From e0e465e31adb22dc26e25e2cf573850233b4df8c Mon Sep 17 00:00:00 2001 From: Alexei Sheplyakov Date: Sun, 1 Feb 2015 16:38:06 +0300 Subject: [PATCH] Added Ceph 0.80.7 HA OpenStack makes use of Ceph. Although a recent enough version of Ceph is shipped with Ubuntu 14.04 several upstream bugfixes which are important for OpenStack have not been incorporated by Ubuntu yet, hence the custom package. The source has been borrowed from Ubuntu 14.04: http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/ceph_0.80.7.orig.tar.bz2 http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/ceph_0.80.7-0ubuntu0.14.04.1.debian.tar.xz The additional (upstream) patches: https://github.com/ceph/ceph/commit/fe7bf06366adaf787816d1e68f5e3f68e8c91134 https://github.com/ceph/ceph/commit/b8fa2ed60b6cce51701df972dbb6f5e02e0d84ba Note: these sources are equivalent (modulo packaging changes) to the ones in https://review.fuel-infra.org/packages/precise/ceph commit fd95b0d17d15240b059a33b3a0f1a5bbba753817 Related-Bug: #1386369 Related-Bug: #1388506 Change-Id: Iaa9f190139c2bfafef195bb1fcd53b353512dde5 --- ceph/AUTHORS | 19 + ceph/COPYING | 140 + ceph/ChangeLog | 0 ceph/INSTALL | 21 + ceph/Makefile.am | 50 + ceph/Makefile.in | 849 + ceph/NEWS | 0 ceph/README | 177 + ceph/aclocal.m4 | 1385 + ceph/ar-lib | 265 + ceph/autogen.sh | 35 + ceph/ceph.spec | 705 + ceph/ceph.spec.in | 705 + ceph/compile | 310 + ceph/config.guess | 1530 + ceph/config.sub | 1773 + ceph/configure | 24717 +++++++++++ ceph/configure.ac | 795 + ceph/depcomp | 688 + ceph/install-sh | 527 + ceph/ltmain.sh | 9661 ++++ ceph/m4/ac_check_classpath.m4 | 24 + ceph/m4/ac_prog_jar.m4 | 39 + ceph/m4/ac_prog_javac.m4 | 45 + ceph/m4/ac_prog_javac_works.m4 | 36 + ceph/m4/ac_prog_javah.m4 | 28 + ceph/m4/acx_pthread.m4 | 397 + ceph/m4/ax_c_pretty_func.m4 | 18 + ceph/m4/ax_c_var_func.m4 | 66 + ceph/m4/ax_check_compile_flag.m4 | 72 + ceph/m4/ax_cxx_static_cast.m4 | 43 + ceph/m4/ax_intel.m4 | 70 + ceph/m4/libtool.m4 | 8001 ++++ ceph/m4/ltoptions.m4 | 384 + ceph/m4/ltsugar.m4 | 123 + ceph/m4/ltversion.m4 | 23 + ceph/m4/lt~obsolete.m4 | 98 + ceph/m4/pkg.m4 | 159 + ceph/man/Makefile.am | 30 + ceph/man/Makefile.in | 555 + ceph/man/ceph-authtool.8 | 299 + ceph/man/ceph-clsinfo.8 | 96 + ceph/man/ceph-conf.8 | 159 + ceph/man/ceph-debugpack.8 | 95 + ceph/man/ceph-dencoder.8 | 193 + ceph/man/ceph-fuse.8 | 120 + ceph/man/ceph-mds.8 | 121 + ceph/man/ceph-mon.8 | 137 + ceph/man/ceph-osd.8 | 169 + ceph/man/ceph-post-file.8 | 130 + ceph/man/ceph-rbdnamer.8 | 92 + ceph/man/ceph-rest-api.8 | 208 + ceph/man/ceph-run.8 | 89 + ceph/man/ceph-syn.8 | 148 + ceph/man/ceph.8 | 162 + ceph/man/cephfs.8 | 144 + ceph/man/crushtool.8 | 433 + ceph/man/librados-config.8 | 94 + ceph/man/mkcephfs.8 | 164 + ceph/man/monmaptool.8 | 188 + ceph/man/mount.ceph.8 | 257 + ceph/man/osdmaptool.8 | 139 + ceph/man/rados.8 | 260 + ceph/man/radosgw-admin.8 | 333 + ceph/man/radosgw.8 | 242 + ceph/man/rbd-fuse.8 | 110 + ceph/man/rbd.8 | 616 + ceph/missing | 331 + ceph/py-compile | 161 + ceph/share/id_dsa_drop.ceph.com | 12 + ceph/share/id_dsa_drop.ceph.com.pub | 1 + ceph/share/known_hosts_drop.ceph.com | 1 + ceph/src/.git_version | 2 + ceph/src/Makefile-env.am | 198 + ceph/src/Makefile.am | 412 + ceph/src/Makefile.in | 16102 +++++++ ceph/src/README | 2 + ceph/src/TODO | 2 + ceph/src/acconfig.h.in | 383 + ceph/src/arch/Makefile.am | 11 + ceph/src/arch/intel.c | 90 + ceph/src/arch/intel.h | 20 + ceph/src/arch/neon.c | 51 + ceph/src/arch/neon.h | 16 + ceph/src/arch/probe.cc | 22 + ceph/src/arch/probe.h | 16 + ceph/src/auth/Auth.h | 247 + ceph/src/auth/AuthAuthorizeHandler.cc | 54 + ceph/src/auth/AuthAuthorizeHandler.h | 55 + ceph/src/auth/AuthClientHandler.cc | 39 + ceph/src/auth/AuthClientHandler.h | 83 + ceph/src/auth/AuthMethodList.cc | 70 + ceph/src/auth/AuthMethodList.h | 42 + ceph/src/auth/AuthServiceHandler.cc | 33 + ceph/src/auth/AuthServiceHandler.h | 44 + ceph/src/auth/AuthSessionHandler.cc | 52 + ceph/src/auth/AuthSessionHandler.h | 71 + ceph/src/auth/Crypto.cc | 398 + ceph/src/auth/Crypto.h | 162 + ceph/src/auth/KeyRing.cc | 276 + ceph/src/auth/KeyRing.h | 110 + ceph/src/auth/Makefile.am | 46 + ceph/src/auth/RotatingKeyRing.cc | 78 + ceph/src/auth/RotatingKeyRing.h | 54 + ceph/src/auth/cephx/CephxAuthorizeHandler.cc | 42 + ceph/src/auth/cephx/CephxAuthorizeHandler.h | 32 + ceph/src/auth/cephx/CephxClientHandler.cc | 219 + ceph/src/auth/cephx/CephxClientHandler.h | 73 + ceph/src/auth/cephx/CephxKeyServer.cc | 461 + ceph/src/auth/cephx/CephxKeyServer.h | 308 + ceph/src/auth/cephx/CephxProtocol.cc | 490 + ceph/src/auth/cephx/CephxProtocol.h | 499 + ceph/src/auth/cephx/CephxServiceHandler.cc | 204 + ceph/src/auth/cephx/CephxServiceHandler.h | 37 + ceph/src/auth/cephx/CephxSessionHandler.cc | 139 + ceph/src/auth/cephx/CephxSessionHandler.h | 49 + .../src/auth/none/AuthNoneAuthorizeHandler.cc | 47 + ceph/src/auth/none/AuthNoneAuthorizeHandler.h | 32 + ceph/src/auth/none/AuthNoneClientHandler.h | 56 + ceph/src/auth/none/AuthNoneProtocol.h | 32 + ceph/src/auth/none/AuthNoneServiceHandler.h | 41 + ceph/src/auth/none/AuthNoneSessionHandler.h | 52 + .../unknown/AuthUnknownAuthorizeHandler.cc | 35 + .../unknown/AuthUnknownAuthorizeHandler.h | 32 + .../auth/unknown/AuthUnknownClientHandler.h | 56 + ceph/src/auth/unknown/AuthUnknownProtocol.h | 32 + .../auth/unknown/AuthUnknownServiceHandler.h | 39 + .../auth/unknown/AuthUnknownSessionHandler.h | 52 + ceph/src/bash_completion/ceph | 60 + ceph/src/bash_completion/rados | 40 + ceph/src/bash_completion/radosgw-admin | 57 + ceph/src/bash_completion/rbd | 44 + ceph/src/brag/Makefile.am | 3 + ceph/src/brag/README.md | 185 + ceph/src/brag/client/ceph-brag | 521 + ceph/src/brag/server/MANIFEST.in | 1 + ceph/src/brag/server/app.wsgi | 5 + .../brag/server/ceph_brag.egg-info/PKG-INFO | 10 + .../server/ceph_brag.egg-info/SOURCES.txt | 21 + .../ceph_brag.egg-info/dependency_links.txt | 1 + .../server/ceph_brag.egg-info/not-zip-safe | 1 + .../server/ceph_brag.egg-info/requires.txt | 1 + .../server/ceph_brag.egg-info/top_level.txt | 1 + ceph/src/brag/server/ceph_brag/__init__.py | 0 ceph/src/brag/server/ceph_brag/app.py | 19 + .../server/ceph_brag/controllers/__init__.py | 0 .../brag/server/ceph_brag/controllers/root.py | 73 + ceph/src/brag/server/ceph_brag/json.py | 115 + .../brag/server/ceph_brag/model/__init__.py | 29 + ceph/src/brag/server/ceph_brag/model/db.py | 284 + .../brag/server/ceph_brag/tests/__init__.py | 22 + .../src/brag/server/ceph_brag/tests/config.py | 54 + .../server/ceph_brag/tests/test_functional.py | 68 + .../brag/server/ceph_brag/tests/test_units.py | 7 + ceph/src/brag/server/config.py | 52 + ceph/src/brag/server/sample.json | 98 + ceph/src/brag/server/setup.cfg | 6 + ceph/src/brag/server/setup.py | 22 + ceph/src/ceph-clsinfo | 85 + ceph/src/ceph-coverage.in | 25 + ceph/src/ceph-create-keys | 227 + ceph/src/ceph-crush-location | 87 + ceph/src/ceph-crush-location.in | 87 + ceph/src/ceph-debugpack.in | 113 + ceph/src/ceph-disk | 2791 ++ ceph/src/ceph-disk-activate | 3 + ceph/src/ceph-disk-prepare | 3 + ceph/src/ceph-disk-udev | 70 + ceph/src/ceph-post-file.in | 168 + ceph/src/ceph-rbdnamer | 12 + ceph/src/ceph-rest-api | 67 + ceph/src/ceph-run | 21 + ceph/src/ceph.in | 844 + ceph/src/ceph_common.sh | 239 + ceph/src/ceph_fuse.cc | 218 + ceph/src/ceph_mds.cc | 332 + ceph/src/ceph_mon.cc | 725 + ceph/src/ceph_osd.cc | 554 + ceph/src/ceph_syn.cc | 107 + ceph/src/ceph_ver.c | 12 + ceph/src/cephfs.cc | 292 + ceph/src/check_version | 19 + ceph/src/civetweb/civetweb.h | 550 + ceph/src/civetweb/include/civetweb.h | 550 + ceph/src/civetweb/src/civetweb.c | 6877 +++ ceph/src/civetweb/src/md5.h | 461 + ceph/src/client/Client.cc | 9025 ++++ ceph/src/client/Client.h | 835 + ceph/src/client/ClientSnapRealm.cc | 29 + ceph/src/client/ClientSnapRealm.h | 60 + ceph/src/client/Dentry.cc | 28 + ceph/src/client/Dentry.h | 55 + ceph/src/client/Dir.h | 19 + ceph/src/client/Fh.h | 31 + ceph/src/client/Inode.cc | 492 + ceph/src/client/Inode.h | 274 + ceph/src/client/Makefile.am | 35 + ceph/src/client/MetaRequest.cc | 114 + ceph/src/client/MetaRequest.h | 188 + ceph/src/client/MetaSession.cc | 40 + ceph/src/client/MetaSession.h | 64 + ceph/src/client/ObjecterWriteback.h | 43 + ceph/src/client/SyntheticClient.cc | 3527 ++ ceph/src/client/SyntheticClient.h | 279 + ceph/src/client/Trace.cc | 81 + ceph/src/client/Trace.h | 67 + ceph/src/client/fuse_ll.cc | 962 + ceph/src/client/fuse_ll.h | 25 + ceph/src/client/ioctl.h | 51 + ceph/src/client/test_ioctls.c | 122 + ceph/src/cls/Makefile.am | 136 + ceph/src/cls/hello/cls_hello.cc | 307 + ceph/src/cls/lock/cls_lock.cc | 463 + ceph/src/cls/lock/cls_lock_client.cc | 224 + ceph/src/cls/lock/cls_lock_client.h | 100 + ceph/src/cls/lock/cls_lock_ops.cc | 172 + ceph/src/cls/lock/cls_lock_ops.h | 173 + ceph/src/cls/lock/cls_lock_types.cc | 70 + ceph/src/cls/lock/cls_lock_types.h | 103 + ceph/src/cls/log/cls_log.cc | 324 + ceph/src/cls/log/cls_log_client.cc | 154 + ceph/src/cls/log/cls_log_client.h | 32 + ceph/src/cls/log/cls_log_ops.h | 152 + ceph/src/cls/log/cls_log_types.h | 67 + ceph/src/cls/rbd/cls_rbd.cc | 2100 + ceph/src/cls/rbd/cls_rbd.h | 146 + ceph/src/cls/rbd/cls_rbd_client.cc | 635 + ceph/src/cls/rbd/cls_rbd_client.h | 114 + ceph/src/cls/refcount/cls_refcount.cc | 243 + ceph/src/cls/refcount/cls_refcount_client.cc | 60 + ceph/src/cls/refcount/cls_refcount_client.h | 34 + ceph/src/cls/refcount/cls_refcount_ops.cc | 80 + ceph/src/cls/refcount/cls_refcount_ops.h | 131 + ceph/src/cls/replica_log/cls_replica_log.cc | 151 + .../cls/replica_log/cls_replica_log_client.cc | 92 + .../cls/replica_log/cls_replica_log_client.h | 84 + .../cls/replica_log/cls_replica_log_ops.cc | 81 + .../src/cls/replica_log/cls_replica_log_ops.h | 112 + .../cls/replica_log/cls_replica_log_types.cc | 104 + .../cls/replica_log/cls_replica_log_types.h | 194 + ceph/src/cls/rgw/cls_rgw.cc | 1628 + ceph/src/cls/rgw/cls_rgw_client.cc | 363 + ceph/src/cls/rgw/cls_rgw_client.h | 69 + ceph/src/cls/rgw/cls_rgw_ops.cc | 228 + ceph/src/cls/rgw/cls_rgw_ops.h | 498 + ceph/src/cls/rgw/cls_rgw_types.cc | 261 + ceph/src/cls/rgw/cls_rgw_types.h | 668 + ceph/src/cls/statelog/cls_statelog.cc | 318 + ceph/src/cls/statelog/cls_statelog_client.cc | 128 + ceph/src/cls/statelog/cls_statelog_client.h | 29 + ceph/src/cls/statelog/cls_statelog_ops.h | 144 + ceph/src/cls/statelog/cls_statelog_types.h | 51 + ceph/src/cls/user/cls_user.cc | 382 + ceph/src/cls/user/cls_user_client.cc | 145 + ceph/src/cls/user/cls_user_client.h | 33 + ceph/src/cls/user/cls_user_ops.cc | 115 + ceph/src/cls/user/cls_user_ops.h | 179 + ceph/src/cls/user/cls_user_types.cc | 108 + ceph/src/cls/user/cls_user_types.h | 187 + ceph/src/cls/version/cls_version.cc | 240 + ceph/src/cls/version/cls_version_client.cc | 102 + ceph/src/cls/version/cls_version_client.h | 25 + ceph/src/cls/version/cls_version_ops.h | 95 + ceph/src/cls/version/cls_version_types.cc | 18 + ceph/src/cls/version/cls_version_types.h | 86 + ceph/src/cls_acl.cc | 59 + ceph/src/cls_crypto.cc | 77 + ceph/src/common/AsyncReserver.h | 114 + ceph/src/common/BackTrace.cc | 69 + ceph/src/common/BackTrace.h | 34 + ceph/src/common/Clock.cc | 39 + ceph/src/common/Clock.h | 27 + ceph/src/common/Cond.h | 191 + ceph/src/common/ConfUtils.cc | 593 + ceph/src/common/ConfUtils.h | 86 + ceph/src/common/DecayCounter.cc | 82 + ceph/src/common/DecayCounter.h | 134 + ceph/src/common/Finisher.cc | 87 + ceph/src/common/Finisher.h | 131 + ceph/src/common/Formatter.cc | 508 + ceph/src/common/Formatter.h | 150 + ceph/src/common/HeartbeatMap.cc | 156 + ceph/src/common/HeartbeatMap.h | 84 + ceph/src/common/LogClient.cc | 188 + ceph/src/common/LogClient.h | 116 + ceph/src/common/LogEntry.cc | 217 + ceph/src/common/LogEntry.h | 131 + ceph/src/common/Makefile.am | 208 + ceph/src/common/MemoryModel.cc | 103 + ceph/src/common/MemoryModel.h | 54 + ceph/src/common/Mutex.cc | 96 + ceph/src/common/Mutex.h | 129 + ceph/src/common/OutputDataSocket.cc | 420 + ceph/src/common/OutputDataSocket.h | 72 + ceph/src/common/PrebufferedStreambuf.cc | 63 + ceph/src/common/PrebufferedStreambuf.h | 42 + ceph/src/common/Preforker.h | 104 + ceph/src/common/PrioritizedQueue.h | 402 + ceph/src/common/RWLock.h | 108 + ceph/src/common/RefCountedObj.cc | 23 + ceph/src/common/RefCountedObj.h | 131 + ceph/src/common/Semaphore.h | 51 + ceph/src/common/SimpleRNG.h | 24 + ceph/src/common/SloppyCRCMap.cc | 180 + ceph/src/common/SloppyCRCMap.h | 78 + ceph/src/common/TextTable.cc | 85 + ceph/src/common/TextTable.h | 159 + ceph/src/common/Thread.cc | 160 + ceph/src/common/Thread.h | 54 + ceph/src/common/Throttle.cc | 276 + ceph/src/common/Throttle.h | 101 + ceph/src/common/Timer.cc | 193 + ceph/src/common/Timer.h | 95 + ceph/src/common/TrackedOp.cc | 279 + ceph/src/common/TrackedOp.h | 180 + ceph/src/common/WorkQueue.cc | 278 + ceph/src/common/WorkQueue.h | 436 + ceph/src/common/addr_parsing.c | 154 + ceph/src/common/admin_socket.cc | 576 + ceph/src/common/admin_socket.h | 105 + ceph/src/common/admin_socket_client.cc | 178 + ceph/src/common/admin_socket_client.h | 36 + ceph/src/common/arch.h | 15 + ceph/src/common/armor.c | 131 + ceph/src/common/armor.h | 20 + ceph/src/common/assert.cc | 89 + ceph/src/common/blkdev.cc | 56 + ceph/src/common/blkdev.h | 6 + ceph/src/common/bloom_filter.cc | 137 + ceph/src/common/bloom_filter.hpp | 708 + ceph/src/common/buffer.cc | 1733 + ceph/src/common/ceph_argparse.cc | 483 + ceph/src/common/ceph_argparse.h | 78 + ceph/src/common/ceph_context.cc | 417 + ceph/src/common/ceph_context.h | 146 + ceph/src/common/ceph_crypto.cc | 73 + ceph/src/common/ceph_crypto.h | 160 + ceph/src/common/ceph_crypto_cms.cc | 360 + ceph/src/common/ceph_crypto_cms.h | 10 + ceph/src/common/ceph_frag.cc | 21 + ceph/src/common/ceph_fs.cc | 79 + ceph/src/common/ceph_hash.cc | 117 + ceph/src/common/ceph_json.cc | 522 + ceph/src/common/ceph_json.h | 361 + ceph/src/common/ceph_strings.cc | 258 + ceph/src/common/cmdparse.cc | 230 + ceph/src/common/cmdparse.h | 72 + ceph/src/common/code_environment.cc | 80 + ceph/src/common/code_environment.h | 42 + ceph/src/common/common_init.cc | 126 + ceph/src/common/common_init.h | 80 + ceph/src/common/compiler_extensions.h | 30 + ceph/src/common/config.cc | 1104 + ceph/src/common/config.h | 279 + ceph/src/common/config_obs.h | 33 + ceph/src/common/config_opts.h | 865 + ceph/src/common/crc32c.cc | 41 + ceph/src/common/crc32c_intel_baseline.c | 135 + ceph/src/common/crc32c_intel_baseline.h | 16 + ceph/src/common/crc32c_intel_fast.c | 51 + ceph/src/common/crc32c_intel_fast.h | 28 + ceph/src/common/crc32c_intel_fast_asm.S | 664 + ceph/src/common/crc32c_intel_fast_zero_asm.S | 646 + ceph/src/common/debug.h | 33 + ceph/src/common/dout.cc | 14 + ceph/src/common/dout.h | 71 + ceph/src/common/entity_name.cc | 174 + ceph/src/common/entity_name.h | 88 + ceph/src/common/environment.cc | 43 + ceph/src/common/environment.h | 21 + ceph/src/common/errno.cc | 30 + ceph/src/common/errno.h | 9 + ceph/src/common/escape.c | 199 + ceph/src/common/escape.h | 52 + ceph/src/common/fd.cc | 57 + ceph/src/common/fd.h | 22 + ceph/src/common/hex.cc | 39 + ceph/src/common/hex.h | 25 + ceph/src/common/histogram.cc | 58 + ceph/src/common/histogram.h | 131 + ceph/src/common/hobject.cc | 281 + ceph/src/common/hobject.h | 340 + ceph/src/common/io_priority.cc | 54 + ceph/src/common/io_priority.h | 44 + ceph/src/common/ipaddr.cc | 152 + ceph/src/common/likely.h | 24 + ceph/src/common/linux_version.c | 25 + ceph/src/common/linux_version.h | 22 + ceph/src/common/lockdep.cc | 277 + ceph/src/common/lockdep.h | 30 + ceph/src/common/lru_map.h | 135 + ceph/src/common/map_cacher.hpp | 200 + ceph/src/common/mime.c | 133 + ceph/src/common/mime.h | 56 + ceph/src/common/obj_bencher.cc | 1191 + ceph/src/common/obj_bencher.h | 109 + ceph/src/common/page.cc | 19 + ceph/src/common/perf_counters.cc | 360 + ceph/src/common/perf_counters.h | 199 + ceph/src/common/pick_address.cc | 157 + ceph/src/common/pick_address.h | 44 + ceph/src/common/pipe.c | 61 + ceph/src/common/pipe.h | 33 + ceph/src/common/random_cache.hpp | 111 + ceph/src/common/run_cmd.cc | 80 + ceph/src/common/run_cmd.h | 33 + ceph/src/common/safe_io.c | 229 + ceph/src/common/safe_io.h | 74 + ceph/src/common/sctp_crc32.c | 788 + ceph/src/common/sctp_crc32.h | 14 + ceph/src/common/secret.c | 137 + ceph/src/common/secret.h | 25 + ceph/src/common/shared_cache.hpp | 183 + ceph/src/common/sharedptr_registry.hpp | 191 + ceph/src/common/signal.cc | 80 + ceph/src/common/signal.h | 42 + ceph/src/common/simple_cache.hpp | 93 + ceph/src/common/simple_spin.cc | 43 + ceph/src/common/simple_spin.h | 39 + ceph/src/common/snap_types.cc | 93 + ceph/src/common/snap_types.h | 74 + ceph/src/common/static_assert.h | 24 + ceph/src/common/str_list.cc | 106 + ceph/src/common/str_map.cc | 68 + ceph/src/common/strtol.cc | 169 + ceph/src/common/strtol.h | 33 + ceph/src/common/sync_filesystem.h | 54 + ceph/src/common/tracked_int_ptr.hpp | 67 + ceph/src/common/utf8.c | 187 + ceph/src/common/utf8.h | 63 + ceph/src/common/util.cc | 106 + ceph/src/common/version.cc | 41 + ceph/src/common/version.h | 29 + ceph/src/common/xattr.c | 262 + ceph/src/common/xattr.h | 38 + ceph/src/crush/CrushCompiler.cc | 860 + ceph/src/crush/CrushCompiler.h | 72 + ceph/src/crush/CrushTester.cc | 599 + ceph/src/crush/CrushTester.h | 330 + ceph/src/crush/CrushWrapper.cc | 1511 + ceph/src/crush/CrushWrapper.h | 957 + ceph/src/crush/CrushWrapper.i | 47 + ceph/src/crush/Makefile.am | 30 + ceph/src/crush/builder.c | 1166 + ceph/src/crush/builder.h | 43 + ceph/src/crush/crush.c | 148 + ceph/src/crush/crush.h | 213 + ceph/src/crush/grammar.h | 169 + ceph/src/crush/hash.c | 155 + ceph/src/crush/hash.h | 17 + ceph/src/crush/mapper.c | 851 + ceph/src/crush/mapper.h | 20 + ceph/src/crush/sample.txt | 47 + ceph/src/crush/types.h | 17 + ceph/src/erasure-code/ErasureCodeInterface.h | 370 + ceph/src/erasure-code/ErasureCodePlugin.cc | 163 + ceph/src/erasure-code/ErasureCodePlugin.h | 76 + ceph/src/erasure-code/Makefile.am | 16 + .../jerasure/ErasureCodeJerasure.cc | 493 + .../jerasure/ErasureCodeJerasure.h | 245 + .../jerasure/ErasureCodePluginJerasure.cc | 89 + .../ErasureCodePluginSelectJerasure.cc | 69 + ceph/src/erasure-code/jerasure/Makefile.am | 110 + .../gf-complete/include/gf_complete.h | 192 + .../jerasure/gf-complete/include/gf_general.h | 61 + .../jerasure/gf-complete/include/gf_int.h | 200 + .../jerasure/gf-complete/include/gf_method.h | 20 + .../jerasure/gf-complete/include/gf_rand.h | 22 + .../jerasure/gf-complete/src/gf.c | 1026 + .../jerasure/gf-complete/src/gf_general.c | 538 + .../jerasure/gf-complete/src/gf_method.c | 185 + .../jerasure/gf-complete/src/gf_rand.c | 80 + .../jerasure/gf-complete/src/gf_w128.c | 1758 + .../jerasure/gf-complete/src/gf_w16.c | 2489 ++ .../jerasure/gf-complete/src/gf_w32.c | 2741 ++ .../jerasure/gf-complete/src/gf_w4.c | 2081 + .../jerasure/gf-complete/src/gf_w64.c | 2244 + .../jerasure/gf-complete/src/gf_w8.c | 2456 + .../jerasure/gf-complete/src/gf_wgen.c | 1019 + .../jerasure/jerasure/include/cauchy.h | 45 + .../jerasure/jerasure/include/galois.h | 99 + .../jerasure/jerasure/include/jerasure.h | 294 + .../jerasure/jerasure/include/liberation.h | 47 + .../jerasure/jerasure/include/reed_sol.h | 50 + .../jerasure/jerasure/src/cauchy.c | 405 + .../jerasure/jerasure/src/galois.c | 353 + .../jerasure/jerasure/src/jerasure.c | 1387 + .../jerasure/jerasure/src/liberation.c | 262 + .../jerasure/jerasure/src/reed_sol.c | 301 + ceph/src/fetch_config | 20 + ceph/src/global/Makefile.am | 14 + ceph/src/global/global_context.cc | 22 + ceph/src/global/global_context.h | 28 + ceph/src/global/global_init.cc | 285 + ceph/src/global/global_init.h | 93 + ceph/src/global/pidfile.cc | 98 + ceph/src/global/pidfile.h | 28 + ceph/src/global/signal_handler.cc | 356 + ceph/src/global/signal_handler.h | 48 + ceph/src/gtest/CHANGES | 98 + ceph/src/gtest/CMakeLists.txt | 384 + ceph/src/gtest/CONTRIBUTORS | 36 + ceph/src/gtest/COPYING | 28 + ceph/src/gtest/Makefile.am | 287 + ceph/src/gtest/Makefile.in | 1381 + ceph/src/gtest/README | 417 + ceph/src/gtest/aclocal.m4 | 1198 + ceph/src/gtest/build-aux/config.guess | 1530 + ceph/src/gtest/build-aux/config.h.in | 69 + ceph/src/gtest/build-aux/config.sub | 1773 + ceph/src/gtest/build-aux/depcomp | 688 + ceph/src/gtest/build-aux/install-sh | 527 + ceph/src/gtest/build-aux/ltmain.sh | 9661 ++++ ceph/src/gtest/build-aux/missing | 331 + ceph/src/gtest/codegear/gtest.cbproj | 138 + ceph/src/gtest/codegear/gtest.groupproj | 54 + ceph/src/gtest/codegear/gtest_all.cc | 38 + ceph/src/gtest/codegear/gtest_link.cc | 40 + ceph/src/gtest/codegear/gtest_main.cbproj | 82 + ceph/src/gtest/codegear/gtest_unittest.cbproj | 88 + ceph/src/gtest/configure | 18222 ++++++++ ceph/src/gtest/configure.ac | 68 + ceph/src/gtest/fused-src/gtest/gtest-all.cc | 8510 ++++ ceph/src/gtest/fused-src/gtest/gtest.h | 18007 ++++++++ ceph/src/gtest/fused-src/gtest/gtest_main.cc | 39 + .../gtest/include/gtest/gtest-death-test.h | 283 + ceph/src/gtest/include/gtest/gtest-message.h | 230 + .../gtest/include/gtest/gtest-param-test.h | 1392 + .../include/gtest/gtest-param-test.h.pump | 457 + ceph/src/gtest/include/gtest/gtest-spi.h | 232 + .../src/gtest/include/gtest/gtest-test-part.h | 176 + .../gtest/include/gtest/gtest-typed-test.h | 259 + ceph/src/gtest/include/gtest/gtest.h | 2052 + .../src/gtest/include/gtest/gtest_pred_impl.h | 368 + ceph/src/gtest/include/gtest/gtest_prod.h | 58 + .../internal/gtest-death-test-internal.h | 275 + .../include/gtest/internal/gtest-filepath.h | 210 + .../include/gtest/internal/gtest-internal.h | 923 + .../include/gtest/internal/gtest-linked_ptr.h | 242 + .../internal/gtest-param-util-generated.h | 4820 ++ .../gtest-param-util-generated.h.pump | 301 + .../include/gtest/internal/gtest-param-util.h | 619 + .../gtest/include/gtest/internal/gtest-port.h | 1497 + .../include/gtest/internal/gtest-string.h | 350 + .../include/gtest/internal/gtest-tuple.h | 968 + .../include/gtest/internal/gtest-tuple.h.pump | 336 + .../include/gtest/internal/gtest-type-util.h | 3321 ++ .../gtest/internal/gtest-type-util.h.pump | 287 + ceph/src/gtest/m4/acx_pthread.m4 | 363 + ceph/src/gtest/m4/gtest.m4 | 74 + ceph/src/gtest/m4/libtool.m4 | 8001 ++++ ceph/src/gtest/m4/ltoptions.m4 | 384 + ceph/src/gtest/m4/ltsugar.m4 | 123 + ceph/src/gtest/m4/ltversion.m4 | 23 + ceph/src/gtest/m4/lt~obsolete.m4 | 98 + ceph/src/gtest/make/Makefile | 80 + ceph/src/gtest/msvc/gtest-md.sln | 45 + ceph/src/gtest/msvc/gtest-md.vcproj | 237 + ceph/src/gtest/msvc/gtest.sln | 45 + ceph/src/gtest/msvc/gtest.vcproj | 237 + ceph/src/gtest/msvc/gtest_main-md.vcproj | 165 + ceph/src/gtest/msvc/gtest_main.vcproj | 165 + ceph/src/gtest/msvc/gtest_prod_test-md.vcproj | 164 + ceph/src/gtest/msvc/gtest_prod_test.vcproj | 164 + ceph/src/gtest/msvc/gtest_unittest-md.vcproj | 147 + ceph/src/gtest/msvc/gtest_unittest.vcproj | 147 + ceph/src/gtest/samples/prime_tables.h | 123 + ceph/src/gtest/samples/sample1.cc | 68 + ceph/src/gtest/samples/sample1.h | 43 + ceph/src/gtest/samples/sample10_unittest.cc | 145 + ceph/src/gtest/samples/sample1_unittest.cc | 153 + ceph/src/gtest/samples/sample2.cc | 56 + ceph/src/gtest/samples/sample2.h | 86 + ceph/src/gtest/samples/sample2_unittest.cc | 109 + ceph/src/gtest/samples/sample3-inl.h | 173 + ceph/src/gtest/samples/sample3_unittest.cc | 151 + ceph/src/gtest/samples/sample4.cc | 46 + ceph/src/gtest/samples/sample4.h | 53 + ceph/src/gtest/samples/sample4_unittest.cc | 45 + ceph/src/gtest/samples/sample5_unittest.cc | 199 + ceph/src/gtest/samples/sample6_unittest.cc | 224 + ceph/src/gtest/samples/sample7_unittest.cc | 132 + ceph/src/gtest/samples/sample8_unittest.cc | 173 + ceph/src/gtest/samples/sample9_unittest.cc | 160 + ceph/src/gtest/scripts/fuse_gtest_files.py | 250 + ceph/src/gtest/scripts/gen_gtest_pred_impl.py | 733 + ceph/src/gtest/scripts/gtest-config.in | 274 + ceph/src/gtest/scripts/pump.py | 835 + ceph/src/gtest/scripts/test/Makefile | 57 + ceph/src/gtest/src/gtest-all.cc | 47 + ceph/src/gtest/src/gtest-death-test.cc | 1172 + ceph/src/gtest/src/gtest-filepath.cc | 380 + ceph/src/gtest/src/gtest-internal-inl.h | 1074 + ceph/src/gtest/src/gtest-port.cc | 711 + ceph/src/gtest/src/gtest-test-part.cc | 110 + ceph/src/gtest/src/gtest-typed-test.cc | 110 + ceph/src/gtest/src/gtest.cc | 4704 ++ ceph/src/gtest/src/gtest_main.cc | 39 + ceph/src/gtest/test/gtest-death-test_test.cc | 1230 + ceph/src/gtest/test/gtest-filepath_test.cc | 690 + ceph/src/gtest/test/gtest-linked_ptr_test.cc | 154 + ceph/src/gtest/test/gtest-listener_test.cc | 313 + ceph/src/gtest/test/gtest-message_test.cc | 167 + ceph/src/gtest/test/gtest-options_test.cc | 212 + ceph/src/gtest/test/gtest-param-test2_test.cc | 65 + ceph/src/gtest/test/gtest-param-test_test.cc | 835 + ceph/src/gtest/test/gtest-param-test_test.h | 55 + ceph/src/gtest/test/gtest-port_test.cc | 1018 + ceph/src/gtest/test/gtest-test-part_test.cc | 208 + ceph/src/gtest/test/gtest-tuple_test.cc | 320 + ceph/src/gtest/test/gtest-typed-test2_test.cc | 45 + ceph/src/gtest/test/gtest-typed-test_test.cc | 360 + ceph/src/gtest/test/gtest-typed-test_test.h | 66 + .../src/gtest/test/gtest-unittest-api_test.cc | 343 + ceph/src/gtest/test/gtest_all_test.cc | 48 + .../test/gtest_break_on_failure_unittest.py | 218 + .../test/gtest_break_on_failure_unittest_.cc | 86 + ceph/src/gtest/test/gtest_color_test.py | 130 + ceph/src/gtest/test/gtest_color_test_.cc | 71 + ceph/src/gtest/test/gtest_env_var_test.py | 105 + ceph/src/gtest/test/gtest_env_var_test_.cc | 126 + ceph/src/gtest/test/gtest_environment_test.cc | 186 + ceph/src/gtest/test/gtest_filter_unittest.py | 633 + ceph/src/gtest/test/gtest_filter_unittest_.cc | 140 + ceph/src/gtest/test/gtest_help_test.py | 169 + ceph/src/gtest/test/gtest_help_test_.cc | 46 + .../gtest/test/gtest_list_tests_unittest.py | 177 + .../gtest/test/gtest_list_tests_unittest_.cc | 85 + ceph/src/gtest/test/gtest_main_unittest.cc | 45 + ceph/src/gtest/test/gtest_no_test_unittest.cc | 54 + ceph/src/gtest/test/gtest_output_test.py | 327 + ceph/src/gtest/test/gtest_output_test_.cc | 1135 + .../test/gtest_output_test_golden_lin.txt | 696 + .../test/gtest_output_test_golden_win.txt | 605 + .../gtest/test/gtest_pred_impl_unittest.cc | 2432 + ceph/src/gtest/test/gtest_prod_test.cc | 57 + ceph/src/gtest/test/gtest_repeat_test.cc | 253 + ceph/src/gtest/test/gtest_shuffle_test.py | 325 + ceph/src/gtest/test/gtest_shuffle_test_.cc | 104 + ceph/src/gtest/test/gtest_sole_header_test.cc | 57 + ceph/src/gtest/test/gtest_stress_test.cc | 257 + ceph/src/gtest/test/gtest_test_utils.py | 309 + .../test/gtest_throw_on_failure_ex_test.cc | 92 + .../gtest/test/gtest_throw_on_failure_test.py | 171 + .../test/gtest_throw_on_failure_test_.cc | 56 + .../gtest/test/gtest_uninitialized_test.py | 70 + .../gtest/test/gtest_uninitialized_test_.cc | 43 + ceph/src/gtest/test/gtest_unittest.cc | 6718 +++ .../gtest/test/gtest_xml_outfile1_test_.cc | 49 + .../gtest/test/gtest_xml_outfile2_test_.cc | 49 + .../src/gtest/test/gtest_xml_outfiles_test.py | 132 + .../gtest/test/gtest_xml_output_unittest.py | 224 + .../gtest/test/gtest_xml_output_unittest_.cc | 145 + ceph/src/gtest/test/gtest_xml_test_utils.py | 172 + ceph/src/gtest/test/production.cc | 36 + ceph/src/gtest/test/production.h | 55 + ceph/src/gtest/test/run_tests_util.py | 466 + ceph/src/gtest/test/run_tests_util_test.py | 676 + .../gtest/xcode/Config/DebugProject.xcconfig | 30 + .../xcode/Config/FrameworkTarget.xcconfig | 17 + ceph/src/gtest/xcode/Config/General.xcconfig | 41 + .../xcode/Config/ReleaseProject.xcconfig | 32 + .../xcode/Config/StaticLibraryTarget.xcconfig | 18 + .../gtest/xcode/Config/TestTarget.xcconfig | 8 + ceph/src/gtest/xcode/Resources/Info.plist | 30 + .../xcode/Samples/FrameworkSample/Info.plist | 28 + .../WidgetFramework.xcodeproj/project.pbxproj | 457 + .../xcode/Samples/FrameworkSample/runtests.sh | 62 + .../xcode/Samples/FrameworkSample/widget.cc | 63 + .../xcode/Samples/FrameworkSample/widget.h | 59 + .../Samples/FrameworkSample/widget_test.cc | 68 + ceph/src/gtest/xcode/Scripts/runtests.sh | 65 + .../gtest/xcode/Scripts/versiongenerate.py | 100 + .../xcode/gtest.xcodeproj/project.pbxproj | 1080 + ceph/src/include/CompatSet.h | 248 + ceph/src/include/Context.h | 382 + ceph/src/include/Distribution.h | 74 + ceph/src/include/Makefile.am | 89 + ceph/src/include/Spinlock.h | 113 + ceph/src/include/addr_parsing.h | 28 + ceph/src/include/assert.h | 140 + ceph/src/include/atomic.h | 142 + ceph/src/include/bitmapper.h | 48 + ceph/src/include/blobhash.h | 47 + ceph/src/include/buffer.h | 540 + ceph/src/include/byteorder.h | 108 + ceph/src/include/ceph_features.h | 140 + ceph/src/include/ceph_frag.h | 109 + ceph/src/include/ceph_fs.h | 785 + ceph/src/include/ceph_hash.h | 13 + ceph/src/include/cephfs/libcephfs.h | 1340 + ceph/src/include/cmp.h | 149 + ceph/src/include/color.h | 13 + ceph/src/include/compat.h | 41 + ceph/src/include/crc32c.h | 32 + ceph/src/include/dlist.h | 127 + ceph/src/include/elist.h | 194 + ceph/src/include/encoding.h | 882 + ceph/src/include/err.h | 29 + ceph/src/include/error.h | 41 + ceph/src/include/filepath.h | 225 + ceph/src/include/frag.h | 601 + ceph/src/include/hash.h | 64 + ceph/src/include/hash_namespace.h | 24 + ceph/src/include/int_types.h | 98 + ceph/src/include/intarith.h | 38 + ceph/src/include/interval_set.h | 550 + ceph/src/include/ipaddr.h | 21 + ceph/src/include/linux_fiemap.h | 75 + ceph/src/include/lru.h | 335 + ceph/src/include/memory.h | 28 + ceph/src/include/msgr.h | 193 + ceph/src/include/object.h | 192 + ceph/src/include/on_exit.h | 49 + ceph/src/include/page.h | 18 + ceph/src/include/rados.h | 498 + ceph/src/include/rados/buffer.h | 540 + ceph/src/include/rados/crc32c.h | 32 + ceph/src/include/rados/librados.h | 2546 ++ ceph/src/include/rados/librados.hpp | 949 + ceph/src/include/rados/librgw.h | 35 + ceph/src/include/rados/memory.h | 28 + ceph/src/include/rados/page.h | 18 + ceph/src/include/rados/rados_types.h | 17 + ceph/src/include/rados/rados_types.hpp | 32 + ceph/src/include/rangeset.h | 252 + ceph/src/include/rbd/features.h | 10 + ceph/src/include/rbd/librbd.h | 392 + ceph/src/include/rbd/librbd.hpp | 238 + ceph/src/include/rbd_types.h | 106 + ceph/src/include/stat.h | 145 + ceph/src/include/statlite.h | 72 + ceph/src/include/str_list.h | 39 + ceph/src/include/str_map.h | 59 + ceph/src/include/stringify.h | 14 + ceph/src/include/triple.h | 29 + ceph/src/include/types.h | 449 + ceph/src/include/unordered_map.h | 24 + ceph/src/include/unordered_set.h | 24 + ceph/src/include/util.h | 20 + ceph/src/include/utime.h | 382 + ceph/src/include/uuid.h | 61 + ceph/src/include/xlist.h | 167 + ceph/src/init-ceph.in | 489 + ceph/src/init-radosgw | 104 + ceph/src/init-radosgw.sysv | 118 + ceph/src/init-rbdmap | 118 + ceph/src/java/Makefile.am | 81 + ceph/src/java/Makefile.in | 564 + ceph/src/java/README | 54 + ceph/src/java/java/com/ceph/crush/Bucket.java | 42 + .../ceph/fs/CephAlreadyMountedException.java | 44 + .../fs/CephFileAlreadyExistsException.java | 44 + .../java/java/com/ceph/fs/CephFileExtent.java | 66 + ceph/src/java/java/com/ceph/fs/CephMount.java | 1052 + .../java/com/ceph/fs/CephNativeLoader.java | 35 + .../ceph/fs/CephNotDirectoryException.java | 44 + .../com/ceph/fs/CephNotMountedException.java | 44 + .../java/com/ceph/fs/CephPoolException.java | 44 + ceph/src/java/java/com/ceph/fs/CephStat.java | 53 + .../java/java/com/ceph/fs/CephStatVFS.java | 33 + ceph/src/java/native/JniConstants.cpp | 42 + ceph/src/java/native/JniConstants.h | 52 + ceph/src/java/native/ScopedLocalRef.h | 63 + ceph/src/java/native/libcephfs_jni.cc | 2942 ++ .../java/test/com/ceph/fs/CephAllTests.java | 44 + .../test/com/ceph/fs/CephDoubleMountTest.java | 45 + .../test/com/ceph/fs/CephMountCreateTest.java | 90 + .../java/test/com/ceph/fs/CephMountTest.java | 998 + .../test/com/ceph/fs/CephUnmountedTest.java | 164 + ceph/src/json_spirit/Makefile.am | 17 + ceph/src/json_spirit/json_spirit.h | 18 + .../json_spirit/json_spirit_error_position.h | 54 + ceph/src/json_spirit/json_spirit_reader.cpp | 137 + ceph/src/json_spirit/json_spirit_reader.h | 62 + .../json_spirit/json_spirit_reader_template.h | 648 + .../json_spirit/json_spirit_stream_reader.h | 70 + ceph/src/json_spirit/json_spirit_utils.h | 63 + ceph/src/json_spirit/json_spirit_value.h | 585 + ceph/src/json_spirit/json_spirit_writer.cpp | 96 + ceph/src/json_spirit/json_spirit_writer.h | 63 + .../json_spirit/json_spirit_writer_options.h | 30 + .../json_spirit/json_spirit_writer_template.h | 383 + ceph/src/key_value_store/Makefile.am | 12 + ceph/src/key_value_store/cls_kvs.cc | 686 + .../src/key_value_store/key_value_structure.h | 149 + .../key_value_store/kv_flat_btree_async.cc | 2350 + .../src/key_value_store/kv_flat_btree_async.h | 884 + ceph/src/key_value_store/kvs_arg_types.h | 145 + ceph/src/libcephfs.cc | 1519 + ceph/src/librados-config.cc | 79 + ceph/src/librados/AioCompletionImpl.h | 247 + ceph/src/librados/IoCtxImpl.cc | 1360 + ceph/src/librados/IoCtxImpl.h | 240 + ceph/src/librados/Makefile.am | 23 + ceph/src/librados/PoolAsyncCompletionImpl.h | 119 + ceph/src/librados/RadosClient.cc | 878 + ceph/src/librados/RadosClient.h | 146 + ceph/src/librados/librados.cc | 3613 ++ ceph/src/librados/snap_set_diff.cc | 104 + ceph/src/librados/snap_set_diff.h | 17 + ceph/src/librbd/AioCompletion.cc | 105 + ceph/src/librbd/AioCompletion.h | 207 + ceph/src/librbd/AioRequest.cc | 256 + ceph/src/librbd/AioRequest.h | 251 + ceph/src/librbd/ImageCtx.cc | 659 + ceph/src/librbd/ImageCtx.h | 154 + ceph/src/librbd/LibrbdWriteback.cc | 197 + ceph/src/librbd/LibrbdWriteback.h | 63 + ceph/src/librbd/Makefile.am | 27 + ceph/src/librbd/SnapInfo.h | 28 + ceph/src/librbd/WatchCtx.cc | 36 + ceph/src/librbd/WatchCtx.h | 32 + ceph/src/librbd/internal.cc | 3191 ++ ceph/src/librbd/internal.h | 215 + ceph/src/librbd/librbd.cc | 1167 + ceph/src/librbd/parent_types.h | 41 + ceph/src/libs3/COPYING | 674 + ceph/src/libs3/ChangeLog | 16 + ceph/src/libs3/GNUmakefile | 414 + ceph/src/libs3/GNUmakefile.mingw | 296 + ceph/src/libs3/GNUmakefile.osx | 305 + ceph/src/libs3/INSTALL | 73 + ceph/src/libs3/LICENSE | 20 + ceph/src/libs3/README | 4 + ceph/src/libs3/TODO | 3 + ceph/src/libs3/archlinux/PKGBUILD | 28 + ceph/src/libs3/debian/changelog | 5 + ceph/src/libs3/debian/changelog.Debian | 6 + ceph/src/libs3/debian/control | 12 + ceph/src/libs3/debian/control.dev | 26 + ceph/src/libs3/debian/postinst | 3 + ceph/src/libs3/doxyfile | 886 + ceph/src/libs3/inc/error_parser.h | 82 + ceph/src/libs3/inc/libs3.h | 1892 + ceph/src/libs3/inc/mingw/pthread.h | 45 + ceph/src/libs3/inc/mingw/sys/select.h | 30 + ceph/src/libs3/inc/mingw/sys/utsname.h | 41 + ceph/src/libs3/inc/request.h | 186 + ceph/src/libs3/inc/request_context.h | 40 + ceph/src/libs3/inc/response_headers_handler.h | 64 + ceph/src/libs3/inc/simplexml.h | 76 + ceph/src/libs3/inc/string_buffer.h | 107 + ceph/src/libs3/inc/util.h | 98 + ceph/src/libs3/libs3.spec | 81 + ceph/src/libs3/mswin/libs3.def | 27 + ceph/src/libs3/mswin/rmrf.bat | 9 + ceph/src/libs3/src/acl.c | 348 + ceph/src/libs3/src/bucket.c | 743 + ceph/src/libs3/src/error_parser.c | 239 + ceph/src/libs3/src/general.c | 473 + ceph/src/libs3/src/mingw_functions.c | 119 + ceph/src/libs3/src/mingw_s3_functions.c | 37 + ceph/src/libs3/src/object.c | 345 + ceph/src/libs3/src/request.c | 1392 + ceph/src/libs3/src/request_context.c | 190 + ceph/src/libs3/src/response_headers_handler.c | 205 + ceph/src/libs3/src/s3.c | 2787 ++ ceph/src/libs3/src/service.c | 191 + ceph/src/libs3/src/service_access_logging.c | 555 + ceph/src/libs3/src/simplexml.c | 207 + ceph/src/libs3/src/testsimplexml.c | 87 + ceph/src/libs3/src/util.c | 560 + ceph/src/libs3/test/badxml_01.xml | 105 + ceph/src/libs3/test/goodxml_01.xml | 7 + ceph/src/libs3/test/goodxml_02.xml | 105 + ceph/src/libs3/test/goodxml_03.xml | 13 + ceph/src/libs3/test/test.sh | 173 + ceph/src/log/Entry.h | 56 + ceph/src/log/EntryQueue.h | 71 + ceph/src/log/Log.cc | 331 + ceph/src/log/Log.h | 76 + ceph/src/log/Makefile.am | 11 + ceph/src/log/SubsystemMap.cc | 31 + ceph/src/log/SubsystemMap.h | 71 + ceph/src/log/test.cc | 189 + ceph/src/logrotate.conf | 29 + ceph/src/make_version | 23 + ceph/src/mds/Anchor.cc | 64 + ceph/src/mds/Anchor.h | 55 + ceph/src/mds/AnchorClient.cc | 129 + ceph/src/mds/AnchorClient.h | 46 + ceph/src/mds/AnchorServer.cc | 351 + ceph/src/mds/AnchorServer.h | 87 + ceph/src/mds/CDentry.cc | 580 + ceph/src/mds/CDentry.h | 408 + ceph/src/mds/CDir.cc | 2659 ++ ceph/src/mds/CDir.h | 632 + ceph/src/mds/CInode.cc | 3374 ++ ceph/src/mds/CInode.h | 865 + ceph/src/mds/Capability.cc | 202 + ceph/src/mds/Capability.h | 358 + ceph/src/mds/Dumper.cc | 274 + ceph/src/mds/Dumper.h | 47 + ceph/src/mds/InoTable.cc | 189 + ceph/src/mds/InoTable.h | 72 + ceph/src/mds/LocalLock.h | 65 + ceph/src/mds/Locker.cc | 4796 ++ ceph/src/mds/Locker.h | 300 + ceph/src/mds/LogEvent.cc | 115 + ceph/src/mds/LogEvent.h | 118 + ceph/src/mds/LogSegment.h | 89 + ceph/src/mds/MDBalancer.cc | 1149 + ceph/src/mds/MDBalancer.h | 125 + ceph/src/mds/MDCache.cc | 12223 +++++ ceph/src/mds/MDCache.h | 1071 + ceph/src/mds/MDLog.cc | 657 + ceph/src/mds/MDLog.h | 250 + ceph/src/mds/MDS.cc | 2263 + ceph/src/mds/MDS.h | 449 + ceph/src/mds/MDSMap.cc | 574 + ceph/src/mds/MDSMap.h | 569 + ceph/src/mds/MDSTable.cc | 165 + ceph/src/mds/MDSTable.h | 85 + ceph/src/mds/MDSTableClient.cc | 242 + ceph/src/mds/MDSTableClient.h | 102 + ceph/src/mds/MDSTableServer.cc | 176 + ceph/src/mds/MDSTableServer.h | 97 + ceph/src/mds/MDSUtility.cc | 160 + ceph/src/mds/MDSUtility.h | 58 + ceph/src/mds/Makefile.am | 94 + ceph/src/mds/Migrator.cc | 3040 ++ ceph/src/mds/Migrator.h | 352 + ceph/src/mds/Mutation.cc | 290 + ceph/src/mds/Mutation.h | 337 + ceph/src/mds/Resetter.cc | 114 + ceph/src/mds/Resetter.h | 37 + ceph/src/mds/ScatterLock.h | 251 + ceph/src/mds/Server.cc | 7607 ++++ ceph/src/mds/Server.h | 259 + ceph/src/mds/SessionMap.cc | 269 + ceph/src/mds/SessionMap.h | 401 + ceph/src/mds/SimpleLock.h | 691 + ceph/src/mds/SnapClient.h | 61 + ceph/src/mds/SnapRealm.cc | 488 + ceph/src/mds/SnapRealm.h | 149 + ceph/src/mds/SnapServer.cc | 340 + ceph/src/mds/SnapServer.h | 92 + ceph/src/mds/events/ECommitted.h | 42 + ceph/src/mds/events/EExport.h | 53 + ceph/src/mds/events/EFragment.h | 78 + ceph/src/mds/events/EImportFinish.h | 52 + ceph/src/mds/events/EImportStart.h | 57 + ceph/src/mds/events/EMetaBlob.h | 601 + ceph/src/mds/events/EOpen.h | 54 + ceph/src/mds/events/EResetJournal.h | 38 + ceph/src/mds/events/ESession.h | 67 + ceph/src/mds/events/ESessions.h | 59 + ceph/src/mds/events/ESlaveUpdate.h | 147 + ceph/src/mds/events/ESubtreeMap.h | 44 + ceph/src/mds/events/ETableClient.h | 48 + ceph/src/mds/events/ETableServer.h | 58 + ceph/src/mds/events/EUpdate.h | 50 + ceph/src/mds/flock.cc | 490 + ceph/src/mds/flock.h | 238 + ceph/src/mds/inode_backtrace.cc | 120 + ceph/src/mds/inode_backtrace.h | 73 + ceph/src/mds/journal.cc | 2787 ++ ceph/src/mds/locks.c | 161 + ceph/src/mds/locks.h | 144 + ceph/src/mds/mds_table_types.h | 75 + ceph/src/mds/mdstypes.cc | 893 + ceph/src/mds/mdstypes.h | 1454 + ceph/src/mds/snap.cc | 195 + ceph/src/mds/snap.h | 94 + ceph/src/messages/MAuth.h | 57 + ceph/src/messages/MAuthReply.h | 65 + ceph/src/messages/MBackfillReserve.h | 91 + ceph/src/messages/MCacheExpire.h | 104 + ceph/src/messages/MClientCapRelease.h | 51 + ceph/src/messages/MClientCaps.h | 225 + ceph/src/messages/MClientLease.h | 83 + ceph/src/messages/MClientReconnect.h | 107 + ceph/src/messages/MClientReply.h | 278 + ceph/src/messages/MClientRequest.h | 204 + ceph/src/messages/MClientRequestForward.h | 65 + ceph/src/messages/MClientSession.h | 67 + ceph/src/messages/MClientSnap.h | 66 + ceph/src/messages/MCommand.h | 59 + ceph/src/messages/MCommandReply.h | 55 + ceph/src/messages/MDentryLink.h | 67 + ceph/src/messages/MDentryUnlink.h | 57 + ceph/src/messages/MDirUpdate.h | 85 + ceph/src/messages/MDiscover.h | 103 + ceph/src/messages/MDiscoverReply.h | 230 + ceph/src/messages/MExportCaps.h | 53 + ceph/src/messages/MExportCapsAck.h | 49 + ceph/src/messages/MExportDir.h | 63 + ceph/src/messages/MExportDirAck.h | 53 + ceph/src/messages/MExportDirCancel.h | 50 + ceph/src/messages/MExportDirDiscover.h | 65 + ceph/src/messages/MExportDirDiscoverAck.h | 60 + ceph/src/messages/MExportDirFinish.h | 54 + ceph/src/messages/MExportDirNotify.h | 81 + ceph/src/messages/MExportDirNotifyAck.h | 52 + ceph/src/messages/MExportDirPrep.h | 85 + ceph/src/messages/MExportDirPrepAck.h | 54 + ceph/src/messages/MForward.h | 91 + ceph/src/messages/MGenericMessage.h | 39 + ceph/src/messages/MGetPoolStats.h | 57 + ceph/src/messages/MGetPoolStatsReply.h | 55 + ceph/src/messages/MHeartbeat.h | 63 + ceph/src/messages/MInodeFileCaps.h | 53 + ceph/src/messages/MLock.h | 96 + ceph/src/messages/MLog.h | 58 + ceph/src/messages/MLogAck.h | 47 + ceph/src/messages/MMDSBeacon.h | 100 + ceph/src/messages/MMDSCacheRejoin.h | 350 + ceph/src/messages/MMDSFindIno.h | 44 + ceph/src/messages/MMDSFindInoReply.h | 44 + ceph/src/messages/MMDSFragmentNotify.h | 64 + ceph/src/messages/MMDSLoadTargets.h | 58 + ceph/src/messages/MMDSMap.h | 97 + ceph/src/messages/MMDSOpenIno.h | 46 + ceph/src/messages/MMDSOpenInoReply.h | 53 + ceph/src/messages/MMDSResolve.h | 73 + ceph/src/messages/MMDSResolveAck.h | 59 + ceph/src/messages/MMDSSlaveRequest.h | 196 + ceph/src/messages/MMDSTableRequest.h | 65 + ceph/src/messages/MMonCommand.h | 61 + ceph/src/messages/MMonCommandAck.h | 54 + ceph/src/messages/MMonElection.h | 120 + ceph/src/messages/MMonGetMap.h | 35 + ceph/src/messages/MMonGetVersion.h | 58 + ceph/src/messages/MMonGetVersionReply.h | 64 + ceph/src/messages/MMonGlobalID.h | 43 + ceph/src/messages/MMonHealth.h | 85 + ceph/src/messages/MMonJoin.h | 59 + ceph/src/messages/MMonMap.h | 53 + ceph/src/messages/MMonPaxos.h | 127 + ceph/src/messages/MMonProbe.h | 128 + ceph/src/messages/MMonQuorumService.h | 72 + ceph/src/messages/MMonScrub.h | 78 + ceph/src/messages/MMonSubscribe.h | 95 + ceph/src/messages/MMonSubscribeAck.h | 50 + ceph/src/messages/MMonSync.h | 111 + ceph/src/messages/MOSDAlive.h | 49 + ceph/src/messages/MOSDBoot.h | 85 + ceph/src/messages/MOSDECSubOpRead.h | 62 + ceph/src/messages/MOSDECSubOpReadReply.h | 62 + ceph/src/messages/MOSDECSubOpWrite.h | 70 + ceph/src/messages/MOSDECSubOpWriteReply.h | 62 + ceph/src/messages/MOSDFailure.h | 78 + ceph/src/messages/MOSDMap.h | 144 + ceph/src/messages/MOSDMarkMeDown.h | 69 + ceph/src/messages/MOSDOp.h | 380 + ceph/src/messages/MOSDOpReply.h | 277 + ceph/src/messages/MOSDPGBackfill.h | 111 + ceph/src/messages/MOSDPGCreate.h | 81 + ceph/src/messages/MOSDPGInfo.h | 137 + ceph/src/messages/MOSDPGLog.h | 101 + ceph/src/messages/MOSDPGMissing.h | 57 + ceph/src/messages/MOSDPGNotify.h | 158 + ceph/src/messages/MOSDPGPull.h | 85 + ceph/src/messages/MOSDPGPush.h | 85 + ceph/src/messages/MOSDPGPushReply.h | 85 + ceph/src/messages/MOSDPGQuery.h | 103 + ceph/src/messages/MOSDPGRemove.h | 90 + ceph/src/messages/MOSDPGScan.h | 106 + ceph/src/messages/MOSDPGTemp.h | 52 + ceph/src/messages/MOSDPGTrim.h | 63 + ceph/src/messages/MOSDPing.h | 96 + ceph/src/messages/MOSDRepScrub.h | 120 + ceph/src/messages/MOSDScrub.h | 79 + ceph/src/messages/MOSDSubOp.h | 283 + ceph/src/messages/MOSDSubOpReply.h | 166 + ceph/src/messages/MPGStats.h | 65 + ceph/src/messages/MPGStatsAck.h | 44 + ceph/src/messages/MPing.h | 35 + ceph/src/messages/MPoolOp.h | 101 + ceph/src/messages/MPoolOpReply.h | 80 + ceph/src/messages/MRecoveryReserve.h | 83 + ceph/src/messages/MRemoveSnaps.h | 51 + ceph/src/messages/MRoute.h | 88 + ceph/src/messages/MStatfs.h | 52 + ceph/src/messages/MStatfsReply.h | 44 + ceph/src/messages/MTimeCheck.h | 87 + ceph/src/messages/MWatchNotify.h | 64 + ceph/src/messages/Makefile.am | 117 + ceph/src/messages/PaxosServiceMessage.h | 70 + ceph/src/mkcephfs.in | 564 + ceph/src/mon/AuthMonitor.cc | 1083 + ceph/src/mon/AuthMonitor.h | 167 + ceph/src/mon/ConfigKeyService.cc | 203 + ceph/src/mon/ConfigKeyService.h | 78 + ceph/src/mon/DataHealthService.cc | 268 + ceph/src/mon/DataHealthService.h | 86 + ceph/src/mon/DumplingMonCommands.h | 531 + ceph/src/mon/Elector.cc | 489 + ceph/src/mon/Elector.h | 436 + ceph/src/mon/HealthMonitor.cc | 103 + ceph/src/mon/HealthMonitor.h | 82 + ceph/src/mon/HealthService.h | 47 + ceph/src/mon/LogMonitor.cc | 550 + ceph/src/mon/LogMonitor.h | 99 + ceph/src/mon/MDSMonitor.cc | 1437 + ceph/src/mon/MDSMonitor.h | 127 + ceph/src/mon/Makefile.am | 49 + ceph/src/mon/MonCap.cc | 450 + ceph/src/mon/MonCap.h | 160 + ceph/src/mon/MonClient.cc | 1048 + ceph/src/mon/MonClient.h | 446 + ceph/src/mon/MonCommands.h | 635 + ceph/src/mon/MonMap.cc | 337 + ceph/src/mon/MonMap.h | 247 + ceph/src/mon/Monitor.cc | 4550 ++ ceph/src/mon/Monitor.h | 965 + ceph/src/mon/MonitorDBStore.h | 577 + ceph/src/mon/MonitorStore.cc | 487 + ceph/src/mon/MonitorStore.h | 109 + ceph/src/mon/MonmapMonitor.cc | 504 + ceph/src/mon/MonmapMonitor.h | 89 + ceph/src/mon/OSDMonitor.cc | 5967 +++ ceph/src/mon/OSDMonitor.h | 412 + ceph/src/mon/PGMap.cc | 1224 + ceph/src/mon/PGMap.h | 316 + ceph/src/mon/PGMonitor.cc | 2129 + ceph/src/mon/PGMonitor.h | 215 + ceph/src/mon/Paxos.cc | 1371 + ceph/src/mon/Paxos.h | 1322 + ceph/src/mon/PaxosService.cc | 386 + ceph/src/mon/PaxosService.h | 911 + ceph/src/mon/QuorumService.h | 135 + ceph/src/mon/Session.h | 206 + ceph/src/mon/mon_types.h | 182 + ceph/src/mount.fuse.ceph | 29 + ceph/src/mount/canonicalize.c | 203 + ceph/src/mount/mount.ceph.c | 380 + ceph/src/mount/mtab.c | 280 + ceph/src/msg/Accepter.cc | 258 + ceph/src/msg/Accepter.h | 44 + ceph/src/msg/DispatchQueue.cc | 177 + ceph/src/msg/DispatchQueue.h | 184 + ceph/src/msg/Dispatcher.h | 120 + ceph/src/msg/Makefile.am | 20 + ceph/src/msg/Message.cc | 787 + ceph/src/msg/Message.h | 544 + ceph/src/msg/Messenger.cc | 13 + ceph/src/msg/Messenger.h | 699 + ceph/src/msg/Pipe.cc | 2358 + ceph/src/msg/Pipe.h | 322 + ceph/src/msg/SimpleMessenger.cc | 737 + ceph/src/msg/SimpleMessenger.h | 472 + ceph/src/msg/msg_types.cc | 154 + ceph/src/msg/msg_types.h | 444 + ceph/src/objclass/class_api.cc | 625 + ceph/src/objclass/objclass.h | 156 + ceph/src/ocf/Makefile.am | 23 + ceph/src/ocf/Makefile.in | 534 + ceph/src/ocf/ceph.in | 177 + ceph/src/ocf/rbd.in | 296 + ceph/src/os/BtrfsFileStoreBackend.cc | 573 + ceph/src/os/BtrfsFileStoreBackend.h | 46 + ceph/src/os/CollectionIndex.h | 190 + ceph/src/os/DBObjectMap.cc | 1268 + ceph/src/os/DBObjectMap.h | 489 + ceph/src/os/FDCache.h | 98 + ceph/src/os/FileJournal.cc | 1877 + ceph/src/os/FileJournal.h | 479 + ceph/src/os/FileStore.cc | 4964 +++ ceph/src/os/FileStore.h | 773 + ceph/src/os/FlatIndex.cc | 430 + ceph/src/os/FlatIndex.h | 87 + ceph/src/os/GenericFileStoreBackend.cc | 366 + ceph/src/os/GenericFileStoreBackend.h | 59 + ceph/src/os/GenericObjectMap.cc | 1111 + ceph/src/os/GenericObjectMap.h | 427 + ceph/src/os/HashIndex.cc | 737 + ceph/src/os/HashIndex.h | 340 + ceph/src/os/IndexManager.cc | 136 + ceph/src/os/IndexManager.h | 112 + ceph/src/os/Journal.h | 78 + ceph/src/os/JournalingObjectStore.cc | 270 + ceph/src/os/JournalingObjectStore.h | 139 + ceph/src/os/KeyValueDB.h | 181 + ceph/src/os/KeyValueStore.cc | 3013 ++ ceph/src/os/KeyValueStore.h | 647 + ceph/src/os/LFNIndex.cc | 1367 + ceph/src/os/LFNIndex.h | 589 + ceph/src/os/LevelDBStore.cc | 288 + ceph/src/os/LevelDBStore.h | 399 + ceph/src/os/Makefile.am | 75 + ceph/src/os/MemStore.cc | 1487 + ceph/src/os/MemStore.h | 357 + ceph/src/os/ObjectMap.h | 161 + ceph/src/os/ObjectStore.cc | 159 + ceph/src/os/ObjectStore.h | 1577 + ceph/src/os/SequencerPosition.h | 59 + ceph/src/os/Transaction.cc | 471 + ceph/src/os/WBThrottle.cc | 260 + ceph/src/os/WBThrottle.h | 173 + ceph/src/os/XfsFileStoreBackend.cc | 137 + ceph/src/os/XfsFileStoreBackend.h | 33 + ceph/src/os/ZFS.cc | 83 + ceph/src/os/ZFS.h | 39 + ceph/src/os/ZFSFileStoreBackend.cc | 260 + ceph/src/os/ZFSFileStoreBackend.h | 30 + ceph/src/os/btrfs_ioctl.h | 201 + ceph/src/os/chain_xattr.cc | 437 + ceph/src/os/chain_xattr.h | 73 + ceph/src/osd/Ager.cc | 271 + ceph/src/osd/Ager.h | 43 + ceph/src/osd/ClassHandler.cc | 267 + ceph/src/osd/ClassHandler.h | 95 + ceph/src/osd/ECBackend.cc | 1785 + ceph/src/osd/ECBackend.h | 480 + ceph/src/osd/ECMsgTypes.cc | 355 + ceph/src/osd/ECMsgTypes.h | 114 + ceph/src/osd/ECTransaction.cc | 283 + ceph/src/osd/ECTransaction.h | 207 + ceph/src/osd/ECUtil.cc | 196 + ceph/src/osd/ECUtil.h | 154 + ceph/src/osd/HitSet.cc | 218 + ceph/src/osd/HitSet.h | 477 + ceph/src/osd/Makefile.am | 52 + ceph/src/osd/OSD.cc | 8043 ++++ ceph/src/osd/OSD.h | 2019 + ceph/src/osd/OSDCap.cc | 260 + ceph/src/osd/OSDCap.h | 158 + ceph/src/osd/OSDMap.cc | 2727 ++ ceph/src/osd/OSDMap.h | 843 + ceph/src/osd/ObjectVersioner.h | 35 + ceph/src/osd/OpRequest.cc | 86 + ceph/src/osd/OpRequest.h | 177 + ceph/src/osd/PG.cc | 7697 ++++ ceph/src/osd/PG.h | 2252 + ceph/src/osd/PGBackend.cc | 577 + ceph/src/osd/PGBackend.h | 646 + ceph/src/osd/PGLog.cc | 1054 + ceph/src/osd/PGLog.h | 587 + ceph/src/osd/ReplicatedBackend.cc | 758 + ceph/src/osd/ReplicatedBackend.h | 420 + ceph/src/osd/ReplicatedPG.cc | 12226 +++++ ceph/src/osd/ReplicatedPG.h | 1453 + ceph/src/osd/SnapMapper.cc | 307 + ceph/src/osd/SnapMapper.h | 233 + ceph/src/osd/TierAgentState.h | 119 + ceph/src/osd/Watch.cc | 457 + ceph/src/osd/Watch.h | 268 + ceph/src/osd/osd_types.cc | 4494 ++ ceph/src/osd/osd_types.h | 3452 ++ ceph/src/osdc/Blinker.h | 92 + ceph/src/osdc/Filer.cc | 304 + ceph/src/osdc/Filer.h | 290 + ceph/src/osdc/Journaler.cc | 1064 + ceph/src/osdc/Journaler.h | 400 + ceph/src/osdc/Makefile.am | 17 + ceph/src/osdc/ObjectCacher.cc | 2049 + ceph/src/osdc/ObjectCacher.h | 728 + ceph/src/osdc/Objecter.cc | 2897 ++ ceph/src/osdc/Objecter.h | 2193 + ceph/src/osdc/Striper.cc | 340 + ceph/src/osdc/Striper.h | 97 + ceph/src/osdc/WritebackHandler.h | 42 + ceph/src/perfglue/Makefile.am | 23 + ceph/src/perfglue/cpu_profiler.cc | 39 + ceph/src/perfglue/cpu_profiler.h | 25 + ceph/src/perfglue/disabled_heap_profiler.cc | 33 + ceph/src/perfglue/disabled_stubs.cc | 25 + ceph/src/perfglue/heap_profiler.cc | 118 + ceph/src/perfglue/heap_profiler.h | 49 + ceph/src/pybind/ceph_argparse.py | 1118 + ceph/src/pybind/ceph_rest_api.py | 499 + ceph/src/pybind/cephfs.py | 356 + ceph/src/pybind/rados.py | 1771 + ceph/src/pybind/rbd.py | 962 + ceph/src/rbd.cc | 3132 ++ ceph/src/rbd_fuse/rbd-fuse.c | 764 + ceph/src/rbdmap | 2 + ceph/src/rgw/Makefile.am | 164 + ceph/src/rgw/librgw.cc | 134 + ceph/src/rgw/logrotate.conf | 26 + ceph/src/rgw/rgw_acl.cc | 117 + ceph/src/rgw/rgw_acl.h | 334 + ceph/src/rgw/rgw_acl_s3.cc | 578 + ceph/src/rgw/rgw_acl_s3.h | 135 + ceph/src/rgw/rgw_acl_swift.cc | 131 + ceph/src/rgw/rgw_acl_swift.h | 25 + ceph/src/rgw/rgw_admin.cc | 2430 + ceph/src/rgw/rgw_auth_s3.cc | 211 + ceph/src/rgw/rgw_auth_s3.h | 15 + ceph/src/rgw/rgw_bucket.cc | 1778 + ceph/src/rgw/rgw_bucket.h | 389 + ceph/src/rgw/rgw_cache.cc | 165 + ceph/src/rgw/rgw_cache.h | 580 + ceph/src/rgw/rgw_civetweb.cc | 172 + ceph/src/rgw/rgw_civetweb.h | 41 + ceph/src/rgw/rgw_client_io.cc | 75 + ceph/src/rgw/rgw_client_io.h | 50 + ceph/src/rgw/rgw_common.cc | 1047 + ceph/src/rgw/rgw_common.h | 1355 + ceph/src/rgw/rgw_cors.cc | 182 + ceph/src/rgw/rgw_cors.h | 135 + ceph/src/rgw/rgw_cors_s3.cc | 239 + ceph/src/rgw/rgw_cors_s3.h | 58 + ceph/src/rgw/rgw_cors_swift.h | 78 + ceph/src/rgw/rgw_dencoder.cc | 458 + ceph/src/rgw/rgw_env.cc | 127 + ceph/src/rgw/rgw_fcgi.cc | 58 + ceph/src/rgw/rgw_fcgi.h | 29 + ceph/src/rgw/rgw_formats.cc | 250 + ceph/src/rgw/rgw_formats.h | 97 + ceph/src/rgw/rgw_gc.cc | 308 + ceph/src/rgw/rgw_gc.h | 63 + ceph/src/rgw/rgw_http_client.cc | 289 + ceph/src/rgw/rgw_http_client.h | 41 + ceph/src/rgw/rgw_http_errors.h | 142 + ceph/src/rgw/rgw_json_enc.cc | 819 + ceph/src/rgw/rgw_jsonparser.cc | 130 + ceph/src/rgw/rgw_keystone.cc | 108 + ceph/src/rgw/rgw_keystone.h | 106 + ceph/src/rgw/rgw_loadgen.cc | 111 + ceph/src/rgw/rgw_loadgen.h | 45 + ceph/src/rgw/rgw_log.cc | 412 + ceph/src/rgw/rgw_log.h | 144 + ceph/src/rgw/rgw_main.cc | 1205 + ceph/src/rgw/rgw_metadata.cc | 632 + ceph/src/rgw/rgw_metadata.h | 216 + ceph/src/rgw/rgw_multi.cc | 64 + ceph/src/rgw/rgw_multi.h | 52 + ceph/src/rgw/rgw_multi_del.cc | 64 + ceph/src/rgw/rgw_multi_del.h | 54 + ceph/src/rgw/rgw_multiparser.cc | 44 + ceph/src/rgw/rgw_op.cc | 3170 ++ ceph/src/rgw/rgw_op.h | 919 + ceph/src/rgw/rgw_policy_s3.cc | 295 + ceph/src/rgw/rgw_policy_s3.h | 56 + ceph/src/rgw/rgw_quota.cc | 751 + ceph/src/rgw/rgw_quota.h | 75 + ceph/src/rgw/rgw_rados.cc | 6924 +++ ceph/src/rgw/rgw_rados.h | 1927 + ceph/src/rgw/rgw_replica_log.cc | 132 + ceph/src/rgw/rgw_replica_log.h | 116 + ceph/src/rgw/rgw_resolve.cc | 183 + ceph/src/rgw/rgw_resolve.h | 22 + ceph/src/rgw/rgw_rest.cc | 1285 + ceph/src/rgw/rgw_rest.h | 382 + ceph/src/rgw/rgw_rest_admin.h | 12 + ceph/src/rgw/rgw_rest_bucket.cc | 257 + ceph/src/rgw/rgw_rest_bucket.h | 33 + ceph/src/rgw/rgw_rest_client.cc | 671 + ceph/src/rgw/rgw_rest_client.h | 98 + ceph/src/rgw/rgw_rest_config.cc | 47 + ceph/src/rgw/rgw_rest_config.h | 55 + ceph/src/rgw/rgw_rest_conn.cc | 119 + ceph/src/rgw/rgw_rest_conn.h | 34 + ceph/src/rgw/rgw_rest_log.cc | 742 + ceph/src/rgw/rgw_rest_log.h | 306 + ceph/src/rgw/rgw_rest_metadata.cc | 291 + ceph/src/rgw/rgw_rest_metadata.h | 123 + ceph/src/rgw/rgw_rest_opstate.cc | 189 + ceph/src/rgw/rgw_rest_opstate.h | 109 + ceph/src/rgw/rgw_rest_replica_log.cc | 301 + ceph/src/rgw/rgw_rest_replica_log.h | 154 + ceph/src/rgw/rgw_rest_s3.cc | 2237 + ceph/src/rgw/rgw_rest_s3.h | 421 + ceph/src/rgw/rgw_rest_swift.cc | 992 + ceph/src/rgw/rgw_rest_swift.h | 226 + ceph/src/rgw/rgw_rest_usage.cc | 96 + ceph/src/rgw/rgw_rest_usage.h | 32 + ceph/src/rgw/rgw_rest_user.cc | 917 + ceph/src/rgw/rgw_rest_user.h | 33 + ceph/src/rgw/rgw_string.h | 94 + ceph/src/rgw/rgw_swift.cc | 740 + ceph/src/rgw/rgw_swift.h | 76 + ceph/src/rgw/rgw_swift_auth.cc | 248 + ceph/src/rgw/rgw_swift_auth.h | 49 + ceph/src/rgw/rgw_tools.cc | 170 + ceph/src/rgw/rgw_tools.h | 23 + ceph/src/rgw/rgw_usage.cc | 140 + ceph/src/rgw/rgw_usage.h | 26 + ceph/src/rgw/rgw_user.cc | 2479 ++ ceph/src/rgw/rgw_user.h | 661 + ceph/src/rgw/rgw_xml.cc | 237 + ceph/src/rgw/rgw_xml.h | 85 + ceph/src/sample.ceph.conf | 364 + ceph/src/stop.sh | 93 + ceph/src/test/Makefile.am | 907 + ceph/src/test/ObjectMap/KeyValueDBMemory.cc | 249 + ceph/src/test/ObjectMap/KeyValueDBMemory.h | 169 + .../ObjectMap/test_keyvaluedb_atomicity.cc | 106 + .../ObjectMap/test_keyvaluedb_iterators.cc | 1787 + ceph/src/test/ObjectMap/test_object_map.cc | 786 + ceph/src/test/TestSignalHandlers.cc | 97 + ceph/src/test/TestTimers.cc | 284 + ceph/src/test/admin_socket.cc | 312 + ceph/src/test/base64.cc | 100 + ceph/src/test/bench/backend.h | 26 + ceph/src/test/bench/bencher.cc | 201 + ceph/src/test/bench/bencher.h | 147 + .../src/test/bench/detailed_stat_collector.cc | 163 + ceph/src/test/bench/detailed_stat_collector.h | 96 + ceph/src/test/bench/distribution.h | 136 + ceph/src/test/bench/dumb_backend.cc | 117 + ceph/src/test/bench/dumb_backend.h | 168 + ceph/src/test/bench/rados_backend.cc | 62 + ceph/src/test/bench/rados_backend.h | 31 + ceph/src/test/bench/rbd_backend.cc | 51 + ceph/src/test/bench/rbd_backend.h | 30 + ceph/src/test/bench/small_io_bench.cc | 206 + ceph/src/test/bench/small_io_bench_dumb.cc | 231 + ceph/src/test/bench/small_io_bench_fs.cc | 246 + ceph/src/test/bench/small_io_bench_rbd.cc | 200 + ceph/src/test/bench/stat_collector.h | 19 + ceph/src/test/bench/testfilestore_backend.cc | 77 + ceph/src/test/bench/testfilestore_backend.h | 37 + ceph/src/test/bench/tp_bench.cc | 205 + ceph/src/test/bench_log.cc | 71 + ceph/src/test/bufferlist.cc | 2195 + ceph/src/test/buildtest_skeleton.cc | 13 + ceph/src/test/ceph_argparse.cc | 445 + ceph/src/test/ceph_compatset.cc | 164 + ceph/src/test/ceph_crypto.cc | 142 + ceph/src/test/cli/.gitignore | 1 + .../src/test/cli/ceph-authtool/add-key-segv.t | 6 + ceph/src/test/cli/ceph-authtool/add-key.t | 14 + ceph/src/test/cli/ceph-authtool/cap-bin.t | 6 + ceph/src/test/cli/ceph-authtool/cap-invalid.t | 12 + .../test/cli/ceph-authtool/cap-overwrite.t | 11 + ceph/src/test/cli/ceph-authtool/cap.t | 11 + .../cli/ceph-authtool/create-gen-list-bin.t | 17 + .../test/cli/ceph-authtool/create-gen-list.t | 20 + ceph/src/test/cli/ceph-authtool/help.t | 25 + .../test/cli/ceph-authtool/list-empty-bin.t | 5 + ceph/src/test/cli/ceph-authtool/list-empty.t | 5 + .../cli/ceph-authtool/list-nonexistent-bin.t | 7 + .../test/cli/ceph-authtool/list-nonexistent.t | 7 + ceph/src/test/cli/ceph-authtool/manpage.t | 53 + ceph/src/test/cli/ceph-authtool/simple.t | 24 + ceph/src/test/cli/ceph-conf/env-vs-args.t | 10 + ceph/src/test/cli/ceph-conf/help.t | 36 + ceph/src/test/cli/ceph-conf/invalid-args.t | 17 + ceph/src/test/cli/ceph-conf/manpage.t | 33 + ceph/src/test/cli/ceph-conf/option.t | 67 + ceph/src/test/cli/ceph-conf/sections.t | 18 + .../test/cli/ceph-conf/show-config-value.t | 20 + ceph/src/test/cli/ceph-conf/show-config.t | 6 + ceph/src/test/cli/ceph-conf/simple.t | 4 + ceph/src/test/cli/crushtool/add-item.t | 15 + .../cli/crushtool/bad-mappings.crushmap.txt | 39 + ceph/src/test/cli/crushtool/bad-mappings.t | 6 + ceph/src/test/cli/crushtool/build.t | 104 + .../crushtool/compile-decompile-recompile.t | 11 + .../test/cli/crushtool/five-devices.crushmap | Bin 0 -> 368 bytes ceph/src/test/cli/crushtool/help.t | 90 + ceph/src/test/cli/crushtool/multitype.after | 87 + ceph/src/test/cli/crushtool/multitype.before | 87 + .../test/cli/crushtool/need_tree_order.crush | 64 + ceph/src/test/cli/crushtool/output-csv.t | 51 + ceph/src/test/cli/crushtool/reweight.t | 8 + .../test/cli/crushtool/reweight_multiple.t | 5 + .../cli/crushtool/set-choose.crushmap.txt | 144 + ceph/src/test/cli/crushtool/set-choose.t | 36941 ++++++++++++++++ ceph/src/test/cli/crushtool/simple.template | Bin 0 -> 316 bytes .../test/cli/crushtool/simple.template.five | 65 + .../test/cli/crushtool/simple.template.four | 56 + .../cli/crushtool/simple.template.multitree | 70 + .../simple.template.multitree.reweighted | 73 + .../test/cli/crushtool/simple.template.one | 58 + .../test/cli/crushtool/simple.template.three | 58 + .../test/cli/crushtool/simple.template.two | 58 + .../test/cli/crushtool/test-map-a.crushmap | Bin 0 -> 31995 bytes .../cli/crushtool/test-map-bobtail-tunables.t | 10253 +++++ .../cli/crushtool/test-map-firefly-tunables.t | 10259 +++++ .../cli/crushtool/test-map-indep.crushmap | Bin 0 -> 31995 bytes ceph/src/test/cli/crushtool/test-map-indep.t | 10253 +++++ .../cli/crushtool/test-map-legacy-tunables.t | 10252 +++++ .../test-map-tries-vs-retries.crushmap | Bin 0 -> 808 bytes .../cli/crushtool/test-map-tries-vs-retries.t | 10259 +++++ .../test/cli/crushtool/test-map-vary-r-0.t | 3081 ++ .../test/cli/crushtool/test-map-vary-r-1.t | 3078 ++ .../test/cli/crushtool/test-map-vary-r-2.t | 3078 ++ .../test/cli/crushtool/test-map-vary-r-3.t | 3078 ++ .../test/cli/crushtool/test-map-vary-r-4.t | 3078 ++ .../cli/crushtool/test-map-vary-r.crushmap | Bin 0 -> 3892 bytes ceph/src/test/cli/monmaptool/add-exists.t | 26 + ceph/src/test/cli/monmaptool/add-many.t | 29 + ceph/src/test/cli/monmaptool/clobber.t | 38 + ceph/src/test/cli/monmaptool/create-print.t | 18 + .../src/test/cli/monmaptool/create-with-add.t | 12 + ceph/src/test/cli/monmaptool/help.t | 3 + ceph/src/test/cli/monmaptool/print-empty.t | 5 + .../test/cli/monmaptool/print-nonexistent.t | 4 + ceph/src/test/cli/monmaptool/rm-nonexistent.t | 24 + ceph/src/test/cli/monmaptool/rm.t | 21 + ceph/src/test/cli/monmaptool/simple.t | 4 + .../test/cli/osdmaptool/ceph.conf.withracks | 1480 + ceph/src/test/cli/osdmaptool/clobber.t | 57 + ceph/src/test/cli/osdmaptool/create-print.t | 104 + ceph/src/test/cli/osdmaptool/create-racks.t | 818 + ceph/src/test/cli/osdmaptool/crush.t | 10 + ceph/src/test/cli/osdmaptool/help.t | 12 + .../test/cli/osdmaptool/missing-argument.t | 12 + ceph/src/test/cli/osdmaptool/pool.t | 54 + ceph/src/test/cli/osdmaptool/print-empty.t | 5 + .../test/cli/osdmaptool/print-nonexistent.t | 4 + ceph/src/test/cli/osdmaptool/test-map-pgs.t | 52 + ceph/src/test/cli/radosgw-admin/help.t | 139 + ceph/src/test/cli/rbd/help.t | 81 + ceph/src/test/cli/rbd/invalid-snap-usage.t | 36 + ceph/src/test/cli/rbd/not-enough-args.t | 33 + ceph/src/test/cls_hello/test_cls_hello.cc | 133 + ceph/src/test/cls_lock/test_cls_lock.cc | 300 + ceph/src/test/cls_log/test_cls_log.cc | 334 + ceph/src/test/cls_rbd/test_cls_rbd.cc | 919 + .../test/cls_refcount/test_cls_refcount.cc | 258 + .../cls_replica_log/test_cls_replica_log.cc | 151 + ceph/src/test/cls_rgw/test_cls_rgw.cc | 539 + .../test/cls_statelog/test_cls_statelog.cc | 211 + ceph/src/test/cls_version/test_cls_version.cc | 318 + ceph/src/test/common/ObjectContents.cc | 128 + ceph/src/test/common/ObjectContents.h | 121 + ceph/src/test/common/Throttle.cc | 261 + .../test/common/get_command_descriptions.cc | 113 + ceph/src/test/common/histogram.cc | 126 + ceph/src/test/common/test_bloom_filter.cc | 289 + ceph/src/test/common/test_config.cc | 187 + ceph/src/test/common/test_context.cc | 64 + ceph/src/test/common/test_crc32c.cc | 255 + .../test/common/test_sharedptr_registry.cc | 342 + ceph/src/test/common/test_sloppy_crc_map.cc | 113 + ceph/src/test/common/test_str_map.cc | 70 + ceph/src/test/common/test_util.cc | 32 + ceph/src/test/confutils.cc | 507 + ceph/src/test/crush/TestCrushWrapper.cc | 638 + ceph/src/test/crush/indep.cc | 262 + ceph/src/test/crypto.cc | 148 + ceph/src/test/crypto_init.cc | 48 + ceph/src/test/daemon_config.cc | 336 + .../cram-0.5.0ceph.2011-01-14.tar.gz | Bin 0 -> 23497 bytes ceph/src/test/encoding.cc | 198 + ceph/src/test/encoding/ceph_dencoder.cc | 411 + ceph/src/test/encoding/check-generated.sh | 77 + ceph/src/test/encoding/readable.sh | 82 + ceph/src/test/encoding/types.h | 528 + .../test/erasure-code/ErasureCodeExample.h | 184 + .../erasure-code/ErasureCodePluginExample.cc | 36 + .../ErasureCodePluginFailToInitialize.cc | 23 + .../ErasureCodePluginFailToRegister.cc | 22 + .../erasure-code/ErasureCodePluginHangs.cc | 24 + .../ErasureCodePluginMissingEntryPoint.cc | 4 + ceph/src/test/erasure-code/Makefile.am | 115 + .../erasure-code/TestErasureCodeExample.cc | 258 + .../erasure-code/TestErasureCodeJerasure.cc | 391 + .../erasure-code/TestErasureCodePlugin.cc | 110 + .../TestErasureCodePluginJerasure.cc | 235 + .../erasure-code/TestJerasurePluginGeneric.cc | 23 + .../erasure-code/TestJerasurePluginSSE3.cc | 23 + .../erasure-code/TestJerasurePluginSSE4.cc | 23 + .../test/erasure-code/ceph_erasure_code.cc | 166 + .../ceph_erasure_code_benchmark.cc | 218 + .../ceph_erasure_code_benchmark.h | 38 + ceph/src/test/escape.cc | 95 + ceph/src/test/formatter.cc | 180 + ceph/src/test/gather.cc | 103 + ceph/src/test/heartbeat_map.cc | 44 + ceph/src/test/kv_store_bench.cc | 560 + ceph/src/test/kv_store_bench.h | 194 + ceph/src/test/libcephfs/caps.cc | 92 + ceph/src/test/libcephfs/multiclient.cc | 97 + ceph/src/test/libcephfs/readdir_r_cb.cc | 63 + ceph/src/test/libcephfs/test.cc | 1163 + ceph/src/test/libcephfs_config.cc | 63 + ceph/src/test/librados/TestCase.cc | 239 + ceph/src/test/librados/TestCase.h | 111 + ceph/src/test/librados/aio.cc | 2865 ++ ceph/src/test/librados/c_read_operations.cc | 539 + ceph/src/test/librados/c_write_operations.cc | 142 + ceph/src/test/librados/cls.cc | 37 + ceph/src/test/librados/cmd.cc | 210 + ceph/src/test/librados/io.cc | 966 + ceph/src/test/librados/librados.cc | 13 + ceph/src/test/librados/librados_config.cc | 98 + ceph/src/test/librados/list.cc | 633 + ceph/src/test/librados/lock.cc | 374 + ceph/src/test/librados/misc.cc | 621 + ceph/src/test/librados/pool.cc | 106 + ceph/src/test/librados/snapshots.cc | 798 + ceph/src/test/librados/stat.cc | 247 + ceph/src/test/librados/test.cc | 257 + ceph/src/test/librados/test.h | 50 + ceph/src/test/librados/tier.cc | 3957 ++ ceph/src/test/librados/watch_notify.cc | 139 + ceph/src/test/librbd/fsx.c | 1622 + ceph/src/test/librbd/test_librbd.cc | 1873 + ceph/src/test/mime.cc | 150 + ceph/src/test/mon/PGMap.cc | 96 + ceph/src/test/mon/mon-test-helpers.sh | 113 + ceph/src/test/mon/moncap.cc | 222 + ceph/src/test/mon/test_mon_workloadgen.cc | 1114 + ceph/src/test/multi_stress_watch.cc | 160 + .../objectstore/DeterministicOpSequence.cc | 526 + .../objectstore/DeterministicOpSequence.h | 98 + ceph/src/test/objectstore/FileStoreDiff.cc | 319 + ceph/src/test/objectstore/FileStoreDiff.h | 43 + ceph/src/test/objectstore/FileStoreTracker.cc | 452 + ceph/src/test/objectstore/FileStoreTracker.h | 138 + .../test/objectstore/TestObjectStoreState.cc | 296 + .../test/objectstore/TestObjectStoreState.h | 148 + ceph/src/test/objectstore/chain_xattr.cc | 217 + ceph/src/test/objectstore/store_test.cc | 1318 + ceph/src/test/objectstore/test_idempotent.cc | 113 + .../objectstore/test_idempotent_sequence.cc | 247 + .../test/objectstore/workload_generator.cc | 575 + .../src/test/objectstore/workload_generator.h | 184 + ceph/src/test/omap_bench.cc | 439 + ceph/src/test/omap_bench.h | 208 + ceph/src/test/on_exit.cc | 113 + ceph/src/test/os/TestFlatIndex.cc | 143 + ceph/src/test/os/TestLFNIndex.cc | 467 + ceph/src/test/osd/Object.cc | 182 + ceph/src/test/osd/Object.h | 367 + ceph/src/test/osd/RadosModel.cc | 37 + ceph/src/test/osd/RadosModel.h | 1996 + ceph/src/test/osd/TestECBackend.cc | 60 + ceph/src/test/osd/TestOSDMap.cc | 401 + ceph/src/test/osd/TestOpStat.cc | 61 + ceph/src/test/osd/TestOpStat.h | 53 + ceph/src/test/osd/TestPGLog.cc | 1878 + ceph/src/test/osd/TestRados.cc | 403 + ceph/src/test/osd/hitset.cc | 197 + ceph/src/test/osd/osd-test-helpers.sh | 55 + ceph/src/test/osd/osdcap.cc | 590 + ceph/src/test/osd/types.cc | 1304 + ceph/src/test/osdc/FakeWriteback.cc | 87 + ceph/src/test/osdc/FakeWriteback.h | 40 + ceph/src/test/osdc/object_cacher_stress.cc | 233 + ceph/src/test/perf_counters.cc | 165 + ceph/src/test/rgw/test_rgw_manifest.cc | 227 + ceph/src/test/run-cli-tests | 57 + .../src/test/run-cli-tests-maybe-unset-ccache | 20 + ceph/src/test/run_cmd.cc | 26 + ceph/src/test/signals.cc | 142 + ceph/src/test/simple_spin.cc | 39 + ceph/src/test/streamtest.cc | 191 + ceph/src/test/strtol.cc | 211 + ceph/src/test/system/cross_process_sem.cc | 107 + ceph/src/test/system/cross_process_sem.h | 40 + .../system/rados_delete_pools_parallel.cc | 110 + ceph/src/test/system/rados_list_parallel.cc | 340 + .../test/system/rados_open_pools_parallel.cc | 136 + ceph/src/test/system/rados_watch_notify.cc | 191 + ceph/src/test/system/st_rados_create_pool.cc | 110 + ceph/src/test/system/st_rados_create_pool.h | 51 + ceph/src/test/system/st_rados_delete_objs.cc | 71 + ceph/src/test/system/st_rados_delete_objs.h | 48 + ceph/src/test/system/st_rados_delete_pool.cc | 59 + ceph/src/test/system/st_rados_delete_pool.h | 43 + ceph/src/test/system/st_rados_list_objects.cc | 103 + ceph/src/test/system/st_rados_list_objects.h | 53 + ceph/src/test/system/st_rados_notify.cc | 74 + ceph/src/test/system/st_rados_notify.h | 52 + ceph/src/test/system/st_rados_watch.cc | 96 + ceph/src/test/system/st_rados_watch.h | 56 + ceph/src/test/system/systest_runnable.cc | 248 + ceph/src/test/system/systest_runnable.h | 88 + ceph/src/test/system/systest_settings.cc | 64 + ceph/src/test/system/systest_settings.h | 36 + ceph/src/test/test_addrs.cc | 64 + ceph/src/test/test_arch.cc | 20 + ceph/src/test/test_c_headers.c | 22 + ceph/src/test/test_cfuse_cache_invalidate.cc | 53 + ceph/src/test/test_cors.cc | 909 + ceph/src/test/test_filejournal.cc | 546 + ceph/src/test/test_get_blkdev_size.cc | 35 + ceph/src/test/test_ipaddr.cc | 478 + ceph/src/test/test_mutate.cc | 110 + ceph/src/test/test_prebufferedstreambuf.cc | 95 + ceph/src/test/test_rewrite_latency.cc | 47 + ceph/src/test/test_rgw_admin_log.cc | 1608 + ceph/src/test/test_rgw_admin_meta.cc | 936 + ceph/src/test/test_rgw_admin_opstate.cc | 834 + ceph/src/test/test_snap_mapper.cc | 670 + ceph/src/test/test_str_list.cc | 37 + ceph/src/test/test_stress_watch.cc | 118 + ceph/src/test/test_striper.cc | 74 + ceph/src/test/test_texttable.cc | 77 + ceph/src/test/test_trans.cc | 77 + ceph/src/test/test_workqueue.cc | 72 + ceph/src/test/testcrypto.cc | 56 + ceph/src/test/testkeys.cc | 64 + ceph/src/test/testmsgr.cc | 145 + ceph/src/test/unit.h | 43 + ceph/src/test/utf8.cc | 66 + ceph/src/test/xattr_bench.cc | 200 + ceph/src/tools/Makefile.am | 104 + ceph/src/tools/ceph-client-debug.cc | 179 + ceph/src/tools/ceph_authtool.cc | 277 + ceph/src/tools/ceph_conf.cc | 229 + ceph/src/tools/ceph_filestore_dump.cc | 1429 + ceph/src/tools/ceph_filestore_tool.cc | 260 + ceph/src/tools/ceph_kvstore_tool.cc | 424 + ceph/src/tools/ceph_monstore_tool.cc | 420 + ceph/src/tools/ceph_osdomap_tool.cc | 162 + ceph/src/tools/common.h | 132 + ceph/src/tools/crushtool.cc | 749 + ceph/src/tools/dupstore.cc | 113 + ceph/src/tools/mon_store_converter.cc | 336 + ceph/src/tools/monmaptool.cc | 208 + ceph/src/tools/osdmaptool.cc | 488 + ceph/src/tools/psim.cc | 114 + ceph/src/tools/rados/rados.cc | 2657 ++ ceph/src/tools/rados/rados_export.cc | 229 + ceph/src/tools/rados/rados_import.cc | 239 + ceph/src/tools/rados/rados_sync.cc | 901 + ceph/src/tools/rados/rados_sync.h | 216 + ceph/src/tools/radosacl.cc | 202 + ceph/src/tools/rest_bench.cc | 802 + ceph/src/tools/scratchtool.c | 312 + ceph/src/tools/scratchtoolpp.cc | 308 + ceph/src/unittest_bufferlist.sh | 19 + ceph/src/upstart/ceph-all.conf | 4 + ceph/src/upstart/ceph-create-keys.conf | 8 + ceph/src/upstart/ceph-mds-all-starter.conf | 18 + ceph/src/upstart/ceph-mds-all.conf | 4 + ceph/src/upstart/ceph-mds.conf | 26 + ceph/src/upstart/ceph-mon-all-starter.conf | 19 + ceph/src/upstart/ceph-mon-all.conf | 4 + ceph/src/upstart/ceph-mon.conf | 31 + ceph/src/upstart/ceph-osd-all-starter.conf | 22 + ceph/src/upstart/ceph-osd-all.conf | 4 + ceph/src/upstart/ceph-osd.conf | 54 + ceph/src/upstart/radosgw-all-starter.conf | 18 + ceph/src/upstart/radosgw-all.conf | 4 + ceph/src/upstart/radosgw.conf | 26 + ceph/src/upstart/rbdmap.conf | 48 + ceph/src/verify-mds-journal.sh | 8 + ceph/src/vstart.sh | 653 + ceph/src/yasm-wrapper | 38 + ceph/udev/50-rbd.rules | 2 + ceph/udev/60-ceph-partuuid-workaround.rules | 37 + ceph/udev/95-ceph-osd-alt.rules | 5 + ceph/udev/95-ceph-osd.rules | 27 + debian/ceph-common.install | 24 + debian/ceph-fs-common.install | 4 + debian/ceph-fuse.install | 3 + debian/ceph-mds.install | 2 + debian/ceph-mds.lintian-overrides | 10 + debian/ceph-mds.postinst | 47 + debian/ceph-mds.prerm | 29 + debian/ceph-resource-agents.install | 1 + debian/ceph-test.install | 28 + debian/ceph.dirs | 8 + debian/ceph.install | 32 + debian/ceph.lintian-overrides | 26 + debian/ceph.postinst | 54 + debian/ceph.postrm | 47 + debian/ceph.prerm | 29 + debian/changelog | 313 + debian/compat | 1 + debian/control | 468 + debian/copyright | 256 + debian/gbp.conf | 6 + debian/libcephfs-dev.install | 3 + debian/libcephfs-java.jlibs | 1 + debian/libcephfs-jni.install | 1 + debian/libcephfs1.install | 1 + debian/librados-dev.install | 12 + debian/librados2.install | 1 + debian/librbd-dev.install | 5 + debian/librbd1.install | 4 + ...s-get_obj-returns-wrong-len-if-len-0.patch | 31 + ...-to-identify-whether-object-has-tail.patch | 41 + debian/patches/modules.patch | 53 + debian/patches/series | 4 + debian/patches/virtualenv-never-download | 16 + debian/python-ceph.install | 1 + debian/radosgw.dirs | 3 + debian/radosgw.install | 5 + debian/radosgw.lintian-overrides | 10 + debian/radosgw.postinst | 62 + debian/radosgw.postrm | 13 + debian/radosgw.preinst | 13 + debian/radosgw.prerm | 30 + debian/rbd-fuse.install | 2 + debian/rest-bench.install | 1 + debian/rules | 105 + debian/source/format | 1 + debian/tests/build-rados | 31 + debian/tests/build-rbd | 24 + debian/tests/ceph-client | 11 + debian/tests/control | 3 + debian/tests/python-ceph | 7 + debian/watch | 3 + tests/integration_tests.conf | 1 + 1785 files changed, 830464 insertions(+) create mode 100644 ceph/AUTHORS create mode 100644 ceph/COPYING create mode 100644 ceph/ChangeLog create mode 100644 ceph/INSTALL create mode 100644 ceph/Makefile.am create mode 100644 ceph/Makefile.in create mode 100644 ceph/NEWS create mode 100644 ceph/README create mode 100644 ceph/aclocal.m4 create mode 100755 ceph/ar-lib create mode 100755 ceph/autogen.sh create mode 100644 ceph/ceph.spec create mode 100644 ceph/ceph.spec.in create mode 100755 ceph/compile create mode 100755 ceph/config.guess create mode 100755 ceph/config.sub create mode 100755 ceph/configure create mode 100644 ceph/configure.ac create mode 100755 ceph/depcomp create mode 100755 ceph/install-sh create mode 100644 ceph/ltmain.sh create mode 100644 ceph/m4/ac_check_classpath.m4 create mode 100644 ceph/m4/ac_prog_jar.m4 create mode 100644 ceph/m4/ac_prog_javac.m4 create mode 100644 ceph/m4/ac_prog_javac_works.m4 create mode 100644 ceph/m4/ac_prog_javah.m4 create mode 100644 ceph/m4/acx_pthread.m4 create mode 100644 ceph/m4/ax_c_pretty_func.m4 create mode 100644 ceph/m4/ax_c_var_func.m4 create mode 100644 ceph/m4/ax_check_compile_flag.m4 create mode 100644 ceph/m4/ax_cxx_static_cast.m4 create mode 100644 ceph/m4/ax_intel.m4 create mode 100644 ceph/m4/libtool.m4 create mode 100644 ceph/m4/ltoptions.m4 create mode 100644 ceph/m4/ltsugar.m4 create mode 100644 ceph/m4/ltversion.m4 create mode 100644 ceph/m4/lt~obsolete.m4 create mode 100644 ceph/m4/pkg.m4 create mode 100644 ceph/man/Makefile.am create mode 100644 ceph/man/Makefile.in create mode 100644 ceph/man/ceph-authtool.8 create mode 100644 ceph/man/ceph-clsinfo.8 create mode 100644 ceph/man/ceph-conf.8 create mode 100644 ceph/man/ceph-debugpack.8 create mode 100644 ceph/man/ceph-dencoder.8 create mode 100644 ceph/man/ceph-fuse.8 create mode 100644 ceph/man/ceph-mds.8 create mode 100644 ceph/man/ceph-mon.8 create mode 100644 ceph/man/ceph-osd.8 create mode 100644 ceph/man/ceph-post-file.8 create mode 100644 ceph/man/ceph-rbdnamer.8 create mode 100644 ceph/man/ceph-rest-api.8 create mode 100644 ceph/man/ceph-run.8 create mode 100644 ceph/man/ceph-syn.8 create mode 100644 ceph/man/ceph.8 create mode 100644 ceph/man/cephfs.8 create mode 100644 ceph/man/crushtool.8 create mode 100644 ceph/man/librados-config.8 create mode 100644 ceph/man/mkcephfs.8 create mode 100644 ceph/man/monmaptool.8 create mode 100644 ceph/man/mount.ceph.8 create mode 100644 ceph/man/osdmaptool.8 create mode 100644 ceph/man/rados.8 create mode 100644 ceph/man/radosgw-admin.8 create mode 100644 ceph/man/radosgw.8 create mode 100644 ceph/man/rbd-fuse.8 create mode 100644 ceph/man/rbd.8 create mode 100755 ceph/missing create mode 100755 ceph/py-compile create mode 100644 ceph/share/id_dsa_drop.ceph.com create mode 100644 ceph/share/id_dsa_drop.ceph.com.pub create mode 100644 ceph/share/known_hosts_drop.ceph.com create mode 100644 ceph/src/.git_version create mode 100644 ceph/src/Makefile-env.am create mode 100644 ceph/src/Makefile.am create mode 100644 ceph/src/Makefile.in create mode 100644 ceph/src/README create mode 100644 ceph/src/TODO create mode 100644 ceph/src/acconfig.h.in create mode 100644 ceph/src/arch/Makefile.am create mode 100644 ceph/src/arch/intel.c create mode 100644 ceph/src/arch/intel.h create mode 100644 ceph/src/arch/neon.c create mode 100644 ceph/src/arch/neon.h create mode 100644 ceph/src/arch/probe.cc create mode 100644 ceph/src/arch/probe.h create mode 100644 ceph/src/auth/Auth.h create mode 100644 ceph/src/auth/AuthAuthorizeHandler.cc create mode 100644 ceph/src/auth/AuthAuthorizeHandler.h create mode 100644 ceph/src/auth/AuthClientHandler.cc create mode 100644 ceph/src/auth/AuthClientHandler.h create mode 100644 ceph/src/auth/AuthMethodList.cc create mode 100644 ceph/src/auth/AuthMethodList.h create mode 100644 ceph/src/auth/AuthServiceHandler.cc create mode 100644 ceph/src/auth/AuthServiceHandler.h create mode 100644 ceph/src/auth/AuthSessionHandler.cc create mode 100644 ceph/src/auth/AuthSessionHandler.h create mode 100644 ceph/src/auth/Crypto.cc create mode 100644 ceph/src/auth/Crypto.h create mode 100644 ceph/src/auth/KeyRing.cc create mode 100644 ceph/src/auth/KeyRing.h create mode 100644 ceph/src/auth/Makefile.am create mode 100644 ceph/src/auth/RotatingKeyRing.cc create mode 100644 ceph/src/auth/RotatingKeyRing.h create mode 100644 ceph/src/auth/cephx/CephxAuthorizeHandler.cc create mode 100644 ceph/src/auth/cephx/CephxAuthorizeHandler.h create mode 100644 ceph/src/auth/cephx/CephxClientHandler.cc create mode 100644 ceph/src/auth/cephx/CephxClientHandler.h create mode 100644 ceph/src/auth/cephx/CephxKeyServer.cc create mode 100644 ceph/src/auth/cephx/CephxKeyServer.h create mode 100644 ceph/src/auth/cephx/CephxProtocol.cc create mode 100644 ceph/src/auth/cephx/CephxProtocol.h create mode 100644 ceph/src/auth/cephx/CephxServiceHandler.cc create mode 100644 ceph/src/auth/cephx/CephxServiceHandler.h create mode 100644 ceph/src/auth/cephx/CephxSessionHandler.cc create mode 100644 ceph/src/auth/cephx/CephxSessionHandler.h create mode 100644 ceph/src/auth/none/AuthNoneAuthorizeHandler.cc create mode 100644 ceph/src/auth/none/AuthNoneAuthorizeHandler.h create mode 100644 ceph/src/auth/none/AuthNoneClientHandler.h create mode 100644 ceph/src/auth/none/AuthNoneProtocol.h create mode 100644 ceph/src/auth/none/AuthNoneServiceHandler.h create mode 100644 ceph/src/auth/none/AuthNoneSessionHandler.h create mode 100644 ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc create mode 100644 ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h create mode 100644 ceph/src/auth/unknown/AuthUnknownClientHandler.h create mode 100644 ceph/src/auth/unknown/AuthUnknownProtocol.h create mode 100644 ceph/src/auth/unknown/AuthUnknownServiceHandler.h create mode 100644 ceph/src/auth/unknown/AuthUnknownSessionHandler.h create mode 100644 ceph/src/bash_completion/ceph create mode 100644 ceph/src/bash_completion/rados create mode 100644 ceph/src/bash_completion/radosgw-admin create mode 100644 ceph/src/bash_completion/rbd create mode 100644 ceph/src/brag/Makefile.am create mode 100644 ceph/src/brag/README.md create mode 100755 ceph/src/brag/client/ceph-brag create mode 100644 ceph/src/brag/server/MANIFEST.in create mode 100644 ceph/src/brag/server/app.wsgi create mode 100644 ceph/src/brag/server/ceph_brag.egg-info/PKG-INFO create mode 100644 ceph/src/brag/server/ceph_brag.egg-info/SOURCES.txt create mode 100644 ceph/src/brag/server/ceph_brag.egg-info/dependency_links.txt create mode 100644 ceph/src/brag/server/ceph_brag.egg-info/not-zip-safe create mode 100644 ceph/src/brag/server/ceph_brag.egg-info/requires.txt create mode 100644 ceph/src/brag/server/ceph_brag.egg-info/top_level.txt create mode 100644 ceph/src/brag/server/ceph_brag/__init__.py create mode 100644 ceph/src/brag/server/ceph_brag/app.py create mode 100644 ceph/src/brag/server/ceph_brag/controllers/__init__.py create mode 100644 ceph/src/brag/server/ceph_brag/controllers/root.py create mode 100644 ceph/src/brag/server/ceph_brag/json.py create mode 100644 ceph/src/brag/server/ceph_brag/model/__init__.py create mode 100644 ceph/src/brag/server/ceph_brag/model/db.py create mode 100644 ceph/src/brag/server/ceph_brag/tests/__init__.py create mode 100644 ceph/src/brag/server/ceph_brag/tests/config.py create mode 100644 ceph/src/brag/server/ceph_brag/tests/test_functional.py create mode 100644 ceph/src/brag/server/ceph_brag/tests/test_units.py create mode 100644 ceph/src/brag/server/config.py create mode 100644 ceph/src/brag/server/sample.json create mode 100644 ceph/src/brag/server/setup.cfg create mode 100644 ceph/src/brag/server/setup.py create mode 100755 ceph/src/ceph-clsinfo create mode 100644 ceph/src/ceph-coverage.in create mode 100755 ceph/src/ceph-create-keys create mode 100755 ceph/src/ceph-crush-location create mode 100755 ceph/src/ceph-crush-location.in create mode 100644 ceph/src/ceph-debugpack.in create mode 100755 ceph/src/ceph-disk create mode 100755 ceph/src/ceph-disk-activate create mode 100755 ceph/src/ceph-disk-prepare create mode 100755 ceph/src/ceph-disk-udev create mode 100755 ceph/src/ceph-post-file.in create mode 100755 ceph/src/ceph-rbdnamer create mode 100755 ceph/src/ceph-rest-api create mode 100755 ceph/src/ceph-run create mode 100755 ceph/src/ceph.in create mode 100644 ceph/src/ceph_common.sh create mode 100644 ceph/src/ceph_fuse.cc create mode 100644 ceph/src/ceph_mds.cc create mode 100644 ceph/src/ceph_mon.cc create mode 100644 ceph/src/ceph_osd.cc create mode 100644 ceph/src/ceph_syn.cc create mode 100644 ceph/src/ceph_ver.c create mode 100644 ceph/src/cephfs.cc create mode 100755 ceph/src/check_version create mode 100644 ceph/src/civetweb/civetweb.h create mode 100644 ceph/src/civetweb/include/civetweb.h create mode 100644 ceph/src/civetweb/src/civetweb.c create mode 100644 ceph/src/civetweb/src/md5.h create mode 100644 ceph/src/client/Client.cc create mode 100644 ceph/src/client/Client.h create mode 100644 ceph/src/client/ClientSnapRealm.cc create mode 100644 ceph/src/client/ClientSnapRealm.h create mode 100644 ceph/src/client/Dentry.cc create mode 100644 ceph/src/client/Dentry.h create mode 100644 ceph/src/client/Dir.h create mode 100644 ceph/src/client/Fh.h create mode 100644 ceph/src/client/Inode.cc create mode 100644 ceph/src/client/Inode.h create mode 100644 ceph/src/client/Makefile.am create mode 100644 ceph/src/client/MetaRequest.cc create mode 100644 ceph/src/client/MetaRequest.h create mode 100644 ceph/src/client/MetaSession.cc create mode 100644 ceph/src/client/MetaSession.h create mode 100644 ceph/src/client/ObjecterWriteback.h create mode 100644 ceph/src/client/SyntheticClient.cc create mode 100644 ceph/src/client/SyntheticClient.h create mode 100644 ceph/src/client/Trace.cc create mode 100644 ceph/src/client/Trace.h create mode 100644 ceph/src/client/fuse_ll.cc create mode 100644 ceph/src/client/fuse_ll.h create mode 100644 ceph/src/client/ioctl.h create mode 100644 ceph/src/client/test_ioctls.c create mode 100644 ceph/src/cls/Makefile.am create mode 100644 ceph/src/cls/hello/cls_hello.cc create mode 100644 ceph/src/cls/lock/cls_lock.cc create mode 100644 ceph/src/cls/lock/cls_lock_client.cc create mode 100644 ceph/src/cls/lock/cls_lock_client.h create mode 100644 ceph/src/cls/lock/cls_lock_ops.cc create mode 100644 ceph/src/cls/lock/cls_lock_ops.h create mode 100644 ceph/src/cls/lock/cls_lock_types.cc create mode 100644 ceph/src/cls/lock/cls_lock_types.h create mode 100644 ceph/src/cls/log/cls_log.cc create mode 100644 ceph/src/cls/log/cls_log_client.cc create mode 100644 ceph/src/cls/log/cls_log_client.h create mode 100644 ceph/src/cls/log/cls_log_ops.h create mode 100644 ceph/src/cls/log/cls_log_types.h create mode 100644 ceph/src/cls/rbd/cls_rbd.cc create mode 100644 ceph/src/cls/rbd/cls_rbd.h create mode 100644 ceph/src/cls/rbd/cls_rbd_client.cc create mode 100644 ceph/src/cls/rbd/cls_rbd_client.h create mode 100644 ceph/src/cls/refcount/cls_refcount.cc create mode 100644 ceph/src/cls/refcount/cls_refcount_client.cc create mode 100644 ceph/src/cls/refcount/cls_refcount_client.h create mode 100644 ceph/src/cls/refcount/cls_refcount_ops.cc create mode 100644 ceph/src/cls/refcount/cls_refcount_ops.h create mode 100644 ceph/src/cls/replica_log/cls_replica_log.cc create mode 100644 ceph/src/cls/replica_log/cls_replica_log_client.cc create mode 100644 ceph/src/cls/replica_log/cls_replica_log_client.h create mode 100644 ceph/src/cls/replica_log/cls_replica_log_ops.cc create mode 100644 ceph/src/cls/replica_log/cls_replica_log_ops.h create mode 100644 ceph/src/cls/replica_log/cls_replica_log_types.cc create mode 100644 ceph/src/cls/replica_log/cls_replica_log_types.h create mode 100644 ceph/src/cls/rgw/cls_rgw.cc create mode 100644 ceph/src/cls/rgw/cls_rgw_client.cc create mode 100644 ceph/src/cls/rgw/cls_rgw_client.h create mode 100644 ceph/src/cls/rgw/cls_rgw_ops.cc create mode 100644 ceph/src/cls/rgw/cls_rgw_ops.h create mode 100644 ceph/src/cls/rgw/cls_rgw_types.cc create mode 100644 ceph/src/cls/rgw/cls_rgw_types.h create mode 100644 ceph/src/cls/statelog/cls_statelog.cc create mode 100644 ceph/src/cls/statelog/cls_statelog_client.cc create mode 100644 ceph/src/cls/statelog/cls_statelog_client.h create mode 100644 ceph/src/cls/statelog/cls_statelog_ops.h create mode 100644 ceph/src/cls/statelog/cls_statelog_types.h create mode 100644 ceph/src/cls/user/cls_user.cc create mode 100644 ceph/src/cls/user/cls_user_client.cc create mode 100644 ceph/src/cls/user/cls_user_client.h create mode 100644 ceph/src/cls/user/cls_user_ops.cc create mode 100644 ceph/src/cls/user/cls_user_ops.h create mode 100644 ceph/src/cls/user/cls_user_types.cc create mode 100644 ceph/src/cls/user/cls_user_types.h create mode 100644 ceph/src/cls/version/cls_version.cc create mode 100644 ceph/src/cls/version/cls_version_client.cc create mode 100644 ceph/src/cls/version/cls_version_client.h create mode 100644 ceph/src/cls/version/cls_version_ops.h create mode 100644 ceph/src/cls/version/cls_version_types.cc create mode 100644 ceph/src/cls/version/cls_version_types.h create mode 100644 ceph/src/cls_acl.cc create mode 100644 ceph/src/cls_crypto.cc create mode 100644 ceph/src/common/AsyncReserver.h create mode 100644 ceph/src/common/BackTrace.cc create mode 100644 ceph/src/common/BackTrace.h create mode 100644 ceph/src/common/Clock.cc create mode 100644 ceph/src/common/Clock.h create mode 100644 ceph/src/common/Cond.h create mode 100644 ceph/src/common/ConfUtils.cc create mode 100644 ceph/src/common/ConfUtils.h create mode 100644 ceph/src/common/DecayCounter.cc create mode 100644 ceph/src/common/DecayCounter.h create mode 100644 ceph/src/common/Finisher.cc create mode 100644 ceph/src/common/Finisher.h create mode 100644 ceph/src/common/Formatter.cc create mode 100644 ceph/src/common/Formatter.h create mode 100644 ceph/src/common/HeartbeatMap.cc create mode 100644 ceph/src/common/HeartbeatMap.h create mode 100644 ceph/src/common/LogClient.cc create mode 100644 ceph/src/common/LogClient.h create mode 100644 ceph/src/common/LogEntry.cc create mode 100644 ceph/src/common/LogEntry.h create mode 100644 ceph/src/common/Makefile.am create mode 100644 ceph/src/common/MemoryModel.cc create mode 100644 ceph/src/common/MemoryModel.h create mode 100644 ceph/src/common/Mutex.cc create mode 100644 ceph/src/common/Mutex.h create mode 100644 ceph/src/common/OutputDataSocket.cc create mode 100644 ceph/src/common/OutputDataSocket.h create mode 100644 ceph/src/common/PrebufferedStreambuf.cc create mode 100644 ceph/src/common/PrebufferedStreambuf.h create mode 100644 ceph/src/common/Preforker.h create mode 100644 ceph/src/common/PrioritizedQueue.h create mode 100644 ceph/src/common/RWLock.h create mode 100644 ceph/src/common/RefCountedObj.cc create mode 100644 ceph/src/common/RefCountedObj.h create mode 100644 ceph/src/common/Semaphore.h create mode 100644 ceph/src/common/SimpleRNG.h create mode 100644 ceph/src/common/SloppyCRCMap.cc create mode 100644 ceph/src/common/SloppyCRCMap.h create mode 100644 ceph/src/common/TextTable.cc create mode 100644 ceph/src/common/TextTable.h create mode 100644 ceph/src/common/Thread.cc create mode 100644 ceph/src/common/Thread.h create mode 100644 ceph/src/common/Throttle.cc create mode 100644 ceph/src/common/Throttle.h create mode 100644 ceph/src/common/Timer.cc create mode 100644 ceph/src/common/Timer.h create mode 100644 ceph/src/common/TrackedOp.cc create mode 100644 ceph/src/common/TrackedOp.h create mode 100644 ceph/src/common/WorkQueue.cc create mode 100644 ceph/src/common/WorkQueue.h create mode 100644 ceph/src/common/addr_parsing.c create mode 100644 ceph/src/common/admin_socket.cc create mode 100644 ceph/src/common/admin_socket.h create mode 100644 ceph/src/common/admin_socket_client.cc create mode 100644 ceph/src/common/admin_socket_client.h create mode 100644 ceph/src/common/arch.h create mode 100644 ceph/src/common/armor.c create mode 100644 ceph/src/common/armor.h create mode 100644 ceph/src/common/assert.cc create mode 100644 ceph/src/common/blkdev.cc create mode 100644 ceph/src/common/blkdev.h create mode 100644 ceph/src/common/bloom_filter.cc create mode 100644 ceph/src/common/bloom_filter.hpp create mode 100644 ceph/src/common/buffer.cc create mode 100644 ceph/src/common/ceph_argparse.cc create mode 100644 ceph/src/common/ceph_argparse.h create mode 100644 ceph/src/common/ceph_context.cc create mode 100644 ceph/src/common/ceph_context.h create mode 100644 ceph/src/common/ceph_crypto.cc create mode 100644 ceph/src/common/ceph_crypto.h create mode 100644 ceph/src/common/ceph_crypto_cms.cc create mode 100644 ceph/src/common/ceph_crypto_cms.h create mode 100644 ceph/src/common/ceph_frag.cc create mode 100644 ceph/src/common/ceph_fs.cc create mode 100644 ceph/src/common/ceph_hash.cc create mode 100644 ceph/src/common/ceph_json.cc create mode 100644 ceph/src/common/ceph_json.h create mode 100644 ceph/src/common/ceph_strings.cc create mode 100644 ceph/src/common/cmdparse.cc create mode 100644 ceph/src/common/cmdparse.h create mode 100644 ceph/src/common/code_environment.cc create mode 100644 ceph/src/common/code_environment.h create mode 100644 ceph/src/common/common_init.cc create mode 100644 ceph/src/common/common_init.h create mode 100644 ceph/src/common/compiler_extensions.h create mode 100644 ceph/src/common/config.cc create mode 100644 ceph/src/common/config.h create mode 100644 ceph/src/common/config_obs.h create mode 100644 ceph/src/common/config_opts.h create mode 100644 ceph/src/common/crc32c.cc create mode 100644 ceph/src/common/crc32c_intel_baseline.c create mode 100644 ceph/src/common/crc32c_intel_baseline.h create mode 100644 ceph/src/common/crc32c_intel_fast.c create mode 100644 ceph/src/common/crc32c_intel_fast.h create mode 100644 ceph/src/common/crc32c_intel_fast_asm.S create mode 100644 ceph/src/common/crc32c_intel_fast_zero_asm.S create mode 100644 ceph/src/common/debug.h create mode 100644 ceph/src/common/dout.cc create mode 100644 ceph/src/common/dout.h create mode 100644 ceph/src/common/entity_name.cc create mode 100644 ceph/src/common/entity_name.h create mode 100644 ceph/src/common/environment.cc create mode 100644 ceph/src/common/environment.h create mode 100644 ceph/src/common/errno.cc create mode 100644 ceph/src/common/errno.h create mode 100644 ceph/src/common/escape.c create mode 100644 ceph/src/common/escape.h create mode 100644 ceph/src/common/fd.cc create mode 100644 ceph/src/common/fd.h create mode 100644 ceph/src/common/hex.cc create mode 100644 ceph/src/common/hex.h create mode 100644 ceph/src/common/histogram.cc create mode 100644 ceph/src/common/histogram.h create mode 100644 ceph/src/common/hobject.cc create mode 100644 ceph/src/common/hobject.h create mode 100644 ceph/src/common/io_priority.cc create mode 100644 ceph/src/common/io_priority.h create mode 100644 ceph/src/common/ipaddr.cc create mode 100644 ceph/src/common/likely.h create mode 100644 ceph/src/common/linux_version.c create mode 100644 ceph/src/common/linux_version.h create mode 100644 ceph/src/common/lockdep.cc create mode 100644 ceph/src/common/lockdep.h create mode 100644 ceph/src/common/lru_map.h create mode 100644 ceph/src/common/map_cacher.hpp create mode 100644 ceph/src/common/mime.c create mode 100644 ceph/src/common/mime.h create mode 100644 ceph/src/common/obj_bencher.cc create mode 100644 ceph/src/common/obj_bencher.h create mode 100644 ceph/src/common/page.cc create mode 100644 ceph/src/common/perf_counters.cc create mode 100644 ceph/src/common/perf_counters.h create mode 100644 ceph/src/common/pick_address.cc create mode 100644 ceph/src/common/pick_address.h create mode 100644 ceph/src/common/pipe.c create mode 100644 ceph/src/common/pipe.h create mode 100644 ceph/src/common/random_cache.hpp create mode 100644 ceph/src/common/run_cmd.cc create mode 100644 ceph/src/common/run_cmd.h create mode 100644 ceph/src/common/safe_io.c create mode 100644 ceph/src/common/safe_io.h create mode 100644 ceph/src/common/sctp_crc32.c create mode 100644 ceph/src/common/sctp_crc32.h create mode 100644 ceph/src/common/secret.c create mode 100644 ceph/src/common/secret.h create mode 100644 ceph/src/common/shared_cache.hpp create mode 100644 ceph/src/common/sharedptr_registry.hpp create mode 100644 ceph/src/common/signal.cc create mode 100644 ceph/src/common/signal.h create mode 100644 ceph/src/common/simple_cache.hpp create mode 100644 ceph/src/common/simple_spin.cc create mode 100644 ceph/src/common/simple_spin.h create mode 100644 ceph/src/common/snap_types.cc create mode 100644 ceph/src/common/snap_types.h create mode 100644 ceph/src/common/static_assert.h create mode 100644 ceph/src/common/str_list.cc create mode 100644 ceph/src/common/str_map.cc create mode 100644 ceph/src/common/strtol.cc create mode 100644 ceph/src/common/strtol.h create mode 100644 ceph/src/common/sync_filesystem.h create mode 100644 ceph/src/common/tracked_int_ptr.hpp create mode 100644 ceph/src/common/utf8.c create mode 100644 ceph/src/common/utf8.h create mode 100644 ceph/src/common/util.cc create mode 100644 ceph/src/common/version.cc create mode 100644 ceph/src/common/version.h create mode 100644 ceph/src/common/xattr.c create mode 100644 ceph/src/common/xattr.h create mode 100644 ceph/src/crush/CrushCompiler.cc create mode 100644 ceph/src/crush/CrushCompiler.h create mode 100644 ceph/src/crush/CrushTester.cc create mode 100644 ceph/src/crush/CrushTester.h create mode 100644 ceph/src/crush/CrushWrapper.cc create mode 100644 ceph/src/crush/CrushWrapper.h create mode 100644 ceph/src/crush/CrushWrapper.i create mode 100644 ceph/src/crush/Makefile.am create mode 100644 ceph/src/crush/builder.c create mode 100644 ceph/src/crush/builder.h create mode 100644 ceph/src/crush/crush.c create mode 100644 ceph/src/crush/crush.h create mode 100644 ceph/src/crush/grammar.h create mode 100644 ceph/src/crush/hash.c create mode 100644 ceph/src/crush/hash.h create mode 100644 ceph/src/crush/mapper.c create mode 100644 ceph/src/crush/mapper.h create mode 100644 ceph/src/crush/sample.txt create mode 100644 ceph/src/crush/types.h create mode 100644 ceph/src/erasure-code/ErasureCodeInterface.h create mode 100644 ceph/src/erasure-code/ErasureCodePlugin.cc create mode 100644 ceph/src/erasure-code/ErasureCodePlugin.h create mode 100644 ceph/src/erasure-code/Makefile.am create mode 100644 ceph/src/erasure-code/jerasure/ErasureCodeJerasure.cc create mode 100644 ceph/src/erasure-code/jerasure/ErasureCodeJerasure.h create mode 100644 ceph/src/erasure-code/jerasure/ErasureCodePluginJerasure.cc create mode 100644 ceph/src/erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc create mode 100644 ceph/src/erasure-code/jerasure/Makefile.am create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/include/gf_complete.h create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/include/gf_general.h create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/include/gf_int.h create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/include/gf_method.h create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/include/gf_rand.h create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_general.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_method.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_rand.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_w128.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_w16.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_w32.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_w4.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_w64.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_w8.c create mode 100644 ceph/src/erasure-code/jerasure/gf-complete/src/gf_wgen.c create mode 100644 ceph/src/erasure-code/jerasure/jerasure/include/cauchy.h create mode 100644 ceph/src/erasure-code/jerasure/jerasure/include/galois.h create mode 100644 ceph/src/erasure-code/jerasure/jerasure/include/jerasure.h create mode 100644 ceph/src/erasure-code/jerasure/jerasure/include/liberation.h create mode 100644 ceph/src/erasure-code/jerasure/jerasure/include/reed_sol.h create mode 100644 ceph/src/erasure-code/jerasure/jerasure/src/cauchy.c create mode 100644 ceph/src/erasure-code/jerasure/jerasure/src/galois.c create mode 100644 ceph/src/erasure-code/jerasure/jerasure/src/jerasure.c create mode 100644 ceph/src/erasure-code/jerasure/jerasure/src/liberation.c create mode 100644 ceph/src/erasure-code/jerasure/jerasure/src/reed_sol.c create mode 100644 ceph/src/fetch_config create mode 100644 ceph/src/global/Makefile.am create mode 100644 ceph/src/global/global_context.cc create mode 100644 ceph/src/global/global_context.h create mode 100644 ceph/src/global/global_init.cc create mode 100644 ceph/src/global/global_init.h create mode 100644 ceph/src/global/pidfile.cc create mode 100644 ceph/src/global/pidfile.h create mode 100644 ceph/src/global/signal_handler.cc create mode 100644 ceph/src/global/signal_handler.h create mode 100644 ceph/src/gtest/CHANGES create mode 100644 ceph/src/gtest/CMakeLists.txt create mode 100644 ceph/src/gtest/CONTRIBUTORS create mode 100644 ceph/src/gtest/COPYING create mode 100644 ceph/src/gtest/Makefile.am create mode 100644 ceph/src/gtest/Makefile.in create mode 100644 ceph/src/gtest/README create mode 100644 ceph/src/gtest/aclocal.m4 create mode 100755 ceph/src/gtest/build-aux/config.guess create mode 100644 ceph/src/gtest/build-aux/config.h.in create mode 100755 ceph/src/gtest/build-aux/config.sub create mode 100755 ceph/src/gtest/build-aux/depcomp create mode 100755 ceph/src/gtest/build-aux/install-sh create mode 100644 ceph/src/gtest/build-aux/ltmain.sh create mode 100755 ceph/src/gtest/build-aux/missing create mode 100644 ceph/src/gtest/codegear/gtest.cbproj create mode 100644 ceph/src/gtest/codegear/gtest.groupproj create mode 100644 ceph/src/gtest/codegear/gtest_all.cc create mode 100644 ceph/src/gtest/codegear/gtest_link.cc create mode 100644 ceph/src/gtest/codegear/gtest_main.cbproj create mode 100644 ceph/src/gtest/codegear/gtest_unittest.cbproj create mode 100755 ceph/src/gtest/configure create mode 100644 ceph/src/gtest/configure.ac create mode 100644 ceph/src/gtest/fused-src/gtest/gtest-all.cc create mode 100644 ceph/src/gtest/fused-src/gtest/gtest.h create mode 100644 ceph/src/gtest/fused-src/gtest/gtest_main.cc create mode 100644 ceph/src/gtest/include/gtest/gtest-death-test.h create mode 100644 ceph/src/gtest/include/gtest/gtest-message.h create mode 100644 ceph/src/gtest/include/gtest/gtest-param-test.h create mode 100644 ceph/src/gtest/include/gtest/gtest-param-test.h.pump create mode 100644 ceph/src/gtest/include/gtest/gtest-spi.h create mode 100644 ceph/src/gtest/include/gtest/gtest-test-part.h create mode 100644 ceph/src/gtest/include/gtest/gtest-typed-test.h create mode 100644 ceph/src/gtest/include/gtest/gtest.h create mode 100644 ceph/src/gtest/include/gtest/gtest_pred_impl.h create mode 100644 ceph/src/gtest/include/gtest/gtest_prod.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-death-test-internal.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-filepath.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-internal.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-linked_ptr.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-param-util-generated.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-param-util-generated.h.pump create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-param-util.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-port.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-string.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-tuple.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-tuple.h.pump create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-type-util.h create mode 100644 ceph/src/gtest/include/gtest/internal/gtest-type-util.h.pump create mode 100644 ceph/src/gtest/m4/acx_pthread.m4 create mode 100644 ceph/src/gtest/m4/gtest.m4 create mode 100644 ceph/src/gtest/m4/libtool.m4 create mode 100644 ceph/src/gtest/m4/ltoptions.m4 create mode 100644 ceph/src/gtest/m4/ltsugar.m4 create mode 100644 ceph/src/gtest/m4/ltversion.m4 create mode 100644 ceph/src/gtest/m4/lt~obsolete.m4 create mode 100644 ceph/src/gtest/make/Makefile create mode 100644 ceph/src/gtest/msvc/gtest-md.sln create mode 100644 ceph/src/gtest/msvc/gtest-md.vcproj create mode 100644 ceph/src/gtest/msvc/gtest.sln create mode 100644 ceph/src/gtest/msvc/gtest.vcproj create mode 100644 ceph/src/gtest/msvc/gtest_main-md.vcproj create mode 100644 ceph/src/gtest/msvc/gtest_main.vcproj create mode 100644 ceph/src/gtest/msvc/gtest_prod_test-md.vcproj create mode 100644 ceph/src/gtest/msvc/gtest_prod_test.vcproj create mode 100644 ceph/src/gtest/msvc/gtest_unittest-md.vcproj create mode 100644 ceph/src/gtest/msvc/gtest_unittest.vcproj create mode 100644 ceph/src/gtest/samples/prime_tables.h create mode 100644 ceph/src/gtest/samples/sample1.cc create mode 100644 ceph/src/gtest/samples/sample1.h create mode 100644 ceph/src/gtest/samples/sample10_unittest.cc create mode 100644 ceph/src/gtest/samples/sample1_unittest.cc create mode 100644 ceph/src/gtest/samples/sample2.cc create mode 100644 ceph/src/gtest/samples/sample2.h create mode 100644 ceph/src/gtest/samples/sample2_unittest.cc create mode 100644 ceph/src/gtest/samples/sample3-inl.h create mode 100644 ceph/src/gtest/samples/sample3_unittest.cc create mode 100644 ceph/src/gtest/samples/sample4.cc create mode 100644 ceph/src/gtest/samples/sample4.h create mode 100644 ceph/src/gtest/samples/sample4_unittest.cc create mode 100644 ceph/src/gtest/samples/sample5_unittest.cc create mode 100644 ceph/src/gtest/samples/sample6_unittest.cc create mode 100644 ceph/src/gtest/samples/sample7_unittest.cc create mode 100644 ceph/src/gtest/samples/sample8_unittest.cc create mode 100644 ceph/src/gtest/samples/sample9_unittest.cc create mode 100755 ceph/src/gtest/scripts/fuse_gtest_files.py create mode 100755 ceph/src/gtest/scripts/gen_gtest_pred_impl.py create mode 100755 ceph/src/gtest/scripts/gtest-config.in create mode 100755 ceph/src/gtest/scripts/pump.py create mode 100644 ceph/src/gtest/scripts/test/Makefile create mode 100644 ceph/src/gtest/src/gtest-all.cc create mode 100644 ceph/src/gtest/src/gtest-death-test.cc create mode 100644 ceph/src/gtest/src/gtest-filepath.cc create mode 100644 ceph/src/gtest/src/gtest-internal-inl.h create mode 100644 ceph/src/gtest/src/gtest-port.cc create mode 100644 ceph/src/gtest/src/gtest-test-part.cc create mode 100644 ceph/src/gtest/src/gtest-typed-test.cc create mode 100644 ceph/src/gtest/src/gtest.cc create mode 100644 ceph/src/gtest/src/gtest_main.cc create mode 100644 ceph/src/gtest/test/gtest-death-test_test.cc create mode 100644 ceph/src/gtest/test/gtest-filepath_test.cc create mode 100644 ceph/src/gtest/test/gtest-linked_ptr_test.cc create mode 100644 ceph/src/gtest/test/gtest-listener_test.cc create mode 100644 ceph/src/gtest/test/gtest-message_test.cc create mode 100644 ceph/src/gtest/test/gtest-options_test.cc create mode 100644 ceph/src/gtest/test/gtest-param-test2_test.cc create mode 100644 ceph/src/gtest/test/gtest-param-test_test.cc create mode 100644 ceph/src/gtest/test/gtest-param-test_test.h create mode 100644 ceph/src/gtest/test/gtest-port_test.cc create mode 100644 ceph/src/gtest/test/gtest-test-part_test.cc create mode 100644 ceph/src/gtest/test/gtest-tuple_test.cc create mode 100644 ceph/src/gtest/test/gtest-typed-test2_test.cc create mode 100644 ceph/src/gtest/test/gtest-typed-test_test.cc create mode 100644 ceph/src/gtest/test/gtest-typed-test_test.h create mode 100644 ceph/src/gtest/test/gtest-unittest-api_test.cc create mode 100644 ceph/src/gtest/test/gtest_all_test.cc create mode 100755 ceph/src/gtest/test/gtest_break_on_failure_unittest.py create mode 100644 ceph/src/gtest/test/gtest_break_on_failure_unittest_.cc create mode 100755 ceph/src/gtest/test/gtest_color_test.py create mode 100644 ceph/src/gtest/test/gtest_color_test_.cc create mode 100755 ceph/src/gtest/test/gtest_env_var_test.py create mode 100644 ceph/src/gtest/test/gtest_env_var_test_.cc create mode 100644 ceph/src/gtest/test/gtest_environment_test.cc create mode 100755 ceph/src/gtest/test/gtest_filter_unittest.py create mode 100644 ceph/src/gtest/test/gtest_filter_unittest_.cc create mode 100755 ceph/src/gtest/test/gtest_help_test.py create mode 100644 ceph/src/gtest/test/gtest_help_test_.cc create mode 100755 ceph/src/gtest/test/gtest_list_tests_unittest.py create mode 100644 ceph/src/gtest/test/gtest_list_tests_unittest_.cc create mode 100644 ceph/src/gtest/test/gtest_main_unittest.cc create mode 100644 ceph/src/gtest/test/gtest_no_test_unittest.cc create mode 100755 ceph/src/gtest/test/gtest_output_test.py create mode 100644 ceph/src/gtest/test/gtest_output_test_.cc create mode 100644 ceph/src/gtest/test/gtest_output_test_golden_lin.txt create mode 100644 ceph/src/gtest/test/gtest_output_test_golden_win.txt create mode 100644 ceph/src/gtest/test/gtest_pred_impl_unittest.cc create mode 100644 ceph/src/gtest/test/gtest_prod_test.cc create mode 100644 ceph/src/gtest/test/gtest_repeat_test.cc create mode 100755 ceph/src/gtest/test/gtest_shuffle_test.py create mode 100644 ceph/src/gtest/test/gtest_shuffle_test_.cc create mode 100644 ceph/src/gtest/test/gtest_sole_header_test.cc create mode 100644 ceph/src/gtest/test/gtest_stress_test.cc create mode 100755 ceph/src/gtest/test/gtest_test_utils.py create mode 100644 ceph/src/gtest/test/gtest_throw_on_failure_ex_test.cc create mode 100755 ceph/src/gtest/test/gtest_throw_on_failure_test.py create mode 100644 ceph/src/gtest/test/gtest_throw_on_failure_test_.cc create mode 100755 ceph/src/gtest/test/gtest_uninitialized_test.py create mode 100644 ceph/src/gtest/test/gtest_uninitialized_test_.cc create mode 100644 ceph/src/gtest/test/gtest_unittest.cc create mode 100644 ceph/src/gtest/test/gtest_xml_outfile1_test_.cc create mode 100644 ceph/src/gtest/test/gtest_xml_outfile2_test_.cc create mode 100755 ceph/src/gtest/test/gtest_xml_outfiles_test.py create mode 100755 ceph/src/gtest/test/gtest_xml_output_unittest.py create mode 100644 ceph/src/gtest/test/gtest_xml_output_unittest_.cc create mode 100755 ceph/src/gtest/test/gtest_xml_test_utils.py create mode 100644 ceph/src/gtest/test/production.cc create mode 100644 ceph/src/gtest/test/production.h create mode 100755 ceph/src/gtest/test/run_tests_util.py create mode 100755 ceph/src/gtest/test/run_tests_util_test.py create mode 100644 ceph/src/gtest/xcode/Config/DebugProject.xcconfig create mode 100644 ceph/src/gtest/xcode/Config/FrameworkTarget.xcconfig create mode 100644 ceph/src/gtest/xcode/Config/General.xcconfig create mode 100644 ceph/src/gtest/xcode/Config/ReleaseProject.xcconfig create mode 100644 ceph/src/gtest/xcode/Config/StaticLibraryTarget.xcconfig create mode 100644 ceph/src/gtest/xcode/Config/TestTarget.xcconfig create mode 100644 ceph/src/gtest/xcode/Resources/Info.plist create mode 100644 ceph/src/gtest/xcode/Samples/FrameworkSample/Info.plist create mode 100644 ceph/src/gtest/xcode/Samples/FrameworkSample/WidgetFramework.xcodeproj/project.pbxproj create mode 100644 ceph/src/gtest/xcode/Samples/FrameworkSample/runtests.sh create mode 100644 ceph/src/gtest/xcode/Samples/FrameworkSample/widget.cc create mode 100644 ceph/src/gtest/xcode/Samples/FrameworkSample/widget.h create mode 100644 ceph/src/gtest/xcode/Samples/FrameworkSample/widget_test.cc create mode 100644 ceph/src/gtest/xcode/Scripts/runtests.sh create mode 100644 ceph/src/gtest/xcode/Scripts/versiongenerate.py create mode 100644 ceph/src/gtest/xcode/gtest.xcodeproj/project.pbxproj create mode 100644 ceph/src/include/CompatSet.h create mode 100644 ceph/src/include/Context.h create mode 100644 ceph/src/include/Distribution.h create mode 100644 ceph/src/include/Makefile.am create mode 100644 ceph/src/include/Spinlock.h create mode 100644 ceph/src/include/addr_parsing.h create mode 100644 ceph/src/include/assert.h create mode 100644 ceph/src/include/atomic.h create mode 100644 ceph/src/include/bitmapper.h create mode 100644 ceph/src/include/blobhash.h create mode 100644 ceph/src/include/buffer.h create mode 100644 ceph/src/include/byteorder.h create mode 100644 ceph/src/include/ceph_features.h create mode 100644 ceph/src/include/ceph_frag.h create mode 100644 ceph/src/include/ceph_fs.h create mode 100644 ceph/src/include/ceph_hash.h create mode 100644 ceph/src/include/cephfs/libcephfs.h create mode 100644 ceph/src/include/cmp.h create mode 100644 ceph/src/include/color.h create mode 100644 ceph/src/include/compat.h create mode 100644 ceph/src/include/crc32c.h create mode 100644 ceph/src/include/dlist.h create mode 100644 ceph/src/include/elist.h create mode 100644 ceph/src/include/encoding.h create mode 100644 ceph/src/include/err.h create mode 100644 ceph/src/include/error.h create mode 100644 ceph/src/include/filepath.h create mode 100644 ceph/src/include/frag.h create mode 100644 ceph/src/include/hash.h create mode 100644 ceph/src/include/hash_namespace.h create mode 100644 ceph/src/include/int_types.h create mode 100644 ceph/src/include/intarith.h create mode 100644 ceph/src/include/interval_set.h create mode 100644 ceph/src/include/ipaddr.h create mode 100644 ceph/src/include/linux_fiemap.h create mode 100644 ceph/src/include/lru.h create mode 100644 ceph/src/include/memory.h create mode 100644 ceph/src/include/msgr.h create mode 100644 ceph/src/include/object.h create mode 100644 ceph/src/include/on_exit.h create mode 100644 ceph/src/include/page.h create mode 100644 ceph/src/include/rados.h create mode 100644 ceph/src/include/rados/buffer.h create mode 100644 ceph/src/include/rados/crc32c.h create mode 100644 ceph/src/include/rados/librados.h create mode 100644 ceph/src/include/rados/librados.hpp create mode 100644 ceph/src/include/rados/librgw.h create mode 100644 ceph/src/include/rados/memory.h create mode 100644 ceph/src/include/rados/page.h create mode 100644 ceph/src/include/rados/rados_types.h create mode 100644 ceph/src/include/rados/rados_types.hpp create mode 100644 ceph/src/include/rangeset.h create mode 100644 ceph/src/include/rbd/features.h create mode 100644 ceph/src/include/rbd/librbd.h create mode 100644 ceph/src/include/rbd/librbd.hpp create mode 100644 ceph/src/include/rbd_types.h create mode 100644 ceph/src/include/stat.h create mode 100644 ceph/src/include/statlite.h create mode 100644 ceph/src/include/str_list.h create mode 100644 ceph/src/include/str_map.h create mode 100644 ceph/src/include/stringify.h create mode 100644 ceph/src/include/triple.h create mode 100644 ceph/src/include/types.h create mode 100644 ceph/src/include/unordered_map.h create mode 100644 ceph/src/include/unordered_set.h create mode 100644 ceph/src/include/util.h create mode 100644 ceph/src/include/utime.h create mode 100644 ceph/src/include/uuid.h create mode 100644 ceph/src/include/xlist.h create mode 100644 ceph/src/init-ceph.in create mode 100644 ceph/src/init-radosgw create mode 100644 ceph/src/init-radosgw.sysv create mode 100755 ceph/src/init-rbdmap create mode 100644 ceph/src/java/Makefile.am create mode 100644 ceph/src/java/Makefile.in create mode 100644 ceph/src/java/README create mode 100644 ceph/src/java/java/com/ceph/crush/Bucket.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephAlreadyMountedException.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephFileAlreadyExistsException.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephFileExtent.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephMount.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephNativeLoader.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephNotDirectoryException.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephNotMountedException.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephPoolException.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephStat.java create mode 100644 ceph/src/java/java/com/ceph/fs/CephStatVFS.java create mode 100644 ceph/src/java/native/JniConstants.cpp create mode 100644 ceph/src/java/native/JniConstants.h create mode 100644 ceph/src/java/native/ScopedLocalRef.h create mode 100644 ceph/src/java/native/libcephfs_jni.cc create mode 100644 ceph/src/java/test/com/ceph/fs/CephAllTests.java create mode 100644 ceph/src/java/test/com/ceph/fs/CephDoubleMountTest.java create mode 100644 ceph/src/java/test/com/ceph/fs/CephMountCreateTest.java create mode 100644 ceph/src/java/test/com/ceph/fs/CephMountTest.java create mode 100644 ceph/src/java/test/com/ceph/fs/CephUnmountedTest.java create mode 100644 ceph/src/json_spirit/Makefile.am create mode 100644 ceph/src/json_spirit/json_spirit.h create mode 100644 ceph/src/json_spirit/json_spirit_error_position.h create mode 100644 ceph/src/json_spirit/json_spirit_reader.cpp create mode 100644 ceph/src/json_spirit/json_spirit_reader.h create mode 100644 ceph/src/json_spirit/json_spirit_reader_template.h create mode 100644 ceph/src/json_spirit/json_spirit_stream_reader.h create mode 100644 ceph/src/json_spirit/json_spirit_utils.h create mode 100644 ceph/src/json_spirit/json_spirit_value.h create mode 100644 ceph/src/json_spirit/json_spirit_writer.cpp create mode 100644 ceph/src/json_spirit/json_spirit_writer.h create mode 100644 ceph/src/json_spirit/json_spirit_writer_options.h create mode 100644 ceph/src/json_spirit/json_spirit_writer_template.h create mode 100644 ceph/src/key_value_store/Makefile.am create mode 100644 ceph/src/key_value_store/cls_kvs.cc create mode 100644 ceph/src/key_value_store/key_value_structure.h create mode 100644 ceph/src/key_value_store/kv_flat_btree_async.cc create mode 100644 ceph/src/key_value_store/kv_flat_btree_async.h create mode 100644 ceph/src/key_value_store/kvs_arg_types.h create mode 100644 ceph/src/libcephfs.cc create mode 100644 ceph/src/librados-config.cc create mode 100644 ceph/src/librados/AioCompletionImpl.h create mode 100644 ceph/src/librados/IoCtxImpl.cc create mode 100644 ceph/src/librados/IoCtxImpl.h create mode 100644 ceph/src/librados/Makefile.am create mode 100644 ceph/src/librados/PoolAsyncCompletionImpl.h create mode 100644 ceph/src/librados/RadosClient.cc create mode 100644 ceph/src/librados/RadosClient.h create mode 100644 ceph/src/librados/librados.cc create mode 100644 ceph/src/librados/snap_set_diff.cc create mode 100644 ceph/src/librados/snap_set_diff.h create mode 100644 ceph/src/librbd/AioCompletion.cc create mode 100644 ceph/src/librbd/AioCompletion.h create mode 100644 ceph/src/librbd/AioRequest.cc create mode 100644 ceph/src/librbd/AioRequest.h create mode 100644 ceph/src/librbd/ImageCtx.cc create mode 100644 ceph/src/librbd/ImageCtx.h create mode 100644 ceph/src/librbd/LibrbdWriteback.cc create mode 100644 ceph/src/librbd/LibrbdWriteback.h create mode 100644 ceph/src/librbd/Makefile.am create mode 100644 ceph/src/librbd/SnapInfo.h create mode 100644 ceph/src/librbd/WatchCtx.cc create mode 100644 ceph/src/librbd/WatchCtx.h create mode 100644 ceph/src/librbd/internal.cc create mode 100644 ceph/src/librbd/internal.h create mode 100644 ceph/src/librbd/librbd.cc create mode 100644 ceph/src/librbd/parent_types.h create mode 100644 ceph/src/libs3/COPYING create mode 100644 ceph/src/libs3/ChangeLog create mode 100644 ceph/src/libs3/GNUmakefile create mode 100644 ceph/src/libs3/GNUmakefile.mingw create mode 100644 ceph/src/libs3/GNUmakefile.osx create mode 100644 ceph/src/libs3/INSTALL create mode 100644 ceph/src/libs3/LICENSE create mode 100644 ceph/src/libs3/README create mode 100644 ceph/src/libs3/TODO create mode 100644 ceph/src/libs3/archlinux/PKGBUILD create mode 100644 ceph/src/libs3/debian/changelog create mode 100644 ceph/src/libs3/debian/changelog.Debian create mode 100644 ceph/src/libs3/debian/control create mode 100644 ceph/src/libs3/debian/control.dev create mode 100755 ceph/src/libs3/debian/postinst create mode 100644 ceph/src/libs3/doxyfile create mode 100644 ceph/src/libs3/inc/error_parser.h create mode 100644 ceph/src/libs3/inc/libs3.h create mode 100644 ceph/src/libs3/inc/mingw/pthread.h create mode 100644 ceph/src/libs3/inc/mingw/sys/select.h create mode 100644 ceph/src/libs3/inc/mingw/sys/utsname.h create mode 100644 ceph/src/libs3/inc/request.h create mode 100644 ceph/src/libs3/inc/request_context.h create mode 100644 ceph/src/libs3/inc/response_headers_handler.h create mode 100644 ceph/src/libs3/inc/simplexml.h create mode 100644 ceph/src/libs3/inc/string_buffer.h create mode 100644 ceph/src/libs3/inc/util.h create mode 100644 ceph/src/libs3/libs3.spec create mode 100644 ceph/src/libs3/mswin/libs3.def create mode 100644 ceph/src/libs3/mswin/rmrf.bat create mode 100644 ceph/src/libs3/src/acl.c create mode 100644 ceph/src/libs3/src/bucket.c create mode 100644 ceph/src/libs3/src/error_parser.c create mode 100644 ceph/src/libs3/src/general.c create mode 100644 ceph/src/libs3/src/mingw_functions.c create mode 100644 ceph/src/libs3/src/mingw_s3_functions.c create mode 100644 ceph/src/libs3/src/object.c create mode 100644 ceph/src/libs3/src/request.c create mode 100644 ceph/src/libs3/src/request_context.c create mode 100644 ceph/src/libs3/src/response_headers_handler.c create mode 100644 ceph/src/libs3/src/s3.c create mode 100644 ceph/src/libs3/src/service.c create mode 100644 ceph/src/libs3/src/service_access_logging.c create mode 100644 ceph/src/libs3/src/simplexml.c create mode 100644 ceph/src/libs3/src/testsimplexml.c create mode 100644 ceph/src/libs3/src/util.c create mode 100644 ceph/src/libs3/test/badxml_01.xml create mode 100644 ceph/src/libs3/test/goodxml_01.xml create mode 100644 ceph/src/libs3/test/goodxml_02.xml create mode 100644 ceph/src/libs3/test/goodxml_03.xml create mode 100755 ceph/src/libs3/test/test.sh create mode 100644 ceph/src/log/Entry.h create mode 100644 ceph/src/log/EntryQueue.h create mode 100644 ceph/src/log/Log.cc create mode 100644 ceph/src/log/Log.h create mode 100644 ceph/src/log/Makefile.am create mode 100644 ceph/src/log/SubsystemMap.cc create mode 100644 ceph/src/log/SubsystemMap.h create mode 100644 ceph/src/log/test.cc create mode 100644 ceph/src/logrotate.conf create mode 100755 ceph/src/make_version create mode 100644 ceph/src/mds/Anchor.cc create mode 100644 ceph/src/mds/Anchor.h create mode 100644 ceph/src/mds/AnchorClient.cc create mode 100644 ceph/src/mds/AnchorClient.h create mode 100644 ceph/src/mds/AnchorServer.cc create mode 100644 ceph/src/mds/AnchorServer.h create mode 100644 ceph/src/mds/CDentry.cc create mode 100644 ceph/src/mds/CDentry.h create mode 100644 ceph/src/mds/CDir.cc create mode 100644 ceph/src/mds/CDir.h create mode 100644 ceph/src/mds/CInode.cc create mode 100644 ceph/src/mds/CInode.h create mode 100644 ceph/src/mds/Capability.cc create mode 100644 ceph/src/mds/Capability.h create mode 100644 ceph/src/mds/Dumper.cc create mode 100644 ceph/src/mds/Dumper.h create mode 100644 ceph/src/mds/InoTable.cc create mode 100644 ceph/src/mds/InoTable.h create mode 100644 ceph/src/mds/LocalLock.h create mode 100644 ceph/src/mds/Locker.cc create mode 100644 ceph/src/mds/Locker.h create mode 100644 ceph/src/mds/LogEvent.cc create mode 100644 ceph/src/mds/LogEvent.h create mode 100644 ceph/src/mds/LogSegment.h create mode 100644 ceph/src/mds/MDBalancer.cc create mode 100644 ceph/src/mds/MDBalancer.h create mode 100644 ceph/src/mds/MDCache.cc create mode 100644 ceph/src/mds/MDCache.h create mode 100644 ceph/src/mds/MDLog.cc create mode 100644 ceph/src/mds/MDLog.h create mode 100644 ceph/src/mds/MDS.cc create mode 100644 ceph/src/mds/MDS.h create mode 100644 ceph/src/mds/MDSMap.cc create mode 100644 ceph/src/mds/MDSMap.h create mode 100644 ceph/src/mds/MDSTable.cc create mode 100644 ceph/src/mds/MDSTable.h create mode 100644 ceph/src/mds/MDSTableClient.cc create mode 100644 ceph/src/mds/MDSTableClient.h create mode 100644 ceph/src/mds/MDSTableServer.cc create mode 100644 ceph/src/mds/MDSTableServer.h create mode 100644 ceph/src/mds/MDSUtility.cc create mode 100644 ceph/src/mds/MDSUtility.h create mode 100644 ceph/src/mds/Makefile.am create mode 100644 ceph/src/mds/Migrator.cc create mode 100644 ceph/src/mds/Migrator.h create mode 100644 ceph/src/mds/Mutation.cc create mode 100644 ceph/src/mds/Mutation.h create mode 100644 ceph/src/mds/Resetter.cc create mode 100644 ceph/src/mds/Resetter.h create mode 100644 ceph/src/mds/ScatterLock.h create mode 100644 ceph/src/mds/Server.cc create mode 100644 ceph/src/mds/Server.h create mode 100644 ceph/src/mds/SessionMap.cc create mode 100644 ceph/src/mds/SessionMap.h create mode 100644 ceph/src/mds/SimpleLock.h create mode 100644 ceph/src/mds/SnapClient.h create mode 100644 ceph/src/mds/SnapRealm.cc create mode 100644 ceph/src/mds/SnapRealm.h create mode 100644 ceph/src/mds/SnapServer.cc create mode 100644 ceph/src/mds/SnapServer.h create mode 100644 ceph/src/mds/events/ECommitted.h create mode 100644 ceph/src/mds/events/EExport.h create mode 100644 ceph/src/mds/events/EFragment.h create mode 100644 ceph/src/mds/events/EImportFinish.h create mode 100644 ceph/src/mds/events/EImportStart.h create mode 100644 ceph/src/mds/events/EMetaBlob.h create mode 100644 ceph/src/mds/events/EOpen.h create mode 100644 ceph/src/mds/events/EResetJournal.h create mode 100644 ceph/src/mds/events/ESession.h create mode 100644 ceph/src/mds/events/ESessions.h create mode 100644 ceph/src/mds/events/ESlaveUpdate.h create mode 100644 ceph/src/mds/events/ESubtreeMap.h create mode 100644 ceph/src/mds/events/ETableClient.h create mode 100644 ceph/src/mds/events/ETableServer.h create mode 100644 ceph/src/mds/events/EUpdate.h create mode 100644 ceph/src/mds/flock.cc create mode 100644 ceph/src/mds/flock.h create mode 100644 ceph/src/mds/inode_backtrace.cc create mode 100644 ceph/src/mds/inode_backtrace.h create mode 100644 ceph/src/mds/journal.cc create mode 100644 ceph/src/mds/locks.c create mode 100644 ceph/src/mds/locks.h create mode 100644 ceph/src/mds/mds_table_types.h create mode 100644 ceph/src/mds/mdstypes.cc create mode 100644 ceph/src/mds/mdstypes.h create mode 100644 ceph/src/mds/snap.cc create mode 100644 ceph/src/mds/snap.h create mode 100644 ceph/src/messages/MAuth.h create mode 100644 ceph/src/messages/MAuthReply.h create mode 100644 ceph/src/messages/MBackfillReserve.h create mode 100644 ceph/src/messages/MCacheExpire.h create mode 100644 ceph/src/messages/MClientCapRelease.h create mode 100644 ceph/src/messages/MClientCaps.h create mode 100644 ceph/src/messages/MClientLease.h create mode 100644 ceph/src/messages/MClientReconnect.h create mode 100644 ceph/src/messages/MClientReply.h create mode 100644 ceph/src/messages/MClientRequest.h create mode 100644 ceph/src/messages/MClientRequestForward.h create mode 100644 ceph/src/messages/MClientSession.h create mode 100644 ceph/src/messages/MClientSnap.h create mode 100644 ceph/src/messages/MCommand.h create mode 100644 ceph/src/messages/MCommandReply.h create mode 100644 ceph/src/messages/MDentryLink.h create mode 100644 ceph/src/messages/MDentryUnlink.h create mode 100644 ceph/src/messages/MDirUpdate.h create mode 100644 ceph/src/messages/MDiscover.h create mode 100644 ceph/src/messages/MDiscoverReply.h create mode 100644 ceph/src/messages/MExportCaps.h create mode 100644 ceph/src/messages/MExportCapsAck.h create mode 100644 ceph/src/messages/MExportDir.h create mode 100644 ceph/src/messages/MExportDirAck.h create mode 100644 ceph/src/messages/MExportDirCancel.h create mode 100644 ceph/src/messages/MExportDirDiscover.h create mode 100644 ceph/src/messages/MExportDirDiscoverAck.h create mode 100644 ceph/src/messages/MExportDirFinish.h create mode 100644 ceph/src/messages/MExportDirNotify.h create mode 100644 ceph/src/messages/MExportDirNotifyAck.h create mode 100644 ceph/src/messages/MExportDirPrep.h create mode 100644 ceph/src/messages/MExportDirPrepAck.h create mode 100644 ceph/src/messages/MForward.h create mode 100644 ceph/src/messages/MGenericMessage.h create mode 100644 ceph/src/messages/MGetPoolStats.h create mode 100644 ceph/src/messages/MGetPoolStatsReply.h create mode 100644 ceph/src/messages/MHeartbeat.h create mode 100644 ceph/src/messages/MInodeFileCaps.h create mode 100644 ceph/src/messages/MLock.h create mode 100644 ceph/src/messages/MLog.h create mode 100644 ceph/src/messages/MLogAck.h create mode 100644 ceph/src/messages/MMDSBeacon.h create mode 100644 ceph/src/messages/MMDSCacheRejoin.h create mode 100644 ceph/src/messages/MMDSFindIno.h create mode 100644 ceph/src/messages/MMDSFindInoReply.h create mode 100644 ceph/src/messages/MMDSFragmentNotify.h create mode 100644 ceph/src/messages/MMDSLoadTargets.h create mode 100644 ceph/src/messages/MMDSMap.h create mode 100644 ceph/src/messages/MMDSOpenIno.h create mode 100644 ceph/src/messages/MMDSOpenInoReply.h create mode 100644 ceph/src/messages/MMDSResolve.h create mode 100644 ceph/src/messages/MMDSResolveAck.h create mode 100644 ceph/src/messages/MMDSSlaveRequest.h create mode 100644 ceph/src/messages/MMDSTableRequest.h create mode 100644 ceph/src/messages/MMonCommand.h create mode 100644 ceph/src/messages/MMonCommandAck.h create mode 100644 ceph/src/messages/MMonElection.h create mode 100644 ceph/src/messages/MMonGetMap.h create mode 100644 ceph/src/messages/MMonGetVersion.h create mode 100644 ceph/src/messages/MMonGetVersionReply.h create mode 100644 ceph/src/messages/MMonGlobalID.h create mode 100644 ceph/src/messages/MMonHealth.h create mode 100644 ceph/src/messages/MMonJoin.h create mode 100644 ceph/src/messages/MMonMap.h create mode 100644 ceph/src/messages/MMonPaxos.h create mode 100644 ceph/src/messages/MMonProbe.h create mode 100644 ceph/src/messages/MMonQuorumService.h create mode 100644 ceph/src/messages/MMonScrub.h create mode 100644 ceph/src/messages/MMonSubscribe.h create mode 100644 ceph/src/messages/MMonSubscribeAck.h create mode 100644 ceph/src/messages/MMonSync.h create mode 100644 ceph/src/messages/MOSDAlive.h create mode 100644 ceph/src/messages/MOSDBoot.h create mode 100644 ceph/src/messages/MOSDECSubOpRead.h create mode 100644 ceph/src/messages/MOSDECSubOpReadReply.h create mode 100644 ceph/src/messages/MOSDECSubOpWrite.h create mode 100644 ceph/src/messages/MOSDECSubOpWriteReply.h create mode 100644 ceph/src/messages/MOSDFailure.h create mode 100644 ceph/src/messages/MOSDMap.h create mode 100644 ceph/src/messages/MOSDMarkMeDown.h create mode 100644 ceph/src/messages/MOSDOp.h create mode 100644 ceph/src/messages/MOSDOpReply.h create mode 100644 ceph/src/messages/MOSDPGBackfill.h create mode 100644 ceph/src/messages/MOSDPGCreate.h create mode 100644 ceph/src/messages/MOSDPGInfo.h create mode 100644 ceph/src/messages/MOSDPGLog.h create mode 100644 ceph/src/messages/MOSDPGMissing.h create mode 100644 ceph/src/messages/MOSDPGNotify.h create mode 100644 ceph/src/messages/MOSDPGPull.h create mode 100644 ceph/src/messages/MOSDPGPush.h create mode 100644 ceph/src/messages/MOSDPGPushReply.h create mode 100644 ceph/src/messages/MOSDPGQuery.h create mode 100644 ceph/src/messages/MOSDPGRemove.h create mode 100644 ceph/src/messages/MOSDPGScan.h create mode 100644 ceph/src/messages/MOSDPGTemp.h create mode 100644 ceph/src/messages/MOSDPGTrim.h create mode 100644 ceph/src/messages/MOSDPing.h create mode 100644 ceph/src/messages/MOSDRepScrub.h create mode 100644 ceph/src/messages/MOSDScrub.h create mode 100644 ceph/src/messages/MOSDSubOp.h create mode 100644 ceph/src/messages/MOSDSubOpReply.h create mode 100644 ceph/src/messages/MPGStats.h create mode 100644 ceph/src/messages/MPGStatsAck.h create mode 100644 ceph/src/messages/MPing.h create mode 100644 ceph/src/messages/MPoolOp.h create mode 100644 ceph/src/messages/MPoolOpReply.h create mode 100644 ceph/src/messages/MRecoveryReserve.h create mode 100644 ceph/src/messages/MRemoveSnaps.h create mode 100644 ceph/src/messages/MRoute.h create mode 100644 ceph/src/messages/MStatfs.h create mode 100644 ceph/src/messages/MStatfsReply.h create mode 100644 ceph/src/messages/MTimeCheck.h create mode 100644 ceph/src/messages/MWatchNotify.h create mode 100644 ceph/src/messages/Makefile.am create mode 100644 ceph/src/messages/PaxosServiceMessage.h create mode 100644 ceph/src/mkcephfs.in create mode 100644 ceph/src/mon/AuthMonitor.cc create mode 100644 ceph/src/mon/AuthMonitor.h create mode 100644 ceph/src/mon/ConfigKeyService.cc create mode 100644 ceph/src/mon/ConfigKeyService.h create mode 100644 ceph/src/mon/DataHealthService.cc create mode 100644 ceph/src/mon/DataHealthService.h create mode 100644 ceph/src/mon/DumplingMonCommands.h create mode 100644 ceph/src/mon/Elector.cc create mode 100644 ceph/src/mon/Elector.h create mode 100644 ceph/src/mon/HealthMonitor.cc create mode 100644 ceph/src/mon/HealthMonitor.h create mode 100644 ceph/src/mon/HealthService.h create mode 100644 ceph/src/mon/LogMonitor.cc create mode 100644 ceph/src/mon/LogMonitor.h create mode 100644 ceph/src/mon/MDSMonitor.cc create mode 100644 ceph/src/mon/MDSMonitor.h create mode 100644 ceph/src/mon/Makefile.am create mode 100644 ceph/src/mon/MonCap.cc create mode 100644 ceph/src/mon/MonCap.h create mode 100644 ceph/src/mon/MonClient.cc create mode 100644 ceph/src/mon/MonClient.h create mode 100644 ceph/src/mon/MonCommands.h create mode 100644 ceph/src/mon/MonMap.cc create mode 100644 ceph/src/mon/MonMap.h create mode 100644 ceph/src/mon/Monitor.cc create mode 100644 ceph/src/mon/Monitor.h create mode 100644 ceph/src/mon/MonitorDBStore.h create mode 100644 ceph/src/mon/MonitorStore.cc create mode 100644 ceph/src/mon/MonitorStore.h create mode 100644 ceph/src/mon/MonmapMonitor.cc create mode 100644 ceph/src/mon/MonmapMonitor.h create mode 100644 ceph/src/mon/OSDMonitor.cc create mode 100644 ceph/src/mon/OSDMonitor.h create mode 100644 ceph/src/mon/PGMap.cc create mode 100644 ceph/src/mon/PGMap.h create mode 100644 ceph/src/mon/PGMonitor.cc create mode 100644 ceph/src/mon/PGMonitor.h create mode 100644 ceph/src/mon/Paxos.cc create mode 100644 ceph/src/mon/Paxos.h create mode 100644 ceph/src/mon/PaxosService.cc create mode 100644 ceph/src/mon/PaxosService.h create mode 100644 ceph/src/mon/QuorumService.h create mode 100644 ceph/src/mon/Session.h create mode 100644 ceph/src/mon/mon_types.h create mode 100755 ceph/src/mount.fuse.ceph create mode 100644 ceph/src/mount/canonicalize.c create mode 100644 ceph/src/mount/mount.ceph.c create mode 100644 ceph/src/mount/mtab.c create mode 100644 ceph/src/msg/Accepter.cc create mode 100644 ceph/src/msg/Accepter.h create mode 100644 ceph/src/msg/DispatchQueue.cc create mode 100644 ceph/src/msg/DispatchQueue.h create mode 100644 ceph/src/msg/Dispatcher.h create mode 100644 ceph/src/msg/Makefile.am create mode 100644 ceph/src/msg/Message.cc create mode 100644 ceph/src/msg/Message.h create mode 100644 ceph/src/msg/Messenger.cc create mode 100644 ceph/src/msg/Messenger.h create mode 100644 ceph/src/msg/Pipe.cc create mode 100644 ceph/src/msg/Pipe.h create mode 100644 ceph/src/msg/SimpleMessenger.cc create mode 100644 ceph/src/msg/SimpleMessenger.h create mode 100644 ceph/src/msg/msg_types.cc create mode 100644 ceph/src/msg/msg_types.h create mode 100644 ceph/src/objclass/class_api.cc create mode 100644 ceph/src/objclass/objclass.h create mode 100644 ceph/src/ocf/Makefile.am create mode 100644 ceph/src/ocf/Makefile.in create mode 100644 ceph/src/ocf/ceph.in create mode 100644 ceph/src/ocf/rbd.in create mode 100644 ceph/src/os/BtrfsFileStoreBackend.cc create mode 100644 ceph/src/os/BtrfsFileStoreBackend.h create mode 100644 ceph/src/os/CollectionIndex.h create mode 100644 ceph/src/os/DBObjectMap.cc create mode 100644 ceph/src/os/DBObjectMap.h create mode 100644 ceph/src/os/FDCache.h create mode 100644 ceph/src/os/FileJournal.cc create mode 100644 ceph/src/os/FileJournal.h create mode 100644 ceph/src/os/FileStore.cc create mode 100644 ceph/src/os/FileStore.h create mode 100644 ceph/src/os/FlatIndex.cc create mode 100644 ceph/src/os/FlatIndex.h create mode 100644 ceph/src/os/GenericFileStoreBackend.cc create mode 100644 ceph/src/os/GenericFileStoreBackend.h create mode 100644 ceph/src/os/GenericObjectMap.cc create mode 100644 ceph/src/os/GenericObjectMap.h create mode 100644 ceph/src/os/HashIndex.cc create mode 100644 ceph/src/os/HashIndex.h create mode 100644 ceph/src/os/IndexManager.cc create mode 100644 ceph/src/os/IndexManager.h create mode 100644 ceph/src/os/Journal.h create mode 100644 ceph/src/os/JournalingObjectStore.cc create mode 100644 ceph/src/os/JournalingObjectStore.h create mode 100644 ceph/src/os/KeyValueDB.h create mode 100644 ceph/src/os/KeyValueStore.cc create mode 100644 ceph/src/os/KeyValueStore.h create mode 100644 ceph/src/os/LFNIndex.cc create mode 100644 ceph/src/os/LFNIndex.h create mode 100644 ceph/src/os/LevelDBStore.cc create mode 100644 ceph/src/os/LevelDBStore.h create mode 100644 ceph/src/os/Makefile.am create mode 100644 ceph/src/os/MemStore.cc create mode 100644 ceph/src/os/MemStore.h create mode 100644 ceph/src/os/ObjectMap.h create mode 100644 ceph/src/os/ObjectStore.cc create mode 100644 ceph/src/os/ObjectStore.h create mode 100644 ceph/src/os/SequencerPosition.h create mode 100644 ceph/src/os/Transaction.cc create mode 100644 ceph/src/os/WBThrottle.cc create mode 100644 ceph/src/os/WBThrottle.h create mode 100644 ceph/src/os/XfsFileStoreBackend.cc create mode 100644 ceph/src/os/XfsFileStoreBackend.h create mode 100644 ceph/src/os/ZFS.cc create mode 100644 ceph/src/os/ZFS.h create mode 100644 ceph/src/os/ZFSFileStoreBackend.cc create mode 100644 ceph/src/os/ZFSFileStoreBackend.h create mode 100644 ceph/src/os/btrfs_ioctl.h create mode 100644 ceph/src/os/chain_xattr.cc create mode 100644 ceph/src/os/chain_xattr.h create mode 100644 ceph/src/osd/Ager.cc create mode 100644 ceph/src/osd/Ager.h create mode 100644 ceph/src/osd/ClassHandler.cc create mode 100644 ceph/src/osd/ClassHandler.h create mode 100644 ceph/src/osd/ECBackend.cc create mode 100644 ceph/src/osd/ECBackend.h create mode 100644 ceph/src/osd/ECMsgTypes.cc create mode 100644 ceph/src/osd/ECMsgTypes.h create mode 100644 ceph/src/osd/ECTransaction.cc create mode 100644 ceph/src/osd/ECTransaction.h create mode 100644 ceph/src/osd/ECUtil.cc create mode 100644 ceph/src/osd/ECUtil.h create mode 100644 ceph/src/osd/HitSet.cc create mode 100644 ceph/src/osd/HitSet.h create mode 100644 ceph/src/osd/Makefile.am create mode 100644 ceph/src/osd/OSD.cc create mode 100644 ceph/src/osd/OSD.h create mode 100644 ceph/src/osd/OSDCap.cc create mode 100644 ceph/src/osd/OSDCap.h create mode 100644 ceph/src/osd/OSDMap.cc create mode 100644 ceph/src/osd/OSDMap.h create mode 100644 ceph/src/osd/ObjectVersioner.h create mode 100644 ceph/src/osd/OpRequest.cc create mode 100644 ceph/src/osd/OpRequest.h create mode 100644 ceph/src/osd/PG.cc create mode 100644 ceph/src/osd/PG.h create mode 100644 ceph/src/osd/PGBackend.cc create mode 100644 ceph/src/osd/PGBackend.h create mode 100644 ceph/src/osd/PGLog.cc create mode 100644 ceph/src/osd/PGLog.h create mode 100644 ceph/src/osd/ReplicatedBackend.cc create mode 100644 ceph/src/osd/ReplicatedBackend.h create mode 100644 ceph/src/osd/ReplicatedPG.cc create mode 100644 ceph/src/osd/ReplicatedPG.h create mode 100644 ceph/src/osd/SnapMapper.cc create mode 100644 ceph/src/osd/SnapMapper.h create mode 100644 ceph/src/osd/TierAgentState.h create mode 100644 ceph/src/osd/Watch.cc create mode 100644 ceph/src/osd/Watch.h create mode 100644 ceph/src/osd/osd_types.cc create mode 100644 ceph/src/osd/osd_types.h create mode 100644 ceph/src/osdc/Blinker.h create mode 100644 ceph/src/osdc/Filer.cc create mode 100644 ceph/src/osdc/Filer.h create mode 100644 ceph/src/osdc/Journaler.cc create mode 100644 ceph/src/osdc/Journaler.h create mode 100644 ceph/src/osdc/Makefile.am create mode 100644 ceph/src/osdc/ObjectCacher.cc create mode 100644 ceph/src/osdc/ObjectCacher.h create mode 100644 ceph/src/osdc/Objecter.cc create mode 100644 ceph/src/osdc/Objecter.h create mode 100644 ceph/src/osdc/Striper.cc create mode 100644 ceph/src/osdc/Striper.h create mode 100644 ceph/src/osdc/WritebackHandler.h create mode 100644 ceph/src/perfglue/Makefile.am create mode 100644 ceph/src/perfglue/cpu_profiler.cc create mode 100644 ceph/src/perfglue/cpu_profiler.h create mode 100644 ceph/src/perfglue/disabled_heap_profiler.cc create mode 100644 ceph/src/perfglue/disabled_stubs.cc create mode 100644 ceph/src/perfglue/heap_profiler.cc create mode 100644 ceph/src/perfglue/heap_profiler.h create mode 100644 ceph/src/pybind/ceph_argparse.py create mode 100755 ceph/src/pybind/ceph_rest_api.py create mode 100644 ceph/src/pybind/cephfs.py create mode 100644 ceph/src/pybind/rados.py create mode 100644 ceph/src/pybind/rbd.py create mode 100644 ceph/src/rbd.cc create mode 100644 ceph/src/rbd_fuse/rbd-fuse.c create mode 100644 ceph/src/rbdmap create mode 100644 ceph/src/rgw/Makefile.am create mode 100644 ceph/src/rgw/librgw.cc create mode 100644 ceph/src/rgw/logrotate.conf create mode 100644 ceph/src/rgw/rgw_acl.cc create mode 100644 ceph/src/rgw/rgw_acl.h create mode 100644 ceph/src/rgw/rgw_acl_s3.cc create mode 100644 ceph/src/rgw/rgw_acl_s3.h create mode 100644 ceph/src/rgw/rgw_acl_swift.cc create mode 100644 ceph/src/rgw/rgw_acl_swift.h create mode 100644 ceph/src/rgw/rgw_admin.cc create mode 100644 ceph/src/rgw/rgw_auth_s3.cc create mode 100644 ceph/src/rgw/rgw_auth_s3.h create mode 100644 ceph/src/rgw/rgw_bucket.cc create mode 100644 ceph/src/rgw/rgw_bucket.h create mode 100644 ceph/src/rgw/rgw_cache.cc create mode 100644 ceph/src/rgw/rgw_cache.h create mode 100644 ceph/src/rgw/rgw_civetweb.cc create mode 100644 ceph/src/rgw/rgw_civetweb.h create mode 100644 ceph/src/rgw/rgw_client_io.cc create mode 100644 ceph/src/rgw/rgw_client_io.h create mode 100644 ceph/src/rgw/rgw_common.cc create mode 100644 ceph/src/rgw/rgw_common.h create mode 100644 ceph/src/rgw/rgw_cors.cc create mode 100644 ceph/src/rgw/rgw_cors.h create mode 100644 ceph/src/rgw/rgw_cors_s3.cc create mode 100644 ceph/src/rgw/rgw_cors_s3.h create mode 100644 ceph/src/rgw/rgw_cors_swift.h create mode 100644 ceph/src/rgw/rgw_dencoder.cc create mode 100644 ceph/src/rgw/rgw_env.cc create mode 100644 ceph/src/rgw/rgw_fcgi.cc create mode 100644 ceph/src/rgw/rgw_fcgi.h create mode 100644 ceph/src/rgw/rgw_formats.cc create mode 100644 ceph/src/rgw/rgw_formats.h create mode 100644 ceph/src/rgw/rgw_gc.cc create mode 100644 ceph/src/rgw/rgw_gc.h create mode 100644 ceph/src/rgw/rgw_http_client.cc create mode 100644 ceph/src/rgw/rgw_http_client.h create mode 100644 ceph/src/rgw/rgw_http_errors.h create mode 100644 ceph/src/rgw/rgw_json_enc.cc create mode 100644 ceph/src/rgw/rgw_jsonparser.cc create mode 100644 ceph/src/rgw/rgw_keystone.cc create mode 100644 ceph/src/rgw/rgw_keystone.h create mode 100644 ceph/src/rgw/rgw_loadgen.cc create mode 100644 ceph/src/rgw/rgw_loadgen.h create mode 100644 ceph/src/rgw/rgw_log.cc create mode 100644 ceph/src/rgw/rgw_log.h create mode 100644 ceph/src/rgw/rgw_main.cc create mode 100644 ceph/src/rgw/rgw_metadata.cc create mode 100644 ceph/src/rgw/rgw_metadata.h create mode 100644 ceph/src/rgw/rgw_multi.cc create mode 100644 ceph/src/rgw/rgw_multi.h create mode 100644 ceph/src/rgw/rgw_multi_del.cc create mode 100644 ceph/src/rgw/rgw_multi_del.h create mode 100644 ceph/src/rgw/rgw_multiparser.cc create mode 100644 ceph/src/rgw/rgw_op.cc create mode 100644 ceph/src/rgw/rgw_op.h create mode 100644 ceph/src/rgw/rgw_policy_s3.cc create mode 100644 ceph/src/rgw/rgw_policy_s3.h create mode 100644 ceph/src/rgw/rgw_quota.cc create mode 100644 ceph/src/rgw/rgw_quota.h create mode 100644 ceph/src/rgw/rgw_rados.cc create mode 100644 ceph/src/rgw/rgw_rados.h create mode 100644 ceph/src/rgw/rgw_replica_log.cc create mode 100644 ceph/src/rgw/rgw_replica_log.h create mode 100644 ceph/src/rgw/rgw_resolve.cc create mode 100644 ceph/src/rgw/rgw_resolve.h create mode 100644 ceph/src/rgw/rgw_rest.cc create mode 100644 ceph/src/rgw/rgw_rest.h create mode 100644 ceph/src/rgw/rgw_rest_admin.h create mode 100644 ceph/src/rgw/rgw_rest_bucket.cc create mode 100644 ceph/src/rgw/rgw_rest_bucket.h create mode 100644 ceph/src/rgw/rgw_rest_client.cc create mode 100644 ceph/src/rgw/rgw_rest_client.h create mode 100644 ceph/src/rgw/rgw_rest_config.cc create mode 100644 ceph/src/rgw/rgw_rest_config.h create mode 100644 ceph/src/rgw/rgw_rest_conn.cc create mode 100644 ceph/src/rgw/rgw_rest_conn.h create mode 100644 ceph/src/rgw/rgw_rest_log.cc create mode 100644 ceph/src/rgw/rgw_rest_log.h create mode 100644 ceph/src/rgw/rgw_rest_metadata.cc create mode 100644 ceph/src/rgw/rgw_rest_metadata.h create mode 100644 ceph/src/rgw/rgw_rest_opstate.cc create mode 100644 ceph/src/rgw/rgw_rest_opstate.h create mode 100644 ceph/src/rgw/rgw_rest_replica_log.cc create mode 100644 ceph/src/rgw/rgw_rest_replica_log.h create mode 100644 ceph/src/rgw/rgw_rest_s3.cc create mode 100644 ceph/src/rgw/rgw_rest_s3.h create mode 100644 ceph/src/rgw/rgw_rest_swift.cc create mode 100644 ceph/src/rgw/rgw_rest_swift.h create mode 100644 ceph/src/rgw/rgw_rest_usage.cc create mode 100644 ceph/src/rgw/rgw_rest_usage.h create mode 100644 ceph/src/rgw/rgw_rest_user.cc create mode 100644 ceph/src/rgw/rgw_rest_user.h create mode 100644 ceph/src/rgw/rgw_string.h create mode 100644 ceph/src/rgw/rgw_swift.cc create mode 100644 ceph/src/rgw/rgw_swift.h create mode 100644 ceph/src/rgw/rgw_swift_auth.cc create mode 100644 ceph/src/rgw/rgw_swift_auth.h create mode 100644 ceph/src/rgw/rgw_tools.cc create mode 100644 ceph/src/rgw/rgw_tools.h create mode 100644 ceph/src/rgw/rgw_usage.cc create mode 100644 ceph/src/rgw/rgw_usage.h create mode 100644 ceph/src/rgw/rgw_user.cc create mode 100644 ceph/src/rgw/rgw_user.h create mode 100644 ceph/src/rgw/rgw_xml.cc create mode 100644 ceph/src/rgw/rgw_xml.h create mode 100644 ceph/src/sample.ceph.conf create mode 100755 ceph/src/stop.sh create mode 100644 ceph/src/test/Makefile.am create mode 100644 ceph/src/test/ObjectMap/KeyValueDBMemory.cc create mode 100644 ceph/src/test/ObjectMap/KeyValueDBMemory.h create mode 100644 ceph/src/test/ObjectMap/test_keyvaluedb_atomicity.cc create mode 100644 ceph/src/test/ObjectMap/test_keyvaluedb_iterators.cc create mode 100644 ceph/src/test/ObjectMap/test_object_map.cc create mode 100644 ceph/src/test/TestSignalHandlers.cc create mode 100644 ceph/src/test/TestTimers.cc create mode 100644 ceph/src/test/admin_socket.cc create mode 100644 ceph/src/test/base64.cc create mode 100644 ceph/src/test/bench/backend.h create mode 100644 ceph/src/test/bench/bencher.cc create mode 100644 ceph/src/test/bench/bencher.h create mode 100644 ceph/src/test/bench/detailed_stat_collector.cc create mode 100644 ceph/src/test/bench/detailed_stat_collector.h create mode 100644 ceph/src/test/bench/distribution.h create mode 100644 ceph/src/test/bench/dumb_backend.cc create mode 100644 ceph/src/test/bench/dumb_backend.h create mode 100644 ceph/src/test/bench/rados_backend.cc create mode 100644 ceph/src/test/bench/rados_backend.h create mode 100644 ceph/src/test/bench/rbd_backend.cc create mode 100644 ceph/src/test/bench/rbd_backend.h create mode 100644 ceph/src/test/bench/small_io_bench.cc create mode 100644 ceph/src/test/bench/small_io_bench_dumb.cc create mode 100644 ceph/src/test/bench/small_io_bench_fs.cc create mode 100644 ceph/src/test/bench/small_io_bench_rbd.cc create mode 100644 ceph/src/test/bench/stat_collector.h create mode 100644 ceph/src/test/bench/testfilestore_backend.cc create mode 100644 ceph/src/test/bench/testfilestore_backend.h create mode 100644 ceph/src/test/bench/tp_bench.cc create mode 100644 ceph/src/test/bench_log.cc create mode 100644 ceph/src/test/bufferlist.cc create mode 100644 ceph/src/test/buildtest_skeleton.cc create mode 100644 ceph/src/test/ceph_argparse.cc create mode 100644 ceph/src/test/ceph_compatset.cc create mode 100644 ceph/src/test/ceph_crypto.cc create mode 100644 ceph/src/test/cli/.gitignore create mode 100644 ceph/src/test/cli/ceph-authtool/add-key-segv.t create mode 100644 ceph/src/test/cli/ceph-authtool/add-key.t create mode 100644 ceph/src/test/cli/ceph-authtool/cap-bin.t create mode 100644 ceph/src/test/cli/ceph-authtool/cap-invalid.t create mode 100644 ceph/src/test/cli/ceph-authtool/cap-overwrite.t create mode 100644 ceph/src/test/cli/ceph-authtool/cap.t create mode 100644 ceph/src/test/cli/ceph-authtool/create-gen-list-bin.t create mode 100644 ceph/src/test/cli/ceph-authtool/create-gen-list.t create mode 100644 ceph/src/test/cli/ceph-authtool/help.t create mode 100644 ceph/src/test/cli/ceph-authtool/list-empty-bin.t create mode 100644 ceph/src/test/cli/ceph-authtool/list-empty.t create mode 100644 ceph/src/test/cli/ceph-authtool/list-nonexistent-bin.t create mode 100644 ceph/src/test/cli/ceph-authtool/list-nonexistent.t create mode 100644 ceph/src/test/cli/ceph-authtool/manpage.t create mode 100644 ceph/src/test/cli/ceph-authtool/simple.t create mode 100644 ceph/src/test/cli/ceph-conf/env-vs-args.t create mode 100644 ceph/src/test/cli/ceph-conf/help.t create mode 100644 ceph/src/test/cli/ceph-conf/invalid-args.t create mode 100644 ceph/src/test/cli/ceph-conf/manpage.t create mode 100644 ceph/src/test/cli/ceph-conf/option.t create mode 100644 ceph/src/test/cli/ceph-conf/sections.t create mode 100644 ceph/src/test/cli/ceph-conf/show-config-value.t create mode 100644 ceph/src/test/cli/ceph-conf/show-config.t create mode 100644 ceph/src/test/cli/ceph-conf/simple.t create mode 100644 ceph/src/test/cli/crushtool/add-item.t create mode 100644 ceph/src/test/cli/crushtool/bad-mappings.crushmap.txt create mode 100644 ceph/src/test/cli/crushtool/bad-mappings.t create mode 100644 ceph/src/test/cli/crushtool/build.t create mode 100644 ceph/src/test/cli/crushtool/compile-decompile-recompile.t create mode 100644 ceph/src/test/cli/crushtool/five-devices.crushmap create mode 100644 ceph/src/test/cli/crushtool/help.t create mode 100644 ceph/src/test/cli/crushtool/multitype.after create mode 100644 ceph/src/test/cli/crushtool/multitype.before create mode 100644 ceph/src/test/cli/crushtool/need_tree_order.crush create mode 100644 ceph/src/test/cli/crushtool/output-csv.t create mode 100644 ceph/src/test/cli/crushtool/reweight.t create mode 100644 ceph/src/test/cli/crushtool/reweight_multiple.t create mode 100644 ceph/src/test/cli/crushtool/set-choose.crushmap.txt create mode 100644 ceph/src/test/cli/crushtool/set-choose.t create mode 100644 ceph/src/test/cli/crushtool/simple.template create mode 100644 ceph/src/test/cli/crushtool/simple.template.five create mode 100644 ceph/src/test/cli/crushtool/simple.template.four create mode 100644 ceph/src/test/cli/crushtool/simple.template.multitree create mode 100644 ceph/src/test/cli/crushtool/simple.template.multitree.reweighted create mode 100644 ceph/src/test/cli/crushtool/simple.template.one create mode 100644 ceph/src/test/cli/crushtool/simple.template.three create mode 100644 ceph/src/test/cli/crushtool/simple.template.two create mode 100644 ceph/src/test/cli/crushtool/test-map-a.crushmap create mode 100644 ceph/src/test/cli/crushtool/test-map-bobtail-tunables.t create mode 100644 ceph/src/test/cli/crushtool/test-map-firefly-tunables.t create mode 100644 ceph/src/test/cli/crushtool/test-map-indep.crushmap create mode 100644 ceph/src/test/cli/crushtool/test-map-indep.t create mode 100644 ceph/src/test/cli/crushtool/test-map-legacy-tunables.t create mode 100644 ceph/src/test/cli/crushtool/test-map-tries-vs-retries.crushmap create mode 100644 ceph/src/test/cli/crushtool/test-map-tries-vs-retries.t create mode 100644 ceph/src/test/cli/crushtool/test-map-vary-r-0.t create mode 100644 ceph/src/test/cli/crushtool/test-map-vary-r-1.t create mode 100644 ceph/src/test/cli/crushtool/test-map-vary-r-2.t create mode 100644 ceph/src/test/cli/crushtool/test-map-vary-r-3.t create mode 100644 ceph/src/test/cli/crushtool/test-map-vary-r-4.t create mode 100644 ceph/src/test/cli/crushtool/test-map-vary-r.crushmap create mode 100644 ceph/src/test/cli/monmaptool/add-exists.t create mode 100644 ceph/src/test/cli/monmaptool/add-many.t create mode 100644 ceph/src/test/cli/monmaptool/clobber.t create mode 100644 ceph/src/test/cli/monmaptool/create-print.t create mode 100644 ceph/src/test/cli/monmaptool/create-with-add.t create mode 100644 ceph/src/test/cli/monmaptool/help.t create mode 100644 ceph/src/test/cli/monmaptool/print-empty.t create mode 100644 ceph/src/test/cli/monmaptool/print-nonexistent.t create mode 100644 ceph/src/test/cli/monmaptool/rm-nonexistent.t create mode 100644 ceph/src/test/cli/monmaptool/rm.t create mode 100644 ceph/src/test/cli/monmaptool/simple.t create mode 100644 ceph/src/test/cli/osdmaptool/ceph.conf.withracks create mode 100644 ceph/src/test/cli/osdmaptool/clobber.t create mode 100644 ceph/src/test/cli/osdmaptool/create-print.t create mode 100644 ceph/src/test/cli/osdmaptool/create-racks.t create mode 100644 ceph/src/test/cli/osdmaptool/crush.t create mode 100644 ceph/src/test/cli/osdmaptool/help.t create mode 100644 ceph/src/test/cli/osdmaptool/missing-argument.t create mode 100644 ceph/src/test/cli/osdmaptool/pool.t create mode 100644 ceph/src/test/cli/osdmaptool/print-empty.t create mode 100644 ceph/src/test/cli/osdmaptool/print-nonexistent.t create mode 100644 ceph/src/test/cli/osdmaptool/test-map-pgs.t create mode 100644 ceph/src/test/cli/radosgw-admin/help.t create mode 100644 ceph/src/test/cli/rbd/help.t create mode 100644 ceph/src/test/cli/rbd/invalid-snap-usage.t create mode 100644 ceph/src/test/cli/rbd/not-enough-args.t create mode 100644 ceph/src/test/cls_hello/test_cls_hello.cc create mode 100644 ceph/src/test/cls_lock/test_cls_lock.cc create mode 100644 ceph/src/test/cls_log/test_cls_log.cc create mode 100644 ceph/src/test/cls_rbd/test_cls_rbd.cc create mode 100644 ceph/src/test/cls_refcount/test_cls_refcount.cc create mode 100644 ceph/src/test/cls_replica_log/test_cls_replica_log.cc create mode 100644 ceph/src/test/cls_rgw/test_cls_rgw.cc create mode 100644 ceph/src/test/cls_statelog/test_cls_statelog.cc create mode 100644 ceph/src/test/cls_version/test_cls_version.cc create mode 100644 ceph/src/test/common/ObjectContents.cc create mode 100644 ceph/src/test/common/ObjectContents.h create mode 100644 ceph/src/test/common/Throttle.cc create mode 100644 ceph/src/test/common/get_command_descriptions.cc create mode 100644 ceph/src/test/common/histogram.cc create mode 100644 ceph/src/test/common/test_bloom_filter.cc create mode 100644 ceph/src/test/common/test_config.cc create mode 100644 ceph/src/test/common/test_context.cc create mode 100644 ceph/src/test/common/test_crc32c.cc create mode 100644 ceph/src/test/common/test_sharedptr_registry.cc create mode 100644 ceph/src/test/common/test_sloppy_crc_map.cc create mode 100644 ceph/src/test/common/test_str_map.cc create mode 100644 ceph/src/test/common/test_util.cc create mode 100644 ceph/src/test/confutils.cc create mode 100644 ceph/src/test/crush/TestCrushWrapper.cc create mode 100644 ceph/src/test/crush/indep.cc create mode 100644 ceph/src/test/crypto.cc create mode 100644 ceph/src/test/crypto_init.cc create mode 100644 ceph/src/test/daemon_config.cc create mode 100644 ceph/src/test/downloads/cram-0.5.0ceph.2011-01-14.tar.gz create mode 100644 ceph/src/test/encoding.cc create mode 100644 ceph/src/test/encoding/ceph_dencoder.cc create mode 100755 ceph/src/test/encoding/check-generated.sh create mode 100755 ceph/src/test/encoding/readable.sh create mode 100644 ceph/src/test/encoding/types.h create mode 100644 ceph/src/test/erasure-code/ErasureCodeExample.h create mode 100644 ceph/src/test/erasure-code/ErasureCodePluginExample.cc create mode 100644 ceph/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc create mode 100644 ceph/src/test/erasure-code/ErasureCodePluginFailToRegister.cc create mode 100644 ceph/src/test/erasure-code/ErasureCodePluginHangs.cc create mode 100644 ceph/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc create mode 100644 ceph/src/test/erasure-code/Makefile.am create mode 100644 ceph/src/test/erasure-code/TestErasureCodeExample.cc create mode 100644 ceph/src/test/erasure-code/TestErasureCodeJerasure.cc create mode 100644 ceph/src/test/erasure-code/TestErasureCodePlugin.cc create mode 100644 ceph/src/test/erasure-code/TestErasureCodePluginJerasure.cc create mode 100644 ceph/src/test/erasure-code/TestJerasurePluginGeneric.cc create mode 100644 ceph/src/test/erasure-code/TestJerasurePluginSSE3.cc create mode 100644 ceph/src/test/erasure-code/TestJerasurePluginSSE4.cc create mode 100644 ceph/src/test/erasure-code/ceph_erasure_code.cc create mode 100644 ceph/src/test/erasure-code/ceph_erasure_code_benchmark.cc create mode 100644 ceph/src/test/erasure-code/ceph_erasure_code_benchmark.h create mode 100644 ceph/src/test/escape.cc create mode 100644 ceph/src/test/formatter.cc create mode 100644 ceph/src/test/gather.cc create mode 100644 ceph/src/test/heartbeat_map.cc create mode 100644 ceph/src/test/kv_store_bench.cc create mode 100644 ceph/src/test/kv_store_bench.h create mode 100644 ceph/src/test/libcephfs/caps.cc create mode 100644 ceph/src/test/libcephfs/multiclient.cc create mode 100644 ceph/src/test/libcephfs/readdir_r_cb.cc create mode 100644 ceph/src/test/libcephfs/test.cc create mode 100644 ceph/src/test/libcephfs_config.cc create mode 100644 ceph/src/test/librados/TestCase.cc create mode 100644 ceph/src/test/librados/TestCase.h create mode 100644 ceph/src/test/librados/aio.cc create mode 100644 ceph/src/test/librados/c_read_operations.cc create mode 100644 ceph/src/test/librados/c_write_operations.cc create mode 100644 ceph/src/test/librados/cls.cc create mode 100644 ceph/src/test/librados/cmd.cc create mode 100644 ceph/src/test/librados/io.cc create mode 100644 ceph/src/test/librados/librados.cc create mode 100644 ceph/src/test/librados/librados_config.cc create mode 100644 ceph/src/test/librados/list.cc create mode 100644 ceph/src/test/librados/lock.cc create mode 100644 ceph/src/test/librados/misc.cc create mode 100644 ceph/src/test/librados/pool.cc create mode 100644 ceph/src/test/librados/snapshots.cc create mode 100644 ceph/src/test/librados/stat.cc create mode 100644 ceph/src/test/librados/test.cc create mode 100644 ceph/src/test/librados/test.h create mode 100644 ceph/src/test/librados/tier.cc create mode 100644 ceph/src/test/librados/watch_notify.cc create mode 100644 ceph/src/test/librbd/fsx.c create mode 100644 ceph/src/test/librbd/test_librbd.cc create mode 100644 ceph/src/test/mime.cc create mode 100644 ceph/src/test/mon/PGMap.cc create mode 100644 ceph/src/test/mon/mon-test-helpers.sh create mode 100644 ceph/src/test/mon/moncap.cc create mode 100644 ceph/src/test/mon/test_mon_workloadgen.cc create mode 100644 ceph/src/test/multi_stress_watch.cc create mode 100644 ceph/src/test/objectstore/DeterministicOpSequence.cc create mode 100644 ceph/src/test/objectstore/DeterministicOpSequence.h create mode 100644 ceph/src/test/objectstore/FileStoreDiff.cc create mode 100644 ceph/src/test/objectstore/FileStoreDiff.h create mode 100644 ceph/src/test/objectstore/FileStoreTracker.cc create mode 100644 ceph/src/test/objectstore/FileStoreTracker.h create mode 100644 ceph/src/test/objectstore/TestObjectStoreState.cc create mode 100644 ceph/src/test/objectstore/TestObjectStoreState.h create mode 100644 ceph/src/test/objectstore/chain_xattr.cc create mode 100644 ceph/src/test/objectstore/store_test.cc create mode 100644 ceph/src/test/objectstore/test_idempotent.cc create mode 100644 ceph/src/test/objectstore/test_idempotent_sequence.cc create mode 100644 ceph/src/test/objectstore/workload_generator.cc create mode 100644 ceph/src/test/objectstore/workload_generator.h create mode 100644 ceph/src/test/omap_bench.cc create mode 100644 ceph/src/test/omap_bench.h create mode 100644 ceph/src/test/on_exit.cc create mode 100644 ceph/src/test/os/TestFlatIndex.cc create mode 100644 ceph/src/test/os/TestLFNIndex.cc create mode 100644 ceph/src/test/osd/Object.cc create mode 100644 ceph/src/test/osd/Object.h create mode 100644 ceph/src/test/osd/RadosModel.cc create mode 100644 ceph/src/test/osd/RadosModel.h create mode 100644 ceph/src/test/osd/TestECBackend.cc create mode 100644 ceph/src/test/osd/TestOSDMap.cc create mode 100644 ceph/src/test/osd/TestOpStat.cc create mode 100644 ceph/src/test/osd/TestOpStat.h create mode 100644 ceph/src/test/osd/TestPGLog.cc create mode 100644 ceph/src/test/osd/TestRados.cc create mode 100644 ceph/src/test/osd/hitset.cc create mode 100644 ceph/src/test/osd/osd-test-helpers.sh create mode 100644 ceph/src/test/osd/osdcap.cc create mode 100644 ceph/src/test/osd/types.cc create mode 100644 ceph/src/test/osdc/FakeWriteback.cc create mode 100644 ceph/src/test/osdc/FakeWriteback.h create mode 100644 ceph/src/test/osdc/object_cacher_stress.cc create mode 100644 ceph/src/test/perf_counters.cc create mode 100644 ceph/src/test/rgw/test_rgw_manifest.cc create mode 100755 ceph/src/test/run-cli-tests create mode 100755 ceph/src/test/run-cli-tests-maybe-unset-ccache create mode 100644 ceph/src/test/run_cmd.cc create mode 100644 ceph/src/test/signals.cc create mode 100644 ceph/src/test/simple_spin.cc create mode 100644 ceph/src/test/streamtest.cc create mode 100644 ceph/src/test/strtol.cc create mode 100644 ceph/src/test/system/cross_process_sem.cc create mode 100644 ceph/src/test/system/cross_process_sem.h create mode 100644 ceph/src/test/system/rados_delete_pools_parallel.cc create mode 100644 ceph/src/test/system/rados_list_parallel.cc create mode 100644 ceph/src/test/system/rados_open_pools_parallel.cc create mode 100644 ceph/src/test/system/rados_watch_notify.cc create mode 100644 ceph/src/test/system/st_rados_create_pool.cc create mode 100644 ceph/src/test/system/st_rados_create_pool.h create mode 100644 ceph/src/test/system/st_rados_delete_objs.cc create mode 100644 ceph/src/test/system/st_rados_delete_objs.h create mode 100644 ceph/src/test/system/st_rados_delete_pool.cc create mode 100644 ceph/src/test/system/st_rados_delete_pool.h create mode 100644 ceph/src/test/system/st_rados_list_objects.cc create mode 100644 ceph/src/test/system/st_rados_list_objects.h create mode 100644 ceph/src/test/system/st_rados_notify.cc create mode 100644 ceph/src/test/system/st_rados_notify.h create mode 100644 ceph/src/test/system/st_rados_watch.cc create mode 100644 ceph/src/test/system/st_rados_watch.h create mode 100644 ceph/src/test/system/systest_runnable.cc create mode 100644 ceph/src/test/system/systest_runnable.h create mode 100644 ceph/src/test/system/systest_settings.cc create mode 100644 ceph/src/test/system/systest_settings.h create mode 100644 ceph/src/test/test_addrs.cc create mode 100644 ceph/src/test/test_arch.cc create mode 100644 ceph/src/test/test_c_headers.c create mode 100644 ceph/src/test/test_cfuse_cache_invalidate.cc create mode 100644 ceph/src/test/test_cors.cc create mode 100644 ceph/src/test/test_filejournal.cc create mode 100644 ceph/src/test/test_get_blkdev_size.cc create mode 100644 ceph/src/test/test_ipaddr.cc create mode 100644 ceph/src/test/test_mutate.cc create mode 100644 ceph/src/test/test_prebufferedstreambuf.cc create mode 100644 ceph/src/test/test_rewrite_latency.cc create mode 100644 ceph/src/test/test_rgw_admin_log.cc create mode 100644 ceph/src/test/test_rgw_admin_meta.cc create mode 100644 ceph/src/test/test_rgw_admin_opstate.cc create mode 100644 ceph/src/test/test_snap_mapper.cc create mode 100644 ceph/src/test/test_str_list.cc create mode 100644 ceph/src/test/test_stress_watch.cc create mode 100644 ceph/src/test/test_striper.cc create mode 100644 ceph/src/test/test_texttable.cc create mode 100644 ceph/src/test/test_trans.cc create mode 100644 ceph/src/test/test_workqueue.cc create mode 100644 ceph/src/test/testcrypto.cc create mode 100644 ceph/src/test/testkeys.cc create mode 100644 ceph/src/test/testmsgr.cc create mode 100644 ceph/src/test/unit.h create mode 100644 ceph/src/test/utf8.cc create mode 100644 ceph/src/test/xattr_bench.cc create mode 100644 ceph/src/tools/Makefile.am create mode 100644 ceph/src/tools/ceph-client-debug.cc create mode 100644 ceph/src/tools/ceph_authtool.cc create mode 100644 ceph/src/tools/ceph_conf.cc create mode 100644 ceph/src/tools/ceph_filestore_dump.cc create mode 100644 ceph/src/tools/ceph_filestore_tool.cc create mode 100644 ceph/src/tools/ceph_kvstore_tool.cc create mode 100644 ceph/src/tools/ceph_monstore_tool.cc create mode 100644 ceph/src/tools/ceph_osdomap_tool.cc create mode 100644 ceph/src/tools/common.h create mode 100644 ceph/src/tools/crushtool.cc create mode 100644 ceph/src/tools/dupstore.cc create mode 100644 ceph/src/tools/mon_store_converter.cc create mode 100644 ceph/src/tools/monmaptool.cc create mode 100644 ceph/src/tools/osdmaptool.cc create mode 100644 ceph/src/tools/psim.cc create mode 100644 ceph/src/tools/rados/rados.cc create mode 100644 ceph/src/tools/rados/rados_export.cc create mode 100644 ceph/src/tools/rados/rados_import.cc create mode 100644 ceph/src/tools/rados/rados_sync.cc create mode 100644 ceph/src/tools/rados/rados_sync.h create mode 100644 ceph/src/tools/radosacl.cc create mode 100644 ceph/src/tools/rest_bench.cc create mode 100644 ceph/src/tools/scratchtool.c create mode 100644 ceph/src/tools/scratchtoolpp.cc create mode 100755 ceph/src/unittest_bufferlist.sh create mode 100644 ceph/src/upstart/ceph-all.conf create mode 100644 ceph/src/upstart/ceph-create-keys.conf create mode 100644 ceph/src/upstart/ceph-mds-all-starter.conf create mode 100644 ceph/src/upstart/ceph-mds-all.conf create mode 100644 ceph/src/upstart/ceph-mds.conf create mode 100644 ceph/src/upstart/ceph-mon-all-starter.conf create mode 100644 ceph/src/upstart/ceph-mon-all.conf create mode 100644 ceph/src/upstart/ceph-mon.conf create mode 100644 ceph/src/upstart/ceph-osd-all-starter.conf create mode 100644 ceph/src/upstart/ceph-osd-all.conf create mode 100644 ceph/src/upstart/ceph-osd.conf create mode 100644 ceph/src/upstart/radosgw-all-starter.conf create mode 100644 ceph/src/upstart/radosgw-all.conf create mode 100644 ceph/src/upstart/radosgw.conf create mode 100644 ceph/src/upstart/rbdmap.conf create mode 100755 ceph/src/verify-mds-journal.sh create mode 100755 ceph/src/vstart.sh create mode 100755 ceph/src/yasm-wrapper create mode 100644 ceph/udev/50-rbd.rules create mode 100644 ceph/udev/60-ceph-partuuid-workaround.rules create mode 100644 ceph/udev/95-ceph-osd-alt.rules create mode 100644 ceph/udev/95-ceph-osd.rules create mode 100644 debian/ceph-common.install create mode 100644 debian/ceph-fs-common.install create mode 100644 debian/ceph-fuse.install create mode 100644 debian/ceph-mds.install create mode 100644 debian/ceph-mds.lintian-overrides create mode 100644 debian/ceph-mds.postinst create mode 100644 debian/ceph-mds.prerm create mode 100644 debian/ceph-resource-agents.install create mode 100644 debian/ceph-test.install create mode 100644 debian/ceph.dirs create mode 100644 debian/ceph.install create mode 100644 debian/ceph.lintian-overrides create mode 100644 debian/ceph.postinst create mode 100644 debian/ceph.postrm create mode 100644 debian/ceph.prerm create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/gbp.conf create mode 100644 debian/libcephfs-dev.install create mode 100644 debian/libcephfs-java.jlibs create mode 100644 debian/libcephfs-jni.install create mode 100644 debian/libcephfs1.install create mode 100644 debian/librados-dev.install create mode 100644 debian/librados2.install create mode 100644 debian/librbd-dev.install create mode 100644 debian/librbd1.install create mode 100644 debian/patches/0001-rgw-RGWRados-get_obj-returns-wrong-len-if-len-0.patch create mode 100644 debian/patches/0001-rgw-fix-test-to-identify-whether-object-has-tail.patch create mode 100644 debian/patches/modules.patch create mode 100644 debian/patches/series create mode 100644 debian/patches/virtualenv-never-download create mode 100644 debian/python-ceph.install create mode 100644 debian/radosgw.dirs create mode 100644 debian/radosgw.install create mode 100644 debian/radosgw.lintian-overrides create mode 100644 debian/radosgw.postinst create mode 100644 debian/radosgw.postrm create mode 100644 debian/radosgw.preinst create mode 100644 debian/radosgw.prerm create mode 100644 debian/rbd-fuse.install create mode 100644 debian/rest-bench.install create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100755 debian/tests/build-rados create mode 100755 debian/tests/build-rbd create mode 100755 debian/tests/ceph-client create mode 100644 debian/tests/control create mode 100755 debian/tests/python-ceph create mode 100644 debian/watch create mode 100644 tests/integration_tests.conf diff --git a/ceph/AUTHORS b/ceph/AUTHORS new file mode 100644 index 00000000..289f54bf --- /dev/null +++ b/ceph/AUTHORS @@ -0,0 +1,19 @@ +Ceph core +--------- + +Sage Weil +Yehuda Sadeh-Weinraub +Greg Farnum + +Kernel client +------------- + +Sage Weil +Patience Warnick +Yehuda Sadeh-Weinraub +Greg Farnum + +Contributors +------------ + +Loic Dachary diff --git a/ceph/COPYING b/ceph/COPYING new file mode 100644 index 00000000..a0034d58 --- /dev/null +++ b/ceph/COPYING @@ -0,0 +1,140 @@ +Format-Specification: http://anonscm.debian.org/viewvc/dep/web/deps/dep5/copyright-format.xml?revision=279&view=markup +Name: ceph +Maintainer: Sage Weil +Source: http://ceph.com/ + +Files: * +Copyright: (c) 2004-2010 by Sage Weil +License: LGPL2.1 (see COPYING-LGPL2.1) + +Files: doc/* +Copyright: (c) 2010-2012 New Dream Network and contributors +License: Creative Commons Attribution-ShareAlike (CC BY-SA) + +Files: src/mount/canonicalize.c +Copyright: Copyright (C) 1993 Rick Sladkey +License: LGPL2 or later + +Files: src/os/btrfs_ioctl.h +Copyright: Copyright (C) 2007 Oracle. All rights reserved. +License: GPL2 + +Files: src/include/ceph_hash.cc +Copyright: None +License: Public domain + +Files: src/common/bloom_filter.hpp +Copyright: Copyright (C) 2000 Arash Partow +License: Boost Software License, Version 1.0 + +Files: m4/acx_pthread.m4 +Copyright: Steven G. Johnson +License: GPLWithACException + +Files: src/common/crc32c_intel*: +Copyright: + Copyright 2012-2013 Intel Corporation All Rights Reserved. +License: BSD 3-clause + +Files: src/common/sctp_crc32.c: +Copyright: + Copyright (c) 2001-2007, by Cisco Systems, Inc. All rights reserved. + Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved +License: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + a) Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + b) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + c) Neither the name of Cisco Systems, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +Files: src/json_spirit +Copyright: + Copyright John W. Wilkinson 2007 - 2011 +License: + The MIT License + + Copyright (c) 2007 - 2010 John W. Wilkinson + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + + +Files: src/test/common/Throttle.cc src/test/filestore/chain_xattr.cc +Copyright: Copyright (C) 2013 Cloudwatt +License: LGPL2 or later + +Files: src/osd/ErasureCodePluginJerasure/*.{c,h} +Copyright: Copyright (c) 2011, James S. Plank +License: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + - Neither the name of the University of Tennessee nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +Packaging: + Copyright (C) 2004-2009 by Sage Weil + Copyright (C) 2010 Canonical, Ltd. + Licensed under LGPL-2.1 diff --git a/ceph/ChangeLog b/ceph/ChangeLog new file mode 100644 index 00000000..e69de29b diff --git a/ceph/INSTALL b/ceph/INSTALL new file mode 100644 index 00000000..0dc8965f --- /dev/null +++ b/ceph/INSTALL @@ -0,0 +1,21 @@ +Installation Instructions +************************* + +If pulling from git, you first need to build the configure script with + +$ ./autogen.sh + +Then, + +$ ./configure +$ make + +Note that if the FUSE library is not found, the user-space fuse client +will not be built. + +If you are doing development, you may want to do + +$ CXXFLAGS="-g -pg" ./configure + +or similar to avoid the default (-g -O2), which includes optimizations +(-O2). diff --git a/ceph/Makefile.am b/ceph/Makefile.am new file mode 100644 index 00000000..cba3af2b --- /dev/null +++ b/ceph/Makefile.am @@ -0,0 +1,50 @@ +AUTOMAKE_OPTIONS = gnu +ACLOCAL_AMFLAGS = -I m4 +EXTRA_DIST = autogen.sh ceph.spec.in ceph.spec +# the "." here makes sure check-local builds gtest before it is used +SUBDIRS = . src man + +EXTRA_DIST += \ + src/test/run-cli-tests \ + src/test/run-cli-tests-maybe-unset-ccache \ + src/test/cli \ + src/test/downloads \ + udev/50-rbd.rules \ + udev/60-ceph-partuuid-workaround.rules \ + udev/95-ceph-osd.rules \ + udev/95-ceph-osd-alt.rules \ + share/known_hosts_drop.ceph.com \ + share/id_dsa_drop.ceph.com \ + share/id_dsa_drop.ceph.com.pub + +# why is it so hard to make autotools to this? +install-data-local: + -mkdir -p $(DESTDIR)$(datadir)/ceph + -install -m 644 share/known_hosts_drop.ceph.com $(DESTDIR)$(datadir)/ceph/known_hosts_drop.ceph.com + -install -m 644 share/id_dsa_drop.ceph.com $(DESTDIR)$(datadir)/ceph/id_dsa_drop.ceph.com + -install -m 644 share/id_dsa_drop.ceph.com.pub $(DESTDIR)$(datadir)/ceph/id_dsa_drop.ceph.com.pub + +all-local: +if WITH_DEBUG +# We need gtest to build the rados-api tests. We only build those in +# a debug build, though. + @cd src/gtest && $(MAKE) $(AM_MAKEFLAGS) lib/libgtest.a lib/libgtest_main.a +endif + +check-local: +# We build gtest this way, instead of using SUBDIRS, because with that, +# gtest's own tests would be run and that would slow us down. + @cd src/gtest && $(MAKE) $(AM_MAKEFLAGS) lib/libgtest.a lib/libgtest_main.a +# exercise cli tools + $(srcdir)/src/test/run-cli-tests '$(top_builddir)/src/test' + +# "make distclean" both runs this and recurses into src/gtest, if +# gtest is in DIST_SUBDIRS. Take extra care to not fail when +# effectively cleaned twice. +clean-local: + @if test -e src/gtest/Makefile; then \ + echo "Making clean in src/gtest"; \ + cd src/gtest && $(MAKE) $(AM_MAKEFLAGS) clean; \ + fi + + @rm -rf src/test/virtualenv diff --git a/ceph/Makefile.in b/ceph/Makefile.in new file mode 100644 index 00000000..eb16431e --- /dev/null +++ b/ceph/Makefile.in @@ -0,0 +1,849 @@ +# Makefile.in generated by automake 1.11.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software +# Foundation, Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +subdir = . +DIST_COMMON = README $(am__configure_deps) $(srcdir)/Makefile.am \ + $(srcdir)/Makefile.in $(srcdir)/ceph.spec.in \ + $(top_srcdir)/configure AUTHORS COPYING ChangeLog INSTALL NEWS \ + ar-lib compile config.guess config.sub depcomp install-sh \ + ltmain.sh missing py-compile +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \ + $(top_srcdir)/m4/ac_prog_jar.m4 \ + $(top_srcdir)/m4/ac_prog_javac.m4 \ + $(top_srcdir)/m4/ac_prog_javac_works.m4 \ + $(top_srcdir)/m4/ac_prog_javah.m4 \ + $(top_srcdir)/m4/acx_pthread.m4 \ + $(top_srcdir)/m4/ax_c_pretty_func.m4 \ + $(top_srcdir)/m4/ax_c_var_func.m4 \ + $(top_srcdir)/m4/ax_check_compile_flag.m4 \ + $(top_srcdir)/m4/ax_cxx_static_cast.m4 \ + $(top_srcdir)/m4/ax_intel.m4 $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/pkg.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \ + configure.lineno config.status.lineno +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/src/acconfig.h +CONFIG_CLEAN_FILES = ceph.spec +CONFIG_CLEAN_VPATH_FILES = +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +SOURCES = +DIST_SOURCES = +RECURSIVE_TARGETS = all-recursive check-recursive dvi-recursive \ + html-recursive info-recursive install-data-recursive \ + install-dvi-recursive install-exec-recursive \ + install-html-recursive install-info-recursive \ + install-pdf-recursive install-ps-recursive install-recursive \ + installcheck-recursive installdirs-recursive pdf-recursive \ + ps-recursive uninstall-recursive +RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ + distclean-recursive maintainer-clean-recursive +AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \ + $(RECURSIVE_CLEAN_TARGETS:-recursive=) tags TAGS ctags CTAGS \ + distdir dist dist-all distcheck +ETAGS = etags +CTAGS = ctags +DIST_SUBDIRS = $(SUBDIRS) +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +distdir = $(PACKAGE)-$(VERSION) +top_distdir = $(distdir) +am__remove_distdir = \ + if test -d "$(distdir)"; then \ + find "$(distdir)" -type d ! -perm -200 -exec chmod u+w {} ';' \ + && rm -rf "$(distdir)" \ + || { sleep 5 && rm -rf "$(distdir)"; }; \ + else :; fi +am__relativize = \ + dir0=`pwd`; \ + sed_first='s,^\([^/]*\)/.*$$,\1,'; \ + sed_rest='s,^[^/]*/*,,'; \ + sed_last='s,^.*/\([^/]*\)$$,\1,'; \ + sed_butlast='s,/*[^/]*$$,,'; \ + while test -n "$$dir1"; do \ + first=`echo "$$dir1" | sed -e "$$sed_first"`; \ + if test "$$first" != "."; then \ + if test "$$first" = ".."; then \ + dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \ + dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \ + else \ + first2=`echo "$$dir2" | sed -e "$$sed_first"`; \ + if test "$$first2" = "$$first"; then \ + dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \ + else \ + dir2="../$$dir2"; \ + fi; \ + dir0="$$dir0"/"$$first"; \ + fi; \ + fi; \ + dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \ + done; \ + reldir="$$dir2" +DIST_ARCHIVES = $(distdir).tar.gz +GZIP_ENV = --best +distuninstallcheck_listfiles = find . -type f -print +am__distuninstallcheck_listfiles = $(distuninstallcheck_listfiles) \ + | sed 's|^\./|$(prefix)/|' | grep -v '$(infodir)/dir$$' +distcleancheck_listfiles = find . -type f -print +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_CXXFLAGS = @AM_CXXFLAGS@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@ +CC = @CC@ +CCAS = @CCAS@ +CCASDEPMODE = @CCASDEPMODE@ +CCASFLAGS = @CCASFLAGS@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPTOPP_CFLAGS = @CRYPTOPP_CFLAGS@ +CRYPTOPP_LIBS = @CRYPTOPP_LIBS@ +CRYPTO_CFLAGS = @CRYPTO_CFLAGS@ +CRYPTO_LIBS = @CRYPTO_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +EXTRA_CLASSPATH_JAR = @EXTRA_CLASSPATH_JAR@ +FGREP = @FGREP@ +GCOV_PREFIX_STRIP = @GCOV_PREFIX_STRIP@ +GIT_CHECK = @GIT_CHECK@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +INTEL_FLAGS = @INTEL_FLAGS@ +INTEL_PCLMUL_FLAGS = @INTEL_PCLMUL_FLAGS@ +INTEL_SSE2_FLAGS = @INTEL_SSE2_FLAGS@ +INTEL_SSE3_FLAGS = @INTEL_SSE3_FLAGS@ +INTEL_SSE4_1_FLAGS = @INTEL_SSE4_1_FLAGS@ +INTEL_SSE4_2_FLAGS = @INTEL_SSE4_2_FLAGS@ +INTEL_SSE_FLAGS = @INTEL_SSE_FLAGS@ +INTEL_SSSE3_FLAGS = @INTEL_SSSE3_FLAGS@ +JAR = @JAR@ +JAVAC = @JAVAC@ +JAVAH = @JAVAH@ +JDK_CPPFLAGS = @JDK_CPPFLAGS@ +KEYUTILS_LIB = @KEYUTILS_LIB@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@ +LIBEDIT_LIBS = @LIBEDIT_LIBS@ +LIBFUSE = @LIBFUSE@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTCMALLOC = @LIBTCMALLOC@ +LIBTOOL = @LIBTOOL@ +LIBZFS_CFLAGS = @LIBZFS_CFLAGS@ +LIBZFS_LIBS = @LIBZFS_LIBS@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NSS_CFLAGS = @NSS_CFLAGS@ +NSS_LIBS = @NSS_LIBS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +PTHREAD_CC = @PTHREAD_CC@ +PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ +PTHREAD_LIBS = @PTHREAD_LIBS@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +RESOLV_LIBS = @RESOLV_LIBS@ +RPM_RELEASE = @RPM_RELEASE@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +WARN_IGNORED_QUALIFIERS = @WARN_IGNORED_QUALIFIERS@ +WARN_TYPE_LIMITS = @WARN_TYPE_LIMITS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +acx_pthread_config = @acx_pthread_config@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +subdirs = @subdirs@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +AUTOMAKE_OPTIONS = gnu +ACLOCAL_AMFLAGS = -I m4 +EXTRA_DIST = autogen.sh ceph.spec.in ceph.spec src/test/run-cli-tests \ + src/test/run-cli-tests-maybe-unset-ccache src/test/cli \ + src/test/downloads udev/50-rbd.rules \ + udev/60-ceph-partuuid-workaround.rules udev/95-ceph-osd.rules \ + udev/95-ceph-osd-alt.rules share/known_hosts_drop.ceph.com \ + share/id_dsa_drop.ceph.com share/id_dsa_drop.ceph.com.pub +# the "." here makes sure check-local builds gtest before it is used +SUBDIRS = . src man +all: all-recursive + +.SUFFIXES: +am--refresh: Makefile + @: +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + echo ' cd $(srcdir) && $(AUTOMAKE) --gnu'; \ + $(am__cd) $(srcdir) && $(AUTOMAKE) --gnu \ + && exit 0; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + echo ' $(SHELL) ./config.status'; \ + $(SHELL) ./config.status;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + $(SHELL) ./config.status --recheck + +$(top_srcdir)/configure: $(am__configure_deps) + $(am__cd) $(srcdir) && $(AUTOCONF) +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + $(am__cd) $(srcdir) && $(ACLOCAL) $(ACLOCAL_AMFLAGS) +$(am__aclocal_m4_deps): +ceph.spec: $(top_builddir)/config.status $(srcdir)/ceph.spec.in + cd $(top_builddir) && $(SHELL) ./config.status $@ + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +distclean-libtool: + -rm -f libtool config.lt + +# This directory's subdirectories are mostly independent; you can cd +# into them and run `make' without going through this Makefile. +# To change the values of `make' variables: instead of editing Makefiles, +# (1) if the variable is set in `config.status', edit `config.status' +# (which will cause the Makefiles to be regenerated when you run `make'); +# (2) otherwise, pass the desired values on the `make' command line. +$(RECURSIVE_TARGETS): + @fail= failcom='exit 1'; \ + for f in x $$MAKEFLAGS; do \ + case $$f in \ + *=* | --[!k]*);; \ + *k*) failcom='fail=yes';; \ + esac; \ + done; \ + dot_seen=no; \ + target=`echo $@ | sed s/-recursive//`; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + dot_seen=yes; \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done; \ + if test "$$dot_seen" = "no"; then \ + $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \ + fi; test -z "$$fail" + +$(RECURSIVE_CLEAN_TARGETS): + @fail= failcom='exit 1'; \ + for f in x $$MAKEFLAGS; do \ + case $$f in \ + *=* | --[!k]*);; \ + *k*) failcom='fail=yes';; \ + esac; \ + done; \ + dot_seen=no; \ + case "$@" in \ + distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \ + *) list='$(SUBDIRS)' ;; \ + esac; \ + rev=''; for subdir in $$list; do \ + if test "$$subdir" = "."; then :; else \ + rev="$$subdir $$rev"; \ + fi; \ + done; \ + rev="$$rev ."; \ + target=`echo $@ | sed s/-recursive//`; \ + for subdir in $$rev; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done && test -z "$$fail" +tags-recursive: + list='$(SUBDIRS)'; for subdir in $$list; do \ + test "$$subdir" = . || ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) tags); \ + done +ctags-recursive: + list='$(SUBDIRS)'; for subdir in $$list; do \ + test "$$subdir" = . || ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) ctags); \ + done + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: tags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \ + include_option=--etags-include; \ + empty_fix=.; \ + else \ + include_option=--include; \ + empty_fix=; \ + fi; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test ! -f $$subdir/TAGS || \ + set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \ + fi; \ + done; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: ctags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + $(am__remove_distdir) + test -d "$(distdir)" || mkdir "$(distdir)" + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test -d "$(distdir)/$$subdir" \ + || $(MKDIR_P) "$(distdir)/$$subdir" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + dir1=$$subdir; dir2="$(distdir)/$$subdir"; \ + $(am__relativize); \ + new_distdir=$$reldir; \ + dir1=$$subdir; dir2="$(top_distdir)"; \ + $(am__relativize); \ + new_top_distdir=$$reldir; \ + echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \ + echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \ + ($(am__cd) $$subdir && \ + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$$new_top_distdir" \ + distdir="$$new_distdir" \ + am__remove_distdir=: \ + am__skip_length_check=: \ + am__skip_mode_fix=: \ + distdir) \ + || exit 1; \ + fi; \ + done + -test -n "$(am__skip_mode_fix)" \ + || find "$(distdir)" -type d ! -perm -755 \ + -exec chmod u+rwx,go+rx {} \; -o \ + ! -type d ! -perm -444 -links 1 -exec chmod a+r {} \; -o \ + ! -type d ! -perm -400 -exec chmod a+r {} \; -o \ + ! -type d ! -perm -444 -exec $(install_sh) -c -m a+r {} {} \; \ + || chmod -R a+r "$(distdir)" +dist-gzip: distdir + tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz + $(am__remove_distdir) + +dist-bzip2: distdir + tardir=$(distdir) && $(am__tar) | BZIP2=$${BZIP2--9} bzip2 -c >$(distdir).tar.bz2 + $(am__remove_distdir) + +dist-lzip: distdir + tardir=$(distdir) && $(am__tar) | lzip -c $${LZIP_OPT--9} >$(distdir).tar.lz + $(am__remove_distdir) + +dist-lzma: distdir + tardir=$(distdir) && $(am__tar) | lzma -9 -c >$(distdir).tar.lzma + $(am__remove_distdir) + +dist-xz: distdir + tardir=$(distdir) && $(am__tar) | XZ_OPT=$${XZ_OPT--e} xz -c >$(distdir).tar.xz + $(am__remove_distdir) + +dist-tarZ: distdir + tardir=$(distdir) && $(am__tar) | compress -c >$(distdir).tar.Z + $(am__remove_distdir) + +dist-shar: distdir + shar $(distdir) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).shar.gz + $(am__remove_distdir) + +dist-zip: distdir + -rm -f $(distdir).zip + zip -rq $(distdir).zip $(distdir) + $(am__remove_distdir) + +dist dist-all: distdir + tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz + $(am__remove_distdir) + +# This target untars the dist file and tries a VPATH configuration. Then +# it guarantees that the distribution is self-contained by making another +# tarfile. +distcheck: dist + case '$(DIST_ARCHIVES)' in \ + *.tar.gz*) \ + GZIP=$(GZIP_ENV) gzip -dc $(distdir).tar.gz | $(am__untar) ;;\ + *.tar.bz2*) \ + bzip2 -dc $(distdir).tar.bz2 | $(am__untar) ;;\ + *.tar.lzma*) \ + lzma -dc $(distdir).tar.lzma | $(am__untar) ;;\ + *.tar.lz*) \ + lzip -dc $(distdir).tar.lz | $(am__untar) ;;\ + *.tar.xz*) \ + xz -dc $(distdir).tar.xz | $(am__untar) ;;\ + *.tar.Z*) \ + uncompress -c $(distdir).tar.Z | $(am__untar) ;;\ + *.shar.gz*) \ + GZIP=$(GZIP_ENV) gzip -dc $(distdir).shar.gz | unshar ;;\ + *.zip*) \ + unzip $(distdir).zip ;;\ + esac + chmod -R a-w $(distdir); chmod a+w $(distdir) + mkdir $(distdir)/_build + mkdir $(distdir)/_inst + chmod a-w $(distdir) + test -d $(distdir)/_build || exit 0; \ + dc_install_base=`$(am__cd) $(distdir)/_inst && pwd | sed -e 's,^[^:\\/]:[\\/],/,'` \ + && dc_destdir="$${TMPDIR-/tmp}/am-dc-$$$$/" \ + && am__cwd=`pwd` \ + && $(am__cd) $(distdir)/_build \ + && ../configure --srcdir=.. --prefix="$$dc_install_base" \ + $(AM_DISTCHECK_CONFIGURE_FLAGS) \ + $(DISTCHECK_CONFIGURE_FLAGS) \ + && $(MAKE) $(AM_MAKEFLAGS) \ + && $(MAKE) $(AM_MAKEFLAGS) dvi \ + && $(MAKE) $(AM_MAKEFLAGS) check \ + && $(MAKE) $(AM_MAKEFLAGS) install \ + && $(MAKE) $(AM_MAKEFLAGS) installcheck \ + && $(MAKE) $(AM_MAKEFLAGS) uninstall \ + && $(MAKE) $(AM_MAKEFLAGS) distuninstallcheck_dir="$$dc_install_base" \ + distuninstallcheck \ + && chmod -R a-w "$$dc_install_base" \ + && ({ \ + (cd ../.. && umask 077 && mkdir "$$dc_destdir") \ + && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" install \ + && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" uninstall \ + && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" \ + distuninstallcheck_dir="$$dc_destdir" distuninstallcheck; \ + } || { rm -rf "$$dc_destdir"; exit 1; }) \ + && rm -rf "$$dc_destdir" \ + && $(MAKE) $(AM_MAKEFLAGS) dist \ + && rm -rf $(DIST_ARCHIVES) \ + && $(MAKE) $(AM_MAKEFLAGS) distcleancheck \ + && cd "$$am__cwd" \ + || exit 1 + $(am__remove_distdir) + @(echo "$(distdir) archives ready for distribution: "; \ + list='$(DIST_ARCHIVES)'; for i in $$list; do echo $$i; done) | \ + sed -e 1h -e 1s/./=/g -e 1p -e 1x -e '$$p' -e '$$x' +distuninstallcheck: + @test -n '$(distuninstallcheck_dir)' || { \ + echo 'ERROR: trying to run $@ with an empty' \ + '$$(distuninstallcheck_dir)' >&2; \ + exit 1; \ + }; \ + $(am__cd) '$(distuninstallcheck_dir)' || { \ + echo 'ERROR: cannot chdir into $(distuninstallcheck_dir)' >&2; \ + exit 1; \ + }; \ + test `$(am__distuninstallcheck_listfiles) | wc -l` -eq 0 \ + || { echo "ERROR: files left after uninstall:" ; \ + if test -n "$(DESTDIR)"; then \ + echo " (check DESTDIR support)"; \ + fi ; \ + $(distuninstallcheck_listfiles) ; \ + exit 1; } >&2 +distcleancheck: distclean + @if test '$(srcdir)' = . ; then \ + echo "ERROR: distcleancheck can only run from a VPATH build" ; \ + exit 1 ; \ + fi + @test `$(distcleancheck_listfiles) | wc -l` -eq 0 \ + || { echo "ERROR: files left in build directory after distclean:" ; \ + $(distcleancheck_listfiles) ; \ + exit 1; } >&2 +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-recursive +all-am: Makefile all-local +installdirs: installdirs-recursive +installdirs-am: +install: install-recursive +install-exec: install-exec-recursive +install-data: install-data-recursive +uninstall: uninstall-recursive + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-recursive +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-recursive + +clean-am: clean-generic clean-libtool clean-local mostlyclean-am + +distclean: distclean-recursive + -rm -f $(am__CONFIG_DISTCLEAN_FILES) + -rm -f Makefile +distclean-am: clean-am distclean-generic distclean-libtool \ + distclean-tags + +dvi: dvi-recursive + +dvi-am: + +html: html-recursive + +html-am: + +info: info-recursive + +info-am: + +install-data-am: install-data-local + +install-dvi: install-dvi-recursive + +install-dvi-am: + +install-exec-am: + +install-html: install-html-recursive + +install-html-am: + +install-info: install-info-recursive + +install-info-am: + +install-man: + +install-pdf: install-pdf-recursive + +install-pdf-am: + +install-ps: install-ps-recursive + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-recursive + -rm -f $(am__CONFIG_DISTCLEAN_FILES) + -rm -rf $(top_srcdir)/autom4te.cache + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-recursive + +mostlyclean-am: mostlyclean-generic mostlyclean-libtool + +pdf: pdf-recursive + +pdf-am: + +ps: ps-recursive + +ps-am: + +uninstall-am: + +.MAKE: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) check-am \ + ctags-recursive install-am install-strip tags-recursive + +.PHONY: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) CTAGS GTAGS \ + all all-am all-local am--refresh check check-am check-local \ + clean clean-generic clean-libtool clean-local ctags \ + ctags-recursive dist dist-all dist-bzip2 dist-gzip dist-lzip \ + dist-lzma dist-shar dist-tarZ dist-xz dist-zip distcheck \ + distclean distclean-generic distclean-libtool distclean-tags \ + distcleancheck distdir distuninstallcheck dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-data-local install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-ps install-ps-am install-strip \ + installcheck installcheck-am installdirs installdirs-am \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags tags-recursive uninstall uninstall-am + + +# why is it so hard to make autotools to this? +install-data-local: + -mkdir -p $(DESTDIR)$(datadir)/ceph + -install -m 644 share/known_hosts_drop.ceph.com $(DESTDIR)$(datadir)/ceph/known_hosts_drop.ceph.com + -install -m 644 share/id_dsa_drop.ceph.com $(DESTDIR)$(datadir)/ceph/id_dsa_drop.ceph.com + -install -m 644 share/id_dsa_drop.ceph.com.pub $(DESTDIR)$(datadir)/ceph/id_dsa_drop.ceph.com.pub + +all-local: +# We need gtest to build the rados-api tests. We only build those in +# a debug build, though. +@WITH_DEBUG_TRUE@ @cd src/gtest && $(MAKE) $(AM_MAKEFLAGS) lib/libgtest.a lib/libgtest_main.a + +check-local: +# We build gtest this way, instead of using SUBDIRS, because with that, +# gtest's own tests would be run and that would slow us down. + @cd src/gtest && $(MAKE) $(AM_MAKEFLAGS) lib/libgtest.a lib/libgtest_main.a +# exercise cli tools + $(srcdir)/src/test/run-cli-tests '$(top_builddir)/src/test' + +# "make distclean" both runs this and recurses into src/gtest, if +# gtest is in DIST_SUBDIRS. Take extra care to not fail when +# effectively cleaned twice. +clean-local: + @if test -e src/gtest/Makefile; then \ + echo "Making clean in src/gtest"; \ + cd src/gtest && $(MAKE) $(AM_MAKEFLAGS) clean; \ + fi + + @rm -rf src/test/virtualenv + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/ceph/NEWS b/ceph/NEWS new file mode 100644 index 00000000..e69de29b diff --git a/ceph/README b/ceph/README new file mode 100644 index 00000000..0eb3ac73 --- /dev/null +++ b/ceph/README @@ -0,0 +1,177 @@ +============================================ +Ceph - a scalable distributed storage system +============================================ + +Please see http://ceph.com/ for current info. + +Contributing Code +================= + +Most of Ceph is licensed under the LGPL version 2.1. Some +miscellaneous code is under BSD-style license or is public domain. +The documentation is licensed under Creative Commons +Attribution-ShareAlike (CC BY-SA). There are a handful of headers +included here that are licensed under the GPL. Please see the file +COPYING for a full inventory of licenses by file. + +Code contributions must include a valid "Signed-off-by" acknowledging +the license for the modified or contributed file. Please see the file +SubmittingPatches for details on what that means and on how to +generate and submit patches. + +We do not require assignment of copyright to contribute code; code is +contributed under the terms of the applicable license. + + +Building Ceph +============= + +To prepare the source tree after it has been git cloned, + + $ git submodule update --init + +To build the server daemons, and FUSE client, execute the following: + + $ ./autogen.sh + $ ./configure + $ make + +(Note that the FUSE client will only be built if libfuse is present.) + +Dependencies +------------ + +The configure script will complain about any missing dependencies as +it goes. You can also refer to debian/control or ceph.spec.in for the +package build dependencies on those platforms. In many cases, +dependencies can be avoided with --with-foo or --without-bar switches. +For example, + +$ ./configure --with-nss # use libnss instead of libcrypto++ +$ ./configure --without-radosgw # do not build radosgw and avoid libfcgi-dev +$ ./configure --without-tcmalloc # avoid google-perftools dependency + + +Building packages +----------------- + +You can build packages for Debian or Debian-derived (e.g., Ubuntu) +systems with + +$ sudo apt-get install dpkg-dev +$ dpkg-checkbuilddeps # make sure we have all dependencies +$ dpkg-buildpackage + +For RPM-based systems (Redhat, Suse, etc.), + +$ rpmbuild + + +Building the Documentation +========================== + +Prerequisites +------------- +To build the documentation, you must install the following: + +- python-dev +- python-pip +- python-virtualenv +- doxygen +- ditaa +- libxml2-dev +- libxslt-dev +- dot +- graphviz + +For example: + + sudo apt-get install python-dev python-pip python-virtualenv doxygen ditaa libxml2-dev libxslt-dev dot graphviz + +Building the Documentation +-------------------------- + +To build the documentation, ensure that you are in the top-level `/ceph directory, and execute the build script. For example: + + $ admin/build-doc + + +Build Prerequisites +=================== + +debian-based +------------ +To build the source code, you must install the following: + +- automake +- autoconf +- pkg-config +- gcc +- g++ +- make +- libboost-dev +- libedit-dev +- libssl-dev +- libtool +- libfcgi +- libfcgi-dev +- xfslibs-dev +- libfuse-dev +- linux-kernel-headers +- libcrypto++-dev +- libaio-dev +- libgoogle-perftools-dev +- libkeyutils-dev +- uuid-dev +- libblkid-dev +- libatomic-ops-dev +- libboost-program-options-dev +- libboost-system-dev +- libboost-thread-dev +- libexpat1-dev +- libleveldb-dev +- libsnappy-dev +- libcurl4-gnutls-dev +- python-argparse +- python-flask + +For example: + + $ apt-get install automake autoconf pkg-config gcc g++ make libboost-dev libedit-dev libssl-dev libtool libfcgi libfcgi-dev xfslibs-dev libfuse-dev linux-kernel-headers libcrypto++-dev libaio-dev libgoogle-perftools-dev libkeyutils-dev uuid-dev libblkid-dev libatomic-ops-dev libboost-program-options-dev libboost-system-dev libboost-thread-dev libexpat1-dev libleveldb-dev libsnappy-dev libcurl4-gnutls-dev python-argparse python-flask python-nose + +Note: libsnappy-dev and libleveldb-dev are not available upstream for natty, oneiric, and squeeze. Backports for Ceph can be found at ceph.com/debian-leveldb. + +rpm-based +--------- +These are the rpm packages needed to install in an rpm-based OS: + + autoconf + automake + gcc + gcc-c++ + make + libtool + python-argparse + python-flask + libuuid-devel + libblkid-devel + keyutils-libs-devel + cryptopp-devel + nss-devel + fcgi-devel + expat-devel + libcurl-devel + xfsprogs-devel + fuse-devel + gperftools-devel + libedit-devel + libatomic_ops-devel + snappy-devel + leveldb-devel + libaio-devel + boost-devel + +For example: + + $ yum install autoconf automake gcc gcc-c++ make libtool python-argparse python-flask libuuid-devel libblkid-devel keyutils-libs-devel cryptopp-devel nss-devel fcgi-devel expat-devel libcurl-devel xfsprogs-devel fuse-devel gperftools-devel libedit-devel libatomic_ops-devel snappy-devel leveldb-devel libaio-devel boost-devel python-nose + diff --git a/ceph/aclocal.m4 b/ceph/aclocal.m4 new file mode 100644 index 00000000..fa0745ae --- /dev/null +++ b/ceph/aclocal.m4 @@ -0,0 +1,1385 @@ +# generated automatically by aclocal 1.11.3 -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, +# 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, +# Inc. +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +m4_ifndef([AC_AUTOCONF_VERSION], + [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl +m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.68],, +[m4_warning([this file was generated for autoconf 2.68. +You have another version of autoconf. It may work, but is not guaranteed to. +If you have problems, you may need to regenerate the build system entirely. +To do so, use the procedure documented by the package, typically `autoreconf'.])]) + +# Copyright (C) 2002, 2003, 2005, 2006, 2007, 2008, 2011 Free Software +# Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 1 + +# AM_AUTOMAKE_VERSION(VERSION) +# ---------------------------- +# Automake X.Y traces this macro to ensure aclocal.m4 has been +# generated from the m4 files accompanying Automake X.Y. +# (This private macro should not be called outside this file.) +AC_DEFUN([AM_AUTOMAKE_VERSION], +[am__api_version='1.11' +dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to +dnl require some minimum version. Point them to the right macro. +m4_if([$1], [1.11.3], [], + [AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl +]) + +# _AM_AUTOCONF_VERSION(VERSION) +# ----------------------------- +# aclocal traces this macro to find the Autoconf version. +# This is a private macro too. Using m4_define simplifies +# the logic in aclocal, which can simply ignore this definition. +m4_define([_AM_AUTOCONF_VERSION], []) + +# AM_SET_CURRENT_AUTOMAKE_VERSION +# ------------------------------- +# Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced. +# This function is AC_REQUIREd by AM_INIT_AUTOMAKE. +AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION], +[AM_AUTOMAKE_VERSION([1.11.3])dnl +m4_ifndef([AC_AUTOCONF_VERSION], + [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl +_AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))]) + +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 1 + +# AM_PROG_AR([ACT-IF-FAIL]) +# ------------------------- +# Try to determine the archiver interface, and trigger the ar-lib wrapper +# if it is needed. If the detection of archiver interface fails, run +# ACT-IF-FAIL (default is to abort configure with a proper error message). +AC_DEFUN([AM_PROG_AR], +[AC_BEFORE([$0], [LT_INIT])dnl +AC_BEFORE([$0], [AC_PROG_LIBTOOL])dnl +AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +AC_REQUIRE_AUX_FILE([ar-lib])dnl +AC_CHECK_TOOLS([AR], [ar lib "link -lib"], [false]) +: ${AR=ar} + +AC_CACHE_CHECK([the archiver ($AR) interface], [am_cv_ar_interface], + [am_cv_ar_interface=ar + AC_COMPILE_IFELSE([AC_LANG_SOURCE([[int some_variable = 0;]])], + [am_ar_try='$AR cru libconftest.a conftest.$ac_objext >&AS_MESSAGE_LOG_FD' + AC_TRY_EVAL([am_ar_try]) + if test "$ac_status" -eq 0; then + am_cv_ar_interface=ar + else + am_ar_try='$AR -NOLOGO -OUT:conftest.lib conftest.$ac_objext >&AS_MESSAGE_LOG_FD' + AC_TRY_EVAL([am_ar_try]) + if test "$ac_status" -eq 0; then + am_cv_ar_interface=lib + else + am_cv_ar_interface=unknown + fi + fi + rm -f conftest.lib libconftest.a + ]) + ]) + +case $am_cv_ar_interface in +ar) + ;; +lib) + # Microsoft lib, so override with the ar-lib wrapper script. + # FIXME: It is wrong to rewrite AR. + # But if we don't then we get into trouble of one sort or another. + # A longer-term fix would be to have automake use am__AR in this case, + # and then we could set am__AR="$am_aux_dir/ar-lib \$(AR)" or something + # similar. + AR="$am_aux_dir/ar-lib $AR" + ;; +unknown) + m4_default([$1], + [AC_MSG_ERROR([could not determine $AR interface])]) + ;; +esac +AC_SUBST([AR])dnl +]) + +# Figure out how to run the assembler. -*- Autoconf -*- + +# Copyright (C) 2001, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 5 + +# AM_PROG_AS +# ---------- +AC_DEFUN([AM_PROG_AS], +[# By default we simply use the C compiler to build assembly code. +AC_REQUIRE([AC_PROG_CC]) +test "${CCAS+set}" = set || CCAS=$CC +test "${CCASFLAGS+set}" = set || CCASFLAGS=$CFLAGS +AC_ARG_VAR([CCAS], [assembler compiler command (defaults to CC)]) +AC_ARG_VAR([CCASFLAGS], [assembler compiler flags (defaults to CFLAGS)]) +_AM_IF_OPTION([no-dependencies],, [_AM_DEPENDENCIES([CCAS])])dnl +]) + +# AM_AUX_DIR_EXPAND -*- Autoconf -*- + +# Copyright (C) 2001, 2003, 2005, 2011 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 1 + +# For projects using AC_CONFIG_AUX_DIR([foo]), Autoconf sets +# $ac_aux_dir to `$srcdir/foo'. In other projects, it is set to +# `$srcdir', `$srcdir/..', or `$srcdir/../..'. +# +# Of course, Automake must honor this variable whenever it calls a +# tool from the auxiliary directory. The problem is that $srcdir (and +# therefore $ac_aux_dir as well) can be either absolute or relative, +# depending on how configure is run. This is pretty annoying, since +# it makes $ac_aux_dir quite unusable in subdirectories: in the top +# source directory, any form will work fine, but in subdirectories a +# relative path needs to be adjusted first. +# +# $ac_aux_dir/missing +# fails when called from a subdirectory if $ac_aux_dir is relative +# $top_srcdir/$ac_aux_dir/missing +# fails if $ac_aux_dir is absolute, +# fails when called from a subdirectory in a VPATH build with +# a relative $ac_aux_dir +# +# The reason of the latter failure is that $top_srcdir and $ac_aux_dir +# are both prefixed by $srcdir. In an in-source build this is usually +# harmless because $srcdir is `.', but things will broke when you +# start a VPATH build or use an absolute $srcdir. +# +# So we could use something similar to $top_srcdir/$ac_aux_dir/missing, +# iff we strip the leading $srcdir from $ac_aux_dir. That would be: +# am_aux_dir='\$(top_srcdir)/'`expr "$ac_aux_dir" : "$srcdir//*\(.*\)"` +# and then we would define $MISSING as +# MISSING="\${SHELL} $am_aux_dir/missing" +# This will work as long as MISSING is not called from configure, because +# unfortunately $(top_srcdir) has no meaning in configure. +# However there are other variables, like CC, which are often used in +# configure, and could therefore not use this "fixed" $ac_aux_dir. +# +# Another solution, used here, is to always expand $ac_aux_dir to an +# absolute PATH. The drawback is that using absolute paths prevent a +# configured tree to be moved without reconfiguration. + +AC_DEFUN([AM_AUX_DIR_EXPAND], +[dnl Rely on autoconf to set up CDPATH properly. +AC_PREREQ([2.50])dnl +# expand $ac_aux_dir to an absolute path +am_aux_dir=`cd $ac_aux_dir && pwd` +]) + +# AM_CONDITIONAL -*- Autoconf -*- + +# Copyright (C) 1997, 2000, 2001, 2003, 2004, 2005, 2006, 2008 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 9 + +# AM_CONDITIONAL(NAME, SHELL-CONDITION) +# ------------------------------------- +# Define a conditional. +AC_DEFUN([AM_CONDITIONAL], +[AC_PREREQ(2.52)dnl + ifelse([$1], [TRUE], [AC_FATAL([$0: invalid condition: $1])], + [$1], [FALSE], [AC_FATAL([$0: invalid condition: $1])])dnl +AC_SUBST([$1_TRUE])dnl +AC_SUBST([$1_FALSE])dnl +_AM_SUBST_NOTMAKE([$1_TRUE])dnl +_AM_SUBST_NOTMAKE([$1_FALSE])dnl +m4_define([_AM_COND_VALUE_$1], [$2])dnl +if $2; then + $1_TRUE= + $1_FALSE='#' +else + $1_TRUE='#' + $1_FALSE= +fi +AC_CONFIG_COMMANDS_PRE( +[if test -z "${$1_TRUE}" && test -z "${$1_FALSE}"; then + AC_MSG_ERROR([[conditional "$1" was never defined. +Usually this means the macro was only invoked conditionally.]]) +fi])]) + +# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2009, +# 2010, 2011 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 12 + +# There are a few dirty hacks below to avoid letting `AC_PROG_CC' be +# written in clear, in which case automake, when reading aclocal.m4, +# will think it sees a *use*, and therefore will trigger all it's +# C support machinery. Also note that it means that autoscan, seeing +# CC etc. in the Makefile, will ask for an AC_PROG_CC use... + + +# _AM_DEPENDENCIES(NAME) +# ---------------------- +# See how the compiler implements dependency checking. +# NAME is "CC", "CXX", "GCJ", or "OBJC". +# We try a few techniques and use that to set a single cache variable. +# +# We don't AC_REQUIRE the corresponding AC_PROG_CC since the latter was +# modified to invoke _AM_DEPENDENCIES(CC); we would have a circular +# dependency, and given that the user is not expected to run this macro, +# just rely on AC_PROG_CC. +AC_DEFUN([_AM_DEPENDENCIES], +[AC_REQUIRE([AM_SET_DEPDIR])dnl +AC_REQUIRE([AM_OUTPUT_DEPENDENCY_COMMANDS])dnl +AC_REQUIRE([AM_MAKE_INCLUDE])dnl +AC_REQUIRE([AM_DEP_TRACK])dnl + +ifelse([$1], CC, [depcc="$CC" am_compiler_list=], + [$1], CXX, [depcc="$CXX" am_compiler_list=], + [$1], OBJC, [depcc="$OBJC" am_compiler_list='gcc3 gcc'], + [$1], UPC, [depcc="$UPC" am_compiler_list=], + [$1], GCJ, [depcc="$GCJ" am_compiler_list='gcc3 gcc'], + [depcc="$$1" am_compiler_list=]) + +AC_CACHE_CHECK([dependency style of $depcc], + [am_cv_$1_dependencies_compiler_type], +[if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + rm -rf conftest.dir + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_$1_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n ['s/^#*\([a-zA-Z0-9]*\))$/\1/p'] < ./depcomp` + fi + am__universal=false + m4_case([$1], [CC], + [case " $depcc " in #( + *\ -arch\ *\ -arch\ *) am__universal=true ;; + esac], + [CXX], + [case " $depcc " in #( + *\ -arch\ *\ -arch\ *) am__universal=true ;; + esac]) + + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. Also, some Intel + # versions had trouble with output in subdirs + am__obj=sub/conftest.${OBJEXT-o} + am__minus_obj="-o $am__obj" + case $depmode in + gcc) + # This depmode causes a compiler race in universal mode. + test "$am__universal" = false || continue + ;; + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + msvc7 | msvc7msys | msvisualcpp | msvcmsys) + # This compiler won't grok `-c -o', but also, the minuso test has + # not run yet. These depmodes are late enough in the game, and + # so weak that their functioning should not be impacted. + am__obj=conftest.${OBJEXT-o} + am__minus_obj= + ;; + none) break ;; + esac + if depmode=$depmode \ + source=sub/conftest.c object=$am__obj \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep $am__obj sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_$1_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_$1_dependencies_compiler_type=none +fi +]) +AC_SUBST([$1DEPMODE], [depmode=$am_cv_$1_dependencies_compiler_type]) +AM_CONDITIONAL([am__fastdep$1], [ + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_$1_dependencies_compiler_type" = gcc3]) +]) + + +# AM_SET_DEPDIR +# ------------- +# Choose a directory name for dependency files. +# This macro is AC_REQUIREd in _AM_DEPENDENCIES +AC_DEFUN([AM_SET_DEPDIR], +[AC_REQUIRE([AM_SET_LEADING_DOT])dnl +AC_SUBST([DEPDIR], ["${am__leading_dot}deps"])dnl +]) + + +# AM_DEP_TRACK +# ------------ +AC_DEFUN([AM_DEP_TRACK], +[AC_ARG_ENABLE(dependency-tracking, +[ --disable-dependency-tracking speeds up one-time build + --enable-dependency-tracking do not reject slow dependency extractors]) +if test "x$enable_dependency_tracking" != xno; then + am_depcomp="$ac_aux_dir/depcomp" + AMDEPBACKSLASH='\' + am__nodep='_no' +fi +AM_CONDITIONAL([AMDEP], [test "x$enable_dependency_tracking" != xno]) +AC_SUBST([AMDEPBACKSLASH])dnl +_AM_SUBST_NOTMAKE([AMDEPBACKSLASH])dnl +AC_SUBST([am__nodep])dnl +_AM_SUBST_NOTMAKE([am__nodep])dnl +]) + +# Generate code to set up dependency tracking. -*- Autoconf -*- + +# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2008 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +#serial 5 + +# _AM_OUTPUT_DEPENDENCY_COMMANDS +# ------------------------------ +AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS], +[{ + # Autoconf 2.62 quotes --file arguments for eval, but not when files + # are listed without --file. Let's play safe and only enable the eval + # if we detect the quoting. + case $CONFIG_FILES in + *\'*) eval set x "$CONFIG_FILES" ;; + *) set x $CONFIG_FILES ;; + esac + shift + for mf + do + # Strip MF so we end up with the name of the file. + mf=`echo "$mf" | sed -e 's/:.*$//'` + # Check whether this is an Automake generated Makefile or not. + # We used to match only the files named `Makefile.in', but + # some people rename them; so instead we look at the file content. + # Grep'ing the first line is not enough: some people post-process + # each Makefile.in and add a new line on top of each file to say so. + # Grep'ing the whole file is not good either: AIX grep has a line + # limit of 2048, but all sed's we know have understand at least 4000. + if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then + dirpart=`AS_DIRNAME("$mf")` + else + continue + fi + # Extract the definition of DEPDIR, am__include, and am__quote + # from the Makefile without running `make'. + DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` + test -z "$DEPDIR" && continue + am__include=`sed -n 's/^am__include = //p' < "$mf"` + test -z "am__include" && continue + am__quote=`sed -n 's/^am__quote = //p' < "$mf"` + # When using ansi2knr, U may be empty or an underscore; expand it + U=`sed -n 's/^U = //p' < "$mf"` + # Find all dependency output files, they are included files with + # $(DEPDIR) in their names. We invoke sed twice because it is the + # simplest approach to changing $(DEPDIR) to its actual value in the + # expansion. + for file in `sed -n " + s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ + sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do + # Make sure the directory exists. + test -f "$dirpart/$file" && continue + fdir=`AS_DIRNAME(["$file"])` + AS_MKDIR_P([$dirpart/$fdir]) + # echo "creating $dirpart/$file" + echo '# dummy' > "$dirpart/$file" + done + done +} +])# _AM_OUTPUT_DEPENDENCY_COMMANDS + + +# AM_OUTPUT_DEPENDENCY_COMMANDS +# ----------------------------- +# This macro should only be invoked once -- use via AC_REQUIRE. +# +# This code is only required when automatic dependency tracking +# is enabled. FIXME. This creates each `.P' file that we will +# need in order to bootstrap the dependency handling code. +AC_DEFUN([AM_OUTPUT_DEPENDENCY_COMMANDS], +[AC_CONFIG_COMMANDS([depfiles], + [test x"$AMDEP_TRUE" != x"" || _AM_OUTPUT_DEPENDENCY_COMMANDS], + [AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"]) +]) + +# Do all the work for Automake. -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, +# 2005, 2006, 2008, 2009 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 16 + +# This macro actually does too much. Some checks are only needed if +# your package does certain things. But this isn't really a big deal. + +# AM_INIT_AUTOMAKE(PACKAGE, VERSION, [NO-DEFINE]) +# AM_INIT_AUTOMAKE([OPTIONS]) +# ----------------------------------------------- +# The call with PACKAGE and VERSION arguments is the old style +# call (pre autoconf-2.50), which is being phased out. PACKAGE +# and VERSION should now be passed to AC_INIT and removed from +# the call to AM_INIT_AUTOMAKE. +# We support both call styles for the transition. After +# the next Automake release, Autoconf can make the AC_INIT +# arguments mandatory, and then we can depend on a new Autoconf +# release and drop the old call support. +AC_DEFUN([AM_INIT_AUTOMAKE], +[AC_PREREQ([2.62])dnl +dnl Autoconf wants to disallow AM_ names. We explicitly allow +dnl the ones we care about. +m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl +AC_REQUIRE([AM_SET_CURRENT_AUTOMAKE_VERSION])dnl +AC_REQUIRE([AC_PROG_INSTALL])dnl +if test "`cd $srcdir && pwd`" != "`pwd`"; then + # Use -I$(srcdir) only when $(srcdir) != ., so that make's output + # is not polluted with repeated "-I." + AC_SUBST([am__isrc], [' -I$(srcdir)'])_AM_SUBST_NOTMAKE([am__isrc])dnl + # test to see if srcdir already configured + if test -f $srcdir/config.status; then + AC_MSG_ERROR([source directory already configured; run "make distclean" there first]) + fi +fi + +# test whether we have cygpath +if test -z "$CYGPATH_W"; then + if (cygpath --version) >/dev/null 2>/dev/null; then + CYGPATH_W='cygpath -w' + else + CYGPATH_W=echo + fi +fi +AC_SUBST([CYGPATH_W]) + +# Define the identity of the package. +dnl Distinguish between old-style and new-style calls. +m4_ifval([$2], +[m4_ifval([$3], [_AM_SET_OPTION([no-define])])dnl + AC_SUBST([PACKAGE], [$1])dnl + AC_SUBST([VERSION], [$2])], +[_AM_SET_OPTIONS([$1])dnl +dnl Diagnose old-style AC_INIT with new-style AM_AUTOMAKE_INIT. +m4_if(m4_ifdef([AC_PACKAGE_NAME], 1)m4_ifdef([AC_PACKAGE_VERSION], 1), 11,, + [m4_fatal([AC_INIT should be called with package and version arguments])])dnl + AC_SUBST([PACKAGE], ['AC_PACKAGE_TARNAME'])dnl + AC_SUBST([VERSION], ['AC_PACKAGE_VERSION'])])dnl + +_AM_IF_OPTION([no-define],, +[AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE", [Name of package]) + AC_DEFINE_UNQUOTED(VERSION, "$VERSION", [Version number of package])])dnl + +# Some tools Automake needs. +AC_REQUIRE([AM_SANITY_CHECK])dnl +AC_REQUIRE([AC_ARG_PROGRAM])dnl +AM_MISSING_PROG(ACLOCAL, aclocal-${am__api_version}) +AM_MISSING_PROG(AUTOCONF, autoconf) +AM_MISSING_PROG(AUTOMAKE, automake-${am__api_version}) +AM_MISSING_PROG(AUTOHEADER, autoheader) +AM_MISSING_PROG(MAKEINFO, makeinfo) +AC_REQUIRE([AM_PROG_INSTALL_SH])dnl +AC_REQUIRE([AM_PROG_INSTALL_STRIP])dnl +AC_REQUIRE([AM_PROG_MKDIR_P])dnl +# We need awk for the "check" target. The system "awk" is bad on +# some platforms. +AC_REQUIRE([AC_PROG_AWK])dnl +AC_REQUIRE([AC_PROG_MAKE_SET])dnl +AC_REQUIRE([AM_SET_LEADING_DOT])dnl +_AM_IF_OPTION([tar-ustar], [_AM_PROG_TAR([ustar])], + [_AM_IF_OPTION([tar-pax], [_AM_PROG_TAR([pax])], + [_AM_PROG_TAR([v7])])]) +_AM_IF_OPTION([no-dependencies],, +[AC_PROVIDE_IFELSE([AC_PROG_CC], + [_AM_DEPENDENCIES(CC)], + [define([AC_PROG_CC], + defn([AC_PROG_CC])[_AM_DEPENDENCIES(CC)])])dnl +AC_PROVIDE_IFELSE([AC_PROG_CXX], + [_AM_DEPENDENCIES(CXX)], + [define([AC_PROG_CXX], + defn([AC_PROG_CXX])[_AM_DEPENDENCIES(CXX)])])dnl +AC_PROVIDE_IFELSE([AC_PROG_OBJC], + [_AM_DEPENDENCIES(OBJC)], + [define([AC_PROG_OBJC], + defn([AC_PROG_OBJC])[_AM_DEPENDENCIES(OBJC)])])dnl +]) +_AM_IF_OPTION([silent-rules], [AC_REQUIRE([AM_SILENT_RULES])])dnl +dnl The `parallel-tests' driver may need to know about EXEEXT, so add the +dnl `am__EXEEXT' conditional if _AM_COMPILER_EXEEXT was seen. This macro +dnl is hooked onto _AC_COMPILER_EXEEXT early, see below. +AC_CONFIG_COMMANDS_PRE(dnl +[m4_provide_if([_AM_COMPILER_EXEEXT], + [AM_CONDITIONAL([am__EXEEXT], [test -n "$EXEEXT"])])])dnl +]) + +dnl Hook into `_AC_COMPILER_EXEEXT' early to learn its expansion. Do not +dnl add the conditional right here, as _AC_COMPILER_EXEEXT may be further +dnl mangled by Autoconf and run in a shell conditional statement. +m4_define([_AC_COMPILER_EXEEXT], +m4_defn([_AC_COMPILER_EXEEXT])[m4_provide([_AM_COMPILER_EXEEXT])]) + + +# When config.status generates a header, we must update the stamp-h file. +# This file resides in the same directory as the config header +# that is generated. The stamp files are numbered to have different names. + +# Autoconf calls _AC_AM_CONFIG_HEADER_HOOK (when defined) in the +# loop where config.status creates the headers, so we can generate +# our stamp files there. +AC_DEFUN([_AC_AM_CONFIG_HEADER_HOOK], +[# Compute $1's index in $config_headers. +_am_arg=$1 +_am_stamp_count=1 +for _am_header in $config_headers :; do + case $_am_header in + $_am_arg | $_am_arg:* ) + break ;; + * ) + _am_stamp_count=`expr $_am_stamp_count + 1` ;; + esac +done +echo "timestamp for $_am_arg" >`AS_DIRNAME(["$_am_arg"])`/stamp-h[]$_am_stamp_count]) + +# Copyright (C) 2001, 2003, 2005, 2008, 2011 Free Software Foundation, +# Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 1 + +# AM_PROG_INSTALL_SH +# ------------------ +# Define $install_sh. +AC_DEFUN([AM_PROG_INSTALL_SH], +[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +if test x"${install_sh}" != xset; then + case $am_aux_dir in + *\ * | *\ *) + install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;; + *) + install_sh="\${SHELL} $am_aux_dir/install-sh" + esac +fi +AC_SUBST(install_sh)]) + +# Copyright (C) 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 2 + +# Check whether the underlying file-system supports filenames +# with a leading dot. For instance MS-DOS doesn't. +AC_DEFUN([AM_SET_LEADING_DOT], +[rm -rf .tst 2>/dev/null +mkdir .tst 2>/dev/null +if test -d .tst; then + am__leading_dot=. +else + am__leading_dot=_ +fi +rmdir .tst 2>/dev/null +AC_SUBST([am__leading_dot])]) + +# Check to see how 'make' treats includes. -*- Autoconf -*- + +# Copyright (C) 2001, 2002, 2003, 2005, 2009 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 4 + +# AM_MAKE_INCLUDE() +# ----------------- +# Check to see how make treats includes. +AC_DEFUN([AM_MAKE_INCLUDE], +[am_make=${MAKE-make} +cat > confinc << 'END' +am__doit: + @echo this is the am__doit target +.PHONY: am__doit +END +# If we don't find an include directive, just comment out the code. +AC_MSG_CHECKING([for style of include used by $am_make]) +am__include="#" +am__quote= +_am_result=none +# First try GNU make style include. +echo "include confinc" > confmf +# Ignore all kinds of additional output from `make'. +case `$am_make -s -f confmf 2> /dev/null` in #( +*the\ am__doit\ target*) + am__include=include + am__quote= + _am_result=GNU + ;; +esac +# Now try BSD make style include. +if test "$am__include" = "#"; then + echo '.include "confinc"' > confmf + case `$am_make -s -f confmf 2> /dev/null` in #( + *the\ am__doit\ target*) + am__include=.include + am__quote="\"" + _am_result=BSD + ;; + esac +fi +AC_SUBST([am__include]) +AC_SUBST([am__quote]) +AC_MSG_RESULT([$_am_result]) +rm -f confinc confmf +]) + +# Copyright (C) 1999, 2000, 2001, 2003, 2004, 2005, 2008 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 6 + +# AM_PROG_CC_C_O +# -------------- +# Like AC_PROG_CC_C_O, but changed for automake. +AC_DEFUN([AM_PROG_CC_C_O], +[AC_REQUIRE([AC_PROG_CC_C_O])dnl +AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +AC_REQUIRE_AUX_FILE([compile])dnl +# FIXME: we rely on the cache variable name because +# there is no other way. +set dummy $CC +am_cc=`echo $[2] | sed ['s/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/']` +eval am_t=\$ac_cv_prog_cc_${am_cc}_c_o +if test "$am_t" != yes; then + # Losing compiler, so override with the script. + # FIXME: It is wrong to rewrite CC. + # But if we don't then we get into trouble of one sort or another. + # A longer-term fix would be to have automake use am__CC in this case, + # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" + CC="$am_aux_dir/compile $CC" +fi +dnl Make sure AC_PROG_CC is never called again, or it will override our +dnl setting of CC. +m4_define([AC_PROG_CC], + [m4_fatal([AC_PROG_CC cannot be called after AM_PROG_CC_C_O])]) +]) + +# Fake the existence of programs that GNU maintainers use. -*- Autoconf -*- + +# Copyright (C) 1997, 1999, 2000, 2001, 2003, 2004, 2005, 2008 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 6 + +# AM_MISSING_PROG(NAME, PROGRAM) +# ------------------------------ +AC_DEFUN([AM_MISSING_PROG], +[AC_REQUIRE([AM_MISSING_HAS_RUN]) +$1=${$1-"${am_missing_run}$2"} +AC_SUBST($1)]) + + +# AM_MISSING_HAS_RUN +# ------------------ +# Define MISSING if not defined so far and test if it supports --run. +# If it does, set am_missing_run to use it, otherwise, to nothing. +AC_DEFUN([AM_MISSING_HAS_RUN], +[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +AC_REQUIRE_AUX_FILE([missing])dnl +if test x"${MISSING+set}" != xset; then + case $am_aux_dir in + *\ * | *\ *) + MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;; + *) + MISSING="\${SHELL} $am_aux_dir/missing" ;; + esac +fi +# Use eval to expand $SHELL +if eval "$MISSING --run true"; then + am_missing_run="$MISSING --run " +else + am_missing_run= + AC_MSG_WARN([`missing' script is too old or missing]) +fi +]) + +# Copyright (C) 2003, 2004, 2005, 2006, 2011 Free Software Foundation, +# Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 1 + +# AM_PROG_MKDIR_P +# --------------- +# Check for `mkdir -p'. +AC_DEFUN([AM_PROG_MKDIR_P], +[AC_PREREQ([2.60])dnl +AC_REQUIRE([AC_PROG_MKDIR_P])dnl +dnl Automake 1.8 to 1.9.6 used to define mkdir_p. We now use MKDIR_P, +dnl while keeping a definition of mkdir_p for backward compatibility. +dnl @MKDIR_P@ is magic: AC_OUTPUT adjusts its value for each Makefile. +dnl However we cannot define mkdir_p as $(MKDIR_P) for the sake of +dnl Makefile.ins that do not define MKDIR_P, so we do our own +dnl adjustment using top_builddir (which is defined more often than +dnl MKDIR_P). +AC_SUBST([mkdir_p], ["$MKDIR_P"])dnl +case $mkdir_p in + [[\\/$]]* | ?:[[\\/]]*) ;; + */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;; +esac +]) + +# Helper functions for option handling. -*- Autoconf -*- + +# Copyright (C) 2001, 2002, 2003, 2005, 2008, 2010 Free Software +# Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 5 + +# _AM_MANGLE_OPTION(NAME) +# ----------------------- +AC_DEFUN([_AM_MANGLE_OPTION], +[[_AM_OPTION_]m4_bpatsubst($1, [[^a-zA-Z0-9_]], [_])]) + +# _AM_SET_OPTION(NAME) +# -------------------- +# Set option NAME. Presently that only means defining a flag for this option. +AC_DEFUN([_AM_SET_OPTION], +[m4_define(_AM_MANGLE_OPTION([$1]), 1)]) + +# _AM_SET_OPTIONS(OPTIONS) +# ------------------------ +# OPTIONS is a space-separated list of Automake options. +AC_DEFUN([_AM_SET_OPTIONS], +[m4_foreach_w([_AM_Option], [$1], [_AM_SET_OPTION(_AM_Option)])]) + +# _AM_IF_OPTION(OPTION, IF-SET, [IF-NOT-SET]) +# ------------------------------------------- +# Execute IF-SET if OPTION is set, IF-NOT-SET otherwise. +AC_DEFUN([_AM_IF_OPTION], +[m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])]) + +# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2008, 2009, +# 2011 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 2 + +# AM_PATH_PYTHON([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +# --------------------------------------------------------------------------- +# Adds support for distributing Python modules and packages. To +# install modules, copy them to $(pythondir), using the python_PYTHON +# automake variable. To install a package with the same name as the +# automake package, install to $(pkgpythondir), or use the +# pkgpython_PYTHON automake variable. +# +# The variables $(pyexecdir) and $(pkgpyexecdir) are provided as +# locations to install python extension modules (shared libraries). +# Another macro is required to find the appropriate flags to compile +# extension modules. +# +# If your package is configured with a different prefix to python, +# users will have to add the install directory to the PYTHONPATH +# environment variable, or create a .pth file (see the python +# documentation for details). +# +# If the MINIMUM-VERSION argument is passed, AM_PATH_PYTHON will +# cause an error if the version of python installed on the system +# doesn't meet the requirement. MINIMUM-VERSION should consist of +# numbers and dots only. +AC_DEFUN([AM_PATH_PYTHON], + [ + dnl Find a Python interpreter. Python versions prior to 2.0 are not + dnl supported. (2.0 was released on October 16, 2000). + m4_define_default([_AM_PYTHON_INTERPRETER_LIST], +[python python2 python3 python3.2 python3.1 python3.0 python2.7 dnl + python2.6 python2.5 python2.4 python2.3 python2.2 python2.1 python2.0]) + + AC_ARG_VAR([PYTHON], [the Python interpreter]) + + m4_if([$1],[],[ + dnl No version check is needed. + # Find any Python interpreter. + if test -z "$PYTHON"; then + AC_PATH_PROGS([PYTHON], _AM_PYTHON_INTERPRETER_LIST, :) + fi + am_display_PYTHON=python + ], [ + dnl A version check is needed. + if test -n "$PYTHON"; then + # If the user set $PYTHON, use it and don't search something else. + AC_MSG_CHECKING([whether $PYTHON version >= $1]) + AM_PYTHON_CHECK_VERSION([$PYTHON], [$1], + [AC_MSG_RESULT(yes)], + [AC_MSG_ERROR(too old)]) + am_display_PYTHON=$PYTHON + else + # Otherwise, try each interpreter until we find one that satisfies + # VERSION. + AC_CACHE_CHECK([for a Python interpreter with version >= $1], + [am_cv_pathless_PYTHON],[ + for am_cv_pathless_PYTHON in _AM_PYTHON_INTERPRETER_LIST none; do + test "$am_cv_pathless_PYTHON" = none && break + AM_PYTHON_CHECK_VERSION([$am_cv_pathless_PYTHON], [$1], [break]) + done]) + # Set $PYTHON to the absolute path of $am_cv_pathless_PYTHON. + if test "$am_cv_pathless_PYTHON" = none; then + PYTHON=: + else + AC_PATH_PROG([PYTHON], [$am_cv_pathless_PYTHON]) + fi + am_display_PYTHON=$am_cv_pathless_PYTHON + fi + ]) + + if test "$PYTHON" = :; then + dnl Run any user-specified action, or abort. + m4_default([$3], [AC_MSG_ERROR([no suitable Python interpreter found])]) + else + + dnl Query Python for its version number. Getting [:3] seems to be + dnl the best way to do this; it's what "site.py" does in the standard + dnl library. + + AC_CACHE_CHECK([for $am_display_PYTHON version], [am_cv_python_version], + [am_cv_python_version=`$PYTHON -c "import sys; sys.stdout.write(sys.version[[:3]])"`]) + AC_SUBST([PYTHON_VERSION], [$am_cv_python_version]) + + dnl Use the values of $prefix and $exec_prefix for the corresponding + dnl values of PYTHON_PREFIX and PYTHON_EXEC_PREFIX. These are made + dnl distinct variables so they can be overridden if need be. However, + dnl general consensus is that you shouldn't need this ability. + + AC_SUBST([PYTHON_PREFIX], ['${prefix}']) + AC_SUBST([PYTHON_EXEC_PREFIX], ['${exec_prefix}']) + + dnl At times (like when building shared libraries) you may want + dnl to know which OS platform Python thinks this is. + + AC_CACHE_CHECK([for $am_display_PYTHON platform], [am_cv_python_platform], + [am_cv_python_platform=`$PYTHON -c "import sys; sys.stdout.write(sys.platform)"`]) + AC_SUBST([PYTHON_PLATFORM], [$am_cv_python_platform]) + + + dnl Set up 4 directories: + + dnl pythondir -- where to install python scripts. This is the + dnl site-packages directory, not the python standard library + dnl directory like in previous automake betas. This behavior + dnl is more consistent with lispdir.m4 for example. + dnl Query distutils for this directory. + AC_CACHE_CHECK([for $am_display_PYTHON script directory], + [am_cv_python_pythondir], + [if test "x$prefix" = xNONE + then + am_py_prefix=$ac_default_prefix + else + am_py_prefix=$prefix + fi + am_cv_python_pythondir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(0,0,prefix='$am_py_prefix'))" 2>/dev/null` + case $am_cv_python_pythondir in + $am_py_prefix*) + am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'` + am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,$PYTHON_PREFIX,"` + ;; + *) + case $am_py_prefix in + /usr|/System*) ;; + *) + am_cv_python_pythondir=$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages + ;; + esac + ;; + esac + ]) + AC_SUBST([pythondir], [$am_cv_python_pythondir]) + + dnl pkgpythondir -- $PACKAGE directory under pythondir. Was + dnl PYTHON_SITE_PACKAGE in previous betas, but this naming is + dnl more consistent with the rest of automake. + + AC_SUBST([pkgpythondir], [\${pythondir}/$PACKAGE]) + + dnl pyexecdir -- directory for installing python extension modules + dnl (shared libraries) + dnl Query distutils for this directory. + AC_CACHE_CHECK([for $am_display_PYTHON extension module directory], + [am_cv_python_pyexecdir], + [if test "x$exec_prefix" = xNONE + then + am_py_exec_prefix=$am_py_prefix + else + am_py_exec_prefix=$exec_prefix + fi + am_cv_python_pyexecdir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(1,0,prefix='$am_py_exec_prefix'))" 2>/dev/null` + case $am_cv_python_pyexecdir in + $am_py_exec_prefix*) + am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'` + am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,$PYTHON_EXEC_PREFIX,"` + ;; + *) + case $am_py_exec_prefix in + /usr|/System*) ;; + *) + am_cv_python_pyexecdir=$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages + ;; + esac + ;; + esac + ]) + AC_SUBST([pyexecdir], [$am_cv_python_pyexecdir]) + + dnl pkgpyexecdir -- $(pyexecdir)/$(PACKAGE) + + AC_SUBST([pkgpyexecdir], [\${pyexecdir}/$PACKAGE]) + + dnl Run any user-specified action. + $2 + fi + +]) + + +# AM_PYTHON_CHECK_VERSION(PROG, VERSION, [ACTION-IF-TRUE], [ACTION-IF-FALSE]) +# --------------------------------------------------------------------------- +# Run ACTION-IF-TRUE if the Python interpreter PROG has version >= VERSION. +# Run ACTION-IF-FALSE otherwise. +# This test uses sys.hexversion instead of the string equivalent (first +# word of sys.version), in order to cope with versions such as 2.2c1. +# This supports Python 2.0 or higher. (2.0 was released on October 16, 2000). +AC_DEFUN([AM_PYTHON_CHECK_VERSION], + [prog="import sys +# split strings by '.' and convert to numeric. Append some zeros +# because we need at least 4 digits for the hex conversion. +# map returns an iterator in Python 3.0 and a list in 2.x +minver = list(map(int, '$2'.split('.'))) + [[0, 0, 0]] +minverhex = 0 +# xrange is not present in Python 3.0 and range returns an iterator +for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[[i]] +sys.exit(sys.hexversion < minverhex)" + AS_IF([AM_RUN_LOG([$1 -c "$prog"])], [$3], [$4])]) + +# Copyright (C) 2001, 2003, 2005, 2011 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 1 + +# AM_RUN_LOG(COMMAND) +# ------------------- +# Run COMMAND, save the exit status in ac_status, and log it. +# (This has been adapted from Autoconf's _AC_RUN_LOG macro.) +AC_DEFUN([AM_RUN_LOG], +[{ echo "$as_me:$LINENO: $1" >&AS_MESSAGE_LOG_FD + ($1) >&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD + (exit $ac_status); }]) + +# Check to make sure that the build environment is sane. -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 2000, 2001, 2003, 2005, 2008 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 5 + +# AM_SANITY_CHECK +# --------------- +AC_DEFUN([AM_SANITY_CHECK], +[AC_MSG_CHECKING([whether build environment is sane]) +# Just in case +sleep 1 +echo timestamp > conftest.file +# Reject unsafe characters in $srcdir or the absolute working directory +# name. Accept space and tab only in the latter. +am_lf=' +' +case `pwd` in + *[[\\\"\#\$\&\'\`$am_lf]]*) + AC_MSG_ERROR([unsafe absolute working directory name]);; +esac +case $srcdir in + *[[\\\"\#\$\&\'\`$am_lf\ \ ]]*) + AC_MSG_ERROR([unsafe srcdir value: `$srcdir']);; +esac + +# Do `set' in a subshell so we don't clobber the current shell's +# arguments. Must try -L first in case configure is actually a +# symlink; some systems play weird games with the mod time of symlinks +# (eg FreeBSD returns the mod time of the symlink's containing +# directory). +if ( + set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null` + if test "$[*]" = "X"; then + # -L didn't work. + set X `ls -t "$srcdir/configure" conftest.file` + fi + rm -f conftest.file + if test "$[*]" != "X $srcdir/configure conftest.file" \ + && test "$[*]" != "X conftest.file $srcdir/configure"; then + + # If neither matched, then we have a broken ls. This can happen + # if, for instance, CONFIG_SHELL is bash and it inherits a + # broken ls alias from the environment. This has actually + # happened. Such a system could not be considered "sane". + AC_MSG_ERROR([ls -t appears to fail. Make sure there is not a broken +alias in your environment]) + fi + + test "$[2]" = conftest.file + ) +then + # Ok. + : +else + AC_MSG_ERROR([newly created file is older than distributed files! +Check your system clock]) +fi +AC_MSG_RESULT(yes)]) + +# Copyright (C) 2009, 2011 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 2 + +# AM_SILENT_RULES([DEFAULT]) +# -------------------------- +# Enable less verbose build rules; with the default set to DEFAULT +# (`yes' being less verbose, `no' or empty being verbose). +AC_DEFUN([AM_SILENT_RULES], +[AC_ARG_ENABLE([silent-rules], +[ --enable-silent-rules less verbose build output (undo: `make V=1') + --disable-silent-rules verbose build output (undo: `make V=0')]) +case $enable_silent_rules in +yes) AM_DEFAULT_VERBOSITY=0;; +no) AM_DEFAULT_VERBOSITY=1;; +*) AM_DEFAULT_VERBOSITY=m4_if([$1], [yes], [0], [1]);; +esac +dnl +dnl A few `make' implementations (e.g., NonStop OS and NextStep) +dnl do not support nested variable expansions. +dnl See automake bug#9928 and bug#10237. +am_make=${MAKE-make} +AC_CACHE_CHECK([whether $am_make supports nested variables], + [am_cv_make_support_nested_variables], + [if AS_ECHO([['TRUE=$(BAR$(V)) +BAR0=false +BAR1=true +V=1 +am__doit: + @$(TRUE) +.PHONY: am__doit']]) | $am_make -f - >/dev/null 2>&1; then + am_cv_make_support_nested_variables=yes +else + am_cv_make_support_nested_variables=no +fi]) +if test $am_cv_make_support_nested_variables = yes; then + dnl Using `$V' instead of `$(V)' breaks IRIX make. + AM_V='$(V)' + AM_DEFAULT_V='$(AM_DEFAULT_VERBOSITY)' +else + AM_V=$AM_DEFAULT_VERBOSITY + AM_DEFAULT_V=$AM_DEFAULT_VERBOSITY +fi +AC_SUBST([AM_V])dnl +AM_SUBST_NOTMAKE([AM_V])dnl +AC_SUBST([AM_DEFAULT_V])dnl +AM_SUBST_NOTMAKE([AM_DEFAULT_V])dnl +AC_SUBST([AM_DEFAULT_VERBOSITY])dnl +AM_BACKSLASH='\' +AC_SUBST([AM_BACKSLASH])dnl +_AM_SUBST_NOTMAKE([AM_BACKSLASH])dnl +]) + +# Copyright (C) 2001, 2003, 2005, 2011 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 1 + +# AM_PROG_INSTALL_STRIP +# --------------------- +# One issue with vendor `install' (even GNU) is that you can't +# specify the program used to strip binaries. This is especially +# annoying in cross-compiling environments, where the build's strip +# is unlikely to handle the host's binaries. +# Fortunately install-sh will honor a STRIPPROG variable, so we +# always use install-sh in `make install-strip', and initialize +# STRIPPROG with the value of the STRIP variable (set by the user). +AC_DEFUN([AM_PROG_INSTALL_STRIP], +[AC_REQUIRE([AM_PROG_INSTALL_SH])dnl +# Installed binaries are usually stripped using `strip' when the user +# run `make install-strip'. However `strip' might not be the right +# tool to use in cross-compilation environments, therefore Automake +# will honor the `STRIP' environment variable to overrule this program. +dnl Don't test for $cross_compiling = yes, because it might be `maybe'. +if test "$cross_compiling" != no; then + AC_CHECK_TOOL([STRIP], [strip], :) +fi +INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s" +AC_SUBST([INSTALL_STRIP_PROGRAM])]) + +# Copyright (C) 2006, 2008, 2010 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 3 + +# _AM_SUBST_NOTMAKE(VARIABLE) +# --------------------------- +# Prevent Automake from outputting VARIABLE = @VARIABLE@ in Makefile.in. +# This macro is traced by Automake. +AC_DEFUN([_AM_SUBST_NOTMAKE]) + +# AM_SUBST_NOTMAKE(VARIABLE) +# -------------------------- +# Public sister of _AM_SUBST_NOTMAKE. +AC_DEFUN([AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE($@)]) + +# Check how to create a tarball. -*- Autoconf -*- + +# Copyright (C) 2004, 2005, 2012 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 2 + +# _AM_PROG_TAR(FORMAT) +# -------------------- +# Check how to create a tarball in format FORMAT. +# FORMAT should be one of `v7', `ustar', or `pax'. +# +# Substitute a variable $(am__tar) that is a command +# writing to stdout a FORMAT-tarball containing the directory +# $tardir. +# tardir=directory && $(am__tar) > result.tar +# +# Substitute a variable $(am__untar) that extract such +# a tarball read from stdin. +# $(am__untar) < result.tar +AC_DEFUN([_AM_PROG_TAR], +[# Always define AMTAR for backward compatibility. Yes, it's still used +# in the wild :-( We should find a proper way to deprecate it ... +AC_SUBST([AMTAR], ['$${TAR-tar}']) +m4_if([$1], [v7], + [am__tar='$${TAR-tar} chof - "$$tardir"' am__untar='$${TAR-tar} xf -'], + [m4_case([$1], [ustar],, [pax],, + [m4_fatal([Unknown tar format])]) +AC_MSG_CHECKING([how to create a $1 tar archive]) +# Loop over all known methods to create a tar archive until one works. +_am_tools='gnutar m4_if([$1], [ustar], [plaintar]) pax cpio none' +_am_tools=${am_cv_prog_tar_$1-$_am_tools} +# Do not fold the above two line into one, because Tru64 sh and +# Solaris sh will not grok spaces in the rhs of `-'. +for _am_tool in $_am_tools +do + case $_am_tool in + gnutar) + for _am_tar in tar gnutar gtar; + do + AM_RUN_LOG([$_am_tar --version]) && break + done + am__tar="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$$tardir"' + am__tar_="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$tardir"' + am__untar="$_am_tar -xf -" + ;; + plaintar) + # Must skip GNU tar: if it does not support --format= it doesn't create + # ustar tarball either. + (tar --version) >/dev/null 2>&1 && continue + am__tar='tar chf - "$$tardir"' + am__tar_='tar chf - "$tardir"' + am__untar='tar xf -' + ;; + pax) + am__tar='pax -L -x $1 -w "$$tardir"' + am__tar_='pax -L -x $1 -w "$tardir"' + am__untar='pax -r' + ;; + cpio) + am__tar='find "$$tardir" -print | cpio -o -H $1 -L' + am__tar_='find "$tardir" -print | cpio -o -H $1 -L' + am__untar='cpio -i -H $1 -d' + ;; + none) + am__tar=false + am__tar_=false + am__untar=false + ;; + esac + + # If the value was cached, stop now. We just wanted to have am__tar + # and am__untar set. + test -n "${am_cv_prog_tar_$1}" && break + + # tar/untar a dummy directory, and stop if the command works + rm -rf conftest.dir + mkdir conftest.dir + echo GrepMe > conftest.dir/file + AM_RUN_LOG([tardir=conftest.dir && eval $am__tar_ >conftest.tar]) + rm -rf conftest.dir + if test -s conftest.tar; then + AM_RUN_LOG([$am__untar /dev/null 2>&1 && break + fi +done +rm -rf conftest.dir + +AC_CACHE_VAL([am_cv_prog_tar_$1], [am_cv_prog_tar_$1=$_am_tool]) +AC_MSG_RESULT([$am_cv_prog_tar_$1])]) +AC_SUBST([am__tar]) +AC_SUBST([am__untar]) +]) # _AM_PROG_TAR + +m4_include([m4/ac_check_classpath.m4]) +m4_include([m4/ac_prog_jar.m4]) +m4_include([m4/ac_prog_javac.m4]) +m4_include([m4/ac_prog_javac_works.m4]) +m4_include([m4/ac_prog_javah.m4]) +m4_include([m4/acx_pthread.m4]) +m4_include([m4/ax_c_pretty_func.m4]) +m4_include([m4/ax_c_var_func.m4]) +m4_include([m4/ax_check_compile_flag.m4]) +m4_include([m4/ax_cxx_static_cast.m4]) +m4_include([m4/ax_intel.m4]) +m4_include([m4/libtool.m4]) +m4_include([m4/ltoptions.m4]) +m4_include([m4/ltsugar.m4]) +m4_include([m4/ltversion.m4]) +m4_include([m4/lt~obsolete.m4]) +m4_include([m4/pkg.m4]) diff --git a/ceph/ar-lib b/ceph/ar-lib new file mode 100755 index 00000000..c0286a4c --- /dev/null +++ b/ceph/ar-lib @@ -0,0 +1,265 @@ +#! /bin/sh +# Wrapper for Microsoft lib.exe + +me=ar-lib +scriptversion=2012-01-30.22; # UTC + +# Copyright (C) 2010, 2012 Free Software Foundation, Inc. +# Written by Peter Rosin . +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + + +# func_error message +func_error () +{ + echo "$me: $1" 1>&2 + exit 1 +} + +file_conv= + +# func_file_conv build_file +# Convert a $build file to $host form and store it in $file +# Currently only supports Windows hosts. +func_file_conv () +{ + file=$1 + case $file in + / | /[!/]*) # absolute file, and not a UNC file + if test -z "$file_conv"; then + # lazily determine how to convert abs files + case `uname -s` in + MINGW*) + file_conv=mingw + ;; + CYGWIN*) + file_conv=cygwin + ;; + *) + file_conv=wine + ;; + esac + fi + case $file_conv in + mingw) + file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` + ;; + cygwin) + file=`cygpath -m "$file" || echo "$file"` + ;; + wine) + file=`winepath -w "$file" || echo "$file"` + ;; + esac + ;; + esac +} + +# func_at_file at_file operation archive +# Iterate over all members in AT_FILE performing OPERATION on ARCHIVE +# for each of them. +# When interpreting the content of the @FILE, do NOT use func_file_conv, +# since the user would need to supply preconverted file names to +# binutils ar, at least for MinGW. +func_at_file () +{ + operation=$2 + archive=$3 + at_file_contents=`cat "$1"` + eval set x "$at_file_contents" + shift + + for member + do + $AR -NOLOGO $operation:"$member" "$archive" || exit $? + done +} + +case $1 in + '') + func_error "no command. Try '$0 --help' for more information." + ;; + -h | --h*) + cat </dev/null && return + + echo + echo "Error: could not find pkg-config" + echo + echo "Please make sure you have pkg-config installed." + echo + exit 1 +} + +if [ `which libtoolize` ]; then + LIBTOOLIZE=libtoolize +elif [ `which glibtoolize` ]; then + LIBTOOLIZE=glibtoolize +else + echo "Error: could not find libtoolize" + echo " Please install libtoolize or glibtoolize." + exit 1 +fi + +rm -f config.cache +aclocal -I m4 --install +check_for_pkg_config +$LIBTOOLIZE --force --copy +aclocal -I m4 --install +autoconf +autoheader +automake -a --add-missing -Wall +( cd src/gtest && autoreconf -fvi; ) +exit diff --git a/ceph/ceph.spec b/ceph/ceph.spec new file mode 100644 index 00000000..1e9a2a62 --- /dev/null +++ b/ceph/ceph.spec @@ -0,0 +1,705 @@ +%bcond_with ocf + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +################################################################################# +# common +################################################################################# +Name: ceph +Version: 0.80.7 +Release: 0%{?dist} +Summary: User space components of the Ceph file system +License: GPL-2.0 +Group: System Environment/Base +URL: http://ceph.com/ +Source0: http://ceph.com/download/%{name}-%{version}.tar.bz2 +Requires: librbd1 = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +Requires: libcephfs1 = %{version}-%{release} +Requires: ceph-common = %{version}-%{release} +Requires: python +Requires: python-argparse +Requires: python-ceph +Requires: python-requests +Requires: xfsprogs +Requires: cryptsetup +Requires: parted +Requires: util-linux +Requires: hdparm +Requires(post): binutils +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: make +BuildRequires: gcc-c++ +BuildRequires: libtool +BuildRequires: boost-devel +BuildRequires: libedit-devel +BuildRequires: perl +BuildRequires: gdbm +BuildRequires: pkgconfig +BuildRequires: python +BuildRequires: python-nose +BuildRequires: python-argparse +BuildRequires: libaio-devel +BuildRequires: libcurl-devel +BuildRequires: libxml2-devel +BuildRequires: libuuid-devel +BuildRequires: libblkid-devel >= 2.17 +BuildRequires: leveldb-devel > 1.2 +BuildRequires: xfsprogs-devel +BuildRequires: yasm +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +BuildRequires: snappy-devel +%endif + +################################################################################# +# specific +################################################################################# +%if ! 0%{?rhel} +BuildRequires: sharutils +%endif + +%if 0%{defined suse_version} +%if 0%{?suse_version} > 1210 +Requires: gptfdisk +BuildRequires: gperftools-devel +%else +Requires: scsirastools +BuildRequires: google-perftools-devel +%endif +Recommends: logrotate +BuildRequires: %insserv_prereq +BuildRequires: mozilla-nss-devel +BuildRequires: keyutils-devel +BuildRequires: libatomic-ops-devel +BuildRequires: fdupes +%else +Requires: gdisk +BuildRequires: nss-devel +BuildRequires: keyutils-libs-devel +BuildRequires: libatomic_ops-devel +Requires: gdisk +Requires(post): chkconfig +Requires(preun):chkconfig +Requires(preun):initscripts +BuildRequires: gperftools-devel +%endif + +%description +Ceph is a massively scalable, open-source, distributed +storage system that runs on commodity hardware and delivers object, +block and file system storage. + + +################################################################################# +# packages +################################################################################# +%package -n ceph-common +Summary: Ceph Common +Group: System Environment/Base +Requires: librbd1 = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +Requires: python-ceph = %{version}-%{release} +Requires: python-requests +Requires: redhat-lsb-core +%description -n ceph-common +common utilities to mount and interact with a ceph storage cluster + +%package fuse +Summary: Ceph fuse-based client +Group: System Environment/Base +Requires: %{name} +BuildRequires: fuse-devel +%description fuse +FUSE based client for Ceph distributed network file system + +%package -n rbd-fuse +Summary: Ceph fuse-based client +Group: System Environment/Base +Requires: %{name} +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +BuildRequires: fuse-devel +%description -n rbd-fuse +FUSE based client to map Ceph rbd images to files + +%package devel +Summary: Ceph headers +Group: Development/Libraries +License: LGPL-2.0 +Requires: %{name} = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +Requires: libcephfs1 = %{version}-%{release} +Requires: libcephfs_jni1 = %{version}-%{release} +%description devel +This package contains libraries and headers needed to develop programs +that use Ceph. + +%package radosgw +Summary: Rados REST gateway +Group: Development/Libraries +Requires: ceph-common = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +%if 0%{defined suse_version} +BuildRequires: libexpat-devel +BuildRequires: FastCGI-devel +Requires: apache2-mod_fcgid +%else +BuildRequires: expat-devel +BuildRequires: fcgi-devel +%endif +%description radosgw +radosgw is an S3 HTTP REST gateway for the RADOS object store. It is +implemented as a FastCGI module using libfcgi, and can be used in +conjunction with any FastCGI capable web server. + +%if %{with ocf} +%package resource-agents +Summary: OCF-compliant resource agents for Ceph daemons +Group: System Environment/Base +License: LGPL-2.0 +Requires: %{name} = %{version} +Requires: resource-agents +%description resource-agents +Resource agents for monitoring and managing Ceph daemons +under Open Cluster Framework (OCF) compliant resource +managers such as Pacemaker. +%endif + +%package -n librados2 +Summary: RADOS distributed object store client library +Group: System Environment/Libraries +License: LGPL-2.0 +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +Obsoletes: ceph-libs +%endif +%description -n librados2 +RADOS is a reliable, autonomic distributed object storage cluster +developed as part of the Ceph distributed storage system. This is a +shared library allowing applications to access the distributed object +store using a simple file-like interface. + +%package -n librbd1 +Summary: RADOS block device client library +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: librados2 = %{version}-%{release} +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +Obsoletes: ceph-libs +%endif +%description -n librbd1 +RBD is a block device striped across multiple distributed objects in +RADOS, a reliable, autonomic distributed object storage cluster +developed as part of the Ceph distributed storage system. This is a +shared library allowing applications to manage these block devices. + +%package -n libcephfs1 +Summary: Ceph distributed file system client library +Group: System Environment/Libraries +License: LGPL-2.0 +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +Obsoletes: ceph-libs +%endif +%description -n libcephfs1 +Ceph is a distributed network file system designed to provide excellent +performance, reliability, and scalability. This is a shared library +allowing applications to access a Ceph distributed file system via a +POSIX-like interface. + +%package -n python-ceph +Summary: Python libraries for the Ceph distributed filesystem +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +Requires: python-flask +%if 0%{defined suse_version} +%py_requires +%endif +%description -n python-ceph +This package contains Python libraries for interacting with Cephs RADOS +object storage. + +%package -n rest-bench +Summary: RESTful benchmark +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: ceph-common = %{version}-%{release} +%description -n rest-bench +RESTful bencher that can be used to benchmark radosgw performance. + +%package -n ceph-test +Summary: Ceph benchmarks and test tools +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +Requires: libcephfs1 = %{version}-%{release} +%description -n ceph-test +This package contains Ceph benchmarks and test tools. + +%package -n libcephfs_jni1 +Summary: Java Native Interface library for CephFS Java bindings. +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: java +Requires: libcephfs1 = %{version}-%{release} +BuildRequires: java-devel +%description -n libcephfs_jni1 +This package contains the Java Native Interface library for CephFS Java +bindings. + +%package -n cephfs-java +Summary: Java libraries for the Ceph File System. +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: java +Requires: libcephfs_jni1 = %{version}-%{release} +BuildRequires: java-devel +Requires: junit4 +BuildRequires: junit4 +%description -n cephfs-java +This package contains the Java libraries for the Ceph File System. + +%if 0%{?opensuse} || 0%{?suse_version} +%debug_package +%endif + +################################################################################# +# common +################################################################################# +%prep +%setup -q + +%build +# Find jni.h +for i in /usr/{lib64,lib}/jvm/java/include{,/linux}; do + [ -d $i ] && java_inc="$java_inc -I$i" +done + +./autogen.sh +MY_CONF_OPT="" + +MY_CONF_OPT="$MY_CONF_OPT --with-radosgw" + +export RPM_OPT_FLAGS=`echo $RPM_OPT_FLAGS | sed -e 's/i386/i486/'` + +%{configure} CPPFLAGS="$java_inc" \ + --prefix=/usr \ + --localstatedir=/var \ + --sysconfdir=/etc \ + --docdir=%{_docdir}/ceph \ + --with-nss \ + --without-cryptopp \ + --with-rest-bench \ + --with-debug \ + --enable-cephfs-java \ + $MY_CONF_OPT \ + %{?_with_ocf} \ + CFLAGS="$RPM_OPT_FLAGS" CXXFLAGS="$RPM_OPT_FLAGS" + +# fix bug in specific version of libedit-devel +%if 0%{defined suse_version} +sed -i -e "s/-lcurses/-lncurses/g" Makefile +sed -i -e "s/-lcurses/-lncurses/g" src/Makefile +sed -i -e "s/-lcurses/-lncurses/g" man/Makefile +sed -i -e "s/-lcurses/-lncurses/g" src/ocf/Makefile +sed -i -e "s/-lcurses/-lncurses/g" src/java/Makefile +%endif + +make -j$(getconf _NPROCESSORS_ONLN) + +%install +make DESTDIR=$RPM_BUILD_ROOT install +find $RPM_BUILD_ROOT -type f -name "*.la" -exec rm -f {} ';' +find $RPM_BUILD_ROOT -type f -name "*.a" -exec rm -f {} ';' +install -D src/init-ceph $RPM_BUILD_ROOT%{_initrddir}/ceph +install -D src/init-radosgw.sysv $RPM_BUILD_ROOT%{_initrddir}/ceph-radosgw +install -D src/init-rbdmap $RPM_BUILD_ROOT%{_initrddir}/rbdmap +install -D src/rbdmap $RPM_BUILD_ROOT%{_sysconfdir}/ceph/rbdmap +mkdir -p $RPM_BUILD_ROOT%{_sbindir} +ln -sf ../../etc/init.d/ceph %{buildroot}/%{_sbindir}/rcceph +ln -sf ../../etc/init.d/ceph-radosgw %{buildroot}/%{_sbindir}/rcceph-radosgw +install -m 0644 -D src/logrotate.conf $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d/ceph +install -m 0644 -D src/rgw/logrotate.conf $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d/radosgw +chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.ceph.conf +chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config + +# udev rules +%if 0%{?rhel} >= 7 +install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/usr/lib/udev/rules.d/50-rbd.rules +install -m 0644 -D udev/60-ceph-partuuid-workaround.rules $RPM_BUILD_ROOT/usr/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +%else +install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules +install -m 0644 -D udev/60-ceph-partuuid-workaround.rules $RPM_BUILD_ROOT/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +%endif + +%if (0%{?rhel} || 0%{?rhel} < 7) +install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%else +install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%endif + +%if 0%{?rhel} >= 7 +mv $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules $RPM_BUILD_ROOT/usr/lib/udev/rules.d/95-ceph-osd.rules +mv $RPM_BUILD_ROOT/sbin/mkcephfs $RPM_BUILD_ROOT/usr/sbin/mkcephfs +mv $RPM_BUILD_ROOT/sbin/mount.ceph $RPM_BUILD_ROOT/usr/sbin/mount.ceph +mv $RPM_BUILD_ROOT/sbin/mount.fuse.ceph $RPM_BUILD_ROOT/usr/sbin/mount.fuse.ceph +%endif + +#set up placeholder directories +mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/run/ceph +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/log/ceph +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/tmp +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/mon +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/osd +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/mds +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-osd +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-mds +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/log/radosgw + +%if %{defined suse_version} +# Fedora seems to have some problems with this macro, use it only on SUSE +%fdupes -s $RPM_BUILD_ROOT/%{python_sitelib} +%fdupes %buildroot +%endif + +%clean +rm -rf $RPM_BUILD_ROOT + +%post +/sbin/ldconfig +/sbin/chkconfig --add ceph +mkdir -p %{_localstatedir}/run/ceph/ + +%preun +%if %{defined suse_version} +%stop_on_removal ceph +%endif +if [ $1 = 0 ] ; then + /sbin/service ceph stop >/dev/null 2>&1 + /sbin/chkconfig --del ceph +fi + +%postun +/sbin/ldconfig +%if %{defined suse_version} +%insserv_cleanup +%endif + + +################################################################################# +# files +################################################################################# +%files +%defattr(-,root,root,-) +%docdir %{_docdir} +%dir %{_docdir}/ceph +%{_docdir}/ceph/sample.ceph.conf +%{_docdir}/ceph/sample.fetch_config +%{_bindir}/cephfs +%{_bindir}/ceph-clsinfo +%{_bindir}/ceph-rest-api +%{_bindir}/crushtool +%{_bindir}/monmaptool +%{_bindir}/osdmaptool +%{_bindir}/ceph-run +%{_bindir}/ceph-mon +%{_bindir}/ceph-mds +%{_bindir}/ceph-osd +%{_bindir}/ceph-rbdnamer +%{_bindir}/librados-config +%{_bindir}/ceph-client-debug +%{_bindir}/ceph-debugpack +%{_bindir}/ceph-coverage +%{_bindir}/ceph_mon_store_converter +%{_initrddir}/ceph +%{_sbindir}/ceph-disk +%{_sbindir}/ceph-disk-activate +%{_sbindir}/ceph-disk-prepare +%{_sbindir}/ceph-disk-udev +%{_sbindir}/ceph-create-keys +%{_sbindir}/rcceph +%if 0%{?rhel} >= 7 +%{_sbindir}/mount.ceph +%else +/sbin/mount.ceph +%endif +%dir %{_libdir}/ceph +%{_libdir}/ceph/ceph_common.sh +%dir %{_libdir}/rados-classes +%{_libdir}/rados-classes/libcls_rbd.so* +%{_libdir}/rados-classes/libcls_hello.so* +%{_libdir}/rados-classes/libcls_rgw.so* +%{_libdir}/rados-classes/libcls_lock.so* +%{_libdir}/rados-classes/libcls_kvs.so* +%{_libdir}/rados-classes/libcls_refcount.so* +%{_libdir}/rados-classes/libcls_log.so* +%{_libdir}/rados-classes/libcls_replica_log.so* +%{_libdir}/rados-classes/libcls_statelog.so* +%{_libdir}/rados-classes/libcls_user.so* +%{_libdir}/rados-classes/libcls_version.so* +%dir %{_libdir}/ceph/erasure-code +%{_libdir}/ceph/erasure-code/libec_example.so* +%{_libdir}/ceph/erasure-code/libec_fail_to_initialize.so* +%{_libdir}/ceph/erasure-code/libec_fail_to_register.so* +%{_libdir}/ceph/erasure-code/libec_hangs.so* +%{_libdir}/ceph/erasure-code/libec_jerasure*.so* +%{_libdir}/ceph/erasure-code/libec_test_jerasure*.so* +%{_libdir}/ceph/erasure-code/libec_missing_entry_point.so* +%if 0%{?rhel} >= 7 +/usr/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +/usr/lib/udev/rules.d/95-ceph-osd.rules +%else +/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +/lib/udev/rules.d/95-ceph-osd.rules +%endif +%config %{_sysconfdir}/bash_completion.d/ceph +%config(noreplace) %{_sysconfdir}/logrotate.d/ceph +%config(noreplace) %{_sysconfdir}/logrotate.d/radosgw +%{_mandir}/man8/ceph-mon.8* +%{_mandir}/man8/ceph-mds.8* +%{_mandir}/man8/ceph-osd.8* +%{_mandir}/man8/mkcephfs.8* +%{_mandir}/man8/ceph-run.8* +%{_mandir}/man8/ceph-rest-api.8* +%{_mandir}/man8/crushtool.8* +%{_mandir}/man8/osdmaptool.8* +%{_mandir}/man8/monmaptool.8* +%{_mandir}/man8/cephfs.8* +%{_mandir}/man8/mount.ceph.8* +%{_mandir}/man8/ceph-rbdnamer.8* +%{_mandir}/man8/ceph-debugpack.8* +%{_mandir}/man8/ceph-clsinfo.8.gz +%{_mandir}/man8/librados-config.8.gz +#set up placeholder directories +%dir %{_localstatedir}/lib/ceph/ +%dir %{_localstatedir}/lib/ceph/tmp +%dir %{_localstatedir}/lib/ceph/mon +%dir %{_localstatedir}/lib/ceph/osd +%dir %{_localstatedir}/lib/ceph/mds +%dir %{_localstatedir}/lib/ceph/bootstrap-osd +%dir %{_localstatedir}/lib/ceph/bootstrap-mds +%ghost %dir %{_localstatedir}/run/ceph/ + +################################################################################# +%files -n ceph-common +%defattr(-,root,root,-) +%{_bindir}/ceph +%{_bindir}/ceph-authtool +%{_bindir}/ceph-conf +%{_bindir}/ceph-dencoder +%{_bindir}/ceph-syn +%{_bindir}/ceph-crush-location +%{_bindir}/rados +%{_bindir}/rbd +%{_bindir}/ceph-post-file +%{_bindir}/ceph-brag +%{_mandir}/man8/ceph-authtool.8* +%{_mandir}/man8/ceph-conf.8* +%{_mandir}/man8/ceph-dencoder.8* +%{_mandir}/man8/ceph-syn.8* +%{_mandir}/man8/ceph-post-file.8* +%{_mandir}/man8/ceph.8* +%{_mandir}/man8/rados.8* +%{_mandir}/man8/rbd.8* +%{_datadir}/ceph/known_hosts_drop.ceph.com +%{_datadir}/ceph/id_dsa_drop.ceph.com +%{_datadir}/ceph/id_dsa_drop.ceph.com.pub +%dir %{_sysconfdir}/ceph/ +%dir %{_localstatedir}/log/ceph/ +%config %{_sysconfdir}/bash_completion.d/rados +%config %{_sysconfdir}/bash_completion.d/rbd +%config(noreplace) %{_sysconfdir}/ceph/rbdmap +%{_initrddir}/rbdmap + +%postun -n ceph-common +# Package removal cleanup +if [ "$1" -eq "0" ] ; then + rm -rf /var/log/ceph + rm -rf /etc/ceph +fi + +################################################################################# +%files fuse +%defattr(-,root,root,-) +%{_bindir}/ceph-fuse +%{_mandir}/man8/ceph-fuse.8* +%if 0%{?rhel} >= 7 +%{_sbindir}/mount.fuse.ceph +%else +/sbin/mount.fuse.ceph +%endif + +################################################################################# +%files -n rbd-fuse +%defattr(-,root,root,-) +%{_bindir}/rbd-fuse +%{_mandir}/man8/rbd-fuse.8* + +################################################################################# +%files devel +%defattr(-,root,root,-) +%dir %{_includedir}/cephfs +%{_includedir}/cephfs/libcephfs.h +%dir %{_includedir}/rados +%{_includedir}/rados/librados.h +%{_includedir}/rados/librados.hpp +%{_includedir}/rados/buffer.h +%{_includedir}/rados/page.h +%{_includedir}/rados/crc32c.h +%{_includedir}/rados/rados_types.h +%{_includedir}/rados/rados_types.hpp +%{_includedir}/rados/memory.h +%dir %{_includedir}/rbd +%{_includedir}/rbd/librbd.h +%{_includedir}/rbd/librbd.hpp +%{_includedir}/rbd/features.h +%{_libdir}/libcephfs.so +%{_libdir}/librbd.so +%{_libdir}/librados.so +%{_libdir}/libcephfs_jni.so + +################################################################################# +%files radosgw +%defattr(-,root,root,-) +%{_initrddir}/ceph-radosgw +%{_bindir}/radosgw +%{_bindir}/radosgw-admin +%{_mandir}/man8/radosgw.8* +%{_mandir}/man8/radosgw-admin.8* +%{_sbindir}/rcceph-radosgw +%config %{_sysconfdir}/bash_completion.d/radosgw-admin +%dir %{_localstatedir}/log/radosgw/ + +%post radosgw +/sbin/ldconfig +%if %{defined suse_version} +%fillup_and_insserv -f -y ceph-radosgw +%endif + +%preun radosgw +%if %{defined suse_version} +%stop_on_removal ceph-radosgw +%endif + +%postun radosgw +/sbin/ldconfig +%if %{defined suse_version} +%restart_on_update ceph-radosgw +%insserv_cleanup +%endif +# Package removal cleanup +if [ "$1" -eq "0" ] ; then + rm -rf /var/log/radosgw +fi + + +################################################################################# +%if %{with ocf} +%files resource-agents +%defattr(0755,root,root,-) +%dir /usr/lib/ocf +%dir /usr/lib/ocf/resource.d +%dir /usr/lib/ocf/resource.d/ceph +/usr/lib/ocf/resource.d/%{name}/* +%endif + +################################################################################# +%files -n librados2 +%defattr(-,root,root,-) +%{_libdir}/librados.so.* + +%post -n librados2 +/sbin/ldconfig + +%postun -n librados2 +/sbin/ldconfig + +################################################################################# +%files -n librbd1 +%defattr(-,root,root,-) +%{_libdir}/librbd.so.* +%if 0%{?rhel} >= 7 +/usr/lib/udev/rules.d/50-rbd.rules +%else +/lib/udev/rules.d/50-rbd.rules +%endif + +%post -n librbd1 +/sbin/ldconfig +mkdir -p /usr/lib64/qemu/ +ln -sf %{_libdir}/librbd.so.1 /usr/lib64/qemu/librbd.so.1 + +%postun -n librbd1 +/sbin/ldconfig + +################################################################################# +%files -n libcephfs1 +%defattr(-,root,root,-) +%{_libdir}/libcephfs.so.* + +%post -n libcephfs1 +/sbin/ldconfig + +%postun -n libcephfs1 +/sbin/ldconfig + +################################################################################# +%files -n python-ceph +%defattr(-,root,root,-) +%{python_sitelib}/rados.py* +%{python_sitelib}/rbd.py* +%{python_sitelib}/cephfs.py* +%{python_sitelib}/ceph_argparse.py* +%{python_sitelib}/ceph_rest_api.py* + +################################################################################# +%files -n rest-bench +%defattr(-,root,root,-) +%{_bindir}/rest-bench + +################################################################################# +%files -n ceph-test +%defattr(-,root,root,-) +%{_bindir}/ceph_bench_log +%{_bindir}/ceph_dupstore +%{_bindir}/ceph_kvstorebench +%{_bindir}/ceph_multi_stress_watch +%{_bindir}/ceph_erasure_code +%{_bindir}/ceph_erasure_code_benchmark +%{_bindir}/ceph_omapbench +%{_bindir}/ceph_psim +%{_bindir}/ceph_radosacl +%{_bindir}/ceph_rgw_jsonparser +%{_bindir}/ceph_rgw_multiparser +%{_bindir}/ceph_scratchtool +%{_bindir}/ceph_scratchtoolpp +%{_bindir}/ceph_smalliobench +%{_bindir}/ceph_smalliobenchdumb +%{_bindir}/ceph_smalliobenchfs +%{_bindir}/ceph_smalliobenchrbd +%{_bindir}/ceph_filestore_dump +%{_bindir}/ceph_filestore_tool +%{_bindir}/ceph_streamtest +%{_bindir}/ceph_test_* +%{_bindir}/ceph_tpbench +%{_bindir}/ceph_xattr_bench +%{_bindir}/ceph-monstore-tool +%{_bindir}/ceph-osdomap-tool +%{_bindir}/ceph-kvstore-tool + +%files -n libcephfs_jni1 +%defattr(-,root,root,-) +%{_libdir}/libcephfs_jni.so.* + +%files -n cephfs-java +%defattr(-,root,root,-) +%{_javadir}/libcephfs.jar +%{_javadir}/libcephfs-test.jar + +%changelog diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in new file mode 100644 index 00000000..54544540 --- /dev/null +++ b/ceph/ceph.spec.in @@ -0,0 +1,705 @@ +%bcond_with ocf + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +################################################################################# +# common +################################################################################# +Name: ceph +Version: @VERSION@ +Release: @RPM_RELEASE@%{?dist} +Summary: User space components of the Ceph file system +License: GPL-2.0 +Group: System Environment/Base +URL: http://ceph.com/ +Source0: http://ceph.com/download/%{name}-%{version}.tar.bz2 +Requires: librbd1 = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +Requires: libcephfs1 = %{version}-%{release} +Requires: ceph-common = %{version}-%{release} +Requires: python +Requires: python-argparse +Requires: python-ceph +Requires: python-requests +Requires: xfsprogs +Requires: cryptsetup +Requires: parted +Requires: util-linux +Requires: hdparm +Requires(post): binutils +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: make +BuildRequires: gcc-c++ +BuildRequires: libtool +BuildRequires: boost-devel +BuildRequires: libedit-devel +BuildRequires: perl +BuildRequires: gdbm +BuildRequires: pkgconfig +BuildRequires: python +BuildRequires: python-nose +BuildRequires: python-argparse +BuildRequires: libaio-devel +BuildRequires: libcurl-devel +BuildRequires: libxml2-devel +BuildRequires: libuuid-devel +BuildRequires: libblkid-devel >= 2.17 +BuildRequires: leveldb-devel > 1.2 +BuildRequires: xfsprogs-devel +BuildRequires: yasm +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +BuildRequires: snappy-devel +%endif + +################################################################################# +# specific +################################################################################# +%if ! 0%{?rhel} +BuildRequires: sharutils +%endif + +%if 0%{defined suse_version} +%if 0%{?suse_version} > 1210 +Requires: gptfdisk +BuildRequires: gperftools-devel +%else +Requires: scsirastools +BuildRequires: google-perftools-devel +%endif +Recommends: logrotate +BuildRequires: %insserv_prereq +BuildRequires: mozilla-nss-devel +BuildRequires: keyutils-devel +BuildRequires: libatomic-ops-devel +BuildRequires: fdupes +%else +Requires: gdisk +BuildRequires: nss-devel +BuildRequires: keyutils-libs-devel +BuildRequires: libatomic_ops-devel +Requires: gdisk +Requires(post): chkconfig +Requires(preun):chkconfig +Requires(preun):initscripts +BuildRequires: gperftools-devel +%endif + +%description +Ceph is a massively scalable, open-source, distributed +storage system that runs on commodity hardware and delivers object, +block and file system storage. + + +################################################################################# +# packages +################################################################################# +%package -n ceph-common +Summary: Ceph Common +Group: System Environment/Base +Requires: librbd1 = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +Requires: python-ceph = %{version}-%{release} +Requires: python-requests +Requires: redhat-lsb-core +%description -n ceph-common +common utilities to mount and interact with a ceph storage cluster + +%package fuse +Summary: Ceph fuse-based client +Group: System Environment/Base +Requires: %{name} +BuildRequires: fuse-devel +%description fuse +FUSE based client for Ceph distributed network file system + +%package -n rbd-fuse +Summary: Ceph fuse-based client +Group: System Environment/Base +Requires: %{name} +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +BuildRequires: fuse-devel +%description -n rbd-fuse +FUSE based client to map Ceph rbd images to files + +%package devel +Summary: Ceph headers +Group: Development/Libraries +License: LGPL-2.0 +Requires: %{name} = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +Requires: libcephfs1 = %{version}-%{release} +Requires: libcephfs_jni1 = %{version}-%{release} +%description devel +This package contains libraries and headers needed to develop programs +that use Ceph. + +%package radosgw +Summary: Rados REST gateway +Group: Development/Libraries +Requires: ceph-common = %{version}-%{release} +Requires: librados2 = %{version}-%{release} +%if 0%{defined suse_version} +BuildRequires: libexpat-devel +BuildRequires: FastCGI-devel +Requires: apache2-mod_fcgid +%else +BuildRequires: expat-devel +BuildRequires: fcgi-devel +%endif +%description radosgw +radosgw is an S3 HTTP REST gateway for the RADOS object store. It is +implemented as a FastCGI module using libfcgi, and can be used in +conjunction with any FastCGI capable web server. + +%if %{with ocf} +%package resource-agents +Summary: OCF-compliant resource agents for Ceph daemons +Group: System Environment/Base +License: LGPL-2.0 +Requires: %{name} = %{version} +Requires: resource-agents +%description resource-agents +Resource agents for monitoring and managing Ceph daemons +under Open Cluster Framework (OCF) compliant resource +managers such as Pacemaker. +%endif + +%package -n librados2 +Summary: RADOS distributed object store client library +Group: System Environment/Libraries +License: LGPL-2.0 +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +Obsoletes: ceph-libs +%endif +%description -n librados2 +RADOS is a reliable, autonomic distributed object storage cluster +developed as part of the Ceph distributed storage system. This is a +shared library allowing applications to access the distributed object +store using a simple file-like interface. + +%package -n librbd1 +Summary: RADOS block device client library +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: librados2 = %{version}-%{release} +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +Obsoletes: ceph-libs +%endif +%description -n librbd1 +RBD is a block device striped across multiple distributed objects in +RADOS, a reliable, autonomic distributed object storage cluster +developed as part of the Ceph distributed storage system. This is a +shared library allowing applications to manage these block devices. + +%package -n libcephfs1 +Summary: Ceph distributed file system client library +Group: System Environment/Libraries +License: LGPL-2.0 +%if 0%{?rhel_version} || 0%{?centos_version} || 0%{?fedora} +Obsoletes: ceph-libs +%endif +%description -n libcephfs1 +Ceph is a distributed network file system designed to provide excellent +performance, reliability, and scalability. This is a shared library +allowing applications to access a Ceph distributed file system via a +POSIX-like interface. + +%package -n python-ceph +Summary: Python libraries for the Ceph distributed filesystem +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +Requires: python-flask +%if 0%{defined suse_version} +%py_requires +%endif +%description -n python-ceph +This package contains Python libraries for interacting with Cephs RADOS +object storage. + +%package -n rest-bench +Summary: RESTful benchmark +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: ceph-common = %{version}-%{release} +%description -n rest-bench +RESTful bencher that can be used to benchmark radosgw performance. + +%package -n ceph-test +Summary: Ceph benchmarks and test tools +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: librados2 = %{version}-%{release} +Requires: librbd1 = %{version}-%{release} +Requires: libcephfs1 = %{version}-%{release} +%description -n ceph-test +This package contains Ceph benchmarks and test tools. + +%package -n libcephfs_jni1 +Summary: Java Native Interface library for CephFS Java bindings. +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: java +Requires: libcephfs1 = %{version}-%{release} +BuildRequires: java-devel +%description -n libcephfs_jni1 +This package contains the Java Native Interface library for CephFS Java +bindings. + +%package -n cephfs-java +Summary: Java libraries for the Ceph File System. +Group: System Environment/Libraries +License: LGPL-2.0 +Requires: java +Requires: libcephfs_jni1 = %{version}-%{release} +BuildRequires: java-devel +Requires: junit4 +BuildRequires: junit4 +%description -n cephfs-java +This package contains the Java libraries for the Ceph File System. + +%if 0%{?opensuse} || 0%{?suse_version} +%debug_package +%endif + +################################################################################# +# common +################################################################################# +%prep +%setup -q + +%build +# Find jni.h +for i in /usr/{lib64,lib}/jvm/java/include{,/linux}; do + [ -d $i ] && java_inc="$java_inc -I$i" +done + +./autogen.sh +MY_CONF_OPT="" + +MY_CONF_OPT="$MY_CONF_OPT --with-radosgw" + +export RPM_OPT_FLAGS=`echo $RPM_OPT_FLAGS | sed -e 's/i386/i486/'` + +%{configure} CPPFLAGS="$java_inc" \ + --prefix=/usr \ + --localstatedir=/var \ + --sysconfdir=/etc \ + --docdir=%{_docdir}/ceph \ + --with-nss \ + --without-cryptopp \ + --with-rest-bench \ + --with-debug \ + --enable-cephfs-java \ + $MY_CONF_OPT \ + %{?_with_ocf} \ + CFLAGS="$RPM_OPT_FLAGS" CXXFLAGS="$RPM_OPT_FLAGS" + +# fix bug in specific version of libedit-devel +%if 0%{defined suse_version} +sed -i -e "s/-lcurses/-lncurses/g" Makefile +sed -i -e "s/-lcurses/-lncurses/g" src/Makefile +sed -i -e "s/-lcurses/-lncurses/g" man/Makefile +sed -i -e "s/-lcurses/-lncurses/g" src/ocf/Makefile +sed -i -e "s/-lcurses/-lncurses/g" src/java/Makefile +%endif + +make -j$(getconf _NPROCESSORS_ONLN) + +%install +make DESTDIR=$RPM_BUILD_ROOT install +find $RPM_BUILD_ROOT -type f -name "*.la" -exec rm -f {} ';' +find $RPM_BUILD_ROOT -type f -name "*.a" -exec rm -f {} ';' +install -D src/init-ceph $RPM_BUILD_ROOT%{_initrddir}/ceph +install -D src/init-radosgw.sysv $RPM_BUILD_ROOT%{_initrddir}/ceph-radosgw +install -D src/init-rbdmap $RPM_BUILD_ROOT%{_initrddir}/rbdmap +install -D src/rbdmap $RPM_BUILD_ROOT%{_sysconfdir}/ceph/rbdmap +mkdir -p $RPM_BUILD_ROOT%{_sbindir} +ln -sf ../../etc/init.d/ceph %{buildroot}/%{_sbindir}/rcceph +ln -sf ../../etc/init.d/ceph-radosgw %{buildroot}/%{_sbindir}/rcceph-radosgw +install -m 0644 -D src/logrotate.conf $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d/ceph +install -m 0644 -D src/rgw/logrotate.conf $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d/radosgw +chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.ceph.conf +chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config + +# udev rules +%if 0%{?rhel} >= 7 +install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/usr/lib/udev/rules.d/50-rbd.rules +install -m 0644 -D udev/60-ceph-partuuid-workaround.rules $RPM_BUILD_ROOT/usr/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +%else +install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules +install -m 0644 -D udev/60-ceph-partuuid-workaround.rules $RPM_BUILD_ROOT/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +%endif + +%if (0%{?rhel} || 0%{?rhel} < 7) +install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%else +install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%endif + +%if 0%{?rhel} >= 7 +mv $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules $RPM_BUILD_ROOT/usr/lib/udev/rules.d/95-ceph-osd.rules +mv $RPM_BUILD_ROOT/sbin/mkcephfs $RPM_BUILD_ROOT/usr/sbin/mkcephfs +mv $RPM_BUILD_ROOT/sbin/mount.ceph $RPM_BUILD_ROOT/usr/sbin/mount.ceph +mv $RPM_BUILD_ROOT/sbin/mount.fuse.ceph $RPM_BUILD_ROOT/usr/sbin/mount.fuse.ceph +%endif + +#set up placeholder directories +mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/run/ceph +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/log/ceph +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/tmp +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/mon +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/osd +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/mds +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-osd +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-mds +mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/log/radosgw + +%if %{defined suse_version} +# Fedora seems to have some problems with this macro, use it only on SUSE +%fdupes -s $RPM_BUILD_ROOT/%{python_sitelib} +%fdupes %buildroot +%endif + +%clean +rm -rf $RPM_BUILD_ROOT + +%post +/sbin/ldconfig +/sbin/chkconfig --add ceph +mkdir -p %{_localstatedir}/run/ceph/ + +%preun +%if %{defined suse_version} +%stop_on_removal ceph +%endif +if [ $1 = 0 ] ; then + /sbin/service ceph stop >/dev/null 2>&1 + /sbin/chkconfig --del ceph +fi + +%postun +/sbin/ldconfig +%if %{defined suse_version} +%insserv_cleanup +%endif + + +################################################################################# +# files +################################################################################# +%files +%defattr(-,root,root,-) +%docdir %{_docdir} +%dir %{_docdir}/ceph +%{_docdir}/ceph/sample.ceph.conf +%{_docdir}/ceph/sample.fetch_config +%{_bindir}/cephfs +%{_bindir}/ceph-clsinfo +%{_bindir}/ceph-rest-api +%{_bindir}/crushtool +%{_bindir}/monmaptool +%{_bindir}/osdmaptool +%{_bindir}/ceph-run +%{_bindir}/ceph-mon +%{_bindir}/ceph-mds +%{_bindir}/ceph-osd +%{_bindir}/ceph-rbdnamer +%{_bindir}/librados-config +%{_bindir}/ceph-client-debug +%{_bindir}/ceph-debugpack +%{_bindir}/ceph-coverage +%{_bindir}/ceph_mon_store_converter +%{_initrddir}/ceph +%{_sbindir}/ceph-disk +%{_sbindir}/ceph-disk-activate +%{_sbindir}/ceph-disk-prepare +%{_sbindir}/ceph-disk-udev +%{_sbindir}/ceph-create-keys +%{_sbindir}/rcceph +%if 0%{?rhel} >= 7 +%{_sbindir}/mount.ceph +%else +/sbin/mount.ceph +%endif +%dir %{_libdir}/ceph +%{_libdir}/ceph/ceph_common.sh +%dir %{_libdir}/rados-classes +%{_libdir}/rados-classes/libcls_rbd.so* +%{_libdir}/rados-classes/libcls_hello.so* +%{_libdir}/rados-classes/libcls_rgw.so* +%{_libdir}/rados-classes/libcls_lock.so* +%{_libdir}/rados-classes/libcls_kvs.so* +%{_libdir}/rados-classes/libcls_refcount.so* +%{_libdir}/rados-classes/libcls_log.so* +%{_libdir}/rados-classes/libcls_replica_log.so* +%{_libdir}/rados-classes/libcls_statelog.so* +%{_libdir}/rados-classes/libcls_user.so* +%{_libdir}/rados-classes/libcls_version.so* +%dir %{_libdir}/ceph/erasure-code +%{_libdir}/ceph/erasure-code/libec_example.so* +%{_libdir}/ceph/erasure-code/libec_fail_to_initialize.so* +%{_libdir}/ceph/erasure-code/libec_fail_to_register.so* +%{_libdir}/ceph/erasure-code/libec_hangs.so* +%{_libdir}/ceph/erasure-code/libec_jerasure*.so* +%{_libdir}/ceph/erasure-code/libec_test_jerasure*.so* +%{_libdir}/ceph/erasure-code/libec_missing_entry_point.so* +%if 0%{?rhel} >= 7 +/usr/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +/usr/lib/udev/rules.d/95-ceph-osd.rules +%else +/lib/udev/rules.d/60-ceph-partuuid-workaround.rules +/lib/udev/rules.d/95-ceph-osd.rules +%endif +%config %{_sysconfdir}/bash_completion.d/ceph +%config(noreplace) %{_sysconfdir}/logrotate.d/ceph +%config(noreplace) %{_sysconfdir}/logrotate.d/radosgw +%{_mandir}/man8/ceph-mon.8* +%{_mandir}/man8/ceph-mds.8* +%{_mandir}/man8/ceph-osd.8* +%{_mandir}/man8/mkcephfs.8* +%{_mandir}/man8/ceph-run.8* +%{_mandir}/man8/ceph-rest-api.8* +%{_mandir}/man8/crushtool.8* +%{_mandir}/man8/osdmaptool.8* +%{_mandir}/man8/monmaptool.8* +%{_mandir}/man8/cephfs.8* +%{_mandir}/man8/mount.ceph.8* +%{_mandir}/man8/ceph-rbdnamer.8* +%{_mandir}/man8/ceph-debugpack.8* +%{_mandir}/man8/ceph-clsinfo.8.gz +%{_mandir}/man8/librados-config.8.gz +#set up placeholder directories +%dir %{_localstatedir}/lib/ceph/ +%dir %{_localstatedir}/lib/ceph/tmp +%dir %{_localstatedir}/lib/ceph/mon +%dir %{_localstatedir}/lib/ceph/osd +%dir %{_localstatedir}/lib/ceph/mds +%dir %{_localstatedir}/lib/ceph/bootstrap-osd +%dir %{_localstatedir}/lib/ceph/bootstrap-mds +%ghost %dir %{_localstatedir}/run/ceph/ + +################################################################################# +%files -n ceph-common +%defattr(-,root,root,-) +%{_bindir}/ceph +%{_bindir}/ceph-authtool +%{_bindir}/ceph-conf +%{_bindir}/ceph-dencoder +%{_bindir}/ceph-syn +%{_bindir}/ceph-crush-location +%{_bindir}/rados +%{_bindir}/rbd +%{_bindir}/ceph-post-file +%{_bindir}/ceph-brag +%{_mandir}/man8/ceph-authtool.8* +%{_mandir}/man8/ceph-conf.8* +%{_mandir}/man8/ceph-dencoder.8* +%{_mandir}/man8/ceph-syn.8* +%{_mandir}/man8/ceph-post-file.8* +%{_mandir}/man8/ceph.8* +%{_mandir}/man8/rados.8* +%{_mandir}/man8/rbd.8* +%{_datadir}/ceph/known_hosts_drop.ceph.com +%{_datadir}/ceph/id_dsa_drop.ceph.com +%{_datadir}/ceph/id_dsa_drop.ceph.com.pub +%dir %{_sysconfdir}/ceph/ +%dir %{_localstatedir}/log/ceph/ +%config %{_sysconfdir}/bash_completion.d/rados +%config %{_sysconfdir}/bash_completion.d/rbd +%config(noreplace) %{_sysconfdir}/ceph/rbdmap +%{_initrddir}/rbdmap + +%postun -n ceph-common +# Package removal cleanup +if [ "$1" -eq "0" ] ; then + rm -rf /var/log/ceph + rm -rf /etc/ceph +fi + +################################################################################# +%files fuse +%defattr(-,root,root,-) +%{_bindir}/ceph-fuse +%{_mandir}/man8/ceph-fuse.8* +%if 0%{?rhel} >= 7 +%{_sbindir}/mount.fuse.ceph +%else +/sbin/mount.fuse.ceph +%endif + +################################################################################# +%files -n rbd-fuse +%defattr(-,root,root,-) +%{_bindir}/rbd-fuse +%{_mandir}/man8/rbd-fuse.8* + +################################################################################# +%files devel +%defattr(-,root,root,-) +%dir %{_includedir}/cephfs +%{_includedir}/cephfs/libcephfs.h +%dir %{_includedir}/rados +%{_includedir}/rados/librados.h +%{_includedir}/rados/librados.hpp +%{_includedir}/rados/buffer.h +%{_includedir}/rados/page.h +%{_includedir}/rados/crc32c.h +%{_includedir}/rados/rados_types.h +%{_includedir}/rados/rados_types.hpp +%{_includedir}/rados/memory.h +%dir %{_includedir}/rbd +%{_includedir}/rbd/librbd.h +%{_includedir}/rbd/librbd.hpp +%{_includedir}/rbd/features.h +%{_libdir}/libcephfs.so +%{_libdir}/librbd.so +%{_libdir}/librados.so +%{_libdir}/libcephfs_jni.so + +################################################################################# +%files radosgw +%defattr(-,root,root,-) +%{_initrddir}/ceph-radosgw +%{_bindir}/radosgw +%{_bindir}/radosgw-admin +%{_mandir}/man8/radosgw.8* +%{_mandir}/man8/radosgw-admin.8* +%{_sbindir}/rcceph-radosgw +%config %{_sysconfdir}/bash_completion.d/radosgw-admin +%dir %{_localstatedir}/log/radosgw/ + +%post radosgw +/sbin/ldconfig +%if %{defined suse_version} +%fillup_and_insserv -f -y ceph-radosgw +%endif + +%preun radosgw +%if %{defined suse_version} +%stop_on_removal ceph-radosgw +%endif + +%postun radosgw +/sbin/ldconfig +%if %{defined suse_version} +%restart_on_update ceph-radosgw +%insserv_cleanup +%endif +# Package removal cleanup +if [ "$1" -eq "0" ] ; then + rm -rf /var/log/radosgw +fi + + +################################################################################# +%if %{with ocf} +%files resource-agents +%defattr(0755,root,root,-) +%dir /usr/lib/ocf +%dir /usr/lib/ocf/resource.d +%dir /usr/lib/ocf/resource.d/ceph +/usr/lib/ocf/resource.d/%{name}/* +%endif + +################################################################################# +%files -n librados2 +%defattr(-,root,root,-) +%{_libdir}/librados.so.* + +%post -n librados2 +/sbin/ldconfig + +%postun -n librados2 +/sbin/ldconfig + +################################################################################# +%files -n librbd1 +%defattr(-,root,root,-) +%{_libdir}/librbd.so.* +%if 0%{?rhel} >= 7 +/usr/lib/udev/rules.d/50-rbd.rules +%else +/lib/udev/rules.d/50-rbd.rules +%endif + +%post -n librbd1 +/sbin/ldconfig +mkdir -p /usr/lib64/qemu/ +ln -sf %{_libdir}/librbd.so.1 /usr/lib64/qemu/librbd.so.1 + +%postun -n librbd1 +/sbin/ldconfig + +################################################################################# +%files -n libcephfs1 +%defattr(-,root,root,-) +%{_libdir}/libcephfs.so.* + +%post -n libcephfs1 +/sbin/ldconfig + +%postun -n libcephfs1 +/sbin/ldconfig + +################################################################################# +%files -n python-ceph +%defattr(-,root,root,-) +%{python_sitelib}/rados.py* +%{python_sitelib}/rbd.py* +%{python_sitelib}/cephfs.py* +%{python_sitelib}/ceph_argparse.py* +%{python_sitelib}/ceph_rest_api.py* + +################################################################################# +%files -n rest-bench +%defattr(-,root,root,-) +%{_bindir}/rest-bench + +################################################################################# +%files -n ceph-test +%defattr(-,root,root,-) +%{_bindir}/ceph_bench_log +%{_bindir}/ceph_dupstore +%{_bindir}/ceph_kvstorebench +%{_bindir}/ceph_multi_stress_watch +%{_bindir}/ceph_erasure_code +%{_bindir}/ceph_erasure_code_benchmark +%{_bindir}/ceph_omapbench +%{_bindir}/ceph_psim +%{_bindir}/ceph_radosacl +%{_bindir}/ceph_rgw_jsonparser +%{_bindir}/ceph_rgw_multiparser +%{_bindir}/ceph_scratchtool +%{_bindir}/ceph_scratchtoolpp +%{_bindir}/ceph_smalliobench +%{_bindir}/ceph_smalliobenchdumb +%{_bindir}/ceph_smalliobenchfs +%{_bindir}/ceph_smalliobenchrbd +%{_bindir}/ceph_filestore_dump +%{_bindir}/ceph_filestore_tool +%{_bindir}/ceph_streamtest +%{_bindir}/ceph_test_* +%{_bindir}/ceph_tpbench +%{_bindir}/ceph_xattr_bench +%{_bindir}/ceph-monstore-tool +%{_bindir}/ceph-osdomap-tool +%{_bindir}/ceph-kvstore-tool + +%files -n libcephfs_jni1 +%defattr(-,root,root,-) +%{_libdir}/libcephfs_jni.so.* + +%files -n cephfs-java +%defattr(-,root,root,-) +%{_javadir}/libcephfs.jar +%{_javadir}/libcephfs-test.jar + +%changelog diff --git a/ceph/compile b/ceph/compile new file mode 100755 index 00000000..b1f47491 --- /dev/null +++ b/ceph/compile @@ -0,0 +1,310 @@ +#! /bin/sh +# Wrapper for compilers which do not understand '-c -o'. + +scriptversion=2012-01-04.17; # UTC + +# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2009, 2010, 2012 Free +# Software Foundation, Inc. +# Written by Tom Tromey . +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +nl=' +' + +# We need space, tab and new line, in precisely that order. Quoting is +# there to prevent tools from complaining about whitespace usage. +IFS=" "" $nl" + +file_conv= + +# func_file_conv build_file lazy +# Convert a $build file to $host form and store it in $file +# Currently only supports Windows hosts. If the determined conversion +# type is listed in (the comma separated) LAZY, no conversion will +# take place. +func_file_conv () +{ + file=$1 + case $file in + / | /[!/]*) # absolute file, and not a UNC file + if test -z "$file_conv"; then + # lazily determine how to convert abs files + case `uname -s` in + MINGW*) + file_conv=mingw + ;; + CYGWIN*) + file_conv=cygwin + ;; + *) + file_conv=wine + ;; + esac + fi + case $file_conv/,$2, in + *,$file_conv,*) + ;; + mingw/*) + file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` + ;; + cygwin/*) + file=`cygpath -m "$file" || echo "$file"` + ;; + wine/*) + file=`winepath -w "$file" || echo "$file"` + ;; + esac + ;; + esac +} + +# func_cl_wrapper cl arg... +# Adjust compile command to suit cl +func_cl_wrapper () +{ + # Assume a capable shell + lib_path= + shared=: + linker_opts= + for arg + do + if test -n "$eat"; then + eat= + else + case $1 in + -o) + # configure might choose to run compile as 'compile cc -o foo foo.c'. + eat=1 + case $2 in + *.o | *.[oO][bB][jJ]) + func_file_conv "$2" + set x "$@" -Fo"$file" + shift + ;; + *) + func_file_conv "$2" + set x "$@" -Fe"$file" + shift + ;; + esac + ;; + -I*) + func_file_conv "${1#-I}" mingw + set x "$@" -I"$file" + shift + ;; + -l*) + lib=${1#-l} + found=no + save_IFS=$IFS + IFS=';' + for dir in $lib_path $LIB + do + IFS=$save_IFS + if $shared && test -f "$dir/$lib.dll.lib"; then + found=yes + set x "$@" "$dir/$lib.dll.lib" + break + fi + if test -f "$dir/$lib.lib"; then + found=yes + set x "$@" "$dir/$lib.lib" + break + fi + done + IFS=$save_IFS + + test "$found" != yes && set x "$@" "$lib.lib" + shift + ;; + -L*) + func_file_conv "${1#-L}" + if test -z "$lib_path"; then + lib_path=$file + else + lib_path="$lib_path;$file" + fi + linker_opts="$linker_opts -LIBPATH:$file" + ;; + -static) + shared=false + ;; + -Wl,*) + arg=${1#-Wl,} + save_ifs="$IFS"; IFS=',' + for flag in $arg; do + IFS="$save_ifs" + linker_opts="$linker_opts $flag" + done + IFS="$save_ifs" + ;; + -Xlinker) + eat=1 + linker_opts="$linker_opts $2" + ;; + -*) + set x "$@" "$1" + shift + ;; + *.cc | *.CC | *.cxx | *.CXX | *.[cC]++) + func_file_conv "$1" + set x "$@" -Tp"$file" + shift + ;; + *.c | *.cpp | *.CPP | *.lib | *.LIB | *.Lib | *.OBJ | *.obj | *.[oO]) + func_file_conv "$1" mingw + set x "$@" "$file" + shift + ;; + *) + set x "$@" "$1" + shift + ;; + esac + fi + shift + done + if test -n "$linker_opts"; then + linker_opts="-link$linker_opts" + fi + exec "$@" $linker_opts + exit 1 +} + +eat= + +case $1 in + '') + echo "$0: No command. Try '$0 --help' for more information." 1>&2 + exit 1; + ;; + -h | --h*) + cat <<\EOF +Usage: compile [--help] [--version] PROGRAM [ARGS] + +Wrapper for compilers which do not understand '-c -o'. +Remove '-o dest.o' from ARGS, run PROGRAM with the remaining +arguments, and rename the output as expected. + +If you are trying to build a whole package this is not the +right script to run: please start by reading the file 'INSTALL'. + +Report bugs to . +EOF + exit $? + ;; + -v | --v*) + echo "compile $scriptversion" + exit $? + ;; + cl | *[/\\]cl | cl.exe | *[/\\]cl.exe ) + func_cl_wrapper "$@" # Doesn't return... + ;; +esac + +ofile= +cfile= + +for arg +do + if test -n "$eat"; then + eat= + else + case $1 in + -o) + # configure might choose to run compile as 'compile cc -o foo foo.c'. + # So we strip '-o arg' only if arg is an object. + eat=1 + case $2 in + *.o | *.obj) + ofile=$2 + ;; + *) + set x "$@" -o "$2" + shift + ;; + esac + ;; + *.c) + cfile=$1 + set x "$@" "$1" + shift + ;; + *) + set x "$@" "$1" + shift + ;; + esac + fi + shift +done + +if test -z "$ofile" || test -z "$cfile"; then + # If no '-o' option was seen then we might have been invoked from a + # pattern rule where we don't need one. That is ok -- this is a + # normal compilation that the losing compiler can handle. If no + # '.c' file was seen then we are probably linking. That is also + # ok. + exec "$@" +fi + +# Name of file we expect compiler to create. +cofile=`echo "$cfile" | sed 's|^.*[\\/]||; s|^[a-zA-Z]:||; s/\.c$/.o/'` + +# Create the lock directory. +# Note: use '[/\\:.-]' here to ensure that we don't use the same name +# that we are using for the .o file. Also, base the name on the expected +# object file name, since that is what matters with a parallel build. +lockdir=`echo "$cofile" | sed -e 's|[/\\:.-]|_|g'`.d +while true; do + if mkdir "$lockdir" >/dev/null 2>&1; then + break + fi + sleep 1 +done +# FIXME: race condition here if user kills between mkdir and trap. +trap "rmdir '$lockdir'; exit 1" 1 2 15 + +# Run the compile. +"$@" +ret=$? + +if test -f "$cofile"; then + test "$cofile" = "$ofile" || mv "$cofile" "$ofile" +elif test -f "${cofile}bj"; then + test "${cofile}bj" = "$ofile" || mv "${cofile}bj" "$ofile" +fi + +rmdir "$lockdir" +exit $ret + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/ceph/config.guess b/ceph/config.guess new file mode 100755 index 00000000..d622a44e --- /dev/null +++ b/ceph/config.guess @@ -0,0 +1,1530 @@ +#! /bin/sh +# Attempt to guess a canonical system name. +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, +# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012 Free Software Foundation, Inc. + +timestamp='2012-02-10' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + + +# Originally written by Per Bothner. Please send patches (context +# diff format) to and include a ChangeLog +# entry. +# +# This script attempts to guess a canonical system name similar to +# config.sub. If it succeeds, it prints the system name on stdout, and +# exits with 0. Otherwise, it exits with 1. +# +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] + +Output the configuration name of the system \`$me' is run on. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.guess ($timestamp) + +Originally written by Per Bothner. +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, +2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 +Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" >&2 + exit 1 ;; + * ) + break ;; + esac +done + +if test $# != 0; then + echo "$me: too many arguments$help" >&2 + exit 1 +fi + +trap 'exit 1' 1 2 15 + +# CC_FOR_BUILD -- compiler used by this script. Note that the use of a +# compiler to aid in system detection is discouraged as it requires +# temporary files to be created and, as you can see below, it is a +# headache to deal with in a portable fashion. + +# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still +# use `HOST_CC' if defined, but it is deprecated. + +# Portable tmp directory creation inspired by the Autoconf team. + +set_cc_for_build=' +trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; +trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; +: ${TMPDIR=/tmp} ; + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; +dummy=$tmp/dummy ; +tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; +case $CC_FOR_BUILD,$HOST_CC,$CC in + ,,) echo "int x;" > $dummy.c ; + for c in cc gcc c89 c99 ; do + if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then + CC_FOR_BUILD="$c"; break ; + fi ; + done ; + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found ; + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; +esac ; set_cc_for_build= ;' + +# This is needed to find uname on a Pyramid OSx when run in the BSD universe. +# (ghazi@noc.rutgers.edu 1994-08-24) +if (test -f /.attbin/uname) >/dev/null 2>&1 ; then + PATH=$PATH:/.attbin ; export PATH +fi + +UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown +UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown +UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown +UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown + +# Note: order is significant - the case branches are not exclusive. + +case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in + *:NetBSD:*:*) + # NetBSD (nbsd) targets should (where applicable) match one or + # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, + # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently + # switched to ELF, *-*-netbsd* would select the old + # object file format. This provides both forward + # compatibility and a consistent mechanism for selecting the + # object file format. + # + # Note: NetBSD doesn't particularly care about the vendor + # portion of the name. We always set it to "unknown". + sysctl="sysctl -n hw.machine_arch" + UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + case "${UNAME_MACHINE_ARCH}" in + armeb) machine=armeb-unknown ;; + arm*) machine=arm-unknown ;; + sh3el) machine=shl-unknown ;; + sh3eb) machine=sh-unknown ;; + sh5el) machine=sh5le-unknown ;; + *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + esac + # The Operating System including object format, if it has switched + # to ELF recently, or will in the future. + case "${UNAME_MACHINE_ARCH}" in + arm*|i386|m68k|ns32k|sh3*|sparc|vax) + eval $set_cc_for_build + if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ELF__ + then + # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). + # Return netbsd for either. FIX? + os=netbsd + else + os=netbsdelf + fi + ;; + *) + os=netbsd + ;; + esac + # The OS release + # Debian GNU/NetBSD machines have a different userland, and + # thus, need a distinct triplet. However, they do not need + # kernel version information, so it can be replaced with a + # suitable tag, in the style of linux-gnu. + case "${UNAME_VERSION}" in + Debian*) + release='-gnu' + ;; + *) + release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + ;; + esac + # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: + # contains redundant information, the shorter form: + # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. + echo "${machine}-${os}${release}" + exit ;; + *:OpenBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + exit ;; + *:ekkoBSD:*:*) + echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + exit ;; + *:SolidBSD:*:*) + echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + exit ;; + macppc:MirBSD:*:*) + echo powerpc-unknown-mirbsd${UNAME_RELEASE} + exit ;; + *:MirBSD:*:*) + echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + exit ;; + alpha:OSF1:*:*) + case $UNAME_RELEASE in + *4.0) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + ;; + *5.*) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + ;; + esac + # According to Compaq, /usr/sbin/psrinfo has been available on + # OSF/1 and Tru64 systems produced since 1995. I hope that + # covers most systems running today. This code pipes the CPU + # types through head -n 1, so we only detect the type of CPU 0. + ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + case "$ALPHA_CPU_TYPE" in + "EV4 (21064)") + UNAME_MACHINE="alpha" ;; + "EV4.5 (21064)") + UNAME_MACHINE="alpha" ;; + "LCA4 (21066/21068)") + UNAME_MACHINE="alpha" ;; + "EV5 (21164)") + UNAME_MACHINE="alphaev5" ;; + "EV5.6 (21164A)") + UNAME_MACHINE="alphaev56" ;; + "EV5.6 (21164PC)") + UNAME_MACHINE="alphapca56" ;; + "EV5.7 (21164PC)") + UNAME_MACHINE="alphapca57" ;; + "EV6 (21264)") + UNAME_MACHINE="alphaev6" ;; + "EV6.7 (21264A)") + UNAME_MACHINE="alphaev67" ;; + "EV6.8CB (21264C)") + UNAME_MACHINE="alphaev68" ;; + "EV6.8AL (21264B)") + UNAME_MACHINE="alphaev68" ;; + "EV6.8CX (21264D)") + UNAME_MACHINE="alphaev68" ;; + "EV6.9A (21264/EV69A)") + UNAME_MACHINE="alphaev69" ;; + "EV7 (21364)") + UNAME_MACHINE="alphaev7" ;; + "EV7.9 (21364A)") + UNAME_MACHINE="alphaev79" ;; + esac + # A Pn.n version is a patched version. + # A Vn.n version is a released version. + # A Tn.n version is a released field test version. + # A Xn.n version is an unreleased experimental baselevel. + # 1.2 uses "1.2" for uname -r. + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + exitcode=$? + trap '' 0 + exit $exitcode ;; + Alpha\ *:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # Should we change UNAME_MACHINE based on the output of uname instead + # of the specific Alpha model? + echo alpha-pc-interix + exit ;; + 21064:Windows_NT:50:3) + echo alpha-dec-winnt3.5 + exit ;; + Amiga*:UNIX_System_V:4.0:*) + echo m68k-unknown-sysv4 + exit ;; + *:[Aa]miga[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-amigaos + exit ;; + *:[Mm]orph[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-morphos + exit ;; + *:OS/390:*:*) + echo i370-ibm-openedition + exit ;; + *:z/VM:*:*) + echo s390-ibm-zvmoe + exit ;; + *:OS400:*:*) + echo powerpc-ibm-os400 + exit ;; + arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) + echo arm-acorn-riscix${UNAME_RELEASE} + exit ;; + arm:riscos:*:*|arm:RISCOS:*:*) + echo arm-unknown-riscos + exit ;; + SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) + echo hppa1.1-hitachi-hiuxmpp + exit ;; + Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) + # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. + if test "`(/bin/universe) 2>/dev/null`" = att ; then + echo pyramid-pyramid-sysv3 + else + echo pyramid-pyramid-bsd + fi + exit ;; + NILE*:*:*:dcosx) + echo pyramid-pyramid-svr4 + exit ;; + DRS?6000:unix:4.0:6*) + echo sparc-icl-nx6 + exit ;; + DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) + case `/usr/bin/uname -p` in + sparc) echo sparc-icl-nx7; exit ;; + esac ;; + s390x:SunOS:*:*) + echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4H:SunOS:5.*:*) + echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) + echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) + echo i386-pc-auroraux${UNAME_RELEASE} + exit ;; + i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) + eval $set_cc_for_build + SUN_ARCH="i386" + # If there is a compiler, see if it is configured for 64-bit objects. + # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. + # This test works for both compilers. + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + SUN_ARCH="x86_64" + fi + fi + echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:6*:*) + # According to config.sub, this is the proper way to canonicalize + # SunOS6. Hard to guess exactly what SunOS6 will be like, but + # it's likely to be more like Solaris than SunOS4. + echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:*:*) + case "`/usr/bin/arch -k`" in + Series*|S4*) + UNAME_RELEASE=`uname -v` + ;; + esac + # Japanese Language versions have a version number like `4.1.3-JL'. + echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + exit ;; + sun3*:SunOS:*:*) + echo m68k-sun-sunos${UNAME_RELEASE} + exit ;; + sun*:*:4.2BSD:*) + UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` + test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + case "`/bin/arch`" in + sun3) + echo m68k-sun-sunos${UNAME_RELEASE} + ;; + sun4) + echo sparc-sun-sunos${UNAME_RELEASE} + ;; + esac + exit ;; + aushp:SunOS:*:*) + echo sparc-auspex-sunos${UNAME_RELEASE} + exit ;; + # The situation for MiNT is a little confusing. The machine name + # can be virtually everything (everything which is not + # "atarist" or "atariste" at least should have a processor + # > m68000). The system name ranges from "MiNT" over "FreeMiNT" + # to the lowercase version "mint" (or "freemint"). Finally + # the system name "TOS" denotes a system which is actually not + # MiNT. But MiNT is downward compatible to TOS, so this should + # be no problem. + atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) + echo m68k-milan-mint${UNAME_RELEASE} + exit ;; + hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) + echo m68k-hades-mint${UNAME_RELEASE} + exit ;; + *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) + echo m68k-unknown-mint${UNAME_RELEASE} + exit ;; + m68k:machten:*:*) + echo m68k-apple-machten${UNAME_RELEASE} + exit ;; + powerpc:machten:*:*) + echo powerpc-apple-machten${UNAME_RELEASE} + exit ;; + RISC*:Mach:*:*) + echo mips-dec-mach_bsd4.3 + exit ;; + RISC*:ULTRIX:*:*) + echo mips-dec-ultrix${UNAME_RELEASE} + exit ;; + VAX*:ULTRIX*:*:*) + echo vax-dec-ultrix${UNAME_RELEASE} + exit ;; + 2020:CLIX:*:* | 2430:CLIX:*:*) + echo clipper-intergraph-clix${UNAME_RELEASE} + exit ;; + mips:*:*:UMIPS | mips:*:*:RISCos) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c +#ifdef __cplusplus +#include /* for printf() prototype */ + int main (int argc, char *argv[]) { +#else + int main (argc, argv) int argc; char *argv[]; { +#endif + #if defined (host_mips) && defined (MIPSEB) + #if defined (SYSTYPE_SYSV) + printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_SVR4) + printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) + printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + #endif + #endif + exit (-1); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && + dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`$dummy $dummyarg` && + { echo "$SYSTEM_NAME"; exit; } + echo mips-mips-riscos${UNAME_RELEASE} + exit ;; + Motorola:PowerMAX_OS:*:*) + echo powerpc-motorola-powermax + exit ;; + Motorola:*:4.3:PL8-*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:Power_UNIX:*:*) + echo powerpc-harris-powerunix + exit ;; + m88k:CX/UX:7*:*) + echo m88k-harris-cxux7 + exit ;; + m88k:*:4*:R4*) + echo m88k-motorola-sysv4 + exit ;; + m88k:*:3*:R3*) + echo m88k-motorola-sysv3 + exit ;; + AViiON:dgux:*:*) + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` + if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + then + if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ + [ ${TARGET_BINARY_INTERFACE}x = x ] + then + echo m88k-dg-dgux${UNAME_RELEASE} + else + echo m88k-dg-dguxbcs${UNAME_RELEASE} + fi + else + echo i586-dg-dgux${UNAME_RELEASE} + fi + exit ;; + M88*:DolphinOS:*:*) # DolphinOS (SVR3) + echo m88k-dolphin-sysv3 + exit ;; + M88*:*:R3*:*) + # Delta 88k system running SVR3 + echo m88k-motorola-sysv3 + exit ;; + XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) + echo m88k-tektronix-sysv3 + exit ;; + Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) + echo m68k-tektronix-bsd + exit ;; + *:IRIX*:*:*) + echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + exit ;; + ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + i*86:AIX:*:*) + echo i386-ibm-aix + exit ;; + ia64:AIX:*:*) + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + exit ;; + *:AIX:2:3) + if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + + main() + { + if (!__power_pc()) + exit(1); + puts("powerpc-ibm-aix3.2.5"); + exit(0); + } +EOF + if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + then + echo "$SYSTEM_NAME" + else + echo rs6000-ibm-aix3.2.5 + fi + elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then + echo rs6000-ibm-aix3.2.4 + else + echo rs6000-ibm-aix3.2 + fi + exit ;; + *:AIX:*:[4567]) + IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` + if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + IBM_ARCH=rs6000 + else + IBM_ARCH=powerpc + fi + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${IBM_ARCH}-ibm-aix${IBM_REV} + exit ;; + *:AIX:*:*) + echo rs6000-ibm-aix + exit ;; + ibmrt:4.4BSD:*|romp-ibm:BSD:*) + echo romp-ibm-bsd4.4 + exit ;; + ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and + echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + exit ;; # report: romp-ibm BSD 4.3 + *:BOSX:*:*) + echo rs6000-bull-bosx + exit ;; + DPX/2?00:B.O.S.:*:*) + echo m68k-bull-sysv3 + exit ;; + 9000/[34]??:4.3bsd:1.*:*) + echo m68k-hp-bsd + exit ;; + hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) + echo m68k-hp-bsd4.4 + exit ;; + 9000/[34678]??:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + case "${UNAME_MACHINE}" in + 9000/31? ) HP_ARCH=m68000 ;; + 9000/[34]?? ) HP_ARCH=m68k ;; + 9000/[678][0-9][0-9]) + if [ -x /usr/bin/getconf ]; then + sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 + 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH="hppa2.0n" ;; + 64) HP_ARCH="hppa2.0w" ;; + '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + esac ;; + esac + fi + if [ "${HP_ARCH}" = "" ]; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + + #define _HPUX_SOURCE + #include + #include + + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); + + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } +EOF + (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + test -z "$HP_ARCH" && HP_ARCH=hppa + fi ;; + esac + if [ ${HP_ARCH} = "hppa2.0w" ] + then + eval $set_cc_for_build + + # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating + # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler + # generating 64-bit code. GNU and HP use different nomenclature: + # + # $ CC_FOR_BUILD=cc ./config.guess + # => hppa2.0w-hp-hpux11.23 + # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess + # => hppa64-hp-hpux11.23 + + if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + grep -q __LP64__ + then + HP_ARCH="hppa2.0w" + else + HP_ARCH="hppa64" + fi + fi + echo ${HP_ARCH}-hp-hpux${HPUX_REV} + exit ;; + ia64:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux${HPUX_REV} + exit ;; + 3050*:HI-UX:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + int + main () + { + long cpu = sysconf (_SC_CPU_VERSION); + /* The order matters, because CPU_IS_HP_MC68K erroneously returns + true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct + results, however. */ + if (CPU_IS_PA_RISC (cpu)) + { + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; + case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; + default: puts ("hppa-hitachi-hiuxwe2"); break; + } + } + else if (CPU_IS_HP_MC68K (cpu)) + puts ("m68k-hitachi-hiuxwe2"); + else puts ("unknown-hitachi-hiuxwe2"); + exit (0); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } + echo unknown-hitachi-hiuxwe2 + exit ;; + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + echo hppa1.1-hp-bsd + exit ;; + 9000/8??:4.3bsd:*:*) + echo hppa1.0-hp-bsd + exit ;; + *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) + echo hppa1.0-hp-mpeix + exit ;; + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + echo hppa1.1-hp-osf + exit ;; + hp8??:OSF1:*:*) + echo hppa1.0-hp-osf + exit ;; + i*86:OSF1:*:*) + if [ -x /usr/sbin/sysversion ] ; then + echo ${UNAME_MACHINE}-unknown-osf1mk + else + echo ${UNAME_MACHINE}-unknown-osf1 + fi + exit ;; + parisc*:Lites*:*:*) + echo hppa1.1-hp-lites + exit ;; + C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) + echo c1-convex-bsd + exit ;; + C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit ;; + C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) + echo c34-convex-bsd + exit ;; + C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) + echo c38-convex-bsd + exit ;; + C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) + echo c4-convex-bsd + exit ;; + CRAY*Y-MP:*:*:*) + echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*[A-Z]90:*:*:*) + echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ + -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ + -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*TS:*:*:*) + echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*T3E:*:*:*) + echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*SV1:*:*:*) + echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + *:UNICOS/mp:*:*) + echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) + FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + 5000:UNIX_System_V:4.*:*) + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) + echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + exit ;; + sparc*:BSD/OS:*:*) + echo sparc-unknown-bsdi${UNAME_RELEASE} + exit ;; + *:BSD/OS:*:*) + echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + exit ;; + *:FreeBSD:*:*) + UNAME_PROCESSOR=`/usr/bin/uname -p` + case ${UNAME_PROCESSOR} in + amd64) + echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + *) + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + esac + exit ;; + i*:CYGWIN*:*) + echo ${UNAME_MACHINE}-pc-cygwin + exit ;; + *:MINGW*:*) + echo ${UNAME_MACHINE}-pc-mingw32 + exit ;; + i*:MSYS*:*) + echo ${UNAME_MACHINE}-pc-msys + exit ;; + i*:windows32*:*) + # uname -m includes "-pc" on this system. + echo ${UNAME_MACHINE}-mingw32 + exit ;; + i*:PW*:*) + echo ${UNAME_MACHINE}-pc-pw32 + exit ;; + *:Interix*:*) + case ${UNAME_MACHINE} in + x86) + echo i586-pc-interix${UNAME_RELEASE} + exit ;; + authenticamd | genuineintel | EM64T) + echo x86_64-unknown-interix${UNAME_RELEASE} + exit ;; + IA64) + echo ia64-unknown-interix${UNAME_RELEASE} + exit ;; + esac ;; + [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) + echo i${UNAME_MACHINE}-pc-mks + exit ;; + 8664:Windows_NT:*) + echo x86_64-pc-mks + exit ;; + i*:Windows_NT*:* | Pentium*:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we + # UNAME_MACHINE based on the output of uname instead of i386? + echo i586-pc-interix + exit ;; + i*:UWIN*:*) + echo ${UNAME_MACHINE}-pc-uwin + exit ;; + amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) + echo x86_64-unknown-cygwin + exit ;; + p*:CYGWIN*:*) + echo powerpcle-unknown-cygwin + exit ;; + prep*:SunOS:5.*:*) + echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + *:GNU:*:*) + # the GNU system + echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + exit ;; + *:GNU/*:*:*) + # other systems with GNU libc and userland + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu + exit ;; + i*86:Minix:*:*) + echo ${UNAME_MACHINE}-pc-minix + exit ;; + aarch64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + aarch64_be:Linux:*:*) + UNAME_MACHINE=aarch64_be + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep -q ld.so.1 + if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi + echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + exit ;; + arm*:Linux:*:*) + eval $set_cc_for_build + if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_EABI__ + then + echo ${UNAME_MACHINE}-unknown-linux-gnu + else + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo ${UNAME_MACHINE}-unknown-linux-gnueabi + else + echo ${UNAME_MACHINE}-unknown-linux-gnueabihf + fi + fi + exit ;; + avr32*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + cris:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-gnu + exit ;; + crisv32:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-gnu + exit ;; + frv:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + hexagon:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + i*86:Linux:*:*) + LIBC=gnu + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #ifdef __dietlibc__ + LIBC=dietlibc + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + echo "${UNAME_MACHINE}-pc-linux-${LIBC}" + exit ;; + ia64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + m32r*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + m68*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + mips:Linux:*:* | mips64:Linux:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #undef CPU + #undef ${UNAME_MACHINE} + #undef ${UNAME_MACHINE}el + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) + CPU=${UNAME_MACHINE}el + #else + #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) + CPU=${UNAME_MACHINE} + #else + CPU= + #endif + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; } + ;; + or32:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + padre:Linux:*:*) + echo sparc-unknown-linux-gnu + exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-gnu + exit ;; + parisc:Linux:*:* | hppa:Linux:*:*) + # Look for CPU level + case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in + PA7*) echo hppa1.1-unknown-linux-gnu ;; + PA8*) echo hppa2.0-unknown-linux-gnu ;; + *) echo hppa-unknown-linux-gnu ;; + esac + exit ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-gnu + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-gnu + exit ;; + s390:Linux:*:* | s390x:Linux:*:*) + echo ${UNAME_MACHINE}-ibm-linux + exit ;; + sh64*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + sh*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + sparc:Linux:*:* | sparc64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + tile*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + vax:Linux:*:*) + echo ${UNAME_MACHINE}-dec-linux-gnu + exit ;; + x86_64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + xtensa*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + i*86:DYNIX/ptx:4*:*) + # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. + # earlier versions are messed up and put the nodename in both + # sysname and nodename. + echo i386-sequent-sysv4 + exit ;; + i*86:UNIX_SV:4.2MP:2.*) + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, + # I just have to hope. -- rms. + # Use sysv4.2uw... so that sysv4* matches it. + echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + exit ;; + i*86:OS/2:*:*) + # If we were able to find `uname', then EMX Unix compatibility + # is probably installed. + echo ${UNAME_MACHINE}-pc-os2-emx + exit ;; + i*86:XTS-300:*:STOP) + echo ${UNAME_MACHINE}-unknown-stop + exit ;; + i*86:atheos:*:*) + echo ${UNAME_MACHINE}-unknown-atheos + exit ;; + i*86:syllable:*:*) + echo ${UNAME_MACHINE}-pc-syllable + exit ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) + echo i386-unknown-lynxos${UNAME_RELEASE} + exit ;; + i*86:*DOS:*:*) + echo ${UNAME_MACHINE}-pc-msdosdjgpp + exit ;; + i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) + UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then + echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + else + echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + fi + exit ;; + i*86:*:5:[678]*) + # UnixWare 7.x, OpenUNIX and OpenServer 6. + case `/bin/uname -X | grep "^Machine"` in + *486*) UNAME_MACHINE=i486 ;; + *Pentium) UNAME_MACHINE=i586 ;; + *Pent*|*Celeron) UNAME_MACHINE=i686 ;; + esac + echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + exit ;; + i*86:*:3.2:*) + if test -f /usr/options/cb.name; then + UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then + UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 + (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ + && UNAME_MACHINE=i586 + (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ + && UNAME_MACHINE=i686 + (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ + && UNAME_MACHINE=i686 + echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + else + echo ${UNAME_MACHINE}-pc-sysv32 + fi + exit ;; + pc:*:*:*) + # Left here for compatibility: + # uname -m prints for DJGPP always 'pc', but it prints nothing about + # the processor, so we play safe by assuming i586. + # Note: whatever this is, it MUST be the same as what config.sub + # prints for the "djgpp" host, or else GDB configury will decide that + # this is a cross-build. + echo i586-pc-msdosdjgpp + exit ;; + Intel:Mach:3*:*) + echo i386-pc-mach3 + exit ;; + paragon:*:*:*) + echo i860-intel-osf1 + exit ;; + i860:*:4.*:*) # i860-SVR4 + if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then + echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + else # Add other i860-SVR4 vendors below as they are discovered. + echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + fi + exit ;; + mini*:CTIX:SYS*5:*) + # "miniframe" + echo m68010-convergent-sysv + exit ;; + mc68k:UNIX:SYSTEM5:3.51m) + echo m68k-convergent-sysv + exit ;; + M680?0:D-NIX:5.3:*) + echo m68k-diab-dnix + exit ;; + M68*:*:R3V[5678]*:*) + test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) + OS_REL='' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; + NCR*:*:4.2:* | MPRAS*:*:4.2:*) + OS_REL='.3' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) + echo m68k-unknown-lynxos${UNAME_RELEASE} + exit ;; + mc68030:UNIX_System_V:4.*:*) + echo m68k-atari-sysv4 + exit ;; + TSUNAMI:LynxOS:2.*:*) + echo sparc-unknown-lynxos${UNAME_RELEASE} + exit ;; + rs6000:LynxOS:2.*:*) + echo rs6000-unknown-lynxos${UNAME_RELEASE} + exit ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) + echo powerpc-unknown-lynxos${UNAME_RELEASE} + exit ;; + SM[BE]S:UNIX_SV:*:*) + echo mips-dde-sysv${UNAME_RELEASE} + exit ;; + RM*:ReliantUNIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + RM*:SINIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + *:SINIX-*:*:*) + if uname -p 2>/dev/null >/dev/null ; then + UNAME_MACHINE=`(uname -p) 2>/dev/null` + echo ${UNAME_MACHINE}-sni-sysv4 + else + echo ns32k-sni-sysv + fi + exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says + echo i586-unisys-sysv4 + exit ;; + *:UNIX_System_V:4*:FTX*) + # From Gerald Hewes . + # How about differentiating between stratus architectures? -djm + echo hppa1.1-stratus-sysv4 + exit ;; + *:*:*:FTX*) + # From seanf@swdc.stratus.com. + echo i860-stratus-sysv4 + exit ;; + i*86:VOS:*:*) + # From Paul.Green@stratus.com. + echo ${UNAME_MACHINE}-stratus-vos + exit ;; + *:VOS:*:*) + # From Paul.Green@stratus.com. + echo hppa1.1-stratus-vos + exit ;; + mc68*:A/UX:*:*) + echo m68k-apple-aux${UNAME_RELEASE} + exit ;; + news*:NEWS-OS:6*:*) + echo mips-sony-newsos6 + exit ;; + R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) + if [ -d /usr/nec ]; then + echo mips-nec-sysv${UNAME_RELEASE} + else + echo mips-unknown-sysv${UNAME_RELEASE} + fi + exit ;; + BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. + echo powerpc-be-beos + exit ;; + BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. + echo powerpc-apple-beos + exit ;; + BePC:BeOS:*:*) # BeOS running on Intel PC compatible. + echo i586-pc-beos + exit ;; + BePC:Haiku:*:*) # Haiku running on Intel PC compatible. + echo i586-pc-haiku + exit ;; + SX-4:SUPER-UX:*:*) + echo sx4-nec-superux${UNAME_RELEASE} + exit ;; + SX-5:SUPER-UX:*:*) + echo sx5-nec-superux${UNAME_RELEASE} + exit ;; + SX-6:SUPER-UX:*:*) + echo sx6-nec-superux${UNAME_RELEASE} + exit ;; + SX-7:SUPER-UX:*:*) + echo sx7-nec-superux${UNAME_RELEASE} + exit ;; + SX-8:SUPER-UX:*:*) + echo sx8-nec-superux${UNAME_RELEASE} + exit ;; + SX-8R:SUPER-UX:*:*) + echo sx8r-nec-superux${UNAME_RELEASE} + exit ;; + Power*:Rhapsody:*:*) + echo powerpc-apple-rhapsody${UNAME_RELEASE} + exit ;; + *:Rhapsody:*:*) + echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + exit ;; + *:Darwin:*:*) + UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown + case $UNAME_PROCESSOR in + i386) + eval $set_cc_for_build + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + UNAME_PROCESSOR="x86_64" + fi + fi ;; + unknown) UNAME_PROCESSOR=powerpc ;; + esac + echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + exit ;; + *:procnto*:*:* | *:QNX:[0123456789]*:*) + UNAME_PROCESSOR=`uname -p` + if test "$UNAME_PROCESSOR" = "x86"; then + UNAME_PROCESSOR=i386 + UNAME_MACHINE=pc + fi + echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + exit ;; + *:QNX:*:4*) + echo i386-pc-qnx + exit ;; + NEO-?:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk${UNAME_RELEASE} + exit ;; + NSE-?:NONSTOP_KERNEL:*:*) + echo nse-tandem-nsk${UNAME_RELEASE} + exit ;; + NSR-?:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk${UNAME_RELEASE} + exit ;; + *:NonStop-UX:*:*) + echo mips-compaq-nonstopux + exit ;; + BS2000:POSIX*:*:*) + echo bs2000-siemens-sysv + exit ;; + DS/*:UNIX_System_V:*:*) + echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + exit ;; + *:Plan9:*:*) + # "uname -m" is not consistent, so use $cputype instead. 386 + # is converted to i386 for consistency with other x86 + # operating systems. + if test "$cputype" = "386"; then + UNAME_MACHINE=i386 + else + UNAME_MACHINE="$cputype" + fi + echo ${UNAME_MACHINE}-unknown-plan9 + exit ;; + *:TOPS-10:*:*) + echo pdp10-unknown-tops10 + exit ;; + *:TENEX:*:*) + echo pdp10-unknown-tenex + exit ;; + KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) + echo pdp10-dec-tops20 + exit ;; + XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) + echo pdp10-xkl-tops20 + exit ;; + *:TOPS-20:*:*) + echo pdp10-unknown-tops20 + exit ;; + *:ITS:*:*) + echo pdp10-unknown-its + exit ;; + SEI:*:*:SEIUX) + echo mips-sei-seiux${UNAME_RELEASE} + exit ;; + *:DragonFly:*:*) + echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + exit ;; + *:*VMS:*:*) + UNAME_MACHINE=`(uname -p) 2>/dev/null` + case "${UNAME_MACHINE}" in + A*) echo alpha-dec-vms ; exit ;; + I*) echo ia64-dec-vms ; exit ;; + V*) echo vax-dec-vms ; exit ;; + esac ;; + *:XENIX:*:SysV) + echo i386-pc-xenix + exit ;; + i*86:skyos:*:*) + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + exit ;; + i*86:rdos:*:*) + echo ${UNAME_MACHINE}-pc-rdos + exit ;; + i*86:AROS:*:*) + echo ${UNAME_MACHINE}-pc-aros + exit ;; + x86_64:VMkernel:*:*) + echo ${UNAME_MACHINE}-unknown-esx + exit ;; +esac + +#echo '(No uname command or uname output not recognized.)' 1>&2 +#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2 + +eval $set_cc_for_build +cat >$dummy.c < +# include +#endif +main () +{ +#if defined (sony) +#if defined (MIPSEB) + /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, + I don't know.... */ + printf ("mips-sony-bsd\n"); exit (0); +#else +#include + printf ("m68k-sony-newsos%s\n", +#ifdef NEWSOS4 + "4" +#else + "" +#endif + ); exit (0); +#endif +#endif + +#if defined (__arm) && defined (__acorn) && defined (__unix) + printf ("arm-acorn-riscix\n"); exit (0); +#endif + +#if defined (hp300) && !defined (hpux) + printf ("m68k-hp-bsd\n"); exit (0); +#endif + +#if defined (NeXT) +#if !defined (__ARCHITECTURE__) +#define __ARCHITECTURE__ "m68k" +#endif + int version; + version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; + if (version < 4) + printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); + else + printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); + exit (0); +#endif + +#if defined (MULTIMAX) || defined (n16) +#if defined (UMAXV) + printf ("ns32k-encore-sysv\n"); exit (0); +#else +#if defined (CMU) + printf ("ns32k-encore-mach\n"); exit (0); +#else + printf ("ns32k-encore-bsd\n"); exit (0); +#endif +#endif +#endif + +#if defined (__386BSD__) + printf ("i386-pc-bsd\n"); exit (0); +#endif + +#if defined (sequent) +#if defined (i386) + printf ("i386-sequent-dynix\n"); exit (0); +#endif +#if defined (ns32000) + printf ("ns32k-sequent-dynix\n"); exit (0); +#endif +#endif + +#if defined (_SEQUENT_) + struct utsname un; + + uname(&un); + + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); + +#endif + +#if defined (vax) +# if !defined (ultrix) +# include +# if defined (BSD) +# if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +# else +# if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +# else + printf ("vax-dec-bsd\n"); exit (0); +# endif +# endif +# else + printf ("vax-dec-bsd\n"); exit (0); +# endif +# else + printf ("vax-dec-ultrix\n"); exit (0); +# endif +#endif + +#if defined (alliant) && defined (i860) + printf ("i860-alliant-bsd\n"); exit (0); +#endif + + exit (1); +} +EOF + +$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } + +# Apollos put the system type in the environment. + +test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } + +# Convex versions that predate uname can use getsysinfo(1) + +if [ -x /usr/convex/getsysinfo ] +then + case `getsysinfo -f cpu_type` in + c1*) + echo c1-convex-bsd + exit ;; + c2*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit ;; + c34*) + echo c34-convex-bsd + exit ;; + c38*) + echo c38-convex-bsd + exit ;; + c4*) + echo c4-convex-bsd + exit ;; + esac +fi + +cat >&2 < in order to provide the needed +information to handle your system. + +config.guess timestamp = $timestamp + +uname -m = `(uname -m) 2>/dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null` + +hostinfo = `(hostinfo) 2>/dev/null` +/bin/universe = `(/bin/universe) 2>/dev/null` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` +/bin/arch = `(/bin/arch) 2>/dev/null` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` + +UNAME_MACHINE = ${UNAME_MACHINE} +UNAME_RELEASE = ${UNAME_RELEASE} +UNAME_SYSTEM = ${UNAME_SYSTEM} +UNAME_VERSION = ${UNAME_VERSION} +EOF + +exit 1 + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/ceph/config.sub b/ceph/config.sub new file mode 100755 index 00000000..c894da45 --- /dev/null +++ b/ceph/config.sub @@ -0,0 +1,1773 @@ +#! /bin/sh +# Configuration validation subroutine script. +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, +# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012 Free Software Foundation, Inc. + +timestamp='2012-02-10' + +# This file is (in principle) common to ALL GNU software. +# The presence of a machine in this file suggests that SOME GNU software +# can handle that machine. It does not imply ALL GNU software can. +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + + +# Please send patches to . Submit a context +# diff and a properly formatted GNU ChangeLog entry. +# +# Configuration subroutine to validate and canonicalize a configuration type. +# Supply the specified configuration type as an argument. +# If it is invalid, we print an error message on stderr and exit with code 1. +# Otherwise, we print the canonical config type on stdout and succeed. + +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + +# This file is supposed to be the same for all GNU packages +# and recognize all the CPU types, system types and aliases +# that are meaningful with *any* GNU software. +# Each package is responsible for reporting which valid configurations +# it does not support. The user should be able to distinguish +# a failure to support a valid configuration from a meaningless +# configuration. + +# The goal of this file is to map all the various variations of a given +# machine specification into a single specification in the form: +# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM +# or in some cases, the newer four-part form: +# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM +# It is wrong to echo any other type of specification. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] CPU-MFR-OPSYS + $0 [OPTION] ALIAS + +Canonicalize a configuration name. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.sub ($timestamp) + +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, +2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 +Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" + exit 1 ;; + + *local*) + # First pass through any local machine types. + echo $1 + exit ;; + + * ) + break ;; + esac +done + +case $# in + 0) echo "$me: missing argument$help" >&2 + exit 1;; + 1) ;; + *) echo "$me: too many arguments$help" >&2 + exit 1;; +esac + +# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). +# Here we must recognize all the valid KERNEL-OS combinations. +maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` +case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ + linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | \ + kopensolaris*-gnu* | \ + storm-chaos* | os2-emx* | rtmk-nova*) + os=-$maybe_os + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` + ;; + android-linux) + os=-linux-android + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown + ;; + *) + basic_machine=`echo $1 | sed 's/-[^-]*$//'` + if [ $basic_machine != $1 ] + then os=`echo $1 | sed 's/.*-/-/'` + else os=; fi + ;; +esac + +### Let's recognize common machines as not being operating systems so +### that things like config.sub decstation-3100 work. We also +### recognize some manufacturers as not being operating systems, so we +### can provide default operating systems below. +case $os in + -sun*os*) + # Prevent following clause from handling this invalid input. + ;; + -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ + -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ + -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ + -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ + -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ + -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ + -apple | -axis | -knuth | -cray | -microblaze) + os= + basic_machine=$1 + ;; + -bluegene*) + os=-cnk + ;; + -sim | -cisco | -oki | -wec | -winbond) + os= + basic_machine=$1 + ;; + -scout) + ;; + -wrs) + os=-vxworks + basic_machine=$1 + ;; + -chorusos*) + os=-chorusos + basic_machine=$1 + ;; + -chorusrdb) + os=-chorusrdb + basic_machine=$1 + ;; + -hiux*) + os=-hiuxwe2 + ;; + -sco6) + os=-sco5v6 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco5) + os=-sco3.2v5 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco4) + os=-sco3.2v4 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2v[4-9]*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco*) + os=-sco3.2v2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -udk*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -isc) + os=-isc2.2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -clix*) + basic_machine=clipper-intergraph + ;; + -isc*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -lynx*) + os=-lynxos + ;; + -ptx*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + ;; + -windowsnt*) + os=`echo $os | sed -e 's/windowsnt/winnt/'` + ;; + -psos*) + os=-psos + ;; + -mint | -mint[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Recognize the basic CPU types without company name. + # Some are omitted here because they have special meanings below. + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ + | am33_2.0 \ + | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \ + | be32 | be64 \ + | bfin \ + | c4x | clipper \ + | d10v | d30v | dlx | dsp16xx \ + | epiphany \ + | fido | fr30 | frv \ + | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i860 | i960 | ia64 \ + | ip2k | iq2000 \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | maxq | mb | microblaze | mcore | mep | metag \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipstx39 | mipstx39el \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nios | nios2 \ + | ns16k | ns32k \ + | open8 \ + | or32 \ + | pdp10 | pdp11 | pj | pjl \ + | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pyramid \ + | rl78 | rx \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu \ + | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ + | ubicom32 \ + | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ + | we32k \ + | x86 | xc16x | xstormy16 | xtensa \ + | z8k | z80) + basic_machine=$basic_machine-unknown + ;; + c54x) + basic_machine=tic54x-unknown + ;; + c55x) + basic_machine=tic55x-unknown + ;; + c6x) + basic_machine=tic6x-unknown + ;; + m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip) + basic_machine=$basic_machine-unknown + os=-none + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + ;; + ms1) + basic_machine=mt-unknown + ;; + + strongarm | thumb | xscale) + basic_machine=arm-unknown + ;; + xgate) + basic_machine=$basic_machine-unknown + os=-none + ;; + xscaleeb) + basic_machine=armeb-unknown + ;; + + xscaleel) + basic_machine=armel-unknown + ;; + + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + basic_machine=$basic_machine-pc + ;; + # Object if more than one company name word. + *-*-*) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; + # Recognize the basic CPU types with company name. + 580-* \ + | a29k-* \ + | aarch64-* | aarch64_be-* \ + | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ + | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ + | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ + | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ + | avr-* | avr32-* \ + | be32-* | be64-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* \ + | clipper-* | craynv-* | cydra-* \ + | d10v-* | d30v-* | dlx-* \ + | elxsi-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ + | h8300-* | h8500-* \ + | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | hexagon-* \ + | i*86-* | i860-* | i960-* | ia64-* \ + | ip2k-* | iq2000-* \ + | le32-* | le64-* \ + | lm32-* \ + | m32c-* | m32r-* | m32rle-* \ + | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ + | m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \ + | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ + | mips16-* \ + | mips64-* | mips64el-* \ + | mips64octeon-* | mips64octeonel-* \ + | mips64orion-* | mips64orionel-* \ + | mips64r5900-* | mips64r5900el-* \ + | mips64vr-* | mips64vrel-* \ + | mips64vr4100-* | mips64vr4100el-* \ + | mips64vr4300-* | mips64vr4300el-* \ + | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ + | mipsisa32-* | mipsisa32el-* \ + | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa64-* | mipsisa64el-* \ + | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64sb1-* | mipsisa64sb1el-* \ + | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mt-* \ + | msp430-* \ + | nds32-* | nds32le-* | nds32be-* \ + | nios-* | nios2-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | open8-* \ + | orion-* \ + | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pyramid-* \ + | rl78-* | romp-* | rs6000-* | rx-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ + | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | tahoe-* \ + | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tile*-* \ + | tron-* \ + | ubicom32-* \ + | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ + | vax-* \ + | we32k-* \ + | x86-* | x86_64-* | xc16x-* | xps100-* \ + | xstormy16-* | xtensa*-* \ + | ymp-* \ + | z8k-* | z80-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown + ;; + # Recognize the various machine names and aliases which stand + # for a CPU type and a company and sometimes even an OS. + 386bsd) + basic_machine=i386-unknown + os=-bsd + ;; + 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) + basic_machine=m68000-att + ;; + 3b*) + basic_machine=we32k-att + ;; + a29khif) + basic_machine=a29k-amd + os=-udi + ;; + abacus) + basic_machine=abacus-unknown + ;; + adobe68k) + basic_machine=m68010-adobe + os=-scout + ;; + alliant | fx80) + basic_machine=fx80-alliant + ;; + altos | altos3068) + basic_machine=m68k-altos + ;; + am29k) + basic_machine=a29k-none + os=-bsd + ;; + amd64) + basic_machine=x86_64-pc + ;; + amd64-*) + basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + amdahl) + basic_machine=580-amdahl + os=-sysv + ;; + amiga | amiga-*) + basic_machine=m68k-unknown + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=-amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=-sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=-sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=-bsd + ;; + aros) + basic_machine=i386-pc + os=-aros + ;; + aux) + basic_machine=m68k-apple + os=-aux + ;; + balance) + basic_machine=ns32k-sequent + os=-dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=-linux + ;; + blackfin-*) + basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + bluegene*) + basic_machine=powerpc-ibm + os=-cnk + ;; + c54x-*) + basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c55x-*) + basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c6x-*) + basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c90) + basic_machine=c90-cray + os=-unicos + ;; + cegcc) + basic_machine=arm-unknown + os=-cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=-bsd + ;; + convex-c2) + basic_machine=c2-convex + os=-bsd + ;; + convex-c32) + basic_machine=c32-convex + os=-bsd + ;; + convex-c34) + basic_machine=c34-convex + os=-bsd + ;; + convex-c38) + basic_machine=c38-convex + os=-bsd + ;; + cray | j90) + basic_machine=j90-cray + os=-unicos + ;; + craynv) + basic_machine=craynv-cray + os=-unicosmp + ;; + cr16 | cr16-*) + basic_machine=cr16-unknown + os=-elf + ;; + crds | unos) + basic_machine=m68k-crds + ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; + cris | cris-* | etrax*) + basic_machine=cris-axis + ;; + crx) + basic_machine=crx-unknown + os=-elf + ;; + da30 | da30-*) + basic_machine=m68k-da30 + ;; + decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) + basic_machine=mips-dec + ;; + decsystem10* | dec10*) + basic_machine=pdp10-dec + os=-tops10 + ;; + decsystem20* | dec20*) + basic_machine=pdp10-dec + os=-tops20 + ;; + delta | 3300 | motorola-3300 | motorola-delta \ + | 3300-motorola | delta-motorola) + basic_machine=m68k-motorola + ;; + delta88) + basic_machine=m88k-motorola + os=-sysv3 + ;; + dicos) + basic_machine=i686-pc + os=-dicos + ;; + djgpp) + basic_machine=i586-pc + os=-msdosdjgpp + ;; + dpx20 | dpx20-*) + basic_machine=rs6000-bull + os=-bosx + ;; + dpx2* | dpx2*-bull) + basic_machine=m68k-bull + os=-sysv3 + ;; + ebmon29k) + basic_machine=a29k-amd + os=-ebmon + ;; + elxsi) + basic_machine=elxsi-elxsi + os=-bsd + ;; + encore | umax | mmax) + basic_machine=ns32k-encore + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=-ose + ;; + fx2800) + basic_machine=i860-alliant + ;; + genix) + basic_machine=ns32k-ns + ;; + gmicro) + basic_machine=tron-gmicro + os=-sysv + ;; + go32) + basic_machine=i386-pc + os=-go32 + ;; + h3050r* | hiux*) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=-hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=-xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=-hms + ;; + harris) + basic_machine=m88k-harris + os=-sysv3 + ;; + hp300-*) + basic_machine=m68k-hp + ;; + hp300bsd) + basic_machine=m68k-hp + os=-bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=-hpux + ;; + hp3k9[0-9][0-9] | hp9[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k2[0-9][0-9] | hp9k31[0-9]) + basic_machine=m68000-hp + ;; + hp9k3[2-9][0-9]) + basic_machine=m68k-hp + ;; + hp9k6[0-9][0-9] | hp6[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k7[0-79][0-9] | hp7[0-79][0-9]) + basic_machine=hppa1.1-hp + ;; + hp9k78[0-9] | hp78[0-9]) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][13679] | hp8[0-9][13679]) + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][0-9] | hp8[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hppa-next) + os=-nextstep3 + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=-osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=-proelf + ;; + i370-ibm* | ibm*) + basic_machine=i370-ibm + ;; + i*86v32) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv32 + ;; + i*86v4*) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv4 + ;; + i*86v) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv + ;; + i*86sol2) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-solaris2 + ;; + i386mach) + basic_machine=i386-mach + os=-mach + ;; + i386-vsta | vsta) + basic_machine=i386-unknown + os=-vsta + ;; + iris | iris4d) + basic_machine=mips-sgi + case $os in + -irix*) + ;; + *) + os=-irix4 + ;; + esac + ;; + isi68 | isi) + basic_machine=m68k-isi + os=-sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=-linux + ;; + m68knommu-*) + basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + m88k-omron*) + basic_machine=m88k-omron + ;; + magnum | m3230) + basic_machine=mips-mips + os=-sysv + ;; + merlin) + basic_machine=ns32k-utek + os=-sysv + ;; + microblaze) + basic_machine=microblaze-xilinx + ;; + mingw32) + basic_machine=i386-pc + os=-mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=-mingw32ce + ;; + miniframe) + basic_machine=m68000-convergent + ;; + *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; + mips3*-*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` + ;; + mips3*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown + ;; + monitor) + basic_machine=m68k-rom68k + os=-coff + ;; + morphos) + basic_machine=powerpc-unknown + os=-morphos + ;; + msdos) + basic_machine=i386-pc + os=-msdos + ;; + ms1-*) + basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` + ;; + msys) + basic_machine=i386-pc + os=-msys + ;; + mvs) + basic_machine=i370-ibm + os=-mvs + ;; + nacl) + basic_machine=le32-unknown + os=-nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=-sysv4 + ;; + netbsd386) + basic_machine=i386-unknown + os=-netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=-linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=-newsos + ;; + news1000) + basic_machine=m68030-sony + os=-newsos + ;; + news-3600 | risc-news) + basic_machine=mips-sony + os=-newsos + ;; + necv70) + basic_machine=v70-nec + os=-sysv + ;; + next | m*-next ) + basic_machine=m68k-next + case $os in + -nextstep* ) + ;; + -ns2*) + os=-nextstep2 + ;; + *) + os=-nextstep3 + ;; + esac + ;; + nh3000) + basic_machine=m68k-harris + os=-cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=-cxux + ;; + nindy960) + basic_machine=i960-intel + os=-nindy + ;; + mon960) + basic_machine=i960-intel + os=-mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=-nonstopux + ;; + np1) + basic_machine=np1-gould + ;; + neo-tandem) + basic_machine=neo-tandem + ;; + nse-tandem) + basic_machine=nse-tandem + ;; + nsr-tandem) + basic_machine=nsr-tandem + ;; + op50n-* | op60c-*) + basic_machine=hppa1.1-oki + os=-proelf + ;; + openrisc | openrisc-*) + basic_machine=or32-unknown + ;; + os400) + basic_machine=powerpc-ibm + os=-os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=-ose + ;; + os68k) + basic_machine=m68k-none + os=-os68k + ;; + pa-hitachi) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + paragon) + basic_machine=i860-intel + os=-osf + ;; + parisc) + basic_machine=hppa-unknown + os=-linux + ;; + parisc-*) + basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + pbd) + basic_machine=sparc-tti + ;; + pbb) + basic_machine=m68k-tti + ;; + pc532 | pc532-*) + basic_machine=ns32k-pc532 + ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentium | p5 | k5 | k6 | nexgen | viac3) + basic_machine=i586-pc + ;; + pentiumpro | p6 | 6x86 | athlon | athlon_*) + basic_machine=i686-pc + ;; + pentiumii | pentium2 | pentiumiii | pentium3) + basic_machine=i686-pc + ;; + pentium4) + basic_machine=i786-pc + ;; + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumpro-* | p6-* | 6x86-* | athlon-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentium4-*) + basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pn) + basic_machine=pn-gould + ;; + power) basic_machine=power-ibm + ;; + ppc | ppcbe) basic_machine=powerpc-unknown + ;; + ppc-* | ppcbe-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppcle | powerpclittle | ppc-le | powerpc-little) + basic_machine=powerpcle-unknown + ;; + ppcle-* | powerpclittle-*) + basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64) basic_machine=powerpc64-unknown + ;; + ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64le | powerpc64little | ppc64-le | powerpc64-little) + basic_machine=powerpc64le-unknown + ;; + ppc64le-* | powerpc64little-*) + basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ps2) + basic_machine=i386-ibm + ;; + pw32) + basic_machine=i586-unknown + os=-pw32 + ;; + rdos) + basic_machine=i386-pc + os=-rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=-coff + ;; + rm[46]00) + basic_machine=mips-siemens + ;; + rtpc | rtpc-*) + basic_machine=romp-ibm + ;; + s390 | s390-*) + basic_machine=s390-ibm + ;; + s390x | s390x-*) + basic_machine=s390x-ibm + ;; + sa29200) + basic_machine=a29k-amd + os=-udi + ;; + sb1) + basic_machine=mipsisa64sb1-unknown + ;; + sb1el) + basic_machine=mipsisa64sb1el-unknown + ;; + sde) + basic_machine=mipsisa32-sde + os=-elf + ;; + sei) + basic_machine=mips-sei + os=-seiux + ;; + sequent) + basic_machine=i386-sequent + ;; + sh) + basic_machine=sh-hitachi + os=-hms + ;; + sh5el) + basic_machine=sh5le-unknown + ;; + sh64) + basic_machine=sh64-unknown + ;; + sparclite-wrs | simso-wrs) + basic_machine=sparclite-wrs + os=-vxworks + ;; + sps7) + basic_machine=m68k-bull + os=-sysv2 + ;; + spur) + basic_machine=spur-unknown + ;; + st2000) + basic_machine=m68k-tandem + ;; + stratus) + basic_machine=i860-stratus + os=-sysv4 + ;; + strongarm-* | thumb-*) + basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + sun2) + basic_machine=m68000-sun + ;; + sun2os3) + basic_machine=m68000-sun + os=-sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=-sunos4 + ;; + sun3os3) + basic_machine=m68k-sun + os=-sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=-sunos4 + ;; + sun4os3) + basic_machine=sparc-sun + os=-sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=-sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=-solaris2 + ;; + sun3 | sun3-*) + basic_machine=m68k-sun + ;; + sun4) + basic_machine=sparc-sun + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + ;; + sv1) + basic_machine=sv1-cray + os=-unicos + ;; + symmetry) + basic_machine=i386-sequent + os=-dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=-unicos + ;; + t90) + basic_machine=t90-cray + os=-unicos + ;; + tile*) + basic_machine=$basic_machine-unknown + os=-linux-gnu + ;; + tx39) + basic_machine=mipstx39-unknown + ;; + tx39el) + basic_machine=mipstx39el-unknown + ;; + toad1) + basic_machine=pdp10-xkl + os=-tops20 + ;; + tower | tower-32) + basic_machine=m68k-ncr + ;; + tpf) + basic_machine=s390x-ibm + os=-tpf + ;; + udi29k) + basic_machine=a29k-amd + os=-udi + ;; + ultra3) + basic_machine=a29k-nyu + os=-sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=-none + ;; + vaxv) + basic_machine=vax-dec + os=-sysv + ;; + vms) + basic_machine=vax-dec + os=-vms + ;; + vpp*|vx|vx-*) + basic_machine=f301-fujitsu + ;; + vxworks960) + basic_machine=i960-wrs + os=-vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=-vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=-vxworks + ;; + w65*) + basic_machine=w65-wdc + os=-none + ;; + w89k-*) + basic_machine=hppa1.1-winbond + os=-proelf + ;; + xbox) + basic_machine=i686-pc + os=-mingw32 + ;; + xps | xps100) + basic_machine=xps100-honeywell + ;; + xscale-* | xscalee[bl]-*) + basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + ;; + ymp) + basic_machine=ymp-cray + os=-unicos + ;; + z8k-*-coff) + basic_machine=z8k-unknown + os=-sim + ;; + z80-*-coff) + basic_machine=z80-unknown + os=-sim + ;; + none) + basic_machine=none-none + os=-none + ;; + +# Here we handle the default manufacturer of certain CPU types. It is in +# some cases the only manufacturer, in others, it is the most popular. + w89k) + basic_machine=hppa1.1-winbond + ;; + op50n) + basic_machine=hppa1.1-oki + ;; + op60c) + basic_machine=hppa1.1-oki + ;; + romp) + basic_machine=romp-ibm + ;; + mmix) + basic_machine=mmix-knuth + ;; + rs6000) + basic_machine=rs6000-ibm + ;; + vax) + basic_machine=vax-dec + ;; + pdp10) + # there are many clones, so DEC is not a safe bet + basic_machine=pdp10-unknown + ;; + pdp11) + basic_machine=pdp11-dec + ;; + we32k) + basic_machine=we32k-att + ;; + sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) + basic_machine=sh-unknown + ;; + sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) + basic_machine=sparc-sun + ;; + cydra) + basic_machine=cydra-cydrome + ;; + orion) + basic_machine=orion-highlevel + ;; + orion105) + basic_machine=clipper-highlevel + ;; + mac | mpw | mac-mpw) + basic_machine=m68k-apple + ;; + pmac | pmac-mpw) + basic_machine=powerpc-apple + ;; + *-unknown) + # Make sure to match an already-canonicalized machine name. + ;; + *) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; +esac + +# Here we canonicalize certain aliases for manufacturers. +case $basic_machine in + *-digital*) + basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` + ;; + *-commodore*) + basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + ;; + *) + ;; +esac + +# Decode manufacturer-specific aliases for certain operating systems. + +if [ x"$os" != x"" ] +then +case $os in + # First match some system type aliases + # that might get confused with valid system types. + # -solaris* is a basic system type, with this one exception. + -auroraux) + os=-auroraux + ;; + -solaris1 | -solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` + ;; + -solaris) + os=-solaris2 + ;; + -svr4*) + os=-sysv4 + ;; + -unixware*) + os=-sysv4.2uw + ;; + -gnu/linux*) + os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` + ;; + # First accept the basic system types. + # The portable systems comes first. + # Each alternative MUST END IN A *, to match a version number. + # -sysv* is not here because it comes later, after sysvr4. + -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ + | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ + | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ + | -sym* | -kopensolaris* \ + | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ + | -aos* | -aros* \ + | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ + | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ + | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ + | -openbsd* | -solidbsd* \ + | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ + | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ + | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ + | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ + | -chorusos* | -chorusrdb* | -cegcc* \ + | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ + | -mingw32* | -linux-gnu* | -linux-android* \ + | -linux-newlib* | -linux-uclibc* \ + | -uxpv* | -beos* | -mpeix* | -udk* \ + | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ + | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ + | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ + | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ + | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ + | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) + # Remember, each alternative MUST END IN *, to match a version number. + ;; + -qnx*) + case $basic_machine in + x86-* | i*86-*) + ;; + *) + os=-nto$os + ;; + esac + ;; + -nto-qnx*) + ;; + -nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` + ;; + -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ + | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ + | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + ;; + -mac*) + os=`echo $os | sed -e 's|mac|macos|'` + ;; + -linux-dietlibc) + os=-linux-dietlibc + ;; + -linux*) + os=`echo $os | sed -e 's|linux|linux-gnu|'` + ;; + -sunos5*) + os=`echo $os | sed -e 's|sunos5|solaris2|'` + ;; + -sunos6*) + os=`echo $os | sed -e 's|sunos6|solaris3|'` + ;; + -opened*) + os=-openedition + ;; + -os400*) + os=-os400 + ;; + -wince*) + os=-wince + ;; + -osfrose*) + os=-osfrose + ;; + -osf*) + os=-osf + ;; + -utek*) + os=-bsd + ;; + -dynix*) + os=-bsd + ;; + -acis*) + os=-aos + ;; + -atheos*) + os=-atheos + ;; + -syllable*) + os=-syllable + ;; + -386bsd) + os=-bsd + ;; + -ctix* | -uts*) + os=-sysv + ;; + -nova*) + os=-rtmk-nova + ;; + -ns2 ) + os=-nextstep2 + ;; + -nsk*) + os=-nsk + ;; + # Preserve the version number of sinix5. + -sinix5.*) + os=`echo $os | sed -e 's|sinix|sysv|'` + ;; + -sinix*) + os=-sysv4 + ;; + -tpf*) + os=-tpf + ;; + -triton*) + os=-sysv3 + ;; + -oss*) + os=-sysv3 + ;; + -svr4) + os=-sysv4 + ;; + -svr3) + os=-sysv3 + ;; + -sysvr4) + os=-sysv4 + ;; + # This must come after -sysvr4. + -sysv*) + ;; + -ose*) + os=-ose + ;; + -es1800*) + os=-ose + ;; + -xenix) + os=-xenix + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + os=-mint + ;; + -aros*) + os=-aros + ;; + -kaos*) + os=-kaos + ;; + -zvmoe) + os=-zvmoe + ;; + -dicos*) + os=-dicos + ;; + -nacl*) + ;; + -none) + ;; + *) + # Get rid of the `-' at the beginning of $os. + os=`echo $os | sed 's/[^-]*-//'` + echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + exit 1 + ;; +esac +else + +# Here we handle the default operating systems that come with various machines. +# The value should be what the vendor currently ships out the door with their +# machine or put another way, the most popular os provided with the machine. + +# Note that if you're going to try to match "-MANUFACTURER" here (say, +# "-sun"), then you have to tell the case statement up towards the top +# that MANUFACTURER isn't an operating system. Otherwise, code above +# will signal an error saying that MANUFACTURER isn't an operating +# system, and we'll never get to this point. + +case $basic_machine in + score-*) + os=-elf + ;; + spu-*) + os=-elf + ;; + *-acorn) + os=-riscix1.2 + ;; + arm*-rebel) + os=-linux + ;; + arm*-semi) + os=-aout + ;; + c4x-* | tic4x-*) + os=-coff + ;; + tic54x-*) + os=-coff + ;; + tic55x-*) + os=-coff + ;; + tic6x-*) + os=-coff + ;; + # This must come before the *-dec entry. + pdp10-*) + os=-tops20 + ;; + pdp11-*) + os=-none + ;; + *-dec | vax-*) + os=-ultrix4.2 + ;; + m68*-apollo) + os=-domain + ;; + i386-sun) + os=-sunos4.0.2 + ;; + m68000-sun) + os=-sunos3 + ;; + m68*-cisco) + os=-aout + ;; + mep-*) + os=-elf + ;; + mips*-cisco) + os=-elf + ;; + mips*-*) + os=-elf + ;; + or32-*) + os=-coff + ;; + *-tti) # must be before sparc entry or we get the wrong os. + os=-sysv3 + ;; + sparc-* | *-sun) + os=-sunos4.1.1 + ;; + *-be) + os=-beos + ;; + *-haiku) + os=-haiku + ;; + *-ibm) + os=-aix + ;; + *-knuth) + os=-mmixware + ;; + *-wec) + os=-proelf + ;; + *-winbond) + os=-proelf + ;; + *-oki) + os=-proelf + ;; + *-hp) + os=-hpux + ;; + *-hitachi) + os=-hiux + ;; + i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) + os=-sysv + ;; + *-cbm) + os=-amigaos + ;; + *-dg) + os=-dgux + ;; + *-dolphin) + os=-sysv3 + ;; + m68k-ccur) + os=-rtu + ;; + m88k-omron*) + os=-luna + ;; + *-next ) + os=-nextstep + ;; + *-sequent) + os=-ptx + ;; + *-crds) + os=-unos + ;; + *-ns) + os=-genix + ;; + i370-*) + os=-mvs + ;; + *-next) + os=-nextstep3 + ;; + *-gould) + os=-sysv + ;; + *-highlevel) + os=-bsd + ;; + *-encore) + os=-bsd + ;; + *-sgi) + os=-irix + ;; + *-siemens) + os=-sysv4 + ;; + *-masscomp) + os=-rtu + ;; + f30[01]-fujitsu | f700-fujitsu) + os=-uxpv + ;; + *-rom68k) + os=-coff + ;; + *-*bug) + os=-coff + ;; + *-apple) + os=-macos + ;; + *-atari*) + os=-mint + ;; + *) + os=-none + ;; +esac +fi + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +vendor=unknown +case $basic_machine in + *-unknown) + case $os in + -riscix*) + vendor=acorn + ;; + -sunos*) + vendor=sun + ;; + -cnk*|-aix*) + vendor=ibm + ;; + -beos*) + vendor=be + ;; + -hpux*) + vendor=hp + ;; + -mpeix*) + vendor=hp + ;; + -hiux*) + vendor=hitachi + ;; + -unos*) + vendor=crds + ;; + -dgux*) + vendor=dg + ;; + -luna*) + vendor=omron + ;; + -genix*) + vendor=ns + ;; + -mvs* | -opened*) + vendor=ibm + ;; + -os400*) + vendor=ibm + ;; + -ptx*) + vendor=sequent + ;; + -tpf*) + vendor=ibm + ;; + -vxsim* | -vxworks* | -windiss*) + vendor=wrs + ;; + -aux*) + vendor=apple + ;; + -hms*) + vendor=hitachi + ;; + -mpw* | -macos*) + vendor=apple + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + vendor=atari + ;; + -vos*) + vendor=stratus + ;; + esac + basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` + ;; +esac + +echo $basic_machine$os +exit + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/ceph/configure b/ceph/configure new file mode 100755 index 00000000..d8d4d5c5 --- /dev/null +++ b/ceph/configure @@ -0,0 +1,24717 @@ +#! /bin/sh +# Guess values for system-dependent variables and create Makefiles. +# Generated by GNU Autoconf 2.68 for ceph 0.80.7. +# +# Report bugs to . +# +# +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, +# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software +# Foundation, Inc. +# +# +# This configure script is free software; the Free Software Foundation +# gives unlimited permission to copy, distribute and modify it. +## -------------------- ## +## M4sh Initialization. ## +## -------------------- ## + +# Be more Bourne compatible +DUALCASE=1; export DUALCASE # for MKS sh +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in #( + *posix*) : + set -o posix ;; #( + *) : + ;; +esac +fi + + +as_nl=' +' +export as_nl +# Printing a long string crashes Solaris 7 /usr/bin/printf. +as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo +# Prefer a ksh shell builtin over an external printf program on Solaris, +# but without wasting forks for bash or zsh. +if test -z "$BASH_VERSION$ZSH_VERSION" \ + && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='print -r --' + as_echo_n='print -rn --' +elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='printf %s\n' + as_echo_n='printf %s' +else + if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then + as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' + as_echo_n='/usr/ucb/echo -n' + else + as_echo_body='eval expr "X$1" : "X\\(.*\\)"' + as_echo_n_body='eval + arg=$1; + case $arg in #( + *"$as_nl"*) + expr "X$arg" : "X\\(.*\\)$as_nl"; + arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; + esac; + expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" + ' + export as_echo_n_body + as_echo_n='sh -c $as_echo_n_body as_echo' + fi + export as_echo_body + as_echo='sh -c $as_echo_body as_echo' +fi + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + PATH_SEPARATOR=: + (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { + (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || + PATH_SEPARATOR=';' + } +fi + + +# IFS +# We need space, tab and new line, in precisely that order. Quoting is +# there to prevent editors from complaining about space-tab. +# (If _AS_PATH_WALK were called with IFS unset, it would disable word +# splitting by setting IFS to empty value.) +IFS=" "" $as_nl" + +# Find who we are. Look in the path if we contain no directory separator. +as_myself= +case $0 in #(( + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break + done +IFS=$as_save_IFS + + ;; +esac +# We did not find ourselves, most probably we were run as `sh COMMAND' +# in which case we are not to be found in the path. +if test "x$as_myself" = x; then + as_myself=$0 +fi +if test ! -f "$as_myself"; then + $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + exit 1 +fi + +# Unset variables that we do not need and which cause bugs (e.g. in +# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" +# suppresses any "Segmentation fault" message there. '((' could +# trigger a bug in pdksh 5.2.14. +for as_var in BASH_ENV ENV MAIL MAILPATH +do eval test x\${$as_var+set} = xset \ + && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : +done +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +LC_ALL=C +export LC_ALL +LANGUAGE=C +export LANGUAGE + +# CDPATH. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +if test "x$CONFIG_SHELL" = x; then + as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then : + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which + # is contrary to our usage. Disable this feature. + alias -g '\${1+\"\$@\"}'='\"\$@\"' + setopt NO_GLOB_SUBST +else + case \`(set -o) 2>/dev/null\` in #( + *posix*) : + set -o posix ;; #( + *) : + ;; +esac +fi +" + as_required="as_fn_return () { (exit \$1); } +as_fn_success () { as_fn_return 0; } +as_fn_failure () { as_fn_return 1; } +as_fn_ret_success () { return 0; } +as_fn_ret_failure () { return 1; } + +exitcode=0 +as_fn_success || { exitcode=1; echo as_fn_success failed.; } +as_fn_failure && { exitcode=1; echo as_fn_failure succeeded.; } +as_fn_ret_success || { exitcode=1; echo as_fn_ret_success failed.; } +as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; } +if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then : + +else + exitcode=1; echo positional parameters were not saved. +fi +test x\$exitcode = x0 || exit 1" + as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO + as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO + eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" && + test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1 + + test -n \"\${ZSH_VERSION+set}\${BASH_VERSION+set}\" || ( + ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' + ECHO=\$ECHO\$ECHO\$ECHO\$ECHO\$ECHO + ECHO=\$ECHO\$ECHO\$ECHO\$ECHO\$ECHO\$ECHO + PATH=/empty FPATH=/empty; export PATH FPATH + test \"X\`printf %s \$ECHO\`\" = \"X\$ECHO\" \\ + || test \"X\`print -r -- \$ECHO\`\" = \"X\$ECHO\" ) || exit 1 +test \$(( 1 + 1 )) = 2 || exit 1" + if (eval "$as_required") 2>/dev/null; then : + as_have_required=yes +else + as_have_required=no +fi + if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null; then : + +else + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +as_found=false +for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + as_found=: + case $as_dir in #( + /*) + for as_base in sh bash ksh sh5; do + # Try only shells that exist, to save several forks. + as_shell=$as_dir/$as_base + if { test -f "$as_shell" || test -f "$as_shell.exe"; } && + { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$as_shell"; } 2>/dev/null; then : + CONFIG_SHELL=$as_shell as_have_required=yes + if { $as_echo "$as_bourne_compatible""$as_suggested" | as_run=a "$as_shell"; } 2>/dev/null; then : + break 2 +fi +fi + done;; + esac + as_found=false +done +$as_found || { if { test -f "$SHELL" || test -f "$SHELL.exe"; } && + { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$SHELL"; } 2>/dev/null; then : + CONFIG_SHELL=$SHELL as_have_required=yes +fi; } +IFS=$as_save_IFS + + + if test "x$CONFIG_SHELL" != x; then : + # We cannot yet assume a decent shell, so we have to provide a + # neutralization value for shells without unset; and this also + # works around shells that cannot unset nonexistent variables. + # Preserve -v and -x to the replacement shell. + BASH_ENV=/dev/null + ENV=/dev/null + (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV + export CONFIG_SHELL + case $- in # (((( + *v*x* | *x*v* ) as_opts=-vx ;; + *v* ) as_opts=-v ;; + *x* ) as_opts=-x ;; + * ) as_opts= ;; + esac + exec "$CONFIG_SHELL" $as_opts "$as_myself" ${1+"$@"} +fi + + if test x$as_have_required = xno; then : + $as_echo "$0: This script requires a shell more modern than all" + $as_echo "$0: the shells that I found on your system." + if test x${ZSH_VERSION+set} = xset ; then + $as_echo "$0: In particular, zsh $ZSH_VERSION has bugs and should" + $as_echo "$0: be upgraded to zsh 4.3.4 or later." + else + $as_echo "$0: Please tell bug-autoconf@gnu.org and +$0: ceph-devel@vger.kernel.org about your system, including +$0: any error possibly output before this message. Then +$0: install a modern shell, or manually run the script +$0: under such a shell if you do have one." + fi + exit 1 +fi +fi +fi +SHELL=${CONFIG_SHELL-/bin/sh} +export SHELL +# Unset more variables known to interfere with behavior of common tools. +CLICOLOR_FORCE= GREP_OPTIONS= +unset CLICOLOR_FORCE GREP_OPTIONS + +## --------------------- ## +## M4sh Shell Functions. ## +## --------------------- ## +# as_fn_unset VAR +# --------------- +# Portably unset VAR. +as_fn_unset () +{ + { eval $1=; unset $1;} +} +as_unset=as_fn_unset + +# as_fn_set_status STATUS +# ----------------------- +# Set $? to STATUS, without forking. +as_fn_set_status () +{ + return $1 +} # as_fn_set_status + +# as_fn_exit STATUS +# ----------------- +# Exit the shell with STATUS, even in a "trap 0" or "set -e" context. +as_fn_exit () +{ + set +e + as_fn_set_status $1 + exit $1 +} # as_fn_exit + +# as_fn_mkdir_p +# ------------- +# Create "$as_dir" as a directory, including parents if necessary. +as_fn_mkdir_p () +{ + + case $as_dir in #( + -*) as_dir=./$as_dir;; + esac + test -d "$as_dir" || eval $as_mkdir_p || { + as_dirs= + while :; do + case $as_dir in #( + *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *) as_qdir=$as_dir;; + esac + as_dirs="'$as_qdir' $as_dirs" + as_dir=`$as_dirname -- "$as_dir" || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + test -d "$as_dir" && break + done + test -z "$as_dirs" || eval "mkdir $as_dirs" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" + + +} # as_fn_mkdir_p +# as_fn_append VAR VALUE +# ---------------------- +# Append the text in VALUE to the end of the definition contained in VAR. Take +# advantage of any shell optimizations that allow amortized linear growth over +# repeated appends, instead of the typical quadratic growth present in naive +# implementations. +if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : + eval 'as_fn_append () + { + eval $1+=\$2 + }' +else + as_fn_append () + { + eval $1=\$$1\$2 + } +fi # as_fn_append + +# as_fn_arith ARG... +# ------------------ +# Perform arithmetic evaluation on the ARGs, and store the result in the +# global $as_val. Take advantage of shells that can avoid forks. The arguments +# must be portable across $(()) and expr. +if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : + eval 'as_fn_arith () + { + as_val=$(( $* )) + }' +else + as_fn_arith () + { + as_val=`expr "$@" || test $? -eq 1` + } +fi # as_fn_arith + + +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- +# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are +# provided, also output the error to LOG_FD, referencing LINENO. Then exit the +# script with STATUS, using 1 if that was 0. +as_fn_error () +{ + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + fi + $as_echo "$as_me: error: $2" >&2 + as_fn_exit $as_status +} # as_fn_error + +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + +if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then + as_dirname=dirname +else + as_dirname=false +fi + +as_me=`$as_basename -- "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ + s//\1/ + q + } + /^X\/\(\/\/\)$/{ + s//\1/ + q + } + /^X\/\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + + + as_lineno_1=$LINENO as_lineno_1a=$LINENO + as_lineno_2=$LINENO as_lineno_2a=$LINENO + eval 'test "x$as_lineno_1'$as_run'" != "x$as_lineno_2'$as_run'" && + test "x`expr $as_lineno_1'$as_run' + 1`" = "x$as_lineno_2'$as_run'"' || { + # Blame Lee E. McMahon (1931-1989) for sed's syntax. :-) + sed -n ' + p + /[$]LINENO/= + ' <$as_myself | + sed ' + s/[$]LINENO.*/&-/ + t lineno + b + :lineno + N + :loop + s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/ + t loop + s/-\n.*// + ' >$as_me.lineno && + chmod +x "$as_me.lineno" || + { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } + + # Don't try to exec as it changes $[0], causing all sort of problems + # (the dirname of $[0] is not the place where we might find the + # original and so on. Autoconf is especially sensitive to this). + . "./$as_me.lineno" + # Exit status is that of the last command. + exit +} + +ECHO_C= ECHO_N= ECHO_T= +case `echo -n x` in #((((( +-n*) + case `echo 'xy\c'` in + *c*) ECHO_T=' ';; # ECHO_T is single tab character. + xy) ECHO_C='\c';; + *) echo `echo ksh88 bug on AIX 6.1` > /dev/null + ECHO_T=' ';; + esac;; +*) + ECHO_N='-n';; +esac + +rm -f conf$$ conf$$.exe conf$$.file +if test -d conf$$.dir; then + rm -f conf$$.dir/conf$$.file +else + rm -f conf$$.dir + mkdir conf$$.dir 2>/dev/null +fi +if (echo >conf$$.file) 2>/dev/null; then + if ln -s conf$$.file conf$$ 2>/dev/null; then + as_ln_s='ln -s' + # ... but there are two gotchas: + # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. + # In both cases, we have to default to `cp -p'. + ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || + as_ln_s='cp -p' + elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln + else + as_ln_s='cp -p' + fi +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file +rmdir conf$$.dir 2>/dev/null + +if mkdir -p . 2>/dev/null; then + as_mkdir_p='mkdir -p "$as_dir"' +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +if test -x / >/dev/null 2>&1; then + as_test_x='test -x' +else + if ls -dL / >/dev/null 2>&1; then + as_ls_L_option=L + else + as_ls_L_option= + fi + as_test_x=' + eval sh -c '\'' + if test -d "$1"; then + test -d "$1/."; + else + case $1 in #( + -*)set "./$1";; + esac; + case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #(( + ???[sx]*):;;*)false;;esac;fi + '\'' sh + ' +fi +as_executable_p=$as_test_x + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + +SHELL=${CONFIG_SHELL-/bin/sh} + + +test -n "$DJDIR" || exec 7<&0 &1 + +# Name of the host. +# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, +# so uname gets run too. +ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` + +# +# Initializations. +# +ac_default_prefix=/usr/local +ac_clean_files= +ac_config_libobj_dir=. +LIBOBJS= +cross_compiling=no +subdirs= +MFLAGS= +MAKEFLAGS= + +# Identity of this package. +PACKAGE_NAME='ceph' +PACKAGE_TARNAME='ceph' +PACKAGE_VERSION='0.80.7' +PACKAGE_STRING='ceph 0.80.7' +PACKAGE_BUGREPORT='ceph-devel@vger.kernel.org' +PACKAGE_URL='' + +enable_option_checking=no +# Factoring default headers for most tests. +ac_includes_default="\ +#include +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif +#ifdef STDC_HEADERS +# include +# include +#else +# ifdef HAVE_STDLIB_H +# include +# endif +#endif +#ifdef HAVE_STRING_H +# if !defined STDC_HEADERS && defined HAVE_MEMORY_H +# include +# endif +# include +#endif +#ifdef HAVE_STRINGS_H +# include +#endif +#ifdef HAVE_INTTYPES_H +# include +#endif +#ifdef HAVE_STDINT_H +# include +#endif +#ifdef HAVE_UNISTD_H +# include +#endif" + +ac_subst_vars='am__EXEEXT_FALSE +am__EXEEXT_TRUE +LTLIBOBJS +LIBOBJS +pkgpyexecdir +pyexecdir +pkgpythondir +pythondir +PYTHON_PLATFORM +PYTHON_EXEC_PREFIX +PYTHON_PREFIX +PYTHON_VERSION +PYTHON +WITH_BUILD_TESTS_FALSE +WITH_BUILD_TESTS_TRUE +BOOST_PROGRAM_OPTIONS_LIBS +USE_BOOST_SPIRIT_OLD_HDR_FALSE +USE_BOOST_SPIRIT_OLD_HDR_TRUE +WITH_LIBZFS_FALSE +WITH_LIBZFS_TRUE +LIBZFS_LIBS +LIBZFS_CFLAGS +WITH_LIBXFS_FALSE +WITH_LIBXFS_TRUE +WITH_LIBAIO_FALSE +WITH_LIBAIO_TRUE +WITH_REST_BENCH_FALSE +WITH_REST_BENCH_TRUE +WITH_SYSTEM_LIBS3_FALSE +WITH_SYSTEM_LIBS3_TRUE +INTEL_FLAGS +INTEL_SSE4_2_FLAGS +INTEL_SSE4_1_FLAGS +INTEL_PCLMUL_FLAGS +INTEL_SSSE3_FLAGS +INTEL_SSE3_FLAGS +INTEL_SSE2_FLAGS +INTEL_SSE_FLAGS +WITH_OCF_FALSE +WITH_OCF_TRUE +WITH_LIBATOMIC_FALSE +WITH_LIBATOMIC_TRUE +LIBEDIT_LIBS +LIBEDIT_CFLAGS +HAVE_JUNIT4_FALSE +HAVE_JUNIT4_TRUE +JDK_CPPFLAGS +JAR +JAVAH +JAVAC +EXTRA_CLASSPATH_JAR +ENABLE_CEPHFS_JAVA_FALSE +ENABLE_CEPHFS_JAVA_TRUE +WITH_TCMALLOC_FALSE +WITH_TCMALLOC_TRUE +LIBTCMALLOC +WITH_FUSE_FALSE +WITH_FUSE_TRUE +LIBFUSE +WITH_RADOSGW_FALSE +WITH_RADOSGW_TRUE +GCOV_PREFIX_STRIP +ENABLE_COVERAGE_FALSE +ENABLE_COVERAGE_TRUE +WITH_DEBUG_FALSE +WITH_DEBUG_TRUE +WITH_PROFILER_FALSE +WITH_PROFILER_TRUE +CRYPTO_LIBS +CRYPTO_CFLAGS +NSS_LIBS +NSS_CFLAGS +CRYPTOPP_LIBS +CRYPTOPP_CFLAGS +PKG_CONFIG_LIBDIR +PKG_CONFIG_PATH +PKG_CONFIG +KEYUTILS_LIB +RESOLV_LIBS +PTHREAD_CFLAGS +PTHREAD_LIBS +PTHREAD_CC +acx_pthread_config +COMPILER_HAS_VTA_FALSE +COMPILER_HAS_VTA_TRUE +ENABLE_FPU_NEON_FALSE +ENABLE_FPU_NEON_TRUE +WARN_IGNORED_QUALIFIERS +WARN_TYPE_LIMITS +WITH_GOOD_YASM_ELF64_FALSE +WITH_GOOD_YASM_ELF64_TRUE +AM_CXXFLAGS +CLANG_FALSE +CLANG_TRUE +CXXCPP +am__fastdepCXX_FALSE +am__fastdepCXX_TRUE +CXXDEPMODE +ac_ct_CXX +CXXFLAGS +CXX +FREEBSD_FALSE +FREEBSD_TRUE +LINUX_FALSE +LINUX_TRUE +AM_BACKSLASH +AM_DEFAULT_VERBOSITY +AM_DEFAULT_V +AM_V +am__fastdepCCAS_FALSE +am__fastdepCCAS_TRUE +CCASDEPMODE +CCASFLAGS +CCAS +CPP +OTOOL64 +OTOOL +LIPO +NMEDIT +DSYMUTIL +MANIFEST_TOOL +RANLIB +DLLTOOL +OBJDUMP +LN_S +NM +ac_ct_DUMPBIN +DUMPBIN +LD +FGREP +EGREP +GREP +SED +LIBTOOL +am__fastdepCC_FALSE +am__fastdepCC_TRUE +CCDEPMODE +am__nodep +AMDEPBACKSLASH +AMDEP_FALSE +AMDEP_TRUE +am__quote +am__include +DEPDIR +am__untar +am__tar +AMTAR +am__leading_dot +SET_MAKE +AWK +mkdir_p +MKDIR_P +INSTALL_STRIP_PROGRAM +STRIP +install_sh +MAKEINFO +AUTOHEADER +AUTOMAKE +AUTOCONF +ACLOCAL +VERSION +PACKAGE +CYGPATH_W +am__isrc +INSTALL_DATA +INSTALL_SCRIPT +INSTALL_PROGRAM +OBJEXT +EXEEXT +ac_ct_CC +CPPFLAGS +LDFLAGS +CFLAGS +CC +ac_ct_AR +AR +target_os +target_vendor +target_cpu +target +host_os +host_vendor +host_cpu +host +build_os +build_vendor +build_cpu +build +subdirs +GIT_CHECK +RPM_RELEASE +target_alias +host_alias +build_alias +LIBS +ECHO_T +ECHO_N +ECHO_C +DEFS +mandir +localedir +libdir +psdir +pdfdir +dvidir +htmldir +infodir +docdir +oldincludedir +includedir +localstatedir +sharedstatedir +sysconfdir +datadir +datarootdir +libexecdir +sbindir +bindir +program_transform_name +prefix +exec_prefix +PACKAGE_URL +PACKAGE_BUGREPORT +PACKAGE_STRING +PACKAGE_VERSION +PACKAGE_TARNAME +PACKAGE_NAME +PATH_SEPARATOR +SHELL' +ac_subst_files='' +ac_user_opts=' +enable_option_checking +enable_dependency_tracking +enable_shared +enable_static +with_pic +enable_fast_install +with_gnu_ld +with_sysroot +enable_libtool_lock +enable_silent_rules +with_cryptopp +with_nss +with_profiler +with_debug +enable_coverage +with_radosgw +with_fuse +with_tcmalloc +enable_pgrefdebugging +enable_cephfs_java +with_jdk_dir +with_libatomic_ops +with_ocf +with_system_libs3 +with_rest_bench +with_libaio +with_libxfs +with_libzfs +' + ac_precious_vars='build_alias +host_alias +target_alias +CC +CFLAGS +LDFLAGS +LIBS +CPPFLAGS +CPP +CCAS +CCASFLAGS +CXX +CXXFLAGS +CCC +CXXCPP +PKG_CONFIG +PKG_CONFIG_PATH +PKG_CONFIG_LIBDIR +CRYPTOPP_CFLAGS +CRYPTOPP_LIBS +NSS_CFLAGS +NSS_LIBS +LIBEDIT_CFLAGS +LIBEDIT_LIBS +LIBZFS_CFLAGS +LIBZFS_LIBS +PYTHON' +ac_subdirs_all='src/gtest' + +# Initialize some variables set by options. +ac_init_help= +ac_init_version=false +ac_unrecognized_opts= +ac_unrecognized_sep= +# The variables have the same names as the options, with +# dashes changed to underlines. +cache_file=/dev/null +exec_prefix=NONE +no_create= +no_recursion= +prefix=NONE +program_prefix=NONE +program_suffix=NONE +program_transform_name=s,x,x, +silent= +site= +srcdir= +verbose= +x_includes=NONE +x_libraries=NONE + +# Installation directory options. +# These are left unexpanded so users can "make install exec_prefix=/foo" +# and all the variables that are supposed to be based on exec_prefix +# by default will actually change. +# Use braces instead of parens because sh, perl, etc. also accept them. +# (The list follows the same order as the GNU Coding Standards.) +bindir='${exec_prefix}/bin' +sbindir='${exec_prefix}/sbin' +libexecdir='${exec_prefix}/libexec' +datarootdir='${prefix}/share' +datadir='${datarootdir}' +sysconfdir='${prefix}/etc' +sharedstatedir='${prefix}/com' +localstatedir='${prefix}/var' +includedir='${prefix}/include' +oldincludedir='/usr/include' +docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' +infodir='${datarootdir}/info' +htmldir='${docdir}' +dvidir='${docdir}' +pdfdir='${docdir}' +psdir='${docdir}' +libdir='${exec_prefix}/lib' +localedir='${datarootdir}/locale' +mandir='${datarootdir}/man' + +ac_prev= +ac_dashdash= +for ac_option +do + # If the previous option needs an argument, assign it. + if test -n "$ac_prev"; then + eval $ac_prev=\$ac_option + ac_prev= + continue + fi + + case $ac_option in + *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; + *=) ac_optarg= ;; + *) ac_optarg=yes ;; + esac + + # Accept the important Cygnus configure options, so we can diagnose typos. + + case $ac_dashdash$ac_option in + --) + ac_dashdash=yes ;; + + -bindir | --bindir | --bindi | --bind | --bin | --bi) + ac_prev=bindir ;; + -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) + bindir=$ac_optarg ;; + + -build | --build | --buil | --bui | --bu) + ac_prev=build_alias ;; + -build=* | --build=* | --buil=* | --bui=* | --bu=*) + build_alias=$ac_optarg ;; + + -cache-file | --cache-file | --cache-fil | --cache-fi \ + | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) + ac_prev=cache_file ;; + -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ + | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) + cache_file=$ac_optarg ;; + + --config-cache | -C) + cache_file=config.cache ;; + + -datadir | --datadir | --datadi | --datad) + ac_prev=datadir ;; + -datadir=* | --datadir=* | --datadi=* | --datad=*) + datadir=$ac_optarg ;; + + -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \ + | --dataroo | --dataro | --datar) + ac_prev=datarootdir ;; + -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \ + | --dataroot=* | --dataroo=* | --dataro=* | --datar=*) + datarootdir=$ac_optarg ;; + + -disable-* | --disable-*) + ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid feature name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"enable_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval enable_$ac_useropt=no ;; + + -docdir | --docdir | --docdi | --doc | --do) + ac_prev=docdir ;; + -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*) + docdir=$ac_optarg ;; + + -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv) + ac_prev=dvidir ;; + -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*) + dvidir=$ac_optarg ;; + + -enable-* | --enable-*) + ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid feature name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"enable_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval enable_$ac_useropt=\$ac_optarg ;; + + -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ + | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ + | --exec | --exe | --ex) + ac_prev=exec_prefix ;; + -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ + | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ + | --exec=* | --exe=* | --ex=*) + exec_prefix=$ac_optarg ;; + + -gas | --gas | --ga | --g) + # Obsolete; use --with-gas. + with_gas=yes ;; + + -help | --help | --hel | --he | -h) + ac_init_help=long ;; + -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) + ac_init_help=recursive ;; + -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) + ac_init_help=short ;; + + -host | --host | --hos | --ho) + ac_prev=host_alias ;; + -host=* | --host=* | --hos=* | --ho=*) + host_alias=$ac_optarg ;; + + -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht) + ac_prev=htmldir ;; + -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \ + | --ht=*) + htmldir=$ac_optarg ;; + + -includedir | --includedir | --includedi | --included | --include \ + | --includ | --inclu | --incl | --inc) + ac_prev=includedir ;; + -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ + | --includ=* | --inclu=* | --incl=* | --inc=*) + includedir=$ac_optarg ;; + + -infodir | --infodir | --infodi | --infod | --info | --inf) + ac_prev=infodir ;; + -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) + infodir=$ac_optarg ;; + + -libdir | --libdir | --libdi | --libd) + ac_prev=libdir ;; + -libdir=* | --libdir=* | --libdi=* | --libd=*) + libdir=$ac_optarg ;; + + -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ + | --libexe | --libex | --libe) + ac_prev=libexecdir ;; + -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ + | --libexe=* | --libex=* | --libe=*) + libexecdir=$ac_optarg ;; + + -localedir | --localedir | --localedi | --localed | --locale) + ac_prev=localedir ;; + -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*) + localedir=$ac_optarg ;; + + -localstatedir | --localstatedir | --localstatedi | --localstated \ + | --localstate | --localstat | --localsta | --localst | --locals) + ac_prev=localstatedir ;; + -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ + | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*) + localstatedir=$ac_optarg ;; + + -mandir | --mandir | --mandi | --mand | --man | --ma | --m) + ac_prev=mandir ;; + -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) + mandir=$ac_optarg ;; + + -nfp | --nfp | --nf) + # Obsolete; use --without-fp. + with_fp=no ;; + + -no-create | --no-create | --no-creat | --no-crea | --no-cre \ + | --no-cr | --no-c | -n) + no_create=yes ;; + + -no-recursion | --no-recursion | --no-recursio | --no-recursi \ + | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) + no_recursion=yes ;; + + -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ + | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ + | --oldin | --oldi | --old | --ol | --o) + ac_prev=oldincludedir ;; + -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ + | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ + | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) + oldincludedir=$ac_optarg ;; + + -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) + ac_prev=prefix ;; + -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) + prefix=$ac_optarg ;; + + -program-prefix | --program-prefix | --program-prefi | --program-pref \ + | --program-pre | --program-pr | --program-p) + ac_prev=program_prefix ;; + -program-prefix=* | --program-prefix=* | --program-prefi=* \ + | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) + program_prefix=$ac_optarg ;; + + -program-suffix | --program-suffix | --program-suffi | --program-suff \ + | --program-suf | --program-su | --program-s) + ac_prev=program_suffix ;; + -program-suffix=* | --program-suffix=* | --program-suffi=* \ + | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) + program_suffix=$ac_optarg ;; + + -program-transform-name | --program-transform-name \ + | --program-transform-nam | --program-transform-na \ + | --program-transform-n | --program-transform- \ + | --program-transform | --program-transfor \ + | --program-transfo | --program-transf \ + | --program-trans | --program-tran \ + | --progr-tra | --program-tr | --program-t) + ac_prev=program_transform_name ;; + -program-transform-name=* | --program-transform-name=* \ + | --program-transform-nam=* | --program-transform-na=* \ + | --program-transform-n=* | --program-transform-=* \ + | --program-transform=* | --program-transfor=* \ + | --program-transfo=* | --program-transf=* \ + | --program-trans=* | --program-tran=* \ + | --progr-tra=* | --program-tr=* | --program-t=*) + program_transform_name=$ac_optarg ;; + + -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd) + ac_prev=pdfdir ;; + -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*) + pdfdir=$ac_optarg ;; + + -psdir | --psdir | --psdi | --psd | --ps) + ac_prev=psdir ;; + -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*) + psdir=$ac_optarg ;; + + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + silent=yes ;; + + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) + ac_prev=sbindir ;; + -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ + | --sbi=* | --sb=*) + sbindir=$ac_optarg ;; + + -sharedstatedir | --sharedstatedir | --sharedstatedi \ + | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ + | --sharedst | --shareds | --shared | --share | --shar \ + | --sha | --sh) + ac_prev=sharedstatedir ;; + -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ + | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ + | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ + | --sha=* | --sh=*) + sharedstatedir=$ac_optarg ;; + + -site | --site | --sit) + ac_prev=site ;; + -site=* | --site=* | --sit=*) + site=$ac_optarg ;; + + -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) + ac_prev=srcdir ;; + -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) + srcdir=$ac_optarg ;; + + -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ + | --syscon | --sysco | --sysc | --sys | --sy) + ac_prev=sysconfdir ;; + -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ + | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) + sysconfdir=$ac_optarg ;; + + -target | --target | --targe | --targ | --tar | --ta | --t) + ac_prev=target_alias ;; + -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) + target_alias=$ac_optarg ;; + + -v | -verbose | --verbose | --verbos | --verbo | --verb) + verbose=yes ;; + + -version | --version | --versio | --versi | --vers | -V) + ac_init_version=: ;; + + -with-* | --with-*) + ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid package name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"with_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval with_$ac_useropt=\$ac_optarg ;; + + -without-* | --without-*) + ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid package name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"with_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval with_$ac_useropt=no ;; + + --x) + # Obsolete; use --with-x. + with_x=yes ;; + + -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ + | --x-incl | --x-inc | --x-in | --x-i) + ac_prev=x_includes ;; + -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ + | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) + x_includes=$ac_optarg ;; + + -x-libraries | --x-libraries | --x-librarie | --x-librari \ + | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) + ac_prev=x_libraries ;; + -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ + | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) + x_libraries=$ac_optarg ;; + + -*) as_fn_error $? "unrecognized option: \`$ac_option' +Try \`$0 --help' for more information" + ;; + + *=*) + ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` + # Reject names that are not valid shell variable names. + case $ac_envvar in #( + '' | [0-9]* | *[!_$as_cr_alnum]* ) + as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; + esac + eval $ac_envvar=\$ac_optarg + export $ac_envvar ;; + + *) + # FIXME: should be removed in autoconf 3.0. + $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 + expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && + $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 + : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" + ;; + + esac +done + +if test -n "$ac_prev"; then + ac_option=--`echo $ac_prev | sed 's/_/-/g'` + as_fn_error $? "missing argument to $ac_option" +fi + +if test -n "$ac_unrecognized_opts"; then + case $enable_option_checking in + no) ;; + fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; + *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; + esac +fi + +# Check all directory arguments for consistency. +for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ + datadir sysconfdir sharedstatedir localstatedir includedir \ + oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ + libdir localedir mandir +do + eval ac_val=\$$ac_var + # Remove trailing slashes. + case $ac_val in + */ ) + ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'` + eval $ac_var=\$ac_val;; + esac + # Be sure to have absolute directory names. + case $ac_val in + [\\/$]* | ?:[\\/]* ) continue;; + NONE | '' ) case $ac_var in *prefix ) continue;; esac;; + esac + as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" +done + +# There might be people who depend on the old broken behavior: `$host' +# used to hold the argument of --host etc. +# FIXME: To remove some day. +build=$build_alias +host=$host_alias +target=$target_alias + +# FIXME: To remove some day. +if test "x$host_alias" != x; then + if test "x$build_alias" = x; then + cross_compiling=maybe + $as_echo "$as_me: WARNING: if you wanted to set the --build type, don't use --host. + If a cross compiler is detected then cross compile mode will be used" >&2 + elif test "x$build_alias" != "x$host_alias"; then + cross_compiling=yes + fi +fi + +ac_tool_prefix= +test -n "$host_alias" && ac_tool_prefix=$host_alias- + +test "$silent" = yes && exec 6>/dev/null + + +ac_pwd=`pwd` && test -n "$ac_pwd" && +ac_ls_di=`ls -di .` && +ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || + as_fn_error $? "working directory cannot be determined" +test "X$ac_ls_di" = "X$ac_pwd_ls_di" || + as_fn_error $? "pwd does not report name of working directory" + + +# Find the source files, if location was not specified. +if test -z "$srcdir"; then + ac_srcdir_defaulted=yes + # Try the directory containing this script, then the parent directory. + ac_confdir=`$as_dirname -- "$as_myself" || +$as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_myself" : 'X\(//\)[^/]' \| \ + X"$as_myself" : 'X\(//\)$' \| \ + X"$as_myself" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_myself" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + srcdir=$ac_confdir + if test ! -r "$srcdir/$ac_unique_file"; then + srcdir=.. + fi +else + ac_srcdir_defaulted=no +fi +if test ! -r "$srcdir/$ac_unique_file"; then + test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." + as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" +fi +ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" +ac_abs_confdir=`( + cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" + pwd)` +# When building in place, set srcdir=. +if test "$ac_abs_confdir" = "$ac_pwd"; then + srcdir=. +fi +# Remove unnecessary trailing slashes from srcdir. +# Double slashes in file names in object file debugging info +# mess up M-x gdb in Emacs. +case $srcdir in +*/) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;; +esac +for ac_var in $ac_precious_vars; do + eval ac_env_${ac_var}_set=\${${ac_var}+set} + eval ac_env_${ac_var}_value=\$${ac_var} + eval ac_cv_env_${ac_var}_set=\${${ac_var}+set} + eval ac_cv_env_${ac_var}_value=\$${ac_var} +done + +# +# Report the --help message. +# +if test "$ac_init_help" = "long"; then + # Omit some internal or obsolete options to make the list less imposing. + # This message is too long to be a string in the A/UX 3.1 sh. + cat <<_ACEOF +\`configure' configures ceph 0.80.7 to adapt to many kinds of systems. + +Usage: $0 [OPTION]... [VAR=VALUE]... + +To assign environment variables (e.g., CC, CFLAGS...), specify them as +VAR=VALUE. See below for descriptions of some of the useful variables. + +Defaults for the options are specified in brackets. + +Configuration: + -h, --help display this help and exit + --help=short display options specific to this package + --help=recursive display the short help of all the included packages + -V, --version display version information and exit + -q, --quiet, --silent do not print \`checking ...' messages + --cache-file=FILE cache test results in FILE [disabled] + -C, --config-cache alias for \`--cache-file=config.cache' + -n, --no-create do not create output files + --srcdir=DIR find the sources in DIR [configure dir or \`..'] + +Installation directories: + --prefix=PREFIX install architecture-independent files in PREFIX + [$ac_default_prefix] + --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX + [PREFIX] + +By default, \`make install' will install all the files in +\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify +an installation prefix other than \`$ac_default_prefix' using \`--prefix', +for instance \`--prefix=\$HOME'. + +For better control, use the options below. + +Fine tuning of the installation directories: + --bindir=DIR user executables [EPREFIX/bin] + --sbindir=DIR system admin executables [EPREFIX/sbin] + --libexecdir=DIR program executables [EPREFIX/libexec] + --sysconfdir=DIR read-only single-machine data [PREFIX/etc] + --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] + --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --libdir=DIR object code libraries [EPREFIX/lib] + --includedir=DIR C header files [PREFIX/include] + --oldincludedir=DIR C header files for non-gcc [/usr/include] + --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] + --datadir=DIR read-only architecture-independent data [DATAROOTDIR] + --infodir=DIR info documentation [DATAROOTDIR/info] + --localedir=DIR locale-dependent data [DATAROOTDIR/locale] + --mandir=DIR man documentation [DATAROOTDIR/man] + --docdir=DIR documentation root [DATAROOTDIR/doc/ceph] + --htmldir=DIR html documentation [DOCDIR] + --dvidir=DIR dvi documentation [DOCDIR] + --pdfdir=DIR pdf documentation [DOCDIR] + --psdir=DIR ps documentation [DOCDIR] +_ACEOF + + cat <<\_ACEOF + +Program names: + --program-prefix=PREFIX prepend PREFIX to installed program names + --program-suffix=SUFFIX append SUFFIX to installed program names + --program-transform-name=PROGRAM run sed PROGRAM on installed program names + +System types: + --build=BUILD configure for building on BUILD [guessed] + --host=HOST cross-compile to build programs to run on HOST [BUILD] + --target=TARGET configure for building compilers for TARGET [HOST] +_ACEOF +fi + +if test -n "$ac_init_help"; then + case $ac_init_help in + short | recursive ) echo "Configuration of ceph 0.80.7:";; + esac + cat <<\_ACEOF + +Optional Features: + --disable-option-checking ignore unrecognized --enable/--with options + --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) + --enable-FEATURE[=ARG] include FEATURE [ARG=yes] + --disable-dependency-tracking speeds up one-time build + --enable-dependency-tracking do not reject slow dependency extractors + --enable-shared[=PKGS] build shared libraries [default=yes] + --enable-static[=PKGS] build static libraries [default=yes] + --enable-fast-install[=PKGS] + optimize for fast installation [default=yes] + --disable-libtool-lock avoid locking (might break parallel builds) + --enable-silent-rules less verbose build output (undo: `make V=1') + --disable-silent-rules verbose build output (undo: `make V=0') + --enable-coverage enable code coverage tracking + --enable-pgrefdebugging enable pg ref debugging + --enable-cephfs-java build libcephfs Java bindings + +Optional Packages: + --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] + --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) + --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use + both] + --with-gnu-ld assume the C compiler uses GNU ld [default=no] + --with-sysroot=DIR Search for dependent libraries within DIR + (or the compiler's sysroot if not specified). + --with-cryptopp Use cryptographic functions from cryptopp + --with-nss Use cryptographic functions from nss + --with-profiler build extra profiler binaries + --with-debug build extra debug binaries + --with-radosgw build RADOS gateway + --without-fuse disable FUSE userspace client + --without-tcmalloc disable tcmalloc for memory allocations + --with-jdk-dir(=DIR) Path to JDK directory + --without-libatomic-ops disable libatomic-ops for the atomic_t type + --with-ocf build OCF-compliant cluster resource agent + --with-system-libs3 use system libs3 + --with-rest-bench enables rest-bench + --without-libaio disable libaio use by journal + --without-libxfs disable libxfs use by FileStore + --with-libzfs build ZFS support + +Some influential environment variables: + CC C compiler command + CFLAGS C compiler flags + LDFLAGS linker flags, e.g. -L if you have libraries in a + nonstandard directory + LIBS libraries to pass to the linker, e.g. -l + CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if + you have headers in a nonstandard directory + CPP C preprocessor + CCAS assembler compiler command (defaults to CC) + CCASFLAGS assembler compiler flags (defaults to CFLAGS) + CXX C++ compiler command + CXXFLAGS C++ compiler flags + CXXCPP C++ preprocessor + PKG_CONFIG path to pkg-config utility + PKG_CONFIG_PATH + directories to add to pkg-config's search path + PKG_CONFIG_LIBDIR + path overriding pkg-config's built-in search path + CRYPTOPP_CFLAGS + C compiler flags for CRYPTOPP, overriding pkg-config + CRYPTOPP_LIBS + linker flags for CRYPTOPP, overriding pkg-config + NSS_CFLAGS C compiler flags for NSS, overriding pkg-config + NSS_LIBS linker flags for NSS, overriding pkg-config + LIBEDIT_CFLAGS + C compiler flags for LIBEDIT, overriding pkg-config + LIBEDIT_LIBS + linker flags for LIBEDIT, overriding pkg-config + LIBZFS_CFLAGS + C compiler flags for LIBZFS, overriding pkg-config + LIBZFS_LIBS linker flags for LIBZFS, overriding pkg-config + PYTHON the Python interpreter + +Use these variables to override the choices made by `configure' or to help +it to find libraries and programs with nonstandard names/locations. + +Report bugs to . +_ACEOF +ac_status=$? +fi + +if test "$ac_init_help" = "recursive"; then + # If there are subdirs, report their specific --help. + for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue + test -d "$ac_dir" || + { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } || + continue + ac_builddir=. + +case "$ac_dir" in +.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; +*) + ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` + # A ".." for each directory in $ac_dir_suffix. + ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + case $ac_top_builddir_sub in + "") ac_top_builddir_sub=. ac_top_build_prefix= ;; + *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; + esac ;; +esac +ac_abs_top_builddir=$ac_pwd +ac_abs_builddir=$ac_pwd$ac_dir_suffix +# for backward compatibility: +ac_top_builddir=$ac_top_build_prefix + +case $srcdir in + .) # We are building in place. + ac_srcdir=. + ac_top_srcdir=$ac_top_builddir_sub + ac_abs_top_srcdir=$ac_pwd ;; + [\\/]* | ?:[\\/]* ) # Absolute name. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir + ac_abs_top_srcdir=$srcdir ;; + *) # Relative name. + ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_build_prefix$srcdir + ac_abs_top_srcdir=$ac_pwd/$srcdir ;; +esac +ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix + + cd "$ac_dir" || { ac_status=$?; continue; } + # Check for guested configure. + if test -f "$ac_srcdir/configure.gnu"; then + echo && + $SHELL "$ac_srcdir/configure.gnu" --help=recursive + elif test -f "$ac_srcdir/configure"; then + echo && + $SHELL "$ac_srcdir/configure" --help=recursive + else + $as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 + fi || ac_status=$? + cd "$ac_pwd" || { ac_status=$?; break; } + done +fi + +test -n "$ac_init_help" && exit $ac_status +if $ac_init_version; then + cat <<\_ACEOF +ceph configure 0.80.7 +generated by GNU Autoconf 2.68 + +Copyright (C) 2010 Free Software Foundation, Inc. +This configure script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it. +_ACEOF + exit +fi + +## ------------------------ ## +## Autoconf initialization. ## +## ------------------------ ## + +# ac_fn_c_try_compile LINENO +# -------------------------- +# Try to compile conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext + if { { ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compile") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_compile + +# ac_fn_c_try_link LINENO +# ----------------------- +# Try to link conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_link () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext conftest$ac_exeext + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information + # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would + # interfere with the next link command; also delete a directory that is + # left behind by Apple's compiler. We do this before executing the actions. + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_link + +# ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES +# ------------------------------------------------------- +# Tests whether HEADER exists and can be compiled using the include files in +# INCLUDES, setting the cache variable VAR accordingly. +ac_fn_c_check_header_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_header_compile + +# ac_fn_c_try_cpp LINENO +# ---------------------- +# Try to preprocess conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_cpp () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } > conftest.i && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || + test ! -s conftest.err + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_cpp + +# ac_fn_c_try_run LINENO +# ---------------------- +# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes +# that executables *can* be run. +ac_fn_c_try_run () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; }; then : + ac_retval=0 +else + $as_echo "$as_me: program exited with status $ac_status" >&5 + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=$ac_status +fi + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_run + +# ac_fn_c_check_func LINENO FUNC VAR +# ---------------------------------- +# Tests whether FUNC exists, setting the cache variable VAR accordingly +ac_fn_c_check_func () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +/* Define $2 to an innocuous variant, in case declares $2. + For example, HP-UX 11i declares gettimeofday. */ +#define $2 innocuous_$2 + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char $2 (); below. + Prefer to if __STDC__ is defined, since + exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include +#else +# include +#endif + +#undef $2 + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char $2 (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined __stub_$2 || defined __stub___$2 +choke me +#endif + +int +main () +{ +return $2 (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_func + +# ac_fn_cxx_try_compile LINENO +# ---------------------------- +# Try to compile conftest.$ac_ext, and return whether this succeeded. +ac_fn_cxx_try_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext + if { { ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compile") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_compile + +# ac_fn_cxx_try_cpp LINENO +# ------------------------ +# Try to preprocess conftest.$ac_ext, and return whether this succeeded. +ac_fn_cxx_try_cpp () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } > conftest.i && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_cpp + +# ac_fn_cxx_try_link LINENO +# ------------------------- +# Try to link conftest.$ac_ext, and return whether this succeeded. +ac_fn_cxx_try_link () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext conftest$ac_exeext + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information + # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would + # interfere with the next link command; also delete a directory that is + # left behind by Apple's compiler. We do this before executing the actions. + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_link + +# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES +# ------------------------------------------------------- +# Tests whether HEADER exists, giving a warning if it cannot be compiled using +# the include files in INCLUDES and setting the cache variable VAR +# accordingly. +ac_fn_c_check_header_mongrel () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if eval \${$3+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +else + # Is the header compilable? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 +$as_echo_n "checking $2 usability... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_header_compiler=yes +else + ac_header_compiler=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 +$as_echo_n "checking $2 presence... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <$2> +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + ac_header_preproc=yes +else + ac_header_preproc=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( + yes:no: ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; + no:yes:* ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} +( $as_echo "## ----------------------------------------- ## +## Report this to ceph-devel@vger.kernel.org ## +## ----------------------------------------- ##" + ) | sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=\$ac_header_compiler" +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_header_mongrel + +# ac_fn_c_compute_int LINENO EXPR VAR INCLUDES +# -------------------------------------------- +# Tries to find the compile-time value of EXPR in a program that includes +# INCLUDES, setting VAR accordingly. Returns whether the value could be +# computed +ac_fn_c_compute_int () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if test "$cross_compiling" = yes; then + # Depending upon the size, compute the lo and hi bounds. +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +static int test_array [1 - 2 * !(($2) >= 0)]; +test_array [0] = 0 + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_lo=0 ac_mid=0 + while :; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +static int test_array [1 - 2 * !(($2) <= $ac_mid)]; +test_array [0] = 0 + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_hi=$ac_mid; break +else + as_fn_arith $ac_mid + 1 && ac_lo=$as_val + if test $ac_lo -le $ac_mid; then + ac_lo= ac_hi= + break + fi + as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + done +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +static int test_array [1 - 2 * !(($2) < 0)]; +test_array [0] = 0 + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_hi=-1 ac_mid=-1 + while :; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +static int test_array [1 - 2 * !(($2) >= $ac_mid)]; +test_array [0] = 0 + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_lo=$ac_mid; break +else + as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val + if test $ac_mid -le $ac_hi; then + ac_lo= ac_hi= + break + fi + as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + done +else + ac_lo= ac_hi= +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +# Binary search between lo and hi bounds. +while test "x$ac_lo" != "x$ac_hi"; do + as_fn_arith '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo && ac_mid=$as_val + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +static int test_array [1 - 2 * !(($2) <= $ac_mid)]; +test_array [0] = 0 + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_hi=$ac_mid +else + as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +done +case $ac_lo in #(( +?*) eval "$3=\$ac_lo"; ac_retval=0 ;; +'') ac_retval=1 ;; +esac + else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +static long int longval () { return $2; } +static unsigned long int ulongval () { return $2; } +#include +#include +int +main () +{ + + FILE *f = fopen ("conftest.val", "w"); + if (! f) + return 1; + if (($2) < 0) + { + long int i = longval (); + if (i != ($2)) + return 1; + fprintf (f, "%ld", i); + } + else + { + unsigned long int i = ulongval (); + if (i != ($2)) + return 1; + fprintf (f, "%lu", i); + } + /* Do not output a trailing newline, as this causes \r\n confusion + on some platforms. */ + return ferror (f) || fclose (f) != 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_run "$LINENO"; then : + echo >>conftest.val; read $3 &5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +else + # Is the header compilable? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 +$as_echo_n "checking $2 usability... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_header_compiler=yes +else + ac_header_compiler=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 +$as_echo_n "checking $2 presence... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <$2> +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + ac_header_preproc=yes +else + ac_header_preproc=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_cxx_preproc_warn_flag in #(( + yes:no: ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; + no:yes:* ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} +( $as_echo "## ----------------------------------------- ## +## Report this to ceph-devel@vger.kernel.org ## +## ----------------------------------------- ##" + ) | sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=\$ac_header_compiler" +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_cxx_check_header_mongrel + +# ac_fn_c_check_member LINENO AGGR MEMBER VAR INCLUDES +# ---------------------------------------------------- +# Tries to find if the field MEMBER exists in type AGGR, after including +# INCLUDES, setting cache variable VAR accordingly. +ac_fn_c_check_member () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2.$3" >&5 +$as_echo_n "checking for $2.$3... " >&6; } +if eval \${$4+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$5 +int +main () +{ +static $2 ac_aggr; +if (ac_aggr.$3) +return 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + eval "$4=yes" +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$5 +int +main () +{ +static $2 ac_aggr; +if (sizeof ac_aggr.$3) +return 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + eval "$4=yes" +else + eval "$4=no" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$4 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_member + +# ac_fn_c_check_type LINENO TYPE VAR INCLUDES +# ------------------------------------------- +# Tests whether TYPE exists after having included INCLUDES, setting cache +# variable VAR accordingly. +ac_fn_c_check_type () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=no" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +if (sizeof ($2)) + return 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +if (sizeof (($2))) + return 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +else + eval "$3=yes" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_type + +# ac_fn_cxx_check_decl LINENO SYMBOL VAR INCLUDES +# ----------------------------------------------- +# Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR +# accordingly. +ac_fn_cxx_check_decl () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + as_decl_name=`echo $2|sed 's/ *(.*//'` + as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5 +$as_echo_n "checking whether $as_decl_name is declared... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +#ifndef $as_decl_name +#ifdef __cplusplus + (void) $as_decl_use; +#else + (void) $as_decl_name; +#endif +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_cxx_check_decl + +# ac_fn_cxx_check_func LINENO FUNC VAR +# ------------------------------------ +# Tests whether FUNC exists, setting the cache variable VAR accordingly +ac_fn_cxx_check_func () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +/* Define $2 to an innocuous variant, in case declares $2. + For example, HP-UX 11i declares gettimeofday. */ +#define $2 innocuous_$2 + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char $2 (); below. + Prefer to if __STDC__ is defined, since + exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include +#else +# include +#endif + +#undef $2 + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char $2 (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined __stub_$2 || defined __stub___$2 +choke me +#endif + +int +main () +{ +return $2 (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_cxx_check_func + +# ac_fn_cxx_try_run LINENO +# ------------------------ +# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes +# that executables *can* be run. +ac_fn_cxx_try_run () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; }; then : + ac_retval=0 +else + $as_echo "$as_me: program exited with status $ac_status" >&5 + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=$ac_status +fi + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_run +cat >config.log <<_ACEOF +This file contains any messages produced by compilers while +running configure, to aid debugging if configure makes a mistake. + +It was created by ceph $as_me 0.80.7, which was +generated by GNU Autoconf 2.68. Invocation command line was + + $ $0 $@ + +_ACEOF +exec 5>>config.log +{ +cat <<_ASUNAME +## --------- ## +## Platform. ## +## --------- ## + +hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` +uname -m = `(uname -m) 2>/dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` + +/bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` +/usr/bin/hostinfo = `(/usr/bin/hostinfo) 2>/dev/null || echo unknown` +/bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` +/bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` + +_ASUNAME + +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + $as_echo "PATH: $as_dir" + done +IFS=$as_save_IFS + +} >&5 + +cat >&5 <<_ACEOF + + +## ----------- ## +## Core tests. ## +## ----------- ## + +_ACEOF + + +# Keep a trace of the command line. +# Strip out --no-create and --no-recursion so they do not pile up. +# Strip out --silent because we don't want to record it for future runs. +# Also quote any args containing shell meta-characters. +# Make two passes to allow for proper duplicate-argument suppression. +ac_configure_args= +ac_configure_args0= +ac_configure_args1= +ac_must_keep_next=false +for ac_pass in 1 2 +do + for ac_arg + do + case $ac_arg in + -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + continue ;; + *\'*) + ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + case $ac_pass in + 1) as_fn_append ac_configure_args0 " '$ac_arg'" ;; + 2) + as_fn_append ac_configure_args1 " '$ac_arg'" + if test $ac_must_keep_next = true; then + ac_must_keep_next=false # Got value, back to normal. + else + case $ac_arg in + *=* | --config-cache | -C | -disable-* | --disable-* \ + | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ + | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ + | -with-* | --with-* | -without-* | --without-* | --x) + case "$ac_configure_args0 " in + "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; + esac + ;; + -* ) ac_must_keep_next=true ;; + esac + fi + as_fn_append ac_configure_args " '$ac_arg'" + ;; + esac + done +done +{ ac_configure_args0=; unset ac_configure_args0;} +{ ac_configure_args1=; unset ac_configure_args1;} + +# When interrupted or exit'd, cleanup temporary files, and complete +# config.log. We remove comments because anyway the quotes in there +# would cause problems or look ugly. +# WARNING: Use '\'' to represent an apostrophe within the trap. +# WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug. +trap 'exit_status=$? + # Save into config.log some information that might help in debugging. + { + echo + + $as_echo "## ---------------- ## +## Cache variables. ## +## ---------------- ##" + echo + # The following way of writing the cache mishandles newlines in values, +( + for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do + eval ac_val=\$$ac_var + case $ac_val in #( + *${as_nl}*) + case $ac_var in #( + *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 +$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; + esac + case $ac_var in #( + _ | IFS | as_nl) ;; #( + BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( + *) { eval $ac_var=; unset $ac_var;} ;; + esac ;; + esac + done + (set) 2>&1 | + case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #( + *${as_nl}ac_space=\ *) + sed -n \ + "s/'\''/'\''\\\\'\'''\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p" + ;; #( + *) + sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" + ;; + esac | + sort +) + echo + + $as_echo "## ----------------- ## +## Output variables. ## +## ----------------- ##" + echo + for ac_var in $ac_subst_vars + do + eval ac_val=\$$ac_var + case $ac_val in + *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; + esac + $as_echo "$ac_var='\''$ac_val'\''" + done | sort + echo + + if test -n "$ac_subst_files"; then + $as_echo "## ------------------- ## +## File substitutions. ## +## ------------------- ##" + echo + for ac_var in $ac_subst_files + do + eval ac_val=\$$ac_var + case $ac_val in + *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; + esac + $as_echo "$ac_var='\''$ac_val'\''" + done | sort + echo + fi + + if test -s confdefs.h; then + $as_echo "## ----------- ## +## confdefs.h. ## +## ----------- ##" + echo + cat confdefs.h + echo + fi + test "$ac_signal" != 0 && + $as_echo "$as_me: caught signal $ac_signal" + $as_echo "$as_me: exit $exit_status" + } >&5 + rm -f core *.core core.conftest.* && + rm -f -r conftest* confdefs* conf$$* $ac_clean_files && + exit $exit_status +' 0 +for ac_signal in 1 2 13 15; do + trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal +done +ac_signal=0 + +# confdefs.h avoids OS command line length limits that DEFS can exceed. +rm -f -r conftest* confdefs.h + +$as_echo "/* confdefs.h */" > confdefs.h + +# Predefined preprocessor variables. + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_NAME "$PACKAGE_NAME" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_TARNAME "$PACKAGE_TARNAME" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_VERSION "$PACKAGE_VERSION" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_STRING "$PACKAGE_STRING" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_URL "$PACKAGE_URL" +_ACEOF + + +# Let the site file select an alternate cache file if it wants to. +# Prefer an explicitly selected file to automatically selected ones. +ac_site_file1=NONE +ac_site_file2=NONE +if test -n "$CONFIG_SITE"; then + # We do not want a PATH search for config.site. + case $CONFIG_SITE in #(( + -*) ac_site_file1=./$CONFIG_SITE;; + */*) ac_site_file1=$CONFIG_SITE;; + *) ac_site_file1=./$CONFIG_SITE;; + esac +elif test "x$prefix" != xNONE; then + ac_site_file1=$prefix/share/config.site + ac_site_file2=$prefix/etc/config.site +else + ac_site_file1=$ac_default_prefix/share/config.site + ac_site_file2=$ac_default_prefix/etc/config.site +fi +for ac_site_file in "$ac_site_file1" "$ac_site_file2" +do + test "x$ac_site_file" = xNONE && continue + if test /dev/null != "$ac_site_file" && test -r "$ac_site_file"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 +$as_echo "$as_me: loading site script $ac_site_file" >&6;} + sed 's/^/| /' "$ac_site_file" >&5 + . "$ac_site_file" \ + || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "failed to load site script $ac_site_file +See \`config.log' for more details" "$LINENO" 5; } + fi +done + +if test -r "$cache_file"; then + # Some versions of bash will fail to source /dev/null (special files + # actually), so we avoid doing that. DJGPP emulates it as a regular file. + if test /dev/null != "$cache_file" && test -f "$cache_file"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5 +$as_echo "$as_me: loading cache $cache_file" >&6;} + case $cache_file in + [\\/]* | ?:[\\/]* ) . "$cache_file";; + *) . "./$cache_file";; + esac + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5 +$as_echo "$as_me: creating cache $cache_file" >&6;} + >$cache_file +fi + +# Check that the precious variables saved in the cache have kept the same +# value. +ac_cache_corrupted=false +for ac_var in $ac_precious_vars; do + eval ac_old_set=\$ac_cv_env_${ac_var}_set + eval ac_new_set=\$ac_env_${ac_var}_set + eval ac_old_val=\$ac_cv_env_${ac_var}_value + eval ac_new_val=\$ac_env_${ac_var}_value + case $ac_old_set,$ac_new_set in + set,) + { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 +$as_echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,set) + { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5 +$as_echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,);; + *) + if test "x$ac_old_val" != "x$ac_new_val"; then + # differences in whitespace do not lead to failure. + ac_old_val_w=`echo x $ac_old_val` + ac_new_val_w=`echo x $ac_new_val` + if test "$ac_old_val_w" != "$ac_new_val_w"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5 +$as_echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} + ac_cache_corrupted=: + else + { $as_echo "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 +$as_echo "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} + eval $ac_var=\$ac_old_val + fi + { $as_echo "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5 +$as_echo "$as_me: former value: \`$ac_old_val'" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5 +$as_echo "$as_me: current value: \`$ac_new_val'" >&2;} + fi;; + esac + # Pass precious variables to config.status. + if test "$ac_new_set" = set; then + case $ac_new_val in + *\'*) ac_arg=$ac_var=`$as_echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; + *) ac_arg=$ac_var=$ac_new_val ;; + esac + case " $ac_configure_args " in + *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. + *) as_fn_append ac_configure_args " '$ac_arg'" ;; + esac + fi +done +if $ac_cache_corrupted; then + { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 +$as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} + as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 +fi +## -------------------- ## +## Main body of script. ## +## -------------------- ## + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + + +# Create release string. Used with VERSION for RPMs. +RPM_RELEASE=0 + +if test -d ".git" ; then + # Extract the first word of "git", so it can be a program name with args. +set dummy git; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_GIT_CHECK+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$GIT_CHECK"; then + ac_cv_prog_GIT_CHECK="$GIT_CHECK" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_GIT_CHECK="yes" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +GIT_CHECK=$ac_cv_prog_GIT_CHECK +if test -n "$GIT_CHECK"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $GIT_CHECK" >&5 +$as_echo "$GIT_CHECK" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + if test x"$GIT_CHECK" = x"yes"; then + RPM_RELEASE=`if expr index $(git describe --always) '-' > /dev/null ; then git describe --always | cut -d- -f2- | tr '-' '.' ; else echo "0"; fi` + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: RPM_RELEASE='$RPM_RELEASE'" >&5 +$as_echo "$as_me: RPM_RELEASE='$RPM_RELEASE'" >&6;} + + + +ac_aux_dir= +for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do + if test -f "$ac_dir/install-sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install-sh -c" + break + elif test -f "$ac_dir/install.sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install.sh -c" + break + elif test -f "$ac_dir/shtool"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/shtool install -c" + break + fi +done +if test -z "$ac_aux_dir"; then + as_fn_error $? "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 +fi + +# These three variables are undocumented and unsupported, +# and are intended to be withdrawn in a future Autoconf release. +# They can cause serious problems if a builder's source tree is in a directory +# whose full name contains unusual characters. +ac_config_guess="$SHELL $ac_aux_dir/config.guess" # Please don't use this var. +ac_config_sub="$SHELL $ac_aux_dir/config.sub" # Please don't use this var. +ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. + + + + +subdirs="$subdirs src/gtest" + + +# Environment +# Make sure we can run config.sub. +$SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 || + as_fn_error $? "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5 + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking build system type" >&5 +$as_echo_n "checking build system type... " >&6; } +if ${ac_cv_build+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_build_alias=$build_alias +test "x$ac_build_alias" = x && + ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"` +test "x$ac_build_alias" = x && + as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5 +ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` || + as_fn_error $? "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5 + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5 +$as_echo "$ac_cv_build" >&6; } +case $ac_cv_build in +*-*-*) ;; +*) as_fn_error $? "invalid value of canonical build" "$LINENO" 5;; +esac +build=$ac_cv_build +ac_save_IFS=$IFS; IFS='-' +set x $ac_cv_build +shift +build_cpu=$1 +build_vendor=$2 +shift; shift +# Remember, the first character of IFS is used to create $*, +# except with old shells: +build_os=$* +IFS=$ac_save_IFS +case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking host system type" >&5 +$as_echo_n "checking host system type... " >&6; } +if ${ac_cv_host+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "x$host_alias" = x; then + ac_cv_host=$ac_cv_build +else + ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` || + as_fn_error $? "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5 +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_host" >&5 +$as_echo "$ac_cv_host" >&6; } +case $ac_cv_host in +*-*-*) ;; +*) as_fn_error $? "invalid value of canonical host" "$LINENO" 5;; +esac +host=$ac_cv_host +ac_save_IFS=$IFS; IFS='-' +set x $ac_cv_host +shift +host_cpu=$1 +host_vendor=$2 +shift; shift +# Remember, the first character of IFS is used to create $*, +# except with old shells: +host_os=$* +IFS=$ac_save_IFS +case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking target system type" >&5 +$as_echo_n "checking target system type... " >&6; } +if ${ac_cv_target+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "x$target_alias" = x; then + ac_cv_target=$ac_cv_host +else + ac_cv_target=`$SHELL "$ac_aux_dir/config.sub" $target_alias` || + as_fn_error $? "$SHELL $ac_aux_dir/config.sub $target_alias failed" "$LINENO" 5 +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_target" >&5 +$as_echo "$ac_cv_target" >&6; } +case $ac_cv_target in +*-*-*) ;; +*) as_fn_error $? "invalid value of canonical target" "$LINENO" 5;; +esac +target=$ac_cv_target +ac_save_IFS=$IFS; IFS='-' +set x $ac_cv_target +shift +target_cpu=$1 +target_vendor=$2 +shift; shift +# Remember, the first character of IFS is used to create $*, +# except with old shells: +target_os=$* +IFS=$ac_save_IFS +case $target_os in *\ *) target_os=`echo "$target_os" | sed 's/ /-/g'`;; esac + + +# The aliases save the names the user supplied, while $host etc. +# will get canonicalized. +test -n "$target_alias" && + test "$program_prefix$program_suffix$program_transform_name" = \ + NONENONEs,x,x, && + program_prefix=${target_alias}- + +# Fix automake problems in 1.12 +# expand $ac_aux_dir to an absolute path +am_aux_dir=`cd $ac_aux_dir && pwd` + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. +set dummy ${ac_tool_prefix}gcc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="${ac_tool_prefix}gcc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_CC"; then + ac_ct_CC=$CC + # Extract the first word of "gcc", so it can be a program name with args. +set dummy gcc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CC="gcc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 +$as_echo "$ac_ct_CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +else + CC="$ac_cv_prog_CC" +fi + +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. +set dummy ${ac_tool_prefix}cc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="${ac_tool_prefix}cc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + fi +fi +if test -z "$CC"; then + # Extract the first word of "cc", so it can be a program name with args. +set dummy cc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else + ac_prog_rejected=no +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then + ac_prog_rejected=yes + continue + fi + ac_cv_prog_CC="cc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +if test $ac_prog_rejected = yes; then + # We found a bogon in the path, so make sure we never use it. + set dummy $ac_cv_prog_CC + shift + if test $# != 0; then + # We chose a different compiler from the bogus one. + # However, it has the same basename, so the bogon will be chosen + # first if we set CC to just the basename; use the full file name. + shift + ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" + fi +fi +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + for ac_prog in cl.exe + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$CC" && break + done +fi +if test -z "$CC"; then + ac_ct_CC=$CC + for ac_prog in cl.exe +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CC="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 +$as_echo "$ac_ct_CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_CC" && break +done + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +fi + +fi + + +test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no acceptable C compiler found in \$PATH +See \`config.log' for more details" "$LINENO" 5; } + +# Provide some information about the compiler. +$as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +for ac_option in --version -v -V -qversion; do + { { ac_try="$ac_compiler $ac_option >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compiler $ac_option >&5") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + sed '10a\ +... rest of stderr output deleted ... + 10q' conftest.err >conftest.er1 + cat conftest.er1 >&5 + fi + rm -f conftest.er1 conftest.err + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } +done + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out" +# Try to create an executable without -o first, disregard a.out. +# It will help us diagnose broken compilers, and finding out an intuition +# of exeext. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5 +$as_echo_n "checking whether the C compiler works... " >&6; } +ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` + +# The possible output files: +ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*" + +ac_rmfiles= +for ac_file in $ac_files +do + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; + * ) ac_rmfiles="$ac_rmfiles $ac_file";; + esac +done +rm -f $ac_rmfiles + +if { { ac_try="$ac_link_default" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link_default") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then : + # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. +# So ignore a value of `no', otherwise this would lead to `EXEEXT = no' +# in a Makefile. We should not override ac_cv_exeext if it was cached, +# so that the user can short-circuit this test for compilers unknown to +# Autoconf. +for ac_file in $ac_files '' +do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) + ;; + [ab].out ) + # We found the default executable, but exeext='' is most + # certainly right. + break;; + *.* ) + if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no; + then :; else + ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + fi + # We set ac_cv_exeext here because the later test for it is not + # safe: cross compilers may not add the suffix if given an `-o' + # argument, so we may need to know it at that point already. + # Even if this section looks crufty: it has the advantage of + # actually working. + break;; + * ) + break;; + esac +done +test "$ac_cv_exeext" = no && ac_cv_exeext= + +else + ac_file='' +fi +if test -z "$ac_file"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +$as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "C compiler cannot create executables +See \`config.log' for more details" "$LINENO" 5; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5 +$as_echo_n "checking for C compiler default output file name... " >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5 +$as_echo "$ac_file" >&6; } +ac_exeext=$ac_cv_exeext + +rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out +ac_clean_files=$ac_clean_files_save +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5 +$as_echo_n "checking for suffix of executables... " >&6; } +if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then : + # If both `conftest.exe' and `conftest' are `present' (well, observable) +# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will +# work properly (i.e., refer to `conftest.exe'), while it won't with +# `rm'. +for ac_file in conftest.exe conftest conftest.*; do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; + *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + break;; + * ) break;; + esac +done +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details" "$LINENO" 5; } +fi +rm -f conftest conftest$ac_cv_exeext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 +$as_echo "$ac_cv_exeext" >&6; } + +rm -f conftest.$ac_ext +EXEEXT=$ac_cv_exeext +ac_exeext=$EXEEXT +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +FILE *f = fopen ("conftest.out", "w"); + return ferror (f) || fclose (f) != 0; + + ; + return 0; +} +_ACEOF +ac_clean_files="$ac_clean_files conftest.out" +# Check that the compiler produces executables we can run. If not, either +# the compiler is broken, or we cross compile. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5 +$as_echo_n "checking whether we are cross compiling... " >&6; } +if test "$cross_compiling" != yes; then + { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + if { ac_try='./conftest$ac_cv_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; }; then + cross_compiling=no + else + if test "$cross_compiling" = maybe; then + cross_compiling=yes + else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot run C compiled programs. +If you meant to cross compile, use \`--host'. +See \`config.log' for more details" "$LINENO" 5; } + fi + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 +$as_echo "$cross_compiling" >&6; } + +rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out +ac_clean_files=$ac_clean_files_save +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 +$as_echo_n "checking for suffix of object files... " >&6; } +if ${ac_cv_objext+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.o conftest.obj +if { { ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compile") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then : + for ac_file in conftest.o conftest.obj conftest.*; do + test -f "$ac_file" || continue; + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;; + *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` + break;; + esac +done +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot compute suffix of object files: cannot compile +See \`config.log' for more details" "$LINENO" 5; } +fi +rm -f conftest.$ac_cv_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 +$as_echo "$ac_cv_objext" >&6; } +OBJEXT=$ac_cv_objext +ac_objext=$OBJEXT +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 +$as_echo_n "checking whether we are using the GNU C compiler... " >&6; } +if ${ac_cv_c_compiler_gnu+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_compiler_gnu=yes +else + ac_compiler_gnu=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_c_compiler_gnu=$ac_compiler_gnu + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 +$as_echo "$ac_cv_c_compiler_gnu" >&6; } +if test $ac_compiler_gnu = yes; then + GCC=yes +else + GCC= +fi +ac_test_CFLAGS=${CFLAGS+set} +ac_save_CFLAGS=$CFLAGS +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 +$as_echo_n "checking whether $CC accepts -g... " >&6; } +if ${ac_cv_prog_cc_g+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_save_c_werror_flag=$ac_c_werror_flag + ac_c_werror_flag=yes + ac_cv_prog_cc_g=no + CFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_prog_cc_g=yes +else + CFLAGS="" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +else + ac_c_werror_flag=$ac_save_c_werror_flag + CFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_prog_cc_g=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_c_werror_flag=$ac_save_c_werror_flag +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 +$as_echo "$ac_cv_prog_cc_g" >&6; } +if test "$ac_test_CFLAGS" = set; then + CFLAGS=$ac_save_CFLAGS +elif test $ac_cv_prog_cc_g = yes; then + if test "$GCC" = yes; then + CFLAGS="-g -O2" + else + CFLAGS="-g" + fi +else + if test "$GCC" = yes; then + CFLAGS="-O2" + else + CFLAGS= + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 +$as_echo_n "checking for $CC option to accept ISO C89... " >&6; } +if ${ac_cv_prog_cc_c89+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_prog_cc_c89=no +ac_save_CC=$CC +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#include +#include +/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ +struct buf { int x; }; +FILE * (*rcsopen) (struct buf *, struct stat *, int); +static char *e (p, i) + char **p; + int i; +{ + return p[i]; +} +static char *f (char * (*g) (char **, int), char **p, ...) +{ + char *s; + va_list v; + va_start (v,p); + s = g (p, va_arg (v,int)); + va_end (v); + return s; +} + +/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has + function prototypes and stuff, but not '\xHH' hex character constants. + These don't provoke an error unfortunately, instead are silently treated + as 'x'. The following induces an error, until -std is added to get + proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an + array size at least. It's necessary to write '\x00'==0 to get something + that's true only with -std. */ +int osf4_cc_array ['\x00' == 0 ? 1 : -1]; + +/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters + inside strings and character constants. */ +#define FOO(x) 'x' +int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1]; + +int test (int i, double x); +struct s1 {int (*f) (int a);}; +struct s2 {int (*f) (double a);}; +int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); +int argc; +char **argv; +int +main () +{ +return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; + ; + return 0; +} +_ACEOF +for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \ + -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" +do + CC="$ac_save_CC $ac_arg" + if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_prog_cc_c89=$ac_arg +fi +rm -f core conftest.err conftest.$ac_objext + test "x$ac_cv_prog_cc_c89" != "xno" && break +done +rm -f conftest.$ac_ext +CC=$ac_save_CC + +fi +# AC_CACHE_VAL +case "x$ac_cv_prog_cc_c89" in + x) + { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 +$as_echo "none needed" >&6; } ;; + xno) + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 +$as_echo "unsupported" >&6; } ;; + *) + CC="$CC $ac_cv_prog_cc_c89" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 +$as_echo "$ac_cv_prog_cc_c89" >&6; } ;; +esac +if test "x$ac_cv_prog_cc_c89" != xno; then : + +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +if test -n "$ac_tool_prefix"; then + for ac_prog in ar lib "link -lib" + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_AR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$AR"; then + ac_cv_prog_AR="$AR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_AR="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +AR=$ac_cv_prog_AR +if test -n "$AR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AR" >&5 +$as_echo "$AR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$AR" && break + done +fi +if test -z "$AR"; then + ac_ct_AR=$AR + for ac_prog in ar lib "link -lib" +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_AR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_AR"; then + ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_AR="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_AR=$ac_cv_prog_ac_ct_AR +if test -n "$ac_ct_AR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_AR" >&5 +$as_echo "$ac_ct_AR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_AR" && break +done + + if test "x$ac_ct_AR" = x; then + AR="false" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + AR=$ac_ct_AR + fi +fi + +: ${AR=ar} + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the archiver ($AR) interface" >&5 +$as_echo_n "checking the archiver ($AR) interface... " >&6; } +if ${am_cv_ar_interface+:} false; then : + $as_echo_n "(cached) " >&6 +else + am_cv_ar_interface=ar + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int some_variable = 0; +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + am_ar_try='$AR cru libconftest.a conftest.$ac_objext >&5' + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$am_ar_try\""; } >&5 + (eval $am_ar_try) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + if test "$ac_status" -eq 0; then + am_cv_ar_interface=ar + else + am_ar_try='$AR -NOLOGO -OUT:conftest.lib conftest.$ac_objext >&5' + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$am_ar_try\""; } >&5 + (eval $am_ar_try) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + if test "$ac_status" -eq 0; then + am_cv_ar_interface=lib + else + am_cv_ar_interface=unknown + fi + fi + rm -f conftest.lib libconftest.a + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_ar_interface" >&5 +$as_echo "$am_cv_ar_interface" >&6; } + +case $am_cv_ar_interface in +ar) + ;; +lib) + # Microsoft lib, so override with the ar-lib wrapper script. + # FIXME: It is wrong to rewrite AR. + # But if we don't then we get into trouble of one sort or another. + # A longer-term fix would be to have automake use am__AR in this case, + # and then we could set am__AR="$am_aux_dir/ar-lib \$(AR)" or something + # similar. + AR="$am_aux_dir/ar-lib $AR" + ;; +unknown) + as_fn_error $? "could not determine $AR interface" "$LINENO" 5 + ;; +esac + + +# Automake +am__api_version='1.11' + +# Find a good install program. We prefer a C program (faster), +# so one script is as good as another. But avoid the broken or +# incompatible versions: +# SysV /etc/install, /usr/sbin/install +# SunOS /usr/etc/install +# IRIX /sbin/install +# AIX /bin/install +# AmigaOS /C/install, which installs bootblocks on floppy discs +# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag +# AFS /usr/afsws/bin/install, which mishandles nonexistent args +# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" +# OS/2's system install, which has a completely different semantic +# ./install, which can be erroneously created by make from ./install.sh. +# Reject install programs that cannot install multiple files. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5 +$as_echo_n "checking for a BSD-compatible install... " >&6; } +if test -z "$INSTALL"; then +if ${ac_cv_path_install+:} false; then : + $as_echo_n "(cached) " >&6 +else + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + # Account for people who put trailing slashes in PATH elements. +case $as_dir/ in #(( + ./ | .// | /[cC]/* | \ + /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ + ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \ + /usr/ucb/* ) ;; + *) + # OSF1 and SCO ODT 3.0 have their own names for install. + # Don't use installbsd from OSF since it installs stuff as root + # by default. + for ac_prog in ginstall scoinst install; do + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; }; then + if test $ac_prog = install && + grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # AIX install. It has an incompatible calling convention. + : + elif test $ac_prog = install && + grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # program-specific install script used by HP pwplus--don't use. + : + else + rm -rf conftest.one conftest.two conftest.dir + echo one > conftest.one + echo two > conftest.two + mkdir conftest.dir + if "$as_dir/$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir" && + test -s conftest.one && test -s conftest.two && + test -s conftest.dir/conftest.one && + test -s conftest.dir/conftest.two + then + ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c" + break 3 + fi + fi + fi + done + done + ;; +esac + + done +IFS=$as_save_IFS + +rm -rf conftest.one conftest.two conftest.dir + +fi + if test "${ac_cv_path_install+set}" = set; then + INSTALL=$ac_cv_path_install + else + # As a last resort, use the slow shell script. Don't cache a + # value for INSTALL within a source directory, because that will + # break other packages using the cache if that directory is + # removed, or if the value is a relative name. + INSTALL=$ac_install_sh + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5 +$as_echo "$INSTALL" >&6; } + +# Use test -z because SunOS4 sh mishandles braces in ${var-val}. +# It thinks the first close brace ends the variable substitution. +test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' + +test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' + +test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether build environment is sane" >&5 +$as_echo_n "checking whether build environment is sane... " >&6; } +# Just in case +sleep 1 +echo timestamp > conftest.file +# Reject unsafe characters in $srcdir or the absolute working directory +# name. Accept space and tab only in the latter. +am_lf=' +' +case `pwd` in + *[\\\"\#\$\&\'\`$am_lf]*) + as_fn_error $? "unsafe absolute working directory name" "$LINENO" 5;; +esac +case $srcdir in + *[\\\"\#\$\&\'\`$am_lf\ \ ]*) + as_fn_error $? "unsafe srcdir value: \`$srcdir'" "$LINENO" 5;; +esac + +# Do `set' in a subshell so we don't clobber the current shell's +# arguments. Must try -L first in case configure is actually a +# symlink; some systems play weird games with the mod time of symlinks +# (eg FreeBSD returns the mod time of the symlink's containing +# directory). +if ( + set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null` + if test "$*" = "X"; then + # -L didn't work. + set X `ls -t "$srcdir/configure" conftest.file` + fi + rm -f conftest.file + if test "$*" != "X $srcdir/configure conftest.file" \ + && test "$*" != "X conftest.file $srcdir/configure"; then + + # If neither matched, then we have a broken ls. This can happen + # if, for instance, CONFIG_SHELL is bash and it inherits a + # broken ls alias from the environment. This has actually + # happened. Such a system could not be considered "sane". + as_fn_error $? "ls -t appears to fail. Make sure there is not a broken +alias in your environment" "$LINENO" 5 + fi + + test "$2" = conftest.file + ) +then + # Ok. + : +else + as_fn_error $? "newly created file is older than distributed files! +Check your system clock" "$LINENO" 5 +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +test "$program_prefix" != NONE && + program_transform_name="s&^&$program_prefix&;$program_transform_name" +# Use a double $ so make ignores it. +test "$program_suffix" != NONE && + program_transform_name="s&\$&$program_suffix&;$program_transform_name" +# Double any \ or $. +# By default was `s,x,x', remove it if useless. +ac_script='s/[\\$]/&&/g;s/;s,x,x,$//' +program_transform_name=`$as_echo "$program_transform_name" | sed "$ac_script"` + +if test x"${MISSING+set}" != xset; then + case $am_aux_dir in + *\ * | *\ *) + MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;; + *) + MISSING="\${SHELL} $am_aux_dir/missing" ;; + esac +fi +# Use eval to expand $SHELL +if eval "$MISSING --run true"; then + am_missing_run="$MISSING --run " +else + am_missing_run= + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \`missing' script is too old or missing" >&5 +$as_echo "$as_me: WARNING: \`missing' script is too old or missing" >&2;} +fi + +if test x"${install_sh}" != xset; then + case $am_aux_dir in + *\ * | *\ *) + install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;; + *) + install_sh="\${SHELL} $am_aux_dir/install-sh" + esac +fi + +# Installed binaries are usually stripped using `strip' when the user +# run `make install-strip'. However `strip' might not be the right +# tool to use in cross-compilation environments, therefore Automake +# will honor the `STRIP' environment variable to overrule this program. +if test "$cross_compiling" != no; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. +set dummy ${ac_tool_prefix}strip; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_STRIP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$STRIP"; then + ac_cv_prog_STRIP="$STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_STRIP="${ac_tool_prefix}strip" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +STRIP=$ac_cv_prog_STRIP +if test -n "$STRIP"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5 +$as_echo "$STRIP" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_STRIP"; then + ac_ct_STRIP=$STRIP + # Extract the first word of "strip", so it can be a program name with args. +set dummy strip; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_STRIP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_STRIP"; then + ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_STRIP="strip" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP +if test -n "$ac_ct_STRIP"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5 +$as_echo "$ac_ct_STRIP" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_STRIP" = x; then + STRIP=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + STRIP=$ac_ct_STRIP + fi +else + STRIP="$ac_cv_prog_STRIP" +fi + +fi +INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s" + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a thread-safe mkdir -p" >&5 +$as_echo_n "checking for a thread-safe mkdir -p... " >&6; } +if test -z "$MKDIR_P"; then + if ${ac_cv_path_mkdir+:} false; then : + $as_echo_n "(cached) " >&6 +else + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/opt/sfw/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in mkdir gmkdir; do + for ac_exec_ext in '' $ac_executable_extensions; do + { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; } || continue + case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #( + 'mkdir (GNU coreutils) '* | \ + 'mkdir (coreutils) '* | \ + 'mkdir (fileutils) '4.1*) + ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext + break 3;; + esac + done + done + done +IFS=$as_save_IFS + +fi + + test -d ./--version && rmdir ./--version + if test "${ac_cv_path_mkdir+set}" = set; then + MKDIR_P="$ac_cv_path_mkdir -p" + else + # As a last resort, use the slow shell script. Don't cache a + # value for MKDIR_P within a source directory, because that will + # break other packages using the cache if that directory is + # removed, or if the value is a relative name. + MKDIR_P="$ac_install_sh -d" + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MKDIR_P" >&5 +$as_echo "$MKDIR_P" >&6; } + +mkdir_p="$MKDIR_P" +case $mkdir_p in + [\\/$]* | ?:[\\/]*) ;; + */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;; +esac + +for ac_prog in gawk mawk nawk awk +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_AWK+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$AWK"; then + ac_cv_prog_AWK="$AWK" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_AWK="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +AWK=$ac_cv_prog_AWK +if test -n "$AWK"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 +$as_echo "$AWK" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$AWK" && break +done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${MAKE-make} sets \$(MAKE)" >&5 +$as_echo_n "checking whether ${MAKE-make} sets \$(MAKE)... " >&6; } +set x ${MAKE-make} +ac_make=`$as_echo "$2" | sed 's/+/p/g; s/[^a-zA-Z0-9_]/_/g'` +if eval \${ac_cv_prog_make_${ac_make}_set+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat >conftest.make <<\_ACEOF +SHELL = /bin/sh +all: + @echo '@@@%%%=$(MAKE)=@@@%%%' +_ACEOF +# GNU make sometimes prints "make[1]: Entering ...", which would confuse us. +case `${MAKE-make} -f conftest.make 2>/dev/null` in + *@@@%%%=?*=@@@%%%*) + eval ac_cv_prog_make_${ac_make}_set=yes;; + *) + eval ac_cv_prog_make_${ac_make}_set=no;; +esac +rm -f conftest.make +fi +if eval test \$ac_cv_prog_make_${ac_make}_set = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + SET_MAKE= +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + SET_MAKE="MAKE=${MAKE-make}" +fi + +rm -rf .tst 2>/dev/null +mkdir .tst 2>/dev/null +if test -d .tst; then + am__leading_dot=. +else + am__leading_dot=_ +fi +rmdir .tst 2>/dev/null + +DEPDIR="${am__leading_dot}deps" + +ac_config_commands="$ac_config_commands depfiles" + + +am_make=${MAKE-make} +cat > confinc << 'END' +am__doit: + @echo this is the am__doit target +.PHONY: am__doit +END +# If we don't find an include directive, just comment out the code. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for style of include used by $am_make" >&5 +$as_echo_n "checking for style of include used by $am_make... " >&6; } +am__include="#" +am__quote= +_am_result=none +# First try GNU make style include. +echo "include confinc" > confmf +# Ignore all kinds of additional output from `make'. +case `$am_make -s -f confmf 2> /dev/null` in #( +*the\ am__doit\ target*) + am__include=include + am__quote= + _am_result=GNU + ;; +esac +# Now try BSD make style include. +if test "$am__include" = "#"; then + echo '.include "confinc"' > confmf + case `$am_make -s -f confmf 2> /dev/null` in #( + *the\ am__doit\ target*) + am__include=.include + am__quote="\"" + _am_result=BSD + ;; + esac +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $_am_result" >&5 +$as_echo "$_am_result" >&6; } +rm -f confinc confmf + +# Check whether --enable-dependency-tracking was given. +if test "${enable_dependency_tracking+set}" = set; then : + enableval=$enable_dependency_tracking; +fi + +if test "x$enable_dependency_tracking" != xno; then + am_depcomp="$ac_aux_dir/depcomp" + AMDEPBACKSLASH='\' + am__nodep='_no' +fi + if test "x$enable_dependency_tracking" != xno; then + AMDEP_TRUE= + AMDEP_FALSE='#' +else + AMDEP_TRUE='#' + AMDEP_FALSE= +fi + + +if test "`cd $srcdir && pwd`" != "`pwd`"; then + # Use -I$(srcdir) only when $(srcdir) != ., so that make's output + # is not polluted with repeated "-I." + am__isrc=' -I$(srcdir)' + # test to see if srcdir already configured + if test -f $srcdir/config.status; then + as_fn_error $? "source directory already configured; run \"make distclean\" there first" "$LINENO" 5 + fi +fi + +# test whether we have cygpath +if test -z "$CYGPATH_W"; then + if (cygpath --version) >/dev/null 2>/dev/null; then + CYGPATH_W='cygpath -w' + else + CYGPATH_W=echo + fi +fi + + +# Define the identity of the package. + PACKAGE='ceph' + VERSION='0.80.7' + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE "$PACKAGE" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define VERSION "$VERSION" +_ACEOF + +# Some tools Automake needs. + +ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"} + + +AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"} + + +AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"} + + +AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"} + + +MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"} + +# We need awk for the "check" target. The system "awk" is bad on +# some platforms. +# Always define AMTAR for backward compatibility. Yes, it's still used +# in the wild :-( We should find a proper way to deprecate it ... +AMTAR='$${TAR-tar}' + +am__tar='$${TAR-tar} chof - "$$tardir"' am__untar='$${TAR-tar} xf -' + + + + +depcc="$CC" am_compiler_list= + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5 +$as_echo_n "checking dependency style of $depcc... " >&6; } +if ${am_cv_CC_dependencies_compiler_type+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + rm -rf conftest.dir + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CC_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + am__universal=false + case " $depcc " in #( + *\ -arch\ *\ -arch\ *) am__universal=true ;; + esac + + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. Also, some Intel + # versions had trouble with output in subdirs + am__obj=sub/conftest.${OBJEXT-o} + am__minus_obj="-o $am__obj" + case $depmode in + gcc) + # This depmode causes a compiler race in universal mode. + test "$am__universal" = false || continue + ;; + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + msvc7 | msvc7msys | msvisualcpp | msvcmsys) + # This compiler won't grok `-c -o', but also, the minuso test has + # not run yet. These depmodes are late enough in the game, and + # so weak that their functioning should not be impacted. + am__obj=conftest.${OBJEXT-o} + am__minus_obj= + ;; + none) break ;; + esac + if depmode=$depmode \ + source=sub/conftest.c object=$am__obj \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep $am__obj sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CC_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CC_dependencies_compiler_type=none +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CC_dependencies_compiler_type" >&5 +$as_echo "$am_cv_CC_dependencies_compiler_type" >&6; } +CCDEPMODE=depmode=$am_cv_CC_dependencies_compiler_type + + if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CC_dependencies_compiler_type" = gcc3; then + am__fastdepCC_TRUE= + am__fastdepCC_FALSE='#' +else + am__fastdepCC_TRUE='#' + am__fastdepCC_FALSE= +fi + + + +if test "x$CC" != xcc; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC and cc understand -c and -o together" >&5 +$as_echo_n "checking whether $CC and cc understand -c and -o together... " >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether cc understands -c and -o together" >&5 +$as_echo_n "checking whether cc understands -c and -o together... " >&6; } +fi +set dummy $CC; ac_cc=`$as_echo "$2" | + sed 's/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/'` +if eval \${ac_cv_prog_cc_${ac_cc}_c_o+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +# Make sure it works both with $CC and with simple cc. +# We do the test twice because some compilers refuse to overwrite an +# existing .o file with -o, though they will create one. +ac_try='$CC -c conftest.$ac_ext -o conftest2.$ac_objext >&5' +rm -f conftest2.* +if { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && + test -f conftest2.$ac_objext && { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; +then + eval ac_cv_prog_cc_${ac_cc}_c_o=yes + if test "x$CC" != xcc; then + # Test first that cc exists at all. + if { ac_try='cc -c conftest.$ac_ext >&5' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; }; then + ac_try='cc -c conftest.$ac_ext -o conftest2.$ac_objext >&5' + rm -f conftest2.* + if { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && + test -f conftest2.$ac_objext && { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; + then + # cc works too. + : + else + # cc exists but doesn't like -o. + eval ac_cv_prog_cc_${ac_cc}_c_o=no + fi + fi + fi +else + eval ac_cv_prog_cc_${ac_cc}_c_o=no +fi +rm -f core conftest* + +fi +if eval test \$ac_cv_prog_cc_${ac_cc}_c_o = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +$as_echo "#define NO_MINUS_C_MINUS_O 1" >>confdefs.h + +fi + +# FIXME: we rely on the cache variable name because +# there is no other way. +set dummy $CC +am_cc=`echo $2 | sed 's/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/'` +eval am_t=\$ac_cv_prog_cc_${am_cc}_c_o +if test "$am_t" != yes; then + # Losing compiler, so override with the script. + # FIXME: It is wrong to rewrite CC. + # But if we don't then we get into trouble of one sort or another. + # A longer-term fix would be to have automake use am__CC in this case, + # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" + CC="$am_aux_dir/compile $CC" +fi + + +case `pwd` in + *\ * | *\ *) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Libtool does not cope well with whitespace in \`pwd\`" >&5 +$as_echo "$as_me: WARNING: Libtool does not cope well with whitespace in \`pwd\`" >&2;} ;; +esac + + + +macro_version='2.4.2' +macro_revision='1.3337' + + + + + + + + + + + + + +ltmain="$ac_aux_dir/ltmain.sh" + +# Backslashify metacharacters that are still active within +# double-quoted strings. +sed_quote_subst='s/\(["`$\\]\)/\\\1/g' + +# Same as above, but do not quote variable references. +double_quote_subst='s/\(["`\\]\)/\\\1/g' + +# Sed substitution to delay expansion of an escaped shell variable in a +# double_quote_subst'ed string. +delay_variable_subst='s/\\\\\\\\\\\$/\\\\\\$/g' + +# Sed substitution to delay expansion of an escaped single quote. +delay_single_quote_subst='s/'\''/'\'\\\\\\\'\''/g' + +# Sed substitution to avoid accidental globbing in evaled expressions +no_glob_subst='s/\*/\\\*/g' + +ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO +ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO$ECHO + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to print strings" >&5 +$as_echo_n "checking how to print strings... " >&6; } +# Test print first, because it will be a builtin if present. +if test "X`( print -r -- -n ) 2>/dev/null`" = X-n && \ + test "X`print -r -- $ECHO 2>/dev/null`" = "X$ECHO"; then + ECHO='print -r --' +elif test "X`printf %s $ECHO 2>/dev/null`" = "X$ECHO"; then + ECHO='printf %s\n' +else + # Use this function as a fallback that always works. + func_fallback_echo () + { + eval 'cat <<_LTECHO_EOF +$1 +_LTECHO_EOF' + } + ECHO='func_fallback_echo' +fi + +# func_echo_all arg... +# Invoke $ECHO with all args, space-separated. +func_echo_all () +{ + $ECHO "" +} + +case "$ECHO" in + printf*) { $as_echo "$as_me:${as_lineno-$LINENO}: result: printf" >&5 +$as_echo "printf" >&6; } ;; + print*) { $as_echo "$as_me:${as_lineno-$LINENO}: result: print -r" >&5 +$as_echo "print -r" >&6; } ;; + *) { $as_echo "$as_me:${as_lineno-$LINENO}: result: cat" >&5 +$as_echo "cat" >&6; } ;; +esac + + + + + + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a sed that does not truncate output" >&5 +$as_echo_n "checking for a sed that does not truncate output... " >&6; } +if ${ac_cv_path_SED+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ + for ac_i in 1 2 3 4 5 6 7; do + ac_script="$ac_script$as_nl$ac_script" + done + echo "$ac_script" 2>/dev/null | sed 99q >conftest.sed + { ac_script=; unset ac_script;} + if test -z "$SED"; then + ac_path_SED_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in sed gsed; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_SED="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_SED" && $as_test_x "$ac_path_SED"; } || continue +# Check for GNU ac_path_SED and select it if it is found. + # Check for GNU $ac_path_SED +case `"$ac_path_SED" --version 2>&1` in +*GNU*) + ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo '' >> "conftest.nl" + "$ac_path_SED" -f conftest.sed < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_SED_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_SED="$ac_path_SED" + ac_path_SED_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_SED_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_SED"; then + as_fn_error $? "no acceptable sed could be found in \$PATH" "$LINENO" 5 + fi +else + ac_cv_path_SED=$SED +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5 +$as_echo "$ac_cv_path_SED" >&6; } + SED="$ac_cv_path_SED" + rm -f conftest.sed + +test -z "$SED" && SED=sed +Xsed="$SED -e 1s/^X//" + + + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 +$as_echo_n "checking for grep that handles long lines and -e... " >&6; } +if ${ac_cv_path_GREP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$GREP"; then + ac_path_GREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in grep ggrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_GREP" && $as_test_x "$ac_path_GREP"; } || continue +# Check for GNU ac_path_GREP and select it if it is found. + # Check for GNU $ac_path_GREP +case `"$ac_path_GREP" --version 2>&1` in +*GNU*) + ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'GREP' >> "conftest.nl" + "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_GREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_GREP="$ac_path_GREP" + ac_path_GREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_GREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_GREP"; then + as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + fi +else + ac_cv_path_GREP=$GREP +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 +$as_echo "$ac_cv_path_GREP" >&6; } + GREP="$ac_cv_path_GREP" + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 +$as_echo_n "checking for egrep... " >&6; } +if ${ac_cv_path_EGREP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 + then ac_cv_path_EGREP="$GREP -E" + else + if test -z "$EGREP"; then + ac_path_EGREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in egrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_EGREP" && $as_test_x "$ac_path_EGREP"; } || continue +# Check for GNU ac_path_EGREP and select it if it is found. + # Check for GNU $ac_path_EGREP +case `"$ac_path_EGREP" --version 2>&1` in +*GNU*) + ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'EGREP' >> "conftest.nl" + "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_EGREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_EGREP="$ac_path_EGREP" + ac_path_EGREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_EGREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_EGREP"; then + as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + fi +else + ac_cv_path_EGREP=$EGREP +fi + + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 +$as_echo "$ac_cv_path_EGREP" >&6; } + EGREP="$ac_cv_path_EGREP" + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for fgrep" >&5 +$as_echo_n "checking for fgrep... " >&6; } +if ${ac_cv_path_FGREP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if echo 'ab*c' | $GREP -F 'ab*c' >/dev/null 2>&1 + then ac_cv_path_FGREP="$GREP -F" + else + if test -z "$FGREP"; then + ac_path_FGREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in fgrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_FGREP="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_FGREP" && $as_test_x "$ac_path_FGREP"; } || continue +# Check for GNU ac_path_FGREP and select it if it is found. + # Check for GNU $ac_path_FGREP +case `"$ac_path_FGREP" --version 2>&1` in +*GNU*) + ac_cv_path_FGREP="$ac_path_FGREP" ac_path_FGREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'FGREP' >> "conftest.nl" + "$ac_path_FGREP" FGREP < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_FGREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_FGREP="$ac_path_FGREP" + ac_path_FGREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_FGREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_FGREP"; then + as_fn_error $? "no acceptable fgrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + fi +else + ac_cv_path_FGREP=$FGREP +fi + + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_FGREP" >&5 +$as_echo "$ac_cv_path_FGREP" >&6; } + FGREP="$ac_cv_path_FGREP" + + +test -z "$GREP" && GREP=grep + + + + + + + + + + + + + + + + + + + +# Check whether --with-gnu-ld was given. +if test "${with_gnu_ld+set}" = set; then : + withval=$with_gnu_ld; test "$withval" = no || with_gnu_ld=yes +else + with_gnu_ld=no +fi + +ac_prog=ld +if test "$GCC" = yes; then + # Check if gcc -print-prog-name=ld gives a path. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ld used by $CC" >&5 +$as_echo_n "checking for ld used by $CC... " >&6; } + case $host in + *-*-mingw*) + # gcc leaves a trailing carriage return which upsets mingw + ac_prog=`($CC -print-prog-name=ld) 2>&5 | tr -d '\015'` ;; + *) + ac_prog=`($CC -print-prog-name=ld) 2>&5` ;; + esac + case $ac_prog in + # Accept absolute paths. + [\\/]* | ?:[\\/]*) + re_direlt='/[^/][^/]*/\.\./' + # Canonicalize the pathname of ld + ac_prog=`$ECHO "$ac_prog"| $SED 's%\\\\%/%g'` + while $ECHO "$ac_prog" | $GREP "$re_direlt" > /dev/null 2>&1; do + ac_prog=`$ECHO $ac_prog| $SED "s%$re_direlt%/%"` + done + test -z "$LD" && LD="$ac_prog" + ;; + "") + # If it fails, then pretend we aren't using GCC. + ac_prog=ld + ;; + *) + # If it is relative, then search for the first ld in PATH. + with_gnu_ld=unknown + ;; + esac +elif test "$with_gnu_ld" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for GNU ld" >&5 +$as_echo_n "checking for GNU ld... " >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for non-GNU ld" >&5 +$as_echo_n "checking for non-GNU ld... " >&6; } +fi +if ${lt_cv_path_LD+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$LD"; then + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR + for ac_dir in $PATH; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + if test -f "$ac_dir/$ac_prog" || test -f "$ac_dir/$ac_prog$ac_exeext"; then + lt_cv_path_LD="$ac_dir/$ac_prog" + # Check to see if the program is GNU ld. I'd rather use --version, + # but apparently some variants of GNU ld only accept -v. + # Break only if it was the GNU/non-GNU ld that we prefer. + case `"$lt_cv_path_LD" -v 2>&1 &5 +$as_echo "$LD" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +test -z "$LD" && as_fn_error $? "no acceptable ld found in \$PATH" "$LINENO" 5 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if the linker ($LD) is GNU ld" >&5 +$as_echo_n "checking if the linker ($LD) is GNU ld... " >&6; } +if ${lt_cv_prog_gnu_ld+:} false; then : + $as_echo_n "(cached) " >&6 +else + # I'd rather use --version here, but apparently some GNU lds only accept -v. +case `$LD -v 2>&1 &5 +$as_echo "$lt_cv_prog_gnu_ld" >&6; } +with_gnu_ld=$lt_cv_prog_gnu_ld + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for BSD- or MS-compatible name lister (nm)" >&5 +$as_echo_n "checking for BSD- or MS-compatible name lister (nm)... " >&6; } +if ${lt_cv_path_NM+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$NM"; then + # Let the user override the test. + lt_cv_path_NM="$NM" +else + lt_nm_to_check="${ac_tool_prefix}nm" + if test -n "$ac_tool_prefix" && test "$build" = "$host"; then + lt_nm_to_check="$lt_nm_to_check nm" + fi + for lt_tmp_nm in $lt_nm_to_check; do + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR + for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + tmp_nm="$ac_dir/$lt_tmp_nm" + if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext" ; then + # Check to see if the nm accepts a BSD-compat flag. + # Adding the `sed 1q' prevents false positives on HP-UX, which says: + # nm: unknown option "B" ignored + # Tru64's nm complains that /dev/null is an invalid object file + case `"$tmp_nm" -B /dev/null 2>&1 | sed '1q'` in + */dev/null* | *'Invalid file or object type'*) + lt_cv_path_NM="$tmp_nm -B" + break + ;; + *) + case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in + */dev/null*) + lt_cv_path_NM="$tmp_nm -p" + break + ;; + *) + lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but + continue # so that we can try to find one that supports BSD flags + ;; + esac + ;; + esac + fi + done + IFS="$lt_save_ifs" + done + : ${lt_cv_path_NM=no} +fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_path_NM" >&5 +$as_echo "$lt_cv_path_NM" >&6; } +if test "$lt_cv_path_NM" != "no"; then + NM="$lt_cv_path_NM" +else + # Didn't find any BSD compatible name lister, look for dumpbin. + if test -n "$DUMPBIN"; then : + # Let the user override the test. + else + if test -n "$ac_tool_prefix"; then + for ac_prog in dumpbin "link -dump" + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_DUMPBIN+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$DUMPBIN"; then + ac_cv_prog_DUMPBIN="$DUMPBIN" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_DUMPBIN="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +DUMPBIN=$ac_cv_prog_DUMPBIN +if test -n "$DUMPBIN"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DUMPBIN" >&5 +$as_echo "$DUMPBIN" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$DUMPBIN" && break + done +fi +if test -z "$DUMPBIN"; then + ac_ct_DUMPBIN=$DUMPBIN + for ac_prog in dumpbin "link -dump" +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_DUMPBIN+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_DUMPBIN"; then + ac_cv_prog_ac_ct_DUMPBIN="$ac_ct_DUMPBIN" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_DUMPBIN="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_DUMPBIN=$ac_cv_prog_ac_ct_DUMPBIN +if test -n "$ac_ct_DUMPBIN"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DUMPBIN" >&5 +$as_echo "$ac_ct_DUMPBIN" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_DUMPBIN" && break +done + + if test "x$ac_ct_DUMPBIN" = x; then + DUMPBIN=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + DUMPBIN=$ac_ct_DUMPBIN + fi +fi + + case `$DUMPBIN -symbols /dev/null 2>&1 | sed '1q'` in + *COFF*) + DUMPBIN="$DUMPBIN -symbols" + ;; + *) + DUMPBIN=: + ;; + esac + fi + + if test "$DUMPBIN" != ":"; then + NM="$DUMPBIN" + fi +fi +test -z "$NM" && NM=nm + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5 +$as_echo_n "checking the name lister ($NM) interface... " >&6; } +if ${lt_cv_nm_interface+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_nm_interface="BSD nm" + echo "int some_variable = 0;" > conftest.$ac_ext + (eval echo "\"\$as_me:$LINENO: $ac_compile\"" >&5) + (eval "$ac_compile" 2>conftest.err) + cat conftest.err >&5 + (eval echo "\"\$as_me:$LINENO: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) + cat conftest.err >&5 + (eval echo "\"\$as_me:$LINENO: output\"" >&5) + cat conftest.out >&5 + if $GREP 'External.*some_variable' conftest.out > /dev/null; then + lt_cv_nm_interface="MS dumpbin" + fi + rm -f conftest* +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5 +$as_echo "$lt_cv_nm_interface" >&6; } + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ln -s works" >&5 +$as_echo_n "checking whether ln -s works... " >&6; } +LN_S=$as_ln_s +if test "$LN_S" = "ln -s"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no, using $LN_S" >&5 +$as_echo "no, using $LN_S" >&6; } +fi + +# find the maximum length of command line arguments +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the maximum length of command line arguments" >&5 +$as_echo_n "checking the maximum length of command line arguments... " >&6; } +if ${lt_cv_sys_max_cmd_len+:} false; then : + $as_echo_n "(cached) " >&6 +else + i=0 + teststring="ABCD" + + case $build_os in + msdosdjgpp*) + # On DJGPP, this test can blow up pretty badly due to problems in libc + # (any single argument exceeding 2000 bytes causes a buffer overrun + # during glob expansion). Even if it were fixed, the result of this + # check would be larger than it should be. + lt_cv_sys_max_cmd_len=12288; # 12K is about right + ;; + + gnu*) + # Under GNU Hurd, this test is not required because there is + # no limit to the length of command line arguments. + # Libtool will interpret -1 as no limit whatsoever + lt_cv_sys_max_cmd_len=-1; + ;; + + cygwin* | mingw* | cegcc*) + # On Win9x/ME, this test blows up -- it succeeds, but takes + # about 5 minutes as the teststring grows exponentially. + # Worse, since 9x/ME are not pre-emptively multitasking, + # you end up with a "frozen" computer, even though with patience + # the test eventually succeeds (with a max line length of 256k). + # Instead, let's just punt: use the minimum linelength reported by + # all of the supported platforms: 8192 (on NT/2K/XP). + lt_cv_sys_max_cmd_len=8192; + ;; + + mint*) + # On MiNT this can take a long time and run out of memory. + lt_cv_sys_max_cmd_len=8192; + ;; + + amigaos*) + # On AmigaOS with pdksh, this test takes hours, literally. + # So we just punt and use a minimum line length of 8192. + lt_cv_sys_max_cmd_len=8192; + ;; + + netbsd* | freebsd* | openbsd* | darwin* | dragonfly*) + # This has been around since 386BSD, at least. Likely further. + if test -x /sbin/sysctl; then + lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax` + elif test -x /usr/sbin/sysctl; then + lt_cv_sys_max_cmd_len=`/usr/sbin/sysctl -n kern.argmax` + else + lt_cv_sys_max_cmd_len=65536 # usable default for all BSDs + fi + # And add a safety zone + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` + ;; + + interix*) + # We know the value 262144 and hardcode it with a safety zone (like BSD) + lt_cv_sys_max_cmd_len=196608 + ;; + + os2*) + # The test takes a long time on OS/2. + lt_cv_sys_max_cmd_len=8192 + ;; + + osf*) + # Dr. Hans Ekkehard Plesser reports seeing a kernel panic running configure + # due to this test when exec_disable_arg_limit is 1 on Tru64. It is not + # nice to cause kernel panics so lets avoid the loop below. + # First set a reasonable default. + lt_cv_sys_max_cmd_len=16384 + # + if test -x /sbin/sysconfig; then + case `/sbin/sysconfig -q proc exec_disable_arg_limit` in + *1*) lt_cv_sys_max_cmd_len=-1 ;; + esac + fi + ;; + sco3.2v5*) + lt_cv_sys_max_cmd_len=102400 + ;; + sysv5* | sco5v6* | sysv4.2uw2*) + kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null` + if test -n "$kargmax"; then + lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[ ]//'` + else + lt_cv_sys_max_cmd_len=32768 + fi + ;; + *) + lt_cv_sys_max_cmd_len=`(getconf ARG_MAX) 2> /dev/null` + if test -n "$lt_cv_sys_max_cmd_len"; then + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` + else + # Make teststring a little bigger before we do anything with it. + # a 1K string should be a reasonable start. + for i in 1 2 3 4 5 6 7 8 ; do + teststring=$teststring$teststring + done + SHELL=${SHELL-${CONFIG_SHELL-/bin/sh}} + # If test is not a shell built-in, we'll probably end up computing a + # maximum length that is only half of the actual maximum length, but + # we can't tell. + while { test "X"`env echo "$teststring$teststring" 2>/dev/null` \ + = "X$teststring$teststring"; } >/dev/null 2>&1 && + test $i != 17 # 1/2 MB should be enough + do + i=`expr $i + 1` + teststring=$teststring$teststring + done + # Only check the string length outside the loop. + lt_cv_sys_max_cmd_len=`expr "X$teststring" : ".*" 2>&1` + teststring= + # Add a significant safety factor because C++ compilers can tack on + # massive amounts of additional arguments before passing them to the + # linker. It appears as though 1/2 is a usable value. + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 2` + fi + ;; + esac + +fi + +if test -n $lt_cv_sys_max_cmd_len ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_sys_max_cmd_len" >&5 +$as_echo "$lt_cv_sys_max_cmd_len" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: none" >&5 +$as_echo "none" >&6; } +fi +max_cmd_len=$lt_cv_sys_max_cmd_len + + + + + + +: ${CP="cp -f"} +: ${MV="mv -f"} +: ${RM="rm -f"} + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the shell understands some XSI constructs" >&5 +$as_echo_n "checking whether the shell understands some XSI constructs... " >&6; } +# Try some XSI features +xsi_shell=no +( _lt_dummy="a/b/c" + test "${_lt_dummy##*/},${_lt_dummy%/*},${_lt_dummy#??}"${_lt_dummy%"$_lt_dummy"}, \ + = c,a/b,b/c, \ + && eval 'test $(( 1 + 1 )) -eq 2 \ + && test "${#_lt_dummy}" -eq 5' ) >/dev/null 2>&1 \ + && xsi_shell=yes +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $xsi_shell" >&5 +$as_echo "$xsi_shell" >&6; } + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the shell understands \"+=\"" >&5 +$as_echo_n "checking whether the shell understands \"+=\"... " >&6; } +lt_shell_append=no +( foo=bar; set foo baz; eval "$1+=\$2" && test "$foo" = barbaz ) \ + >/dev/null 2>&1 \ + && lt_shell_append=yes +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_shell_append" >&5 +$as_echo "$lt_shell_append" >&6; } + + +if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then + lt_unset=unset +else + lt_unset=false +fi + + + + + +# test EBCDIC or ASCII +case `echo X|tr X '\101'` in + A) # ASCII based system + # \n is not interpreted correctly by Solaris 8 /usr/ucb/tr + lt_SP2NL='tr \040 \012' + lt_NL2SP='tr \015\012 \040\040' + ;; + *) # EBCDIC based system + lt_SP2NL='tr \100 \n' + lt_NL2SP='tr \r\n \100\100' + ;; +esac + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to convert $build file names to $host format" >&5 +$as_echo_n "checking how to convert $build file names to $host format... " >&6; } +if ${lt_cv_to_host_file_cmd+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $host in + *-*-mingw* ) + case $build in + *-*-mingw* ) # actually msys + lt_cv_to_host_file_cmd=func_convert_file_msys_to_w32 + ;; + *-*-cygwin* ) + lt_cv_to_host_file_cmd=func_convert_file_cygwin_to_w32 + ;; + * ) # otherwise, assume *nix + lt_cv_to_host_file_cmd=func_convert_file_nix_to_w32 + ;; + esac + ;; + *-*-cygwin* ) + case $build in + *-*-mingw* ) # actually msys + lt_cv_to_host_file_cmd=func_convert_file_msys_to_cygwin + ;; + *-*-cygwin* ) + lt_cv_to_host_file_cmd=func_convert_file_noop + ;; + * ) # otherwise, assume *nix + lt_cv_to_host_file_cmd=func_convert_file_nix_to_cygwin + ;; + esac + ;; + * ) # unhandled hosts (and "normal" native builds) + lt_cv_to_host_file_cmd=func_convert_file_noop + ;; +esac + +fi + +to_host_file_cmd=$lt_cv_to_host_file_cmd +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_to_host_file_cmd" >&5 +$as_echo "$lt_cv_to_host_file_cmd" >&6; } + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to convert $build file names to toolchain format" >&5 +$as_echo_n "checking how to convert $build file names to toolchain format... " >&6; } +if ${lt_cv_to_tool_file_cmd+:} false; then : + $as_echo_n "(cached) " >&6 +else + #assume ordinary cross tools, or native build. +lt_cv_to_tool_file_cmd=func_convert_file_noop +case $host in + *-*-mingw* ) + case $build in + *-*-mingw* ) # actually msys + lt_cv_to_tool_file_cmd=func_convert_file_msys_to_w32 + ;; + esac + ;; +esac + +fi + +to_tool_file_cmd=$lt_cv_to_tool_file_cmd +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_to_tool_file_cmd" >&5 +$as_echo "$lt_cv_to_tool_file_cmd" >&6; } + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $LD option to reload object files" >&5 +$as_echo_n "checking for $LD option to reload object files... " >&6; } +if ${lt_cv_ld_reload_flag+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_ld_reload_flag='-r' +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_reload_flag" >&5 +$as_echo "$lt_cv_ld_reload_flag" >&6; } +reload_flag=$lt_cv_ld_reload_flag +case $reload_flag in +"" | " "*) ;; +*) reload_flag=" $reload_flag" ;; +esac +reload_cmds='$LD$reload_flag -o $output$reload_objs' +case $host_os in + cygwin* | mingw* | pw32* | cegcc*) + if test "$GCC" != yes; then + reload_cmds=false + fi + ;; + darwin*) + if test "$GCC" = yes; then + reload_cmds='$LTCC $LTCFLAGS -nostdlib ${wl}-r -o $output$reload_objs' + else + reload_cmds='$LD$reload_flag -o $output$reload_objs' + fi + ;; +esac + + + + + + + + + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}objdump", so it can be a program name with args. +set dummy ${ac_tool_prefix}objdump; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_OBJDUMP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$OBJDUMP"; then + ac_cv_prog_OBJDUMP="$OBJDUMP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_OBJDUMP="${ac_tool_prefix}objdump" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +OBJDUMP=$ac_cv_prog_OBJDUMP +if test -n "$OBJDUMP"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OBJDUMP" >&5 +$as_echo "$OBJDUMP" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_OBJDUMP"; then + ac_ct_OBJDUMP=$OBJDUMP + # Extract the first word of "objdump", so it can be a program name with args. +set dummy objdump; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_OBJDUMP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_OBJDUMP"; then + ac_cv_prog_ac_ct_OBJDUMP="$ac_ct_OBJDUMP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_OBJDUMP="objdump" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_OBJDUMP=$ac_cv_prog_ac_ct_OBJDUMP +if test -n "$ac_ct_OBJDUMP"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OBJDUMP" >&5 +$as_echo "$ac_ct_OBJDUMP" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_OBJDUMP" = x; then + OBJDUMP="false" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + OBJDUMP=$ac_ct_OBJDUMP + fi +else + OBJDUMP="$ac_cv_prog_OBJDUMP" +fi + +test -z "$OBJDUMP" && OBJDUMP=objdump + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to recognize dependent libraries" >&5 +$as_echo_n "checking how to recognize dependent libraries... " >&6; } +if ${lt_cv_deplibs_check_method+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_file_magic_cmd='$MAGIC_CMD' +lt_cv_file_magic_test_file= +lt_cv_deplibs_check_method='unknown' +# Need to set the preceding variable on all platforms that support +# interlibrary dependencies. +# 'none' -- dependencies not supported. +# `unknown' -- same as none, but documents that we really don't know. +# 'pass_all' -- all dependencies passed with no checks. +# 'test_compile' -- check by making test program. +# 'file_magic [[regex]]' -- check by looking for files in library path +# which responds to the $file_magic_cmd with a given extended regex. +# If you have `file' or equivalent on your system and you're not sure +# whether `pass_all' will *always* work, you probably want this one. + +case $host_os in +aix[4-9]*) + lt_cv_deplibs_check_method=pass_all + ;; + +beos*) + lt_cv_deplibs_check_method=pass_all + ;; + +bsdi[45]*) + lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (shared object|dynamic lib)' + lt_cv_file_magic_cmd='/usr/bin/file -L' + lt_cv_file_magic_test_file=/shlib/libc.so + ;; + +cygwin*) + # func_win32_libid is a shell function defined in ltmain.sh + lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL' + lt_cv_file_magic_cmd='func_win32_libid' + ;; + +mingw* | pw32*) + # Base MSYS/MinGW do not provide the 'file' command needed by + # func_win32_libid shell function, so use a weaker test based on 'objdump', + # unless we find 'file', for example because we are cross-compiling. + # func_win32_libid assumes BSD nm, so disallow it if using MS dumpbin. + if ( test "$lt_cv_nm_interface" = "BSD nm" && file / ) >/dev/null 2>&1; then + lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL' + lt_cv_file_magic_cmd='func_win32_libid' + else + # Keep this pattern in sync with the one in func_win32_libid. + lt_cv_deplibs_check_method='file_magic file format (pei*-i386(.*architecture: i386)?|pe-arm-wince|pe-x86-64)' + lt_cv_file_magic_cmd='$OBJDUMP -f' + fi + ;; + +cegcc*) + # use the weaker test based on 'objdump'. See mingw*. + lt_cv_deplibs_check_method='file_magic file format pe-arm-.*little(.*architecture: arm)?' + lt_cv_file_magic_cmd='$OBJDUMP -f' + ;; + +darwin* | rhapsody*) + lt_cv_deplibs_check_method=pass_all + ;; + +freebsd* | dragonfly*) + if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then + case $host_cpu in + i*86 ) + # Not sure whether the presence of OpenBSD here was a mistake. + # Let's accept both of them until this is cleared up. + lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[3-9]86 (compact )?demand paged shared library' + lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*` + ;; + esac + else + lt_cv_deplibs_check_method=pass_all + fi + ;; + +gnu*) + lt_cv_deplibs_check_method=pass_all + ;; + +haiku*) + lt_cv_deplibs_check_method=pass_all + ;; + +hpux10.20* | hpux11*) + lt_cv_file_magic_cmd=/usr/bin/file + case $host_cpu in + ia64*) + lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF-[0-9][0-9]) shared object file - IA64' + lt_cv_file_magic_test_file=/usr/lib/hpux32/libc.so + ;; + hppa*64*) + lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF[ -][0-9][0-9])(-bit)?( [LM]SB)? shared object( file)?[, -]* PA-RISC [0-9]\.[0-9]' + lt_cv_file_magic_test_file=/usr/lib/pa20_64/libc.sl + ;; + *) + lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|PA-RISC[0-9]\.[0-9]) shared library' + lt_cv_file_magic_test_file=/usr/lib/libc.sl + ;; + esac + ;; + +interix[3-9]*) + # PIC code is broken on Interix 3.x, that's why |\.a not |_pic\.a here + lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so|\.a)$' + ;; + +irix5* | irix6* | nonstopux*) + case $LD in + *-32|*"-32 ") libmagic=32-bit;; + *-n32|*"-n32 ") libmagic=N32;; + *-64|*"-64 ") libmagic=64-bit;; + *) libmagic=never-match;; + esac + lt_cv_deplibs_check_method=pass_all + ;; + +# This must be glibc/ELF. +linux* | k*bsd*-gnu | kopensolaris*-gnu) + lt_cv_deplibs_check_method=pass_all + ;; + +netbsd* | netbsdelf*-gnu) + if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then + lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|_pic\.a)$' + else + lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so|_pic\.a)$' + fi + ;; + +newos6*) + lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (executable|dynamic lib)' + lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_test_file=/usr/lib/libnls.so + ;; + +*nto* | *qnx*) + lt_cv_deplibs_check_method=pass_all + ;; + +openbsd*) + if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|\.so|_pic\.a)$' + else + lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|_pic\.a)$' + fi + ;; + +osf3* | osf4* | osf5*) + lt_cv_deplibs_check_method=pass_all + ;; + +rdos*) + lt_cv_deplibs_check_method=pass_all + ;; + +solaris*) + lt_cv_deplibs_check_method=pass_all + ;; + +sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) + lt_cv_deplibs_check_method=pass_all + ;; + +sysv4 | sysv4.3*) + case $host_vendor in + motorola) + lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (shared object|dynamic lib) M[0-9][0-9]* Version [0-9]' + lt_cv_file_magic_test_file=`echo /usr/lib/libc.so*` + ;; + ncr) + lt_cv_deplibs_check_method=pass_all + ;; + sequent) + lt_cv_file_magic_cmd='/bin/file' + lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [LM]SB (shared object|dynamic lib )' + ;; + sni) + lt_cv_file_magic_cmd='/bin/file' + lt_cv_deplibs_check_method="file_magic ELF [0-9][0-9]*-bit [LM]SB dynamic lib" + lt_cv_file_magic_test_file=/lib/libc.so + ;; + siemens) + lt_cv_deplibs_check_method=pass_all + ;; + pc) + lt_cv_deplibs_check_method=pass_all + ;; + esac + ;; + +tpf*) + lt_cv_deplibs_check_method=pass_all + ;; +esac + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_deplibs_check_method" >&5 +$as_echo "$lt_cv_deplibs_check_method" >&6; } + +file_magic_glob= +want_nocaseglob=no +if test "$build" = "$host"; then + case $host_os in + mingw* | pw32*) + if ( shopt | grep nocaseglob ) >/dev/null 2>&1; then + want_nocaseglob=yes + else + file_magic_glob=`echo aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ | $SED -e "s/\(..\)/s\/[\1]\/[\1]\/g;/g"` + fi + ;; + esac +fi + +file_magic_cmd=$lt_cv_file_magic_cmd +deplibs_check_method=$lt_cv_deplibs_check_method +test -z "$deplibs_check_method" && deplibs_check_method=unknown + + + + + + + + + + + + + + + + + + + + + + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}dlltool", so it can be a program name with args. +set dummy ${ac_tool_prefix}dlltool; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_DLLTOOL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$DLLTOOL"; then + ac_cv_prog_DLLTOOL="$DLLTOOL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_DLLTOOL="${ac_tool_prefix}dlltool" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +DLLTOOL=$ac_cv_prog_DLLTOOL +if test -n "$DLLTOOL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DLLTOOL" >&5 +$as_echo "$DLLTOOL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_DLLTOOL"; then + ac_ct_DLLTOOL=$DLLTOOL + # Extract the first word of "dlltool", so it can be a program name with args. +set dummy dlltool; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_DLLTOOL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_DLLTOOL"; then + ac_cv_prog_ac_ct_DLLTOOL="$ac_ct_DLLTOOL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_DLLTOOL="dlltool" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_DLLTOOL=$ac_cv_prog_ac_ct_DLLTOOL +if test -n "$ac_ct_DLLTOOL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DLLTOOL" >&5 +$as_echo "$ac_ct_DLLTOOL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_DLLTOOL" = x; then + DLLTOOL="false" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + DLLTOOL=$ac_ct_DLLTOOL + fi +else + DLLTOOL="$ac_cv_prog_DLLTOOL" +fi + +test -z "$DLLTOOL" && DLLTOOL=dlltool + + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to associate runtime and link libraries" >&5 +$as_echo_n "checking how to associate runtime and link libraries... " >&6; } +if ${lt_cv_sharedlib_from_linklib_cmd+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_sharedlib_from_linklib_cmd='unknown' + +case $host_os in +cygwin* | mingw* | pw32* | cegcc*) + # two different shell functions defined in ltmain.sh + # decide which to use based on capabilities of $DLLTOOL + case `$DLLTOOL --help 2>&1` in + *--identify-strict*) + lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib + ;; + *) + lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib_fallback + ;; + esac + ;; +*) + # fallback: assume linklib IS sharedlib + lt_cv_sharedlib_from_linklib_cmd="$ECHO" + ;; +esac + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_sharedlib_from_linklib_cmd" >&5 +$as_echo "$lt_cv_sharedlib_from_linklib_cmd" >&6; } +sharedlib_from_linklib_cmd=$lt_cv_sharedlib_from_linklib_cmd +test -z "$sharedlib_from_linklib_cmd" && sharedlib_from_linklib_cmd=$ECHO + + + + + + + +if test -n "$ac_tool_prefix"; then + for ac_prog in ar + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_AR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$AR"; then + ac_cv_prog_AR="$AR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_AR="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +AR=$ac_cv_prog_AR +if test -n "$AR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AR" >&5 +$as_echo "$AR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$AR" && break + done +fi +if test -z "$AR"; then + ac_ct_AR=$AR + for ac_prog in ar +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_AR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_AR"; then + ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_AR="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_AR=$ac_cv_prog_ac_ct_AR +if test -n "$ac_ct_AR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_AR" >&5 +$as_echo "$ac_ct_AR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_AR" && break +done + + if test "x$ac_ct_AR" = x; then + AR="false" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + AR=$ac_ct_AR + fi +fi + +: ${AR=ar} +: ${AR_FLAGS=cru} + + + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for archiver @FILE support" >&5 +$as_echo_n "checking for archiver @FILE support... " >&6; } +if ${lt_cv_ar_at_file+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_ar_at_file=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + echo conftest.$ac_objext > conftest.lst + lt_ar_try='$AR $AR_FLAGS libconftest.a @conftest.lst >&5' + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$lt_ar_try\""; } >&5 + (eval $lt_ar_try) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + if test "$ac_status" -eq 0; then + # Ensure the archiver fails upon bogus file names. + rm -f conftest.$ac_objext libconftest.a + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$lt_ar_try\""; } >&5 + (eval $lt_ar_try) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + if test "$ac_status" -ne 0; then + lt_cv_ar_at_file=@ + fi + fi + rm -f conftest.* libconftest.a + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ar_at_file" >&5 +$as_echo "$lt_cv_ar_at_file" >&6; } + +if test "x$lt_cv_ar_at_file" = xno; then + archiver_list_spec= +else + archiver_list_spec=$lt_cv_ar_at_file +fi + + + + + + + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. +set dummy ${ac_tool_prefix}strip; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_STRIP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$STRIP"; then + ac_cv_prog_STRIP="$STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_STRIP="${ac_tool_prefix}strip" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +STRIP=$ac_cv_prog_STRIP +if test -n "$STRIP"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5 +$as_echo "$STRIP" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_STRIP"; then + ac_ct_STRIP=$STRIP + # Extract the first word of "strip", so it can be a program name with args. +set dummy strip; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_STRIP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_STRIP"; then + ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_STRIP="strip" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP +if test -n "$ac_ct_STRIP"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5 +$as_echo "$ac_ct_STRIP" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_STRIP" = x; then + STRIP=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + STRIP=$ac_ct_STRIP + fi +else + STRIP="$ac_cv_prog_STRIP" +fi + +test -z "$STRIP" && STRIP=: + + + + + + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args. +set dummy ${ac_tool_prefix}ranlib; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_RANLIB+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$RANLIB"; then + ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +RANLIB=$ac_cv_prog_RANLIB +if test -n "$RANLIB"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $RANLIB" >&5 +$as_echo "$RANLIB" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_RANLIB"; then + ac_ct_RANLIB=$RANLIB + # Extract the first word of "ranlib", so it can be a program name with args. +set dummy ranlib; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_RANLIB+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_RANLIB"; then + ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_RANLIB="ranlib" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB +if test -n "$ac_ct_RANLIB"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_RANLIB" >&5 +$as_echo "$ac_ct_RANLIB" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_RANLIB" = x; then + RANLIB=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + RANLIB=$ac_ct_RANLIB + fi +else + RANLIB="$ac_cv_prog_RANLIB" +fi + +test -z "$RANLIB" && RANLIB=: + + + + + + +# Determine commands to create old-style static archives. +old_archive_cmds='$AR $AR_FLAGS $oldlib$oldobjs' +old_postinstall_cmds='chmod 644 $oldlib' +old_postuninstall_cmds= + +if test -n "$RANLIB"; then + case $host_os in + openbsd*) + old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB -t \$tool_oldlib" + ;; + *) + old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB \$tool_oldlib" + ;; + esac + old_archive_cmds="$old_archive_cmds~\$RANLIB \$tool_oldlib" +fi + +case $host_os in + darwin*) + lock_old_archive_extraction=yes ;; + *) + lock_old_archive_extraction=no ;; +esac + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# If no C compiler was specified, use CC. +LTCC=${LTCC-"$CC"} + +# If no C compiler flags were specified, use CFLAGS. +LTCFLAGS=${LTCFLAGS-"$CFLAGS"} + +# Allow CC to be a program name with arguments. +compiler=$CC + + +# Check for command to grab the raw symbol name followed by C symbol from nm. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking command to parse $NM output from $compiler object" >&5 +$as_echo_n "checking command to parse $NM output from $compiler object... " >&6; } +if ${lt_cv_sys_global_symbol_pipe+:} false; then : + $as_echo_n "(cached) " >&6 +else + +# These are sane defaults that work on at least a few old systems. +# [They come from Ultrix. What could be older than Ultrix?!! ;)] + +# Character class describing NM global symbol codes. +symcode='[BCDEGRST]' + +# Regexp to match symbols that can be accessed directly from C. +sympat='\([_A-Za-z][_A-Za-z0-9]*\)' + +# Define system-specific variables. +case $host_os in +aix*) + symcode='[BCDT]' + ;; +cygwin* | mingw* | pw32* | cegcc*) + symcode='[ABCDGISTW]' + ;; +hpux*) + if test "$host_cpu" = ia64; then + symcode='[ABCDEGRST]' + fi + ;; +irix* | nonstopux*) + symcode='[BCDEGRST]' + ;; +osf*) + symcode='[BCDEGQRST]' + ;; +solaris*) + symcode='[BDRT]' + ;; +sco3.2v5*) + symcode='[DT]' + ;; +sysv4.2uw2*) + symcode='[DT]' + ;; +sysv5* | sco5v6* | unixware* | OpenUNIX*) + symcode='[ABDT]' + ;; +sysv4) + symcode='[DFNSTU]' + ;; +esac + +# If we're using GNU nm, then use its standard symbol codes. +case `$NM -V 2>&1` in +*GNU* | *'with BFD'*) + symcode='[ABCDGIRSTW]' ;; +esac + +# Transform an extracted symbol line into a proper C declaration. +# Some systems (esp. on ia64) link data and code symbols differently, +# so use this general approach. +lt_cv_sys_global_symbol_to_cdecl="sed -n -e 's/^T .* \(.*\)$/extern int \1();/p' -e 's/^$symcode* .* \(.*\)$/extern char \1;/p'" + +# Transform an extracted symbol line into symbol name and symbol address +lt_cv_sys_global_symbol_to_c_name_address="sed -n -e 's/^: \([^ ]*\)[ ]*$/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([^ ]*\) \([^ ]*\)$/ {\"\2\", (void *) \&\2},/p'" +lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n -e 's/^: \([^ ]*\)[ ]*$/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([^ ]*\) \(lib[^ ]*\)$/ {\"\2\", (void *) \&\2},/p' -e 's/^$symcode* \([^ ]*\) \([^ ]*\)$/ {\"lib\2\", (void *) \&\2},/p'" + +# Handle CRLF in mingw tool chain +opt_cr= +case $build_os in +mingw*) + opt_cr=`$ECHO 'x\{0,1\}' | tr x '\015'` # option cr in regexp + ;; +esac + +# Try without a prefix underscore, then with it. +for ac_symprfx in "" "_"; do + + # Transform symcode, sympat, and symprfx into a raw symbol and a C symbol. + symxfrm="\\1 $ac_symprfx\\2 \\2" + + # Write the raw and C identifiers. + if test "$lt_cv_nm_interface" = "MS dumpbin"; then + # Fake it for dumpbin and say T for any non-static function + # and D for any global variable. + # Also find C++ and __fastcall symbols from MSVC++, + # which start with @ or ?. + lt_cv_sys_global_symbol_pipe="$AWK '"\ +" {last_section=section; section=\$ 3};"\ +" /^COFF SYMBOL TABLE/{for(i in hide) delete hide[i]};"\ +" /Section length .*#relocs.*(pick any)/{hide[last_section]=1};"\ +" \$ 0!~/External *\|/{next};"\ +" / 0+ UNDEF /{next}; / UNDEF \([^|]\)*()/{next};"\ +" {if(hide[section]) next};"\ +" {f=0}; \$ 0~/\(\).*\|/{f=1}; {printf f ? \"T \" : \"D \"};"\ +" {split(\$ 0, a, /\||\r/); split(a[2], s)};"\ +" s[1]~/^[@?]/{print s[1], s[1]; next};"\ +" s[1]~prfx {split(s[1],t,\"@\"); print t[1], substr(t[1],length(prfx))}"\ +" ' prfx=^$ac_symprfx" + else + lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[ ]\($symcode$symcode*\)[ ][ ]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" + fi + lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | sed '/ __gnu_lto/d'" + + # Check to see that the pipe works correctly. + pipe_works=no + + rm -f conftest* + cat > conftest.$ac_ext <<_LT_EOF +#ifdef __cplusplus +extern "C" { +#endif +char nm_test_var; +void nm_test_func(void); +void nm_test_func(void){} +#ifdef __cplusplus +} +#endif +int main(){nm_test_var='a';nm_test_func();return(0);} +_LT_EOF + + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + # Now try to grab the symbols. + nlist=conftest.nm + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$NM conftest.$ac_objext \| "$lt_cv_sys_global_symbol_pipe" \> $nlist\""; } >&5 + (eval $NM conftest.$ac_objext \| "$lt_cv_sys_global_symbol_pipe" \> $nlist) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && test -s "$nlist"; then + # Try sorting and uniquifying the output. + if sort "$nlist" | uniq > "$nlist"T; then + mv -f "$nlist"T "$nlist" + else + rm -f "$nlist"T + fi + + # Make sure that we snagged all the symbols we need. + if $GREP ' nm_test_var$' "$nlist" >/dev/null; then + if $GREP ' nm_test_func$' "$nlist" >/dev/null; then + cat <<_LT_EOF > conftest.$ac_ext +/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ +#if defined(_WIN32) || defined(__CYGWIN__) || defined(_WIN32_WCE) +/* DATA imports from DLLs on WIN32 con't be const, because runtime + relocations are performed -- see ld's documentation on pseudo-relocs. */ +# define LT_DLSYM_CONST +#elif defined(__osf__) +/* This system does not cope well with relocations in const data. */ +# define LT_DLSYM_CONST +#else +# define LT_DLSYM_CONST const +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +_LT_EOF + # Now generate the symbol file. + eval "$lt_cv_sys_global_symbol_to_cdecl"' < "$nlist" | $GREP -v main >> conftest.$ac_ext' + + cat <<_LT_EOF >> conftest.$ac_ext + +/* The mapping between symbol names and symbols. */ +LT_DLSYM_CONST struct { + const char *name; + void *address; +} +lt__PROGRAM__LTX_preloaded_symbols[] = +{ + { "@PROGRAM@", (void *) 0 }, +_LT_EOF + $SED "s/^$symcode$symcode* \(.*\) \(.*\)$/ {\"\2\", (void *) \&\2},/" < "$nlist" | $GREP -v main >> conftest.$ac_ext + cat <<\_LT_EOF >> conftest.$ac_ext + {0, (void *) 0} +}; + +/* This works around a problem in FreeBSD linker */ +#ifdef FREEBSD_WORKAROUND +static const void *lt_preloaded_setup() { + return lt__PROGRAM__LTX_preloaded_symbols; +} +#endif + +#ifdef __cplusplus +} +#endif +_LT_EOF + # Now try linking the two files. + mv conftest.$ac_objext conftstm.$ac_objext + lt_globsym_save_LIBS=$LIBS + lt_globsym_save_CFLAGS=$CFLAGS + LIBS="conftstm.$ac_objext" + CFLAGS="$CFLAGS$lt_prog_compiler_no_builtin_flag" + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5 + (eval $ac_link) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && test -s conftest${ac_exeext}; then + pipe_works=yes + fi + LIBS=$lt_globsym_save_LIBS + CFLAGS=$lt_globsym_save_CFLAGS + else + echo "cannot find nm_test_func in $nlist" >&5 + fi + else + echo "cannot find nm_test_var in $nlist" >&5 + fi + else + echo "cannot run $lt_cv_sys_global_symbol_pipe" >&5 + fi + else + echo "$progname: failed program was:" >&5 + cat conftest.$ac_ext >&5 + fi + rm -rf conftest* conftst* + + # Do not use the global_symbol_pipe unless it works. + if test "$pipe_works" = yes; then + break + else + lt_cv_sys_global_symbol_pipe= + fi +done + +fi + +if test -z "$lt_cv_sys_global_symbol_pipe"; then + lt_cv_sys_global_symbol_to_cdecl= +fi +if test -z "$lt_cv_sys_global_symbol_pipe$lt_cv_sys_global_symbol_to_cdecl"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: failed" >&5 +$as_echo "failed" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: ok" >&5 +$as_echo "ok" >&6; } +fi + +# Response file support. +if test "$lt_cv_nm_interface" = "MS dumpbin"; then + nm_file_list_spec='@' +elif $NM --help 2>/dev/null | grep '[@]FILE' >/dev/null; then + nm_file_list_spec='@' +fi + + + + + + + + + + + + + + + + + + + + + + + + + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for sysroot" >&5 +$as_echo_n "checking for sysroot... " >&6; } + +# Check whether --with-sysroot was given. +if test "${with_sysroot+set}" = set; then : + withval=$with_sysroot; +else + with_sysroot=no +fi + + +lt_sysroot= +case ${with_sysroot} in #( + yes) + if test "$GCC" = yes; then + lt_sysroot=`$CC --print-sysroot 2>/dev/null` + fi + ;; #( + /*) + lt_sysroot=`echo "$with_sysroot" | sed -e "$sed_quote_subst"` + ;; #( + no|'') + ;; #( + *) + { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${with_sysroot}" >&5 +$as_echo "${with_sysroot}" >&6; } + as_fn_error $? "The sysroot must be an absolute path." "$LINENO" 5 + ;; +esac + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${lt_sysroot:-no}" >&5 +$as_echo "${lt_sysroot:-no}" >&6; } + + + + + +# Check whether --enable-libtool-lock was given. +if test "${enable_libtool_lock+set}" = set; then : + enableval=$enable_libtool_lock; +fi + +test "x$enable_libtool_lock" != xno && enable_libtool_lock=yes + +# Some flags need to be propagated to the compiler or linker for good +# libtool support. +case $host in +ia64-*-hpux*) + # Find out which ABI we are using. + echo 'int i;' > conftest.$ac_ext + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + case `/usr/bin/file conftest.$ac_objext` in + *ELF-32*) + HPUX_IA64_MODE="32" + ;; + *ELF-64*) + HPUX_IA64_MODE="64" + ;; + esac + fi + rm -rf conftest* + ;; +*-*-irix6*) + # Find out which ABI we are using. + echo '#line '$LINENO' "configure"' > conftest.$ac_ext + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + if test "$lt_cv_prog_gnu_ld" = yes; then + case `/usr/bin/file conftest.$ac_objext` in + *32-bit*) + LD="${LD-ld} -melf32bsmip" + ;; + *N32*) + LD="${LD-ld} -melf32bmipn32" + ;; + *64-bit*) + LD="${LD-ld} -melf64bmip" + ;; + esac + else + case `/usr/bin/file conftest.$ac_objext` in + *32-bit*) + LD="${LD-ld} -32" + ;; + *N32*) + LD="${LD-ld} -n32" + ;; + *64-bit*) + LD="${LD-ld} -64" + ;; + esac + fi + fi + rm -rf conftest* + ;; + +x86_64-*kfreebsd*-gnu|x86_64-*linux*|ppc*-*linux*|powerpc*-*linux*| \ +s390*-*linux*|s390*-*tpf*|sparc*-*linux*) + # Find out which ABI we are using. + echo 'int i;' > conftest.$ac_ext + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + case `/usr/bin/file conftest.o` in + *32-bit*) + case $host in + x86_64-*kfreebsd*-gnu) + LD="${LD-ld} -m elf_i386_fbsd" + ;; + x86_64-*linux*) + LD="${LD-ld} -m elf_i386" + ;; + ppc64-*linux*|powerpc64-*linux*) + LD="${LD-ld} -m elf32ppclinux" + ;; + s390x-*linux*) + LD="${LD-ld} -m elf_s390" + ;; + sparc64-*linux*) + LD="${LD-ld} -m elf32_sparc" + ;; + esac + ;; + *64-bit*) + case $host in + x86_64-*kfreebsd*-gnu) + LD="${LD-ld} -m elf_x86_64_fbsd" + ;; + x86_64-*linux*) + LD="${LD-ld} -m elf_x86_64" + ;; + ppc*-*linux*|powerpc*-*linux*) + LD="${LD-ld} -m elf64ppc" + ;; + s390*-*linux*|s390*-*tpf*) + LD="${LD-ld} -m elf64_s390" + ;; + sparc*-*linux*) + LD="${LD-ld} -m elf64_sparc" + ;; + esac + ;; + esac + fi + rm -rf conftest* + ;; + +*-*-sco3.2v5*) + # On SCO OpenServer 5, we need -belf to get full-featured binaries. + SAVE_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS -belf" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler needs -belf" >&5 +$as_echo_n "checking whether the C compiler needs -belf... " >&6; } +if ${lt_cv_cc_needs_belf+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + lt_cv_cc_needs_belf=yes +else + lt_cv_cc_needs_belf=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_cc_needs_belf" >&5 +$as_echo "$lt_cv_cc_needs_belf" >&6; } + if test x"$lt_cv_cc_needs_belf" != x"yes"; then + # this is probably gcc 2.8.0, egcs 1.0 or newer; no need for -belf + CFLAGS="$SAVE_CFLAGS" + fi + ;; +*-*solaris*) + # Find out which ABI we are using. + echo 'int i;' > conftest.$ac_ext + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + case `/usr/bin/file conftest.o` in + *64-bit*) + case $lt_cv_prog_gnu_ld in + yes*) + case $host in + i?86-*-solaris*) + LD="${LD-ld} -m elf_x86_64" + ;; + sparc*-*-solaris*) + LD="${LD-ld} -m elf64_sparc" + ;; + esac + # GNU ld 2.21 introduced _sol2 emulations. Use them if available. + if ${LD-ld} -V | grep _sol2 >/dev/null 2>&1; then + LD="${LD-ld}_sol2" + fi + ;; + *) + if ${LD-ld} -64 -r -o conftest2.o conftest.o >/dev/null 2>&1; then + LD="${LD-ld} -64" + fi + ;; + esac + ;; + esac + fi + rm -rf conftest* + ;; +esac + +need_locks="$enable_libtool_lock" + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}mt", so it can be a program name with args. +set dummy ${ac_tool_prefix}mt; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_MANIFEST_TOOL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$MANIFEST_TOOL"; then + ac_cv_prog_MANIFEST_TOOL="$MANIFEST_TOOL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_MANIFEST_TOOL="${ac_tool_prefix}mt" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +MANIFEST_TOOL=$ac_cv_prog_MANIFEST_TOOL +if test -n "$MANIFEST_TOOL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MANIFEST_TOOL" >&5 +$as_echo "$MANIFEST_TOOL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_MANIFEST_TOOL"; then + ac_ct_MANIFEST_TOOL=$MANIFEST_TOOL + # Extract the first word of "mt", so it can be a program name with args. +set dummy mt; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_MANIFEST_TOOL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_MANIFEST_TOOL"; then + ac_cv_prog_ac_ct_MANIFEST_TOOL="$ac_ct_MANIFEST_TOOL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_MANIFEST_TOOL="mt" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_MANIFEST_TOOL=$ac_cv_prog_ac_ct_MANIFEST_TOOL +if test -n "$ac_ct_MANIFEST_TOOL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_MANIFEST_TOOL" >&5 +$as_echo "$ac_ct_MANIFEST_TOOL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_MANIFEST_TOOL" = x; then + MANIFEST_TOOL=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + MANIFEST_TOOL=$ac_ct_MANIFEST_TOOL + fi +else + MANIFEST_TOOL="$ac_cv_prog_MANIFEST_TOOL" +fi + +test -z "$MANIFEST_TOOL" && MANIFEST_TOOL=mt +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $MANIFEST_TOOL is a manifest tool" >&5 +$as_echo_n "checking if $MANIFEST_TOOL is a manifest tool... " >&6; } +if ${lt_cv_path_mainfest_tool+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_path_mainfest_tool=no + echo "$as_me:$LINENO: $MANIFEST_TOOL '-?'" >&5 + $MANIFEST_TOOL '-?' 2>conftest.err > conftest.out + cat conftest.err >&5 + if $GREP 'Manifest Tool' conftest.out > /dev/null; then + lt_cv_path_mainfest_tool=yes + fi + rm -f conftest* +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_path_mainfest_tool" >&5 +$as_echo "$lt_cv_path_mainfest_tool" >&6; } +if test "x$lt_cv_path_mainfest_tool" != xyes; then + MANIFEST_TOOL=: +fi + + + + + + + case $host_os in + rhapsody* | darwin*) + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}dsymutil", so it can be a program name with args. +set dummy ${ac_tool_prefix}dsymutil; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_DSYMUTIL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$DSYMUTIL"; then + ac_cv_prog_DSYMUTIL="$DSYMUTIL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_DSYMUTIL="${ac_tool_prefix}dsymutil" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +DSYMUTIL=$ac_cv_prog_DSYMUTIL +if test -n "$DSYMUTIL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DSYMUTIL" >&5 +$as_echo "$DSYMUTIL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_DSYMUTIL"; then + ac_ct_DSYMUTIL=$DSYMUTIL + # Extract the first word of "dsymutil", so it can be a program name with args. +set dummy dsymutil; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_DSYMUTIL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_DSYMUTIL"; then + ac_cv_prog_ac_ct_DSYMUTIL="$ac_ct_DSYMUTIL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_DSYMUTIL="dsymutil" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_DSYMUTIL=$ac_cv_prog_ac_ct_DSYMUTIL +if test -n "$ac_ct_DSYMUTIL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DSYMUTIL" >&5 +$as_echo "$ac_ct_DSYMUTIL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_DSYMUTIL" = x; then + DSYMUTIL=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + DSYMUTIL=$ac_ct_DSYMUTIL + fi +else + DSYMUTIL="$ac_cv_prog_DSYMUTIL" +fi + + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}nmedit", so it can be a program name with args. +set dummy ${ac_tool_prefix}nmedit; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_NMEDIT+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$NMEDIT"; then + ac_cv_prog_NMEDIT="$NMEDIT" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_NMEDIT="${ac_tool_prefix}nmedit" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +NMEDIT=$ac_cv_prog_NMEDIT +if test -n "$NMEDIT"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NMEDIT" >&5 +$as_echo "$NMEDIT" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_NMEDIT"; then + ac_ct_NMEDIT=$NMEDIT + # Extract the first word of "nmedit", so it can be a program name with args. +set dummy nmedit; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_NMEDIT+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_NMEDIT"; then + ac_cv_prog_ac_ct_NMEDIT="$ac_ct_NMEDIT" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_NMEDIT="nmedit" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_NMEDIT=$ac_cv_prog_ac_ct_NMEDIT +if test -n "$ac_ct_NMEDIT"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_NMEDIT" >&5 +$as_echo "$ac_ct_NMEDIT" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_NMEDIT" = x; then + NMEDIT=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + NMEDIT=$ac_ct_NMEDIT + fi +else + NMEDIT="$ac_cv_prog_NMEDIT" +fi + + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}lipo", so it can be a program name with args. +set dummy ${ac_tool_prefix}lipo; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_LIPO+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$LIPO"; then + ac_cv_prog_LIPO="$LIPO" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_LIPO="${ac_tool_prefix}lipo" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +LIPO=$ac_cv_prog_LIPO +if test -n "$LIPO"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LIPO" >&5 +$as_echo "$LIPO" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_LIPO"; then + ac_ct_LIPO=$LIPO + # Extract the first word of "lipo", so it can be a program name with args. +set dummy lipo; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_LIPO+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_LIPO"; then + ac_cv_prog_ac_ct_LIPO="$ac_ct_LIPO" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_LIPO="lipo" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_LIPO=$ac_cv_prog_ac_ct_LIPO +if test -n "$ac_ct_LIPO"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_LIPO" >&5 +$as_echo "$ac_ct_LIPO" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_LIPO" = x; then + LIPO=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + LIPO=$ac_ct_LIPO + fi +else + LIPO="$ac_cv_prog_LIPO" +fi + + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}otool", so it can be a program name with args. +set dummy ${ac_tool_prefix}otool; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_OTOOL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$OTOOL"; then + ac_cv_prog_OTOOL="$OTOOL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_OTOOL="${ac_tool_prefix}otool" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +OTOOL=$ac_cv_prog_OTOOL +if test -n "$OTOOL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OTOOL" >&5 +$as_echo "$OTOOL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_OTOOL"; then + ac_ct_OTOOL=$OTOOL + # Extract the first word of "otool", so it can be a program name with args. +set dummy otool; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_OTOOL+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_OTOOL"; then + ac_cv_prog_ac_ct_OTOOL="$ac_ct_OTOOL" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_OTOOL="otool" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_OTOOL=$ac_cv_prog_ac_ct_OTOOL +if test -n "$ac_ct_OTOOL"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OTOOL" >&5 +$as_echo "$ac_ct_OTOOL" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_OTOOL" = x; then + OTOOL=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + OTOOL=$ac_ct_OTOOL + fi +else + OTOOL="$ac_cv_prog_OTOOL" +fi + + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}otool64", so it can be a program name with args. +set dummy ${ac_tool_prefix}otool64; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_OTOOL64+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$OTOOL64"; then + ac_cv_prog_OTOOL64="$OTOOL64" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_OTOOL64="${ac_tool_prefix}otool64" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +OTOOL64=$ac_cv_prog_OTOOL64 +if test -n "$OTOOL64"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OTOOL64" >&5 +$as_echo "$OTOOL64" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_OTOOL64"; then + ac_ct_OTOOL64=$OTOOL64 + # Extract the first word of "otool64", so it can be a program name with args. +set dummy otool64; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_OTOOL64+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_OTOOL64"; then + ac_cv_prog_ac_ct_OTOOL64="$ac_ct_OTOOL64" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_OTOOL64="otool64" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_OTOOL64=$ac_cv_prog_ac_ct_OTOOL64 +if test -n "$ac_ct_OTOOL64"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OTOOL64" >&5 +$as_echo "$ac_ct_OTOOL64" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_OTOOL64" = x; then + OTOOL64=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + OTOOL64=$ac_ct_OTOOL64 + fi +else + OTOOL64="$ac_cv_prog_OTOOL64" +fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -single_module linker flag" >&5 +$as_echo_n "checking for -single_module linker flag... " >&6; } +if ${lt_cv_apple_cc_single_mod+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_apple_cc_single_mod=no + if test -z "${LT_MULTI_MODULE}"; then + # By default we will add the -single_module flag. You can override + # by either setting the environment variable LT_MULTI_MODULE + # non-empty at configure time, or by adding -multi_module to the + # link flags. + rm -rf libconftest.dylib* + echo "int foo(void){return 1;}" > conftest.c + echo "$LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ +-dynamiclib -Wl,-single_module conftest.c" >&5 + $LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ + -dynamiclib -Wl,-single_module conftest.c 2>conftest.err + _lt_result=$? + # If there is a non-empty error log, and "single_module" + # appears in it, assume the flag caused a linker warning + if test -s conftest.err && $GREP single_module conftest.err; then + cat conftest.err >&5 + # Otherwise, if the output was created with a 0 exit code from + # the compiler, it worked. + elif test -f libconftest.dylib && test $_lt_result -eq 0; then + lt_cv_apple_cc_single_mod=yes + else + cat conftest.err >&5 + fi + rm -rf libconftest.dylib* + rm -f conftest.* + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_apple_cc_single_mod" >&5 +$as_echo "$lt_cv_apple_cc_single_mod" >&6; } + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -exported_symbols_list linker flag" >&5 +$as_echo_n "checking for -exported_symbols_list linker flag... " >&6; } +if ${lt_cv_ld_exported_symbols_list+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_ld_exported_symbols_list=no + save_LDFLAGS=$LDFLAGS + echo "_main" > conftest.sym + LDFLAGS="$LDFLAGS -Wl,-exported_symbols_list,conftest.sym" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + lt_cv_ld_exported_symbols_list=yes +else + lt_cv_ld_exported_symbols_list=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LDFLAGS="$save_LDFLAGS" + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_exported_symbols_list" >&5 +$as_echo "$lt_cv_ld_exported_symbols_list" >&6; } + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -force_load linker flag" >&5 +$as_echo_n "checking for -force_load linker flag... " >&6; } +if ${lt_cv_ld_force_load+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_ld_force_load=no + cat > conftest.c << _LT_EOF +int forced_loaded() { return 2;} +_LT_EOF + echo "$LTCC $LTCFLAGS -c -o conftest.o conftest.c" >&5 + $LTCC $LTCFLAGS -c -o conftest.o conftest.c 2>&5 + echo "$AR cru libconftest.a conftest.o" >&5 + $AR cru libconftest.a conftest.o 2>&5 + echo "$RANLIB libconftest.a" >&5 + $RANLIB libconftest.a 2>&5 + cat > conftest.c << _LT_EOF +int main() { return 0;} +_LT_EOF + echo "$LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a" >&5 + $LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a 2>conftest.err + _lt_result=$? + if test -s conftest.err && $GREP force_load conftest.err; then + cat conftest.err >&5 + elif test -f conftest && test $_lt_result -eq 0 && $GREP forced_load conftest >/dev/null 2>&1 ; then + lt_cv_ld_force_load=yes + else + cat conftest.err >&5 + fi + rm -f conftest.err libconftest.a conftest conftest.c + rm -rf conftest.dSYM + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_force_load" >&5 +$as_echo "$lt_cv_ld_force_load" >&6; } + case $host_os in + rhapsody* | darwin1.[012]) + _lt_dar_allow_undefined='${wl}-undefined ${wl}suppress' ;; + darwin1.*) + _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;; + darwin*) # darwin 5.x on + # if running on 10.5 or later, the deployment target defaults + # to the OS version, if on x86, and 10.4, the deployment + # target defaults to 10.4. Don't you love it? + case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in + 10.0,*86*-darwin8*|10.0,*-darwin[91]*) + _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;; + 10.[012]*) + _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;; + 10.*) + _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;; + esac + ;; + esac + if test "$lt_cv_apple_cc_single_mod" = "yes"; then + _lt_dar_single_mod='$single_module' + fi + if test "$lt_cv_ld_exported_symbols_list" = "yes"; then + _lt_dar_export_syms=' ${wl}-exported_symbols_list,$output_objdir/${libname}-symbols.expsym' + else + _lt_dar_export_syms='~$NMEDIT -s $output_objdir/${libname}-symbols.expsym ${lib}' + fi + if test "$DSYMUTIL" != ":" && test "$lt_cv_ld_force_load" = "no"; then + _lt_dsymutil='~$DSYMUTIL $lib || :' + else + _lt_dsymutil= + fi + ;; + esac + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 +$as_echo_n "checking how to run the C preprocessor... " >&6; } +# On Suns, sometimes $CPP names a directory. +if test -n "$CPP" && test -d "$CPP"; then + CPP= +fi +if test -z "$CPP"; then + if ${ac_cv_prog_CPP+:} false; then : + $as_echo_n "(cached) " >&6 +else + # Double quotes because CPP needs to be expanded + for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" + do + ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + break +fi + + done + ac_cv_prog_CPP=$CPP + +fi + CPP=$ac_cv_prog_CPP +else + ac_cv_prog_CPP=$CPP +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5 +$as_echo "$CPP" >&6; } +ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details" "$LINENO" 5; } +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 +$as_echo_n "checking for ANSI C header files... " >&6; } +if ${ac_cv_header_stdc+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#include +#include + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_header_stdc=yes +else + ac_cv_header_stdc=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +if test $ac_cv_header_stdc = yes; then + # SunOS 4.x string.h does not declare mem*, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "memchr" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "free" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. + if test "$cross_compiling" = yes; then : + : +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#if ((' ' & 0x0FF) == 0x020) +# define ISLOWER(c) ('a' <= (c) && (c) <= 'z') +# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) +#else +# define ISLOWER(c) \ + (('a' <= (c) && (c) <= 'i') \ + || ('j' <= (c) && (c) <= 'r') \ + || ('s' <= (c) && (c) <= 'z')) +# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) +#endif + +#define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) +int +main () +{ + int i; + for (i = 0; i < 256; i++) + if (XOR (islower (i), ISLOWER (i)) + || toupper (i) != TOUPPER (i)) + return 2; + return 0; +} +_ACEOF +if ac_fn_c_try_run "$LINENO"; then : + +else + ac_cv_header_stdc=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5 +$as_echo "$ac_cv_header_stdc" >&6; } +if test $ac_cv_header_stdc = yes; then + +$as_echo "#define STDC_HEADERS 1" >>confdefs.h + +fi + +# On IRIX 5.3, sys/types and inttypes.h are conflicting. +for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ + inttypes.h stdint.h unistd.h +do : + as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default +" +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + +fi + +done + + +for ac_header in dlfcn.h +do : + ac_fn_c_check_header_compile "$LINENO" "dlfcn.h" "ac_cv_header_dlfcn_h" "$ac_includes_default +" +if test "x$ac_cv_header_dlfcn_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_DLFCN_H 1 +_ACEOF + +fi + +done + + + + + +# Set options + + + + enable_dlopen=no + + + enable_win32_dll=no + + + # Check whether --enable-shared was given. +if test "${enable_shared+set}" = set; then : + enableval=$enable_shared; p=${PACKAGE-default} + case $enableval in + yes) enable_shared=yes ;; + no) enable_shared=no ;; + *) + enable_shared=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for pkg in $enableval; do + IFS="$lt_save_ifs" + if test "X$pkg" = "X$p"; then + enable_shared=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac +else + enable_shared=yes +fi + + + + + + + + + + # Check whether --enable-static was given. +if test "${enable_static+set}" = set; then : + enableval=$enable_static; p=${PACKAGE-default} + case $enableval in + yes) enable_static=yes ;; + no) enable_static=no ;; + *) + enable_static=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for pkg in $enableval; do + IFS="$lt_save_ifs" + if test "X$pkg" = "X$p"; then + enable_static=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac +else + enable_static=yes +fi + + + + + + + + + + +# Check whether --with-pic was given. +if test "${with_pic+set}" = set; then : + withval=$with_pic; lt_p=${PACKAGE-default} + case $withval in + yes|no) pic_mode=$withval ;; + *) + pic_mode=default + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for lt_pkg in $withval; do + IFS="$lt_save_ifs" + if test "X$lt_pkg" = "X$lt_p"; then + pic_mode=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac +else + pic_mode=default +fi + + +test -z "$pic_mode" && pic_mode=default + + + + + + + + # Check whether --enable-fast-install was given. +if test "${enable_fast_install+set}" = set; then : + enableval=$enable_fast_install; p=${PACKAGE-default} + case $enableval in + yes) enable_fast_install=yes ;; + no) enable_fast_install=no ;; + *) + enable_fast_install=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for pkg in $enableval; do + IFS="$lt_save_ifs" + if test "X$pkg" = "X$p"; then + enable_fast_install=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac +else + enable_fast_install=yes +fi + + + + + + + + + + + +# This can be used to rebuild libtool when needed +LIBTOOL_DEPS="$ltmain" + +# Always use our own libtool. +LIBTOOL='$(SHELL) $(top_builddir)/libtool' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +test -z "$LN_S" && LN_S="ln -s" + + + + + + + + + + + + + + +if test -n "${ZSH_VERSION+set}" ; then + setopt NO_GLOB_SUBST +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for objdir" >&5 +$as_echo_n "checking for objdir... " >&6; } +if ${lt_cv_objdir+:} false; then : + $as_echo_n "(cached) " >&6 +else + rm -f .libs 2>/dev/null +mkdir .libs 2>/dev/null +if test -d .libs; then + lt_cv_objdir=.libs +else + # MS-DOS does not allow filenames that begin with a dot. + lt_cv_objdir=_libs +fi +rmdir .libs 2>/dev/null +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_objdir" >&5 +$as_echo "$lt_cv_objdir" >&6; } +objdir=$lt_cv_objdir + + + + + +cat >>confdefs.h <<_ACEOF +#define LT_OBJDIR "$lt_cv_objdir/" +_ACEOF + + + + +case $host_os in +aix3*) + # AIX sometimes has problems with the GCC collect2 program. For some + # reason, if we set the COLLECT_NAMES environment variable, the problems + # vanish in a puff of smoke. + if test "X${COLLECT_NAMES+set}" != Xset; then + COLLECT_NAMES= + export COLLECT_NAMES + fi + ;; +esac + +# Global variables: +ofile=libtool +can_build_shared=yes + +# All known linkers require a `.a' archive for static linking (except MSVC, +# which needs '.lib'). +libext=a + +with_gnu_ld="$lt_cv_prog_gnu_ld" + +old_CC="$CC" +old_CFLAGS="$CFLAGS" + +# Set sane defaults for various variables +test -z "$CC" && CC=cc +test -z "$LTCC" && LTCC=$CC +test -z "$LTCFLAGS" && LTCFLAGS=$CFLAGS +test -z "$LD" && LD=ld +test -z "$ac_objext" && ac_objext=o + +for cc_temp in $compiler""; do + case $cc_temp in + compile | *[\\/]compile | ccache | *[\\/]ccache ) ;; + distcc | *[\\/]distcc | purify | *[\\/]purify ) ;; + \-*) ;; + *) break;; + esac +done +cc_basename=`$ECHO "$cc_temp" | $SED "s%.*/%%; s%^$host_alias-%%"` + + +# Only perform the check for file, if the check method requires it +test -z "$MAGIC_CMD" && MAGIC_CMD=file +case $deplibs_check_method in +file_magic*) + if test "$file_magic_cmd" = '$MAGIC_CMD'; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ${ac_tool_prefix}file" >&5 +$as_echo_n "checking for ${ac_tool_prefix}file... " >&6; } +if ${lt_cv_path_MAGIC_CMD+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $MAGIC_CMD in +[\\/*] | ?:[\\/]*) + lt_cv_path_MAGIC_CMD="$MAGIC_CMD" # Let the user override the test with a path. + ;; +*) + lt_save_MAGIC_CMD="$MAGIC_CMD" + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR + ac_dummy="/usr/bin$PATH_SEPARATOR$PATH" + for ac_dir in $ac_dummy; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + if test -f $ac_dir/${ac_tool_prefix}file; then + lt_cv_path_MAGIC_CMD="$ac_dir/${ac_tool_prefix}file" + if test -n "$file_magic_test_file"; then + case $deplibs_check_method in + "file_magic "*) + file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"` + MAGIC_CMD="$lt_cv_path_MAGIC_CMD" + if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null | + $EGREP "$file_magic_regex" > /dev/null; then + : + else + cat <<_LT_EOF 1>&2 + +*** Warning: the command libtool uses to detect shared libraries, +*** $file_magic_cmd, produces output that libtool cannot recognize. +*** The result is that libtool may fail to recognize shared libraries +*** as such. This will affect the creation of libtool libraries that +*** depend on shared libraries, but programs linked with such libtool +*** libraries will work regardless of this problem. Nevertheless, you +*** may want to report the problem to your system manager and/or to +*** bug-libtool@gnu.org + +_LT_EOF + fi ;; + esac + fi + break + fi + done + IFS="$lt_save_ifs" + MAGIC_CMD="$lt_save_MAGIC_CMD" + ;; +esac +fi + +MAGIC_CMD="$lt_cv_path_MAGIC_CMD" +if test -n "$MAGIC_CMD"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MAGIC_CMD" >&5 +$as_echo "$MAGIC_CMD" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + + + +if test -z "$lt_cv_path_MAGIC_CMD"; then + if test -n "$ac_tool_prefix"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for file" >&5 +$as_echo_n "checking for file... " >&6; } +if ${lt_cv_path_MAGIC_CMD+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $MAGIC_CMD in +[\\/*] | ?:[\\/]*) + lt_cv_path_MAGIC_CMD="$MAGIC_CMD" # Let the user override the test with a path. + ;; +*) + lt_save_MAGIC_CMD="$MAGIC_CMD" + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR + ac_dummy="/usr/bin$PATH_SEPARATOR$PATH" + for ac_dir in $ac_dummy; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + if test -f $ac_dir/file; then + lt_cv_path_MAGIC_CMD="$ac_dir/file" + if test -n "$file_magic_test_file"; then + case $deplibs_check_method in + "file_magic "*) + file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"` + MAGIC_CMD="$lt_cv_path_MAGIC_CMD" + if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null | + $EGREP "$file_magic_regex" > /dev/null; then + : + else + cat <<_LT_EOF 1>&2 + +*** Warning: the command libtool uses to detect shared libraries, +*** $file_magic_cmd, produces output that libtool cannot recognize. +*** The result is that libtool may fail to recognize shared libraries +*** as such. This will affect the creation of libtool libraries that +*** depend on shared libraries, but programs linked with such libtool +*** libraries will work regardless of this problem. Nevertheless, you +*** may want to report the problem to your system manager and/or to +*** bug-libtool@gnu.org + +_LT_EOF + fi ;; + esac + fi + break + fi + done + IFS="$lt_save_ifs" + MAGIC_CMD="$lt_save_MAGIC_CMD" + ;; +esac +fi + +MAGIC_CMD="$lt_cv_path_MAGIC_CMD" +if test -n "$MAGIC_CMD"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MAGIC_CMD" >&5 +$as_echo "$MAGIC_CMD" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + else + MAGIC_CMD=: + fi +fi + + fi + ;; +esac + +# Use C for the default configuration in the libtool script + +lt_save_CC="$CC" +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +# Source file extension for C test sources. +ac_ext=c + +# Object file extension for compiled C test sources. +objext=o +objext=$objext + +# Code to be used in simple compile tests +lt_simple_compile_test_code="int some_variable = 0;" + +# Code to be used in simple link tests +lt_simple_link_test_code='int main(){return(0);}' + + + + + + + +# If no C compiler was specified, use CC. +LTCC=${LTCC-"$CC"} + +# If no C compiler flags were specified, use CFLAGS. +LTCFLAGS=${LTCFLAGS-"$CFLAGS"} + +# Allow CC to be a program name with arguments. +compiler=$CC + +# Save the default compiler, since it gets overwritten when the other +# tags are being tested, and _LT_TAGVAR(compiler, []) is a NOP. +compiler_DEFAULT=$CC + +# save warnings/boilerplate of simple test code +ac_outfile=conftest.$ac_objext +echo "$lt_simple_compile_test_code" >conftest.$ac_ext +eval "$ac_compile" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err +_lt_compiler_boilerplate=`cat conftest.err` +$RM conftest* + +ac_outfile=conftest.$ac_objext +echo "$lt_simple_link_test_code" >conftest.$ac_ext +eval "$ac_link" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err +_lt_linker_boilerplate=`cat conftest.err` +$RM -r conftest* + + +## CAVEAT EMPTOR: +## There is no encapsulation within the following macros, do not change +## the running order or otherwise move them around unless you know exactly +## what you are doing... +if test -n "$compiler"; then + +lt_prog_compiler_no_builtin_flag= + +if test "$GCC" = yes; then + case $cc_basename in + nvcc*) + lt_prog_compiler_no_builtin_flag=' -Xcompiler -fno-builtin' ;; + *) + lt_prog_compiler_no_builtin_flag=' -fno-builtin' ;; + esac + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -fno-rtti -fno-exceptions" >&5 +$as_echo_n "checking if $compiler supports -fno-rtti -fno-exceptions... " >&6; } +if ${lt_cv_prog_compiler_rtti_exceptions+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_rtti_exceptions=no + ac_outfile=conftest.$ac_objext + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + lt_compiler_flag="-fno-rtti -fno-exceptions" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + # The option is referenced via a variable to avoid confusing sed. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) + (eval "$lt_compile" 2>conftest.err) + ac_status=$? + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + if (exit $ac_status) && test -s "$ac_outfile"; then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings other than the usual output. + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' >conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then + lt_cv_prog_compiler_rtti_exceptions=yes + fi + fi + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_rtti_exceptions" >&5 +$as_echo "$lt_cv_prog_compiler_rtti_exceptions" >&6; } + +if test x"$lt_cv_prog_compiler_rtti_exceptions" = xyes; then + lt_prog_compiler_no_builtin_flag="$lt_prog_compiler_no_builtin_flag -fno-rtti -fno-exceptions" +else + : +fi + +fi + + + + + + + lt_prog_compiler_wl= +lt_prog_compiler_pic= +lt_prog_compiler_static= + + + if test "$GCC" = yes; then + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_static='-static' + + case $host_os in + aix*) + # All AIX code is PIC. + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + lt_prog_compiler_static='-Bstatic' + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + lt_prog_compiler_pic='-fPIC' + ;; + m68k) + # FIXME: we need at least 68020 code to build shared libraries, but + # adding the `-m68020' flag to GCC prevents building anything better, + # like `-m68040'. + lt_prog_compiler_pic='-m68020 -resident32 -malways-restore-a4' + ;; + esac + ;; + + beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*) + # PIC is the default for these OSes. + ;; + + mingw* | cygwin* | pw32* | os2* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + # Although the cygwin gcc ignores -fPIC, still need this for old-style + # (--disable-auto-import) libraries + lt_prog_compiler_pic='-DDLL_EXPORT' + ;; + + darwin* | rhapsody*) + # PIC is the default on this platform + # Common symbols not allowed in MH_DYLIB files + lt_prog_compiler_pic='-fno-common' + ;; + + haiku*) + # PIC is the default for Haiku. + # The "-static" flag exists, but is broken. + lt_prog_compiler_static= + ;; + + hpux*) + # PIC is the default for 64-bit PA HP-UX, but not for 32-bit + # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag + # sets the default TLS model and affects inlining. + case $host_cpu in + hppa*64*) + # +Z the default + ;; + *) + lt_prog_compiler_pic='-fPIC' + ;; + esac + ;; + + interix[3-9]*) + # Interix 3.x gcc -fpic/-fPIC options generate broken code. + # Instead, we relocate shared libraries at runtime. + ;; + + msdosdjgpp*) + # Just because we use GCC doesn't mean we suddenly get shared libraries + # on systems that don't support them. + lt_prog_compiler_can_build_shared=no + enable_shared=no + ;; + + *nto* | *qnx*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + lt_prog_compiler_pic='-fPIC -shared' + ;; + + sysv4*MP*) + if test -d /usr/nec; then + lt_prog_compiler_pic=-Kconform_pic + fi + ;; + + *) + lt_prog_compiler_pic='-fPIC' + ;; + esac + + case $cc_basename in + nvcc*) # Cuda Compiler Driver 2.2 + lt_prog_compiler_wl='-Xlinker ' + if test -n "$lt_prog_compiler_pic"; then + lt_prog_compiler_pic="-Xcompiler $lt_prog_compiler_pic" + fi + ;; + esac + else + # PORTME Check for flag to pass linker flags through the system compiler. + case $host_os in + aix*) + lt_prog_compiler_wl='-Wl,' + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + lt_prog_compiler_static='-Bstatic' + else + lt_prog_compiler_static='-bnso -bI:/lib/syscalls.exp' + fi + ;; + + mingw* | cygwin* | pw32* | os2* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + lt_prog_compiler_pic='-DDLL_EXPORT' + ;; + + hpux9* | hpux10* | hpux11*) + lt_prog_compiler_wl='-Wl,' + # PIC is the default for IA64 HP-UX and 64-bit HP-UX, but + # not for PA HP-UX. + case $host_cpu in + hppa*64*|ia64*) + # +Z the default + ;; + *) + lt_prog_compiler_pic='+Z' + ;; + esac + # Is there a better lt_prog_compiler_static that works with the bundled CC? + lt_prog_compiler_static='${wl}-a ${wl}archive' + ;; + + irix5* | irix6* | nonstopux*) + lt_prog_compiler_wl='-Wl,' + # PIC (with -KPIC) is the default. + lt_prog_compiler_static='-non_shared' + ;; + + linux* | k*bsd*-gnu | kopensolaris*-gnu) + case $cc_basename in + # old Intel for x86_64 which still supported -KPIC. + ecc*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-static' + ;; + # icc used to be incompatible with GCC. + # ICC 10 doesn't accept -KPIC any more. + icc* | ifort*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-fPIC' + lt_prog_compiler_static='-static' + ;; + # Lahey Fortran 8.1. + lf95*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='--shared' + lt_prog_compiler_static='--static' + ;; + nagfor*) + # NAG Fortran compiler + lt_prog_compiler_wl='-Wl,-Wl,,' + lt_prog_compiler_pic='-PIC' + lt_prog_compiler_static='-Bstatic' + ;; + pgcc* | pgf77* | pgf90* | pgf95* | pgfortran*) + # Portland Group compilers (*not* the Pentium gcc compiler, + # which looks to be a dead project) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-fpic' + lt_prog_compiler_static='-Bstatic' + ;; + ccc*) + lt_prog_compiler_wl='-Wl,' + # All Alpha code is PIC. + lt_prog_compiler_static='-non_shared' + ;; + xl* | bgxl* | bgf* | mpixl*) + # IBM XL C 8.0/Fortran 10.1, 11.1 on PPC and BlueGene + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-qpic' + lt_prog_compiler_static='-qstaticlink' + ;; + *) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ Ceres\ Fortran* | *Sun*Fortran*\ [1-7].* | *Sun*Fortran*\ 8.[0-3]*) + # Sun Fortran 8.3 passes all unrecognized flags to the linker + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-Bstatic' + lt_prog_compiler_wl='' + ;; + *Sun\ F* | *Sun*Fortran*) + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-Bstatic' + lt_prog_compiler_wl='-Qoption ld ' + ;; + *Sun\ C*) + # Sun C 5.9 + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-Bstatic' + lt_prog_compiler_wl='-Wl,' + ;; + *Intel*\ [CF]*Compiler*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-fPIC' + lt_prog_compiler_static='-static' + ;; + *Portland\ Group*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-fpic' + lt_prog_compiler_static='-Bstatic' + ;; + esac + ;; + esac + ;; + + newsos6) + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-Bstatic' + ;; + + *nto* | *qnx*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + lt_prog_compiler_pic='-fPIC -shared' + ;; + + osf3* | osf4* | osf5*) + lt_prog_compiler_wl='-Wl,' + # All OSF/1 code is PIC. + lt_prog_compiler_static='-non_shared' + ;; + + rdos*) + lt_prog_compiler_static='-non_shared' + ;; + + solaris*) + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-Bstatic' + case $cc_basename in + f77* | f90* | f95* | sunf77* | sunf90* | sunf95*) + lt_prog_compiler_wl='-Qoption ld ';; + *) + lt_prog_compiler_wl='-Wl,';; + esac + ;; + + sunos4*) + lt_prog_compiler_wl='-Qoption ld ' + lt_prog_compiler_pic='-PIC' + lt_prog_compiler_static='-Bstatic' + ;; + + sysv4 | sysv4.2uw2* | sysv4.3*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-Bstatic' + ;; + + sysv4*MP*) + if test -d /usr/nec ;then + lt_prog_compiler_pic='-Kconform_pic' + lt_prog_compiler_static='-Bstatic' + fi + ;; + + sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_pic='-KPIC' + lt_prog_compiler_static='-Bstatic' + ;; + + unicos*) + lt_prog_compiler_wl='-Wl,' + lt_prog_compiler_can_build_shared=no + ;; + + uts4*) + lt_prog_compiler_pic='-pic' + lt_prog_compiler_static='-Bstatic' + ;; + + *) + lt_prog_compiler_can_build_shared=no + ;; + esac + fi + +case $host_os in + # For platforms which do not support PIC, -DPIC is meaningless: + *djgpp*) + lt_prog_compiler_pic= + ;; + *) + lt_prog_compiler_pic="$lt_prog_compiler_pic -DPIC" + ;; +esac + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $compiler option to produce PIC" >&5 +$as_echo_n "checking for $compiler option to produce PIC... " >&6; } +if ${lt_cv_prog_compiler_pic+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_pic=$lt_prog_compiler_pic +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_pic" >&5 +$as_echo "$lt_cv_prog_compiler_pic" >&6; } +lt_prog_compiler_pic=$lt_cv_prog_compiler_pic + +# +# Check to make sure the PIC flag actually works. +# +if test -n "$lt_prog_compiler_pic"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler PIC flag $lt_prog_compiler_pic works" >&5 +$as_echo_n "checking if $compiler PIC flag $lt_prog_compiler_pic works... " >&6; } +if ${lt_cv_prog_compiler_pic_works+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_pic_works=no + ac_outfile=conftest.$ac_objext + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + lt_compiler_flag="$lt_prog_compiler_pic -DPIC" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + # The option is referenced via a variable to avoid confusing sed. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) + (eval "$lt_compile" 2>conftest.err) + ac_status=$? + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + if (exit $ac_status) && test -s "$ac_outfile"; then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings other than the usual output. + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' >conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then + lt_cv_prog_compiler_pic_works=yes + fi + fi + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_pic_works" >&5 +$as_echo "$lt_cv_prog_compiler_pic_works" >&6; } + +if test x"$lt_cv_prog_compiler_pic_works" = xyes; then + case $lt_prog_compiler_pic in + "" | " "*) ;; + *) lt_prog_compiler_pic=" $lt_prog_compiler_pic" ;; + esac +else + lt_prog_compiler_pic= + lt_prog_compiler_can_build_shared=no +fi + +fi + + + + + + + + + + + +# +# Check to make sure the static flag actually works. +# +wl=$lt_prog_compiler_wl eval lt_tmp_static_flag=\"$lt_prog_compiler_static\" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler static flag $lt_tmp_static_flag works" >&5 +$as_echo_n "checking if $compiler static flag $lt_tmp_static_flag works... " >&6; } +if ${lt_cv_prog_compiler_static_works+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_static_works=no + save_LDFLAGS="$LDFLAGS" + LDFLAGS="$LDFLAGS $lt_tmp_static_flag" + echo "$lt_simple_link_test_code" > conftest.$ac_ext + if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then + # The linker can only warn and ignore the option if not recognized + # So say no if there are warnings + if test -s conftest.err; then + # Append any errors to the config.log. + cat conftest.err 1>&5 + $ECHO "$_lt_linker_boilerplate" | $SED '/^$/d' > conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if diff conftest.exp conftest.er2 >/dev/null; then + lt_cv_prog_compiler_static_works=yes + fi + else + lt_cv_prog_compiler_static_works=yes + fi + fi + $RM -r conftest* + LDFLAGS="$save_LDFLAGS" + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_static_works" >&5 +$as_echo "$lt_cv_prog_compiler_static_works" >&6; } + +if test x"$lt_cv_prog_compiler_static_works" = xyes; then + : +else + lt_prog_compiler_static= +fi + + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5 +$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; } +if ${lt_cv_prog_compiler_c_o+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_c_o=no + $RM -r conftest 2>/dev/null + mkdir conftest + cd conftest + mkdir out + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + lt_compiler_flag="-o out/conftest2.$ac_objext" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) + (eval "$lt_compile" 2>out/conftest.err) + ac_status=$? + cat out/conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + if (exit $ac_status) && test -s out/conftest2.$ac_objext + then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp + $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 + if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then + lt_cv_prog_compiler_c_o=yes + fi + fi + chmod u+w . 2>&5 + $RM conftest* + # SGI C++ compiler will create directory out/ii_files/ for + # template instantiation + test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files + $RM out/* && rmdir out + cd .. + $RM -r conftest + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o" >&5 +$as_echo "$lt_cv_prog_compiler_c_o" >&6; } + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5 +$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; } +if ${lt_cv_prog_compiler_c_o+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_c_o=no + $RM -r conftest 2>/dev/null + mkdir conftest + cd conftest + mkdir out + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + lt_compiler_flag="-o out/conftest2.$ac_objext" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) + (eval "$lt_compile" 2>out/conftest.err) + ac_status=$? + cat out/conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + if (exit $ac_status) && test -s out/conftest2.$ac_objext + then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp + $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 + if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then + lt_cv_prog_compiler_c_o=yes + fi + fi + chmod u+w . 2>&5 + $RM conftest* + # SGI C++ compiler will create directory out/ii_files/ for + # template instantiation + test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files + $RM out/* && rmdir out + cd .. + $RM -r conftest + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o" >&5 +$as_echo "$lt_cv_prog_compiler_c_o" >&6; } + + + + +hard_links="nottested" +if test "$lt_cv_prog_compiler_c_o" = no && test "$need_locks" != no; then + # do not overwrite the value of need_locks provided by the user + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if we can lock with hard links" >&5 +$as_echo_n "checking if we can lock with hard links... " >&6; } + hard_links=yes + $RM conftest* + ln conftest.a conftest.b 2>/dev/null && hard_links=no + touch conftest.a + ln conftest.a conftest.b 2>&5 || hard_links=no + ln conftest.a conftest.b 2>/dev/null && hard_links=no + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $hard_links" >&5 +$as_echo "$hard_links" >&6; } + if test "$hard_links" = no; then + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \`$CC' does not support \`-c -o', so \`make -j' may be unsafe" >&5 +$as_echo "$as_me: WARNING: \`$CC' does not support \`-c -o', so \`make -j' may be unsafe" >&2;} + need_locks=warn + fi +else + need_locks=no +fi + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the $compiler linker ($LD) supports shared libraries" >&5 +$as_echo_n "checking whether the $compiler linker ($LD) supports shared libraries... " >&6; } + + runpath_var= + allow_undefined_flag= + always_export_symbols=no + archive_cmds= + archive_expsym_cmds= + compiler_needs_object=no + enable_shared_with_static_runtimes=no + export_dynamic_flag_spec= + export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' + hardcode_automatic=no + hardcode_direct=no + hardcode_direct_absolute=no + hardcode_libdir_flag_spec= + hardcode_libdir_separator= + hardcode_minus_L=no + hardcode_shlibpath_var=unsupported + inherit_rpath=no + link_all_deplibs=unknown + module_cmds= + module_expsym_cmds= + old_archive_from_new_cmds= + old_archive_from_expsyms_cmds= + thread_safe_flag_spec= + whole_archive_flag_spec= + # include_expsyms should be a list of space-separated symbols to be *always* + # included in the symbol list + include_expsyms= + # exclude_expsyms can be an extended regexp of symbols to exclude + # it will be wrapped by ` (' and `)$', so one must not match beginning or + # end of line. Example: `a|bc|.*d.*' will exclude the symbols `a' and `bc', + # as well as any symbol that contains `d'. + exclude_expsyms='_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*' + # Although _GLOBAL_OFFSET_TABLE_ is a valid symbol C name, most a.out + # platforms (ab)use it in PIC code, but their linkers get confused if + # the symbol is explicitly referenced. Since portable code cannot + # rely on this symbol name, it's probably fine to never include it in + # preloaded symbol tables. + # Exclude shared library initialization/finalization symbols. + extract_expsyms_cmds= + + case $host_os in + cygwin* | mingw* | pw32* | cegcc*) + # FIXME: the MSVC++ port hasn't been tested in a loooong time + # When not using gcc, we currently assume that we are using + # Microsoft Visual C++. + if test "$GCC" != yes; then + with_gnu_ld=no + fi + ;; + interix*) + # we just hope/assume this is gcc and not c89 (= MSVC++) + with_gnu_ld=yes + ;; + openbsd*) + with_gnu_ld=no + ;; + linux* | k*bsd*-gnu | gnu*) + link_all_deplibs=no + ;; + esac + + ld_shlibs=yes + + # On some targets, GNU ld is compatible enough with the native linker + # that we're better off using the native interface for both. + lt_use_gnu_ld_interface=no + if test "$with_gnu_ld" = yes; then + case $host_os in + aix*) + # The AIX port of GNU ld has always aspired to compatibility + # with the native linker. However, as the warning in the GNU ld + # block says, versions before 2.19.5* couldn't really create working + # shared libraries, regardless of the interface used. + case `$LD -v 2>&1` in + *\ \(GNU\ Binutils\)\ 2.19.5*) ;; + *\ \(GNU\ Binutils\)\ 2.[2-9]*) ;; + *\ \(GNU\ Binutils\)\ [3-9]*) ;; + *) + lt_use_gnu_ld_interface=yes + ;; + esac + ;; + *) + lt_use_gnu_ld_interface=yes + ;; + esac + fi + + if test "$lt_use_gnu_ld_interface" = yes; then + # If archive_cmds runs LD, not CC, wlarc should be empty + wlarc='${wl}' + + # Set some defaults for GNU ld with shared library support. These + # are reset later if shared libraries are not supported. Putting them + # here allows them to be overridden if necessary. + runpath_var=LD_RUN_PATH + hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir' + export_dynamic_flag_spec='${wl}--export-dynamic' + # ancient GNU ld didn't support --whole-archive et. al. + if $LD --help 2>&1 | $GREP 'no-whole-archive' > /dev/null; then + whole_archive_flag_spec="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive' + else + whole_archive_flag_spec= + fi + supports_anon_versioning=no + case `$LD -v 2>&1` in + *GNU\ gold*) supports_anon_versioning=yes ;; + *\ [01].* | *\ 2.[0-9].* | *\ 2.10.*) ;; # catch versions < 2.11 + *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ... + *\ 2.11.92.0.12\ *) supports_anon_versioning=yes ;; # Mandrake 8.2 ... + *\ 2.11.*) ;; # other 2.11 versions + *) supports_anon_versioning=yes ;; + esac + + # See if GNU ld supports shared libraries. + case $host_os in + aix[3-9]*) + # On AIX/PPC, the GNU linker is very broken + if test "$host_cpu" != ia64; then + ld_shlibs=no + cat <<_LT_EOF 1>&2 + +*** Warning: the GNU linker, at least up to release 2.19, is reported +*** to be unable to reliably create shared libraries on AIX. +*** Therefore, libtool is disabling shared libraries support. If you +*** really care for shared libraries, you may want to install binutils +*** 2.20 or above, or modify your PATH so that a non-GNU linker is found. +*** You will then need to restart the configuration process. + +_LT_EOF + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds='' + ;; + m68k) + archive_cmds='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' + hardcode_libdir_flag_spec='-L$libdir' + hardcode_minus_L=yes + ;; + esac + ;; + + beos*) + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + allow_undefined_flag=unsupported + # Joseph Beckenbach says some releases of gcc + # support --undefined. This deserves some investigation. FIXME + archive_cmds='$CC -nostart $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + else + ld_shlibs=no + fi + ;; + + cygwin* | mingw* | pw32* | cegcc*) + # _LT_TAGVAR(hardcode_libdir_flag_spec, ) is actually meaningless, + # as there is no search path for DLLs. + hardcode_libdir_flag_spec='-L$libdir' + export_dynamic_flag_spec='${wl}--export-all-symbols' + allow_undefined_flag=unsupported + always_export_symbols=no + enable_shared_with_static_runtimes=yes + export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[BCDGRS][ ]/s/.*[ ]\([^ ]*\)/\1 DATA/;s/^.*[ ]__nm__\([^ ]*\)[ ][^ ]*/\1 DATA/;/^I[ ]/d;/^[AITW][ ]/s/.* //'\'' | sort | uniq > $export_symbols' + exclude_expsyms='[_]+GLOBAL_OFFSET_TABLE_|[_]+GLOBAL__[FID]_.*|[_]+head_[A-Za-z0-9_]+_dll|[A-Za-z0-9_]+_dll_iname' + + if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then + archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + # If the export-symbols file already is a .def file (1st line + # is EXPORTS), use it as is; otherwise, prepend... + archive_expsym_cmds='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + cp $export_symbols $output_objdir/$soname.def; + else + echo EXPORTS > $output_objdir/$soname.def; + cat $export_symbols >> $output_objdir/$soname.def; + fi~ + $CC -shared $output_objdir/$soname.def $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + else + ld_shlibs=no + fi + ;; + + haiku*) + archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + link_all_deplibs=yes + ;; + + interix[3-9]*) + hardcode_direct=no + hardcode_shlibpath_var=no + hardcode_libdir_flag_spec='${wl}-rpath,$libdir' + export_dynamic_flag_spec='${wl}-E' + # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc. + # Instead, shared libraries are loaded at an image base (0x10000000 by + # default) and relocated if they conflict, which is a slow very memory + # consuming and fragmenting process. To avoid this, we pick a random, + # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link + # time. Moving up from 0x10000000 also allows more sbrk(2) space. + archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + archive_expsym_cmds='sed "s,^,_," $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--retain-symbols-file,$output_objdir/$soname.expsym ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + ;; + + gnu* | linux* | tpf* | k*bsd*-gnu | kopensolaris*-gnu) + tmp_diet=no + if test "$host_os" = linux-dietlibc; then + case $cc_basename in + diet\ *) tmp_diet=yes;; # linux-dietlibc with static linking (!diet-dyn) + esac + fi + if $LD --help 2>&1 | $EGREP ': supported targets:.* elf' > /dev/null \ + && test "$tmp_diet" = no + then + tmp_addflag=' $pic_flag' + tmp_sharedflag='-shared' + case $cc_basename,$host_cpu in + pgcc*) # Portland Group C compiler + whole_archive_flag_spec='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + tmp_addflag=' $pic_flag' + ;; + pgf77* | pgf90* | pgf95* | pgfortran*) + # Portland Group f77 and f90 compilers + whole_archive_flag_spec='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + tmp_addflag=' $pic_flag -Mnomain' ;; + ecc*,ia64* | icc*,ia64*) # Intel C compiler on ia64 + tmp_addflag=' -i_dynamic' ;; + efc*,ia64* | ifort*,ia64*) # Intel Fortran compiler on ia64 + tmp_addflag=' -i_dynamic -nofor_main' ;; + ifc* | ifort*) # Intel Fortran compiler + tmp_addflag=' -nofor_main' ;; + lf95*) # Lahey Fortran 8.1 + whole_archive_flag_spec= + tmp_sharedflag='--shared' ;; + xl[cC]* | bgxl[cC]* | mpixl[cC]*) # IBM XL C 8.0 on PPC (deal with xlf below) + tmp_sharedflag='-qmkshrobj' + tmp_addflag= ;; + nvcc*) # Cuda Compiler Driver 2.2 + whole_archive_flag_spec='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + compiler_needs_object=yes + ;; + esac + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) # Sun C 5.9 + whole_archive_flag_spec='${wl}--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + compiler_needs_object=yes + tmp_sharedflag='-G' ;; + *Sun\ F*) # Sun Fortran 8.3 + tmp_sharedflag='-G' ;; + esac + archive_cmds='$CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + + if test "x$supports_anon_versioning" = xyes; then + archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~ + cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + echo "local: *; };" >> $output_objdir/$libname.ver~ + $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-version-script ${wl}$output_objdir/$libname.ver -o $lib' + fi + + case $cc_basename in + xlf* | bgf* | bgxlf* | mpixlf*) + # IBM XL Fortran 10.1 on PPC cannot create shared libs itself + whole_archive_flag_spec='--whole-archive$convenience --no-whole-archive' + hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir' + archive_cmds='$LD -shared $libobjs $deplibs $linker_flags -soname $soname -o $lib' + if test "x$supports_anon_versioning" = xyes; then + archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~ + cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + echo "local: *; };" >> $output_objdir/$libname.ver~ + $LD -shared $libobjs $deplibs $linker_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib' + fi + ;; + esac + else + ld_shlibs=no + fi + ;; + + netbsd* | netbsdelf*-gnu) + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + archive_cmds='$LD -Bshareable $libobjs $deplibs $linker_flags -o $lib' + wlarc= + else + archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + fi + ;; + + solaris*) + if $LD -v 2>&1 | $GREP 'BFD 2\.8' > /dev/null; then + ld_shlibs=no + cat <<_LT_EOF 1>&2 + +*** Warning: The releases 2.8.* of the GNU linker cannot reliably +*** create shared libraries on Solaris systems. Therefore, libtool +*** is disabling shared libraries support. We urge you to upgrade GNU +*** binutils to release 2.9.1 or newer. Another option is to modify +*** your PATH or compiler configuration so that the native linker is +*** used, and then restart. + +_LT_EOF + elif $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + else + ld_shlibs=no + fi + ;; + + sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX*) + case `$LD -v 2>&1` in + *\ [01].* | *\ 2.[0-9].* | *\ 2.1[0-5].*) + ld_shlibs=no + cat <<_LT_EOF 1>&2 + +*** Warning: Releases of the GNU linker prior to 2.16.91.0.3 can not +*** reliably create shared libraries on SCO systems. Therefore, libtool +*** is disabling shared libraries support. We urge you to upgrade GNU +*** binutils to release 2.16.91.0.3 or newer. Another option is to modify +*** your PATH or compiler configuration so that the native linker is +*** used, and then restart. + +_LT_EOF + ;; + *) + # For security reasons, it is highly recommended that you always + # use absolute paths for naming shared libraries, and exclude the + # DT_RUNPATH tag from executables and libraries. But doing so + # requires that you compile everything twice, which is a pain. + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir' + archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + else + ld_shlibs=no + fi + ;; + esac + ;; + + sunos4*) + archive_cmds='$LD -assert pure-text -Bshareable -o $lib $libobjs $deplibs $linker_flags' + wlarc= + hardcode_direct=yes + hardcode_shlibpath_var=no + ;; + + *) + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + else + ld_shlibs=no + fi + ;; + esac + + if test "$ld_shlibs" = no; then + runpath_var= + hardcode_libdir_flag_spec= + export_dynamic_flag_spec= + whole_archive_flag_spec= + fi + else + # PORTME fill in a description of your system's linker (not GNU ld) + case $host_os in + aix3*) + allow_undefined_flag=unsupported + always_export_symbols=yes + archive_expsym_cmds='$LD -o $output_objdir/$soname $libobjs $deplibs $linker_flags -bE:$export_symbols -T512 -H512 -bM:SRE~$AR $AR_FLAGS $lib $output_objdir/$soname' + # Note: this linker hardcodes the directories in LIBPATH if there + # are no directories specified by -L. + hardcode_minus_L=yes + if test "$GCC" = yes && test -z "$lt_prog_compiler_static"; then + # Neither direct hardcoding nor static linking is supported with a + # broken collect2. + hardcode_direct=unsupported + fi + ;; + + aix[4-9]*) + if test "$host_cpu" = ia64; then + # On IA64, the linker does run time linking by default, so we don't + # have to do anything special. + aix_use_runtimelinking=no + exp_sym_flag='-Bexport' + no_entry_flag="" + else + # If we're using GNU nm, then we don't want the "-C" option. + # -C means demangle to AIX nm, but means don't demangle with GNU nm + # Also, AIX nm treats weak defined symbols like other global + # defined symbols, whereas GNU nm marks them as "W". + if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then + export_symbols_cmds='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && (substr(\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + else + export_symbols_cmds='$NM -BCpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && (substr(\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + fi + aix_use_runtimelinking=no + + # Test if we are trying to use run time linking or normal + # AIX style linking. If -brtl is somewhere in LDFLAGS, we + # need to do runtime linking. + case $host_os in aix4.[23]|aix4.[23].*|aix[5-9]*) + for ld_flag in $LDFLAGS; do + if (test $ld_flag = "-brtl" || test $ld_flag = "-Wl,-brtl"); then + aix_use_runtimelinking=yes + break + fi + done + ;; + esac + + exp_sym_flag='-bexport' + no_entry_flag='-bnoentry' + fi + + # When large executables or shared objects are built, AIX ld can + # have problems creating the table of contents. If linking a library + # or program results in "error TOC overflow" add -mminimal-toc to + # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not + # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS. + + archive_cmds='' + hardcode_direct=yes + hardcode_direct_absolute=yes + hardcode_libdir_separator=':' + link_all_deplibs=yes + file_list_spec='${wl}-f,' + + if test "$GCC" = yes; then + case $host_os in aix4.[012]|aix4.[012].*) + # We only want to do this on AIX 4.2 and lower, the check + # below for broken collect2 doesn't work under 4.3+ + collect2name=`${CC} -print-prog-name=collect2` + if test -f "$collect2name" && + strings "$collect2name" | $GREP resolve_lib_name >/dev/null + then + # We have reworked collect2 + : + else + # We have old collect2 + hardcode_direct=unsupported + # It fails to find uninstalled libraries when the uninstalled + # path is not listed in the libpath. Setting hardcode_minus_L + # to unsupported forces relinking + hardcode_minus_L=yes + hardcode_libdir_flag_spec='-L$libdir' + hardcode_libdir_separator= + fi + ;; + esac + shared_flag='-shared' + if test "$aix_use_runtimelinking" = yes; then + shared_flag="$shared_flag "'${wl}-G' + fi + link_all_deplibs=no + else + # not using gcc + if test "$host_cpu" = ia64; then + # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release + # chokes on -Wl,-G. The following line is correct: + shared_flag='-G' + else + if test "$aix_use_runtimelinking" = yes; then + shared_flag='${wl}-G' + else + shared_flag='${wl}-bM:SRE' + fi + fi + fi + + export_dynamic_flag_spec='${wl}-bexpall' + # It seems that -bexpall does not export symbols beginning with + # underscore (_), so it is better to generate a list of symbols to export. + always_export_symbols=yes + if test "$aix_use_runtimelinking" = yes; then + # Warning - without using the other runtime loading flags (-brtl), + # -berok will link without error, but may produce a broken library. + allow_undefined_flag='-berok' + # Determine the default libpath from the value encoded in an + # empty executable. + if test "${lt_cv_aix_libpath+set}" = set; then + aix_libpath=$lt_cv_aix_libpath +else + if ${lt_cv_aix_libpath_+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + + lt_aix_libpath_sed=' + /Import File Strings/,/^$/ { + /^0/ { + s/^0 *\([^ ]*\) *$/\1/ + p + } + }' + lt_cv_aix_libpath_=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + # Check for a 64-bit object if we didn't find anything. + if test -z "$lt_cv_aix_libpath_"; then + lt_cv_aix_libpath_=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + fi +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + if test -z "$lt_cv_aix_libpath_"; then + lt_cv_aix_libpath_="/usr/lib:/lib" + fi + +fi + + aix_libpath=$lt_cv_aix_libpath_ +fi + + hardcode_libdir_flag_spec='${wl}-blibpath:$libdir:'"$aix_libpath" + archive_expsym_cmds='$CC -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags `if test "x${allow_undefined_flag}" != "x"; then func_echo_all "${wl}${allow_undefined_flag}"; else :; fi` '"\${wl}$exp_sym_flag:\$export_symbols $shared_flag" + else + if test "$host_cpu" = ia64; then + hardcode_libdir_flag_spec='${wl}-R $libdir:/usr/lib:/lib' + allow_undefined_flag="-z nodefs" + archive_expsym_cmds="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags ${wl}${allow_undefined_flag} '"\${wl}$exp_sym_flag:\$export_symbols" + else + # Determine the default libpath from the value encoded in an + # empty executable. + if test "${lt_cv_aix_libpath+set}" = set; then + aix_libpath=$lt_cv_aix_libpath +else + if ${lt_cv_aix_libpath_+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + + lt_aix_libpath_sed=' + /Import File Strings/,/^$/ { + /^0/ { + s/^0 *\([^ ]*\) *$/\1/ + p + } + }' + lt_cv_aix_libpath_=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + # Check for a 64-bit object if we didn't find anything. + if test -z "$lt_cv_aix_libpath_"; then + lt_cv_aix_libpath_=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + fi +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + if test -z "$lt_cv_aix_libpath_"; then + lt_cv_aix_libpath_="/usr/lib:/lib" + fi + +fi + + aix_libpath=$lt_cv_aix_libpath_ +fi + + hardcode_libdir_flag_spec='${wl}-blibpath:$libdir:'"$aix_libpath" + # Warning - without using the other run time loading flags, + # -berok will link without error, but may produce a broken library. + no_undefined_flag=' ${wl}-bernotok' + allow_undefined_flag=' ${wl}-berok' + if test "$with_gnu_ld" = yes; then + # We only use this code for GNU lds that support --whole-archive. + whole_archive_flag_spec='${wl}--whole-archive$convenience ${wl}--no-whole-archive' + else + # Exported symbols can be pulled into shared objects from archives + whole_archive_flag_spec='$convenience' + fi + archive_cmds_need_lc=yes + # This is similar to how AIX traditionally builds its shared libraries. + archive_expsym_cmds="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs ${wl}-bnoentry $compiler_flags ${wl}-bE:$export_symbols${allow_undefined_flag}~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$soname' + fi + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds='' + ;; + m68k) + archive_cmds='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' + hardcode_libdir_flag_spec='-L$libdir' + hardcode_minus_L=yes + ;; + esac + ;; + + bsdi[45]*) + export_dynamic_flag_spec=-rdynamic + ;; + + cygwin* | mingw* | pw32* | cegcc*) + # When not using gcc, we currently assume that we are using + # Microsoft Visual C++. + # hardcode_libdir_flag_spec is actually meaningless, as there is + # no search path for DLLs. + case $cc_basename in + cl*) + # Native MSVC + hardcode_libdir_flag_spec=' ' + allow_undefined_flag=unsupported + always_export_symbols=yes + file_list_spec='@' + # Tell ltmain to make .lib files, not .a files. + libext=lib + # Tell ltmain to make .dll files, not .so files. + shrext_cmds=".dll" + # FIXME: Setting linknames here is a bad hack. + archive_cmds='$CC -o $output_objdir/$soname $libobjs $compiler_flags $deplibs -Wl,-dll~linknames=' + archive_expsym_cmds='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + sed -n -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' -e '1\\\!p' < $export_symbols > $output_objdir/$soname.exp; + else + sed -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' < $export_symbols > $output_objdir/$soname.exp; + fi~ + $CC -o $tool_output_objdir$soname $libobjs $compiler_flags $deplibs "@$tool_output_objdir$soname.exp" -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~ + linknames=' + # The linker will not automatically build a static lib if we build a DLL. + # _LT_TAGVAR(old_archive_from_new_cmds, )='true' + enable_shared_with_static_runtimes=yes + exclude_expsyms='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' + export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[BCDGRS][ ]/s/.*[ ]\([^ ]*\)/\1,DATA/'\'' | $SED -e '\''/^[AITW][ ]/s/.*[ ]//'\'' | sort | uniq > $export_symbols' + # Don't use ranlib + old_postinstall_cmds='chmod 644 $oldlib' + postlink_cmds='lt_outputfile="@OUTPUT@"~ + lt_tool_outputfile="@TOOL_OUTPUT@"~ + case $lt_outputfile in + *.exe|*.EXE) ;; + *) + lt_outputfile="$lt_outputfile.exe" + lt_tool_outputfile="$lt_tool_outputfile.exe" + ;; + esac~ + if test "$MANIFEST_TOOL" != ":" && test -f "$lt_outputfile.manifest"; then + $MANIFEST_TOOL -manifest "$lt_tool_outputfile.manifest" -outputresource:"$lt_tool_outputfile" || exit 1; + $RM "$lt_outputfile.manifest"; + fi' + ;; + *) + # Assume MSVC wrapper + hardcode_libdir_flag_spec=' ' + allow_undefined_flag=unsupported + # Tell ltmain to make .lib files, not .a files. + libext=lib + # Tell ltmain to make .dll files, not .so files. + shrext_cmds=".dll" + # FIXME: Setting linknames here is a bad hack. + archive_cmds='$CC -o $lib $libobjs $compiler_flags `func_echo_all "$deplibs" | $SED '\''s/ -lc$//'\''` -link -dll~linknames=' + # The linker will automatically build a .lib file if we build a DLL. + old_archive_from_new_cmds='true' + # FIXME: Should let the user specify the lib program. + old_archive_cmds='lib -OUT:$oldlib$oldobjs$old_deplibs' + enable_shared_with_static_runtimes=yes + ;; + esac + ;; + + darwin* | rhapsody*) + + + archive_cmds_need_lc=no + hardcode_direct=no + hardcode_automatic=yes + hardcode_shlibpath_var=unsupported + if test "$lt_cv_ld_force_load" = "yes"; then + whole_archive_flag_spec='`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience ${wl}-force_load,$conv\"; done; func_echo_all \"$new_convenience\"`' + + else + whole_archive_flag_spec='' + fi + link_all_deplibs=yes + allow_undefined_flag="$_lt_dar_allow_undefined" + case $cc_basename in + ifort*) _lt_dar_can_shared=yes ;; + *) _lt_dar_can_shared=$GCC ;; + esac + if test "$_lt_dar_can_shared" = "yes"; then + output_verbose_link_cmd=func_echo_all + archive_cmds="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod${_lt_dsymutil}" + module_cmds="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dsymutil}" + archive_expsym_cmds="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring ${_lt_dar_single_mod}${_lt_dar_export_syms}${_lt_dsymutil}" + module_expsym_cmds="sed -e 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dar_export_syms}${_lt_dsymutil}" + + else + ld_shlibs=no + fi + + ;; + + dgux*) + archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + hardcode_libdir_flag_spec='-L$libdir' + hardcode_shlibpath_var=no + ;; + + # FreeBSD 2.2.[012] allows us to include c++rt0.o to get C++ constructor + # support. Future versions do this automatically, but an explicit c++rt0.o + # does not break anything, and helps significantly (at the cost of a little + # extra space). + freebsd2.2*) + archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags /usr/lib/c++rt0.o' + hardcode_libdir_flag_spec='-R$libdir' + hardcode_direct=yes + hardcode_shlibpath_var=no + ;; + + # Unfortunately, older versions of FreeBSD 2 do not have this feature. + freebsd2.*) + archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' + hardcode_direct=yes + hardcode_minus_L=yes + hardcode_shlibpath_var=no + ;; + + # FreeBSD 3 and greater uses gcc -shared to do shared libraries. + freebsd* | dragonfly*) + archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + hardcode_libdir_flag_spec='-R$libdir' + hardcode_direct=yes + hardcode_shlibpath_var=no + ;; + + hpux9*) + if test "$GCC" = yes; then + archive_cmds='$RM $output_objdir/$soname~$CC -shared $pic_flag ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $libobjs $deplibs $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + else + archive_cmds='$RM $output_objdir/$soname~$LD -b +b $install_libdir -o $output_objdir/$soname $libobjs $deplibs $linker_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + fi + hardcode_libdir_flag_spec='${wl}+b ${wl}$libdir' + hardcode_libdir_separator=: + hardcode_direct=yes + + # hardcode_minus_L: Not really in the search PATH, + # but as the default location of the library. + hardcode_minus_L=yes + export_dynamic_flag_spec='${wl}-E' + ;; + + hpux10*) + if test "$GCC" = yes && test "$with_gnu_ld" = no; then + archive_cmds='$CC -shared $pic_flag ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags' + else + archive_cmds='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags' + fi + if test "$with_gnu_ld" = no; then + hardcode_libdir_flag_spec='${wl}+b ${wl}$libdir' + hardcode_libdir_separator=: + hardcode_direct=yes + hardcode_direct_absolute=yes + export_dynamic_flag_spec='${wl}-E' + # hardcode_minus_L: Not really in the search PATH, + # but as the default location of the library. + hardcode_minus_L=yes + fi + ;; + + hpux11*) + if test "$GCC" = yes && test "$with_gnu_ld" = no; then + case $host_cpu in + hppa*64*) + archive_cmds='$CC -shared ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + ia64*) + archive_cmds='$CC -shared $pic_flag ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' + ;; + *) + archive_cmds='$CC -shared $pic_flag ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags' + ;; + esac + else + case $host_cpu in + hppa*64*) + archive_cmds='$CC -b ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + ia64*) + archive_cmds='$CC -b ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' + ;; + *) + + # Older versions of the 11.00 compiler do not understand -b yet + # (HP92453-01 A.11.01.20 doesn't, HP92453-01 B.11.X.35175-35176.GP does) + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $CC understands -b" >&5 +$as_echo_n "checking if $CC understands -b... " >&6; } +if ${lt_cv_prog_compiler__b+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler__b=no + save_LDFLAGS="$LDFLAGS" + LDFLAGS="$LDFLAGS -b" + echo "$lt_simple_link_test_code" > conftest.$ac_ext + if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then + # The linker can only warn and ignore the option if not recognized + # So say no if there are warnings + if test -s conftest.err; then + # Append any errors to the config.log. + cat conftest.err 1>&5 + $ECHO "$_lt_linker_boilerplate" | $SED '/^$/d' > conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if diff conftest.exp conftest.er2 >/dev/null; then + lt_cv_prog_compiler__b=yes + fi + else + lt_cv_prog_compiler__b=yes + fi + fi + $RM -r conftest* + LDFLAGS="$save_LDFLAGS" + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler__b" >&5 +$as_echo "$lt_cv_prog_compiler__b" >&6; } + +if test x"$lt_cv_prog_compiler__b" = xyes; then + archive_cmds='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags' +else + archive_cmds='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags' +fi + + ;; + esac + fi + if test "$with_gnu_ld" = no; then + hardcode_libdir_flag_spec='${wl}+b ${wl}$libdir' + hardcode_libdir_separator=: + + case $host_cpu in + hppa*64*|ia64*) + hardcode_direct=no + hardcode_shlibpath_var=no + ;; + *) + hardcode_direct=yes + hardcode_direct_absolute=yes + export_dynamic_flag_spec='${wl}-E' + + # hardcode_minus_L: Not really in the search PATH, + # but as the default location of the library. + hardcode_minus_L=yes + ;; + esac + fi + ;; + + irix5* | irix6* | nonstopux*) + if test "$GCC" = yes; then + archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + # Try to use the -exported_symbol ld option, if it does not + # work, assume that -exports_file does not work either and + # implicitly export all symbols. + # This should be the same for all languages, so no per-tag cache variable. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the $host_os linker accepts -exported_symbol" >&5 +$as_echo_n "checking whether the $host_os linker accepts -exported_symbol... " >&6; } +if ${lt_cv_irix_exported_symbol+:} false; then : + $as_echo_n "(cached) " >&6 +else + save_LDFLAGS="$LDFLAGS" + LDFLAGS="$LDFLAGS -shared ${wl}-exported_symbol ${wl}foo ${wl}-update_registry ${wl}/dev/null" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int foo (void) { return 0; } +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + lt_cv_irix_exported_symbol=yes +else + lt_cv_irix_exported_symbol=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LDFLAGS="$save_LDFLAGS" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_irix_exported_symbol" >&5 +$as_echo "$lt_cv_irix_exported_symbol" >&6; } + if test "$lt_cv_irix_exported_symbol" = yes; then + archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations ${wl}-exports_file ${wl}$export_symbols -o $lib' + fi + else + archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -exports_file $export_symbols -o $lib' + fi + archive_cmds_need_lc='no' + hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir' + hardcode_libdir_separator=: + inherit_rpath=yes + link_all_deplibs=yes + ;; + + netbsd* | netbsdelf*-gnu) + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' # a.out + else + archive_cmds='$LD -shared -o $lib $libobjs $deplibs $linker_flags' # ELF + fi + hardcode_libdir_flag_spec='-R$libdir' + hardcode_direct=yes + hardcode_shlibpath_var=no + ;; + + newsos6) + archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + hardcode_direct=yes + hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir' + hardcode_libdir_separator=: + hardcode_shlibpath_var=no + ;; + + *nto* | *qnx*) + ;; + + openbsd*) + if test -f /usr/libexec/ld.so; then + hardcode_direct=yes + hardcode_shlibpath_var=no + hardcode_direct_absolute=yes + if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags ${wl}-retain-symbols-file,$export_symbols' + hardcode_libdir_flag_spec='${wl}-rpath,$libdir' + export_dynamic_flag_spec='${wl}-E' + else + case $host_os in + openbsd[01].* | openbsd2.[0-7] | openbsd2.[0-7].*) + archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' + hardcode_libdir_flag_spec='-R$libdir' + ;; + *) + archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + hardcode_libdir_flag_spec='${wl}-rpath,$libdir' + ;; + esac + fi + else + ld_shlibs=no + fi + ;; + + os2*) + hardcode_libdir_flag_spec='-L$libdir' + hardcode_minus_L=yes + allow_undefined_flag=unsupported + archive_cmds='$ECHO "LIBRARY $libname INITINSTANCE" > $output_objdir/$libname.def~$ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~echo DATA >> $output_objdir/$libname.def~echo " SINGLE NONSHARED" >> $output_objdir/$libname.def~echo EXPORTS >> $output_objdir/$libname.def~emxexp $libobjs >> $output_objdir/$libname.def~$CC -Zdll -Zcrtdll -o $lib $libobjs $deplibs $compiler_flags $output_objdir/$libname.def' + old_archive_from_new_cmds='emximp -o $output_objdir/$libname.a $output_objdir/$libname.def' + ;; + + osf3*) + if test "$GCC" = yes; then + allow_undefined_flag=' ${wl}-expect_unresolved ${wl}\*' + archive_cmds='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + else + allow_undefined_flag=' -expect_unresolved \*' + archive_cmds='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + fi + archive_cmds_need_lc='no' + hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir' + hardcode_libdir_separator=: + ;; + + osf4* | osf5*) # as osf3* with the addition of -msym flag + if test "$GCC" = yes; then + allow_undefined_flag=' ${wl}-expect_unresolved ${wl}\*' + archive_cmds='$CC -shared${allow_undefined_flag} $pic_flag $libobjs $deplibs $compiler_flags ${wl}-msym ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir' + else + allow_undefined_flag=' -expect_unresolved \*' + archive_cmds='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -msym -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + archive_expsym_cmds='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done; printf "%s\\n" "-hidden">> $lib.exp~ + $CC -shared${allow_undefined_flag} ${wl}-input ${wl}$lib.exp $compiler_flags $libobjs $deplibs -soname $soname `test -n "$verstring" && $ECHO "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib~$RM $lib.exp' + + # Both c and cxx compiler support -rpath directly + hardcode_libdir_flag_spec='-rpath $libdir' + fi + archive_cmds_need_lc='no' + hardcode_libdir_separator=: + ;; + + solaris*) + no_undefined_flag=' -z defs' + if test "$GCC" = yes; then + wlarc='${wl}' + archive_cmds='$CC -shared $pic_flag ${wl}-z ${wl}text ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -shared $pic_flag ${wl}-z ${wl}text ${wl}-M ${wl}$lib.exp ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' + else + case `$CC -V 2>&1` in + *"Compilers 5.0"*) + wlarc='' + archive_cmds='$LD -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $linker_flags' + archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $LD -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $linker_flags~$RM $lib.exp' + ;; + *) + wlarc='${wl}' + archive_cmds='$CC -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' + ;; + esac + fi + hardcode_libdir_flag_spec='-R$libdir' + hardcode_shlibpath_var=no + case $host_os in + solaris2.[0-5] | solaris2.[0-5].*) ;; + *) + # The compiler driver will combine and reorder linker options, + # but understands `-z linker_flag'. GCC discards it without `$wl', + # but is careful enough not to reorder. + # Supported since Solaris 2.6 (maybe 2.5.1?) + if test "$GCC" = yes; then + whole_archive_flag_spec='${wl}-z ${wl}allextract$convenience ${wl}-z ${wl}defaultextract' + else + whole_archive_flag_spec='-z allextract$convenience -z defaultextract' + fi + ;; + esac + link_all_deplibs=yes + ;; + + sunos4*) + if test "x$host_vendor" = xsequent; then + # Use $CC to link under sequent, because it throws in some extra .o + # files that make .init and .fini sections work. + archive_cmds='$CC -G ${wl}-h $soname -o $lib $libobjs $deplibs $compiler_flags' + else + archive_cmds='$LD -assert pure-text -Bstatic -o $lib $libobjs $deplibs $linker_flags' + fi + hardcode_libdir_flag_spec='-L$libdir' + hardcode_direct=yes + hardcode_minus_L=yes + hardcode_shlibpath_var=no + ;; + + sysv4) + case $host_vendor in + sni) + archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + hardcode_direct=yes # is this really true??? + ;; + siemens) + ## LD is ld it makes a PLAMLIB + ## CC just makes a GrossModule. + archive_cmds='$LD -G -o $lib $libobjs $deplibs $linker_flags' + reload_cmds='$CC -r -o $output$reload_objs' + hardcode_direct=no + ;; + motorola) + archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + hardcode_direct=no #Motorola manual says yes, but my tests say they lie + ;; + esac + runpath_var='LD_RUN_PATH' + hardcode_shlibpath_var=no + ;; + + sysv4.3*) + archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + hardcode_shlibpath_var=no + export_dynamic_flag_spec='-Bexport' + ;; + + sysv4*MP*) + if test -d /usr/nec; then + archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + hardcode_shlibpath_var=no + runpath_var=LD_RUN_PATH + hardcode_runpath_var=yes + ld_shlibs=yes + fi + ;; + + sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[01].[10]* | unixware7* | sco3.2v5.0.[024]*) + no_undefined_flag='${wl}-z,text' + archive_cmds_need_lc=no + hardcode_shlibpath_var=no + runpath_var='LD_RUN_PATH' + + if test "$GCC" = yes; then + archive_cmds='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + else + archive_cmds='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + fi + ;; + + sysv5* | sco3.2v5* | sco5v6*) + # Note: We can NOT use -z defs as we might desire, because we do not + # link with -lc, and that would cause any symbols used from libc to + # always be unresolved, which means just about no library would + # ever link correctly. If we're not using GNU ld we use -z text + # though, which does catch some bad symbols but isn't as heavy-handed + # as -z defs. + no_undefined_flag='${wl}-z,text' + allow_undefined_flag='${wl}-z,nodefs' + archive_cmds_need_lc=no + hardcode_shlibpath_var=no + hardcode_libdir_flag_spec='${wl}-R,$libdir' + hardcode_libdir_separator=':' + link_all_deplibs=yes + export_dynamic_flag_spec='${wl}-Bexport' + runpath_var='LD_RUN_PATH' + + if test "$GCC" = yes; then + archive_cmds='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + else + archive_cmds='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + fi + ;; + + uts4*) + archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + hardcode_libdir_flag_spec='-L$libdir' + hardcode_shlibpath_var=no + ;; + + *) + ld_shlibs=no + ;; + esac + + if test x$host_vendor = xsni; then + case $host in + sysv4 | sysv4.2uw2* | sysv4.3* | sysv5*) + export_dynamic_flag_spec='${wl}-Blargedynsym' + ;; + esac + fi + fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ld_shlibs" >&5 +$as_echo "$ld_shlibs" >&6; } +test "$ld_shlibs" = no && can_build_shared=no + +with_gnu_ld=$with_gnu_ld + + + + + + + + + + + + + + + +# +# Do we need to explicitly link libc? +# +case "x$archive_cmds_need_lc" in +x|xyes) + # Assume -lc should be added + archive_cmds_need_lc=yes + + if test "$enable_shared" = yes && test "$GCC" = yes; then + case $archive_cmds in + *'~'*) + # FIXME: we may have to deal with multi-command sequences. + ;; + '$CC '*) + # Test whether the compiler implicitly links with -lc since on some + # systems, -lgcc has to come before -lc. If gcc already passes -lc + # to ld, don't add -lc before -lgcc. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -lc should be explicitly linked in" >&5 +$as_echo_n "checking whether -lc should be explicitly linked in... " >&6; } +if ${lt_cv_archive_cmds_need_lc+:} false; then : + $as_echo_n "(cached) " >&6 +else + $RM conftest* + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } 2>conftest.err; then + soname=conftest + lib=conftest + libobjs=conftest.$ac_objext + deplibs= + wl=$lt_prog_compiler_wl + pic_flag=$lt_prog_compiler_pic + compiler_flags=-v + linker_flags=-v + verstring= + output_objdir=. + libname=conftest + lt_save_allow_undefined_flag=$allow_undefined_flag + allow_undefined_flag= + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$archive_cmds 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1\""; } >&5 + (eval $archive_cmds 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + then + lt_cv_archive_cmds_need_lc=no + else + lt_cv_archive_cmds_need_lc=yes + fi + allow_undefined_flag=$lt_save_allow_undefined_flag + else + cat conftest.err 1>&5 + fi + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_archive_cmds_need_lc" >&5 +$as_echo "$lt_cv_archive_cmds_need_lc" >&6; } + archive_cmds_need_lc=$lt_cv_archive_cmds_need_lc + ;; + esac + fi + ;; +esac + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking dynamic linker characteristics" >&5 +$as_echo_n "checking dynamic linker characteristics... " >&6; } + +if test "$GCC" = yes; then + case $host_os in + darwin*) lt_awk_arg="/^libraries:/,/LR/" ;; + *) lt_awk_arg="/^libraries:/" ;; + esac + case $host_os in + mingw* | cegcc*) lt_sed_strip_eq="s,=\([A-Za-z]:\),\1,g" ;; + *) lt_sed_strip_eq="s,=/,/,g" ;; + esac + lt_search_path_spec=`$CC -print-search-dirs | awk $lt_awk_arg | $SED -e "s/^libraries://" -e $lt_sed_strip_eq` + case $lt_search_path_spec in + *\;*) + # if the path contains ";" then we assume it to be the separator + # otherwise default to the standard path separator (i.e. ":") - it is + # assumed that no part of a normal pathname contains ";" but that should + # okay in the real world where ";" in dirpaths is itself problematic. + lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED 's/;/ /g'` + ;; + *) + lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED "s/$PATH_SEPARATOR/ /g"` + ;; + esac + # Ok, now we have the path, separated by spaces, we can step through it + # and add multilib dir if necessary. + lt_tmp_lt_search_path_spec= + lt_multi_os_dir=`$CC $CPPFLAGS $CFLAGS $LDFLAGS -print-multi-os-directory 2>/dev/null` + for lt_sys_path in $lt_search_path_spec; do + if test -d "$lt_sys_path/$lt_multi_os_dir"; then + lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path/$lt_multi_os_dir" + else + test -d "$lt_sys_path" && \ + lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path" + fi + done + lt_search_path_spec=`$ECHO "$lt_tmp_lt_search_path_spec" | awk ' +BEGIN {RS=" "; FS="/|\n";} { + lt_foo=""; + lt_count=0; + for (lt_i = NF; lt_i > 0; lt_i--) { + if ($lt_i != "" && $lt_i != ".") { + if ($lt_i == "..") { + lt_count++; + } else { + if (lt_count == 0) { + lt_foo="/" $lt_i lt_foo; + } else { + lt_count--; + } + } + } + } + if (lt_foo != "") { lt_freq[lt_foo]++; } + if (lt_freq[lt_foo] == 1) { print lt_foo; } +}'` + # AWK program above erroneously prepends '/' to C:/dos/paths + # for these hosts. + case $host_os in + mingw* | cegcc*) lt_search_path_spec=`$ECHO "$lt_search_path_spec" |\ + $SED 's,/\([A-Za-z]:\),\1,g'` ;; + esac + sys_lib_search_path_spec=`$ECHO "$lt_search_path_spec" | $lt_NL2SP` +else + sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib" +fi +library_names_spec= +libname_spec='lib$name' +soname_spec= +shrext_cmds=".so" +postinstall_cmds= +postuninstall_cmds= +finish_cmds= +finish_eval= +shlibpath_var= +shlibpath_overrides_runpath=unknown +version_type=none +dynamic_linker="$host_os ld.so" +sys_lib_dlsearch_path_spec="/lib /usr/lib" +need_lib_prefix=unknown +hardcode_into_libs=no + +# when you set need_version to no, make sure it does not cause -set_version +# flags to be left without arguments +need_version=unknown + +case $host_os in +aix3*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix $libname.a' + shlibpath_var=LIBPATH + + # AIX 3 has no versioning support, so we append a major version to the name. + soname_spec='${libname}${release}${shared_ext}$major' + ;; + +aix[4-9]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + hardcode_into_libs=yes + if test "$host_cpu" = ia64; then + # AIX 5 supports IA64 + library_names_spec='${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext}$versuffix $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + else + # With GCC up to 2.95.x, collect2 would create an import file + # for dependence libraries. The import file would start with + # the line `#! .'. This would cause the generated library to + # depend on `.', always an invalid library. This was fixed in + # development snapshots of GCC prior to 3.0. + case $host_os in + aix4 | aix4.[01] | aix4.[01].*) + if { echo '#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 97)' + echo ' yes ' + echo '#endif'; } | ${CC} -E - | $GREP yes > /dev/null; then + : + else + can_build_shared=no + fi + ;; + esac + # AIX (on Power*) has no versioning support, so currently we can not hardcode correct + # soname into executable. Probably we can add versioning support to + # collect2, so additional links can be useful in future. + if test "$aix_use_runtimelinking" = yes; then + # If using run time linking (on AIX 4.2 or later) use lib.so + # instead of lib.a to let people know that these are not + # typical AIX shared libraries. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + else + # We preserve .a as extension for shared libraries through AIX4.2 + # and later when we are not doing run time linking. + library_names_spec='${libname}${release}.a $libname.a' + soname_spec='${libname}${release}${shared_ext}$major' + fi + shlibpath_var=LIBPATH + fi + ;; + +amigaos*) + case $host_cpu in + powerpc) + # Since July 2007 AmigaOS4 officially supports .so libraries. + # When compiling the executable, add -use-dynld -Lsobjs: to the compileline. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + ;; + m68k) + library_names_spec='$libname.ixlibrary $libname.a' + # Create ${libname}_ixlibrary.a entries in /sys/libs. + finish_eval='for lib in `ls $libdir/*.ixlibrary 2>/dev/null`; do libname=`func_echo_all "$lib" | $SED '\''s%^.*/\([^/]*\)\.ixlibrary$%\1%'\''`; test $RM /sys/libs/${libname}_ixlibrary.a; $show "cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a"; cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a || exit 1; done' + ;; + esac + ;; + +beos*) + library_names_spec='${libname}${shared_ext}' + dynamic_linker="$host_os ld.so" + shlibpath_var=LIBRARY_PATH + ;; + +bsdi[45]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + finish_cmds='PATH="\$PATH:/sbin" ldconfig $libdir' + shlibpath_var=LD_LIBRARY_PATH + sys_lib_search_path_spec="/shlib /usr/lib /usr/X11/lib /usr/contrib/lib /lib /usr/local/lib" + sys_lib_dlsearch_path_spec="/shlib /usr/lib /usr/local/lib" + # the default ld.so.conf also contains /usr/contrib/lib and + # /usr/X11R6/lib (/usr/X11 is a link to /usr/X11R6), but let us allow + # libtool to hard-code these into programs + ;; + +cygwin* | mingw* | pw32* | cegcc*) + version_type=windows + shrext_cmds=".dll" + need_version=no + need_lib_prefix=no + + case $GCC,$cc_basename in + yes,*) + # gcc + library_names_spec='$libname.dll.a' + # DLL is installed to $(libdir)/../bin by postinstall_cmds + postinstall_cmds='base_file=`basename \${file}`~ + dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~ + dldir=$destdir/`dirname \$dlpath`~ + test -d \$dldir || mkdir -p \$dldir~ + $install_prog $dir/$dlname \$dldir/$dlname~ + chmod a+x \$dldir/$dlname~ + if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then + eval '\''$striplib \$dldir/$dlname'\'' || exit \$?; + fi' + postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ + dlpath=$dir/\$dldll~ + $RM \$dlpath' + shlibpath_overrides_runpath=yes + + case $host_os in + cygwin*) + # Cygwin DLLs use 'cyg' prefix rather than 'lib' + soname_spec='`echo ${libname} | sed -e 's/^lib/cyg/'``echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + + sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/lib/w32api" + ;; + mingw* | cegcc*) + # MinGW DLLs use traditional 'lib' prefix + soname_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + ;; + pw32*) + # pw32 DLLs use 'pw' prefix rather than 'lib' + library_names_spec='`echo ${libname} | sed -e 's/^lib/pw/'``echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + ;; + esac + dynamic_linker='Win32 ld.exe' + ;; + + *,cl*) + # Native MSVC + libname_spec='$name' + soname_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + library_names_spec='${libname}.dll.lib' + + case $build_os in + mingw*) + sys_lib_search_path_spec= + lt_save_ifs=$IFS + IFS=';' + for lt_path in $LIB + do + IFS=$lt_save_ifs + # Let DOS variable expansion print the short 8.3 style file name. + lt_path=`cd "$lt_path" 2>/dev/null && cmd //C "for %i in (".") do @echo %~si"` + sys_lib_search_path_spec="$sys_lib_search_path_spec $lt_path" + done + IFS=$lt_save_ifs + # Convert to MSYS style. + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([a-zA-Z]\\):| /\\1|g' -e 's|^ ||'` + ;; + cygwin*) + # Convert to unix form, then to dos form, then back to unix form + # but this time dos style (no spaces!) so that the unix form looks + # like /cygdrive/c/PROGRA~1:/cygdr... + sys_lib_search_path_spec=`cygpath --path --unix "$LIB"` + sys_lib_search_path_spec=`cygpath --path --dos "$sys_lib_search_path_spec" 2>/dev/null` + sys_lib_search_path_spec=`cygpath --path --unix "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` + ;; + *) + sys_lib_search_path_spec="$LIB" + if $ECHO "$sys_lib_search_path_spec" | $GREP ';[c-zC-Z]:/' >/dev/null; then + # It is most probably a Windows format PATH. + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's/;/ /g'` + else + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` + fi + # FIXME: find the short name or the path components, as spaces are + # common. (e.g. "Program Files" -> "PROGRA~1") + ;; + esac + + # DLL is installed to $(libdir)/../bin by postinstall_cmds + postinstall_cmds='base_file=`basename \${file}`~ + dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~ + dldir=$destdir/`dirname \$dlpath`~ + test -d \$dldir || mkdir -p \$dldir~ + $install_prog $dir/$dlname \$dldir/$dlname' + postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ + dlpath=$dir/\$dldll~ + $RM \$dlpath' + shlibpath_overrides_runpath=yes + dynamic_linker='Win32 link.exe' + ;; + + *) + # Assume MSVC wrapper + library_names_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext} $libname.lib' + dynamic_linker='Win32 ld.exe' + ;; + esac + # FIXME: first we should search . and the directory the executable is in + shlibpath_var=PATH + ;; + +darwin* | rhapsody*) + dynamic_linker="$host_os dyld" + version_type=darwin + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${major}$shared_ext ${libname}$shared_ext' + soname_spec='${libname}${release}${major}$shared_ext' + shlibpath_overrides_runpath=yes + shlibpath_var=DYLD_LIBRARY_PATH + shrext_cmds='`test .$module = .yes && echo .so || echo .dylib`' + + sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/local/lib" + sys_lib_dlsearch_path_spec='/usr/local/lib /lib /usr/lib' + ;; + +dgux*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname$shared_ext' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + ;; + +freebsd* | dragonfly*) + # DragonFly does not have aout. When/if they implement a new + # versioning mechanism, adjust this. + if test -x /usr/bin/objformat; then + objformat=`/usr/bin/objformat` + else + case $host_os in + freebsd[23].*) objformat=aout ;; + *) objformat=elf ;; + esac + fi + version_type=freebsd-$objformat + case $version_type in + freebsd-elf*) + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}' + need_version=no + need_lib_prefix=no + ;; + freebsd-*) + library_names_spec='${libname}${release}${shared_ext}$versuffix $libname${shared_ext}$versuffix' + need_version=yes + ;; + esac + shlibpath_var=LD_LIBRARY_PATH + case $host_os in + freebsd2.*) + shlibpath_overrides_runpath=yes + ;; + freebsd3.[01]* | freebsdelf3.[01]*) + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + freebsd3.[2-9]* | freebsdelf3.[2-9]* | \ + freebsd4.[0-5] | freebsdelf4.[0-5] | freebsd4.1.1 | freebsdelf4.1.1) + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + *) # from 4.6 on, and DragonFly + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + esac + ;; + +gnu*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +haiku*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + dynamic_linker="$host_os runtime_loader" + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LIBRARY_PATH + shlibpath_overrides_runpath=yes + sys_lib_dlsearch_path_spec='/boot/home/config/lib /boot/common/lib /boot/system/lib' + hardcode_into_libs=yes + ;; + +hpux9* | hpux10* | hpux11*) + # Give a soname corresponding to the major version so that dld.sl refuses to + # link against other versions. + version_type=sunos + need_lib_prefix=no + need_version=no + case $host_cpu in + ia64*) + shrext_cmds='.so' + hardcode_into_libs=yes + dynamic_linker="$host_os dld.so" + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + if test "X$HPUX_IA64_MODE" = X32; then + sys_lib_search_path_spec="/usr/lib/hpux32 /usr/local/lib/hpux32 /usr/local/lib" + else + sys_lib_search_path_spec="/usr/lib/hpux64 /usr/local/lib/hpux64" + fi + sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec + ;; + hppa*64*) + shrext_cmds='.sl' + hardcode_into_libs=yes + dynamic_linker="$host_os dld.sl" + shlibpath_var=LD_LIBRARY_PATH # How should we handle SHLIB_PATH + shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + sys_lib_search_path_spec="/usr/lib/pa20_64 /usr/ccs/lib/pa20_64" + sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec + ;; + *) + shrext_cmds='.sl' + dynamic_linker="$host_os dld.sl" + shlibpath_var=SHLIB_PATH + shlibpath_overrides_runpath=no # +s is required to enable SHLIB_PATH + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + ;; + esac + # HP-UX runs *really* slowly unless shared libraries are mode 555, ... + postinstall_cmds='chmod 555 $lib' + # or fails outright, so override atomically: + install_override_mode=555 + ;; + +interix[3-9]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + dynamic_linker='Interix 3.x ld.so.1 (PE, like ELF)' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +irix5* | irix6* | nonstopux*) + case $host_os in + nonstopux*) version_type=nonstopux ;; + *) + if test "$lt_cv_prog_gnu_ld" = yes; then + version_type=linux # correct to gnu/linux during the next big refactor + else + version_type=irix + fi ;; + esac + need_lib_prefix=no + need_version=no + soname_spec='${libname}${release}${shared_ext}$major' + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext} $libname${shared_ext}' + case $host_os in + irix5* | nonstopux*) + libsuff= shlibsuff= + ;; + *) + case $LD in # libtool.m4 will add one of these switches to LD + *-32|*"-32 "|*-melf32bsmip|*"-melf32bsmip ") + libsuff= shlibsuff= libmagic=32-bit;; + *-n32|*"-n32 "|*-melf32bmipn32|*"-melf32bmipn32 ") + libsuff=32 shlibsuff=N32 libmagic=N32;; + *-64|*"-64 "|*-melf64bmip|*"-melf64bmip ") + libsuff=64 shlibsuff=64 libmagic=64-bit;; + *) libsuff= shlibsuff= libmagic=never-match;; + esac + ;; + esac + shlibpath_var=LD_LIBRARY${shlibsuff}_PATH + shlibpath_overrides_runpath=no + sys_lib_search_path_spec="/usr/lib${libsuff} /lib${libsuff} /usr/local/lib${libsuff}" + sys_lib_dlsearch_path_spec="/usr/lib${libsuff} /lib${libsuff}" + hardcode_into_libs=yes + ;; + +# No shared lib support for Linux oldld, aout, or coff. +linux*oldld* | linux*aout* | linux*coff*) + dynamic_linker=no + ;; + +# This must be glibc/ELF. +linux* | k*bsd*-gnu | kopensolaris*-gnu) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -n $libdir' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + + # Some binutils ld are patched to set DT_RUNPATH + if ${lt_cv_shlibpath_overrides_runpath+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_shlibpath_overrides_runpath=no + save_LDFLAGS=$LDFLAGS + save_libdir=$libdir + eval "libdir=/foo; wl=\"$lt_prog_compiler_wl\"; \ + LDFLAGS=\"\$LDFLAGS $hardcode_libdir_flag_spec\"" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + if ($OBJDUMP -p conftest$ac_exeext) 2>/dev/null | grep "RUNPATH.*$libdir" >/dev/null; then : + lt_cv_shlibpath_overrides_runpath=yes +fi +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LDFLAGS=$save_LDFLAGS + libdir=$save_libdir + +fi + + shlibpath_overrides_runpath=$lt_cv_shlibpath_overrides_runpath + + # This implies no fast_install, which is unacceptable. + # Some rework will be needed to allow for fast_install + # before this can be enabled. + hardcode_into_libs=yes + + # Append ld.so.conf contents to the search path + if test -f /etc/ld.so.conf; then + lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \$2)); skip = 1; } { if (!skip) print \$0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;/^[ ]*hwcap[ ]/d;s/[:, ]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;s/"//g;/^$/d' | tr '\n' ' '` + sys_lib_dlsearch_path_spec="/lib /usr/lib $lt_ld_extra" + fi + + # We used to test for /lib/ld.so.1 and disable shared libraries on + # powerpc, because MkLinux only supported shared libraries with the + # GNU dynamic linker. Since this was broken with cross compilers, + # most powerpc-linux boxes support dynamic linking these days and + # people can always --disable-shared, the test was removed, and we + # assume the GNU/Linux dynamic linker is in use. + dynamic_linker='GNU/Linux ld.so' + ;; + +netbsdelf*-gnu) + version_type=linux + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + dynamic_linker='NetBSD ld.elf_so' + ;; + +netbsd*) + version_type=sunos + need_lib_prefix=no + need_version=no + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' + dynamic_linker='NetBSD (a.out) ld.so' + else + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + dynamic_linker='NetBSD ld.elf_so' + fi + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + +newsos6) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + ;; + +*nto* | *qnx*) + version_type=qnx + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + dynamic_linker='ldqnx.so' + ;; + +openbsd*) + version_type=sunos + sys_lib_dlsearch_path_spec="/usr/lib" + need_lib_prefix=no + # Some older versions of OpenBSD (3.3 at least) *do* need versioned libs. + case $host_os in + openbsd3.3 | openbsd3.3.*) need_version=yes ;; + *) need_version=no ;; + esac + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' + shlibpath_var=LD_LIBRARY_PATH + if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + case $host_os in + openbsd2.[89] | openbsd2.[89].*) + shlibpath_overrides_runpath=no + ;; + *) + shlibpath_overrides_runpath=yes + ;; + esac + else + shlibpath_overrides_runpath=yes + fi + ;; + +os2*) + libname_spec='$name' + shrext_cmds=".dll" + need_lib_prefix=no + library_names_spec='$libname${shared_ext} $libname.a' + dynamic_linker='OS/2 ld.exe' + shlibpath_var=LIBPATH + ;; + +osf3* | osf4* | osf5*) + version_type=osf + need_lib_prefix=no + need_version=no + soname_spec='${libname}${release}${shared_ext}$major' + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + sys_lib_search_path_spec="/usr/shlib /usr/ccs/lib /usr/lib/cmplrs/cc /usr/lib /usr/local/lib /var/shlib" + sys_lib_dlsearch_path_spec="$sys_lib_search_path_spec" + ;; + +rdos*) + dynamic_linker=no + ;; + +solaris*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + # ldd complains unless libraries are executable + postinstall_cmds='chmod +x $lib' + ;; + +sunos4*) + version_type=sunos + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/usr/etc" ldconfig $libdir' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + if test "$with_gnu_ld" = yes; then + need_lib_prefix=no + fi + need_version=yes + ;; + +sysv4 | sysv4.3*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + case $host_vendor in + sni) + shlibpath_overrides_runpath=no + need_lib_prefix=no + runpath_var=LD_RUN_PATH + ;; + siemens) + need_lib_prefix=no + ;; + motorola) + need_lib_prefix=no + need_version=no + shlibpath_overrides_runpath=no + sys_lib_search_path_spec='/lib /usr/lib /usr/ccs/lib' + ;; + esac + ;; + +sysv4*MP*) + if test -d /usr/nec ;then + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='$libname${shared_ext}.$versuffix $libname${shared_ext}.$major $libname${shared_ext}' + soname_spec='$libname${shared_ext}.$major' + shlibpath_var=LD_LIBRARY_PATH + fi + ;; + +sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) + version_type=freebsd-elf + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + if test "$with_gnu_ld" = yes; then + sys_lib_search_path_spec='/usr/local/lib /usr/gnu/lib /usr/ccs/lib /usr/lib /lib' + else + sys_lib_search_path_spec='/usr/ccs/lib /usr/lib' + case $host_os in + sco3.2v5*) + sys_lib_search_path_spec="$sys_lib_search_path_spec /lib" + ;; + esac + fi + sys_lib_dlsearch_path_spec='/usr/lib' + ;; + +tpf*) + # TPF is a cross-target only. Preferred cross-host = GNU/Linux. + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +uts4*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + ;; + +*) + dynamic_linker=no + ;; +esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $dynamic_linker" >&5 +$as_echo "$dynamic_linker" >&6; } +test "$dynamic_linker" = no && can_build_shared=no + +variables_saved_for_relink="PATH $shlibpath_var $runpath_var" +if test "$GCC" = yes; then + variables_saved_for_relink="$variables_saved_for_relink GCC_EXEC_PREFIX COMPILER_PATH LIBRARY_PATH" +fi + +if test "${lt_cv_sys_lib_search_path_spec+set}" = set; then + sys_lib_search_path_spec="$lt_cv_sys_lib_search_path_spec" +fi +if test "${lt_cv_sys_lib_dlsearch_path_spec+set}" = set; then + sys_lib_dlsearch_path_spec="$lt_cv_sys_lib_dlsearch_path_spec" +fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking how to hardcode library paths into programs" >&5 +$as_echo_n "checking how to hardcode library paths into programs... " >&6; } +hardcode_action= +if test -n "$hardcode_libdir_flag_spec" || + test -n "$runpath_var" || + test "X$hardcode_automatic" = "Xyes" ; then + + # We can hardcode non-existent directories. + if test "$hardcode_direct" != no && + # If the only mechanism to avoid hardcoding is shlibpath_var, we + # have to relink, otherwise we might link with an installed library + # when we should be linking with a yet-to-be-installed one + ## test "$_LT_TAGVAR(hardcode_shlibpath_var, )" != no && + test "$hardcode_minus_L" != no; then + # Linking always hardcodes the temporary library directory. + hardcode_action=relink + else + # We can link without hardcoding, and we can hardcode nonexisting dirs. + hardcode_action=immediate + fi +else + # We cannot hardcode anything, or else we can only hardcode existing + # directories. + hardcode_action=unsupported +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $hardcode_action" >&5 +$as_echo "$hardcode_action" >&6; } + +if test "$hardcode_action" = relink || + test "$inherit_rpath" = yes; then + # Fast installation is not supported + enable_fast_install=no +elif test "$shlibpath_overrides_runpath" = yes || + test "$enable_shared" = no; then + # Fast installation is not necessary + enable_fast_install=needless +fi + + + + + + + if test "x$enable_dlopen" != xyes; then + enable_dlopen=unknown + enable_dlopen_self=unknown + enable_dlopen_self_static=unknown +else + lt_cv_dlopen=no + lt_cv_dlopen_libs= + + case $host_os in + beos*) + lt_cv_dlopen="load_add_on" + lt_cv_dlopen_libs= + lt_cv_dlopen_self=yes + ;; + + mingw* | pw32* | cegcc*) + lt_cv_dlopen="LoadLibrary" + lt_cv_dlopen_libs= + ;; + + cygwin*) + lt_cv_dlopen="dlopen" + lt_cv_dlopen_libs= + ;; + + darwin*) + # if libdl is installed we need to link against it + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5 +$as_echo_n "checking for dlopen in -ldl... " >&6; } +if ${ac_cv_lib_dl_dlopen+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ldl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char dlopen (); +int +main () +{ +return dlopen (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_dl_dlopen=yes +else + ac_cv_lib_dl_dlopen=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 +$as_echo "$ac_cv_lib_dl_dlopen" >&6; } +if test "x$ac_cv_lib_dl_dlopen" = xyes; then : + lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl" +else + + lt_cv_dlopen="dyld" + lt_cv_dlopen_libs= + lt_cv_dlopen_self=yes + +fi + + ;; + + *) + ac_fn_c_check_func "$LINENO" "shl_load" "ac_cv_func_shl_load" +if test "x$ac_cv_func_shl_load" = xyes; then : + lt_cv_dlopen="shl_load" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shl_load in -ldld" >&5 +$as_echo_n "checking for shl_load in -ldld... " >&6; } +if ${ac_cv_lib_dld_shl_load+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ldld $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char shl_load (); +int +main () +{ +return shl_load (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_dld_shl_load=yes +else + ac_cv_lib_dld_shl_load=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_shl_load" >&5 +$as_echo "$ac_cv_lib_dld_shl_load" >&6; } +if test "x$ac_cv_lib_dld_shl_load" = xyes; then : + lt_cv_dlopen="shl_load" lt_cv_dlopen_libs="-ldld" +else + ac_fn_c_check_func "$LINENO" "dlopen" "ac_cv_func_dlopen" +if test "x$ac_cv_func_dlopen" = xyes; then : + lt_cv_dlopen="dlopen" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5 +$as_echo_n "checking for dlopen in -ldl... " >&6; } +if ${ac_cv_lib_dl_dlopen+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ldl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char dlopen (); +int +main () +{ +return dlopen (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_dl_dlopen=yes +else + ac_cv_lib_dl_dlopen=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 +$as_echo "$ac_cv_lib_dl_dlopen" >&6; } +if test "x$ac_cv_lib_dl_dlopen" = xyes; then : + lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -lsvld" >&5 +$as_echo_n "checking for dlopen in -lsvld... " >&6; } +if ${ac_cv_lib_svld_dlopen+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsvld $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char dlopen (); +int +main () +{ +return dlopen (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_svld_dlopen=yes +else + ac_cv_lib_svld_dlopen=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_svld_dlopen" >&5 +$as_echo "$ac_cv_lib_svld_dlopen" >&6; } +if test "x$ac_cv_lib_svld_dlopen" = xyes; then : + lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-lsvld" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dld_link in -ldld" >&5 +$as_echo_n "checking for dld_link in -ldld... " >&6; } +if ${ac_cv_lib_dld_dld_link+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ldld $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char dld_link (); +int +main () +{ +return dld_link (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_dld_dld_link=yes +else + ac_cv_lib_dld_dld_link=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_dld_link" >&5 +$as_echo "$ac_cv_lib_dld_dld_link" >&6; } +if test "x$ac_cv_lib_dld_dld_link" = xyes; then : + lt_cv_dlopen="dld_link" lt_cv_dlopen_libs="-ldld" +fi + + +fi + + +fi + + +fi + + +fi + + +fi + + ;; + esac + + if test "x$lt_cv_dlopen" != xno; then + enable_dlopen=yes + else + enable_dlopen=no + fi + + case $lt_cv_dlopen in + dlopen) + save_CPPFLAGS="$CPPFLAGS" + test "x$ac_cv_header_dlfcn_h" = xyes && CPPFLAGS="$CPPFLAGS -DHAVE_DLFCN_H" + + save_LDFLAGS="$LDFLAGS" + wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $export_dynamic_flag_spec\" + + save_LIBS="$LIBS" + LIBS="$lt_cv_dlopen_libs $LIBS" + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a program can dlopen itself" >&5 +$as_echo_n "checking whether a program can dlopen itself... " >&6; } +if ${lt_cv_dlopen_self+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : + lt_cv_dlopen_self=cross +else + lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 + lt_status=$lt_dlunknown + cat > conftest.$ac_ext <<_LT_EOF +#line $LINENO "configure" +#include "confdefs.h" + +#if HAVE_DLFCN_H +#include +#endif + +#include + +#ifdef RTLD_GLOBAL +# define LT_DLGLOBAL RTLD_GLOBAL +#else +# ifdef DL_GLOBAL +# define LT_DLGLOBAL DL_GLOBAL +# else +# define LT_DLGLOBAL 0 +# endif +#endif + +/* We may have to define LT_DLLAZY_OR_NOW in the command line if we + find out it does not work in some platform. */ +#ifndef LT_DLLAZY_OR_NOW +# ifdef RTLD_LAZY +# define LT_DLLAZY_OR_NOW RTLD_LAZY +# else +# ifdef DL_LAZY +# define LT_DLLAZY_OR_NOW DL_LAZY +# else +# ifdef RTLD_NOW +# define LT_DLLAZY_OR_NOW RTLD_NOW +# else +# ifdef DL_NOW +# define LT_DLLAZY_OR_NOW DL_NOW +# else +# define LT_DLLAZY_OR_NOW 0 +# endif +# endif +# endif +# endif +#endif + +/* When -fvisbility=hidden is used, assume the code has been annotated + correspondingly for the symbols needed. */ +#if defined(__GNUC__) && (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3)) || (__GNUC__ > 3)) +int fnord () __attribute__((visibility("default"))); +#endif + +int fnord () { return 42; } +int main () +{ + void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW); + int status = $lt_dlunknown; + + if (self) + { + if (dlsym (self,"fnord")) status = $lt_dlno_uscore; + else + { + if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore; + else puts (dlerror ()); + } + /* dlclose (self); */ + } + else + puts (dlerror ()); + + return status; +} +_LT_EOF + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5 + (eval $ac_link) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && test -s conftest${ac_exeext} 2>/dev/null; then + (./conftest; exit; ) >&5 2>/dev/null + lt_status=$? + case x$lt_status in + x$lt_dlno_uscore) lt_cv_dlopen_self=yes ;; + x$lt_dlneed_uscore) lt_cv_dlopen_self=yes ;; + x$lt_dlunknown|x*) lt_cv_dlopen_self=no ;; + esac + else : + # compilation failed + lt_cv_dlopen_self=no + fi +fi +rm -fr conftest* + + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_dlopen_self" >&5 +$as_echo "$lt_cv_dlopen_self" >&6; } + + if test "x$lt_cv_dlopen_self" = xyes; then + wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $lt_prog_compiler_static\" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a statically linked program can dlopen itself" >&5 +$as_echo_n "checking whether a statically linked program can dlopen itself... " >&6; } +if ${lt_cv_dlopen_self_static+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : + lt_cv_dlopen_self_static=cross +else + lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 + lt_status=$lt_dlunknown + cat > conftest.$ac_ext <<_LT_EOF +#line $LINENO "configure" +#include "confdefs.h" + +#if HAVE_DLFCN_H +#include +#endif + +#include + +#ifdef RTLD_GLOBAL +# define LT_DLGLOBAL RTLD_GLOBAL +#else +# ifdef DL_GLOBAL +# define LT_DLGLOBAL DL_GLOBAL +# else +# define LT_DLGLOBAL 0 +# endif +#endif + +/* We may have to define LT_DLLAZY_OR_NOW in the command line if we + find out it does not work in some platform. */ +#ifndef LT_DLLAZY_OR_NOW +# ifdef RTLD_LAZY +# define LT_DLLAZY_OR_NOW RTLD_LAZY +# else +# ifdef DL_LAZY +# define LT_DLLAZY_OR_NOW DL_LAZY +# else +# ifdef RTLD_NOW +# define LT_DLLAZY_OR_NOW RTLD_NOW +# else +# ifdef DL_NOW +# define LT_DLLAZY_OR_NOW DL_NOW +# else +# define LT_DLLAZY_OR_NOW 0 +# endif +# endif +# endif +# endif +#endif + +/* When -fvisbility=hidden is used, assume the code has been annotated + correspondingly for the symbols needed. */ +#if defined(__GNUC__) && (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3)) || (__GNUC__ > 3)) +int fnord () __attribute__((visibility("default"))); +#endif + +int fnord () { return 42; } +int main () +{ + void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW); + int status = $lt_dlunknown; + + if (self) + { + if (dlsym (self,"fnord")) status = $lt_dlno_uscore; + else + { + if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore; + else puts (dlerror ()); + } + /* dlclose (self); */ + } + else + puts (dlerror ()); + + return status; +} +_LT_EOF + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5 + (eval $ac_link) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && test -s conftest${ac_exeext} 2>/dev/null; then + (./conftest; exit; ) >&5 2>/dev/null + lt_status=$? + case x$lt_status in + x$lt_dlno_uscore) lt_cv_dlopen_self_static=yes ;; + x$lt_dlneed_uscore) lt_cv_dlopen_self_static=yes ;; + x$lt_dlunknown|x*) lt_cv_dlopen_self_static=no ;; + esac + else : + # compilation failed + lt_cv_dlopen_self_static=no + fi +fi +rm -fr conftest* + + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_dlopen_self_static" >&5 +$as_echo "$lt_cv_dlopen_self_static" >&6; } + fi + + CPPFLAGS="$save_CPPFLAGS" + LDFLAGS="$save_LDFLAGS" + LIBS="$save_LIBS" + ;; + esac + + case $lt_cv_dlopen_self in + yes|no) enable_dlopen_self=$lt_cv_dlopen_self ;; + *) enable_dlopen_self=unknown ;; + esac + + case $lt_cv_dlopen_self_static in + yes|no) enable_dlopen_self_static=$lt_cv_dlopen_self_static ;; + *) enable_dlopen_self_static=unknown ;; + esac +fi + + + + + + + + + + + + + + + + + +striplib= +old_striplib= +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether stripping libraries is possible" >&5 +$as_echo_n "checking whether stripping libraries is possible... " >&6; } +if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then + test -z "$old_striplib" && old_striplib="$STRIP --strip-debug" + test -z "$striplib" && striplib="$STRIP --strip-unneeded" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else +# FIXME - insert some real tests, host_os isn't really good enough + case $host_os in + darwin*) + if test -n "$STRIP" ; then + striplib="$STRIP -x" + old_striplib="$STRIP -S" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi + ;; + *) + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + ;; + esac +fi + + + + + + + + + + + + + # Report which library types will actually be built + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if libtool supports shared libraries" >&5 +$as_echo_n "checking if libtool supports shared libraries... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $can_build_shared" >&5 +$as_echo "$can_build_shared" >&6; } + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build shared libraries" >&5 +$as_echo_n "checking whether to build shared libraries... " >&6; } + test "$can_build_shared" = "no" && enable_shared=no + + # On AIX, shared libraries and static libraries use the same namespace, and + # are all built from PIC. + case $host_os in + aix3*) + test "$enable_shared" = yes && enable_static=no + if test -n "$RANLIB"; then + archive_cmds="$archive_cmds~\$RANLIB \$lib" + postinstall_cmds='$RANLIB $lib' + fi + ;; + + aix[4-9]*) + if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then + test "$enable_shared" = yes && enable_static=no + fi + ;; + esac + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_shared" >&5 +$as_echo "$enable_shared" >&6; } + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build static libraries" >&5 +$as_echo_n "checking whether to build static libraries... " >&6; } + # Make sure either enable_shared or enable_static is yes. + test "$enable_shared" = yes || enable_static=yes + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_static" >&5 +$as_echo "$enable_static" >&6; } + + + + +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +CC="$lt_save_CC" + + + + + + + + + + + + + + + + ac_config_commands="$ac_config_commands libtool" + + + + +# Only expand once: + + +# By default we simply use the C compiler to build assembly code. + +test "${CCAS+set}" = set || CCAS=$CC +test "${CCASFLAGS+set}" = set || CCASFLAGS=$CFLAGS + + + +depcc="$CCAS" am_compiler_list= + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5 +$as_echo_n "checking dependency style of $depcc... " >&6; } +if ${am_cv_CCAS_dependencies_compiler_type+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + rm -rf conftest.dir + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CCAS_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + am__universal=false + + + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. Also, some Intel + # versions had trouble with output in subdirs + am__obj=sub/conftest.${OBJEXT-o} + am__minus_obj="-o $am__obj" + case $depmode in + gcc) + # This depmode causes a compiler race in universal mode. + test "$am__universal" = false || continue + ;; + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + msvc7 | msvc7msys | msvisualcpp | msvcmsys) + # This compiler won't grok `-c -o', but also, the minuso test has + # not run yet. These depmodes are late enough in the game, and + # so weak that their functioning should not be impacted. + am__obj=conftest.${OBJEXT-o} + am__minus_obj= + ;; + none) break ;; + esac + if depmode=$depmode \ + source=sub/conftest.c object=$am__obj \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep $am__obj sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CCAS_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CCAS_dependencies_compiler_type=none +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CCAS_dependencies_compiler_type" >&5 +$as_echo "$am_cv_CCAS_dependencies_compiler_type" >&6; } +CCASDEPMODE=depmode=$am_cv_CCAS_dependencies_compiler_type + + if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CCAS_dependencies_compiler_type" = gcc3; then + am__fastdepCCAS_TRUE= + am__fastdepCCAS_FALSE='#' +else + am__fastdepCCAS_TRUE='#' + am__fastdepCCAS_FALSE= +fi + + + + +# enable make V=0 (if automake >1.11) +if test "`cd $srcdir && pwd`" != "`pwd`"; then + # Use -I$(srcdir) only when $(srcdir) != ., so that make's output + # is not polluted with repeated "-I." + am__isrc=' -I$(srcdir)' + # test to see if srcdir already configured + if test -f $srcdir/config.status; then + as_fn_error $? "source directory already configured; run \"make distclean\" there first" "$LINENO" 5 + fi +fi + +# test whether we have cygpath +if test -z "$CYGPATH_W"; then + if (cygpath --version) >/dev/null 2>/dev/null; then + CYGPATH_W='cygpath -w' + else + CYGPATH_W=echo + fi +fi + + +# Define the identity of the package. + PACKAGE='ceph' + VERSION='0.80.7' + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE "$PACKAGE" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define VERSION "$VERSION" +_ACEOF + +# Some tools Automake needs. + +ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"} + + +AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"} + + +AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"} + + +AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"} + + +MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"} + +# We need awk for the "check" target. The system "awk" is bad on +# some platforms. +# Always define AMTAR for backward compatibility. Yes, it's still used +# in the wild :-( We should find a proper way to deprecate it ... +AMTAR='$${TAR-tar}' + +am__tar='$${TAR-tar} chof - "$$tardir"' am__untar='$${TAR-tar} xf -' + + + + +depcc="$CC" am_compiler_list= + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5 +$as_echo_n "checking dependency style of $depcc... " >&6; } +if ${am_cv_CC_dependencies_compiler_type+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + rm -rf conftest.dir + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CC_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + am__universal=false + case " $depcc " in #( + *\ -arch\ *\ -arch\ *) am__universal=true ;; + esac + + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. Also, some Intel + # versions had trouble with output in subdirs + am__obj=sub/conftest.${OBJEXT-o} + am__minus_obj="-o $am__obj" + case $depmode in + gcc) + # This depmode causes a compiler race in universal mode. + test "$am__universal" = false || continue + ;; + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + msvc7 | msvc7msys | msvisualcpp | msvcmsys) + # This compiler won't grok `-c -o', but also, the minuso test has + # not run yet. These depmodes are late enough in the game, and + # so weak that their functioning should not be impacted. + am__obj=conftest.${OBJEXT-o} + am__minus_obj= + ;; + none) break ;; + esac + if depmode=$depmode \ + source=sub/conftest.c object=$am__obj \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep $am__obj sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CC_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CC_dependencies_compiler_type=none +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CC_dependencies_compiler_type" >&5 +$as_echo "$am_cv_CC_dependencies_compiler_type" >&6; } +CCDEPMODE=depmode=$am_cv_CC_dependencies_compiler_type + + if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CC_dependencies_compiler_type" = gcc3; then + am__fastdepCC_TRUE= + am__fastdepCC_FALSE='#' +else + am__fastdepCC_TRUE='#' + am__fastdepCC_FALSE= +fi + + + +# Check whether --enable-silent-rules was given. +if test "${enable_silent_rules+set}" = set; then : + enableval=$enable_silent_rules; +fi + +case $enable_silent_rules in +yes) AM_DEFAULT_VERBOSITY=0;; +no) AM_DEFAULT_VERBOSITY=1;; +*) AM_DEFAULT_VERBOSITY=0;; +esac +am_make=${MAKE-make} +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $am_make supports nested variables" >&5 +$as_echo_n "checking whether $am_make supports nested variables... " >&6; } +if ${am_cv_make_support_nested_variables+:} false; then : + $as_echo_n "(cached) " >&6 +else + if $as_echo 'TRUE=$(BAR$(V)) +BAR0=false +BAR1=true +V=1 +am__doit: + @$(TRUE) +.PHONY: am__doit' | $am_make -f - >/dev/null 2>&1; then + am_cv_make_support_nested_variables=yes +else + am_cv_make_support_nested_variables=no +fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_make_support_nested_variables" >&5 +$as_echo "$am_cv_make_support_nested_variables" >&6; } +if test $am_cv_make_support_nested_variables = yes; then + AM_V='$(V)' + AM_DEFAULT_V='$(AM_DEFAULT_VERBOSITY)' +else + AM_V=$AM_DEFAULT_VERBOSITY + AM_DEFAULT_V=$AM_DEFAULT_VERBOSITY +fi +AM_BACKSLASH='\' + + +# Platform +case "${target_os}" in +darwin*) + +$as_echo "#define DARWIN 1" >>confdefs.h + + ;; +linux*) + linux="yes" + ;; +freebsd*) + freebsd="yes" + ;; +esac + if test x"$linux" = x"yes"; then + LINUX_TRUE= + LINUX_FALSE='#' +else + LINUX_TRUE='#' + LINUX_FALSE= +fi + + if test x"$freebsd" = x"yes"; then + FREEBSD_TRUE= + FREEBSD_FALSE='#' +else + FREEBSD_TRUE='#' + FREEBSD_FALSE= +fi + + +# Checks for programs. +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +if test -z "$CXX"; then + if test -n "$CCC"; then + CXX=$CCC + else + if test -n "$ac_tool_prefix"; then + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CXX"; then + ac_cv_prog_CXX="$CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CXX="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CXX=$ac_cv_prog_CXX +if test -n "$CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +$as_echo "$CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$CXX" && break + done +fi +if test -z "$CXX"; then + ac_ct_CXX=$CXX + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CXX"; then + ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CXX="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CXX=$ac_cv_prog_ac_ct_CXX +if test -n "$ac_ct_CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CXX" >&5 +$as_echo "$ac_ct_CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_CXX" && break +done + + if test "x$ac_ct_CXX" = x; then + CXX="g++" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CXX=$ac_ct_CXX + fi +fi + + fi +fi +# Provide some information about the compiler. +$as_echo "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +for ac_option in --version -v -V -qversion; do + { { ac_try="$ac_compiler $ac_option >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compiler $ac_option >&5") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + sed '10a\ +... rest of stderr output deleted ... + 10q' conftest.err >conftest.er1 + cat conftest.er1 >&5 + fi + rm -f conftest.er1 conftest.err + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } +done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C++ compiler" >&5 +$as_echo_n "checking whether we are using the GNU C++ compiler... " >&6; } +if ${ac_cv_cxx_compiler_gnu+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_compiler_gnu=yes +else + ac_compiler_gnu=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_cxx_compiler_gnu=$ac_compiler_gnu + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_compiler_gnu" >&5 +$as_echo "$ac_cv_cxx_compiler_gnu" >&6; } +if test $ac_compiler_gnu = yes; then + GXX=yes +else + GXX= +fi +ac_test_CXXFLAGS=${CXXFLAGS+set} +ac_save_CXXFLAGS=$CXXFLAGS +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CXX accepts -g" >&5 +$as_echo_n "checking whether $CXX accepts -g... " >&6; } +if ${ac_cv_prog_cxx_g+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_save_cxx_werror_flag=$ac_cxx_werror_flag + ac_cxx_werror_flag=yes + ac_cv_prog_cxx_g=no + CXXFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_prog_cxx_g=yes +else + CXXFLAGS="" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + +else + ac_cxx_werror_flag=$ac_save_cxx_werror_flag + CXXFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_prog_cxx_g=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_cxx_werror_flag=$ac_save_cxx_werror_flag +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_g" >&5 +$as_echo "$ac_cv_prog_cxx_g" >&6; } +if test "$ac_test_CXXFLAGS" = set; then + CXXFLAGS=$ac_save_CXXFLAGS +elif test $ac_cv_prog_cxx_g = yes; then + if test "$GXX" = yes; then + CXXFLAGS="-g -O2" + else + CXXFLAGS="-g" + fi +else + if test "$GXX" = yes; then + CXXFLAGS="-O2" + else + CXXFLAGS= + fi +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +depcc="$CXX" am_compiler_list= + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5 +$as_echo_n "checking dependency style of $depcc... " >&6; } +if ${am_cv_CXX_dependencies_compiler_type+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + rm -rf conftest.dir + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CXX_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + am__universal=false + case " $depcc " in #( + *\ -arch\ *\ -arch\ *) am__universal=true ;; + esac + + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. Also, some Intel + # versions had trouble with output in subdirs + am__obj=sub/conftest.${OBJEXT-o} + am__minus_obj="-o $am__obj" + case $depmode in + gcc) + # This depmode causes a compiler race in universal mode. + test "$am__universal" = false || continue + ;; + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + msvc7 | msvc7msys | msvisualcpp | msvcmsys) + # This compiler won't grok `-c -o', but also, the minuso test has + # not run yet. These depmodes are late enough in the game, and + # so weak that their functioning should not be impacted. + am__obj=conftest.${OBJEXT-o} + am__minus_obj= + ;; + none) break ;; + esac + if depmode=$depmode \ + source=sub/conftest.c object=$am__obj \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep $am__obj sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CXX_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CXX_dependencies_compiler_type=none +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CXX_dependencies_compiler_type" >&5 +$as_echo "$am_cv_CXX_dependencies_compiler_type" >&6; } +CXXDEPMODE=depmode=$am_cv_CXX_dependencies_compiler_type + + if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CXX_dependencies_compiler_type" = gcc3; then + am__fastdepCXX_TRUE= + am__fastdepCXX_FALSE='#' +else + am__fastdepCXX_TRUE='#' + am__fastdepCXX_FALSE= +fi + + + + +func_stripname_cnf () +{ + case ${2} in + .*) func_stripname_result=`$ECHO "${3}" | $SED "s%^${1}%%; s%\\\\${2}\$%%"`;; + *) func_stripname_result=`$ECHO "${3}" | $SED "s%^${1}%%; s%${2}\$%%"`;; + esac +} # func_stripname_cnf + + if test -n "$CXX" && ( test "X$CXX" != "Xno" && + ( (test "X$CXX" = "Xg++" && `g++ -v >/dev/null 2>&1` ) || + (test "X$CXX" != "Xg++"))) ; then + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C++ preprocessor" >&5 +$as_echo_n "checking how to run the C++ preprocessor... " >&6; } +if test -z "$CXXCPP"; then + if ${ac_cv_prog_CXXCPP+:} false; then : + $as_echo_n "(cached) " >&6 +else + # Double quotes because CXXCPP needs to be expanded + for CXXCPP in "$CXX -E" "/lib/cpp" + do + ac_preproc_ok=false +for ac_cxx_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + break +fi + + done + ac_cv_prog_CXXCPP=$CXXCPP + +fi + CXXCPP=$ac_cv_prog_CXXCPP +else + ac_cv_prog_CXXCPP=$CXXCPP +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CXXCPP" >&5 +$as_echo "$CXXCPP" >&6; } +ac_preproc_ok=false +for ac_cxx_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "C++ preprocessor \"$CXXCPP\" fails sanity check +See \`config.log' for more details" "$LINENO" 5; } +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +else + _lt_caught_CXX_error=yes +fi + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + +archive_cmds_need_lc_CXX=no +allow_undefined_flag_CXX= +always_export_symbols_CXX=no +archive_expsym_cmds_CXX= +compiler_needs_object_CXX=no +export_dynamic_flag_spec_CXX= +hardcode_direct_CXX=no +hardcode_direct_absolute_CXX=no +hardcode_libdir_flag_spec_CXX= +hardcode_libdir_separator_CXX= +hardcode_minus_L_CXX=no +hardcode_shlibpath_var_CXX=unsupported +hardcode_automatic_CXX=no +inherit_rpath_CXX=no +module_cmds_CXX= +module_expsym_cmds_CXX= +link_all_deplibs_CXX=unknown +old_archive_cmds_CXX=$old_archive_cmds +reload_flag_CXX=$reload_flag +reload_cmds_CXX=$reload_cmds +no_undefined_flag_CXX= +whole_archive_flag_spec_CXX= +enable_shared_with_static_runtimes_CXX=no + +# Source file extension for C++ test sources. +ac_ext=cpp + +# Object file extension for compiled C++ test sources. +objext=o +objext_CXX=$objext + +# No sense in running all these tests if we already determined that +# the CXX compiler isn't working. Some variables (like enable_shared) +# are currently assumed to apply to all compilers on this platform, +# and will be corrupted by setting them based on a non-working compiler. +if test "$_lt_caught_CXX_error" != yes; then + # Code to be used in simple compile tests + lt_simple_compile_test_code="int some_variable = 0;" + + # Code to be used in simple link tests + lt_simple_link_test_code='int main(int, char *[]) { return(0); }' + + # ltmain only uses $CC for tagged configurations so make sure $CC is set. + + + + + + +# If no C compiler was specified, use CC. +LTCC=${LTCC-"$CC"} + +# If no C compiler flags were specified, use CFLAGS. +LTCFLAGS=${LTCFLAGS-"$CFLAGS"} + +# Allow CC to be a program name with arguments. +compiler=$CC + + + # save warnings/boilerplate of simple test code + ac_outfile=conftest.$ac_objext +echo "$lt_simple_compile_test_code" >conftest.$ac_ext +eval "$ac_compile" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err +_lt_compiler_boilerplate=`cat conftest.err` +$RM conftest* + + ac_outfile=conftest.$ac_objext +echo "$lt_simple_link_test_code" >conftest.$ac_ext +eval "$ac_link" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err +_lt_linker_boilerplate=`cat conftest.err` +$RM -r conftest* + + + # Allow CC to be a program name with arguments. + lt_save_CC=$CC + lt_save_CFLAGS=$CFLAGS + lt_save_LD=$LD + lt_save_GCC=$GCC + GCC=$GXX + lt_save_with_gnu_ld=$with_gnu_ld + lt_save_path_LD=$lt_cv_path_LD + if test -n "${lt_cv_prog_gnu_ldcxx+set}"; then + lt_cv_prog_gnu_ld=$lt_cv_prog_gnu_ldcxx + else + $as_unset lt_cv_prog_gnu_ld + fi + if test -n "${lt_cv_path_LDCXX+set}"; then + lt_cv_path_LD=$lt_cv_path_LDCXX + else + $as_unset lt_cv_path_LD + fi + test -z "${LDCXX+set}" || LD=$LDCXX + CC=${CXX-"c++"} + CFLAGS=$CXXFLAGS + compiler=$CC + compiler_CXX=$CC + for cc_temp in $compiler""; do + case $cc_temp in + compile | *[\\/]compile | ccache | *[\\/]ccache ) ;; + distcc | *[\\/]distcc | purify | *[\\/]purify ) ;; + \-*) ;; + *) break;; + esac +done +cc_basename=`$ECHO "$cc_temp" | $SED "s%.*/%%; s%^$host_alias-%%"` + + + if test -n "$compiler"; then + # We don't want -fno-exception when compiling C++ code, so set the + # no_builtin_flag separately + if test "$GXX" = yes; then + lt_prog_compiler_no_builtin_flag_CXX=' -fno-builtin' + else + lt_prog_compiler_no_builtin_flag_CXX= + fi + + if test "$GXX" = yes; then + # Set up default GNU C++ configuration + + + +# Check whether --with-gnu-ld was given. +if test "${with_gnu_ld+set}" = set; then : + withval=$with_gnu_ld; test "$withval" = no || with_gnu_ld=yes +else + with_gnu_ld=no +fi + +ac_prog=ld +if test "$GCC" = yes; then + # Check if gcc -print-prog-name=ld gives a path. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ld used by $CC" >&5 +$as_echo_n "checking for ld used by $CC... " >&6; } + case $host in + *-*-mingw*) + # gcc leaves a trailing carriage return which upsets mingw + ac_prog=`($CC -print-prog-name=ld) 2>&5 | tr -d '\015'` ;; + *) + ac_prog=`($CC -print-prog-name=ld) 2>&5` ;; + esac + case $ac_prog in + # Accept absolute paths. + [\\/]* | ?:[\\/]*) + re_direlt='/[^/][^/]*/\.\./' + # Canonicalize the pathname of ld + ac_prog=`$ECHO "$ac_prog"| $SED 's%\\\\%/%g'` + while $ECHO "$ac_prog" | $GREP "$re_direlt" > /dev/null 2>&1; do + ac_prog=`$ECHO $ac_prog| $SED "s%$re_direlt%/%"` + done + test -z "$LD" && LD="$ac_prog" + ;; + "") + # If it fails, then pretend we aren't using GCC. + ac_prog=ld + ;; + *) + # If it is relative, then search for the first ld in PATH. + with_gnu_ld=unknown + ;; + esac +elif test "$with_gnu_ld" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for GNU ld" >&5 +$as_echo_n "checking for GNU ld... " >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for non-GNU ld" >&5 +$as_echo_n "checking for non-GNU ld... " >&6; } +fi +if ${lt_cv_path_LD+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$LD"; then + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR + for ac_dir in $PATH; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + if test -f "$ac_dir/$ac_prog" || test -f "$ac_dir/$ac_prog$ac_exeext"; then + lt_cv_path_LD="$ac_dir/$ac_prog" + # Check to see if the program is GNU ld. I'd rather use --version, + # but apparently some variants of GNU ld only accept -v. + # Break only if it was the GNU/non-GNU ld that we prefer. + case `"$lt_cv_path_LD" -v 2>&1 &5 +$as_echo "$LD" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +test -z "$LD" && as_fn_error $? "no acceptable ld found in \$PATH" "$LINENO" 5 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if the linker ($LD) is GNU ld" >&5 +$as_echo_n "checking if the linker ($LD) is GNU ld... " >&6; } +if ${lt_cv_prog_gnu_ld+:} false; then : + $as_echo_n "(cached) " >&6 +else + # I'd rather use --version here, but apparently some GNU lds only accept -v. +case `$LD -v 2>&1 &5 +$as_echo "$lt_cv_prog_gnu_ld" >&6; } +with_gnu_ld=$lt_cv_prog_gnu_ld + + + + + + + + # Check if GNU C++ uses GNU ld as the underlying linker, since the + # archiving commands below assume that GNU ld is being used. + if test "$with_gnu_ld" = yes; then + archive_cmds_CXX='$CC $pic_flag -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds_CXX='$CC $pic_flag -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + + hardcode_libdir_flag_spec_CXX='${wl}-rpath ${wl}$libdir' + export_dynamic_flag_spec_CXX='${wl}--export-dynamic' + + # If archive_cmds runs LD, not CC, wlarc should be empty + # XXX I think wlarc can be eliminated in ltcf-cxx, but I need to + # investigate it a little bit more. (MM) + wlarc='${wl}' + + # ancient GNU ld didn't support --whole-archive et. al. + if eval "`$CC -print-prog-name=ld` --help 2>&1" | + $GREP 'no-whole-archive' > /dev/null; then + whole_archive_flag_spec_CXX="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive' + else + whole_archive_flag_spec_CXX= + fi + else + with_gnu_ld=no + wlarc= + + # A generic and very simple default shared library creation + # command for GNU C++ for the case where it uses the native + # linker, instead of GNU ld. If possible, this setting should + # overridden to take advantage of the native linker features on + # the platform it is being used on. + archive_cmds_CXX='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib' + fi + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + + else + GXX=no + with_gnu_ld=no + wlarc= + fi + + # PORTME: fill in a description of your system's C++ link characteristics + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the $compiler linker ($LD) supports shared libraries" >&5 +$as_echo_n "checking whether the $compiler linker ($LD) supports shared libraries... " >&6; } + ld_shlibs_CXX=yes + case $host_os in + aix3*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + aix[4-9]*) + if test "$host_cpu" = ia64; then + # On IA64, the linker does run time linking by default, so we don't + # have to do anything special. + aix_use_runtimelinking=no + exp_sym_flag='-Bexport' + no_entry_flag="" + else + aix_use_runtimelinking=no + + # Test if we are trying to use run time linking or normal + # AIX style linking. If -brtl is somewhere in LDFLAGS, we + # need to do runtime linking. + case $host_os in aix4.[23]|aix4.[23].*|aix[5-9]*) + for ld_flag in $LDFLAGS; do + case $ld_flag in + *-brtl*) + aix_use_runtimelinking=yes + break + ;; + esac + done + ;; + esac + + exp_sym_flag='-bexport' + no_entry_flag='-bnoentry' + fi + + # When large executables or shared objects are built, AIX ld can + # have problems creating the table of contents. If linking a library + # or program results in "error TOC overflow" add -mminimal-toc to + # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not + # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS. + + archive_cmds_CXX='' + hardcode_direct_CXX=yes + hardcode_direct_absolute_CXX=yes + hardcode_libdir_separator_CXX=':' + link_all_deplibs_CXX=yes + file_list_spec_CXX='${wl}-f,' + + if test "$GXX" = yes; then + case $host_os in aix4.[012]|aix4.[012].*) + # We only want to do this on AIX 4.2 and lower, the check + # below for broken collect2 doesn't work under 4.3+ + collect2name=`${CC} -print-prog-name=collect2` + if test -f "$collect2name" && + strings "$collect2name" | $GREP resolve_lib_name >/dev/null + then + # We have reworked collect2 + : + else + # We have old collect2 + hardcode_direct_CXX=unsupported + # It fails to find uninstalled libraries when the uninstalled + # path is not listed in the libpath. Setting hardcode_minus_L + # to unsupported forces relinking + hardcode_minus_L_CXX=yes + hardcode_libdir_flag_spec_CXX='-L$libdir' + hardcode_libdir_separator_CXX= + fi + esac + shared_flag='-shared' + if test "$aix_use_runtimelinking" = yes; then + shared_flag="$shared_flag "'${wl}-G' + fi + else + # not using gcc + if test "$host_cpu" = ia64; then + # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release + # chokes on -Wl,-G. The following line is correct: + shared_flag='-G' + else + if test "$aix_use_runtimelinking" = yes; then + shared_flag='${wl}-G' + else + shared_flag='${wl}-bM:SRE' + fi + fi + fi + + export_dynamic_flag_spec_CXX='${wl}-bexpall' + # It seems that -bexpall does not export symbols beginning with + # underscore (_), so it is better to generate a list of symbols to + # export. + always_export_symbols_CXX=yes + if test "$aix_use_runtimelinking" = yes; then + # Warning - without using the other runtime loading flags (-brtl), + # -berok will link without error, but may produce a broken library. + allow_undefined_flag_CXX='-berok' + # Determine the default libpath from the value encoded in an empty + # executable. + if test "${lt_cv_aix_libpath+set}" = set; then + aix_libpath=$lt_cv_aix_libpath +else + if ${lt_cv_aix_libpath__CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + + lt_aix_libpath_sed=' + /Import File Strings/,/^$/ { + /^0/ { + s/^0 *\([^ ]*\) *$/\1/ + p + } + }' + lt_cv_aix_libpath__CXX=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + # Check for a 64-bit object if we didn't find anything. + if test -z "$lt_cv_aix_libpath__CXX"; then + lt_cv_aix_libpath__CXX=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + fi +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + if test -z "$lt_cv_aix_libpath__CXX"; then + lt_cv_aix_libpath__CXX="/usr/lib:/lib" + fi + +fi + + aix_libpath=$lt_cv_aix_libpath__CXX +fi + + hardcode_libdir_flag_spec_CXX='${wl}-blibpath:$libdir:'"$aix_libpath" + + archive_expsym_cmds_CXX='$CC -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags `if test "x${allow_undefined_flag}" != "x"; then func_echo_all "${wl}${allow_undefined_flag}"; else :; fi` '"\${wl}$exp_sym_flag:\$export_symbols $shared_flag" + else + if test "$host_cpu" = ia64; then + hardcode_libdir_flag_spec_CXX='${wl}-R $libdir:/usr/lib:/lib' + allow_undefined_flag_CXX="-z nodefs" + archive_expsym_cmds_CXX="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags ${wl}${allow_undefined_flag} '"\${wl}$exp_sym_flag:\$export_symbols" + else + # Determine the default libpath from the value encoded in an + # empty executable. + if test "${lt_cv_aix_libpath+set}" = set; then + aix_libpath=$lt_cv_aix_libpath +else + if ${lt_cv_aix_libpath__CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + + lt_aix_libpath_sed=' + /Import File Strings/,/^$/ { + /^0/ { + s/^0 *\([^ ]*\) *$/\1/ + p + } + }' + lt_cv_aix_libpath__CXX=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + # Check for a 64-bit object if we didn't find anything. + if test -z "$lt_cv_aix_libpath__CXX"; then + lt_cv_aix_libpath__CXX=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + fi +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + if test -z "$lt_cv_aix_libpath__CXX"; then + lt_cv_aix_libpath__CXX="/usr/lib:/lib" + fi + +fi + + aix_libpath=$lt_cv_aix_libpath__CXX +fi + + hardcode_libdir_flag_spec_CXX='${wl}-blibpath:$libdir:'"$aix_libpath" + # Warning - without using the other run time loading flags, + # -berok will link without error, but may produce a broken library. + no_undefined_flag_CXX=' ${wl}-bernotok' + allow_undefined_flag_CXX=' ${wl}-berok' + if test "$with_gnu_ld" = yes; then + # We only use this code for GNU lds that support --whole-archive. + whole_archive_flag_spec_CXX='${wl}--whole-archive$convenience ${wl}--no-whole-archive' + else + # Exported symbols can be pulled into shared objects from archives + whole_archive_flag_spec_CXX='$convenience' + fi + archive_cmds_need_lc_CXX=yes + # This is similar to how AIX traditionally builds its shared + # libraries. + archive_expsym_cmds_CXX="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs ${wl}-bnoentry $compiler_flags ${wl}-bE:$export_symbols${allow_undefined_flag}~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$soname' + fi + fi + ;; + + beos*) + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + allow_undefined_flag_CXX=unsupported + # Joseph Beckenbach says some releases of gcc + # support --undefined. This deserves some investigation. FIXME + archive_cmds_CXX='$CC -nostart $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + else + ld_shlibs_CXX=no + fi + ;; + + chorus*) + case $cc_basename in + *) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + esac + ;; + + cygwin* | mingw* | pw32* | cegcc*) + case $GXX,$cc_basename in + ,cl* | no,cl*) + # Native MSVC + # hardcode_libdir_flag_spec is actually meaningless, as there is + # no search path for DLLs. + hardcode_libdir_flag_spec_CXX=' ' + allow_undefined_flag_CXX=unsupported + always_export_symbols_CXX=yes + file_list_spec_CXX='@' + # Tell ltmain to make .lib files, not .a files. + libext=lib + # Tell ltmain to make .dll files, not .so files. + shrext_cmds=".dll" + # FIXME: Setting linknames here is a bad hack. + archive_cmds_CXX='$CC -o $output_objdir/$soname $libobjs $compiler_flags $deplibs -Wl,-dll~linknames=' + archive_expsym_cmds_CXX='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + $SED -n -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' -e '1\\\!p' < $export_symbols > $output_objdir/$soname.exp; + else + $SED -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' < $export_symbols > $output_objdir/$soname.exp; + fi~ + $CC -o $tool_output_objdir$soname $libobjs $compiler_flags $deplibs "@$tool_output_objdir$soname.exp" -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~ + linknames=' + # The linker will not automatically build a static lib if we build a DLL. + # _LT_TAGVAR(old_archive_from_new_cmds, CXX)='true' + enable_shared_with_static_runtimes_CXX=yes + # Don't use ranlib + old_postinstall_cmds_CXX='chmod 644 $oldlib' + postlink_cmds_CXX='lt_outputfile="@OUTPUT@"~ + lt_tool_outputfile="@TOOL_OUTPUT@"~ + case $lt_outputfile in + *.exe|*.EXE) ;; + *) + lt_outputfile="$lt_outputfile.exe" + lt_tool_outputfile="$lt_tool_outputfile.exe" + ;; + esac~ + func_to_tool_file "$lt_outputfile"~ + if test "$MANIFEST_TOOL" != ":" && test -f "$lt_outputfile.manifest"; then + $MANIFEST_TOOL -manifest "$lt_tool_outputfile.manifest" -outputresource:"$lt_tool_outputfile" || exit 1; + $RM "$lt_outputfile.manifest"; + fi' + ;; + *) + # g++ + # _LT_TAGVAR(hardcode_libdir_flag_spec, CXX) is actually meaningless, + # as there is no search path for DLLs. + hardcode_libdir_flag_spec_CXX='-L$libdir' + export_dynamic_flag_spec_CXX='${wl}--export-all-symbols' + allow_undefined_flag_CXX=unsupported + always_export_symbols_CXX=no + enable_shared_with_static_runtimes_CXX=yes + + if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then + archive_cmds_CXX='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + # If the export-symbols file already is a .def file (1st line + # is EXPORTS), use it as is; otherwise, prepend... + archive_expsym_cmds_CXX='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + cp $export_symbols $output_objdir/$soname.def; + else + echo EXPORTS > $output_objdir/$soname.def; + cat $export_symbols >> $output_objdir/$soname.def; + fi~ + $CC -shared -nostdlib $output_objdir/$soname.def $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + else + ld_shlibs_CXX=no + fi + ;; + esac + ;; + darwin* | rhapsody*) + + + archive_cmds_need_lc_CXX=no + hardcode_direct_CXX=no + hardcode_automatic_CXX=yes + hardcode_shlibpath_var_CXX=unsupported + if test "$lt_cv_ld_force_load" = "yes"; then + whole_archive_flag_spec_CXX='`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience ${wl}-force_load,$conv\"; done; func_echo_all \"$new_convenience\"`' + + else + whole_archive_flag_spec_CXX='' + fi + link_all_deplibs_CXX=yes + allow_undefined_flag_CXX="$_lt_dar_allow_undefined" + case $cc_basename in + ifort*) _lt_dar_can_shared=yes ;; + *) _lt_dar_can_shared=$GCC ;; + esac + if test "$_lt_dar_can_shared" = "yes"; then + output_verbose_link_cmd=func_echo_all + archive_cmds_CXX="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod${_lt_dsymutil}" + module_cmds_CXX="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dsymutil}" + archive_expsym_cmds_CXX="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring ${_lt_dar_single_mod}${_lt_dar_export_syms}${_lt_dsymutil}" + module_expsym_cmds_CXX="sed -e 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dar_export_syms}${_lt_dsymutil}" + if test "$lt_cv_apple_cc_single_mod" != "yes"; then + archive_cmds_CXX="\$CC -r -keep_private_externs -nostdlib -o \${lib}-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \${lib}-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring${_lt_dsymutil}" + archive_expsym_cmds_CXX="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \${lib}-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \${lib}-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring${_lt_dar_export_syms}${_lt_dsymutil}" + fi + + else + ld_shlibs_CXX=no + fi + + ;; + + dgux*) + case $cc_basename in + ec++*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + ghcx*) + # Green Hills C++ Compiler + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + *) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + esac + ;; + + freebsd2.*) + # C++ shared libraries reported to be fairly broken before + # switch to ELF + ld_shlibs_CXX=no + ;; + + freebsd-elf*) + archive_cmds_need_lc_CXX=no + ;; + + freebsd* | dragonfly*) + # FreeBSD 3 and later use GNU C++ and GNU ld with standard ELF + # conventions + ld_shlibs_CXX=yes + ;; + + gnu*) + ;; + + haiku*) + archive_cmds_CXX='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + link_all_deplibs_CXX=yes + ;; + + hpux9*) + hardcode_libdir_flag_spec_CXX='${wl}+b ${wl}$libdir' + hardcode_libdir_separator_CXX=: + export_dynamic_flag_spec_CXX='${wl}-E' + hardcode_direct_CXX=yes + hardcode_minus_L_CXX=yes # Not in the search PATH, + # but as the default + # location of the library. + + case $cc_basename in + CC*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + aCC*) + archive_cmds_CXX='$RM $output_objdir/$soname~$CC -b ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $EGREP "\-L"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + ;; + *) + if test "$GXX" = yes; then + archive_cmds_CXX='$RM $output_objdir/$soname~$CC -shared -nostdlib $pic_flag ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + else + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + fi + ;; + esac + ;; + + hpux10*|hpux11*) + if test $with_gnu_ld = no; then + hardcode_libdir_flag_spec_CXX='${wl}+b ${wl}$libdir' + hardcode_libdir_separator_CXX=: + + case $host_cpu in + hppa*64*|ia64*) + ;; + *) + export_dynamic_flag_spec_CXX='${wl}-E' + ;; + esac + fi + case $host_cpu in + hppa*64*|ia64*) + hardcode_direct_CXX=no + hardcode_shlibpath_var_CXX=no + ;; + *) + hardcode_direct_CXX=yes + hardcode_direct_absolute_CXX=yes + hardcode_minus_L_CXX=yes # Not in the search PATH, + # but as the default + # location of the library. + ;; + esac + + case $cc_basename in + CC*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + aCC*) + case $host_cpu in + hppa*64*) + archive_cmds_CXX='$CC -b ${wl}+h ${wl}$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + ia64*) + archive_cmds_CXX='$CC -b ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + *) + archive_cmds_CXX='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + esac + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $GREP "\-L"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + ;; + *) + if test "$GXX" = yes; then + if test $with_gnu_ld = no; then + case $host_cpu in + hppa*64*) + archive_cmds_CXX='$CC -shared -nostdlib -fPIC ${wl}+h ${wl}$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + ia64*) + archive_cmds_CXX='$CC -shared -nostdlib $pic_flag ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + *) + archive_cmds_CXX='$CC -shared -nostdlib $pic_flag ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + esac + fi + else + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + fi + ;; + esac + ;; + + interix[3-9]*) + hardcode_direct_CXX=no + hardcode_shlibpath_var_CXX=no + hardcode_libdir_flag_spec_CXX='${wl}-rpath,$libdir' + export_dynamic_flag_spec_CXX='${wl}-E' + # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc. + # Instead, shared libraries are loaded at an image base (0x10000000 by + # default) and relocated if they conflict, which is a slow very memory + # consuming and fragmenting process. To avoid this, we pick a random, + # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link + # time. Moving up from 0x10000000 also allows more sbrk(2) space. + archive_cmds_CXX='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + archive_expsym_cmds_CXX='sed "s,^,_," $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--retain-symbols-file,$output_objdir/$soname.expsym ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + ;; + irix5* | irix6*) + case $cc_basename in + CC*) + # SGI C++ + archive_cmds_CXX='$CC -shared -all -multigot $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + + # Archives containing C++ object files must be created using + # "CC -ar", where "CC" is the IRIX C++ compiler. This is + # necessary to make sure instantiated templates are included + # in the archive. + old_archive_cmds_CXX='$CC -ar -WR,-u -o $oldlib $oldobjs' + ;; + *) + if test "$GXX" = yes; then + if test "$with_gnu_ld" = no; then + archive_cmds_CXX='$CC -shared $pic_flag -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + else + archive_cmds_CXX='$CC -shared $pic_flag -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` -o $lib' + fi + fi + link_all_deplibs_CXX=yes + ;; + esac + hardcode_libdir_flag_spec_CXX='${wl}-rpath ${wl}$libdir' + hardcode_libdir_separator_CXX=: + inherit_rpath_CXX=yes + ;; + + linux* | k*bsd*-gnu | kopensolaris*-gnu) + case $cc_basename in + KCC*) + # Kuck and Associates, Inc. (KAI) C++ Compiler + + # KCC will only create a shared library if the output file + # ends with ".so" (or ".sl" for HP-UX), so rename the library + # to its proper name (with version) after linking. + archive_cmds_CXX='tempext=`echo $shared_ext | $SED -e '\''s/\([^()0-9A-Za-z{}]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib' + archive_expsym_cmds_CXX='tempext=`echo $shared_ext | $SED -e '\''s/\([^()0-9A-Za-z{}]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib ${wl}-retain-symbols-file,$export_symbols; mv \$templib $lib' + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`$CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1 | $GREP "ld"`; rm -f libconftest$shared_ext; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + + hardcode_libdir_flag_spec_CXX='${wl}-rpath,$libdir' + export_dynamic_flag_spec_CXX='${wl}--export-dynamic' + + # Archives containing C++ object files must be created using + # "CC -Bstatic", where "CC" is the KAI C++ compiler. + old_archive_cmds_CXX='$CC -Bstatic -o $oldlib $oldobjs' + ;; + icpc* | ecpc* ) + # Intel C++ + with_gnu_ld=yes + # version 8.0 and above of icpc choke on multiply defined symbols + # if we add $predep_objects and $postdep_objects, however 7.1 and + # earlier do not add the objects themselves. + case `$CC -V 2>&1` in + *"Version 7."*) + archive_cmds_CXX='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds_CXX='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + ;; + *) # Version 8.0 or newer + tmp_idyn= + case $host_cpu in + ia64*) tmp_idyn=' -i_dynamic';; + esac + archive_cmds_CXX='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds_CXX='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + ;; + esac + archive_cmds_need_lc_CXX=no + hardcode_libdir_flag_spec_CXX='${wl}-rpath,$libdir' + export_dynamic_flag_spec_CXX='${wl}--export-dynamic' + whole_archive_flag_spec_CXX='${wl}--whole-archive$convenience ${wl}--no-whole-archive' + ;; + pgCC* | pgcpp*) + # Portland Group C++ compiler + case `$CC -V` in + *pgCC\ [1-5].* | *pgcpp\ [1-5].*) + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ + compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ + $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ + $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ + $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + esac + + hardcode_libdir_flag_spec_CXX='${wl}--rpath ${wl}$libdir' + export_dynamic_flag_spec_CXX='${wl}--export-dynamic' + whole_archive_flag_spec_CXX='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + ;; + cxx*) + # Compaq C++ + archive_cmds_CXX='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib' + archive_expsym_cmds_CXX='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib ${wl}-retain-symbols-file $wl$export_symbols' + + runpath_var=LD_RUN_PATH + hardcode_libdir_flag_spec_CXX='-rpath $libdir' + hardcode_libdir_separator_CXX=: + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld"`; templist=`func_echo_all "$templist" | $SED "s/\(^.*ld.*\)\( .*ld .*$\)/\1/"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "X$list" | $Xsed' + ;; + xl* | mpixl* | bgxl*) + # IBM XL 8.0 on PPC, with GNU ld + hardcode_libdir_flag_spec_CXX='${wl}-rpath ${wl}$libdir' + export_dynamic_flag_spec_CXX='${wl}--export-dynamic' + archive_cmds_CXX='$CC -qmkshrobj $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + if test "x$supports_anon_versioning" = xyes; then + archive_expsym_cmds_CXX='echo "{ global:" > $output_objdir/$libname.ver~ + cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + echo "local: *; };" >> $output_objdir/$libname.ver~ + $CC -qmkshrobj $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-version-script ${wl}$output_objdir/$libname.ver -o $lib' + fi + ;; + *) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) + # Sun C++ 5.9 + no_undefined_flag_CXX=' -zdefs' + archive_cmds_CXX='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + archive_expsym_cmds_CXX='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-retain-symbols-file ${wl}$export_symbols' + hardcode_libdir_flag_spec_CXX='-R$libdir' + whole_archive_flag_spec_CXX='${wl}--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + compiler_needs_object_CXX=yes + + # Not sure whether something based on + # $CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1 + # would be better. + output_verbose_link_cmd='func_echo_all' + + # Archives containing C++ object files must be created using + # "CC -xar", where "CC" is the Sun C++ compiler. This is + # necessary to make sure instantiated templates are included + # in the archive. + old_archive_cmds_CXX='$CC -xar -o $oldlib $oldobjs' + ;; + esac + ;; + esac + ;; + + lynxos*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + + m88k*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + + mvs*) + case $cc_basename in + cxx*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + *) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + esac + ;; + + netbsd*) + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + archive_cmds_CXX='$LD -Bshareable -o $lib $predep_objects $libobjs $deplibs $postdep_objects $linker_flags' + wlarc= + hardcode_libdir_flag_spec_CXX='-R$libdir' + hardcode_direct_CXX=yes + hardcode_shlibpath_var_CXX=no + fi + # Workaround some broken pre-1.5 toolchains + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP conftest.$objext | $SED -e "s:-lgcc -lc -lgcc::"' + ;; + + *nto* | *qnx*) + ld_shlibs_CXX=yes + ;; + + openbsd2*) + # C++ shared libraries are fairly broken + ld_shlibs_CXX=no + ;; + + openbsd*) + if test -f /usr/libexec/ld.so; then + hardcode_direct_CXX=yes + hardcode_shlibpath_var_CXX=no + hardcode_direct_absolute_CXX=yes + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib' + hardcode_libdir_flag_spec_CXX='${wl}-rpath,$libdir' + if test -z "`echo __ELF__ | $CC -E - | grep __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + archive_expsym_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-retain-symbols-file,$export_symbols -o $lib' + export_dynamic_flag_spec_CXX='${wl}-E' + whole_archive_flag_spec_CXX="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive' + fi + output_verbose_link_cmd=func_echo_all + else + ld_shlibs_CXX=no + fi + ;; + + osf3* | osf4* | osf5*) + case $cc_basename in + KCC*) + # Kuck and Associates, Inc. (KAI) C++ Compiler + + # KCC will only create a shared library if the output file + # ends with ".so" (or ".sl" for HP-UX), so rename the library + # to its proper name (with version) after linking. + archive_cmds_CXX='tempext=`echo $shared_ext | $SED -e '\''s/\([^()0-9A-Za-z{}]\)/\\\\\1/g'\''`; templib=`echo "$lib" | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib' + + hardcode_libdir_flag_spec_CXX='${wl}-rpath,$libdir' + hardcode_libdir_separator_CXX=: + + # Archives containing C++ object files must be created using + # the KAI C++ compiler. + case $host in + osf3*) old_archive_cmds_CXX='$CC -Bstatic -o $oldlib $oldobjs' ;; + *) old_archive_cmds_CXX='$CC -o $oldlib $oldobjs' ;; + esac + ;; + RCC*) + # Rational C++ 2.4.1 + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + cxx*) + case $host in + osf3*) + allow_undefined_flag_CXX=' ${wl}-expect_unresolved ${wl}\*' + archive_cmds_CXX='$CC -shared${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $soname `test -n "$verstring" && func_echo_all "${wl}-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + hardcode_libdir_flag_spec_CXX='${wl}-rpath ${wl}$libdir' + ;; + *) + allow_undefined_flag_CXX=' -expect_unresolved \*' + archive_cmds_CXX='$CC -shared${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + archive_expsym_cmds_CXX='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done~ + echo "-hidden">> $lib.exp~ + $CC -shared$allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname ${wl}-input ${wl}$lib.exp `test -n "$verstring" && $ECHO "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib~ + $RM $lib.exp' + hardcode_libdir_flag_spec_CXX='-rpath $libdir' + ;; + esac + + hardcode_libdir_separator_CXX=: + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld" | $GREP -v "ld:"`; templist=`func_echo_all "$templist" | $SED "s/\(^.*ld.*\)\( .*ld.*$\)/\1/"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + ;; + *) + if test "$GXX" = yes && test "$with_gnu_ld" = no; then + allow_undefined_flag_CXX=' ${wl}-expect_unresolved ${wl}\*' + case $host in + osf3*) + archive_cmds_CXX='$CC -shared -nostdlib ${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + ;; + *) + archive_cmds_CXX='$CC -shared $pic_flag -nostdlib ${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-msym ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + ;; + esac + + hardcode_libdir_flag_spec_CXX='${wl}-rpath ${wl}$libdir' + hardcode_libdir_separator_CXX=: + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + + else + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + fi + ;; + esac + ;; + + psos*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + + sunos4*) + case $cc_basename in + CC*) + # Sun C++ 4.x + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + lcc*) + # Lucid + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + *) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + esac + ;; + + solaris*) + case $cc_basename in + CC* | sunCC*) + # Sun C++ 4.2, 5.x and Centerline C++ + archive_cmds_need_lc_CXX=yes + no_undefined_flag_CXX=' -zdefs' + archive_cmds_CXX='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + archive_expsym_cmds_CXX='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -G${allow_undefined_flag} ${wl}-M ${wl}$lib.exp -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' + + hardcode_libdir_flag_spec_CXX='-R$libdir' + hardcode_shlibpath_var_CXX=no + case $host_os in + solaris2.[0-5] | solaris2.[0-5].*) ;; + *) + # The compiler driver will combine and reorder linker options, + # but understands `-z linker_flag'. + # Supported since Solaris 2.6 (maybe 2.5.1?) + whole_archive_flag_spec_CXX='-z allextract$convenience -z defaultextract' + ;; + esac + link_all_deplibs_CXX=yes + + output_verbose_link_cmd='func_echo_all' + + # Archives containing C++ object files must be created using + # "CC -xar", where "CC" is the Sun C++ compiler. This is + # necessary to make sure instantiated templates are included + # in the archive. + old_archive_cmds_CXX='$CC -xar -o $oldlib $oldobjs' + ;; + gcx*) + # Green Hills C++ Compiler + archive_cmds_CXX='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib' + + # The C++ compiler must be used to create the archive. + old_archive_cmds_CXX='$CC $LDFLAGS -archive -o $oldlib $oldobjs' + ;; + *) + # GNU C++ compiler with Solaris linker + if test "$GXX" = yes && test "$with_gnu_ld" = no; then + no_undefined_flag_CXX=' ${wl}-z ${wl}defs' + if $CC --version | $GREP -v '^2\.7' > /dev/null; then + archive_cmds_CXX='$CC -shared $pic_flag -nostdlib $LDFLAGS $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib' + archive_expsym_cmds_CXX='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -shared $pic_flag -nostdlib ${wl}-M $wl$lib.exp -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + else + # g++ 2.7 appears to require `-G' NOT `-shared' on this + # platform. + archive_cmds_CXX='$CC -G -nostdlib $LDFLAGS $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib' + archive_expsym_cmds_CXX='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -G -nostdlib ${wl}-M $wl$lib.exp -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -G $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + fi + + hardcode_libdir_flag_spec_CXX='${wl}-R $wl$libdir' + case $host_os in + solaris2.[0-5] | solaris2.[0-5].*) ;; + *) + whole_archive_flag_spec_CXX='${wl}-z ${wl}allextract$convenience ${wl}-z ${wl}defaultextract' + ;; + esac + fi + ;; + esac + ;; + + sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[01].[10]* | unixware7* | sco3.2v5.0.[024]*) + no_undefined_flag_CXX='${wl}-z,text' + archive_cmds_need_lc_CXX=no + hardcode_shlibpath_var_CXX=no + runpath_var='LD_RUN_PATH' + + case $cc_basename in + CC*) + archive_cmds_CXX='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds_CXX='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + *) + archive_cmds_CXX='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds_CXX='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + esac + ;; + + sysv5* | sco3.2v5* | sco5v6*) + # Note: We can NOT use -z defs as we might desire, because we do not + # link with -lc, and that would cause any symbols used from libc to + # always be unresolved, which means just about no library would + # ever link correctly. If we're not using GNU ld we use -z text + # though, which does catch some bad symbols but isn't as heavy-handed + # as -z defs. + no_undefined_flag_CXX='${wl}-z,text' + allow_undefined_flag_CXX='${wl}-z,nodefs' + archive_cmds_need_lc_CXX=no + hardcode_shlibpath_var_CXX=no + hardcode_libdir_flag_spec_CXX='${wl}-R,$libdir' + hardcode_libdir_separator_CXX=':' + link_all_deplibs_CXX=yes + export_dynamic_flag_spec_CXX='${wl}-Bexport' + runpath_var='LD_RUN_PATH' + + case $cc_basename in + CC*) + archive_cmds_CXX='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds_CXX='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + old_archive_cmds_CXX='$CC -Tprelink_objects $oldobjs~ + '"$old_archive_cmds_CXX" + reload_cmds_CXX='$CC -Tprelink_objects $reload_objs~ + '"$reload_cmds_CXX" + ;; + *) + archive_cmds_CXX='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + archive_expsym_cmds_CXX='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + esac + ;; + + tandem*) + case $cc_basename in + NCC*) + # NonStop-UX NCC 3.20 + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + *) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + esac + ;; + + vxworks*) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + + *) + # FIXME: insert proper C++ library support + ld_shlibs_CXX=no + ;; + esac + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ld_shlibs_CXX" >&5 +$as_echo "$ld_shlibs_CXX" >&6; } + test "$ld_shlibs_CXX" = no && can_build_shared=no + + GCC_CXX="$GXX" + LD_CXX="$LD" + + ## CAVEAT EMPTOR: + ## There is no encapsulation within the following macros, do not change + ## the running order or otherwise move them around unless you know exactly + ## what you are doing... + # Dependencies to place before and after the object being linked: +predep_objects_CXX= +postdep_objects_CXX= +predeps_CXX= +postdeps_CXX= +compiler_lib_search_path_CXX= + +cat > conftest.$ac_ext <<_LT_EOF +class Foo +{ +public: + Foo (void) { a = 0; } +private: + int a; +}; +_LT_EOF + + +_lt_libdeps_save_CFLAGS=$CFLAGS +case "$CC $CFLAGS " in #( +*\ -flto*\ *) CFLAGS="$CFLAGS -fno-lto" ;; +*\ -fwhopr*\ *) CFLAGS="$CFLAGS -fno-whopr" ;; +*\ -fuse-linker-plugin*\ *) CFLAGS="$CFLAGS -fno-use-linker-plugin" ;; +esac + +if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + # Parse the compiler output and extract the necessary + # objects, libraries and library flags. + + # Sentinel used to keep track of whether or not we are before + # the conftest object file. + pre_test_object_deps_done=no + + for p in `eval "$output_verbose_link_cmd"`; do + case ${prev}${p} in + + -L* | -R* | -l*) + # Some compilers place space between "-{L,R}" and the path. + # Remove the space. + if test $p = "-L" || + test $p = "-R"; then + prev=$p + continue + fi + + # Expand the sysroot to ease extracting the directories later. + if test -z "$prev"; then + case $p in + -L*) func_stripname_cnf '-L' '' "$p"; prev=-L; p=$func_stripname_result ;; + -R*) func_stripname_cnf '-R' '' "$p"; prev=-R; p=$func_stripname_result ;; + -l*) func_stripname_cnf '-l' '' "$p"; prev=-l; p=$func_stripname_result ;; + esac + fi + case $p in + =*) func_stripname_cnf '=' '' "$p"; p=$lt_sysroot$func_stripname_result ;; + esac + if test "$pre_test_object_deps_done" = no; then + case ${prev} in + -L | -R) + # Internal compiler library paths should come after those + # provided the user. The postdeps already come after the + # user supplied libs so there is no need to process them. + if test -z "$compiler_lib_search_path_CXX"; then + compiler_lib_search_path_CXX="${prev}${p}" + else + compiler_lib_search_path_CXX="${compiler_lib_search_path_CXX} ${prev}${p}" + fi + ;; + # The "-l" case would never come before the object being + # linked, so don't bother handling this case. + esac + else + if test -z "$postdeps_CXX"; then + postdeps_CXX="${prev}${p}" + else + postdeps_CXX="${postdeps_CXX} ${prev}${p}" + fi + fi + prev= + ;; + + *.lto.$objext) ;; # Ignore GCC LTO objects + *.$objext) + # This assumes that the test object file only shows up + # once in the compiler output. + if test "$p" = "conftest.$objext"; then + pre_test_object_deps_done=yes + continue + fi + + if test "$pre_test_object_deps_done" = no; then + if test -z "$predep_objects_CXX"; then + predep_objects_CXX="$p" + else + predep_objects_CXX="$predep_objects_CXX $p" + fi + else + if test -z "$postdep_objects_CXX"; then + postdep_objects_CXX="$p" + else + postdep_objects_CXX="$postdep_objects_CXX $p" + fi + fi + ;; + + *) ;; # Ignore the rest. + + esac + done + + # Clean up. + rm -f a.out a.exe +else + echo "libtool.m4: error: problem compiling CXX test program" +fi + +$RM -f confest.$objext +CFLAGS=$_lt_libdeps_save_CFLAGS + +# PORTME: override above test on systems where it is broken +case $host_os in +interix[3-9]*) + # Interix 3.5 installs completely hosed .la files for C++, so rather than + # hack all around it, let's just trust "g++" to DTRT. + predep_objects_CXX= + postdep_objects_CXX= + postdeps_CXX= + ;; + +linux*) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) + # Sun C++ 5.9 + + # The more standards-conforming stlport4 library is + # incompatible with the Cstd library. Avoid specifying + # it if it's in CXXFLAGS. Ignore libCrun as + # -library=stlport4 depends on it. + case " $CXX $CXXFLAGS " in + *" -library=stlport4 "*) + solaris_use_stlport4=yes + ;; + esac + + if test "$solaris_use_stlport4" != yes; then + postdeps_CXX='-library=Cstd -library=Crun' + fi + ;; + esac + ;; + +solaris*) + case $cc_basename in + CC* | sunCC*) + # The more standards-conforming stlport4 library is + # incompatible with the Cstd library. Avoid specifying + # it if it's in CXXFLAGS. Ignore libCrun as + # -library=stlport4 depends on it. + case " $CXX $CXXFLAGS " in + *" -library=stlport4 "*) + solaris_use_stlport4=yes + ;; + esac + + # Adding this requires a known-good setup of shared libraries for + # Sun compiler versions before 5.6, else PIC objects from an old + # archive will be linked into the output, leading to subtle bugs. + if test "$solaris_use_stlport4" != yes; then + postdeps_CXX='-library=Cstd -library=Crun' + fi + ;; + esac + ;; +esac + + +case " $postdeps_CXX " in +*" -lc "*) archive_cmds_need_lc_CXX=no ;; +esac + compiler_lib_search_dirs_CXX= +if test -n "${compiler_lib_search_path_CXX}"; then + compiler_lib_search_dirs_CXX=`echo " ${compiler_lib_search_path_CXX}" | ${SED} -e 's! -L! !g' -e 's!^ !!'` +fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + lt_prog_compiler_wl_CXX= +lt_prog_compiler_pic_CXX= +lt_prog_compiler_static_CXX= + + + # C++ specific cases for pic, static, wl, etc. + if test "$GXX" = yes; then + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_static_CXX='-static' + + case $host_os in + aix*) + # All AIX code is PIC. + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + lt_prog_compiler_static_CXX='-Bstatic' + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + lt_prog_compiler_pic_CXX='-fPIC' + ;; + m68k) + # FIXME: we need at least 68020 code to build shared libraries, but + # adding the `-m68020' flag to GCC prevents building anything better, + # like `-m68040'. + lt_prog_compiler_pic_CXX='-m68020 -resident32 -malways-restore-a4' + ;; + esac + ;; + + beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*) + # PIC is the default for these OSes. + ;; + mingw* | cygwin* | os2* | pw32* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + # Although the cygwin gcc ignores -fPIC, still need this for old-style + # (--disable-auto-import) libraries + lt_prog_compiler_pic_CXX='-DDLL_EXPORT' + ;; + darwin* | rhapsody*) + # PIC is the default on this platform + # Common symbols not allowed in MH_DYLIB files + lt_prog_compiler_pic_CXX='-fno-common' + ;; + *djgpp*) + # DJGPP does not support shared libraries at all + lt_prog_compiler_pic_CXX= + ;; + haiku*) + # PIC is the default for Haiku. + # The "-static" flag exists, but is broken. + lt_prog_compiler_static_CXX= + ;; + interix[3-9]*) + # Interix 3.x gcc -fpic/-fPIC options generate broken code. + # Instead, we relocate shared libraries at runtime. + ;; + sysv4*MP*) + if test -d /usr/nec; then + lt_prog_compiler_pic_CXX=-Kconform_pic + fi + ;; + hpux*) + # PIC is the default for 64-bit PA HP-UX, but not for 32-bit + # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag + # sets the default TLS model and affects inlining. + case $host_cpu in + hppa*64*) + ;; + *) + lt_prog_compiler_pic_CXX='-fPIC' + ;; + esac + ;; + *qnx* | *nto*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + lt_prog_compiler_pic_CXX='-fPIC -shared' + ;; + *) + lt_prog_compiler_pic_CXX='-fPIC' + ;; + esac + else + case $host_os in + aix[4-9]*) + # All AIX code is PIC. + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + lt_prog_compiler_static_CXX='-Bstatic' + else + lt_prog_compiler_static_CXX='-bnso -bI:/lib/syscalls.exp' + fi + ;; + chorus*) + case $cc_basename in + cxch68*) + # Green Hills C++ Compiler + # _LT_TAGVAR(lt_prog_compiler_static, CXX)="--no_auto_instantiation -u __main -u __premain -u _abort -r $COOL_DIR/lib/libOrb.a $MVME_DIR/lib/CC/libC.a $MVME_DIR/lib/classix/libcx.s.a" + ;; + esac + ;; + mingw* | cygwin* | os2* | pw32* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + lt_prog_compiler_pic_CXX='-DDLL_EXPORT' + ;; + dgux*) + case $cc_basename in + ec++*) + lt_prog_compiler_pic_CXX='-KPIC' + ;; + ghcx*) + # Green Hills C++ Compiler + lt_prog_compiler_pic_CXX='-pic' + ;; + *) + ;; + esac + ;; + freebsd* | dragonfly*) + # FreeBSD uses GNU C++ + ;; + hpux9* | hpux10* | hpux11*) + case $cc_basename in + CC*) + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_static_CXX='${wl}-a ${wl}archive' + if test "$host_cpu" != ia64; then + lt_prog_compiler_pic_CXX='+Z' + fi + ;; + aCC*) + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_static_CXX='${wl}-a ${wl}archive' + case $host_cpu in + hppa*64*|ia64*) + # +Z the default + ;; + *) + lt_prog_compiler_pic_CXX='+Z' + ;; + esac + ;; + *) + ;; + esac + ;; + interix*) + # This is c89, which is MS Visual C++ (no shared libs) + # Anyone wants to do a port? + ;; + irix5* | irix6* | nonstopux*) + case $cc_basename in + CC*) + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_static_CXX='-non_shared' + # CC pic flag -KPIC is the default. + ;; + *) + ;; + esac + ;; + linux* | k*bsd*-gnu | kopensolaris*-gnu) + case $cc_basename in + KCC*) + # KAI C++ Compiler + lt_prog_compiler_wl_CXX='--backend -Wl,' + lt_prog_compiler_pic_CXX='-fPIC' + ;; + ecpc* ) + # old Intel C++ for x86_64 which still supported -KPIC. + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_pic_CXX='-KPIC' + lt_prog_compiler_static_CXX='-static' + ;; + icpc* ) + # Intel C++, used to be incompatible with GCC. + # ICC 10 doesn't accept -KPIC any more. + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_pic_CXX='-fPIC' + lt_prog_compiler_static_CXX='-static' + ;; + pgCC* | pgcpp*) + # Portland Group C++ compiler + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_pic_CXX='-fpic' + lt_prog_compiler_static_CXX='-Bstatic' + ;; + cxx*) + # Compaq C++ + # Make sure the PIC flag is empty. It appears that all Alpha + # Linux and Compaq Tru64 Unix objects are PIC. + lt_prog_compiler_pic_CXX= + lt_prog_compiler_static_CXX='-non_shared' + ;; + xlc* | xlC* | bgxl[cC]* | mpixl[cC]*) + # IBM XL 8.0, 9.0 on PPC and BlueGene + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_pic_CXX='-qpic' + lt_prog_compiler_static_CXX='-qstaticlink' + ;; + *) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) + # Sun C++ 5.9 + lt_prog_compiler_pic_CXX='-KPIC' + lt_prog_compiler_static_CXX='-Bstatic' + lt_prog_compiler_wl_CXX='-Qoption ld ' + ;; + esac + ;; + esac + ;; + lynxos*) + ;; + m88k*) + ;; + mvs*) + case $cc_basename in + cxx*) + lt_prog_compiler_pic_CXX='-W c,exportall' + ;; + *) + ;; + esac + ;; + netbsd* | netbsdelf*-gnu) + ;; + *qnx* | *nto*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + lt_prog_compiler_pic_CXX='-fPIC -shared' + ;; + osf3* | osf4* | osf5*) + case $cc_basename in + KCC*) + lt_prog_compiler_wl_CXX='--backend -Wl,' + ;; + RCC*) + # Rational C++ 2.4.1 + lt_prog_compiler_pic_CXX='-pic' + ;; + cxx*) + # Digital/Compaq C++ + lt_prog_compiler_wl_CXX='-Wl,' + # Make sure the PIC flag is empty. It appears that all Alpha + # Linux and Compaq Tru64 Unix objects are PIC. + lt_prog_compiler_pic_CXX= + lt_prog_compiler_static_CXX='-non_shared' + ;; + *) + ;; + esac + ;; + psos*) + ;; + solaris*) + case $cc_basename in + CC* | sunCC*) + # Sun C++ 4.2, 5.x and Centerline C++ + lt_prog_compiler_pic_CXX='-KPIC' + lt_prog_compiler_static_CXX='-Bstatic' + lt_prog_compiler_wl_CXX='-Qoption ld ' + ;; + gcx*) + # Green Hills C++ Compiler + lt_prog_compiler_pic_CXX='-PIC' + ;; + *) + ;; + esac + ;; + sunos4*) + case $cc_basename in + CC*) + # Sun C++ 4.x + lt_prog_compiler_pic_CXX='-pic' + lt_prog_compiler_static_CXX='-Bstatic' + ;; + lcc*) + # Lucid + lt_prog_compiler_pic_CXX='-pic' + ;; + *) + ;; + esac + ;; + sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*) + case $cc_basename in + CC*) + lt_prog_compiler_wl_CXX='-Wl,' + lt_prog_compiler_pic_CXX='-KPIC' + lt_prog_compiler_static_CXX='-Bstatic' + ;; + esac + ;; + tandem*) + case $cc_basename in + NCC*) + # NonStop-UX NCC 3.20 + lt_prog_compiler_pic_CXX='-KPIC' + ;; + *) + ;; + esac + ;; + vxworks*) + ;; + *) + lt_prog_compiler_can_build_shared_CXX=no + ;; + esac + fi + +case $host_os in + # For platforms which do not support PIC, -DPIC is meaningless: + *djgpp*) + lt_prog_compiler_pic_CXX= + ;; + *) + lt_prog_compiler_pic_CXX="$lt_prog_compiler_pic_CXX -DPIC" + ;; +esac + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $compiler option to produce PIC" >&5 +$as_echo_n "checking for $compiler option to produce PIC... " >&6; } +if ${lt_cv_prog_compiler_pic_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_pic_CXX=$lt_prog_compiler_pic_CXX +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_pic_CXX" >&5 +$as_echo "$lt_cv_prog_compiler_pic_CXX" >&6; } +lt_prog_compiler_pic_CXX=$lt_cv_prog_compiler_pic_CXX + +# +# Check to make sure the PIC flag actually works. +# +if test -n "$lt_prog_compiler_pic_CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler PIC flag $lt_prog_compiler_pic_CXX works" >&5 +$as_echo_n "checking if $compiler PIC flag $lt_prog_compiler_pic_CXX works... " >&6; } +if ${lt_cv_prog_compiler_pic_works_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_pic_works_CXX=no + ac_outfile=conftest.$ac_objext + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + lt_compiler_flag="$lt_prog_compiler_pic_CXX -DPIC" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + # The option is referenced via a variable to avoid confusing sed. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) + (eval "$lt_compile" 2>conftest.err) + ac_status=$? + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + if (exit $ac_status) && test -s "$ac_outfile"; then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings other than the usual output. + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' >conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then + lt_cv_prog_compiler_pic_works_CXX=yes + fi + fi + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_pic_works_CXX" >&5 +$as_echo "$lt_cv_prog_compiler_pic_works_CXX" >&6; } + +if test x"$lt_cv_prog_compiler_pic_works_CXX" = xyes; then + case $lt_prog_compiler_pic_CXX in + "" | " "*) ;; + *) lt_prog_compiler_pic_CXX=" $lt_prog_compiler_pic_CXX" ;; + esac +else + lt_prog_compiler_pic_CXX= + lt_prog_compiler_can_build_shared_CXX=no +fi + +fi + + + + + +# +# Check to make sure the static flag actually works. +# +wl=$lt_prog_compiler_wl_CXX eval lt_tmp_static_flag=\"$lt_prog_compiler_static_CXX\" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler static flag $lt_tmp_static_flag works" >&5 +$as_echo_n "checking if $compiler static flag $lt_tmp_static_flag works... " >&6; } +if ${lt_cv_prog_compiler_static_works_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_static_works_CXX=no + save_LDFLAGS="$LDFLAGS" + LDFLAGS="$LDFLAGS $lt_tmp_static_flag" + echo "$lt_simple_link_test_code" > conftest.$ac_ext + if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then + # The linker can only warn and ignore the option if not recognized + # So say no if there are warnings + if test -s conftest.err; then + # Append any errors to the config.log. + cat conftest.err 1>&5 + $ECHO "$_lt_linker_boilerplate" | $SED '/^$/d' > conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if diff conftest.exp conftest.er2 >/dev/null; then + lt_cv_prog_compiler_static_works_CXX=yes + fi + else + lt_cv_prog_compiler_static_works_CXX=yes + fi + fi + $RM -r conftest* + LDFLAGS="$save_LDFLAGS" + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_static_works_CXX" >&5 +$as_echo "$lt_cv_prog_compiler_static_works_CXX" >&6; } + +if test x"$lt_cv_prog_compiler_static_works_CXX" = xyes; then + : +else + lt_prog_compiler_static_CXX= +fi + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5 +$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; } +if ${lt_cv_prog_compiler_c_o_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_c_o_CXX=no + $RM -r conftest 2>/dev/null + mkdir conftest + cd conftest + mkdir out + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + lt_compiler_flag="-o out/conftest2.$ac_objext" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) + (eval "$lt_compile" 2>out/conftest.err) + ac_status=$? + cat out/conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + if (exit $ac_status) && test -s out/conftest2.$ac_objext + then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp + $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 + if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then + lt_cv_prog_compiler_c_o_CXX=yes + fi + fi + chmod u+w . 2>&5 + $RM conftest* + # SGI C++ compiler will create directory out/ii_files/ for + # template instantiation + test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files + $RM out/* && rmdir out + cd .. + $RM -r conftest + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o_CXX" >&5 +$as_echo "$lt_cv_prog_compiler_c_o_CXX" >&6; } + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5 +$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; } +if ${lt_cv_prog_compiler_c_o_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_prog_compiler_c_o_CXX=no + $RM -r conftest 2>/dev/null + mkdir conftest + cd conftest + mkdir out + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + lt_compiler_flag="-o out/conftest2.$ac_objext" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) + (eval "$lt_compile" 2>out/conftest.err) + ac_status=$? + cat out/conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + if (exit $ac_status) && test -s out/conftest2.$ac_objext + then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp + $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 + if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then + lt_cv_prog_compiler_c_o_CXX=yes + fi + fi + chmod u+w . 2>&5 + $RM conftest* + # SGI C++ compiler will create directory out/ii_files/ for + # template instantiation + test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files + $RM out/* && rmdir out + cd .. + $RM -r conftest + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o_CXX" >&5 +$as_echo "$lt_cv_prog_compiler_c_o_CXX" >&6; } + + + + +hard_links="nottested" +if test "$lt_cv_prog_compiler_c_o_CXX" = no && test "$need_locks" != no; then + # do not overwrite the value of need_locks provided by the user + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if we can lock with hard links" >&5 +$as_echo_n "checking if we can lock with hard links... " >&6; } + hard_links=yes + $RM conftest* + ln conftest.a conftest.b 2>/dev/null && hard_links=no + touch conftest.a + ln conftest.a conftest.b 2>&5 || hard_links=no + ln conftest.a conftest.b 2>/dev/null && hard_links=no + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $hard_links" >&5 +$as_echo "$hard_links" >&6; } + if test "$hard_links" = no; then + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \`$CC' does not support \`-c -o', so \`make -j' may be unsafe" >&5 +$as_echo "$as_me: WARNING: \`$CC' does not support \`-c -o', so \`make -j' may be unsafe" >&2;} + need_locks=warn + fi +else + need_locks=no +fi + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the $compiler linker ($LD) supports shared libraries" >&5 +$as_echo_n "checking whether the $compiler linker ($LD) supports shared libraries... " >&6; } + + export_symbols_cmds_CXX='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' + exclude_expsyms_CXX='_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*' + case $host_os in + aix[4-9]*) + # If we're using GNU nm, then we don't want the "-C" option. + # -C means demangle to AIX nm, but means don't demangle with GNU nm + # Also, AIX nm treats weak defined symbols like other global defined + # symbols, whereas GNU nm marks them as "W". + if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then + export_symbols_cmds_CXX='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && (substr(\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + else + export_symbols_cmds_CXX='$NM -BCpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && (substr(\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + fi + ;; + pw32*) + export_symbols_cmds_CXX="$ltdll_cmds" + ;; + cygwin* | mingw* | cegcc*) + case $cc_basename in + cl*) + exclude_expsyms_CXX='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' + ;; + *) + export_symbols_cmds_CXX='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[BCDGRS][ ]/s/.*[ ]\([^ ]*\)/\1 DATA/;s/^.*[ ]__nm__\([^ ]*\)[ ][^ ]*/\1 DATA/;/^I[ ]/d;/^[AITW][ ]/s/.* //'\'' | sort | uniq > $export_symbols' + exclude_expsyms_CXX='[_]+GLOBAL_OFFSET_TABLE_|[_]+GLOBAL__[FID]_.*|[_]+head_[A-Za-z0-9_]+_dll|[A-Za-z0-9_]+_dll_iname' + ;; + esac + ;; + linux* | k*bsd*-gnu | gnu*) + link_all_deplibs_CXX=no + ;; + *) + export_symbols_cmds_CXX='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' + ;; + esac + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ld_shlibs_CXX" >&5 +$as_echo "$ld_shlibs_CXX" >&6; } +test "$ld_shlibs_CXX" = no && can_build_shared=no + +with_gnu_ld_CXX=$with_gnu_ld + + + + + + +# +# Do we need to explicitly link libc? +# +case "x$archive_cmds_need_lc_CXX" in +x|xyes) + # Assume -lc should be added + archive_cmds_need_lc_CXX=yes + + if test "$enable_shared" = yes && test "$GCC" = yes; then + case $archive_cmds_CXX in + *'~'*) + # FIXME: we may have to deal with multi-command sequences. + ;; + '$CC '*) + # Test whether the compiler implicitly links with -lc since on some + # systems, -lgcc has to come before -lc. If gcc already passes -lc + # to ld, don't add -lc before -lgcc. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -lc should be explicitly linked in" >&5 +$as_echo_n "checking whether -lc should be explicitly linked in... " >&6; } +if ${lt_cv_archive_cmds_need_lc_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + $RM conftest* + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } 2>conftest.err; then + soname=conftest + lib=conftest + libobjs=conftest.$ac_objext + deplibs= + wl=$lt_prog_compiler_wl_CXX + pic_flag=$lt_prog_compiler_pic_CXX + compiler_flags=-v + linker_flags=-v + verstring= + output_objdir=. + libname=conftest + lt_save_allow_undefined_flag=$allow_undefined_flag_CXX + allow_undefined_flag_CXX= + if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$archive_cmds_CXX 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1\""; } >&5 + (eval $archive_cmds_CXX 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + then + lt_cv_archive_cmds_need_lc_CXX=no + else + lt_cv_archive_cmds_need_lc_CXX=yes + fi + allow_undefined_flag_CXX=$lt_save_allow_undefined_flag + else + cat conftest.err 1>&5 + fi + $RM conftest* + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_archive_cmds_need_lc_CXX" >&5 +$as_echo "$lt_cv_archive_cmds_need_lc_CXX" >&6; } + archive_cmds_need_lc_CXX=$lt_cv_archive_cmds_need_lc_CXX + ;; + esac + fi + ;; +esac + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking dynamic linker characteristics" >&5 +$as_echo_n "checking dynamic linker characteristics... " >&6; } + +library_names_spec= +libname_spec='lib$name' +soname_spec= +shrext_cmds=".so" +postinstall_cmds= +postuninstall_cmds= +finish_cmds= +finish_eval= +shlibpath_var= +shlibpath_overrides_runpath=unknown +version_type=none +dynamic_linker="$host_os ld.so" +sys_lib_dlsearch_path_spec="/lib /usr/lib" +need_lib_prefix=unknown +hardcode_into_libs=no + +# when you set need_version to no, make sure it does not cause -set_version +# flags to be left without arguments +need_version=unknown + +case $host_os in +aix3*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix $libname.a' + shlibpath_var=LIBPATH + + # AIX 3 has no versioning support, so we append a major version to the name. + soname_spec='${libname}${release}${shared_ext}$major' + ;; + +aix[4-9]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + hardcode_into_libs=yes + if test "$host_cpu" = ia64; then + # AIX 5 supports IA64 + library_names_spec='${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext}$versuffix $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + else + # With GCC up to 2.95.x, collect2 would create an import file + # for dependence libraries. The import file would start with + # the line `#! .'. This would cause the generated library to + # depend on `.', always an invalid library. This was fixed in + # development snapshots of GCC prior to 3.0. + case $host_os in + aix4 | aix4.[01] | aix4.[01].*) + if { echo '#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 97)' + echo ' yes ' + echo '#endif'; } | ${CC} -E - | $GREP yes > /dev/null; then + : + else + can_build_shared=no + fi + ;; + esac + # AIX (on Power*) has no versioning support, so currently we can not hardcode correct + # soname into executable. Probably we can add versioning support to + # collect2, so additional links can be useful in future. + if test "$aix_use_runtimelinking" = yes; then + # If using run time linking (on AIX 4.2 or later) use lib.so + # instead of lib.a to let people know that these are not + # typical AIX shared libraries. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + else + # We preserve .a as extension for shared libraries through AIX4.2 + # and later when we are not doing run time linking. + library_names_spec='${libname}${release}.a $libname.a' + soname_spec='${libname}${release}${shared_ext}$major' + fi + shlibpath_var=LIBPATH + fi + ;; + +amigaos*) + case $host_cpu in + powerpc) + # Since July 2007 AmigaOS4 officially supports .so libraries. + # When compiling the executable, add -use-dynld -Lsobjs: to the compileline. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + ;; + m68k) + library_names_spec='$libname.ixlibrary $libname.a' + # Create ${libname}_ixlibrary.a entries in /sys/libs. + finish_eval='for lib in `ls $libdir/*.ixlibrary 2>/dev/null`; do libname=`func_echo_all "$lib" | $SED '\''s%^.*/\([^/]*\)\.ixlibrary$%\1%'\''`; test $RM /sys/libs/${libname}_ixlibrary.a; $show "cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a"; cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a || exit 1; done' + ;; + esac + ;; + +beos*) + library_names_spec='${libname}${shared_ext}' + dynamic_linker="$host_os ld.so" + shlibpath_var=LIBRARY_PATH + ;; + +bsdi[45]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + finish_cmds='PATH="\$PATH:/sbin" ldconfig $libdir' + shlibpath_var=LD_LIBRARY_PATH + sys_lib_search_path_spec="/shlib /usr/lib /usr/X11/lib /usr/contrib/lib /lib /usr/local/lib" + sys_lib_dlsearch_path_spec="/shlib /usr/lib /usr/local/lib" + # the default ld.so.conf also contains /usr/contrib/lib and + # /usr/X11R6/lib (/usr/X11 is a link to /usr/X11R6), but let us allow + # libtool to hard-code these into programs + ;; + +cygwin* | mingw* | pw32* | cegcc*) + version_type=windows + shrext_cmds=".dll" + need_version=no + need_lib_prefix=no + + case $GCC,$cc_basename in + yes,*) + # gcc + library_names_spec='$libname.dll.a' + # DLL is installed to $(libdir)/../bin by postinstall_cmds + postinstall_cmds='base_file=`basename \${file}`~ + dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~ + dldir=$destdir/`dirname \$dlpath`~ + test -d \$dldir || mkdir -p \$dldir~ + $install_prog $dir/$dlname \$dldir/$dlname~ + chmod a+x \$dldir/$dlname~ + if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then + eval '\''$striplib \$dldir/$dlname'\'' || exit \$?; + fi' + postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ + dlpath=$dir/\$dldll~ + $RM \$dlpath' + shlibpath_overrides_runpath=yes + + case $host_os in + cygwin*) + # Cygwin DLLs use 'cyg' prefix rather than 'lib' + soname_spec='`echo ${libname} | sed -e 's/^lib/cyg/'``echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + + ;; + mingw* | cegcc*) + # MinGW DLLs use traditional 'lib' prefix + soname_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + ;; + pw32*) + # pw32 DLLs use 'pw' prefix rather than 'lib' + library_names_spec='`echo ${libname} | sed -e 's/^lib/pw/'``echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + ;; + esac + dynamic_linker='Win32 ld.exe' + ;; + + *,cl*) + # Native MSVC + libname_spec='$name' + soname_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}' + library_names_spec='${libname}.dll.lib' + + case $build_os in + mingw*) + sys_lib_search_path_spec= + lt_save_ifs=$IFS + IFS=';' + for lt_path in $LIB + do + IFS=$lt_save_ifs + # Let DOS variable expansion print the short 8.3 style file name. + lt_path=`cd "$lt_path" 2>/dev/null && cmd //C "for %i in (".") do @echo %~si"` + sys_lib_search_path_spec="$sys_lib_search_path_spec $lt_path" + done + IFS=$lt_save_ifs + # Convert to MSYS style. + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([a-zA-Z]\\):| /\\1|g' -e 's|^ ||'` + ;; + cygwin*) + # Convert to unix form, then to dos form, then back to unix form + # but this time dos style (no spaces!) so that the unix form looks + # like /cygdrive/c/PROGRA~1:/cygdr... + sys_lib_search_path_spec=`cygpath --path --unix "$LIB"` + sys_lib_search_path_spec=`cygpath --path --dos "$sys_lib_search_path_spec" 2>/dev/null` + sys_lib_search_path_spec=`cygpath --path --unix "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` + ;; + *) + sys_lib_search_path_spec="$LIB" + if $ECHO "$sys_lib_search_path_spec" | $GREP ';[c-zC-Z]:/' >/dev/null; then + # It is most probably a Windows format PATH. + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's/;/ /g'` + else + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` + fi + # FIXME: find the short name or the path components, as spaces are + # common. (e.g. "Program Files" -> "PROGRA~1") + ;; + esac + + # DLL is installed to $(libdir)/../bin by postinstall_cmds + postinstall_cmds='base_file=`basename \${file}`~ + dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~ + dldir=$destdir/`dirname \$dlpath`~ + test -d \$dldir || mkdir -p \$dldir~ + $install_prog $dir/$dlname \$dldir/$dlname' + postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ + dlpath=$dir/\$dldll~ + $RM \$dlpath' + shlibpath_overrides_runpath=yes + dynamic_linker='Win32 link.exe' + ;; + + *) + # Assume MSVC wrapper + library_names_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext} $libname.lib' + dynamic_linker='Win32 ld.exe' + ;; + esac + # FIXME: first we should search . and the directory the executable is in + shlibpath_var=PATH + ;; + +darwin* | rhapsody*) + dynamic_linker="$host_os dyld" + version_type=darwin + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${major}$shared_ext ${libname}$shared_ext' + soname_spec='${libname}${release}${major}$shared_ext' + shlibpath_overrides_runpath=yes + shlibpath_var=DYLD_LIBRARY_PATH + shrext_cmds='`test .$module = .yes && echo .so || echo .dylib`' + + sys_lib_dlsearch_path_spec='/usr/local/lib /lib /usr/lib' + ;; + +dgux*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname$shared_ext' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + ;; + +freebsd* | dragonfly*) + # DragonFly does not have aout. When/if they implement a new + # versioning mechanism, adjust this. + if test -x /usr/bin/objformat; then + objformat=`/usr/bin/objformat` + else + case $host_os in + freebsd[23].*) objformat=aout ;; + *) objformat=elf ;; + esac + fi + version_type=freebsd-$objformat + case $version_type in + freebsd-elf*) + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}' + need_version=no + need_lib_prefix=no + ;; + freebsd-*) + library_names_spec='${libname}${release}${shared_ext}$versuffix $libname${shared_ext}$versuffix' + need_version=yes + ;; + esac + shlibpath_var=LD_LIBRARY_PATH + case $host_os in + freebsd2.*) + shlibpath_overrides_runpath=yes + ;; + freebsd3.[01]* | freebsdelf3.[01]*) + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + freebsd3.[2-9]* | freebsdelf3.[2-9]* | \ + freebsd4.[0-5] | freebsdelf4.[0-5] | freebsd4.1.1 | freebsdelf4.1.1) + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + *) # from 4.6 on, and DragonFly + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + esac + ;; + +gnu*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +haiku*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + dynamic_linker="$host_os runtime_loader" + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LIBRARY_PATH + shlibpath_overrides_runpath=yes + sys_lib_dlsearch_path_spec='/boot/home/config/lib /boot/common/lib /boot/system/lib' + hardcode_into_libs=yes + ;; + +hpux9* | hpux10* | hpux11*) + # Give a soname corresponding to the major version so that dld.sl refuses to + # link against other versions. + version_type=sunos + need_lib_prefix=no + need_version=no + case $host_cpu in + ia64*) + shrext_cmds='.so' + hardcode_into_libs=yes + dynamic_linker="$host_os dld.so" + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + if test "X$HPUX_IA64_MODE" = X32; then + sys_lib_search_path_spec="/usr/lib/hpux32 /usr/local/lib/hpux32 /usr/local/lib" + else + sys_lib_search_path_spec="/usr/lib/hpux64 /usr/local/lib/hpux64" + fi + sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec + ;; + hppa*64*) + shrext_cmds='.sl' + hardcode_into_libs=yes + dynamic_linker="$host_os dld.sl" + shlibpath_var=LD_LIBRARY_PATH # How should we handle SHLIB_PATH + shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + sys_lib_search_path_spec="/usr/lib/pa20_64 /usr/ccs/lib/pa20_64" + sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec + ;; + *) + shrext_cmds='.sl' + dynamic_linker="$host_os dld.sl" + shlibpath_var=SHLIB_PATH + shlibpath_overrides_runpath=no # +s is required to enable SHLIB_PATH + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + ;; + esac + # HP-UX runs *really* slowly unless shared libraries are mode 555, ... + postinstall_cmds='chmod 555 $lib' + # or fails outright, so override atomically: + install_override_mode=555 + ;; + +interix[3-9]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + dynamic_linker='Interix 3.x ld.so.1 (PE, like ELF)' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +irix5* | irix6* | nonstopux*) + case $host_os in + nonstopux*) version_type=nonstopux ;; + *) + if test "$lt_cv_prog_gnu_ld" = yes; then + version_type=linux # correct to gnu/linux during the next big refactor + else + version_type=irix + fi ;; + esac + need_lib_prefix=no + need_version=no + soname_spec='${libname}${release}${shared_ext}$major' + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext} $libname${shared_ext}' + case $host_os in + irix5* | nonstopux*) + libsuff= shlibsuff= + ;; + *) + case $LD in # libtool.m4 will add one of these switches to LD + *-32|*"-32 "|*-melf32bsmip|*"-melf32bsmip ") + libsuff= shlibsuff= libmagic=32-bit;; + *-n32|*"-n32 "|*-melf32bmipn32|*"-melf32bmipn32 ") + libsuff=32 shlibsuff=N32 libmagic=N32;; + *-64|*"-64 "|*-melf64bmip|*"-melf64bmip ") + libsuff=64 shlibsuff=64 libmagic=64-bit;; + *) libsuff= shlibsuff= libmagic=never-match;; + esac + ;; + esac + shlibpath_var=LD_LIBRARY${shlibsuff}_PATH + shlibpath_overrides_runpath=no + sys_lib_search_path_spec="/usr/lib${libsuff} /lib${libsuff} /usr/local/lib${libsuff}" + sys_lib_dlsearch_path_spec="/usr/lib${libsuff} /lib${libsuff}" + hardcode_into_libs=yes + ;; + +# No shared lib support for Linux oldld, aout, or coff. +linux*oldld* | linux*aout* | linux*coff*) + dynamic_linker=no + ;; + +# This must be glibc/ELF. +linux* | k*bsd*-gnu | kopensolaris*-gnu) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -n $libdir' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + + # Some binutils ld are patched to set DT_RUNPATH + if ${lt_cv_shlibpath_overrides_runpath+:} false; then : + $as_echo_n "(cached) " >&6 +else + lt_cv_shlibpath_overrides_runpath=no + save_LDFLAGS=$LDFLAGS + save_libdir=$libdir + eval "libdir=/foo; wl=\"$lt_prog_compiler_wl_CXX\"; \ + LDFLAGS=\"\$LDFLAGS $hardcode_libdir_flag_spec_CXX\"" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + if ($OBJDUMP -p conftest$ac_exeext) 2>/dev/null | grep "RUNPATH.*$libdir" >/dev/null; then : + lt_cv_shlibpath_overrides_runpath=yes +fi +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LDFLAGS=$save_LDFLAGS + libdir=$save_libdir + +fi + + shlibpath_overrides_runpath=$lt_cv_shlibpath_overrides_runpath + + # This implies no fast_install, which is unacceptable. + # Some rework will be needed to allow for fast_install + # before this can be enabled. + hardcode_into_libs=yes + + # Append ld.so.conf contents to the search path + if test -f /etc/ld.so.conf; then + lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \$2)); skip = 1; } { if (!skip) print \$0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;/^[ ]*hwcap[ ]/d;s/[:, ]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;s/"//g;/^$/d' | tr '\n' ' '` + sys_lib_dlsearch_path_spec="/lib /usr/lib $lt_ld_extra" + fi + + # We used to test for /lib/ld.so.1 and disable shared libraries on + # powerpc, because MkLinux only supported shared libraries with the + # GNU dynamic linker. Since this was broken with cross compilers, + # most powerpc-linux boxes support dynamic linking these days and + # people can always --disable-shared, the test was removed, and we + # assume the GNU/Linux dynamic linker is in use. + dynamic_linker='GNU/Linux ld.so' + ;; + +netbsdelf*-gnu) + version_type=linux + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + dynamic_linker='NetBSD ld.elf_so' + ;; + +netbsd*) + version_type=sunos + need_lib_prefix=no + need_version=no + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' + dynamic_linker='NetBSD (a.out) ld.so' + else + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + dynamic_linker='NetBSD ld.elf_so' + fi + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + +newsos6) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + ;; + +*nto* | *qnx*) + version_type=qnx + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + dynamic_linker='ldqnx.so' + ;; + +openbsd*) + version_type=sunos + sys_lib_dlsearch_path_spec="/usr/lib" + need_lib_prefix=no + # Some older versions of OpenBSD (3.3 at least) *do* need versioned libs. + case $host_os in + openbsd3.3 | openbsd3.3.*) need_version=yes ;; + *) need_version=no ;; + esac + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' + shlibpath_var=LD_LIBRARY_PATH + if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + case $host_os in + openbsd2.[89] | openbsd2.[89].*) + shlibpath_overrides_runpath=no + ;; + *) + shlibpath_overrides_runpath=yes + ;; + esac + else + shlibpath_overrides_runpath=yes + fi + ;; + +os2*) + libname_spec='$name' + shrext_cmds=".dll" + need_lib_prefix=no + library_names_spec='$libname${shared_ext} $libname.a' + dynamic_linker='OS/2 ld.exe' + shlibpath_var=LIBPATH + ;; + +osf3* | osf4* | osf5*) + version_type=osf + need_lib_prefix=no + need_version=no + soname_spec='${libname}${release}${shared_ext}$major' + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + sys_lib_search_path_spec="/usr/shlib /usr/ccs/lib /usr/lib/cmplrs/cc /usr/lib /usr/local/lib /var/shlib" + sys_lib_dlsearch_path_spec="$sys_lib_search_path_spec" + ;; + +rdos*) + dynamic_linker=no + ;; + +solaris*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + # ldd complains unless libraries are executable + postinstall_cmds='chmod +x $lib' + ;; + +sunos4*) + version_type=sunos + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/usr/etc" ldconfig $libdir' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + if test "$with_gnu_ld" = yes; then + need_lib_prefix=no + fi + need_version=yes + ;; + +sysv4 | sysv4.3*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + case $host_vendor in + sni) + shlibpath_overrides_runpath=no + need_lib_prefix=no + runpath_var=LD_RUN_PATH + ;; + siemens) + need_lib_prefix=no + ;; + motorola) + need_lib_prefix=no + need_version=no + shlibpath_overrides_runpath=no + sys_lib_search_path_spec='/lib /usr/lib /usr/ccs/lib' + ;; + esac + ;; + +sysv4*MP*) + if test -d /usr/nec ;then + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='$libname${shared_ext}.$versuffix $libname${shared_ext}.$major $libname${shared_ext}' + soname_spec='$libname${shared_ext}.$major' + shlibpath_var=LD_LIBRARY_PATH + fi + ;; + +sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) + version_type=freebsd-elf + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + if test "$with_gnu_ld" = yes; then + sys_lib_search_path_spec='/usr/local/lib /usr/gnu/lib /usr/ccs/lib /usr/lib /lib' + else + sys_lib_search_path_spec='/usr/ccs/lib /usr/lib' + case $host_os in + sco3.2v5*) + sys_lib_search_path_spec="$sys_lib_search_path_spec /lib" + ;; + esac + fi + sys_lib_dlsearch_path_spec='/usr/lib' + ;; + +tpf*) + # TPF is a cross-target only. Preferred cross-host = GNU/Linux. + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +uts4*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + ;; + +*) + dynamic_linker=no + ;; +esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $dynamic_linker" >&5 +$as_echo "$dynamic_linker" >&6; } +test "$dynamic_linker" = no && can_build_shared=no + +variables_saved_for_relink="PATH $shlibpath_var $runpath_var" +if test "$GCC" = yes; then + variables_saved_for_relink="$variables_saved_for_relink GCC_EXEC_PREFIX COMPILER_PATH LIBRARY_PATH" +fi + +if test "${lt_cv_sys_lib_search_path_spec+set}" = set; then + sys_lib_search_path_spec="$lt_cv_sys_lib_search_path_spec" +fi +if test "${lt_cv_sys_lib_dlsearch_path_spec+set}" = set; then + sys_lib_dlsearch_path_spec="$lt_cv_sys_lib_dlsearch_path_spec" +fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking how to hardcode library paths into programs" >&5 +$as_echo_n "checking how to hardcode library paths into programs... " >&6; } +hardcode_action_CXX= +if test -n "$hardcode_libdir_flag_spec_CXX" || + test -n "$runpath_var_CXX" || + test "X$hardcode_automatic_CXX" = "Xyes" ; then + + # We can hardcode non-existent directories. + if test "$hardcode_direct_CXX" != no && + # If the only mechanism to avoid hardcoding is shlibpath_var, we + # have to relink, otherwise we might link with an installed library + # when we should be linking with a yet-to-be-installed one + ## test "$_LT_TAGVAR(hardcode_shlibpath_var, CXX)" != no && + test "$hardcode_minus_L_CXX" != no; then + # Linking always hardcodes the temporary library directory. + hardcode_action_CXX=relink + else + # We can link without hardcoding, and we can hardcode nonexisting dirs. + hardcode_action_CXX=immediate + fi +else + # We cannot hardcode anything, or else we can only hardcode existing + # directories. + hardcode_action_CXX=unsupported +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $hardcode_action_CXX" >&5 +$as_echo "$hardcode_action_CXX" >&6; } + +if test "$hardcode_action_CXX" = relink || + test "$inherit_rpath_CXX" = yes; then + # Fast installation is not supported + enable_fast_install=no +elif test "$shlibpath_overrides_runpath" = yes || + test "$enable_shared" = no; then + # Fast installation is not necessary + enable_fast_install=needless +fi + + + + + + + + fi # test -n "$compiler" + + CC=$lt_save_CC + CFLAGS=$lt_save_CFLAGS + LDCXX=$LD + LD=$lt_save_LD + GCC=$lt_save_GCC + with_gnu_ld=$lt_save_with_gnu_ld + lt_cv_path_LDCXX=$lt_cv_path_LD + lt_cv_path_LD=$lt_save_path_LD + lt_cv_prog_gnu_ldcxx=$lt_cv_prog_gnu_ld + lt_cv_prog_gnu_ld=$lt_save_with_gnu_ld +fi # test "$_lt_caught_CXX_error" != yes + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +depcc="$CXX" am_compiler_list= + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5 +$as_echo_n "checking dependency style of $depcc... " >&6; } +if ${am_cv_CXX_dependencies_compiler_type+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + rm -rf conftest.dir + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CXX_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + am__universal=false + case " $depcc " in #( + *\ -arch\ *\ -arch\ *) am__universal=true ;; + esac + + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. Also, some Intel + # versions had trouble with output in subdirs + am__obj=sub/conftest.${OBJEXT-o} + am__minus_obj="-o $am__obj" + case $depmode in + gcc) + # This depmode causes a compiler race in universal mode. + test "$am__universal" = false || continue + ;; + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + msvc7 | msvc7msys | msvisualcpp | msvcmsys) + # This compiler won't grok `-c -o', but also, the minuso test has + # not run yet. These depmodes are late enough in the game, and + # so weak that their functioning should not be impacted. + am__obj=conftest.${OBJEXT-o} + am__minus_obj= + ;; + none) break ;; + esac + if depmode=$depmode \ + source=sub/conftest.c object=$am__obj \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep $am__obj sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CXX_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CXX_dependencies_compiler_type=none +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CXX_dependencies_compiler_type" >&5 +$as_echo "$am_cv_CXX_dependencies_compiler_type" >&6; } +CXXDEPMODE=depmode=$am_cv_CXX_dependencies_compiler_type + + if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CXX_dependencies_compiler_type" = gcc3; then + am__fastdepCXX_TRUE= + am__fastdepCXX_FALSE='#' +else + am__fastdepCXX_TRUE='#' + am__fastdepCXX_FALSE= +fi + + +if test "$CXX" = no || test "$CXX:$GXX" = "g++:"; then + as_fn_error $? "no C++ compiler found" "$LINENO" 5 +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if compiler is clang" >&5 +$as_echo_n "checking if compiler is clang... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + +#ifndef __clang__ +#error "Not Clang" +#endif +return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + CLANG=yes +else + CLANG=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CLANG" >&5 +$as_echo "$CLANG" >&6; } + if test "$CLANG" = "yes"; then + CLANG_TRUE= + CLANG_FALSE='#' +else + CLANG_TRUE='#' + CLANG_FALSE= +fi + + +#AC_PROG_CC +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${MAKE-make} sets \$(MAKE)" >&5 +$as_echo_n "checking whether ${MAKE-make} sets \$(MAKE)... " >&6; } +set x ${MAKE-make} +ac_make=`$as_echo "$2" | sed 's/+/p/g; s/[^a-zA-Z0-9_]/_/g'` +if eval \${ac_cv_prog_make_${ac_make}_set+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat >conftest.make <<\_ACEOF +SHELL = /bin/sh +all: + @echo '@@@%%%=$(MAKE)=@@@%%%' +_ACEOF +# GNU make sometimes prints "make[1]: Entering ...", which would confuse us. +case `${MAKE-make} -f conftest.make 2>/dev/null` in + *@@@%%%=?*=@@@%%%*) + eval ac_cv_prog_make_${ac_make}_set=yes;; + *) + eval ac_cv_prog_make_${ac_make}_set=no;; +esac +rm -f conftest.make +fi +if eval test \$ac_cv_prog_make_${ac_make}_set = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + SET_MAKE= +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + SET_MAKE="MAKE=${MAKE-make}" +fi + + + +# Compiler flags + + +AM_CXXFLAGS="${AM_CXXFLAGS}" + +# Check for yasm +if yasm -f elf64 src/common/crc32c_intel_fast_asm.S -o /dev/null; then + echo 'we have a modern and working yasm' + if test `arch` = "x86_64"; then + echo 'we are x86_64' + +$as_echo "#define HAVE_GOOD_YASM_ELF64 1" >>confdefs.h + + with_good_yasm=yes + fi +else + echo 'we do not have a modern/working yasm' +fi + if test "$with_good_yasm" = "yes"; then + WITH_GOOD_YASM_ELF64_TRUE= + WITH_GOOD_YASM_ELF64_FALSE='#' +else + WITH_GOOD_YASM_ELF64_TRUE='#' + WITH_GOOD_YASM_ELF64_FALSE= +fi + + +# Checks for compiler warning types + +# AC_CHECK_CC_FLAG(FLAG_TO_TEST, VARIABLE_TO_SET_IF_SUPPORTED) +# --------- + + +{ + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + my_cflags_save="$CFLAGS" + CFLAGS="$my_cflags_save -Wtype-limits" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -Wtype-limits" >&5 +$as_echo_n "checking whether $CC accepts -Wtype-limits... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; }; WARN_TYPE_LIMITS="-Wtype-limits" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS="$my_cflags_save" + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +} +{ + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + my_cflags_save="$CFLAGS" + CFLAGS="$my_cflags_save -Wignored-qualifiers" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -Wignored-qualifiers" >&5 +$as_echo_n "checking whether $CC accepts -Wignored-qualifiers... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; }; WARN_IGNORED_QUALIFIERS="-Wignored-qualifiers" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS="$my_cflags_save" + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +} + +# Checks for architecture stuff + if case $target_cpu in arm*) true;; *) false;; esac; then + ENABLE_FPU_NEON_TRUE= + ENABLE_FPU_NEON_FALSE='#' +else + ENABLE_FPU_NEON_TRUE='#' + ENABLE_FPU_NEON_FALSE= +fi + + +# Check for compiler VTA support +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fvar-tracking-assignments" >&5 +$as_echo_n "checking whether C compiler accepts -fvar-tracking-assignments... " >&6; } +if ${ax_cv_check_cflags___fvar_tracking_assignments+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -fvar-tracking-assignments" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___fvar_tracking_assignments=yes +else + ax_cv_check_cflags___fvar_tracking_assignments=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fvar_tracking_assignments" >&5 +$as_echo "$ax_cv_check_cflags___fvar_tracking_assignments" >&6; } +if test x"$ax_cv_check_cflags___fvar_tracking_assignments" = xyes; then : + HAS_VTA_SUPPORT=1 +else + HAS_VTA_SUPPORT=0 +fi + + if test "$HAS_VTA_SUPPORT" = 1; then + COMPILER_HAS_VTA_TRUE= + COMPILER_HAS_VTA_FALSE='#' +else + COMPILER_HAS_VTA_TRUE='#' + COMPILER_HAS_VTA_FALSE= +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports static_cast<>" >&5 +$as_echo_n "checking whether the compiler supports static_cast<>... " >&6; } +if ${ax_cv_cxx_static_cast+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +class Base { public : Base () {} virtual void f () = 0; }; +class Derived : public Base { public : Derived () {} virtual void f () {} }; +int g (Derived&) { return 0; } +int +main () +{ + +Derived d; Base& b = d; Derived& s = static_cast (b); return g (s); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ax_cv_cxx_static_cast=yes +else + ax_cv_cxx_static_cast=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_cxx_static_cast" >&5 +$as_echo "$ax_cv_cxx_static_cast" >&6; } +if test "$ax_cv_cxx_static_cast" = yes; then + +$as_echo "#define HAVE_STATIC_CAST /**/" >>confdefs.h + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC recognizes __func__" >&5 +$as_echo_n "checking whether $CC recognizes __func__... " >&6; } +if ${ac_cv_c_var_func+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + +char *s = __func__; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_FUNC /**/" >>confdefs.h + ac_cv_c_var_func=yes +else + ac_cv_c_var_func=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_var_func" >&5 +$as_echo "$ac_cv_c_var_func" >&6; } + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC recognizes __PRETTY_FUNCTION__" >&5 +$as_echo_n "checking whether $CC recognizes __PRETTY_FUNCTION__... " >&6; } +if ${ac_cv_c_pretty_func+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + +char *s = __PRETTY_FUNCTION__; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_PRETTY_FUNC /**/" >>confdefs.h + ac_cv_c_pretty_func=yes +else + ac_cv_c_pretty_func=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_pretty_func" >&5 +$as_echo "$ac_cv_c_pretty_func" >&6; } + + +# Checks for libraries. + + + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +acx_pthread_ok=no + +# We used to check for pthread.h first, but this fails if pthread.h +# requires special compiler flags (e.g. on True64 or Sequent). +# It gets checked for in the link test anyway. + +# First of all, check if the user has set any of the PTHREAD_LIBS, +# etcetera environment variables, and if threads linking works using +# them: +if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS" >&5 +$as_echo_n "checking for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_join (); +int +main () +{ +return pthread_join (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + acx_pthread_ok=yes +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $acx_pthread_ok" >&5 +$as_echo "$acx_pthread_ok" >&6; } + if test x"$acx_pthread_ok" = xno; then + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" + fi + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" +fi + +# We must check for the threads library under a number of different +# names; the ordering is very important because some systems +# (e.g. DEC) have both -lpthread and -lpthreads, where one of the +# libraries is broken (non-POSIX). + +# Create a list of thread flags to try. Items starting with a "-" are +# C compiler flags, and other items are library names, except for "none" +# which indicates that we try without any flags at all, and "pthread-config" +# which is a program returning the flags for the Pth emulation library. + +acx_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" + +# The ordering *is* (sometimes) important. Some notes on the +# individual items follow: + +# pthreads: AIX (must check this before -lpthread) +# none: in case threads are in libc; should be tried before -Kthread and +# other compiler flags to prevent continual compiler warnings +# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) +# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) +# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) +# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads) +# -pthreads: Solaris/gcc +# -mthreads: Mingw32/gcc, Lynx/gcc +# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it +# doesn't hurt to check since this sometimes defines pthreads too; +# also defines -D_REENTRANT) +# ... -mt is also the pthreads flag for HP/aCC +# pthread: Linux, etcetera +# --thread-safe: KAI C++ +# pthread-config: use pthread-config program (for GNU Pth library) + +case "${host_cpu}-${host_os}" in + *solaris*) + + # On Solaris (at least, for some versions), libc contains stubbed + # (non-functional) versions of the pthreads routines, so link-based + # tests will erroneously succeed. (We need to link with -pthreads/-mt/ + # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather + # a function called by this macro, so we could check for that, but + # who knows whether they'll stub that too in a future libc.) So, + # we'll just look for -pthreads and -lpthread first: + + acx_pthread_flags="-pthreads pthread -mt -pthread $acx_pthread_flags" + ;; +esac + +if test x"$acx_pthread_ok" = xno; then +for flag in $acx_pthread_flags; do + + case $flag in + none) + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether pthreads work without any flags" >&5 +$as_echo_n "checking whether pthreads work without any flags... " >&6; } + ;; + + -*) + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether pthreads work with $flag" >&5 +$as_echo_n "checking whether pthreads work with $flag... " >&6; } + PTHREAD_CFLAGS="$flag" + ;; + + pthread-config) + # Extract the first word of "pthread-config", so it can be a program name with args. +set dummy pthread-config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_acx_pthread_config+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$acx_pthread_config"; then + ac_cv_prog_acx_pthread_config="$acx_pthread_config" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_acx_pthread_config="yes" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_acx_pthread_config" && ac_cv_prog_acx_pthread_config="no" +fi +fi +acx_pthread_config=$ac_cv_prog_acx_pthread_config +if test -n "$acx_pthread_config"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $acx_pthread_config" >&5 +$as_echo "$acx_pthread_config" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + if test x"$acx_pthread_config" = xno; then continue; fi + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; + + *) + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for the pthreads library -l$flag" >&5 +$as_echo_n "checking for the pthreads library -l$flag... " >&6; } + PTHREAD_LIBS="-l$flag" + ;; + esac + + save_LIBS="$LIBS" + save_CFLAGS="$CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Check for various functions. We must include pthread.h, + # since some functions may be macros. (On the Sequent, we + # need a special flag -Kthread to make this header compile.) + # We check for pthread_join because it is in -lpthread on IRIX + # while pthread_create is in libc. We check for pthread_attr_init + # due to DEC craziness with -lpthreads. We check for + # pthread_cleanup_push because it is one of the few pthread + # functions on Solaris that doesn't have a non-functional libc stub. + # We try pthread_create on general principles. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + acx_pthread_ok=yes +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $acx_pthread_ok" >&5 +$as_echo "$acx_pthread_ok" >&6; } + if test "x$acx_pthread_ok" = xyes; then + break; + fi + + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" +done +fi + +# Various other checks: +if test "x$acx_pthread_ok" = xyes; then + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for joinable pthread attribute" >&5 +$as_echo_n "checking for joinable pthread attribute... " >&6; } + attr_name=unknown + for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +int attr=$attr; return attr; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + attr_name=$attr; break +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + done + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $attr_name" >&5 +$as_echo "$attr_name" >&6; } + if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then + +cat >>confdefs.h <<_ACEOF +#define PTHREAD_CREATE_JOINABLE $attr_name +_ACEOF + + fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if more special flags are required for pthreads" >&5 +$as_echo_n "checking if more special flags are required for pthreads... " >&6; } + flag=no + case "${host_cpu}-${host_os}" in + *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";; + *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";; + esac + { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${flag}" >&5 +$as_echo "${flag}" >&6; } + if test "x$flag" != xno; then + PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS" + fi + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + # More AIX lossage: must compile with xlc_r or cc_r + if test x"$GCC" != xyes; then + for ac_prog in xlc_r cc_r +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_PTHREAD_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$PTHREAD_CC"; then + ac_cv_prog_PTHREAD_CC="$PTHREAD_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_PTHREAD_CC="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +PTHREAD_CC=$ac_cv_prog_PTHREAD_CC +if test -n "$PTHREAD_CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PTHREAD_CC" >&5 +$as_echo "$PTHREAD_CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$PTHREAD_CC" && break +done +test -n "$PTHREAD_CC" || PTHREAD_CC="${CC}" + + else + PTHREAD_CC=$CC + fi + + # The next part tries to detect GCC inconsistency with -shared on some + # architectures and systems. The problem is that in certain + # configurations, when -shared is specified, GCC "forgets" to + # internally use various flags which are still necessary. + + # + # Prepare the flags + # + save_CFLAGS="$CFLAGS" + save_LIBS="$LIBS" + save_CC="$CC" + + # Try with the flags determined by the earlier checks. + # + # -Wl,-z,defs forces link-time symbol resolution, so that the + # linking checks with -shared actually have any value + # + # FIXME: -fPIC is required for -shared on many architectures, + # so we specify it here, but the right way would probably be to + # properly detect whether it is actually required. + CFLAGS="-shared -fPIC -Wl,-z,defs $CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + CC="$PTHREAD_CC" + + # In order not to create several levels of indentation, we test + # the value of "$done" until we find the cure or run out of ideas. + done="no" + + # First, make sure the CFLAGS we added are actually accepted by our + # compiler. If not (and OS X's ld, for instance, does not accept -z), + # then we can't do this test. + if test x"$done" = xno; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to check for GCC pthread/shared inconsistencies" >&5 +$as_echo_n "checking whether to check for GCC pthread/shared inconsistencies... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + +else + done=yes +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + + if test "x$done" = xyes ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + fi + fi + + if test x"$done" = xno; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -pthread is sufficient with -shared" >&5 +$as_echo_n "checking whether -pthread is sufficient with -shared... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + done=yes +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + + if test "x$done" = xyes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi + fi + + # + # Linux gcc on some architectures such as mips/mipsel forgets + # about -lpthread + # + if test x"$done" = xno; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -lpthread fixes that" >&5 +$as_echo_n "checking whether -lpthread fixes that... " >&6; } + LIBS="-lpthread $PTHREAD_LIBS $save_LIBS" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + done=yes +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + + if test "x$done" = xyes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + PTHREAD_LIBS="-lpthread $PTHREAD_LIBS" + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi + fi + # + # FreeBSD 4.10 gcc forgets to use -lc_r instead of -lc + # + if test x"$done" = xno; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -lc_r fixes that" >&5 +$as_echo_n "checking whether -lc_r fixes that... " >&6; } + LIBS="-lc_r $PTHREAD_LIBS $save_LIBS" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + done=yes +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + + if test "x$done" = xyes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + PTHREAD_LIBS="-lc_r $PTHREAD_LIBS" + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi + fi + if test x"$done" = xno; then + # OK, we have run out of ideas + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Impossible to determine how to use pthreads with shared libraries" >&5 +$as_echo "$as_me: WARNING: Impossible to determine how to use pthreads with shared libraries" >&2;} + + # so it's not safe to assume that we may use pthreads + acx_pthread_ok=no + fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether what we have so far is sufficient with -nostdlib" >&5 +$as_echo_n "checking whether what we have so far is sufficient with -nostdlib... " >&6; } + CFLAGS="-nostdlib $CFLAGS" + # we need c with nostdlib + LIBS="$LIBS -lc" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + done=yes +else + done=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + + if test "x$done" = xyes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi + + if test x"$done" = xno; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -lpthread saves the day" >&5 +$as_echo_n "checking whether -lpthread saves the day... " >&6; } + LIBS="-lpthread $LIBS" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + done=yes +else + done=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + + if test "x$done" = xyes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + PTHREAD_LIBS="$PTHREAD_LIBS -lpthread" + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Impossible to determine how to use pthreads with shared libraries and -nostdlib" >&5 +$as_echo "$as_me: WARNING: Impossible to determine how to use pthreads with shared libraries and -nostdlib" >&2;} + fi + fi + + CFLAGS="$save_CFLAGS" + LIBS="$save_LIBS" + CC="$save_CC" +else + PTHREAD_CC="$CC" +fi + + + + + +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test x"$acx_pthread_ok" = xyes; then + +$as_echo "#define HAVE_PTHREAD 1" >>confdefs.h + + : +else + acx_pthread_ok=no + +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for uuid_parse in -luuid" >&5 +$as_echo_n "checking for uuid_parse in -luuid... " >&6; } +if ${ac_cv_lib_uuid_uuid_parse+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-luuid $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_parse (); +int +main () +{ +return uuid_parse (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_uuid_uuid_parse=yes +else + ac_cv_lib_uuid_uuid_parse=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_parse" >&5 +$as_echo "$ac_cv_lib_uuid_uuid_parse" >&6; } +if test "x$ac_cv_lib_uuid_uuid_parse" = xyes; then : + true +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libuuid not found +See \`config.log' for more details" "$LINENO" 5; } +fi + + +if test x"$linux" = x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for blkid_devno_to_wholedisk in -lblkid" >&5 +$as_echo_n "checking for blkid_devno_to_wholedisk in -lblkid... " >&6; } +if ${ac_cv_lib_blkid_blkid_devno_to_wholedisk+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lblkid $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char blkid_devno_to_wholedisk (); +int +main () +{ +return blkid_devno_to_wholedisk (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_blkid_blkid_devno_to_wholedisk=yes +else + ac_cv_lib_blkid_blkid_devno_to_wholedisk=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_blkid_blkid_devno_to_wholedisk" >&5 +$as_echo "$ac_cv_lib_blkid_blkid_devno_to_wholedisk" >&6; } +if test "x$ac_cv_lib_blkid_blkid_devno_to_wholedisk" = xyes; then : + true +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libblkid not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi + +# +# Check for res_nquery in libresolv. There are several variations. On OSX +# res_nquery is a macro defined in resolv.h, so the typical AC_CHECK_LIB +# doesn't work. On FreeBSD res_nquery can be found in libc. The required +# library for linking (if any) is defined RESOLV_LIBS. +# +ac_fn_c_check_header_compile "$LINENO" "resolv.h" "ac_cv_header_resolv_h" "#include +" +if test "x$ac_cv_header_resolv_h" = xyes; then : + +fi + + + + + +RESOLV_LIBS="" +{ + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if res_nquery will link (LIBS=$RESOLV_LIBS)" >&5 +$as_echo_n "checking if res_nquery will link (LIBS=$RESOLV_LIBS)... " >&6; } + saved_LIBS="${LIBS}" + LIBS="$RESOLV_LIBS" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include + #include + +int +main () +{ +res_nquery(0, 0, 0, 0, 0, 0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + resolv_libs="ok" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LIBS="${saved_LIBS}" +} +if test x"$resolv_libs" != "xok"; then + RESOLV_LIBS="-lresolv" + { + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if res_nquery will link (LIBS=$RESOLV_LIBS)" >&5 +$as_echo_n "checking if res_nquery will link (LIBS=$RESOLV_LIBS)... " >&6; } + saved_LIBS="${LIBS}" + LIBS="$RESOLV_LIBS" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include + #include + +int +main () +{ +res_nquery(0, 0, 0, 0, 0, 0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + resolv_libs="ok" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LIBS="${saved_LIBS}" +} + if test x"$resolv_libs" != "xok"; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no resolv library found +See \`config.log' for more details" "$LINENO" 5; } + fi +fi + + +KEYUTILS_LIB="" +if test x"$linux" = x"yes"; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for add_key in -lkeyutils" >&5 +$as_echo_n "checking for add_key in -lkeyutils... " >&6; } +if ${ac_cv_lib_keyutils_add_key+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lkeyutils $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char add_key (); +int +main () +{ +return add_key (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_keyutils_add_key=yes +else + ac_cv_lib_keyutils_add_key=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_keyutils_add_key" >&5 +$as_echo "$ac_cv_lib_keyutils_add_key" >&6; } +if test "x$ac_cv_lib_keyutils_add_key" = xyes; then : + KEYUTILS_LIB="-lkeyutils" +else + + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libkeyutils not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for pow in -lm" >&5 +$as_echo_n "checking for pow in -lm... " >&6; } +if ${ac_cv_lib_m_pow+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lm $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char pow (); +int +main () +{ +return pow (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_m_pow=yes +else + ac_cv_lib_m_pow=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_m_pow" >&5 +$as_echo "$ac_cv_lib_m_pow" >&6; } +if test "x$ac_cv_lib_m_pow" = xyes; then : + true +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libm not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +for ac_func in syncfs +do : + ac_fn_c_check_func "$LINENO" "syncfs" "ac_cv_func_syncfs" +if test "x$ac_cv_func_syncfs" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_SYNCFS 1 +_ACEOF + +$as_echo "#define HAVE_SYS_SYNCFS 1" >>confdefs.h + +fi +done + + +# Find some crypto library for us to use, while letting user to decide which one to use. + +# Check whether --with-cryptopp was given. +if test "${with_cryptopp+set}" = set; then : + withval=$with_cryptopp; +else + with_cryptopp=check +fi + +have_cryptopp=no +# this looks clumsy but it's just if A then { success } else { if B then success } + + + + + + + +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}pkg-config", so it can be a program name with args. +set dummy ${ac_tool_prefix}pkg-config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PKG_CONFIG+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_path_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PKG_CONFIG=$ac_cv_path_PKG_CONFIG +if test -n "$PKG_CONFIG"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PKG_CONFIG" >&5 +$as_echo "$PKG_CONFIG" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_PKG_CONFIG"; then + ac_pt_PKG_CONFIG=$PKG_CONFIG + # Extract the first word of "pkg-config", so it can be a program name with args. +set dummy pkg-config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_PKG_CONFIG+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_path_ac_pt_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG +if test -n "$ac_pt_PKG_CONFIG"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_PKG_CONFIG" >&5 +$as_echo "$ac_pt_PKG_CONFIG" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_PKG_CONFIG" = x; then + PKG_CONFIG="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + PKG_CONFIG=$ac_pt_PKG_CONFIG + fi +else + PKG_CONFIG="$ac_cv_path_PKG_CONFIG" +fi + +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=0.9.0 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking pkg-config is at least version $_pkg_min_version" >&5 +$as_echo_n "checking pkg-config is at least version $_pkg_min_version... " >&6; } + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + PKG_CONFIG="" + fi +fi +if test "x$with_cryptopp" != "xno"; then : + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRYPTOPP" >&5 +$as_echo_n "checking for CRYPTOPP... " >&6; } + +if test -n "$CRYPTOPP_CFLAGS"; then + pkg_cv_CRYPTOPP_CFLAGS="$CRYPTOPP_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libcrypto++\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libcrypto++") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CRYPTOPP_CFLAGS=`$PKG_CONFIG --cflags "libcrypto++" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$CRYPTOPP_LIBS"; then + pkg_cv_CRYPTOPP_LIBS="$CRYPTOPP_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libcrypto++\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libcrypto++") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CRYPTOPP_LIBS=`$PKG_CONFIG --libs "libcrypto++" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CRYPTOPP_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libcrypto++" 2>&1` + else + CRYPTOPP_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libcrypto++" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CRYPTOPP_PKG_ERRORS" >&5 + + + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + SAVED_CXXFLAGS="${CXXFLAGS}" + SAVED_LIBS="${LIBS}" + LIBS="${LIBS} ${PTHREAD_LIBS}" + CXXFLAGS="${CXXFLAGS} ${PTHREAD_CFLAGS}" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing _ZTIN8CryptoPP14CBC_EncryptionE" >&5 +$as_echo_n "checking for library containing _ZTIN8CryptoPP14CBC_EncryptionE... " >&6; } +if ${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char _ZTIN8CryptoPP14CBC_EncryptionE (); +int +main () +{ +return _ZTIN8CryptoPP14CBC_EncryptionE (); + ; + return 0; +} +_ACEOF +for ac_lib in '' crypto++ cryptopp; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE+:} false; then : + break +fi +done +if ${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE+:} false; then : + +else + ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE" >&5 +$as_echo "$ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE" >&6; } +ac_res=$ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + have_cryptopp=yes +else + true +fi + + CRYPTOPP_LIBS="${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE}" + LIBS="${SAVED_LIBS}" + CXXFLAGS="${SAVED_CXXFLAGS}" + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + SAVED_CXXFLAGS="${CXXFLAGS}" + SAVED_LIBS="${LIBS}" + LIBS="${LIBS} ${PTHREAD_LIBS}" + CXXFLAGS="${CXXFLAGS} ${PTHREAD_CFLAGS}" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing _ZTIN8CryptoPP14CBC_EncryptionE" >&5 +$as_echo_n "checking for library containing _ZTIN8CryptoPP14CBC_EncryptionE... " >&6; } +if ${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char _ZTIN8CryptoPP14CBC_EncryptionE (); +int +main () +{ +return _ZTIN8CryptoPP14CBC_EncryptionE (); + ; + return 0; +} +_ACEOF +for ac_lib in '' crypto++ cryptopp; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE+:} false; then : + break +fi +done +if ${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE+:} false; then : + +else + ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE" >&5 +$as_echo "$ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE" >&6; } +ac_res=$ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + have_cryptopp=yes +else + true +fi + + CRYPTOPP_LIBS="${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE}" + LIBS="${SAVED_LIBS}" + CXXFLAGS="${SAVED_CXXFLAGS}" + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +else + CRYPTOPP_CFLAGS=$pkg_cv_CRYPTOPP_CFLAGS + CRYPTOPP_LIBS=$pkg_cv_CRYPTOPP_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + have_cryptopp=yes +fi +fi +# bail out if given explicit --with-cryptopp +if test "x$have_cryptopp" = "xno" -a "x$with_cryptopp" != "xcheck" -a "x$with_cryptopp" != "xno"; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "--with-cryptopp was given, but library was not found +See \`config.log' for more details" "$LINENO" 5; } +fi + + +# Check whether --with-nss was given. +if test "${with_nss+set}" = set; then : + withval=$with_nss; +else + with_nss=check +fi + +have_nss=no +if test "x$with_nss" != "xno"; then : + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for NSS" >&5 +$as_echo_n "checking for NSS... " >&6; } + +if test -n "$NSS_CFLAGS"; then + pkg_cv_NSS_CFLAGS="$NSS_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"nss\""; } >&5 + ($PKG_CONFIG --exists --print-errors "nss") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_NSS_CFLAGS=`$PKG_CONFIG --cflags "nss" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$NSS_LIBS"; then + pkg_cv_NSS_LIBS="$NSS_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"nss\""; } >&5 + ($PKG_CONFIG --exists --print-errors "nss") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_NSS_LIBS=`$PKG_CONFIG --libs "nss" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + NSS_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "nss" 2>&1` + else + NSS_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "nss" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$NSS_PKG_ERRORS" >&5 + + true +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + true +else + NSS_CFLAGS=$pkg_cv_NSS_CFLAGS + NSS_LIBS=$pkg_cv_NSS_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + have_nss=yes +fi +fi +# bail out if given explicit --with-nss +if test "x$have_nss" = "xno" -a "x$with_nss" != "xcheck" -a "x$with_nss" != "xno"; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "--with-nss was given, but library was not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +# now decide which crypto library to really use +if test "x$have_cryptopp" = "xyes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: using cryptopp for cryptography" >&5 +$as_echo "$as_me: using cryptopp for cryptography" >&6;} + +$as_echo "#define USE_CRYPTOPP 1" >>confdefs.h + + CRYPTO_CFLAGS=$CRYPTOPP_CFLAGS + + #AC_SUBST([CRYPTO_CXXFLAGS], [$CRYPTOPP_CXXFLAGS]) + AM_CXXFLAGS="${AM_CXXFLAGS} ${CRYPTOPP_CXXFLAGS}" + CRYPTO_LIBS=$CRYPTOPP_LIBS + +elif test "x$have_nss" = "xyes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: using nss for cryptography" >&5 +$as_echo "$as_me: using nss for cryptography" >&6;} + +$as_echo "#define USE_NSS 1" >>confdefs.h + + CRYPTO_CFLAGS=$NSS_CFLAGS + + # this needs CFLAGS too in practise to get the includes right. ugly. + #AC_SUBST([CRYPTO_CXXFLAGS], [$NSS_CFLAGS $NSS_CXXFLAGS]) + AM_CXXFLAGS="${AM_CXXFLAGS} ${NSS_CFLAGS} ${NSS_CXXFLAGS}" + CRYPTO_LIBS=$NSS_LIBS + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no suitable crypto library found +See \`config.log' for more details" "$LINENO" 5; } +fi + +# profiler? + +# Check whether --with-profiler was given. +if test "${with_profiler+set}" = set; then : + withval=$with_profiler; case "${withval}" in + yes) with_profiler=yes ;; + no) with_profiler=no ;; + *) as_fn_error $? "bad value ${withval} for --with-profiler" "$LINENO" 5 ;; + esac +else + with_profiler=no +fi + +if test "x$with_profiler" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ProfilerFlush in -lprofiler" >&5 +$as_echo_n "checking for ProfilerFlush in -lprofiler... " >&6; } +if ${ac_cv_lib_profiler_ProfilerFlush+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lprofiler $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char ProfilerFlush (); +int +main () +{ +return ProfilerFlush (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_profiler_ProfilerFlush=yes +else + ac_cv_lib_profiler_ProfilerFlush=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_profiler_ProfilerFlush" >&5 +$as_echo "$ac_cv_lib_profiler_ProfilerFlush" >&6; } +if test "x$ac_cv_lib_profiler_ProfilerFlush" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBPROFILER 1 +_ACEOF + + LIBS="-lprofiler $LIBS" + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "--with-profiler was given but libprofiler (libgoogle-perftools-dev on debian) not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi + if test "$with_profiler" = "yes"; then + WITH_PROFILER_TRUE= + WITH_PROFILER_FALSE='#' +else + WITH_PROFILER_TRUE='#' + WITH_PROFILER_FALSE= +fi + +if test "$with_profiler" = "yes"; then : + +$as_echo "#define HAVE_PROFILER 1" >>confdefs.h + +fi + +# debug crap? + +# Check whether --with-debug was given. +if test "${with_debug+set}" = set; then : + withval=$with_debug; case "${withval}" in + yes) with_debug=yes ;; + no) with_debug=no ;; + *) as_fn_error $? "bad value ${withval} for --with-debug" "$LINENO" 5 ;; + esac +else + with_debug=no +fi + + if test "$with_debug" = "yes"; then + WITH_DEBUG_TRUE= + WITH_DEBUG_FALSE='#' +else + WITH_DEBUG_TRUE='#' + WITH_DEBUG_FALSE= +fi + + + +$as_echo "#define DEBUG_GATHER 1" >>confdefs.h + + +# code coverage? +# Check whether --enable-coverage was given. +if test "${enable_coverage+set}" = set; then : + enableval=$enable_coverage; +else + enable_coverage=no +fi + + if test "x$enable_coverage" != xno; then + ENABLE_COVERAGE_TRUE= + ENABLE_COVERAGE_FALSE='#' +else + ENABLE_COVERAGE_TRUE='#' + ENABLE_COVERAGE_FALSE= +fi + +if test "x$enable_coverage" != xno; then + +$as_echo "#define ENABLE_COVERAGE 1" >>confdefs.h + +fi +GCOV_PREFIX_STRIP=`echo $(pwd)/src | tr -dc / | wc -c` + + +# radosgw? + +# Check whether --with-radosgw was given. +if test "${with_radosgw+set}" = set; then : + withval=$with_radosgw; +else + with_radosgw=check +fi + +RADOSGW=0 +if test "x$with_radosgw" != xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for FCGX_Init in -lfcgi" >&5 +$as_echo_n "checking for FCGX_Init in -lfcgi... " >&6; } +if ${ac_cv_lib_fcgi_FCGX_Init+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lfcgi $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char FCGX_Init (); +int +main () +{ +return FCGX_Init (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_fcgi_FCGX_Init=yes +else + ac_cv_lib_fcgi_FCGX_Init=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_fcgi_FCGX_Init" >&5 +$as_echo "$ac_cv_lib_fcgi_FCGX_Init" >&6; } +if test "x$ac_cv_lib_fcgi_FCGX_Init" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for XML_Parse in -lexpat" >&5 +$as_echo_n "checking for XML_Parse in -lexpat... " >&6; } +if ${ac_cv_lib_expat_XML_Parse+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lexpat $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char XML_Parse (); +int +main () +{ +return XML_Parse (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_expat_XML_Parse=yes +else + ac_cv_lib_expat_XML_Parse=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_expat_XML_Parse" >&5 +$as_echo "$ac_cv_lib_expat_XML_Parse" >&6; } +if test "x$ac_cv_lib_expat_XML_Parse" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_easy_init in -lcurl" >&5 +$as_echo_n "checking for curl_easy_init in -lcurl... " >&6; } +if ${ac_cv_lib_curl_curl_easy_init+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lcurl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char curl_easy_init (); +int +main () +{ +return curl_easy_init (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_curl_curl_easy_init=yes +else + ac_cv_lib_curl_curl_easy_init=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_init" >&5 +$as_echo "$ac_cv_lib_curl_curl_easy_init" >&6; } +if test "x$ac_cv_lib_curl_curl_easy_init" = xyes; then : + RADOSGW=1 + ac_fn_c_check_header_mongrel "$LINENO" "fastcgi/fcgiapp.h" "ac_cv_header_fastcgi_fcgiapp_h" "$ac_includes_default" +if test "x$ac_cv_header_fastcgi_fcgiapp_h" = xyes; then : + +$as_echo "#define FASTCGI_INCLUDE_DIR 1" >>confdefs.h + +fi + + + +else + if test "x$with_radosgw" != "xcheck"; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "--with-radosgw was given but libcurl (libcurl-dev on debian) not found +See \`config.log' for more details" "$LINENO" 5; } + fi +fi + + +else + if test "x$with_radosgw" != "xcheck"; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "--with-radosgw was given but libexpat (libexpat1-dev on debian) not found +See \`config.log' for more details" "$LINENO" 5; } + fi +fi + + +else + if test "x$with_radosgw" != "xcheck"; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "--with-radosgw was given but libfcgi (libfcgi-dev on debian) not found +See \`config.log' for more details" "$LINENO" 5; } + fi +fi + +fi + if test "$RADOSGW" = "1"; then + WITH_RADOSGW_TRUE= + WITH_RADOSGW_FALSE='#' +else + WITH_RADOSGW_TRUE='#' + WITH_RADOSGW_FALSE= +fi + + +if test "$RADOSGW" = "1"; then : + +$as_echo "#define WITH_RADOSGW 1" >>confdefs.h + +fi + +if test "$RADOSGW" = "1"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_multi_wait in -lcurl" >&5 +$as_echo_n "checking for curl_multi_wait in -lcurl... " >&6; } +if ${ac_cv_lib_curl_curl_multi_wait+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lcurl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char curl_multi_wait (); +int +main () +{ +return curl_multi_wait (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_curl_curl_multi_wait=yes +else + ac_cv_lib_curl_curl_multi_wait=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_multi_wait" >&5 +$as_echo "$ac_cv_lib_curl_curl_multi_wait" >&6; } +if test "x$ac_cv_lib_curl_curl_multi_wait" = xyes; then : + +$as_echo "#define HAVE_CURL_MULTI_WAIT 1" >>confdefs.h + +fi + + +fi + +# fuse? + +# Check whether --with-fuse was given. +if test "${with_fuse+set}" = set; then : + withval=$with_fuse; +else + with_fuse=yes +fi + +LIBFUSE= +if test "x$with_fuse" != xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for fuse_main in -lfuse" >&5 +$as_echo_n "checking for fuse_main in -lfuse... " >&6; } +if ${ac_cv_lib_fuse_fuse_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lfuse $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char fuse_main (); +int +main () +{ +return fuse_main (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_fuse_fuse_main=yes +else + ac_cv_lib_fuse_fuse_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_fuse_fuse_main" >&5 +$as_echo "$ac_cv_lib_fuse_fuse_main" >&6; } +if test "x$ac_cv_lib_fuse_fuse_main" = xyes; then : + LIBFUSE="-lfuse" + + +$as_echo "#define HAVE_LIBFUSE 1" >>confdefs.h + + HAVE_LIBFUSE=1 + # look for fuse_getgroups and define FUSE_GETGROUPS if found + LIBS_saved="$LIBS" + LIBS="$LIBS -lfuse" + for ac_func in fuse_getgroups +do : + ac_fn_c_check_func "$LINENO" "fuse_getgroups" "ac_cv_func_fuse_getgroups" +if test "x$ac_cv_func_fuse_getgroups" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_FUSE_GETGROUPS 1 +_ACEOF + +fi +done + + LIBS="$LIBS_saved" + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no FUSE found (use --without-fuse to disable) +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi + if test "$HAVE_LIBFUSE" = "1"; then + WITH_FUSE_TRUE= + WITH_FUSE_FALSE='#' +else + WITH_FUSE_TRUE='#' + WITH_FUSE_FALSE= +fi + + +# tcmalloc? + +# Check whether --with-tcmalloc was given. +if test "${with_tcmalloc+set}" = set; then : + withval=$with_tcmalloc; +else + with_tcmalloc=yes +fi + +TCMALLOC= +if test "x$with_tcmalloc" != xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for malloc in -ltcmalloc" >&5 +$as_echo_n "checking for malloc in -ltcmalloc... " >&6; } +if ${ac_cv_lib_tcmalloc_malloc+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ltcmalloc $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char malloc (); +int +main () +{ +return malloc (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_tcmalloc_malloc=yes +else + ac_cv_lib_tcmalloc_malloc=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_tcmalloc_malloc" >&5 +$as_echo "$ac_cv_lib_tcmalloc_malloc" >&6; } +if test "x$ac_cv_lib_tcmalloc_malloc" = xyes; then : + LIBTCMALLOC="-ltcmalloc" + + +$as_echo "#define HAVE_LIBTCMALLOC 1" >>confdefs.h + + HAVE_LIBTCMALLOC=1 + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no tcmalloc found (use --without-tcmalloc to disable) +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi + if test "$HAVE_LIBTCMALLOC" = "1"; then + WITH_TCMALLOC_TRUE= + WITH_TCMALLOC_FALSE='#' +else + WITH_TCMALLOC_TRUE='#' + WITH_TCMALLOC_FALSE= +fi + + +#set pg ref debugging? +# Check whether --enable-pgrefdebugging was given. +if test "${enable_pgrefdebugging+set}" = set; then : + enableval=$enable_pgrefdebugging; +$as_echo "#define PG_DEBUG_REFS 1" >>confdefs.h + +fi + + +# +# Java is painful +# - adapted from OMPI wrappers package +# - this might become bigger. maybe should be own m4 file +# +# Check whether --enable-cephfs-java was given. +if test "${enable_cephfs_java+set}" = set; then : + enableval=$enable_cephfs_java; +else + enable_cephfs_java=no +fi + + + if test "x$enable_cephfs_java" = "xyes"; then + ENABLE_CEPHFS_JAVA_TRUE= + ENABLE_CEPHFS_JAVA_FALSE='#' +else + ENABLE_CEPHFS_JAVA_TRUE='#' + ENABLE_CEPHFS_JAVA_FALSE= +fi + + + +# Check whether --with-jdk-dir was given. +if test "${with_jdk_dir+set}" = set; then : + withval=$with_jdk_dir; +fi + + +if test "x$enable_cephfs_java" = "xyes"; then + + # setup bin/include dirs from --with-jdk-dir (search for jni.h, javac) + if test -n "$with_jdk_dir"; then : + + javac_prog=`find $with_jdk_dir/ -name javac | head -n 1` + if test -x "$javac_prog"; then : + + EXTRA_JDK_BIN_DIR=`dirname $javac_prog` +fi + jnih=`find $with_jdk_dir/ -name jni.h | head -n 1` + if test -r "$jnih"; then : + + EXTRA_JDK_INC_DIR=`dirname $jnih` +fi +fi + + # setup defaults for Debian default-jdk package (without --with-jdk-dir) + if test -z "$with_jdk_dir"; then : + + # This works with Debian's and CentOS' default-jdk package + for dir in '/usr/lib/jvm/default-java/' '/usr/lib/jvm/java/' '/usr/lib/jvm/java-gcj/'; do + # only test if a suitable path has not yet been found + if test "$EXTRA_JDK_BIN_DIR" == ""; then : + + if test -x "$javac_prog"; then : + + EXTRA_JDK_BIN_DIR=`dirname $javac_prog` +fi + jnih=`find $dir -name jni.h | head -n 1` + if test -r "$jnih"; then : + + EXTRA_JDK_INC_DIR=`dirname $jnih` +fi + +fi + done + +fi + + # cephfs_java_test only makes sense if java is already turned on + # setup CLASSPATH for Debian default junit4.jar package + # + # Configuring --with-debug and --enable-cephfs-java will throw an error if + # JUnit4 cannot be found. While currently this works for users who have + # installed via the package manager on Ubuntu, we need to expand this + # check to 1 support other distrubtions and 2 allow users to influence + # the search path. + if test "x$with_debug" = "xyes"; then : + + dir='/usr/share/java' + junit4_jar=`find $dir -name junit4.jar | head -n 1` + if test -r "$junit4_jar"; then : + + EXTRA_CLASSPATH_JAR=`dirname $junit4_jar`/junit4.jar + + have_junit4=1 +else + + { $as_echo "$as_me:${as_lineno-$LINENO}: Cannot find junit4.jar (apt-get install junit4)" >&5 +$as_echo "$as_me: Cannot find junit4.jar (apt-get install junit4)" >&6;} + have_junit4=0 +fi +fi + + +if test "x$CLASSPATH" = x; then + echo "You have no CLASSPATH, I hope it is good" +else + echo "You have CLASSPATH $CLASSPATH, hope it is correct" +fi + + + +if test "x$JAVAPREFIX" = x; then + test "x$JAVAC" = x && for ac_prog in javac$EXEEXT "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_JAVAC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$JAVAC"; then + ac_cv_prog_JAVAC="$JAVAC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_JAVAC="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +JAVAC=$ac_cv_prog_JAVAC +if test -n "$JAVAC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAC" >&5 +$as_echo "$JAVAC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$JAVAC" && break +done + +else + test "x$JAVAC" = x && for ac_prog in javac$EXEEXT "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_JAVAC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$JAVAC"; then + ac_cv_prog_JAVAC="$JAVAC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_JAVAC="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +JAVAC=$ac_cv_prog_JAVAC +if test -n "$JAVAC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAC" >&5 +$as_echo "$JAVAC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$JAVAC" && break +done +test -n "$JAVAC" || JAVAC="$JAVAPREFIX" + +fi +test "x$JAVAC" = x && as_fn_error $? "no acceptable Java compiler found in \$PATH" "$LINENO" 5 + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $JAVAC works" >&5 +$as_echo_n "checking if $JAVAC works... " >&6; } +if ${ac_cv_prog_javac_works+:} false; then : + $as_echo_n "(cached) " >&6 +else + +JAVA_TEST=Test.java +CLASS_TEST=Test.class +cat << \EOF > $JAVA_TEST +/* #line 19087 "configure" */ +public class Test { +} +EOF +if { ac_try='$JAVAC $JAVACFLAGS $JAVA_TEST' + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5 + (eval $ac_try) 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; } >/dev/null 2>&1; then + ac_cv_prog_javac_works=yes +else + as_fn_error $? "The Java compiler $JAVAC failed (see config.log, check the CLASSPATH?)" "$LINENO" 5 + echo "configure: failed program was:" >&5 + cat $JAVA_TEST >&5 +fi +rm -f $JAVA_TEST $CLASS_TEST + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_javac_works" >&5 +$as_echo "$ac_cv_prog_javac_works" >&6; } + + + +# Extract the first word of "javah", so it can be a program name with args. +set dummy javah; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_JAVAH+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $JAVAH in + [\\/]* | ?:[\\/]*) + ac_cv_path_JAVAH="$JAVAH" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_path_JAVAH="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +JAVAH=$ac_cv_path_JAVAH +if test -n "$JAVAH"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAVAH" >&5 +$as_echo "$JAVAH" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +if test x"`eval 'echo $ac_cv_path_JAVAH'`" != x ; then + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + +else + + ac_save_CPPFLAGS="$CPPFLAGS" + ac_dir=`echo $ac_cv_path_JAVAH | sed 's,\(.*\)/[^/]*/[^/]*$,\1/include,'` + ac_machdep=`echo $build_os | sed 's,[-0-9].*,,' | sed 's,cygwin,win32,'` + CPPFLAGS="$ac_save_CPPFLAGS -I$ac_dir -I$ac_dir/$ac_machdep" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + ac_save_CPPFLAGS="$CPPFLAGS" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unable to include " >&5 +$as_echo "$as_me: WARNING: unable to include " >&2;} +fi +rm -f conftest.err conftest.i conftest.$ac_ext + CPPFLAGS="$ac_save_CPPFLAGS" +fi +rm -f conftest.err conftest.i conftest.$ac_ext +fi + +if test "x$JAVAPREFIX" = x; then + test "x$JAR" = x && for ac_prog in jar$EXEEXT +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_JAR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$JAR"; then + ac_cv_prog_JAR="$JAR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_JAR="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +JAR=$ac_cv_prog_JAR +if test -n "$JAR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAR" >&5 +$as_echo "$JAR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$JAR" && break +done + +else + test "x$JAR" = x && for ac_prog in jar +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_JAR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$JAR"; then + ac_cv_prog_JAR="$JAR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_JAR="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +JAR=$ac_cv_prog_JAR +if test -n "$JAR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JAR" >&5 +$as_echo "$JAR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$JAR" && break +done +test -n "$JAR" || JAR="$JAVAPREFIX" + +fi +test "x$JAR" = x && as_fn_error $? "no acceptable jar program found in \$PATH" "$LINENO" 5 + + + CLASSPATH=$CLASSPATH:$EXTRA_CLASSPATH_JAR + export CLASSPATH + { $as_echo "$as_me:${as_lineno-$LINENO}: classpath - $CLASSPATH" >&5 +$as_echo "$as_me: classpath - $CLASSPATH" >&6;} + + # Check for jni.h + CPPFLAGS_save=$CPPFLAGS + + if test -n "$EXTRA_JDK_INC_DIR"; then : + JDK_CPPFLAGS="-I$EXTRA_JDK_INC_DIR" + if test -d "$EXTRA_JDK_INC_DIR/linux"; then : + JDK_CPPFLAGS="$JDK_CPPFLAGS -I$EXTRA_JDK_INC_DIR/linux" +fi + CPPFLAGS="$CPPFLAGS $JDK_CPPFLAGS" +fi + + ac_fn_c_check_header_mongrel "$LINENO" "jni.h" "ac_cv_header_jni_h" "$ac_includes_default" +if test "x$ac_cv_header_jni_h" = xyes; then : + +else + as_fn_error $? "Cannot find header 'jni.h'. Try setting --with-jdk-dir" "$LINENO" 5 +fi + + + + CPPFLAGS=$CPPFLAGS_save + + # Setup output var + +fi + if test "$have_junit4" = "1"; then + HAVE_JUNIT4_TRUE= + HAVE_JUNIT4_FALSE='#' +else + HAVE_JUNIT4_TRUE='#' + HAVE_JUNIT4_FALSE= +fi + + +# +# FreeBSD has it in base. +# +if test x"$freebsd" != x"yes"; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBEDIT" >&5 +$as_echo_n "checking for LIBEDIT... " >&6; } + +if test -n "$LIBEDIT_CFLAGS"; then + pkg_cv_LIBEDIT_CFLAGS="$LIBEDIT_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libedit >= 2.11\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libedit >= 2.11") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBEDIT_CFLAGS=`$PKG_CONFIG --cflags "libedit >= 2.11" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LIBEDIT_LIBS"; then + pkg_cv_LIBEDIT_LIBS="$LIBEDIT_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libedit >= 2.11\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libedit >= 2.11") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBEDIT_LIBS=`$PKG_CONFIG --libs "libedit >= 2.11" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libedit >= 2.11" 2>&1` + else + LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libedit >= 2.11" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LIBEDIT_PKG_ERRORS" >&5 + + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "No usable version of libedit found. +See \`config.log' for more details" "$LINENO" 5; } +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "No usable version of libedit found. +See \`config.log' for more details" "$LINENO" 5; } +else + LIBEDIT_CFLAGS=$pkg_cv_LIBEDIT_CFLAGS + LIBEDIT_LIBS=$pkg_cv_LIBEDIT_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi +else + LIBEDIT_LIBS="-ledit" +fi + +#libatomic-ops? You want it! + +# Check whether --with-libatomic-ops was given. +if test "${with_libatomic_ops+set}" = set; then : + withval=$with_libatomic_ops; +else + with_libatomic_ops=yes +fi + +if test "x$with_libatomic_ops" != xno; then : + ac_fn_c_check_header_mongrel "$LINENO" "atomic_ops.h" "ac_cv_header_atomic_ops_h" "$ac_includes_default" +if test "x$ac_cv_header_atomic_ops_h" = xyes; then : + HAVE_ATOMIC_OPS=1 +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no libatomic-ops found (use --without-libatomic-ops to disable) +See \`config.log' for more details" "$LINENO" 5; } + +fi + + +fi +if test "$HAVE_ATOMIC_OPS" = "1"; then : + + # The cast to long int works around a bug in the HP C Compiler +# version HP92453-01 B.11.11.23709.GP, which incorrectly rejects +# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# This bug is HP SR number 8606223364. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking size of AO_t" >&5 +$as_echo_n "checking size of AO_t... " >&6; } +if ${ac_cv_sizeof_AO_t+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (AO_t))" "ac_cv_sizeof_AO_t" " + #include + +"; then : + +else + if test "$ac_cv_type_AO_t" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute sizeof (AO_t) +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_sizeof_AO_t=0 + fi +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_AO_t" >&5 +$as_echo "$ac_cv_sizeof_AO_t" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define SIZEOF_AO_T $ac_cv_sizeof_AO_t +_ACEOF + + + +else + +$as_echo "#define NO_ATOMIC_OPS 1" >>confdefs.h + +fi + + + if test "$HAVE_ATOMIC_OPS" = "1"; then + WITH_LIBATOMIC_TRUE= + WITH_LIBATOMIC_FALSE='#' +else + WITH_LIBATOMIC_TRUE='#' + WITH_LIBATOMIC_FALSE= +fi + + +# newsyn? requires mpi. +#AC_ARG_WITH([newsyn], +# [AS_HELP_STRING([--with-newsyn], [build newsyn target requires mpi])], +# [], +# [with_newsyn=no]) + + +# Check whether --with-ocf was given. +if test "${with_ocf+set}" = set; then : + withval=$with_ocf; +else + with_ocf=no +fi + + if test "$with_ocf" = "yes" ; then + WITH_OCF_TRUE= + WITH_OCF_FALSE='#' +else + WITH_OCF_TRUE='#' + WITH_OCF_FALSE= +fi + + +# check is snappy-devel is installed, needed by leveldb +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for snappy_compress in -lsnappy" >&5 +$as_echo_n "checking for snappy_compress in -lsnappy... " >&6; } +if ${ac_cv_lib_snappy_snappy_compress+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lsnappy $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char snappy_compress (); +int +main () +{ +return snappy_compress (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_snappy_snappy_compress=yes +else + ac_cv_lib_snappy_snappy_compress=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_snappy_snappy_compress" >&5 +$as_echo "$ac_cv_lib_snappy_snappy_compress" >&6; } +if test "x$ac_cv_lib_snappy_snappy_compress" = xyes; then : + true +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libsnappy not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +# use system leveldb +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for leveldb_open in -lleveldb" >&5 +$as_echo_n "checking for leveldb_open in -lleveldb... " >&6; } +if ${ac_cv_lib_leveldb_leveldb_open+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lleveldb -lsnappy -lpthread $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char leveldb_open (); +int +main () +{ +return leveldb_open (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_leveldb_leveldb_open=yes +else + ac_cv_lib_leveldb_leveldb_open=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_leveldb_leveldb_open" >&5 +$as_echo "$ac_cv_lib_leveldb_leveldb_open" >&6; } +if test "x$ac_cv_lib_leveldb_leveldb_open" = xyes; then : + true +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libleveldb not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +# see if we can use bloom filters with leveldb +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + +ac_fn_cxx_check_header_mongrel "$LINENO" "leveldb/filter_policy.h" "ac_cv_header_leveldb_filter_policy_h" "$ac_includes_default" +if test "x$ac_cv_header_leveldb_filter_policy_h" = xyes; then : + +$as_echo "#define HAVE_LEVELDB_FILTER_POLICY 1" >>confdefs.h + +fi + + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +# Find supported SIMD / SSE extensions supported by the compiler + + + + case $target_cpu in + i[3456]86*|x86_64*|amd64*) + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse" >&5 +$as_echo_n "checking whether C compiler accepts -msse... " >&6; } +if ${ax_cv_check_cflags___msse+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -msse" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___msse=yes +else + ax_cv_check_cflags___msse=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___msse" >&5 +$as_echo "$ax_cv_check_cflags___msse" >&6; } +if test x"$ax_cv_check_cflags___msse" = xyes; then : + ax_cv_support_sse_ext=yes +else + : +fi + + if test x"$ax_cv_support_sse_ext" = x"yes"; then + INTEL_SSE_FLAGS="-msse -DINTEL_SSE" + + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE_FLAGS" + +$as_echo "#define HAVE_SSE /**/" >>confdefs.h + + fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse2" >&5 +$as_echo_n "checking whether C compiler accepts -msse2... " >&6; } +if ${ax_cv_check_cflags___msse2+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -msse2" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___msse2=yes +else + ax_cv_check_cflags___msse2=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___msse2" >&5 +$as_echo "$ax_cv_check_cflags___msse2" >&6; } +if test x"$ax_cv_check_cflags___msse2" = xyes; then : + ax_cv_support_sse2_ext=yes +else + : +fi + + if test x"$ax_cv_support_sse2_ext" = x"yes"; then + INTEL_SSE2_FLAGS="-msse2 -DINTEL_SSE2" + + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE2_FLAGS" + +$as_echo "#define HAVE_SSE2 /**/" >>confdefs.h + + fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse3" >&5 +$as_echo_n "checking whether C compiler accepts -msse3... " >&6; } +if ${ax_cv_check_cflags___msse3+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -msse3" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___msse3=yes +else + ax_cv_check_cflags___msse3=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___msse3" >&5 +$as_echo "$ax_cv_check_cflags___msse3" >&6; } +if test x"$ax_cv_check_cflags___msse3" = xyes; then : + ax_cv_support_sse3_ext=yes +else + : +fi + + if test x"$ax_cv_support_sse3_ext" = x"yes"; then + INTEL_SSE3_FLAGS="-msse3 -DINTEL_SSE3" + + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE3_FLAGS" + +$as_echo "#define HAVE_SSE3 /**/" >>confdefs.h + + fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mssse3" >&5 +$as_echo_n "checking whether C compiler accepts -mssse3... " >&6; } +if ${ax_cv_check_cflags___mssse3+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -mssse3" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___mssse3=yes +else + ax_cv_check_cflags___mssse3=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___mssse3" >&5 +$as_echo "$ax_cv_check_cflags___mssse3" >&6; } +if test x"$ax_cv_check_cflags___mssse3" = xyes; then : + ax_cv_support_ssse3_ext=yes +else + : +fi + + if test x"$ax_cv_support_ssse3_ext" = x"yes"; then + INTEL_SSSE3_FLAGS="-mssse3 -DINTEL_SSSE3" + + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSSE3_FLAGS" + +$as_echo "#define HAVE_SSSE3 /**/" >>confdefs.h + + fi + ;; + esac + + case $target_cpu in + x86_64*|amd64*) + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mpclmul" >&5 +$as_echo_n "checking whether C compiler accepts -mpclmul... " >&6; } +if ${ax_cv_check_cflags___mpclmul+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -mpclmul" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___mpclmul=yes +else + ax_cv_check_cflags___mpclmul=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___mpclmul" >&5 +$as_echo "$ax_cv_check_cflags___mpclmul" >&6; } +if test x"$ax_cv_check_cflags___mpclmul" = xyes; then : + ax_cv_support_pclmuldq_ext=yes +else + : +fi + + if test x"$ax_cv_support_pclmuldq_ext" = x"yes"; then + INTEL_PCLMUL_FLAGS="-mpclmul -DINTEL_SSE4_PCLMUL" + + INTEL_FLAGS="$INTEL_FLAGS $INTEL_PCLMUL_FLAGS" + +$as_echo "#define HAVE_PCLMUL /**/" >>confdefs.h + + fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse4.1" >&5 +$as_echo_n "checking whether C compiler accepts -msse4.1... " >&6; } +if ${ax_cv_check_cflags___msse4_1+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -msse4.1" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___msse4_1=yes +else + ax_cv_check_cflags___msse4_1=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___msse4_1" >&5 +$as_echo "$ax_cv_check_cflags___msse4_1" >&6; } +if test x"$ax_cv_check_cflags___msse4_1" = xyes; then : + ax_cv_support_sse41_ext=yes +else + : +fi + + if test x"$ax_cv_support_sse41_ext" = x"yes"; then + INTEL_SSE4_1_FLAGS="-msse4.1 -DINTEL_SSE4" + + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE4_1_FLAGS" + +$as_echo "#define HAVE_SSE4_1 /**/" >>confdefs.h + + fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse4.2" >&5 +$as_echo_n "checking whether C compiler accepts -msse4.2... " >&6; } +if ${ax_cv_check_cflags___msse4_2+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -msse4.2" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___msse4_2=yes +else + ax_cv_check_cflags___msse4_2=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___msse4_2" >&5 +$as_echo "$ax_cv_check_cflags___msse4_2" >&6; } +if test x"$ax_cv_check_cflags___msse4_2" = xyes; then : + ax_cv_support_sse42_ext=yes +else + : +fi + + if test x"$ax_cv_support_sse42_ext" = x"yes"; then + INTEL_SSE4_2_FLAGS="-msse4.2 -DINTEL_SSE4" + + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE4_2_FLAGS" + +$as_echo "#define HAVE_SSE4_2 /**/" >>confdefs.h + + fi + ;; + esac + + + + +# use system libs3? + +# Check whether --with-system-libs3 was given. +if test "${with_system_libs3+set}" = set; then : + withval=$with_system_libs3; +else + with_system_libs3=no +fi + +if test "x$with_system_libs3" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for S3_initialize in -ls3" >&5 +$as_echo_n "checking for S3_initialize in -ls3... " >&6; } +if ${ac_cv_lib_s3_S3_initialize+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ls3 -lpthread $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char S3_initialize (); +int +main () +{ +return S3_initialize (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_s3_S3_initialize=yes +else + ac_cv_lib_s3_S3_initialize=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_s3_S3_initialize" >&5 +$as_echo "$ac_cv_lib_s3_S3_initialize" >&6; } +if test "x$ac_cv_lib_s3_S3_initialize" = xyes; then : + true +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libs3 not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi +if test "x$with_system_libs3" = xcheck; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing S3_initialize" >&5 +$as_echo_n "checking for library containing S3_initialize... " >&6; } +if ${ac_cv_search_S3_initialize+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char S3_initialize (); +int +main () +{ +return S3_initialize (); + ; + return 0; +} +_ACEOF +for ac_lib in '' s3; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib -lpthread $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_S3_initialize=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_S3_initialize+:} false; then : + break +fi +done +if ${ac_cv_search_S3_initialize+:} false; then : + +else + ac_cv_search_S3_initialize=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_S3_initialize" >&5 +$as_echo "$ac_cv_search_S3_initialize" >&6; } +ac_res=$ac_cv_search_S3_initialize +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + with_system_libs3=yes +else + true +fi + +fi + if test "$with_system_libs3" = "yes" ; then + WITH_SYSTEM_LIBS3_TRUE= + WITH_SYSTEM_LIBS3_FALSE='#' +else + WITH_SYSTEM_LIBS3_TRUE='#' + WITH_SYSTEM_LIBS3_FALSE= +fi + + +# rest-bench? + +# Check whether --with-rest-bench was given. +if test "${with_rest_bench+set}" = set; then : + withval=$with_rest_bench; +else + with_rest_bench=no +fi + + if test "$with_rest_bench" = "yes" ; then + WITH_REST_BENCH_TRUE= + WITH_REST_BENCH_FALSE='#' +else + WITH_REST_BENCH_TRUE='#' + WITH_REST_BENCH_FALSE= +fi + + +# use libaio? + +# Check whether --with-libaio was given. +if test "${with_libaio+set}" = set; then : + withval=$with_libaio; +else + with_libaio=yes +fi + +if test "x$with_libaio" != xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for io_submit in -laio" >&5 +$as_echo_n "checking for io_submit in -laio... " >&6; } +if ${ac_cv_lib_aio_io_submit+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-laio $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char io_submit (); +int +main () +{ +return io_submit (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_aio_io_submit=yes +else + ac_cv_lib_aio_io_submit=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_aio_io_submit" >&5 +$as_echo "$ac_cv_lib_aio_io_submit" >&6; } +if test "x$ac_cv_lib_aio_io_submit" = xyes; then : + true +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libaio not found +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi +if test "x$with_libaio" != xno; then : + ac_fn_c_check_header_mongrel "$LINENO" "libaio.h" "ac_cv_header_libaio_h" "$ac_includes_default" +if test "x$ac_cv_header_libaio_h" = xyes; then : + +fi + + +fi +if test "$with_libaio" = "yes"; then : + +$as_echo "#define HAVE_LIBAIO 1" >>confdefs.h + +fi + if test "$with_libaio" = "yes" ; then + WITH_LIBAIO_TRUE= + WITH_LIBAIO_FALSE='#' +else + WITH_LIBAIO_TRUE='#' + WITH_LIBAIO_FALSE= +fi + + +# use libxfs? + +# Check whether --with-libxfs was given. +if test "${with_libxfs+set}" = set; then : + withval=$with_libxfs; +else + with_libxfs=yes +fi + +if test "x$with_libxfs" != "xno"; then : + + # xfs/xfs.h presence and XFS_XFLAG_EXTSIZE define + ac_fn_c_check_header_mongrel "$LINENO" "xfs/xfs.h" "ac_cv_header_xfs_xfs_h" "$ac_includes_default" +if test "x$ac_cv_header_xfs_xfs_h" = xyes; then : + +else + as_fn_error $? "xfs/xfs.h not found (--without-libxfs to disable)" "$LINENO" 5 +fi + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for XFS_XFLAG_EXTSIZE in xfs/xfs.h" >&5 +$as_echo_n "checking for XFS_XFLAG_EXTSIZE in xfs/xfs.h... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include + #ifdef XFS_XFLAG_EXTSIZE + yes_have_xfs_xflag_extsize + #endif + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "yes_have_xfs_xflag_extsize" >/dev/null 2>&1; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +$as_echo "#define HAVE_LIBXFS 1" >>confdefs.h + + +else + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "XFS_XFLAG_EXTSIZE not found (--without-libxfs to disable)" "$LINENO" 5 + +fi +rm -f conftest* + + +fi + if test "x$with_libxfs" != "xno"; then + WITH_LIBXFS_TRUE= + WITH_LIBXFS_FALSE='#' +else + WITH_LIBXFS_TRUE='#' + WITH_LIBXFS_FALSE= +fi + + +# use libzfs + +# Check whether --with-libzfs was given. +if test "${with_libzfs+set}" = set; then : + withval=$with_libzfs; +else + with_libzfs=no +fi + +if test "x$with_libzfs" = xyes; then : + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBZFS" >&5 +$as_echo_n "checking for LIBZFS... " >&6; } + +if test -n "$LIBZFS_CFLAGS"; then + pkg_cv_LIBZFS_CFLAGS="$LIBZFS_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zfs\""; } >&5 + ($PKG_CONFIG --exists --print-errors "zfs") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBZFS_CFLAGS=`$PKG_CONFIG --cflags "zfs" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LIBZFS_LIBS"; then + pkg_cv_LIBZFS_LIBS="$LIBZFS_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zfs\""; } >&5 + ($PKG_CONFIG --exists --print-errors "zfs") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBZFS_LIBS=`$PKG_CONFIG --libs "zfs" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LIBZFS_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zfs" 2>&1` + else + LIBZFS_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zfs" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LIBZFS_PKG_ERRORS" >&5 + + true +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + true +else + LIBZFS_CFLAGS=$pkg_cv_LIBZFS_CFLAGS + LIBZFS_LIBS=$pkg_cv_LIBZFS_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi +fi +if test "x$with_libzfs" = xyes; then : + +$as_echo "#define HAVE_LIBZFS 1" >>confdefs.h + +fi + if test "$with_libzfs" = "yes" ; then + WITH_LIBZFS_TRUE= + WITH_LIBZFS_FALSE='#' +else + WITH_LIBZFS_TRUE='#' + WITH_LIBZFS_FALSE= +fi + + +# Checks for header files. +ac_header_dirent=no +for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do + as_ac_Header=`$as_echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh` +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_hdr that defines DIR" >&5 +$as_echo_n "checking for $ac_hdr that defines DIR... " >&6; } +if eval \${$as_ac_Header+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include <$ac_hdr> + +int +main () +{ +if ((DIR *) 0) +return 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + eval "$as_ac_Header=yes" +else + eval "$as_ac_Header=no" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$as_ac_Header + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_hdr" | $as_tr_cpp` 1 +_ACEOF + +ac_header_dirent=$ac_hdr; break +fi + +done +# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix. +if test $ac_header_dirent = dirent.h; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing opendir" >&5 +$as_echo_n "checking for library containing opendir... " >&6; } +if ${ac_cv_search_opendir+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char opendir (); +int +main () +{ +return opendir (); + ; + return 0; +} +_ACEOF +for ac_lib in '' dir; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_opendir=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_opendir+:} false; then : + break +fi +done +if ${ac_cv_search_opendir+:} false; then : + +else + ac_cv_search_opendir=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5 +$as_echo "$ac_cv_search_opendir" >&6; } +ac_res=$ac_cv_search_opendir +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing opendir" >&5 +$as_echo_n "checking for library containing opendir... " >&6; } +if ${ac_cv_search_opendir+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char opendir (); +int +main () +{ +return opendir (); + ; + return 0; +} +_ACEOF +for ac_lib in '' x; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_opendir=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_opendir+:} false; then : + break +fi +done +if ${ac_cv_search_opendir+:} false; then : + +else + ac_cv_search_opendir=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5 +$as_echo "$ac_cv_search_opendir" >&6; } +ac_res=$ac_cv_search_opendir +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 +$as_echo_n "checking for ANSI C header files... " >&6; } +if ${ac_cv_header_stdc+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#include +#include + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_header_stdc=yes +else + ac_cv_header_stdc=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +if test $ac_cv_header_stdc = yes; then + # SunOS 4.x string.h does not declare mem*, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "memchr" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "free" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. + if test "$cross_compiling" = yes; then : + : +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#if ((' ' & 0x0FF) == 0x020) +# define ISLOWER(c) ('a' <= (c) && (c) <= 'z') +# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) +#else +# define ISLOWER(c) \ + (('a' <= (c) && (c) <= 'i') \ + || ('j' <= (c) && (c) <= 'r') \ + || ('s' <= (c) && (c) <= 'z')) +# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) +#endif + +#define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) +int +main () +{ + int i; + for (i = 0; i < 256; i++) + if (XOR (islower (i), ISLOWER (i)) + || toupper (i) != TOUPPER (i)) + return 2; + return 0; +} +_ACEOF +if ac_fn_c_try_run "$LINENO"; then : + +else + ac_cv_header_stdc=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5 +$as_echo "$ac_cv_header_stdc" >&6; } +if test $ac_cv_header_stdc = yes; then + +$as_echo "#define STDC_HEADERS 1" >>confdefs.h + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for sys/wait.h that is POSIX.1 compatible" >&5 +$as_echo_n "checking for sys/wait.h that is POSIX.1 compatible... " >&6; } +if ${ac_cv_header_sys_wait_h+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#ifndef WEXITSTATUS +# define WEXITSTATUS(stat_val) ((unsigned int) (stat_val) >> 8) +#endif +#ifndef WIFEXITED +# define WIFEXITED(stat_val) (((stat_val) & 255) == 0) +#endif + +int +main () +{ + int s; + wait (&s); + s = WIFEXITED (s) ? WEXITSTATUS (s) : 1; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_header_sys_wait_h=yes +else + ac_cv_header_sys_wait_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_sys_wait_h" >&5 +$as_echo "$ac_cv_header_sys_wait_h" >&6; } +if test $ac_cv_header_sys_wait_h = yes; then + +$as_echo "#define HAVE_SYS_WAIT_H 1" >>confdefs.h + +fi + + + +# spirit? +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + +ac_fn_cxx_check_header_mongrel "$LINENO" "boost/spirit/include/classic_core.hpp" "ac_cv_header_boost_spirit_include_classic_core_hpp" "$ac_includes_default" +if test "x$ac_cv_header_boost_spirit_include_classic_core_hpp" = xyes; then : + +else + ac_fn_cxx_check_header_mongrel "$LINENO" "boost/spirit.hpp" "ac_cv_header_boost_spirit_hpp" "$ac_includes_default" +if test "x$ac_cv_header_boost_spirit_hpp" = xyes; then : + use_bspirit_old_hdr=yes +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "\"Can't find boost spirit headers\" +See \`config.log' for more details" "$LINENO" 5; } +fi + + +fi + + + if test "$use_bspirit_old_hdr" = "yes"; then + USE_BOOST_SPIRIT_OLD_HDR_TRUE= + USE_BOOST_SPIRIT_OLD_HDR_FALSE='#' +else + USE_BOOST_SPIRIT_OLD_HDR_TRUE='#' + USE_BOOST_SPIRIT_OLD_HDR_FALSE= +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "boost/random/discrete_distribution.hpp" "ac_cv_header_boost_random_discrete_distribution_hpp" "$ac_includes_default" +if test "x$ac_cv_header_boost_random_discrete_distribution_hpp" = xyes; then : + +$as_echo "#define HAVE_BOOST_RANDOM_DISCRETE_DISTRIBUTION /**/" >>confdefs.h + +fi + + + +ac_fn_cxx_check_header_mongrel "$LINENO" "boost/statechart/state.hpp" "ac_cv_header_boost_statechart_state_hpp" "$ac_includes_default" +if test "x$ac_cv_header_boost_statechart_state_hpp" = xyes; then : + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "\"Can't find boost statechart headers; need 1.34 or later\" +See \`config.log' for more details" "$LINENO" 5; } +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "boost/program_options/option.hpp" "ac_cv_header_boost_program_options_option_hpp" "$ac_includes_default" +if test "x$ac_cv_header_boost_program_options_option_hpp" = xyes; then : + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "\"Can't find boost program_options headers\" +See \`config.log' for more details" "$LINENO" 5; } +fi + + + +# If we have the boost system library installed, then we may want to link +# with it. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_system-mt" >&5 +$as_echo_n "checking for main in -lboost_system-mt... " >&6; } +if ${ac_cv_lib_boost_system_mt_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lboost_system-mt $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_boost_system_mt_main=yes +else + ac_cv_lib_boost_system_mt_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_system_mt_main" >&5 +$as_echo "$ac_cv_lib_boost_system_mt_main" >&6; } +if test "x$ac_cv_lib_boost_system_mt_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBBOOST_SYSTEM_MT 1 +_ACEOF + + LIBS="-lboost_system-mt $LIBS" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_system" >&5 +$as_echo_n "checking for main in -lboost_system... " >&6; } +if ${ac_cv_lib_boost_system_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lboost_system $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_boost_system_main=yes +else + ac_cv_lib_boost_system_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_system_main" >&5 +$as_echo "$ac_cv_lib_boost_system_main" >&6; } +if test "x$ac_cv_lib_boost_system_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBBOOST_SYSTEM 1 +_ACEOF + + LIBS="-lboost_system $LIBS" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: \"Boost system library not found.\"" >&5 +$as_echo "$as_me: \"Boost system library not found.\"" >&6;} +fi + +fi + + +# Find the right boost_thread library. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_thread-mt" >&5 +$as_echo_n "checking for main in -lboost_thread-mt... " >&6; } +if ${ac_cv_lib_boost_thread_mt_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lboost_thread-mt $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_boost_thread_mt_main=yes +else + ac_cv_lib_boost_thread_mt_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_thread_mt_main" >&5 +$as_echo "$ac_cv_lib_boost_thread_mt_main" >&6; } +if test "x$ac_cv_lib_boost_thread_mt_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBBOOST_THREAD_MT 1 +_ACEOF + + LIBS="-lboost_thread-mt $LIBS" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_thread" >&5 +$as_echo_n "checking for main in -lboost_thread... " >&6; } +if ${ac_cv_lib_boost_thread_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lboost_thread $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_boost_thread_main=yes +else + ac_cv_lib_boost_thread_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_thread_main" >&5 +$as_echo "$ac_cv_lib_boost_thread_main" >&6; } +if test "x$ac_cv_lib_boost_thread_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBBOOST_THREAD 1 +_ACEOF + + LIBS="-lboost_thread $LIBS" + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "\"Boost thread library not found.\" +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi + + +# +# Check for boost_program_options library (defines BOOST_PROGRAM_OPTIONS_LIBS). +# +BOOST_PROGRAM_OPTIONS_LIBS="" +saved_LIBS="${LIBS}" +LIBS="" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_program_options-mt" >&5 +$as_echo_n "checking for main in -lboost_program_options-mt... " >&6; } +if ${ac_cv_lib_boost_program_options_mt_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lboost_program_options-mt $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_boost_program_options_mt_main=yes +else + ac_cv_lib_boost_program_options_mt_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_program_options_mt_main" >&5 +$as_echo "$ac_cv_lib_boost_program_options_mt_main" >&6; } +if test "x$ac_cv_lib_boost_program_options_mt_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBBOOST_PROGRAM_OPTIONS_MT 1 +_ACEOF + + LIBS="-lboost_program_options-mt $LIBS" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lboost_program_options" >&5 +$as_echo_n "checking for main in -lboost_program_options... " >&6; } +if ${ac_cv_lib_boost_program_options_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lboost_program_options $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_boost_program_options_main=yes +else + ac_cv_lib_boost_program_options_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_boost_program_options_main" >&5 +$as_echo "$ac_cv_lib_boost_program_options_main" >&6; } +if test "x$ac_cv_lib_boost_program_options_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBBOOST_PROGRAM_OPTIONS 1 +_ACEOF + + LIBS="-lboost_program_options $LIBS" + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "\"Boost program options library not found.\" +See \`config.log' for more details" "$LINENO" 5; } +fi + +fi + +BOOST_PROGRAM_OPTIONS_LIBS="${LIBS}" +LIBS="${saved_LIBS}" + + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +ac_fn_c_check_member "$LINENO" "struct fiemap_extent" "fe_logical" "ac_cv_member_struct_fiemap_extent_fe_logical" "#include +" +if test "x$ac_cv_member_struct_fiemap_extent_fe_logical" = xyes; then : + +$as_echo "#define HAVE_FIEMAP_H /**/" >>confdefs.h + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: linux/fiemap.h was not found or not usable; using local Ceph copy" >&5 +$as_echo "$as_me: linux/fiemap.h was not found or not usable; using local Ceph copy" >&6;} +fi + + +for ac_header in \ + arpa/inet.h \ + arpa/nameser_compat.h \ + linux/version.h \ + netdb.h \ + netinet/in.h \ + sys/file.h \ + sys/ioctl.h \ + sys/mount.h \ + sys/param.h \ + sys/socket.h \ + sys/statvfs.h \ + sys/time.h \ + sys/vfs.h \ + sys/xattr.h \ + syslog.h \ + utime.h \ + +do : + as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` +ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + +fi + +done + + +# sync_file_range +ac_fn_c_check_func "$LINENO" "sync_file_range" "ac_cv_func_sync_file_range" +if test "x$ac_cv_func_sync_file_range" = xyes; then : + +$as_echo "#define HAVE_SYNC_FILE_RANGE /**/" >>confdefs.h + +fi + + +# fallocate +ac_fn_c_check_func "$LINENO" "fallocate" "ac_cv_func_fallocate" +if test "x$ac_cv_func_fallocate" = xyes; then : + +$as_echo "#define CEPH_HAVE_FALLOCATE /**/" >>confdefs.h + +fi + + +# +# Test for time-related `struct stat` members. +# + +ac_fn_c_check_member "$LINENO" "struct stat" "st_mtim.tv_nsec" "ac_cv_member_struct_stat_st_mtim_tv_nsec" "$ac_includes_default" +if test "x$ac_cv_member_struct_stat_st_mtim_tv_nsec" = xyes; then : + +$as_echo "#define HAVE_STAT_ST_MTIM_TV_NSEC 1" >>confdefs.h + +fi + + +ac_fn_c_check_member "$LINENO" "struct stat" "st_mtimespec.tv_nsec" "ac_cv_member_struct_stat_st_mtimespec_tv_nsec" "$ac_includes_default" +if test "x$ac_cv_member_struct_stat_st_mtimespec_tv_nsec" = xyes; then : + +$as_echo "#define HAVE_STAT_ST_MTIMESPEC_TV_NSEC 1" >>confdefs.h + +fi + + +# splice/tee +ac_fn_c_check_func "$LINENO" "splice" "ac_cv_func_splice" +if test "x$ac_cv_func_splice" = xyes; then : + +$as_echo "#define CEPH_HAVE_SPLICE /**/" >>confdefs.h + +fi + + +# F_SETPIPE_SZ in fcntl.h +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for F_SETPIPE_SZ in fcntl.h" >&5 +$as_echo_n "checking for F_SETPIPE_SZ in fcntl.h... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #define _GNU_SOURCE + #include + #ifdef F_SETPIPE_SZ + yes_have_f_setpipe_sz + #endif + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "yes_have_f_setpipe_sz" >/dev/null 2>&1; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +$as_echo "#define CEPH_HAVE_SETPIPE_SZ /**/" >>confdefs.h + + +else + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: F_SETPIPE_SZ not found, zero-copy may be less efficent" >&5 +$as_echo "$as_me: F_SETPIPE_SZ not found, zero-copy may be less efficent" >&6;} + +fi +rm -f conftest* + + +for ac_func in posix_fallocate +do : + ac_fn_c_check_func "$LINENO" "posix_fallocate" "ac_cv_func_posix_fallocate" +if test "x$ac_cv_func_posix_fallocate" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_POSIX_FALLOCATE 1 +_ACEOF + +fi +done + +for ac_header in sys/prctl.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "sys/prctl.h" "ac_cv_header_sys_prctl_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_prctl_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_SYS_PRCTL_H 1 +_ACEOF + +fi + +done + +for ac_func in prctl +do : + ac_fn_c_check_func "$LINENO" "prctl" "ac_cv_func_prctl" +if test "x$ac_cv_func_prctl" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_PRCTL 1 +_ACEOF + +fi +done + +for ac_func in pipe2 +do : + ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" +if test "x$ac_cv_func_pipe2" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_PIPE2 1 +_ACEOF + +fi +done + +for ac_func in posix_fadvise +do : + ac_fn_c_check_func "$LINENO" "posix_fadvise" "ac_cv_func_posix_fadvise" +if test "x$ac_cv_func_posix_fadvise" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_POSIX_FADVISE 1 +_ACEOF + +fi +done + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for fdatasync" >&5 +$as_echo_n "checking for fdatasync... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include + +int +main () +{ + +#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0 +return fdatasync(0); +#else +#error Not supported +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +$as_echo "#define HAVE_FDATASYNC 1" >>confdefs.h + + +else + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +# +# Check for pthread spinlock (depends on ACX_PTHREAD) +# +saved_LIBS="$LIBS" +saved_CFLAGS="$CFLAGS" +LIBS="$PTHREAD_LIBS $LIBS" +CFLAGS="$PTHREAD_CFLAGS $CFLAGS" +ac_fn_c_check_func "$LINENO" "pthread_spin_init" "ac_cv_func_pthread_spin_init" +if test "x$ac_cv_func_pthread_spin_init" = xyes; then : + +$as_echo "#define HAVE_PTHREAD_SPINLOCK 1" >>confdefs.h + +fi + +LIBS="$saved_LIBS" +CFLAGS="$saved_CFLAGS" + +ac_fn_c_check_type "$LINENO" "int8_t" "ac_cv_type_int8_t" "$ac_includes_default" +if test "x$ac_cv_type_int8_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_INT8_T 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "uint8_t" "ac_cv_type_uint8_t" "$ac_includes_default" +if test "x$ac_cv_type_uint8_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_UINT8_T 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "int16_t" "ac_cv_type_int16_t" "$ac_includes_default" +if test "x$ac_cv_type_int16_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_INT16_T 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "uint16_t" "ac_cv_type_uint16_t" "$ac_includes_default" +if test "x$ac_cv_type_uint16_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_UINT16_T 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "int32_t" "ac_cv_type_int32_t" "$ac_includes_default" +if test "x$ac_cv_type_int32_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_INT32_T 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "uint32_t" "ac_cv_type_uint32_t" "$ac_includes_default" +if test "x$ac_cv_type_uint32_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_UINT32_T 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "int64_t" "ac_cv_type_int64_t" "$ac_includes_default" +if test "x$ac_cv_type_int64_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_INT64_T 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "uint64_t" "ac_cv_type_uint64_t" "$ac_includes_default" +if test "x$ac_cv_type_uint64_t" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_UINT64_T 1 +_ACEOF + + +fi + + +for ac_header in linux/types.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "linux/types.h" "ac_cv_header_linux_types_h" "$ac_includes_default" +if test "x$ac_cv_header_linux_types_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LINUX_TYPES_H 1 +_ACEOF + +fi + +done + +ac_fn_c_check_type "$LINENO" "__u8" "ac_cv_type___u8" "#include +" +if test "x$ac_cv_type___u8" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___U8 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__s8" "ac_cv_type___s8" "#include +" +if test "x$ac_cv_type___s8" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___S8 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__u16" "ac_cv_type___u16" "#include +" +if test "x$ac_cv_type___u16" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___U16 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__s16" "ac_cv_type___s16" "#include +" +if test "x$ac_cv_type___s16" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___S16 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__u32" "ac_cv_type___u32" "#include +" +if test "x$ac_cv_type___u32" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___U32 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__s32" "ac_cv_type___s32" "#include +" +if test "x$ac_cv_type___s32" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___S32 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__u64" "ac_cv_type___u64" "#include +" +if test "x$ac_cv_type___u64" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___U64 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__s64" "ac_cv_type___s64" "#include +" +if test "x$ac_cv_type___s64" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___S64 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__le16" "ac_cv_type___le16" "#include +" +if test "x$ac_cv_type___le16" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___LE16 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__be16" "ac_cv_type___be16" "#include +" +if test "x$ac_cv_type___be16" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___BE16 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__le32" "ac_cv_type___le32" "#include +" +if test "x$ac_cv_type___le32" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___LE32 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__be32" "ac_cv_type___be32" "#include +" +if test "x$ac_cv_type___be32" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___BE32 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__le64" "ac_cv_type___le64" "#include +" +if test "x$ac_cv_type___le64" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___LE64 1 +_ACEOF + + +fi +ac_fn_c_check_type "$LINENO" "__be64" "ac_cv_type___be64" "#include +" +if test "x$ac_cv_type___be64" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE___BE64 1 +_ACEOF + + +fi + + +# Checks for typedefs, structures, and compiler characteristics. +#AC_HEADER_STDBOOL +#AC_C_CONST +#AC_TYPE_UID_T +#AC_C_INLINE +#AC_TYPE_INT16_T +#AC_TYPE_INT32_T +#AC_TYPE_INT64_T +#AC_TYPE_INT8_T +#AC_TYPE_MODE_T +#AC_TYPE_OFF_T +#AC_TYPE_PID_T +#AC_TYPE_SIZE_T +#AC_TYPE_SSIZE_T +#AC_CHECK_MEMBERS([struct stat.st_blksize]) +#AC_STRUCT_ST_BLOCKS +#AC_CHECK_MEMBERS([struct stat.st_rdev]) +#AC_HEADER_TIME +#AC_STRUCT_TM +#AC_TYPE_UINT16_T +#AC_TYPE_UINT32_T +#AC_TYPE_UINT64_T +#AC_TYPE_UINT8_T + +# Checks for library functions. +#AC_FUNC_CHOWN +#AC_FUNC_CLOSEDIR_VOID +#AC_FUNC_ERROR_AT_LINE +#AC_FUNC_FORK +#AC_PROG_GCC_TRADITIONAL +#AC_FUNC_LSTAT +#AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK +#AC_FUNC_MALLOC # this causes annoying rpl_malloc error on some machines; skip it +#AC_FUNC_MEMCMP +#AC_FUNC_MMAP +#AC_FUNC_REALLOC +#AC_FUNC_SELECT_ARGTYPES +#AC_TYPE_SIGNAL +#AC_FUNC_STAT +#AC_FUNC_UTIME_NULL +#AC_CHECK_FUNCS([bzero fchdir fdatasync floor ftruncate getcwd gethostbyname gethostname gettimeofday inet_ntoa localtime_r memmove memset mkdir munmap pow rmdir select socket sqrt strcasecmp strchr strerror strstr utime]) + +# check for return type (and presence) if strerror_r in C++ mode +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + +ac_fn_cxx_check_decl "$LINENO" "strerror_r" "ac_cv_have_decl_strerror_r" "$ac_includes_default" +if test "x$ac_cv_have_decl_strerror_r" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_STRERROR_R $ac_have_decl +_ACEOF + +for ac_func in strerror_r +do : + ac_fn_cxx_check_func "$LINENO" "strerror_r" "ac_cv_func_strerror_r" +if test "x$ac_cv_func_strerror_r" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_STRERROR_R 1 +_ACEOF + +fi +done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether strerror_r returns char *" >&5 +$as_echo_n "checking whether strerror_r returns char *... " >&6; } +if ${ac_cv_func_strerror_r_char_p+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ac_cv_func_strerror_r_char_p=no + if test $ac_cv_have_decl_strerror_r = yes; then + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ + + char buf[100]; + char x = *strerror_r (0, buf, sizeof buf); + char *p = strerror_r (0, buf, sizeof buf); + return !p || x; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_func_strerror_r_char_p=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + else + # strerror_r is not declared. Choose between + # systems that have relatively inaccessible declarations for the + # function. BeOS and DEC UNIX 4.0 fall in this category, but the + # former has a strerror_r that returns char*, while the latter + # has a strerror_r that returns `int'. + # This test should segfault on the DEC system. + if test "$cross_compiling" = yes; then : + : +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_includes_default + extern char *strerror_r (); +int +main () +{ +char buf[100]; + char x = *strerror_r (0, buf, sizeof buf); + return ! isalpha (x); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_run "$LINENO"; then : + ac_cv_func_strerror_r_char_p=yes +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_strerror_r_char_p" >&5 +$as_echo "$ac_cv_func_strerror_r_char_p" >&6; } +if test $ac_cv_func_strerror_r_char_p = yes; then + +$as_echo "#define STRERROR_R_CHAR_P 1" >>confdefs.h + +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + + if test "$WITH_BUILD_TESTS" = "1"; then + WITH_BUILD_TESTS_TRUE= + WITH_BUILD_TESTS_FALSE='#' +else + WITH_BUILD_TESTS_TRUE='#' + WITH_BUILD_TESTS_FALSE= +fi + + + + + + + + + if test -n "$PYTHON"; then + # If the user set $PYTHON, use it and don't search something else. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $PYTHON version >= 2.4" >&5 +$as_echo_n "checking whether $PYTHON version >= 2.4... " >&6; } + prog="import sys +# split strings by '.' and convert to numeric. Append some zeros +# because we need at least 4 digits for the hex conversion. +# map returns an iterator in Python 3.0 and a list in 2.x +minver = list(map(int, '2.4'.split('.'))) + [0, 0, 0] +minverhex = 0 +# xrange is not present in Python 3.0 and range returns an iterator +for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[i] +sys.exit(sys.hexversion < minverhex)" + if { echo "$as_me:$LINENO: $PYTHON -c "$prog"" >&5 + ($PYTHON -c "$prog") >&5 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + as_fn_error $? "too old" "$LINENO" 5 +fi + am_display_PYTHON=$PYTHON + else + # Otherwise, try each interpreter until we find one that satisfies + # VERSION. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for a Python interpreter with version >= 2.4" >&5 +$as_echo_n "checking for a Python interpreter with version >= 2.4... " >&6; } +if ${am_cv_pathless_PYTHON+:} false; then : + $as_echo_n "(cached) " >&6 +else + + for am_cv_pathless_PYTHON in python python2 python3 python3.2 python3.1 python3.0 python2.7 python2.6 python2.5 python2.4 python2.3 python2.2 python2.1 python2.0 none; do + test "$am_cv_pathless_PYTHON" = none && break + prog="import sys +# split strings by '.' and convert to numeric. Append some zeros +# because we need at least 4 digits for the hex conversion. +# map returns an iterator in Python 3.0 and a list in 2.x +minver = list(map(int, '2.4'.split('.'))) + [0, 0, 0] +minverhex = 0 +# xrange is not present in Python 3.0 and range returns an iterator +for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[i] +sys.exit(sys.hexversion < minverhex)" + if { echo "$as_me:$LINENO: $am_cv_pathless_PYTHON -c "$prog"" >&5 + ($am_cv_pathless_PYTHON -c "$prog") >&5 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then : + break +fi + done +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_pathless_PYTHON" >&5 +$as_echo "$am_cv_pathless_PYTHON" >&6; } + # Set $PYTHON to the absolute path of $am_cv_pathless_PYTHON. + if test "$am_cv_pathless_PYTHON" = none; then + PYTHON=: + else + # Extract the first word of "$am_cv_pathless_PYTHON", so it can be a program name with args. +set dummy $am_cv_pathless_PYTHON; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTHON+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTHON in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTHON="$PYTHON" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_path_PYTHON="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTHON=$ac_cv_path_PYTHON +if test -n "$PYTHON"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON" >&5 +$as_echo "$PYTHON" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + fi + am_display_PYTHON=$am_cv_pathless_PYTHON + fi + + + if test "$PYTHON" = :; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "Failed to find Python 2.4 or newer +See \`config.log' for more details" "$LINENO" 5; } + else + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $am_display_PYTHON version" >&5 +$as_echo_n "checking for $am_display_PYTHON version... " >&6; } +if ${am_cv_python_version+:} false; then : + $as_echo_n "(cached) " >&6 +else + am_cv_python_version=`$PYTHON -c "import sys; sys.stdout.write(sys.version[:3])"` +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_python_version" >&5 +$as_echo "$am_cv_python_version" >&6; } + PYTHON_VERSION=$am_cv_python_version + + + + PYTHON_PREFIX='${prefix}' + + PYTHON_EXEC_PREFIX='${exec_prefix}' + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $am_display_PYTHON platform" >&5 +$as_echo_n "checking for $am_display_PYTHON platform... " >&6; } +if ${am_cv_python_platform+:} false; then : + $as_echo_n "(cached) " >&6 +else + am_cv_python_platform=`$PYTHON -c "import sys; sys.stdout.write(sys.platform)"` +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_python_platform" >&5 +$as_echo "$am_cv_python_platform" >&6; } + PYTHON_PLATFORM=$am_cv_python_platform + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $am_display_PYTHON script directory" >&5 +$as_echo_n "checking for $am_display_PYTHON script directory... " >&6; } +if ${am_cv_python_pythondir+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "x$prefix" = xNONE + then + am_py_prefix=$ac_default_prefix + else + am_py_prefix=$prefix + fi + am_cv_python_pythondir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(0,0,prefix='$am_py_prefix'))" 2>/dev/null` + case $am_cv_python_pythondir in + $am_py_prefix*) + am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'` + am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,$PYTHON_PREFIX,"` + ;; + *) + case $am_py_prefix in + /usr|/System*) ;; + *) + am_cv_python_pythondir=$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages + ;; + esac + ;; + esac + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_python_pythondir" >&5 +$as_echo "$am_cv_python_pythondir" >&6; } + pythondir=$am_cv_python_pythondir + + + + pkgpythondir=\${pythondir}/$PACKAGE + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $am_display_PYTHON extension module directory" >&5 +$as_echo_n "checking for $am_display_PYTHON extension module directory... " >&6; } +if ${am_cv_python_pyexecdir+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "x$exec_prefix" = xNONE + then + am_py_exec_prefix=$am_py_prefix + else + am_py_exec_prefix=$exec_prefix + fi + am_cv_python_pyexecdir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(1,0,prefix='$am_py_exec_prefix'))" 2>/dev/null` + case $am_cv_python_pyexecdir in + $am_py_exec_prefix*) + am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'` + am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,$PYTHON_EXEC_PREFIX,"` + ;; + *) + case $am_py_exec_prefix in + /usr|/System*) ;; + *) + am_cv_python_pyexecdir=$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages + ;; + esac + ;; + esac + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_python_pyexecdir" >&5 +$as_echo "$am_cv_python_pyexecdir" >&6; } + pyexecdir=$am_cv_python_pyexecdir + + + + pkgpyexecdir=\${pyexecdir}/$PACKAGE + + + + fi + + + +ac_config_headers="$ac_config_headers src/acconfig.h" + +ac_config_files="$ac_config_files Makefile src/Makefile src/ocf/Makefile src/ocf/ceph src/ocf/rbd src/java/Makefile man/Makefile ceph.spec" + +cat >confcache <<\_ACEOF +# This file is a shell script that caches the results of configure +# tests run on this system so they can be shared between configure +# scripts and configure runs, see configure's option --config-cache. +# It is not useful on other systems. If it contains results you don't +# want to keep, you may remove or edit it. +# +# config.status only pays attention to the cache file if you give it +# the --recheck option to rerun configure. +# +# `ac_cv_env_foo' variables (set or unset) will be overridden when +# loading this file, other *unset* `ac_cv_foo' will be assigned the +# following values. + +_ACEOF + +# The following way of writing the cache mishandles newlines in values, +# but we know of no workaround that is simple, portable, and efficient. +# So, we kill variables containing newlines. +# Ultrix sh set writes to stderr and can't be redirected directly, +# and sets the high bit in the cache file unless we assign to the vars. +( + for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do + eval ac_val=\$$ac_var + case $ac_val in #( + *${as_nl}*) + case $ac_var in #( + *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 +$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; + esac + case $ac_var in #( + _ | IFS | as_nl) ;; #( + BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( + *) { eval $ac_var=; unset $ac_var;} ;; + esac ;; + esac + done + + (set) 2>&1 | + case $as_nl`(ac_space=' '; set) 2>&1` in #( + *${as_nl}ac_space=\ *) + # `set' does not quote correctly, so add quotes: double-quote + # substitution turns \\\\ into \\, and sed turns \\ into \. + sed -n \ + "s/'/'\\\\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" + ;; #( + *) + # `set' quotes correctly as required by POSIX, so do not add quotes. + sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" + ;; + esac | + sort +) | + sed ' + /^ac_cv_env_/b end + t clear + :clear + s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ + t end + s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ + :end' >>confcache +if diff "$cache_file" confcache >/dev/null 2>&1; then :; else + if test -w "$cache_file"; then + if test "x$cache_file" != "x/dev/null"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 +$as_echo "$as_me: updating cache $cache_file" >&6;} + if test ! -f "$cache_file" || test -h "$cache_file"; then + cat confcache >"$cache_file" + else + case $cache_file in #( + */* | ?:*) + mv -f confcache "$cache_file"$$ && + mv -f "$cache_file"$$ "$cache_file" ;; #( + *) + mv -f confcache "$cache_file" ;; + esac + fi + fi + else + { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 +$as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} + fi +fi +rm -f confcache + +test "x$prefix" = xNONE && prefix=$ac_default_prefix +# Let make expand exec_prefix. +test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' + +DEFS=-DHAVE_CONFIG_H + +ac_libobjs= +ac_ltlibobjs= +U= +for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue + # 1. Remove the extension, and $U if already installed. + ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' + ac_i=`$as_echo "$ac_i" | sed "$ac_script"` + # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR + # will be set to the directory where LIBOBJS objects are built. + as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext" + as_fn_append ac_ltlibobjs " \${LIBOBJDIR}$ac_i"'$U.lo' +done +LIBOBJS=$ac_libobjs + +LTLIBOBJS=$ac_ltlibobjs + + +if test -z "${AMDEP_TRUE}" && test -z "${AMDEP_FALSE}"; then + as_fn_error $? "conditional \"AMDEP\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then + as_fn_error $? "conditional \"am__fastdepCC\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi + if test -n "$EXEEXT"; then + am__EXEEXT_TRUE= + am__EXEEXT_FALSE='#' +else + am__EXEEXT_TRUE='#' + am__EXEEXT_FALSE= +fi + +if test -z "${am__fastdepCCAS_TRUE}" && test -z "${am__fastdepCCAS_FALSE}"; then + as_fn_error $? "conditional \"am__fastdepCCAS\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then + as_fn_error $? "conditional \"am__fastdepCC\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi + if test -n "$EXEEXT"; then + am__EXEEXT_TRUE= + am__EXEEXT_FALSE='#' +else + am__EXEEXT_TRUE='#' + am__EXEEXT_FALSE= +fi + +if test -z "${LINUX_TRUE}" && test -z "${LINUX_FALSE}"; then + as_fn_error $? "conditional \"LINUX\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${FREEBSD_TRUE}" && test -z "${FREEBSD_FALSE}"; then + as_fn_error $? "conditional \"FREEBSD\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${am__fastdepCXX_TRUE}" && test -z "${am__fastdepCXX_FALSE}"; then + as_fn_error $? "conditional \"am__fastdepCXX\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${am__fastdepCXX_TRUE}" && test -z "${am__fastdepCXX_FALSE}"; then + as_fn_error $? "conditional \"am__fastdepCXX\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${CLANG_TRUE}" && test -z "${CLANG_FALSE}"; then + as_fn_error $? "conditional \"CLANG\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_GOOD_YASM_ELF64_TRUE}" && test -z "${WITH_GOOD_YASM_ELF64_FALSE}"; then + as_fn_error $? "conditional \"WITH_GOOD_YASM_ELF64\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${ENABLE_FPU_NEON_TRUE}" && test -z "${ENABLE_FPU_NEON_FALSE}"; then + as_fn_error $? "conditional \"ENABLE_FPU_NEON\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${COMPILER_HAS_VTA_TRUE}" && test -z "${COMPILER_HAS_VTA_FALSE}"; then + as_fn_error $? "conditional \"COMPILER_HAS_VTA\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_PROFILER_TRUE}" && test -z "${WITH_PROFILER_FALSE}"; then + as_fn_error $? "conditional \"WITH_PROFILER\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_DEBUG_TRUE}" && test -z "${WITH_DEBUG_FALSE}"; then + as_fn_error $? "conditional \"WITH_DEBUG\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${ENABLE_COVERAGE_TRUE}" && test -z "${ENABLE_COVERAGE_FALSE}"; then + as_fn_error $? "conditional \"ENABLE_COVERAGE\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_RADOSGW_TRUE}" && test -z "${WITH_RADOSGW_FALSE}"; then + as_fn_error $? "conditional \"WITH_RADOSGW\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_FUSE_TRUE}" && test -z "${WITH_FUSE_FALSE}"; then + as_fn_error $? "conditional \"WITH_FUSE\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_TCMALLOC_TRUE}" && test -z "${WITH_TCMALLOC_FALSE}"; then + as_fn_error $? "conditional \"WITH_TCMALLOC\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${ENABLE_CEPHFS_JAVA_TRUE}" && test -z "${ENABLE_CEPHFS_JAVA_FALSE}"; then + as_fn_error $? "conditional \"ENABLE_CEPHFS_JAVA\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${HAVE_JUNIT4_TRUE}" && test -z "${HAVE_JUNIT4_FALSE}"; then + as_fn_error $? "conditional \"HAVE_JUNIT4\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_LIBATOMIC_TRUE}" && test -z "${WITH_LIBATOMIC_FALSE}"; then + as_fn_error $? "conditional \"WITH_LIBATOMIC\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_OCF_TRUE}" && test -z "${WITH_OCF_FALSE}"; then + as_fn_error $? "conditional \"WITH_OCF\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_SYSTEM_LIBS3_TRUE}" && test -z "${WITH_SYSTEM_LIBS3_FALSE}"; then + as_fn_error $? "conditional \"WITH_SYSTEM_LIBS3\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_REST_BENCH_TRUE}" && test -z "${WITH_REST_BENCH_FALSE}"; then + as_fn_error $? "conditional \"WITH_REST_BENCH\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_LIBAIO_TRUE}" && test -z "${WITH_LIBAIO_FALSE}"; then + as_fn_error $? "conditional \"WITH_LIBAIO\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_LIBXFS_TRUE}" && test -z "${WITH_LIBXFS_FALSE}"; then + as_fn_error $? "conditional \"WITH_LIBXFS\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_LIBZFS_TRUE}" && test -z "${WITH_LIBZFS_FALSE}"; then + as_fn_error $? "conditional \"WITH_LIBZFS\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${USE_BOOST_SPIRIT_OLD_HDR_TRUE}" && test -z "${USE_BOOST_SPIRIT_OLD_HDR_FALSE}"; then + as_fn_error $? "conditional \"USE_BOOST_SPIRIT_OLD_HDR\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${WITH_BUILD_TESTS_TRUE}" && test -z "${WITH_BUILD_TESTS_FALSE}"; then + as_fn_error $? "conditional \"WITH_BUILD_TESTS\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi + +: "${CONFIG_STATUS=./config.status}" +ac_write_fail=0 +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files $CONFIG_STATUS" +{ $as_echo "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5 +$as_echo "$as_me: creating $CONFIG_STATUS" >&6;} +as_write_fail=0 +cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1 +#! $SHELL +# Generated by $as_me. +# Run this file to recreate the current configuration. +# Compiler output produced by configure, useful for debugging +# configure, is in config.log if it exists. + +debug=false +ac_cs_recheck=false +ac_cs_silent=false + +SHELL=\${CONFIG_SHELL-$SHELL} +export SHELL +_ASEOF +cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 +## -------------------- ## +## M4sh Initialization. ## +## -------------------- ## + +# Be more Bourne compatible +DUALCASE=1; export DUALCASE # for MKS sh +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in #( + *posix*) : + set -o posix ;; #( + *) : + ;; +esac +fi + + +as_nl=' +' +export as_nl +# Printing a long string crashes Solaris 7 /usr/bin/printf. +as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo +# Prefer a ksh shell builtin over an external printf program on Solaris, +# but without wasting forks for bash or zsh. +if test -z "$BASH_VERSION$ZSH_VERSION" \ + && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='print -r --' + as_echo_n='print -rn --' +elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='printf %s\n' + as_echo_n='printf %s' +else + if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then + as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' + as_echo_n='/usr/ucb/echo -n' + else + as_echo_body='eval expr "X$1" : "X\\(.*\\)"' + as_echo_n_body='eval + arg=$1; + case $arg in #( + *"$as_nl"*) + expr "X$arg" : "X\\(.*\\)$as_nl"; + arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; + esac; + expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" + ' + export as_echo_n_body + as_echo_n='sh -c $as_echo_n_body as_echo' + fi + export as_echo_body + as_echo='sh -c $as_echo_body as_echo' +fi + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + PATH_SEPARATOR=: + (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { + (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || + PATH_SEPARATOR=';' + } +fi + + +# IFS +# We need space, tab and new line, in precisely that order. Quoting is +# there to prevent editors from complaining about space-tab. +# (If _AS_PATH_WALK were called with IFS unset, it would disable word +# splitting by setting IFS to empty value.) +IFS=" "" $as_nl" + +# Find who we are. Look in the path if we contain no directory separator. +as_myself= +case $0 in #(( + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break + done +IFS=$as_save_IFS + + ;; +esac +# We did not find ourselves, most probably we were run as `sh COMMAND' +# in which case we are not to be found in the path. +if test "x$as_myself" = x; then + as_myself=$0 +fi +if test ! -f "$as_myself"; then + $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + exit 1 +fi + +# Unset variables that we do not need and which cause bugs (e.g. in +# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" +# suppresses any "Segmentation fault" message there. '((' could +# trigger a bug in pdksh 5.2.14. +for as_var in BASH_ENV ENV MAIL MAILPATH +do eval test x\${$as_var+set} = xset \ + && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : +done +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +LC_ALL=C +export LC_ALL +LANGUAGE=C +export LANGUAGE + +# CDPATH. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + + +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- +# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are +# provided, also output the error to LOG_FD, referencing LINENO. Then exit the +# script with STATUS, using 1 if that was 0. +as_fn_error () +{ + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + fi + $as_echo "$as_me: error: $2" >&2 + as_fn_exit $as_status +} # as_fn_error + + +# as_fn_set_status STATUS +# ----------------------- +# Set $? to STATUS, without forking. +as_fn_set_status () +{ + return $1 +} # as_fn_set_status + +# as_fn_exit STATUS +# ----------------- +# Exit the shell with STATUS, even in a "trap 0" or "set -e" context. +as_fn_exit () +{ + set +e + as_fn_set_status $1 + exit $1 +} # as_fn_exit + +# as_fn_unset VAR +# --------------- +# Portably unset VAR. +as_fn_unset () +{ + { eval $1=; unset $1;} +} +as_unset=as_fn_unset +# as_fn_append VAR VALUE +# ---------------------- +# Append the text in VALUE to the end of the definition contained in VAR. Take +# advantage of any shell optimizations that allow amortized linear growth over +# repeated appends, instead of the typical quadratic growth present in naive +# implementations. +if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : + eval 'as_fn_append () + { + eval $1+=\$2 + }' +else + as_fn_append () + { + eval $1=\$$1\$2 + } +fi # as_fn_append + +# as_fn_arith ARG... +# ------------------ +# Perform arithmetic evaluation on the ARGs, and store the result in the +# global $as_val. Take advantage of shells that can avoid forks. The arguments +# must be portable across $(()) and expr. +if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : + eval 'as_fn_arith () + { + as_val=$(( $* )) + }' +else + as_fn_arith () + { + as_val=`expr "$@" || test $? -eq 1` + } +fi # as_fn_arith + + +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + +if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then + as_dirname=dirname +else + as_dirname=false +fi + +as_me=`$as_basename -- "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ + s//\1/ + q + } + /^X\/\(\/\/\)$/{ + s//\1/ + q + } + /^X\/\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + +ECHO_C= ECHO_N= ECHO_T= +case `echo -n x` in #((((( +-n*) + case `echo 'xy\c'` in + *c*) ECHO_T=' ';; # ECHO_T is single tab character. + xy) ECHO_C='\c';; + *) echo `echo ksh88 bug on AIX 6.1` > /dev/null + ECHO_T=' ';; + esac;; +*) + ECHO_N='-n';; +esac + +rm -f conf$$ conf$$.exe conf$$.file +if test -d conf$$.dir; then + rm -f conf$$.dir/conf$$.file +else + rm -f conf$$.dir + mkdir conf$$.dir 2>/dev/null +fi +if (echo >conf$$.file) 2>/dev/null; then + if ln -s conf$$.file conf$$ 2>/dev/null; then + as_ln_s='ln -s' + # ... but there are two gotchas: + # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. + # In both cases, we have to default to `cp -p'. + ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || + as_ln_s='cp -p' + elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln + else + as_ln_s='cp -p' + fi +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file +rmdir conf$$.dir 2>/dev/null + + +# as_fn_mkdir_p +# ------------- +# Create "$as_dir" as a directory, including parents if necessary. +as_fn_mkdir_p () +{ + + case $as_dir in #( + -*) as_dir=./$as_dir;; + esac + test -d "$as_dir" || eval $as_mkdir_p || { + as_dirs= + while :; do + case $as_dir in #( + *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *) as_qdir=$as_dir;; + esac + as_dirs="'$as_qdir' $as_dirs" + as_dir=`$as_dirname -- "$as_dir" || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + test -d "$as_dir" && break + done + test -z "$as_dirs" || eval "mkdir $as_dirs" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" + + +} # as_fn_mkdir_p +if mkdir -p . 2>/dev/null; then + as_mkdir_p='mkdir -p "$as_dir"' +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +if test -x / >/dev/null 2>&1; then + as_test_x='test -x' +else + if ls -dL / >/dev/null 2>&1; then + as_ls_L_option=L + else + as_ls_L_option= + fi + as_test_x=' + eval sh -c '\'' + if test -d "$1"; then + test -d "$1/."; + else + case $1 in #( + -*)set "./$1";; + esac; + case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #(( + ???[sx]*):;;*)false;;esac;fi + '\'' sh + ' +fi +as_executable_p=$as_test_x + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + +exec 6>&1 +## ----------------------------------- ## +## Main body of $CONFIG_STATUS script. ## +## ----------------------------------- ## +_ASEOF +test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# Save the log message, to keep $0 and so on meaningful, and to +# report actual input values of CONFIG_FILES etc. instead of their +# values after options handling. +ac_log=" +This file was extended by ceph $as_me 0.80.7, which was +generated by GNU Autoconf 2.68. Invocation command line was + + CONFIG_FILES = $CONFIG_FILES + CONFIG_HEADERS = $CONFIG_HEADERS + CONFIG_LINKS = $CONFIG_LINKS + CONFIG_COMMANDS = $CONFIG_COMMANDS + $ $0 $@ + +on `(hostname || uname -n) 2>/dev/null | sed 1q` +" + +_ACEOF + +case $ac_config_files in *" +"*) set x $ac_config_files; shift; ac_config_files=$*;; +esac + +case $ac_config_headers in *" +"*) set x $ac_config_headers; shift; ac_config_headers=$*;; +esac + + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +# Files that config.status was made for. +config_files="$ac_config_files" +config_headers="$ac_config_headers" +config_commands="$ac_config_commands" + +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +ac_cs_usage="\ +\`$as_me' instantiates files and other configuration actions +from templates according to the current configuration. Unless the files +and actions are specified as TAGs, all are instantiated by default. + +Usage: $0 [OPTION]... [TAG]... + + -h, --help print this help, then exit + -V, --version print version number and configuration settings, then exit + --config print configuration, then exit + -q, --quiet, --silent + do not print progress messages + -d, --debug don't remove temporary files + --recheck update $as_me by reconfiguring in the same conditions + --file=FILE[:TEMPLATE] + instantiate the configuration file FILE + --header=FILE[:TEMPLATE] + instantiate the configuration header FILE + +Configuration files: +$config_files + +Configuration headers: +$config_headers + +Configuration commands: +$config_commands + +Report bugs to ." + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" +ac_cs_version="\\ +ceph config.status 0.80.7 +configured by $0, generated by GNU Autoconf 2.68, + with options \\"\$ac_cs_config\\" + +Copyright (C) 2010 Free Software Foundation, Inc. +This config.status script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it." + +ac_pwd='$ac_pwd' +srcdir='$srcdir' +INSTALL='$INSTALL' +MKDIR_P='$MKDIR_P' +AWK='$AWK' +test -n "\$AWK" || AWK=awk +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# The default lists apply if the user does not specify any file. +ac_need_defaults=: +while test $# != 0 +do + case $1 in + --*=?*) + ac_option=`expr "X$1" : 'X\([^=]*\)='` + ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` + ac_shift=: + ;; + --*=) + ac_option=`expr "X$1" : 'X\([^=]*\)='` + ac_optarg= + ac_shift=: + ;; + *) + ac_option=$1 + ac_optarg=$2 + ac_shift=shift + ;; + esac + + case $ac_option in + # Handling of the options. + -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) + ac_cs_recheck=: ;; + --version | --versio | --versi | --vers | --ver | --ve | --v | -V ) + $as_echo "$ac_cs_version"; exit ;; + --config | --confi | --conf | --con | --co | --c ) + $as_echo "$ac_cs_config"; exit ;; + --debug | --debu | --deb | --de | --d | -d ) + debug=: ;; + --file | --fil | --fi | --f ) + $ac_shift + case $ac_optarg in + *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + '') as_fn_error $? "missing file argument" ;; + esac + as_fn_append CONFIG_FILES " '$ac_optarg'" + ac_need_defaults=false;; + --header | --heade | --head | --hea ) + $ac_shift + case $ac_optarg in + *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + as_fn_append CONFIG_HEADERS " '$ac_optarg'" + ac_need_defaults=false;; + --he | --h) + # Conflict between --help and --header + as_fn_error $? "ambiguous option: \`$1' +Try \`$0 --help' for more information.";; + --help | --hel | -h ) + $as_echo "$ac_cs_usage"; exit ;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil | --si | --s) + ac_cs_silent=: ;; + + # This is an error. + -*) as_fn_error $? "unrecognized option: \`$1' +Try \`$0 --help' for more information." ;; + + *) as_fn_append ac_config_targets " $1" + ac_need_defaults=false ;; + + esac + shift +done + +ac_configure_extra_args= + +if $ac_cs_silent; then + exec 6>/dev/null + ac_configure_extra_args="$ac_configure_extra_args --silent" +fi + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +if \$ac_cs_recheck; then + set X '$SHELL' '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion + shift + \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6 + CONFIG_SHELL='$SHELL' + export CONFIG_SHELL + exec "\$@" +fi + +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +exec 5>>config.log +{ + echo + sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX +## Running $as_me. ## +_ASBOX + $as_echo "$ac_log" +} >&5 + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +# +# INIT-COMMANDS +# +AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir" + + +# The HP-UX ksh and POSIX shell print the target directory to stdout +# if CDPATH is set. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +sed_quote_subst='$sed_quote_subst' +double_quote_subst='$double_quote_subst' +delay_variable_subst='$delay_variable_subst' +macro_version='`$ECHO "$macro_version" | $SED "$delay_single_quote_subst"`' +macro_revision='`$ECHO "$macro_revision" | $SED "$delay_single_quote_subst"`' +enable_shared='`$ECHO "$enable_shared" | $SED "$delay_single_quote_subst"`' +enable_static='`$ECHO "$enable_static" | $SED "$delay_single_quote_subst"`' +pic_mode='`$ECHO "$pic_mode" | $SED "$delay_single_quote_subst"`' +enable_fast_install='`$ECHO "$enable_fast_install" | $SED "$delay_single_quote_subst"`' +SHELL='`$ECHO "$SHELL" | $SED "$delay_single_quote_subst"`' +ECHO='`$ECHO "$ECHO" | $SED "$delay_single_quote_subst"`' +PATH_SEPARATOR='`$ECHO "$PATH_SEPARATOR" | $SED "$delay_single_quote_subst"`' +host_alias='`$ECHO "$host_alias" | $SED "$delay_single_quote_subst"`' +host='`$ECHO "$host" | $SED "$delay_single_quote_subst"`' +host_os='`$ECHO "$host_os" | $SED "$delay_single_quote_subst"`' +build_alias='`$ECHO "$build_alias" | $SED "$delay_single_quote_subst"`' +build='`$ECHO "$build" | $SED "$delay_single_quote_subst"`' +build_os='`$ECHO "$build_os" | $SED "$delay_single_quote_subst"`' +SED='`$ECHO "$SED" | $SED "$delay_single_quote_subst"`' +Xsed='`$ECHO "$Xsed" | $SED "$delay_single_quote_subst"`' +GREP='`$ECHO "$GREP" | $SED "$delay_single_quote_subst"`' +EGREP='`$ECHO "$EGREP" | $SED "$delay_single_quote_subst"`' +FGREP='`$ECHO "$FGREP" | $SED "$delay_single_quote_subst"`' +LD='`$ECHO "$LD" | $SED "$delay_single_quote_subst"`' +NM='`$ECHO "$NM" | $SED "$delay_single_quote_subst"`' +LN_S='`$ECHO "$LN_S" | $SED "$delay_single_quote_subst"`' +max_cmd_len='`$ECHO "$max_cmd_len" | $SED "$delay_single_quote_subst"`' +ac_objext='`$ECHO "$ac_objext" | $SED "$delay_single_quote_subst"`' +exeext='`$ECHO "$exeext" | $SED "$delay_single_quote_subst"`' +lt_unset='`$ECHO "$lt_unset" | $SED "$delay_single_quote_subst"`' +lt_SP2NL='`$ECHO "$lt_SP2NL" | $SED "$delay_single_quote_subst"`' +lt_NL2SP='`$ECHO "$lt_NL2SP" | $SED "$delay_single_quote_subst"`' +lt_cv_to_host_file_cmd='`$ECHO "$lt_cv_to_host_file_cmd" | $SED "$delay_single_quote_subst"`' +lt_cv_to_tool_file_cmd='`$ECHO "$lt_cv_to_tool_file_cmd" | $SED "$delay_single_quote_subst"`' +reload_flag='`$ECHO "$reload_flag" | $SED "$delay_single_quote_subst"`' +reload_cmds='`$ECHO "$reload_cmds" | $SED "$delay_single_quote_subst"`' +OBJDUMP='`$ECHO "$OBJDUMP" | $SED "$delay_single_quote_subst"`' +deplibs_check_method='`$ECHO "$deplibs_check_method" | $SED "$delay_single_quote_subst"`' +file_magic_cmd='`$ECHO "$file_magic_cmd" | $SED "$delay_single_quote_subst"`' +file_magic_glob='`$ECHO "$file_magic_glob" | $SED "$delay_single_quote_subst"`' +want_nocaseglob='`$ECHO "$want_nocaseglob" | $SED "$delay_single_quote_subst"`' +DLLTOOL='`$ECHO "$DLLTOOL" | $SED "$delay_single_quote_subst"`' +sharedlib_from_linklib_cmd='`$ECHO "$sharedlib_from_linklib_cmd" | $SED "$delay_single_quote_subst"`' +AR='`$ECHO "$AR" | $SED "$delay_single_quote_subst"`' +AR_FLAGS='`$ECHO "$AR_FLAGS" | $SED "$delay_single_quote_subst"`' +archiver_list_spec='`$ECHO "$archiver_list_spec" | $SED "$delay_single_quote_subst"`' +STRIP='`$ECHO "$STRIP" | $SED "$delay_single_quote_subst"`' +RANLIB='`$ECHO "$RANLIB" | $SED "$delay_single_quote_subst"`' +old_postinstall_cmds='`$ECHO "$old_postinstall_cmds" | $SED "$delay_single_quote_subst"`' +old_postuninstall_cmds='`$ECHO "$old_postuninstall_cmds" | $SED "$delay_single_quote_subst"`' +old_archive_cmds='`$ECHO "$old_archive_cmds" | $SED "$delay_single_quote_subst"`' +lock_old_archive_extraction='`$ECHO "$lock_old_archive_extraction" | $SED "$delay_single_quote_subst"`' +CC='`$ECHO "$CC" | $SED "$delay_single_quote_subst"`' +CFLAGS='`$ECHO "$CFLAGS" | $SED "$delay_single_quote_subst"`' +compiler='`$ECHO "$compiler" | $SED "$delay_single_quote_subst"`' +GCC='`$ECHO "$GCC" | $SED "$delay_single_quote_subst"`' +lt_cv_sys_global_symbol_pipe='`$ECHO "$lt_cv_sys_global_symbol_pipe" | $SED "$delay_single_quote_subst"`' +lt_cv_sys_global_symbol_to_cdecl='`$ECHO "$lt_cv_sys_global_symbol_to_cdecl" | $SED "$delay_single_quote_subst"`' +lt_cv_sys_global_symbol_to_c_name_address='`$ECHO "$lt_cv_sys_global_symbol_to_c_name_address" | $SED "$delay_single_quote_subst"`' +lt_cv_sys_global_symbol_to_c_name_address_lib_prefix='`$ECHO "$lt_cv_sys_global_symbol_to_c_name_address_lib_prefix" | $SED "$delay_single_quote_subst"`' +nm_file_list_spec='`$ECHO "$nm_file_list_spec" | $SED "$delay_single_quote_subst"`' +lt_sysroot='`$ECHO "$lt_sysroot" | $SED "$delay_single_quote_subst"`' +objdir='`$ECHO "$objdir" | $SED "$delay_single_quote_subst"`' +MAGIC_CMD='`$ECHO "$MAGIC_CMD" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_no_builtin_flag='`$ECHO "$lt_prog_compiler_no_builtin_flag" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_pic='`$ECHO "$lt_prog_compiler_pic" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_wl='`$ECHO "$lt_prog_compiler_wl" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_static='`$ECHO "$lt_prog_compiler_static" | $SED "$delay_single_quote_subst"`' +lt_cv_prog_compiler_c_o='`$ECHO "$lt_cv_prog_compiler_c_o" | $SED "$delay_single_quote_subst"`' +need_locks='`$ECHO "$need_locks" | $SED "$delay_single_quote_subst"`' +MANIFEST_TOOL='`$ECHO "$MANIFEST_TOOL" | $SED "$delay_single_quote_subst"`' +DSYMUTIL='`$ECHO "$DSYMUTIL" | $SED "$delay_single_quote_subst"`' +NMEDIT='`$ECHO "$NMEDIT" | $SED "$delay_single_quote_subst"`' +LIPO='`$ECHO "$LIPO" | $SED "$delay_single_quote_subst"`' +OTOOL='`$ECHO "$OTOOL" | $SED "$delay_single_quote_subst"`' +OTOOL64='`$ECHO "$OTOOL64" | $SED "$delay_single_quote_subst"`' +libext='`$ECHO "$libext" | $SED "$delay_single_quote_subst"`' +shrext_cmds='`$ECHO "$shrext_cmds" | $SED "$delay_single_quote_subst"`' +extract_expsyms_cmds='`$ECHO "$extract_expsyms_cmds" | $SED "$delay_single_quote_subst"`' +archive_cmds_need_lc='`$ECHO "$archive_cmds_need_lc" | $SED "$delay_single_quote_subst"`' +enable_shared_with_static_runtimes='`$ECHO "$enable_shared_with_static_runtimes" | $SED "$delay_single_quote_subst"`' +export_dynamic_flag_spec='`$ECHO "$export_dynamic_flag_spec" | $SED "$delay_single_quote_subst"`' +whole_archive_flag_spec='`$ECHO "$whole_archive_flag_spec" | $SED "$delay_single_quote_subst"`' +compiler_needs_object='`$ECHO "$compiler_needs_object" | $SED "$delay_single_quote_subst"`' +old_archive_from_new_cmds='`$ECHO "$old_archive_from_new_cmds" | $SED "$delay_single_quote_subst"`' +old_archive_from_expsyms_cmds='`$ECHO "$old_archive_from_expsyms_cmds" | $SED "$delay_single_quote_subst"`' +archive_cmds='`$ECHO "$archive_cmds" | $SED "$delay_single_quote_subst"`' +archive_expsym_cmds='`$ECHO "$archive_expsym_cmds" | $SED "$delay_single_quote_subst"`' +module_cmds='`$ECHO "$module_cmds" | $SED "$delay_single_quote_subst"`' +module_expsym_cmds='`$ECHO "$module_expsym_cmds" | $SED "$delay_single_quote_subst"`' +with_gnu_ld='`$ECHO "$with_gnu_ld" | $SED "$delay_single_quote_subst"`' +allow_undefined_flag='`$ECHO "$allow_undefined_flag" | $SED "$delay_single_quote_subst"`' +no_undefined_flag='`$ECHO "$no_undefined_flag" | $SED "$delay_single_quote_subst"`' +hardcode_libdir_flag_spec='`$ECHO "$hardcode_libdir_flag_spec" | $SED "$delay_single_quote_subst"`' +hardcode_libdir_separator='`$ECHO "$hardcode_libdir_separator" | $SED "$delay_single_quote_subst"`' +hardcode_direct='`$ECHO "$hardcode_direct" | $SED "$delay_single_quote_subst"`' +hardcode_direct_absolute='`$ECHO "$hardcode_direct_absolute" | $SED "$delay_single_quote_subst"`' +hardcode_minus_L='`$ECHO "$hardcode_minus_L" | $SED "$delay_single_quote_subst"`' +hardcode_shlibpath_var='`$ECHO "$hardcode_shlibpath_var" | $SED "$delay_single_quote_subst"`' +hardcode_automatic='`$ECHO "$hardcode_automatic" | $SED "$delay_single_quote_subst"`' +inherit_rpath='`$ECHO "$inherit_rpath" | $SED "$delay_single_quote_subst"`' +link_all_deplibs='`$ECHO "$link_all_deplibs" | $SED "$delay_single_quote_subst"`' +always_export_symbols='`$ECHO "$always_export_symbols" | $SED "$delay_single_quote_subst"`' +export_symbols_cmds='`$ECHO "$export_symbols_cmds" | $SED "$delay_single_quote_subst"`' +exclude_expsyms='`$ECHO "$exclude_expsyms" | $SED "$delay_single_quote_subst"`' +include_expsyms='`$ECHO "$include_expsyms" | $SED "$delay_single_quote_subst"`' +prelink_cmds='`$ECHO "$prelink_cmds" | $SED "$delay_single_quote_subst"`' +postlink_cmds='`$ECHO "$postlink_cmds" | $SED "$delay_single_quote_subst"`' +file_list_spec='`$ECHO "$file_list_spec" | $SED "$delay_single_quote_subst"`' +variables_saved_for_relink='`$ECHO "$variables_saved_for_relink" | $SED "$delay_single_quote_subst"`' +need_lib_prefix='`$ECHO "$need_lib_prefix" | $SED "$delay_single_quote_subst"`' +need_version='`$ECHO "$need_version" | $SED "$delay_single_quote_subst"`' +version_type='`$ECHO "$version_type" | $SED "$delay_single_quote_subst"`' +runpath_var='`$ECHO "$runpath_var" | $SED "$delay_single_quote_subst"`' +shlibpath_var='`$ECHO "$shlibpath_var" | $SED "$delay_single_quote_subst"`' +shlibpath_overrides_runpath='`$ECHO "$shlibpath_overrides_runpath" | $SED "$delay_single_quote_subst"`' +libname_spec='`$ECHO "$libname_spec" | $SED "$delay_single_quote_subst"`' +library_names_spec='`$ECHO "$library_names_spec" | $SED "$delay_single_quote_subst"`' +soname_spec='`$ECHO "$soname_spec" | $SED "$delay_single_quote_subst"`' +install_override_mode='`$ECHO "$install_override_mode" | $SED "$delay_single_quote_subst"`' +postinstall_cmds='`$ECHO "$postinstall_cmds" | $SED "$delay_single_quote_subst"`' +postuninstall_cmds='`$ECHO "$postuninstall_cmds" | $SED "$delay_single_quote_subst"`' +finish_cmds='`$ECHO "$finish_cmds" | $SED "$delay_single_quote_subst"`' +finish_eval='`$ECHO "$finish_eval" | $SED "$delay_single_quote_subst"`' +hardcode_into_libs='`$ECHO "$hardcode_into_libs" | $SED "$delay_single_quote_subst"`' +sys_lib_search_path_spec='`$ECHO "$sys_lib_search_path_spec" | $SED "$delay_single_quote_subst"`' +sys_lib_dlsearch_path_spec='`$ECHO "$sys_lib_dlsearch_path_spec" | $SED "$delay_single_quote_subst"`' +hardcode_action='`$ECHO "$hardcode_action" | $SED "$delay_single_quote_subst"`' +enable_dlopen='`$ECHO "$enable_dlopen" | $SED "$delay_single_quote_subst"`' +enable_dlopen_self='`$ECHO "$enable_dlopen_self" | $SED "$delay_single_quote_subst"`' +enable_dlopen_self_static='`$ECHO "$enable_dlopen_self_static" | $SED "$delay_single_quote_subst"`' +old_striplib='`$ECHO "$old_striplib" | $SED "$delay_single_quote_subst"`' +striplib='`$ECHO "$striplib" | $SED "$delay_single_quote_subst"`' +compiler_lib_search_dirs='`$ECHO "$compiler_lib_search_dirs" | $SED "$delay_single_quote_subst"`' +predep_objects='`$ECHO "$predep_objects" | $SED "$delay_single_quote_subst"`' +postdep_objects='`$ECHO "$postdep_objects" | $SED "$delay_single_quote_subst"`' +predeps='`$ECHO "$predeps" | $SED "$delay_single_quote_subst"`' +postdeps='`$ECHO "$postdeps" | $SED "$delay_single_quote_subst"`' +compiler_lib_search_path='`$ECHO "$compiler_lib_search_path" | $SED "$delay_single_quote_subst"`' +LD_CXX='`$ECHO "$LD_CXX" | $SED "$delay_single_quote_subst"`' +reload_flag_CXX='`$ECHO "$reload_flag_CXX" | $SED "$delay_single_quote_subst"`' +reload_cmds_CXX='`$ECHO "$reload_cmds_CXX" | $SED "$delay_single_quote_subst"`' +old_archive_cmds_CXX='`$ECHO "$old_archive_cmds_CXX" | $SED "$delay_single_quote_subst"`' +compiler_CXX='`$ECHO "$compiler_CXX" | $SED "$delay_single_quote_subst"`' +GCC_CXX='`$ECHO "$GCC_CXX" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_no_builtin_flag_CXX='`$ECHO "$lt_prog_compiler_no_builtin_flag_CXX" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_pic_CXX='`$ECHO "$lt_prog_compiler_pic_CXX" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_wl_CXX='`$ECHO "$lt_prog_compiler_wl_CXX" | $SED "$delay_single_quote_subst"`' +lt_prog_compiler_static_CXX='`$ECHO "$lt_prog_compiler_static_CXX" | $SED "$delay_single_quote_subst"`' +lt_cv_prog_compiler_c_o_CXX='`$ECHO "$lt_cv_prog_compiler_c_o_CXX" | $SED "$delay_single_quote_subst"`' +archive_cmds_need_lc_CXX='`$ECHO "$archive_cmds_need_lc_CXX" | $SED "$delay_single_quote_subst"`' +enable_shared_with_static_runtimes_CXX='`$ECHO "$enable_shared_with_static_runtimes_CXX" | $SED "$delay_single_quote_subst"`' +export_dynamic_flag_spec_CXX='`$ECHO "$export_dynamic_flag_spec_CXX" | $SED "$delay_single_quote_subst"`' +whole_archive_flag_spec_CXX='`$ECHO "$whole_archive_flag_spec_CXX" | $SED "$delay_single_quote_subst"`' +compiler_needs_object_CXX='`$ECHO "$compiler_needs_object_CXX" | $SED "$delay_single_quote_subst"`' +old_archive_from_new_cmds_CXX='`$ECHO "$old_archive_from_new_cmds_CXX" | $SED "$delay_single_quote_subst"`' +old_archive_from_expsyms_cmds_CXX='`$ECHO "$old_archive_from_expsyms_cmds_CXX" | $SED "$delay_single_quote_subst"`' +archive_cmds_CXX='`$ECHO "$archive_cmds_CXX" | $SED "$delay_single_quote_subst"`' +archive_expsym_cmds_CXX='`$ECHO "$archive_expsym_cmds_CXX" | $SED "$delay_single_quote_subst"`' +module_cmds_CXX='`$ECHO "$module_cmds_CXX" | $SED "$delay_single_quote_subst"`' +module_expsym_cmds_CXX='`$ECHO "$module_expsym_cmds_CXX" | $SED "$delay_single_quote_subst"`' +with_gnu_ld_CXX='`$ECHO "$with_gnu_ld_CXX" | $SED "$delay_single_quote_subst"`' +allow_undefined_flag_CXX='`$ECHO "$allow_undefined_flag_CXX" | $SED "$delay_single_quote_subst"`' +no_undefined_flag_CXX='`$ECHO "$no_undefined_flag_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_libdir_flag_spec_CXX='`$ECHO "$hardcode_libdir_flag_spec_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_libdir_separator_CXX='`$ECHO "$hardcode_libdir_separator_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_direct_CXX='`$ECHO "$hardcode_direct_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_direct_absolute_CXX='`$ECHO "$hardcode_direct_absolute_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_minus_L_CXX='`$ECHO "$hardcode_minus_L_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_shlibpath_var_CXX='`$ECHO "$hardcode_shlibpath_var_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_automatic_CXX='`$ECHO "$hardcode_automatic_CXX" | $SED "$delay_single_quote_subst"`' +inherit_rpath_CXX='`$ECHO "$inherit_rpath_CXX" | $SED "$delay_single_quote_subst"`' +link_all_deplibs_CXX='`$ECHO "$link_all_deplibs_CXX" | $SED "$delay_single_quote_subst"`' +always_export_symbols_CXX='`$ECHO "$always_export_symbols_CXX" | $SED "$delay_single_quote_subst"`' +export_symbols_cmds_CXX='`$ECHO "$export_symbols_cmds_CXX" | $SED "$delay_single_quote_subst"`' +exclude_expsyms_CXX='`$ECHO "$exclude_expsyms_CXX" | $SED "$delay_single_quote_subst"`' +include_expsyms_CXX='`$ECHO "$include_expsyms_CXX" | $SED "$delay_single_quote_subst"`' +prelink_cmds_CXX='`$ECHO "$prelink_cmds_CXX" | $SED "$delay_single_quote_subst"`' +postlink_cmds_CXX='`$ECHO "$postlink_cmds_CXX" | $SED "$delay_single_quote_subst"`' +file_list_spec_CXX='`$ECHO "$file_list_spec_CXX" | $SED "$delay_single_quote_subst"`' +hardcode_action_CXX='`$ECHO "$hardcode_action_CXX" | $SED "$delay_single_quote_subst"`' +compiler_lib_search_dirs_CXX='`$ECHO "$compiler_lib_search_dirs_CXX" | $SED "$delay_single_quote_subst"`' +predep_objects_CXX='`$ECHO "$predep_objects_CXX" | $SED "$delay_single_quote_subst"`' +postdep_objects_CXX='`$ECHO "$postdep_objects_CXX" | $SED "$delay_single_quote_subst"`' +predeps_CXX='`$ECHO "$predeps_CXX" | $SED "$delay_single_quote_subst"`' +postdeps_CXX='`$ECHO "$postdeps_CXX" | $SED "$delay_single_quote_subst"`' +compiler_lib_search_path_CXX='`$ECHO "$compiler_lib_search_path_CXX" | $SED "$delay_single_quote_subst"`' + +LTCC='$LTCC' +LTCFLAGS='$LTCFLAGS' +compiler='$compiler_DEFAULT' + +# A function that is used when there is no print builtin or printf. +func_fallback_echo () +{ + eval 'cat <<_LTECHO_EOF +\$1 +_LTECHO_EOF' +} + +# Quote evaled strings. +for var in SHELL \ +ECHO \ +PATH_SEPARATOR \ +SED \ +GREP \ +EGREP \ +FGREP \ +LD \ +NM \ +LN_S \ +lt_SP2NL \ +lt_NL2SP \ +reload_flag \ +OBJDUMP \ +deplibs_check_method \ +file_magic_cmd \ +file_magic_glob \ +want_nocaseglob \ +DLLTOOL \ +sharedlib_from_linklib_cmd \ +AR \ +AR_FLAGS \ +archiver_list_spec \ +STRIP \ +RANLIB \ +CC \ +CFLAGS \ +compiler \ +lt_cv_sys_global_symbol_pipe \ +lt_cv_sys_global_symbol_to_cdecl \ +lt_cv_sys_global_symbol_to_c_name_address \ +lt_cv_sys_global_symbol_to_c_name_address_lib_prefix \ +nm_file_list_spec \ +lt_prog_compiler_no_builtin_flag \ +lt_prog_compiler_pic \ +lt_prog_compiler_wl \ +lt_prog_compiler_static \ +lt_cv_prog_compiler_c_o \ +need_locks \ +MANIFEST_TOOL \ +DSYMUTIL \ +NMEDIT \ +LIPO \ +OTOOL \ +OTOOL64 \ +shrext_cmds \ +export_dynamic_flag_spec \ +whole_archive_flag_spec \ +compiler_needs_object \ +with_gnu_ld \ +allow_undefined_flag \ +no_undefined_flag \ +hardcode_libdir_flag_spec \ +hardcode_libdir_separator \ +exclude_expsyms \ +include_expsyms \ +file_list_spec \ +variables_saved_for_relink \ +libname_spec \ +library_names_spec \ +soname_spec \ +install_override_mode \ +finish_eval \ +old_striplib \ +striplib \ +compiler_lib_search_dirs \ +predep_objects \ +postdep_objects \ +predeps \ +postdeps \ +compiler_lib_search_path \ +LD_CXX \ +reload_flag_CXX \ +compiler_CXX \ +lt_prog_compiler_no_builtin_flag_CXX \ +lt_prog_compiler_pic_CXX \ +lt_prog_compiler_wl_CXX \ +lt_prog_compiler_static_CXX \ +lt_cv_prog_compiler_c_o_CXX \ +export_dynamic_flag_spec_CXX \ +whole_archive_flag_spec_CXX \ +compiler_needs_object_CXX \ +with_gnu_ld_CXX \ +allow_undefined_flag_CXX \ +no_undefined_flag_CXX \ +hardcode_libdir_flag_spec_CXX \ +hardcode_libdir_separator_CXX \ +exclude_expsyms_CXX \ +include_expsyms_CXX \ +file_list_spec_CXX \ +compiler_lib_search_dirs_CXX \ +predep_objects_CXX \ +postdep_objects_CXX \ +predeps_CXX \ +postdeps_CXX \ +compiler_lib_search_path_CXX; do + case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in + *[\\\\\\\`\\"\\\$]*) + eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED \\"\\\$sed_quote_subst\\"\\\`\\\\\\"" + ;; + *) + eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" + ;; + esac +done + +# Double-quote double-evaled strings. +for var in reload_cmds \ +old_postinstall_cmds \ +old_postuninstall_cmds \ +old_archive_cmds \ +extract_expsyms_cmds \ +old_archive_from_new_cmds \ +old_archive_from_expsyms_cmds \ +archive_cmds \ +archive_expsym_cmds \ +module_cmds \ +module_expsym_cmds \ +export_symbols_cmds \ +prelink_cmds \ +postlink_cmds \ +postinstall_cmds \ +postuninstall_cmds \ +finish_cmds \ +sys_lib_search_path_spec \ +sys_lib_dlsearch_path_spec \ +reload_cmds_CXX \ +old_archive_cmds_CXX \ +old_archive_from_new_cmds_CXX \ +old_archive_from_expsyms_cmds_CXX \ +archive_cmds_CXX \ +archive_expsym_cmds_CXX \ +module_cmds_CXX \ +module_expsym_cmds_CXX \ +export_symbols_cmds_CXX \ +prelink_cmds_CXX \ +postlink_cmds_CXX; do + case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in + *[\\\\\\\`\\"\\\$]*) + eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED -e \\"\\\$double_quote_subst\\" -e \\"\\\$sed_quote_subst\\" -e \\"\\\$delay_variable_subst\\"\\\`\\\\\\"" + ;; + *) + eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" + ;; + esac +done + +ac_aux_dir='$ac_aux_dir' +xsi_shell='$xsi_shell' +lt_shell_append='$lt_shell_append' + +# See if we are running on zsh, and set the options which allow our +# commands through without removal of \ escapes INIT. +if test -n "\${ZSH_VERSION+set}" ; then + setopt NO_GLOB_SUBST +fi + + + PACKAGE='$PACKAGE' + VERSION='$VERSION' + TIMESTAMP='$TIMESTAMP' + RM='$RM' + ofile='$ofile' + + + + + + +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 + +# Handling of arguments. +for ac_config_target in $ac_config_targets +do + case $ac_config_target in + "depfiles") CONFIG_COMMANDS="$CONFIG_COMMANDS depfiles" ;; + "libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;; + "src/acconfig.h") CONFIG_HEADERS="$CONFIG_HEADERS src/acconfig.h" ;; + "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; + "src/Makefile") CONFIG_FILES="$CONFIG_FILES src/Makefile" ;; + "src/ocf/Makefile") CONFIG_FILES="$CONFIG_FILES src/ocf/Makefile" ;; + "src/ocf/ceph") CONFIG_FILES="$CONFIG_FILES src/ocf/ceph" ;; + "src/ocf/rbd") CONFIG_FILES="$CONFIG_FILES src/ocf/rbd" ;; + "src/java/Makefile") CONFIG_FILES="$CONFIG_FILES src/java/Makefile" ;; + "man/Makefile") CONFIG_FILES="$CONFIG_FILES man/Makefile" ;; + "ceph.spec") CONFIG_FILES="$CONFIG_FILES ceph.spec" ;; + + *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; + esac +done + + +# If the user did not use the arguments to specify the items to instantiate, +# then the envvar interface is used. Set only those that are not. +# We use the long form for the default assignment because of an extremely +# bizarre bug on SunOS 4.1.3. +if $ac_need_defaults; then + test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files + test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers + test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands +fi + +# Have a temporary directory for convenience. Make it in the build tree +# simply because there is no reason against having it here, and in addition, +# creating and moving files from /tmp can sometimes cause problems. +# Hook for its removal unless debugging. +# Note that there is a small window in which the directory will not be cleaned: +# after its creation but before its name has been assigned to `$tmp'. +$debug || +{ + tmp= ac_tmp= + trap 'exit_status=$? + : "${ac_tmp:=$tmp}" + { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status +' 0 + trap 'as_fn_exit 1' 1 2 13 15 +} +# Create a (secure) tmp directory for tmp files. + +{ + tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && + test -d "$tmp" +} || +{ + tmp=./conf$$-$RANDOM + (umask 077 && mkdir "$tmp") +} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 +ac_tmp=$tmp + +# Set up the scripts for CONFIG_FILES section. +# No need to generate them if there are no CONFIG_FILES. +# This happens for instance with `./config.status config.h'. +if test -n "$CONFIG_FILES"; then + + +ac_cr=`echo X | tr X '\015'` +# On cygwin, bash can eat \r inside `` if the user requested igncr. +# But we know of no other shell where ac_cr would be empty at this +# point, so we can use a bashism as a fallback. +if test "x$ac_cr" = x; then + eval ac_cr=\$\'\\r\' +fi +ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` +if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then + ac_cs_awk_cr='\\r' +else + ac_cs_awk_cr=$ac_cr +fi + +echo 'BEGIN {' >"$ac_tmp/subs1.awk" && +_ACEOF + + +{ + echo "cat >conf$$subs.awk <<_ACEOF" && + echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && + echo "_ACEOF" +} >conf$$subs.sh || + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 +ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` +ac_delim='%!_!# ' +for ac_last_try in false false false false false :; do + . ./conf$$subs.sh || + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 + + ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` + if test $ac_delim_n = $ac_delim_num; then + break + elif $ac_last_try; then + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 + else + ac_delim="$ac_delim!$ac_delim _$ac_delim!! " + fi +done +rm -f conf$$subs.sh + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && +_ACEOF +sed -n ' +h +s/^/S["/; s/!.*/"]=/ +p +g +s/^[^!]*!// +:repl +t repl +s/'"$ac_delim"'$// +t delim +:nl +h +s/\(.\{148\}\)..*/\1/ +t more1 +s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/ +p +n +b repl +:more1 +s/["\\]/\\&/g; s/^/"/; s/$/"\\/ +p +g +s/.\{148\}// +t nl +:delim +h +s/\(.\{148\}\)..*/\1/ +t more2 +s/["\\]/\\&/g; s/^/"/; s/$/"/ +p +b +:more2 +s/["\\]/\\&/g; s/^/"/; s/$/"\\/ +p +g +s/.\{148\}// +t delim +' >$CONFIG_STATUS || ac_write_fail=1 +rm -f conf$$subs.awk +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +_ACAWK +cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && + for (key in S) S_is_set[key] = 1 + FS = "" + +} +{ + line = $ 0 + nfields = split(line, field, "@") + substed = 0 + len = length(field[1]) + for (i = 2; i < nfields; i++) { + key = field[i] + keylen = length(key) + if (S_is_set[key]) { + value = S[key] + line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3) + len += length(value) + length(field[++i]) + substed = 1 + } else + len += 1 + keylen + } + + print line +} + +_ACAWK +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then + sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" +else + cat +fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \ + || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 +_ACEOF + +# VPATH may cause trouble with some makes, so we remove sole $(srcdir), +# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and +# trailing colons and then remove the whole line if VPATH becomes empty +# (actually we leave an empty line to preserve line numbers). +if test "x$srcdir" = x.; then + ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ +h +s/// +s/^/:/ +s/[ ]*$/:/ +s/:\$(srcdir):/:/g +s/:\${srcdir}:/:/g +s/:@srcdir@:/:/g +s/^:*// +s/:*$// +x +s/\(=[ ]*\).*/\1/ +G +s/\n// +s/^[^=]*=[ ]*$// +}' +fi + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +fi # test -n "$CONFIG_FILES" + +# Set up the scripts for CONFIG_HEADERS section. +# No need to generate them if there are no CONFIG_HEADERS. +# This happens for instance with `./config.status Makefile'. +if test -n "$CONFIG_HEADERS"; then +cat >"$ac_tmp/defines.awk" <<\_ACAWK || +BEGIN { +_ACEOF + +# Transform confdefs.h into an awk script `defines.awk', embedded as +# here-document in config.status, that substitutes the proper values into +# config.h.in to produce config.h. + +# Create a delimiter string that does not exist in confdefs.h, to ease +# handling of long lines. +ac_delim='%!_!# ' +for ac_last_try in false false :; do + ac_tt=`sed -n "/$ac_delim/p" confdefs.h` + if test -z "$ac_tt"; then + break + elif $ac_last_try; then + as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5 + else + ac_delim="$ac_delim!$ac_delim _$ac_delim!! " + fi +done + +# For the awk script, D is an array of macro values keyed by name, +# likewise P contains macro parameters if any. Preserve backslash +# newline sequences. + +ac_word_re=[_$as_cr_Letters][_$as_cr_alnum]* +sed -n ' +s/.\{148\}/&'"$ac_delim"'/g +t rset +:rset +s/^[ ]*#[ ]*define[ ][ ]*/ / +t def +d +:def +s/\\$// +t bsnl +s/["\\]/\\&/g +s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ +D["\1"]=" \3"/p +s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2"/p +d +:bsnl +s/["\\]/\\&/g +s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ +D["\1"]=" \3\\\\\\n"\\/p +t cont +s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2\\\\\\n"\\/p +t cont +d +:cont +n +s/.\{148\}/&'"$ac_delim"'/g +t clear +:clear +s/\\$// +t bsnlc +s/["\\]/\\&/g; s/^/"/; s/$/"/p +d +:bsnlc +s/["\\]/\\&/g; s/^/"/; s/$/\\\\\\n"\\/p +b cont +' >$CONFIG_STATUS || ac_write_fail=1 + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 + for (key in D) D_is_set[key] = 1 + FS = "" +} +/^[\t ]*#[\t ]*(define|undef)[\t ]+$ac_word_re([\t (]|\$)/ { + line = \$ 0 + split(line, arg, " ") + if (arg[1] == "#") { + defundef = arg[2] + mac1 = arg[3] + } else { + defundef = substr(arg[1], 2) + mac1 = arg[2] + } + split(mac1, mac2, "(") #) + macro = mac2[1] + prefix = substr(line, 1, index(line, defundef) - 1) + if (D_is_set[macro]) { + # Preserve the white space surrounding the "#". + print prefix "define", macro P[macro] D[macro] + next + } else { + # Replace #undef with comments. This is necessary, for example, + # in the case of _POSIX_SOURCE, which is predefined and required + # on some systems where configure will not decide to define it. + if (defundef == "undef") { + print "/*", prefix defundef, macro, "*/" + next + } + } +} +{ print } +_ACAWK +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 + as_fn_error $? "could not setup config headers machinery" "$LINENO" 5 +fi # test -n "$CONFIG_HEADERS" + + +eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS :C $CONFIG_COMMANDS" +shift +for ac_tag +do + case $ac_tag in + :[FHLC]) ac_mode=$ac_tag; continue;; + esac + case $ac_mode$ac_tag in + :[FHL]*:*);; + :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; + :[FH]-) ac_tag=-:-;; + :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; + esac + ac_save_IFS=$IFS + IFS=: + set x $ac_tag + IFS=$ac_save_IFS + shift + ac_file=$1 + shift + + case $ac_mode in + :L) ac_source=$1;; + :[FH]) + ac_file_inputs= + for ac_f + do + case $ac_f in + -) ac_f="$ac_tmp/stdin";; + *) # Look for the file first in the build tree, then in the source tree + # (if the path is not absolute). The absolute path cannot be DOS-style, + # because $ac_f cannot contain `:'. + test -f "$ac_f" || + case $ac_f in + [\\/$]*) false;; + *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; + esac || + as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; + esac + case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac + as_fn_append ac_file_inputs " '$ac_f'" + done + + # Let's still pretend it is `configure' which instantiates (i.e., don't + # use $as_me), people would be surprised to read: + # /* config.h. Generated by config.status. */ + configure_input='Generated from '` + $as_echo "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' + `' by configure.' + if test x"$ac_file" != x-; then + configure_input="$ac_file. $configure_input" + { $as_echo "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5 +$as_echo "$as_me: creating $ac_file" >&6;} + fi + # Neutralize special characters interpreted by sed in replacement strings. + case $configure_input in #( + *\&* | *\|* | *\\* ) + ac_sed_conf_input=`$as_echo "$configure_input" | + sed 's/[\\\\&|]/\\\\&/g'`;; #( + *) ac_sed_conf_input=$configure_input;; + esac + + case $ac_tag in + *:-:* | *:-) cat >"$ac_tmp/stdin" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; + esac + ;; + esac + + ac_dir=`$as_dirname -- "$ac_file" || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + as_dir="$ac_dir"; as_fn_mkdir_p + ac_builddir=. + +case "$ac_dir" in +.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; +*) + ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` + # A ".." for each directory in $ac_dir_suffix. + ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + case $ac_top_builddir_sub in + "") ac_top_builddir_sub=. ac_top_build_prefix= ;; + *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; + esac ;; +esac +ac_abs_top_builddir=$ac_pwd +ac_abs_builddir=$ac_pwd$ac_dir_suffix +# for backward compatibility: +ac_top_builddir=$ac_top_build_prefix + +case $srcdir in + .) # We are building in place. + ac_srcdir=. + ac_top_srcdir=$ac_top_builddir_sub + ac_abs_top_srcdir=$ac_pwd ;; + [\\/]* | ?:[\\/]* ) # Absolute name. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir + ac_abs_top_srcdir=$srcdir ;; + *) # Relative name. + ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_build_prefix$srcdir + ac_abs_top_srcdir=$ac_pwd/$srcdir ;; +esac +ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix + + + case $ac_mode in + :F) + # + # CONFIG_FILE + # + + case $INSTALL in + [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;; + *) ac_INSTALL=$ac_top_build_prefix$INSTALL ;; + esac + ac_MKDIR_P=$MKDIR_P + case $MKDIR_P in + [\\/$]* | ?:[\\/]* ) ;; + */*) ac_MKDIR_P=$ac_top_build_prefix$MKDIR_P ;; + esac +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# If the template does not know about datarootdir, expand it. +# FIXME: This hack should be removed a few years after 2.60. +ac_datarootdir_hack=; ac_datarootdir_seen= +ac_sed_dataroot=' +/datarootdir/ { + p + q +} +/@datadir@/p +/@docdir@/p +/@infodir@/p +/@localedir@/p +/@mandir@/p' +case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in +*datarootdir*) ac_datarootdir_seen=yes;; +*@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 +$as_echo "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 + ac_datarootdir_hack=' + s&@datadir@&$datadir&g + s&@docdir@&$docdir&g + s&@infodir@&$infodir&g + s&@localedir@&$localedir&g + s&@mandir@&$mandir&g + s&\\\${datarootdir}&$datarootdir&g' ;; +esac +_ACEOF + +# Neutralize VPATH when `$srcdir' = `.'. +# Shell code in configure.ac might set extrasub. +# FIXME: do we really want to maintain this feature? +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +ac_sed_extra="$ac_vpsub +$extrasub +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +:t +/@[a-zA-Z_][a-zA-Z_0-9]*@/!b +s|@configure_input@|$ac_sed_conf_input|;t t +s&@top_builddir@&$ac_top_builddir_sub&;t t +s&@top_build_prefix@&$ac_top_build_prefix&;t t +s&@srcdir@&$ac_srcdir&;t t +s&@abs_srcdir@&$ac_abs_srcdir&;t t +s&@top_srcdir@&$ac_top_srcdir&;t t +s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t +s&@builddir@&$ac_builddir&;t t +s&@abs_builddir@&$ac_abs_builddir&;t t +s&@abs_top_builddir@&$ac_abs_top_builddir&;t t +s&@INSTALL@&$ac_INSTALL&;t t +s&@MKDIR_P@&$ac_MKDIR_P&;t t +$ac_datarootdir_hack +" +eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \ + >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + +test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && + { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && + { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ + "$ac_tmp/out"`; test -z "$ac_out"; } && + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' +which seems to be undefined. Please make sure it is defined" >&5 +$as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' +which seems to be undefined. Please make sure it is defined" >&2;} + + rm -f "$ac_tmp/stdin" + case $ac_file in + -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";; + *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";; + esac \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + ;; + :H) + # + # CONFIG_HEADER + # + if test x"$ac_file" != x-; then + { + $as_echo "/* $configure_input */" \ + && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" + } >"$ac_tmp/config.h" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + if diff "$ac_file" "$ac_tmp/config.h" >/dev/null 2>&1; then + { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5 +$as_echo "$as_me: $ac_file is unchanged" >&6;} + else + rm -f "$ac_file" + mv "$ac_tmp/config.h" "$ac_file" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + fi + else + $as_echo "/* $configure_input */" \ + && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" \ + || as_fn_error $? "could not create -" "$LINENO" 5 + fi +# Compute "$ac_file"'s index in $config_headers. +_am_arg="$ac_file" +_am_stamp_count=1 +for _am_header in $config_headers :; do + case $_am_header in + $_am_arg | $_am_arg:* ) + break ;; + * ) + _am_stamp_count=`expr $_am_stamp_count + 1` ;; + esac +done +echo "timestamp for $_am_arg" >`$as_dirname -- "$_am_arg" || +$as_expr X"$_am_arg" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$_am_arg" : 'X\(//\)[^/]' \| \ + X"$_am_arg" : 'X\(//\)$' \| \ + X"$_am_arg" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$_am_arg" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'`/stamp-h$_am_stamp_count + ;; + + :C) { $as_echo "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5 +$as_echo "$as_me: executing $ac_file commands" >&6;} + ;; + esac + + + case $ac_file$ac_mode in + "depfiles":C) test x"$AMDEP_TRUE" != x"" || { + # Autoconf 2.62 quotes --file arguments for eval, but not when files + # are listed without --file. Let's play safe and only enable the eval + # if we detect the quoting. + case $CONFIG_FILES in + *\'*) eval set x "$CONFIG_FILES" ;; + *) set x $CONFIG_FILES ;; + esac + shift + for mf + do + # Strip MF so we end up with the name of the file. + mf=`echo "$mf" | sed -e 's/:.*$//'` + # Check whether this is an Automake generated Makefile or not. + # We used to match only the files named `Makefile.in', but + # some people rename them; so instead we look at the file content. + # Grep'ing the first line is not enough: some people post-process + # each Makefile.in and add a new line on top of each file to say so. + # Grep'ing the whole file is not good either: AIX grep has a line + # limit of 2048, but all sed's we know have understand at least 4000. + if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then + dirpart=`$as_dirname -- "$mf" || +$as_expr X"$mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$mf" : 'X\(//\)[^/]' \| \ + X"$mf" : 'X\(//\)$' \| \ + X"$mf" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$mf" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + else + continue + fi + # Extract the definition of DEPDIR, am__include, and am__quote + # from the Makefile without running `make'. + DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` + test -z "$DEPDIR" && continue + am__include=`sed -n 's/^am__include = //p' < "$mf"` + test -z "am__include" && continue + am__quote=`sed -n 's/^am__quote = //p' < "$mf"` + # When using ansi2knr, U may be empty or an underscore; expand it + U=`sed -n 's/^U = //p' < "$mf"` + # Find all dependency output files, they are included files with + # $(DEPDIR) in their names. We invoke sed twice because it is the + # simplest approach to changing $(DEPDIR) to its actual value in the + # expansion. + for file in `sed -n " + s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ + sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do + # Make sure the directory exists. + test -f "$dirpart/$file" && continue + fdir=`$as_dirname -- "$file" || +$as_expr X"$file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$file" : 'X\(//\)[^/]' \| \ + X"$file" : 'X\(//\)$' \| \ + X"$file" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + as_dir=$dirpart/$fdir; as_fn_mkdir_p + # echo "creating $dirpart/$file" + echo '# dummy' > "$dirpart/$file" + done + done +} + ;; + "libtool":C) + + # See if we are running on zsh, and set the options which allow our + # commands through without removal of \ escapes. + if test -n "${ZSH_VERSION+set}" ; then + setopt NO_GLOB_SUBST + fi + + cfgfile="${ofile}T" + trap "$RM \"$cfgfile\"; exit 1" 1 2 15 + $RM "$cfgfile" + + cat <<_LT_EOF >> "$cfgfile" +#! $SHELL + +# `$ECHO "$ofile" | sed 's%^.*/%%'` - Provide generalized library-building support services. +# Generated automatically by $as_me ($PACKAGE$TIMESTAMP) $VERSION +# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: +# NOTE: Changes made to this file will be lost: look at ltmain.sh. +# +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005, +# 2006, 2007, 2008, 2009, 2010, 2011 Free Software +# Foundation, Inc. +# Written by Gordon Matzigkeit, 1996 +# +# This file is part of GNU Libtool. +# +# GNU Libtool is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# As a special exception to the GNU General Public License, +# if you distribute this file as part of a program or library that +# is built using GNU Libtool, you may include this file under the +# same distribution terms that you use for the rest of that program. +# +# GNU Libtool is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Libtool; see the file COPYING. If not, a copy +# can be downloaded from http://www.gnu.org/licenses/gpl.html, or +# obtained by writing to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +# The names of the tagged configurations supported by this script. +available_tags="CXX " + +# ### BEGIN LIBTOOL CONFIG + +# Which release of libtool.m4 was used? +macro_version=$macro_version +macro_revision=$macro_revision + +# Whether or not to build shared libraries. +build_libtool_libs=$enable_shared + +# Whether or not to build static libraries. +build_old_libs=$enable_static + +# What type of objects to build. +pic_mode=$pic_mode + +# Whether or not to optimize for fast installation. +fast_install=$enable_fast_install + +# Shell to use when invoking shell scripts. +SHELL=$lt_SHELL + +# An echo program that protects backslashes. +ECHO=$lt_ECHO + +# The PATH separator for the build system. +PATH_SEPARATOR=$lt_PATH_SEPARATOR + +# The host system. +host_alias=$host_alias +host=$host +host_os=$host_os + +# The build system. +build_alias=$build_alias +build=$build +build_os=$build_os + +# A sed program that does not truncate output. +SED=$lt_SED + +# Sed that helps us avoid accidentally triggering echo(1) options like -n. +Xsed="\$SED -e 1s/^X//" + +# A grep program that handles long lines. +GREP=$lt_GREP + +# An ERE matcher. +EGREP=$lt_EGREP + +# A literal string matcher. +FGREP=$lt_FGREP + +# A BSD- or MS-compatible name lister. +NM=$lt_NM + +# Whether we need soft or hard links. +LN_S=$lt_LN_S + +# What is the maximum length of a command? +max_cmd_len=$max_cmd_len + +# Object file suffix (normally "o"). +objext=$ac_objext + +# Executable file suffix (normally ""). +exeext=$exeext + +# whether the shell understands "unset". +lt_unset=$lt_unset + +# turn spaces into newlines. +SP2NL=$lt_lt_SP2NL + +# turn newlines into spaces. +NL2SP=$lt_lt_NL2SP + +# convert \$build file names to \$host format. +to_host_file_cmd=$lt_cv_to_host_file_cmd + +# convert \$build files to toolchain format. +to_tool_file_cmd=$lt_cv_to_tool_file_cmd + +# An object symbol dumper. +OBJDUMP=$lt_OBJDUMP + +# Method to check whether dependent libraries are shared objects. +deplibs_check_method=$lt_deplibs_check_method + +# Command to use when deplibs_check_method = "file_magic". +file_magic_cmd=$lt_file_magic_cmd + +# How to find potential files when deplibs_check_method = "file_magic". +file_magic_glob=$lt_file_magic_glob + +# Find potential files using nocaseglob when deplibs_check_method = "file_magic". +want_nocaseglob=$lt_want_nocaseglob + +# DLL creation program. +DLLTOOL=$lt_DLLTOOL + +# Command to associate shared and link libraries. +sharedlib_from_linklib_cmd=$lt_sharedlib_from_linklib_cmd + +# The archiver. +AR=$lt_AR + +# Flags to create an archive. +AR_FLAGS=$lt_AR_FLAGS + +# How to feed a file listing to the archiver. +archiver_list_spec=$lt_archiver_list_spec + +# A symbol stripping program. +STRIP=$lt_STRIP + +# Commands used to install an old-style archive. +RANLIB=$lt_RANLIB +old_postinstall_cmds=$lt_old_postinstall_cmds +old_postuninstall_cmds=$lt_old_postuninstall_cmds + +# Whether to use a lock for old archive extraction. +lock_old_archive_extraction=$lock_old_archive_extraction + +# A C compiler. +LTCC=$lt_CC + +# LTCC compiler flags. +LTCFLAGS=$lt_CFLAGS + +# Take the output of nm and produce a listing of raw symbols and C names. +global_symbol_pipe=$lt_lt_cv_sys_global_symbol_pipe + +# Transform the output of nm in a proper C declaration. +global_symbol_to_cdecl=$lt_lt_cv_sys_global_symbol_to_cdecl + +# Transform the output of nm in a C name address pair. +global_symbol_to_c_name_address=$lt_lt_cv_sys_global_symbol_to_c_name_address + +# Transform the output of nm in a C name address pair when lib prefix is needed. +global_symbol_to_c_name_address_lib_prefix=$lt_lt_cv_sys_global_symbol_to_c_name_address_lib_prefix + +# Specify filename containing input files for \$NM. +nm_file_list_spec=$lt_nm_file_list_spec + +# The root where to search for dependent libraries,and in which our libraries should be installed. +lt_sysroot=$lt_sysroot + +# The name of the directory that contains temporary libtool files. +objdir=$objdir + +# Used to examine libraries when file_magic_cmd begins with "file". +MAGIC_CMD=$MAGIC_CMD + +# Must we lock files when doing compilation? +need_locks=$lt_need_locks + +# Manifest tool. +MANIFEST_TOOL=$lt_MANIFEST_TOOL + +# Tool to manipulate archived DWARF debug symbol files on Mac OS X. +DSYMUTIL=$lt_DSYMUTIL + +# Tool to change global to local symbols on Mac OS X. +NMEDIT=$lt_NMEDIT + +# Tool to manipulate fat objects and archives on Mac OS X. +LIPO=$lt_LIPO + +# ldd/readelf like tool for Mach-O binaries on Mac OS X. +OTOOL=$lt_OTOOL + +# ldd/readelf like tool for 64 bit Mach-O binaries on Mac OS X 10.4. +OTOOL64=$lt_OTOOL64 + +# Old archive suffix (normally "a"). +libext=$libext + +# Shared library suffix (normally ".so"). +shrext_cmds=$lt_shrext_cmds + +# The commands to extract the exported symbol list from a shared archive. +extract_expsyms_cmds=$lt_extract_expsyms_cmds + +# Variables whose values should be saved in libtool wrapper scripts and +# restored at link time. +variables_saved_for_relink=$lt_variables_saved_for_relink + +# Do we need the "lib" prefix for modules? +need_lib_prefix=$need_lib_prefix + +# Do we need a version for libraries? +need_version=$need_version + +# Library versioning type. +version_type=$version_type + +# Shared library runtime path variable. +runpath_var=$runpath_var + +# Shared library path variable. +shlibpath_var=$shlibpath_var + +# Is shlibpath searched before the hard-coded library search path? +shlibpath_overrides_runpath=$shlibpath_overrides_runpath + +# Format of library name prefix. +libname_spec=$lt_libname_spec + +# List of archive names. First name is the real one, the rest are links. +# The last name is the one that the linker finds with -lNAME +library_names_spec=$lt_library_names_spec + +# The coded name of the library, if different from the real name. +soname_spec=$lt_soname_spec + +# Permission mode override for installation of shared libraries. +install_override_mode=$lt_install_override_mode + +# Command to use after installation of a shared archive. +postinstall_cmds=$lt_postinstall_cmds + +# Command to use after uninstallation of a shared archive. +postuninstall_cmds=$lt_postuninstall_cmds + +# Commands used to finish a libtool library installation in a directory. +finish_cmds=$lt_finish_cmds + +# As "finish_cmds", except a single script fragment to be evaled but +# not shown. +finish_eval=$lt_finish_eval + +# Whether we should hardcode library paths into libraries. +hardcode_into_libs=$hardcode_into_libs + +# Compile-time system search path for libraries. +sys_lib_search_path_spec=$lt_sys_lib_search_path_spec + +# Run-time system search path for libraries. +sys_lib_dlsearch_path_spec=$lt_sys_lib_dlsearch_path_spec + +# Whether dlopen is supported. +dlopen_support=$enable_dlopen + +# Whether dlopen of programs is supported. +dlopen_self=$enable_dlopen_self + +# Whether dlopen of statically linked programs is supported. +dlopen_self_static=$enable_dlopen_self_static + +# Commands to strip libraries. +old_striplib=$lt_old_striplib +striplib=$lt_striplib + + +# The linker used to build libraries. +LD=$lt_LD + +# How to create reloadable object files. +reload_flag=$lt_reload_flag +reload_cmds=$lt_reload_cmds + +# Commands used to build an old-style archive. +old_archive_cmds=$lt_old_archive_cmds + +# A language specific compiler. +CC=$lt_compiler + +# Is the compiler the GNU compiler? +with_gcc=$GCC + +# Compiler flag to turn off builtin functions. +no_builtin_flag=$lt_lt_prog_compiler_no_builtin_flag + +# Additional compiler flags for building library objects. +pic_flag=$lt_lt_prog_compiler_pic + +# How to pass a linker flag through the compiler. +wl=$lt_lt_prog_compiler_wl + +# Compiler flag to prevent dynamic linking. +link_static_flag=$lt_lt_prog_compiler_static + +# Does compiler simultaneously support -c and -o options? +compiler_c_o=$lt_lt_cv_prog_compiler_c_o + +# Whether or not to add -lc for building shared libraries. +build_libtool_need_lc=$archive_cmds_need_lc + +# Whether or not to disallow shared libs when runtime libs are static. +allow_libtool_libs_with_static_runtimes=$enable_shared_with_static_runtimes + +# Compiler flag to allow reflexive dlopens. +export_dynamic_flag_spec=$lt_export_dynamic_flag_spec + +# Compiler flag to generate shared objects directly from archives. +whole_archive_flag_spec=$lt_whole_archive_flag_spec + +# Whether the compiler copes with passing no objects directly. +compiler_needs_object=$lt_compiler_needs_object + +# Create an old-style archive from a shared archive. +old_archive_from_new_cmds=$lt_old_archive_from_new_cmds + +# Create a temporary old-style archive to link instead of a shared archive. +old_archive_from_expsyms_cmds=$lt_old_archive_from_expsyms_cmds + +# Commands used to build a shared archive. +archive_cmds=$lt_archive_cmds +archive_expsym_cmds=$lt_archive_expsym_cmds + +# Commands used to build a loadable module if different from building +# a shared archive. +module_cmds=$lt_module_cmds +module_expsym_cmds=$lt_module_expsym_cmds + +# Whether we are building with GNU ld or not. +with_gnu_ld=$lt_with_gnu_ld + +# Flag that allows shared libraries with undefined symbols to be built. +allow_undefined_flag=$lt_allow_undefined_flag + +# Flag that enforces no undefined symbols. +no_undefined_flag=$lt_no_undefined_flag + +# Flag to hardcode \$libdir into a binary during linking. +# This must work even if \$libdir does not exist +hardcode_libdir_flag_spec=$lt_hardcode_libdir_flag_spec + +# Whether we need a single "-rpath" flag with a separated argument. +hardcode_libdir_separator=$lt_hardcode_libdir_separator + +# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes +# DIR into the resulting binary. +hardcode_direct=$hardcode_direct + +# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes +# DIR into the resulting binary and the resulting library dependency is +# "absolute",i.e impossible to change by setting \${shlibpath_var} if the +# library is relocated. +hardcode_direct_absolute=$hardcode_direct_absolute + +# Set to "yes" if using the -LDIR flag during linking hardcodes DIR +# into the resulting binary. +hardcode_minus_L=$hardcode_minus_L + +# Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR +# into the resulting binary. +hardcode_shlibpath_var=$hardcode_shlibpath_var + +# Set to "yes" if building a shared library automatically hardcodes DIR +# into the library and all subsequent libraries and executables linked +# against it. +hardcode_automatic=$hardcode_automatic + +# Set to yes if linker adds runtime paths of dependent libraries +# to runtime path list. +inherit_rpath=$inherit_rpath + +# Whether libtool must link a program against all its dependency libraries. +link_all_deplibs=$link_all_deplibs + +# Set to "yes" if exported symbols are required. +always_export_symbols=$always_export_symbols + +# The commands to list exported symbols. +export_symbols_cmds=$lt_export_symbols_cmds + +# Symbols that should not be listed in the preloaded symbols. +exclude_expsyms=$lt_exclude_expsyms + +# Symbols that must always be exported. +include_expsyms=$lt_include_expsyms + +# Commands necessary for linking programs (against libraries) with templates. +prelink_cmds=$lt_prelink_cmds + +# Commands necessary for finishing linking programs. +postlink_cmds=$lt_postlink_cmds + +# Specify filename containing input files. +file_list_spec=$lt_file_list_spec + +# How to hardcode a shared library path into an executable. +hardcode_action=$hardcode_action + +# The directories searched by this compiler when creating a shared library. +compiler_lib_search_dirs=$lt_compiler_lib_search_dirs + +# Dependencies to place before and after the objects being linked to +# create a shared library. +predep_objects=$lt_predep_objects +postdep_objects=$lt_postdep_objects +predeps=$lt_predeps +postdeps=$lt_postdeps + +# The library search path used internally by the compiler when linking +# a shared library. +compiler_lib_search_path=$lt_compiler_lib_search_path + +# ### END LIBTOOL CONFIG + +_LT_EOF + + case $host_os in + aix3*) + cat <<\_LT_EOF >> "$cfgfile" +# AIX sometimes has problems with the GCC collect2 program. For some +# reason, if we set the COLLECT_NAMES environment variable, the problems +# vanish in a puff of smoke. +if test "X${COLLECT_NAMES+set}" != Xset; then + COLLECT_NAMES= + export COLLECT_NAMES +fi +_LT_EOF + ;; + esac + + +ltmain="$ac_aux_dir/ltmain.sh" + + + # We use sed instead of cat because bash on DJGPP gets confused if + # if finds mixed CR/LF and LF-only lines. Since sed operates in + # text mode, it properly converts lines to CR/LF. This bash problem + # is reportedly fixed, but why not run on old versions too? + sed '$q' "$ltmain" >> "$cfgfile" \ + || (rm -f "$cfgfile"; exit 1) + + if test x"$xsi_shell" = xyes; then + sed -e '/^func_dirname ()$/,/^} # func_dirname /c\ +func_dirname ()\ +{\ +\ case ${1} in\ +\ */*) func_dirname_result="${1%/*}${2}" ;;\ +\ * ) func_dirname_result="${3}" ;;\ +\ esac\ +} # Extended-shell func_dirname implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_basename ()$/,/^} # func_basename /c\ +func_basename ()\ +{\ +\ func_basename_result="${1##*/}"\ +} # Extended-shell func_basename implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_dirname_and_basename ()$/,/^} # func_dirname_and_basename /c\ +func_dirname_and_basename ()\ +{\ +\ case ${1} in\ +\ */*) func_dirname_result="${1%/*}${2}" ;;\ +\ * ) func_dirname_result="${3}" ;;\ +\ esac\ +\ func_basename_result="${1##*/}"\ +} # Extended-shell func_dirname_and_basename implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_stripname ()$/,/^} # func_stripname /c\ +func_stripname ()\ +{\ +\ # pdksh 5.2.14 does not do ${X%$Y} correctly if both X and Y are\ +\ # positional parameters, so assign one to ordinary parameter first.\ +\ func_stripname_result=${3}\ +\ func_stripname_result=${func_stripname_result#"${1}"}\ +\ func_stripname_result=${func_stripname_result%"${2}"}\ +} # Extended-shell func_stripname implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_split_long_opt ()$/,/^} # func_split_long_opt /c\ +func_split_long_opt ()\ +{\ +\ func_split_long_opt_name=${1%%=*}\ +\ func_split_long_opt_arg=${1#*=}\ +} # Extended-shell func_split_long_opt implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_split_short_opt ()$/,/^} # func_split_short_opt /c\ +func_split_short_opt ()\ +{\ +\ func_split_short_opt_arg=${1#??}\ +\ func_split_short_opt_name=${1%"$func_split_short_opt_arg"}\ +} # Extended-shell func_split_short_opt implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_lo2o ()$/,/^} # func_lo2o /c\ +func_lo2o ()\ +{\ +\ case ${1} in\ +\ *.lo) func_lo2o_result=${1%.lo}.${objext} ;;\ +\ *) func_lo2o_result=${1} ;;\ +\ esac\ +} # Extended-shell func_lo2o implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_xform ()$/,/^} # func_xform /c\ +func_xform ()\ +{\ + func_xform_result=${1%.*}.lo\ +} # Extended-shell func_xform implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_arith ()$/,/^} # func_arith /c\ +func_arith ()\ +{\ + func_arith_result=$(( $* ))\ +} # Extended-shell func_arith implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_len ()$/,/^} # func_len /c\ +func_len ()\ +{\ + func_len_result=${#1}\ +} # Extended-shell func_len implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + +fi + +if test x"$lt_shell_append" = xyes; then + sed -e '/^func_append ()$/,/^} # func_append /c\ +func_append ()\ +{\ + eval "${1}+=\\${2}"\ +} # Extended-shell func_append implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + sed -e '/^func_append_quoted ()$/,/^} # func_append_quoted /c\ +func_append_quoted ()\ +{\ +\ func_quote_for_eval "${2}"\ +\ eval "${1}+=\\\\ \\$func_quote_for_eval_result"\ +} # Extended-shell func_append_quoted implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: + + + # Save a `func_append' function call where possible by direct use of '+=' + sed -e 's%func_append \([a-zA-Z_]\{1,\}\) "%\1+="%g' $cfgfile > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") + test 0 -eq $? || _lt_function_replace_fail=: +else + # Save a `func_append' function call even when '+=' is not available + sed -e 's%func_append \([a-zA-Z_]\{1,\}\) "%\1="$\1%g' $cfgfile > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") + test 0 -eq $? || _lt_function_replace_fail=: +fi + +if test x"$_lt_function_replace_fail" = x":"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Unable to substitute extended shell functions in $ofile" >&5 +$as_echo "$as_me: WARNING: Unable to substitute extended shell functions in $ofile" >&2;} +fi + + + mv -f "$cfgfile" "$ofile" || + (rm -f "$ofile" && cp "$cfgfile" "$ofile" && rm -f "$cfgfile") + chmod +x "$ofile" + + + cat <<_LT_EOF >> "$ofile" + +# ### BEGIN LIBTOOL TAG CONFIG: CXX + +# The linker used to build libraries. +LD=$lt_LD_CXX + +# How to create reloadable object files. +reload_flag=$lt_reload_flag_CXX +reload_cmds=$lt_reload_cmds_CXX + +# Commands used to build an old-style archive. +old_archive_cmds=$lt_old_archive_cmds_CXX + +# A language specific compiler. +CC=$lt_compiler_CXX + +# Is the compiler the GNU compiler? +with_gcc=$GCC_CXX + +# Compiler flag to turn off builtin functions. +no_builtin_flag=$lt_lt_prog_compiler_no_builtin_flag_CXX + +# Additional compiler flags for building library objects. +pic_flag=$lt_lt_prog_compiler_pic_CXX + +# How to pass a linker flag through the compiler. +wl=$lt_lt_prog_compiler_wl_CXX + +# Compiler flag to prevent dynamic linking. +link_static_flag=$lt_lt_prog_compiler_static_CXX + +# Does compiler simultaneously support -c and -o options? +compiler_c_o=$lt_lt_cv_prog_compiler_c_o_CXX + +# Whether or not to add -lc for building shared libraries. +build_libtool_need_lc=$archive_cmds_need_lc_CXX + +# Whether or not to disallow shared libs when runtime libs are static. +allow_libtool_libs_with_static_runtimes=$enable_shared_with_static_runtimes_CXX + +# Compiler flag to allow reflexive dlopens. +export_dynamic_flag_spec=$lt_export_dynamic_flag_spec_CXX + +# Compiler flag to generate shared objects directly from archives. +whole_archive_flag_spec=$lt_whole_archive_flag_spec_CXX + +# Whether the compiler copes with passing no objects directly. +compiler_needs_object=$lt_compiler_needs_object_CXX + +# Create an old-style archive from a shared archive. +old_archive_from_new_cmds=$lt_old_archive_from_new_cmds_CXX + +# Create a temporary old-style archive to link instead of a shared archive. +old_archive_from_expsyms_cmds=$lt_old_archive_from_expsyms_cmds_CXX + +# Commands used to build a shared archive. +archive_cmds=$lt_archive_cmds_CXX +archive_expsym_cmds=$lt_archive_expsym_cmds_CXX + +# Commands used to build a loadable module if different from building +# a shared archive. +module_cmds=$lt_module_cmds_CXX +module_expsym_cmds=$lt_module_expsym_cmds_CXX + +# Whether we are building with GNU ld or not. +with_gnu_ld=$lt_with_gnu_ld_CXX + +# Flag that allows shared libraries with undefined symbols to be built. +allow_undefined_flag=$lt_allow_undefined_flag_CXX + +# Flag that enforces no undefined symbols. +no_undefined_flag=$lt_no_undefined_flag_CXX + +# Flag to hardcode \$libdir into a binary during linking. +# This must work even if \$libdir does not exist +hardcode_libdir_flag_spec=$lt_hardcode_libdir_flag_spec_CXX + +# Whether we need a single "-rpath" flag with a separated argument. +hardcode_libdir_separator=$lt_hardcode_libdir_separator_CXX + +# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes +# DIR into the resulting binary. +hardcode_direct=$hardcode_direct_CXX + +# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes +# DIR into the resulting binary and the resulting library dependency is +# "absolute",i.e impossible to change by setting \${shlibpath_var} if the +# library is relocated. +hardcode_direct_absolute=$hardcode_direct_absolute_CXX + +# Set to "yes" if using the -LDIR flag during linking hardcodes DIR +# into the resulting binary. +hardcode_minus_L=$hardcode_minus_L_CXX + +# Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR +# into the resulting binary. +hardcode_shlibpath_var=$hardcode_shlibpath_var_CXX + +# Set to "yes" if building a shared library automatically hardcodes DIR +# into the library and all subsequent libraries and executables linked +# against it. +hardcode_automatic=$hardcode_automatic_CXX + +# Set to yes if linker adds runtime paths of dependent libraries +# to runtime path list. +inherit_rpath=$inherit_rpath_CXX + +# Whether libtool must link a program against all its dependency libraries. +link_all_deplibs=$link_all_deplibs_CXX + +# Set to "yes" if exported symbols are required. +always_export_symbols=$always_export_symbols_CXX + +# The commands to list exported symbols. +export_symbols_cmds=$lt_export_symbols_cmds_CXX + +# Symbols that should not be listed in the preloaded symbols. +exclude_expsyms=$lt_exclude_expsyms_CXX + +# Symbols that must always be exported. +include_expsyms=$lt_include_expsyms_CXX + +# Commands necessary for linking programs (against libraries) with templates. +prelink_cmds=$lt_prelink_cmds_CXX + +# Commands necessary for finishing linking programs. +postlink_cmds=$lt_postlink_cmds_CXX + +# Specify filename containing input files. +file_list_spec=$lt_file_list_spec_CXX + +# How to hardcode a shared library path into an executable. +hardcode_action=$hardcode_action_CXX + +# The directories searched by this compiler when creating a shared library. +compiler_lib_search_dirs=$lt_compiler_lib_search_dirs_CXX + +# Dependencies to place before and after the objects being linked to +# create a shared library. +predep_objects=$lt_predep_objects_CXX +postdep_objects=$lt_postdep_objects_CXX +predeps=$lt_predeps_CXX +postdeps=$lt_postdeps_CXX + +# The library search path used internally by the compiler when linking +# a shared library. +compiler_lib_search_path=$lt_compiler_lib_search_path_CXX + +# ### END LIBTOOL TAG CONFIG: CXX +_LT_EOF + + ;; + + esac +done # for ac_tag + + +as_fn_exit 0 +_ACEOF +ac_clean_files=$ac_clean_files_save + +test $ac_write_fail = 0 || + as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 + + +# configure is writing to config.log, and then calls config.status. +# config.status does its own redirection, appending to config.log. +# Unfortunately, on DOS this fails, as config.log is still kept open +# by configure, so config.status won't be able to write to it; its +# output is simply discarded. So we exec the FD to /dev/null, +# effectively closing config.log, so it can be properly (re)opened and +# appended to by config.status. When coming back to configure, we +# need to make the FD available again. +if test "$no_create" != yes; then + ac_cs_success=: + ac_config_status_args= + test "$silent" = yes && + ac_config_status_args="$ac_config_status_args --quiet" + exec 5>/dev/null + $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false + exec 5>>config.log + # Use ||, not &&, to avoid exiting from the if with $? = 1, which + # would make configure fail if this is the last instruction. + $ac_cs_success || as_fn_exit 1 +fi + +# +# CONFIG_SUBDIRS section. +# +if test "$no_recursion" != yes; then + + # Remove --cache-file, --srcdir, and --disable-option-checking arguments + # so they do not pile up. + ac_sub_configure_args= + ac_prev= + eval "set x $ac_configure_args" + shift + for ac_arg + do + if test -n "$ac_prev"; then + ac_prev= + continue + fi + case $ac_arg in + -cache-file | --cache-file | --cache-fil | --cache-fi \ + | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) + ac_prev=cache_file ;; + -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ + | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* \ + | --c=*) + ;; + --config-cache | -C) + ;; + -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) + ac_prev=srcdir ;; + -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) + ;; + -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) + ac_prev=prefix ;; + -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) + ;; + --disable-option-checking) + ;; + *) + case $ac_arg in + *\'*) ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + as_fn_append ac_sub_configure_args " '$ac_arg'" ;; + esac + done + + # Always prepend --prefix to ensure using the same prefix + # in subdir configurations. + ac_arg="--prefix=$prefix" + case $ac_arg in + *\'*) ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + ac_sub_configure_args="'$ac_arg' $ac_sub_configure_args" + + # Pass --silent + if test "$silent" = yes; then + ac_sub_configure_args="--silent $ac_sub_configure_args" + fi + + # Always prepend --disable-option-checking to silence warnings, since + # different subdirs can have different --enable and --with options. + ac_sub_configure_args="--disable-option-checking $ac_sub_configure_args" + + ac_popdir=`pwd` + for ac_dir in : $subdirs; do test "x$ac_dir" = x: && continue + + # Do not complain, so a configure script can configure whichever + # parts of a large source tree are present. + test -d "$srcdir/$ac_dir" || continue + + ac_msg="=== configuring in $ac_dir (`pwd`/$ac_dir)" + $as_echo "$as_me:${as_lineno-$LINENO}: $ac_msg" >&5 + $as_echo "$ac_msg" >&6 + as_dir="$ac_dir"; as_fn_mkdir_p + ac_builddir=. + +case "$ac_dir" in +.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; +*) + ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` + # A ".." for each directory in $ac_dir_suffix. + ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + case $ac_top_builddir_sub in + "") ac_top_builddir_sub=. ac_top_build_prefix= ;; + *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; + esac ;; +esac +ac_abs_top_builddir=$ac_pwd +ac_abs_builddir=$ac_pwd$ac_dir_suffix +# for backward compatibility: +ac_top_builddir=$ac_top_build_prefix + +case $srcdir in + .) # We are building in place. + ac_srcdir=. + ac_top_srcdir=$ac_top_builddir_sub + ac_abs_top_srcdir=$ac_pwd ;; + [\\/]* | ?:[\\/]* ) # Absolute name. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir + ac_abs_top_srcdir=$srcdir ;; + *) # Relative name. + ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_build_prefix$srcdir + ac_abs_top_srcdir=$ac_pwd/$srcdir ;; +esac +ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix + + + cd "$ac_dir" + + # Check for guested configure; otherwise get Cygnus style configure. + if test -f "$ac_srcdir/configure.gnu"; then + ac_sub_configure=$ac_srcdir/configure.gnu + elif test -f "$ac_srcdir/configure"; then + ac_sub_configure=$ac_srcdir/configure + elif test -f "$ac_srcdir/configure.in"; then + # This should be Cygnus configure. + ac_sub_configure=$ac_aux_dir/configure + else + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: no configuration information is in $ac_dir" >&5 +$as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2;} + ac_sub_configure= + fi + + # The recursion is here. + if test -n "$ac_sub_configure"; then + # Make the cache file name correct relative to the subdirectory. + case $cache_file in + [\\/]* | ?:[\\/]* ) ac_sub_cache_file=$cache_file ;; + *) # Relative name. + ac_sub_cache_file=$ac_top_build_prefix$cache_file ;; + esac + + { $as_echo "$as_me:${as_lineno-$LINENO}: running $SHELL $ac_sub_configure $ac_sub_configure_args --cache-file=$ac_sub_cache_file --srcdir=$ac_srcdir" >&5 +$as_echo "$as_me: running $SHELL $ac_sub_configure $ac_sub_configure_args --cache-file=$ac_sub_cache_file --srcdir=$ac_srcdir" >&6;} + # The eval makes quoting arguments work. + eval "\$SHELL \"\$ac_sub_configure\" $ac_sub_configure_args \ + --cache-file=\"\$ac_sub_cache_file\" --srcdir=\"\$ac_srcdir\"" || + as_fn_error $? "$ac_sub_configure failed for $ac_dir" "$LINENO" 5 + fi + + cd "$ac_popdir" + done +fi +if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 +$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} +fi + diff --git a/ceph/configure.ac b/ceph/configure.ac new file mode 100644 index 00000000..7255c7c9 --- /dev/null +++ b/ceph/configure.ac @@ -0,0 +1,795 @@ +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. + +# Autoconf +AC_PREREQ(2.59) + +# NOTE: This version is _only_ used for naming the tarball. The +# VERSION define is not used by the code. It gets a version string +# from 'git describe'; see src/ceph_ver.[ch] + +AC_INIT([ceph], [0.80.7], [ceph-devel@vger.kernel.org]) + +# Create release string. Used with VERSION for RPMs. +RPM_RELEASE=0 +AC_SUBST(RPM_RELEASE) +if test -d ".git" ; then + AC_CHECK_PROG(GIT_CHECK, git, yes) + if test x"$GIT_CHECK" = x"yes"; then + RPM_RELEASE=`if expr index $(git describe --always) '-' > /dev/null ; then git describe --always | cut -d- -f2- | tr '-' '.' ; else echo "0"; fi` + fi +fi +AC_MSG_NOTICE([RPM_RELEASE='$RPM_RELEASE']) + +AC_CONFIG_MACRO_DIR([m4]) + +AC_CONFIG_SUBDIRS([src/gtest]) + +# Environment +AC_CANONICAL_HOST +AC_CANONICAL_TARGET + +# Fix automake problems in 1.12 +m4_ifdef([AM_PROG_AR], [AM_PROG_AR]) + +# Automake +AM_INIT_AUTOMAKE +AM_PROG_CC_C_O +AM_PROG_LIBTOOL +AM_PROG_AS + + +# enable make V=0 (if automake >1.11) +AM_INIT_AUTOMAKE([foreign]) +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) + +# Platform +case "${target_os}" in +darwin*) + AC_DEFINE([DARWIN], [1], [Define if darwin/osx]) + ;; +linux*) + linux="yes" + ;; +freebsd*) + freebsd="yes" + ;; +esac +AM_CONDITIONAL(LINUX, test x"$linux" = x"yes") +AM_CONDITIONAL(FREEBSD, test x"$freebsd" = x"yes") + +# Checks for programs. +AC_PROG_CXX +if test "$CXX" = no || test "$CXX:$GXX" = "g++:"; then + AC_MSG_ERROR([no C++ compiler found]) +fi + +AC_MSG_CHECKING([if compiler is clang]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [[ +#ifndef __clang__ +#error "Not Clang" +#endif +return 0; +]])], +[CLANG=yes], [CLANG=no]) +AC_MSG_RESULT([$CLANG]) +AM_CONDITIONAL(CLANG, test "$CLANG" = "yes") + +#AC_PROG_CC +AC_PROG_MAKE_SET +AC_PROG_LIBTOOL + +# Compiler flags + +AC_SUBST(AM_CXXFLAGS) +AM_CXXFLAGS="${AM_CXXFLAGS}" + +# Check for yasm +if yasm -f elf64 src/common/crc32c_intel_fast_asm.S -o /dev/null; then + echo 'we have a modern and working yasm' + if test `arch` = "x86_64"; then + echo 'we are x86_64' + AC_DEFINE([HAVE_GOOD_YASM_ELF64], [1], [we have a recent yasm and are x86_64]) + with_good_yasm=yes + fi +else + echo 'we do not have a modern/working yasm' +fi +AM_CONDITIONAL(WITH_GOOD_YASM_ELF64, test "$with_good_yasm" = "yes") + +# Checks for compiler warning types + +# AC_CHECK_CC_FLAG(FLAG_TO_TEST, VARIABLE_TO_SET_IF_SUPPORTED) +# --------- +AC_DEFUN([AC_CHECK_CC_FLAG], +[{ + AC_LANG_PUSH([C]) + my_cflags_save="$CFLAGS" + CFLAGS="$my_cflags_save $1" + AC_MSG_CHECKING([whether $CC accepts $1]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ ]])], + [AC_MSG_RESULT([yes]); AC_SUBST([$2], ["$1"])], + [AC_MSG_RESULT([no])] + ) + CFLAGS="$my_cflags_save" + AC_LANG_POP([C]) +}]) + +AC_CHECK_CC_FLAG([-Wtype-limits], [WARN_TYPE_LIMITS]) +AC_CHECK_CC_FLAG([-Wignored-qualifiers], [WARN_IGNORED_QUALIFIERS]) + +# Checks for architecture stuff +AM_CONDITIONAL([ENABLE_FPU_NEON], [case $target_cpu in arm*) true;; *) false;; esac]) + +# Check for compiler VTA support +AX_CHECK_COMPILE_FLAG([-fvar-tracking-assignments], [HAS_VTA_SUPPORT=1], [HAS_VTA_SUPPORT=0]) +AM_CONDITIONAL(COMPILER_HAS_VTA, [test "$HAS_VTA_SUPPORT" = 1]) + +AX_CXX_STATIC_CAST +AX_C_VAR_FUNC +AX_C_PRETTY_FUNC + +# Checks for libraries. +ACX_PTHREAD + +AC_CHECK_LIB([uuid], [uuid_parse], [true], AC_MSG_FAILURE([libuuid not found])) + +if test x"$linux" = x"yes"; then + AC_CHECK_LIB([blkid], [blkid_devno_to_wholedisk], [true], AC_MSG_FAILURE([libblkid not found])) +fi + +# +# Check for res_nquery in libresolv. There are several variations. On OSX +# res_nquery is a macro defined in resolv.h, so the typical AC_CHECK_LIB +# doesn't work. On FreeBSD res_nquery can be found in libc. The required +# library for linking (if any) is defined RESOLV_LIBS. +# +AC_CHECK_HEADER([resolv.h], [], [], [#include ]) + +AC_DEFUN([CHECK_RESOLV_LIBS], [{ + AC_MSG_CHECKING([if res_nquery will link (LIBS=$1)]) + saved_LIBS="${LIBS}" + LIBS="$1" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + ]], [[res_nquery(0, 0, 0, 0, 0, 0);]])], + [AC_MSG_RESULT([yes]) + [$2]], + AC_MSG_RESULT([no])) + LIBS="${saved_LIBS}" +}]) + +RESOLV_LIBS="" +CHECK_RESOLV_LIBS([$RESOLV_LIBS], [resolv_libs="ok"]) +if test x"$resolv_libs" != "xok"; then + RESOLV_LIBS="-lresolv" + CHECK_RESOLV_LIBS([$RESOLV_LIBS], [resolv_libs="ok"]) + if test x"$resolv_libs" != "xok"; then + AC_MSG_FAILURE([no resolv library found]) + fi +fi +AC_SUBST([RESOLV_LIBS]) + +dnl check for libkeyutils on linux +KEYUTILS_LIB="" +AS_IF([test x"$linux" = x"yes"], [ + AC_CHECK_LIB([keyutils], [add_key], [KEYUTILS_LIB="-lkeyutils"], [ + AC_MSG_FAILURE([libkeyutils not found])])]) +AC_SUBST(KEYUTILS_LIB) + +AC_CHECK_LIB([m], [pow], [true], AC_MSG_FAILURE([libm not found])) +AC_CHECK_FUNCS([syncfs], AC_DEFINE([HAVE_SYS_SYNCFS], [1], [we have syncfs]), []) + +# Find some crypto library for us to use, while letting user to decide which one to use. +AC_ARG_WITH([cryptopp], + [AS_HELP_STRING([--with-cryptopp], [Use cryptographic functions from cryptopp])], + [], + [with_cryptopp=check]) +have_cryptopp=no +# this looks clumsy but it's just if A then { success } else { if B then success } +AS_IF([test "x$with_cryptopp" != "xno"], + [PKG_CHECK_MODULES([CRYPTOPP], + [libcrypto++], + [have_cryptopp=yes], + [ + AC_LANG_PUSH([C++]) + SAVED_CXXFLAGS="${CXXFLAGS}" + SAVED_LIBS="${LIBS}" + LIBS="${LIBS} ${PTHREAD_LIBS}" + CXXFLAGS="${CXXFLAGS} ${PTHREAD_CFLAGS}" + AC_SEARCH_LIBS([_ZTIN8CryptoPP14CBC_EncryptionE], [crypto++ cryptopp], + [have_cryptopp=yes], + [true], []) + CRYPTOPP_LIBS="${ac_cv_search__ZTIN8CryptoPP14CBC_EncryptionE}" + LIBS="${SAVED_LIBS}" + CXXFLAGS="${SAVED_CXXFLAGS}" + AC_LANG_POP([C++]) + ])]) +# bail out if given explicit --with-cryptopp +if test "x$have_cryptopp" = "xno" -a "x$with_cryptopp" != "xcheck" -a "x$with_cryptopp" != "xno"; then + AC_MSG_FAILURE([--with-cryptopp was given, but library was not found]) +fi + +AC_ARG_WITH([nss], + [AS_HELP_STRING([--with-nss], [Use cryptographic functions from nss])], + [], + [with_nss=check]) +have_nss=no +AS_IF([test "x$with_nss" != "xno"], + [PKG_CHECK_MODULES([NSS], [nss], [have_nss=yes], [true])]) +# bail out if given explicit --with-nss +if test "x$have_nss" = "xno" -a "x$with_nss" != "xcheck" -a "x$with_nss" != "xno"; then + AC_MSG_FAILURE([--with-nss was given, but library was not found]) +fi + +# now decide which crypto library to really use +if test "x$have_cryptopp" = "xyes"; then + AC_MSG_NOTICE([using cryptopp for cryptography]) + AC_DEFINE([USE_CRYPTOPP], [1], [Define if using CryptoPP.]) + AC_SUBST([CRYPTO_CFLAGS], [$CRYPTOPP_CFLAGS]) + #AC_SUBST([CRYPTO_CXXFLAGS], [$CRYPTOPP_CXXFLAGS]) + AM_CXXFLAGS="${AM_CXXFLAGS} ${CRYPTOPP_CXXFLAGS}" + AC_SUBST([CRYPTO_LIBS], [$CRYPTOPP_LIBS]) +elif test "x$have_nss" = "xyes"; then + AC_MSG_NOTICE([using nss for cryptography]) + AC_DEFINE([USE_NSS], [1], [Define if using NSS.]) + AC_SUBST([CRYPTO_CFLAGS], [$NSS_CFLAGS]) + # this needs CFLAGS too in practise to get the includes right. ugly. + #AC_SUBST([CRYPTO_CXXFLAGS], [$NSS_CFLAGS $NSS_CXXFLAGS]) + AM_CXXFLAGS="${AM_CXXFLAGS} ${NSS_CFLAGS} ${NSS_CXXFLAGS}" + AC_SUBST([CRYPTO_LIBS], [$NSS_LIBS]) +else + AC_MSG_FAILURE([no suitable crypto library found]) +fi + +# profiler? +AC_ARG_WITH([profiler], + [AS_HELP_STRING([--with-profiler], [build extra profiler binaries])], + [case "${withval}" in + yes) with_profiler=yes ;; + no) with_profiler=no ;; + *) AC_MSG_ERROR([bad value ${withval} for --with-profiler]) ;; + esac], + [with_profiler=no]) +AS_IF([test "x$with_profiler" = xyes], + [AC_CHECK_LIB([profiler], [ProfilerFlush], [], + [AC_MSG_FAILURE([--with-profiler was given but libprofiler (libgoogle-perftools-dev on debian) not found])])], + []) +AM_CONDITIONAL(WITH_PROFILER, test "$with_profiler" = "yes") +AS_IF([test "$with_profiler" = "yes"], + [AC_DEFINE([HAVE_PROFILER], [1], [Define if you have perftools profiler enabled])], + []) + +# debug crap? +AC_ARG_WITH([debug], + [AS_HELP_STRING([--with-debug], [build extra debug binaries])], + [case "${withval}" in + yes) with_debug=yes ;; + no) with_debug=no ;; + *) AC_MSG_ERROR([bad value ${withval} for --with-debug]) ;; + esac], + [with_debug=no]) +AM_CONDITIONAL(WITH_DEBUG, test "$with_debug" = "yes") + +AC_DEFINE([DEBUG_GATHER], [1], [Define if you want C_Gather debugging]) + +# code coverage? +AC_ARG_ENABLE([coverage], + [AS_HELP_STRING([--enable-coverage], [enable code coverage tracking])], + [], + [enable_coverage=no]) +AM_CONDITIONAL(ENABLE_COVERAGE, test "x$enable_coverage" != xno) +if test "x$enable_coverage" != xno; then + AC_DEFINE([ENABLE_COVERAGE], [1], [Define if enabling coverage.]) +fi +AC_SUBST(GCOV_PREFIX_STRIP, `echo $(pwd)/src | tr -dc / | wc -c`) + +# radosgw? +AC_ARG_WITH([radosgw], + [AS_HELP_STRING([--with-radosgw], [build RADOS gateway])], + [], + [with_radosgw=check]) +RADOSGW=0 +AS_IF([test "x$with_radosgw" != xno], + [AC_CHECK_LIB([fcgi], [FCGX_Init], + [AC_CHECK_LIB([expat], [XML_Parse], + [AC_CHECK_LIB([curl], [curl_easy_init], + [RADOSGW=1 + AC_CHECK_HEADER([fastcgi/fcgiapp.h], + [AC_DEFINE([FASTCGI_INCLUDE_DIR], [1], [FastCGI headers are in /usr/include/fastcgi])]) + ], + [if test "x$with_radosgw" != "xcheck"; then + AC_MSG_FAILURE([--with-radosgw was given but libcurl (libcurl-dev on debian) not found]) + fi]) + ], + [if test "x$with_radosgw" != "xcheck"; then + AC_MSG_FAILURE([--with-radosgw was given but libexpat (libexpat1-dev on debian) not found]) + fi]) + ], + [if test "x$with_radosgw" != "xcheck"; then + AC_MSG_FAILURE([--with-radosgw was given but libfcgi (libfcgi-dev on debian) not found]) + fi])]) +AM_CONDITIONAL(WITH_RADOSGW, test "$RADOSGW" = "1") + +AS_IF([test "$RADOSGW" = "1"], [AC_DEFINE([WITH_RADOSGW], [1], [define if radosgw enabled])]) + +AS_IF([test "$RADOSGW" = "1"], + [AC_CHECK_LIB([curl], [curl_multi_wait], + AC_DEFINE([HAVE_CURL_MULTI_WAIT], [1], [Define if have curl_multi_wait()])) + ]) + +# fuse? +AC_ARG_WITH([fuse], + [AS_HELP_STRING([--without-fuse], [disable FUSE userspace client])], + [], + [with_fuse=yes]) +LIBFUSE= +AS_IF([test "x$with_fuse" != xno], + [AC_CHECK_LIB([fuse], [fuse_main], + [AC_SUBST([LIBFUSE], ["-lfuse"]) + AC_DEFINE([HAVE_LIBFUSE], [1], + [Define if you have fuse]) + HAVE_LIBFUSE=1 + # look for fuse_getgroups and define FUSE_GETGROUPS if found + LIBS_saved="$LIBS" + LIBS="$LIBS -lfuse" + AC_CHECK_FUNCS([fuse_getgroups]) + LIBS="$LIBS_saved" + ], + [AC_MSG_FAILURE( + [no FUSE found (use --without-fuse to disable)])])]) +AM_CONDITIONAL(WITH_FUSE, [test "$HAVE_LIBFUSE" = "1"]) + +# tcmalloc? +AC_ARG_WITH([tcmalloc], + [AS_HELP_STRING([--without-tcmalloc], [disable tcmalloc for memory allocations])], + [], + [with_tcmalloc=yes]) +TCMALLOC= +AS_IF([test "x$with_tcmalloc" != xno], + [AC_CHECK_LIB([tcmalloc], [malloc], + [AC_SUBST([LIBTCMALLOC], ["-ltcmalloc"]) + AC_DEFINE([HAVE_LIBTCMALLOC], [1], + [Define if you have tcmalloc]) + HAVE_LIBTCMALLOC=1 + ], + [AC_MSG_FAILURE( + [no tcmalloc found (use --without-tcmalloc to disable)])])]) +AM_CONDITIONAL(WITH_TCMALLOC, [test "$HAVE_LIBTCMALLOC" = "1"]) + +#set pg ref debugging? +AC_ARG_ENABLE([pgrefdebugging], + [AS_HELP_STRING([--enable-pgrefdebugging], [enable pg ref debugging])], + [AC_DEFINE([PG_DEBUG_REFS], [1], [Defined if you want pg ref debugging])], + []) + +# +# Java is painful +# - adapted from OMPI wrappers package +# - this might become bigger. maybe should be own m4 file +# +AC_ARG_ENABLE(cephfs-java, + [AC_HELP_STRING([--enable-cephfs-java], [build libcephfs Java bindings])], + [], [enable_cephfs_java=no]) + +AM_CONDITIONAL(ENABLE_CEPHFS_JAVA, [test "x$enable_cephfs_java" = "xyes"]) + +AC_ARG_WITH(jdk-dir, + AC_HELP_STRING([--with-jdk-dir(=DIR)], [Path to JDK directory])) + +if test "x$enable_cephfs_java" = "xyes"; then + + # setup bin/include dirs from --with-jdk-dir (search for jni.h, javac) + AS_IF([test -n "$with_jdk_dir"], [ + javac_prog=`find $with_jdk_dir/ -name javac | head -n 1` + AS_IF([test -x "$javac_prog"], [ + EXTRA_JDK_BIN_DIR=`dirname $javac_prog`]) + jnih=`find $with_jdk_dir/ -name jni.h | head -n 1` + AS_IF([test -r "$jnih"], [ + EXTRA_JDK_INC_DIR=`dirname $jnih`])]) + + # setup defaults for Debian default-jdk package (without --with-jdk-dir) + AS_IF([test -z "$with_jdk_dir"], [ + # This works with Debian's and CentOS' default-jdk package + for dir in '/usr/lib/jvm/default-java/' '/usr/lib/jvm/java/' '/usr/lib/jvm/java-gcj/'; do + # only test if a suitable path has not yet been found + AS_IF([test "$EXTRA_JDK_BIN_DIR" == ""], [ + AS_IF([test -x "$javac_prog"], [ + EXTRA_JDK_BIN_DIR=`dirname $javac_prog`]) + jnih=`find $dir -name jni.h | head -n 1` + AS_IF([test -r "$jnih"], [ + EXTRA_JDK_INC_DIR=`dirname $jnih`]) + ]) + done + ]) + + # cephfs_java_test only makes sense if java is already turned on + # setup CLASSPATH for Debian default junit4.jar package + # + # Configuring --with-debug and --enable-cephfs-java will throw an error if + # JUnit4 cannot be found. While currently this works for users who have + # installed via the package manager on Ubuntu, we need to expand this + # check to 1 support other distrubtions and 2 allow users to influence + # the search path. + AS_IF([test "x$with_debug" = "xyes"], [ + dir='/usr/share/java' + junit4_jar=`find $dir -name junit4.jar | head -n 1` + AS_IF([test -r "$junit4_jar"], [ + EXTRA_CLASSPATH_JAR=`dirname $junit4_jar`/junit4.jar + AC_SUBST(EXTRA_CLASSPATH_JAR) + [have_junit4=1]], [ + AC_MSG_NOTICE([Cannot find junit4.jar (apt-get install junit4)]) + [have_junit4=0]])]) + + AC_CHECK_CLASSPATH + AC_PROG_JAVAC + AC_PROG_JAVAH + AC_PROG_JAR + + CLASSPATH=$CLASSPATH:$EXTRA_CLASSPATH_JAR + export CLASSPATH + AC_MSG_NOTICE([classpath - $CLASSPATH]) + + # Check for jni.h + CPPFLAGS_save=$CPPFLAGS + + AS_IF([test -n "$EXTRA_JDK_INC_DIR"], + [JDK_CPPFLAGS="-I$EXTRA_JDK_INC_DIR" + AS_IF([test -d "$EXTRA_JDK_INC_DIR/linux"], + [JDK_CPPFLAGS="$JDK_CPPFLAGS -I$EXTRA_JDK_INC_DIR/linux"]) + CPPFLAGS="$CPPFLAGS $JDK_CPPFLAGS"]) + + AC_CHECK_HEADER([jni.h], [], AC_MSG_ERROR([Cannot find header 'jni.h'. Try setting --with-jdk-dir])) + + CPPFLAGS=$CPPFLAGS_save + + # Setup output var + AC_SUBST(JDK_CPPFLAGS) +fi +AM_CONDITIONAL(HAVE_JUNIT4, [test "$have_junit4" = "1"]) + +# +# FreeBSD has it in base. +# +if test x"$freebsd" != x"yes"; then +PKG_CHECK_MODULES([LIBEDIT], [libedit >= 2.11], + [], AC_MSG_FAILURE([No usable version of libedit found.])) +else + LIBEDIT_LIBS="-ledit" +fi + +#libatomic-ops? You want it! +AC_ARG_WITH([libatomic-ops], + [AS_HELP_STRING([--without-libatomic-ops], + [disable libatomic-ops for the atomic_t type])], + [], + [with_libatomic_ops=yes]) +AS_IF([test "x$with_libatomic_ops" != xno], + [AC_CHECK_HEADER([atomic_ops.h], + [HAVE_ATOMIC_OPS=1], + [AC_MSG_FAILURE( + [no libatomic-ops found (use --without-libatomic-ops to disable)]) + ])]) +AS_IF([test "$HAVE_ATOMIC_OPS" = "1"], + [ + AC_CHECK_SIZEOF(AO_t, [], [ + #include + ]) + ], + [AC_DEFINE([NO_ATOMIC_OPS], [1], [Defined if you do not have atomic_ops])]) + + +AM_CONDITIONAL(WITH_LIBATOMIC, [test "$HAVE_ATOMIC_OPS" = "1"]) + +# newsyn? requires mpi. +#AC_ARG_WITH([newsyn], +# [AS_HELP_STRING([--with-newsyn], [build newsyn target requires mpi])], +# [], +# [with_newsyn=no]) + +AC_ARG_WITH([ocf], + [AS_HELP_STRING([--with-ocf], [build OCF-compliant cluster resource agent])], + , + [with_ocf=no]) +AM_CONDITIONAL(WITH_OCF, [ test "$with_ocf" = "yes" ]) + +# check is snappy-devel is installed, needed by leveldb +AC_CHECK_LIB([snappy], [snappy_compress], [true], [AC_MSG_FAILURE([libsnappy not found])]) +# use system leveldb +AC_CHECK_LIB([leveldb], [leveldb_open], [true], [AC_MSG_FAILURE([libleveldb not found])], [-lsnappy -lpthread]) +# see if we can use bloom filters with leveldb +AC_LANG_PUSH([C++]) +AC_CHECK_HEADER([leveldb/filter_policy.h], [AC_DEFINE([HAVE_LEVELDB_FILTER_POLICY], [1], [Defined if LevelDB supports bloom filters ])]) +AC_LANG_POP([C++]) + +# Find supported SIMD / SSE extensions supported by the compiler +AX_INTEL_FEATURES() + +# use system libs3? +AC_ARG_WITH([system-libs3], + [AS_HELP_STRING([--with-system-libs3], [use system libs3])], + , + [with_system_libs3=no]) +AS_IF([test "x$with_system_libs3" = xyes], + [AC_CHECK_LIB([s3], [S3_initialize], [true], [AC_MSG_FAILURE([libs3 not found])], [-lpthread])]) +AS_IF([test "x$with_system_libs3" = xcheck], + [AC_SEARCH_LIBS([S3_initialize], [s3], [with_system_libs3=yes], [true], [-lpthread])]) +AM_CONDITIONAL(WITH_SYSTEM_LIBS3, [ test "$with_system_libs3" = "yes" ]) + +# rest-bench? +AC_ARG_WITH([rest-bench], + [AS_HELP_STRING([--with-rest-bench], [enables rest-bench])], + [], + [with_rest_bench=no]) +AM_CONDITIONAL(WITH_REST_BENCH, [ test "$with_rest_bench" = "yes" ]) + +# use libaio? +AC_ARG_WITH([libaio], + [AS_HELP_STRING([--without-libaio], [disable libaio use by journal])], + , + [with_libaio=yes]) +AS_IF([test "x$with_libaio" != xno], + [AC_CHECK_LIB([aio], [io_submit], [true], AC_MSG_FAILURE([libaio not found]))]) +AS_IF([test "x$with_libaio" != xno], + [AC_CHECK_HEADER([libaio.h])]) +AS_IF([test "$with_libaio" = "yes"], + [AC_DEFINE([HAVE_LIBAIO], [1], [Defined if you don't have atomic_ops])]) +AM_CONDITIONAL(WITH_LIBAIO, [ test "$with_libaio" = "yes" ]) + +# use libxfs? +AC_ARG_WITH([libxfs], + [AS_HELP_STRING([--without-libxfs], [disable libxfs use by FileStore])], + [], + [with_libxfs=yes]) +AS_IF([test "x$with_libxfs" != "xno"], [ + # xfs/xfs.h presence and XFS_XFLAG_EXTSIZE define + AC_CHECK_HEADER([xfs/xfs.h], [], AC_MSG_ERROR( + [xfs/xfs.h not found (--without-libxfs to disable)])) + AC_MSG_CHECKING([for XFS_XFLAG_EXTSIZE in xfs/xfs.h]) + AC_EGREP_CPP([yes_have_xfs_xflag_extsize], [ + #include + #ifdef XFS_XFLAG_EXTSIZE + yes_have_xfs_xflag_extsize + #endif + ], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_LIBXFS], [1], [Define to 1 if you have libxfs]) + ], [ + AC_MSG_RESULT([no]) + AC_MSG_ERROR([XFS_XFLAG_EXTSIZE not found (--without-libxfs to disable)]) + ]) +]) +AM_CONDITIONAL(WITH_LIBXFS, [test "x$with_libxfs" != "xno"]) + +# use libzfs +AC_ARG_WITH([libzfs], + [AS_HELP_STRING([--with-libzfs], [build ZFS support])], + , + [with_libzfs=no]) +AS_IF([test "x$with_libzfs" = xyes], + [PKG_CHECK_MODULES([LIBZFS], [zfs], [], [true])]) +AS_IF([test "x$with_libzfs" = xyes], + [AC_DEFINE([HAVE_LIBZFS], [1], [Defined if you have libzfs enabled])]) +AM_CONDITIONAL(WITH_LIBZFS, [ test "$with_libzfs" = "yes" ]) + +# Checks for header files. +AC_HEADER_DIRENT +AC_HEADER_STDC +AC_HEADER_SYS_WAIT + + +# spirit? +AC_LANG([C++]) + +AC_CHECK_HEADER([boost/spirit/include/classic_core.hpp], [], + [AC_CHECK_HEADER([boost/spirit.hpp], [use_bspirit_old_hdr=yes], + AC_MSG_FAILURE(["Can't find boost spirit headers"]))]) +AM_CONDITIONAL(USE_BOOST_SPIRIT_OLD_HDR, [test "$use_bspirit_old_hdr" = "yes"]) + +AC_CHECK_HEADER([boost/random/discrete_distribution.hpp], + [AC_DEFINE([HAVE_BOOST_RANDOM_DISCRETE_DISTRIBUTION], [], [have boost::random::discrete_distribution])], + []) + +AC_CHECK_HEADER([boost/statechart/state.hpp], [], + AC_MSG_FAILURE(["Can't find boost statechart headers; need 1.34 or later"])) +AC_CHECK_HEADER([boost/program_options/option.hpp], [], + AC_MSG_FAILURE(["Can't find boost program_options headers"])) + +# If we have the boost system library installed, then we may want to link +# with it. +AC_CHECK_LIB(boost_system-mt, main, [], + [AC_CHECK_LIB(boost_system, main, [], + AC_MSG_NOTICE(["Boost system library not found."]))]) + +# Find the right boost_thread library. +AC_CHECK_LIB(boost_thread-mt, main, [], + [AC_CHECK_LIB(boost_thread, main, [], + AC_MSG_FAILURE(["Boost thread library not found."]))]) + +# +# Check for boost_program_options library (defines BOOST_PROGRAM_OPTIONS_LIBS). +# +BOOST_PROGRAM_OPTIONS_LIBS="" +saved_LIBS="${LIBS}" +LIBS="" +AC_CHECK_LIB(boost_program_options-mt, main, [], + [AC_CHECK_LIB(boost_program_options, main, [], + AC_MSG_FAILURE(["Boost program options library not found."]))]) +BOOST_PROGRAM_OPTIONS_LIBS="${LIBS}" +LIBS="${saved_LIBS}" +AC_SUBST(BOOST_PROGRAM_OPTIONS_LIBS) + +AC_LANG([C]) + +AC_CHECK_MEMBER([struct fiemap_extent.fe_logical], + [AC_DEFINE([HAVE_FIEMAP_H], [], [linux/fiemap.h was found, fiemap ioctl will be used])], + [AC_MSG_NOTICE([linux/fiemap.h was not found or not usable; using local Ceph copy])], + [[#include ]]) + +AC_CHECK_HEADERS([ \ + arpa/inet.h \ + arpa/nameser_compat.h \ + linux/version.h \ + netdb.h \ + netinet/in.h \ + sys/file.h \ + sys/ioctl.h \ + sys/mount.h \ + sys/param.h \ + sys/socket.h \ + sys/statvfs.h \ + sys/time.h \ + sys/vfs.h \ + sys/xattr.h \ + syslog.h \ + utime.h \ +]) + +# sync_file_range +AC_CHECK_FUNC([sync_file_range], + [AC_DEFINE([HAVE_SYNC_FILE_RANGE], [], [sync_file_range(2) is supported])], + []) + +# fallocate +AC_CHECK_FUNC([fallocate], + [AC_DEFINE([CEPH_HAVE_FALLOCATE], [], [fallocate(2) is supported])], + []) + +# +# Test for time-related `struct stat` members. +# + +AC_CHECK_MEMBER([struct stat.st_mtim.tv_nsec], + [AC_DEFINE(HAVE_STAT_ST_MTIM_TV_NSEC, 1, + [Define if you have struct stat.st_mtim.tv_nsec])]) + +AC_CHECK_MEMBER([struct stat.st_mtimespec.tv_nsec], + [AC_DEFINE(HAVE_STAT_ST_MTIMESPEC_TV_NSEC, 1, + [Define if you have struct stat.st_mtimespec.tv_nsec])]) + +# splice/tee +AC_CHECK_FUNC([splice], + [AC_DEFINE([CEPH_HAVE_SPLICE], [], [splice(2) is supported])], + []) + +# F_SETPIPE_SZ in fcntl.h +AC_MSG_CHECKING([for F_SETPIPE_SZ in fcntl.h]) +AC_EGREP_CPP([yes_have_f_setpipe_sz], [ + #define _GNU_SOURCE + #include + #ifdef F_SETPIPE_SZ + yes_have_f_setpipe_sz + #endif +], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([CEPH_HAVE_SETPIPE_SZ], [], [F_SETPIPE_SZ is supported]) +], [ + AC_MSG_RESULT([no]) + AC_MSG_NOTICE([F_SETPIPE_SZ not found, zero-copy may be less efficent]) +]) + +AC_CHECK_FUNCS([posix_fallocate]) +AC_CHECK_HEADERS([sys/prctl.h]) +AC_CHECK_FUNCS([prctl]) +AC_CHECK_FUNCS([pipe2]) +AC_CHECK_FUNCS([posix_fadvise]) + +AC_MSG_CHECKING([for fdatasync]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#include +]], [[ +#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0 +return fdatasync(0); +#else +#error Not supported +#endif +]])], [ +AC_MSG_RESULT([yes]) +AC_DEFINE([HAVE_FDATASYNC], 1, [Define to 1 if you have fdatasync.]) +], [ +AC_MSG_RESULT([no]) +]) + +# +# Check for pthread spinlock (depends on ACX_PTHREAD) +# +saved_LIBS="$LIBS" +saved_CFLAGS="$CFLAGS" +LIBS="$PTHREAD_LIBS $LIBS" +CFLAGS="$PTHREAD_CFLAGS $CFLAGS" +AC_CHECK_FUNC([pthread_spin_init], + [AC_DEFINE(HAVE_PTHREAD_SPINLOCK, 1, [Define if you have pthread_spin_init])]) +LIBS="$saved_LIBS" +CFLAGS="$saved_CFLAGS" + +AC_CHECK_TYPES([int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, + int64_t, uint64_t]) + +dnl check for Linux types +AC_CHECK_HEADERS([linux/types.h]) +AC_CHECK_TYPES([__u8, __s8, __u16, __s16, __u32, __s32, __u64, __s64, __le16, + __be16, __le32, __be32, __le64, __be64], [], [], [[#include ]]) + +# Checks for typedefs, structures, and compiler characteristics. +#AC_HEADER_STDBOOL +#AC_C_CONST +#AC_TYPE_UID_T +#AC_C_INLINE +#AC_TYPE_INT16_T +#AC_TYPE_INT32_T +#AC_TYPE_INT64_T +#AC_TYPE_INT8_T +#AC_TYPE_MODE_T +#AC_TYPE_OFF_T +#AC_TYPE_PID_T +#AC_TYPE_SIZE_T +#AC_TYPE_SSIZE_T +#AC_CHECK_MEMBERS([struct stat.st_blksize]) +#AC_STRUCT_ST_BLOCKS +#AC_CHECK_MEMBERS([struct stat.st_rdev]) +#AC_HEADER_TIME +#AC_STRUCT_TM +#AC_TYPE_UINT16_T +#AC_TYPE_UINT32_T +#AC_TYPE_UINT64_T +#AC_TYPE_UINT8_T + +# Checks for library functions. +#AC_FUNC_CHOWN +#AC_FUNC_CLOSEDIR_VOID +#AC_FUNC_ERROR_AT_LINE +#AC_FUNC_FORK +#AC_PROG_GCC_TRADITIONAL +#AC_FUNC_LSTAT +#AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK +#AC_FUNC_MALLOC # this causes annoying rpl_malloc error on some machines; skip it +#AC_FUNC_MEMCMP +#AC_FUNC_MMAP +#AC_FUNC_REALLOC +#AC_FUNC_SELECT_ARGTYPES +#AC_TYPE_SIGNAL +#AC_FUNC_STAT +#AC_FUNC_UTIME_NULL +#AC_CHECK_FUNCS([bzero fchdir fdatasync floor ftruncate getcwd gethostbyname gethostname gettimeofday inet_ntoa localtime_r memmove memset mkdir munmap pow rmdir select socket sqrt strcasecmp strchr strerror strstr utime]) + +# check for return type (and presence) if strerror_r in C++ mode +AC_LANG_PUSH([C++]) +AC_FUNC_STRERROR_R +AC_LANG_POP([C++]) + +AM_CONDITIONAL(WITH_BUILD_TESTS, test "$WITH_BUILD_TESTS" = "1") + +AM_PATH_PYTHON([2.4], + [], [AC_MSG_FAILURE([Failed to find Python 2.4 or newer])]) + +AC_CONFIG_HEADERS([src/acconfig.h]) +AC_CONFIG_FILES([Makefile + src/Makefile + src/ocf/Makefile + src/ocf/ceph + src/ocf/rbd + src/java/Makefile + man/Makefile + ceph.spec]) +AC_OUTPUT diff --git a/ceph/depcomp b/ceph/depcomp new file mode 100755 index 00000000..bd0ac089 --- /dev/null +++ b/ceph/depcomp @@ -0,0 +1,688 @@ +#! /bin/sh +# depcomp - compile a program generating dependencies as side-effects + +scriptversion=2011-12-04.11; # UTC + +# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007, 2009, 2010, +# 2011 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# Originally written by Alexandre Oliva . + +case $1 in + '') + echo "$0: No command. Try \`$0 --help' for more information." 1>&2 + exit 1; + ;; + -h | --h*) + cat <<\EOF +Usage: depcomp [--help] [--version] PROGRAM [ARGS] + +Run PROGRAMS ARGS to compile a file, generating dependencies +as side-effects. + +Environment variables: + depmode Dependency tracking mode. + source Source file read by `PROGRAMS ARGS'. + object Object file output by `PROGRAMS ARGS'. + DEPDIR directory where to store dependencies. + depfile Dependency file to output. + tmpdepfile Temporary file to use when outputting dependencies. + libtool Whether libtool is used (yes/no). + +Report bugs to . +EOF + exit $? + ;; + -v | --v*) + echo "depcomp $scriptversion" + exit $? + ;; +esac + +if test -z "$depmode" || test -z "$source" || test -z "$object"; then + echo "depcomp: Variables source, object and depmode must be set" 1>&2 + exit 1 +fi + +# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po. +depfile=${depfile-`echo "$object" | + sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`} +tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`} + +rm -f "$tmpdepfile" + +# Some modes work just like other modes, but use different flags. We +# parameterize here, but still list the modes in the big case below, +# to make depend.m4 easier to write. Note that we *cannot* use a case +# here, because this file can only contain one case statement. +if test "$depmode" = hp; then + # HP compiler uses -M and no extra arg. + gccflag=-M + depmode=gcc +fi + +if test "$depmode" = dashXmstdout; then + # This is just like dashmstdout with a different argument. + dashmflag=-xM + depmode=dashmstdout +fi + +cygpath_u="cygpath -u -f -" +if test "$depmode" = msvcmsys; then + # This is just like msvisualcpp but w/o cygpath translation. + # Just convert the backslash-escaped backslashes to single forward + # slashes to satisfy depend.m4 + cygpath_u='sed s,\\\\,/,g' + depmode=msvisualcpp +fi + +if test "$depmode" = msvc7msys; then + # This is just like msvc7 but w/o cygpath translation. + # Just convert the backslash-escaped backslashes to single forward + # slashes to satisfy depend.m4 + cygpath_u='sed s,\\\\,/,g' + depmode=msvc7 +fi + +case "$depmode" in +gcc3) +## gcc 3 implements dependency tracking that does exactly what +## we want. Yay! Note: for some reason libtool 1.4 doesn't like +## it if -MD -MP comes after the -MF stuff. Hmm. +## Unfortunately, FreeBSD c89 acceptance of flags depends upon +## the command line argument order; so add the flags where they +## appear in depend2.am. Note that the slowdown incurred here +## affects only configure: in makefiles, %FASTDEP% shortcuts this. + for arg + do + case $arg in + -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;; + *) set fnord "$@" "$arg" ;; + esac + shift # fnord + shift # $arg + done + "$@" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + mv "$tmpdepfile" "$depfile" + ;; + +gcc) +## There are various ways to get dependency output from gcc. Here's +## why we pick this rather obscure method: +## - Don't want to use -MD because we'd like the dependencies to end +## up in a subdir. Having to rename by hand is ugly. +## (We might end up doing this anyway to support other compilers.) +## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like +## -MM, not -M (despite what the docs say). +## - Using -M directly means running the compiler twice (even worse +## than renaming). + if test -z "$gccflag"; then + gccflag=-MD, + fi + "$@" -Wp,"$gccflag$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + echo "$object : \\" > "$depfile" + alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz +## The second -e expression handles DOS-style file names with drive letters. + sed -e 's/^[^:]*: / /' \ + -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile" +## This next piece of magic avoids the `deleted header file' problem. +## The problem is that when a header file which appears in a .P file +## is deleted, the dependency causes make to die (because there is +## typically no way to rebuild the header). We avoid this by adding +## dummy dependencies for each header file. Too bad gcc doesn't do +## this for us directly. + tr ' ' ' +' < "$tmpdepfile" | +## Some versions of gcc put a space before the `:'. On the theory +## that the space means something, we add a space to the output as +## well. hp depmode also adds that space, but also prefixes the VPATH +## to the object. Take care to not repeat it in the output. +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \ + | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +hp) + # This case exists only to let depend.m4 do its work. It works by + # looking at the text of this script. This case will never be run, + # since it is checked for above. + exit 1 + ;; + +sgi) + if test "$libtool" = yes; then + "$@" "-Wp,-MDupdate,$tmpdepfile" + else + "$@" -MDupdate "$tmpdepfile" + fi + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + + if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files + echo "$object : \\" > "$depfile" + + # Clip off the initial element (the dependent). Don't try to be + # clever and replace this with sed code, as IRIX sed won't handle + # lines with more than a fixed number of characters (4096 in + # IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines; + # the IRIX cc adds comments like `#:fec' to the end of the + # dependency line. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \ + tr ' +' ' ' >> "$depfile" + echo >> "$depfile" + + # The second pass generates a dummy entry for each header file. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \ + >> "$depfile" + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +aix) + # The C for AIX Compiler uses -M and outputs the dependencies + # in a .u file. In older versions, this file always lives in the + # current directory. Also, the AIX compiler puts `$object:' at the + # start of each line; $object doesn't have directory information. + # Version 6 uses the directory in both cases. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + if test "$libtool" = yes; then + tmpdepfile1=$dir$base.u + tmpdepfile2=$base.u + tmpdepfile3=$dir.libs/$base.u + "$@" -Wc,-M + else + tmpdepfile1=$dir$base.u + tmpdepfile2=$dir$base.u + tmpdepfile3=$dir$base.u + "$@" -M + fi + stat=$? + + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + # Each line is of the form `foo.o: dependent.h'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile" + # That's a tab and a space in the []. + sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile" + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +icc) + # Intel's C compiler understands `-MD -MF file'. However on + # icc -MD -MF foo.d -c -o sub/foo.o sub/foo.c + # ICC 7.0 will fill foo.d with something like + # foo.o: sub/foo.c + # foo.o: sub/foo.h + # which is wrong. We want: + # sub/foo.o: sub/foo.c + # sub/foo.o: sub/foo.h + # sub/foo.c: + # sub/foo.h: + # ICC 7.1 will output + # foo.o: sub/foo.c sub/foo.h + # and will wrap long lines using \ : + # foo.o: sub/foo.c ... \ + # sub/foo.h ... \ + # ... + + "$@" -MD -MF "$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + # Each line is of the form `foo.o: dependent.h', + # or `foo.o: dep1.h dep2.h \', or ` dep3.h dep4.h \'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed "s,^[^:]*:,$object :," < "$tmpdepfile" > "$depfile" + # Some versions of the HPUX 10.20 sed can't process this invocation + # correctly. Breaking it into two sed invocations is a workaround. + sed 's,^[^:]*: \(.*\)$,\1,;s/^\\$//;/^$/d;/:$/d' < "$tmpdepfile" | + sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +hp2) + # The "hp" stanza above does not work with aCC (C++) and HP's ia64 + # compilers, which have integrated preprocessors. The correct option + # to use with these is +Maked; it writes dependencies to a file named + # 'foo.d', which lands next to the object file, wherever that + # happens to be. + # Much of this is similar to the tru64 case; see comments there. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + if test "$libtool" = yes; then + tmpdepfile1=$dir$base.d + tmpdepfile2=$dir.libs/$base.d + "$@" -Wc,+Maked + else + tmpdepfile1=$dir$base.d + tmpdepfile2=$dir$base.d + "$@" +Maked + fi + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile" + # Add `dependent.h:' lines. + sed -ne '2,${ + s/^ *// + s/ \\*$// + s/$/:/ + p + }' "$tmpdepfile" >> "$depfile" + else + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" "$tmpdepfile2" + ;; + +tru64) + # The Tru64 compiler uses -MD to generate dependencies as a side + # effect. `cc -MD -o foo.o ...' puts the dependencies into `foo.o.d'. + # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put + # dependencies in `foo.d' instead, so we check for that too. + # Subdirectories are respected. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + + if test "$libtool" = yes; then + # With Tru64 cc, shared objects can also be used to make a + # static library. This mechanism is used in libtool 1.4 series to + # handle both shared and static libraries in a single compilation. + # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d. + # + # With libtool 1.5 this exception was removed, and libtool now + # generates 2 separate objects for the 2 libraries. These two + # compilations output dependencies in $dir.libs/$base.o.d and + # in $dir$base.o.d. We have to check for both files, because + # one of the two compilations can be disabled. We should prefer + # $dir$base.o.d over $dir.libs/$base.o.d because the latter is + # automatically cleaned when .libs/ is deleted, while ignoring + # the former would cause a distcleancheck panic. + tmpdepfile1=$dir.libs/$base.lo.d # libtool 1.4 + tmpdepfile2=$dir$base.o.d # libtool 1.5 + tmpdepfile3=$dir.libs/$base.o.d # libtool 1.5 + tmpdepfile4=$dir.libs/$base.d # Compaq CCC V6.2-504 + "$@" -Wc,-MD + else + tmpdepfile1=$dir$base.o.d + tmpdepfile2=$dir$base.d + tmpdepfile3=$dir$base.d + tmpdepfile4=$dir$base.d + "$@" -MD + fi + + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile" + # That's a tab and a space in the []. + sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile" + else + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +msvc7) + if test "$libtool" = yes; then + showIncludes=-Wc,-showIncludes + else + showIncludes=-showIncludes + fi + "$@" $showIncludes > "$tmpdepfile" + stat=$? + grep -v '^Note: including file: ' "$tmpdepfile" + if test "$stat" = 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + echo "$object : \\" > "$depfile" + # The first sed program below extracts the file names and escapes + # backslashes for cygpath. The second sed program outputs the file + # name when reading, but also accumulates all include files in the + # hold buffer in order to output them again at the end. This only + # works with sed implementations that can handle large buffers. + sed < "$tmpdepfile" -n ' +/^Note: including file: *\(.*\)/ { + s//\1/ + s/\\/\\\\/g + p +}' | $cygpath_u | sort -u | sed -n ' +s/ /\\ /g +s/\(.*\)/ \1 \\/p +s/.\(.*\) \\/\1:/ +H +$ { + s/.*/ / + G + p +}' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +msvc7msys) + # This case exists only to let depend.m4 do its work. It works by + # looking at the text of this script. This case will never be run, + # since it is checked for above. + exit 1 + ;; + +#nosideeffect) + # This comment above is used by automake to tell side-effect + # dependency tracking mechanisms from slower ones. + +dashmstdout) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout, regardless of -o. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test "X$1" != 'X--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + test -z "$dashmflag" && dashmflag=-M + # Require at least two characters before searching for `:' + # in the target name. This is to cope with DOS-style filenames: + # a dependency such as `c:/foo/bar' could be seen as target `c' otherwise. + "$@" $dashmflag | + sed 's:^[ ]*[^: ][^:][^:]*\:[ ]*:'"$object"'\: :' > "$tmpdepfile" + rm -f "$depfile" + cat < "$tmpdepfile" > "$depfile" + tr ' ' ' +' < "$tmpdepfile" | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +dashXmstdout) + # This case only exists to satisfy depend.m4. It is never actually + # run, as this mode is specially recognized in the preamble. + exit 1 + ;; + +makedepend) + "$@" || exit $? + # Remove any Libtool call + if test "$libtool" = yes; then + while test "X$1" != 'X--mode=compile'; do + shift + done + shift + fi + # X makedepend + shift + cleared=no eat=no + for arg + do + case $cleared in + no) + set ""; shift + cleared=yes ;; + esac + if test $eat = yes; then + eat=no + continue + fi + case "$arg" in + -D*|-I*) + set fnord "$@" "$arg"; shift ;; + # Strip any option that makedepend may not understand. Remove + # the object too, otherwise makedepend will parse it as a source file. + -arch) + eat=yes ;; + -*|$object) + ;; + *) + set fnord "$@" "$arg"; shift ;; + esac + done + obj_suffix=`echo "$object" | sed 's/^.*\././'` + touch "$tmpdepfile" + ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@" + rm -f "$depfile" + # makedepend may prepend the VPATH from the source file name to the object. + # No need to regex-escape $object, excess matching of '.' is harmless. + sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile" + sed '1,2d' "$tmpdepfile" | tr ' ' ' +' | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" "$tmpdepfile".bak + ;; + +cpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test "X$1" != 'X--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + "$@" -E | + sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ + -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' | + sed '$ s: \\$::' > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + cat < "$tmpdepfile" >> "$depfile" + sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +msvisualcpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test "X$1" != 'X--mode=compile'; do + shift + done + shift + fi + + IFS=" " + for arg + do + case "$arg" in + -o) + shift + ;; + $object) + shift + ;; + "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI") + set fnord "$@" + shift + shift + ;; + *) + set fnord "$@" "$arg" + shift + shift + ;; + esac + done + "$@" -E 2>/dev/null | + sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s:: \1 \\:p' >> "$depfile" + echo " " >> "$depfile" + sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +msvcmsys) + # This case exists only to let depend.m4 do its work. It works by + # looking at the text of this script. This case will never be run, + # since it is checked for above. + exit 1 + ;; + +none) + exec "$@" + ;; + +*) + echo "Unknown depmode $depmode" 1>&2 + exit 1 + ;; +esac + +exit 0 + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/ceph/install-sh b/ceph/install-sh new file mode 100755 index 00000000..a9244eb0 --- /dev/null +++ b/ceph/install-sh @@ -0,0 +1,527 @@ +#!/bin/sh +# install - install a program, script, or datafile + +scriptversion=2011-01-19.21; # UTC + +# This originates from X11R5 (mit/util/scripts/install.sh), which was +# later released in X11R6 (xc/config/util/install.sh) with the +# following copyright and license. +# +# Copyright (C) 1994 X Consortium +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- +# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name of the X Consortium shall not +# be used in advertising or otherwise to promote the sale, use or other deal- +# ings in this Software without prior written authorization from the X Consor- +# tium. +# +# +# FSF changes to this file are in the public domain. +# +# Calling this script install-sh is preferred over install.sh, to prevent +# `make' implicit rules from creating a file called install from it +# when there is no Makefile. +# +# This script is compatible with the BSD install script, but was written +# from scratch. + +nl=' +' +IFS=" "" $nl" + +# set DOITPROG to echo to test this script + +# Don't use :- since 4.3BSD and earlier shells don't like it. +doit=${DOITPROG-} +if test -z "$doit"; then + doit_exec=exec +else + doit_exec=$doit +fi + +# Put in absolute file names if you don't have them in your path; +# or use environment vars. + +chgrpprog=${CHGRPPROG-chgrp} +chmodprog=${CHMODPROG-chmod} +chownprog=${CHOWNPROG-chown} +cmpprog=${CMPPROG-cmp} +cpprog=${CPPROG-cp} +mkdirprog=${MKDIRPROG-mkdir} +mvprog=${MVPROG-mv} +rmprog=${RMPROG-rm} +stripprog=${STRIPPROG-strip} + +posix_glob='?' +initialize_posix_glob=' + test "$posix_glob" != "?" || { + if (set -f) 2>/dev/null; then + posix_glob= + else + posix_glob=: + fi + } +' + +posix_mkdir= + +# Desired mode of installed file. +mode=0755 + +chgrpcmd= +chmodcmd=$chmodprog +chowncmd= +mvcmd=$mvprog +rmcmd="$rmprog -f" +stripcmd= + +src= +dst= +dir_arg= +dst_arg= + +copy_on_change=false +no_target_directory= + +usage="\ +Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE + or: $0 [OPTION]... SRCFILES... DIRECTORY + or: $0 [OPTION]... -t DIRECTORY SRCFILES... + or: $0 [OPTION]... -d DIRECTORIES... + +In the 1st form, copy SRCFILE to DSTFILE. +In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. +In the 4th, create DIRECTORIES. + +Options: + --help display this help and exit. + --version display version info and exit. + + -c (ignored) + -C install only if different (preserve the last data modification time) + -d create directories instead of installing files. + -g GROUP $chgrpprog installed files to GROUP. + -m MODE $chmodprog installed files to MODE. + -o USER $chownprog installed files to USER. + -s $stripprog installed files. + -t DIRECTORY install into DIRECTORY. + -T report an error if DSTFILE is a directory. + +Environment variables override the default commands: + CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG + RMPROG STRIPPROG +" + +while test $# -ne 0; do + case $1 in + -c) ;; + + -C) copy_on_change=true;; + + -d) dir_arg=true;; + + -g) chgrpcmd="$chgrpprog $2" + shift;; + + --help) echo "$usage"; exit $?;; + + -m) mode=$2 + case $mode in + *' '* | *' '* | *' +'* | *'*'* | *'?'* | *'['*) + echo "$0: invalid mode: $mode" >&2 + exit 1;; + esac + shift;; + + -o) chowncmd="$chownprog $2" + shift;; + + -s) stripcmd=$stripprog;; + + -t) dst_arg=$2 + # Protect names problematic for `test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + shift;; + + -T) no_target_directory=true;; + + --version) echo "$0 $scriptversion"; exit $?;; + + --) shift + break;; + + -*) echo "$0: invalid option: $1" >&2 + exit 1;; + + *) break;; + esac + shift +done + +if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then + # When -d is used, all remaining arguments are directories to create. + # When -t is used, the destination is already specified. + # Otherwise, the last argument is the destination. Remove it from $@. + for arg + do + if test -n "$dst_arg"; then + # $@ is not empty: it contains at least $arg. + set fnord "$@" "$dst_arg" + shift # fnord + fi + shift # arg + dst_arg=$arg + # Protect names problematic for `test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + done +fi + +if test $# -eq 0; then + if test -z "$dir_arg"; then + echo "$0: no input file specified." >&2 + exit 1 + fi + # It's OK to call `install-sh -d' without argument. + # This can happen when creating conditional directories. + exit 0 +fi + +if test -z "$dir_arg"; then + do_exit='(exit $ret); exit $ret' + trap "ret=129; $do_exit" 1 + trap "ret=130; $do_exit" 2 + trap "ret=141; $do_exit" 13 + trap "ret=143; $do_exit" 15 + + # Set umask so as not to create temps with too-generous modes. + # However, 'strip' requires both read and write access to temps. + case $mode in + # Optimize common cases. + *644) cp_umask=133;; + *755) cp_umask=22;; + + *[0-7]) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw='% 200' + fi + cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; + *) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw=,u+rw + fi + cp_umask=$mode$u_plus_rw;; + esac +fi + +for src +do + # Protect names problematic for `test' and other utilities. + case $src in + -* | [=\(\)!]) src=./$src;; + esac + + if test -n "$dir_arg"; then + dst=$src + dstdir=$dst + test -d "$dstdir" + dstdir_status=$? + else + + # Waiting for this to be detected by the "$cpprog $src $dsttmp" command + # might cause directories to be created, which would be especially bad + # if $src (and thus $dsttmp) contains '*'. + if test ! -f "$src" && test ! -d "$src"; then + echo "$0: $src does not exist." >&2 + exit 1 + fi + + if test -z "$dst_arg"; then + echo "$0: no destination specified." >&2 + exit 1 + fi + dst=$dst_arg + + # If destination is a directory, append the input filename; won't work + # if double slashes aren't ignored. + if test -d "$dst"; then + if test -n "$no_target_directory"; then + echo "$0: $dst_arg: Is a directory" >&2 + exit 1 + fi + dstdir=$dst + dst=$dstdir/`basename "$src"` + dstdir_status=0 + else + # Prefer dirname, but fall back on a substitute if dirname fails. + dstdir=` + (dirname "$dst") 2>/dev/null || + expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$dst" : 'X\(//\)[^/]' \| \ + X"$dst" : 'X\(//\)$' \| \ + X"$dst" : 'X\(/\)' \| . 2>/dev/null || + echo X"$dst" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q' + ` + + test -d "$dstdir" + dstdir_status=$? + fi + fi + + obsolete_mkdir_used=false + + if test $dstdir_status != 0; then + case $posix_mkdir in + '') + # Create intermediate dirs using mode 755 as modified by the umask. + # This is like FreeBSD 'install' as of 1997-10-28. + umask=`umask` + case $stripcmd.$umask in + # Optimize common cases. + *[2367][2367]) mkdir_umask=$umask;; + .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; + + *[0-7]) + mkdir_umask=`expr $umask + 22 \ + - $umask % 100 % 40 + $umask % 20 \ + - $umask % 10 % 4 + $umask % 2 + `;; + *) mkdir_umask=$umask,go-w;; + esac + + # With -d, create the new directory with the user-specified mode. + # Otherwise, rely on $mkdir_umask. + if test -n "$dir_arg"; then + mkdir_mode=-m$mode + else + mkdir_mode= + fi + + posix_mkdir=false + case $umask in + *[123567][0-7][0-7]) + # POSIX mkdir -p sets u+wx bits regardless of umask, which + # is incompatible with FreeBSD 'install' when (umask & 300) != 0. + ;; + *) + tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ + trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 + + if (umask $mkdir_umask && + exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 + then + if test -z "$dir_arg" || { + # Check for POSIX incompatibilities with -m. + # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or + # other-writeable bit of parent directory when it shouldn't. + # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. + ls_ld_tmpdir=`ls -ld "$tmpdir"` + case $ls_ld_tmpdir in + d????-?r-*) different_mode=700;; + d????-?--*) different_mode=755;; + *) false;; + esac && + $mkdirprog -m$different_mode -p -- "$tmpdir" && { + ls_ld_tmpdir_1=`ls -ld "$tmpdir"` + test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" + } + } + then posix_mkdir=: + fi + rmdir "$tmpdir/d" "$tmpdir" + else + # Remove any dirs left behind by ancient mkdir implementations. + rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null + fi + trap '' 0;; + esac;; + esac + + if + $posix_mkdir && ( + umask $mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" + ) + then : + else + + # The umask is ridiculous, or mkdir does not conform to POSIX, + # or it failed possibly due to a race condition. Create the + # directory the slow way, step by step, checking for races as we go. + + case $dstdir in + /*) prefix='/';; + [-=\(\)!]*) prefix='./';; + *) prefix='';; + esac + + eval "$initialize_posix_glob" + + oIFS=$IFS + IFS=/ + $posix_glob set -f + set fnord $dstdir + shift + $posix_glob set +f + IFS=$oIFS + + prefixes= + + for d + do + test X"$d" = X && continue + + prefix=$prefix$d + if test -d "$prefix"; then + prefixes= + else + if $posix_mkdir; then + (umask=$mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break + # Don't fail if two instances are running concurrently. + test -d "$prefix" || exit 1 + else + case $prefix in + *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; + *) qprefix=$prefix;; + esac + prefixes="$prefixes '$qprefix'" + fi + fi + prefix=$prefix/ + done + + if test -n "$prefixes"; then + # Don't fail if two instances are running concurrently. + (umask $mkdir_umask && + eval "\$doit_exec \$mkdirprog $prefixes") || + test -d "$dstdir" || exit 1 + obsolete_mkdir_used=true + fi + fi + fi + + if test -n "$dir_arg"; then + { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && + { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || + test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 + else + + # Make a couple of temp file names in the proper directory. + dsttmp=$dstdir/_inst.$$_ + rmtmp=$dstdir/_rm.$$_ + + # Trap to clean up those temp files at exit. + trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 + + # Copy the file name to the temp name. + (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + + # and set any options; do chmod last to preserve setuid bits. + # + # If any of these fail, we abort the whole thing. If we want to + # ignore errors from any of these, just make sure not to ignore + # errors from the above "$doit $cpprog $src $dsttmp" command. + # + { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && + { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && + { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && + + # If -C, don't bother to copy if it wouldn't change the file. + if $copy_on_change && + old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && + new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && + + eval "$initialize_posix_glob" && + $posix_glob set -f && + set X $old && old=:$2:$4:$5:$6 && + set X $new && new=:$2:$4:$5:$6 && + $posix_glob set +f && + + test "$old" = "$new" && + $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 + then + rm -f "$dsttmp" + else + # Rename the file to the real destination. + $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || + + # The rename failed, perhaps because mv can't rename something else + # to itself, or perhaps because mv is so ancient that it does not + # support -f. + { + # Now remove or move aside any old file at destination location. + # We try this two ways since rm can't unlink itself on some + # systems and the destination file might be busy for other + # reasons. In this case, the final cleanup might fail but the new + # file should still install successfully. + { + test ! -f "$dst" || + $doit $rmcmd -f "$dst" 2>/dev/null || + { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && + { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } + } || + { echo "$0: cannot unlink or rename $dst" >&2 + (exit 1); exit 1 + } + } && + + # Now rename the file to the real destination. + $doit $mvcmd "$dsttmp" "$dst" + } + fi || exit 1 + + trap '' 0 + fi +done + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/ceph/ltmain.sh b/ceph/ltmain.sh new file mode 100644 index 00000000..c2852d85 --- /dev/null +++ b/ceph/ltmain.sh @@ -0,0 +1,9661 @@ + +# libtool (GNU libtool) 2.4.2 +# Written by Gordon Matzigkeit , 1996 + +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005, 2006, +# 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc. +# This is free software; see the source for copying conditions. There is NO +# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +# GNU Libtool is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# As a special exception to the GNU General Public License, +# if you distribute this file as part of a program or library that +# is built using GNU Libtool, you may include this file under the +# same distribution terms that you use for the rest of that program. +# +# GNU Libtool is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Libtool; see the file COPYING. If not, a copy +# can be downloaded from http://www.gnu.org/licenses/gpl.html, +# or obtained by writing to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Usage: $progname [OPTION]... [MODE-ARG]... +# +# Provide generalized library-building support services. +# +# --config show all configuration variables +# --debug enable verbose shell tracing +# -n, --dry-run display commands without modifying any files +# --features display basic configuration information and exit +# --mode=MODE use operation mode MODE +# --preserve-dup-deps don't remove duplicate dependency libraries +# --quiet, --silent don't print informational messages +# --no-quiet, --no-silent +# print informational messages (default) +# --no-warn don't display warning messages +# --tag=TAG use configuration variables from tag TAG +# -v, --verbose print more informational messages than default +# --no-verbose don't print the extra informational messages +# --version print version information +# -h, --help, --help-all print short, long, or detailed help message +# +# MODE must be one of the following: +# +# clean remove files from the build directory +# compile compile a source file into a libtool object +# execute automatically set library path, then run a program +# finish complete the installation of libtool libraries +# install install libraries or executables +# link create a library or an executable +# uninstall remove libraries from an installed directory +# +# MODE-ARGS vary depending on the MODE. When passed as first option, +# `--mode=MODE' may be abbreviated as `MODE' or a unique abbreviation of that. +# Try `$progname --help --mode=MODE' for a more detailed description of MODE. +# +# When reporting a bug, please describe a test case to reproduce it and +# include the following information: +# +# host-triplet: $host +# shell: $SHELL +# compiler: $LTCC +# compiler flags: $LTCFLAGS +# linker: $LD (gnu? $with_gnu_ld) +# $progname: (GNU libtool) 2.4.2 Debian-2.4.2-1ubuntu1 +# automake: $automake_version +# autoconf: $autoconf_version +# +# Report bugs to . +# GNU libtool home page: . +# General help using GNU software: . + +PROGRAM=libtool +PACKAGE=libtool +VERSION="2.4.2 Debian-2.4.2-1ubuntu1" +TIMESTAMP="" +package_revision=1.3337 + +# Be Bourne compatible +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac +fi +BIN_SH=xpg4; export BIN_SH # for Tru64 +DUALCASE=1; export DUALCASE # for MKS sh + +# A function that is used when there is no print builtin or printf. +func_fallback_echo () +{ + eval 'cat <<_LTECHO_EOF +$1 +_LTECHO_EOF' +} + +# NLS nuisances: We save the old values to restore during execute mode. +lt_user_locale= +lt_safe_locale= +for lt_var in LANG LANGUAGE LC_ALL LC_CTYPE LC_COLLATE LC_MESSAGES +do + eval "if test \"\${$lt_var+set}\" = set; then + save_$lt_var=\$$lt_var + $lt_var=C + export $lt_var + lt_user_locale=\"$lt_var=\\\$save_\$lt_var; \$lt_user_locale\" + lt_safe_locale=\"$lt_var=C; \$lt_safe_locale\" + fi" +done +LC_ALL=C +LANGUAGE=C +export LANGUAGE LC_ALL + +$lt_unset CDPATH + + +# Work around backward compatibility issue on IRIX 6.5. On IRIX 6.4+, sh +# is ksh but when the shell is invoked as "sh" and the current value of +# the _XPG environment variable is not equal to 1 (one), the special +# positional parameter $0, within a function call, is the name of the +# function. +progpath="$0" + + + +: ${CP="cp -f"} +test "${ECHO+set}" = set || ECHO=${as_echo-'printf %s\n'} +: ${MAKE="make"} +: ${MKDIR="mkdir"} +: ${MV="mv -f"} +: ${RM="rm -f"} +: ${SHELL="${CONFIG_SHELL-/bin/sh}"} +: ${Xsed="$SED -e 1s/^X//"} + +# Global variables: +EXIT_SUCCESS=0 +EXIT_FAILURE=1 +EXIT_MISMATCH=63 # $? = 63 is used to indicate version mismatch to missing. +EXIT_SKIP=77 # $? = 77 is used to indicate a skipped test to automake. + +exit_status=$EXIT_SUCCESS + +# Make sure IFS has a sensible default +lt_nl=' +' +IFS=" $lt_nl" + +dirname="s,/[^/]*$,," +basename="s,^.*/,," + +# func_dirname file append nondir_replacement +# Compute the dirname of FILE. If nonempty, add APPEND to the result, +# otherwise set result to NONDIR_REPLACEMENT. +func_dirname () +{ + func_dirname_result=`$ECHO "${1}" | $SED "$dirname"` + if test "X$func_dirname_result" = "X${1}"; then + func_dirname_result="${3}" + else + func_dirname_result="$func_dirname_result${2}" + fi +} # func_dirname may be replaced by extended shell implementation + + +# func_basename file +func_basename () +{ + func_basename_result=`$ECHO "${1}" | $SED "$basename"` +} # func_basename may be replaced by extended shell implementation + + +# func_dirname_and_basename file append nondir_replacement +# perform func_basename and func_dirname in a single function +# call: +# dirname: Compute the dirname of FILE. If nonempty, +# add APPEND to the result, otherwise set result +# to NONDIR_REPLACEMENT. +# value returned in "$func_dirname_result" +# basename: Compute filename of FILE. +# value retuned in "$func_basename_result" +# Implementation must be kept synchronized with func_dirname +# and func_basename. For efficiency, we do not delegate to +# those functions but instead duplicate the functionality here. +func_dirname_and_basename () +{ + # Extract subdirectory from the argument. + func_dirname_result=`$ECHO "${1}" | $SED -e "$dirname"` + if test "X$func_dirname_result" = "X${1}"; then + func_dirname_result="${3}" + else + func_dirname_result="$func_dirname_result${2}" + fi + func_basename_result=`$ECHO "${1}" | $SED -e "$basename"` +} # func_dirname_and_basename may be replaced by extended shell implementation + + +# func_stripname prefix suffix name +# strip PREFIX and SUFFIX off of NAME. +# PREFIX and SUFFIX must not contain globbing or regex special +# characters, hashes, percent signs, but SUFFIX may contain a leading +# dot (in which case that matches only a dot). +# func_strip_suffix prefix name +func_stripname () +{ + case ${2} in + .*) func_stripname_result=`$ECHO "${3}" | $SED "s%^${1}%%; s%\\\\${2}\$%%"`;; + *) func_stripname_result=`$ECHO "${3}" | $SED "s%^${1}%%; s%${2}\$%%"`;; + esac +} # func_stripname may be replaced by extended shell implementation + + +# These SED scripts presuppose an absolute path with a trailing slash. +pathcar='s,^/\([^/]*\).*$,\1,' +pathcdr='s,^/[^/]*,,' +removedotparts=':dotsl + s@/\./@/@g + t dotsl + s,/\.$,/,' +collapseslashes='s@/\{1,\}@/@g' +finalslash='s,/*$,/,' + +# func_normal_abspath PATH +# Remove doubled-up and trailing slashes, "." path components, +# and cancel out any ".." path components in PATH after making +# it an absolute path. +# value returned in "$func_normal_abspath_result" +func_normal_abspath () +{ + # Start from root dir and reassemble the path. + func_normal_abspath_result= + func_normal_abspath_tpath=$1 + func_normal_abspath_altnamespace= + case $func_normal_abspath_tpath in + "") + # Empty path, that just means $cwd. + func_stripname '' '/' "`pwd`" + func_normal_abspath_result=$func_stripname_result + return + ;; + # The next three entries are used to spot a run of precisely + # two leading slashes without using negated character classes; + # we take advantage of case's first-match behaviour. + ///*) + # Unusual form of absolute path, do nothing. + ;; + //*) + # Not necessarily an ordinary path; POSIX reserves leading '//' + # and for example Cygwin uses it to access remote file shares + # over CIFS/SMB, so we conserve a leading double slash if found. + func_normal_abspath_altnamespace=/ + ;; + /*) + # Absolute path, do nothing. + ;; + *) + # Relative path, prepend $cwd. + func_normal_abspath_tpath=`pwd`/$func_normal_abspath_tpath + ;; + esac + # Cancel out all the simple stuff to save iterations. We also want + # the path to end with a slash for ease of parsing, so make sure + # there is one (and only one) here. + func_normal_abspath_tpath=`$ECHO "$func_normal_abspath_tpath" | $SED \ + -e "$removedotparts" -e "$collapseslashes" -e "$finalslash"` + while :; do + # Processed it all yet? + if test "$func_normal_abspath_tpath" = / ; then + # If we ascended to the root using ".." the result may be empty now. + if test -z "$func_normal_abspath_result" ; then + func_normal_abspath_result=/ + fi + break + fi + func_normal_abspath_tcomponent=`$ECHO "$func_normal_abspath_tpath" | $SED \ + -e "$pathcar"` + func_normal_abspath_tpath=`$ECHO "$func_normal_abspath_tpath" | $SED \ + -e "$pathcdr"` + # Figure out what to do with it + case $func_normal_abspath_tcomponent in + "") + # Trailing empty path component, ignore it. + ;; + ..) + # Parent dir; strip last assembled component from result. + func_dirname "$func_normal_abspath_result" + func_normal_abspath_result=$func_dirname_result + ;; + *) + # Actual path component, append it. + func_normal_abspath_result=$func_normal_abspath_result/$func_normal_abspath_tcomponent + ;; + esac + done + # Restore leading double-slash if one was found on entry. + func_normal_abspath_result=$func_normal_abspath_altnamespace$func_normal_abspath_result +} + +# func_relative_path SRCDIR DSTDIR +# generates a relative path from SRCDIR to DSTDIR, with a trailing +# slash if non-empty, suitable for immediately appending a filename +# without needing to append a separator. +# value returned in "$func_relative_path_result" +func_relative_path () +{ + func_relative_path_result= + func_normal_abspath "$1" + func_relative_path_tlibdir=$func_normal_abspath_result + func_normal_abspath "$2" + func_relative_path_tbindir=$func_normal_abspath_result + + # Ascend the tree starting from libdir + while :; do + # check if we have found a prefix of bindir + case $func_relative_path_tbindir in + $func_relative_path_tlibdir) + # found an exact match + func_relative_path_tcancelled= + break + ;; + $func_relative_path_tlibdir*) + # found a matching prefix + func_stripname "$func_relative_path_tlibdir" '' "$func_relative_path_tbindir" + func_relative_path_tcancelled=$func_stripname_result + if test -z "$func_relative_path_result"; then + func_relative_path_result=. + fi + break + ;; + *) + func_dirname $func_relative_path_tlibdir + func_relative_path_tlibdir=${func_dirname_result} + if test "x$func_relative_path_tlibdir" = x ; then + # Have to descend all the way to the root! + func_relative_path_result=../$func_relative_path_result + func_relative_path_tcancelled=$func_relative_path_tbindir + break + fi + func_relative_path_result=../$func_relative_path_result + ;; + esac + done + + # Now calculate path; take care to avoid doubling-up slashes. + func_stripname '' '/' "$func_relative_path_result" + func_relative_path_result=$func_stripname_result + func_stripname '/' '/' "$func_relative_path_tcancelled" + if test "x$func_stripname_result" != x ; then + func_relative_path_result=${func_relative_path_result}/${func_stripname_result} + fi + + # Normalisation. If bindir is libdir, return empty string, + # else relative path ending with a slash; either way, target + # file name can be directly appended. + if test ! -z "$func_relative_path_result"; then + func_stripname './' '' "$func_relative_path_result/" + func_relative_path_result=$func_stripname_result + fi +} + +# The name of this program: +func_dirname_and_basename "$progpath" +progname=$func_basename_result + +# Make sure we have an absolute path for reexecution: +case $progpath in + [\\/]*|[A-Za-z]:\\*) ;; + *[\\/]*) + progdir=$func_dirname_result + progdir=`cd "$progdir" && pwd` + progpath="$progdir/$progname" + ;; + *) + save_IFS="$IFS" + IFS=${PATH_SEPARATOR-:} + for progdir in $PATH; do + IFS="$save_IFS" + test -x "$progdir/$progname" && break + done + IFS="$save_IFS" + test -n "$progdir" || progdir=`pwd` + progpath="$progdir/$progname" + ;; +esac + +# Sed substitution that helps us do robust quoting. It backslashifies +# metacharacters that are still active within double-quoted strings. +Xsed="${SED}"' -e 1s/^X//' +sed_quote_subst='s/\([`"$\\]\)/\\\1/g' + +# Same as above, but do not quote variable references. +double_quote_subst='s/\(["`\\]\)/\\\1/g' + +# Sed substitution that turns a string into a regex matching for the +# string literally. +sed_make_literal_regex='s,[].[^$\\*\/],\\&,g' + +# Sed substitution that converts a w32 file name or path +# which contains forward slashes, into one that contains +# (escaped) backslashes. A very naive implementation. +lt_sed_naive_backslashify='s|\\\\*|\\|g;s|/|\\|g;s|\\|\\\\|g' + +# Re-`\' parameter expansions in output of double_quote_subst that were +# `\'-ed in input to the same. If an odd number of `\' preceded a '$' +# in input to double_quote_subst, that '$' was protected from expansion. +# Since each input `\' is now two `\'s, look for any number of runs of +# four `\'s followed by two `\'s and then a '$'. `\' that '$'. +bs='\\' +bs2='\\\\' +bs4='\\\\\\\\' +dollar='\$' +sed_double_backslash="\ + s/$bs4/&\\ +/g + s/^$bs2$dollar/$bs&/ + s/\\([^$bs]\\)$bs2$dollar/\\1$bs2$bs$dollar/g + s/\n//g" + +# Standard options: +opt_dry_run=false +opt_help=false +opt_quiet=false +opt_verbose=false +opt_warning=: + +# func_echo arg... +# Echo program name prefixed message, along with the current mode +# name if it has been set yet. +func_echo () +{ + $ECHO "$progname: ${opt_mode+$opt_mode: }$*" +} + +# func_verbose arg... +# Echo program name prefixed message in verbose mode only. +func_verbose () +{ + $opt_verbose && func_echo ${1+"$@"} + + # A bug in bash halts the script if the last line of a function + # fails when set -e is in force, so we need another command to + # work around that: + : +} + +# func_echo_all arg... +# Invoke $ECHO with all args, space-separated. +func_echo_all () +{ + $ECHO "$*" +} + +# func_error arg... +# Echo program name prefixed message to standard error. +func_error () +{ + $ECHO "$progname: ${opt_mode+$opt_mode: }"${1+"$@"} 1>&2 +} + +# func_warning arg... +# Echo program name prefixed warning message to standard error. +func_warning () +{ + $opt_warning && $ECHO "$progname: ${opt_mode+$opt_mode: }warning: "${1+"$@"} 1>&2 + + # bash bug again: + : +} + +# func_fatal_error arg... +# Echo program name prefixed message to standard error, and exit. +func_fatal_error () +{ + func_error ${1+"$@"} + exit $EXIT_FAILURE +} + +# func_fatal_help arg... +# Echo program name prefixed message to standard error, followed by +# a help hint, and exit. +func_fatal_help () +{ + func_error ${1+"$@"} + func_fatal_error "$help" +} +help="Try \`$progname --help' for more information." ## default + + +# func_grep expression filename +# Check whether EXPRESSION matches any line of FILENAME, without output. +func_grep () +{ + $GREP "$1" "$2" >/dev/null 2>&1 +} + + +# func_mkdir_p directory-path +# Make sure the entire path to DIRECTORY-PATH is available. +func_mkdir_p () +{ + my_directory_path="$1" + my_dir_list= + + if test -n "$my_directory_path" && test "$opt_dry_run" != ":"; then + + # Protect directory names starting with `-' + case $my_directory_path in + -*) my_directory_path="./$my_directory_path" ;; + esac + + # While some portion of DIR does not yet exist... + while test ! -d "$my_directory_path"; do + # ...make a list in topmost first order. Use a colon delimited + # list incase some portion of path contains whitespace. + my_dir_list="$my_directory_path:$my_dir_list" + + # If the last portion added has no slash in it, the list is done + case $my_directory_path in */*) ;; *) break ;; esac + + # ...otherwise throw away the child directory and loop + my_directory_path=`$ECHO "$my_directory_path" | $SED -e "$dirname"` + done + my_dir_list=`$ECHO "$my_dir_list" | $SED 's,:*$,,'` + + save_mkdir_p_IFS="$IFS"; IFS=':' + for my_dir in $my_dir_list; do + IFS="$save_mkdir_p_IFS" + # mkdir can fail with a `File exist' error if two processes + # try to create one of the directories concurrently. Don't + # stop in that case! + $MKDIR "$my_dir" 2>/dev/null || : + done + IFS="$save_mkdir_p_IFS" + + # Bail out if we (or some other process) failed to create a directory. + test -d "$my_directory_path" || \ + func_fatal_error "Failed to create \`$1'" + fi +} + + +# func_mktempdir [string] +# Make a temporary directory that won't clash with other running +# libtool processes, and avoids race conditions if possible. If +# given, STRING is the basename for that directory. +func_mktempdir () +{ + my_template="${TMPDIR-/tmp}/${1-$progname}" + + if test "$opt_dry_run" = ":"; then + # Return a directory name, but don't create it in dry-run mode + my_tmpdir="${my_template}-$$" + else + + # If mktemp works, use that first and foremost + my_tmpdir=`mktemp -d "${my_template}-XXXXXXXX" 2>/dev/null` + + if test ! -d "$my_tmpdir"; then + # Failing that, at least try and use $RANDOM to avoid a race + my_tmpdir="${my_template}-${RANDOM-0}$$" + + save_mktempdir_umask=`umask` + umask 0077 + $MKDIR "$my_tmpdir" + umask $save_mktempdir_umask + fi + + # If we're not in dry-run mode, bomb out on failure + test -d "$my_tmpdir" || \ + func_fatal_error "cannot create temporary directory \`$my_tmpdir'" + fi + + $ECHO "$my_tmpdir" +} + + +# func_quote_for_eval arg +# Aesthetically quote ARG to be evaled later. +# This function returns two values: FUNC_QUOTE_FOR_EVAL_RESULT +# is double-quoted, suitable for a subsequent eval, whereas +# FUNC_QUOTE_FOR_EVAL_UNQUOTED_RESULT has merely all characters +# which are still active within double quotes backslashified. +func_quote_for_eval () +{ + case $1 in + *[\\\`\"\$]*) + func_quote_for_eval_unquoted_result=`$ECHO "$1" | $SED "$sed_quote_subst"` ;; + *) + func_quote_for_eval_unquoted_result="$1" ;; + esac + + case $func_quote_for_eval_unquoted_result in + # Double-quote args containing shell metacharacters to delay + # word splitting, command substitution and and variable + # expansion for a subsequent eval. + # Many Bourne shells cannot handle close brackets correctly + # in scan sets, so we specify it separately. + *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"") + func_quote_for_eval_result="\"$func_quote_for_eval_unquoted_result\"" + ;; + *) + func_quote_for_eval_result="$func_quote_for_eval_unquoted_result" + esac +} + + +# func_quote_for_expand arg +# Aesthetically quote ARG to be evaled later; same as above, +# but do not quote variable references. +func_quote_for_expand () +{ + case $1 in + *[\\\`\"]*) + my_arg=`$ECHO "$1" | $SED \ + -e "$double_quote_subst" -e "$sed_double_backslash"` ;; + *) + my_arg="$1" ;; + esac + + case $my_arg in + # Double-quote args containing shell metacharacters to delay + # word splitting and command substitution for a subsequent eval. + # Many Bourne shells cannot handle close brackets correctly + # in scan sets, so we specify it separately. + *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"") + my_arg="\"$my_arg\"" + ;; + esac + + func_quote_for_expand_result="$my_arg" +} + + +# func_show_eval cmd [fail_exp] +# Unless opt_silent is true, then output CMD. Then, if opt_dryrun is +# not true, evaluate CMD. If the evaluation of CMD fails, and FAIL_EXP +# is given, then evaluate it. +func_show_eval () +{ + my_cmd="$1" + my_fail_exp="${2-:}" + + ${opt_silent-false} || { + func_quote_for_expand "$my_cmd" + eval "func_echo $func_quote_for_expand_result" + } + + if ${opt_dry_run-false}; then :; else + eval "$my_cmd" + my_status=$? + if test "$my_status" -eq 0; then :; else + eval "(exit $my_status); $my_fail_exp" + fi + fi +} + + +# func_show_eval_locale cmd [fail_exp] +# Unless opt_silent is true, then output CMD. Then, if opt_dryrun is +# not true, evaluate CMD. If the evaluation of CMD fails, and FAIL_EXP +# is given, then evaluate it. Use the saved locale for evaluation. +func_show_eval_locale () +{ + my_cmd="$1" + my_fail_exp="${2-:}" + + ${opt_silent-false} || { + func_quote_for_expand "$my_cmd" + eval "func_echo $func_quote_for_expand_result" + } + + if ${opt_dry_run-false}; then :; else + eval "$lt_user_locale + $my_cmd" + my_status=$? + eval "$lt_safe_locale" + if test "$my_status" -eq 0; then :; else + eval "(exit $my_status); $my_fail_exp" + fi + fi +} + +# func_tr_sh +# Turn $1 into a string suitable for a shell variable name. +# Result is stored in $func_tr_sh_result. All characters +# not in the set a-zA-Z0-9_ are replaced with '_'. Further, +# if $1 begins with a digit, a '_' is prepended as well. +func_tr_sh () +{ + case $1 in + [0-9]* | *[!a-zA-Z0-9_]*) + func_tr_sh_result=`$ECHO "$1" | $SED 's/^\([0-9]\)/_\1/; s/[^a-zA-Z0-9_]/_/g'` + ;; + * ) + func_tr_sh_result=$1 + ;; + esac +} + + +# func_version +# Echo version message to standard output and exit. +func_version () +{ + $opt_debug + + $SED -n '/(C)/!b go + :more + /\./!{ + N + s/\n# / / + b more + } + :go + /^# '$PROGRAM' (GNU /,/# warranty; / { + s/^# // + s/^# *$// + s/\((C)\)[ 0-9,-]*\( [1-9][0-9]*\)/\1\2/ + p + }' < "$progpath" + exit $? +} + +# func_usage +# Echo short help message to standard output and exit. +func_usage () +{ + $opt_debug + + $SED -n '/^# Usage:/,/^# *.*--help/ { + s/^# // + s/^# *$// + s/\$progname/'$progname'/ + p + }' < "$progpath" + echo + $ECHO "run \`$progname --help | more' for full usage" + exit $? +} + +# func_help [NOEXIT] +# Echo long help message to standard output and exit, +# unless 'noexit' is passed as argument. +func_help () +{ + $opt_debug + + $SED -n '/^# Usage:/,/# Report bugs to/ { + :print + s/^# // + s/^# *$// + s*\$progname*'$progname'* + s*\$host*'"$host"'* + s*\$SHELL*'"$SHELL"'* + s*\$LTCC*'"$LTCC"'* + s*\$LTCFLAGS*'"$LTCFLAGS"'* + s*\$LD*'"$LD"'* + s/\$with_gnu_ld/'"$with_gnu_ld"'/ + s/\$automake_version/'"`(${AUTOMAKE-automake} --version) 2>/dev/null |$SED 1q`"'/ + s/\$autoconf_version/'"`(${AUTOCONF-autoconf} --version) 2>/dev/null |$SED 1q`"'/ + p + d + } + /^# .* home page:/b print + /^# General help using/b print + ' < "$progpath" + ret=$? + if test -z "$1"; then + exit $ret + fi +} + +# func_missing_arg argname +# Echo program name prefixed message to standard error and set global +# exit_cmd. +func_missing_arg () +{ + $opt_debug + + func_error "missing argument for $1." + exit_cmd=exit +} + + +# func_split_short_opt shortopt +# Set func_split_short_opt_name and func_split_short_opt_arg shell +# variables after splitting SHORTOPT after the 2nd character. +func_split_short_opt () +{ + my_sed_short_opt='1s/^\(..\).*$/\1/;q' + my_sed_short_rest='1s/^..\(.*\)$/\1/;q' + + func_split_short_opt_name=`$ECHO "$1" | $SED "$my_sed_short_opt"` + func_split_short_opt_arg=`$ECHO "$1" | $SED "$my_sed_short_rest"` +} # func_split_short_opt may be replaced by extended shell implementation + + +# func_split_long_opt longopt +# Set func_split_long_opt_name and func_split_long_opt_arg shell +# variables after splitting LONGOPT at the `=' sign. +func_split_long_opt () +{ + my_sed_long_opt='1s/^\(--[^=]*\)=.*/\1/;q' + my_sed_long_arg='1s/^--[^=]*=//' + + func_split_long_opt_name=`$ECHO "$1" | $SED "$my_sed_long_opt"` + func_split_long_opt_arg=`$ECHO "$1" | $SED "$my_sed_long_arg"` +} # func_split_long_opt may be replaced by extended shell implementation + +exit_cmd=: + + + + + +magic="%%%MAGIC variable%%%" +magic_exe="%%%MAGIC EXE variable%%%" + +# Global variables. +nonopt= +preserve_args= +lo2o="s/\\.lo\$/.${objext}/" +o2lo="s/\\.${objext}\$/.lo/" +extracted_archives= +extracted_serial=0 + +# If this variable is set in any of the actions, the command in it +# will be execed at the end. This prevents here-documents from being +# left over by shells. +exec_cmd= + +# func_append var value +# Append VALUE to the end of shell variable VAR. +func_append () +{ + eval "${1}=\$${1}\${2}" +} # func_append may be replaced by extended shell implementation + +# func_append_quoted var value +# Quote VALUE and append to the end of shell variable VAR, separated +# by a space. +func_append_quoted () +{ + func_quote_for_eval "${2}" + eval "${1}=\$${1}\\ \$func_quote_for_eval_result" +} # func_append_quoted may be replaced by extended shell implementation + + +# func_arith arithmetic-term... +func_arith () +{ + func_arith_result=`expr "${@}"` +} # func_arith may be replaced by extended shell implementation + + +# func_len string +# STRING may not start with a hyphen. +func_len () +{ + func_len_result=`expr "${1}" : ".*" 2>/dev/null || echo $max_cmd_len` +} # func_len may be replaced by extended shell implementation + + +# func_lo2o object +func_lo2o () +{ + func_lo2o_result=`$ECHO "${1}" | $SED "$lo2o"` +} # func_lo2o may be replaced by extended shell implementation + + +# func_xform libobj-or-source +func_xform () +{ + func_xform_result=`$ECHO "${1}" | $SED 's/\.[^.]*$/.lo/'` +} # func_xform may be replaced by extended shell implementation + + +# func_fatal_configuration arg... +# Echo program name prefixed message to standard error, followed by +# a configuration failure hint, and exit. +func_fatal_configuration () +{ + func_error ${1+"$@"} + func_error "See the $PACKAGE documentation for more information." + func_fatal_error "Fatal configuration error." +} + + +# func_config +# Display the configuration for all the tags in this script. +func_config () +{ + re_begincf='^# ### BEGIN LIBTOOL' + re_endcf='^# ### END LIBTOOL' + + # Default configuration. + $SED "1,/$re_begincf CONFIG/d;/$re_endcf CONFIG/,\$d" < "$progpath" + + # Now print the configurations for the tags. + for tagname in $taglist; do + $SED -n "/$re_begincf TAG CONFIG: $tagname\$/,/$re_endcf TAG CONFIG: $tagname\$/p" < "$progpath" + done + + exit $? +} + +# func_features +# Display the features supported by this script. +func_features () +{ + echo "host: $host" + if test "$build_libtool_libs" = yes; then + echo "enable shared libraries" + else + echo "disable shared libraries" + fi + if test "$build_old_libs" = yes; then + echo "enable static libraries" + else + echo "disable static libraries" + fi + + exit $? +} + +# func_enable_tag tagname +# Verify that TAGNAME is valid, and either flag an error and exit, or +# enable the TAGNAME tag. We also add TAGNAME to the global $taglist +# variable here. +func_enable_tag () +{ + # Global variable: + tagname="$1" + + re_begincf="^# ### BEGIN LIBTOOL TAG CONFIG: $tagname\$" + re_endcf="^# ### END LIBTOOL TAG CONFIG: $tagname\$" + sed_extractcf="/$re_begincf/,/$re_endcf/p" + + # Validate tagname. + case $tagname in + *[!-_A-Za-z0-9,/]*) + func_fatal_error "invalid tag name: $tagname" + ;; + esac + + # Don't test for the "default" C tag, as we know it's + # there but not specially marked. + case $tagname in + CC) ;; + *) + if $GREP "$re_begincf" "$progpath" >/dev/null 2>&1; then + taglist="$taglist $tagname" + + # Evaluate the configuration. Be careful to quote the path + # and the sed script, to avoid splitting on whitespace, but + # also don't use non-portable quotes within backquotes within + # quotes we have to do it in 2 steps: + extractedcf=`$SED -n -e "$sed_extractcf" < "$progpath"` + eval "$extractedcf" + else + func_error "ignoring unknown tag $tagname" + fi + ;; + esac +} + +# func_check_version_match +# Ensure that we are using m4 macros, and libtool script from the same +# release of libtool. +func_check_version_match () +{ + if test "$package_revision" != "$macro_revision"; then + if test "$VERSION" != "$macro_version"; then + if test -z "$macro_version"; then + cat >&2 <<_LT_EOF +$progname: Version mismatch error. This is $PACKAGE $VERSION, but the +$progname: definition of this LT_INIT comes from an older release. +$progname: You should recreate aclocal.m4 with macros from $PACKAGE $VERSION +$progname: and run autoconf again. +_LT_EOF + else + cat >&2 <<_LT_EOF +$progname: Version mismatch error. This is $PACKAGE $VERSION, but the +$progname: definition of this LT_INIT comes from $PACKAGE $macro_version. +$progname: You should recreate aclocal.m4 with macros from $PACKAGE $VERSION +$progname: and run autoconf again. +_LT_EOF + fi + else + cat >&2 <<_LT_EOF +$progname: Version mismatch error. This is $PACKAGE $VERSION, revision $package_revision, +$progname: but the definition of this LT_INIT comes from revision $macro_revision. +$progname: You should recreate aclocal.m4 with macros from revision $package_revision +$progname: of $PACKAGE $VERSION and run autoconf again. +_LT_EOF + fi + + exit $EXIT_MISMATCH + fi +} + + +# Shorthand for --mode=foo, only valid as the first argument +case $1 in +clean|clea|cle|cl) + shift; set dummy --mode clean ${1+"$@"}; shift + ;; +compile|compil|compi|comp|com|co|c) + shift; set dummy --mode compile ${1+"$@"}; shift + ;; +execute|execut|execu|exec|exe|ex|e) + shift; set dummy --mode execute ${1+"$@"}; shift + ;; +finish|finis|fini|fin|fi|f) + shift; set dummy --mode finish ${1+"$@"}; shift + ;; +install|instal|insta|inst|ins|in|i) + shift; set dummy --mode install ${1+"$@"}; shift + ;; +link|lin|li|l) + shift; set dummy --mode link ${1+"$@"}; shift + ;; +uninstall|uninstal|uninsta|uninst|unins|unin|uni|un|u) + shift; set dummy --mode uninstall ${1+"$@"}; shift + ;; +esac + + + +# Option defaults: +opt_debug=: +opt_dry_run=false +opt_config=false +opt_preserve_dup_deps=false +opt_features=false +opt_finish=false +opt_help=false +opt_help_all=false +opt_silent=: +opt_warning=: +opt_verbose=: +opt_silent=false +opt_verbose=false + + +# Parse options once, thoroughly. This comes as soon as possible in the +# script to make things like `--version' happen as quickly as we can. +{ + # this just eases exit handling + while test $# -gt 0; do + opt="$1" + shift + case $opt in + --debug|-x) opt_debug='set -x' + func_echo "enabling shell trace mode" + $opt_debug + ;; + --dry-run|--dryrun|-n) + opt_dry_run=: + ;; + --config) + opt_config=: +func_config + ;; + --dlopen|-dlopen) + optarg="$1" + opt_dlopen="${opt_dlopen+$opt_dlopen +}$optarg" + shift + ;; + --preserve-dup-deps) + opt_preserve_dup_deps=: + ;; + --features) + opt_features=: +func_features + ;; + --finish) + opt_finish=: +set dummy --mode finish ${1+"$@"}; shift + ;; + --help) + opt_help=: + ;; + --help-all) + opt_help_all=: +opt_help=': help-all' + ;; + --mode) + test $# = 0 && func_missing_arg $opt && break + optarg="$1" + opt_mode="$optarg" +case $optarg in + # Valid mode arguments: + clean|compile|execute|finish|install|link|relink|uninstall) ;; + + # Catch anything else as an error + *) func_error "invalid argument for $opt" + exit_cmd=exit + break + ;; +esac + shift + ;; + --no-silent|--no-quiet) + opt_silent=false +func_append preserve_args " $opt" + ;; + --no-warning|--no-warn) + opt_warning=false +func_append preserve_args " $opt" + ;; + --no-verbose) + opt_verbose=false +func_append preserve_args " $opt" + ;; + --silent|--quiet) + opt_silent=: +func_append preserve_args " $opt" + opt_verbose=false + ;; + --verbose|-v) + opt_verbose=: +func_append preserve_args " $opt" +opt_silent=false + ;; + --tag) + test $# = 0 && func_missing_arg $opt && break + optarg="$1" + opt_tag="$optarg" +func_append preserve_args " $opt $optarg" +func_enable_tag "$optarg" + shift + ;; + + -\?|-h) func_usage ;; + --help) func_help ;; + --version) func_version ;; + + # Separate optargs to long options: + --*=*) + func_split_long_opt "$opt" + set dummy "$func_split_long_opt_name" "$func_split_long_opt_arg" ${1+"$@"} + shift + ;; + + # Separate non-argument short options: + -\?*|-h*|-n*|-v*) + func_split_short_opt "$opt" + set dummy "$func_split_short_opt_name" "-$func_split_short_opt_arg" ${1+"$@"} + shift + ;; + + --) break ;; + -*) func_fatal_help "unrecognized option \`$opt'" ;; + *) set dummy "$opt" ${1+"$@"}; shift; break ;; + esac + done + + # Validate options: + + # save first non-option argument + if test "$#" -gt 0; then + nonopt="$opt" + shift + fi + + # preserve --debug + test "$opt_debug" = : || func_append preserve_args " --debug" + + case $host in + *cygwin* | *mingw* | *pw32* | *cegcc*) + # don't eliminate duplications in $postdeps and $predeps + opt_duplicate_compiler_generated_deps=: + ;; + *) + opt_duplicate_compiler_generated_deps=$opt_preserve_dup_deps + ;; + esac + + $opt_help || { + # Sanity checks first: + func_check_version_match + + if test "$build_libtool_libs" != yes && test "$build_old_libs" != yes; then + func_fatal_configuration "not configured to build any kind of library" + fi + + # Darwin sucks + eval std_shrext=\"$shrext_cmds\" + + # Only execute mode is allowed to have -dlopen flags. + if test -n "$opt_dlopen" && test "$opt_mode" != execute; then + func_error "unrecognized option \`-dlopen'" + $ECHO "$help" 1>&2 + exit $EXIT_FAILURE + fi + + # Change the help message to a mode-specific one. + generic_help="$help" + help="Try \`$progname --help --mode=$opt_mode' for more information." + } + + + # Bail if the options were screwed + $exit_cmd $EXIT_FAILURE +} + + + + +## ----------- ## +## Main. ## +## ----------- ## + +# func_lalib_p file +# True iff FILE is a libtool `.la' library or `.lo' object file. +# This function is only a basic sanity check; it will hardly flush out +# determined imposters. +func_lalib_p () +{ + test -f "$1" && + $SED -e 4q "$1" 2>/dev/null \ + | $GREP "^# Generated by .*$PACKAGE" > /dev/null 2>&1 +} + +# func_lalib_unsafe_p file +# True iff FILE is a libtool `.la' library or `.lo' object file. +# This function implements the same check as func_lalib_p without +# resorting to external programs. To this end, it redirects stdin and +# closes it afterwards, without saving the original file descriptor. +# As a safety measure, use it only where a negative result would be +# fatal anyway. Works if `file' does not exist. +func_lalib_unsafe_p () +{ + lalib_p=no + if test -f "$1" && test -r "$1" && exec 5<&0 <"$1"; then + for lalib_p_l in 1 2 3 4 + do + read lalib_p_line + case "$lalib_p_line" in + \#\ Generated\ by\ *$PACKAGE* ) lalib_p=yes; break;; + esac + done + exec 0<&5 5<&- + fi + test "$lalib_p" = yes +} + +# func_ltwrapper_script_p file +# True iff FILE is a libtool wrapper script +# This function is only a basic sanity check; it will hardly flush out +# determined imposters. +func_ltwrapper_script_p () +{ + func_lalib_p "$1" +} + +# func_ltwrapper_executable_p file +# True iff FILE is a libtool wrapper executable +# This function is only a basic sanity check; it will hardly flush out +# determined imposters. +func_ltwrapper_executable_p () +{ + func_ltwrapper_exec_suffix= + case $1 in + *.exe) ;; + *) func_ltwrapper_exec_suffix=.exe ;; + esac + $GREP "$magic_exe" "$1$func_ltwrapper_exec_suffix" >/dev/null 2>&1 +} + +# func_ltwrapper_scriptname file +# Assumes file is an ltwrapper_executable +# uses $file to determine the appropriate filename for a +# temporary ltwrapper_script. +func_ltwrapper_scriptname () +{ + func_dirname_and_basename "$1" "" "." + func_stripname '' '.exe' "$func_basename_result" + func_ltwrapper_scriptname_result="$func_dirname_result/$objdir/${func_stripname_result}_ltshwrapper" +} + +# func_ltwrapper_p file +# True iff FILE is a libtool wrapper script or wrapper executable +# This function is only a basic sanity check; it will hardly flush out +# determined imposters. +func_ltwrapper_p () +{ + func_ltwrapper_script_p "$1" || func_ltwrapper_executable_p "$1" +} + + +# func_execute_cmds commands fail_cmd +# Execute tilde-delimited COMMANDS. +# If FAIL_CMD is given, eval that upon failure. +# FAIL_CMD may read-access the current command in variable CMD! +func_execute_cmds () +{ + $opt_debug + save_ifs=$IFS; IFS='~' + for cmd in $1; do + IFS=$save_ifs + eval cmd=\"$cmd\" + func_show_eval "$cmd" "${2-:}" + done + IFS=$save_ifs +} + + +# func_source file +# Source FILE, adding directory component if necessary. +# Note that it is not necessary on cygwin/mingw to append a dot to +# FILE even if both FILE and FILE.exe exist: automatic-append-.exe +# behavior happens only for exec(3), not for open(2)! Also, sourcing +# `FILE.' does not work on cygwin managed mounts. +func_source () +{ + $opt_debug + case $1 in + */* | *\\*) . "$1" ;; + *) . "./$1" ;; + esac +} + + +# func_resolve_sysroot PATH +# Replace a leading = in PATH with a sysroot. Store the result into +# func_resolve_sysroot_result +func_resolve_sysroot () +{ + func_resolve_sysroot_result=$1 + case $func_resolve_sysroot_result in + =*) + func_stripname '=' '' "$func_resolve_sysroot_result" + func_resolve_sysroot_result=$lt_sysroot$func_stripname_result + ;; + esac +} + +# func_replace_sysroot PATH +# If PATH begins with the sysroot, replace it with = and +# store the result into func_replace_sysroot_result. +func_replace_sysroot () +{ + case "$lt_sysroot:$1" in + ?*:"$lt_sysroot"*) + func_stripname "$lt_sysroot" '' "$1" + func_replace_sysroot_result="=$func_stripname_result" + ;; + *) + # Including no sysroot. + func_replace_sysroot_result=$1 + ;; + esac +} + +# func_infer_tag arg +# Infer tagged configuration to use if any are available and +# if one wasn't chosen via the "--tag" command line option. +# Only attempt this if the compiler in the base compile +# command doesn't match the default compiler. +# arg is usually of the form 'gcc ...' +func_infer_tag () +{ + $opt_debug + if test -n "$available_tags" && test -z "$tagname"; then + CC_quoted= + for arg in $CC; do + func_append_quoted CC_quoted "$arg" + done + CC_expanded=`func_echo_all $CC` + CC_quoted_expanded=`func_echo_all $CC_quoted` + case $@ in + # Blanks in the command may have been stripped by the calling shell, + # but not from the CC environment variable when configure was run. + " $CC "* | "$CC "* | " $CC_expanded "* | "$CC_expanded "* | \ + " $CC_quoted"* | "$CC_quoted "* | " $CC_quoted_expanded "* | "$CC_quoted_expanded "*) ;; + # Blanks at the start of $base_compile will cause this to fail + # if we don't check for them as well. + *) + for z in $available_tags; do + if $GREP "^# ### BEGIN LIBTOOL TAG CONFIG: $z$" < "$progpath" > /dev/null; then + # Evaluate the configuration. + eval "`${SED} -n -e '/^# ### BEGIN LIBTOOL TAG CONFIG: '$z'$/,/^# ### END LIBTOOL TAG CONFIG: '$z'$/p' < $progpath`" + CC_quoted= + for arg in $CC; do + # Double-quote args containing other shell metacharacters. + func_append_quoted CC_quoted "$arg" + done + CC_expanded=`func_echo_all $CC` + CC_quoted_expanded=`func_echo_all $CC_quoted` + case "$@ " in + " $CC "* | "$CC "* | " $CC_expanded "* | "$CC_expanded "* | \ + " $CC_quoted"* | "$CC_quoted "* | " $CC_quoted_expanded "* | "$CC_quoted_expanded "*) + # The compiler in the base compile command matches + # the one in the tagged configuration. + # Assume this is the tagged configuration we want. + tagname=$z + break + ;; + esac + fi + done + # If $tagname still isn't set, then no tagged configuration + # was found and let the user know that the "--tag" command + # line option must be used. + if test -z "$tagname"; then + func_echo "unable to infer tagged configuration" + func_fatal_error "specify a tag with \`--tag'" +# else +# func_verbose "using $tagname tagged configuration" + fi + ;; + esac + fi +} + + + +# func_write_libtool_object output_name pic_name nonpic_name +# Create a libtool object file (analogous to a ".la" file), +# but don't create it if we're doing a dry run. +func_write_libtool_object () +{ + write_libobj=${1} + if test "$build_libtool_libs" = yes; then + write_lobj=\'${2}\' + else + write_lobj=none + fi + + if test "$build_old_libs" = yes; then + write_oldobj=\'${3}\' + else + write_oldobj=none + fi + + $opt_dry_run || { + cat >${write_libobj}T </dev/null` + if test "$?" -eq 0 && test -n "${func_convert_core_file_wine_to_w32_tmp}"; then + func_convert_core_file_wine_to_w32_result=`$ECHO "$func_convert_core_file_wine_to_w32_tmp" | + $SED -e "$lt_sed_naive_backslashify"` + else + func_convert_core_file_wine_to_w32_result= + fi + fi +} +# end: func_convert_core_file_wine_to_w32 + + +# func_convert_core_path_wine_to_w32 ARG +# Helper function used by path conversion functions when $build is *nix, and +# $host is mingw, cygwin, or some other w32 environment. Relies on a correctly +# configured wine environment available, with the winepath program in $build's +# $PATH. Assumes ARG has no leading or trailing path separator characters. +# +# ARG is path to be converted from $build format to win32. +# Result is available in $func_convert_core_path_wine_to_w32_result. +# Unconvertible file (directory) names in ARG are skipped; if no directory names +# are convertible, then the result may be empty. +func_convert_core_path_wine_to_w32 () +{ + $opt_debug + # unfortunately, winepath doesn't convert paths, only file names + func_convert_core_path_wine_to_w32_result="" + if test -n "$1"; then + oldIFS=$IFS + IFS=: + for func_convert_core_path_wine_to_w32_f in $1; do + IFS=$oldIFS + func_convert_core_file_wine_to_w32 "$func_convert_core_path_wine_to_w32_f" + if test -n "$func_convert_core_file_wine_to_w32_result" ; then + if test -z "$func_convert_core_path_wine_to_w32_result"; then + func_convert_core_path_wine_to_w32_result="$func_convert_core_file_wine_to_w32_result" + else + func_append func_convert_core_path_wine_to_w32_result ";$func_convert_core_file_wine_to_w32_result" + fi + fi + done + IFS=$oldIFS + fi +} +# end: func_convert_core_path_wine_to_w32 + + +# func_cygpath ARGS... +# Wrapper around calling the cygpath program via LT_CYGPATH. This is used when +# when (1) $build is *nix and Cygwin is hosted via a wine environment; or (2) +# $build is MSYS and $host is Cygwin, or (3) $build is Cygwin. In case (1) or +# (2), returns the Cygwin file name or path in func_cygpath_result (input +# file name or path is assumed to be in w32 format, as previously converted +# from $build's *nix or MSYS format). In case (3), returns the w32 file name +# or path in func_cygpath_result (input file name or path is assumed to be in +# Cygwin format). Returns an empty string on error. +# +# ARGS are passed to cygpath, with the last one being the file name or path to +# be converted. +# +# Specify the absolute *nix (or w32) name to cygpath in the LT_CYGPATH +# environment variable; do not put it in $PATH. +func_cygpath () +{ + $opt_debug + if test -n "$LT_CYGPATH" && test -f "$LT_CYGPATH"; then + func_cygpath_result=`$LT_CYGPATH "$@" 2>/dev/null` + if test "$?" -ne 0; then + # on failure, ensure result is empty + func_cygpath_result= + fi + else + func_cygpath_result= + func_error "LT_CYGPATH is empty or specifies non-existent file: \`$LT_CYGPATH'" + fi +} +#end: func_cygpath + + +# func_convert_core_msys_to_w32 ARG +# Convert file name or path ARG from MSYS format to w32 format. Return +# result in func_convert_core_msys_to_w32_result. +func_convert_core_msys_to_w32 () +{ + $opt_debug + # awkward: cmd appends spaces to result + func_convert_core_msys_to_w32_result=`( cmd //c echo "$1" ) 2>/dev/null | + $SED -e 's/[ ]*$//' -e "$lt_sed_naive_backslashify"` +} +#end: func_convert_core_msys_to_w32 + + +# func_convert_file_check ARG1 ARG2 +# Verify that ARG1 (a file name in $build format) was converted to $host +# format in ARG2. Otherwise, emit an error message, but continue (resetting +# func_to_host_file_result to ARG1). +func_convert_file_check () +{ + $opt_debug + if test -z "$2" && test -n "$1" ; then + func_error "Could not determine host file name corresponding to" + func_error " \`$1'" + func_error "Continuing, but uninstalled executables may not work." + # Fallback: + func_to_host_file_result="$1" + fi +} +# end func_convert_file_check + + +# func_convert_path_check FROM_PATHSEP TO_PATHSEP FROM_PATH TO_PATH +# Verify that FROM_PATH (a path in $build format) was converted to $host +# format in TO_PATH. Otherwise, emit an error message, but continue, resetting +# func_to_host_file_result to a simplistic fallback value (see below). +func_convert_path_check () +{ + $opt_debug + if test -z "$4" && test -n "$3"; then + func_error "Could not determine the host path corresponding to" + func_error " \`$3'" + func_error "Continuing, but uninstalled executables may not work." + # Fallback. This is a deliberately simplistic "conversion" and + # should not be "improved". See libtool.info. + if test "x$1" != "x$2"; then + lt_replace_pathsep_chars="s|$1|$2|g" + func_to_host_path_result=`echo "$3" | + $SED -e "$lt_replace_pathsep_chars"` + else + func_to_host_path_result="$3" + fi + fi +} +# end func_convert_path_check + + +# func_convert_path_front_back_pathsep FRONTPAT BACKPAT REPL ORIG +# Modifies func_to_host_path_result by prepending REPL if ORIG matches FRONTPAT +# and appending REPL if ORIG matches BACKPAT. +func_convert_path_front_back_pathsep () +{ + $opt_debug + case $4 in + $1 ) func_to_host_path_result="$3$func_to_host_path_result" + ;; + esac + case $4 in + $2 ) func_append func_to_host_path_result "$3" + ;; + esac +} +# end func_convert_path_front_back_pathsep + + +################################################## +# $build to $host FILE NAME CONVERSION FUNCTIONS # +################################################## +# invoked via `$to_host_file_cmd ARG' +# +# In each case, ARG is the path to be converted from $build to $host format. +# Result will be available in $func_to_host_file_result. + + +# func_to_host_file ARG +# Converts the file name ARG from $build format to $host format. Return result +# in func_to_host_file_result. +func_to_host_file () +{ + $opt_debug + $to_host_file_cmd "$1" +} +# end func_to_host_file + + +# func_to_tool_file ARG LAZY +# converts the file name ARG from $build format to toolchain format. Return +# result in func_to_tool_file_result. If the conversion in use is listed +# in (the comma separated) LAZY, no conversion takes place. +func_to_tool_file () +{ + $opt_debug + case ,$2, in + *,"$to_tool_file_cmd",*) + func_to_tool_file_result=$1 + ;; + *) + $to_tool_file_cmd "$1" + func_to_tool_file_result=$func_to_host_file_result + ;; + esac +} +# end func_to_tool_file + + +# func_convert_file_noop ARG +# Copy ARG to func_to_host_file_result. +func_convert_file_noop () +{ + func_to_host_file_result="$1" +} +# end func_convert_file_noop + + +# func_convert_file_msys_to_w32 ARG +# Convert file name ARG from (mingw) MSYS to (mingw) w32 format; automatic +# conversion to w32 is not available inside the cwrapper. Returns result in +# func_to_host_file_result. +func_convert_file_msys_to_w32 () +{ + $opt_debug + func_to_host_file_result="$1" + if test -n "$1"; then + func_convert_core_msys_to_w32 "$1" + func_to_host_file_result="$func_convert_core_msys_to_w32_result" + fi + func_convert_file_check "$1" "$func_to_host_file_result" +} +# end func_convert_file_msys_to_w32 + + +# func_convert_file_cygwin_to_w32 ARG +# Convert file name ARG from Cygwin to w32 format. Returns result in +# func_to_host_file_result. +func_convert_file_cygwin_to_w32 () +{ + $opt_debug + func_to_host_file_result="$1" + if test -n "$1"; then + # because $build is cygwin, we call "the" cygpath in $PATH; no need to use + # LT_CYGPATH in this case. + func_to_host_file_result=`cygpath -m "$1"` + fi + func_convert_file_check "$1" "$func_to_host_file_result" +} +# end func_convert_file_cygwin_to_w32 + + +# func_convert_file_nix_to_w32 ARG +# Convert file name ARG from *nix to w32 format. Requires a wine environment +# and a working winepath. Returns result in func_to_host_file_result. +func_convert_file_nix_to_w32 () +{ + $opt_debug + func_to_host_file_result="$1" + if test -n "$1"; then + func_convert_core_file_wine_to_w32 "$1" + func_to_host_file_result="$func_convert_core_file_wine_to_w32_result" + fi + func_convert_file_check "$1" "$func_to_host_file_result" +} +# end func_convert_file_nix_to_w32 + + +# func_convert_file_msys_to_cygwin ARG +# Convert file name ARG from MSYS to Cygwin format. Requires LT_CYGPATH set. +# Returns result in func_to_host_file_result. +func_convert_file_msys_to_cygwin () +{ + $opt_debug + func_to_host_file_result="$1" + if test -n "$1"; then + func_convert_core_msys_to_w32 "$1" + func_cygpath -u "$func_convert_core_msys_to_w32_result" + func_to_host_file_result="$func_cygpath_result" + fi + func_convert_file_check "$1" "$func_to_host_file_result" +} +# end func_convert_file_msys_to_cygwin + + +# func_convert_file_nix_to_cygwin ARG +# Convert file name ARG from *nix to Cygwin format. Requires Cygwin installed +# in a wine environment, working winepath, and LT_CYGPATH set. Returns result +# in func_to_host_file_result. +func_convert_file_nix_to_cygwin () +{ + $opt_debug + func_to_host_file_result="$1" + if test -n "$1"; then + # convert from *nix to w32, then use cygpath to convert from w32 to cygwin. + func_convert_core_file_wine_to_w32 "$1" + func_cygpath -u "$func_convert_core_file_wine_to_w32_result" + func_to_host_file_result="$func_cygpath_result" + fi + func_convert_file_check "$1" "$func_to_host_file_result" +} +# end func_convert_file_nix_to_cygwin + + +############################################# +# $build to $host PATH CONVERSION FUNCTIONS # +############################################# +# invoked via `$to_host_path_cmd ARG' +# +# In each case, ARG is the path to be converted from $build to $host format. +# The result will be available in $func_to_host_path_result. +# +# Path separators are also converted from $build format to $host format. If +# ARG begins or ends with a path separator character, it is preserved (but +# converted to $host format) on output. +# +# All path conversion functions are named using the following convention: +# file name conversion function : func_convert_file_X_to_Y () +# path conversion function : func_convert_path_X_to_Y () +# where, for any given $build/$host combination the 'X_to_Y' value is the +# same. If conversion functions are added for new $build/$host combinations, +# the two new functions must follow this pattern, or func_init_to_host_path_cmd +# will break. + + +# func_init_to_host_path_cmd +# Ensures that function "pointer" variable $to_host_path_cmd is set to the +# appropriate value, based on the value of $to_host_file_cmd. +to_host_path_cmd= +func_init_to_host_path_cmd () +{ + $opt_debug + if test -z "$to_host_path_cmd"; then + func_stripname 'func_convert_file_' '' "$to_host_file_cmd" + to_host_path_cmd="func_convert_path_${func_stripname_result}" + fi +} + + +# func_to_host_path ARG +# Converts the path ARG from $build format to $host format. Return result +# in func_to_host_path_result. +func_to_host_path () +{ + $opt_debug + func_init_to_host_path_cmd + $to_host_path_cmd "$1" +} +# end func_to_host_path + + +# func_convert_path_noop ARG +# Copy ARG to func_to_host_path_result. +func_convert_path_noop () +{ + func_to_host_path_result="$1" +} +# end func_convert_path_noop + + +# func_convert_path_msys_to_w32 ARG +# Convert path ARG from (mingw) MSYS to (mingw) w32 format; automatic +# conversion to w32 is not available inside the cwrapper. Returns result in +# func_to_host_path_result. +func_convert_path_msys_to_w32 () +{ + $opt_debug + func_to_host_path_result="$1" + if test -n "$1"; then + # Remove leading and trailing path separator characters from ARG. MSYS + # behavior is inconsistent here; cygpath turns them into '.;' and ';.'; + # and winepath ignores them completely. + func_stripname : : "$1" + func_to_host_path_tmp1=$func_stripname_result + func_convert_core_msys_to_w32 "$func_to_host_path_tmp1" + func_to_host_path_result="$func_convert_core_msys_to_w32_result" + func_convert_path_check : ";" \ + "$func_to_host_path_tmp1" "$func_to_host_path_result" + func_convert_path_front_back_pathsep ":*" "*:" ";" "$1" + fi +} +# end func_convert_path_msys_to_w32 + + +# func_convert_path_cygwin_to_w32 ARG +# Convert path ARG from Cygwin to w32 format. Returns result in +# func_to_host_file_result. +func_convert_path_cygwin_to_w32 () +{ + $opt_debug + func_to_host_path_result="$1" + if test -n "$1"; then + # See func_convert_path_msys_to_w32: + func_stripname : : "$1" + func_to_host_path_tmp1=$func_stripname_result + func_to_host_path_result=`cygpath -m -p "$func_to_host_path_tmp1"` + func_convert_path_check : ";" \ + "$func_to_host_path_tmp1" "$func_to_host_path_result" + func_convert_path_front_back_pathsep ":*" "*:" ";" "$1" + fi +} +# end func_convert_path_cygwin_to_w32 + + +# func_convert_path_nix_to_w32 ARG +# Convert path ARG from *nix to w32 format. Requires a wine environment and +# a working winepath. Returns result in func_to_host_file_result. +func_convert_path_nix_to_w32 () +{ + $opt_debug + func_to_host_path_result="$1" + if test -n "$1"; then + # See func_convert_path_msys_to_w32: + func_stripname : : "$1" + func_to_host_path_tmp1=$func_stripname_result + func_convert_core_path_wine_to_w32 "$func_to_host_path_tmp1" + func_to_host_path_result="$func_convert_core_path_wine_to_w32_result" + func_convert_path_check : ";" \ + "$func_to_host_path_tmp1" "$func_to_host_path_result" + func_convert_path_front_back_pathsep ":*" "*:" ";" "$1" + fi +} +# end func_convert_path_nix_to_w32 + + +# func_convert_path_msys_to_cygwin ARG +# Convert path ARG from MSYS to Cygwin format. Requires LT_CYGPATH set. +# Returns result in func_to_host_file_result. +func_convert_path_msys_to_cygwin () +{ + $opt_debug + func_to_host_path_result="$1" + if test -n "$1"; then + # See func_convert_path_msys_to_w32: + func_stripname : : "$1" + func_to_host_path_tmp1=$func_stripname_result + func_convert_core_msys_to_w32 "$func_to_host_path_tmp1" + func_cygpath -u -p "$func_convert_core_msys_to_w32_result" + func_to_host_path_result="$func_cygpath_result" + func_convert_path_check : : \ + "$func_to_host_path_tmp1" "$func_to_host_path_result" + func_convert_path_front_back_pathsep ":*" "*:" : "$1" + fi +} +# end func_convert_path_msys_to_cygwin + + +# func_convert_path_nix_to_cygwin ARG +# Convert path ARG from *nix to Cygwin format. Requires Cygwin installed in a +# a wine environment, working winepath, and LT_CYGPATH set. Returns result in +# func_to_host_file_result. +func_convert_path_nix_to_cygwin () +{ + $opt_debug + func_to_host_path_result="$1" + if test -n "$1"; then + # Remove leading and trailing path separator characters from + # ARG. msys behavior is inconsistent here, cygpath turns them + # into '.;' and ';.', and winepath ignores them completely. + func_stripname : : "$1" + func_to_host_path_tmp1=$func_stripname_result + func_convert_core_path_wine_to_w32 "$func_to_host_path_tmp1" + func_cygpath -u -p "$func_convert_core_path_wine_to_w32_result" + func_to_host_path_result="$func_cygpath_result" + func_convert_path_check : : \ + "$func_to_host_path_tmp1" "$func_to_host_path_result" + func_convert_path_front_back_pathsep ":*" "*:" : "$1" + fi +} +# end func_convert_path_nix_to_cygwin + + +# func_mode_compile arg... +func_mode_compile () +{ + $opt_debug + # Get the compilation command and the source file. + base_compile= + srcfile="$nonopt" # always keep a non-empty value in "srcfile" + suppress_opt=yes + suppress_output= + arg_mode=normal + libobj= + later= + pie_flag= + + for arg + do + case $arg_mode in + arg ) + # do not "continue". Instead, add this to base_compile + lastarg="$arg" + arg_mode=normal + ;; + + target ) + libobj="$arg" + arg_mode=normal + continue + ;; + + normal ) + # Accept any command-line options. + case $arg in + -o) + test -n "$libobj" && \ + func_fatal_error "you cannot specify \`-o' more than once" + arg_mode=target + continue + ;; + + -pie | -fpie | -fPIE) + func_append pie_flag " $arg" + continue + ;; + + -shared | -static | -prefer-pic | -prefer-non-pic) + func_append later " $arg" + continue + ;; + + -no-suppress) + suppress_opt=no + continue + ;; + + -Xcompiler) + arg_mode=arg # the next one goes into the "base_compile" arg list + continue # The current "srcfile" will either be retained or + ;; # replaced later. I would guess that would be a bug. + + -Wc,*) + func_stripname '-Wc,' '' "$arg" + args=$func_stripname_result + lastarg= + save_ifs="$IFS"; IFS=',' + for arg in $args; do + IFS="$save_ifs" + func_append_quoted lastarg "$arg" + done + IFS="$save_ifs" + func_stripname ' ' '' "$lastarg" + lastarg=$func_stripname_result + + # Add the arguments to base_compile. + func_append base_compile " $lastarg" + continue + ;; + + *) + # Accept the current argument as the source file. + # The previous "srcfile" becomes the current argument. + # + lastarg="$srcfile" + srcfile="$arg" + ;; + esac # case $arg + ;; + esac # case $arg_mode + + # Aesthetically quote the previous argument. + func_append_quoted base_compile "$lastarg" + done # for arg + + case $arg_mode in + arg) + func_fatal_error "you must specify an argument for -Xcompile" + ;; + target) + func_fatal_error "you must specify a target with \`-o'" + ;; + *) + # Get the name of the library object. + test -z "$libobj" && { + func_basename "$srcfile" + libobj="$func_basename_result" + } + ;; + esac + + # Recognize several different file suffixes. + # If the user specifies -o file.o, it is replaced with file.lo + case $libobj in + *.[cCFSifmso] | \ + *.ada | *.adb | *.ads | *.asm | \ + *.c++ | *.cc | *.ii | *.class | *.cpp | *.cxx | \ + *.[fF][09]? | *.for | *.java | *.go | *.obj | *.sx | *.cu | *.cup) + func_xform "$libobj" + libobj=$func_xform_result + ;; + esac + + case $libobj in + *.lo) func_lo2o "$libobj"; obj=$func_lo2o_result ;; + *) + func_fatal_error "cannot determine name of library object from \`$libobj'" + ;; + esac + + func_infer_tag $base_compile + + for arg in $later; do + case $arg in + -shared) + test "$build_libtool_libs" != yes && \ + func_fatal_configuration "can not build a shared library" + build_old_libs=no + continue + ;; + + -static) + build_libtool_libs=no + build_old_libs=yes + continue + ;; + + -prefer-pic) + pic_mode=yes + continue + ;; + + -prefer-non-pic) + pic_mode=no + continue + ;; + esac + done + + func_quote_for_eval "$libobj" + test "X$libobj" != "X$func_quote_for_eval_result" \ + && $ECHO "X$libobj" | $GREP '[]~#^*{};<>?"'"'"' &()|`$[]' \ + && func_warning "libobj name \`$libobj' may not contain shell special characters." + func_dirname_and_basename "$obj" "/" "" + objname="$func_basename_result" + xdir="$func_dirname_result" + lobj=${xdir}$objdir/$objname + + test -z "$base_compile" && \ + func_fatal_help "you must specify a compilation command" + + # Delete any leftover library objects. + if test "$build_old_libs" = yes; then + removelist="$obj $lobj $libobj ${libobj}T" + else + removelist="$lobj $libobj ${libobj}T" + fi + + # On Cygwin there's no "real" PIC flag so we must build both object types + case $host_os in + cygwin* | mingw* | pw32* | os2* | cegcc*) + pic_mode=default + ;; + esac + if test "$pic_mode" = no && test "$deplibs_check_method" != pass_all; then + # non-PIC code in shared libraries is not supported + pic_mode=default + fi + + # Calculate the filename of the output object if compiler does + # not support -o with -c + if test "$compiler_c_o" = no; then + output_obj=`$ECHO "$srcfile" | $SED 's%^.*/%%; s%\.[^.]*$%%'`.${objext} + lockfile="$output_obj.lock" + else + output_obj= + need_locks=no + lockfile= + fi + + # Lock this critical section if it is needed + # We use this script file to make the link, it avoids creating a new file + if test "$need_locks" = yes; then + until $opt_dry_run || ln "$progpath" "$lockfile" 2>/dev/null; do + func_echo "Waiting for $lockfile to be removed" + sleep 2 + done + elif test "$need_locks" = warn; then + if test -f "$lockfile"; then + $ECHO "\ +*** ERROR, $lockfile exists and contains: +`cat $lockfile 2>/dev/null` + +This indicates that another process is trying to use the same +temporary object file, and libtool could not work around it because +your compiler does not support \`-c' and \`-o' together. If you +repeat this compilation, it may succeed, by chance, but you had better +avoid parallel builds (make -j) in this platform, or get a better +compiler." + + $opt_dry_run || $RM $removelist + exit $EXIT_FAILURE + fi + func_append removelist " $output_obj" + $ECHO "$srcfile" > "$lockfile" + fi + + $opt_dry_run || $RM $removelist + func_append removelist " $lockfile" + trap '$opt_dry_run || $RM $removelist; exit $EXIT_FAILURE' 1 2 15 + + func_to_tool_file "$srcfile" func_convert_file_msys_to_w32 + srcfile=$func_to_tool_file_result + func_quote_for_eval "$srcfile" + qsrcfile=$func_quote_for_eval_result + + # Only build a PIC object if we are building libtool libraries. + if test "$build_libtool_libs" = yes; then + # Without this assignment, base_compile gets emptied. + fbsd_hideous_sh_bug=$base_compile + + if test "$pic_mode" != no; then + command="$base_compile $qsrcfile $pic_flag" + else + # Don't build PIC code + command="$base_compile $qsrcfile" + fi + + func_mkdir_p "$xdir$objdir" + + if test -z "$output_obj"; then + # Place PIC objects in $objdir + func_append command " -o $lobj" + fi + + func_show_eval_locale "$command" \ + 'test -n "$output_obj" && $RM $removelist; exit $EXIT_FAILURE' + + if test "$need_locks" = warn && + test "X`cat $lockfile 2>/dev/null`" != "X$srcfile"; then + $ECHO "\ +*** ERROR, $lockfile contains: +`cat $lockfile 2>/dev/null` + +but it should contain: +$srcfile + +This indicates that another process is trying to use the same +temporary object file, and libtool could not work around it because +your compiler does not support \`-c' and \`-o' together. If you +repeat this compilation, it may succeed, by chance, but you had better +avoid parallel builds (make -j) in this platform, or get a better +compiler." + + $opt_dry_run || $RM $removelist + exit $EXIT_FAILURE + fi + + # Just move the object if needed, then go on to compile the next one + if test -n "$output_obj" && test "X$output_obj" != "X$lobj"; then + func_show_eval '$MV "$output_obj" "$lobj"' \ + 'error=$?; $opt_dry_run || $RM $removelist; exit $error' + fi + + # Allow error messages only from the first compilation. + if test "$suppress_opt" = yes; then + suppress_output=' >/dev/null 2>&1' + fi + fi + + # Only build a position-dependent object if we build old libraries. + if test "$build_old_libs" = yes; then + if test "$pic_mode" != yes; then + # Don't build PIC code + command="$base_compile $qsrcfile$pie_flag" + else + command="$base_compile $qsrcfile $pic_flag" + fi + if test "$compiler_c_o" = yes; then + func_append command " -o $obj" + fi + + # Suppress compiler output if we already did a PIC compilation. + func_append command "$suppress_output" + func_show_eval_locale "$command" \ + '$opt_dry_run || $RM $removelist; exit $EXIT_FAILURE' + + if test "$need_locks" = warn && + test "X`cat $lockfile 2>/dev/null`" != "X$srcfile"; then + $ECHO "\ +*** ERROR, $lockfile contains: +`cat $lockfile 2>/dev/null` + +but it should contain: +$srcfile + +This indicates that another process is trying to use the same +temporary object file, and libtool could not work around it because +your compiler does not support \`-c' and \`-o' together. If you +repeat this compilation, it may succeed, by chance, but you had better +avoid parallel builds (make -j) in this platform, or get a better +compiler." + + $opt_dry_run || $RM $removelist + exit $EXIT_FAILURE + fi + + # Just move the object if needed + if test -n "$output_obj" && test "X$output_obj" != "X$obj"; then + func_show_eval '$MV "$output_obj" "$obj"' \ + 'error=$?; $opt_dry_run || $RM $removelist; exit $error' + fi + fi + + $opt_dry_run || { + func_write_libtool_object "$libobj" "$objdir/$objname" "$objname" + + # Unlock the critical section if it was locked + if test "$need_locks" != no; then + removelist=$lockfile + $RM "$lockfile" + fi + } + + exit $EXIT_SUCCESS +} + +$opt_help || { + test "$opt_mode" = compile && func_mode_compile ${1+"$@"} +} + +func_mode_help () +{ + # We need to display help for each of the modes. + case $opt_mode in + "") + # Generic help is extracted from the usage comments + # at the start of this file. + func_help + ;; + + clean) + $ECHO \ +"Usage: $progname [OPTION]... --mode=clean RM [RM-OPTION]... FILE... + +Remove files from the build directory. + +RM is the name of the program to use to delete files associated with each FILE +(typically \`/bin/rm'). RM-OPTIONS are options (such as \`-f') to be passed +to RM. + +If FILE is a libtool library, object or program, all the files associated +with it are deleted. Otherwise, only FILE itself is deleted using RM." + ;; + + compile) + $ECHO \ +"Usage: $progname [OPTION]... --mode=compile COMPILE-COMMAND... SOURCEFILE + +Compile a source file into a libtool library object. + +This mode accepts the following additional options: + + -o OUTPUT-FILE set the output file name to OUTPUT-FILE + -no-suppress do not suppress compiler output for multiple passes + -prefer-pic try to build PIC objects only + -prefer-non-pic try to build non-PIC objects only + -shared do not build a \`.o' file suitable for static linking + -static only build a \`.o' file suitable for static linking + -Wc,FLAG pass FLAG directly to the compiler + +COMPILE-COMMAND is a command to be used in creating a \`standard' object file +from the given SOURCEFILE. + +The output file name is determined by removing the directory component from +SOURCEFILE, then substituting the C source code suffix \`.c' with the +library object suffix, \`.lo'." + ;; + + execute) + $ECHO \ +"Usage: $progname [OPTION]... --mode=execute COMMAND [ARGS]... + +Automatically set library path, then run a program. + +This mode accepts the following additional options: + + -dlopen FILE add the directory containing FILE to the library path + +This mode sets the library path environment variable according to \`-dlopen' +flags. + +If any of the ARGS are libtool executable wrappers, then they are translated +into their corresponding uninstalled binary, and any of their required library +directories are added to the library path. + +Then, COMMAND is executed, with ARGS as arguments." + ;; + + finish) + $ECHO \ +"Usage: $progname [OPTION]... --mode=finish [LIBDIR]... + +Complete the installation of libtool libraries. + +Each LIBDIR is a directory that contains libtool libraries. + +The commands that this mode executes may require superuser privileges. Use +the \`--dry-run' option if you just want to see what would be executed." + ;; + + install) + $ECHO \ +"Usage: $progname [OPTION]... --mode=install INSTALL-COMMAND... + +Install executables or libraries. + +INSTALL-COMMAND is the installation command. The first component should be +either the \`install' or \`cp' program. + +The following components of INSTALL-COMMAND are treated specially: + + -inst-prefix-dir PREFIX-DIR Use PREFIX-DIR as a staging area for installation + +The rest of the components are interpreted as arguments to that command (only +BSD-compatible install options are recognized)." + ;; + + link) + $ECHO \ +"Usage: $progname [OPTION]... --mode=link LINK-COMMAND... + +Link object files or libraries together to form another library, or to +create an executable program. + +LINK-COMMAND is a command using the C compiler that you would use to create +a program from several object files. + +The following components of LINK-COMMAND are treated specially: + + -all-static do not do any dynamic linking at all + -avoid-version do not add a version suffix if possible + -bindir BINDIR specify path to binaries directory (for systems where + libraries must be found in the PATH setting at runtime) + -dlopen FILE \`-dlpreopen' FILE if it cannot be dlopened at runtime + -dlpreopen FILE link in FILE and add its symbols to lt_preloaded_symbols + -export-dynamic allow symbols from OUTPUT-FILE to be resolved with dlsym(3) + -export-symbols SYMFILE + try to export only the symbols listed in SYMFILE + -export-symbols-regex REGEX + try to export only the symbols matching REGEX + -LLIBDIR search LIBDIR for required installed libraries + -lNAME OUTPUT-FILE requires the installed library libNAME + -module build a library that can dlopened + -no-fast-install disable the fast-install mode + -no-install link a not-installable executable + -no-undefined declare that a library does not refer to external symbols + -o OUTPUT-FILE create OUTPUT-FILE from the specified objects + -objectlist FILE Use a list of object files found in FILE to specify objects + -precious-files-regex REGEX + don't remove output files matching REGEX + -release RELEASE specify package release information + -rpath LIBDIR the created library will eventually be installed in LIBDIR + -R[ ]LIBDIR add LIBDIR to the runtime path of programs and libraries + -shared only do dynamic linking of libtool libraries + -shrext SUFFIX override the standard shared library file extension + -static do not do any dynamic linking of uninstalled libtool libraries + -static-libtool-libs + do not do any dynamic linking of libtool libraries + -version-info CURRENT[:REVISION[:AGE]] + specify library version info [each variable defaults to 0] + -weak LIBNAME declare that the target provides the LIBNAME interface + -Wc,FLAG + -Xcompiler FLAG pass linker-specific FLAG directly to the compiler + -Wl,FLAG + -Xlinker FLAG pass linker-specific FLAG directly to the linker + -XCClinker FLAG pass link-specific FLAG to the compiler driver (CC) + +All other options (arguments beginning with \`-') are ignored. + +Every other argument is treated as a filename. Files ending in \`.la' are +treated as uninstalled libtool libraries, other files are standard or library +object files. + +If the OUTPUT-FILE ends in \`.la', then a libtool library is created, +only library objects (\`.lo' files) may be specified, and \`-rpath' is +required, except when creating a convenience library. + +If OUTPUT-FILE ends in \`.a' or \`.lib', then a standard library is created +using \`ar' and \`ranlib', or on Windows using \`lib'. + +If OUTPUT-FILE ends in \`.lo' or \`.${objext}', then a reloadable object file +is created, otherwise an executable program is created." + ;; + + uninstall) + $ECHO \ +"Usage: $progname [OPTION]... --mode=uninstall RM [RM-OPTION]... FILE... + +Remove libraries from an installation directory. + +RM is the name of the program to use to delete files associated with each FILE +(typically \`/bin/rm'). RM-OPTIONS are options (such as \`-f') to be passed +to RM. + +If FILE is a libtool library, all the files associated with it are deleted. +Otherwise, only FILE itself is deleted using RM." + ;; + + *) + func_fatal_help "invalid operation mode \`$opt_mode'" + ;; + esac + + echo + $ECHO "Try \`$progname --help' for more information about other modes." +} + +# Now that we've collected a possible --mode arg, show help if necessary +if $opt_help; then + if test "$opt_help" = :; then + func_mode_help + else + { + func_help noexit + for opt_mode in compile link execute install finish uninstall clean; do + func_mode_help + done + } | sed -n '1p; 2,$s/^Usage:/ or: /p' + { + func_help noexit + for opt_mode in compile link execute install finish uninstall clean; do + echo + func_mode_help + done + } | + sed '1d + /^When reporting/,/^Report/{ + H + d + } + $x + /information about other modes/d + /more detailed .*MODE/d + s/^Usage:.*--mode=\([^ ]*\) .*/Description of \1 mode:/' + fi + exit $? +fi + + +# func_mode_execute arg... +func_mode_execute () +{ + $opt_debug + # The first argument is the command name. + cmd="$nonopt" + test -z "$cmd" && \ + func_fatal_help "you must specify a COMMAND" + + # Handle -dlopen flags immediately. + for file in $opt_dlopen; do + test -f "$file" \ + || func_fatal_help "\`$file' is not a file" + + dir= + case $file in + *.la) + func_resolve_sysroot "$file" + file=$func_resolve_sysroot_result + + # Check to see that this really is a libtool archive. + func_lalib_unsafe_p "$file" \ + || func_fatal_help "\`$lib' is not a valid libtool archive" + + # Read the libtool library. + dlname= + library_names= + func_source "$file" + + # Skip this library if it cannot be dlopened. + if test -z "$dlname"; then + # Warn if it was a shared library. + test -n "$library_names" && \ + func_warning "\`$file' was not linked with \`-export-dynamic'" + continue + fi + + func_dirname "$file" "" "." + dir="$func_dirname_result" + + if test -f "$dir/$objdir/$dlname"; then + func_append dir "/$objdir" + else + if test ! -f "$dir/$dlname"; then + func_fatal_error "cannot find \`$dlname' in \`$dir' or \`$dir/$objdir'" + fi + fi + ;; + + *.lo) + # Just add the directory containing the .lo file. + func_dirname "$file" "" "." + dir="$func_dirname_result" + ;; + + *) + func_warning "\`-dlopen' is ignored for non-libtool libraries and objects" + continue + ;; + esac + + # Get the absolute pathname. + absdir=`cd "$dir" && pwd` + test -n "$absdir" && dir="$absdir" + + # Now add the directory to shlibpath_var. + if eval "test -z \"\$$shlibpath_var\""; then + eval "$shlibpath_var=\"\$dir\"" + else + eval "$shlibpath_var=\"\$dir:\$$shlibpath_var\"" + fi + done + + # This variable tells wrapper scripts just to set shlibpath_var + # rather than running their programs. + libtool_execute_magic="$magic" + + # Check if any of the arguments is a wrapper script. + args= + for file + do + case $file in + -* | *.la | *.lo ) ;; + *) + # Do a test to see if this is really a libtool program. + if func_ltwrapper_script_p "$file"; then + func_source "$file" + # Transform arg to wrapped name. + file="$progdir/$program" + elif func_ltwrapper_executable_p "$file"; then + func_ltwrapper_scriptname "$file" + func_source "$func_ltwrapper_scriptname_result" + # Transform arg to wrapped name. + file="$progdir/$program" + fi + ;; + esac + # Quote arguments (to preserve shell metacharacters). + func_append_quoted args "$file" + done + + if test "X$opt_dry_run" = Xfalse; then + if test -n "$shlibpath_var"; then + # Export the shlibpath_var. + eval "export $shlibpath_var" + fi + + # Restore saved environment variables + for lt_var in LANG LANGUAGE LC_ALL LC_CTYPE LC_COLLATE LC_MESSAGES + do + eval "if test \"\${save_$lt_var+set}\" = set; then + $lt_var=\$save_$lt_var; export $lt_var + else + $lt_unset $lt_var + fi" + done + + # Now prepare to actually exec the command. + exec_cmd="\$cmd$args" + else + # Display what would be done. + if test -n "$shlibpath_var"; then + eval "\$ECHO \"\$shlibpath_var=\$$shlibpath_var\"" + echo "export $shlibpath_var" + fi + $ECHO "$cmd$args" + exit $EXIT_SUCCESS + fi +} + +test "$opt_mode" = execute && func_mode_execute ${1+"$@"} + + +# func_mode_finish arg... +func_mode_finish () +{ + $opt_debug + libs= + libdirs= + admincmds= + + for opt in "$nonopt" ${1+"$@"} + do + if test -d "$opt"; then + func_append libdirs " $opt" + + elif test -f "$opt"; then + if func_lalib_unsafe_p "$opt"; then + func_append libs " $opt" + else + func_warning "\`$opt' is not a valid libtool archive" + fi + + else + func_fatal_error "invalid argument \`$opt'" + fi + done + + if test -n "$libs"; then + if test -n "$lt_sysroot"; then + sysroot_regex=`$ECHO "$lt_sysroot" | $SED "$sed_make_literal_regex"` + sysroot_cmd="s/\([ ']\)$sysroot_regex/\1/g;" + else + sysroot_cmd= + fi + + # Remove sysroot references + if $opt_dry_run; then + for lib in $libs; do + echo "removing references to $lt_sysroot and \`=' prefixes from $lib" + done + else + tmpdir=`func_mktempdir` + for lib in $libs; do + sed -e "${sysroot_cmd} s/\([ ']-[LR]\)=/\1/g; s/\([ ']\)=/\1/g" $lib \ + > $tmpdir/tmp-la + mv -f $tmpdir/tmp-la $lib + done + ${RM}r "$tmpdir" + fi + fi + + if test -n "$finish_cmds$finish_eval" && test -n "$libdirs"; then + for libdir in $libdirs; do + if test -n "$finish_cmds"; then + # Do each command in the finish commands. + func_execute_cmds "$finish_cmds" 'admincmds="$admincmds +'"$cmd"'"' + fi + if test -n "$finish_eval"; then + # Do the single finish_eval. + eval cmds=\"$finish_eval\" + $opt_dry_run || eval "$cmds" || func_append admincmds " + $cmds" + fi + done + fi + + # Exit here if they wanted silent mode. + $opt_silent && exit $EXIT_SUCCESS + + if test -n "$finish_cmds$finish_eval" && test -n "$libdirs"; then + echo "----------------------------------------------------------------------" + echo "Libraries have been installed in:" + for libdir in $libdirs; do + $ECHO " $libdir" + done + echo + echo "If you ever happen to want to link against installed libraries" + echo "in a given directory, LIBDIR, you must either use libtool, and" + echo "specify the full pathname of the library, or use the \`-LLIBDIR'" + echo "flag during linking and do at least one of the following:" + if test -n "$shlibpath_var"; then + echo " - add LIBDIR to the \`$shlibpath_var' environment variable" + echo " during execution" + fi + if test -n "$runpath_var"; then + echo " - add LIBDIR to the \`$runpath_var' environment variable" + echo " during linking" + fi + if test -n "$hardcode_libdir_flag_spec"; then + libdir=LIBDIR + eval flag=\"$hardcode_libdir_flag_spec\" + + $ECHO " - use the \`$flag' linker flag" + fi + if test -n "$admincmds"; then + $ECHO " - have your system administrator run these commands:$admincmds" + fi + if test -f /etc/ld.so.conf; then + echo " - have your system administrator add LIBDIR to \`/etc/ld.so.conf'" + fi + echo + + echo "See any operating system documentation about shared libraries for" + case $host in + solaris2.[6789]|solaris2.1[0-9]) + echo "more information, such as the ld(1), crle(1) and ld.so(8) manual" + echo "pages." + ;; + *) + echo "more information, such as the ld(1) and ld.so(8) manual pages." + ;; + esac + echo "----------------------------------------------------------------------" + fi + exit $EXIT_SUCCESS +} + +test "$opt_mode" = finish && func_mode_finish ${1+"$@"} + + +# func_mode_install arg... +func_mode_install () +{ + $opt_debug + # There may be an optional sh(1) argument at the beginning of + # install_prog (especially on Windows NT). + if test "$nonopt" = "$SHELL" || test "$nonopt" = /bin/sh || + # Allow the use of GNU shtool's install command. + case $nonopt in *shtool*) :;; *) false;; esac; then + # Aesthetically quote it. + func_quote_for_eval "$nonopt" + install_prog="$func_quote_for_eval_result " + arg=$1 + shift + else + install_prog= + arg=$nonopt + fi + + # The real first argument should be the name of the installation program. + # Aesthetically quote it. + func_quote_for_eval "$arg" + func_append install_prog "$func_quote_for_eval_result" + install_shared_prog=$install_prog + case " $install_prog " in + *[\\\ /]cp\ *) install_cp=: ;; + *) install_cp=false ;; + esac + + # We need to accept at least all the BSD install flags. + dest= + files= + opts= + prev= + install_type= + isdir=no + stripme= + no_mode=: + for arg + do + arg2= + if test -n "$dest"; then + func_append files " $dest" + dest=$arg + continue + fi + + case $arg in + -d) isdir=yes ;; + -f) + if $install_cp; then :; else + prev=$arg + fi + ;; + -g | -m | -o) + prev=$arg + ;; + -s) + stripme=" -s" + continue + ;; + -*) + ;; + *) + # If the previous option needed an argument, then skip it. + if test -n "$prev"; then + if test "x$prev" = x-m && test -n "$install_override_mode"; then + arg2=$install_override_mode + no_mode=false + fi + prev= + else + dest=$arg + continue + fi + ;; + esac + + # Aesthetically quote the argument. + func_quote_for_eval "$arg" + func_append install_prog " $func_quote_for_eval_result" + if test -n "$arg2"; then + func_quote_for_eval "$arg2" + fi + func_append install_shared_prog " $func_quote_for_eval_result" + done + + test -z "$install_prog" && \ + func_fatal_help "you must specify an install program" + + test -n "$prev" && \ + func_fatal_help "the \`$prev' option requires an argument" + + if test -n "$install_override_mode" && $no_mode; then + if $install_cp; then :; else + func_quote_for_eval "$install_override_mode" + func_append install_shared_prog " -m $func_quote_for_eval_result" + fi + fi + + if test -z "$files"; then + if test -z "$dest"; then + func_fatal_help "no file or destination specified" + else + func_fatal_help "you must specify a destination" + fi + fi + + # Strip any trailing slash from the destination. + func_stripname '' '/' "$dest" + dest=$func_stripname_result + + # Check to see that the destination is a directory. + test -d "$dest" && isdir=yes + if test "$isdir" = yes; then + destdir="$dest" + destname= + else + func_dirname_and_basename "$dest" "" "." + destdir="$func_dirname_result" + destname="$func_basename_result" + + # Not a directory, so check to see that there is only one file specified. + set dummy $files; shift + test "$#" -gt 1 && \ + func_fatal_help "\`$dest' is not a directory" + fi + case $destdir in + [\\/]* | [A-Za-z]:[\\/]*) ;; + *) + for file in $files; do + case $file in + *.lo) ;; + *) + func_fatal_help "\`$destdir' must be an absolute directory name" + ;; + esac + done + ;; + esac + + # This variable tells wrapper scripts just to set variables rather + # than running their programs. + libtool_install_magic="$magic" + + staticlibs= + future_libdirs= + current_libdirs= + for file in $files; do + + # Do each installation. + case $file in + *.$libext) + # Do the static libraries later. + func_append staticlibs " $file" + ;; + + *.la) + func_resolve_sysroot "$file" + file=$func_resolve_sysroot_result + + # Check to see that this really is a libtool archive. + func_lalib_unsafe_p "$file" \ + || func_fatal_help "\`$file' is not a valid libtool archive" + + library_names= + old_library= + relink_command= + func_source "$file" + + # Add the libdir to current_libdirs if it is the destination. + if test "X$destdir" = "X$libdir"; then + case "$current_libdirs " in + *" $libdir "*) ;; + *) func_append current_libdirs " $libdir" ;; + esac + else + # Note the libdir as a future libdir. + case "$future_libdirs " in + *" $libdir "*) ;; + *) func_append future_libdirs " $libdir" ;; + esac + fi + + func_dirname "$file" "/" "" + dir="$func_dirname_result" + func_append dir "$objdir" + + if test -n "$relink_command"; then + # Determine the prefix the user has applied to our future dir. + inst_prefix_dir=`$ECHO "$destdir" | $SED -e "s%$libdir\$%%"` + + # Don't allow the user to place us outside of our expected + # location b/c this prevents finding dependent libraries that + # are installed to the same prefix. + # At present, this check doesn't affect windows .dll's that + # are installed into $libdir/../bin (currently, that works fine) + # but it's something to keep an eye on. + test "$inst_prefix_dir" = "$destdir" && \ + func_fatal_error "error: cannot install \`$file' to a directory not ending in $libdir" + + if test -n "$inst_prefix_dir"; then + # Stick the inst_prefix_dir data into the link command. + relink_command=`$ECHO "$relink_command" | $SED "s%@inst_prefix_dir@%-inst-prefix-dir $inst_prefix_dir%"` + else + relink_command=`$ECHO "$relink_command" | $SED "s%@inst_prefix_dir@%%"` + fi + + func_warning "relinking \`$file'" + func_show_eval "$relink_command" \ + 'func_fatal_error "error: relink \`$file'\'' with the above command before installing it"' + fi + + # See the names of the shared library. + set dummy $library_names; shift + if test -n "$1"; then + realname="$1" + shift + + srcname="$realname" + test -n "$relink_command" && srcname="$realname"T + + # Install the shared library and build the symlinks. + func_show_eval "$install_shared_prog $dir/$srcname $destdir/$realname" \ + 'exit $?' + tstripme="$stripme" + case $host_os in + cygwin* | mingw* | pw32* | cegcc*) + case $realname in + *.dll.a) + tstripme="" + ;; + esac + ;; + esac + if test -n "$tstripme" && test -n "$striplib"; then + func_show_eval "$striplib $destdir/$realname" 'exit $?' + fi + + if test "$#" -gt 0; then + # Delete the old symlinks, and create new ones. + # Try `ln -sf' first, because the `ln' binary might depend on + # the symlink we replace! Solaris /bin/ln does not understand -f, + # so we also need to try rm && ln -s. + for linkname + do + test "$linkname" != "$realname" \ + && func_show_eval "(cd $destdir && { $LN_S -f $realname $linkname || { $RM $linkname && $LN_S $realname $linkname; }; })" + done + fi + + # Do each command in the postinstall commands. + lib="$destdir/$realname" + func_execute_cmds "$postinstall_cmds" 'exit $?' + fi + + # Install the pseudo-library for information purposes. + func_basename "$file" + name="$func_basename_result" + instname="$dir/$name"i + func_show_eval "$install_prog $instname $destdir/$name" 'exit $?' + + # Maybe install the static library, too. + test -n "$old_library" && func_append staticlibs " $dir/$old_library" + ;; + + *.lo) + # Install (i.e. copy) a libtool object. + + # Figure out destination file name, if it wasn't already specified. + if test -n "$destname"; then + destfile="$destdir/$destname" + else + func_basename "$file" + destfile="$func_basename_result" + destfile="$destdir/$destfile" + fi + + # Deduce the name of the destination old-style object file. + case $destfile in + *.lo) + func_lo2o "$destfile" + staticdest=$func_lo2o_result + ;; + *.$objext) + staticdest="$destfile" + destfile= + ;; + *) + func_fatal_help "cannot copy a libtool object to \`$destfile'" + ;; + esac + + # Install the libtool object if requested. + test -n "$destfile" && \ + func_show_eval "$install_prog $file $destfile" 'exit $?' + + # Install the old object if enabled. + if test "$build_old_libs" = yes; then + # Deduce the name of the old-style object file. + func_lo2o "$file" + staticobj=$func_lo2o_result + func_show_eval "$install_prog \$staticobj \$staticdest" 'exit $?' + fi + exit $EXIT_SUCCESS + ;; + + *) + # Figure out destination file name, if it wasn't already specified. + if test -n "$destname"; then + destfile="$destdir/$destname" + else + func_basename "$file" + destfile="$func_basename_result" + destfile="$destdir/$destfile" + fi + + # If the file is missing, and there is a .exe on the end, strip it + # because it is most likely a libtool script we actually want to + # install + stripped_ext="" + case $file in + *.exe) + if test ! -f "$file"; then + func_stripname '' '.exe' "$file" + file=$func_stripname_result + stripped_ext=".exe" + fi + ;; + esac + + # Do a test to see if this is really a libtool program. + case $host in + *cygwin* | *mingw*) + if func_ltwrapper_executable_p "$file"; then + func_ltwrapper_scriptname "$file" + wrapper=$func_ltwrapper_scriptname_result + else + func_stripname '' '.exe' "$file" + wrapper=$func_stripname_result + fi + ;; + *) + wrapper=$file + ;; + esac + if func_ltwrapper_script_p "$wrapper"; then + notinst_deplibs= + relink_command= + + func_source "$wrapper" + + # Check the variables that should have been set. + test -z "$generated_by_libtool_version" && \ + func_fatal_error "invalid libtool wrapper script \`$wrapper'" + + finalize=yes + for lib in $notinst_deplibs; do + # Check to see that each library is installed. + libdir= + if test -f "$lib"; then + func_source "$lib" + fi + libfile="$libdir/"`$ECHO "$lib" | $SED 's%^.*/%%g'` ### testsuite: skip nested quoting test + if test -n "$libdir" && test ! -f "$libfile"; then + func_warning "\`$lib' has not been installed in \`$libdir'" + finalize=no + fi + done + + relink_command= + func_source "$wrapper" + + outputname= + if test "$fast_install" = no && test -n "$relink_command"; then + $opt_dry_run || { + if test "$finalize" = yes; then + tmpdir=`func_mktempdir` + func_basename "$file$stripped_ext" + file="$func_basename_result" + outputname="$tmpdir/$file" + # Replace the output file specification. + relink_command=`$ECHO "$relink_command" | $SED 's%@OUTPUT@%'"$outputname"'%g'` + + $opt_silent || { + func_quote_for_expand "$relink_command" + eval "func_echo $func_quote_for_expand_result" + } + if eval "$relink_command"; then : + else + func_error "error: relink \`$file' with the above command before installing it" + $opt_dry_run || ${RM}r "$tmpdir" + continue + fi + file="$outputname" + else + func_warning "cannot relink \`$file'" + fi + } + else + # Install the binary that we compiled earlier. + file=`$ECHO "$file$stripped_ext" | $SED "s%\([^/]*\)$%$objdir/\1%"` + fi + fi + + # remove .exe since cygwin /usr/bin/install will append another + # one anyway + case $install_prog,$host in + */usr/bin/install*,*cygwin*) + case $file:$destfile in + *.exe:*.exe) + # this is ok + ;; + *.exe:*) + destfile=$destfile.exe + ;; + *:*.exe) + func_stripname '' '.exe' "$destfile" + destfile=$func_stripname_result + ;; + esac + ;; + esac + func_show_eval "$install_prog\$stripme \$file \$destfile" 'exit $?' + $opt_dry_run || if test -n "$outputname"; then + ${RM}r "$tmpdir" + fi + ;; + esac + done + + for file in $staticlibs; do + func_basename "$file" + name="$func_basename_result" + + # Set up the ranlib parameters. + oldlib="$destdir/$name" + func_to_tool_file "$oldlib" func_convert_file_msys_to_w32 + tool_oldlib=$func_to_tool_file_result + + func_show_eval "$install_prog \$file \$oldlib" 'exit $?' + + if test -n "$stripme" && test -n "$old_striplib"; then + func_show_eval "$old_striplib $tool_oldlib" 'exit $?' + fi + + # Do each command in the postinstall commands. + func_execute_cmds "$old_postinstall_cmds" 'exit $?' + done + + test -n "$future_libdirs" && \ + func_warning "remember to run \`$progname --finish$future_libdirs'" + + if test -n "$current_libdirs"; then + # Maybe just do a dry run. + $opt_dry_run && current_libdirs=" -n$current_libdirs" + exec_cmd='$SHELL $progpath $preserve_args --finish$current_libdirs' + else + exit $EXIT_SUCCESS + fi +} + +test "$opt_mode" = install && func_mode_install ${1+"$@"} + + +# func_generate_dlsyms outputname originator pic_p +# Extract symbols from dlprefiles and create ${outputname}S.o with +# a dlpreopen symbol table. +func_generate_dlsyms () +{ + $opt_debug + my_outputname="$1" + my_originator="$2" + my_pic_p="${3-no}" + my_prefix=`$ECHO "$my_originator" | sed 's%[^a-zA-Z0-9]%_%g'` + my_dlsyms= + + if test -n "$dlfiles$dlprefiles" || test "$dlself" != no; then + if test -n "$NM" && test -n "$global_symbol_pipe"; then + my_dlsyms="${my_outputname}S.c" + else + func_error "not configured to extract global symbols from dlpreopened files" + fi + fi + + if test -n "$my_dlsyms"; then + case $my_dlsyms in + "") ;; + *.c) + # Discover the nlist of each of the dlfiles. + nlist="$output_objdir/${my_outputname}.nm" + + func_show_eval "$RM $nlist ${nlist}S ${nlist}T" + + # Parse the name list into a source file. + func_verbose "creating $output_objdir/$my_dlsyms" + + $opt_dry_run || $ECHO > "$output_objdir/$my_dlsyms" "\ +/* $my_dlsyms - symbol resolution table for \`$my_outputname' dlsym emulation. */ +/* Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION */ + +#ifdef __cplusplus +extern \"C\" { +#endif + +#if defined(__GNUC__) && (((__GNUC__ == 4) && (__GNUC_MINOR__ >= 4)) || (__GNUC__ > 4)) +#pragma GCC diagnostic ignored \"-Wstrict-prototypes\" +#endif + +/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ +#if defined(_WIN32) || defined(__CYGWIN__) || defined(_WIN32_WCE) +/* DATA imports from DLLs on WIN32 con't be const, because runtime + relocations are performed -- see ld's documentation on pseudo-relocs. */ +# define LT_DLSYM_CONST +#elif defined(__osf__) +/* This system does not cope well with relocations in const data. */ +# define LT_DLSYM_CONST +#else +# define LT_DLSYM_CONST const +#endif + +/* External symbol declarations for the compiler. */\ +" + + if test "$dlself" = yes; then + func_verbose "generating symbol list for \`$output'" + + $opt_dry_run || echo ': @PROGRAM@ ' > "$nlist" + + # Add our own program objects to the symbol list. + progfiles=`$ECHO "$objs$old_deplibs" | $SP2NL | $SED "$lo2o" | $NL2SP` + for progfile in $progfiles; do + func_to_tool_file "$progfile" func_convert_file_msys_to_w32 + func_verbose "extracting global C symbols from \`$func_to_tool_file_result'" + $opt_dry_run || eval "$NM $func_to_tool_file_result | $global_symbol_pipe >> '$nlist'" + done + + if test -n "$exclude_expsyms"; then + $opt_dry_run || { + eval '$EGREP -v " ($exclude_expsyms)$" "$nlist" > "$nlist"T' + eval '$MV "$nlist"T "$nlist"' + } + fi + + if test -n "$export_symbols_regex"; then + $opt_dry_run || { + eval '$EGREP -e "$export_symbols_regex" "$nlist" > "$nlist"T' + eval '$MV "$nlist"T "$nlist"' + } + fi + + # Prepare the list of exported symbols + if test -z "$export_symbols"; then + export_symbols="$output_objdir/$outputname.exp" + $opt_dry_run || { + $RM $export_symbols + eval "${SED} -n -e '/^: @PROGRAM@ $/d' -e 's/^.* \(.*\)$/\1/p' "'< "$nlist" > "$export_symbols"' + case $host in + *cygwin* | *mingw* | *cegcc* ) + eval "echo EXPORTS "'> "$output_objdir/$outputname.def"' + eval 'cat "$export_symbols" >> "$output_objdir/$outputname.def"' + ;; + esac + } + else + $opt_dry_run || { + eval "${SED} -e 's/\([].[*^$]\)/\\\\\1/g' -e 's/^/ /' -e 's/$/$/'"' < "$export_symbols" > "$output_objdir/$outputname.exp"' + eval '$GREP -f "$output_objdir/$outputname.exp" < "$nlist" > "$nlist"T' + eval '$MV "$nlist"T "$nlist"' + case $host in + *cygwin* | *mingw* | *cegcc* ) + eval "echo EXPORTS "'> "$output_objdir/$outputname.def"' + eval 'cat "$nlist" >> "$output_objdir/$outputname.def"' + ;; + esac + } + fi + fi + + for dlprefile in $dlprefiles; do + func_verbose "extracting global C symbols from \`$dlprefile'" + func_basename "$dlprefile" + name="$func_basename_result" + case $host in + *cygwin* | *mingw* | *cegcc* ) + # if an import library, we need to obtain dlname + if func_win32_import_lib_p "$dlprefile"; then + func_tr_sh "$dlprefile" + eval "curr_lafile=\$libfile_$func_tr_sh_result" + dlprefile_dlbasename="" + if test -n "$curr_lafile" && func_lalib_p "$curr_lafile"; then + # Use subshell, to avoid clobbering current variable values + dlprefile_dlname=`source "$curr_lafile" && echo "$dlname"` + if test -n "$dlprefile_dlname" ; then + func_basename "$dlprefile_dlname" + dlprefile_dlbasename="$func_basename_result" + else + # no lafile. user explicitly requested -dlpreopen . + $sharedlib_from_linklib_cmd "$dlprefile" + dlprefile_dlbasename=$sharedlib_from_linklib_result + fi + fi + $opt_dry_run || { + if test -n "$dlprefile_dlbasename" ; then + eval '$ECHO ": $dlprefile_dlbasename" >> "$nlist"' + else + func_warning "Could not compute DLL name from $name" + eval '$ECHO ": $name " >> "$nlist"' + fi + func_to_tool_file "$dlprefile" func_convert_file_msys_to_w32 + eval "$NM \"$func_to_tool_file_result\" 2>/dev/null | $global_symbol_pipe | + $SED -e '/I __imp/d' -e 's/I __nm_/D /;s/_nm__//' >> '$nlist'" + } + else # not an import lib + $opt_dry_run || { + eval '$ECHO ": $name " >> "$nlist"' + func_to_tool_file "$dlprefile" func_convert_file_msys_to_w32 + eval "$NM \"$func_to_tool_file_result\" 2>/dev/null | $global_symbol_pipe >> '$nlist'" + } + fi + ;; + *) + $opt_dry_run || { + eval '$ECHO ": $name " >> "$nlist"' + func_to_tool_file "$dlprefile" func_convert_file_msys_to_w32 + eval "$NM \"$func_to_tool_file_result\" 2>/dev/null | $global_symbol_pipe >> '$nlist'" + } + ;; + esac + done + + $opt_dry_run || { + # Make sure we have at least an empty file. + test -f "$nlist" || : > "$nlist" + + if test -n "$exclude_expsyms"; then + $EGREP -v " ($exclude_expsyms)$" "$nlist" > "$nlist"T + $MV "$nlist"T "$nlist" + fi + + # Try sorting and uniquifying the output. + if $GREP -v "^: " < "$nlist" | + if sort -k 3 /dev/null 2>&1; then + sort -k 3 + else + sort +2 + fi | + uniq > "$nlist"S; then + : + else + $GREP -v "^: " < "$nlist" > "$nlist"S + fi + + if test -f "$nlist"S; then + eval "$global_symbol_to_cdecl"' < "$nlist"S >> "$output_objdir/$my_dlsyms"' + else + echo '/* NONE */' >> "$output_objdir/$my_dlsyms" + fi + + echo >> "$output_objdir/$my_dlsyms" "\ + +/* The mapping between symbol names and symbols. */ +typedef struct { + const char *name; + void *address; +} lt_dlsymlist; +extern LT_DLSYM_CONST lt_dlsymlist +lt_${my_prefix}_LTX_preloaded_symbols[]; +LT_DLSYM_CONST lt_dlsymlist +lt_${my_prefix}_LTX_preloaded_symbols[] = +{\ + { \"$my_originator\", (void *) 0 }," + + case $need_lib_prefix in + no) + eval "$global_symbol_to_c_name_address" < "$nlist" >> "$output_objdir/$my_dlsyms" + ;; + *) + eval "$global_symbol_to_c_name_address_lib_prefix" < "$nlist" >> "$output_objdir/$my_dlsyms" + ;; + esac + echo >> "$output_objdir/$my_dlsyms" "\ + {0, (void *) 0} +}; + +/* This works around a problem in FreeBSD linker */ +#ifdef FREEBSD_WORKAROUND +static const void *lt_preloaded_setup() { + return lt_${my_prefix}_LTX_preloaded_symbols; +} +#endif + +#ifdef __cplusplus +} +#endif\ +" + } # !$opt_dry_run + + pic_flag_for_symtable= + case "$compile_command " in + *" -static "*) ;; + *) + case $host in + # compiling the symbol table file with pic_flag works around + # a FreeBSD bug that causes programs to crash when -lm is + # linked before any other PIC object. But we must not use + # pic_flag when linking with -static. The problem exists in + # FreeBSD 2.2.6 and is fixed in FreeBSD 3.1. + *-*-freebsd2.*|*-*-freebsd3.0*|*-*-freebsdelf3.0*) + pic_flag_for_symtable=" $pic_flag -DFREEBSD_WORKAROUND" ;; + *-*-hpux*) + pic_flag_for_symtable=" $pic_flag" ;; + *) + if test "X$my_pic_p" != Xno; then + pic_flag_for_symtable=" $pic_flag" + fi + ;; + esac + ;; + esac + symtab_cflags= + for arg in $LTCFLAGS; do + case $arg in + -pie | -fpie | -fPIE) ;; + *) func_append symtab_cflags " $arg" ;; + esac + done + + # Now compile the dynamic symbol file. + func_show_eval '(cd $output_objdir && $LTCC$symtab_cflags -c$no_builtin_flag$pic_flag_for_symtable "$my_dlsyms")' 'exit $?' + + # Clean up the generated files. + func_show_eval '$RM "$output_objdir/$my_dlsyms" "$nlist" "${nlist}S" "${nlist}T"' + + # Transform the symbol file into the correct name. + symfileobj="$output_objdir/${my_outputname}S.$objext" + case $host in + *cygwin* | *mingw* | *cegcc* ) + if test -f "$output_objdir/$my_outputname.def"; then + compile_command=`$ECHO "$compile_command" | $SED "s%@SYMFILE@%$output_objdir/$my_outputname.def $symfileobj%"` + finalize_command=`$ECHO "$finalize_command" | $SED "s%@SYMFILE@%$output_objdir/$my_outputname.def $symfileobj%"` + else + compile_command=`$ECHO "$compile_command" | $SED "s%@SYMFILE@%$symfileobj%"` + finalize_command=`$ECHO "$finalize_command" | $SED "s%@SYMFILE@%$symfileobj%"` + fi + ;; + *) + compile_command=`$ECHO "$compile_command" | $SED "s%@SYMFILE@%$symfileobj%"` + finalize_command=`$ECHO "$finalize_command" | $SED "s%@SYMFILE@%$symfileobj%"` + ;; + esac + ;; + *) + func_fatal_error "unknown suffix for \`$my_dlsyms'" + ;; + esac + else + # We keep going just in case the user didn't refer to + # lt_preloaded_symbols. The linker will fail if global_symbol_pipe + # really was required. + + # Nullify the symbol file. + compile_command=`$ECHO "$compile_command" | $SED "s% @SYMFILE@%%"` + finalize_command=`$ECHO "$finalize_command" | $SED "s% @SYMFILE@%%"` + fi +} + +# func_win32_libid arg +# return the library type of file 'arg' +# +# Need a lot of goo to handle *both* DLLs and import libs +# Has to be a shell function in order to 'eat' the argument +# that is supplied when $file_magic_command is called. +# Despite the name, also deal with 64 bit binaries. +func_win32_libid () +{ + $opt_debug + win32_libid_type="unknown" + win32_fileres=`file -L $1 2>/dev/null` + case $win32_fileres in + *ar\ archive\ import\ library*) # definitely import + win32_libid_type="x86 archive import" + ;; + *ar\ archive*) # could be an import, or static + # Keep the egrep pattern in sync with the one in _LT_CHECK_MAGIC_METHOD. + if eval $OBJDUMP -f $1 | $SED -e '10q' 2>/dev/null | + $EGREP 'file format (pei*-i386(.*architecture: i386)?|pe-arm-wince|pe-x86-64)' >/dev/null; then + func_to_tool_file "$1" func_convert_file_msys_to_w32 + win32_nmres=`eval $NM -f posix -A \"$func_to_tool_file_result\" | + $SED -n -e ' + 1,100{ + / I /{ + s,.*,import, + p + q + } + }'` + case $win32_nmres in + import*) win32_libid_type="x86 archive import";; + *) win32_libid_type="x86 archive static";; + esac + fi + ;; + *DLL*) + win32_libid_type="x86 DLL" + ;; + *executable*) # but shell scripts are "executable" too... + case $win32_fileres in + *MS\ Windows\ PE\ Intel*) + win32_libid_type="x86 DLL" + ;; + esac + ;; + esac + $ECHO "$win32_libid_type" +} + +# func_cygming_dll_for_implib ARG +# +# Platform-specific function to extract the +# name of the DLL associated with the specified +# import library ARG. +# Invoked by eval'ing the libtool variable +# $sharedlib_from_linklib_cmd +# Result is available in the variable +# $sharedlib_from_linklib_result +func_cygming_dll_for_implib () +{ + $opt_debug + sharedlib_from_linklib_result=`$DLLTOOL --identify-strict --identify "$1"` +} + +# func_cygming_dll_for_implib_fallback_core SECTION_NAME LIBNAMEs +# +# The is the core of a fallback implementation of a +# platform-specific function to extract the name of the +# DLL associated with the specified import library LIBNAME. +# +# SECTION_NAME is either .idata$6 or .idata$7, depending +# on the platform and compiler that created the implib. +# +# Echos the name of the DLL associated with the +# specified import library. +func_cygming_dll_for_implib_fallback_core () +{ + $opt_debug + match_literal=`$ECHO "$1" | $SED "$sed_make_literal_regex"` + $OBJDUMP -s --section "$1" "$2" 2>/dev/null | + $SED '/^Contents of section '"$match_literal"':/{ + # Place marker at beginning of archive member dllname section + s/.*/====MARK====/ + p + d + } + # These lines can sometimes be longer than 43 characters, but + # are always uninteresting + /:[ ]*file format pe[i]\{,1\}-/d + /^In archive [^:]*:/d + # Ensure marker is printed + /^====MARK====/p + # Remove all lines with less than 43 characters + /^.\{43\}/!d + # From remaining lines, remove first 43 characters + s/^.\{43\}//' | + $SED -n ' + # Join marker and all lines until next marker into a single line + /^====MARK====/ b para + H + $ b para + b + :para + x + s/\n//g + # Remove the marker + s/^====MARK====// + # Remove trailing dots and whitespace + s/[\. \t]*$// + # Print + /./p' | + # we now have a list, one entry per line, of the stringified + # contents of the appropriate section of all members of the + # archive which possess that section. Heuristic: eliminate + # all those which have a first or second character that is + # a '.' (that is, objdump's representation of an unprintable + # character.) This should work for all archives with less than + # 0x302f exports -- but will fail for DLLs whose name actually + # begins with a literal '.' or a single character followed by + # a '.'. + # + # Of those that remain, print the first one. + $SED -e '/^\./d;/^.\./d;q' +} + +# func_cygming_gnu_implib_p ARG +# This predicate returns with zero status (TRUE) if +# ARG is a GNU/binutils-style import library. Returns +# with nonzero status (FALSE) otherwise. +func_cygming_gnu_implib_p () +{ + $opt_debug + func_to_tool_file "$1" func_convert_file_msys_to_w32 + func_cygming_gnu_implib_tmp=`$NM "$func_to_tool_file_result" | eval "$global_symbol_pipe" | $EGREP ' (_head_[A-Za-z0-9_]+_[ad]l*|[A-Za-z0-9_]+_[ad]l*_iname)$'` + test -n "$func_cygming_gnu_implib_tmp" +} + +# func_cygming_ms_implib_p ARG +# This predicate returns with zero status (TRUE) if +# ARG is an MS-style import library. Returns +# with nonzero status (FALSE) otherwise. +func_cygming_ms_implib_p () +{ + $opt_debug + func_to_tool_file "$1" func_convert_file_msys_to_w32 + func_cygming_ms_implib_tmp=`$NM "$func_to_tool_file_result" | eval "$global_symbol_pipe" | $GREP '_NULL_IMPORT_DESCRIPTOR'` + test -n "$func_cygming_ms_implib_tmp" +} + +# func_cygming_dll_for_implib_fallback ARG +# Platform-specific function to extract the +# name of the DLL associated with the specified +# import library ARG. +# +# This fallback implementation is for use when $DLLTOOL +# does not support the --identify-strict option. +# Invoked by eval'ing the libtool variable +# $sharedlib_from_linklib_cmd +# Result is available in the variable +# $sharedlib_from_linklib_result +func_cygming_dll_for_implib_fallback () +{ + $opt_debug + if func_cygming_gnu_implib_p "$1" ; then + # binutils import library + sharedlib_from_linklib_result=`func_cygming_dll_for_implib_fallback_core '.idata$7' "$1"` + elif func_cygming_ms_implib_p "$1" ; then + # ms-generated import library + sharedlib_from_linklib_result=`func_cygming_dll_for_implib_fallback_core '.idata$6' "$1"` + else + # unknown + sharedlib_from_linklib_result="" + fi +} + + +# func_extract_an_archive dir oldlib +func_extract_an_archive () +{ + $opt_debug + f_ex_an_ar_dir="$1"; shift + f_ex_an_ar_oldlib="$1" + if test "$lock_old_archive_extraction" = yes; then + lockfile=$f_ex_an_ar_oldlib.lock + until $opt_dry_run || ln "$progpath" "$lockfile" 2>/dev/null; do + func_echo "Waiting for $lockfile to be removed" + sleep 2 + done + fi + func_show_eval "(cd \$f_ex_an_ar_dir && $AR x \"\$f_ex_an_ar_oldlib\")" \ + 'stat=$?; rm -f "$lockfile"; exit $stat' + if test "$lock_old_archive_extraction" = yes; then + $opt_dry_run || rm -f "$lockfile" + fi + if ($AR t "$f_ex_an_ar_oldlib" | sort | sort -uc >/dev/null 2>&1); then + : + else + func_fatal_error "object name conflicts in archive: $f_ex_an_ar_dir/$f_ex_an_ar_oldlib" + fi +} + + +# func_extract_archives gentop oldlib ... +func_extract_archives () +{ + $opt_debug + my_gentop="$1"; shift + my_oldlibs=${1+"$@"} + my_oldobjs="" + my_xlib="" + my_xabs="" + my_xdir="" + + for my_xlib in $my_oldlibs; do + # Extract the objects. + case $my_xlib in + [\\/]* | [A-Za-z]:[\\/]*) my_xabs="$my_xlib" ;; + *) my_xabs=`pwd`"/$my_xlib" ;; + esac + func_basename "$my_xlib" + my_xlib="$func_basename_result" + my_xlib_u=$my_xlib + while :; do + case " $extracted_archives " in + *" $my_xlib_u "*) + func_arith $extracted_serial + 1 + extracted_serial=$func_arith_result + my_xlib_u=lt$extracted_serial-$my_xlib ;; + *) break ;; + esac + done + extracted_archives="$extracted_archives $my_xlib_u" + my_xdir="$my_gentop/$my_xlib_u" + + func_mkdir_p "$my_xdir" + + case $host in + *-darwin*) + func_verbose "Extracting $my_xabs" + # Do not bother doing anything if just a dry run + $opt_dry_run || { + darwin_orig_dir=`pwd` + cd $my_xdir || exit $? + darwin_archive=$my_xabs + darwin_curdir=`pwd` + darwin_base_archive=`basename "$darwin_archive"` + darwin_arches=`$LIPO -info "$darwin_archive" 2>/dev/null | $GREP Architectures 2>/dev/null || true` + if test -n "$darwin_arches"; then + darwin_arches=`$ECHO "$darwin_arches" | $SED -e 's/.*are://'` + darwin_arch= + func_verbose "$darwin_base_archive has multiple architectures $darwin_arches" + for darwin_arch in $darwin_arches ; do + func_mkdir_p "unfat-$$/${darwin_base_archive}-${darwin_arch}" + $LIPO -thin $darwin_arch -output "unfat-$$/${darwin_base_archive}-${darwin_arch}/${darwin_base_archive}" "${darwin_archive}" + cd "unfat-$$/${darwin_base_archive}-${darwin_arch}" + func_extract_an_archive "`pwd`" "${darwin_base_archive}" + cd "$darwin_curdir" + $RM "unfat-$$/${darwin_base_archive}-${darwin_arch}/${darwin_base_archive}" + done # $darwin_arches + ## Okay now we've a bunch of thin objects, gotta fatten them up :) + darwin_filelist=`find unfat-$$ -type f -name \*.o -print -o -name \*.lo -print | $SED -e "$basename" | sort -u` + darwin_file= + darwin_files= + for darwin_file in $darwin_filelist; do + darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` + $LIPO -create -output "$darwin_file" $darwin_files + done # $darwin_filelist + $RM -rf unfat-$$ + cd "$darwin_orig_dir" + else + cd $darwin_orig_dir + func_extract_an_archive "$my_xdir" "$my_xabs" + fi # $darwin_arches + } # !$opt_dry_run + ;; + *) + func_extract_an_archive "$my_xdir" "$my_xabs" + ;; + esac + my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` + done + + func_extract_archives_result="$my_oldobjs" +} + + +# func_emit_wrapper [arg=no] +# +# Emit a libtool wrapper script on stdout. +# Don't directly open a file because we may want to +# incorporate the script contents within a cygwin/mingw +# wrapper executable. Must ONLY be called from within +# func_mode_link because it depends on a number of variables +# set therein. +# +# ARG is the value that the WRAPPER_SCRIPT_BELONGS_IN_OBJDIR +# variable will take. If 'yes', then the emitted script +# will assume that the directory in which it is stored is +# the $objdir directory. This is a cygwin/mingw-specific +# behavior. +func_emit_wrapper () +{ + func_emit_wrapper_arg1=${1-no} + + $ECHO "\ +#! $SHELL + +# $output - temporary wrapper script for $objdir/$outputname +# Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION +# +# The $output program cannot be directly executed until all the libtool +# libraries that it depends on are installed. +# +# This wrapper script should never be moved out of the build directory. +# If it is, it will not operate correctly. + +# Sed substitution that helps us do robust quoting. It backslashifies +# metacharacters that are still active within double-quoted strings. +sed_quote_subst='$sed_quote_subst' + +# Be Bourne compatible +if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on \${1+\"\$@\"}, which + # is contrary to our usage. Disable this feature. + alias -g '\${1+\"\$@\"}'='\"\$@\"' + setopt NO_GLOB_SUBST +else + case \`(set -o) 2>/dev/null\` in *posix*) set -o posix;; esac +fi +BIN_SH=xpg4; export BIN_SH # for Tru64 +DUALCASE=1; export DUALCASE # for MKS sh + +# The HP-UX ksh and POSIX shell print the target directory to stdout +# if CDPATH is set. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +relink_command=\"$relink_command\" + +# This environment variable determines our operation mode. +if test \"\$libtool_install_magic\" = \"$magic\"; then + # install mode needs the following variables: + generated_by_libtool_version='$macro_version' + notinst_deplibs='$notinst_deplibs' +else + # When we are sourced in execute mode, \$file and \$ECHO are already set. + if test \"\$libtool_execute_magic\" != \"$magic\"; then + file=\"\$0\"" + + qECHO=`$ECHO "$ECHO" | $SED "$sed_quote_subst"` + $ECHO "\ + +# A function that is used when there is no print builtin or printf. +func_fallback_echo () +{ + eval 'cat <<_LTECHO_EOF +\$1 +_LTECHO_EOF' +} + ECHO=\"$qECHO\" + fi + +# Very basic option parsing. These options are (a) specific to +# the libtool wrapper, (b) are identical between the wrapper +# /script/ and the wrapper /executable/ which is used only on +# windows platforms, and (c) all begin with the string "--lt-" +# (application programs are unlikely to have options which match +# this pattern). +# +# There are only two supported options: --lt-debug and +# --lt-dump-script. There is, deliberately, no --lt-help. +# +# The first argument to this parsing function should be the +# script's $0 value, followed by "$@". +lt_option_debug= +func_parse_lt_options () +{ + lt_script_arg0=\$0 + shift + for lt_opt + do + case \"\$lt_opt\" in + --lt-debug) lt_option_debug=1 ;; + --lt-dump-script) + lt_dump_D=\`\$ECHO \"X\$lt_script_arg0\" | $SED -e 's/^X//' -e 's%/[^/]*$%%'\` + test \"X\$lt_dump_D\" = \"X\$lt_script_arg0\" && lt_dump_D=. + lt_dump_F=\`\$ECHO \"X\$lt_script_arg0\" | $SED -e 's/^X//' -e 's%^.*/%%'\` + cat \"\$lt_dump_D/\$lt_dump_F\" + exit 0 + ;; + --lt-*) + \$ECHO \"Unrecognized --lt- option: '\$lt_opt'\" 1>&2 + exit 1 + ;; + esac + done + + # Print the debug banner immediately: + if test -n \"\$lt_option_debug\"; then + echo \"${outputname}:${output}:\${LINENO}: libtool wrapper (GNU $PACKAGE$TIMESTAMP) $VERSION\" 1>&2 + fi +} + +# Used when --lt-debug. Prints its arguments to stdout +# (redirection is the responsibility of the caller) +func_lt_dump_args () +{ + lt_dump_args_N=1; + for lt_arg + do + \$ECHO \"${outputname}:${output}:\${LINENO}: newargv[\$lt_dump_args_N]: \$lt_arg\" + lt_dump_args_N=\`expr \$lt_dump_args_N + 1\` + done +} + +# Core function for launching the target application +func_exec_program_core () +{ +" + case $host in + # Backslashes separate directories on plain windows + *-*-mingw | *-*-os2* | *-cegcc*) + $ECHO "\ + if test -n \"\$lt_option_debug\"; then + \$ECHO \"${outputname}:${output}:\${LINENO}: newargv[0]: \$progdir\\\\\$program\" 1>&2 + func_lt_dump_args \${1+\"\$@\"} 1>&2 + fi + exec \"\$progdir\\\\\$program\" \${1+\"\$@\"} +" + ;; + + *) + $ECHO "\ + if test -n \"\$lt_option_debug\"; then + \$ECHO \"${outputname}:${output}:\${LINENO}: newargv[0]: \$progdir/\$program\" 1>&2 + func_lt_dump_args \${1+\"\$@\"} 1>&2 + fi + exec \"\$progdir/\$program\" \${1+\"\$@\"} +" + ;; + esac + $ECHO "\ + \$ECHO \"\$0: cannot exec \$program \$*\" 1>&2 + exit 1 +} + +# A function to encapsulate launching the target application +# Strips options in the --lt-* namespace from \$@ and +# launches target application with the remaining arguments. +func_exec_program () +{ + case \" \$* \" in + *\\ --lt-*) + for lt_wr_arg + do + case \$lt_wr_arg in + --lt-*) ;; + *) set x \"\$@\" \"\$lt_wr_arg\"; shift;; + esac + shift + done ;; + esac + func_exec_program_core \${1+\"\$@\"} +} + + # Parse options + func_parse_lt_options \"\$0\" \${1+\"\$@\"} + + # Find the directory that this script lives in. + thisdir=\`\$ECHO \"\$file\" | $SED 's%/[^/]*$%%'\` + test \"x\$thisdir\" = \"x\$file\" && thisdir=. + + # Follow symbolic links until we get to the real thisdir. + file=\`ls -ld \"\$file\" | $SED -n 's/.*-> //p'\` + while test -n \"\$file\"; do + destdir=\`\$ECHO \"\$file\" | $SED 's%/[^/]*\$%%'\` + + # If there was a directory component, then change thisdir. + if test \"x\$destdir\" != \"x\$file\"; then + case \"\$destdir\" in + [\\\\/]* | [A-Za-z]:[\\\\/]*) thisdir=\"\$destdir\" ;; + *) thisdir=\"\$thisdir/\$destdir\" ;; + esac + fi + + file=\`\$ECHO \"\$file\" | $SED 's%^.*/%%'\` + file=\`ls -ld \"\$thisdir/\$file\" | $SED -n 's/.*-> //p'\` + done + + # Usually 'no', except on cygwin/mingw when embedded into + # the cwrapper. + WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=$func_emit_wrapper_arg1 + if test \"\$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR\" = \"yes\"; then + # special case for '.' + if test \"\$thisdir\" = \".\"; then + thisdir=\`pwd\` + fi + # remove .libs from thisdir + case \"\$thisdir\" in + *[\\\\/]$objdir ) thisdir=\`\$ECHO \"\$thisdir\" | $SED 's%[\\\\/][^\\\\/]*$%%'\` ;; + $objdir ) thisdir=. ;; + esac + fi + + # Try to get the absolute directory name. + absdir=\`cd \"\$thisdir\" && pwd\` + test -n \"\$absdir\" && thisdir=\"\$absdir\" +" + + if test "$fast_install" = yes; then + $ECHO "\ + program=lt-'$outputname'$exeext + progdir=\"\$thisdir/$objdir\" + + if test ! -f \"\$progdir/\$program\" || + { file=\`ls -1dt \"\$progdir/\$program\" \"\$progdir/../\$program\" 2>/dev/null | ${SED} 1q\`; \\ + test \"X\$file\" != \"X\$progdir/\$program\"; }; then + + file=\"\$\$-\$program\" + + if test ! -d \"\$progdir\"; then + $MKDIR \"\$progdir\" + else + $RM \"\$progdir/\$file\" + fi" + + $ECHO "\ + + # relink executable if necessary + if test -n \"\$relink_command\"; then + if relink_command_output=\`eval \$relink_command 2>&1\`; then : + else + $ECHO \"\$relink_command_output\" >&2 + $RM \"\$progdir/\$file\" + exit 1 + fi + fi + + $MV \"\$progdir/\$file\" \"\$progdir/\$program\" 2>/dev/null || + { $RM \"\$progdir/\$program\"; + $MV \"\$progdir/\$file\" \"\$progdir/\$program\"; } + $RM \"\$progdir/\$file\" + fi" + else + $ECHO "\ + program='$outputname' + progdir=\"\$thisdir/$objdir\" +" + fi + + $ECHO "\ + + if test -f \"\$progdir/\$program\"; then" + + # fixup the dll searchpath if we need to. + # + # Fix the DLL searchpath if we need to. Do this before prepending + # to shlibpath, because on Windows, both are PATH and uninstalled + # libraries must come first. + if test -n "$dllsearchpath"; then + $ECHO "\ + # Add the dll search path components to the executable PATH + PATH=$dllsearchpath:\$PATH +" + fi + + # Export our shlibpath_var if we have one. + if test "$shlibpath_overrides_runpath" = yes && test -n "$shlibpath_var" && test -n "$temp_rpath"; then + $ECHO "\ + # Add our own library path to $shlibpath_var + $shlibpath_var=\"$temp_rpath\$$shlibpath_var\" + + # Some systems cannot cope with colon-terminated $shlibpath_var + # The second colon is a workaround for a bug in BeOS R4 sed + $shlibpath_var=\`\$ECHO \"\$$shlibpath_var\" | $SED 's/::*\$//'\` + + export $shlibpath_var +" + fi + + $ECHO "\ + if test \"\$libtool_execute_magic\" != \"$magic\"; then + # Run the actual program with our arguments. + func_exec_program \${1+\"\$@\"} + fi + else + # The program doesn't exist. + \$ECHO \"\$0: error: \\\`\$progdir/\$program' does not exist\" 1>&2 + \$ECHO \"This script is just a wrapper for \$program.\" 1>&2 + \$ECHO \"See the $PACKAGE documentation for more information.\" 1>&2 + exit 1 + fi +fi\ +" +} + + +# func_emit_cwrapperexe_src +# emit the source code for a wrapper executable on stdout +# Must ONLY be called from within func_mode_link because +# it depends on a number of variable set therein. +func_emit_cwrapperexe_src () +{ + cat < +#include +#ifdef _MSC_VER +# include +# include +# include +#else +# include +# include +# ifdef __CYGWIN__ +# include +# endif +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +/* declarations of non-ANSI functions */ +#if defined(__MINGW32__) +# ifdef __STRICT_ANSI__ +int _putenv (const char *); +# endif +#elif defined(__CYGWIN__) +# ifdef __STRICT_ANSI__ +char *realpath (const char *, char *); +int putenv (char *); +int setenv (const char *, const char *, int); +# endif +/* #elif defined (other platforms) ... */ +#endif + +/* portability defines, excluding path handling macros */ +#if defined(_MSC_VER) +# define setmode _setmode +# define stat _stat +# define chmod _chmod +# define getcwd _getcwd +# define putenv _putenv +# define S_IXUSR _S_IEXEC +# ifndef _INTPTR_T_DEFINED +# define _INTPTR_T_DEFINED +# define intptr_t int +# endif +#elif defined(__MINGW32__) +# define setmode _setmode +# define stat _stat +# define chmod _chmod +# define getcwd _getcwd +# define putenv _putenv +#elif defined(__CYGWIN__) +# define HAVE_SETENV +# define FOPEN_WB "wb" +/* #elif defined (other platforms) ... */ +#endif + +#if defined(PATH_MAX) +# define LT_PATHMAX PATH_MAX +#elif defined(MAXPATHLEN) +# define LT_PATHMAX MAXPATHLEN +#else +# define LT_PATHMAX 1024 +#endif + +#ifndef S_IXOTH +# define S_IXOTH 0 +#endif +#ifndef S_IXGRP +# define S_IXGRP 0 +#endif + +/* path handling portability macros */ +#ifndef DIR_SEPARATOR +# define DIR_SEPARATOR '/' +# define PATH_SEPARATOR ':' +#endif + +#if defined (_WIN32) || defined (__MSDOS__) || defined (__DJGPP__) || \ + defined (__OS2__) +# define HAVE_DOS_BASED_FILE_SYSTEM +# define FOPEN_WB "wb" +# ifndef DIR_SEPARATOR_2 +# define DIR_SEPARATOR_2 '\\' +# endif +# ifndef PATH_SEPARATOR_2 +# define PATH_SEPARATOR_2 ';' +# endif +#endif + +#ifndef DIR_SEPARATOR_2 +# define IS_DIR_SEPARATOR(ch) ((ch) == DIR_SEPARATOR) +#else /* DIR_SEPARATOR_2 */ +# define IS_DIR_SEPARATOR(ch) \ + (((ch) == DIR_SEPARATOR) || ((ch) == DIR_SEPARATOR_2)) +#endif /* DIR_SEPARATOR_2 */ + +#ifndef PATH_SEPARATOR_2 +# define IS_PATH_SEPARATOR(ch) ((ch) == PATH_SEPARATOR) +#else /* PATH_SEPARATOR_2 */ +# define IS_PATH_SEPARATOR(ch) ((ch) == PATH_SEPARATOR_2) +#endif /* PATH_SEPARATOR_2 */ + +#ifndef FOPEN_WB +# define FOPEN_WB "w" +#endif +#ifndef _O_BINARY +# define _O_BINARY 0 +#endif + +#define XMALLOC(type, num) ((type *) xmalloc ((num) * sizeof(type))) +#define XFREE(stale) do { \ + if (stale) { free ((void *) stale); stale = 0; } \ +} while (0) + +#if defined(LT_DEBUGWRAPPER) +static int lt_debug = 1; +#else +static int lt_debug = 0; +#endif + +const char *program_name = "libtool-wrapper"; /* in case xstrdup fails */ + +void *xmalloc (size_t num); +char *xstrdup (const char *string); +const char *base_name (const char *name); +char *find_executable (const char *wrapper); +char *chase_symlinks (const char *pathspec); +int make_executable (const char *path); +int check_executable (const char *path); +char *strendzap (char *str, const char *pat); +void lt_debugprintf (const char *file, int line, const char *fmt, ...); +void lt_fatal (const char *file, int line, const char *message, ...); +static const char *nonnull (const char *s); +static const char *nonempty (const char *s); +void lt_setenv (const char *name, const char *value); +char *lt_extend_str (const char *orig_value, const char *add, int to_end); +void lt_update_exe_path (const char *name, const char *value); +void lt_update_lib_path (const char *name, const char *value); +char **prepare_spawn (char **argv); +void lt_dump_script (FILE *f); +EOF + + cat <= 0) + && (st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) + return 1; + else + return 0; +} + +int +make_executable (const char *path) +{ + int rval = 0; + struct stat st; + + lt_debugprintf (__FILE__, __LINE__, "(make_executable): %s\n", + nonempty (path)); + if ((!path) || (!*path)) + return 0; + + if (stat (path, &st) >= 0) + { + rval = chmod (path, st.st_mode | S_IXOTH | S_IXGRP | S_IXUSR); + } + return rval; +} + +/* Searches for the full path of the wrapper. Returns + newly allocated full path name if found, NULL otherwise + Does not chase symlinks, even on platforms that support them. +*/ +char * +find_executable (const char *wrapper) +{ + int has_slash = 0; + const char *p; + const char *p_next; + /* static buffer for getcwd */ + char tmp[LT_PATHMAX + 1]; + int tmp_len; + char *concat_name; + + lt_debugprintf (__FILE__, __LINE__, "(find_executable): %s\n", + nonempty (wrapper)); + + if ((wrapper == NULL) || (*wrapper == '\0')) + return NULL; + + /* Absolute path? */ +#if defined (HAVE_DOS_BASED_FILE_SYSTEM) + if (isalpha ((unsigned char) wrapper[0]) && wrapper[1] == ':') + { + concat_name = xstrdup (wrapper); + if (check_executable (concat_name)) + return concat_name; + XFREE (concat_name); + } + else + { +#endif + if (IS_DIR_SEPARATOR (wrapper[0])) + { + concat_name = xstrdup (wrapper); + if (check_executable (concat_name)) + return concat_name; + XFREE (concat_name); + } +#if defined (HAVE_DOS_BASED_FILE_SYSTEM) + } +#endif + + for (p = wrapper; *p; p++) + if (*p == '/') + { + has_slash = 1; + break; + } + if (!has_slash) + { + /* no slashes; search PATH */ + const char *path = getenv ("PATH"); + if (path != NULL) + { + for (p = path; *p; p = p_next) + { + const char *q; + size_t p_len; + for (q = p; *q; q++) + if (IS_PATH_SEPARATOR (*q)) + break; + p_len = q - p; + p_next = (*q == '\0' ? q : q + 1); + if (p_len == 0) + { + /* empty path: current directory */ + if (getcwd (tmp, LT_PATHMAX) == NULL) + lt_fatal (__FILE__, __LINE__, "getcwd failed: %s", + nonnull (strerror (errno))); + tmp_len = strlen (tmp); + concat_name = + XMALLOC (char, tmp_len + 1 + strlen (wrapper) + 1); + memcpy (concat_name, tmp, tmp_len); + concat_name[tmp_len] = '/'; + strcpy (concat_name + tmp_len + 1, wrapper); + } + else + { + concat_name = + XMALLOC (char, p_len + 1 + strlen (wrapper) + 1); + memcpy (concat_name, p, p_len); + concat_name[p_len] = '/'; + strcpy (concat_name + p_len + 1, wrapper); + } + if (check_executable (concat_name)) + return concat_name; + XFREE (concat_name); + } + } + /* not found in PATH; assume curdir */ + } + /* Relative path | not found in path: prepend cwd */ + if (getcwd (tmp, LT_PATHMAX) == NULL) + lt_fatal (__FILE__, __LINE__, "getcwd failed: %s", + nonnull (strerror (errno))); + tmp_len = strlen (tmp); + concat_name = XMALLOC (char, tmp_len + 1 + strlen (wrapper) + 1); + memcpy (concat_name, tmp, tmp_len); + concat_name[tmp_len] = '/'; + strcpy (concat_name + tmp_len + 1, wrapper); + + if (check_executable (concat_name)) + return concat_name; + XFREE (concat_name); + return NULL; +} + +char * +chase_symlinks (const char *pathspec) +{ +#ifndef S_ISLNK + return xstrdup (pathspec); +#else + char buf[LT_PATHMAX]; + struct stat s; + char *tmp_pathspec = xstrdup (pathspec); + char *p; + int has_symlinks = 0; + while (strlen (tmp_pathspec) && !has_symlinks) + { + lt_debugprintf (__FILE__, __LINE__, + "checking path component for symlinks: %s\n", + tmp_pathspec); + if (lstat (tmp_pathspec, &s) == 0) + { + if (S_ISLNK (s.st_mode) != 0) + { + has_symlinks = 1; + break; + } + + /* search backwards for last DIR_SEPARATOR */ + p = tmp_pathspec + strlen (tmp_pathspec) - 1; + while ((p > tmp_pathspec) && (!IS_DIR_SEPARATOR (*p))) + p--; + if ((p == tmp_pathspec) && (!IS_DIR_SEPARATOR (*p))) + { + /* no more DIR_SEPARATORS left */ + break; + } + *p = '\0'; + } + else + { + lt_fatal (__FILE__, __LINE__, + "error accessing file \"%s\": %s", + tmp_pathspec, nonnull (strerror (errno))); + } + } + XFREE (tmp_pathspec); + + if (!has_symlinks) + { + return xstrdup (pathspec); + } + + tmp_pathspec = realpath (pathspec, buf); + if (tmp_pathspec == 0) + { + lt_fatal (__FILE__, __LINE__, + "could not follow symlinks for %s", pathspec); + } + return xstrdup (tmp_pathspec); +#endif +} + +char * +strendzap (char *str, const char *pat) +{ + size_t len, patlen; + + assert (str != NULL); + assert (pat != NULL); + + len = strlen (str); + patlen = strlen (pat); + + if (patlen <= len) + { + str += len - patlen; + if (strcmp (str, pat) == 0) + *str = '\0'; + } + return str; +} + +void +lt_debugprintf (const char *file, int line, const char *fmt, ...) +{ + va_list args; + if (lt_debug) + { + (void) fprintf (stderr, "%s:%s:%d: ", program_name, file, line); + va_start (args, fmt); + (void) vfprintf (stderr, fmt, args); + va_end (args); + } +} + +static void +lt_error_core (int exit_status, const char *file, + int line, const char *mode, + const char *message, va_list ap) +{ + fprintf (stderr, "%s:%s:%d: %s: ", program_name, file, line, mode); + vfprintf (stderr, message, ap); + fprintf (stderr, ".\n"); + + if (exit_status >= 0) + exit (exit_status); +} + +void +lt_fatal (const char *file, int line, const char *message, ...) +{ + va_list ap; + va_start (ap, message); + lt_error_core (EXIT_FAILURE, file, line, "FATAL", message, ap); + va_end (ap); +} + +static const char * +nonnull (const char *s) +{ + return s ? s : "(null)"; +} + +static const char * +nonempty (const char *s) +{ + return (s && !*s) ? "(empty)" : nonnull (s); +} + +void +lt_setenv (const char *name, const char *value) +{ + lt_debugprintf (__FILE__, __LINE__, + "(lt_setenv) setting '%s' to '%s'\n", + nonnull (name), nonnull (value)); + { +#ifdef HAVE_SETENV + /* always make a copy, for consistency with !HAVE_SETENV */ + char *str = xstrdup (value); + setenv (name, str, 1); +#else + int len = strlen (name) + 1 + strlen (value) + 1; + char *str = XMALLOC (char, len); + sprintf (str, "%s=%s", name, value); + if (putenv (str) != EXIT_SUCCESS) + { + XFREE (str); + } +#endif + } +} + +char * +lt_extend_str (const char *orig_value, const char *add, int to_end) +{ + char *new_value; + if (orig_value && *orig_value) + { + int orig_value_len = strlen (orig_value); + int add_len = strlen (add); + new_value = XMALLOC (char, add_len + orig_value_len + 1); + if (to_end) + { + strcpy (new_value, orig_value); + strcpy (new_value + orig_value_len, add); + } + else + { + strcpy (new_value, add); + strcpy (new_value + add_len, orig_value); + } + } + else + { + new_value = xstrdup (add); + } + return new_value; +} + +void +lt_update_exe_path (const char *name, const char *value) +{ + lt_debugprintf (__FILE__, __LINE__, + "(lt_update_exe_path) modifying '%s' by prepending '%s'\n", + nonnull (name), nonnull (value)); + + if (name && *name && value && *value) + { + char *new_value = lt_extend_str (getenv (name), value, 0); + /* some systems can't cope with a ':'-terminated path #' */ + int len = strlen (new_value); + while (((len = strlen (new_value)) > 0) && IS_PATH_SEPARATOR (new_value[len-1])) + { + new_value[len-1] = '\0'; + } + lt_setenv (name, new_value); + XFREE (new_value); + } +} + +void +lt_update_lib_path (const char *name, const char *value) +{ + lt_debugprintf (__FILE__, __LINE__, + "(lt_update_lib_path) modifying '%s' by prepending '%s'\n", + nonnull (name), nonnull (value)); + + if (name && *name && value && *value) + { + char *new_value = lt_extend_str (getenv (name), value, 0); + lt_setenv (name, new_value); + XFREE (new_value); + } +} + +EOF + case $host_os in + mingw*) + cat <<"EOF" + +/* Prepares an argument vector before calling spawn(). + Note that spawn() does not by itself call the command interpreter + (getenv ("COMSPEC") != NULL ? getenv ("COMSPEC") : + ({ OSVERSIONINFO v; v.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); + GetVersionEx(&v); + v.dwPlatformId == VER_PLATFORM_WIN32_NT; + }) ? "cmd.exe" : "command.com"). + Instead it simply concatenates the arguments, separated by ' ', and calls + CreateProcess(). We must quote the arguments since Win32 CreateProcess() + interprets characters like ' ', '\t', '\\', '"' (but not '<' and '>') in a + special way: + - Space and tab are interpreted as delimiters. They are not treated as + delimiters if they are surrounded by double quotes: "...". + - Unescaped double quotes are removed from the input. Their only effect is + that within double quotes, space and tab are treated like normal + characters. + - Backslashes not followed by double quotes are not special. + - But 2*n+1 backslashes followed by a double quote become + n backslashes followed by a double quote (n >= 0): + \" -> " + \\\" -> \" + \\\\\" -> \\" + */ +#define SHELL_SPECIAL_CHARS "\"\\ \001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" +#define SHELL_SPACE_CHARS " \001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" +char ** +prepare_spawn (char **argv) +{ + size_t argc; + char **new_argv; + size_t i; + + /* Count number of arguments. */ + for (argc = 0; argv[argc] != NULL; argc++) + ; + + /* Allocate new argument vector. */ + new_argv = XMALLOC (char *, argc + 1); + + /* Put quoted arguments into the new argument vector. */ + for (i = 0; i < argc; i++) + { + const char *string = argv[i]; + + if (string[0] == '\0') + new_argv[i] = xstrdup ("\"\""); + else if (strpbrk (string, SHELL_SPECIAL_CHARS) != NULL) + { + int quote_around = (strpbrk (string, SHELL_SPACE_CHARS) != NULL); + size_t length; + unsigned int backslashes; + const char *s; + char *quoted_string; + char *p; + + length = 0; + backslashes = 0; + if (quote_around) + length++; + for (s = string; *s != '\0'; s++) + { + char c = *s; + if (c == '"') + length += backslashes + 1; + length++; + if (c == '\\') + backslashes++; + else + backslashes = 0; + } + if (quote_around) + length += backslashes + 1; + + quoted_string = XMALLOC (char, length + 1); + + p = quoted_string; + backslashes = 0; + if (quote_around) + *p++ = '"'; + for (s = string; *s != '\0'; s++) + { + char c = *s; + if (c == '"') + { + unsigned int j; + for (j = backslashes + 1; j > 0; j--) + *p++ = '\\'; + } + *p++ = c; + if (c == '\\') + backslashes++; + else + backslashes = 0; + } + if (quote_around) + { + unsigned int j; + for (j = backslashes; j > 0; j--) + *p++ = '\\'; + *p++ = '"'; + } + *p = '\0'; + + new_argv[i] = quoted_string; + } + else + new_argv[i] = (char *) string; + } + new_argv[argc] = NULL; + + return new_argv; +} +EOF + ;; + esac + + cat <<"EOF" +void lt_dump_script (FILE* f) +{ +EOF + func_emit_wrapper yes | + $SED -n -e ' +s/^\(.\{79\}\)\(..*\)/\1\ +\2/ +h +s/\([\\"]\)/\\\1/g +s/$/\\n/ +s/\([^\n]*\).*/ fputs ("\1", f);/p +g +D' + cat <<"EOF" +} +EOF +} +# end: func_emit_cwrapperexe_src + +# func_win32_import_lib_p ARG +# True if ARG is an import lib, as indicated by $file_magic_cmd +func_win32_import_lib_p () +{ + $opt_debug + case `eval $file_magic_cmd \"\$1\" 2>/dev/null | $SED -e 10q` in + *import*) : ;; + *) false ;; + esac +} + +# func_mode_link arg... +func_mode_link () +{ + $opt_debug + case $host in + *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*) + # It is impossible to link a dll without this setting, and + # we shouldn't force the makefile maintainer to figure out + # which system we are compiling for in order to pass an extra + # flag for every libtool invocation. + # allow_undefined=no + + # FIXME: Unfortunately, there are problems with the above when trying + # to make a dll which has undefined symbols, in which case not + # even a static library is built. For now, we need to specify + # -no-undefined on the libtool link line when we can be certain + # that all symbols are satisfied, otherwise we get a static library. + allow_undefined=yes + ;; + *) + allow_undefined=yes + ;; + esac + libtool_args=$nonopt + base_compile="$nonopt $@" + compile_command=$nonopt + finalize_command=$nonopt + + compile_rpath= + finalize_rpath= + compile_shlibpath= + finalize_shlibpath= + convenience= + old_convenience= + deplibs= + old_deplibs= + compiler_flags= + linker_flags= + dllsearchpath= + lib_search_path=`pwd` + inst_prefix_dir= + new_inherited_linker_flags= + + avoid_version=no + bindir= + dlfiles= + dlprefiles= + dlself=no + export_dynamic=no + export_symbols= + export_symbols_regex= + generated= + libobjs= + ltlibs= + module=no + no_install=no + objs= + non_pic_objects= + precious_files_regex= + prefer_static_libs=no + preload=no + prev= + prevarg= + release= + rpath= + xrpath= + perm_rpath= + temp_rpath= + thread_safe=no + vinfo= + vinfo_number=no + weak_libs= + single_module="${wl}-single_module" + func_infer_tag $base_compile + + # We need to know -static, to get the right output filenames. + for arg + do + case $arg in + -shared) + test "$build_libtool_libs" != yes && \ + func_fatal_configuration "can not build a shared library" + build_old_libs=no + break + ;; + -all-static | -static | -static-libtool-libs) + case $arg in + -all-static) + if test "$build_libtool_libs" = yes && test -z "$link_static_flag"; then + func_warning "complete static linking is impossible in this configuration" + fi + if test -n "$link_static_flag"; then + dlopen_self=$dlopen_self_static + fi + prefer_static_libs=yes + ;; + -static) + if test -z "$pic_flag" && test -n "$link_static_flag"; then + dlopen_self=$dlopen_self_static + fi + prefer_static_libs=built + ;; + -static-libtool-libs) + if test -z "$pic_flag" && test -n "$link_static_flag"; then + dlopen_self=$dlopen_self_static + fi + prefer_static_libs=yes + ;; + esac + build_libtool_libs=no + build_old_libs=yes + break + ;; + esac + done + + # See if our shared archives depend on static archives. + test -n "$old_archive_from_new_cmds" && build_old_libs=yes + + # Go through the arguments, transforming them on the way. + while test "$#" -gt 0; do + arg="$1" + shift + func_quote_for_eval "$arg" + qarg=$func_quote_for_eval_unquoted_result + func_append libtool_args " $func_quote_for_eval_result" + + # If the previous option needs an argument, assign it. + if test -n "$prev"; then + case $prev in + output) + func_append compile_command " @OUTPUT@" + func_append finalize_command " @OUTPUT@" + ;; + esac + + case $prev in + bindir) + bindir="$arg" + prev= + continue + ;; + dlfiles|dlprefiles) + if test "$preload" = no; then + # Add the symbol object into the linking commands. + func_append compile_command " @SYMFILE@" + func_append finalize_command " @SYMFILE@" + preload=yes + fi + case $arg in + *.la | *.lo) ;; # We handle these cases below. + force) + if test "$dlself" = no; then + dlself=needless + export_dynamic=yes + fi + prev= + continue + ;; + self) + if test "$prev" = dlprefiles; then + dlself=yes + elif test "$prev" = dlfiles && test "$dlopen_self" != yes; then + dlself=yes + else + dlself=needless + export_dynamic=yes + fi + prev= + continue + ;; + *) + if test "$prev" = dlfiles; then + func_append dlfiles " $arg" + else + func_append dlprefiles " $arg" + fi + prev= + continue + ;; + esac + ;; + expsyms) + export_symbols="$arg" + test -f "$arg" \ + || func_fatal_error "symbol file \`$arg' does not exist" + prev= + continue + ;; + expsyms_regex) + export_symbols_regex="$arg" + prev= + continue + ;; + framework) + case $host in + *-*-darwin*) + case "$deplibs " in + *" $qarg.ltframework "*) ;; + *) func_append deplibs " $qarg.ltframework" # this is fixed later + ;; + esac + ;; + esac + prev= + continue + ;; + inst_prefix) + inst_prefix_dir="$arg" + prev= + continue + ;; + objectlist) + if test -f "$arg"; then + save_arg=$arg + moreargs= + for fil in `cat "$save_arg"` + do +# func_append moreargs " $fil" + arg=$fil + # A libtool-controlled object. + + # Check to see that this really is a libtool object. + if func_lalib_unsafe_p "$arg"; then + pic_object= + non_pic_object= + + # Read the .lo file + func_source "$arg" + + if test -z "$pic_object" || + test -z "$non_pic_object" || + test "$pic_object" = none && + test "$non_pic_object" = none; then + func_fatal_error "cannot find name of object for \`$arg'" + fi + + # Extract subdirectory from the argument. + func_dirname "$arg" "/" "" + xdir="$func_dirname_result" + + if test "$pic_object" != none; then + # Prepend the subdirectory the object is found in. + pic_object="$xdir$pic_object" + + if test "$prev" = dlfiles; then + if test "$build_libtool_libs" = yes && test "$dlopen_support" = yes; then + func_append dlfiles " $pic_object" + prev= + continue + else + # If libtool objects are unsupported, then we need to preload. + prev=dlprefiles + fi + fi + + # CHECK ME: I think I busted this. -Ossama + if test "$prev" = dlprefiles; then + # Preload the old-style object. + func_append dlprefiles " $pic_object" + prev= + fi + + # A PIC object. + func_append libobjs " $pic_object" + arg="$pic_object" + fi + + # Non-PIC object. + if test "$non_pic_object" != none; then + # Prepend the subdirectory the object is found in. + non_pic_object="$xdir$non_pic_object" + + # A standard non-PIC object + func_append non_pic_objects " $non_pic_object" + if test -z "$pic_object" || test "$pic_object" = none ; then + arg="$non_pic_object" + fi + else + # If the PIC object exists, use it instead. + # $xdir was prepended to $pic_object above. + non_pic_object="$pic_object" + func_append non_pic_objects " $non_pic_object" + fi + else + # Only an error if not doing a dry-run. + if $opt_dry_run; then + # Extract subdirectory from the argument. + func_dirname "$arg" "/" "" + xdir="$func_dirname_result" + + func_lo2o "$arg" + pic_object=$xdir$objdir/$func_lo2o_result + non_pic_object=$xdir$func_lo2o_result + func_append libobjs " $pic_object" + func_append non_pic_objects " $non_pic_object" + else + func_fatal_error "\`$arg' is not a valid libtool object" + fi + fi + done + else + func_fatal_error "link input file \`$arg' does not exist" + fi + arg=$save_arg + prev= + continue + ;; + precious_regex) + precious_files_regex="$arg" + prev= + continue + ;; + release) + release="-$arg" + prev= + continue + ;; + rpath | xrpath) + # We need an absolute path. + case $arg in + [\\/]* | [A-Za-z]:[\\/]*) ;; + *) + func_fatal_error "only absolute run-paths are allowed" + ;; + esac + if test "$prev" = rpath; then + case "$rpath " in + *" $arg "*) ;; + *) func_append rpath " $arg" ;; + esac + else + case "$xrpath " in + *" $arg "*) ;; + *) func_append xrpath " $arg" ;; + esac + fi + prev= + continue + ;; + shrext) + shrext_cmds="$arg" + prev= + continue + ;; + weak) + func_append weak_libs " $arg" + prev= + continue + ;; + xcclinker) + func_append linker_flags " $qarg" + func_append compiler_flags " $qarg" + prev= + func_append compile_command " $qarg" + func_append finalize_command " $qarg" + continue + ;; + xcompiler) + func_append compiler_flags " $qarg" + prev= + func_append compile_command " $qarg" + func_append finalize_command " $qarg" + continue + ;; + xlinker) + func_append linker_flags " $qarg" + func_append compiler_flags " $wl$qarg" + prev= + func_append compile_command " $wl$qarg" + func_append finalize_command " $wl$qarg" + continue + ;; + *) + eval "$prev=\"\$arg\"" + prev= + continue + ;; + esac + fi # test -n "$prev" + + prevarg="$arg" + + case $arg in + -all-static) + if test -n "$link_static_flag"; then + # See comment for -static flag below, for more details. + func_append compile_command " $link_static_flag" + func_append finalize_command " $link_static_flag" + fi + continue + ;; + + -allow-undefined) + # FIXME: remove this flag sometime in the future. + func_fatal_error "\`-allow-undefined' must not be used because it is the default" + ;; + + -avoid-version) + avoid_version=yes + continue + ;; + + -bindir) + prev=bindir + continue + ;; + + -dlopen) + prev=dlfiles + continue + ;; + + -dlpreopen) + prev=dlprefiles + continue + ;; + + -export-dynamic) + export_dynamic=yes + continue + ;; + + -export-symbols | -export-symbols-regex) + if test -n "$export_symbols" || test -n "$export_symbols_regex"; then + func_fatal_error "more than one -exported-symbols argument is not allowed" + fi + if test "X$arg" = "X-export-symbols"; then + prev=expsyms + else + prev=expsyms_regex + fi + continue + ;; + + -framework) + prev=framework + continue + ;; + + -inst-prefix-dir) + prev=inst_prefix + continue + ;; + + # The native IRIX linker understands -LANG:*, -LIST:* and -LNO:* + # so, if we see these flags be careful not to treat them like -L + -L[A-Z][A-Z]*:*) + case $with_gcc/$host in + no/*-*-irix* | /*-*-irix*) + func_append compile_command " $arg" + func_append finalize_command " $arg" + ;; + esac + continue + ;; + + -L*) + func_stripname "-L" '' "$arg" + if test -z "$func_stripname_result"; then + if test "$#" -gt 0; then + func_fatal_error "require no space between \`-L' and \`$1'" + else + func_fatal_error "need path for \`-L' option" + fi + fi + func_resolve_sysroot "$func_stripname_result" + dir=$func_resolve_sysroot_result + # We need an absolute path. + case $dir in + [\\/]* | [A-Za-z]:[\\/]*) ;; + *) + absdir=`cd "$dir" && pwd` + test -z "$absdir" && \ + func_fatal_error "cannot determine absolute directory name of \`$dir'" + dir="$absdir" + ;; + esac + case "$deplibs " in + *" -L$dir "* | *" $arg "*) + # Will only happen for absolute or sysroot arguments + ;; + *) + # Preserve sysroot, but never include relative directories + case $dir in + [\\/]* | [A-Za-z]:[\\/]* | =*) func_append deplibs " $arg" ;; + *) func_append deplibs " -L$dir" ;; + esac + func_append lib_search_path " $dir" + ;; + esac + case $host in + *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*) + testbindir=`$ECHO "$dir" | $SED 's*/lib$*/bin*'` + case :$dllsearchpath: in + *":$dir:"*) ;; + ::) dllsearchpath=$dir;; + *) func_append dllsearchpath ":$dir";; + esac + case :$dllsearchpath: in + *":$testbindir:"*) ;; + ::) dllsearchpath=$testbindir;; + *) func_append dllsearchpath ":$testbindir";; + esac + ;; + esac + continue + ;; + + -l*) + if test "X$arg" = "X-lc" || test "X$arg" = "X-lm"; then + case $host in + *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-beos* | *-cegcc* | *-*-haiku*) + # These systems don't actually have a C or math library (as such) + continue + ;; + *-*-os2*) + # These systems don't actually have a C library (as such) + test "X$arg" = "X-lc" && continue + ;; + *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*) + # Do not include libc due to us having libc/libc_r. + test "X$arg" = "X-lc" && continue + ;; + *-*-rhapsody* | *-*-darwin1.[012]) + # Rhapsody C and math libraries are in the System framework + func_append deplibs " System.ltframework" + continue + ;; + *-*-sco3.2v5* | *-*-sco5v6*) + # Causes problems with __ctype + test "X$arg" = "X-lc" && continue + ;; + *-*-sysv4.2uw2* | *-*-sysv5* | *-*-unixware* | *-*-OpenUNIX*) + # Compiler inserts libc in the correct place for threads to work + test "X$arg" = "X-lc" && continue + ;; + esac + elif test "X$arg" = "X-lc_r"; then + case $host in + *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*) + # Do not include libc_r directly, use -pthread flag. + continue + ;; + esac + fi + func_append deplibs " $arg" + continue + ;; + + -module) + module=yes + continue + ;; + + # Tru64 UNIX uses -model [arg] to determine the layout of C++ + # classes, name mangling, and exception handling. + # Darwin uses the -arch flag to determine output architecture. + -model|-arch|-isysroot|--sysroot) + func_append compiler_flags " $arg" + func_append compile_command " $arg" + func_append finalize_command " $arg" + prev=xcompiler + continue + ;; + + -mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe \ + |-threads|-fopenmp|-openmp|-mp|-xopenmp|-omp|-qsmp=*) + func_append compiler_flags " $arg" + func_append compile_command " $arg" + func_append finalize_command " $arg" + case "$new_inherited_linker_flags " in + *" $arg "*) ;; + * ) func_append new_inherited_linker_flags " $arg" ;; + esac + continue + ;; + + -multi_module) + single_module="${wl}-multi_module" + continue + ;; + + -no-fast-install) + fast_install=no + continue + ;; + + -no-install) + case $host in + *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-*-darwin* | *-cegcc*) + # The PATH hackery in wrapper scripts is required on Windows + # and Darwin in order for the loader to find any dlls it needs. + func_warning "\`-no-install' is ignored for $host" + func_warning "assuming \`-no-fast-install' instead" + fast_install=no + ;; + *) no_install=yes ;; + esac + continue + ;; + + -no-undefined) + allow_undefined=no + continue + ;; + + -objectlist) + prev=objectlist + continue + ;; + + -o) prev=output ;; + + -precious-files-regex) + prev=precious_regex + continue + ;; + + -release) + prev=release + continue + ;; + + -rpath) + prev=rpath + continue + ;; + + -R) + prev=xrpath + continue + ;; + + -R*) + func_stripname '-R' '' "$arg" + dir=$func_stripname_result + # We need an absolute path. + case $dir in + [\\/]* | [A-Za-z]:[\\/]*) ;; + =*) + func_stripname '=' '' "$dir" + dir=$lt_sysroot$func_stripname_result + ;; + *) + func_fatal_error "only absolute run-paths are allowed" + ;; + esac + case "$xrpath " in + *" $dir "*) ;; + *) func_append xrpath " $dir" ;; + esac + continue + ;; + + -shared) + # The effects of -shared are defined in a previous loop. + continue + ;; + + -shrext) + prev=shrext + continue + ;; + + -static | -static-libtool-libs) + # The effects of -static are defined in a previous loop. + # We used to do the same as -all-static on platforms that + # didn't have a PIC flag, but the assumption that the effects + # would be equivalent was wrong. It would break on at least + # Digital Unix and AIX. + continue + ;; + + -thread-safe) + thread_safe=yes + continue + ;; + + -version-info) + prev=vinfo + continue + ;; + + -version-number) + prev=vinfo + vinfo_number=yes + continue + ;; + + -weak) + prev=weak + continue + ;; + + -Wc,*) + func_stripname '-Wc,' '' "$arg" + args=$func_stripname_result + arg= + save_ifs="$IFS"; IFS=',' + for flag in $args; do + IFS="$save_ifs" + func_quote_for_eval "$flag" + func_append arg " $func_quote_for_eval_result" + func_append compiler_flags " $func_quote_for_eval_result" + done + IFS="$save_ifs" + func_stripname ' ' '' "$arg" + arg=$func_stripname_result + ;; + + -Wl,*) + func_stripname '-Wl,' '' "$arg" + args=$func_stripname_result + arg= + save_ifs="$IFS"; IFS=',' + for flag in $args; do + IFS="$save_ifs" + func_quote_for_eval "$flag" + func_append arg " $wl$func_quote_for_eval_result" + func_append compiler_flags " $wl$func_quote_for_eval_result" + func_append linker_flags " $func_quote_for_eval_result" + done + IFS="$save_ifs" + func_stripname ' ' '' "$arg" + arg=$func_stripname_result + ;; + + -Xcompiler) + prev=xcompiler + continue + ;; + + -Xlinker) + prev=xlinker + continue + ;; + + -XCClinker) + prev=xcclinker + continue + ;; + + # -msg_* for osf cc + -msg_*) + func_quote_for_eval "$arg" + arg="$func_quote_for_eval_result" + ;; + + # Flags to be passed through unchanged, with rationale: + # -64, -mips[0-9] enable 64-bit mode for the SGI compiler + # -r[0-9][0-9]* specify processor for the SGI compiler + # -xarch=*, -xtarget=* enable 64-bit mode for the Sun compiler + # +DA*, +DD* enable 64-bit mode for the HP compiler + # -q* compiler args for the IBM compiler + # -m*, -t[45]*, -txscale* architecture-specific flags for GCC + # -F/path path to uninstalled frameworks, gcc on darwin + # -p, -pg, --coverage, -fprofile-* profiling flags for GCC + # @file GCC response files + # -tp=* Portland pgcc target processor selection + # --sysroot=* for sysroot support + # -O*, -flto*, -fwhopr*, -fuse-linker-plugin GCC link-time optimization + -64|-mips[0-9]|-r[0-9][0-9]*|-xarch=*|-xtarget=*|+DA*|+DD*|-q*|-m*| \ + -t[45]*|-txscale*|-p|-pg|--coverage|-fprofile-*|-F*|@*|-tp=*|--sysroot=*| \ + -O*|-flto*|-fwhopr*|-fuse-linker-plugin) + func_quote_for_eval "$arg" + arg="$func_quote_for_eval_result" + func_append compile_command " $arg" + func_append finalize_command " $arg" + func_append compiler_flags " $arg" + continue + ;; + + # Some other compiler flag. + -* | +*) + func_quote_for_eval "$arg" + arg="$func_quote_for_eval_result" + ;; + + *.$objext) + # A standard object. + func_append objs " $arg" + ;; + + *.lo) + # A libtool-controlled object. + + # Check to see that this really is a libtool object. + if func_lalib_unsafe_p "$arg"; then + pic_object= + non_pic_object= + + # Read the .lo file + func_source "$arg" + + if test -z "$pic_object" || + test -z "$non_pic_object" || + test "$pic_object" = none && + test "$non_pic_object" = none; then + func_fatal_error "cannot find name of object for \`$arg'" + fi + + # Extract subdirectory from the argument. + func_dirname "$arg" "/" "" + xdir="$func_dirname_result" + + if test "$pic_object" != none; then + # Prepend the subdirectory the object is found in. + pic_object="$xdir$pic_object" + + if test "$prev" = dlfiles; then + if test "$build_libtool_libs" = yes && test "$dlopen_support" = yes; then + func_append dlfiles " $pic_object" + prev= + continue + else + # If libtool objects are unsupported, then we need to preload. + prev=dlprefiles + fi + fi + + # CHECK ME: I think I busted this. -Ossama + if test "$prev" = dlprefiles; then + # Preload the old-style object. + func_append dlprefiles " $pic_object" + prev= + fi + + # A PIC object. + func_append libobjs " $pic_object" + arg="$pic_object" + fi + + # Non-PIC object. + if test "$non_pic_object" != none; then + # Prepend the subdirectory the object is found in. + non_pic_object="$xdir$non_pic_object" + + # A standard non-PIC object + func_append non_pic_objects " $non_pic_object" + if test -z "$pic_object" || test "$pic_object" = none ; then + arg="$non_pic_object" + fi + else + # If the PIC object exists, use it instead. + # $xdir was prepended to $pic_object above. + non_pic_object="$pic_object" + func_append non_pic_objects " $non_pic_object" + fi + else + # Only an error if not doing a dry-run. + if $opt_dry_run; then + # Extract subdirectory from the argument. + func_dirname "$arg" "/" "" + xdir="$func_dirname_result" + + func_lo2o "$arg" + pic_object=$xdir$objdir/$func_lo2o_result + non_pic_object=$xdir$func_lo2o_result + func_append libobjs " $pic_object" + func_append non_pic_objects " $non_pic_object" + else + func_fatal_error "\`$arg' is not a valid libtool object" + fi + fi + ;; + + *.$libext) + # An archive. + func_append deplibs " $arg" + func_append old_deplibs " $arg" + continue + ;; + + *.la) + # A libtool-controlled library. + + func_resolve_sysroot "$arg" + if test "$prev" = dlfiles; then + # This library was specified with -dlopen. + func_append dlfiles " $func_resolve_sysroot_result" + prev= + elif test "$prev" = dlprefiles; then + # The library was specified with -dlpreopen. + func_append dlprefiles " $func_resolve_sysroot_result" + prev= + else + func_append deplibs " $func_resolve_sysroot_result" + fi + continue + ;; + + # Some other compiler argument. + *) + # Unknown arguments in both finalize_command and compile_command need + # to be aesthetically quoted because they are evaled later. + func_quote_for_eval "$arg" + arg="$func_quote_for_eval_result" + ;; + esac # arg + + # Now actually substitute the argument into the commands. + if test -n "$arg"; then + func_append compile_command " $arg" + func_append finalize_command " $arg" + fi + done # argument parsing loop + + test -n "$prev" && \ + func_fatal_help "the \`$prevarg' option requires an argument" + + if test "$export_dynamic" = yes && test -n "$export_dynamic_flag_spec"; then + eval arg=\"$export_dynamic_flag_spec\" + func_append compile_command " $arg" + func_append finalize_command " $arg" + fi + + oldlibs= + # calculate the name of the file, without its directory + func_basename "$output" + outputname="$func_basename_result" + libobjs_save="$libobjs" + + if test -n "$shlibpath_var"; then + # get the directories listed in $shlibpath_var + eval shlib_search_path=\`\$ECHO \"\${$shlibpath_var}\" \| \$SED \'s/:/ /g\'\` + else + shlib_search_path= + fi + eval sys_lib_search_path=\"$sys_lib_search_path_spec\" + eval sys_lib_dlsearch_path=\"$sys_lib_dlsearch_path_spec\" + + func_dirname "$output" "/" "" + output_objdir="$func_dirname_result$objdir" + func_to_tool_file "$output_objdir/" + tool_output_objdir=$func_to_tool_file_result + # Create the object directory. + func_mkdir_p "$output_objdir" + + # Determine the type of output + case $output in + "") + func_fatal_help "you must specify an output file" + ;; + *.$libext) linkmode=oldlib ;; + *.lo | *.$objext) linkmode=obj ;; + *.la) linkmode=lib ;; + *) linkmode=prog ;; # Anything else should be a program. + esac + + specialdeplibs= + + libs= + # Find all interdependent deplibs by searching for libraries + # that are linked more than once (e.g. -la -lb -la) + for deplib in $deplibs; do + if $opt_preserve_dup_deps ; then + case "$libs " in + *" $deplib "*) func_append specialdeplibs " $deplib" ;; + esac + fi + func_append libs " $deplib" + done + + if test "$linkmode" = lib; then + libs="$predeps $libs $compiler_lib_search_path $postdeps" + + # Compute libraries that are listed more than once in $predeps + # $postdeps and mark them as special (i.e., whose duplicates are + # not to be eliminated). + pre_post_deps= + if $opt_duplicate_compiler_generated_deps; then + for pre_post_dep in $predeps $postdeps; do + case "$pre_post_deps " in + *" $pre_post_dep "*) func_append specialdeplibs " $pre_post_deps" ;; + esac + func_append pre_post_deps " $pre_post_dep" + done + fi + pre_post_deps= + fi + + deplibs= + newdependency_libs= + newlib_search_path= + need_relink=no # whether we're linking any uninstalled libtool libraries + notinst_deplibs= # not-installed libtool libraries + notinst_path= # paths that contain not-installed libtool libraries + + case $linkmode in + lib) + passes="conv dlpreopen link" + for file in $dlfiles $dlprefiles; do + case $file in + *.la) ;; + *) + func_fatal_help "libraries can \`-dlopen' only libtool libraries: $file" + ;; + esac + done + ;; + prog) + compile_deplibs= + finalize_deplibs= + alldeplibs=no + newdlfiles= + newdlprefiles= + passes="conv scan dlopen dlpreopen link" + ;; + *) passes="conv" + ;; + esac + + for pass in $passes; do + # The preopen pass in lib mode reverses $deplibs; put it back here + # so that -L comes before libs that need it for instance... + if test "$linkmode,$pass" = "lib,link"; then + ## FIXME: Find the place where the list is rebuilt in the wrong + ## order, and fix it there properly + tmp_deplibs= + for deplib in $deplibs; do + tmp_deplibs="$deplib $tmp_deplibs" + done + deplibs="$tmp_deplibs" + fi + + if test "$linkmode,$pass" = "lib,link" || + test "$linkmode,$pass" = "prog,scan"; then + libs="$deplibs" + deplibs= + fi + if test "$linkmode" = prog; then + case $pass in + dlopen) libs="$dlfiles" ;; + dlpreopen) libs="$dlprefiles" ;; + link) + libs="$deplibs %DEPLIBS%" + test "X$link_all_deplibs" != Xno && libs="$libs $dependency_libs" + ;; + esac + fi + if test "$linkmode,$pass" = "lib,dlpreopen"; then + # Collect and forward deplibs of preopened libtool libs + for lib in $dlprefiles; do + # Ignore non-libtool-libs + dependency_libs= + func_resolve_sysroot "$lib" + case $lib in + *.la) func_source "$func_resolve_sysroot_result" ;; + esac + + # Collect preopened libtool deplibs, except any this library + # has declared as weak libs + for deplib in $dependency_libs; do + func_basename "$deplib" + deplib_base=$func_basename_result + case " $weak_libs " in + *" $deplib_base "*) ;; + *) func_append deplibs " $deplib" ;; + esac + done + done + libs="$dlprefiles" + fi + if test "$pass" = dlopen; then + # Collect dlpreopened libraries + save_deplibs="$deplibs" + deplibs= + fi + + for deplib in $libs; do + lib= + found=no + case $deplib in + -mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe \ + |-threads|-fopenmp|-openmp|-mp|-xopenmp|-omp|-qsmp=*) + if test "$linkmode,$pass" = "prog,link"; then + compile_deplibs="$deplib $compile_deplibs" + finalize_deplibs="$deplib $finalize_deplibs" + else + func_append compiler_flags " $deplib" + if test "$linkmode" = lib ; then + case "$new_inherited_linker_flags " in + *" $deplib "*) ;; + * ) func_append new_inherited_linker_flags " $deplib" ;; + esac + fi + fi + continue + ;; + -l*) + if test "$linkmode" != lib && test "$linkmode" != prog; then + func_warning "\`-l' is ignored for archives/objects" + continue + fi + func_stripname '-l' '' "$deplib" + name=$func_stripname_result + if test "$linkmode" = lib; then + searchdirs="$newlib_search_path $lib_search_path $compiler_lib_search_dirs $sys_lib_search_path $shlib_search_path" + else + searchdirs="$newlib_search_path $lib_search_path $sys_lib_search_path $shlib_search_path" + fi + for searchdir in $searchdirs; do + for search_ext in .la $std_shrext .so .a; do + # Search the libtool library + lib="$searchdir/lib${name}${search_ext}" + if test -f "$lib"; then + if test "$search_ext" = ".la"; then + found=yes + else + found=no + fi + break 2 + fi + done + done + if test "$found" != yes; then + # deplib doesn't seem to be a libtool library + if test "$linkmode,$pass" = "prog,link"; then + compile_deplibs="$deplib $compile_deplibs" + finalize_deplibs="$deplib $finalize_deplibs" + else + deplibs="$deplib $deplibs" + test "$linkmode" = lib && newdependency_libs="$deplib $newdependency_libs" + fi + continue + else # deplib is a libtool library + # If $allow_libtool_libs_with_static_runtimes && $deplib is a stdlib, + # We need to do some special things here, and not later. + if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then + case " $predeps $postdeps " in + *" $deplib "*) + if func_lalib_p "$lib"; then + library_names= + old_library= + func_source "$lib" + for l in $old_library $library_names; do + ll="$l" + done + if test "X$ll" = "X$old_library" ; then # only static version available + found=no + func_dirname "$lib" "" "." + ladir="$func_dirname_result" + lib=$ladir/$old_library + if test "$linkmode,$pass" = "prog,link"; then + compile_deplibs="$deplib $compile_deplibs" + finalize_deplibs="$deplib $finalize_deplibs" + else + deplibs="$deplib $deplibs" + test "$linkmode" = lib && newdependency_libs="$deplib $newdependency_libs" + fi + continue + fi + fi + ;; + *) ;; + esac + fi + fi + ;; # -l + *.ltframework) + if test "$linkmode,$pass" = "prog,link"; then + compile_deplibs="$deplib $compile_deplibs" + finalize_deplibs="$deplib $finalize_deplibs" + else + deplibs="$deplib $deplibs" + if test "$linkmode" = lib ; then + case "$new_inherited_linker_flags " in + *" $deplib "*) ;; + * ) func_append new_inherited_linker_flags " $deplib" ;; + esac + fi + fi + continue + ;; + -L*) + case $linkmode in + lib) + deplibs="$deplib $deplibs" + test "$pass" = conv && continue + newdependency_libs="$deplib $newdependency_libs" + func_stripname '-L' '' "$deplib" + func_resolve_sysroot "$func_stripname_result" + func_append newlib_search_path " $func_resolve_sysroot_result" + ;; + prog) + if test "$pass" = conv; then + deplibs="$deplib $deplibs" + continue + fi + if test "$pass" = scan; then + deplibs="$deplib $deplibs" + else + compile_deplibs="$deplib $compile_deplibs" + finalize_deplibs="$deplib $finalize_deplibs" + fi + func_stripname '-L' '' "$deplib" + func_resolve_sysroot "$func_stripname_result" + func_append newlib_search_path " $func_resolve_sysroot_result" + ;; + *) + func_warning "\`-L' is ignored for archives/objects" + ;; + esac # linkmode + continue + ;; # -L + -R*) + if test "$pass" = link; then + func_stripname '-R' '' "$deplib" + func_resolve_sysroot "$func_stripname_result" + dir=$func_resolve_sysroot_result + # Make sure the xrpath contains only unique directories. + case "$xrpath " in + *" $dir "*) ;; + *) func_append xrpath " $dir" ;; + esac + fi + deplibs="$deplib $deplibs" + continue + ;; + *.la) + func_resolve_sysroot "$deplib" + lib=$func_resolve_sysroot_result + ;; + *.$libext) + if test "$pass" = conv; then + deplibs="$deplib $deplibs" + continue + fi + case $linkmode in + lib) + # Linking convenience modules into shared libraries is allowed, + # but linking other static libraries is non-portable. + case " $dlpreconveniencelibs " in + *" $deplib "*) ;; + *) + valid_a_lib=no + case $deplibs_check_method in + match_pattern*) + set dummy $deplibs_check_method; shift + match_pattern_regex=`expr "$deplibs_check_method" : "$1 \(.*\)"` + if eval "\$ECHO \"$deplib\"" 2>/dev/null | $SED 10q \ + | $EGREP "$match_pattern_regex" > /dev/null; then + valid_a_lib=yes + fi + ;; + pass_all) + valid_a_lib=yes + ;; + esac + if test "$valid_a_lib" != yes; then + echo + $ECHO "*** Warning: Trying to link with static lib archive $deplib." + echo "*** I have the capability to make that library automatically link in when" + echo "*** you link to this library. But I can only do this if you have a" + echo "*** shared version of the library, which you do not appear to have" + echo "*** because the file extensions .$libext of this argument makes me believe" + echo "*** that it is just a static archive that I should not use here." + else + echo + $ECHO "*** Warning: Linking the shared library $output against the" + $ECHO "*** static library $deplib is not portable!" + deplibs="$deplib $deplibs" + fi + ;; + esac + continue + ;; + prog) + if test "$pass" != link; then + deplibs="$deplib $deplibs" + else + compile_deplibs="$deplib $compile_deplibs" + finalize_deplibs="$deplib $finalize_deplibs" + fi + continue + ;; + esac # linkmode + ;; # *.$libext + *.lo | *.$objext) + if test "$pass" = conv; then + deplibs="$deplib $deplibs" + elif test "$linkmode" = prog; then + if test "$pass" = dlpreopen || test "$dlopen_support" != yes || test "$build_libtool_libs" = no; then + # If there is no dlopen support or we're linking statically, + # we need to preload. + func_append newdlprefiles " $deplib" + compile_deplibs="$deplib $compile_deplibs" + finalize_deplibs="$deplib $finalize_deplibs" + else + func_append newdlfiles " $deplib" + fi + fi + continue + ;; + %DEPLIBS%) + alldeplibs=yes + continue + ;; + esac # case $deplib + + if test "$found" = yes || test -f "$lib"; then : + else + func_fatal_error "cannot find the library \`$lib' or unhandled argument \`$deplib'" + fi + + # Check to see that this really is a libtool archive. + func_lalib_unsafe_p "$lib" \ + || func_fatal_error "\`$lib' is not a valid libtool archive" + + func_dirname "$lib" "" "." + ladir="$func_dirname_result" + + dlname= + dlopen= + dlpreopen= + libdir= + library_names= + old_library= + inherited_linker_flags= + # If the library was installed with an old release of libtool, + # it will not redefine variables installed, or shouldnotlink + installed=yes + shouldnotlink=no + avoidtemprpath= + + + # Read the .la file + func_source "$lib" + + # Convert "-framework foo" to "foo.ltframework" + if test -n "$inherited_linker_flags"; then + tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g'` + for tmp_inherited_linker_flag in $tmp_inherited_linker_flags; do + case " $new_inherited_linker_flags " in + *" $tmp_inherited_linker_flag "*) ;; + *) func_append new_inherited_linker_flags " $tmp_inherited_linker_flag";; + esac + done + fi + dependency_libs=`$ECHO " $dependency_libs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` + if test "$linkmode,$pass" = "lib,link" || + test "$linkmode,$pass" = "prog,scan" || + { test "$linkmode" != prog && test "$linkmode" != lib; }; then + test -n "$dlopen" && func_append dlfiles " $dlopen" + test -n "$dlpreopen" && func_append dlprefiles " $dlpreopen" + fi + + if test "$pass" = conv; then + # Only check for convenience libraries + deplibs="$lib $deplibs" + if test -z "$libdir"; then + if test -z "$old_library"; then + func_fatal_error "cannot find name of link library for \`$lib'" + fi + # It is a libtool convenience library, so add in its objects. + func_append convenience " $ladir/$objdir/$old_library" + func_append old_convenience " $ladir/$objdir/$old_library" + tmp_libs= + for deplib in $dependency_libs; do + deplibs="$deplib $deplibs" + if $opt_preserve_dup_deps ; then + case "$tmp_libs " in + *" $deplib "*) func_append specialdeplibs " $deplib" ;; + esac + fi + func_append tmp_libs " $deplib" + done + elif test "$linkmode" != prog && test "$linkmode" != lib; then + func_fatal_error "\`$lib' is not a convenience library" + fi + continue + fi # $pass = conv + + + # Get the name of the library we link against. + linklib= + if test -n "$old_library" && + { test "$prefer_static_libs" = yes || + test "$prefer_static_libs,$installed" = "built,no"; }; then + linklib=$old_library + else + for l in $old_library $library_names; do + linklib="$l" + done + fi + if test -z "$linklib"; then + func_fatal_error "cannot find name of link library for \`$lib'" + fi + + # This library was specified with -dlopen. + if test "$pass" = dlopen; then + if test -z "$libdir"; then + func_fatal_error "cannot -dlopen a convenience library: \`$lib'" + fi + if test -z "$dlname" || + test "$dlopen_support" != yes || + test "$build_libtool_libs" = no; then + # If there is no dlname, no dlopen support or we're linking + # statically, we need to preload. We also need to preload any + # dependent libraries so libltdl's deplib preloader doesn't + # bomb out in the load deplibs phase. + func_append dlprefiles " $lib $dependency_libs" + else + func_append newdlfiles " $lib" + fi + continue + fi # $pass = dlopen + + # We need an absolute path. + case $ladir in + [\\/]* | [A-Za-z]:[\\/]*) abs_ladir="$ladir" ;; + *) + abs_ladir=`cd "$ladir" && pwd` + if test -z "$abs_ladir"; then + func_warning "cannot determine absolute directory name of \`$ladir'" + func_warning "passing it literally to the linker, although it might fail" + abs_ladir="$ladir" + fi + ;; + esac + func_basename "$lib" + laname="$func_basename_result" + + # Find the relevant object directory and library name. + if test "X$installed" = Xyes; then + if test ! -f "$lt_sysroot$libdir/$linklib" && test -f "$abs_ladir/$linklib"; then + func_warning "library \`$lib' was moved." + dir="$ladir" + absdir="$abs_ladir" + libdir="$abs_ladir" + else + dir="$lt_sysroot$libdir" + absdir="$lt_sysroot$libdir" + fi + test "X$hardcode_automatic" = Xyes && avoidtemprpath=yes + else + if test ! -f "$ladir/$objdir/$linklib" && test -f "$abs_ladir/$linklib"; then + dir="$ladir" + absdir="$abs_ladir" + # Remove this search path later + func_append notinst_path " $abs_ladir" + else + dir="$ladir/$objdir" + absdir="$abs_ladir/$objdir" + # Remove this search path later + func_append notinst_path " $abs_ladir" + fi + fi # $installed = yes + func_stripname 'lib' '.la' "$laname" + name=$func_stripname_result + + # This library was specified with -dlpreopen. + if test "$pass" = dlpreopen; then + if test -z "$libdir" && test "$linkmode" = prog; then + func_fatal_error "only libraries may -dlpreopen a convenience library: \`$lib'" + fi + case "$host" in + # special handling for platforms with PE-DLLs. + *cygwin* | *mingw* | *cegcc* ) + # Linker will automatically link against shared library if both + # static and shared are present. Therefore, ensure we extract + # symbols from the import library if a shared library is present + # (otherwise, the dlopen module name will be incorrect). We do + # this by putting the import library name into $newdlprefiles. + # We recover the dlopen module name by 'saving' the la file + # name in a special purpose variable, and (later) extracting the + # dlname from the la file. + if test -n "$dlname"; then + func_tr_sh "$dir/$linklib" + eval "libfile_$func_tr_sh_result=\$abs_ladir/\$laname" + func_append newdlprefiles " $dir/$linklib" + else + func_append newdlprefiles " $dir/$old_library" + # Keep a list of preopened convenience libraries to check + # that they are being used correctly in the link pass. + test -z "$libdir" && \ + func_append dlpreconveniencelibs " $dir/$old_library" + fi + ;; + * ) + # Prefer using a static library (so that no silly _DYNAMIC symbols + # are required to link). + if test -n "$old_library"; then + func_append newdlprefiles " $dir/$old_library" + # Keep a list of preopened convenience libraries to check + # that they are being used correctly in the link pass. + test -z "$libdir" && \ + func_append dlpreconveniencelibs " $dir/$old_library" + # Otherwise, use the dlname, so that lt_dlopen finds it. + elif test -n "$dlname"; then + func_append newdlprefiles " $dir/$dlname" + else + func_append newdlprefiles " $dir/$linklib" + fi + ;; + esac + fi # $pass = dlpreopen + + if test -z "$libdir"; then + # Link the convenience library + if test "$linkmode" = lib; then + deplibs="$dir/$old_library $deplibs" + elif test "$linkmode,$pass" = "prog,link"; then + compile_deplibs="$dir/$old_library $compile_deplibs" + finalize_deplibs="$dir/$old_library $finalize_deplibs" + else + deplibs="$lib $deplibs" # used for prog,scan pass + fi + continue + fi + + + if test "$linkmode" = prog && test "$pass" != link; then + func_append newlib_search_path " $ladir" + deplibs="$lib $deplibs" + + linkalldeplibs=no + if test "$link_all_deplibs" != no || test -z "$library_names" || + test "$build_libtool_libs" = no; then + linkalldeplibs=yes + fi + + tmp_libs= + for deplib in $dependency_libs; do + case $deplib in + -L*) func_stripname '-L' '' "$deplib" + func_resolve_sysroot "$func_stripname_result" + func_append newlib_search_path " $func_resolve_sysroot_result" + ;; + esac + # Need to link against all dependency_libs? + if test "$linkalldeplibs" = yes; then + deplibs="$deplib $deplibs" + else + # Need to hardcode shared library paths + # or/and link against static libraries + newdependency_libs="$deplib $newdependency_libs" + fi + if $opt_preserve_dup_deps ; then + case "$tmp_libs " in + *" $deplib "*) func_append specialdeplibs " $deplib" ;; + esac + fi + func_append tmp_libs " $deplib" + done # for deplib + continue + fi # $linkmode = prog... + + if test "$linkmode,$pass" = "prog,link"; then + if test -n "$library_names" && + { { test "$prefer_static_libs" = no || + test "$prefer_static_libs,$installed" = "built,yes"; } || + test -z "$old_library"; }; then + # We need to hardcode the library path + if test -n "$shlibpath_var" && test -z "$avoidtemprpath" ; then + # Make sure the rpath contains only unique directories. + case "$temp_rpath:" in + *"$absdir:"*) ;; + *) func_append temp_rpath "$absdir:" ;; + esac + fi + + # Hardcode the library path. + # Skip directories that are in the system default run-time + # search path. + case " $sys_lib_dlsearch_path " in + *" $absdir "*) ;; + *) + case "$compile_rpath " in + *" $absdir "*) ;; + *) func_append compile_rpath " $absdir" ;; + esac + ;; + esac + case " $sys_lib_dlsearch_path " in + *" $libdir "*) ;; + *) + case "$finalize_rpath " in + *" $libdir "*) ;; + *) func_append finalize_rpath " $libdir" ;; + esac + ;; + esac + fi # $linkmode,$pass = prog,link... + + if test "$alldeplibs" = yes && + { test "$deplibs_check_method" = pass_all || + { test "$build_libtool_libs" = yes && + test -n "$library_names"; }; }; then + # We only need to search for static libraries + continue + fi + fi + + link_static=no # Whether the deplib will be linked statically + use_static_libs=$prefer_static_libs + if test "$use_static_libs" = built && test "$installed" = yes; then + use_static_libs=no + fi + if test -n "$library_names" && + { test "$use_static_libs" = no || test -z "$old_library"; }; then + case $host in + *cygwin* | *mingw* | *cegcc*) + # No point in relinking DLLs because paths are not encoded + func_append notinst_deplibs " $lib" + need_relink=no + ;; + *) + if test "$installed" = no; then + func_append notinst_deplibs " $lib" + need_relink=yes + fi + ;; + esac + # This is a shared library + + # Warn about portability, can't link against -module's on some + # systems (darwin). Don't bleat about dlopened modules though! + dlopenmodule="" + for dlpremoduletest in $dlprefiles; do + if test "X$dlpremoduletest" = "X$lib"; then + dlopenmodule="$dlpremoduletest" + break + fi + done + if test -z "$dlopenmodule" && test "$shouldnotlink" = yes && test "$pass" = link; then + echo + if test "$linkmode" = prog; then + $ECHO "*** Warning: Linking the executable $output against the loadable module" + else + $ECHO "*** Warning: Linking the shared library $output against the loadable module" + fi + $ECHO "*** $linklib is not portable!" + fi + if test "$linkmode" = lib && + test "$hardcode_into_libs" = yes; then + # Hardcode the library path. + # Skip directories that are in the system default run-time + # search path. + case " $sys_lib_dlsearch_path " in + *" $absdir "*) ;; + *) + case "$compile_rpath " in + *" $absdir "*) ;; + *) func_append compile_rpath " $absdir" ;; + esac + ;; + esac + case " $sys_lib_dlsearch_path " in + *" $libdir "*) ;; + *) + case "$finalize_rpath " in + *" $libdir "*) ;; + *) func_append finalize_rpath " $libdir" ;; + esac + ;; + esac + fi + + if test -n "$old_archive_from_expsyms_cmds"; then + # figure out the soname + set dummy $library_names + shift + realname="$1" + shift + libname=`eval "\\$ECHO \"$libname_spec\""` + # use dlname if we got it. it's perfectly good, no? + if test -n "$dlname"; then + soname="$dlname" + elif test -n "$soname_spec"; then + # bleh windows + case $host in + *cygwin* | mingw* | *cegcc*) + func_arith $current - $age + major=$func_arith_result + versuffix="-$major" + ;; + esac + eval soname=\"$soname_spec\" + else + soname="$realname" + fi + + # Make a new name for the extract_expsyms_cmds to use + soroot="$soname" + func_basename "$soroot" + soname="$func_basename_result" + func_stripname 'lib' '.dll' "$soname" + newlib=libimp-$func_stripname_result.a + + # If the library has no export list, then create one now + if test -f "$output_objdir/$soname-def"; then : + else + func_verbose "extracting exported symbol list from \`$soname'" + func_execute_cmds "$extract_expsyms_cmds" 'exit $?' + fi + + # Create $newlib + if test -f "$output_objdir/$newlib"; then :; else + func_verbose "generating import library for \`$soname'" + func_execute_cmds "$old_archive_from_expsyms_cmds" 'exit $?' + fi + # make sure the library variables are pointing to the new library + dir=$output_objdir + linklib=$newlib + fi # test -n "$old_archive_from_expsyms_cmds" + + if test "$linkmode" = prog || test "$opt_mode" != relink; then + add_shlibpath= + add_dir= + add= + lib_linked=yes + case $hardcode_action in + immediate | unsupported) + if test "$hardcode_direct" = no; then + add="$dir/$linklib" + case $host in + *-*-sco3.2v5.0.[024]*) add_dir="-L$dir" ;; + *-*-sysv4*uw2*) add_dir="-L$dir" ;; + *-*-sysv5OpenUNIX* | *-*-sysv5UnixWare7.[01].[10]* | \ + *-*-unixware7*) add_dir="-L$dir" ;; + *-*-darwin* ) + # if the lib is a (non-dlopened) module then we can not + # link against it, someone is ignoring the earlier warnings + if /usr/bin/file -L $add 2> /dev/null | + $GREP ": [^:]* bundle" >/dev/null ; then + if test "X$dlopenmodule" != "X$lib"; then + $ECHO "*** Warning: lib $linklib is a module, not a shared library" + if test -z "$old_library" ; then + echo + echo "*** And there doesn't seem to be a static archive available" + echo "*** The link will probably fail, sorry" + else + add="$dir/$old_library" + fi + elif test -n "$old_library"; then + add="$dir/$old_library" + fi + fi + esac + elif test "$hardcode_minus_L" = no; then + case $host in + *-*-sunos*) add_shlibpath="$dir" ;; + esac + add_dir="-L$dir" + add="-l$name" + elif test "$hardcode_shlibpath_var" = no; then + add_shlibpath="$dir" + add="-l$name" + else + lib_linked=no + fi + ;; + relink) + if test "$hardcode_direct" = yes && + test "$hardcode_direct_absolute" = no; then + add="$dir/$linklib" + elif test "$hardcode_minus_L" = yes; then + add_dir="-L$absdir" + # Try looking first in the location we're being installed to. + if test -n "$inst_prefix_dir"; then + case $libdir in + [\\/]*) + func_append add_dir " -L$inst_prefix_dir$libdir" + ;; + esac + fi + add="-l$name" + elif test "$hardcode_shlibpath_var" = yes; then + add_shlibpath="$dir" + add="-l$name" + else + lib_linked=no + fi + ;; + *) lib_linked=no ;; + esac + + if test "$lib_linked" != yes; then + func_fatal_configuration "unsupported hardcode properties" + fi + + if test -n "$add_shlibpath"; then + case :$compile_shlibpath: in + *":$add_shlibpath:"*) ;; + *) func_append compile_shlibpath "$add_shlibpath:" ;; + esac + fi + if test "$linkmode" = prog; then + test -n "$add_dir" && compile_deplibs="$add_dir $compile_deplibs" + test -n "$add" && compile_deplibs="$add $compile_deplibs" + else + test -n "$add_dir" && deplibs="$add_dir $deplibs" + test -n "$add" && deplibs="$add $deplibs" + if test "$hardcode_direct" != yes && + test "$hardcode_minus_L" != yes && + test "$hardcode_shlibpath_var" = yes; then + case :$finalize_shlibpath: in + *":$libdir:"*) ;; + *) func_append finalize_shlibpath "$libdir:" ;; + esac + fi + fi + fi + + if test "$linkmode" = prog || test "$opt_mode" = relink; then + add_shlibpath= + add_dir= + add= + # Finalize command for both is simple: just hardcode it. + if test "$hardcode_direct" = yes && + test "$hardcode_direct_absolute" = no; then + add="$libdir/$linklib" + elif test "$hardcode_minus_L" = yes; then + add_dir="-L$libdir" + add="-l$name" + elif test "$hardcode_shlibpath_var" = yes; then + case :$finalize_shlibpath: in + *":$libdir:"*) ;; + *) func_append finalize_shlibpath "$libdir:" ;; + esac + add="-l$name" + elif test "$hardcode_automatic" = yes; then + if test -n "$inst_prefix_dir" && + test -f "$inst_prefix_dir$libdir/$linklib" ; then + add="$inst_prefix_dir$libdir/$linklib" + else + add="$libdir/$linklib" + fi + else + # We cannot seem to hardcode it, guess we'll fake it. + add_dir="-L$libdir" + # Try looking first in the location we're being installed to. + if test -n "$inst_prefix_dir"; then + case $libdir in + [\\/]*) + func_append add_dir " -L$inst_prefix_dir$libdir" + ;; + esac + fi + add="-l$name" + fi + + if test "$linkmode" = prog; then + test -n "$add_dir" && finalize_deplibs="$add_dir $finalize_deplibs" + test -n "$add" && finalize_deplibs="$add $finalize_deplibs" + else + test -n "$add_dir" && deplibs="$add_dir $deplibs" + test -n "$add" && deplibs="$add $deplibs" + fi + fi + elif test "$linkmode" = prog; then + # Here we assume that one of hardcode_direct or hardcode_minus_L + # is not unsupported. This is valid on all known static and + # shared platforms. + if test "$hardcode_direct" != unsupported; then + test -n "$old_library" && linklib="$old_library" + compile_deplibs="$dir/$linklib $compile_deplibs" + finalize_deplibs="$dir/$linklib $finalize_deplibs" + else + compile_deplibs="-l$name -L$dir $compile_deplibs" + finalize_deplibs="-l$name -L$dir $finalize_deplibs" + fi + elif test "$build_libtool_libs" = yes; then + # Not a shared library + if test "$deplibs_check_method" != pass_all; then + # We're trying link a shared library against a static one + # but the system doesn't support it. + + # Just print a warning and add the library to dependency_libs so + # that the program can be linked against the static library. + echo + $ECHO "*** Warning: This system can not link to static lib archive $lib." + echo "*** I have the capability to make that library automatically link in when" + echo "*** you link to this library. But I can only do this if you have a" + echo "*** shared version of the library, which you do not appear to have." + if test "$module" = yes; then + echo "*** But as you try to build a module library, libtool will still create " + echo "*** a static module, that should work as long as the dlopening application" + echo "*** is linked with the -dlopen flag to resolve symbols at runtime." + if test -z "$global_symbol_pipe"; then + echo + echo "*** However, this would only work if libtool was able to extract symbol" + echo "*** lists from a program, using \`nm' or equivalent, but libtool could" + echo "*** not find such a program. So, this module is probably useless." + echo "*** \`nm' from GNU binutils and a full rebuild may help." + fi + if test "$build_old_libs" = no; then + build_libtool_libs=module + build_old_libs=yes + else + build_libtool_libs=no + fi + fi + else + deplibs="$dir/$old_library $deplibs" + link_static=yes + fi + fi # link shared/static library? + + if test "$linkmode" = lib; then + if test -n "$dependency_libs" && + { test "$hardcode_into_libs" != yes || + test "$build_old_libs" = yes || + test "$link_static" = yes; }; then + # Extract -R from dependency_libs + temp_deplibs= + for libdir in $dependency_libs; do + case $libdir in + -R*) func_stripname '-R' '' "$libdir" + temp_xrpath=$func_stripname_result + case " $xrpath " in + *" $temp_xrpath "*) ;; + *) func_append xrpath " $temp_xrpath";; + esac;; + *) func_append temp_deplibs " $libdir";; + esac + done + dependency_libs="$temp_deplibs" + fi + + func_append newlib_search_path " $absdir" + # Link against this library + test "$link_static" = no && newdependency_libs="$abs_ladir/$laname $newdependency_libs" + # ... and its dependency_libs + tmp_libs= + for deplib in $dependency_libs; do + newdependency_libs="$deplib $newdependency_libs" + case $deplib in + -L*) func_stripname '-L' '' "$deplib" + func_resolve_sysroot "$func_stripname_result";; + *) func_resolve_sysroot "$deplib" ;; + esac + if $opt_preserve_dup_deps ; then + case "$tmp_libs " in + *" $func_resolve_sysroot_result "*) + func_append specialdeplibs " $func_resolve_sysroot_result" ;; + esac + fi + func_append tmp_libs " $func_resolve_sysroot_result" + done + + if test "$link_all_deplibs" != no; then + # Add the search paths of all dependency libraries + for deplib in $dependency_libs; do + path= + case $deplib in + -L*) path="$deplib" ;; + *.la) + func_resolve_sysroot "$deplib" + deplib=$func_resolve_sysroot_result + func_dirname "$deplib" "" "." + dir=$func_dirname_result + # We need an absolute path. + case $dir in + [\\/]* | [A-Za-z]:[\\/]*) absdir="$dir" ;; + *) + absdir=`cd "$dir" && pwd` + if test -z "$absdir"; then + func_warning "cannot determine absolute directory name of \`$dir'" + absdir="$dir" + fi + ;; + esac + if $GREP "^installed=no" $deplib > /dev/null; then + case $host in + *-*-darwin*) + depdepl= + eval deplibrary_names=`${SED} -n -e 's/^library_names=\(.*\)$/\1/p' $deplib` + if test -n "$deplibrary_names" ; then + for tmp in $deplibrary_names ; do + depdepl=$tmp + done + if test -f "$absdir/$objdir/$depdepl" ; then + depdepl="$absdir/$objdir/$depdepl" + darwin_install_name=`${OTOOL} -L $depdepl | awk '{if (NR == 2) {print $1;exit}}'` + if test -z "$darwin_install_name"; then + darwin_install_name=`${OTOOL64} -L $depdepl | awk '{if (NR == 2) {print $1;exit}}'` + fi + func_append compiler_flags " ${wl}-dylib_file ${wl}${darwin_install_name}:${depdepl}" + func_append linker_flags " -dylib_file ${darwin_install_name}:${depdepl}" + path= + fi + fi + ;; + *) + path="-L$absdir/$objdir" + ;; + esac + else + eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $deplib` + test -z "$libdir" && \ + func_fatal_error "\`$deplib' is not a valid libtool archive" + test "$absdir" != "$libdir" && \ + func_warning "\`$deplib' seems to be moved" + + path="-L$absdir" + fi + ;; + esac + case " $deplibs " in + *" $path "*) ;; + *) deplibs="$path $deplibs" ;; + esac + done + fi # link_all_deplibs != no + fi # linkmode = lib + done # for deplib in $libs + if test "$pass" = link; then + if test "$linkmode" = "prog"; then + compile_deplibs="$new_inherited_linker_flags $compile_deplibs" + finalize_deplibs="$new_inherited_linker_flags $finalize_deplibs" + else + compiler_flags="$compiler_flags "`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` + fi + fi + dependency_libs="$newdependency_libs" + if test "$pass" = dlpreopen; then + # Link the dlpreopened libraries before other libraries + for deplib in $save_deplibs; do + deplibs="$deplib $deplibs" + done + fi + if test "$pass" != dlopen; then + if test "$pass" != conv; then + # Make sure lib_search_path contains only unique directories. + lib_search_path= + for dir in $newlib_search_path; do + case "$lib_search_path " in + *" $dir "*) ;; + *) func_append lib_search_path " $dir" ;; + esac + done + newlib_search_path= + fi + + if test "$linkmode,$pass" != "prog,link"; then + vars="deplibs" + else + vars="compile_deplibs finalize_deplibs" + fi + for var in $vars dependency_libs; do + # Add libraries to $var in reverse order + eval tmp_libs=\"\$$var\" + new_libs= + for deplib in $tmp_libs; do + # FIXME: Pedantically, this is the right thing to do, so + # that some nasty dependency loop isn't accidentally + # broken: + #new_libs="$deplib $new_libs" + # Pragmatically, this seems to cause very few problems in + # practice: + case $deplib in + -L*) new_libs="$deplib $new_libs" ;; + -R*) ;; + *) + # And here is the reason: when a library appears more + # than once as an explicit dependence of a library, or + # is implicitly linked in more than once by the + # compiler, it is considered special, and multiple + # occurrences thereof are not removed. Compare this + # with having the same library being listed as a + # dependency of multiple other libraries: in this case, + # we know (pedantically, we assume) the library does not + # need to be listed more than once, so we keep only the + # last copy. This is not always right, but it is rare + # enough that we require users that really mean to play + # such unportable linking tricks to link the library + # using -Wl,-lname, so that libtool does not consider it + # for duplicate removal. + case " $specialdeplibs " in + *" $deplib "*) new_libs="$deplib $new_libs" ;; + *) + case " $new_libs " in + *" $deplib "*) ;; + *) new_libs="$deplib $new_libs" ;; + esac + ;; + esac + ;; + esac + done + tmp_libs= + for deplib in $new_libs; do + case $deplib in + -L*) + case " $tmp_libs " in + *" $deplib "*) ;; + *) func_append tmp_libs " $deplib" ;; + esac + ;; + *) func_append tmp_libs " $deplib" ;; + esac + done + eval $var=\"$tmp_libs\" + done # for var + fi + # Last step: remove runtime libs from dependency_libs + # (they stay in deplibs) + tmp_libs= + for i in $dependency_libs ; do + case " $predeps $postdeps $compiler_lib_search_path " in + *" $i "*) + i="" + ;; + esac + if test -n "$i" ; then + func_append tmp_libs " $i" + fi + done + dependency_libs=$tmp_libs + done # for pass + if test "$linkmode" = prog; then + dlfiles="$newdlfiles" + fi + if test "$linkmode" = prog || test "$linkmode" = lib; then + dlprefiles="$newdlprefiles" + fi + + case $linkmode in + oldlib) + if test -n "$dlfiles$dlprefiles" || test "$dlself" != no; then + func_warning "\`-dlopen' is ignored for archives" + fi + + case " $deplibs" in + *\ -l* | *\ -L*) + func_warning "\`-l' and \`-L' are ignored for archives" ;; + esac + + test -n "$rpath" && \ + func_warning "\`-rpath' is ignored for archives" + + test -n "$xrpath" && \ + func_warning "\`-R' is ignored for archives" + + test -n "$vinfo" && \ + func_warning "\`-version-info/-version-number' is ignored for archives" + + test -n "$release" && \ + func_warning "\`-release' is ignored for archives" + + test -n "$export_symbols$export_symbols_regex" && \ + func_warning "\`-export-symbols' is ignored for archives" + + # Now set the variables for building old libraries. + build_libtool_libs=no + oldlibs="$output" + func_append objs "$old_deplibs" + ;; + + lib) + # Make sure we only generate libraries of the form `libNAME.la'. + case $outputname in + lib*) + func_stripname 'lib' '.la' "$outputname" + name=$func_stripname_result + eval shared_ext=\"$shrext_cmds\" + eval libname=\"$libname_spec\" + ;; + *) + test "$module" = no && \ + func_fatal_help "libtool library \`$output' must begin with \`lib'" + + if test "$need_lib_prefix" != no; then + # Add the "lib" prefix for modules if required + func_stripname '' '.la' "$outputname" + name=$func_stripname_result + eval shared_ext=\"$shrext_cmds\" + eval libname=\"$libname_spec\" + else + func_stripname '' '.la' "$outputname" + libname=$func_stripname_result + fi + ;; + esac + + if test -n "$objs"; then + if test "$deplibs_check_method" != pass_all; then + func_fatal_error "cannot build libtool library \`$output' from non-libtool objects on this host:$objs" + else + echo + $ECHO "*** Warning: Linking the shared library $output against the non-libtool" + $ECHO "*** objects $objs is not portable!" + func_append libobjs " $objs" + fi + fi + + test "$dlself" != no && \ + func_warning "\`-dlopen self' is ignored for libtool libraries" + + set dummy $rpath + shift + test "$#" -gt 1 && \ + func_warning "ignoring multiple \`-rpath's for a libtool library" + + install_libdir="$1" + + oldlibs= + if test -z "$rpath"; then + if test "$build_libtool_libs" = yes; then + # Building a libtool convenience library. + # Some compilers have problems with a `.al' extension so + # convenience libraries should have the same extension an + # archive normally would. + oldlibs="$output_objdir/$libname.$libext $oldlibs" + build_libtool_libs=convenience + build_old_libs=yes + fi + + test -n "$vinfo" && \ + func_warning "\`-version-info/-version-number' is ignored for convenience libraries" + + test -n "$release" && \ + func_warning "\`-release' is ignored for convenience libraries" + else + + # Parse the version information argument. + save_ifs="$IFS"; IFS=':' + set dummy $vinfo 0 0 0 + shift + IFS="$save_ifs" + + test -n "$7" && \ + func_fatal_help "too many parameters to \`-version-info'" + + # convert absolute version numbers to libtool ages + # this retains compatibility with .la files and attempts + # to make the code below a bit more comprehensible + + case $vinfo_number in + yes) + number_major="$1" + number_minor="$2" + number_revision="$3" + # + # There are really only two kinds -- those that + # use the current revision as the major version + # and those that subtract age and use age as + # a minor version. But, then there is irix + # which has an extra 1 added just for fun + # + case $version_type in + # correct linux to gnu/linux during the next big refactor + darwin|linux|osf|windows|none) + func_arith $number_major + $number_minor + current=$func_arith_result + age="$number_minor" + revision="$number_revision" + ;; + freebsd-aout|freebsd-elf|qnx|sunos) + current="$number_major" + revision="$number_minor" + age="0" + ;; + irix|nonstopux) + func_arith $number_major + $number_minor + current=$func_arith_result + age="$number_minor" + revision="$number_minor" + lt_irix_increment=no + ;; + *) + func_fatal_configuration "$modename: unknown library version type \`$version_type'" + ;; + esac + ;; + no) + current="$1" + revision="$2" + age="$3" + ;; + esac + + # Check that each of the things are valid numbers. + case $current in + 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;; + *) + func_error "CURRENT \`$current' must be a nonnegative integer" + func_fatal_error "\`$vinfo' is not valid version information" + ;; + esac + + case $revision in + 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;; + *) + func_error "REVISION \`$revision' must be a nonnegative integer" + func_fatal_error "\`$vinfo' is not valid version information" + ;; + esac + + case $age in + 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;; + *) + func_error "AGE \`$age' must be a nonnegative integer" + func_fatal_error "\`$vinfo' is not valid version information" + ;; + esac + + if test "$age" -gt "$current"; then + func_error "AGE \`$age' is greater than the current interface number \`$current'" + func_fatal_error "\`$vinfo' is not valid version information" + fi + + # Calculate the version variables. + major= + versuffix= + verstring= + case $version_type in + none) ;; + + darwin) + # Like Linux, but with the current version available in + # verstring for coding it into the library header + func_arith $current - $age + major=.$func_arith_result + versuffix="$major.$age.$revision" + # Darwin ld doesn't like 0 for these options... + func_arith $current + 1 + minor_current=$func_arith_result + xlcverstring="${wl}-compatibility_version ${wl}$minor_current ${wl}-current_version ${wl}$minor_current.$revision" + verstring="-compatibility_version $minor_current -current_version $minor_current.$revision" + ;; + + freebsd-aout) + major=".$current" + versuffix=".$current.$revision"; + ;; + + freebsd-elf) + major=".$current" + versuffix=".$current" + ;; + + irix | nonstopux) + if test "X$lt_irix_increment" = "Xno"; then + func_arith $current - $age + else + func_arith $current - $age + 1 + fi + major=$func_arith_result + + case $version_type in + nonstopux) verstring_prefix=nonstopux ;; + *) verstring_prefix=sgi ;; + esac + verstring="$verstring_prefix$major.$revision" + + # Add in all the interfaces that we are compatible with. + loop=$revision + while test "$loop" -ne 0; do + func_arith $revision - $loop + iface=$func_arith_result + func_arith $loop - 1 + loop=$func_arith_result + verstring="$verstring_prefix$major.$iface:$verstring" + done + + # Before this point, $major must not contain `.'. + major=.$major + versuffix="$major.$revision" + ;; + + linux) # correct to gnu/linux during the next big refactor + func_arith $current - $age + major=.$func_arith_result + versuffix="$major.$age.$revision" + ;; + + osf) + func_arith $current - $age + major=.$func_arith_result + versuffix=".$current.$age.$revision" + verstring="$current.$age.$revision" + + # Add in all the interfaces that we are compatible with. + loop=$age + while test "$loop" -ne 0; do + func_arith $current - $loop + iface=$func_arith_result + func_arith $loop - 1 + loop=$func_arith_result + verstring="$verstring:${iface}.0" + done + + # Make executables depend on our current version. + func_append verstring ":${current}.0" + ;; + + qnx) + major=".$current" + versuffix=".$current" + ;; + + sunos) + major=".$current" + versuffix=".$current.$revision" + ;; + + windows) + # Use '-' rather than '.', since we only want one + # extension on DOS 8.3 filesystems. + func_arith $current - $age + major=$func_arith_result + versuffix="-$major" + ;; + + *) + func_fatal_configuration "unknown library version type \`$version_type'" + ;; + esac + + # Clear the version info if we defaulted, and they specified a release. + if test -z "$vinfo" && test -n "$release"; then + major= + case $version_type in + darwin) + # we can't check for "0.0" in archive_cmds due to quoting + # problems, so we reset it completely + verstring= + ;; + *) + verstring="0.0" + ;; + esac + if test "$need_version" = no; then + versuffix= + else + versuffix=".0.0" + fi + fi + + # Remove version info from name if versioning should be avoided + if test "$avoid_version" = yes && test "$need_version" = no; then + major= + versuffix= + verstring="" + fi + + # Check to see if the archive will have undefined symbols. + if test "$allow_undefined" = yes; then + if test "$allow_undefined_flag" = unsupported; then + func_warning "undefined symbols not allowed in $host shared libraries" + build_libtool_libs=no + build_old_libs=yes + fi + else + # Don't allow undefined symbols. + allow_undefined_flag="$no_undefined_flag" + fi + + fi + + func_generate_dlsyms "$libname" "$libname" "yes" + func_append libobjs " $symfileobj" + test "X$libobjs" = "X " && libobjs= + + if test "$opt_mode" != relink; then + # Remove our outputs, but don't remove object files since they + # may have been created when compiling PIC objects. + removelist= + tempremovelist=`$ECHO "$output_objdir/*"` + for p in $tempremovelist; do + case $p in + *.$objext | *.gcno) + ;; + $output_objdir/$outputname | $output_objdir/$libname.* | $output_objdir/${libname}${release}.*) + if test "X$precious_files_regex" != "X"; then + if $ECHO "$p" | $EGREP -e "$precious_files_regex" >/dev/null 2>&1 + then + continue + fi + fi + func_append removelist " $p" + ;; + *) ;; + esac + done + test -n "$removelist" && \ + func_show_eval "${RM}r \$removelist" + fi + + # Now set the variables for building old libraries. + if test "$build_old_libs" = yes && test "$build_libtool_libs" != convenience ; then + func_append oldlibs " $output_objdir/$libname.$libext" + + # Transform .lo files to .o files. + oldobjs="$objs "`$ECHO "$libobjs" | $SP2NL | $SED "/\.${libext}$/d; $lo2o" | $NL2SP` + fi + + # Eliminate all temporary directories. + #for path in $notinst_path; do + # lib_search_path=`$ECHO "$lib_search_path " | $SED "s% $path % %g"` + # deplibs=`$ECHO "$deplibs " | $SED "s% -L$path % %g"` + # dependency_libs=`$ECHO "$dependency_libs " | $SED "s% -L$path % %g"` + #done + + if test -n "$xrpath"; then + # If the user specified any rpath flags, then add them. + temp_xrpath= + for libdir in $xrpath; do + func_replace_sysroot "$libdir" + func_append temp_xrpath " -R$func_replace_sysroot_result" + case "$finalize_rpath " in + *" $libdir "*) ;; + *) func_append finalize_rpath " $libdir" ;; + esac + done + if test "$hardcode_into_libs" != yes || test "$build_old_libs" = yes; then + dependency_libs="$temp_xrpath $dependency_libs" + fi + fi + + # Make sure dlfiles contains only unique files that won't be dlpreopened + old_dlfiles="$dlfiles" + dlfiles= + for lib in $old_dlfiles; do + case " $dlprefiles $dlfiles " in + *" $lib "*) ;; + *) func_append dlfiles " $lib" ;; + esac + done + + # Make sure dlprefiles contains only unique files + old_dlprefiles="$dlprefiles" + dlprefiles= + for lib in $old_dlprefiles; do + case "$dlprefiles " in + *" $lib "*) ;; + *) func_append dlprefiles " $lib" ;; + esac + done + + if test "$build_libtool_libs" = yes; then + if test -n "$rpath"; then + case $host in + *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-*-beos* | *-cegcc* | *-*-haiku*) + # these systems don't actually have a c library (as such)! + ;; + *-*-rhapsody* | *-*-darwin1.[012]) + # Rhapsody C library is in the System framework + func_append deplibs " System.ltframework" + ;; + *-*-netbsd*) + # Don't link with libc until the a.out ld.so is fixed. + ;; + *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*) + # Do not include libc due to us having libc/libc_r. + ;; + *-*-sco3.2v5* | *-*-sco5v6*) + # Causes problems with __ctype + ;; + *-*-sysv4.2uw2* | *-*-sysv5* | *-*-unixware* | *-*-OpenUNIX*) + # Compiler inserts libc in the correct place for threads to work + ;; + *) + # Add libc to deplibs on all other systems if necessary. + if test "$build_libtool_need_lc" = "yes"; then + func_append deplibs " -lc" + fi + ;; + esac + fi + + # Transform deplibs into only deplibs that can be linked in shared. + name_save=$name + libname_save=$libname + release_save=$release + versuffix_save=$versuffix + major_save=$major + # I'm not sure if I'm treating the release correctly. I think + # release should show up in the -l (ie -lgmp5) so we don't want to + # add it in twice. Is that correct? + release="" + versuffix="" + major="" + newdeplibs= + droppeddeps=no + case $deplibs_check_method in + pass_all) + # Don't check for shared/static. Everything works. + # This might be a little naive. We might want to check + # whether the library exists or not. But this is on + # osf3 & osf4 and I'm not really sure... Just + # implementing what was already the behavior. + newdeplibs=$deplibs + ;; + test_compile) + # This code stresses the "libraries are programs" paradigm to its + # limits. Maybe even breaks it. We compile a program, linking it + # against the deplibs as a proxy for the library. Then we can check + # whether they linked in statically or dynamically with ldd. + $opt_dry_run || $RM conftest.c + cat > conftest.c </dev/null` + $nocaseglob + else + potential_libs=`ls $i/$libnameglob[.-]* 2>/dev/null` + fi + for potent_lib in $potential_libs; do + # Follow soft links. + if ls -lLd "$potent_lib" 2>/dev/null | + $GREP " -> " >/dev/null; then + continue + fi + # The statement above tries to avoid entering an + # endless loop below, in case of cyclic links. + # We might still enter an endless loop, since a link + # loop can be closed while we follow links, + # but so what? + potlib="$potent_lib" + while test -h "$potlib" 2>/dev/null; do + potliblink=`ls -ld $potlib | ${SED} 's/.* -> //'` + case $potliblink in + [\\/]* | [A-Za-z]:[\\/]*) potlib="$potliblink";; + *) potlib=`$ECHO "$potlib" | $SED 's,[^/]*$,,'`"$potliblink";; + esac + done + if eval $file_magic_cmd \"\$potlib\" 2>/dev/null | + $SED -e 10q | + $EGREP "$file_magic_regex" > /dev/null; then + func_append newdeplibs " $a_deplib" + a_deplib="" + break 2 + fi + done + done + fi + if test -n "$a_deplib" ; then + droppeddeps=yes + echo + $ECHO "*** Warning: linker path does not have real file for library $a_deplib." + echo "*** I have the capability to make that library automatically link in when" + echo "*** you link to this library. But I can only do this if you have a" + echo "*** shared version of the library, which you do not appear to have" + echo "*** because I did check the linker path looking for a file starting" + if test -z "$potlib" ; then + $ECHO "*** with $libname but no candidates were found. (...for file magic test)" + else + $ECHO "*** with $libname and none of the candidates passed a file format test" + $ECHO "*** using a file magic. Last file checked: $potlib" + fi + fi + ;; + *) + # Add a -L argument. + func_append newdeplibs " $a_deplib" + ;; + esac + done # Gone through all deplibs. + ;; + match_pattern*) + set dummy $deplibs_check_method; shift + match_pattern_regex=`expr "$deplibs_check_method" : "$1 \(.*\)"` + for a_deplib in $deplibs; do + case $a_deplib in + -l*) + func_stripname -l '' "$a_deplib" + name=$func_stripname_result + if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then + case " $predeps $postdeps " in + *" $a_deplib "*) + func_append newdeplibs " $a_deplib" + a_deplib="" + ;; + esac + fi + if test -n "$a_deplib" ; then + libname=`eval "\\$ECHO \"$libname_spec\""` + for i in $lib_search_path $sys_lib_search_path $shlib_search_path; do + potential_libs=`ls $i/$libname[.-]* 2>/dev/null` + for potent_lib in $potential_libs; do + potlib="$potent_lib" # see symlink-check above in file_magic test + if eval "\$ECHO \"$potent_lib\"" 2>/dev/null | $SED 10q | \ + $EGREP "$match_pattern_regex" > /dev/null; then + func_append newdeplibs " $a_deplib" + a_deplib="" + break 2 + fi + done + done + fi + if test -n "$a_deplib" ; then + droppeddeps=yes + echo + $ECHO "*** Warning: linker path does not have real file for library $a_deplib." + echo "*** I have the capability to make that library automatically link in when" + echo "*** you link to this library. But I can only do this if you have a" + echo "*** shared version of the library, which you do not appear to have" + echo "*** because I did check the linker path looking for a file starting" + if test -z "$potlib" ; then + $ECHO "*** with $libname but no candidates were found. (...for regex pattern test)" + else + $ECHO "*** with $libname and none of the candidates passed a file format test" + $ECHO "*** using a regex pattern. Last file checked: $potlib" + fi + fi + ;; + *) + # Add a -L argument. + func_append newdeplibs " $a_deplib" + ;; + esac + done # Gone through all deplibs. + ;; + none | unknown | *) + newdeplibs="" + tmp_deplibs=`$ECHO " $deplibs" | $SED 's/ -lc$//; s/ -[LR][^ ]*//g'` + if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then + for i in $predeps $postdeps ; do + # can't use Xsed below, because $i might contain '/' + tmp_deplibs=`$ECHO " $tmp_deplibs" | $SED "s,$i,,"` + done + fi + case $tmp_deplibs in + *[!\ \ ]*) + echo + if test "X$deplibs_check_method" = "Xnone"; then + echo "*** Warning: inter-library dependencies are not supported in this platform." + else + echo "*** Warning: inter-library dependencies are not known to be supported." + fi + echo "*** All declared inter-library dependencies are being dropped." + droppeddeps=yes + ;; + esac + ;; + esac + versuffix=$versuffix_save + major=$major_save + release=$release_save + libname=$libname_save + name=$name_save + + case $host in + *-*-rhapsody* | *-*-darwin1.[012]) + # On Rhapsody replace the C library with the System framework + newdeplibs=`$ECHO " $newdeplibs" | $SED 's/ -lc / System.ltframework /'` + ;; + esac + + if test "$droppeddeps" = yes; then + if test "$module" = yes; then + echo + echo "*** Warning: libtool could not satisfy all declared inter-library" + $ECHO "*** dependencies of module $libname. Therefore, libtool will create" + echo "*** a static module, that should work as long as the dlopening" + echo "*** application is linked with the -dlopen flag." + if test -z "$global_symbol_pipe"; then + echo + echo "*** However, this would only work if libtool was able to extract symbol" + echo "*** lists from a program, using \`nm' or equivalent, but libtool could" + echo "*** not find such a program. So, this module is probably useless." + echo "*** \`nm' from GNU binutils and a full rebuild may help." + fi + if test "$build_old_libs" = no; then + oldlibs="$output_objdir/$libname.$libext" + build_libtool_libs=module + build_old_libs=yes + else + build_libtool_libs=no + fi + else + echo "*** The inter-library dependencies that have been dropped here will be" + echo "*** automatically added whenever a program is linked with this library" + echo "*** or is declared to -dlopen it." + + if test "$allow_undefined" = no; then + echo + echo "*** Since this library must not contain undefined symbols," + echo "*** because either the platform does not support them or" + echo "*** it was explicitly requested with -no-undefined," + echo "*** libtool will only create a static version of it." + if test "$build_old_libs" = no; then + oldlibs="$output_objdir/$libname.$libext" + build_libtool_libs=module + build_old_libs=yes + else + build_libtool_libs=no + fi + fi + fi + fi + # Done checking deplibs! + deplibs=$newdeplibs + fi + # Time to change all our "foo.ltframework" stuff back to "-framework foo" + case $host in + *-*-darwin*) + newdeplibs=`$ECHO " $newdeplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` + new_inherited_linker_flags=`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` + deplibs=`$ECHO " $deplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` + ;; + esac + + # move library search paths that coincide with paths to not yet + # installed libraries to the beginning of the library search list + new_libs= + for path in $notinst_path; do + case " $new_libs " in + *" -L$path/$objdir "*) ;; + *) + case " $deplibs " in + *" -L$path/$objdir "*) + func_append new_libs " -L$path/$objdir" ;; + esac + ;; + esac + done + for deplib in $deplibs; do + case $deplib in + -L*) + case " $new_libs " in + *" $deplib "*) ;; + *) func_append new_libs " $deplib" ;; + esac + ;; + *) func_append new_libs " $deplib" ;; + esac + done + deplibs="$new_libs" + + # All the library-specific variables (install_libdir is set above). + library_names= + old_library= + dlname= + + # Test again, we may have decided not to build it any more + if test "$build_libtool_libs" = yes; then + # Remove ${wl} instances when linking with ld. + # FIXME: should test the right _cmds variable. + case $archive_cmds in + *\$LD\ *) wl= ;; + esac + if test "$hardcode_into_libs" = yes; then + # Hardcode the library paths + hardcode_libdirs= + dep_rpath= + rpath="$finalize_rpath" + test "$opt_mode" != relink && rpath="$compile_rpath$rpath" + for libdir in $rpath; do + if test -n "$hardcode_libdir_flag_spec"; then + if test -n "$hardcode_libdir_separator"; then + func_replace_sysroot "$libdir" + libdir=$func_replace_sysroot_result + if test -z "$hardcode_libdirs"; then + hardcode_libdirs="$libdir" + else + # Just accumulate the unique libdirs. + case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in + *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*) + ;; + *) + func_append hardcode_libdirs "$hardcode_libdir_separator$libdir" + ;; + esac + fi + else + eval flag=\"$hardcode_libdir_flag_spec\" + func_append dep_rpath " $flag" + fi + elif test -n "$runpath_var"; then + case "$perm_rpath " in + *" $libdir "*) ;; + *) func_append perm_rpath " $libdir" ;; + esac + fi + done + # Substitute the hardcoded libdirs into the rpath. + if test -n "$hardcode_libdir_separator" && + test -n "$hardcode_libdirs"; then + libdir="$hardcode_libdirs" + eval "dep_rpath=\"$hardcode_libdir_flag_spec\"" + fi + if test -n "$runpath_var" && test -n "$perm_rpath"; then + # We should set the runpath_var. + rpath= + for dir in $perm_rpath; do + func_append rpath "$dir:" + done + eval "$runpath_var='$rpath\$$runpath_var'; export $runpath_var" + fi + test -n "$dep_rpath" && deplibs="$dep_rpath $deplibs" + fi + + shlibpath="$finalize_shlibpath" + test "$opt_mode" != relink && shlibpath="$compile_shlibpath$shlibpath" + if test -n "$shlibpath"; then + eval "$shlibpath_var='$shlibpath\$$shlibpath_var'; export $shlibpath_var" + fi + + # Get the real and link names of the library. + eval shared_ext=\"$shrext_cmds\" + eval library_names=\"$library_names_spec\" + set dummy $library_names + shift + realname="$1" + shift + + if test -n "$soname_spec"; then + eval soname=\"$soname_spec\" + else + soname="$realname" + fi + if test -z "$dlname"; then + dlname=$soname + fi + + lib="$output_objdir/$realname" + linknames= + for link + do + func_append linknames " $link" + done + + # Use standard objects if they are pic + test -z "$pic_flag" && libobjs=`$ECHO "$libobjs" | $SP2NL | $SED "$lo2o" | $NL2SP` + test "X$libobjs" = "X " && libobjs= + + delfiles= + if test -n "$export_symbols" && test -n "$include_expsyms"; then + $opt_dry_run || cp "$export_symbols" "$output_objdir/$libname.uexp" + export_symbols="$output_objdir/$libname.uexp" + func_append delfiles " $export_symbols" + fi + + orig_export_symbols= + case $host_os in + cygwin* | mingw* | cegcc*) + if test -n "$export_symbols" && test -z "$export_symbols_regex"; then + # exporting using user supplied symfile + if test "x`$SED 1q $export_symbols`" != xEXPORTS; then + # and it's NOT already a .def file. Must figure out + # which of the given symbols are data symbols and tag + # them as such. So, trigger use of export_symbols_cmds. + # export_symbols gets reassigned inside the "prepare + # the list of exported symbols" if statement, so the + # include_expsyms logic still works. + orig_export_symbols="$export_symbols" + export_symbols= + always_export_symbols=yes + fi + fi + ;; + esac + + # Prepare the list of exported symbols + if test -z "$export_symbols"; then + if test "$always_export_symbols" = yes || test -n "$export_symbols_regex"; then + func_verbose "generating symbol list for \`$libname.la'" + export_symbols="$output_objdir/$libname.exp" + $opt_dry_run || $RM $export_symbols + cmds=$export_symbols_cmds + save_ifs="$IFS"; IFS='~' + for cmd1 in $cmds; do + IFS="$save_ifs" + # Take the normal branch if the nm_file_list_spec branch + # doesn't work or if tool conversion is not needed. + case $nm_file_list_spec~$to_tool_file_cmd in + *~func_convert_file_noop | *~func_convert_file_msys_to_w32 | ~*) + try_normal_branch=yes + eval cmd=\"$cmd1\" + func_len " $cmd" + len=$func_len_result + ;; + *) + try_normal_branch=no + ;; + esac + if test "$try_normal_branch" = yes \ + && { test "$len" -lt "$max_cmd_len" \ + || test "$max_cmd_len" -le -1; } + then + func_show_eval "$cmd" 'exit $?' + skipped_export=false + elif test -n "$nm_file_list_spec"; then + func_basename "$output" + output_la=$func_basename_result + save_libobjs=$libobjs + save_output=$output + output=${output_objdir}/${output_la}.nm + func_to_tool_file "$output" + libobjs=$nm_file_list_spec$func_to_tool_file_result + func_append delfiles " $output" + func_verbose "creating $NM input file list: $output" + for obj in $save_libobjs; do + func_to_tool_file "$obj" + $ECHO "$func_to_tool_file_result" + done > "$output" + eval cmd=\"$cmd1\" + func_show_eval "$cmd" 'exit $?' + output=$save_output + libobjs=$save_libobjs + skipped_export=false + else + # The command line is too long to execute in one step. + func_verbose "using reloadable object file for export list..." + skipped_export=: + # Break out early, otherwise skipped_export may be + # set to false by a later but shorter cmd. + break + fi + done + IFS="$save_ifs" + if test -n "$export_symbols_regex" && test "X$skipped_export" != "X:"; then + func_show_eval '$EGREP -e "$export_symbols_regex" "$export_symbols" > "${export_symbols}T"' + func_show_eval '$MV "${export_symbols}T" "$export_symbols"' + fi + fi + fi + + if test -n "$export_symbols" && test -n "$include_expsyms"; then + tmp_export_symbols="$export_symbols" + test -n "$orig_export_symbols" && tmp_export_symbols="$orig_export_symbols" + $opt_dry_run || eval '$ECHO "$include_expsyms" | $SP2NL >> "$tmp_export_symbols"' + fi + + if test "X$skipped_export" != "X:" && test -n "$orig_export_symbols"; then + # The given exports_symbols file has to be filtered, so filter it. + func_verbose "filter symbol list for \`$libname.la' to tag DATA exports" + # FIXME: $output_objdir/$libname.filter potentially contains lots of + # 's' commands which not all seds can handle. GNU sed should be fine + # though. Also, the filter scales superlinearly with the number of + # global variables. join(1) would be nice here, but unfortunately + # isn't a blessed tool. + $opt_dry_run || $SED -e '/[ ,]DATA/!d;s,\(.*\)\([ \,].*\),s|^\1$|\1\2|,' < $export_symbols > $output_objdir/$libname.filter + func_append delfiles " $export_symbols $output_objdir/$libname.filter" + export_symbols=$output_objdir/$libname.def + $opt_dry_run || $SED -f $output_objdir/$libname.filter < $orig_export_symbols > $export_symbols + fi + + tmp_deplibs= + for test_deplib in $deplibs; do + case " $convenience " in + *" $test_deplib "*) ;; + *) + func_append tmp_deplibs " $test_deplib" + ;; + esac + done + deplibs="$tmp_deplibs" + + if test -n "$convenience"; then + if test -n "$whole_archive_flag_spec" && + test "$compiler_needs_object" = yes && + test -z "$libobjs"; then + # extract the archives, so we have objects to list. + # TODO: could optimize this to just extract one archive. + whole_archive_flag_spec= + fi + if test -n "$whole_archive_flag_spec"; then + save_libobjs=$libobjs + eval libobjs=\"\$libobjs $whole_archive_flag_spec\" + test "X$libobjs" = "X " && libobjs= + else + gentop="$output_objdir/${outputname}x" + func_append generated " $gentop" + + func_extract_archives $gentop $convenience + func_append libobjs " $func_extract_archives_result" + test "X$libobjs" = "X " && libobjs= + fi + fi + + if test "$thread_safe" = yes && test -n "$thread_safe_flag_spec"; then + eval flag=\"$thread_safe_flag_spec\" + func_append linker_flags " $flag" + fi + + # Make a backup of the uninstalled library when relinking + if test "$opt_mode" = relink; then + $opt_dry_run || eval '(cd $output_objdir && $RM ${realname}U && $MV $realname ${realname}U)' || exit $? + fi + + # Do each of the archive commands. + if test "$module" = yes && test -n "$module_cmds" ; then + if test -n "$export_symbols" && test -n "$module_expsym_cmds"; then + eval test_cmds=\"$module_expsym_cmds\" + cmds=$module_expsym_cmds + else + eval test_cmds=\"$module_cmds\" + cmds=$module_cmds + fi + else + if test -n "$export_symbols" && test -n "$archive_expsym_cmds"; then + eval test_cmds=\"$archive_expsym_cmds\" + cmds=$archive_expsym_cmds + else + eval test_cmds=\"$archive_cmds\" + cmds=$archive_cmds + fi + fi + + if test "X$skipped_export" != "X:" && + func_len " $test_cmds" && + len=$func_len_result && + test "$len" -lt "$max_cmd_len" || test "$max_cmd_len" -le -1; then + : + else + # The command line is too long to link in one step, link piecewise + # or, if using GNU ld and skipped_export is not :, use a linker + # script. + + # Save the value of $output and $libobjs because we want to + # use them later. If we have whole_archive_flag_spec, we + # want to use save_libobjs as it was before + # whole_archive_flag_spec was expanded, because we can't + # assume the linker understands whole_archive_flag_spec. + # This may have to be revisited, in case too many + # convenience libraries get linked in and end up exceeding + # the spec. + if test -z "$convenience" || test -z "$whole_archive_flag_spec"; then + save_libobjs=$libobjs + fi + save_output=$output + func_basename "$output" + output_la=$func_basename_result + + # Clear the reloadable object creation command queue and + # initialize k to one. + test_cmds= + concat_cmds= + objlist= + last_robj= + k=1 + + if test -n "$save_libobjs" && test "X$skipped_export" != "X:" && test "$with_gnu_ld" = yes; then + output=${output_objdir}/${output_la}.lnkscript + func_verbose "creating GNU ld script: $output" + echo 'INPUT (' > $output + for obj in $save_libobjs + do + func_to_tool_file "$obj" + $ECHO "$func_to_tool_file_result" >> $output + done + echo ')' >> $output + func_append delfiles " $output" + func_to_tool_file "$output" + output=$func_to_tool_file_result + elif test -n "$save_libobjs" && test "X$skipped_export" != "X:" && test "X$file_list_spec" != X; then + output=${output_objdir}/${output_la}.lnk + func_verbose "creating linker input file list: $output" + : > $output + set x $save_libobjs + shift + firstobj= + if test "$compiler_needs_object" = yes; then + firstobj="$1 " + shift + fi + for obj + do + func_to_tool_file "$obj" + $ECHO "$func_to_tool_file_result" >> $output + done + func_append delfiles " $output" + func_to_tool_file "$output" + output=$firstobj\"$file_list_spec$func_to_tool_file_result\" + else + if test -n "$save_libobjs"; then + func_verbose "creating reloadable object files..." + output=$output_objdir/$output_la-${k}.$objext + eval test_cmds=\"$reload_cmds\" + func_len " $test_cmds" + len0=$func_len_result + len=$len0 + + # Loop over the list of objects to be linked. + for obj in $save_libobjs + do + func_len " $obj" + func_arith $len + $func_len_result + len=$func_arith_result + if test "X$objlist" = X || + test "$len" -lt "$max_cmd_len"; then + func_append objlist " $obj" + else + # The command $test_cmds is almost too long, add a + # command to the queue. + if test "$k" -eq 1 ; then + # The first file doesn't have a previous command to add. + reload_objs=$objlist + eval concat_cmds=\"$reload_cmds\" + else + # All subsequent reloadable object files will link in + # the last one created. + reload_objs="$objlist $last_robj" + eval concat_cmds=\"\$concat_cmds~$reload_cmds~\$RM $last_robj\" + fi + last_robj=$output_objdir/$output_la-${k}.$objext + func_arith $k + 1 + k=$func_arith_result + output=$output_objdir/$output_la-${k}.$objext + objlist=" $obj" + func_len " $last_robj" + func_arith $len0 + $func_len_result + len=$func_arith_result + fi + done + # Handle the remaining objects by creating one last + # reloadable object file. All subsequent reloadable object + # files will link in the last one created. + test -z "$concat_cmds" || concat_cmds=$concat_cmds~ + reload_objs="$objlist $last_robj" + eval concat_cmds=\"\${concat_cmds}$reload_cmds\" + if test -n "$last_robj"; then + eval concat_cmds=\"\${concat_cmds}~\$RM $last_robj\" + fi + func_append delfiles " $output" + + else + output= + fi + + if ${skipped_export-false}; then + func_verbose "generating symbol list for \`$libname.la'" + export_symbols="$output_objdir/$libname.exp" + $opt_dry_run || $RM $export_symbols + libobjs=$output + # Append the command to create the export file. + test -z "$concat_cmds" || concat_cmds=$concat_cmds~ + eval concat_cmds=\"\$concat_cmds$export_symbols_cmds\" + if test -n "$last_robj"; then + eval concat_cmds=\"\$concat_cmds~\$RM $last_robj\" + fi + fi + + test -n "$save_libobjs" && + func_verbose "creating a temporary reloadable object file: $output" + + # Loop through the commands generated above and execute them. + save_ifs="$IFS"; IFS='~' + for cmd in $concat_cmds; do + IFS="$save_ifs" + $opt_silent || { + func_quote_for_expand "$cmd" + eval "func_echo $func_quote_for_expand_result" + } + $opt_dry_run || eval "$cmd" || { + lt_exit=$? + + # Restore the uninstalled library and exit + if test "$opt_mode" = relink; then + ( cd "$output_objdir" && \ + $RM "${realname}T" && \ + $MV "${realname}U" "$realname" ) + fi + + exit $lt_exit + } + done + IFS="$save_ifs" + + if test -n "$export_symbols_regex" && ${skipped_export-false}; then + func_show_eval '$EGREP -e "$export_symbols_regex" "$export_symbols" > "${export_symbols}T"' + func_show_eval '$MV "${export_symbols}T" "$export_symbols"' + fi + fi + + if ${skipped_export-false}; then + if test -n "$export_symbols" && test -n "$include_expsyms"; then + tmp_export_symbols="$export_symbols" + test -n "$orig_export_symbols" && tmp_export_symbols="$orig_export_symbols" + $opt_dry_run || eval '$ECHO "$include_expsyms" | $SP2NL >> "$tmp_export_symbols"' + fi + + if test -n "$orig_export_symbols"; then + # The given exports_symbols file has to be filtered, so filter it. + func_verbose "filter symbol list for \`$libname.la' to tag DATA exports" + # FIXME: $output_objdir/$libname.filter potentially contains lots of + # 's' commands which not all seds can handle. GNU sed should be fine + # though. Also, the filter scales superlinearly with the number of + # global variables. join(1) would be nice here, but unfortunately + # isn't a blessed tool. + $opt_dry_run || $SED -e '/[ ,]DATA/!d;s,\(.*\)\([ \,].*\),s|^\1$|\1\2|,' < $export_symbols > $output_objdir/$libname.filter + func_append delfiles " $export_symbols $output_objdir/$libname.filter" + export_symbols=$output_objdir/$libname.def + $opt_dry_run || $SED -f $output_objdir/$libname.filter < $orig_export_symbols > $export_symbols + fi + fi + + libobjs=$output + # Restore the value of output. + output=$save_output + + if test -n "$convenience" && test -n "$whole_archive_flag_spec"; then + eval libobjs=\"\$libobjs $whole_archive_flag_spec\" + test "X$libobjs" = "X " && libobjs= + fi + # Expand the library linking commands again to reset the + # value of $libobjs for piecewise linking. + + # Do each of the archive commands. + if test "$module" = yes && test -n "$module_cmds" ; then + if test -n "$export_symbols" && test -n "$module_expsym_cmds"; then + cmds=$module_expsym_cmds + else + cmds=$module_cmds + fi + else + if test -n "$export_symbols" && test -n "$archive_expsym_cmds"; then + cmds=$archive_expsym_cmds + else + cmds=$archive_cmds + fi + fi + fi + + if test -n "$delfiles"; then + # Append the command to remove temporary files to $cmds. + eval cmds=\"\$cmds~\$RM $delfiles\" + fi + + # Add any objects from preloaded convenience libraries + if test -n "$dlprefiles"; then + gentop="$output_objdir/${outputname}x" + func_append generated " $gentop" + + func_extract_archives $gentop $dlprefiles + func_append libobjs " $func_extract_archives_result" + test "X$libobjs" = "X " && libobjs= + fi + + save_ifs="$IFS"; IFS='~' + for cmd in $cmds; do + IFS="$save_ifs" + eval cmd=\"$cmd\" + $opt_silent || { + func_quote_for_expand "$cmd" + eval "func_echo $func_quote_for_expand_result" + } + $opt_dry_run || eval "$cmd" || { + lt_exit=$? + + # Restore the uninstalled library and exit + if test "$opt_mode" = relink; then + ( cd "$output_objdir" && \ + $RM "${realname}T" && \ + $MV "${realname}U" "$realname" ) + fi + + exit $lt_exit + } + done + IFS="$save_ifs" + + # Restore the uninstalled library and exit + if test "$opt_mode" = relink; then + $opt_dry_run || eval '(cd $output_objdir && $RM ${realname}T && $MV $realname ${realname}T && $MV ${realname}U $realname)' || exit $? + + if test -n "$convenience"; then + if test -z "$whole_archive_flag_spec"; then + func_show_eval '${RM}r "$gentop"' + fi + fi + + exit $EXIT_SUCCESS + fi + + # Create links to the real library. + for linkname in $linknames; do + if test "$realname" != "$linkname"; then + func_show_eval '(cd "$output_objdir" && $RM "$linkname" && $LN_S "$realname" "$linkname")' 'exit $?' + fi + done + + # If -module or -export-dynamic was specified, set the dlname. + if test "$module" = yes || test "$export_dynamic" = yes; then + # On all known operating systems, these are identical. + dlname="$soname" + fi + fi + ;; + + obj) + if test -n "$dlfiles$dlprefiles" || test "$dlself" != no; then + func_warning "\`-dlopen' is ignored for objects" + fi + + case " $deplibs" in + *\ -l* | *\ -L*) + func_warning "\`-l' and \`-L' are ignored for objects" ;; + esac + + test -n "$rpath" && \ + func_warning "\`-rpath' is ignored for objects" + + test -n "$xrpath" && \ + func_warning "\`-R' is ignored for objects" + + test -n "$vinfo" && \ + func_warning "\`-version-info' is ignored for objects" + + test -n "$release" && \ + func_warning "\`-release' is ignored for objects" + + case $output in + *.lo) + test -n "$objs$old_deplibs" && \ + func_fatal_error "cannot build library object \`$output' from non-libtool objects" + + libobj=$output + func_lo2o "$libobj" + obj=$func_lo2o_result + ;; + *) + libobj= + obj="$output" + ;; + esac + + # Delete the old objects. + $opt_dry_run || $RM $obj $libobj + + # Objects from convenience libraries. This assumes + # single-version convenience libraries. Whenever we create + # different ones for PIC/non-PIC, this we'll have to duplicate + # the extraction. + reload_conv_objs= + gentop= + # reload_cmds runs $LD directly, so let us get rid of + # -Wl from whole_archive_flag_spec and hope we can get by with + # turning comma into space.. + wl= + + if test -n "$convenience"; then + if test -n "$whole_archive_flag_spec"; then + eval tmp_whole_archive_flags=\"$whole_archive_flag_spec\" + reload_conv_objs=$reload_objs\ `$ECHO "$tmp_whole_archive_flags" | $SED 's|,| |g'` + else + gentop="$output_objdir/${obj}x" + func_append generated " $gentop" + + func_extract_archives $gentop $convenience + reload_conv_objs="$reload_objs $func_extract_archives_result" + fi + fi + + # If we're not building shared, we need to use non_pic_objs + test "$build_libtool_libs" != yes && libobjs="$non_pic_objects" + + # Create the old-style object. + reload_objs="$objs$old_deplibs "`$ECHO "$libobjs" | $SP2NL | $SED "/\.${libext}$/d; /\.lib$/d; $lo2o" | $NL2SP`" $reload_conv_objs" ### testsuite: skip nested quoting test + + output="$obj" + func_execute_cmds "$reload_cmds" 'exit $?' + + # Exit if we aren't doing a library object file. + if test -z "$libobj"; then + if test -n "$gentop"; then + func_show_eval '${RM}r "$gentop"' + fi + + exit $EXIT_SUCCESS + fi + + if test "$build_libtool_libs" != yes; then + if test -n "$gentop"; then + func_show_eval '${RM}r "$gentop"' + fi + + # Create an invalid libtool object if no PIC, so that we don't + # accidentally link it into a program. + # $show "echo timestamp > $libobj" + # $opt_dry_run || eval "echo timestamp > $libobj" || exit $? + exit $EXIT_SUCCESS + fi + + if test -n "$pic_flag" || test "$pic_mode" != default; then + # Only do commands if we really have different PIC objects. + reload_objs="$libobjs $reload_conv_objs" + output="$libobj" + func_execute_cmds "$reload_cmds" 'exit $?' + fi + + if test -n "$gentop"; then + func_show_eval '${RM}r "$gentop"' + fi + + exit $EXIT_SUCCESS + ;; + + prog) + case $host in + *cygwin*) func_stripname '' '.exe' "$output" + output=$func_stripname_result.exe;; + esac + test -n "$vinfo" && \ + func_warning "\`-version-info' is ignored for programs" + + test -n "$release" && \ + func_warning "\`-release' is ignored for programs" + + test "$preload" = yes \ + && test "$dlopen_support" = unknown \ + && test "$dlopen_self" = unknown \ + && test "$dlopen_self_static" = unknown && \ + func_warning "\`LT_INIT([dlopen])' not used. Assuming no dlopen support." + + case $host in + *-*-rhapsody* | *-*-darwin1.[012]) + # On Rhapsody replace the C library is the System framework + compile_deplibs=`$ECHO " $compile_deplibs" | $SED 's/ -lc / System.ltframework /'` + finalize_deplibs=`$ECHO " $finalize_deplibs" | $SED 's/ -lc / System.ltframework /'` + ;; + esac + + case $host in + *-*-darwin*) + # Don't allow lazy linking, it breaks C++ global constructors + # But is supposedly fixed on 10.4 or later (yay!). + if test "$tagname" = CXX ; then + case ${MACOSX_DEPLOYMENT_TARGET-10.0} in + 10.[0123]) + func_append compile_command " ${wl}-bind_at_load" + func_append finalize_command " ${wl}-bind_at_load" + ;; + esac + fi + # Time to change all our "foo.ltframework" stuff back to "-framework foo" + compile_deplibs=`$ECHO " $compile_deplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` + finalize_deplibs=`$ECHO " $finalize_deplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` + ;; + esac + + + # move library search paths that coincide with paths to not yet + # installed libraries to the beginning of the library search list + new_libs= + for path in $notinst_path; do + case " $new_libs " in + *" -L$path/$objdir "*) ;; + *) + case " $compile_deplibs " in + *" -L$path/$objdir "*) + func_append new_libs " -L$path/$objdir" ;; + esac + ;; + esac + done + for deplib in $compile_deplibs; do + case $deplib in + -L*) + case " $new_libs " in + *" $deplib "*) ;; + *) func_append new_libs " $deplib" ;; + esac + ;; + *) func_append new_libs " $deplib" ;; + esac + done + compile_deplibs="$new_libs" + + + func_append compile_command " $compile_deplibs" + func_append finalize_command " $finalize_deplibs" + + if test -n "$rpath$xrpath"; then + # If the user specified any rpath flags, then add them. + for libdir in $rpath $xrpath; do + # This is the magic to use -rpath. + case "$finalize_rpath " in + *" $libdir "*) ;; + *) func_append finalize_rpath " $libdir" ;; + esac + done + fi + + # Now hardcode the library paths + rpath= + hardcode_libdirs= + for libdir in $compile_rpath $finalize_rpath; do + if test -n "$hardcode_libdir_flag_spec"; then + if test -n "$hardcode_libdir_separator"; then + if test -z "$hardcode_libdirs"; then + hardcode_libdirs="$libdir" + else + # Just accumulate the unique libdirs. + case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in + *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*) + ;; + *) + func_append hardcode_libdirs "$hardcode_libdir_separator$libdir" + ;; + esac + fi + else + eval flag=\"$hardcode_libdir_flag_spec\" + func_append rpath " $flag" + fi + elif test -n "$runpath_var"; then + case "$perm_rpath " in + *" $libdir "*) ;; + *) func_append perm_rpath " $libdir" ;; + esac + fi + case $host in + *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*) + testbindir=`${ECHO} "$libdir" | ${SED} -e 's*/lib$*/bin*'` + case :$dllsearchpath: in + *":$libdir:"*) ;; + ::) dllsearchpath=$libdir;; + *) func_append dllsearchpath ":$libdir";; + esac + case :$dllsearchpath: in + *":$testbindir:"*) ;; + ::) dllsearchpath=$testbindir;; + *) func_append dllsearchpath ":$testbindir";; + esac + ;; + esac + done + # Substitute the hardcoded libdirs into the rpath. + if test -n "$hardcode_libdir_separator" && + test -n "$hardcode_libdirs"; then + libdir="$hardcode_libdirs" + eval rpath=\" $hardcode_libdir_flag_spec\" + fi + compile_rpath="$rpath" + + rpath= + hardcode_libdirs= + for libdir in $finalize_rpath; do + if test -n "$hardcode_libdir_flag_spec"; then + if test -n "$hardcode_libdir_separator"; then + if test -z "$hardcode_libdirs"; then + hardcode_libdirs="$libdir" + else + # Just accumulate the unique libdirs. + case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in + *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*) + ;; + *) + func_append hardcode_libdirs "$hardcode_libdir_separator$libdir" + ;; + esac + fi + else + eval flag=\"$hardcode_libdir_flag_spec\" + func_append rpath " $flag" + fi + elif test -n "$runpath_var"; then + case "$finalize_perm_rpath " in + *" $libdir "*) ;; + *) func_append finalize_perm_rpath " $libdir" ;; + esac + fi + done + # Substitute the hardcoded libdirs into the rpath. + if test -n "$hardcode_libdir_separator" && + test -n "$hardcode_libdirs"; then + libdir="$hardcode_libdirs" + eval rpath=\" $hardcode_libdir_flag_spec\" + fi + finalize_rpath="$rpath" + + if test -n "$libobjs" && test "$build_old_libs" = yes; then + # Transform all the library objects into standard objects. + compile_command=`$ECHO "$compile_command" | $SP2NL | $SED "$lo2o" | $NL2SP` + finalize_command=`$ECHO "$finalize_command" | $SP2NL | $SED "$lo2o" | $NL2SP` + fi + + func_generate_dlsyms "$outputname" "@PROGRAM@" "no" + + # template prelinking step + if test -n "$prelink_cmds"; then + func_execute_cmds "$prelink_cmds" 'exit $?' + fi + + wrappers_required=yes + case $host in + *cegcc* | *mingw32ce*) + # Disable wrappers for cegcc and mingw32ce hosts, we are cross compiling anyway. + wrappers_required=no + ;; + *cygwin* | *mingw* ) + if test "$build_libtool_libs" != yes; then + wrappers_required=no + fi + ;; + *) + if test "$need_relink" = no || test "$build_libtool_libs" != yes; then + wrappers_required=no + fi + ;; + esac + if test "$wrappers_required" = no; then + # Replace the output file specification. + compile_command=`$ECHO "$compile_command" | $SED 's%@OUTPUT@%'"$output"'%g'` + link_command="$compile_command$compile_rpath" + + # We have no uninstalled library dependencies, so finalize right now. + exit_status=0 + func_show_eval "$link_command" 'exit_status=$?' + + if test -n "$postlink_cmds"; then + func_to_tool_file "$output" + postlink_cmds=`func_echo_all "$postlink_cmds" | $SED -e 's%@OUTPUT@%'"$output"'%g' -e 's%@TOOL_OUTPUT@%'"$func_to_tool_file_result"'%g'` + func_execute_cmds "$postlink_cmds" 'exit $?' + fi + + # Delete the generated files. + if test -f "$output_objdir/${outputname}S.${objext}"; then + func_show_eval '$RM "$output_objdir/${outputname}S.${objext}"' + fi + + exit $exit_status + fi + + if test -n "$compile_shlibpath$finalize_shlibpath"; then + compile_command="$shlibpath_var=\"$compile_shlibpath$finalize_shlibpath\$$shlibpath_var\" $compile_command" + fi + if test -n "$finalize_shlibpath"; then + finalize_command="$shlibpath_var=\"$finalize_shlibpath\$$shlibpath_var\" $finalize_command" + fi + + compile_var= + finalize_var= + if test -n "$runpath_var"; then + if test -n "$perm_rpath"; then + # We should set the runpath_var. + rpath= + for dir in $perm_rpath; do + func_append rpath "$dir:" + done + compile_var="$runpath_var=\"$rpath\$$runpath_var\" " + fi + if test -n "$finalize_perm_rpath"; then + # We should set the runpath_var. + rpath= + for dir in $finalize_perm_rpath; do + func_append rpath "$dir:" + done + finalize_var="$runpath_var=\"$rpath\$$runpath_var\" " + fi + fi + + if test "$no_install" = yes; then + # We don't need to create a wrapper script. + link_command="$compile_var$compile_command$compile_rpath" + # Replace the output file specification. + link_command=`$ECHO "$link_command" | $SED 's%@OUTPUT@%'"$output"'%g'` + # Delete the old output file. + $opt_dry_run || $RM $output + # Link the executable and exit + func_show_eval "$link_command" 'exit $?' + + if test -n "$postlink_cmds"; then + func_to_tool_file "$output" + postlink_cmds=`func_echo_all "$postlink_cmds" | $SED -e 's%@OUTPUT@%'"$output"'%g' -e 's%@TOOL_OUTPUT@%'"$func_to_tool_file_result"'%g'` + func_execute_cmds "$postlink_cmds" 'exit $?' + fi + + exit $EXIT_SUCCESS + fi + + if test "$hardcode_action" = relink; then + # Fast installation is not supported + link_command="$compile_var$compile_command$compile_rpath" + relink_command="$finalize_var$finalize_command$finalize_rpath" + + func_warning "this platform does not like uninstalled shared libraries" + func_warning "\`$output' will be relinked during installation" + else + if test "$fast_install" != no; then + link_command="$finalize_var$compile_command$finalize_rpath" + if test "$fast_install" = yes; then + relink_command=`$ECHO "$compile_var$compile_command$compile_rpath" | $SED 's%@OUTPUT@%\$progdir/\$file%g'` + else + # fast_install is set to needless + relink_command= + fi + else + link_command="$compile_var$compile_command$compile_rpath" + relink_command="$finalize_var$finalize_command$finalize_rpath" + fi + fi + + # Replace the output file specification. + link_command=`$ECHO "$link_command" | $SED 's%@OUTPUT@%'"$output_objdir/$outputname"'%g'` + + # Delete the old output files. + $opt_dry_run || $RM $output $output_objdir/$outputname $output_objdir/lt-$outputname + + func_show_eval "$link_command" 'exit $?' + + if test -n "$postlink_cmds"; then + func_to_tool_file "$output_objdir/$outputname" + postlink_cmds=`func_echo_all "$postlink_cmds" | $SED -e 's%@OUTPUT@%'"$output_objdir/$outputname"'%g' -e 's%@TOOL_OUTPUT@%'"$func_to_tool_file_result"'%g'` + func_execute_cmds "$postlink_cmds" 'exit $?' + fi + + # Now create the wrapper script. + func_verbose "creating $output" + + # Quote the relink command for shipping. + if test -n "$relink_command"; then + # Preserve any variables that may affect compiler behavior + for var in $variables_saved_for_relink; do + if eval test -z \"\${$var+set}\"; then + relink_command="{ test -z \"\${$var+set}\" || $lt_unset $var || { $var=; export $var; }; }; $relink_command" + elif eval var_value=\$$var; test -z "$var_value"; then + relink_command="$var=; export $var; $relink_command" + else + func_quote_for_eval "$var_value" + relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command" + fi + done + relink_command="(cd `pwd`; $relink_command)" + relink_command=`$ECHO "$relink_command" | $SED "$sed_quote_subst"` + fi + + # Only actually do things if not in dry run mode. + $opt_dry_run || { + # win32 will think the script is a binary if it has + # a .exe suffix, so we strip it off here. + case $output in + *.exe) func_stripname '' '.exe' "$output" + output=$func_stripname_result ;; + esac + # test for cygwin because mv fails w/o .exe extensions + case $host in + *cygwin*) + exeext=.exe + func_stripname '' '.exe' "$outputname" + outputname=$func_stripname_result ;; + *) exeext= ;; + esac + case $host in + *cygwin* | *mingw* ) + func_dirname_and_basename "$output" "" "." + output_name=$func_basename_result + output_path=$func_dirname_result + cwrappersource="$output_path/$objdir/lt-$output_name.c" + cwrapper="$output_path/$output_name.exe" + $RM $cwrappersource $cwrapper + trap "$RM $cwrappersource $cwrapper; exit $EXIT_FAILURE" 1 2 15 + + func_emit_cwrapperexe_src > $cwrappersource + + # The wrapper executable is built using the $host compiler, + # because it contains $host paths and files. If cross- + # compiling, it, like the target executable, must be + # executed on the $host or under an emulation environment. + $opt_dry_run || { + $LTCC $LTCFLAGS -o $cwrapper $cwrappersource + $STRIP $cwrapper + } + + # Now, create the wrapper script for func_source use: + func_ltwrapper_scriptname $cwrapper + $RM $func_ltwrapper_scriptname_result + trap "$RM $func_ltwrapper_scriptname_result; exit $EXIT_FAILURE" 1 2 15 + $opt_dry_run || { + # note: this script will not be executed, so do not chmod. + if test "x$build" = "x$host" ; then + $cwrapper --lt-dump-script > $func_ltwrapper_scriptname_result + else + func_emit_wrapper no > $func_ltwrapper_scriptname_result + fi + } + ;; + * ) + $RM $output + trap "$RM $output; exit $EXIT_FAILURE" 1 2 15 + + func_emit_wrapper no > $output + chmod +x $output + ;; + esac + } + exit $EXIT_SUCCESS + ;; + esac + + # See if we need to build an old-fashioned archive. + for oldlib in $oldlibs; do + + if test "$build_libtool_libs" = convenience; then + oldobjs="$libobjs_save $symfileobj" + addlibs="$convenience" + build_libtool_libs=no + else + if test "$build_libtool_libs" = module; then + oldobjs="$libobjs_save" + build_libtool_libs=no + else + oldobjs="$old_deplibs $non_pic_objects" + if test "$preload" = yes && test -f "$symfileobj"; then + func_append oldobjs " $symfileobj" + fi + fi + addlibs="$old_convenience" + fi + + if test -n "$addlibs"; then + gentop="$output_objdir/${outputname}x" + func_append generated " $gentop" + + func_extract_archives $gentop $addlibs + func_append oldobjs " $func_extract_archives_result" + fi + + # Do each command in the archive commands. + if test -n "$old_archive_from_new_cmds" && test "$build_libtool_libs" = yes; then + cmds=$old_archive_from_new_cmds + else + + # Add any objects from preloaded convenience libraries + if test -n "$dlprefiles"; then + gentop="$output_objdir/${outputname}x" + func_append generated " $gentop" + + func_extract_archives $gentop $dlprefiles + func_append oldobjs " $func_extract_archives_result" + fi + + # POSIX demands no paths to be encoded in archives. We have + # to avoid creating archives with duplicate basenames if we + # might have to extract them afterwards, e.g., when creating a + # static archive out of a convenience library, or when linking + # the entirety of a libtool archive into another (currently + # not supported by libtool). + if (for obj in $oldobjs + do + func_basename "$obj" + $ECHO "$func_basename_result" + done | sort | sort -uc >/dev/null 2>&1); then + : + else + echo "copying selected object files to avoid basename conflicts..." + gentop="$output_objdir/${outputname}x" + func_append generated " $gentop" + func_mkdir_p "$gentop" + save_oldobjs=$oldobjs + oldobjs= + counter=1 + for obj in $save_oldobjs + do + func_basename "$obj" + objbase="$func_basename_result" + case " $oldobjs " in + " ") oldobjs=$obj ;; + *[\ /]"$objbase "*) + while :; do + # Make sure we don't pick an alternate name that also + # overlaps. + newobj=lt$counter-$objbase + func_arith $counter + 1 + counter=$func_arith_result + case " $oldobjs " in + *[\ /]"$newobj "*) ;; + *) if test ! -f "$gentop/$newobj"; then break; fi ;; + esac + done + func_show_eval "ln $obj $gentop/$newobj || cp $obj $gentop/$newobj" + func_append oldobjs " $gentop/$newobj" + ;; + *) func_append oldobjs " $obj" ;; + esac + done + fi + func_to_tool_file "$oldlib" func_convert_file_msys_to_w32 + tool_oldlib=$func_to_tool_file_result + eval cmds=\"$old_archive_cmds\" + + func_len " $cmds" + len=$func_len_result + if test "$len" -lt "$max_cmd_len" || test "$max_cmd_len" -le -1; then + cmds=$old_archive_cmds + elif test -n "$archiver_list_spec"; then + func_verbose "using command file archive linking..." + for obj in $oldobjs + do + func_to_tool_file "$obj" + $ECHO "$func_to_tool_file_result" + done > $output_objdir/$libname.libcmd + func_to_tool_file "$output_objdir/$libname.libcmd" + oldobjs=" $archiver_list_spec$func_to_tool_file_result" + cmds=$old_archive_cmds + else + # the command line is too long to link in one step, link in parts + func_verbose "using piecewise archive linking..." + save_RANLIB=$RANLIB + RANLIB=: + objlist= + concat_cmds= + save_oldobjs=$oldobjs + oldobjs= + # Is there a better way of finding the last object in the list? + for obj in $save_oldobjs + do + last_oldobj=$obj + done + eval test_cmds=\"$old_archive_cmds\" + func_len " $test_cmds" + len0=$func_len_result + len=$len0 + for obj in $save_oldobjs + do + func_len " $obj" + func_arith $len + $func_len_result + len=$func_arith_result + func_append objlist " $obj" + if test "$len" -lt "$max_cmd_len"; then + : + else + # the above command should be used before it gets too long + oldobjs=$objlist + if test "$obj" = "$last_oldobj" ; then + RANLIB=$save_RANLIB + fi + test -z "$concat_cmds" || concat_cmds=$concat_cmds~ + eval concat_cmds=\"\${concat_cmds}$old_archive_cmds\" + objlist= + len=$len0 + fi + done + RANLIB=$save_RANLIB + oldobjs=$objlist + if test "X$oldobjs" = "X" ; then + eval cmds=\"\$concat_cmds\" + else + eval cmds=\"\$concat_cmds~\$old_archive_cmds\" + fi + fi + fi + func_execute_cmds "$cmds" 'exit $?' + done + + test -n "$generated" && \ + func_show_eval "${RM}r$generated" + + # Now create the libtool archive. + case $output in + *.la) + old_library= + test "$build_old_libs" = yes && old_library="$libname.$libext" + func_verbose "creating $output" + + # Preserve any variables that may affect compiler behavior + for var in $variables_saved_for_relink; do + if eval test -z \"\${$var+set}\"; then + relink_command="{ test -z \"\${$var+set}\" || $lt_unset $var || { $var=; export $var; }; }; $relink_command" + elif eval var_value=\$$var; test -z "$var_value"; then + relink_command="$var=; export $var; $relink_command" + else + func_quote_for_eval "$var_value" + relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command" + fi + done + # Quote the link command for shipping. + relink_command="(cd `pwd`; $SHELL $progpath $preserve_args --mode=relink $libtool_args @inst_prefix_dir@)" + relink_command=`$ECHO "$relink_command" | $SED "$sed_quote_subst"` + if test "$hardcode_automatic" = yes ; then + relink_command= + fi + + # Only create the output if not a dry run. + $opt_dry_run || { + for installed in no yes; do + if test "$installed" = yes; then + if test -z "$install_libdir"; then + break + fi + output="$output_objdir/$outputname"i + # Replace all uninstalled libtool libraries with the installed ones + newdependency_libs= + for deplib in $dependency_libs; do + case $deplib in + *.la) + func_basename "$deplib" + name="$func_basename_result" + func_resolve_sysroot "$deplib" + eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $func_resolve_sysroot_result` + test -z "$libdir" && \ + func_fatal_error "\`$deplib' is not a valid libtool archive" + func_append newdependency_libs " ${lt_sysroot:+=}$libdir/$name" + ;; + -L*) + func_stripname -L '' "$deplib" + func_replace_sysroot "$func_stripname_result" + func_append newdependency_libs " -L$func_replace_sysroot_result" + ;; + -R*) + func_stripname -R '' "$deplib" + func_replace_sysroot "$func_stripname_result" + func_append newdependency_libs " -R$func_replace_sysroot_result" + ;; + *) func_append newdependency_libs " $deplib" ;; + esac + done + dependency_libs="$newdependency_libs" + newdlfiles= + + for lib in $dlfiles; do + case $lib in + *.la) + func_basename "$lib" + name="$func_basename_result" + eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $lib` + test -z "$libdir" && \ + func_fatal_error "\`$lib' is not a valid libtool archive" + func_append newdlfiles " ${lt_sysroot:+=}$libdir/$name" + ;; + *) func_append newdlfiles " $lib" ;; + esac + done + dlfiles="$newdlfiles" + newdlprefiles= + for lib in $dlprefiles; do + case $lib in + *.la) + # Only pass preopened files to the pseudo-archive (for + # eventual linking with the app. that links it) if we + # didn't already link the preopened objects directly into + # the library: + func_basename "$lib" + name="$func_basename_result" + eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $lib` + test -z "$libdir" && \ + func_fatal_error "\`$lib' is not a valid libtool archive" + func_append newdlprefiles " ${lt_sysroot:+=}$libdir/$name" + ;; + esac + done + dlprefiles="$newdlprefiles" + else + newdlfiles= + for lib in $dlfiles; do + case $lib in + [\\/]* | [A-Za-z]:[\\/]*) abs="$lib" ;; + *) abs=`pwd`"/$lib" ;; + esac + func_append newdlfiles " $abs" + done + dlfiles="$newdlfiles" + newdlprefiles= + for lib in $dlprefiles; do + case $lib in + [\\/]* | [A-Za-z]:[\\/]*) abs="$lib" ;; + *) abs=`pwd`"/$lib" ;; + esac + func_append newdlprefiles " $abs" + done + dlprefiles="$newdlprefiles" + fi + $RM $output + # place dlname in correct position for cygwin + # In fact, it would be nice if we could use this code for all target + # systems that can't hard-code library paths into their executables + # and that have no shared library path variable independent of PATH, + # but it turns out we can't easily determine that from inspecting + # libtool variables, so we have to hard-code the OSs to which it + # applies here; at the moment, that means platforms that use the PE + # object format with DLL files. See the long comment at the top of + # tests/bindir.at for full details. + tdlname=$dlname + case $host,$output,$installed,$module,$dlname in + *cygwin*,*lai,yes,no,*.dll | *mingw*,*lai,yes,no,*.dll | *cegcc*,*lai,yes,no,*.dll) + # If a -bindir argument was supplied, place the dll there. + if test "x$bindir" != x ; + then + func_relative_path "$install_libdir" "$bindir" + tdlname=$func_relative_path_result$dlname + else + # Otherwise fall back on heuristic. + tdlname=../bin/$dlname + fi + ;; + esac + $ECHO > $output "\ +# $outputname - a libtool library file +# Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='$tdlname' + +# Names of this library. +library_names='$library_names' + +# The name of the static archive. +old_library='$old_library' + +# Linker flags that can not go in dependency_libs. +inherited_linker_flags='$new_inherited_linker_flags' + +# Libraries that this one depends upon. +dependency_libs='$dependency_libs' + +# Names of additional weak libraries provided by this library +weak_library_names='$weak_libs' + +# Version information for $libname. +current=$current +age=$age +revision=$revision + +# Is this an already installed library? +installed=$installed + +# Should we warn about portability when linking against -modules? +shouldnotlink=$module + +# Files to dlopen/dlpreopen +dlopen='$dlfiles' +dlpreopen='$dlprefiles' + +# Directory that this library needs to be installed in: +libdir='$install_libdir'" + if test "$installed" = no && test "$need_relink" = yes; then + $ECHO >> $output "\ +relink_command=\"$relink_command\"" + fi + done + } + + # Do a symbolic link so that the libtool archive can be found in + # LD_LIBRARY_PATH before the program is installed. + func_show_eval '( cd "$output_objdir" && $RM "$outputname" && $LN_S "../$outputname" "$outputname" )' 'exit $?' + ;; + esac + exit $EXIT_SUCCESS +} + +{ test "$opt_mode" = link || test "$opt_mode" = relink; } && + func_mode_link ${1+"$@"} + + +# func_mode_uninstall arg... +func_mode_uninstall () +{ + $opt_debug + RM="$nonopt" + files= + rmforce= + exit_status=0 + + # This variable tells wrapper scripts just to set variables rather + # than running their programs. + libtool_install_magic="$magic" + + for arg + do + case $arg in + -f) func_append RM " $arg"; rmforce=yes ;; + -*) func_append RM " $arg" ;; + *) func_append files " $arg" ;; + esac + done + + test -z "$RM" && \ + func_fatal_help "you must specify an RM program" + + rmdirs= + + for file in $files; do + func_dirname "$file" "" "." + dir="$func_dirname_result" + if test "X$dir" = X.; then + odir="$objdir" + else + odir="$dir/$objdir" + fi + func_basename "$file" + name="$func_basename_result" + test "$opt_mode" = uninstall && odir="$dir" + + # Remember odir for removal later, being careful to avoid duplicates + if test "$opt_mode" = clean; then + case " $rmdirs " in + *" $odir "*) ;; + *) func_append rmdirs " $odir" ;; + esac + fi + + # Don't error if the file doesn't exist and rm -f was used. + if { test -L "$file"; } >/dev/null 2>&1 || + { test -h "$file"; } >/dev/null 2>&1 || + test -f "$file"; then + : + elif test -d "$file"; then + exit_status=1 + continue + elif test "$rmforce" = yes; then + continue + fi + + rmfiles="$file" + + case $name in + *.la) + # Possibly a libtool archive, so verify it. + if func_lalib_p "$file"; then + func_source $dir/$name + + # Delete the libtool libraries and symlinks. + for n in $library_names; do + func_append rmfiles " $odir/$n" + done + test -n "$old_library" && func_append rmfiles " $odir/$old_library" + + case "$opt_mode" in + clean) + case " $library_names " in + *" $dlname "*) ;; + *) test -n "$dlname" && func_append rmfiles " $odir/$dlname" ;; + esac + test -n "$libdir" && func_append rmfiles " $odir/$name $odir/${name}i" + ;; + uninstall) + if test -n "$library_names"; then + # Do each command in the postuninstall commands. + func_execute_cmds "$postuninstall_cmds" 'test "$rmforce" = yes || exit_status=1' + fi + + if test -n "$old_library"; then + # Do each command in the old_postuninstall commands. + func_execute_cmds "$old_postuninstall_cmds" 'test "$rmforce" = yes || exit_status=1' + fi + # FIXME: should reinstall the best remaining shared library. + ;; + esac + fi + ;; + + *.lo) + # Possibly a libtool object, so verify it. + if func_lalib_p "$file"; then + + # Read the .lo file + func_source $dir/$name + + # Add PIC object to the list of files to remove. + if test -n "$pic_object" && + test "$pic_object" != none; then + func_append rmfiles " $dir/$pic_object" + fi + + # Add non-PIC object to the list of files to remove. + if test -n "$non_pic_object" && + test "$non_pic_object" != none; then + func_append rmfiles " $dir/$non_pic_object" + fi + fi + ;; + + *) + if test "$opt_mode" = clean ; then + noexename=$name + case $file in + *.exe) + func_stripname '' '.exe' "$file" + file=$func_stripname_result + func_stripname '' '.exe' "$name" + noexename=$func_stripname_result + # $file with .exe has already been added to rmfiles, + # add $file without .exe + func_append rmfiles " $file" + ;; + esac + # Do a test to see if this is a libtool program. + if func_ltwrapper_p "$file"; then + if func_ltwrapper_executable_p "$file"; then + func_ltwrapper_scriptname "$file" + relink_command= + func_source $func_ltwrapper_scriptname_result + func_append rmfiles " $func_ltwrapper_scriptname_result" + else + relink_command= + func_source $dir/$noexename + fi + + # note $name still contains .exe if it was in $file originally + # as does the version of $file that was added into $rmfiles + func_append rmfiles " $odir/$name $odir/${name}S.${objext}" + if test "$fast_install" = yes && test -n "$relink_command"; then + func_append rmfiles " $odir/lt-$name" + fi + if test "X$noexename" != "X$name" ; then + func_append rmfiles " $odir/lt-${noexename}.c" + fi + fi + fi + ;; + esac + func_show_eval "$RM $rmfiles" 'exit_status=1' + done + + # Try to remove the ${objdir}s in the directories where we deleted files + for dir in $rmdirs; do + if test -d "$dir"; then + func_show_eval "rmdir $dir >/dev/null 2>&1" + fi + done + + exit $exit_status +} + +{ test "$opt_mode" = uninstall || test "$opt_mode" = clean; } && + func_mode_uninstall ${1+"$@"} + +test -z "$opt_mode" && { + help="$generic_help" + func_fatal_help "you must specify a MODE" +} + +test -z "$exec_cmd" && \ + func_fatal_help "invalid operation mode \`$opt_mode'" + +if test -n "$exec_cmd"; then + eval exec "$exec_cmd" + exit $EXIT_FAILURE +fi + +exit $exit_status + + +# The TAGs below are defined such that we never get into a situation +# in which we disable both kinds of libraries. Given conflicting +# choices, we go for a static library, that is the most portable, +# since we can't tell whether shared libraries were disabled because +# the user asked for that or because the platform doesn't support +# them. This is particularly important on AIX, because we don't +# support having both static and shared libraries enabled at the same +# time on that platform, so we default to a shared-only configuration. +# If a disable-shared tag is given, we'll fallback to a static-only +# configuration. But we'll never go from static-only to shared-only. + +# ### BEGIN LIBTOOL TAG CONFIG: disable-shared +build_libtool_libs=no +build_old_libs=yes +# ### END LIBTOOL TAG CONFIG: disable-shared + +# ### BEGIN LIBTOOL TAG CONFIG: disable-static +build_old_libs=`case $build_libtool_libs in yes) echo no;; *) echo yes;; esac` +# ### END LIBTOOL TAG CONFIG: disable-static + +# Local Variables: +# mode:shell-script +# sh-indentation:2 +# End: +# vi:sw=2 + diff --git a/ceph/m4/ac_check_classpath.m4 b/ceph/m4/ac_check_classpath.m4 new file mode 100644 index 00000000..1782b5d3 --- /dev/null +++ b/ceph/m4/ac_check_classpath.m4 @@ -0,0 +1,24 @@ +dnl @synopsis AC_CHECK_CLASSPATH +dnl +dnl AC_CHECK_CLASSPATH just displays the CLASSPATH, for the edification +dnl of the user. +dnl +dnl Note: This is part of the set of autoconf M4 macros for Java +dnl programs. It is VERY IMPORTANT that you download the whole set, +dnl some macros depend on other. Unfortunately, the autoconf archive +dnl does not support the concept of set of macros, so I had to break it +dnl for submission. The general documentation, as well as the sample +dnl configure.in, is included in the AC_PROG_JAVA macro. +dnl +dnl @category Java +dnl @author Stephane Bortzmeyer +dnl @version 2000-07-19 +dnl @license GPLWithACException + +AC_DEFUN([AC_CHECK_CLASSPATH],[ +if test "x$CLASSPATH" = x; then + echo "You have no CLASSPATH, I hope it is good" +else + echo "You have CLASSPATH $CLASSPATH, hope it is correct" +fi +]) diff --git a/ceph/m4/ac_prog_jar.m4 b/ceph/m4/ac_prog_jar.m4 new file mode 100644 index 00000000..e5bf57c0 --- /dev/null +++ b/ceph/m4/ac_prog_jar.m4 @@ -0,0 +1,39 @@ +dnl @synopsis AC_PROG_JAR +dnl +dnl AC_PROG_JAR tests for an existing jar program. It uses the +dnl environment variable JAR then tests in sequence various common jar +dnl programs. +dnl +dnl If you want to force a specific compiler: +dnl +dnl - at the configure.in level, set JAR=yourcompiler before calling +dnl AC_PROG_JAR +dnl +dnl - at the configure level, setenv JAR +dnl +dnl You can use the JAR variable in your Makefile.in, with @JAR@. +dnl +dnl Note: This macro depends on the autoconf M4 macros for Java +dnl programs. It is VERY IMPORTANT that you download that whole set, +dnl some macros depend on other. Unfortunately, the autoconf archive +dnl does not support the concept of set of macros, so I had to break it +dnl for submission. +dnl +dnl The general documentation of those macros, as well as the sample +dnl configure.in, is included in the AC_PROG_JAVA macro. +dnl +dnl @category Java +dnl @author Egon Willighagen +dnl @version 2000-07-19 +dnl @license AllPermissive + +AC_DEFUN([AC_PROG_JAR],[ +AC_REQUIRE([AC_EXEEXT])dnl +if test "x$JAVAPREFIX" = x; then + test "x$JAR" = x && AC_CHECK_PROGS(JAR, jar$EXEEXT) +else + test "x$JAR" = x && AC_CHECK_PROGS(JAR, jar, $JAVAPREFIX) +fi +test "x$JAR" = x && AC_MSG_ERROR([no acceptable jar program found in \$PATH]) +AC_PROVIDE([$0])dnl +]) diff --git a/ceph/m4/ac_prog_javac.m4 b/ceph/m4/ac_prog_javac.m4 new file mode 100644 index 00000000..f6a2fb27 --- /dev/null +++ b/ceph/m4/ac_prog_javac.m4 @@ -0,0 +1,45 @@ +dnl @synopsis AC_PROG_JAVAC +dnl +dnl AC_PROG_JAVAC tests an existing Java compiler. It uses the +dnl environment variable JAVAC then tests in sequence various common +dnl Java compilers. For political reasons, it starts with the free +dnl ones. +dnl +dnl If you want to force a specific compiler: +dnl +dnl - at the configure.in level, set JAVAC=yourcompiler before calling +dnl AC_PROG_JAVAC +dnl +dnl - at the configure level, setenv JAVAC +dnl +dnl You can use the JAVAC variable in your Makefile.in, with @JAVAC@. +dnl +dnl *Warning*: its success or failure can depend on a proper setting of +dnl the CLASSPATH env. variable. +dnl +dnl TODO: allow to exclude compilers (rationale: most Java programs +dnl cannot compile with some compilers like guavac). +dnl +dnl Note: This is part of the set of autoconf M4 macros for Java +dnl programs. It is VERY IMPORTANT that you download the whole set, +dnl some macros depend on other. Unfortunately, the autoconf archive +dnl does not support the concept of set of macros, so I had to break it +dnl for submission. The general documentation, as well as the sample +dnl configure.in, is included in the AC_PROG_JAVA macro. +dnl +dnl @category Java +dnl @author Stephane Bortzmeyer +dnl @version 2000-07-19 +dnl @license GPLWithACException + +AC_DEFUN([AC_PROG_JAVAC],[ +AC_REQUIRE([AC_EXEEXT])dnl +if test "x$JAVAPREFIX" = x; then + test "x$JAVAC" = x && AC_CHECK_PROGS(JAVAC, javac$EXEEXT "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT) +else + test "x$JAVAC" = x && AC_CHECK_PROGS(JAVAC, javac$EXEEXT "gcj$EXEEXT -C" guavac$EXEEXT jikes$EXEEXT, $JAVAPREFIX) +fi +test "x$JAVAC" = x && AC_MSG_ERROR([no acceptable Java compiler found in \$PATH]) +AC_PROG_JAVAC_WORKS +AC_PROVIDE([$0])dnl +]) diff --git a/ceph/m4/ac_prog_javac_works.m4 b/ceph/m4/ac_prog_javac_works.m4 new file mode 100644 index 00000000..8727e7fd --- /dev/null +++ b/ceph/m4/ac_prog_javac_works.m4 @@ -0,0 +1,36 @@ +dnl @synopsis AC_PROG_JAVAC_WORKS +dnl +dnl Internal use ONLY. +dnl +dnl Note: This is part of the set of autoconf M4 macros for Java +dnl programs. It is VERY IMPORTANT that you download the whole set, +dnl some macros depend on other. Unfortunately, the autoconf archive +dnl does not support the concept of set of macros, so I had to break it +dnl for submission. The general documentation, as well as the sample +dnl configure.in, is included in the AC_PROG_JAVA macro. +dnl +dnl @category Java +dnl @author Stephane Bortzmeyer +dnl @version 2000-07-19 +dnl @license GPLWithACException + +AC_DEFUN([AC_PROG_JAVAC_WORKS],[ +AC_CACHE_CHECK([if $JAVAC works], ac_cv_prog_javac_works, [ +JAVA_TEST=Test.java +CLASS_TEST=Test.class +cat << \EOF > $JAVA_TEST +/* [#]line __oline__ "configure" */ +public class Test { +} +EOF +if AC_TRY_COMMAND($JAVAC $JAVACFLAGS $JAVA_TEST) >/dev/null 2>&1; then + ac_cv_prog_javac_works=yes +else + AC_MSG_ERROR([The Java compiler $JAVAC failed (see config.log, check the CLASSPATH?)]) + echo "configure: failed program was:" >&AC_FD_CC + cat $JAVA_TEST >&AC_FD_CC +fi +rm -f $JAVA_TEST $CLASS_TEST +]) +AC_PROVIDE([$0])dnl +]) diff --git a/ceph/m4/ac_prog_javah.m4 b/ceph/m4/ac_prog_javah.m4 new file mode 100644 index 00000000..a1c0556a --- /dev/null +++ b/ceph/m4/ac_prog_javah.m4 @@ -0,0 +1,28 @@ +dnl @synopsis AC_PROG_JAVAH +dnl +dnl AC_PROG_JAVAH tests the availability of the javah header generator +dnl and looks for the jni.h header file. If available, JAVAH is set to +dnl the full path of javah and CPPFLAGS is updated accordingly. +dnl +dnl @category Java +dnl @author Luc Maisonobe +dnl @version 2002-03-25 +dnl @license AllPermissive + +AC_DEFUN([AC_PROG_JAVAH],[ +AC_REQUIRE([AC_CANONICAL_TARGET])dnl +AC_REQUIRE([AC_PROG_CPP])dnl +AC_PATH_PROG(JAVAH,javah) +if test x"`eval 'echo $ac_cv_path_JAVAH'`" != x ; then + AC_TRY_CPP([#include ],,[ + ac_save_CPPFLAGS="$CPPFLAGS" +changequote(, )dnl + ac_dir=`echo $ac_cv_path_JAVAH | sed 's,\(.*\)/[^/]*/[^/]*$,\1/include,'` + ac_machdep=`echo $build_os | sed 's,[-0-9].*,,' | sed 's,cygwin,win32,'` +changequote([, ])dnl + CPPFLAGS="$ac_save_CPPFLAGS -I$ac_dir -I$ac_dir/$ac_machdep" + AC_TRY_CPP([#include ], + ac_save_CPPFLAGS="$CPPFLAGS", + AC_MSG_WARN([unable to include ])) + CPPFLAGS="$ac_save_CPPFLAGS"]) +fi]) diff --git a/ceph/m4/acx_pthread.m4 b/ceph/m4/acx_pthread.m4 new file mode 100644 index 00000000..89d42c74 --- /dev/null +++ b/ceph/m4/acx_pthread.m4 @@ -0,0 +1,397 @@ +# This was retrieved from +# http://svn.0pointer.de/viewvc/trunk/common/acx_pthread.m4?revision=1277&root=avahi +# See also (perhaps for new versions?) +# http://svn.0pointer.de/viewvc/trunk/common/acx_pthread.m4?root=avahi +# +# We've rewritten the inconsistency check code (from avahi), to work +# more broadly. In particular, it no longer assumes ld accepts -zdefs. +# This caused a restructing of the code, but the functionality has only +# changed a little. + +dnl @synopsis ACX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) +dnl +dnl @summary figure out how to build C programs using POSIX threads +dnl +dnl This macro figures out how to build C programs using POSIX threads. +dnl It sets the PTHREAD_LIBS output variable to the threads library and +dnl linker flags, and the PTHREAD_CFLAGS output variable to any special +dnl C compiler flags that are needed. (The user can also force certain +dnl compiler flags/libs to be tested by setting these environment +dnl variables.) +dnl +dnl Also sets PTHREAD_CC to any special C compiler that is needed for +dnl multi-threaded programs (defaults to the value of CC otherwise). +dnl (This is necessary on AIX to use the special cc_r compiler alias.) +dnl +dnl NOTE: You are assumed to not only compile your program with these +dnl flags, but also link it with them as well. e.g. you should link +dnl with $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS +dnl $LIBS +dnl +dnl If you are only building threads programs, you may wish to use +dnl these variables in your default LIBS, CFLAGS, and CC: +dnl +dnl LIBS="$PTHREAD_LIBS $LIBS" +dnl CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +dnl CC="$PTHREAD_CC" +dnl +dnl In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute +dnl constant has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to +dnl that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX). +dnl +dnl ACTION-IF-FOUND is a list of shell commands to run if a threads +dnl library is found, and ACTION-IF-NOT-FOUND is a list of commands to +dnl run it if it is not found. If ACTION-IF-FOUND is not specified, the +dnl default action will define HAVE_PTHREAD. +dnl +dnl Please let the authors know if this macro fails on any platform, or +dnl if you have any other suggestions or comments. This macro was based +dnl on work by SGJ on autoconf scripts for FFTW (www.fftw.org) (with +dnl help from M. Frigo), as well as ac_pthread and hb_pthread macros +dnl posted by Alejandro Forero Cuervo to the autoconf macro repository. +dnl We are also grateful for the helpful feedback of numerous users. +dnl +dnl @category InstalledPackages +dnl @author Steven G. Johnson +dnl @version 2006-05-29 +dnl @license GPLWithACException +dnl +dnl Checks for GCC shared/pthread inconsistency based on work by +dnl Marcin Owsiany + + +AC_DEFUN([ACX_PTHREAD], [ +AC_REQUIRE([AC_CANONICAL_HOST]) +AC_LANG_SAVE +AC_LANG_C +acx_pthread_ok=no + +# We used to check for pthread.h first, but this fails if pthread.h +# requires special compiler flags (e.g. on True64 or Sequent). +# It gets checked for in the link test anyway. + +# First of all, check if the user has set any of the PTHREAD_LIBS, +# etcetera environment variables, and if threads linking works using +# them: +if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS]) + AC_TRY_LINK_FUNC(pthread_join, acx_pthread_ok=yes) + AC_MSG_RESULT($acx_pthread_ok) + if test x"$acx_pthread_ok" = xno; then + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" + fi + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" +fi + +# We must check for the threads library under a number of different +# names; the ordering is very important because some systems +# (e.g. DEC) have both -lpthread and -lpthreads, where one of the +# libraries is broken (non-POSIX). + +# Create a list of thread flags to try. Items starting with a "-" are +# C compiler flags, and other items are library names, except for "none" +# which indicates that we try without any flags at all, and "pthread-config" +# which is a program returning the flags for the Pth emulation library. + +acx_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" + +# The ordering *is* (sometimes) important. Some notes on the +# individual items follow: + +# pthreads: AIX (must check this before -lpthread) +# none: in case threads are in libc; should be tried before -Kthread and +# other compiler flags to prevent continual compiler warnings +# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) +# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) +# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) +# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads) +# -pthreads: Solaris/gcc +# -mthreads: Mingw32/gcc, Lynx/gcc +# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it +# doesn't hurt to check since this sometimes defines pthreads too; +# also defines -D_REENTRANT) +# ... -mt is also the pthreads flag for HP/aCC +# pthread: Linux, etcetera +# --thread-safe: KAI C++ +# pthread-config: use pthread-config program (for GNU Pth library) + +case "${host_cpu}-${host_os}" in + *solaris*) + + # On Solaris (at least, for some versions), libc contains stubbed + # (non-functional) versions of the pthreads routines, so link-based + # tests will erroneously succeed. (We need to link with -pthreads/-mt/ + # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather + # a function called by this macro, so we could check for that, but + # who knows whether they'll stub that too in a future libc.) So, + # we'll just look for -pthreads and -lpthread first: + + acx_pthread_flags="-pthreads pthread -mt -pthread $acx_pthread_flags" + ;; +esac + +if test x"$acx_pthread_ok" = xno; then +for flag in $acx_pthread_flags; do + + case $flag in + none) + AC_MSG_CHECKING([whether pthreads work without any flags]) + ;; + + -*) + AC_MSG_CHECKING([whether pthreads work with $flag]) + PTHREAD_CFLAGS="$flag" + ;; + + pthread-config) + AC_CHECK_PROG(acx_pthread_config, pthread-config, yes, no) + if test x"$acx_pthread_config" = xno; then continue; fi + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; + + *) + AC_MSG_CHECKING([for the pthreads library -l$flag]) + PTHREAD_LIBS="-l$flag" + ;; + esac + + save_LIBS="$LIBS" + save_CFLAGS="$CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Check for various functions. We must include pthread.h, + # since some functions may be macros. (On the Sequent, we + # need a special flag -Kthread to make this header compile.) + # We check for pthread_join because it is in -lpthread on IRIX + # while pthread_create is in libc. We check for pthread_attr_init + # due to DEC craziness with -lpthreads. We check for + # pthread_cleanup_push because it is one of the few pthread + # functions on Solaris that doesn't have a non-functional libc stub. + # We try pthread_create on general principles. + AC_TRY_LINK([#include ], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [acx_pthread_ok=yes]) + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + + AC_MSG_RESULT($acx_pthread_ok) + if test "x$acx_pthread_ok" = xyes; then + break; + fi + + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" +done +fi + +# Various other checks: +if test "x$acx_pthread_ok" = xyes; then + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. + AC_MSG_CHECKING([for joinable pthread attribute]) + attr_name=unknown + for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do + AC_TRY_LINK([#include ], [int attr=$attr; return attr;], + [attr_name=$attr; break]) + done + AC_MSG_RESULT($attr_name) + if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then + AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name, + [Define to necessary symbol if this constant + uses a non-standard name on your system.]) + fi + + AC_MSG_CHECKING([if more special flags are required for pthreads]) + flag=no + case "${host_cpu}-${host_os}" in + *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";; + *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";; + esac + AC_MSG_RESULT(${flag}) + if test "x$flag" != xno; then + PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS" + fi + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + # More AIX lossage: must compile with xlc_r or cc_r + if test x"$GCC" != xyes; then + AC_CHECK_PROGS(PTHREAD_CC, xlc_r cc_r, ${CC}) + else + PTHREAD_CC=$CC + fi + + # The next part tries to detect GCC inconsistency with -shared on some + # architectures and systems. The problem is that in certain + # configurations, when -shared is specified, GCC "forgets" to + # internally use various flags which are still necessary. + + # + # Prepare the flags + # + save_CFLAGS="$CFLAGS" + save_LIBS="$LIBS" + save_CC="$CC" + + # Try with the flags determined by the earlier checks. + # + # -Wl,-z,defs forces link-time symbol resolution, so that the + # linking checks with -shared actually have any value + # + # FIXME: -fPIC is required for -shared on many architectures, + # so we specify it here, but the right way would probably be to + # properly detect whether it is actually required. + CFLAGS="-shared -fPIC -Wl,-z,defs $CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + CC="$PTHREAD_CC" + + # In order not to create several levels of indentation, we test + # the value of "$done" until we find the cure or run out of ideas. + done="no" + + # First, make sure the CFLAGS we added are actually accepted by our + # compiler. If not (and OS X's ld, for instance, does not accept -z), + # then we can't do this test. + if test x"$done" = xno; then + AC_MSG_CHECKING([whether to check for GCC pthread/shared inconsistencies]) + AC_TRY_LINK(,, , [done=yes]) + + if test "x$done" = xyes ; then + AC_MSG_RESULT([no]) + else + AC_MSG_RESULT([yes]) + fi + fi + + if test x"$done" = xno; then + AC_MSG_CHECKING([whether -pthread is sufficient with -shared]) + AC_TRY_LINK([#include ], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + fi + + # + # Linux gcc on some architectures such as mips/mipsel forgets + # about -lpthread + # + if test x"$done" = xno; then + AC_MSG_CHECKING([whether -lpthread fixes that]) + LIBS="-lpthread $PTHREAD_LIBS $save_LIBS" + AC_TRY_LINK([#include ], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + PTHREAD_LIBS="-lpthread $PTHREAD_LIBS" + else + AC_MSG_RESULT([no]) + fi + fi + # + # FreeBSD 4.10 gcc forgets to use -lc_r instead of -lc + # + if test x"$done" = xno; then + AC_MSG_CHECKING([whether -lc_r fixes that]) + LIBS="-lc_r $PTHREAD_LIBS $save_LIBS" + AC_TRY_LINK([#include ], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + PTHREAD_LIBS="-lc_r $PTHREAD_LIBS" + else + AC_MSG_RESULT([no]) + fi + fi + if test x"$done" = xno; then + # OK, we have run out of ideas + AC_MSG_WARN([Impossible to determine how to use pthreads with shared libraries]) + + # so it's not safe to assume that we may use pthreads + acx_pthread_ok=no + fi + + AC_MSG_CHECKING([whether what we have so far is sufficient with -nostdlib]) + CFLAGS="-nostdlib $CFLAGS" + # we need c with nostdlib + LIBS="$LIBS -lc" + AC_TRY_LINK([#include ], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes],[done=no]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + + if test x"$done" = xno; then + AC_MSG_CHECKING([whether -lpthread saves the day]) + LIBS="-lpthread $LIBS" + AC_TRY_LINK([#include ], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes],[done=no]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + PTHREAD_LIBS="$PTHREAD_LIBS -lpthread" + else + AC_MSG_RESULT([no]) + AC_MSG_WARN([Impossible to determine how to use pthreads with shared libraries and -nostdlib]) + fi + fi + + CFLAGS="$save_CFLAGS" + LIBS="$save_LIBS" + CC="$save_CC" +else + PTHREAD_CC="$CC" +fi + +AC_SUBST(PTHREAD_LIBS) +AC_SUBST(PTHREAD_CFLAGS) +AC_SUBST(PTHREAD_CC) + +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test x"$acx_pthread_ok" = xyes; then + ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1]) + : +else + acx_pthread_ok=no + $2 +fi +AC_LANG_RESTORE +])dnl ACX_PTHREAD diff --git a/ceph/m4/ax_c_pretty_func.m4 b/ceph/m4/ax_c_pretty_func.m4 new file mode 100644 index 00000000..ad76709c --- /dev/null +++ b/ceph/m4/ax_c_pretty_func.m4 @@ -0,0 +1,18 @@ +# +# Test for C compiler support of __PRETTY_FUNCTION__ +# +# - Adapted from ax_c_var_func (Noah Watkins) +# + +AU_ALIAS([AC_C_PRETTY_FUNC], [AX_C_PRETTY_FUNC]) +AC_DEFUN([AX_C_PRETTY_FUNC], +[AC_REQUIRE([AC_PROG_CC]) +AC_CACHE_CHECK(whether $CC recognizes __PRETTY_FUNCTION__, ac_cv_c_pretty_func, +AC_TRY_COMPILE(, +[ +char *s = __PRETTY_FUNCTION__; +], +AC_DEFINE(HAVE_PRETTY_FUNC,, +[Define if the C complier supports __PRETTY_FUNCTION__]) ac_cv_c_pretty_func=yes, +ac_cv_c_pretty_func=no) ) +])dnl diff --git a/ceph/m4/ax_c_var_func.m4 b/ceph/m4/ax_c_var_func.m4 new file mode 100644 index 00000000..8b575636 --- /dev/null +++ b/ceph/m4/ax_c_var_func.m4 @@ -0,0 +1,66 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_c_var_func.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_C_VAR_FUNC +# +# DESCRIPTION +# +# This macro tests if the C complier supports the C9X standard __func__ +# indentifier. +# +# The new C9X standard for the C language stipulates that the identifier +# __func__ shall be implictly declared by the compiler as if, immediately +# following the opening brace of each function definition, the declaration +# +# static const char __func__[] = "function-name"; +# +# appeared, where function-name is the name of the function where the +# __func__ identifier is used. +# +# LICENSE +# +# Copyright (c) 2008 Christopher Currie +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 5 + +AU_ALIAS([AC_C_VAR_FUNC], [AX_C_VAR_FUNC]) +AC_DEFUN([AX_C_VAR_FUNC], +[AC_REQUIRE([AC_PROG_CC]) +AC_CACHE_CHECK(whether $CC recognizes __func__, ac_cv_c_var_func, +AC_TRY_COMPILE(, +[ +char *s = __func__; +], +AC_DEFINE(HAVE_FUNC,, +[Define if the C complier supports __func__]) ac_cv_c_var_func=yes, +ac_cv_c_var_func=no) ) +])dnl diff --git a/ceph/m4/ax_check_compile_flag.m4 b/ceph/m4/ax_check_compile_flag.m4 new file mode 100644 index 00000000..c3a8d695 --- /dev/null +++ b/ceph/m4/ax_check_compile_flag.m4 @@ -0,0 +1,72 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/ceph/m4/ax_cxx_static_cast.m4 b/ceph/m4/ax_cxx_static_cast.m4 new file mode 100644 index 00000000..e09e6c46 --- /dev/null +++ b/ceph/m4/ax_cxx_static_cast.m4 @@ -0,0 +1,43 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_cxx_static_cast.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_STATIC_CAST +# +# DESCRIPTION +# +# If the compiler supports static_cast<>, define HAVE_STATIC_CAST. +# +# LICENSE +# +# Copyright (c) 2008 Todd Veldhuizen +# Copyright (c) 2008 Luc Maisonobe +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AU_ALIAS([AC_CXX_STATIC_CAST], [AX_CXX_STATIC_CAST]) +AC_DEFUN([AX_CXX_STATIC_CAST], +[AC_CACHE_CHECK(whether the compiler supports static_cast<>, +ax_cv_cxx_static_cast, +[AC_LANG_SAVE + AC_LANG_CPLUSPLUS + AC_TRY_COMPILE([#include +class Base { public : Base () {} virtual void f () = 0; }; +class Derived : public Base { public : Derived () {} virtual void f () {} }; +int g (Derived&) { return 0; }],[ +Derived d; Base& b = d; Derived& s = static_cast (b); return g (s);], + ax_cv_cxx_static_cast=yes, ax_cv_cxx_static_cast=no) + AC_LANG_RESTORE +]) +if test "$ax_cv_cxx_static_cast" = yes; then + AC_DEFINE(HAVE_STATIC_CAST,, + [define if the compiler supports static_cast<>]) +fi +]) diff --git a/ceph/m4/ax_intel.m4 b/ceph/m4/ax_intel.m4 new file mode 100644 index 00000000..347a3a60 --- /dev/null +++ b/ceph/m4/ax_intel.m4 @@ -0,0 +1,70 @@ +AC_DEFUN([AX_INTEL_FEATURES], +[ + AC_REQUIRE([AC_CANONICAL_HOST]) + + case $target_cpu in + i[[3456]]86*|x86_64*|amd64*) + AX_CHECK_COMPILE_FLAG(-msse, ax_cv_support_sse_ext=yes, []) + if test x"$ax_cv_support_sse_ext" = x"yes"; then + INTEL_SSE_FLAGS="-msse -DINTEL_SSE" + AC_SUBST(INTEL_SSE_FLAGS) + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE_FLAGS" + AC_DEFINE(HAVE_SSE,,[Support SSE (Streaming SIMD Extensions) instructions]) + fi + + AX_CHECK_COMPILE_FLAG(-msse2, ax_cv_support_sse2_ext=yes, []) + if test x"$ax_cv_support_sse2_ext" = x"yes"; then + INTEL_SSE2_FLAGS="-msse2 -DINTEL_SSE2" + AC_SUBST(INTEL_SSE2_FLAGS) + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE2_FLAGS" + AC_DEFINE(HAVE_SSE2,,[Support SSE2 (Streaming SIMD Extensions 2) instructions]) + fi + + AX_CHECK_COMPILE_FLAG(-msse3, ax_cv_support_sse3_ext=yes, []) + if test x"$ax_cv_support_sse3_ext" = x"yes"; then + INTEL_SSE3_FLAGS="-msse3 -DINTEL_SSE3" + AC_SUBST(INTEL_SSE3_FLAGS) + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE3_FLAGS" + AC_DEFINE(HAVE_SSE3,,[Support SSE3 (Streaming SIMD Extensions 3) instructions]) + fi + + AX_CHECK_COMPILE_FLAG(-mssse3, ax_cv_support_ssse3_ext=yes, []) + if test x"$ax_cv_support_ssse3_ext" = x"yes"; then + INTEL_SSSE3_FLAGS="-mssse3 -DINTEL_SSSE3" + AC_SUBST(INTEL_SSSE3_FLAGS) + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSSE3_FLAGS" + AC_DEFINE(HAVE_SSSE3,,[Support SSSE3 (Supplemental Streaming SIMD Extensions 3) instructions]) + fi + ;; + esac + + case $target_cpu in + x86_64*|amd64*) + AX_CHECK_COMPILE_FLAG(-mpclmul, ax_cv_support_pclmuldq_ext=yes, []) + if test x"$ax_cv_support_pclmuldq_ext" = x"yes"; then + INTEL_PCLMUL_FLAGS="-mpclmul -DINTEL_SSE4_PCLMUL" + AC_SUBST(INTEL_PCLMUL_FLAGS) + INTEL_FLAGS="$INTEL_FLAGS $INTEL_PCLMUL_FLAGS" + AC_DEFINE(HAVE_PCLMUL,,[Support (PCLMUL) Carry-Free Muliplication]) + fi + + AX_CHECK_COMPILE_FLAG(-msse4.1, ax_cv_support_sse41_ext=yes, []) + if test x"$ax_cv_support_sse41_ext" = x"yes"; then + INTEL_SSE4_1_FLAGS="-msse4.1 -DINTEL_SSE4" + AC_SUBST(INTEL_SSE4_1_FLAGS) + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE4_1_FLAGS" + AC_DEFINE(HAVE_SSE4_1,,[Support SSE4.1 (Streaming SIMD Extensions 4.1) instructions]) + fi + + AX_CHECK_COMPILE_FLAG(-msse4.2, ax_cv_support_sse42_ext=yes, []) + if test x"$ax_cv_support_sse42_ext" = x"yes"; then + INTEL_SSE4_2_FLAGS="-msse4.2 -DINTEL_SSE4" + AC_SUBST(INTEL_SSE4_2_FLAGS) + INTEL_FLAGS="$INTEL_FLAGS $INTEL_SSE4_2_FLAGS" + AC_DEFINE(HAVE_SSE4_2,,[Support SSE4.2 (Streaming SIMD Extensions 4.2) instructions]) + fi + ;; + esac + + AC_SUBST(INTEL_FLAGS) +]) diff --git a/ceph/m4/libtool.m4 b/ceph/m4/libtool.m4 new file mode 100644 index 00000000..828104cf --- /dev/null +++ b/ceph/m4/libtool.m4 @@ -0,0 +1,8001 @@ +# libtool.m4 - Configure libtool for the host system. -*-Autoconf-*- +# +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005, +# 2006, 2007, 2008, 2009, 2010, 2011 Free Software +# Foundation, Inc. +# Written by Gordon Matzigkeit, 1996 +# +# This file is free software; the Free Software Foundation gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. + +m4_define([_LT_COPYING], [dnl +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005, +# 2006, 2007, 2008, 2009, 2010, 2011 Free Software +# Foundation, Inc. +# Written by Gordon Matzigkeit, 1996 +# +# This file is part of GNU Libtool. +# +# GNU Libtool is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# As a special exception to the GNU General Public License, +# if you distribute this file as part of a program or library that +# is built using GNU Libtool, you may include this file under the +# same distribution terms that you use for the rest of that program. +# +# GNU Libtool is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Libtool; see the file COPYING. If not, a copy +# can be downloaded from http://www.gnu.org/licenses/gpl.html, or +# obtained by writing to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +]) + +# serial 57 LT_INIT + + +# LT_PREREQ(VERSION) +# ------------------ +# Complain and exit if this libtool version is less that VERSION. +m4_defun([LT_PREREQ], +[m4_if(m4_version_compare(m4_defn([LT_PACKAGE_VERSION]), [$1]), -1, + [m4_default([$3], + [m4_fatal([Libtool version $1 or higher is required], + 63)])], + [$2])]) + + +# _LT_CHECK_BUILDDIR +# ------------------ +# Complain if the absolute build directory name contains unusual characters +m4_defun([_LT_CHECK_BUILDDIR], +[case `pwd` in + *\ * | *\ *) + AC_MSG_WARN([Libtool does not cope well with whitespace in `pwd`]) ;; +esac +]) + + +# LT_INIT([OPTIONS]) +# ------------------ +AC_DEFUN([LT_INIT], +[AC_PREREQ([2.58])dnl We use AC_INCLUDES_DEFAULT +AC_REQUIRE([AC_CONFIG_AUX_DIR_DEFAULT])dnl +AC_BEFORE([$0], [LT_LANG])dnl +AC_BEFORE([$0], [LT_OUTPUT])dnl +AC_BEFORE([$0], [LTDL_INIT])dnl +m4_require([_LT_CHECK_BUILDDIR])dnl + +dnl Autoconf doesn't catch unexpanded LT_ macros by default: +m4_pattern_forbid([^_?LT_[A-Z_]+$])dnl +m4_pattern_allow([^(_LT_EOF|LT_DLGLOBAL|LT_DLLAZY_OR_NOW|LT_MULTI_MODULE)$])dnl +dnl aclocal doesn't pull ltoptions.m4, ltsugar.m4, or ltversion.m4 +dnl unless we require an AC_DEFUNed macro: +AC_REQUIRE([LTOPTIONS_VERSION])dnl +AC_REQUIRE([LTSUGAR_VERSION])dnl +AC_REQUIRE([LTVERSION_VERSION])dnl +AC_REQUIRE([LTOBSOLETE_VERSION])dnl +m4_require([_LT_PROG_LTMAIN])dnl + +_LT_SHELL_INIT([SHELL=${CONFIG_SHELL-/bin/sh}]) + +dnl Parse OPTIONS +_LT_SET_OPTIONS([$0], [$1]) + +# This can be used to rebuild libtool when needed +LIBTOOL_DEPS="$ltmain" + +# Always use our own libtool. +LIBTOOL='$(SHELL) $(top_builddir)/libtool' +AC_SUBST(LIBTOOL)dnl + +_LT_SETUP + +# Only expand once: +m4_define([LT_INIT]) +])# LT_INIT + +# Old names: +AU_ALIAS([AC_PROG_LIBTOOL], [LT_INIT]) +AU_ALIAS([AM_PROG_LIBTOOL], [LT_INIT]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_PROG_LIBTOOL], []) +dnl AC_DEFUN([AM_PROG_LIBTOOL], []) + + +# _LT_CC_BASENAME(CC) +# ------------------- +# Calculate cc_basename. Skip known compiler wrappers and cross-prefix. +m4_defun([_LT_CC_BASENAME], +[for cc_temp in $1""; do + case $cc_temp in + compile | *[[\\/]]compile | ccache | *[[\\/]]ccache ) ;; + distcc | *[[\\/]]distcc | purify | *[[\\/]]purify ) ;; + \-*) ;; + *) break;; + esac +done +cc_basename=`$ECHO "$cc_temp" | $SED "s%.*/%%; s%^$host_alias-%%"` +]) + + +# _LT_FILEUTILS_DEFAULTS +# ---------------------- +# It is okay to use these file commands and assume they have been set +# sensibly after `m4_require([_LT_FILEUTILS_DEFAULTS])'. +m4_defun([_LT_FILEUTILS_DEFAULTS], +[: ${CP="cp -f"} +: ${MV="mv -f"} +: ${RM="rm -f"} +])# _LT_FILEUTILS_DEFAULTS + + +# _LT_SETUP +# --------- +m4_defun([_LT_SETUP], +[AC_REQUIRE([AC_CANONICAL_HOST])dnl +AC_REQUIRE([AC_CANONICAL_BUILD])dnl +AC_REQUIRE([_LT_PREPARE_SED_QUOTE_VARS])dnl +AC_REQUIRE([_LT_PROG_ECHO_BACKSLASH])dnl + +_LT_DECL([], [PATH_SEPARATOR], [1], [The PATH separator for the build system])dnl +dnl +_LT_DECL([], [host_alias], [0], [The host system])dnl +_LT_DECL([], [host], [0])dnl +_LT_DECL([], [host_os], [0])dnl +dnl +_LT_DECL([], [build_alias], [0], [The build system])dnl +_LT_DECL([], [build], [0])dnl +_LT_DECL([], [build_os], [0])dnl +dnl +AC_REQUIRE([AC_PROG_CC])dnl +AC_REQUIRE([LT_PATH_LD])dnl +AC_REQUIRE([LT_PATH_NM])dnl +dnl +AC_REQUIRE([AC_PROG_LN_S])dnl +test -z "$LN_S" && LN_S="ln -s" +_LT_DECL([], [LN_S], [1], [Whether we need soft or hard links])dnl +dnl +AC_REQUIRE([LT_CMD_MAX_LEN])dnl +_LT_DECL([objext], [ac_objext], [0], [Object file suffix (normally "o")])dnl +_LT_DECL([], [exeext], [0], [Executable file suffix (normally "")])dnl +dnl +m4_require([_LT_FILEUTILS_DEFAULTS])dnl +m4_require([_LT_CHECK_SHELL_FEATURES])dnl +m4_require([_LT_PATH_CONVERSION_FUNCTIONS])dnl +m4_require([_LT_CMD_RELOAD])dnl +m4_require([_LT_CHECK_MAGIC_METHOD])dnl +m4_require([_LT_CHECK_SHAREDLIB_FROM_LINKLIB])dnl +m4_require([_LT_CMD_OLD_ARCHIVE])dnl +m4_require([_LT_CMD_GLOBAL_SYMBOLS])dnl +m4_require([_LT_WITH_SYSROOT])dnl + +_LT_CONFIG_LIBTOOL_INIT([ +# See if we are running on zsh, and set the options which allow our +# commands through without removal of \ escapes INIT. +if test -n "\${ZSH_VERSION+set}" ; then + setopt NO_GLOB_SUBST +fi +]) +if test -n "${ZSH_VERSION+set}" ; then + setopt NO_GLOB_SUBST +fi + +_LT_CHECK_OBJDIR + +m4_require([_LT_TAG_COMPILER])dnl + +case $host_os in +aix3*) + # AIX sometimes has problems with the GCC collect2 program. For some + # reason, if we set the COLLECT_NAMES environment variable, the problems + # vanish in a puff of smoke. + if test "X${COLLECT_NAMES+set}" != Xset; then + COLLECT_NAMES= + export COLLECT_NAMES + fi + ;; +esac + +# Global variables: +ofile=libtool +can_build_shared=yes + +# All known linkers require a `.a' archive for static linking (except MSVC, +# which needs '.lib'). +libext=a + +with_gnu_ld="$lt_cv_prog_gnu_ld" + +old_CC="$CC" +old_CFLAGS="$CFLAGS" + +# Set sane defaults for various variables +test -z "$CC" && CC=cc +test -z "$LTCC" && LTCC=$CC +test -z "$LTCFLAGS" && LTCFLAGS=$CFLAGS +test -z "$LD" && LD=ld +test -z "$ac_objext" && ac_objext=o + +_LT_CC_BASENAME([$compiler]) + +# Only perform the check for file, if the check method requires it +test -z "$MAGIC_CMD" && MAGIC_CMD=file +case $deplibs_check_method in +file_magic*) + if test "$file_magic_cmd" = '$MAGIC_CMD'; then + _LT_PATH_MAGIC + fi + ;; +esac + +# Use C for the default configuration in the libtool script +LT_SUPPORTED_TAG([CC]) +_LT_LANG_C_CONFIG +_LT_LANG_DEFAULT_CONFIG +_LT_CONFIG_COMMANDS +])# _LT_SETUP + + +# _LT_PREPARE_SED_QUOTE_VARS +# -------------------------- +# Define a few sed substitution that help us do robust quoting. +m4_defun([_LT_PREPARE_SED_QUOTE_VARS], +[# Backslashify metacharacters that are still active within +# double-quoted strings. +sed_quote_subst='s/\([["`$\\]]\)/\\\1/g' + +# Same as above, but do not quote variable references. +double_quote_subst='s/\([["`\\]]\)/\\\1/g' + +# Sed substitution to delay expansion of an escaped shell variable in a +# double_quote_subst'ed string. +delay_variable_subst='s/\\\\\\\\\\\$/\\\\\\$/g' + +# Sed substitution to delay expansion of an escaped single quote. +delay_single_quote_subst='s/'\''/'\'\\\\\\\'\''/g' + +# Sed substitution to avoid accidental globbing in evaled expressions +no_glob_subst='s/\*/\\\*/g' +]) + +# _LT_PROG_LTMAIN +# --------------- +# Note that this code is called both from `configure', and `config.status' +# now that we use AC_CONFIG_COMMANDS to generate libtool. Notably, +# `config.status' has no value for ac_aux_dir unless we are using Automake, +# so we pass a copy along to make sure it has a sensible value anyway. +m4_defun([_LT_PROG_LTMAIN], +[m4_ifdef([AC_REQUIRE_AUX_FILE], [AC_REQUIRE_AUX_FILE([ltmain.sh])])dnl +_LT_CONFIG_LIBTOOL_INIT([ac_aux_dir='$ac_aux_dir']) +ltmain="$ac_aux_dir/ltmain.sh" +])# _LT_PROG_LTMAIN + + +## ------------------------------------- ## +## Accumulate code for creating libtool. ## +## ------------------------------------- ## + +# So that we can recreate a full libtool script including additional +# tags, we accumulate the chunks of code to send to AC_CONFIG_COMMANDS +# in macros and then make a single call at the end using the `libtool' +# label. + + +# _LT_CONFIG_LIBTOOL_INIT([INIT-COMMANDS]) +# ---------------------------------------- +# Register INIT-COMMANDS to be passed to AC_CONFIG_COMMANDS later. +m4_define([_LT_CONFIG_LIBTOOL_INIT], +[m4_ifval([$1], + [m4_append([_LT_OUTPUT_LIBTOOL_INIT], + [$1 +])])]) + +# Initialize. +m4_define([_LT_OUTPUT_LIBTOOL_INIT]) + + +# _LT_CONFIG_LIBTOOL([COMMANDS]) +# ------------------------------ +# Register COMMANDS to be passed to AC_CONFIG_COMMANDS later. +m4_define([_LT_CONFIG_LIBTOOL], +[m4_ifval([$1], + [m4_append([_LT_OUTPUT_LIBTOOL_COMMANDS], + [$1 +])])]) + +# Initialize. +m4_define([_LT_OUTPUT_LIBTOOL_COMMANDS]) + + +# _LT_CONFIG_SAVE_COMMANDS([COMMANDS], [INIT_COMMANDS]) +# ----------------------------------------------------- +m4_defun([_LT_CONFIG_SAVE_COMMANDS], +[_LT_CONFIG_LIBTOOL([$1]) +_LT_CONFIG_LIBTOOL_INIT([$2]) +]) + + +# _LT_FORMAT_COMMENT([COMMENT]) +# ----------------------------- +# Add leading comment marks to the start of each line, and a trailing +# full-stop to the whole comment if one is not present already. +m4_define([_LT_FORMAT_COMMENT], +[m4_ifval([$1], [ +m4_bpatsubst([m4_bpatsubst([$1], [^ *], [# ])], + [['`$\]], [\\\&])]m4_bmatch([$1], [[!?.]$], [], [.]) +)]) + + + +## ------------------------ ## +## FIXME: Eliminate VARNAME ## +## ------------------------ ## + + +# _LT_DECL([CONFIGNAME], VARNAME, VALUE, [DESCRIPTION], [IS-TAGGED?]) +# ------------------------------------------------------------------- +# CONFIGNAME is the name given to the value in the libtool script. +# VARNAME is the (base) name used in the configure script. +# VALUE may be 0, 1 or 2 for a computed quote escaped value based on +# VARNAME. Any other value will be used directly. +m4_define([_LT_DECL], +[lt_if_append_uniq([lt_decl_varnames], [$2], [, ], + [lt_dict_add_subkey([lt_decl_dict], [$2], [libtool_name], + [m4_ifval([$1], [$1], [$2])]) + lt_dict_add_subkey([lt_decl_dict], [$2], [value], [$3]) + m4_ifval([$4], + [lt_dict_add_subkey([lt_decl_dict], [$2], [description], [$4])]) + lt_dict_add_subkey([lt_decl_dict], [$2], + [tagged?], [m4_ifval([$5], [yes], [no])])]) +]) + + +# _LT_TAGDECL([CONFIGNAME], VARNAME, VALUE, [DESCRIPTION]) +# -------------------------------------------------------- +m4_define([_LT_TAGDECL], [_LT_DECL([$1], [$2], [$3], [$4], [yes])]) + + +# lt_decl_tag_varnames([SEPARATOR], [VARNAME1...]) +# ------------------------------------------------ +m4_define([lt_decl_tag_varnames], +[_lt_decl_filter([tagged?], [yes], $@)]) + + +# _lt_decl_filter(SUBKEY, VALUE, [SEPARATOR], [VARNAME1..]) +# --------------------------------------------------------- +m4_define([_lt_decl_filter], +[m4_case([$#], + [0], [m4_fatal([$0: too few arguments: $#])], + [1], [m4_fatal([$0: too few arguments: $#: $1])], + [2], [lt_dict_filter([lt_decl_dict], [$1], [$2], [], lt_decl_varnames)], + [3], [lt_dict_filter([lt_decl_dict], [$1], [$2], [$3], lt_decl_varnames)], + [lt_dict_filter([lt_decl_dict], $@)])[]dnl +]) + + +# lt_decl_quote_varnames([SEPARATOR], [VARNAME1...]) +# -------------------------------------------------- +m4_define([lt_decl_quote_varnames], +[_lt_decl_filter([value], [1], $@)]) + + +# lt_decl_dquote_varnames([SEPARATOR], [VARNAME1...]) +# --------------------------------------------------- +m4_define([lt_decl_dquote_varnames], +[_lt_decl_filter([value], [2], $@)]) + + +# lt_decl_varnames_tagged([SEPARATOR], [VARNAME1...]) +# --------------------------------------------------- +m4_define([lt_decl_varnames_tagged], +[m4_assert([$# <= 2])dnl +_$0(m4_quote(m4_default([$1], [[, ]])), + m4_ifval([$2], [[$2]], [m4_dquote(lt_decl_tag_varnames)]), + m4_split(m4_normalize(m4_quote(_LT_TAGS)), [ ]))]) +m4_define([_lt_decl_varnames_tagged], +[m4_ifval([$3], [lt_combine([$1], [$2], [_], $3)])]) + + +# lt_decl_all_varnames([SEPARATOR], [VARNAME1...]) +# ------------------------------------------------ +m4_define([lt_decl_all_varnames], +[_$0(m4_quote(m4_default([$1], [[, ]])), + m4_if([$2], [], + m4_quote(lt_decl_varnames), + m4_quote(m4_shift($@))))[]dnl +]) +m4_define([_lt_decl_all_varnames], +[lt_join($@, lt_decl_varnames_tagged([$1], + lt_decl_tag_varnames([[, ]], m4_shift($@))))dnl +]) + + +# _LT_CONFIG_STATUS_DECLARE([VARNAME]) +# ------------------------------------ +# Quote a variable value, and forward it to `config.status' so that its +# declaration there will have the same value as in `configure'. VARNAME +# must have a single quote delimited value for this to work. +m4_define([_LT_CONFIG_STATUS_DECLARE], +[$1='`$ECHO "$][$1" | $SED "$delay_single_quote_subst"`']) + + +# _LT_CONFIG_STATUS_DECLARATIONS +# ------------------------------ +# We delimit libtool config variables with single quotes, so when +# we write them to config.status, we have to be sure to quote all +# embedded single quotes properly. In configure, this macro expands +# each variable declared with _LT_DECL (and _LT_TAGDECL) into: +# +# ='`$ECHO "$" | $SED "$delay_single_quote_subst"`' +m4_defun([_LT_CONFIG_STATUS_DECLARATIONS], +[m4_foreach([_lt_var], m4_quote(lt_decl_all_varnames), + [m4_n([_LT_CONFIG_STATUS_DECLARE(_lt_var)])])]) + + +# _LT_LIBTOOL_TAGS +# ---------------- +# Output comment and list of tags supported by the script +m4_defun([_LT_LIBTOOL_TAGS], +[_LT_FORMAT_COMMENT([The names of the tagged configurations supported by this script])dnl +available_tags="_LT_TAGS"dnl +]) + + +# _LT_LIBTOOL_DECLARE(VARNAME, [TAG]) +# ----------------------------------- +# Extract the dictionary values for VARNAME (optionally with TAG) and +# expand to a commented shell variable setting: +# +# # Some comment about what VAR is for. +# visible_name=$lt_internal_name +m4_define([_LT_LIBTOOL_DECLARE], +[_LT_FORMAT_COMMENT(m4_quote(lt_dict_fetch([lt_decl_dict], [$1], + [description])))[]dnl +m4_pushdef([_libtool_name], + m4_quote(lt_dict_fetch([lt_decl_dict], [$1], [libtool_name])))[]dnl +m4_case(m4_quote(lt_dict_fetch([lt_decl_dict], [$1], [value])), + [0], [_libtool_name=[$]$1], + [1], [_libtool_name=$lt_[]$1], + [2], [_libtool_name=$lt_[]$1], + [_libtool_name=lt_dict_fetch([lt_decl_dict], [$1], [value])])[]dnl +m4_ifval([$2], [_$2])[]m4_popdef([_libtool_name])[]dnl +]) + + +# _LT_LIBTOOL_CONFIG_VARS +# ----------------------- +# Produce commented declarations of non-tagged libtool config variables +# suitable for insertion in the LIBTOOL CONFIG section of the `libtool' +# script. Tagged libtool config variables (even for the LIBTOOL CONFIG +# section) are produced by _LT_LIBTOOL_TAG_VARS. +m4_defun([_LT_LIBTOOL_CONFIG_VARS], +[m4_foreach([_lt_var], + m4_quote(_lt_decl_filter([tagged?], [no], [], lt_decl_varnames)), + [m4_n([_LT_LIBTOOL_DECLARE(_lt_var)])])]) + + +# _LT_LIBTOOL_TAG_VARS(TAG) +# ------------------------- +m4_define([_LT_LIBTOOL_TAG_VARS], +[m4_foreach([_lt_var], m4_quote(lt_decl_tag_varnames), + [m4_n([_LT_LIBTOOL_DECLARE(_lt_var, [$1])])])]) + + +# _LT_TAGVAR(VARNAME, [TAGNAME]) +# ------------------------------ +m4_define([_LT_TAGVAR], [m4_ifval([$2], [$1_$2], [$1])]) + + +# _LT_CONFIG_COMMANDS +# ------------------- +# Send accumulated output to $CONFIG_STATUS. Thanks to the lists of +# variables for single and double quote escaping we saved from calls +# to _LT_DECL, we can put quote escaped variables declarations +# into `config.status', and then the shell code to quote escape them in +# for loops in `config.status'. Finally, any additional code accumulated +# from calls to _LT_CONFIG_LIBTOOL_INIT is expanded. +m4_defun([_LT_CONFIG_COMMANDS], +[AC_PROVIDE_IFELSE([LT_OUTPUT], + dnl If the libtool generation code has been placed in $CONFIG_LT, + dnl instead of duplicating it all over again into config.status, + dnl then we will have config.status run $CONFIG_LT later, so it + dnl needs to know what name is stored there: + [AC_CONFIG_COMMANDS([libtool], + [$SHELL $CONFIG_LT || AS_EXIT(1)], [CONFIG_LT='$CONFIG_LT'])], + dnl If the libtool generation code is destined for config.status, + dnl expand the accumulated commands and init code now: + [AC_CONFIG_COMMANDS([libtool], + [_LT_OUTPUT_LIBTOOL_COMMANDS], [_LT_OUTPUT_LIBTOOL_COMMANDS_INIT])]) +])#_LT_CONFIG_COMMANDS + + +# Initialize. +m4_define([_LT_OUTPUT_LIBTOOL_COMMANDS_INIT], +[ + +# The HP-UX ksh and POSIX shell print the target directory to stdout +# if CDPATH is set. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +sed_quote_subst='$sed_quote_subst' +double_quote_subst='$double_quote_subst' +delay_variable_subst='$delay_variable_subst' +_LT_CONFIG_STATUS_DECLARATIONS +LTCC='$LTCC' +LTCFLAGS='$LTCFLAGS' +compiler='$compiler_DEFAULT' + +# A function that is used when there is no print builtin or printf. +func_fallback_echo () +{ + eval 'cat <<_LTECHO_EOF +\$[]1 +_LTECHO_EOF' +} + +# Quote evaled strings. +for var in lt_decl_all_varnames([[ \ +]], lt_decl_quote_varnames); do + case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in + *[[\\\\\\\`\\"\\\$]]*) + eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED \\"\\\$sed_quote_subst\\"\\\`\\\\\\"" + ;; + *) + eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" + ;; + esac +done + +# Double-quote double-evaled strings. +for var in lt_decl_all_varnames([[ \ +]], lt_decl_dquote_varnames); do + case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in + *[[\\\\\\\`\\"\\\$]]*) + eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED -e \\"\\\$double_quote_subst\\" -e \\"\\\$sed_quote_subst\\" -e \\"\\\$delay_variable_subst\\"\\\`\\\\\\"" + ;; + *) + eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" + ;; + esac +done + +_LT_OUTPUT_LIBTOOL_INIT +]) + +# _LT_GENERATED_FILE_INIT(FILE, [COMMENT]) +# ------------------------------------ +# Generate a child script FILE with all initialization necessary to +# reuse the environment learned by the parent script, and make the +# file executable. If COMMENT is supplied, it is inserted after the +# `#!' sequence but before initialization text begins. After this +# macro, additional text can be appended to FILE to form the body of +# the child script. The macro ends with non-zero status if the +# file could not be fully written (such as if the disk is full). +m4_ifdef([AS_INIT_GENERATED], +[m4_defun([_LT_GENERATED_FILE_INIT],[AS_INIT_GENERATED($@)])], +[m4_defun([_LT_GENERATED_FILE_INIT], +[m4_require([AS_PREPARE])]dnl +[m4_pushdef([AS_MESSAGE_LOG_FD])]dnl +[lt_write_fail=0 +cat >$1 <<_ASEOF || lt_write_fail=1 +#! $SHELL +# Generated by $as_me. +$2 +SHELL=\${CONFIG_SHELL-$SHELL} +export SHELL +_ASEOF +cat >>$1 <<\_ASEOF || lt_write_fail=1 +AS_SHELL_SANITIZE +_AS_PREPARE +exec AS_MESSAGE_FD>&1 +_ASEOF +test $lt_write_fail = 0 && chmod +x $1[]dnl +m4_popdef([AS_MESSAGE_LOG_FD])])])# _LT_GENERATED_FILE_INIT + +# LT_OUTPUT +# --------- +# This macro allows early generation of the libtool script (before +# AC_OUTPUT is called), incase it is used in configure for compilation +# tests. +AC_DEFUN([LT_OUTPUT], +[: ${CONFIG_LT=./config.lt} +AC_MSG_NOTICE([creating $CONFIG_LT]) +_LT_GENERATED_FILE_INIT(["$CONFIG_LT"], +[# Run this file to recreate a libtool stub with the current configuration.]) + +cat >>"$CONFIG_LT" <<\_LTEOF +lt_cl_silent=false +exec AS_MESSAGE_LOG_FD>>config.log +{ + echo + AS_BOX([Running $as_me.]) +} >&AS_MESSAGE_LOG_FD + +lt_cl_help="\ +\`$as_me' creates a local libtool stub from the current configuration, +for use in further configure time tests before the real libtool is +generated. + +Usage: $[0] [[OPTIONS]] + + -h, --help print this help, then exit + -V, --version print version number, then exit + -q, --quiet do not print progress messages + -d, --debug don't remove temporary files + +Report bugs to ." + +lt_cl_version="\ +m4_ifset([AC_PACKAGE_NAME], [AC_PACKAGE_NAME ])config.lt[]dnl +m4_ifset([AC_PACKAGE_VERSION], [ AC_PACKAGE_VERSION]) +configured by $[0], generated by m4_PACKAGE_STRING. + +Copyright (C) 2011 Free Software Foundation, Inc. +This config.lt script is free software; the Free Software Foundation +gives unlimited permision to copy, distribute and modify it." + +while test $[#] != 0 +do + case $[1] in + --version | --v* | -V ) + echo "$lt_cl_version"; exit 0 ;; + --help | --h* | -h ) + echo "$lt_cl_help"; exit 0 ;; + --debug | --d* | -d ) + debug=: ;; + --quiet | --q* | --silent | --s* | -q ) + lt_cl_silent=: ;; + + -*) AC_MSG_ERROR([unrecognized option: $[1] +Try \`$[0] --help' for more information.]) ;; + + *) AC_MSG_ERROR([unrecognized argument: $[1] +Try \`$[0] --help' for more information.]) ;; + esac + shift +done + +if $lt_cl_silent; then + exec AS_MESSAGE_FD>/dev/null +fi +_LTEOF + +cat >>"$CONFIG_LT" <<_LTEOF +_LT_OUTPUT_LIBTOOL_COMMANDS_INIT +_LTEOF + +cat >>"$CONFIG_LT" <<\_LTEOF +AC_MSG_NOTICE([creating $ofile]) +_LT_OUTPUT_LIBTOOL_COMMANDS +AS_EXIT(0) +_LTEOF +chmod +x "$CONFIG_LT" + +# configure is writing to config.log, but config.lt does its own redirection, +# appending to config.log, which fails on DOS, as config.log is still kept +# open by configure. Here we exec the FD to /dev/null, effectively closing +# config.log, so it can be properly (re)opened and appended to by config.lt. +lt_cl_success=: +test "$silent" = yes && + lt_config_lt_args="$lt_config_lt_args --quiet" +exec AS_MESSAGE_LOG_FD>/dev/null +$SHELL "$CONFIG_LT" $lt_config_lt_args || lt_cl_success=false +exec AS_MESSAGE_LOG_FD>>config.log +$lt_cl_success || AS_EXIT(1) +])# LT_OUTPUT + + +# _LT_CONFIG(TAG) +# --------------- +# If TAG is the built-in tag, create an initial libtool script with a +# default configuration from the untagged config vars. Otherwise add code +# to config.status for appending the configuration named by TAG from the +# matching tagged config vars. +m4_defun([_LT_CONFIG], +[m4_require([_LT_FILEUTILS_DEFAULTS])dnl +_LT_CONFIG_SAVE_COMMANDS([ + m4_define([_LT_TAG], m4_if([$1], [], [C], [$1]))dnl + m4_if(_LT_TAG, [C], [ + # See if we are running on zsh, and set the options which allow our + # commands through without removal of \ escapes. + if test -n "${ZSH_VERSION+set}" ; then + setopt NO_GLOB_SUBST + fi + + cfgfile="${ofile}T" + trap "$RM \"$cfgfile\"; exit 1" 1 2 15 + $RM "$cfgfile" + + cat <<_LT_EOF >> "$cfgfile" +#! $SHELL + +# `$ECHO "$ofile" | sed 's%^.*/%%'` - Provide generalized library-building support services. +# Generated automatically by $as_me ($PACKAGE$TIMESTAMP) $VERSION +# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: +# NOTE: Changes made to this file will be lost: look at ltmain.sh. +# +_LT_COPYING +_LT_LIBTOOL_TAGS + +# ### BEGIN LIBTOOL CONFIG +_LT_LIBTOOL_CONFIG_VARS +_LT_LIBTOOL_TAG_VARS +# ### END LIBTOOL CONFIG + +_LT_EOF + + case $host_os in + aix3*) + cat <<\_LT_EOF >> "$cfgfile" +# AIX sometimes has problems with the GCC collect2 program. For some +# reason, if we set the COLLECT_NAMES environment variable, the problems +# vanish in a puff of smoke. +if test "X${COLLECT_NAMES+set}" != Xset; then + COLLECT_NAMES= + export COLLECT_NAMES +fi +_LT_EOF + ;; + esac + + _LT_PROG_LTMAIN + + # We use sed instead of cat because bash on DJGPP gets confused if + # if finds mixed CR/LF and LF-only lines. Since sed operates in + # text mode, it properly converts lines to CR/LF. This bash problem + # is reportedly fixed, but why not run on old versions too? + sed '$q' "$ltmain" >> "$cfgfile" \ + || (rm -f "$cfgfile"; exit 1) + + _LT_PROG_REPLACE_SHELLFNS + + mv -f "$cfgfile" "$ofile" || + (rm -f "$ofile" && cp "$cfgfile" "$ofile" && rm -f "$cfgfile") + chmod +x "$ofile" +], +[cat <<_LT_EOF >> "$ofile" + +dnl Unfortunately we have to use $1 here, since _LT_TAG is not expanded +dnl in a comment (ie after a #). +# ### BEGIN LIBTOOL TAG CONFIG: $1 +_LT_LIBTOOL_TAG_VARS(_LT_TAG) +# ### END LIBTOOL TAG CONFIG: $1 +_LT_EOF +])dnl /m4_if +], +[m4_if([$1], [], [ + PACKAGE='$PACKAGE' + VERSION='$VERSION' + TIMESTAMP='$TIMESTAMP' + RM='$RM' + ofile='$ofile'], []) +])dnl /_LT_CONFIG_SAVE_COMMANDS +])# _LT_CONFIG + + +# LT_SUPPORTED_TAG(TAG) +# --------------------- +# Trace this macro to discover what tags are supported by the libtool +# --tag option, using: +# autoconf --trace 'LT_SUPPORTED_TAG:$1' +AC_DEFUN([LT_SUPPORTED_TAG], []) + + +# C support is built-in for now +m4_define([_LT_LANG_C_enabled], []) +m4_define([_LT_TAGS], []) + + +# LT_LANG(LANG) +# ------------- +# Enable libtool support for the given language if not already enabled. +AC_DEFUN([LT_LANG], +[AC_BEFORE([$0], [LT_OUTPUT])dnl +m4_case([$1], + [C], [_LT_LANG(C)], + [C++], [_LT_LANG(CXX)], + [Go], [_LT_LANG(GO)], + [Java], [_LT_LANG(GCJ)], + [Fortran 77], [_LT_LANG(F77)], + [Fortran], [_LT_LANG(FC)], + [Windows Resource], [_LT_LANG(RC)], + [m4_ifdef([_LT_LANG_]$1[_CONFIG], + [_LT_LANG($1)], + [m4_fatal([$0: unsupported language: "$1"])])])dnl +])# LT_LANG + + +# _LT_LANG(LANGNAME) +# ------------------ +m4_defun([_LT_LANG], +[m4_ifdef([_LT_LANG_]$1[_enabled], [], + [LT_SUPPORTED_TAG([$1])dnl + m4_append([_LT_TAGS], [$1 ])dnl + m4_define([_LT_LANG_]$1[_enabled], [])dnl + _LT_LANG_$1_CONFIG($1)])dnl +])# _LT_LANG + + +m4_ifndef([AC_PROG_GO], [ +############################################################ +# NOTE: This macro has been submitted for inclusion into # +# GNU Autoconf as AC_PROG_GO. When it is available in # +# a released version of Autoconf we should remove this # +# macro and use it instead. # +############################################################ +m4_defun([AC_PROG_GO], +[AC_LANG_PUSH(Go)dnl +AC_ARG_VAR([GOC], [Go compiler command])dnl +AC_ARG_VAR([GOFLAGS], [Go compiler flags])dnl +_AC_ARG_VAR_LDFLAGS()dnl +AC_CHECK_TOOL(GOC, gccgo) +if test -z "$GOC"; then + if test -n "$ac_tool_prefix"; then + AC_CHECK_PROG(GOC, [${ac_tool_prefix}gccgo], [${ac_tool_prefix}gccgo]) + fi +fi +if test -z "$GOC"; then + AC_CHECK_PROG(GOC, gccgo, gccgo, false) +fi +])#m4_defun +])#m4_ifndef + + +# _LT_LANG_DEFAULT_CONFIG +# ----------------------- +m4_defun([_LT_LANG_DEFAULT_CONFIG], +[AC_PROVIDE_IFELSE([AC_PROG_CXX], + [LT_LANG(CXX)], + [m4_define([AC_PROG_CXX], defn([AC_PROG_CXX])[LT_LANG(CXX)])]) + +AC_PROVIDE_IFELSE([AC_PROG_F77], + [LT_LANG(F77)], + [m4_define([AC_PROG_F77], defn([AC_PROG_F77])[LT_LANG(F77)])]) + +AC_PROVIDE_IFELSE([AC_PROG_FC], + [LT_LANG(FC)], + [m4_define([AC_PROG_FC], defn([AC_PROG_FC])[LT_LANG(FC)])]) + +dnl The call to [A][M_PROG_GCJ] is quoted like that to stop aclocal +dnl pulling things in needlessly. +AC_PROVIDE_IFELSE([AC_PROG_GCJ], + [LT_LANG(GCJ)], + [AC_PROVIDE_IFELSE([A][M_PROG_GCJ], + [LT_LANG(GCJ)], + [AC_PROVIDE_IFELSE([LT_PROG_GCJ], + [LT_LANG(GCJ)], + [m4_ifdef([AC_PROG_GCJ], + [m4_define([AC_PROG_GCJ], defn([AC_PROG_GCJ])[LT_LANG(GCJ)])]) + m4_ifdef([A][M_PROG_GCJ], + [m4_define([A][M_PROG_GCJ], defn([A][M_PROG_GCJ])[LT_LANG(GCJ)])]) + m4_ifdef([LT_PROG_GCJ], + [m4_define([LT_PROG_GCJ], defn([LT_PROG_GCJ])[LT_LANG(GCJ)])])])])]) + +AC_PROVIDE_IFELSE([AC_PROG_GO], + [LT_LANG(GO)], + [m4_define([AC_PROG_GO], defn([AC_PROG_GO])[LT_LANG(GO)])]) + +AC_PROVIDE_IFELSE([LT_PROG_RC], + [LT_LANG(RC)], + [m4_define([LT_PROG_RC], defn([LT_PROG_RC])[LT_LANG(RC)])]) +])# _LT_LANG_DEFAULT_CONFIG + +# Obsolete macros: +AU_DEFUN([AC_LIBTOOL_CXX], [LT_LANG(C++)]) +AU_DEFUN([AC_LIBTOOL_F77], [LT_LANG(Fortran 77)]) +AU_DEFUN([AC_LIBTOOL_FC], [LT_LANG(Fortran)]) +AU_DEFUN([AC_LIBTOOL_GCJ], [LT_LANG(Java)]) +AU_DEFUN([AC_LIBTOOL_RC], [LT_LANG(Windows Resource)]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_CXX], []) +dnl AC_DEFUN([AC_LIBTOOL_F77], []) +dnl AC_DEFUN([AC_LIBTOOL_FC], []) +dnl AC_DEFUN([AC_LIBTOOL_GCJ], []) +dnl AC_DEFUN([AC_LIBTOOL_RC], []) + + +# _LT_TAG_COMPILER +# ---------------- +m4_defun([_LT_TAG_COMPILER], +[AC_REQUIRE([AC_PROG_CC])dnl + +_LT_DECL([LTCC], [CC], [1], [A C compiler])dnl +_LT_DECL([LTCFLAGS], [CFLAGS], [1], [LTCC compiler flags])dnl +_LT_TAGDECL([CC], [compiler], [1], [A language specific compiler])dnl +_LT_TAGDECL([with_gcc], [GCC], [0], [Is the compiler the GNU compiler?])dnl + +# If no C compiler was specified, use CC. +LTCC=${LTCC-"$CC"} + +# If no C compiler flags were specified, use CFLAGS. +LTCFLAGS=${LTCFLAGS-"$CFLAGS"} + +# Allow CC to be a program name with arguments. +compiler=$CC +])# _LT_TAG_COMPILER + + +# _LT_COMPILER_BOILERPLATE +# ------------------------ +# Check for compiler boilerplate output or warnings with +# the simple compiler test code. +m4_defun([_LT_COMPILER_BOILERPLATE], +[m4_require([_LT_DECL_SED])dnl +ac_outfile=conftest.$ac_objext +echo "$lt_simple_compile_test_code" >conftest.$ac_ext +eval "$ac_compile" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err +_lt_compiler_boilerplate=`cat conftest.err` +$RM conftest* +])# _LT_COMPILER_BOILERPLATE + + +# _LT_LINKER_BOILERPLATE +# ---------------------- +# Check for linker boilerplate output or warnings with +# the simple link test code. +m4_defun([_LT_LINKER_BOILERPLATE], +[m4_require([_LT_DECL_SED])dnl +ac_outfile=conftest.$ac_objext +echo "$lt_simple_link_test_code" >conftest.$ac_ext +eval "$ac_link" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err +_lt_linker_boilerplate=`cat conftest.err` +$RM -r conftest* +])# _LT_LINKER_BOILERPLATE + +# _LT_REQUIRED_DARWIN_CHECKS +# ------------------------- +m4_defun_once([_LT_REQUIRED_DARWIN_CHECKS],[ + case $host_os in + rhapsody* | darwin*) + AC_CHECK_TOOL([DSYMUTIL], [dsymutil], [:]) + AC_CHECK_TOOL([NMEDIT], [nmedit], [:]) + AC_CHECK_TOOL([LIPO], [lipo], [:]) + AC_CHECK_TOOL([OTOOL], [otool], [:]) + AC_CHECK_TOOL([OTOOL64], [otool64], [:]) + _LT_DECL([], [DSYMUTIL], [1], + [Tool to manipulate archived DWARF debug symbol files on Mac OS X]) + _LT_DECL([], [NMEDIT], [1], + [Tool to change global to local symbols on Mac OS X]) + _LT_DECL([], [LIPO], [1], + [Tool to manipulate fat objects and archives on Mac OS X]) + _LT_DECL([], [OTOOL], [1], + [ldd/readelf like tool for Mach-O binaries on Mac OS X]) + _LT_DECL([], [OTOOL64], [1], + [ldd/readelf like tool for 64 bit Mach-O binaries on Mac OS X 10.4]) + + AC_CACHE_CHECK([for -single_module linker flag],[lt_cv_apple_cc_single_mod], + [lt_cv_apple_cc_single_mod=no + if test -z "${LT_MULTI_MODULE}"; then + # By default we will add the -single_module flag. You can override + # by either setting the environment variable LT_MULTI_MODULE + # non-empty at configure time, or by adding -multi_module to the + # link flags. + rm -rf libconftest.dylib* + echo "int foo(void){return 1;}" > conftest.c + echo "$LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ +-dynamiclib -Wl,-single_module conftest.c" >&AS_MESSAGE_LOG_FD + $LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ + -dynamiclib -Wl,-single_module conftest.c 2>conftest.err + _lt_result=$? + # If there is a non-empty error log, and "single_module" + # appears in it, assume the flag caused a linker warning + if test -s conftest.err && $GREP single_module conftest.err; then + cat conftest.err >&AS_MESSAGE_LOG_FD + # Otherwise, if the output was created with a 0 exit code from + # the compiler, it worked. + elif test -f libconftest.dylib && test $_lt_result -eq 0; then + lt_cv_apple_cc_single_mod=yes + else + cat conftest.err >&AS_MESSAGE_LOG_FD + fi + rm -rf libconftest.dylib* + rm -f conftest.* + fi]) + + AC_CACHE_CHECK([for -exported_symbols_list linker flag], + [lt_cv_ld_exported_symbols_list], + [lt_cv_ld_exported_symbols_list=no + save_LDFLAGS=$LDFLAGS + echo "_main" > conftest.sym + LDFLAGS="$LDFLAGS -Wl,-exported_symbols_list,conftest.sym" + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[])], + [lt_cv_ld_exported_symbols_list=yes], + [lt_cv_ld_exported_symbols_list=no]) + LDFLAGS="$save_LDFLAGS" + ]) + + AC_CACHE_CHECK([for -force_load linker flag],[lt_cv_ld_force_load], + [lt_cv_ld_force_load=no + cat > conftest.c << _LT_EOF +int forced_loaded() { return 2;} +_LT_EOF + echo "$LTCC $LTCFLAGS -c -o conftest.o conftest.c" >&AS_MESSAGE_LOG_FD + $LTCC $LTCFLAGS -c -o conftest.o conftest.c 2>&AS_MESSAGE_LOG_FD + echo "$AR cru libconftest.a conftest.o" >&AS_MESSAGE_LOG_FD + $AR cru libconftest.a conftest.o 2>&AS_MESSAGE_LOG_FD + echo "$RANLIB libconftest.a" >&AS_MESSAGE_LOG_FD + $RANLIB libconftest.a 2>&AS_MESSAGE_LOG_FD + cat > conftest.c << _LT_EOF +int main() { return 0;} +_LT_EOF + echo "$LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a" >&AS_MESSAGE_LOG_FD + $LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a 2>conftest.err + _lt_result=$? + if test -s conftest.err && $GREP force_load conftest.err; then + cat conftest.err >&AS_MESSAGE_LOG_FD + elif test -f conftest && test $_lt_result -eq 0 && $GREP forced_load conftest >/dev/null 2>&1 ; then + lt_cv_ld_force_load=yes + else + cat conftest.err >&AS_MESSAGE_LOG_FD + fi + rm -f conftest.err libconftest.a conftest conftest.c + rm -rf conftest.dSYM + ]) + case $host_os in + rhapsody* | darwin1.[[012]]) + _lt_dar_allow_undefined='${wl}-undefined ${wl}suppress' ;; + darwin1.*) + _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;; + darwin*) # darwin 5.x on + # if running on 10.5 or later, the deployment target defaults + # to the OS version, if on x86, and 10.4, the deployment + # target defaults to 10.4. Don't you love it? + case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in + 10.0,*86*-darwin8*|10.0,*-darwin[[91]]*) + _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;; + 10.[[012]]*) + _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;; + 10.*) + _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;; + esac + ;; + esac + if test "$lt_cv_apple_cc_single_mod" = "yes"; then + _lt_dar_single_mod='$single_module' + fi + if test "$lt_cv_ld_exported_symbols_list" = "yes"; then + _lt_dar_export_syms=' ${wl}-exported_symbols_list,$output_objdir/${libname}-symbols.expsym' + else + _lt_dar_export_syms='~$NMEDIT -s $output_objdir/${libname}-symbols.expsym ${lib}' + fi + if test "$DSYMUTIL" != ":" && test "$lt_cv_ld_force_load" = "no"; then + _lt_dsymutil='~$DSYMUTIL $lib || :' + else + _lt_dsymutil= + fi + ;; + esac +]) + + +# _LT_DARWIN_LINKER_FEATURES([TAG]) +# --------------------------------- +# Checks for linker and compiler features on darwin +m4_defun([_LT_DARWIN_LINKER_FEATURES], +[ + m4_require([_LT_REQUIRED_DARWIN_CHECKS]) + _LT_TAGVAR(archive_cmds_need_lc, $1)=no + _LT_TAGVAR(hardcode_direct, $1)=no + _LT_TAGVAR(hardcode_automatic, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported + if test "$lt_cv_ld_force_load" = "yes"; then + _LT_TAGVAR(whole_archive_flag_spec, $1)='`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience ${wl}-force_load,$conv\"; done; func_echo_all \"$new_convenience\"`' + m4_case([$1], [F77], [_LT_TAGVAR(compiler_needs_object, $1)=yes], + [FC], [_LT_TAGVAR(compiler_needs_object, $1)=yes]) + else + _LT_TAGVAR(whole_archive_flag_spec, $1)='' + fi + _LT_TAGVAR(link_all_deplibs, $1)=yes + _LT_TAGVAR(allow_undefined_flag, $1)="$_lt_dar_allow_undefined" + case $cc_basename in + ifort*) _lt_dar_can_shared=yes ;; + *) _lt_dar_can_shared=$GCC ;; + esac + if test "$_lt_dar_can_shared" = "yes"; then + output_verbose_link_cmd=func_echo_all + _LT_TAGVAR(archive_cmds, $1)="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod${_lt_dsymutil}" + _LT_TAGVAR(module_cmds, $1)="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dsymutil}" + _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring ${_lt_dar_single_mod}${_lt_dar_export_syms}${_lt_dsymutil}" + _LT_TAGVAR(module_expsym_cmds, $1)="sed -e 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dar_export_syms}${_lt_dsymutil}" + m4_if([$1], [CXX], +[ if test "$lt_cv_apple_cc_single_mod" != "yes"; then + _LT_TAGVAR(archive_cmds, $1)="\$CC -r -keep_private_externs -nostdlib -o \${lib}-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \${lib}-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring${_lt_dsymutil}" + _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \${lib}-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \${lib}-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring${_lt_dar_export_syms}${_lt_dsymutil}" + fi +],[]) + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi +]) + +# _LT_SYS_MODULE_PATH_AIX([TAGNAME]) +# ---------------------------------- +# Links a minimal program and checks the executable +# for the system default hardcoded library path. In most cases, +# this is /usr/lib:/lib, but when the MPI compilers are used +# the location of the communication and MPI libs are included too. +# If we don't find anything, use the default library path according +# to the aix ld manual. +# Store the results from the different compilers for each TAGNAME. +# Allow to override them for all tags through lt_cv_aix_libpath. +m4_defun([_LT_SYS_MODULE_PATH_AIX], +[m4_require([_LT_DECL_SED])dnl +if test "${lt_cv_aix_libpath+set}" = set; then + aix_libpath=$lt_cv_aix_libpath +else + AC_CACHE_VAL([_LT_TAGVAR([lt_cv_aix_libpath_], [$1])], + [AC_LINK_IFELSE([AC_LANG_PROGRAM],[ + lt_aix_libpath_sed='[ + /Import File Strings/,/^$/ { + /^0/ { + s/^0 *\([^ ]*\) *$/\1/ + p + } + }]' + _LT_TAGVAR([lt_cv_aix_libpath_], [$1])=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + # Check for a 64-bit object if we didn't find anything. + if test -z "$_LT_TAGVAR([lt_cv_aix_libpath_], [$1])"; then + _LT_TAGVAR([lt_cv_aix_libpath_], [$1])=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` + fi],[]) + if test -z "$_LT_TAGVAR([lt_cv_aix_libpath_], [$1])"; then + _LT_TAGVAR([lt_cv_aix_libpath_], [$1])="/usr/lib:/lib" + fi + ]) + aix_libpath=$_LT_TAGVAR([lt_cv_aix_libpath_], [$1]) +fi +])# _LT_SYS_MODULE_PATH_AIX + + +# _LT_SHELL_INIT(ARG) +# ------------------- +m4_define([_LT_SHELL_INIT], +[m4_divert_text([M4SH-INIT], [$1 +])])# _LT_SHELL_INIT + + + +# _LT_PROG_ECHO_BACKSLASH +# ----------------------- +# Find how we can fake an echo command that does not interpret backslash. +# In particular, with Autoconf 2.60 or later we add some code to the start +# of the generated configure script which will find a shell with a builtin +# printf (which we can use as an echo command). +m4_defun([_LT_PROG_ECHO_BACKSLASH], +[ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO +ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO$ECHO + +AC_MSG_CHECKING([how to print strings]) +# Test print first, because it will be a builtin if present. +if test "X`( print -r -- -n ) 2>/dev/null`" = X-n && \ + test "X`print -r -- $ECHO 2>/dev/null`" = "X$ECHO"; then + ECHO='print -r --' +elif test "X`printf %s $ECHO 2>/dev/null`" = "X$ECHO"; then + ECHO='printf %s\n' +else + # Use this function as a fallback that always works. + func_fallback_echo () + { + eval 'cat <<_LTECHO_EOF +$[]1 +_LTECHO_EOF' + } + ECHO='func_fallback_echo' +fi + +# func_echo_all arg... +# Invoke $ECHO with all args, space-separated. +func_echo_all () +{ + $ECHO "$*" +} + +case "$ECHO" in + printf*) AC_MSG_RESULT([printf]) ;; + print*) AC_MSG_RESULT([print -r]) ;; + *) AC_MSG_RESULT([cat]) ;; +esac + +m4_ifdef([_AS_DETECT_SUGGESTED], +[_AS_DETECT_SUGGESTED([ + test -n "${ZSH_VERSION+set}${BASH_VERSION+set}" || ( + ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' + ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO + ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO$ECHO + PATH=/empty FPATH=/empty; export PATH FPATH + test "X`printf %s $ECHO`" = "X$ECHO" \ + || test "X`print -r -- $ECHO`" = "X$ECHO" )])]) + +_LT_DECL([], [SHELL], [1], [Shell to use when invoking shell scripts]) +_LT_DECL([], [ECHO], [1], [An echo program that protects backslashes]) +])# _LT_PROG_ECHO_BACKSLASH + + +# _LT_WITH_SYSROOT +# ---------------- +AC_DEFUN([_LT_WITH_SYSROOT], +[AC_MSG_CHECKING([for sysroot]) +AC_ARG_WITH([sysroot], +[ --with-sysroot[=DIR] Search for dependent libraries within DIR + (or the compiler's sysroot if not specified).], +[], [with_sysroot=no]) + +dnl lt_sysroot will always be passed unquoted. We quote it here +dnl in case the user passed a directory name. +lt_sysroot= +case ${with_sysroot} in #( + yes) + if test "$GCC" = yes; then + lt_sysroot=`$CC --print-sysroot 2>/dev/null` + fi + ;; #( + /*) + lt_sysroot=`echo "$with_sysroot" | sed -e "$sed_quote_subst"` + ;; #( + no|'') + ;; #( + *) + AC_MSG_RESULT([${with_sysroot}]) + AC_MSG_ERROR([The sysroot must be an absolute path.]) + ;; +esac + + AC_MSG_RESULT([${lt_sysroot:-no}]) +_LT_DECL([], [lt_sysroot], [0], [The root where to search for ]dnl +[dependent libraries, and in which our libraries should be installed.])]) + +# _LT_ENABLE_LOCK +# --------------- +m4_defun([_LT_ENABLE_LOCK], +[AC_ARG_ENABLE([libtool-lock], + [AS_HELP_STRING([--disable-libtool-lock], + [avoid locking (might break parallel builds)])]) +test "x$enable_libtool_lock" != xno && enable_libtool_lock=yes + +# Some flags need to be propagated to the compiler or linker for good +# libtool support. +case $host in +ia64-*-hpux*) + # Find out which ABI we are using. + echo 'int i;' > conftest.$ac_ext + if AC_TRY_EVAL(ac_compile); then + case `/usr/bin/file conftest.$ac_objext` in + *ELF-32*) + HPUX_IA64_MODE="32" + ;; + *ELF-64*) + HPUX_IA64_MODE="64" + ;; + esac + fi + rm -rf conftest* + ;; +*-*-irix6*) + # Find out which ABI we are using. + echo '[#]line '$LINENO' "configure"' > conftest.$ac_ext + if AC_TRY_EVAL(ac_compile); then + if test "$lt_cv_prog_gnu_ld" = yes; then + case `/usr/bin/file conftest.$ac_objext` in + *32-bit*) + LD="${LD-ld} -melf32bsmip" + ;; + *N32*) + LD="${LD-ld} -melf32bmipn32" + ;; + *64-bit*) + LD="${LD-ld} -melf64bmip" + ;; + esac + else + case `/usr/bin/file conftest.$ac_objext` in + *32-bit*) + LD="${LD-ld} -32" + ;; + *N32*) + LD="${LD-ld} -n32" + ;; + *64-bit*) + LD="${LD-ld} -64" + ;; + esac + fi + fi + rm -rf conftest* + ;; + +x86_64-*kfreebsd*-gnu|x86_64-*linux*|ppc*-*linux*|powerpc*-*linux*| \ +s390*-*linux*|s390*-*tpf*|sparc*-*linux*) + # Find out which ABI we are using. + echo 'int i;' > conftest.$ac_ext + if AC_TRY_EVAL(ac_compile); then + case `/usr/bin/file conftest.o` in + *32-bit*) + case $host in + x86_64-*kfreebsd*-gnu) + LD="${LD-ld} -m elf_i386_fbsd" + ;; + x86_64-*linux*) + LD="${LD-ld} -m elf_i386" + ;; + ppc64-*linux*|powerpc64-*linux*) + LD="${LD-ld} -m elf32ppclinux" + ;; + s390x-*linux*) + LD="${LD-ld} -m elf_s390" + ;; + sparc64-*linux*) + LD="${LD-ld} -m elf32_sparc" + ;; + esac + ;; + *64-bit*) + case $host in + x86_64-*kfreebsd*-gnu) + LD="${LD-ld} -m elf_x86_64_fbsd" + ;; + x86_64-*linux*) + LD="${LD-ld} -m elf_x86_64" + ;; + ppc*-*linux*|powerpc*-*linux*) + LD="${LD-ld} -m elf64ppc" + ;; + s390*-*linux*|s390*-*tpf*) + LD="${LD-ld} -m elf64_s390" + ;; + sparc*-*linux*) + LD="${LD-ld} -m elf64_sparc" + ;; + esac + ;; + esac + fi + rm -rf conftest* + ;; + +*-*-sco3.2v5*) + # On SCO OpenServer 5, we need -belf to get full-featured binaries. + SAVE_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS -belf" + AC_CACHE_CHECK([whether the C compiler needs -belf], lt_cv_cc_needs_belf, + [AC_LANG_PUSH(C) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[]],[[]])],[lt_cv_cc_needs_belf=yes],[lt_cv_cc_needs_belf=no]) + AC_LANG_POP]) + if test x"$lt_cv_cc_needs_belf" != x"yes"; then + # this is probably gcc 2.8.0, egcs 1.0 or newer; no need for -belf + CFLAGS="$SAVE_CFLAGS" + fi + ;; +*-*solaris*) + # Find out which ABI we are using. + echo 'int i;' > conftest.$ac_ext + if AC_TRY_EVAL(ac_compile); then + case `/usr/bin/file conftest.o` in + *64-bit*) + case $lt_cv_prog_gnu_ld in + yes*) + case $host in + i?86-*-solaris*) + LD="${LD-ld} -m elf_x86_64" + ;; + sparc*-*-solaris*) + LD="${LD-ld} -m elf64_sparc" + ;; + esac + # GNU ld 2.21 introduced _sol2 emulations. Use them if available. + if ${LD-ld} -V | grep _sol2 >/dev/null 2>&1; then + LD="${LD-ld}_sol2" + fi + ;; + *) + if ${LD-ld} -64 -r -o conftest2.o conftest.o >/dev/null 2>&1; then + LD="${LD-ld} -64" + fi + ;; + esac + ;; + esac + fi + rm -rf conftest* + ;; +esac + +need_locks="$enable_libtool_lock" +])# _LT_ENABLE_LOCK + + +# _LT_PROG_AR +# ----------- +m4_defun([_LT_PROG_AR], +[AC_CHECK_TOOLS(AR, [ar], false) +: ${AR=ar} +: ${AR_FLAGS=cru} +_LT_DECL([], [AR], [1], [The archiver]) +_LT_DECL([], [AR_FLAGS], [1], [Flags to create an archive]) + +AC_CACHE_CHECK([for archiver @FILE support], [lt_cv_ar_at_file], + [lt_cv_ar_at_file=no + AC_COMPILE_IFELSE([AC_LANG_PROGRAM], + [echo conftest.$ac_objext > conftest.lst + lt_ar_try='$AR $AR_FLAGS libconftest.a @conftest.lst >&AS_MESSAGE_LOG_FD' + AC_TRY_EVAL([lt_ar_try]) + if test "$ac_status" -eq 0; then + # Ensure the archiver fails upon bogus file names. + rm -f conftest.$ac_objext libconftest.a + AC_TRY_EVAL([lt_ar_try]) + if test "$ac_status" -ne 0; then + lt_cv_ar_at_file=@ + fi + fi + rm -f conftest.* libconftest.a + ]) + ]) + +if test "x$lt_cv_ar_at_file" = xno; then + archiver_list_spec= +else + archiver_list_spec=$lt_cv_ar_at_file +fi +_LT_DECL([], [archiver_list_spec], [1], + [How to feed a file listing to the archiver]) +])# _LT_PROG_AR + + +# _LT_CMD_OLD_ARCHIVE +# ------------------- +m4_defun([_LT_CMD_OLD_ARCHIVE], +[_LT_PROG_AR + +AC_CHECK_TOOL(STRIP, strip, :) +test -z "$STRIP" && STRIP=: +_LT_DECL([], [STRIP], [1], [A symbol stripping program]) + +AC_CHECK_TOOL(RANLIB, ranlib, :) +test -z "$RANLIB" && RANLIB=: +_LT_DECL([], [RANLIB], [1], + [Commands used to install an old-style archive]) + +# Determine commands to create old-style static archives. +old_archive_cmds='$AR $AR_FLAGS $oldlib$oldobjs' +old_postinstall_cmds='chmod 644 $oldlib' +old_postuninstall_cmds= + +if test -n "$RANLIB"; then + case $host_os in + openbsd*) + old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB -t \$tool_oldlib" + ;; + *) + old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB \$tool_oldlib" + ;; + esac + old_archive_cmds="$old_archive_cmds~\$RANLIB \$tool_oldlib" +fi + +case $host_os in + darwin*) + lock_old_archive_extraction=yes ;; + *) + lock_old_archive_extraction=no ;; +esac +_LT_DECL([], [old_postinstall_cmds], [2]) +_LT_DECL([], [old_postuninstall_cmds], [2]) +_LT_TAGDECL([], [old_archive_cmds], [2], + [Commands used to build an old-style archive]) +_LT_DECL([], [lock_old_archive_extraction], [0], + [Whether to use a lock for old archive extraction]) +])# _LT_CMD_OLD_ARCHIVE + + +# _LT_COMPILER_OPTION(MESSAGE, VARIABLE-NAME, FLAGS, +# [OUTPUT-FILE], [ACTION-SUCCESS], [ACTION-FAILURE]) +# ---------------------------------------------------------------- +# Check whether the given compiler option works +AC_DEFUN([_LT_COMPILER_OPTION], +[m4_require([_LT_FILEUTILS_DEFAULTS])dnl +m4_require([_LT_DECL_SED])dnl +AC_CACHE_CHECK([$1], [$2], + [$2=no + m4_if([$4], , [ac_outfile=conftest.$ac_objext], [ac_outfile=$4]) + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + lt_compiler_flag="$3" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + # The option is referenced via a variable to avoid confusing sed. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [[^ ]]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&AS_MESSAGE_LOG_FD) + (eval "$lt_compile" 2>conftest.err) + ac_status=$? + cat conftest.err >&AS_MESSAGE_LOG_FD + echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD + if (exit $ac_status) && test -s "$ac_outfile"; then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings other than the usual output. + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' >conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then + $2=yes + fi + fi + $RM conftest* +]) + +if test x"[$]$2" = xyes; then + m4_if([$5], , :, [$5]) +else + m4_if([$6], , :, [$6]) +fi +])# _LT_COMPILER_OPTION + +# Old name: +AU_ALIAS([AC_LIBTOOL_COMPILER_OPTION], [_LT_COMPILER_OPTION]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_COMPILER_OPTION], []) + + +# _LT_LINKER_OPTION(MESSAGE, VARIABLE-NAME, FLAGS, +# [ACTION-SUCCESS], [ACTION-FAILURE]) +# ---------------------------------------------------- +# Check whether the given linker option works +AC_DEFUN([_LT_LINKER_OPTION], +[m4_require([_LT_FILEUTILS_DEFAULTS])dnl +m4_require([_LT_DECL_SED])dnl +AC_CACHE_CHECK([$1], [$2], + [$2=no + save_LDFLAGS="$LDFLAGS" + LDFLAGS="$LDFLAGS $3" + echo "$lt_simple_link_test_code" > conftest.$ac_ext + if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then + # The linker can only warn and ignore the option if not recognized + # So say no if there are warnings + if test -s conftest.err; then + # Append any errors to the config.log. + cat conftest.err 1>&AS_MESSAGE_LOG_FD + $ECHO "$_lt_linker_boilerplate" | $SED '/^$/d' > conftest.exp + $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 + if diff conftest.exp conftest.er2 >/dev/null; then + $2=yes + fi + else + $2=yes + fi + fi + $RM -r conftest* + LDFLAGS="$save_LDFLAGS" +]) + +if test x"[$]$2" = xyes; then + m4_if([$4], , :, [$4]) +else + m4_if([$5], , :, [$5]) +fi +])# _LT_LINKER_OPTION + +# Old name: +AU_ALIAS([AC_LIBTOOL_LINKER_OPTION], [_LT_LINKER_OPTION]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_LINKER_OPTION], []) + + +# LT_CMD_MAX_LEN +#--------------- +AC_DEFUN([LT_CMD_MAX_LEN], +[AC_REQUIRE([AC_CANONICAL_HOST])dnl +# find the maximum length of command line arguments +AC_MSG_CHECKING([the maximum length of command line arguments]) +AC_CACHE_VAL([lt_cv_sys_max_cmd_len], [dnl + i=0 + teststring="ABCD" + + case $build_os in + msdosdjgpp*) + # On DJGPP, this test can blow up pretty badly due to problems in libc + # (any single argument exceeding 2000 bytes causes a buffer overrun + # during glob expansion). Even if it were fixed, the result of this + # check would be larger than it should be. + lt_cv_sys_max_cmd_len=12288; # 12K is about right + ;; + + gnu*) + # Under GNU Hurd, this test is not required because there is + # no limit to the length of command line arguments. + # Libtool will interpret -1 as no limit whatsoever + lt_cv_sys_max_cmd_len=-1; + ;; + + cygwin* | mingw* | cegcc*) + # On Win9x/ME, this test blows up -- it succeeds, but takes + # about 5 minutes as the teststring grows exponentially. + # Worse, since 9x/ME are not pre-emptively multitasking, + # you end up with a "frozen" computer, even though with patience + # the test eventually succeeds (with a max line length of 256k). + # Instead, let's just punt: use the minimum linelength reported by + # all of the supported platforms: 8192 (on NT/2K/XP). + lt_cv_sys_max_cmd_len=8192; + ;; + + mint*) + # On MiNT this can take a long time and run out of memory. + lt_cv_sys_max_cmd_len=8192; + ;; + + amigaos*) + # On AmigaOS with pdksh, this test takes hours, literally. + # So we just punt and use a minimum line length of 8192. + lt_cv_sys_max_cmd_len=8192; + ;; + + netbsd* | freebsd* | openbsd* | darwin* | dragonfly*) + # This has been around since 386BSD, at least. Likely further. + if test -x /sbin/sysctl; then + lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax` + elif test -x /usr/sbin/sysctl; then + lt_cv_sys_max_cmd_len=`/usr/sbin/sysctl -n kern.argmax` + else + lt_cv_sys_max_cmd_len=65536 # usable default for all BSDs + fi + # And add a safety zone + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` + ;; + + interix*) + # We know the value 262144 and hardcode it with a safety zone (like BSD) + lt_cv_sys_max_cmd_len=196608 + ;; + + os2*) + # The test takes a long time on OS/2. + lt_cv_sys_max_cmd_len=8192 + ;; + + osf*) + # Dr. Hans Ekkehard Plesser reports seeing a kernel panic running configure + # due to this test when exec_disable_arg_limit is 1 on Tru64. It is not + # nice to cause kernel panics so lets avoid the loop below. + # First set a reasonable default. + lt_cv_sys_max_cmd_len=16384 + # + if test -x /sbin/sysconfig; then + case `/sbin/sysconfig -q proc exec_disable_arg_limit` in + *1*) lt_cv_sys_max_cmd_len=-1 ;; + esac + fi + ;; + sco3.2v5*) + lt_cv_sys_max_cmd_len=102400 + ;; + sysv5* | sco5v6* | sysv4.2uw2*) + kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null` + if test -n "$kargmax"; then + lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[[ ]]//'` + else + lt_cv_sys_max_cmd_len=32768 + fi + ;; + *) + lt_cv_sys_max_cmd_len=`(getconf ARG_MAX) 2> /dev/null` + if test -n "$lt_cv_sys_max_cmd_len"; then + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` + else + # Make teststring a little bigger before we do anything with it. + # a 1K string should be a reasonable start. + for i in 1 2 3 4 5 6 7 8 ; do + teststring=$teststring$teststring + done + SHELL=${SHELL-${CONFIG_SHELL-/bin/sh}} + # If test is not a shell built-in, we'll probably end up computing a + # maximum length that is only half of the actual maximum length, but + # we can't tell. + while { test "X"`env echo "$teststring$teststring" 2>/dev/null` \ + = "X$teststring$teststring"; } >/dev/null 2>&1 && + test $i != 17 # 1/2 MB should be enough + do + i=`expr $i + 1` + teststring=$teststring$teststring + done + # Only check the string length outside the loop. + lt_cv_sys_max_cmd_len=`expr "X$teststring" : ".*" 2>&1` + teststring= + # Add a significant safety factor because C++ compilers can tack on + # massive amounts of additional arguments before passing them to the + # linker. It appears as though 1/2 is a usable value. + lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 2` + fi + ;; + esac +]) +if test -n $lt_cv_sys_max_cmd_len ; then + AC_MSG_RESULT($lt_cv_sys_max_cmd_len) +else + AC_MSG_RESULT(none) +fi +max_cmd_len=$lt_cv_sys_max_cmd_len +_LT_DECL([], [max_cmd_len], [0], + [What is the maximum length of a command?]) +])# LT_CMD_MAX_LEN + +# Old name: +AU_ALIAS([AC_LIBTOOL_SYS_MAX_CMD_LEN], [LT_CMD_MAX_LEN]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_SYS_MAX_CMD_LEN], []) + + +# _LT_HEADER_DLFCN +# ---------------- +m4_defun([_LT_HEADER_DLFCN], +[AC_CHECK_HEADERS([dlfcn.h], [], [], [AC_INCLUDES_DEFAULT])dnl +])# _LT_HEADER_DLFCN + + +# _LT_TRY_DLOPEN_SELF (ACTION-IF-TRUE, ACTION-IF-TRUE-W-USCORE, +# ACTION-IF-FALSE, ACTION-IF-CROSS-COMPILING) +# ---------------------------------------------------------------- +m4_defun([_LT_TRY_DLOPEN_SELF], +[m4_require([_LT_HEADER_DLFCN])dnl +if test "$cross_compiling" = yes; then : + [$4] +else + lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 + lt_status=$lt_dlunknown + cat > conftest.$ac_ext <<_LT_EOF +[#line $LINENO "configure" +#include "confdefs.h" + +#if HAVE_DLFCN_H +#include +#endif + +#include + +#ifdef RTLD_GLOBAL +# define LT_DLGLOBAL RTLD_GLOBAL +#else +# ifdef DL_GLOBAL +# define LT_DLGLOBAL DL_GLOBAL +# else +# define LT_DLGLOBAL 0 +# endif +#endif + +/* We may have to define LT_DLLAZY_OR_NOW in the command line if we + find out it does not work in some platform. */ +#ifndef LT_DLLAZY_OR_NOW +# ifdef RTLD_LAZY +# define LT_DLLAZY_OR_NOW RTLD_LAZY +# else +# ifdef DL_LAZY +# define LT_DLLAZY_OR_NOW DL_LAZY +# else +# ifdef RTLD_NOW +# define LT_DLLAZY_OR_NOW RTLD_NOW +# else +# ifdef DL_NOW +# define LT_DLLAZY_OR_NOW DL_NOW +# else +# define LT_DLLAZY_OR_NOW 0 +# endif +# endif +# endif +# endif +#endif + +/* When -fvisbility=hidden is used, assume the code has been annotated + correspondingly for the symbols needed. */ +#if defined(__GNUC__) && (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3)) || (__GNUC__ > 3)) +int fnord () __attribute__((visibility("default"))); +#endif + +int fnord () { return 42; } +int main () +{ + void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW); + int status = $lt_dlunknown; + + if (self) + { + if (dlsym (self,"fnord")) status = $lt_dlno_uscore; + else + { + if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore; + else puts (dlerror ()); + } + /* dlclose (self); */ + } + else + puts (dlerror ()); + + return status; +}] +_LT_EOF + if AC_TRY_EVAL(ac_link) && test -s conftest${ac_exeext} 2>/dev/null; then + (./conftest; exit; ) >&AS_MESSAGE_LOG_FD 2>/dev/null + lt_status=$? + case x$lt_status in + x$lt_dlno_uscore) $1 ;; + x$lt_dlneed_uscore) $2 ;; + x$lt_dlunknown|x*) $3 ;; + esac + else : + # compilation failed + $3 + fi +fi +rm -fr conftest* +])# _LT_TRY_DLOPEN_SELF + + +# LT_SYS_DLOPEN_SELF +# ------------------ +AC_DEFUN([LT_SYS_DLOPEN_SELF], +[m4_require([_LT_HEADER_DLFCN])dnl +if test "x$enable_dlopen" != xyes; then + enable_dlopen=unknown + enable_dlopen_self=unknown + enable_dlopen_self_static=unknown +else + lt_cv_dlopen=no + lt_cv_dlopen_libs= + + case $host_os in + beos*) + lt_cv_dlopen="load_add_on" + lt_cv_dlopen_libs= + lt_cv_dlopen_self=yes + ;; + + mingw* | pw32* | cegcc*) + lt_cv_dlopen="LoadLibrary" + lt_cv_dlopen_libs= + ;; + + cygwin*) + lt_cv_dlopen="dlopen" + lt_cv_dlopen_libs= + ;; + + darwin*) + # if libdl is installed we need to link against it + AC_CHECK_LIB([dl], [dlopen], + [lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl"],[ + lt_cv_dlopen="dyld" + lt_cv_dlopen_libs= + lt_cv_dlopen_self=yes + ]) + ;; + + *) + AC_CHECK_FUNC([shl_load], + [lt_cv_dlopen="shl_load"], + [AC_CHECK_LIB([dld], [shl_load], + [lt_cv_dlopen="shl_load" lt_cv_dlopen_libs="-ldld"], + [AC_CHECK_FUNC([dlopen], + [lt_cv_dlopen="dlopen"], + [AC_CHECK_LIB([dl], [dlopen], + [lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl"], + [AC_CHECK_LIB([svld], [dlopen], + [lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-lsvld"], + [AC_CHECK_LIB([dld], [dld_link], + [lt_cv_dlopen="dld_link" lt_cv_dlopen_libs="-ldld"]) + ]) + ]) + ]) + ]) + ]) + ;; + esac + + if test "x$lt_cv_dlopen" != xno; then + enable_dlopen=yes + else + enable_dlopen=no + fi + + case $lt_cv_dlopen in + dlopen) + save_CPPFLAGS="$CPPFLAGS" + test "x$ac_cv_header_dlfcn_h" = xyes && CPPFLAGS="$CPPFLAGS -DHAVE_DLFCN_H" + + save_LDFLAGS="$LDFLAGS" + wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $export_dynamic_flag_spec\" + + save_LIBS="$LIBS" + LIBS="$lt_cv_dlopen_libs $LIBS" + + AC_CACHE_CHECK([whether a program can dlopen itself], + lt_cv_dlopen_self, [dnl + _LT_TRY_DLOPEN_SELF( + lt_cv_dlopen_self=yes, lt_cv_dlopen_self=yes, + lt_cv_dlopen_self=no, lt_cv_dlopen_self=cross) + ]) + + if test "x$lt_cv_dlopen_self" = xyes; then + wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $lt_prog_compiler_static\" + AC_CACHE_CHECK([whether a statically linked program can dlopen itself], + lt_cv_dlopen_self_static, [dnl + _LT_TRY_DLOPEN_SELF( + lt_cv_dlopen_self_static=yes, lt_cv_dlopen_self_static=yes, + lt_cv_dlopen_self_static=no, lt_cv_dlopen_self_static=cross) + ]) + fi + + CPPFLAGS="$save_CPPFLAGS" + LDFLAGS="$save_LDFLAGS" + LIBS="$save_LIBS" + ;; + esac + + case $lt_cv_dlopen_self in + yes|no) enable_dlopen_self=$lt_cv_dlopen_self ;; + *) enable_dlopen_self=unknown ;; + esac + + case $lt_cv_dlopen_self_static in + yes|no) enable_dlopen_self_static=$lt_cv_dlopen_self_static ;; + *) enable_dlopen_self_static=unknown ;; + esac +fi +_LT_DECL([dlopen_support], [enable_dlopen], [0], + [Whether dlopen is supported]) +_LT_DECL([dlopen_self], [enable_dlopen_self], [0], + [Whether dlopen of programs is supported]) +_LT_DECL([dlopen_self_static], [enable_dlopen_self_static], [0], + [Whether dlopen of statically linked programs is supported]) +])# LT_SYS_DLOPEN_SELF + +# Old name: +AU_ALIAS([AC_LIBTOOL_DLOPEN_SELF], [LT_SYS_DLOPEN_SELF]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_DLOPEN_SELF], []) + + +# _LT_COMPILER_C_O([TAGNAME]) +# --------------------------- +# Check to see if options -c and -o are simultaneously supported by compiler. +# This macro does not hard code the compiler like AC_PROG_CC_C_O. +m4_defun([_LT_COMPILER_C_O], +[m4_require([_LT_DECL_SED])dnl +m4_require([_LT_FILEUTILS_DEFAULTS])dnl +m4_require([_LT_TAG_COMPILER])dnl +AC_CACHE_CHECK([if $compiler supports -c -o file.$ac_objext], + [_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)], + [_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=no + $RM -r conftest 2>/dev/null + mkdir conftest + cd conftest + mkdir out + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + lt_compiler_flag="-o out/conftest2.$ac_objext" + # Insert the option either (1) after the last *FLAGS variable, or + # (2) before a word containing "conftest.", or (3) at the end. + # Note that $ac_compile itself does not contain backslashes and begins + # with a dollar sign (not a hyphen), so the echo should work correctly. + lt_compile=`echo "$ac_compile" | $SED \ + -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ + -e 's: [[^ ]]*conftest\.: $lt_compiler_flag&:; t' \ + -e 's:$: $lt_compiler_flag:'` + (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&AS_MESSAGE_LOG_FD) + (eval "$lt_compile" 2>out/conftest.err) + ac_status=$? + cat out/conftest.err >&AS_MESSAGE_LOG_FD + echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD + if (exit $ac_status) && test -s out/conftest2.$ac_objext + then + # The compiler can only warn and ignore the option if not recognized + # So say no if there are warnings + $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp + $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 + if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then + _LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=yes + fi + fi + chmod u+w . 2>&AS_MESSAGE_LOG_FD + $RM conftest* + # SGI C++ compiler will create directory out/ii_files/ for + # template instantiation + test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files + $RM out/* && rmdir out + cd .. + $RM -r conftest + $RM conftest* +]) +_LT_TAGDECL([compiler_c_o], [lt_cv_prog_compiler_c_o], [1], + [Does compiler simultaneously support -c and -o options?]) +])# _LT_COMPILER_C_O + + +# _LT_COMPILER_FILE_LOCKS([TAGNAME]) +# ---------------------------------- +# Check to see if we can do hard links to lock some files if needed +m4_defun([_LT_COMPILER_FILE_LOCKS], +[m4_require([_LT_ENABLE_LOCK])dnl +m4_require([_LT_FILEUTILS_DEFAULTS])dnl +_LT_COMPILER_C_O([$1]) + +hard_links="nottested" +if test "$_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)" = no && test "$need_locks" != no; then + # do not overwrite the value of need_locks provided by the user + AC_MSG_CHECKING([if we can lock with hard links]) + hard_links=yes + $RM conftest* + ln conftest.a conftest.b 2>/dev/null && hard_links=no + touch conftest.a + ln conftest.a conftest.b 2>&5 || hard_links=no + ln conftest.a conftest.b 2>/dev/null && hard_links=no + AC_MSG_RESULT([$hard_links]) + if test "$hard_links" = no; then + AC_MSG_WARN([`$CC' does not support `-c -o', so `make -j' may be unsafe]) + need_locks=warn + fi +else + need_locks=no +fi +_LT_DECL([], [need_locks], [1], [Must we lock files when doing compilation?]) +])# _LT_COMPILER_FILE_LOCKS + + +# _LT_CHECK_OBJDIR +# ---------------- +m4_defun([_LT_CHECK_OBJDIR], +[AC_CACHE_CHECK([for objdir], [lt_cv_objdir], +[rm -f .libs 2>/dev/null +mkdir .libs 2>/dev/null +if test -d .libs; then + lt_cv_objdir=.libs +else + # MS-DOS does not allow filenames that begin with a dot. + lt_cv_objdir=_libs +fi +rmdir .libs 2>/dev/null]) +objdir=$lt_cv_objdir +_LT_DECL([], [objdir], [0], + [The name of the directory that contains temporary libtool files])dnl +m4_pattern_allow([LT_OBJDIR])dnl +AC_DEFINE_UNQUOTED(LT_OBJDIR, "$lt_cv_objdir/", + [Define to the sub-directory in which libtool stores uninstalled libraries.]) +])# _LT_CHECK_OBJDIR + + +# _LT_LINKER_HARDCODE_LIBPATH([TAGNAME]) +# -------------------------------------- +# Check hardcoding attributes. +m4_defun([_LT_LINKER_HARDCODE_LIBPATH], +[AC_MSG_CHECKING([how to hardcode library paths into programs]) +_LT_TAGVAR(hardcode_action, $1)= +if test -n "$_LT_TAGVAR(hardcode_libdir_flag_spec, $1)" || + test -n "$_LT_TAGVAR(runpath_var, $1)" || + test "X$_LT_TAGVAR(hardcode_automatic, $1)" = "Xyes" ; then + + # We can hardcode non-existent directories. + if test "$_LT_TAGVAR(hardcode_direct, $1)" != no && + # If the only mechanism to avoid hardcoding is shlibpath_var, we + # have to relink, otherwise we might link with an installed library + # when we should be linking with a yet-to-be-installed one + ## test "$_LT_TAGVAR(hardcode_shlibpath_var, $1)" != no && + test "$_LT_TAGVAR(hardcode_minus_L, $1)" != no; then + # Linking always hardcodes the temporary library directory. + _LT_TAGVAR(hardcode_action, $1)=relink + else + # We can link without hardcoding, and we can hardcode nonexisting dirs. + _LT_TAGVAR(hardcode_action, $1)=immediate + fi +else + # We cannot hardcode anything, or else we can only hardcode existing + # directories. + _LT_TAGVAR(hardcode_action, $1)=unsupported +fi +AC_MSG_RESULT([$_LT_TAGVAR(hardcode_action, $1)]) + +if test "$_LT_TAGVAR(hardcode_action, $1)" = relink || + test "$_LT_TAGVAR(inherit_rpath, $1)" = yes; then + # Fast installation is not supported + enable_fast_install=no +elif test "$shlibpath_overrides_runpath" = yes || + test "$enable_shared" = no; then + # Fast installation is not necessary + enable_fast_install=needless +fi +_LT_TAGDECL([], [hardcode_action], [0], + [How to hardcode a shared library path into an executable]) +])# _LT_LINKER_HARDCODE_LIBPATH + + +# _LT_CMD_STRIPLIB +# ---------------- +m4_defun([_LT_CMD_STRIPLIB], +[m4_require([_LT_DECL_EGREP]) +striplib= +old_striplib= +AC_MSG_CHECKING([whether stripping libraries is possible]) +if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then + test -z "$old_striplib" && old_striplib="$STRIP --strip-debug" + test -z "$striplib" && striplib="$STRIP --strip-unneeded" + AC_MSG_RESULT([yes]) +else +# FIXME - insert some real tests, host_os isn't really good enough + case $host_os in + darwin*) + if test -n "$STRIP" ; then + striplib="$STRIP -x" + old_striplib="$STRIP -S" + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + ;; + *) + AC_MSG_RESULT([no]) + ;; + esac +fi +_LT_DECL([], [old_striplib], [1], [Commands to strip libraries]) +_LT_DECL([], [striplib], [1]) +])# _LT_CMD_STRIPLIB + + +# _LT_SYS_DYNAMIC_LINKER([TAG]) +# ----------------------------- +# PORTME Fill in your ld.so characteristics +m4_defun([_LT_SYS_DYNAMIC_LINKER], +[AC_REQUIRE([AC_CANONICAL_HOST])dnl +m4_require([_LT_DECL_EGREP])dnl +m4_require([_LT_FILEUTILS_DEFAULTS])dnl +m4_require([_LT_DECL_OBJDUMP])dnl +m4_require([_LT_DECL_SED])dnl +m4_require([_LT_CHECK_SHELL_FEATURES])dnl +AC_MSG_CHECKING([dynamic linker characteristics]) +m4_if([$1], + [], [ +if test "$GCC" = yes; then + case $host_os in + darwin*) lt_awk_arg="/^libraries:/,/LR/" ;; + *) lt_awk_arg="/^libraries:/" ;; + esac + case $host_os in + mingw* | cegcc*) lt_sed_strip_eq="s,=\([[A-Za-z]]:\),\1,g" ;; + *) lt_sed_strip_eq="s,=/,/,g" ;; + esac + lt_search_path_spec=`$CC -print-search-dirs | awk $lt_awk_arg | $SED -e "s/^libraries://" -e $lt_sed_strip_eq` + case $lt_search_path_spec in + *\;*) + # if the path contains ";" then we assume it to be the separator + # otherwise default to the standard path separator (i.e. ":") - it is + # assumed that no part of a normal pathname contains ";" but that should + # okay in the real world where ";" in dirpaths is itself problematic. + lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED 's/;/ /g'` + ;; + *) + lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED "s/$PATH_SEPARATOR/ /g"` + ;; + esac + # Ok, now we have the path, separated by spaces, we can step through it + # and add multilib dir if necessary. + lt_tmp_lt_search_path_spec= + lt_multi_os_dir=`$CC $CPPFLAGS $CFLAGS $LDFLAGS -print-multi-os-directory 2>/dev/null` + for lt_sys_path in $lt_search_path_spec; do + if test -d "$lt_sys_path/$lt_multi_os_dir"; then + lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path/$lt_multi_os_dir" + else + test -d "$lt_sys_path" && \ + lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path" + fi + done + lt_search_path_spec=`$ECHO "$lt_tmp_lt_search_path_spec" | awk ' +BEGIN {RS=" "; FS="/|\n";} { + lt_foo=""; + lt_count=0; + for (lt_i = NF; lt_i > 0; lt_i--) { + if ($lt_i != "" && $lt_i != ".") { + if ($lt_i == "..") { + lt_count++; + } else { + if (lt_count == 0) { + lt_foo="/" $lt_i lt_foo; + } else { + lt_count--; + } + } + } + } + if (lt_foo != "") { lt_freq[[lt_foo]]++; } + if (lt_freq[[lt_foo]] == 1) { print lt_foo; } +}'` + # AWK program above erroneously prepends '/' to C:/dos/paths + # for these hosts. + case $host_os in + mingw* | cegcc*) lt_search_path_spec=`$ECHO "$lt_search_path_spec" |\ + $SED 's,/\([[A-Za-z]]:\),\1,g'` ;; + esac + sys_lib_search_path_spec=`$ECHO "$lt_search_path_spec" | $lt_NL2SP` +else + sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib" +fi]) +library_names_spec= +libname_spec='lib$name' +soname_spec= +shrext_cmds=".so" +postinstall_cmds= +postuninstall_cmds= +finish_cmds= +finish_eval= +shlibpath_var= +shlibpath_overrides_runpath=unknown +version_type=none +dynamic_linker="$host_os ld.so" +sys_lib_dlsearch_path_spec="/lib /usr/lib" +need_lib_prefix=unknown +hardcode_into_libs=no + +# when you set need_version to no, make sure it does not cause -set_version +# flags to be left without arguments +need_version=unknown + +case $host_os in +aix3*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix $libname.a' + shlibpath_var=LIBPATH + + # AIX 3 has no versioning support, so we append a major version to the name. + soname_spec='${libname}${release}${shared_ext}$major' + ;; + +aix[[4-9]]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + hardcode_into_libs=yes + if test "$host_cpu" = ia64; then + # AIX 5 supports IA64 + library_names_spec='${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext}$versuffix $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + else + # With GCC up to 2.95.x, collect2 would create an import file + # for dependence libraries. The import file would start with + # the line `#! .'. This would cause the generated library to + # depend on `.', always an invalid library. This was fixed in + # development snapshots of GCC prior to 3.0. + case $host_os in + aix4 | aix4.[[01]] | aix4.[[01]].*) + if { echo '#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 97)' + echo ' yes ' + echo '#endif'; } | ${CC} -E - | $GREP yes > /dev/null; then + : + else + can_build_shared=no + fi + ;; + esac + # AIX (on Power*) has no versioning support, so currently we can not hardcode correct + # soname into executable. Probably we can add versioning support to + # collect2, so additional links can be useful in future. + if test "$aix_use_runtimelinking" = yes; then + # If using run time linking (on AIX 4.2 or later) use lib.so + # instead of lib.a to let people know that these are not + # typical AIX shared libraries. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + else + # We preserve .a as extension for shared libraries through AIX4.2 + # and later when we are not doing run time linking. + library_names_spec='${libname}${release}.a $libname.a' + soname_spec='${libname}${release}${shared_ext}$major' + fi + shlibpath_var=LIBPATH + fi + ;; + +amigaos*) + case $host_cpu in + powerpc) + # Since July 2007 AmigaOS4 officially supports .so libraries. + # When compiling the executable, add -use-dynld -Lsobjs: to the compileline. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + ;; + m68k) + library_names_spec='$libname.ixlibrary $libname.a' + # Create ${libname}_ixlibrary.a entries in /sys/libs. + finish_eval='for lib in `ls $libdir/*.ixlibrary 2>/dev/null`; do libname=`func_echo_all "$lib" | $SED '\''s%^.*/\([[^/]]*\)\.ixlibrary$%\1%'\''`; test $RM /sys/libs/${libname}_ixlibrary.a; $show "cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a"; cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a || exit 1; done' + ;; + esac + ;; + +beos*) + library_names_spec='${libname}${shared_ext}' + dynamic_linker="$host_os ld.so" + shlibpath_var=LIBRARY_PATH + ;; + +bsdi[[45]]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + finish_cmds='PATH="\$PATH:/sbin" ldconfig $libdir' + shlibpath_var=LD_LIBRARY_PATH + sys_lib_search_path_spec="/shlib /usr/lib /usr/X11/lib /usr/contrib/lib /lib /usr/local/lib" + sys_lib_dlsearch_path_spec="/shlib /usr/lib /usr/local/lib" + # the default ld.so.conf also contains /usr/contrib/lib and + # /usr/X11R6/lib (/usr/X11 is a link to /usr/X11R6), but let us allow + # libtool to hard-code these into programs + ;; + +cygwin* | mingw* | pw32* | cegcc*) + version_type=windows + shrext_cmds=".dll" + need_version=no + need_lib_prefix=no + + case $GCC,$cc_basename in + yes,*) + # gcc + library_names_spec='$libname.dll.a' + # DLL is installed to $(libdir)/../bin by postinstall_cmds + postinstall_cmds='base_file=`basename \${file}`~ + dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~ + dldir=$destdir/`dirname \$dlpath`~ + test -d \$dldir || mkdir -p \$dldir~ + $install_prog $dir/$dlname \$dldir/$dlname~ + chmod a+x \$dldir/$dlname~ + if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then + eval '\''$striplib \$dldir/$dlname'\'' || exit \$?; + fi' + postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ + dlpath=$dir/\$dldll~ + $RM \$dlpath' + shlibpath_overrides_runpath=yes + + case $host_os in + cygwin*) + # Cygwin DLLs use 'cyg' prefix rather than 'lib' + soname_spec='`echo ${libname} | sed -e 's/^lib/cyg/'``echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext}' +m4_if([$1], [],[ + sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/lib/w32api"]) + ;; + mingw* | cegcc*) + # MinGW DLLs use traditional 'lib' prefix + soname_spec='${libname}`echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext}' + ;; + pw32*) + # pw32 DLLs use 'pw' prefix rather than 'lib' + library_names_spec='`echo ${libname} | sed -e 's/^lib/pw/'``echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext}' + ;; + esac + dynamic_linker='Win32 ld.exe' + ;; + + *,cl*) + # Native MSVC + libname_spec='$name' + soname_spec='${libname}`echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext}' + library_names_spec='${libname}.dll.lib' + + case $build_os in + mingw*) + sys_lib_search_path_spec= + lt_save_ifs=$IFS + IFS=';' + for lt_path in $LIB + do + IFS=$lt_save_ifs + # Let DOS variable expansion print the short 8.3 style file name. + lt_path=`cd "$lt_path" 2>/dev/null && cmd //C "for %i in (".") do @echo %~si"` + sys_lib_search_path_spec="$sys_lib_search_path_spec $lt_path" + done + IFS=$lt_save_ifs + # Convert to MSYS style. + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([[a-zA-Z]]\\):| /\\1|g' -e 's|^ ||'` + ;; + cygwin*) + # Convert to unix form, then to dos form, then back to unix form + # but this time dos style (no spaces!) so that the unix form looks + # like /cygdrive/c/PROGRA~1:/cygdr... + sys_lib_search_path_spec=`cygpath --path --unix "$LIB"` + sys_lib_search_path_spec=`cygpath --path --dos "$sys_lib_search_path_spec" 2>/dev/null` + sys_lib_search_path_spec=`cygpath --path --unix "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` + ;; + *) + sys_lib_search_path_spec="$LIB" + if $ECHO "$sys_lib_search_path_spec" | [$GREP ';[c-zC-Z]:/' >/dev/null]; then + # It is most probably a Windows format PATH. + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's/;/ /g'` + else + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` + fi + # FIXME: find the short name or the path components, as spaces are + # common. (e.g. "Program Files" -> "PROGRA~1") + ;; + esac + + # DLL is installed to $(libdir)/../bin by postinstall_cmds + postinstall_cmds='base_file=`basename \${file}`~ + dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~ + dldir=$destdir/`dirname \$dlpath`~ + test -d \$dldir || mkdir -p \$dldir~ + $install_prog $dir/$dlname \$dldir/$dlname' + postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ + dlpath=$dir/\$dldll~ + $RM \$dlpath' + shlibpath_overrides_runpath=yes + dynamic_linker='Win32 link.exe' + ;; + + *) + # Assume MSVC wrapper + library_names_spec='${libname}`echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext} $libname.lib' + dynamic_linker='Win32 ld.exe' + ;; + esac + # FIXME: first we should search . and the directory the executable is in + shlibpath_var=PATH + ;; + +darwin* | rhapsody*) + dynamic_linker="$host_os dyld" + version_type=darwin + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${major}$shared_ext ${libname}$shared_ext' + soname_spec='${libname}${release}${major}$shared_ext' + shlibpath_overrides_runpath=yes + shlibpath_var=DYLD_LIBRARY_PATH + shrext_cmds='`test .$module = .yes && echo .so || echo .dylib`' +m4_if([$1], [],[ + sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/local/lib"]) + sys_lib_dlsearch_path_spec='/usr/local/lib /lib /usr/lib' + ;; + +dgux*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname$shared_ext' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + ;; + +freebsd* | dragonfly*) + # DragonFly does not have aout. When/if they implement a new + # versioning mechanism, adjust this. + if test -x /usr/bin/objformat; then + objformat=`/usr/bin/objformat` + else + case $host_os in + freebsd[[23]].*) objformat=aout ;; + *) objformat=elf ;; + esac + fi + version_type=freebsd-$objformat + case $version_type in + freebsd-elf*) + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}' + need_version=no + need_lib_prefix=no + ;; + freebsd-*) + library_names_spec='${libname}${release}${shared_ext}$versuffix $libname${shared_ext}$versuffix' + need_version=yes + ;; + esac + shlibpath_var=LD_LIBRARY_PATH + case $host_os in + freebsd2.*) + shlibpath_overrides_runpath=yes + ;; + freebsd3.[[01]]* | freebsdelf3.[[01]]*) + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + freebsd3.[[2-9]]* | freebsdelf3.[[2-9]]* | \ + freebsd4.[[0-5]] | freebsdelf4.[[0-5]] | freebsd4.1.1 | freebsdelf4.1.1) + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + *) # from 4.6 on, and DragonFly + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + esac + ;; + +gnu*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +haiku*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + dynamic_linker="$host_os runtime_loader" + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LIBRARY_PATH + shlibpath_overrides_runpath=yes + sys_lib_dlsearch_path_spec='/boot/home/config/lib /boot/common/lib /boot/system/lib' + hardcode_into_libs=yes + ;; + +hpux9* | hpux10* | hpux11*) + # Give a soname corresponding to the major version so that dld.sl refuses to + # link against other versions. + version_type=sunos + need_lib_prefix=no + need_version=no + case $host_cpu in + ia64*) + shrext_cmds='.so' + hardcode_into_libs=yes + dynamic_linker="$host_os dld.so" + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + if test "X$HPUX_IA64_MODE" = X32; then + sys_lib_search_path_spec="/usr/lib/hpux32 /usr/local/lib/hpux32 /usr/local/lib" + else + sys_lib_search_path_spec="/usr/lib/hpux64 /usr/local/lib/hpux64" + fi + sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec + ;; + hppa*64*) + shrext_cmds='.sl' + hardcode_into_libs=yes + dynamic_linker="$host_os dld.sl" + shlibpath_var=LD_LIBRARY_PATH # How should we handle SHLIB_PATH + shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + sys_lib_search_path_spec="/usr/lib/pa20_64 /usr/ccs/lib/pa20_64" + sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec + ;; + *) + shrext_cmds='.sl' + dynamic_linker="$host_os dld.sl" + shlibpath_var=SHLIB_PATH + shlibpath_overrides_runpath=no # +s is required to enable SHLIB_PATH + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + ;; + esac + # HP-UX runs *really* slowly unless shared libraries are mode 555, ... + postinstall_cmds='chmod 555 $lib' + # or fails outright, so override atomically: + install_override_mode=555 + ;; + +interix[[3-9]]*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + dynamic_linker='Interix 3.x ld.so.1 (PE, like ELF)' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +irix5* | irix6* | nonstopux*) + case $host_os in + nonstopux*) version_type=nonstopux ;; + *) + if test "$lt_cv_prog_gnu_ld" = yes; then + version_type=linux # correct to gnu/linux during the next big refactor + else + version_type=irix + fi ;; + esac + need_lib_prefix=no + need_version=no + soname_spec='${libname}${release}${shared_ext}$major' + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext} $libname${shared_ext}' + case $host_os in + irix5* | nonstopux*) + libsuff= shlibsuff= + ;; + *) + case $LD in # libtool.m4 will add one of these switches to LD + *-32|*"-32 "|*-melf32bsmip|*"-melf32bsmip ") + libsuff= shlibsuff= libmagic=32-bit;; + *-n32|*"-n32 "|*-melf32bmipn32|*"-melf32bmipn32 ") + libsuff=32 shlibsuff=N32 libmagic=N32;; + *-64|*"-64 "|*-melf64bmip|*"-melf64bmip ") + libsuff=64 shlibsuff=64 libmagic=64-bit;; + *) libsuff= shlibsuff= libmagic=never-match;; + esac + ;; + esac + shlibpath_var=LD_LIBRARY${shlibsuff}_PATH + shlibpath_overrides_runpath=no + sys_lib_search_path_spec="/usr/lib${libsuff} /lib${libsuff} /usr/local/lib${libsuff}" + sys_lib_dlsearch_path_spec="/usr/lib${libsuff} /lib${libsuff}" + hardcode_into_libs=yes + ;; + +# No shared lib support for Linux oldld, aout, or coff. +linux*oldld* | linux*aout* | linux*coff*) + dynamic_linker=no + ;; + +# This must be glibc/ELF. +linux* | k*bsd*-gnu | kopensolaris*-gnu) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -n $libdir' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + + # Some binutils ld are patched to set DT_RUNPATH + AC_CACHE_VAL([lt_cv_shlibpath_overrides_runpath], + [lt_cv_shlibpath_overrides_runpath=no + save_LDFLAGS=$LDFLAGS + save_libdir=$libdir + eval "libdir=/foo; wl=\"$_LT_TAGVAR(lt_prog_compiler_wl, $1)\"; \ + LDFLAGS=\"\$LDFLAGS $_LT_TAGVAR(hardcode_libdir_flag_spec, $1)\"" + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[])], + [AS_IF([ ($OBJDUMP -p conftest$ac_exeext) 2>/dev/null | grep "RUNPATH.*$libdir" >/dev/null], + [lt_cv_shlibpath_overrides_runpath=yes])]) + LDFLAGS=$save_LDFLAGS + libdir=$save_libdir + ]) + shlibpath_overrides_runpath=$lt_cv_shlibpath_overrides_runpath + + # This implies no fast_install, which is unacceptable. + # Some rework will be needed to allow for fast_install + # before this can be enabled. + hardcode_into_libs=yes + + # Append ld.so.conf contents to the search path + if test -f /etc/ld.so.conf; then + lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;/^[ ]*hwcap[ ]/d;s/[:, ]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;s/"//g;/^$/d' | tr '\n' ' '` + sys_lib_dlsearch_path_spec="/lib /usr/lib $lt_ld_extra" + fi + + # We used to test for /lib/ld.so.1 and disable shared libraries on + # powerpc, because MkLinux only supported shared libraries with the + # GNU dynamic linker. Since this was broken with cross compilers, + # most powerpc-linux boxes support dynamic linking these days and + # people can always --disable-shared, the test was removed, and we + # assume the GNU/Linux dynamic linker is in use. + dynamic_linker='GNU/Linux ld.so' + ;; + +netbsdelf*-gnu) + version_type=linux + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + dynamic_linker='NetBSD ld.elf_so' + ;; + +netbsd*) + version_type=sunos + need_lib_prefix=no + need_version=no + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' + dynamic_linker='NetBSD (a.out) ld.so' + else + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + dynamic_linker='NetBSD ld.elf_so' + fi + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + ;; + +newsos6) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + ;; + +*nto* | *qnx*) + version_type=qnx + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + dynamic_linker='ldqnx.so' + ;; + +openbsd*) + version_type=sunos + sys_lib_dlsearch_path_spec="/usr/lib" + need_lib_prefix=no + # Some older versions of OpenBSD (3.3 at least) *do* need versioned libs. + case $host_os in + openbsd3.3 | openbsd3.3.*) need_version=yes ;; + *) need_version=no ;; + esac + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' + shlibpath_var=LD_LIBRARY_PATH + if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + case $host_os in + openbsd2.[[89]] | openbsd2.[[89]].*) + shlibpath_overrides_runpath=no + ;; + *) + shlibpath_overrides_runpath=yes + ;; + esac + else + shlibpath_overrides_runpath=yes + fi + ;; + +os2*) + libname_spec='$name' + shrext_cmds=".dll" + need_lib_prefix=no + library_names_spec='$libname${shared_ext} $libname.a' + dynamic_linker='OS/2 ld.exe' + shlibpath_var=LIBPATH + ;; + +osf3* | osf4* | osf5*) + version_type=osf + need_lib_prefix=no + need_version=no + soname_spec='${libname}${release}${shared_ext}$major' + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + sys_lib_search_path_spec="/usr/shlib /usr/ccs/lib /usr/lib/cmplrs/cc /usr/lib /usr/local/lib /var/shlib" + sys_lib_dlsearch_path_spec="$sys_lib_search_path_spec" + ;; + +rdos*) + dynamic_linker=no + ;; + +solaris*) + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + # ldd complains unless libraries are executable + postinstall_cmds='chmod +x $lib' + ;; + +sunos4*) + version_type=sunos + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix' + finish_cmds='PATH="\$PATH:/usr/etc" ldconfig $libdir' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + if test "$with_gnu_ld" = yes; then + need_lib_prefix=no + fi + need_version=yes + ;; + +sysv4 | sysv4.3*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + case $host_vendor in + sni) + shlibpath_overrides_runpath=no + need_lib_prefix=no + runpath_var=LD_RUN_PATH + ;; + siemens) + need_lib_prefix=no + ;; + motorola) + need_lib_prefix=no + need_version=no + shlibpath_overrides_runpath=no + sys_lib_search_path_spec='/lib /usr/lib /usr/ccs/lib' + ;; + esac + ;; + +sysv4*MP*) + if test -d /usr/nec ;then + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='$libname${shared_ext}.$versuffix $libname${shared_ext}.$major $libname${shared_ext}' + soname_spec='$libname${shared_ext}.$major' + shlibpath_var=LD_LIBRARY_PATH + fi + ;; + +sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) + version_type=freebsd-elf + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=yes + hardcode_into_libs=yes + if test "$with_gnu_ld" = yes; then + sys_lib_search_path_spec='/usr/local/lib /usr/gnu/lib /usr/ccs/lib /usr/lib /lib' + else + sys_lib_search_path_spec='/usr/ccs/lib /usr/lib' + case $host_os in + sco3.2v5*) + sys_lib_search_path_spec="$sys_lib_search_path_spec /lib" + ;; + esac + fi + sys_lib_dlsearch_path_spec='/usr/lib' + ;; + +tpf*) + # TPF is a cross-target only. Preferred cross-host = GNU/Linux. + version_type=linux # correct to gnu/linux during the next big refactor + need_lib_prefix=no + need_version=no + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + shlibpath_var=LD_LIBRARY_PATH + shlibpath_overrides_runpath=no + hardcode_into_libs=yes + ;; + +uts4*) + version_type=linux # correct to gnu/linux during the next big refactor + library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}' + soname_spec='${libname}${release}${shared_ext}$major' + shlibpath_var=LD_LIBRARY_PATH + ;; + +*) + dynamic_linker=no + ;; +esac +AC_MSG_RESULT([$dynamic_linker]) +test "$dynamic_linker" = no && can_build_shared=no + +variables_saved_for_relink="PATH $shlibpath_var $runpath_var" +if test "$GCC" = yes; then + variables_saved_for_relink="$variables_saved_for_relink GCC_EXEC_PREFIX COMPILER_PATH LIBRARY_PATH" +fi + +if test "${lt_cv_sys_lib_search_path_spec+set}" = set; then + sys_lib_search_path_spec="$lt_cv_sys_lib_search_path_spec" +fi +if test "${lt_cv_sys_lib_dlsearch_path_spec+set}" = set; then + sys_lib_dlsearch_path_spec="$lt_cv_sys_lib_dlsearch_path_spec" +fi + +_LT_DECL([], [variables_saved_for_relink], [1], + [Variables whose values should be saved in libtool wrapper scripts and + restored at link time]) +_LT_DECL([], [need_lib_prefix], [0], + [Do we need the "lib" prefix for modules?]) +_LT_DECL([], [need_version], [0], [Do we need a version for libraries?]) +_LT_DECL([], [version_type], [0], [Library versioning type]) +_LT_DECL([], [runpath_var], [0], [Shared library runtime path variable]) +_LT_DECL([], [shlibpath_var], [0],[Shared library path variable]) +_LT_DECL([], [shlibpath_overrides_runpath], [0], + [Is shlibpath searched before the hard-coded library search path?]) +_LT_DECL([], [libname_spec], [1], [Format of library name prefix]) +_LT_DECL([], [library_names_spec], [1], + [[List of archive names. First name is the real one, the rest are links. + The last name is the one that the linker finds with -lNAME]]) +_LT_DECL([], [soname_spec], [1], + [[The coded name of the library, if different from the real name]]) +_LT_DECL([], [install_override_mode], [1], + [Permission mode override for installation of shared libraries]) +_LT_DECL([], [postinstall_cmds], [2], + [Command to use after installation of a shared archive]) +_LT_DECL([], [postuninstall_cmds], [2], + [Command to use after uninstallation of a shared archive]) +_LT_DECL([], [finish_cmds], [2], + [Commands used to finish a libtool library installation in a directory]) +_LT_DECL([], [finish_eval], [1], + [[As "finish_cmds", except a single script fragment to be evaled but + not shown]]) +_LT_DECL([], [hardcode_into_libs], [0], + [Whether we should hardcode library paths into libraries]) +_LT_DECL([], [sys_lib_search_path_spec], [2], + [Compile-time system search path for libraries]) +_LT_DECL([], [sys_lib_dlsearch_path_spec], [2], + [Run-time system search path for libraries]) +])# _LT_SYS_DYNAMIC_LINKER + + +# _LT_PATH_TOOL_PREFIX(TOOL) +# -------------------------- +# find a file program which can recognize shared library +AC_DEFUN([_LT_PATH_TOOL_PREFIX], +[m4_require([_LT_DECL_EGREP])dnl +AC_MSG_CHECKING([for $1]) +AC_CACHE_VAL(lt_cv_path_MAGIC_CMD, +[case $MAGIC_CMD in +[[\\/*] | ?:[\\/]*]) + lt_cv_path_MAGIC_CMD="$MAGIC_CMD" # Let the user override the test with a path. + ;; +*) + lt_save_MAGIC_CMD="$MAGIC_CMD" + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR +dnl $ac_dummy forces splitting on constant user-supplied paths. +dnl POSIX.2 word splitting is done only on the output of word expansions, +dnl not every word. This closes a longstanding sh security hole. + ac_dummy="m4_if([$2], , $PATH, [$2])" + for ac_dir in $ac_dummy; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + if test -f $ac_dir/$1; then + lt_cv_path_MAGIC_CMD="$ac_dir/$1" + if test -n "$file_magic_test_file"; then + case $deplibs_check_method in + "file_magic "*) + file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"` + MAGIC_CMD="$lt_cv_path_MAGIC_CMD" + if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null | + $EGREP "$file_magic_regex" > /dev/null; then + : + else + cat <<_LT_EOF 1>&2 + +*** Warning: the command libtool uses to detect shared libraries, +*** $file_magic_cmd, produces output that libtool cannot recognize. +*** The result is that libtool may fail to recognize shared libraries +*** as such. This will affect the creation of libtool libraries that +*** depend on shared libraries, but programs linked with such libtool +*** libraries will work regardless of this problem. Nevertheless, you +*** may want to report the problem to your system manager and/or to +*** bug-libtool@gnu.org + +_LT_EOF + fi ;; + esac + fi + break + fi + done + IFS="$lt_save_ifs" + MAGIC_CMD="$lt_save_MAGIC_CMD" + ;; +esac]) +MAGIC_CMD="$lt_cv_path_MAGIC_CMD" +if test -n "$MAGIC_CMD"; then + AC_MSG_RESULT($MAGIC_CMD) +else + AC_MSG_RESULT(no) +fi +_LT_DECL([], [MAGIC_CMD], [0], + [Used to examine libraries when file_magic_cmd begins with "file"])dnl +])# _LT_PATH_TOOL_PREFIX + +# Old name: +AU_ALIAS([AC_PATH_TOOL_PREFIX], [_LT_PATH_TOOL_PREFIX]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_PATH_TOOL_PREFIX], []) + + +# _LT_PATH_MAGIC +# -------------- +# find a file program which can recognize a shared library +m4_defun([_LT_PATH_MAGIC], +[_LT_PATH_TOOL_PREFIX(${ac_tool_prefix}file, /usr/bin$PATH_SEPARATOR$PATH) +if test -z "$lt_cv_path_MAGIC_CMD"; then + if test -n "$ac_tool_prefix"; then + _LT_PATH_TOOL_PREFIX(file, /usr/bin$PATH_SEPARATOR$PATH) + else + MAGIC_CMD=: + fi +fi +])# _LT_PATH_MAGIC + + +# LT_PATH_LD +# ---------- +# find the pathname to the GNU or non-GNU linker +AC_DEFUN([LT_PATH_LD], +[AC_REQUIRE([AC_PROG_CC])dnl +AC_REQUIRE([AC_CANONICAL_HOST])dnl +AC_REQUIRE([AC_CANONICAL_BUILD])dnl +m4_require([_LT_DECL_SED])dnl +m4_require([_LT_DECL_EGREP])dnl +m4_require([_LT_PROG_ECHO_BACKSLASH])dnl + +AC_ARG_WITH([gnu-ld], + [AS_HELP_STRING([--with-gnu-ld], + [assume the C compiler uses GNU ld @<:@default=no@:>@])], + [test "$withval" = no || with_gnu_ld=yes], + [with_gnu_ld=no])dnl + +ac_prog=ld +if test "$GCC" = yes; then + # Check if gcc -print-prog-name=ld gives a path. + AC_MSG_CHECKING([for ld used by $CC]) + case $host in + *-*-mingw*) + # gcc leaves a trailing carriage return which upsets mingw + ac_prog=`($CC -print-prog-name=ld) 2>&5 | tr -d '\015'` ;; + *) + ac_prog=`($CC -print-prog-name=ld) 2>&5` ;; + esac + case $ac_prog in + # Accept absolute paths. + [[\\/]]* | ?:[[\\/]]*) + re_direlt='/[[^/]][[^/]]*/\.\./' + # Canonicalize the pathname of ld + ac_prog=`$ECHO "$ac_prog"| $SED 's%\\\\%/%g'` + while $ECHO "$ac_prog" | $GREP "$re_direlt" > /dev/null 2>&1; do + ac_prog=`$ECHO $ac_prog| $SED "s%$re_direlt%/%"` + done + test -z "$LD" && LD="$ac_prog" + ;; + "") + # If it fails, then pretend we aren't using GCC. + ac_prog=ld + ;; + *) + # If it is relative, then search for the first ld in PATH. + with_gnu_ld=unknown + ;; + esac +elif test "$with_gnu_ld" = yes; then + AC_MSG_CHECKING([for GNU ld]) +else + AC_MSG_CHECKING([for non-GNU ld]) +fi +AC_CACHE_VAL(lt_cv_path_LD, +[if test -z "$LD"; then + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR + for ac_dir in $PATH; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + if test -f "$ac_dir/$ac_prog" || test -f "$ac_dir/$ac_prog$ac_exeext"; then + lt_cv_path_LD="$ac_dir/$ac_prog" + # Check to see if the program is GNU ld. I'd rather use --version, + # but apparently some variants of GNU ld only accept -v. + # Break only if it was the GNU/non-GNU ld that we prefer. + case `"$lt_cv_path_LD" -v 2>&1 &1 /dev/null 2>&1; then + lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL' + lt_cv_file_magic_cmd='func_win32_libid' + else + # Keep this pattern in sync with the one in func_win32_libid. + lt_cv_deplibs_check_method='file_magic file format (pei*-i386(.*architecture: i386)?|pe-arm-wince|pe-x86-64)' + lt_cv_file_magic_cmd='$OBJDUMP -f' + fi + ;; + +cegcc*) + # use the weaker test based on 'objdump'. See mingw*. + lt_cv_deplibs_check_method='file_magic file format pe-arm-.*little(.*architecture: arm)?' + lt_cv_file_magic_cmd='$OBJDUMP -f' + ;; + +darwin* | rhapsody*) + lt_cv_deplibs_check_method=pass_all + ;; + +freebsd* | dragonfly*) + if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then + case $host_cpu in + i*86 ) + # Not sure whether the presence of OpenBSD here was a mistake. + # Let's accept both of them until this is cleared up. + lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[[3-9]]86 (compact )?demand paged shared library' + lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*` + ;; + esac + else + lt_cv_deplibs_check_method=pass_all + fi + ;; + +gnu*) + lt_cv_deplibs_check_method=pass_all + ;; + +haiku*) + lt_cv_deplibs_check_method=pass_all + ;; + +hpux10.20* | hpux11*) + lt_cv_file_magic_cmd=/usr/bin/file + case $host_cpu in + ia64*) + lt_cv_deplibs_check_method='file_magic (s[[0-9]][[0-9]][[0-9]]|ELF-[[0-9]][[0-9]]) shared object file - IA64' + lt_cv_file_magic_test_file=/usr/lib/hpux32/libc.so + ;; + hppa*64*) + [lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF[ -][0-9][0-9])(-bit)?( [LM]SB)? shared object( file)?[, -]* PA-RISC [0-9]\.[0-9]'] + lt_cv_file_magic_test_file=/usr/lib/pa20_64/libc.sl + ;; + *) + lt_cv_deplibs_check_method='file_magic (s[[0-9]][[0-9]][[0-9]]|PA-RISC[[0-9]]\.[[0-9]]) shared library' + lt_cv_file_magic_test_file=/usr/lib/libc.sl + ;; + esac + ;; + +interix[[3-9]]*) + # PIC code is broken on Interix 3.x, that's why |\.a not |_pic\.a here + lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so|\.a)$' + ;; + +irix5* | irix6* | nonstopux*) + case $LD in + *-32|*"-32 ") libmagic=32-bit;; + *-n32|*"-n32 ") libmagic=N32;; + *-64|*"-64 ") libmagic=64-bit;; + *) libmagic=never-match;; + esac + lt_cv_deplibs_check_method=pass_all + ;; + +# This must be glibc/ELF. +linux* | k*bsd*-gnu | kopensolaris*-gnu) + lt_cv_deplibs_check_method=pass_all + ;; + +netbsd* | netbsdelf*-gnu) + if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then + lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|_pic\.a)$' + else + lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so|_pic\.a)$' + fi + ;; + +newos6*) + lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (executable|dynamic lib)' + lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_test_file=/usr/lib/libnls.so + ;; + +*nto* | *qnx*) + lt_cv_deplibs_check_method=pass_all + ;; + +openbsd*) + if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|\.so|_pic\.a)$' + else + lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|_pic\.a)$' + fi + ;; + +osf3* | osf4* | osf5*) + lt_cv_deplibs_check_method=pass_all + ;; + +rdos*) + lt_cv_deplibs_check_method=pass_all + ;; + +solaris*) + lt_cv_deplibs_check_method=pass_all + ;; + +sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) + lt_cv_deplibs_check_method=pass_all + ;; + +sysv4 | sysv4.3*) + case $host_vendor in + motorola) + lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (shared object|dynamic lib) M[[0-9]][[0-9]]* Version [[0-9]]' + lt_cv_file_magic_test_file=`echo /usr/lib/libc.so*` + ;; + ncr) + lt_cv_deplibs_check_method=pass_all + ;; + sequent) + lt_cv_file_magic_cmd='/bin/file' + lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[LM]]SB (shared object|dynamic lib )' + ;; + sni) + lt_cv_file_magic_cmd='/bin/file' + lt_cv_deplibs_check_method="file_magic ELF [[0-9]][[0-9]]*-bit [[LM]]SB dynamic lib" + lt_cv_file_magic_test_file=/lib/libc.so + ;; + siemens) + lt_cv_deplibs_check_method=pass_all + ;; + pc) + lt_cv_deplibs_check_method=pass_all + ;; + esac + ;; + +tpf*) + lt_cv_deplibs_check_method=pass_all + ;; +esac +]) + +file_magic_glob= +want_nocaseglob=no +if test "$build" = "$host"; then + case $host_os in + mingw* | pw32*) + if ( shopt | grep nocaseglob ) >/dev/null 2>&1; then + want_nocaseglob=yes + else + file_magic_glob=`echo aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ | $SED -e "s/\(..\)/s\/[[\1]]\/[[\1]]\/g;/g"` + fi + ;; + esac +fi + +file_magic_cmd=$lt_cv_file_magic_cmd +deplibs_check_method=$lt_cv_deplibs_check_method +test -z "$deplibs_check_method" && deplibs_check_method=unknown + +_LT_DECL([], [deplibs_check_method], [1], + [Method to check whether dependent libraries are shared objects]) +_LT_DECL([], [file_magic_cmd], [1], + [Command to use when deplibs_check_method = "file_magic"]) +_LT_DECL([], [file_magic_glob], [1], + [How to find potential files when deplibs_check_method = "file_magic"]) +_LT_DECL([], [want_nocaseglob], [1], + [Find potential files using nocaseglob when deplibs_check_method = "file_magic"]) +])# _LT_CHECK_MAGIC_METHOD + + +# LT_PATH_NM +# ---------- +# find the pathname to a BSD- or MS-compatible name lister +AC_DEFUN([LT_PATH_NM], +[AC_REQUIRE([AC_PROG_CC])dnl +AC_CACHE_CHECK([for BSD- or MS-compatible name lister (nm)], lt_cv_path_NM, +[if test -n "$NM"; then + # Let the user override the test. + lt_cv_path_NM="$NM" +else + lt_nm_to_check="${ac_tool_prefix}nm" + if test -n "$ac_tool_prefix" && test "$build" = "$host"; then + lt_nm_to_check="$lt_nm_to_check nm" + fi + for lt_tmp_nm in $lt_nm_to_check; do + lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR + for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do + IFS="$lt_save_ifs" + test -z "$ac_dir" && ac_dir=. + tmp_nm="$ac_dir/$lt_tmp_nm" + if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext" ; then + # Check to see if the nm accepts a BSD-compat flag. + # Adding the `sed 1q' prevents false positives on HP-UX, which says: + # nm: unknown option "B" ignored + # Tru64's nm complains that /dev/null is an invalid object file + case `"$tmp_nm" -B /dev/null 2>&1 | sed '1q'` in + */dev/null* | *'Invalid file or object type'*) + lt_cv_path_NM="$tmp_nm -B" + break + ;; + *) + case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in + */dev/null*) + lt_cv_path_NM="$tmp_nm -p" + break + ;; + *) + lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but + continue # so that we can try to find one that supports BSD flags + ;; + esac + ;; + esac + fi + done + IFS="$lt_save_ifs" + done + : ${lt_cv_path_NM=no} +fi]) +if test "$lt_cv_path_NM" != "no"; then + NM="$lt_cv_path_NM" +else + # Didn't find any BSD compatible name lister, look for dumpbin. + if test -n "$DUMPBIN"; then : + # Let the user override the test. + else + AC_CHECK_TOOLS(DUMPBIN, [dumpbin "link -dump"], :) + case `$DUMPBIN -symbols /dev/null 2>&1 | sed '1q'` in + *COFF*) + DUMPBIN="$DUMPBIN -symbols" + ;; + *) + DUMPBIN=: + ;; + esac + fi + AC_SUBST([DUMPBIN]) + if test "$DUMPBIN" != ":"; then + NM="$DUMPBIN" + fi +fi +test -z "$NM" && NM=nm +AC_SUBST([NM]) +_LT_DECL([], [NM], [1], [A BSD- or MS-compatible name lister])dnl + +AC_CACHE_CHECK([the name lister ($NM) interface], [lt_cv_nm_interface], + [lt_cv_nm_interface="BSD nm" + echo "int some_variable = 0;" > conftest.$ac_ext + (eval echo "\"\$as_me:$LINENO: $ac_compile\"" >&AS_MESSAGE_LOG_FD) + (eval "$ac_compile" 2>conftest.err) + cat conftest.err >&AS_MESSAGE_LOG_FD + (eval echo "\"\$as_me:$LINENO: $NM \\\"conftest.$ac_objext\\\"\"" >&AS_MESSAGE_LOG_FD) + (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) + cat conftest.err >&AS_MESSAGE_LOG_FD + (eval echo "\"\$as_me:$LINENO: output\"" >&AS_MESSAGE_LOG_FD) + cat conftest.out >&AS_MESSAGE_LOG_FD + if $GREP 'External.*some_variable' conftest.out > /dev/null; then + lt_cv_nm_interface="MS dumpbin" + fi + rm -f conftest*]) +])# LT_PATH_NM + +# Old names: +AU_ALIAS([AM_PROG_NM], [LT_PATH_NM]) +AU_ALIAS([AC_PROG_NM], [LT_PATH_NM]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AM_PROG_NM], []) +dnl AC_DEFUN([AC_PROG_NM], []) + +# _LT_CHECK_SHAREDLIB_FROM_LINKLIB +# -------------------------------- +# how to determine the name of the shared library +# associated with a specific link library. +# -- PORTME fill in with the dynamic library characteristics +m4_defun([_LT_CHECK_SHAREDLIB_FROM_LINKLIB], +[m4_require([_LT_DECL_EGREP]) +m4_require([_LT_DECL_OBJDUMP]) +m4_require([_LT_DECL_DLLTOOL]) +AC_CACHE_CHECK([how to associate runtime and link libraries], +lt_cv_sharedlib_from_linklib_cmd, +[lt_cv_sharedlib_from_linklib_cmd='unknown' + +case $host_os in +cygwin* | mingw* | pw32* | cegcc*) + # two different shell functions defined in ltmain.sh + # decide which to use based on capabilities of $DLLTOOL + case `$DLLTOOL --help 2>&1` in + *--identify-strict*) + lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib + ;; + *) + lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib_fallback + ;; + esac + ;; +*) + # fallback: assume linklib IS sharedlib + lt_cv_sharedlib_from_linklib_cmd="$ECHO" + ;; +esac +]) +sharedlib_from_linklib_cmd=$lt_cv_sharedlib_from_linklib_cmd +test -z "$sharedlib_from_linklib_cmd" && sharedlib_from_linklib_cmd=$ECHO + +_LT_DECL([], [sharedlib_from_linklib_cmd], [1], + [Command to associate shared and link libraries]) +])# _LT_CHECK_SHAREDLIB_FROM_LINKLIB + + +# _LT_PATH_MANIFEST_TOOL +# ---------------------- +# locate the manifest tool +m4_defun([_LT_PATH_MANIFEST_TOOL], +[AC_CHECK_TOOL(MANIFEST_TOOL, mt, :) +test -z "$MANIFEST_TOOL" && MANIFEST_TOOL=mt +AC_CACHE_CHECK([if $MANIFEST_TOOL is a manifest tool], [lt_cv_path_mainfest_tool], + [lt_cv_path_mainfest_tool=no + echo "$as_me:$LINENO: $MANIFEST_TOOL '-?'" >&AS_MESSAGE_LOG_FD + $MANIFEST_TOOL '-?' 2>conftest.err > conftest.out + cat conftest.err >&AS_MESSAGE_LOG_FD + if $GREP 'Manifest Tool' conftest.out > /dev/null; then + lt_cv_path_mainfest_tool=yes + fi + rm -f conftest*]) +if test "x$lt_cv_path_mainfest_tool" != xyes; then + MANIFEST_TOOL=: +fi +_LT_DECL([], [MANIFEST_TOOL], [1], [Manifest tool])dnl +])# _LT_PATH_MANIFEST_TOOL + + +# LT_LIB_M +# -------- +# check for math library +AC_DEFUN([LT_LIB_M], +[AC_REQUIRE([AC_CANONICAL_HOST])dnl +LIBM= +case $host in +*-*-beos* | *-*-cegcc* | *-*-cygwin* | *-*-haiku* | *-*-pw32* | *-*-darwin*) + # These system don't have libm, or don't need it + ;; +*-ncr-sysv4.3*) + AC_CHECK_LIB(mw, _mwvalidcheckl, LIBM="-lmw") + AC_CHECK_LIB(m, cos, LIBM="$LIBM -lm") + ;; +*) + AC_CHECK_LIB(m, cos, LIBM="-lm") + ;; +esac +AC_SUBST([LIBM]) +])# LT_LIB_M + +# Old name: +AU_ALIAS([AC_CHECK_LIBM], [LT_LIB_M]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_CHECK_LIBM], []) + + +# _LT_COMPILER_NO_RTTI([TAGNAME]) +# ------------------------------- +m4_defun([_LT_COMPILER_NO_RTTI], +[m4_require([_LT_TAG_COMPILER])dnl + +_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)= + +if test "$GCC" = yes; then + case $cc_basename in + nvcc*) + _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -Xcompiler -fno-builtin' ;; + *) + _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -fno-builtin' ;; + esac + + _LT_COMPILER_OPTION([if $compiler supports -fno-rtti -fno-exceptions], + lt_cv_prog_compiler_rtti_exceptions, + [-fno-rtti -fno-exceptions], [], + [_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)="$_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1) -fno-rtti -fno-exceptions"]) +fi +_LT_TAGDECL([no_builtin_flag], [lt_prog_compiler_no_builtin_flag], [1], + [Compiler flag to turn off builtin functions]) +])# _LT_COMPILER_NO_RTTI + + +# _LT_CMD_GLOBAL_SYMBOLS +# ---------------------- +m4_defun([_LT_CMD_GLOBAL_SYMBOLS], +[AC_REQUIRE([AC_CANONICAL_HOST])dnl +AC_REQUIRE([AC_PROG_CC])dnl +AC_REQUIRE([AC_PROG_AWK])dnl +AC_REQUIRE([LT_PATH_NM])dnl +AC_REQUIRE([LT_PATH_LD])dnl +m4_require([_LT_DECL_SED])dnl +m4_require([_LT_DECL_EGREP])dnl +m4_require([_LT_TAG_COMPILER])dnl + +# Check for command to grab the raw symbol name followed by C symbol from nm. +AC_MSG_CHECKING([command to parse $NM output from $compiler object]) +AC_CACHE_VAL([lt_cv_sys_global_symbol_pipe], +[ +# These are sane defaults that work on at least a few old systems. +# [They come from Ultrix. What could be older than Ultrix?!! ;)] + +# Character class describing NM global symbol codes. +symcode='[[BCDEGRST]]' + +# Regexp to match symbols that can be accessed directly from C. +sympat='\([[_A-Za-z]][[_A-Za-z0-9]]*\)' + +# Define system-specific variables. +case $host_os in +aix*) + symcode='[[BCDT]]' + ;; +cygwin* | mingw* | pw32* | cegcc*) + symcode='[[ABCDGISTW]]' + ;; +hpux*) + if test "$host_cpu" = ia64; then + symcode='[[ABCDEGRST]]' + fi + ;; +irix* | nonstopux*) + symcode='[[BCDEGRST]]' + ;; +osf*) + symcode='[[BCDEGQRST]]' + ;; +solaris*) + symcode='[[BDRT]]' + ;; +sco3.2v5*) + symcode='[[DT]]' + ;; +sysv4.2uw2*) + symcode='[[DT]]' + ;; +sysv5* | sco5v6* | unixware* | OpenUNIX*) + symcode='[[ABDT]]' + ;; +sysv4) + symcode='[[DFNSTU]]' + ;; +esac + +# If we're using GNU nm, then use its standard symbol codes. +case `$NM -V 2>&1` in +*GNU* | *'with BFD'*) + symcode='[[ABCDGIRSTW]]' ;; +esac + +# Transform an extracted symbol line into a proper C declaration. +# Some systems (esp. on ia64) link data and code symbols differently, +# so use this general approach. +lt_cv_sys_global_symbol_to_cdecl="sed -n -e 's/^T .* \(.*\)$/extern int \1();/p' -e 's/^$symcode* .* \(.*\)$/extern char \1;/p'" + +# Transform an extracted symbol line into symbol name and symbol address +lt_cv_sys_global_symbol_to_c_name_address="sed -n -e 's/^: \([[^ ]]*\)[[ ]]*$/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([[^ ]]*\) \([[^ ]]*\)$/ {\"\2\", (void *) \&\2},/p'" +lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n -e 's/^: \([[^ ]]*\)[[ ]]*$/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([[^ ]]*\) \(lib[[^ ]]*\)$/ {\"\2\", (void *) \&\2},/p' -e 's/^$symcode* \([[^ ]]*\) \([[^ ]]*\)$/ {\"lib\2\", (void *) \&\2},/p'" + +# Handle CRLF in mingw tool chain +opt_cr= +case $build_os in +mingw*) + opt_cr=`$ECHO 'x\{0,1\}' | tr x '\015'` # option cr in regexp + ;; +esac + +# Try without a prefix underscore, then with it. +for ac_symprfx in "" "_"; do + + # Transform symcode, sympat, and symprfx into a raw symbol and a C symbol. + symxfrm="\\1 $ac_symprfx\\2 \\2" + + # Write the raw and C identifiers. + if test "$lt_cv_nm_interface" = "MS dumpbin"; then + # Fake it for dumpbin and say T for any non-static function + # and D for any global variable. + # Also find C++ and __fastcall symbols from MSVC++, + # which start with @ or ?. + lt_cv_sys_global_symbol_pipe="$AWK ['"\ +" {last_section=section; section=\$ 3};"\ +" /^COFF SYMBOL TABLE/{for(i in hide) delete hide[i]};"\ +" /Section length .*#relocs.*(pick any)/{hide[last_section]=1};"\ +" \$ 0!~/External *\|/{next};"\ +" / 0+ UNDEF /{next}; / UNDEF \([^|]\)*()/{next};"\ +" {if(hide[section]) next};"\ +" {f=0}; \$ 0~/\(\).*\|/{f=1}; {printf f ? \"T \" : \"D \"};"\ +" {split(\$ 0, a, /\||\r/); split(a[2], s)};"\ +" s[1]~/^[@?]/{print s[1], s[1]; next};"\ +" s[1]~prfx {split(s[1],t,\"@\"); print t[1], substr(t[1],length(prfx))}"\ +" ' prfx=^$ac_symprfx]" + else + lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[[ ]]\($symcode$symcode*\)[[ ]][[ ]]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" + fi + lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | sed '/ __gnu_lto/d'" + + # Check to see that the pipe works correctly. + pipe_works=no + + rm -f conftest* + cat > conftest.$ac_ext <<_LT_EOF +#ifdef __cplusplus +extern "C" { +#endif +char nm_test_var; +void nm_test_func(void); +void nm_test_func(void){} +#ifdef __cplusplus +} +#endif +int main(){nm_test_var='a';nm_test_func();return(0);} +_LT_EOF + + if AC_TRY_EVAL(ac_compile); then + # Now try to grab the symbols. + nlist=conftest.nm + if AC_TRY_EVAL(NM conftest.$ac_objext \| "$lt_cv_sys_global_symbol_pipe" \> $nlist) && test -s "$nlist"; then + # Try sorting and uniquifying the output. + if sort "$nlist" | uniq > "$nlist"T; then + mv -f "$nlist"T "$nlist" + else + rm -f "$nlist"T + fi + + # Make sure that we snagged all the symbols we need. + if $GREP ' nm_test_var$' "$nlist" >/dev/null; then + if $GREP ' nm_test_func$' "$nlist" >/dev/null; then + cat <<_LT_EOF > conftest.$ac_ext +/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ +#if defined(_WIN32) || defined(__CYGWIN__) || defined(_WIN32_WCE) +/* DATA imports from DLLs on WIN32 con't be const, because runtime + relocations are performed -- see ld's documentation on pseudo-relocs. */ +# define LT@&t@_DLSYM_CONST +#elif defined(__osf__) +/* This system does not cope well with relocations in const data. */ +# define LT@&t@_DLSYM_CONST +#else +# define LT@&t@_DLSYM_CONST const +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +_LT_EOF + # Now generate the symbol file. + eval "$lt_cv_sys_global_symbol_to_cdecl"' < "$nlist" | $GREP -v main >> conftest.$ac_ext' + + cat <<_LT_EOF >> conftest.$ac_ext + +/* The mapping between symbol names and symbols. */ +LT@&t@_DLSYM_CONST struct { + const char *name; + void *address; +} +lt__PROGRAM__LTX_preloaded_symbols[[]] = +{ + { "@PROGRAM@", (void *) 0 }, +_LT_EOF + $SED "s/^$symcode$symcode* \(.*\) \(.*\)$/ {\"\2\", (void *) \&\2},/" < "$nlist" | $GREP -v main >> conftest.$ac_ext + cat <<\_LT_EOF >> conftest.$ac_ext + {0, (void *) 0} +}; + +/* This works around a problem in FreeBSD linker */ +#ifdef FREEBSD_WORKAROUND +static const void *lt_preloaded_setup() { + return lt__PROGRAM__LTX_preloaded_symbols; +} +#endif + +#ifdef __cplusplus +} +#endif +_LT_EOF + # Now try linking the two files. + mv conftest.$ac_objext conftstm.$ac_objext + lt_globsym_save_LIBS=$LIBS + lt_globsym_save_CFLAGS=$CFLAGS + LIBS="conftstm.$ac_objext" + CFLAGS="$CFLAGS$_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)" + if AC_TRY_EVAL(ac_link) && test -s conftest${ac_exeext}; then + pipe_works=yes + fi + LIBS=$lt_globsym_save_LIBS + CFLAGS=$lt_globsym_save_CFLAGS + else + echo "cannot find nm_test_func in $nlist" >&AS_MESSAGE_LOG_FD + fi + else + echo "cannot find nm_test_var in $nlist" >&AS_MESSAGE_LOG_FD + fi + else + echo "cannot run $lt_cv_sys_global_symbol_pipe" >&AS_MESSAGE_LOG_FD + fi + else + echo "$progname: failed program was:" >&AS_MESSAGE_LOG_FD + cat conftest.$ac_ext >&5 + fi + rm -rf conftest* conftst* + + # Do not use the global_symbol_pipe unless it works. + if test "$pipe_works" = yes; then + break + else + lt_cv_sys_global_symbol_pipe= + fi +done +]) +if test -z "$lt_cv_sys_global_symbol_pipe"; then + lt_cv_sys_global_symbol_to_cdecl= +fi +if test -z "$lt_cv_sys_global_symbol_pipe$lt_cv_sys_global_symbol_to_cdecl"; then + AC_MSG_RESULT(failed) +else + AC_MSG_RESULT(ok) +fi + +# Response file support. +if test "$lt_cv_nm_interface" = "MS dumpbin"; then + nm_file_list_spec='@' +elif $NM --help 2>/dev/null | grep '[[@]]FILE' >/dev/null; then + nm_file_list_spec='@' +fi + +_LT_DECL([global_symbol_pipe], [lt_cv_sys_global_symbol_pipe], [1], + [Take the output of nm and produce a listing of raw symbols and C names]) +_LT_DECL([global_symbol_to_cdecl], [lt_cv_sys_global_symbol_to_cdecl], [1], + [Transform the output of nm in a proper C declaration]) +_LT_DECL([global_symbol_to_c_name_address], + [lt_cv_sys_global_symbol_to_c_name_address], [1], + [Transform the output of nm in a C name address pair]) +_LT_DECL([global_symbol_to_c_name_address_lib_prefix], + [lt_cv_sys_global_symbol_to_c_name_address_lib_prefix], [1], + [Transform the output of nm in a C name address pair when lib prefix is needed]) +_LT_DECL([], [nm_file_list_spec], [1], + [Specify filename containing input files for $NM]) +]) # _LT_CMD_GLOBAL_SYMBOLS + + +# _LT_COMPILER_PIC([TAGNAME]) +# --------------------------- +m4_defun([_LT_COMPILER_PIC], +[m4_require([_LT_TAG_COMPILER])dnl +_LT_TAGVAR(lt_prog_compiler_wl, $1)= +_LT_TAGVAR(lt_prog_compiler_pic, $1)= +_LT_TAGVAR(lt_prog_compiler_static, $1)= + +m4_if([$1], [CXX], [ + # C++ specific cases for pic, static, wl, etc. + if test "$GXX" = yes; then + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' + + case $host_os in + aix*) + # All AIX code is PIC. + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + ;; + m68k) + # FIXME: we need at least 68020 code to build shared libraries, but + # adding the `-m68020' flag to GCC prevents building anything better, + # like `-m68040'. + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-m68020 -resident32 -malways-restore-a4' + ;; + esac + ;; + + beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*) + # PIC is the default for these OSes. + ;; + mingw* | cygwin* | os2* | pw32* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + # Although the cygwin gcc ignores -fPIC, still need this for old-style + # (--disable-auto-import) libraries + m4_if([$1], [GCJ], [], + [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) + ;; + darwin* | rhapsody*) + # PIC is the default on this platform + # Common symbols not allowed in MH_DYLIB files + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fno-common' + ;; + *djgpp*) + # DJGPP does not support shared libraries at all + _LT_TAGVAR(lt_prog_compiler_pic, $1)= + ;; + haiku*) + # PIC is the default for Haiku. + # The "-static" flag exists, but is broken. + _LT_TAGVAR(lt_prog_compiler_static, $1)= + ;; + interix[[3-9]]*) + # Interix 3.x gcc -fpic/-fPIC options generate broken code. + # Instead, we relocate shared libraries at runtime. + ;; + sysv4*MP*) + if test -d /usr/nec; then + _LT_TAGVAR(lt_prog_compiler_pic, $1)=-Kconform_pic + fi + ;; + hpux*) + # PIC is the default for 64-bit PA HP-UX, but not for 32-bit + # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag + # sets the default TLS model and affects inlining. + case $host_cpu in + hppa*64*) + ;; + *) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + ;; + esac + ;; + *qnx* | *nto*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' + ;; + *) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + ;; + esac + else + case $host_os in + aix[[4-9]]*) + # All AIX code is PIC. + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + else + _LT_TAGVAR(lt_prog_compiler_static, $1)='-bnso -bI:/lib/syscalls.exp' + fi + ;; + chorus*) + case $cc_basename in + cxch68*) + # Green Hills C++ Compiler + # _LT_TAGVAR(lt_prog_compiler_static, $1)="--no_auto_instantiation -u __main -u __premain -u _abort -r $COOL_DIR/lib/libOrb.a $MVME_DIR/lib/CC/libC.a $MVME_DIR/lib/classix/libcx.s.a" + ;; + esac + ;; + mingw* | cygwin* | os2* | pw32* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + m4_if([$1], [GCJ], [], + [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) + ;; + dgux*) + case $cc_basename in + ec++*) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + ;; + ghcx*) + # Green Hills C++ Compiler + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' + ;; + *) + ;; + esac + ;; + freebsd* | dragonfly*) + # FreeBSD uses GNU C++ + ;; + hpux9* | hpux10* | hpux11*) + case $cc_basename in + CC*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_static, $1)='${wl}-a ${wl}archive' + if test "$host_cpu" != ia64; then + _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z' + fi + ;; + aCC*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_static, $1)='${wl}-a ${wl}archive' + case $host_cpu in + hppa*64*|ia64*) + # +Z the default + ;; + *) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z' + ;; + esac + ;; + *) + ;; + esac + ;; + interix*) + # This is c89, which is MS Visual C++ (no shared libs) + # Anyone wants to do a port? + ;; + irix5* | irix6* | nonstopux*) + case $cc_basename in + CC*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' + # CC pic flag -KPIC is the default. + ;; + *) + ;; + esac + ;; + linux* | k*bsd*-gnu | kopensolaris*-gnu) + case $cc_basename in + KCC*) + # KAI C++ Compiler + _LT_TAGVAR(lt_prog_compiler_wl, $1)='--backend -Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + ;; + ecpc* ) + # old Intel C++ for x86_64 which still supported -KPIC. + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' + ;; + icpc* ) + # Intel C++, used to be incompatible with GCC. + # ICC 10 doesn't accept -KPIC any more. + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' + ;; + pgCC* | pgcpp*) + # Portland Group C++ compiler + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + cxx*) + # Compaq C++ + # Make sure the PIC flag is empty. It appears that all Alpha + # Linux and Compaq Tru64 Unix objects are PIC. + _LT_TAGVAR(lt_prog_compiler_pic, $1)= + _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' + ;; + xlc* | xlC* | bgxl[[cC]]* | mpixl[[cC]]*) + # IBM XL 8.0, 9.0 on PPC and BlueGene + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-qpic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink' + ;; + *) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) + # Sun C++ 5.9 + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' + ;; + esac + ;; + esac + ;; + lynxos*) + ;; + m88k*) + ;; + mvs*) + case $cc_basename in + cxx*) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-W c,exportall' + ;; + *) + ;; + esac + ;; + netbsd* | netbsdelf*-gnu) + ;; + *qnx* | *nto*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' + ;; + osf3* | osf4* | osf5*) + case $cc_basename in + KCC*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='--backend -Wl,' + ;; + RCC*) + # Rational C++ 2.4.1 + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' + ;; + cxx*) + # Digital/Compaq C++ + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + # Make sure the PIC flag is empty. It appears that all Alpha + # Linux and Compaq Tru64 Unix objects are PIC. + _LT_TAGVAR(lt_prog_compiler_pic, $1)= + _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' + ;; + *) + ;; + esac + ;; + psos*) + ;; + solaris*) + case $cc_basename in + CC* | sunCC*) + # Sun C++ 4.2, 5.x and Centerline C++ + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' + ;; + gcx*) + # Green Hills C++ Compiler + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC' + ;; + *) + ;; + esac + ;; + sunos4*) + case $cc_basename in + CC*) + # Sun C++ 4.x + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + lcc*) + # Lucid + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' + ;; + *) + ;; + esac + ;; + sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*) + case $cc_basename in + CC*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + esac + ;; + tandem*) + case $cc_basename in + NCC*) + # NonStop-UX NCC 3.20 + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + ;; + *) + ;; + esac + ;; + vxworks*) + ;; + *) + _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no + ;; + esac + fi +], +[ + if test "$GCC" = yes; then + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' + + case $host_os in + aix*) + # All AIX code is PIC. + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + ;; + m68k) + # FIXME: we need at least 68020 code to build shared libraries, but + # adding the `-m68020' flag to GCC prevents building anything better, + # like `-m68040'. + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-m68020 -resident32 -malways-restore-a4' + ;; + esac + ;; + + beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*) + # PIC is the default for these OSes. + ;; + + mingw* | cygwin* | pw32* | os2* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + # Although the cygwin gcc ignores -fPIC, still need this for old-style + # (--disable-auto-import) libraries + m4_if([$1], [GCJ], [], + [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) + ;; + + darwin* | rhapsody*) + # PIC is the default on this platform + # Common symbols not allowed in MH_DYLIB files + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fno-common' + ;; + + haiku*) + # PIC is the default for Haiku. + # The "-static" flag exists, but is broken. + _LT_TAGVAR(lt_prog_compiler_static, $1)= + ;; + + hpux*) + # PIC is the default for 64-bit PA HP-UX, but not for 32-bit + # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag + # sets the default TLS model and affects inlining. + case $host_cpu in + hppa*64*) + # +Z the default + ;; + *) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + ;; + esac + ;; + + interix[[3-9]]*) + # Interix 3.x gcc -fpic/-fPIC options generate broken code. + # Instead, we relocate shared libraries at runtime. + ;; + + msdosdjgpp*) + # Just because we use GCC doesn't mean we suddenly get shared libraries + # on systems that don't support them. + _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no + enable_shared=no + ;; + + *nto* | *qnx*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' + ;; + + sysv4*MP*) + if test -d /usr/nec; then + _LT_TAGVAR(lt_prog_compiler_pic, $1)=-Kconform_pic + fi + ;; + + *) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + ;; + esac + + case $cc_basename in + nvcc*) # Cuda Compiler Driver 2.2 + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Xlinker ' + if test -n "$_LT_TAGVAR(lt_prog_compiler_pic, $1)"; then + _LT_TAGVAR(lt_prog_compiler_pic, $1)="-Xcompiler $_LT_TAGVAR(lt_prog_compiler_pic, $1)" + fi + ;; + esac + else + # PORTME Check for flag to pass linker flags through the system compiler. + case $host_os in + aix*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + if test "$host_cpu" = ia64; then + # AIX 5 now supports IA64 processor + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + else + _LT_TAGVAR(lt_prog_compiler_static, $1)='-bnso -bI:/lib/syscalls.exp' + fi + ;; + + mingw* | cygwin* | pw32* | os2* | cegcc*) + # This hack is so that the source file can tell whether it is being + # built for inclusion in a dll (and should export symbols for example). + m4_if([$1], [GCJ], [], + [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) + ;; + + hpux9* | hpux10* | hpux11*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + # PIC is the default for IA64 HP-UX and 64-bit HP-UX, but + # not for PA HP-UX. + case $host_cpu in + hppa*64*|ia64*) + # +Z the default + ;; + *) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z' + ;; + esac + # Is there a better lt_prog_compiler_static that works with the bundled CC? + _LT_TAGVAR(lt_prog_compiler_static, $1)='${wl}-a ${wl}archive' + ;; + + irix5* | irix6* | nonstopux*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + # PIC (with -KPIC) is the default. + _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' + ;; + + linux* | k*bsd*-gnu | kopensolaris*-gnu) + case $cc_basename in + # old Intel for x86_64 which still supported -KPIC. + ecc*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' + ;; + # icc used to be incompatible with GCC. + # ICC 10 doesn't accept -KPIC any more. + icc* | ifort*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' + ;; + # Lahey Fortran 8.1. + lf95*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='--shared' + _LT_TAGVAR(lt_prog_compiler_static, $1)='--static' + ;; + nagfor*) + # NAG Fortran compiler + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,-Wl,,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + pgcc* | pgf77* | pgf90* | pgf95* | pgfortran*) + # Portland Group compilers (*not* the Pentium gcc compiler, + # which looks to be a dead project) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + ccc*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + # All Alpha code is PIC. + _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' + ;; + xl* | bgxl* | bgf* | mpixl*) + # IBM XL C 8.0/Fortran 10.1, 11.1 on PPC and BlueGene + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-qpic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink' + ;; + *) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ Ceres\ Fortran* | *Sun*Fortran*\ [[1-7]].* | *Sun*Fortran*\ 8.[[0-3]]*) + # Sun Fortran 8.3 passes all unrecognized flags to the linker + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + _LT_TAGVAR(lt_prog_compiler_wl, $1)='' + ;; + *Sun\ F* | *Sun*Fortran*) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' + ;; + *Sun\ C*) + # Sun C 5.9 + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + ;; + *Intel*\ [[CF]]*Compiler*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' + ;; + *Portland\ Group*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + esac + ;; + esac + ;; + + newsos6) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + + *nto* | *qnx*) + # QNX uses GNU C++, but need to define -shared option too, otherwise + # it will coredump. + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' + ;; + + osf3* | osf4* | osf5*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + # All OSF/1 code is PIC. + _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' + ;; + + rdos*) + _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' + ;; + + solaris*) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + case $cc_basename in + f77* | f90* | f95* | sunf77* | sunf90* | sunf95*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ';; + *) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,';; + esac + ;; + + sunos4*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + + sysv4 | sysv4.2uw2* | sysv4.3*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + + sysv4*MP*) + if test -d /usr/nec ;then + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-Kconform_pic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + fi + ;; + + sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + + unicos*) + _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' + _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no + ;; + + uts4*) + _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' + _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' + ;; + + *) + _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no + ;; + esac + fi +]) +case $host_os in + # For platforms which do not support PIC, -DPIC is meaningless: + *djgpp*) + _LT_TAGVAR(lt_prog_compiler_pic, $1)= + ;; + *) + _LT_TAGVAR(lt_prog_compiler_pic, $1)="$_LT_TAGVAR(lt_prog_compiler_pic, $1)@&t@m4_if([$1],[],[ -DPIC],[m4_if([$1],[CXX],[ -DPIC],[])])" + ;; +esac + +AC_CACHE_CHECK([for $compiler option to produce PIC], + [_LT_TAGVAR(lt_cv_prog_compiler_pic, $1)], + [_LT_TAGVAR(lt_cv_prog_compiler_pic, $1)=$_LT_TAGVAR(lt_prog_compiler_pic, $1)]) +_LT_TAGVAR(lt_prog_compiler_pic, $1)=$_LT_TAGVAR(lt_cv_prog_compiler_pic, $1) + +# +# Check to make sure the PIC flag actually works. +# +if test -n "$_LT_TAGVAR(lt_prog_compiler_pic, $1)"; then + _LT_COMPILER_OPTION([if $compiler PIC flag $_LT_TAGVAR(lt_prog_compiler_pic, $1) works], + [_LT_TAGVAR(lt_cv_prog_compiler_pic_works, $1)], + [$_LT_TAGVAR(lt_prog_compiler_pic, $1)@&t@m4_if([$1],[],[ -DPIC],[m4_if([$1],[CXX],[ -DPIC],[])])], [], + [case $_LT_TAGVAR(lt_prog_compiler_pic, $1) in + "" | " "*) ;; + *) _LT_TAGVAR(lt_prog_compiler_pic, $1)=" $_LT_TAGVAR(lt_prog_compiler_pic, $1)" ;; + esac], + [_LT_TAGVAR(lt_prog_compiler_pic, $1)= + _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no]) +fi +_LT_TAGDECL([pic_flag], [lt_prog_compiler_pic], [1], + [Additional compiler flags for building library objects]) + +_LT_TAGDECL([wl], [lt_prog_compiler_wl], [1], + [How to pass a linker flag through the compiler]) +# +# Check to make sure the static flag actually works. +# +wl=$_LT_TAGVAR(lt_prog_compiler_wl, $1) eval lt_tmp_static_flag=\"$_LT_TAGVAR(lt_prog_compiler_static, $1)\" +_LT_LINKER_OPTION([if $compiler static flag $lt_tmp_static_flag works], + _LT_TAGVAR(lt_cv_prog_compiler_static_works, $1), + $lt_tmp_static_flag, + [], + [_LT_TAGVAR(lt_prog_compiler_static, $1)=]) +_LT_TAGDECL([link_static_flag], [lt_prog_compiler_static], [1], + [Compiler flag to prevent dynamic linking]) +])# _LT_COMPILER_PIC + + +# _LT_LINKER_SHLIBS([TAGNAME]) +# ---------------------------- +# See if the linker supports building shared libraries. +m4_defun([_LT_LINKER_SHLIBS], +[AC_REQUIRE([LT_PATH_LD])dnl +AC_REQUIRE([LT_PATH_NM])dnl +m4_require([_LT_PATH_MANIFEST_TOOL])dnl +m4_require([_LT_FILEUTILS_DEFAULTS])dnl +m4_require([_LT_DECL_EGREP])dnl +m4_require([_LT_DECL_SED])dnl +m4_require([_LT_CMD_GLOBAL_SYMBOLS])dnl +m4_require([_LT_TAG_COMPILER])dnl +AC_MSG_CHECKING([whether the $compiler linker ($LD) supports shared libraries]) +m4_if([$1], [CXX], [ + _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' + _LT_TAGVAR(exclude_expsyms, $1)=['_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*'] + case $host_os in + aix[[4-9]]*) + # If we're using GNU nm, then we don't want the "-C" option. + # -C means demangle to AIX nm, but means don't demangle with GNU nm + # Also, AIX nm treats weak defined symbols like other global defined + # symbols, whereas GNU nm marks them as "W". + if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then + _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + else + _LT_TAGVAR(export_symbols_cmds, $1)='$NM -BCpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + fi + ;; + pw32*) + _LT_TAGVAR(export_symbols_cmds, $1)="$ltdll_cmds" + ;; + cygwin* | mingw* | cegcc*) + case $cc_basename in + cl*) + _LT_TAGVAR(exclude_expsyms, $1)='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' + ;; + *) + _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1 DATA/;s/^.*[[ ]]__nm__\([[^ ]]*\)[[ ]][[^ ]]*/\1 DATA/;/^I[[ ]]/d;/^[[AITW]][[ ]]/s/.* //'\'' | sort | uniq > $export_symbols' + _LT_TAGVAR(exclude_expsyms, $1)=['[_]+GLOBAL_OFFSET_TABLE_|[_]+GLOBAL__[FID]_.*|[_]+head_[A-Za-z0-9_]+_dll|[A-Za-z0-9_]+_dll_iname'] + ;; + esac + ;; + linux* | k*bsd*-gnu | gnu*) + _LT_TAGVAR(link_all_deplibs, $1)=no + ;; + *) + _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' + ;; + esac +], [ + runpath_var= + _LT_TAGVAR(allow_undefined_flag, $1)= + _LT_TAGVAR(always_export_symbols, $1)=no + _LT_TAGVAR(archive_cmds, $1)= + _LT_TAGVAR(archive_expsym_cmds, $1)= + _LT_TAGVAR(compiler_needs_object, $1)=no + _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no + _LT_TAGVAR(export_dynamic_flag_spec, $1)= + _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' + _LT_TAGVAR(hardcode_automatic, $1)=no + _LT_TAGVAR(hardcode_direct, $1)=no + _LT_TAGVAR(hardcode_direct_absolute, $1)=no + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)= + _LT_TAGVAR(hardcode_libdir_separator, $1)= + _LT_TAGVAR(hardcode_minus_L, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported + _LT_TAGVAR(inherit_rpath, $1)=no + _LT_TAGVAR(link_all_deplibs, $1)=unknown + _LT_TAGVAR(module_cmds, $1)= + _LT_TAGVAR(module_expsym_cmds, $1)= + _LT_TAGVAR(old_archive_from_new_cmds, $1)= + _LT_TAGVAR(old_archive_from_expsyms_cmds, $1)= + _LT_TAGVAR(thread_safe_flag_spec, $1)= + _LT_TAGVAR(whole_archive_flag_spec, $1)= + # include_expsyms should be a list of space-separated symbols to be *always* + # included in the symbol list + _LT_TAGVAR(include_expsyms, $1)= + # exclude_expsyms can be an extended regexp of symbols to exclude + # it will be wrapped by ` (' and `)$', so one must not match beginning or + # end of line. Example: `a|bc|.*d.*' will exclude the symbols `a' and `bc', + # as well as any symbol that contains `d'. + _LT_TAGVAR(exclude_expsyms, $1)=['_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*'] + # Although _GLOBAL_OFFSET_TABLE_ is a valid symbol C name, most a.out + # platforms (ab)use it in PIC code, but their linkers get confused if + # the symbol is explicitly referenced. Since portable code cannot + # rely on this symbol name, it's probably fine to never include it in + # preloaded symbol tables. + # Exclude shared library initialization/finalization symbols. +dnl Note also adjust exclude_expsyms for C++ above. + extract_expsyms_cmds= + + case $host_os in + cygwin* | mingw* | pw32* | cegcc*) + # FIXME: the MSVC++ port hasn't been tested in a loooong time + # When not using gcc, we currently assume that we are using + # Microsoft Visual C++. + if test "$GCC" != yes; then + with_gnu_ld=no + fi + ;; + interix*) + # we just hope/assume this is gcc and not c89 (= MSVC++) + with_gnu_ld=yes + ;; + openbsd*) + with_gnu_ld=no + ;; + linux* | k*bsd*-gnu | gnu*) + _LT_TAGVAR(link_all_deplibs, $1)=no + ;; + esac + + _LT_TAGVAR(ld_shlibs, $1)=yes + + # On some targets, GNU ld is compatible enough with the native linker + # that we're better off using the native interface for both. + lt_use_gnu_ld_interface=no + if test "$with_gnu_ld" = yes; then + case $host_os in + aix*) + # The AIX port of GNU ld has always aspired to compatibility + # with the native linker. However, as the warning in the GNU ld + # block says, versions before 2.19.5* couldn't really create working + # shared libraries, regardless of the interface used. + case `$LD -v 2>&1` in + *\ \(GNU\ Binutils\)\ 2.19.5*) ;; + *\ \(GNU\ Binutils\)\ 2.[[2-9]]*) ;; + *\ \(GNU\ Binutils\)\ [[3-9]]*) ;; + *) + lt_use_gnu_ld_interface=yes + ;; + esac + ;; + *) + lt_use_gnu_ld_interface=yes + ;; + esac + fi + + if test "$lt_use_gnu_ld_interface" = yes; then + # If archive_cmds runs LD, not CC, wlarc should be empty + wlarc='${wl}' + + # Set some defaults for GNU ld with shared library support. These + # are reset later if shared libraries are not supported. Putting them + # here allows them to be overridden if necessary. + runpath_var=LD_RUN_PATH + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic' + # ancient GNU ld didn't support --whole-archive et. al. + if $LD --help 2>&1 | $GREP 'no-whole-archive' > /dev/null; then + _LT_TAGVAR(whole_archive_flag_spec, $1)="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive' + else + _LT_TAGVAR(whole_archive_flag_spec, $1)= + fi + supports_anon_versioning=no + case `$LD -v 2>&1` in + *GNU\ gold*) supports_anon_versioning=yes ;; + *\ [[01]].* | *\ 2.[[0-9]].* | *\ 2.10.*) ;; # catch versions < 2.11 + *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ... + *\ 2.11.92.0.12\ *) supports_anon_versioning=yes ;; # Mandrake 8.2 ... + *\ 2.11.*) ;; # other 2.11 versions + *) supports_anon_versioning=yes ;; + esac + + # See if GNU ld supports shared libraries. + case $host_os in + aix[[3-9]]*) + # On AIX/PPC, the GNU linker is very broken + if test "$host_cpu" != ia64; then + _LT_TAGVAR(ld_shlibs, $1)=no + cat <<_LT_EOF 1>&2 + +*** Warning: the GNU linker, at least up to release 2.19, is reported +*** to be unable to reliably create shared libraries on AIX. +*** Therefore, libtool is disabling shared libraries support. If you +*** really care for shared libraries, you may want to install binutils +*** 2.20 or above, or modify your PATH so that a non-GNU linker is found. +*** You will then need to restart the configuration process. + +_LT_EOF + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='' + ;; + m68k) + _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_minus_L, $1)=yes + ;; + esac + ;; + + beos*) + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + # Joseph Beckenbach says some releases of gcc + # support --undefined. This deserves some investigation. FIXME + _LT_TAGVAR(archive_cmds, $1)='$CC -nostart $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + + cygwin* | mingw* | pw32* | cegcc*) + # _LT_TAGVAR(hardcode_libdir_flag_spec, $1) is actually meaningless, + # as there is no search path for DLLs. + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-all-symbols' + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + _LT_TAGVAR(always_export_symbols, $1)=no + _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1 DATA/;s/^.*[[ ]]__nm__\([[^ ]]*\)[[ ]][[^ ]]*/\1 DATA/;/^I[[ ]]/d;/^[[AITW]][[ ]]/s/.* //'\'' | sort | uniq > $export_symbols' + _LT_TAGVAR(exclude_expsyms, $1)=['[_]+GLOBAL_OFFSET_TABLE_|[_]+GLOBAL__[FID]_.*|[_]+head_[A-Za-z0-9_]+_dll|[A-Za-z0-9_]+_dll_iname'] + + if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + # If the export-symbols file already is a .def file (1st line + # is EXPORTS), use it as is; otherwise, prepend... + _LT_TAGVAR(archive_expsym_cmds, $1)='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + cp $export_symbols $output_objdir/$soname.def; + else + echo EXPORTS > $output_objdir/$soname.def; + cat $export_symbols >> $output_objdir/$soname.def; + fi~ + $CC -shared $output_objdir/$soname.def $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + + haiku*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(link_all_deplibs, $1)=yes + ;; + + interix[[3-9]]*) + _LT_TAGVAR(hardcode_direct, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc. + # Instead, shared libraries are loaded at an image base (0x10000000 by + # default) and relocated if they conflict, which is a slow very memory + # consuming and fragmenting process. To avoid this, we pick a random, + # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link + # time. Moving up from 0x10000000 also allows more sbrk(2) space. + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s,^,_," $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--retain-symbols-file,$output_objdir/$soname.expsym ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + ;; + + gnu* | linux* | tpf* | k*bsd*-gnu | kopensolaris*-gnu) + tmp_diet=no + if test "$host_os" = linux-dietlibc; then + case $cc_basename in + diet\ *) tmp_diet=yes;; # linux-dietlibc with static linking (!diet-dyn) + esac + fi + if $LD --help 2>&1 | $EGREP ': supported targets:.* elf' > /dev/null \ + && test "$tmp_diet" = no + then + tmp_addflag=' $pic_flag' + tmp_sharedflag='-shared' + case $cc_basename,$host_cpu in + pgcc*) # Portland Group C compiler + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + tmp_addflag=' $pic_flag' + ;; + pgf77* | pgf90* | pgf95* | pgfortran*) + # Portland Group f77 and f90 compilers + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + tmp_addflag=' $pic_flag -Mnomain' ;; + ecc*,ia64* | icc*,ia64*) # Intel C compiler on ia64 + tmp_addflag=' -i_dynamic' ;; + efc*,ia64* | ifort*,ia64*) # Intel Fortran compiler on ia64 + tmp_addflag=' -i_dynamic -nofor_main' ;; + ifc* | ifort*) # Intel Fortran compiler + tmp_addflag=' -nofor_main' ;; + lf95*) # Lahey Fortran 8.1 + _LT_TAGVAR(whole_archive_flag_spec, $1)= + tmp_sharedflag='--shared' ;; + xl[[cC]]* | bgxl[[cC]]* | mpixl[[cC]]*) # IBM XL C 8.0 on PPC (deal with xlf below) + tmp_sharedflag='-qmkshrobj' + tmp_addflag= ;; + nvcc*) # Cuda Compiler Driver 2.2 + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + _LT_TAGVAR(compiler_needs_object, $1)=yes + ;; + esac + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) # Sun C 5.9 + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + _LT_TAGVAR(compiler_needs_object, $1)=yes + tmp_sharedflag='-G' ;; + *Sun\ F*) # Sun Fortran 8.3 + tmp_sharedflag='-G' ;; + esac + _LT_TAGVAR(archive_cmds, $1)='$CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + + if test "x$supports_anon_versioning" = xyes; then + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ + cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + echo "local: *; };" >> $output_objdir/$libname.ver~ + $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-version-script ${wl}$output_objdir/$libname.ver -o $lib' + fi + + case $cc_basename in + xlf* | bgf* | bgxlf* | mpixlf*) + # IBM XL Fortran 10.1 on PPC cannot create shared libs itself + _LT_TAGVAR(whole_archive_flag_spec, $1)='--whole-archive$convenience --no-whole-archive' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(archive_cmds, $1)='$LD -shared $libobjs $deplibs $linker_flags -soname $soname -o $lib' + if test "x$supports_anon_versioning" = xyes; then + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ + cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + echo "local: *; };" >> $output_objdir/$libname.ver~ + $LD -shared $libobjs $deplibs $linker_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib' + fi + ;; + esac + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + + netbsd* | netbsdelf*-gnu) + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable $libobjs $deplibs $linker_flags -o $lib' + wlarc= + else + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + fi + ;; + + solaris*) + if $LD -v 2>&1 | $GREP 'BFD 2\.8' > /dev/null; then + _LT_TAGVAR(ld_shlibs, $1)=no + cat <<_LT_EOF 1>&2 + +*** Warning: The releases 2.8.* of the GNU linker cannot reliably +*** create shared libraries on Solaris systems. Therefore, libtool +*** is disabling shared libraries support. We urge you to upgrade GNU +*** binutils to release 2.9.1 or newer. Another option is to modify +*** your PATH or compiler configuration so that the native linker is +*** used, and then restart. + +_LT_EOF + elif $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + + sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX*) + case `$LD -v 2>&1` in + *\ [[01]].* | *\ 2.[[0-9]].* | *\ 2.1[[0-5]].*) + _LT_TAGVAR(ld_shlibs, $1)=no + cat <<_LT_EOF 1>&2 + +*** Warning: Releases of the GNU linker prior to 2.16.91.0.3 can not +*** reliably create shared libraries on SCO systems. Therefore, libtool +*** is disabling shared libraries support. We urge you to upgrade GNU +*** binutils to release 2.16.91.0.3 or newer. Another option is to modify +*** your PATH or compiler configuration so that the native linker is +*** used, and then restart. + +_LT_EOF + ;; + *) + # For security reasons, it is highly recommended that you always + # use absolute paths for naming shared libraries, and exclude the + # DT_RUNPATH tag from executables and libraries. But doing so + # requires that you compile everything twice, which is a pain. + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + esac + ;; + + sunos4*) + _LT_TAGVAR(archive_cmds, $1)='$LD -assert pure-text -Bshareable -o $lib $libobjs $deplibs $linker_flags' + wlarc= + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + *) + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + esac + + if test "$_LT_TAGVAR(ld_shlibs, $1)" = no; then + runpath_var= + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)= + _LT_TAGVAR(export_dynamic_flag_spec, $1)= + _LT_TAGVAR(whole_archive_flag_spec, $1)= + fi + else + # PORTME fill in a description of your system's linker (not GNU ld) + case $host_os in + aix3*) + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + _LT_TAGVAR(always_export_symbols, $1)=yes + _LT_TAGVAR(archive_expsym_cmds, $1)='$LD -o $output_objdir/$soname $libobjs $deplibs $linker_flags -bE:$export_symbols -T512 -H512 -bM:SRE~$AR $AR_FLAGS $lib $output_objdir/$soname' + # Note: this linker hardcodes the directories in LIBPATH if there + # are no directories specified by -L. + _LT_TAGVAR(hardcode_minus_L, $1)=yes + if test "$GCC" = yes && test -z "$lt_prog_compiler_static"; then + # Neither direct hardcoding nor static linking is supported with a + # broken collect2. + _LT_TAGVAR(hardcode_direct, $1)=unsupported + fi + ;; + + aix[[4-9]]*) + if test "$host_cpu" = ia64; then + # On IA64, the linker does run time linking by default, so we don't + # have to do anything special. + aix_use_runtimelinking=no + exp_sym_flag='-Bexport' + no_entry_flag="" + else + # If we're using GNU nm, then we don't want the "-C" option. + # -C means demangle to AIX nm, but means don't demangle with GNU nm + # Also, AIX nm treats weak defined symbols like other global + # defined symbols, whereas GNU nm marks them as "W". + if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then + _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + else + _LT_TAGVAR(export_symbols_cmds, $1)='$NM -BCpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols' + fi + aix_use_runtimelinking=no + + # Test if we are trying to use run time linking or normal + # AIX style linking. If -brtl is somewhere in LDFLAGS, we + # need to do runtime linking. + case $host_os in aix4.[[23]]|aix4.[[23]].*|aix[[5-9]]*) + for ld_flag in $LDFLAGS; do + if (test $ld_flag = "-brtl" || test $ld_flag = "-Wl,-brtl"); then + aix_use_runtimelinking=yes + break + fi + done + ;; + esac + + exp_sym_flag='-bexport' + no_entry_flag='-bnoentry' + fi + + # When large executables or shared objects are built, AIX ld can + # have problems creating the table of contents. If linking a library + # or program results in "error TOC overflow" add -mminimal-toc to + # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not + # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS. + + _LT_TAGVAR(archive_cmds, $1)='' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_direct_absolute, $1)=yes + _LT_TAGVAR(hardcode_libdir_separator, $1)=':' + _LT_TAGVAR(link_all_deplibs, $1)=yes + _LT_TAGVAR(file_list_spec, $1)='${wl}-f,' + + if test "$GCC" = yes; then + case $host_os in aix4.[[012]]|aix4.[[012]].*) + # We only want to do this on AIX 4.2 and lower, the check + # below for broken collect2 doesn't work under 4.3+ + collect2name=`${CC} -print-prog-name=collect2` + if test -f "$collect2name" && + strings "$collect2name" | $GREP resolve_lib_name >/dev/null + then + # We have reworked collect2 + : + else + # We have old collect2 + _LT_TAGVAR(hardcode_direct, $1)=unsupported + # It fails to find uninstalled libraries when the uninstalled + # path is not listed in the libpath. Setting hardcode_minus_L + # to unsupported forces relinking + _LT_TAGVAR(hardcode_minus_L, $1)=yes + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)= + fi + ;; + esac + shared_flag='-shared' + if test "$aix_use_runtimelinking" = yes; then + shared_flag="$shared_flag "'${wl}-G' + fi + _LT_TAGVAR(link_all_deplibs, $1)=no + else + # not using gcc + if test "$host_cpu" = ia64; then + # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release + # chokes on -Wl,-G. The following line is correct: + shared_flag='-G' + else + if test "$aix_use_runtimelinking" = yes; then + shared_flag='${wl}-G' + else + shared_flag='${wl}-bM:SRE' + fi + fi + fi + + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-bexpall' + # It seems that -bexpall does not export symbols beginning with + # underscore (_), so it is better to generate a list of symbols to export. + _LT_TAGVAR(always_export_symbols, $1)=yes + if test "$aix_use_runtimelinking" = yes; then + # Warning - without using the other runtime loading flags (-brtl), + # -berok will link without error, but may produce a broken library. + _LT_TAGVAR(allow_undefined_flag, $1)='-berok' + # Determine the default libpath from the value encoded in an + # empty executable. + _LT_SYS_MODULE_PATH_AIX([$1]) + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath" + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags `if test "x${allow_undefined_flag}" != "x"; then func_echo_all "${wl}${allow_undefined_flag}"; else :; fi` '"\${wl}$exp_sym_flag:\$export_symbols $shared_flag" + else + if test "$host_cpu" = ia64; then + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R $libdir:/usr/lib:/lib' + _LT_TAGVAR(allow_undefined_flag, $1)="-z nodefs" + _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags ${wl}${allow_undefined_flag} '"\${wl}$exp_sym_flag:\$export_symbols" + else + # Determine the default libpath from the value encoded in an + # empty executable. + _LT_SYS_MODULE_PATH_AIX([$1]) + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath" + # Warning - without using the other run time loading flags, + # -berok will link without error, but may produce a broken library. + _LT_TAGVAR(no_undefined_flag, $1)=' ${wl}-bernotok' + _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-berok' + if test "$with_gnu_ld" = yes; then + # We only use this code for GNU lds that support --whole-archive. + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive$convenience ${wl}--no-whole-archive' + else + # Exported symbols can be pulled into shared objects from archives + _LT_TAGVAR(whole_archive_flag_spec, $1)='$convenience' + fi + _LT_TAGVAR(archive_cmds_need_lc, $1)=yes + # This is similar to how AIX traditionally builds its shared libraries. + _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs ${wl}-bnoentry $compiler_flags ${wl}-bE:$export_symbols${allow_undefined_flag}~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$soname' + fi + fi + ;; + + amigaos*) + case $host_cpu in + powerpc) + # see comment about AmigaOS4 .so support + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='' + ;; + m68k) + _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_minus_L, $1)=yes + ;; + esac + ;; + + bsdi[[45]]*) + _LT_TAGVAR(export_dynamic_flag_spec, $1)=-rdynamic + ;; + + cygwin* | mingw* | pw32* | cegcc*) + # When not using gcc, we currently assume that we are using + # Microsoft Visual C++. + # hardcode_libdir_flag_spec is actually meaningless, as there is + # no search path for DLLs. + case $cc_basename in + cl*) + # Native MSVC + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + _LT_TAGVAR(always_export_symbols, $1)=yes + _LT_TAGVAR(file_list_spec, $1)='@' + # Tell ltmain to make .lib files, not .a files. + libext=lib + # Tell ltmain to make .dll files, not .so files. + shrext_cmds=".dll" + # FIXME: Setting linknames here is a bad hack. + _LT_TAGVAR(archive_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $compiler_flags $deplibs -Wl,-dll~linknames=' + _LT_TAGVAR(archive_expsym_cmds, $1)='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + sed -n -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' -e '1\\\!p' < $export_symbols > $output_objdir/$soname.exp; + else + sed -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' < $export_symbols > $output_objdir/$soname.exp; + fi~ + $CC -o $tool_output_objdir$soname $libobjs $compiler_flags $deplibs "@$tool_output_objdir$soname.exp" -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~ + linknames=' + # The linker will not automatically build a static lib if we build a DLL. + # _LT_TAGVAR(old_archive_from_new_cmds, $1)='true' + _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + _LT_TAGVAR(exclude_expsyms, $1)='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' + _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1,DATA/'\'' | $SED -e '\''/^[[AITW]][[ ]]/s/.*[[ ]]//'\'' | sort | uniq > $export_symbols' + # Don't use ranlib + _LT_TAGVAR(old_postinstall_cmds, $1)='chmod 644 $oldlib' + _LT_TAGVAR(postlink_cmds, $1)='lt_outputfile="@OUTPUT@"~ + lt_tool_outputfile="@TOOL_OUTPUT@"~ + case $lt_outputfile in + *.exe|*.EXE) ;; + *) + lt_outputfile="$lt_outputfile.exe" + lt_tool_outputfile="$lt_tool_outputfile.exe" + ;; + esac~ + if test "$MANIFEST_TOOL" != ":" && test -f "$lt_outputfile.manifest"; then + $MANIFEST_TOOL -manifest "$lt_tool_outputfile.manifest" -outputresource:"$lt_tool_outputfile" || exit 1; + $RM "$lt_outputfile.manifest"; + fi' + ;; + *) + # Assume MSVC wrapper + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + # Tell ltmain to make .lib files, not .a files. + libext=lib + # Tell ltmain to make .dll files, not .so files. + shrext_cmds=".dll" + # FIXME: Setting linknames here is a bad hack. + _LT_TAGVAR(archive_cmds, $1)='$CC -o $lib $libobjs $compiler_flags `func_echo_all "$deplibs" | $SED '\''s/ -lc$//'\''` -link -dll~linknames=' + # The linker will automatically build a .lib file if we build a DLL. + _LT_TAGVAR(old_archive_from_new_cmds, $1)='true' + # FIXME: Should let the user specify the lib program. + _LT_TAGVAR(old_archive_cmds, $1)='lib -OUT:$oldlib$oldobjs$old_deplibs' + _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + ;; + esac + ;; + + darwin* | rhapsody*) + _LT_DARWIN_LINKER_FEATURES($1) + ;; + + dgux*) + _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + # FreeBSD 2.2.[012] allows us to include c++rt0.o to get C++ constructor + # support. Future versions do this automatically, but an explicit c++rt0.o + # does not break anything, and helps significantly (at the cost of a little + # extra space). + freebsd2.2*) + _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags /usr/lib/c++rt0.o' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + # Unfortunately, older versions of FreeBSD 2 do not have this feature. + freebsd2.*) + _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_minus_L, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + # FreeBSD 3 and greater uses gcc -shared to do shared libraries. + freebsd* | dragonfly*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + hpux9*) + if test "$GCC" = yes; then + _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -shared $pic_flag ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $libobjs $deplibs $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + else + _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$LD -b +b $install_libdir -o $output_objdir/$soname $libobjs $deplibs $linker_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + fi + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + _LT_TAGVAR(hardcode_direct, $1)=yes + + # hardcode_minus_L: Not really in the search PATH, + # but as the default location of the library. + _LT_TAGVAR(hardcode_minus_L, $1)=yes + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + ;; + + hpux10*) + if test "$GCC" = yes && test "$with_gnu_ld" = no; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags' + else + _LT_TAGVAR(archive_cmds, $1)='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags' + fi + if test "$with_gnu_ld" = no; then + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_direct_absolute, $1)=yes + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + # hardcode_minus_L: Not really in the search PATH, + # but as the default location of the library. + _LT_TAGVAR(hardcode_minus_L, $1)=yes + fi + ;; + + hpux11*) + if test "$GCC" = yes && test "$with_gnu_ld" = no; then + case $host_cpu in + hppa*64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + ia64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' + ;; + *) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags' + ;; + esac + else + case $host_cpu in + hppa*64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + ia64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' + ;; + *) + m4_if($1, [], [ + # Older versions of the 11.00 compiler do not understand -b yet + # (HP92453-01 A.11.01.20 doesn't, HP92453-01 B.11.X.35175-35176.GP does) + _LT_LINKER_OPTION([if $CC understands -b], + _LT_TAGVAR(lt_cv_prog_compiler__b, $1), [-b], + [_LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags'], + [_LT_TAGVAR(archive_cmds, $1)='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags'])], + [_LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags']) + ;; + esac + fi + if test "$with_gnu_ld" = no; then + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + + case $host_cpu in + hppa*64*|ia64*) + _LT_TAGVAR(hardcode_direct, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + *) + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_direct_absolute, $1)=yes + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + + # hardcode_minus_L: Not really in the search PATH, + # but as the default location of the library. + _LT_TAGVAR(hardcode_minus_L, $1)=yes + ;; + esac + fi + ;; + + irix5* | irix6* | nonstopux*) + if test "$GCC" = yes; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + # Try to use the -exported_symbol ld option, if it does not + # work, assume that -exports_file does not work either and + # implicitly export all symbols. + # This should be the same for all languages, so no per-tag cache variable. + AC_CACHE_CHECK([whether the $host_os linker accepts -exported_symbol], + [lt_cv_irix_exported_symbol], + [save_LDFLAGS="$LDFLAGS" + LDFLAGS="$LDFLAGS -shared ${wl}-exported_symbol ${wl}foo ${wl}-update_registry ${wl}/dev/null" + AC_LINK_IFELSE( + [AC_LANG_SOURCE( + [AC_LANG_CASE([C], [[int foo (void) { return 0; }]], + [C++], [[int foo (void) { return 0; }]], + [Fortran 77], [[ + subroutine foo + end]], + [Fortran], [[ + subroutine foo + end]])])], + [lt_cv_irix_exported_symbol=yes], + [lt_cv_irix_exported_symbol=no]) + LDFLAGS="$save_LDFLAGS"]) + if test "$lt_cv_irix_exported_symbol" = yes; then + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations ${wl}-exports_file ${wl}$export_symbols -o $lib' + fi + else + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -exports_file $export_symbols -o $lib' + fi + _LT_TAGVAR(archive_cmds_need_lc, $1)='no' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + _LT_TAGVAR(inherit_rpath, $1)=yes + _LT_TAGVAR(link_all_deplibs, $1)=yes + ;; + + netbsd* | netbsdelf*-gnu) + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' # a.out + else + _LT_TAGVAR(archive_cmds, $1)='$LD -shared -o $lib $libobjs $deplibs $linker_flags' # ELF + fi + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + newsos6) + _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + *nto* | *qnx*) + ;; + + openbsd*) + if test -f /usr/libexec/ld.so; then + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + _LT_TAGVAR(hardcode_direct_absolute, $1)=yes + if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags ${wl}-retain-symbols-file,$export_symbols' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + else + case $host_os in + openbsd[[01]].* | openbsd2.[[0-7]] | openbsd2.[[0-7]].*) + _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + ;; + *) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + ;; + esac + fi + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + + os2*) + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_minus_L, $1)=yes + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + _LT_TAGVAR(archive_cmds, $1)='$ECHO "LIBRARY $libname INITINSTANCE" > $output_objdir/$libname.def~$ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~echo DATA >> $output_objdir/$libname.def~echo " SINGLE NONSHARED" >> $output_objdir/$libname.def~echo EXPORTS >> $output_objdir/$libname.def~emxexp $libobjs >> $output_objdir/$libname.def~$CC -Zdll -Zcrtdll -o $lib $libobjs $deplibs $compiler_flags $output_objdir/$libname.def' + _LT_TAGVAR(old_archive_from_new_cmds, $1)='emximp -o $output_objdir/$libname.a $output_objdir/$libname.def' + ;; + + osf3*) + if test "$GCC" = yes; then + _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + else + _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + fi + _LT_TAGVAR(archive_cmds_need_lc, $1)='no' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + ;; + + osf4* | osf5*) # as osf3* with the addition of -msym flag + if test "$GCC" = yes; then + _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $pic_flag $libobjs $deplibs $compiler_flags ${wl}-msym ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + else + _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -msym -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done; printf "%s\\n" "-hidden">> $lib.exp~ + $CC -shared${allow_undefined_flag} ${wl}-input ${wl}$lib.exp $compiler_flags $libobjs $deplibs -soname $soname `test -n "$verstring" && $ECHO "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib~$RM $lib.exp' + + # Both c and cxx compiler support -rpath directly + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir' + fi + _LT_TAGVAR(archive_cmds_need_lc, $1)='no' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + ;; + + solaris*) + _LT_TAGVAR(no_undefined_flag, $1)=' -z defs' + if test "$GCC" = yes; then + wlarc='${wl}' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag ${wl}-z ${wl}text ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -shared $pic_flag ${wl}-z ${wl}text ${wl}-M ${wl}$lib.exp ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' + else + case `$CC -V 2>&1` in + *"Compilers 5.0"*) + wlarc='' + _LT_TAGVAR(archive_cmds, $1)='$LD -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $LD -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $linker_flags~$RM $lib.exp' + ;; + *) + wlarc='${wl}' + _LT_TAGVAR(archive_cmds, $1)='$CC -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' + ;; + esac + fi + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + case $host_os in + solaris2.[[0-5]] | solaris2.[[0-5]].*) ;; + *) + # The compiler driver will combine and reorder linker options, + # but understands `-z linker_flag'. GCC discards it without `$wl', + # but is careful enough not to reorder. + # Supported since Solaris 2.6 (maybe 2.5.1?) + if test "$GCC" = yes; then + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}-z ${wl}allextract$convenience ${wl}-z ${wl}defaultextract' + else + _LT_TAGVAR(whole_archive_flag_spec, $1)='-z allextract$convenience -z defaultextract' + fi + ;; + esac + _LT_TAGVAR(link_all_deplibs, $1)=yes + ;; + + sunos4*) + if test "x$host_vendor" = xsequent; then + # Use $CC to link under sequent, because it throws in some extra .o + # files that make .init and .fini sections work. + _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h $soname -o $lib $libobjs $deplibs $compiler_flags' + else + _LT_TAGVAR(archive_cmds, $1)='$LD -assert pure-text -Bstatic -o $lib $libobjs $deplibs $linker_flags' + fi + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_minus_L, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + sysv4) + case $host_vendor in + sni) + _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_direct, $1)=yes # is this really true??? + ;; + siemens) + ## LD is ld it makes a PLAMLIB + ## CC just makes a GrossModule. + _LT_TAGVAR(archive_cmds, $1)='$LD -G -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(reload_cmds, $1)='$CC -r -o $output$reload_objs' + _LT_TAGVAR(hardcode_direct, $1)=no + ;; + motorola) + _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_direct, $1)=no #Motorola manual says yes, but my tests say they lie + ;; + esac + runpath_var='LD_RUN_PATH' + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + sysv4.3*) + _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + _LT_TAGVAR(export_dynamic_flag_spec, $1)='-Bexport' + ;; + + sysv4*MP*) + if test -d /usr/nec; then + _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + runpath_var=LD_RUN_PATH + hardcode_runpath_var=yes + _LT_TAGVAR(ld_shlibs, $1)=yes + fi + ;; + + sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[[01]].[[10]]* | unixware7* | sco3.2v5.0.[[024]]*) + _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text' + _LT_TAGVAR(archive_cmds_need_lc, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + runpath_var='LD_RUN_PATH' + + if test "$GCC" = yes; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + else + _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + fi + ;; + + sysv5* | sco3.2v5* | sco5v6*) + # Note: We can NOT use -z defs as we might desire, because we do not + # link with -lc, and that would cause any symbols used from libc to + # always be unresolved, which means just about no library would + # ever link correctly. If we're not using GNU ld we use -z text + # though, which does catch some bad symbols but isn't as heavy-handed + # as -z defs. + _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text' + _LT_TAGVAR(allow_undefined_flag, $1)='${wl}-z,nodefs' + _LT_TAGVAR(archive_cmds_need_lc, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R,$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=':' + _LT_TAGVAR(link_all_deplibs, $1)=yes + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-Bexport' + runpath_var='LD_RUN_PATH' + + if test "$GCC" = yes; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + else + _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + fi + ;; + + uts4*) + _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + + *) + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + esac + + if test x$host_vendor = xsni; then + case $host in + sysv4 | sysv4.2uw2* | sysv4.3* | sysv5*) + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-Blargedynsym' + ;; + esac + fi + fi +]) +AC_MSG_RESULT([$_LT_TAGVAR(ld_shlibs, $1)]) +test "$_LT_TAGVAR(ld_shlibs, $1)" = no && can_build_shared=no + +_LT_TAGVAR(with_gnu_ld, $1)=$with_gnu_ld + +_LT_DECL([], [libext], [0], [Old archive suffix (normally "a")])dnl +_LT_DECL([], [shrext_cmds], [1], [Shared library suffix (normally ".so")])dnl +_LT_DECL([], [extract_expsyms_cmds], [2], + [The commands to extract the exported symbol list from a shared archive]) + +# +# Do we need to explicitly link libc? +# +case "x$_LT_TAGVAR(archive_cmds_need_lc, $1)" in +x|xyes) + # Assume -lc should be added + _LT_TAGVAR(archive_cmds_need_lc, $1)=yes + + if test "$enable_shared" = yes && test "$GCC" = yes; then + case $_LT_TAGVAR(archive_cmds, $1) in + *'~'*) + # FIXME: we may have to deal with multi-command sequences. + ;; + '$CC '*) + # Test whether the compiler implicitly links with -lc since on some + # systems, -lgcc has to come before -lc. If gcc already passes -lc + # to ld, don't add -lc before -lgcc. + AC_CACHE_CHECK([whether -lc should be explicitly linked in], + [lt_cv_]_LT_TAGVAR(archive_cmds_need_lc, $1), + [$RM conftest* + echo "$lt_simple_compile_test_code" > conftest.$ac_ext + + if AC_TRY_EVAL(ac_compile) 2>conftest.err; then + soname=conftest + lib=conftest + libobjs=conftest.$ac_objext + deplibs= + wl=$_LT_TAGVAR(lt_prog_compiler_wl, $1) + pic_flag=$_LT_TAGVAR(lt_prog_compiler_pic, $1) + compiler_flags=-v + linker_flags=-v + verstring= + output_objdir=. + libname=conftest + lt_save_allow_undefined_flag=$_LT_TAGVAR(allow_undefined_flag, $1) + _LT_TAGVAR(allow_undefined_flag, $1)= + if AC_TRY_EVAL(_LT_TAGVAR(archive_cmds, $1) 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1) + then + lt_cv_[]_LT_TAGVAR(archive_cmds_need_lc, $1)=no + else + lt_cv_[]_LT_TAGVAR(archive_cmds_need_lc, $1)=yes + fi + _LT_TAGVAR(allow_undefined_flag, $1)=$lt_save_allow_undefined_flag + else + cat conftest.err 1>&5 + fi + $RM conftest* + ]) + _LT_TAGVAR(archive_cmds_need_lc, $1)=$lt_cv_[]_LT_TAGVAR(archive_cmds_need_lc, $1) + ;; + esac + fi + ;; +esac + +_LT_TAGDECL([build_libtool_need_lc], [archive_cmds_need_lc], [0], + [Whether or not to add -lc for building shared libraries]) +_LT_TAGDECL([allow_libtool_libs_with_static_runtimes], + [enable_shared_with_static_runtimes], [0], + [Whether or not to disallow shared libs when runtime libs are static]) +_LT_TAGDECL([], [export_dynamic_flag_spec], [1], + [Compiler flag to allow reflexive dlopens]) +_LT_TAGDECL([], [whole_archive_flag_spec], [1], + [Compiler flag to generate shared objects directly from archives]) +_LT_TAGDECL([], [compiler_needs_object], [1], + [Whether the compiler copes with passing no objects directly]) +_LT_TAGDECL([], [old_archive_from_new_cmds], [2], + [Create an old-style archive from a shared archive]) +_LT_TAGDECL([], [old_archive_from_expsyms_cmds], [2], + [Create a temporary old-style archive to link instead of a shared archive]) +_LT_TAGDECL([], [archive_cmds], [2], [Commands used to build a shared archive]) +_LT_TAGDECL([], [archive_expsym_cmds], [2]) +_LT_TAGDECL([], [module_cmds], [2], + [Commands used to build a loadable module if different from building + a shared archive.]) +_LT_TAGDECL([], [module_expsym_cmds], [2]) +_LT_TAGDECL([], [with_gnu_ld], [1], + [Whether we are building with GNU ld or not]) +_LT_TAGDECL([], [allow_undefined_flag], [1], + [Flag that allows shared libraries with undefined symbols to be built]) +_LT_TAGDECL([], [no_undefined_flag], [1], + [Flag that enforces no undefined symbols]) +_LT_TAGDECL([], [hardcode_libdir_flag_spec], [1], + [Flag to hardcode $libdir into a binary during linking. + This must work even if $libdir does not exist]) +_LT_TAGDECL([], [hardcode_libdir_separator], [1], + [Whether we need a single "-rpath" flag with a separated argument]) +_LT_TAGDECL([], [hardcode_direct], [0], + [Set to "yes" if using DIR/libNAME${shared_ext} during linking hardcodes + DIR into the resulting binary]) +_LT_TAGDECL([], [hardcode_direct_absolute], [0], + [Set to "yes" if using DIR/libNAME${shared_ext} during linking hardcodes + DIR into the resulting binary and the resulting library dependency is + "absolute", i.e impossible to change by setting ${shlibpath_var} if the + library is relocated]) +_LT_TAGDECL([], [hardcode_minus_L], [0], + [Set to "yes" if using the -LDIR flag during linking hardcodes DIR + into the resulting binary]) +_LT_TAGDECL([], [hardcode_shlibpath_var], [0], + [Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR + into the resulting binary]) +_LT_TAGDECL([], [hardcode_automatic], [0], + [Set to "yes" if building a shared library automatically hardcodes DIR + into the library and all subsequent libraries and executables linked + against it]) +_LT_TAGDECL([], [inherit_rpath], [0], + [Set to yes if linker adds runtime paths of dependent libraries + to runtime path list]) +_LT_TAGDECL([], [link_all_deplibs], [0], + [Whether libtool must link a program against all its dependency libraries]) +_LT_TAGDECL([], [always_export_symbols], [0], + [Set to "yes" if exported symbols are required]) +_LT_TAGDECL([], [export_symbols_cmds], [2], + [The commands to list exported symbols]) +_LT_TAGDECL([], [exclude_expsyms], [1], + [Symbols that should not be listed in the preloaded symbols]) +_LT_TAGDECL([], [include_expsyms], [1], + [Symbols that must always be exported]) +_LT_TAGDECL([], [prelink_cmds], [2], + [Commands necessary for linking programs (against libraries) with templates]) +_LT_TAGDECL([], [postlink_cmds], [2], + [Commands necessary for finishing linking programs]) +_LT_TAGDECL([], [file_list_spec], [1], + [Specify filename containing input files]) +dnl FIXME: Not yet implemented +dnl _LT_TAGDECL([], [thread_safe_flag_spec], [1], +dnl [Compiler flag to generate thread safe objects]) +])# _LT_LINKER_SHLIBS + + +# _LT_LANG_C_CONFIG([TAG]) +# ------------------------ +# Ensure that the configuration variables for a C compiler are suitably +# defined. These variables are subsequently used by _LT_CONFIG to write +# the compiler configuration to `libtool'. +m4_defun([_LT_LANG_C_CONFIG], +[m4_require([_LT_DECL_EGREP])dnl +lt_save_CC="$CC" +AC_LANG_PUSH(C) + +# Source file extension for C test sources. +ac_ext=c + +# Object file extension for compiled C test sources. +objext=o +_LT_TAGVAR(objext, $1)=$objext + +# Code to be used in simple compile tests +lt_simple_compile_test_code="int some_variable = 0;" + +# Code to be used in simple link tests +lt_simple_link_test_code='int main(){return(0);}' + +_LT_TAG_COMPILER +# Save the default compiler, since it gets overwritten when the other +# tags are being tested, and _LT_TAGVAR(compiler, []) is a NOP. +compiler_DEFAULT=$CC + +# save warnings/boilerplate of simple test code +_LT_COMPILER_BOILERPLATE +_LT_LINKER_BOILERPLATE + +## CAVEAT EMPTOR: +## There is no encapsulation within the following macros, do not change +## the running order or otherwise move them around unless you know exactly +## what you are doing... +if test -n "$compiler"; then + _LT_COMPILER_NO_RTTI($1) + _LT_COMPILER_PIC($1) + _LT_COMPILER_C_O($1) + _LT_COMPILER_FILE_LOCKS($1) + _LT_LINKER_SHLIBS($1) + _LT_SYS_DYNAMIC_LINKER($1) + _LT_LINKER_HARDCODE_LIBPATH($1) + LT_SYS_DLOPEN_SELF + _LT_CMD_STRIPLIB + + # Report which library types will actually be built + AC_MSG_CHECKING([if libtool supports shared libraries]) + AC_MSG_RESULT([$can_build_shared]) + + AC_MSG_CHECKING([whether to build shared libraries]) + test "$can_build_shared" = "no" && enable_shared=no + + # On AIX, shared libraries and static libraries use the same namespace, and + # are all built from PIC. + case $host_os in + aix3*) + test "$enable_shared" = yes && enable_static=no + if test -n "$RANLIB"; then + archive_cmds="$archive_cmds~\$RANLIB \$lib" + postinstall_cmds='$RANLIB $lib' + fi + ;; + + aix[[4-9]]*) + if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then + test "$enable_shared" = yes && enable_static=no + fi + ;; + esac + AC_MSG_RESULT([$enable_shared]) + + AC_MSG_CHECKING([whether to build static libraries]) + # Make sure either enable_shared or enable_static is yes. + test "$enable_shared" = yes || enable_static=yes + AC_MSG_RESULT([$enable_static]) + + _LT_CONFIG($1) +fi +AC_LANG_POP +CC="$lt_save_CC" +])# _LT_LANG_C_CONFIG + + +# _LT_LANG_CXX_CONFIG([TAG]) +# -------------------------- +# Ensure that the configuration variables for a C++ compiler are suitably +# defined. These variables are subsequently used by _LT_CONFIG to write +# the compiler configuration to `libtool'. +m4_defun([_LT_LANG_CXX_CONFIG], +[m4_require([_LT_FILEUTILS_DEFAULTS])dnl +m4_require([_LT_DECL_EGREP])dnl +m4_require([_LT_PATH_MANIFEST_TOOL])dnl +if test -n "$CXX" && ( test "X$CXX" != "Xno" && + ( (test "X$CXX" = "Xg++" && `g++ -v >/dev/null 2>&1` ) || + (test "X$CXX" != "Xg++"))) ; then + AC_PROG_CXXCPP +else + _lt_caught_CXX_error=yes +fi + +AC_LANG_PUSH(C++) +_LT_TAGVAR(archive_cmds_need_lc, $1)=no +_LT_TAGVAR(allow_undefined_flag, $1)= +_LT_TAGVAR(always_export_symbols, $1)=no +_LT_TAGVAR(archive_expsym_cmds, $1)= +_LT_TAGVAR(compiler_needs_object, $1)=no +_LT_TAGVAR(export_dynamic_flag_spec, $1)= +_LT_TAGVAR(hardcode_direct, $1)=no +_LT_TAGVAR(hardcode_direct_absolute, $1)=no +_LT_TAGVAR(hardcode_libdir_flag_spec, $1)= +_LT_TAGVAR(hardcode_libdir_separator, $1)= +_LT_TAGVAR(hardcode_minus_L, $1)=no +_LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported +_LT_TAGVAR(hardcode_automatic, $1)=no +_LT_TAGVAR(inherit_rpath, $1)=no +_LT_TAGVAR(module_cmds, $1)= +_LT_TAGVAR(module_expsym_cmds, $1)= +_LT_TAGVAR(link_all_deplibs, $1)=unknown +_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds +_LT_TAGVAR(reload_flag, $1)=$reload_flag +_LT_TAGVAR(reload_cmds, $1)=$reload_cmds +_LT_TAGVAR(no_undefined_flag, $1)= +_LT_TAGVAR(whole_archive_flag_spec, $1)= +_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no + +# Source file extension for C++ test sources. +ac_ext=cpp + +# Object file extension for compiled C++ test sources. +objext=o +_LT_TAGVAR(objext, $1)=$objext + +# No sense in running all these tests if we already determined that +# the CXX compiler isn't working. Some variables (like enable_shared) +# are currently assumed to apply to all compilers on this platform, +# and will be corrupted by setting them based on a non-working compiler. +if test "$_lt_caught_CXX_error" != yes; then + # Code to be used in simple compile tests + lt_simple_compile_test_code="int some_variable = 0;" + + # Code to be used in simple link tests + lt_simple_link_test_code='int main(int, char *[[]]) { return(0); }' + + # ltmain only uses $CC for tagged configurations so make sure $CC is set. + _LT_TAG_COMPILER + + # save warnings/boilerplate of simple test code + _LT_COMPILER_BOILERPLATE + _LT_LINKER_BOILERPLATE + + # Allow CC to be a program name with arguments. + lt_save_CC=$CC + lt_save_CFLAGS=$CFLAGS + lt_save_LD=$LD + lt_save_GCC=$GCC + GCC=$GXX + lt_save_with_gnu_ld=$with_gnu_ld + lt_save_path_LD=$lt_cv_path_LD + if test -n "${lt_cv_prog_gnu_ldcxx+set}"; then + lt_cv_prog_gnu_ld=$lt_cv_prog_gnu_ldcxx + else + $as_unset lt_cv_prog_gnu_ld + fi + if test -n "${lt_cv_path_LDCXX+set}"; then + lt_cv_path_LD=$lt_cv_path_LDCXX + else + $as_unset lt_cv_path_LD + fi + test -z "${LDCXX+set}" || LD=$LDCXX + CC=${CXX-"c++"} + CFLAGS=$CXXFLAGS + compiler=$CC + _LT_TAGVAR(compiler, $1)=$CC + _LT_CC_BASENAME([$compiler]) + + if test -n "$compiler"; then + # We don't want -fno-exception when compiling C++ code, so set the + # no_builtin_flag separately + if test "$GXX" = yes; then + _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -fno-builtin' + else + _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)= + fi + + if test "$GXX" = yes; then + # Set up default GNU C++ configuration + + LT_PATH_LD + + # Check if GNU C++ uses GNU ld as the underlying linker, since the + # archiving commands below assume that GNU ld is being used. + if test "$with_gnu_ld" = yes; then + _LT_TAGVAR(archive_cmds, $1)='$CC $pic_flag -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC $pic_flag -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic' + + # If archive_cmds runs LD, not CC, wlarc should be empty + # XXX I think wlarc can be eliminated in ltcf-cxx, but I need to + # investigate it a little bit more. (MM) + wlarc='${wl}' + + # ancient GNU ld didn't support --whole-archive et. al. + if eval "`$CC -print-prog-name=ld` --help 2>&1" | + $GREP 'no-whole-archive' > /dev/null; then + _LT_TAGVAR(whole_archive_flag_spec, $1)="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive' + else + _LT_TAGVAR(whole_archive_flag_spec, $1)= + fi + else + with_gnu_ld=no + wlarc= + + # A generic and very simple default shared library creation + # command for GNU C++ for the case where it uses the native + # linker, instead of GNU ld. If possible, this setting should + # overridden to take advantage of the native linker features on + # the platform it is being used on. + _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib' + fi + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + + else + GXX=no + with_gnu_ld=no + wlarc= + fi + + # PORTME: fill in a description of your system's C++ link characteristics + AC_MSG_CHECKING([whether the $compiler linker ($LD) supports shared libraries]) + _LT_TAGVAR(ld_shlibs, $1)=yes + case $host_os in + aix3*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + aix[[4-9]]*) + if test "$host_cpu" = ia64; then + # On IA64, the linker does run time linking by default, so we don't + # have to do anything special. + aix_use_runtimelinking=no + exp_sym_flag='-Bexport' + no_entry_flag="" + else + aix_use_runtimelinking=no + + # Test if we are trying to use run time linking or normal + # AIX style linking. If -brtl is somewhere in LDFLAGS, we + # need to do runtime linking. + case $host_os in aix4.[[23]]|aix4.[[23]].*|aix[[5-9]]*) + for ld_flag in $LDFLAGS; do + case $ld_flag in + *-brtl*) + aix_use_runtimelinking=yes + break + ;; + esac + done + ;; + esac + + exp_sym_flag='-bexport' + no_entry_flag='-bnoentry' + fi + + # When large executables or shared objects are built, AIX ld can + # have problems creating the table of contents. If linking a library + # or program results in "error TOC overflow" add -mminimal-toc to + # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not + # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS. + + _LT_TAGVAR(archive_cmds, $1)='' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_direct_absolute, $1)=yes + _LT_TAGVAR(hardcode_libdir_separator, $1)=':' + _LT_TAGVAR(link_all_deplibs, $1)=yes + _LT_TAGVAR(file_list_spec, $1)='${wl}-f,' + + if test "$GXX" = yes; then + case $host_os in aix4.[[012]]|aix4.[[012]].*) + # We only want to do this on AIX 4.2 and lower, the check + # below for broken collect2 doesn't work under 4.3+ + collect2name=`${CC} -print-prog-name=collect2` + if test -f "$collect2name" && + strings "$collect2name" | $GREP resolve_lib_name >/dev/null + then + # We have reworked collect2 + : + else + # We have old collect2 + _LT_TAGVAR(hardcode_direct, $1)=unsupported + # It fails to find uninstalled libraries when the uninstalled + # path is not listed in the libpath. Setting hardcode_minus_L + # to unsupported forces relinking + _LT_TAGVAR(hardcode_minus_L, $1)=yes + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)= + fi + esac + shared_flag='-shared' + if test "$aix_use_runtimelinking" = yes; then + shared_flag="$shared_flag "'${wl}-G' + fi + else + # not using gcc + if test "$host_cpu" = ia64; then + # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release + # chokes on -Wl,-G. The following line is correct: + shared_flag='-G' + else + if test "$aix_use_runtimelinking" = yes; then + shared_flag='${wl}-G' + else + shared_flag='${wl}-bM:SRE' + fi + fi + fi + + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-bexpall' + # It seems that -bexpall does not export symbols beginning with + # underscore (_), so it is better to generate a list of symbols to + # export. + _LT_TAGVAR(always_export_symbols, $1)=yes + if test "$aix_use_runtimelinking" = yes; then + # Warning - without using the other runtime loading flags (-brtl), + # -berok will link without error, but may produce a broken library. + _LT_TAGVAR(allow_undefined_flag, $1)='-berok' + # Determine the default libpath from the value encoded in an empty + # executable. + _LT_SYS_MODULE_PATH_AIX([$1]) + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath" + + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags `if test "x${allow_undefined_flag}" != "x"; then func_echo_all "${wl}${allow_undefined_flag}"; else :; fi` '"\${wl}$exp_sym_flag:\$export_symbols $shared_flag" + else + if test "$host_cpu" = ia64; then + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R $libdir:/usr/lib:/lib' + _LT_TAGVAR(allow_undefined_flag, $1)="-z nodefs" + _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags ${wl}${allow_undefined_flag} '"\${wl}$exp_sym_flag:\$export_symbols" + else + # Determine the default libpath from the value encoded in an + # empty executable. + _LT_SYS_MODULE_PATH_AIX([$1]) + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath" + # Warning - without using the other run time loading flags, + # -berok will link without error, but may produce a broken library. + _LT_TAGVAR(no_undefined_flag, $1)=' ${wl}-bernotok' + _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-berok' + if test "$with_gnu_ld" = yes; then + # We only use this code for GNU lds that support --whole-archive. + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive$convenience ${wl}--no-whole-archive' + else + # Exported symbols can be pulled into shared objects from archives + _LT_TAGVAR(whole_archive_flag_spec, $1)='$convenience' + fi + _LT_TAGVAR(archive_cmds_need_lc, $1)=yes + # This is similar to how AIX traditionally builds its shared + # libraries. + _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs ${wl}-bnoentry $compiler_flags ${wl}-bE:$export_symbols${allow_undefined_flag}~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$soname' + fi + fi + ;; + + beos*) + if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + # Joseph Beckenbach says some releases of gcc + # support --undefined. This deserves some investigation. FIXME + _LT_TAGVAR(archive_cmds, $1)='$CC -nostart $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + + chorus*) + case $cc_basename in + *) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + esac + ;; + + cygwin* | mingw* | pw32* | cegcc*) + case $GXX,$cc_basename in + ,cl* | no,cl*) + # Native MSVC + # hardcode_libdir_flag_spec is actually meaningless, as there is + # no search path for DLLs. + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + _LT_TAGVAR(always_export_symbols, $1)=yes + _LT_TAGVAR(file_list_spec, $1)='@' + # Tell ltmain to make .lib files, not .a files. + libext=lib + # Tell ltmain to make .dll files, not .so files. + shrext_cmds=".dll" + # FIXME: Setting linknames here is a bad hack. + _LT_TAGVAR(archive_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $compiler_flags $deplibs -Wl,-dll~linknames=' + _LT_TAGVAR(archive_expsym_cmds, $1)='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + $SED -n -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' -e '1\\\!p' < $export_symbols > $output_objdir/$soname.exp; + else + $SED -e 's/\\\\\\\(.*\\\\\\\)/-link\\\ -EXPORT:\\\\\\\1/' < $export_symbols > $output_objdir/$soname.exp; + fi~ + $CC -o $tool_output_objdir$soname $libobjs $compiler_flags $deplibs "@$tool_output_objdir$soname.exp" -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~ + linknames=' + # The linker will not automatically build a static lib if we build a DLL. + # _LT_TAGVAR(old_archive_from_new_cmds, $1)='true' + _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + # Don't use ranlib + _LT_TAGVAR(old_postinstall_cmds, $1)='chmod 644 $oldlib' + _LT_TAGVAR(postlink_cmds, $1)='lt_outputfile="@OUTPUT@"~ + lt_tool_outputfile="@TOOL_OUTPUT@"~ + case $lt_outputfile in + *.exe|*.EXE) ;; + *) + lt_outputfile="$lt_outputfile.exe" + lt_tool_outputfile="$lt_tool_outputfile.exe" + ;; + esac~ + func_to_tool_file "$lt_outputfile"~ + if test "$MANIFEST_TOOL" != ":" && test -f "$lt_outputfile.manifest"; then + $MANIFEST_TOOL -manifest "$lt_tool_outputfile.manifest" -outputresource:"$lt_tool_outputfile" || exit 1; + $RM "$lt_outputfile.manifest"; + fi' + ;; + *) + # g++ + # _LT_TAGVAR(hardcode_libdir_flag_spec, $1) is actually meaningless, + # as there is no search path for DLLs. + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-all-symbols' + _LT_TAGVAR(allow_undefined_flag, $1)=unsupported + _LT_TAGVAR(always_export_symbols, $1)=no + _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + + if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + # If the export-symbols file already is a .def file (1st line + # is EXPORTS), use it as is; otherwise, prepend... + _LT_TAGVAR(archive_expsym_cmds, $1)='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then + cp $export_symbols $output_objdir/$soname.def; + else + echo EXPORTS > $output_objdir/$soname.def; + cat $export_symbols >> $output_objdir/$soname.def; + fi~ + $CC -shared -nostdlib $output_objdir/$soname.def $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + esac + ;; + darwin* | rhapsody*) + _LT_DARWIN_LINKER_FEATURES($1) + ;; + + dgux*) + case $cc_basename in + ec++*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + ghcx*) + # Green Hills C++ Compiler + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + *) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + esac + ;; + + freebsd2.*) + # C++ shared libraries reported to be fairly broken before + # switch to ELF + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + + freebsd-elf*) + _LT_TAGVAR(archive_cmds_need_lc, $1)=no + ;; + + freebsd* | dragonfly*) + # FreeBSD 3 and later use GNU C++ and GNU ld with standard ELF + # conventions + _LT_TAGVAR(ld_shlibs, $1)=yes + ;; + + gnu*) + ;; + + haiku*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(link_all_deplibs, $1)=yes + ;; + + hpux9*) + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_minus_L, $1)=yes # Not in the search PATH, + # but as the default + # location of the library. + + case $cc_basename in + CC*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + aCC*) + _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -b ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $EGREP "\-L"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + ;; + *) + if test "$GXX" = yes; then + _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -shared -nostdlib $pic_flag ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib' + else + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + esac + ;; + + hpux10*|hpux11*) + if test $with_gnu_ld = no; then + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + + case $host_cpu in + hppa*64*|ia64*) + ;; + *) + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + ;; + esac + fi + case $host_cpu in + hppa*64*|ia64*) + _LT_TAGVAR(hardcode_direct, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + ;; + *) + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_direct_absolute, $1)=yes + _LT_TAGVAR(hardcode_minus_L, $1)=yes # Not in the search PATH, + # but as the default + # location of the library. + ;; + esac + + case $cc_basename in + CC*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + aCC*) + case $host_cpu in + hppa*64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + ia64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + *) + _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + esac + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $GREP "\-L"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + ;; + *) + if test "$GXX" = yes; then + if test $with_gnu_ld = no; then + case $host_cpu in + hppa*64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib -fPIC ${wl}+h ${wl}$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + ia64*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $pic_flag ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + *) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $pic_flag ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + ;; + esac + fi + else + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + esac + ;; + + interix[[3-9]]*) + _LT_TAGVAR(hardcode_direct, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc. + # Instead, shared libraries are loaded at an image base (0x10000000 by + # default) and relocated if they conflict, which is a slow very memory + # consuming and fragmenting process. To avoid this, we pick a random, + # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link + # time. Moving up from 0x10000000 also allows more sbrk(2) space. + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s,^,_," $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--retain-symbols-file,$output_objdir/$soname.expsym ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + ;; + irix5* | irix6*) + case $cc_basename in + CC*) + # SGI C++ + _LT_TAGVAR(archive_cmds, $1)='$CC -shared -all -multigot $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + + # Archives containing C++ object files must be created using + # "CC -ar", where "CC" is the IRIX C++ compiler. This is + # necessary to make sure instantiated templates are included + # in the archive. + _LT_TAGVAR(old_archive_cmds, $1)='$CC -ar -WR,-u -o $oldlib $oldobjs' + ;; + *) + if test "$GXX" = yes; then + if test "$with_gnu_ld" = no; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + else + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` -o $lib' + fi + fi + _LT_TAGVAR(link_all_deplibs, $1)=yes + ;; + esac + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + _LT_TAGVAR(inherit_rpath, $1)=yes + ;; + + linux* | k*bsd*-gnu | kopensolaris*-gnu) + case $cc_basename in + KCC*) + # Kuck and Associates, Inc. (KAI) C++ Compiler + + # KCC will only create a shared library if the output file + # ends with ".so" (or ".sl" for HP-UX), so rename the library + # to its proper name (with version) after linking. + _LT_TAGVAR(archive_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib ${wl}-retain-symbols-file,$export_symbols; mv \$templib $lib' + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`$CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1 | $GREP "ld"`; rm -f libconftest$shared_ext; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic' + + # Archives containing C++ object files must be created using + # "CC -Bstatic", where "CC" is the KAI C++ compiler. + _LT_TAGVAR(old_archive_cmds, $1)='$CC -Bstatic -o $oldlib $oldobjs' + ;; + icpc* | ecpc* ) + # Intel C++ + with_gnu_ld=yes + # version 8.0 and above of icpc choke on multiply defined symbols + # if we add $predep_objects and $postdep_objects, however 7.1 and + # earlier do not add the objects themselves. + case `$CC -V 2>&1` in + *"Version 7."*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + ;; + *) # Version 8.0 or newer + tmp_idyn= + case $host_cpu in + ia64*) tmp_idyn=' -i_dynamic';; + esac + _LT_TAGVAR(archive_cmds, $1)='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib' + ;; + esac + _LT_TAGVAR(archive_cmds_need_lc, $1)=no + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic' + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive$convenience ${wl}--no-whole-archive' + ;; + pgCC* | pgcpp*) + # Portland Group C++ compiler + case `$CC -V` in + *pgCC\ [[1-5]].* | *pgcpp\ [[1-5]].*) + _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ + compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ + $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ + $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ + $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + esac + + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}--rpath ${wl}$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic' + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + ;; + cxx*) + # Compaq C++ + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib ${wl}-retain-symbols-file $wl$export_symbols' + + runpath_var=LD_RUN_PATH + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld"`; templist=`func_echo_all "$templist" | $SED "s/\(^.*ld.*\)\( .*ld .*$\)/\1/"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "X$list" | $Xsed' + ;; + xl* | mpixl* | bgxl*) + # IBM XL 8.0 on PPC, with GNU ld + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic' + _LT_TAGVAR(archive_cmds, $1)='$CC -qmkshrobj $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib' + if test "x$supports_anon_versioning" = xyes; then + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ + cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + echo "local: *; };" >> $output_objdir/$libname.ver~ + $CC -qmkshrobj $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-version-script ${wl}$output_objdir/$libname.ver -o $lib' + fi + ;; + *) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) + # Sun C++ 5.9 + _LT_TAGVAR(no_undefined_flag, $1)=' -zdefs' + _LT_TAGVAR(archive_cmds, $1)='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-retain-symbols-file ${wl}$export_symbols' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` ${wl}--no-whole-archive' + _LT_TAGVAR(compiler_needs_object, $1)=yes + + # Not sure whether something based on + # $CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1 + # would be better. + output_verbose_link_cmd='func_echo_all' + + # Archives containing C++ object files must be created using + # "CC -xar", where "CC" is the Sun C++ compiler. This is + # necessary to make sure instantiated templates are included + # in the archive. + _LT_TAGVAR(old_archive_cmds, $1)='$CC -xar -o $oldlib $oldobjs' + ;; + esac + ;; + esac + ;; + + lynxos*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + + m88k*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + + mvs*) + case $cc_basename in + cxx*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + *) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + esac + ;; + + netbsd*) + if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $predep_objects $libobjs $deplibs $postdep_objects $linker_flags' + wlarc= + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + fi + # Workaround some broken pre-1.5 toolchains + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP conftest.$objext | $SED -e "s:-lgcc -lc -lgcc::"' + ;; + + *nto* | *qnx*) + _LT_TAGVAR(ld_shlibs, $1)=yes + ;; + + openbsd2*) + # C++ shared libraries are fairly broken + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + + openbsd*) + if test -f /usr/libexec/ld.so; then + _LT_TAGVAR(hardcode_direct, $1)=yes + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + _LT_TAGVAR(hardcode_direct_absolute, $1)=yes + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + if test -z "`echo __ELF__ | $CC -E - | grep __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-retain-symbols-file,$export_symbols -o $lib' + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E' + _LT_TAGVAR(whole_archive_flag_spec, $1)="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive' + fi + output_verbose_link_cmd=func_echo_all + else + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + + osf3* | osf4* | osf5*) + case $cc_basename in + KCC*) + # Kuck and Associates, Inc. (KAI) C++ Compiler + + # KCC will only create a shared library if the output file + # ends with ".so" (or ".sl" for HP-UX), so rename the library + # to its proper name (with version) after linking. + _LT_TAGVAR(archive_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo "$lib" | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib' + + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + + # Archives containing C++ object files must be created using + # the KAI C++ compiler. + case $host in + osf3*) _LT_TAGVAR(old_archive_cmds, $1)='$CC -Bstatic -o $oldlib $oldobjs' ;; + *) _LT_TAGVAR(old_archive_cmds, $1)='$CC -o $oldlib $oldobjs' ;; + esac + ;; + RCC*) + # Rational C++ 2.4.1 + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + cxx*) + case $host in + osf3*) + _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $soname `test -n "$verstring" && func_echo_all "${wl}-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + ;; + *) + _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*' + _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done~ + echo "-hidden">> $lib.exp~ + $CC -shared$allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname ${wl}-input ${wl}$lib.exp `test -n "$verstring" && $ECHO "-set_version $verstring"` -update_registry ${output_objdir}/so_locations -o $lib~ + $RM $lib.exp' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir' + ;; + esac + + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + # + # There doesn't appear to be a way to prevent this compiler from + # explicitly linking system object files so we need to strip them + # from the output so that they don't get included in the library + # dependencies. + output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld" | $GREP -v "ld:"`; templist=`func_echo_all "$templist" | $SED "s/\(^.*ld.*\)\( .*ld.*$\)/\1/"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' + ;; + *) + if test "$GXX" = yes && test "$with_gnu_ld" = no; then + _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*' + case $host in + osf3*) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib ${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + ;; + *) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib ${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-msym ${wl}-soname ${wl}$soname `test -n "$verstring" && func_echo_all "${wl}-set_version ${wl}$verstring"` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib' + ;; + esac + + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=: + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + + else + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + fi + ;; + esac + ;; + + psos*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + + sunos4*) + case $cc_basename in + CC*) + # Sun C++ 4.x + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + lcc*) + # Lucid + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + *) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + esac + ;; + + solaris*) + case $cc_basename in + CC* | sunCC*) + # Sun C++ 4.2, 5.x and Centerline C++ + _LT_TAGVAR(archive_cmds_need_lc,$1)=yes + _LT_TAGVAR(no_undefined_flag, $1)=' -zdefs' + _LT_TAGVAR(archive_cmds, $1)='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -G${allow_undefined_flag} ${wl}-M ${wl}$lib.exp -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' + + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + case $host_os in + solaris2.[[0-5]] | solaris2.[[0-5]].*) ;; + *) + # The compiler driver will combine and reorder linker options, + # but understands `-z linker_flag'. + # Supported since Solaris 2.6 (maybe 2.5.1?) + _LT_TAGVAR(whole_archive_flag_spec, $1)='-z allextract$convenience -z defaultextract' + ;; + esac + _LT_TAGVAR(link_all_deplibs, $1)=yes + + output_verbose_link_cmd='func_echo_all' + + # Archives containing C++ object files must be created using + # "CC -xar", where "CC" is the Sun C++ compiler. This is + # necessary to make sure instantiated templates are included + # in the archive. + _LT_TAGVAR(old_archive_cmds, $1)='$CC -xar -o $oldlib $oldobjs' + ;; + gcx*) + # Green Hills C++ Compiler + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib' + + # The C++ compiler must be used to create the archive. + _LT_TAGVAR(old_archive_cmds, $1)='$CC $LDFLAGS -archive -o $oldlib $oldobjs' + ;; + *) + # GNU C++ compiler with Solaris linker + if test "$GXX" = yes && test "$with_gnu_ld" = no; then + _LT_TAGVAR(no_undefined_flag, $1)=' ${wl}-z ${wl}defs' + if $CC --version | $GREP -v '^2\.7' > /dev/null; then + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib $LDFLAGS $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -shared $pic_flag -nostdlib ${wl}-M $wl$lib.exp -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + else + # g++ 2.7 appears to require `-G' NOT `-shared' on this + # platform. + _LT_TAGVAR(archive_cmds, $1)='$CC -G -nostdlib $LDFLAGS $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ + $CC -G -nostdlib ${wl}-M $wl$lib.exp -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' + + # Commands to make compiler produce verbose output that lists + # what "hidden" libraries, object files and flags are used when + # linking a shared library. + output_verbose_link_cmd='$CC -G $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP "\-L"' + fi + + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R $wl$libdir' + case $host_os in + solaris2.[[0-5]] | solaris2.[[0-5]].*) ;; + *) + _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}-z ${wl}allextract$convenience ${wl}-z ${wl}defaultextract' + ;; + esac + fi + ;; + esac + ;; + + sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[[01]].[[10]]* | unixware7* | sco3.2v5.0.[[024]]*) + _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text' + _LT_TAGVAR(archive_cmds_need_lc, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + runpath_var='LD_RUN_PATH' + + case $cc_basename in + CC*) + _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + *) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + esac + ;; + + sysv5* | sco3.2v5* | sco5v6*) + # Note: We can NOT use -z defs as we might desire, because we do not + # link with -lc, and that would cause any symbols used from libc to + # always be unresolved, which means just about no library would + # ever link correctly. If we're not using GNU ld we use -z text + # though, which does catch some bad symbols but isn't as heavy-handed + # as -z defs. + _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text' + _LT_TAGVAR(allow_undefined_flag, $1)='${wl}-z,nodefs' + _LT_TAGVAR(archive_cmds_need_lc, $1)=no + _LT_TAGVAR(hardcode_shlibpath_var, $1)=no + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R,$libdir' + _LT_TAGVAR(hardcode_libdir_separator, $1)=':' + _LT_TAGVAR(link_all_deplibs, $1)=yes + _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-Bexport' + runpath_var='LD_RUN_PATH' + + case $cc_basename in + CC*) + _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(old_archive_cmds, $1)='$CC -Tprelink_objects $oldobjs~ + '"$_LT_TAGVAR(old_archive_cmds, $1)" + _LT_TAGVAR(reload_cmds, $1)='$CC -Tprelink_objects $reload_objs~ + '"$_LT_TAGVAR(reload_cmds, $1)" + ;; + *) + _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags' + ;; + esac + ;; + + tandem*) + case $cc_basename in + NCC*) + # NonStop-UX NCC 3.20 + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + *) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + esac + ;; + + vxworks*) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + + *) + # FIXME: insert proper C++ library support + _LT_TAGVAR(ld_shlibs, $1)=no + ;; + esac + + AC_MSG_RESULT([$_LT_TAGVAR(ld_shlibs, $1)]) + test "$_LT_TAGVAR(ld_shlibs, $1)" = no && can_build_shared=no + + _LT_TAGVAR(GCC, $1)="$GXX" + _LT_TAGVAR(LD, $1)="$LD" + + ## CAVEAT EMPTOR: + ## There is no encapsulation within the following macros, do not change + ## the running order or otherwise move them around unless you know exactly + ## what you are doing... + _LT_SYS_HIDDEN_LIBDEPS($1) + _LT_COMPILER_PIC($1) + _LT_COMPILER_C_O($1) + _LT_COMPILER_FILE_LOCKS($1) + _LT_LINKER_SHLIBS($1) + _LT_SYS_DYNAMIC_LINKER($1) + _LT_LINKER_HARDCODE_LIBPATH($1) + + _LT_CONFIG($1) + fi # test -n "$compiler" + + CC=$lt_save_CC + CFLAGS=$lt_save_CFLAGS + LDCXX=$LD + LD=$lt_save_LD + GCC=$lt_save_GCC + with_gnu_ld=$lt_save_with_gnu_ld + lt_cv_path_LDCXX=$lt_cv_path_LD + lt_cv_path_LD=$lt_save_path_LD + lt_cv_prog_gnu_ldcxx=$lt_cv_prog_gnu_ld + lt_cv_prog_gnu_ld=$lt_save_with_gnu_ld +fi # test "$_lt_caught_CXX_error" != yes + +AC_LANG_POP +])# _LT_LANG_CXX_CONFIG + + +# _LT_FUNC_STRIPNAME_CNF +# ---------------------- +# func_stripname_cnf prefix suffix name +# strip PREFIX and SUFFIX off of NAME. +# PREFIX and SUFFIX must not contain globbing or regex special +# characters, hashes, percent signs, but SUFFIX may contain a leading +# dot (in which case that matches only a dot). +# +# This function is identical to the (non-XSI) version of func_stripname, +# except this one can be used by m4 code that may be executed by configure, +# rather than the libtool script. +m4_defun([_LT_FUNC_STRIPNAME_CNF],[dnl +AC_REQUIRE([_LT_DECL_SED]) +AC_REQUIRE([_LT_PROG_ECHO_BACKSLASH]) +func_stripname_cnf () +{ + case ${2} in + .*) func_stripname_result=`$ECHO "${3}" | $SED "s%^${1}%%; s%\\\\${2}\$%%"`;; + *) func_stripname_result=`$ECHO "${3}" | $SED "s%^${1}%%; s%${2}\$%%"`;; + esac +} # func_stripname_cnf +])# _LT_FUNC_STRIPNAME_CNF + +# _LT_SYS_HIDDEN_LIBDEPS([TAGNAME]) +# --------------------------------- +# Figure out "hidden" library dependencies from verbose +# compiler output when linking a shared library. +# Parse the compiler output and extract the necessary +# objects, libraries and library flags. +m4_defun([_LT_SYS_HIDDEN_LIBDEPS], +[m4_require([_LT_FILEUTILS_DEFAULTS])dnl +AC_REQUIRE([_LT_FUNC_STRIPNAME_CNF])dnl +# Dependencies to place before and after the object being linked: +_LT_TAGVAR(predep_objects, $1)= +_LT_TAGVAR(postdep_objects, $1)= +_LT_TAGVAR(predeps, $1)= +_LT_TAGVAR(postdeps, $1)= +_LT_TAGVAR(compiler_lib_search_path, $1)= + +dnl we can't use the lt_simple_compile_test_code here, +dnl because it contains code intended for an executable, +dnl not a library. It's possible we should let each +dnl tag define a new lt_????_link_test_code variable, +dnl but it's only used here... +m4_if([$1], [], [cat > conftest.$ac_ext <<_LT_EOF +int a; +void foo (void) { a = 0; } +_LT_EOF +], [$1], [CXX], [cat > conftest.$ac_ext <<_LT_EOF +class Foo +{ +public: + Foo (void) { a = 0; } +private: + int a; +}; +_LT_EOF +], [$1], [F77], [cat > conftest.$ac_ext <<_LT_EOF + subroutine foo + implicit none + integer*4 a + a=0 + return + end +_LT_EOF +], [$1], [FC], [cat > conftest.$ac_ext <<_LT_EOF + subroutine foo + implicit none + integer a + a=0 + return + end +_LT_EOF +], [$1], [GCJ], [cat > conftest.$ac_ext <<_LT_EOF +public class foo { + private int a; + public void bar (void) { + a = 0; + } +}; +_LT_EOF +], [$1], [GO], [cat > conftest.$ac_ext <<_LT_EOF +package foo +func foo() { +} +_LT_EOF +]) + +_lt_libdeps_save_CFLAGS=$CFLAGS +case "$CC $CFLAGS " in #( +*\ -flto*\ *) CFLAGS="$CFLAGS -fno-lto" ;; +*\ -fwhopr*\ *) CFLAGS="$CFLAGS -fno-whopr" ;; +*\ -fuse-linker-plugin*\ *) CFLAGS="$CFLAGS -fno-use-linker-plugin" ;; +esac + +dnl Parse the compiler output and extract the necessary +dnl objects, libraries and library flags. +if AC_TRY_EVAL(ac_compile); then + # Parse the compiler output and extract the necessary + # objects, libraries and library flags. + + # Sentinel used to keep track of whether or not we are before + # the conftest object file. + pre_test_object_deps_done=no + + for p in `eval "$output_verbose_link_cmd"`; do + case ${prev}${p} in + + -L* | -R* | -l*) + # Some compilers place space between "-{L,R}" and the path. + # Remove the space. + if test $p = "-L" || + test $p = "-R"; then + prev=$p + continue + fi + + # Expand the sysroot to ease extracting the directories later. + if test -z "$prev"; then + case $p in + -L*) func_stripname_cnf '-L' '' "$p"; prev=-L; p=$func_stripname_result ;; + -R*) func_stripname_cnf '-R' '' "$p"; prev=-R; p=$func_stripname_result ;; + -l*) func_stripname_cnf '-l' '' "$p"; prev=-l; p=$func_stripname_result ;; + esac + fi + case $p in + =*) func_stripname_cnf '=' '' "$p"; p=$lt_sysroot$func_stripname_result ;; + esac + if test "$pre_test_object_deps_done" = no; then + case ${prev} in + -L | -R) + # Internal compiler library paths should come after those + # provided the user. The postdeps already come after the + # user supplied libs so there is no need to process them. + if test -z "$_LT_TAGVAR(compiler_lib_search_path, $1)"; then + _LT_TAGVAR(compiler_lib_search_path, $1)="${prev}${p}" + else + _LT_TAGVAR(compiler_lib_search_path, $1)="${_LT_TAGVAR(compiler_lib_search_path, $1)} ${prev}${p}" + fi + ;; + # The "-l" case would never come before the object being + # linked, so don't bother handling this case. + esac + else + if test -z "$_LT_TAGVAR(postdeps, $1)"; then + _LT_TAGVAR(postdeps, $1)="${prev}${p}" + else + _LT_TAGVAR(postdeps, $1)="${_LT_TAGVAR(postdeps, $1)} ${prev}${p}" + fi + fi + prev= + ;; + + *.lto.$objext) ;; # Ignore GCC LTO objects + *.$objext) + # This assumes that the test object file only shows up + # once in the compiler output. + if test "$p" = "conftest.$objext"; then + pre_test_object_deps_done=yes + continue + fi + + if test "$pre_test_object_deps_done" = no; then + if test -z "$_LT_TAGVAR(predep_objects, $1)"; then + _LT_TAGVAR(predep_objects, $1)="$p" + else + _LT_TAGVAR(predep_objects, $1)="$_LT_TAGVAR(predep_objects, $1) $p" + fi + else + if test -z "$_LT_TAGVAR(postdep_objects, $1)"; then + _LT_TAGVAR(postdep_objects, $1)="$p" + else + _LT_TAGVAR(postdep_objects, $1)="$_LT_TAGVAR(postdep_objects, $1) $p" + fi + fi + ;; + + *) ;; # Ignore the rest. + + esac + done + + # Clean up. + rm -f a.out a.exe +else + echo "libtool.m4: error: problem compiling $1 test program" +fi + +$RM -f confest.$objext +CFLAGS=$_lt_libdeps_save_CFLAGS + +# PORTME: override above test on systems where it is broken +m4_if([$1], [CXX], +[case $host_os in +interix[[3-9]]*) + # Interix 3.5 installs completely hosed .la files for C++, so rather than + # hack all around it, let's just trust "g++" to DTRT. + _LT_TAGVAR(predep_objects,$1)= + _LT_TAGVAR(postdep_objects,$1)= + _LT_TAGVAR(postdeps,$1)= + ;; + +linux*) + case `$CC -V 2>&1 | sed 5q` in + *Sun\ C*) + # Sun C++ 5.9 + + # The more standards-conforming stlport4 library is + # incompatible with the Cstd library. Avoid specifying + # it if it's in CXXFLAGS. Ignore libCrun as + # -library=stlport4 depends on it. + case " $CXX $CXXFLAGS " in + *" -library=stlport4 "*) + solaris_use_stlport4=yes + ;; + esac + + if test "$solaris_use_stlport4" != yes; then + _LT_TAGVAR(postdeps,$1)='-library=Cstd -library=Crun' + fi + ;; + esac + ;; + +solaris*) + case $cc_basename in + CC* | sunCC*) + # The more standards-conforming stlport4 library is + # incompatible with the Cstd library. Avoid specifying + # it if it's in CXXFLAGS. Ignore libCrun as + # -library=stlport4 depends on it. + case " $CXX $CXXFLAGS " in + *" -library=stlport4 "*) + solaris_use_stlport4=yes + ;; + esac + + # Adding this requires a known-good setup of shared libraries for + # Sun compiler versions before 5.6, else PIC objects from an old + # archive will be linked into the output, leading to subtle bugs. + if test "$solaris_use_stlport4" != yes; then + _LT_TAGVAR(postdeps,$1)='-library=Cstd -library=Crun' + fi + ;; + esac + ;; +esac +]) + +case " $_LT_TAGVAR(postdeps, $1) " in +*" -lc "*) _LT_TAGVAR(archive_cmds_need_lc, $1)=no ;; +esac + _LT_TAGVAR(compiler_lib_search_dirs, $1)= +if test -n "${_LT_TAGVAR(compiler_lib_search_path, $1)}"; then + _LT_TAGVAR(compiler_lib_search_dirs, $1)=`echo " ${_LT_TAGVAR(compiler_lib_search_path, $1)}" | ${SED} -e 's! -L! !g' -e 's!^ !!'` +fi +_LT_TAGDECL([], [compiler_lib_search_dirs], [1], + [The directories searched by this compiler when creating a shared library]) +_LT_TAGDECL([], [predep_objects], [1], + [Dependencies to place before and after the objects being linked to + create a shared library]) +_LT_TAGDECL([], [postdep_objects], [1]) +_LT_TAGDECL([], [predeps], [1]) +_LT_TAGDECL([], [postdeps], [1]) +_LT_TAGDECL([], [compiler_lib_search_path], [1], + [The library search path used internally by the compiler when linking + a shared library]) +])# _LT_SYS_HIDDEN_LIBDEPS + + +# _LT_LANG_F77_CONFIG([TAG]) +# -------------------------- +# Ensure that the configuration variables for a Fortran 77 compiler are +# suitably defined. These variables are subsequently used by _LT_CONFIG +# to write the compiler configuration to `libtool'. +m4_defun([_LT_LANG_F77_CONFIG], +[AC_LANG_PUSH(Fortran 77) +if test -z "$F77" || test "X$F77" = "Xno"; then + _lt_disable_F77=yes +fi + +_LT_TAGVAR(archive_cmds_need_lc, $1)=no +_LT_TAGVAR(allow_undefined_flag, $1)= +_LT_TAGVAR(always_export_symbols, $1)=no +_LT_TAGVAR(archive_expsym_cmds, $1)= +_LT_TAGVAR(export_dynamic_flag_spec, $1)= +_LT_TAGVAR(hardcode_direct, $1)=no +_LT_TAGVAR(hardcode_direct_absolute, $1)=no +_LT_TAGVAR(hardcode_libdir_flag_spec, $1)= +_LT_TAGVAR(hardcode_libdir_separator, $1)= +_LT_TAGVAR(hardcode_minus_L, $1)=no +_LT_TAGVAR(hardcode_automatic, $1)=no +_LT_TAGVAR(inherit_rpath, $1)=no +_LT_TAGVAR(module_cmds, $1)= +_LT_TAGVAR(module_expsym_cmds, $1)= +_LT_TAGVAR(link_all_deplibs, $1)=unknown +_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds +_LT_TAGVAR(reload_flag, $1)=$reload_flag +_LT_TAGVAR(reload_cmds, $1)=$reload_cmds +_LT_TAGVAR(no_undefined_flag, $1)= +_LT_TAGVAR(whole_archive_flag_spec, $1)= +_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no + +# Source file extension for f77 test sources. +ac_ext=f + +# Object file extension for compiled f77 test sources. +objext=o +_LT_TAGVAR(objext, $1)=$objext + +# No sense in running all these tests if we already determined that +# the F77 compiler isn't working. Some variables (like enable_shared) +# are currently assumed to apply to all compilers on this platform, +# and will be corrupted by setting them based on a non-working compiler. +if test "$_lt_disable_F77" != yes; then + # Code to be used in simple compile tests + lt_simple_compile_test_code="\ + subroutine t + return + end +" + + # Code to be used in simple link tests + lt_simple_link_test_code="\ + program t + end +" + + # ltmain only uses $CC for tagged configurations so make sure $CC is set. + _LT_TAG_COMPILER + + # save warnings/boilerplate of simple test code + _LT_COMPILER_BOILERPLATE + _LT_LINKER_BOILERPLATE + + # Allow CC to be a program name with arguments. + lt_save_CC="$CC" + lt_save_GCC=$GCC + lt_save_CFLAGS=$CFLAGS + CC=${F77-"f77"} + CFLAGS=$FFLAGS + compiler=$CC + _LT_TAGVAR(compiler, $1)=$CC + _LT_CC_BASENAME([$compiler]) + GCC=$G77 + if test -n "$compiler"; then + AC_MSG_CHECKING([if libtool supports shared libraries]) + AC_MSG_RESULT([$can_build_shared]) + + AC_MSG_CHECKING([whether to build shared libraries]) + test "$can_build_shared" = "no" && enable_shared=no + + # On AIX, shared libraries and static libraries use the same namespace, and + # are all built from PIC. + case $host_os in + aix3*) + test "$enable_shared" = yes && enable_static=no + if test -n "$RANLIB"; then + archive_cmds="$archive_cmds~\$RANLIB \$lib" + postinstall_cmds='$RANLIB $lib' + fi + ;; + aix[[4-9]]*) + if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then + test "$enable_shared" = yes && enable_static=no + fi + ;; + esac + AC_MSG_RESULT([$enable_shared]) + + AC_MSG_CHECKING([whether to build static libraries]) + # Make sure either enable_shared or enable_static is yes. + test "$enable_shared" = yes || enable_static=yes + AC_MSG_RESULT([$enable_static]) + + _LT_TAGVAR(GCC, $1)="$G77" + _LT_TAGVAR(LD, $1)="$LD" + + ## CAVEAT EMPTOR: + ## There is no encapsulation within the following macros, do not change + ## the running order or otherwise move them around unless you know exactly + ## what you are doing... + _LT_COMPILER_PIC($1) + _LT_COMPILER_C_O($1) + _LT_COMPILER_FILE_LOCKS($1) + _LT_LINKER_SHLIBS($1) + _LT_SYS_DYNAMIC_LINKER($1) + _LT_LINKER_HARDCODE_LIBPATH($1) + + _LT_CONFIG($1) + fi # test -n "$compiler" + + GCC=$lt_save_GCC + CC="$lt_save_CC" + CFLAGS="$lt_save_CFLAGS" +fi # test "$_lt_disable_F77" != yes + +AC_LANG_POP +])# _LT_LANG_F77_CONFIG + + +# _LT_LANG_FC_CONFIG([TAG]) +# ------------------------- +# Ensure that the configuration variables for a Fortran compiler are +# suitably defined. These variables are subsequently used by _LT_CONFIG +# to write the compiler configuration to `libtool'. +m4_defun([_LT_LANG_FC_CONFIG], +[AC_LANG_PUSH(Fortran) + +if test -z "$FC" || test "X$FC" = "Xno"; then + _lt_disable_FC=yes +fi + +_LT_TAGVAR(archive_cmds_need_lc, $1)=no +_LT_TAGVAR(allow_undefined_flag, $1)= +_LT_TAGVAR(always_export_symbols, $1)=no +_LT_TAGVAR(archive_expsym_cmds, $1)= +_LT_TAGVAR(export_dynamic_flag_spec, $1)= +_LT_TAGVAR(hardcode_direct, $1)=no +_LT_TAGVAR(hardcode_direct_absolute, $1)=no +_LT_TAGVAR(hardcode_libdir_flag_spec, $1)= +_LT_TAGVAR(hardcode_libdir_separator, $1)= +_LT_TAGVAR(hardcode_minus_L, $1)=no +_LT_TAGVAR(hardcode_automatic, $1)=no +_LT_TAGVAR(inherit_rpath, $1)=no +_LT_TAGVAR(module_cmds, $1)= +_LT_TAGVAR(module_expsym_cmds, $1)= +_LT_TAGVAR(link_all_deplibs, $1)=unknown +_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds +_LT_TAGVAR(reload_flag, $1)=$reload_flag +_LT_TAGVAR(reload_cmds, $1)=$reload_cmds +_LT_TAGVAR(no_undefined_flag, $1)= +_LT_TAGVAR(whole_archive_flag_spec, $1)= +_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no + +# Source file extension for fc test sources. +ac_ext=${ac_fc_srcext-f} + +# Object file extension for compiled fc test sources. +objext=o +_LT_TAGVAR(objext, $1)=$objext + +# No sense in running all these tests if we already determined that +# the FC compiler isn't working. Some variables (like enable_shared) +# are currently assumed to apply to all compilers on this platform, +# and will be corrupted by setting them based on a non-working compiler. +if test "$_lt_disable_FC" != yes; then + # Code to be used in simple compile tests + lt_simple_compile_test_code="\ + subroutine t + return + end +" + + # Code to be used in simple link tests + lt_simple_link_test_code="\ + program t + end +" + + # ltmain only uses $CC for tagged configurations so make sure $CC is set. + _LT_TAG_COMPILER + + # save warnings/boilerplate of simple test code + _LT_COMPILER_BOILERPLATE + _LT_LINKER_BOILERPLATE + + # Allow CC to be a program name with arguments. + lt_save_CC="$CC" + lt_save_GCC=$GCC + lt_save_CFLAGS=$CFLAGS + CC=${FC-"f95"} + CFLAGS=$FCFLAGS + compiler=$CC + GCC=$ac_cv_fc_compiler_gnu + + _LT_TAGVAR(compiler, $1)=$CC + _LT_CC_BASENAME([$compiler]) + + if test -n "$compiler"; then + AC_MSG_CHECKING([if libtool supports shared libraries]) + AC_MSG_RESULT([$can_build_shared]) + + AC_MSG_CHECKING([whether to build shared libraries]) + test "$can_build_shared" = "no" && enable_shared=no + + # On AIX, shared libraries and static libraries use the same namespace, and + # are all built from PIC. + case $host_os in + aix3*) + test "$enable_shared" = yes && enable_static=no + if test -n "$RANLIB"; then + archive_cmds="$archive_cmds~\$RANLIB \$lib" + postinstall_cmds='$RANLIB $lib' + fi + ;; + aix[[4-9]]*) + if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then + test "$enable_shared" = yes && enable_static=no + fi + ;; + esac + AC_MSG_RESULT([$enable_shared]) + + AC_MSG_CHECKING([whether to build static libraries]) + # Make sure either enable_shared or enable_static is yes. + test "$enable_shared" = yes || enable_static=yes + AC_MSG_RESULT([$enable_static]) + + _LT_TAGVAR(GCC, $1)="$ac_cv_fc_compiler_gnu" + _LT_TAGVAR(LD, $1)="$LD" + + ## CAVEAT EMPTOR: + ## There is no encapsulation within the following macros, do not change + ## the running order or otherwise move them around unless you know exactly + ## what you are doing... + _LT_SYS_HIDDEN_LIBDEPS($1) + _LT_COMPILER_PIC($1) + _LT_COMPILER_C_O($1) + _LT_COMPILER_FILE_LOCKS($1) + _LT_LINKER_SHLIBS($1) + _LT_SYS_DYNAMIC_LINKER($1) + _LT_LINKER_HARDCODE_LIBPATH($1) + + _LT_CONFIG($1) + fi # test -n "$compiler" + + GCC=$lt_save_GCC + CC=$lt_save_CC + CFLAGS=$lt_save_CFLAGS +fi # test "$_lt_disable_FC" != yes + +AC_LANG_POP +])# _LT_LANG_FC_CONFIG + + +# _LT_LANG_GCJ_CONFIG([TAG]) +# -------------------------- +# Ensure that the configuration variables for the GNU Java Compiler compiler +# are suitably defined. These variables are subsequently used by _LT_CONFIG +# to write the compiler configuration to `libtool'. +m4_defun([_LT_LANG_GCJ_CONFIG], +[AC_REQUIRE([LT_PROG_GCJ])dnl +AC_LANG_SAVE + +# Source file extension for Java test sources. +ac_ext=java + +# Object file extension for compiled Java test sources. +objext=o +_LT_TAGVAR(objext, $1)=$objext + +# Code to be used in simple compile tests +lt_simple_compile_test_code="class foo {}" + +# Code to be used in simple link tests +lt_simple_link_test_code='public class conftest { public static void main(String[[]] argv) {}; }' + +# ltmain only uses $CC for tagged configurations so make sure $CC is set. +_LT_TAG_COMPILER + +# save warnings/boilerplate of simple test code +_LT_COMPILER_BOILERPLATE +_LT_LINKER_BOILERPLATE + +# Allow CC to be a program name with arguments. +lt_save_CC=$CC +lt_save_CFLAGS=$CFLAGS +lt_save_GCC=$GCC +GCC=yes +CC=${GCJ-"gcj"} +CFLAGS=$GCJFLAGS +compiler=$CC +_LT_TAGVAR(compiler, $1)=$CC +_LT_TAGVAR(LD, $1)="$LD" +_LT_CC_BASENAME([$compiler]) + +# GCJ did not exist at the time GCC didn't implicitly link libc in. +_LT_TAGVAR(archive_cmds_need_lc, $1)=no + +_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds +_LT_TAGVAR(reload_flag, $1)=$reload_flag +_LT_TAGVAR(reload_cmds, $1)=$reload_cmds + +## CAVEAT EMPTOR: +## There is no encapsulation within the following macros, do not change +## the running order or otherwise move them around unless you know exactly +## what you are doing... +if test -n "$compiler"; then + _LT_COMPILER_NO_RTTI($1) + _LT_COMPILER_PIC($1) + _LT_COMPILER_C_O($1) + _LT_COMPILER_FILE_LOCKS($1) + _LT_LINKER_SHLIBS($1) + _LT_LINKER_HARDCODE_LIBPATH($1) + + _LT_CONFIG($1) +fi + +AC_LANG_RESTORE + +GCC=$lt_save_GCC +CC=$lt_save_CC +CFLAGS=$lt_save_CFLAGS +])# _LT_LANG_GCJ_CONFIG + + +# _LT_LANG_GO_CONFIG([TAG]) +# -------------------------- +# Ensure that the configuration variables for the GNU Go compiler +# are suitably defined. These variables are subsequently used by _LT_CONFIG +# to write the compiler configuration to `libtool'. +m4_defun([_LT_LANG_GO_CONFIG], +[AC_REQUIRE([LT_PROG_GO])dnl +AC_LANG_SAVE + +# Source file extension for Go test sources. +ac_ext=go + +# Object file extension for compiled Go test sources. +objext=o +_LT_TAGVAR(objext, $1)=$objext + +# Code to be used in simple compile tests +lt_simple_compile_test_code="package main; func main() { }" + +# Code to be used in simple link tests +lt_simple_link_test_code='package main; func main() { }' + +# ltmain only uses $CC for tagged configurations so make sure $CC is set. +_LT_TAG_COMPILER + +# save warnings/boilerplate of simple test code +_LT_COMPILER_BOILERPLATE +_LT_LINKER_BOILERPLATE + +# Allow CC to be a program name with arguments. +lt_save_CC=$CC +lt_save_CFLAGS=$CFLAGS +lt_save_GCC=$GCC +GCC=yes +CC=${GOC-"gccgo"} +CFLAGS=$GOFLAGS +compiler=$CC +_LT_TAGVAR(compiler, $1)=$CC +_LT_TAGVAR(LD, $1)="$LD" +_LT_CC_BASENAME([$compiler]) + +# Go did not exist at the time GCC didn't implicitly link libc in. +_LT_TAGVAR(archive_cmds_need_lc, $1)=no + +_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds +_LT_TAGVAR(reload_flag, $1)=$reload_flag +_LT_TAGVAR(reload_cmds, $1)=$reload_cmds + +## CAVEAT EMPTOR: +## There is no encapsulation within the following macros, do not change +## the running order or otherwise move them around unless you know exactly +## what you are doing... +if test -n "$compiler"; then + _LT_COMPILER_NO_RTTI($1) + _LT_COMPILER_PIC($1) + _LT_COMPILER_C_O($1) + _LT_COMPILER_FILE_LOCKS($1) + _LT_LINKER_SHLIBS($1) + _LT_LINKER_HARDCODE_LIBPATH($1) + + _LT_CONFIG($1) +fi + +AC_LANG_RESTORE + +GCC=$lt_save_GCC +CC=$lt_save_CC +CFLAGS=$lt_save_CFLAGS +])# _LT_LANG_GO_CONFIG + + +# _LT_LANG_RC_CONFIG([TAG]) +# ------------------------- +# Ensure that the configuration variables for the Windows resource compiler +# are suitably defined. These variables are subsequently used by _LT_CONFIG +# to write the compiler configuration to `libtool'. +m4_defun([_LT_LANG_RC_CONFIG], +[AC_REQUIRE([LT_PROG_RC])dnl +AC_LANG_SAVE + +# Source file extension for RC test sources. +ac_ext=rc + +# Object file extension for compiled RC test sources. +objext=o +_LT_TAGVAR(objext, $1)=$objext + +# Code to be used in simple compile tests +lt_simple_compile_test_code='sample MENU { MENUITEM "&Soup", 100, CHECKED }' + +# Code to be used in simple link tests +lt_simple_link_test_code="$lt_simple_compile_test_code" + +# ltmain only uses $CC for tagged configurations so make sure $CC is set. +_LT_TAG_COMPILER + +# save warnings/boilerplate of simple test code +_LT_COMPILER_BOILERPLATE +_LT_LINKER_BOILERPLATE + +# Allow CC to be a program name with arguments. +lt_save_CC="$CC" +lt_save_CFLAGS=$CFLAGS +lt_save_GCC=$GCC +GCC= +CC=${RC-"windres"} +CFLAGS= +compiler=$CC +_LT_TAGVAR(compiler, $1)=$CC +_LT_CC_BASENAME([$compiler]) +_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=yes + +if test -n "$compiler"; then + : + _LT_CONFIG($1) +fi + +GCC=$lt_save_GCC +AC_LANG_RESTORE +CC=$lt_save_CC +CFLAGS=$lt_save_CFLAGS +])# _LT_LANG_RC_CONFIG + + +# LT_PROG_GCJ +# ----------- +AC_DEFUN([LT_PROG_GCJ], +[m4_ifdef([AC_PROG_GCJ], [AC_PROG_GCJ], + [m4_ifdef([A][M_PROG_GCJ], [A][M_PROG_GCJ], + [AC_CHECK_TOOL(GCJ, gcj,) + test "x${GCJFLAGS+set}" = xset || GCJFLAGS="-g -O2" + AC_SUBST(GCJFLAGS)])])[]dnl +]) + +# Old name: +AU_ALIAS([LT_AC_PROG_GCJ], [LT_PROG_GCJ]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([LT_AC_PROG_GCJ], []) + + +# LT_PROG_GO +# ---------- +AC_DEFUN([LT_PROG_GO], +[AC_CHECK_TOOL(GOC, gccgo,) +]) + + +# LT_PROG_RC +# ---------- +AC_DEFUN([LT_PROG_RC], +[AC_CHECK_TOOL(RC, windres,) +]) + +# Old name: +AU_ALIAS([LT_AC_PROG_RC], [LT_PROG_RC]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([LT_AC_PROG_RC], []) + + +# _LT_DECL_EGREP +# -------------- +# If we don't have a new enough Autoconf to choose the best grep +# available, choose the one first in the user's PATH. +m4_defun([_LT_DECL_EGREP], +[AC_REQUIRE([AC_PROG_EGREP])dnl +AC_REQUIRE([AC_PROG_FGREP])dnl +test -z "$GREP" && GREP=grep +_LT_DECL([], [GREP], [1], [A grep program that handles long lines]) +_LT_DECL([], [EGREP], [1], [An ERE matcher]) +_LT_DECL([], [FGREP], [1], [A literal string matcher]) +dnl Non-bleeding-edge autoconf doesn't subst GREP, so do it here too +AC_SUBST([GREP]) +]) + + +# _LT_DECL_OBJDUMP +# -------------- +# If we don't have a new enough Autoconf to choose the best objdump +# available, choose the one first in the user's PATH. +m4_defun([_LT_DECL_OBJDUMP], +[AC_CHECK_TOOL(OBJDUMP, objdump, false) +test -z "$OBJDUMP" && OBJDUMP=objdump +_LT_DECL([], [OBJDUMP], [1], [An object symbol dumper]) +AC_SUBST([OBJDUMP]) +]) + +# _LT_DECL_DLLTOOL +# ---------------- +# Ensure DLLTOOL variable is set. +m4_defun([_LT_DECL_DLLTOOL], +[AC_CHECK_TOOL(DLLTOOL, dlltool, false) +test -z "$DLLTOOL" && DLLTOOL=dlltool +_LT_DECL([], [DLLTOOL], [1], [DLL creation program]) +AC_SUBST([DLLTOOL]) +]) + +# _LT_DECL_SED +# ------------ +# Check for a fully-functional sed program, that truncates +# as few characters as possible. Prefer GNU sed if found. +m4_defun([_LT_DECL_SED], +[AC_PROG_SED +test -z "$SED" && SED=sed +Xsed="$SED -e 1s/^X//" +_LT_DECL([], [SED], [1], [A sed program that does not truncate output]) +_LT_DECL([], [Xsed], ["\$SED -e 1s/^X//"], + [Sed that helps us avoid accidentally triggering echo(1) options like -n]) +])# _LT_DECL_SED + +m4_ifndef([AC_PROG_SED], [ +############################################################ +# NOTE: This macro has been submitted for inclusion into # +# GNU Autoconf as AC_PROG_SED. When it is available in # +# a released version of Autoconf we should remove this # +# macro and use it instead. # +############################################################ + +m4_defun([AC_PROG_SED], +[AC_MSG_CHECKING([for a sed that does not truncate output]) +AC_CACHE_VAL(lt_cv_path_SED, +[# Loop through the user's path and test for sed and gsed. +# Then use that list of sed's as ones to test for truncation. +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for lt_ac_prog in sed gsed; do + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$lt_ac_prog$ac_exec_ext"; then + lt_ac_sed_list="$lt_ac_sed_list $as_dir/$lt_ac_prog$ac_exec_ext" + fi + done + done +done +IFS=$as_save_IFS +lt_ac_max=0 +lt_ac_count=0 +# Add /usr/xpg4/bin/sed as it is typically found on Solaris +# along with /bin/sed that truncates output. +for lt_ac_sed in $lt_ac_sed_list /usr/xpg4/bin/sed; do + test ! -f $lt_ac_sed && continue + cat /dev/null > conftest.in + lt_ac_count=0 + echo $ECHO_N "0123456789$ECHO_C" >conftest.in + # Check for GNU sed and select it if it is found. + if "$lt_ac_sed" --version 2>&1 < /dev/null | grep 'GNU' > /dev/null; then + lt_cv_path_SED=$lt_ac_sed + break + fi + while true; do + cat conftest.in conftest.in >conftest.tmp + mv conftest.tmp conftest.in + cp conftest.in conftest.nl + echo >>conftest.nl + $lt_ac_sed -e 's/a$//' < conftest.nl >conftest.out || break + cmp -s conftest.out conftest.nl || break + # 10000 chars as input seems more than enough + test $lt_ac_count -gt 10 && break + lt_ac_count=`expr $lt_ac_count + 1` + if test $lt_ac_count -gt $lt_ac_max; then + lt_ac_max=$lt_ac_count + lt_cv_path_SED=$lt_ac_sed + fi + done +done +]) +SED=$lt_cv_path_SED +AC_SUBST([SED]) +AC_MSG_RESULT([$SED]) +])#AC_PROG_SED +])#m4_ifndef + +# Old name: +AU_ALIAS([LT_AC_PROG_SED], [AC_PROG_SED]) +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([LT_AC_PROG_SED], []) + + +# _LT_CHECK_SHELL_FEATURES +# ------------------------ +# Find out whether the shell is Bourne or XSI compatible, +# or has some other useful features. +m4_defun([_LT_CHECK_SHELL_FEATURES], +[AC_MSG_CHECKING([whether the shell understands some XSI constructs]) +# Try some XSI features +xsi_shell=no +( _lt_dummy="a/b/c" + test "${_lt_dummy##*/},${_lt_dummy%/*},${_lt_dummy#??}"${_lt_dummy%"$_lt_dummy"}, \ + = c,a/b,b/c, \ + && eval 'test $(( 1 + 1 )) -eq 2 \ + && test "${#_lt_dummy}" -eq 5' ) >/dev/null 2>&1 \ + && xsi_shell=yes +AC_MSG_RESULT([$xsi_shell]) +_LT_CONFIG_LIBTOOL_INIT([xsi_shell='$xsi_shell']) + +AC_MSG_CHECKING([whether the shell understands "+="]) +lt_shell_append=no +( foo=bar; set foo baz; eval "$[1]+=\$[2]" && test "$foo" = barbaz ) \ + >/dev/null 2>&1 \ + && lt_shell_append=yes +AC_MSG_RESULT([$lt_shell_append]) +_LT_CONFIG_LIBTOOL_INIT([lt_shell_append='$lt_shell_append']) + +if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then + lt_unset=unset +else + lt_unset=false +fi +_LT_DECL([], [lt_unset], [0], [whether the shell understands "unset"])dnl + +# test EBCDIC or ASCII +case `echo X|tr X '\101'` in + A) # ASCII based system + # \n is not interpreted correctly by Solaris 8 /usr/ucb/tr + lt_SP2NL='tr \040 \012' + lt_NL2SP='tr \015\012 \040\040' + ;; + *) # EBCDIC based system + lt_SP2NL='tr \100 \n' + lt_NL2SP='tr \r\n \100\100' + ;; +esac +_LT_DECL([SP2NL], [lt_SP2NL], [1], [turn spaces into newlines])dnl +_LT_DECL([NL2SP], [lt_NL2SP], [1], [turn newlines into spaces])dnl +])# _LT_CHECK_SHELL_FEATURES + + +# _LT_PROG_FUNCTION_REPLACE (FUNCNAME, REPLACEMENT-BODY) +# ------------------------------------------------------ +# In `$cfgfile', look for function FUNCNAME delimited by `^FUNCNAME ()$' and +# '^} FUNCNAME ', and replace its body with REPLACEMENT-BODY. +m4_defun([_LT_PROG_FUNCTION_REPLACE], +[dnl { +sed -e '/^$1 ()$/,/^} # $1 /c\ +$1 ()\ +{\ +m4_bpatsubsts([$2], [$], [\\], [^\([ ]\)], [\\\1]) +} # Extended-shell $1 implementation' "$cfgfile" > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") +test 0 -eq $? || _lt_function_replace_fail=: +]) + + +# _LT_PROG_REPLACE_SHELLFNS +# ------------------------- +# Replace existing portable implementations of several shell functions with +# equivalent extended shell implementations where those features are available.. +m4_defun([_LT_PROG_REPLACE_SHELLFNS], +[if test x"$xsi_shell" = xyes; then + _LT_PROG_FUNCTION_REPLACE([func_dirname], [dnl + case ${1} in + */*) func_dirname_result="${1%/*}${2}" ;; + * ) func_dirname_result="${3}" ;; + esac]) + + _LT_PROG_FUNCTION_REPLACE([func_basename], [dnl + func_basename_result="${1##*/}"]) + + _LT_PROG_FUNCTION_REPLACE([func_dirname_and_basename], [dnl + case ${1} in + */*) func_dirname_result="${1%/*}${2}" ;; + * ) func_dirname_result="${3}" ;; + esac + func_basename_result="${1##*/}"]) + + _LT_PROG_FUNCTION_REPLACE([func_stripname], [dnl + # pdksh 5.2.14 does not do ${X%$Y} correctly if both X and Y are + # positional parameters, so assign one to ordinary parameter first. + func_stripname_result=${3} + func_stripname_result=${func_stripname_result#"${1}"} + func_stripname_result=${func_stripname_result%"${2}"}]) + + _LT_PROG_FUNCTION_REPLACE([func_split_long_opt], [dnl + func_split_long_opt_name=${1%%=*} + func_split_long_opt_arg=${1#*=}]) + + _LT_PROG_FUNCTION_REPLACE([func_split_short_opt], [dnl + func_split_short_opt_arg=${1#??} + func_split_short_opt_name=${1%"$func_split_short_opt_arg"}]) + + _LT_PROG_FUNCTION_REPLACE([func_lo2o], [dnl + case ${1} in + *.lo) func_lo2o_result=${1%.lo}.${objext} ;; + *) func_lo2o_result=${1} ;; + esac]) + + _LT_PROG_FUNCTION_REPLACE([func_xform], [ func_xform_result=${1%.*}.lo]) + + _LT_PROG_FUNCTION_REPLACE([func_arith], [ func_arith_result=$(( $[*] ))]) + + _LT_PROG_FUNCTION_REPLACE([func_len], [ func_len_result=${#1}]) +fi + +if test x"$lt_shell_append" = xyes; then + _LT_PROG_FUNCTION_REPLACE([func_append], [ eval "${1}+=\\${2}"]) + + _LT_PROG_FUNCTION_REPLACE([func_append_quoted], [dnl + func_quote_for_eval "${2}" +dnl m4 expansion turns \\\\ into \\, and then the shell eval turns that into \ + eval "${1}+=\\\\ \\$func_quote_for_eval_result"]) + + # Save a `func_append' function call where possible by direct use of '+=' + sed -e 's%func_append \([[a-zA-Z_]]\{1,\}\) "%\1+="%g' $cfgfile > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") + test 0 -eq $? || _lt_function_replace_fail=: +else + # Save a `func_append' function call even when '+=' is not available + sed -e 's%func_append \([[a-zA-Z_]]\{1,\}\) "%\1="$\1%g' $cfgfile > $cfgfile.tmp \ + && mv -f "$cfgfile.tmp" "$cfgfile" \ + || (rm -f "$cfgfile" && cp "$cfgfile.tmp" "$cfgfile" && rm -f "$cfgfile.tmp") + test 0 -eq $? || _lt_function_replace_fail=: +fi + +if test x"$_lt_function_replace_fail" = x":"; then + AC_MSG_WARN([Unable to substitute extended shell functions in $ofile]) +fi +]) + +# _LT_PATH_CONVERSION_FUNCTIONS +# ----------------------------- +# Determine which file name conversion functions should be used by +# func_to_host_file (and, implicitly, by func_to_host_path). These are needed +# for certain cross-compile configurations and native mingw. +m4_defun([_LT_PATH_CONVERSION_FUNCTIONS], +[AC_REQUIRE([AC_CANONICAL_HOST])dnl +AC_REQUIRE([AC_CANONICAL_BUILD])dnl +AC_MSG_CHECKING([how to convert $build file names to $host format]) +AC_CACHE_VAL(lt_cv_to_host_file_cmd, +[case $host in + *-*-mingw* ) + case $build in + *-*-mingw* ) # actually msys + lt_cv_to_host_file_cmd=func_convert_file_msys_to_w32 + ;; + *-*-cygwin* ) + lt_cv_to_host_file_cmd=func_convert_file_cygwin_to_w32 + ;; + * ) # otherwise, assume *nix + lt_cv_to_host_file_cmd=func_convert_file_nix_to_w32 + ;; + esac + ;; + *-*-cygwin* ) + case $build in + *-*-mingw* ) # actually msys + lt_cv_to_host_file_cmd=func_convert_file_msys_to_cygwin + ;; + *-*-cygwin* ) + lt_cv_to_host_file_cmd=func_convert_file_noop + ;; + * ) # otherwise, assume *nix + lt_cv_to_host_file_cmd=func_convert_file_nix_to_cygwin + ;; + esac + ;; + * ) # unhandled hosts (and "normal" native builds) + lt_cv_to_host_file_cmd=func_convert_file_noop + ;; +esac +]) +to_host_file_cmd=$lt_cv_to_host_file_cmd +AC_MSG_RESULT([$lt_cv_to_host_file_cmd]) +_LT_DECL([to_host_file_cmd], [lt_cv_to_host_file_cmd], + [0], [convert $build file names to $host format])dnl + +AC_MSG_CHECKING([how to convert $build file names to toolchain format]) +AC_CACHE_VAL(lt_cv_to_tool_file_cmd, +[#assume ordinary cross tools, or native build. +lt_cv_to_tool_file_cmd=func_convert_file_noop +case $host in + *-*-mingw* ) + case $build in + *-*-mingw* ) # actually msys + lt_cv_to_tool_file_cmd=func_convert_file_msys_to_w32 + ;; + esac + ;; +esac +]) +to_tool_file_cmd=$lt_cv_to_tool_file_cmd +AC_MSG_RESULT([$lt_cv_to_tool_file_cmd]) +_LT_DECL([to_tool_file_cmd], [lt_cv_to_tool_file_cmd], + [0], [convert $build files to toolchain format])dnl +])# _LT_PATH_CONVERSION_FUNCTIONS diff --git a/ceph/m4/ltoptions.m4 b/ceph/m4/ltoptions.m4 new file mode 100644 index 00000000..5d9acd8e --- /dev/null +++ b/ceph/m4/ltoptions.m4 @@ -0,0 +1,384 @@ +# Helper functions for option handling. -*- Autoconf -*- +# +# Copyright (C) 2004, 2005, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# Written by Gary V. Vaughan, 2004 +# +# This file is free software; the Free Software Foundation gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. + +# serial 7 ltoptions.m4 + +# This is to help aclocal find these macros, as it can't see m4_define. +AC_DEFUN([LTOPTIONS_VERSION], [m4_if([1])]) + + +# _LT_MANGLE_OPTION(MACRO-NAME, OPTION-NAME) +# ------------------------------------------ +m4_define([_LT_MANGLE_OPTION], +[[_LT_OPTION_]m4_bpatsubst($1__$2, [[^a-zA-Z0-9_]], [_])]) + + +# _LT_SET_OPTION(MACRO-NAME, OPTION-NAME) +# --------------------------------------- +# Set option OPTION-NAME for macro MACRO-NAME, and if there is a +# matching handler defined, dispatch to it. Other OPTION-NAMEs are +# saved as a flag. +m4_define([_LT_SET_OPTION], +[m4_define(_LT_MANGLE_OPTION([$1], [$2]))dnl +m4_ifdef(_LT_MANGLE_DEFUN([$1], [$2]), + _LT_MANGLE_DEFUN([$1], [$2]), + [m4_warning([Unknown $1 option `$2'])])[]dnl +]) + + +# _LT_IF_OPTION(MACRO-NAME, OPTION-NAME, IF-SET, [IF-NOT-SET]) +# ------------------------------------------------------------ +# Execute IF-SET if OPTION is set, IF-NOT-SET otherwise. +m4_define([_LT_IF_OPTION], +[m4_ifdef(_LT_MANGLE_OPTION([$1], [$2]), [$3], [$4])]) + + +# _LT_UNLESS_OPTIONS(MACRO-NAME, OPTION-LIST, IF-NOT-SET) +# ------------------------------------------------------- +# Execute IF-NOT-SET unless all options in OPTION-LIST for MACRO-NAME +# are set. +m4_define([_LT_UNLESS_OPTIONS], +[m4_foreach([_LT_Option], m4_split(m4_normalize([$2])), + [m4_ifdef(_LT_MANGLE_OPTION([$1], _LT_Option), + [m4_define([$0_found])])])[]dnl +m4_ifdef([$0_found], [m4_undefine([$0_found])], [$3 +])[]dnl +]) + + +# _LT_SET_OPTIONS(MACRO-NAME, OPTION-LIST) +# ---------------------------------------- +# OPTION-LIST is a space-separated list of Libtool options associated +# with MACRO-NAME. If any OPTION has a matching handler declared with +# LT_OPTION_DEFINE, dispatch to that macro; otherwise complain about +# the unknown option and exit. +m4_defun([_LT_SET_OPTIONS], +[# Set options +m4_foreach([_LT_Option], m4_split(m4_normalize([$2])), + [_LT_SET_OPTION([$1], _LT_Option)]) + +m4_if([$1],[LT_INIT],[ + dnl + dnl Simply set some default values (i.e off) if boolean options were not + dnl specified: + _LT_UNLESS_OPTIONS([LT_INIT], [dlopen], [enable_dlopen=no + ]) + _LT_UNLESS_OPTIONS([LT_INIT], [win32-dll], [enable_win32_dll=no + ]) + dnl + dnl If no reference was made to various pairs of opposing options, then + dnl we run the default mode handler for the pair. For example, if neither + dnl `shared' nor `disable-shared' was passed, we enable building of shared + dnl archives by default: + _LT_UNLESS_OPTIONS([LT_INIT], [shared disable-shared], [_LT_ENABLE_SHARED]) + _LT_UNLESS_OPTIONS([LT_INIT], [static disable-static], [_LT_ENABLE_STATIC]) + _LT_UNLESS_OPTIONS([LT_INIT], [pic-only no-pic], [_LT_WITH_PIC]) + _LT_UNLESS_OPTIONS([LT_INIT], [fast-install disable-fast-install], + [_LT_ENABLE_FAST_INSTALL]) + ]) +])# _LT_SET_OPTIONS + + +## --------------------------------- ## +## Macros to handle LT_INIT options. ## +## --------------------------------- ## + +# _LT_MANGLE_DEFUN(MACRO-NAME, OPTION-NAME) +# ----------------------------------------- +m4_define([_LT_MANGLE_DEFUN], +[[_LT_OPTION_DEFUN_]m4_bpatsubst(m4_toupper([$1__$2]), [[^A-Z0-9_]], [_])]) + + +# LT_OPTION_DEFINE(MACRO-NAME, OPTION-NAME, CODE) +# ----------------------------------------------- +m4_define([LT_OPTION_DEFINE], +[m4_define(_LT_MANGLE_DEFUN([$1], [$2]), [$3])[]dnl +])# LT_OPTION_DEFINE + + +# dlopen +# ------ +LT_OPTION_DEFINE([LT_INIT], [dlopen], [enable_dlopen=yes +]) + +AU_DEFUN([AC_LIBTOOL_DLOPEN], +[_LT_SET_OPTION([LT_INIT], [dlopen]) +AC_DIAGNOSE([obsolete], +[$0: Remove this warning and the call to _LT_SET_OPTION when you +put the `dlopen' option into LT_INIT's first parameter.]) +]) + +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_DLOPEN], []) + + +# win32-dll +# --------- +# Declare package support for building win32 dll's. +LT_OPTION_DEFINE([LT_INIT], [win32-dll], +[enable_win32_dll=yes + +case $host in +*-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-cegcc*) + AC_CHECK_TOOL(AS, as, false) + AC_CHECK_TOOL(DLLTOOL, dlltool, false) + AC_CHECK_TOOL(OBJDUMP, objdump, false) + ;; +esac + +test -z "$AS" && AS=as +_LT_DECL([], [AS], [1], [Assembler program])dnl + +test -z "$DLLTOOL" && DLLTOOL=dlltool +_LT_DECL([], [DLLTOOL], [1], [DLL creation program])dnl + +test -z "$OBJDUMP" && OBJDUMP=objdump +_LT_DECL([], [OBJDUMP], [1], [Object dumper program])dnl +])# win32-dll + +AU_DEFUN([AC_LIBTOOL_WIN32_DLL], +[AC_REQUIRE([AC_CANONICAL_HOST])dnl +_LT_SET_OPTION([LT_INIT], [win32-dll]) +AC_DIAGNOSE([obsolete], +[$0: Remove this warning and the call to _LT_SET_OPTION when you +put the `win32-dll' option into LT_INIT's first parameter.]) +]) + +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_WIN32_DLL], []) + + +# _LT_ENABLE_SHARED([DEFAULT]) +# ---------------------------- +# implement the --enable-shared flag, and supports the `shared' and +# `disable-shared' LT_INIT options. +# DEFAULT is either `yes' or `no'. If omitted, it defaults to `yes'. +m4_define([_LT_ENABLE_SHARED], +[m4_define([_LT_ENABLE_SHARED_DEFAULT], [m4_if($1, no, no, yes)])dnl +AC_ARG_ENABLE([shared], + [AS_HELP_STRING([--enable-shared@<:@=PKGS@:>@], + [build shared libraries @<:@default=]_LT_ENABLE_SHARED_DEFAULT[@:>@])], + [p=${PACKAGE-default} + case $enableval in + yes) enable_shared=yes ;; + no) enable_shared=no ;; + *) + enable_shared=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for pkg in $enableval; do + IFS="$lt_save_ifs" + if test "X$pkg" = "X$p"; then + enable_shared=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac], + [enable_shared=]_LT_ENABLE_SHARED_DEFAULT) + + _LT_DECL([build_libtool_libs], [enable_shared], [0], + [Whether or not to build shared libraries]) +])# _LT_ENABLE_SHARED + +LT_OPTION_DEFINE([LT_INIT], [shared], [_LT_ENABLE_SHARED([yes])]) +LT_OPTION_DEFINE([LT_INIT], [disable-shared], [_LT_ENABLE_SHARED([no])]) + +# Old names: +AC_DEFUN([AC_ENABLE_SHARED], +[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[shared]) +]) + +AC_DEFUN([AC_DISABLE_SHARED], +[_LT_SET_OPTION([LT_INIT], [disable-shared]) +]) + +AU_DEFUN([AM_ENABLE_SHARED], [AC_ENABLE_SHARED($@)]) +AU_DEFUN([AM_DISABLE_SHARED], [AC_DISABLE_SHARED($@)]) + +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AM_ENABLE_SHARED], []) +dnl AC_DEFUN([AM_DISABLE_SHARED], []) + + + +# _LT_ENABLE_STATIC([DEFAULT]) +# ---------------------------- +# implement the --enable-static flag, and support the `static' and +# `disable-static' LT_INIT options. +# DEFAULT is either `yes' or `no'. If omitted, it defaults to `yes'. +m4_define([_LT_ENABLE_STATIC], +[m4_define([_LT_ENABLE_STATIC_DEFAULT], [m4_if($1, no, no, yes)])dnl +AC_ARG_ENABLE([static], + [AS_HELP_STRING([--enable-static@<:@=PKGS@:>@], + [build static libraries @<:@default=]_LT_ENABLE_STATIC_DEFAULT[@:>@])], + [p=${PACKAGE-default} + case $enableval in + yes) enable_static=yes ;; + no) enable_static=no ;; + *) + enable_static=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for pkg in $enableval; do + IFS="$lt_save_ifs" + if test "X$pkg" = "X$p"; then + enable_static=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac], + [enable_static=]_LT_ENABLE_STATIC_DEFAULT) + + _LT_DECL([build_old_libs], [enable_static], [0], + [Whether or not to build static libraries]) +])# _LT_ENABLE_STATIC + +LT_OPTION_DEFINE([LT_INIT], [static], [_LT_ENABLE_STATIC([yes])]) +LT_OPTION_DEFINE([LT_INIT], [disable-static], [_LT_ENABLE_STATIC([no])]) + +# Old names: +AC_DEFUN([AC_ENABLE_STATIC], +[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[static]) +]) + +AC_DEFUN([AC_DISABLE_STATIC], +[_LT_SET_OPTION([LT_INIT], [disable-static]) +]) + +AU_DEFUN([AM_ENABLE_STATIC], [AC_ENABLE_STATIC($@)]) +AU_DEFUN([AM_DISABLE_STATIC], [AC_DISABLE_STATIC($@)]) + +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AM_ENABLE_STATIC], []) +dnl AC_DEFUN([AM_DISABLE_STATIC], []) + + + +# _LT_ENABLE_FAST_INSTALL([DEFAULT]) +# ---------------------------------- +# implement the --enable-fast-install flag, and support the `fast-install' +# and `disable-fast-install' LT_INIT options. +# DEFAULT is either `yes' or `no'. If omitted, it defaults to `yes'. +m4_define([_LT_ENABLE_FAST_INSTALL], +[m4_define([_LT_ENABLE_FAST_INSTALL_DEFAULT], [m4_if($1, no, no, yes)])dnl +AC_ARG_ENABLE([fast-install], + [AS_HELP_STRING([--enable-fast-install@<:@=PKGS@:>@], + [optimize for fast installation @<:@default=]_LT_ENABLE_FAST_INSTALL_DEFAULT[@:>@])], + [p=${PACKAGE-default} + case $enableval in + yes) enable_fast_install=yes ;; + no) enable_fast_install=no ;; + *) + enable_fast_install=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for pkg in $enableval; do + IFS="$lt_save_ifs" + if test "X$pkg" = "X$p"; then + enable_fast_install=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac], + [enable_fast_install=]_LT_ENABLE_FAST_INSTALL_DEFAULT) + +_LT_DECL([fast_install], [enable_fast_install], [0], + [Whether or not to optimize for fast installation])dnl +])# _LT_ENABLE_FAST_INSTALL + +LT_OPTION_DEFINE([LT_INIT], [fast-install], [_LT_ENABLE_FAST_INSTALL([yes])]) +LT_OPTION_DEFINE([LT_INIT], [disable-fast-install], [_LT_ENABLE_FAST_INSTALL([no])]) + +# Old names: +AU_DEFUN([AC_ENABLE_FAST_INSTALL], +[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[fast-install]) +AC_DIAGNOSE([obsolete], +[$0: Remove this warning and the call to _LT_SET_OPTION when you put +the `fast-install' option into LT_INIT's first parameter.]) +]) + +AU_DEFUN([AC_DISABLE_FAST_INSTALL], +[_LT_SET_OPTION([LT_INIT], [disable-fast-install]) +AC_DIAGNOSE([obsolete], +[$0: Remove this warning and the call to _LT_SET_OPTION when you put +the `disable-fast-install' option into LT_INIT's first parameter.]) +]) + +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_ENABLE_FAST_INSTALL], []) +dnl AC_DEFUN([AM_DISABLE_FAST_INSTALL], []) + + +# _LT_WITH_PIC([MODE]) +# -------------------- +# implement the --with-pic flag, and support the `pic-only' and `no-pic' +# LT_INIT options. +# MODE is either `yes' or `no'. If omitted, it defaults to `both'. +m4_define([_LT_WITH_PIC], +[AC_ARG_WITH([pic], + [AS_HELP_STRING([--with-pic@<:@=PKGS@:>@], + [try to use only PIC/non-PIC objects @<:@default=use both@:>@])], + [lt_p=${PACKAGE-default} + case $withval in + yes|no) pic_mode=$withval ;; + *) + pic_mode=default + # Look at the argument we got. We use all the common list separators. + lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR," + for lt_pkg in $withval; do + IFS="$lt_save_ifs" + if test "X$lt_pkg" = "X$lt_p"; then + pic_mode=yes + fi + done + IFS="$lt_save_ifs" + ;; + esac], + [pic_mode=default]) + +test -z "$pic_mode" && pic_mode=m4_default([$1], [default]) + +_LT_DECL([], [pic_mode], [0], [What type of objects to build])dnl +])# _LT_WITH_PIC + +LT_OPTION_DEFINE([LT_INIT], [pic-only], [_LT_WITH_PIC([yes])]) +LT_OPTION_DEFINE([LT_INIT], [no-pic], [_LT_WITH_PIC([no])]) + +# Old name: +AU_DEFUN([AC_LIBTOOL_PICMODE], +[_LT_SET_OPTION([LT_INIT], [pic-only]) +AC_DIAGNOSE([obsolete], +[$0: Remove this warning and the call to _LT_SET_OPTION when you +put the `pic-only' option into LT_INIT's first parameter.]) +]) + +dnl aclocal-1.4 backwards compatibility: +dnl AC_DEFUN([AC_LIBTOOL_PICMODE], []) + +## ----------------- ## +## LTDL_INIT Options ## +## ----------------- ## + +m4_define([_LTDL_MODE], []) +LT_OPTION_DEFINE([LTDL_INIT], [nonrecursive], + [m4_define([_LTDL_MODE], [nonrecursive])]) +LT_OPTION_DEFINE([LTDL_INIT], [recursive], + [m4_define([_LTDL_MODE], [recursive])]) +LT_OPTION_DEFINE([LTDL_INIT], [subproject], + [m4_define([_LTDL_MODE], [subproject])]) + +m4_define([_LTDL_TYPE], []) +LT_OPTION_DEFINE([LTDL_INIT], [installable], + [m4_define([_LTDL_TYPE], [installable])]) +LT_OPTION_DEFINE([LTDL_INIT], [convenience], + [m4_define([_LTDL_TYPE], [convenience])]) diff --git a/ceph/m4/ltsugar.m4 b/ceph/m4/ltsugar.m4 new file mode 100644 index 00000000..9000a057 --- /dev/null +++ b/ceph/m4/ltsugar.m4 @@ -0,0 +1,123 @@ +# ltsugar.m4 -- libtool m4 base layer. -*-Autoconf-*- +# +# Copyright (C) 2004, 2005, 2007, 2008 Free Software Foundation, Inc. +# Written by Gary V. Vaughan, 2004 +# +# This file is free software; the Free Software Foundation gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. + +# serial 6 ltsugar.m4 + +# This is to help aclocal find these macros, as it can't see m4_define. +AC_DEFUN([LTSUGAR_VERSION], [m4_if([0.1])]) + + +# lt_join(SEP, ARG1, [ARG2...]) +# ----------------------------- +# Produce ARG1SEPARG2...SEPARGn, omitting [] arguments and their +# associated separator. +# Needed until we can rely on m4_join from Autoconf 2.62, since all earlier +# versions in m4sugar had bugs. +m4_define([lt_join], +[m4_if([$#], [1], [], + [$#], [2], [[$2]], + [m4_if([$2], [], [], [[$2]_])$0([$1], m4_shift(m4_shift($@)))])]) +m4_define([_lt_join], +[m4_if([$#$2], [2], [], + [m4_if([$2], [], [], [[$1$2]])$0([$1], m4_shift(m4_shift($@)))])]) + + +# lt_car(LIST) +# lt_cdr(LIST) +# ------------ +# Manipulate m4 lists. +# These macros are necessary as long as will still need to support +# Autoconf-2.59 which quotes differently. +m4_define([lt_car], [[$1]]) +m4_define([lt_cdr], +[m4_if([$#], 0, [m4_fatal([$0: cannot be called without arguments])], + [$#], 1, [], + [m4_dquote(m4_shift($@))])]) +m4_define([lt_unquote], $1) + + +# lt_append(MACRO-NAME, STRING, [SEPARATOR]) +# ------------------------------------------ +# Redefine MACRO-NAME to hold its former content plus `SEPARATOR'`STRING'. +# Note that neither SEPARATOR nor STRING are expanded; they are appended +# to MACRO-NAME as is (leaving the expansion for when MACRO-NAME is invoked). +# No SEPARATOR is output if MACRO-NAME was previously undefined (different +# than defined and empty). +# +# This macro is needed until we can rely on Autoconf 2.62, since earlier +# versions of m4sugar mistakenly expanded SEPARATOR but not STRING. +m4_define([lt_append], +[m4_define([$1], + m4_ifdef([$1], [m4_defn([$1])[$3]])[$2])]) + + + +# lt_combine(SEP, PREFIX-LIST, INFIX, SUFFIX1, [SUFFIX2...]) +# ---------------------------------------------------------- +# Produce a SEP delimited list of all paired combinations of elements of +# PREFIX-LIST with SUFFIX1 through SUFFIXn. Each element of the list +# has the form PREFIXmINFIXSUFFIXn. +# Needed until we can rely on m4_combine added in Autoconf 2.62. +m4_define([lt_combine], +[m4_if(m4_eval([$# > 3]), [1], + [m4_pushdef([_Lt_sep], [m4_define([_Lt_sep], m4_defn([lt_car]))])]]dnl +[[m4_foreach([_Lt_prefix], [$2], + [m4_foreach([_Lt_suffix], + ]m4_dquote(m4_dquote(m4_shift(m4_shift(m4_shift($@)))))[, + [_Lt_sep([$1])[]m4_defn([_Lt_prefix])[$3]m4_defn([_Lt_suffix])])])])]) + + +# lt_if_append_uniq(MACRO-NAME, VARNAME, [SEPARATOR], [UNIQ], [NOT-UNIQ]) +# ----------------------------------------------------------------------- +# Iff MACRO-NAME does not yet contain VARNAME, then append it (delimited +# by SEPARATOR if supplied) and expand UNIQ, else NOT-UNIQ. +m4_define([lt_if_append_uniq], +[m4_ifdef([$1], + [m4_if(m4_index([$3]m4_defn([$1])[$3], [$3$2$3]), [-1], + [lt_append([$1], [$2], [$3])$4], + [$5])], + [lt_append([$1], [$2], [$3])$4])]) + + +# lt_dict_add(DICT, KEY, VALUE) +# ----------------------------- +m4_define([lt_dict_add], +[m4_define([$1($2)], [$3])]) + + +# lt_dict_add_subkey(DICT, KEY, SUBKEY, VALUE) +# -------------------------------------------- +m4_define([lt_dict_add_subkey], +[m4_define([$1($2:$3)], [$4])]) + + +# lt_dict_fetch(DICT, KEY, [SUBKEY]) +# ---------------------------------- +m4_define([lt_dict_fetch], +[m4_ifval([$3], + m4_ifdef([$1($2:$3)], [m4_defn([$1($2:$3)])]), + m4_ifdef([$1($2)], [m4_defn([$1($2)])]))]) + + +# lt_if_dict_fetch(DICT, KEY, [SUBKEY], VALUE, IF-TRUE, [IF-FALSE]) +# ----------------------------------------------------------------- +m4_define([lt_if_dict_fetch], +[m4_if(lt_dict_fetch([$1], [$2], [$3]), [$4], + [$5], + [$6])]) + + +# lt_dict_filter(DICT, [SUBKEY], VALUE, [SEPARATOR], KEY, [...]) +# -------------------------------------------------------------- +m4_define([lt_dict_filter], +[m4_if([$5], [], [], + [lt_join(m4_quote(m4_default([$4], [[, ]])), + lt_unquote(m4_split(m4_normalize(m4_foreach(_Lt_key, lt_car([m4_shiftn(4, $@)]), + [lt_if_dict_fetch([$1], _Lt_key, [$2], [$3], [_Lt_key ])])))))])[]dnl +]) diff --git a/ceph/m4/ltversion.m4 b/ceph/m4/ltversion.m4 new file mode 100644 index 00000000..07a8602d --- /dev/null +++ b/ceph/m4/ltversion.m4 @@ -0,0 +1,23 @@ +# ltversion.m4 -- version numbers -*- Autoconf -*- +# +# Copyright (C) 2004 Free Software Foundation, Inc. +# Written by Scott James Remnant, 2004 +# +# This file is free software; the Free Software Foundation gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. + +# @configure_input@ + +# serial 3337 ltversion.m4 +# This file is part of GNU Libtool + +m4_define([LT_PACKAGE_VERSION], [2.4.2]) +m4_define([LT_PACKAGE_REVISION], [1.3337]) + +AC_DEFUN([LTVERSION_VERSION], +[macro_version='2.4.2' +macro_revision='1.3337' +_LT_DECL(, macro_version, 0, [Which release of libtool.m4 was used?]) +_LT_DECL(, macro_revision, 0) +]) diff --git a/ceph/m4/lt~obsolete.m4 b/ceph/m4/lt~obsolete.m4 new file mode 100644 index 00000000..c573da90 --- /dev/null +++ b/ceph/m4/lt~obsolete.m4 @@ -0,0 +1,98 @@ +# lt~obsolete.m4 -- aclocal satisfying obsolete definitions. -*-Autoconf-*- +# +# Copyright (C) 2004, 2005, 2007, 2009 Free Software Foundation, Inc. +# Written by Scott James Remnant, 2004. +# +# This file is free software; the Free Software Foundation gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. + +# serial 5 lt~obsolete.m4 + +# These exist entirely to fool aclocal when bootstrapping libtool. +# +# In the past libtool.m4 has provided macros via AC_DEFUN (or AU_DEFUN) +# which have later been changed to m4_define as they aren't part of the +# exported API, or moved to Autoconf or Automake where they belong. +# +# The trouble is, aclocal is a bit thick. It'll see the old AC_DEFUN +# in /usr/share/aclocal/libtool.m4 and remember it, then when it sees us +# using a macro with the same name in our local m4/libtool.m4 it'll +# pull the old libtool.m4 in (it doesn't see our shiny new m4_define +# and doesn't know about Autoconf macros at all.) +# +# So we provide this file, which has a silly filename so it's always +# included after everything else. This provides aclocal with the +# AC_DEFUNs it wants, but when m4 processes it, it doesn't do anything +# because those macros already exist, or will be overwritten later. +# We use AC_DEFUN over AU_DEFUN for compatibility with aclocal-1.6. +# +# Anytime we withdraw an AC_DEFUN or AU_DEFUN, remember to add it here. +# Yes, that means every name once taken will need to remain here until +# we give up compatibility with versions before 1.7, at which point +# we need to keep only those names which we still refer to. + +# This is to help aclocal find these macros, as it can't see m4_define. +AC_DEFUN([LTOBSOLETE_VERSION], [m4_if([1])]) + +m4_ifndef([AC_LIBTOOL_LINKER_OPTION], [AC_DEFUN([AC_LIBTOOL_LINKER_OPTION])]) +m4_ifndef([AC_PROG_EGREP], [AC_DEFUN([AC_PROG_EGREP])]) +m4_ifndef([_LT_AC_PROG_ECHO_BACKSLASH], [AC_DEFUN([_LT_AC_PROG_ECHO_BACKSLASH])]) +m4_ifndef([_LT_AC_SHELL_INIT], [AC_DEFUN([_LT_AC_SHELL_INIT])]) +m4_ifndef([_LT_AC_SYS_LIBPATH_AIX], [AC_DEFUN([_LT_AC_SYS_LIBPATH_AIX])]) +m4_ifndef([_LT_PROG_LTMAIN], [AC_DEFUN([_LT_PROG_LTMAIN])]) +m4_ifndef([_LT_AC_TAGVAR], [AC_DEFUN([_LT_AC_TAGVAR])]) +m4_ifndef([AC_LTDL_ENABLE_INSTALL], [AC_DEFUN([AC_LTDL_ENABLE_INSTALL])]) +m4_ifndef([AC_LTDL_PREOPEN], [AC_DEFUN([AC_LTDL_PREOPEN])]) +m4_ifndef([_LT_AC_SYS_COMPILER], [AC_DEFUN([_LT_AC_SYS_COMPILER])]) +m4_ifndef([_LT_AC_LOCK], [AC_DEFUN([_LT_AC_LOCK])]) +m4_ifndef([AC_LIBTOOL_SYS_OLD_ARCHIVE], [AC_DEFUN([AC_LIBTOOL_SYS_OLD_ARCHIVE])]) +m4_ifndef([_LT_AC_TRY_DLOPEN_SELF], [AC_DEFUN([_LT_AC_TRY_DLOPEN_SELF])]) +m4_ifndef([AC_LIBTOOL_PROG_CC_C_O], [AC_DEFUN([AC_LIBTOOL_PROG_CC_C_O])]) +m4_ifndef([AC_LIBTOOL_SYS_HARD_LINK_LOCKS], [AC_DEFUN([AC_LIBTOOL_SYS_HARD_LINK_LOCKS])]) +m4_ifndef([AC_LIBTOOL_OBJDIR], [AC_DEFUN([AC_LIBTOOL_OBJDIR])]) +m4_ifndef([AC_LTDL_OBJDIR], [AC_DEFUN([AC_LTDL_OBJDIR])]) +m4_ifndef([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH], [AC_DEFUN([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH])]) +m4_ifndef([AC_LIBTOOL_SYS_LIB_STRIP], [AC_DEFUN([AC_LIBTOOL_SYS_LIB_STRIP])]) +m4_ifndef([AC_PATH_MAGIC], [AC_DEFUN([AC_PATH_MAGIC])]) +m4_ifndef([AC_PROG_LD_GNU], [AC_DEFUN([AC_PROG_LD_GNU])]) +m4_ifndef([AC_PROG_LD_RELOAD_FLAG], [AC_DEFUN([AC_PROG_LD_RELOAD_FLAG])]) +m4_ifndef([AC_DEPLIBS_CHECK_METHOD], [AC_DEFUN([AC_DEPLIBS_CHECK_METHOD])]) +m4_ifndef([AC_LIBTOOL_PROG_COMPILER_NO_RTTI], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_NO_RTTI])]) +m4_ifndef([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE], [AC_DEFUN([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE])]) +m4_ifndef([AC_LIBTOOL_PROG_COMPILER_PIC], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_PIC])]) +m4_ifndef([AC_LIBTOOL_PROG_LD_SHLIBS], [AC_DEFUN([AC_LIBTOOL_PROG_LD_SHLIBS])]) +m4_ifndef([AC_LIBTOOL_POSTDEP_PREDEP], [AC_DEFUN([AC_LIBTOOL_POSTDEP_PREDEP])]) +m4_ifndef([LT_AC_PROG_EGREP], [AC_DEFUN([LT_AC_PROG_EGREP])]) +m4_ifndef([LT_AC_PROG_SED], [AC_DEFUN([LT_AC_PROG_SED])]) +m4_ifndef([_LT_CC_BASENAME], [AC_DEFUN([_LT_CC_BASENAME])]) +m4_ifndef([_LT_COMPILER_BOILERPLATE], [AC_DEFUN([_LT_COMPILER_BOILERPLATE])]) +m4_ifndef([_LT_LINKER_BOILERPLATE], [AC_DEFUN([_LT_LINKER_BOILERPLATE])]) +m4_ifndef([_AC_PROG_LIBTOOL], [AC_DEFUN([_AC_PROG_LIBTOOL])]) +m4_ifndef([AC_LIBTOOL_SETUP], [AC_DEFUN([AC_LIBTOOL_SETUP])]) +m4_ifndef([_LT_AC_CHECK_DLFCN], [AC_DEFUN([_LT_AC_CHECK_DLFCN])]) +m4_ifndef([AC_LIBTOOL_SYS_DYNAMIC_LINKER], [AC_DEFUN([AC_LIBTOOL_SYS_DYNAMIC_LINKER])]) +m4_ifndef([_LT_AC_TAGCONFIG], [AC_DEFUN([_LT_AC_TAGCONFIG])]) +m4_ifndef([AC_DISABLE_FAST_INSTALL], [AC_DEFUN([AC_DISABLE_FAST_INSTALL])]) +m4_ifndef([_LT_AC_LANG_CXX], [AC_DEFUN([_LT_AC_LANG_CXX])]) +m4_ifndef([_LT_AC_LANG_F77], [AC_DEFUN([_LT_AC_LANG_F77])]) +m4_ifndef([_LT_AC_LANG_GCJ], [AC_DEFUN([_LT_AC_LANG_GCJ])]) +m4_ifndef([AC_LIBTOOL_LANG_C_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_C_CONFIG])]) +m4_ifndef([_LT_AC_LANG_C_CONFIG], [AC_DEFUN([_LT_AC_LANG_C_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_CXX_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_CXX_CONFIG])]) +m4_ifndef([_LT_AC_LANG_CXX_CONFIG], [AC_DEFUN([_LT_AC_LANG_CXX_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_F77_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_F77_CONFIG])]) +m4_ifndef([_LT_AC_LANG_F77_CONFIG], [AC_DEFUN([_LT_AC_LANG_F77_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_GCJ_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_GCJ_CONFIG])]) +m4_ifndef([_LT_AC_LANG_GCJ_CONFIG], [AC_DEFUN([_LT_AC_LANG_GCJ_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_RC_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_RC_CONFIG])]) +m4_ifndef([_LT_AC_LANG_RC_CONFIG], [AC_DEFUN([_LT_AC_LANG_RC_CONFIG])]) +m4_ifndef([AC_LIBTOOL_CONFIG], [AC_DEFUN([AC_LIBTOOL_CONFIG])]) +m4_ifndef([_LT_AC_FILE_LTDLL_C], [AC_DEFUN([_LT_AC_FILE_LTDLL_C])]) +m4_ifndef([_LT_REQUIRED_DARWIN_CHECKS], [AC_DEFUN([_LT_REQUIRED_DARWIN_CHECKS])]) +m4_ifndef([_LT_AC_PROG_CXXCPP], [AC_DEFUN([_LT_AC_PROG_CXXCPP])]) +m4_ifndef([_LT_PREPARE_SED_QUOTE_VARS], [AC_DEFUN([_LT_PREPARE_SED_QUOTE_VARS])]) +m4_ifndef([_LT_PROG_ECHO_BACKSLASH], [AC_DEFUN([_LT_PROG_ECHO_BACKSLASH])]) +m4_ifndef([_LT_PROG_F77], [AC_DEFUN([_LT_PROG_F77])]) +m4_ifndef([_LT_PROG_FC], [AC_DEFUN([_LT_PROG_FC])]) +m4_ifndef([_LT_PROG_CXX], [AC_DEFUN([_LT_PROG_CXX])]) diff --git a/ceph/m4/pkg.m4 b/ceph/m4/pkg.m4 new file mode 100644 index 00000000..9a71878c --- /dev/null +++ b/ceph/m4/pkg.m4 @@ -0,0 +1,159 @@ +# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +# serial 1 (pkg-config-0.24) +# +# Copyright © 2004 Scott James Remnant . +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# PKG_PROG_PKG_CONFIG([MIN-VERSION]) +# ---------------------------------- +AC_DEFUN([PKG_PROG_PKG_CONFIG], +[m4_pattern_forbid([^_?PKG_[A-Z_]+$]) +m4_pattern_allow([^PKG_CONFIG(_(PATH|LIBDIR|SYSROOT_DIR|ALLOW_SYSTEM_(CFLAGS|LIBS)))?$]) +m4_pattern_allow([^PKG_CONFIG_(DISABLE_UNINSTALLED|TOP_BUILD_DIR|DEBUG_SPEW)$]) +AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility]) +AC_ARG_VAR([PKG_CONFIG_PATH], [directories to add to pkg-config's search path]) +AC_ARG_VAR([PKG_CONFIG_LIBDIR], [path overriding pkg-config's built-in search path]) + +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + AC_PATH_TOOL([PKG_CONFIG], [pkg-config]) +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=m4_default([$1], [0.9.0]) + AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version]) + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + PKG_CONFIG="" + fi +fi[]dnl +])# PKG_PROG_PKG_CONFIG + +# PKG_CHECK_EXISTS(MODULES, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +# +# Check to see whether a particular set of modules exists. Similar +# to PKG_CHECK_MODULES(), but does not set variables or print errors. +# +# Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +# only at the first occurence in configure.ac, so if the first place +# it's called might be skipped (such as if it is within an "if", you +# have to call PKG_CHECK_EXISTS manually +# -------------------------------------------------------------- +AC_DEFUN([PKG_CHECK_EXISTS], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +if test -n "$PKG_CONFIG" && \ + AC_RUN_LOG([$PKG_CONFIG --exists --print-errors "$1"]); then + m4_default([$2], [:]) +m4_ifvaln([$3], [else + $3])dnl +fi]) + +# _PKG_CONFIG([VARIABLE], [COMMAND], [MODULES]) +# --------------------------------------------- +m4_define([_PKG_CONFIG], +[if test -n "$$1"; then + pkg_cv_[]$1="$$1" + elif test -n "$PKG_CONFIG"; then + PKG_CHECK_EXISTS([$3], + [pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes ], + [pkg_failed=yes]) + else + pkg_failed=untried +fi[]dnl +])# _PKG_CONFIG + +# _PKG_SHORT_ERRORS_SUPPORTED +# ----------------------------- +AC_DEFUN([_PKG_SHORT_ERRORS_SUPPORTED], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi[]dnl +])# _PKG_SHORT_ERRORS_SUPPORTED + + +# PKG_CHECK_MODULES(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], +# [ACTION-IF-NOT-FOUND]) +# +# +# Note that if there is a possibility the first call to +# PKG_CHECK_MODULES might not happen, you should be sure to include an +# explicit call to PKG_PROG_PKG_CONFIG in your configure.ac +# +# +# -------------------------------------------------------------- +AC_DEFUN([PKG_CHECK_MODULES], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl +AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl + +pkg_failed=no +AC_MSG_CHECKING([for $1]) + +_PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) +_PKG_CONFIG([$1][_LIBS], [libs], [$2]) + +m4_define([_PKG_TEXT], [Alternatively, you may set the environment variables $1[]_CFLAGS +and $1[]_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details.]) + +if test $pkg_failed = yes; then + AC_MSG_RESULT([no]) + _PKG_SHORT_ERRORS_SUPPORTED + if test $_pkg_short_errors_supported = yes; then + $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` + else + $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD + + m4_default([$4], [AC_MSG_ERROR( +[Package requirements ($2) were not met: + +$$1_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +_PKG_TEXT])[]dnl + ]) +elif test $pkg_failed = untried; then + AC_MSG_RESULT([no]) + m4_default([$4], [AC_MSG_FAILURE( +[The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +_PKG_TEXT + +To get pkg-config, see .])[]dnl + ]) +else + $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS + $1[]_LIBS=$pkg_cv_[]$1[]_LIBS + AC_MSG_RESULT([yes]) + $3 +fi[]dnl +])# PKG_CHECK_MODULES diff --git a/ceph/man/Makefile.am b/ceph/man/Makefile.am new file mode 100644 index 00000000..be071b17 --- /dev/null +++ b/ceph/man/Makefile.am @@ -0,0 +1,30 @@ +AUTOMAKE_OPTIONS = gnu + +dist_man_MANS = \ + ceph-osd.8 \ + ceph-mds.8 \ + ceph-mon.8 \ + mkcephfs.8 \ + ceph-fuse.8 \ + ceph-syn.8 \ + crushtool.8 \ + osdmaptool.8 \ + monmaptool.8 \ + ceph-conf.8 \ + ceph-run.8 \ + ceph.8 \ + mount.ceph.8 \ + radosgw.8 \ + radosgw-admin.8 \ + ceph-authtool.8 \ + rados.8 \ + librados-config.8 \ + rbd.8 \ + ceph-clsinfo.8 \ + ceph-debugpack.8 \ + cephfs.8 \ + ceph-dencoder.8 \ + ceph-rest-api.8 \ + ceph-rbdnamer.8 \ + ceph-post-file.8 \ + rbd-fuse.8 diff --git a/ceph/man/Makefile.in b/ceph/man/Makefile.in new file mode 100644 index 00000000..e5732312 --- /dev/null +++ b/ceph/man/Makefile.in @@ -0,0 +1,555 @@ +# Makefile.in generated by automake 1.11.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software +# Foundation, Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +subdir = man +DIST_COMMON = $(dist_man_MANS) $(srcdir)/Makefile.am \ + $(srcdir)/Makefile.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \ + $(top_srcdir)/m4/ac_prog_jar.m4 \ + $(top_srcdir)/m4/ac_prog_javac.m4 \ + $(top_srcdir)/m4/ac_prog_javac_works.m4 \ + $(top_srcdir)/m4/ac_prog_javah.m4 \ + $(top_srcdir)/m4/acx_pthread.m4 \ + $(top_srcdir)/m4/ax_c_pretty_func.m4 \ + $(top_srcdir)/m4/ax_c_var_func.m4 \ + $(top_srcdir)/m4/ax_check_compile_flag.m4 \ + $(top_srcdir)/m4/ax_cxx_static_cast.m4 \ + $(top_srcdir)/m4/ax_intel.m4 $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/pkg.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/src/acconfig.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +SOURCES = +DIST_SOURCES = +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +man8dir = $(mandir)/man8 +am__installdirs = "$(DESTDIR)$(man8dir)" +NROFF = nroff +MANS = $(dist_man_MANS) +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_CXXFLAGS = @AM_CXXFLAGS@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@ +CC = @CC@ +CCAS = @CCAS@ +CCASDEPMODE = @CCASDEPMODE@ +CCASFLAGS = @CCASFLAGS@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPTOPP_CFLAGS = @CRYPTOPP_CFLAGS@ +CRYPTOPP_LIBS = @CRYPTOPP_LIBS@ +CRYPTO_CFLAGS = @CRYPTO_CFLAGS@ +CRYPTO_LIBS = @CRYPTO_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +EXTRA_CLASSPATH_JAR = @EXTRA_CLASSPATH_JAR@ +FGREP = @FGREP@ +GCOV_PREFIX_STRIP = @GCOV_PREFIX_STRIP@ +GIT_CHECK = @GIT_CHECK@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +INTEL_FLAGS = @INTEL_FLAGS@ +INTEL_PCLMUL_FLAGS = @INTEL_PCLMUL_FLAGS@ +INTEL_SSE2_FLAGS = @INTEL_SSE2_FLAGS@ +INTEL_SSE3_FLAGS = @INTEL_SSE3_FLAGS@ +INTEL_SSE4_1_FLAGS = @INTEL_SSE4_1_FLAGS@ +INTEL_SSE4_2_FLAGS = @INTEL_SSE4_2_FLAGS@ +INTEL_SSE_FLAGS = @INTEL_SSE_FLAGS@ +INTEL_SSSE3_FLAGS = @INTEL_SSSE3_FLAGS@ +JAR = @JAR@ +JAVAC = @JAVAC@ +JAVAH = @JAVAH@ +JDK_CPPFLAGS = @JDK_CPPFLAGS@ +KEYUTILS_LIB = @KEYUTILS_LIB@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@ +LIBEDIT_LIBS = @LIBEDIT_LIBS@ +LIBFUSE = @LIBFUSE@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTCMALLOC = @LIBTCMALLOC@ +LIBTOOL = @LIBTOOL@ +LIBZFS_CFLAGS = @LIBZFS_CFLAGS@ +LIBZFS_LIBS = @LIBZFS_LIBS@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NSS_CFLAGS = @NSS_CFLAGS@ +NSS_LIBS = @NSS_LIBS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +PTHREAD_CC = @PTHREAD_CC@ +PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ +PTHREAD_LIBS = @PTHREAD_LIBS@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +RESOLV_LIBS = @RESOLV_LIBS@ +RPM_RELEASE = @RPM_RELEASE@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +WARN_IGNORED_QUALIFIERS = @WARN_IGNORED_QUALIFIERS@ +WARN_TYPE_LIMITS = @WARN_TYPE_LIMITS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +acx_pthread_config = @acx_pthread_config@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +subdirs = @subdirs@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +AUTOMAKE_OPTIONS = gnu +dist_man_MANS = \ + ceph-osd.8 \ + ceph-mds.8 \ + ceph-mon.8 \ + mkcephfs.8 \ + ceph-fuse.8 \ + ceph-syn.8 \ + crushtool.8 \ + osdmaptool.8 \ + monmaptool.8 \ + ceph-conf.8 \ + ceph-run.8 \ + ceph.8 \ + mount.ceph.8 \ + radosgw.8 \ + radosgw-admin.8 \ + ceph-authtool.8 \ + rados.8 \ + librados-config.8 \ + rbd.8 \ + ceph-clsinfo.8 \ + ceph-debugpack.8 \ + cephfs.8 \ + ceph-dencoder.8 \ + ceph-rest-api.8 \ + ceph-rbdnamer.8 \ + ceph-post-file.8 \ + rbd-fuse.8 + +all: all-am + +.SUFFIXES: +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu man/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu man/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-man8: $(dist_man_MANS) + @$(NORMAL_INSTALL) + test -z "$(man8dir)" || $(MKDIR_P) "$(DESTDIR)$(man8dir)" + @list=''; test -n "$(man8dir)" || exit 0; \ + { for i in $$list; do echo "$$i"; done; \ + l2='$(dist_man_MANS)'; for i in $$l2; do echo "$$i"; done | \ + sed -n '/\.8[a-z]*$$/p'; \ + } | while read p; do \ + if test -f $$p; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; echo "$$p"; \ + done | \ + sed -e 'n;s,.*/,,;p;h;s,.*\.,,;s,^[^8][0-9a-z]*$$,8,;x' \ + -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,' | \ + sed 'N;N;s,\n, ,g' | { \ + list=; while read file base inst; do \ + if test "$$base" = "$$inst"; then list="$$list $$file"; else \ + echo " $(INSTALL_DATA) '$$file' '$(DESTDIR)$(man8dir)/$$inst'"; \ + $(INSTALL_DATA) "$$file" "$(DESTDIR)$(man8dir)/$$inst" || exit $$?; \ + fi; \ + done; \ + for i in $$list; do echo "$$i"; done | $(am__base_list) | \ + while read files; do \ + test -z "$$files" || { \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(man8dir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(man8dir)" || exit $$?; }; \ + done; } + +uninstall-man8: + @$(NORMAL_UNINSTALL) + @list=''; test -n "$(man8dir)" || exit 0; \ + files=`{ for i in $$list; do echo "$$i"; done; \ + l2='$(dist_man_MANS)'; for i in $$l2; do echo "$$i"; done | \ + sed -n '/\.8[a-z]*$$/p'; \ + } | sed -e 's,.*/,,;h;s,.*\.,,;s,^[^8][0-9a-z]*$$,8,;x' \ + -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,'`; \ + dir='$(DESTDIR)$(man8dir)'; $(am__uninstall_files_from_dir) +tags: TAGS +TAGS: + +ctags: CTAGS +CTAGS: + + +distdir: $(DISTFILES) + @list='$(MANS)'; if test -n "$$list"; then \ + list=`for p in $$list; do \ + if test -f $$p; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; else :; fi; done`; \ + if test -n "$$list" && \ + grep 'ab help2man is required to generate this page' $$list >/dev/null; then \ + echo "error: found man pages containing the \`missing help2man' replacement text:" >&2; \ + grep -l 'ab help2man is required to generate this page' $$list | sed 's/^/ /' >&2; \ + echo " to fix them, install help2man, remove and regenerate the man pages;" >&2; \ + echo " typically \`make maintainer-clean' will remove them" >&2; \ + exit 1; \ + else :; fi; \ + else :; fi + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(MANS) +installdirs: + for dir in "$(DESTDIR)$(man8dir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool mostlyclean-am + +distclean: distclean-am + -rm -f Makefile +distclean-am: clean-am distclean-generic + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-man + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: install-man8 + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-generic mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-man + +uninstall-man: uninstall-man8 + +.MAKE: install-am install-strip + +.PHONY: all all-am check check-am clean clean-generic clean-libtool \ + distclean distclean-generic distclean-libtool distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-man8 \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + uninstall uninstall-am uninstall-man uninstall-man8 + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/ceph/man/ceph-authtool.8 b/ceph/man/ceph-authtool.8 new file mode 100644 index 00000000..aae41d38 --- /dev/null +++ b/ceph/man/ceph-authtool.8 @@ -0,0 +1,299 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-AUTHTOOL" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-authtool \- ceph keyring manipulation tool +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-authtool\fP \fIkeyringfile\fP [ \-l | \-\-list ] [ \-C | \-\-create\-keyring +] [ \-p | \-\-print ] [ \-n | \-\-name \fIentityname\fP ] [ \-\-gen\-key ] [ \-a | +\-\-add\-key \fIbase64_key\fP ] [ \-\-caps \fIcapfile\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-authtool\fP is a utility to create, view, and modify a Ceph keyring +file. A keyring file stores one or more Ceph authentication keys and +possibly an associated capability specification. Each key is +associated with an entity name, of the form +\fB{client,mon,mds,osd}.name\fP\&. +.sp +\fBWARNING\fP Ceph provides authentication and protection against +man\-in\-the\-middle attacks once secret keys are in place. However, +data over the wire is not encrypted, which may include the messages +used to configure said keys. The system is primarily intended to be +used in trusted environments. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-l, \-\-list +will list all keys and capabilities present in the keyring +.UNINDENT +.INDENT 0.0 +.TP +.B \-p, \-\-print +will print an encoded key for the specified entityname. This is +suitable for the \fBmount \-o secret=\fP argument +.UNINDENT +.INDENT 0.0 +.TP +.B \-C, \-\-create\-keyring +will create a new keyring, overwriting any existing keyringfile +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-gen\-key +will generate a new secret key for the specified entityname +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-add\-key +will add an encoded key to the keyring +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-cap subsystem capability +will set the capability for given subsystem +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-caps capsfile +will set all of capabilities associated with a given key, for all subsystems +.UNINDENT +.SH CAPABILITIES +.sp +The subsystem is the name of a Ceph subsystem: \fBmon\fP, \fBmds\fP, or +\fBosd\fP\&. +.sp +The capability is a string describing what the given user is allowed +to do. This takes the form of a comma separated list of allow +clauses with a permission specifier containing one or more of rwx for +read, write, and execute permission. The \fBallow *\fP grants full +superuser permissions for the given subsystem. +.sp +For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# can read, write, and execute objects +osd = "allow rwx" + +# can access mds server +mds = "allow" + +# can modify cluster state (i.e., is a server daemon) +mon = "allow rwx" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +A librados user restricted to a single pool might look like: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mon = "allow r" + +osd = "allow rw pool foo" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +A client using rbd with read access to one pool and read/write access to another: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mon = "allow r" + +osd = "allow class\-read object_prefix rbd_children, allow pool templates r class\-read, allow pool vms rwx" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +A client mounting the file system with minimal permissions would need caps like: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mds = "allow" + +osd = "allow rw pool data" + +mon = "allow r" +.ft P +.fi +.UNINDENT +.UNINDENT +.SH OSD CAPABILITIES +.sp +In general, an osd capability follows the grammar: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +osdcap := grant[,grant...] +grant := allow (match capspec | capspec match) +match := [pool[=] | object_prefix ] +capspec := * | [r][w][x] [class\-read] [class\-write] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The capspec determines what kind of operations the entity can perform: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +r = read access to objects +w = write access to objects +x = can call any class method (same as class\-read class\-write) +class\-read = can call class methods that are reads +class\-write = can call class methods that are writes +* = equivalent to rwx, plus the ability to run osd admin commands, + i.e. ceph osd tell ... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The match criteria restrict a grant based on the pool being accessed. +Grants are additive if the client fulfills the match condition. For +example, if a client has the osd capabilities: "allow r object_prefix +prefix, allow w pool foo, allow x pool bar", then it has rw access to +pool foo, rx access to pool bar, and r access to objects whose +names begin with \(aqprefix\(aq in any pool. +.SH CAPS FILE FORMAT +.sp +The caps file format consists of zero or more key/value pairs, one per +line. The key and value are separated by an \fB=\fP, and the value must +be quoted (with \fB\(aq\fP or \fB"\fP) if it contains any whitespace. The key +is the name of the Ceph subsystem (\fBosd\fP, \fBmds\fP, \fBmon\fP), and the +value is the capability string (see above). +.SH EXAMPLE +.sp +To create a new keyring containing a key for client.foo: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-authtool \-C \-n client.foo \-\-gen\-key keyring +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To associate some capabilities with the key (namely, the ability to +mount a Ceph filesystem): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-authtool \-n client.foo \-\-cap mds \(aqallow\(aq \-\-cap osd \(aqallow rw pool=data\(aq \-\-cap mon \(aqallow r\(aq keyring +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To display the contents of the keyring: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-authtool \-l keyring +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +When mount a Ceph file system, you can grab the appropriately encoded secret key with: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount \-t ceph serverhost:/ mountpoint \-o name=foo,secret=\(gaceph\-authtool \-p \-n client.foo keyring\(ga +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-authtool\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-clsinfo.8 b/ceph/man/ceph-clsinfo.8 new file mode 100644 index 00000000..4fb82748 --- /dev/null +++ b/ceph/man/ceph-clsinfo.8 @@ -0,0 +1,96 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-CLSINFO" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-clsinfo \- show class object information +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-clsinfo\fP [ \fIoptions\fP ] ... \fIfilename\fP +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-clsinfo\fP can show name, version, and architecture information +about a specific class object. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-n, \-\-name +Shows the class name +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-version +Shows the class version +.UNINDENT +.INDENT 0.0 +.TP +.B \-a, \-\-arch +Shows the class architecture +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-clsinfo\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-conf.8 b/ceph/man/ceph-conf.8 new file mode 100644 index 00000000..754f7fc9 --- /dev/null +++ b/ceph/man/ceph-conf.8 @@ -0,0 +1,159 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-CONF" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-conf \- ceph conf file tool +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-conf\fP \-c \fIconffile\fP \-\-list\-all\-sections +\fBceph\-conf\fP \-c \fIconffile\fP \-L +\fBceph\-conf\fP \-c \fIconffile\fP \-l \fIprefix\fP +\fBceph\-conf\fP \fIkey\fP \-s \fIsection1\fP ... +\fBceph\-conf\fP [\-s \fIsection\fP ] \-\-lookup \fIkey\fP +\fBceph\-conf\fP [\-s \fIsection\fP ] \fIkey\fP +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-conf\fP is a utility for getting information about a ceph +configuration file. As with most Ceph programs, you can specify which +Ceph configuration file to use with the \fB\-c\fP flag. +.SH ACTIONS +.sp +\fBceph\-conf\fP will perform one of the following actions: +.sp +\-\-list\-all\-sections or \-L prints out a list of all the section names in the configuration +file. +.sp +\-\-list\-sections or \-l prints out a list of all the sections that begin +with a given prefix. For example, \-\-list\-sections mon would list all +sections beginning with mon. +.sp +\-\-lookup will search the configuration for a given value. By default, the sections that +are searched are determined by the Ceph name that we are using. The Ceph name defaults to +client.admin. It can be specified with \-\-name. +.sp +For example, if we specify \-\-name osd.0, the following sections will be searched: +[osd.0], [osd], [global] +.sp +You can specify additional sections to search with \-\-section or \-s. These additional +sections will be searched before the sections that would normally be searched. As always, +the first matching entry we find will be returned. +.sp +Note: \-\-lookup is the default action. If no other actions are given on the command line, +we will default to doing a lookup. +.SH EXAMPLES +.sp +To find out what value osd 0 will use for the "osd data" option: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-conf \-c foo.conf \-\-name osd.0 \-\-lookup "osd data" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To find out what value will mds a use for the "log file" option: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-conf \-c foo.conf \-\-name mds.a "log file" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To list all sections that begin with osd: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-conf \-c foo.conf \-l osd +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To list all sections: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-conf \-c foo.conf \-L +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-conf\fP is part of the Ceph distributed storage system. Please refer +to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-debugpack.8 b/ceph/man/ceph-debugpack.8 new file mode 100644 index 00000000..9fc016ff --- /dev/null +++ b/ceph/man/ceph-debugpack.8 @@ -0,0 +1,95 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-DEBUGPACK" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-debugpack \- ceph debug packer utility +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-debugpack\fP [ \fIoptions\fP ] \fIfilename.tar.gz\fP +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-debugpack\fP will build a tarball containing various items that are +useful for debugging crashes. The resulting tarball can be shared with +Ceph developers when debugging a problem. +.sp +The tarball will include the binaries for ceph\-mds, ceph\-osd, and ceph\-mon, radosgw, any +log files, the ceph.conf configuration file, any core files we can +find, and (if the system is running) dumps of the current cluster state +as reported by \(aqceph report\(aq. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during +startup. +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-debugpack\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +\fBceph\-post\-file\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-dencoder.8 b/ceph/man/ceph-dencoder.8 new file mode 100644 index 00000000..51b27045 --- /dev/null +++ b/ceph/man/ceph-dencoder.8 @@ -0,0 +1,193 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-DENCODER" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-dencoder \- ceph encoder/decoder utility +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-dencoder\fP [commands...] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-dencoder\fP is a utility to encode, decode, and dump ceph data +structures. It is used for debugging and for testing inter\-version +compatibility. +.sp +\fBceph\-dencoder\fP takes a simple list of commands and performs them +in order. +.SH COMMANDS +.INDENT 0.0 +.TP +.B version +Print the version string for the \fBceph\-dencoder\fP binary. +.UNINDENT +.INDENT 0.0 +.TP +.B import +Read a binary blob of encoded data from the given file. It will be +placed in an in\-memory buffer. +.UNINDENT +.INDENT 0.0 +.TP +.B export +Write the contents of the current in\-memory buffer to the given +file. +.UNINDENT +.INDENT 0.0 +.TP +.B list_types +List the data types known to this build of \fBceph\-dencoder\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B type +Select the given type for future \fBencode\fP or \fBdecode\fP operations. +.UNINDENT +.INDENT 0.0 +.TP +.B decode +Decode the contents of the in\-memory buffer into an instance of the +previously selected type. If there is an error, report it. +.UNINDENT +.INDENT 0.0 +.TP +.B encode +Encode the contents of the in\-memory instance of the previously +selected type to the in\-memory buffer. +.UNINDENT +.INDENT 0.0 +.TP +.B dump_json +Print a JSON\-formatted description of the in\-memory object. +.UNINDENT +.INDENT 0.0 +.TP +.B count_tests +Print the number of built\-in test instances of the previosly +selected type that \fBceph\-dencoder\fP is able to generate. +.UNINDENT +.INDENT 0.0 +.TP +.B select_test +Select the given build\-in test instance as a the in\-memory instance +of the type. +.UNINDENT +.INDENT 0.0 +.TP +.B get_features +Print the decimal value of the feature set supported by this version +of \fBceph\-dencoder\fP\&. Each bit represents a feature. These correspond to +CEPH_FEATURE_* defines in src/include/ceph_features.h. +.UNINDENT +.INDENT 0.0 +.TP +.B set_features +Set the feature bits provided to \fBencode\fP to \fIf\fP\&. This allows +you to encode objects such that they can be understood by old +versions of the software (for those types that support it). +.UNINDENT +.SH EXAMPLE +.sp +Say you want to examine an attribute on an object stored by \fBceph\-osd\fP\&. You can do: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ cd /mnt/osd.12/current/2.b_head +$ attr \-l foo_bar_head_EFE6384B +Attribute "ceph.snapset" has a 31 byte value for foo_bar_head_EFE6384B +Attribute "ceph._" has a 195 byte value for foo_bar_head_EFE6384B +$ attr foo_bar_head_EFE6384B \-g ceph._ \-q > /tmp/a +$ ceph\-dencoder type object_info_t import /tmp/a decode dump_json +{ "oid": { "oid": "foo", + "key": "bar", + "snapid": \-2, + "hash": 4024842315, + "max": 0}, + "locator": { "pool": 2, + "preferred": \-1, + "key": "bar"}, + "category": "", + "version": "9\(aq1", + "prior_version": "0\(aq0", + "last_reqid": "client.4116.0:1", + "size": 1681, + "mtime": "2012\-02\-21 08:58:23.666639", + "lost": 0, + "wrlock_by": "unknown.0.0:0", + "snaps": [], + "truncate_seq": 0, + "truncate_size": 0, + "watchers": {}} +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-dencoder\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-fuse.8 b/ceph/man/ceph-fuse.8 new file mode 100644 index 00000000..f41b7b91 --- /dev/null +++ b/ceph/man/ceph-fuse.8 @@ -0,0 +1,120 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-FUSE" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-fuse \- FUSE-based client for ceph +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-fuse\fP [ \-m \fImonaddr\fP:\fIport\fP ] \fImountpoint\fP [ \fIfuse options\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-fuse\fP is a FUSE (File system in USErspace) client for Ceph +distributed file system. It will mount a ceph file system (specified +via the \-m option for described by ceph.conf (see below) at the +specific mount point. +.sp +The file system can be unmounted with: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +fusermount \-u mountpoint +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +or by sending \fBSIGINT\fP to the \fBceph\-fuse\fP process. +.SH OPTIONS +.sp +Any options not recognized by ceph\-fuse will be passed on to libfuse. +.INDENT 0.0 +.TP +.B \-d +Detach from console and daemonize after startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through ceph.conf). +.UNINDENT +.INDENT 0.0 +.TP +.B \-r root_directory +Use root_directory as the mounted root, rather than the full Ceph tree. +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-fuse\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +fusermount(8), +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-mds.8 b/ceph/man/ceph-mds.8 new file mode 100644 index 00000000..398b0102 --- /dev/null +++ b/ceph/man/ceph-mds.8 @@ -0,0 +1,121 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-MDS" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-mds \- ceph metadata server daemon +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-mds\fP \-i \fIname\fP [[ \-\-hot\-standby [\fIrank\fP] ]|[\-\-journal_check \fIrank\fP]] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-mds\fP is the metadata server daemon for the Ceph distributed file +system. One or more instances of ceph\-mds collectively manage the file +system namespace, coordinating access to the shared OSD cluster. +.sp +Each ceph\-mds daemon instance should have a unique name. The name is used +to identify daemon instances in the ceph.conf. +.sp +Once the daemon has started, the monitor cluster will normally assign +it a logical rank, or put it in a standby pool to take over for +another daemon that crashes. Some of the specified options can cause +other behaviors. +.sp +If you specify hot\-standby or journal\-check, you must either specify +the rank on the command line, or specify one of the +mds_standby_for_[rank|name] parameters in the config. The command +line specification overrides the config, and specifying the rank +overrides specifying the name. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-f, \-\-foreground +Foreground: do not daemonize after startup (run in foreground). Do +not generate a pid file. Useful when run via \fBceph\-run\fP(8). +.UNINDENT +.INDENT 0.0 +.TP +.B \-d +Debug mode: like \fB\-f\fP, but also send all log output to stderr. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during +startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through +\fBceph.conf\fP). +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-mon\fP is part of the Ceph distributed storage system. Please refer to the Ceph documentation at +\fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBceph\-mon\fP(8), +\fBceph\-osd\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-mon.8 b/ceph/man/ceph-mon.8 new file mode 100644 index 00000000..64e3bf9c --- /dev/null +++ b/ceph/man/ceph-mon.8 @@ -0,0 +1,137 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-MON" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-mon \- ceph monitor daemon +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-mon\fP \-i \fImonid\fP [ \-\-mon\-data \fImondatapath\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-mon\fP is the cluster monitor daemon for the Ceph distributed +file system. One or more instances of \fBceph\-mon\fP form a Paxos +part\-time parliament cluster that provides extremely reliable and +durable storage of cluster membership, configuration, and state. +.sp +The \fImondatapath\fP refers to a directory on a local file system storing +monitor data. It is normally specified via the \fBmon data\fP option in +the configuration file. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-f, \-\-foreground +Foreground: do not daemonize after startup (run in foreground). Do +not generate a pid file. Useful when run via \fBceph\-run\fP(8). +.UNINDENT +.INDENT 0.0 +.TP +.B \-d +Debug mode: like \fB\-f\fP, but also send all log output to stderr. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during +startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-mkfs +Initialize the \fBmon data\fP directory with seed information to form +and initial ceph file system or to join an existing monitor +cluster. Three pieces of information must be provided: +.INDENT 7.0 +.IP \(bu 2 +The cluster fsid. This can come from a monmap (\fB\-\-monmap \fP) or +explicitly via \fB\-\-fsid \fP\&. +.IP \(bu 2 +A list of monitors and their addresses. This list of monitors +can come from a monmap (\fB\-\-monmap \fP), the \fBmon host\fP +configuration value (in \fIceph.conf\fP or via \fB\-m +host1,host2,...\fP), or \fBmon addr\fP lines in \fIceph.conf\fP\&. If this +monitor is to be part of the initial monitor quorum for a new +Ceph cluster, then it must be included in the initial list, +matching either the name or address of a monitor in the list. +When matching by address, either the \fBpublic addr\fP or \fBpublic +subnet\fP options may be used. +.IP \(bu 2 +The monitor secret key \fBmon.\fP\&. This must be included in the +keyring provided via \fB\-\-keyring \fP\&. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keyring +Specify a keyring for use with \fB\-\-mkfs\fP\&. +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-mon\fP is part of the Ceph distributed storage system. Please refer +to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBceph\-mds\fP(8), +\fBceph\-osd\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-osd.8 b/ceph/man/ceph-osd.8 new file mode 100644 index 00000000..24ec595c --- /dev/null +++ b/ceph/man/ceph-osd.8 @@ -0,0 +1,169 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-OSD" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-osd \- ceph object storage daemon +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-osd\fP \-i \fIosdnum\fP [ \-\-osd\-data \fIdatapath\fP ] [ \-\-osd\-journal +\fIjournal\fP ] [ \-\-mkfs ] [ \-\-mkjournal ] [ \-\-mkkey ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-osd\fP is the object storage daemon for the Ceph distributed file +system. It is responsible for storing objects on a local file system +and providing access to them over the network. +.sp +The datapath argument should be a directory on a btrfs file system +where the object data resides. The journal is optional, and is only +useful performance\-wise when it resides on a different disk than +datapath with low latency (ideally, an NVRAM device). +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-f, \-\-foreground +Foreground: do not daemonize after startup (run in foreground). Do +not generate a pid file. Useful when run via \fBceph\-run\fP(8). +.UNINDENT +.INDENT 0.0 +.TP +.B \-d +Debug mode: like \fB\-f\fP, but also send all log output to stderr. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-osd\-data osddata +Use object store at \fIosddata\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-osd\-journal journal +Journal updates to \fIjournal\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-mkfs +Create an empty object repository. This also initializes the journal +(if one is defined). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-mkkey +Generate a new secret key. This is normally used in combination +with \fB\-\-mkfs\fP as it is more convenient than generating a key by +hand with \fBceph\-authtool\fP(8). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-mkjournal +Create a new journal file to match an existing object repository. +This is useful if the journal device or file is wiped out due to a +disk or file system failure. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-flush\-journal +Flush the journal to permanent store. This runs in the foreground +so you know when it\(aqs completed. This can be useful if you want to +resize the journal or need to otherwise destroy it: this guarantees +you won\(aqt lose data. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-get\-cluster\-fsid +Print the cluster fsid (uuid) and exit. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-get\-osd\-fsid +Print the OSD\(aqs fsid and exit. The OSD\(aqs uuid is generated at +\-\-mkfs time and is thus unique to a particular instantiation of +this OSD. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-get\-journal\-fsid +Print the journal\(aqs uuid. The journal fsid is set to match the OSD +fsid at \-\-mkfs time. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP for runtime configuration options. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through +\fBceph.conf\fP). +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-osd\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBceph\-mds\fP(8), +\fBceph\-mon\fP(8), +\fBceph\-authtool\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-post-file.8 b/ceph/man/ceph-post-file.8 new file mode 100644 index 00000000..a0378215 --- /dev/null +++ b/ceph/man/ceph-post-file.8 @@ -0,0 +1,130 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-POST-FILE" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-post-file \- post files for ceph developers +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-post\-file\fP [\-d \fIdescription] [\-u *user\fP] \fIfile or dir\fP ... +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-post\-file\fP will upload files or directories to ceph.com for +later analysis by Ceph developers. +.sp +Each invocation uploads files or directories to a separate directory +with a unique tag. That tag can be passed to a developer or +referenced in a bug report (\fI\%http://tracker.ceph.com/\fP). Once the +upload completes, the directory is marked non\-readable and +non\-writeable to prevent access or modification by other users. +.SH WARNING +.sp +Basic measures are taken to make posted data be visible only to +developers with access to ceph.com infrastructure. However, users +should think twice and/or take appropriate precautions before +posting potentially sensitive data (for example, logs or data +directories that contain Ceph secrets). +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-d *description*, \-\-description *description* +Add a short description for the upload. This is a good opportunity +to reference a bug number. There is no default value. +.UNINDENT +.INDENT 0.0 +.TP +.B \-u *user* +Set the user metadata for the upload. This defaults to \fIwhoami\(ga@\(gahostname \-f\fP\&. +.UNINDENT +.SH EXAMPLES +.sp +To upload a single log: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-post\-file /var/log/ceph/ceph\-mon.\(gahostname\(ga.log +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To upload several directories: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-post\-file \-d \(aqmon data directories\(aq /var/log/ceph/mon/* +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-post\-file\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBceph\-debugpack\fP(8), +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-rbdnamer.8 b/ceph/man/ceph-rbdnamer.8 new file mode 100644 index 00000000..dab2c106 --- /dev/null +++ b/ceph/man/ceph-rbdnamer.8 @@ -0,0 +1,92 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-RBDNAMER" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-rbdnamer \- udev helper to name RBD devices +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-rbdnamer\fP \fInum\fP +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-rbdnamer\fP prints the pool and image name for the given RBD devices +to stdout. It is used by \fIudev\fP (using a rule like the one below) to +set up a device symlink. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +KERNEL=="rbd[0\-9]*", PROGRAM="/usr/bin/ceph\-rbdnamer %n", SYMLINK+="rbd/%c{1}/%c{2}" +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-rbdnamer\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBrbd\fP(8), +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-rest-api.8 b/ceph/man/ceph-rest-api.8 new file mode 100644 index 00000000..0d443f65 --- /dev/null +++ b/ceph/man/ceph-rest-api.8 @@ -0,0 +1,208 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-REST-API" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-rest-api \- ceph RESTlike administration server +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-rest\-api\fP [ \-c \fIconffile\fP ] [\-\-cluster \fIclustername\fP ] [ \-n \fIname\fP ] [\-i \fIid\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-rest\-api\fP is a WSGI application that can run as a +standalone web service or run under a web server that supports +WSGI. It provides much of the functionality of the \fBceph\fP +command\-line tool through an HTTP\-accessible interface. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-c/\-\-conf conffile +names the ceph.conf file to use for configuration. If \-c is not +specified, the default depends on the state of the \-\-cluster option +(default \(aqceph\(aq; see below). The configuration file is searched +for in this order: +.INDENT 7.0 +.IP \(bu 2 +$CEPH_CONF +.IP \(bu 2 +/etc/ceph/${cluster}.conf +.IP \(bu 2 +~/.ceph/${cluster}.conf +.IP \(bu 2 +${cluster}.conf (in the current directory) +.UNINDENT +.sp +so you can also pass this option in the environment as CEPH_CONF. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-cluster clustername +set \fIclustername\fP for use in the $cluster metavariable, for +locating the ceph.conf file. The default is \(aqceph\(aq. +.UNINDENT +.INDENT 0.0 +.TP +.B \-n/\-\-name name +specifies the client \(aqname\(aq, which is used to find the +client\-specific configuration options in the config file, and +also is the name used for authentication when connecting +to the cluster (the entity name appearing in ceph auth list output, +for example). The default is \(aqclient.restapi\(aq. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i/\-\-id id +specifies the client \(aqid\(aq, which will form the clientname +as \(aqclient.\(aq if clientname is not set. If \-n/\-name is +set, that takes precedence. +.sp +Also, global Ceph options are supported. +.UNINDENT +.SH CONFIGURATION PARAMETERS +.sp +Supported configuration parameters include: +.INDENT 0.0 +.IP \(bu 2 +\fBkeyring\fP the keyring file holding the key for \(aqclientname\(aq +.IP \(bu 2 +\fBpublic addr\fP ip:port to listen on (default 0.0.0.0:5000) +.IP \(bu 2 +\fBlog file\fP (usual Ceph default) +.IP \(bu 2 +\fBrestapi base url\fP the base URL to answer requests on (default /api/v0.1) +.IP \(bu 2 +\fBrestapi log level\fP critical, error, warning, info, debug (default warning) +.UNINDENT +.sp +Configuration parameters are searched in the standard order: +first in the section named \(aq\(aq, then \(aqclient\(aq, then \(aqglobal\(aq. +.sp + is either supplied by \-n/\-\-name, "client." where + is supplied by \-i/\-\-id, or \(aqclient.restapi\(aq if neither option +is present. +.sp +A single\-threaded server will run on \fBpublic addr\fP if the ceph\-rest\-api +executed directly; otherwise, configuration is specified by the enclosing +WSGI web server. +.SH COMMANDS +.sp +Commands are submitted with HTTP GET requests (for commands that +primarily return data) or PUT (for commands that affect cluster state). +HEAD and OPTIONS are also supported. Standard HTTP status codes +are returned. +.sp +For commands that return bulk data, the request can include +Accept: application/json or Accept: application/xml to select the +desired structured output, or you may use a .json or .xml addition +to the requested PATH. Parameters are supplied as query parameters +in the request; for parameters that take more than one value, repeat +the key=val construct. For instance, to remove OSDs 2 and 3, +send a PUT request to \fBosd/rm?ids=2&ids=3\fP\&. +.SH DISCOVERY +.sp +Human\-readable discovery of supported commands and parameters, along +with a small description of each command, is provided when the requested +path is incomplete/partially matching. Requesting / will redirect to +the value of \fBrestapi base url\fP, and that path will give a full list +of all known commands. +For example, requesting \fBapi/vX.X/mon\fP will return the list of API calls for +monitors \- \fBapi/vX.X/osd\fP will return the list of API calls for OSD and so on. +.sp +The command set is very similar to the commands +supported by the \fBceph\fP tool. One notable exception is that the +\fBceph pg \fP style of commands is supported here +as \fBtell//command?args\fP\&. +.SH DEPLOYMENT AS WSGI APPLICATION +.sp +When deploying as WSGI application (say, with Apache/mod_wsgi, +or nginx/uwsgi, or gunicorn, etc.), use the \fBceph_rest_api.py\fP module +(\fBceph\-rest\-api\fP is a thin layer around this module). The standalone web +server is of course not used, so address/port configuration is done in +the WSGI server. Use a python .wsgi module or the equivalent to call +\fBapp = generate_app(conf, cluster, clientname, clientid, args)\fP where: +.INDENT 0.0 +.IP \(bu 2 +conf is as \-c/\-\-conf above +.IP \(bu 2 +cluster is as \-\-cluster above +.IP \(bu 2 +clientname, \-n/\-\-name +.IP \(bu 2 +clientid, \-i/\-\-id, and +.IP \(bu 2 +args are any other generic Ceph arguments +.UNINDENT +.sp +When app is returned, it will have attributes \(aqceph_addr\(aq and \(aqceph_port\(aq +set to what the address and port are in the Ceph configuration; +those may be used for the server, or ignored. +.sp +Any errors reading configuration or connecting to the cluster cause an +exception to be raised; see your WSGI server documentation for how to +see those messages in case of problem. +.SH AVAILABILITY +.sp +\fBceph\-rest\-api\fP is part of the Ceph distributed storage system. Please refer to the Ceph documentation at +\fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-run.8 b/ceph/man/ceph-run.8 new file mode 100644 index 00000000..0baa632b --- /dev/null +++ b/ceph/man/ceph-run.8 @@ -0,0 +1,89 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-RUN" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-run \- restart daemon on core dump +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-run\fP \fIcommand\fP ... +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-run\fP is a simple wrapper that will restart a daemon if it exits +with a signal indicating it crashed and possibly core dumped (that is, +signals 3, 4, 5, 6, 8, or 11). +.sp +The command should run the daemon in the foreground. For Ceph daemons, +that means the \fB\-f\fP option. +.SH OPTIONS +.sp +None +.SH AVAILABILITY +.sp +\fBceph\-run\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBceph\-mon\fP(8), +\fBceph\-mds\fP(8), +\fBceph\-osd\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph-syn.8 b/ceph/man/ceph-syn.8 new file mode 100644 index 00000000..6b12a09b --- /dev/null +++ b/ceph/man/ceph-syn.8 @@ -0,0 +1,148 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH-SYN" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph-syn \- ceph synthetic workload generator +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\-syn\fP [ \-m \fImonaddr\fP:\fIport\fP ] \-\-syn \fIcommand\fP \fI\&...\fP +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\-syn\fP is a simple synthetic workload generator for the Ceph +distributed file system. It uses the userspace client library to +generate simple workloads against a currently running file system. The +file system need not be mounted via ceph\-fuse(8) or the kernel client. +.sp +One or more \fB\-\-syn\fP command arguments specify the particular +workload, as documented below. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-d +Detach from console and daemonize after startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during +startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through +\fBceph.conf\fP). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-num_client num +Run num different clients, each in a separate thread. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-syn workloadspec +Run the given workload. May be specified as many times as +needed. Workloads will normally run sequentially. +.UNINDENT +.SH WORKLOADS +.sp +Each workload should be preceded by \fB\-\-syn\fP on the command +line. This is not a complete list. +.INDENT 0.0 +.TP +.B \fBmknap\fP \fIpath\fP \fIsnapname\fP +Create a snapshot called \fIsnapname\fP on \fIpath\fP\&. +.TP +.B \fBrmsnap\fP \fIpath\fP \fIsnapname\fP +Delete snapshot called \fIsnapname\fP on \fIpath\fP\&. +.TP +.B \fBrmfile\fP \fIpath\fP +Delete/unlink \fIpath\fP\&. +.TP +.B \fBwritefile\fP \fIsizeinmb\fP \fIblocksize\fP +Create a file, named after our client id, that is \fIsizeinmb\fP MB by +writing \fIblocksize\fP chunks. +.TP +.B \fBreadfile\fP \fIsizeinmb\fP \fIblocksize\fP +Read file, named after our client id, that is \fIsizeinmb\fP MB by +writing \fIblocksize\fP chunks. +.TP +.B \fBrw\fP \fIsizeinmb\fP \fIblocksize\fP +Write file, then read it back, as above. +.TP +.B \fBmakedirs\fP \fInumsubdirs\fP \fInumfiles\fP \fIdepth\fP +Create a hierarchy of directories that is \fIdepth\fP levels deep. Give +each directory \fInumsubdirs\fP subdirectories and \fInumfiles\fP files. +.TP +.B \fBwalk\fP +Recursively walk the file system (like find). +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\-syn\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBceph\-fuse\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/ceph.8 b/ceph/man/ceph.8 new file mode 100644 index 00000000..9bb903c0 --- /dev/null +++ b/ceph/man/ceph.8 @@ -0,0 +1,162 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPH" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +ceph \- ceph file system control utility +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBceph\fP [ \-m \fImonaddr\fP ] [ \-w | \fIcommand\fP ... ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBceph\fP is a control utility for communicating with the monitor +cluster of a running Ceph distributed storage system. +.sp +There are three basic modes of operation. +.SS Interactive mode +.sp +To start in interactive mode, no arguments are necessary. Control\-d or +\(aqquit\(aq will exit. +.SS Watch mode +.sp +Watch mode shows cluster state changes as they occur. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph \-w +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Command line mode +.sp +Finally, to send a single instruction to the monitor cluster (and wait +for a response), the command can be specified on the command line. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-i infile +will specify an input file to be passed along as a payload with the +command to the monitor cluster. This is only used for specific +monitor commands. +.UNINDENT +.INDENT 0.0 +.TP +.B \-o outfile +will write any payload returned by the monitor cluster with its +reply to outfile. Only specific monitor commands (e.g. osd getmap) +return a payload. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use ceph.conf configuration file instead of the default +/etc/ceph/ceph.conf to determine monitor addresses during startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through ceph.conf). +.UNINDENT +.SH EXAMPLES +.sp +To grab a copy of the current OSD map: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph \-m 1.2.3.4:6789 osd getmap \-o osdmap +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To get a dump of placement group (PG) state: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph pg dump \-o pg.txt +.ft P +.fi +.UNINDENT +.UNINDENT +.SH MONITOR COMMANDS +.sp +A more complete summary of commands understood by the monitor cluster can be found in the +online documentation, at +.INDENT 0.0 +.INDENT 3.5 +\fI\%http://ceph.com/docs/master/rados/operations/control\fP +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBceph\fP is part of the Ceph distributed storage system. Please refer to the Ceph documentation at +\fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/cephfs.8 b/ceph/man/cephfs.8 new file mode 100644 index 00000000..2fa55839 --- /dev/null +++ b/ceph/man/cephfs.8 @@ -0,0 +1,144 @@ +.\" Man page generated from reStructuredText. +. +.TH "CEPHFS" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +cephfs \- ceph file system options utility +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBcephfs\fP [ \fIpath\fP \fIcommand\fP \fIoptions\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBcephfs\fP is a control utility for accessing and manipulating file +layout and location data in the Ceph distributed storage system. +.sp +Choose one of the following three commands: +.INDENT 0.0 +.IP \(bu 2 +\fBshow_layout\fP View the layout information on a file or directory +.IP \(bu 2 +\fBset_layout\fP Set the layout information on a file or directory +.IP \(bu 2 +\fBshow_location\fP View the location information on a file +.UNINDENT +.SH OPTIONS +.sp +Your applicable options differ depending on whether you are setting or viewing layout/location. +.SS Viewing options: +.INDENT 0.0 +.TP +.B \-l \-\-offset +Specify an offset for which to retrieve location data +.UNINDENT +.SS Setting options: +.INDENT 0.0 +.TP +.B \-u \-\-stripe_unit +Set the size of each stripe +.UNINDENT +.INDENT 0.0 +.TP +.B \-c \-\-stripe_count +Set the number of objects to stripe across +.UNINDENT +.INDENT 0.0 +.TP +.B \-s \-\-object_size +Set the size of the objects to stripe across +.UNINDENT +.INDENT 0.0 +.TP +.B \-p \-\-pool +Set the pool (by numeric value, not name!) to use +.UNINDENT +.INDENT 0.0 +.TP +.B \-o \-\-osd +Set the preferred OSD to use as the primary +.UNINDENT +.SH LIMITATIONS +.sp +When setting layout data, the specified object size must evenly divide +by the specified stripe unit. Any parameters you don\(aqt set +explicitly are left at the system defaults. +.sp +Obviously setting the layout of a file and a directory means different +things. Setting the layout of a file specifies exactly how to place +the individual file. This must be done before writing \fIany\fP data to +it. Truncating a file does not allow you to change the layout either. +.sp +Setting the layout of a directory sets the "default layout", which is +used to set the file layouts on any files subsequently created in the +directory (or any subdirectory). Pre\-existing files do not have their +layouts changed. +.sp +You\(aqll notice that the layout information allows you to specify a +preferred OSD for placement. This feature is unsupported and ignored +in modern versions of the Ceph servers; do not use it. +.SH AVAILABILITY +.sp +\fBcephfs\fP is part of the Ceph distributed storage system. Please refer +to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/crushtool.8 b/ceph/man/crushtool.8 new file mode 100644 index 00000000..bb518f77 --- /dev/null +++ b/ceph/man/crushtool.8 @@ -0,0 +1,433 @@ +.\" Man page generated from reStructuredText. +. +.TH "CRUSHTOOL" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +crushtool \- CRUSH map manipulation tool +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBcrushtool\fP ( \-d \fImap\fP | \-c \fImap.txt\fP | \-\-build \-\-num_osds \fInumosds\fP +\fIlayer1\fP \fI\&...\fP | \-\-test ) [ \-o \fIoutfile\fP ] +.fi +.sp +.SH DESCRIPTION +.INDENT 0.0 +.TP +.B \fBcrushtool\fP is a utility that lets you create, compile, decompile +and test CRUSH map files. +.UNINDENT +.sp +CRUSH is a pseudo\-random data distribution algorithm that efficiently +maps input values (typically data objects) across a heterogeneous, +hierarchically structured device map. The algorithm was originally +described in detail in the following paper (although it has evolved +some since then): +.INDENT 0.0 +.INDENT 3.5 +\fI\%http://www.ssrc.ucsc.edu/Papers/weil-sc06.pdf\fP +.UNINDENT +.UNINDENT +.sp +The tool has four modes of operation. +.INDENT 0.0 +.TP +.B \-\-compile|\-c map.txt +will compile a plaintext map.txt into a binary map file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-decompile|\-d map +will take the compiled map and decompile it into a plaintext source +file, suitable for editing. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-build \-\-num_osds {num\-osds} layer1 ... +will create map with the given layer structure. See below for a +detailed explanation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-test +will perform a dry run of a CRUSH mapping for a range of input +object names. See below for a detailed explanation. +.UNINDENT +.sp +Unlike other Ceph tools, \fBcrushtool\fP does not accept generic options +such as \fB\-\-debug\-crush\fP from the command line. They can however be +provided via the CEPH_ARGS environment variable. For instance, to +silence all output from the CRUSH subsystem: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +CEPH_ARGS="\-\-debug\-crush 0" crushtool ... +.ft P +.fi +.UNINDENT +.UNINDENT +.SH RUNNING TESTS WITH --TEST +.sp +The test mode will use the input crush map ( as specified with \fB\-i +map\fP ) and perform a dry run of CRUSH mapping or random placement ( +if \fB\-\-simulate\fP is set ). On completion, two kinds of reports can be +created. The \fB\-\-show\-...\fP options output human readable information +on stderr. The \fB\-\-output\-csv\fP option creates CSV files that are +documented by the \fB\-\-help\-output\fP option. +.INDENT 0.0 +.TP +.B \-\-show\-statistics +for each rule display the mapping of each object. For instance: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +CRUSH rule 1 x 24 [11,6] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +shows that object \fB24\fP is mapped to devices \fB[11,6]\fP by rule +\fB1\fP\&. At the end of the mapping details, a summary of the +distribution is displayed. For instance: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +rule 1 (metadata) num_rep 5 result size == 5: 1024/1024 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +shows that rule \fB1\fP which is named \fBmetadata\fP successfully +mapped \fB1024\fP objects to \fBresult size == 5\fP devices when trying +to map them to \fBnum_rep 5\fP replicas. When it fails to provide the +required mapping, presumably because the number of \fBtries\fP must +be increased, a breakdown of the failures is displays. For instance: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +rule 1 (metadata) num_rep 10 result size == 8: 4/1024 +rule 1 (metadata) num_rep 10 result size == 9: 93/1024 +rule 1 (metadata) num_rep 10 result size == 10: 927/1024 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +shows that although \fBnum_rep 10\fP replicas were required, \fB4\fP +out of \fB1024\fP objects ( \fB4/1024\fP ) were mapped to \fBresult size +== 8\fP devices only. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-show\-bad\-mappings +display which object failed to be mapped to the required number of +devices. For instance: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +shows that when rule \fB1\fP was required to map \fB7\fP devices, it +could only map six : \fB[8,10,2,11,6,9]\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-show\-utilization +display the expected and actual utilisation for each device, for +each number of replicas. For instance: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +device 0: stored : 951 expected : 853.333 +device 1: stored : 963 expected : 853.333 +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +shows that device \fB0\fP stored \fB951\fP objects and was expected to store \fB853\fP\&. +Implies \fB\-\-show\-statistics\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-show\-utilization\-all +displays the same as \fB\-\-show\-utilization\fP but does not suppress +output when the weight of a device is zero. +Implies \fB\-\-show\-statistics\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-show\-choose\-tries +display how many attempts were needed to find a device mapping. +For instance: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +0: 95224 +1: 3745 +2: 2225 +\&.. +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +shows that \fB95224\fP mappings succeeded without retries, \fB3745\fP +mappings succeeded with one attempts, etc. There are as many rows +as the value of the \fB\-\-set\-choose\-total\-tries\fP option. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-output\-csv +create CSV files (in the current directory) containing information +documented by \fB\-\-help\-output\fP\&. The files are named after the rule +used when collecting the statistics. For instance, if the rule +metadata is used, the CSV files will be: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +metadata\-absolute_weights.csv +metadata\-device_utilization.csv +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The first line of the file shortly explains the column layout. For +instance: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +metadata\-absolute_weights.csv +Device ID, Absolute Weight +0,1 +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-output\-name NAME +prepend \fBNAME\fP to the file names generated when \fB\-\-output\-csv\fP +is specified. For instance \fB\-\-output\-name FOO\fP will create +files: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +FOO\-metadata\-absolute_weights.csv +FOO\-metadata\-device_utilization.csv +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.sp +The \fB\-\-set\-...\fP options can be used to modify the tunables of the +input crush map. The input crush map is modified in +memory. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ crushtool \-i mymap \-\-test \-\-show\-bad\-mappings +bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +could be fixed by increasing the \fBchoose\-total\-tries\fP as follows: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.B $ crushtool \-i mymap \-\-test +\-\-show\-bad\-mappings \-\-set\-choose\-total\-tries 500 +.UNINDENT +.UNINDENT +.UNINDENT +.SH BUILDING A MAP WITH --BUILD +.sp +The build mode will generate hierarchical maps. The first argument +specifies the number of devices (leaves) in the CRUSH hierarchy. Each +layer describes how the layer (or devices) preceding it should be +grouped. +.sp +Each layer consists of: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +bucket ( uniform | list | tree | straw ) size +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The \fBbucket\fP is the type of the buckets in the layer +(e.g. "rack"). Each bucket name will be built by appending a unique +number to the \fBbucket\fP string (e.g. "rack0", "rack1"...). +.sp +The second component is the type of bucket: \fBstraw\fP should be used +most of the time. +.sp +The third component is the maximum size of the bucket. A size of zero +means a bucket of infinite capacity. +.SH EXAMPLE +.sp +Suppose we have two rows with two racks each and 20 nodes per rack. Suppose +each node contains 4 storage devices for Ceph OSD Daemons. This configuration +allows us to deploy 320 Ceph OSD Daemons. Lets assume a 42U rack with 2U nodes, +leaving an extra 2U for a rack switch. +.sp +To reflect our hierarchy of devices, nodes, racks and rows, we would execute +the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ crushtool \-o crushmap \-\-build \-\-num_osds 320 \e + node straw 4 \e + rack straw 20 \e + row straw 2 \e + root straw 0 +# id weight type name reweight +\-87 320 root root +\-85 160 row row0 +\-81 80 rack rack0 +\-1 4 node node0 +0 1 osd.0 1 +1 1 osd.1 1 +2 1 osd.2 1 +3 1 osd.3 1 +\-2 4 node node1 +4 1 osd.4 1 +5 1 osd.5 1 +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +CRUSH rulesets are created so the generated crushmap can be +tested. They are the same rulesets as the one created by default when +creating a new Ceph cluster. They can be further edited with: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# decompile +crushtool \-d crushmap \-o map.txt + +# edit +emacs map.txt + +# recompile +crushtool \-c map.txt \-o crushmap +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBcrushtool\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBosdmaptool\fP(8), +.SH AUTHORS +.sp +John Wilkins, Sage Weil, Loic Dachary +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/librados-config.8 b/ceph/man/librados-config.8 new file mode 100644 index 00000000..48238fe1 --- /dev/null +++ b/ceph/man/librados-config.8 @@ -0,0 +1,94 @@ +.\" Man page generated from reStructuredText. +. +.TH "LIBRADOS-CONFIG" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +librados-config \- display information about librados +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBlibrados\-config\fP [ \-\-version ] [ \-\-vernum ] +.fi +.sp +.SH DESCRIPTION +.INDENT 0.0 +.TP +.B \fBlibrados\-config\fP is a utility that displays information about the +installed \fBlibrados\fP\&. +.UNINDENT +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-\-version +Display \fBlibrados\fP version +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-vernum +Display the \fBlibrados\fP version code +.UNINDENT +.SH AVAILABILITY +.sp +\fBlibrados\-config\fP is part of the Ceph distributed storage system. +Please refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for +more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBrados\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/mkcephfs.8 b/ceph/man/mkcephfs.8 new file mode 100644 index 00000000..0341f495 --- /dev/null +++ b/ceph/man/mkcephfs.8 @@ -0,0 +1,164 @@ +.TH "MKCEPHFS" "8" "April 29, 2013" "dev" "Ceph" +.SH NAME +mkcephfs \- create a ceph file system +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.\" Man page generated from reStructuredText. +. +.SH SYNOPSIS +.nf +\fBmkcephfs\fP \-c \fIceph.conf\fP [ \-\-mkfs ] [ \-a, \-\-all\-hosts [ \-k +\fI/path/to/admin.keyring\fP ] ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBmkcephfs\fP is used to create an empty Ceph file system, possibly +spanning multiple hosts. The ceph.conf file describes the composition +of the entire Ceph cluster, including which hosts are participating, +which daemons run where, and which paths are used to store file system +data or metadata. +.sp +The mkcephfs tool can be used in two ways. If \-a is used, it will use +ssh and scp to connect to remote hosts on your behalf and do the setup +of the entire cluster. This is the easiest solution, but can also be +inconvenient (if you don\(aqt have ssh to connect without prompting for +passwords) or slow (if you have a large cluster). +.sp +Alternatively, you can run each setup phase manually. First, you need +to prepare a monmap that will be shared by each node: +.sp +.nf +.ft C +# prepare +master# mkdir /tmp/foo +master# mkcephfs \-c /etc/ceph/ceph.conf \e + \-\-prepare\-monmap \-d /tmp/foo +.ft P +.fi +.sp +Share the \fB/tmp/foo\fP directory with other nodes in whatever way is +convenient for you. On each OSD and MDS node: +.sp +.nf +.ft C +osdnode# mkcephfs \-\-init\-local\-daemons osd \-d /tmp/foo +mdsnode# mkcephfs \-\-init\-local\-daemons mds \-d /tmp/foo +.ft P +.fi +.sp +Collect the contents of the /tmp/foo directories back onto a single +node, and then: +.sp +.nf +.ft C +master# mkcephfs \-\-prepare\-mon \-d /tmp/foo +.ft P +.fi +.sp +Finally, distribute \fB/tmp/foo\fP to all monitor nodes and, on each of +those nodes: +.sp +.nf +.ft C +monnode# mkcephfs \-\-init\-local\-daemons mon \-d /tmp/foo +.ft P +.fi +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-a, \-\-allhosts +Performs the necessary initialization steps on all hosts in the +cluster, executing commands via SSH. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use the given conf file instead of the default \fB/etc/ceph/ceph.conf\fP. +.UNINDENT +.INDENT 0.0 +.TP +.B \-k /path/to/keyring +When \fB\-a\fP is used, we can specify a location to copy the +client.admin keyring, which is used to administer the cluster. The +default is \fB/etc/ceph/keyring\fP (or whatever is specified in the +config file). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-mkfs +Create and mount the file systems specified in the ceph.conf for +OSD data storage using mkfs.$type. The \fBdevs\fP option in ceph.conf +must specify the device(s) and the \fBosd mkfs type\fP option must +specify the file system type (normally one of btrfs, xfs, or ext4). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-copy\-conf +By default, mkcephfs with \-a will copy the new configuration to +/etc/ceph/ceph.conf on each node in the cluster. This option +disables that behavior. +.UNINDENT +.SH SUBCOMMANDS +.sp +The sub\-commands performed during cluster setup can be run individually with +.INDENT 0.0 +.TP +.B \-\-prepare\-monmap \-d dir \-c ceph.conf +Create an initial monmap with a random fsid/uuid and store it and +the ceph.conf in dir. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-init\-local\-daemons type \-d dir +Initialize any daemons of type type on the local host using the +monmap in dir. For types osd and mds, the resulting authentication +keys will be placed in dir. For type mon, the initial data files +generated by \-\-prepare\-mon (below) are expected in dir. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-prepare\-mon \-d dir +Prepare the initial monitor data based on the monmap, OSD, and MDS +authentication keys collected in dir, and put the result in dir. +.UNINDENT +.SH AVAILABILITY +.sp +\fBmkcephfs\fP is part of the Ceph distributed file system. Please refer +to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBmonmaptool\fP(8), +\fBosdmaptool\fP(8), +\fBcrushtool\fP(8) +.SH COPYRIGHT +2010-2013, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/monmaptool.8 b/ceph/man/monmaptool.8 new file mode 100644 index 00000000..1bd9f306 --- /dev/null +++ b/ceph/man/monmaptool.8 @@ -0,0 +1,188 @@ +.\" Man page generated from reStructuredText. +. +.TH "MONMAPTOOL" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +monmaptool \- ceph monitor cluster map manipulation tool +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBmonmaptool\fP \fImapfilename\fP [ \-\-clobber ] [ \-\-print ] [ \-\-create ] +[ \-\-add \fIip\fP:\fIport\fP \fI\&...\fP ] [ \-\-rm \fIip\fP:\fIport\fP \fI\&...\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBmonmaptool\fP is a utility to create, view, and modify a monitor +cluster map for the Ceph distributed storage system. The monitor map +specifies the only fixed addresses in the Ceph distributed system. +All other daemons bind to arbitrary addresses and register themselves +with the monitors. +.sp +When creating a map with \-\-create, a new monitor map with a new, +random UUID will be created. It should be followed by one or more +monitor addresses. +.sp +The default Ceph monitor port is 6789. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-\-print +will print a plaintext dump of the map, after any modifications are +made. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clobber +will allow monmaptool to overwrite mapfilename if changes are made. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-create +will create a new monitor map with a new UUID (and with it, a new, +empty Ceph file system). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-generate +generate a new monmap based on the values on the command line or specified +in the ceph configuration. This is, in order of preference, +.INDENT 7.0 +.INDENT 3.5 +.INDENT 0.0 +.IP 1. 3 +\fB\-\-monmap filename\fP to specify a monmap to load +.IP 2. 3 +\fB\-\-mon\-host \(aqhost1,ip2\(aq\fP to specify a list of hosts or ip addresses +.IP 3. 3 +\fB[mon.foo]\fP sections containing \fBmon addr\fP settings in the config +.UNINDENT +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-filter\-initial\-members +filter the initial monmap by applying the \fBmon initial members\fP +setting. Monitors not present in that list will be removed, and +initial members not present in the map will be added with dummy +addresses. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-add name ip:port +will add a monitor with the specified ip:port to the map. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-rm name +will remove the monitor with the specified ip:port from the map. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-fsid uuid +will set the fsid to the given uuid. If not specified with \-\-create, a random fsid will be generated. +.UNINDENT +.SH EXAMPLE +.sp +To create a new map with three monitors (for a fresh Ceph file system): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +monmaptool \-\-create \-\-add mon.a 192.168.0.10:6789 \-\-add mon.b 192.168.0.11:6789 \e + \-\-add mon.c 192.168.0.12:6789 \-\-clobber monmap +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To display the contents of the map: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +monmaptool \-\-print monmap +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To replace one monitor: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +monmaptool \-\-rm mon.a \-\-add mon.a 192.168.0.9:6789 \-\-clobber monmap +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBmonmaptool\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBcrushtool\fP(8), +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/mount.ceph.8 b/ceph/man/mount.ceph.8 new file mode 100644 index 00000000..c3549c08 --- /dev/null +++ b/ceph/man/mount.ceph.8 @@ -0,0 +1,257 @@ +.\" Man page generated from reStructuredText. +. +.TH "MOUNT.CEPH" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +mount.ceph \- mount a ceph file system +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBmount.ceph\fP \fImonaddr1\fP[,\fImonaddr2\fP,...]:/[\fIsubdir\fP] \fIdir\fP [ +\-o \fIoptions\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBmount.ceph\fP is a simple helper for mounting the Ceph file system on +a Linux host. It serves to resolve monitor hostname(s) into IP +addresses and read authentication keys from disk; the Linux kernel +client component does most of the real work. In fact, it is possible +to mount a non\-authenticated Ceph file system without mount.ceph by +specifying monitor address(es) by IP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount \-t ceph 1.2.3.4:/ mountpoint +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Each monitor address monaddr takes the form host[:port]. If the port +is not specified, the Ceph default of 6789 is assumed. +.sp +Multiple monitor addresses can be separated by commas. Only one +responsible monitor is needed to successfully mount; the client will +learn about all monitors from any responsive monitor. However, it is a +good idea to specify more than one in case one happens to be down at +the time of mount. +.sp +A subdirectory subdir may be specified if a subset of the file system +is to be mounted. +.sp +Mount helper application conventions dictate that the first two +options are device to be mounted and destination path. Options must be +passed only after these fixed arguments. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \fBwsize\fP +int, max write size. Default: none (writeback uses smaller of wsize +and stripe unit) +.TP +.B \fBrsize\fP +int (bytes), max readahead, multiple of 1024, Default: 524288 +(512*1024) +.TP +.B \fBosdtimeout\fP +int (seconds), Default: 60 +.TP +.B \fBosdkeepalivetimeout\fP +int, Default: 5 +.TP +.B \fBmount_timeout\fP +int (seconds), Default: 60 +.TP +.B \fBosd_idle_ttl\fP +int (seconds), Default: 60 +.TP +.B \fBcaps_wanted_delay_min\fP +int, cap release delay, Default: 5 +.TP +.B \fBcaps_wanted_delay_max\fP +int, cap release delay, Default: 60 +.TP +.B \fBcap_release_safety\fP +int, Default: calculated +.TP +.B \fBreaddir_max_entries\fP +int, Default: 1024 +.TP +.B \fBreaddir_max_bytes\fP +int, Default: 524288 (512*1024) +.TP +.B \fBwrite_congestion_kb\fP +int (kb), max writeback in flight. scale with available +memory. Default: calculated from available memory +.TP +.B \fBsnapdirname\fP +string, set the name of the hidden snapdir. Default: .snap +.TP +.B \fBname\fP +RADOS user to authenticate as when using cephx. Default: guest +.TP +.B \fBsecret\fP +secret key for use with cephx. This option is insecure because it exposes +the secret on the command line. To avoid this, use the secretfile option. +.TP +.B \fBsecretfile\fP +path to file containing the secret key to use with cephx +.TP +.B \fBip\fP +my ip +.TP +.B \fBnoshare\fP +create a new client instance, instead of sharing an existing +instance of a client mounting the same cluster +.TP +.B \fBdirstat\fP +funky \fIcat dirname\fP for stats, Default: off +.TP +.B \fBnodirstat\fP +no funky \fIcat dirname\fP for stats +.TP +.B \fBrbytes\fP +Report the recursive size of the directory contents for st_size on +directories. Default: on +.TP +.B \fBnorbytes\fP +Do not report the recursive size of the directory contents for +st_size on directories. +.TP +.B \fBnocrc\fP +no data crc on writes +.TP +.B \fBnoasyncreaddir\fP +no dcache readdir +.UNINDENT +.SH EXAMPLES +.sp +Mount the full file system: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount.ceph monhost:/ /mnt/foo +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +If there are multiple monitors: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount.ceph monhost1,monhost2,monhost3:/ /mnt/foo +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +If \fBceph\-mon\fP(8) is running on a non\-standard +port: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount.ceph monhost1:7000,monhost2:7000,monhost3:7000:/ /mnt/foo +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To mount only part of the namespace: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount.ceph monhost1:/some/small/thing /mnt/thing +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Assuming mount.ceph(8) is installed properly, it should be +automatically invoked by mount(8) like so: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount \-t ceph monhost:/ /mnt/foo +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBmount.ceph\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\-fuse\fP(8), +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/osdmaptool.8 b/ceph/man/osdmaptool.8 new file mode 100644 index 00000000..cf3660bf --- /dev/null +++ b/ceph/man/osdmaptool.8 @@ -0,0 +1,139 @@ +.\" Man page generated from reStructuredText. +. +.TH "OSDMAPTOOL" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +osdmaptool \- ceph osd cluster map manipulation tool +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBosdmaptool\fP \fImapfilename\fP [\-\-print] [\-\-createsimple \fInumosd\fP +[\-\-pgbits \fIbitsperosd\fP ] ] [\-\-clobber] +.fi +.sp +.SH DESCRIPTION +.sp +\fBosdmaptool\fP is a utility that lets you create, view, and manipulate +OSD cluster maps from the Ceph distributed storage system. Notably, it +lets you extract the embedded CRUSH map or import a new CRUSH map. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-\-print +will simply make the tool print a plaintext dump of the map, after +any modifications are made. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clobber +will allow osdmaptool to overwrite mapfilename if changes are made. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-import\-crush mapfile +will load the CRUSH map from mapfile and embed it in the OSD map. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-export\-crush mapfile +will extract the CRUSH map from the OSD map and write it to +mapfile. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-createsimple numosd [\-\-pgbits bitsperosd] +will create a relatively generic OSD map with the numosd devices. +If \-\-pgbits is specified, the initial placement group counts will +be set with bitsperosd bits per OSD. That is, the pg_num map +attribute will be set to numosd shifted by bitsperosd. +.UNINDENT +.SH EXAMPLE +.sp +To create a simple map with 16 devices: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +osdmaptool \-\-createsimple 16 osdmap \-\-clobber +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To view the result: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +osdmaptool \-\-print osdmap +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBosdmaptool\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBcrushtool\fP(8), +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/rados.8 b/ceph/man/rados.8 new file mode 100644 index 00000000..0b9881f3 --- /dev/null +++ b/ceph/man/rados.8 @@ -0,0 +1,260 @@ +.\" Man page generated from reStructuredText. +. +.TH "RADOS" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +rados \- rados object storage utility +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBrados\fP [ \-m \fImonaddr\fP ] [ mkpool | rmpool \fIfoo\fP ] [ \-p | \-\-pool +\fIpool\fP ] [ \-s | \-\-snap \fIsnap\fP ] [ \-i \fIinfile\fP ] [ \-o \fIoutfile\fP ] +\fIcommand\fP ... +.fi +.sp +.SH DESCRIPTION +.sp +\fBrados\fP is a utility for interacting with a Ceph object storage +cluster (RADOS), part of the Ceph distributed storage system. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-p pool, \-\-pool pool +Interact with the given pool. Required by most commands. +.UNINDENT +.INDENT 0.0 +.TP +.B \-s snap, \-\-snap snap +Read from the given pool snapshot. Valid for all pool\-specific read operations. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i infile +will specify an input file to be passed along as a payload with the +command to the monitor cluster. This is only used for specific +monitor commands. +.UNINDENT +.INDENT 0.0 +.TP +.B \-o outfile +will write any payload returned by the monitor cluster with its +reply to outfile. Only specific monitor commands (e.g. osd getmap) +return a payload. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use ceph.conf configuration file instead of the default +/etc/ceph/ceph.conf to determine monitor addresses during startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through ceph.conf). +.UNINDENT +.SH GLOBAL COMMANDS +.INDENT 0.0 +.TP +.B \fBlspools\fP +List object pools +.TP +.B \fBdf\fP +Show utilization statistics, including disk usage (bytes) and object +counts, over the entire system and broken down by pool. +.TP +.B \fBmkpool\fP \fIfoo\fP +Create a pool with name foo. +.TP +.B \fBrmpool\fP \fIfoo\fP [ \fIfoo\fP \-\-yes\-i\-really\-really\-mean\-it ] +Delete the pool foo (and all its data) +.UNINDENT +.SH POOL SPECIFIC COMMANDS +.INDENT 0.0 +.TP +.B \fBget\fP \fIname\fP \fIoutfile\fP +Read object name from the cluster and write it to outfile. +.TP +.B \fBput\fP \fIname\fP \fIinfile\fP +Write object name to the cluster with contents from infile. +.TP +.B \fBrm\fP \fIname\fP +Remove object name. +.TP +.B \fBls\fP \fIoutfile\fP +List objects in given pool and write to outfile. +.TP +.B \fBlssnap\fP +List snapshots for given pool. +.TP +.B \fBclonedata\fP \fIsrcname\fP \fIdstname\fP \-\-object\-locator \fIkey\fP +Clone object byte data from \fIsrcname\fP to \fIdstname\fP\&. Both objects must be stored with the locator key \fIkey\fP (usually either \fIsrcname\fP or \fIdstname\fP). Object attributes and omap keys are not copied or cloned. +.TP +.B \fBmksnap\fP \fIfoo\fP +Create pool snapshot named \fIfoo\fP\&. +.TP +.B \fBrmsnap\fP \fIfoo\fP +Remove pool snapshot named \fIfoo\fP\&. +.TP +.B \fBbench\fP \fIseconds\fP \fImode\fP [ \-b \fIobjsize\fP ] [ \-t \fIthreads\fP ] +Benchmark for seconds. The mode can be write or read. The default +object size is 4 MB, and the default number of simulated threads +(parallel writes) is 16. +.TP +.B \fBlistomapkeys\fP \fIname\fP +List all the keys stored in the object map of object name. +.TP +.B \fBlistomapvals\fP \fIname\fP +List all key/value pairs stored in the object map of object name. +The values are dumped in hexadecimal. +.TP +.B \fBgetomapval\fP \fIname\fP \fIkey\fP +Dump the hexadecimal value of key in the object map of object name. +.TP +.B \fBsetomapval\fP \fIname\fP \fIkey\fP \fIvalue\fP +Set the value of key in the object map of object name. +.TP +.B \fBrmomapkey\fP \fIname\fP \fIkey\fP +Remove key from the object map of object name. +.TP +.B \fBgetomapheader\fP \fIname\fP +Dump the hexadecimal value of the object map header of object name. +.TP +.B \fBsetomapheader\fP \fIname\fP \fIvalue\fP +Set the value of the object map header of object name. +.UNINDENT +.SH EXAMPLES +.sp +To view cluster utilization: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rados df +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To get a list object in pool foo sent to stdout: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rados \-p foo ls \- +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To write an object: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rados \-p foo put myobject blah.txt +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To create a snapshot: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rados \-p foo mksnap mysnap +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To delete the object: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rados \-p foo rm myobject +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To read a previously snapshotted version of an object: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rados \-p foo \-s mysnap get myobject blah.txt.old +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBrados\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/radosgw-admin.8 b/ceph/man/radosgw-admin.8 new file mode 100644 index 00000000..64c54ae2 --- /dev/null +++ b/ceph/man/radosgw-admin.8 @@ -0,0 +1,333 @@ +.\" Man page generated from reStructuredText. +. +.TH "RADOSGW-ADMIN" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +radosgw-admin \- rados REST gateway user administration utility +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBradosgw\-admin\fP \fIcommand\fP [ \fIoptions\fP \fI\&...\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBradosgw\-admin\fP is a RADOS gateway user administration utility. It +allows creating and modifying users. +.SH COMMANDS +.sp +\fIcommand\fP can be one of the following options: +.INDENT 0.0 +.TP +.B \fBuser create\fP +Create a new user +.TP +.B \fBuser modify\fP +Modify a user +.TP +.B \fBuser info\fP +Display information of a user, and any potentially available +subusers and keys +.TP +.B \fBuser rm\fP +Remove a user +.TP +.B \fBsubuser create\fP +Create a new subuser (primarily useful for clients using the Swift API) +.TP +.B \fBsubuser modify\fP +Modify a subuser +.TP +.B \fBsubuser rm\fP +Remove a subuser +.TP +.B \fBbucket list\fP +List all buckets +.TP +.B \fBbucket unlink\fP +Remove a bucket +.TP +.B \fBbucket rm\fP +Remove a bucket +.TP +.B \fBobject rm\fP +Remove an object +.TP +.B \fBkey create\fP +Create an access key +.TP +.B \fBkey rm\fP +Remove an access key +.TP +.B \fBpool add\fP +Add an existing pool for data placement +.TP +.B \fBpool rm\fP +Remove an existing pool from data placement set +.TP +.B \fBpools list\fP +List placement active set +.TP +.B \fBpolicy\fP +Display bucket/object policy +.TP +.B \fBlog show\fP +Show the log of a bucket (with a specified date) +.TP +.B \fBusage show\fP +Show the usage information (with optional user and date range) +.TP +.B \fBusage trim\fP +Trim usage information (with optional user and date range) +.UNINDENT +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during +startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through ceph.conf). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-uid=uid +The radosgw user ID. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-secret=secret +The secret associated with a given key. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-display\-name=name +Configure the display name of the user. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-email=email +The e\-mail address of the user +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-bucket=bucket +Specify the bucket name. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-object=object +Specify the object name. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-date=yyyy\-mm\-dd +The date needed for some commands +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-start\-date=yyyy\-mm\-dd +The start date needed for some commands +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-end\-date=yyyy\-mm\-dd +The end date needed for some commands +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-auth\-uid=auid +The librados auid +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-purge\-data +Remove user data before user removal +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-purge\-objects +Remove all objects before bucket removal +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-lazy\-remove +Defer removal of object tail +.UNINDENT +.SH EXAMPLES +.sp +Generate a new user: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin user create \-\-display\-name="johnny rotten" \-\-uid=johnny +{ "user_id": "johnny", + "rados_uid": 0, + "display_name": "johnny rotten", + "email": "", + "suspended": 0, + "subusers": [], + "keys": [ + { "user": "johnny", + "access_key": "TCICW53D9BQ2VGC46I44", + "secret_key": "tfm9aHMI8X76L3UdgE+ZQaJag1vJQmE6HDb5Lbrz"}], + "swift_keys": []} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Remove a user: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin user rm \-\-uid=johnny +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Remove a user and all associated buckets with their contents: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin user rm \-\-uid=johnny \-\-purge\-data +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Remove a bucket: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin bucket unlink \-\-bucket=foo +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Show the logs of a bucket from April 1st, 2012: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin log show \-\-bucket=foo \-\-date=2012=04\-01 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Show usage information for user from March 1st to (but not including) April 1st, 2012: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin usage show \-\-uid=johnny \e + \-\-start\-date=2012\-03\-01 \-\-end\-date=2012\-04\-01 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Show only summary of usage information for all users: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin usage show \-\-show\-log\-entries=false +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Trim usage information for user until March 1st, 2012: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ radosgw\-admin usage trim \-\-uid=johnny \-\-end\-date=2012\-04\-01 +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBradosgw\-admin\fP is part of the Ceph distributed storage system. Please +refer to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/radosgw.8 b/ceph/man/radosgw.8 new file mode 100644 index 00000000..054fb793 --- /dev/null +++ b/ceph/man/radosgw.8 @@ -0,0 +1,242 @@ +.\" Man page generated from reStructuredText. +. +.TH "RADOSGW" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +radosgw \- rados REST gateway +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBradosgw\fP +.fi +.sp +.SH DESCRIPTION +.sp +\fBradosgw\fP is an HTTP REST gateway for the RADOS object store, a part +of the Ceph distributed storage system. It is implemented as a FastCGI +module using libfcgi, and can be used in conjunction with any FastCGI +capable web server. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf=ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through +\fBceph.conf\fP). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-rgw\-socket\-path=path +Specify a unix domain socket path. +.UNINDENT +.SH CONFIGURATION +.sp +Currently it\(aqs the easiest to use the RADOS Gateway with Apache and mod_fastcgi: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +FastCgiExternalServer /var/www/s3gw.fcgi \-socket /tmp/radosgw.sock + + + ServerName rgw.example1.com + ServerAlias rgw + ServerAdmin webmaster@example1.com + DocumentRoot /var/www + + RewriteEngine On + RewriteRule ^/([a\-zA\-Z0\-9\-_.]*)([/]?.*) /s3gw.fcgi?page=$1¶ms=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L] + + + + Options +ExecCGI + AllowOverride All + SetHandler fastcgi\-script + Order allow,deny + Allow from all + AuthBasicAuthoritative Off + + + + AllowEncodedSlashes On + ServerSignature Off + +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And the corresponding radosgw script (/var/www/s3gw.fcgi): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +#!/bin/sh +exec /usr/bin/radosgw \-c /etc/ceph/ceph.conf \-n client.radosgw.gateway +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The radosgw daemon is a standalone process which needs a configuration +section in the ceph.conf The section name should start with +\(aqclient.radosgw.\(aq as specified in /etc/init.d/radosgw: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[client.radosgw.gateway] + host = gateway + keyring = /etc/ceph/keyring.radosgw.gateway + rgw socket path = /tmp/radosgw.sock +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You will also have to generate a key for the radosgw to use for +authentication with the cluster: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph\-authtool \-C \-n client.radosgw.gateway \-\-gen\-key /etc/ceph/keyring.radosgw.gateway +ceph\-authtool \-n client.radosgw.gateway \-\-cap mon \(aqallow rw\(aq \-\-cap osd \(aqallow rwx\(aq /etc/ceph/keyring.radosgw.gateway +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And add the key to the auth entries: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +ceph auth add client.radosgw.gateway \-\-in\-file=keyring.radosgw.gateway +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Now you can start Apache and the radosgw daemon: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +/etc/init.d/apache2 start +/etc/init.d/radosgw start +.ft P +.fi +.UNINDENT +.UNINDENT +.SH USAGE LOGGING +.sp +The \fBradosgw\fP maintains an asynchronous usage log. It accumulates +statistics about user operations and flushes it periodically. The +logs can be accessed and managed through \fBradosgw\-admin\fP\&. +.sp +The information that is being logged contains total data transfer, +total operations, and total successful operations. The data is being +accounted in an hourly resolution under the bucket owner, unless the +operation was done on the service (e.g., when listing a bucket) in +which case it is accounted under the operating user. +.sp +Following is an example configuration: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[client.radosgw.gateway] + rgw enable usage log = true + rgw usage log tick interval = 30 + rgw usage log flush threshold = 1024 + rgw usage max shards = 32 + rgw usage max user shards = 1 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The total number of shards determines how many total objects hold the +usage log information. The per\-user number of shards specify how many +objects hold usage information for a single user. The tick interval +configures the number of seconds between log flushes, and the flush +threshold specify how many entries can be kept before resorting to +synchronous flush. +.SH AVAILABILITY +.sp +\fBradosgw\fP is part of the Ceph distributed storage system. Please refer +to the Ceph documentation at \fI\%http://ceph.com/docs\fP for more +information. +.SH SEE ALSO +.sp +\fBceph\fP(8) +\fBradosgw\-admin\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/rbd-fuse.8 b/ceph/man/rbd-fuse.8 new file mode 100644 index 00000000..3ba0636e --- /dev/null +++ b/ceph/man/rbd-fuse.8 @@ -0,0 +1,110 @@ +.\" Man page generated from reStructuredText. +. +.TH "RBD-FUSE" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +rbd-fuse \- expose rbd images as files +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBrbd\-fuse\fP [ \-p pool ] [\-c conffile] \fImountpoint\fP [ \fIfuse options\fP ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBrbd\-fuse\fP is a FUSE (File system in USErspace) client for RADOS +block device (rbd) images. Given a pool containing rbd images, +it will mount a userspace filesystem allowing access to those images +as regular files at \fBmountpoint\fP\&. +.sp +The file system can be unmounted with: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +fusermount \-u mountpoint +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +or by sending \fBSIGINT\fP to the \fBrbd\-fuse\fP process. +.SH OPTIONS +.sp +Any options not recognized by rbd\-fuse will be passed on to libfuse. +.INDENT 0.0 +.TP +.B \-c ceph.conf +Use \fIceph.conf\fP configuration file instead of the default +\fB/etc/ceph/ceph.conf\fP to determine monitor addresses during startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-p pool +Use \fIpool\fP as the pool to search for rbd images. Default is \fBrbd\fP\&. +.UNINDENT +.SH AVAILABILITY +.sp +\fBrbd\-fuse\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +fusermount(8), +\fBrbd\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/man/rbd.8 b/ceph/man/rbd.8 new file mode 100644 index 00000000..1bec65dd --- /dev/null +++ b/ceph/man/rbd.8 @@ -0,0 +1,616 @@ +.\" Man page generated from reStructuredText. +. +.TH "RBD" "8" "January 12, 2014" "dev" "Ceph" +.SH NAME +rbd \- manage rados block device (RBD) images +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +\fBrbd\fP [ \-c \fIceph.conf\fP ] [ \-m \fImonaddr\fP ] [ \-p | \-\-pool \fIpool\fP ] [ +\-\-size \fIsize\fP ] [ \-\-order \fIbits\fP ] [ \fIcommand\fP ... ] +.fi +.sp +.SH DESCRIPTION +.sp +\fBrbd\fP is a utility for manipulating rados block device (RBD) images, +used by the Linux rbd driver and the rbd storage driver for Qemu/KVM. +RBD images are simple block devices that are striped over objects and +stored in a RADOS object store. The size of the objects the image is +striped over must be a power of two. +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-c ceph.conf, \-\-conf ceph.conf +Use ceph.conf configuration file instead of the default /etc/ceph/ceph.conf to +determine monitor addresses during startup. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m monaddress[:port] +Connect to specified monitor (instead of looking through ceph.conf). +.UNINDENT +.INDENT 0.0 +.TP +.B \-p pool, \-\-pool pool +Interact with the given pool. Required by most commands. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-progress +Do not output progress information (goes to standard error by +default for some commands). +.UNINDENT +.SH PARAMETERS +.INDENT 0.0 +.TP +.B \-\-image\-format format +Specifies which object layout to use. The default is 1. +.INDENT 7.0 +.IP \(bu 2 +format 1 \- Use the original format for a new rbd image. This format is +understood by all versions of librbd and the kernel rbd module, but +does not support newer features like cloning. +.IP \(bu 2 +format 2 \- Use the second rbd format, which is supported by +librbd (but not the kernel rbd module) at this time. This adds +support for cloning and is more easily extensible to allow more +features in the future. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-size size\-in\-mb +Specifies the size (in megabytes) of the new rbd image. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-order bits +Specifies the object size expressed as a number of bits, such that +the object size is \fB1 << order\fP\&. The default is 22 (4 MB). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-stripe\-unit size\-in\-bytes +Specifies the stripe unit size in bytes. See striping section (below) for more details. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-stripe\-count num +Specifies the number of objects to stripe over before looping back +to the first object. See striping section (below) for more details. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-snap snap +Specifies the snapshot name for the specific operation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-id username +Specifies the username (without the \fBclient.\fP prefix) to use with the map command. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keyfile filename +Specifies a file containing the secret to use with the map command. +If not specified, \fBclient.admin\fP will be used by default. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keyring filename +Specifies a keyring file containing a secret for the specified user +to use with the map command. If not specified, the default keyring +locations will be searched. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-shared tag +Option for \fIlock add\fP that allows multiple clients to lock the +same image if they use the same tag. The tag is an arbitrary +string. This is useful for situations where an image must +be open from more than one client at once, like during +live migration of a virtual machine, or for use underneath +a clustered filesystem. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-format format +Specifies output formatting (default: plain, json, xml) +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pretty\-format +Make json or xml formatted output more human\-readable. +.UNINDENT +.INDENT 0.0 +.TP +.B \-o map\-options, \-\-options map\-options +Specifies which options to use when mapping an image. map\-options is +a comma\-separated string of options (similar to mount(8) mount options). +See map options section below for more details. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-read\-only +Map the image read\-only. Equivalent to \-o ro. +.UNINDENT +.SH COMMANDS +.INDENT 0.0 +.TP +.B \fBls\fP [\-l | \-\-long] [pool\-name] +Will list all rbd images listed in the rbd_directory object. With +\-l, also show snapshots, and use longer\-format output including +size, parent (if clone), format, etc. +.TP +.B \fBinfo\fP [\fIimage\-name\fP] +Will dump information (such as size and order) about a specific rbd image. +If image is a clone, information about its parent is also displayed. +If a snapshot is specified, whether it is protected is shown as well. +.TP +.B \fBcreate\fP [\fIimage\-name\fP] +Will create a new rbd image. You must also specify the size via \-\-size. The +\-\-stripe\-unit and \-\-stripe\-count arguments are optional, but must be used together. +.TP +.B \fBclone\fP [\fIparent\-snapname\fP] [\fIimage\-name\fP] +Will create a clone (copy\-on\-write child) of the parent snapshot. +Object order will be identical to that of the parent image unless +specified. Size will be the same as the parent snapshot. +.sp +The parent snapshot must be protected (see \fIrbd snap protect\fP). +This requires image format 2. +.TP +.B \fBflatten\fP [\fIimage\-name\fP] +If image is a clone, copy all shared blocks from the parent snapshot and +make the child independent of the parent, severing the link between +parent snap and child. The parent snapshot can be unprotected and +deleted if it has no further dependent clones. +.sp +This requires image format 2. +.TP +.B \fBchildren\fP [\fIimage\-name\fP] +List the clones of the image at the given snapshot. This checks +every pool, and outputs the resulting poolname/imagename. +.sp +This requires image format 2. +.TP +.B \fBresize\fP [\fIimage\-name\fP] [\-\-allow\-shrink] +Resizes rbd image. The size parameter also needs to be specified. +The \-\-allow\-shrink option lets the size be reduced. +.TP +.B \fBrm\fP [\fIimage\-name\fP] +Deletes an rbd image (including all data blocks). If the image has +snapshots, this fails and nothing is deleted. +.TP +.B \fBexport\fP [\fIimage\-name\fP] [\fIdest\-path\fP] +Exports image to dest path (use \- for stdout). +.TP +.B \fBimport\fP [\fIpath\fP] [\fIdest\-image\fP] +Creates a new image and imports its data from path (use \- for +stdin). The import operation will try to create sparse rbd images +if possible. For import from stdin, the sparsification unit is +the data block size of the destination image (1 << order). +.TP +.B \fBexport\-diff\fP [\fIimage\-name\fP] [\fIdest\-path\fP] [\-\-from\-snap \fIsnapname\fP] +Exports an incremental diff for an image to dest path (use \- for stdout). If +an initial snapshot is specified, only changes since that snapshot are included; otherwise, +any regions of the image that contain data are included. The end snapshot is specified +using the standard \-\-snap option or @snap syntax (see below). The image diff format includes +metadata about image size changes, and the start and end snapshots. It efficiently represents +discarded or \(aqzero\(aq regions of the image. +.TP +.B \fBimport\-diff\fP [\fIsrc\-path\fP] [\fIimage\-name\fP] +Imports an incremental diff of an image and applies it to the current image. If the diff +was generated relative to a start snapshot, we verify that snapshot already exists before +continuing. If there was an end snapshot we verify it does not already exist before +applying the changes, and create the snapshot when we are done. +.TP +.B \fBdiff\fP [\fIimage\-name\fP] [\-\-from\-snap \fIsnapname\fP] +Dump a list of byte extents in the image that have changed since the specified start +snapshot, or since the image was created. Each output line includes the starting offset +(in bytes), the length of the region (in bytes), and either \(aqzero\(aq or \(aqdata\(aq to indicate +whether the region is known to be zeros or may contain other data. +.TP +.B \fBcp\fP [\fIsrc\-image\fP] [\fIdest\-image\fP] +Copies the content of a src\-image into the newly created dest\-image. +dest\-image will have the same size, order, and image format as src\-image. +.TP +.B \fBmv\fP [\fIsrc\-image\fP] [\fIdest\-image\fP] +Renames an image. Note: rename across pools is not supported. +.TP +.B \fBsnap\fP ls [\fIimage\-name\fP] +Dumps the list of snapshots inside a specific image. +.TP +.B \fBsnap\fP create [\fIimage\-name\fP] +Creates a new snapshot. Requires the snapshot name parameter specified. +.TP +.B \fBsnap\fP rollback [\fIimage\-name\fP] +Rollback image content to snapshot. This will iterate through the entire blocks +array and update the data head content to the snapshotted version. +.TP +.B \fBsnap\fP rm [\fIimage\-name\fP] +Removes the specified snapshot. +.TP +.B \fBsnap\fP purge [\fIimage\-name\fP] +Removes all snapshots from an image. +.TP +.B \fBsnap\fP protect [\fIimage\-name\fP] +Protect a snapshot from deletion, so that clones can be made of it +(see \fIrbd clone\fP). Snapshots must be protected before clones are made; +protection implies that there exist dependent cloned children that +refer to this snapshot. \fIrbd clone\fP will fail on a nonprotected +snapshot. +.sp +This requires image format 2. +.TP +.B \fBsnap\fP unprotect [\fIimage\-name\fP] +Unprotect a snapshot from deletion (undo \fIsnap protect\fP). If cloned +children remain, \fIsnap unprotect\fP fails. (Note that clones may exist +in different pools than the parent snapshot.) +.sp +This requires image format 2. +.TP +.B \fBmap\fP [\fIimage\-name\fP] [\-o | \-\-options \fImap\-options\fP ] [\-\-read\-only] +Maps the specified image to a block device via the rbd kernel module. +.TP +.B \fBunmap\fP [\fIdevice\-path\fP] +Unmaps the block device that was mapped via the rbd kernel module. +.TP +.B \fBshowmapped\fP +Show the rbd images that are mapped via the rbd kernel module. +.TP +.B \fBlock\fP list [\fIimage\-name\fP] +Show locks held on the image. The first column is the locker +to use with the \fIlock remove\fP command. +.TP +.B \fBlock\fP add [\fIimage\-name\fP] [\fIlock\-id\fP] +Lock an image. The lock\-id is an arbitrary name for the user\(aqs +convenience. By default, this is an exclusive lock, meaning it +will fail if the image is already locked. The \-\-shared option +changes this behavior. Note that locking does not affect +any operation other than adding a lock. It does not +protect an image from being deleted. +.TP +.B \fBlock\fP remove [\fIimage\-name\fP] [\fIlock\-id\fP] [\fIlocker\fP] +Release a lock on an image. The lock id and locker are +as output by lock ls. +.TP +.B \fBbench\-write\fP [\fIimage\-name\fP] \-\-io\-size [\fIio\-size\-in\-bytes\fP] \-\-io\-threads [\fInum\-ios\-in\-flight\fP] \-\-io\-total [\fItotal\-bytes\-to\-write\fP] +Generate a series of sequential writes to the image and measure the +write throughput and latency. Defaults are: \-\-io\-size 4096, \-\-io\-threads 16, +\-\-io\-total 1GB +.UNINDENT +.SH IMAGE NAME +.sp +In addition to using the \-\-pool and the \-\-snap options, the image name can include both +the pool name and the snapshot name. The image name format is as follows: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[pool/]image\-name[@snap] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Thus an image name that contains a slash character (\(aq/\(aq) requires specifying the pool +name explicitly. +.SH STRIPING +.sp +RBD images are striped over many objects, which are then stored by the +Ceph distributed object store (RADOS). As a result, read and write +requests for the image are distributed across many nodes in the +cluster, generally preventing any single node from becoming a +bottleneck when individual images get large or busy. +.sp +The striping is controlled by three parameters: +.INDENT 0.0 +.TP +.B order +.TP +.B The size of objects we stripe over is a power of two, specifially 2^[*order*] bytes. The default +.TP +.B is 22, or 4 MB. +.UNINDENT +.INDENT 0.0 +.TP +.B stripe_unit +.TP +.B Each [*stripe_unit*] contiguous bytes are stored adjacently in the same object, before we move on +.TP +.B to the next object. +.UNINDENT +.INDENT 0.0 +.TP +.B stripe_count +.TP +.B After we write [*stripe_unit*] bytes to [*stripe_count*] objects, we loop back to the initial object +.TP +.B and write another stripe, until the object reaches its maximum size (as specified by [*order*]. At that +.TP +.B point, we move on to the next [*stripe_count*] objects. +.UNINDENT +.sp +By default, [\fIstripe_unit\fP] is the same as the object size and [\fIstripe_count\fP] is 1. Specifying a different +[\fIstripe_unit\fP] requires that the STRIPINGV2 feature be supported (added in Ceph v0.53) and format 2 images be +used. +.SH MAP OPTIONS +.sp +Most of these options are useful mainly for debugging and benchmarking. The +default values are set in the kernel and may therefore depend on the version of +the running kernel. +.INDENT 0.0 +.IP \(bu 2 +fsid=aaaaaaaa\-bbbb\-cccc\-dddd\-eeeeeeeeeeee \- FSID that should be assumed by +the client. +.IP \(bu 2 +ip=a.b.c.d[:p] \- IP and, optionally, port the client should use. +.IP \(bu 2 +share \- Enable sharing of client instances with other mappings (default). +.IP \(bu 2 +noshare \- Disable sharing of client instances with other mappings. +.IP \(bu 2 +crc \- Enable CRC32C checksumming for data writes (default). +.IP \(bu 2 +nocrc \- Disable CRC32C checksumming for data writes. +.IP \(bu 2 +osdkeepalive=x \- OSD keepalive timeout (default is 5 seconds). +.IP \(bu 2 +osd_idle_ttl=x \- OSD idle TTL (default is 60 seconds). +.IP \(bu 2 +rw \- Map the image read\-write (default). +.IP \(bu 2 +ro \- Map the image read\-only. Equivalent to \-\-read\-only. +.UNINDENT +.SH EXAMPLES +.sp +To create a new rbd image that is 100 GB: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd \-p mypool create myimage \-\-size 102400 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +or alternatively: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd create mypool/myimage \-\-size 102400 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To use a non\-default object size (8 MB): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd create mypool/myimage \-\-size 102400 \-\-order 23 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To delete an rbd image (be careful!): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd rm mypool/myimage +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To create a new snapshot: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd snap create mypool/myimage@mysnap +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To create a copy\-on\-write clone of a protected snapshot: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd clone mypool/myimage@mysnap otherpool/cloneimage +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To see which clones of a snapshot exist: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd children mypool/myimage@mysnap +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To delete a snapshot: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd snap rm mypool/myimage@mysnap +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To map an image via the kernel with cephx enabled: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd map mypool/myimage \-\-id admin \-\-keyfile secretfile +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To unmap an image: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd unmap /dev/rbd0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To create an image and a clone from it: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd import \-\-image\-format 2 image mypool/parent +rbd snap create \-\-snap snapname mypool/parent +rbd snap protect mypool/parent@snap +rbd clone mypool/parent@snap otherpool/child +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To create an image with a smaller stripe_unit (to better distribute small writes in some workloads): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd \-p mypool create myimage \-\-size 102400 \-\-stripe\-unit 65536 \-\-stripe\-count 16 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To change an image from one image format to another, export it and then +import it as the desired image format: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd export mypool/myimage@snap /tmp/img +rbd import \-\-image\-format 2 /tmp/img mypool/myimage2 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To lock an image for exclusive use: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd lock add mypool/myimage mylockid +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To release a lock: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rbd lock remove mypool/myimage mylockid client.2485 +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AVAILABILITY +.sp +\fBrbd\fP is part of the Ceph distributed storage system. Please refer to +the Ceph documentation at \fI\%http://ceph.com/docs\fP for more information. +.SH SEE ALSO +.sp +\fBceph\fP(8), +\fBrados\fP(8) +.SH COPYRIGHT +2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA +.\" Generated by docutils manpage writer. +. diff --git a/ceph/missing b/ceph/missing new file mode 100755 index 00000000..86a8fc31 --- /dev/null +++ b/ceph/missing @@ -0,0 +1,331 @@ +#! /bin/sh +# Common stub for a few missing GNU programs while installing. + +scriptversion=2012-01-06.13; # UTC + +# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006, +# 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. +# Originally by Fran,cois Pinard , 1996. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +if test $# -eq 0; then + echo 1>&2 "Try \`$0 --help' for more information" + exit 1 +fi + +run=: +sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p' +sed_minuso='s/.* -o \([^ ]*\).*/\1/p' + +# In the cases where this matters, `missing' is being run in the +# srcdir already. +if test -f configure.ac; then + configure_ac=configure.ac +else + configure_ac=configure.in +fi + +msg="missing on your system" + +case $1 in +--run) + # Try to run requested program, and just exit if it succeeds. + run= + shift + "$@" && exit 0 + # Exit code 63 means version mismatch. This often happens + # when the user try to use an ancient version of a tool on + # a file that requires a minimum version. In this case we + # we should proceed has if the program had been absent, or + # if --run hadn't been passed. + if test $? = 63; then + run=: + msg="probably too old" + fi + ;; + + -h|--h|--he|--hel|--help) + echo "\ +$0 [OPTION]... PROGRAM [ARGUMENT]... + +Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an +error status if there is no known handling for PROGRAM. + +Options: + -h, --help display this help and exit + -v, --version output version information and exit + --run try to run the given command, and emulate it if it fails + +Supported PROGRAM values: + aclocal touch file \`aclocal.m4' + autoconf touch file \`configure' + autoheader touch file \`config.h.in' + autom4te touch the output file, or create a stub one + automake touch all \`Makefile.in' files + bison create \`y.tab.[ch]', if possible, from existing .[ch] + flex create \`lex.yy.c', if possible, from existing .c + help2man touch the output file + lex create \`lex.yy.c', if possible, from existing .c + makeinfo touch the output file + yacc create \`y.tab.[ch]', if possible, from existing .[ch] + +Version suffixes to PROGRAM as well as the prefixes \`gnu-', \`gnu', and +\`g' are ignored when checking the name. + +Send bug reports to ." + exit $? + ;; + + -v|--v|--ve|--ver|--vers|--versi|--versio|--version) + echo "missing $scriptversion (GNU Automake)" + exit $? + ;; + + -*) + echo 1>&2 "$0: Unknown \`$1' option" + echo 1>&2 "Try \`$0 --help' for more information" + exit 1 + ;; + +esac + +# normalize program name to check for. +program=`echo "$1" | sed ' + s/^gnu-//; t + s/^gnu//; t + s/^g//; t'` + +# Now exit if we have it, but it failed. Also exit now if we +# don't have it and --version was passed (most likely to detect +# the program). This is about non-GNU programs, so use $1 not +# $program. +case $1 in + lex*|yacc*) + # Not GNU programs, they don't have --version. + ;; + + *) + if test -z "$run" && ($1 --version) > /dev/null 2>&1; then + # We have it, but it failed. + exit 1 + elif test "x$2" = "x--version" || test "x$2" = "x--help"; then + # Could not run --version or --help. This is probably someone + # running `$TOOL --version' or `$TOOL --help' to check whether + # $TOOL exists and not knowing $TOOL uses missing. + exit 1 + fi + ;; +esac + +# If it does not exist, or fails to run (possibly an outdated version), +# try to emulate it. +case $program in + aclocal*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`acinclude.m4' or \`${configure_ac}'. You might want + to install the \`Automake' and \`Perl' packages. Grab them from + any GNU archive site." + touch aclocal.m4 + ;; + + autoconf*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`${configure_ac}'. You might want to install the + \`Autoconf' and \`GNU m4' packages. Grab them from any GNU + archive site." + touch configure + ;; + + autoheader*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`acconfig.h' or \`${configure_ac}'. You might want + to install the \`Autoconf' and \`GNU m4' packages. Grab them + from any GNU archive site." + files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}` + test -z "$files" && files="config.h" + touch_files= + for f in $files; do + case $f in + *:*) touch_files="$touch_files "`echo "$f" | + sed -e 's/^[^:]*://' -e 's/:.*//'`;; + *) touch_files="$touch_files $f.in";; + esac + done + touch $touch_files + ;; + + automake*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'. + You might want to install the \`Automake' and \`Perl' packages. + Grab them from any GNU archive site." + find . -type f -name Makefile.am -print | + sed 's/\.am$/.in/' | + while read f; do touch "$f"; done + ;; + + autom4te*) + echo 1>&2 "\ +WARNING: \`$1' is needed, but is $msg. + You might have modified some files without having the + proper tools for further handling them. + You can get \`$1' as part of \`Autoconf' from any GNU + archive site." + + file=`echo "$*" | sed -n "$sed_output"` + test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` + if test -f "$file"; then + touch $file + else + test -z "$file" || exec >$file + echo "#! /bin/sh" + echo "# Created by GNU Automake missing as a replacement of" + echo "# $ $@" + echo "exit 0" + chmod +x $file + exit 1 + fi + ;; + + bison*|yacc*) + echo 1>&2 "\ +WARNING: \`$1' $msg. You should only need it if + you modified a \`.y' file. You may need the \`Bison' package + in order for those modifications to take effect. You can get + \`Bison' from any GNU archive site." + rm -f y.tab.c y.tab.h + if test $# -ne 1; then + eval LASTARG=\${$#} + case $LASTARG in + *.y) + SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'` + if test -f "$SRCFILE"; then + cp "$SRCFILE" y.tab.c + fi + SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'` + if test -f "$SRCFILE"; then + cp "$SRCFILE" y.tab.h + fi + ;; + esac + fi + if test ! -f y.tab.h; then + echo >y.tab.h + fi + if test ! -f y.tab.c; then + echo 'main() { return 0; }' >y.tab.c + fi + ;; + + lex*|flex*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified a \`.l' file. You may need the \`Flex' package + in order for those modifications to take effect. You can get + \`Flex' from any GNU archive site." + rm -f lex.yy.c + if test $# -ne 1; then + eval LASTARG=\${$#} + case $LASTARG in + *.l) + SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'` + if test -f "$SRCFILE"; then + cp "$SRCFILE" lex.yy.c + fi + ;; + esac + fi + if test ! -f lex.yy.c; then + echo 'main() { return 0; }' >lex.yy.c + fi + ;; + + help2man*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified a dependency of a manual page. You may need the + \`Help2man' package in order for those modifications to take + effect. You can get \`Help2man' from any GNU archive site." + + file=`echo "$*" | sed -n "$sed_output"` + test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` + if test -f "$file"; then + touch $file + else + test -z "$file" || exec >$file + echo ".ab help2man is required to generate this page" + exit $? + fi + ;; + + makeinfo*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified a \`.texi' or \`.texinfo' file, or any other file + indirectly affecting the aspect of the manual. The spurious + call might also be the consequence of using a buggy \`make' (AIX, + DU, IRIX). You might want to install the \`Texinfo' package or + the \`GNU make' package. Grab either from any GNU archive site." + # The file to touch is that specified with -o ... + file=`echo "$*" | sed -n "$sed_output"` + test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` + if test -z "$file"; then + # ... or it is the one specified with @setfilename ... + infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'` + file=`sed -n ' + /^@setfilename/{ + s/.* \([^ ]*\) *$/\1/ + p + q + }' $infile` + # ... or it is derived from the source name (dir/f.texi becomes f.info) + test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info + fi + # If the file does not exist, the user really needs makeinfo; + # let's fail without touching anything. + test -f $file || exit 1 + touch $file + ;; + + *) + echo 1>&2 "\ +WARNING: \`$1' is needed, and is $msg. + You might have modified some files without having the + proper tools for further handling them. Check the \`README' file, + it often tells you about the needed prerequisites for installing + this package. You may also peek at any GNU archive site, in case + some other package would contain this missing \`$1' program." + exit 1 + ;; +esac + +exit 0 + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/ceph/py-compile b/ceph/py-compile new file mode 100755 index 00000000..15c834c4 --- /dev/null +++ b/ceph/py-compile @@ -0,0 +1,161 @@ +#!/bin/sh +# py-compile - Compile a Python program + +scriptversion=2011-06-08.12; # UTC + +# Copyright (C) 2000, 2001, 2003, 2004, 2005, 2008, 2009, 2011 Free +# Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +if [ -z "$PYTHON" ]; then + PYTHON=python +fi + +me=py-compile + +usage_error () +{ + echo "$me: $*" >&2 + echo "Try \`$me --help' for more information." >&2 + exit 1 +} + +basedir= +destdir= +while test $# -ne 0; do + case "$1" in + --basedir) + if test $# -lt 2; then + usage_error "option '--basedir' requires an argument" + else + basedir=$2 + fi + shift + ;; + --destdir) + if test $# -lt 2; then + usage_error "option '--destdir' requires an argument" + else + destdir=$2 + fi + shift + ;; + -h|--help) + cat <<\EOF +Usage: py-compile [--help] [--version] [--basedir DIR] [--destdir DIR] FILES..." + +Byte compile some python scripts FILES. Use --destdir to specify any +leading directory path to the FILES that you don't want to include in the +byte compiled file. Specify --basedir for any additional path information you +do want to be shown in the byte compiled file. + +Example: + py-compile --destdir /tmp/pkg-root --basedir /usr/share/test test.py test2.py + +Report bugs to . +EOF + exit $? + ;; + -v|--version) + echo "$me $scriptversion" + exit $? + ;; + --) + shift + break + ;; + -*) + usage_error "unrecognized option '$1'" + ;; + *) + break + ;; + esac + shift +done + +files=$* +if test -z "$files"; then + usage_error "no files given" +fi + +# if basedir was given, then it should be prepended to filenames before +# byte compilation. +if [ -z "$basedir" ]; then + pathtrans="path = file" +else + pathtrans="path = os.path.join('$basedir', file)" +fi + +# if destdir was given, then it needs to be prepended to the filename to +# byte compile but not go into the compiled file. +if [ -z "$destdir" ]; then + filetrans="filepath = path" +else + filetrans="filepath = os.path.normpath('$destdir' + os.sep + path)" +fi + +$PYTHON -c " +import sys, os, py_compile + +files = '''$files''' + +sys.stdout.write('Byte-compiling python modules...\n') +for file in files.split(): + $pathtrans + $filetrans + if not os.path.exists(filepath) or not (len(filepath) >= 3 + and filepath[-3:] == '.py'): + continue + sys.stdout.write(file) + sys.stdout.flush() + py_compile.compile(filepath, filepath + 'c', path) +sys.stdout.write('\n')" || exit $? + +# this will fail for python < 1.5, but that doesn't matter ... +$PYTHON -O -c " +import sys, os, py_compile + +files = '''$files''' +sys.stdout.write('Byte-compiling python modules (optimized versions) ...\n') +for file in files.split(): + $pathtrans + $filetrans + if not os.path.exists(filepath) or not (len(filepath) >= 3 + and filepath[-3:] == '.py'): + continue + sys.stdout.write(file) + sys.stdout.flush() + py_compile.compile(filepath, filepath + 'o', path) +sys.stdout.write('\n')" 2>/dev/null || : + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/ceph/share/id_dsa_drop.ceph.com b/ceph/share/id_dsa_drop.ceph.com new file mode 100644 index 00000000..3efc985a --- /dev/null +++ b/ceph/share/id_dsa_drop.ceph.com @@ -0,0 +1,12 @@ +-----BEGIN DSA PRIVATE KEY----- +MIIBugIBAAKBgQDv8F/WToUDOc2HRWUOqtq5ilORE+5P53yZUo7ugr8XD3wM0H7Q +IIl9F9fizwUtL2gh3n1BnBxmPhkVU6VYsiDpn1P3dWvRmf+jyqPuk+b185L0Erb8 +QsExADv6v33Yyd+9i5oTI988Rm1VWY6QhP7neW6yMPt2noi1TwleLm6z2wIVAKHL +ciT2S0w/dbTFQDFHSEOCAif3AoGAHwOYd8YEInrcBrXPFJuPFbQKr8ceO3/ItY0r +/W/L92nXUJbdl1JEt2KfkdwaxkBhlYT7E1JR5MRoTNBTEMCFjHxemZCdH+03+Jzq ++RAQ28p77przbqOFaMuZuQoGlqMy3gYrhnPRGEJGjh+pkhMePqUPCCKFtRntNzlH +lDh4uOACgYBLGpqu3Pthhd4fnawv8Md16gc/p1Vg/5vyAzi9Gshhgf1hXvFHdeJv +AN/5mgE/Ekg7fqeNUhui9LYkuuOMgP267naGkAAgxV3bbiy439Vj8SzXdOQk4agA +YgebWkmJrdMtUSzeBYBkqBZTZODvQwCmYdR6INuNuZtA+rHgKwiAHQIUZak7aJD8 +y4kap9GmduDYmp6/JxU= +-----END DSA PRIVATE KEY----- diff --git a/ceph/share/id_dsa_drop.ceph.com.pub b/ceph/share/id_dsa_drop.ceph.com.pub new file mode 100644 index 00000000..e7e53834 --- /dev/null +++ b/ceph/share/id_dsa_drop.ceph.com.pub @@ -0,0 +1 @@ +ssh-dss AAAAB3NzaC1kc3MAAACBAO/wX9ZOhQM5zYdFZQ6q2rmKU5ET7k/nfJlSju6CvxcPfAzQftAgiX0X1+LPBS0vaCHefUGcHGY+GRVTpViyIOmfU/d1a9GZ/6PKo+6T5vXzkvQStvxCwTEAO/q/fdjJ372LmhMj3zxGbVVZjpCE/ud5brIw+3aeiLVPCV4ubrPbAAAAFQChy3Ik9ktMP3W0xUAxR0hDggIn9wAAAIAfA5h3xgQietwGtc8Um48VtAqvxx47f8i1jSv9b8v3addQlt2XUkS3Yp+R3BrGQGGVhPsTUlHkxGhM0FMQwIWMfF6ZkJ0f7Tf4nOr5EBDbynvumvNuo4Voy5m5CgaWozLeBiuGc9EYQkaOH6mSEx4+pQ8IIoW1Ge03OUeUOHi44AAAAIBLGpqu3Pthhd4fnawv8Md16gc/p1Vg/5vyAzi9Gshhgf1hXvFHdeJvAN/5mgE/Ekg7fqeNUhui9LYkuuOMgP267naGkAAgxV3bbiy439Vj8SzXdOQk4agAYgebWkmJrdMtUSzeBYBkqBZTZODvQwCmYdR6INuNuZtA+rHgKwiAHQ== public_ceph_post_key_2013-08-16 diff --git a/ceph/share/known_hosts_drop.ceph.com b/ceph/share/known_hosts_drop.ceph.com new file mode 100644 index 00000000..862df67a --- /dev/null +++ b/ceph/share/known_hosts_drop.ceph.com @@ -0,0 +1 @@ +drop.ceph.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCjl2XzIpS92snr4SymcBVQx7y6d/ZjzCKJAlGZYkgknWWf+JBEpXp+cOoMk2Td5pIPkOdn72pGAuuPzL9HSJpN/o75tzbv0mAd//3t9D5/Kpnd+tWIDDgj+RIz8ZKRxSR8lnVjsUHlTrwQwaUkZ3KDiVgtQXDp0+1rU1+02cEkeBStoMLQt/6xw1hmPGSIAMH2HRkyge+/I8RwK7jbTwwcxh61Vxe0qMGkDO6vUVXw+K6hoXV4uGaqZ9/B2GirXJPz6ulvLC/mtEdgtfKS3eiMTaJS5Cpne6rJw2Wm7kHfQPstJaUq06BJiRe6R+JHC897NVZd0yc1bZe+BI0PmQJL diff --git a/ceph/src/.git_version b/ceph/src/.git_version new file mode 100644 index 00000000..1727fed6 --- /dev/null +++ b/ceph/src/.git_version @@ -0,0 +1,2 @@ +6c0127fcb58008793d3c8b62d925bc91963672a3 +v0.80.7 diff --git a/ceph/src/Makefile-env.am b/ceph/src/Makefile-env.am new file mode 100644 index 00000000..d62247b3 --- /dev/null +++ b/ceph/src/Makefile-env.am @@ -0,0 +1,198 @@ +AUTOMAKE_OPTIONS = gnu subdir-objects + +SUBDIRS = +DIST_SUBDIRS = +BUILT_SOURCES = +EXTRA_DIST = +CLEANFILES = + +noinst_HEADERS = +bin_PROGRAMS = +noinst_PROGRAMS = +bin_SCRIPTS = +sbin_PROGRAMS = +sbin_SCRIPTS = +su_sbin_PROGRAMS = +su_sbin_SCRIPTS = +dist_bin_SCRIPTS = +lib_LTLIBRARIES = +noinst_LTLIBRARIES = +noinst_LIBRARIES = +radoslib_LTLIBRARIES = + +# like bin_PROGRAMS, but these targets are only built for debug builds +bin_DEBUGPROGRAMS = + +# like sbin_SCRIPTS but can be used to install to e.g. /usr/sbin +ceph_sbindir = $(sbindir) + +# certain things go straight into /sbin, though! +su_sbindir = /sbin + +# C/C++ tests to build will be appended to this +check_PROGRAMS = + +# tests scripts will be appended to this +check_SCRIPTS = + +# python unit tests need to know where the scripts are located +export PYTHONPATH=$(top_srcdir)/src/pybind + +# when doing a debug build, make sure to make the targets +if WITH_DEBUG +bin_PROGRAMS += $(bin_DEBUGPROGRAMS) +endif + + +################################## +## automake environment + +AM_COMMON_CPPFLAGS = \ + -D__CEPH__ \ + -D_FILE_OFFSET_BITS=64 \ + -D_REENTRANT \ + -D_THREAD_SAFE \ + -D__STDC_FORMAT_MACROS \ + -D_GNU_SOURCE \ + -DCEPH_LIBDIR=\"${libdir}\" \ + -DCEPH_PKGLIBDIR=\"${pkglibdir}\" \ + -DGTEST_HAS_TR1_TUPLE=0 + +AM_COMMON_CFLAGS = \ + -Wall \ + ${WARN_TYPE_LIMITS} \ + ${WARN_IGNORED_QUALIFIERS} \ + -Winit-self \ + -Wpointer-arith \ + -Werror=format-security \ + -fno-strict-aliasing \ + -fsigned-char +if !CLANG + AM_COMMON_CFLAGS += -rdynamic +endif + +AM_CFLAGS = $(AM_COMMON_CFLAGS) +AM_CPPFLAGS = $(AM_COMMON_CPPFLAGS) +AM_CXXFLAGS = \ + @AM_CXXFLAGS@ \ + $(AM_COMMON_CFLAGS) \ + -ftemplate-depth-1024 \ + -Wnon-virtual-dtor \ + -Wno-invalid-offsetof +if !CLANG + AM_CXXFLAGS += -Wstrict-null-sentinel +endif + +# note: this is position dependant, it affects the -l options that +# come after it on the command line. when you use ${AM_LDFLAGS} in +# later rules, take care where you place it. for more information, see +# http://blog.flameeyes.eu/2008/11/19/relationship-between-as-needed-and-no-undefined-part-1-what-do-they-do +# http://blog.flameeyes.eu/2008/11/20/misguided-link-and-as-needed +# http://www.gentoo.org/proj/en/qa/asneeded.xml +# http://gcc.gnu.org/ml/gcc-help/2010-12/msg00338.html +# http://sigquit.wordpress.com/2011/02/16/why-asneeded-doesnt-work-as-expected-for-your-libraries-on-your-autotools-project/ +AM_LDFLAGS = +if LINUX +AM_LDFLAGS += -Wl,--as-needed +endif + +if USE_BOOST_SPIRIT_OLD_HDR +AM_CXXFLAGS += -DUSE_BOOST_SPIRIT_OLD_HDR +endif + +if WITH_LIBATOMIC +AM_LDFLAGS += -latomic_ops +endif + +if ENABLE_COVERAGE +AM_CFLAGS += -fprofile-arcs -ftest-coverage +AM_CXXFLAGS += -fprofile-arcs -ftest-coverage -O0 +endif + +CCAS = ${srcdir}/yasm-wrapper +AM_CCASFLAGS = -f elf64 + + +##################### +## library definitions and dependencies + +EXTRALIBS = -luuid -lm +if FREEBSD +EXTRALIBS += -lexecinfo +endif # FREEBSD + +if LINUX +EXTRALIBS += -lrt +endif # LINUX + +if WITH_PROFILER +EXTRALIBS += -lprofiler +endif # PROFILER + +LIBGLOBAL = libglobal.la +LIBCOMMON = libcommon.la +LIBARCH = libarch.la +LIBPERFGLUE = libperfglue.la +LIBAUTH = libauth.la +LIBMSG = libmsg.la +LIBCRUSH = libcrush.la +LIBJSON_SPIRIT = libjson_spirit.la +LIBLOG = liblog.la +LIBOS = libos.la +LIBOS_TYPES = libos_types.la +LIBOSD = libosd.la +LIBOSD_TYPES = libosd_types.la +LIBOSDC = libosdc.la +LIBMON = libmon.la +LIBMON_TYPES = libmon_types.la +LIBMDS = libmds.la +LIBCLIENT = libclient.la +LIBCLIENT_FUSE = libclient_fuse.la +LIBRADOS = librados.la +LIBRGW = librgw.la +LIBRBD = librbd.la +LIBCEPHFS = libcephfs.la +LIBERASURE_CODE = liberasure_code.la + +if WITH_LIBAIO +LIBOS += -laio +endif # WITH_LIBAIO + +if WITH_LIBZFS +LIBOS += libos_zfs.a -lzfs +endif # WITH_LIBZFS + +if WITH_TCMALLOC +LIBPERFGLUE += -ltcmalloc +endif # WITH_TCMALLOC + +if ENABLE_COVERAGE +EXTRALIBS += -lgcov +endif # ENABLE_COVERAGE + +# Libosd always needs osdc and os +LIBOSD += $(LIBOSDC) $(LIBOS) + +# These have references to syms like ceph_using_tcmalloc(), glue libperfglue to them +LIBMON += $(LIBPERFGLUE) +LIBOSD += $(LIBPERFGLUE) +LIBMDS += $(LIBPERFGLUE) + +# Always use system leveldb +LIBOS += -lleveldb -lsnappy + +# Use this for binaries requiring libglobal +CEPH_GLOBAL = $(LIBGLOBAL) $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) + +# This is set by [lib]/Makefile.am and used for build tests +LIBCOMMON_DEPS = +LIBRADOS_DEPS = +LIBRGW_DEPS = + +# This is used by the dencoder test +DENCODER_SOURCES = +DENCODER_DEPS = + + +radoslibdir = $(libdir)/rados-classes + diff --git a/ceph/src/Makefile.am b/ceph/src/Makefile.am new file mode 100644 index 00000000..edec05e6 --- /dev/null +++ b/ceph/src/Makefile.am @@ -0,0 +1,412 @@ +include Makefile-env.am + +SUBDIRS += ocf java +DIST_SUBDIRS += gtest ocf libs3 java + + +# subdirs + +include arch/Makefile.am +include auth/Makefile.am +include brag/Makefile.am +include crush/Makefile.am +include mon/Makefile.am +include mds/Makefile.am +include os/Makefile.am +include osd/Makefile.am +include erasure-code/Makefile.am +include osdc/Makefile.am +include client/Makefile.am +include global/Makefile.am +include json_spirit/Makefile.am +include log/Makefile.am +include perfglue/Makefile.am +include common/Makefile.am +include msg/Makefile.am +include messages/Makefile.am +include include/Makefile.am +include librados/Makefile.am +include librbd/Makefile.am +include rgw/Makefile.am +include cls/Makefile.am +include key_value_store/Makefile.am +include test/Makefile.am +include tools/Makefile.am + + +# core daemons + +ceph_mon_SOURCES = ceph_mon.cc +ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) $(LIBCOMMON) +bin_PROGRAMS += ceph-mon + +ceph_osd_SOURCES = ceph_osd.cc +ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(LIBCOMMON) +bin_PROGRAMS += ceph-osd + +ceph_mds_SOURCES = ceph_mds.cc +ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL) $(LIBCOMMON) +bin_PROGRAMS += ceph-mds + + +# admin tools + + +# user tools + +mount_ceph_SOURCES = mount/mount.ceph.c common/secret.c +mount_ceph_LDADD = $(LIBCOMMON) $(KEYUTILS_LIB) +if LINUX +su_sbin_PROGRAMS += mount.ceph +endif # LINUX +su_sbin_SCRIPTS += mount.fuse.ceph + +cephfs_SOURCES = cephfs.cc +cephfs_LDADD = $(LIBCOMMON) +bin_PROGRAMS += cephfs + +librados_config_SOURCES = librados-config.cc +librados_config_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +bin_PROGRAMS += librados-config + +ceph_syn_SOURCES = ceph_syn.cc +ceph_syn_SOURCES += client/SyntheticClient.cc # uses g_conf.. needs cleanup +ceph_syn_LDADD = $(LIBCLIENT) $(CEPH_GLOBAL) +bin_PROGRAMS += ceph-syn + +rbd_SOURCES = rbd.cc common/secret.c +rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) -lblkid $(KEYUTILS_LIB) +if LINUX +bin_PROGRAMS += rbd +endif #LINUX + + + +# Fuse targets + +if WITH_FUSE +ceph_fuse_SOURCES = ceph_fuse.cc +ceph_fuse_LDADD = $(LIBCLIENT_FUSE) $(CEPH_GLOBAL) +bin_PROGRAMS += ceph-fuse + +rbd_fuse_SOURCES = rbd_fuse/rbd-fuse.c +rbd_fuse_LDADD = -lfuse $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) +bin_PROGRAMS += rbd-fuse +endif # WITH_FUSE + + +# libcephfs (this should go somewhere else in the future) + +libcephfs_la_SOURCES = libcephfs.cc +libcephfs_la_LIBADD = $(LIBCLIENT) $(LIBCOMMON) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +libcephfs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^ceph_.*' +lib_LTLIBRARIES += libcephfs.la + +# jni library (java source is in src/java) + +if ENABLE_CEPHFS_JAVA +libcephfs_jni_la_SOURCES = \ + java/native/libcephfs_jni.cc \ + java/native/ScopedLocalRef.h \ + java/native/JniConstants.cpp \ + java/native/JniConstants.h +libcephfs_jni_la_LIBADD = $(LIBCEPHFS) $(EXTRALIBS) +libcephfs_jni_la_CPPFLAGS = $(JDK_CPPFLAGS) $(AM_CPPFLAGS) +libcephfs_jni_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 +lib_LTLIBRARIES += libcephfs_jni.la +endif + + +# shell scripts + +editpaths = sed \ + -e 's|@bindir[@]|$(bindir)|g' \ + -e 's|@sbindir[@]|$(sbindir)|g' \ + -e 's|@libdir[@]|$(libdir)|g' \ + -e 's|@sysconfdir[@]|$(sysconfdir)|g' \ + -e 's|@datadir[@]|$(pkgdatadir)|g' \ + -e 's|@prefix[@]|$(prefix)|g' \ + -e 's|@@GCOV_PREFIX_STRIP[@][@]|$(GCOV_PREFIX_STRIP)|g' +shell_scripts = ceph-debugpack ceph-post-file ceph-crush-location +$(shell_scripts): Makefile +$(shell_scripts): %: %.in + rm -f $@ $@.tmp + $(editpaths) '$(srcdir)/$@.in' >$@.tmp + chmod +x $@.tmp + chmod a-w $@.tmp + mv $@.tmp $@ + +EXTRA_DIST += $(srcdir)/$(shell_scripts:%=%.in) +CLEANFILES += $(shell_scripts) + + +# extra bits + +EXTRA_DIST += \ + $(srcdir)/verify-mds-journal.sh \ + $(srcdir)/vstart.sh \ + $(srcdir)/stop.sh \ + ceph-run \ + $(srcdir)/ceph_common.sh \ + $(srcdir)/init-radosgw \ + $(srcdir)/init-radosgw.sysv \ + $(srcdir)/init-rbdmap \ + $(srcdir)/ceph-clsinfo \ + $(srcdir)/make_version \ + $(srcdir)/check_version \ + $(srcdir)/.git_version \ + $(srcdir)/ceph-rbdnamer \ + $(srcdir)/test/encoding/readable.sh \ + $(srcdir)/test/encoding/check-generated.sh \ + $(srcdir)/upstart/ceph-all.conf \ + $(srcdir)/upstart/ceph-mon.conf \ + $(srcdir)/upstart/ceph-mon-all.conf \ + $(srcdir)/upstart/ceph-mon-all-starter.conf \ + $(srcdir)/upstart/ceph-create-keys.conf \ + $(srcdir)/upstart/ceph-osd.conf \ + $(srcdir)/upstart/ceph-osd-all.conf \ + $(srcdir)/upstart/ceph-osd-all-starter.conf \ + $(srcdir)/upstart/ceph-mds.conf \ + $(srcdir)/upstart/ceph-mds-all.conf \ + $(srcdir)/upstart/ceph-mds-all-starter.conf \ + $(srcdir)/upstart/radosgw.conf \ + $(srcdir)/upstart/radosgw-all.conf \ + $(srcdir)/upstart/radosgw-all-starter.conf \ + $(srcdir)/upstart/rbdmap.conf \ + ceph.in \ + ceph-disk \ + ceph-disk-prepare \ + ceph-disk-activate \ + ceph-disk-udev \ + ceph-create-keys \ + ceph-rest-api \ + ceph-crush-location \ + mount.fuse.ceph \ + rbdmap \ + unittest_bufferlist.sh \ + yasm-wrapper + +EXTRA_DIST += \ + libs3/COPYING \ + libs3/ChangeLog \ + libs3/GNUmakefile \ + libs3/GNUmakefile.mingw \ + libs3/GNUmakefile.osx \ + libs3/INSTALL \ + libs3/LICENSE \ + libs3/README \ + libs3/TODO \ + libs3/archlinux \ + libs3/debian \ + libs3/doxyfile \ + libs3/inc \ + libs3/libs3.spec \ + libs3/mswin \ + libs3/src \ + libs3/test \ + unittest_bufferlist.sh + +# work around old versions of automake that don't define $docdir +# NOTE: this won't work on suse, where docdir is /usr/share/doc/packages/$package. +docdir ?= ${datadir}/doc/ceph +doc_DATA = $(srcdir)/sample.ceph.conf sample.fetch_config + + +# various scripts + +shell_commondir = $(libdir)/ceph +shell_common_SCRIPTS = ceph_common.sh + +bash_completiondir = $(sysconfdir)/bash_completion.d +bash_completion_DATA = $(srcdir)/bash_completion/ceph \ + $(srcdir)/bash_completion/rados \ + $(srcdir)/bash_completion/rbd \ + $(srcdir)/bash_completion/radosgw-admin + +ceph_sbin_SCRIPTS = \ + ceph-disk \ + ceph-disk-prepare \ + ceph-disk-activate \ + ceph-disk-udev \ + ceph-create-keys + +bin_SCRIPTS += \ + ceph \ + ceph-run \ + ceph-rest-api \ + ceph-clsinfo \ + ceph-debugpack \ + ceph-rbdnamer \ + ceph-post-file \ + ceph-crush-location + +BUILT_SOURCES += init-ceph +su_sbin_SCRIPTS += mkcephfs + +shell_scripts += init-ceph mkcephfs + + + + + +# tests to actually run on "make check"; if you need extra, non-test, +# executables built, you need to replace this with manual assignments +# target by target + +TESTS = \ + $(check_PROGRAMS) \ + $(check_SCRIPTS) + +check-local: + $(srcdir)/test/encoding/readable.sh ../ceph-object-corpus + + +# base targets + +core-daemons: ceph-mon ceph-osd ceph-mds radosgw +admin-tools: monmaptool osdmaptool crushtool ceph-authtool +base: core-daemons admin-tools \ + cephfs ceph-syn ceph-conf \ + rados librados-config \ + init-ceph mkcephfs ceph_mon_store_converter ceph-post-file + + +# version stuff + +FORCE: +.git_version: FORCE + $(srcdir)/check_version $(srcdir)/.git_version + +# if NO_VERSION is set, only generate a new ceph_ver.h if there currently +# is none, and call "make_version -n" to fill it with a fixed string. +# Otherwise, set it from the contents of .git_version. + +ceph_ver.h: .git_version + if [ -n "$$NO_VERSION" ] ; then \ + if [ ! -f ./ceph_ver.h ] ; then \ + $(srcdir)/make_version -n ./ceph_ver.h ; \ + fi; \ + else \ + $(srcdir)/make_version $(srcdir)/.git_version ./ceph_ver.h ; \ + fi + +ceph_ver.c: ./ceph_ver.h +common/version.cc: ./ceph_ver.h +test/encoding/ceph_dencoder.cc: ./ceph_ver.h + +sample.fetch_config: fetch_config + cp -f $(srcdir)/fetch_config ./sample.fetch_config + +dist-hook: + $(srcdir)/check_version $(srcdir)/.git_version + +CLEANFILES += ceph_ver.h sample.fetch_config + + +# assemble Python script with global version variables +# NB: depends on format of ceph_ver.h + +ceph: ceph.in ./ceph_ver.h Makefile + rm -f $@ $@.tmp + echo "#!/usr/bin/env python" >$@.tmp + grep "#define CEPH_GIT_NICE_VER" ./ceph_ver.h | \ + sed -e 's/#define \(.*VER\) /\1=/' >>$@.tmp + grep "#define CEPH_GIT_VER" ./ceph_ver.h | \ + sed -e 's/#define \(.*VER\) /\1=/' -e 's/=\(.*\)$$/="\1"/' >>$@.tmp + cat $(srcdir)/$@.in >>$@.tmp + chmod a+x $@.tmp + chmod a-w $@.tmp + mv $@.tmp $@ + +# cleaning + +clean-local: + rm -f *.so + find . -name '*.gcno' -o -name '*.gcda' -o -name '*.lcov' | xargs rm -f + rm -f ceph java/java/com/ceph/crush/Bucket.class + + +# pybind + +python_PYTHON = pybind/rados.py \ + pybind/rbd.py \ + pybind/cephfs.py \ + pybind/ceph_argparse.py \ + pybind/ceph_rest_api.py + + +# everything else we want to include in a 'make dist' + +noinst_HEADERS += \ + cls_acl.cc\ + cls_crypto.cc\ + fetch_config\ + logrotate.conf\ + sample.ceph.conf\ + bash_completion/ceph \ + bash_completion/rados \ + bash_completion/rbd \ + bash_completion/radosgw-admin \ + mount/canonicalize.c \ + mount/mtab.c \ + objclass/objclass.h + + +# coverage + +shell_scripts += ceph-coverage +bin_SCRIPTS += ceph-coverage + + +if ENABLE_COVERAGE +COV_DIR = $(DESTDIR)$(libdir)/ceph/coverage +COV_FILES = $(srcdir)/*.gcno +COV_LIB_FILES = $(srcdir)/.libs/*.gcno +endif + +install-coverage: +if ENABLE_COVERAGE + -mkdir -p $(COV_DIR)/.libs + -$(INSTALL_DATA) $(COV_FILES) $(COV_DIR) + -$(INSTALL_DATA) $(COV_LIB_FILES) $(COV_DIR)/.libs +endif + +uninstall-coverage: +if ENABLE_COVERAGE + -rm $(COV_DIR)/*.gcno + -rm $(COV_DIR)/.libs/*.gcno + -rmdir -p $(COV_DIR)/.libs + -rmdir -p $(COV_DIR) +endif + +check-coverage: +if ENABLE_COVERAGE + -test/coverage.sh -d $(srcdir) -o check-coverage make check +endif + +install-data-local: install-coverage + -mkdir -p $(DESTDIR)$(sysconfdir)/ceph + -mkdir -p $(DESTDIR)$(localstatedir)/log/ceph + -mkdir -p $(DESTDIR)$(localstatedir)/lib/ceph/tmp + +uninstall-local: uninstall-coverage + -rmdir -p $(DESTDIR)$(sysconfdir)/ceph/ + -rmdir -p $(DESTDIR)$(localstatedir)/log/ceph + -rmdir -p $(DESTDIR)$(localstatedir)/lib/ceph/tmp + +# +# coverity rules expect: +# - cov-build to be in the path +# - password in ~/coverity.build.pass.txt +# - ability to scp into the ceph.com directory +# +project.tgz: clean + rm -rf cov-int + cov-build --dir cov-int make + echo Sage Weil sage@newdream.net ceph >> README + tar czvf project.tgz README cov-int + rm -f README + +coverity-submit: + scp project.tgz ceph.com:/home/ceph_site/ceph.com/coverity/`git describe`.tgz + curl --data "project=ceph&password=`cat ~/coverity.build.pass.txt`&email=sage@newdream.net&url=http://ceph.com/coverity/`git describe`.tgz" http://scan5.coverity.com/cgi-bin/submit_build.py diff --git a/ceph/src/Makefile.in b/ceph/src/Makefile.in new file mode 100644 index 00000000..afa524ba --- /dev/null +++ b/ceph/src/Makefile.in @@ -0,0 +1,16102 @@ +# Makefile.in generated by automake 1.11.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software +# Foundation, Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + + + + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +DIST_COMMON = README $(am__noinst_HEADERS_DIST) $(dist_bin_SCRIPTS) \ + $(python_PYTHON) $(srcdir)/Makefile-env.am \ + $(srcdir)/Makefile.am $(srcdir)/Makefile.in \ + $(srcdir)/acconfig.h.in $(srcdir)/arch/Makefile.am \ + $(srcdir)/auth/Makefile.am $(srcdir)/brag/Makefile.am \ + $(srcdir)/client/Makefile.am $(srcdir)/cls/Makefile.am \ + $(srcdir)/common/Makefile.am $(srcdir)/crush/Makefile.am \ + $(srcdir)/erasure-code/Makefile.am \ + $(srcdir)/erasure-code/jerasure/Makefile.am \ + $(srcdir)/global/Makefile.am $(srcdir)/include/Makefile.am \ + $(srcdir)/json_spirit/Makefile.am \ + $(srcdir)/key_value_store/Makefile.am \ + $(srcdir)/librados/Makefile.am $(srcdir)/librbd/Makefile.am \ + $(srcdir)/log/Makefile.am $(srcdir)/mds/Makefile.am \ + $(srcdir)/messages/Makefile.am $(srcdir)/mon/Makefile.am \ + $(srcdir)/msg/Makefile.am $(srcdir)/os/Makefile.am \ + $(srcdir)/osd/Makefile.am $(srcdir)/osdc/Makefile.am \ + $(srcdir)/perfglue/Makefile.am $(srcdir)/rgw/Makefile.am \ + $(srcdir)/test/Makefile.am \ + $(srcdir)/test/erasure-code/Makefile.am \ + $(srcdir)/tools/Makefile.am TODO +bin_PROGRAMS = $(am__EXEEXT_9) $(am__EXEEXT_10) ceph-dencoder$(EXEEXT) \ + ceph_filestore_tool$(EXEEXT) ceph_filestore_dump$(EXEEXT) \ + monmaptool$(EXEEXT) crushtool$(EXEEXT) osdmaptool$(EXEEXT) \ + rados$(EXEEXT) $(am__EXEEXT_11) ceph-conf$(EXEEXT) \ + ceph-authtool$(EXEEXT) ceph_mon_store_converter$(EXEEXT) \ + ceph-mon$(EXEEXT) ceph-osd$(EXEEXT) ceph-mds$(EXEEXT) \ + cephfs$(EXEEXT) librados-config$(EXEEXT) ceph-syn$(EXEEXT) \ + $(am__EXEEXT_12) $(am__EXEEXT_13) +noinst_PROGRAMS = get_command_descriptions$(EXEEXT) +sbin_PROGRAMS = +su_sbin_PROGRAMS = $(am__EXEEXT_14) +check_PROGRAMS = unittest_erasure_code_plugin$(EXEEXT) \ + unittest_erasure_code_jerasure$(EXEEXT) \ + unittest_erasure_code_plugin_jerasure$(EXEEXT) \ + unittest_erasure_code_example$(EXEEXT) \ + unittest_encoding$(EXEEXT) unittest_addrs$(EXEEXT) \ + unittest_bloom_filter$(EXEEXT) unittest_histogram$(EXEEXT) \ + unittest_str_map$(EXEEXT) unittest_sharedptr_registry$(EXEEXT) \ + unittest_sloppy_crc_map$(EXEEXT) unittest_util$(EXEEXT) \ + unittest_crush_indep$(EXEEXT) unittest_osdmap$(EXEEXT) \ + unittest_workqueue$(EXEEXT) unittest_striper$(EXEEXT) \ + unittest_prebufferedstreambuf$(EXEEXT) \ + unittest_str_list$(EXEEXT) unittest_log$(EXEEXT) \ + unittest_throttle$(EXEEXT) unittest_crush_wrapper$(EXEEXT) \ + unittest_base64$(EXEEXT) unittest_ceph_argparse$(EXEEXT) \ + unittest_ceph_compatset$(EXEEXT) unittest_osd_types$(EXEEXT) \ + unittest_pglog$(EXEEXT) unittest_ecbackend$(EXEEXT) \ + unittest_hitset$(EXEEXT) unittest_gather$(EXEEXT) \ + unittest_run_cmd$(EXEEXT) unittest_signals$(EXEEXT) \ + unittest_simple_spin$(EXEEXT) unittest_librados$(EXEEXT) \ + unittest_bufferlist$(EXEEXT) unittest_crc32c$(EXEEXT) \ + unittest_arch$(EXEEXT) unittest_crypto$(EXEEXT) \ + unittest_crypto_init$(EXEEXT) unittest_perf_counters$(EXEEXT) \ + unittest_admin_socket$(EXEEXT) unittest_ceph_crypto$(EXEEXT) \ + unittest_utf8$(EXEEXT) unittest_mime$(EXEEXT) \ + unittest_escape$(EXEEXT) unittest_chain_xattr$(EXEEXT) \ + unittest_flatindex$(EXEEXT) unittest_strtol$(EXEEXT) \ + unittest_confutils$(EXEEXT) unittest_config$(EXEEXT) \ + unittest_context$(EXEEXT) unittest_heartbeatmap$(EXEEXT) \ + unittest_formatter$(EXEEXT) unittest_libcephfs_config$(EXEEXT) \ + unittest_lfnindex$(EXEEXT) unittest_librados_config$(EXEEXT) \ + unittest_daemon_config$(EXEEXT) unittest_osd_osdcap$(EXEEXT) \ + unittest_mon_moncap$(EXEEXT) unittest_mon_pgmap$(EXEEXT) \ + unittest_ipaddr$(EXEEXT) unittest_texttable$(EXEEXT) \ + unittest_on_exit$(EXEEXT) + +# when doing a debug build, make sure to make the targets +@WITH_DEBUG_TRUE@am__append_1 = $(bin_DEBUGPROGRAMS) +@LINUX_TRUE@am__append_2 = -Wl,--as-needed +@USE_BOOST_SPIRIT_OLD_HDR_TRUE@am__append_3 = -DUSE_BOOST_SPIRIT_OLD_HDR +@WITH_LIBATOMIC_TRUE@am__append_4 = -latomic_ops +@ENABLE_COVERAGE_TRUE@am__append_5 = -fprofile-arcs -ftest-coverage +@ENABLE_COVERAGE_TRUE@am__append_6 = -fprofile-arcs -ftest-coverage -O0 +@FREEBSD_TRUE@am__append_7 = -lexecinfo +@LINUX_TRUE@am__append_8 = -lrt +@WITH_PROFILER_TRUE@am__append_9 = -lprofiler +@WITH_LIBAIO_TRUE@am__append_10 = -laio +@WITH_LIBZFS_TRUE@am__append_11 = libos_zfs.a -lzfs +@WITH_TCMALLOC_TRUE@am__append_12 = -ltcmalloc +@ENABLE_COVERAGE_TRUE@am__append_13 = -lgcov +@LINUX_TRUE@am__append_14 = os/BtrfsFileStoreBackend.cc +@WITH_LIBXFS_TRUE@am__append_15 = os/XfsFileStoreBackend.cc +@WITH_LIBZFS_TRUE@am__append_16 = os/ZFSFileStoreBackend.cc +@WITH_LIBZFS_TRUE@am__append_17 = libos_zfs.a +@WITH_LIBZFS_TRUE@am__append_18 = os/ZFS.h +@LINUX_TRUE@am__append_19 = -export-symbols-regex '.*__erasure_code_.*' +@LINUX_TRUE@am__append_20 = -export-symbols-regex '.*__erasure_code_.*' +@LINUX_TRUE@am__append_21 = -export-symbols-regex '.*__erasure_code_.*' +@LINUX_TRUE@am__append_22 = -export-symbols-regex '.*__erasure_code_.*' +@WITH_FUSE_TRUE@am__append_23 = libclient_fuse.la +@WITH_FUSE_TRUE@am__append_24 = client/fuse_ll.h +@WITH_TCMALLOC_TRUE@am__append_25 = perfglue/heap_profiler.cc +@WITH_TCMALLOC_TRUE@am__append_26 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free +@WITH_TCMALLOC_TRUE@am__append_27 = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free +@WITH_TCMALLOC_FALSE@am__append_28 = perfglue/disabled_heap_profiler.cc +@WITH_PROFILER_TRUE@am__append_29 = perfglue/cpu_profiler.cc +@WITH_PROFILER_FALSE@am__append_30 = perfglue/disabled_stubs.cc +@WITH_GOOD_YASM_ELF64_TRUE@am__append_31 = common/crc32c_intel_fast_asm.S common/crc32c_intel_fast_zero_asm.S +@LINUX_TRUE@am__append_32 = -lrt +@LINUX_TRUE@am__append_33 = -export-symbols-regex '^rados_.*' +@LINUX_TRUE@am__append_34 = -export-symbols-regex '^rbd_.*' +@WITH_RADOSGW_TRUE@am__append_35 = librgw.la +@WITH_RADOSGW_TRUE@am__append_36 = \ +@WITH_RADOSGW_TRUE@ $(LIBRADOS) \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la \ +@WITH_RADOSGW_TRUE@ libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \ +@WITH_RADOSGW_TRUE@ libcls_user_client.a \ +@WITH_RADOSGW_TRUE@ libcls_replica_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_lock_client.la \ +@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a \ +@WITH_RADOSGW_TRUE@ -lcurl \ +@WITH_RADOSGW_TRUE@ -lexpat \ +@WITH_RADOSGW_TRUE@ -lm \ +@WITH_RADOSGW_TRUE@ -lfcgi \ +@WITH_RADOSGW_TRUE@ -ldl + +@WITH_RADOSGW_TRUE@am__append_37 = radosgw radosgw-admin +@WITH_RADOSGW_TRUE@am__append_38 = ceph_rgw_multiparser \ +@WITH_RADOSGW_TRUE@ ceph_rgw_jsonparser + +# inject rgw stuff in the decoder testcase +@WITH_RADOSGW_TRUE@am__append_39 = \ +@WITH_RADOSGW_TRUE@ rgw/rgw_dencoder.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_acl.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_common.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_env.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_json_enc.cc + +@LINUX_TRUE@am__append_40 = libcls_kvs.la +@LINUX_TRUE@am__append_41 = -ldl +@LINUX_TRUE@am__append_42 = -ldl +@LINUX_TRUE@am__append_43 = -ldl +@LINUX_TRUE@am__append_44 = -ldl +@LINUX_TRUE@am__append_45 = -ldl +@COMPILER_HAS_VTA_TRUE@am__append_46 = -fno-var-tracking-assignments +@COMPILER_HAS_VTA_TRUE@am__append_47 = -fno-var-tracking-assignments +@WITH_BUILD_TESTS_TRUE@am__append_48 = test_build_libcommon \ +@WITH_BUILD_TESTS_TRUE@ test_build_librados test_build_librgw \ +@WITH_BUILD_TESTS_TRUE@ test_build_libcephfs +@LINUX_TRUE@am__append_49 = ceph_kvstorebench \ +@LINUX_TRUE@ ceph_test_rados_list_parallel \ +@LINUX_TRUE@ ceph_test_rados_open_pools_parallel \ +@LINUX_TRUE@ ceph_test_rados_delete_pools_parallel \ +@LINUX_TRUE@ ceph_test_rados_watch_notify +@LINUX_TRUE@am__append_50 = libsystest.la +@LINUX_TRUE@am__append_51 = -ldl +@WITH_RADOSGW_TRUE@am__append_52 = ceph_test_cors \ +@WITH_RADOSGW_TRUE@ ceph_test_rgw_manifest \ +@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_meta \ +@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_log \ +@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_opstate +@LINUX_TRUE@am__append_53 = ceph_test_librbd_fsx +@WITH_RADOSGW_TRUE@am__append_54 = ceph_test_cls_rgw +@LINUX_TRUE@am__append_55 = ceph_test_objectstore +@LINUX_TRUE@am__append_56 = -ldl +@LINUX_TRUE@am__append_57 = -ldl +@WITH_REST_BENCH_TRUE@am__append_58 = rest-bench +@WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_TRUE@am__append_59 = -ls3 +@WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE@am__append_60 = libs3/build/lib/libs3.a -lcurl -lxml2 +@WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE@am__append_61 = libs3 +@LINUX_TRUE@am__append_62 = mount.ceph +@LINUX_TRUE@am__append_63 = rbd +@WITH_FUSE_TRUE@am__append_64 = ceph-fuse rbd-fuse +@ENABLE_CEPHFS_JAVA_TRUE@am__append_65 = libcephfs_jni.la +subdir = src +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \ + $(top_srcdir)/m4/ac_prog_jar.m4 \ + $(top_srcdir)/m4/ac_prog_javac.m4 \ + $(top_srcdir)/m4/ac_prog_javac_works.m4 \ + $(top_srcdir)/m4/ac_prog_javah.m4 \ + $(top_srcdir)/m4/acx_pthread.m4 \ + $(top_srcdir)/m4/ax_c_pretty_func.m4 \ + $(top_srcdir)/m4/ax_c_var_func.m4 \ + $(top_srcdir)/m4/ax_check_compile_flag.m4 \ + $(top_srcdir)/m4/ax_cxx_static_cast.m4 \ + $(top_srcdir)/m4/ax_intel.m4 $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/pkg.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = acconfig.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LIBRARIES = $(noinst_LIBRARIES) +ARFLAGS = cru +AM_V_AR = $(am__v_AR_@AM_V@) +am__v_AR_ = $(am__v_AR_@AM_DEFAULT_V@) +am__v_AR_0 = @echo " AR " $@; +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +libcls_log_client_a_AR = $(AR) $(ARFLAGS) +libcls_log_client_a_LIBADD = +am__dirstamp = $(am__leading_dot)dirstamp +am_libcls_log_client_a_OBJECTS = cls/log/cls_log_client.$(OBJEXT) +libcls_log_client_a_OBJECTS = $(am_libcls_log_client_a_OBJECTS) +libcls_replica_log_client_a_AR = $(AR) $(ARFLAGS) +libcls_replica_log_client_a_LIBADD = +am_libcls_replica_log_client_a_OBJECTS = \ + cls/replica_log/cls_replica_log_types.$(OBJEXT) \ + cls/replica_log/cls_replica_log_ops.$(OBJEXT) \ + cls/replica_log/cls_replica_log_client.$(OBJEXT) +libcls_replica_log_client_a_OBJECTS = \ + $(am_libcls_replica_log_client_a_OBJECTS) +libcls_statelog_client_a_AR = $(AR) $(ARFLAGS) +libcls_statelog_client_a_LIBADD = +am_libcls_statelog_client_a_OBJECTS = \ + cls/statelog/cls_statelog_client.$(OBJEXT) +libcls_statelog_client_a_OBJECTS = \ + $(am_libcls_statelog_client_a_OBJECTS) +libcls_user_client_a_AR = $(AR) $(ARFLAGS) +libcls_user_client_a_LIBADD = +am_libcls_user_client_a_OBJECTS = cls/user/cls_user_client.$(OBJEXT) \ + cls/user/cls_user_types.$(OBJEXT) \ + cls/user/cls_user_ops.$(OBJEXT) +libcls_user_client_a_OBJECTS = $(am_libcls_user_client_a_OBJECTS) +libcls_version_client_a_AR = $(AR) $(ARFLAGS) +libcls_version_client_a_LIBADD = +am_libcls_version_client_a_OBJECTS = \ + cls/version/cls_version_client.$(OBJEXT) \ + cls/version/cls_version_types.$(OBJEXT) +libcls_version_client_a_OBJECTS = \ + $(am_libcls_version_client_a_OBJECTS) +libos_zfs_a_AR = $(AR) $(ARFLAGS) +libos_zfs_a_LIBADD = +am__libos_zfs_a_SOURCES_DIST = os/ZFS.cc +@WITH_LIBZFS_TRUE@am_libos_zfs_a_OBJECTS = \ +@WITH_LIBZFS_TRUE@ os/libos_zfs_a-ZFS.$(OBJEXT) +libos_zfs_a_OBJECTS = $(am_libos_zfs_a_OBJECTS) +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(erasure_codelibdir)" \ + "$(DESTDIR)$(libdir)" "$(DESTDIR)$(radoslibdir)" \ + "$(DESTDIR)$(bindir)" "$(DESTDIR)$(sbindir)" \ + "$(DESTDIR)$(su_sbindir)" "$(DESTDIR)$(bindir)" \ + "$(DESTDIR)$(ceph_sbindir)" "$(DESTDIR)$(bindir)" \ + "$(DESTDIR)$(sbindir)" "$(DESTDIR)$(shell_commondir)" \ + "$(DESTDIR)$(su_sbindir)" "$(DESTDIR)$(pythondir)" \ + "$(DESTDIR)$(bash_completiondir)" "$(DESTDIR)$(docdir)" \ + "$(DESTDIR)$(libcephfs_includedir)" \ + "$(DESTDIR)$(librbd_includedir)" \ + "$(DESTDIR)$(rados_includedir)" +LTLIBRARIES = $(erasure_codelib_LTLIBRARIES) $(lib_LTLIBRARIES) \ + $(noinst_LTLIBRARIES) $(radoslib_LTLIBRARIES) +libarch_la_LIBADD = +am_libarch_la_OBJECTS = arch/intel.lo arch/neon.lo arch/probe.lo +libarch_la_OBJECTS = $(am_libarch_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +libauth_la_LIBADD = +am_libauth_la_OBJECTS = auth/AuthAuthorizeHandler.lo \ + auth/AuthClientHandler.lo auth/AuthSessionHandler.lo \ + auth/AuthServiceHandler.lo auth/AuthMethodList.lo \ + auth/cephx/CephxAuthorizeHandler.lo \ + auth/cephx/CephxClientHandler.lo auth/cephx/CephxProtocol.lo \ + auth/cephx/CephxServiceHandler.lo \ + auth/cephx/CephxSessionHandler.lo auth/cephx/CephxKeyServer.lo \ + auth/none/AuthNoneAuthorizeHandler.lo \ + auth/unknown/AuthUnknownAuthorizeHandler.lo auth/Crypto.lo \ + auth/KeyRing.lo auth/RotatingKeyRing.lo +libauth_la_OBJECTS = $(am_libauth_la_OBJECTS) +am__DEPENDENCIES_1 = +am__DEPENDENCIES_2 = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) +libcephfs_la_DEPENDENCIES = $(LIBCLIENT) $(LIBCOMMON) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcephfs_la_OBJECTS = libcephfs.lo +libcephfs_la_OBJECTS = $(am_libcephfs_la_OBJECTS) +libcephfs_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(libcephfs_la_LDFLAGS) $(LDFLAGS) -o $@ +@ENABLE_CEPHFS_JAVA_TRUE@libcephfs_jni_la_DEPENDENCIES = $(LIBCEPHFS) \ +@ENABLE_CEPHFS_JAVA_TRUE@ $(am__DEPENDENCIES_2) +am__libcephfs_jni_la_SOURCES_DIST = java/native/libcephfs_jni.cc \ + java/native/ScopedLocalRef.h java/native/JniConstants.cpp \ + java/native/JniConstants.h +@ENABLE_CEPHFS_JAVA_TRUE@am_libcephfs_jni_la_OBJECTS = java/native/libcephfs_jni_la-libcephfs_jni.lo \ +@ENABLE_CEPHFS_JAVA_TRUE@ java/native/libcephfs_jni_la-JniConstants.lo +libcephfs_jni_la_OBJECTS = $(am_libcephfs_jni_la_OBJECTS) +libcephfs_jni_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcephfs_jni_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +@ENABLE_CEPHFS_JAVA_TRUE@am_libcephfs_jni_la_rpath = -rpath $(libdir) +libclient_la_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_1) +am_libclient_la_OBJECTS = client/Client.lo client/Inode.lo \ + client/Dentry.lo client/MetaRequest.lo \ + client/ClientSnapRealm.lo client/MetaSession.lo \ + client/Trace.lo +libclient_la_OBJECTS = $(am_libclient_la_OBJECTS) +@WITH_FUSE_TRUE@libclient_fuse_la_DEPENDENCIES = libclient.la +am__libclient_fuse_la_SOURCES_DIST = client/fuse_ll.cc +@WITH_FUSE_TRUE@am_libclient_fuse_la_OBJECTS = client/fuse_ll.lo +libclient_fuse_la_OBJECTS = $(am_libclient_fuse_la_OBJECTS) +@WITH_FUSE_TRUE@am_libclient_fuse_la_rpath = +libcls_hello_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_hello_la_OBJECTS = cls/hello/cls_hello.lo +libcls_hello_la_OBJECTS = $(am_libcls_hello_la_OBJECTS) +libcls_hello_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_hello_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +@LINUX_TRUE@libcls_kvs_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ +@LINUX_TRUE@ $(am__DEPENDENCIES_2) +am__libcls_kvs_la_SOURCES_DIST = key_value_store/cls_kvs.cc +@LINUX_TRUE@am_libcls_kvs_la_OBJECTS = key_value_store/cls_kvs.lo +libcls_kvs_la_OBJECTS = $(am_libcls_kvs_la_OBJECTS) +libcls_kvs_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_kvs_la_LDFLAGS) $(LDFLAGS) \ + -o $@ +@LINUX_TRUE@am_libcls_kvs_la_rpath = -rpath $(radoslibdir) +libcls_lock_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_lock_la_OBJECTS = cls/lock/cls_lock.lo +libcls_lock_la_OBJECTS = $(am_libcls_lock_la_OBJECTS) +libcls_lock_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_lock_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +libcls_lock_client_la_LIBADD = +am_libcls_lock_client_la_OBJECTS = cls/lock/cls_lock_client.lo \ + cls/lock/cls_lock_types.lo cls/lock/cls_lock_ops.lo +libcls_lock_client_la_OBJECTS = $(am_libcls_lock_client_la_OBJECTS) +libcls_log_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_log_la_OBJECTS = cls/log/cls_log.lo +libcls_log_la_OBJECTS = $(am_libcls_log_la_OBJECTS) +libcls_log_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_log_la_LDFLAGS) $(LDFLAGS) \ + -o $@ +libcls_rbd_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_rbd_la_OBJECTS = cls/rbd/cls_rbd.lo +libcls_rbd_la_OBJECTS = $(am_libcls_rbd_la_OBJECTS) +libcls_rbd_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_rbd_la_LDFLAGS) $(LDFLAGS) \ + -o $@ +libcls_rbd_client_la_LIBADD = +am_libcls_rbd_client_la_OBJECTS = cls/rbd/cls_rbd_client.lo +libcls_rbd_client_la_OBJECTS = $(am_libcls_rbd_client_la_OBJECTS) +libcls_refcount_la_DEPENDENCIES = libjson_spirit.la \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) +am_libcls_refcount_la_OBJECTS = cls/refcount/cls_refcount.lo \ + cls/refcount/cls_refcount_ops.lo common/ceph_json.lo +libcls_refcount_la_OBJECTS = $(am_libcls_refcount_la_OBJECTS) +libcls_refcount_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_refcount_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +libcls_refcount_client_la_LIBADD = +am_libcls_refcount_client_la_OBJECTS = \ + cls/refcount/cls_refcount_client.lo \ + cls/refcount/cls_refcount_ops.lo +libcls_refcount_client_la_OBJECTS = \ + $(am_libcls_refcount_client_la_OBJECTS) +libcls_replica_log_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_replica_log_la_OBJECTS = cls/replica_log/cls_replica_log.lo +libcls_replica_log_la_OBJECTS = $(am_libcls_replica_log_la_OBJECTS) +libcls_replica_log_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_replica_log_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +libcls_rgw_la_DEPENDENCIES = libjson_spirit.la $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_rgw_la_OBJECTS = cls/rgw/cls_rgw.lo cls/rgw/cls_rgw_ops.lo \ + cls/rgw/cls_rgw_types.lo common/ceph_json.lo +libcls_rgw_la_OBJECTS = $(am_libcls_rgw_la_OBJECTS) +libcls_rgw_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_rgw_la_LDFLAGS) $(LDFLAGS) \ + -o $@ +libcls_rgw_client_la_LIBADD = +am_libcls_rgw_client_la_OBJECTS = cls/rgw/cls_rgw_client.lo \ + cls/rgw/cls_rgw_types.lo cls/rgw/cls_rgw_ops.lo +libcls_rgw_client_la_OBJECTS = $(am_libcls_rgw_client_la_OBJECTS) +libcls_statelog_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_statelog_la_OBJECTS = cls/statelog/cls_statelog.lo +libcls_statelog_la_OBJECTS = $(am_libcls_statelog_la_OBJECTS) +libcls_statelog_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_statelog_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +libcls_user_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_user_la_OBJECTS = cls/user/cls_user.lo +libcls_user_la_OBJECTS = $(am_libcls_user_la_OBJECTS) +libcls_user_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_user_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +libcls_version_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libcls_version_la_OBJECTS = cls/version/cls_version.lo +libcls_version_la_OBJECTS = $(am_libcls_version_la_OBJECTS) +libcls_version_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(AM_CXXFLAGS) $(CXXFLAGS) $(libcls_version_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +am__DEPENDENCIES_3 = libcommon_crc.la $(LIBERASURE_CODE) $(LIBMSG) \ + $(LIBAUTH) $(LIBCRUSH) $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) \ + $(am__DEPENDENCIES_1) +libcommon_la_DEPENDENCIES = $(am__DEPENDENCIES_3) +am_libcommon_la_OBJECTS = ceph_ver.lo common/DecayCounter.lo \ + common/LogClient.lo common/LogEntry.lo \ + common/PrebufferedStreambuf.lo common/SloppyCRCMap.lo \ + common/BackTrace.lo common/perf_counters.lo common/Mutex.lo \ + common/OutputDataSocket.lo common/admin_socket.lo \ + common/admin_socket_client.lo common/cmdparse.lo \ + common/escape.lo common/io_priority.lo common/Clock.lo \ + common/Throttle.lo common/Timer.lo common/Finisher.lo \ + common/environment.lo common/assert.lo common/run_cmd.lo \ + common/WorkQueue.lo common/ConfUtils.lo common/MemoryModel.lo \ + common/armor.lo common/fd.lo common/xattr.lo common/safe_io.lo \ + common/snap_types.lo common/str_list.lo common/str_map.lo \ + common/errno.lo common/RefCountedObj.lo common/blkdev.lo \ + common/common_init.lo common/pipe.lo common/ceph_argparse.lo \ + common/ceph_context.lo common/buffer.lo \ + common/code_environment.lo common/dout.lo common/histogram.lo \ + common/signal.lo common/simple_spin.lo common/Thread.lo \ + common/Formatter.lo common/HeartbeatMap.lo common/config.lo \ + common/utf8.lo common/mime.lo common/strtol.lo common/page.lo \ + common/lockdep.lo common/version.lo common/hex.lo \ + common/entity_name.lo common/ceph_crypto.lo \ + common/ceph_crypto_cms.lo common/ceph_json.lo common/ipaddr.lo \ + common/pick_address.lo common/util.lo common/TextTable.lo \ + common/ceph_fs.lo common/ceph_hash.lo common/ceph_strings.lo \ + common/ceph_frag.lo common/addr_parsing.lo common/hobject.lo \ + common/bloom_filter.lo common/linux_version.lo mon/MonCap.lo \ + mon/MonClient.lo mon/MonMap.lo osd/OSDMap.lo osd/osd_types.lo \ + osd/ECMsgTypes.lo osd/HitSet.lo mds/MDSMap.lo \ + mds/inode_backtrace.lo mds/mdstypes.lo +libcommon_la_OBJECTS = $(am_libcommon_la_OBJECTS) +libcommon_crc_la_LIBADD = +am__libcommon_crc_la_SOURCES_DIST = common/sctp_crc32.c \ + common/crc32c.cc common/crc32c_intel_baseline.c \ + common/crc32c_intel_fast.c common/crc32c_intel_fast_asm.S \ + common/crc32c_intel_fast_zero_asm.S +@WITH_GOOD_YASM_ELF64_TRUE@am__objects_1 = common/libcommon_crc_la-crc32c_intel_fast_asm.lo \ +@WITH_GOOD_YASM_ELF64_TRUE@ common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo +am_libcommon_crc_la_OBJECTS = common/libcommon_crc_la-sctp_crc32.lo \ + common/libcommon_crc_la-crc32c.lo \ + common/libcommon_crc_la-crc32c_intel_baseline.lo \ + common/libcommon_crc_la-crc32c_intel_fast.lo $(am__objects_1) +libcommon_crc_la_OBJECTS = $(am_libcommon_crc_la_OBJECTS) +libcommon_crc_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +libcrush_la_LIBADD = +am_libcrush_la_OBJECTS = crush/builder.lo crush/mapper.lo \ + crush/crush.lo crush/hash.lo crush/CrushWrapper.lo \ + crush/CrushCompiler.lo crush/CrushTester.lo +libcrush_la_OBJECTS = $(am_libcrush_la_OBJECTS) +libec_example_la_DEPENDENCIES = $(LIBCRUSH) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_example_la_OBJECTS = test/erasure-code/libec_example_la-ErasureCodePluginExample.lo +libec_example_la_OBJECTS = $(am_libec_example_la_OBJECTS) +libec_example_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_example_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_example_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_fail_to_initialize_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_fail_to_initialize_la_OBJECTS = test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo +libec_fail_to_initialize_la_OBJECTS = \ + $(am_libec_fail_to_initialize_la_OBJECTS) +libec_fail_to_initialize_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_fail_to_initialize_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_fail_to_initialize_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_fail_to_register_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_fail_to_register_la_OBJECTS = test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo +libec_fail_to_register_la_OBJECTS = \ + $(am_libec_fail_to_register_la_OBJECTS) +libec_fail_to_register_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_fail_to_register_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_fail_to_register_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_hangs_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_hangs_la_OBJECTS = \ + test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo +libec_hangs_la_OBJECTS = $(am_libec_hangs_la_OBJECTS) +libec_hangs_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_hangs_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_jerasure_la_DEPENDENCIES = $(LIBCRUSH) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_jerasure_la_OBJECTS = erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo +libec_jerasure_la_OBJECTS = $(am_libec_jerasure_la_OBJECTS) +libec_jerasure_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_jerasure_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_jerasure_generic_la_DEPENDENCIES = $(LIBCRUSH) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) +am__objects_2 = erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo \ + erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo \ + erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo +am_libec_jerasure_generic_la_OBJECTS = $(am__objects_2) +libec_jerasure_generic_la_OBJECTS = \ + $(am_libec_jerasure_generic_la_OBJECTS) +libec_jerasure_generic_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_jerasure_generic_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_jerasure_sse3_la_DEPENDENCIES = $(LIBCRUSH) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) +am__objects_3 = erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo \ + erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo \ + erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo +am_libec_jerasure_sse3_la_OBJECTS = $(am__objects_3) +libec_jerasure_sse3_la_OBJECTS = $(am_libec_jerasure_sse3_la_OBJECTS) +libec_jerasure_sse3_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_jerasure_sse3_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_jerasure_sse4_la_DEPENDENCIES = $(LIBCRUSH) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) +am__objects_4 = erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo \ + erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo \ + erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo \ + erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo \ + erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo +am_libec_jerasure_sse4_la_OBJECTS = $(am__objects_4) +libec_jerasure_sse4_la_OBJECTS = $(am_libec_jerasure_sse4_la_OBJECTS) +libec_jerasure_sse4_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_jerasure_sse4_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_missing_entry_point_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_missing_entry_point_la_OBJECTS = test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo +libec_missing_entry_point_la_OBJECTS = \ + $(am_libec_missing_entry_point_la_OBJECTS) +libec_missing_entry_point_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_missing_entry_point_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_test_jerasure_generic_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_test_jerasure_generic_la_OBJECTS = test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo +libec_test_jerasure_generic_la_OBJECTS = \ + $(am_libec_test_jerasure_generic_la_OBJECTS) +libec_test_jerasure_generic_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_test_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_test_jerasure_generic_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_test_jerasure_sse3_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_test_jerasure_sse3_la_OBJECTS = test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo +libec_test_jerasure_sse3_la_OBJECTS = \ + $(am_libec_test_jerasure_sse3_la_OBJECTS) +libec_test_jerasure_sse3_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_test_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_test_jerasure_sse3_la_LDFLAGS) $(LDFLAGS) -o $@ +libec_test_jerasure_sse4_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_libec_test_jerasure_sse4_la_OBJECTS = test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo +libec_test_jerasure_sse4_la_OBJECTS = \ + $(am_libec_test_jerasure_sse4_la_OBJECTS) +libec_test_jerasure_sse4_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libec_test_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) \ + $(libec_test_jerasure_sse4_la_LDFLAGS) $(LDFLAGS) -o $@ +liberasure_code_la_DEPENDENCIES = +am_liberasure_code_la_OBJECTS = erasure-code/ErasureCodePlugin.lo +liberasure_code_la_OBJECTS = $(am_liberasure_code_la_OBJECTS) +libglobal_la_DEPENDENCIES = $(LIBCOMMON) +am_libglobal_la_OBJECTS = global/global_context.lo \ + global/global_init.lo global/pidfile.lo \ + global/signal_handler.lo +libglobal_la_OBJECTS = $(am_libglobal_la_OBJECTS) +libjson_spirit_la_LIBADD = +am_libjson_spirit_la_OBJECTS = json_spirit/json_spirit_reader.lo \ + json_spirit/json_spirit_writer.lo +libjson_spirit_la_OBJECTS = $(am_libjson_spirit_la_OBJECTS) +liblog_la_LIBADD = +am_liblog_la_OBJECTS = log/Log.lo log/SubsystemMap.lo +liblog_la_OBJECTS = $(am_liblog_la_OBJECTS) +libmds_la_DEPENDENCIES = $(LIBOSDC) +am_libmds_la_OBJECTS = mds/Anchor.lo mds/Capability.lo mds/Dumper.lo \ + mds/Resetter.lo mds/MDS.lo mds/flock.lo mds/locks.lo \ + mds/journal.lo mds/Server.lo mds/Mutation.lo mds/MDCache.lo \ + mds/Locker.lo mds/Migrator.lo mds/MDBalancer.lo mds/CDentry.lo \ + mds/CDir.lo mds/CInode.lo mds/LogEvent.lo mds/MDSTable.lo \ + mds/InoTable.lo mds/MDSTableClient.lo mds/MDSTableServer.lo \ + mds/AnchorServer.lo mds/AnchorClient.lo mds/SnapRealm.lo \ + mds/SnapServer.lo mds/snap.lo mds/SessionMap.lo mds/MDLog.lo \ + mds/MDSUtility.lo +libmds_la_OBJECTS = $(am_libmds_la_OBJECTS) +@WITH_LIBZFS_TRUE@am__DEPENDENCIES_4 = libos_zfs.a +am__DEPENDENCIES_5 = libos.la $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_4) +libmon_la_DEPENDENCIES = $(LIBAUTH) $(LIBCOMMON) $(am__DEPENDENCIES_5) \ + $(LIBMON_TYPES) +am_libmon_la_OBJECTS = mon/Monitor.lo mon/Paxos.lo mon/PaxosService.lo \ + mon/OSDMonitor.lo mon/MDSMonitor.lo mon/MonmapMonitor.lo \ + mon/PGMonitor.lo mon/LogMonitor.lo mon/AuthMonitor.lo \ + mon/Elector.lo mon/MonitorStore.lo mon/HealthMonitor.lo \ + mon/DataHealthService.lo mon/ConfigKeyService.lo +libmon_la_OBJECTS = $(am_libmon_la_OBJECTS) +libmon_types_la_LIBADD = +am_libmon_types_la_OBJECTS = mon/PGMap.lo +libmon_types_la_OBJECTS = $(am_libmon_types_la_OBJECTS) +libmsg_la_LIBADD = +am_libmsg_la_OBJECTS = msg/Accepter.lo msg/DispatchQueue.lo \ + msg/Message.lo msg/Messenger.lo msg/Pipe.lo \ + msg/SimpleMessenger.lo msg/msg_types.lo +libmsg_la_OBJECTS = $(am_libmsg_la_OBJECTS) +libos_la_DEPENDENCIES = $(LIBOS_TYPES) +am__libos_la_SOURCES_DIST = os/chain_xattr.cc os/DBObjectMap.cc \ + os/GenericObjectMap.cc os/FileJournal.cc os/FileStore.cc \ + os/FlatIndex.cc os/GenericFileStoreBackend.cc os/HashIndex.cc \ + os/IndexManager.cc os/JournalingObjectStore.cc \ + os/LevelDBStore.cc os/LFNIndex.cc os/MemStore.cc \ + os/KeyValueStore.cc os/ObjectStore.cc os/WBThrottle.cc \ + common/TrackedOp.cc os/BtrfsFileStoreBackend.cc \ + os/XfsFileStoreBackend.cc os/ZFSFileStoreBackend.cc +@LINUX_TRUE@am__objects_5 = os/libos_la-BtrfsFileStoreBackend.lo +@WITH_LIBXFS_TRUE@am__objects_6 = os/libos_la-XfsFileStoreBackend.lo +@WITH_LIBZFS_TRUE@am__objects_7 = os/libos_la-ZFSFileStoreBackend.lo +am_libos_la_OBJECTS = os/libos_la-chain_xattr.lo \ + os/libos_la-DBObjectMap.lo os/libos_la-GenericObjectMap.lo \ + os/libos_la-FileJournal.lo os/libos_la-FileStore.lo \ + os/libos_la-FlatIndex.lo \ + os/libos_la-GenericFileStoreBackend.lo \ + os/libos_la-HashIndex.lo os/libos_la-IndexManager.lo \ + os/libos_la-JournalingObjectStore.lo \ + os/libos_la-LevelDBStore.lo os/libos_la-LFNIndex.lo \ + os/libos_la-MemStore.lo os/libos_la-KeyValueStore.lo \ + os/libos_la-ObjectStore.lo os/libos_la-WBThrottle.lo \ + common/libos_la-TrackedOp.lo $(am__objects_5) $(am__objects_6) \ + $(am__objects_7) +libos_la_OBJECTS = $(am_libos_la_OBJECTS) +libos_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(libos_la_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +libos_types_la_LIBADD = +am_libos_types_la_OBJECTS = os/libos_types_la-Transaction.lo +libos_types_la_OBJECTS = $(am_libos_types_la_OBJECTS) +libos_types_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libos_types_la_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +libosd_la_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_5) \ + $(LIBOSD_TYPES) $(LIBOS_TYPES) +am_libosd_la_OBJECTS = osd/libosd_la-PG.lo \ + osd/libosd_la-ReplicatedPG.lo \ + osd/libosd_la-ReplicatedBackend.lo osd/libosd_la-ECBackend.lo \ + osd/libosd_la-ECMsgTypes.lo osd/libosd_la-ECTransaction.lo \ + osd/libosd_la-PGBackend.lo osd/libosd_la-Ager.lo \ + osd/libosd_la-HitSet.lo osd/libosd_la-OSD.lo \ + osd/libosd_la-OSDCap.lo osd/libosd_la-Watch.lo \ + osd/libosd_la-ClassHandler.lo osd/libosd_la-OpRequest.lo \ + common/libosd_la-TrackedOp.lo osd/libosd_la-SnapMapper.lo \ + objclass/libosd_la-class_api.lo +libosd_la_OBJECTS = $(am_libosd_la_OBJECTS) +libosd_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(libosd_la_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +libosd_types_la_LIBADD = +am_libosd_types_la_OBJECTS = osd/libosd_types_la-PGLog.lo \ + osd/libosd_types_la-osd_types.lo osd/libosd_types_la-ECUtil.lo +libosd_types_la_OBJECTS = $(am_libosd_types_la_OBJECTS) +libosd_types_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libosd_types_la_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +libosdc_la_LIBADD = +am_libosdc_la_OBJECTS = osdc/Objecter.lo osdc/ObjectCacher.lo \ + osdc/Filer.lo osdc/Striper.lo osdc/Journaler.lo +libosdc_la_OBJECTS = $(am_libosdc_la_OBJECTS) +libperfglue_la_DEPENDENCIES = +am__libperfglue_la_SOURCES_DIST = perfglue/heap_profiler.cc \ + perfglue/disabled_heap_profiler.cc perfglue/cpu_profiler.cc \ + perfglue/disabled_stubs.cc +@WITH_TCMALLOC_TRUE@am__objects_8 = perfglue/heap_profiler.lo +@WITH_TCMALLOC_FALSE@am__objects_9 = \ +@WITH_TCMALLOC_FALSE@ perfglue/disabled_heap_profiler.lo +@WITH_PROFILER_TRUE@am__objects_10 = perfglue/cpu_profiler.lo +@WITH_PROFILER_FALSE@am__objects_11 = perfglue/disabled_stubs.lo +am_libperfglue_la_OBJECTS = $(am__objects_8) $(am__objects_9) \ + $(am__objects_10) $(am__objects_11) +libperfglue_la_OBJECTS = $(am_libperfglue_la_OBJECTS) +librados_la_DEPENDENCIES = $(LIBRADOS_DEPS) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) +am_librados_la_OBJECTS = librados/librados_la-librados.lo \ + librados/librados_la-RadosClient.lo \ + librados/librados_la-IoCtxImpl.lo \ + librados/librados_la-snap_set_diff.lo +librados_la_OBJECTS = $(am_librados_la_OBJECTS) +librados_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(librados_la_CXXFLAGS) \ + $(CXXFLAGS) $(librados_la_LDFLAGS) $(LDFLAGS) -o $@ +libradostest_la_LIBADD = +am_libradostest_la_OBJECTS = test/librados/libradostest_la-test.lo \ + test/librados/libradostest_la-TestCase.lo +libradostest_la_OBJECTS = $(am_libradostest_la_OBJECTS) +libradostest_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(libradostest_la_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +librbd_la_DEPENDENCIES = $(LIBRADOS) $(LIBOSDC) libcls_rbd_client.la \ + libcls_lock_client.la $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am_librbd_la_OBJECTS = librbd/librbd.lo librbd/AioCompletion.lo \ + librbd/AioRequest.lo librbd/ImageCtx.lo librbd/internal.lo \ + librbd/LibrbdWriteback.lo librbd/WatchCtx.lo +librbd_la_OBJECTS = $(am_librbd_la_OBJECTS) +librbd_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(librbd_la_LDFLAGS) $(LDFLAGS) -o $@ +librgw_la_LIBADD = +am__librgw_la_SOURCES_DIST = rgw/librgw.cc rgw/rgw_acl.cc \ + rgw/rgw_acl_s3.cc rgw/rgw_acl_swift.cc rgw/rgw_client_io.cc \ + rgw/rgw_fcgi.cc rgw/rgw_xml.cc rgw/rgw_usage.cc \ + rgw/rgw_json_enc.cc rgw/rgw_user.cc rgw/rgw_bucket.cc \ + rgw/rgw_tools.cc rgw/rgw_rados.cc rgw/rgw_http_client.cc \ + rgw/rgw_rest_client.cc rgw/rgw_rest_conn.cc rgw/rgw_op.cc \ + rgw/rgw_common.cc rgw/rgw_cache.cc rgw/rgw_formats.cc \ + rgw/rgw_log.cc rgw/rgw_multi.cc rgw/rgw_policy_s3.cc \ + rgw/rgw_gc.cc rgw/rgw_multi_del.cc rgw/rgw_env.cc \ + rgw/rgw_cors.cc rgw/rgw_cors_s3.cc rgw/rgw_auth_s3.cc \ + rgw/rgw_metadata.cc rgw/rgw_replica_log.cc rgw/rgw_keystone.cc \ + rgw/rgw_quota.cc rgw/rgw_dencoder.cc +@WITH_RADOSGW_TRUE@am_librgw_la_OBJECTS = rgw/librgw_la-librgw.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_acl.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_acl_s3.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_acl_swift.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_client_io.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_fcgi.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_xml.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_usage.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_json_enc.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_user.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_bucket.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_tools.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_rados.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_http_client.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_rest_client.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_rest_conn.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_op.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_common.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_cache.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_formats.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_log.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_multi.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_policy_s3.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_gc.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_multi_del.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_env.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_cors.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_cors_s3.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_auth_s3.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_metadata.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_replica_log.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_keystone.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_quota.lo \ +@WITH_RADOSGW_TRUE@ rgw/librgw_la-rgw_dencoder.lo +librgw_la_OBJECTS = $(am_librgw_la_OBJECTS) +librgw_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(librgw_la_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +@WITH_RADOSGW_TRUE@am_librgw_la_rpath = +am__DEPENDENCIES_6 = $(LIBGLOBAL) $(LIBCOMMON) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) +@LINUX_TRUE@libsystest_la_DEPENDENCIES = $(am__DEPENDENCIES_6) +am__libsystest_la_SOURCES_DIST = test/system/cross_process_sem.cc \ + test/system/systest_runnable.cc \ + test/system/systest_settings.cc +@LINUX_TRUE@am_libsystest_la_OBJECTS = \ +@LINUX_TRUE@ test/system/cross_process_sem.lo \ +@LINUX_TRUE@ test/system/systest_runnable.lo \ +@LINUX_TRUE@ test/system/systest_settings.lo +libsystest_la_OBJECTS = $(am_libsystest_la_OBJECTS) +@LINUX_TRUE@am_libsystest_la_rpath = +@WITH_RADOSGW_TRUE@am__EXEEXT_1 = ceph_rgw_multiparser$(EXEEXT) \ +@WITH_RADOSGW_TRUE@ ceph_rgw_jsonparser$(EXEEXT) +@WITH_BUILD_TESTS_TRUE@am__EXEEXT_2 = test_build_libcommon$(EXEEXT) \ +@WITH_BUILD_TESTS_TRUE@ test_build_librados$(EXEEXT) \ +@WITH_BUILD_TESTS_TRUE@ test_build_librgw$(EXEEXT) \ +@WITH_BUILD_TESTS_TRUE@ test_build_libcephfs$(EXEEXT) +@LINUX_TRUE@am__EXEEXT_3 = ceph_kvstorebench$(EXEEXT) \ +@LINUX_TRUE@ ceph_test_rados_list_parallel$(EXEEXT) \ +@LINUX_TRUE@ ceph_test_rados_open_pools_parallel$(EXEEXT) \ +@LINUX_TRUE@ ceph_test_rados_delete_pools_parallel$(EXEEXT) \ +@LINUX_TRUE@ ceph_test_rados_watch_notify$(EXEEXT) +@WITH_RADOSGW_TRUE@am__EXEEXT_4 = ceph_test_cors$(EXEEXT) \ +@WITH_RADOSGW_TRUE@ ceph_test_rgw_manifest$(EXEEXT) \ +@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_meta$(EXEEXT) \ +@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_log$(EXEEXT) \ +@WITH_RADOSGW_TRUE@ ceph_test_cls_rgw_opstate$(EXEEXT) +@LINUX_TRUE@am__EXEEXT_5 = ceph_test_librbd_fsx$(EXEEXT) +@WITH_RADOSGW_TRUE@am__EXEEXT_6 = ceph_test_cls_rgw$(EXEEXT) +@LINUX_TRUE@am__EXEEXT_7 = ceph_test_objectstore$(EXEEXT) +am__EXEEXT_8 = ceph_test_ioctls$(EXEEXT) $(am__EXEEXT_1) \ + ceph_erasure_code_benchmark$(EXEEXT) \ + ceph_erasure_code$(EXEEXT) ceph_test_timers$(EXEEXT) \ + ceph_test_signal_handlers$(EXEEXT) ceph_test_rados$(EXEEXT) \ + ceph_test_mutate$(EXEEXT) ceph_test_rewrite_latency$(EXEEXT) \ + ceph_test_msgr$(EXEEXT) ceph_streamtest$(EXEEXT) \ + ceph_test_trans$(EXEEXT) ceph_test_crypto$(EXEEXT) \ + ceph_test_keys$(EXEEXT) $(am__EXEEXT_2) \ + ceph_smalliobench$(EXEEXT) ceph_smalliobenchfs$(EXEEXT) \ + ceph_smalliobenchdumb$(EXEEXT) ceph_smalliobenchrbd$(EXEEXT) \ + ceph_tpbench$(EXEEXT) ceph_omapbench$(EXEEXT) $(am__EXEEXT_3) \ + ceph_bench_log$(EXEEXT) $(am__EXEEXT_4) \ + ceph_multi_stress_watch$(EXEEXT) ceph_test_librbd$(EXEEXT) \ + $(am__EXEEXT_5) ceph_test_cls_rbd$(EXEEXT) \ + ceph_test_cls_refcount$(EXEEXT) ceph_test_cls_version$(EXEEXT) \ + ceph_test_cls_log$(EXEEXT) ceph_test_cls_statelog$(EXEEXT) \ + ceph_test_cls_replica_log$(EXEEXT) ceph_test_cls_lock$(EXEEXT) \ + ceph_test_cls_hello$(EXEEXT) $(am__EXEEXT_6) \ + ceph_test_mon_workloadgen$(EXEEXT) \ + ceph_test_rados_api_cmd$(EXEEXT) \ + ceph_test_rados_api_io$(EXEEXT) \ + ceph_test_rados_api_c_write_operations$(EXEEXT) \ + ceph_test_rados_api_c_read_operations$(EXEEXT) \ + ceph_test_rados_api_aio$(EXEEXT) \ + ceph_test_rados_api_list$(EXEEXT) \ + ceph_test_rados_api_pool$(EXEEXT) \ + ceph_test_rados_api_stat$(EXEEXT) \ + ceph_test_rados_api_watch_notify$(EXEEXT) \ + ceph_test_rados_api_snapshots$(EXEEXT) \ + ceph_test_rados_api_cls$(EXEEXT) \ + ceph_test_rados_api_misc$(EXEEXT) \ + ceph_test_rados_api_tier$(EXEEXT) \ + ceph_test_rados_api_lock$(EXEEXT) ceph_test_libcephfs$(EXEEXT) \ + $(am__EXEEXT_7) ceph_test_objectstore_workloadgen$(EXEEXT) \ + ceph_test_filestore_idempotent$(EXEEXT) \ + ceph_test_filestore_idempotent_sequence$(EXEEXT) \ + ceph_xattr_bench$(EXEEXT) ceph_test_filejournal$(EXEEXT) \ + ceph_test_stress_watch$(EXEEXT) \ + ceph_test_objectcacher_stress$(EXEEXT) \ + ceph_test_snap_mapper$(EXEEXT) ceph_test_object_map$(EXEEXT) \ + ceph_test_keyvaluedb_atomicity$(EXEEXT) \ + ceph_test_keyvaluedb_iterators$(EXEEXT) \ + ceph_test_cfuse_cache_invalidate$(EXEEXT) \ + ceph_test_c_headers$(EXEEXT) \ + ceph_test_get_blkdev_size$(EXEEXT) ceph-osdomap-tool$(EXEEXT) \ + ceph-monstore-tool$(EXEEXT) ceph-kvstore-tool$(EXEEXT) \ + ceph_scratchtool$(EXEEXT) ceph_scratchtoolpp$(EXEEXT) \ + ceph_psim$(EXEEXT) ceph_dupstore$(EXEEXT) \ + ceph_radosacl$(EXEEXT) ceph-client-debug$(EXEEXT) +@WITH_DEBUG_TRUE@am__EXEEXT_9 = $(am__EXEEXT_8) +@WITH_RADOSGW_TRUE@am__EXEEXT_10 = radosgw$(EXEEXT) \ +@WITH_RADOSGW_TRUE@ radosgw-admin$(EXEEXT) +@WITH_REST_BENCH_TRUE@am__EXEEXT_11 = rest-bench$(EXEEXT) +@LINUX_TRUE@am__EXEEXT_12 = rbd$(EXEEXT) +@WITH_FUSE_TRUE@am__EXEEXT_13 = ceph-fuse$(EXEEXT) rbd-fuse$(EXEEXT) +@LINUX_TRUE@am__EXEEXT_14 = mount.ceph$(EXEEXT) +PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) $(sbin_PROGRAMS) \ + $(su_sbin_PROGRAMS) +am_ceph_authtool_OBJECTS = tools/ceph_authtool.$(OBJEXT) +ceph_authtool_OBJECTS = $(am_ceph_authtool_OBJECTS) +ceph_authtool_DEPENDENCIES = $(am__DEPENDENCIES_6) $(LIBCOMMON) +am_ceph_client_debug_OBJECTS = tools/ceph-client-debug.$(OBJEXT) +ceph_client_debug_OBJECTS = $(am_ceph_client_debug_OBJECTS) +ceph_client_debug_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_6) \ + $(LIBCOMMON) +am_ceph_conf_OBJECTS = tools/ceph_conf.$(OBJEXT) +ceph_conf_OBJECTS = $(am_ceph_conf_OBJECTS) +ceph_conf_DEPENDENCIES = $(am__DEPENDENCIES_6) $(LIBCOMMON) +am__ceph_dencoder_SOURCES_DIST = test/encoding/ceph_dencoder.cc \ + rgw/rgw_dencoder.cc rgw/rgw_acl.cc rgw/rgw_common.cc \ + rgw/rgw_env.cc rgw/rgw_json_enc.cc +@WITH_RADOSGW_TRUE@am__objects_12 = \ +@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_dencoder.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_acl.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_common.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_env.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/ceph_dencoder-rgw_json_enc.$(OBJEXT) +am__objects_13 = $(am__objects_12) +am_ceph_dencoder_OBJECTS = \ + test/encoding/ceph_dencoder-ceph_dencoder.$(OBJEXT) \ + $(am__objects_13) +ceph_dencoder_OBJECTS = $(am_ceph_dencoder_OBJECTS) +am__DEPENDENCIES_7 = libperfglue.la $(am__DEPENDENCIES_1) +am__DEPENDENCIES_8 = libmds.la $(am__DEPENDENCIES_7) +ceph_dencoder_DEPENDENCIES = $(LIBOSD_TYPES) $(LIBOS_TYPES) \ + $(am__DEPENDENCIES_8) $(LIBMON_TYPES) $(DENCODER_DEPS) \ + $(am__DEPENDENCIES_6) +ceph_dencoder_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +am__ceph_fuse_SOURCES_DIST = ceph_fuse.cc +@WITH_FUSE_TRUE@am_ceph_fuse_OBJECTS = ceph_fuse.$(OBJEXT) +ceph_fuse_OBJECTS = $(am_ceph_fuse_OBJECTS) +@WITH_FUSE_TRUE@ceph_fuse_DEPENDENCIES = $(LIBCLIENT_FUSE) \ +@WITH_FUSE_TRUE@ $(am__DEPENDENCIES_6) +am_ceph_kvstore_tool_OBJECTS = \ + tools/ceph_kvstore_tool-ceph_kvstore_tool.$(OBJEXT) +ceph_kvstore_tool_OBJECTS = $(am_ceph_kvstore_tool_OBJECTS) +ceph_kvstore_tool_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) +ceph_kvstore_tool_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_kvstore_tool_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_mds_OBJECTS = ceph_mds.$(OBJEXT) +ceph_mds_OBJECTS = $(am_ceph_mds_OBJECTS) +ceph_mds_DEPENDENCIES = $(am__DEPENDENCIES_8) $(LIBOSDC) \ + $(am__DEPENDENCIES_6) $(LIBCOMMON) +am_ceph_mon_OBJECTS = ceph_mon.$(OBJEXT) +ceph_mon_OBJECTS = $(am_ceph_mon_OBJECTS) +am__DEPENDENCIES_9 = libmon.la $(am__DEPENDENCIES_7) +ceph_mon_DEPENDENCIES = $(am__DEPENDENCIES_9) $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) $(LIBCOMMON) +am_ceph_monstore_tool_OBJECTS = tools/ceph_monstore_tool.$(OBJEXT) +ceph_monstore_tool_OBJECTS = $(am_ceph_monstore_tool_OBJECTS) +ceph_monstore_tool_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) $(am__DEPENDENCIES_1) +am_ceph_osd_OBJECTS = ceph_osd.$(OBJEXT) +ceph_osd_OBJECTS = $(am_ceph_osd_OBJECTS) +am__DEPENDENCIES_10 = libosd.la $(LIBOSDC) $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_7) +ceph_osd_DEPENDENCIES = $(am__DEPENDENCIES_10) $(am__DEPENDENCIES_6) \ + $(LIBCOMMON) +am_ceph_osdomap_tool_OBJECTS = tools/ceph_osdomap_tool.$(OBJEXT) +ceph_osdomap_tool_OBJECTS = $(am_ceph_osdomap_tool_OBJECTS) +ceph_osdomap_tool_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) $(am__DEPENDENCIES_1) +am_ceph_syn_OBJECTS = ceph_syn.$(OBJEXT) \ + client/SyntheticClient.$(OBJEXT) +ceph_syn_OBJECTS = $(am_ceph_syn_OBJECTS) +ceph_syn_DEPENDENCIES = $(LIBCLIENT) $(am__DEPENDENCIES_6) +am_ceph_bench_log_OBJECTS = test/bench_log.$(OBJEXT) +ceph_bench_log_OBJECTS = $(am_ceph_bench_log_OBJECTS) +ceph_bench_log_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_ceph_dupstore_OBJECTS = tools/dupstore.$(OBJEXT) +ceph_dupstore_OBJECTS = $(am_ceph_dupstore_OBJECTS) +ceph_dupstore_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) +am_ceph_erasure_code_OBJECTS = \ + test/erasure-code/ceph_erasure_code.$(OBJEXT) +ceph_erasure_code_OBJECTS = $(am_ceph_erasure_code_OBJECTS) +ceph_erasure_code_DEPENDENCIES = $(am__DEPENDENCIES_10) $(LIBCOMMON) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) +am_ceph_erasure_code_benchmark_OBJECTS = \ + test/erasure-code/ceph_erasure_code_benchmark.$(OBJEXT) +ceph_erasure_code_benchmark_OBJECTS = \ + $(am_ceph_erasure_code_benchmark_OBJECTS) +ceph_erasure_code_benchmark_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(LIBCOMMON) $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) +am_ceph_filestore_dump_OBJECTS = tools/ceph_filestore_dump.$(OBJEXT) +ceph_filestore_dump_OBJECTS = $(am_ceph_filestore_dump_OBJECTS) +ceph_filestore_dump_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) +am_ceph_filestore_tool_OBJECTS = tools/ceph_filestore_tool.$(OBJEXT) +ceph_filestore_tool_OBJECTS = $(am_ceph_filestore_tool_OBJECTS) +ceph_filestore_tool_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) +am__ceph_kvstorebench_SOURCES_DIST = test/kv_store_bench.cc \ + key_value_store/kv_flat_btree_async.cc +@LINUX_TRUE@am_ceph_kvstorebench_OBJECTS = \ +@LINUX_TRUE@ test/kv_store_bench.$(OBJEXT) \ +@LINUX_TRUE@ key_value_store/kv_flat_btree_async.$(OBJEXT) +ceph_kvstorebench_OBJECTS = $(am_ceph_kvstorebench_OBJECTS) +@LINUX_TRUE@ceph_kvstorebench_DEPENDENCIES = $(LIBRADOS) \ +@LINUX_TRUE@ $(am__DEPENDENCIES_6) +am_ceph_mon_store_converter_OBJECTS = \ + tools/mon_store_converter.$(OBJEXT) +ceph_mon_store_converter_OBJECTS = \ + $(am_ceph_mon_store_converter_OBJECTS) +ceph_mon_store_converter_DEPENDENCIES = $(am__DEPENDENCIES_9) \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) +am_ceph_multi_stress_watch_OBJECTS = \ + test/multi_stress_watch.$(OBJEXT) +ceph_multi_stress_watch_OBJECTS = \ + $(am_ceph_multi_stress_watch_OBJECTS) +ceph_multi_stress_watch_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_6) $(RADOS_TEST_LDADD) +am_ceph_omapbench_OBJECTS = test/omap_bench.$(OBJEXT) +ceph_omapbench_OBJECTS = $(am_ceph_omapbench_OBJECTS) +ceph_omapbench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6) +am_ceph_psim_OBJECTS = tools/psim.$(OBJEXT) +ceph_psim_OBJECTS = $(am_ceph_psim_OBJECTS) +ceph_psim_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_ceph_radosacl_OBJECTS = tools/radosacl.$(OBJEXT) +ceph_radosacl_OBJECTS = $(am_ceph_radosacl_OBJECTS) +ceph_radosacl_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6) +am__ceph_rgw_jsonparser_SOURCES_DIST = rgw/rgw_jsonparser.cc \ + rgw/rgw_common.cc rgw/rgw_env.cc rgw/rgw_json_enc.cc +@WITH_RADOSGW_TRUE@am_ceph_rgw_jsonparser_OBJECTS = \ +@WITH_RADOSGW_TRUE@ rgw/rgw_jsonparser.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_common.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_env.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_json_enc.$(OBJEXT) +ceph_rgw_jsonparser_OBJECTS = $(am_ceph_rgw_jsonparser_OBJECTS) +@WITH_RADOSGW_TRUE@am__DEPENDENCIES_11 = $(LIBRADOS) \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \ +@WITH_RADOSGW_TRUE@ libcls_user_client.a \ +@WITH_RADOSGW_TRUE@ libcls_replica_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_lock_client.la \ +@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a +am__DEPENDENCIES_12 = $(am__DEPENDENCIES_11) +@WITH_RADOSGW_TRUE@ceph_rgw_jsonparser_DEPENDENCIES = $(LIBRGW) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_12) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6) +am__ceph_rgw_multiparser_SOURCES_DIST = rgw/rgw_multiparser.cc +@WITH_RADOSGW_TRUE@am_ceph_rgw_multiparser_OBJECTS = \ +@WITH_RADOSGW_TRUE@ rgw/rgw_multiparser.$(OBJEXT) +ceph_rgw_multiparser_OBJECTS = $(am_ceph_rgw_multiparser_OBJECTS) +@WITH_RADOSGW_TRUE@ceph_rgw_multiparser_DEPENDENCIES = $(LIBRGW) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_12) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6) +am_ceph_scratchtool_OBJECTS = tools/scratchtool.$(OBJEXT) +ceph_scratchtool_OBJECTS = $(am_ceph_scratchtool_OBJECTS) +ceph_scratchtool_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6) +am_ceph_scratchtoolpp_OBJECTS = tools/scratchtoolpp.$(OBJEXT) +ceph_scratchtoolpp_OBJECTS = $(am_ceph_scratchtoolpp_OBJECTS) +ceph_scratchtoolpp_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6) +am_ceph_smalliobench_OBJECTS = test/bench/small_io_bench.$(OBJEXT) \ + test/bench/rados_backend.$(OBJEXT) \ + test/bench/detailed_stat_collector.$(OBJEXT) \ + test/bench/bencher.$(OBJEXT) +ceph_smalliobench_OBJECTS = $(am_ceph_smalliobench_OBJECTS) +ceph_smalliobench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_6) +am_ceph_smalliobenchdumb_OBJECTS = \ + test/bench/small_io_bench_dumb.$(OBJEXT) \ + test/bench/dumb_backend.$(OBJEXT) \ + test/bench/detailed_stat_collector.$(OBJEXT) \ + test/bench/bencher.$(OBJEXT) +ceph_smalliobenchdumb_OBJECTS = $(am_ceph_smalliobenchdumb_OBJECTS) +ceph_smalliobenchdumb_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) +am_ceph_smalliobenchfs_OBJECTS = \ + test/bench/small_io_bench_fs.$(OBJEXT) \ + test/bench/testfilestore_backend.$(OBJEXT) \ + test/bench/detailed_stat_collector.$(OBJEXT) \ + test/bench/bencher.$(OBJEXT) +ceph_smalliobenchfs_OBJECTS = $(am_ceph_smalliobenchfs_OBJECTS) +ceph_smalliobenchfs_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) +am_ceph_smalliobenchrbd_OBJECTS = \ + test/bench/small_io_bench_rbd.$(OBJEXT) \ + test/bench/rbd_backend.$(OBJEXT) \ + test/bench/detailed_stat_collector.$(OBJEXT) \ + test/bench/bencher.$(OBJEXT) +ceph_smalliobenchrbd_OBJECTS = $(am_ceph_smalliobenchrbd_OBJECTS) +ceph_smalliobenchrbd_DEPENDENCIES = $(LIBRBD) $(LIBRADOS) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6) +am_ceph_streamtest_OBJECTS = test/streamtest.$(OBJEXT) +ceph_streamtest_OBJECTS = $(am_ceph_streamtest_OBJECTS) +ceph_streamtest_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) +am_ceph_test_c_headers_OBJECTS = test/test_c_headers.$(OBJEXT) +ceph_test_c_headers_OBJECTS = $(am_ceph_test_c_headers_OBJECTS) +ceph_test_c_headers_DEPENDENCIES = $(LIBRADOS) $(LIBCEPHFS) +am_ceph_test_cfuse_cache_invalidate_OBJECTS = \ + test/test_cfuse_cache_invalidate.$(OBJEXT) +ceph_test_cfuse_cache_invalidate_OBJECTS = \ + $(am_ceph_test_cfuse_cache_invalidate_OBJECTS) +ceph_test_cfuse_cache_invalidate_LDADD = $(LDADD) +am_ceph_test_cls_hello_OBJECTS = \ + test/cls_hello/ceph_test_cls_hello-test_cls_hello.$(OBJEXT) +ceph_test_cls_hello_OBJECTS = $(am_ceph_test_cls_hello_OBJECTS) +am__DEPENDENCIES_13 = $(top_builddir)/src/gtest/lib/libgtest.a \ + $(top_builddir)/src/gtest/lib/libgtest_main.a \ + $(am__DEPENDENCIES_1) +ceph_test_cls_hello_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(RADOS_TEST_LDADD) +ceph_test_cls_hello_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_cls_lock_OBJECTS = \ + test/cls_lock/ceph_test_cls_lock-test_cls_lock.$(OBJEXT) +ceph_test_cls_lock_OBJECTS = $(am_ceph_test_cls_lock_OBJECTS) +ceph_test_cls_lock_DEPENDENCIES = $(LIBRADOS) libcls_lock_client.la \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_cls_lock_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_cls_log_OBJECTS = \ + test/cls_log/ceph_test_cls_log-test_cls_log.$(OBJEXT) +ceph_test_cls_log_OBJECTS = $(am_ceph_test_cls_log_OBJECTS) +ceph_test_cls_log_DEPENDENCIES = $(LIBRADOS) libcls_log_client.a \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(RADOS_TEST_LDADD) +ceph_test_cls_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_cls_rbd_OBJECTS = \ + test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.$(OBJEXT) +ceph_test_cls_rbd_OBJECTS = $(am_ceph_test_cls_rbd_OBJECTS) +ceph_test_cls_rbd_DEPENDENCIES = $(LIBRADOS) libcls_rbd_client.la \ + libcls_lock_client.la $(am__DEPENDENCIES_13) \ + $(RADOS_TEST_LDADD) +ceph_test_cls_rbd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_cls_refcount_OBJECTS = test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.$(OBJEXT) +ceph_test_cls_refcount_OBJECTS = $(am_ceph_test_cls_refcount_OBJECTS) +ceph_test_cls_refcount_DEPENDENCIES = $(LIBRADOS) \ + libcls_refcount_client.la $(am__DEPENDENCIES_13) \ + $(RADOS_TEST_LDADD) +ceph_test_cls_refcount_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_cls_replica_log_OBJECTS = test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.$(OBJEXT) +ceph_test_cls_replica_log_OBJECTS = \ + $(am_ceph_test_cls_replica_log_OBJECTS) +ceph_test_cls_replica_log_DEPENDENCIES = $(LIBRADOS) \ + libcls_replica_log_client.a $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) $(RADOS_TEST_LDADD) +ceph_test_cls_replica_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am__ceph_test_cls_rgw_SOURCES_DIST = test/cls_rgw/test_cls_rgw.cc +@WITH_RADOSGW_TRUE@am_ceph_test_cls_rgw_OBJECTS = test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.$(OBJEXT) +ceph_test_cls_rgw_OBJECTS = $(am_ceph_test_cls_rgw_OBJECTS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_DEPENDENCIES = $(LIBRADOS) \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la $(am__DEPENDENCIES_13) \ +@WITH_RADOSGW_TRUE@ $(RADOS_TEST_LDADD) +ceph_test_cls_rgw_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am__ceph_test_cls_rgw_log_SOURCES_DIST = test/test_rgw_admin_log.cc +@WITH_RADOSGW_TRUE@am_ceph_test_cls_rgw_log_OBJECTS = test/ceph_test_cls_rgw_log-test_rgw_admin_log.$(OBJEXT) +ceph_test_cls_rgw_log_OBJECTS = $(am_ceph_test_cls_rgw_log_OBJECTS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_log_DEPENDENCIES = $(LIBRADOS) \ +@WITH_RADOSGW_TRUE@ $(LIBRGW) $(am__DEPENDENCIES_6) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \ +@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a \ +@WITH_RADOSGW_TRUE@ libcls_lock_client.la +ceph_test_cls_rgw_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_rgw_log_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am__ceph_test_cls_rgw_meta_SOURCES_DIST = test/test_rgw_admin_meta.cc +@WITH_RADOSGW_TRUE@am_ceph_test_cls_rgw_meta_OBJECTS = test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.$(OBJEXT) +ceph_test_cls_rgw_meta_OBJECTS = $(am_ceph_test_cls_rgw_meta_OBJECTS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_meta_DEPENDENCIES = $(LIBRADOS) \ +@WITH_RADOSGW_TRUE@ $(LIBRGW) $(am__DEPENDENCIES_6) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \ +@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a \ +@WITH_RADOSGW_TRUE@ libcls_lock_client.la +ceph_test_cls_rgw_meta_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_rgw_meta_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am__ceph_test_cls_rgw_opstate_SOURCES_DIST = \ + test/test_rgw_admin_opstate.cc +@WITH_RADOSGW_TRUE@am_ceph_test_cls_rgw_opstate_OBJECTS = test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.$(OBJEXT) +ceph_test_cls_rgw_opstate_OBJECTS = \ + $(am_ceph_test_cls_rgw_opstate_OBJECTS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_opstate_DEPENDENCIES = \ +@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(am__DEPENDENCIES_6) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a \ +@WITH_RADOSGW_TRUE@ libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a \ +@WITH_RADOSGW_TRUE@ libcls_lock_client.la +ceph_test_cls_rgw_opstate_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_rgw_opstate_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_ceph_test_cls_statelog_OBJECTS = test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.$(OBJEXT) +ceph_test_cls_statelog_OBJECTS = $(am_ceph_test_cls_statelog_OBJECTS) +ceph_test_cls_statelog_DEPENDENCIES = $(LIBRADOS) \ + libcls_statelog_client.a $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) $(RADOS_TEST_LDADD) +ceph_test_cls_statelog_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_cls_version_OBJECTS = test/cls_version/ceph_test_cls_version-test_cls_version.$(OBJEXT) +ceph_test_cls_version_OBJECTS = $(am_ceph_test_cls_version_OBJECTS) +ceph_test_cls_version_DEPENDENCIES = $(LIBRADOS) \ + libcls_version_client.a $(am__DEPENDENCIES_13) \ + $(RADOS_TEST_LDADD) +ceph_test_cls_version_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am__ceph_test_cors_SOURCES_DIST = test/test_cors.cc +@WITH_RADOSGW_TRUE@am_ceph_test_cors_OBJECTS = \ +@WITH_RADOSGW_TRUE@ test/ceph_test_cors-test_cors.$(OBJEXT) +ceph_test_cors_OBJECTS = $(am_ceph_test_cors_OBJECTS) +@WITH_RADOSGW_TRUE@ceph_test_cors_DEPENDENCIES = $(LIBRADOS) $(LIBRGW) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) +ceph_test_cors_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_cors_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_crypto_OBJECTS = test/testcrypto.$(OBJEXT) +ceph_test_crypto_OBJECTS = $(am_ceph_test_crypto_OBJECTS) +ceph_test_crypto_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_ceph_test_filejournal_OBJECTS = \ + test/ceph_test_filejournal-test_filejournal.$(OBJEXT) +ceph_test_filejournal_OBJECTS = $(am_ceph_test_filejournal_OBJECTS) +ceph_test_filejournal_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +ceph_test_filejournal_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_filestore_idempotent_OBJECTS = \ + test/objectstore/test_idempotent.$(OBJEXT) \ + test/objectstore/FileStoreTracker.$(OBJEXT) \ + test/common/ObjectContents.$(OBJEXT) +ceph_test_filestore_idempotent_OBJECTS = \ + $(am_ceph_test_filestore_idempotent_OBJECTS) +ceph_test_filestore_idempotent_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) +am_ceph_test_filestore_idempotent_sequence_OBJECTS = \ + test/objectstore/test_idempotent_sequence.$(OBJEXT) \ + test/objectstore/DeterministicOpSequence.$(OBJEXT) \ + test/objectstore/TestObjectStoreState.$(OBJEXT) \ + test/objectstore/FileStoreDiff.$(OBJEXT) +ceph_test_filestore_idempotent_sequence_OBJECTS = \ + $(am_ceph_test_filestore_idempotent_sequence_OBJECTS) +ceph_test_filestore_idempotent_sequence_DEPENDENCIES = \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) +am_ceph_test_get_blkdev_size_OBJECTS = \ + test/test_get_blkdev_size.$(OBJEXT) +ceph_test_get_blkdev_size_OBJECTS = \ + $(am_ceph_test_get_blkdev_size_OBJECTS) +ceph_test_get_blkdev_size_DEPENDENCIES = $(LIBCOMMON) +am_ceph_test_ioctls_OBJECTS = client/test_ioctls.$(OBJEXT) +ceph_test_ioctls_OBJECTS = $(am_ceph_test_ioctls_OBJECTS) +ceph_test_ioctls_LDADD = $(LDADD) +am_ceph_test_keys_OBJECTS = test/testkeys.$(OBJEXT) +ceph_test_keys_OBJECTS = $(am_ceph_test_keys_OBJECTS) +ceph_test_keys_DEPENDENCIES = $(am__DEPENDENCIES_9) \ + $(am__DEPENDENCIES_6) +am_ceph_test_keyvaluedb_atomicity_OBJECTS = test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.$(OBJEXT) +ceph_test_keyvaluedb_atomicity_OBJECTS = \ + $(am_ceph_test_keyvaluedb_atomicity_OBJECTS) +ceph_test_keyvaluedb_atomicity_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +ceph_test_keyvaluedb_atomicity_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_keyvaluedb_atomicity_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_ceph_test_keyvaluedb_iterators_OBJECTS = test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.$(OBJEXT) \ + test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.$(OBJEXT) +ceph_test_keyvaluedb_iterators_OBJECTS = \ + $(am_ceph_test_keyvaluedb_iterators_OBJECTS) +ceph_test_keyvaluedb_iterators_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +ceph_test_keyvaluedb_iterators_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_ceph_test_libcephfs_OBJECTS = \ + test/libcephfs/ceph_test_libcephfs-test.$(OBJEXT) \ + test/libcephfs/ceph_test_libcephfs-readdir_r_cb.$(OBJEXT) \ + test/libcephfs/ceph_test_libcephfs-caps.$(OBJEXT) \ + test/libcephfs/ceph_test_libcephfs-multiclient.$(OBJEXT) +ceph_test_libcephfs_OBJECTS = $(am_ceph_test_libcephfs_OBJECTS) +ceph_test_libcephfs_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_13) +ceph_test_libcephfs_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_librbd_OBJECTS = \ + test/librbd/ceph_test_librbd-test_librbd.$(OBJEXT) +ceph_test_librbd_OBJECTS = $(am_ceph_test_librbd_OBJECTS) +ceph_test_librbd_DEPENDENCIES = $(LIBRBD) $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(RADOS_TEST_LDADD) +ceph_test_librbd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am__ceph_test_librbd_fsx_SOURCES_DIST = test/librbd/fsx.c +@LINUX_TRUE@am_ceph_test_librbd_fsx_OBJECTS = \ +@LINUX_TRUE@ test/librbd/ceph_test_librbd_fsx-fsx.$(OBJEXT) +ceph_test_librbd_fsx_OBJECTS = $(am_ceph_test_librbd_fsx_OBJECTS) +@LINUX_TRUE@ceph_test_librbd_fsx_DEPENDENCIES = $(LIBRBD) $(LIBRADOS) +ceph_test_librbd_fsx_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_mon_workloadgen_OBJECTS = \ + test/mon/test_mon_workloadgen.$(OBJEXT) +ceph_test_mon_workloadgen_OBJECTS = \ + $(am_ceph_test_mon_workloadgen_OBJECTS) +ceph_test_mon_workloadgen_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(LIBOSDC) $(am__DEPENDENCIES_6) +am_ceph_test_msgr_OBJECTS = test/testmsgr.$(OBJEXT) +ceph_test_msgr_OBJECTS = $(am_ceph_test_msgr_OBJECTS) +ceph_test_msgr_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_ceph_test_mutate_OBJECTS = test/test_mutate.$(OBJEXT) +ceph_test_mutate_OBJECTS = $(am_ceph_test_mutate_OBJECTS) +ceph_test_mutate_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6) +am_ceph_test_object_map_OBJECTS = \ + test/ObjectMap/ceph_test_object_map-test_object_map.$(OBJEXT) \ + test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.$(OBJEXT) +ceph_test_object_map_OBJECTS = $(am_ceph_test_object_map_OBJECTS) +ceph_test_object_map_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +ceph_test_object_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_objectcacher_stress_OBJECTS = \ + test/osdc/object_cacher_stress.$(OBJEXT) \ + test/osdc/FakeWriteback.$(OBJEXT) +ceph_test_objectcacher_stress_OBJECTS = \ + $(am_ceph_test_objectcacher_stress_OBJECTS) +ceph_test_objectcacher_stress_DEPENDENCIES = $(LIBOSDC) \ + $(am__DEPENDENCIES_6) +am__ceph_test_objectstore_SOURCES_DIST = \ + test/objectstore/store_test.cc +@LINUX_TRUE@am_ceph_test_objectstore_OBJECTS = test/objectstore/ceph_test_objectstore-store_test.$(OBJEXT) +ceph_test_objectstore_OBJECTS = $(am_ceph_test_objectstore_OBJECTS) +@LINUX_TRUE@ceph_test_objectstore_DEPENDENCIES = \ +@LINUX_TRUE@ $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_13) \ +@LINUX_TRUE@ $(am__DEPENDENCIES_6) +ceph_test_objectstore_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_objectstore_workloadgen_OBJECTS = \ + test/objectstore/workload_generator.$(OBJEXT) \ + test/objectstore/TestObjectStoreState.$(OBJEXT) +ceph_test_objectstore_workloadgen_OBJECTS = \ + $(am_ceph_test_objectstore_workloadgen_OBJECTS) +ceph_test_objectstore_workloadgen_DEPENDENCIES = \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) +am_ceph_test_rados_OBJECTS = test/osd/TestRados.$(OBJEXT) \ + test/osd/TestOpStat.$(OBJEXT) test/osd/Object.$(OBJEXT) \ + test/osd/RadosModel.$(OBJEXT) +ceph_test_rados_OBJECTS = $(am_ceph_test_rados_OBJECTS) +ceph_test_rados_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6) +am_ceph_test_rados_api_aio_OBJECTS = \ + test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT) +ceph_test_rados_api_aio_OBJECTS = \ + $(am_ceph_test_rados_api_aio_OBJECTS) +ceph_test_rados_api_aio_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_aio_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_c_read_operations_OBJECTS = test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.$(OBJEXT) +ceph_test_rados_api_c_read_operations_OBJECTS = \ + $(am_ceph_test_rados_api_c_read_operations_OBJECTS) +ceph_test_rados_api_c_read_operations_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_c_read_operations_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CXXLD) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_ceph_test_rados_api_c_write_operations_OBJECTS = test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT) +ceph_test_rados_api_c_write_operations_OBJECTS = \ + $(am_ceph_test_rados_api_c_write_operations_OBJECTS) +ceph_test_rados_api_c_write_operations_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_c_write_operations_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CXXLD) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_ceph_test_rados_api_cls_OBJECTS = \ + test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT) +ceph_test_rados_api_cls_OBJECTS = \ + $(am_ceph_test_rados_api_cls_OBJECTS) +ceph_test_rados_api_cls_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_cls_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_cmd_OBJECTS = \ + test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT) +ceph_test_rados_api_cmd_OBJECTS = \ + $(am_ceph_test_rados_api_cmd_OBJECTS) +ceph_test_rados_api_cmd_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_cmd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_io_OBJECTS = \ + test/librados/ceph_test_rados_api_io-io.$(OBJEXT) +ceph_test_rados_api_io_OBJECTS = $(am_ceph_test_rados_api_io_OBJECTS) +ceph_test_rados_api_io_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_io_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_list_OBJECTS = \ + test/librados/ceph_test_rados_api_list-list.$(OBJEXT) +ceph_test_rados_api_list_OBJECTS = \ + $(am_ceph_test_rados_api_list_OBJECTS) +ceph_test_rados_api_list_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_list_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_lock_OBJECTS = \ + test/librados/ceph_test_rados_api_lock-lock.$(OBJEXT) +ceph_test_rados_api_lock_OBJECTS = \ + $(am_ceph_test_rados_api_lock_OBJECTS) +ceph_test_rados_api_lock_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_lock_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_misc_OBJECTS = \ + test/librados/ceph_test_rados_api_misc-misc.$(OBJEXT) +ceph_test_rados_api_misc_OBJECTS = \ + $(am_ceph_test_rados_api_misc_OBJECTS) +ceph_test_rados_api_misc_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(RADOS_TEST_LDADD) +ceph_test_rados_api_misc_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_pool_OBJECTS = \ + test/librados/ceph_test_rados_api_pool-pool.$(OBJEXT) +ceph_test_rados_api_pool_OBJECTS = \ + $(am_ceph_test_rados_api_pool_OBJECTS) +ceph_test_rados_api_pool_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_pool_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_snapshots_OBJECTS = test/librados/ceph_test_rados_api_snapshots-snapshots.$(OBJEXT) +ceph_test_rados_api_snapshots_OBJECTS = \ + $(am_ceph_test_rados_api_snapshots_OBJECTS) +ceph_test_rados_api_snapshots_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_snapshots_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_ceph_test_rados_api_stat_OBJECTS = \ + test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT) +ceph_test_rados_api_stat_OBJECTS = \ + $(am_ceph_test_rados_api_stat_OBJECTS) +ceph_test_rados_api_stat_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_stat_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_tier_OBJECTS = \ + test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT) \ + osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT) +ceph_test_rados_api_tier_OBJECTS = \ + $(am_ceph_test_rados_api_tier_OBJECTS) +ceph_test_rados_api_tier_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(RADOS_TEST_LDADD) +ceph_test_rados_api_tier_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_rados_api_watch_notify_OBJECTS = test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT) +ceph_test_rados_api_watch_notify_OBJECTS = \ + $(am_ceph_test_rados_api_watch_notify_OBJECTS) +ceph_test_rados_api_watch_notify_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_rados_api_watch_notify_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CXXLD) $(ceph_test_rados_api_watch_notify_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am__ceph_test_rados_delete_pools_parallel_SOURCES_DIST = \ + test/system/rados_delete_pools_parallel.cc \ + test/system/st_rados_create_pool.cc \ + test/system/st_rados_delete_pool.cc \ + test/system/st_rados_list_objects.cc +@LINUX_TRUE@am_ceph_test_rados_delete_pools_parallel_OBJECTS = \ +@LINUX_TRUE@ test/system/rados_delete_pools_parallel.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_delete_pool.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_list_objects.$(OBJEXT) +ceph_test_rados_delete_pools_parallel_OBJECTS = \ + $(am_ceph_test_rados_delete_pools_parallel_OBJECTS) +@LINUX_TRUE@ceph_test_rados_delete_pools_parallel_DEPENDENCIES = \ +@LINUX_TRUE@ $(LIBRADOS) libsystest.la $(am__DEPENDENCIES_1) +am__ceph_test_rados_list_parallel_SOURCES_DIST = \ + test/system/rados_list_parallel.cc \ + test/system/st_rados_create_pool.cc \ + test/system/st_rados_list_objects.cc +@LINUX_TRUE@am_ceph_test_rados_list_parallel_OBJECTS = \ +@LINUX_TRUE@ test/system/rados_list_parallel.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_list_objects.$(OBJEXT) +ceph_test_rados_list_parallel_OBJECTS = \ + $(am_ceph_test_rados_list_parallel_OBJECTS) +@LINUX_TRUE@ceph_test_rados_list_parallel_DEPENDENCIES = $(LIBRADOS) \ +@LINUX_TRUE@ libsystest.la $(am__DEPENDENCIES_1) +am__ceph_test_rados_open_pools_parallel_SOURCES_DIST = \ + test/system/rados_open_pools_parallel.cc \ + test/system/st_rados_create_pool.cc +@LINUX_TRUE@am_ceph_test_rados_open_pools_parallel_OBJECTS = \ +@LINUX_TRUE@ test/system/rados_open_pools_parallel.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT) +ceph_test_rados_open_pools_parallel_OBJECTS = \ + $(am_ceph_test_rados_open_pools_parallel_OBJECTS) +@LINUX_TRUE@ceph_test_rados_open_pools_parallel_DEPENDENCIES = \ +@LINUX_TRUE@ $(LIBRADOS) libsystest.la $(am__DEPENDENCIES_1) +am__ceph_test_rados_watch_notify_SOURCES_DIST = \ + test/system/rados_watch_notify.cc \ + test/system/st_rados_create_pool.cc \ + test/system/st_rados_delete_pool.cc \ + test/system/st_rados_delete_objs.cc \ + test/system/st_rados_watch.cc test/system/st_rados_notify.cc +@LINUX_TRUE@am_ceph_test_rados_watch_notify_OBJECTS = \ +@LINUX_TRUE@ test/system/rados_watch_notify.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_create_pool.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_delete_pool.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_delete_objs.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_watch.$(OBJEXT) \ +@LINUX_TRUE@ test/system/st_rados_notify.$(OBJEXT) +ceph_test_rados_watch_notify_OBJECTS = \ + $(am_ceph_test_rados_watch_notify_OBJECTS) +@LINUX_TRUE@ceph_test_rados_watch_notify_DEPENDENCIES = $(LIBRADOS) \ +@LINUX_TRUE@ libsystest.la $(am__DEPENDENCIES_1) +am_ceph_test_rewrite_latency_OBJECTS = \ + test/test_rewrite_latency.$(OBJEXT) +ceph_test_rewrite_latency_OBJECTS = \ + $(am_ceph_test_rewrite_latency_OBJECTS) +ceph_test_rewrite_latency_DEPENDENCIES = $(LIBCOMMON) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_2) +am__ceph_test_rgw_manifest_SOURCES_DIST = \ + test/rgw/test_rgw_manifest.cc +@WITH_RADOSGW_TRUE@am_ceph_test_rgw_manifest_OBJECTS = test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.$(OBJEXT) +ceph_test_rgw_manifest_OBJECTS = $(am_ceph_test_rgw_manifest_OBJECTS) +@WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_DEPENDENCIES = $(LIBRADOS) \ +@WITH_RADOSGW_TRUE@ $(LIBRGW) $(am__DEPENDENCIES_12) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_13) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) +ceph_test_rgw_manifest_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_signal_handlers_OBJECTS = \ + test/TestSignalHandlers.$(OBJEXT) +ceph_test_signal_handlers_OBJECTS = \ + $(am_ceph_test_signal_handlers_OBJECTS) +ceph_test_signal_handlers_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_ceph_test_snap_mapper_OBJECTS = \ + test/ceph_test_snap_mapper-test_snap_mapper.$(OBJEXT) +ceph_test_snap_mapper_OBJECTS = $(am_ceph_test_snap_mapper_OBJECTS) +ceph_test_snap_mapper_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +ceph_test_snap_mapper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_snap_mapper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_stress_watch_OBJECTS = \ + test/ceph_test_stress_watch-test_stress_watch.$(OBJEXT) +ceph_test_stress_watch_OBJECTS = $(am_ceph_test_stress_watch_OBJECTS) +ceph_test_stress_watch_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) $(RADOS_TEST_LDADD) +ceph_test_stress_watch_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_ceph_test_timers_OBJECTS = test/TestTimers.$(OBJEXT) +ceph_test_timers_OBJECTS = $(am_ceph_test_timers_OBJECTS) +ceph_test_timers_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_ceph_test_trans_OBJECTS = test/test_trans.$(OBJEXT) +ceph_test_trans_OBJECTS = $(am_ceph_test_trans_OBJECTS) +ceph_test_trans_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_6) +am_ceph_tpbench_OBJECTS = test/bench/tp_bench.$(OBJEXT) \ + test/bench/detailed_stat_collector.$(OBJEXT) +ceph_tpbench_OBJECTS = $(am_ceph_tpbench_OBJECTS) +ceph_tpbench_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_5) $(am__DEPENDENCIES_6) +am_ceph_xattr_bench_OBJECTS = \ + test/ceph_xattr_bench-xattr_bench.$(OBJEXT) +ceph_xattr_bench_OBJECTS = $(am_ceph_xattr_bench_OBJECTS) +ceph_xattr_bench_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +ceph_xattr_bench_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(ceph_xattr_bench_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_cephfs_OBJECTS = cephfs.$(OBJEXT) +cephfs_OBJECTS = $(am_cephfs_OBJECTS) +cephfs_DEPENDENCIES = $(LIBCOMMON) +am_crushtool_OBJECTS = tools/crushtool.$(OBJEXT) +crushtool_OBJECTS = $(am_crushtool_OBJECTS) +crushtool_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_get_command_descriptions_OBJECTS = \ + test/common/get_command_descriptions.$(OBJEXT) +get_command_descriptions_OBJECTS = \ + $(am_get_command_descriptions_OBJECTS) +get_command_descriptions_DEPENDENCIES = $(am__DEPENDENCIES_9) \ + $(LIBCOMMON) $(am__DEPENDENCIES_6) +am_librados_config_OBJECTS = librados-config.$(OBJEXT) +librados_config_OBJECTS = $(am_librados_config_OBJECTS) +librados_config_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_6) +am_monmaptool_OBJECTS = tools/monmaptool.$(OBJEXT) +monmaptool_OBJECTS = $(am_monmaptool_OBJECTS) +monmaptool_DEPENDENCIES = $(am__DEPENDENCIES_6) $(LIBCOMMON) +am_mount_ceph_OBJECTS = mount/mount.ceph.$(OBJEXT) \ + common/secret.$(OBJEXT) +mount_ceph_OBJECTS = $(am_mount_ceph_OBJECTS) +mount_ceph_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_1) +am_osdmaptool_OBJECTS = tools/osdmaptool.$(OBJEXT) +osdmaptool_OBJECTS = $(am_osdmaptool_OBJECTS) +osdmaptool_DEPENDENCIES = $(am__DEPENDENCIES_6) +am_rados_OBJECTS = tools/rados/rados.$(OBJEXT) \ + tools/rados/rados_import.$(OBJEXT) \ + tools/rados/rados_export.$(OBJEXT) \ + tools/rados/rados_sync.$(OBJEXT) common/obj_bencher.$(OBJEXT) +rados_OBJECTS = $(am_rados_OBJECTS) +rados_DEPENDENCIES = libcls_lock_client.la $(LIBRADOS) \ + $(am__DEPENDENCIES_6) +am__radosgw_SOURCES_DIST = rgw/rgw_resolve.cc rgw/rgw_rest.cc \ + rgw/rgw_rest_swift.cc rgw/rgw_rest_s3.cc rgw/rgw_rest_usage.cc \ + rgw/rgw_rest_user.cc rgw/rgw_rest_bucket.cc \ + rgw/rgw_rest_metadata.cc rgw/rgw_replica_log.cc \ + rgw/rgw_rest_log.cc rgw/rgw_rest_opstate.cc \ + rgw/rgw_rest_replica_log.cc rgw/rgw_rest_config.cc \ + rgw/rgw_http_client.cc rgw/rgw_swift.cc rgw/rgw_swift_auth.cc \ + rgw/rgw_loadgen.cc rgw/rgw_civetweb.cc civetweb/src/civetweb.c \ + rgw/rgw_main.cc +@WITH_RADOSGW_TRUE@am_radosgw_OBJECTS = rgw/rgw_resolve.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_swift.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_s3.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_usage.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_user.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_bucket.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_metadata.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_replica_log.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_log.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_opstate.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_replica_log.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_config.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_http_client.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_swift.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_swift_auth.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_loadgen.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_civetweb.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ civetweb/src/radosgw-civetweb.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/rgw_main.$(OBJEXT) +radosgw_OBJECTS = $(am_radosgw_OBJECTS) +@WITH_RADOSGW_TRUE@radosgw_DEPENDENCIES = $(LIBRGW) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_12) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6) +am__radosgw_admin_SOURCES_DIST = rgw/rgw_admin.cc +@WITH_RADOSGW_TRUE@am_radosgw_admin_OBJECTS = rgw/rgw_admin.$(OBJEXT) +radosgw_admin_OBJECTS = $(am_radosgw_admin_OBJECTS) +@WITH_RADOSGW_TRUE@radosgw_admin_DEPENDENCIES = $(LIBRGW) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_12) \ +@WITH_RADOSGW_TRUE@ $(am__DEPENDENCIES_6) +am_rbd_OBJECTS = rbd.$(OBJEXT) common/secret.$(OBJEXT) +rbd_OBJECTS = $(am_rbd_OBJECTS) +rbd_DEPENDENCIES = $(LIBRBD) $(LIBRADOS) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) +am__rbd_fuse_SOURCES_DIST = rbd_fuse/rbd-fuse.c +@WITH_FUSE_TRUE@am_rbd_fuse_OBJECTS = rbd_fuse/rbd-fuse.$(OBJEXT) +rbd_fuse_OBJECTS = $(am_rbd_fuse_OBJECTS) +@WITH_FUSE_TRUE@rbd_fuse_DEPENDENCIES = $(LIBRBD) $(LIBRADOS) \ +@WITH_FUSE_TRUE@ $(am__DEPENDENCIES_6) +am__rest_bench_SOURCES_DIST = tools/rest_bench.cc \ + common/obj_bencher.cc +@WITH_REST_BENCH_TRUE@am_rest_bench_OBJECTS = \ +@WITH_REST_BENCH_TRUE@ tools/rest_bench-rest_bench.$(OBJEXT) \ +@WITH_REST_BENCH_TRUE@ common/rest_bench-obj_bencher.$(OBJEXT) +rest_bench_OBJECTS = $(am_rest_bench_OBJECTS) +@WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE@am__DEPENDENCIES_14 = libs3/build/lib/libs3.a +@WITH_REST_BENCH_TRUE@rest_bench_DEPENDENCIES = $(am__DEPENDENCIES_6) \ +@WITH_REST_BENCH_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_REST_BENCH_TRUE@ $(am__DEPENDENCIES_14) +rest_bench_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(rest_bench_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am__test_build_libcephfs_SOURCES_DIST = test/buildtest_skeleton.cc \ + osdc/Objecter.cc osdc/ObjectCacher.cc osdc/Filer.cc \ + osdc/Striper.cc osdc/Journaler.cc +am__objects_14 = osdc/test_build_libcephfs-Objecter.$(OBJEXT) \ + osdc/test_build_libcephfs-ObjectCacher.$(OBJEXT) \ + osdc/test_build_libcephfs-Filer.$(OBJEXT) \ + osdc/test_build_libcephfs-Striper.$(OBJEXT) \ + osdc/test_build_libcephfs-Journaler.$(OBJEXT) +@WITH_BUILD_TESTS_TRUE@am_test_build_libcephfs_OBJECTS = test/test_build_libcephfs-buildtest_skeleton.$(OBJEXT) \ +@WITH_BUILD_TESTS_TRUE@ $(am__objects_14) +test_build_libcephfs_OBJECTS = $(am_test_build_libcephfs_OBJECTS) +@WITH_BUILD_TESTS_TRUE@test_build_libcephfs_DEPENDENCIES = \ +@WITH_BUILD_TESTS_TRUE@ $(LIBCEPHFS) $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_2) +test_build_libcephfs_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) \ + $(test_build_libcephfs_LDFLAGS) $(LDFLAGS) -o $@ +am__test_build_libcommon_SOURCES_DIST = test/buildtest_skeleton.cc \ + ceph_ver.c common/DecayCounter.cc common/LogClient.cc \ + common/LogEntry.cc common/PrebufferedStreambuf.cc \ + common/SloppyCRCMap.cc common/BackTrace.cc \ + common/perf_counters.cc common/Mutex.cc \ + common/OutputDataSocket.cc common/admin_socket.cc \ + common/admin_socket_client.cc common/cmdparse.cc \ + common/escape.c common/io_priority.cc common/Clock.cc \ + common/Throttle.cc common/Timer.cc common/Finisher.cc \ + common/environment.cc common/assert.cc common/run_cmd.cc \ + common/WorkQueue.cc common/ConfUtils.cc common/MemoryModel.cc \ + common/armor.c common/fd.cc common/xattr.c common/safe_io.c \ + common/snap_types.cc common/str_list.cc common/str_map.cc \ + common/errno.cc common/RefCountedObj.cc common/blkdev.cc \ + common/common_init.cc common/pipe.c common/ceph_argparse.cc \ + common/ceph_context.cc common/buffer.cc \ + common/code_environment.cc common/dout.cc common/histogram.cc \ + common/signal.cc common/simple_spin.cc common/Thread.cc \ + common/Formatter.cc common/HeartbeatMap.cc common/config.cc \ + common/utf8.c common/mime.c common/strtol.cc common/page.cc \ + common/lockdep.cc common/version.cc common/hex.cc \ + common/entity_name.cc common/ceph_crypto.cc \ + common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \ + common/pick_address.cc common/util.cc common/TextTable.cc \ + common/ceph_fs.cc common/ceph_hash.cc common/ceph_strings.cc \ + common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \ + common/bloom_filter.cc common/linux_version.c mon/MonCap.cc \ + mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc osd/osd_types.cc \ + osd/ECMsgTypes.cc osd/HitSet.cc mds/MDSMap.cc \ + mds/inode_backtrace.cc mds/mdstypes.cc +am__objects_15 = test_build_libcommon-ceph_ver.$(OBJEXT) \ + common/test_build_libcommon-DecayCounter.$(OBJEXT) \ + common/test_build_libcommon-LogClient.$(OBJEXT) \ + common/test_build_libcommon-LogEntry.$(OBJEXT) \ + common/test_build_libcommon-PrebufferedStreambuf.$(OBJEXT) \ + common/test_build_libcommon-SloppyCRCMap.$(OBJEXT) \ + common/test_build_libcommon-BackTrace.$(OBJEXT) \ + common/test_build_libcommon-perf_counters.$(OBJEXT) \ + common/test_build_libcommon-Mutex.$(OBJEXT) \ + common/test_build_libcommon-OutputDataSocket.$(OBJEXT) \ + common/test_build_libcommon-admin_socket.$(OBJEXT) \ + common/test_build_libcommon-admin_socket_client.$(OBJEXT) \ + common/test_build_libcommon-cmdparse.$(OBJEXT) \ + common/test_build_libcommon-escape.$(OBJEXT) \ + common/test_build_libcommon-io_priority.$(OBJEXT) \ + common/test_build_libcommon-Clock.$(OBJEXT) \ + common/test_build_libcommon-Throttle.$(OBJEXT) \ + common/test_build_libcommon-Timer.$(OBJEXT) \ + common/test_build_libcommon-Finisher.$(OBJEXT) \ + common/test_build_libcommon-environment.$(OBJEXT) \ + common/test_build_libcommon-assert.$(OBJEXT) \ + common/test_build_libcommon-run_cmd.$(OBJEXT) \ + common/test_build_libcommon-WorkQueue.$(OBJEXT) \ + common/test_build_libcommon-ConfUtils.$(OBJEXT) \ + common/test_build_libcommon-MemoryModel.$(OBJEXT) \ + common/test_build_libcommon-armor.$(OBJEXT) \ + common/test_build_libcommon-fd.$(OBJEXT) \ + common/test_build_libcommon-xattr.$(OBJEXT) \ + common/test_build_libcommon-safe_io.$(OBJEXT) \ + common/test_build_libcommon-snap_types.$(OBJEXT) \ + common/test_build_libcommon-str_list.$(OBJEXT) \ + common/test_build_libcommon-str_map.$(OBJEXT) \ + common/test_build_libcommon-errno.$(OBJEXT) \ + common/test_build_libcommon-RefCountedObj.$(OBJEXT) \ + common/test_build_libcommon-blkdev.$(OBJEXT) \ + common/test_build_libcommon-common_init.$(OBJEXT) \ + common/test_build_libcommon-pipe.$(OBJEXT) \ + common/test_build_libcommon-ceph_argparse.$(OBJEXT) \ + common/test_build_libcommon-ceph_context.$(OBJEXT) \ + common/test_build_libcommon-buffer.$(OBJEXT) \ + common/test_build_libcommon-code_environment.$(OBJEXT) \ + common/test_build_libcommon-dout.$(OBJEXT) \ + common/test_build_libcommon-histogram.$(OBJEXT) \ + common/test_build_libcommon-signal.$(OBJEXT) \ + common/test_build_libcommon-simple_spin.$(OBJEXT) \ + common/test_build_libcommon-Thread.$(OBJEXT) \ + common/test_build_libcommon-Formatter.$(OBJEXT) \ + common/test_build_libcommon-HeartbeatMap.$(OBJEXT) \ + common/test_build_libcommon-config.$(OBJEXT) \ + common/test_build_libcommon-utf8.$(OBJEXT) \ + common/test_build_libcommon-mime.$(OBJEXT) \ + common/test_build_libcommon-strtol.$(OBJEXT) \ + common/test_build_libcommon-page.$(OBJEXT) \ + common/test_build_libcommon-lockdep.$(OBJEXT) \ + common/test_build_libcommon-version.$(OBJEXT) \ + common/test_build_libcommon-hex.$(OBJEXT) \ + common/test_build_libcommon-entity_name.$(OBJEXT) \ + common/test_build_libcommon-ceph_crypto.$(OBJEXT) \ + common/test_build_libcommon-ceph_crypto_cms.$(OBJEXT) \ + common/test_build_libcommon-ceph_json.$(OBJEXT) \ + common/test_build_libcommon-ipaddr.$(OBJEXT) \ + common/test_build_libcommon-pick_address.$(OBJEXT) \ + common/test_build_libcommon-util.$(OBJEXT) \ + common/test_build_libcommon-TextTable.$(OBJEXT) \ + common/test_build_libcommon-ceph_fs.$(OBJEXT) \ + common/test_build_libcommon-ceph_hash.$(OBJEXT) \ + common/test_build_libcommon-ceph_strings.$(OBJEXT) \ + common/test_build_libcommon-ceph_frag.$(OBJEXT) \ + common/test_build_libcommon-addr_parsing.$(OBJEXT) \ + common/test_build_libcommon-hobject.$(OBJEXT) \ + common/test_build_libcommon-bloom_filter.$(OBJEXT) \ + common/test_build_libcommon-linux_version.$(OBJEXT) \ + mon/test_build_libcommon-MonCap.$(OBJEXT) \ + mon/test_build_libcommon-MonClient.$(OBJEXT) \ + mon/test_build_libcommon-MonMap.$(OBJEXT) \ + osd/test_build_libcommon-OSDMap.$(OBJEXT) \ + osd/test_build_libcommon-osd_types.$(OBJEXT) \ + osd/test_build_libcommon-ECMsgTypes.$(OBJEXT) \ + osd/test_build_libcommon-HitSet.$(OBJEXT) \ + mds/test_build_libcommon-MDSMap.$(OBJEXT) \ + mds/test_build_libcommon-inode_backtrace.$(OBJEXT) \ + mds/test_build_libcommon-mdstypes.$(OBJEXT) +@WITH_BUILD_TESTS_TRUE@am_test_build_libcommon_OBJECTS = test/test_build_libcommon-buildtest_skeleton.$(OBJEXT) \ +@WITH_BUILD_TESTS_TRUE@ $(am__objects_15) +test_build_libcommon_OBJECTS = $(am_test_build_libcommon_OBJECTS) +@WITH_BUILD_TESTS_TRUE@test_build_libcommon_DEPENDENCIES = \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_3) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_2) +test_build_libcommon_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) \ + $(test_build_libcommon_LDFLAGS) $(LDFLAGS) -o $@ +am__test_build_librados_SOURCES_DIST = test/buildtest_skeleton.cc \ + librados/librados.cc librados/RadosClient.cc \ + librados/IoCtxImpl.cc librados/snap_set_diff.cc +am__objects_16 = librados/test_build_librados-librados.$(OBJEXT) \ + librados/test_build_librados-RadosClient.$(OBJEXT) \ + librados/test_build_librados-IoCtxImpl.$(OBJEXT) \ + librados/test_build_librados-snap_set_diff.$(OBJEXT) +@WITH_BUILD_TESTS_TRUE@am_test_build_librados_OBJECTS = test/test_build_librados-buildtest_skeleton.$(OBJEXT) \ +@WITH_BUILD_TESTS_TRUE@ $(am__objects_16) +test_build_librados_OBJECTS = $(am_test_build_librados_OBJECTS) +@WITH_BUILD_TESTS_TRUE@test_build_librados_DEPENDENCIES = \ +@WITH_BUILD_TESTS_TRUE@ $(LIBRADOS_DEPS) $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_2) +test_build_librados_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(test_build_librados_CXXFLAGS) $(CXXFLAGS) \ + $(test_build_librados_LDFLAGS) $(LDFLAGS) -o $@ +am__test_build_librgw_SOURCES_DIST = test/buildtest_skeleton.cc \ + rgw/librgw.cc rgw/rgw_acl.cc rgw/rgw_acl_s3.cc \ + rgw/rgw_acl_swift.cc rgw/rgw_client_io.cc rgw/rgw_fcgi.cc \ + rgw/rgw_xml.cc rgw/rgw_usage.cc rgw/rgw_json_enc.cc \ + rgw/rgw_user.cc rgw/rgw_bucket.cc rgw/rgw_tools.cc \ + rgw/rgw_rados.cc rgw/rgw_http_client.cc rgw/rgw_rest_client.cc \ + rgw/rgw_rest_conn.cc rgw/rgw_op.cc rgw/rgw_common.cc \ + rgw/rgw_cache.cc rgw/rgw_formats.cc rgw/rgw_log.cc \ + rgw/rgw_multi.cc rgw/rgw_policy_s3.cc rgw/rgw_gc.cc \ + rgw/rgw_multi_del.cc rgw/rgw_env.cc rgw/rgw_cors.cc \ + rgw/rgw_cors_s3.cc rgw/rgw_auth_s3.cc rgw/rgw_metadata.cc \ + rgw/rgw_replica_log.cc rgw/rgw_keystone.cc rgw/rgw_quota.cc \ + rgw/rgw_dencoder.cc +@WITH_RADOSGW_TRUE@am__objects_17 = \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-librgw.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_acl.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_acl_s3.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_acl_swift.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_client_io.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_fcgi.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_xml.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_usage.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_json_enc.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_user.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_bucket.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_tools.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_rados.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_http_client.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_rest_client.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_rest_conn.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_op.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_common.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_cache.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_formats.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_log.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_multi.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_policy_s3.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_gc.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_multi_del.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_env.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_cors.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_cors_s3.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_auth_s3.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_metadata.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_replica_log.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_keystone.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_quota.$(OBJEXT) \ +@WITH_RADOSGW_TRUE@ rgw/test_build_librgw-rgw_dencoder.$(OBJEXT) +@WITH_BUILD_TESTS_TRUE@am_test_build_librgw_OBJECTS = test/test_build_librgw-buildtest_skeleton.$(OBJEXT) \ +@WITH_BUILD_TESTS_TRUE@ $(am__objects_17) +test_build_librgw_OBJECTS = $(am_test_build_librgw_OBJECTS) +@WITH_BUILD_TESTS_TRUE@test_build_librgw_DEPENDENCIES = \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_12) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_1) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_2) \ +@WITH_BUILD_TESTS_TRUE@ $(am__DEPENDENCIES_6) +test_build_librgw_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) \ + $(test_build_librgw_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_addrs_OBJECTS = test/unittest_addrs-test_addrs.$(OBJEXT) +unittest_addrs_OBJECTS = $(am_unittest_addrs_OBJECTS) +unittest_addrs_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_addrs_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_addrs_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_admin_socket_OBJECTS = \ + test/unittest_admin_socket-admin_socket.$(OBJEXT) +unittest_admin_socket_OBJECTS = $(am_unittest_admin_socket_OBJECTS) +unittest_admin_socket_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_admin_socket_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_admin_socket_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_arch_OBJECTS = test/unittest_arch-test_arch.$(OBJEXT) +unittest_arch_OBJECTS = $(am_unittest_arch_OBJECTS) +unittest_arch_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_arch_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_arch_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +am_unittest_base64_OBJECTS = test/unittest_base64-base64.$(OBJEXT) +unittest_base64_OBJECTS = $(am_unittest_base64_OBJECTS) +unittest_base64_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_13) +unittest_base64_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_base64_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_bloom_filter_OBJECTS = \ + test/common/unittest_bloom_filter-test_bloom_filter.$(OBJEXT) +unittest_bloom_filter_OBJECTS = $(am_unittest_bloom_filter_OBJECTS) +unittest_bloom_filter_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_bloom_filter_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_bloom_filter_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_bufferlist_OBJECTS = \ + test/unittest_bufferlist-bufferlist.$(OBJEXT) +unittest_bufferlist_OBJECTS = $(am_unittest_bufferlist_OBJECTS) +unittest_bufferlist_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_bufferlist_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_bufferlist_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_ceph_argparse_OBJECTS = \ + test/unittest_ceph_argparse-ceph_argparse.$(OBJEXT) +unittest_ceph_argparse_OBJECTS = $(am_unittest_ceph_argparse_OBJECTS) +unittest_ceph_argparse_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_ceph_argparse_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_ceph_argparse_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_ceph_compatset_OBJECTS = \ + test/unittest_ceph_compatset-ceph_compatset.$(OBJEXT) +unittest_ceph_compatset_OBJECTS = \ + $(am_unittest_ceph_compatset_OBJECTS) +unittest_ceph_compatset_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_ceph_compatset_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_ceph_compatset_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_ceph_crypto_OBJECTS = \ + test/unittest_ceph_crypto-ceph_crypto.$(OBJEXT) +unittest_ceph_crypto_OBJECTS = $(am_unittest_ceph_crypto_OBJECTS) +unittest_ceph_crypto_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_ceph_crypto_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_ceph_crypto_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_chain_xattr_OBJECTS = \ + test/objectstore/unittest_chain_xattr-chain_xattr.$(OBJEXT) +unittest_chain_xattr_OBJECTS = $(am_unittest_chain_xattr_OBJECTS) +unittest_chain_xattr_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_chain_xattr_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_config_OBJECTS = \ + test/common/unittest_config-test_config.$(OBJEXT) +unittest_config_OBJECTS = $(am_unittest_config_OBJECTS) +unittest_config_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_config_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_confutils_OBJECTS = \ + test/unittest_confutils-confutils.$(OBJEXT) +unittest_confutils_OBJECTS = $(am_unittest_confutils_OBJECTS) +unittest_confutils_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_confutils_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_confutils_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_context_OBJECTS = \ + test/common/unittest_context-test_context.$(OBJEXT) +unittest_context_OBJECTS = $(am_unittest_context_OBJECTS) +unittest_context_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_context_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_context_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_crc32c_OBJECTS = \ + test/common/unittest_crc32c-test_crc32c.$(OBJEXT) +unittest_crc32c_OBJECTS = $(am_unittest_crc32c_OBJECTS) +unittest_crc32c_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_crc32c_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_crush_indep_OBJECTS = \ + test/crush/unittest_crush_indep-indep.$(OBJEXT) +unittest_crush_indep_OBJECTS = $(am_unittest_crush_indep_OBJECTS) +unittest_crush_indep_DEPENDENCIES = $(LIBCOMMON) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_2) \ + $(am__DEPENDENCIES_6) +unittest_crush_indep_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_crush_wrapper_OBJECTS = \ + test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT) +unittest_crush_wrapper_OBJECTS = $(am_unittest_crush_wrapper_OBJECTS) +unittest_crush_wrapper_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) $(LIBCRUSH) +unittest_crush_wrapper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_crypto_OBJECTS = test/unittest_crypto-crypto.$(OBJEXT) +unittest_crypto_OBJECTS = $(am_unittest_crypto_OBJECTS) +unittest_crypto_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_crypto_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_crypto_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_crypto_init_OBJECTS = \ + test/unittest_crypto_init-crypto_init.$(OBJEXT) +unittest_crypto_init_OBJECTS = $(am_unittest_crypto_init_OBJECTS) +unittest_crypto_init_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_crypto_init_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_crypto_init_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_daemon_config_OBJECTS = \ + test/unittest_daemon_config-daemon_config.$(OBJEXT) +unittest_daemon_config_OBJECTS = $(am_unittest_daemon_config_OBJECTS) +unittest_daemon_config_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_daemon_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_daemon_config_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_ecbackend_OBJECTS = \ + test/osd/unittest_ecbackend-TestECBackend.$(OBJEXT) +unittest_ecbackend_OBJECTS = $(am_unittest_ecbackend_OBJECTS) +unittest_ecbackend_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_ecbackend_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_encoding_OBJECTS = \ + test/unittest_encoding-encoding.$(OBJEXT) +unittest_encoding_OBJECTS = $(am_unittest_encoding_OBJECTS) +unittest_encoding_DEPENDENCIES = $(LIBCEPHFS) $(LIBRADOS) \ + $(am__DEPENDENCIES_13) +unittest_encoding_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_encoding_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_erasure_code_example_OBJECTS = test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT) +unittest_erasure_code_example_OBJECTS = \ + $(am_unittest_erasure_code_example_OBJECTS) +unittest_erasure_code_example_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(LIBCOMMON) $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_erasure_code_example_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am__objects_18 = erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.$(OBJEXT) \ + erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.$(OBJEXT) \ + erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.$(OBJEXT) \ + erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.$(OBJEXT) \ + erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.$(OBJEXT) \ + erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.$(OBJEXT) \ + erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT) \ + erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT) +am_unittest_erasure_code_jerasure_OBJECTS = test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT) \ + $(am__objects_18) +unittest_erasure_code_jerasure_OBJECTS = \ + $(am_unittest_erasure_code_jerasure_OBJECTS) +unittest_erasure_code_jerasure_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(LIBCOMMON) $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) +unittest_erasure_code_jerasure_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_erasure_code_plugin_OBJECTS = test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT) +unittest_erasure_code_plugin_OBJECTS = \ + $(am_unittest_erasure_code_plugin_OBJECTS) +unittest_erasure_code_plugin_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(LIBCOMMON) $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) +unittest_erasure_code_plugin_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_erasure_code_plugin_jerasure_OBJECTS = test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT) +unittest_erasure_code_plugin_jerasure_OBJECTS = \ + $(am_unittest_erasure_code_plugin_jerasure_OBJECTS) +unittest_erasure_code_plugin_jerasure_DEPENDENCIES = \ + $(am__DEPENDENCIES_10) $(LIBCOMMON) $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) $(am__DEPENDENCIES_1) +unittest_erasure_code_plugin_jerasure_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CXXLD) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_escape_OBJECTS = test/unittest_escape-escape.$(OBJEXT) +unittest_escape_OBJECTS = $(am_unittest_escape_OBJECTS) +unittest_escape_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_escape_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_escape_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_flatindex_OBJECTS = \ + test/os/unittest_flatindex-TestFlatIndex.$(OBJEXT) +unittest_flatindex_OBJECTS = $(am_unittest_flatindex_OBJECTS) +unittest_flatindex_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_flatindex_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_flatindex_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_formatter_OBJECTS = \ + test/unittest_formatter-formatter.$(OBJEXT) \ + rgw/unittest_formatter-rgw_formats.$(OBJEXT) +unittest_formatter_OBJECTS = $(am_unittest_formatter_OBJECTS) +unittest_formatter_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_formatter_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_gather_OBJECTS = test/unittest_gather-gather.$(OBJEXT) +unittest_gather_OBJECTS = $(am_unittest_gather_OBJECTS) +unittest_gather_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_gather_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_gather_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_heartbeatmap_OBJECTS = \ + test/unittest_heartbeatmap-heartbeat_map.$(OBJEXT) +unittest_heartbeatmap_OBJECTS = $(am_unittest_heartbeatmap_OBJECTS) +unittest_heartbeatmap_DEPENDENCIES = $(LIBCOMMON) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_heartbeatmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_histogram_OBJECTS = \ + test/common/unittest_histogram-histogram.$(OBJEXT) +unittest_histogram_OBJECTS = $(am_unittest_histogram_OBJECTS) +unittest_histogram_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_histogram_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_hitset_OBJECTS = \ + test/osd/unittest_hitset-hitset.$(OBJEXT) +unittest_hitset_OBJECTS = $(am_unittest_hitset_OBJECTS) +unittest_hitset_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_hitset_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_ipaddr_OBJECTS = \ + test/unittest_ipaddr-test_ipaddr.$(OBJEXT) +unittest_ipaddr_OBJECTS = $(am_unittest_ipaddr_OBJECTS) +unittest_ipaddr_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_ipaddr_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_ipaddr_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_lfnindex_OBJECTS = \ + test/os/unittest_lfnindex-TestLFNIndex.$(OBJEXT) +unittest_lfnindex_OBJECTS = $(am_unittest_lfnindex_OBJECTS) +unittest_lfnindex_DEPENDENCIES = $(am__DEPENDENCIES_5) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_lfnindex_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_lfnindex_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_libcephfs_config_OBJECTS = \ + test/unittest_libcephfs_config-libcephfs_config.$(OBJEXT) +unittest_libcephfs_config_OBJECTS = \ + $(am_unittest_libcephfs_config_OBJECTS) +unittest_libcephfs_config_DEPENDENCIES = $(LIBCEPHFS) \ + $(am__DEPENDENCIES_13) +unittest_libcephfs_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_libcephfs_config_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_librados_OBJECTS = \ + test/librados/unittest_librados-librados.$(OBJEXT) +unittest_librados_OBJECTS = $(am_unittest_librados_OBJECTS) +unittest_librados_DEPENDENCIES = $(LIBRADOS) $(am__DEPENDENCIES_13) +unittest_librados_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_librados_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_librados_config_OBJECTS = test/librados/unittest_librados_config-librados_config.$(OBJEXT) +unittest_librados_config_OBJECTS = \ + $(am_unittest_librados_config_OBJECTS) +unittest_librados_config_DEPENDENCIES = $(LIBRADOS) \ + $(am__DEPENDENCIES_13) +unittest_librados_config_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_librados_config_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_log_OBJECTS = log/unittest_log-test.$(OBJEXT) +unittest_log_OBJECTS = $(am_unittest_log_OBJECTS) +unittest_log_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_13) +unittest_log_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(unittest_log_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_mime_OBJECTS = test/unittest_mime-mime.$(OBJEXT) +unittest_mime_OBJECTS = $(am_unittest_mime_OBJECTS) +unittest_mime_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_mime_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_mime_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +am_unittest_mon_moncap_OBJECTS = \ + test/mon/unittest_mon_moncap-moncap.$(OBJEXT) +unittest_mon_moncap_OBJECTS = $(am_unittest_mon_moncap_OBJECTS) +unittest_mon_moncap_DEPENDENCIES = $(am__DEPENDENCIES_9) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_mon_moncap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_mon_pgmap_OBJECTS = \ + test/mon/unittest_mon_pgmap-PGMap.$(OBJEXT) +unittest_mon_pgmap_OBJECTS = $(am_unittest_mon_pgmap_OBJECTS) +unittest_mon_pgmap_DEPENDENCIES = $(am__DEPENDENCIES_9) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_mon_pgmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_on_exit_OBJECTS = test/on_exit.$(OBJEXT) +unittest_on_exit_OBJECTS = $(am_unittest_on_exit_OBJECTS) +unittest_on_exit_DEPENDENCIES = $(am__DEPENDENCIES_1) +am_unittest_osd_osdcap_OBJECTS = \ + test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT) +unittest_osd_osdcap_OBJECTS = $(am_unittest_osd_osdcap_OBJECTS) +unittest_osd_osdcap_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) +unittest_osd_osdcap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_osd_types_OBJECTS = \ + test/osd/unittest_osd_types-types.$(OBJEXT) +unittest_osd_types_OBJECTS = $(am_unittest_osd_types_OBJECTS) +unittest_osd_types_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_osd_types_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_osdmap_OBJECTS = \ + test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT) +unittest_osdmap_OBJECTS = $(am_unittest_osdmap_OBJECTS) +unittest_osdmap_DEPENDENCIES = $(am__DEPENDENCIES_13) $(LIBCOMMON) \ + $(am__DEPENDENCIES_6) +unittest_osdmap_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_perf_counters_OBJECTS = \ + test/unittest_perf_counters-perf_counters.$(OBJEXT) +unittest_perf_counters_OBJECTS = $(am_unittest_perf_counters_OBJECTS) +unittest_perf_counters_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_perf_counters_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_perf_counters_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_pglog_OBJECTS = \ + test/osd/unittest_pglog-TestPGLog.$(OBJEXT) +unittest_pglog_OBJECTS = $(am_unittest_pglog_OBJECTS) +unittest_pglog_DEPENDENCIES = $(am__DEPENDENCIES_10) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_6) \ + $(am__DEPENDENCIES_1) +unittest_pglog_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_pglog_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_prebufferedstreambuf_OBJECTS = test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.$(OBJEXT) +unittest_prebufferedstreambuf_OBJECTS = \ + $(am_unittest_prebufferedstreambuf_OBJECTS) +unittest_prebufferedstreambuf_DEPENDENCIES = $(LIBCOMMON) \ + $(am__DEPENDENCIES_13) $(am__DEPENDENCIES_2) +unittest_prebufferedstreambuf_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_prebufferedstreambuf_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_run_cmd_OBJECTS = test/unittest_run_cmd-run_cmd.$(OBJEXT) +unittest_run_cmd_OBJECTS = $(am_unittest_run_cmd_OBJECTS) +unittest_run_cmd_DEPENDENCIES = $(LIBCEPHFS) $(am__DEPENDENCIES_13) +unittest_run_cmd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_run_cmd_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_sharedptr_registry_OBJECTS = test/common/unittest_sharedptr_registry-test_sharedptr_registry.$(OBJEXT) +unittest_sharedptr_registry_OBJECTS = \ + $(am_unittest_sharedptr_registry_OBJECTS) +unittest_sharedptr_registry_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_sharedptr_registry_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_sharedptr_registry_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +am_unittest_signals_OBJECTS = test/unittest_signals-signals.$(OBJEXT) +unittest_signals_OBJECTS = $(am_unittest_signals_OBJECTS) +unittest_signals_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_signals_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_signals_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_simple_spin_OBJECTS = \ + test/unittest_simple_spin-simple_spin.$(OBJEXT) +unittest_simple_spin_OBJECTS = $(am_unittest_simple_spin_OBJECTS) +unittest_simple_spin_DEPENDENCIES = $(LIBCEPHFS) \ + $(am__DEPENDENCIES_13) +unittest_simple_spin_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_simple_spin_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_sloppy_crc_map_OBJECTS = test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.$(OBJEXT) +unittest_sloppy_crc_map_OBJECTS = \ + $(am_unittest_sloppy_crc_map_OBJECTS) +unittest_sloppy_crc_map_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_sloppy_crc_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_sloppy_crc_map_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_str_list_OBJECTS = \ + test/unittest_str_list-test_str_list.$(OBJEXT) +unittest_str_list_OBJECTS = $(am_unittest_str_list_OBJECTS) +unittest_str_list_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_str_list_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_str_list_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_str_map_OBJECTS = \ + test/common/unittest_str_map-test_str_map.$(OBJEXT) +unittest_str_map_OBJECTS = $(am_unittest_str_map_OBJECTS) +unittest_str_map_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_str_map_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_striper_OBJECTS = \ + test/unittest_striper-test_striper.$(OBJEXT) +unittest_striper_OBJECTS = $(am_unittest_striper_OBJECTS) +unittest_striper_DEPENDENCIES = $(LIBOSDC) $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_striper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_striper_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_strtol_OBJECTS = test/unittest_strtol-strtol.$(OBJEXT) +unittest_strtol_OBJECTS = $(am_unittest_strtol_OBJECTS) +unittest_strtol_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_strtol_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_strtol_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_texttable_OBJECTS = \ + test/unittest_texttable-test_texttable.$(OBJEXT) +unittest_texttable_OBJECTS = $(am_unittest_texttable_OBJECTS) +unittest_texttable_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_13) +unittest_texttable_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_texttable_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_throttle_OBJECTS = \ + test/common/unittest_throttle-Throttle.$(OBJEXT) +unittest_throttle_OBJECTS = $(am_unittest_throttle_OBJECTS) +unittest_throttle_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_throttle_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_throttle_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_unittest_utf8_OBJECTS = test/unittest_utf8-utf8.$(OBJEXT) +unittest_utf8_OBJECTS = $(am_unittest_utf8_OBJECTS) +unittest_utf8_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_utf8_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_utf8_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +am_unittest_util_OBJECTS = \ + test/common/unittest_util-test_util.$(OBJEXT) +unittest_util_OBJECTS = $(am_unittest_util_OBJECTS) +unittest_util_DEPENDENCIES = $(LIBCOMMON) $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) +unittest_util_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_util_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +am_unittest_workqueue_OBJECTS = \ + test/unittest_workqueue-test_workqueue.$(OBJEXT) +unittest_workqueue_OBJECTS = $(am_unittest_workqueue_OBJECTS) +unittest_workqueue_DEPENDENCIES = $(am__DEPENDENCIES_13) \ + $(am__DEPENDENCIES_6) +unittest_workqueue_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(unittest_workqueue_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +SCRIPTS = $(bin_SCRIPTS) $(ceph_sbin_SCRIPTS) $(dist_bin_SCRIPTS) \ + $(sbin_SCRIPTS) $(shell_common_SCRIPTS) $(su_sbin_SCRIPTS) +DEFAULT_INCLUDES = -I.@am__isrc@ +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +CPPASCOMPILE = $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CCASFLAGS) $(CCASFLAGS) +LTCPPASCOMPILE = $(LIBTOOL) $(AM_V_lt) $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CCAS) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CCASFLAGS) $(CCASFLAGS) +AM_V_CPPAS = $(am__v_CPPAS_@AM_V@) +am__v_CPPAS_ = $(am__v_CPPAS_@AM_DEFAULT_V@) +am__v_CPPAS_0 = @echo " CPPAS " $@; +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CXXFLAGS) $(CXXFLAGS) +AM_V_CXX = $(am__v_CXX_@AM_V@) +am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@) +am__v_CXX_0 = @echo " CXX " $@; +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) +am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) +am__v_CXXLD_0 = @echo " CXXLD " $@; +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +SOURCES = $(libcls_log_client_a_SOURCES) \ + $(libcls_replica_log_client_a_SOURCES) \ + $(libcls_statelog_client_a_SOURCES) \ + $(libcls_user_client_a_SOURCES) \ + $(libcls_version_client_a_SOURCES) $(libos_zfs_a_SOURCES) \ + $(libarch_la_SOURCES) $(libauth_la_SOURCES) \ + $(libcephfs_la_SOURCES) $(libcephfs_jni_la_SOURCES) \ + $(libclient_la_SOURCES) $(libclient_fuse_la_SOURCES) \ + $(libcls_hello_la_SOURCES) $(libcls_kvs_la_SOURCES) \ + $(libcls_lock_la_SOURCES) $(libcls_lock_client_la_SOURCES) \ + $(libcls_log_la_SOURCES) $(libcls_rbd_la_SOURCES) \ + $(libcls_rbd_client_la_SOURCES) $(libcls_refcount_la_SOURCES) \ + $(libcls_refcount_client_la_SOURCES) \ + $(libcls_replica_log_la_SOURCES) $(libcls_rgw_la_SOURCES) \ + $(libcls_rgw_client_la_SOURCES) $(libcls_statelog_la_SOURCES) \ + $(libcls_user_la_SOURCES) $(libcls_version_la_SOURCES) \ + $(libcommon_la_SOURCES) $(libcommon_crc_la_SOURCES) \ + $(libcrush_la_SOURCES) $(libec_example_la_SOURCES) \ + $(libec_fail_to_initialize_la_SOURCES) \ + $(libec_fail_to_register_la_SOURCES) $(libec_hangs_la_SOURCES) \ + $(libec_jerasure_la_SOURCES) \ + $(libec_jerasure_generic_la_SOURCES) \ + $(libec_jerasure_sse3_la_SOURCES) \ + $(libec_jerasure_sse4_la_SOURCES) \ + $(libec_missing_entry_point_la_SOURCES) \ + $(libec_test_jerasure_generic_la_SOURCES) \ + $(libec_test_jerasure_sse3_la_SOURCES) \ + $(libec_test_jerasure_sse4_la_SOURCES) \ + $(liberasure_code_la_SOURCES) $(libglobal_la_SOURCES) \ + $(libjson_spirit_la_SOURCES) $(liblog_la_SOURCES) \ + $(libmds_la_SOURCES) $(libmon_la_SOURCES) \ + $(libmon_types_la_SOURCES) $(libmsg_la_SOURCES) \ + $(libos_la_SOURCES) $(libos_types_la_SOURCES) \ + $(libosd_la_SOURCES) $(libosd_types_la_SOURCES) \ + $(libosdc_la_SOURCES) $(libperfglue_la_SOURCES) \ + $(librados_la_SOURCES) $(libradostest_la_SOURCES) \ + $(librbd_la_SOURCES) $(librgw_la_SOURCES) \ + $(libsystest_la_SOURCES) $(ceph_authtool_SOURCES) \ + $(ceph_client_debug_SOURCES) $(ceph_conf_SOURCES) \ + $(ceph_dencoder_SOURCES) $(ceph_fuse_SOURCES) \ + $(ceph_kvstore_tool_SOURCES) $(ceph_mds_SOURCES) \ + $(ceph_mon_SOURCES) $(ceph_monstore_tool_SOURCES) \ + $(ceph_osd_SOURCES) $(ceph_osdomap_tool_SOURCES) \ + $(ceph_syn_SOURCES) $(ceph_bench_log_SOURCES) \ + $(ceph_dupstore_SOURCES) $(ceph_erasure_code_SOURCES) \ + $(ceph_erasure_code_benchmark_SOURCES) \ + $(ceph_filestore_dump_SOURCES) $(ceph_filestore_tool_SOURCES) \ + $(ceph_kvstorebench_SOURCES) \ + $(ceph_mon_store_converter_SOURCES) \ + $(ceph_multi_stress_watch_SOURCES) $(ceph_omapbench_SOURCES) \ + $(ceph_psim_SOURCES) $(ceph_radosacl_SOURCES) \ + $(ceph_rgw_jsonparser_SOURCES) $(ceph_rgw_multiparser_SOURCES) \ + $(ceph_scratchtool_SOURCES) $(ceph_scratchtoolpp_SOURCES) \ + $(ceph_smalliobench_SOURCES) $(ceph_smalliobenchdumb_SOURCES) \ + $(ceph_smalliobenchfs_SOURCES) $(ceph_smalliobenchrbd_SOURCES) \ + $(ceph_streamtest_SOURCES) $(ceph_test_c_headers_SOURCES) \ + $(ceph_test_cfuse_cache_invalidate_SOURCES) \ + $(ceph_test_cls_hello_SOURCES) $(ceph_test_cls_lock_SOURCES) \ + $(ceph_test_cls_log_SOURCES) $(ceph_test_cls_rbd_SOURCES) \ + $(ceph_test_cls_refcount_SOURCES) \ + $(ceph_test_cls_replica_log_SOURCES) \ + $(ceph_test_cls_rgw_SOURCES) $(ceph_test_cls_rgw_log_SOURCES) \ + $(ceph_test_cls_rgw_meta_SOURCES) \ + $(ceph_test_cls_rgw_opstate_SOURCES) \ + $(ceph_test_cls_statelog_SOURCES) \ + $(ceph_test_cls_version_SOURCES) $(ceph_test_cors_SOURCES) \ + $(ceph_test_crypto_SOURCES) $(ceph_test_filejournal_SOURCES) \ + $(ceph_test_filestore_idempotent_SOURCES) \ + $(ceph_test_filestore_idempotent_sequence_SOURCES) \ + $(ceph_test_get_blkdev_size_SOURCES) \ + $(ceph_test_ioctls_SOURCES) $(ceph_test_keys_SOURCES) \ + $(ceph_test_keyvaluedb_atomicity_SOURCES) \ + $(ceph_test_keyvaluedb_iterators_SOURCES) \ + $(ceph_test_libcephfs_SOURCES) $(ceph_test_librbd_SOURCES) \ + $(ceph_test_librbd_fsx_SOURCES) \ + $(ceph_test_mon_workloadgen_SOURCES) $(ceph_test_msgr_SOURCES) \ + $(ceph_test_mutate_SOURCES) $(ceph_test_object_map_SOURCES) \ + $(ceph_test_objectcacher_stress_SOURCES) \ + $(ceph_test_objectstore_SOURCES) \ + $(ceph_test_objectstore_workloadgen_SOURCES) \ + $(ceph_test_rados_SOURCES) $(ceph_test_rados_api_aio_SOURCES) \ + $(ceph_test_rados_api_c_read_operations_SOURCES) \ + $(ceph_test_rados_api_c_write_operations_SOURCES) \ + $(ceph_test_rados_api_cls_SOURCES) \ + $(ceph_test_rados_api_cmd_SOURCES) \ + $(ceph_test_rados_api_io_SOURCES) \ + $(ceph_test_rados_api_list_SOURCES) \ + $(ceph_test_rados_api_lock_SOURCES) \ + $(ceph_test_rados_api_misc_SOURCES) \ + $(ceph_test_rados_api_pool_SOURCES) \ + $(ceph_test_rados_api_snapshots_SOURCES) \ + $(ceph_test_rados_api_stat_SOURCES) \ + $(ceph_test_rados_api_tier_SOURCES) \ + $(ceph_test_rados_api_watch_notify_SOURCES) \ + $(ceph_test_rados_delete_pools_parallel_SOURCES) \ + $(ceph_test_rados_list_parallel_SOURCES) \ + $(ceph_test_rados_open_pools_parallel_SOURCES) \ + $(ceph_test_rados_watch_notify_SOURCES) \ + $(ceph_test_rewrite_latency_SOURCES) \ + $(ceph_test_rgw_manifest_SOURCES) \ + $(ceph_test_signal_handlers_SOURCES) \ + $(ceph_test_snap_mapper_SOURCES) \ + $(ceph_test_stress_watch_SOURCES) $(ceph_test_timers_SOURCES) \ + $(ceph_test_trans_SOURCES) $(ceph_tpbench_SOURCES) \ + $(ceph_xattr_bench_SOURCES) $(cephfs_SOURCES) \ + $(crushtool_SOURCES) $(get_command_descriptions_SOURCES) \ + $(librados_config_SOURCES) $(monmaptool_SOURCES) \ + $(mount_ceph_SOURCES) $(osdmaptool_SOURCES) $(rados_SOURCES) \ + $(radosgw_SOURCES) $(radosgw_admin_SOURCES) $(rbd_SOURCES) \ + $(rbd_fuse_SOURCES) $(rest_bench_SOURCES) \ + $(test_build_libcephfs_SOURCES) \ + $(test_build_libcommon_SOURCES) $(test_build_librados_SOURCES) \ + $(test_build_librgw_SOURCES) $(unittest_addrs_SOURCES) \ + $(unittest_admin_socket_SOURCES) $(unittest_arch_SOURCES) \ + $(unittest_base64_SOURCES) $(unittest_bloom_filter_SOURCES) \ + $(unittest_bufferlist_SOURCES) \ + $(unittest_ceph_argparse_SOURCES) \ + $(unittest_ceph_compatset_SOURCES) \ + $(unittest_ceph_crypto_SOURCES) \ + $(unittest_chain_xattr_SOURCES) $(unittest_config_SOURCES) \ + $(unittest_confutils_SOURCES) $(unittest_context_SOURCES) \ + $(unittest_crc32c_SOURCES) $(unittest_crush_indep_SOURCES) \ + $(unittest_crush_wrapper_SOURCES) $(unittest_crypto_SOURCES) \ + $(unittest_crypto_init_SOURCES) \ + $(unittest_daemon_config_SOURCES) \ + $(unittest_ecbackend_SOURCES) $(unittest_encoding_SOURCES) \ + $(unittest_erasure_code_example_SOURCES) \ + $(unittest_erasure_code_jerasure_SOURCES) \ + $(unittest_erasure_code_plugin_SOURCES) \ + $(unittest_erasure_code_plugin_jerasure_SOURCES) \ + $(unittest_escape_SOURCES) $(unittest_flatindex_SOURCES) \ + $(unittest_formatter_SOURCES) $(unittest_gather_SOURCES) \ + $(unittest_heartbeatmap_SOURCES) $(unittest_histogram_SOURCES) \ + $(unittest_hitset_SOURCES) $(unittest_ipaddr_SOURCES) \ + $(unittest_lfnindex_SOURCES) \ + $(unittest_libcephfs_config_SOURCES) \ + $(unittest_librados_SOURCES) \ + $(unittest_librados_config_SOURCES) $(unittest_log_SOURCES) \ + $(unittest_mime_SOURCES) $(unittest_mon_moncap_SOURCES) \ + $(unittest_mon_pgmap_SOURCES) $(unittest_on_exit_SOURCES) \ + $(unittest_osd_osdcap_SOURCES) $(unittest_osd_types_SOURCES) \ + $(unittest_osdmap_SOURCES) $(unittest_perf_counters_SOURCES) \ + $(unittest_pglog_SOURCES) \ + $(unittest_prebufferedstreambuf_SOURCES) \ + $(unittest_run_cmd_SOURCES) \ + $(unittest_sharedptr_registry_SOURCES) \ + $(unittest_signals_SOURCES) $(unittest_simple_spin_SOURCES) \ + $(unittest_sloppy_crc_map_SOURCES) \ + $(unittest_str_list_SOURCES) $(unittest_str_map_SOURCES) \ + $(unittest_striper_SOURCES) $(unittest_strtol_SOURCES) \ + $(unittest_texttable_SOURCES) $(unittest_throttle_SOURCES) \ + $(unittest_utf8_SOURCES) $(unittest_util_SOURCES) \ + $(unittest_workqueue_SOURCES) +DIST_SOURCES = $(libcls_log_client_a_SOURCES) \ + $(libcls_replica_log_client_a_SOURCES) \ + $(libcls_statelog_client_a_SOURCES) \ + $(libcls_user_client_a_SOURCES) \ + $(libcls_version_client_a_SOURCES) \ + $(am__libos_zfs_a_SOURCES_DIST) $(libarch_la_SOURCES) \ + $(libauth_la_SOURCES) $(libcephfs_la_SOURCES) \ + $(am__libcephfs_jni_la_SOURCES_DIST) $(libclient_la_SOURCES) \ + $(am__libclient_fuse_la_SOURCES_DIST) \ + $(libcls_hello_la_SOURCES) $(am__libcls_kvs_la_SOURCES_DIST) \ + $(libcls_lock_la_SOURCES) $(libcls_lock_client_la_SOURCES) \ + $(libcls_log_la_SOURCES) $(libcls_rbd_la_SOURCES) \ + $(libcls_rbd_client_la_SOURCES) $(libcls_refcount_la_SOURCES) \ + $(libcls_refcount_client_la_SOURCES) \ + $(libcls_replica_log_la_SOURCES) $(libcls_rgw_la_SOURCES) \ + $(libcls_rgw_client_la_SOURCES) $(libcls_statelog_la_SOURCES) \ + $(libcls_user_la_SOURCES) $(libcls_version_la_SOURCES) \ + $(libcommon_la_SOURCES) $(am__libcommon_crc_la_SOURCES_DIST) \ + $(libcrush_la_SOURCES) $(libec_example_la_SOURCES) \ + $(libec_fail_to_initialize_la_SOURCES) \ + $(libec_fail_to_register_la_SOURCES) $(libec_hangs_la_SOURCES) \ + $(libec_jerasure_la_SOURCES) \ + $(libec_jerasure_generic_la_SOURCES) \ + $(libec_jerasure_sse3_la_SOURCES) \ + $(libec_jerasure_sse4_la_SOURCES) \ + $(libec_missing_entry_point_la_SOURCES) \ + $(libec_test_jerasure_generic_la_SOURCES) \ + $(libec_test_jerasure_sse3_la_SOURCES) \ + $(libec_test_jerasure_sse4_la_SOURCES) \ + $(liberasure_code_la_SOURCES) $(libglobal_la_SOURCES) \ + $(libjson_spirit_la_SOURCES) $(liblog_la_SOURCES) \ + $(libmds_la_SOURCES) $(libmon_la_SOURCES) \ + $(libmon_types_la_SOURCES) $(libmsg_la_SOURCES) \ + $(am__libos_la_SOURCES_DIST) $(libos_types_la_SOURCES) \ + $(libosd_la_SOURCES) $(libosd_types_la_SOURCES) \ + $(libosdc_la_SOURCES) $(am__libperfglue_la_SOURCES_DIST) \ + $(librados_la_SOURCES) $(libradostest_la_SOURCES) \ + $(librbd_la_SOURCES) $(am__librgw_la_SOURCES_DIST) \ + $(am__libsystest_la_SOURCES_DIST) $(ceph_authtool_SOURCES) \ + $(ceph_client_debug_SOURCES) $(ceph_conf_SOURCES) \ + $(am__ceph_dencoder_SOURCES_DIST) \ + $(am__ceph_fuse_SOURCES_DIST) $(ceph_kvstore_tool_SOURCES) \ + $(ceph_mds_SOURCES) $(ceph_mon_SOURCES) \ + $(ceph_monstore_tool_SOURCES) $(ceph_osd_SOURCES) \ + $(ceph_osdomap_tool_SOURCES) $(ceph_syn_SOURCES) \ + $(ceph_bench_log_SOURCES) $(ceph_dupstore_SOURCES) \ + $(ceph_erasure_code_SOURCES) \ + $(ceph_erasure_code_benchmark_SOURCES) \ + $(ceph_filestore_dump_SOURCES) $(ceph_filestore_tool_SOURCES) \ + $(am__ceph_kvstorebench_SOURCES_DIST) \ + $(ceph_mon_store_converter_SOURCES) \ + $(ceph_multi_stress_watch_SOURCES) $(ceph_omapbench_SOURCES) \ + $(ceph_psim_SOURCES) $(ceph_radosacl_SOURCES) \ + $(am__ceph_rgw_jsonparser_SOURCES_DIST) \ + $(am__ceph_rgw_multiparser_SOURCES_DIST) \ + $(ceph_scratchtool_SOURCES) $(ceph_scratchtoolpp_SOURCES) \ + $(ceph_smalliobench_SOURCES) $(ceph_smalliobenchdumb_SOURCES) \ + $(ceph_smalliobenchfs_SOURCES) $(ceph_smalliobenchrbd_SOURCES) \ + $(ceph_streamtest_SOURCES) $(ceph_test_c_headers_SOURCES) \ + $(ceph_test_cfuse_cache_invalidate_SOURCES) \ + $(ceph_test_cls_hello_SOURCES) $(ceph_test_cls_lock_SOURCES) \ + $(ceph_test_cls_log_SOURCES) $(ceph_test_cls_rbd_SOURCES) \ + $(ceph_test_cls_refcount_SOURCES) \ + $(ceph_test_cls_replica_log_SOURCES) \ + $(am__ceph_test_cls_rgw_SOURCES_DIST) \ + $(am__ceph_test_cls_rgw_log_SOURCES_DIST) \ + $(am__ceph_test_cls_rgw_meta_SOURCES_DIST) \ + $(am__ceph_test_cls_rgw_opstate_SOURCES_DIST) \ + $(ceph_test_cls_statelog_SOURCES) \ + $(ceph_test_cls_version_SOURCES) \ + $(am__ceph_test_cors_SOURCES_DIST) $(ceph_test_crypto_SOURCES) \ + $(ceph_test_filejournal_SOURCES) \ + $(ceph_test_filestore_idempotent_SOURCES) \ + $(ceph_test_filestore_idempotent_sequence_SOURCES) \ + $(ceph_test_get_blkdev_size_SOURCES) \ + $(ceph_test_ioctls_SOURCES) $(ceph_test_keys_SOURCES) \ + $(ceph_test_keyvaluedb_atomicity_SOURCES) \ + $(ceph_test_keyvaluedb_iterators_SOURCES) \ + $(ceph_test_libcephfs_SOURCES) $(ceph_test_librbd_SOURCES) \ + $(am__ceph_test_librbd_fsx_SOURCES_DIST) \ + $(ceph_test_mon_workloadgen_SOURCES) $(ceph_test_msgr_SOURCES) \ + $(ceph_test_mutate_SOURCES) $(ceph_test_object_map_SOURCES) \ + $(ceph_test_objectcacher_stress_SOURCES) \ + $(am__ceph_test_objectstore_SOURCES_DIST) \ + $(ceph_test_objectstore_workloadgen_SOURCES) \ + $(ceph_test_rados_SOURCES) $(ceph_test_rados_api_aio_SOURCES) \ + $(ceph_test_rados_api_c_read_operations_SOURCES) \ + $(ceph_test_rados_api_c_write_operations_SOURCES) \ + $(ceph_test_rados_api_cls_SOURCES) \ + $(ceph_test_rados_api_cmd_SOURCES) \ + $(ceph_test_rados_api_io_SOURCES) \ + $(ceph_test_rados_api_list_SOURCES) \ + $(ceph_test_rados_api_lock_SOURCES) \ + $(ceph_test_rados_api_misc_SOURCES) \ + $(ceph_test_rados_api_pool_SOURCES) \ + $(ceph_test_rados_api_snapshots_SOURCES) \ + $(ceph_test_rados_api_stat_SOURCES) \ + $(ceph_test_rados_api_tier_SOURCES) \ + $(ceph_test_rados_api_watch_notify_SOURCES) \ + $(am__ceph_test_rados_delete_pools_parallel_SOURCES_DIST) \ + $(am__ceph_test_rados_list_parallel_SOURCES_DIST) \ + $(am__ceph_test_rados_open_pools_parallel_SOURCES_DIST) \ + $(am__ceph_test_rados_watch_notify_SOURCES_DIST) \ + $(ceph_test_rewrite_latency_SOURCES) \ + $(am__ceph_test_rgw_manifest_SOURCES_DIST) \ + $(ceph_test_signal_handlers_SOURCES) \ + $(ceph_test_snap_mapper_SOURCES) \ + $(ceph_test_stress_watch_SOURCES) $(ceph_test_timers_SOURCES) \ + $(ceph_test_trans_SOURCES) $(ceph_tpbench_SOURCES) \ + $(ceph_xattr_bench_SOURCES) $(cephfs_SOURCES) \ + $(crushtool_SOURCES) $(get_command_descriptions_SOURCES) \ + $(librados_config_SOURCES) $(monmaptool_SOURCES) \ + $(mount_ceph_SOURCES) $(osdmaptool_SOURCES) $(rados_SOURCES) \ + $(am__radosgw_SOURCES_DIST) $(am__radosgw_admin_SOURCES_DIST) \ + $(rbd_SOURCES) $(am__rbd_fuse_SOURCES_DIST) \ + $(am__rest_bench_SOURCES_DIST) \ + $(am__test_build_libcephfs_SOURCES_DIST) \ + $(am__test_build_libcommon_SOURCES_DIST) \ + $(am__test_build_librados_SOURCES_DIST) \ + $(am__test_build_librgw_SOURCES_DIST) \ + $(unittest_addrs_SOURCES) $(unittest_admin_socket_SOURCES) \ + $(unittest_arch_SOURCES) $(unittest_base64_SOURCES) \ + $(unittest_bloom_filter_SOURCES) \ + $(unittest_bufferlist_SOURCES) \ + $(unittest_ceph_argparse_SOURCES) \ + $(unittest_ceph_compatset_SOURCES) \ + $(unittest_ceph_crypto_SOURCES) \ + $(unittest_chain_xattr_SOURCES) $(unittest_config_SOURCES) \ + $(unittest_confutils_SOURCES) $(unittest_context_SOURCES) \ + $(unittest_crc32c_SOURCES) $(unittest_crush_indep_SOURCES) \ + $(unittest_crush_wrapper_SOURCES) $(unittest_crypto_SOURCES) \ + $(unittest_crypto_init_SOURCES) \ + $(unittest_daemon_config_SOURCES) \ + $(unittest_ecbackend_SOURCES) $(unittest_encoding_SOURCES) \ + $(unittest_erasure_code_example_SOURCES) \ + $(unittest_erasure_code_jerasure_SOURCES) \ + $(unittest_erasure_code_plugin_SOURCES) \ + $(unittest_erasure_code_plugin_jerasure_SOURCES) \ + $(unittest_escape_SOURCES) $(unittest_flatindex_SOURCES) \ + $(unittest_formatter_SOURCES) $(unittest_gather_SOURCES) \ + $(unittest_heartbeatmap_SOURCES) $(unittest_histogram_SOURCES) \ + $(unittest_hitset_SOURCES) $(unittest_ipaddr_SOURCES) \ + $(unittest_lfnindex_SOURCES) \ + $(unittest_libcephfs_config_SOURCES) \ + $(unittest_librados_SOURCES) \ + $(unittest_librados_config_SOURCES) $(unittest_log_SOURCES) \ + $(unittest_mime_SOURCES) $(unittest_mon_moncap_SOURCES) \ + $(unittest_mon_pgmap_SOURCES) $(unittest_on_exit_SOURCES) \ + $(unittest_osd_osdcap_SOURCES) $(unittest_osd_types_SOURCES) \ + $(unittest_osdmap_SOURCES) $(unittest_perf_counters_SOURCES) \ + $(unittest_pglog_SOURCES) \ + $(unittest_prebufferedstreambuf_SOURCES) \ + $(unittest_run_cmd_SOURCES) \ + $(unittest_sharedptr_registry_SOURCES) \ + $(unittest_signals_SOURCES) $(unittest_simple_spin_SOURCES) \ + $(unittest_sloppy_crc_map_SOURCES) \ + $(unittest_str_list_SOURCES) $(unittest_str_map_SOURCES) \ + $(unittest_striper_SOURCES) $(unittest_strtol_SOURCES) \ + $(unittest_texttable_SOURCES) $(unittest_throttle_SOURCES) \ + $(unittest_utf8_SOURCES) $(unittest_util_SOURCES) \ + $(unittest_workqueue_SOURCES) +RECURSIVE_TARGETS = all-recursive check-recursive dvi-recursive \ + html-recursive info-recursive install-data-recursive \ + install-dvi-recursive install-exec-recursive \ + install-html-recursive install-info-recursive \ + install-pdf-recursive install-ps-recursive install-recursive \ + installcheck-recursive installdirs-recursive pdf-recursive \ + ps-recursive uninstall-recursive +am__py_compile = PYTHON=$(PYTHON) $(SHELL) $(py_compile) +py_compile = $(top_srcdir)/py-compile +DATA = $(bash_completion_DATA) $(doc_DATA) $(libcephfs_include_DATA) \ + $(librbd_include_DATA) $(rados_include_DATA) +am__noinst_HEADERS_DIST = arch/intel.h arch/neon.h arch/probe.h \ + auth/cephx/CephxAuthorizeHandler.h auth/cephx/CephxKeyServer.h \ + auth/cephx/CephxProtocol.h auth/cephx/CephxClientHandler.h \ + auth/cephx/CephxServiceHandler.h \ + auth/cephx/CephxSessionHandler.h \ + auth/none/AuthNoneAuthorizeHandler.h \ + auth/none/AuthNoneClientHandler.h \ + auth/none/AuthNoneServiceHandler.h \ + auth/none/AuthNoneSessionHandler.h \ + auth/none/AuthNoneProtocol.h \ + auth/unknown/AuthUnknownAuthorizeHandler.h \ + auth/unknown/AuthUnknownClientHandler.h \ + auth/unknown/AuthUnknownServiceHandler.h \ + auth/unknown/AuthUnknownSessionHandler.h \ + auth/unknown/AuthUnknownProtocol.h auth/Auth.h \ + auth/AuthMethodList.h auth/AuthClientHandler.h \ + auth/AuthServiceHandler.h auth/AuthSessionHandler.h \ + auth/AuthAuthorizeHandler.h auth/KeyRing.h \ + auth/RotatingKeyRing.h auth/Crypto.h crush/CrushCompiler.h \ + crush/CrushTester.h crush/CrushWrapper.h crush/CrushWrapper.i \ + crush/builder.h crush/crush.h crush/grammar.h crush/hash.h \ + crush/mapper.h crush/sample.txt crush/types.h \ + mon/AuthMonitor.h mon/DataHealthService.h mon/Elector.h \ + mon/LogMonitor.h mon/ConfigKeyService.h mon/HealthMonitor.h \ + mon/HealthService.h mon/MDSMonitor.h mon/MonmapMonitor.h \ + mon/MonCap.h mon/MonClient.h mon/MonCommands.h \ + mon/DumplingMonCommands.h mon/MonMap.h mon/Monitor.h \ + mon/MonitorStore.h mon/MonitorDBStore.h mon/OSDMonitor.h \ + mon/PGMap.h mon/PGMonitor.h mon/Paxos.h mon/PaxosService.h \ + mon/QuorumService.h mon/Session.h mon/mon_types.h \ + mds/inode_backtrace.h mds/flock.h mds/locks.c mds/locks.h \ + mds/Anchor.h mds/AnchorClient.h mds/AnchorServer.h \ + mds/CDentry.h mds/CDir.h mds/CInode.h mds/Capability.h \ + mds/Dumper.h mds/InoTable.h mds/LocalLock.h mds/Locker.h \ + mds/LogEvent.h mds/LogSegment.h mds/MDBalancer.h mds/MDCache.h \ + mds/MDLog.h mds/MDS.h mds/MDSMap.h mds/MDSTable.h \ + mds/MDSTableServer.h mds/MDSTableClient.h mds/MDSUtility.h \ + mds/Mutation.h mds/Migrator.h mds/Resetter.h mds/ScatterLock.h \ + mds/Server.h mds/SessionMap.h mds/SimpleLock.h \ + mds/SnapClient.h mds/SnapRealm.h mds/SnapServer.h \ + mds/mds_table_types.h mds/mdstypes.h mds/snap.h \ + mds/events/ECommitted.h mds/events/EExport.h \ + mds/events/EFragment.h mds/events/EImportFinish.h \ + mds/events/EImportStart.h mds/events/EMetaBlob.h \ + mds/events/EOpen.h mds/events/EResetJournal.h \ + mds/events/ESession.h mds/events/ESessions.h \ + mds/events/ESlaveUpdate.h mds/events/ESubtreeMap.h \ + mds/events/ETableClient.h mds/events/ETableServer.h \ + mds/events/EUpdate.h os/btrfs_ioctl.h os/chain_xattr.h \ + os/BtrfsFileStoreBackend.h os/CollectionIndex.h \ + os/DBObjectMap.h os/GenericObjectMap.h os/FileJournal.h \ + os/FileStore.h os/FlatIndex.h os/FDCache.h \ + os/GenericFileStoreBackend.h os/HashIndex.h os/IndexManager.h \ + os/Journal.h os/JournalingObjectStore.h os/KeyValueDB.h \ + os/LevelDBStore.h os/LFNIndex.h os/MemStore.h \ + os/KeyValueStore.h os/ObjectMap.h os/ObjectStore.h \ + os/SequencerPosition.h os/WBThrottle.h \ + os/XfsFileStoreBackend.h os/ZFSFileStoreBackend.h os/ZFS.h \ + osd/Ager.h osd/ClassHandler.h osd/HitSet.h osd/OSD.h \ + osd/OSDCap.h osd/OSDMap.h osd/ObjectVersioner.h \ + osd/OpRequest.h osd/SnapMapper.h osd/PG.h osd/PGLog.h \ + osd/ReplicatedPG.h osd/PGBackend.h osd/ReplicatedBackend.h \ + osd/TierAgentState.h osd/ECBackend.h osd/ECUtil.h \ + osd/ECMsgTypes.h osd/ECTransaction.h osd/Watch.h \ + osd/osd_types.h \ + erasure-code/jerasure/jerasure/include/cauchy.h \ + erasure-code/jerasure/jerasure/include/galois.h \ + erasure-code/jerasure/jerasure/include/jerasure.h \ + erasure-code/jerasure/jerasure/include/liberation.h \ + erasure-code/jerasure/jerasure/include/reed_sol.h \ + erasure-code/jerasure/gf-complete/include/gf_int.h \ + erasure-code/jerasure/gf-complete/include/gf_complete.h \ + erasure-code/jerasure/gf-complete/include/gf_rand.h \ + erasure-code/jerasure/gf-complete/include/gf_method.h \ + erasure-code/jerasure/gf-complete/include/gf_general.h \ + erasure-code/jerasure/ErasureCodeJerasure.h \ + erasure-code/ErasureCodeInterface.h \ + erasure-code/ErasureCodePlugin.h osdc/Blinker.h osdc/Filer.h \ + osdc/Journaler.h osdc/ObjectCacher.h osdc/Objecter.h \ + osdc/Striper.h osdc/WritebackHandler.h client/Client.h \ + client/Dentry.h client/Dir.h client/Fh.h client/Inode.h \ + client/MetaRequest.h client/MetaSession.h \ + client/ClientSnapRealm.h client/SyntheticClient.h \ + client/Trace.h client/ioctl.h client/ObjecterWriteback.h \ + client/fuse_ll.h global/pidfile.h global/global_init.h \ + global/global_context.h global/signal_handler.h \ + json_spirit/json_spirit.h \ + json_spirit/json_spirit_error_position.h \ + json_spirit/json_spirit_reader.h \ + json_spirit/json_spirit_reader_template.h \ + json_spirit/json_spirit_stream_reader.h \ + json_spirit/json_spirit_utils.h \ + json_spirit/json_spirit_value.h \ + json_spirit/json_spirit_writer.h \ + json_spirit/json_spirit_writer_options.h \ + json_spirit/json_spirit_writer_template.h log/Entry.h \ + log/EntryQueue.h log/Log.h log/SubsystemMap.h \ + perfglue/cpu_profiler.h perfglue/heap_profiler.h \ + common/bloom_filter.hpp common/sctp_crc32.h \ + common/crc32c_intel_baseline.h common/crc32c_intel_fast.h \ + common/BackTrace.h common/RefCountedObj.h \ + common/HeartbeatMap.h common/LogClient.h common/LogEntry.h \ + common/Preforker.h common/SloppyCRCMap.h common/WorkQueue.h \ + common/PrioritizedQueue.h common/ceph_argparse.h \ + common/ceph_context.h common/xattr.h common/blkdev.h \ + common/compiler_extensions.h common/debug.h common/dout.h \ + common/escape.h common/fd.h common/version.h common/hex.h \ + common/histogram.h common/entity_name.h common/errno.h \ + common/environment.h common/likely.h common/lockdep.h \ + common/obj_bencher.h common/snap_types.h common/Clock.h \ + common/Cond.h common/ConfUtils.h common/DecayCounter.h \ + common/Finisher.h common/Formatter.h common/perf_counters.h \ + common/OutputDataSocket.h common/admin_socket.h \ + common/admin_socket_client.h common/random_cache.hpp \ + common/shared_cache.hpp common/tracked_int_ptr.hpp \ + common/simple_cache.hpp common/sharedptr_registry.hpp \ + common/map_cacher.hpp common/MemoryModel.h common/Mutex.h \ + common/PrebufferedStreambuf.h common/RWLock.h \ + common/Semaphore.h common/SimpleRNG.h common/TextTable.h \ + common/Thread.h common/Throttle.h common/Timer.h \ + common/TrackedOp.h common/arch.h common/armor.h \ + common/common_init.h common/io_priority.h common/pipe.h \ + common/code_environment.h common/signal.h common/simple_spin.h \ + common/run_cmd.h common/safe_io.h common/config.h \ + common/config_obs.h common/config_opts.h common/ceph_crypto.h \ + common/ceph_crypto_cms.h common/ceph_json.h common/lru_map.h \ + common/utf8.h common/mime.h common/pick_address.h \ + common/secret.h common/strtol.h common/static_assert.h \ + common/AsyncReserver.h common/sync_filesystem.h \ + common/cmdparse.h common/hobject.h common/linux_version.h \ + msg/Accepter.h msg/DispatchQueue.h msg/Dispatcher.h \ + msg/Message.h msg/Messenger.h msg/Pipe.h msg/SimpleMessenger.h \ + msg/msg_types.h messages/MAuth.h messages/MAuthReply.h \ + messages/MCacheExpire.h messages/MClientCaps.h \ + messages/MClientCapRelease.h messages/MClientLease.h \ + messages/MClientReconnect.h messages/MClientReply.h \ + messages/MClientRequest.h messages/MClientRequestForward.h \ + messages/MClientSession.h messages/MClientSnap.h \ + messages/MCommand.h messages/MCommandReply.h \ + messages/MDentryLink.h messages/MDentryUnlink.h \ + messages/MDirUpdate.h messages/MDiscover.h \ + messages/MDiscoverReply.h messages/MExportCaps.h \ + messages/MExportCapsAck.h messages/MExportDir.h \ + messages/MExportDirAck.h messages/MExportDirCancel.h \ + messages/MExportDirDiscover.h messages/MExportDirDiscoverAck.h \ + messages/MExportDirFinish.h messages/MExportDirNotify.h \ + messages/MExportDirNotifyAck.h messages/MExportDirPrep.h \ + messages/MExportDirPrepAck.h messages/MGenericMessage.h \ + messages/MGetPoolStats.h messages/MGetPoolStatsReply.h \ + messages/MHeartbeat.h messages/MInodeFileCaps.h \ + messages/MLock.h messages/MLog.h messages/MLogAck.h \ + messages/MMDSBeacon.h messages/MMDSCacheRejoin.h \ + messages/MMDSLoadTargets.h messages/MMDSFindIno.h \ + messages/MMDSFindInoReply.h messages/MMDSFragmentNotify.h \ + messages/MMDSMap.h messages/MMDSOpenIno.h \ + messages/MMDSOpenInoReply.h messages/MMDSResolve.h \ + messages/MMDSResolveAck.h messages/MMDSSlaveRequest.h \ + messages/MMDSTableRequest.h messages/MMonCommand.h \ + messages/MMonCommandAck.h messages/MMonElection.h \ + messages/MMonGetMap.h messages/MMonGetVersion.h \ + messages/MMonGetVersionReply.h messages/MMonGlobalID.h \ + messages/MMonHealth.h messages/MMonJoin.h messages/MMonMap.h \ + messages/MMonPaxos.h messages/MMonProbe.h messages/MMonScrub.h \ + messages/MMonSubscribe.h messages/MMonSubscribeAck.h \ + messages/MMonSync.h messages/MOSDAlive.h messages/MOSDBoot.h \ + messages/MOSDFailure.h messages/MOSDMarkMeDown.h \ + messages/MOSDMap.h messages/MOSDOp.h messages/MOSDOpReply.h \ + messages/MOSDPGBackfill.h messages/MOSDPGCreate.h \ + messages/MOSDPGPush.h messages/MOSDPGPull.h \ + messages/MOSDPGPushReply.h messages/MOSDPGInfo.h \ + messages/MOSDPGLog.h messages/MOSDPGMissing.h \ + messages/MOSDPGNotify.h messages/MOSDPGQuery.h \ + messages/MOSDPGRemove.h messages/MOSDPGScan.h \ + messages/MOSDECSubOpWrite.h messages/MOSDECSubOpWriteReply.h \ + messages/MOSDECSubOpRead.h messages/MOSDECSubOpReadReply.h \ + messages/MBackfillReserve.h messages/MRecoveryReserve.h \ + messages/MMonQuorumService.h messages/MOSDPGTemp.h \ + messages/MOSDPGTrim.h messages/MOSDPing.h \ + messages/MOSDRepScrub.h messages/MOSDScrub.h \ + messages/MOSDSubOp.h messages/MOSDSubOpReply.h \ + messages/MPGStats.h messages/MPGStatsAck.h messages/MPing.h \ + messages/MPoolOp.h messages/MPoolOpReply.h \ + messages/MRemoveSnaps.h messages/MRoute.h messages/MForward.h \ + messages/MStatfs.h messages/MStatfsReply.h \ + messages/MTimeCheck.h messages/MWatchNotify.h \ + messages/PaxosServiceMessage.h include/Context.h \ + include/CompatSet.h include/Distribution.h include/Spinlock.h \ + include/addr_parsing.h include/assert.h include/atomic.h \ + include/bitmapper.h include/blobhash.h include/buffer.h \ + include/byteorder.h include/cephfs/libcephfs.h \ + include/ceph_features.h include/ceph_frag.h include/ceph_fs.h \ + include/ceph_hash.h include/cmp.h include/color.h \ + include/compat.h include/crc32c.h include/encoding.h \ + include/err.h include/error.h include/filepath.h \ + include/frag.h include/hash.h include/intarith.h \ + include/interval_set.h include/int_types.h include/ipaddr.h \ + include/linux_fiemap.h include/lru.h include/msgr.h \ + include/object.h include/page.h include/rangeset.h \ + include/rados.h include/rbd_types.h include/statlite.h \ + include/str_list.h include/str_map.h include/stringify.h \ + include/triple.h include/types.h include/utime.h \ + include/dlist.h include/elist.h include/uuid.h include/xlist.h \ + include/rados/librados.h include/rados/rados_types.h \ + include/rados/rados_types.hpp include/rados/librados.hpp \ + include/rados/librgw.h include/rados/page.h \ + include/rados/crc32c.h include/rados/buffer.h \ + include/rbd/features.h include/rbd/librbd.h \ + include/rbd/librbd.hpp include/util.h include/stat.h \ + include/on_exit.h include/memory.h include/rados/memory.h \ + include/hash_namespace.h include/unordered_set.h \ + include/unordered_map.h librados/snap_set_diff.h \ + librados/AioCompletionImpl.h librados/IoCtxImpl.h \ + librados/PoolAsyncCompletionImpl.h librados/RadosClient.h \ + librbd/AioCompletion.h librbd/AioRequest.h librbd/ImageCtx.h \ + librbd/internal.h librbd/LibrbdWriteback.h \ + librbd/parent_types.h librbd/SnapInfo.h librbd/WatchCtx.h \ + rgw/logrotate.conf rgw/rgw_acl.h rgw/rgw_acl_s3.h \ + rgw/rgw_acl_swift.h rgw/rgw_client_io.h rgw/rgw_fcgi.h \ + rgw/rgw_xml.h rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \ + rgw/rgw_cors_s3.h rgw/rgw_cors_swift.h rgw/rgw_string.h \ + rgw/rgw_formats.h rgw/rgw_http_errors.h rgw/rgw_log.h \ + rgw/rgw_loadgen.h rgw/rgw_multi.h rgw/rgw_policy_s3.h \ + rgw/rgw_gc.h rgw/rgw_metadata.h rgw/rgw_multi_del.h \ + rgw/rgw_op.h rgw/rgw_http_client.h rgw/rgw_swift.h \ + rgw/rgw_swift_auth.h rgw/rgw_quota.h rgw/rgw_rados.h \ + rgw/rgw_replica_log.h rgw/rgw_resolve.h rgw/rgw_rest.h \ + rgw/rgw_rest_swift.h rgw/rgw_rest_s3.h rgw/rgw_auth_s3.h \ + rgw/rgw_rest_admin.h rgw/rgw_rest_usage.h rgw/rgw_rest_user.h \ + rgw/rgw_rest_bucket.h rgw/rgw_rest_client.h \ + rgw/rgw_rest_conn.h rgw/rgw_tools.h rgw/rgw_rest_metadata.h \ + rgw/rgw_rest_log.h rgw/rgw_rest_opstate.h \ + rgw/rgw_rest_replica_log.h rgw/rgw_rest_config.h \ + rgw/rgw_usage.h rgw/rgw_user.h rgw/rgw_bucket.h \ + rgw/rgw_keystone.h rgw/rgw_civetweb.h civetweb/civetweb.h \ + civetweb/include/civetweb.h civetweb/src/md5.h \ + cls/lock/cls_lock_types.h cls/lock/cls_lock_ops.h \ + cls/lock/cls_lock_client.h cls/rbd/cls_rbd.h \ + cls/rbd/cls_rbd_client.h cls/refcount/cls_refcount_ops.h \ + cls/refcount/cls_refcount_client.h \ + cls/version/cls_version_types.h cls/version/cls_version_ops.h \ + cls/version/cls_version_client.h cls/log/cls_log_types.h \ + cls/log/cls_log_ops.h cls/log/cls_log_client.h \ + cls/statelog/cls_statelog_types.h \ + cls/statelog/cls_statelog_ops.h \ + cls/statelog/cls_statelog_client.h \ + cls/replica_log/cls_replica_log_types.h \ + cls/replica_log/cls_replica_log_ops.h \ + cls/replica_log/cls_replica_log_client.h \ + cls/rgw/cls_rgw_client.h cls/rgw/cls_rgw_ops.h \ + cls/rgw/cls_rgw_types.h cls/user/cls_user_client.h \ + cls/user/cls_user_ops.h cls/user/cls_user_types.h \ + key_value_store/key_value_structure.h \ + key_value_store/kv_flat_btree_async.h \ + key_value_store/kvs_arg_types.h \ + test/erasure-code/ErasureCodeExample.h \ + test/erasure-code/ceph_erasure_code_benchmark.h \ + test/bench/backend.h test/bench/bencher.h \ + test/bench/detailed_stat_collector.h test/bench/distribution.h \ + test/bench/dumb_backend.h test/bench/rados_backend.h \ + test/bench/rbd_backend.h test/bench/stat_collector.h \ + test/bench/testfilestore_backend.h \ + test/common/ObjectContents.h test/encoding/types.h \ + test/objectstore/DeterministicOpSequence.h \ + test/objectstore/FileStoreDiff.h \ + test/objectstore/FileStoreTracker.h \ + test/objectstore/TestObjectStoreState.h \ + test/objectstore/workload_generator.h test/kv_store_bench.h \ + test/librados/test.h test/librados/TestCase.h \ + test/ObjectMap/KeyValueDBMemory.h test/omap_bench.h \ + test/osdc/FakeWriteback.h test/osd/Object.h \ + test/osd/RadosModel.h test/osd/TestOpStat.h \ + test/system/cross_process_sem.h \ + test/system/st_rados_create_pool.h \ + test/system/st_rados_delete_objs.h \ + test/system/st_rados_delete_pool.h \ + test/system/st_rados_list_objects.h \ + test/system/st_rados_notify.h test/system/st_rados_watch.h \ + test/system/systest_runnable.h test/system/systest_settings.h \ + test/unit.h tools/rados/rados_sync.h tools/common.h cls_acl.cc \ + cls_crypto.cc fetch_config logrotate.conf sample.ceph.conf \ + bash_completion/ceph bash_completion/rados bash_completion/rbd \ + bash_completion/radosgw-admin mount/canonicalize.c \ + mount/mtab.c objclass/objclass.h +HEADERS = $(noinst_HEADERS) +RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ + distclean-recursive maintainer-clean-recursive +AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \ + $(RECURSIVE_CLEAN_TARGETS:-recursive=) tags TAGS ctags CTAGS \ + distdir +ETAGS = etags +CTAGS = ctags +am__tty_colors = \ +red=; grn=; lgn=; blu=; std= +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +am__relativize = \ + dir0=`pwd`; \ + sed_first='s,^\([^/]*\)/.*$$,\1,'; \ + sed_rest='s,^[^/]*/*,,'; \ + sed_last='s,^.*/\([^/]*\)$$,\1,'; \ + sed_butlast='s,/*[^/]*$$,,'; \ + while test -n "$$dir1"; do \ + first=`echo "$$dir1" | sed -e "$$sed_first"`; \ + if test "$$first" != "."; then \ + if test "$$first" = ".."; then \ + dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \ + dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \ + else \ + first2=`echo "$$dir2" | sed -e "$$sed_first"`; \ + if test "$$first2" = "$$first"; then \ + dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \ + else \ + dir2="../$$dir2"; \ + fi; \ + dir0="$$dir0"/"$$first"; \ + fi; \ + fi; \ + dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \ + done; \ + reldir="$$dir2" +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_CXXFLAGS = @AM_CXXFLAGS@ $(AM_COMMON_CFLAGS) -ftemplate-depth-1024 \ + -Wnon-virtual-dtor -Wno-invalid-offsetof $(am__append_3) \ + $(am__append_6) $(am__append_27) +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@ +CC = @CC@ +CCAS = ${srcdir}/yasm-wrapper +CCASDEPMODE = @CCASDEPMODE@ +CCASFLAGS = @CCASFLAGS@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPTOPP_CFLAGS = @CRYPTOPP_CFLAGS@ +CRYPTOPP_LIBS = @CRYPTOPP_LIBS@ +CRYPTO_CFLAGS = @CRYPTO_CFLAGS@ +CRYPTO_LIBS = @CRYPTO_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +EXTRA_CLASSPATH_JAR = @EXTRA_CLASSPATH_JAR@ +FGREP = @FGREP@ +GCOV_PREFIX_STRIP = @GCOV_PREFIX_STRIP@ +GIT_CHECK = @GIT_CHECK@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +INTEL_FLAGS = @INTEL_FLAGS@ +INTEL_PCLMUL_FLAGS = @INTEL_PCLMUL_FLAGS@ +INTEL_SSE2_FLAGS = @INTEL_SSE2_FLAGS@ +INTEL_SSE3_FLAGS = @INTEL_SSE3_FLAGS@ +INTEL_SSE4_1_FLAGS = @INTEL_SSE4_1_FLAGS@ +INTEL_SSE4_2_FLAGS = @INTEL_SSE4_2_FLAGS@ +INTEL_SSE_FLAGS = @INTEL_SSE_FLAGS@ +INTEL_SSSE3_FLAGS = @INTEL_SSSE3_FLAGS@ +JAR = @JAR@ +JAVAC = @JAVAC@ +JAVAH = @JAVAH@ +JDK_CPPFLAGS = @JDK_CPPFLAGS@ +KEYUTILS_LIB = @KEYUTILS_LIB@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@ +LIBEDIT_LIBS = @LIBEDIT_LIBS@ +LIBFUSE = @LIBFUSE@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTCMALLOC = @LIBTCMALLOC@ +LIBTOOL = @LIBTOOL@ +LIBZFS_CFLAGS = @LIBZFS_CFLAGS@ +LIBZFS_LIBS = @LIBZFS_LIBS@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NSS_CFLAGS = @NSS_CFLAGS@ +NSS_LIBS = @NSS_LIBS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +PTHREAD_CC = @PTHREAD_CC@ +PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ +PTHREAD_LIBS = @PTHREAD_LIBS@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +RESOLV_LIBS = @RESOLV_LIBS@ +RPM_RELEASE = @RPM_RELEASE@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +WARN_IGNORED_QUALIFIERS = @WARN_IGNORED_QUALIFIERS@ +WARN_TYPE_LIMITS = @WARN_TYPE_LIMITS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +acx_pthread_config = @acx_pthread_config@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +subdirs = @subdirs@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +AUTOMAKE_OPTIONS = gnu subdir-objects +SUBDIRS = ocf java $(am__append_61) +DIST_SUBDIRS = gtest ocf libs3 java +BUILT_SOURCES = init-ceph + +# extra bits +EXTRA_DIST = brag/server brag/README.md brag/client \ + $(srcdir)/test/mon/mon-test-helpers.sh \ + $(srcdir)/test/osd/osd-test-helpers.sh \ + $(srcdir)/$(shell_scripts:%=%.in) \ + $(srcdir)/verify-mds-journal.sh $(srcdir)/vstart.sh \ + $(srcdir)/stop.sh ceph-run $(srcdir)/ceph_common.sh \ + $(srcdir)/init-radosgw $(srcdir)/init-radosgw.sysv \ + $(srcdir)/init-rbdmap $(srcdir)/ceph-clsinfo \ + $(srcdir)/make_version $(srcdir)/check_version \ + $(srcdir)/.git_version $(srcdir)/ceph-rbdnamer \ + $(srcdir)/test/encoding/readable.sh \ + $(srcdir)/test/encoding/check-generated.sh \ + $(srcdir)/upstart/ceph-all.conf \ + $(srcdir)/upstart/ceph-mon.conf \ + $(srcdir)/upstart/ceph-mon-all.conf \ + $(srcdir)/upstart/ceph-mon-all-starter.conf \ + $(srcdir)/upstart/ceph-create-keys.conf \ + $(srcdir)/upstart/ceph-osd.conf \ + $(srcdir)/upstart/ceph-osd-all.conf \ + $(srcdir)/upstart/ceph-osd-all-starter.conf \ + $(srcdir)/upstart/ceph-mds.conf \ + $(srcdir)/upstart/ceph-mds-all.conf \ + $(srcdir)/upstart/ceph-mds-all-starter.conf \ + $(srcdir)/upstart/radosgw.conf \ + $(srcdir)/upstart/radosgw-all.conf \ + $(srcdir)/upstart/radosgw-all-starter.conf \ + $(srcdir)/upstart/rbdmap.conf ceph.in ceph-disk \ + ceph-disk-prepare ceph-disk-activate ceph-disk-udev \ + ceph-create-keys ceph-rest-api ceph-crush-location \ + mount.fuse.ceph rbdmap unittest_bufferlist.sh yasm-wrapper \ + libs3/COPYING libs3/ChangeLog libs3/GNUmakefile \ + libs3/GNUmakefile.mingw libs3/GNUmakefile.osx libs3/INSTALL \ + libs3/LICENSE libs3/README libs3/TODO libs3/archlinux \ + libs3/debian libs3/doxyfile libs3/inc libs3/libs3.spec \ + libs3/mswin libs3/src libs3/test unittest_bufferlist.sh +CLEANFILES = $(shell_scripts) ceph_ver.h sample.fetch_config + +# jerasure plugin + +# everything else we want to include in a 'make dist' +noinst_HEADERS = arch/intel.h arch/neon.h arch/probe.h \ + auth/cephx/CephxAuthorizeHandler.h auth/cephx/CephxKeyServer.h \ + auth/cephx/CephxProtocol.h auth/cephx/CephxClientHandler.h \ + auth/cephx/CephxServiceHandler.h \ + auth/cephx/CephxSessionHandler.h \ + auth/none/AuthNoneAuthorizeHandler.h \ + auth/none/AuthNoneClientHandler.h \ + auth/none/AuthNoneServiceHandler.h \ + auth/none/AuthNoneSessionHandler.h \ + auth/none/AuthNoneProtocol.h \ + auth/unknown/AuthUnknownAuthorizeHandler.h \ + auth/unknown/AuthUnknownClientHandler.h \ + auth/unknown/AuthUnknownServiceHandler.h \ + auth/unknown/AuthUnknownSessionHandler.h \ + auth/unknown/AuthUnknownProtocol.h auth/Auth.h \ + auth/AuthMethodList.h auth/AuthClientHandler.h \ + auth/AuthServiceHandler.h auth/AuthSessionHandler.h \ + auth/AuthAuthorizeHandler.h auth/KeyRing.h \ + auth/RotatingKeyRing.h auth/Crypto.h crush/CrushCompiler.h \ + crush/CrushTester.h crush/CrushWrapper.h crush/CrushWrapper.i \ + crush/builder.h crush/crush.h crush/grammar.h crush/hash.h \ + crush/mapper.h crush/sample.txt crush/types.h \ + mon/AuthMonitor.h mon/DataHealthService.h mon/Elector.h \ + mon/LogMonitor.h mon/ConfigKeyService.h mon/HealthMonitor.h \ + mon/HealthService.h mon/MDSMonitor.h mon/MonmapMonitor.h \ + mon/MonCap.h mon/MonClient.h mon/MonCommands.h \ + mon/DumplingMonCommands.h mon/MonMap.h mon/Monitor.h \ + mon/MonitorStore.h mon/MonitorDBStore.h mon/OSDMonitor.h \ + mon/PGMap.h mon/PGMonitor.h mon/Paxos.h mon/PaxosService.h \ + mon/QuorumService.h mon/Session.h mon/mon_types.h \ + mds/inode_backtrace.h mds/flock.h mds/locks.c mds/locks.h \ + mds/Anchor.h mds/AnchorClient.h mds/AnchorServer.h \ + mds/CDentry.h mds/CDir.h mds/CInode.h mds/Capability.h \ + mds/Dumper.h mds/InoTable.h mds/LocalLock.h mds/Locker.h \ + mds/LogEvent.h mds/LogSegment.h mds/MDBalancer.h mds/MDCache.h \ + mds/MDLog.h mds/MDS.h mds/MDSMap.h mds/MDSTable.h \ + mds/MDSTableServer.h mds/MDSTableClient.h mds/MDSUtility.h \ + mds/Mutation.h mds/Migrator.h mds/Resetter.h mds/ScatterLock.h \ + mds/Server.h mds/SessionMap.h mds/SimpleLock.h \ + mds/SnapClient.h mds/SnapRealm.h mds/SnapServer.h \ + mds/inode_backtrace.h mds/mds_table_types.h mds/mdstypes.h \ + mds/snap.h mds/events/ECommitted.h mds/events/EExport.h \ + mds/events/EFragment.h mds/events/EImportFinish.h \ + mds/events/EImportStart.h mds/events/EMetaBlob.h \ + mds/events/EOpen.h mds/events/EResetJournal.h \ + mds/events/ESession.h mds/events/ESessions.h \ + mds/events/ESlaveUpdate.h mds/events/ESubtreeMap.h \ + mds/events/ETableClient.h mds/events/ETableServer.h \ + mds/events/EUpdate.h os/btrfs_ioctl.h os/chain_xattr.h \ + os/BtrfsFileStoreBackend.h os/CollectionIndex.h \ + os/DBObjectMap.h os/GenericObjectMap.h os/FileJournal.h \ + os/FileStore.h os/FlatIndex.h os/FDCache.h \ + os/GenericFileStoreBackend.h os/HashIndex.h os/IndexManager.h \ + os/Journal.h os/JournalingObjectStore.h os/KeyValueDB.h \ + os/LevelDBStore.h os/LFNIndex.h os/MemStore.h \ + os/KeyValueStore.h os/ObjectMap.h os/ObjectStore.h \ + os/SequencerPosition.h os/WBThrottle.h \ + os/XfsFileStoreBackend.h os/ZFSFileStoreBackend.h \ + $(am__append_18) osd/Ager.h osd/ClassHandler.h osd/HitSet.h \ + osd/OSD.h osd/OSDCap.h osd/OSDMap.h osd/ObjectVersioner.h \ + osd/OpRequest.h osd/SnapMapper.h osd/PG.h osd/PGLog.h \ + osd/ReplicatedPG.h osd/PGBackend.h osd/ReplicatedBackend.h \ + osd/TierAgentState.h osd/ECBackend.h osd/ECUtil.h \ + osd/ECMsgTypes.h osd/ECTransaction.h osd/Watch.h \ + osd/osd_types.h \ + erasure-code/jerasure/jerasure/include/cauchy.h \ + erasure-code/jerasure/jerasure/include/galois.h \ + erasure-code/jerasure/jerasure/include/jerasure.h \ + erasure-code/jerasure/jerasure/include/liberation.h \ + erasure-code/jerasure/jerasure/include/reed_sol.h \ + erasure-code/jerasure/gf-complete/include/gf_int.h \ + erasure-code/jerasure/gf-complete/include/gf_complete.h \ + erasure-code/jerasure/gf-complete/include/gf_rand.h \ + erasure-code/jerasure/gf-complete/include/gf_method.h \ + erasure-code/jerasure/gf-complete/include/gf_general.h \ + erasure-code/jerasure/ErasureCodeJerasure.h \ + erasure-code/ErasureCodeInterface.h \ + erasure-code/ErasureCodePlugin.h osdc/Blinker.h osdc/Filer.h \ + osdc/Journaler.h osdc/ObjectCacher.h osdc/Objecter.h \ + osdc/Striper.h osdc/WritebackHandler.h client/Client.h \ + client/Dentry.h client/Dir.h client/Fh.h client/Inode.h \ + client/MetaRequest.h client/MetaSession.h \ + client/ClientSnapRealm.h client/SyntheticClient.h \ + client/Trace.h client/ioctl.h client/ObjecterWriteback.h \ + $(am__append_24) global/pidfile.h global/global_init.h \ + global/global_context.h global/signal_handler.h \ + json_spirit/json_spirit.h \ + json_spirit/json_spirit_error_position.h \ + json_spirit/json_spirit_reader.h \ + json_spirit/json_spirit_reader_template.h \ + json_spirit/json_spirit_stream_reader.h \ + json_spirit/json_spirit_utils.h \ + json_spirit/json_spirit_value.h \ + json_spirit/json_spirit_writer.h \ + json_spirit/json_spirit_writer_options.h \ + json_spirit/json_spirit_writer_template.h log/Entry.h \ + log/EntryQueue.h log/Log.h log/SubsystemMap.h \ + perfglue/cpu_profiler.h perfglue/heap_profiler.h \ + common/bloom_filter.hpp common/sctp_crc32.h \ + common/crc32c_intel_baseline.h common/crc32c_intel_fast.h \ + common/BackTrace.h common/RefCountedObj.h \ + common/HeartbeatMap.h common/LogClient.h common/LogEntry.h \ + common/Preforker.h common/SloppyCRCMap.h common/WorkQueue.h \ + common/PrioritizedQueue.h common/ceph_argparse.h \ + common/ceph_context.h common/xattr.h common/blkdev.h \ + common/compiler_extensions.h common/debug.h common/dout.h \ + common/escape.h common/fd.h common/version.h common/hex.h \ + common/histogram.h common/entity_name.h common/errno.h \ + common/environment.h common/likely.h common/lockdep.h \ + common/obj_bencher.h common/snap_types.h common/Clock.h \ + common/Cond.h common/ConfUtils.h common/DecayCounter.h \ + common/Finisher.h common/Formatter.h common/perf_counters.h \ + common/OutputDataSocket.h common/admin_socket.h \ + common/admin_socket_client.h common/random_cache.hpp \ + common/shared_cache.hpp common/tracked_int_ptr.hpp \ + common/simple_cache.hpp common/sharedptr_registry.hpp \ + common/map_cacher.hpp common/MemoryModel.h common/Mutex.h \ + common/PrebufferedStreambuf.h common/RWLock.h \ + common/Semaphore.h common/SimpleRNG.h common/TextTable.h \ + common/Thread.h common/Throttle.h common/Timer.h \ + common/TrackedOp.h common/arch.h common/armor.h \ + common/common_init.h common/io_priority.h common/pipe.h \ + common/code_environment.h common/signal.h common/simple_spin.h \ + common/run_cmd.h common/safe_io.h common/config.h \ + common/config_obs.h common/config_opts.h common/ceph_crypto.h \ + common/ceph_crypto_cms.h common/ceph_json.h common/lru_map.h \ + common/utf8.h common/mime.h common/pick_address.h \ + common/secret.h common/strtol.h common/static_assert.h \ + common/AsyncReserver.h common/sync_filesystem.h \ + common/cmdparse.h common/hobject.h common/linux_version.h \ + msg/Accepter.h msg/DispatchQueue.h msg/Dispatcher.h \ + msg/Message.h msg/Messenger.h msg/Pipe.h msg/SimpleMessenger.h \ + msg/msg_types.h messages/MAuth.h messages/MAuthReply.h \ + messages/MCacheExpire.h messages/MClientCaps.h \ + messages/MClientCapRelease.h messages/MClientLease.h \ + messages/MClientReconnect.h messages/MClientReply.h \ + messages/MClientRequest.h messages/MClientRequestForward.h \ + messages/MClientSession.h messages/MClientSnap.h \ + messages/MCommand.h messages/MCommandReply.h \ + messages/MDentryLink.h messages/MDentryUnlink.h \ + messages/MDirUpdate.h messages/MDiscover.h \ + messages/MDiscoverReply.h messages/MExportCaps.h \ + messages/MExportCapsAck.h messages/MExportDir.h \ + messages/MExportDirAck.h messages/MExportDirCancel.h \ + messages/MExportDirDiscover.h messages/MExportDirDiscoverAck.h \ + messages/MExportDirFinish.h messages/MExportDirNotify.h \ + messages/MExportDirNotifyAck.h messages/MExportDirPrep.h \ + messages/MExportDirPrepAck.h messages/MGenericMessage.h \ + messages/MGetPoolStats.h messages/MGetPoolStatsReply.h \ + messages/MHeartbeat.h messages/MInodeFileCaps.h \ + messages/MLock.h messages/MLog.h messages/MLogAck.h \ + messages/MMDSBeacon.h messages/MMDSCacheRejoin.h \ + messages/MMDSLoadTargets.h messages/MMDSFindIno.h \ + messages/MMDSFindInoReply.h messages/MMDSFragmentNotify.h \ + messages/MMDSMap.h messages/MMDSOpenIno.h \ + messages/MMDSOpenInoReply.h messages/MMDSResolve.h \ + messages/MMDSResolveAck.h messages/MMDSSlaveRequest.h \ + messages/MMDSTableRequest.h messages/MMonCommand.h \ + messages/MMonCommandAck.h messages/MMonElection.h \ + messages/MMonGetMap.h messages/MMonGetVersion.h \ + messages/MMonGetVersionReply.h messages/MMonGlobalID.h \ + messages/MMonHealth.h messages/MMonJoin.h messages/MMonMap.h \ + messages/MMonPaxos.h messages/MMonProbe.h messages/MMonScrub.h \ + messages/MMonSubscribe.h messages/MMonSubscribeAck.h \ + messages/MMonSync.h messages/MOSDAlive.h messages/MOSDBoot.h \ + messages/MOSDFailure.h messages/MOSDMarkMeDown.h \ + messages/MOSDMap.h messages/MOSDOp.h messages/MOSDOpReply.h \ + messages/MOSDPGBackfill.h messages/MOSDPGCreate.h \ + messages/MOSDPGPush.h messages/MOSDPGPull.h \ + messages/MOSDPGPushReply.h messages/MOSDPGInfo.h \ + messages/MOSDPGLog.h messages/MOSDPGMissing.h \ + messages/MOSDPGNotify.h messages/MOSDPGQuery.h \ + messages/MOSDPGRemove.h messages/MOSDPGScan.h \ + messages/MOSDECSubOpWrite.h messages/MOSDECSubOpWriteReply.h \ + messages/MOSDECSubOpRead.h messages/MOSDECSubOpReadReply.h \ + messages/MBackfillReserve.h messages/MRecoveryReserve.h \ + messages/MMonQuorumService.h messages/MOSDPGTemp.h \ + messages/MOSDPGTrim.h messages/MOSDPing.h \ + messages/MOSDRepScrub.h messages/MOSDScrub.h \ + messages/MOSDSubOp.h messages/MOSDSubOpReply.h \ + messages/MPGStats.h messages/MPGStatsAck.h messages/MPing.h \ + messages/MPoolOp.h messages/MPoolOpReply.h \ + messages/MRemoveSnaps.h messages/MRoute.h messages/MForward.h \ + messages/MStatfs.h messages/MStatfsReply.h \ + messages/MTimeCheck.h messages/MWatchNotify.h \ + messages/PaxosServiceMessage.h include/Context.h \ + include/CompatSet.h include/Distribution.h include/Spinlock.h \ + include/addr_parsing.h include/assert.h include/atomic.h \ + include/bitmapper.h include/blobhash.h include/buffer.h \ + include/byteorder.h include/cephfs/libcephfs.h \ + include/ceph_features.h include/ceph_frag.h include/ceph_fs.h \ + include/ceph_hash.h include/cmp.h include/color.h \ + include/compat.h include/crc32c.h include/encoding.h \ + include/err.h include/error.h include/filepath.h \ + include/frag.h include/hash.h include/intarith.h \ + include/interval_set.h include/int_types.h include/ipaddr.h \ + include/linux_fiemap.h include/lru.h include/msgr.h \ + include/object.h include/page.h include/rangeset.h \ + include/rados.h include/rbd_types.h include/statlite.h \ + include/str_list.h include/str_map.h include/stringify.h \ + include/triple.h include/types.h include/utime.h \ + include/dlist.h include/elist.h include/uuid.h include/xlist.h \ + include/rados/librados.h include/rados/rados_types.h \ + include/rados/rados_types.hpp include/rados/librados.hpp \ + include/rados/librgw.h include/rados/page.h \ + include/rados/crc32c.h include/rados/buffer.h \ + include/rbd/features.h include/rbd/librbd.h \ + include/rbd/librbd.hpp include/util.h include/stat.h \ + include/on_exit.h include/memory.h include/rados/memory.h \ + include/hash_namespace.h include/unordered_set.h \ + include/unordered_map.h librados/snap_set_diff.h \ + librados/AioCompletionImpl.h librados/IoCtxImpl.h \ + librados/PoolAsyncCompletionImpl.h librados/RadosClient.h \ + librbd/AioCompletion.h librbd/AioRequest.h librbd/ImageCtx.h \ + librbd/internal.h librbd/LibrbdWriteback.h \ + librbd/parent_types.h librbd/SnapInfo.h librbd/WatchCtx.h \ + rgw/logrotate.conf rgw/rgw_acl.h rgw/rgw_acl_s3.h \ + rgw/rgw_acl_swift.h rgw/rgw_client_io.h rgw/rgw_fcgi.h \ + rgw/rgw_xml.h rgw/rgw_cache.h rgw/rgw_common.h rgw/rgw_cors.h \ + rgw/rgw_cors_s3.h rgw/rgw_cors_swift.h rgw/rgw_string.h \ + rgw/rgw_formats.h rgw/rgw_http_errors.h rgw/rgw_log.h \ + rgw/rgw_loadgen.h rgw/rgw_multi.h rgw/rgw_policy_s3.h \ + rgw/rgw_gc.h rgw/rgw_metadata.h rgw/rgw_multi_del.h \ + rgw/rgw_op.h rgw/rgw_http_client.h rgw/rgw_swift.h \ + rgw/rgw_swift_auth.h rgw/rgw_quota.h rgw/rgw_rados.h \ + rgw/rgw_replica_log.h rgw/rgw_resolve.h rgw/rgw_rest.h \ + rgw/rgw_rest_swift.h rgw/rgw_rest_s3.h rgw/rgw_auth_s3.h \ + rgw/rgw_rest_admin.h rgw/rgw_rest_usage.h rgw/rgw_rest_user.h \ + rgw/rgw_rest_bucket.h rgw/rgw_rest_client.h \ + rgw/rgw_rest_conn.h rgw/rgw_tools.h rgw/rgw_rest_metadata.h \ + rgw/rgw_rest_log.h rgw/rgw_rest_opstate.h \ + rgw/rgw_rest_replica_log.h rgw/rgw_rest_config.h \ + rgw/rgw_usage.h rgw/rgw_user.h rgw/rgw_bucket.h \ + rgw/rgw_keystone.h rgw/rgw_civetweb.h civetweb/civetweb.h \ + civetweb/include/civetweb.h civetweb/src/md5.h \ + cls/lock/cls_lock_types.h cls/lock/cls_lock_ops.h \ + cls/lock/cls_lock_client.h cls/rbd/cls_rbd.h \ + cls/rbd/cls_rbd_client.h cls/refcount/cls_refcount_ops.h \ + cls/refcount/cls_refcount_client.h \ + cls/version/cls_version_types.h cls/version/cls_version_ops.h \ + cls/version/cls_version_client.h cls/log/cls_log_types.h \ + cls/log/cls_log_ops.h cls/log/cls_log_client.h \ + cls/statelog/cls_statelog_types.h \ + cls/statelog/cls_statelog_ops.h \ + cls/statelog/cls_statelog_client.h \ + cls/replica_log/cls_replica_log_types.h \ + cls/replica_log/cls_replica_log_ops.h \ + cls/replica_log/cls_replica_log_client.h \ + cls/rgw/cls_rgw_client.h cls/rgw/cls_rgw_ops.h \ + cls/rgw/cls_rgw_types.h cls/user/cls_user_client.h \ + cls/user/cls_user_ops.h cls/user/cls_user_types.h \ + key_value_store/key_value_structure.h \ + key_value_store/kv_flat_btree_async.h \ + key_value_store/kvs_arg_types.h \ + test/erasure-code/ErasureCodeExample.h \ + test/erasure-code/ceph_erasure_code_benchmark.h \ + test/bench/backend.h test/bench/bencher.h \ + test/bench/detailed_stat_collector.h test/bench/distribution.h \ + test/bench/dumb_backend.h test/bench/rados_backend.h \ + test/bench/rbd_backend.h test/bench/stat_collector.h \ + test/bench/testfilestore_backend.h \ + test/common/ObjectContents.h test/encoding/types.h \ + test/objectstore/DeterministicOpSequence.h \ + test/objectstore/FileStoreDiff.h \ + test/objectstore/FileStoreTracker.h \ + test/objectstore/TestObjectStoreState.h \ + test/objectstore/workload_generator.h test/kv_store_bench.h \ + test/librados/test.h test/librados/TestCase.h \ + test/ObjectMap/KeyValueDBMemory.h test/omap_bench.h \ + test/osdc/FakeWriteback.h test/osd/Object.h \ + test/osd/RadosModel.h test/osd/TestOpStat.h \ + test/system/cross_process_sem.h \ + test/system/st_rados_create_pool.h \ + test/system/st_rados_delete_objs.h \ + test/system/st_rados_delete_pool.h \ + test/system/st_rados_list_objects.h \ + test/system/st_rados_notify.h test/system/st_rados_watch.h \ + test/system/systest_runnable.h test/system/systest_settings.h \ + test/unit.h tools/rados/rados_sync.h tools/common.h cls_acl.cc \ + cls_crypto.cc fetch_config logrotate.conf sample.ceph.conf \ + bash_completion/ceph bash_completion/rados bash_completion/rbd \ + bash_completion/radosgw-admin mount/canonicalize.c \ + mount/mtab.c objclass/objclass.h +bin_SCRIPTS = brag/client/ceph-brag ceph ceph-run ceph-rest-api \ + ceph-clsinfo ceph-debugpack ceph-rbdnamer ceph-post-file \ + ceph-crush-location ceph-coverage +sbin_SCRIPTS = +su_sbin_SCRIPTS = mount.fuse.ceph mkcephfs +dist_bin_SCRIPTS = +lib_LTLIBRARIES = librados.la librbd.la libcephfs.la $(am__append_65) +noinst_LTLIBRARIES = libarch.la libauth.la libcrush.la libmon_types.la \ + libmon.la libmds.la libos_types.la libos.la libosd_types.la \ + libosd.la liberasure_code.la libosdc.la libclient.la \ + $(am__append_23) libglobal.la libjson_spirit.la liblog.la \ + libperfglue.la libcommon_crc.la libcommon.la libmsg.la \ + $(am__append_35) libcls_lock_client.la \ + libcls_refcount_client.la libcls_rgw_client.la \ + libcls_rbd_client.la $(am__append_50) libradostest.la +noinst_LIBRARIES = $(am__append_17) libcls_version_client.a \ + libcls_log_client.a libcls_statelog_client.a \ + libcls_replica_log_client.a libcls_user_client.a +radoslib_LTLIBRARIES = libcls_hello.la libcls_rbd.la libcls_lock.la \ + libcls_refcount.la libcls_version.la libcls_log.la \ + libcls_statelog.la libcls_replica_log.la libcls_user.la \ + libcls_rgw.la $(am__append_40) + +# like bin_PROGRAMS, but these targets are only built for debug builds +bin_DEBUGPROGRAMS = ceph_test_ioctls $(am__append_38) \ + ceph_erasure_code_benchmark ceph_erasure_code ceph_test_timers \ + ceph_test_signal_handlers ceph_test_rados ceph_test_mutate \ + ceph_test_rewrite_latency ceph_test_msgr ceph_streamtest \ + ceph_test_trans ceph_test_crypto ceph_test_keys \ + $(am__append_48) ceph_smalliobench ceph_smalliobenchfs \ + ceph_smalliobenchdumb ceph_smalliobenchrbd ceph_tpbench \ + ceph_omapbench $(am__append_49) ceph_bench_log \ + $(am__append_52) ceph_multi_stress_watch ceph_test_librbd \ + $(am__append_53) ceph_test_cls_rbd ceph_test_cls_refcount \ + ceph_test_cls_version ceph_test_cls_log ceph_test_cls_statelog \ + ceph_test_cls_replica_log ceph_test_cls_lock \ + ceph_test_cls_hello $(am__append_54) ceph_test_mon_workloadgen \ + ceph_test_rados_api_cmd ceph_test_rados_api_io \ + ceph_test_rados_api_c_write_operations \ + ceph_test_rados_api_c_read_operations ceph_test_rados_api_aio \ + ceph_test_rados_api_list ceph_test_rados_api_pool \ + ceph_test_rados_api_stat ceph_test_rados_api_watch_notify \ + ceph_test_rados_api_snapshots ceph_test_rados_api_cls \ + ceph_test_rados_api_misc ceph_test_rados_api_tier \ + ceph_test_rados_api_lock ceph_test_libcephfs $(am__append_55) \ + ceph_test_objectstore_workloadgen \ + ceph_test_filestore_idempotent \ + ceph_test_filestore_idempotent_sequence ceph_xattr_bench \ + ceph_test_filejournal ceph_test_stress_watch \ + ceph_test_objectcacher_stress ceph_test_snap_mapper \ + ceph_test_object_map ceph_test_keyvaluedb_atomicity \ + ceph_test_keyvaluedb_iterators \ + ceph_test_cfuse_cache_invalidate ceph_test_c_headers \ + ceph_test_get_blkdev_size ceph-osdomap-tool ceph-monstore-tool \ + ceph-kvstore-tool ceph_scratchtool ceph_scratchtoolpp \ + ceph_psim ceph_dupstore ceph_radosacl ceph-client-debug + +# like sbin_SCRIPTS but can be used to install to e.g. /usr/sbin +ceph_sbindir = $(sbindir) + +# certain things go straight into /sbin, though! +su_sbindir = /sbin + +# tests scripts will be appended to this +check_SCRIPTS = test/erasure-code/test-erasure-code.sh \ + unittest_bufferlist.sh test/encoding/check-generated.sh \ + test/mon/osd-pool-create.sh test/mon/misc.sh \ + test/mon/osd-crush.sh test/mon/osd-erasure-code-profile.sh \ + test/mon/mkfs.sh test/ceph-disk.sh \ + test/mon/mon-handle-forward.sh test/vstart_wrapped_tests.sh \ + test/pybind/test_ceph_argparse.py + +################################## +AM_COMMON_CPPFLAGS = \ + -D__CEPH__ \ + -D_FILE_OFFSET_BITS=64 \ + -D_REENTRANT \ + -D_THREAD_SAFE \ + -D__STDC_FORMAT_MACROS \ + -D_GNU_SOURCE \ + -DCEPH_LIBDIR=\"${libdir}\" \ + -DCEPH_PKGLIBDIR=\"${pkglibdir}\" \ + -DGTEST_HAS_TR1_TUPLE=0 + +AM_COMMON_CFLAGS = \ + -Wall \ + ${WARN_TYPE_LIMITS} \ + ${WARN_IGNORED_QUALIFIERS} \ + -Winit-self \ + -Wpointer-arith \ + -Werror=format-security \ + -fno-strict-aliasing \ + -fsigned-char + +AM_CFLAGS = $(AM_COMMON_CFLAGS) $(am__append_5) $(am__append_26) +AM_CPPFLAGS = $(AM_COMMON_CPPFLAGS) + +# note: this is position dependant, it affects the -l options that +# come after it on the command line. when you use ${AM_LDFLAGS} in +# later rules, take care where you place it. for more information, see +# http://blog.flameeyes.eu/2008/11/19/relationship-between-as-needed-and-no-undefined-part-1-what-do-they-do +# http://blog.flameeyes.eu/2008/11/20/misguided-link-and-as-needed +# http://www.gentoo.org/proj/en/qa/asneeded.xml +# http://gcc.gnu.org/ml/gcc-help/2010-12/msg00338.html +# http://sigquit.wordpress.com/2011/02/16/why-asneeded-doesnt-work-as-expected-for-your-libraries-on-your-autotools-project/ +AM_LDFLAGS = $(am__append_2) $(am__append_4) +AM_CCASFLAGS = -f elf64 + +##################### +EXTRALIBS = -luuid -lm $(am__append_7) $(am__append_8) $(am__append_9) \ + $(am__append_13) +LIBGLOBAL = libglobal.la +LIBCOMMON = libcommon.la +LIBARCH = libarch.la +LIBPERFGLUE = libperfglue.la $(am__append_12) +LIBAUTH = libauth.la +LIBMSG = libmsg.la +LIBCRUSH = libcrush.la +LIBJSON_SPIRIT = libjson_spirit.la +LIBLOG = liblog.la + +# Always use system leveldb +LIBOS = libos.la $(am__append_10) $(am__append_11) -lleveldb -lsnappy +LIBOS_TYPES = libos_types.la + +# Libosd always needs osdc and os +LIBOSD = libosd.la $(LIBOSDC) $(LIBOS) $(LIBPERFGLUE) +LIBOSD_TYPES = libosd_types.la +LIBOSDC = libosdc.la + +# These have references to syms like ceph_using_tcmalloc(), glue libperfglue to them +LIBMON = libmon.la $(LIBPERFGLUE) +LIBMON_TYPES = libmon_types.la +LIBMDS = libmds.la $(LIBPERFGLUE) +LIBCLIENT = libclient.la +LIBCLIENT_FUSE = libclient_fuse.la +LIBRADOS = librados.la +LIBRGW = librgw.la +LIBRBD = librbd.la +LIBCEPHFS = libcephfs.la +LIBERASURE_CODE = liberasure_code.la + +# Use this for binaries requiring libglobal +CEPH_GLOBAL = $(LIBGLOBAL) $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) + +# This is set by [lib]/Makefile.am and used for build tests + +# important; libmsg before libauth! +LIBCOMMON_DEPS = libcommon_crc.la $(LIBERASURE_CODE) $(LIBMSG) \ + $(LIBAUTH) $(LIBCRUSH) $(LIBJSON_SPIRIT) $(LIBLOG) $(LIBARCH) \ + $(am__append_32) +LIBRADOS_DEPS = libcls_lock_client.la $(LIBOSDC) $(LIBCOMMON) +LIBRGW_DEPS = $(am__append_36) + +# This is used by the dencoder test +DENCODER_SOURCES = $(am__append_39) +DENCODER_DEPS = libcls_lock_client.la libcls_refcount_client.la \ + libcls_replica_log_client.a libcls_rgw_client.la \ + libcls_user_client.a +radoslibdir = $(libdir)/rados-classes +libarch_la_SOURCES = \ + arch/intel.c \ + arch/neon.c \ + arch/probe.cc + +libauth_la_SOURCES = \ + auth/AuthAuthorizeHandler.cc \ + auth/AuthClientHandler.cc \ + auth/AuthSessionHandler.cc \ + auth/AuthServiceHandler.cc \ + auth/AuthMethodList.cc \ + auth/cephx/CephxAuthorizeHandler.cc \ + auth/cephx/CephxClientHandler.cc \ + auth/cephx/CephxProtocol.cc \ + auth/cephx/CephxServiceHandler.cc \ + auth/cephx/CephxSessionHandler.cc \ + auth/cephx/CephxKeyServer.cc \ + auth/none/AuthNoneAuthorizeHandler.cc \ + auth/unknown/AuthUnknownAuthorizeHandler.cc \ + auth/Crypto.cc \ + auth/KeyRing.cc \ + auth/RotatingKeyRing.cc + +libcrush_la_SOURCES = \ + crush/builder.c \ + crush/mapper.c \ + crush/crush.c \ + crush/hash.c \ + crush/CrushWrapper.cc \ + crush/CrushCompiler.cc \ + crush/CrushTester.cc + +libmon_types_la_SOURCES = \ + mon/PGMap.cc + +libmon_la_SOURCES = \ + mon/Monitor.cc \ + mon/Paxos.cc \ + mon/PaxosService.cc \ + mon/OSDMonitor.cc \ + mon/MDSMonitor.cc \ + mon/MonmapMonitor.cc \ + mon/PGMonitor.cc \ + mon/LogMonitor.cc \ + mon/AuthMonitor.cc \ + mon/Elector.cc \ + mon/MonitorStore.cc \ + mon/HealthMonitor.cc \ + mon/DataHealthService.cc \ + mon/ConfigKeyService.cc + +libmon_la_LIBADD = $(LIBAUTH) $(LIBCOMMON) $(LIBOS) $(LIBMON_TYPES) +libmds_la_SOURCES = \ + mds/Anchor.cc \ + mds/Capability.cc \ + mds/Dumper.cc \ + mds/Resetter.cc \ + mds/MDS.cc \ + mds/flock.cc \ + mds/locks.c \ + mds/journal.cc \ + mds/Server.cc \ + mds/Mutation.cc \ + mds/MDCache.cc \ + mds/Locker.cc \ + mds/Migrator.cc \ + mds/MDBalancer.cc \ + mds/CDentry.cc \ + mds/CDir.cc \ + mds/CInode.cc \ + mds/LogEvent.cc \ + mds/MDSTable.cc \ + mds/InoTable.cc \ + mds/MDSTableClient.cc \ + mds/MDSTableServer.cc \ + mds/AnchorServer.cc \ + mds/AnchorClient.cc \ + mds/SnapRealm.cc \ + mds/SnapServer.cc \ + mds/snap.cc \ + mds/SessionMap.cc \ + mds/MDLog.cc \ + mds/MDSUtility.cc + +libmds_la_LIBADD = $(LIBOSDC) +libos_types_la_SOURCES = \ + os/Transaction.cc + +libos_types_la_CXXFLAGS = ${AM_CXXFLAGS} +libos_la_SOURCES = os/chain_xattr.cc os/DBObjectMap.cc \ + os/GenericObjectMap.cc os/FileJournal.cc os/FileStore.cc \ + os/FlatIndex.cc os/GenericFileStoreBackend.cc os/HashIndex.cc \ + os/IndexManager.cc os/JournalingObjectStore.cc \ + os/LevelDBStore.cc os/LFNIndex.cc os/MemStore.cc \ + os/KeyValueStore.cc os/ObjectStore.cc os/WBThrottle.cc \ + common/TrackedOp.cc $(am__append_14) $(am__append_15) \ + $(am__append_16) +libos_la_CXXFLAGS = ${AM_CXXFLAGS} +libos_la_LIBADD = $(LIBOS_TYPES) +@WITH_LIBZFS_TRUE@libos_zfs_a_SOURCES = os/ZFS.cc +@WITH_LIBZFS_TRUE@libos_zfs_a_CXXFLAGS = ${AM_CXXFLAGS} ${LIBZFS_CFLAGS} +libosd_types_la_SOURCES = \ + osd/PGLog.cc \ + osd/osd_types.cc \ + osd/ECUtil.cc + +libosd_types_la_CXXFLAGS = ${AM_CXXFLAGS} +libosd_la_SOURCES = \ + osd/PG.cc \ + osd/ReplicatedPG.cc \ + osd/ReplicatedBackend.cc \ + osd/ECBackend.cc \ + osd/ECMsgTypes.cc \ + osd/ECTransaction.cc \ + osd/PGBackend.cc \ + osd/Ager.cc \ + osd/HitSet.cc \ + osd/OSD.cc \ + osd/OSDCap.cc \ + osd/Watch.cc \ + osd/ClassHandler.cc \ + osd/OpRequest.cc \ + common/TrackedOp.cc \ + osd/SnapMapper.cc \ + objclass/class_api.cc + +libosd_la_CXXFLAGS = ${AM_CXXFLAGS} +libosd_la_LIBADD = $(LIBOSDC) $(LIBOS) $(LIBOSD_TYPES) $(LIBOS_TYPES) +erasure_codelibdir = $(pkglibdir)/erasure-code +erasure_codelib_LTLIBRARIES = libec_jerasure_generic.la \ + libec_jerasure_sse3.la libec_jerasure_sse4.la \ + libec_jerasure.la libec_example.la \ + libec_missing_entry_point.la libec_hangs.la \ + libec_fail_to_initialize.la libec_fail_to_register.la \ + libec_test_jerasure_sse4.la libec_test_jerasure_sse3.la \ + libec_test_jerasure_generic.la +jerasure_sources = \ + erasure-code/jerasure/jerasure/src/cauchy.c \ + erasure-code/jerasure/jerasure/src/galois.c \ + erasure-code/jerasure/jerasure/src/jerasure.c \ + erasure-code/jerasure/jerasure/src/liberation.c \ + erasure-code/jerasure/jerasure/src/reed_sol.c \ + erasure-code/jerasure/gf-complete/src/gf_wgen.c \ + erasure-code/jerasure/gf-complete/src/gf_method.c \ + erasure-code/jerasure/gf-complete/src/gf_w16.c \ + erasure-code/jerasure/gf-complete/src/gf.c \ + erasure-code/jerasure/gf-complete/src/gf_w32.c \ + erasure-code/jerasure/gf-complete/src/gf_w64.c \ + erasure-code/jerasure/gf-complete/src/gf_w128.c \ + erasure-code/jerasure/gf-complete/src/gf_general.c \ + erasure-code/jerasure/gf-complete/src/gf_w4.c \ + erasure-code/jerasure/gf-complete/src/gf_rand.c \ + erasure-code/jerasure/gf-complete/src/gf_w8.c \ + erasure-code/jerasure/ErasureCodePluginJerasure.cc \ + erasure-code/jerasure/ErasureCodeJerasure.cc + +libec_jerasure_generic_la_SOURCES = ${jerasure_sources} +libec_jerasure_generic_la_CFLAGS = ${AM_CFLAGS} \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +libec_jerasure_generic_la_CXXFLAGS = ${AM_CXXFLAGS} \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +libec_jerasure_generic_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) +libec_jerasure_generic_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 \ + $(am__append_19) +libec_jerasure_sse3_la_SOURCES = ${jerasure_sources} +libec_jerasure_sse3_la_CFLAGS = ${AM_CFLAGS} \ + ${INTEL_SSE_FLAGS} \ + ${INTEL_SSE2_FLAGS} \ + ${INTEL_SSE3_FLAGS} \ + ${INTEL_SSSE3_FLAGS} \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +libec_jerasure_sse3_la_CXXFLAGS = ${AM_CXXFLAGS} \ + ${INTEL_SSE_FLAGS} \ + ${INTEL_SSE2_FLAGS} \ + ${INTEL_SSE3_FLAGS} \ + ${INTEL_SSSE3_FLAGS} \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +libec_jerasure_sse3_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) +libec_jerasure_sse3_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 \ + $(am__append_20) +libec_jerasure_sse4_la_SOURCES = ${jerasure_sources} +libec_jerasure_sse4_la_CFLAGS = ${AM_CFLAGS} \ + ${INTEL_SSE_FLAGS} \ + ${INTEL_SSE2_FLAGS} \ + ${INTEL_SSE3_FLAGS} \ + ${INTEL_SSSE3_FLAGS} \ + ${INTEL_SSE4_1_FLAGS} \ + ${INTEL_SSE4_2_FLAGS} \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +libec_jerasure_sse4_la_CXXFLAGS = ${AM_CXXFLAGS} \ + ${INTEL_SSE_FLAGS} \ + ${INTEL_SSE2_FLAGS} \ + ${INTEL_SSE3_FLAGS} \ + ${INTEL_SSSE3_FLAGS} \ + ${INTEL_SSE4_1_FLAGS} \ + ${INTEL_SSE4_2_FLAGS} \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +libec_jerasure_sse4_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) +libec_jerasure_sse4_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 \ + $(am__append_21) +libec_jerasure_la_SOURCES = \ + erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc + +libec_jerasure_la_CFLAGS = ${AM_CFLAGS} +libec_jerasure_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_jerasure_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) +libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 \ + $(am__append_22) +liberasure_code_la_SOURCES = \ + erasure-code/ErasureCodePlugin.cc + +@LINUX_TRUE@liberasure_code_la_LIBADD = -ldl +libosdc_la_SOURCES = \ + osdc/Objecter.cc \ + osdc/ObjectCacher.cc \ + osdc/Filer.cc \ + osdc/Striper.cc \ + osdc/Journaler.cc + +libclient_la_SOURCES = \ + client/Client.cc \ + client/Inode.cc \ + client/Dentry.cc \ + client/MetaRequest.cc \ + client/ClientSnapRealm.cc \ + client/MetaSession.cc \ + client/Trace.cc + +libclient_la_LIBADD = $(LIBOSDC) $(LIBEDIT_LIBS) +@WITH_FUSE_TRUE@libclient_fuse_la_SOURCES = client/fuse_ll.cc +@WITH_FUSE_TRUE@libclient_fuse_la_LIBADD = libclient.la -lfuse +ceph_test_ioctls_SOURCES = client/test_ioctls.c +libglobal_la_SOURCES = \ + global/global_context.cc \ + global/global_init.cc \ + global/pidfile.cc \ + global/signal_handler.cc + +libglobal_la_LIBADD = $(LIBCOMMON) +libjson_spirit_la_SOURCES = \ + json_spirit/json_spirit_reader.cpp \ + json_spirit/json_spirit_writer.cpp + +liblog_la_SOURCES = \ + log/Log.cc \ + log/SubsystemMap.cc + +libperfglue_la_SOURCES = $(am__append_25) $(am__append_28) \ + $(am__append_29) $(am__append_30) +@WITH_TCMALLOC_TRUE@libperfglue_la_LIBADD = -ltcmalloc + +# these should go out of libcommon +libcommon_la_SOURCES = ceph_ver.c common/DecayCounter.cc \ + common/LogClient.cc common/LogEntry.cc \ + common/PrebufferedStreambuf.cc common/SloppyCRCMap.cc \ + common/BackTrace.cc common/perf_counters.cc common/Mutex.cc \ + common/OutputDataSocket.cc common/admin_socket.cc \ + common/admin_socket_client.cc common/cmdparse.cc \ + common/escape.c common/io_priority.cc common/Clock.cc \ + common/Throttle.cc common/Timer.cc common/Finisher.cc \ + common/environment.cc common/assert.cc common/run_cmd.cc \ + common/WorkQueue.cc common/ConfUtils.cc common/MemoryModel.cc \ + common/armor.c common/fd.cc common/xattr.c common/safe_io.c \ + common/snap_types.cc common/str_list.cc common/str_map.cc \ + common/errno.cc common/RefCountedObj.cc common/blkdev.cc \ + common/common_init.cc common/pipe.c common/ceph_argparse.cc \ + common/ceph_context.cc common/buffer.cc \ + common/code_environment.cc common/dout.cc common/histogram.cc \ + common/signal.cc common/simple_spin.cc common/Thread.cc \ + common/Formatter.cc common/HeartbeatMap.cc common/config.cc \ + common/utf8.c common/mime.c common/strtol.cc common/page.cc \ + common/lockdep.cc common/version.cc common/hex.cc \ + common/entity_name.cc common/ceph_crypto.cc \ + common/ceph_crypto_cms.cc common/ceph_json.cc common/ipaddr.cc \ + common/pick_address.cc common/util.cc common/TextTable.cc \ + common/ceph_fs.cc common/ceph_hash.cc common/ceph_strings.cc \ + common/ceph_frag.cc common/addr_parsing.c common/hobject.cc \ + common/bloom_filter.cc common/linux_version.c mon/MonCap.cc \ + mon/MonClient.cc mon/MonMap.cc osd/OSDMap.cc osd/osd_types.cc \ + osd/ECMsgTypes.cc osd/HitSet.cc mds/MDSMap.cc \ + mds/inode_backtrace.cc mds/mdstypes.cc + +# inject crc in common +libcommon_crc_la_SOURCES = common/sctp_crc32.c common/crc32c.cc \ + common/crc32c_intel_baseline.c common/crc32c_intel_fast.c \ + $(am__append_31) +@WITH_GOOD_YASM_ELF64_TRUE@libcommon_crc_la_LIBTOOLFLAGS = --tag=CC +libcommon_la_LIBADD = $(LIBCOMMON_DEPS) +libmsg_la_SOURCES = \ + msg/Accepter.cc \ + msg/DispatchQueue.cc \ + msg/Message.cc \ + msg/Messenger.cc \ + msg/Pipe.cc \ + msg/SimpleMessenger.cc \ + msg/msg_types.cc + +libcephfs_includedir = $(includedir)/cephfs +libcephfs_include_DATA = $(srcdir)/include/cephfs/libcephfs.h +librbd_includedir = $(includedir)/rbd +librbd_include_DATA = \ + $(srcdir)/include/rbd/features.h \ + $(srcdir)/include/rbd/librbd.h \ + $(srcdir)/include/rbd/librbd.hpp + +rados_includedir = $(includedir)/rados +rados_include_DATA = \ + $(srcdir)/include/rados/librados.h \ + $(srcdir)/include/rados/rados_types.h \ + $(srcdir)/include/rados/rados_types.hpp \ + $(srcdir)/include/rados/librados.hpp \ + $(srcdir)/include/buffer.h \ + $(srcdir)/include/page.h \ + $(srcdir)/include/crc32c.h \ + $(srcdir)/include/memory.h + +librados_la_SOURCES = \ + librados/librados.cc \ + librados/RadosClient.cc \ + librados/IoCtxImpl.cc \ + librados/snap_set_diff.cc + + +# We need this to avoid basename conflicts with the librados build tests in test/Makefile.am +librados_la_CXXFLAGS = ${AM_CXXFLAGS} +librados_la_LIBADD = $(LIBRADOS_DEPS) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +librados_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0 \ + $(am__append_33) +librbd_la_SOURCES = \ + librbd/librbd.cc \ + librbd/AioCompletion.cc \ + librbd/AioRequest.cc \ + librbd/ImageCtx.cc \ + librbd/internal.cc \ + librbd/LibrbdWriteback.cc \ + librbd/WatchCtx.cc + +librbd_la_LIBADD = \ + $(LIBRADOS) $(LIBOSDC) \ + libcls_rbd_client.la libcls_lock_client.la \ + $(PTHREAD_LIBS) $(EXTRALIBS) + +librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 $(am__append_34) +@WITH_RADOSGW_TRUE@librgw_la_SOURCES = \ +@WITH_RADOSGW_TRUE@ rgw/librgw.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_acl.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_acl_s3.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_acl_swift.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_client_io.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_fcgi.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_xml.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_usage.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_json_enc.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_user.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_bucket.cc\ +@WITH_RADOSGW_TRUE@ rgw/rgw_tools.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rados.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_http_client.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_client.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_conn.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_op.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_common.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_cache.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_formats.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_log.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_multi.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_policy_s3.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_gc.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_multi_del.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_env.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_cors.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_cors_s3.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_auth_s3.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_metadata.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_replica_log.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_keystone.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_quota.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_dencoder.cc + +@WITH_RADOSGW_TRUE@librgw_la_CXXFLAGS = -Woverloaded-virtual ${AM_CXXFLAGS} +@WITH_RADOSGW_TRUE@radosgw_SOURCES = \ +@WITH_RADOSGW_TRUE@ rgw/rgw_resolve.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_swift.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_s3.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_usage.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_user.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_bucket.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_metadata.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_replica_log.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_log.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_opstate.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_replica_log.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_rest_config.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_http_client.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_swift.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_swift_auth.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_loadgen.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_civetweb.cc \ +@WITH_RADOSGW_TRUE@ civetweb/src/civetweb.c \ +@WITH_RADOSGW_TRUE@ rgw/rgw_main.cc + +@WITH_RADOSGW_TRUE@radosgw_CFLAGS = -Icivetweb/include +@WITH_RADOSGW_TRUE@radosgw_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(RESOLV_LIBS) $(CEPH_GLOBAL) +@WITH_RADOSGW_TRUE@radosgw_admin_SOURCES = rgw/rgw_admin.cc +@WITH_RADOSGW_TRUE@radosgw_admin_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) +@WITH_RADOSGW_TRUE@ceph_rgw_multiparser_SOURCES = rgw/rgw_multiparser.cc +@WITH_RADOSGW_TRUE@ceph_rgw_multiparser_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) +@WITH_RADOSGW_TRUE@ceph_rgw_jsonparser_SOURCES = \ +@WITH_RADOSGW_TRUE@ rgw/rgw_jsonparser.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_common.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_env.cc \ +@WITH_RADOSGW_TRUE@ rgw/rgw_json_enc.cc + +@WITH_RADOSGW_TRUE@ceph_rgw_jsonparser_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) +libcls_hello_la_SOURCES = cls/hello/cls_hello.cc +libcls_hello_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_hello_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_rbd_la_SOURCES = cls/rbd/cls_rbd.cc +libcls_rbd_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_rbd_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_lock_la_SOURCES = cls/lock/cls_lock.cc +libcls_lock_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_lock_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_refcount_la_SOURCES = \ + cls/refcount/cls_refcount.cc \ + cls/refcount/cls_refcount_ops.cc \ + common/ceph_json.cc + +libcls_refcount_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_refcount_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_version_la_SOURCES = cls/version/cls_version.cc +libcls_version_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_version_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_log_la_SOURCES = cls/log/cls_log.cc +libcls_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_log_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_statelog_la_SOURCES = cls/statelog/cls_statelog.cc +libcls_statelog_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_statelog_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_replica_log_la_SOURCES = cls/replica_log/cls_replica_log.cc +libcls_replica_log_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_replica_log_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_user_la_SOURCES = cls/user/cls_user.cc +libcls_user_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_user_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*' +libcls_rgw_la_SOURCES = \ + cls/rgw/cls_rgw.cc \ + cls/rgw/cls_rgw_ops.cc \ + cls/rgw/cls_rgw_types.cc \ + common/ceph_json.cc + +libcls_rgw_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +libcls_lock_client_la_SOURCES = \ + cls/lock/cls_lock_client.cc \ + cls/lock/cls_lock_types.cc \ + cls/lock/cls_lock_ops.cc + +libcls_refcount_client_la_SOURCES = \ + cls/refcount/cls_refcount_client.cc \ + cls/refcount/cls_refcount_ops.cc + +libcls_version_client_a_SOURCES = \ + cls/version/cls_version_client.cc \ + cls/version/cls_version_types.cc + +libcls_log_client_a_SOURCES = cls/log/cls_log_client.cc +libcls_statelog_client_a_SOURCES = cls/statelog/cls_statelog_client.cc +libcls_replica_log_client_a_SOURCES = \ + cls/replica_log/cls_replica_log_types.cc \ + cls/replica_log/cls_replica_log_ops.cc \ + cls/replica_log/cls_replica_log_client.cc + +libcls_rgw_client_la_SOURCES = \ + cls/rgw/cls_rgw_client.cc \ + cls/rgw/cls_rgw_types.cc \ + cls/rgw/cls_rgw_ops.cc + +libcls_rbd_client_la_SOURCES = cls/rbd/cls_rbd_client.cc +libcls_user_client_a_SOURCES = cls/user/cls_user_client.cc \ + cls/user/cls_user_types.cc \ + cls/user/cls_user_ops.cc + +@LINUX_TRUE@libcls_kvs_la_SOURCES = key_value_store/cls_kvs.cc +@LINUX_TRUE@libcls_kvs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +@LINUX_TRUE@libcls_kvs_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +ceph_erasure_code_benchmark_SOURCES = \ + test/erasure-code/ceph_erasure_code_benchmark.cc + +ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) \ + $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) $(am__append_41) +ceph_erasure_code_SOURCES = \ + test/erasure-code/ceph_erasure_code.cc + +ceph_erasure_code_LDADD = $(LIBOSD) $(LIBCOMMON) \ + $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) $(am__append_42) +libec_example_la_SOURCES = test/erasure-code/ErasureCodePluginExample.cc +libec_example_la_CFLAGS = ${AM_CFLAGS} +libec_example_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) +libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +libec_missing_entry_point_la_SOURCES = test/erasure-code/ErasureCodePluginMissingEntryPoint.cc +libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS} +libec_missing_entry_point_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +libec_hangs_la_SOURCES = test/erasure-code/ErasureCodePluginHangs.cc +libec_hangs_la_CFLAGS = ${AM_CFLAGS} +libec_hangs_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +libec_fail_to_initialize_la_SOURCES = test/erasure-code/ErasureCodePluginFailToInitialize.cc +libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS} +libec_fail_to_initialize_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +libec_fail_to_register_la_SOURCES = test/erasure-code/ErasureCodePluginFailToRegister.cc +libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS} +libec_fail_to_register_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +libec_test_jerasure_sse4_la_SOURCES = test/erasure-code/TestJerasurePluginSSE4.cc +libec_test_jerasure_sse4_la_CFLAGS = ${AM_CFLAGS} +libec_test_jerasure_sse4_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_test_jerasure_sse4_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_test_jerasure_sse4_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +libec_test_jerasure_sse3_la_SOURCES = test/erasure-code/TestJerasurePluginSSE3.cc +libec_test_jerasure_sse3_la_CFLAGS = ${AM_CFLAGS} +libec_test_jerasure_sse3_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_test_jerasure_sse3_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_test_jerasure_sse3_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +libec_test_jerasure_generic_la_SOURCES = test/erasure-code/TestJerasurePluginGeneric.cc +libec_test_jerasure_generic_la_CFLAGS = ${AM_CFLAGS} +libec_test_jerasure_generic_la_CXXFLAGS = ${AM_CXXFLAGS} +libec_test_jerasure_generic_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_test_jerasure_generic_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +unittest_erasure_code_plugin_SOURCES = test/erasure-code/TestErasureCodePlugin.cc +unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) \ + $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_43) +unittest_erasure_code_jerasure_SOURCES = \ + test/erasure-code/TestErasureCodeJerasure.cc \ + ${jerasure_sources} + +unittest_erasure_code_jerasure_CFLAGS = $(AM_CFLAGS) \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS) \ + -Ierasure-code/jerasure/gf-complete/include \ + -Ierasure-code/jerasure/jerasure/include + +unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \ + $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_44) +unittest_erasure_code_plugin_jerasure_SOURCES = \ + test/erasure-code/TestErasureCodePluginJerasure.cc + +unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} +unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) \ + $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(am__append_45) +unittest_erasure_code_example_SOURCES = test/erasure-code/TestErasureCodeExample.cc +unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_timers_SOURCES = test/TestTimers.cc +ceph_test_timers_LDADD = $(CEPH_GLOBAL) +ceph_test_signal_handlers_SOURCES = test/TestSignalHandlers.cc +ceph_test_signal_handlers_LDADD = $(CEPH_GLOBAL) +ceph_test_rados_SOURCES = \ + test/osd/TestRados.cc \ + test/osd/TestOpStat.cc \ + test/osd/Object.cc \ + test/osd/RadosModel.cc + +ceph_test_rados_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +ceph_test_mutate_SOURCES = test/test_mutate.cc +ceph_test_mutate_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +ceph_test_rewrite_latency_SOURCES = test/test_rewrite_latency.cc +ceph_test_rewrite_latency_LDADD = $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) +ceph_test_msgr_SOURCES = test/testmsgr.cc +ceph_test_msgr_LDADD = $(CEPH_GLOBAL) +ceph_streamtest_SOURCES = test/streamtest.cc +ceph_streamtest_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_test_trans_SOURCES = test/test_trans.cc +ceph_test_trans_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_test_crypto_SOURCES = test/testcrypto.cc +ceph_test_crypto_LDADD = $(CEPH_GLOBAL) +ceph_test_keys_SOURCES = test/testkeys.cc +ceph_test_keys_LDADD = $(LIBMON) $(CEPH_GLOBAL) +ceph_dencoder_SOURCES = \ + test/encoding/ceph_dencoder.cc \ + $(DENCODER_SOURCES) + +ceph_dencoder_LDADD = \ + $(LIBOSD_TYPES) \ + $(LIBOS_TYPES) \ + $(LIBMDS) \ + $(LIBMON_TYPES) \ + $(DENCODER_DEPS) \ + $(CEPH_GLOBAL) + + +# These should always use explicit _CFLAGS/_CXXFLAGS so avoid basename conflicts +ceph_dencoder_CFLAGS = ${AM_CFLAGS} $(am__append_46) +ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS} $(am__append_47) +get_command_descriptions_SOURCES = test/common/get_command_descriptions.cc +get_command_descriptions_LDADD = $(LIBMON) $(LIBCOMMON) $(CEPH_GLOBAL) + +# These should all use explicit _CXXFLAGS so avoid basename conflicts +@WITH_BUILD_TESTS_TRUE@test_build_libcommon_SOURCES = \ +@WITH_BUILD_TESTS_TRUE@ test/buildtest_skeleton.cc \ +@WITH_BUILD_TESTS_TRUE@ $(libcommon_la_SOURCES) + +@WITH_BUILD_TESTS_TRUE@test_build_libcommon_LDADD = \ +@WITH_BUILD_TESTS_TRUE@ $(LIBCOMMON_DEPS) \ +@WITH_BUILD_TESTS_TRUE@ $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) + +@WITH_BUILD_TESTS_TRUE@test_build_libcommon_LDFLAGS = -static-libtool-libs +@WITH_BUILD_TESTS_TRUE@test_build_libcommon_CFLAGS = $(AM_CFLAGS) +@WITH_BUILD_TESTS_TRUE@test_build_libcommon_CXXFLAGS = $(AM_CXXFLAGS) +@WITH_BUILD_TESTS_TRUE@test_build_librados_SOURCES = \ +@WITH_BUILD_TESTS_TRUE@ test/buildtest_skeleton.cc \ +@WITH_BUILD_TESTS_TRUE@ $(librados_la_SOURCES) + +@WITH_BUILD_TESTS_TRUE@test_build_librados_LDADD = \ +@WITH_BUILD_TESTS_TRUE@ $(LIBRADOS_DEPS) \ +@WITH_BUILD_TESTS_TRUE@ $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) + +@WITH_BUILD_TESTS_TRUE@test_build_librados_LDFLAGS = -static-libtool-libs +@WITH_BUILD_TESTS_TRUE@test_build_librados_CFLAGS = $(AM_CFLAGS) +@WITH_BUILD_TESTS_TRUE@test_build_librados_CXXFLAGS = $(AM_CXXFLAGS) +@WITH_BUILD_TESTS_TRUE@test_build_librgw_SOURCES = \ +@WITH_BUILD_TESTS_TRUE@ test/buildtest_skeleton.cc \ +@WITH_BUILD_TESTS_TRUE@ $(librgw_la_SOURCES) + +@WITH_BUILD_TESTS_TRUE@test_build_librgw_LDADD = \ +@WITH_BUILD_TESTS_TRUE@ $(LIBRGW_DEPS) \ +@WITH_BUILD_TESTS_TRUE@ $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) \ +@WITH_BUILD_TESTS_TRUE@ $(CEPH_GLOBAL) + +@WITH_BUILD_TESTS_TRUE@test_build_librgw_LDFLAGS = -static-libtool-libs +@WITH_BUILD_TESTS_TRUE@test_build_librgw_CFLAGS = $(AM_CFLAGS) +@WITH_BUILD_TESTS_TRUE@test_build_librgw_CXXFLAGS = $(AM_CXXFLAGS) + +# I dont get this one... testing the osdc build but link in libcephfs? +@WITH_BUILD_TESTS_TRUE@test_build_libcephfs_SOURCES = \ +@WITH_BUILD_TESTS_TRUE@ test/buildtest_skeleton.cc \ +@WITH_BUILD_TESTS_TRUE@ $(libosdc_la_SOURCES) + +@WITH_BUILD_TESTS_TRUE@test_build_libcephfs_LDADD = \ +@WITH_BUILD_TESTS_TRUE@ $(LIBCEPHFS) -lexpat \ +@WITH_BUILD_TESTS_TRUE@ $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) + +@WITH_BUILD_TESTS_TRUE@test_build_libcephfs_LDFLAGS = -static-libtool-libs +@WITH_BUILD_TESTS_TRUE@test_build_libcephfs_CFLAGS = $(AM_CFLAGS) +@WITH_BUILD_TESTS_TRUE@test_build_libcephfs_CXXFLAGS = $(AM_CXXFLAGS) +ceph_smalliobench_SOURCES = \ + test/bench/small_io_bench.cc \ + test/bench/rados_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc + +ceph_smalliobench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) +ceph_smalliobenchfs_SOURCES = \ + test/bench/small_io_bench_fs.cc \ + test/bench/testfilestore_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc + +ceph_smalliobenchfs_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL) +ceph_smalliobenchdumb_SOURCES = \ + test/bench/small_io_bench_dumb.cc \ + test/bench/dumb_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc + +ceph_smalliobenchdumb_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL) +ceph_smalliobenchrbd_SOURCES = \ + test/bench/small_io_bench_rbd.cc \ + test/bench/rbd_backend.cc \ + test/bench/detailed_stat_collector.cc \ + test/bench/bencher.cc + +ceph_smalliobenchrbd_LDADD = $(LIBRBD) $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL) +ceph_tpbench_SOURCES = \ + test/bench/tp_bench.cc \ + test/bench/detailed_stat_collector.cc + +ceph_tpbench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL) +ceph_omapbench_SOURCES = test/omap_bench.cc +ceph_omapbench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +@LINUX_TRUE@ceph_kvstorebench_SOURCES = \ +@LINUX_TRUE@ test/kv_store_bench.cc \ +@LINUX_TRUE@ key_value_store/kv_flat_btree_async.cc + +@LINUX_TRUE@ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +@LINUX_TRUE@libsystest_la_SOURCES = \ +@LINUX_TRUE@ test/system/cross_process_sem.cc \ +@LINUX_TRUE@ test/system/systest_runnable.cc \ +@LINUX_TRUE@ test/system/systest_settings.cc + +@LINUX_TRUE@libsystest_la_LIBADD = $(CEPH_GLOBAL) +@LINUX_TRUE@ceph_test_rados_list_parallel_SOURCES = \ +@LINUX_TRUE@ test/system/rados_list_parallel.cc \ +@LINUX_TRUE@ test/system/st_rados_create_pool.cc \ +@LINUX_TRUE@ test/system/st_rados_list_objects.cc + +@LINUX_TRUE@ceph_test_rados_list_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +@LINUX_TRUE@ceph_test_rados_open_pools_parallel_SOURCES = \ +@LINUX_TRUE@ test/system/rados_open_pools_parallel.cc \ +@LINUX_TRUE@ test/system/st_rados_create_pool.cc + +@LINUX_TRUE@ceph_test_rados_open_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +@LINUX_TRUE@ceph_test_rados_delete_pools_parallel_SOURCES = \ +@LINUX_TRUE@ test/system/rados_delete_pools_parallel.cc \ +@LINUX_TRUE@ test/system/st_rados_create_pool.cc \ +@LINUX_TRUE@ test/system/st_rados_delete_pool.cc \ +@LINUX_TRUE@ test/system/st_rados_list_objects.cc + +@LINUX_TRUE@ceph_test_rados_delete_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +@LINUX_TRUE@ceph_test_rados_watch_notify_SOURCES = \ +@LINUX_TRUE@ test/system/rados_watch_notify.cc \ +@LINUX_TRUE@ test/system/st_rados_create_pool.cc \ +@LINUX_TRUE@ test/system/st_rados_delete_pool.cc \ +@LINUX_TRUE@ test/system/st_rados_delete_objs.cc \ +@LINUX_TRUE@ test/system/st_rados_watch.cc \ +@LINUX_TRUE@ test/system/st_rados_notify.cc + +@LINUX_TRUE@ceph_test_rados_watch_notify_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS) +ceph_bench_log_SOURCES = test/bench_log.cc +ceph_bench_log_LDADD = $(CEPH_GLOBAL) +UNITTEST_CXXFLAGS = \ + $(AM_CXXFLAGS) \ + -I$(top_srcdir)/src/gtest/include \ + -I$(top_builddir)/src/gtest/include + +UNITTEST_LDADD = \ + $(top_builddir)/src/gtest/lib/libgtest.a \ + $(top_builddir)/src/gtest/lib/libgtest_main.a \ + $(PTHREAD_LIBS) + +unittest_encoding_SOURCES = test/encoding.cc +unittest_encoding_LDADD = $(LIBCEPHFS) $(LIBRADOS) -lm $(UNITTEST_LDADD) +unittest_encoding_CXXFLAGS = $(UNITTEST_CXXFLAGS) -fno-strict-aliasing +unittest_addrs_SOURCES = test/test_addrs.cc +unittest_addrs_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_addrs_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_bloom_filter_SOURCES = test/common/test_bloom_filter.cc +unittest_bloom_filter_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_bloom_filter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_histogram_SOURCES = test/common/histogram.cc +unittest_histogram_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_histogram_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_str_map_SOURCES = test/common/test_str_map.cc +unittest_str_map_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_str_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc +unittest_sharedptr_registry_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_sharedptr_registry_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_sloppy_crc_map_SOURCES = test/common/test_sloppy_crc_map.cc +unittest_sloppy_crc_map_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_sloppy_crc_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_util_SOURCES = test/common/test_util.cc +unittest_util_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_util_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CRYPTO_LIBS) $(EXTRALIBS) +unittest_crush_indep_SOURCES = test/crush/indep.cc +unittest_crush_indep_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_crush_indep_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CEPH_CRUSH) $(EXTRALIBS) $(CEPH_GLOBAL) +unittest_osdmap_SOURCES = test/osd/TestOSDMap.cc +unittest_osdmap_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_osdmap_LDADD = $(UNITTEST_LDADD) $(LIBCOMMON) $(CEPH_GLOBAL) +unittest_workqueue_SOURCES = test/test_workqueue.cc +unittest_workqueue_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_workqueue_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_striper_SOURCES = test/test_striper.cc +unittest_striper_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_striper_LDADD = $(LIBOSDC) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_prebufferedstreambuf_SOURCES = test/test_prebufferedstreambuf.cc +unittest_prebufferedstreambuf_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_prebufferedstreambuf_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(EXTRALIBS) +unittest_str_list_SOURCES = test/test_str_list.cc +unittest_str_list_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_str_list_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_log_SOURCES = log/test.cc +unittest_log_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) +unittest_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2 +unittest_throttle_SOURCES = test/common/Throttle.cc +unittest_throttle_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_throttle_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2 +unittest_crush_wrapper_SOURCES = test/crush/TestCrushWrapper.cc +unittest_crush_wrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH) +unittest_crush_wrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2 +unittest_base64_SOURCES = test/base64.cc +unittest_base64_LDADD = $(LIBCEPHFS) -lm $(UNITTEST_LDADD) +unittest_base64_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_ceph_argparse_SOURCES = test/ceph_argparse.cc +unittest_ceph_argparse_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_ceph_argparse_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_ceph_compatset_SOURCES = test/ceph_compatset.cc +unittest_ceph_compatset_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_ceph_compatset_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_osd_types_SOURCES = test/osd/types.cc +unittest_osd_types_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_pglog_SOURCES = test/osd/TestPGLog.cc +unittest_pglog_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_pglog_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) \ + $(am__append_51) +unittest_ecbackend_SOURCES = test/osd/TestECBackend.cc +unittest_ecbackend_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_ecbackend_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_hitset_SOURCES = test/osd/hitset.cc +unittest_hitset_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_hitset_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_gather_SOURCES = test/gather.cc +unittest_gather_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_gather_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_run_cmd_SOURCES = test/run_cmd.cc +unittest_run_cmd_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +unittest_run_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_signals_SOURCES = test/signals.cc +unittest_signals_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_signals_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_simple_spin_SOURCES = test/simple_spin.cc +unittest_simple_spin_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +unittest_simple_spin_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_librados_SOURCES = test/librados/librados.cc +unittest_librados_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +unittest_librados_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_bufferlist_SOURCES = test/bufferlist.cc +unittest_bufferlist_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_bufferlist_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_crc32c_SOURCES = test/common/test_crc32c.cc +unittest_crc32c_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_crc32c_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_arch_SOURCES = test/test_arch.cc +unittest_arch_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_arch_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_crypto_SOURCES = test/crypto.cc +unittest_crypto_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_crypto_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_crypto_init_SOURCES = test/crypto_init.cc +unittest_crypto_init_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_crypto_init_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_perf_counters_SOURCES = test/perf_counters.cc +unittest_perf_counters_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_perf_counters_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_admin_socket_SOURCES = test/admin_socket.cc +unittest_admin_socket_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_admin_socket_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_ceph_crypto_SOURCES = test/ceph_crypto.cc +unittest_ceph_crypto_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_ceph_crypto_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_utf8_SOURCES = test/utf8.cc +unittest_utf8_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_utf8_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_mime_SOURCES = test/mime.cc +unittest_mime_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_mime_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_escape_SOURCES = test/escape.cc +unittest_escape_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_escape_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_chain_xattr_SOURCES = test/objectstore/chain_xattr.cc +unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_flatindex_SOURCES = test/os/TestFlatIndex.cc +unittest_flatindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_flatindex_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_strtol_SOURCES = test/strtol.cc +unittest_strtol_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_strtol_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_confutils_SOURCES = test/confutils.cc +unittest_confutils_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_confutils_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_config_SOURCES = test/common/test_config.cc +unittest_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_config_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_context_SOURCES = test/common/test_context.cc +unittest_context_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_context_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc +unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS) + +# why does this include rgw/rgw_formats.cc...? +unittest_formatter_SOURCES = \ + test/formatter.cc \ + rgw/rgw_formats.cc + +unittest_formatter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_formatter_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_libcephfs_config_SOURCES = test/libcephfs_config.cc +unittest_libcephfs_config_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +unittest_libcephfs_config_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_lfnindex_SOURCES = test/os/TestLFNIndex.cc +unittest_lfnindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_lfnindex_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_librados_config_SOURCES = test/librados/librados_config.cc +unittest_librados_config_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) +unittest_librados_config_CXXFLAGS = $(UNITTEST_CXXFLAGS) + +#unittest_librgw_link_SOURCES = test/librgw_link.cc +#unittest_librgw_link_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS} +#unittest_librgw_link_LDADD = $(LIBRGW) ${UNITTEST_LDADD} +#unittest_librgw_link_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} +#check_PROGRAMS += unittest_librgw_link +unittest_daemon_config_SOURCES = test/daemon_config.cc +unittest_daemon_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_daemon_config_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_osd_osdcap_SOURCES = test/osd/osdcap.cc +unittest_osd_osdcap_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_osd_osdcap_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_mon_moncap_SOURCES = test/mon/moncap.cc +unittest_mon_moncap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_mon_moncap_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_mon_pgmap_SOURCES = test/mon/PGMap.cc +unittest_mon_pgmap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_mon_pgmap_CXXFLAGS = $(UNITTEST_CXXFLAGS) + +#if WITH_RADOSGW +#unittest_librgw_SOURCES = test/librgw.cc +#unittest_librgw_LDFLAGS = -lrt $(PTHREAD_CFLAGS) -lcurl ${AM_LDFLAGS} +#unittest_librgw_LDADD = librgw.la $(LIBRADOS) ${UNITTEST_LDADD} -lexpat $(CEPH_GLOBAL) +#unittest_librgw_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} +#check_PROGRAMS += unittest_librgw +#endif # WITH_RADOSGW +unittest_ipaddr_SOURCES = test/test_ipaddr.cc +unittest_ipaddr_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) +unittest_ipaddr_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_texttable_SOURCES = test/test_texttable.cc +unittest_texttable_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) +unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_on_exit_SOURCES = test/on_exit.cc +unittest_on_exit_LDADD = $(PTHREAD_LIBS) +@WITH_RADOSGW_TRUE@ceph_test_cors_SOURCES = test/test_cors.cc +@WITH_RADOSGW_TRUE@ceph_test_cors_LDADD = \ +@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ +@WITH_RADOSGW_TRUE@ $(UNITTEST_LDADD) \ +@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat + +@WITH_RADOSGW_TRUE@ceph_test_cors_CXXFLAGS = $(UNITTEST_CXXFLAGS) +@WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_SOURCES = test/rgw/test_rgw_manifest.cc +@WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_LDADD = \ +@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) \ +@WITH_RADOSGW_TRUE@ $(UNITTEST_LDADD) $(CRYPTO_LIBS) \ +@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat + +@WITH_RADOSGW_TRUE@ceph_test_rgw_manifest_CXXFLAGS = $(UNITTEST_CXXFLAGS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_meta_SOURCES = test/test_rgw_admin_meta.cc +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_meta_LDADD = \ +@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ +@WITH_RADOSGW_TRUE@ $(UNITTEST_LDADD) $(CRYPTO_LIBS) \ +@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la + +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_meta_CXXFLAGS = $(UNITTEST_CXXFLAGS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_log_SOURCES = test/test_rgw_admin_log.cc +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_log_LDADD = \ +@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ +@WITH_RADOSGW_TRUE@ $(UNITTEST_LDADD) $(CRYPTO_LIBS) \ +@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la + +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_opstate_SOURCES = test/test_rgw_admin_opstate.cc +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_opstate_LDADD = \ +@WITH_RADOSGW_TRUE@ $(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \ +@WITH_RADOSGW_TRUE@ $(UNITTEST_LDADD) $(CRYPTO_LIBS) \ +@WITH_RADOSGW_TRUE@ -lcurl -luuid -lexpat \ +@WITH_RADOSGW_TRUE@ libcls_version_client.a libcls_log_client.a \ +@WITH_RADOSGW_TRUE@ libcls_statelog_client.a libcls_refcount_client.la \ +@WITH_RADOSGW_TRUE@ libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la + +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_opstate_CXXFLAGS = $(UNITTEST_CXXFLAGS) +libradostest_la_SOURCES = \ + test/librados/test.cc \ + test/librados/TestCase.cc + +libradostest_la_CXXFLAGS = $(UNITTEST_CXXFLAGS) +RADOS_TEST_LDADD = libradostest.la +ceph_multi_stress_watch_SOURCES = test/multi_stress_watch.cc +ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) +ceph_test_librbd_SOURCES = test/librbd/test_librbd.cc +ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) +ceph_test_librbd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +@LINUX_TRUE@ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c +@LINUX_TRUE@ceph_test_librbd_fsx_LDADD = $(LIBRBD) $(LIBRADOS) -lm +@LINUX_TRUE@ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format +ceph_test_cls_rbd_SOURCES = test/cls_rbd/test_cls_rbd.cc +ceph_test_cls_rbd_LDADD = $(LIBRADOS) libcls_rbd_client.la libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_cls_rbd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cls_refcount_SOURCES = test/cls_refcount/test_cls_refcount.cc +ceph_test_cls_refcount_LDADD = $(LIBRADOS) libcls_refcount_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_cls_refcount_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cls_version_SOURCES = test/cls_version/test_cls_version.cc +ceph_test_cls_version_LDADD = $(LIBRADOS) libcls_version_client.a $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_cls_version_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cls_log_SOURCES = test/cls_log/test_cls_log.cc +ceph_test_cls_log_LDADD = $(LIBRADOS) libcls_log_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) +ceph_test_cls_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cls_statelog_SOURCES = test/cls_statelog/test_cls_statelog.cc +ceph_test_cls_statelog_LDADD = $(LIBRADOS) libcls_statelog_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) +ceph_test_cls_statelog_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cls_replica_log_SOURCES = test/cls_replica_log/test_cls_replica_log.cc +ceph_test_cls_replica_log_LDADD = \ + $(LIBRADOS) libcls_replica_log_client.a \ + $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) + +ceph_test_cls_replica_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cls_lock_SOURCES = test/cls_lock/test_cls_lock.cc +ceph_test_cls_lock_LDADD = $(LIBRADOS) libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_cls_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cls_hello_SOURCES = test/cls_hello/test_cls_hello.cc +ceph_test_cls_hello_LDADD = \ + $(LIBRADOS) $(CRYPTO_LIBS) \ + $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) + +ceph_test_cls_hello_CXXFLAGS = $(UNITTEST_CXXFLAGS) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_SOURCES = test/cls_rgw/test_cls_rgw.cc +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_LDADD = $(LIBRADOS) libcls_rgw_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +@WITH_RADOSGW_TRUE@ceph_test_cls_rgw_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_mon_workloadgen_SOURCES = test/mon/test_mon_workloadgen.cc +ceph_test_mon_workloadgen_LDADD = $(LIBOS) $(LIBOSDC) $(CEPH_GLOBAL) +ceph_test_rados_api_cmd_SOURCES = test/librados/cmd.cc +ceph_test_rados_api_cmd_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_io_SOURCES = test/librados/io.cc +ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_io_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_c_write_operations_SOURCES = \ + test/librados/c_write_operations.cc + +ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_c_write_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_c_read_operations_SOURCES = \ + test/librados/c_read_operations.cc + +ceph_test_rados_api_c_read_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_c_read_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_aio_SOURCES = test/librados/aio.cc +ceph_test_rados_api_aio_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_aio_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_list_SOURCES = test/librados/list.cc +ceph_test_rados_api_list_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_list_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_pool_SOURCES = test/librados/pool.cc +ceph_test_rados_api_pool_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_pool_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_stat_SOURCES = test/librados/stat.cc +ceph_test_rados_api_stat_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_stat_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_watch_notify_SOURCES = test/librados/watch_notify.cc +ceph_test_rados_api_watch_notify_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_watch_notify_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_snapshots_SOURCES = test/librados/snapshots.cc +ceph_test_rados_api_snapshots_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_snapshots_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_cls_SOURCES = test/librados/cls.cc +ceph_test_rados_api_cls_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_cls_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_misc_SOURCES = test/librados/misc.cc +ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) +ceph_test_rados_api_misc_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_tier_SOURCES = \ + test/librados/tier.cc \ + osd/HitSet.cc + +ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD) +ceph_test_rados_api_tier_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_rados_api_lock_SOURCES = test/librados/lock.cc +ceph_test_rados_api_lock_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_rados_api_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_libcephfs_SOURCES = \ + test/libcephfs/test.cc \ + test/libcephfs/readdir_r_cb.cc \ + test/libcephfs/caps.cc \ + test/libcephfs/multiclient.cc + +ceph_test_libcephfs_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD) +ceph_test_libcephfs_CXXFLAGS = $(UNITTEST_CXXFLAGS) +@LINUX_TRUE@ceph_test_objectstore_SOURCES = test/objectstore/store_test.cc +@LINUX_TRUE@ceph_test_objectstore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +@LINUX_TRUE@ceph_test_objectstore_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_objectstore_workloadgen_SOURCES = \ + test/objectstore/workload_generator.cc \ + test/objectstore/TestObjectStoreState.cc + +ceph_test_objectstore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_test_filestore_idempotent_SOURCES = \ + test/objectstore/test_idempotent.cc \ + test/objectstore/FileStoreTracker.cc \ + test/common/ObjectContents.cc + +ceph_test_filestore_idempotent_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_test_filestore_idempotent_sequence_SOURCES = \ + test/objectstore/test_idempotent_sequence.cc \ + test/objectstore/DeterministicOpSequence.cc \ + test/objectstore/TestObjectStoreState.cc \ + test/objectstore/FileStoreDiff.cc + +ceph_test_filestore_idempotent_sequence_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_xattr_bench_SOURCES = test/xattr_bench.cc +ceph_xattr_bench_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_xattr_bench_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_filejournal_SOURCES = test/test_filejournal.cc +ceph_test_filejournal_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_filejournal_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_stress_watch_SOURCES = test/test_stress_watch.cc +ceph_test_stress_watch_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD) +ceph_test_stress_watch_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_objectcacher_stress_SOURCES = \ + test/osdc/object_cacher_stress.cc \ + test/osdc/FakeWriteback.cc + +ceph_test_objectcacher_stress_LDADD = $(LIBOSDC) $(CEPH_GLOBAL) +ceph_test_snap_mapper_SOURCES = test/test_snap_mapper.cc +ceph_test_snap_mapper_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_snap_mapper_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_object_map_SOURCES = \ + test/ObjectMap/test_object_map.cc \ + test/ObjectMap/KeyValueDBMemory.cc + +ceph_test_object_map_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_object_map_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_keyvaluedb_atomicity_SOURCES = test/ObjectMap/test_keyvaluedb_atomicity.cc +ceph_test_keyvaluedb_atomicity_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_keyvaluedb_atomicity_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_keyvaluedb_iterators_SOURCES = \ + test/ObjectMap/test_keyvaluedb_iterators.cc \ + test/ObjectMap/KeyValueDBMemory.cc + +ceph_test_keyvaluedb_iterators_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +ceph_test_keyvaluedb_iterators_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc +ceph_test_c_headers_SOURCES = test/test_c_headers.c +ceph_test_c_headers_LDADD = $(LIBRADOS) $(LIBCEPHFS) +ceph_test_get_blkdev_size_SOURCES = test/test_get_blkdev_size.cc +ceph_test_get_blkdev_size_LDADD = $(LIBCOMMON) +ceph_osdomap_tool_SOURCES = tools/ceph_osdomap_tool.cc +ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) +ceph_monstore_tool_SOURCES = tools/ceph_monstore_tool.cc +ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) +ceph_kvstore_tool_SOURCES = tools/ceph_kvstore_tool.cc +ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS) +ceph_filestore_tool_SOURCES = tools/ceph_filestore_tool.cc +ceph_filestore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) \ + -lboost_program_options $(am__append_56) +ceph_filestore_dump_SOURCES = tools/ceph_filestore_dump.cc +ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) \ + $(BOOST_PROGRAM_OPTIONS_LIBS) $(am__append_57) +monmaptool_SOURCES = tools/monmaptool.cc +monmaptool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON) +crushtool_SOURCES = tools/crushtool.cc +crushtool_LDADD = $(CEPH_GLOBAL) +osdmaptool_SOURCES = tools/osdmaptool.cc +osdmaptool_LDADD = $(CEPH_GLOBAL) +ceph_scratchtool_SOURCES = tools/scratchtool.c +ceph_scratchtool_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +ceph_scratchtoolpp_SOURCES = tools/scratchtoolpp.cc +ceph_scratchtoolpp_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +ceph_psim_SOURCES = tools/psim.cc +ceph_psim_LDADD = $(CEPH_GLOBAL) +ceph_dupstore_SOURCES = tools/dupstore.cc +ceph_dupstore_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_radosacl_SOURCES = tools/radosacl.cc +ceph_radosacl_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +ceph_client_debug_SOURCES = tools/ceph-client-debug.cc +ceph_client_debug_LDADD = $(LIBCEPHFS) $(CEPH_GLOBAL) $(LIBCOMMON) +rados_SOURCES = tools/rados/rados.cc tools/rados/rados_import.cc \ + tools/rados/rados_export.cc tools/rados/rados_sync.cc \ + common/obj_bencher.cc # needs cleanup so it can go in \ + libcommon.la +rados_LDADD = libcls_lock_client.la $(LIBRADOS) $(CEPH_GLOBAL) +@WITH_REST_BENCH_TRUE@rest_bench_SOURCES = tools/rest_bench.cc \ +@WITH_REST_BENCH_TRUE@ common/obj_bencher.cc # needs cleanup so \ +@WITH_REST_BENCH_TRUE@ it can go in libcommon.la +@WITH_REST_BENCH_TRUE@rest_bench_LDADD = $(CEPH_GLOBAL) \ +@WITH_REST_BENCH_TRUE@ $(am__append_59) $(am__append_60) +@WITH_REST_BENCH_TRUE@@WITH_SYSTEM_LIBS3_FALSE@rest_bench_CXXFLAGS = ${AM_CXXFLAGS} -I$(top_srcdir)/src/libs3/inc +ceph_conf_SOURCES = tools/ceph_conf.cc +ceph_conf_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON) +ceph_authtool_SOURCES = tools/ceph_authtool.cc +ceph_authtool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON) +ceph_mon_store_converter_SOURCES = tools/mon_store_converter.cc +ceph_mon_store_converter_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) + +# subdirs + +# core daemons +ceph_mon_SOURCES = ceph_mon.cc +ceph_mon_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL) $(LIBCOMMON) +ceph_osd_SOURCES = ceph_osd.cc +ceph_osd_LDADD = $(LIBOSD) $(CEPH_GLOBAL) $(LIBCOMMON) +ceph_mds_SOURCES = ceph_mds.cc +ceph_mds_LDADD = $(LIBMDS) $(LIBOSDC) $(CEPH_GLOBAL) $(LIBCOMMON) + +# admin tools + +# user tools +mount_ceph_SOURCES = mount/mount.ceph.c common/secret.c +mount_ceph_LDADD = $(LIBCOMMON) $(KEYUTILS_LIB) +cephfs_SOURCES = cephfs.cc +cephfs_LDADD = $(LIBCOMMON) +librados_config_SOURCES = librados-config.cc +librados_config_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) +ceph_syn_SOURCES = ceph_syn.cc client/SyntheticClient.cc # uses \ + g_conf.. needs cleanup +ceph_syn_LDADD = $(LIBCLIENT) $(CEPH_GLOBAL) +rbd_SOURCES = rbd.cc common/secret.c +rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) -lblkid $(KEYUTILS_LIB) + +# Fuse targets +@WITH_FUSE_TRUE@ceph_fuse_SOURCES = ceph_fuse.cc +@WITH_FUSE_TRUE@ceph_fuse_LDADD = $(LIBCLIENT_FUSE) $(CEPH_GLOBAL) +@WITH_FUSE_TRUE@rbd_fuse_SOURCES = rbd_fuse/rbd-fuse.c +@WITH_FUSE_TRUE@rbd_fuse_LDADD = -lfuse $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) + +# libcephfs (this should go somewhere else in the future) +libcephfs_la_SOURCES = libcephfs.cc +libcephfs_la_LIBADD = $(LIBCLIENT) $(LIBCOMMON) $(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) +libcephfs_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '^ceph_.*' + +# jni library (java source is in src/java) +@ENABLE_CEPHFS_JAVA_TRUE@libcephfs_jni_la_SOURCES = \ +@ENABLE_CEPHFS_JAVA_TRUE@ java/native/libcephfs_jni.cc \ +@ENABLE_CEPHFS_JAVA_TRUE@ java/native/ScopedLocalRef.h \ +@ENABLE_CEPHFS_JAVA_TRUE@ java/native/JniConstants.cpp \ +@ENABLE_CEPHFS_JAVA_TRUE@ java/native/JniConstants.h + +@ENABLE_CEPHFS_JAVA_TRUE@libcephfs_jni_la_LIBADD = $(LIBCEPHFS) $(EXTRALIBS) +@ENABLE_CEPHFS_JAVA_TRUE@libcephfs_jni_la_CPPFLAGS = $(JDK_CPPFLAGS) $(AM_CPPFLAGS) +@ENABLE_CEPHFS_JAVA_TRUE@libcephfs_jni_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 + +# shell scripts +editpaths = sed \ + -e 's|@bindir[@]|$(bindir)|g' \ + -e 's|@sbindir[@]|$(sbindir)|g' \ + -e 's|@libdir[@]|$(libdir)|g' \ + -e 's|@sysconfdir[@]|$(sysconfdir)|g' \ + -e 's|@datadir[@]|$(pkgdatadir)|g' \ + -e 's|@prefix[@]|$(prefix)|g' \ + -e 's|@@GCOV_PREFIX_STRIP[@][@]|$(GCOV_PREFIX_STRIP)|g' + + +# coverage +shell_scripts = ceph-debugpack ceph-post-file ceph-crush-location \ + init-ceph mkcephfs ceph-coverage +doc_DATA = $(srcdir)/sample.ceph.conf sample.fetch_config + +# various scripts +shell_commondir = $(libdir)/ceph +shell_common_SCRIPTS = ceph_common.sh +bash_completiondir = $(sysconfdir)/bash_completion.d +bash_completion_DATA = $(srcdir)/bash_completion/ceph \ + $(srcdir)/bash_completion/rados \ + $(srcdir)/bash_completion/rbd \ + $(srcdir)/bash_completion/radosgw-admin + +ceph_sbin_SCRIPTS = \ + ceph-disk \ + ceph-disk-prepare \ + ceph-disk-activate \ + ceph-disk-udev \ + ceph-create-keys + + +# tests to actually run on "make check"; if you need extra, non-test, +# executables built, you need to replace this with manual assignments +# target by target +TESTS = \ + $(check_PROGRAMS) \ + $(check_SCRIPTS) + + +# pybind +python_PYTHON = pybind/rados.py \ + pybind/rbd.py \ + pybind/cephfs.py \ + pybind/ceph_argparse.py \ + pybind/ceph_rest_api.py + +@ENABLE_COVERAGE_TRUE@COV_DIR = $(DESTDIR)$(libdir)/ceph/coverage +@ENABLE_COVERAGE_TRUE@COV_FILES = $(srcdir)/*.gcno +@ENABLE_COVERAGE_TRUE@COV_LIB_FILES = $(srcdir)/.libs/*.gcno +all: $(BUILT_SOURCES) acconfig.h + $(MAKE) $(AM_MAKEFLAGS) all-recursive + +.SUFFIXES: +.SUFFIXES: .S .c .cc .cpp .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(srcdir)/Makefile-env.am $(srcdir)/arch/Makefile.am $(srcdir)/auth/Makefile.am $(srcdir)/brag/Makefile.am $(srcdir)/crush/Makefile.am $(srcdir)/mon/Makefile.am $(srcdir)/mds/Makefile.am $(srcdir)/os/Makefile.am $(srcdir)/osd/Makefile.am $(srcdir)/erasure-code/Makefile.am $(srcdir)/erasure-code/jerasure/Makefile.am $(srcdir)/osdc/Makefile.am $(srcdir)/client/Makefile.am $(srcdir)/global/Makefile.am $(srcdir)/json_spirit/Makefile.am $(srcdir)/log/Makefile.am $(srcdir)/perfglue/Makefile.am $(srcdir)/common/Makefile.am $(srcdir)/msg/Makefile.am $(srcdir)/messages/Makefile.am $(srcdir)/include/Makefile.am $(srcdir)/librados/Makefile.am $(srcdir)/librbd/Makefile.am $(srcdir)/rgw/Makefile.am $(srcdir)/cls/Makefile.am $(srcdir)/key_value_store/Makefile.am $(srcdir)/test/Makefile.am $(srcdir)/test/erasure-code/Makefile.am $(srcdir)/tools/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu src/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu src/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; +$(srcdir)/Makefile-env.am $(srcdir)/arch/Makefile.am $(srcdir)/auth/Makefile.am $(srcdir)/brag/Makefile.am $(srcdir)/crush/Makefile.am $(srcdir)/mon/Makefile.am $(srcdir)/mds/Makefile.am $(srcdir)/os/Makefile.am $(srcdir)/osd/Makefile.am $(srcdir)/erasure-code/Makefile.am $(srcdir)/erasure-code/jerasure/Makefile.am $(srcdir)/osdc/Makefile.am $(srcdir)/client/Makefile.am $(srcdir)/global/Makefile.am $(srcdir)/json_spirit/Makefile.am $(srcdir)/log/Makefile.am $(srcdir)/perfglue/Makefile.am $(srcdir)/common/Makefile.am $(srcdir)/msg/Makefile.am $(srcdir)/messages/Makefile.am $(srcdir)/include/Makefile.am $(srcdir)/librados/Makefile.am $(srcdir)/librbd/Makefile.am $(srcdir)/rgw/Makefile.am $(srcdir)/cls/Makefile.am $(srcdir)/key_value_store/Makefile.am $(srcdir)/test/Makefile.am $(srcdir)/test/erasure-code/Makefile.am $(srcdir)/tools/Makefile.am: + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +acconfig.h: stamp-h1 + @if test ! -f $@; then rm -f stamp-h1; else :; fi + @if test ! -f $@; then $(MAKE) $(AM_MAKEFLAGS) stamp-h1; else :; fi + +stamp-h1: $(srcdir)/acconfig.h.in $(top_builddir)/config.status + @rm -f stamp-h1 + cd $(top_builddir) && $(SHELL) ./config.status src/acconfig.h +$(srcdir)/acconfig.h.in: $(am__configure_deps) + ($(am__cd) $(top_srcdir) && $(AUTOHEADER)) + rm -f stamp-h1 + touch $@ + +distclean-hdr: + -rm -f acconfig.h stamp-h1 + +clean-noinstLIBRARIES: + -test -z "$(noinst_LIBRARIES)" || rm -f $(noinst_LIBRARIES) +cls/log/$(am__dirstamp): + @$(MKDIR_P) cls/log + @: > cls/log/$(am__dirstamp) +cls/log/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/log/$(DEPDIR) + @: > cls/log/$(DEPDIR)/$(am__dirstamp) +cls/log/cls_log_client.$(OBJEXT): cls/log/$(am__dirstamp) \ + cls/log/$(DEPDIR)/$(am__dirstamp) +libcls_log_client.a: $(libcls_log_client_a_OBJECTS) $(libcls_log_client_a_DEPENDENCIES) $(EXTRA_libcls_log_client_a_DEPENDENCIES) + $(AM_V_at)-rm -f libcls_log_client.a + $(AM_V_AR)$(libcls_log_client_a_AR) libcls_log_client.a $(libcls_log_client_a_OBJECTS) $(libcls_log_client_a_LIBADD) + $(AM_V_at)$(RANLIB) libcls_log_client.a +cls/replica_log/$(am__dirstamp): + @$(MKDIR_P) cls/replica_log + @: > cls/replica_log/$(am__dirstamp) +cls/replica_log/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/replica_log/$(DEPDIR) + @: > cls/replica_log/$(DEPDIR)/$(am__dirstamp) +cls/replica_log/cls_replica_log_types.$(OBJEXT): \ + cls/replica_log/$(am__dirstamp) \ + cls/replica_log/$(DEPDIR)/$(am__dirstamp) +cls/replica_log/cls_replica_log_ops.$(OBJEXT): \ + cls/replica_log/$(am__dirstamp) \ + cls/replica_log/$(DEPDIR)/$(am__dirstamp) +cls/replica_log/cls_replica_log_client.$(OBJEXT): \ + cls/replica_log/$(am__dirstamp) \ + cls/replica_log/$(DEPDIR)/$(am__dirstamp) +libcls_replica_log_client.a: $(libcls_replica_log_client_a_OBJECTS) $(libcls_replica_log_client_a_DEPENDENCIES) $(EXTRA_libcls_replica_log_client_a_DEPENDENCIES) + $(AM_V_at)-rm -f libcls_replica_log_client.a + $(AM_V_AR)$(libcls_replica_log_client_a_AR) libcls_replica_log_client.a $(libcls_replica_log_client_a_OBJECTS) $(libcls_replica_log_client_a_LIBADD) + $(AM_V_at)$(RANLIB) libcls_replica_log_client.a +cls/statelog/$(am__dirstamp): + @$(MKDIR_P) cls/statelog + @: > cls/statelog/$(am__dirstamp) +cls/statelog/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/statelog/$(DEPDIR) + @: > cls/statelog/$(DEPDIR)/$(am__dirstamp) +cls/statelog/cls_statelog_client.$(OBJEXT): \ + cls/statelog/$(am__dirstamp) \ + cls/statelog/$(DEPDIR)/$(am__dirstamp) +libcls_statelog_client.a: $(libcls_statelog_client_a_OBJECTS) $(libcls_statelog_client_a_DEPENDENCIES) $(EXTRA_libcls_statelog_client_a_DEPENDENCIES) + $(AM_V_at)-rm -f libcls_statelog_client.a + $(AM_V_AR)$(libcls_statelog_client_a_AR) libcls_statelog_client.a $(libcls_statelog_client_a_OBJECTS) $(libcls_statelog_client_a_LIBADD) + $(AM_V_at)$(RANLIB) libcls_statelog_client.a +cls/user/$(am__dirstamp): + @$(MKDIR_P) cls/user + @: > cls/user/$(am__dirstamp) +cls/user/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/user/$(DEPDIR) + @: > cls/user/$(DEPDIR)/$(am__dirstamp) +cls/user/cls_user_client.$(OBJEXT): cls/user/$(am__dirstamp) \ + cls/user/$(DEPDIR)/$(am__dirstamp) +cls/user/cls_user_types.$(OBJEXT): cls/user/$(am__dirstamp) \ + cls/user/$(DEPDIR)/$(am__dirstamp) +cls/user/cls_user_ops.$(OBJEXT): cls/user/$(am__dirstamp) \ + cls/user/$(DEPDIR)/$(am__dirstamp) +libcls_user_client.a: $(libcls_user_client_a_OBJECTS) $(libcls_user_client_a_DEPENDENCIES) $(EXTRA_libcls_user_client_a_DEPENDENCIES) + $(AM_V_at)-rm -f libcls_user_client.a + $(AM_V_AR)$(libcls_user_client_a_AR) libcls_user_client.a $(libcls_user_client_a_OBJECTS) $(libcls_user_client_a_LIBADD) + $(AM_V_at)$(RANLIB) libcls_user_client.a +cls/version/$(am__dirstamp): + @$(MKDIR_P) cls/version + @: > cls/version/$(am__dirstamp) +cls/version/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/version/$(DEPDIR) + @: > cls/version/$(DEPDIR)/$(am__dirstamp) +cls/version/cls_version_client.$(OBJEXT): cls/version/$(am__dirstamp) \ + cls/version/$(DEPDIR)/$(am__dirstamp) +cls/version/cls_version_types.$(OBJEXT): cls/version/$(am__dirstamp) \ + cls/version/$(DEPDIR)/$(am__dirstamp) +libcls_version_client.a: $(libcls_version_client_a_OBJECTS) $(libcls_version_client_a_DEPENDENCIES) $(EXTRA_libcls_version_client_a_DEPENDENCIES) + $(AM_V_at)-rm -f libcls_version_client.a + $(AM_V_AR)$(libcls_version_client_a_AR) libcls_version_client.a $(libcls_version_client_a_OBJECTS) $(libcls_version_client_a_LIBADD) + $(AM_V_at)$(RANLIB) libcls_version_client.a +os/$(am__dirstamp): + @$(MKDIR_P) os + @: > os/$(am__dirstamp) +os/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) os/$(DEPDIR) + @: > os/$(DEPDIR)/$(am__dirstamp) +os/libos_zfs_a-ZFS.$(OBJEXT): os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +libos_zfs.a: $(libos_zfs_a_OBJECTS) $(libos_zfs_a_DEPENDENCIES) $(EXTRA_libos_zfs_a_DEPENDENCIES) + $(AM_V_at)-rm -f libos_zfs.a + $(AM_V_AR)$(libos_zfs_a_AR) libos_zfs.a $(libos_zfs_a_OBJECTS) $(libos_zfs_a_LIBADD) + $(AM_V_at)$(RANLIB) libos_zfs.a +install-erasure_codelibLTLIBRARIES: $(erasure_codelib_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(erasure_codelibdir)" || $(MKDIR_P) "$(DESTDIR)$(erasure_codelibdir)" + @list='$(erasure_codelib_LTLIBRARIES)'; test -n "$(erasure_codelibdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(erasure_codelibdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(erasure_codelibdir)"; \ + } + +uninstall-erasure_codelibLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(erasure_codelib_LTLIBRARIES)'; test -n "$(erasure_codelibdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(erasure_codelibdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(erasure_codelibdir)/$$f"; \ + done + +clean-erasure_codelibLTLIBRARIES: + -test -z "$(erasure_codelib_LTLIBRARIES)" || rm -f $(erasure_codelib_LTLIBRARIES) + @list='$(erasure_codelib_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +install-libLTLIBRARIES: $(lib_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(libdir)" || $(MKDIR_P) "$(DESTDIR)$(libdir)" + @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(libdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(libdir)"; \ + } + +uninstall-libLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(libdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(libdir)/$$f"; \ + done + +clean-libLTLIBRARIES: + -test -z "$(lib_LTLIBRARIES)" || rm -f $(lib_LTLIBRARIES) + @list='$(lib_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +install-radoslibLTLIBRARIES: $(radoslib_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(radoslibdir)" || $(MKDIR_P) "$(DESTDIR)$(radoslibdir)" + @list='$(radoslib_LTLIBRARIES)'; test -n "$(radoslibdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(radoslibdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(radoslibdir)"; \ + } + +uninstall-radoslibLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(radoslib_LTLIBRARIES)'; test -n "$(radoslibdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(radoslibdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(radoslibdir)/$$f"; \ + done + +clean-radoslibLTLIBRARIES: + -test -z "$(radoslib_LTLIBRARIES)" || rm -f $(radoslib_LTLIBRARIES) + @list='$(radoslib_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +arch/$(am__dirstamp): + @$(MKDIR_P) arch + @: > arch/$(am__dirstamp) +arch/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) arch/$(DEPDIR) + @: > arch/$(DEPDIR)/$(am__dirstamp) +arch/intel.lo: arch/$(am__dirstamp) arch/$(DEPDIR)/$(am__dirstamp) +arch/neon.lo: arch/$(am__dirstamp) arch/$(DEPDIR)/$(am__dirstamp) +arch/probe.lo: arch/$(am__dirstamp) arch/$(DEPDIR)/$(am__dirstamp) +libarch.la: $(libarch_la_OBJECTS) $(libarch_la_DEPENDENCIES) $(EXTRA_libarch_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libarch_la_OBJECTS) $(libarch_la_LIBADD) $(LIBS) +auth/$(am__dirstamp): + @$(MKDIR_P) auth + @: > auth/$(am__dirstamp) +auth/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) auth/$(DEPDIR) + @: > auth/$(DEPDIR)/$(am__dirstamp) +auth/AuthAuthorizeHandler.lo: auth/$(am__dirstamp) \ + auth/$(DEPDIR)/$(am__dirstamp) +auth/AuthClientHandler.lo: auth/$(am__dirstamp) \ + auth/$(DEPDIR)/$(am__dirstamp) +auth/AuthSessionHandler.lo: auth/$(am__dirstamp) \ + auth/$(DEPDIR)/$(am__dirstamp) +auth/AuthServiceHandler.lo: auth/$(am__dirstamp) \ + auth/$(DEPDIR)/$(am__dirstamp) +auth/AuthMethodList.lo: auth/$(am__dirstamp) \ + auth/$(DEPDIR)/$(am__dirstamp) +auth/cephx/$(am__dirstamp): + @$(MKDIR_P) auth/cephx + @: > auth/cephx/$(am__dirstamp) +auth/cephx/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) auth/cephx/$(DEPDIR) + @: > auth/cephx/$(DEPDIR)/$(am__dirstamp) +auth/cephx/CephxAuthorizeHandler.lo: auth/cephx/$(am__dirstamp) \ + auth/cephx/$(DEPDIR)/$(am__dirstamp) +auth/cephx/CephxClientHandler.lo: auth/cephx/$(am__dirstamp) \ + auth/cephx/$(DEPDIR)/$(am__dirstamp) +auth/cephx/CephxProtocol.lo: auth/cephx/$(am__dirstamp) \ + auth/cephx/$(DEPDIR)/$(am__dirstamp) +auth/cephx/CephxServiceHandler.lo: auth/cephx/$(am__dirstamp) \ + auth/cephx/$(DEPDIR)/$(am__dirstamp) +auth/cephx/CephxSessionHandler.lo: auth/cephx/$(am__dirstamp) \ + auth/cephx/$(DEPDIR)/$(am__dirstamp) +auth/cephx/CephxKeyServer.lo: auth/cephx/$(am__dirstamp) \ + auth/cephx/$(DEPDIR)/$(am__dirstamp) +auth/none/$(am__dirstamp): + @$(MKDIR_P) auth/none + @: > auth/none/$(am__dirstamp) +auth/none/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) auth/none/$(DEPDIR) + @: > auth/none/$(DEPDIR)/$(am__dirstamp) +auth/none/AuthNoneAuthorizeHandler.lo: auth/none/$(am__dirstamp) \ + auth/none/$(DEPDIR)/$(am__dirstamp) +auth/unknown/$(am__dirstamp): + @$(MKDIR_P) auth/unknown + @: > auth/unknown/$(am__dirstamp) +auth/unknown/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) auth/unknown/$(DEPDIR) + @: > auth/unknown/$(DEPDIR)/$(am__dirstamp) +auth/unknown/AuthUnknownAuthorizeHandler.lo: \ + auth/unknown/$(am__dirstamp) \ + auth/unknown/$(DEPDIR)/$(am__dirstamp) +auth/Crypto.lo: auth/$(am__dirstamp) auth/$(DEPDIR)/$(am__dirstamp) +auth/KeyRing.lo: auth/$(am__dirstamp) auth/$(DEPDIR)/$(am__dirstamp) +auth/RotatingKeyRing.lo: auth/$(am__dirstamp) \ + auth/$(DEPDIR)/$(am__dirstamp) +libauth.la: $(libauth_la_OBJECTS) $(libauth_la_DEPENDENCIES) $(EXTRA_libauth_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libauth_la_OBJECTS) $(libauth_la_LIBADD) $(LIBS) +libcephfs.la: $(libcephfs_la_OBJECTS) $(libcephfs_la_DEPENDENCIES) $(EXTRA_libcephfs_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcephfs_la_LINK) -rpath $(libdir) $(libcephfs_la_OBJECTS) $(libcephfs_la_LIBADD) $(LIBS) +java/native/$(am__dirstamp): + @$(MKDIR_P) java/native + @: > java/native/$(am__dirstamp) +java/native/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) java/native/$(DEPDIR) + @: > java/native/$(DEPDIR)/$(am__dirstamp) +java/native/libcephfs_jni_la-libcephfs_jni.lo: \ + java/native/$(am__dirstamp) \ + java/native/$(DEPDIR)/$(am__dirstamp) +java/native/libcephfs_jni_la-JniConstants.lo: \ + java/native/$(am__dirstamp) \ + java/native/$(DEPDIR)/$(am__dirstamp) +libcephfs_jni.la: $(libcephfs_jni_la_OBJECTS) $(libcephfs_jni_la_DEPENDENCIES) $(EXTRA_libcephfs_jni_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcephfs_jni_la_LINK) $(am_libcephfs_jni_la_rpath) $(libcephfs_jni_la_OBJECTS) $(libcephfs_jni_la_LIBADD) $(LIBS) +client/$(am__dirstamp): + @$(MKDIR_P) client + @: > client/$(am__dirstamp) +client/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) client/$(DEPDIR) + @: > client/$(DEPDIR)/$(am__dirstamp) +client/Client.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +client/Inode.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +client/Dentry.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +client/MetaRequest.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +client/ClientSnapRealm.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +client/MetaSession.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +client/Trace.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +libclient.la: $(libclient_la_OBJECTS) $(libclient_la_DEPENDENCIES) $(EXTRA_libclient_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libclient_la_OBJECTS) $(libclient_la_LIBADD) $(LIBS) +client/fuse_ll.lo: client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +libclient_fuse.la: $(libclient_fuse_la_OBJECTS) $(libclient_fuse_la_DEPENDENCIES) $(EXTRA_libclient_fuse_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(am_libclient_fuse_la_rpath) $(libclient_fuse_la_OBJECTS) $(libclient_fuse_la_LIBADD) $(LIBS) +cls/hello/$(am__dirstamp): + @$(MKDIR_P) cls/hello + @: > cls/hello/$(am__dirstamp) +cls/hello/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/hello/$(DEPDIR) + @: > cls/hello/$(DEPDIR)/$(am__dirstamp) +cls/hello/cls_hello.lo: cls/hello/$(am__dirstamp) \ + cls/hello/$(DEPDIR)/$(am__dirstamp) +libcls_hello.la: $(libcls_hello_la_OBJECTS) $(libcls_hello_la_DEPENDENCIES) $(EXTRA_libcls_hello_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_hello_la_LINK) -rpath $(radoslibdir) $(libcls_hello_la_OBJECTS) $(libcls_hello_la_LIBADD) $(LIBS) +key_value_store/$(am__dirstamp): + @$(MKDIR_P) key_value_store + @: > key_value_store/$(am__dirstamp) +key_value_store/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) key_value_store/$(DEPDIR) + @: > key_value_store/$(DEPDIR)/$(am__dirstamp) +key_value_store/cls_kvs.lo: key_value_store/$(am__dirstamp) \ + key_value_store/$(DEPDIR)/$(am__dirstamp) +libcls_kvs.la: $(libcls_kvs_la_OBJECTS) $(libcls_kvs_la_DEPENDENCIES) $(EXTRA_libcls_kvs_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_kvs_la_LINK) $(am_libcls_kvs_la_rpath) $(libcls_kvs_la_OBJECTS) $(libcls_kvs_la_LIBADD) $(LIBS) +cls/lock/$(am__dirstamp): + @$(MKDIR_P) cls/lock + @: > cls/lock/$(am__dirstamp) +cls/lock/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/lock/$(DEPDIR) + @: > cls/lock/$(DEPDIR)/$(am__dirstamp) +cls/lock/cls_lock.lo: cls/lock/$(am__dirstamp) \ + cls/lock/$(DEPDIR)/$(am__dirstamp) +libcls_lock.la: $(libcls_lock_la_OBJECTS) $(libcls_lock_la_DEPENDENCIES) $(EXTRA_libcls_lock_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_lock_la_LINK) -rpath $(radoslibdir) $(libcls_lock_la_OBJECTS) $(libcls_lock_la_LIBADD) $(LIBS) +cls/lock/cls_lock_client.lo: cls/lock/$(am__dirstamp) \ + cls/lock/$(DEPDIR)/$(am__dirstamp) +cls/lock/cls_lock_types.lo: cls/lock/$(am__dirstamp) \ + cls/lock/$(DEPDIR)/$(am__dirstamp) +cls/lock/cls_lock_ops.lo: cls/lock/$(am__dirstamp) \ + cls/lock/$(DEPDIR)/$(am__dirstamp) +libcls_lock_client.la: $(libcls_lock_client_la_OBJECTS) $(libcls_lock_client_la_DEPENDENCIES) $(EXTRA_libcls_lock_client_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libcls_lock_client_la_OBJECTS) $(libcls_lock_client_la_LIBADD) $(LIBS) +cls/log/cls_log.lo: cls/log/$(am__dirstamp) \ + cls/log/$(DEPDIR)/$(am__dirstamp) +libcls_log.la: $(libcls_log_la_OBJECTS) $(libcls_log_la_DEPENDENCIES) $(EXTRA_libcls_log_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_log_la_LINK) -rpath $(radoslibdir) $(libcls_log_la_OBJECTS) $(libcls_log_la_LIBADD) $(LIBS) +cls/rbd/$(am__dirstamp): + @$(MKDIR_P) cls/rbd + @: > cls/rbd/$(am__dirstamp) +cls/rbd/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/rbd/$(DEPDIR) + @: > cls/rbd/$(DEPDIR)/$(am__dirstamp) +cls/rbd/cls_rbd.lo: cls/rbd/$(am__dirstamp) \ + cls/rbd/$(DEPDIR)/$(am__dirstamp) +libcls_rbd.la: $(libcls_rbd_la_OBJECTS) $(libcls_rbd_la_DEPENDENCIES) $(EXTRA_libcls_rbd_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_rbd_la_LINK) -rpath $(radoslibdir) $(libcls_rbd_la_OBJECTS) $(libcls_rbd_la_LIBADD) $(LIBS) +cls/rbd/cls_rbd_client.lo: cls/rbd/$(am__dirstamp) \ + cls/rbd/$(DEPDIR)/$(am__dirstamp) +libcls_rbd_client.la: $(libcls_rbd_client_la_OBJECTS) $(libcls_rbd_client_la_DEPENDENCIES) $(EXTRA_libcls_rbd_client_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libcls_rbd_client_la_OBJECTS) $(libcls_rbd_client_la_LIBADD) $(LIBS) +cls/refcount/$(am__dirstamp): + @$(MKDIR_P) cls/refcount + @: > cls/refcount/$(am__dirstamp) +cls/refcount/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/refcount/$(DEPDIR) + @: > cls/refcount/$(DEPDIR)/$(am__dirstamp) +cls/refcount/cls_refcount.lo: cls/refcount/$(am__dirstamp) \ + cls/refcount/$(DEPDIR)/$(am__dirstamp) +cls/refcount/cls_refcount_ops.lo: cls/refcount/$(am__dirstamp) \ + cls/refcount/$(DEPDIR)/$(am__dirstamp) +common/$(am__dirstamp): + @$(MKDIR_P) common + @: > common/$(am__dirstamp) +common/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) common/$(DEPDIR) + @: > common/$(DEPDIR)/$(am__dirstamp) +common/ceph_json.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +libcls_refcount.la: $(libcls_refcount_la_OBJECTS) $(libcls_refcount_la_DEPENDENCIES) $(EXTRA_libcls_refcount_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_refcount_la_LINK) -rpath $(radoslibdir) $(libcls_refcount_la_OBJECTS) $(libcls_refcount_la_LIBADD) $(LIBS) +cls/refcount/cls_refcount_client.lo: cls/refcount/$(am__dirstamp) \ + cls/refcount/$(DEPDIR)/$(am__dirstamp) +libcls_refcount_client.la: $(libcls_refcount_client_la_OBJECTS) $(libcls_refcount_client_la_DEPENDENCIES) $(EXTRA_libcls_refcount_client_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libcls_refcount_client_la_OBJECTS) $(libcls_refcount_client_la_LIBADD) $(LIBS) +cls/replica_log/cls_replica_log.lo: cls/replica_log/$(am__dirstamp) \ + cls/replica_log/$(DEPDIR)/$(am__dirstamp) +libcls_replica_log.la: $(libcls_replica_log_la_OBJECTS) $(libcls_replica_log_la_DEPENDENCIES) $(EXTRA_libcls_replica_log_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_replica_log_la_LINK) -rpath $(radoslibdir) $(libcls_replica_log_la_OBJECTS) $(libcls_replica_log_la_LIBADD) $(LIBS) +cls/rgw/$(am__dirstamp): + @$(MKDIR_P) cls/rgw + @: > cls/rgw/$(am__dirstamp) +cls/rgw/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) cls/rgw/$(DEPDIR) + @: > cls/rgw/$(DEPDIR)/$(am__dirstamp) +cls/rgw/cls_rgw.lo: cls/rgw/$(am__dirstamp) \ + cls/rgw/$(DEPDIR)/$(am__dirstamp) +cls/rgw/cls_rgw_ops.lo: cls/rgw/$(am__dirstamp) \ + cls/rgw/$(DEPDIR)/$(am__dirstamp) +cls/rgw/cls_rgw_types.lo: cls/rgw/$(am__dirstamp) \ + cls/rgw/$(DEPDIR)/$(am__dirstamp) +libcls_rgw.la: $(libcls_rgw_la_OBJECTS) $(libcls_rgw_la_DEPENDENCIES) $(EXTRA_libcls_rgw_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_rgw_la_LINK) -rpath $(radoslibdir) $(libcls_rgw_la_OBJECTS) $(libcls_rgw_la_LIBADD) $(LIBS) +cls/rgw/cls_rgw_client.lo: cls/rgw/$(am__dirstamp) \ + cls/rgw/$(DEPDIR)/$(am__dirstamp) +libcls_rgw_client.la: $(libcls_rgw_client_la_OBJECTS) $(libcls_rgw_client_la_DEPENDENCIES) $(EXTRA_libcls_rgw_client_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libcls_rgw_client_la_OBJECTS) $(libcls_rgw_client_la_LIBADD) $(LIBS) +cls/statelog/cls_statelog.lo: cls/statelog/$(am__dirstamp) \ + cls/statelog/$(DEPDIR)/$(am__dirstamp) +libcls_statelog.la: $(libcls_statelog_la_OBJECTS) $(libcls_statelog_la_DEPENDENCIES) $(EXTRA_libcls_statelog_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_statelog_la_LINK) -rpath $(radoslibdir) $(libcls_statelog_la_OBJECTS) $(libcls_statelog_la_LIBADD) $(LIBS) +cls/user/cls_user.lo: cls/user/$(am__dirstamp) \ + cls/user/$(DEPDIR)/$(am__dirstamp) +libcls_user.la: $(libcls_user_la_OBJECTS) $(libcls_user_la_DEPENDENCIES) $(EXTRA_libcls_user_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_user_la_LINK) -rpath $(radoslibdir) $(libcls_user_la_OBJECTS) $(libcls_user_la_LIBADD) $(LIBS) +cls/version/cls_version.lo: cls/version/$(am__dirstamp) \ + cls/version/$(DEPDIR)/$(am__dirstamp) +libcls_version.la: $(libcls_version_la_OBJECTS) $(libcls_version_la_DEPENDENCIES) $(EXTRA_libcls_version_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcls_version_la_LINK) -rpath $(radoslibdir) $(libcls_version_la_OBJECTS) $(libcls_version_la_LIBADD) $(LIBS) +common/DecayCounter.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/LogClient.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/LogEntry.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/PrebufferedStreambuf.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/SloppyCRCMap.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/BackTrace.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/perf_counters.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/Mutex.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/OutputDataSocket.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/admin_socket.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/admin_socket_client.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/cmdparse.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/escape.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/io_priority.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/Clock.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/Throttle.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/Timer.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/Finisher.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/environment.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/assert.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/run_cmd.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/WorkQueue.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ConfUtils.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/MemoryModel.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/armor.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/fd.lo: common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/xattr.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/safe_io.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/snap_types.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/str_list.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/str_map.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/errno.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/RefCountedObj.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/blkdev.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/common_init.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/pipe.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_argparse.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_context.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/buffer.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/code_environment.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/dout.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/histogram.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/signal.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/simple_spin.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/Thread.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/Formatter.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/HeartbeatMap.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/config.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/utf8.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/mime.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/strtol.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/page.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/lockdep.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/version.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/hex.lo: common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/entity_name.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_crypto.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_crypto_cms.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ipaddr.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/pick_address.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/util.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/TextTable.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_fs.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_hash.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_strings.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/ceph_frag.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/addr_parsing.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/hobject.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/bloom_filter.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/linux_version.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +mon/$(am__dirstamp): + @$(MKDIR_P) mon + @: > mon/$(am__dirstamp) +mon/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) mon/$(DEPDIR) + @: > mon/$(DEPDIR)/$(am__dirstamp) +mon/MonCap.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/MonClient.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/MonMap.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +osd/$(am__dirstamp): + @$(MKDIR_P) osd + @: > osd/$(am__dirstamp) +osd/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) osd/$(DEPDIR) + @: > osd/$(DEPDIR)/$(am__dirstamp) +osd/OSDMap.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp) +osd/osd_types.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp) +osd/ECMsgTypes.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp) +osd/HitSet.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp) +mds/$(am__dirstamp): + @$(MKDIR_P) mds + @: > mds/$(am__dirstamp) +mds/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) mds/$(DEPDIR) + @: > mds/$(DEPDIR)/$(am__dirstamp) +mds/MDSMap.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/inode_backtrace.lo: mds/$(am__dirstamp) \ + mds/$(DEPDIR)/$(am__dirstamp) +mds/mdstypes.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +libcommon.la: $(libcommon_la_OBJECTS) $(libcommon_la_DEPENDENCIES) $(EXTRA_libcommon_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libcommon_la_OBJECTS) $(libcommon_la_LIBADD) $(LIBS) +common/libcommon_crc_la-sctp_crc32.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/libcommon_crc_la-crc32c.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/libcommon_crc_la-crc32c_intel_baseline.lo: \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/libcommon_crc_la-crc32c_intel_fast.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/libcommon_crc_la-crc32c_intel_fast_asm.lo: \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo: \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +libcommon_crc.la: $(libcommon_crc_la_OBJECTS) $(libcommon_crc_la_DEPENDENCIES) $(EXTRA_libcommon_crc_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libcommon_crc_la_LINK) $(libcommon_crc_la_OBJECTS) $(libcommon_crc_la_LIBADD) $(LIBS) +crush/$(am__dirstamp): + @$(MKDIR_P) crush + @: > crush/$(am__dirstamp) +crush/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) crush/$(DEPDIR) + @: > crush/$(DEPDIR)/$(am__dirstamp) +crush/builder.lo: crush/$(am__dirstamp) \ + crush/$(DEPDIR)/$(am__dirstamp) +crush/mapper.lo: crush/$(am__dirstamp) crush/$(DEPDIR)/$(am__dirstamp) +crush/crush.lo: crush/$(am__dirstamp) crush/$(DEPDIR)/$(am__dirstamp) +crush/hash.lo: crush/$(am__dirstamp) crush/$(DEPDIR)/$(am__dirstamp) +crush/CrushWrapper.lo: crush/$(am__dirstamp) \ + crush/$(DEPDIR)/$(am__dirstamp) +crush/CrushCompiler.lo: crush/$(am__dirstamp) \ + crush/$(DEPDIR)/$(am__dirstamp) +crush/CrushTester.lo: crush/$(am__dirstamp) \ + crush/$(DEPDIR)/$(am__dirstamp) +libcrush.la: $(libcrush_la_OBJECTS) $(libcrush_la_DEPENDENCIES) $(EXTRA_libcrush_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libcrush_la_OBJECTS) $(libcrush_la_LIBADD) $(LIBS) +test/erasure-code/$(am__dirstamp): + @$(MKDIR_P) test/erasure-code + @: > test/erasure-code/$(am__dirstamp) +test/erasure-code/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/erasure-code/$(DEPDIR) + @: > test/erasure-code/$(DEPDIR)/$(am__dirstamp) +test/erasure-code/libec_example_la-ErasureCodePluginExample.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_example.la: $(libec_example_la_OBJECTS) $(libec_example_la_DEPENDENCIES) $(EXTRA_libec_example_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_example_la_LINK) -rpath $(erasure_codelibdir) $(libec_example_la_OBJECTS) $(libec_example_la_LIBADD) $(LIBS) +test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_fail_to_initialize.la: $(libec_fail_to_initialize_la_OBJECTS) $(libec_fail_to_initialize_la_DEPENDENCIES) $(EXTRA_libec_fail_to_initialize_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_fail_to_initialize_la_LINK) -rpath $(erasure_codelibdir) $(libec_fail_to_initialize_la_OBJECTS) $(libec_fail_to_initialize_la_LIBADD) $(LIBS) +test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_fail_to_register.la: $(libec_fail_to_register_la_OBJECTS) $(libec_fail_to_register_la_DEPENDENCIES) $(EXTRA_libec_fail_to_register_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_fail_to_register_la_LINK) -rpath $(erasure_codelibdir) $(libec_fail_to_register_la_OBJECTS) $(libec_fail_to_register_la_LIBADD) $(LIBS) +test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_hangs.la: $(libec_hangs_la_OBJECTS) $(libec_hangs_la_DEPENDENCIES) $(EXTRA_libec_hangs_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_hangs_la_LINK) -rpath $(erasure_codelibdir) $(libec_hangs_la_OBJECTS) $(libec_hangs_la_LIBADD) $(LIBS) +erasure-code/jerasure/$(am__dirstamp): + @$(MKDIR_P) erasure-code/jerasure + @: > erasure-code/jerasure/$(am__dirstamp) +erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) erasure-code/jerasure/$(DEPDIR) + @: > erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo: \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +libec_jerasure.la: $(libec_jerasure_la_OBJECTS) $(libec_jerasure_la_DEPENDENCIES) $(EXTRA_libec_jerasure_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_jerasure_la_LINK) -rpath $(erasure_codelibdir) $(libec_jerasure_la_OBJECTS) $(libec_jerasure_la_LIBADD) $(LIBS) +erasure-code/jerasure/jerasure/src/$(am__dirstamp): + @$(MKDIR_P) erasure-code/jerasure/jerasure/src + @: > erasure-code/jerasure/jerasure/src/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) erasure-code/jerasure/jerasure/src/$(DEPDIR) + @: > erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/$(am__dirstamp): + @$(MKDIR_P) erasure-code/jerasure/gf-complete/src + @: > erasure-code/jerasure/gf-complete/src/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) erasure-code/jerasure/gf-complete/src/$(DEPDIR) + @: > erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo: \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo: \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +libec_jerasure_generic.la: $(libec_jerasure_generic_la_OBJECTS) $(libec_jerasure_generic_la_DEPENDENCIES) $(EXTRA_libec_jerasure_generic_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_jerasure_generic_la_LINK) -rpath $(erasure_codelibdir) $(libec_jerasure_generic_la_OBJECTS) $(libec_jerasure_generic_la_LIBADD) $(LIBS) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo: \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo: \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +libec_jerasure_sse3.la: $(libec_jerasure_sse3_la_OBJECTS) $(libec_jerasure_sse3_la_DEPENDENCIES) $(EXTRA_libec_jerasure_sse3_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_jerasure_sse3_la_LINK) -rpath $(erasure_codelibdir) $(libec_jerasure_sse3_la_OBJECTS) $(libec_jerasure_sse3_la_LIBADD) $(LIBS) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo: \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo: \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo: \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo: \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +libec_jerasure_sse4.la: $(libec_jerasure_sse4_la_OBJECTS) $(libec_jerasure_sse4_la_DEPENDENCIES) $(EXTRA_libec_jerasure_sse4_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_jerasure_sse4_la_LINK) -rpath $(erasure_codelibdir) $(libec_jerasure_sse4_la_OBJECTS) $(libec_jerasure_sse4_la_LIBADD) $(LIBS) +test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_missing_entry_point.la: $(libec_missing_entry_point_la_OBJECTS) $(libec_missing_entry_point_la_DEPENDENCIES) $(EXTRA_libec_missing_entry_point_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_missing_entry_point_la_LINK) -rpath $(erasure_codelibdir) $(libec_missing_entry_point_la_OBJECTS) $(libec_missing_entry_point_la_LIBADD) $(LIBS) +test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_test_jerasure_generic.la: $(libec_test_jerasure_generic_la_OBJECTS) $(libec_test_jerasure_generic_la_DEPENDENCIES) $(EXTRA_libec_test_jerasure_generic_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_test_jerasure_generic_la_LINK) -rpath $(erasure_codelibdir) $(libec_test_jerasure_generic_la_OBJECTS) $(libec_test_jerasure_generic_la_LIBADD) $(LIBS) +test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_test_jerasure_sse3.la: $(libec_test_jerasure_sse3_la_OBJECTS) $(libec_test_jerasure_sse3_la_DEPENDENCIES) $(EXTRA_libec_test_jerasure_sse3_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_test_jerasure_sse3_la_LINK) -rpath $(erasure_codelibdir) $(libec_test_jerasure_sse3_la_OBJECTS) $(libec_test_jerasure_sse3_la_LIBADD) $(LIBS) +test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo: \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +libec_test_jerasure_sse4.la: $(libec_test_jerasure_sse4_la_OBJECTS) $(libec_test_jerasure_sse4_la_DEPENDENCIES) $(EXTRA_libec_test_jerasure_sse4_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libec_test_jerasure_sse4_la_LINK) -rpath $(erasure_codelibdir) $(libec_test_jerasure_sse4_la_OBJECTS) $(libec_test_jerasure_sse4_la_LIBADD) $(LIBS) +erasure-code/$(am__dirstamp): + @$(MKDIR_P) erasure-code + @: > erasure-code/$(am__dirstamp) +erasure-code/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) erasure-code/$(DEPDIR) + @: > erasure-code/$(DEPDIR)/$(am__dirstamp) +erasure-code/ErasureCodePlugin.lo: erasure-code/$(am__dirstamp) \ + erasure-code/$(DEPDIR)/$(am__dirstamp) +liberasure_code.la: $(liberasure_code_la_OBJECTS) $(liberasure_code_la_DEPENDENCIES) $(EXTRA_liberasure_code_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(liberasure_code_la_OBJECTS) $(liberasure_code_la_LIBADD) $(LIBS) +global/$(am__dirstamp): + @$(MKDIR_P) global + @: > global/$(am__dirstamp) +global/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) global/$(DEPDIR) + @: > global/$(DEPDIR)/$(am__dirstamp) +global/global_context.lo: global/$(am__dirstamp) \ + global/$(DEPDIR)/$(am__dirstamp) +global/global_init.lo: global/$(am__dirstamp) \ + global/$(DEPDIR)/$(am__dirstamp) +global/pidfile.lo: global/$(am__dirstamp) \ + global/$(DEPDIR)/$(am__dirstamp) +global/signal_handler.lo: global/$(am__dirstamp) \ + global/$(DEPDIR)/$(am__dirstamp) +libglobal.la: $(libglobal_la_OBJECTS) $(libglobal_la_DEPENDENCIES) $(EXTRA_libglobal_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libglobal_la_OBJECTS) $(libglobal_la_LIBADD) $(LIBS) +json_spirit/$(am__dirstamp): + @$(MKDIR_P) json_spirit + @: > json_spirit/$(am__dirstamp) +json_spirit/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) json_spirit/$(DEPDIR) + @: > json_spirit/$(DEPDIR)/$(am__dirstamp) +json_spirit/json_spirit_reader.lo: json_spirit/$(am__dirstamp) \ + json_spirit/$(DEPDIR)/$(am__dirstamp) +json_spirit/json_spirit_writer.lo: json_spirit/$(am__dirstamp) \ + json_spirit/$(DEPDIR)/$(am__dirstamp) +libjson_spirit.la: $(libjson_spirit_la_OBJECTS) $(libjson_spirit_la_DEPENDENCIES) $(EXTRA_libjson_spirit_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libjson_spirit_la_OBJECTS) $(libjson_spirit_la_LIBADD) $(LIBS) +log/$(am__dirstamp): + @$(MKDIR_P) log + @: > log/$(am__dirstamp) +log/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) log/$(DEPDIR) + @: > log/$(DEPDIR)/$(am__dirstamp) +log/Log.lo: log/$(am__dirstamp) log/$(DEPDIR)/$(am__dirstamp) +log/SubsystemMap.lo: log/$(am__dirstamp) log/$(DEPDIR)/$(am__dirstamp) +liblog.la: $(liblog_la_OBJECTS) $(liblog_la_DEPENDENCIES) $(EXTRA_liblog_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(liblog_la_OBJECTS) $(liblog_la_LIBADD) $(LIBS) +mds/Anchor.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/Capability.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/Dumper.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/Resetter.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/MDS.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/flock.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/locks.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/journal.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/Server.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/Mutation.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/MDCache.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/Locker.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/Migrator.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/MDBalancer.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/CDentry.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/CDir.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/CInode.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/LogEvent.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/MDSTable.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/InoTable.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/MDSTableClient.lo: mds/$(am__dirstamp) \ + mds/$(DEPDIR)/$(am__dirstamp) +mds/MDSTableServer.lo: mds/$(am__dirstamp) \ + mds/$(DEPDIR)/$(am__dirstamp) +mds/AnchorServer.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/AnchorClient.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/SnapRealm.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/SnapServer.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/snap.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/SessionMap.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/MDLog.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/MDSUtility.lo: mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +libmds.la: $(libmds_la_OBJECTS) $(libmds_la_DEPENDENCIES) $(EXTRA_libmds_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libmds_la_OBJECTS) $(libmds_la_LIBADD) $(LIBS) +mon/Monitor.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/Paxos.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/PaxosService.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/OSDMonitor.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/MDSMonitor.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/MonmapMonitor.lo: mon/$(am__dirstamp) \ + mon/$(DEPDIR)/$(am__dirstamp) +mon/PGMonitor.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/LogMonitor.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/AuthMonitor.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/Elector.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/MonitorStore.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +mon/HealthMonitor.lo: mon/$(am__dirstamp) \ + mon/$(DEPDIR)/$(am__dirstamp) +mon/DataHealthService.lo: mon/$(am__dirstamp) \ + mon/$(DEPDIR)/$(am__dirstamp) +mon/ConfigKeyService.lo: mon/$(am__dirstamp) \ + mon/$(DEPDIR)/$(am__dirstamp) +libmon.la: $(libmon_la_OBJECTS) $(libmon_la_DEPENDENCIES) $(EXTRA_libmon_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libmon_la_OBJECTS) $(libmon_la_LIBADD) $(LIBS) +mon/PGMap.lo: mon/$(am__dirstamp) mon/$(DEPDIR)/$(am__dirstamp) +libmon_types.la: $(libmon_types_la_OBJECTS) $(libmon_types_la_DEPENDENCIES) $(EXTRA_libmon_types_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libmon_types_la_OBJECTS) $(libmon_types_la_LIBADD) $(LIBS) +msg/$(am__dirstamp): + @$(MKDIR_P) msg + @: > msg/$(am__dirstamp) +msg/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) msg/$(DEPDIR) + @: > msg/$(DEPDIR)/$(am__dirstamp) +msg/Accepter.lo: msg/$(am__dirstamp) msg/$(DEPDIR)/$(am__dirstamp) +msg/DispatchQueue.lo: msg/$(am__dirstamp) \ + msg/$(DEPDIR)/$(am__dirstamp) +msg/Message.lo: msg/$(am__dirstamp) msg/$(DEPDIR)/$(am__dirstamp) +msg/Messenger.lo: msg/$(am__dirstamp) msg/$(DEPDIR)/$(am__dirstamp) +msg/Pipe.lo: msg/$(am__dirstamp) msg/$(DEPDIR)/$(am__dirstamp) +msg/SimpleMessenger.lo: msg/$(am__dirstamp) \ + msg/$(DEPDIR)/$(am__dirstamp) +msg/msg_types.lo: msg/$(am__dirstamp) msg/$(DEPDIR)/$(am__dirstamp) +libmsg.la: $(libmsg_la_OBJECTS) $(libmsg_la_DEPENDENCIES) $(EXTRA_libmsg_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libmsg_la_OBJECTS) $(libmsg_la_LIBADD) $(LIBS) +os/libos_la-chain_xattr.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-DBObjectMap.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-GenericObjectMap.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-FileJournal.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-FileStore.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-FlatIndex.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-GenericFileStoreBackend.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-HashIndex.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-IndexManager.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-JournalingObjectStore.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-LevelDBStore.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-LFNIndex.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-MemStore.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-KeyValueStore.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-ObjectStore.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-WBThrottle.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +common/libos_la-TrackedOp.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +os/libos_la-BtrfsFileStoreBackend.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-XfsFileStoreBackend.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +os/libos_la-ZFSFileStoreBackend.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +libos.la: $(libos_la_OBJECTS) $(libos_la_DEPENDENCIES) $(EXTRA_libos_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libos_la_LINK) $(libos_la_OBJECTS) $(libos_la_LIBADD) $(LIBS) +os/libos_types_la-Transaction.lo: os/$(am__dirstamp) \ + os/$(DEPDIR)/$(am__dirstamp) +libos_types.la: $(libos_types_la_OBJECTS) $(libos_types_la_DEPENDENCIES) $(EXTRA_libos_types_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libos_types_la_LINK) $(libos_types_la_OBJECTS) $(libos_types_la_LIBADD) $(LIBS) +osd/libosd_la-PG.lo: osd/$(am__dirstamp) osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-ReplicatedPG.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-ReplicatedBackend.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-ECBackend.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-ECMsgTypes.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-ECTransaction.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-PGBackend.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-Ager.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-HitSet.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-OSD.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-OSDCap.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-Watch.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-ClassHandler.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-OpRequest.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +common/libosd_la-TrackedOp.lo: common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +osd/libosd_la-SnapMapper.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +objclass/$(am__dirstamp): + @$(MKDIR_P) objclass + @: > objclass/$(am__dirstamp) +objclass/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) objclass/$(DEPDIR) + @: > objclass/$(DEPDIR)/$(am__dirstamp) +objclass/libosd_la-class_api.lo: objclass/$(am__dirstamp) \ + objclass/$(DEPDIR)/$(am__dirstamp) +libosd.la: $(libosd_la_OBJECTS) $(libosd_la_DEPENDENCIES) $(EXTRA_libosd_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libosd_la_LINK) $(libosd_la_OBJECTS) $(libosd_la_LIBADD) $(LIBS) +osd/libosd_types_la-PGLog.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_types_la-osd_types.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/libosd_types_la-ECUtil.lo: osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +libosd_types.la: $(libosd_types_la_OBJECTS) $(libosd_types_la_DEPENDENCIES) $(EXTRA_libosd_types_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libosd_types_la_LINK) $(libosd_types_la_OBJECTS) $(libosd_types_la_LIBADD) $(LIBS) +osdc/$(am__dirstamp): + @$(MKDIR_P) osdc + @: > osdc/$(am__dirstamp) +osdc/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) osdc/$(DEPDIR) + @: > osdc/$(DEPDIR)/$(am__dirstamp) +osdc/Objecter.lo: osdc/$(am__dirstamp) osdc/$(DEPDIR)/$(am__dirstamp) +osdc/ObjectCacher.lo: osdc/$(am__dirstamp) \ + osdc/$(DEPDIR)/$(am__dirstamp) +osdc/Filer.lo: osdc/$(am__dirstamp) osdc/$(DEPDIR)/$(am__dirstamp) +osdc/Striper.lo: osdc/$(am__dirstamp) osdc/$(DEPDIR)/$(am__dirstamp) +osdc/Journaler.lo: osdc/$(am__dirstamp) osdc/$(DEPDIR)/$(am__dirstamp) +libosdc.la: $(libosdc_la_OBJECTS) $(libosdc_la_DEPENDENCIES) $(EXTRA_libosdc_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libosdc_la_OBJECTS) $(libosdc_la_LIBADD) $(LIBS) +perfglue/$(am__dirstamp): + @$(MKDIR_P) perfglue + @: > perfglue/$(am__dirstamp) +perfglue/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) perfglue/$(DEPDIR) + @: > perfglue/$(DEPDIR)/$(am__dirstamp) +perfglue/heap_profiler.lo: perfglue/$(am__dirstamp) \ + perfglue/$(DEPDIR)/$(am__dirstamp) +perfglue/disabled_heap_profiler.lo: perfglue/$(am__dirstamp) \ + perfglue/$(DEPDIR)/$(am__dirstamp) +perfglue/cpu_profiler.lo: perfglue/$(am__dirstamp) \ + perfglue/$(DEPDIR)/$(am__dirstamp) +perfglue/disabled_stubs.lo: perfglue/$(am__dirstamp) \ + perfglue/$(DEPDIR)/$(am__dirstamp) +libperfglue.la: $(libperfglue_la_OBJECTS) $(libperfglue_la_DEPENDENCIES) $(EXTRA_libperfglue_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libperfglue_la_OBJECTS) $(libperfglue_la_LIBADD) $(LIBS) +librados/$(am__dirstamp): + @$(MKDIR_P) librados + @: > librados/$(am__dirstamp) +librados/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) librados/$(DEPDIR) + @: > librados/$(DEPDIR)/$(am__dirstamp) +librados/librados_la-librados.lo: librados/$(am__dirstamp) \ + librados/$(DEPDIR)/$(am__dirstamp) +librados/librados_la-RadosClient.lo: librados/$(am__dirstamp) \ + librados/$(DEPDIR)/$(am__dirstamp) +librados/librados_la-IoCtxImpl.lo: librados/$(am__dirstamp) \ + librados/$(DEPDIR)/$(am__dirstamp) +librados/librados_la-snap_set_diff.lo: librados/$(am__dirstamp) \ + librados/$(DEPDIR)/$(am__dirstamp) +librados.la: $(librados_la_OBJECTS) $(librados_la_DEPENDENCIES) $(EXTRA_librados_la_DEPENDENCIES) + $(AM_V_CXXLD)$(librados_la_LINK) -rpath $(libdir) $(librados_la_OBJECTS) $(librados_la_LIBADD) $(LIBS) +test/librados/$(am__dirstamp): + @$(MKDIR_P) test/librados + @: > test/librados/$(am__dirstamp) +test/librados/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/librados/$(DEPDIR) + @: > test/librados/$(DEPDIR)/$(am__dirstamp) +test/librados/libradostest_la-test.lo: test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +test/librados/libradostest_la-TestCase.lo: \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +libradostest.la: $(libradostest_la_OBJECTS) $(libradostest_la_DEPENDENCIES) $(EXTRA_libradostest_la_DEPENDENCIES) + $(AM_V_CXXLD)$(libradostest_la_LINK) $(libradostest_la_OBJECTS) $(libradostest_la_LIBADD) $(LIBS) +librbd/$(am__dirstamp): + @$(MKDIR_P) librbd + @: > librbd/$(am__dirstamp) +librbd/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) librbd/$(DEPDIR) + @: > librbd/$(DEPDIR)/$(am__dirstamp) +librbd/librbd.lo: librbd/$(am__dirstamp) \ + librbd/$(DEPDIR)/$(am__dirstamp) +librbd/AioCompletion.lo: librbd/$(am__dirstamp) \ + librbd/$(DEPDIR)/$(am__dirstamp) +librbd/AioRequest.lo: librbd/$(am__dirstamp) \ + librbd/$(DEPDIR)/$(am__dirstamp) +librbd/ImageCtx.lo: librbd/$(am__dirstamp) \ + librbd/$(DEPDIR)/$(am__dirstamp) +librbd/internal.lo: librbd/$(am__dirstamp) \ + librbd/$(DEPDIR)/$(am__dirstamp) +librbd/LibrbdWriteback.lo: librbd/$(am__dirstamp) \ + librbd/$(DEPDIR)/$(am__dirstamp) +librbd/WatchCtx.lo: librbd/$(am__dirstamp) \ + librbd/$(DEPDIR)/$(am__dirstamp) +librbd.la: $(librbd_la_OBJECTS) $(librbd_la_DEPENDENCIES) $(EXTRA_librbd_la_DEPENDENCIES) + $(AM_V_CXXLD)$(librbd_la_LINK) -rpath $(libdir) $(librbd_la_OBJECTS) $(librbd_la_LIBADD) $(LIBS) +rgw/$(am__dirstamp): + @$(MKDIR_P) rgw + @: > rgw/$(am__dirstamp) +rgw/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) rgw/$(DEPDIR) + @: > rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-librgw.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_acl.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_acl_s3.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_acl_swift.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_client_io.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_fcgi.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_xml.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_usage.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_json_enc.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_user.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_bucket.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_tools.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_rados.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_http_client.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_rest_client.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_rest_conn.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_op.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_common.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_cache.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_formats.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_log.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_multi.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_policy_s3.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_gc.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_multi_del.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_env.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_cors.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_cors_s3.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_auth_s3.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_metadata.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_replica_log.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_keystone.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_quota.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/librgw_la-rgw_dencoder.lo: rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +librgw.la: $(librgw_la_OBJECTS) $(librgw_la_DEPENDENCIES) $(EXTRA_librgw_la_DEPENDENCIES) + $(AM_V_CXXLD)$(librgw_la_LINK) $(am_librgw_la_rpath) $(librgw_la_OBJECTS) $(librgw_la_LIBADD) $(LIBS) +test/system/$(am__dirstamp): + @$(MKDIR_P) test/system + @: > test/system/$(am__dirstamp) +test/system/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/system/$(DEPDIR) + @: > test/system/$(DEPDIR)/$(am__dirstamp) +test/system/cross_process_sem.lo: test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/systest_runnable.lo: test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/systest_settings.lo: test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +libsystest.la: $(libsystest_la_OBJECTS) $(libsystest_la_DEPENDENCIES) $(EXTRA_libsystest_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(am_libsystest_la_rpath) $(libsystest_la_OBJECTS) $(libsystest_la_LIBADD) $(LIBS) +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p || test -f $$p1; \ + then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-checkPROGRAMS: + @list='$(check_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +install-sbinPROGRAMS: $(sbin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(sbindir)" || $(MKDIR_P) "$(DESTDIR)$(sbindir)" + @list='$(sbin_PROGRAMS)'; test -n "$(sbindir)" || list=; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p || test -f $$p1; \ + then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(sbindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(sbindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-sbinPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(sbin_PROGRAMS)'; test -n "$(sbindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(sbindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(sbindir)" && rm -f $$files + +clean-sbinPROGRAMS: + @list='$(sbin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +install-su_sbinPROGRAMS: $(su_sbin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(su_sbindir)" || $(MKDIR_P) "$(DESTDIR)$(su_sbindir)" + @list='$(su_sbin_PROGRAMS)'; test -n "$(su_sbindir)" || list=; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p || test -f $$p1; \ + then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(su_sbindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(su_sbindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-su_sbinPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(su_sbin_PROGRAMS)'; test -n "$(su_sbindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(su_sbindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(su_sbindir)" && rm -f $$files + +clean-su_sbinPROGRAMS: + @list='$(su_sbin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +tools/$(am__dirstamp): + @$(MKDIR_P) tools + @: > tools/$(am__dirstamp) +tools/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) tools/$(DEPDIR) + @: > tools/$(DEPDIR)/$(am__dirstamp) +tools/ceph_authtool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph-authtool$(EXEEXT): $(ceph_authtool_OBJECTS) $(ceph_authtool_DEPENDENCIES) $(EXTRA_ceph_authtool_DEPENDENCIES) + @rm -f ceph-authtool$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_authtool_OBJECTS) $(ceph_authtool_LDADD) $(LIBS) +tools/ceph-client-debug.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph-client-debug$(EXEEXT): $(ceph_client_debug_OBJECTS) $(ceph_client_debug_DEPENDENCIES) $(EXTRA_ceph_client_debug_DEPENDENCIES) + @rm -f ceph-client-debug$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_client_debug_OBJECTS) $(ceph_client_debug_LDADD) $(LIBS) +tools/ceph_conf.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph-conf$(EXEEXT): $(ceph_conf_OBJECTS) $(ceph_conf_DEPENDENCIES) $(EXTRA_ceph_conf_DEPENDENCIES) + @rm -f ceph-conf$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_conf_OBJECTS) $(ceph_conf_LDADD) $(LIBS) +test/encoding/$(am__dirstamp): + @$(MKDIR_P) test/encoding + @: > test/encoding/$(am__dirstamp) +test/encoding/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/encoding/$(DEPDIR) + @: > test/encoding/$(DEPDIR)/$(am__dirstamp) +test/encoding/ceph_dencoder-ceph_dencoder.$(OBJEXT): \ + test/encoding/$(am__dirstamp) \ + test/encoding/$(DEPDIR)/$(am__dirstamp) +rgw/ceph_dencoder-rgw_dencoder.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/ceph_dencoder-rgw_acl.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/ceph_dencoder-rgw_common.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/ceph_dencoder-rgw_env.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/ceph_dencoder-rgw_json_enc.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +ceph-dencoder$(EXEEXT): $(ceph_dencoder_OBJECTS) $(ceph_dencoder_DEPENDENCIES) $(EXTRA_ceph_dencoder_DEPENDENCIES) + @rm -f ceph-dencoder$(EXEEXT) + $(AM_V_CXXLD)$(ceph_dencoder_LINK) $(ceph_dencoder_OBJECTS) $(ceph_dencoder_LDADD) $(LIBS) +ceph-fuse$(EXEEXT): $(ceph_fuse_OBJECTS) $(ceph_fuse_DEPENDENCIES) $(EXTRA_ceph_fuse_DEPENDENCIES) + @rm -f ceph-fuse$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_fuse_OBJECTS) $(ceph_fuse_LDADD) $(LIBS) +tools/ceph_kvstore_tool-ceph_kvstore_tool.$(OBJEXT): \ + tools/$(am__dirstamp) tools/$(DEPDIR)/$(am__dirstamp) +ceph-kvstore-tool$(EXEEXT): $(ceph_kvstore_tool_OBJECTS) $(ceph_kvstore_tool_DEPENDENCIES) $(EXTRA_ceph_kvstore_tool_DEPENDENCIES) + @rm -f ceph-kvstore-tool$(EXEEXT) + $(AM_V_CXXLD)$(ceph_kvstore_tool_LINK) $(ceph_kvstore_tool_OBJECTS) $(ceph_kvstore_tool_LDADD) $(LIBS) +ceph-mds$(EXEEXT): $(ceph_mds_OBJECTS) $(ceph_mds_DEPENDENCIES) $(EXTRA_ceph_mds_DEPENDENCIES) + @rm -f ceph-mds$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_mds_OBJECTS) $(ceph_mds_LDADD) $(LIBS) +ceph-mon$(EXEEXT): $(ceph_mon_OBJECTS) $(ceph_mon_DEPENDENCIES) $(EXTRA_ceph_mon_DEPENDENCIES) + @rm -f ceph-mon$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_mon_OBJECTS) $(ceph_mon_LDADD) $(LIBS) +tools/ceph_monstore_tool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph-monstore-tool$(EXEEXT): $(ceph_monstore_tool_OBJECTS) $(ceph_monstore_tool_DEPENDENCIES) $(EXTRA_ceph_monstore_tool_DEPENDENCIES) + @rm -f ceph-monstore-tool$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_monstore_tool_OBJECTS) $(ceph_monstore_tool_LDADD) $(LIBS) +ceph-osd$(EXEEXT): $(ceph_osd_OBJECTS) $(ceph_osd_DEPENDENCIES) $(EXTRA_ceph_osd_DEPENDENCIES) + @rm -f ceph-osd$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_osd_OBJECTS) $(ceph_osd_LDADD) $(LIBS) +tools/ceph_osdomap_tool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph-osdomap-tool$(EXEEXT): $(ceph_osdomap_tool_OBJECTS) $(ceph_osdomap_tool_DEPENDENCIES) $(EXTRA_ceph_osdomap_tool_DEPENDENCIES) + @rm -f ceph-osdomap-tool$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_osdomap_tool_OBJECTS) $(ceph_osdomap_tool_LDADD) $(LIBS) +client/SyntheticClient.$(OBJEXT): client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +ceph-syn$(EXEEXT): $(ceph_syn_OBJECTS) $(ceph_syn_DEPENDENCIES) $(EXTRA_ceph_syn_DEPENDENCIES) + @rm -f ceph-syn$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_syn_OBJECTS) $(ceph_syn_LDADD) $(LIBS) +test/$(am__dirstamp): + @$(MKDIR_P) test + @: > test/$(am__dirstamp) +test/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/$(DEPDIR) + @: > test/$(DEPDIR)/$(am__dirstamp) +test/bench_log.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_bench_log$(EXEEXT): $(ceph_bench_log_OBJECTS) $(ceph_bench_log_DEPENDENCIES) $(EXTRA_ceph_bench_log_DEPENDENCIES) + @rm -f ceph_bench_log$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_bench_log_OBJECTS) $(ceph_bench_log_LDADD) $(LIBS) +tools/dupstore.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_dupstore$(EXEEXT): $(ceph_dupstore_OBJECTS) $(ceph_dupstore_DEPENDENCIES) $(EXTRA_ceph_dupstore_DEPENDENCIES) + @rm -f ceph_dupstore$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_dupstore_OBJECTS) $(ceph_dupstore_LDADD) $(LIBS) +test/erasure-code/ceph_erasure_code.$(OBJEXT): \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +ceph_erasure_code$(EXEEXT): $(ceph_erasure_code_OBJECTS) $(ceph_erasure_code_DEPENDENCIES) $(EXTRA_ceph_erasure_code_DEPENDENCIES) + @rm -f ceph_erasure_code$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_erasure_code_OBJECTS) $(ceph_erasure_code_LDADD) $(LIBS) +test/erasure-code/ceph_erasure_code_benchmark.$(OBJEXT): \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +ceph_erasure_code_benchmark$(EXEEXT): $(ceph_erasure_code_benchmark_OBJECTS) $(ceph_erasure_code_benchmark_DEPENDENCIES) $(EXTRA_ceph_erasure_code_benchmark_DEPENDENCIES) + @rm -f ceph_erasure_code_benchmark$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_erasure_code_benchmark_OBJECTS) $(ceph_erasure_code_benchmark_LDADD) $(LIBS) +tools/ceph_filestore_dump.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_filestore_dump$(EXEEXT): $(ceph_filestore_dump_OBJECTS) $(ceph_filestore_dump_DEPENDENCIES) $(EXTRA_ceph_filestore_dump_DEPENDENCIES) + @rm -f ceph_filestore_dump$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_filestore_dump_OBJECTS) $(ceph_filestore_dump_LDADD) $(LIBS) +tools/ceph_filestore_tool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_filestore_tool$(EXEEXT): $(ceph_filestore_tool_OBJECTS) $(ceph_filestore_tool_DEPENDENCIES) $(EXTRA_ceph_filestore_tool_DEPENDENCIES) + @rm -f ceph_filestore_tool$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_filestore_tool_OBJECTS) $(ceph_filestore_tool_LDADD) $(LIBS) +test/kv_store_bench.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +key_value_store/kv_flat_btree_async.$(OBJEXT): \ + key_value_store/$(am__dirstamp) \ + key_value_store/$(DEPDIR)/$(am__dirstamp) +ceph_kvstorebench$(EXEEXT): $(ceph_kvstorebench_OBJECTS) $(ceph_kvstorebench_DEPENDENCIES) $(EXTRA_ceph_kvstorebench_DEPENDENCIES) + @rm -f ceph_kvstorebench$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_kvstorebench_OBJECTS) $(ceph_kvstorebench_LDADD) $(LIBS) +tools/mon_store_converter.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_mon_store_converter$(EXEEXT): $(ceph_mon_store_converter_OBJECTS) $(ceph_mon_store_converter_DEPENDENCIES) $(EXTRA_ceph_mon_store_converter_DEPENDENCIES) + @rm -f ceph_mon_store_converter$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_mon_store_converter_OBJECTS) $(ceph_mon_store_converter_LDADD) $(LIBS) +test/multi_stress_watch.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_multi_stress_watch$(EXEEXT): $(ceph_multi_stress_watch_OBJECTS) $(ceph_multi_stress_watch_DEPENDENCIES) $(EXTRA_ceph_multi_stress_watch_DEPENDENCIES) + @rm -f ceph_multi_stress_watch$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_multi_stress_watch_OBJECTS) $(ceph_multi_stress_watch_LDADD) $(LIBS) +test/omap_bench.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_omapbench$(EXEEXT): $(ceph_omapbench_OBJECTS) $(ceph_omapbench_DEPENDENCIES) $(EXTRA_ceph_omapbench_DEPENDENCIES) + @rm -f ceph_omapbench$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_omapbench_OBJECTS) $(ceph_omapbench_LDADD) $(LIBS) +tools/psim.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_psim$(EXEEXT): $(ceph_psim_OBJECTS) $(ceph_psim_DEPENDENCIES) $(EXTRA_ceph_psim_DEPENDENCIES) + @rm -f ceph_psim$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_psim_OBJECTS) $(ceph_psim_LDADD) $(LIBS) +tools/radosacl.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_radosacl$(EXEEXT): $(ceph_radosacl_OBJECTS) $(ceph_radosacl_DEPENDENCIES) $(EXTRA_ceph_radosacl_DEPENDENCIES) + @rm -f ceph_radosacl$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_radosacl_OBJECTS) $(ceph_radosacl_LDADD) $(LIBS) +rgw/rgw_jsonparser.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_common.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_env.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_json_enc.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +ceph_rgw_jsonparser$(EXEEXT): $(ceph_rgw_jsonparser_OBJECTS) $(ceph_rgw_jsonparser_DEPENDENCIES) $(EXTRA_ceph_rgw_jsonparser_DEPENDENCIES) + @rm -f ceph_rgw_jsonparser$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_rgw_jsonparser_OBJECTS) $(ceph_rgw_jsonparser_LDADD) $(LIBS) +rgw/rgw_multiparser.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +ceph_rgw_multiparser$(EXEEXT): $(ceph_rgw_multiparser_OBJECTS) $(ceph_rgw_multiparser_DEPENDENCIES) $(EXTRA_ceph_rgw_multiparser_DEPENDENCIES) + @rm -f ceph_rgw_multiparser$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_rgw_multiparser_OBJECTS) $(ceph_rgw_multiparser_LDADD) $(LIBS) +tools/scratchtool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_scratchtool$(EXEEXT): $(ceph_scratchtool_OBJECTS) $(ceph_scratchtool_DEPENDENCIES) $(EXTRA_ceph_scratchtool_DEPENDENCIES) + @rm -f ceph_scratchtool$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(ceph_scratchtool_OBJECTS) $(ceph_scratchtool_LDADD) $(LIBS) +tools/scratchtoolpp.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +ceph_scratchtoolpp$(EXEEXT): $(ceph_scratchtoolpp_OBJECTS) $(ceph_scratchtoolpp_DEPENDENCIES) $(EXTRA_ceph_scratchtoolpp_DEPENDENCIES) + @rm -f ceph_scratchtoolpp$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_scratchtoolpp_OBJECTS) $(ceph_scratchtoolpp_LDADD) $(LIBS) +test/bench/$(am__dirstamp): + @$(MKDIR_P) test/bench + @: > test/bench/$(am__dirstamp) +test/bench/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/bench/$(DEPDIR) + @: > test/bench/$(DEPDIR)/$(am__dirstamp) +test/bench/small_io_bench.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +test/bench/rados_backend.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +test/bench/detailed_stat_collector.$(OBJEXT): \ + test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +test/bench/bencher.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +ceph_smalliobench$(EXEEXT): $(ceph_smalliobench_OBJECTS) $(ceph_smalliobench_DEPENDENCIES) $(EXTRA_ceph_smalliobench_DEPENDENCIES) + @rm -f ceph_smalliobench$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_smalliobench_OBJECTS) $(ceph_smalliobench_LDADD) $(LIBS) +test/bench/small_io_bench_dumb.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +test/bench/dumb_backend.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +ceph_smalliobenchdumb$(EXEEXT): $(ceph_smalliobenchdumb_OBJECTS) $(ceph_smalliobenchdumb_DEPENDENCIES) $(EXTRA_ceph_smalliobenchdumb_DEPENDENCIES) + @rm -f ceph_smalliobenchdumb$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_smalliobenchdumb_OBJECTS) $(ceph_smalliobenchdumb_LDADD) $(LIBS) +test/bench/small_io_bench_fs.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +test/bench/testfilestore_backend.$(OBJEXT): \ + test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +ceph_smalliobenchfs$(EXEEXT): $(ceph_smalliobenchfs_OBJECTS) $(ceph_smalliobenchfs_DEPENDENCIES) $(EXTRA_ceph_smalliobenchfs_DEPENDENCIES) + @rm -f ceph_smalliobenchfs$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_smalliobenchfs_OBJECTS) $(ceph_smalliobenchfs_LDADD) $(LIBS) +test/bench/small_io_bench_rbd.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +test/bench/rbd_backend.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +ceph_smalliobenchrbd$(EXEEXT): $(ceph_smalliobenchrbd_OBJECTS) $(ceph_smalliobenchrbd_DEPENDENCIES) $(EXTRA_ceph_smalliobenchrbd_DEPENDENCIES) + @rm -f ceph_smalliobenchrbd$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_smalliobenchrbd_OBJECTS) $(ceph_smalliobenchrbd_LDADD) $(LIBS) +test/streamtest.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_streamtest$(EXEEXT): $(ceph_streamtest_OBJECTS) $(ceph_streamtest_DEPENDENCIES) $(EXTRA_ceph_streamtest_DEPENDENCIES) + @rm -f ceph_streamtest$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_streamtest_OBJECTS) $(ceph_streamtest_LDADD) $(LIBS) +test/test_c_headers.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_c_headers$(EXEEXT): $(ceph_test_c_headers_OBJECTS) $(ceph_test_c_headers_DEPENDENCIES) $(EXTRA_ceph_test_c_headers_DEPENDENCIES) + @rm -f ceph_test_c_headers$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(ceph_test_c_headers_OBJECTS) $(ceph_test_c_headers_LDADD) $(LIBS) +test/test_cfuse_cache_invalidate.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_cfuse_cache_invalidate$(EXEEXT): $(ceph_test_cfuse_cache_invalidate_OBJECTS) $(ceph_test_cfuse_cache_invalidate_DEPENDENCIES) $(EXTRA_ceph_test_cfuse_cache_invalidate_DEPENDENCIES) + @rm -f ceph_test_cfuse_cache_invalidate$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_cfuse_cache_invalidate_OBJECTS) $(ceph_test_cfuse_cache_invalidate_LDADD) $(LIBS) +test/cls_hello/$(am__dirstamp): + @$(MKDIR_P) test/cls_hello + @: > test/cls_hello/$(am__dirstamp) +test/cls_hello/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_hello/$(DEPDIR) + @: > test/cls_hello/$(DEPDIR)/$(am__dirstamp) +test/cls_hello/ceph_test_cls_hello-test_cls_hello.$(OBJEXT): \ + test/cls_hello/$(am__dirstamp) \ + test/cls_hello/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_hello$(EXEEXT): $(ceph_test_cls_hello_OBJECTS) $(ceph_test_cls_hello_DEPENDENCIES) $(EXTRA_ceph_test_cls_hello_DEPENDENCIES) + @rm -f ceph_test_cls_hello$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_hello_LINK) $(ceph_test_cls_hello_OBJECTS) $(ceph_test_cls_hello_LDADD) $(LIBS) +test/cls_lock/$(am__dirstamp): + @$(MKDIR_P) test/cls_lock + @: > test/cls_lock/$(am__dirstamp) +test/cls_lock/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_lock/$(DEPDIR) + @: > test/cls_lock/$(DEPDIR)/$(am__dirstamp) +test/cls_lock/ceph_test_cls_lock-test_cls_lock.$(OBJEXT): \ + test/cls_lock/$(am__dirstamp) \ + test/cls_lock/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_lock$(EXEEXT): $(ceph_test_cls_lock_OBJECTS) $(ceph_test_cls_lock_DEPENDENCIES) $(EXTRA_ceph_test_cls_lock_DEPENDENCIES) + @rm -f ceph_test_cls_lock$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_lock_LINK) $(ceph_test_cls_lock_OBJECTS) $(ceph_test_cls_lock_LDADD) $(LIBS) +test/cls_log/$(am__dirstamp): + @$(MKDIR_P) test/cls_log + @: > test/cls_log/$(am__dirstamp) +test/cls_log/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_log/$(DEPDIR) + @: > test/cls_log/$(DEPDIR)/$(am__dirstamp) +test/cls_log/ceph_test_cls_log-test_cls_log.$(OBJEXT): \ + test/cls_log/$(am__dirstamp) \ + test/cls_log/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_log$(EXEEXT): $(ceph_test_cls_log_OBJECTS) $(ceph_test_cls_log_DEPENDENCIES) $(EXTRA_ceph_test_cls_log_DEPENDENCIES) + @rm -f ceph_test_cls_log$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_log_LINK) $(ceph_test_cls_log_OBJECTS) $(ceph_test_cls_log_LDADD) $(LIBS) +test/cls_rbd/$(am__dirstamp): + @$(MKDIR_P) test/cls_rbd + @: > test/cls_rbd/$(am__dirstamp) +test/cls_rbd/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_rbd/$(DEPDIR) + @: > test/cls_rbd/$(DEPDIR)/$(am__dirstamp) +test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.$(OBJEXT): \ + test/cls_rbd/$(am__dirstamp) \ + test/cls_rbd/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_rbd$(EXEEXT): $(ceph_test_cls_rbd_OBJECTS) $(ceph_test_cls_rbd_DEPENDENCIES) $(EXTRA_ceph_test_cls_rbd_DEPENDENCIES) + @rm -f ceph_test_cls_rbd$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_rbd_LINK) $(ceph_test_cls_rbd_OBJECTS) $(ceph_test_cls_rbd_LDADD) $(LIBS) +test/cls_refcount/$(am__dirstamp): + @$(MKDIR_P) test/cls_refcount + @: > test/cls_refcount/$(am__dirstamp) +test/cls_refcount/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_refcount/$(DEPDIR) + @: > test/cls_refcount/$(DEPDIR)/$(am__dirstamp) +test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.$(OBJEXT): \ + test/cls_refcount/$(am__dirstamp) \ + test/cls_refcount/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_refcount$(EXEEXT): $(ceph_test_cls_refcount_OBJECTS) $(ceph_test_cls_refcount_DEPENDENCIES) $(EXTRA_ceph_test_cls_refcount_DEPENDENCIES) + @rm -f ceph_test_cls_refcount$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_refcount_LINK) $(ceph_test_cls_refcount_OBJECTS) $(ceph_test_cls_refcount_LDADD) $(LIBS) +test/cls_replica_log/$(am__dirstamp): + @$(MKDIR_P) test/cls_replica_log + @: > test/cls_replica_log/$(am__dirstamp) +test/cls_replica_log/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_replica_log/$(DEPDIR) + @: > test/cls_replica_log/$(DEPDIR)/$(am__dirstamp) +test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.$(OBJEXT): \ + test/cls_replica_log/$(am__dirstamp) \ + test/cls_replica_log/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_replica_log$(EXEEXT): $(ceph_test_cls_replica_log_OBJECTS) $(ceph_test_cls_replica_log_DEPENDENCIES) $(EXTRA_ceph_test_cls_replica_log_DEPENDENCIES) + @rm -f ceph_test_cls_replica_log$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_replica_log_LINK) $(ceph_test_cls_replica_log_OBJECTS) $(ceph_test_cls_replica_log_LDADD) $(LIBS) +test/cls_rgw/$(am__dirstamp): + @$(MKDIR_P) test/cls_rgw + @: > test/cls_rgw/$(am__dirstamp) +test/cls_rgw/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_rgw/$(DEPDIR) + @: > test/cls_rgw/$(DEPDIR)/$(am__dirstamp) +test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.$(OBJEXT): \ + test/cls_rgw/$(am__dirstamp) \ + test/cls_rgw/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_rgw$(EXEEXT): $(ceph_test_cls_rgw_OBJECTS) $(ceph_test_cls_rgw_DEPENDENCIES) $(EXTRA_ceph_test_cls_rgw_DEPENDENCIES) + @rm -f ceph_test_cls_rgw$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_rgw_LINK) $(ceph_test_cls_rgw_OBJECTS) $(ceph_test_cls_rgw_LDADD) $(LIBS) +test/ceph_test_cls_rgw_log-test_rgw_admin_log.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_rgw_log$(EXEEXT): $(ceph_test_cls_rgw_log_OBJECTS) $(ceph_test_cls_rgw_log_DEPENDENCIES) $(EXTRA_ceph_test_cls_rgw_log_DEPENDENCIES) + @rm -f ceph_test_cls_rgw_log$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_rgw_log_LINK) $(ceph_test_cls_rgw_log_OBJECTS) $(ceph_test_cls_rgw_log_LDADD) $(LIBS) +test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_rgw_meta$(EXEEXT): $(ceph_test_cls_rgw_meta_OBJECTS) $(ceph_test_cls_rgw_meta_DEPENDENCIES) $(EXTRA_ceph_test_cls_rgw_meta_DEPENDENCIES) + @rm -f ceph_test_cls_rgw_meta$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_rgw_meta_LINK) $(ceph_test_cls_rgw_meta_OBJECTS) $(ceph_test_cls_rgw_meta_LDADD) $(LIBS) +test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_rgw_opstate$(EXEEXT): $(ceph_test_cls_rgw_opstate_OBJECTS) $(ceph_test_cls_rgw_opstate_DEPENDENCIES) $(EXTRA_ceph_test_cls_rgw_opstate_DEPENDENCIES) + @rm -f ceph_test_cls_rgw_opstate$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_rgw_opstate_LINK) $(ceph_test_cls_rgw_opstate_OBJECTS) $(ceph_test_cls_rgw_opstate_LDADD) $(LIBS) +test/cls_statelog/$(am__dirstamp): + @$(MKDIR_P) test/cls_statelog + @: > test/cls_statelog/$(am__dirstamp) +test/cls_statelog/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_statelog/$(DEPDIR) + @: > test/cls_statelog/$(DEPDIR)/$(am__dirstamp) +test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.$(OBJEXT): \ + test/cls_statelog/$(am__dirstamp) \ + test/cls_statelog/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_statelog$(EXEEXT): $(ceph_test_cls_statelog_OBJECTS) $(ceph_test_cls_statelog_DEPENDENCIES) $(EXTRA_ceph_test_cls_statelog_DEPENDENCIES) + @rm -f ceph_test_cls_statelog$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_statelog_LINK) $(ceph_test_cls_statelog_OBJECTS) $(ceph_test_cls_statelog_LDADD) $(LIBS) +test/cls_version/$(am__dirstamp): + @$(MKDIR_P) test/cls_version + @: > test/cls_version/$(am__dirstamp) +test/cls_version/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/cls_version/$(DEPDIR) + @: > test/cls_version/$(DEPDIR)/$(am__dirstamp) +test/cls_version/ceph_test_cls_version-test_cls_version.$(OBJEXT): \ + test/cls_version/$(am__dirstamp) \ + test/cls_version/$(DEPDIR)/$(am__dirstamp) +ceph_test_cls_version$(EXEEXT): $(ceph_test_cls_version_OBJECTS) $(ceph_test_cls_version_DEPENDENCIES) $(EXTRA_ceph_test_cls_version_DEPENDENCIES) + @rm -f ceph_test_cls_version$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cls_version_LINK) $(ceph_test_cls_version_OBJECTS) $(ceph_test_cls_version_LDADD) $(LIBS) +test/ceph_test_cors-test_cors.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_cors$(EXEEXT): $(ceph_test_cors_OBJECTS) $(ceph_test_cors_DEPENDENCIES) $(EXTRA_ceph_test_cors_DEPENDENCIES) + @rm -f ceph_test_cors$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_cors_LINK) $(ceph_test_cors_OBJECTS) $(ceph_test_cors_LDADD) $(LIBS) +test/testcrypto.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_crypto$(EXEEXT): $(ceph_test_crypto_OBJECTS) $(ceph_test_crypto_DEPENDENCIES) $(EXTRA_ceph_test_crypto_DEPENDENCIES) + @rm -f ceph_test_crypto$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_crypto_OBJECTS) $(ceph_test_crypto_LDADD) $(LIBS) +test/ceph_test_filejournal-test_filejournal.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +ceph_test_filejournal$(EXEEXT): $(ceph_test_filejournal_OBJECTS) $(ceph_test_filejournal_DEPENDENCIES) $(EXTRA_ceph_test_filejournal_DEPENDENCIES) + @rm -f ceph_test_filejournal$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_filejournal_LINK) $(ceph_test_filejournal_OBJECTS) $(ceph_test_filejournal_LDADD) $(LIBS) +test/objectstore/$(am__dirstamp): + @$(MKDIR_P) test/objectstore + @: > test/objectstore/$(am__dirstamp) +test/objectstore/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/objectstore/$(DEPDIR) + @: > test/objectstore/$(DEPDIR)/$(am__dirstamp) +test/objectstore/test_idempotent.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +test/objectstore/FileStoreTracker.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +test/common/$(am__dirstamp): + @$(MKDIR_P) test/common + @: > test/common/$(am__dirstamp) +test/common/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/common/$(DEPDIR) + @: > test/common/$(DEPDIR)/$(am__dirstamp) +test/common/ObjectContents.$(OBJEXT): test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +ceph_test_filestore_idempotent$(EXEEXT): $(ceph_test_filestore_idempotent_OBJECTS) $(ceph_test_filestore_idempotent_DEPENDENCIES) $(EXTRA_ceph_test_filestore_idempotent_DEPENDENCIES) + @rm -f ceph_test_filestore_idempotent$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_filestore_idempotent_OBJECTS) $(ceph_test_filestore_idempotent_LDADD) $(LIBS) +test/objectstore/test_idempotent_sequence.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +test/objectstore/DeterministicOpSequence.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +test/objectstore/TestObjectStoreState.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +test/objectstore/FileStoreDiff.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +ceph_test_filestore_idempotent_sequence$(EXEEXT): $(ceph_test_filestore_idempotent_sequence_OBJECTS) $(ceph_test_filestore_idempotent_sequence_DEPENDENCIES) $(EXTRA_ceph_test_filestore_idempotent_sequence_DEPENDENCIES) + @rm -f ceph_test_filestore_idempotent_sequence$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_filestore_idempotent_sequence_OBJECTS) $(ceph_test_filestore_idempotent_sequence_LDADD) $(LIBS) +test/test_get_blkdev_size.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_get_blkdev_size$(EXEEXT): $(ceph_test_get_blkdev_size_OBJECTS) $(ceph_test_get_blkdev_size_DEPENDENCIES) $(EXTRA_ceph_test_get_blkdev_size_DEPENDENCIES) + @rm -f ceph_test_get_blkdev_size$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_get_blkdev_size_OBJECTS) $(ceph_test_get_blkdev_size_LDADD) $(LIBS) +client/test_ioctls.$(OBJEXT): client/$(am__dirstamp) \ + client/$(DEPDIR)/$(am__dirstamp) +ceph_test_ioctls$(EXEEXT): $(ceph_test_ioctls_OBJECTS) $(ceph_test_ioctls_DEPENDENCIES) $(EXTRA_ceph_test_ioctls_DEPENDENCIES) + @rm -f ceph_test_ioctls$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(ceph_test_ioctls_OBJECTS) $(ceph_test_ioctls_LDADD) $(LIBS) +test/testkeys.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_keys$(EXEEXT): $(ceph_test_keys_OBJECTS) $(ceph_test_keys_DEPENDENCIES) $(EXTRA_ceph_test_keys_DEPENDENCIES) + @rm -f ceph_test_keys$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_keys_OBJECTS) $(ceph_test_keys_LDADD) $(LIBS) +test/ObjectMap/$(am__dirstamp): + @$(MKDIR_P) test/ObjectMap + @: > test/ObjectMap/$(am__dirstamp) +test/ObjectMap/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/ObjectMap/$(DEPDIR) + @: > test/ObjectMap/$(DEPDIR)/$(am__dirstamp) +test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.$(OBJEXT): \ + test/ObjectMap/$(am__dirstamp) \ + test/ObjectMap/$(DEPDIR)/$(am__dirstamp) +ceph_test_keyvaluedb_atomicity$(EXEEXT): $(ceph_test_keyvaluedb_atomicity_OBJECTS) $(ceph_test_keyvaluedb_atomicity_DEPENDENCIES) $(EXTRA_ceph_test_keyvaluedb_atomicity_DEPENDENCIES) + @rm -f ceph_test_keyvaluedb_atomicity$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_keyvaluedb_atomicity_LINK) $(ceph_test_keyvaluedb_atomicity_OBJECTS) $(ceph_test_keyvaluedb_atomicity_LDADD) $(LIBS) +test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.$(OBJEXT): \ + test/ObjectMap/$(am__dirstamp) \ + test/ObjectMap/$(DEPDIR)/$(am__dirstamp) +test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.$(OBJEXT): \ + test/ObjectMap/$(am__dirstamp) \ + test/ObjectMap/$(DEPDIR)/$(am__dirstamp) +ceph_test_keyvaluedb_iterators$(EXEEXT): $(ceph_test_keyvaluedb_iterators_OBJECTS) $(ceph_test_keyvaluedb_iterators_DEPENDENCIES) $(EXTRA_ceph_test_keyvaluedb_iterators_DEPENDENCIES) + @rm -f ceph_test_keyvaluedb_iterators$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_keyvaluedb_iterators_LINK) $(ceph_test_keyvaluedb_iterators_OBJECTS) $(ceph_test_keyvaluedb_iterators_LDADD) $(LIBS) +test/libcephfs/$(am__dirstamp): + @$(MKDIR_P) test/libcephfs + @: > test/libcephfs/$(am__dirstamp) +test/libcephfs/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/libcephfs/$(DEPDIR) + @: > test/libcephfs/$(DEPDIR)/$(am__dirstamp) +test/libcephfs/ceph_test_libcephfs-test.$(OBJEXT): \ + test/libcephfs/$(am__dirstamp) \ + test/libcephfs/$(DEPDIR)/$(am__dirstamp) +test/libcephfs/ceph_test_libcephfs-readdir_r_cb.$(OBJEXT): \ + test/libcephfs/$(am__dirstamp) \ + test/libcephfs/$(DEPDIR)/$(am__dirstamp) +test/libcephfs/ceph_test_libcephfs-caps.$(OBJEXT): \ + test/libcephfs/$(am__dirstamp) \ + test/libcephfs/$(DEPDIR)/$(am__dirstamp) +test/libcephfs/ceph_test_libcephfs-multiclient.$(OBJEXT): \ + test/libcephfs/$(am__dirstamp) \ + test/libcephfs/$(DEPDIR)/$(am__dirstamp) +ceph_test_libcephfs$(EXEEXT): $(ceph_test_libcephfs_OBJECTS) $(ceph_test_libcephfs_DEPENDENCIES) $(EXTRA_ceph_test_libcephfs_DEPENDENCIES) + @rm -f ceph_test_libcephfs$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_libcephfs_LINK) $(ceph_test_libcephfs_OBJECTS) $(ceph_test_libcephfs_LDADD) $(LIBS) +test/librbd/$(am__dirstamp): + @$(MKDIR_P) test/librbd + @: > test/librbd/$(am__dirstamp) +test/librbd/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/librbd/$(DEPDIR) + @: > test/librbd/$(DEPDIR)/$(am__dirstamp) +test/librbd/ceph_test_librbd-test_librbd.$(OBJEXT): \ + test/librbd/$(am__dirstamp) \ + test/librbd/$(DEPDIR)/$(am__dirstamp) +ceph_test_librbd$(EXEEXT): $(ceph_test_librbd_OBJECTS) $(ceph_test_librbd_DEPENDENCIES) $(EXTRA_ceph_test_librbd_DEPENDENCIES) + @rm -f ceph_test_librbd$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_librbd_LINK) $(ceph_test_librbd_OBJECTS) $(ceph_test_librbd_LDADD) $(LIBS) +test/librbd/ceph_test_librbd_fsx-fsx.$(OBJEXT): \ + test/librbd/$(am__dirstamp) \ + test/librbd/$(DEPDIR)/$(am__dirstamp) +ceph_test_librbd_fsx$(EXEEXT): $(ceph_test_librbd_fsx_OBJECTS) $(ceph_test_librbd_fsx_DEPENDENCIES) $(EXTRA_ceph_test_librbd_fsx_DEPENDENCIES) + @rm -f ceph_test_librbd_fsx$(EXEEXT) + $(AM_V_CCLD)$(ceph_test_librbd_fsx_LINK) $(ceph_test_librbd_fsx_OBJECTS) $(ceph_test_librbd_fsx_LDADD) $(LIBS) +test/mon/$(am__dirstamp): + @$(MKDIR_P) test/mon + @: > test/mon/$(am__dirstamp) +test/mon/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/mon/$(DEPDIR) + @: > test/mon/$(DEPDIR)/$(am__dirstamp) +test/mon/test_mon_workloadgen.$(OBJEXT): test/mon/$(am__dirstamp) \ + test/mon/$(DEPDIR)/$(am__dirstamp) +ceph_test_mon_workloadgen$(EXEEXT): $(ceph_test_mon_workloadgen_OBJECTS) $(ceph_test_mon_workloadgen_DEPENDENCIES) $(EXTRA_ceph_test_mon_workloadgen_DEPENDENCIES) + @rm -f ceph_test_mon_workloadgen$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_mon_workloadgen_OBJECTS) $(ceph_test_mon_workloadgen_LDADD) $(LIBS) +test/testmsgr.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_msgr$(EXEEXT): $(ceph_test_msgr_OBJECTS) $(ceph_test_msgr_DEPENDENCIES) $(EXTRA_ceph_test_msgr_DEPENDENCIES) + @rm -f ceph_test_msgr$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_msgr_OBJECTS) $(ceph_test_msgr_LDADD) $(LIBS) +test/test_mutate.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_mutate$(EXEEXT): $(ceph_test_mutate_OBJECTS) $(ceph_test_mutate_DEPENDENCIES) $(EXTRA_ceph_test_mutate_DEPENDENCIES) + @rm -f ceph_test_mutate$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_mutate_OBJECTS) $(ceph_test_mutate_LDADD) $(LIBS) +test/ObjectMap/ceph_test_object_map-test_object_map.$(OBJEXT): \ + test/ObjectMap/$(am__dirstamp) \ + test/ObjectMap/$(DEPDIR)/$(am__dirstamp) +test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.$(OBJEXT): \ + test/ObjectMap/$(am__dirstamp) \ + test/ObjectMap/$(DEPDIR)/$(am__dirstamp) +ceph_test_object_map$(EXEEXT): $(ceph_test_object_map_OBJECTS) $(ceph_test_object_map_DEPENDENCIES) $(EXTRA_ceph_test_object_map_DEPENDENCIES) + @rm -f ceph_test_object_map$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_object_map_LINK) $(ceph_test_object_map_OBJECTS) $(ceph_test_object_map_LDADD) $(LIBS) +test/osdc/$(am__dirstamp): + @$(MKDIR_P) test/osdc + @: > test/osdc/$(am__dirstamp) +test/osdc/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/osdc/$(DEPDIR) + @: > test/osdc/$(DEPDIR)/$(am__dirstamp) +test/osdc/object_cacher_stress.$(OBJEXT): test/osdc/$(am__dirstamp) \ + test/osdc/$(DEPDIR)/$(am__dirstamp) +test/osdc/FakeWriteback.$(OBJEXT): test/osdc/$(am__dirstamp) \ + test/osdc/$(DEPDIR)/$(am__dirstamp) +ceph_test_objectcacher_stress$(EXEEXT): $(ceph_test_objectcacher_stress_OBJECTS) $(ceph_test_objectcacher_stress_DEPENDENCIES) $(EXTRA_ceph_test_objectcacher_stress_DEPENDENCIES) + @rm -f ceph_test_objectcacher_stress$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_objectcacher_stress_OBJECTS) $(ceph_test_objectcacher_stress_LDADD) $(LIBS) +test/objectstore/ceph_test_objectstore-store_test.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +ceph_test_objectstore$(EXEEXT): $(ceph_test_objectstore_OBJECTS) $(ceph_test_objectstore_DEPENDENCIES) $(EXTRA_ceph_test_objectstore_DEPENDENCIES) + @rm -f ceph_test_objectstore$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_objectstore_LINK) $(ceph_test_objectstore_OBJECTS) $(ceph_test_objectstore_LDADD) $(LIBS) +test/objectstore/workload_generator.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +ceph_test_objectstore_workloadgen$(EXEEXT): $(ceph_test_objectstore_workloadgen_OBJECTS) $(ceph_test_objectstore_workloadgen_DEPENDENCIES) $(EXTRA_ceph_test_objectstore_workloadgen_DEPENDENCIES) + @rm -f ceph_test_objectstore_workloadgen$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_objectstore_workloadgen_OBJECTS) $(ceph_test_objectstore_workloadgen_LDADD) $(LIBS) +test/osd/$(am__dirstamp): + @$(MKDIR_P) test/osd + @: > test/osd/$(am__dirstamp) +test/osd/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/osd/$(DEPDIR) + @: > test/osd/$(DEPDIR)/$(am__dirstamp) +test/osd/TestRados.$(OBJEXT): test/osd/$(am__dirstamp) \ + test/osd/$(DEPDIR)/$(am__dirstamp) +test/osd/TestOpStat.$(OBJEXT): test/osd/$(am__dirstamp) \ + test/osd/$(DEPDIR)/$(am__dirstamp) +test/osd/Object.$(OBJEXT): test/osd/$(am__dirstamp) \ + test/osd/$(DEPDIR)/$(am__dirstamp) +test/osd/RadosModel.$(OBJEXT): test/osd/$(am__dirstamp) \ + test/osd/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados$(EXEEXT): $(ceph_test_rados_OBJECTS) $(ceph_test_rados_DEPENDENCIES) $(EXTRA_ceph_test_rados_DEPENDENCIES) + @rm -f ceph_test_rados$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_rados_OBJECTS) $(ceph_test_rados_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_aio$(EXEEXT): $(ceph_test_rados_api_aio_OBJECTS) $(ceph_test_rados_api_aio_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_aio_DEPENDENCIES) + @rm -f ceph_test_rados_api_aio$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_aio_LINK) $(ceph_test_rados_api_aio_OBJECTS) $(ceph_test_rados_api_aio_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_c_read_operations$(EXEEXT): $(ceph_test_rados_api_c_read_operations_OBJECTS) $(ceph_test_rados_api_c_read_operations_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_c_read_operations_DEPENDENCIES) + @rm -f ceph_test_rados_api_c_read_operations$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_c_read_operations_LINK) $(ceph_test_rados_api_c_read_operations_OBJECTS) $(ceph_test_rados_api_c_read_operations_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_c_write_operations$(EXEEXT): $(ceph_test_rados_api_c_write_operations_OBJECTS) $(ceph_test_rados_api_c_write_operations_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_c_write_operations_DEPENDENCIES) + @rm -f ceph_test_rados_api_c_write_operations$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_c_write_operations_LINK) $(ceph_test_rados_api_c_write_operations_OBJECTS) $(ceph_test_rados_api_c_write_operations_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_cls$(EXEEXT): $(ceph_test_rados_api_cls_OBJECTS) $(ceph_test_rados_api_cls_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_cls_DEPENDENCIES) + @rm -f ceph_test_rados_api_cls$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_cls_LINK) $(ceph_test_rados_api_cls_OBJECTS) $(ceph_test_rados_api_cls_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_cmd$(EXEEXT): $(ceph_test_rados_api_cmd_OBJECTS) $(ceph_test_rados_api_cmd_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_cmd_DEPENDENCIES) + @rm -f ceph_test_rados_api_cmd$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_cmd_LINK) $(ceph_test_rados_api_cmd_OBJECTS) $(ceph_test_rados_api_cmd_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_io-io.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_io$(EXEEXT): $(ceph_test_rados_api_io_OBJECTS) $(ceph_test_rados_api_io_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_io_DEPENDENCIES) + @rm -f ceph_test_rados_api_io$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_io_LINK) $(ceph_test_rados_api_io_OBJECTS) $(ceph_test_rados_api_io_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_list-list.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_list$(EXEEXT): $(ceph_test_rados_api_list_OBJECTS) $(ceph_test_rados_api_list_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_list_DEPENDENCIES) + @rm -f ceph_test_rados_api_list$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_list_LINK) $(ceph_test_rados_api_list_OBJECTS) $(ceph_test_rados_api_list_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_lock-lock.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_lock$(EXEEXT): $(ceph_test_rados_api_lock_OBJECTS) $(ceph_test_rados_api_lock_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_lock_DEPENDENCIES) + @rm -f ceph_test_rados_api_lock$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_lock_LINK) $(ceph_test_rados_api_lock_OBJECTS) $(ceph_test_rados_api_lock_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_misc-misc.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_misc$(EXEEXT): $(ceph_test_rados_api_misc_OBJECTS) $(ceph_test_rados_api_misc_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_misc_DEPENDENCIES) + @rm -f ceph_test_rados_api_misc$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_misc_LINK) $(ceph_test_rados_api_misc_OBJECTS) $(ceph_test_rados_api_misc_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_pool-pool.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_pool$(EXEEXT): $(ceph_test_rados_api_pool_OBJECTS) $(ceph_test_rados_api_pool_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_pool_DEPENDENCIES) + @rm -f ceph_test_rados_api_pool$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_pool_LINK) $(ceph_test_rados_api_pool_OBJECTS) $(ceph_test_rados_api_pool_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_snapshots-snapshots.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_snapshots$(EXEEXT): $(ceph_test_rados_api_snapshots_OBJECTS) $(ceph_test_rados_api_snapshots_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_snapshots_DEPENDENCIES) + @rm -f ceph_test_rados_api_snapshots$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_snapshots_LINK) $(ceph_test_rados_api_snapshots_OBJECTS) $(ceph_test_rados_api_snapshots_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_stat$(EXEEXT): $(ceph_test_rados_api_stat_OBJECTS) $(ceph_test_rados_api_stat_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_stat_DEPENDENCIES) + @rm -f ceph_test_rados_api_stat$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_stat_LINK) $(ceph_test_rados_api_stat_OBJECTS) $(ceph_test_rados_api_stat_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT): osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_tier$(EXEEXT): $(ceph_test_rados_api_tier_OBJECTS) $(ceph_test_rados_api_tier_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_tier_DEPENDENCIES) + @rm -f ceph_test_rados_api_tier$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_tier_LINK) $(ceph_test_rados_api_tier_OBJECTS) $(ceph_test_rados_api_tier_LDADD) $(LIBS) +test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_api_watch_notify$(EXEEXT): $(ceph_test_rados_api_watch_notify_OBJECTS) $(ceph_test_rados_api_watch_notify_DEPENDENCIES) $(EXTRA_ceph_test_rados_api_watch_notify_DEPENDENCIES) + @rm -f ceph_test_rados_api_watch_notify$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rados_api_watch_notify_LINK) $(ceph_test_rados_api_watch_notify_OBJECTS) $(ceph_test_rados_api_watch_notify_LDADD) $(LIBS) +test/system/rados_delete_pools_parallel.$(OBJEXT): \ + test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/st_rados_create_pool.$(OBJEXT): \ + test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/st_rados_delete_pool.$(OBJEXT): \ + test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/st_rados_list_objects.$(OBJEXT): \ + test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_delete_pools_parallel$(EXEEXT): $(ceph_test_rados_delete_pools_parallel_OBJECTS) $(ceph_test_rados_delete_pools_parallel_DEPENDENCIES) $(EXTRA_ceph_test_rados_delete_pools_parallel_DEPENDENCIES) + @rm -f ceph_test_rados_delete_pools_parallel$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_rados_delete_pools_parallel_OBJECTS) $(ceph_test_rados_delete_pools_parallel_LDADD) $(LIBS) +test/system/rados_list_parallel.$(OBJEXT): \ + test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_list_parallel$(EXEEXT): $(ceph_test_rados_list_parallel_OBJECTS) $(ceph_test_rados_list_parallel_DEPENDENCIES) $(EXTRA_ceph_test_rados_list_parallel_DEPENDENCIES) + @rm -f ceph_test_rados_list_parallel$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_rados_list_parallel_OBJECTS) $(ceph_test_rados_list_parallel_LDADD) $(LIBS) +test/system/rados_open_pools_parallel.$(OBJEXT): \ + test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_open_pools_parallel$(EXEEXT): $(ceph_test_rados_open_pools_parallel_OBJECTS) $(ceph_test_rados_open_pools_parallel_DEPENDENCIES) $(EXTRA_ceph_test_rados_open_pools_parallel_DEPENDENCIES) + @rm -f ceph_test_rados_open_pools_parallel$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_rados_open_pools_parallel_OBJECTS) $(ceph_test_rados_open_pools_parallel_LDADD) $(LIBS) +test/system/rados_watch_notify.$(OBJEXT): test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/st_rados_delete_objs.$(OBJEXT): \ + test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/st_rados_watch.$(OBJEXT): test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +test/system/st_rados_notify.$(OBJEXT): test/system/$(am__dirstamp) \ + test/system/$(DEPDIR)/$(am__dirstamp) +ceph_test_rados_watch_notify$(EXEEXT): $(ceph_test_rados_watch_notify_OBJECTS) $(ceph_test_rados_watch_notify_DEPENDENCIES) $(EXTRA_ceph_test_rados_watch_notify_DEPENDENCIES) + @rm -f ceph_test_rados_watch_notify$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_rados_watch_notify_OBJECTS) $(ceph_test_rados_watch_notify_LDADD) $(LIBS) +test/test_rewrite_latency.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_rewrite_latency$(EXEEXT): $(ceph_test_rewrite_latency_OBJECTS) $(ceph_test_rewrite_latency_DEPENDENCIES) $(EXTRA_ceph_test_rewrite_latency_DEPENDENCIES) + @rm -f ceph_test_rewrite_latency$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_rewrite_latency_OBJECTS) $(ceph_test_rewrite_latency_LDADD) $(LIBS) +test/rgw/$(am__dirstamp): + @$(MKDIR_P) test/rgw + @: > test/rgw/$(am__dirstamp) +test/rgw/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/rgw/$(DEPDIR) + @: > test/rgw/$(DEPDIR)/$(am__dirstamp) +test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.$(OBJEXT): \ + test/rgw/$(am__dirstamp) test/rgw/$(DEPDIR)/$(am__dirstamp) +ceph_test_rgw_manifest$(EXEEXT): $(ceph_test_rgw_manifest_OBJECTS) $(ceph_test_rgw_manifest_DEPENDENCIES) $(EXTRA_ceph_test_rgw_manifest_DEPENDENCIES) + @rm -f ceph_test_rgw_manifest$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_rgw_manifest_LINK) $(ceph_test_rgw_manifest_OBJECTS) $(ceph_test_rgw_manifest_LDADD) $(LIBS) +test/TestSignalHandlers.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_signal_handlers$(EXEEXT): $(ceph_test_signal_handlers_OBJECTS) $(ceph_test_signal_handlers_DEPENDENCIES) $(EXTRA_ceph_test_signal_handlers_DEPENDENCIES) + @rm -f ceph_test_signal_handlers$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_signal_handlers_OBJECTS) $(ceph_test_signal_handlers_LDADD) $(LIBS) +test/ceph_test_snap_mapper-test_snap_mapper.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +ceph_test_snap_mapper$(EXEEXT): $(ceph_test_snap_mapper_OBJECTS) $(ceph_test_snap_mapper_DEPENDENCIES) $(EXTRA_ceph_test_snap_mapper_DEPENDENCIES) + @rm -f ceph_test_snap_mapper$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_snap_mapper_LINK) $(ceph_test_snap_mapper_OBJECTS) $(ceph_test_snap_mapper_LDADD) $(LIBS) +test/ceph_test_stress_watch-test_stress_watch.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +ceph_test_stress_watch$(EXEEXT): $(ceph_test_stress_watch_OBJECTS) $(ceph_test_stress_watch_DEPENDENCIES) $(EXTRA_ceph_test_stress_watch_DEPENDENCIES) + @rm -f ceph_test_stress_watch$(EXEEXT) + $(AM_V_CXXLD)$(ceph_test_stress_watch_LINK) $(ceph_test_stress_watch_OBJECTS) $(ceph_test_stress_watch_LDADD) $(LIBS) +test/TestTimers.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_timers$(EXEEXT): $(ceph_test_timers_OBJECTS) $(ceph_test_timers_DEPENDENCIES) $(EXTRA_ceph_test_timers_DEPENDENCIES) + @rm -f ceph_test_timers$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_timers_OBJECTS) $(ceph_test_timers_LDADD) $(LIBS) +test/test_trans.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_test_trans$(EXEEXT): $(ceph_test_trans_OBJECTS) $(ceph_test_trans_DEPENDENCIES) $(EXTRA_ceph_test_trans_DEPENDENCIES) + @rm -f ceph_test_trans$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_test_trans_OBJECTS) $(ceph_test_trans_LDADD) $(LIBS) +test/bench/tp_bench.$(OBJEXT): test/bench/$(am__dirstamp) \ + test/bench/$(DEPDIR)/$(am__dirstamp) +ceph_tpbench$(EXEEXT): $(ceph_tpbench_OBJECTS) $(ceph_tpbench_DEPENDENCIES) $(EXTRA_ceph_tpbench_DEPENDENCIES) + @rm -f ceph_tpbench$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(ceph_tpbench_OBJECTS) $(ceph_tpbench_LDADD) $(LIBS) +test/ceph_xattr_bench-xattr_bench.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +ceph_xattr_bench$(EXEEXT): $(ceph_xattr_bench_OBJECTS) $(ceph_xattr_bench_DEPENDENCIES) $(EXTRA_ceph_xattr_bench_DEPENDENCIES) + @rm -f ceph_xattr_bench$(EXEEXT) + $(AM_V_CXXLD)$(ceph_xattr_bench_LINK) $(ceph_xattr_bench_OBJECTS) $(ceph_xattr_bench_LDADD) $(LIBS) +cephfs$(EXEEXT): $(cephfs_OBJECTS) $(cephfs_DEPENDENCIES) $(EXTRA_cephfs_DEPENDENCIES) + @rm -f cephfs$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(cephfs_OBJECTS) $(cephfs_LDADD) $(LIBS) +tools/crushtool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +crushtool$(EXEEXT): $(crushtool_OBJECTS) $(crushtool_DEPENDENCIES) $(EXTRA_crushtool_DEPENDENCIES) + @rm -f crushtool$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(crushtool_OBJECTS) $(crushtool_LDADD) $(LIBS) +test/common/get_command_descriptions.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +get_command_descriptions$(EXEEXT): $(get_command_descriptions_OBJECTS) $(get_command_descriptions_DEPENDENCIES) $(EXTRA_get_command_descriptions_DEPENDENCIES) + @rm -f get_command_descriptions$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(get_command_descriptions_OBJECTS) $(get_command_descriptions_LDADD) $(LIBS) +librados-config$(EXEEXT): $(librados_config_OBJECTS) $(librados_config_DEPENDENCIES) $(EXTRA_librados_config_DEPENDENCIES) + @rm -f librados-config$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(librados_config_OBJECTS) $(librados_config_LDADD) $(LIBS) +tools/monmaptool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +monmaptool$(EXEEXT): $(monmaptool_OBJECTS) $(monmaptool_DEPENDENCIES) $(EXTRA_monmaptool_DEPENDENCIES) + @rm -f monmaptool$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(monmaptool_OBJECTS) $(monmaptool_LDADD) $(LIBS) +mount/$(am__dirstamp): + @$(MKDIR_P) mount + @: > mount/$(am__dirstamp) +mount/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) mount/$(DEPDIR) + @: > mount/$(DEPDIR)/$(am__dirstamp) +mount/mount.ceph.$(OBJEXT): mount/$(am__dirstamp) \ + mount/$(DEPDIR)/$(am__dirstamp) +common/secret.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +mount.ceph$(EXEEXT): $(mount_ceph_OBJECTS) $(mount_ceph_DEPENDENCIES) $(EXTRA_mount_ceph_DEPENDENCIES) + @rm -f mount.ceph$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(mount_ceph_OBJECTS) $(mount_ceph_LDADD) $(LIBS) +tools/osdmaptool.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +osdmaptool$(EXEEXT): $(osdmaptool_OBJECTS) $(osdmaptool_DEPENDENCIES) $(EXTRA_osdmaptool_DEPENDENCIES) + @rm -f osdmaptool$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(osdmaptool_OBJECTS) $(osdmaptool_LDADD) $(LIBS) +tools/rados/$(am__dirstamp): + @$(MKDIR_P) tools/rados + @: > tools/rados/$(am__dirstamp) +tools/rados/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) tools/rados/$(DEPDIR) + @: > tools/rados/$(DEPDIR)/$(am__dirstamp) +tools/rados/rados.$(OBJEXT): tools/rados/$(am__dirstamp) \ + tools/rados/$(DEPDIR)/$(am__dirstamp) +tools/rados/rados_import.$(OBJEXT): tools/rados/$(am__dirstamp) \ + tools/rados/$(DEPDIR)/$(am__dirstamp) +tools/rados/rados_export.$(OBJEXT): tools/rados/$(am__dirstamp) \ + tools/rados/$(DEPDIR)/$(am__dirstamp) +tools/rados/rados_sync.$(OBJEXT): tools/rados/$(am__dirstamp) \ + tools/rados/$(DEPDIR)/$(am__dirstamp) +common/obj_bencher.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +rados$(EXEEXT): $(rados_OBJECTS) $(rados_DEPENDENCIES) $(EXTRA_rados_DEPENDENCIES) + @rm -f rados$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(rados_OBJECTS) $(rados_LDADD) $(LIBS) +rgw/rgw_resolve.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_swift.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_s3.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_usage.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_user.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_bucket.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_metadata.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_replica_log.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_log.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_opstate.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_replica_log.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_rest_config.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_http_client.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_swift.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_swift_auth.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_loadgen.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_civetweb.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +civetweb/src/$(am__dirstamp): + @$(MKDIR_P) civetweb/src + @: > civetweb/src/$(am__dirstamp) +civetweb/src/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) civetweb/src/$(DEPDIR) + @: > civetweb/src/$(DEPDIR)/$(am__dirstamp) +civetweb/src/radosgw-civetweb.$(OBJEXT): civetweb/src/$(am__dirstamp) \ + civetweb/src/$(DEPDIR)/$(am__dirstamp) +rgw/rgw_main.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +radosgw$(EXEEXT): $(radosgw_OBJECTS) $(radosgw_DEPENDENCIES) $(EXTRA_radosgw_DEPENDENCIES) + @rm -f radosgw$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(radosgw_OBJECTS) $(radosgw_LDADD) $(LIBS) +rgw/rgw_admin.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +radosgw-admin$(EXEEXT): $(radosgw_admin_OBJECTS) $(radosgw_admin_DEPENDENCIES) $(EXTRA_radosgw_admin_DEPENDENCIES) + @rm -f radosgw-admin$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(radosgw_admin_OBJECTS) $(radosgw_admin_LDADD) $(LIBS) +rbd$(EXEEXT): $(rbd_OBJECTS) $(rbd_DEPENDENCIES) $(EXTRA_rbd_DEPENDENCIES) + @rm -f rbd$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(rbd_OBJECTS) $(rbd_LDADD) $(LIBS) +rbd_fuse/$(am__dirstamp): + @$(MKDIR_P) rbd_fuse + @: > rbd_fuse/$(am__dirstamp) +rbd_fuse/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) rbd_fuse/$(DEPDIR) + @: > rbd_fuse/$(DEPDIR)/$(am__dirstamp) +rbd_fuse/rbd-fuse.$(OBJEXT): rbd_fuse/$(am__dirstamp) \ + rbd_fuse/$(DEPDIR)/$(am__dirstamp) +rbd-fuse$(EXEEXT): $(rbd_fuse_OBJECTS) $(rbd_fuse_DEPENDENCIES) $(EXTRA_rbd_fuse_DEPENDENCIES) + @rm -f rbd-fuse$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(rbd_fuse_OBJECTS) $(rbd_fuse_LDADD) $(LIBS) +tools/rest_bench-rest_bench.$(OBJEXT): tools/$(am__dirstamp) \ + tools/$(DEPDIR)/$(am__dirstamp) +common/rest_bench-obj_bencher.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +rest-bench$(EXEEXT): $(rest_bench_OBJECTS) $(rest_bench_DEPENDENCIES) $(EXTRA_rest_bench_DEPENDENCIES) + @rm -f rest-bench$(EXEEXT) + $(AM_V_CXXLD)$(rest_bench_LINK) $(rest_bench_OBJECTS) $(rest_bench_LDADD) $(LIBS) +test/test_build_libcephfs-buildtest_skeleton.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +osdc/test_build_libcephfs-Objecter.$(OBJEXT): osdc/$(am__dirstamp) \ + osdc/$(DEPDIR)/$(am__dirstamp) +osdc/test_build_libcephfs-ObjectCacher.$(OBJEXT): \ + osdc/$(am__dirstamp) osdc/$(DEPDIR)/$(am__dirstamp) +osdc/test_build_libcephfs-Filer.$(OBJEXT): osdc/$(am__dirstamp) \ + osdc/$(DEPDIR)/$(am__dirstamp) +osdc/test_build_libcephfs-Striper.$(OBJEXT): osdc/$(am__dirstamp) \ + osdc/$(DEPDIR)/$(am__dirstamp) +osdc/test_build_libcephfs-Journaler.$(OBJEXT): osdc/$(am__dirstamp) \ + osdc/$(DEPDIR)/$(am__dirstamp) +test_build_libcephfs$(EXEEXT): $(test_build_libcephfs_OBJECTS) $(test_build_libcephfs_DEPENDENCIES) $(EXTRA_test_build_libcephfs_DEPENDENCIES) + @rm -f test_build_libcephfs$(EXEEXT) + $(AM_V_CXXLD)$(test_build_libcephfs_LINK) $(test_build_libcephfs_OBJECTS) $(test_build_libcephfs_LDADD) $(LIBS) +test/test_build_libcommon-buildtest_skeleton.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-DecayCounter.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-LogClient.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-LogEntry.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-PrebufferedStreambuf.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-SloppyCRCMap.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-BackTrace.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-perf_counters.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-Mutex.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-OutputDataSocket.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-admin_socket.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-admin_socket_client.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-cmdparse.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-escape.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-io_priority.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-Clock.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-Throttle.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-Timer.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-Finisher.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-environment.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-assert.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-run_cmd.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-WorkQueue.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ConfUtils.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-MemoryModel.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-armor.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-fd.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-xattr.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-safe_io.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-snap_types.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-str_list.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-str_map.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-errno.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-RefCountedObj.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-blkdev.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-common_init.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-pipe.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_argparse.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_context.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-buffer.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-code_environment.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-dout.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-histogram.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-signal.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-simple_spin.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-Thread.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-Formatter.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-HeartbeatMap.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-config.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-utf8.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-mime.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-strtol.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-page.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-lockdep.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-version.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-hex.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-entity_name.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_crypto.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_crypto_cms.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_json.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ipaddr.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-pick_address.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-util.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-TextTable.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_fs.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_hash.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_strings.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-ceph_frag.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-addr_parsing.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-hobject.$(OBJEXT): common/$(am__dirstamp) \ + common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-bloom_filter.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +common/test_build_libcommon-linux_version.$(OBJEXT): \ + common/$(am__dirstamp) common/$(DEPDIR)/$(am__dirstamp) +mon/test_build_libcommon-MonCap.$(OBJEXT): mon/$(am__dirstamp) \ + mon/$(DEPDIR)/$(am__dirstamp) +mon/test_build_libcommon-MonClient.$(OBJEXT): mon/$(am__dirstamp) \ + mon/$(DEPDIR)/$(am__dirstamp) +mon/test_build_libcommon-MonMap.$(OBJEXT): mon/$(am__dirstamp) \ + mon/$(DEPDIR)/$(am__dirstamp) +osd/test_build_libcommon-OSDMap.$(OBJEXT): osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/test_build_libcommon-osd_types.$(OBJEXT): osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/test_build_libcommon-ECMsgTypes.$(OBJEXT): osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +osd/test_build_libcommon-HitSet.$(OBJEXT): osd/$(am__dirstamp) \ + osd/$(DEPDIR)/$(am__dirstamp) +mds/test_build_libcommon-MDSMap.$(OBJEXT): mds/$(am__dirstamp) \ + mds/$(DEPDIR)/$(am__dirstamp) +mds/test_build_libcommon-inode_backtrace.$(OBJEXT): \ + mds/$(am__dirstamp) mds/$(DEPDIR)/$(am__dirstamp) +mds/test_build_libcommon-mdstypes.$(OBJEXT): mds/$(am__dirstamp) \ + mds/$(DEPDIR)/$(am__dirstamp) +test_build_libcommon$(EXEEXT): $(test_build_libcommon_OBJECTS) $(test_build_libcommon_DEPENDENCIES) $(EXTRA_test_build_libcommon_DEPENDENCIES) + @rm -f test_build_libcommon$(EXEEXT) + $(AM_V_CXXLD)$(test_build_libcommon_LINK) $(test_build_libcommon_OBJECTS) $(test_build_libcommon_LDADD) $(LIBS) +test/test_build_librados-buildtest_skeleton.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +librados/test_build_librados-librados.$(OBJEXT): \ + librados/$(am__dirstamp) librados/$(DEPDIR)/$(am__dirstamp) +librados/test_build_librados-RadosClient.$(OBJEXT): \ + librados/$(am__dirstamp) librados/$(DEPDIR)/$(am__dirstamp) +librados/test_build_librados-IoCtxImpl.$(OBJEXT): \ + librados/$(am__dirstamp) librados/$(DEPDIR)/$(am__dirstamp) +librados/test_build_librados-snap_set_diff.$(OBJEXT): \ + librados/$(am__dirstamp) librados/$(DEPDIR)/$(am__dirstamp) +test_build_librados$(EXEEXT): $(test_build_librados_OBJECTS) $(test_build_librados_DEPENDENCIES) $(EXTRA_test_build_librados_DEPENDENCIES) + @rm -f test_build_librados$(EXEEXT) + $(AM_V_CXXLD)$(test_build_librados_LINK) $(test_build_librados_OBJECTS) $(test_build_librados_LDADD) $(LIBS) +test/test_build_librgw-buildtest_skeleton.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-librgw.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_acl.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_acl_s3.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_acl_swift.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_client_io.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_fcgi.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_xml.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_usage.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_json_enc.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_user.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_bucket.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_tools.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_rados.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_http_client.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_rest_client.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_rest_conn.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_op.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_common.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_cache.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_formats.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_log.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_multi.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_policy_s3.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_gc.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_multi_del.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_env.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_cors.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_cors_s3.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_auth_s3.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_metadata.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_replica_log.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_keystone.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_quota.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +rgw/test_build_librgw-rgw_dencoder.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +test_build_librgw$(EXEEXT): $(test_build_librgw_OBJECTS) $(test_build_librgw_DEPENDENCIES) $(EXTRA_test_build_librgw_DEPENDENCIES) + @rm -f test_build_librgw$(EXEEXT) + $(AM_V_CXXLD)$(test_build_librgw_LINK) $(test_build_librgw_OBJECTS) $(test_build_librgw_LDADD) $(LIBS) +test/unittest_addrs-test_addrs.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_addrs$(EXEEXT): $(unittest_addrs_OBJECTS) $(unittest_addrs_DEPENDENCIES) $(EXTRA_unittest_addrs_DEPENDENCIES) + @rm -f unittest_addrs$(EXEEXT) + $(AM_V_CXXLD)$(unittest_addrs_LINK) $(unittest_addrs_OBJECTS) $(unittest_addrs_LDADD) $(LIBS) +test/unittest_admin_socket-admin_socket.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_admin_socket$(EXEEXT): $(unittest_admin_socket_OBJECTS) $(unittest_admin_socket_DEPENDENCIES) $(EXTRA_unittest_admin_socket_DEPENDENCIES) + @rm -f unittest_admin_socket$(EXEEXT) + $(AM_V_CXXLD)$(unittest_admin_socket_LINK) $(unittest_admin_socket_OBJECTS) $(unittest_admin_socket_LDADD) $(LIBS) +test/unittest_arch-test_arch.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_arch$(EXEEXT): $(unittest_arch_OBJECTS) $(unittest_arch_DEPENDENCIES) $(EXTRA_unittest_arch_DEPENDENCIES) + @rm -f unittest_arch$(EXEEXT) + $(AM_V_CXXLD)$(unittest_arch_LINK) $(unittest_arch_OBJECTS) $(unittest_arch_LDADD) $(LIBS) +test/unittest_base64-base64.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_base64$(EXEEXT): $(unittest_base64_OBJECTS) $(unittest_base64_DEPENDENCIES) $(EXTRA_unittest_base64_DEPENDENCIES) + @rm -f unittest_base64$(EXEEXT) + $(AM_V_CXXLD)$(unittest_base64_LINK) $(unittest_base64_OBJECTS) $(unittest_base64_LDADD) $(LIBS) +test/common/unittest_bloom_filter-test_bloom_filter.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_bloom_filter$(EXEEXT): $(unittest_bloom_filter_OBJECTS) $(unittest_bloom_filter_DEPENDENCIES) $(EXTRA_unittest_bloom_filter_DEPENDENCIES) + @rm -f unittest_bloom_filter$(EXEEXT) + $(AM_V_CXXLD)$(unittest_bloom_filter_LINK) $(unittest_bloom_filter_OBJECTS) $(unittest_bloom_filter_LDADD) $(LIBS) +test/unittest_bufferlist-bufferlist.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_bufferlist$(EXEEXT): $(unittest_bufferlist_OBJECTS) $(unittest_bufferlist_DEPENDENCIES) $(EXTRA_unittest_bufferlist_DEPENDENCIES) + @rm -f unittest_bufferlist$(EXEEXT) + $(AM_V_CXXLD)$(unittest_bufferlist_LINK) $(unittest_bufferlist_OBJECTS) $(unittest_bufferlist_LDADD) $(LIBS) +test/unittest_ceph_argparse-ceph_argparse.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_ceph_argparse$(EXEEXT): $(unittest_ceph_argparse_OBJECTS) $(unittest_ceph_argparse_DEPENDENCIES) $(EXTRA_unittest_ceph_argparse_DEPENDENCIES) + @rm -f unittest_ceph_argparse$(EXEEXT) + $(AM_V_CXXLD)$(unittest_ceph_argparse_LINK) $(unittest_ceph_argparse_OBJECTS) $(unittest_ceph_argparse_LDADD) $(LIBS) +test/unittest_ceph_compatset-ceph_compatset.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_ceph_compatset$(EXEEXT): $(unittest_ceph_compatset_OBJECTS) $(unittest_ceph_compatset_DEPENDENCIES) $(EXTRA_unittest_ceph_compatset_DEPENDENCIES) + @rm -f unittest_ceph_compatset$(EXEEXT) + $(AM_V_CXXLD)$(unittest_ceph_compatset_LINK) $(unittest_ceph_compatset_OBJECTS) $(unittest_ceph_compatset_LDADD) $(LIBS) +test/unittest_ceph_crypto-ceph_crypto.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_ceph_crypto$(EXEEXT): $(unittest_ceph_crypto_OBJECTS) $(unittest_ceph_crypto_DEPENDENCIES) $(EXTRA_unittest_ceph_crypto_DEPENDENCIES) + @rm -f unittest_ceph_crypto$(EXEEXT) + $(AM_V_CXXLD)$(unittest_ceph_crypto_LINK) $(unittest_ceph_crypto_OBJECTS) $(unittest_ceph_crypto_LDADD) $(LIBS) +test/objectstore/unittest_chain_xattr-chain_xattr.$(OBJEXT): \ + test/objectstore/$(am__dirstamp) \ + test/objectstore/$(DEPDIR)/$(am__dirstamp) +unittest_chain_xattr$(EXEEXT): $(unittest_chain_xattr_OBJECTS) $(unittest_chain_xattr_DEPENDENCIES) $(EXTRA_unittest_chain_xattr_DEPENDENCIES) + @rm -f unittest_chain_xattr$(EXEEXT) + $(AM_V_CXXLD)$(unittest_chain_xattr_LINK) $(unittest_chain_xattr_OBJECTS) $(unittest_chain_xattr_LDADD) $(LIBS) +test/common/unittest_config-test_config.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_config$(EXEEXT): $(unittest_config_OBJECTS) $(unittest_config_DEPENDENCIES) $(EXTRA_unittest_config_DEPENDENCIES) + @rm -f unittest_config$(EXEEXT) + $(AM_V_CXXLD)$(unittest_config_LINK) $(unittest_config_OBJECTS) $(unittest_config_LDADD) $(LIBS) +test/unittest_confutils-confutils.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_confutils$(EXEEXT): $(unittest_confutils_OBJECTS) $(unittest_confutils_DEPENDENCIES) $(EXTRA_unittest_confutils_DEPENDENCIES) + @rm -f unittest_confutils$(EXEEXT) + $(AM_V_CXXLD)$(unittest_confutils_LINK) $(unittest_confutils_OBJECTS) $(unittest_confutils_LDADD) $(LIBS) +test/common/unittest_context-test_context.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_context$(EXEEXT): $(unittest_context_OBJECTS) $(unittest_context_DEPENDENCIES) $(EXTRA_unittest_context_DEPENDENCIES) + @rm -f unittest_context$(EXEEXT) + $(AM_V_CXXLD)$(unittest_context_LINK) $(unittest_context_OBJECTS) $(unittest_context_LDADD) $(LIBS) +test/common/unittest_crc32c-test_crc32c.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_crc32c$(EXEEXT): $(unittest_crc32c_OBJECTS) $(unittest_crc32c_DEPENDENCIES) $(EXTRA_unittest_crc32c_DEPENDENCIES) + @rm -f unittest_crc32c$(EXEEXT) + $(AM_V_CXXLD)$(unittest_crc32c_LINK) $(unittest_crc32c_OBJECTS) $(unittest_crc32c_LDADD) $(LIBS) +test/crush/$(am__dirstamp): + @$(MKDIR_P) test/crush + @: > test/crush/$(am__dirstamp) +test/crush/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/crush/$(DEPDIR) + @: > test/crush/$(DEPDIR)/$(am__dirstamp) +test/crush/unittest_crush_indep-indep.$(OBJEXT): \ + test/crush/$(am__dirstamp) \ + test/crush/$(DEPDIR)/$(am__dirstamp) +unittest_crush_indep$(EXEEXT): $(unittest_crush_indep_OBJECTS) $(unittest_crush_indep_DEPENDENCIES) $(EXTRA_unittest_crush_indep_DEPENDENCIES) + @rm -f unittest_crush_indep$(EXEEXT) + $(AM_V_CXXLD)$(unittest_crush_indep_LINK) $(unittest_crush_indep_OBJECTS) $(unittest_crush_indep_LDADD) $(LIBS) +test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT): \ + test/crush/$(am__dirstamp) \ + test/crush/$(DEPDIR)/$(am__dirstamp) +unittest_crush_wrapper$(EXEEXT): $(unittest_crush_wrapper_OBJECTS) $(unittest_crush_wrapper_DEPENDENCIES) $(EXTRA_unittest_crush_wrapper_DEPENDENCIES) + @rm -f unittest_crush_wrapper$(EXEEXT) + $(AM_V_CXXLD)$(unittest_crush_wrapper_LINK) $(unittest_crush_wrapper_OBJECTS) $(unittest_crush_wrapper_LDADD) $(LIBS) +test/unittest_crypto-crypto.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_crypto$(EXEEXT): $(unittest_crypto_OBJECTS) $(unittest_crypto_DEPENDENCIES) $(EXTRA_unittest_crypto_DEPENDENCIES) + @rm -f unittest_crypto$(EXEEXT) + $(AM_V_CXXLD)$(unittest_crypto_LINK) $(unittest_crypto_OBJECTS) $(unittest_crypto_LDADD) $(LIBS) +test/unittest_crypto_init-crypto_init.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_crypto_init$(EXEEXT): $(unittest_crypto_init_OBJECTS) $(unittest_crypto_init_DEPENDENCIES) $(EXTRA_unittest_crypto_init_DEPENDENCIES) + @rm -f unittest_crypto_init$(EXEEXT) + $(AM_V_CXXLD)$(unittest_crypto_init_LINK) $(unittest_crypto_init_OBJECTS) $(unittest_crypto_init_LDADD) $(LIBS) +test/unittest_daemon_config-daemon_config.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_daemon_config$(EXEEXT): $(unittest_daemon_config_OBJECTS) $(unittest_daemon_config_DEPENDENCIES) $(EXTRA_unittest_daemon_config_DEPENDENCIES) + @rm -f unittest_daemon_config$(EXEEXT) + $(AM_V_CXXLD)$(unittest_daemon_config_LINK) $(unittest_daemon_config_OBJECTS) $(unittest_daemon_config_LDADD) $(LIBS) +test/osd/unittest_ecbackend-TestECBackend.$(OBJEXT): \ + test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp) +unittest_ecbackend$(EXEEXT): $(unittest_ecbackend_OBJECTS) $(unittest_ecbackend_DEPENDENCIES) $(EXTRA_unittest_ecbackend_DEPENDENCIES) + @rm -f unittest_ecbackend$(EXEEXT) + $(AM_V_CXXLD)$(unittest_ecbackend_LINK) $(unittest_ecbackend_OBJECTS) $(unittest_ecbackend_LDADD) $(LIBS) +test/unittest_encoding-encoding.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_encoding$(EXEEXT): $(unittest_encoding_OBJECTS) $(unittest_encoding_DEPENDENCIES) $(EXTRA_unittest_encoding_DEPENDENCIES) + @rm -f unittest_encoding$(EXEEXT) + $(AM_V_CXXLD)$(unittest_encoding_LINK) $(unittest_encoding_OBJECTS) $(unittest_encoding_LDADD) $(LIBS) +test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT): \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +unittest_erasure_code_example$(EXEEXT): $(unittest_erasure_code_example_OBJECTS) $(unittest_erasure_code_example_DEPENDENCIES) $(EXTRA_unittest_erasure_code_example_DEPENDENCIES) + @rm -f unittest_erasure_code_example$(EXEEXT) + $(AM_V_CXXLD)$(unittest_erasure_code_example_LINK) $(unittest_erasure_code_example_OBJECTS) $(unittest_erasure_code_example_LDADD) $(LIBS) +test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT): \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.$(OBJEXT): \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.$(OBJEXT): \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.$(OBJEXT): \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.$(OBJEXT): \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.$(OBJEXT): \ + erasure-code/jerasure/jerasure/src/$(am__dirstamp) \ + erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.$(OBJEXT): \ + erasure-code/jerasure/gf-complete/src/$(am__dirstamp) \ + erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT): \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT): \ + erasure-code/jerasure/$(am__dirstamp) \ + erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) +unittest_erasure_code_jerasure$(EXEEXT): $(unittest_erasure_code_jerasure_OBJECTS) $(unittest_erasure_code_jerasure_DEPENDENCIES) $(EXTRA_unittest_erasure_code_jerasure_DEPENDENCIES) + @rm -f unittest_erasure_code_jerasure$(EXEEXT) + $(AM_V_CXXLD)$(unittest_erasure_code_jerasure_LINK) $(unittest_erasure_code_jerasure_OBJECTS) $(unittest_erasure_code_jerasure_LDADD) $(LIBS) +test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT): \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +unittest_erasure_code_plugin$(EXEEXT): $(unittest_erasure_code_plugin_OBJECTS) $(unittest_erasure_code_plugin_DEPENDENCIES) $(EXTRA_unittest_erasure_code_plugin_DEPENDENCIES) + @rm -f unittest_erasure_code_plugin$(EXEEXT) + $(AM_V_CXXLD)$(unittest_erasure_code_plugin_LINK) $(unittest_erasure_code_plugin_OBJECTS) $(unittest_erasure_code_plugin_LDADD) $(LIBS) +test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT): \ + test/erasure-code/$(am__dirstamp) \ + test/erasure-code/$(DEPDIR)/$(am__dirstamp) +unittest_erasure_code_plugin_jerasure$(EXEEXT): $(unittest_erasure_code_plugin_jerasure_OBJECTS) $(unittest_erasure_code_plugin_jerasure_DEPENDENCIES) $(EXTRA_unittest_erasure_code_plugin_jerasure_DEPENDENCIES) + @rm -f unittest_erasure_code_plugin_jerasure$(EXEEXT) + $(AM_V_CXXLD)$(unittest_erasure_code_plugin_jerasure_LINK) $(unittest_erasure_code_plugin_jerasure_OBJECTS) $(unittest_erasure_code_plugin_jerasure_LDADD) $(LIBS) +test/unittest_escape-escape.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_escape$(EXEEXT): $(unittest_escape_OBJECTS) $(unittest_escape_DEPENDENCIES) $(EXTRA_unittest_escape_DEPENDENCIES) + @rm -f unittest_escape$(EXEEXT) + $(AM_V_CXXLD)$(unittest_escape_LINK) $(unittest_escape_OBJECTS) $(unittest_escape_LDADD) $(LIBS) +test/os/$(am__dirstamp): + @$(MKDIR_P) test/os + @: > test/os/$(am__dirstamp) +test/os/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) test/os/$(DEPDIR) + @: > test/os/$(DEPDIR)/$(am__dirstamp) +test/os/unittest_flatindex-TestFlatIndex.$(OBJEXT): \ + test/os/$(am__dirstamp) test/os/$(DEPDIR)/$(am__dirstamp) +unittest_flatindex$(EXEEXT): $(unittest_flatindex_OBJECTS) $(unittest_flatindex_DEPENDENCIES) $(EXTRA_unittest_flatindex_DEPENDENCIES) + @rm -f unittest_flatindex$(EXEEXT) + $(AM_V_CXXLD)$(unittest_flatindex_LINK) $(unittest_flatindex_OBJECTS) $(unittest_flatindex_LDADD) $(LIBS) +test/unittest_formatter-formatter.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +rgw/unittest_formatter-rgw_formats.$(OBJEXT): rgw/$(am__dirstamp) \ + rgw/$(DEPDIR)/$(am__dirstamp) +unittest_formatter$(EXEEXT): $(unittest_formatter_OBJECTS) $(unittest_formatter_DEPENDENCIES) $(EXTRA_unittest_formatter_DEPENDENCIES) + @rm -f unittest_formatter$(EXEEXT) + $(AM_V_CXXLD)$(unittest_formatter_LINK) $(unittest_formatter_OBJECTS) $(unittest_formatter_LDADD) $(LIBS) +test/unittest_gather-gather.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_gather$(EXEEXT): $(unittest_gather_OBJECTS) $(unittest_gather_DEPENDENCIES) $(EXTRA_unittest_gather_DEPENDENCIES) + @rm -f unittest_gather$(EXEEXT) + $(AM_V_CXXLD)$(unittest_gather_LINK) $(unittest_gather_OBJECTS) $(unittest_gather_LDADD) $(LIBS) +test/unittest_heartbeatmap-heartbeat_map.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_heartbeatmap$(EXEEXT): $(unittest_heartbeatmap_OBJECTS) $(unittest_heartbeatmap_DEPENDENCIES) $(EXTRA_unittest_heartbeatmap_DEPENDENCIES) + @rm -f unittest_heartbeatmap$(EXEEXT) + $(AM_V_CXXLD)$(unittest_heartbeatmap_LINK) $(unittest_heartbeatmap_OBJECTS) $(unittest_heartbeatmap_LDADD) $(LIBS) +test/common/unittest_histogram-histogram.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_histogram$(EXEEXT): $(unittest_histogram_OBJECTS) $(unittest_histogram_DEPENDENCIES) $(EXTRA_unittest_histogram_DEPENDENCIES) + @rm -f unittest_histogram$(EXEEXT) + $(AM_V_CXXLD)$(unittest_histogram_LINK) $(unittest_histogram_OBJECTS) $(unittest_histogram_LDADD) $(LIBS) +test/osd/unittest_hitset-hitset.$(OBJEXT): test/osd/$(am__dirstamp) \ + test/osd/$(DEPDIR)/$(am__dirstamp) +unittest_hitset$(EXEEXT): $(unittest_hitset_OBJECTS) $(unittest_hitset_DEPENDENCIES) $(EXTRA_unittest_hitset_DEPENDENCIES) + @rm -f unittest_hitset$(EXEEXT) + $(AM_V_CXXLD)$(unittest_hitset_LINK) $(unittest_hitset_OBJECTS) $(unittest_hitset_LDADD) $(LIBS) +test/unittest_ipaddr-test_ipaddr.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_ipaddr$(EXEEXT): $(unittest_ipaddr_OBJECTS) $(unittest_ipaddr_DEPENDENCIES) $(EXTRA_unittest_ipaddr_DEPENDENCIES) + @rm -f unittest_ipaddr$(EXEEXT) + $(AM_V_CXXLD)$(unittest_ipaddr_LINK) $(unittest_ipaddr_OBJECTS) $(unittest_ipaddr_LDADD) $(LIBS) +test/os/unittest_lfnindex-TestLFNIndex.$(OBJEXT): \ + test/os/$(am__dirstamp) test/os/$(DEPDIR)/$(am__dirstamp) +unittest_lfnindex$(EXEEXT): $(unittest_lfnindex_OBJECTS) $(unittest_lfnindex_DEPENDENCIES) $(EXTRA_unittest_lfnindex_DEPENDENCIES) + @rm -f unittest_lfnindex$(EXEEXT) + $(AM_V_CXXLD)$(unittest_lfnindex_LINK) $(unittest_lfnindex_OBJECTS) $(unittest_lfnindex_LDADD) $(LIBS) +test/unittest_libcephfs_config-libcephfs_config.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_libcephfs_config$(EXEEXT): $(unittest_libcephfs_config_OBJECTS) $(unittest_libcephfs_config_DEPENDENCIES) $(EXTRA_unittest_libcephfs_config_DEPENDENCIES) + @rm -f unittest_libcephfs_config$(EXEEXT) + $(AM_V_CXXLD)$(unittest_libcephfs_config_LINK) $(unittest_libcephfs_config_OBJECTS) $(unittest_libcephfs_config_LDADD) $(LIBS) +test/librados/unittest_librados-librados.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +unittest_librados$(EXEEXT): $(unittest_librados_OBJECTS) $(unittest_librados_DEPENDENCIES) $(EXTRA_unittest_librados_DEPENDENCIES) + @rm -f unittest_librados$(EXEEXT) + $(AM_V_CXXLD)$(unittest_librados_LINK) $(unittest_librados_OBJECTS) $(unittest_librados_LDADD) $(LIBS) +test/librados/unittest_librados_config-librados_config.$(OBJEXT): \ + test/librados/$(am__dirstamp) \ + test/librados/$(DEPDIR)/$(am__dirstamp) +unittest_librados_config$(EXEEXT): $(unittest_librados_config_OBJECTS) $(unittest_librados_config_DEPENDENCIES) $(EXTRA_unittest_librados_config_DEPENDENCIES) + @rm -f unittest_librados_config$(EXEEXT) + $(AM_V_CXXLD)$(unittest_librados_config_LINK) $(unittest_librados_config_OBJECTS) $(unittest_librados_config_LDADD) $(LIBS) +log/unittest_log-test.$(OBJEXT): log/$(am__dirstamp) \ + log/$(DEPDIR)/$(am__dirstamp) +unittest_log$(EXEEXT): $(unittest_log_OBJECTS) $(unittest_log_DEPENDENCIES) $(EXTRA_unittest_log_DEPENDENCIES) + @rm -f unittest_log$(EXEEXT) + $(AM_V_CXXLD)$(unittest_log_LINK) $(unittest_log_OBJECTS) $(unittest_log_LDADD) $(LIBS) +test/unittest_mime-mime.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_mime$(EXEEXT): $(unittest_mime_OBJECTS) $(unittest_mime_DEPENDENCIES) $(EXTRA_unittest_mime_DEPENDENCIES) + @rm -f unittest_mime$(EXEEXT) + $(AM_V_CXXLD)$(unittest_mime_LINK) $(unittest_mime_OBJECTS) $(unittest_mime_LDADD) $(LIBS) +test/mon/unittest_mon_moncap-moncap.$(OBJEXT): \ + test/mon/$(am__dirstamp) test/mon/$(DEPDIR)/$(am__dirstamp) +unittest_mon_moncap$(EXEEXT): $(unittest_mon_moncap_OBJECTS) $(unittest_mon_moncap_DEPENDENCIES) $(EXTRA_unittest_mon_moncap_DEPENDENCIES) + @rm -f unittest_mon_moncap$(EXEEXT) + $(AM_V_CXXLD)$(unittest_mon_moncap_LINK) $(unittest_mon_moncap_OBJECTS) $(unittest_mon_moncap_LDADD) $(LIBS) +test/mon/unittest_mon_pgmap-PGMap.$(OBJEXT): test/mon/$(am__dirstamp) \ + test/mon/$(DEPDIR)/$(am__dirstamp) +unittest_mon_pgmap$(EXEEXT): $(unittest_mon_pgmap_OBJECTS) $(unittest_mon_pgmap_DEPENDENCIES) $(EXTRA_unittest_mon_pgmap_DEPENDENCIES) + @rm -f unittest_mon_pgmap$(EXEEXT) + $(AM_V_CXXLD)$(unittest_mon_pgmap_LINK) $(unittest_mon_pgmap_OBJECTS) $(unittest_mon_pgmap_LDADD) $(LIBS) +test/on_exit.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_on_exit$(EXEEXT): $(unittest_on_exit_OBJECTS) $(unittest_on_exit_DEPENDENCIES) $(EXTRA_unittest_on_exit_DEPENDENCIES) + @rm -f unittest_on_exit$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(unittest_on_exit_OBJECTS) $(unittest_on_exit_LDADD) $(LIBS) +test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT): \ + test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp) +unittest_osd_osdcap$(EXEEXT): $(unittest_osd_osdcap_OBJECTS) $(unittest_osd_osdcap_DEPENDENCIES) $(EXTRA_unittest_osd_osdcap_DEPENDENCIES) + @rm -f unittest_osd_osdcap$(EXEEXT) + $(AM_V_CXXLD)$(unittest_osd_osdcap_LINK) $(unittest_osd_osdcap_OBJECTS) $(unittest_osd_osdcap_LDADD) $(LIBS) +test/osd/unittest_osd_types-types.$(OBJEXT): test/osd/$(am__dirstamp) \ + test/osd/$(DEPDIR)/$(am__dirstamp) +unittest_osd_types$(EXEEXT): $(unittest_osd_types_OBJECTS) $(unittest_osd_types_DEPENDENCIES) $(EXTRA_unittest_osd_types_DEPENDENCIES) + @rm -f unittest_osd_types$(EXEEXT) + $(AM_V_CXXLD)$(unittest_osd_types_LINK) $(unittest_osd_types_OBJECTS) $(unittest_osd_types_LDADD) $(LIBS) +test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT): \ + test/osd/$(am__dirstamp) test/osd/$(DEPDIR)/$(am__dirstamp) +unittest_osdmap$(EXEEXT): $(unittest_osdmap_OBJECTS) $(unittest_osdmap_DEPENDENCIES) $(EXTRA_unittest_osdmap_DEPENDENCIES) + @rm -f unittest_osdmap$(EXEEXT) + $(AM_V_CXXLD)$(unittest_osdmap_LINK) $(unittest_osdmap_OBJECTS) $(unittest_osdmap_LDADD) $(LIBS) +test/unittest_perf_counters-perf_counters.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_perf_counters$(EXEEXT): $(unittest_perf_counters_OBJECTS) $(unittest_perf_counters_DEPENDENCIES) $(EXTRA_unittest_perf_counters_DEPENDENCIES) + @rm -f unittest_perf_counters$(EXEEXT) + $(AM_V_CXXLD)$(unittest_perf_counters_LINK) $(unittest_perf_counters_OBJECTS) $(unittest_perf_counters_LDADD) $(LIBS) +test/osd/unittest_pglog-TestPGLog.$(OBJEXT): test/osd/$(am__dirstamp) \ + test/osd/$(DEPDIR)/$(am__dirstamp) +unittest_pglog$(EXEEXT): $(unittest_pglog_OBJECTS) $(unittest_pglog_DEPENDENCIES) $(EXTRA_unittest_pglog_DEPENDENCIES) + @rm -f unittest_pglog$(EXEEXT) + $(AM_V_CXXLD)$(unittest_pglog_LINK) $(unittest_pglog_OBJECTS) $(unittest_pglog_LDADD) $(LIBS) +test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_prebufferedstreambuf$(EXEEXT): $(unittest_prebufferedstreambuf_OBJECTS) $(unittest_prebufferedstreambuf_DEPENDENCIES) $(EXTRA_unittest_prebufferedstreambuf_DEPENDENCIES) + @rm -f unittest_prebufferedstreambuf$(EXEEXT) + $(AM_V_CXXLD)$(unittest_prebufferedstreambuf_LINK) $(unittest_prebufferedstreambuf_OBJECTS) $(unittest_prebufferedstreambuf_LDADD) $(LIBS) +test/unittest_run_cmd-run_cmd.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_run_cmd$(EXEEXT): $(unittest_run_cmd_OBJECTS) $(unittest_run_cmd_DEPENDENCIES) $(EXTRA_unittest_run_cmd_DEPENDENCIES) + @rm -f unittest_run_cmd$(EXEEXT) + $(AM_V_CXXLD)$(unittest_run_cmd_LINK) $(unittest_run_cmd_OBJECTS) $(unittest_run_cmd_LDADD) $(LIBS) +test/common/unittest_sharedptr_registry-test_sharedptr_registry.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_sharedptr_registry$(EXEEXT): $(unittest_sharedptr_registry_OBJECTS) $(unittest_sharedptr_registry_DEPENDENCIES) $(EXTRA_unittest_sharedptr_registry_DEPENDENCIES) + @rm -f unittest_sharedptr_registry$(EXEEXT) + $(AM_V_CXXLD)$(unittest_sharedptr_registry_LINK) $(unittest_sharedptr_registry_OBJECTS) $(unittest_sharedptr_registry_LDADD) $(LIBS) +test/unittest_signals-signals.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_signals$(EXEEXT): $(unittest_signals_OBJECTS) $(unittest_signals_DEPENDENCIES) $(EXTRA_unittest_signals_DEPENDENCIES) + @rm -f unittest_signals$(EXEEXT) + $(AM_V_CXXLD)$(unittest_signals_LINK) $(unittest_signals_OBJECTS) $(unittest_signals_LDADD) $(LIBS) +test/unittest_simple_spin-simple_spin.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_simple_spin$(EXEEXT): $(unittest_simple_spin_OBJECTS) $(unittest_simple_spin_DEPENDENCIES) $(EXTRA_unittest_simple_spin_DEPENDENCIES) + @rm -f unittest_simple_spin$(EXEEXT) + $(AM_V_CXXLD)$(unittest_simple_spin_LINK) $(unittest_simple_spin_OBJECTS) $(unittest_simple_spin_LDADD) $(LIBS) +test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_sloppy_crc_map$(EXEEXT): $(unittest_sloppy_crc_map_OBJECTS) $(unittest_sloppy_crc_map_DEPENDENCIES) $(EXTRA_unittest_sloppy_crc_map_DEPENDENCIES) + @rm -f unittest_sloppy_crc_map$(EXEEXT) + $(AM_V_CXXLD)$(unittest_sloppy_crc_map_LINK) $(unittest_sloppy_crc_map_OBJECTS) $(unittest_sloppy_crc_map_LDADD) $(LIBS) +test/unittest_str_list-test_str_list.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_str_list$(EXEEXT): $(unittest_str_list_OBJECTS) $(unittest_str_list_DEPENDENCIES) $(EXTRA_unittest_str_list_DEPENDENCIES) + @rm -f unittest_str_list$(EXEEXT) + $(AM_V_CXXLD)$(unittest_str_list_LINK) $(unittest_str_list_OBJECTS) $(unittest_str_list_LDADD) $(LIBS) +test/common/unittest_str_map-test_str_map.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_str_map$(EXEEXT): $(unittest_str_map_OBJECTS) $(unittest_str_map_DEPENDENCIES) $(EXTRA_unittest_str_map_DEPENDENCIES) + @rm -f unittest_str_map$(EXEEXT) + $(AM_V_CXXLD)$(unittest_str_map_LINK) $(unittest_str_map_OBJECTS) $(unittest_str_map_LDADD) $(LIBS) +test/unittest_striper-test_striper.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_striper$(EXEEXT): $(unittest_striper_OBJECTS) $(unittest_striper_DEPENDENCIES) $(EXTRA_unittest_striper_DEPENDENCIES) + @rm -f unittest_striper$(EXEEXT) + $(AM_V_CXXLD)$(unittest_striper_LINK) $(unittest_striper_OBJECTS) $(unittest_striper_LDADD) $(LIBS) +test/unittest_strtol-strtol.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_strtol$(EXEEXT): $(unittest_strtol_OBJECTS) $(unittest_strtol_DEPENDENCIES) $(EXTRA_unittest_strtol_DEPENDENCIES) + @rm -f unittest_strtol$(EXEEXT) + $(AM_V_CXXLD)$(unittest_strtol_LINK) $(unittest_strtol_OBJECTS) $(unittest_strtol_LDADD) $(LIBS) +test/unittest_texttable-test_texttable.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_texttable$(EXEEXT): $(unittest_texttable_OBJECTS) $(unittest_texttable_DEPENDENCIES) $(EXTRA_unittest_texttable_DEPENDENCIES) + @rm -f unittest_texttable$(EXEEXT) + $(AM_V_CXXLD)$(unittest_texttable_LINK) $(unittest_texttable_OBJECTS) $(unittest_texttable_LDADD) $(LIBS) +test/common/unittest_throttle-Throttle.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_throttle$(EXEEXT): $(unittest_throttle_OBJECTS) $(unittest_throttle_DEPENDENCIES) $(EXTRA_unittest_throttle_DEPENDENCIES) + @rm -f unittest_throttle$(EXEEXT) + $(AM_V_CXXLD)$(unittest_throttle_LINK) $(unittest_throttle_OBJECTS) $(unittest_throttle_LDADD) $(LIBS) +test/unittest_utf8-utf8.$(OBJEXT): test/$(am__dirstamp) \ + test/$(DEPDIR)/$(am__dirstamp) +unittest_utf8$(EXEEXT): $(unittest_utf8_OBJECTS) $(unittest_utf8_DEPENDENCIES) $(EXTRA_unittest_utf8_DEPENDENCIES) + @rm -f unittest_utf8$(EXEEXT) + $(AM_V_CXXLD)$(unittest_utf8_LINK) $(unittest_utf8_OBJECTS) $(unittest_utf8_LDADD) $(LIBS) +test/common/unittest_util-test_util.$(OBJEXT): \ + test/common/$(am__dirstamp) \ + test/common/$(DEPDIR)/$(am__dirstamp) +unittest_util$(EXEEXT): $(unittest_util_OBJECTS) $(unittest_util_DEPENDENCIES) $(EXTRA_unittest_util_DEPENDENCIES) + @rm -f unittest_util$(EXEEXT) + $(AM_V_CXXLD)$(unittest_util_LINK) $(unittest_util_OBJECTS) $(unittest_util_LDADD) $(LIBS) +test/unittest_workqueue-test_workqueue.$(OBJEXT): \ + test/$(am__dirstamp) test/$(DEPDIR)/$(am__dirstamp) +unittest_workqueue$(EXEEXT): $(unittest_workqueue_OBJECTS) $(unittest_workqueue_DEPENDENCIES) $(EXTRA_unittest_workqueue_DEPENDENCIES) + @rm -f unittest_workqueue$(EXEEXT) + $(AM_V_CXXLD)$(unittest_workqueue_LINK) $(unittest_workqueue_OBJECTS) $(unittest_workqueue_LDADD) $(LIBS) +install-binSCRIPTS: $(bin_SCRIPTS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(bin_SCRIPTS)'; test -n "$(bindir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n' \ + -e 'h;s|.*|.|' \ + -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) { files[d] = files[d] " " $$1; \ + if (++n[d] == $(am__install_max)) { \ + print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ + else { print "f", d "/" $$4, $$1 } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binSCRIPTS: + @$(NORMAL_UNINSTALL) + @list='$(bin_SCRIPTS)'; test -n "$(bindir)" || exit 0; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 's,.*/,,;$(transform)'`; \ + dir='$(DESTDIR)$(bindir)'; $(am__uninstall_files_from_dir) +install-ceph_sbinSCRIPTS: $(ceph_sbin_SCRIPTS) + @$(NORMAL_INSTALL) + test -z "$(ceph_sbindir)" || $(MKDIR_P) "$(DESTDIR)$(ceph_sbindir)" + @list='$(ceph_sbin_SCRIPTS)'; test -n "$(ceph_sbindir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n' \ + -e 'h;s|.*|.|' \ + -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) { files[d] = files[d] " " $$1; \ + if (++n[d] == $(am__install_max)) { \ + print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ + else { print "f", d "/" $$4, $$1 } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(ceph_sbindir)$$dir'"; \ + $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(ceph_sbindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-ceph_sbinSCRIPTS: + @$(NORMAL_UNINSTALL) + @list='$(ceph_sbin_SCRIPTS)'; test -n "$(ceph_sbindir)" || exit 0; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 's,.*/,,;$(transform)'`; \ + dir='$(DESTDIR)$(ceph_sbindir)'; $(am__uninstall_files_from_dir) +install-dist_binSCRIPTS: $(dist_bin_SCRIPTS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(dist_bin_SCRIPTS)'; test -n "$(bindir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n' \ + -e 'h;s|.*|.|' \ + -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) { files[d] = files[d] " " $$1; \ + if (++n[d] == $(am__install_max)) { \ + print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ + else { print "f", d "/" $$4, $$1 } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-dist_binSCRIPTS: + @$(NORMAL_UNINSTALL) + @list='$(dist_bin_SCRIPTS)'; test -n "$(bindir)" || exit 0; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 's,.*/,,;$(transform)'`; \ + dir='$(DESTDIR)$(bindir)'; $(am__uninstall_files_from_dir) +install-sbinSCRIPTS: $(sbin_SCRIPTS) + @$(NORMAL_INSTALL) + test -z "$(sbindir)" || $(MKDIR_P) "$(DESTDIR)$(sbindir)" + @list='$(sbin_SCRIPTS)'; test -n "$(sbindir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n' \ + -e 'h;s|.*|.|' \ + -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) { files[d] = files[d] " " $$1; \ + if (++n[d] == $(am__install_max)) { \ + print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ + else { print "f", d "/" $$4, $$1 } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(sbindir)$$dir'"; \ + $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(sbindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-sbinSCRIPTS: + @$(NORMAL_UNINSTALL) + @list='$(sbin_SCRIPTS)'; test -n "$(sbindir)" || exit 0; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 's,.*/,,;$(transform)'`; \ + dir='$(DESTDIR)$(sbindir)'; $(am__uninstall_files_from_dir) +install-shell_commonSCRIPTS: $(shell_common_SCRIPTS) + @$(NORMAL_INSTALL) + test -z "$(shell_commondir)" || $(MKDIR_P) "$(DESTDIR)$(shell_commondir)" + @list='$(shell_common_SCRIPTS)'; test -n "$(shell_commondir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n' \ + -e 'h;s|.*|.|' \ + -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) { files[d] = files[d] " " $$1; \ + if (++n[d] == $(am__install_max)) { \ + print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ + else { print "f", d "/" $$4, $$1 } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(shell_commondir)$$dir'"; \ + $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(shell_commondir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-shell_commonSCRIPTS: + @$(NORMAL_UNINSTALL) + @list='$(shell_common_SCRIPTS)'; test -n "$(shell_commondir)" || exit 0; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 's,.*/,,;$(transform)'`; \ + dir='$(DESTDIR)$(shell_commondir)'; $(am__uninstall_files_from_dir) +install-su_sbinSCRIPTS: $(su_sbin_SCRIPTS) + @$(NORMAL_INSTALL) + test -z "$(su_sbindir)" || $(MKDIR_P) "$(DESTDIR)$(su_sbindir)" + @list='$(su_sbin_SCRIPTS)'; test -n "$(su_sbindir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n' \ + -e 'h;s|.*|.|' \ + -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) { files[d] = files[d] " " $$1; \ + if (++n[d] == $(am__install_max)) { \ + print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ + else { print "f", d "/" $$4, $$1 } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(su_sbindir)$$dir'"; \ + $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(su_sbindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-su_sbinSCRIPTS: + @$(NORMAL_UNINSTALL) + @list='$(su_sbin_SCRIPTS)'; test -n "$(su_sbindir)" || exit 0; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 's,.*/,,;$(transform)'`; \ + dir='$(DESTDIR)$(su_sbindir)'; $(am__uninstall_files_from_dir) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + -rm -f arch/intel.$(OBJEXT) + -rm -f arch/intel.lo + -rm -f arch/neon.$(OBJEXT) + -rm -f arch/neon.lo + -rm -f arch/probe.$(OBJEXT) + -rm -f arch/probe.lo + -rm -f auth/AuthAuthorizeHandler.$(OBJEXT) + -rm -f auth/AuthAuthorizeHandler.lo + -rm -f auth/AuthClientHandler.$(OBJEXT) + -rm -f auth/AuthClientHandler.lo + -rm -f auth/AuthMethodList.$(OBJEXT) + -rm -f auth/AuthMethodList.lo + -rm -f auth/AuthServiceHandler.$(OBJEXT) + -rm -f auth/AuthServiceHandler.lo + -rm -f auth/AuthSessionHandler.$(OBJEXT) + -rm -f auth/AuthSessionHandler.lo + -rm -f auth/Crypto.$(OBJEXT) + -rm -f auth/Crypto.lo + -rm -f auth/KeyRing.$(OBJEXT) + -rm -f auth/KeyRing.lo + -rm -f auth/RotatingKeyRing.$(OBJEXT) + -rm -f auth/RotatingKeyRing.lo + -rm -f auth/cephx/CephxAuthorizeHandler.$(OBJEXT) + -rm -f auth/cephx/CephxAuthorizeHandler.lo + -rm -f auth/cephx/CephxClientHandler.$(OBJEXT) + -rm -f auth/cephx/CephxClientHandler.lo + -rm -f auth/cephx/CephxKeyServer.$(OBJEXT) + -rm -f auth/cephx/CephxKeyServer.lo + -rm -f auth/cephx/CephxProtocol.$(OBJEXT) + -rm -f auth/cephx/CephxProtocol.lo + -rm -f auth/cephx/CephxServiceHandler.$(OBJEXT) + -rm -f auth/cephx/CephxServiceHandler.lo + -rm -f auth/cephx/CephxSessionHandler.$(OBJEXT) + -rm -f auth/cephx/CephxSessionHandler.lo + -rm -f auth/none/AuthNoneAuthorizeHandler.$(OBJEXT) + -rm -f auth/none/AuthNoneAuthorizeHandler.lo + -rm -f auth/unknown/AuthUnknownAuthorizeHandler.$(OBJEXT) + -rm -f auth/unknown/AuthUnknownAuthorizeHandler.lo + -rm -f civetweb/src/radosgw-civetweb.$(OBJEXT) + -rm -f client/Client.$(OBJEXT) + -rm -f client/Client.lo + -rm -f client/ClientSnapRealm.$(OBJEXT) + -rm -f client/ClientSnapRealm.lo + -rm -f client/Dentry.$(OBJEXT) + -rm -f client/Dentry.lo + -rm -f client/Inode.$(OBJEXT) + -rm -f client/Inode.lo + -rm -f client/MetaRequest.$(OBJEXT) + -rm -f client/MetaRequest.lo + -rm -f client/MetaSession.$(OBJEXT) + -rm -f client/MetaSession.lo + -rm -f client/SyntheticClient.$(OBJEXT) + -rm -f client/Trace.$(OBJEXT) + -rm -f client/Trace.lo + -rm -f client/fuse_ll.$(OBJEXT) + -rm -f client/fuse_ll.lo + -rm -f client/test_ioctls.$(OBJEXT) + -rm -f cls/hello/cls_hello.$(OBJEXT) + -rm -f cls/hello/cls_hello.lo + -rm -f cls/lock/cls_lock.$(OBJEXT) + -rm -f cls/lock/cls_lock.lo + -rm -f cls/lock/cls_lock_client.$(OBJEXT) + -rm -f cls/lock/cls_lock_client.lo + -rm -f cls/lock/cls_lock_ops.$(OBJEXT) + -rm -f cls/lock/cls_lock_ops.lo + -rm -f cls/lock/cls_lock_types.$(OBJEXT) + -rm -f cls/lock/cls_lock_types.lo + -rm -f cls/log/cls_log.$(OBJEXT) + -rm -f cls/log/cls_log.lo + -rm -f cls/log/cls_log_client.$(OBJEXT) + -rm -f cls/rbd/cls_rbd.$(OBJEXT) + -rm -f cls/rbd/cls_rbd.lo + -rm -f cls/rbd/cls_rbd_client.$(OBJEXT) + -rm -f cls/rbd/cls_rbd_client.lo + -rm -f cls/refcount/cls_refcount.$(OBJEXT) + -rm -f cls/refcount/cls_refcount.lo + -rm -f cls/refcount/cls_refcount_client.$(OBJEXT) + -rm -f cls/refcount/cls_refcount_client.lo + -rm -f cls/refcount/cls_refcount_ops.$(OBJEXT) + -rm -f cls/refcount/cls_refcount_ops.lo + -rm -f cls/replica_log/cls_replica_log.$(OBJEXT) + -rm -f cls/replica_log/cls_replica_log.lo + -rm -f cls/replica_log/cls_replica_log_client.$(OBJEXT) + -rm -f cls/replica_log/cls_replica_log_ops.$(OBJEXT) + -rm -f cls/replica_log/cls_replica_log_types.$(OBJEXT) + -rm -f cls/rgw/cls_rgw.$(OBJEXT) + -rm -f cls/rgw/cls_rgw.lo + -rm -f cls/rgw/cls_rgw_client.$(OBJEXT) + -rm -f cls/rgw/cls_rgw_client.lo + -rm -f cls/rgw/cls_rgw_ops.$(OBJEXT) + -rm -f cls/rgw/cls_rgw_ops.lo + -rm -f cls/rgw/cls_rgw_types.$(OBJEXT) + -rm -f cls/rgw/cls_rgw_types.lo + -rm -f cls/statelog/cls_statelog.$(OBJEXT) + -rm -f cls/statelog/cls_statelog.lo + -rm -f cls/statelog/cls_statelog_client.$(OBJEXT) + -rm -f cls/user/cls_user.$(OBJEXT) + -rm -f cls/user/cls_user.lo + -rm -f cls/user/cls_user_client.$(OBJEXT) + -rm -f cls/user/cls_user_ops.$(OBJEXT) + -rm -f cls/user/cls_user_types.$(OBJEXT) + -rm -f cls/version/cls_version.$(OBJEXT) + -rm -f cls/version/cls_version.lo + -rm -f cls/version/cls_version_client.$(OBJEXT) + -rm -f cls/version/cls_version_types.$(OBJEXT) + -rm -f common/BackTrace.$(OBJEXT) + -rm -f common/BackTrace.lo + -rm -f common/Clock.$(OBJEXT) + -rm -f common/Clock.lo + -rm -f common/ConfUtils.$(OBJEXT) + -rm -f common/ConfUtils.lo + -rm -f common/DecayCounter.$(OBJEXT) + -rm -f common/DecayCounter.lo + -rm -f common/Finisher.$(OBJEXT) + -rm -f common/Finisher.lo + -rm -f common/Formatter.$(OBJEXT) + -rm -f common/Formatter.lo + -rm -f common/HeartbeatMap.$(OBJEXT) + -rm -f common/HeartbeatMap.lo + -rm -f common/LogClient.$(OBJEXT) + -rm -f common/LogClient.lo + -rm -f common/LogEntry.$(OBJEXT) + -rm -f common/LogEntry.lo + -rm -f common/MemoryModel.$(OBJEXT) + -rm -f common/MemoryModel.lo + -rm -f common/Mutex.$(OBJEXT) + -rm -f common/Mutex.lo + -rm -f common/OutputDataSocket.$(OBJEXT) + -rm -f common/OutputDataSocket.lo + -rm -f common/PrebufferedStreambuf.$(OBJEXT) + -rm -f common/PrebufferedStreambuf.lo + -rm -f common/RefCountedObj.$(OBJEXT) + -rm -f common/RefCountedObj.lo + -rm -f common/SloppyCRCMap.$(OBJEXT) + -rm -f common/SloppyCRCMap.lo + -rm -f common/TextTable.$(OBJEXT) + -rm -f common/TextTable.lo + -rm -f common/Thread.$(OBJEXT) + -rm -f common/Thread.lo + -rm -f common/Throttle.$(OBJEXT) + -rm -f common/Throttle.lo + -rm -f common/Timer.$(OBJEXT) + -rm -f common/Timer.lo + -rm -f common/WorkQueue.$(OBJEXT) + -rm -f common/WorkQueue.lo + -rm -f common/addr_parsing.$(OBJEXT) + -rm -f common/addr_parsing.lo + -rm -f common/admin_socket.$(OBJEXT) + -rm -f common/admin_socket.lo + -rm -f common/admin_socket_client.$(OBJEXT) + -rm -f common/admin_socket_client.lo + -rm -f common/armor.$(OBJEXT) + -rm -f common/armor.lo + -rm -f common/assert.$(OBJEXT) + -rm -f common/assert.lo + -rm -f common/blkdev.$(OBJEXT) + -rm -f common/blkdev.lo + -rm -f common/bloom_filter.$(OBJEXT) + -rm -f common/bloom_filter.lo + -rm -f common/buffer.$(OBJEXT) + -rm -f common/buffer.lo + -rm -f common/ceph_argparse.$(OBJEXT) + -rm -f common/ceph_argparse.lo + -rm -f common/ceph_context.$(OBJEXT) + -rm -f common/ceph_context.lo + -rm -f common/ceph_crypto.$(OBJEXT) + -rm -f common/ceph_crypto.lo + -rm -f common/ceph_crypto_cms.$(OBJEXT) + -rm -f common/ceph_crypto_cms.lo + -rm -f common/ceph_frag.$(OBJEXT) + -rm -f common/ceph_frag.lo + -rm -f common/ceph_fs.$(OBJEXT) + -rm -f common/ceph_fs.lo + -rm -f common/ceph_hash.$(OBJEXT) + -rm -f common/ceph_hash.lo + -rm -f common/ceph_json.$(OBJEXT) + -rm -f common/ceph_json.lo + -rm -f common/ceph_strings.$(OBJEXT) + -rm -f common/ceph_strings.lo + -rm -f common/cmdparse.$(OBJEXT) + -rm -f common/cmdparse.lo + -rm -f common/code_environment.$(OBJEXT) + -rm -f common/code_environment.lo + -rm -f common/common_init.$(OBJEXT) + -rm -f common/common_init.lo + -rm -f common/config.$(OBJEXT) + -rm -f common/config.lo + -rm -f common/dout.$(OBJEXT) + -rm -f common/dout.lo + -rm -f common/entity_name.$(OBJEXT) + -rm -f common/entity_name.lo + -rm -f common/environment.$(OBJEXT) + -rm -f common/environment.lo + -rm -f common/errno.$(OBJEXT) + -rm -f common/errno.lo + -rm -f common/escape.$(OBJEXT) + -rm -f common/escape.lo + -rm -f common/fd.$(OBJEXT) + -rm -f common/fd.lo + -rm -f common/hex.$(OBJEXT) + -rm -f common/hex.lo + -rm -f common/histogram.$(OBJEXT) + -rm -f common/histogram.lo + -rm -f common/hobject.$(OBJEXT) + -rm -f common/hobject.lo + -rm -f common/io_priority.$(OBJEXT) + -rm -f common/io_priority.lo + -rm -f common/ipaddr.$(OBJEXT) + -rm -f common/ipaddr.lo + -rm -f common/libcommon_crc_la-crc32c.$(OBJEXT) + -rm -f common/libcommon_crc_la-crc32c.lo + -rm -f common/libcommon_crc_la-crc32c_intel_baseline.$(OBJEXT) + -rm -f common/libcommon_crc_la-crc32c_intel_baseline.lo + -rm -f common/libcommon_crc_la-crc32c_intel_fast.$(OBJEXT) + -rm -f common/libcommon_crc_la-crc32c_intel_fast.lo + -rm -f common/libcommon_crc_la-crc32c_intel_fast_asm.$(OBJEXT) + -rm -f common/libcommon_crc_la-crc32c_intel_fast_asm.lo + -rm -f common/libcommon_crc_la-crc32c_intel_fast_zero_asm.$(OBJEXT) + -rm -f common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo + -rm -f common/libcommon_crc_la-sctp_crc32.$(OBJEXT) + -rm -f common/libcommon_crc_la-sctp_crc32.lo + -rm -f common/libos_la-TrackedOp.$(OBJEXT) + -rm -f common/libos_la-TrackedOp.lo + -rm -f common/libosd_la-TrackedOp.$(OBJEXT) + -rm -f common/libosd_la-TrackedOp.lo + -rm -f common/linux_version.$(OBJEXT) + -rm -f common/linux_version.lo + -rm -f common/lockdep.$(OBJEXT) + -rm -f common/lockdep.lo + -rm -f common/mime.$(OBJEXT) + -rm -f common/mime.lo + -rm -f common/obj_bencher.$(OBJEXT) + -rm -f common/page.$(OBJEXT) + -rm -f common/page.lo + -rm -f common/perf_counters.$(OBJEXT) + -rm -f common/perf_counters.lo + -rm -f common/pick_address.$(OBJEXT) + -rm -f common/pick_address.lo + -rm -f common/pipe.$(OBJEXT) + -rm -f common/pipe.lo + -rm -f common/rest_bench-obj_bencher.$(OBJEXT) + -rm -f common/run_cmd.$(OBJEXT) + -rm -f common/run_cmd.lo + -rm -f common/safe_io.$(OBJEXT) + -rm -f common/safe_io.lo + -rm -f common/secret.$(OBJEXT) + -rm -f common/signal.$(OBJEXT) + -rm -f common/signal.lo + -rm -f common/simple_spin.$(OBJEXT) + -rm -f common/simple_spin.lo + -rm -f common/snap_types.$(OBJEXT) + -rm -f common/snap_types.lo + -rm -f common/str_list.$(OBJEXT) + -rm -f common/str_list.lo + -rm -f common/str_map.$(OBJEXT) + -rm -f common/str_map.lo + -rm -f common/strtol.$(OBJEXT) + -rm -f common/strtol.lo + -rm -f common/test_build_libcommon-BackTrace.$(OBJEXT) + -rm -f common/test_build_libcommon-Clock.$(OBJEXT) + -rm -f common/test_build_libcommon-ConfUtils.$(OBJEXT) + -rm -f common/test_build_libcommon-DecayCounter.$(OBJEXT) + -rm -f common/test_build_libcommon-Finisher.$(OBJEXT) + -rm -f common/test_build_libcommon-Formatter.$(OBJEXT) + -rm -f common/test_build_libcommon-HeartbeatMap.$(OBJEXT) + -rm -f common/test_build_libcommon-LogClient.$(OBJEXT) + -rm -f common/test_build_libcommon-LogEntry.$(OBJEXT) + -rm -f common/test_build_libcommon-MemoryModel.$(OBJEXT) + -rm -f common/test_build_libcommon-Mutex.$(OBJEXT) + -rm -f common/test_build_libcommon-OutputDataSocket.$(OBJEXT) + -rm -f common/test_build_libcommon-PrebufferedStreambuf.$(OBJEXT) + -rm -f common/test_build_libcommon-RefCountedObj.$(OBJEXT) + -rm -f common/test_build_libcommon-SloppyCRCMap.$(OBJEXT) + -rm -f common/test_build_libcommon-TextTable.$(OBJEXT) + -rm -f common/test_build_libcommon-Thread.$(OBJEXT) + -rm -f common/test_build_libcommon-Throttle.$(OBJEXT) + -rm -f common/test_build_libcommon-Timer.$(OBJEXT) + -rm -f common/test_build_libcommon-WorkQueue.$(OBJEXT) + -rm -f common/test_build_libcommon-addr_parsing.$(OBJEXT) + -rm -f common/test_build_libcommon-admin_socket.$(OBJEXT) + -rm -f common/test_build_libcommon-admin_socket_client.$(OBJEXT) + -rm -f common/test_build_libcommon-armor.$(OBJEXT) + -rm -f common/test_build_libcommon-assert.$(OBJEXT) + -rm -f common/test_build_libcommon-blkdev.$(OBJEXT) + -rm -f common/test_build_libcommon-bloom_filter.$(OBJEXT) + -rm -f common/test_build_libcommon-buffer.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_argparse.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_context.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_crypto.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_crypto_cms.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_frag.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_fs.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_hash.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_json.$(OBJEXT) + -rm -f common/test_build_libcommon-ceph_strings.$(OBJEXT) + -rm -f common/test_build_libcommon-cmdparse.$(OBJEXT) + -rm -f common/test_build_libcommon-code_environment.$(OBJEXT) + -rm -f common/test_build_libcommon-common_init.$(OBJEXT) + -rm -f common/test_build_libcommon-config.$(OBJEXT) + -rm -f common/test_build_libcommon-dout.$(OBJEXT) + -rm -f common/test_build_libcommon-entity_name.$(OBJEXT) + -rm -f common/test_build_libcommon-environment.$(OBJEXT) + -rm -f common/test_build_libcommon-errno.$(OBJEXT) + -rm -f common/test_build_libcommon-escape.$(OBJEXT) + -rm -f common/test_build_libcommon-fd.$(OBJEXT) + -rm -f common/test_build_libcommon-hex.$(OBJEXT) + -rm -f common/test_build_libcommon-histogram.$(OBJEXT) + -rm -f common/test_build_libcommon-hobject.$(OBJEXT) + -rm -f common/test_build_libcommon-io_priority.$(OBJEXT) + -rm -f common/test_build_libcommon-ipaddr.$(OBJEXT) + -rm -f common/test_build_libcommon-linux_version.$(OBJEXT) + -rm -f common/test_build_libcommon-lockdep.$(OBJEXT) + -rm -f common/test_build_libcommon-mime.$(OBJEXT) + -rm -f common/test_build_libcommon-page.$(OBJEXT) + -rm -f common/test_build_libcommon-perf_counters.$(OBJEXT) + -rm -f common/test_build_libcommon-pick_address.$(OBJEXT) + -rm -f common/test_build_libcommon-pipe.$(OBJEXT) + -rm -f common/test_build_libcommon-run_cmd.$(OBJEXT) + -rm -f common/test_build_libcommon-safe_io.$(OBJEXT) + -rm -f common/test_build_libcommon-signal.$(OBJEXT) + -rm -f common/test_build_libcommon-simple_spin.$(OBJEXT) + -rm -f common/test_build_libcommon-snap_types.$(OBJEXT) + -rm -f common/test_build_libcommon-str_list.$(OBJEXT) + -rm -f common/test_build_libcommon-str_map.$(OBJEXT) + -rm -f common/test_build_libcommon-strtol.$(OBJEXT) + -rm -f common/test_build_libcommon-utf8.$(OBJEXT) + -rm -f common/test_build_libcommon-util.$(OBJEXT) + -rm -f common/test_build_libcommon-version.$(OBJEXT) + -rm -f common/test_build_libcommon-xattr.$(OBJEXT) + -rm -f common/utf8.$(OBJEXT) + -rm -f common/utf8.lo + -rm -f common/util.$(OBJEXT) + -rm -f common/util.lo + -rm -f common/version.$(OBJEXT) + -rm -f common/version.lo + -rm -f common/xattr.$(OBJEXT) + -rm -f common/xattr.lo + -rm -f crush/CrushCompiler.$(OBJEXT) + -rm -f crush/CrushCompiler.lo + -rm -f crush/CrushTester.$(OBJEXT) + -rm -f crush/CrushTester.lo + -rm -f crush/CrushWrapper.$(OBJEXT) + -rm -f crush/CrushWrapper.lo + -rm -f crush/builder.$(OBJEXT) + -rm -f crush/builder.lo + -rm -f crush/crush.$(OBJEXT) + -rm -f crush/crush.lo + -rm -f crush/hash.$(OBJEXT) + -rm -f crush/hash.lo + -rm -f crush/mapper.$(OBJEXT) + -rm -f crush/mapper.lo + -rm -f erasure-code/ErasureCodePlugin.$(OBJEXT) + -rm -f erasure-code/ErasureCodePlugin.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.$(OBJEXT) + -rm -f erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo + -rm -f erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.$(OBJEXT) + -rm -f erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo + -rm -f erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo + -rm -f erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo + -rm -f erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo + -rm -f erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo + -rm -f erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo + -rm -f erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo + -rm -f erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.$(OBJEXT) + -rm -f erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.$(OBJEXT) + -rm -f global/global_context.$(OBJEXT) + -rm -f global/global_context.lo + -rm -f global/global_init.$(OBJEXT) + -rm -f global/global_init.lo + -rm -f global/pidfile.$(OBJEXT) + -rm -f global/pidfile.lo + -rm -f global/signal_handler.$(OBJEXT) + -rm -f global/signal_handler.lo + -rm -f java/native/libcephfs_jni_la-JniConstants.$(OBJEXT) + -rm -f java/native/libcephfs_jni_la-JniConstants.lo + -rm -f java/native/libcephfs_jni_la-libcephfs_jni.$(OBJEXT) + -rm -f java/native/libcephfs_jni_la-libcephfs_jni.lo + -rm -f json_spirit/json_spirit_reader.$(OBJEXT) + -rm -f json_spirit/json_spirit_reader.lo + -rm -f json_spirit/json_spirit_writer.$(OBJEXT) + -rm -f json_spirit/json_spirit_writer.lo + -rm -f key_value_store/cls_kvs.$(OBJEXT) + -rm -f key_value_store/cls_kvs.lo + -rm -f key_value_store/kv_flat_btree_async.$(OBJEXT) + -rm -f librados/librados_la-IoCtxImpl.$(OBJEXT) + -rm -f librados/librados_la-IoCtxImpl.lo + -rm -f librados/librados_la-RadosClient.$(OBJEXT) + -rm -f librados/librados_la-RadosClient.lo + -rm -f librados/librados_la-librados.$(OBJEXT) + -rm -f librados/librados_la-librados.lo + -rm -f librados/librados_la-snap_set_diff.$(OBJEXT) + -rm -f librados/librados_la-snap_set_diff.lo + -rm -f librados/test_build_librados-IoCtxImpl.$(OBJEXT) + -rm -f librados/test_build_librados-RadosClient.$(OBJEXT) + -rm -f librados/test_build_librados-librados.$(OBJEXT) + -rm -f librados/test_build_librados-snap_set_diff.$(OBJEXT) + -rm -f librbd/AioCompletion.$(OBJEXT) + -rm -f librbd/AioCompletion.lo + -rm -f librbd/AioRequest.$(OBJEXT) + -rm -f librbd/AioRequest.lo + -rm -f librbd/ImageCtx.$(OBJEXT) + -rm -f librbd/ImageCtx.lo + -rm -f librbd/LibrbdWriteback.$(OBJEXT) + -rm -f librbd/LibrbdWriteback.lo + -rm -f librbd/WatchCtx.$(OBJEXT) + -rm -f librbd/WatchCtx.lo + -rm -f librbd/internal.$(OBJEXT) + -rm -f librbd/internal.lo + -rm -f librbd/librbd.$(OBJEXT) + -rm -f librbd/librbd.lo + -rm -f log/Log.$(OBJEXT) + -rm -f log/Log.lo + -rm -f log/SubsystemMap.$(OBJEXT) + -rm -f log/SubsystemMap.lo + -rm -f log/unittest_log-test.$(OBJEXT) + -rm -f mds/Anchor.$(OBJEXT) + -rm -f mds/Anchor.lo + -rm -f mds/AnchorClient.$(OBJEXT) + -rm -f mds/AnchorClient.lo + -rm -f mds/AnchorServer.$(OBJEXT) + -rm -f mds/AnchorServer.lo + -rm -f mds/CDentry.$(OBJEXT) + -rm -f mds/CDentry.lo + -rm -f mds/CDir.$(OBJEXT) + -rm -f mds/CDir.lo + -rm -f mds/CInode.$(OBJEXT) + -rm -f mds/CInode.lo + -rm -f mds/Capability.$(OBJEXT) + -rm -f mds/Capability.lo + -rm -f mds/Dumper.$(OBJEXT) + -rm -f mds/Dumper.lo + -rm -f mds/InoTable.$(OBJEXT) + -rm -f mds/InoTable.lo + -rm -f mds/Locker.$(OBJEXT) + -rm -f mds/Locker.lo + -rm -f mds/LogEvent.$(OBJEXT) + -rm -f mds/LogEvent.lo + -rm -f mds/MDBalancer.$(OBJEXT) + -rm -f mds/MDBalancer.lo + -rm -f mds/MDCache.$(OBJEXT) + -rm -f mds/MDCache.lo + -rm -f mds/MDLog.$(OBJEXT) + -rm -f mds/MDLog.lo + -rm -f mds/MDS.$(OBJEXT) + -rm -f mds/MDS.lo + -rm -f mds/MDSMap.$(OBJEXT) + -rm -f mds/MDSMap.lo + -rm -f mds/MDSTable.$(OBJEXT) + -rm -f mds/MDSTable.lo + -rm -f mds/MDSTableClient.$(OBJEXT) + -rm -f mds/MDSTableClient.lo + -rm -f mds/MDSTableServer.$(OBJEXT) + -rm -f mds/MDSTableServer.lo + -rm -f mds/MDSUtility.$(OBJEXT) + -rm -f mds/MDSUtility.lo + -rm -f mds/Migrator.$(OBJEXT) + -rm -f mds/Migrator.lo + -rm -f mds/Mutation.$(OBJEXT) + -rm -f mds/Mutation.lo + -rm -f mds/Resetter.$(OBJEXT) + -rm -f mds/Resetter.lo + -rm -f mds/Server.$(OBJEXT) + -rm -f mds/Server.lo + -rm -f mds/SessionMap.$(OBJEXT) + -rm -f mds/SessionMap.lo + -rm -f mds/SnapRealm.$(OBJEXT) + -rm -f mds/SnapRealm.lo + -rm -f mds/SnapServer.$(OBJEXT) + -rm -f mds/SnapServer.lo + -rm -f mds/flock.$(OBJEXT) + -rm -f mds/flock.lo + -rm -f mds/inode_backtrace.$(OBJEXT) + -rm -f mds/inode_backtrace.lo + -rm -f mds/journal.$(OBJEXT) + -rm -f mds/journal.lo + -rm -f mds/locks.$(OBJEXT) + -rm -f mds/locks.lo + -rm -f mds/mdstypes.$(OBJEXT) + -rm -f mds/mdstypes.lo + -rm -f mds/snap.$(OBJEXT) + -rm -f mds/snap.lo + -rm -f mds/test_build_libcommon-MDSMap.$(OBJEXT) + -rm -f mds/test_build_libcommon-inode_backtrace.$(OBJEXT) + -rm -f mds/test_build_libcommon-mdstypes.$(OBJEXT) + -rm -f mon/AuthMonitor.$(OBJEXT) + -rm -f mon/AuthMonitor.lo + -rm -f mon/ConfigKeyService.$(OBJEXT) + -rm -f mon/ConfigKeyService.lo + -rm -f mon/DataHealthService.$(OBJEXT) + -rm -f mon/DataHealthService.lo + -rm -f mon/Elector.$(OBJEXT) + -rm -f mon/Elector.lo + -rm -f mon/HealthMonitor.$(OBJEXT) + -rm -f mon/HealthMonitor.lo + -rm -f mon/LogMonitor.$(OBJEXT) + -rm -f mon/LogMonitor.lo + -rm -f mon/MDSMonitor.$(OBJEXT) + -rm -f mon/MDSMonitor.lo + -rm -f mon/MonCap.$(OBJEXT) + -rm -f mon/MonCap.lo + -rm -f mon/MonClient.$(OBJEXT) + -rm -f mon/MonClient.lo + -rm -f mon/MonMap.$(OBJEXT) + -rm -f mon/MonMap.lo + -rm -f mon/Monitor.$(OBJEXT) + -rm -f mon/Monitor.lo + -rm -f mon/MonitorStore.$(OBJEXT) + -rm -f mon/MonitorStore.lo + -rm -f mon/MonmapMonitor.$(OBJEXT) + -rm -f mon/MonmapMonitor.lo + -rm -f mon/OSDMonitor.$(OBJEXT) + -rm -f mon/OSDMonitor.lo + -rm -f mon/PGMap.$(OBJEXT) + -rm -f mon/PGMap.lo + -rm -f mon/PGMonitor.$(OBJEXT) + -rm -f mon/PGMonitor.lo + -rm -f mon/Paxos.$(OBJEXT) + -rm -f mon/Paxos.lo + -rm -f mon/PaxosService.$(OBJEXT) + -rm -f mon/PaxosService.lo + -rm -f mon/test_build_libcommon-MonCap.$(OBJEXT) + -rm -f mon/test_build_libcommon-MonClient.$(OBJEXT) + -rm -f mon/test_build_libcommon-MonMap.$(OBJEXT) + -rm -f mount/mount.ceph.$(OBJEXT) + -rm -f msg/Accepter.$(OBJEXT) + -rm -f msg/Accepter.lo + -rm -f msg/DispatchQueue.$(OBJEXT) + -rm -f msg/DispatchQueue.lo + -rm -f msg/Message.$(OBJEXT) + -rm -f msg/Message.lo + -rm -f msg/Messenger.$(OBJEXT) + -rm -f msg/Messenger.lo + -rm -f msg/Pipe.$(OBJEXT) + -rm -f msg/Pipe.lo + -rm -f msg/SimpleMessenger.$(OBJEXT) + -rm -f msg/SimpleMessenger.lo + -rm -f msg/msg_types.$(OBJEXT) + -rm -f msg/msg_types.lo + -rm -f objclass/libosd_la-class_api.$(OBJEXT) + -rm -f objclass/libosd_la-class_api.lo + -rm -f os/libos_la-BtrfsFileStoreBackend.$(OBJEXT) + -rm -f os/libos_la-BtrfsFileStoreBackend.lo + -rm -f os/libos_la-DBObjectMap.$(OBJEXT) + -rm -f os/libos_la-DBObjectMap.lo + -rm -f os/libos_la-FileJournal.$(OBJEXT) + -rm -f os/libos_la-FileJournal.lo + -rm -f os/libos_la-FileStore.$(OBJEXT) + -rm -f os/libos_la-FileStore.lo + -rm -f os/libos_la-FlatIndex.$(OBJEXT) + -rm -f os/libos_la-FlatIndex.lo + -rm -f os/libos_la-GenericFileStoreBackend.$(OBJEXT) + -rm -f os/libos_la-GenericFileStoreBackend.lo + -rm -f os/libos_la-GenericObjectMap.$(OBJEXT) + -rm -f os/libos_la-GenericObjectMap.lo + -rm -f os/libos_la-HashIndex.$(OBJEXT) + -rm -f os/libos_la-HashIndex.lo + -rm -f os/libos_la-IndexManager.$(OBJEXT) + -rm -f os/libos_la-IndexManager.lo + -rm -f os/libos_la-JournalingObjectStore.$(OBJEXT) + -rm -f os/libos_la-JournalingObjectStore.lo + -rm -f os/libos_la-KeyValueStore.$(OBJEXT) + -rm -f os/libos_la-KeyValueStore.lo + -rm -f os/libos_la-LFNIndex.$(OBJEXT) + -rm -f os/libos_la-LFNIndex.lo + -rm -f os/libos_la-LevelDBStore.$(OBJEXT) + -rm -f os/libos_la-LevelDBStore.lo + -rm -f os/libos_la-MemStore.$(OBJEXT) + -rm -f os/libos_la-MemStore.lo + -rm -f os/libos_la-ObjectStore.$(OBJEXT) + -rm -f os/libos_la-ObjectStore.lo + -rm -f os/libos_la-WBThrottle.$(OBJEXT) + -rm -f os/libos_la-WBThrottle.lo + -rm -f os/libos_la-XfsFileStoreBackend.$(OBJEXT) + -rm -f os/libos_la-XfsFileStoreBackend.lo + -rm -f os/libos_la-ZFSFileStoreBackend.$(OBJEXT) + -rm -f os/libos_la-ZFSFileStoreBackend.lo + -rm -f os/libos_la-chain_xattr.$(OBJEXT) + -rm -f os/libos_la-chain_xattr.lo + -rm -f os/libos_types_la-Transaction.$(OBJEXT) + -rm -f os/libos_types_la-Transaction.lo + -rm -f os/libos_zfs_a-ZFS.$(OBJEXT) + -rm -f osd/ECMsgTypes.$(OBJEXT) + -rm -f osd/ECMsgTypes.lo + -rm -f osd/HitSet.$(OBJEXT) + -rm -f osd/HitSet.lo + -rm -f osd/OSDMap.$(OBJEXT) + -rm -f osd/OSDMap.lo + -rm -f osd/ceph_test_rados_api_tier-HitSet.$(OBJEXT) + -rm -f osd/libosd_la-Ager.$(OBJEXT) + -rm -f osd/libosd_la-Ager.lo + -rm -f osd/libosd_la-ClassHandler.$(OBJEXT) + -rm -f osd/libosd_la-ClassHandler.lo + -rm -f osd/libosd_la-ECBackend.$(OBJEXT) + -rm -f osd/libosd_la-ECBackend.lo + -rm -f osd/libosd_la-ECMsgTypes.$(OBJEXT) + -rm -f osd/libosd_la-ECMsgTypes.lo + -rm -f osd/libosd_la-ECTransaction.$(OBJEXT) + -rm -f osd/libosd_la-ECTransaction.lo + -rm -f osd/libosd_la-HitSet.$(OBJEXT) + -rm -f osd/libosd_la-HitSet.lo + -rm -f osd/libosd_la-OSD.$(OBJEXT) + -rm -f osd/libosd_la-OSD.lo + -rm -f osd/libosd_la-OSDCap.$(OBJEXT) + -rm -f osd/libosd_la-OSDCap.lo + -rm -f osd/libosd_la-OpRequest.$(OBJEXT) + -rm -f osd/libosd_la-OpRequest.lo + -rm -f osd/libosd_la-PG.$(OBJEXT) + -rm -f osd/libosd_la-PG.lo + -rm -f osd/libosd_la-PGBackend.$(OBJEXT) + -rm -f osd/libosd_la-PGBackend.lo + -rm -f osd/libosd_la-ReplicatedBackend.$(OBJEXT) + -rm -f osd/libosd_la-ReplicatedBackend.lo + -rm -f osd/libosd_la-ReplicatedPG.$(OBJEXT) + -rm -f osd/libosd_la-ReplicatedPG.lo + -rm -f osd/libosd_la-SnapMapper.$(OBJEXT) + -rm -f osd/libosd_la-SnapMapper.lo + -rm -f osd/libosd_la-Watch.$(OBJEXT) + -rm -f osd/libosd_la-Watch.lo + -rm -f osd/libosd_types_la-ECUtil.$(OBJEXT) + -rm -f osd/libosd_types_la-ECUtil.lo + -rm -f osd/libosd_types_la-PGLog.$(OBJEXT) + -rm -f osd/libosd_types_la-PGLog.lo + -rm -f osd/libosd_types_la-osd_types.$(OBJEXT) + -rm -f osd/libosd_types_la-osd_types.lo + -rm -f osd/osd_types.$(OBJEXT) + -rm -f osd/osd_types.lo + -rm -f osd/test_build_libcommon-ECMsgTypes.$(OBJEXT) + -rm -f osd/test_build_libcommon-HitSet.$(OBJEXT) + -rm -f osd/test_build_libcommon-OSDMap.$(OBJEXT) + -rm -f osd/test_build_libcommon-osd_types.$(OBJEXT) + -rm -f osdc/Filer.$(OBJEXT) + -rm -f osdc/Filer.lo + -rm -f osdc/Journaler.$(OBJEXT) + -rm -f osdc/Journaler.lo + -rm -f osdc/ObjectCacher.$(OBJEXT) + -rm -f osdc/ObjectCacher.lo + -rm -f osdc/Objecter.$(OBJEXT) + -rm -f osdc/Objecter.lo + -rm -f osdc/Striper.$(OBJEXT) + -rm -f osdc/Striper.lo + -rm -f osdc/test_build_libcephfs-Filer.$(OBJEXT) + -rm -f osdc/test_build_libcephfs-Journaler.$(OBJEXT) + -rm -f osdc/test_build_libcephfs-ObjectCacher.$(OBJEXT) + -rm -f osdc/test_build_libcephfs-Objecter.$(OBJEXT) + -rm -f osdc/test_build_libcephfs-Striper.$(OBJEXT) + -rm -f perfglue/cpu_profiler.$(OBJEXT) + -rm -f perfglue/cpu_profiler.lo + -rm -f perfglue/disabled_heap_profiler.$(OBJEXT) + -rm -f perfglue/disabled_heap_profiler.lo + -rm -f perfglue/disabled_stubs.$(OBJEXT) + -rm -f perfglue/disabled_stubs.lo + -rm -f perfglue/heap_profiler.$(OBJEXT) + -rm -f perfglue/heap_profiler.lo + -rm -f rbd_fuse/rbd-fuse.$(OBJEXT) + -rm -f rgw/ceph_dencoder-rgw_acl.$(OBJEXT) + -rm -f rgw/ceph_dencoder-rgw_common.$(OBJEXT) + -rm -f rgw/ceph_dencoder-rgw_dencoder.$(OBJEXT) + -rm -f rgw/ceph_dencoder-rgw_env.$(OBJEXT) + -rm -f rgw/ceph_dencoder-rgw_json_enc.$(OBJEXT) + -rm -f rgw/librgw_la-librgw.$(OBJEXT) + -rm -f rgw/librgw_la-librgw.lo + -rm -f rgw/librgw_la-rgw_acl.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_acl.lo + -rm -f rgw/librgw_la-rgw_acl_s3.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_acl_s3.lo + -rm -f rgw/librgw_la-rgw_acl_swift.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_acl_swift.lo + -rm -f rgw/librgw_la-rgw_auth_s3.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_auth_s3.lo + -rm -f rgw/librgw_la-rgw_bucket.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_bucket.lo + -rm -f rgw/librgw_la-rgw_cache.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_cache.lo + -rm -f rgw/librgw_la-rgw_client_io.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_client_io.lo + -rm -f rgw/librgw_la-rgw_common.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_common.lo + -rm -f rgw/librgw_la-rgw_cors.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_cors.lo + -rm -f rgw/librgw_la-rgw_cors_s3.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_cors_s3.lo + -rm -f rgw/librgw_la-rgw_dencoder.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_dencoder.lo + -rm -f rgw/librgw_la-rgw_env.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_env.lo + -rm -f rgw/librgw_la-rgw_fcgi.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_fcgi.lo + -rm -f rgw/librgw_la-rgw_formats.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_formats.lo + -rm -f rgw/librgw_la-rgw_gc.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_gc.lo + -rm -f rgw/librgw_la-rgw_http_client.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_http_client.lo + -rm -f rgw/librgw_la-rgw_json_enc.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_json_enc.lo + -rm -f rgw/librgw_la-rgw_keystone.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_keystone.lo + -rm -f rgw/librgw_la-rgw_log.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_log.lo + -rm -f rgw/librgw_la-rgw_metadata.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_metadata.lo + -rm -f rgw/librgw_la-rgw_multi.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_multi.lo + -rm -f rgw/librgw_la-rgw_multi_del.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_multi_del.lo + -rm -f rgw/librgw_la-rgw_op.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_op.lo + -rm -f rgw/librgw_la-rgw_policy_s3.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_policy_s3.lo + -rm -f rgw/librgw_la-rgw_quota.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_quota.lo + -rm -f rgw/librgw_la-rgw_rados.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_rados.lo + -rm -f rgw/librgw_la-rgw_replica_log.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_replica_log.lo + -rm -f rgw/librgw_la-rgw_rest_client.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_rest_client.lo + -rm -f rgw/librgw_la-rgw_rest_conn.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_rest_conn.lo + -rm -f rgw/librgw_la-rgw_tools.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_tools.lo + -rm -f rgw/librgw_la-rgw_usage.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_usage.lo + -rm -f rgw/librgw_la-rgw_user.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_user.lo + -rm -f rgw/librgw_la-rgw_xml.$(OBJEXT) + -rm -f rgw/librgw_la-rgw_xml.lo + -rm -f rgw/rgw_admin.$(OBJEXT) + -rm -f rgw/rgw_civetweb.$(OBJEXT) + -rm -f rgw/rgw_common.$(OBJEXT) + -rm -f rgw/rgw_env.$(OBJEXT) + -rm -f rgw/rgw_http_client.$(OBJEXT) + -rm -f rgw/rgw_json_enc.$(OBJEXT) + -rm -f rgw/rgw_jsonparser.$(OBJEXT) + -rm -f rgw/rgw_loadgen.$(OBJEXT) + -rm -f rgw/rgw_main.$(OBJEXT) + -rm -f rgw/rgw_multiparser.$(OBJEXT) + -rm -f rgw/rgw_replica_log.$(OBJEXT) + -rm -f rgw/rgw_resolve.$(OBJEXT) + -rm -f rgw/rgw_rest.$(OBJEXT) + -rm -f rgw/rgw_rest_bucket.$(OBJEXT) + -rm -f rgw/rgw_rest_config.$(OBJEXT) + -rm -f rgw/rgw_rest_log.$(OBJEXT) + -rm -f rgw/rgw_rest_metadata.$(OBJEXT) + -rm -f rgw/rgw_rest_opstate.$(OBJEXT) + -rm -f rgw/rgw_rest_replica_log.$(OBJEXT) + -rm -f rgw/rgw_rest_s3.$(OBJEXT) + -rm -f rgw/rgw_rest_swift.$(OBJEXT) + -rm -f rgw/rgw_rest_usage.$(OBJEXT) + -rm -f rgw/rgw_rest_user.$(OBJEXT) + -rm -f rgw/rgw_swift.$(OBJEXT) + -rm -f rgw/rgw_swift_auth.$(OBJEXT) + -rm -f rgw/test_build_librgw-librgw.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_acl.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_acl_s3.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_acl_swift.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_auth_s3.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_bucket.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_cache.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_client_io.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_common.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_cors.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_cors_s3.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_dencoder.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_env.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_fcgi.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_formats.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_gc.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_http_client.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_json_enc.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_keystone.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_log.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_metadata.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_multi.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_multi_del.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_op.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_policy_s3.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_quota.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_rados.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_replica_log.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_rest_client.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_rest_conn.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_tools.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_usage.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_user.$(OBJEXT) + -rm -f rgw/test_build_librgw-rgw_xml.$(OBJEXT) + -rm -f rgw/unittest_formatter-rgw_formats.$(OBJEXT) + -rm -f test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.$(OBJEXT) + -rm -f test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.$(OBJEXT) + -rm -f test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.$(OBJEXT) + -rm -f test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.$(OBJEXT) + -rm -f test/ObjectMap/ceph_test_object_map-test_object_map.$(OBJEXT) + -rm -f test/TestSignalHandlers.$(OBJEXT) + -rm -f test/TestTimers.$(OBJEXT) + -rm -f test/bench/bencher.$(OBJEXT) + -rm -f test/bench/detailed_stat_collector.$(OBJEXT) + -rm -f test/bench/dumb_backend.$(OBJEXT) + -rm -f test/bench/rados_backend.$(OBJEXT) + -rm -f test/bench/rbd_backend.$(OBJEXT) + -rm -f test/bench/small_io_bench.$(OBJEXT) + -rm -f test/bench/small_io_bench_dumb.$(OBJEXT) + -rm -f test/bench/small_io_bench_fs.$(OBJEXT) + -rm -f test/bench/small_io_bench_rbd.$(OBJEXT) + -rm -f test/bench/testfilestore_backend.$(OBJEXT) + -rm -f test/bench/tp_bench.$(OBJEXT) + -rm -f test/bench_log.$(OBJEXT) + -rm -f test/ceph_test_cls_rgw_log-test_rgw_admin_log.$(OBJEXT) + -rm -f test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.$(OBJEXT) + -rm -f test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.$(OBJEXT) + -rm -f test/ceph_test_cors-test_cors.$(OBJEXT) + -rm -f test/ceph_test_filejournal-test_filejournal.$(OBJEXT) + -rm -f test/ceph_test_snap_mapper-test_snap_mapper.$(OBJEXT) + -rm -f test/ceph_test_stress_watch-test_stress_watch.$(OBJEXT) + -rm -f test/ceph_xattr_bench-xattr_bench.$(OBJEXT) + -rm -f test/cls_hello/ceph_test_cls_hello-test_cls_hello.$(OBJEXT) + -rm -f test/cls_lock/ceph_test_cls_lock-test_cls_lock.$(OBJEXT) + -rm -f test/cls_log/ceph_test_cls_log-test_cls_log.$(OBJEXT) + -rm -f test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.$(OBJEXT) + -rm -f test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.$(OBJEXT) + -rm -f test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.$(OBJEXT) + -rm -f test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.$(OBJEXT) + -rm -f test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.$(OBJEXT) + -rm -f test/cls_version/ceph_test_cls_version-test_cls_version.$(OBJEXT) + -rm -f test/common/ObjectContents.$(OBJEXT) + -rm -f test/common/get_command_descriptions.$(OBJEXT) + -rm -f test/common/unittest_bloom_filter-test_bloom_filter.$(OBJEXT) + -rm -f test/common/unittest_config-test_config.$(OBJEXT) + -rm -f test/common/unittest_context-test_context.$(OBJEXT) + -rm -f test/common/unittest_crc32c-test_crc32c.$(OBJEXT) + -rm -f test/common/unittest_histogram-histogram.$(OBJEXT) + -rm -f test/common/unittest_sharedptr_registry-test_sharedptr_registry.$(OBJEXT) + -rm -f test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.$(OBJEXT) + -rm -f test/common/unittest_str_map-test_str_map.$(OBJEXT) + -rm -f test/common/unittest_throttle-Throttle.$(OBJEXT) + -rm -f test/common/unittest_util-test_util.$(OBJEXT) + -rm -f test/crush/unittest_crush_indep-indep.$(OBJEXT) + -rm -f test/crush/unittest_crush_wrapper-TestCrushWrapper.$(OBJEXT) + -rm -f test/encoding/ceph_dencoder-ceph_dencoder.$(OBJEXT) + -rm -f test/erasure-code/ceph_erasure_code.$(OBJEXT) + -rm -f test/erasure-code/ceph_erasure_code_benchmark.$(OBJEXT) + -rm -f test/erasure-code/libec_example_la-ErasureCodePluginExample.$(OBJEXT) + -rm -f test/erasure-code/libec_example_la-ErasureCodePluginExample.lo + -rm -f test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.$(OBJEXT) + -rm -f test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo + -rm -f test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.$(OBJEXT) + -rm -f test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo + -rm -f test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.$(OBJEXT) + -rm -f test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo + -rm -f test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.$(OBJEXT) + -rm -f test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo + -rm -f test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.$(OBJEXT) + -rm -f test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo + -rm -f test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.$(OBJEXT) + -rm -f test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo + -rm -f test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.$(OBJEXT) + -rm -f test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo + -rm -f test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.$(OBJEXT) + -rm -f test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.$(OBJEXT) + -rm -f test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.$(OBJEXT) + -rm -f test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.$(OBJEXT) + -rm -f test/kv_store_bench.$(OBJEXT) + -rm -f test/libcephfs/ceph_test_libcephfs-caps.$(OBJEXT) + -rm -f test/libcephfs/ceph_test_libcephfs-multiclient.$(OBJEXT) + -rm -f test/libcephfs/ceph_test_libcephfs-readdir_r_cb.$(OBJEXT) + -rm -f test/libcephfs/ceph_test_libcephfs-test.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_aio-aio.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_cls-cls.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_cmd-cmd.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_io-io.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_list-list.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_lock-lock.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_misc-misc.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_pool-pool.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_snapshots-snapshots.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_stat-stat.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_tier-tier.$(OBJEXT) + -rm -f test/librados/ceph_test_rados_api_watch_notify-watch_notify.$(OBJEXT) + -rm -f test/librados/libradostest_la-TestCase.$(OBJEXT) + -rm -f test/librados/libradostest_la-TestCase.lo + -rm -f test/librados/libradostest_la-test.$(OBJEXT) + -rm -f test/librados/libradostest_la-test.lo + -rm -f test/librados/unittest_librados-librados.$(OBJEXT) + -rm -f test/librados/unittest_librados_config-librados_config.$(OBJEXT) + -rm -f test/librbd/ceph_test_librbd-test_librbd.$(OBJEXT) + -rm -f test/librbd/ceph_test_librbd_fsx-fsx.$(OBJEXT) + -rm -f test/mon/test_mon_workloadgen.$(OBJEXT) + -rm -f test/mon/unittest_mon_moncap-moncap.$(OBJEXT) + -rm -f test/mon/unittest_mon_pgmap-PGMap.$(OBJEXT) + -rm -f test/multi_stress_watch.$(OBJEXT) + -rm -f test/objectstore/DeterministicOpSequence.$(OBJEXT) + -rm -f test/objectstore/FileStoreDiff.$(OBJEXT) + -rm -f test/objectstore/FileStoreTracker.$(OBJEXT) + -rm -f test/objectstore/TestObjectStoreState.$(OBJEXT) + -rm -f test/objectstore/ceph_test_objectstore-store_test.$(OBJEXT) + -rm -f test/objectstore/test_idempotent.$(OBJEXT) + -rm -f test/objectstore/test_idempotent_sequence.$(OBJEXT) + -rm -f test/objectstore/unittest_chain_xattr-chain_xattr.$(OBJEXT) + -rm -f test/objectstore/workload_generator.$(OBJEXT) + -rm -f test/omap_bench.$(OBJEXT) + -rm -f test/on_exit.$(OBJEXT) + -rm -f test/os/unittest_flatindex-TestFlatIndex.$(OBJEXT) + -rm -f test/os/unittest_lfnindex-TestLFNIndex.$(OBJEXT) + -rm -f test/osd/Object.$(OBJEXT) + -rm -f test/osd/RadosModel.$(OBJEXT) + -rm -f test/osd/TestOpStat.$(OBJEXT) + -rm -f test/osd/TestRados.$(OBJEXT) + -rm -f test/osd/unittest_ecbackend-TestECBackend.$(OBJEXT) + -rm -f test/osd/unittest_hitset-hitset.$(OBJEXT) + -rm -f test/osd/unittest_osd_osdcap-osdcap.$(OBJEXT) + -rm -f test/osd/unittest_osd_types-types.$(OBJEXT) + -rm -f test/osd/unittest_osdmap-TestOSDMap.$(OBJEXT) + -rm -f test/osd/unittest_pglog-TestPGLog.$(OBJEXT) + -rm -f test/osdc/FakeWriteback.$(OBJEXT) + -rm -f test/osdc/object_cacher_stress.$(OBJEXT) + -rm -f test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.$(OBJEXT) + -rm -f test/streamtest.$(OBJEXT) + -rm -f test/system/cross_process_sem.$(OBJEXT) + -rm -f test/system/cross_process_sem.lo + -rm -f test/system/rados_delete_pools_parallel.$(OBJEXT) + -rm -f test/system/rados_list_parallel.$(OBJEXT) + -rm -f test/system/rados_open_pools_parallel.$(OBJEXT) + -rm -f test/system/rados_watch_notify.$(OBJEXT) + -rm -f test/system/st_rados_create_pool.$(OBJEXT) + -rm -f test/system/st_rados_delete_objs.$(OBJEXT) + -rm -f test/system/st_rados_delete_pool.$(OBJEXT) + -rm -f test/system/st_rados_list_objects.$(OBJEXT) + -rm -f test/system/st_rados_notify.$(OBJEXT) + -rm -f test/system/st_rados_watch.$(OBJEXT) + -rm -f test/system/systest_runnable.$(OBJEXT) + -rm -f test/system/systest_runnable.lo + -rm -f test/system/systest_settings.$(OBJEXT) + -rm -f test/system/systest_settings.lo + -rm -f test/test_build_libcephfs-buildtest_skeleton.$(OBJEXT) + -rm -f test/test_build_libcommon-buildtest_skeleton.$(OBJEXT) + -rm -f test/test_build_librados-buildtest_skeleton.$(OBJEXT) + -rm -f test/test_build_librgw-buildtest_skeleton.$(OBJEXT) + -rm -f test/test_c_headers.$(OBJEXT) + -rm -f test/test_cfuse_cache_invalidate.$(OBJEXT) + -rm -f test/test_get_blkdev_size.$(OBJEXT) + -rm -f test/test_mutate.$(OBJEXT) + -rm -f test/test_rewrite_latency.$(OBJEXT) + -rm -f test/test_trans.$(OBJEXT) + -rm -f test/testcrypto.$(OBJEXT) + -rm -f test/testkeys.$(OBJEXT) + -rm -f test/testmsgr.$(OBJEXT) + -rm -f test/unittest_addrs-test_addrs.$(OBJEXT) + -rm -f test/unittest_admin_socket-admin_socket.$(OBJEXT) + -rm -f test/unittest_arch-test_arch.$(OBJEXT) + -rm -f test/unittest_base64-base64.$(OBJEXT) + -rm -f test/unittest_bufferlist-bufferlist.$(OBJEXT) + -rm -f test/unittest_ceph_argparse-ceph_argparse.$(OBJEXT) + -rm -f test/unittest_ceph_compatset-ceph_compatset.$(OBJEXT) + -rm -f test/unittest_ceph_crypto-ceph_crypto.$(OBJEXT) + -rm -f test/unittest_confutils-confutils.$(OBJEXT) + -rm -f test/unittest_crypto-crypto.$(OBJEXT) + -rm -f test/unittest_crypto_init-crypto_init.$(OBJEXT) + -rm -f test/unittest_daemon_config-daemon_config.$(OBJEXT) + -rm -f test/unittest_encoding-encoding.$(OBJEXT) + -rm -f test/unittest_escape-escape.$(OBJEXT) + -rm -f test/unittest_formatter-formatter.$(OBJEXT) + -rm -f test/unittest_gather-gather.$(OBJEXT) + -rm -f test/unittest_heartbeatmap-heartbeat_map.$(OBJEXT) + -rm -f test/unittest_ipaddr-test_ipaddr.$(OBJEXT) + -rm -f test/unittest_libcephfs_config-libcephfs_config.$(OBJEXT) + -rm -f test/unittest_mime-mime.$(OBJEXT) + -rm -f test/unittest_perf_counters-perf_counters.$(OBJEXT) + -rm -f test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.$(OBJEXT) + -rm -f test/unittest_run_cmd-run_cmd.$(OBJEXT) + -rm -f test/unittest_signals-signals.$(OBJEXT) + -rm -f test/unittest_simple_spin-simple_spin.$(OBJEXT) + -rm -f test/unittest_str_list-test_str_list.$(OBJEXT) + -rm -f test/unittest_striper-test_striper.$(OBJEXT) + -rm -f test/unittest_strtol-strtol.$(OBJEXT) + -rm -f test/unittest_texttable-test_texttable.$(OBJEXT) + -rm -f test/unittest_utf8-utf8.$(OBJEXT) + -rm -f test/unittest_workqueue-test_workqueue.$(OBJEXT) + -rm -f tools/ceph-client-debug.$(OBJEXT) + -rm -f tools/ceph_authtool.$(OBJEXT) + -rm -f tools/ceph_conf.$(OBJEXT) + -rm -f tools/ceph_filestore_dump.$(OBJEXT) + -rm -f tools/ceph_filestore_tool.$(OBJEXT) + -rm -f tools/ceph_kvstore_tool-ceph_kvstore_tool.$(OBJEXT) + -rm -f tools/ceph_monstore_tool.$(OBJEXT) + -rm -f tools/ceph_osdomap_tool.$(OBJEXT) + -rm -f tools/crushtool.$(OBJEXT) + -rm -f tools/dupstore.$(OBJEXT) + -rm -f tools/mon_store_converter.$(OBJEXT) + -rm -f tools/monmaptool.$(OBJEXT) + -rm -f tools/osdmaptool.$(OBJEXT) + -rm -f tools/psim.$(OBJEXT) + -rm -f tools/rados/rados.$(OBJEXT) + -rm -f tools/rados/rados_export.$(OBJEXT) + -rm -f tools/rados/rados_import.$(OBJEXT) + -rm -f tools/rados/rados_sync.$(OBJEXT) + -rm -f tools/radosacl.$(OBJEXT) + -rm -f tools/rest_bench-rest_bench.$(OBJEXT) + -rm -f tools/scratchtool.$(OBJEXT) + -rm -f tools/scratchtoolpp.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ceph_fuse.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ceph_mds.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ceph_mon.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ceph_osd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ceph_syn.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ceph_ver.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cephfs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcephfs.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/librados-config.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rbd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_build_libcommon-ceph_ver.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@arch/$(DEPDIR)/intel.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@arch/$(DEPDIR)/neon.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@arch/$(DEPDIR)/probe.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/AuthAuthorizeHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/AuthClientHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/AuthMethodList.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/AuthServiceHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/AuthSessionHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/Crypto.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/KeyRing.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/$(DEPDIR)/RotatingKeyRing.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/cephx/$(DEPDIR)/CephxAuthorizeHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/cephx/$(DEPDIR)/CephxClientHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/cephx/$(DEPDIR)/CephxKeyServer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/cephx/$(DEPDIR)/CephxProtocol.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/cephx/$(DEPDIR)/CephxServiceHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/cephx/$(DEPDIR)/CephxSessionHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/none/$(DEPDIR)/AuthNoneAuthorizeHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@auth/unknown/$(DEPDIR)/AuthUnknownAuthorizeHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@civetweb/src/$(DEPDIR)/radosgw-civetweb.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/Client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/ClientSnapRealm.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/Dentry.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/Inode.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/MetaRequest.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/MetaSession.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/SyntheticClient.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/Trace.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/fuse_ll.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@client/$(DEPDIR)/test_ioctls.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/hello/$(DEPDIR)/cls_hello.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/lock/$(DEPDIR)/cls_lock.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/lock/$(DEPDIR)/cls_lock_client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/lock/$(DEPDIR)/cls_lock_ops.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/lock/$(DEPDIR)/cls_lock_types.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/log/$(DEPDIR)/cls_log.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/log/$(DEPDIR)/cls_log_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/rbd/$(DEPDIR)/cls_rbd.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/rbd/$(DEPDIR)/cls_rbd_client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/refcount/$(DEPDIR)/cls_refcount.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/refcount/$(DEPDIR)/cls_refcount_client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/refcount/$(DEPDIR)/cls_refcount_ops.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/replica_log/$(DEPDIR)/cls_replica_log.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/replica_log/$(DEPDIR)/cls_replica_log_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/replica_log/$(DEPDIR)/cls_replica_log_ops.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/replica_log/$(DEPDIR)/cls_replica_log_types.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/rgw/$(DEPDIR)/cls_rgw.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/rgw/$(DEPDIR)/cls_rgw_client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/rgw/$(DEPDIR)/cls_rgw_ops.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/rgw/$(DEPDIR)/cls_rgw_types.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/statelog/$(DEPDIR)/cls_statelog.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/statelog/$(DEPDIR)/cls_statelog_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/user/$(DEPDIR)/cls_user.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/user/$(DEPDIR)/cls_user_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/user/$(DEPDIR)/cls_user_ops.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/user/$(DEPDIR)/cls_user_types.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/version/$(DEPDIR)/cls_version.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/version/$(DEPDIR)/cls_version_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@cls/version/$(DEPDIR)/cls_version_types.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/BackTrace.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/Clock.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ConfUtils.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/DecayCounter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/Finisher.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/Formatter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/HeartbeatMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/LogClient.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/LogEntry.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/MemoryModel.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/Mutex.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/OutputDataSocket.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/PrebufferedStreambuf.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/RefCountedObj.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/SloppyCRCMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/TextTable.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/Thread.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/Throttle.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/Timer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/WorkQueue.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/addr_parsing.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/admin_socket.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/admin_socket_client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/armor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/assert.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/blkdev.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/bloom_filter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/buffer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_argparse.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_context.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_crypto.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_crypto_cms.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_frag.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_fs.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_hash.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_json.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ceph_strings.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/cmdparse.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/code_environment.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/common_init.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/config.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/dout.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/entity_name.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/environment.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/errno.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/escape.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/fd.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/hex.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/histogram.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/hobject.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/io_priority.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/ipaddr.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libcommon_crc_la-crc32c.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_baseline.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_asm.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_zero_asm.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libcommon_crc_la-sctp_crc32.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libos_la-TrackedOp.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/libosd_la-TrackedOp.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/linux_version.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/lockdep.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/mime.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/obj_bencher.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/page.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/perf_counters.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/pick_address.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/pipe.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/rest_bench-obj_bencher.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/run_cmd.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/safe_io.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/secret.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/signal.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/simple_spin.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/snap_types.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/str_list.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/str_map.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/strtol.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-BackTrace.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-Clock.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ConfUtils.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-DecayCounter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-Finisher.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-Formatter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-HeartbeatMap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-LogClient.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-LogEntry.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-MemoryModel.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-Mutex.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-OutputDataSocket.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-PrebufferedStreambuf.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-RefCountedObj.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-SloppyCRCMap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-TextTable.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-Thread.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-Throttle.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-Timer.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-WorkQueue.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-addr_parsing.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-admin_socket.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-admin_socket_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-armor.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-assert.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-blkdev.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-bloom_filter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-buffer.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_argparse.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_context.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_crypto.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_crypto_cms.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_frag.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_fs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_hash.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_json.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ceph_strings.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-cmdparse.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-code_environment.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-common_init.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-config.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-dout.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-entity_name.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-environment.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-errno.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-escape.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-fd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-hex.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-histogram.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-hobject.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-io_priority.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-ipaddr.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-linux_version.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-lockdep.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-mime.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-page.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-perf_counters.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-pick_address.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-pipe.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-run_cmd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-safe_io.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-signal.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-simple_spin.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-snap_types.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-str_list.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-str_map.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-strtol.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-utf8.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-util.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-version.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/test_build_libcommon-xattr.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/utf8.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/util.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/version.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@common/$(DEPDIR)/xattr.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@crush/$(DEPDIR)/CrushCompiler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@crush/$(DEPDIR)/CrushTester.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@crush/$(DEPDIR)/CrushWrapper.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@crush/$(DEPDIR)/builder.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@crush/$(DEPDIR)/crush.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@crush/$(DEPDIR)/hash.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@crush/$(DEPDIR)/mapper.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/$(DEPDIR)/ErasureCodePlugin.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodeJerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodePluginJerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginSelectJerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodeJerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodePluginJerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodeJerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodePluginJerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_general.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_method.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_rand.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w128.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w16.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w32.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w4.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w64.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w8.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_wgen.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_general.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_method.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_rand.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w128.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w16.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w32.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w4.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w64.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w8.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_wgen.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_general.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_method.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_rand.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w128.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w16.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w32.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w4.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w64.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w8.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_wgen.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_general.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_method.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_rand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w128.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w16.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w32.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w4.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w64.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w8.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_wgen.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-cauchy.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-galois.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-jerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-liberation.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-reed_sol.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-cauchy.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-galois.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-jerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-liberation.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-reed_sol.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-cauchy.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-galois.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-jerasure.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-liberation.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-reed_sol.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-cauchy.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-galois.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-jerasure.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-liberation.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-reed_sol.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@global/$(DEPDIR)/global_context.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@global/$(DEPDIR)/global_init.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@global/$(DEPDIR)/pidfile.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@global/$(DEPDIR)/signal_handler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@java/native/$(DEPDIR)/libcephfs_jni_la-JniConstants.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@java/native/$(DEPDIR)/libcephfs_jni_la-libcephfs_jni.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@json_spirit/$(DEPDIR)/json_spirit_reader.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@json_spirit/$(DEPDIR)/json_spirit_writer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@key_value_store/$(DEPDIR)/cls_kvs.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@key_value_store/$(DEPDIR)/kv_flat_btree_async.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/librados_la-IoCtxImpl.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/librados_la-RadosClient.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/librados_la-librados.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/librados_la-snap_set_diff.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/test_build_librados-RadosClient.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/test_build_librados-librados.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librados/$(DEPDIR)/test_build_librados-snap_set_diff.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librbd/$(DEPDIR)/AioCompletion.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librbd/$(DEPDIR)/AioRequest.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librbd/$(DEPDIR)/ImageCtx.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librbd/$(DEPDIR)/LibrbdWriteback.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librbd/$(DEPDIR)/WatchCtx.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librbd/$(DEPDIR)/internal.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@librbd/$(DEPDIR)/librbd.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@log/$(DEPDIR)/Log.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@log/$(DEPDIR)/SubsystemMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@log/$(DEPDIR)/unittest_log-test.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Anchor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/AnchorClient.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/AnchorServer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/CDentry.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/CDir.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/CInode.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Capability.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Dumper.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/InoTable.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Locker.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/LogEvent.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDBalancer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDCache.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDLog.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDS.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDSMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDSTable.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDSTableClient.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDSTableServer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/MDSUtility.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Migrator.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Mutation.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Resetter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/Server.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/SessionMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/SnapRealm.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/SnapServer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/flock.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/inode_backtrace.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/journal.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/locks.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/mdstypes.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/snap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/test_build_libcommon-MDSMap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/test_build_libcommon-inode_backtrace.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mds/$(DEPDIR)/test_build_libcommon-mdstypes.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/AuthMonitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/ConfigKeyService.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/DataHealthService.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/Elector.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/HealthMonitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/LogMonitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/MDSMonitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/MonCap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/MonClient.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/MonMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/Monitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/MonitorStore.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/MonmapMonitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/OSDMonitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/PGMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/PGMonitor.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/Paxos.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/PaxosService.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/test_build_libcommon-MonCap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/test_build_libcommon-MonClient.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mon/$(DEPDIR)/test_build_libcommon-MonMap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@mount/$(DEPDIR)/mount.ceph.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@msg/$(DEPDIR)/Accepter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@msg/$(DEPDIR)/DispatchQueue.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@msg/$(DEPDIR)/Message.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@msg/$(DEPDIR)/Messenger.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@msg/$(DEPDIR)/Pipe.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@msg/$(DEPDIR)/SimpleMessenger.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@msg/$(DEPDIR)/msg_types.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@objclass/$(DEPDIR)/libosd_la-class_api.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-BtrfsFileStoreBackend.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-DBObjectMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-FileJournal.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-FileStore.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-FlatIndex.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-GenericFileStoreBackend.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-GenericObjectMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-HashIndex.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-IndexManager.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-JournalingObjectStore.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-KeyValueStore.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-LFNIndex.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-LevelDBStore.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-MemStore.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-ObjectStore.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-WBThrottle.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-XfsFileStoreBackend.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-ZFSFileStoreBackend.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_la-chain_xattr.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_types_la-Transaction.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@os/$(DEPDIR)/libos_zfs_a-ZFS.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/ECMsgTypes.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/HitSet.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/OSDMap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-Ager.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-ClassHandler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-ECBackend.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-ECMsgTypes.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-ECTransaction.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-HitSet.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-OSD.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-OSDCap.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-OpRequest.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-PG.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-PGBackend.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-ReplicatedBackend.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-ReplicatedPG.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-SnapMapper.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_la-Watch.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_types_la-ECUtil.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_types_la-PGLog.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/libosd_types_la-osd_types.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/osd_types.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/test_build_libcommon-HitSet.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/test_build_libcommon-OSDMap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osd/$(DEPDIR)/test_build_libcommon-osd_types.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/Filer.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/Journaler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/ObjectCacher.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/Objecter.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/Striper.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/test_build_libcephfs-Filer.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/test_build_libcephfs-Journaler.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/test_build_libcephfs-ObjectCacher.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/test_build_libcephfs-Objecter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@osdc/$(DEPDIR)/test_build_libcephfs-Striper.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@perfglue/$(DEPDIR)/cpu_profiler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@perfglue/$(DEPDIR)/disabled_heap_profiler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@perfglue/$(DEPDIR)/disabled_stubs.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@perfglue/$(DEPDIR)/heap_profiler.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rbd_fuse/$(DEPDIR)/rbd-fuse.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/ceph_dencoder-rgw_acl.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/ceph_dencoder-rgw_common.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/ceph_dencoder-rgw_dencoder.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/ceph_dencoder-rgw_env.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/ceph_dencoder-rgw_json_enc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-librgw.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_acl.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_acl_s3.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_acl_swift.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_auth_s3.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_bucket.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_cache.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_client_io.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_common.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_cors.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_cors_s3.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_dencoder.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_env.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_fcgi.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_formats.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_gc.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_http_client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_json_enc.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_keystone.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_log.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_metadata.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_multi.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_multi_del.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_op.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_policy_s3.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_quota.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_rados.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_replica_log.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_rest_client.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_rest_conn.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_tools.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_usage.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_user.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/librgw_la-rgw_xml.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_admin.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_civetweb.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_common.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_env.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_http_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_json_enc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_jsonparser.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_loadgen.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_main.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_multiparser.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_replica_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_resolve.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_bucket.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_config.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_metadata.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_opstate.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_replica_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_s3.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_swift.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_usage.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_rest_user.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_swift.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/rgw_swift_auth.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-librgw.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_acl.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_acl_s3.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_acl_swift.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_auth_s3.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_bucket.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_cache.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_client_io.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_common.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_cors.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_cors_s3.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_dencoder.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_env.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_fcgi.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_formats.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_gc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_http_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_json_enc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_keystone.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_metadata.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_multi.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_multi_del.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_op.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_policy_s3.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_quota.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_rados.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_replica_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_rest_client.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_rest_conn.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_tools.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_usage.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_user.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/test_build_librgw-rgw_xml.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@rgw/$(DEPDIR)/unittest_formatter-rgw_formats.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/TestSignalHandlers.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/TestTimers.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/bench_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_test_cls_rgw_meta-test_rgw_admin_meta.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_test_cors-test_cors.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_test_filejournal-test_filejournal.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_test_stress_watch-test_stress_watch.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/kv_store_bench.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/multi_stress_watch.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/omap_bench.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/on_exit.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/streamtest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_c_headers.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_cfuse_cache_invalidate.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_get_blkdev_size.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_mutate.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_rewrite_latency.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/test_trans.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/testcrypto.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/testkeys.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/testmsgr.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_addrs-test_addrs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_admin_socket-admin_socket.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_arch-test_arch.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_base64-base64.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_bufferlist-bufferlist.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_confutils-confutils.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_crypto-crypto.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_crypto_init-crypto_init.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_daemon_config-daemon_config.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_encoding-encoding.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_escape-escape.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_formatter-formatter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_gather-gather.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_heartbeatmap-heartbeat_map.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_mime-mime.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_perf_counters-perf_counters.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_run_cmd-run_cmd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_signals-signals.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_simple_spin-simple_spin.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_str_list-test_str_list.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_striper-test_striper.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_strtol-strtol.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_texttable-test_texttable.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_utf8-utf8.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/$(DEPDIR)/unittest_workqueue-test_workqueue.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/ObjectMap/$(DEPDIR)/ceph_test_object_map-KeyValueDBMemory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/bencher.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/detailed_stat_collector.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/dumb_backend.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/rados_backend.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/rbd_backend.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/small_io_bench.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/small_io_bench_dumb.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/small_io_bench_fs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/small_io_bench_rbd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/testfilestore_backend.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/bench/$(DEPDIR)/tp_bench.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_hello/$(DEPDIR)/ceph_test_cls_hello-test_cls_hello.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_statelog/$(DEPDIR)/ceph_test_cls_statelog-test_cls_statelog.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/ObjectContents.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/get_command_descriptions.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_config-test_config.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_context-test_context.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_histogram-histogram.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_str_map-test_str_map.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_throttle-Throttle.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/common/$(DEPDIR)/unittest_util-test_util.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/crush/$(DEPDIR)/unittest_crush_indep-indep.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/ceph_erasure_code.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/ceph_erasure_code_benchmark.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/libradostest_la-TestCase.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/libradostest_la-test.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/unittest_librados-librados.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/mon/$(DEPDIR)/test_mon_workloadgen.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/DeterministicOpSequence.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/FileStoreDiff.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/FileStoreTracker.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/TestObjectStoreState.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/test_idempotent.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/test_idempotent_sequence.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/objectstore/$(DEPDIR)/workload_generator.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/Object.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/RadosModel.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/TestOpStat.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/TestRados.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/unittest_hitset-hitset.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/unittest_osd_types-types.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osdc/$(DEPDIR)/FakeWriteback.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/osdc/$(DEPDIR)/object_cacher_stress.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/cross_process_sem.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/rados_delete_pools_parallel.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/rados_list_parallel.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/rados_open_pools_parallel.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/rados_watch_notify.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/st_rados_create_pool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/st_rados_delete_objs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/st_rados_delete_pool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/st_rados_list_objects.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/st_rados_notify.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/st_rados_watch.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/systest_runnable.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@test/system/$(DEPDIR)/systest_settings.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph-client-debug.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph_authtool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph_conf.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph_filestore_dump.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph_filestore_tool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph_kvstore_tool-ceph_kvstore_tool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph_monstore_tool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/ceph_osdomap_tool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/crushtool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/dupstore.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/mon_store_converter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/monmaptool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/osdmaptool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/psim.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/radosacl.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/rest_bench-rest_bench.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/scratchtool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/$(DEPDIR)/scratchtoolpp.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/rados/$(DEPDIR)/rados.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/rados/$(DEPDIR)/rados_export.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/rados/$(DEPDIR)/rados_import.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tools/rados/$(DEPDIR)/rados_sync.Po@am__quote@ + +.S.o: +@am__fastdepCCAS_TRUE@ $(AM_V_CPPAS)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ +@am__fastdepCCAS_TRUE@ $(CPPASCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCCAS_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS@am__nodep@)$(CPPASCOMPILE) -c -o $@ $< + +.S.obj: +@am__fastdepCCAS_TRUE@ $(AM_V_CPPAS)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ +@am__fastdepCCAS_TRUE@ $(CPPASCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ +@am__fastdepCCAS_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS@am__nodep@)$(CPPASCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.S.lo: +@am__fastdepCCAS_TRUE@ $(AM_V_CPPAS)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ +@am__fastdepCCAS_TRUE@ $(LTCPPASCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCCAS_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS@am__nodep@)$(LTCPPASCOMPILE) -c -o $@ $< + +common/libcommon_crc_la-crc32c_intel_fast_asm.lo: common/crc32c_intel_fast_asm.S +@am__fastdepCCAS_TRUE@ $(AM_V_CPPAS)$(LIBTOOL) $(AM_V_lt) $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CCASFLAGS) $(CCASFLAGS) -MT common/libcommon_crc_la-crc32c_intel_fast_asm.lo -MD -MP -MF common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_asm.Tpo -c -o common/libcommon_crc_la-crc32c_intel_fast_asm.lo `test -f 'common/crc32c_intel_fast_asm.S' || echo '$(srcdir)/'`common/crc32c_intel_fast_asm.S +@am__fastdepCCAS_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_asm.Tpo common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_asm.Plo +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS)source='common/crc32c_intel_fast_asm.S' object='common/libcommon_crc_la-crc32c_intel_fast_asm.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS@am__nodep@)$(LIBTOOL) $(AM_V_lt) $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CCASFLAGS) $(CCASFLAGS) -c -o common/libcommon_crc_la-crc32c_intel_fast_asm.lo `test -f 'common/crc32c_intel_fast_asm.S' || echo '$(srcdir)/'`common/crc32c_intel_fast_asm.S + +common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo: common/crc32c_intel_fast_zero_asm.S +@am__fastdepCCAS_TRUE@ $(AM_V_CPPAS)$(LIBTOOL) $(AM_V_lt) $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CCASFLAGS) $(CCASFLAGS) -MT common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo -MD -MP -MF common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_zero_asm.Tpo -c -o common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo `test -f 'common/crc32c_intel_fast_zero_asm.S' || echo '$(srcdir)/'`common/crc32c_intel_fast_zero_asm.S +@am__fastdepCCAS_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_zero_asm.Tpo common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast_zero_asm.Plo +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS)source='common/crc32c_intel_fast_zero_asm.S' object='common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(AM_V_CPPAS@am__nodep@)$(LIBTOOL) $(AM_V_lt) $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CCASFLAGS) $(CCASFLAGS) -c -o common/libcommon_crc_la-crc32c_intel_fast_zero_asm.lo `test -f 'common/crc32c_intel_fast_zero_asm.S' || echo '$(srcdir)/'`common/crc32c_intel_fast_zero_asm.S + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ +@am__fastdepCC_TRUE@ $(LTCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +common/libcommon_crc_la-sctp_crc32.lo: common/sctp_crc32.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT common/libcommon_crc_la-sctp_crc32.lo -MD -MP -MF common/$(DEPDIR)/libcommon_crc_la-sctp_crc32.Tpo -c -o common/libcommon_crc_la-sctp_crc32.lo `test -f 'common/sctp_crc32.c' || echo '$(srcdir)/'`common/sctp_crc32.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libcommon_crc_la-sctp_crc32.Tpo common/$(DEPDIR)/libcommon_crc_la-sctp_crc32.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/sctp_crc32.c' object='common/libcommon_crc_la-sctp_crc32.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o common/libcommon_crc_la-sctp_crc32.lo `test -f 'common/sctp_crc32.c' || echo '$(srcdir)/'`common/sctp_crc32.c + +common/libcommon_crc_la-crc32c_intel_baseline.lo: common/crc32c_intel_baseline.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT common/libcommon_crc_la-crc32c_intel_baseline.lo -MD -MP -MF common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_baseline.Tpo -c -o common/libcommon_crc_la-crc32c_intel_baseline.lo `test -f 'common/crc32c_intel_baseline.c' || echo '$(srcdir)/'`common/crc32c_intel_baseline.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_baseline.Tpo common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_baseline.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/crc32c_intel_baseline.c' object='common/libcommon_crc_la-crc32c_intel_baseline.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o common/libcommon_crc_la-crc32c_intel_baseline.lo `test -f 'common/crc32c_intel_baseline.c' || echo '$(srcdir)/'`common/crc32c_intel_baseline.c + +common/libcommon_crc_la-crc32c_intel_fast.lo: common/crc32c_intel_fast.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT common/libcommon_crc_la-crc32c_intel_fast.lo -MD -MP -MF common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast.Tpo -c -o common/libcommon_crc_la-crc32c_intel_fast.lo `test -f 'common/crc32c_intel_fast.c' || echo '$(srcdir)/'`common/crc32c_intel_fast.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast.Tpo common/$(DEPDIR)/libcommon_crc_la-crc32c_intel_fast.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/crc32c_intel_fast.c' object='common/libcommon_crc_la-crc32c_intel_fast.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o common/libcommon_crc_la-crc32c_intel_fast.lo `test -f 'common/crc32c_intel_fast.c' || echo '$(srcdir)/'`common/crc32c_intel_fast.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo: erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-cauchy.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-cauchy.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-cauchy.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/cauchy.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-cauchy.lo `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo: erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-galois.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-galois.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-galois.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/galois.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-galois.lo `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo: erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-jerasure.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-jerasure.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-jerasure.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/jerasure.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo: erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-liberation.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-liberation.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-liberation.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/liberation.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-liberation.lo `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo: erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-reed_sol.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-reed_sol.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_generic_la-reed_sol.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/reed_sol.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_generic_la-reed_sol.lo `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo: erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_wgen.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_wgen.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_wgen.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_wgen.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_wgen.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo: erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_method.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_method.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_method.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_method.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_method.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo: erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w16.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w16.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w16.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w16.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w16.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo: erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo: erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w32.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w32.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w32.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w32.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w32.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo: erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w64.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w64.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w64.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w64.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w64.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo: erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w128.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w128.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w128.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w128.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w128.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo: erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_general.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_general.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_general.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_general.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_general.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo: erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w4.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w4.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w4.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w4.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w4.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo: erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_rand.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_rand.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_rand.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_rand.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_rand.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo: erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w8.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w8.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_generic_la-gf_w8.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w8.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_generic_la-gf_w8.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo: erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-cauchy.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-cauchy.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-cauchy.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/cauchy.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-cauchy.lo `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo: erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-galois.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-galois.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-galois.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/galois.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-galois.lo `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo: erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-jerasure.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-jerasure.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-jerasure.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/jerasure.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo: erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-liberation.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-liberation.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-liberation.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/liberation.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-liberation.lo `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo: erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-reed_sol.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-reed_sol.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse3_la-reed_sol.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/reed_sol.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse3_la-reed_sol.lo `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo: erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_wgen.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_wgen.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_wgen.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_wgen.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_wgen.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo: erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_method.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_method.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_method.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_method.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_method.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo: erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w16.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w16.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w16.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w16.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w16.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo: erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo: erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w32.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w32.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w32.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w32.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w32.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo: erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w64.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w64.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w64.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w64.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w64.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo: erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w128.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w128.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w128.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w128.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w128.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo: erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_general.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_general.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_general.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_general.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_general.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo: erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w4.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w4.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w4.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w4.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w4.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo: erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_rand.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_rand.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_rand.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_rand.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_rand.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo: erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w8.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w8.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse3_la-gf_w8.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w8.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse3_la-gf_w8.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo: erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-cauchy.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-cauchy.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-cauchy.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/cauchy.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-cauchy.lo `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo: erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-galois.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-galois.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-galois.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/galois.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-galois.lo `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo: erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-jerasure.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-jerasure.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-jerasure.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/jerasure.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-jerasure.lo `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo: erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-liberation.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-liberation.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-liberation.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/liberation.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-liberation.lo `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c + +erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo: erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-reed_sol.Tpo -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-reed_sol.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/libec_jerasure_sse4_la-reed_sol.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/reed_sol.c' object='erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/libec_jerasure_sse4_la-reed_sol.lo `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo: erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_wgen.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_wgen.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_wgen.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_wgen.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_wgen.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo: erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_method.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_method.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_method.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_method.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_method.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo: erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w16.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w16.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w16.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w16.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w16.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo: erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo: erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w32.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w32.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w32.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w32.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w32.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo: erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w64.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w64.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w64.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w64.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w64.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo: erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w128.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w128.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w128.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w128.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w128.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo: erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_general.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_general.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_general.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_general.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_general.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo: erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w4.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w4.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w4.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w4.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w4.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo: erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_rand.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_rand.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_rand.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_rand.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_rand.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c + +erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo: erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w8.Tpo -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w8.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/libec_jerasure_sse4_la-gf_w8.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w8.c' object='erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/libec_jerasure_sse4_la-gf_w8.lo `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c + +test/librbd/ceph_test_librbd_fsx-fsx.o: test/librbd/fsx.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) -MT test/librbd/ceph_test_librbd_fsx-fsx.o -MD -MP -MF test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Tpo -c -o test/librbd/ceph_test_librbd_fsx-fsx.o `test -f 'test/librbd/fsx.c' || echo '$(srcdir)/'`test/librbd/fsx.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Tpo test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='test/librbd/fsx.c' object='test/librbd/ceph_test_librbd_fsx-fsx.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) -c -o test/librbd/ceph_test_librbd_fsx-fsx.o `test -f 'test/librbd/fsx.c' || echo '$(srcdir)/'`test/librbd/fsx.c + +test/librbd/ceph_test_librbd_fsx-fsx.obj: test/librbd/fsx.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) -MT test/librbd/ceph_test_librbd_fsx-fsx.obj -MD -MP -MF test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Tpo -c -o test/librbd/ceph_test_librbd_fsx-fsx.obj `if test -f 'test/librbd/fsx.c'; then $(CYGPATH_W) 'test/librbd/fsx.c'; else $(CYGPATH_W) '$(srcdir)/test/librbd/fsx.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Tpo test/librbd/$(DEPDIR)/ceph_test_librbd_fsx-fsx.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='test/librbd/fsx.c' object='test/librbd/ceph_test_librbd_fsx-fsx.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_fsx_CFLAGS) $(CFLAGS) -c -o test/librbd/ceph_test_librbd_fsx-fsx.obj `if test -f 'test/librbd/fsx.c'; then $(CYGPATH_W) 'test/librbd/fsx.c'; else $(CYGPATH_W) '$(srcdir)/test/librbd/fsx.c'; fi` + +civetweb/src/radosgw-civetweb.o: civetweb/src/civetweb.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -MT civetweb/src/radosgw-civetweb.o -MD -MP -MF civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo -c -o civetweb/src/radosgw-civetweb.o `test -f 'civetweb/src/civetweb.c' || echo '$(srcdir)/'`civetweb/src/civetweb.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo civetweb/src/$(DEPDIR)/radosgw-civetweb.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='civetweb/src/civetweb.c' object='civetweb/src/radosgw-civetweb.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -c -o civetweb/src/radosgw-civetweb.o `test -f 'civetweb/src/civetweb.c' || echo '$(srcdir)/'`civetweb/src/civetweb.c + +civetweb/src/radosgw-civetweb.obj: civetweb/src/civetweb.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -MT civetweb/src/radosgw-civetweb.obj -MD -MP -MF civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo -c -o civetweb/src/radosgw-civetweb.obj `if test -f 'civetweb/src/civetweb.c'; then $(CYGPATH_W) 'civetweb/src/civetweb.c'; else $(CYGPATH_W) '$(srcdir)/civetweb/src/civetweb.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) civetweb/src/$(DEPDIR)/radosgw-civetweb.Tpo civetweb/src/$(DEPDIR)/radosgw-civetweb.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='civetweb/src/civetweb.c' object='civetweb/src/radosgw-civetweb.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(radosgw_CFLAGS) $(CFLAGS) -c -o civetweb/src/radosgw-civetweb.obj `if test -f 'civetweb/src/civetweb.c'; then $(CYGPATH_W) 'civetweb/src/civetweb.c'; else $(CYGPATH_W) '$(srcdir)/civetweb/src/civetweb.c'; fi` + +test_build_libcommon-ceph_ver.o: ceph_ver.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT test_build_libcommon-ceph_ver.o -MD -MP -MF $(DEPDIR)/test_build_libcommon-ceph_ver.Tpo -c -o test_build_libcommon-ceph_ver.o `test -f 'ceph_ver.c' || echo '$(srcdir)/'`ceph_ver.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/test_build_libcommon-ceph_ver.Tpo $(DEPDIR)/test_build_libcommon-ceph_ver.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ceph_ver.c' object='test_build_libcommon-ceph_ver.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o test_build_libcommon-ceph_ver.o `test -f 'ceph_ver.c' || echo '$(srcdir)/'`ceph_ver.c + +test_build_libcommon-ceph_ver.obj: ceph_ver.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT test_build_libcommon-ceph_ver.obj -MD -MP -MF $(DEPDIR)/test_build_libcommon-ceph_ver.Tpo -c -o test_build_libcommon-ceph_ver.obj `if test -f 'ceph_ver.c'; then $(CYGPATH_W) 'ceph_ver.c'; else $(CYGPATH_W) '$(srcdir)/ceph_ver.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/test_build_libcommon-ceph_ver.Tpo $(DEPDIR)/test_build_libcommon-ceph_ver.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ceph_ver.c' object='test_build_libcommon-ceph_ver.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o test_build_libcommon-ceph_ver.obj `if test -f 'ceph_ver.c'; then $(CYGPATH_W) 'ceph_ver.c'; else $(CYGPATH_W) '$(srcdir)/ceph_ver.c'; fi` + +common/test_build_libcommon-escape.o: common/escape.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-escape.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-escape.Tpo -c -o common/test_build_libcommon-escape.o `test -f 'common/escape.c' || echo '$(srcdir)/'`common/escape.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-escape.Tpo common/$(DEPDIR)/test_build_libcommon-escape.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/escape.c' object='common/test_build_libcommon-escape.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-escape.o `test -f 'common/escape.c' || echo '$(srcdir)/'`common/escape.c + +common/test_build_libcommon-escape.obj: common/escape.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-escape.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-escape.Tpo -c -o common/test_build_libcommon-escape.obj `if test -f 'common/escape.c'; then $(CYGPATH_W) 'common/escape.c'; else $(CYGPATH_W) '$(srcdir)/common/escape.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-escape.Tpo common/$(DEPDIR)/test_build_libcommon-escape.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/escape.c' object='common/test_build_libcommon-escape.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-escape.obj `if test -f 'common/escape.c'; then $(CYGPATH_W) 'common/escape.c'; else $(CYGPATH_W) '$(srcdir)/common/escape.c'; fi` + +common/test_build_libcommon-armor.o: common/armor.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-armor.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-armor.Tpo -c -o common/test_build_libcommon-armor.o `test -f 'common/armor.c' || echo '$(srcdir)/'`common/armor.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-armor.Tpo common/$(DEPDIR)/test_build_libcommon-armor.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/armor.c' object='common/test_build_libcommon-armor.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-armor.o `test -f 'common/armor.c' || echo '$(srcdir)/'`common/armor.c + +common/test_build_libcommon-armor.obj: common/armor.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-armor.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-armor.Tpo -c -o common/test_build_libcommon-armor.obj `if test -f 'common/armor.c'; then $(CYGPATH_W) 'common/armor.c'; else $(CYGPATH_W) '$(srcdir)/common/armor.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-armor.Tpo common/$(DEPDIR)/test_build_libcommon-armor.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/armor.c' object='common/test_build_libcommon-armor.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-armor.obj `if test -f 'common/armor.c'; then $(CYGPATH_W) 'common/armor.c'; else $(CYGPATH_W) '$(srcdir)/common/armor.c'; fi` + +common/test_build_libcommon-xattr.o: common/xattr.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-xattr.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-xattr.Tpo -c -o common/test_build_libcommon-xattr.o `test -f 'common/xattr.c' || echo '$(srcdir)/'`common/xattr.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-xattr.Tpo common/$(DEPDIR)/test_build_libcommon-xattr.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/xattr.c' object='common/test_build_libcommon-xattr.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-xattr.o `test -f 'common/xattr.c' || echo '$(srcdir)/'`common/xattr.c + +common/test_build_libcommon-xattr.obj: common/xattr.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-xattr.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-xattr.Tpo -c -o common/test_build_libcommon-xattr.obj `if test -f 'common/xattr.c'; then $(CYGPATH_W) 'common/xattr.c'; else $(CYGPATH_W) '$(srcdir)/common/xattr.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-xattr.Tpo common/$(DEPDIR)/test_build_libcommon-xattr.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/xattr.c' object='common/test_build_libcommon-xattr.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-xattr.obj `if test -f 'common/xattr.c'; then $(CYGPATH_W) 'common/xattr.c'; else $(CYGPATH_W) '$(srcdir)/common/xattr.c'; fi` + +common/test_build_libcommon-safe_io.o: common/safe_io.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-safe_io.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-safe_io.Tpo -c -o common/test_build_libcommon-safe_io.o `test -f 'common/safe_io.c' || echo '$(srcdir)/'`common/safe_io.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-safe_io.Tpo common/$(DEPDIR)/test_build_libcommon-safe_io.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/safe_io.c' object='common/test_build_libcommon-safe_io.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-safe_io.o `test -f 'common/safe_io.c' || echo '$(srcdir)/'`common/safe_io.c + +common/test_build_libcommon-safe_io.obj: common/safe_io.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-safe_io.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-safe_io.Tpo -c -o common/test_build_libcommon-safe_io.obj `if test -f 'common/safe_io.c'; then $(CYGPATH_W) 'common/safe_io.c'; else $(CYGPATH_W) '$(srcdir)/common/safe_io.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-safe_io.Tpo common/$(DEPDIR)/test_build_libcommon-safe_io.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/safe_io.c' object='common/test_build_libcommon-safe_io.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-safe_io.obj `if test -f 'common/safe_io.c'; then $(CYGPATH_W) 'common/safe_io.c'; else $(CYGPATH_W) '$(srcdir)/common/safe_io.c'; fi` + +common/test_build_libcommon-pipe.o: common/pipe.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-pipe.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-pipe.Tpo -c -o common/test_build_libcommon-pipe.o `test -f 'common/pipe.c' || echo '$(srcdir)/'`common/pipe.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-pipe.Tpo common/$(DEPDIR)/test_build_libcommon-pipe.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/pipe.c' object='common/test_build_libcommon-pipe.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-pipe.o `test -f 'common/pipe.c' || echo '$(srcdir)/'`common/pipe.c + +common/test_build_libcommon-pipe.obj: common/pipe.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-pipe.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-pipe.Tpo -c -o common/test_build_libcommon-pipe.obj `if test -f 'common/pipe.c'; then $(CYGPATH_W) 'common/pipe.c'; else $(CYGPATH_W) '$(srcdir)/common/pipe.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-pipe.Tpo common/$(DEPDIR)/test_build_libcommon-pipe.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/pipe.c' object='common/test_build_libcommon-pipe.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-pipe.obj `if test -f 'common/pipe.c'; then $(CYGPATH_W) 'common/pipe.c'; else $(CYGPATH_W) '$(srcdir)/common/pipe.c'; fi` + +common/test_build_libcommon-utf8.o: common/utf8.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-utf8.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-utf8.Tpo -c -o common/test_build_libcommon-utf8.o `test -f 'common/utf8.c' || echo '$(srcdir)/'`common/utf8.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-utf8.Tpo common/$(DEPDIR)/test_build_libcommon-utf8.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/utf8.c' object='common/test_build_libcommon-utf8.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-utf8.o `test -f 'common/utf8.c' || echo '$(srcdir)/'`common/utf8.c + +common/test_build_libcommon-utf8.obj: common/utf8.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-utf8.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-utf8.Tpo -c -o common/test_build_libcommon-utf8.obj `if test -f 'common/utf8.c'; then $(CYGPATH_W) 'common/utf8.c'; else $(CYGPATH_W) '$(srcdir)/common/utf8.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-utf8.Tpo common/$(DEPDIR)/test_build_libcommon-utf8.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/utf8.c' object='common/test_build_libcommon-utf8.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-utf8.obj `if test -f 'common/utf8.c'; then $(CYGPATH_W) 'common/utf8.c'; else $(CYGPATH_W) '$(srcdir)/common/utf8.c'; fi` + +common/test_build_libcommon-mime.o: common/mime.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-mime.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-mime.Tpo -c -o common/test_build_libcommon-mime.o `test -f 'common/mime.c' || echo '$(srcdir)/'`common/mime.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-mime.Tpo common/$(DEPDIR)/test_build_libcommon-mime.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/mime.c' object='common/test_build_libcommon-mime.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-mime.o `test -f 'common/mime.c' || echo '$(srcdir)/'`common/mime.c + +common/test_build_libcommon-mime.obj: common/mime.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-mime.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-mime.Tpo -c -o common/test_build_libcommon-mime.obj `if test -f 'common/mime.c'; then $(CYGPATH_W) 'common/mime.c'; else $(CYGPATH_W) '$(srcdir)/common/mime.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-mime.Tpo common/$(DEPDIR)/test_build_libcommon-mime.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/mime.c' object='common/test_build_libcommon-mime.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-mime.obj `if test -f 'common/mime.c'; then $(CYGPATH_W) 'common/mime.c'; else $(CYGPATH_W) '$(srcdir)/common/mime.c'; fi` + +common/test_build_libcommon-addr_parsing.o: common/addr_parsing.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-addr_parsing.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-addr_parsing.Tpo -c -o common/test_build_libcommon-addr_parsing.o `test -f 'common/addr_parsing.c' || echo '$(srcdir)/'`common/addr_parsing.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-addr_parsing.Tpo common/$(DEPDIR)/test_build_libcommon-addr_parsing.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/addr_parsing.c' object='common/test_build_libcommon-addr_parsing.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-addr_parsing.o `test -f 'common/addr_parsing.c' || echo '$(srcdir)/'`common/addr_parsing.c + +common/test_build_libcommon-addr_parsing.obj: common/addr_parsing.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-addr_parsing.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-addr_parsing.Tpo -c -o common/test_build_libcommon-addr_parsing.obj `if test -f 'common/addr_parsing.c'; then $(CYGPATH_W) 'common/addr_parsing.c'; else $(CYGPATH_W) '$(srcdir)/common/addr_parsing.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-addr_parsing.Tpo common/$(DEPDIR)/test_build_libcommon-addr_parsing.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/addr_parsing.c' object='common/test_build_libcommon-addr_parsing.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-addr_parsing.obj `if test -f 'common/addr_parsing.c'; then $(CYGPATH_W) 'common/addr_parsing.c'; else $(CYGPATH_W) '$(srcdir)/common/addr_parsing.c'; fi` + +common/test_build_libcommon-linux_version.o: common/linux_version.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-linux_version.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo -c -o common/test_build_libcommon-linux_version.o `test -f 'common/linux_version.c' || echo '$(srcdir)/'`common/linux_version.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo common/$(DEPDIR)/test_build_libcommon-linux_version.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/linux_version.c' object='common/test_build_libcommon-linux_version.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-linux_version.o `test -f 'common/linux_version.c' || echo '$(srcdir)/'`common/linux_version.c + +common/test_build_libcommon-linux_version.obj: common/linux_version.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -MT common/test_build_libcommon-linux_version.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo -c -o common/test_build_libcommon-linux_version.obj `if test -f 'common/linux_version.c'; then $(CYGPATH_W) 'common/linux_version.c'; else $(CYGPATH_W) '$(srcdir)/common/linux_version.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-linux_version.Tpo common/$(DEPDIR)/test_build_libcommon-linux_version.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='common/linux_version.c' object='common/test_build_libcommon-linux_version.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CFLAGS) $(CFLAGS) -c -o common/test_build_libcommon-linux_version.obj `if test -f 'common/linux_version.c'; then $(CYGPATH_W) 'common/linux_version.c'; else $(CYGPATH_W) '$(srcdir)/common/linux_version.c'; fi` + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.o: erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.o -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-cauchy.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.o `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-cauchy.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-cauchy.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/cauchy.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.o `test -f 'erasure-code/jerasure/jerasure/src/cauchy.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/cauchy.c + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.obj: erasure-code/jerasure/jerasure/src/cauchy.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.obj -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-cauchy.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.obj `if test -f 'erasure-code/jerasure/jerasure/src/cauchy.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/cauchy.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/cauchy.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-cauchy.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-cauchy.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/cauchy.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-cauchy.obj `if test -f 'erasure-code/jerasure/jerasure/src/cauchy.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/cauchy.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/cauchy.c'; fi` + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.o: erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.o -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-galois.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.o `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-galois.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-galois.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/galois.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.o `test -f 'erasure-code/jerasure/jerasure/src/galois.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/galois.c + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.obj: erasure-code/jerasure/jerasure/src/galois.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.obj -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-galois.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.obj `if test -f 'erasure-code/jerasure/jerasure/src/galois.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/galois.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/galois.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-galois.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-galois.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/galois.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-galois.obj `if test -f 'erasure-code/jerasure/jerasure/src/galois.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/galois.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/galois.c'; fi` + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.o: erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.o -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-jerasure.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.o `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-jerasure.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-jerasure.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/jerasure.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.o `test -f 'erasure-code/jerasure/jerasure/src/jerasure.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/jerasure.c + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.obj: erasure-code/jerasure/jerasure/src/jerasure.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.obj -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-jerasure.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.obj `if test -f 'erasure-code/jerasure/jerasure/src/jerasure.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/jerasure.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/jerasure.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-jerasure.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-jerasure.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/jerasure.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-jerasure.obj `if test -f 'erasure-code/jerasure/jerasure/src/jerasure.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/jerasure.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/jerasure.c'; fi` + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.o: erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.o -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-liberation.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.o `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-liberation.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-liberation.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/liberation.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.o `test -f 'erasure-code/jerasure/jerasure/src/liberation.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/liberation.c + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.obj: erasure-code/jerasure/jerasure/src/liberation.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.obj -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-liberation.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.obj `if test -f 'erasure-code/jerasure/jerasure/src/liberation.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/liberation.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/liberation.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-liberation.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-liberation.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/liberation.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-liberation.obj `if test -f 'erasure-code/jerasure/jerasure/src/liberation.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/liberation.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/liberation.c'; fi` + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.o: erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.o -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-reed_sol.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.o `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-reed_sol.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-reed_sol.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/reed_sol.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.o `test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c' || echo '$(srcdir)/'`erasure-code/jerasure/jerasure/src/reed_sol.c + +erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.obj: erasure-code/jerasure/jerasure/src/reed_sol.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.obj -MD -MP -MF erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-reed_sol.Tpo -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.obj `if test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/reed_sol.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/reed_sol.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-reed_sol.Tpo erasure-code/jerasure/jerasure/src/$(DEPDIR)/unittest_erasure_code_jerasure-reed_sol.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/jerasure/src/reed_sol.c' object='erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/jerasure/src/unittest_erasure_code_jerasure-reed_sol.obj `if test -f 'erasure-code/jerasure/jerasure/src/reed_sol.c'; then $(CYGPATH_W) 'erasure-code/jerasure/jerasure/src/reed_sol.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/jerasure/src/reed_sol.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.o: erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_wgen.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_wgen.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_wgen.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_wgen.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_wgen.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.obj: erasure-code/jerasure/gf-complete/src/gf_wgen.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_wgen.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_wgen.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_wgen.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_wgen.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_wgen.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_wgen.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_wgen.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_wgen.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_wgen.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_wgen.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.o: erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_method.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_method.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_method.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_method.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_method.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.obj: erasure-code/jerasure/gf-complete/src/gf_method.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_method.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_method.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_method.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_method.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_method.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_method.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_method.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_method.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_method.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_method.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.o: erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w16.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w16.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w16.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w16.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w16.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.obj: erasure-code/jerasure/gf-complete/src/gf_w16.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w16.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w16.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w16.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w16.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w16.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w16.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w16.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w16.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w16.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w16.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.o: erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.o `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.o `test -f 'erasure-code/jerasure/gf-complete/src/gf.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.obj: erasure-code/jerasure/gf-complete/src/gf.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.o: erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w32.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w32.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w32.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w32.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w32.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.obj: erasure-code/jerasure/gf-complete/src/gf_w32.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w32.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w32.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w32.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w32.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w32.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w32.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w32.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w32.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w32.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w32.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.o: erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w64.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w64.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w64.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w64.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w64.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.obj: erasure-code/jerasure/gf-complete/src/gf_w64.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w64.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w64.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w64.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w64.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w64.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w64.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w64.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w64.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w64.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w64.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.o: erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w128.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w128.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w128.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w128.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w128.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.obj: erasure-code/jerasure/gf-complete/src/gf_w128.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w128.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w128.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w128.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w128.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w128.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w128.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w128.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w128.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w128.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w128.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.o: erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_general.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_general.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_general.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_general.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_general.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.obj: erasure-code/jerasure/gf-complete/src/gf_general.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_general.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_general.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_general.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_general.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_general.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_general.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_general.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_general.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_general.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_general.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.o: erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w4.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w4.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w4.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w4.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w4.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.obj: erasure-code/jerasure/gf-complete/src/gf_w4.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w4.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w4.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w4.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w4.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w4.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w4.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w4.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w4.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w4.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w4.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.o: erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_rand.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_rand.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_rand.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_rand.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_rand.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.obj: erasure-code/jerasure/gf-complete/src/gf_rand.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_rand.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_rand.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_rand.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_rand.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_rand.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_rand.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_rand.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_rand.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_rand.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_rand.c'; fi` + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.o: erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.o -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w8.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w8.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w8.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w8.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.o `test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c' || echo '$(srcdir)/'`erasure-code/jerasure/gf-complete/src/gf_w8.c + +erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.obj: erasure-code/jerasure/gf-complete/src/gf_w8.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -MT erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.obj -MD -MP -MF erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w8.Tpo -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w8.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w8.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w8.Tpo erasure-code/jerasure/gf-complete/src/$(DEPDIR)/unittest_erasure_code_jerasure-gf_w8.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='erasure-code/jerasure/gf-complete/src/gf_w8.c' object='erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CFLAGS) $(CFLAGS) -c -o erasure-code/jerasure/gf-complete/src/unittest_erasure_code_jerasure-gf_w8.obj `if test -f 'erasure-code/jerasure/gf-complete/src/gf_w8.c'; then $(CYGPATH_W) 'erasure-code/jerasure/gf-complete/src/gf_w8.c'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/gf-complete/src/gf_w8.c'; fi` + +.cc.o: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< + +os/libos_zfs_a-ZFS.o: os/ZFS.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_zfs_a_CXXFLAGS) $(CXXFLAGS) -MT os/libos_zfs_a-ZFS.o -MD -MP -MF os/$(DEPDIR)/libos_zfs_a-ZFS.Tpo -c -o os/libos_zfs_a-ZFS.o `test -f 'os/ZFS.cc' || echo '$(srcdir)/'`os/ZFS.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_zfs_a-ZFS.Tpo os/$(DEPDIR)/libos_zfs_a-ZFS.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/ZFS.cc' object='os/libos_zfs_a-ZFS.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_zfs_a_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_zfs_a-ZFS.o `test -f 'os/ZFS.cc' || echo '$(srcdir)/'`os/ZFS.cc + +os/libos_zfs_a-ZFS.obj: os/ZFS.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_zfs_a_CXXFLAGS) $(CXXFLAGS) -MT os/libos_zfs_a-ZFS.obj -MD -MP -MF os/$(DEPDIR)/libos_zfs_a-ZFS.Tpo -c -o os/libos_zfs_a-ZFS.obj `if test -f 'os/ZFS.cc'; then $(CYGPATH_W) 'os/ZFS.cc'; else $(CYGPATH_W) '$(srcdir)/os/ZFS.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_zfs_a-ZFS.Tpo os/$(DEPDIR)/libos_zfs_a-ZFS.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/ZFS.cc' object='os/libos_zfs_a-ZFS.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_zfs_a_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_zfs_a-ZFS.obj `if test -f 'os/ZFS.cc'; then $(CYGPATH_W) 'os/ZFS.cc'; else $(CYGPATH_W) '$(srcdir)/os/ZFS.cc'; fi` + +java/native/libcephfs_jni_la-libcephfs_jni.lo: java/native/libcephfs_jni.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcephfs_jni_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT java/native/libcephfs_jni_la-libcephfs_jni.lo -MD -MP -MF java/native/$(DEPDIR)/libcephfs_jni_la-libcephfs_jni.Tpo -c -o java/native/libcephfs_jni_la-libcephfs_jni.lo `test -f 'java/native/libcephfs_jni.cc' || echo '$(srcdir)/'`java/native/libcephfs_jni.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) java/native/$(DEPDIR)/libcephfs_jni_la-libcephfs_jni.Tpo java/native/$(DEPDIR)/libcephfs_jni_la-libcephfs_jni.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='java/native/libcephfs_jni.cc' object='java/native/libcephfs_jni_la-libcephfs_jni.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcephfs_jni_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o java/native/libcephfs_jni_la-libcephfs_jni.lo `test -f 'java/native/libcephfs_jni.cc' || echo '$(srcdir)/'`java/native/libcephfs_jni.cc + +java/native/libcephfs_jni_la-JniConstants.lo: java/native/JniConstants.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcephfs_jni_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT java/native/libcephfs_jni_la-JniConstants.lo -MD -MP -MF java/native/$(DEPDIR)/libcephfs_jni_la-JniConstants.Tpo -c -o java/native/libcephfs_jni_la-JniConstants.lo `test -f 'java/native/JniConstants.cpp' || echo '$(srcdir)/'`java/native/JniConstants.cpp +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) java/native/$(DEPDIR)/libcephfs_jni_la-JniConstants.Tpo java/native/$(DEPDIR)/libcephfs_jni_la-JniConstants.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='java/native/JniConstants.cpp' object='java/native/libcephfs_jni_la-JniConstants.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcephfs_jni_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o java/native/libcephfs_jni_la-JniConstants.lo `test -f 'java/native/JniConstants.cpp' || echo '$(srcdir)/'`java/native/JniConstants.cpp + +common/libcommon_crc_la-crc32c.lo: common/crc32c.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT common/libcommon_crc_la-crc32c.lo -MD -MP -MF common/$(DEPDIR)/libcommon_crc_la-crc32c.Tpo -c -o common/libcommon_crc_la-crc32c.lo `test -f 'common/crc32c.cc' || echo '$(srcdir)/'`common/crc32c.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libcommon_crc_la-crc32c.Tpo common/$(DEPDIR)/libcommon_crc_la-crc32c.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/crc32c.cc' object='common/libcommon_crc_la-crc32c.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(libcommon_crc_la_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o common/libcommon_crc_la-crc32c.lo `test -f 'common/crc32c.cc' || echo '$(srcdir)/'`common/crc32c.cc + +test/erasure-code/libec_example_la-ErasureCodePluginExample.lo: test/erasure-code/ErasureCodePluginExample.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_example_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_example_la-ErasureCodePluginExample.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Tpo -c -o test/erasure-code/libec_example_la-ErasureCodePluginExample.lo `test -f 'test/erasure-code/ErasureCodePluginExample.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginExample.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Tpo test/erasure-code/$(DEPDIR)/libec_example_la-ErasureCodePluginExample.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginExample.cc' object='test/erasure-code/libec_example_la-ErasureCodePluginExample.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_example_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_example_la-ErasureCodePluginExample.lo `test -f 'test/erasure-code/ErasureCodePluginExample.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginExample.cc + +test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo: test/erasure-code/ErasureCodePluginFailToInitialize.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_initialize_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Tpo -c -o test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo `test -f 'test/erasure-code/ErasureCodePluginFailToInitialize.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginFailToInitialize.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Tpo test/erasure-code/$(DEPDIR)/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginFailToInitialize.cc' object='test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_initialize_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_fail_to_initialize_la-ErasureCodePluginFailToInitialize.lo `test -f 'test/erasure-code/ErasureCodePluginFailToInitialize.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginFailToInitialize.cc + +test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo: test/erasure-code/ErasureCodePluginFailToRegister.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_register_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Tpo -c -o test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo `test -f 'test/erasure-code/ErasureCodePluginFailToRegister.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginFailToRegister.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Tpo test/erasure-code/$(DEPDIR)/libec_fail_to_register_la-ErasureCodePluginFailToRegister.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginFailToRegister.cc' object='test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_fail_to_register_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_fail_to_register_la-ErasureCodePluginFailToRegister.lo `test -f 'test/erasure-code/ErasureCodePluginFailToRegister.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginFailToRegister.cc + +test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo: test/erasure-code/ErasureCodePluginHangs.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Tpo -c -o test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo `test -f 'test/erasure-code/ErasureCodePluginHangs.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginHangs.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Tpo test/erasure-code/$(DEPDIR)/libec_hangs_la-ErasureCodePluginHangs.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginHangs.cc' object='test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_hangs_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_hangs_la-ErasureCodePluginHangs.lo `test -f 'test/erasure-code/ErasureCodePluginHangs.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginHangs.cc + +erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo: erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginSelectJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginSelectJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_la-ErasureCodePluginSelectJerasure.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc' object='erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_la-ErasureCodePluginSelectJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginSelectJerasure.cc + +erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo: erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodePluginJerasure.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodePluginJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc + +erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo: erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_generic_la-ErasureCodeJerasure.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_generic_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc + +erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo: erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodePluginJerasure.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodePluginJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc + +erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo: erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse3_la-ErasureCodeJerasure.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_sse3_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc + +erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo: erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodePluginJerasure.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodePluginJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc + +erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo: erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/libec_jerasure_sse4_la-ErasureCodeJerasure.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/libec_jerasure_sse4_la-ErasureCodeJerasure.lo `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc + +test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo: test/erasure-code/ErasureCodePluginMissingEntryPoint.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Tpo -c -o test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo `test -f 'test/erasure-code/ErasureCodePluginMissingEntryPoint.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginMissingEntryPoint.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Tpo test/erasure-code/$(DEPDIR)/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/ErasureCodePluginMissingEntryPoint.cc' object='test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_missing_entry_point_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_missing_entry_point_la-ErasureCodePluginMissingEntryPoint.lo `test -f 'test/erasure-code/ErasureCodePluginMissingEntryPoint.cc' || echo '$(srcdir)/'`test/erasure-code/ErasureCodePluginMissingEntryPoint.cc + +test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo: test/erasure-code/TestJerasurePluginGeneric.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_test_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.Tpo -c -o test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo `test -f 'test/erasure-code/TestJerasurePluginGeneric.cc' || echo '$(srcdir)/'`test/erasure-code/TestJerasurePluginGeneric.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.Tpo test/erasure-code/$(DEPDIR)/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestJerasurePluginGeneric.cc' object='test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_test_jerasure_generic_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_test_jerasure_generic_la-TestJerasurePluginGeneric.lo `test -f 'test/erasure-code/TestJerasurePluginGeneric.cc' || echo '$(srcdir)/'`test/erasure-code/TestJerasurePluginGeneric.cc + +test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo: test/erasure-code/TestJerasurePluginSSE3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_test_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.Tpo -c -o test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo `test -f 'test/erasure-code/TestJerasurePluginSSE3.cc' || echo '$(srcdir)/'`test/erasure-code/TestJerasurePluginSSE3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.Tpo test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestJerasurePluginSSE3.cc' object='test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_test_jerasure_sse3_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_test_jerasure_sse3_la-TestJerasurePluginSSE3.lo `test -f 'test/erasure-code/TestJerasurePluginSSE3.cc' || echo '$(srcdir)/'`test/erasure-code/TestJerasurePluginSSE3.cc + +test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo: test/erasure-code/TestJerasurePluginSSE4.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_test_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo -MD -MP -MF test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.Tpo -c -o test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo `test -f 'test/erasure-code/TestJerasurePluginSSE4.cc' || echo '$(srcdir)/'`test/erasure-code/TestJerasurePluginSSE4.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.Tpo test/erasure-code/$(DEPDIR)/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestJerasurePluginSSE4.cc' object='test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libec_test_jerasure_sse4_la_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/libec_test_jerasure_sse4_la-TestJerasurePluginSSE4.lo `test -f 'test/erasure-code/TestJerasurePluginSSE4.cc' || echo '$(srcdir)/'`test/erasure-code/TestJerasurePluginSSE4.cc + +os/libos_la-chain_xattr.lo: os/chain_xattr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-chain_xattr.lo -MD -MP -MF os/$(DEPDIR)/libos_la-chain_xattr.Tpo -c -o os/libos_la-chain_xattr.lo `test -f 'os/chain_xattr.cc' || echo '$(srcdir)/'`os/chain_xattr.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-chain_xattr.Tpo os/$(DEPDIR)/libos_la-chain_xattr.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/chain_xattr.cc' object='os/libos_la-chain_xattr.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-chain_xattr.lo `test -f 'os/chain_xattr.cc' || echo '$(srcdir)/'`os/chain_xattr.cc + +os/libos_la-DBObjectMap.lo: os/DBObjectMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-DBObjectMap.lo -MD -MP -MF os/$(DEPDIR)/libos_la-DBObjectMap.Tpo -c -o os/libos_la-DBObjectMap.lo `test -f 'os/DBObjectMap.cc' || echo '$(srcdir)/'`os/DBObjectMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-DBObjectMap.Tpo os/$(DEPDIR)/libos_la-DBObjectMap.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/DBObjectMap.cc' object='os/libos_la-DBObjectMap.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-DBObjectMap.lo `test -f 'os/DBObjectMap.cc' || echo '$(srcdir)/'`os/DBObjectMap.cc + +os/libos_la-GenericObjectMap.lo: os/GenericObjectMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-GenericObjectMap.lo -MD -MP -MF os/$(DEPDIR)/libos_la-GenericObjectMap.Tpo -c -o os/libos_la-GenericObjectMap.lo `test -f 'os/GenericObjectMap.cc' || echo '$(srcdir)/'`os/GenericObjectMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-GenericObjectMap.Tpo os/$(DEPDIR)/libos_la-GenericObjectMap.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/GenericObjectMap.cc' object='os/libos_la-GenericObjectMap.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-GenericObjectMap.lo `test -f 'os/GenericObjectMap.cc' || echo '$(srcdir)/'`os/GenericObjectMap.cc + +os/libos_la-FileJournal.lo: os/FileJournal.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-FileJournal.lo -MD -MP -MF os/$(DEPDIR)/libos_la-FileJournal.Tpo -c -o os/libos_la-FileJournal.lo `test -f 'os/FileJournal.cc' || echo '$(srcdir)/'`os/FileJournal.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-FileJournal.Tpo os/$(DEPDIR)/libos_la-FileJournal.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/FileJournal.cc' object='os/libos_la-FileJournal.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-FileJournal.lo `test -f 'os/FileJournal.cc' || echo '$(srcdir)/'`os/FileJournal.cc + +os/libos_la-FileStore.lo: os/FileStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-FileStore.lo -MD -MP -MF os/$(DEPDIR)/libos_la-FileStore.Tpo -c -o os/libos_la-FileStore.lo `test -f 'os/FileStore.cc' || echo '$(srcdir)/'`os/FileStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-FileStore.Tpo os/$(DEPDIR)/libos_la-FileStore.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/FileStore.cc' object='os/libos_la-FileStore.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-FileStore.lo `test -f 'os/FileStore.cc' || echo '$(srcdir)/'`os/FileStore.cc + +os/libos_la-FlatIndex.lo: os/FlatIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-FlatIndex.lo -MD -MP -MF os/$(DEPDIR)/libos_la-FlatIndex.Tpo -c -o os/libos_la-FlatIndex.lo `test -f 'os/FlatIndex.cc' || echo '$(srcdir)/'`os/FlatIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-FlatIndex.Tpo os/$(DEPDIR)/libos_la-FlatIndex.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/FlatIndex.cc' object='os/libos_la-FlatIndex.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-FlatIndex.lo `test -f 'os/FlatIndex.cc' || echo '$(srcdir)/'`os/FlatIndex.cc + +os/libos_la-GenericFileStoreBackend.lo: os/GenericFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-GenericFileStoreBackend.lo -MD -MP -MF os/$(DEPDIR)/libos_la-GenericFileStoreBackend.Tpo -c -o os/libos_la-GenericFileStoreBackend.lo `test -f 'os/GenericFileStoreBackend.cc' || echo '$(srcdir)/'`os/GenericFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-GenericFileStoreBackend.Tpo os/$(DEPDIR)/libos_la-GenericFileStoreBackend.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/GenericFileStoreBackend.cc' object='os/libos_la-GenericFileStoreBackend.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-GenericFileStoreBackend.lo `test -f 'os/GenericFileStoreBackend.cc' || echo '$(srcdir)/'`os/GenericFileStoreBackend.cc + +os/libos_la-HashIndex.lo: os/HashIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-HashIndex.lo -MD -MP -MF os/$(DEPDIR)/libos_la-HashIndex.Tpo -c -o os/libos_la-HashIndex.lo `test -f 'os/HashIndex.cc' || echo '$(srcdir)/'`os/HashIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-HashIndex.Tpo os/$(DEPDIR)/libos_la-HashIndex.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/HashIndex.cc' object='os/libos_la-HashIndex.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-HashIndex.lo `test -f 'os/HashIndex.cc' || echo '$(srcdir)/'`os/HashIndex.cc + +os/libos_la-IndexManager.lo: os/IndexManager.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-IndexManager.lo -MD -MP -MF os/$(DEPDIR)/libos_la-IndexManager.Tpo -c -o os/libos_la-IndexManager.lo `test -f 'os/IndexManager.cc' || echo '$(srcdir)/'`os/IndexManager.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-IndexManager.Tpo os/$(DEPDIR)/libos_la-IndexManager.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/IndexManager.cc' object='os/libos_la-IndexManager.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-IndexManager.lo `test -f 'os/IndexManager.cc' || echo '$(srcdir)/'`os/IndexManager.cc + +os/libos_la-JournalingObjectStore.lo: os/JournalingObjectStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-JournalingObjectStore.lo -MD -MP -MF os/$(DEPDIR)/libos_la-JournalingObjectStore.Tpo -c -o os/libos_la-JournalingObjectStore.lo `test -f 'os/JournalingObjectStore.cc' || echo '$(srcdir)/'`os/JournalingObjectStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-JournalingObjectStore.Tpo os/$(DEPDIR)/libos_la-JournalingObjectStore.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/JournalingObjectStore.cc' object='os/libos_la-JournalingObjectStore.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-JournalingObjectStore.lo `test -f 'os/JournalingObjectStore.cc' || echo '$(srcdir)/'`os/JournalingObjectStore.cc + +os/libos_la-LevelDBStore.lo: os/LevelDBStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-LevelDBStore.lo -MD -MP -MF os/$(DEPDIR)/libos_la-LevelDBStore.Tpo -c -o os/libos_la-LevelDBStore.lo `test -f 'os/LevelDBStore.cc' || echo '$(srcdir)/'`os/LevelDBStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-LevelDBStore.Tpo os/$(DEPDIR)/libos_la-LevelDBStore.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/LevelDBStore.cc' object='os/libos_la-LevelDBStore.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-LevelDBStore.lo `test -f 'os/LevelDBStore.cc' || echo '$(srcdir)/'`os/LevelDBStore.cc + +os/libos_la-LFNIndex.lo: os/LFNIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-LFNIndex.lo -MD -MP -MF os/$(DEPDIR)/libos_la-LFNIndex.Tpo -c -o os/libos_la-LFNIndex.lo `test -f 'os/LFNIndex.cc' || echo '$(srcdir)/'`os/LFNIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-LFNIndex.Tpo os/$(DEPDIR)/libos_la-LFNIndex.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/LFNIndex.cc' object='os/libos_la-LFNIndex.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-LFNIndex.lo `test -f 'os/LFNIndex.cc' || echo '$(srcdir)/'`os/LFNIndex.cc + +os/libos_la-MemStore.lo: os/MemStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-MemStore.lo -MD -MP -MF os/$(DEPDIR)/libos_la-MemStore.Tpo -c -o os/libos_la-MemStore.lo `test -f 'os/MemStore.cc' || echo '$(srcdir)/'`os/MemStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-MemStore.Tpo os/$(DEPDIR)/libos_la-MemStore.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/MemStore.cc' object='os/libos_la-MemStore.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-MemStore.lo `test -f 'os/MemStore.cc' || echo '$(srcdir)/'`os/MemStore.cc + +os/libos_la-KeyValueStore.lo: os/KeyValueStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-KeyValueStore.lo -MD -MP -MF os/$(DEPDIR)/libos_la-KeyValueStore.Tpo -c -o os/libos_la-KeyValueStore.lo `test -f 'os/KeyValueStore.cc' || echo '$(srcdir)/'`os/KeyValueStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-KeyValueStore.Tpo os/$(DEPDIR)/libos_la-KeyValueStore.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/KeyValueStore.cc' object='os/libos_la-KeyValueStore.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-KeyValueStore.lo `test -f 'os/KeyValueStore.cc' || echo '$(srcdir)/'`os/KeyValueStore.cc + +os/libos_la-ObjectStore.lo: os/ObjectStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-ObjectStore.lo -MD -MP -MF os/$(DEPDIR)/libos_la-ObjectStore.Tpo -c -o os/libos_la-ObjectStore.lo `test -f 'os/ObjectStore.cc' || echo '$(srcdir)/'`os/ObjectStore.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-ObjectStore.Tpo os/$(DEPDIR)/libos_la-ObjectStore.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/ObjectStore.cc' object='os/libos_la-ObjectStore.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-ObjectStore.lo `test -f 'os/ObjectStore.cc' || echo '$(srcdir)/'`os/ObjectStore.cc + +os/libos_la-WBThrottle.lo: os/WBThrottle.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-WBThrottle.lo -MD -MP -MF os/$(DEPDIR)/libos_la-WBThrottle.Tpo -c -o os/libos_la-WBThrottle.lo `test -f 'os/WBThrottle.cc' || echo '$(srcdir)/'`os/WBThrottle.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-WBThrottle.Tpo os/$(DEPDIR)/libos_la-WBThrottle.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/WBThrottle.cc' object='os/libos_la-WBThrottle.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-WBThrottle.lo `test -f 'os/WBThrottle.cc' || echo '$(srcdir)/'`os/WBThrottle.cc + +common/libos_la-TrackedOp.lo: common/TrackedOp.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT common/libos_la-TrackedOp.lo -MD -MP -MF common/$(DEPDIR)/libos_la-TrackedOp.Tpo -c -o common/libos_la-TrackedOp.lo `test -f 'common/TrackedOp.cc' || echo '$(srcdir)/'`common/TrackedOp.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libos_la-TrackedOp.Tpo common/$(DEPDIR)/libos_la-TrackedOp.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/TrackedOp.cc' object='common/libos_la-TrackedOp.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o common/libos_la-TrackedOp.lo `test -f 'common/TrackedOp.cc' || echo '$(srcdir)/'`common/TrackedOp.cc + +os/libos_la-BtrfsFileStoreBackend.lo: os/BtrfsFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-BtrfsFileStoreBackend.lo -MD -MP -MF os/$(DEPDIR)/libos_la-BtrfsFileStoreBackend.Tpo -c -o os/libos_la-BtrfsFileStoreBackend.lo `test -f 'os/BtrfsFileStoreBackend.cc' || echo '$(srcdir)/'`os/BtrfsFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-BtrfsFileStoreBackend.Tpo os/$(DEPDIR)/libos_la-BtrfsFileStoreBackend.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/BtrfsFileStoreBackend.cc' object='os/libos_la-BtrfsFileStoreBackend.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-BtrfsFileStoreBackend.lo `test -f 'os/BtrfsFileStoreBackend.cc' || echo '$(srcdir)/'`os/BtrfsFileStoreBackend.cc + +os/libos_la-XfsFileStoreBackend.lo: os/XfsFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-XfsFileStoreBackend.lo -MD -MP -MF os/$(DEPDIR)/libos_la-XfsFileStoreBackend.Tpo -c -o os/libos_la-XfsFileStoreBackend.lo `test -f 'os/XfsFileStoreBackend.cc' || echo '$(srcdir)/'`os/XfsFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-XfsFileStoreBackend.Tpo os/$(DEPDIR)/libos_la-XfsFileStoreBackend.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/XfsFileStoreBackend.cc' object='os/libos_la-XfsFileStoreBackend.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-XfsFileStoreBackend.lo `test -f 'os/XfsFileStoreBackend.cc' || echo '$(srcdir)/'`os/XfsFileStoreBackend.cc + +os/libos_la-ZFSFileStoreBackend.lo: os/ZFSFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_la-ZFSFileStoreBackend.lo -MD -MP -MF os/$(DEPDIR)/libos_la-ZFSFileStoreBackend.Tpo -c -o os/libos_la-ZFSFileStoreBackend.lo `test -f 'os/ZFSFileStoreBackend.cc' || echo '$(srcdir)/'`os/ZFSFileStoreBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_la-ZFSFileStoreBackend.Tpo os/$(DEPDIR)/libos_la-ZFSFileStoreBackend.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/ZFSFileStoreBackend.cc' object='os/libos_la-ZFSFileStoreBackend.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_la-ZFSFileStoreBackend.lo `test -f 'os/ZFSFileStoreBackend.cc' || echo '$(srcdir)/'`os/ZFSFileStoreBackend.cc + +os/libos_types_la-Transaction.lo: os/Transaction.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_types_la_CXXFLAGS) $(CXXFLAGS) -MT os/libos_types_la-Transaction.lo -MD -MP -MF os/$(DEPDIR)/libos_types_la-Transaction.Tpo -c -o os/libos_types_la-Transaction.lo `test -f 'os/Transaction.cc' || echo '$(srcdir)/'`os/Transaction.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) os/$(DEPDIR)/libos_types_la-Transaction.Tpo os/$(DEPDIR)/libos_types_la-Transaction.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='os/Transaction.cc' object='os/libos_types_la-Transaction.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libos_types_la_CXXFLAGS) $(CXXFLAGS) -c -o os/libos_types_la-Transaction.lo `test -f 'os/Transaction.cc' || echo '$(srcdir)/'`os/Transaction.cc + +osd/libosd_la-PG.lo: osd/PG.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-PG.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-PG.Tpo -c -o osd/libosd_la-PG.lo `test -f 'osd/PG.cc' || echo '$(srcdir)/'`osd/PG.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-PG.Tpo osd/$(DEPDIR)/libosd_la-PG.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/PG.cc' object='osd/libosd_la-PG.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-PG.lo `test -f 'osd/PG.cc' || echo '$(srcdir)/'`osd/PG.cc + +osd/libosd_la-ReplicatedPG.lo: osd/ReplicatedPG.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-ReplicatedPG.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-ReplicatedPG.Tpo -c -o osd/libosd_la-ReplicatedPG.lo `test -f 'osd/ReplicatedPG.cc' || echo '$(srcdir)/'`osd/ReplicatedPG.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-ReplicatedPG.Tpo osd/$(DEPDIR)/libosd_la-ReplicatedPG.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ReplicatedPG.cc' object='osd/libosd_la-ReplicatedPG.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-ReplicatedPG.lo `test -f 'osd/ReplicatedPG.cc' || echo '$(srcdir)/'`osd/ReplicatedPG.cc + +osd/libosd_la-ReplicatedBackend.lo: osd/ReplicatedBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-ReplicatedBackend.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-ReplicatedBackend.Tpo -c -o osd/libosd_la-ReplicatedBackend.lo `test -f 'osd/ReplicatedBackend.cc' || echo '$(srcdir)/'`osd/ReplicatedBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-ReplicatedBackend.Tpo osd/$(DEPDIR)/libosd_la-ReplicatedBackend.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ReplicatedBackend.cc' object='osd/libosd_la-ReplicatedBackend.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-ReplicatedBackend.lo `test -f 'osd/ReplicatedBackend.cc' || echo '$(srcdir)/'`osd/ReplicatedBackend.cc + +osd/libosd_la-ECBackend.lo: osd/ECBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-ECBackend.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-ECBackend.Tpo -c -o osd/libosd_la-ECBackend.lo `test -f 'osd/ECBackend.cc' || echo '$(srcdir)/'`osd/ECBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-ECBackend.Tpo osd/$(DEPDIR)/libosd_la-ECBackend.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECBackend.cc' object='osd/libosd_la-ECBackend.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-ECBackend.lo `test -f 'osd/ECBackend.cc' || echo '$(srcdir)/'`osd/ECBackend.cc + +osd/libosd_la-ECMsgTypes.lo: osd/ECMsgTypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-ECMsgTypes.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-ECMsgTypes.Tpo -c -o osd/libosd_la-ECMsgTypes.lo `test -f 'osd/ECMsgTypes.cc' || echo '$(srcdir)/'`osd/ECMsgTypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-ECMsgTypes.Tpo osd/$(DEPDIR)/libosd_la-ECMsgTypes.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECMsgTypes.cc' object='osd/libosd_la-ECMsgTypes.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-ECMsgTypes.lo `test -f 'osd/ECMsgTypes.cc' || echo '$(srcdir)/'`osd/ECMsgTypes.cc + +osd/libosd_la-ECTransaction.lo: osd/ECTransaction.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-ECTransaction.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-ECTransaction.Tpo -c -o osd/libosd_la-ECTransaction.lo `test -f 'osd/ECTransaction.cc' || echo '$(srcdir)/'`osd/ECTransaction.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-ECTransaction.Tpo osd/$(DEPDIR)/libosd_la-ECTransaction.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECTransaction.cc' object='osd/libosd_la-ECTransaction.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-ECTransaction.lo `test -f 'osd/ECTransaction.cc' || echo '$(srcdir)/'`osd/ECTransaction.cc + +osd/libosd_la-PGBackend.lo: osd/PGBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-PGBackend.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-PGBackend.Tpo -c -o osd/libosd_la-PGBackend.lo `test -f 'osd/PGBackend.cc' || echo '$(srcdir)/'`osd/PGBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-PGBackend.Tpo osd/$(DEPDIR)/libosd_la-PGBackend.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/PGBackend.cc' object='osd/libosd_la-PGBackend.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-PGBackend.lo `test -f 'osd/PGBackend.cc' || echo '$(srcdir)/'`osd/PGBackend.cc + +osd/libosd_la-Ager.lo: osd/Ager.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-Ager.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-Ager.Tpo -c -o osd/libosd_la-Ager.lo `test -f 'osd/Ager.cc' || echo '$(srcdir)/'`osd/Ager.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-Ager.Tpo osd/$(DEPDIR)/libosd_la-Ager.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/Ager.cc' object='osd/libosd_la-Ager.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-Ager.lo `test -f 'osd/Ager.cc' || echo '$(srcdir)/'`osd/Ager.cc + +osd/libosd_la-HitSet.lo: osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-HitSet.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-HitSet.Tpo -c -o osd/libosd_la-HitSet.lo `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-HitSet.Tpo osd/$(DEPDIR)/libosd_la-HitSet.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/libosd_la-HitSet.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-HitSet.lo `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc + +osd/libosd_la-OSD.lo: osd/OSD.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-OSD.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-OSD.Tpo -c -o osd/libosd_la-OSD.lo `test -f 'osd/OSD.cc' || echo '$(srcdir)/'`osd/OSD.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-OSD.Tpo osd/$(DEPDIR)/libosd_la-OSD.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/OSD.cc' object='osd/libosd_la-OSD.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-OSD.lo `test -f 'osd/OSD.cc' || echo '$(srcdir)/'`osd/OSD.cc + +osd/libosd_la-OSDCap.lo: osd/OSDCap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-OSDCap.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-OSDCap.Tpo -c -o osd/libosd_la-OSDCap.lo `test -f 'osd/OSDCap.cc' || echo '$(srcdir)/'`osd/OSDCap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-OSDCap.Tpo osd/$(DEPDIR)/libosd_la-OSDCap.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/OSDCap.cc' object='osd/libosd_la-OSDCap.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-OSDCap.lo `test -f 'osd/OSDCap.cc' || echo '$(srcdir)/'`osd/OSDCap.cc + +osd/libosd_la-Watch.lo: osd/Watch.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-Watch.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-Watch.Tpo -c -o osd/libosd_la-Watch.lo `test -f 'osd/Watch.cc' || echo '$(srcdir)/'`osd/Watch.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-Watch.Tpo osd/$(DEPDIR)/libosd_la-Watch.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/Watch.cc' object='osd/libosd_la-Watch.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-Watch.lo `test -f 'osd/Watch.cc' || echo '$(srcdir)/'`osd/Watch.cc + +osd/libosd_la-ClassHandler.lo: osd/ClassHandler.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-ClassHandler.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-ClassHandler.Tpo -c -o osd/libosd_la-ClassHandler.lo `test -f 'osd/ClassHandler.cc' || echo '$(srcdir)/'`osd/ClassHandler.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-ClassHandler.Tpo osd/$(DEPDIR)/libosd_la-ClassHandler.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ClassHandler.cc' object='osd/libosd_la-ClassHandler.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-ClassHandler.lo `test -f 'osd/ClassHandler.cc' || echo '$(srcdir)/'`osd/ClassHandler.cc + +osd/libosd_la-OpRequest.lo: osd/OpRequest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-OpRequest.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-OpRequest.Tpo -c -o osd/libosd_la-OpRequest.lo `test -f 'osd/OpRequest.cc' || echo '$(srcdir)/'`osd/OpRequest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-OpRequest.Tpo osd/$(DEPDIR)/libosd_la-OpRequest.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/OpRequest.cc' object='osd/libosd_la-OpRequest.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-OpRequest.lo `test -f 'osd/OpRequest.cc' || echo '$(srcdir)/'`osd/OpRequest.cc + +common/libosd_la-TrackedOp.lo: common/TrackedOp.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT common/libosd_la-TrackedOp.lo -MD -MP -MF common/$(DEPDIR)/libosd_la-TrackedOp.Tpo -c -o common/libosd_la-TrackedOp.lo `test -f 'common/TrackedOp.cc' || echo '$(srcdir)/'`common/TrackedOp.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/libosd_la-TrackedOp.Tpo common/$(DEPDIR)/libosd_la-TrackedOp.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/TrackedOp.cc' object='common/libosd_la-TrackedOp.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o common/libosd_la-TrackedOp.lo `test -f 'common/TrackedOp.cc' || echo '$(srcdir)/'`common/TrackedOp.cc + +osd/libosd_la-SnapMapper.lo: osd/SnapMapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_la-SnapMapper.lo -MD -MP -MF osd/$(DEPDIR)/libosd_la-SnapMapper.Tpo -c -o osd/libosd_la-SnapMapper.lo `test -f 'osd/SnapMapper.cc' || echo '$(srcdir)/'`osd/SnapMapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_la-SnapMapper.Tpo osd/$(DEPDIR)/libosd_la-SnapMapper.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/SnapMapper.cc' object='osd/libosd_la-SnapMapper.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_la-SnapMapper.lo `test -f 'osd/SnapMapper.cc' || echo '$(srcdir)/'`osd/SnapMapper.cc + +objclass/libosd_la-class_api.lo: objclass/class_api.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -MT objclass/libosd_la-class_api.lo -MD -MP -MF objclass/$(DEPDIR)/libosd_la-class_api.Tpo -c -o objclass/libosd_la-class_api.lo `test -f 'objclass/class_api.cc' || echo '$(srcdir)/'`objclass/class_api.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) objclass/$(DEPDIR)/libosd_la-class_api.Tpo objclass/$(DEPDIR)/libosd_la-class_api.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='objclass/class_api.cc' object='objclass/libosd_la-class_api.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_la_CXXFLAGS) $(CXXFLAGS) -c -o objclass/libosd_la-class_api.lo `test -f 'objclass/class_api.cc' || echo '$(srcdir)/'`objclass/class_api.cc + +osd/libosd_types_la-PGLog.lo: osd/PGLog.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_types_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_types_la-PGLog.lo -MD -MP -MF osd/$(DEPDIR)/libosd_types_la-PGLog.Tpo -c -o osd/libosd_types_la-PGLog.lo `test -f 'osd/PGLog.cc' || echo '$(srcdir)/'`osd/PGLog.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_types_la-PGLog.Tpo osd/$(DEPDIR)/libosd_types_la-PGLog.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/PGLog.cc' object='osd/libosd_types_la-PGLog.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_types_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_types_la-PGLog.lo `test -f 'osd/PGLog.cc' || echo '$(srcdir)/'`osd/PGLog.cc + +osd/libosd_types_la-osd_types.lo: osd/osd_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_types_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_types_la-osd_types.lo -MD -MP -MF osd/$(DEPDIR)/libosd_types_la-osd_types.Tpo -c -o osd/libosd_types_la-osd_types.lo `test -f 'osd/osd_types.cc' || echo '$(srcdir)/'`osd/osd_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_types_la-osd_types.Tpo osd/$(DEPDIR)/libosd_types_la-osd_types.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/osd_types.cc' object='osd/libosd_types_la-osd_types.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_types_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_types_la-osd_types.lo `test -f 'osd/osd_types.cc' || echo '$(srcdir)/'`osd/osd_types.cc + +osd/libosd_types_la-ECUtil.lo: osd/ECUtil.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_types_la_CXXFLAGS) $(CXXFLAGS) -MT osd/libosd_types_la-ECUtil.lo -MD -MP -MF osd/$(DEPDIR)/libosd_types_la-ECUtil.Tpo -c -o osd/libosd_types_la-ECUtil.lo `test -f 'osd/ECUtil.cc' || echo '$(srcdir)/'`osd/ECUtil.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/libosd_types_la-ECUtil.Tpo osd/$(DEPDIR)/libosd_types_la-ECUtil.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECUtil.cc' object='osd/libosd_types_la-ECUtil.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libosd_types_la_CXXFLAGS) $(CXXFLAGS) -c -o osd/libosd_types_la-ECUtil.lo `test -f 'osd/ECUtil.cc' || echo '$(srcdir)/'`osd/ECUtil.cc + +librados/librados_la-librados.lo: librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-librados.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-librados.Tpo -c -o librados/librados_la-librados.lo `test -f 'librados/librados.cc' || echo '$(srcdir)/'`librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-librados.Tpo librados/$(DEPDIR)/librados_la-librados.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/librados.cc' object='librados/librados_la-librados.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-librados.lo `test -f 'librados/librados.cc' || echo '$(srcdir)/'`librados/librados.cc + +librados/librados_la-RadosClient.lo: librados/RadosClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-RadosClient.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-RadosClient.Tpo -c -o librados/librados_la-RadosClient.lo `test -f 'librados/RadosClient.cc' || echo '$(srcdir)/'`librados/RadosClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-RadosClient.Tpo librados/$(DEPDIR)/librados_la-RadosClient.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/RadosClient.cc' object='librados/librados_la-RadosClient.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-RadosClient.lo `test -f 'librados/RadosClient.cc' || echo '$(srcdir)/'`librados/RadosClient.cc + +librados/librados_la-IoCtxImpl.lo: librados/IoCtxImpl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-IoCtxImpl.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-IoCtxImpl.Tpo -c -o librados/librados_la-IoCtxImpl.lo `test -f 'librados/IoCtxImpl.cc' || echo '$(srcdir)/'`librados/IoCtxImpl.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-IoCtxImpl.Tpo librados/$(DEPDIR)/librados_la-IoCtxImpl.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/IoCtxImpl.cc' object='librados/librados_la-IoCtxImpl.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-IoCtxImpl.lo `test -f 'librados/IoCtxImpl.cc' || echo '$(srcdir)/'`librados/IoCtxImpl.cc + +librados/librados_la-snap_set_diff.lo: librados/snap_set_diff.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -MT librados/librados_la-snap_set_diff.lo -MD -MP -MF librados/$(DEPDIR)/librados_la-snap_set_diff.Tpo -c -o librados/librados_la-snap_set_diff.lo `test -f 'librados/snap_set_diff.cc' || echo '$(srcdir)/'`librados/snap_set_diff.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/librados_la-snap_set_diff.Tpo librados/$(DEPDIR)/librados_la-snap_set_diff.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/snap_set_diff.cc' object='librados/librados_la-snap_set_diff.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librados_la_CXXFLAGS) $(CXXFLAGS) -c -o librados/librados_la-snap_set_diff.lo `test -f 'librados/snap_set_diff.cc' || echo '$(srcdir)/'`librados/snap_set_diff.cc + +test/librados/libradostest_la-test.lo: test/librados/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -MT test/librados/libradostest_la-test.lo -MD -MP -MF test/librados/$(DEPDIR)/libradostest_la-test.Tpo -c -o test/librados/libradostest_la-test.lo `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/libradostest_la-test.Tpo test/librados/$(DEPDIR)/libradostest_la-test.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/test.cc' object='test/librados/libradostest_la-test.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/libradostest_la-test.lo `test -f 'test/librados/test.cc' || echo '$(srcdir)/'`test/librados/test.cc + +test/librados/libradostest_la-TestCase.lo: test/librados/TestCase.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -MT test/librados/libradostest_la-TestCase.lo -MD -MP -MF test/librados/$(DEPDIR)/libradostest_la-TestCase.Tpo -c -o test/librados/libradostest_la-TestCase.lo `test -f 'test/librados/TestCase.cc' || echo '$(srcdir)/'`test/librados/TestCase.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/libradostest_la-TestCase.Tpo test/librados/$(DEPDIR)/libradostest_la-TestCase.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/TestCase.cc' object='test/librados/libradostest_la-TestCase.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libradostest_la_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/libradostest_la-TestCase.lo `test -f 'test/librados/TestCase.cc' || echo '$(srcdir)/'`test/librados/TestCase.cc + +rgw/librgw_la-librgw.lo: rgw/librgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-librgw.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-librgw.Tpo -c -o rgw/librgw_la-librgw.lo `test -f 'rgw/librgw.cc' || echo '$(srcdir)/'`rgw/librgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-librgw.Tpo rgw/$(DEPDIR)/librgw_la-librgw.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/librgw.cc' object='rgw/librgw_la-librgw.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-librgw.lo `test -f 'rgw/librgw.cc' || echo '$(srcdir)/'`rgw/librgw.cc + +rgw/librgw_la-rgw_acl.lo: rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_acl.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_acl.Tpo -c -o rgw/librgw_la-rgw_acl.lo `test -f 'rgw/rgw_acl.cc' || echo '$(srcdir)/'`rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_acl.Tpo rgw/$(DEPDIR)/librgw_la-rgw_acl.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl.cc' object='rgw/librgw_la-rgw_acl.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_acl.lo `test -f 'rgw/rgw_acl.cc' || echo '$(srcdir)/'`rgw/rgw_acl.cc + +rgw/librgw_la-rgw_acl_s3.lo: rgw/rgw_acl_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_acl_s3.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_acl_s3.Tpo -c -o rgw/librgw_la-rgw_acl_s3.lo `test -f 'rgw/rgw_acl_s3.cc' || echo '$(srcdir)/'`rgw/rgw_acl_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_acl_s3.Tpo rgw/$(DEPDIR)/librgw_la-rgw_acl_s3.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl_s3.cc' object='rgw/librgw_la-rgw_acl_s3.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_acl_s3.lo `test -f 'rgw/rgw_acl_s3.cc' || echo '$(srcdir)/'`rgw/rgw_acl_s3.cc + +rgw/librgw_la-rgw_acl_swift.lo: rgw/rgw_acl_swift.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_acl_swift.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_acl_swift.Tpo -c -o rgw/librgw_la-rgw_acl_swift.lo `test -f 'rgw/rgw_acl_swift.cc' || echo '$(srcdir)/'`rgw/rgw_acl_swift.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_acl_swift.Tpo rgw/$(DEPDIR)/librgw_la-rgw_acl_swift.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl_swift.cc' object='rgw/librgw_la-rgw_acl_swift.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_acl_swift.lo `test -f 'rgw/rgw_acl_swift.cc' || echo '$(srcdir)/'`rgw/rgw_acl_swift.cc + +rgw/librgw_la-rgw_client_io.lo: rgw/rgw_client_io.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_client_io.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_client_io.Tpo -c -o rgw/librgw_la-rgw_client_io.lo `test -f 'rgw/rgw_client_io.cc' || echo '$(srcdir)/'`rgw/rgw_client_io.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_client_io.Tpo rgw/$(DEPDIR)/librgw_la-rgw_client_io.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_client_io.cc' object='rgw/librgw_la-rgw_client_io.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_client_io.lo `test -f 'rgw/rgw_client_io.cc' || echo '$(srcdir)/'`rgw/rgw_client_io.cc + +rgw/librgw_la-rgw_fcgi.lo: rgw/rgw_fcgi.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_fcgi.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_fcgi.Tpo -c -o rgw/librgw_la-rgw_fcgi.lo `test -f 'rgw/rgw_fcgi.cc' || echo '$(srcdir)/'`rgw/rgw_fcgi.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_fcgi.Tpo rgw/$(DEPDIR)/librgw_la-rgw_fcgi.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_fcgi.cc' object='rgw/librgw_la-rgw_fcgi.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_fcgi.lo `test -f 'rgw/rgw_fcgi.cc' || echo '$(srcdir)/'`rgw/rgw_fcgi.cc + +rgw/librgw_la-rgw_xml.lo: rgw/rgw_xml.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_xml.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_xml.Tpo -c -o rgw/librgw_la-rgw_xml.lo `test -f 'rgw/rgw_xml.cc' || echo '$(srcdir)/'`rgw/rgw_xml.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_xml.Tpo rgw/$(DEPDIR)/librgw_la-rgw_xml.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_xml.cc' object='rgw/librgw_la-rgw_xml.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_xml.lo `test -f 'rgw/rgw_xml.cc' || echo '$(srcdir)/'`rgw/rgw_xml.cc + +rgw/librgw_la-rgw_usage.lo: rgw/rgw_usage.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_usage.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_usage.Tpo -c -o rgw/librgw_la-rgw_usage.lo `test -f 'rgw/rgw_usage.cc' || echo '$(srcdir)/'`rgw/rgw_usage.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_usage.Tpo rgw/$(DEPDIR)/librgw_la-rgw_usage.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_usage.cc' object='rgw/librgw_la-rgw_usage.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_usage.lo `test -f 'rgw/rgw_usage.cc' || echo '$(srcdir)/'`rgw/rgw_usage.cc + +rgw/librgw_la-rgw_json_enc.lo: rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_json_enc.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_json_enc.Tpo -c -o rgw/librgw_la-rgw_json_enc.lo `test -f 'rgw/rgw_json_enc.cc' || echo '$(srcdir)/'`rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_json_enc.Tpo rgw/$(DEPDIR)/librgw_la-rgw_json_enc.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_json_enc.cc' object='rgw/librgw_la-rgw_json_enc.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_json_enc.lo `test -f 'rgw/rgw_json_enc.cc' || echo '$(srcdir)/'`rgw/rgw_json_enc.cc + +rgw/librgw_la-rgw_user.lo: rgw/rgw_user.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_user.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_user.Tpo -c -o rgw/librgw_la-rgw_user.lo `test -f 'rgw/rgw_user.cc' || echo '$(srcdir)/'`rgw/rgw_user.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_user.Tpo rgw/$(DEPDIR)/librgw_la-rgw_user.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_user.cc' object='rgw/librgw_la-rgw_user.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_user.lo `test -f 'rgw/rgw_user.cc' || echo '$(srcdir)/'`rgw/rgw_user.cc + +rgw/librgw_la-rgw_bucket.lo: rgw/rgw_bucket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_bucket.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_bucket.Tpo -c -o rgw/librgw_la-rgw_bucket.lo `test -f 'rgw/rgw_bucket.cc' || echo '$(srcdir)/'`rgw/rgw_bucket.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_bucket.Tpo rgw/$(DEPDIR)/librgw_la-rgw_bucket.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_bucket.cc' object='rgw/librgw_la-rgw_bucket.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_bucket.lo `test -f 'rgw/rgw_bucket.cc' || echo '$(srcdir)/'`rgw/rgw_bucket.cc + +rgw/librgw_la-rgw_tools.lo: rgw/rgw_tools.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_tools.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_tools.Tpo -c -o rgw/librgw_la-rgw_tools.lo `test -f 'rgw/rgw_tools.cc' || echo '$(srcdir)/'`rgw/rgw_tools.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_tools.Tpo rgw/$(DEPDIR)/librgw_la-rgw_tools.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_tools.cc' object='rgw/librgw_la-rgw_tools.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_tools.lo `test -f 'rgw/rgw_tools.cc' || echo '$(srcdir)/'`rgw/rgw_tools.cc + +rgw/librgw_la-rgw_rados.lo: rgw/rgw_rados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_rados.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_rados.Tpo -c -o rgw/librgw_la-rgw_rados.lo `test -f 'rgw/rgw_rados.cc' || echo '$(srcdir)/'`rgw/rgw_rados.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_rados.Tpo rgw/$(DEPDIR)/librgw_la-rgw_rados.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rados.cc' object='rgw/librgw_la-rgw_rados.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_rados.lo `test -f 'rgw/rgw_rados.cc' || echo '$(srcdir)/'`rgw/rgw_rados.cc + +rgw/librgw_la-rgw_http_client.lo: rgw/rgw_http_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_http_client.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_http_client.Tpo -c -o rgw/librgw_la-rgw_http_client.lo `test -f 'rgw/rgw_http_client.cc' || echo '$(srcdir)/'`rgw/rgw_http_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_http_client.Tpo rgw/$(DEPDIR)/librgw_la-rgw_http_client.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_http_client.cc' object='rgw/librgw_la-rgw_http_client.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_http_client.lo `test -f 'rgw/rgw_http_client.cc' || echo '$(srcdir)/'`rgw/rgw_http_client.cc + +rgw/librgw_la-rgw_rest_client.lo: rgw/rgw_rest_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_rest_client.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_rest_client.Tpo -c -o rgw/librgw_la-rgw_rest_client.lo `test -f 'rgw/rgw_rest_client.cc' || echo '$(srcdir)/'`rgw/rgw_rest_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_rest_client.Tpo rgw/$(DEPDIR)/librgw_la-rgw_rest_client.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rest_client.cc' object='rgw/librgw_la-rgw_rest_client.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_rest_client.lo `test -f 'rgw/rgw_rest_client.cc' || echo '$(srcdir)/'`rgw/rgw_rest_client.cc + +rgw/librgw_la-rgw_rest_conn.lo: rgw/rgw_rest_conn.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_rest_conn.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_rest_conn.Tpo -c -o rgw/librgw_la-rgw_rest_conn.lo `test -f 'rgw/rgw_rest_conn.cc' || echo '$(srcdir)/'`rgw/rgw_rest_conn.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_rest_conn.Tpo rgw/$(DEPDIR)/librgw_la-rgw_rest_conn.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rest_conn.cc' object='rgw/librgw_la-rgw_rest_conn.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_rest_conn.lo `test -f 'rgw/rgw_rest_conn.cc' || echo '$(srcdir)/'`rgw/rgw_rest_conn.cc + +rgw/librgw_la-rgw_op.lo: rgw/rgw_op.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_op.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_op.Tpo -c -o rgw/librgw_la-rgw_op.lo `test -f 'rgw/rgw_op.cc' || echo '$(srcdir)/'`rgw/rgw_op.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_op.Tpo rgw/$(DEPDIR)/librgw_la-rgw_op.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_op.cc' object='rgw/librgw_la-rgw_op.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_op.lo `test -f 'rgw/rgw_op.cc' || echo '$(srcdir)/'`rgw/rgw_op.cc + +rgw/librgw_la-rgw_common.lo: rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_common.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_common.Tpo -c -o rgw/librgw_la-rgw_common.lo `test -f 'rgw/rgw_common.cc' || echo '$(srcdir)/'`rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_common.Tpo rgw/$(DEPDIR)/librgw_la-rgw_common.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_common.cc' object='rgw/librgw_la-rgw_common.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_common.lo `test -f 'rgw/rgw_common.cc' || echo '$(srcdir)/'`rgw/rgw_common.cc + +rgw/librgw_la-rgw_cache.lo: rgw/rgw_cache.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_cache.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_cache.Tpo -c -o rgw/librgw_la-rgw_cache.lo `test -f 'rgw/rgw_cache.cc' || echo '$(srcdir)/'`rgw/rgw_cache.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_cache.Tpo rgw/$(DEPDIR)/librgw_la-rgw_cache.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cache.cc' object='rgw/librgw_la-rgw_cache.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_cache.lo `test -f 'rgw/rgw_cache.cc' || echo '$(srcdir)/'`rgw/rgw_cache.cc + +rgw/librgw_la-rgw_formats.lo: rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_formats.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_formats.Tpo -c -o rgw/librgw_la-rgw_formats.lo `test -f 'rgw/rgw_formats.cc' || echo '$(srcdir)/'`rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_formats.Tpo rgw/$(DEPDIR)/librgw_la-rgw_formats.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_formats.cc' object='rgw/librgw_la-rgw_formats.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_formats.lo `test -f 'rgw/rgw_formats.cc' || echo '$(srcdir)/'`rgw/rgw_formats.cc + +rgw/librgw_la-rgw_log.lo: rgw/rgw_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_log.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_log.Tpo -c -o rgw/librgw_la-rgw_log.lo `test -f 'rgw/rgw_log.cc' || echo '$(srcdir)/'`rgw/rgw_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_log.Tpo rgw/$(DEPDIR)/librgw_la-rgw_log.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_log.cc' object='rgw/librgw_la-rgw_log.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_log.lo `test -f 'rgw/rgw_log.cc' || echo '$(srcdir)/'`rgw/rgw_log.cc + +rgw/librgw_la-rgw_multi.lo: rgw/rgw_multi.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_multi.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_multi.Tpo -c -o rgw/librgw_la-rgw_multi.lo `test -f 'rgw/rgw_multi.cc' || echo '$(srcdir)/'`rgw/rgw_multi.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_multi.Tpo rgw/$(DEPDIR)/librgw_la-rgw_multi.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_multi.cc' object='rgw/librgw_la-rgw_multi.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_multi.lo `test -f 'rgw/rgw_multi.cc' || echo '$(srcdir)/'`rgw/rgw_multi.cc + +rgw/librgw_la-rgw_policy_s3.lo: rgw/rgw_policy_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_policy_s3.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_policy_s3.Tpo -c -o rgw/librgw_la-rgw_policy_s3.lo `test -f 'rgw/rgw_policy_s3.cc' || echo '$(srcdir)/'`rgw/rgw_policy_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_policy_s3.Tpo rgw/$(DEPDIR)/librgw_la-rgw_policy_s3.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_policy_s3.cc' object='rgw/librgw_la-rgw_policy_s3.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_policy_s3.lo `test -f 'rgw/rgw_policy_s3.cc' || echo '$(srcdir)/'`rgw/rgw_policy_s3.cc + +rgw/librgw_la-rgw_gc.lo: rgw/rgw_gc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_gc.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_gc.Tpo -c -o rgw/librgw_la-rgw_gc.lo `test -f 'rgw/rgw_gc.cc' || echo '$(srcdir)/'`rgw/rgw_gc.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_gc.Tpo rgw/$(DEPDIR)/librgw_la-rgw_gc.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_gc.cc' object='rgw/librgw_la-rgw_gc.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_gc.lo `test -f 'rgw/rgw_gc.cc' || echo '$(srcdir)/'`rgw/rgw_gc.cc + +rgw/librgw_la-rgw_multi_del.lo: rgw/rgw_multi_del.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_multi_del.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_multi_del.Tpo -c -o rgw/librgw_la-rgw_multi_del.lo `test -f 'rgw/rgw_multi_del.cc' || echo '$(srcdir)/'`rgw/rgw_multi_del.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_multi_del.Tpo rgw/$(DEPDIR)/librgw_la-rgw_multi_del.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_multi_del.cc' object='rgw/librgw_la-rgw_multi_del.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_multi_del.lo `test -f 'rgw/rgw_multi_del.cc' || echo '$(srcdir)/'`rgw/rgw_multi_del.cc + +rgw/librgw_la-rgw_env.lo: rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_env.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_env.Tpo -c -o rgw/librgw_la-rgw_env.lo `test -f 'rgw/rgw_env.cc' || echo '$(srcdir)/'`rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_env.Tpo rgw/$(DEPDIR)/librgw_la-rgw_env.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_env.cc' object='rgw/librgw_la-rgw_env.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_env.lo `test -f 'rgw/rgw_env.cc' || echo '$(srcdir)/'`rgw/rgw_env.cc + +rgw/librgw_la-rgw_cors.lo: rgw/rgw_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_cors.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_cors.Tpo -c -o rgw/librgw_la-rgw_cors.lo `test -f 'rgw/rgw_cors.cc' || echo '$(srcdir)/'`rgw/rgw_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_cors.Tpo rgw/$(DEPDIR)/librgw_la-rgw_cors.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cors.cc' object='rgw/librgw_la-rgw_cors.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_cors.lo `test -f 'rgw/rgw_cors.cc' || echo '$(srcdir)/'`rgw/rgw_cors.cc + +rgw/librgw_la-rgw_cors_s3.lo: rgw/rgw_cors_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_cors_s3.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_cors_s3.Tpo -c -o rgw/librgw_la-rgw_cors_s3.lo `test -f 'rgw/rgw_cors_s3.cc' || echo '$(srcdir)/'`rgw/rgw_cors_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_cors_s3.Tpo rgw/$(DEPDIR)/librgw_la-rgw_cors_s3.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cors_s3.cc' object='rgw/librgw_la-rgw_cors_s3.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_cors_s3.lo `test -f 'rgw/rgw_cors_s3.cc' || echo '$(srcdir)/'`rgw/rgw_cors_s3.cc + +rgw/librgw_la-rgw_auth_s3.lo: rgw/rgw_auth_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_auth_s3.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_auth_s3.Tpo -c -o rgw/librgw_la-rgw_auth_s3.lo `test -f 'rgw/rgw_auth_s3.cc' || echo '$(srcdir)/'`rgw/rgw_auth_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_auth_s3.Tpo rgw/$(DEPDIR)/librgw_la-rgw_auth_s3.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_auth_s3.cc' object='rgw/librgw_la-rgw_auth_s3.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_auth_s3.lo `test -f 'rgw/rgw_auth_s3.cc' || echo '$(srcdir)/'`rgw/rgw_auth_s3.cc + +rgw/librgw_la-rgw_metadata.lo: rgw/rgw_metadata.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_metadata.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_metadata.Tpo -c -o rgw/librgw_la-rgw_metadata.lo `test -f 'rgw/rgw_metadata.cc' || echo '$(srcdir)/'`rgw/rgw_metadata.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_metadata.Tpo rgw/$(DEPDIR)/librgw_la-rgw_metadata.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_metadata.cc' object='rgw/librgw_la-rgw_metadata.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_metadata.lo `test -f 'rgw/rgw_metadata.cc' || echo '$(srcdir)/'`rgw/rgw_metadata.cc + +rgw/librgw_la-rgw_replica_log.lo: rgw/rgw_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_replica_log.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_replica_log.Tpo -c -o rgw/librgw_la-rgw_replica_log.lo `test -f 'rgw/rgw_replica_log.cc' || echo '$(srcdir)/'`rgw/rgw_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_replica_log.Tpo rgw/$(DEPDIR)/librgw_la-rgw_replica_log.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_replica_log.cc' object='rgw/librgw_la-rgw_replica_log.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_replica_log.lo `test -f 'rgw/rgw_replica_log.cc' || echo '$(srcdir)/'`rgw/rgw_replica_log.cc + +rgw/librgw_la-rgw_keystone.lo: rgw/rgw_keystone.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_keystone.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_keystone.Tpo -c -o rgw/librgw_la-rgw_keystone.lo `test -f 'rgw/rgw_keystone.cc' || echo '$(srcdir)/'`rgw/rgw_keystone.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_keystone.Tpo rgw/$(DEPDIR)/librgw_la-rgw_keystone.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_keystone.cc' object='rgw/librgw_la-rgw_keystone.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_keystone.lo `test -f 'rgw/rgw_keystone.cc' || echo '$(srcdir)/'`rgw/rgw_keystone.cc + +rgw/librgw_la-rgw_quota.lo: rgw/rgw_quota.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_quota.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_quota.Tpo -c -o rgw/librgw_la-rgw_quota.lo `test -f 'rgw/rgw_quota.cc' || echo '$(srcdir)/'`rgw/rgw_quota.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_quota.Tpo rgw/$(DEPDIR)/librgw_la-rgw_quota.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_quota.cc' object='rgw/librgw_la-rgw_quota.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_quota.lo `test -f 'rgw/rgw_quota.cc' || echo '$(srcdir)/'`rgw/rgw_quota.cc + +rgw/librgw_la-rgw_dencoder.lo: rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -MT rgw/librgw_la-rgw_dencoder.lo -MD -MP -MF rgw/$(DEPDIR)/librgw_la-rgw_dencoder.Tpo -c -o rgw/librgw_la-rgw_dencoder.lo `test -f 'rgw/rgw_dencoder.cc' || echo '$(srcdir)/'`rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/librgw_la-rgw_dencoder.Tpo rgw/$(DEPDIR)/librgw_la-rgw_dencoder.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_dencoder.cc' object='rgw/librgw_la-rgw_dencoder.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(librgw_la_CXXFLAGS) $(CXXFLAGS) -c -o rgw/librgw_la-rgw_dencoder.lo `test -f 'rgw/rgw_dencoder.cc' || echo '$(srcdir)/'`rgw/rgw_dencoder.cc + +test/encoding/ceph_dencoder-ceph_dencoder.o: test/encoding/ceph_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT test/encoding/ceph_dencoder-ceph_dencoder.o -MD -MP -MF test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Tpo -c -o test/encoding/ceph_dencoder-ceph_dencoder.o `test -f 'test/encoding/ceph_dencoder.cc' || echo '$(srcdir)/'`test/encoding/ceph_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Tpo test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/encoding/ceph_dencoder.cc' object='test/encoding/ceph_dencoder-ceph_dencoder.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o test/encoding/ceph_dencoder-ceph_dencoder.o `test -f 'test/encoding/ceph_dencoder.cc' || echo '$(srcdir)/'`test/encoding/ceph_dencoder.cc + +test/encoding/ceph_dencoder-ceph_dencoder.obj: test/encoding/ceph_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT test/encoding/ceph_dencoder-ceph_dencoder.obj -MD -MP -MF test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Tpo -c -o test/encoding/ceph_dencoder-ceph_dencoder.obj `if test -f 'test/encoding/ceph_dencoder.cc'; then $(CYGPATH_W) 'test/encoding/ceph_dencoder.cc'; else $(CYGPATH_W) '$(srcdir)/test/encoding/ceph_dencoder.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Tpo test/encoding/$(DEPDIR)/ceph_dencoder-ceph_dencoder.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/encoding/ceph_dencoder.cc' object='test/encoding/ceph_dencoder-ceph_dencoder.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o test/encoding/ceph_dencoder-ceph_dencoder.obj `if test -f 'test/encoding/ceph_dencoder.cc'; then $(CYGPATH_W) 'test/encoding/ceph_dencoder.cc'; else $(CYGPATH_W) '$(srcdir)/test/encoding/ceph_dencoder.cc'; fi` + +rgw/ceph_dencoder-rgw_dencoder.o: rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_dencoder.o -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_dencoder.Tpo -c -o rgw/ceph_dencoder-rgw_dencoder.o `test -f 'rgw/rgw_dencoder.cc' || echo '$(srcdir)/'`rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_dencoder.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_dencoder.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_dencoder.cc' object='rgw/ceph_dencoder-rgw_dencoder.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_dencoder.o `test -f 'rgw/rgw_dencoder.cc' || echo '$(srcdir)/'`rgw/rgw_dencoder.cc + +rgw/ceph_dencoder-rgw_dencoder.obj: rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_dencoder.obj -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_dencoder.Tpo -c -o rgw/ceph_dencoder-rgw_dencoder.obj `if test -f 'rgw/rgw_dencoder.cc'; then $(CYGPATH_W) 'rgw/rgw_dencoder.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_dencoder.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_dencoder.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_dencoder.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_dencoder.cc' object='rgw/ceph_dencoder-rgw_dencoder.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_dencoder.obj `if test -f 'rgw/rgw_dencoder.cc'; then $(CYGPATH_W) 'rgw/rgw_dencoder.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_dencoder.cc'; fi` + +rgw/ceph_dencoder-rgw_acl.o: rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_acl.o -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_acl.Tpo -c -o rgw/ceph_dencoder-rgw_acl.o `test -f 'rgw/rgw_acl.cc' || echo '$(srcdir)/'`rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_acl.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_acl.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl.cc' object='rgw/ceph_dencoder-rgw_acl.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_acl.o `test -f 'rgw/rgw_acl.cc' || echo '$(srcdir)/'`rgw/rgw_acl.cc + +rgw/ceph_dencoder-rgw_acl.obj: rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_acl.obj -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_acl.Tpo -c -o rgw/ceph_dencoder-rgw_acl.obj `if test -f 'rgw/rgw_acl.cc'; then $(CYGPATH_W) 'rgw/rgw_acl.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_acl.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_acl.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl.cc' object='rgw/ceph_dencoder-rgw_acl.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_acl.obj `if test -f 'rgw/rgw_acl.cc'; then $(CYGPATH_W) 'rgw/rgw_acl.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl.cc'; fi` + +rgw/ceph_dencoder-rgw_common.o: rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_common.o -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_common.Tpo -c -o rgw/ceph_dencoder-rgw_common.o `test -f 'rgw/rgw_common.cc' || echo '$(srcdir)/'`rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_common.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_common.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_common.cc' object='rgw/ceph_dencoder-rgw_common.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_common.o `test -f 'rgw/rgw_common.cc' || echo '$(srcdir)/'`rgw/rgw_common.cc + +rgw/ceph_dencoder-rgw_common.obj: rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_common.obj -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_common.Tpo -c -o rgw/ceph_dencoder-rgw_common.obj `if test -f 'rgw/rgw_common.cc'; then $(CYGPATH_W) 'rgw/rgw_common.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_common.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_common.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_common.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_common.cc' object='rgw/ceph_dencoder-rgw_common.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_common.obj `if test -f 'rgw/rgw_common.cc'; then $(CYGPATH_W) 'rgw/rgw_common.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_common.cc'; fi` + +rgw/ceph_dencoder-rgw_env.o: rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_env.o -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_env.Tpo -c -o rgw/ceph_dencoder-rgw_env.o `test -f 'rgw/rgw_env.cc' || echo '$(srcdir)/'`rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_env.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_env.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_env.cc' object='rgw/ceph_dencoder-rgw_env.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_env.o `test -f 'rgw/rgw_env.cc' || echo '$(srcdir)/'`rgw/rgw_env.cc + +rgw/ceph_dencoder-rgw_env.obj: rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_env.obj -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_env.Tpo -c -o rgw/ceph_dencoder-rgw_env.obj `if test -f 'rgw/rgw_env.cc'; then $(CYGPATH_W) 'rgw/rgw_env.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_env.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_env.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_env.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_env.cc' object='rgw/ceph_dencoder-rgw_env.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_env.obj `if test -f 'rgw/rgw_env.cc'; then $(CYGPATH_W) 'rgw/rgw_env.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_env.cc'; fi` + +rgw/ceph_dencoder-rgw_json_enc.o: rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_json_enc.o -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_json_enc.Tpo -c -o rgw/ceph_dencoder-rgw_json_enc.o `test -f 'rgw/rgw_json_enc.cc' || echo '$(srcdir)/'`rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_json_enc.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_json_enc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_json_enc.cc' object='rgw/ceph_dencoder-rgw_json_enc.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_json_enc.o `test -f 'rgw/rgw_json_enc.cc' || echo '$(srcdir)/'`rgw/rgw_json_enc.cc + +rgw/ceph_dencoder-rgw_json_enc.obj: rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -MT rgw/ceph_dencoder-rgw_json_enc.obj -MD -MP -MF rgw/$(DEPDIR)/ceph_dencoder-rgw_json_enc.Tpo -c -o rgw/ceph_dencoder-rgw_json_enc.obj `if test -f 'rgw/rgw_json_enc.cc'; then $(CYGPATH_W) 'rgw/rgw_json_enc.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_json_enc.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/ceph_dencoder-rgw_json_enc.Tpo rgw/$(DEPDIR)/ceph_dencoder-rgw_json_enc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_json_enc.cc' object='rgw/ceph_dencoder-rgw_json_enc.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_dencoder_CXXFLAGS) $(CXXFLAGS) -c -o rgw/ceph_dencoder-rgw_json_enc.obj `if test -f 'rgw/rgw_json_enc.cc'; then $(CYGPATH_W) 'rgw/rgw_json_enc.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_json_enc.cc'; fi` + +tools/ceph_kvstore_tool-ceph_kvstore_tool.o: tools/ceph_kvstore_tool.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_kvstore_tool_CXXFLAGS) $(CXXFLAGS) -MT tools/ceph_kvstore_tool-ceph_kvstore_tool.o -MD -MP -MF tools/$(DEPDIR)/ceph_kvstore_tool-ceph_kvstore_tool.Tpo -c -o tools/ceph_kvstore_tool-ceph_kvstore_tool.o `test -f 'tools/ceph_kvstore_tool.cc' || echo '$(srcdir)/'`tools/ceph_kvstore_tool.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) tools/$(DEPDIR)/ceph_kvstore_tool-ceph_kvstore_tool.Tpo tools/$(DEPDIR)/ceph_kvstore_tool-ceph_kvstore_tool.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tools/ceph_kvstore_tool.cc' object='tools/ceph_kvstore_tool-ceph_kvstore_tool.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_kvstore_tool_CXXFLAGS) $(CXXFLAGS) -c -o tools/ceph_kvstore_tool-ceph_kvstore_tool.o `test -f 'tools/ceph_kvstore_tool.cc' || echo '$(srcdir)/'`tools/ceph_kvstore_tool.cc + +tools/ceph_kvstore_tool-ceph_kvstore_tool.obj: tools/ceph_kvstore_tool.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_kvstore_tool_CXXFLAGS) $(CXXFLAGS) -MT tools/ceph_kvstore_tool-ceph_kvstore_tool.obj -MD -MP -MF tools/$(DEPDIR)/ceph_kvstore_tool-ceph_kvstore_tool.Tpo -c -o tools/ceph_kvstore_tool-ceph_kvstore_tool.obj `if test -f 'tools/ceph_kvstore_tool.cc'; then $(CYGPATH_W) 'tools/ceph_kvstore_tool.cc'; else $(CYGPATH_W) '$(srcdir)/tools/ceph_kvstore_tool.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) tools/$(DEPDIR)/ceph_kvstore_tool-ceph_kvstore_tool.Tpo tools/$(DEPDIR)/ceph_kvstore_tool-ceph_kvstore_tool.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tools/ceph_kvstore_tool.cc' object='tools/ceph_kvstore_tool-ceph_kvstore_tool.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_kvstore_tool_CXXFLAGS) $(CXXFLAGS) -c -o tools/ceph_kvstore_tool-ceph_kvstore_tool.obj `if test -f 'tools/ceph_kvstore_tool.cc'; then $(CYGPATH_W) 'tools/ceph_kvstore_tool.cc'; else $(CYGPATH_W) '$(srcdir)/tools/ceph_kvstore_tool.cc'; fi` + +test/cls_hello/ceph_test_cls_hello-test_cls_hello.o: test/cls_hello/test_cls_hello.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -MT test/cls_hello/ceph_test_cls_hello-test_cls_hello.o -MD -MP -MF test/cls_hello/$(DEPDIR)/ceph_test_cls_hello-test_cls_hello.Tpo -c -o test/cls_hello/ceph_test_cls_hello-test_cls_hello.o `test -f 'test/cls_hello/test_cls_hello.cc' || echo '$(srcdir)/'`test/cls_hello/test_cls_hello.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_hello/$(DEPDIR)/ceph_test_cls_hello-test_cls_hello.Tpo test/cls_hello/$(DEPDIR)/ceph_test_cls_hello-test_cls_hello.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_hello/test_cls_hello.cc' object='test/cls_hello/ceph_test_cls_hello-test_cls_hello.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_hello/ceph_test_cls_hello-test_cls_hello.o `test -f 'test/cls_hello/test_cls_hello.cc' || echo '$(srcdir)/'`test/cls_hello/test_cls_hello.cc + +test/cls_hello/ceph_test_cls_hello-test_cls_hello.obj: test/cls_hello/test_cls_hello.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -MT test/cls_hello/ceph_test_cls_hello-test_cls_hello.obj -MD -MP -MF test/cls_hello/$(DEPDIR)/ceph_test_cls_hello-test_cls_hello.Tpo -c -o test/cls_hello/ceph_test_cls_hello-test_cls_hello.obj `if test -f 'test/cls_hello/test_cls_hello.cc'; then $(CYGPATH_W) 'test/cls_hello/test_cls_hello.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_hello/test_cls_hello.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_hello/$(DEPDIR)/ceph_test_cls_hello-test_cls_hello.Tpo test/cls_hello/$(DEPDIR)/ceph_test_cls_hello-test_cls_hello.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_hello/test_cls_hello.cc' object='test/cls_hello/ceph_test_cls_hello-test_cls_hello.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_hello_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_hello/ceph_test_cls_hello-test_cls_hello.obj `if test -f 'test/cls_hello/test_cls_hello.cc'; then $(CYGPATH_W) 'test/cls_hello/test_cls_hello.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_hello/test_cls_hello.cc'; fi` + +test/cls_lock/ceph_test_cls_lock-test_cls_lock.o: test/cls_lock/test_cls_lock.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -MT test/cls_lock/ceph_test_cls_lock-test_cls_lock.o -MD -MP -MF test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Tpo -c -o test/cls_lock/ceph_test_cls_lock-test_cls_lock.o `test -f 'test/cls_lock/test_cls_lock.cc' || echo '$(srcdir)/'`test/cls_lock/test_cls_lock.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Tpo test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_lock/test_cls_lock.cc' object='test/cls_lock/ceph_test_cls_lock-test_cls_lock.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_lock/ceph_test_cls_lock-test_cls_lock.o `test -f 'test/cls_lock/test_cls_lock.cc' || echo '$(srcdir)/'`test/cls_lock/test_cls_lock.cc + +test/cls_lock/ceph_test_cls_lock-test_cls_lock.obj: test/cls_lock/test_cls_lock.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -MT test/cls_lock/ceph_test_cls_lock-test_cls_lock.obj -MD -MP -MF test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Tpo -c -o test/cls_lock/ceph_test_cls_lock-test_cls_lock.obj `if test -f 'test/cls_lock/test_cls_lock.cc'; then $(CYGPATH_W) 'test/cls_lock/test_cls_lock.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_lock/test_cls_lock.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Tpo test/cls_lock/$(DEPDIR)/ceph_test_cls_lock-test_cls_lock.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_lock/test_cls_lock.cc' object='test/cls_lock/ceph_test_cls_lock-test_cls_lock.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_lock/ceph_test_cls_lock-test_cls_lock.obj `if test -f 'test/cls_lock/test_cls_lock.cc'; then $(CYGPATH_W) 'test/cls_lock/test_cls_lock.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_lock/test_cls_lock.cc'; fi` + +test/cls_log/ceph_test_cls_log-test_cls_log.o: test/cls_log/test_cls_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -MT test/cls_log/ceph_test_cls_log-test_cls_log.o -MD -MP -MF test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Tpo -c -o test/cls_log/ceph_test_cls_log-test_cls_log.o `test -f 'test/cls_log/test_cls_log.cc' || echo '$(srcdir)/'`test/cls_log/test_cls_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Tpo test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_log/test_cls_log.cc' object='test/cls_log/ceph_test_cls_log-test_cls_log.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_log/ceph_test_cls_log-test_cls_log.o `test -f 'test/cls_log/test_cls_log.cc' || echo '$(srcdir)/'`test/cls_log/test_cls_log.cc + +test/cls_log/ceph_test_cls_log-test_cls_log.obj: test/cls_log/test_cls_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -MT test/cls_log/ceph_test_cls_log-test_cls_log.obj -MD -MP -MF test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Tpo -c -o test/cls_log/ceph_test_cls_log-test_cls_log.obj `if test -f 'test/cls_log/test_cls_log.cc'; then $(CYGPATH_W) 'test/cls_log/test_cls_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_log/test_cls_log.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Tpo test/cls_log/$(DEPDIR)/ceph_test_cls_log-test_cls_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_log/test_cls_log.cc' object='test/cls_log/ceph_test_cls_log-test_cls_log.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_log_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_log/ceph_test_cls_log-test_cls_log.obj `if test -f 'test/cls_log/test_cls_log.cc'; then $(CYGPATH_W) 'test/cls_log/test_cls_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_log/test_cls_log.cc'; fi` + +test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o: test/cls_rbd/test_cls_rbd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -MT test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o -MD -MP -MF test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Tpo -c -o test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o `test -f 'test/cls_rbd/test_cls_rbd.cc' || echo '$(srcdir)/'`test/cls_rbd/test_cls_rbd.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Tpo test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_rbd/test_cls_rbd.cc' object='test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.o `test -f 'test/cls_rbd/test_cls_rbd.cc' || echo '$(srcdir)/'`test/cls_rbd/test_cls_rbd.cc + +test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.obj: test/cls_rbd/test_cls_rbd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -MT test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.obj -MD -MP -MF test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Tpo -c -o test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.obj `if test -f 'test/cls_rbd/test_cls_rbd.cc'; then $(CYGPATH_W) 'test/cls_rbd/test_cls_rbd.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_rbd/test_cls_rbd.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Tpo test/cls_rbd/$(DEPDIR)/ceph_test_cls_rbd-test_cls_rbd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_rbd/test_cls_rbd.cc' object='test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rbd_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_rbd/ceph_test_cls_rbd-test_cls_rbd.obj `if test -f 'test/cls_rbd/test_cls_rbd.cc'; then $(CYGPATH_W) 'test/cls_rbd/test_cls_rbd.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_rbd/test_cls_rbd.cc'; fi` + +test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o: test/cls_refcount/test_cls_refcount.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -MT test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o -MD -MP -MF test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Tpo -c -o test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o `test -f 'test/cls_refcount/test_cls_refcount.cc' || echo '$(srcdir)/'`test/cls_refcount/test_cls_refcount.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Tpo test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_refcount/test_cls_refcount.cc' object='test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.o `test -f 'test/cls_refcount/test_cls_refcount.cc' || echo '$(srcdir)/'`test/cls_refcount/test_cls_refcount.cc + +test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.obj: test/cls_refcount/test_cls_refcount.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -MT test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.obj -MD -MP -MF test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Tpo -c -o test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.obj `if test -f 'test/cls_refcount/test_cls_refcount.cc'; then $(CYGPATH_W) 'test/cls_refcount/test_cls_refcount.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_refcount/test_cls_refcount.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Tpo test/cls_refcount/$(DEPDIR)/ceph_test_cls_refcount-test_cls_refcount.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_refcount/test_cls_refcount.cc' object='test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_refcount_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_refcount/ceph_test_cls_refcount-test_cls_refcount.obj `if test -f 'test/cls_refcount/test_cls_refcount.cc'; then $(CYGPATH_W) 'test/cls_refcount/test_cls_refcount.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_refcount/test_cls_refcount.cc'; fi` + +test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o: test/cls_replica_log/test_cls_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -MT test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o -MD -MP -MF test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Tpo -c -o test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o `test -f 'test/cls_replica_log/test_cls_replica_log.cc' || echo '$(srcdir)/'`test/cls_replica_log/test_cls_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Tpo test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_replica_log/test_cls_replica_log.cc' object='test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.o `test -f 'test/cls_replica_log/test_cls_replica_log.cc' || echo '$(srcdir)/'`test/cls_replica_log/test_cls_replica_log.cc + +test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.obj: test/cls_replica_log/test_cls_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -MT test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.obj -MD -MP -MF test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Tpo -c -o test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.obj `if test -f 'test/cls_replica_log/test_cls_replica_log.cc'; then $(CYGPATH_W) 'test/cls_replica_log/test_cls_replica_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_replica_log/test_cls_replica_log.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Tpo test/cls_replica_log/$(DEPDIR)/ceph_test_cls_replica_log-test_cls_replica_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_replica_log/test_cls_replica_log.cc' object='test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_replica_log_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_replica_log/ceph_test_cls_replica_log-test_cls_replica_log.obj `if test -f 'test/cls_replica_log/test_cls_replica_log.cc'; then $(CYGPATH_W) 'test/cls_replica_log/test_cls_replica_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_replica_log/test_cls_replica_log.cc'; fi` + +test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o: test/cls_rgw/test_cls_rgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -MT test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o -MD -MP -MF test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Tpo -c -o test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o `test -f 'test/cls_rgw/test_cls_rgw.cc' || echo '$(srcdir)/'`test/cls_rgw/test_cls_rgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Tpo test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_rgw/test_cls_rgw.cc' object='test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.o `test -f 'test/cls_rgw/test_cls_rgw.cc' || echo '$(srcdir)/'`test/cls_rgw/test_cls_rgw.cc + +test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.obj: test/cls_rgw/test_cls_rgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -MT test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.obj -MD -MP -MF test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Tpo -c -o test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.obj `if test -f 'test/cls_rgw/test_cls_rgw.cc'; then $(CYGPATH_W) 'test/cls_rgw/test_cls_rgw.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_rgw/test_cls_rgw.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Tpo test/cls_rgw/$(DEPDIR)/ceph_test_cls_rgw-test_cls_rgw.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_rgw/test_cls_rgw.cc' object='test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_rgw/ceph_test_cls_rgw-test_cls_rgw.obj `if test -f 'test/cls_rgw/test_cls_rgw.cc'; then $(CYGPATH_W) 'test/cls_rgw/test_cls_rgw.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_rgw/test_cls_rgw.cc'; fi` + +test/ceph_test_cls_rgw_log-test_rgw_admin_log.o: test/test_rgw_admin_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_log_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cls_rgw_log-test_rgw_admin_log.o -MD -MP -MF test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Tpo -c -o test/ceph_test_cls_rgw_log-test_rgw_admin_log.o `test -f 'test/test_rgw_admin_log.cc' || echo '$(srcdir)/'`test/test_rgw_admin_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Tpo test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_rgw_admin_log.cc' object='test/ceph_test_cls_rgw_log-test_rgw_admin_log.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_log_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cls_rgw_log-test_rgw_admin_log.o `test -f 'test/test_rgw_admin_log.cc' || echo '$(srcdir)/'`test/test_rgw_admin_log.cc + +test/ceph_test_cls_rgw_log-test_rgw_admin_log.obj: test/test_rgw_admin_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_log_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cls_rgw_log-test_rgw_admin_log.obj -MD -MP -MF test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Tpo -c -o test/ceph_test_cls_rgw_log-test_rgw_admin_log.obj `if test -f 'test/test_rgw_admin_log.cc'; then $(CYGPATH_W) 'test/test_rgw_admin_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_rgw_admin_log.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Tpo test/$(DEPDIR)/ceph_test_cls_rgw_log-test_rgw_admin_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_rgw_admin_log.cc' object='test/ceph_test_cls_rgw_log-test_rgw_admin_log.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_log_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cls_rgw_log-test_rgw_admin_log.obj `if test -f 'test/test_rgw_admin_log.cc'; then $(CYGPATH_W) 'test/test_rgw_admin_log.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_rgw_admin_log.cc'; fi` + +test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.o: test/test_rgw_admin_meta.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_meta_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.o -MD -MP -MF test/$(DEPDIR)/ceph_test_cls_rgw_meta-test_rgw_admin_meta.Tpo -c -o test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.o `test -f 'test/test_rgw_admin_meta.cc' || echo '$(srcdir)/'`test/test_rgw_admin_meta.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cls_rgw_meta-test_rgw_admin_meta.Tpo test/$(DEPDIR)/ceph_test_cls_rgw_meta-test_rgw_admin_meta.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_rgw_admin_meta.cc' object='test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_meta_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.o `test -f 'test/test_rgw_admin_meta.cc' || echo '$(srcdir)/'`test/test_rgw_admin_meta.cc + +test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.obj: test/test_rgw_admin_meta.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_meta_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.obj -MD -MP -MF test/$(DEPDIR)/ceph_test_cls_rgw_meta-test_rgw_admin_meta.Tpo -c -o test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.obj `if test -f 'test/test_rgw_admin_meta.cc'; then $(CYGPATH_W) 'test/test_rgw_admin_meta.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_rgw_admin_meta.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cls_rgw_meta-test_rgw_admin_meta.Tpo test/$(DEPDIR)/ceph_test_cls_rgw_meta-test_rgw_admin_meta.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_rgw_admin_meta.cc' object='test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_meta_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cls_rgw_meta-test_rgw_admin_meta.obj `if test -f 'test/test_rgw_admin_meta.cc'; then $(CYGPATH_W) 'test/test_rgw_admin_meta.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_rgw_admin_meta.cc'; fi` + +test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.o: test/test_rgw_admin_opstate.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_opstate_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.o -MD -MP -MF test/$(DEPDIR)/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.Tpo -c -o test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.o `test -f 'test/test_rgw_admin_opstate.cc' || echo '$(srcdir)/'`test/test_rgw_admin_opstate.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.Tpo test/$(DEPDIR)/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_rgw_admin_opstate.cc' object='test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_opstate_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.o `test -f 'test/test_rgw_admin_opstate.cc' || echo '$(srcdir)/'`test/test_rgw_admin_opstate.cc + +test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.obj: test/test_rgw_admin_opstate.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_opstate_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.obj -MD -MP -MF test/$(DEPDIR)/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.Tpo -c -o test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.obj `if test -f 'test/test_rgw_admin_opstate.cc'; then $(CYGPATH_W) 'test/test_rgw_admin_opstate.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_rgw_admin_opstate.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.Tpo test/$(DEPDIR)/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_rgw_admin_opstate.cc' object='test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_rgw_opstate_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cls_rgw_opstate-test_rgw_admin_opstate.obj `if test -f 'test/test_rgw_admin_opstate.cc'; then $(CYGPATH_W) 'test/test_rgw_admin_opstate.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_rgw_admin_opstate.cc'; fi` + +test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.o: test/cls_statelog/test_cls_statelog.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -MT test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.o -MD -MP -MF test/cls_statelog/$(DEPDIR)/ceph_test_cls_statelog-test_cls_statelog.Tpo -c -o test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.o `test -f 'test/cls_statelog/test_cls_statelog.cc' || echo '$(srcdir)/'`test/cls_statelog/test_cls_statelog.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_statelog/$(DEPDIR)/ceph_test_cls_statelog-test_cls_statelog.Tpo test/cls_statelog/$(DEPDIR)/ceph_test_cls_statelog-test_cls_statelog.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_statelog/test_cls_statelog.cc' object='test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.o `test -f 'test/cls_statelog/test_cls_statelog.cc' || echo '$(srcdir)/'`test/cls_statelog/test_cls_statelog.cc + +test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.obj: test/cls_statelog/test_cls_statelog.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -MT test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.obj -MD -MP -MF test/cls_statelog/$(DEPDIR)/ceph_test_cls_statelog-test_cls_statelog.Tpo -c -o test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.obj `if test -f 'test/cls_statelog/test_cls_statelog.cc'; then $(CYGPATH_W) 'test/cls_statelog/test_cls_statelog.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_statelog/test_cls_statelog.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_statelog/$(DEPDIR)/ceph_test_cls_statelog-test_cls_statelog.Tpo test/cls_statelog/$(DEPDIR)/ceph_test_cls_statelog-test_cls_statelog.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_statelog/test_cls_statelog.cc' object='test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_statelog_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_statelog/ceph_test_cls_statelog-test_cls_statelog.obj `if test -f 'test/cls_statelog/test_cls_statelog.cc'; then $(CYGPATH_W) 'test/cls_statelog/test_cls_statelog.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_statelog/test_cls_statelog.cc'; fi` + +test/cls_version/ceph_test_cls_version-test_cls_version.o: test/cls_version/test_cls_version.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -MT test/cls_version/ceph_test_cls_version-test_cls_version.o -MD -MP -MF test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Tpo -c -o test/cls_version/ceph_test_cls_version-test_cls_version.o `test -f 'test/cls_version/test_cls_version.cc' || echo '$(srcdir)/'`test/cls_version/test_cls_version.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Tpo test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_version/test_cls_version.cc' object='test/cls_version/ceph_test_cls_version-test_cls_version.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_version/ceph_test_cls_version-test_cls_version.o `test -f 'test/cls_version/test_cls_version.cc' || echo '$(srcdir)/'`test/cls_version/test_cls_version.cc + +test/cls_version/ceph_test_cls_version-test_cls_version.obj: test/cls_version/test_cls_version.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -MT test/cls_version/ceph_test_cls_version-test_cls_version.obj -MD -MP -MF test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Tpo -c -o test/cls_version/ceph_test_cls_version-test_cls_version.obj `if test -f 'test/cls_version/test_cls_version.cc'; then $(CYGPATH_W) 'test/cls_version/test_cls_version.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_version/test_cls_version.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Tpo test/cls_version/$(DEPDIR)/ceph_test_cls_version-test_cls_version.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/cls_version/test_cls_version.cc' object='test/cls_version/ceph_test_cls_version-test_cls_version.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cls_version_CXXFLAGS) $(CXXFLAGS) -c -o test/cls_version/ceph_test_cls_version-test_cls_version.obj `if test -f 'test/cls_version/test_cls_version.cc'; then $(CYGPATH_W) 'test/cls_version/test_cls_version.cc'; else $(CYGPATH_W) '$(srcdir)/test/cls_version/test_cls_version.cc'; fi` + +test/ceph_test_cors-test_cors.o: test/test_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cors_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cors-test_cors.o -MD -MP -MF test/$(DEPDIR)/ceph_test_cors-test_cors.Tpo -c -o test/ceph_test_cors-test_cors.o `test -f 'test/test_cors.cc' || echo '$(srcdir)/'`test/test_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cors-test_cors.Tpo test/$(DEPDIR)/ceph_test_cors-test_cors.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_cors.cc' object='test/ceph_test_cors-test_cors.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cors_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cors-test_cors.o `test -f 'test/test_cors.cc' || echo '$(srcdir)/'`test/test_cors.cc + +test/ceph_test_cors-test_cors.obj: test/test_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cors_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_cors-test_cors.obj -MD -MP -MF test/$(DEPDIR)/ceph_test_cors-test_cors.Tpo -c -o test/ceph_test_cors-test_cors.obj `if test -f 'test/test_cors.cc'; then $(CYGPATH_W) 'test/test_cors.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_cors.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_cors-test_cors.Tpo test/$(DEPDIR)/ceph_test_cors-test_cors.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_cors.cc' object='test/ceph_test_cors-test_cors.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_cors_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_cors-test_cors.obj `if test -f 'test/test_cors.cc'; then $(CYGPATH_W) 'test/test_cors.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_cors.cc'; fi` + +test/ceph_test_filejournal-test_filejournal.o: test/test_filejournal.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_filejournal-test_filejournal.o -MD -MP -MF test/$(DEPDIR)/ceph_test_filejournal-test_filejournal.Tpo -c -o test/ceph_test_filejournal-test_filejournal.o `test -f 'test/test_filejournal.cc' || echo '$(srcdir)/'`test/test_filejournal.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_filejournal-test_filejournal.Tpo test/$(DEPDIR)/ceph_test_filejournal-test_filejournal.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_filejournal.cc' object='test/ceph_test_filejournal-test_filejournal.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_filejournal-test_filejournal.o `test -f 'test/test_filejournal.cc' || echo '$(srcdir)/'`test/test_filejournal.cc + +test/ceph_test_filejournal-test_filejournal.obj: test/test_filejournal.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_filejournal-test_filejournal.obj -MD -MP -MF test/$(DEPDIR)/ceph_test_filejournal-test_filejournal.Tpo -c -o test/ceph_test_filejournal-test_filejournal.obj `if test -f 'test/test_filejournal.cc'; then $(CYGPATH_W) 'test/test_filejournal.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_filejournal.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_filejournal-test_filejournal.Tpo test/$(DEPDIR)/ceph_test_filejournal-test_filejournal.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_filejournal.cc' object='test/ceph_test_filejournal-test_filejournal.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_filejournal_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_filejournal-test_filejournal.obj `if test -f 'test/test_filejournal.cc'; then $(CYGPATH_W) 'test/test_filejournal.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_filejournal.cc'; fi` + +test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o: test/ObjectMap/test_keyvaluedb_atomicity.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_atomicity_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Tpo -c -o test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o `test -f 'test/ObjectMap/test_keyvaluedb_atomicity.cc' || echo '$(srcdir)/'`test/ObjectMap/test_keyvaluedb_atomicity.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/test_keyvaluedb_atomicity.cc' object='test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_atomicity_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.o `test -f 'test/ObjectMap/test_keyvaluedb_atomicity.cc' || echo '$(srcdir)/'`test/ObjectMap/test_keyvaluedb_atomicity.cc + +test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.obj: test/ObjectMap/test_keyvaluedb_atomicity.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_atomicity_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.obj -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Tpo -c -o test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.obj `if test -f 'test/ObjectMap/test_keyvaluedb_atomicity.cc'; then $(CYGPATH_W) 'test/ObjectMap/test_keyvaluedb_atomicity.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/test_keyvaluedb_atomicity.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/test_keyvaluedb_atomicity.cc' object='test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_atomicity_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_keyvaluedb_atomicity-test_keyvaluedb_atomicity.obj `if test -f 'test/ObjectMap/test_keyvaluedb_atomicity.cc'; then $(CYGPATH_W) 'test/ObjectMap/test_keyvaluedb_atomicity.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/test_keyvaluedb_atomicity.cc'; fi` + +test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.o: test/ObjectMap/test_keyvaluedb_iterators.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.o -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.Tpo -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.o `test -f 'test/ObjectMap/test_keyvaluedb_iterators.cc' || echo '$(srcdir)/'`test/ObjectMap/test_keyvaluedb_iterators.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/test_keyvaluedb_iterators.cc' object='test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.o `test -f 'test/ObjectMap/test_keyvaluedb_iterators.cc' || echo '$(srcdir)/'`test/ObjectMap/test_keyvaluedb_iterators.cc + +test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.obj: test/ObjectMap/test_keyvaluedb_iterators.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.obj -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.Tpo -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.obj `if test -f 'test/ObjectMap/test_keyvaluedb_iterators.cc'; then $(CYGPATH_W) 'test/ObjectMap/test_keyvaluedb_iterators.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/test_keyvaluedb_iterators.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/test_keyvaluedb_iterators.cc' object='test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-test_keyvaluedb_iterators.obj `if test -f 'test/ObjectMap/test_keyvaluedb_iterators.cc'; then $(CYGPATH_W) 'test/ObjectMap/test_keyvaluedb_iterators.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/test_keyvaluedb_iterators.cc'; fi` + +test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.o: test/ObjectMap/KeyValueDBMemory.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.o -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.Tpo -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.o `test -f 'test/ObjectMap/KeyValueDBMemory.cc' || echo '$(srcdir)/'`test/ObjectMap/KeyValueDBMemory.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/KeyValueDBMemory.cc' object='test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.o `test -f 'test/ObjectMap/KeyValueDBMemory.cc' || echo '$(srcdir)/'`test/ObjectMap/KeyValueDBMemory.cc + +test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.obj: test/ObjectMap/KeyValueDBMemory.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.obj -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.Tpo -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.obj `if test -f 'test/ObjectMap/KeyValueDBMemory.cc'; then $(CYGPATH_W) 'test/ObjectMap/KeyValueDBMemory.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/KeyValueDBMemory.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/KeyValueDBMemory.cc' object='test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_keyvaluedb_iterators_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_keyvaluedb_iterators-KeyValueDBMemory.obj `if test -f 'test/ObjectMap/KeyValueDBMemory.cc'; then $(CYGPATH_W) 'test/ObjectMap/KeyValueDBMemory.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/KeyValueDBMemory.cc'; fi` + +test/libcephfs/ceph_test_libcephfs-test.o: test/libcephfs/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-test.o -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Tpo -c -o test/libcephfs/ceph_test_libcephfs-test.o `test -f 'test/libcephfs/test.cc' || echo '$(srcdir)/'`test/libcephfs/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/test.cc' object='test/libcephfs/ceph_test_libcephfs-test.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-test.o `test -f 'test/libcephfs/test.cc' || echo '$(srcdir)/'`test/libcephfs/test.cc + +test/libcephfs/ceph_test_libcephfs-test.obj: test/libcephfs/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-test.obj -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Tpo -c -o test/libcephfs/ceph_test_libcephfs-test.obj `if test -f 'test/libcephfs/test.cc'; then $(CYGPATH_W) 'test/libcephfs/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/test.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/test.cc' object='test/libcephfs/ceph_test_libcephfs-test.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-test.obj `if test -f 'test/libcephfs/test.cc'; then $(CYGPATH_W) 'test/libcephfs/test.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/test.cc'; fi` + +test/libcephfs/ceph_test_libcephfs-readdir_r_cb.o: test/libcephfs/readdir_r_cb.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-readdir_r_cb.o -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Tpo -c -o test/libcephfs/ceph_test_libcephfs-readdir_r_cb.o `test -f 'test/libcephfs/readdir_r_cb.cc' || echo '$(srcdir)/'`test/libcephfs/readdir_r_cb.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/readdir_r_cb.cc' object='test/libcephfs/ceph_test_libcephfs-readdir_r_cb.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-readdir_r_cb.o `test -f 'test/libcephfs/readdir_r_cb.cc' || echo '$(srcdir)/'`test/libcephfs/readdir_r_cb.cc + +test/libcephfs/ceph_test_libcephfs-readdir_r_cb.obj: test/libcephfs/readdir_r_cb.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-readdir_r_cb.obj -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Tpo -c -o test/libcephfs/ceph_test_libcephfs-readdir_r_cb.obj `if test -f 'test/libcephfs/readdir_r_cb.cc'; then $(CYGPATH_W) 'test/libcephfs/readdir_r_cb.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/readdir_r_cb.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-readdir_r_cb.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/readdir_r_cb.cc' object='test/libcephfs/ceph_test_libcephfs-readdir_r_cb.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-readdir_r_cb.obj `if test -f 'test/libcephfs/readdir_r_cb.cc'; then $(CYGPATH_W) 'test/libcephfs/readdir_r_cb.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/readdir_r_cb.cc'; fi` + +test/libcephfs/ceph_test_libcephfs-caps.o: test/libcephfs/caps.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-caps.o -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Tpo -c -o test/libcephfs/ceph_test_libcephfs-caps.o `test -f 'test/libcephfs/caps.cc' || echo '$(srcdir)/'`test/libcephfs/caps.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/caps.cc' object='test/libcephfs/ceph_test_libcephfs-caps.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-caps.o `test -f 'test/libcephfs/caps.cc' || echo '$(srcdir)/'`test/libcephfs/caps.cc + +test/libcephfs/ceph_test_libcephfs-caps.obj: test/libcephfs/caps.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-caps.obj -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Tpo -c -o test/libcephfs/ceph_test_libcephfs-caps.obj `if test -f 'test/libcephfs/caps.cc'; then $(CYGPATH_W) 'test/libcephfs/caps.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/caps.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-caps.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/caps.cc' object='test/libcephfs/ceph_test_libcephfs-caps.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-caps.obj `if test -f 'test/libcephfs/caps.cc'; then $(CYGPATH_W) 'test/libcephfs/caps.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/caps.cc'; fi` + +test/libcephfs/ceph_test_libcephfs-multiclient.o: test/libcephfs/multiclient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-multiclient.o -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Tpo -c -o test/libcephfs/ceph_test_libcephfs-multiclient.o `test -f 'test/libcephfs/multiclient.cc' || echo '$(srcdir)/'`test/libcephfs/multiclient.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/multiclient.cc' object='test/libcephfs/ceph_test_libcephfs-multiclient.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-multiclient.o `test -f 'test/libcephfs/multiclient.cc' || echo '$(srcdir)/'`test/libcephfs/multiclient.cc + +test/libcephfs/ceph_test_libcephfs-multiclient.obj: test/libcephfs/multiclient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/libcephfs/ceph_test_libcephfs-multiclient.obj -MD -MP -MF test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Tpo -c -o test/libcephfs/ceph_test_libcephfs-multiclient.obj `if test -f 'test/libcephfs/multiclient.cc'; then $(CYGPATH_W) 'test/libcephfs/multiclient.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/multiclient.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Tpo test/libcephfs/$(DEPDIR)/ceph_test_libcephfs-multiclient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs/multiclient.cc' object='test/libcephfs/ceph_test_libcephfs-multiclient.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/libcephfs/ceph_test_libcephfs-multiclient.obj `if test -f 'test/libcephfs/multiclient.cc'; then $(CYGPATH_W) 'test/libcephfs/multiclient.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs/multiclient.cc'; fi` + +test/librbd/ceph_test_librbd-test_librbd.o: test/librbd/test_librbd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -MT test/librbd/ceph_test_librbd-test_librbd.o -MD -MP -MF test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Tpo -c -o test/librbd/ceph_test_librbd-test_librbd.o `test -f 'test/librbd/test_librbd.cc' || echo '$(srcdir)/'`test/librbd/test_librbd.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Tpo test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librbd/test_librbd.cc' object='test/librbd/ceph_test_librbd-test_librbd.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -c -o test/librbd/ceph_test_librbd-test_librbd.o `test -f 'test/librbd/test_librbd.cc' || echo '$(srcdir)/'`test/librbd/test_librbd.cc + +test/librbd/ceph_test_librbd-test_librbd.obj: test/librbd/test_librbd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -MT test/librbd/ceph_test_librbd-test_librbd.obj -MD -MP -MF test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Tpo -c -o test/librbd/ceph_test_librbd-test_librbd.obj `if test -f 'test/librbd/test_librbd.cc'; then $(CYGPATH_W) 'test/librbd/test_librbd.cc'; else $(CYGPATH_W) '$(srcdir)/test/librbd/test_librbd.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Tpo test/librbd/$(DEPDIR)/ceph_test_librbd-test_librbd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librbd/test_librbd.cc' object='test/librbd/ceph_test_librbd-test_librbd.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_librbd_CXXFLAGS) $(CXXFLAGS) -c -o test/librbd/ceph_test_librbd-test_librbd.obj `if test -f 'test/librbd/test_librbd.cc'; then $(CYGPATH_W) 'test/librbd/test_librbd.cc'; else $(CYGPATH_W) '$(srcdir)/test/librbd/test_librbd.cc'; fi` + +test/ObjectMap/ceph_test_object_map-test_object_map.o: test/ObjectMap/test_object_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_object_map-test_object_map.o -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Tpo -c -o test/ObjectMap/ceph_test_object_map-test_object_map.o `test -f 'test/ObjectMap/test_object_map.cc' || echo '$(srcdir)/'`test/ObjectMap/test_object_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/test_object_map.cc' object='test/ObjectMap/ceph_test_object_map-test_object_map.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_object_map-test_object_map.o `test -f 'test/ObjectMap/test_object_map.cc' || echo '$(srcdir)/'`test/ObjectMap/test_object_map.cc + +test/ObjectMap/ceph_test_object_map-test_object_map.obj: test/ObjectMap/test_object_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_object_map-test_object_map.obj -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Tpo -c -o test/ObjectMap/ceph_test_object_map-test_object_map.obj `if test -f 'test/ObjectMap/test_object_map.cc'; then $(CYGPATH_W) 'test/ObjectMap/test_object_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/test_object_map.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_object_map-test_object_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/test_object_map.cc' object='test/ObjectMap/ceph_test_object_map-test_object_map.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_object_map-test_object_map.obj `if test -f 'test/ObjectMap/test_object_map.cc'; then $(CYGPATH_W) 'test/ObjectMap/test_object_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/test_object_map.cc'; fi` + +test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.o: test/ObjectMap/KeyValueDBMemory.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.o -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_object_map-KeyValueDBMemory.Tpo -c -o test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.o `test -f 'test/ObjectMap/KeyValueDBMemory.cc' || echo '$(srcdir)/'`test/ObjectMap/KeyValueDBMemory.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_object_map-KeyValueDBMemory.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_object_map-KeyValueDBMemory.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/KeyValueDBMemory.cc' object='test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.o `test -f 'test/ObjectMap/KeyValueDBMemory.cc' || echo '$(srcdir)/'`test/ObjectMap/KeyValueDBMemory.cc + +test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.obj: test/ObjectMap/KeyValueDBMemory.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -MT test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.obj -MD -MP -MF test/ObjectMap/$(DEPDIR)/ceph_test_object_map-KeyValueDBMemory.Tpo -c -o test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.obj `if test -f 'test/ObjectMap/KeyValueDBMemory.cc'; then $(CYGPATH_W) 'test/ObjectMap/KeyValueDBMemory.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/KeyValueDBMemory.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/ObjectMap/$(DEPDIR)/ceph_test_object_map-KeyValueDBMemory.Tpo test/ObjectMap/$(DEPDIR)/ceph_test_object_map-KeyValueDBMemory.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ObjectMap/KeyValueDBMemory.cc' object='test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_object_map_CXXFLAGS) $(CXXFLAGS) -c -o test/ObjectMap/ceph_test_object_map-KeyValueDBMemory.obj `if test -f 'test/ObjectMap/KeyValueDBMemory.cc'; then $(CYGPATH_W) 'test/ObjectMap/KeyValueDBMemory.cc'; else $(CYGPATH_W) '$(srcdir)/test/ObjectMap/KeyValueDBMemory.cc'; fi` + +test/objectstore/ceph_test_objectstore-store_test.o: test/objectstore/store_test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/ceph_test_objectstore-store_test.o -MD -MP -MF test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo -c -o test/objectstore/ceph_test_objectstore-store_test.o `test -f 'test/objectstore/store_test.cc' || echo '$(srcdir)/'`test/objectstore/store_test.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/store_test.cc' object='test/objectstore/ceph_test_objectstore-store_test.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/ceph_test_objectstore-store_test.o `test -f 'test/objectstore/store_test.cc' || echo '$(srcdir)/'`test/objectstore/store_test.cc + +test/objectstore/ceph_test_objectstore-store_test.obj: test/objectstore/store_test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/ceph_test_objectstore-store_test.obj -MD -MP -MF test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo -c -o test/objectstore/ceph_test_objectstore-store_test.obj `if test -f 'test/objectstore/store_test.cc'; then $(CYGPATH_W) 'test/objectstore/store_test.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/store_test.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Tpo test/objectstore/$(DEPDIR)/ceph_test_objectstore-store_test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/store_test.cc' object='test/objectstore/ceph_test_objectstore-store_test.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_objectstore_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/ceph_test_objectstore-store_test.obj `if test -f 'test/objectstore/store_test.cc'; then $(CYGPATH_W) 'test/objectstore/store_test.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/store_test.cc'; fi` + +test/librados/ceph_test_rados_api_aio-aio.o: test/librados/aio.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_aio-aio.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Tpo -c -o test/librados/ceph_test_rados_api_aio-aio.o `test -f 'test/librados/aio.cc' || echo '$(srcdir)/'`test/librados/aio.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/aio.cc' object='test/librados/ceph_test_rados_api_aio-aio.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_aio-aio.o `test -f 'test/librados/aio.cc' || echo '$(srcdir)/'`test/librados/aio.cc + +test/librados/ceph_test_rados_api_aio-aio.obj: test/librados/aio.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_aio-aio.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Tpo -c -o test/librados/ceph_test_rados_api_aio-aio.obj `if test -f 'test/librados/aio.cc'; then $(CYGPATH_W) 'test/librados/aio.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/aio.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_aio-aio.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/aio.cc' object='test/librados/ceph_test_rados_api_aio-aio.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_aio_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_aio-aio.obj `if test -f 'test/librados/aio.cc'; then $(CYGPATH_W) 'test/librados/aio.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/aio.cc'; fi` + +test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o: test/librados/c_read_operations.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o `test -f 'test/librados/c_read_operations.cc' || echo '$(srcdir)/'`test/librados/c_read_operations.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_read_operations.cc' object='test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.o `test -f 'test/librados/c_read_operations.cc' || echo '$(srcdir)/'`test/librados/c_read_operations.cc + +test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj: test/librados/c_read_operations.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj `if test -f 'test/librados/c_read_operations.cc'; then $(CYGPATH_W) 'test/librados/c_read_operations.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/c_read_operations.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_read_operations-c_read_operations.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_read_operations.cc' object='test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_read_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_read_operations-c_read_operations.obj `if test -f 'test/librados/c_read_operations.cc'; then $(CYGPATH_W) 'test/librados/c_read_operations.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/c_read_operations.cc'; fi` + +test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o: test/librados/c_write_operations.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o `test -f 'test/librados/c_write_operations.cc' || echo '$(srcdir)/'`test/librados/c_write_operations.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_write_operations.cc' object='test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.o `test -f 'test/librados/c_write_operations.cc' || echo '$(srcdir)/'`test/librados/c_write_operations.cc + +test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj: test/librados/c_write_operations.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj `if test -f 'test/librados/c_write_operations.cc'; then $(CYGPATH_W) 'test/librados/c_write_operations.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/c_write_operations.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_c_write_operations-c_write_operations.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/c_write_operations.cc' object='test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_c_write_operations_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_c_write_operations-c_write_operations.obj `if test -f 'test/librados/c_write_operations.cc'; then $(CYGPATH_W) 'test/librados/c_write_operations.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/c_write_operations.cc'; fi` + +test/librados/ceph_test_rados_api_cls-cls.o: test/librados/cls.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cls-cls.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo -c -o test/librados/ceph_test_rados_api_cls-cls.o `test -f 'test/librados/cls.cc' || echo '$(srcdir)/'`test/librados/cls.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/cls.cc' object='test/librados/ceph_test_rados_api_cls-cls.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cls-cls.o `test -f 'test/librados/cls.cc' || echo '$(srcdir)/'`test/librados/cls.cc + +test/librados/ceph_test_rados_api_cls-cls.obj: test/librados/cls.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cls-cls.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo -c -o test/librados/ceph_test_rados_api_cls-cls.obj `if test -f 'test/librados/cls.cc'; then $(CYGPATH_W) 'test/librados/cls.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/cls.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cls-cls.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/cls.cc' object='test/librados/ceph_test_rados_api_cls-cls.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cls_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cls-cls.obj `if test -f 'test/librados/cls.cc'; then $(CYGPATH_W) 'test/librados/cls.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/cls.cc'; fi` + +test/librados/ceph_test_rados_api_cmd-cmd.o: test/librados/cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cmd-cmd.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Tpo -c -o test/librados/ceph_test_rados_api_cmd-cmd.o `test -f 'test/librados/cmd.cc' || echo '$(srcdir)/'`test/librados/cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/cmd.cc' object='test/librados/ceph_test_rados_api_cmd-cmd.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cmd-cmd.o `test -f 'test/librados/cmd.cc' || echo '$(srcdir)/'`test/librados/cmd.cc + +test/librados/ceph_test_rados_api_cmd-cmd.obj: test/librados/cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_cmd-cmd.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Tpo -c -o test/librados/ceph_test_rados_api_cmd-cmd.obj `if test -f 'test/librados/cmd.cc'; then $(CYGPATH_W) 'test/librados/cmd.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/cmd.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_cmd-cmd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/cmd.cc' object='test/librados/ceph_test_rados_api_cmd-cmd.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_cmd_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_cmd-cmd.obj `if test -f 'test/librados/cmd.cc'; then $(CYGPATH_W) 'test/librados/cmd.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/cmd.cc'; fi` + +test/librados/ceph_test_rados_api_io-io.o: test/librados/io.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_io-io.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Tpo -c -o test/librados/ceph_test_rados_api_io-io.o `test -f 'test/librados/io.cc' || echo '$(srcdir)/'`test/librados/io.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/io.cc' object='test/librados/ceph_test_rados_api_io-io.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_io-io.o `test -f 'test/librados/io.cc' || echo '$(srcdir)/'`test/librados/io.cc + +test/librados/ceph_test_rados_api_io-io.obj: test/librados/io.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_io-io.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Tpo -c -o test/librados/ceph_test_rados_api_io-io.obj `if test -f 'test/librados/io.cc'; then $(CYGPATH_W) 'test/librados/io.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/io.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_io-io.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/io.cc' object='test/librados/ceph_test_rados_api_io-io.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_io_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_io-io.obj `if test -f 'test/librados/io.cc'; then $(CYGPATH_W) 'test/librados/io.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/io.cc'; fi` + +test/librados/ceph_test_rados_api_list-list.o: test/librados/list.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_list-list.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Tpo -c -o test/librados/ceph_test_rados_api_list-list.o `test -f 'test/librados/list.cc' || echo '$(srcdir)/'`test/librados/list.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/list.cc' object='test/librados/ceph_test_rados_api_list-list.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_list-list.o `test -f 'test/librados/list.cc' || echo '$(srcdir)/'`test/librados/list.cc + +test/librados/ceph_test_rados_api_list-list.obj: test/librados/list.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_list-list.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Tpo -c -o test/librados/ceph_test_rados_api_list-list.obj `if test -f 'test/librados/list.cc'; then $(CYGPATH_W) 'test/librados/list.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/list.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_list-list.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/list.cc' object='test/librados/ceph_test_rados_api_list-list.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_list_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_list-list.obj `if test -f 'test/librados/list.cc'; then $(CYGPATH_W) 'test/librados/list.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/list.cc'; fi` + +test/librados/ceph_test_rados_api_lock-lock.o: test/librados/lock.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_lock-lock.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Tpo -c -o test/librados/ceph_test_rados_api_lock-lock.o `test -f 'test/librados/lock.cc' || echo '$(srcdir)/'`test/librados/lock.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/lock.cc' object='test/librados/ceph_test_rados_api_lock-lock.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_lock-lock.o `test -f 'test/librados/lock.cc' || echo '$(srcdir)/'`test/librados/lock.cc + +test/librados/ceph_test_rados_api_lock-lock.obj: test/librados/lock.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_lock-lock.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Tpo -c -o test/librados/ceph_test_rados_api_lock-lock.obj `if test -f 'test/librados/lock.cc'; then $(CYGPATH_W) 'test/librados/lock.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/lock.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_lock-lock.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/lock.cc' object='test/librados/ceph_test_rados_api_lock-lock.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_lock_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_lock-lock.obj `if test -f 'test/librados/lock.cc'; then $(CYGPATH_W) 'test/librados/lock.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/lock.cc'; fi` + +test/librados/ceph_test_rados_api_misc-misc.o: test/librados/misc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_misc-misc.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Tpo -c -o test/librados/ceph_test_rados_api_misc-misc.o `test -f 'test/librados/misc.cc' || echo '$(srcdir)/'`test/librados/misc.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/misc.cc' object='test/librados/ceph_test_rados_api_misc-misc.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_misc-misc.o `test -f 'test/librados/misc.cc' || echo '$(srcdir)/'`test/librados/misc.cc + +test/librados/ceph_test_rados_api_misc-misc.obj: test/librados/misc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_misc-misc.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Tpo -c -o test/librados/ceph_test_rados_api_misc-misc.obj `if test -f 'test/librados/misc.cc'; then $(CYGPATH_W) 'test/librados/misc.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/misc.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_misc-misc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/misc.cc' object='test/librados/ceph_test_rados_api_misc-misc.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_misc_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_misc-misc.obj `if test -f 'test/librados/misc.cc'; then $(CYGPATH_W) 'test/librados/misc.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/misc.cc'; fi` + +test/librados/ceph_test_rados_api_pool-pool.o: test/librados/pool.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_pool-pool.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Tpo -c -o test/librados/ceph_test_rados_api_pool-pool.o `test -f 'test/librados/pool.cc' || echo '$(srcdir)/'`test/librados/pool.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/pool.cc' object='test/librados/ceph_test_rados_api_pool-pool.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_pool-pool.o `test -f 'test/librados/pool.cc' || echo '$(srcdir)/'`test/librados/pool.cc + +test/librados/ceph_test_rados_api_pool-pool.obj: test/librados/pool.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_pool-pool.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Tpo -c -o test/librados/ceph_test_rados_api_pool-pool.obj `if test -f 'test/librados/pool.cc'; then $(CYGPATH_W) 'test/librados/pool.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/pool.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_pool-pool.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/pool.cc' object='test/librados/ceph_test_rados_api_pool-pool.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_pool_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_pool-pool.obj `if test -f 'test/librados/pool.cc'; then $(CYGPATH_W) 'test/librados/pool.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/pool.cc'; fi` + +test/librados/ceph_test_rados_api_snapshots-snapshots.o: test/librados/snapshots.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_snapshots-snapshots.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Tpo -c -o test/librados/ceph_test_rados_api_snapshots-snapshots.o `test -f 'test/librados/snapshots.cc' || echo '$(srcdir)/'`test/librados/snapshots.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/snapshots.cc' object='test/librados/ceph_test_rados_api_snapshots-snapshots.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_snapshots-snapshots.o `test -f 'test/librados/snapshots.cc' || echo '$(srcdir)/'`test/librados/snapshots.cc + +test/librados/ceph_test_rados_api_snapshots-snapshots.obj: test/librados/snapshots.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_snapshots-snapshots.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Tpo -c -o test/librados/ceph_test_rados_api_snapshots-snapshots.obj `if test -f 'test/librados/snapshots.cc'; then $(CYGPATH_W) 'test/librados/snapshots.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/snapshots.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_snapshots-snapshots.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/snapshots.cc' object='test/librados/ceph_test_rados_api_snapshots-snapshots.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_snapshots_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_snapshots-snapshots.obj `if test -f 'test/librados/snapshots.cc'; then $(CYGPATH_W) 'test/librados/snapshots.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/snapshots.cc'; fi` + +test/librados/ceph_test_rados_api_stat-stat.o: test/librados/stat.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_stat-stat.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Tpo -c -o test/librados/ceph_test_rados_api_stat-stat.o `test -f 'test/librados/stat.cc' || echo '$(srcdir)/'`test/librados/stat.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/stat.cc' object='test/librados/ceph_test_rados_api_stat-stat.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_stat-stat.o `test -f 'test/librados/stat.cc' || echo '$(srcdir)/'`test/librados/stat.cc + +test/librados/ceph_test_rados_api_stat-stat.obj: test/librados/stat.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_stat-stat.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Tpo -c -o test/librados/ceph_test_rados_api_stat-stat.obj `if test -f 'test/librados/stat.cc'; then $(CYGPATH_W) 'test/librados/stat.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/stat.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_stat-stat.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/stat.cc' object='test/librados/ceph_test_rados_api_stat-stat.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_stat_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_stat-stat.obj `if test -f 'test/librados/stat.cc'; then $(CYGPATH_W) 'test/librados/stat.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/stat.cc'; fi` + +test/librados/ceph_test_rados_api_tier-tier.o: test/librados/tier.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-tier.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo -c -o test/librados/ceph_test_rados_api_tier-tier.o `test -f 'test/librados/tier.cc' || echo '$(srcdir)/'`test/librados/tier.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/tier.cc' object='test/librados/ceph_test_rados_api_tier-tier.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-tier.o `test -f 'test/librados/tier.cc' || echo '$(srcdir)/'`test/librados/tier.cc + +test/librados/ceph_test_rados_api_tier-tier.obj: test/librados/tier.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_tier-tier.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo -c -o test/librados/ceph_test_rados_api_tier-tier.obj `if test -f 'test/librados/tier.cc'; then $(CYGPATH_W) 'test/librados/tier.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/tier.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_tier-tier.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/tier.cc' object='test/librados/ceph_test_rados_api_tier-tier.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_tier-tier.obj `if test -f 'test/librados/tier.cc'; then $(CYGPATH_W) 'test/librados/tier.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/tier.cc'; fi` + +osd/ceph_test_rados_api_tier-HitSet.o: osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT osd/ceph_test_rados_api_tier-HitSet.o -MD -MP -MF osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo -c -o osd/ceph_test_rados_api_tier-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/ceph_test_rados_api_tier-HitSet.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o osd/ceph_test_rados_api_tier-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc + +osd/ceph_test_rados_api_tier-HitSet.obj: osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -MT osd/ceph_test_rados_api_tier-HitSet.obj -MD -MP -MF osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo -c -o osd/ceph_test_rados_api_tier-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Tpo osd/$(DEPDIR)/ceph_test_rados_api_tier-HitSet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/ceph_test_rados_api_tier-HitSet.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_tier_CXXFLAGS) $(CXXFLAGS) -c -o osd/ceph_test_rados_api_tier-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi` + +test/librados/ceph_test_rados_api_watch_notify-watch_notify.o: test/librados/watch_notify.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_watch_notify-watch_notify.o -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Tpo -c -o test/librados/ceph_test_rados_api_watch_notify-watch_notify.o `test -f 'test/librados/watch_notify.cc' || echo '$(srcdir)/'`test/librados/watch_notify.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/watch_notify.cc' object='test/librados/ceph_test_rados_api_watch_notify-watch_notify.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_watch_notify-watch_notify.o `test -f 'test/librados/watch_notify.cc' || echo '$(srcdir)/'`test/librados/watch_notify.cc + +test/librados/ceph_test_rados_api_watch_notify-watch_notify.obj: test/librados/watch_notify.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -MT test/librados/ceph_test_rados_api_watch_notify-watch_notify.obj -MD -MP -MF test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Tpo -c -o test/librados/ceph_test_rados_api_watch_notify-watch_notify.obj `if test -f 'test/librados/watch_notify.cc'; then $(CYGPATH_W) 'test/librados/watch_notify.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/watch_notify.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Tpo test/librados/$(DEPDIR)/ceph_test_rados_api_watch_notify-watch_notify.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/watch_notify.cc' object='test/librados/ceph_test_rados_api_watch_notify-watch_notify.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rados_api_watch_notify_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/ceph_test_rados_api_watch_notify-watch_notify.obj `if test -f 'test/librados/watch_notify.cc'; then $(CYGPATH_W) 'test/librados/watch_notify.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/watch_notify.cc'; fi` + +test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o: test/rgw/test_rgw_manifest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -MT test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o -MD -MP -MF test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o `test -f 'test/rgw/test_rgw_manifest.cc' || echo '$(srcdir)/'`test/rgw/test_rgw_manifest.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/rgw/test_rgw_manifest.cc' object='test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.o `test -f 'test/rgw/test_rgw_manifest.cc' || echo '$(srcdir)/'`test/rgw/test_rgw_manifest.cc + +test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj: test/rgw/test_rgw_manifest.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -MT test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj -MD -MP -MF test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj `if test -f 'test/rgw/test_rgw_manifest.cc'; then $(CYGPATH_W) 'test/rgw/test_rgw_manifest.cc'; else $(CYGPATH_W) '$(srcdir)/test/rgw/test_rgw_manifest.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Tpo test/rgw/$(DEPDIR)/ceph_test_rgw_manifest-test_rgw_manifest.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/rgw/test_rgw_manifest.cc' object='test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_rgw_manifest_CXXFLAGS) $(CXXFLAGS) -c -o test/rgw/ceph_test_rgw_manifest-test_rgw_manifest.obj `if test -f 'test/rgw/test_rgw_manifest.cc'; then $(CYGPATH_W) 'test/rgw/test_rgw_manifest.cc'; else $(CYGPATH_W) '$(srcdir)/test/rgw/test_rgw_manifest.cc'; fi` + +test/ceph_test_snap_mapper-test_snap_mapper.o: test/test_snap_mapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_snap_mapper_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_snap_mapper-test_snap_mapper.o -MD -MP -MF test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Tpo -c -o test/ceph_test_snap_mapper-test_snap_mapper.o `test -f 'test/test_snap_mapper.cc' || echo '$(srcdir)/'`test/test_snap_mapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Tpo test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_snap_mapper.cc' object='test/ceph_test_snap_mapper-test_snap_mapper.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_snap_mapper_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_snap_mapper-test_snap_mapper.o `test -f 'test/test_snap_mapper.cc' || echo '$(srcdir)/'`test/test_snap_mapper.cc + +test/ceph_test_snap_mapper-test_snap_mapper.obj: test/test_snap_mapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_snap_mapper_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_snap_mapper-test_snap_mapper.obj -MD -MP -MF test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Tpo -c -o test/ceph_test_snap_mapper-test_snap_mapper.obj `if test -f 'test/test_snap_mapper.cc'; then $(CYGPATH_W) 'test/test_snap_mapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_snap_mapper.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Tpo test/$(DEPDIR)/ceph_test_snap_mapper-test_snap_mapper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_snap_mapper.cc' object='test/ceph_test_snap_mapper-test_snap_mapper.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_snap_mapper_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_snap_mapper-test_snap_mapper.obj `if test -f 'test/test_snap_mapper.cc'; then $(CYGPATH_W) 'test/test_snap_mapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_snap_mapper.cc'; fi` + +test/ceph_test_stress_watch-test_stress_watch.o: test/test_stress_watch.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_stress_watch-test_stress_watch.o -MD -MP -MF test/$(DEPDIR)/ceph_test_stress_watch-test_stress_watch.Tpo -c -o test/ceph_test_stress_watch-test_stress_watch.o `test -f 'test/test_stress_watch.cc' || echo '$(srcdir)/'`test/test_stress_watch.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_stress_watch-test_stress_watch.Tpo test/$(DEPDIR)/ceph_test_stress_watch-test_stress_watch.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_stress_watch.cc' object='test/ceph_test_stress_watch-test_stress_watch.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_stress_watch-test_stress_watch.o `test -f 'test/test_stress_watch.cc' || echo '$(srcdir)/'`test/test_stress_watch.cc + +test/ceph_test_stress_watch-test_stress_watch.obj: test/test_stress_watch.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_test_stress_watch-test_stress_watch.obj -MD -MP -MF test/$(DEPDIR)/ceph_test_stress_watch-test_stress_watch.Tpo -c -o test/ceph_test_stress_watch-test_stress_watch.obj `if test -f 'test/test_stress_watch.cc'; then $(CYGPATH_W) 'test/test_stress_watch.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_stress_watch.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_test_stress_watch-test_stress_watch.Tpo test/$(DEPDIR)/ceph_test_stress_watch-test_stress_watch.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_stress_watch.cc' object='test/ceph_test_stress_watch-test_stress_watch.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_test_stress_watch_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_test_stress_watch-test_stress_watch.obj `if test -f 'test/test_stress_watch.cc'; then $(CYGPATH_W) 'test/test_stress_watch.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_stress_watch.cc'; fi` + +test/ceph_xattr_bench-xattr_bench.o: test/xattr_bench.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_xattr_bench_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_xattr_bench-xattr_bench.o -MD -MP -MF test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Tpo -c -o test/ceph_xattr_bench-xattr_bench.o `test -f 'test/xattr_bench.cc' || echo '$(srcdir)/'`test/xattr_bench.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Tpo test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/xattr_bench.cc' object='test/ceph_xattr_bench-xattr_bench.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_xattr_bench_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_xattr_bench-xattr_bench.o `test -f 'test/xattr_bench.cc' || echo '$(srcdir)/'`test/xattr_bench.cc + +test/ceph_xattr_bench-xattr_bench.obj: test/xattr_bench.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_xattr_bench_CXXFLAGS) $(CXXFLAGS) -MT test/ceph_xattr_bench-xattr_bench.obj -MD -MP -MF test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Tpo -c -o test/ceph_xattr_bench-xattr_bench.obj `if test -f 'test/xattr_bench.cc'; then $(CYGPATH_W) 'test/xattr_bench.cc'; else $(CYGPATH_W) '$(srcdir)/test/xattr_bench.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Tpo test/$(DEPDIR)/ceph_xattr_bench-xattr_bench.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/xattr_bench.cc' object='test/ceph_xattr_bench-xattr_bench.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(ceph_xattr_bench_CXXFLAGS) $(CXXFLAGS) -c -o test/ceph_xattr_bench-xattr_bench.obj `if test -f 'test/xattr_bench.cc'; then $(CYGPATH_W) 'test/xattr_bench.cc'; else $(CYGPATH_W) '$(srcdir)/test/xattr_bench.cc'; fi` + +tools/rest_bench-rest_bench.o: tools/rest_bench.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -MT tools/rest_bench-rest_bench.o -MD -MP -MF tools/$(DEPDIR)/rest_bench-rest_bench.Tpo -c -o tools/rest_bench-rest_bench.o `test -f 'tools/rest_bench.cc' || echo '$(srcdir)/'`tools/rest_bench.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) tools/$(DEPDIR)/rest_bench-rest_bench.Tpo tools/$(DEPDIR)/rest_bench-rest_bench.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tools/rest_bench.cc' object='tools/rest_bench-rest_bench.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -c -o tools/rest_bench-rest_bench.o `test -f 'tools/rest_bench.cc' || echo '$(srcdir)/'`tools/rest_bench.cc + +tools/rest_bench-rest_bench.obj: tools/rest_bench.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -MT tools/rest_bench-rest_bench.obj -MD -MP -MF tools/$(DEPDIR)/rest_bench-rest_bench.Tpo -c -o tools/rest_bench-rest_bench.obj `if test -f 'tools/rest_bench.cc'; then $(CYGPATH_W) 'tools/rest_bench.cc'; else $(CYGPATH_W) '$(srcdir)/tools/rest_bench.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) tools/$(DEPDIR)/rest_bench-rest_bench.Tpo tools/$(DEPDIR)/rest_bench-rest_bench.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tools/rest_bench.cc' object='tools/rest_bench-rest_bench.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -c -o tools/rest_bench-rest_bench.obj `if test -f 'tools/rest_bench.cc'; then $(CYGPATH_W) 'tools/rest_bench.cc'; else $(CYGPATH_W) '$(srcdir)/tools/rest_bench.cc'; fi` + +common/rest_bench-obj_bencher.o: common/obj_bencher.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -MT common/rest_bench-obj_bencher.o -MD -MP -MF common/$(DEPDIR)/rest_bench-obj_bencher.Tpo -c -o common/rest_bench-obj_bencher.o `test -f 'common/obj_bencher.cc' || echo '$(srcdir)/'`common/obj_bencher.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/rest_bench-obj_bencher.Tpo common/$(DEPDIR)/rest_bench-obj_bencher.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/obj_bencher.cc' object='common/rest_bench-obj_bencher.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -c -o common/rest_bench-obj_bencher.o `test -f 'common/obj_bencher.cc' || echo '$(srcdir)/'`common/obj_bencher.cc + +common/rest_bench-obj_bencher.obj: common/obj_bencher.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -MT common/rest_bench-obj_bencher.obj -MD -MP -MF common/$(DEPDIR)/rest_bench-obj_bencher.Tpo -c -o common/rest_bench-obj_bencher.obj `if test -f 'common/obj_bencher.cc'; then $(CYGPATH_W) 'common/obj_bencher.cc'; else $(CYGPATH_W) '$(srcdir)/common/obj_bencher.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/rest_bench-obj_bencher.Tpo common/$(DEPDIR)/rest_bench-obj_bencher.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/obj_bencher.cc' object='common/rest_bench-obj_bencher.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(rest_bench_CXXFLAGS) $(CXXFLAGS) -c -o common/rest_bench-obj_bencher.obj `if test -f 'common/obj_bencher.cc'; then $(CYGPATH_W) 'common/obj_bencher.cc'; else $(CYGPATH_W) '$(srcdir)/common/obj_bencher.cc'; fi` + +test/test_build_libcephfs-buildtest_skeleton.o: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_libcephfs-buildtest_skeleton.o -MD -MP -MF test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Tpo -c -o test/test_build_libcephfs-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_libcephfs-buildtest_skeleton.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_libcephfs-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc + +test/test_build_libcephfs-buildtest_skeleton.obj: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_libcephfs-buildtest_skeleton.obj -MD -MP -MF test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Tpo -c -o test/test_build_libcephfs-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_libcephfs-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_libcephfs-buildtest_skeleton.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_libcephfs-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` + +osdc/test_build_libcephfs-Objecter.o: osdc/Objecter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Objecter.o -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Objecter.Tpo -c -o osdc/test_build_libcephfs-Objecter.o `test -f 'osdc/Objecter.cc' || echo '$(srcdir)/'`osdc/Objecter.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Objecter.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Objecter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Objecter.cc' object='osdc/test_build_libcephfs-Objecter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Objecter.o `test -f 'osdc/Objecter.cc' || echo '$(srcdir)/'`osdc/Objecter.cc + +osdc/test_build_libcephfs-Objecter.obj: osdc/Objecter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Objecter.obj -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Objecter.Tpo -c -o osdc/test_build_libcephfs-Objecter.obj `if test -f 'osdc/Objecter.cc'; then $(CYGPATH_W) 'osdc/Objecter.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Objecter.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Objecter.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Objecter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Objecter.cc' object='osdc/test_build_libcephfs-Objecter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Objecter.obj `if test -f 'osdc/Objecter.cc'; then $(CYGPATH_W) 'osdc/Objecter.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Objecter.cc'; fi` + +osdc/test_build_libcephfs-ObjectCacher.o: osdc/ObjectCacher.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-ObjectCacher.o -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-ObjectCacher.Tpo -c -o osdc/test_build_libcephfs-ObjectCacher.o `test -f 'osdc/ObjectCacher.cc' || echo '$(srcdir)/'`osdc/ObjectCacher.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-ObjectCacher.Tpo osdc/$(DEPDIR)/test_build_libcephfs-ObjectCacher.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/ObjectCacher.cc' object='osdc/test_build_libcephfs-ObjectCacher.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-ObjectCacher.o `test -f 'osdc/ObjectCacher.cc' || echo '$(srcdir)/'`osdc/ObjectCacher.cc + +osdc/test_build_libcephfs-ObjectCacher.obj: osdc/ObjectCacher.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-ObjectCacher.obj -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-ObjectCacher.Tpo -c -o osdc/test_build_libcephfs-ObjectCacher.obj `if test -f 'osdc/ObjectCacher.cc'; then $(CYGPATH_W) 'osdc/ObjectCacher.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/ObjectCacher.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-ObjectCacher.Tpo osdc/$(DEPDIR)/test_build_libcephfs-ObjectCacher.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/ObjectCacher.cc' object='osdc/test_build_libcephfs-ObjectCacher.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-ObjectCacher.obj `if test -f 'osdc/ObjectCacher.cc'; then $(CYGPATH_W) 'osdc/ObjectCacher.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/ObjectCacher.cc'; fi` + +osdc/test_build_libcephfs-Filer.o: osdc/Filer.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Filer.o -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Filer.Tpo -c -o osdc/test_build_libcephfs-Filer.o `test -f 'osdc/Filer.cc' || echo '$(srcdir)/'`osdc/Filer.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Filer.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Filer.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Filer.cc' object='osdc/test_build_libcephfs-Filer.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Filer.o `test -f 'osdc/Filer.cc' || echo '$(srcdir)/'`osdc/Filer.cc + +osdc/test_build_libcephfs-Filer.obj: osdc/Filer.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Filer.obj -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Filer.Tpo -c -o osdc/test_build_libcephfs-Filer.obj `if test -f 'osdc/Filer.cc'; then $(CYGPATH_W) 'osdc/Filer.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Filer.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Filer.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Filer.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Filer.cc' object='osdc/test_build_libcephfs-Filer.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Filer.obj `if test -f 'osdc/Filer.cc'; then $(CYGPATH_W) 'osdc/Filer.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Filer.cc'; fi` + +osdc/test_build_libcephfs-Striper.o: osdc/Striper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Striper.o -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Striper.Tpo -c -o osdc/test_build_libcephfs-Striper.o `test -f 'osdc/Striper.cc' || echo '$(srcdir)/'`osdc/Striper.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Striper.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Striper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Striper.cc' object='osdc/test_build_libcephfs-Striper.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Striper.o `test -f 'osdc/Striper.cc' || echo '$(srcdir)/'`osdc/Striper.cc + +osdc/test_build_libcephfs-Striper.obj: osdc/Striper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Striper.obj -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Striper.Tpo -c -o osdc/test_build_libcephfs-Striper.obj `if test -f 'osdc/Striper.cc'; then $(CYGPATH_W) 'osdc/Striper.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Striper.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Striper.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Striper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Striper.cc' object='osdc/test_build_libcephfs-Striper.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Striper.obj `if test -f 'osdc/Striper.cc'; then $(CYGPATH_W) 'osdc/Striper.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Striper.cc'; fi` + +osdc/test_build_libcephfs-Journaler.o: osdc/Journaler.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Journaler.o -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Journaler.Tpo -c -o osdc/test_build_libcephfs-Journaler.o `test -f 'osdc/Journaler.cc' || echo '$(srcdir)/'`osdc/Journaler.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Journaler.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Journaler.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Journaler.cc' object='osdc/test_build_libcephfs-Journaler.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Journaler.o `test -f 'osdc/Journaler.cc' || echo '$(srcdir)/'`osdc/Journaler.cc + +osdc/test_build_libcephfs-Journaler.obj: osdc/Journaler.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -MT osdc/test_build_libcephfs-Journaler.obj -MD -MP -MF osdc/$(DEPDIR)/test_build_libcephfs-Journaler.Tpo -c -o osdc/test_build_libcephfs-Journaler.obj `if test -f 'osdc/Journaler.cc'; then $(CYGPATH_W) 'osdc/Journaler.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Journaler.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osdc/$(DEPDIR)/test_build_libcephfs-Journaler.Tpo osdc/$(DEPDIR)/test_build_libcephfs-Journaler.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osdc/Journaler.cc' object='osdc/test_build_libcephfs-Journaler.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcephfs_CXXFLAGS) $(CXXFLAGS) -c -o osdc/test_build_libcephfs-Journaler.obj `if test -f 'osdc/Journaler.cc'; then $(CYGPATH_W) 'osdc/Journaler.cc'; else $(CYGPATH_W) '$(srcdir)/osdc/Journaler.cc'; fi` + +test/test_build_libcommon-buildtest_skeleton.o: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_libcommon-buildtest_skeleton.o -MD -MP -MF test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Tpo -c -o test/test_build_libcommon-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_libcommon-buildtest_skeleton.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_libcommon-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc + +test/test_build_libcommon-buildtest_skeleton.obj: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_libcommon-buildtest_skeleton.obj -MD -MP -MF test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Tpo -c -o test/test_build_libcommon-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_libcommon-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_libcommon-buildtest_skeleton.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_libcommon-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` + +common/test_build_libcommon-DecayCounter.o: common/DecayCounter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-DecayCounter.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-DecayCounter.Tpo -c -o common/test_build_libcommon-DecayCounter.o `test -f 'common/DecayCounter.cc' || echo '$(srcdir)/'`common/DecayCounter.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-DecayCounter.Tpo common/$(DEPDIR)/test_build_libcommon-DecayCounter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/DecayCounter.cc' object='common/test_build_libcommon-DecayCounter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-DecayCounter.o `test -f 'common/DecayCounter.cc' || echo '$(srcdir)/'`common/DecayCounter.cc + +common/test_build_libcommon-DecayCounter.obj: common/DecayCounter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-DecayCounter.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-DecayCounter.Tpo -c -o common/test_build_libcommon-DecayCounter.obj `if test -f 'common/DecayCounter.cc'; then $(CYGPATH_W) 'common/DecayCounter.cc'; else $(CYGPATH_W) '$(srcdir)/common/DecayCounter.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-DecayCounter.Tpo common/$(DEPDIR)/test_build_libcommon-DecayCounter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/DecayCounter.cc' object='common/test_build_libcommon-DecayCounter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-DecayCounter.obj `if test -f 'common/DecayCounter.cc'; then $(CYGPATH_W) 'common/DecayCounter.cc'; else $(CYGPATH_W) '$(srcdir)/common/DecayCounter.cc'; fi` + +common/test_build_libcommon-LogClient.o: common/LogClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-LogClient.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-LogClient.Tpo -c -o common/test_build_libcommon-LogClient.o `test -f 'common/LogClient.cc' || echo '$(srcdir)/'`common/LogClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-LogClient.Tpo common/$(DEPDIR)/test_build_libcommon-LogClient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/LogClient.cc' object='common/test_build_libcommon-LogClient.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-LogClient.o `test -f 'common/LogClient.cc' || echo '$(srcdir)/'`common/LogClient.cc + +common/test_build_libcommon-LogClient.obj: common/LogClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-LogClient.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-LogClient.Tpo -c -o common/test_build_libcommon-LogClient.obj `if test -f 'common/LogClient.cc'; then $(CYGPATH_W) 'common/LogClient.cc'; else $(CYGPATH_W) '$(srcdir)/common/LogClient.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-LogClient.Tpo common/$(DEPDIR)/test_build_libcommon-LogClient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/LogClient.cc' object='common/test_build_libcommon-LogClient.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-LogClient.obj `if test -f 'common/LogClient.cc'; then $(CYGPATH_W) 'common/LogClient.cc'; else $(CYGPATH_W) '$(srcdir)/common/LogClient.cc'; fi` + +common/test_build_libcommon-LogEntry.o: common/LogEntry.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-LogEntry.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-LogEntry.Tpo -c -o common/test_build_libcommon-LogEntry.o `test -f 'common/LogEntry.cc' || echo '$(srcdir)/'`common/LogEntry.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-LogEntry.Tpo common/$(DEPDIR)/test_build_libcommon-LogEntry.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/LogEntry.cc' object='common/test_build_libcommon-LogEntry.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-LogEntry.o `test -f 'common/LogEntry.cc' || echo '$(srcdir)/'`common/LogEntry.cc + +common/test_build_libcommon-LogEntry.obj: common/LogEntry.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-LogEntry.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-LogEntry.Tpo -c -o common/test_build_libcommon-LogEntry.obj `if test -f 'common/LogEntry.cc'; then $(CYGPATH_W) 'common/LogEntry.cc'; else $(CYGPATH_W) '$(srcdir)/common/LogEntry.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-LogEntry.Tpo common/$(DEPDIR)/test_build_libcommon-LogEntry.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/LogEntry.cc' object='common/test_build_libcommon-LogEntry.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-LogEntry.obj `if test -f 'common/LogEntry.cc'; then $(CYGPATH_W) 'common/LogEntry.cc'; else $(CYGPATH_W) '$(srcdir)/common/LogEntry.cc'; fi` + +common/test_build_libcommon-PrebufferedStreambuf.o: common/PrebufferedStreambuf.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-PrebufferedStreambuf.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-PrebufferedStreambuf.Tpo -c -o common/test_build_libcommon-PrebufferedStreambuf.o `test -f 'common/PrebufferedStreambuf.cc' || echo '$(srcdir)/'`common/PrebufferedStreambuf.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-PrebufferedStreambuf.Tpo common/$(DEPDIR)/test_build_libcommon-PrebufferedStreambuf.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/PrebufferedStreambuf.cc' object='common/test_build_libcommon-PrebufferedStreambuf.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-PrebufferedStreambuf.o `test -f 'common/PrebufferedStreambuf.cc' || echo '$(srcdir)/'`common/PrebufferedStreambuf.cc + +common/test_build_libcommon-PrebufferedStreambuf.obj: common/PrebufferedStreambuf.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-PrebufferedStreambuf.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-PrebufferedStreambuf.Tpo -c -o common/test_build_libcommon-PrebufferedStreambuf.obj `if test -f 'common/PrebufferedStreambuf.cc'; then $(CYGPATH_W) 'common/PrebufferedStreambuf.cc'; else $(CYGPATH_W) '$(srcdir)/common/PrebufferedStreambuf.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-PrebufferedStreambuf.Tpo common/$(DEPDIR)/test_build_libcommon-PrebufferedStreambuf.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/PrebufferedStreambuf.cc' object='common/test_build_libcommon-PrebufferedStreambuf.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-PrebufferedStreambuf.obj `if test -f 'common/PrebufferedStreambuf.cc'; then $(CYGPATH_W) 'common/PrebufferedStreambuf.cc'; else $(CYGPATH_W) '$(srcdir)/common/PrebufferedStreambuf.cc'; fi` + +common/test_build_libcommon-SloppyCRCMap.o: common/SloppyCRCMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-SloppyCRCMap.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-SloppyCRCMap.Tpo -c -o common/test_build_libcommon-SloppyCRCMap.o `test -f 'common/SloppyCRCMap.cc' || echo '$(srcdir)/'`common/SloppyCRCMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-SloppyCRCMap.Tpo common/$(DEPDIR)/test_build_libcommon-SloppyCRCMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/SloppyCRCMap.cc' object='common/test_build_libcommon-SloppyCRCMap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-SloppyCRCMap.o `test -f 'common/SloppyCRCMap.cc' || echo '$(srcdir)/'`common/SloppyCRCMap.cc + +common/test_build_libcommon-SloppyCRCMap.obj: common/SloppyCRCMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-SloppyCRCMap.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-SloppyCRCMap.Tpo -c -o common/test_build_libcommon-SloppyCRCMap.obj `if test -f 'common/SloppyCRCMap.cc'; then $(CYGPATH_W) 'common/SloppyCRCMap.cc'; else $(CYGPATH_W) '$(srcdir)/common/SloppyCRCMap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-SloppyCRCMap.Tpo common/$(DEPDIR)/test_build_libcommon-SloppyCRCMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/SloppyCRCMap.cc' object='common/test_build_libcommon-SloppyCRCMap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-SloppyCRCMap.obj `if test -f 'common/SloppyCRCMap.cc'; then $(CYGPATH_W) 'common/SloppyCRCMap.cc'; else $(CYGPATH_W) '$(srcdir)/common/SloppyCRCMap.cc'; fi` + +common/test_build_libcommon-BackTrace.o: common/BackTrace.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-BackTrace.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-BackTrace.Tpo -c -o common/test_build_libcommon-BackTrace.o `test -f 'common/BackTrace.cc' || echo '$(srcdir)/'`common/BackTrace.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-BackTrace.Tpo common/$(DEPDIR)/test_build_libcommon-BackTrace.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/BackTrace.cc' object='common/test_build_libcommon-BackTrace.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-BackTrace.o `test -f 'common/BackTrace.cc' || echo '$(srcdir)/'`common/BackTrace.cc + +common/test_build_libcommon-BackTrace.obj: common/BackTrace.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-BackTrace.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-BackTrace.Tpo -c -o common/test_build_libcommon-BackTrace.obj `if test -f 'common/BackTrace.cc'; then $(CYGPATH_W) 'common/BackTrace.cc'; else $(CYGPATH_W) '$(srcdir)/common/BackTrace.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-BackTrace.Tpo common/$(DEPDIR)/test_build_libcommon-BackTrace.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/BackTrace.cc' object='common/test_build_libcommon-BackTrace.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-BackTrace.obj `if test -f 'common/BackTrace.cc'; then $(CYGPATH_W) 'common/BackTrace.cc'; else $(CYGPATH_W) '$(srcdir)/common/BackTrace.cc'; fi` + +common/test_build_libcommon-perf_counters.o: common/perf_counters.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-perf_counters.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-perf_counters.Tpo -c -o common/test_build_libcommon-perf_counters.o `test -f 'common/perf_counters.cc' || echo '$(srcdir)/'`common/perf_counters.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-perf_counters.Tpo common/$(DEPDIR)/test_build_libcommon-perf_counters.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/perf_counters.cc' object='common/test_build_libcommon-perf_counters.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-perf_counters.o `test -f 'common/perf_counters.cc' || echo '$(srcdir)/'`common/perf_counters.cc + +common/test_build_libcommon-perf_counters.obj: common/perf_counters.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-perf_counters.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-perf_counters.Tpo -c -o common/test_build_libcommon-perf_counters.obj `if test -f 'common/perf_counters.cc'; then $(CYGPATH_W) 'common/perf_counters.cc'; else $(CYGPATH_W) '$(srcdir)/common/perf_counters.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-perf_counters.Tpo common/$(DEPDIR)/test_build_libcommon-perf_counters.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/perf_counters.cc' object='common/test_build_libcommon-perf_counters.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-perf_counters.obj `if test -f 'common/perf_counters.cc'; then $(CYGPATH_W) 'common/perf_counters.cc'; else $(CYGPATH_W) '$(srcdir)/common/perf_counters.cc'; fi` + +common/test_build_libcommon-Mutex.o: common/Mutex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Mutex.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Mutex.Tpo -c -o common/test_build_libcommon-Mutex.o `test -f 'common/Mutex.cc' || echo '$(srcdir)/'`common/Mutex.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Mutex.Tpo common/$(DEPDIR)/test_build_libcommon-Mutex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Mutex.cc' object='common/test_build_libcommon-Mutex.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Mutex.o `test -f 'common/Mutex.cc' || echo '$(srcdir)/'`common/Mutex.cc + +common/test_build_libcommon-Mutex.obj: common/Mutex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Mutex.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Mutex.Tpo -c -o common/test_build_libcommon-Mutex.obj `if test -f 'common/Mutex.cc'; then $(CYGPATH_W) 'common/Mutex.cc'; else $(CYGPATH_W) '$(srcdir)/common/Mutex.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Mutex.Tpo common/$(DEPDIR)/test_build_libcommon-Mutex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Mutex.cc' object='common/test_build_libcommon-Mutex.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Mutex.obj `if test -f 'common/Mutex.cc'; then $(CYGPATH_W) 'common/Mutex.cc'; else $(CYGPATH_W) '$(srcdir)/common/Mutex.cc'; fi` + +common/test_build_libcommon-OutputDataSocket.o: common/OutputDataSocket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-OutputDataSocket.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-OutputDataSocket.Tpo -c -o common/test_build_libcommon-OutputDataSocket.o `test -f 'common/OutputDataSocket.cc' || echo '$(srcdir)/'`common/OutputDataSocket.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-OutputDataSocket.Tpo common/$(DEPDIR)/test_build_libcommon-OutputDataSocket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/OutputDataSocket.cc' object='common/test_build_libcommon-OutputDataSocket.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-OutputDataSocket.o `test -f 'common/OutputDataSocket.cc' || echo '$(srcdir)/'`common/OutputDataSocket.cc + +common/test_build_libcommon-OutputDataSocket.obj: common/OutputDataSocket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-OutputDataSocket.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-OutputDataSocket.Tpo -c -o common/test_build_libcommon-OutputDataSocket.obj `if test -f 'common/OutputDataSocket.cc'; then $(CYGPATH_W) 'common/OutputDataSocket.cc'; else $(CYGPATH_W) '$(srcdir)/common/OutputDataSocket.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-OutputDataSocket.Tpo common/$(DEPDIR)/test_build_libcommon-OutputDataSocket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/OutputDataSocket.cc' object='common/test_build_libcommon-OutputDataSocket.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-OutputDataSocket.obj `if test -f 'common/OutputDataSocket.cc'; then $(CYGPATH_W) 'common/OutputDataSocket.cc'; else $(CYGPATH_W) '$(srcdir)/common/OutputDataSocket.cc'; fi` + +common/test_build_libcommon-admin_socket.o: common/admin_socket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-admin_socket.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-admin_socket.Tpo -c -o common/test_build_libcommon-admin_socket.o `test -f 'common/admin_socket.cc' || echo '$(srcdir)/'`common/admin_socket.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-admin_socket.Tpo common/$(DEPDIR)/test_build_libcommon-admin_socket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/admin_socket.cc' object='common/test_build_libcommon-admin_socket.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-admin_socket.o `test -f 'common/admin_socket.cc' || echo '$(srcdir)/'`common/admin_socket.cc + +common/test_build_libcommon-admin_socket.obj: common/admin_socket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-admin_socket.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-admin_socket.Tpo -c -o common/test_build_libcommon-admin_socket.obj `if test -f 'common/admin_socket.cc'; then $(CYGPATH_W) 'common/admin_socket.cc'; else $(CYGPATH_W) '$(srcdir)/common/admin_socket.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-admin_socket.Tpo common/$(DEPDIR)/test_build_libcommon-admin_socket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/admin_socket.cc' object='common/test_build_libcommon-admin_socket.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-admin_socket.obj `if test -f 'common/admin_socket.cc'; then $(CYGPATH_W) 'common/admin_socket.cc'; else $(CYGPATH_W) '$(srcdir)/common/admin_socket.cc'; fi` + +common/test_build_libcommon-admin_socket_client.o: common/admin_socket_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-admin_socket_client.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-admin_socket_client.Tpo -c -o common/test_build_libcommon-admin_socket_client.o `test -f 'common/admin_socket_client.cc' || echo '$(srcdir)/'`common/admin_socket_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-admin_socket_client.Tpo common/$(DEPDIR)/test_build_libcommon-admin_socket_client.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/admin_socket_client.cc' object='common/test_build_libcommon-admin_socket_client.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-admin_socket_client.o `test -f 'common/admin_socket_client.cc' || echo '$(srcdir)/'`common/admin_socket_client.cc + +common/test_build_libcommon-admin_socket_client.obj: common/admin_socket_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-admin_socket_client.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-admin_socket_client.Tpo -c -o common/test_build_libcommon-admin_socket_client.obj `if test -f 'common/admin_socket_client.cc'; then $(CYGPATH_W) 'common/admin_socket_client.cc'; else $(CYGPATH_W) '$(srcdir)/common/admin_socket_client.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-admin_socket_client.Tpo common/$(DEPDIR)/test_build_libcommon-admin_socket_client.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/admin_socket_client.cc' object='common/test_build_libcommon-admin_socket_client.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-admin_socket_client.obj `if test -f 'common/admin_socket_client.cc'; then $(CYGPATH_W) 'common/admin_socket_client.cc'; else $(CYGPATH_W) '$(srcdir)/common/admin_socket_client.cc'; fi` + +common/test_build_libcommon-cmdparse.o: common/cmdparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-cmdparse.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-cmdparse.Tpo -c -o common/test_build_libcommon-cmdparse.o `test -f 'common/cmdparse.cc' || echo '$(srcdir)/'`common/cmdparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-cmdparse.Tpo common/$(DEPDIR)/test_build_libcommon-cmdparse.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/cmdparse.cc' object='common/test_build_libcommon-cmdparse.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-cmdparse.o `test -f 'common/cmdparse.cc' || echo '$(srcdir)/'`common/cmdparse.cc + +common/test_build_libcommon-cmdparse.obj: common/cmdparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-cmdparse.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-cmdparse.Tpo -c -o common/test_build_libcommon-cmdparse.obj `if test -f 'common/cmdparse.cc'; then $(CYGPATH_W) 'common/cmdparse.cc'; else $(CYGPATH_W) '$(srcdir)/common/cmdparse.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-cmdparse.Tpo common/$(DEPDIR)/test_build_libcommon-cmdparse.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/cmdparse.cc' object='common/test_build_libcommon-cmdparse.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-cmdparse.obj `if test -f 'common/cmdparse.cc'; then $(CYGPATH_W) 'common/cmdparse.cc'; else $(CYGPATH_W) '$(srcdir)/common/cmdparse.cc'; fi` + +common/test_build_libcommon-io_priority.o: common/io_priority.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-io_priority.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-io_priority.Tpo -c -o common/test_build_libcommon-io_priority.o `test -f 'common/io_priority.cc' || echo '$(srcdir)/'`common/io_priority.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-io_priority.Tpo common/$(DEPDIR)/test_build_libcommon-io_priority.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/io_priority.cc' object='common/test_build_libcommon-io_priority.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-io_priority.o `test -f 'common/io_priority.cc' || echo '$(srcdir)/'`common/io_priority.cc + +common/test_build_libcommon-io_priority.obj: common/io_priority.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-io_priority.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-io_priority.Tpo -c -o common/test_build_libcommon-io_priority.obj `if test -f 'common/io_priority.cc'; then $(CYGPATH_W) 'common/io_priority.cc'; else $(CYGPATH_W) '$(srcdir)/common/io_priority.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-io_priority.Tpo common/$(DEPDIR)/test_build_libcommon-io_priority.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/io_priority.cc' object='common/test_build_libcommon-io_priority.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-io_priority.obj `if test -f 'common/io_priority.cc'; then $(CYGPATH_W) 'common/io_priority.cc'; else $(CYGPATH_W) '$(srcdir)/common/io_priority.cc'; fi` + +common/test_build_libcommon-Clock.o: common/Clock.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Clock.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Clock.Tpo -c -o common/test_build_libcommon-Clock.o `test -f 'common/Clock.cc' || echo '$(srcdir)/'`common/Clock.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Clock.Tpo common/$(DEPDIR)/test_build_libcommon-Clock.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Clock.cc' object='common/test_build_libcommon-Clock.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Clock.o `test -f 'common/Clock.cc' || echo '$(srcdir)/'`common/Clock.cc + +common/test_build_libcommon-Clock.obj: common/Clock.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Clock.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Clock.Tpo -c -o common/test_build_libcommon-Clock.obj `if test -f 'common/Clock.cc'; then $(CYGPATH_W) 'common/Clock.cc'; else $(CYGPATH_W) '$(srcdir)/common/Clock.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Clock.Tpo common/$(DEPDIR)/test_build_libcommon-Clock.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Clock.cc' object='common/test_build_libcommon-Clock.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Clock.obj `if test -f 'common/Clock.cc'; then $(CYGPATH_W) 'common/Clock.cc'; else $(CYGPATH_W) '$(srcdir)/common/Clock.cc'; fi` + +common/test_build_libcommon-Throttle.o: common/Throttle.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Throttle.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Throttle.Tpo -c -o common/test_build_libcommon-Throttle.o `test -f 'common/Throttle.cc' || echo '$(srcdir)/'`common/Throttle.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Throttle.Tpo common/$(DEPDIR)/test_build_libcommon-Throttle.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Throttle.cc' object='common/test_build_libcommon-Throttle.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Throttle.o `test -f 'common/Throttle.cc' || echo '$(srcdir)/'`common/Throttle.cc + +common/test_build_libcommon-Throttle.obj: common/Throttle.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Throttle.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Throttle.Tpo -c -o common/test_build_libcommon-Throttle.obj `if test -f 'common/Throttle.cc'; then $(CYGPATH_W) 'common/Throttle.cc'; else $(CYGPATH_W) '$(srcdir)/common/Throttle.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Throttle.Tpo common/$(DEPDIR)/test_build_libcommon-Throttle.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Throttle.cc' object='common/test_build_libcommon-Throttle.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Throttle.obj `if test -f 'common/Throttle.cc'; then $(CYGPATH_W) 'common/Throttle.cc'; else $(CYGPATH_W) '$(srcdir)/common/Throttle.cc'; fi` + +common/test_build_libcommon-Timer.o: common/Timer.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Timer.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Timer.Tpo -c -o common/test_build_libcommon-Timer.o `test -f 'common/Timer.cc' || echo '$(srcdir)/'`common/Timer.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Timer.Tpo common/$(DEPDIR)/test_build_libcommon-Timer.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Timer.cc' object='common/test_build_libcommon-Timer.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Timer.o `test -f 'common/Timer.cc' || echo '$(srcdir)/'`common/Timer.cc + +common/test_build_libcommon-Timer.obj: common/Timer.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Timer.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Timer.Tpo -c -o common/test_build_libcommon-Timer.obj `if test -f 'common/Timer.cc'; then $(CYGPATH_W) 'common/Timer.cc'; else $(CYGPATH_W) '$(srcdir)/common/Timer.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Timer.Tpo common/$(DEPDIR)/test_build_libcommon-Timer.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Timer.cc' object='common/test_build_libcommon-Timer.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Timer.obj `if test -f 'common/Timer.cc'; then $(CYGPATH_W) 'common/Timer.cc'; else $(CYGPATH_W) '$(srcdir)/common/Timer.cc'; fi` + +common/test_build_libcommon-Finisher.o: common/Finisher.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Finisher.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Finisher.Tpo -c -o common/test_build_libcommon-Finisher.o `test -f 'common/Finisher.cc' || echo '$(srcdir)/'`common/Finisher.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Finisher.Tpo common/$(DEPDIR)/test_build_libcommon-Finisher.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Finisher.cc' object='common/test_build_libcommon-Finisher.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Finisher.o `test -f 'common/Finisher.cc' || echo '$(srcdir)/'`common/Finisher.cc + +common/test_build_libcommon-Finisher.obj: common/Finisher.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Finisher.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Finisher.Tpo -c -o common/test_build_libcommon-Finisher.obj `if test -f 'common/Finisher.cc'; then $(CYGPATH_W) 'common/Finisher.cc'; else $(CYGPATH_W) '$(srcdir)/common/Finisher.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Finisher.Tpo common/$(DEPDIR)/test_build_libcommon-Finisher.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Finisher.cc' object='common/test_build_libcommon-Finisher.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Finisher.obj `if test -f 'common/Finisher.cc'; then $(CYGPATH_W) 'common/Finisher.cc'; else $(CYGPATH_W) '$(srcdir)/common/Finisher.cc'; fi` + +common/test_build_libcommon-environment.o: common/environment.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-environment.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-environment.Tpo -c -o common/test_build_libcommon-environment.o `test -f 'common/environment.cc' || echo '$(srcdir)/'`common/environment.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-environment.Tpo common/$(DEPDIR)/test_build_libcommon-environment.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/environment.cc' object='common/test_build_libcommon-environment.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-environment.o `test -f 'common/environment.cc' || echo '$(srcdir)/'`common/environment.cc + +common/test_build_libcommon-environment.obj: common/environment.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-environment.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-environment.Tpo -c -o common/test_build_libcommon-environment.obj `if test -f 'common/environment.cc'; then $(CYGPATH_W) 'common/environment.cc'; else $(CYGPATH_W) '$(srcdir)/common/environment.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-environment.Tpo common/$(DEPDIR)/test_build_libcommon-environment.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/environment.cc' object='common/test_build_libcommon-environment.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-environment.obj `if test -f 'common/environment.cc'; then $(CYGPATH_W) 'common/environment.cc'; else $(CYGPATH_W) '$(srcdir)/common/environment.cc'; fi` + +common/test_build_libcommon-assert.o: common/assert.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-assert.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-assert.Tpo -c -o common/test_build_libcommon-assert.o `test -f 'common/assert.cc' || echo '$(srcdir)/'`common/assert.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-assert.Tpo common/$(DEPDIR)/test_build_libcommon-assert.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/assert.cc' object='common/test_build_libcommon-assert.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-assert.o `test -f 'common/assert.cc' || echo '$(srcdir)/'`common/assert.cc + +common/test_build_libcommon-assert.obj: common/assert.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-assert.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-assert.Tpo -c -o common/test_build_libcommon-assert.obj `if test -f 'common/assert.cc'; then $(CYGPATH_W) 'common/assert.cc'; else $(CYGPATH_W) '$(srcdir)/common/assert.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-assert.Tpo common/$(DEPDIR)/test_build_libcommon-assert.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/assert.cc' object='common/test_build_libcommon-assert.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-assert.obj `if test -f 'common/assert.cc'; then $(CYGPATH_W) 'common/assert.cc'; else $(CYGPATH_W) '$(srcdir)/common/assert.cc'; fi` + +common/test_build_libcommon-run_cmd.o: common/run_cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-run_cmd.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-run_cmd.Tpo -c -o common/test_build_libcommon-run_cmd.o `test -f 'common/run_cmd.cc' || echo '$(srcdir)/'`common/run_cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-run_cmd.Tpo common/$(DEPDIR)/test_build_libcommon-run_cmd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/run_cmd.cc' object='common/test_build_libcommon-run_cmd.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-run_cmd.o `test -f 'common/run_cmd.cc' || echo '$(srcdir)/'`common/run_cmd.cc + +common/test_build_libcommon-run_cmd.obj: common/run_cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-run_cmd.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-run_cmd.Tpo -c -o common/test_build_libcommon-run_cmd.obj `if test -f 'common/run_cmd.cc'; then $(CYGPATH_W) 'common/run_cmd.cc'; else $(CYGPATH_W) '$(srcdir)/common/run_cmd.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-run_cmd.Tpo common/$(DEPDIR)/test_build_libcommon-run_cmd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/run_cmd.cc' object='common/test_build_libcommon-run_cmd.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-run_cmd.obj `if test -f 'common/run_cmd.cc'; then $(CYGPATH_W) 'common/run_cmd.cc'; else $(CYGPATH_W) '$(srcdir)/common/run_cmd.cc'; fi` + +common/test_build_libcommon-WorkQueue.o: common/WorkQueue.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-WorkQueue.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-WorkQueue.Tpo -c -o common/test_build_libcommon-WorkQueue.o `test -f 'common/WorkQueue.cc' || echo '$(srcdir)/'`common/WorkQueue.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-WorkQueue.Tpo common/$(DEPDIR)/test_build_libcommon-WorkQueue.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/WorkQueue.cc' object='common/test_build_libcommon-WorkQueue.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-WorkQueue.o `test -f 'common/WorkQueue.cc' || echo '$(srcdir)/'`common/WorkQueue.cc + +common/test_build_libcommon-WorkQueue.obj: common/WorkQueue.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-WorkQueue.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-WorkQueue.Tpo -c -o common/test_build_libcommon-WorkQueue.obj `if test -f 'common/WorkQueue.cc'; then $(CYGPATH_W) 'common/WorkQueue.cc'; else $(CYGPATH_W) '$(srcdir)/common/WorkQueue.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-WorkQueue.Tpo common/$(DEPDIR)/test_build_libcommon-WorkQueue.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/WorkQueue.cc' object='common/test_build_libcommon-WorkQueue.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-WorkQueue.obj `if test -f 'common/WorkQueue.cc'; then $(CYGPATH_W) 'common/WorkQueue.cc'; else $(CYGPATH_W) '$(srcdir)/common/WorkQueue.cc'; fi` + +common/test_build_libcommon-ConfUtils.o: common/ConfUtils.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ConfUtils.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ConfUtils.Tpo -c -o common/test_build_libcommon-ConfUtils.o `test -f 'common/ConfUtils.cc' || echo '$(srcdir)/'`common/ConfUtils.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ConfUtils.Tpo common/$(DEPDIR)/test_build_libcommon-ConfUtils.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ConfUtils.cc' object='common/test_build_libcommon-ConfUtils.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ConfUtils.o `test -f 'common/ConfUtils.cc' || echo '$(srcdir)/'`common/ConfUtils.cc + +common/test_build_libcommon-ConfUtils.obj: common/ConfUtils.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ConfUtils.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ConfUtils.Tpo -c -o common/test_build_libcommon-ConfUtils.obj `if test -f 'common/ConfUtils.cc'; then $(CYGPATH_W) 'common/ConfUtils.cc'; else $(CYGPATH_W) '$(srcdir)/common/ConfUtils.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ConfUtils.Tpo common/$(DEPDIR)/test_build_libcommon-ConfUtils.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ConfUtils.cc' object='common/test_build_libcommon-ConfUtils.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ConfUtils.obj `if test -f 'common/ConfUtils.cc'; then $(CYGPATH_W) 'common/ConfUtils.cc'; else $(CYGPATH_W) '$(srcdir)/common/ConfUtils.cc'; fi` + +common/test_build_libcommon-MemoryModel.o: common/MemoryModel.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-MemoryModel.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-MemoryModel.Tpo -c -o common/test_build_libcommon-MemoryModel.o `test -f 'common/MemoryModel.cc' || echo '$(srcdir)/'`common/MemoryModel.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-MemoryModel.Tpo common/$(DEPDIR)/test_build_libcommon-MemoryModel.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/MemoryModel.cc' object='common/test_build_libcommon-MemoryModel.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-MemoryModel.o `test -f 'common/MemoryModel.cc' || echo '$(srcdir)/'`common/MemoryModel.cc + +common/test_build_libcommon-MemoryModel.obj: common/MemoryModel.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-MemoryModel.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-MemoryModel.Tpo -c -o common/test_build_libcommon-MemoryModel.obj `if test -f 'common/MemoryModel.cc'; then $(CYGPATH_W) 'common/MemoryModel.cc'; else $(CYGPATH_W) '$(srcdir)/common/MemoryModel.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-MemoryModel.Tpo common/$(DEPDIR)/test_build_libcommon-MemoryModel.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/MemoryModel.cc' object='common/test_build_libcommon-MemoryModel.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-MemoryModel.obj `if test -f 'common/MemoryModel.cc'; then $(CYGPATH_W) 'common/MemoryModel.cc'; else $(CYGPATH_W) '$(srcdir)/common/MemoryModel.cc'; fi` + +common/test_build_libcommon-fd.o: common/fd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-fd.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-fd.Tpo -c -o common/test_build_libcommon-fd.o `test -f 'common/fd.cc' || echo '$(srcdir)/'`common/fd.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-fd.Tpo common/$(DEPDIR)/test_build_libcommon-fd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/fd.cc' object='common/test_build_libcommon-fd.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-fd.o `test -f 'common/fd.cc' || echo '$(srcdir)/'`common/fd.cc + +common/test_build_libcommon-fd.obj: common/fd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-fd.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-fd.Tpo -c -o common/test_build_libcommon-fd.obj `if test -f 'common/fd.cc'; then $(CYGPATH_W) 'common/fd.cc'; else $(CYGPATH_W) '$(srcdir)/common/fd.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-fd.Tpo common/$(DEPDIR)/test_build_libcommon-fd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/fd.cc' object='common/test_build_libcommon-fd.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-fd.obj `if test -f 'common/fd.cc'; then $(CYGPATH_W) 'common/fd.cc'; else $(CYGPATH_W) '$(srcdir)/common/fd.cc'; fi` + +common/test_build_libcommon-snap_types.o: common/snap_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-snap_types.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-snap_types.Tpo -c -o common/test_build_libcommon-snap_types.o `test -f 'common/snap_types.cc' || echo '$(srcdir)/'`common/snap_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-snap_types.Tpo common/$(DEPDIR)/test_build_libcommon-snap_types.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/snap_types.cc' object='common/test_build_libcommon-snap_types.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-snap_types.o `test -f 'common/snap_types.cc' || echo '$(srcdir)/'`common/snap_types.cc + +common/test_build_libcommon-snap_types.obj: common/snap_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-snap_types.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-snap_types.Tpo -c -o common/test_build_libcommon-snap_types.obj `if test -f 'common/snap_types.cc'; then $(CYGPATH_W) 'common/snap_types.cc'; else $(CYGPATH_W) '$(srcdir)/common/snap_types.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-snap_types.Tpo common/$(DEPDIR)/test_build_libcommon-snap_types.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/snap_types.cc' object='common/test_build_libcommon-snap_types.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-snap_types.obj `if test -f 'common/snap_types.cc'; then $(CYGPATH_W) 'common/snap_types.cc'; else $(CYGPATH_W) '$(srcdir)/common/snap_types.cc'; fi` + +common/test_build_libcommon-str_list.o: common/str_list.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-str_list.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-str_list.Tpo -c -o common/test_build_libcommon-str_list.o `test -f 'common/str_list.cc' || echo '$(srcdir)/'`common/str_list.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-str_list.Tpo common/$(DEPDIR)/test_build_libcommon-str_list.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/str_list.cc' object='common/test_build_libcommon-str_list.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-str_list.o `test -f 'common/str_list.cc' || echo '$(srcdir)/'`common/str_list.cc + +common/test_build_libcommon-str_list.obj: common/str_list.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-str_list.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-str_list.Tpo -c -o common/test_build_libcommon-str_list.obj `if test -f 'common/str_list.cc'; then $(CYGPATH_W) 'common/str_list.cc'; else $(CYGPATH_W) '$(srcdir)/common/str_list.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-str_list.Tpo common/$(DEPDIR)/test_build_libcommon-str_list.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/str_list.cc' object='common/test_build_libcommon-str_list.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-str_list.obj `if test -f 'common/str_list.cc'; then $(CYGPATH_W) 'common/str_list.cc'; else $(CYGPATH_W) '$(srcdir)/common/str_list.cc'; fi` + +common/test_build_libcommon-str_map.o: common/str_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-str_map.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-str_map.Tpo -c -o common/test_build_libcommon-str_map.o `test -f 'common/str_map.cc' || echo '$(srcdir)/'`common/str_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-str_map.Tpo common/$(DEPDIR)/test_build_libcommon-str_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/str_map.cc' object='common/test_build_libcommon-str_map.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-str_map.o `test -f 'common/str_map.cc' || echo '$(srcdir)/'`common/str_map.cc + +common/test_build_libcommon-str_map.obj: common/str_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-str_map.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-str_map.Tpo -c -o common/test_build_libcommon-str_map.obj `if test -f 'common/str_map.cc'; then $(CYGPATH_W) 'common/str_map.cc'; else $(CYGPATH_W) '$(srcdir)/common/str_map.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-str_map.Tpo common/$(DEPDIR)/test_build_libcommon-str_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/str_map.cc' object='common/test_build_libcommon-str_map.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-str_map.obj `if test -f 'common/str_map.cc'; then $(CYGPATH_W) 'common/str_map.cc'; else $(CYGPATH_W) '$(srcdir)/common/str_map.cc'; fi` + +common/test_build_libcommon-errno.o: common/errno.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-errno.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-errno.Tpo -c -o common/test_build_libcommon-errno.o `test -f 'common/errno.cc' || echo '$(srcdir)/'`common/errno.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-errno.Tpo common/$(DEPDIR)/test_build_libcommon-errno.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/errno.cc' object='common/test_build_libcommon-errno.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-errno.o `test -f 'common/errno.cc' || echo '$(srcdir)/'`common/errno.cc + +common/test_build_libcommon-errno.obj: common/errno.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-errno.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-errno.Tpo -c -o common/test_build_libcommon-errno.obj `if test -f 'common/errno.cc'; then $(CYGPATH_W) 'common/errno.cc'; else $(CYGPATH_W) '$(srcdir)/common/errno.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-errno.Tpo common/$(DEPDIR)/test_build_libcommon-errno.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/errno.cc' object='common/test_build_libcommon-errno.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-errno.obj `if test -f 'common/errno.cc'; then $(CYGPATH_W) 'common/errno.cc'; else $(CYGPATH_W) '$(srcdir)/common/errno.cc'; fi` + +common/test_build_libcommon-RefCountedObj.o: common/RefCountedObj.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-RefCountedObj.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-RefCountedObj.Tpo -c -o common/test_build_libcommon-RefCountedObj.o `test -f 'common/RefCountedObj.cc' || echo '$(srcdir)/'`common/RefCountedObj.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-RefCountedObj.Tpo common/$(DEPDIR)/test_build_libcommon-RefCountedObj.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/RefCountedObj.cc' object='common/test_build_libcommon-RefCountedObj.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-RefCountedObj.o `test -f 'common/RefCountedObj.cc' || echo '$(srcdir)/'`common/RefCountedObj.cc + +common/test_build_libcommon-RefCountedObj.obj: common/RefCountedObj.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-RefCountedObj.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-RefCountedObj.Tpo -c -o common/test_build_libcommon-RefCountedObj.obj `if test -f 'common/RefCountedObj.cc'; then $(CYGPATH_W) 'common/RefCountedObj.cc'; else $(CYGPATH_W) '$(srcdir)/common/RefCountedObj.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-RefCountedObj.Tpo common/$(DEPDIR)/test_build_libcommon-RefCountedObj.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/RefCountedObj.cc' object='common/test_build_libcommon-RefCountedObj.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-RefCountedObj.obj `if test -f 'common/RefCountedObj.cc'; then $(CYGPATH_W) 'common/RefCountedObj.cc'; else $(CYGPATH_W) '$(srcdir)/common/RefCountedObj.cc'; fi` + +common/test_build_libcommon-blkdev.o: common/blkdev.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-blkdev.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-blkdev.Tpo -c -o common/test_build_libcommon-blkdev.o `test -f 'common/blkdev.cc' || echo '$(srcdir)/'`common/blkdev.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-blkdev.Tpo common/$(DEPDIR)/test_build_libcommon-blkdev.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/blkdev.cc' object='common/test_build_libcommon-blkdev.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-blkdev.o `test -f 'common/blkdev.cc' || echo '$(srcdir)/'`common/blkdev.cc + +common/test_build_libcommon-blkdev.obj: common/blkdev.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-blkdev.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-blkdev.Tpo -c -o common/test_build_libcommon-blkdev.obj `if test -f 'common/blkdev.cc'; then $(CYGPATH_W) 'common/blkdev.cc'; else $(CYGPATH_W) '$(srcdir)/common/blkdev.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-blkdev.Tpo common/$(DEPDIR)/test_build_libcommon-blkdev.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/blkdev.cc' object='common/test_build_libcommon-blkdev.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-blkdev.obj `if test -f 'common/blkdev.cc'; then $(CYGPATH_W) 'common/blkdev.cc'; else $(CYGPATH_W) '$(srcdir)/common/blkdev.cc'; fi` + +common/test_build_libcommon-common_init.o: common/common_init.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-common_init.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-common_init.Tpo -c -o common/test_build_libcommon-common_init.o `test -f 'common/common_init.cc' || echo '$(srcdir)/'`common/common_init.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-common_init.Tpo common/$(DEPDIR)/test_build_libcommon-common_init.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/common_init.cc' object='common/test_build_libcommon-common_init.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-common_init.o `test -f 'common/common_init.cc' || echo '$(srcdir)/'`common/common_init.cc + +common/test_build_libcommon-common_init.obj: common/common_init.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-common_init.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-common_init.Tpo -c -o common/test_build_libcommon-common_init.obj `if test -f 'common/common_init.cc'; then $(CYGPATH_W) 'common/common_init.cc'; else $(CYGPATH_W) '$(srcdir)/common/common_init.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-common_init.Tpo common/$(DEPDIR)/test_build_libcommon-common_init.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/common_init.cc' object='common/test_build_libcommon-common_init.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-common_init.obj `if test -f 'common/common_init.cc'; then $(CYGPATH_W) 'common/common_init.cc'; else $(CYGPATH_W) '$(srcdir)/common/common_init.cc'; fi` + +common/test_build_libcommon-ceph_argparse.o: common/ceph_argparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_argparse.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_argparse.Tpo -c -o common/test_build_libcommon-ceph_argparse.o `test -f 'common/ceph_argparse.cc' || echo '$(srcdir)/'`common/ceph_argparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_argparse.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_argparse.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_argparse.cc' object='common/test_build_libcommon-ceph_argparse.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_argparse.o `test -f 'common/ceph_argparse.cc' || echo '$(srcdir)/'`common/ceph_argparse.cc + +common/test_build_libcommon-ceph_argparse.obj: common/ceph_argparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_argparse.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_argparse.Tpo -c -o common/test_build_libcommon-ceph_argparse.obj `if test -f 'common/ceph_argparse.cc'; then $(CYGPATH_W) 'common/ceph_argparse.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_argparse.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_argparse.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_argparse.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_argparse.cc' object='common/test_build_libcommon-ceph_argparse.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_argparse.obj `if test -f 'common/ceph_argparse.cc'; then $(CYGPATH_W) 'common/ceph_argparse.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_argparse.cc'; fi` + +common/test_build_libcommon-ceph_context.o: common/ceph_context.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_context.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_context.Tpo -c -o common/test_build_libcommon-ceph_context.o `test -f 'common/ceph_context.cc' || echo '$(srcdir)/'`common/ceph_context.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_context.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_context.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_context.cc' object='common/test_build_libcommon-ceph_context.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_context.o `test -f 'common/ceph_context.cc' || echo '$(srcdir)/'`common/ceph_context.cc + +common/test_build_libcommon-ceph_context.obj: common/ceph_context.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_context.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_context.Tpo -c -o common/test_build_libcommon-ceph_context.obj `if test -f 'common/ceph_context.cc'; then $(CYGPATH_W) 'common/ceph_context.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_context.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_context.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_context.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_context.cc' object='common/test_build_libcommon-ceph_context.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_context.obj `if test -f 'common/ceph_context.cc'; then $(CYGPATH_W) 'common/ceph_context.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_context.cc'; fi` + +common/test_build_libcommon-buffer.o: common/buffer.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-buffer.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-buffer.Tpo -c -o common/test_build_libcommon-buffer.o `test -f 'common/buffer.cc' || echo '$(srcdir)/'`common/buffer.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-buffer.Tpo common/$(DEPDIR)/test_build_libcommon-buffer.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/buffer.cc' object='common/test_build_libcommon-buffer.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-buffer.o `test -f 'common/buffer.cc' || echo '$(srcdir)/'`common/buffer.cc + +common/test_build_libcommon-buffer.obj: common/buffer.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-buffer.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-buffer.Tpo -c -o common/test_build_libcommon-buffer.obj `if test -f 'common/buffer.cc'; then $(CYGPATH_W) 'common/buffer.cc'; else $(CYGPATH_W) '$(srcdir)/common/buffer.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-buffer.Tpo common/$(DEPDIR)/test_build_libcommon-buffer.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/buffer.cc' object='common/test_build_libcommon-buffer.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-buffer.obj `if test -f 'common/buffer.cc'; then $(CYGPATH_W) 'common/buffer.cc'; else $(CYGPATH_W) '$(srcdir)/common/buffer.cc'; fi` + +common/test_build_libcommon-code_environment.o: common/code_environment.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-code_environment.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-code_environment.Tpo -c -o common/test_build_libcommon-code_environment.o `test -f 'common/code_environment.cc' || echo '$(srcdir)/'`common/code_environment.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-code_environment.Tpo common/$(DEPDIR)/test_build_libcommon-code_environment.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/code_environment.cc' object='common/test_build_libcommon-code_environment.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-code_environment.o `test -f 'common/code_environment.cc' || echo '$(srcdir)/'`common/code_environment.cc + +common/test_build_libcommon-code_environment.obj: common/code_environment.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-code_environment.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-code_environment.Tpo -c -o common/test_build_libcommon-code_environment.obj `if test -f 'common/code_environment.cc'; then $(CYGPATH_W) 'common/code_environment.cc'; else $(CYGPATH_W) '$(srcdir)/common/code_environment.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-code_environment.Tpo common/$(DEPDIR)/test_build_libcommon-code_environment.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/code_environment.cc' object='common/test_build_libcommon-code_environment.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-code_environment.obj `if test -f 'common/code_environment.cc'; then $(CYGPATH_W) 'common/code_environment.cc'; else $(CYGPATH_W) '$(srcdir)/common/code_environment.cc'; fi` + +common/test_build_libcommon-dout.o: common/dout.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-dout.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-dout.Tpo -c -o common/test_build_libcommon-dout.o `test -f 'common/dout.cc' || echo '$(srcdir)/'`common/dout.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-dout.Tpo common/$(DEPDIR)/test_build_libcommon-dout.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/dout.cc' object='common/test_build_libcommon-dout.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-dout.o `test -f 'common/dout.cc' || echo '$(srcdir)/'`common/dout.cc + +common/test_build_libcommon-dout.obj: common/dout.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-dout.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-dout.Tpo -c -o common/test_build_libcommon-dout.obj `if test -f 'common/dout.cc'; then $(CYGPATH_W) 'common/dout.cc'; else $(CYGPATH_W) '$(srcdir)/common/dout.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-dout.Tpo common/$(DEPDIR)/test_build_libcommon-dout.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/dout.cc' object='common/test_build_libcommon-dout.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-dout.obj `if test -f 'common/dout.cc'; then $(CYGPATH_W) 'common/dout.cc'; else $(CYGPATH_W) '$(srcdir)/common/dout.cc'; fi` + +common/test_build_libcommon-histogram.o: common/histogram.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-histogram.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-histogram.Tpo -c -o common/test_build_libcommon-histogram.o `test -f 'common/histogram.cc' || echo '$(srcdir)/'`common/histogram.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-histogram.Tpo common/$(DEPDIR)/test_build_libcommon-histogram.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/histogram.cc' object='common/test_build_libcommon-histogram.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-histogram.o `test -f 'common/histogram.cc' || echo '$(srcdir)/'`common/histogram.cc + +common/test_build_libcommon-histogram.obj: common/histogram.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-histogram.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-histogram.Tpo -c -o common/test_build_libcommon-histogram.obj `if test -f 'common/histogram.cc'; then $(CYGPATH_W) 'common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/common/histogram.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-histogram.Tpo common/$(DEPDIR)/test_build_libcommon-histogram.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/histogram.cc' object='common/test_build_libcommon-histogram.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-histogram.obj `if test -f 'common/histogram.cc'; then $(CYGPATH_W) 'common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/common/histogram.cc'; fi` + +common/test_build_libcommon-signal.o: common/signal.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-signal.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-signal.Tpo -c -o common/test_build_libcommon-signal.o `test -f 'common/signal.cc' || echo '$(srcdir)/'`common/signal.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-signal.Tpo common/$(DEPDIR)/test_build_libcommon-signal.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/signal.cc' object='common/test_build_libcommon-signal.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-signal.o `test -f 'common/signal.cc' || echo '$(srcdir)/'`common/signal.cc + +common/test_build_libcommon-signal.obj: common/signal.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-signal.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-signal.Tpo -c -o common/test_build_libcommon-signal.obj `if test -f 'common/signal.cc'; then $(CYGPATH_W) 'common/signal.cc'; else $(CYGPATH_W) '$(srcdir)/common/signal.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-signal.Tpo common/$(DEPDIR)/test_build_libcommon-signal.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/signal.cc' object='common/test_build_libcommon-signal.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-signal.obj `if test -f 'common/signal.cc'; then $(CYGPATH_W) 'common/signal.cc'; else $(CYGPATH_W) '$(srcdir)/common/signal.cc'; fi` + +common/test_build_libcommon-simple_spin.o: common/simple_spin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-simple_spin.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-simple_spin.Tpo -c -o common/test_build_libcommon-simple_spin.o `test -f 'common/simple_spin.cc' || echo '$(srcdir)/'`common/simple_spin.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-simple_spin.Tpo common/$(DEPDIR)/test_build_libcommon-simple_spin.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/simple_spin.cc' object='common/test_build_libcommon-simple_spin.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-simple_spin.o `test -f 'common/simple_spin.cc' || echo '$(srcdir)/'`common/simple_spin.cc + +common/test_build_libcommon-simple_spin.obj: common/simple_spin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-simple_spin.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-simple_spin.Tpo -c -o common/test_build_libcommon-simple_spin.obj `if test -f 'common/simple_spin.cc'; then $(CYGPATH_W) 'common/simple_spin.cc'; else $(CYGPATH_W) '$(srcdir)/common/simple_spin.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-simple_spin.Tpo common/$(DEPDIR)/test_build_libcommon-simple_spin.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/simple_spin.cc' object='common/test_build_libcommon-simple_spin.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-simple_spin.obj `if test -f 'common/simple_spin.cc'; then $(CYGPATH_W) 'common/simple_spin.cc'; else $(CYGPATH_W) '$(srcdir)/common/simple_spin.cc'; fi` + +common/test_build_libcommon-Thread.o: common/Thread.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Thread.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Thread.Tpo -c -o common/test_build_libcommon-Thread.o `test -f 'common/Thread.cc' || echo '$(srcdir)/'`common/Thread.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Thread.Tpo common/$(DEPDIR)/test_build_libcommon-Thread.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Thread.cc' object='common/test_build_libcommon-Thread.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Thread.o `test -f 'common/Thread.cc' || echo '$(srcdir)/'`common/Thread.cc + +common/test_build_libcommon-Thread.obj: common/Thread.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Thread.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Thread.Tpo -c -o common/test_build_libcommon-Thread.obj `if test -f 'common/Thread.cc'; then $(CYGPATH_W) 'common/Thread.cc'; else $(CYGPATH_W) '$(srcdir)/common/Thread.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Thread.Tpo common/$(DEPDIR)/test_build_libcommon-Thread.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Thread.cc' object='common/test_build_libcommon-Thread.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Thread.obj `if test -f 'common/Thread.cc'; then $(CYGPATH_W) 'common/Thread.cc'; else $(CYGPATH_W) '$(srcdir)/common/Thread.cc'; fi` + +common/test_build_libcommon-Formatter.o: common/Formatter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Formatter.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Formatter.Tpo -c -o common/test_build_libcommon-Formatter.o `test -f 'common/Formatter.cc' || echo '$(srcdir)/'`common/Formatter.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Formatter.Tpo common/$(DEPDIR)/test_build_libcommon-Formatter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Formatter.cc' object='common/test_build_libcommon-Formatter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Formatter.o `test -f 'common/Formatter.cc' || echo '$(srcdir)/'`common/Formatter.cc + +common/test_build_libcommon-Formatter.obj: common/Formatter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-Formatter.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-Formatter.Tpo -c -o common/test_build_libcommon-Formatter.obj `if test -f 'common/Formatter.cc'; then $(CYGPATH_W) 'common/Formatter.cc'; else $(CYGPATH_W) '$(srcdir)/common/Formatter.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-Formatter.Tpo common/$(DEPDIR)/test_build_libcommon-Formatter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/Formatter.cc' object='common/test_build_libcommon-Formatter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-Formatter.obj `if test -f 'common/Formatter.cc'; then $(CYGPATH_W) 'common/Formatter.cc'; else $(CYGPATH_W) '$(srcdir)/common/Formatter.cc'; fi` + +common/test_build_libcommon-HeartbeatMap.o: common/HeartbeatMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-HeartbeatMap.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-HeartbeatMap.Tpo -c -o common/test_build_libcommon-HeartbeatMap.o `test -f 'common/HeartbeatMap.cc' || echo '$(srcdir)/'`common/HeartbeatMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-HeartbeatMap.Tpo common/$(DEPDIR)/test_build_libcommon-HeartbeatMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/HeartbeatMap.cc' object='common/test_build_libcommon-HeartbeatMap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-HeartbeatMap.o `test -f 'common/HeartbeatMap.cc' || echo '$(srcdir)/'`common/HeartbeatMap.cc + +common/test_build_libcommon-HeartbeatMap.obj: common/HeartbeatMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-HeartbeatMap.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-HeartbeatMap.Tpo -c -o common/test_build_libcommon-HeartbeatMap.obj `if test -f 'common/HeartbeatMap.cc'; then $(CYGPATH_W) 'common/HeartbeatMap.cc'; else $(CYGPATH_W) '$(srcdir)/common/HeartbeatMap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-HeartbeatMap.Tpo common/$(DEPDIR)/test_build_libcommon-HeartbeatMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/HeartbeatMap.cc' object='common/test_build_libcommon-HeartbeatMap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-HeartbeatMap.obj `if test -f 'common/HeartbeatMap.cc'; then $(CYGPATH_W) 'common/HeartbeatMap.cc'; else $(CYGPATH_W) '$(srcdir)/common/HeartbeatMap.cc'; fi` + +common/test_build_libcommon-config.o: common/config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-config.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-config.Tpo -c -o common/test_build_libcommon-config.o `test -f 'common/config.cc' || echo '$(srcdir)/'`common/config.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-config.Tpo common/$(DEPDIR)/test_build_libcommon-config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/config.cc' object='common/test_build_libcommon-config.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-config.o `test -f 'common/config.cc' || echo '$(srcdir)/'`common/config.cc + +common/test_build_libcommon-config.obj: common/config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-config.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-config.Tpo -c -o common/test_build_libcommon-config.obj `if test -f 'common/config.cc'; then $(CYGPATH_W) 'common/config.cc'; else $(CYGPATH_W) '$(srcdir)/common/config.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-config.Tpo common/$(DEPDIR)/test_build_libcommon-config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/config.cc' object='common/test_build_libcommon-config.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-config.obj `if test -f 'common/config.cc'; then $(CYGPATH_W) 'common/config.cc'; else $(CYGPATH_W) '$(srcdir)/common/config.cc'; fi` + +common/test_build_libcommon-strtol.o: common/strtol.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-strtol.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-strtol.Tpo -c -o common/test_build_libcommon-strtol.o `test -f 'common/strtol.cc' || echo '$(srcdir)/'`common/strtol.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-strtol.Tpo common/$(DEPDIR)/test_build_libcommon-strtol.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/strtol.cc' object='common/test_build_libcommon-strtol.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-strtol.o `test -f 'common/strtol.cc' || echo '$(srcdir)/'`common/strtol.cc + +common/test_build_libcommon-strtol.obj: common/strtol.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-strtol.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-strtol.Tpo -c -o common/test_build_libcommon-strtol.obj `if test -f 'common/strtol.cc'; then $(CYGPATH_W) 'common/strtol.cc'; else $(CYGPATH_W) '$(srcdir)/common/strtol.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-strtol.Tpo common/$(DEPDIR)/test_build_libcommon-strtol.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/strtol.cc' object='common/test_build_libcommon-strtol.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-strtol.obj `if test -f 'common/strtol.cc'; then $(CYGPATH_W) 'common/strtol.cc'; else $(CYGPATH_W) '$(srcdir)/common/strtol.cc'; fi` + +common/test_build_libcommon-page.o: common/page.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-page.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-page.Tpo -c -o common/test_build_libcommon-page.o `test -f 'common/page.cc' || echo '$(srcdir)/'`common/page.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-page.Tpo common/$(DEPDIR)/test_build_libcommon-page.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/page.cc' object='common/test_build_libcommon-page.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-page.o `test -f 'common/page.cc' || echo '$(srcdir)/'`common/page.cc + +common/test_build_libcommon-page.obj: common/page.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-page.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-page.Tpo -c -o common/test_build_libcommon-page.obj `if test -f 'common/page.cc'; then $(CYGPATH_W) 'common/page.cc'; else $(CYGPATH_W) '$(srcdir)/common/page.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-page.Tpo common/$(DEPDIR)/test_build_libcommon-page.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/page.cc' object='common/test_build_libcommon-page.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-page.obj `if test -f 'common/page.cc'; then $(CYGPATH_W) 'common/page.cc'; else $(CYGPATH_W) '$(srcdir)/common/page.cc'; fi` + +common/test_build_libcommon-lockdep.o: common/lockdep.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-lockdep.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-lockdep.Tpo -c -o common/test_build_libcommon-lockdep.o `test -f 'common/lockdep.cc' || echo '$(srcdir)/'`common/lockdep.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-lockdep.Tpo common/$(DEPDIR)/test_build_libcommon-lockdep.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/lockdep.cc' object='common/test_build_libcommon-lockdep.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-lockdep.o `test -f 'common/lockdep.cc' || echo '$(srcdir)/'`common/lockdep.cc + +common/test_build_libcommon-lockdep.obj: common/lockdep.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-lockdep.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-lockdep.Tpo -c -o common/test_build_libcommon-lockdep.obj `if test -f 'common/lockdep.cc'; then $(CYGPATH_W) 'common/lockdep.cc'; else $(CYGPATH_W) '$(srcdir)/common/lockdep.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-lockdep.Tpo common/$(DEPDIR)/test_build_libcommon-lockdep.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/lockdep.cc' object='common/test_build_libcommon-lockdep.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-lockdep.obj `if test -f 'common/lockdep.cc'; then $(CYGPATH_W) 'common/lockdep.cc'; else $(CYGPATH_W) '$(srcdir)/common/lockdep.cc'; fi` + +common/test_build_libcommon-version.o: common/version.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-version.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-version.Tpo -c -o common/test_build_libcommon-version.o `test -f 'common/version.cc' || echo '$(srcdir)/'`common/version.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-version.Tpo common/$(DEPDIR)/test_build_libcommon-version.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/version.cc' object='common/test_build_libcommon-version.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-version.o `test -f 'common/version.cc' || echo '$(srcdir)/'`common/version.cc + +common/test_build_libcommon-version.obj: common/version.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-version.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-version.Tpo -c -o common/test_build_libcommon-version.obj `if test -f 'common/version.cc'; then $(CYGPATH_W) 'common/version.cc'; else $(CYGPATH_W) '$(srcdir)/common/version.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-version.Tpo common/$(DEPDIR)/test_build_libcommon-version.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/version.cc' object='common/test_build_libcommon-version.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-version.obj `if test -f 'common/version.cc'; then $(CYGPATH_W) 'common/version.cc'; else $(CYGPATH_W) '$(srcdir)/common/version.cc'; fi` + +common/test_build_libcommon-hex.o: common/hex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-hex.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-hex.Tpo -c -o common/test_build_libcommon-hex.o `test -f 'common/hex.cc' || echo '$(srcdir)/'`common/hex.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-hex.Tpo common/$(DEPDIR)/test_build_libcommon-hex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/hex.cc' object='common/test_build_libcommon-hex.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-hex.o `test -f 'common/hex.cc' || echo '$(srcdir)/'`common/hex.cc + +common/test_build_libcommon-hex.obj: common/hex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-hex.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-hex.Tpo -c -o common/test_build_libcommon-hex.obj `if test -f 'common/hex.cc'; then $(CYGPATH_W) 'common/hex.cc'; else $(CYGPATH_W) '$(srcdir)/common/hex.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-hex.Tpo common/$(DEPDIR)/test_build_libcommon-hex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/hex.cc' object='common/test_build_libcommon-hex.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-hex.obj `if test -f 'common/hex.cc'; then $(CYGPATH_W) 'common/hex.cc'; else $(CYGPATH_W) '$(srcdir)/common/hex.cc'; fi` + +common/test_build_libcommon-entity_name.o: common/entity_name.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-entity_name.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-entity_name.Tpo -c -o common/test_build_libcommon-entity_name.o `test -f 'common/entity_name.cc' || echo '$(srcdir)/'`common/entity_name.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-entity_name.Tpo common/$(DEPDIR)/test_build_libcommon-entity_name.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/entity_name.cc' object='common/test_build_libcommon-entity_name.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-entity_name.o `test -f 'common/entity_name.cc' || echo '$(srcdir)/'`common/entity_name.cc + +common/test_build_libcommon-entity_name.obj: common/entity_name.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-entity_name.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-entity_name.Tpo -c -o common/test_build_libcommon-entity_name.obj `if test -f 'common/entity_name.cc'; then $(CYGPATH_W) 'common/entity_name.cc'; else $(CYGPATH_W) '$(srcdir)/common/entity_name.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-entity_name.Tpo common/$(DEPDIR)/test_build_libcommon-entity_name.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/entity_name.cc' object='common/test_build_libcommon-entity_name.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-entity_name.obj `if test -f 'common/entity_name.cc'; then $(CYGPATH_W) 'common/entity_name.cc'; else $(CYGPATH_W) '$(srcdir)/common/entity_name.cc'; fi` + +common/test_build_libcommon-ceph_crypto.o: common/ceph_crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_crypto.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_crypto.Tpo -c -o common/test_build_libcommon-ceph_crypto.o `test -f 'common/ceph_crypto.cc' || echo '$(srcdir)/'`common/ceph_crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_crypto.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_crypto.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_crypto.cc' object='common/test_build_libcommon-ceph_crypto.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_crypto.o `test -f 'common/ceph_crypto.cc' || echo '$(srcdir)/'`common/ceph_crypto.cc + +common/test_build_libcommon-ceph_crypto.obj: common/ceph_crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_crypto.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_crypto.Tpo -c -o common/test_build_libcommon-ceph_crypto.obj `if test -f 'common/ceph_crypto.cc'; then $(CYGPATH_W) 'common/ceph_crypto.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_crypto.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_crypto.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_crypto.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_crypto.cc' object='common/test_build_libcommon-ceph_crypto.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_crypto.obj `if test -f 'common/ceph_crypto.cc'; then $(CYGPATH_W) 'common/ceph_crypto.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_crypto.cc'; fi` + +common/test_build_libcommon-ceph_crypto_cms.o: common/ceph_crypto_cms.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_crypto_cms.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_crypto_cms.Tpo -c -o common/test_build_libcommon-ceph_crypto_cms.o `test -f 'common/ceph_crypto_cms.cc' || echo '$(srcdir)/'`common/ceph_crypto_cms.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_crypto_cms.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_crypto_cms.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_crypto_cms.cc' object='common/test_build_libcommon-ceph_crypto_cms.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_crypto_cms.o `test -f 'common/ceph_crypto_cms.cc' || echo '$(srcdir)/'`common/ceph_crypto_cms.cc + +common/test_build_libcommon-ceph_crypto_cms.obj: common/ceph_crypto_cms.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_crypto_cms.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_crypto_cms.Tpo -c -o common/test_build_libcommon-ceph_crypto_cms.obj `if test -f 'common/ceph_crypto_cms.cc'; then $(CYGPATH_W) 'common/ceph_crypto_cms.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_crypto_cms.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_crypto_cms.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_crypto_cms.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_crypto_cms.cc' object='common/test_build_libcommon-ceph_crypto_cms.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_crypto_cms.obj `if test -f 'common/ceph_crypto_cms.cc'; then $(CYGPATH_W) 'common/ceph_crypto_cms.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_crypto_cms.cc'; fi` + +common/test_build_libcommon-ceph_json.o: common/ceph_json.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_json.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_json.Tpo -c -o common/test_build_libcommon-ceph_json.o `test -f 'common/ceph_json.cc' || echo '$(srcdir)/'`common/ceph_json.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_json.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_json.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_json.cc' object='common/test_build_libcommon-ceph_json.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_json.o `test -f 'common/ceph_json.cc' || echo '$(srcdir)/'`common/ceph_json.cc + +common/test_build_libcommon-ceph_json.obj: common/ceph_json.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_json.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_json.Tpo -c -o common/test_build_libcommon-ceph_json.obj `if test -f 'common/ceph_json.cc'; then $(CYGPATH_W) 'common/ceph_json.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_json.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_json.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_json.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_json.cc' object='common/test_build_libcommon-ceph_json.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_json.obj `if test -f 'common/ceph_json.cc'; then $(CYGPATH_W) 'common/ceph_json.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_json.cc'; fi` + +common/test_build_libcommon-ipaddr.o: common/ipaddr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ipaddr.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ipaddr.Tpo -c -o common/test_build_libcommon-ipaddr.o `test -f 'common/ipaddr.cc' || echo '$(srcdir)/'`common/ipaddr.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ipaddr.Tpo common/$(DEPDIR)/test_build_libcommon-ipaddr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ipaddr.cc' object='common/test_build_libcommon-ipaddr.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ipaddr.o `test -f 'common/ipaddr.cc' || echo '$(srcdir)/'`common/ipaddr.cc + +common/test_build_libcommon-ipaddr.obj: common/ipaddr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ipaddr.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ipaddr.Tpo -c -o common/test_build_libcommon-ipaddr.obj `if test -f 'common/ipaddr.cc'; then $(CYGPATH_W) 'common/ipaddr.cc'; else $(CYGPATH_W) '$(srcdir)/common/ipaddr.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ipaddr.Tpo common/$(DEPDIR)/test_build_libcommon-ipaddr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ipaddr.cc' object='common/test_build_libcommon-ipaddr.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ipaddr.obj `if test -f 'common/ipaddr.cc'; then $(CYGPATH_W) 'common/ipaddr.cc'; else $(CYGPATH_W) '$(srcdir)/common/ipaddr.cc'; fi` + +common/test_build_libcommon-pick_address.o: common/pick_address.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-pick_address.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-pick_address.Tpo -c -o common/test_build_libcommon-pick_address.o `test -f 'common/pick_address.cc' || echo '$(srcdir)/'`common/pick_address.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-pick_address.Tpo common/$(DEPDIR)/test_build_libcommon-pick_address.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/pick_address.cc' object='common/test_build_libcommon-pick_address.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-pick_address.o `test -f 'common/pick_address.cc' || echo '$(srcdir)/'`common/pick_address.cc + +common/test_build_libcommon-pick_address.obj: common/pick_address.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-pick_address.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-pick_address.Tpo -c -o common/test_build_libcommon-pick_address.obj `if test -f 'common/pick_address.cc'; then $(CYGPATH_W) 'common/pick_address.cc'; else $(CYGPATH_W) '$(srcdir)/common/pick_address.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-pick_address.Tpo common/$(DEPDIR)/test_build_libcommon-pick_address.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/pick_address.cc' object='common/test_build_libcommon-pick_address.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-pick_address.obj `if test -f 'common/pick_address.cc'; then $(CYGPATH_W) 'common/pick_address.cc'; else $(CYGPATH_W) '$(srcdir)/common/pick_address.cc'; fi` + +common/test_build_libcommon-util.o: common/util.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-util.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-util.Tpo -c -o common/test_build_libcommon-util.o `test -f 'common/util.cc' || echo '$(srcdir)/'`common/util.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-util.Tpo common/$(DEPDIR)/test_build_libcommon-util.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/util.cc' object='common/test_build_libcommon-util.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-util.o `test -f 'common/util.cc' || echo '$(srcdir)/'`common/util.cc + +common/test_build_libcommon-util.obj: common/util.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-util.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-util.Tpo -c -o common/test_build_libcommon-util.obj `if test -f 'common/util.cc'; then $(CYGPATH_W) 'common/util.cc'; else $(CYGPATH_W) '$(srcdir)/common/util.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-util.Tpo common/$(DEPDIR)/test_build_libcommon-util.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/util.cc' object='common/test_build_libcommon-util.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-util.obj `if test -f 'common/util.cc'; then $(CYGPATH_W) 'common/util.cc'; else $(CYGPATH_W) '$(srcdir)/common/util.cc'; fi` + +common/test_build_libcommon-TextTable.o: common/TextTable.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-TextTable.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-TextTable.Tpo -c -o common/test_build_libcommon-TextTable.o `test -f 'common/TextTable.cc' || echo '$(srcdir)/'`common/TextTable.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-TextTable.Tpo common/$(DEPDIR)/test_build_libcommon-TextTable.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/TextTable.cc' object='common/test_build_libcommon-TextTable.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-TextTable.o `test -f 'common/TextTable.cc' || echo '$(srcdir)/'`common/TextTable.cc + +common/test_build_libcommon-TextTable.obj: common/TextTable.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-TextTable.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-TextTable.Tpo -c -o common/test_build_libcommon-TextTable.obj `if test -f 'common/TextTable.cc'; then $(CYGPATH_W) 'common/TextTable.cc'; else $(CYGPATH_W) '$(srcdir)/common/TextTable.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-TextTable.Tpo common/$(DEPDIR)/test_build_libcommon-TextTable.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/TextTable.cc' object='common/test_build_libcommon-TextTable.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-TextTable.obj `if test -f 'common/TextTable.cc'; then $(CYGPATH_W) 'common/TextTable.cc'; else $(CYGPATH_W) '$(srcdir)/common/TextTable.cc'; fi` + +common/test_build_libcommon-ceph_fs.o: common/ceph_fs.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_fs.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_fs.Tpo -c -o common/test_build_libcommon-ceph_fs.o `test -f 'common/ceph_fs.cc' || echo '$(srcdir)/'`common/ceph_fs.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_fs.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_fs.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_fs.cc' object='common/test_build_libcommon-ceph_fs.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_fs.o `test -f 'common/ceph_fs.cc' || echo '$(srcdir)/'`common/ceph_fs.cc + +common/test_build_libcommon-ceph_fs.obj: common/ceph_fs.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_fs.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_fs.Tpo -c -o common/test_build_libcommon-ceph_fs.obj `if test -f 'common/ceph_fs.cc'; then $(CYGPATH_W) 'common/ceph_fs.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_fs.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_fs.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_fs.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_fs.cc' object='common/test_build_libcommon-ceph_fs.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_fs.obj `if test -f 'common/ceph_fs.cc'; then $(CYGPATH_W) 'common/ceph_fs.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_fs.cc'; fi` + +common/test_build_libcommon-ceph_hash.o: common/ceph_hash.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_hash.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_hash.Tpo -c -o common/test_build_libcommon-ceph_hash.o `test -f 'common/ceph_hash.cc' || echo '$(srcdir)/'`common/ceph_hash.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_hash.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_hash.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_hash.cc' object='common/test_build_libcommon-ceph_hash.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_hash.o `test -f 'common/ceph_hash.cc' || echo '$(srcdir)/'`common/ceph_hash.cc + +common/test_build_libcommon-ceph_hash.obj: common/ceph_hash.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_hash.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_hash.Tpo -c -o common/test_build_libcommon-ceph_hash.obj `if test -f 'common/ceph_hash.cc'; then $(CYGPATH_W) 'common/ceph_hash.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_hash.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_hash.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_hash.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_hash.cc' object='common/test_build_libcommon-ceph_hash.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_hash.obj `if test -f 'common/ceph_hash.cc'; then $(CYGPATH_W) 'common/ceph_hash.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_hash.cc'; fi` + +common/test_build_libcommon-ceph_strings.o: common/ceph_strings.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_strings.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_strings.Tpo -c -o common/test_build_libcommon-ceph_strings.o `test -f 'common/ceph_strings.cc' || echo '$(srcdir)/'`common/ceph_strings.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_strings.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_strings.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_strings.cc' object='common/test_build_libcommon-ceph_strings.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_strings.o `test -f 'common/ceph_strings.cc' || echo '$(srcdir)/'`common/ceph_strings.cc + +common/test_build_libcommon-ceph_strings.obj: common/ceph_strings.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_strings.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_strings.Tpo -c -o common/test_build_libcommon-ceph_strings.obj `if test -f 'common/ceph_strings.cc'; then $(CYGPATH_W) 'common/ceph_strings.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_strings.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_strings.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_strings.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_strings.cc' object='common/test_build_libcommon-ceph_strings.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_strings.obj `if test -f 'common/ceph_strings.cc'; then $(CYGPATH_W) 'common/ceph_strings.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_strings.cc'; fi` + +common/test_build_libcommon-ceph_frag.o: common/ceph_frag.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_frag.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_frag.Tpo -c -o common/test_build_libcommon-ceph_frag.o `test -f 'common/ceph_frag.cc' || echo '$(srcdir)/'`common/ceph_frag.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_frag.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_frag.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_frag.cc' object='common/test_build_libcommon-ceph_frag.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_frag.o `test -f 'common/ceph_frag.cc' || echo '$(srcdir)/'`common/ceph_frag.cc + +common/test_build_libcommon-ceph_frag.obj: common/ceph_frag.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-ceph_frag.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-ceph_frag.Tpo -c -o common/test_build_libcommon-ceph_frag.obj `if test -f 'common/ceph_frag.cc'; then $(CYGPATH_W) 'common/ceph_frag.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_frag.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-ceph_frag.Tpo common/$(DEPDIR)/test_build_libcommon-ceph_frag.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/ceph_frag.cc' object='common/test_build_libcommon-ceph_frag.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-ceph_frag.obj `if test -f 'common/ceph_frag.cc'; then $(CYGPATH_W) 'common/ceph_frag.cc'; else $(CYGPATH_W) '$(srcdir)/common/ceph_frag.cc'; fi` + +common/test_build_libcommon-hobject.o: common/hobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-hobject.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-hobject.Tpo -c -o common/test_build_libcommon-hobject.o `test -f 'common/hobject.cc' || echo '$(srcdir)/'`common/hobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-hobject.Tpo common/$(DEPDIR)/test_build_libcommon-hobject.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/hobject.cc' object='common/test_build_libcommon-hobject.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-hobject.o `test -f 'common/hobject.cc' || echo '$(srcdir)/'`common/hobject.cc + +common/test_build_libcommon-hobject.obj: common/hobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-hobject.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-hobject.Tpo -c -o common/test_build_libcommon-hobject.obj `if test -f 'common/hobject.cc'; then $(CYGPATH_W) 'common/hobject.cc'; else $(CYGPATH_W) '$(srcdir)/common/hobject.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-hobject.Tpo common/$(DEPDIR)/test_build_libcommon-hobject.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/hobject.cc' object='common/test_build_libcommon-hobject.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-hobject.obj `if test -f 'common/hobject.cc'; then $(CYGPATH_W) 'common/hobject.cc'; else $(CYGPATH_W) '$(srcdir)/common/hobject.cc'; fi` + +common/test_build_libcommon-bloom_filter.o: common/bloom_filter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-bloom_filter.o -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-bloom_filter.Tpo -c -o common/test_build_libcommon-bloom_filter.o `test -f 'common/bloom_filter.cc' || echo '$(srcdir)/'`common/bloom_filter.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-bloom_filter.Tpo common/$(DEPDIR)/test_build_libcommon-bloom_filter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/bloom_filter.cc' object='common/test_build_libcommon-bloom_filter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-bloom_filter.o `test -f 'common/bloom_filter.cc' || echo '$(srcdir)/'`common/bloom_filter.cc + +common/test_build_libcommon-bloom_filter.obj: common/bloom_filter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT common/test_build_libcommon-bloom_filter.obj -MD -MP -MF common/$(DEPDIR)/test_build_libcommon-bloom_filter.Tpo -c -o common/test_build_libcommon-bloom_filter.obj `if test -f 'common/bloom_filter.cc'; then $(CYGPATH_W) 'common/bloom_filter.cc'; else $(CYGPATH_W) '$(srcdir)/common/bloom_filter.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) common/$(DEPDIR)/test_build_libcommon-bloom_filter.Tpo common/$(DEPDIR)/test_build_libcommon-bloom_filter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='common/bloom_filter.cc' object='common/test_build_libcommon-bloom_filter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o common/test_build_libcommon-bloom_filter.obj `if test -f 'common/bloom_filter.cc'; then $(CYGPATH_W) 'common/bloom_filter.cc'; else $(CYGPATH_W) '$(srcdir)/common/bloom_filter.cc'; fi` + +mon/test_build_libcommon-MonCap.o: mon/MonCap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mon/test_build_libcommon-MonCap.o -MD -MP -MF mon/$(DEPDIR)/test_build_libcommon-MonCap.Tpo -c -o mon/test_build_libcommon-MonCap.o `test -f 'mon/MonCap.cc' || echo '$(srcdir)/'`mon/MonCap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mon/$(DEPDIR)/test_build_libcommon-MonCap.Tpo mon/$(DEPDIR)/test_build_libcommon-MonCap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mon/MonCap.cc' object='mon/test_build_libcommon-MonCap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mon/test_build_libcommon-MonCap.o `test -f 'mon/MonCap.cc' || echo '$(srcdir)/'`mon/MonCap.cc + +mon/test_build_libcommon-MonCap.obj: mon/MonCap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mon/test_build_libcommon-MonCap.obj -MD -MP -MF mon/$(DEPDIR)/test_build_libcommon-MonCap.Tpo -c -o mon/test_build_libcommon-MonCap.obj `if test -f 'mon/MonCap.cc'; then $(CYGPATH_W) 'mon/MonCap.cc'; else $(CYGPATH_W) '$(srcdir)/mon/MonCap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mon/$(DEPDIR)/test_build_libcommon-MonCap.Tpo mon/$(DEPDIR)/test_build_libcommon-MonCap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mon/MonCap.cc' object='mon/test_build_libcommon-MonCap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mon/test_build_libcommon-MonCap.obj `if test -f 'mon/MonCap.cc'; then $(CYGPATH_W) 'mon/MonCap.cc'; else $(CYGPATH_W) '$(srcdir)/mon/MonCap.cc'; fi` + +mon/test_build_libcommon-MonClient.o: mon/MonClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mon/test_build_libcommon-MonClient.o -MD -MP -MF mon/$(DEPDIR)/test_build_libcommon-MonClient.Tpo -c -o mon/test_build_libcommon-MonClient.o `test -f 'mon/MonClient.cc' || echo '$(srcdir)/'`mon/MonClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mon/$(DEPDIR)/test_build_libcommon-MonClient.Tpo mon/$(DEPDIR)/test_build_libcommon-MonClient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mon/MonClient.cc' object='mon/test_build_libcommon-MonClient.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mon/test_build_libcommon-MonClient.o `test -f 'mon/MonClient.cc' || echo '$(srcdir)/'`mon/MonClient.cc + +mon/test_build_libcommon-MonClient.obj: mon/MonClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mon/test_build_libcommon-MonClient.obj -MD -MP -MF mon/$(DEPDIR)/test_build_libcommon-MonClient.Tpo -c -o mon/test_build_libcommon-MonClient.obj `if test -f 'mon/MonClient.cc'; then $(CYGPATH_W) 'mon/MonClient.cc'; else $(CYGPATH_W) '$(srcdir)/mon/MonClient.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mon/$(DEPDIR)/test_build_libcommon-MonClient.Tpo mon/$(DEPDIR)/test_build_libcommon-MonClient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mon/MonClient.cc' object='mon/test_build_libcommon-MonClient.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mon/test_build_libcommon-MonClient.obj `if test -f 'mon/MonClient.cc'; then $(CYGPATH_W) 'mon/MonClient.cc'; else $(CYGPATH_W) '$(srcdir)/mon/MonClient.cc'; fi` + +mon/test_build_libcommon-MonMap.o: mon/MonMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mon/test_build_libcommon-MonMap.o -MD -MP -MF mon/$(DEPDIR)/test_build_libcommon-MonMap.Tpo -c -o mon/test_build_libcommon-MonMap.o `test -f 'mon/MonMap.cc' || echo '$(srcdir)/'`mon/MonMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mon/$(DEPDIR)/test_build_libcommon-MonMap.Tpo mon/$(DEPDIR)/test_build_libcommon-MonMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mon/MonMap.cc' object='mon/test_build_libcommon-MonMap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mon/test_build_libcommon-MonMap.o `test -f 'mon/MonMap.cc' || echo '$(srcdir)/'`mon/MonMap.cc + +mon/test_build_libcommon-MonMap.obj: mon/MonMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mon/test_build_libcommon-MonMap.obj -MD -MP -MF mon/$(DEPDIR)/test_build_libcommon-MonMap.Tpo -c -o mon/test_build_libcommon-MonMap.obj `if test -f 'mon/MonMap.cc'; then $(CYGPATH_W) 'mon/MonMap.cc'; else $(CYGPATH_W) '$(srcdir)/mon/MonMap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mon/$(DEPDIR)/test_build_libcommon-MonMap.Tpo mon/$(DEPDIR)/test_build_libcommon-MonMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mon/MonMap.cc' object='mon/test_build_libcommon-MonMap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mon/test_build_libcommon-MonMap.obj `if test -f 'mon/MonMap.cc'; then $(CYGPATH_W) 'mon/MonMap.cc'; else $(CYGPATH_W) '$(srcdir)/mon/MonMap.cc'; fi` + +osd/test_build_libcommon-OSDMap.o: osd/OSDMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-OSDMap.o -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-OSDMap.Tpo -c -o osd/test_build_libcommon-OSDMap.o `test -f 'osd/OSDMap.cc' || echo '$(srcdir)/'`osd/OSDMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-OSDMap.Tpo osd/$(DEPDIR)/test_build_libcommon-OSDMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/OSDMap.cc' object='osd/test_build_libcommon-OSDMap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-OSDMap.o `test -f 'osd/OSDMap.cc' || echo '$(srcdir)/'`osd/OSDMap.cc + +osd/test_build_libcommon-OSDMap.obj: osd/OSDMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-OSDMap.obj -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-OSDMap.Tpo -c -o osd/test_build_libcommon-OSDMap.obj `if test -f 'osd/OSDMap.cc'; then $(CYGPATH_W) 'osd/OSDMap.cc'; else $(CYGPATH_W) '$(srcdir)/osd/OSDMap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-OSDMap.Tpo osd/$(DEPDIR)/test_build_libcommon-OSDMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/OSDMap.cc' object='osd/test_build_libcommon-OSDMap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-OSDMap.obj `if test -f 'osd/OSDMap.cc'; then $(CYGPATH_W) 'osd/OSDMap.cc'; else $(CYGPATH_W) '$(srcdir)/osd/OSDMap.cc'; fi` + +osd/test_build_libcommon-osd_types.o: osd/osd_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-osd_types.o -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-osd_types.Tpo -c -o osd/test_build_libcommon-osd_types.o `test -f 'osd/osd_types.cc' || echo '$(srcdir)/'`osd/osd_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-osd_types.Tpo osd/$(DEPDIR)/test_build_libcommon-osd_types.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/osd_types.cc' object='osd/test_build_libcommon-osd_types.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-osd_types.o `test -f 'osd/osd_types.cc' || echo '$(srcdir)/'`osd/osd_types.cc + +osd/test_build_libcommon-osd_types.obj: osd/osd_types.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-osd_types.obj -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-osd_types.Tpo -c -o osd/test_build_libcommon-osd_types.obj `if test -f 'osd/osd_types.cc'; then $(CYGPATH_W) 'osd/osd_types.cc'; else $(CYGPATH_W) '$(srcdir)/osd/osd_types.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-osd_types.Tpo osd/$(DEPDIR)/test_build_libcommon-osd_types.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/osd_types.cc' object='osd/test_build_libcommon-osd_types.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-osd_types.obj `if test -f 'osd/osd_types.cc'; then $(CYGPATH_W) 'osd/osd_types.cc'; else $(CYGPATH_W) '$(srcdir)/osd/osd_types.cc'; fi` + +osd/test_build_libcommon-ECMsgTypes.o: osd/ECMsgTypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-ECMsgTypes.o -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo -c -o osd/test_build_libcommon-ECMsgTypes.o `test -f 'osd/ECMsgTypes.cc' || echo '$(srcdir)/'`osd/ECMsgTypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECMsgTypes.cc' object='osd/test_build_libcommon-ECMsgTypes.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-ECMsgTypes.o `test -f 'osd/ECMsgTypes.cc' || echo '$(srcdir)/'`osd/ECMsgTypes.cc + +osd/test_build_libcommon-ECMsgTypes.obj: osd/ECMsgTypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-ECMsgTypes.obj -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo -c -o osd/test_build_libcommon-ECMsgTypes.obj `if test -f 'osd/ECMsgTypes.cc'; then $(CYGPATH_W) 'osd/ECMsgTypes.cc'; else $(CYGPATH_W) '$(srcdir)/osd/ECMsgTypes.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Tpo osd/$(DEPDIR)/test_build_libcommon-ECMsgTypes.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/ECMsgTypes.cc' object='osd/test_build_libcommon-ECMsgTypes.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-ECMsgTypes.obj `if test -f 'osd/ECMsgTypes.cc'; then $(CYGPATH_W) 'osd/ECMsgTypes.cc'; else $(CYGPATH_W) '$(srcdir)/osd/ECMsgTypes.cc'; fi` + +osd/test_build_libcommon-HitSet.o: osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-HitSet.o -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo -c -o osd/test_build_libcommon-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo osd/$(DEPDIR)/test_build_libcommon-HitSet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/test_build_libcommon-HitSet.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-HitSet.o `test -f 'osd/HitSet.cc' || echo '$(srcdir)/'`osd/HitSet.cc + +osd/test_build_libcommon-HitSet.obj: osd/HitSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT osd/test_build_libcommon-HitSet.obj -MD -MP -MF osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo -c -o osd/test_build_libcommon-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) osd/$(DEPDIR)/test_build_libcommon-HitSet.Tpo osd/$(DEPDIR)/test_build_libcommon-HitSet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='osd/HitSet.cc' object='osd/test_build_libcommon-HitSet.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o osd/test_build_libcommon-HitSet.obj `if test -f 'osd/HitSet.cc'; then $(CYGPATH_W) 'osd/HitSet.cc'; else $(CYGPATH_W) '$(srcdir)/osd/HitSet.cc'; fi` + +mds/test_build_libcommon-MDSMap.o: mds/MDSMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mds/test_build_libcommon-MDSMap.o -MD -MP -MF mds/$(DEPDIR)/test_build_libcommon-MDSMap.Tpo -c -o mds/test_build_libcommon-MDSMap.o `test -f 'mds/MDSMap.cc' || echo '$(srcdir)/'`mds/MDSMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mds/$(DEPDIR)/test_build_libcommon-MDSMap.Tpo mds/$(DEPDIR)/test_build_libcommon-MDSMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mds/MDSMap.cc' object='mds/test_build_libcommon-MDSMap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mds/test_build_libcommon-MDSMap.o `test -f 'mds/MDSMap.cc' || echo '$(srcdir)/'`mds/MDSMap.cc + +mds/test_build_libcommon-MDSMap.obj: mds/MDSMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mds/test_build_libcommon-MDSMap.obj -MD -MP -MF mds/$(DEPDIR)/test_build_libcommon-MDSMap.Tpo -c -o mds/test_build_libcommon-MDSMap.obj `if test -f 'mds/MDSMap.cc'; then $(CYGPATH_W) 'mds/MDSMap.cc'; else $(CYGPATH_W) '$(srcdir)/mds/MDSMap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mds/$(DEPDIR)/test_build_libcommon-MDSMap.Tpo mds/$(DEPDIR)/test_build_libcommon-MDSMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mds/MDSMap.cc' object='mds/test_build_libcommon-MDSMap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mds/test_build_libcommon-MDSMap.obj `if test -f 'mds/MDSMap.cc'; then $(CYGPATH_W) 'mds/MDSMap.cc'; else $(CYGPATH_W) '$(srcdir)/mds/MDSMap.cc'; fi` + +mds/test_build_libcommon-inode_backtrace.o: mds/inode_backtrace.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mds/test_build_libcommon-inode_backtrace.o -MD -MP -MF mds/$(DEPDIR)/test_build_libcommon-inode_backtrace.Tpo -c -o mds/test_build_libcommon-inode_backtrace.o `test -f 'mds/inode_backtrace.cc' || echo '$(srcdir)/'`mds/inode_backtrace.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mds/$(DEPDIR)/test_build_libcommon-inode_backtrace.Tpo mds/$(DEPDIR)/test_build_libcommon-inode_backtrace.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mds/inode_backtrace.cc' object='mds/test_build_libcommon-inode_backtrace.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mds/test_build_libcommon-inode_backtrace.o `test -f 'mds/inode_backtrace.cc' || echo '$(srcdir)/'`mds/inode_backtrace.cc + +mds/test_build_libcommon-inode_backtrace.obj: mds/inode_backtrace.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mds/test_build_libcommon-inode_backtrace.obj -MD -MP -MF mds/$(DEPDIR)/test_build_libcommon-inode_backtrace.Tpo -c -o mds/test_build_libcommon-inode_backtrace.obj `if test -f 'mds/inode_backtrace.cc'; then $(CYGPATH_W) 'mds/inode_backtrace.cc'; else $(CYGPATH_W) '$(srcdir)/mds/inode_backtrace.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mds/$(DEPDIR)/test_build_libcommon-inode_backtrace.Tpo mds/$(DEPDIR)/test_build_libcommon-inode_backtrace.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mds/inode_backtrace.cc' object='mds/test_build_libcommon-inode_backtrace.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mds/test_build_libcommon-inode_backtrace.obj `if test -f 'mds/inode_backtrace.cc'; then $(CYGPATH_W) 'mds/inode_backtrace.cc'; else $(CYGPATH_W) '$(srcdir)/mds/inode_backtrace.cc'; fi` + +mds/test_build_libcommon-mdstypes.o: mds/mdstypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mds/test_build_libcommon-mdstypes.o -MD -MP -MF mds/$(DEPDIR)/test_build_libcommon-mdstypes.Tpo -c -o mds/test_build_libcommon-mdstypes.o `test -f 'mds/mdstypes.cc' || echo '$(srcdir)/'`mds/mdstypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mds/$(DEPDIR)/test_build_libcommon-mdstypes.Tpo mds/$(DEPDIR)/test_build_libcommon-mdstypes.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mds/mdstypes.cc' object='mds/test_build_libcommon-mdstypes.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mds/test_build_libcommon-mdstypes.o `test -f 'mds/mdstypes.cc' || echo '$(srcdir)/'`mds/mdstypes.cc + +mds/test_build_libcommon-mdstypes.obj: mds/mdstypes.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -MT mds/test_build_libcommon-mdstypes.obj -MD -MP -MF mds/$(DEPDIR)/test_build_libcommon-mdstypes.Tpo -c -o mds/test_build_libcommon-mdstypes.obj `if test -f 'mds/mdstypes.cc'; then $(CYGPATH_W) 'mds/mdstypes.cc'; else $(CYGPATH_W) '$(srcdir)/mds/mdstypes.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) mds/$(DEPDIR)/test_build_libcommon-mdstypes.Tpo mds/$(DEPDIR)/test_build_libcommon-mdstypes.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mds/mdstypes.cc' object='mds/test_build_libcommon-mdstypes.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_libcommon_CXXFLAGS) $(CXXFLAGS) -c -o mds/test_build_libcommon-mdstypes.obj `if test -f 'mds/mdstypes.cc'; then $(CYGPATH_W) 'mds/mdstypes.cc'; else $(CYGPATH_W) '$(srcdir)/mds/mdstypes.cc'; fi` + +test/test_build_librados-buildtest_skeleton.o: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_librados-buildtest_skeleton.o -MD -MP -MF test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Tpo -c -o test/test_build_librados-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_librados-buildtest_skeleton.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_librados-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc + +test/test_build_librados-buildtest_skeleton.obj: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_librados-buildtest_skeleton.obj -MD -MP -MF test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Tpo -c -o test/test_build_librados-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_librados-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_librados-buildtest_skeleton.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_librados-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` + +librados/test_build_librados-librados.o: librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-librados.o -MD -MP -MF librados/$(DEPDIR)/test_build_librados-librados.Tpo -c -o librados/test_build_librados-librados.o `test -f 'librados/librados.cc' || echo '$(srcdir)/'`librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-librados.Tpo librados/$(DEPDIR)/test_build_librados-librados.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/librados.cc' object='librados/test_build_librados-librados.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-librados.o `test -f 'librados/librados.cc' || echo '$(srcdir)/'`librados/librados.cc + +librados/test_build_librados-librados.obj: librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-librados.obj -MD -MP -MF librados/$(DEPDIR)/test_build_librados-librados.Tpo -c -o librados/test_build_librados-librados.obj `if test -f 'librados/librados.cc'; then $(CYGPATH_W) 'librados/librados.cc'; else $(CYGPATH_W) '$(srcdir)/librados/librados.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-librados.Tpo librados/$(DEPDIR)/test_build_librados-librados.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/librados.cc' object='librados/test_build_librados-librados.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-librados.obj `if test -f 'librados/librados.cc'; then $(CYGPATH_W) 'librados/librados.cc'; else $(CYGPATH_W) '$(srcdir)/librados/librados.cc'; fi` + +librados/test_build_librados-RadosClient.o: librados/RadosClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-RadosClient.o -MD -MP -MF librados/$(DEPDIR)/test_build_librados-RadosClient.Tpo -c -o librados/test_build_librados-RadosClient.o `test -f 'librados/RadosClient.cc' || echo '$(srcdir)/'`librados/RadosClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-RadosClient.Tpo librados/$(DEPDIR)/test_build_librados-RadosClient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/RadosClient.cc' object='librados/test_build_librados-RadosClient.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-RadosClient.o `test -f 'librados/RadosClient.cc' || echo '$(srcdir)/'`librados/RadosClient.cc + +librados/test_build_librados-RadosClient.obj: librados/RadosClient.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-RadosClient.obj -MD -MP -MF librados/$(DEPDIR)/test_build_librados-RadosClient.Tpo -c -o librados/test_build_librados-RadosClient.obj `if test -f 'librados/RadosClient.cc'; then $(CYGPATH_W) 'librados/RadosClient.cc'; else $(CYGPATH_W) '$(srcdir)/librados/RadosClient.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-RadosClient.Tpo librados/$(DEPDIR)/test_build_librados-RadosClient.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/RadosClient.cc' object='librados/test_build_librados-RadosClient.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-RadosClient.obj `if test -f 'librados/RadosClient.cc'; then $(CYGPATH_W) 'librados/RadosClient.cc'; else $(CYGPATH_W) '$(srcdir)/librados/RadosClient.cc'; fi` + +librados/test_build_librados-IoCtxImpl.o: librados/IoCtxImpl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-IoCtxImpl.o -MD -MP -MF librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Tpo -c -o librados/test_build_librados-IoCtxImpl.o `test -f 'librados/IoCtxImpl.cc' || echo '$(srcdir)/'`librados/IoCtxImpl.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Tpo librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/IoCtxImpl.cc' object='librados/test_build_librados-IoCtxImpl.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-IoCtxImpl.o `test -f 'librados/IoCtxImpl.cc' || echo '$(srcdir)/'`librados/IoCtxImpl.cc + +librados/test_build_librados-IoCtxImpl.obj: librados/IoCtxImpl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-IoCtxImpl.obj -MD -MP -MF librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Tpo -c -o librados/test_build_librados-IoCtxImpl.obj `if test -f 'librados/IoCtxImpl.cc'; then $(CYGPATH_W) 'librados/IoCtxImpl.cc'; else $(CYGPATH_W) '$(srcdir)/librados/IoCtxImpl.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Tpo librados/$(DEPDIR)/test_build_librados-IoCtxImpl.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/IoCtxImpl.cc' object='librados/test_build_librados-IoCtxImpl.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-IoCtxImpl.obj `if test -f 'librados/IoCtxImpl.cc'; then $(CYGPATH_W) 'librados/IoCtxImpl.cc'; else $(CYGPATH_W) '$(srcdir)/librados/IoCtxImpl.cc'; fi` + +librados/test_build_librados-snap_set_diff.o: librados/snap_set_diff.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-snap_set_diff.o -MD -MP -MF librados/$(DEPDIR)/test_build_librados-snap_set_diff.Tpo -c -o librados/test_build_librados-snap_set_diff.o `test -f 'librados/snap_set_diff.cc' || echo '$(srcdir)/'`librados/snap_set_diff.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-snap_set_diff.Tpo librados/$(DEPDIR)/test_build_librados-snap_set_diff.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/snap_set_diff.cc' object='librados/test_build_librados-snap_set_diff.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-snap_set_diff.o `test -f 'librados/snap_set_diff.cc' || echo '$(srcdir)/'`librados/snap_set_diff.cc + +librados/test_build_librados-snap_set_diff.obj: librados/snap_set_diff.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -MT librados/test_build_librados-snap_set_diff.obj -MD -MP -MF librados/$(DEPDIR)/test_build_librados-snap_set_diff.Tpo -c -o librados/test_build_librados-snap_set_diff.obj `if test -f 'librados/snap_set_diff.cc'; then $(CYGPATH_W) 'librados/snap_set_diff.cc'; else $(CYGPATH_W) '$(srcdir)/librados/snap_set_diff.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) librados/$(DEPDIR)/test_build_librados-snap_set_diff.Tpo librados/$(DEPDIR)/test_build_librados-snap_set_diff.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='librados/snap_set_diff.cc' object='librados/test_build_librados-snap_set_diff.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librados_CXXFLAGS) $(CXXFLAGS) -c -o librados/test_build_librados-snap_set_diff.obj `if test -f 'librados/snap_set_diff.cc'; then $(CYGPATH_W) 'librados/snap_set_diff.cc'; else $(CYGPATH_W) '$(srcdir)/librados/snap_set_diff.cc'; fi` + +test/test_build_librgw-buildtest_skeleton.o: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_librgw-buildtest_skeleton.o -MD -MP -MF test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Tpo -c -o test/test_build_librgw-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_librgw-buildtest_skeleton.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_librgw-buildtest_skeleton.o `test -f 'test/buildtest_skeleton.cc' || echo '$(srcdir)/'`test/buildtest_skeleton.cc + +test/test_build_librgw-buildtest_skeleton.obj: test/buildtest_skeleton.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT test/test_build_librgw-buildtest_skeleton.obj -MD -MP -MF test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Tpo -c -o test/test_build_librgw-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Tpo test/$(DEPDIR)/test_build_librgw-buildtest_skeleton.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/buildtest_skeleton.cc' object='test/test_build_librgw-buildtest_skeleton.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o test/test_build_librgw-buildtest_skeleton.obj `if test -f 'test/buildtest_skeleton.cc'; then $(CYGPATH_W) 'test/buildtest_skeleton.cc'; else $(CYGPATH_W) '$(srcdir)/test/buildtest_skeleton.cc'; fi` + +rgw/test_build_librgw-librgw.o: rgw/librgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-librgw.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-librgw.Tpo -c -o rgw/test_build_librgw-librgw.o `test -f 'rgw/librgw.cc' || echo '$(srcdir)/'`rgw/librgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-librgw.Tpo rgw/$(DEPDIR)/test_build_librgw-librgw.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/librgw.cc' object='rgw/test_build_librgw-librgw.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-librgw.o `test -f 'rgw/librgw.cc' || echo '$(srcdir)/'`rgw/librgw.cc + +rgw/test_build_librgw-librgw.obj: rgw/librgw.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-librgw.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-librgw.Tpo -c -o rgw/test_build_librgw-librgw.obj `if test -f 'rgw/librgw.cc'; then $(CYGPATH_W) 'rgw/librgw.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/librgw.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-librgw.Tpo rgw/$(DEPDIR)/test_build_librgw-librgw.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/librgw.cc' object='rgw/test_build_librgw-librgw.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-librgw.obj `if test -f 'rgw/librgw.cc'; then $(CYGPATH_W) 'rgw/librgw.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/librgw.cc'; fi` + +rgw/test_build_librgw-rgw_acl.o: rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_acl.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_acl.Tpo -c -o rgw/test_build_librgw-rgw_acl.o `test -f 'rgw/rgw_acl.cc' || echo '$(srcdir)/'`rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_acl.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_acl.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl.cc' object='rgw/test_build_librgw-rgw_acl.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_acl.o `test -f 'rgw/rgw_acl.cc' || echo '$(srcdir)/'`rgw/rgw_acl.cc + +rgw/test_build_librgw-rgw_acl.obj: rgw/rgw_acl.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_acl.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_acl.Tpo -c -o rgw/test_build_librgw-rgw_acl.obj `if test -f 'rgw/rgw_acl.cc'; then $(CYGPATH_W) 'rgw/rgw_acl.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_acl.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_acl.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl.cc' object='rgw/test_build_librgw-rgw_acl.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_acl.obj `if test -f 'rgw/rgw_acl.cc'; then $(CYGPATH_W) 'rgw/rgw_acl.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl.cc'; fi` + +rgw/test_build_librgw-rgw_acl_s3.o: rgw/rgw_acl_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_acl_s3.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_acl_s3.Tpo -c -o rgw/test_build_librgw-rgw_acl_s3.o `test -f 'rgw/rgw_acl_s3.cc' || echo '$(srcdir)/'`rgw/rgw_acl_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_acl_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_acl_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl_s3.cc' object='rgw/test_build_librgw-rgw_acl_s3.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_acl_s3.o `test -f 'rgw/rgw_acl_s3.cc' || echo '$(srcdir)/'`rgw/rgw_acl_s3.cc + +rgw/test_build_librgw-rgw_acl_s3.obj: rgw/rgw_acl_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_acl_s3.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_acl_s3.Tpo -c -o rgw/test_build_librgw-rgw_acl_s3.obj `if test -f 'rgw/rgw_acl_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_acl_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl_s3.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_acl_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_acl_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl_s3.cc' object='rgw/test_build_librgw-rgw_acl_s3.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_acl_s3.obj `if test -f 'rgw/rgw_acl_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_acl_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl_s3.cc'; fi` + +rgw/test_build_librgw-rgw_acl_swift.o: rgw/rgw_acl_swift.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_acl_swift.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_acl_swift.Tpo -c -o rgw/test_build_librgw-rgw_acl_swift.o `test -f 'rgw/rgw_acl_swift.cc' || echo '$(srcdir)/'`rgw/rgw_acl_swift.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_acl_swift.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_acl_swift.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl_swift.cc' object='rgw/test_build_librgw-rgw_acl_swift.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_acl_swift.o `test -f 'rgw/rgw_acl_swift.cc' || echo '$(srcdir)/'`rgw/rgw_acl_swift.cc + +rgw/test_build_librgw-rgw_acl_swift.obj: rgw/rgw_acl_swift.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_acl_swift.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_acl_swift.Tpo -c -o rgw/test_build_librgw-rgw_acl_swift.obj `if test -f 'rgw/rgw_acl_swift.cc'; then $(CYGPATH_W) 'rgw/rgw_acl_swift.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl_swift.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_acl_swift.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_acl_swift.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_acl_swift.cc' object='rgw/test_build_librgw-rgw_acl_swift.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_acl_swift.obj `if test -f 'rgw/rgw_acl_swift.cc'; then $(CYGPATH_W) 'rgw/rgw_acl_swift.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_acl_swift.cc'; fi` + +rgw/test_build_librgw-rgw_client_io.o: rgw/rgw_client_io.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_client_io.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_client_io.Tpo -c -o rgw/test_build_librgw-rgw_client_io.o `test -f 'rgw/rgw_client_io.cc' || echo '$(srcdir)/'`rgw/rgw_client_io.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_client_io.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_client_io.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_client_io.cc' object='rgw/test_build_librgw-rgw_client_io.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_client_io.o `test -f 'rgw/rgw_client_io.cc' || echo '$(srcdir)/'`rgw/rgw_client_io.cc + +rgw/test_build_librgw-rgw_client_io.obj: rgw/rgw_client_io.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_client_io.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_client_io.Tpo -c -o rgw/test_build_librgw-rgw_client_io.obj `if test -f 'rgw/rgw_client_io.cc'; then $(CYGPATH_W) 'rgw/rgw_client_io.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_client_io.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_client_io.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_client_io.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_client_io.cc' object='rgw/test_build_librgw-rgw_client_io.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_client_io.obj `if test -f 'rgw/rgw_client_io.cc'; then $(CYGPATH_W) 'rgw/rgw_client_io.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_client_io.cc'; fi` + +rgw/test_build_librgw-rgw_fcgi.o: rgw/rgw_fcgi.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_fcgi.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_fcgi.Tpo -c -o rgw/test_build_librgw-rgw_fcgi.o `test -f 'rgw/rgw_fcgi.cc' || echo '$(srcdir)/'`rgw/rgw_fcgi.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_fcgi.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_fcgi.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_fcgi.cc' object='rgw/test_build_librgw-rgw_fcgi.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_fcgi.o `test -f 'rgw/rgw_fcgi.cc' || echo '$(srcdir)/'`rgw/rgw_fcgi.cc + +rgw/test_build_librgw-rgw_fcgi.obj: rgw/rgw_fcgi.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_fcgi.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_fcgi.Tpo -c -o rgw/test_build_librgw-rgw_fcgi.obj `if test -f 'rgw/rgw_fcgi.cc'; then $(CYGPATH_W) 'rgw/rgw_fcgi.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_fcgi.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_fcgi.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_fcgi.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_fcgi.cc' object='rgw/test_build_librgw-rgw_fcgi.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_fcgi.obj `if test -f 'rgw/rgw_fcgi.cc'; then $(CYGPATH_W) 'rgw/rgw_fcgi.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_fcgi.cc'; fi` + +rgw/test_build_librgw-rgw_xml.o: rgw/rgw_xml.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_xml.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_xml.Tpo -c -o rgw/test_build_librgw-rgw_xml.o `test -f 'rgw/rgw_xml.cc' || echo '$(srcdir)/'`rgw/rgw_xml.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_xml.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_xml.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_xml.cc' object='rgw/test_build_librgw-rgw_xml.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_xml.o `test -f 'rgw/rgw_xml.cc' || echo '$(srcdir)/'`rgw/rgw_xml.cc + +rgw/test_build_librgw-rgw_xml.obj: rgw/rgw_xml.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_xml.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_xml.Tpo -c -o rgw/test_build_librgw-rgw_xml.obj `if test -f 'rgw/rgw_xml.cc'; then $(CYGPATH_W) 'rgw/rgw_xml.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_xml.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_xml.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_xml.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_xml.cc' object='rgw/test_build_librgw-rgw_xml.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_xml.obj `if test -f 'rgw/rgw_xml.cc'; then $(CYGPATH_W) 'rgw/rgw_xml.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_xml.cc'; fi` + +rgw/test_build_librgw-rgw_usage.o: rgw/rgw_usage.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_usage.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_usage.Tpo -c -o rgw/test_build_librgw-rgw_usage.o `test -f 'rgw/rgw_usage.cc' || echo '$(srcdir)/'`rgw/rgw_usage.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_usage.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_usage.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_usage.cc' object='rgw/test_build_librgw-rgw_usage.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_usage.o `test -f 'rgw/rgw_usage.cc' || echo '$(srcdir)/'`rgw/rgw_usage.cc + +rgw/test_build_librgw-rgw_usage.obj: rgw/rgw_usage.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_usage.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_usage.Tpo -c -o rgw/test_build_librgw-rgw_usage.obj `if test -f 'rgw/rgw_usage.cc'; then $(CYGPATH_W) 'rgw/rgw_usage.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_usage.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_usage.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_usage.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_usage.cc' object='rgw/test_build_librgw-rgw_usage.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_usage.obj `if test -f 'rgw/rgw_usage.cc'; then $(CYGPATH_W) 'rgw/rgw_usage.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_usage.cc'; fi` + +rgw/test_build_librgw-rgw_json_enc.o: rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_json_enc.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_json_enc.Tpo -c -o rgw/test_build_librgw-rgw_json_enc.o `test -f 'rgw/rgw_json_enc.cc' || echo '$(srcdir)/'`rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_json_enc.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_json_enc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_json_enc.cc' object='rgw/test_build_librgw-rgw_json_enc.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_json_enc.o `test -f 'rgw/rgw_json_enc.cc' || echo '$(srcdir)/'`rgw/rgw_json_enc.cc + +rgw/test_build_librgw-rgw_json_enc.obj: rgw/rgw_json_enc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_json_enc.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_json_enc.Tpo -c -o rgw/test_build_librgw-rgw_json_enc.obj `if test -f 'rgw/rgw_json_enc.cc'; then $(CYGPATH_W) 'rgw/rgw_json_enc.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_json_enc.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_json_enc.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_json_enc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_json_enc.cc' object='rgw/test_build_librgw-rgw_json_enc.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_json_enc.obj `if test -f 'rgw/rgw_json_enc.cc'; then $(CYGPATH_W) 'rgw/rgw_json_enc.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_json_enc.cc'; fi` + +rgw/test_build_librgw-rgw_user.o: rgw/rgw_user.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_user.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_user.Tpo -c -o rgw/test_build_librgw-rgw_user.o `test -f 'rgw/rgw_user.cc' || echo '$(srcdir)/'`rgw/rgw_user.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_user.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_user.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_user.cc' object='rgw/test_build_librgw-rgw_user.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_user.o `test -f 'rgw/rgw_user.cc' || echo '$(srcdir)/'`rgw/rgw_user.cc + +rgw/test_build_librgw-rgw_user.obj: rgw/rgw_user.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_user.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_user.Tpo -c -o rgw/test_build_librgw-rgw_user.obj `if test -f 'rgw/rgw_user.cc'; then $(CYGPATH_W) 'rgw/rgw_user.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_user.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_user.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_user.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_user.cc' object='rgw/test_build_librgw-rgw_user.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_user.obj `if test -f 'rgw/rgw_user.cc'; then $(CYGPATH_W) 'rgw/rgw_user.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_user.cc'; fi` + +rgw/test_build_librgw-rgw_bucket.o: rgw/rgw_bucket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_bucket.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_bucket.Tpo -c -o rgw/test_build_librgw-rgw_bucket.o `test -f 'rgw/rgw_bucket.cc' || echo '$(srcdir)/'`rgw/rgw_bucket.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_bucket.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_bucket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_bucket.cc' object='rgw/test_build_librgw-rgw_bucket.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_bucket.o `test -f 'rgw/rgw_bucket.cc' || echo '$(srcdir)/'`rgw/rgw_bucket.cc + +rgw/test_build_librgw-rgw_bucket.obj: rgw/rgw_bucket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_bucket.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_bucket.Tpo -c -o rgw/test_build_librgw-rgw_bucket.obj `if test -f 'rgw/rgw_bucket.cc'; then $(CYGPATH_W) 'rgw/rgw_bucket.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_bucket.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_bucket.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_bucket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_bucket.cc' object='rgw/test_build_librgw-rgw_bucket.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_bucket.obj `if test -f 'rgw/rgw_bucket.cc'; then $(CYGPATH_W) 'rgw/rgw_bucket.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_bucket.cc'; fi` + +rgw/test_build_librgw-rgw_tools.o: rgw/rgw_tools.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_tools.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_tools.Tpo -c -o rgw/test_build_librgw-rgw_tools.o `test -f 'rgw/rgw_tools.cc' || echo '$(srcdir)/'`rgw/rgw_tools.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_tools.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_tools.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_tools.cc' object='rgw/test_build_librgw-rgw_tools.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_tools.o `test -f 'rgw/rgw_tools.cc' || echo '$(srcdir)/'`rgw/rgw_tools.cc + +rgw/test_build_librgw-rgw_tools.obj: rgw/rgw_tools.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_tools.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_tools.Tpo -c -o rgw/test_build_librgw-rgw_tools.obj `if test -f 'rgw/rgw_tools.cc'; then $(CYGPATH_W) 'rgw/rgw_tools.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_tools.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_tools.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_tools.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_tools.cc' object='rgw/test_build_librgw-rgw_tools.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_tools.obj `if test -f 'rgw/rgw_tools.cc'; then $(CYGPATH_W) 'rgw/rgw_tools.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_tools.cc'; fi` + +rgw/test_build_librgw-rgw_rados.o: rgw/rgw_rados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_rados.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_rados.Tpo -c -o rgw/test_build_librgw-rgw_rados.o `test -f 'rgw/rgw_rados.cc' || echo '$(srcdir)/'`rgw/rgw_rados.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_rados.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_rados.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rados.cc' object='rgw/test_build_librgw-rgw_rados.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_rados.o `test -f 'rgw/rgw_rados.cc' || echo '$(srcdir)/'`rgw/rgw_rados.cc + +rgw/test_build_librgw-rgw_rados.obj: rgw/rgw_rados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_rados.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_rados.Tpo -c -o rgw/test_build_librgw-rgw_rados.obj `if test -f 'rgw/rgw_rados.cc'; then $(CYGPATH_W) 'rgw/rgw_rados.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_rados.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_rados.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_rados.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rados.cc' object='rgw/test_build_librgw-rgw_rados.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_rados.obj `if test -f 'rgw/rgw_rados.cc'; then $(CYGPATH_W) 'rgw/rgw_rados.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_rados.cc'; fi` + +rgw/test_build_librgw-rgw_http_client.o: rgw/rgw_http_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_http_client.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_http_client.Tpo -c -o rgw/test_build_librgw-rgw_http_client.o `test -f 'rgw/rgw_http_client.cc' || echo '$(srcdir)/'`rgw/rgw_http_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_http_client.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_http_client.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_http_client.cc' object='rgw/test_build_librgw-rgw_http_client.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_http_client.o `test -f 'rgw/rgw_http_client.cc' || echo '$(srcdir)/'`rgw/rgw_http_client.cc + +rgw/test_build_librgw-rgw_http_client.obj: rgw/rgw_http_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_http_client.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_http_client.Tpo -c -o rgw/test_build_librgw-rgw_http_client.obj `if test -f 'rgw/rgw_http_client.cc'; then $(CYGPATH_W) 'rgw/rgw_http_client.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_http_client.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_http_client.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_http_client.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_http_client.cc' object='rgw/test_build_librgw-rgw_http_client.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_http_client.obj `if test -f 'rgw/rgw_http_client.cc'; then $(CYGPATH_W) 'rgw/rgw_http_client.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_http_client.cc'; fi` + +rgw/test_build_librgw-rgw_rest_client.o: rgw/rgw_rest_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_rest_client.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_rest_client.Tpo -c -o rgw/test_build_librgw-rgw_rest_client.o `test -f 'rgw/rgw_rest_client.cc' || echo '$(srcdir)/'`rgw/rgw_rest_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_rest_client.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_rest_client.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rest_client.cc' object='rgw/test_build_librgw-rgw_rest_client.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_rest_client.o `test -f 'rgw/rgw_rest_client.cc' || echo '$(srcdir)/'`rgw/rgw_rest_client.cc + +rgw/test_build_librgw-rgw_rest_client.obj: rgw/rgw_rest_client.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_rest_client.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_rest_client.Tpo -c -o rgw/test_build_librgw-rgw_rest_client.obj `if test -f 'rgw/rgw_rest_client.cc'; then $(CYGPATH_W) 'rgw/rgw_rest_client.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_rest_client.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_rest_client.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_rest_client.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rest_client.cc' object='rgw/test_build_librgw-rgw_rest_client.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_rest_client.obj `if test -f 'rgw/rgw_rest_client.cc'; then $(CYGPATH_W) 'rgw/rgw_rest_client.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_rest_client.cc'; fi` + +rgw/test_build_librgw-rgw_rest_conn.o: rgw/rgw_rest_conn.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_rest_conn.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_rest_conn.Tpo -c -o rgw/test_build_librgw-rgw_rest_conn.o `test -f 'rgw/rgw_rest_conn.cc' || echo '$(srcdir)/'`rgw/rgw_rest_conn.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_rest_conn.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_rest_conn.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rest_conn.cc' object='rgw/test_build_librgw-rgw_rest_conn.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_rest_conn.o `test -f 'rgw/rgw_rest_conn.cc' || echo '$(srcdir)/'`rgw/rgw_rest_conn.cc + +rgw/test_build_librgw-rgw_rest_conn.obj: rgw/rgw_rest_conn.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_rest_conn.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_rest_conn.Tpo -c -o rgw/test_build_librgw-rgw_rest_conn.obj `if test -f 'rgw/rgw_rest_conn.cc'; then $(CYGPATH_W) 'rgw/rgw_rest_conn.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_rest_conn.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_rest_conn.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_rest_conn.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_rest_conn.cc' object='rgw/test_build_librgw-rgw_rest_conn.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_rest_conn.obj `if test -f 'rgw/rgw_rest_conn.cc'; then $(CYGPATH_W) 'rgw/rgw_rest_conn.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_rest_conn.cc'; fi` + +rgw/test_build_librgw-rgw_op.o: rgw/rgw_op.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_op.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_op.Tpo -c -o rgw/test_build_librgw-rgw_op.o `test -f 'rgw/rgw_op.cc' || echo '$(srcdir)/'`rgw/rgw_op.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_op.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_op.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_op.cc' object='rgw/test_build_librgw-rgw_op.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_op.o `test -f 'rgw/rgw_op.cc' || echo '$(srcdir)/'`rgw/rgw_op.cc + +rgw/test_build_librgw-rgw_op.obj: rgw/rgw_op.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_op.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_op.Tpo -c -o rgw/test_build_librgw-rgw_op.obj `if test -f 'rgw/rgw_op.cc'; then $(CYGPATH_W) 'rgw/rgw_op.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_op.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_op.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_op.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_op.cc' object='rgw/test_build_librgw-rgw_op.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_op.obj `if test -f 'rgw/rgw_op.cc'; then $(CYGPATH_W) 'rgw/rgw_op.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_op.cc'; fi` + +rgw/test_build_librgw-rgw_common.o: rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_common.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_common.Tpo -c -o rgw/test_build_librgw-rgw_common.o `test -f 'rgw/rgw_common.cc' || echo '$(srcdir)/'`rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_common.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_common.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_common.cc' object='rgw/test_build_librgw-rgw_common.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_common.o `test -f 'rgw/rgw_common.cc' || echo '$(srcdir)/'`rgw/rgw_common.cc + +rgw/test_build_librgw-rgw_common.obj: rgw/rgw_common.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_common.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_common.Tpo -c -o rgw/test_build_librgw-rgw_common.obj `if test -f 'rgw/rgw_common.cc'; then $(CYGPATH_W) 'rgw/rgw_common.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_common.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_common.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_common.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_common.cc' object='rgw/test_build_librgw-rgw_common.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_common.obj `if test -f 'rgw/rgw_common.cc'; then $(CYGPATH_W) 'rgw/rgw_common.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_common.cc'; fi` + +rgw/test_build_librgw-rgw_cache.o: rgw/rgw_cache.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_cache.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_cache.Tpo -c -o rgw/test_build_librgw-rgw_cache.o `test -f 'rgw/rgw_cache.cc' || echo '$(srcdir)/'`rgw/rgw_cache.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_cache.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_cache.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cache.cc' object='rgw/test_build_librgw-rgw_cache.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_cache.o `test -f 'rgw/rgw_cache.cc' || echo '$(srcdir)/'`rgw/rgw_cache.cc + +rgw/test_build_librgw-rgw_cache.obj: rgw/rgw_cache.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_cache.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_cache.Tpo -c -o rgw/test_build_librgw-rgw_cache.obj `if test -f 'rgw/rgw_cache.cc'; then $(CYGPATH_W) 'rgw/rgw_cache.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_cache.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_cache.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_cache.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cache.cc' object='rgw/test_build_librgw-rgw_cache.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_cache.obj `if test -f 'rgw/rgw_cache.cc'; then $(CYGPATH_W) 'rgw/rgw_cache.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_cache.cc'; fi` + +rgw/test_build_librgw-rgw_formats.o: rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_formats.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_formats.Tpo -c -o rgw/test_build_librgw-rgw_formats.o `test -f 'rgw/rgw_formats.cc' || echo '$(srcdir)/'`rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_formats.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_formats.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_formats.cc' object='rgw/test_build_librgw-rgw_formats.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_formats.o `test -f 'rgw/rgw_formats.cc' || echo '$(srcdir)/'`rgw/rgw_formats.cc + +rgw/test_build_librgw-rgw_formats.obj: rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_formats.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_formats.Tpo -c -o rgw/test_build_librgw-rgw_formats.obj `if test -f 'rgw/rgw_formats.cc'; then $(CYGPATH_W) 'rgw/rgw_formats.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_formats.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_formats.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_formats.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_formats.cc' object='rgw/test_build_librgw-rgw_formats.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_formats.obj `if test -f 'rgw/rgw_formats.cc'; then $(CYGPATH_W) 'rgw/rgw_formats.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_formats.cc'; fi` + +rgw/test_build_librgw-rgw_log.o: rgw/rgw_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_log.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_log.Tpo -c -o rgw/test_build_librgw-rgw_log.o `test -f 'rgw/rgw_log.cc' || echo '$(srcdir)/'`rgw/rgw_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_log.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_log.cc' object='rgw/test_build_librgw-rgw_log.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_log.o `test -f 'rgw/rgw_log.cc' || echo '$(srcdir)/'`rgw/rgw_log.cc + +rgw/test_build_librgw-rgw_log.obj: rgw/rgw_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_log.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_log.Tpo -c -o rgw/test_build_librgw-rgw_log.obj `if test -f 'rgw/rgw_log.cc'; then $(CYGPATH_W) 'rgw/rgw_log.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_log.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_log.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_log.cc' object='rgw/test_build_librgw-rgw_log.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_log.obj `if test -f 'rgw/rgw_log.cc'; then $(CYGPATH_W) 'rgw/rgw_log.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_log.cc'; fi` + +rgw/test_build_librgw-rgw_multi.o: rgw/rgw_multi.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_multi.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_multi.Tpo -c -o rgw/test_build_librgw-rgw_multi.o `test -f 'rgw/rgw_multi.cc' || echo '$(srcdir)/'`rgw/rgw_multi.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_multi.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_multi.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_multi.cc' object='rgw/test_build_librgw-rgw_multi.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_multi.o `test -f 'rgw/rgw_multi.cc' || echo '$(srcdir)/'`rgw/rgw_multi.cc + +rgw/test_build_librgw-rgw_multi.obj: rgw/rgw_multi.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_multi.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_multi.Tpo -c -o rgw/test_build_librgw-rgw_multi.obj `if test -f 'rgw/rgw_multi.cc'; then $(CYGPATH_W) 'rgw/rgw_multi.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_multi.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_multi.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_multi.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_multi.cc' object='rgw/test_build_librgw-rgw_multi.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_multi.obj `if test -f 'rgw/rgw_multi.cc'; then $(CYGPATH_W) 'rgw/rgw_multi.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_multi.cc'; fi` + +rgw/test_build_librgw-rgw_policy_s3.o: rgw/rgw_policy_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_policy_s3.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_policy_s3.Tpo -c -o rgw/test_build_librgw-rgw_policy_s3.o `test -f 'rgw/rgw_policy_s3.cc' || echo '$(srcdir)/'`rgw/rgw_policy_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_policy_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_policy_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_policy_s3.cc' object='rgw/test_build_librgw-rgw_policy_s3.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_policy_s3.o `test -f 'rgw/rgw_policy_s3.cc' || echo '$(srcdir)/'`rgw/rgw_policy_s3.cc + +rgw/test_build_librgw-rgw_policy_s3.obj: rgw/rgw_policy_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_policy_s3.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_policy_s3.Tpo -c -o rgw/test_build_librgw-rgw_policy_s3.obj `if test -f 'rgw/rgw_policy_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_policy_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_policy_s3.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_policy_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_policy_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_policy_s3.cc' object='rgw/test_build_librgw-rgw_policy_s3.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_policy_s3.obj `if test -f 'rgw/rgw_policy_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_policy_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_policy_s3.cc'; fi` + +rgw/test_build_librgw-rgw_gc.o: rgw/rgw_gc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_gc.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_gc.Tpo -c -o rgw/test_build_librgw-rgw_gc.o `test -f 'rgw/rgw_gc.cc' || echo '$(srcdir)/'`rgw/rgw_gc.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_gc.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_gc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_gc.cc' object='rgw/test_build_librgw-rgw_gc.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_gc.o `test -f 'rgw/rgw_gc.cc' || echo '$(srcdir)/'`rgw/rgw_gc.cc + +rgw/test_build_librgw-rgw_gc.obj: rgw/rgw_gc.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_gc.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_gc.Tpo -c -o rgw/test_build_librgw-rgw_gc.obj `if test -f 'rgw/rgw_gc.cc'; then $(CYGPATH_W) 'rgw/rgw_gc.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_gc.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_gc.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_gc.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_gc.cc' object='rgw/test_build_librgw-rgw_gc.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_gc.obj `if test -f 'rgw/rgw_gc.cc'; then $(CYGPATH_W) 'rgw/rgw_gc.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_gc.cc'; fi` + +rgw/test_build_librgw-rgw_multi_del.o: rgw/rgw_multi_del.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_multi_del.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_multi_del.Tpo -c -o rgw/test_build_librgw-rgw_multi_del.o `test -f 'rgw/rgw_multi_del.cc' || echo '$(srcdir)/'`rgw/rgw_multi_del.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_multi_del.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_multi_del.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_multi_del.cc' object='rgw/test_build_librgw-rgw_multi_del.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_multi_del.o `test -f 'rgw/rgw_multi_del.cc' || echo '$(srcdir)/'`rgw/rgw_multi_del.cc + +rgw/test_build_librgw-rgw_multi_del.obj: rgw/rgw_multi_del.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_multi_del.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_multi_del.Tpo -c -o rgw/test_build_librgw-rgw_multi_del.obj `if test -f 'rgw/rgw_multi_del.cc'; then $(CYGPATH_W) 'rgw/rgw_multi_del.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_multi_del.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_multi_del.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_multi_del.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_multi_del.cc' object='rgw/test_build_librgw-rgw_multi_del.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_multi_del.obj `if test -f 'rgw/rgw_multi_del.cc'; then $(CYGPATH_W) 'rgw/rgw_multi_del.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_multi_del.cc'; fi` + +rgw/test_build_librgw-rgw_env.o: rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_env.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_env.Tpo -c -o rgw/test_build_librgw-rgw_env.o `test -f 'rgw/rgw_env.cc' || echo '$(srcdir)/'`rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_env.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_env.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_env.cc' object='rgw/test_build_librgw-rgw_env.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_env.o `test -f 'rgw/rgw_env.cc' || echo '$(srcdir)/'`rgw/rgw_env.cc + +rgw/test_build_librgw-rgw_env.obj: rgw/rgw_env.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_env.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_env.Tpo -c -o rgw/test_build_librgw-rgw_env.obj `if test -f 'rgw/rgw_env.cc'; then $(CYGPATH_W) 'rgw/rgw_env.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_env.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_env.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_env.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_env.cc' object='rgw/test_build_librgw-rgw_env.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_env.obj `if test -f 'rgw/rgw_env.cc'; then $(CYGPATH_W) 'rgw/rgw_env.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_env.cc'; fi` + +rgw/test_build_librgw-rgw_cors.o: rgw/rgw_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_cors.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_cors.Tpo -c -o rgw/test_build_librgw-rgw_cors.o `test -f 'rgw/rgw_cors.cc' || echo '$(srcdir)/'`rgw/rgw_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_cors.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_cors.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cors.cc' object='rgw/test_build_librgw-rgw_cors.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_cors.o `test -f 'rgw/rgw_cors.cc' || echo '$(srcdir)/'`rgw/rgw_cors.cc + +rgw/test_build_librgw-rgw_cors.obj: rgw/rgw_cors.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_cors.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_cors.Tpo -c -o rgw/test_build_librgw-rgw_cors.obj `if test -f 'rgw/rgw_cors.cc'; then $(CYGPATH_W) 'rgw/rgw_cors.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_cors.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_cors.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_cors.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cors.cc' object='rgw/test_build_librgw-rgw_cors.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_cors.obj `if test -f 'rgw/rgw_cors.cc'; then $(CYGPATH_W) 'rgw/rgw_cors.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_cors.cc'; fi` + +rgw/test_build_librgw-rgw_cors_s3.o: rgw/rgw_cors_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_cors_s3.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_cors_s3.Tpo -c -o rgw/test_build_librgw-rgw_cors_s3.o `test -f 'rgw/rgw_cors_s3.cc' || echo '$(srcdir)/'`rgw/rgw_cors_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_cors_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_cors_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cors_s3.cc' object='rgw/test_build_librgw-rgw_cors_s3.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_cors_s3.o `test -f 'rgw/rgw_cors_s3.cc' || echo '$(srcdir)/'`rgw/rgw_cors_s3.cc + +rgw/test_build_librgw-rgw_cors_s3.obj: rgw/rgw_cors_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_cors_s3.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_cors_s3.Tpo -c -o rgw/test_build_librgw-rgw_cors_s3.obj `if test -f 'rgw/rgw_cors_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_cors_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_cors_s3.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_cors_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_cors_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_cors_s3.cc' object='rgw/test_build_librgw-rgw_cors_s3.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_cors_s3.obj `if test -f 'rgw/rgw_cors_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_cors_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_cors_s3.cc'; fi` + +rgw/test_build_librgw-rgw_auth_s3.o: rgw/rgw_auth_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_auth_s3.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_auth_s3.Tpo -c -o rgw/test_build_librgw-rgw_auth_s3.o `test -f 'rgw/rgw_auth_s3.cc' || echo '$(srcdir)/'`rgw/rgw_auth_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_auth_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_auth_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_auth_s3.cc' object='rgw/test_build_librgw-rgw_auth_s3.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_auth_s3.o `test -f 'rgw/rgw_auth_s3.cc' || echo '$(srcdir)/'`rgw/rgw_auth_s3.cc + +rgw/test_build_librgw-rgw_auth_s3.obj: rgw/rgw_auth_s3.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_auth_s3.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_auth_s3.Tpo -c -o rgw/test_build_librgw-rgw_auth_s3.obj `if test -f 'rgw/rgw_auth_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_auth_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_auth_s3.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_auth_s3.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_auth_s3.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_auth_s3.cc' object='rgw/test_build_librgw-rgw_auth_s3.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_auth_s3.obj `if test -f 'rgw/rgw_auth_s3.cc'; then $(CYGPATH_W) 'rgw/rgw_auth_s3.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_auth_s3.cc'; fi` + +rgw/test_build_librgw-rgw_metadata.o: rgw/rgw_metadata.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_metadata.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_metadata.Tpo -c -o rgw/test_build_librgw-rgw_metadata.o `test -f 'rgw/rgw_metadata.cc' || echo '$(srcdir)/'`rgw/rgw_metadata.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_metadata.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_metadata.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_metadata.cc' object='rgw/test_build_librgw-rgw_metadata.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_metadata.o `test -f 'rgw/rgw_metadata.cc' || echo '$(srcdir)/'`rgw/rgw_metadata.cc + +rgw/test_build_librgw-rgw_metadata.obj: rgw/rgw_metadata.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_metadata.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_metadata.Tpo -c -o rgw/test_build_librgw-rgw_metadata.obj `if test -f 'rgw/rgw_metadata.cc'; then $(CYGPATH_W) 'rgw/rgw_metadata.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_metadata.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_metadata.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_metadata.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_metadata.cc' object='rgw/test_build_librgw-rgw_metadata.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_metadata.obj `if test -f 'rgw/rgw_metadata.cc'; then $(CYGPATH_W) 'rgw/rgw_metadata.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_metadata.cc'; fi` + +rgw/test_build_librgw-rgw_replica_log.o: rgw/rgw_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_replica_log.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_replica_log.Tpo -c -o rgw/test_build_librgw-rgw_replica_log.o `test -f 'rgw/rgw_replica_log.cc' || echo '$(srcdir)/'`rgw/rgw_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_replica_log.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_replica_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_replica_log.cc' object='rgw/test_build_librgw-rgw_replica_log.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_replica_log.o `test -f 'rgw/rgw_replica_log.cc' || echo '$(srcdir)/'`rgw/rgw_replica_log.cc + +rgw/test_build_librgw-rgw_replica_log.obj: rgw/rgw_replica_log.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_replica_log.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_replica_log.Tpo -c -o rgw/test_build_librgw-rgw_replica_log.obj `if test -f 'rgw/rgw_replica_log.cc'; then $(CYGPATH_W) 'rgw/rgw_replica_log.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_replica_log.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_replica_log.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_replica_log.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_replica_log.cc' object='rgw/test_build_librgw-rgw_replica_log.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_replica_log.obj `if test -f 'rgw/rgw_replica_log.cc'; then $(CYGPATH_W) 'rgw/rgw_replica_log.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_replica_log.cc'; fi` + +rgw/test_build_librgw-rgw_keystone.o: rgw/rgw_keystone.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_keystone.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_keystone.Tpo -c -o rgw/test_build_librgw-rgw_keystone.o `test -f 'rgw/rgw_keystone.cc' || echo '$(srcdir)/'`rgw/rgw_keystone.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_keystone.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_keystone.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_keystone.cc' object='rgw/test_build_librgw-rgw_keystone.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_keystone.o `test -f 'rgw/rgw_keystone.cc' || echo '$(srcdir)/'`rgw/rgw_keystone.cc + +rgw/test_build_librgw-rgw_keystone.obj: rgw/rgw_keystone.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_keystone.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_keystone.Tpo -c -o rgw/test_build_librgw-rgw_keystone.obj `if test -f 'rgw/rgw_keystone.cc'; then $(CYGPATH_W) 'rgw/rgw_keystone.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_keystone.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_keystone.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_keystone.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_keystone.cc' object='rgw/test_build_librgw-rgw_keystone.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_keystone.obj `if test -f 'rgw/rgw_keystone.cc'; then $(CYGPATH_W) 'rgw/rgw_keystone.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_keystone.cc'; fi` + +rgw/test_build_librgw-rgw_quota.o: rgw/rgw_quota.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_quota.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_quota.Tpo -c -o rgw/test_build_librgw-rgw_quota.o `test -f 'rgw/rgw_quota.cc' || echo '$(srcdir)/'`rgw/rgw_quota.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_quota.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_quota.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_quota.cc' object='rgw/test_build_librgw-rgw_quota.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_quota.o `test -f 'rgw/rgw_quota.cc' || echo '$(srcdir)/'`rgw/rgw_quota.cc + +rgw/test_build_librgw-rgw_quota.obj: rgw/rgw_quota.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_quota.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_quota.Tpo -c -o rgw/test_build_librgw-rgw_quota.obj `if test -f 'rgw/rgw_quota.cc'; then $(CYGPATH_W) 'rgw/rgw_quota.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_quota.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_quota.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_quota.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_quota.cc' object='rgw/test_build_librgw-rgw_quota.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_quota.obj `if test -f 'rgw/rgw_quota.cc'; then $(CYGPATH_W) 'rgw/rgw_quota.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_quota.cc'; fi` + +rgw/test_build_librgw-rgw_dencoder.o: rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_dencoder.o -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_dencoder.Tpo -c -o rgw/test_build_librgw-rgw_dencoder.o `test -f 'rgw/rgw_dencoder.cc' || echo '$(srcdir)/'`rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_dencoder.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_dencoder.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_dencoder.cc' object='rgw/test_build_librgw-rgw_dencoder.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_dencoder.o `test -f 'rgw/rgw_dencoder.cc' || echo '$(srcdir)/'`rgw/rgw_dencoder.cc + +rgw/test_build_librgw-rgw_dencoder.obj: rgw/rgw_dencoder.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -MT rgw/test_build_librgw-rgw_dencoder.obj -MD -MP -MF rgw/$(DEPDIR)/test_build_librgw-rgw_dencoder.Tpo -c -o rgw/test_build_librgw-rgw_dencoder.obj `if test -f 'rgw/rgw_dencoder.cc'; then $(CYGPATH_W) 'rgw/rgw_dencoder.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_dencoder.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/test_build_librgw-rgw_dencoder.Tpo rgw/$(DEPDIR)/test_build_librgw-rgw_dencoder.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_dencoder.cc' object='rgw/test_build_librgw-rgw_dencoder.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(test_build_librgw_CXXFLAGS) $(CXXFLAGS) -c -o rgw/test_build_librgw-rgw_dencoder.obj `if test -f 'rgw/rgw_dencoder.cc'; then $(CYGPATH_W) 'rgw/rgw_dencoder.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_dencoder.cc'; fi` + +test/unittest_addrs-test_addrs.o: test/test_addrs.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_addrs_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_addrs-test_addrs.o -MD -MP -MF test/$(DEPDIR)/unittest_addrs-test_addrs.Tpo -c -o test/unittest_addrs-test_addrs.o `test -f 'test/test_addrs.cc' || echo '$(srcdir)/'`test/test_addrs.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_addrs-test_addrs.Tpo test/$(DEPDIR)/unittest_addrs-test_addrs.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_addrs.cc' object='test/unittest_addrs-test_addrs.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_addrs_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_addrs-test_addrs.o `test -f 'test/test_addrs.cc' || echo '$(srcdir)/'`test/test_addrs.cc + +test/unittest_addrs-test_addrs.obj: test/test_addrs.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_addrs_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_addrs-test_addrs.obj -MD -MP -MF test/$(DEPDIR)/unittest_addrs-test_addrs.Tpo -c -o test/unittest_addrs-test_addrs.obj `if test -f 'test/test_addrs.cc'; then $(CYGPATH_W) 'test/test_addrs.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_addrs.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_addrs-test_addrs.Tpo test/$(DEPDIR)/unittest_addrs-test_addrs.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_addrs.cc' object='test/unittest_addrs-test_addrs.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_addrs_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_addrs-test_addrs.obj `if test -f 'test/test_addrs.cc'; then $(CYGPATH_W) 'test/test_addrs.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_addrs.cc'; fi` + +test/unittest_admin_socket-admin_socket.o: test/admin_socket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_admin_socket_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_admin_socket-admin_socket.o -MD -MP -MF test/$(DEPDIR)/unittest_admin_socket-admin_socket.Tpo -c -o test/unittest_admin_socket-admin_socket.o `test -f 'test/admin_socket.cc' || echo '$(srcdir)/'`test/admin_socket.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_admin_socket-admin_socket.Tpo test/$(DEPDIR)/unittest_admin_socket-admin_socket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/admin_socket.cc' object='test/unittest_admin_socket-admin_socket.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_admin_socket_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_admin_socket-admin_socket.o `test -f 'test/admin_socket.cc' || echo '$(srcdir)/'`test/admin_socket.cc + +test/unittest_admin_socket-admin_socket.obj: test/admin_socket.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_admin_socket_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_admin_socket-admin_socket.obj -MD -MP -MF test/$(DEPDIR)/unittest_admin_socket-admin_socket.Tpo -c -o test/unittest_admin_socket-admin_socket.obj `if test -f 'test/admin_socket.cc'; then $(CYGPATH_W) 'test/admin_socket.cc'; else $(CYGPATH_W) '$(srcdir)/test/admin_socket.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_admin_socket-admin_socket.Tpo test/$(DEPDIR)/unittest_admin_socket-admin_socket.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/admin_socket.cc' object='test/unittest_admin_socket-admin_socket.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_admin_socket_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_admin_socket-admin_socket.obj `if test -f 'test/admin_socket.cc'; then $(CYGPATH_W) 'test/admin_socket.cc'; else $(CYGPATH_W) '$(srcdir)/test/admin_socket.cc'; fi` + +test/unittest_arch-test_arch.o: test/test_arch.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_arch-test_arch.o -MD -MP -MF test/$(DEPDIR)/unittest_arch-test_arch.Tpo -c -o test/unittest_arch-test_arch.o `test -f 'test/test_arch.cc' || echo '$(srcdir)/'`test/test_arch.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_arch-test_arch.Tpo test/$(DEPDIR)/unittest_arch-test_arch.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_arch.cc' object='test/unittest_arch-test_arch.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_arch-test_arch.o `test -f 'test/test_arch.cc' || echo '$(srcdir)/'`test/test_arch.cc + +test/unittest_arch-test_arch.obj: test/test_arch.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_arch-test_arch.obj -MD -MP -MF test/$(DEPDIR)/unittest_arch-test_arch.Tpo -c -o test/unittest_arch-test_arch.obj `if test -f 'test/test_arch.cc'; then $(CYGPATH_W) 'test/test_arch.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_arch.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_arch-test_arch.Tpo test/$(DEPDIR)/unittest_arch-test_arch.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_arch.cc' object='test/unittest_arch-test_arch.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_arch_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_arch-test_arch.obj `if test -f 'test/test_arch.cc'; then $(CYGPATH_W) 'test/test_arch.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_arch.cc'; fi` + +test/unittest_base64-base64.o: test/base64.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_base64_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_base64-base64.o -MD -MP -MF test/$(DEPDIR)/unittest_base64-base64.Tpo -c -o test/unittest_base64-base64.o `test -f 'test/base64.cc' || echo '$(srcdir)/'`test/base64.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_base64-base64.Tpo test/$(DEPDIR)/unittest_base64-base64.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/base64.cc' object='test/unittest_base64-base64.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_base64_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_base64-base64.o `test -f 'test/base64.cc' || echo '$(srcdir)/'`test/base64.cc + +test/unittest_base64-base64.obj: test/base64.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_base64_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_base64-base64.obj -MD -MP -MF test/$(DEPDIR)/unittest_base64-base64.Tpo -c -o test/unittest_base64-base64.obj `if test -f 'test/base64.cc'; then $(CYGPATH_W) 'test/base64.cc'; else $(CYGPATH_W) '$(srcdir)/test/base64.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_base64-base64.Tpo test/$(DEPDIR)/unittest_base64-base64.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/base64.cc' object='test/unittest_base64-base64.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_base64_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_base64-base64.obj `if test -f 'test/base64.cc'; then $(CYGPATH_W) 'test/base64.cc'; else $(CYGPATH_W) '$(srcdir)/test/base64.cc'; fi` + +test/common/unittest_bloom_filter-test_bloom_filter.o: test/common/test_bloom_filter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bloom_filter_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_bloom_filter-test_bloom_filter.o -MD -MP -MF test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Tpo -c -o test/common/unittest_bloom_filter-test_bloom_filter.o `test -f 'test/common/test_bloom_filter.cc' || echo '$(srcdir)/'`test/common/test_bloom_filter.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Tpo test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_bloom_filter.cc' object='test/common/unittest_bloom_filter-test_bloom_filter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bloom_filter_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_bloom_filter-test_bloom_filter.o `test -f 'test/common/test_bloom_filter.cc' || echo '$(srcdir)/'`test/common/test_bloom_filter.cc + +test/common/unittest_bloom_filter-test_bloom_filter.obj: test/common/test_bloom_filter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bloom_filter_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_bloom_filter-test_bloom_filter.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Tpo -c -o test/common/unittest_bloom_filter-test_bloom_filter.obj `if test -f 'test/common/test_bloom_filter.cc'; then $(CYGPATH_W) 'test/common/test_bloom_filter.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_bloom_filter.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Tpo test/common/$(DEPDIR)/unittest_bloom_filter-test_bloom_filter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_bloom_filter.cc' object='test/common/unittest_bloom_filter-test_bloom_filter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bloom_filter_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_bloom_filter-test_bloom_filter.obj `if test -f 'test/common/test_bloom_filter.cc'; then $(CYGPATH_W) 'test/common/test_bloom_filter.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_bloom_filter.cc'; fi` + +test/unittest_bufferlist-bufferlist.o: test/bufferlist.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bufferlist_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_bufferlist-bufferlist.o -MD -MP -MF test/$(DEPDIR)/unittest_bufferlist-bufferlist.Tpo -c -o test/unittest_bufferlist-bufferlist.o `test -f 'test/bufferlist.cc' || echo '$(srcdir)/'`test/bufferlist.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_bufferlist-bufferlist.Tpo test/$(DEPDIR)/unittest_bufferlist-bufferlist.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/bufferlist.cc' object='test/unittest_bufferlist-bufferlist.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bufferlist_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_bufferlist-bufferlist.o `test -f 'test/bufferlist.cc' || echo '$(srcdir)/'`test/bufferlist.cc + +test/unittest_bufferlist-bufferlist.obj: test/bufferlist.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bufferlist_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_bufferlist-bufferlist.obj -MD -MP -MF test/$(DEPDIR)/unittest_bufferlist-bufferlist.Tpo -c -o test/unittest_bufferlist-bufferlist.obj `if test -f 'test/bufferlist.cc'; then $(CYGPATH_W) 'test/bufferlist.cc'; else $(CYGPATH_W) '$(srcdir)/test/bufferlist.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_bufferlist-bufferlist.Tpo test/$(DEPDIR)/unittest_bufferlist-bufferlist.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/bufferlist.cc' object='test/unittest_bufferlist-bufferlist.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_bufferlist_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_bufferlist-bufferlist.obj `if test -f 'test/bufferlist.cc'; then $(CYGPATH_W) 'test/bufferlist.cc'; else $(CYGPATH_W) '$(srcdir)/test/bufferlist.cc'; fi` + +test/unittest_ceph_argparse-ceph_argparse.o: test/ceph_argparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_argparse_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ceph_argparse-ceph_argparse.o -MD -MP -MF test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Tpo -c -o test/unittest_ceph_argparse-ceph_argparse.o `test -f 'test/ceph_argparse.cc' || echo '$(srcdir)/'`test/ceph_argparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Tpo test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ceph_argparse.cc' object='test/unittest_ceph_argparse-ceph_argparse.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_argparse_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ceph_argparse-ceph_argparse.o `test -f 'test/ceph_argparse.cc' || echo '$(srcdir)/'`test/ceph_argparse.cc + +test/unittest_ceph_argparse-ceph_argparse.obj: test/ceph_argparse.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_argparse_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ceph_argparse-ceph_argparse.obj -MD -MP -MF test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Tpo -c -o test/unittest_ceph_argparse-ceph_argparse.obj `if test -f 'test/ceph_argparse.cc'; then $(CYGPATH_W) 'test/ceph_argparse.cc'; else $(CYGPATH_W) '$(srcdir)/test/ceph_argparse.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Tpo test/$(DEPDIR)/unittest_ceph_argparse-ceph_argparse.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ceph_argparse.cc' object='test/unittest_ceph_argparse-ceph_argparse.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_argparse_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ceph_argparse-ceph_argparse.obj `if test -f 'test/ceph_argparse.cc'; then $(CYGPATH_W) 'test/ceph_argparse.cc'; else $(CYGPATH_W) '$(srcdir)/test/ceph_argparse.cc'; fi` + +test/unittest_ceph_compatset-ceph_compatset.o: test/ceph_compatset.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_compatset_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ceph_compatset-ceph_compatset.o -MD -MP -MF test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Tpo -c -o test/unittest_ceph_compatset-ceph_compatset.o `test -f 'test/ceph_compatset.cc' || echo '$(srcdir)/'`test/ceph_compatset.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Tpo test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ceph_compatset.cc' object='test/unittest_ceph_compatset-ceph_compatset.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_compatset_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ceph_compatset-ceph_compatset.o `test -f 'test/ceph_compatset.cc' || echo '$(srcdir)/'`test/ceph_compatset.cc + +test/unittest_ceph_compatset-ceph_compatset.obj: test/ceph_compatset.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_compatset_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ceph_compatset-ceph_compatset.obj -MD -MP -MF test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Tpo -c -o test/unittest_ceph_compatset-ceph_compatset.obj `if test -f 'test/ceph_compatset.cc'; then $(CYGPATH_W) 'test/ceph_compatset.cc'; else $(CYGPATH_W) '$(srcdir)/test/ceph_compatset.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Tpo test/$(DEPDIR)/unittest_ceph_compatset-ceph_compatset.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ceph_compatset.cc' object='test/unittest_ceph_compatset-ceph_compatset.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_compatset_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ceph_compatset-ceph_compatset.obj `if test -f 'test/ceph_compatset.cc'; then $(CYGPATH_W) 'test/ceph_compatset.cc'; else $(CYGPATH_W) '$(srcdir)/test/ceph_compatset.cc'; fi` + +test/unittest_ceph_crypto-ceph_crypto.o: test/ceph_crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_crypto_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ceph_crypto-ceph_crypto.o -MD -MP -MF test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Tpo -c -o test/unittest_ceph_crypto-ceph_crypto.o `test -f 'test/ceph_crypto.cc' || echo '$(srcdir)/'`test/ceph_crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Tpo test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ceph_crypto.cc' object='test/unittest_ceph_crypto-ceph_crypto.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_crypto_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ceph_crypto-ceph_crypto.o `test -f 'test/ceph_crypto.cc' || echo '$(srcdir)/'`test/ceph_crypto.cc + +test/unittest_ceph_crypto-ceph_crypto.obj: test/ceph_crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_crypto_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ceph_crypto-ceph_crypto.obj -MD -MP -MF test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Tpo -c -o test/unittest_ceph_crypto-ceph_crypto.obj `if test -f 'test/ceph_crypto.cc'; then $(CYGPATH_W) 'test/ceph_crypto.cc'; else $(CYGPATH_W) '$(srcdir)/test/ceph_crypto.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Tpo test/$(DEPDIR)/unittest_ceph_crypto-ceph_crypto.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/ceph_crypto.cc' object='test/unittest_ceph_crypto-ceph_crypto.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ceph_crypto_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ceph_crypto-ceph_crypto.obj `if test -f 'test/ceph_crypto.cc'; then $(CYGPATH_W) 'test/ceph_crypto.cc'; else $(CYGPATH_W) '$(srcdir)/test/ceph_crypto.cc'; fi` + +test/objectstore/unittest_chain_xattr-chain_xattr.o: test/objectstore/chain_xattr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/unittest_chain_xattr-chain_xattr.o -MD -MP -MF test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo -c -o test/objectstore/unittest_chain_xattr-chain_xattr.o `test -f 'test/objectstore/chain_xattr.cc' || echo '$(srcdir)/'`test/objectstore/chain_xattr.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/chain_xattr.cc' object='test/objectstore/unittest_chain_xattr-chain_xattr.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/unittest_chain_xattr-chain_xattr.o `test -f 'test/objectstore/chain_xattr.cc' || echo '$(srcdir)/'`test/objectstore/chain_xattr.cc + +test/objectstore/unittest_chain_xattr-chain_xattr.obj: test/objectstore/chain_xattr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -MT test/objectstore/unittest_chain_xattr-chain_xattr.obj -MD -MP -MF test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo -c -o test/objectstore/unittest_chain_xattr-chain_xattr.obj `if test -f 'test/objectstore/chain_xattr.cc'; then $(CYGPATH_W) 'test/objectstore/chain_xattr.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/chain_xattr.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Tpo test/objectstore/$(DEPDIR)/unittest_chain_xattr-chain_xattr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/objectstore/chain_xattr.cc' object='test/objectstore/unittest_chain_xattr-chain_xattr.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_chain_xattr_CXXFLAGS) $(CXXFLAGS) -c -o test/objectstore/unittest_chain_xattr-chain_xattr.obj `if test -f 'test/objectstore/chain_xattr.cc'; then $(CYGPATH_W) 'test/objectstore/chain_xattr.cc'; else $(CYGPATH_W) '$(srcdir)/test/objectstore/chain_xattr.cc'; fi` + +test/common/unittest_config-test_config.o: test/common/test_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_config-test_config.o -MD -MP -MF test/common/$(DEPDIR)/unittest_config-test_config.Tpo -c -o test/common/unittest_config-test_config.o `test -f 'test/common/test_config.cc' || echo '$(srcdir)/'`test/common/test_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_config-test_config.Tpo test/common/$(DEPDIR)/unittest_config-test_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_config.cc' object='test/common/unittest_config-test_config.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_config-test_config.o `test -f 'test/common/test_config.cc' || echo '$(srcdir)/'`test/common/test_config.cc + +test/common/unittest_config-test_config.obj: test/common/test_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_config-test_config.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_config-test_config.Tpo -c -o test/common/unittest_config-test_config.obj `if test -f 'test/common/test_config.cc'; then $(CYGPATH_W) 'test/common/test_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_config.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_config-test_config.Tpo test/common/$(DEPDIR)/unittest_config-test_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_config.cc' object='test/common/unittest_config-test_config.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_config_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_config-test_config.obj `if test -f 'test/common/test_config.cc'; then $(CYGPATH_W) 'test/common/test_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_config.cc'; fi` + +test/unittest_confutils-confutils.o: test/confutils.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_confutils_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_confutils-confutils.o -MD -MP -MF test/$(DEPDIR)/unittest_confutils-confutils.Tpo -c -o test/unittest_confutils-confutils.o `test -f 'test/confutils.cc' || echo '$(srcdir)/'`test/confutils.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_confutils-confutils.Tpo test/$(DEPDIR)/unittest_confutils-confutils.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/confutils.cc' object='test/unittest_confutils-confutils.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_confutils_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_confutils-confutils.o `test -f 'test/confutils.cc' || echo '$(srcdir)/'`test/confutils.cc + +test/unittest_confutils-confutils.obj: test/confutils.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_confutils_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_confutils-confutils.obj -MD -MP -MF test/$(DEPDIR)/unittest_confutils-confutils.Tpo -c -o test/unittest_confutils-confutils.obj `if test -f 'test/confutils.cc'; then $(CYGPATH_W) 'test/confutils.cc'; else $(CYGPATH_W) '$(srcdir)/test/confutils.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_confutils-confutils.Tpo test/$(DEPDIR)/unittest_confutils-confutils.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/confutils.cc' object='test/unittest_confutils-confutils.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_confutils_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_confutils-confutils.obj `if test -f 'test/confutils.cc'; then $(CYGPATH_W) 'test/confutils.cc'; else $(CYGPATH_W) '$(srcdir)/test/confutils.cc'; fi` + +test/common/unittest_context-test_context.o: test/common/test_context.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_context-test_context.o -MD -MP -MF test/common/$(DEPDIR)/unittest_context-test_context.Tpo -c -o test/common/unittest_context-test_context.o `test -f 'test/common/test_context.cc' || echo '$(srcdir)/'`test/common/test_context.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_context-test_context.Tpo test/common/$(DEPDIR)/unittest_context-test_context.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_context.cc' object='test/common/unittest_context-test_context.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_context-test_context.o `test -f 'test/common/test_context.cc' || echo '$(srcdir)/'`test/common/test_context.cc + +test/common/unittest_context-test_context.obj: test/common/test_context.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_context-test_context.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_context-test_context.Tpo -c -o test/common/unittest_context-test_context.obj `if test -f 'test/common/test_context.cc'; then $(CYGPATH_W) 'test/common/test_context.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_context.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_context-test_context.Tpo test/common/$(DEPDIR)/unittest_context-test_context.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_context.cc' object='test/common/unittest_context-test_context.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_context_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_context-test_context.obj `if test -f 'test/common/test_context.cc'; then $(CYGPATH_W) 'test/common/test_context.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_context.cc'; fi` + +test/common/unittest_crc32c-test_crc32c.o: test/common/test_crc32c.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_crc32c-test_crc32c.o -MD -MP -MF test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Tpo -c -o test/common/unittest_crc32c-test_crc32c.o `test -f 'test/common/test_crc32c.cc' || echo '$(srcdir)/'`test/common/test_crc32c.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Tpo test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_crc32c.cc' object='test/common/unittest_crc32c-test_crc32c.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_crc32c-test_crc32c.o `test -f 'test/common/test_crc32c.cc' || echo '$(srcdir)/'`test/common/test_crc32c.cc + +test/common/unittest_crc32c-test_crc32c.obj: test/common/test_crc32c.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_crc32c-test_crc32c.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Tpo -c -o test/common/unittest_crc32c-test_crc32c.obj `if test -f 'test/common/test_crc32c.cc'; then $(CYGPATH_W) 'test/common/test_crc32c.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_crc32c.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Tpo test/common/$(DEPDIR)/unittest_crc32c-test_crc32c.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_crc32c.cc' object='test/common/unittest_crc32c-test_crc32c.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crc32c_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_crc32c-test_crc32c.obj `if test -f 'test/common/test_crc32c.cc'; then $(CYGPATH_W) 'test/common/test_crc32c.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_crc32c.cc'; fi` + +test/crush/unittest_crush_indep-indep.o: test/crush/indep.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_indep-indep.o -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo -c -o test/crush/unittest_crush_indep-indep.o `test -f 'test/crush/indep.cc' || echo '$(srcdir)/'`test/crush/indep.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo test/crush/$(DEPDIR)/unittest_crush_indep-indep.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/indep.cc' object='test/crush/unittest_crush_indep-indep.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_indep-indep.o `test -f 'test/crush/indep.cc' || echo '$(srcdir)/'`test/crush/indep.cc + +test/crush/unittest_crush_indep-indep.obj: test/crush/indep.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_indep-indep.obj -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo -c -o test/crush/unittest_crush_indep-indep.obj `if test -f 'test/crush/indep.cc'; then $(CYGPATH_W) 'test/crush/indep.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/indep.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_indep-indep.Tpo test/crush/$(DEPDIR)/unittest_crush_indep-indep.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/indep.cc' object='test/crush/unittest_crush_indep-indep.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_indep_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_indep-indep.obj `if test -f 'test/crush/indep.cc'; then $(CYGPATH_W) 'test/crush/indep.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/indep.cc'; fi` + +test/crush/unittest_crush_wrapper-TestCrushWrapper.o: test/crush/TestCrushWrapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_wrapper-TestCrushWrapper.o -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.o `test -f 'test/crush/TestCrushWrapper.cc' || echo '$(srcdir)/'`test/crush/TestCrushWrapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/TestCrushWrapper.cc' object='test/crush/unittest_crush_wrapper-TestCrushWrapper.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.o `test -f 'test/crush/TestCrushWrapper.cc' || echo '$(srcdir)/'`test/crush/TestCrushWrapper.cc + +test/crush/unittest_crush_wrapper-TestCrushWrapper.obj: test/crush/TestCrushWrapper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -MT test/crush/unittest_crush_wrapper-TestCrushWrapper.obj -MD -MP -MF test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.obj `if test -f 'test/crush/TestCrushWrapper.cc'; then $(CYGPATH_W) 'test/crush/TestCrushWrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/TestCrushWrapper.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Tpo test/crush/$(DEPDIR)/unittest_crush_wrapper-TestCrushWrapper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crush/TestCrushWrapper.cc' object='test/crush/unittest_crush_wrapper-TestCrushWrapper.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crush_wrapper_CXXFLAGS) $(CXXFLAGS) -c -o test/crush/unittest_crush_wrapper-TestCrushWrapper.obj `if test -f 'test/crush/TestCrushWrapper.cc'; then $(CYGPATH_W) 'test/crush/TestCrushWrapper.cc'; else $(CYGPATH_W) '$(srcdir)/test/crush/TestCrushWrapper.cc'; fi` + +test/unittest_crypto-crypto.o: test/crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crypto-crypto.o -MD -MP -MF test/$(DEPDIR)/unittest_crypto-crypto.Tpo -c -o test/unittest_crypto-crypto.o `test -f 'test/crypto.cc' || echo '$(srcdir)/'`test/crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crypto-crypto.Tpo test/$(DEPDIR)/unittest_crypto-crypto.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crypto.cc' object='test/unittest_crypto-crypto.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crypto-crypto.o `test -f 'test/crypto.cc' || echo '$(srcdir)/'`test/crypto.cc + +test/unittest_crypto-crypto.obj: test/crypto.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crypto-crypto.obj -MD -MP -MF test/$(DEPDIR)/unittest_crypto-crypto.Tpo -c -o test/unittest_crypto-crypto.obj `if test -f 'test/crypto.cc'; then $(CYGPATH_W) 'test/crypto.cc'; else $(CYGPATH_W) '$(srcdir)/test/crypto.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crypto-crypto.Tpo test/$(DEPDIR)/unittest_crypto-crypto.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crypto.cc' object='test/unittest_crypto-crypto.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crypto-crypto.obj `if test -f 'test/crypto.cc'; then $(CYGPATH_W) 'test/crypto.cc'; else $(CYGPATH_W) '$(srcdir)/test/crypto.cc'; fi` + +test/unittest_crypto_init-crypto_init.o: test/crypto_init.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_init_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crypto_init-crypto_init.o -MD -MP -MF test/$(DEPDIR)/unittest_crypto_init-crypto_init.Tpo -c -o test/unittest_crypto_init-crypto_init.o `test -f 'test/crypto_init.cc' || echo '$(srcdir)/'`test/crypto_init.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crypto_init-crypto_init.Tpo test/$(DEPDIR)/unittest_crypto_init-crypto_init.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crypto_init.cc' object='test/unittest_crypto_init-crypto_init.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_init_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crypto_init-crypto_init.o `test -f 'test/crypto_init.cc' || echo '$(srcdir)/'`test/crypto_init.cc + +test/unittest_crypto_init-crypto_init.obj: test/crypto_init.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_init_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_crypto_init-crypto_init.obj -MD -MP -MF test/$(DEPDIR)/unittest_crypto_init-crypto_init.Tpo -c -o test/unittest_crypto_init-crypto_init.obj `if test -f 'test/crypto_init.cc'; then $(CYGPATH_W) 'test/crypto_init.cc'; else $(CYGPATH_W) '$(srcdir)/test/crypto_init.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_crypto_init-crypto_init.Tpo test/$(DEPDIR)/unittest_crypto_init-crypto_init.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/crypto_init.cc' object='test/unittest_crypto_init-crypto_init.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_crypto_init_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_crypto_init-crypto_init.obj `if test -f 'test/crypto_init.cc'; then $(CYGPATH_W) 'test/crypto_init.cc'; else $(CYGPATH_W) '$(srcdir)/test/crypto_init.cc'; fi` + +test/unittest_daemon_config-daemon_config.o: test/daemon_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_daemon_config_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_daemon_config-daemon_config.o -MD -MP -MF test/$(DEPDIR)/unittest_daemon_config-daemon_config.Tpo -c -o test/unittest_daemon_config-daemon_config.o `test -f 'test/daemon_config.cc' || echo '$(srcdir)/'`test/daemon_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_daemon_config-daemon_config.Tpo test/$(DEPDIR)/unittest_daemon_config-daemon_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/daemon_config.cc' object='test/unittest_daemon_config-daemon_config.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_daemon_config_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_daemon_config-daemon_config.o `test -f 'test/daemon_config.cc' || echo '$(srcdir)/'`test/daemon_config.cc + +test/unittest_daemon_config-daemon_config.obj: test/daemon_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_daemon_config_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_daemon_config-daemon_config.obj -MD -MP -MF test/$(DEPDIR)/unittest_daemon_config-daemon_config.Tpo -c -o test/unittest_daemon_config-daemon_config.obj `if test -f 'test/daemon_config.cc'; then $(CYGPATH_W) 'test/daemon_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/daemon_config.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_daemon_config-daemon_config.Tpo test/$(DEPDIR)/unittest_daemon_config-daemon_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/daemon_config.cc' object='test/unittest_daemon_config-daemon_config.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_daemon_config_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_daemon_config-daemon_config.obj `if test -f 'test/daemon_config.cc'; then $(CYGPATH_W) 'test/daemon_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/daemon_config.cc'; fi` + +test/osd/unittest_ecbackend-TestECBackend.o: test/osd/TestECBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_ecbackend-TestECBackend.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo -c -o test/osd/unittest_ecbackend-TestECBackend.o `test -f 'test/osd/TestECBackend.cc' || echo '$(srcdir)/'`test/osd/TestECBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestECBackend.cc' object='test/osd/unittest_ecbackend-TestECBackend.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_ecbackend-TestECBackend.o `test -f 'test/osd/TestECBackend.cc' || echo '$(srcdir)/'`test/osd/TestECBackend.cc + +test/osd/unittest_ecbackend-TestECBackend.obj: test/osd/TestECBackend.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_ecbackend-TestECBackend.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo -c -o test/osd/unittest_ecbackend-TestECBackend.obj `if test -f 'test/osd/TestECBackend.cc'; then $(CYGPATH_W) 'test/osd/TestECBackend.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestECBackend.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Tpo test/osd/$(DEPDIR)/unittest_ecbackend-TestECBackend.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestECBackend.cc' object='test/osd/unittest_ecbackend-TestECBackend.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ecbackend_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_ecbackend-TestECBackend.obj `if test -f 'test/osd/TestECBackend.cc'; then $(CYGPATH_W) 'test/osd/TestECBackend.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestECBackend.cc'; fi` + +test/unittest_encoding-encoding.o: test/encoding.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_encoding_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_encoding-encoding.o -MD -MP -MF test/$(DEPDIR)/unittest_encoding-encoding.Tpo -c -o test/unittest_encoding-encoding.o `test -f 'test/encoding.cc' || echo '$(srcdir)/'`test/encoding.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_encoding-encoding.Tpo test/$(DEPDIR)/unittest_encoding-encoding.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/encoding.cc' object='test/unittest_encoding-encoding.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_encoding_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_encoding-encoding.o `test -f 'test/encoding.cc' || echo '$(srcdir)/'`test/encoding.cc + +test/unittest_encoding-encoding.obj: test/encoding.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_encoding_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_encoding-encoding.obj -MD -MP -MF test/$(DEPDIR)/unittest_encoding-encoding.Tpo -c -o test/unittest_encoding-encoding.obj `if test -f 'test/encoding.cc'; then $(CYGPATH_W) 'test/encoding.cc'; else $(CYGPATH_W) '$(srcdir)/test/encoding.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_encoding-encoding.Tpo test/$(DEPDIR)/unittest_encoding-encoding.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/encoding.cc' object='test/unittest_encoding-encoding.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_encoding_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_encoding-encoding.obj `if test -f 'test/encoding.cc'; then $(CYGPATH_W) 'test/encoding.cc'; else $(CYGPATH_W) '$(srcdir)/test/encoding.cc'; fi` + +test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o: test/erasure-code/TestErasureCodeExample.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o `test -f 'test/erasure-code/TestErasureCodeExample.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodeExample.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeExample.cc' object='test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.o `test -f 'test/erasure-code/TestErasureCodeExample.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodeExample.cc + +test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj: test/erasure-code/TestErasureCodeExample.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj `if test -f 'test/erasure-code/TestErasureCodeExample.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodeExample.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodeExample.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_example-TestErasureCodeExample.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeExample.cc' object='test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_example_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_example-TestErasureCodeExample.obj `if test -f 'test/erasure-code/TestErasureCodeExample.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodeExample.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodeExample.cc'; fi` + +test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o: test/erasure-code/TestErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o `test -f 'test/erasure-code/TestErasureCodeJerasure.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeJerasure.cc' object='test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.o `test -f 'test/erasure-code/TestErasureCodeJerasure.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodeJerasure.cc + +test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj: test/erasure-code/TestErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj `if test -f 'test/erasure-code/TestErasureCodeJerasure.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodeJerasure.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_jerasure-TestErasureCodeJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodeJerasure.cc' object='test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_jerasure-TestErasureCodeJerasure.obj `if test -f 'test/erasure-code/TestErasureCodeJerasure.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodeJerasure.cc'; fi` + +erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o: erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.o `test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodePluginJerasure.cc + +erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj: erasure-code/jerasure/ErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'erasure-code/jerasure/ErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/ErasureCodePluginJerasure.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodePluginJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodePluginJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'erasure-code/jerasure/ErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/ErasureCodePluginJerasure.cc'; fi` + +erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o: erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.o `test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc' || echo '$(srcdir)/'`erasure-code/jerasure/ErasureCodeJerasure.cc + +erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj: erasure-code/jerasure/ErasureCodeJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -MT erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj -MD -MP -MF erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc'; then $(CYGPATH_W) 'erasure-code/jerasure/ErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/ErasureCodeJerasure.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Tpo erasure-code/jerasure/$(DEPDIR)/unittest_erasure_code_jerasure-ErasureCodeJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='erasure-code/jerasure/ErasureCodeJerasure.cc' object='erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o erasure-code/jerasure/unittest_erasure_code_jerasure-ErasureCodeJerasure.obj `if test -f 'erasure-code/jerasure/ErasureCodeJerasure.cc'; then $(CYGPATH_W) 'erasure-code/jerasure/ErasureCodeJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/erasure-code/jerasure/ErasureCodeJerasure.cc'; fi` + +test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o: test/erasure-code/TestErasureCodePlugin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o `test -f 'test/erasure-code/TestErasureCodePlugin.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodePlugin.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePlugin.cc' object='test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.o `test -f 'test/erasure-code/TestErasureCodePlugin.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodePlugin.cc + +test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj: test/erasure-code/TestErasureCodePlugin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj `if test -f 'test/erasure-code/TestErasureCodePlugin.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodePlugin.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodePlugin.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin-TestErasureCodePlugin.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePlugin.cc' object='test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin-TestErasureCodePlugin.obj `if test -f 'test/erasure-code/TestErasureCodePlugin.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodePlugin.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodePlugin.cc'; fi` + +test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o: test/erasure-code/TestErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o `test -f 'test/erasure-code/TestErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePluginJerasure.cc' object='test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.o `test -f 'test/erasure-code/TestErasureCodePluginJerasure.cc' || echo '$(srcdir)/'`test/erasure-code/TestErasureCodePluginJerasure.cc + +test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj: test/erasure-code/TestErasureCodePluginJerasure.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -MT test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj -MD -MP -MF test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj `if test -f 'test/erasure-code/TestErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodePluginJerasure.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Tpo test/erasure-code/$(DEPDIR)/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/erasure-code/TestErasureCodePluginJerasure.cc' object='test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_erasure_code_plugin_jerasure_CXXFLAGS) $(CXXFLAGS) -c -o test/erasure-code/unittest_erasure_code_plugin_jerasure-TestErasureCodePluginJerasure.obj `if test -f 'test/erasure-code/TestErasureCodePluginJerasure.cc'; then $(CYGPATH_W) 'test/erasure-code/TestErasureCodePluginJerasure.cc'; else $(CYGPATH_W) '$(srcdir)/test/erasure-code/TestErasureCodePluginJerasure.cc'; fi` + +test/unittest_escape-escape.o: test/escape.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_escape_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_escape-escape.o -MD -MP -MF test/$(DEPDIR)/unittest_escape-escape.Tpo -c -o test/unittest_escape-escape.o `test -f 'test/escape.cc' || echo '$(srcdir)/'`test/escape.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_escape-escape.Tpo test/$(DEPDIR)/unittest_escape-escape.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/escape.cc' object='test/unittest_escape-escape.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_escape_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_escape-escape.o `test -f 'test/escape.cc' || echo '$(srcdir)/'`test/escape.cc + +test/unittest_escape-escape.obj: test/escape.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_escape_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_escape-escape.obj -MD -MP -MF test/$(DEPDIR)/unittest_escape-escape.Tpo -c -o test/unittest_escape-escape.obj `if test -f 'test/escape.cc'; then $(CYGPATH_W) 'test/escape.cc'; else $(CYGPATH_W) '$(srcdir)/test/escape.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_escape-escape.Tpo test/$(DEPDIR)/unittest_escape-escape.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/escape.cc' object='test/unittest_escape-escape.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_escape_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_escape-escape.obj `if test -f 'test/escape.cc'; then $(CYGPATH_W) 'test/escape.cc'; else $(CYGPATH_W) '$(srcdir)/test/escape.cc'; fi` + +test/os/unittest_flatindex-TestFlatIndex.o: test/os/TestFlatIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_flatindex_CXXFLAGS) $(CXXFLAGS) -MT test/os/unittest_flatindex-TestFlatIndex.o -MD -MP -MF test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Tpo -c -o test/os/unittest_flatindex-TestFlatIndex.o `test -f 'test/os/TestFlatIndex.cc' || echo '$(srcdir)/'`test/os/TestFlatIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Tpo test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/os/TestFlatIndex.cc' object='test/os/unittest_flatindex-TestFlatIndex.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_flatindex_CXXFLAGS) $(CXXFLAGS) -c -o test/os/unittest_flatindex-TestFlatIndex.o `test -f 'test/os/TestFlatIndex.cc' || echo '$(srcdir)/'`test/os/TestFlatIndex.cc + +test/os/unittest_flatindex-TestFlatIndex.obj: test/os/TestFlatIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_flatindex_CXXFLAGS) $(CXXFLAGS) -MT test/os/unittest_flatindex-TestFlatIndex.obj -MD -MP -MF test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Tpo -c -o test/os/unittest_flatindex-TestFlatIndex.obj `if test -f 'test/os/TestFlatIndex.cc'; then $(CYGPATH_W) 'test/os/TestFlatIndex.cc'; else $(CYGPATH_W) '$(srcdir)/test/os/TestFlatIndex.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Tpo test/os/$(DEPDIR)/unittest_flatindex-TestFlatIndex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/os/TestFlatIndex.cc' object='test/os/unittest_flatindex-TestFlatIndex.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_flatindex_CXXFLAGS) $(CXXFLAGS) -c -o test/os/unittest_flatindex-TestFlatIndex.obj `if test -f 'test/os/TestFlatIndex.cc'; then $(CYGPATH_W) 'test/os/TestFlatIndex.cc'; else $(CYGPATH_W) '$(srcdir)/test/os/TestFlatIndex.cc'; fi` + +test/unittest_formatter-formatter.o: test/formatter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_formatter-formatter.o -MD -MP -MF test/$(DEPDIR)/unittest_formatter-formatter.Tpo -c -o test/unittest_formatter-formatter.o `test -f 'test/formatter.cc' || echo '$(srcdir)/'`test/formatter.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_formatter-formatter.Tpo test/$(DEPDIR)/unittest_formatter-formatter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/formatter.cc' object='test/unittest_formatter-formatter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_formatter-formatter.o `test -f 'test/formatter.cc' || echo '$(srcdir)/'`test/formatter.cc + +test/unittest_formatter-formatter.obj: test/formatter.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_formatter-formatter.obj -MD -MP -MF test/$(DEPDIR)/unittest_formatter-formatter.Tpo -c -o test/unittest_formatter-formatter.obj `if test -f 'test/formatter.cc'; then $(CYGPATH_W) 'test/formatter.cc'; else $(CYGPATH_W) '$(srcdir)/test/formatter.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_formatter-formatter.Tpo test/$(DEPDIR)/unittest_formatter-formatter.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/formatter.cc' object='test/unittest_formatter-formatter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_formatter-formatter.obj `if test -f 'test/formatter.cc'; then $(CYGPATH_W) 'test/formatter.cc'; else $(CYGPATH_W) '$(srcdir)/test/formatter.cc'; fi` + +rgw/unittest_formatter-rgw_formats.o: rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -MT rgw/unittest_formatter-rgw_formats.o -MD -MP -MF rgw/$(DEPDIR)/unittest_formatter-rgw_formats.Tpo -c -o rgw/unittest_formatter-rgw_formats.o `test -f 'rgw/rgw_formats.cc' || echo '$(srcdir)/'`rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/unittest_formatter-rgw_formats.Tpo rgw/$(DEPDIR)/unittest_formatter-rgw_formats.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_formats.cc' object='rgw/unittest_formatter-rgw_formats.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -c -o rgw/unittest_formatter-rgw_formats.o `test -f 'rgw/rgw_formats.cc' || echo '$(srcdir)/'`rgw/rgw_formats.cc + +rgw/unittest_formatter-rgw_formats.obj: rgw/rgw_formats.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -MT rgw/unittest_formatter-rgw_formats.obj -MD -MP -MF rgw/$(DEPDIR)/unittest_formatter-rgw_formats.Tpo -c -o rgw/unittest_formatter-rgw_formats.obj `if test -f 'rgw/rgw_formats.cc'; then $(CYGPATH_W) 'rgw/rgw_formats.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_formats.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) rgw/$(DEPDIR)/unittest_formatter-rgw_formats.Tpo rgw/$(DEPDIR)/unittest_formatter-rgw_formats.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='rgw/rgw_formats.cc' object='rgw/unittest_formatter-rgw_formats.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_formatter_CXXFLAGS) $(CXXFLAGS) -c -o rgw/unittest_formatter-rgw_formats.obj `if test -f 'rgw/rgw_formats.cc'; then $(CYGPATH_W) 'rgw/rgw_formats.cc'; else $(CYGPATH_W) '$(srcdir)/rgw/rgw_formats.cc'; fi` + +test/unittest_gather-gather.o: test/gather.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_gather_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_gather-gather.o -MD -MP -MF test/$(DEPDIR)/unittest_gather-gather.Tpo -c -o test/unittest_gather-gather.o `test -f 'test/gather.cc' || echo '$(srcdir)/'`test/gather.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_gather-gather.Tpo test/$(DEPDIR)/unittest_gather-gather.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/gather.cc' object='test/unittest_gather-gather.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_gather_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_gather-gather.o `test -f 'test/gather.cc' || echo '$(srcdir)/'`test/gather.cc + +test/unittest_gather-gather.obj: test/gather.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_gather_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_gather-gather.obj -MD -MP -MF test/$(DEPDIR)/unittest_gather-gather.Tpo -c -o test/unittest_gather-gather.obj `if test -f 'test/gather.cc'; then $(CYGPATH_W) 'test/gather.cc'; else $(CYGPATH_W) '$(srcdir)/test/gather.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_gather-gather.Tpo test/$(DEPDIR)/unittest_gather-gather.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/gather.cc' object='test/unittest_gather-gather.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_gather_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_gather-gather.obj `if test -f 'test/gather.cc'; then $(CYGPATH_W) 'test/gather.cc'; else $(CYGPATH_W) '$(srcdir)/test/gather.cc'; fi` + +test/unittest_heartbeatmap-heartbeat_map.o: test/heartbeat_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_heartbeatmap-heartbeat_map.o -MD -MP -MF test/$(DEPDIR)/unittest_heartbeatmap-heartbeat_map.Tpo -c -o test/unittest_heartbeatmap-heartbeat_map.o `test -f 'test/heartbeat_map.cc' || echo '$(srcdir)/'`test/heartbeat_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_heartbeatmap-heartbeat_map.Tpo test/$(DEPDIR)/unittest_heartbeatmap-heartbeat_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/heartbeat_map.cc' object='test/unittest_heartbeatmap-heartbeat_map.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_heartbeatmap-heartbeat_map.o `test -f 'test/heartbeat_map.cc' || echo '$(srcdir)/'`test/heartbeat_map.cc + +test/unittest_heartbeatmap-heartbeat_map.obj: test/heartbeat_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_heartbeatmap-heartbeat_map.obj -MD -MP -MF test/$(DEPDIR)/unittest_heartbeatmap-heartbeat_map.Tpo -c -o test/unittest_heartbeatmap-heartbeat_map.obj `if test -f 'test/heartbeat_map.cc'; then $(CYGPATH_W) 'test/heartbeat_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/heartbeat_map.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_heartbeatmap-heartbeat_map.Tpo test/$(DEPDIR)/unittest_heartbeatmap-heartbeat_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/heartbeat_map.cc' object='test/unittest_heartbeatmap-heartbeat_map.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_heartbeatmap_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_heartbeatmap-heartbeat_map.obj `if test -f 'test/heartbeat_map.cc'; then $(CYGPATH_W) 'test/heartbeat_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/heartbeat_map.cc'; fi` + +test/common/unittest_histogram-histogram.o: test/common/histogram.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_histogram-histogram.o -MD -MP -MF test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo -c -o test/common/unittest_histogram-histogram.o `test -f 'test/common/histogram.cc' || echo '$(srcdir)/'`test/common/histogram.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo test/common/$(DEPDIR)/unittest_histogram-histogram.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/histogram.cc' object='test/common/unittest_histogram-histogram.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_histogram-histogram.o `test -f 'test/common/histogram.cc' || echo '$(srcdir)/'`test/common/histogram.cc + +test/common/unittest_histogram-histogram.obj: test/common/histogram.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_histogram-histogram.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo -c -o test/common/unittest_histogram-histogram.obj `if test -f 'test/common/histogram.cc'; then $(CYGPATH_W) 'test/common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/histogram.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_histogram-histogram.Tpo test/common/$(DEPDIR)/unittest_histogram-histogram.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/histogram.cc' object='test/common/unittest_histogram-histogram.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_histogram_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_histogram-histogram.obj `if test -f 'test/common/histogram.cc'; then $(CYGPATH_W) 'test/common/histogram.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/histogram.cc'; fi` + +test/osd/unittest_hitset-hitset.o: test/osd/hitset.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_hitset-hitset.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo -c -o test/osd/unittest_hitset-hitset.o `test -f 'test/osd/hitset.cc' || echo '$(srcdir)/'`test/osd/hitset.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo test/osd/$(DEPDIR)/unittest_hitset-hitset.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/hitset.cc' object='test/osd/unittest_hitset-hitset.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_hitset-hitset.o `test -f 'test/osd/hitset.cc' || echo '$(srcdir)/'`test/osd/hitset.cc + +test/osd/unittest_hitset-hitset.obj: test/osd/hitset.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_hitset-hitset.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo -c -o test/osd/unittest_hitset-hitset.obj `if test -f 'test/osd/hitset.cc'; then $(CYGPATH_W) 'test/osd/hitset.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/hitset.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_hitset-hitset.Tpo test/osd/$(DEPDIR)/unittest_hitset-hitset.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/hitset.cc' object='test/osd/unittest_hitset-hitset.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_hitset_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_hitset-hitset.obj `if test -f 'test/osd/hitset.cc'; then $(CYGPATH_W) 'test/osd/hitset.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/hitset.cc'; fi` + +test/unittest_ipaddr-test_ipaddr.o: test/test_ipaddr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ipaddr_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ipaddr-test_ipaddr.o -MD -MP -MF test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Tpo -c -o test/unittest_ipaddr-test_ipaddr.o `test -f 'test/test_ipaddr.cc' || echo '$(srcdir)/'`test/test_ipaddr.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Tpo test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_ipaddr.cc' object='test/unittest_ipaddr-test_ipaddr.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ipaddr_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ipaddr-test_ipaddr.o `test -f 'test/test_ipaddr.cc' || echo '$(srcdir)/'`test/test_ipaddr.cc + +test/unittest_ipaddr-test_ipaddr.obj: test/test_ipaddr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ipaddr_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_ipaddr-test_ipaddr.obj -MD -MP -MF test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Tpo -c -o test/unittest_ipaddr-test_ipaddr.obj `if test -f 'test/test_ipaddr.cc'; then $(CYGPATH_W) 'test/test_ipaddr.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_ipaddr.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Tpo test/$(DEPDIR)/unittest_ipaddr-test_ipaddr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_ipaddr.cc' object='test/unittest_ipaddr-test_ipaddr.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_ipaddr_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_ipaddr-test_ipaddr.obj `if test -f 'test/test_ipaddr.cc'; then $(CYGPATH_W) 'test/test_ipaddr.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_ipaddr.cc'; fi` + +test/os/unittest_lfnindex-TestLFNIndex.o: test/os/TestLFNIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_lfnindex_CXXFLAGS) $(CXXFLAGS) -MT test/os/unittest_lfnindex-TestLFNIndex.o -MD -MP -MF test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Tpo -c -o test/os/unittest_lfnindex-TestLFNIndex.o `test -f 'test/os/TestLFNIndex.cc' || echo '$(srcdir)/'`test/os/TestLFNIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Tpo test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/os/TestLFNIndex.cc' object='test/os/unittest_lfnindex-TestLFNIndex.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_lfnindex_CXXFLAGS) $(CXXFLAGS) -c -o test/os/unittest_lfnindex-TestLFNIndex.o `test -f 'test/os/TestLFNIndex.cc' || echo '$(srcdir)/'`test/os/TestLFNIndex.cc + +test/os/unittest_lfnindex-TestLFNIndex.obj: test/os/TestLFNIndex.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_lfnindex_CXXFLAGS) $(CXXFLAGS) -MT test/os/unittest_lfnindex-TestLFNIndex.obj -MD -MP -MF test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Tpo -c -o test/os/unittest_lfnindex-TestLFNIndex.obj `if test -f 'test/os/TestLFNIndex.cc'; then $(CYGPATH_W) 'test/os/TestLFNIndex.cc'; else $(CYGPATH_W) '$(srcdir)/test/os/TestLFNIndex.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Tpo test/os/$(DEPDIR)/unittest_lfnindex-TestLFNIndex.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/os/TestLFNIndex.cc' object='test/os/unittest_lfnindex-TestLFNIndex.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_lfnindex_CXXFLAGS) $(CXXFLAGS) -c -o test/os/unittest_lfnindex-TestLFNIndex.obj `if test -f 'test/os/TestLFNIndex.cc'; then $(CYGPATH_W) 'test/os/TestLFNIndex.cc'; else $(CYGPATH_W) '$(srcdir)/test/os/TestLFNIndex.cc'; fi` + +test/unittest_libcephfs_config-libcephfs_config.o: test/libcephfs_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_libcephfs_config_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_libcephfs_config-libcephfs_config.o -MD -MP -MF test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Tpo -c -o test/unittest_libcephfs_config-libcephfs_config.o `test -f 'test/libcephfs_config.cc' || echo '$(srcdir)/'`test/libcephfs_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Tpo test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs_config.cc' object='test/unittest_libcephfs_config-libcephfs_config.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_libcephfs_config_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_libcephfs_config-libcephfs_config.o `test -f 'test/libcephfs_config.cc' || echo '$(srcdir)/'`test/libcephfs_config.cc + +test/unittest_libcephfs_config-libcephfs_config.obj: test/libcephfs_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_libcephfs_config_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_libcephfs_config-libcephfs_config.obj -MD -MP -MF test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Tpo -c -o test/unittest_libcephfs_config-libcephfs_config.obj `if test -f 'test/libcephfs_config.cc'; then $(CYGPATH_W) 'test/libcephfs_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs_config.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Tpo test/$(DEPDIR)/unittest_libcephfs_config-libcephfs_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/libcephfs_config.cc' object='test/unittest_libcephfs_config-libcephfs_config.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_libcephfs_config_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_libcephfs_config-libcephfs_config.obj `if test -f 'test/libcephfs_config.cc'; then $(CYGPATH_W) 'test/libcephfs_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/libcephfs_config.cc'; fi` + +test/librados/unittest_librados-librados.o: test/librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_CXXFLAGS) $(CXXFLAGS) -MT test/librados/unittest_librados-librados.o -MD -MP -MF test/librados/$(DEPDIR)/unittest_librados-librados.Tpo -c -o test/librados/unittest_librados-librados.o `test -f 'test/librados/librados.cc' || echo '$(srcdir)/'`test/librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/unittest_librados-librados.Tpo test/librados/$(DEPDIR)/unittest_librados-librados.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/librados.cc' object='test/librados/unittest_librados-librados.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/unittest_librados-librados.o `test -f 'test/librados/librados.cc' || echo '$(srcdir)/'`test/librados/librados.cc + +test/librados/unittest_librados-librados.obj: test/librados/librados.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_CXXFLAGS) $(CXXFLAGS) -MT test/librados/unittest_librados-librados.obj -MD -MP -MF test/librados/$(DEPDIR)/unittest_librados-librados.Tpo -c -o test/librados/unittest_librados-librados.obj `if test -f 'test/librados/librados.cc'; then $(CYGPATH_W) 'test/librados/librados.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/librados.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/unittest_librados-librados.Tpo test/librados/$(DEPDIR)/unittest_librados-librados.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/librados.cc' object='test/librados/unittest_librados-librados.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/unittest_librados-librados.obj `if test -f 'test/librados/librados.cc'; then $(CYGPATH_W) 'test/librados/librados.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/librados.cc'; fi` + +test/librados/unittest_librados_config-librados_config.o: test/librados/librados_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_config_CXXFLAGS) $(CXXFLAGS) -MT test/librados/unittest_librados_config-librados_config.o -MD -MP -MF test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Tpo -c -o test/librados/unittest_librados_config-librados_config.o `test -f 'test/librados/librados_config.cc' || echo '$(srcdir)/'`test/librados/librados_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Tpo test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/librados_config.cc' object='test/librados/unittest_librados_config-librados_config.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_config_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/unittest_librados_config-librados_config.o `test -f 'test/librados/librados_config.cc' || echo '$(srcdir)/'`test/librados/librados_config.cc + +test/librados/unittest_librados_config-librados_config.obj: test/librados/librados_config.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_config_CXXFLAGS) $(CXXFLAGS) -MT test/librados/unittest_librados_config-librados_config.obj -MD -MP -MF test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Tpo -c -o test/librados/unittest_librados_config-librados_config.obj `if test -f 'test/librados/librados_config.cc'; then $(CYGPATH_W) 'test/librados/librados_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/librados_config.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Tpo test/librados/$(DEPDIR)/unittest_librados_config-librados_config.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/librados/librados_config.cc' object='test/librados/unittest_librados_config-librados_config.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_librados_config_CXXFLAGS) $(CXXFLAGS) -c -o test/librados/unittest_librados_config-librados_config.obj `if test -f 'test/librados/librados_config.cc'; then $(CYGPATH_W) 'test/librados/librados_config.cc'; else $(CYGPATH_W) '$(srcdir)/test/librados/librados_config.cc'; fi` + +log/unittest_log-test.o: log/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_log_CXXFLAGS) $(CXXFLAGS) -MT log/unittest_log-test.o -MD -MP -MF log/$(DEPDIR)/unittest_log-test.Tpo -c -o log/unittest_log-test.o `test -f 'log/test.cc' || echo '$(srcdir)/'`log/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) log/$(DEPDIR)/unittest_log-test.Tpo log/$(DEPDIR)/unittest_log-test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='log/test.cc' object='log/unittest_log-test.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_log_CXXFLAGS) $(CXXFLAGS) -c -o log/unittest_log-test.o `test -f 'log/test.cc' || echo '$(srcdir)/'`log/test.cc + +log/unittest_log-test.obj: log/test.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_log_CXXFLAGS) $(CXXFLAGS) -MT log/unittest_log-test.obj -MD -MP -MF log/$(DEPDIR)/unittest_log-test.Tpo -c -o log/unittest_log-test.obj `if test -f 'log/test.cc'; then $(CYGPATH_W) 'log/test.cc'; else $(CYGPATH_W) '$(srcdir)/log/test.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) log/$(DEPDIR)/unittest_log-test.Tpo log/$(DEPDIR)/unittest_log-test.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='log/test.cc' object='log/unittest_log-test.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_log_CXXFLAGS) $(CXXFLAGS) -c -o log/unittest_log-test.obj `if test -f 'log/test.cc'; then $(CYGPATH_W) 'log/test.cc'; else $(CYGPATH_W) '$(srcdir)/log/test.cc'; fi` + +test/unittest_mime-mime.o: test/mime.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mime_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_mime-mime.o -MD -MP -MF test/$(DEPDIR)/unittest_mime-mime.Tpo -c -o test/unittest_mime-mime.o `test -f 'test/mime.cc' || echo '$(srcdir)/'`test/mime.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_mime-mime.Tpo test/$(DEPDIR)/unittest_mime-mime.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mime.cc' object='test/unittest_mime-mime.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mime_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_mime-mime.o `test -f 'test/mime.cc' || echo '$(srcdir)/'`test/mime.cc + +test/unittest_mime-mime.obj: test/mime.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mime_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_mime-mime.obj -MD -MP -MF test/$(DEPDIR)/unittest_mime-mime.Tpo -c -o test/unittest_mime-mime.obj `if test -f 'test/mime.cc'; then $(CYGPATH_W) 'test/mime.cc'; else $(CYGPATH_W) '$(srcdir)/test/mime.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_mime-mime.Tpo test/$(DEPDIR)/unittest_mime-mime.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mime.cc' object='test/unittest_mime-mime.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mime_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_mime-mime.obj `if test -f 'test/mime.cc'; then $(CYGPATH_W) 'test/mime.cc'; else $(CYGPATH_W) '$(srcdir)/test/mime.cc'; fi` + +test/mon/unittest_mon_moncap-moncap.o: test/mon/moncap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) -MT test/mon/unittest_mon_moncap-moncap.o -MD -MP -MF test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Tpo -c -o test/mon/unittest_mon_moncap-moncap.o `test -f 'test/mon/moncap.cc' || echo '$(srcdir)/'`test/mon/moncap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Tpo test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mon/moncap.cc' object='test/mon/unittest_mon_moncap-moncap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) -c -o test/mon/unittest_mon_moncap-moncap.o `test -f 'test/mon/moncap.cc' || echo '$(srcdir)/'`test/mon/moncap.cc + +test/mon/unittest_mon_moncap-moncap.obj: test/mon/moncap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) -MT test/mon/unittest_mon_moncap-moncap.obj -MD -MP -MF test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Tpo -c -o test/mon/unittest_mon_moncap-moncap.obj `if test -f 'test/mon/moncap.cc'; then $(CYGPATH_W) 'test/mon/moncap.cc'; else $(CYGPATH_W) '$(srcdir)/test/mon/moncap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Tpo test/mon/$(DEPDIR)/unittest_mon_moncap-moncap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mon/moncap.cc' object='test/mon/unittest_mon_moncap-moncap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_moncap_CXXFLAGS) $(CXXFLAGS) -c -o test/mon/unittest_mon_moncap-moncap.obj `if test -f 'test/mon/moncap.cc'; then $(CYGPATH_W) 'test/mon/moncap.cc'; else $(CYGPATH_W) '$(srcdir)/test/mon/moncap.cc'; fi` + +test/mon/unittest_mon_pgmap-PGMap.o: test/mon/PGMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -MT test/mon/unittest_mon_pgmap-PGMap.o -MD -MP -MF test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo -c -o test/mon/unittest_mon_pgmap-PGMap.o `test -f 'test/mon/PGMap.cc' || echo '$(srcdir)/'`test/mon/PGMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mon/PGMap.cc' object='test/mon/unittest_mon_pgmap-PGMap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -c -o test/mon/unittest_mon_pgmap-PGMap.o `test -f 'test/mon/PGMap.cc' || echo '$(srcdir)/'`test/mon/PGMap.cc + +test/mon/unittest_mon_pgmap-PGMap.obj: test/mon/PGMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -MT test/mon/unittest_mon_pgmap-PGMap.obj -MD -MP -MF test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo -c -o test/mon/unittest_mon_pgmap-PGMap.obj `if test -f 'test/mon/PGMap.cc'; then $(CYGPATH_W) 'test/mon/PGMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/mon/PGMap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Tpo test/mon/$(DEPDIR)/unittest_mon_pgmap-PGMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/mon/PGMap.cc' object='test/mon/unittest_mon_pgmap-PGMap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_mon_pgmap_CXXFLAGS) $(CXXFLAGS) -c -o test/mon/unittest_mon_pgmap-PGMap.obj `if test -f 'test/mon/PGMap.cc'; then $(CYGPATH_W) 'test/mon/PGMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/mon/PGMap.cc'; fi` + +test/osd/unittest_osd_osdcap-osdcap.o: test/osd/osdcap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osd_osdcap-osdcap.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Tpo -c -o test/osd/unittest_osd_osdcap-osdcap.o `test -f 'test/osd/osdcap.cc' || echo '$(srcdir)/'`test/osd/osdcap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Tpo test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/osdcap.cc' object='test/osd/unittest_osd_osdcap-osdcap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osd_osdcap-osdcap.o `test -f 'test/osd/osdcap.cc' || echo '$(srcdir)/'`test/osd/osdcap.cc + +test/osd/unittest_osd_osdcap-osdcap.obj: test/osd/osdcap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osd_osdcap-osdcap.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Tpo -c -o test/osd/unittest_osd_osdcap-osdcap.obj `if test -f 'test/osd/osdcap.cc'; then $(CYGPATH_W) 'test/osd/osdcap.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/osdcap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Tpo test/osd/$(DEPDIR)/unittest_osd_osdcap-osdcap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/osdcap.cc' object='test/osd/unittest_osd_osdcap-osdcap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_osdcap_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osd_osdcap-osdcap.obj `if test -f 'test/osd/osdcap.cc'; then $(CYGPATH_W) 'test/osd/osdcap.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/osdcap.cc'; fi` + +test/osd/unittest_osd_types-types.o: test/osd/types.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osd_types-types.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo -c -o test/osd/unittest_osd_types-types.o `test -f 'test/osd/types.cc' || echo '$(srcdir)/'`test/osd/types.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo test/osd/$(DEPDIR)/unittest_osd_types-types.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/types.cc' object='test/osd/unittest_osd_types-types.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osd_types-types.o `test -f 'test/osd/types.cc' || echo '$(srcdir)/'`test/osd/types.cc + +test/osd/unittest_osd_types-types.obj: test/osd/types.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osd_types-types.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo -c -o test/osd/unittest_osd_types-types.obj `if test -f 'test/osd/types.cc'; then $(CYGPATH_W) 'test/osd/types.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/types.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osd_types-types.Tpo test/osd/$(DEPDIR)/unittest_osd_types-types.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/types.cc' object='test/osd/unittest_osd_types-types.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osd_types_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osd_types-types.obj `if test -f 'test/osd/types.cc'; then $(CYGPATH_W) 'test/osd/types.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/types.cc'; fi` + +test/osd/unittest_osdmap-TestOSDMap.o: test/osd/TestOSDMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osdmap-TestOSDMap.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo -c -o test/osd/unittest_osdmap-TestOSDMap.o `test -f 'test/osd/TestOSDMap.cc' || echo '$(srcdir)/'`test/osd/TestOSDMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestOSDMap.cc' object='test/osd/unittest_osdmap-TestOSDMap.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osdmap-TestOSDMap.o `test -f 'test/osd/TestOSDMap.cc' || echo '$(srcdir)/'`test/osd/TestOSDMap.cc + +test/osd/unittest_osdmap-TestOSDMap.obj: test/osd/TestOSDMap.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_osdmap-TestOSDMap.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo -c -o test/osd/unittest_osdmap-TestOSDMap.obj `if test -f 'test/osd/TestOSDMap.cc'; then $(CYGPATH_W) 'test/osd/TestOSDMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestOSDMap.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Tpo test/osd/$(DEPDIR)/unittest_osdmap-TestOSDMap.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestOSDMap.cc' object='test/osd/unittest_osdmap-TestOSDMap.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_osdmap_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_osdmap-TestOSDMap.obj `if test -f 'test/osd/TestOSDMap.cc'; then $(CYGPATH_W) 'test/osd/TestOSDMap.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestOSDMap.cc'; fi` + +test/unittest_perf_counters-perf_counters.o: test/perf_counters.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_perf_counters_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_perf_counters-perf_counters.o -MD -MP -MF test/$(DEPDIR)/unittest_perf_counters-perf_counters.Tpo -c -o test/unittest_perf_counters-perf_counters.o `test -f 'test/perf_counters.cc' || echo '$(srcdir)/'`test/perf_counters.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_perf_counters-perf_counters.Tpo test/$(DEPDIR)/unittest_perf_counters-perf_counters.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/perf_counters.cc' object='test/unittest_perf_counters-perf_counters.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_perf_counters_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_perf_counters-perf_counters.o `test -f 'test/perf_counters.cc' || echo '$(srcdir)/'`test/perf_counters.cc + +test/unittest_perf_counters-perf_counters.obj: test/perf_counters.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_perf_counters_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_perf_counters-perf_counters.obj -MD -MP -MF test/$(DEPDIR)/unittest_perf_counters-perf_counters.Tpo -c -o test/unittest_perf_counters-perf_counters.obj `if test -f 'test/perf_counters.cc'; then $(CYGPATH_W) 'test/perf_counters.cc'; else $(CYGPATH_W) '$(srcdir)/test/perf_counters.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_perf_counters-perf_counters.Tpo test/$(DEPDIR)/unittest_perf_counters-perf_counters.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/perf_counters.cc' object='test/unittest_perf_counters-perf_counters.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_perf_counters_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_perf_counters-perf_counters.obj `if test -f 'test/perf_counters.cc'; then $(CYGPATH_W) 'test/perf_counters.cc'; else $(CYGPATH_W) '$(srcdir)/test/perf_counters.cc'; fi` + +test/osd/unittest_pglog-TestPGLog.o: test/osd/TestPGLog.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_pglog_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_pglog-TestPGLog.o -MD -MP -MF test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Tpo -c -o test/osd/unittest_pglog-TestPGLog.o `test -f 'test/osd/TestPGLog.cc' || echo '$(srcdir)/'`test/osd/TestPGLog.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Tpo test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestPGLog.cc' object='test/osd/unittest_pglog-TestPGLog.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_pglog_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_pglog-TestPGLog.o `test -f 'test/osd/TestPGLog.cc' || echo '$(srcdir)/'`test/osd/TestPGLog.cc + +test/osd/unittest_pglog-TestPGLog.obj: test/osd/TestPGLog.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_pglog_CXXFLAGS) $(CXXFLAGS) -MT test/osd/unittest_pglog-TestPGLog.obj -MD -MP -MF test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Tpo -c -o test/osd/unittest_pglog-TestPGLog.obj `if test -f 'test/osd/TestPGLog.cc'; then $(CYGPATH_W) 'test/osd/TestPGLog.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestPGLog.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Tpo test/osd/$(DEPDIR)/unittest_pglog-TestPGLog.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/osd/TestPGLog.cc' object='test/osd/unittest_pglog-TestPGLog.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_pglog_CXXFLAGS) $(CXXFLAGS) -c -o test/osd/unittest_pglog-TestPGLog.obj `if test -f 'test/osd/TestPGLog.cc'; then $(CYGPATH_W) 'test/osd/TestPGLog.cc'; else $(CYGPATH_W) '$(srcdir)/test/osd/TestPGLog.cc'; fi` + +test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.o: test/test_prebufferedstreambuf.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_prebufferedstreambuf_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.o -MD -MP -MF test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Tpo -c -o test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.o `test -f 'test/test_prebufferedstreambuf.cc' || echo '$(srcdir)/'`test/test_prebufferedstreambuf.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Tpo test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_prebufferedstreambuf.cc' object='test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_prebufferedstreambuf_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.o `test -f 'test/test_prebufferedstreambuf.cc' || echo '$(srcdir)/'`test/test_prebufferedstreambuf.cc + +test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.obj: test/test_prebufferedstreambuf.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_prebufferedstreambuf_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.obj -MD -MP -MF test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Tpo -c -o test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.obj `if test -f 'test/test_prebufferedstreambuf.cc'; then $(CYGPATH_W) 'test/test_prebufferedstreambuf.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_prebufferedstreambuf.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Tpo test/$(DEPDIR)/unittest_prebufferedstreambuf-test_prebufferedstreambuf.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_prebufferedstreambuf.cc' object='test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_prebufferedstreambuf_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_prebufferedstreambuf-test_prebufferedstreambuf.obj `if test -f 'test/test_prebufferedstreambuf.cc'; then $(CYGPATH_W) 'test/test_prebufferedstreambuf.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_prebufferedstreambuf.cc'; fi` + +test/unittest_run_cmd-run_cmd.o: test/run_cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_run_cmd_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_run_cmd-run_cmd.o -MD -MP -MF test/$(DEPDIR)/unittest_run_cmd-run_cmd.Tpo -c -o test/unittest_run_cmd-run_cmd.o `test -f 'test/run_cmd.cc' || echo '$(srcdir)/'`test/run_cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_run_cmd-run_cmd.Tpo test/$(DEPDIR)/unittest_run_cmd-run_cmd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/run_cmd.cc' object='test/unittest_run_cmd-run_cmd.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_run_cmd_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_run_cmd-run_cmd.o `test -f 'test/run_cmd.cc' || echo '$(srcdir)/'`test/run_cmd.cc + +test/unittest_run_cmd-run_cmd.obj: test/run_cmd.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_run_cmd_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_run_cmd-run_cmd.obj -MD -MP -MF test/$(DEPDIR)/unittest_run_cmd-run_cmd.Tpo -c -o test/unittest_run_cmd-run_cmd.obj `if test -f 'test/run_cmd.cc'; then $(CYGPATH_W) 'test/run_cmd.cc'; else $(CYGPATH_W) '$(srcdir)/test/run_cmd.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_run_cmd-run_cmd.Tpo test/$(DEPDIR)/unittest_run_cmd-run_cmd.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/run_cmd.cc' object='test/unittest_run_cmd-run_cmd.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_run_cmd_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_run_cmd-run_cmd.obj `if test -f 'test/run_cmd.cc'; then $(CYGPATH_W) 'test/run_cmd.cc'; else $(CYGPATH_W) '$(srcdir)/test/run_cmd.cc'; fi` + +test/common/unittest_sharedptr_registry-test_sharedptr_registry.o: test/common/test_sharedptr_registry.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sharedptr_registry_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_sharedptr_registry-test_sharedptr_registry.o -MD -MP -MF test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Tpo -c -o test/common/unittest_sharedptr_registry-test_sharedptr_registry.o `test -f 'test/common/test_sharedptr_registry.cc' || echo '$(srcdir)/'`test/common/test_sharedptr_registry.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Tpo test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_sharedptr_registry.cc' object='test/common/unittest_sharedptr_registry-test_sharedptr_registry.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sharedptr_registry_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_sharedptr_registry-test_sharedptr_registry.o `test -f 'test/common/test_sharedptr_registry.cc' || echo '$(srcdir)/'`test/common/test_sharedptr_registry.cc + +test/common/unittest_sharedptr_registry-test_sharedptr_registry.obj: test/common/test_sharedptr_registry.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sharedptr_registry_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_sharedptr_registry-test_sharedptr_registry.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Tpo -c -o test/common/unittest_sharedptr_registry-test_sharedptr_registry.obj `if test -f 'test/common/test_sharedptr_registry.cc'; then $(CYGPATH_W) 'test/common/test_sharedptr_registry.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_sharedptr_registry.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Tpo test/common/$(DEPDIR)/unittest_sharedptr_registry-test_sharedptr_registry.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_sharedptr_registry.cc' object='test/common/unittest_sharedptr_registry-test_sharedptr_registry.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sharedptr_registry_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_sharedptr_registry-test_sharedptr_registry.obj `if test -f 'test/common/test_sharedptr_registry.cc'; then $(CYGPATH_W) 'test/common/test_sharedptr_registry.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_sharedptr_registry.cc'; fi` + +test/unittest_signals-signals.o: test/signals.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_signals_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_signals-signals.o -MD -MP -MF test/$(DEPDIR)/unittest_signals-signals.Tpo -c -o test/unittest_signals-signals.o `test -f 'test/signals.cc' || echo '$(srcdir)/'`test/signals.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_signals-signals.Tpo test/$(DEPDIR)/unittest_signals-signals.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/signals.cc' object='test/unittest_signals-signals.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_signals_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_signals-signals.o `test -f 'test/signals.cc' || echo '$(srcdir)/'`test/signals.cc + +test/unittest_signals-signals.obj: test/signals.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_signals_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_signals-signals.obj -MD -MP -MF test/$(DEPDIR)/unittest_signals-signals.Tpo -c -o test/unittest_signals-signals.obj `if test -f 'test/signals.cc'; then $(CYGPATH_W) 'test/signals.cc'; else $(CYGPATH_W) '$(srcdir)/test/signals.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_signals-signals.Tpo test/$(DEPDIR)/unittest_signals-signals.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/signals.cc' object='test/unittest_signals-signals.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_signals_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_signals-signals.obj `if test -f 'test/signals.cc'; then $(CYGPATH_W) 'test/signals.cc'; else $(CYGPATH_W) '$(srcdir)/test/signals.cc'; fi` + +test/unittest_simple_spin-simple_spin.o: test/simple_spin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_simple_spin_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_simple_spin-simple_spin.o -MD -MP -MF test/$(DEPDIR)/unittest_simple_spin-simple_spin.Tpo -c -o test/unittest_simple_spin-simple_spin.o `test -f 'test/simple_spin.cc' || echo '$(srcdir)/'`test/simple_spin.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_simple_spin-simple_spin.Tpo test/$(DEPDIR)/unittest_simple_spin-simple_spin.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/simple_spin.cc' object='test/unittest_simple_spin-simple_spin.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_simple_spin_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_simple_spin-simple_spin.o `test -f 'test/simple_spin.cc' || echo '$(srcdir)/'`test/simple_spin.cc + +test/unittest_simple_spin-simple_spin.obj: test/simple_spin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_simple_spin_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_simple_spin-simple_spin.obj -MD -MP -MF test/$(DEPDIR)/unittest_simple_spin-simple_spin.Tpo -c -o test/unittest_simple_spin-simple_spin.obj `if test -f 'test/simple_spin.cc'; then $(CYGPATH_W) 'test/simple_spin.cc'; else $(CYGPATH_W) '$(srcdir)/test/simple_spin.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_simple_spin-simple_spin.Tpo test/$(DEPDIR)/unittest_simple_spin-simple_spin.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/simple_spin.cc' object='test/unittest_simple_spin-simple_spin.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_simple_spin_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_simple_spin-simple_spin.obj `if test -f 'test/simple_spin.cc'; then $(CYGPATH_W) 'test/simple_spin.cc'; else $(CYGPATH_W) '$(srcdir)/test/simple_spin.cc'; fi` + +test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.o: test/common/test_sloppy_crc_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sloppy_crc_map_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.o -MD -MP -MF test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Tpo -c -o test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.o `test -f 'test/common/test_sloppy_crc_map.cc' || echo '$(srcdir)/'`test/common/test_sloppy_crc_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Tpo test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_sloppy_crc_map.cc' object='test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sloppy_crc_map_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.o `test -f 'test/common/test_sloppy_crc_map.cc' || echo '$(srcdir)/'`test/common/test_sloppy_crc_map.cc + +test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.obj: test/common/test_sloppy_crc_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sloppy_crc_map_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Tpo -c -o test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.obj `if test -f 'test/common/test_sloppy_crc_map.cc'; then $(CYGPATH_W) 'test/common/test_sloppy_crc_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_sloppy_crc_map.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Tpo test/common/$(DEPDIR)/unittest_sloppy_crc_map-test_sloppy_crc_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_sloppy_crc_map.cc' object='test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_sloppy_crc_map_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_sloppy_crc_map-test_sloppy_crc_map.obj `if test -f 'test/common/test_sloppy_crc_map.cc'; then $(CYGPATH_W) 'test/common/test_sloppy_crc_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_sloppy_crc_map.cc'; fi` + +test/unittest_str_list-test_str_list.o: test/test_str_list.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_list_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_str_list-test_str_list.o -MD -MP -MF test/$(DEPDIR)/unittest_str_list-test_str_list.Tpo -c -o test/unittest_str_list-test_str_list.o `test -f 'test/test_str_list.cc' || echo '$(srcdir)/'`test/test_str_list.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_str_list-test_str_list.Tpo test/$(DEPDIR)/unittest_str_list-test_str_list.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_str_list.cc' object='test/unittest_str_list-test_str_list.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_list_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_str_list-test_str_list.o `test -f 'test/test_str_list.cc' || echo '$(srcdir)/'`test/test_str_list.cc + +test/unittest_str_list-test_str_list.obj: test/test_str_list.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_list_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_str_list-test_str_list.obj -MD -MP -MF test/$(DEPDIR)/unittest_str_list-test_str_list.Tpo -c -o test/unittest_str_list-test_str_list.obj `if test -f 'test/test_str_list.cc'; then $(CYGPATH_W) 'test/test_str_list.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_str_list.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_str_list-test_str_list.Tpo test/$(DEPDIR)/unittest_str_list-test_str_list.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_str_list.cc' object='test/unittest_str_list-test_str_list.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_list_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_str_list-test_str_list.obj `if test -f 'test/test_str_list.cc'; then $(CYGPATH_W) 'test/test_str_list.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_str_list.cc'; fi` + +test/common/unittest_str_map-test_str_map.o: test/common/test_str_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_str_map-test_str_map.o -MD -MP -MF test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo -c -o test/common/unittest_str_map-test_str_map.o `test -f 'test/common/test_str_map.cc' || echo '$(srcdir)/'`test/common/test_str_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo test/common/$(DEPDIR)/unittest_str_map-test_str_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_str_map.cc' object='test/common/unittest_str_map-test_str_map.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_str_map-test_str_map.o `test -f 'test/common/test_str_map.cc' || echo '$(srcdir)/'`test/common/test_str_map.cc + +test/common/unittest_str_map-test_str_map.obj: test/common/test_str_map.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_str_map-test_str_map.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo -c -o test/common/unittest_str_map-test_str_map.obj `if test -f 'test/common/test_str_map.cc'; then $(CYGPATH_W) 'test/common/test_str_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_str_map.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_str_map-test_str_map.Tpo test/common/$(DEPDIR)/unittest_str_map-test_str_map.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_str_map.cc' object='test/common/unittest_str_map-test_str_map.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_str_map_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_str_map-test_str_map.obj `if test -f 'test/common/test_str_map.cc'; then $(CYGPATH_W) 'test/common/test_str_map.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_str_map.cc'; fi` + +test/unittest_striper-test_striper.o: test/test_striper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_striper_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_striper-test_striper.o -MD -MP -MF test/$(DEPDIR)/unittest_striper-test_striper.Tpo -c -o test/unittest_striper-test_striper.o `test -f 'test/test_striper.cc' || echo '$(srcdir)/'`test/test_striper.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_striper-test_striper.Tpo test/$(DEPDIR)/unittest_striper-test_striper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_striper.cc' object='test/unittest_striper-test_striper.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_striper_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_striper-test_striper.o `test -f 'test/test_striper.cc' || echo '$(srcdir)/'`test/test_striper.cc + +test/unittest_striper-test_striper.obj: test/test_striper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_striper_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_striper-test_striper.obj -MD -MP -MF test/$(DEPDIR)/unittest_striper-test_striper.Tpo -c -o test/unittest_striper-test_striper.obj `if test -f 'test/test_striper.cc'; then $(CYGPATH_W) 'test/test_striper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_striper.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_striper-test_striper.Tpo test/$(DEPDIR)/unittest_striper-test_striper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_striper.cc' object='test/unittest_striper-test_striper.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_striper_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_striper-test_striper.obj `if test -f 'test/test_striper.cc'; then $(CYGPATH_W) 'test/test_striper.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_striper.cc'; fi` + +test/unittest_strtol-strtol.o: test/strtol.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_strtol_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_strtol-strtol.o -MD -MP -MF test/$(DEPDIR)/unittest_strtol-strtol.Tpo -c -o test/unittest_strtol-strtol.o `test -f 'test/strtol.cc' || echo '$(srcdir)/'`test/strtol.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_strtol-strtol.Tpo test/$(DEPDIR)/unittest_strtol-strtol.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/strtol.cc' object='test/unittest_strtol-strtol.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_strtol_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_strtol-strtol.o `test -f 'test/strtol.cc' || echo '$(srcdir)/'`test/strtol.cc + +test/unittest_strtol-strtol.obj: test/strtol.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_strtol_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_strtol-strtol.obj -MD -MP -MF test/$(DEPDIR)/unittest_strtol-strtol.Tpo -c -o test/unittest_strtol-strtol.obj `if test -f 'test/strtol.cc'; then $(CYGPATH_W) 'test/strtol.cc'; else $(CYGPATH_W) '$(srcdir)/test/strtol.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_strtol-strtol.Tpo test/$(DEPDIR)/unittest_strtol-strtol.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/strtol.cc' object='test/unittest_strtol-strtol.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_strtol_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_strtol-strtol.obj `if test -f 'test/strtol.cc'; then $(CYGPATH_W) 'test/strtol.cc'; else $(CYGPATH_W) '$(srcdir)/test/strtol.cc'; fi` + +test/unittest_texttable-test_texttable.o: test/test_texttable.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_texttable_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_texttable-test_texttable.o -MD -MP -MF test/$(DEPDIR)/unittest_texttable-test_texttable.Tpo -c -o test/unittest_texttable-test_texttable.o `test -f 'test/test_texttable.cc' || echo '$(srcdir)/'`test/test_texttable.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_texttable-test_texttable.Tpo test/$(DEPDIR)/unittest_texttable-test_texttable.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_texttable.cc' object='test/unittest_texttable-test_texttable.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_texttable_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_texttable-test_texttable.o `test -f 'test/test_texttable.cc' || echo '$(srcdir)/'`test/test_texttable.cc + +test/unittest_texttable-test_texttable.obj: test/test_texttable.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_texttable_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_texttable-test_texttable.obj -MD -MP -MF test/$(DEPDIR)/unittest_texttable-test_texttable.Tpo -c -o test/unittest_texttable-test_texttable.obj `if test -f 'test/test_texttable.cc'; then $(CYGPATH_W) 'test/test_texttable.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_texttable.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_texttable-test_texttable.Tpo test/$(DEPDIR)/unittest_texttable-test_texttable.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_texttable.cc' object='test/unittest_texttable-test_texttable.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_texttable_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_texttable-test_texttable.obj `if test -f 'test/test_texttable.cc'; then $(CYGPATH_W) 'test/test_texttable.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_texttable.cc'; fi` + +test/common/unittest_throttle-Throttle.o: test/common/Throttle.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_throttle_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_throttle-Throttle.o -MD -MP -MF test/common/$(DEPDIR)/unittest_throttle-Throttle.Tpo -c -o test/common/unittest_throttle-Throttle.o `test -f 'test/common/Throttle.cc' || echo '$(srcdir)/'`test/common/Throttle.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_throttle-Throttle.Tpo test/common/$(DEPDIR)/unittest_throttle-Throttle.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/Throttle.cc' object='test/common/unittest_throttle-Throttle.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_throttle_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_throttle-Throttle.o `test -f 'test/common/Throttle.cc' || echo '$(srcdir)/'`test/common/Throttle.cc + +test/common/unittest_throttle-Throttle.obj: test/common/Throttle.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_throttle_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_throttle-Throttle.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_throttle-Throttle.Tpo -c -o test/common/unittest_throttle-Throttle.obj `if test -f 'test/common/Throttle.cc'; then $(CYGPATH_W) 'test/common/Throttle.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/Throttle.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_throttle-Throttle.Tpo test/common/$(DEPDIR)/unittest_throttle-Throttle.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/Throttle.cc' object='test/common/unittest_throttle-Throttle.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_throttle_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_throttle-Throttle.obj `if test -f 'test/common/Throttle.cc'; then $(CYGPATH_W) 'test/common/Throttle.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/Throttle.cc'; fi` + +test/unittest_utf8-utf8.o: test/utf8.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_utf8_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_utf8-utf8.o -MD -MP -MF test/$(DEPDIR)/unittest_utf8-utf8.Tpo -c -o test/unittest_utf8-utf8.o `test -f 'test/utf8.cc' || echo '$(srcdir)/'`test/utf8.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_utf8-utf8.Tpo test/$(DEPDIR)/unittest_utf8-utf8.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/utf8.cc' object='test/unittest_utf8-utf8.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_utf8_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_utf8-utf8.o `test -f 'test/utf8.cc' || echo '$(srcdir)/'`test/utf8.cc + +test/unittest_utf8-utf8.obj: test/utf8.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_utf8_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_utf8-utf8.obj -MD -MP -MF test/$(DEPDIR)/unittest_utf8-utf8.Tpo -c -o test/unittest_utf8-utf8.obj `if test -f 'test/utf8.cc'; then $(CYGPATH_W) 'test/utf8.cc'; else $(CYGPATH_W) '$(srcdir)/test/utf8.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_utf8-utf8.Tpo test/$(DEPDIR)/unittest_utf8-utf8.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/utf8.cc' object='test/unittest_utf8-utf8.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_utf8_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_utf8-utf8.obj `if test -f 'test/utf8.cc'; then $(CYGPATH_W) 'test/utf8.cc'; else $(CYGPATH_W) '$(srcdir)/test/utf8.cc'; fi` + +test/common/unittest_util-test_util.o: test/common/test_util.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_util_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_util-test_util.o -MD -MP -MF test/common/$(DEPDIR)/unittest_util-test_util.Tpo -c -o test/common/unittest_util-test_util.o `test -f 'test/common/test_util.cc' || echo '$(srcdir)/'`test/common/test_util.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_util-test_util.Tpo test/common/$(DEPDIR)/unittest_util-test_util.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_util.cc' object='test/common/unittest_util-test_util.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_util_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_util-test_util.o `test -f 'test/common/test_util.cc' || echo '$(srcdir)/'`test/common/test_util.cc + +test/common/unittest_util-test_util.obj: test/common/test_util.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_util_CXXFLAGS) $(CXXFLAGS) -MT test/common/unittest_util-test_util.obj -MD -MP -MF test/common/$(DEPDIR)/unittest_util-test_util.Tpo -c -o test/common/unittest_util-test_util.obj `if test -f 'test/common/test_util.cc'; then $(CYGPATH_W) 'test/common/test_util.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_util.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/common/$(DEPDIR)/unittest_util-test_util.Tpo test/common/$(DEPDIR)/unittest_util-test_util.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/common/test_util.cc' object='test/common/unittest_util-test_util.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_util_CXXFLAGS) $(CXXFLAGS) -c -o test/common/unittest_util-test_util.obj `if test -f 'test/common/test_util.cc'; then $(CYGPATH_W) 'test/common/test_util.cc'; else $(CYGPATH_W) '$(srcdir)/test/common/test_util.cc'; fi` + +test/unittest_workqueue-test_workqueue.o: test/test_workqueue.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_workqueue_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_workqueue-test_workqueue.o -MD -MP -MF test/$(DEPDIR)/unittest_workqueue-test_workqueue.Tpo -c -o test/unittest_workqueue-test_workqueue.o `test -f 'test/test_workqueue.cc' || echo '$(srcdir)/'`test/test_workqueue.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_workqueue-test_workqueue.Tpo test/$(DEPDIR)/unittest_workqueue-test_workqueue.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_workqueue.cc' object='test/unittest_workqueue-test_workqueue.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_workqueue_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_workqueue-test_workqueue.o `test -f 'test/test_workqueue.cc' || echo '$(srcdir)/'`test/test_workqueue.cc + +test/unittest_workqueue-test_workqueue.obj: test/test_workqueue.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_workqueue_CXXFLAGS) $(CXXFLAGS) -MT test/unittest_workqueue-test_workqueue.obj -MD -MP -MF test/$(DEPDIR)/unittest_workqueue-test_workqueue.Tpo -c -o test/unittest_workqueue-test_workqueue.obj `if test -f 'test/test_workqueue.cc'; then $(CYGPATH_W) 'test/test_workqueue.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_workqueue.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) test/$(DEPDIR)/unittest_workqueue-test_workqueue.Tpo test/$(DEPDIR)/unittest_workqueue-test_workqueue.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='test/test_workqueue.cc' object='test/unittest_workqueue-test_workqueue.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(unittest_workqueue_CXXFLAGS) $(CXXFLAGS) -c -o test/unittest_workqueue-test_workqueue.obj `if test -f 'test/test_workqueue.cc'; then $(CYGPATH_W) 'test/test_workqueue.cc'; else $(CYGPATH_W) '$(srcdir)/test/test_workqueue.cc'; fi` + +.cpp.o: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< + +.cpp.obj: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cpp.lo: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + -rm -rf arch/.libs arch/_libs + -rm -rf auth/.libs auth/_libs + -rm -rf auth/cephx/.libs auth/cephx/_libs + -rm -rf auth/none/.libs auth/none/_libs + -rm -rf auth/unknown/.libs auth/unknown/_libs + -rm -rf client/.libs client/_libs + -rm -rf cls/hello/.libs cls/hello/_libs + -rm -rf cls/lock/.libs cls/lock/_libs + -rm -rf cls/log/.libs cls/log/_libs + -rm -rf cls/rbd/.libs cls/rbd/_libs + -rm -rf cls/refcount/.libs cls/refcount/_libs + -rm -rf cls/replica_log/.libs cls/replica_log/_libs + -rm -rf cls/rgw/.libs cls/rgw/_libs + -rm -rf cls/statelog/.libs cls/statelog/_libs + -rm -rf cls/user/.libs cls/user/_libs + -rm -rf cls/version/.libs cls/version/_libs + -rm -rf common/.libs common/_libs + -rm -rf crush/.libs crush/_libs + -rm -rf erasure-code/.libs erasure-code/_libs + -rm -rf erasure-code/jerasure/.libs erasure-code/jerasure/_libs + -rm -rf erasure-code/jerasure/gf-complete/src/.libs erasure-code/jerasure/gf-complete/src/_libs + -rm -rf erasure-code/jerasure/jerasure/src/.libs erasure-code/jerasure/jerasure/src/_libs + -rm -rf global/.libs global/_libs + -rm -rf java/native/.libs java/native/_libs + -rm -rf json_spirit/.libs json_spirit/_libs + -rm -rf key_value_store/.libs key_value_store/_libs + -rm -rf librados/.libs librados/_libs + -rm -rf librbd/.libs librbd/_libs + -rm -rf log/.libs log/_libs + -rm -rf mds/.libs mds/_libs + -rm -rf mon/.libs mon/_libs + -rm -rf msg/.libs msg/_libs + -rm -rf objclass/.libs objclass/_libs + -rm -rf os/.libs os/_libs + -rm -rf osd/.libs osd/_libs + -rm -rf osdc/.libs osdc/_libs + -rm -rf perfglue/.libs perfglue/_libs + -rm -rf rgw/.libs rgw/_libs + -rm -rf test/erasure-code/.libs test/erasure-code/_libs + -rm -rf test/librados/.libs test/librados/_libs + -rm -rf test/system/.libs test/system/_libs +install-pythonPYTHON: $(python_PYTHON) + @$(NORMAL_INSTALL) + test -z "$(pythondir)" || $(MKDIR_P) "$(DESTDIR)$(pythondir)" + @list='$(python_PYTHON)'; dlist=; list2=; test -n "$(pythondir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then b=; else b="$(srcdir)/"; fi; \ + if test -f $$b$$p; then \ + $(am__strip_dir) \ + dlist="$$dlist $$f"; \ + list2="$$list2 $$b$$p"; \ + else :; fi; \ + done; \ + for file in $$list2; do echo $$file; done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(pythondir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(pythondir)" || exit $$?; \ + done || exit $$?; \ + if test -n "$$dlist"; then \ + $(am__py_compile) --destdir "$(DESTDIR)" \ + --basedir "$(pythondir)" $$dlist; \ + else :; fi + +uninstall-pythonPYTHON: + @$(NORMAL_UNINSTALL) + @list='$(python_PYTHON)'; test -n "$(pythondir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + test -n "$$files" || exit 0; \ + dir='$(DESTDIR)$(pythondir)'; \ + filesc=`echo "$$files" | sed 's|$$|c|'`; \ + fileso=`echo "$$files" | sed 's|$$|o|'`; \ + st=0; \ + for files in "$$files" "$$filesc" "$$fileso"; do \ + $(am__uninstall_files_from_dir) || st=$$?; \ + done; \ + exit $$st +install-bash_completionDATA: $(bash_completion_DATA) + @$(NORMAL_INSTALL) + test -z "$(bash_completiondir)" || $(MKDIR_P) "$(DESTDIR)$(bash_completiondir)" + @list='$(bash_completion_DATA)'; test -n "$(bash_completiondir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(bash_completiondir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(bash_completiondir)" || exit $$?; \ + done + +uninstall-bash_completionDATA: + @$(NORMAL_UNINSTALL) + @list='$(bash_completion_DATA)'; test -n "$(bash_completiondir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(bash_completiondir)'; $(am__uninstall_files_from_dir) +install-docDATA: $(doc_DATA) + @$(NORMAL_INSTALL) + test -z "$(docdir)" || $(MKDIR_P) "$(DESTDIR)$(docdir)" + @list='$(doc_DATA)'; test -n "$(docdir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(docdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(docdir)" || exit $$?; \ + done + +uninstall-docDATA: + @$(NORMAL_UNINSTALL) + @list='$(doc_DATA)'; test -n "$(docdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(docdir)'; $(am__uninstall_files_from_dir) +install-libcephfs_includeDATA: $(libcephfs_include_DATA) + @$(NORMAL_INSTALL) + test -z "$(libcephfs_includedir)" || $(MKDIR_P) "$(DESTDIR)$(libcephfs_includedir)" + @list='$(libcephfs_include_DATA)'; test -n "$(libcephfs_includedir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(libcephfs_includedir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(libcephfs_includedir)" || exit $$?; \ + done + +uninstall-libcephfs_includeDATA: + @$(NORMAL_UNINSTALL) + @list='$(libcephfs_include_DATA)'; test -n "$(libcephfs_includedir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(libcephfs_includedir)'; $(am__uninstall_files_from_dir) +install-librbd_includeDATA: $(librbd_include_DATA) + @$(NORMAL_INSTALL) + test -z "$(librbd_includedir)" || $(MKDIR_P) "$(DESTDIR)$(librbd_includedir)" + @list='$(librbd_include_DATA)'; test -n "$(librbd_includedir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(librbd_includedir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(librbd_includedir)" || exit $$?; \ + done + +uninstall-librbd_includeDATA: + @$(NORMAL_UNINSTALL) + @list='$(librbd_include_DATA)'; test -n "$(librbd_includedir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(librbd_includedir)'; $(am__uninstall_files_from_dir) +install-rados_includeDATA: $(rados_include_DATA) + @$(NORMAL_INSTALL) + test -z "$(rados_includedir)" || $(MKDIR_P) "$(DESTDIR)$(rados_includedir)" + @list='$(rados_include_DATA)'; test -n "$(rados_includedir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(rados_includedir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(rados_includedir)" || exit $$?; \ + done + +uninstall-rados_includeDATA: + @$(NORMAL_UNINSTALL) + @list='$(rados_include_DATA)'; test -n "$(rados_includedir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(rados_includedir)'; $(am__uninstall_files_from_dir) + +# This directory's subdirectories are mostly independent; you can cd +# into them and run `make' without going through this Makefile. +# To change the values of `make' variables: instead of editing Makefiles, +# (1) if the variable is set in `config.status', edit `config.status' +# (which will cause the Makefiles to be regenerated when you run `make'); +# (2) otherwise, pass the desired values on the `make' command line. +$(RECURSIVE_TARGETS): + @fail= failcom='exit 1'; \ + for f in x $$MAKEFLAGS; do \ + case $$f in \ + *=* | --[!k]*);; \ + *k*) failcom='fail=yes';; \ + esac; \ + done; \ + dot_seen=no; \ + target=`echo $@ | sed s/-recursive//`; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + dot_seen=yes; \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done; \ + if test "$$dot_seen" = "no"; then \ + $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \ + fi; test -z "$$fail" + +$(RECURSIVE_CLEAN_TARGETS): + @fail= failcom='exit 1'; \ + for f in x $$MAKEFLAGS; do \ + case $$f in \ + *=* | --[!k]*);; \ + *k*) failcom='fail=yes';; \ + esac; \ + done; \ + dot_seen=no; \ + case "$@" in \ + distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \ + *) list='$(SUBDIRS)' ;; \ + esac; \ + rev=''; for subdir in $$list; do \ + if test "$$subdir" = "."; then :; else \ + rev="$$subdir $$rev"; \ + fi; \ + done; \ + rev="$$rev ."; \ + target=`echo $@ | sed s/-recursive//`; \ + for subdir in $$rev; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done && test -z "$$fail" +tags-recursive: + list='$(SUBDIRS)'; for subdir in $$list; do \ + test "$$subdir" = . || ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) tags); \ + done +ctags-recursive: + list='$(SUBDIRS)'; for subdir in $$list; do \ + test "$$subdir" = . || ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) ctags); \ + done + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: tags-recursive $(HEADERS) $(SOURCES) acconfig.h.in $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \ + include_option=--etags-include; \ + empty_fix=.; \ + else \ + include_option=--include; \ + empty_fix=; \ + fi; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test ! -f $$subdir/TAGS || \ + set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \ + fi; \ + done; \ + list='$(SOURCES) $(HEADERS) acconfig.h.in $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: ctags-recursive $(HEADERS) $(SOURCES) acconfig.h.in $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) acconfig.h.in $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +check-TESTS: $(TESTS) + @failed=0; all=0; xfail=0; xpass=0; skip=0; \ + srcdir=$(srcdir); export srcdir; \ + list=' $(TESTS) '; \ + $(am__tty_colors); \ + if test -n "$$list"; then \ + for tst in $$list; do \ + if test -f ./$$tst; then dir=./; \ + elif test -f $$tst; then dir=; \ + else dir="$(srcdir)/"; fi; \ + if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xpass=`expr $$xpass + 1`; \ + failed=`expr $$failed + 1`; \ + col=$$red; res=XPASS; \ + ;; \ + *) \ + col=$$grn; res=PASS; \ + ;; \ + esac; \ + elif test $$? -ne 77; then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xfail=`expr $$xfail + 1`; \ + col=$$lgn; res=XFAIL; \ + ;; \ + *) \ + failed=`expr $$failed + 1`; \ + col=$$red; res=FAIL; \ + ;; \ + esac; \ + else \ + skip=`expr $$skip + 1`; \ + col=$$blu; res=SKIP; \ + fi; \ + echo "$${col}$$res$${std}: $$tst"; \ + done; \ + if test "$$all" -eq 1; then \ + tests="test"; \ + All=""; \ + else \ + tests="tests"; \ + All="All "; \ + fi; \ + if test "$$failed" -eq 0; then \ + if test "$$xfail" -eq 0; then \ + banner="$$All$$all $$tests passed"; \ + else \ + if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \ + banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \ + fi; \ + else \ + if test "$$xpass" -eq 0; then \ + banner="$$failed of $$all $$tests failed"; \ + else \ + if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \ + banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \ + fi; \ + fi; \ + dashes="$$banner"; \ + skipped=""; \ + if test "$$skip" -ne 0; then \ + if test "$$skip" -eq 1; then \ + skipped="($$skip test was not run)"; \ + else \ + skipped="($$skip tests were not run)"; \ + fi; \ + test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$skipped"; \ + fi; \ + report=""; \ + if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \ + report="Please report to $(PACKAGE_BUGREPORT)"; \ + test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$report"; \ + fi; \ + dashes=`echo "$$dashes" | sed s/./=/g`; \ + if test "$$failed" -eq 0; then \ + col="$$grn"; \ + else \ + col="$$red"; \ + fi; \ + echo "$${col}$$dashes$${std}"; \ + echo "$${col}$$banner$${std}"; \ + test -z "$$skipped" || echo "$${col}$$skipped$${std}"; \ + test -z "$$report" || echo "$${col}$$report$${std}"; \ + echo "$${col}$$dashes$${std}"; \ + test "$$failed" -eq 0; \ + else :; fi + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test -d "$(distdir)/$$subdir" \ + || $(MKDIR_P) "$(distdir)/$$subdir" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + dir1=$$subdir; dir2="$(distdir)/$$subdir"; \ + $(am__relativize); \ + new_distdir=$$reldir; \ + dir1=$$subdir; dir2="$(top_distdir)"; \ + $(am__relativize); \ + new_top_distdir=$$reldir; \ + echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \ + echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \ + ($(am__cd) $$subdir && \ + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$$new_top_distdir" \ + distdir="$$new_distdir" \ + am__remove_distdir=: \ + am__skip_length_check=: \ + am__skip_mode_fix=: \ + distdir) \ + || exit 1; \ + fi; \ + done + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$(top_distdir)" distdir="$(distdir)" \ + dist-hook +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) $(check_PROGRAMS) $(check_SCRIPTS) + $(MAKE) $(AM_MAKEFLAGS) check-TESTS check-local +check: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) check-recursive +all-am: Makefile $(LIBRARIES) $(LTLIBRARIES) $(PROGRAMS) $(SCRIPTS) \ + $(DATA) $(HEADERS) acconfig.h +install-binPROGRAMS: install-libLTLIBRARIES + +installdirs: installdirs-recursive +installdirs-am: + for dir in "$(DESTDIR)$(erasure_codelibdir)" "$(DESTDIR)$(libdir)" "$(DESTDIR)$(radoslibdir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(sbindir)" "$(DESTDIR)$(su_sbindir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(ceph_sbindir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(sbindir)" "$(DESTDIR)$(shell_commondir)" "$(DESTDIR)$(su_sbindir)" "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(bash_completiondir)" "$(DESTDIR)$(docdir)" "$(DESTDIR)$(libcephfs_includedir)" "$(DESTDIR)$(librbd_includedir)" "$(DESTDIR)$(rados_includedir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) install-recursive +install-exec: install-exec-recursive +install-data: install-data-recursive +uninstall: uninstall-recursive + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-recursive +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + -rm -f arch/$(DEPDIR)/$(am__dirstamp) + -rm -f arch/$(am__dirstamp) + -rm -f auth/$(DEPDIR)/$(am__dirstamp) + -rm -f auth/$(am__dirstamp) + -rm -f auth/cephx/$(DEPDIR)/$(am__dirstamp) + -rm -f auth/cephx/$(am__dirstamp) + -rm -f auth/none/$(DEPDIR)/$(am__dirstamp) + -rm -f auth/none/$(am__dirstamp) + -rm -f auth/unknown/$(DEPDIR)/$(am__dirstamp) + -rm -f auth/unknown/$(am__dirstamp) + -rm -f civetweb/src/$(DEPDIR)/$(am__dirstamp) + -rm -f civetweb/src/$(am__dirstamp) + -rm -f client/$(DEPDIR)/$(am__dirstamp) + -rm -f client/$(am__dirstamp) + -rm -f cls/hello/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/hello/$(am__dirstamp) + -rm -f cls/lock/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/lock/$(am__dirstamp) + -rm -f cls/log/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/log/$(am__dirstamp) + -rm -f cls/rbd/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/rbd/$(am__dirstamp) + -rm -f cls/refcount/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/refcount/$(am__dirstamp) + -rm -f cls/replica_log/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/replica_log/$(am__dirstamp) + -rm -f cls/rgw/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/rgw/$(am__dirstamp) + -rm -f cls/statelog/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/statelog/$(am__dirstamp) + -rm -f cls/user/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/user/$(am__dirstamp) + -rm -f cls/version/$(DEPDIR)/$(am__dirstamp) + -rm -f cls/version/$(am__dirstamp) + -rm -f common/$(DEPDIR)/$(am__dirstamp) + -rm -f common/$(am__dirstamp) + -rm -f crush/$(DEPDIR)/$(am__dirstamp) + -rm -f crush/$(am__dirstamp) + -rm -f erasure-code/$(DEPDIR)/$(am__dirstamp) + -rm -f erasure-code/$(am__dirstamp) + -rm -f erasure-code/jerasure/$(DEPDIR)/$(am__dirstamp) + -rm -f erasure-code/jerasure/$(am__dirstamp) + -rm -f erasure-code/jerasure/gf-complete/src/$(DEPDIR)/$(am__dirstamp) + -rm -f erasure-code/jerasure/gf-complete/src/$(am__dirstamp) + -rm -f erasure-code/jerasure/jerasure/src/$(DEPDIR)/$(am__dirstamp) + -rm -f erasure-code/jerasure/jerasure/src/$(am__dirstamp) + -rm -f global/$(DEPDIR)/$(am__dirstamp) + -rm -f global/$(am__dirstamp) + -rm -f java/native/$(DEPDIR)/$(am__dirstamp) + -rm -f java/native/$(am__dirstamp) + -rm -f json_spirit/$(DEPDIR)/$(am__dirstamp) + -rm -f json_spirit/$(am__dirstamp) + -rm -f key_value_store/$(DEPDIR)/$(am__dirstamp) + -rm -f key_value_store/$(am__dirstamp) + -rm -f librados/$(DEPDIR)/$(am__dirstamp) + -rm -f librados/$(am__dirstamp) + -rm -f librbd/$(DEPDIR)/$(am__dirstamp) + -rm -f librbd/$(am__dirstamp) + -rm -f log/$(DEPDIR)/$(am__dirstamp) + -rm -f log/$(am__dirstamp) + -rm -f mds/$(DEPDIR)/$(am__dirstamp) + -rm -f mds/$(am__dirstamp) + -rm -f mon/$(DEPDIR)/$(am__dirstamp) + -rm -f mon/$(am__dirstamp) + -rm -f mount/$(DEPDIR)/$(am__dirstamp) + -rm -f mount/$(am__dirstamp) + -rm -f msg/$(DEPDIR)/$(am__dirstamp) + -rm -f msg/$(am__dirstamp) + -rm -f objclass/$(DEPDIR)/$(am__dirstamp) + -rm -f objclass/$(am__dirstamp) + -rm -f os/$(DEPDIR)/$(am__dirstamp) + -rm -f os/$(am__dirstamp) + -rm -f osd/$(DEPDIR)/$(am__dirstamp) + -rm -f osd/$(am__dirstamp) + -rm -f osdc/$(DEPDIR)/$(am__dirstamp) + -rm -f osdc/$(am__dirstamp) + -rm -f perfglue/$(DEPDIR)/$(am__dirstamp) + -rm -f perfglue/$(am__dirstamp) + -rm -f rbd_fuse/$(DEPDIR)/$(am__dirstamp) + -rm -f rbd_fuse/$(am__dirstamp) + -rm -f rgw/$(DEPDIR)/$(am__dirstamp) + -rm -f rgw/$(am__dirstamp) + -rm -f test/$(DEPDIR)/$(am__dirstamp) + -rm -f test/$(am__dirstamp) + -rm -f test/ObjectMap/$(DEPDIR)/$(am__dirstamp) + -rm -f test/ObjectMap/$(am__dirstamp) + -rm -f test/bench/$(DEPDIR)/$(am__dirstamp) + -rm -f test/bench/$(am__dirstamp) + -rm -f test/cls_hello/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_hello/$(am__dirstamp) + -rm -f test/cls_lock/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_lock/$(am__dirstamp) + -rm -f test/cls_log/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_log/$(am__dirstamp) + -rm -f test/cls_rbd/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_rbd/$(am__dirstamp) + -rm -f test/cls_refcount/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_refcount/$(am__dirstamp) + -rm -f test/cls_replica_log/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_replica_log/$(am__dirstamp) + -rm -f test/cls_rgw/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_rgw/$(am__dirstamp) + -rm -f test/cls_statelog/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_statelog/$(am__dirstamp) + -rm -f test/cls_version/$(DEPDIR)/$(am__dirstamp) + -rm -f test/cls_version/$(am__dirstamp) + -rm -f test/common/$(DEPDIR)/$(am__dirstamp) + -rm -f test/common/$(am__dirstamp) + -rm -f test/crush/$(DEPDIR)/$(am__dirstamp) + -rm -f test/crush/$(am__dirstamp) + -rm -f test/encoding/$(DEPDIR)/$(am__dirstamp) + -rm -f test/encoding/$(am__dirstamp) + -rm -f test/erasure-code/$(DEPDIR)/$(am__dirstamp) + -rm -f test/erasure-code/$(am__dirstamp) + -rm -f test/libcephfs/$(DEPDIR)/$(am__dirstamp) + -rm -f test/libcephfs/$(am__dirstamp) + -rm -f test/librados/$(DEPDIR)/$(am__dirstamp) + -rm -f test/librados/$(am__dirstamp) + -rm -f test/librbd/$(DEPDIR)/$(am__dirstamp) + -rm -f test/librbd/$(am__dirstamp) + -rm -f test/mon/$(DEPDIR)/$(am__dirstamp) + -rm -f test/mon/$(am__dirstamp) + -rm -f test/objectstore/$(DEPDIR)/$(am__dirstamp) + -rm -f test/objectstore/$(am__dirstamp) + -rm -f test/os/$(DEPDIR)/$(am__dirstamp) + -rm -f test/os/$(am__dirstamp) + -rm -f test/osd/$(DEPDIR)/$(am__dirstamp) + -rm -f test/osd/$(am__dirstamp) + -rm -f test/osdc/$(DEPDIR)/$(am__dirstamp) + -rm -f test/osdc/$(am__dirstamp) + -rm -f test/rgw/$(DEPDIR)/$(am__dirstamp) + -rm -f test/rgw/$(am__dirstamp) + -rm -f test/system/$(DEPDIR)/$(am__dirstamp) + -rm -f test/system/$(am__dirstamp) + -rm -f tools/$(DEPDIR)/$(am__dirstamp) + -rm -f tools/$(am__dirstamp) + -rm -f tools/rados/$(DEPDIR)/$(am__dirstamp) + -rm -f tools/rados/$(am__dirstamp) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." + -test -z "$(BUILT_SOURCES)" || rm -f $(BUILT_SOURCES) +clean: clean-recursive + +clean-am: clean-binPROGRAMS clean-checkPROGRAMS \ + clean-erasure_codelibLTLIBRARIES clean-generic \ + clean-libLTLIBRARIES clean-libtool clean-local \ + clean-noinstLIBRARIES clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS clean-radoslibLTLIBRARIES \ + clean-sbinPROGRAMS clean-su_sbinPROGRAMS mostlyclean-am + +distclean: distclean-recursive + -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) erasure-code/$(DEPDIR) erasure-code/jerasure/$(DEPDIR) erasure-code/jerasure/gf-complete/src/$(DEPDIR) erasure-code/jerasure/jerasure/src/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$(DEPDIR) librbd/$(DEPDIR) log/$(DEPDIR) mds/$(DEPDIR) mon/$(DEPDIR) mount/$(DEPDIR) msg/$(DEPDIR) objclass/$(DEPDIR) os/$(DEPDIR) osd/$(DEPDIR) osdc/$(DEPDIR) perfglue/$(DEPDIR) rbd_fuse/$(DEPDIR) rgw/$(DEPDIR) test/$(DEPDIR) test/ObjectMap/$(DEPDIR) test/bench/$(DEPDIR) test/cls_hello/$(DEPDIR) test/cls_lock/$(DEPDIR) test/cls_log/$(DEPDIR) test/cls_rbd/$(DEPDIR) test/cls_refcount/$(DEPDIR) test/cls_replica_log/$(DEPDIR) test/cls_rgw/$(DEPDIR) test/cls_statelog/$(DEPDIR) test/cls_version/$(DEPDIR) test/common/$(DEPDIR) test/crush/$(DEPDIR) test/encoding/$(DEPDIR) test/erasure-code/$(DEPDIR) test/libcephfs/$(DEPDIR) test/librados/$(DEPDIR) test/librbd/$(DEPDIR) test/mon/$(DEPDIR) test/objectstore/$(DEPDIR) test/os/$(DEPDIR) test/osd/$(DEPDIR) test/osdc/$(DEPDIR) test/rgw/$(DEPDIR) test/system/$(DEPDIR) tools/$(DEPDIR) tools/rados/$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-hdr distclean-tags + +dvi: dvi-recursive + +dvi-am: + +html: html-recursive + +html-am: + +info: info-recursive + +info-am: + +install-data-am: install-bash_completionDATA install-ceph_sbinSCRIPTS \ + install-data-local install-docDATA \ + install-erasure_codelibLTLIBRARIES \ + install-libcephfs_includeDATA install-librbd_includeDATA \ + install-pythonPYTHON install-rados_includeDATA \ + install-radoslibLTLIBRARIES install-shell_commonSCRIPTS \ + install-su_sbinPROGRAMS install-su_sbinSCRIPTS + +install-dvi: install-dvi-recursive + +install-dvi-am: + +install-exec-am: install-binPROGRAMS install-binSCRIPTS \ + install-dist_binSCRIPTS install-libLTLIBRARIES \ + install-sbinPROGRAMS install-sbinSCRIPTS + +install-html: install-html-recursive + +install-html-am: + +install-info: install-info-recursive + +install-info-am: + +install-man: + +install-pdf: install-pdf-recursive + +install-pdf-am: + +install-ps: install-ps-recursive + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-recursive + -rm -rf ./$(DEPDIR) arch/$(DEPDIR) auth/$(DEPDIR) auth/cephx/$(DEPDIR) auth/none/$(DEPDIR) auth/unknown/$(DEPDIR) civetweb/src/$(DEPDIR) client/$(DEPDIR) cls/hello/$(DEPDIR) cls/lock/$(DEPDIR) cls/log/$(DEPDIR) cls/rbd/$(DEPDIR) cls/refcount/$(DEPDIR) cls/replica_log/$(DEPDIR) cls/rgw/$(DEPDIR) cls/statelog/$(DEPDIR) cls/user/$(DEPDIR) cls/version/$(DEPDIR) common/$(DEPDIR) crush/$(DEPDIR) erasure-code/$(DEPDIR) erasure-code/jerasure/$(DEPDIR) erasure-code/jerasure/gf-complete/src/$(DEPDIR) erasure-code/jerasure/jerasure/src/$(DEPDIR) global/$(DEPDIR) java/native/$(DEPDIR) json_spirit/$(DEPDIR) key_value_store/$(DEPDIR) librados/$(DEPDIR) librbd/$(DEPDIR) log/$(DEPDIR) mds/$(DEPDIR) mon/$(DEPDIR) mount/$(DEPDIR) msg/$(DEPDIR) objclass/$(DEPDIR) os/$(DEPDIR) osd/$(DEPDIR) osdc/$(DEPDIR) perfglue/$(DEPDIR) rbd_fuse/$(DEPDIR) rgw/$(DEPDIR) test/$(DEPDIR) test/ObjectMap/$(DEPDIR) test/bench/$(DEPDIR) test/cls_hello/$(DEPDIR) test/cls_lock/$(DEPDIR) test/cls_log/$(DEPDIR) test/cls_rbd/$(DEPDIR) test/cls_refcount/$(DEPDIR) test/cls_replica_log/$(DEPDIR) test/cls_rgw/$(DEPDIR) test/cls_statelog/$(DEPDIR) test/cls_version/$(DEPDIR) test/common/$(DEPDIR) test/crush/$(DEPDIR) test/encoding/$(DEPDIR) test/erasure-code/$(DEPDIR) test/libcephfs/$(DEPDIR) test/librados/$(DEPDIR) test/librbd/$(DEPDIR) test/mon/$(DEPDIR) test/objectstore/$(DEPDIR) test/os/$(DEPDIR) test/osd/$(DEPDIR) test/osdc/$(DEPDIR) test/rgw/$(DEPDIR) test/system/$(DEPDIR) tools/$(DEPDIR) tools/rados/$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-recursive + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-recursive + +pdf-am: + +ps: ps-recursive + +ps-am: + +uninstall-am: uninstall-bash_completionDATA uninstall-binPROGRAMS \ + uninstall-binSCRIPTS uninstall-ceph_sbinSCRIPTS \ + uninstall-dist_binSCRIPTS uninstall-docDATA \ + uninstall-erasure_codelibLTLIBRARIES uninstall-libLTLIBRARIES \ + uninstall-libcephfs_includeDATA uninstall-librbd_includeDATA \ + uninstall-local uninstall-pythonPYTHON \ + uninstall-rados_includeDATA uninstall-radoslibLTLIBRARIES \ + uninstall-sbinPROGRAMS uninstall-sbinSCRIPTS \ + uninstall-shell_commonSCRIPTS uninstall-su_sbinPROGRAMS \ + uninstall-su_sbinSCRIPTS + +.MAKE: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) all check \ + check-am ctags-recursive install install-am install-strip \ + tags-recursive + +.PHONY: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) CTAGS GTAGS \ + all all-am check check-TESTS check-am check-local clean \ + clean-binPROGRAMS clean-checkPROGRAMS \ + clean-erasure_codelibLTLIBRARIES clean-generic \ + clean-libLTLIBRARIES clean-libtool clean-local \ + clean-noinstLIBRARIES clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS clean-radoslibLTLIBRARIES \ + clean-sbinPROGRAMS clean-su_sbinPROGRAMS ctags ctags-recursive \ + dist-hook distclean distclean-compile distclean-generic \ + distclean-hdr distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-bash_completionDATA install-binPROGRAMS \ + install-binSCRIPTS install-ceph_sbinSCRIPTS install-data \ + install-data-am install-data-local install-dist_binSCRIPTS \ + install-docDATA install-dvi install-dvi-am \ + install-erasure_codelibLTLIBRARIES install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-libLTLIBRARIES \ + install-libcephfs_includeDATA install-librbd_includeDATA \ + install-man install-pdf install-pdf-am install-ps \ + install-ps-am install-pythonPYTHON install-rados_includeDATA \ + install-radoslibLTLIBRARIES install-sbinPROGRAMS \ + install-sbinSCRIPTS install-shell_commonSCRIPTS install-strip \ + install-su_sbinPROGRAMS install-su_sbinSCRIPTS installcheck \ + installcheck-am installdirs installdirs-am maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags tags-recursive uninstall uninstall-am \ + uninstall-bash_completionDATA uninstall-binPROGRAMS \ + uninstall-binSCRIPTS uninstall-ceph_sbinSCRIPTS \ + uninstall-dist_binSCRIPTS uninstall-docDATA \ + uninstall-erasure_codelibLTLIBRARIES uninstall-libLTLIBRARIES \ + uninstall-libcephfs_includeDATA uninstall-librbd_includeDATA \ + uninstall-local uninstall-pythonPYTHON \ + uninstall-rados_includeDATA uninstall-radoslibLTLIBRARIES \ + uninstall-sbinPROGRAMS uninstall-sbinSCRIPTS \ + uninstall-shell_commonSCRIPTS uninstall-su_sbinPROGRAMS \ + uninstall-su_sbinSCRIPTS + + +# python unit tests need to know where the scripts are located +export PYTHONPATH=$(top_srcdir)/src/pybind +@CLANG_FALSE@ AM_COMMON_CFLAGS += -rdynamic +@CLANG_FALSE@ AM_CXXFLAGS += -Wstrict-null-sentinel + +#crush_includedir = $(includedir)/crush +#crush_include_DATA = \ +# $(srcdir)/crush/hash.h \ +# $(srcdir)/crush/crush.h \ +# $(srcdir)/crush/mapper.h \ +# $(srcdir)/crush/types.h + +# target to build but not run the unit tests +unittests:: $(check_PROGRAMS) +$(shell_scripts): Makefile +$(shell_scripts): %: %.in + rm -f $@ $@.tmp + $(editpaths) '$(srcdir)/$@.in' >$@.tmp + chmod +x $@.tmp + chmod a-w $@.tmp + mv $@.tmp $@ + +# work around old versions of automake that don't define $docdir +# NOTE: this won't work on suse, where docdir is /usr/share/doc/packages/$package. +docdir ?= ${datadir}/doc/ceph + +check-local: + $(srcdir)/test/encoding/readable.sh ../ceph-object-corpus + +# base targets + +core-daemons: ceph-mon ceph-osd ceph-mds radosgw +admin-tools: monmaptool osdmaptool crushtool ceph-authtool +base: core-daemons admin-tools \ + cephfs ceph-syn ceph-conf \ + rados librados-config \ + init-ceph mkcephfs ceph_mon_store_converter ceph-post-file + +# version stuff + +FORCE: +.git_version: FORCE + $(srcdir)/check_version $(srcdir)/.git_version + +# if NO_VERSION is set, only generate a new ceph_ver.h if there currently +# is none, and call "make_version -n" to fill it with a fixed string. +# Otherwise, set it from the contents of .git_version. + +ceph_ver.h: .git_version + if [ -n "$$NO_VERSION" ] ; then \ + if [ ! -f ./ceph_ver.h ] ; then \ + $(srcdir)/make_version -n ./ceph_ver.h ; \ + fi; \ + else \ + $(srcdir)/make_version $(srcdir)/.git_version ./ceph_ver.h ; \ + fi + +ceph_ver.c: ./ceph_ver.h +common/version.cc: ./ceph_ver.h +test/encoding/ceph_dencoder.cc: ./ceph_ver.h + +sample.fetch_config: fetch_config + cp -f $(srcdir)/fetch_config ./sample.fetch_config + +dist-hook: + $(srcdir)/check_version $(srcdir)/.git_version + +# assemble Python script with global version variables +# NB: depends on format of ceph_ver.h + +ceph: ceph.in ./ceph_ver.h Makefile + rm -f $@ $@.tmp + echo "#!/usr/bin/env python" >$@.tmp + grep "#define CEPH_GIT_NICE_VER" ./ceph_ver.h | \ + sed -e 's/#define \(.*VER\) /\1=/' >>$@.tmp + grep "#define CEPH_GIT_VER" ./ceph_ver.h | \ + sed -e 's/#define \(.*VER\) /\1=/' -e 's/=\(.*\)$$/="\1"/' >>$@.tmp + cat $(srcdir)/$@.in >>$@.tmp + chmod a+x $@.tmp + chmod a-w $@.tmp + mv $@.tmp $@ + +# cleaning + +clean-local: + rm -f *.so + find . -name '*.gcno' -o -name '*.gcda' -o -name '*.lcov' | xargs rm -f + rm -f ceph java/java/com/ceph/crush/Bucket.class + +install-coverage: +@ENABLE_COVERAGE_TRUE@ -mkdir -p $(COV_DIR)/.libs +@ENABLE_COVERAGE_TRUE@ -$(INSTALL_DATA) $(COV_FILES) $(COV_DIR) +@ENABLE_COVERAGE_TRUE@ -$(INSTALL_DATA) $(COV_LIB_FILES) $(COV_DIR)/.libs + +uninstall-coverage: +@ENABLE_COVERAGE_TRUE@ -rm $(COV_DIR)/*.gcno +@ENABLE_COVERAGE_TRUE@ -rm $(COV_DIR)/.libs/*.gcno +@ENABLE_COVERAGE_TRUE@ -rmdir -p $(COV_DIR)/.libs +@ENABLE_COVERAGE_TRUE@ -rmdir -p $(COV_DIR) + +check-coverage: +@ENABLE_COVERAGE_TRUE@ -test/coverage.sh -d $(srcdir) -o check-coverage make check + +install-data-local: install-coverage + -mkdir -p $(DESTDIR)$(sysconfdir)/ceph + -mkdir -p $(DESTDIR)$(localstatedir)/log/ceph + -mkdir -p $(DESTDIR)$(localstatedir)/lib/ceph/tmp + +uninstall-local: uninstall-coverage + -rmdir -p $(DESTDIR)$(sysconfdir)/ceph/ + -rmdir -p $(DESTDIR)$(localstatedir)/log/ceph + -rmdir -p $(DESTDIR)$(localstatedir)/lib/ceph/tmp + +# +# coverity rules expect: +# - cov-build to be in the path +# - password in ~/coverity.build.pass.txt +# - ability to scp into the ceph.com directory +# +project.tgz: clean + rm -rf cov-int + cov-build --dir cov-int make + echo Sage Weil sage@newdream.net ceph >> README + tar czvf project.tgz README cov-int + rm -f README + +coverity-submit: + scp project.tgz ceph.com:/home/ceph_site/ceph.com/coverity/`git describe`.tgz + curl --data "project=ceph&password=`cat ~/coverity.build.pass.txt`&email=sage@newdream.net&url=http://ceph.com/coverity/`git describe`.tgz" http://scan5.coverity.com/cgi-bin/submit_build.py + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/ceph/src/README b/ceph/src/README new file mode 100644 index 00000000..a0b4e24d --- /dev/null +++ b/ceph/src/README @@ -0,0 +1,2 @@ +Sage Weil +Ceph - scalable distributed storage system diff --git a/ceph/src/TODO b/ceph/src/TODO new file mode 100644 index 00000000..b3da2072 --- /dev/null +++ b/ceph/src/TODO @@ -0,0 +1,2 @@ + +See http://tracker.newdream.net/projects/ceph/roadmap \ No newline at end of file diff --git a/ceph/src/acconfig.h.in b/ceph/src/acconfig.h.in new file mode 100644 index 00000000..bed7a051 --- /dev/null +++ b/ceph/src/acconfig.h.in @@ -0,0 +1,383 @@ +/* src/acconfig.h.in. Generated from configure.ac by autoheader. */ + +/* fallocate(2) is supported */ +#undef CEPH_HAVE_FALLOCATE + +/* F_SETPIPE_SZ is supported */ +#undef CEPH_HAVE_SETPIPE_SZ + +/* splice(2) is supported */ +#undef CEPH_HAVE_SPLICE + +/* Define if darwin/osx */ +#undef DARWIN + +/* Define if you want C_Gather debugging */ +#undef DEBUG_GATHER + +/* Define if enabling coverage. */ +#undef ENABLE_COVERAGE + +/* FastCGI headers are in /usr/include/fastcgi */ +#undef FASTCGI_INCLUDE_DIR + +/* Define to 1 if you have the header file. */ +#undef HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_ARPA_NAMESER_COMPAT_H + +/* have boost::random::discrete_distribution */ +#undef HAVE_BOOST_RANDOM_DISCRETE_DISTRIBUTION + +/* Define if have curl_multi_wait() */ +#undef HAVE_CURL_MULTI_WAIT + +/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you + don't. */ +#undef HAVE_DECL_STRERROR_R + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#undef HAVE_DIRENT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_DLFCN_H + +/* Define to 1 if you have fdatasync. */ +#undef HAVE_FDATASYNC + +/* linux/fiemap.h was found, fiemap ioctl will be used */ +#undef HAVE_FIEMAP_H + +/* Define if the C complier supports __func__ */ +#undef HAVE_FUNC + +/* Define to 1 if you have the `fuse_getgroups' function. */ +#undef HAVE_FUSE_GETGROUPS + +/* we have a recent yasm and are x86_64 */ +#undef HAVE_GOOD_YASM_ELF64 + +/* Define to 1 if the system has the type `int16_t'. */ +#undef HAVE_INT16_T + +/* Define to 1 if the system has the type `int32_t'. */ +#undef HAVE_INT32_T + +/* Define to 1 if the system has the type `int64_t'. */ +#undef HAVE_INT64_T + +/* Define to 1 if the system has the type `int8_t'. */ +#undef HAVE_INT8_T + +/* Define to 1 if you have the header file. */ +#undef HAVE_INTTYPES_H + +/* Defined if LevelDB supports bloom filters */ +#undef HAVE_LEVELDB_FILTER_POLICY + +/* Defined if you don't have atomic_ops */ +#undef HAVE_LIBAIO + +/* Define to 1 if you have the `boost_program_options' library + (-lboost_program_options). */ +#undef HAVE_LIBBOOST_PROGRAM_OPTIONS + +/* Define to 1 if you have the `boost_program_options-mt' library + (-lboost_program_options-mt). */ +#undef HAVE_LIBBOOST_PROGRAM_OPTIONS_MT + +/* Define to 1 if you have the `boost_system' library (-lboost_system). */ +#undef HAVE_LIBBOOST_SYSTEM + +/* Define to 1 if you have the `boost_system-mt' library (-lboost_system-mt). + */ +#undef HAVE_LIBBOOST_SYSTEM_MT + +/* Define to 1 if you have the `boost_thread' library (-lboost_thread). */ +#undef HAVE_LIBBOOST_THREAD + +/* Define to 1 if you have the `boost_thread-mt' library (-lboost_thread-mt). + */ +#undef HAVE_LIBBOOST_THREAD_MT + +/* Define if you have fuse */ +#undef HAVE_LIBFUSE + +/* Define to 1 if you have the `profiler' library (-lprofiler). */ +#undef HAVE_LIBPROFILER + +/* Define if you have tcmalloc */ +#undef HAVE_LIBTCMALLOC + +/* Define to 1 if you have libxfs */ +#undef HAVE_LIBXFS + +/* Defined if you have libzfs enabled */ +#undef HAVE_LIBZFS + +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_TYPES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_VERSION_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_MEMORY_H + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +#undef HAVE_NDIR_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NETDB_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NETINET_IN_H + +/* Support (PCLMUL) Carry-Free Muliplication */ +#undef HAVE_PCLMUL + +/* Define to 1 if you have the `pipe2' function. */ +#undef HAVE_PIPE2 + +/* Define to 1 if you have the `posix_fadvise' function. */ +#undef HAVE_POSIX_FADVISE + +/* Define to 1 if you have the `posix_fallocate' function. */ +#undef HAVE_POSIX_FALLOCATE + +/* Define to 1 if you have the `prctl' function. */ +#undef HAVE_PRCTL + +/* Define if the C complier supports __PRETTY_FUNCTION__ */ +#undef HAVE_PRETTY_FUNC + +/* Define if you have perftools profiler enabled */ +#undef HAVE_PROFILER + +/* Define if you have POSIX threads libraries and header files. */ +#undef HAVE_PTHREAD + +/* Define if you have pthread_spin_init */ +#undef HAVE_PTHREAD_SPINLOCK + +/* Support SSE (Streaming SIMD Extensions) instructions */ +#undef HAVE_SSE + +/* Support SSE2 (Streaming SIMD Extensions 2) instructions */ +#undef HAVE_SSE2 + +/* Support SSE3 (Streaming SIMD Extensions 3) instructions */ +#undef HAVE_SSE3 + +/* Support SSE4.1 (Streaming SIMD Extensions 4.1) instructions */ +#undef HAVE_SSE4_1 + +/* Support SSE4.2 (Streaming SIMD Extensions 4.2) instructions */ +#undef HAVE_SSE4_2 + +/* Support SSSE3 (Supplemental Streaming SIMD Extensions 3) instructions */ +#undef HAVE_SSSE3 + +/* define if the compiler supports static_cast<> */ +#undef HAVE_STATIC_CAST + +/* Define if you have struct stat.st_mtimespec.tv_nsec */ +#undef HAVE_STAT_ST_MTIMESPEC_TV_NSEC + +/* Define if you have struct stat.st_mtim.tv_nsec */ +#undef HAVE_STAT_ST_MTIM_TV_NSEC + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDLIB_H + +/* Define to 1 if you have the `strerror_r' function. */ +#undef HAVE_STRERROR_R + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRING_H + +/* Define to 1 if you have the `syncfs' function. */ +#undef HAVE_SYNCFS + +/* sync_file_range(2) is supported */ +#undef HAVE_SYNC_FILE_RANGE + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYSLOG_H + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#undef HAVE_SYS_DIR_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_FILE_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_MOUNT_H + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#undef HAVE_SYS_NDIR_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_PRCTL_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_STATVFS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_STAT_H + +/* we have syncfs */ +#undef HAVE_SYS_SYNCFS + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_VFS_H + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#undef HAVE_SYS_WAIT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_XATTR_H + +/* Define to 1 if the system has the type `uint16_t'. */ +#undef HAVE_UINT16_T + +/* Define to 1 if the system has the type `uint32_t'. */ +#undef HAVE_UINT32_T + +/* Define to 1 if the system has the type `uint64_t'. */ +#undef HAVE_UINT64_T + +/* Define to 1 if the system has the type `uint8_t'. */ +#undef HAVE_UINT8_T + +/* Define to 1 if you have the header file. */ +#undef HAVE_UNISTD_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_UTIME_H + +/* Define to 1 if the system has the type `__be16'. */ +#undef HAVE___BE16 + +/* Define to 1 if the system has the type `__be32'. */ +#undef HAVE___BE32 + +/* Define to 1 if the system has the type `__be64'. */ +#undef HAVE___BE64 + +/* Define to 1 if the system has the type `__le16'. */ +#undef HAVE___LE16 + +/* Define to 1 if the system has the type `__le32'. */ +#undef HAVE___LE32 + +/* Define to 1 if the system has the type `__le64'. */ +#undef HAVE___LE64 + +/* Define to 1 if the system has the type `__s16'. */ +#undef HAVE___S16 + +/* Define to 1 if the system has the type `__s32'. */ +#undef HAVE___S32 + +/* Define to 1 if the system has the type `__s64'. */ +#undef HAVE___S64 + +/* Define to 1 if the system has the type `__s8'. */ +#undef HAVE___S8 + +/* Define to 1 if the system has the type `__u16'. */ +#undef HAVE___U16 + +/* Define to 1 if the system has the type `__u32'. */ +#undef HAVE___U32 + +/* Define to 1 if the system has the type `__u64'. */ +#undef HAVE___U64 + +/* Define to 1 if the system has the type `__u8'. */ +#undef HAVE___U8 + +/* Define to the sub-directory in which libtool stores uninstalled libraries. + */ +#undef LT_OBJDIR + +/* Defined if you do not have atomic_ops */ +#undef NO_ATOMIC_OPS + +/* Define to 1 if your C compiler doesn't accept -c and -o together. */ +#undef NO_MINUS_C_MINUS_O + +/* Name of package */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the home page for this package. */ +#undef PACKAGE_URL + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* Defined if you want pg ref debugging */ +#undef PG_DEBUG_REFS + +/* Define to necessary symbol if this constant uses a non-standard name on + your system. */ +#undef PTHREAD_CREATE_JOINABLE + +/* The size of `AO_t', as computed by sizeof. */ +#undef SIZEOF_AO_T + +/* Define to 1 if you have the ANSI C header files. */ +#undef STDC_HEADERS + +/* Define to 1 if strerror_r returns char *. */ +#undef STRERROR_R_CHAR_P + +/* Define if using CryptoPP. */ +#undef USE_CRYPTOPP + +/* Define if using NSS. */ +#undef USE_NSS + +/* Version number of package */ +#undef VERSION + +/* define if radosgw enabled */ +#undef WITH_RADOSGW diff --git a/ceph/src/arch/Makefile.am b/ceph/src/arch/Makefile.am new file mode 100644 index 00000000..27342078 --- /dev/null +++ b/ceph/src/arch/Makefile.am @@ -0,0 +1,11 @@ +libarch_la_SOURCES = \ + arch/intel.c \ + arch/neon.c \ + arch/probe.cc + +noinst_LTLIBRARIES += libarch.la + +noinst_HEADERS += \ + arch/intel.h \ + arch/neon.h \ + arch/probe.h diff --git a/ceph/src/arch/intel.c b/ceph/src/arch/intel.c new file mode 100644 index 00000000..e487da4f --- /dev/null +++ b/ceph/src/arch/intel.c @@ -0,0 +1,90 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013,2014 Inktank Storage, Inc. + * Copyright (C) 2014 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ +#include +#include "arch/probe.h" + +/* flags we export */ +int ceph_arch_intel_pclmul = 0; +int ceph_arch_intel_sse42 = 0; +int ceph_arch_intel_sse41 = 0; +int ceph_arch_intel_ssse3 = 0; +int ceph_arch_intel_sse3 = 0; +int ceph_arch_intel_sse2 = 0; + +#ifdef __x86_64__ + +/* Note: valgrind redefines cpuid : it is different from the native processor. */ +/* intel cpu? */ +static void do_cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx, + unsigned int *edx) +{ + int id = *eax; + + asm("movl %4, %%eax;" + "cpuid;" + "movl %%eax, %0;" + "movl %%ebx, %1;" + "movl %%ecx, %2;" + "movl %%edx, %3;" + : "=r" (*eax), "=r" (*ebx), "=r" (*ecx), "=r" (*edx) + : "r" (id) + : "eax", "ebx", "ecx", "edx"); +} + +/* http://en.wikipedia.org/wiki/CPUID#EAX.3D1:_Processor_Info_and_Feature_Bits */ + +#define CPUID_PCLMUL (1 << 1) +#define CPUID_SSE42 (1 << 20) +#define CPUID_SSE41 (1 << 19) +#define CPUID_SSSE3 (1 << 9) +#define CPUID_SSE3 (1) +#define CPUID_SSE2 (1 << 26) + +int ceph_arch_intel_probe(void) +{ + /* i know how to check this on x86_64... */ + unsigned int eax = 1, ebx, ecx, edx; + do_cpuid(&eax, &ebx, &ecx, &edx); + if ((ecx & CPUID_PCLMUL) != 0) { + ceph_arch_intel_pclmul = 1; + } + if ((ecx & CPUID_SSE42) != 0) { + ceph_arch_intel_sse42 = 1; + } + if ((ecx & CPUID_SSE41) != 0) { + ceph_arch_intel_sse41 = 1; + } + if ((ecx & CPUID_SSSE3) != 0) { + ceph_arch_intel_ssse3 = 1; + } + if ((ecx & CPUID_SSE3) != 0) { + ceph_arch_intel_sse3 = 1; + } + if ((edx & CPUID_SSE2) != 0) { + ceph_arch_intel_sse2 = 1; + } + + return 0; +} + +#else // __x86_64__ + +int ceph_arch_intel_probe(void) +{ + /* no features */ + return 0; +} + +#endif // __x86_64__ diff --git a/ceph/src/arch/intel.h b/ceph/src/arch/intel.h new file mode 100644 index 00000000..2c3b8099 --- /dev/null +++ b/ceph/src/arch/intel.h @@ -0,0 +1,20 @@ +#ifndef CEPH_ARCH_INTEL_H +#define CEPH_ARCH_INTEL_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern int ceph_arch_intel_pclmul; /* true if we have PCLMUL features */ +extern int ceph_arch_intel_sse42; /* true if we have sse 4.2 features */ +extern int ceph_arch_intel_sse41; /* true if we have sse 4.1 features */ +extern int ceph_arch_intel_ssse3; /* true if we have ssse 3 features */ +extern int ceph_arch_intel_sse3; /* true if we have sse 3 features */ +extern int ceph_arch_intel_sse2; /* true if we have sse 2 features */ +extern int ceph_arch_intel_probe(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ceph/src/arch/neon.c b/ceph/src/arch/neon.c new file mode 100644 index 00000000..32c1f621 --- /dev/null +++ b/ceph/src/arch/neon.c @@ -0,0 +1,51 @@ +#include "arch/probe.h" + +/* flags we export */ +int ceph_arch_neon = 0; + +#include + +#if __linux__ + +#include +#include // ElfW macro + +#if __arm__ +#include +#endif // __arm__ + +static unsigned long get_auxval(unsigned long type) +{ + unsigned long result = 0; + FILE *f = fopen("/proc/self/auxv", "r"); + if (f) { + ElfW(auxv_t) entry; + while (fread(&entry, sizeof(entry), 1, f)) { + if (entry.a_type == type) { + result = entry.a_un.a_val; + break; + } + } + fclose(f); + } + return result; +} + +static unsigned long get_hwcap(void) +{ + return get_auxval(AT_HWCAP); +} + +#endif // __linux__ + +int ceph_arch_neon_probe(void) +{ +#if __arm__ && __linux__ + ceph_arch_neon = (get_hwcap() & HWCAP_NEON) == HWCAP_NEON; +#else + if (0) + get_hwcap(); // make compiler shut up +#endif + return 0; +} + diff --git a/ceph/src/arch/neon.h b/ceph/src/arch/neon.h new file mode 100644 index 00000000..0c8aacf5 --- /dev/null +++ b/ceph/src/arch/neon.h @@ -0,0 +1,16 @@ +#ifndef CEPH_ARCH_NEON_H +#define CEPH_ARCH_NEON_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern int ceph_arch_neon; /* true if we have ARM NEON abilities */ + +extern int ceph_arch_neon_probe(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ceph/src/arch/probe.cc b/ceph/src/arch/probe.cc new file mode 100644 index 00000000..8648e54d --- /dev/null +++ b/ceph/src/arch/probe.cc @@ -0,0 +1,22 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "arch/probe.h" + +#include "arch/intel.h" +#include "arch/neon.h" + +int ceph_arch_probe(void) +{ + if (ceph_arch_probed) + return 1; + + ceph_arch_intel_probe(); + ceph_arch_neon_probe(); + + ceph_arch_probed = 1; + return 1; +} + +// do this once using the magic of c++. +int ceph_arch_probed = ceph_arch_probe(); diff --git a/ceph/src/arch/probe.h b/ceph/src/arch/probe.h new file mode 100644 index 00000000..a789c4e8 --- /dev/null +++ b/ceph/src/arch/probe.h @@ -0,0 +1,16 @@ +#ifndef CEPH_ARCH_PROBE_H +#define CEPH_ARCH_PROBE_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern int ceph_arch_probed; /* non-zero if we've probed features */ + +extern int ceph_arch_probe(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ceph/src/auth/Auth.h b/ceph/src/auth/Auth.h new file mode 100644 index 00000000..0adc9009 --- /dev/null +++ b/ceph/src/auth/Auth.h @@ -0,0 +1,247 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHTYPES_H +#define CEPH_AUTHTYPES_H + +#include "Crypto.h" +#include "msg/msg_types.h" + +#include "common/config.h" +#include "common/entity_name.h" + +class Cond; + +struct EntityAuth { + uint64_t auid; + CryptoKey key; + map caps; + + EntityAuth() : auid(CEPH_AUTH_UID_DEFAULT) {} + + void encode(bufferlist& bl) const { + __u8 struct_v = 2; + ::encode(struct_v, bl); + ::encode(auid, bl); + ::encode(key, bl); + ::encode(caps, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + if (struct_v >= 2) + ::decode(auid, bl); + else auid = CEPH_AUTH_UID_DEFAULT; + ::decode(key, bl); + ::decode(caps, bl); + } +}; +WRITE_CLASS_ENCODER(EntityAuth) + +static inline ostream& operator<<(ostream& out, const EntityAuth& a) { + return out << "auth(auid = " << a.auid << " key=" << a.key << " with " << a.caps.size() << " caps)"; +} + +struct AuthCapsInfo { + bool allow_all; + bufferlist caps; + + AuthCapsInfo() : allow_all(false) {} + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + __u8 a = (__u8)allow_all; + ::encode(a, bl); + ::encode(caps, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + __u8 a; + ::decode(a, bl); + allow_all = (bool)a; + ::decode(caps, bl); + } +}; +WRITE_CLASS_ENCODER(AuthCapsInfo) + +/* + * The ticket (if properly validated) authorizes the principal use + * services as described by 'caps' during the specified validity + * period. + */ +struct AuthTicket { + EntityName name; + uint64_t global_id; /* global instance id */ + uint64_t auid; + utime_t created, renew_after, expires; + AuthCapsInfo caps; + __u32 flags; + + AuthTicket() : global_id(0), auid(CEPH_AUTH_UID_DEFAULT), flags(0){} + + void init_timestamps(utime_t now, double ttl) { + created = now; + expires = now; + expires += ttl; + renew_after = now; + renew_after += ttl / 2.0; + } + + void encode(bufferlist& bl) const { + __u8 struct_v = 2; + ::encode(struct_v, bl); + ::encode(name, bl); + ::encode(global_id, bl); + ::encode(auid, bl); + ::encode(created, bl); + ::encode(expires, bl); + ::encode(caps, bl); + ::encode(flags, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(name, bl); + ::decode(global_id, bl); + if (struct_v >= 2) + ::decode(auid, bl); + else auid = CEPH_AUTH_UID_DEFAULT; + ::decode(created, bl); + ::decode(expires, bl); + ::decode(caps, bl); + ::decode(flags, bl); + } +}; +WRITE_CLASS_ENCODER(AuthTicket) + + +/* + * abstract authorizer class + */ +struct AuthAuthorizer { + __u32 protocol; + bufferlist bl; + CryptoKey session_key; + + AuthAuthorizer(__u32 p) : protocol(p) {} + virtual ~AuthAuthorizer() {} + virtual bool verify_reply(bufferlist::iterator& reply) = 0; +}; + + +/* + * Key management + */ +#define KEY_ROTATE_NUM 3 /* prev, current, next */ + +struct ExpiringCryptoKey { + CryptoKey key; + utime_t expiration; + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(key, bl); + ::encode(expiration, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(key, bl); + ::decode(expiration, bl); + } +}; +WRITE_CLASS_ENCODER(ExpiringCryptoKey); + +static inline ostream& operator<<(ostream& out, const ExpiringCryptoKey& c) +{ + return out << c.key << " expires " << c.expiration; +} + +struct RotatingSecrets { + map secrets; + version_t max_ver; + + RotatingSecrets() : max_ver(0) {} + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(secrets, bl); + ::encode(max_ver, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(secrets, bl); + ::decode(max_ver, bl); + } + + uint64_t add(ExpiringCryptoKey& key) { + secrets[++max_ver] = key; + while (secrets.size() > KEY_ROTATE_NUM) + secrets.erase(secrets.begin()); + return max_ver; + } + + bool need_new_secrets() const { + return secrets.size() < KEY_ROTATE_NUM; + } + bool need_new_secrets(utime_t now) const { + return secrets.size() < KEY_ROTATE_NUM || current().expiration <= now; + } + + ExpiringCryptoKey& previous() { + return secrets.begin()->second; + } + ExpiringCryptoKey& current() { + map::iterator p = secrets.begin(); + ++p; + return p->second; + } + const ExpiringCryptoKey& current() const { + map::const_iterator p = secrets.begin(); + ++p; + return p->second; + } + ExpiringCryptoKey& next() { + return secrets.rbegin()->second; + } + bool empty() { + return secrets.empty(); + } + + void dump(); +}; +WRITE_CLASS_ENCODER(RotatingSecrets); + + + +class KeyStore { +public: + virtual ~KeyStore() {} + virtual bool get_secret(const EntityName& name, CryptoKey& secret) const = 0; + virtual bool get_service_secret(uint32_t service_id, uint64_t secret_id, + CryptoKey& secret) const = 0; +}; + +static inline bool auth_principal_needs_rotating_keys(EntityName& name) +{ + uint32_t ty(name.get_type()); + return ((ty == CEPH_ENTITY_TYPE_OSD) || (ty == CEPH_ENTITY_TYPE_MDS)); +} + +#endif diff --git a/ceph/src/auth/AuthAuthorizeHandler.cc b/ceph/src/auth/AuthAuthorizeHandler.cc new file mode 100644 index 00000000..c9c25900 --- /dev/null +++ b/ceph/src/auth/AuthAuthorizeHandler.cc @@ -0,0 +1,54 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2009-2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "common/debug.h" +#include "Auth.h" +#include "AuthAuthorizeHandler.h" +#include "cephx/CephxAuthorizeHandler.h" +#include "none/AuthNoneAuthorizeHandler.h" +#include "AuthMethodList.h" +#include "common/Mutex.h" + +#define dout_subsys ceph_subsys_auth + +AuthAuthorizeHandler *AuthAuthorizeHandlerRegistry::get_handler(int protocol) +{ + if (!supported.is_supported_auth(protocol)) { + return NULL; + } + + Mutex::Locker l(m_lock); + map::iterator iter = m_authorizers.find(protocol); + if (iter != m_authorizers.end()) + return iter->second; + + switch (protocol) { + case CEPH_AUTH_NONE: + m_authorizers[protocol] = new AuthNoneAuthorizeHandler(); + return m_authorizers[protocol]; + + case CEPH_AUTH_CEPHX: + m_authorizers[protocol] = new CephxAuthorizeHandler(); + return m_authorizers[protocol]; + } + return NULL; +} + +AuthAuthorizeHandlerRegistry::~AuthAuthorizeHandlerRegistry() +{ + for (map::iterator iter = m_authorizers.begin(); + iter != m_authorizers.end(); + ++iter) + delete iter->second; +} diff --git a/ceph/src/auth/AuthAuthorizeHandler.h b/ceph/src/auth/AuthAuthorizeHandler.h new file mode 100644 index 00000000..7b67876e --- /dev/null +++ b/ceph/src/auth/AuthAuthorizeHandler.h @@ -0,0 +1,55 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHAUTHORIZEHANDLER_H +#define CEPH_AUTHAUTHORIZEHANDLER_H + +#include "Auth.h" +#include "AuthMethodList.h" +#include "include/types.h" + +// Different classes of session crypto handling + +#define SESSION_CRYPTO_NONE 0 +#define SESSION_SYMMETRIC_AUTHENTICATE 1 +#define SESSION_SYMMETRIC_ENCRYPT 2 + +class CephContext; +class KeyRing; +class RotatingKeyRing; + +struct AuthAuthorizeHandler { + virtual ~AuthAuthorizeHandler() {} + virtual bool verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid = NULL) = 0; + virtual int authorizer_session_crypto() = 0; +}; + +class AuthAuthorizeHandlerRegistry { + Mutex m_lock; + map m_authorizers; + AuthMethodList supported; + +public: + AuthAuthorizeHandlerRegistry(CephContext *cct_, std::string methods) + : m_lock("AuthAuthorizeHandlerRegistry::m_lock"), supported(cct_, methods) + {} + ~AuthAuthorizeHandlerRegistry(); + + AuthAuthorizeHandler *get_handler(int protocol); +}; + +#endif diff --git a/ceph/src/auth/AuthClientHandler.cc b/ceph/src/auth/AuthClientHandler.cc new file mode 100644 index 00000000..5b02a258 --- /dev/null +++ b/ceph/src/auth/AuthClientHandler.cc @@ -0,0 +1,39 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include + +#include "AuthClientHandler.h" +#include "KeyRing.h" + +#include "messages/MAuth.h" +#include "messages/MAuthReply.h" + +#include "cephx/CephxClientHandler.h" +#include "none/AuthNoneClientHandler.h" + +AuthClientHandler *get_auth_client_handler(CephContext *cct, int proto, + RotatingKeyRing *rkeys) +{ + switch (proto) { + case CEPH_AUTH_CEPHX: + return new CephxClientHandler(cct, rkeys); + case CEPH_AUTH_NONE: + return new AuthNoneClientHandler(cct, rkeys); + default: + return NULL; + } +} + diff --git a/ceph/src/auth/AuthClientHandler.h b/ceph/src/auth/AuthClientHandler.h new file mode 100644 index 00000000..e12cbb91 --- /dev/null +++ b/ceph/src/auth/AuthClientHandler.h @@ -0,0 +1,83 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHCLIENTHANDLER_H +#define CEPH_AUTHCLIENTHANDLER_H + + +#include "auth/Auth.h" + +#include "common/Mutex.h" +#include "common/Cond.h" +#include "common/RWLock.h" + +#include "common/Timer.h" + +class CephContext; +struct MAuthReply; +class AuthClientHandler; +class RotatingKeyRing; + +class AuthClientHandler { +protected: + CephContext *cct; + EntityName name; + uint64_t global_id; + uint32_t want; + uint32_t have; + uint32_t need; + RWLock lock; + +public: + AuthClientHandler(CephContext *cct_) + : cct(cct_), global_id(0), want(CEPH_ENTITY_TYPE_AUTH), have(0), need(0), + lock("AuthClientHandler::lock") {} + virtual ~AuthClientHandler() {} + + void init(EntityName& n) { name = n; } + + void set_want_keys(__u32 keys) { + RWLock::WLocker l(lock); + want = keys | CEPH_ENTITY_TYPE_AUTH; + validate_tickets(); + } + void add_want_keys(__u32 keys) { + RWLock::WLocker l(lock); + want |= keys; + validate_tickets(); + } + + virtual int get_protocol() const = 0; + + virtual void reset() = 0; + virtual void prepare_build_request() = 0; + virtual int build_request(bufferlist& bl) const = 0; + virtual int handle_response(int ret, bufferlist::iterator& iter) = 0; + virtual bool build_rotating_request(bufferlist& bl) const = 0; + + virtual AuthAuthorizer *build_authorizer(uint32_t service_id) const = 0; + + virtual bool need_tickets() = 0; + + virtual void set_global_id(uint64_t id) = 0; +protected: + virtual void validate_tickets() = 0; +}; + + +extern AuthClientHandler *get_auth_client_handler(CephContext *cct, + int proto, RotatingKeyRing *rkeys); + +#endif + diff --git a/ceph/src/auth/AuthMethodList.cc b/ceph/src/auth/AuthMethodList.cc new file mode 100644 index 00000000..0e507fbf --- /dev/null +++ b/ceph/src/auth/AuthMethodList.cc @@ -0,0 +1,70 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include + +#include "common/Mutex.h" +#include "common/config.h" +#include "common/debug.h" +#include "include/str_list.h" + +#include "AuthMethodList.h" + +const static int dout_subsys = ceph_subsys_auth; + + +AuthMethodList::AuthMethodList(CephContext *cct, string str) +{ + list sup_list; + get_str_list(str, sup_list); + if (sup_list.empty()) { + lderr(cct) << "WARNING: empty auth protocol list" << dendl; + } + for (list::iterator iter = sup_list.begin(); iter != sup_list.end(); ++iter) { + ldout(cct, 5) << "adding auth protocol: " << *iter << dendl; + if (iter->compare("cephx") == 0) { + auth_supported.push_back(CEPH_AUTH_CEPHX); + } else if (iter->compare("none") == 0) { + auth_supported.push_back(CEPH_AUTH_NONE); + } else { + lderr(cct) << "WARNING: unknown auth protocol defined: " << *iter << dendl; + } + } + if (auth_supported.empty()) { + auth_supported.push_back(CEPH_AUTH_CEPHX); + } +} + +bool AuthMethodList::is_supported_auth(int auth_type) +{ + return std::find(auth_supported.begin(), auth_supported.end(), auth_type) != auth_supported.end(); +} + +int AuthMethodList::pick(const std::set<__u32>& supported) +{ + for (set<__u32>::const_reverse_iterator p = supported.rbegin(); p != supported.rend(); ++p) + if (is_supported_auth(*p)) + return *p; + return CEPH_AUTH_UNKNOWN; +} + +void AuthMethodList::remove_supported_auth(int auth_type) +{ + for (list<__u32>::iterator p = auth_supported.begin(); p != auth_supported.end(); ) { + if (*p == (__u32)auth_type) + auth_supported.erase(p++); + else + ++p; + } +} diff --git a/ceph/src/auth/AuthMethodList.h b/ceph/src/auth/AuthMethodList.h new file mode 100644 index 00000000..b5aae0bb --- /dev/null +++ b/ceph/src/auth/AuthMethodList.h @@ -0,0 +1,42 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHMETHODLIST_H +#define CEPH_AUTHMETHODLIST_H + +#include "include/int_types.h" + +#include +#include +#include + +class CephContext; + +class AuthMethodList { + std::list<__u32> auth_supported; +public: + AuthMethodList(CephContext *cct, std::string str); + + bool is_supported_auth(int auth_type); + int pick(const std::set<__u32>& supported); + + const std::list<__u32>& get_supported_set() const { + return auth_supported; + } + + void remove_supported_auth(int auth_type); +}; + + +#endif diff --git a/ceph/src/auth/AuthServiceHandler.cc b/ceph/src/auth/AuthServiceHandler.cc new file mode 100644 index 00000000..8e2d9661 --- /dev/null +++ b/ceph/src/auth/AuthServiceHandler.cc @@ -0,0 +1,33 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "AuthServiceHandler.h" +#include "cephx/CephxServiceHandler.h" +#include "none/AuthNoneServiceHandler.h" +#include "AuthMethodList.h" +#include "common/config.h" + +#define dout_subsys ceph_subsys_auth + + +AuthServiceHandler *get_auth_service_handler(int type, CephContext *cct, KeyServer *ks) +{ + switch (type) { + case CEPH_AUTH_CEPHX: + return new CephxServiceHandler(cct, ks); + case CEPH_AUTH_NONE: + return new AuthNoneServiceHandler(cct); + } + return NULL; +} diff --git a/ceph/src/auth/AuthServiceHandler.h b/ceph/src/auth/AuthServiceHandler.h new file mode 100644 index 00000000..429a5501 --- /dev/null +++ b/ceph/src/auth/AuthServiceHandler.h @@ -0,0 +1,44 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHSERVICEHANDLER_H +#define CEPH_AUTHSERVICEHANDLER_H + +#include "include/types.h" +#include "common/config.h" +#include "Auth.h" + +class CephContext; +class KeyServer; + +struct AuthServiceHandler { +protected: + CephContext *cct; +public: + EntityName entity_name; + uint64_t global_id; + + AuthServiceHandler(CephContext *cct_) : cct(cct_), global_id(0) {} + + virtual ~AuthServiceHandler() { } + + virtual int start_session(EntityName& name, bufferlist::iterator& indata, bufferlist& result, AuthCapsInfo& caps) = 0; + virtual int handle_request(bufferlist::iterator& indata, bufferlist& result, uint64_t& global_id, AuthCapsInfo& caps, uint64_t *auid = NULL) = 0; + + EntityName& get_entity_name() { return entity_name; } +}; + +extern AuthServiceHandler *get_auth_service_handler(int type, CephContext *cct, KeyServer *ks); + +#endif diff --git a/ceph/src/auth/AuthSessionHandler.cc b/ceph/src/auth/AuthSessionHandler.cc new file mode 100644 index 00000000..e928cf74 --- /dev/null +++ b/ceph/src/auth/AuthSessionHandler.cc @@ -0,0 +1,52 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "common/debug.h" +#include "AuthSessionHandler.h" +#include "cephx/CephxSessionHandler.h" +#include "none/AuthNoneSessionHandler.h" +#include "unknown/AuthUnknownSessionHandler.h" +#include "common/config.h" + +#define dout_subsys ceph_subsys_auth + + +AuthSessionHandler *get_auth_session_handler(CephContext *cct, int protocol, CryptoKey key, uint64_t features) +{ + + // Should add code to only print the SHA1 hash of the key, unless in secure debugging mode + + ldout(cct,10) << "In get_auth_session_handler for protocol " << protocol << dendl; + + switch (protocol) { + case CEPH_AUTH_CEPHX: + return new CephxSessionHandler(cct, key, features); + case CEPH_AUTH_NONE: + return new AuthNoneSessionHandler(cct, key); + case CEPH_AUTH_UNKNOWN: + return new AuthUnknownSessionHandler(cct, key); + } + return NULL; +} + + +void AuthSessionHandler::print_auth_session_handler_stats() { + ldout(cct,10) << "Auth Session Handler Stats " << this << dendl; + ldout(cct,10) << " Messages Signed = " << messages_signed << dendl; + ldout(cct,10) << " Signatures Checked = " << signatures_checked << dendl; + ldout(cct,10) << " Signatures Matched = " << signatures_matched << dendl; + ldout(cct,10) << " Signatures Did Not Match = " << signatures_failed << dendl; + ldout(cct,10) << " Messages Encrypted = " << messages_encrypted << dendl; + ldout(cct,10) << " Messages Decrypted = " << messages_decrypted << dendl; +} diff --git a/ceph/src/auth/AuthSessionHandler.h b/ceph/src/auth/AuthSessionHandler.h new file mode 100644 index 00000000..684b83a2 --- /dev/null +++ b/ceph/src/auth/AuthSessionHandler.h @@ -0,0 +1,71 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#ifndef CEPH_AUTHSESSIONHANDLER_H +#define CEPH_AUTHSESSIONHANDLER_H + +#include "include/types.h" +#include "common/config.h" +#include "msg/Message.h" +#include "Auth.h" + +#define SESSION_SIGNATURE_FAILURE -1 + +// Defines the security applied to ongoing messages in a session, once the session is established. PLR + +class CephContext; +class KeyServer; + +struct AuthSessionHandler { +protected: + CephContext *cct; + int protocol; + CryptoKey key; + +public: + // Keep stats on how many messages were signed, how many messages were encrypted, how many + // signatures were properly checked, and how many messages were decrypted. PLR + int messages_signed; + int signatures_checked; + int signatures_matched; + int signatures_failed; + int messages_encrypted; + int messages_decrypted; + + AuthSessionHandler(CephContext *cct_) : cct(cct_), messages_signed(0), signatures_checked(0), + signatures_matched(0), signatures_failed(0), messages_encrypted(0), messages_decrypted(0) {} + + AuthSessionHandler(CephContext *cct_, int protocol_, CryptoKey key_) : cct(cct_), + protocol(protocol_), key(key_), messages_signed(0), signatures_checked(0), signatures_matched(0), + signatures_failed(0), messages_encrypted(0), messages_decrypted(0) {} + virtual ~AuthSessionHandler() { } + + void print_auth_session_handler_stats() ; + + virtual bool no_security() = 0; + virtual int sign_message(Message *message) = 0; + virtual int check_message_signature(Message *message) = 0; + virtual int encrypt_message(Message *message) = 0; + virtual int decrypt_message(Message *message) = 0; + + int get_protocol() {return protocol;} + CryptoKey get_key() {return key;} + +}; + +extern AuthSessionHandler *get_auth_session_handler(CephContext *cct, int protocol, CryptoKey key, + uint64_t features); + +#endif diff --git a/ceph/src/auth/Crypto.cc b/ceph/src/auth/Crypto.cc new file mode 100644 index 00000000..e401c960 --- /dev/null +++ b/ceph/src/auth/Crypto.cc @@ -0,0 +1,398 @@ +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include "Crypto.h" +#ifdef USE_CRYPTOPP +# include +# include +# include +#elif USE_NSS +# include +# include +# include +#endif + +#include "include/assert.h" +#include "common/Clock.h" +#include "common/armor.h" +#include "common/ceph_crypto.h" +#include "common/config.h" +#include "common/debug.h" +#include "common/hex.h" +#include "common/safe_io.h" +#include "include/ceph_fs.h" +#include "include/compat.h" + +#include + +int get_random_bytes(char *buf, int len) +{ + int fd = TEMP_FAILURE_RETRY(::open("/dev/urandom", O_RDONLY)); + if (fd < 0) + return -errno; + int ret = safe_read_exact(fd, buf, len); + VOID_TEMP_FAILURE_RETRY(::close(fd)); + return ret; +} + +static int get_random_bytes(int len, bufferlist& bl) +{ + char buf[len]; + get_random_bytes(buf, len); + bl.append(buf, len); + return 0; +} + +uint64_t get_random(uint64_t min_val, uint64_t max_val) +{ + uint64_t r; + get_random_bytes((char *)&r, sizeof(r)); + r = min_val + r % (max_val - min_val + 1); + return r; +} + +// --------------------------------------------------- + +int CryptoNone::create(bufferptr& secret) +{ + return 0; +} + +int CryptoNone::validate_secret(bufferptr& secret) +{ + return 0; +} + +void CryptoNone::encrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const +{ + out = in; +} + +void CryptoNone::decrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const +{ + out = in; +} + + +// --------------------------------------------------- +#ifdef USE_CRYPTOPP +# define AES_KEY_LEN ((size_t)CryptoPP::AES::DEFAULT_KEYLENGTH) +# define AES_BLOCK_LEN ((size_t)CryptoPP::AES::BLOCKSIZE) +#elif USE_NSS +// when we say AES, we mean AES-128 +# define AES_KEY_LEN 16 +# define AES_BLOCK_LEN 16 + +static void nss_aes_operation(CK_ATTRIBUTE_TYPE op, const bufferptr& secret, + const bufferlist& in, bufferlist& out, std::string &error) +{ + const CK_MECHANISM_TYPE mechanism = CKM_AES_CBC_PAD; + + // sample source said this has to be at least size of input + 8, + // but i see 15 still fail with SEC_ERROR_OUTPUT_LEN + bufferptr out_tmp(in.length()+16); + + PK11SlotInfo *slot; + + slot = PK11_GetBestSlot(mechanism, NULL); + if (!slot) { + ostringstream oss; + oss << "cannot find NSS slot to use: " << PR_GetError(); + error = oss.str(); + goto err; + } + + SECItem keyItem; + + keyItem.type = siBuffer; + keyItem.data = (unsigned char*)secret.c_str(); + keyItem.len = secret.length(); + + PK11SymKey *key; + + key = PK11_ImportSymKey(slot, mechanism, PK11_OriginUnwrap, CKA_ENCRYPT, + &keyItem, NULL); + if (!key) { + ostringstream oss; + oss << "cannot convert AES key for NSS: " << PR_GetError(); + error = oss.str(); + goto err_slot; + } + + SECItem ivItem; + + ivItem.type = siBuffer; + // losing constness due to SECItem.data; IV should never be + // modified, regardless + ivItem.data = (unsigned char*)CEPH_AES_IV; + ivItem.len = sizeof(CEPH_AES_IV); + + SECItem *param; + + param = PK11_ParamFromIV(mechanism, &ivItem); + if (!param) { + ostringstream oss; + oss << "cannot set NSS IV param: " << PR_GetError(); + error = oss.str(); + goto err_key; + } + + PK11Context *ctx; + + ctx = PK11_CreateContextBySymKey(mechanism, op, key, param); + if (!ctx) { + ostringstream oss; + oss << "cannot create NSS context: " << PR_GetError(); + error = oss.str(); + goto err_param; + } + + SECStatus ret; + int written; + // in is const, and PK11_CipherOp is not; C++ makes this hard to cheat, + // so just copy it to a temp buffer, at least for now + unsigned in_len; + unsigned char *in_buf; + in_len = in.length(); + in_buf = (unsigned char*)malloc(in_len); + if (!in_buf) + throw std::bad_alloc(); + in.copy(0, in_len, (char*)in_buf); + ret = PK11_CipherOp(ctx, (unsigned char*)out_tmp.c_str(), &written, out_tmp.length(), + in_buf, in.length()); + free(in_buf); + if (ret != SECSuccess) { + ostringstream oss; + oss << "NSS AES failed: " << PR_GetError(); + error = oss.str(); + goto err_op; + } + + unsigned int written2; + ret = PK11_DigestFinal(ctx, (unsigned char*)out_tmp.c_str()+written, &written2, + out_tmp.length()-written); + if (ret != SECSuccess) { + ostringstream oss; + oss << "NSS AES final round failed: " << PR_GetError(); + error = oss.str(); + goto err_op; + } + + out_tmp.set_length(written + written2); + out.append(out_tmp); + + PK11_DestroyContext(ctx, PR_TRUE); + SECITEM_FreeItem(param, PR_TRUE); + PK11_FreeSymKey(key); + PK11_FreeSlot(slot); + return; + + err_op: + PK11_DestroyContext(ctx, PR_TRUE); + err_param: + SECITEM_FreeItem(param, PR_TRUE); + err_key: + PK11_FreeSymKey(key); + err_slot: + PK11_FreeSlot(slot); + err: + ; +} + +#else +# error "No supported crypto implementation found." +#endif + +int CryptoAES::create(bufferptr& secret) +{ + bufferlist bl; + int r = get_random_bytes(AES_KEY_LEN, bl); + if (r < 0) + return r; + secret = buffer::ptr(bl.c_str(), bl.length()); + return 0; +} + +int CryptoAES::validate_secret(bufferptr& secret) +{ + if (secret.length() < (size_t)AES_KEY_LEN) { + return -EINVAL; + } + + return 0; +} + +void CryptoAES::encrypt(const bufferptr& secret, const bufferlist& in, bufferlist& out, + std::string &error) const +{ + if (secret.length() < AES_KEY_LEN) { + error = "key is too short"; + return; + } +#ifdef USE_CRYPTOPP + { + const unsigned char *key = (const unsigned char *)secret.c_str(); + + string ciphertext; + CryptoPP::AES::Encryption aesEncryption(key, CryptoPP::AES::DEFAULT_KEYLENGTH); + CryptoPP::CBC_Mode_ExternalCipher::Encryption cbcEncryption( aesEncryption, (const byte*)CEPH_AES_IV ); + CryptoPP::StringSink *sink = new CryptoPP::StringSink(ciphertext); + CryptoPP::StreamTransformationFilter stfEncryptor(cbcEncryption, sink); + + for (std::list::const_iterator it = in.buffers().begin(); + it != in.buffers().end(); ++it) { + const unsigned char *in_buf = (const unsigned char *)it->c_str(); + stfEncryptor.Put(in_buf, it->length()); + } + try { + stfEncryptor.MessageEnd(); + } catch (CryptoPP::Exception& e) { + ostringstream oss; + oss << "encryptor.MessageEnd::Exception: " << e.GetWhat(); + error = oss.str(); + return; + } + out.append((const char *)ciphertext.c_str(), ciphertext.length()); + } +#elif USE_NSS + nss_aes_operation(CKA_ENCRYPT, secret, in, out, error); +#else +# error "No supported crypto implementation found." +#endif +} + +void CryptoAES::decrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const +{ +#ifdef USE_CRYPTOPP + const unsigned char *key = (const unsigned char *)secret.c_str(); + + CryptoPP::AES::Decryption aesDecryption(key, CryptoPP::AES::DEFAULT_KEYLENGTH); + CryptoPP::CBC_Mode_ExternalCipher::Decryption cbcDecryption( aesDecryption, (const byte*)CEPH_AES_IV ); + + string decryptedtext; + CryptoPP::StringSink *sink = new CryptoPP::StringSink(decryptedtext); + CryptoPP::StreamTransformationFilter stfDecryptor(cbcDecryption, sink); + for (std::list::const_iterator it = in.buffers().begin(); + it != in.buffers().end(); ++it) { + const unsigned char *in_buf = (const unsigned char *)it->c_str(); + stfDecryptor.Put(in_buf, it->length()); + } + + try { + stfDecryptor.MessageEnd(); + } catch (CryptoPP::Exception& e) { + ostringstream oss; + oss << "decryptor.MessageEnd::Exception: " << e.GetWhat(); + error = oss.str(); + return; + } + + out.append((const char *)decryptedtext.c_str(), decryptedtext.length()); +#elif USE_NSS + nss_aes_operation(CKA_DECRYPT, secret, in, out, error); +#else +# error "No supported crypto implementation found." +#endif +} + + +// --------------------------------------------------- + +int CryptoKey::set_secret(CephContext *cct, int type, bufferptr& s) +{ + this->type = type; + created = ceph_clock_now(cct); + + CryptoHandler *h = cct->get_crypto_handler(type); + if (!h) { + lderr(cct) << "ERROR: cct->get_crypto_handler(type=" << type << ") returned NULL" << dendl; + return -EOPNOTSUPP; + } + int ret = h->validate_secret(s); + + if (ret < 0) + return ret; + + secret = s; + + return 0; +} + +int CryptoKey::create(CephContext *cct, int t) +{ + type = t; + created = ceph_clock_now(cct); + + CryptoHandler *h = cct->get_crypto_handler(type); + if (!h) { + lderr(cct) << "ERROR: cct->get_crypto_handler(type=" << type << ") returned NULL" << dendl; + return -EOPNOTSUPP; + } + return h->create(secret); +} + +void CryptoKey::encrypt(CephContext *cct, const bufferlist& in, bufferlist& out, std::string &error) const +{ + if (!ch || ch->get_type() != type) { + ch = cct->get_crypto_handler(type); + if (!ch) { + ostringstream oss; + oss << "CryptoKey::encrypt: key type " << type << " not supported."; + return; + } + } + ch->encrypt(this->secret, in, out, error); +} + +void CryptoKey::decrypt(CephContext *cct, const bufferlist& in, bufferlist& out, std::string &error) const +{ + if (!ch || ch->get_type() != type) { + ch = cct->get_crypto_handler(type); + if (!ch) { + ostringstream oss; + oss << "CryptoKey::decrypt: key type " << type << " not supported."; + return; + } + } + ch->decrypt(this->secret, in, out, error); +} + +void CryptoKey::print(std::ostream &out) const +{ + out << encode_base64(); +} + +void CryptoKey::to_str(std::string& s) const +{ + int len = secret.length() * 4; + char buf[len]; + hex2str(secret.c_str(), secret.length(), buf, len); + s = buf; +} + +void CryptoKey::encode_formatted(string label, Formatter *f, bufferlist &bl) +{ + f->open_object_section(label.c_str()); + f->dump_string("key", encode_base64()); + f->close_section(); + f->flush(bl); +} + +void CryptoKey::encode_plaintext(bufferlist &bl) +{ + bl.append(encode_base64()); +} diff --git a/ceph/src/auth/Crypto.h b/ceph/src/auth/Crypto.h new file mode 100644 index 00000000..f0fc97b5 --- /dev/null +++ b/ceph/src/auth/Crypto.h @@ -0,0 +1,162 @@ +// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTH_CRYPTO_H +#define CEPH_AUTH_CRYPTO_H + +#include "include/types.h" +#include "include/utime.h" + +#include "common/Formatter.h" +#include "include/buffer.h" + +#include + +class CephContext; +class CryptoHandler; + +/* + * match encoding of struct ceph_secret + */ +class CryptoKey { +protected: + __u16 type; + utime_t created; + bufferptr secret; + + // cache a pointer to the handler, so we don't have to look it up + // for each crypto operation + mutable CryptoHandler *ch; + +public: + CryptoKey() : type(0), ch(NULL) { } + CryptoKey(int t, utime_t c, bufferptr& s) : type(t), created(c), secret(s), ch(NULL) { } + + void encode(bufferlist& bl) const { + ::encode(type, bl); + ::encode(created, bl); + __u16 len = secret.length(); + ::encode(len, bl); + bl.append(secret); + } + void decode(bufferlist::iterator& bl) { + ::decode(type, bl); + ::decode(created, bl); + __u16 len; + ::decode(len, bl); + bl.copy(len, secret); + secret.c_str(); // make sure it's a single buffer! + } + + int get_type() const { return type; } + utime_t get_created() const { return created; } + void print(std::ostream& out) const; + + int set_secret(CephContext *cct, int type, bufferptr& s); + bufferptr& get_secret() { return secret; } + const bufferptr& get_secret() const { return secret; } + + void encode_base64(string& s) const { + bufferlist bl; + encode(bl); + bufferlist e; + bl.encode_base64(e); + e.append('\0'); + s = e.c_str(); + } + string encode_base64() const { + string s; + encode_base64(s); + return s; + } + void decode_base64(const string& s) { + bufferlist e; + e.append(s); + bufferlist bl; + bl.decode_base64(e); + bufferlist::iterator p = bl.begin(); + decode(p); + } + + void encode_formatted(string label, Formatter *f, bufferlist &bl); + void encode_plaintext(bufferlist &bl); + + // -- + int create(CephContext *cct, int type); + void encrypt(CephContext *cct, const bufferlist& in, bufferlist& out, std::string &error) const; + void decrypt(CephContext *cct, const bufferlist& in, bufferlist& out, std::string &error) const; + + void to_str(std::string& s) const; +}; +WRITE_CLASS_ENCODER(CryptoKey); + +static inline ostream& operator<<(ostream& out, const CryptoKey& k) +{ + k.print(out); + return out; +} + + +/* + * Driver for a particular algorithm + * + * To use these functions, you need to call ceph::crypto::init(), see + * common/ceph_crypto.h. common_init_finish does this for you. + */ +class CryptoHandler { +public: + virtual ~CryptoHandler() {} + virtual int get_type() const = 0; + virtual int create(bufferptr& secret) = 0; + virtual int validate_secret(bufferptr& secret) = 0; + virtual void encrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const = 0; + virtual void decrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const = 0; +}; + +extern int get_random_bytes(char *buf, int len); +extern uint64_t get_random(uint64_t min_val, uint64_t max_val); + +class CryptoNone : public CryptoHandler { +public: + CryptoNone() { } + ~CryptoNone() {} + int get_type() const { + return CEPH_CRYPTO_NONE; + } + int create(bufferptr& secret); + int validate_secret(bufferptr& secret); + void encrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const; + void decrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const; +}; + +class CryptoAES : public CryptoHandler { +public: + CryptoAES() { } + ~CryptoAES() {} + int get_type() const { + return CEPH_CRYPTO_AES; + } + int create(bufferptr& secret); + int validate_secret(bufferptr& secret); + void encrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const; + void decrypt(const bufferptr& secret, const bufferlist& in, + bufferlist& out, std::string &error) const; +}; + +#endif diff --git a/ceph/src/auth/KeyRing.cc b/ceph/src/auth/KeyRing.cc new file mode 100644 index 00000000..96e56cd4 --- /dev/null +++ b/ceph/src/auth/KeyRing.cc @@ -0,0 +1,276 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +#include +#include + +#include "auth/AuthMethodList.h" +#include "auth/Crypto.h" +#include "auth/KeyRing.h" +#include "common/ConfUtils.h" +#include "common/config.h" +#include "common/debug.h" +#include "common/errno.h" +#include "include/str_list.h" +#include "common/Formatter.h" + +#define dout_subsys ceph_subsys_auth + +#undef dout_prefix +#define dout_prefix *_dout << "auth: " + +using std::auto_ptr; +using namespace std; + +int KeyRing::from_ceph_context(CephContext *cct) +{ + const md_config_t *conf = cct->_conf; + + int ret = -ENOENT; + string filename; + + if (ceph_resolve_file_search(conf->keyring, filename)) { + ret = load(cct, filename); + if (ret < 0) + lderr(cct) << "failed to load " << filename + << ": " << cpp_strerror(ret) << dendl; + } + + if (!conf->key.empty()) { + EntityAuth ea; + try { + ea.key.decode_base64(conf->key); + add(conf->name, ea); + return 0; + } + catch (buffer::error& e) { + lderr(cct) << "failed to decode key '" << conf->key << "'" << dendl; + return -EINVAL; + } + } + + if (!conf->keyfile.empty()) { + bufferlist bl; + string err; + int r = bl.read_file(conf->keyfile.c_str(), &err); + if (r < 0) { + lderr(cct) << err << dendl; + return r; + } + string k(bl.c_str(), bl.length()); + EntityAuth ea; + try { + ea.key.decode_base64(k); + add(conf->name, ea); + } + catch (buffer::error& e) { + lderr(cct) << "failed to decode key '" << k << "'" << dendl; + return -EINVAL; + } + return 0; + } + + return ret; +} + +KeyRing *KeyRing::create_empty() +{ + return new KeyRing(); +} + +int KeyRing::set_modifier(const char *type, const char *val, EntityName& name, map& caps) +{ + if (!val) + return -EINVAL; + + if (strcmp(type, "key") == 0) { + CryptoKey key; + string l(val); + try { + key.decode_base64(l); + } catch (const buffer::error& err) { + return -EINVAL; + } + set_key(name, key); + } else if (strncmp(type, "caps ", 5) == 0) { + const char *caps_entity = type + 5; + if (!*caps_entity) + return -EINVAL; + string l(val); + bufferlist bl; + ::encode(l, bl); + caps[caps_entity] = bl; + set_caps(name, caps); + } else if (strcmp(type, "auid") == 0) { + uint64_t auid = strtoull(val, NULL, 0); + set_uid(name, auid); + } else + return -EINVAL; + + return 0; +} + +void KeyRing::encode_plaintext(bufferlist& bl) +{ + std::ostringstream os; + print(os); + string str = os.str(); + bl.append(str); +} + +void KeyRing::encode_formatted(string label, Formatter *f, bufferlist& bl) +{ + std::ostringstream(os); + f->open_array_section(label.c_str()); + for (map::iterator p = keys.begin(); + p != keys.end(); + ++p) { + + f->open_object_section("auth_entities"); + f->dump_string("entity", p->first.to_str().c_str()); + std::ostringstream keyss; + keyss << p->second.key; + f->dump_string("key", keyss.str()); + f->open_object_section("caps"); + for (map::iterator q = p->second.caps.begin(); + q != p->second.caps.end(); + ++q) { + bufferlist::iterator dataiter = q->second.begin(); + string caps; + ::decode(caps, dataiter); + f->dump_string(q->first.c_str(), caps); + } + f->close_section(); /* caps */ + f->close_section(); /* auth_entities */ + } + f->close_section(); /* auth_dump */ + f->flush(bl); +} + +void KeyRing::decode_plaintext(bufferlist::iterator& bli) +{ + int ret; + bufferlist bl; + bli.copy_all(bl); + ConfFile cf; + std::deque parse_errors; + + if (cf.parse_bufferlist(&bl, &parse_errors, NULL) != 0) { + throw buffer::malformed_input("cannot parse buffer"); + } + + for (ConfFile::const_section_iter_t s = cf.sections_begin(); + s != cf.sections_end(); ++s) { + string name = s->first; + if (name == "global") + continue; + + EntityName ename; + map caps; + if (!ename.from_str(name)) { + ostringstream oss; + oss << "bad entity name in keyring: " << name; + throw buffer::malformed_input(oss.str().c_str()); + } + + for (ConfSection::const_line_iter_t l = s->second.lines.begin(); + l != s->second.lines.end(); ++l) { + if (l->key.empty()) + continue; + string k(l->key); + std::replace(k.begin(), k.end(), '_', ' '); + ret = set_modifier(k.c_str(), l->val.c_str(), ename, caps); + if (ret < 0) { + ostringstream oss; + oss << "error setting modifier for [" << name << "] type=" << k + << " val=" << l->val; + throw buffer::malformed_input(oss.str().c_str()); + } + } + } +} + +void KeyRing::decode(bufferlist::iterator& bl) { + __u8 struct_v; + bufferlist::iterator start_pos = bl; + try { + ::decode(struct_v, bl); + ::decode(keys, bl); + } catch (buffer::error& err) { + keys.clear(); + decode_plaintext(start_pos); + } +} + +int KeyRing::load(CephContext *cct, const std::string &filename) +{ + if (filename.empty()) + return -EINVAL; + + bufferlist bl; + std::string err; + int ret = bl.read_file(filename.c_str(), &err); + if (ret < 0) { + lderr(cct) << "error reading file: " << filename << ": " << err << dendl; + return ret; + } + + try { + bufferlist::iterator iter = bl.begin(); + decode(iter); + } + catch (const buffer::error& err) { + lderr(cct) << "error parsing file " << filename << dendl; + } + + ldout(cct, 2) << "KeyRing::load: loaded key file " << filename << dendl; + return 0; +} + +void KeyRing::print(ostream& out) +{ + for (map::iterator p = keys.begin(); + p != keys.end(); + ++p) { + out << "[" << p->first << "]" << std::endl; + out << "\tkey = " << p->second.key << std::endl; + if (p->second.auid != CEPH_AUTH_UID_DEFAULT) + out << "\tauid = " << p->second.auid << std::endl; + + for (map::iterator q = p->second.caps.begin(); + q != p->second.caps.end(); + ++q) { + bufferlist::iterator dataiter = q->second.begin(); + string caps; + ::decode(caps, dataiter); + out << "\tcaps " << q->first << " = \"" << caps << '"' << std::endl; + } + } +} + +void KeyRing::import(CephContext *cct, KeyRing& other) +{ + for (map::iterator p = other.keys.begin(); + p != other.keys.end(); + ++p) { + ldout(cct, 10) << " importing " << p->first << dendl; + ldout(cct, 30) << " " << p->second << dendl; + keys[p->first] = p->second; + } +} + + diff --git a/ceph/src/auth/KeyRing.h b/ceph/src/auth/KeyRing.h new file mode 100644 index 00000000..b69921c6 --- /dev/null +++ b/ceph/src/auth/KeyRing.h @@ -0,0 +1,110 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_KEYRING_H +#define CEPH_KEYRING_H + +#include "common/config.h" + +#include "auth/Crypto.h" +#include "auth/Auth.h" + + +class KeyRing : public KeyStore { + map keys; + + int set_modifier(const char *type, const char *val, EntityName& name, map& caps); +public: + void decode_plaintext(bufferlist::iterator& bl); + /* Create a KeyRing from a Ceph context. + * We will use the configuration stored inside the context. */ + int from_ceph_context(CephContext *cct); + + /* Create an empty KeyRing */ + static KeyRing *create_empty(); + + map& get_keys() { return keys; } // yuck + + int load(CephContext *cct, const std::string &filename); + void print(ostream& out); + + // accessors + bool get_auth(const EntityName& name, EntityAuth &a) const { + map::const_iterator k = keys.find(name); + if (k == keys.end()) + return false; + a = k->second; + return true; + } + bool get_secret(const EntityName& name, CryptoKey& secret) const { + map::const_iterator k = keys.find(name); + if (k == keys.end()) + return false; + secret = k->second.key; + return true; + } + bool get_service_secret(uint32_t service_id, uint64_t secret_id, + CryptoKey& secret) const { + return false; + } + bool get_caps(const EntityName& name, + const std::string& type, AuthCapsInfo& caps) const { + map::const_iterator k = keys.find(name); + if (k == keys.end()) + return false; + map::const_iterator i = k->second.caps.find(type); + if (i != k->second.caps.end()) { + caps.caps = i->second; + } + return true; + } + + // modifiers + void add(const EntityName& name, EntityAuth &a) { + keys[name] = a; + } + void add(const EntityName& name, CryptoKey &k) { + EntityAuth a; + a.key = k; + keys[name] = a; + } + void remove(const EntityName& name) { + keys.erase(name); + } + void set_caps(EntityName& name, map& caps) { + keys[name].caps = caps; + } + void set_uid(EntityName& ename, uint64_t auid) { + keys[ename].auid = auid; + } + void set_key(EntityName& ename, CryptoKey& key) { + keys[ename].key = key; + } + void import(CephContext *cct, KeyRing& other); + + // encoders + void decode(bufferlist::iterator& bl); + + void encode_plaintext(bufferlist& bl); + void encode_formatted(string label, Formatter *f, bufferlist& bl); +}; + +// don't use WRITE_CLASS_ENCODER macro because we don't have an encode +// macro. don't juse encode_plaintext in that case because it is not +// wrappable; it assumes it gets the entire bufferlist. +static inline void decode(KeyRing& kr, bufferlist::iterator& p) { + kr.decode(p); +} + +#endif diff --git a/ceph/src/auth/Makefile.am b/ceph/src/auth/Makefile.am new file mode 100644 index 00000000..f7f3b386 --- /dev/null +++ b/ceph/src/auth/Makefile.am @@ -0,0 +1,46 @@ +libauth_la_SOURCES = \ + auth/AuthAuthorizeHandler.cc \ + auth/AuthClientHandler.cc \ + auth/AuthSessionHandler.cc \ + auth/AuthServiceHandler.cc \ + auth/AuthMethodList.cc \ + auth/cephx/CephxAuthorizeHandler.cc \ + auth/cephx/CephxClientHandler.cc \ + auth/cephx/CephxProtocol.cc \ + auth/cephx/CephxServiceHandler.cc \ + auth/cephx/CephxSessionHandler.cc \ + auth/cephx/CephxKeyServer.cc \ + auth/none/AuthNoneAuthorizeHandler.cc \ + auth/unknown/AuthUnknownAuthorizeHandler.cc \ + auth/Crypto.cc \ + auth/KeyRing.cc \ + auth/RotatingKeyRing.cc +noinst_LTLIBRARIES += libauth.la + +noinst_HEADERS += \ + auth/cephx/CephxAuthorizeHandler.h \ + auth/cephx/CephxKeyServer.h \ + auth/cephx/CephxProtocol.h \ + auth/cephx/CephxClientHandler.h \ + auth/cephx/CephxServiceHandler.h \ + auth/cephx/CephxSessionHandler.h \ + auth/none/AuthNoneAuthorizeHandler.h \ + auth/none/AuthNoneClientHandler.h \ + auth/none/AuthNoneServiceHandler.h \ + auth/none/AuthNoneSessionHandler.h \ + auth/none/AuthNoneProtocol.h \ + auth/unknown/AuthUnknownAuthorizeHandler.h \ + auth/unknown/AuthUnknownClientHandler.h \ + auth/unknown/AuthUnknownServiceHandler.h \ + auth/unknown/AuthUnknownSessionHandler.h \ + auth/unknown/AuthUnknownProtocol.h \ + auth/Auth.h \ + auth/AuthMethodList.h \ + auth/AuthClientHandler.h \ + auth/AuthServiceHandler.h \ + auth/AuthSessionHandler.h \ + auth/AuthAuthorizeHandler.h \ + auth/KeyRing.h \ + auth/RotatingKeyRing.h \ + auth/Crypto.h + diff --git a/ceph/src/auth/RotatingKeyRing.cc b/ceph/src/auth/RotatingKeyRing.cc new file mode 100644 index 00000000..9470801f --- /dev/null +++ b/ceph/src/auth/RotatingKeyRing.cc @@ -0,0 +1,78 @@ +#include +#include + +#include "common/config.h" +#include "common/debug.h" +#include "include/str_list.h" + +#include "Crypto.h" +#include "auth/RotatingKeyRing.h" +#include "auth/KeyRing.h" + +#define dout_subsys ceph_subsys_auth +#undef dout_prefix +#define dout_prefix *_dout << "auth: " + + +bool RotatingKeyRing::need_new_secrets() const +{ + Mutex::Locker l(lock); + return secrets.need_new_secrets(); +} + +bool RotatingKeyRing::need_new_secrets(utime_t now) const +{ + Mutex::Locker l(lock); + return secrets.need_new_secrets(now); +} + +void RotatingKeyRing::set_secrets(RotatingSecrets& s) +{ + Mutex::Locker l(lock); + secrets = s; + dump_rotating(); +} + +void RotatingKeyRing::dump_rotating() const +{ + ldout(cct, 10) << "dump_rotating:" << dendl; + for (map::const_iterator iter = secrets.secrets.begin(); + iter != secrets.secrets.end(); + ++iter) + ldout(cct, 10) << " id " << iter->first << " " << iter->second << dendl; +} + +bool RotatingKeyRing::get_secret(const EntityName& name, CryptoKey& secret) const +{ + Mutex::Locker l(lock); + return keyring->get_secret(name, secret); +} + +bool RotatingKeyRing::get_service_secret(uint32_t service_id_, uint64_t secret_id, + CryptoKey& secret) const +{ + Mutex::Locker l(lock); + + if (service_id_ != this->service_id) { + ldout(cct, 0) << "do not have service " << ceph_entity_type_name(service_id_) + << ", i am " << ceph_entity_type_name(this->service_id) << dendl; + return false; + } + + map::const_iterator iter = + secrets.secrets.find(secret_id); + if (iter == secrets.secrets.end()) { + ldout(cct, 0) << "could not find secret_id=" << secret_id << dendl; + dump_rotating(); + return false; + } + + secret = iter->second.key; + return true; +} + +KeyRing *RotatingKeyRing:: +get_keyring() +{ + return keyring; +} diff --git a/ceph/src/auth/RotatingKeyRing.h b/ceph/src/auth/RotatingKeyRing.h new file mode 100644 index 00000000..efcb6382 --- /dev/null +++ b/ceph/src/auth/RotatingKeyRing.h @@ -0,0 +1,54 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_ROTATINGKEYRING_H +#define CEPH_ROTATINGKEYRING_H + +#include "common/config.h" +#include "common/Mutex.h" + +#include "auth/Crypto.h" +#include "auth/Auth.h" + +/* + * mediate access to a service's keyring and rotating secrets + */ + +class KeyRing; + +class RotatingKeyRing : public KeyStore { + CephContext *cct; + uint32_t service_id; + RotatingSecrets secrets; + KeyRing *keyring; + mutable Mutex lock; + +public: + RotatingKeyRing(CephContext *cct_, uint32_t s, KeyRing *kr) : + cct(cct_), + service_id(s), + keyring(kr), + lock("RotatingKeyRing::lock") {} + + bool need_new_secrets() const; + bool need_new_secrets(utime_t now) const; + void set_secrets(RotatingSecrets& s); + void dump_rotating() const; + bool get_secret(const EntityName& name, CryptoKey& secret) const; + bool get_service_secret(uint32_t service_id, uint64_t secret_id, + CryptoKey& secret) const; + KeyRing *get_keyring(); +}; + +#endif diff --git a/ceph/src/auth/cephx/CephxAuthorizeHandler.cc b/ceph/src/auth/cephx/CephxAuthorizeHandler.cc new file mode 100644 index 00000000..5b28b9e5 --- /dev/null +++ b/ceph/src/auth/cephx/CephxAuthorizeHandler.cc @@ -0,0 +1,42 @@ + +#include "../KeyRing.h" + +#include "CephxProtocol.h" +#include "CephxAuthorizeHandler.h" + +#define dout_subsys ceph_subsys_auth + + + +bool CephxAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid) +{ + bufferlist::iterator iter = authorizer_data.begin(); + + if (!authorizer_data.length()) { + ldout(cct, 1) << "verify authorizer, authorizer_data.length()=0" << dendl; + return false; + } + + CephXServiceTicketInfo auth_ticket_info; + + bool isvalid = cephx_verify_authorizer(cct, keys, iter, auth_ticket_info, authorizer_reply); + + if (isvalid) { + caps_info = auth_ticket_info.ticket.caps; + entity_name = auth_ticket_info.ticket.name; + global_id = auth_ticket_info.ticket.global_id; + session_key = auth_ticket_info.session_key; + if (auid) *auid = auth_ticket_info.ticket.auid; + } + + return isvalid; +} + +// Return type of crypto used for this session's data; for cephx, symmetric authentication + +int CephxAuthorizeHandler::authorizer_session_crypto() +{ + return SESSION_SYMMETRIC_AUTHENTICATE; +} diff --git a/ceph/src/auth/cephx/CephxAuthorizeHandler.h b/ceph/src/auth/cephx/CephxAuthorizeHandler.h new file mode 100644 index 00000000..d17a692f --- /dev/null +++ b/ceph/src/auth/cephx/CephxAuthorizeHandler.h @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_CEPHXAUTHORIZEHANDLER_H +#define CEPH_CEPHXAUTHORIZEHANDLER_H + +#include "../AuthAuthorizeHandler.h" + +class CephContext; + +struct CephxAuthorizeHandler : public AuthAuthorizeHandler { + bool verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid = NULL); + int authorizer_session_crypto(); +}; + + + +#endif diff --git a/ceph/src/auth/cephx/CephxClientHandler.cc b/ceph/src/auth/cephx/CephxClientHandler.cc new file mode 100644 index 00000000..b6d3501e --- /dev/null +++ b/ceph/src/auth/cephx/CephxClientHandler.cc @@ -0,0 +1,219 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include + +#include "CephxClientHandler.h" +#include "CephxProtocol.h" + +#include "../KeyRing.h" + +#include "common/config.h" + +#define dout_subsys ceph_subsys_auth +#undef dout_prefix +#define dout_prefix *_dout << "cephx client: " + + +int CephxClientHandler::build_request(bufferlist& bl) const +{ + ldout(cct, 10) << "build_request" << dendl; + + RWLock::RLocker l(lock); + + if (need & CEPH_ENTITY_TYPE_AUTH) { + /* authenticate */ + CephXRequestHeader header; + header.request_type = CEPHX_GET_AUTH_SESSION_KEY; + ::encode(header, bl); + + CryptoKey secret; + keyring->get_secret(cct->_conf->name, secret); + + CephXAuthenticate req; + get_random_bytes((char *)&req.client_challenge, sizeof(req.client_challenge)); + std::string error; + cephx_calc_client_server_challenge(cct, secret, server_challenge, + req.client_challenge, &req.key, error); + if (!error.empty()) { + ldout(cct, 20) << "cephx_calc_client_server_challenge error: " << error << dendl; + return -EIO; + } + + req.old_ticket = ticket_handler->ticket; + + if (req.old_ticket.blob.length()) { + ldout(cct, 20) << "old ticket len=" << req.old_ticket.blob.length() << dendl; + } + + ::encode(req, bl); + + ldout(cct, 10) << "get auth session key: client_challenge " << req.client_challenge << dendl; + return 0; + } + + if (need) { + /* get service tickets */ + ldout(cct, 10) << "get service keys: want=" << want << " need=" << need << " have=" << have << dendl; + + CephXRequestHeader header; + header.request_type = CEPHX_GET_PRINCIPAL_SESSION_KEY; + ::encode(header, bl); + + CephXAuthorizer *authorizer = ticket_handler->build_authorizer(global_id); + if (!authorizer) + return -EINVAL; + bl.claim_append(authorizer->bl); + delete authorizer; + + CephXServiceTicketRequest req; + req.keys = need; + ::encode(req, bl); + } + + return 0; +} + +int CephxClientHandler::handle_response(int ret, bufferlist::iterator& indata) +{ + ldout(cct, 10) << "handle_response ret = " << ret << dendl; + RWLock::WLocker l(lock); + + if (ret < 0) + return ret; // hrm! + + if (starting) { + CephXServerChallenge ch; + ::decode(ch, indata); + server_challenge = ch.server_challenge; + ldout(cct, 10) << " got initial server challenge " << server_challenge << dendl; + starting = false; + + tickets.invalidate_ticket(CEPH_ENTITY_TYPE_AUTH); + return -EAGAIN; + } + + struct CephXResponseHeader header; + ::decode(header, indata); + + switch (header.request_type) { + case CEPHX_GET_AUTH_SESSION_KEY: + { + ldout(cct, 10) << " get_auth_session_key" << dendl; + CryptoKey secret; + keyring->get_secret(cct->_conf->name, secret); + + if (!tickets.verify_service_ticket_reply(secret, indata)) { + ldout(cct, 0) << "could not verify service_ticket reply" << dendl; + return -EPERM; + } + ldout(cct, 10) << " want=" << want << " need=" << need << " have=" << have << dendl; + validate_tickets(); + if (need) + ret = -EAGAIN; + else + ret = 0; + } + break; + + case CEPHX_GET_PRINCIPAL_SESSION_KEY: + { + CephXTicketHandler& ticket_handler = tickets.get_handler(CEPH_ENTITY_TYPE_AUTH); + ldout(cct, 10) << " get_principal_session_key session_key " << ticket_handler.session_key << dendl; + + if (!tickets.verify_service_ticket_reply(ticket_handler.session_key, indata)) { + ldout(cct, 0) << "could not verify service_ticket reply" << dendl; + return -EPERM; + } + validate_tickets(); + if (!need) { + ret = 0; + } + } + break; + + case CEPHX_GET_ROTATING_KEY: + { + ldout(cct, 10) << " get_rotating_key" << dendl; + if (rotating_secrets) { + RotatingSecrets secrets; + CryptoKey secret_key; + keyring->get_secret(cct->_conf->name, secret_key); + std::string error; + if (decode_decrypt(cct, secrets, secret_key, indata, error)) { + ldout(cct, 0) << "could not set rotating key: decode_decrypt failed. error:" + << error << dendl; + error.clear(); + } else { + rotating_secrets->set_secrets(secrets); + } + } + } + break; + + default: + ldout(cct, 0) << " unknown request_type " << header.request_type << dendl; + assert(0); + } + return ret; +} + + + +AuthAuthorizer *CephxClientHandler::build_authorizer(uint32_t service_id) const +{ + RWLock::RLocker l(lock); + ldout(cct, 10) << "build_authorizer for service " << ceph_entity_type_name(service_id) << dendl; + return tickets.build_authorizer(service_id); +} + + +bool CephxClientHandler::build_rotating_request(bufferlist& bl) const +{ + ldout(cct, 10) << "build_rotating_request" << dendl; + CephXRequestHeader header; + header.request_type = CEPHX_GET_ROTATING_KEY; + ::encode(header, bl); + return true; +} + +void CephxClientHandler::prepare_build_request() +{ + RWLock::WLocker l(lock); + ldout(cct, 10) << "validate_tickets: want=" << want << " need=" << need + << " have=" << have << dendl; + validate_tickets(); + ldout(cct, 10) << "want=" << want << " need=" << need << " have=" << have + << dendl; + + ticket_handler = &(tickets.get_handler(CEPH_ENTITY_TYPE_AUTH)); +} + +void CephxClientHandler::validate_tickets() +{ + // lock should be held for write + tickets.validate_tickets(want, have, need); +} + +bool CephxClientHandler::need_tickets() +{ + RWLock::WLocker l(lock); + validate_tickets(); + + ldout(cct, 20) << "need_tickets: want=" << want << " need=" << need << " have=" << have << dendl; + + return (need != 0); +} + diff --git a/ceph/src/auth/cephx/CephxClientHandler.h b/ceph/src/auth/cephx/CephxClientHandler.h new file mode 100644 index 00000000..d200ac99 --- /dev/null +++ b/ceph/src/auth/cephx/CephxClientHandler.h @@ -0,0 +1,73 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_CEPHXCLIENTHANDLER_H +#define CEPH_CEPHXCLIENTHANDLER_H + +#include "../AuthClientHandler.h" +#include "CephxProtocol.h" + +class CephContext; + +class CephxClientHandler : public AuthClientHandler { + bool starting; + + /* envelope protocol parameters */ + uint64_t server_challenge; + + CephXTicketManager tickets; + CephXTicketHandler* ticket_handler; + + RotatingKeyRing *rotating_secrets; + KeyRing *keyring; + +public: + CephxClientHandler(CephContext *cct_, RotatingKeyRing *rsecrets) + : AuthClientHandler(cct_), + starting(false), + server_challenge(0), + tickets(cct_), + ticket_handler(NULL), + rotating_secrets(rsecrets), + keyring(rsecrets->get_keyring()) + { + reset(); + } + + void reset() { + RWLock::WLocker l(lock); + starting = true; + server_challenge = 0; + } + void prepare_build_request(); + int build_request(bufferlist& bl) const; + int handle_response(int ret, bufferlist::iterator& iter); + bool build_rotating_request(bufferlist& bl) const; + + int get_protocol() const { return CEPH_AUTH_CEPHX; } + + AuthAuthorizer *build_authorizer(uint32_t service_id) const; + + bool need_tickets(); + + void set_global_id(uint64_t id) { + RWLock::WLocker l(lock); + global_id = id; + tickets.global_id = id; + } +private: + void validate_tickets(); +}; + +#endif diff --git a/ceph/src/auth/cephx/CephxKeyServer.cc b/ceph/src/auth/cephx/CephxKeyServer.cc new file mode 100644 index 00000000..e57b5575 --- /dev/null +++ b/ceph/src/auth/cephx/CephxKeyServer.cc @@ -0,0 +1,461 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "common/config.h" + +#include "CephxKeyServer.h" +#include "common/Timer.h" + +#include + +#define dout_subsys ceph_subsys_auth +#undef dout_prefix +#define dout_prefix *_dout << "cephx keyserverdata: " + +bool KeyServerData::get_service_secret(CephContext *cct, uint32_t service_id, + ExpiringCryptoKey& secret, uint64_t& secret_id) const +{ + map::const_iterator iter = + rotating_secrets.find(service_id); + if (iter == rotating_secrets.end()) { + ldout(cct, 10) << "get_service_secret service " << ceph_entity_type_name(service_id) << " not found " << dendl; + return false; + } + + const RotatingSecrets& secrets = iter->second; + + // second to oldest, unless it's expired + map::const_iterator riter = + secrets.secrets.begin(); + if (secrets.secrets.size() > 1) + ++riter; + + if (riter->second.expiration < ceph_clock_now(cct)) + ++riter; // "current" key has expired, use "next" key instead + + secret_id = riter->first; + secret = riter->second; + ldout(cct, 30) << "get_service_secret service " << ceph_entity_type_name(service_id) + << " id " << secret_id << " " << secret << dendl; + return true; +} + +bool KeyServerData::get_service_secret(CephContext *cct, uint32_t service_id, + CryptoKey& secret, uint64_t& secret_id) const +{ + ExpiringCryptoKey e; + + if (!get_service_secret(cct, service_id, e, secret_id)) + return false; + + secret = e.key; + return true; +} + +bool KeyServerData::get_service_secret(CephContext *cct, uint32_t service_id, + uint64_t secret_id, CryptoKey& secret) const +{ + map::const_iterator iter = + rotating_secrets.find(service_id); + if (iter == rotating_secrets.end()) + return false; + + const RotatingSecrets& secrets = iter->second; + map::const_iterator riter = + secrets.secrets.find(secret_id); + + if (riter == secrets.secrets.end()) { + ldout(cct, 10) << "get_service_secret service " << ceph_entity_type_name(service_id) + << " secret " << secret_id << " not found" << dendl; + ldout(cct, 30) << " I have:" << dendl; + for (map::const_iterator iter = + secrets.secrets.begin(); + iter != secrets.secrets.end(); + ++iter) + ldout(cct, 30) << " id " << iter->first << " " << iter->second << dendl; + return false; + } + + secret = riter->second.key; + + return true; +} +bool KeyServerData::get_auth(const EntityName& name, EntityAuth& auth) const { + map::const_iterator iter = secrets.find(name); + if (iter != secrets.end()) { + auth = iter->second; + return true; + } + return extra_secrets->get_auth(name, auth); +} + +bool KeyServerData::get_secret(const EntityName& name, CryptoKey& secret) const { + map::const_iterator iter = secrets.find(name); + if (iter != secrets.end()) { + secret = iter->second.key; + return true; + } + return extra_secrets->get_secret(name, secret); +} + +bool KeyServerData::get_caps(CephContext *cct, const EntityName& name, + const string& type, AuthCapsInfo& caps_info) const +{ + caps_info.allow_all = false; + + ldout(cct, 10) << "get_caps: name=" << name.to_str() << dendl; + map::const_iterator iter = secrets.find(name); + if (iter != secrets.end()) { + ldout(cct, 10) << "get_secret: num of caps=" << iter->second.caps.size() << dendl; + map::const_iterator capsiter = iter->second.caps.find(type); + if (capsiter != iter->second.caps.end()) { + caps_info.caps = capsiter->second; + } + return true; + } + + return extra_secrets->get_caps(name, type, caps_info); +} + + +#undef dout_prefix +#define dout_prefix *_dout << "cephx keyserver: " + + +KeyServer::KeyServer(CephContext *cct_, KeyRing *extra_secrets) + : cct(cct_), + data(extra_secrets), + lock("KeyServer::lock") +{ +} + +int KeyServer::start_server() +{ + Mutex::Locker l(lock); + + _check_rotating_secrets(); + _dump_rotating_secrets(); + return 0; +} + +bool KeyServer::_check_rotating_secrets() +{ + ldout(cct, 10) << "_check_rotating_secrets" << dendl; + + int added = 0; + added += _rotate_secret(CEPH_ENTITY_TYPE_AUTH); + added += _rotate_secret(CEPH_ENTITY_TYPE_MON); + added += _rotate_secret(CEPH_ENTITY_TYPE_OSD); + added += _rotate_secret(CEPH_ENTITY_TYPE_MDS); + + if (added) { + ldout(cct, 10) << __func__ << " added " << added << dendl; + data.rotating_ver++; + //data.next_rotating_time = ceph_clock_now(cct); + //data.next_rotating_time += MIN(cct->_conf->auth_mon_ticket_ttl, cct->_conf->auth_service_ticket_ttl); + _dump_rotating_secrets(); + return true; + } + return false; +} + +void KeyServer::_dump_rotating_secrets() +{ + ldout(cct, 30) << "_dump_rotating_secrets" << dendl; + for (map::iterator iter = data.rotating_secrets.begin(); + iter != data.rotating_secrets.end(); + ++iter) { + RotatingSecrets& key = iter->second; + for (map::iterator mapiter = key.secrets.begin(); + mapiter != key.secrets.end(); + ++mapiter) + ldout(cct, 30) << "service " << ceph_entity_type_name(iter->first) + << " id " << mapiter->first + << " key " << mapiter->second << dendl; + } +} + +int KeyServer::_rotate_secret(uint32_t service_id) +{ + RotatingSecrets& r = data.rotating_secrets[service_id]; + int added = 0; + utime_t now = ceph_clock_now(cct); + double ttl = service_id == CEPH_ENTITY_TYPE_AUTH ? cct->_conf->auth_mon_ticket_ttl : cct->_conf->auth_service_ticket_ttl; + + while (r.need_new_secrets(now)) { + ExpiringCryptoKey ek; + generate_secret(ek.key); + if (r.empty()) { + ek.expiration = now; + } else { + utime_t next_ttl = now; + next_ttl += ttl; + ek.expiration = MAX(next_ttl, r.next().expiration); + } + ek.expiration += ttl; + uint64_t secret_id = r.add(ek); + ldout(cct, 10) << "_rotate_secret adding " << ceph_entity_type_name(service_id) << dendl; + ldout(cct, 30) << "_rotate_secret adding " << ceph_entity_type_name(service_id) + << " id " << secret_id << " " << ek + << dendl; + added++; + } + return added; +} + +bool KeyServer::get_secret(const EntityName& name, CryptoKey& secret) const +{ + Mutex::Locker l(lock); + return data.get_secret(name, secret); +} + +bool KeyServer::get_auth(const EntityName& name, EntityAuth& auth) const +{ + Mutex::Locker l(lock); + return data.get_auth(name, auth); +} + +bool KeyServer::get_caps(const EntityName& name, const string& type, + AuthCapsInfo& caps_info) const +{ + Mutex::Locker l(lock); + + return data.get_caps(cct, name, type, caps_info); +} + +bool KeyServer::get_service_secret(uint32_t service_id, + ExpiringCryptoKey& secret, uint64_t& secret_id) const +{ + Mutex::Locker l(lock); + + return data.get_service_secret(cct, service_id, secret, secret_id); +} + +bool KeyServer::get_service_secret(uint32_t service_id, + CryptoKey& secret, uint64_t& secret_id) const +{ + Mutex::Locker l(lock); + + return data.get_service_secret(cct, service_id, secret, secret_id); +} + +bool KeyServer::get_service_secret(uint32_t service_id, + uint64_t secret_id, CryptoKey& secret) const +{ + Mutex::Locker l(lock); + + return data.get_service_secret(cct, service_id, secret_id, secret); +} + +bool KeyServer::generate_secret(CryptoKey& secret) +{ + bufferptr bp; + CryptoHandler *crypto = cct->get_crypto_handler(CEPH_CRYPTO_AES); + if (!crypto) + return false; + + if (crypto->create(bp) < 0) + return false; + + secret.set_secret(cct, CEPH_CRYPTO_AES, bp); + + return true; +} + +bool KeyServer::generate_secret(EntityName& name, CryptoKey& secret) +{ + if (!generate_secret(secret)) + return false; + + Mutex::Locker l(lock); + + EntityAuth auth; + auth.key = secret; + + data.add_auth(name, auth); + + return true; +} + +bool KeyServer::contains(const EntityName& name) const +{ + Mutex::Locker l(lock); + + return data.contains(name); +} + +int KeyServer::encode_secrets(Formatter *f, stringstream *ds) const +{ + Mutex::Locker l(lock); + + if (f) + f->open_array_section("auth_dump"); + + map::const_iterator mapiter = data.secrets_begin(); + + if (mapiter == data.secrets_end()) + return -ENOENT; + + while (mapiter != data.secrets_end()) { + const EntityName& name = mapiter->first; + if (ds) { + *ds << name.to_str() << std::endl; + *ds << "\tkey: " << mapiter->second.key << std::endl; + } + if (f) { + f->open_object_section("auth_entities"); + f->dump_string("entity", name.to_str()); + f->dump_stream("key") << mapiter->second.key; + f->open_object_section("caps"); + } + + map::const_iterator capsiter = + mapiter->second.caps.begin(); + for (; capsiter != mapiter->second.caps.end(); ++capsiter) { + // FIXME: need a const_iterator for bufferlist, but it doesn't exist yet. + bufferlist *bl = const_cast(&capsiter->second); + bufferlist::iterator dataiter = bl->begin(); + string caps; + ::decode(caps, dataiter); + if (ds) + *ds << "\tcaps: [" << capsiter->first << "] " << caps << std::endl; + if (f) + f->dump_string(capsiter->first.c_str(), caps); + } + if (f) { + f->close_section(); // caps + f->close_section(); // auth_entities + } + + ++mapiter; + } + + if (f) + f->close_section(); // auth_dump + return 0; +} + +void KeyServer::encode_formatted(string label, Formatter *f, bufferlist &bl) +{ + assert(f != NULL); + f->open_object_section(label.c_str()); + encode_secrets(f, NULL); + f->close_section(); + f->flush(bl); +} + +void KeyServer::encode_plaintext(bufferlist &bl) +{ + stringstream os; + encode_secrets(NULL, &os); + bl.append(os.str()); +} + +bool KeyServer::updated_rotating(bufferlist& rotating_bl, version_t& rotating_ver) +{ + Mutex::Locker l(lock); + + _check_rotating_secrets(); + + if (data.rotating_ver <= rotating_ver) + return false; + + data.encode_rotating(rotating_bl); + + rotating_ver = data.rotating_ver; + + return true; +} + +bool KeyServer::get_rotating_encrypted(const EntityName& name, + bufferlist& enc_bl) const +{ + Mutex::Locker l(lock); + + map::const_iterator mapiter = data.find_name(name); + if (mapiter == data.secrets_end()) + return false; + + const CryptoKey& specific_key = mapiter->second.key; + + map::const_iterator rotate_iter = + data.rotating_secrets.find(name.get_type()); + if (rotate_iter == data.rotating_secrets.end()) + return false; + + RotatingSecrets secrets = rotate_iter->second; + + std::string error; + if (encode_encrypt(cct, secrets, specific_key, enc_bl, error)) + return false; + + return true; +} + +bool KeyServer::_get_service_caps(const EntityName& name, uint32_t service_id, + AuthCapsInfo& caps_info) const +{ + string s = ceph_entity_type_name(service_id); + + return data.get_caps(cct, name, s, caps_info); +} + +bool KeyServer::get_service_caps(const EntityName& name, uint32_t service_id, + AuthCapsInfo& caps_info) const +{ + Mutex::Locker l(lock); + return _get_service_caps(name, service_id, caps_info); +} + + +int KeyServer::_build_session_auth_info(uint32_t service_id, CephXServiceTicketInfo& auth_ticket_info, + CephXSessionAuthInfo& info) +{ + info.service_id = service_id; + info.ticket = auth_ticket_info.ticket; + info.ticket.init_timestamps(ceph_clock_now(cct), cct->_conf->auth_service_ticket_ttl); + + generate_secret(info.session_key); + + // mon keys are stored externally. and the caps are blank anyway. + if (service_id != CEPH_ENTITY_TYPE_MON) { + string s = ceph_entity_type_name(service_id); + if (!data.get_caps(cct, info.ticket.name, s, info.ticket.caps)) { + return -EINVAL; + } + } + return 0; +} + +int KeyServer::build_session_auth_info(uint32_t service_id, CephXServiceTicketInfo& auth_ticket_info, + CephXSessionAuthInfo& info) +{ + if (!get_service_secret(service_id, info.service_secret, info.secret_id)) { + return -EPERM; + } + + Mutex::Locker l(lock); + + return _build_session_auth_info(service_id, auth_ticket_info, info); +} + +int KeyServer::build_session_auth_info(uint32_t service_id, CephXServiceTicketInfo& auth_ticket_info, CephXSessionAuthInfo& info, + CryptoKey& service_secret, uint64_t secret_id) +{ + info.service_secret = service_secret; + info.secret_id = secret_id; + + return _build_session_auth_info(service_id, auth_ticket_info, info); +} + diff --git a/ceph/src/auth/cephx/CephxKeyServer.h b/ceph/src/auth/cephx/CephxKeyServer.h new file mode 100644 index 00000000..bc30fef6 --- /dev/null +++ b/ceph/src/auth/cephx/CephxKeyServer.h @@ -0,0 +1,308 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_KEYSSERVER_H +#define CEPH_KEYSSERVER_H + +#include "common/config.h" + +#include "auth/KeyRing.h" +#include "CephxProtocol.h" + +#include "common/Timer.h" + +class CephContext; + +struct KeyServerData { + version_t version; + + /* for each entity */ + map secrets; + KeyRing *extra_secrets; + + /* for each service type */ + version_t rotating_ver; + map rotating_secrets; + + KeyServerData(KeyRing *extra) + : version(0), + extra_secrets(extra), + rotating_ver(0) {} + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(version, bl); + ::encode(rotating_ver, bl); + ::encode(secrets, bl); + ::encode(rotating_secrets, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(version, bl); + ::decode(rotating_ver, bl); + ::decode(secrets, bl); + ::decode(rotating_secrets, bl); + } + + void encode_rotating(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(rotating_ver, bl); + ::encode(rotating_secrets, bl); + } + void decode_rotating(bufferlist& rotating_bl) { + bufferlist::iterator iter = rotating_bl.begin(); + __u8 struct_v; + ::decode(struct_v, iter); + ::decode(rotating_ver, iter); + ::decode(rotating_secrets, iter); + } + + bool contains(const EntityName& name) const { + return (secrets.find(name) != secrets.end()); + } + + void clear_secrets() { + secrets.clear(); + } + + void add_auth(const EntityName& name, EntityAuth& auth) { + secrets[name] = auth; + } + + void remove_secret(const EntityName& name) { + map::iterator iter = secrets.find(name); + if (iter == secrets.end()) + return; + secrets.erase(iter); + } + + bool get_service_secret(CephContext *cct, uint32_t service_id, + ExpiringCryptoKey& secret, uint64_t& secret_id) const; + bool get_service_secret(CephContext *cct, uint32_t service_id, + CryptoKey& secret, uint64_t& secret_id) const; + bool get_service_secret(CephContext *cct, uint32_t service_id, + uint64_t secret_id, CryptoKey& secret) const; + bool get_auth(const EntityName& name, EntityAuth& auth) const; + bool get_secret(const EntityName& name, CryptoKey& secret) const; + bool get_caps(CephContext *cct, const EntityName& name, + const std::string& type, AuthCapsInfo& caps) const; + + map::iterator secrets_begin() + { return secrets.begin(); } + map::const_iterator secrets_begin() const + { return secrets.begin(); } + map::iterator secrets_end() + { return secrets.end(); } + map::const_iterator secrets_end() const + { return secrets.end(); } + map::iterator find_name(const EntityName& name) + { return secrets.find(name); } + map::const_iterator find_name(const EntityName& name) const + { return secrets.find(name); } + + + // -- incremental updates -- + typedef enum { + AUTH_INC_NOP, + AUTH_INC_ADD, + AUTH_INC_DEL, + AUTH_INC_SET_ROTATING, + } IncrementalOp; + + struct Incremental { + IncrementalOp op; + bufferlist rotating_bl; // if SET_ROTATING. otherwise, + EntityName name; + EntityAuth auth; + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + __u32 _op = (__u32)op; + ::encode(_op, bl); + if (op == AUTH_INC_SET_ROTATING) { + ::encode(rotating_bl, bl); + } else { + ::encode(name, bl); + ::encode(auth, bl); + } + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + __u32 _op; + ::decode(_op, bl); + op = (IncrementalOp)_op; + assert(op >= AUTH_INC_NOP && op <= AUTH_INC_SET_ROTATING); + if (op == AUTH_INC_SET_ROTATING) { + ::decode(rotating_bl, bl); + } else { + ::decode(name, bl); + ::decode(auth, bl); + } + } + }; + + void apply_incremental(Incremental& inc) { + switch (inc.op) { + case AUTH_INC_ADD: + add_auth(inc.name, inc.auth); + break; + + case AUTH_INC_DEL: + remove_secret(inc.name); + break; + + case AUTH_INC_SET_ROTATING: + decode_rotating(inc.rotating_bl); + break; + + case AUTH_INC_NOP: + break; + + default: + assert(0); + } + } + +}; +WRITE_CLASS_ENCODER(KeyServerData); +WRITE_CLASS_ENCODER(KeyServerData::Incremental); + + + + +class KeyServer : public KeyStore { + CephContext *cct; + KeyServerData data; + mutable Mutex lock; + + int _rotate_secret(uint32_t service_id); + bool _check_rotating_secrets(); + void _dump_rotating_secrets(); + int _build_session_auth_info(uint32_t service_id, + CephXServiceTicketInfo& auth_ticket_info, CephXSessionAuthInfo& info); + bool _get_service_caps(const EntityName& name, uint32_t service_id, + AuthCapsInfo& caps) const; +public: + KeyServer(CephContext *cct_, KeyRing *extra_secrets); + bool generate_secret(CryptoKey& secret); + + bool get_secret(const EntityName& name, CryptoKey& secret) const; + bool get_auth(const EntityName& name, EntityAuth& auth) const; + bool get_caps(const EntityName& name, const string& type, AuthCapsInfo& caps) const; + bool get_active_rotating_secret(const EntityName& name, CryptoKey& secret) const; + int start_server(); + void rotate_timeout(double timeout); + + int build_session_auth_info(uint32_t service_id, CephXServiceTicketInfo& auth_ticket_info, CephXSessionAuthInfo& info); + int build_session_auth_info(uint32_t service_id, CephXServiceTicketInfo& auth_ticket_info, CephXSessionAuthInfo& info, + CryptoKey& service_secret, uint64_t secret_id); + + /* get current secret for specific service type */ + bool get_service_secret(uint32_t service_id, ExpiringCryptoKey& service_key, + uint64_t& secret_id) const; + bool get_service_secret(uint32_t service_id, CryptoKey& service_key, + uint64_t& secret_id) const; + bool get_service_secret(uint32_t service_id, uint64_t secret_id, + CryptoKey& secret) const; + + bool generate_secret(EntityName& name, CryptoKey& secret); + + void encode(bufferlist& bl) const { + ::encode(data, bl); + } + void decode(bufferlist::iterator& bl) { + Mutex::Locker l(lock); + ::decode(data, bl); + } + bool contains(const EntityName& name) const; + int encode_secrets(Formatter *f, stringstream *ds) const; + void encode_formatted(string label, Formatter *f, bufferlist &bl); + void encode_plaintext(bufferlist &bl); + int list_secrets(stringstream& ds) const { + return encode_secrets(NULL, &ds); + } + version_t get_ver() const { + Mutex::Locker l(lock); + return data.version; + } + + void clear_secrets() { + data.clear_secrets(); + } + + void apply_data_incremental(KeyServerData::Incremental& inc) { + data.apply_incremental(inc); + } + void set_ver(version_t ver) { + Mutex::Locker l(lock); + data.version = ver; + } + + void add_auth(const EntityName& name, EntityAuth& auth) { + Mutex::Locker l(lock); + data.add_auth(name, auth); + } + + void remove_secret(const EntityName& name) { + Mutex::Locker l(lock); + data.remove_secret(name); + } + + bool has_secrets() { + map::const_iterator b = data.secrets_begin(); + return (b != data.secrets_end()); + } + int get_num_secrets() { + return data.secrets.size(); + } + + /*void add_rotating_secret(uint32_t service_id, ExpiringCryptoKey& key) { + Mutex::Locker l(lock); + data.add_rotating_secret(service_id, key); + } + */ + void clone_to(KeyServerData& dst) const { + Mutex::Locker l(lock); + dst = data; + } + void export_keyring(KeyRing& keyring) { + for (map::iterator p = data.secrets.begin(); + p != data.secrets.end(); + ++p) { + keyring.add(p->first, p->second); + } + } + + bool updated_rotating(bufferlist& rotating_bl, version_t& rotating_ver); + + bool get_rotating_encrypted(const EntityName& name, bufferlist& enc_bl) const; + + Mutex& get_lock() const { return lock; } + bool get_service_caps(const EntityName& name, uint32_t service_id, + AuthCapsInfo& caps) const; + + map::iterator secrets_begin() + { return data.secrets_begin(); } + map::iterator secrets_end() + { return data.secrets_end(); } +}; +WRITE_CLASS_ENCODER(KeyServer); + + +#endif diff --git a/ceph/src/auth/cephx/CephxProtocol.cc b/ceph/src/auth/cephx/CephxProtocol.cc new file mode 100644 index 00000000..f57f0635 --- /dev/null +++ b/ceph/src/auth/cephx/CephxProtocol.cc @@ -0,0 +1,490 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2009-2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "CephxProtocol.h" +#include "common/Clock.h" +#include "common/config.h" +#include "common/debug.h" +#include "include/buffer.h" + +#define dout_subsys ceph_subsys_auth +#undef dout_prefix +#define dout_prefix *_dout << "cephx: " + + + +void cephx_calc_client_server_challenge(CephContext *cct, CryptoKey& secret, uint64_t server_challenge, + uint64_t client_challenge, uint64_t *key, std::string &ret) +{ + CephXChallengeBlob b; + b.server_challenge = server_challenge; + b.client_challenge = client_challenge; + + bufferlist enc; + std::string error; + if (encode_encrypt(cct, b, secret, enc, error)) + return; + + uint64_t k = 0; + const uint64_t *p = (const uint64_t *)enc.c_str(); + for (int pos = 0; pos + sizeof(k) <= enc.length(); pos+=sizeof(k), p++) + k ^= mswab64(*p); + *key = k; +} + + +/* + * Authentication + */ + +bool cephx_build_service_ticket_blob(CephContext *cct, CephXSessionAuthInfo& info, + CephXTicketBlob& blob) +{ + CephXServiceTicketInfo ticket_info; + ticket_info.session_key = info.session_key; + ticket_info.ticket = info.ticket; + ticket_info.ticket.caps = info.ticket.caps; + + ldout(cct, 10) << "build_service_ticket service " << ceph_entity_type_name(info.service_id) + << " secret_id " << info.secret_id + << " ticket_info.ticket.name=" << ticket_info.ticket.name.to_str() << dendl; + blob.secret_id = info.secret_id; + std::string error; + encode_encrypt_enc_bl(cct, ticket_info, info.service_secret, blob.blob, error); + if (!error.empty()) { + ldout(cct, -1) << "cephx_build_service_ticket_blob failed with error " + << error << dendl; + return false; + } + return true; +} + +/* + * AUTH SERVER: authenticate + * + * Authenticate principal, respond with AuthServiceTicketInfo + * + * {session key, validity}^principal_secret + * {principal_ticket, session key}^service_secret ... "enc_ticket" + */ +bool cephx_build_service_ticket_reply(CephContext *cct, + CryptoKey& principal_secret, + vector ticket_info_vec, + bool should_encrypt_ticket, + CryptoKey& ticket_enc_key, + bufferlist& reply) +{ + __u8 service_ticket_reply_v = 1; + ::encode(service_ticket_reply_v, reply); + + uint32_t num = ticket_info_vec.size(); + ::encode(num, reply); + ldout(cct, 10) << "build_service_ticket_reply encoding " << num + << " tickets with secret " << principal_secret << dendl; + + for (vector::iterator ticket_iter = ticket_info_vec.begin(); + ticket_iter != ticket_info_vec.end(); + ++ticket_iter) { + CephXSessionAuthInfo& info = *ticket_iter; + ::encode(info.service_id, reply); + + __u8 service_ticket_v = 1; + ::encode(service_ticket_v, reply); + + CephXServiceTicket msg_a; + msg_a.session_key = info.session_key; + msg_a.validity = info.validity; + std::string error; + if (encode_encrypt(cct, msg_a, principal_secret, reply, error)) { + ldout(cct, -1) << "error encoding encrypted: " << error << dendl; + return false; + } + + bufferlist service_ticket_bl; + CephXTicketBlob blob; + if (!cephx_build_service_ticket_blob(cct, info, blob)) { + return false; + } + ::encode(blob, service_ticket_bl); + + ldout(cct, 30) << "service_ticket_blob is "; + service_ticket_bl.hexdump(*_dout); + *_dout << dendl; + + ::encode((__u8)should_encrypt_ticket, reply); + if (should_encrypt_ticket) { + if (encode_encrypt(cct, service_ticket_bl, ticket_enc_key, reply, error)) { + ldout(cct, -1) << "error encoding encrypted ticket: " << error << dendl; + return false; + } + } else { + ::encode(service_ticket_bl, reply); + } + } + return true; +} + +/* + * PRINCIPAL: verify our attempt to authenticate succeeded. fill out + * this ServiceTicket with the result. + */ +bool CephXTicketHandler::verify_service_ticket_reply(CryptoKey& secret, + bufferlist::iterator& indata) +{ + __u8 service_ticket_v; + ::decode(service_ticket_v, indata); + + CephXServiceTicket msg_a; + std::string error; + if (decode_decrypt(cct, msg_a, secret, indata, error)) { + ldout(cct, 0) << "verify_service_ticket_reply: failed decode_decrypt, error is: " << error << dendl; + return false; + } + + __u8 ticket_enc; + ::decode(ticket_enc, indata); + + bufferlist service_ticket_bl; + if (ticket_enc) { + ldout(cct, 10) << " got encrypted ticket" << dendl; + std::string error; + if (decode_decrypt(cct, service_ticket_bl, session_key, indata, error)) { + ldout(cct, 10) << "verify_service_ticket_reply: decode_decrypt failed " + << "with " << error << dendl; + return false; + } + } else { + ::decode(service_ticket_bl, indata); + } + bufferlist::iterator iter = service_ticket_bl.begin(); + ::decode(ticket, iter); + ldout(cct, 10) << " ticket.secret_id=" << ticket.secret_id << dendl; + + ldout(cct, 10) << "verify_service_ticket_reply service " << ceph_entity_type_name(service_id) + << " secret_id " << ticket.secret_id + << " session_key " << msg_a.session_key + << " validity=" << msg_a.validity << dendl; + session_key = msg_a.session_key; + if (!msg_a.validity.is_zero()) { + expires = ceph_clock_now(cct); + expires += msg_a.validity; + renew_after = expires; + renew_after -= ((double)msg_a.validity.sec() / 4); + ldout(cct, 10) << "ticket expires=" << expires << " renew_after=" << renew_after << dendl; + } + + have_key_flag = true; + return true; +} + +bool CephXTicketHandler::have_key() +{ + if (have_key_flag) { + have_key_flag = ceph_clock_now(cct) < expires; + } + + return have_key_flag; +} + +bool CephXTicketHandler::need_key() const +{ + if (have_key_flag) { + return (!expires.is_zero()) && (ceph_clock_now(cct) >= renew_after); + } + + return true; +} + +bool CephXTicketManager::have_key(uint32_t service_id) +{ + map::iterator iter = tickets_map.find(service_id); + if (iter == tickets_map.end()) + return false; + return iter->second.have_key(); +} + +bool CephXTicketManager::need_key(uint32_t service_id) const +{ + map::const_iterator iter = tickets_map.find(service_id); + if (iter == tickets_map.end()) + return true; + return iter->second.need_key(); +} + +void CephXTicketManager::set_have_need_key(uint32_t service_id, uint32_t& have, uint32_t& need) +{ + map::iterator iter = tickets_map.find(service_id); + if (iter == tickets_map.end()) { + have &= ~service_id; + need |= service_id; + ldout(cct, 10) << "set_have_need_key no handler for service " + << ceph_entity_type_name(service_id) << dendl; + return; + } + + //ldout(cct, 10) << "set_have_need_key service " << ceph_entity_type_name(service_id) + //<< " (" << service_id << ")" + //<< " need=" << iter->second.need_key() << " have=" << iter->second.have_key() << dendl; + if (iter->second.need_key()) + need |= service_id; + else + need &= ~service_id; + + if (iter->second.have_key()) + have |= service_id; + else + have &= ~service_id; +} + +void CephXTicketManager::invalidate_ticket(uint32_t service_id) +{ + map::iterator iter = tickets_map.find(service_id); + if (iter != tickets_map.end()) + iter->second.invalidate_ticket(); +} + +/* + * PRINCIPAL: verify our attempt to authenticate succeeded. fill out + * this ServiceTicket with the result. + */ +bool CephXTicketManager::verify_service_ticket_reply(CryptoKey& secret, + bufferlist::iterator& indata) +{ + __u8 service_ticket_reply_v; + ::decode(service_ticket_reply_v, indata); + + uint32_t num; + ::decode(num, indata); + ldout(cct, 10) << "verify_service_ticket_reply got " << num << " keys" << dendl; + + for (int i=0; i<(int)num; i++) { + uint32_t type; + ::decode(type, indata); + ldout(cct, 10) << "got key for service_id " << ceph_entity_type_name(type) << dendl; + CephXTicketHandler& handler = get_handler(type); + if (!handler.verify_service_ticket_reply(secret, indata)) { + return false; + } + handler.service_id = type; + } + + if (!indata.end()) + return false; + + return true; +} + +/* + * PRINCIPAL: build authorizer to access the service. + * + * ticket, {timestamp}^session_key + */ +CephXAuthorizer *CephXTicketHandler::build_authorizer(uint64_t global_id) const +{ + CephXAuthorizer *a = new CephXAuthorizer(cct); + a->session_key = session_key; + a->nonce = ((uint64_t)rand() << 32) + rand(); + + __u8 authorizer_v = 1; + ::encode(authorizer_v, a->bl); + ::encode(global_id, a->bl); + ::encode(service_id, a->bl); + + ::encode(ticket, a->bl); + + CephXAuthorize msg; + msg.nonce = a->nonce; + + std::string error; + if (encode_encrypt(cct, msg, session_key, a->bl, error)) { + ldout(cct, 0) << "failed to encrypt authorizer: " << error << dendl; + delete a; + return 0; + } + return a; +} + +/* + * PRINCIPAL: build authorizer to access the service. + * + * ticket, {timestamp}^session_key + */ +CephXAuthorizer *CephXTicketManager::build_authorizer(uint32_t service_id) const +{ + map::const_iterator iter = tickets_map.find(service_id); + if (iter == tickets_map.end()) { + ldout(cct, 0) << "no TicketHandler for service " + << ceph_entity_type_name(service_id) << dendl; + return NULL; + } + + const CephXTicketHandler& handler = iter->second; + return handler.build_authorizer(global_id); +} + +void CephXTicketManager::validate_tickets(uint32_t mask, uint32_t& have, uint32_t& need) +{ + uint32_t i; + need = 0; + for (i = 1; i<=mask; i<<=1) { + if (mask & i) { + set_have_need_key(i, have, need); + } + } + ldout(cct, 10) << "validate_tickets want " << mask << " have " << have + << " need " << need << dendl; +} + +bool cephx_decode_ticket(CephContext *cct, KeyStore *keys, uint32_t service_id, + CephXTicketBlob& ticket_blob, CephXServiceTicketInfo& ticket_info) +{ + uint64_t secret_id = ticket_blob.secret_id; + CryptoKey service_secret; + + if (!ticket_blob.blob.length()) { + return false; + } + + if (secret_id == (uint64_t)-1) { + if (!keys->get_secret(cct->_conf->name, service_secret)) { + ldout(cct, 0) << "ceph_decode_ticket could not get general service secret for service_id=" + << ceph_entity_type_name(service_id) << " secret_id=" << secret_id << dendl; + return false; + } + } else { + if (!keys->get_service_secret(service_id, secret_id, service_secret)) { + ldout(cct, 0) << "ceph_decode_ticket could not get service secret for service_id=" + << ceph_entity_type_name(service_id) << " secret_id=" << secret_id << dendl; + return false; + } + } + + std::string error; + decode_decrypt_enc_bl(cct, ticket_info, service_secret, ticket_blob.blob, error); + if (!error.empty()) { + ldout(cct, 0) << "ceph_decode_ticket could not decrypt ticket info. error:" + << error << dendl; + return false; + } + + return true; +} + +/* + * SERVICE: verify authorizer and generate reply authorizer + * + * {timestamp + 1}^session_key + */ +bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist::iterator& indata, + CephXServiceTicketInfo& ticket_info, bufferlist& reply_bl) +{ + __u8 authorizer_v; + uint32_t service_id; + uint64_t global_id; + CryptoKey service_secret; + // ticket blob + CephXTicketBlob ticket; + + + try { + ::decode(authorizer_v, indata); + ::decode(global_id, indata); + ::decode(service_id, indata); + ::decode(ticket, indata); + } catch (buffer::end_of_buffer &e) { + // Unable to decode! + return false; + } + ldout(cct, 10) << "verify_authorizer decrypted service " + << ceph_entity_type_name(service_id) + << " secret_id=" << ticket.secret_id << dendl; + + if (ticket.secret_id == (uint64_t)-1) { + EntityName name; + name.set_type(service_id); + if (!keys->get_secret(name, service_secret)) { + ldout(cct, 0) << "verify_authorizer could not get general service secret for service " + << ceph_entity_type_name(service_id) << " secret_id=" << ticket.secret_id << dendl; + return false; + } + } else { + if (!keys->get_service_secret(service_id, ticket.secret_id, service_secret)) { + ldout(cct, 0) << "verify_authorizer could not get service secret for service " + << ceph_entity_type_name(service_id) << " secret_id=" << ticket.secret_id << dendl; + if (cct->_conf->auth_debug && ticket.secret_id == 0) + assert(0 == "got secret_id=0"); + return false; + } + } + std::string error; + decode_decrypt_enc_bl(cct, ticket_info, service_secret, ticket.blob, error); + if (!error.empty()) { + ldout(cct, 0) << "verify_authorizer could not decrypt ticket info: error: " + << error << dendl; + return false; + } + + if (ticket_info.ticket.global_id != global_id) { + ldout(cct, 0) << "verify_authorizer global_id mismatch: declared id=" << global_id + << " ticket_id=" << ticket_info.ticket.global_id << dendl; + return false; + } + + ldout(cct, 10) << "verify_authorizer global_id=" << global_id << dendl; + + // CephXAuthorize + CephXAuthorize auth_msg; + if (decode_decrypt(cct, auth_msg, ticket_info.session_key, indata, error)) { + ldout(cct, 0) << "verify_authorizercould not decrypt authorize request with error: " + << error << dendl; + return false; + } + + /* + * Reply authorizer: + * {timestamp + 1}^session_key + */ + CephXAuthorizeReply reply; + // reply.trans_id = auth_msg.trans_id; + reply.nonce_plus_one = auth_msg.nonce + 1; + if (encode_encrypt(cct, reply, ticket_info.session_key, reply_bl, error)) { + ldout(cct, 10) << "verify_authorizer: encode_encrypt error: " << error << dendl; + return false; + } + + ldout(cct, 10) << "verify_authorizer ok nonce " << hex << auth_msg.nonce << dec + << " reply_bl.length()=" << reply_bl.length() << dendl; + return true; +} + +bool CephXAuthorizer::verify_reply(bufferlist::iterator& indata) +{ + CephXAuthorizeReply reply; + + std::string error; + if (decode_decrypt(cct, reply, session_key, indata, error)) { + ldout(cct, 0) << "verify_reply couldn't decrypt with error: " << error << dendl; + return false; + } + + uint64_t expect = nonce + 1; + if (expect != reply.nonce_plus_one) { + ldout(cct, 0) << "verify_authorizer_reply bad nonce got " << reply.nonce_plus_one << " expected " << expect + << " sent " << nonce << dendl; + return false; + } + return true; +} + diff --git a/ceph/src/auth/cephx/CephxProtocol.h b/ceph/src/auth/cephx/CephxProtocol.h new file mode 100644 index 00000000..8a3e0946 --- /dev/null +++ b/ceph/src/auth/cephx/CephxProtocol.h @@ -0,0 +1,499 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_CEPHXPROTOCOL_H +#define CEPH_CEPHXPROTOCOL_H + +/* + Ceph X protocol + + First, the principal has to authenticate with the authenticator. A + shared-secret mechanism is being used, and the negotitaion goes like this: + + A = Authenticator + P = Principle + S = Service + + 1. Obtaining principal/auth session key + + (Authenticate Request) + p->a : principal, principal_addr. authenticate me! + + ...authenticator does lookup in database... + + a->p : A= {principal/auth session key, validity}^principal_secret (*) + B= {principal ticket, validity, principal/auth session key}^authsecret + + + [principal/auth session key, validity] = service ticket + [principal ticket, validity, principal/auth session key] = service ticket info + + (*) annotation: ^ signifies 'encrypted by' + + At this point, if is genuine, the principal should have the principal/auth + session key at hand. The next step would be to request an authorization to + use some other service: + + 2. Obtaining principal/service session key + + p->a : B, {principal_addr, timestamp}^principal/auth session key. authorize + me! + a->p : E= {service ticket}^svcsecret + F= {principal/service session key, validity}^principal/auth session key + + principal_addr, timestamp = authenticator + + service ticket = principal name, client network address, validity, principal/service session key + + Note that steps 1 and 2 are pretty much the same thing; contacting the + authenticator and requesting for a key. + + Following this the principal should have a principal/service session key that + could be used later on for creating a session: + + 3. Opening a session to a service + + p->s : E + {principal_addr, timestamp}^principal/service session key + s->p : {timestamp+1}^principal/service/session key + + timestamp+1 = reply authenticator + + Now, the principal is fully authenticated with the service. So, logically we + have 2 main actions here. The first one would be to obtain a session key to + the service (steps 1 and 2), and the second one would be to authenticate with + the service, using that ticket. +*/ + +/* authenticate requests */ +#define CEPHX_GET_AUTH_SESSION_KEY 0x0100 +#define CEPHX_GET_PRINCIPAL_SESSION_KEY 0x0200 +#define CEPHX_GET_ROTATING_KEY 0x0400 + +#define CEPHX_REQUEST_TYPE_MASK 0x0F00 +#define CEPHX_CRYPT_ERR 1 + +#include "../Auth.h" +#include "../RotatingKeyRing.h" +#include "common/debug.h" + +#include +#include + +class CephContext; + +/* + * Authentication + */ + +// initial server -> client challenge +struct CephXServerChallenge { + uint64_t server_challenge; + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(server_challenge, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(server_challenge, bl); + } +}; +WRITE_CLASS_ENCODER(CephXServerChallenge); + + +// request/reply headers, for subsequent exchanges. + +struct CephXRequestHeader { + __u16 request_type; + + void encode(bufferlist& bl) const { + ::encode(request_type, bl); + } + void decode(bufferlist::iterator& bl) { + ::decode(request_type, bl); + } +}; +WRITE_CLASS_ENCODER(CephXRequestHeader); + +struct CephXResponseHeader { + uint16_t request_type; + int32_t status; + + void encode(bufferlist& bl) const { + ::encode(request_type, bl); + ::encode(status, bl); + } + void decode(bufferlist::iterator& bl) { + ::decode(request_type, bl); + ::decode(status, bl); + } +}; +WRITE_CLASS_ENCODER(CephXResponseHeader); + +struct CephXTicketBlob { + uint64_t secret_id; + bufferlist blob; + + CephXTicketBlob() : secret_id(0) {} + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(secret_id, bl); + ::encode(blob, bl); + } + + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(secret_id, bl); + ::decode(blob, bl); + } +}; +WRITE_CLASS_ENCODER(CephXTicketBlob); + +// client -> server response to challenge +struct CephXAuthenticate { + uint64_t client_challenge; + uint64_t key; + CephXTicketBlob old_ticket; + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(client_challenge, bl); + ::encode(key, bl); + ::encode(old_ticket, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(client_challenge, bl); + ::decode(key, bl); + ::decode(old_ticket, bl); + } +}; +WRITE_CLASS_ENCODER(CephXAuthenticate) + +struct CephXChallengeBlob { + uint64_t server_challenge, client_challenge; + + void encode(bufferlist& bl) const { + ::encode(server_challenge, bl); + ::encode(client_challenge, bl); + } + void decode(bufferlist::iterator& bl) { + ::decode(server_challenge, bl); + ::decode(client_challenge, bl); + } +}; +WRITE_CLASS_ENCODER(CephXChallengeBlob) + +void cephx_calc_client_server_challenge(CephContext *cct, + CryptoKey& secret, uint64_t server_challenge, uint64_t client_challenge, + uint64_t *key, std::string &error); + + +/* + * getting service tickets + */ +struct CephXSessionAuthInfo { + uint32_t service_id; + uint64_t secret_id; + AuthTicket ticket; + CryptoKey session_key; + CryptoKey service_secret; + utime_t validity; +}; + + +extern bool cephx_build_service_ticket_blob(CephContext *cct, + CephXSessionAuthInfo& ticket_info, CephXTicketBlob& blob); + +extern void cephx_build_service_ticket_request(CephContext *cct, + uint32_t keys, + bufferlist& request); + +extern bool cephx_build_service_ticket_reply(CephContext *cct, + CryptoKey& principal_secret, + vector ticket_info, + bool should_encrypt_ticket, + CryptoKey& ticket_enc_key, + bufferlist& reply); + +struct CephXServiceTicketRequest { + uint32_t keys; + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(keys, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(keys, bl); + } +}; +WRITE_CLASS_ENCODER(CephXServiceTicketRequest); + + +/* + * Authorize + */ + +struct CephXAuthorizeReply { + uint64_t nonce_plus_one; + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(nonce_plus_one, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(nonce_plus_one, bl); + } +}; +WRITE_CLASS_ENCODER(CephXAuthorizeReply); + + +struct CephXAuthorizer : public AuthAuthorizer { +private: + CephContext *cct; +public: + uint64_t nonce; + + CephXAuthorizer(CephContext *cct_) + : AuthAuthorizer(CEPH_AUTH_CEPHX), cct(cct_), nonce(0) {} + + bool build_authorizer(); + bool verify_reply(bufferlist::iterator& reply); +}; + + + +/* + * TicketHandler + */ +struct CephXTicketHandler { + uint32_t service_id; + CryptoKey session_key; + CephXTicketBlob ticket; // opaque to us + utime_t renew_after, expires; + bool have_key_flag; + + CephXTicketHandler(CephContext *cct_, uint32_t service_id_) + : service_id(service_id_), have_key_flag(false), cct(cct_) { } + + // to build our ServiceTicket + bool verify_service_ticket_reply(CryptoKey& principal_secret, + bufferlist::iterator& indata); + // to access the service + CephXAuthorizer *build_authorizer(uint64_t global_id) const; + + bool have_key(); + bool need_key() const; + + void invalidate_ticket() { + have_key_flag = 0; + } +private: + CephContext *cct; +}; + +struct CephXTicketManager { + typedef map tickets_map_t; + tickets_map_t tickets_map; + uint64_t global_id; + + CephXTicketManager(CephContext *cct_) : global_id(0), cct(cct_) {} + + bool verify_service_ticket_reply(CryptoKey& principal_secret, + bufferlist::iterator& indata); + + CephXTicketHandler& get_handler(uint32_t type) { + tickets_map_t::iterator i = tickets_map.find(type); + if (i != tickets_map.end()) + return i->second; + CephXTicketHandler newTicketHandler(cct, type); + std::pair < tickets_map_t::iterator, bool > res = + tickets_map.insert(std::make_pair(type, newTicketHandler)); + assert(res.second); + return res.first->second; + } + CephXAuthorizer *build_authorizer(uint32_t service_id) const; + bool have_key(uint32_t service_id); + bool need_key(uint32_t service_id) const; + void set_have_need_key(uint32_t service_id, uint32_t& have, uint32_t& need); + void validate_tickets(uint32_t mask, uint32_t& have, uint32_t& need); + void invalidate_ticket(uint32_t service_id); + +private: + CephContext *cct; +}; + + +/* A */ +struct CephXServiceTicket { + CryptoKey session_key; + utime_t validity; + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(session_key, bl); + ::encode(validity, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(session_key, bl); + ::decode(validity, bl); + } +}; +WRITE_CLASS_ENCODER(CephXServiceTicket); + +/* B */ +struct CephXServiceTicketInfo { + AuthTicket ticket; + CryptoKey session_key; + + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(ticket, bl); + ::encode(session_key, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(ticket, bl); + ::decode(session_key, bl); + } +}; +WRITE_CLASS_ENCODER(CephXServiceTicketInfo); + +struct CephXAuthorize { + uint64_t nonce; + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(nonce, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(nonce, bl); + } +}; +WRITE_CLASS_ENCODER(CephXAuthorize); + +/* + * Decode an extract ticket + */ +bool cephx_decode_ticket(CephContext *cct, KeyStore *keys, + uint32_t service_id, CephXTicketBlob& ticket_blob, + CephXServiceTicketInfo& ticket_info); + +/* + * Verify authorizer and generate reply authorizer + */ +extern bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist::iterator& indata, + CephXServiceTicketInfo& ticket_info, bufferlist& reply_bl); + + + + + + +/* + * encode+encrypt macros + */ +#define AUTH_ENC_MAGIC 0xff009cad8826aa55ull + +template +void decode_decrypt_enc_bl(CephContext *cct, T& t, CryptoKey key, bufferlist& bl_enc, + std::string &error) +{ + uint64_t magic; + bufferlist bl; + + key.decrypt(cct, bl_enc, bl, error); + if (!error.empty()) + return; + + bufferlist::iterator iter2 = bl.begin(); + __u8 struct_v; + ::decode(struct_v, iter2); + ::decode(magic, iter2); + if (magic != AUTH_ENC_MAGIC) { + ostringstream oss; + oss << "bad magic in decode_decrypt, " << magic << " != " << AUTH_ENC_MAGIC; + error = oss.str(); + return; + } + + ::decode(t, iter2); +} + +template +void encode_encrypt_enc_bl(CephContext *cct, const T& t, const CryptoKey& key, + bufferlist& out, std::string &error) +{ + bufferlist bl; + __u8 struct_v = 1; + ::encode(struct_v, bl); + uint64_t magic = AUTH_ENC_MAGIC; + ::encode(magic, bl); + ::encode(t, bl); + + key.encrypt(cct, bl, out, error); +} + +template +int decode_decrypt(CephContext *cct, T& t, const CryptoKey& key, + bufferlist::iterator& iter, std::string &error) +{ + bufferlist bl_enc; + try { + ::decode(bl_enc, iter); + decode_decrypt_enc_bl(cct, t, key, bl_enc, error); + } + catch (buffer::error &e) { + error = "error decoding block for decryption"; + } + if (!error.empty()) + return CEPHX_CRYPT_ERR; + return 0; +} + +template +int encode_encrypt(CephContext *cct, const T& t, const CryptoKey& key, + bufferlist& out, std::string &error) +{ + bufferlist bl_enc; + encode_encrypt_enc_bl(cct, t, key, bl_enc, error); + if (!error.empty()){ + return CEPHX_CRYPT_ERR; + } + ::encode(bl_enc, out); + return 0; +} + + +#endif diff --git a/ceph/src/auth/cephx/CephxServiceHandler.cc b/ceph/src/auth/cephx/CephxServiceHandler.cc new file mode 100644 index 00000000..c5d91d98 --- /dev/null +++ b/ceph/src/auth/cephx/CephxServiceHandler.cc @@ -0,0 +1,204 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include "CephxServiceHandler.h" +#include "CephxProtocol.h" + +#include "../Auth.h" + +#include "mon/Monitor.h" + +#include +#include + +#include "common/config.h" +#include "include/assert.h" + +#define dout_subsys ceph_subsys_auth +#undef dout_prefix +#define dout_prefix *_dout << "cephx server " << entity_name << ": " + +int CephxServiceHandler::start_session(EntityName& name, bufferlist::iterator& indata, bufferlist& result_bl, AuthCapsInfo& caps) +{ + entity_name = name; + + get_random_bytes((char *)&server_challenge, sizeof(server_challenge)); + if (!server_challenge) + server_challenge = 1; // always non-zero. + ldout(cct, 10) << "start_session server_challenge " << hex << server_challenge << dec << dendl; + + CephXServerChallenge ch; + ch.server_challenge = server_challenge; + ::encode(ch, result_bl); + return CEPH_AUTH_CEPHX; +} + +int CephxServiceHandler::handle_request(bufferlist::iterator& indata, bufferlist& result_bl, uint64_t& global_id, AuthCapsInfo& caps, uint64_t *auid) +{ + int ret = 0; + + struct CephXRequestHeader cephx_header; + ::decode(cephx_header, indata); + + + switch (cephx_header.request_type) { + case CEPHX_GET_AUTH_SESSION_KEY: + { + ldout(cct, 10) << "handle_request get_auth_session_key for " << entity_name << dendl; + + CephXAuthenticate req; + ::decode(req, indata); + + CryptoKey secret; + if (!key_server->get_secret(entity_name, secret)) { + ldout(cct, 0) << "couldn't find entity name: " << entity_name << dendl; + ret = -EPERM; + break; + } + + if (!server_challenge) { + ret = -EPERM; + break; + } + + uint64_t expected_key; + std::string error; + cephx_calc_client_server_challenge(cct, secret, server_challenge, + req.client_challenge, &expected_key, error); + if (!error.empty()) { + ldout(cct, 0) << " cephx_calc_client_server_challenge error: " << error << dendl; + ret = -EPERM; + break; + } + + ldout(cct, 20) << " checking key: req.key=" << hex << req.key + << " expected_key=" << expected_key << dec << dendl; + if (req.key != expected_key) { + ldout(cct, 0) << " unexpected key: req.key=" << hex << req.key + << " expected_key=" << expected_key << dec << dendl; + ret = -EPERM; + break; + } + + CryptoKey session_key; + CephXSessionAuthInfo info; + bool should_enc_ticket = false; + + EntityAuth eauth; + if (key_server->get_auth(entity_name, eauth) < 0) { + ret = -EPERM; + break; + } + CephXServiceTicketInfo old_ticket_info; + + if (cephx_decode_ticket(cct, key_server, CEPH_ENTITY_TYPE_AUTH, + req.old_ticket, old_ticket_info)) { + global_id = old_ticket_info.ticket.global_id; + ldout(cct, 10) << "decoded old_ticket with global_id=" << global_id << dendl; + should_enc_ticket = true; + } + + info.ticket.init_timestamps(ceph_clock_now(cct), cct->_conf->auth_mon_ticket_ttl); + info.ticket.name = entity_name; + info.ticket.global_id = global_id; + info.ticket.auid = eauth.auid; + info.validity += cct->_conf->auth_mon_ticket_ttl; + + if (auid) *auid = eauth.auid; + + key_server->generate_secret(session_key); + + info.session_key = session_key; + info.service_id = CEPH_ENTITY_TYPE_AUTH; + if (!key_server->get_service_secret(CEPH_ENTITY_TYPE_AUTH, info.service_secret, info.secret_id)) { + ldout(cct, 0) << " could not get service secret for auth subsystem" << dendl; + ret = -EIO; + break; + } + + vector info_vec; + info_vec.push_back(info); + + build_cephx_response_header(cephx_header.request_type, 0, result_bl); + if (!cephx_build_service_ticket_reply(cct, eauth.key, info_vec, should_enc_ticket, + old_ticket_info.session_key, result_bl)) { + ret = -EIO; + } + + if (!key_server->get_service_caps(entity_name, CEPH_ENTITY_TYPE_MON, caps)) { + ldout(cct, 0) << " could not get mon caps for " << entity_name << dendl; + } + } + break; + + case CEPHX_GET_PRINCIPAL_SESSION_KEY: + { + ldout(cct, 10) << "handle_request get_principal_session_key" << dendl; + + bufferlist tmp_bl; + CephXServiceTicketInfo auth_ticket_info; + if (!cephx_verify_authorizer(cct, key_server, indata, auth_ticket_info, tmp_bl)) { + ret = -EPERM; + break; + } + + CephXServiceTicketRequest ticket_req; + ::decode(ticket_req, indata); + ldout(cct, 10) << " ticket_req.keys = " << ticket_req.keys << dendl; + + ret = 0; + vector info_vec; + for (uint32_t service_id = 1; service_id <= ticket_req.keys; service_id <<= 1) { + if (ticket_req.keys & service_id) { + ldout(cct, 10) << " adding key for service " << ceph_entity_type_name(service_id) << dendl; + CephXSessionAuthInfo info; + int r = key_server->build_session_auth_info(service_id, auth_ticket_info, info); + if (r < 0) { + ret = r; + break; + } + info.validity += cct->_conf->auth_service_ticket_ttl; + info_vec.push_back(info); + } + } + CryptoKey no_key; + build_cephx_response_header(cephx_header.request_type, ret, result_bl); + cephx_build_service_ticket_reply(cct, auth_ticket_info.session_key, info_vec, false, no_key, result_bl); + } + break; + + case CEPHX_GET_ROTATING_KEY: + { + ldout(cct, 10) << "handle_request getting rotating secret for " << entity_name << dendl; + build_cephx_response_header(cephx_header.request_type, 0, result_bl); + key_server->get_rotating_encrypted(entity_name, result_bl); + ret = 0; + } + break; + + default: + ldout(cct, 10) << "handle_request unknown op " << cephx_header.request_type << dendl; + return -EINVAL; + } + return ret; +} + +void CephxServiceHandler::build_cephx_response_header(int request_type, int status, bufferlist& bl) +{ + struct CephXResponseHeader header; + header.request_type = request_type; + header.status = status; + ::encode(header, bl); +} diff --git a/ceph/src/auth/cephx/CephxServiceHandler.h b/ceph/src/auth/cephx/CephxServiceHandler.h new file mode 100644 index 00000000..3649d3b5 --- /dev/null +++ b/ceph/src/auth/cephx/CephxServiceHandler.h @@ -0,0 +1,37 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_CEPHXSERVICEHANDLER_H +#define CEPH_CEPHXSERVICEHANDLER_H + +#include "../AuthServiceHandler.h" +#include "../Auth.h" + +class KeyServer; + +class CephxServiceHandler : public AuthServiceHandler { + KeyServer *key_server; + uint64_t server_challenge; + +public: + CephxServiceHandler(CephContext *cct_, KeyServer *ks) + : AuthServiceHandler(cct_), key_server(ks), server_challenge(0) {} + ~CephxServiceHandler() {} + + int start_session(EntityName& name, bufferlist::iterator& indata, bufferlist& result_bl, AuthCapsInfo& caps); + int handle_request(bufferlist::iterator& indata, bufferlist& result_bl, uint64_t& global_id, AuthCapsInfo& caps, uint64_t *auid = NULL); + void build_cephx_response_header(int request_type, int status, bufferlist& bl); +}; + +#endif diff --git a/ceph/src/auth/cephx/CephxSessionHandler.cc b/ceph/src/auth/cephx/CephxSessionHandler.cc new file mode 100644 index 00000000..b2d402d2 --- /dev/null +++ b/ceph/src/auth/cephx/CephxSessionHandler.cc @@ -0,0 +1,139 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "CephxSessionHandler.h" +#include "CephxProtocol.h" + +#include +#include + +#include "common/config.h" +#include "include/assert.h" +#include "include/ceph_features.h" + +#define dout_subsys ceph_subsys_auth + +int CephxSessionHandler::sign_message(Message *m) +{ + // If runtime signing option is off, just return success without signing. + if (!cct->_conf->cephx_sign_messages) { + return 0; + } + bufferlist bl_plaintext, bl_encrypted; + ceph_msg_header header = m->get_header(); + std::string error; + + ceph_msg_footer& en_footer = m->get_footer(); + + ::encode(header.crc, bl_plaintext); + ::encode(en_footer.front_crc, bl_plaintext); + ::encode(en_footer.middle_crc, bl_plaintext); + ::encode(en_footer.data_crc, bl_plaintext); + + ldout(cct, 10) << "sign_message: seq # " << header.seq << " CRCs are: header " << header.crc + << " front " << en_footer.front_crc << " middle " << en_footer.middle_crc + << " data " << en_footer.data_crc << dendl; + + if (encode_encrypt(cct, bl_plaintext, key, bl_encrypted, error)) { + ldout(cct, 0) << "error encrypting message signature: " << error << dendl; + ldout(cct, 0) << "no signature put on message" << dendl; + return SESSION_SIGNATURE_FAILURE; + } + + bufferlist::iterator ci = bl_encrypted.begin(); + // Skip the magic number up front. PLR + ci.advance(4); + ::decode(en_footer.sig, ci); + + // There's potentially an issue with whether the encoding and decoding done here will work + // properly when a big endian and little endian machine are talking. We think it's OK, + // but it should be tested to be sure. PLR + + // Receiver won't trust this flag to decide if msg should have been signed. It's primarily + // to debug problems where sender and receiver disagree on need to sign msg. PLR + en_footer.flags = (unsigned)en_footer.flags | CEPH_MSG_FOOTER_SIGNED; + messages_signed++; + ldout(cct, 20) << "Putting signature in client message(seq # " << header.seq << "): sig = " << en_footer.sig << dendl; + return 0; +} + +int CephxSessionHandler::check_message_signature(Message *m) +{ + // If runtime signing option is off, just return success without checking signature. + if (!cct->_conf->cephx_sign_messages) { + return 0; + } + + bufferlist bl_plaintext, bl_ciphertext; + std::string sig_error; + ceph_msg_header& header = m->get_header(); + ceph_msg_footer& footer = m->get_footer(); + + if ((features & CEPH_FEATURE_MSG_AUTH) == 0) { + // it's fine, we didn't negotiate this feature. + return 0; + } + + signatures_checked++; + + ldout(cct, 10) << "check_message_signature: seq # = " << m->get_seq() << " front_crc_ = " << footer.front_crc + << " middle_crc = " << footer.middle_crc << " data_crc = " << footer.data_crc << dendl; + ::encode(header.crc, bl_plaintext); + ::encode(footer.front_crc, bl_plaintext); + ::encode(footer.middle_crc, bl_plaintext); + ::encode(footer.data_crc, bl_plaintext); + + // Encrypt the buffer containing the checksums to calculate the signature. PLR + if (encode_encrypt(cct, bl_plaintext, key, bl_ciphertext, sig_error)) { + ldout(cct, 0) << "error in encryption for checking message signature: " << sig_error << dendl; + return (SESSION_SIGNATURE_FAILURE); + } + + bufferlist::iterator ci = bl_ciphertext.begin(); + // Skip the magic number at the front. PLR + ci.advance(4); + uint64_t sig_check; + ::decode(sig_check, ci); + + // There's potentially an issue with whether the encoding and decoding done here will work + // properly when a big endian and little endian machine are talking. We think it's OK, + // but it should be tested to be sure. PLR + + if (sig_check != footer.sig) { + // Should have been signed, but signature check failed. PLR + if (!(footer.flags & CEPH_MSG_FOOTER_SIGNED)) { + ldout(cct, 0) << "SIGN: MSG " << header.seq << " Sender did not set CEPH_MSG_FOOTER_SIGNED." << dendl; + } + ldout(cct, 0) << "SIGN: MSG " << header.seq << " Message signature does not match contents." << dendl; + ldout(cct, 0) << "SIGN: MSG " << header.seq << "Signature on message:" << dendl; + ldout(cct, 0) << "SIGN: MSG " << header.seq << " sig: " << footer.sig << dendl; + ldout(cct, 0) << "SIGN: MSG " << header.seq << "Locally calculated signature:" << dendl; + ldout(cct, 0) << "SIGN: MSG " << header.seq << " sig_check:" << sig_check << dendl; + + // For the moment, printing an error message to the log and returning failure is sufficient. + // In the long term, we should probably have code parsing the log looking for this kind + // of security failure, particularly when there are large numbers of them, since the latter + // is a potential sign of an attack. PLR + + signatures_failed++; + ldout(cct, 0) << "Signature failed." << dendl; + return (SESSION_SIGNATURE_FAILURE); + } + + // If we get here, the signature checked. PLR + signatures_matched++; + + return 0; +} + diff --git a/ceph/src/auth/cephx/CephxSessionHandler.h b/ceph/src/auth/cephx/CephxSessionHandler.h new file mode 100644 index 00000000..52a112e2 --- /dev/null +++ b/ceph/src/auth/cephx/CephxSessionHandler.h @@ -0,0 +1,49 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include "../AuthSessionHandler.h" +#include "../Auth.h" + +class CephContext; + +class CephxSessionHandler : public AuthSessionHandler { + uint64_t features; + +public: + CephxSessionHandler(CephContext *cct_, CryptoKey session_key, uint64_t features) + : AuthSessionHandler(cct_, CEPH_AUTH_CEPHX, session_key), + features(features) {} + ~CephxSessionHandler() {} + + bool no_security() { + return false; + } + + int sign_message(Message *m); + + int check_message_signature(Message *m) ; + + // Cephx does not currently encrypt messages, so just return 0 if called. PLR + + int encrypt_message(Message *m) { + return 0; + } + + int decrypt_message(Message *m) { + return 0; + } + +}; + diff --git a/ceph/src/auth/none/AuthNoneAuthorizeHandler.cc b/ceph/src/auth/none/AuthNoneAuthorizeHandler.cc new file mode 100644 index 00000000..8b55e9e9 --- /dev/null +++ b/ceph/src/auth/none/AuthNoneAuthorizeHandler.cc @@ -0,0 +1,47 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2009-2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "AuthNoneAuthorizeHandler.h" +#include "common/debug.h" + +#define dout_subsys ceph_subsys_auth + +bool AuthNoneAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, +uint64_t *auid) +{ + bufferlist::iterator iter = authorizer_data.begin(); + + try { + __u8 struct_v = 1; + ::decode(struct_v, iter); + ::decode(entity_name, iter); + ::decode(global_id, iter); + } catch (const buffer::error &err) { + ldout(cct, 0) << "AuthNoneAuthorizeHandle::verify_authorizer() failed to decode" << dendl; + return false; + } + + caps_info.allow_all = true; + + return true; +} + +// Return type of crypto used for this session's data; for none, no crypt used + +int AuthNoneAuthorizeHandler::authorizer_session_crypto() +{ + return SESSION_CRYPTO_NONE; +} diff --git a/ceph/src/auth/none/AuthNoneAuthorizeHandler.h b/ceph/src/auth/none/AuthNoneAuthorizeHandler.h new file mode 100644 index 00000000..cc00d3d6 --- /dev/null +++ b/ceph/src/auth/none/AuthNoneAuthorizeHandler.h @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHNONEAUTHORIZEHANDLER_H +#define CEPH_AUTHNONEAUTHORIZEHANDLER_H + +#include "../AuthAuthorizeHandler.h" + +class CephContext; + +struct AuthNoneAuthorizeHandler : public AuthAuthorizeHandler { + bool verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid=NULL); + int authorizer_session_crypto(); +}; + + + +#endif diff --git a/ceph/src/auth/none/AuthNoneClientHandler.h b/ceph/src/auth/none/AuthNoneClientHandler.h new file mode 100644 index 00000000..203687e4 --- /dev/null +++ b/ceph/src/auth/none/AuthNoneClientHandler.h @@ -0,0 +1,56 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHNONECLIENTHANDLER_H +#define CEPH_AUTHNONECLIENTHANDLER_H + +#include "../AuthClientHandler.h" +#include "AuthNoneProtocol.h" + +class CephContext; + +class AuthNoneClientHandler : public AuthClientHandler { +public: + AuthNoneClientHandler(CephContext *cct_, RotatingKeyRing *rkeys) + : AuthClientHandler(cct_) {} + + void reset() { } + + void prepare_build_request() {} + int build_request(bufferlist& bl) const { return 0; } + int handle_response(int ret, bufferlist::iterator& iter) { return 0; } + bool build_rotating_request(bufferlist& bl) const { return false; } + + int get_protocol() const { return CEPH_AUTH_NONE; } + + AuthAuthorizer *build_authorizer(uint32_t service_id) const { + RWLock::RLocker l(lock); + AuthNoneAuthorizer *auth = new AuthNoneAuthorizer(); + if (auth) { + auth->build_authorizer(cct->_conf->name, global_id); + } + return auth; + } + + bool need_tickets() { return false; } + + void set_global_id(uint64_t id) { + RWLock::WLocker l(lock); + global_id = id; + } +private: + void validate_tickets() {} +}; + +#endif diff --git a/ceph/src/auth/none/AuthNoneProtocol.h b/ceph/src/auth/none/AuthNoneProtocol.h new file mode 100644 index 00000000..a3c98912 --- /dev/null +++ b/ceph/src/auth/none/AuthNoneProtocol.h @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHNONEPROTOCOL_H +#define CEPH_AUTHNONEPROTOCOL_H + +#include "../Auth.h" + +struct AuthNoneAuthorizer : public AuthAuthorizer { + AuthNoneAuthorizer() : AuthAuthorizer(CEPH_AUTH_NONE) { } + bool build_authorizer(const EntityName &ename, uint64_t global_id) { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(ename, bl); + ::encode(global_id, bl); + return 0; + } + bool verify_reply(bufferlist::iterator& reply) { return true; } +}; + +#endif diff --git a/ceph/src/auth/none/AuthNoneServiceHandler.h b/ceph/src/auth/none/AuthNoneServiceHandler.h new file mode 100644 index 00000000..1c37d79e --- /dev/null +++ b/ceph/src/auth/none/AuthNoneServiceHandler.h @@ -0,0 +1,41 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHNONESERVICEHANDLER_H +#define CEPH_AUTHNONESERVICEHANDLER_H + +#include "../AuthServiceHandler.h" +#include "../Auth.h" + +class CephContext; + +class AuthNoneServiceHandler : public AuthServiceHandler { +public: + AuthNoneServiceHandler(CephContext *cct_) + : AuthServiceHandler(cct_) {} + ~AuthNoneServiceHandler() {} + + int start_session(EntityName& name, bufferlist::iterator& indata, bufferlist& result_bl, AuthCapsInfo& caps) { + entity_name = name; + caps.allow_all = true; + return CEPH_AUTH_NONE; + } + int handle_request(bufferlist::iterator& indata, bufferlist& result_bl, uint64_t& global_id, AuthCapsInfo& caps, uint64_t *auid = NULL) { + assert(0); // shouldn't get called + return 0; + } + void build_cephx_response_header(int request_type, int status, bufferlist& bl) { } +}; + +#endif diff --git a/ceph/src/auth/none/AuthNoneSessionHandler.h b/ceph/src/auth/none/AuthNoneSessionHandler.h new file mode 100644 index 00000000..85261bf0 --- /dev/null +++ b/ceph/src/auth/none/AuthNoneSessionHandler.h @@ -0,0 +1,52 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "../AuthSessionHandler.h" +#include "../Auth.h" + +#define dout_subsys ceph_subsys_auth + +class CephContext; + +class AuthNoneSessionHandler : public AuthSessionHandler { +public: + AuthNoneSessionHandler(CephContext *cct_, CryptoKey session_key) + : AuthSessionHandler(cct_, CEPH_AUTH_NONE, session_key) {} + ~AuthNoneSessionHandler() {} + + bool no_security() { + return true; + } + + // The None suite neither signs nor encrypts messages, so these functions just return success. + // Since nothing was signed or encrypted, don't increment the stats. PLR + + int sign_message(Message *m) { + return 0; + } + + int check_message_signature(Message *m) { + return 0; + } + + int encrypt_message(Message *m) { + return 0; + } + + int decrypt_message(Message *m) { + return 0; + } + +}; + diff --git a/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc new file mode 100644 index 00000000..de66058c --- /dev/null +++ b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc @@ -0,0 +1,35 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2009-2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "AuthUnknownAuthorizeHandler.h" +#include "common/debug.h" + +#define dout_subsys ceph_subsys_auth + +bool AuthUnknownAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, +uint64_t *auid) +{ + // For unknown authorizers, there's nothing to verify. They're "OK" by definition. PLR + + return true; +} + +// Return type of crypto used for this session's data; for unknown, no crypt used + +int AuthUnknownAuthorizeHandler::authorizer_session_crypto() +{ + return SESSION_CRYPTO_NONE; +} diff --git a/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h new file mode 100644 index 00000000..0156fbd8 --- /dev/null +++ b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHUNKNOWNAUTHORIZEHANDLER_H +#define CEPH_AUTHUNKNOWNAUTHORIZEHANDLER_H + +#include "../AuthAuthorizeHandler.h" + +class CephContext; + +struct AuthUnknownAuthorizeHandler : public AuthAuthorizeHandler { + bool verify_authorizer(CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid=NULL); + int authorizer_session_crypto(); +}; + + + +#endif diff --git a/ceph/src/auth/unknown/AuthUnknownClientHandler.h b/ceph/src/auth/unknown/AuthUnknownClientHandler.h new file mode 100644 index 00000000..088b816c --- /dev/null +++ b/ceph/src/auth/unknown/AuthUnknownClientHandler.h @@ -0,0 +1,56 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHUNKNOWNCLIENTHANDLER_H +#define CEPH_AUTHUNKNOWNCLIENTHANDLER_H + +#include "../AuthClientHandler.h" +#include "AuthUnknownProtocol.h" + +class CephContext; + +class AuthUnknownClientHandler : public AuthClientHandler { +public: + AuthUnknownClientHandler(CephContext *cct_, RotatingKeyRing *rkeys) + : AuthClientHandler(cct_) {} + + void reset() { } + + void prepare_build_request() {} + int build_request(bufferlist& bl) const { return 0; } + int handle_response(int ret, bufferlist::iterator& iter) { return 0; } + bool build_rotating_request(bufferlist& bl) const { return false; } + + int get_protocol() const { return CEPH_AUTH_UNKNOWN; } + + AuthAuthorizer *build_authorizer(uint32_t service_id) const { + RWLock::RLocker l(lock); + AuthUnknownAuthorizer *auth = new AuthUnknownAuthorizer(); + if (auth) { + auth->build_authorizer(cct->_conf->name, global_id); + } + return auth; + } + + bool need_tickets() { return false; } + + void set_global_id(uint64_t id) { + RWLock::WLocker l(lock); + global_id = id; + } +private: + void validate_tickets() { } +}; + +#endif diff --git a/ceph/src/auth/unknown/AuthUnknownProtocol.h b/ceph/src/auth/unknown/AuthUnknownProtocol.h new file mode 100644 index 00000000..94504fb9 --- /dev/null +++ b/ceph/src/auth/unknown/AuthUnknownProtocol.h @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHUNKNOWNPROTOCOL_H +#define CEPH_AUTHUNKNOWNPROTOCOL_H + +#include "../Auth.h" + +struct AuthUnknownAuthorizer : public AuthAuthorizer { + AuthUnknownAuthorizer() : AuthAuthorizer(CEPH_AUTH_UNKNOWN) { } + bool build_authorizer(const EntityName &ename, uint64_t global_id) { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(ename, bl); + ::encode(global_id, bl); + return 0; + } + bool verify_reply(bufferlist::iterator& reply) { return true; } +}; + +#endif diff --git a/ceph/src/auth/unknown/AuthUnknownServiceHandler.h b/ceph/src/auth/unknown/AuthUnknownServiceHandler.h new file mode 100644 index 00000000..db039b69 --- /dev/null +++ b/ceph/src/auth/unknown/AuthUnknownServiceHandler.h @@ -0,0 +1,39 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_AUTHUNKNOWNSERVICEHANDLER_H +#define CEPH_AUTHUNKNOWNSERVICEHANDLER_H + +#include "../AuthServiceHandler.h" +#include "../Auth.h" + +class CephContext; + +class AuthUnknownServiceHandler : public AuthServiceHandler { +public: + AuthUnknownServiceHandler(CephContext *cct_) + : AuthServiceHandler(cct_) {} + ~AuthUnknownServiceHandler() {} + + int start_session(EntityName& name, bufferlist::iterator& indata, bufferlist& result_bl, AuthCapsInfo& caps) { + return CEPH_AUTH_UNKNOWN; + } + int handle_request(bufferlist::iterator& indata, bufferlist& result_bl, uint64_t& global_id, AuthCapsInfo& caps, uint64_t *auid = NULL) { + assert(0); // shouldn't get called + return 0; + } + void build_cephx_response_header(int request_type, int status, bufferlist& bl) { } +}; + +#endif diff --git a/ceph/src/auth/unknown/AuthUnknownSessionHandler.h b/ceph/src/auth/unknown/AuthUnknownSessionHandler.h new file mode 100644 index 00000000..4f811ab5 --- /dev/null +++ b/ceph/src/auth/unknown/AuthUnknownSessionHandler.h @@ -0,0 +1,52 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "../AuthSessionHandler.h" +#include "../Auth.h" + +#define dout_subsys ceph_subsys_auth + +class CephContext; + +class AuthUnknownSessionHandler : public AuthSessionHandler { +public: + AuthUnknownSessionHandler(CephContext *cct_, CryptoKey session_key) + : AuthSessionHandler(cct_, CEPH_AUTH_UNKNOWN, session_key) {} + ~AuthUnknownSessionHandler() {} + + bool no_security() { + return true; + } + + // The Unknown suite neither signs nor encrypts messages, so these functions just return success. + // Since nothing was signed or encrypted, don't increment the stats. PLR + + int sign_message(Message *m) { + return 0; + } + + int check_message_signature(Message *m) { + return 0; + } + + int encrypt_message(Message *m) { + return 0; + } + + int decrypt_message(Message *m) { + return 0; + } + +}; + diff --git a/ceph/src/bash_completion/ceph b/ceph/src/bash_completion/ceph new file mode 100644 index 00000000..2ea53c60 --- /dev/null +++ b/ceph/src/bash_completion/ceph @@ -0,0 +1,60 @@ +# +# Ceph - scalable distributed file system +# +# Copyright (C) 2011 Wido den Hollander +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# + +_ceph() +{ + local cur prev + + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [[ ${cur} == -* ]] ; then + COMPREPLY=( $(compgen -W "--conf -c --name --id -m --version -s --status -w --watch -o --out-file -i --in-file" -- ${cur}) ) + return 0 + fi + + case "${prev}" in + -o | --out-file | -i | --in-file | --conf | -c) + COMPREPLY=( $(compgen -f ${cur}) ) + return 0 + ;; + -m) + COMPREPLY=( $(compgen -A hostname ${cur}) ) + return 0 + ;; + auth) + COMPREPLY=( $(compgen -W "list add del" -- ${cur}) ) + return 0 + ;; + pg) + COMPREPLY=( $(compgen -W "stat dump getmap map send_pg_creates scrub deep-scrub repair" -- ${cur}) ) + return 0 + ;; + osd) + COMPREPLY=( $(compgen -W "tell stat pool dump getmaxosd tree getmap getcrushmap lspools" -- ${cur}) ) + return 0 + ;; + mon) + COMPREPLY=( $(compgen -W "tell stat getmap" -- ${cur}) ) + return 0 + ;; + mds) + COMPREPLY=( $(compgen -W "tell stat stat getmap dump compat" -- ${cur}) ) + return 0 + ;; + *) + COMPREPLY=( $(compgen -W "osd mon mds pg auth" -- ${cur}) ) + return 0 + ;; + esac +} +complete -F _ceph ceph diff --git a/ceph/src/bash_completion/rados b/ceph/src/bash_completion/rados new file mode 100644 index 00000000..aa277c34 --- /dev/null +++ b/ceph/src/bash_completion/rados @@ -0,0 +1,40 @@ +# +# Ceph - scalable distributed file system +# +# Copyright (C) 2011 Wido den Hollander +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# + +_rados() +{ + local cur prev + + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [[ ${cur} == -* ]] ; then + COMPREPLY=( $(compgen -W "-c --conf -m -d -f -p --pool -b --snap -i -o --create" -- ${cur}) ) + return 0 + fi + + case "${prev}" in + --conf | -c | -o | -i) + COMPREPLY=( $(compgen -f ${cur}) ) + return 0 + ;; + -m) + COMPREPLY=( $(compgen -A hostname ${cur}) ) + return 0 + ;; + *) + COMPREPLY=( $(compgen -W "lspools mkpool rmpool df ls chown get put create rm listxattr getxattr setxattr rmxattr stat mapext lssnap mksnap rmsnap rollback bench" -- ${cur}) ) + return 0 + ;; + esac +} +complete -F _rados rados diff --git a/ceph/src/bash_completion/radosgw-admin b/ceph/src/bash_completion/radosgw-admin new file mode 100644 index 00000000..023a83f8 --- /dev/null +++ b/ceph/src/bash_completion/radosgw-admin @@ -0,0 +1,57 @@ +# +# Ceph - scalable distributed file system +# +# Copyright (C) 2011 Wido den Hollander +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# + +_radosgw_admin() +{ + local cur prev + + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [[ ${cur} == -* ]] ; then + COMPREPLY=( $(compgen -W "--uid --subuser --access-key --os-user --email --auth_uid --secret --os-secret --gen-access-key --gen-secret \ + --access --display-name --bucket --object --date --conf --name --id --version -s -w" -- ${cur}) ) + return 0 + fi + + case "${prev}" in + --conf | -c) + COMPREPLY=( $(compgen -f ${cur}) ) + return 0 + ;; + -m) + COMPREPLY=( $(compgen -A hostname ${cur}) ) + return 0 + ;; + user) + COMPREPLY=( $(compgen -W "create modify info rm" -- ${cur}) ) + return 0 + ;; + subuser) + COMPREPLY=( $(compgen -W "create modify rm" -- ${cur}) ) + return 0 + ;; + key) + COMPREPLY=( $(compgen -W "create rm" -- ${cur}) ) + return 0 + ;; + buckets) + COMPREPLY=( $(compgen -W "list unlink" -- ${cur}) ) + return 0 + ;; + *) + COMPREPLY=( $(compgen -W "user subuser key buckets policy log" -- ${cur}) ) + return 0 + ;; + esac +} +complete -F _radosgw_admin radosgw-admin diff --git a/ceph/src/bash_completion/rbd b/ceph/src/bash_completion/rbd new file mode 100644 index 00000000..7fe8bee7 --- /dev/null +++ b/ceph/src/bash_completion/rbd @@ -0,0 +1,44 @@ +# +# Ceph - scalable distributed file system +# +# Copyright (C) 2011 Wido den Hollander +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# + +_rbd() +{ + local cur prev + + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [[ ${cur} == -* ]] ; then + COMPREPLY=( $(compgen -W "-c --conf -m -d -f -p --pool --snap -i -o --image --dest --dest-pool --path --size --id --keyfile" -- ${cur}) ) + return 0 + fi + + case "${prev}" in + --conf | -c | --path | --keyfile | --keyring) + COMPREPLY=( $(compgen -f ${cur}) ) + return 0 + ;; + -m) + COMPREPLY=( $(compgen -A hostname ${cur}) ) + return 0 + ;; + snap) + COMPREPLY=( $(compgen -W "ls create rollback rm" -- ${cur}) ) + return 0 + ;; + *) + COMPREPLY=( $(compgen -W "ls list info create resize rm export import cp copy mv rename snap watch map unmap showmapped" -- ${cur}) ) + return 0 + ;; + esac +} +complete -F _rbd rbd diff --git a/ceph/src/brag/Makefile.am b/ceph/src/brag/Makefile.am new file mode 100644 index 00000000..35c735ba --- /dev/null +++ b/ceph/src/brag/Makefile.am @@ -0,0 +1,3 @@ + +bin_SCRIPTS += brag/client/ceph-brag +EXTRA_DIST += brag/server brag/README.md brag/client diff --git a/ceph/src/brag/README.md b/ceph/src/brag/README.md new file mode 100644 index 00000000..574d7fdb --- /dev/null +++ b/ceph/src/brag/README.md @@ -0,0 +1,185 @@ +# Ceph-brag + +`ceph-brag` is going to be an anonymized cluster reporting tool designed to collect a "registry" of Ceph clusters for community knowledge. +This data will be displayed on a public web page using UUID by default, but users can claim their cluster and publish information about ownership if they so desire. + +For more information please visit: + +* [Blueprint](http://wiki.ceph.com/Planning/Blueprints/Firefly/Ceph-Brag) +* [CDS Etherpad](http://pad.ceph.com/p/cdsfirefly-ceph-brag) + +# Client + +## How to use: + +### Pre-requisites: +ceph-brag uses 'ceph' python script. Hence, before executing ceph-brag script ensure that ceph services are all running and 'ceph' script is in 'PATH' environment + +### Runtime instructions: +Run 'ceph-brag -h' to get the usage information of this tool. + +### Sample output: + + { + "cluster_creation_date": "2014-01-16 13:38:41.928551", + "uuid": "20679d0e-04b1-4004-8ee9-45ac271510e9", + "components_count": { + "num_data_bytes": 0, + "num_bytes_total": 1209312904, + "num_osds": 1, + "num_objects": 0, + "num_pgs": 192, + "num_pools": 3, + "num_mdss": 1, + "num_mons": 1 + }, + "crush_types": [ + { + "type": "osd" + "count": 2, + }, + { + "type": "rack" + "count": 1, + }, + { + "type": "host" + "count": 1, + }, + { + "type": "root" + "count": 1, + } + ], + "ownership": { + "organization": "eNovance", + "description": "Use case1", + "email": "mail@enovance.com", + "name": "Cluster1" + }, + "pool_metadata": [ + { + "size": 3, + "id": 0, + "type": 1 + }, + { + "size": 3, + "id": 1, + "type": 1 + }, + { + "size": 3, + "id": 2, + "name": 1 + } + ], + "sysinfo": { + "kernel_types": [ + { + "count": 1, + "type": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012" + } + ], + "cpu_archs": [ + { + "count": 1, + "arch": "x86_64" + } + ], + "cpus": [ + { + "count": 1, + "cpu": "Intel Xeon E312xx (Sandy Bridge)" + } + ], + "kernel_versions": [ + { + "count": 1, + "version": "3.2.0-23-virtual" + } + ], + "ceph_versions": [ + { + "count": 1, + "version": "0.75-229-g4050eae(4050eae32cd77a1c210ca11d0f12c74daecb1bd3)" + } + ], + "os_info": [ + { + "count": 1, + "os": "Linux" + } + ], + "distros": [ + { + "count": 1, + "distro": "Ubuntu 12.04 precise (Ubuntu 12.04 LTS)" + } + ] + } + } + + +# Server + +## Info +The ceph-brag server code is a python based web application. + +## How to use + +### Prerequisites +* [pecan](http://pecanpy.org) is the web framework that is used by this application. +* [sqlalchemy](www.sqlalchemy.org) is the ORM that is used by this application + +### How to deploy +* [Common recipes to deploy](http://pecan.readthedocs.org/en/latest/deployment.html#common-recipes) +* Modify server/config.py:sqlalchemy['url'] to point the correct database connection + +## URLs +Following are the REST urls that are implemented with 'url-prefix' being the mount point for the WSGI script + +### GET + +##### * GET /url-prefix/ +Returns the list of clusters that are registered so far. +Outputs - On success application/json of the following format is returned + + [ + { + "num_versions": 3, + "cluster_creation_date": "2014-01-16 13:38:41.928551", + "uuid": "20679d0e-04b1-4004-8ee9-45ac271510e9", + "cluster_name": "Cluster1", + "organization": "eNovance", + "email": "mail@enovance.com" + }, + ... + ] + +##### * GET /url-prefix/UUID +Returns the list of version information for a particular UUID. +Outputs - On success application/json of the following format is returned + + [ + { + "version_number": 1, + "version_date": "2014-02-10 10:17:56.283499" + }, + ... + ] + +##### * GET /url-prefix/UUID/version\_number +Returns the entire brag report as mentioned in client's sample output for a particular version of a UUID + +### PUT + +##### * PUT /url-prefix +Uploads the brag report and creates a new version for the UUID mentioned in the payload + +### DELETE + +##### * DELETE /url-prefix?uuid=xxxx +Deletes all the versions of a cluster whose UUID is sent as a parameter + + diff --git a/ceph/src/brag/client/ceph-brag b/ceph/src/brag/client/ceph-brag new file mode 100755 index 00000000..9ae8e5ac --- /dev/null +++ b/ceph/src/brag/client/ceph-brag @@ -0,0 +1,521 @@ +#!/usr/bin/env python + +import subprocess +import uuid +import re +import json +import sys +import ast +import requests +from operator import itemgetter +from heapq import nlargest +from itertools import repeat, ifilter + + +CLUSTER_UUID_NAME='cluster-uuid' +CLUSTER_OWNERSHIP_NAME='cluster-ownership' + + +class Counter(dict): + '''Dict subclass for counting hashable objects. Sometimes called a bag + or multiset. Elements are stored as dictionary keys and their counts + are stored as dictionary values. + + >>> Counter('zyzygy') + Counter({'y': 3, 'z': 2, 'g': 1}) + + ''' + + def __init__(self, iterable=None, **kwds): + '''Create a new, empty Counter object. And if given, count elements + from an input iterable. Or, initialize the count from another mapping + of elements to their counts. + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping + >>> c = Counter(a=4, b=2) # a new counter from keyword args + + ''' + self.update(iterable, **kwds) + + def __missing__(self, key): + return 0 + + def most_common(self, n=None): + '''List the n most common elements and their counts from the most + common to the least. If n is None, then list all element counts. + + >>> Counter('abracadabra').most_common(3) + [('a', 5), ('r', 2), ('b', 2)] + + ''' + if n is None: + return sorted(self.iteritems(), key=itemgetter(1), reverse=True) + return nlargest(n, self.iteritems(), key=itemgetter(1)) + + def elements(self): + '''Iterator over elements repeating each as many times as its count. + + >>> c = Counter('ABCABC') + >>> sorted(c.elements()) + ['A', 'A', 'B', 'B', 'C', 'C'] + + If an element's count has been set to zero or is a negative number, + elements() will ignore it. + + ''' + for elem, count in self.iteritems(): + for _ in repeat(None, count): + yield elem + + # Override dict methods where the meaning changes for Counter objects. + + @classmethod + def fromkeys(cls, iterable, v=None): + raise NotImplementedError( + 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') + + def update(self, iterable=None, **kwds): + '''Like dict.update() but add counts instead of replacing them. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.update('witch') # add elements from another iterable + >>> d = Counter('watch') + >>> c.update(d) # add elements from another counter + >>> c['h'] # four 'h' in which, witch, and watch + 4 + + ''' + if iterable is not None: + if hasattr(iterable, 'iteritems'): + if self: + self_get = self.get + for elem, count in iterable.iteritems(): + self[elem] = self_get(elem, 0) + count + else: + dict.update(self, iterable) # fast path when counter is empty + else: + self_get = self.get + for elem in iterable: + self[elem] = self_get(elem, 0) + 1 + if kwds: + self.update(kwds) + + def copy(self): + 'Like dict.copy() but returns a Counter instance instead of a dict.' + return Counter(self) + + def __delitem__(self, elem): + 'Like dict.__delitem__() but does not raise KeyError for missing values.' + if elem in self: + dict.__delitem__(self, elem) + + def __repr__(self): + if not self: + return '%s()' % self.__class__.__name__ + items = ', '.join(map('%r: %r'.__mod__, self.most_common())) + return '%s({%s})' % (self.__class__.__name__, items) + + # Multiset-style mathematical operations discussed in: + # Knuth TAOCP Volume II section 4.6.3 exercise 19 + # and at http://en.wikipedia.org/wiki/Multiset + # + # Outputs guaranteed to only include positive counts. + # + # To strip negative and zero counts, add-in an empty counter: + # c += Counter() + + def __add__(self, other): + '''Add counts from two counters. + + >>> Counter('abbb') + Counter('bcc') + Counter({'b': 4, 'c': 2, 'a': 1}) + + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem in set(self) | set(other): + newcount = self[elem] + other[elem] + if newcount > 0: + result[elem] = newcount + return result + + def __sub__(self, other): + ''' Subtract count, but keep only results with positive counts. + + >>> Counter('abbbc') - Counter('bccd') + Counter({'b': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem in set(self) | set(other): + newcount = self[elem] - other[elem] + if newcount > 0: + result[elem] = newcount + return result + + def __or__(self, other): + '''Union is the maximum of value in either of the input counters. + + >>> Counter('abbb') | Counter('bcc') + Counter({'b': 3, 'c': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + _max = max + result = Counter() + for elem in set(self) | set(other): + newcount = _max(self[elem], other[elem]) + if newcount > 0: + result[elem] = newcount + return result + + def __and__(self, other): + ''' Intersection is the minimum of corresponding counts. + + >>> Counter('abbb') & Counter('bcc') + Counter({'b': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + _min = min + result = Counter() + if len(self) < len(other): + self, other = other, self + for elem in ifilter(self.__contains__, other): + newcount = _min(self[elem], other[elem]) + if newcount > 0: + result[elem] = newcount + return result + + +def run_command(cmd): + child = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (o, e) = child.communicate() + return (child.returncode, o, e) + + +def get_uuid(): + (rc,uid,e) = run_command(['ceph', 'config-key', 'get', CLUSTER_UUID_NAME]) + if rc is not 0: + #uuid is not yet set. + uid = str(uuid.uuid4()) + (rc, o, e) = run_command(['ceph', 'config-key', 'put', + CLUSTER_UUID_NAME, uid]) + if rc is not 0: + raise RuntimeError("\'ceph config-key put\' failed -" + e) + + return uid + +def bytes_pretty_to_raw(byte_count, byte_scale): + if byte_scale == 'kB': + return byte_count >> 10 + if byte_scale == 'MB': + return byte_count >> 20 + if byte_scale == 'GB': + return byte_count >> 30 + if byte_scale == 'TB': + return byte_count >> 40 + if byte_scale == 'PB': + return byte_count >> 50 + if byte_scale == 'EB': + return byte_count >> 60 + + return byte_count + +def get_nums(): + (rc, o, e) = run_command(['ceph', '-s', '-f', 'json']) + if rc is not 0: + raise RuntimeError("\'ceph -s\' failed - " + e) + + oj = json.loads(o) + num_mons = len(oj['monmap']['mons']) + num_osds = int(oj['osdmap']['osdmap']['num_in_osds']) + num_mdss = oj['mdsmap']['in'] + + pgmap = oj['pgmap'] + num_pgs = pgmap['num_pgs'] + num_data_bytes = pgmap['data_bytes'] + num_bytes_total = pgmap['bytes_total'] + + (rc, o, e) = run_command(['ceph', 'pg', 'dump', 'pools', '-f', 'json-pretty']) + if rc is not 0: + raise RuntimeError("\'ceph pg dump pools\' failed - " + e) + + pools = json.loads(o) + num_pools = len(pools) + num_objs = 0 + for p in pools: + num_objs += p['stat_sum']['num_objects'] + + nums = {'num_mons':num_mons, + 'num_osds':num_osds, + 'num_mdss':num_mdss, + 'num_pgs':num_pgs, + 'num_data_bytes':num_data_bytes, + 'num_bytes_total':num_bytes_total, + 'num_pools':num_pools, + 'num_objects':num_objs} + return nums + +def get_crush_types(): + (rc, o, e) = run_command(['ceph', 'osd', 'crush', 'dump']) + if rc is not 0: + raise RuntimeError("\'ceph osd crush dump\' failed - " + e) + + crush_dump = json.loads(o) + if crush_dump['types'] is None: + raise RuntimeError("\'types\' item missing in \'ceph osd crush dump\'") + + crush_types = {} + for t in crush_dump['types']: + crush_types[t['type_id']] = t['name'] + + types_list = [] + for bucket in crush_dump['buckets']: + types_list.append(bucket['type_id']) + + crush_map = [] + types_counter = Counter(types_list) + append = lambda t,c: crush_map.append({'type':t, 'count':c}) + for id,count in types_counter.items(): + append(crush_types[id], + count) + + if 'devices' in crush_dump: + append('devices', len(crush_dump['devices'])) + + return crush_map + +def get_osd_dump_info(): + (rc, o, e) = run_command(['ceph', 'osd', 'dump', '-f', 'json']) + if rc is not 0: + raise RuntimeError("\'ceph osd dump\' failed - " + e) + + pool_meta = [] + oj = json.loads(o) + proc = lambda x: {'id':x['pool'], 'type':x['type'], 'size':x['size']} + for p in oj['pools']: + pool_meta.append(proc(p)) + + return oj['created'], pool_meta + +def get_sysinfo(max_osds): + count = 0 + osd_metadata_available = False + + os = {} + kern_version = {} + kern_description = {} + distro = {} + cpu = {} + arch = {} + ceph_version = {} + + incr = lambda a,k: 1 if k not in a else a[k]+1 + while count < max_osds: + meta = {'id':count} + (rc, o, e) = run_command(['ceph', 'osd', 'metadata', str(count)]) + if rc is 0: + if osd_metadata_available is False: + osd_metadata_available = True + + jmeta = json.loads(o) + + version = jmeta['ceph_version'].split() + cv = version[2] + if (len(version) > 3): + cv += version[3] + + ceph_version[cv] = incr(ceph_version, cv) + os[jmeta['os']] = incr(os, jmeta['os']) + kern_version[jmeta['kernel_version']] = \ + incr(kern_version, jmeta['kernel_version']) + kern_description[jmeta['kernel_description']] = \ + incr(kern_description, jmeta['kernel_description']) + + try: + dstr = jmeta['distro'] + ' ' + dstr += jmeta['distro_version'] + ' ' + dstr += jmeta['distro_codename'] + ' (' + dstr += jmeta['distro_description'] + ')' + distro[dstr] = incr(distro, dstr) + except KeyError as ke: + pass + + cpu[jmeta['cpu']] = incr(cpu, jmeta['cpu']) + arch[jmeta['arch']] = incr(arch, jmeta['arch']) + + count = count + 1 + + sysinfo = {} + if osd_metadata_available is False: + print >> sys.stderr, "'ceph osd metadata' is not available at all" + return sysinfo + + def jsonify(type_count, name, type_name): + tmp = [] + for k, v in type_count.items(): + tmp.append({type_name:k, 'count':v}) + sysinfo[name] = tmp + + jsonify(os, 'os_info', 'os') + jsonify(kern_version, 'kernel_versions', 'version') + jsonify(kern_description, 'kernel_types', 'type') + jsonify(distro, 'distros', 'distro') + jsonify(cpu, 'cpus', 'cpu') + jsonify(arch, 'cpu_archs', 'arch') + jsonify(ceph_version, 'ceph_versions', 'version') + return sysinfo + +def get_ownership_info(): + (rc, o, e) = run_command(['ceph', 'config-key', 'get', + CLUSTER_OWNERSHIP_NAME]) + if rc is not 0: + return {} + + return ast.literal_eval(o) + +def output_json(): + out = {} + url = None + + out['uuid'] = get_uuid() + nums = get_nums() + num_osds = int(nums['num_osds']) + out['components_count'] = nums + out['crush_types'] = get_crush_types() + out['cluster_creation_date'], out['pool_metadata'] = get_osd_dump_info() + out['sysinfo'] = get_sysinfo(num_osds) + + owner = get_ownership_info() + if owner is not None: + out['ownership'] = owner + if 'url' in owner: + url = owner.pop('url') + + return json.dumps(out, indent=2, separators=(',', ': ')), url + +def describe_usage(): + print >> sys.stderr, "Usage:" + print >> sys.stderr, "======\n" + + print >> sys.stderr, sys.argv[0] + " [command-options]\n" + print >> sys.stderr, "commands:" + print >> sys.stderr, "publish - publish the brag report to the server" + print >> sys.stderr, "update-metadata - Update" + print >> sys.stderr, " ownership information for bragging" + print >> sys.stderr, "clear-metadata - Clear information set by update-metadata" + print >> sys.stderr, "unpublish --yes-i-am-shy - delete the brag report from the server" + print >> sys.stderr, "" + + print >> sys.stderr, "update-metadata options:" + print >> sys.stderr, "--name= - Name of the cluster" + print >> sys.stderr, "--organization= - Name of the organization" + print >> sys.stderr, "--email= - Email contact address" + print >> sys.stderr, "--description= - Reporting use-case" + print >> sys.stderr, "--url= - The URL that is used to publish and unpublish" + print >> sys.stderr, "" + +def update_metadata(): + info = {} + possibles = ['name', 'organization', 'email', 'description', 'url'] + + #get the existing values + info = get_ownership_info(); + + for index in range(2, len(sys.argv)): + mo = re.search("--(\S+)=(.*)", sys.argv[index]) + if not mo: + describe_usage() + return 22 + + k = mo.group(1) + v = mo.group(2) + + if k in possibles: + info[k] = v + else: + print >> sys.stderr, "Unexpect option --" + k + describe_usage() + return 22 + + (rc, o, e) = run_command(['ceph', 'config-key', 'put', + CLUSTER_OWNERSHIP_NAME, str(info)]) + return rc + +def clear_metadata(): + (rc, o, e) = run_command(['ceph', 'config-key', 'del', + CLUSTER_OWNERSHIP_NAME]) + return rc + +def publish(): + data, url = output_json() + if url is None: + print >> sys.stderr, "Cannot publish until a URL is set using update-metadata" + return 1 + + req = requests.put(url, data=data) + if req.status_code is not 201: + print >> sys.stderr, "Failed to publish, server responded with code " + str(req.status_code) + print >> sys.stderr, req.text + return 1 + + return 0 + +def unpublish(): + if len(sys.argv) <= 2 or sys.argv[2] != '--yes-i-am-shy': + print >> sys.stderr, "unpublish should be followed by --yes-i-am-shy" + return 22 + + fail = False + owner = get_ownership_info() + if owner is None: + fail = True + try: + url = owner['url'] + except KeyError as e: + fail = True + + if fail: + print >> sys.stderr, "URL is not updated yet" + return 1 + + uuid = get_uuid() + + params = {'uuid':uuid} + req = requests.delete(url, params=params) + if req.status_code is not 200: + print >> sys.stderr, "Failed to unpublish, server responsed with code " + str(req.status_code) + return 1 + + return 0 + +def main(): + if len(sys.argv) is 1: + print output_json()[0] + return 0 + elif sys.argv[1] == 'update-metadata': + return update_metadata() + elif sys.argv[1] == 'clear-metadata': + return clear_metadata() + elif sys.argv[1] == 'publish': + return publish() + elif sys.argv[1] == 'unpublish': + return unpublish() + else: + describe_usage() + return 22 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ceph/src/brag/server/MANIFEST.in b/ceph/src/brag/server/MANIFEST.in new file mode 100644 index 00000000..c922f11a --- /dev/null +++ b/ceph/src/brag/server/MANIFEST.in @@ -0,0 +1 @@ +recursive-include public * diff --git a/ceph/src/brag/server/app.wsgi b/ceph/src/brag/server/app.wsgi new file mode 100644 index 00000000..01dfc72a --- /dev/null +++ b/ceph/src/brag/server/app.wsgi @@ -0,0 +1,5 @@ +import os +from pecan.deploy import deploy + +cur_path = os.path.dirname(__file__) +application = deploy(cur_path + '/config.py') diff --git a/ceph/src/brag/server/ceph_brag.egg-info/PKG-INFO b/ceph/src/brag/server/ceph_brag.egg-info/PKG-INFO new file mode 100644 index 00000000..eeb2b1b7 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag.egg-info/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: ceph-brag +Version: 0.1 +Summary: UNKNOWN +Home-page: UNKNOWN +Author: UNKNOWN +Author-email: UNKNOWN +License: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN diff --git a/ceph/src/brag/server/ceph_brag.egg-info/SOURCES.txt b/ceph/src/brag/server/ceph_brag.egg-info/SOURCES.txt new file mode 100644 index 00000000..fc3b81db --- /dev/null +++ b/ceph/src/brag/server/ceph_brag.egg-info/SOURCES.txt @@ -0,0 +1,21 @@ +MANIFEST.in +config.py +setup.cfg +setup.py +ceph_brag/__init__.py +ceph_brag/app.py +ceph_brag/json.py +ceph_brag.egg-info/PKG-INFO +ceph_brag.egg-info/SOURCES.txt +ceph_brag.egg-info/dependency_links.txt +ceph_brag.egg-info/not-zip-safe +ceph_brag.egg-info/requires.txt +ceph_brag.egg-info/top_level.txt +ceph_brag/controllers/__init__.py +ceph_brag/controllers/root.py +ceph_brag/model/__init__.py +ceph_brag/model/db.py +ceph_brag/tests/__init__.py +ceph_brag/tests/config.py +ceph_brag/tests/test_functional.py +ceph_brag/tests/test_units.py \ No newline at end of file diff --git a/ceph/src/brag/server/ceph_brag.egg-info/dependency_links.txt b/ceph/src/brag/server/ceph_brag.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/ceph/src/brag/server/ceph_brag.egg-info/not-zip-safe b/ceph/src/brag/server/ceph_brag.egg-info/not-zip-safe new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/ceph/src/brag/server/ceph_brag.egg-info/requires.txt b/ceph/src/brag/server/ceph_brag.egg-info/requires.txt new file mode 100644 index 00000000..ea2fd7bf --- /dev/null +++ b/ceph/src/brag/server/ceph_brag.egg-info/requires.txt @@ -0,0 +1 @@ +pecan \ No newline at end of file diff --git a/ceph/src/brag/server/ceph_brag.egg-info/top_level.txt b/ceph/src/brag/server/ceph_brag.egg-info/top_level.txt new file mode 100644 index 00000000..9e1c53c8 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag.egg-info/top_level.txt @@ -0,0 +1 @@ +ceph_brag diff --git a/ceph/src/brag/server/ceph_brag/__init__.py b/ceph/src/brag/server/ceph_brag/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ceph/src/brag/server/ceph_brag/app.py b/ceph/src/brag/server/ceph_brag/app.py new file mode 100644 index 00000000..340c60d9 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/app.py @@ -0,0 +1,19 @@ +from pecan import make_app +from ceph_brag import model, json +from pecan.hooks import TransactionHook + +def setup_app(config): + + model.init_model() + app_conf = dict(config.app) + + return make_app( + app_conf.pop('root'), + logging=getattr(config, 'logging', {}), + hooks=[TransactionHook(model.start, + model.start, + model.commit, + model.rollback, + model.clear)], + **app_conf + ) diff --git a/ceph/src/brag/server/ceph_brag/controllers/__init__.py b/ceph/src/brag/server/ceph_brag/controllers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ceph/src/brag/server/ceph_brag/controllers/root.py b/ceph/src/brag/server/ceph_brag/controllers/root.py new file mode 100644 index 00000000..254ab86c --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/controllers/root.py @@ -0,0 +1,73 @@ +from pecan import expose, request, abort, response +from webob import exc +from pecan.rest import RestController +from ceph_brag.model import db +import sys, traceback + +class RootController(RestController): + def fail(self, status_code=200, msg="OK"): + response.status = status_code + return msg + + @expose('json') + def get(self, *args, **kwargs): + if len(args) is 0: + #return the list of uuids + try: + result = db.get_uuids() + except Exception as e: + return self.fail(500, msg="Internal Server Error") + elif len(args) is 1 or len(args) is 2 and args[1] == '': + #/uuid + try: + result = db.get_versions(args[0]) + except Exception as e: + return self.fail(status_code=500, msg="Internal Server Error") + + if result is None: + return self.fail(400, msg="Invalid UUID") + elif len(args) is 2 or len(args) is 3 and args[2] == '': + #/uuid/version_number + try: + result = db.get_brag(args[0], args[1]) + except Exception as e: + return self.fail(status_code=500, msg="Internal Server Error") + + if result is None: + return self.fail(status_code=400, msg="Invalid UUID,version combination") + else: + return self.fail(status_code=400, msg="Invalid args") + + return result + + @expose(content_type='application/json') + def put(self, *args, **kwargs): + try: + db.put_new_version(request.body) + except ValueError as ve: + return self.fail(status_code=422, msg="Improper payload") + except KeyError as ke: + msg = "Payload not as expected, some keys are missing" + return self.fail(status_code=422, msg=msg) + except Exception as e: + return self.fail(status_code=500, msg="Internal Server Error") + + response.status = 201 + return "CREATED" + + @expose() + def delete(self, *args, **kwargs): + if 'uuid' not in kwargs: + return self.fail(status_code=400, msg="Required uuid parameter") + + uuid = kwargs['uuid'] + try: + status = db.delete_uuid(uuid) + except Exception as e: + return self.fail(status_code=500, msg="Internal Server Error") + + if status is not None: + return self.fail(status_code=status['status'], msg=status['msg']) + + response.status=200 + return "DELETED" diff --git a/ceph/src/brag/server/ceph_brag/json.py b/ceph/src/brag/server/ceph_brag/json.py new file mode 100644 index 00000000..7f36eb6d --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/json.py @@ -0,0 +1,115 @@ +from pecan.jsonify import jsonify +from ceph_brag.model import db + +@jsonify.register(db.version_info) +def jsonify_version(vi): + return dict( + version_number=vi.version_number, + version_date=str(vi.version_date) + ) + +@jsonify.register(db.cluster_info) +def jsonify_cluster_info(ci): + return dict( + uuid=ci.uuid, + organization=ci.organization, + email=ci.contact_email, + cluster_name=ci.cluster_name, + cluster_creation_date=str(ci.cluster_creation_date), + num_versions=ci.num_versions + ) + +@jsonify.register(db.components_info) +def jsonify_components_info(comps): + return dict( + num_data_bytes=comps.num_data_bytes, + num_bytes_total=comps.num_bytes_total, + num_osds=comps.num_osds, + num_objects=comps.num_objects, + num_pgs=comps.num_pgs, + num_pools=comps.num_pools, + num_mdss=comps.num_mdss, + num_mons=comps.num_mons) + +@jsonify.register(db.crush_types) +def jsonify_crush_types(crush): + return dict(type=crush.crush_type, + count=crush.crush_count) + +@jsonify.register(db.pools_info) +def jsonify_pools_info(pool): + return dict(size=pool.pool_rep_size, + type=pool.pool_type, + id=pool.pool_id) + +@jsonify.register(db.os_info) +def jsonify_os_info(value): + return dict(os=value.os, + count=value.count) + +@jsonify.register(db.kernel_versions) +def jsonify_kernel_versions(value): + return dict(version=value.version, + count=value.count) + +@jsonify.register(db.kernel_types) +def jsonify_kernel_types(value): + return dict(type=value.type, + count=value.count) + +@jsonify.register(db.distros) +def jsonify_distros(value): + return dict(distro=value.distro, + count=value.count) + +@jsonify.register(db.cpus) +def jsonify_cpus(value): + return dict(cpu=value.cpu, + count=value.count) + +@jsonify.register(db.cpu_archs) +def jsonify_cpu_archs(value): + return dict(arch=value.arch, + count=value.count) + +@jsonify.register(db.ceph_versions) +def jsonify_ceph_versions(value): + return dict(version=value.version, + count=value.count) + +@jsonify.register(db.sysinfo) +def jsonify_sysinfo(value): + retval = {} + + if value.os: + retval['os_info'] = value.os + if value.kern_vers: + retval['kernel_versions'] = value.kern_vers + if value.kern_types: + retval['kernel_types'] = value.kern_types + if value.distros: + retval['distros'] = value.distros + if value.cpus: + retval['cpus'] = value.cpus + if value.cpu_archs: + retval['cpu_archs'] = value.cpu_archs + if value.ceph_vers: + retval['ceph_versions'] = value.ceph_vers + + return retval + +@jsonify.register(db.brag) +def jsonify_brag(b): + ownership = {'organization':b.ci.organization, + 'description':b.ci.description, + 'email':b.ci.contact_email, + 'name':b.ci.cluster_name + } + return dict(uuid=b.ci.uuid, + cluster_creation_date=str(b.ci.cluster_creation_date), + components_count=b.comps, + crush_types=b.crush, + ownership=ownership, + pool_metadata=b.pools, + sysinfo=b.sysinfo + ) diff --git a/ceph/src/brag/server/ceph_brag/model/__init__.py b/ceph/src/brag/server/ceph_brag/model/__init__.py new file mode 100644 index 00000000..664427d2 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/model/__init__.py @@ -0,0 +1,29 @@ +from sqlalchemy import create_engine +from pecan import conf # noqa +from db import Session, Base +import sys + +def create_from_conf(): + configs = dict(conf.sqlalchemy) + url = configs.pop('url') + return create_engine(url, **configs) + +def init_model(): + engine = create_from_conf() + conf.sqlalchemy.engine = engine + engine.connect() + #create the tables if not existing + Base.metadata.create_all(engine) + +def start(): + Session.bind = conf.sqlalchemy.engine + +def commit(): + Session.commit() + +def rollback(): + Session.rollback() + +def clear(): + Session.remove() + diff --git a/ceph/src/brag/server/ceph_brag/model/db.py b/ceph/src/brag/server/ceph_brag/model/db.py new file mode 100644 index 00000000..5dfc745d --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/model/db.py @@ -0,0 +1,284 @@ +import json +from datetime import datetime +from sqlalchemy.orm import sessionmaker, scoped_session +from sqlalchemy import Column, Integer, String, \ + DateTime, ForeignKey, BigInteger +from sqlalchemy import PrimaryKeyConstraint +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.declarative import declared_attr + +Base = declarative_base() +Session = scoped_session(sessionmaker()) + +class cluster_info(Base): + __tablename__ = 'cluster_info' + + index = Column(Integer, primary_key=True) + uuid = Column(String(36), unique=True) + organization = Column(String(64)) + contact_email = Column(String(32)) + cluster_name = Column(String(32)) + cluster_creation_date = Column(DateTime) + description = Column(String(32)) + num_versions = Column(Integer) + +class version_info(Base): + __tablename__ = 'version_info' + + index = Column(Integer, primary_key=True) + cluster_id = Column(ForeignKey('cluster_info.index')) + version_number = Column(Integer) + version_date = Column(DateTime) + +class components_info(Base): + __tablename__ = 'components_info' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + num_data_bytes = Column(BigInteger) + num_bytes_total = Column(BigInteger) + num_osds = Column(Integer) + num_objects = Column(Integer) + num_pgs = Column(Integer) + num_pools = Column(Integer) + num_mdss = Column(Integer) + num_mons = Column(Integer) + +class crush_types(Base): + __tablename__ = 'crush_types' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + crush_type = Column(String(16)) + crush_count = Column(Integer) + +class pools_info(Base): + __tablename__ = 'pools_info' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + pool_id = Column(Integer) + pool_type = Column(Integer) + pool_rep_size = Column(Integer) + +class os_info(Base): + __tablename__ = 'os_info' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + os = Column(String(16)) + count = Column(Integer) + +class kernel_versions(Base): + __tablename__ = 'kernel_versions' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + version = Column(String(16)) + count = Column(Integer) + +class kernel_types(Base): + __tablename__ = 'kernel_types' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + type = Column(String(64)) + count = Column(Integer) + +class distros(Base): + __tablename__ = 'distros' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + distro = Column(String(64)) + count = Column(Integer) + +class cpus(Base): + __tablename__ = 'cpus' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + cpu = Column(String(16)) + count = Column(Integer) + +class cpu_archs(Base): + __tablename__ = 'cpu_archs' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + arch = Column(String(16)) + count = Column(Integer) + +class ceph_versions(Base): + __tablename__ = 'ceph_versions' + + index = Column(Integer, primary_key=True) + vid = Column(ForeignKey('version_info.index')) + version = Column(String(16)) + count = Column(Integer) + +class sysinfo(object): + def __init__(self, vindex): + self.os = Session.query(os_info).filter_by(vid=vindex).all() + self.kern_vers = Session.query(kernel_versions).filter_by(vid=vindex).all() + self.kern_types = Session.query(kernel_types).filter_by(vid=vindex).all() + self.distros = Session.query(distros).filter_by(vid=vindex).all() + self.cpus = Session.query(cpus).filter_by(vid=vindex).all() + self.cpu_archs = Session.query(cpu_archs).filter_by(vid=vindex).all() + self.ceph_vers = Session.query(ceph_versions).filter_by(vid=vindex).all() + +class brag(object): + def __init__(self, uuid, version_number): + self.ci = Session.query(cluster_info).filter_by(uuid=uuid).first() + if self.ci is not None: + self.vi = Session.query(version_info).filter_by(cluster_id=self.ci.index, version_number=version_number).first() + + if self.ci is not None and self.vi is not None: + self.comps = Session.query(components_info).filter_by(vid=self.vi.index).first() + self.crush = Session.query(crush_types).filter_by(vid=self.vi.index).all() + self.pools = Session.query(pools_info).filter_by(vid=self.vi.index).all() + self.sysinfo = sysinfo(self.vi.index) + +def put_new_version(data): + info = json.loads(data) + def add_cluster_info(): + ci = Session.query(cluster_info).filter_by(uuid=info['uuid']).first() + if ci is None: + dt = datetime.strptime(info['cluster_creation_date'], "%Y-%m-%d %H:%M:%S.%f") + ci = cluster_info(uuid=info['uuid'], + organization=info['ownership']['organization'], + contact_email=info['ownership']['email'], + cluster_name=info['ownership']['name'], + description=info['ownership']['description'], + cluster_creation_date=dt, + num_versions=1) + Session.add(ci) + Session.commit() + else: + ci.num_versions += 1 + + return ci + + def add_version_info(ci): + vi = version_info(cluster_id=ci.index, + version_number=ci.num_versions, + version_date=datetime.now()) + Session.add(vi) + return vi + + def add_components_info(vi): + comps_count= info['components_count'] + comps_info = components_info(vid=vi.index, + num_data_bytes=comps_count['num_data_bytes'], + num_bytes_total=comps_count['num_bytes_total'], + num_osds=comps_count['num_osds'], + num_objects=comps_count['num_objects'], + num_pgs=comps_count['num_pgs'], + num_pools=comps_count['num_pools'], + num_mdss=comps_count['num_mdss'], + num_mons=comps_count['num_mons']) + Session.add(comps_info) + + def add_crush_types(vi): + for c in info['crush_types']: + Session.add(crush_types(vid=vi.index, + crush_type=c['type'], + crush_count=c['count'])) + + def add_pools_info(vi): + pools = info['pool_metadata'] + for p in pools: + Session.add(pools_info(vid=vi.index, + pool_id=p['id'], + pool_type=p['type'], + pool_rep_size=p['size'])) + + def add_sys_info(vi): + si = info['sysinfo'] + while si: + k,v = si.popitem() + if k == 'os_info': + for o in v: + Session.add(os_info(vid=vi.index, + os=o['os'], + count=o['count'])) + elif k == 'kernel_versions': + for k in v: + Session.add(kernel_versions(vid=vi.index, + version=k['version'], + count=k['count'])) + elif k == 'kernel_types': + for k in v: + Session.add(kernel_types(vid=vi.index, + type=k['type'], + count=k['count'])) + elif k == 'distros': + for d in v: + Session.add(distros(vid=vi.index, + distro=d['distro'], + count=d['count'])) + elif k == 'cpus': + for c in v: + Session.add(cpus(vid=vi.index, + cpu=c['cpu'], + count=c['count'])) + elif k == 'cpu_archs': + for c in v: + Session.add(cpu_archs(vid=vi.index, + arch=c['arch'], + count=c['count'])) + elif k == 'ceph_versions': + for c in v: + Session.add(ceph_versions(vid=vi.index, + version=c['version'], + count=c['count'])) + + ci = add_cluster_info() + add_version_info(ci) + vi = Session.query(version_info).filter_by(cluster_id=ci.index, + version_number=ci.num_versions).first() + add_components_info(vi) + add_crush_types(vi) + add_pools_info(vi) + add_sys_info(vi) + +def delete_uuid(uuid): + ci = Session.query(cluster_info).filter_by(uuid=uuid).first() + if ci is None: + return {'status':400, 'msg':'No information for this UUID'} + + for v in Session.query(version_info).filter_by(cluster_id=ci.index).all(): + Session.query(components_info).filter_by(vid=v.index).delete() + Session.query(crush_types).filter_by(vid=v.index).delete() + Session.query(pools_info).filter_by(vid=v.index).delete() + Session.query(os_info).filter_by(vid=v.index).delete() + Session.query(kernel_versions).filter_by(vid=v.index).delete() + Session.query(kernel_types).filter_by(vid=v.index).delete() + Session.query(distros).filter_by(vid=v.index).delete() + Session.query(cpus).filter_by(vid=v.index).delete() + Session.query(cpu_archs).filter_by(vid=v.index).delete() + Session.query(ceph_versions).filter_by(vid=v.index).delete() + + Session.flush() + Session.delete(v) + Session.flush() + + Session.delete(ci) + return None + +def get_uuids(): + return Session.query(cluster_info).all() + +def get_versions(uuid): + ci = Session.query(cluster_info).filter_by(uuid=uuid).first() + if ci is None: + return None + + return Session.query(version_info).filter_by(cluster_id=ci.index).all() + +def get_brag(uuid, version_id): + b = brag(uuid, version_id) + if b.ci is None or b.vi is None: + return None + + return b diff --git a/ceph/src/brag/server/ceph_brag/tests/__init__.py b/ceph/src/brag/server/ceph_brag/tests/__init__.py new file mode 100644 index 00000000..78ea5274 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/tests/__init__.py @@ -0,0 +1,22 @@ +import os +from unittest import TestCase +from pecan import set_config +from pecan.testing import load_test_app + +__all__ = ['FunctionalTest'] + + +class FunctionalTest(TestCase): + """ + Used for functional tests where you need to test your + literal application and its integration with the framework. + """ + + def setUp(self): + self.app = load_test_app(os.path.join( + os.path.dirname(__file__), + 'config.py' + )) + + def tearDown(self): + set_config({}, overwrite=True) diff --git a/ceph/src/brag/server/ceph_brag/tests/config.py b/ceph/src/brag/server/ceph_brag/tests/config.py new file mode 100644 index 00000000..b6190da4 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/tests/config.py @@ -0,0 +1,54 @@ +# Server Specific Configurations +server = { + 'port': '8080', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'ceph_brag.controllers.root.RootController', + 'modules': ['ceph_brag'], + 'static_root': '%(confdir)s/public', + 'template_path': '%(confdir)s/ceph_brag/templates', + 'debug': True, + 'errors': { + 404: '/error/404', + '__force_dict__': True + } +} + +logging = { + 'loggers': { + 'root': {'level': 'INFO', 'handlers': ['console']}, + 'ceph_brag': {'level': 'DEBUG', 'handlers': ['console']}, + 'py.warnings': {'handlers': ['console']}, + '__force_dict__': True + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + } + }, + 'formatters': { + 'simple': { + 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' + '[%(threadName)s] %(message)s') + } + } +} + +sqlalchemy = { + 'url' : 'sqlite:////tmp/test.db', + 'echo' : False, + 'encoding' : 'utf-8' +} + + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/ceph/src/brag/server/ceph_brag/tests/test_functional.py b/ceph/src/brag/server/ceph_brag/tests/test_functional.py new file mode 100644 index 00000000..13285c41 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/tests/test_functional.py @@ -0,0 +1,68 @@ +from unittest import TestCase +from webtest import TestApp +from ceph_brag.tests import FunctionalTest +import json, sys +from pecan import request + +class TestRootController(FunctionalTest): + def test_1_get_invalid_url_format(self): + response = self.app.get('/1/2/3', expect_errors=True) + assert response.status_int == 400 + + def test_2_put(self): + with open ("sample.json", "r") as myfile: + data=myfile.read().replace('\n', '') + response = self.app.request('/', method='PUT', body=data) + assert response.status_int == 201 + + def test_3_put_invalid_json(self): + response = self.app.request('/', method='PUT', body='{asdfg', expect_errors=True) + assert response.status_int == 422 + + def test_4_put_invalid_entries_1(self): + response = self.app.request('/', method='PUT', body='{}', expect_errors=True) + assert response.status_int == 422 + + def test_5_put_incomplete_json(self): + response = self.app.request('/', method='PUT', body='{\"uuid\":\"adfs-12312ad\"}', + expect_errors=True) + assert response.status_int == 422 + + def test_6_get(self): + response = self.app.get('/') + js = json.loads(response.body) + for entry in js: + ci = entry + break + + response = self.app.get('/'+ci['uuid']+'/'+str(ci['num_versions'])) + assert response.status_int == 200 + + def test_7_get_invalid_uuid(self): + response = self.app.get('/xxxxxx', expect_errors=True) + assert response.status_int == 400 + + def test_8_get_invalid_version(self): + response = self.app.get('/') + js = json.loads(response.body) + for entry in js: + ci = entry + break + + response = self.app.get('/'+ci['uuid']+'/'+str(0), expect_errors=True) + assert response.status_int == 400 + + def test_9_delete_invalid_parameters(self): + response = self.app.delete('/', expect_errors=True) + assert response.status_int == 400 + + def test_91_delete_wrong_uuid(self): + response = self.app.delete('/?uuid=xxxx', expect_errors=True) + assert response.status_int == 400 + + def test_92_delete(self): + response = self.app.get('/') + js = json.loads(response.body) + for entry in js: + response = self.app.delete('/?uuid='+entry['uuid']) + assert response.status_int == 200 diff --git a/ceph/src/brag/server/ceph_brag/tests/test_units.py b/ceph/src/brag/server/ceph_brag/tests/test_units.py new file mode 100644 index 00000000..573fb682 --- /dev/null +++ b/ceph/src/brag/server/ceph_brag/tests/test_units.py @@ -0,0 +1,7 @@ +from unittest import TestCase + + +class TestUnits(TestCase): + + def test_units(self): + assert 5 * 5 == 25 diff --git a/ceph/src/brag/server/config.py b/ceph/src/brag/server/config.py new file mode 100644 index 00000000..429a2ef2 --- /dev/null +++ b/ceph/src/brag/server/config.py @@ -0,0 +1,52 @@ +# Server Specific Configurations +server = { + 'port': '8080', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'ceph_brag.controllers.root.RootController', + 'modules': ['ceph_brag'], + 'debug': True, + 'errors': { + 404: '/error/404', + '__force_dict__': True + } +} + +logging = { + 'loggers': { + 'root': {'level': 'INFO', 'handlers': ['console']}, + 'ceph_brag': {'level': 'DEBUG', 'handlers': ['console']}, + 'py.warnings': {'handlers': ['console']}, + '__force_dict__': True + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + } + }, + 'formatters': { + 'simple': { + 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' + '[%(threadName)s] %(message)s') + } + } +} + +sqlalchemy = { + 'url' : 'sqlite:////tmp/test.db', + 'echo' : False, + 'encoding' : 'utf-8' +} + + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/ceph/src/brag/server/sample.json b/ceph/src/brag/server/sample.json new file mode 100644 index 00000000..194ec637 --- /dev/null +++ b/ceph/src/brag/server/sample.json @@ -0,0 +1,98 @@ +{ + "cluster_creation_date": "2014-01-16 13:38:41.928551", + "uuid": "20679d0e-04b1-4004-8ee9-45ac271510e9", + "components_count": { + "num_pgs": 192, + "num_mdss": 1, + "num_osds": 1, + "num_bytes": 0, + "num_pools": 3, + "num_mons": 1, + "num_objects": 0 + }, + "crush_types": [ + { + "count": 1, + "type": "osd" + }, + { + "count": 1, + "type": "rack" + }, + { + "count": 1, + "type": "host" + }, + { + "count": 1, + "type": "root" + } + ], + "ownership": { + "organization": "eNovance", + "description": "Use case1", + "name": "Cluster1", + "email": "mail@enovance.com" + }, + "pool_metadata": [ + { + "type": 1, + "id": 0, + "size": 3 + }, + { + "type": 1, + "id": 1, + "size": 3 + }, + { + "type": 1, + "id": 2, + "size": 3 + } + ], + "sysinfo": { + "kernel_types": [ + { + "count": 1, + "type": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012" + } + ], + "cpu_archs": [ + { + "count": 1, + "arch": "x86_64" + } + ], + "cpus": [ + { + "count": 1, + "cpu": "Intel Xeon E312xx (Sandy Bridge)" + } + ], + "kernel_versions": [ + { + "count": 1, + "version": "3.2.0-23-virtual" + } + ], + "ceph_versions": [ + { + "count": 1, + "version": "0.75-229-g4050eae(4050eae32cd77a1c210ca11d0f12c74daecb1bd3)" + } + ], + "os_info": [ + { + "count": 1, + "os": "Linux" + } + ], + "distros": [ + { + "count": 1, + "distro": "Ubuntu 12.04 precise (Ubuntu 12.04 LTS)" + } + ] + } +} diff --git a/ceph/src/brag/server/setup.cfg b/ceph/src/brag/server/setup.cfg new file mode 100644 index 00000000..20d3a716 --- /dev/null +++ b/ceph/src/brag/server/setup.cfg @@ -0,0 +1,6 @@ +[nosetests] +match=^test +where=ceph_brag +nocapture=1 +cover-package=ceph_brag +cover-erase=1 diff --git a/ceph/src/brag/server/setup.py b/ceph/src/brag/server/setup.py new file mode 100644 index 00000000..51443203 --- /dev/null +++ b/ceph/src/brag/server/setup.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +setup( + name='ceph_brag', + version='0.1', + description='', + author='', + author_email='', + install_requires=[ + "pecan", + ], + test_suite='ceph_brag', + zip_safe=False, + include_package_data=True, + packages=find_packages(exclude=['ez_setup']) +) diff --git a/ceph/src/ceph-clsinfo b/ceph/src/ceph-clsinfo new file mode 100755 index 00000000..5615af48 --- /dev/null +++ b/ceph/src/ceph-clsinfo @@ -0,0 +1,85 @@ +#!/bin/sh + + +show_name=0 +show_ver=0 +show_arch=0 +show_default=1 +fname="" + +usage="usage: $0 [option]... \n" +usage=$usage"options:\n" +usage=$usage"\t-n, --name\n" +usage=$usage"\t-v, --ver\n" +usage=$usage"\t-a, --arch\n" + +usage_exit() { + printf "$usage" + exit 1 +} + +err_exit() { + echo "Error: $*" + exit 1 +} + +while [ $# -ge 1 ]; do +case $1 in + -n | --name ) + show_name=1 + show_default=0 + ;; + -v | --ver ) + show_ver=1 + show_default=0 + ;; + -a | --arch ) + show_arch=1 + show_default=0 + ;; + * ) + [ -n "$fname" ] && usage_exit + fname=$1 + ;; + +esac +shift +done + +[ -z "$fname" ] && usage_exit + +if [ $show_default -eq 1 ]; then + show_name=1 + show_ver=1 + show_arch=1 +fi + +[ -e "$fname" ] || err_exit "File not found: ${fname}" + +if [ $show_name -eq 1 ]; then + raw_name=`nm $fname | grep __cls_name__` + name=`echo $raw_name | sed 's/.*cls_name__//g'` + [ -z "$name" ] && err_exit "Could not detect class name" + s=$name + c=" " +fi + +if [ $show_ver -eq 1 ]; then + raw_ver=`nm $fname | grep __cls_ver__` + ver=`echo $raw_ver | sed 's/.*cls_ver__//g; s/_/./g'` + [ -z "$ver" ] && err_exit "Could not detect class version" + s=$s$c$ver + c=" " +fi + +if [ $show_arch -eq 1 ]; then + raw_arch=`readelf -h $fname | grep Machine` + arch="" + [ `echo $raw_arch | grep -c 386` -gt 0 ] && arch="i386" + [ `echo $raw_arch | grep -c 86-64` -gt 0 ] && arch="x86-64" + [ -z "$arch" ] && err_exit "unknown file architecture" + s=$s$c$arch + c=" " +fi + +echo $s diff --git a/ceph/src/ceph-coverage.in b/ceph/src/ceph-coverage.in new file mode 100644 index 00000000..85e83395 --- /dev/null +++ b/ceph/src/ceph-coverage.in @@ -0,0 +1,25 @@ +#!/bin/sh +set -e + +export GCOV_PREFIX_STRIP=@@GCOV_PREFIX_STRIP@@ + +usage () { + printf '%s: usage: %s OUTPUTDIR COMMAND [ARGS..]\n' "$(basename "$0")" "$(basename "$0")" 1>&2 + exit 1 +} + +export GCOV_PREFIX="$1" +[ -n "$GCOV_PREFIX" ] || usage +shift + +case "$GCOV_PREFIX" in + /*) + # absolute path -> ok + ;; + *) + # make it absolute + GCOV_PREFIX="$PWD/$GCOV_PREFIX" + ;; +esac + +exec "$@" diff --git a/ceph/src/ceph-create-keys b/ceph/src/ceph-create-keys new file mode 100755 index 00000000..0359228d --- /dev/null +++ b/ceph/src/ceph-create-keys @@ -0,0 +1,227 @@ +#!/usr/bin/env python +import argparse +import errno +import json +import logging +import os +import subprocess +import sys +import time + + +LOG = logging.getLogger(os.path.basename(sys.argv[0])) + +QUORUM_STATES = ['leader', 'peon'] + +def wait_for_quorum(cluster, mon_id): + while True: + p = subprocess.Popen( + args=[ + 'ceph', + '--cluster={cluster}'.format(cluster=cluster), + '--admin-daemon=/var/run/ceph/{cluster}-mon.{mon_id}.asok'.format( + cluster=cluster, + mon_id=mon_id, + ), + 'mon_status', + ], + stdout=subprocess.PIPE, + ) + out = p.stdout.read() + returncode = p.wait() + if returncode != 0: + LOG.info('ceph-mon admin socket not ready yet.') + time.sleep(1) + continue + + if out == '': + LOG.info('ceph-mon admin socket returned no data') + time.sleep(1) + continue + + try: + data = json.loads(out) + except: + LOG.info('failed to parse json %s', out) + sys.exit(errno.EINVAL) + + state = data['state'] + if state not in QUORUM_STATES: + LOG.info('ceph-mon is not in quorum: %r', state) + time.sleep(1) + continue + + break + + +def get_key(cluster, mon_id): + path = '/etc/ceph/{cluster}.client.admin.keyring'.format( + cluster=cluster, + ) + if os.path.exists(path): + LOG.info('Key exists already: %s', path) + return + tmp = '{path}.{pid}.tmp'.format( + path=path, + pid=os.getpid(), + ) + pathdir = os.path.dirname(path) + if not os.path.exists(pathdir): + os.makedirs(pathdir) + while True: + try: + with file(tmp, 'w') as f: + os.fchmod(f.fileno(), 0600) + LOG.info('Talking to monitor...') + returncode = subprocess.call( + args=[ + 'ceph', + '--cluster={cluster}'.format(cluster=cluster), + '--name=mon.', + '--keyring=/var/lib/ceph/mon/{cluster}-{mon_id}/keyring'.format( + cluster=cluster, + mon_id=mon_id, + ), + 'auth', + 'get-or-create', + 'client.admin', + 'mon', 'allow *', + 'osd', 'allow *', + 'mds', 'allow', + ], + stdout=f, + ) + if returncode != 0: + if returncode == errno.EPERM or returncode == errno.EACCES: + LOG.info('Cannot get or create admin key, permission denied') + sys.exit(returncode) + else: + LOG.info('Cannot get or create admin key') + time.sleep(1) + continue + + os.rename(tmp, path) + break + finally: + try: + os.unlink(tmp) + except OSError as e: + if e.errno == errno.ENOENT: + pass + else: + raise + +def bootstrap_key(cluster, type_): + path = '/var/lib/ceph/bootstrap-{type}/{cluster}.keyring'.format( + type=type_, + cluster=cluster, + ) + if os.path.exists(path): + LOG.info('Key exists already: %s', path) + return + tmp = '{path}.{pid}.tmp'.format( + path=path, + pid=os.getpid(), + ) + + args = [ + 'ceph', + '--cluster={cluster}'.format(cluster=cluster), + 'auth', + 'get-or-create', + 'client.bootstrap-{type}'.format(type=type_), + 'mon', + 'allow profile bootstrap-{type}'.format(type=type_), + ] + + pathdir = os.path.dirname(path) + if not os.path.exists(pathdir): + os.makedirs(pathdir) + + while True: + try: + with file(tmp, 'w') as f: + os.fchmod(f.fileno(), 0600) + LOG.info('Talking to monitor...') + returncode = subprocess.call( + args=args, + stdout=f, + ) + if returncode != 0: + if returncode == errno.EPERM or returncode == errno.EACCES: + LOG.info('Cannot get or create bootstrap key for %s, permission denied', type_) + break + else: + LOG.info('Cannot get or create bootstrap key for %s', type_) + time.sleep(1) + continue + + os.rename(tmp, path) + break + finally: + try: + os.unlink(tmp) + except OSError as e: + if e.errno == errno.ENOENT: + pass + else: + raise + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Create Ceph client.admin key when ceph-mon is ready', + ) + parser.add_argument( + '-v', '--verbose', + action='store_true', default=None, + help='be more verbose', + ) + parser.add_argument( + '--cluster', + metavar='NAME', + help='name of the cluster', + ) + parser.add_argument( + '--id', '-i', + metavar='ID', + help='id of a ceph-mon that is coming up', + required=True, + ) + parser.set_defaults( + cluster='ceph', + ) + parser.set_defaults( + # we want to hold on to this, for later + prog=parser.prog, + ) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + loglevel = logging.INFO + if args.verbose: + loglevel = logging.DEBUG + + logging.basicConfig( + level=loglevel, + ) + + wait_for_quorum(cluster=args.cluster, mon_id=args.id) + get_key(cluster=args.cluster, mon_id=args.id) + + bootstrap_key( + cluster=args.cluster, + type_='osd', + ) + bootstrap_key( + cluster=args.cluster, + type_='mds', + ) + + +if __name__ == '__main__': + main() diff --git a/ceph/src/ceph-crush-location b/ceph/src/ceph-crush-location new file mode 100755 index 00000000..1f33f38f --- /dev/null +++ b/ceph/src/ceph-crush-location @@ -0,0 +1,87 @@ +#!/bin/sh +# +# Generate a CRUSH location for the given entity +# +# The CRUSH location consists of a list of key=value pairs, separated +# by spaces, all on a single line. This describes where in CRUSH +# hierarhcy this entity should be placed. +# +# Arguments: +# --cluster name of the cluster (see /etc/ceph/$cluster.conf) +# --type daemon/entity type +# --id id (osd number, mds name, client name) +# + +# if we start up as ./ceph-crush-location, assume everything else is +# in the current directory too. +if [ `dirname $0` = "." ] && [ $PWD != "/usr/bin" ]; then + BINDIR=. + SBINDIR=. + LIBDIR=. + ETCDIR=. +else + BINDIR=/usr/bin + SBINDIR=/usr/sbin + LIBDIR=/usr/lib/ceph + ETCDIR=/etc/ceph +fi + +usage_exit() { + echo "usage: $0 [--cluster ] --id --type " + exit +} + +cluster="ceph" +type="" +id="" +while [ $# -ge 1 ]; do + case $1 in + --cluster | -C) + shift + cluster="$1" + shift + ;; + --id | -i) + shift + id="$1" + shift + ;; + --type | -t) + shift + type="$1" + shift + ;; + *) + echo "unrecognized option '$1'" + usage_exit + ;; + esac +done + +if [ -z "$type" ]; then + echo "must specify entity type" + usage_exit +fi + +if [ -z "$id" ]; then + echo "must specify id" + usage_exit +fi + +# try a type-specific config, e.g. 'osd crush location' +location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup ${type}_crush_location || :)" +if [ -n "$location" ]; then + echo $location + exit 0 +fi + +# try a generic location +location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup crush_location || :)" +if [ -n "$location" ]; then + echo $location + exit 0 +fi + +# spit out something generic +echo "host=$(hostname -s) root=default" + diff --git a/ceph/src/ceph-crush-location.in b/ceph/src/ceph-crush-location.in new file mode 100755 index 00000000..b5043f26 --- /dev/null +++ b/ceph/src/ceph-crush-location.in @@ -0,0 +1,87 @@ +#!/bin/sh +# +# Generate a CRUSH location for the given entity +# +# The CRUSH location consists of a list of key=value pairs, separated +# by spaces, all on a single line. This describes where in CRUSH +# hierarhcy this entity should be placed. +# +# Arguments: +# --cluster name of the cluster (see /etc/ceph/$cluster.conf) +# --type daemon/entity type +# --id id (osd number, mds name, client name) +# + +# if we start up as ./ceph-crush-location, assume everything else is +# in the current directory too. +if [ `dirname $0` = "." ] && [ $PWD != "/usr/bin" ]; then + BINDIR=. + SBINDIR=. + LIBDIR=. + ETCDIR=. +else + BINDIR=@bindir@ + SBINDIR=@prefix@/sbin + LIBDIR=@libdir@/ceph + ETCDIR=@sysconfdir@/ceph +fi + +usage_exit() { + echo "usage: $0 [--cluster ] --id --type " + exit +} + +cluster="ceph" +type="" +id="" +while [ $# -ge 1 ]; do + case $1 in + --cluster | -C) + shift + cluster="$1" + shift + ;; + --id | -i) + shift + id="$1" + shift + ;; + --type | -t) + shift + type="$1" + shift + ;; + *) + echo "unrecognized option '$1'" + usage_exit + ;; + esac +done + +if [ -z "$type" ]; then + echo "must specify entity type" + usage_exit +fi + +if [ -z "$id" ]; then + echo "must specify id" + usage_exit +fi + +# try a type-specific config, e.g. 'osd crush location' +location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup ${type}_crush_location || :)" +if [ -n "$location" ]; then + echo $location + exit 0 +fi + +# try a generic location +location="$($BINDIR/ceph-conf --cluster=${cluster:-ceph} --name=$type.$id --lookup crush_location || :)" +if [ -n "$location" ]; then + echo $location + exit 0 +fi + +# spit out something generic +echo "host=$(hostname -s) root=default" + diff --git a/ceph/src/ceph-debugpack.in b/ceph/src/ceph-debugpack.in new file mode 100644 index 00000000..7be14e44 --- /dev/null +++ b/ceph/src/ceph-debugpack.in @@ -0,0 +1,113 @@ +#!/bin/sh + +# if we start up as ./init-ceph, assume everything else is in the +# current directory too. +if [ `dirname $0` = "." ] && [ $PWD != "/etc/init.d" ]; then + BINDIR=. + LIBDIR=. + ETCDIR=. +else + BINDIR=@bindir@ + LIBDIR=@libdir@/ceph + ETCDIR=@sysconfdir@/ceph +fi + +BINDBGDIR="/usr/lib/debug/usr/bin" + +usage_exit() { + echo "usage: $0 [-c ceph.conf] " + exit +} + +wait_pid_exit() { + pid=$1 + + for i in {1..10}; do + [ -e /proc/$pid ] || return + sleep 1 + done + if [ -e /proc/$pid ]; then + echo Killing pid $pid + kill $pid + fi +} + +. $LIBDIR/ceph_common.sh + +dest_tar='' +while [ $# -ge 1 ]; do +case $1 in + --conf | -c) + [ -z "$2" ] && usage_exit + shift + conf=$1 + ;; + *) + if [ -n "$dest_tar" ]; then + echo unrecognized option \'$1\' + usage_exit + fi + dest_tar=$1 + ;; +esac +shift +done + +[ "$dest_tar" = "" ] && usage_exit + +echo "$0: generating debugpack tarball..." + +if [ -e $dest_tar ]; then + echo "$0: dest $dest_tar already exists, aborting" + exit 1 +fi + +# get absolute path for dest_tar +bins="ceph-mon ceph-mds ceph-osd radosgw" +core_paths="/ $BINDIR $BINDBGDIR" +[ "$conf" = "" ] && conf=$ETCDIR/ceph.conf +log_path=`$CCONF -c $conf "log dir"` + +[ -z "$conf" ] && usage_exit + +# all configs +files='/etc/ceph' + +# binaries +for bin in bins; do + if [ -e "/usr/bin/$bin" ]; then + files="$files /usr/bin/$bin" + fi + if [ -e "/usr/lib/debug/usr/bin/$bin" ]; then + files="$files /usr/lib/debug/usr/bin/$bin" + fi +done + +# logs (the non-rotated ones) +for f in `find $path -maxdepth 1 -name 'core*'`; do + files="$files $f" +done + +# copy cores (if exist) +for path in $core_paths; do + if [ -d $path ]; then + for f in `find $path -maxdepth 1 -name 'core*'`; do + files="$files $f" + done + fi +done + +# cluster state +tmp_path=`mktemp -d /tmp/ceph-debugpack.XXXXXXXXXX` + +$BINDIR/ceph report > $tmp_path/ceph-report & +wait_pid_exit $! + +files="$files $tmp_path" + +# now create a tarball +tar cvfz $dest_tar $files +rm -rf $tmp_path + +echo "$0: created debugpack tarball at $dest_tar" + diff --git a/ceph/src/ceph-disk b/ceph/src/ceph-disk new file mode 100755 index 00000000..5d6071db --- /dev/null +++ b/ceph/src/ceph-disk @@ -0,0 +1,2791 @@ +#!/usr/bin/env python +# +# Copyright (C) 2014 Inktank +# Copyright (C) 2014 Cloudwatt +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +import argparse +import errno +import fcntl +import logging +import os +import os.path +import platform +import re +import subprocess +import stat +import sys +import tempfile +import uuid + +""" +Prepare: + - create GPT partition + - mark the partition with the ceph type uuid + - create a file system + - mark the fs as ready for ceph consumption + - entire data disk is used (one big partition) + - a new partition is added to the journal disk (so it can be easily shared) + + - triggered by administrator or ceph-deploy, e.g. 'ceph-disk [journal disk] + +Activate: + - mount the volume in a temp location + - allocate an osd id (if needed) + - remount in the correct location /var/lib/ceph/osd/$cluster-$id + - start ceph-osd + + - triggered by udev when it sees the OSD gpt partition type + - triggered by admin 'ceph-disk activate ' + - triggered on ceph service startup with 'ceph-disk activate-all' + +We rely on /dev/disk/by-partuuid to find partitions by their UUID; +this is what the journal symlink inside the osd data volume normally +points to. + +activate-all relies on /dev/disk/by-parttype-uuid/$typeuuid.$uuid to +find all partitions. We install special udev rules to create these +links. + +udev triggers 'ceph-disk activate ' or 'ceph-disk +activate-journal ' based on the partition type. + +On old distros (e.g., RHEL6), the blkid installed does not recognized +GPT partition metadata and the /dev/disk/by-partuuid etc. links aren't +present. We have a horrible hack in the form of ceph-disk-udev that +parses gparted output to create the symlinks above and triggers the +'ceph-disk activate' etc commands that udev normally would do if it +knew the GPT partition type. + +""" + +CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026' + +JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106' +DMCRYPT_JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-5ec00ceff106' +OSD_UUID = '4fbd7e29-9d25-41b8-afd0-062c0ceff05d' +DMCRYPT_OSD_UUID = '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d' +TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be' +DMCRYPT_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be' + +DEFAULT_FS_TYPE = 'xfs' + +MOUNT_OPTIONS = dict( + btrfs='noatime,user_subvol_rm_allowed', + # user_xattr is default ever since linux 2.6.39 / 3.0, but we'll + # delay a moment before removing it fully because we did have some + # issues with ext4 before the xatts-in-leveldb work, and it seemed + # that user_xattr helped + ext4='noatime,user_xattr', + xfs='noatime', + ) + +MKFS_ARGS = dict( + btrfs=[ + '-m', 'single', + '-l', '32768', + '-n', '32768', + ], + xfs=[ + # xfs insists on not overwriting previous fs; even if we wipe + # partition table, we often recreate it exactly the same way, + # so we'll see ghosts of filesystems past + '-f', + '-i', 'size=2048', + ], + ) + +INIT_SYSTEMS = [ + 'upstart', + 'sysvinit', + 'systemd', + 'auto', + 'none', + ] + +STATEDIR = '/var/lib/ceph' + +SYSCONFDIR = '/etc/ceph' + +# only warn once about some things +warned_about = {} + +# Nuke the TERM variable to avoid confusing any subprocesses we call. +# For example, libreadline will print weird control sequences for some +# TERM values. +if 'TERM' in os.environ: + del os.environ['TERM'] + +LOG_NAME = __name__ +if LOG_NAME == '__main__': + LOG_NAME = os.path.basename(sys.argv[0]) +LOG = logging.getLogger(LOG_NAME) + + +###### lock ######## + +class filelock(object): + def __init__(self, fn): + self.fn = fn + self.fd = None + + def acquire(self): + assert not self.fd + self.fd = file(self.fn, 'w') + fcntl.lockf(self.fd, fcntl.LOCK_EX) + + def release(self): + assert self.fd + fcntl.lockf(self.fd, fcntl.LOCK_UN) + self.fd = None + + +###### exceptions ######## + + +class Error(Exception): + """ + Error + """ + + def __str__(self): + doc = self.__doc__.strip() + return ': '.join([doc] + [str(a) for a in self.args]) + + +class MountError(Error): + """ + Mounting filesystem failed + """ + + +class UnmountError(Error): + """ + Unmounting filesystem failed + """ + + +class BadMagicError(Error): + """ + Does not look like a Ceph OSD, or incompatible version + """ + + +class TruncatedLineError(Error): + """ + Line is truncated + """ + + +class TooManyLinesError(Error): + """ + Too many lines + """ + + +class FilesystemTypeError(Error): + """ + Cannot discover filesystem type + """ + + +class CephDiskException(Exception): + """ + A base exception for ceph-disk to provide custom (ad-hoc) messages that + will be caught and dealt with when main() is executed + """ + pass + + +class ExecutableNotFound(CephDiskException): + """ + Exception to report on executables not available in PATH + """ + pass + + +####### utils + + +def maybe_mkdir(*a, **kw): + """ + Creates a new directory if it doesn't exist, removes + existing symlink before creating the directory. + """ + # remove any symlink, if it is there.. + if os.path.exists(*a) and stat.S_ISLNK(os.lstat(*a).st_mode): + LOG.debug('Removing old symlink at %s', *a) + os.unlink(*a) + try: + os.mkdir(*a, **kw) + except OSError, e: + if e.errno == errno.EEXIST: + pass + else: + raise + + +def which(executable): + """find the location of an executable""" + if 'PATH' in os.environ: + envpath = os.environ['PATH'] + else: + envpath = os.defpath + PATH = envpath.split(os.pathsep) + + locations = PATH + [ + '/usr/local/bin', + '/bin', + '/usr/bin', + '/usr/local/sbin', + '/usr/sbin', + '/sbin', + ] + + for location in locations: + executable_path = os.path.join(location, executable) + if os.path.exists(executable_path): + return executable_path + + +def _get_command_executable(arguments): + """ + Return the full path for an executable, raise if the executable is not + found. If the executable has already a full path do not perform any checks. + """ + if arguments[0].startswith('/'): # an absolute path + return arguments + executable = which(arguments[0]) + if not executable: + command_msg = 'Could not run command: %s' % ' '.join(arguments) + executable_msg = '%s not in path.' % arguments[0] + raise ExecutableNotFound('%s %s' % (executable_msg, command_msg)) + + # swap the old executable for the new one + arguments[0] = executable + return arguments + + +def command(arguments, **kwargs): + """ + Safely execute a ``subprocess.Popen`` call making sure that the + executable exists and raising a helpful error message + if it does not. + + .. note:: This should be the prefered way of calling ``subprocess.Popen`` + since it provides the caller with the safety net of making sure that + executables *will* be found and will error nicely otherwise. + + This returns the output of the command and the return code of the + process in a tuple: (output, returncode). + """ + arguments = _get_command_executable(arguments) + LOG.info('Running command: %s' % ' '.join(arguments)) + process = subprocess.Popen( + arguments, + stdout=subprocess.PIPE, + **kwargs) + out, _ = process.communicate() + return out, process.returncode + + +def command_check_call(arguments): + """ + Safely execute a ``subprocess.check_call`` call making sure that the + executable exists and raising a helpful error message if it does not. + + .. note:: This should be the prefered way of calling + ``subprocess.check_call`` since it provides the caller with the safety net + of making sure that executables *will* be found and will error nicely + otherwise. + """ + arguments = _get_command_executable(arguments) + LOG.info('Running command: %s', ' '.join(arguments)) + return subprocess.check_call(arguments) + + +def platform_distro(): + """ + Returns a normalized, lower case string without any leading nor trailing + whitespace that represents the distribution name of the current machine. + """ + distro = platform_information()[0] or '' + return distro.strip().lower() + + +def platform_information(): + distro, release, codename = platform.linux_distribution() + if not codename and 'debian' in distro.lower(): # this could be an empty string in Debian + debian_codenames = { + '8': 'jessie', + '7': 'wheezy', + '6': 'squeeze', + } + major_version = release.split('.')[0] + codename = debian_codenames.get(major_version, '') + + # In order to support newer jessie/sid or wheezy/sid strings we test this + # if sid is buried in the minor, we should use sid anyway. + if not codename and '/' in release: + major, minor = release.split('/') + if minor == 'sid': + codename = minor + else: + codename = major + + return ( + str(distro).strip(), + str(release).strip(), + str(codename).strip() + ) + + +def get_dev_name(path): + """ + get device name from path. e.g.:: + + /dev/sda -> sdas, /dev/cciss/c0d1 -> cciss!c0d1 + + a device "name" is something like:: + + sdb + cciss!c0d1 + + """ + assert path.startswith('/dev/') + base = path[5:] + return base.replace('/', '!') + + +def get_dev_path(name): + """ + get a path (/dev/...) from a name (cciss!c0d1) + a device "path" is something like:: + + /dev/sdb + /dev/cciss/c0d1 + + """ + return '/dev/' + name.replace('!', '/') + + +def get_dev_relpath(name): + """ + get a relative path to /dev from a name (cciss!c0d1) + """ + return name.replace('!', '/') + + +def get_dev_size(dev, size='megabytes'): + """ + Attempt to get the size of a device so that we can prevent errors + from actions to devices that are smaller, and improve error reporting. + + Because we want to avoid breakage in case this approach is not robust, we + will issue a warning if we failed to get the size. + + :param size: bytes or megabytes + :param dev: the device to calculate the size + """ + fd = os.open(dev, os.O_RDONLY) + dividers = {'bytes': 1, 'megabytes': 1024*1024} + try: + device_size = os.lseek(fd, 0, os.SEEK_END) + divider = dividers.get(size, 1024*1024) # default to megabytes + return device_size/divider + except Exception as error: + LOG.warning('failed to get size of %s: %s' % (dev, str(error))) + finally: + os.close(fd) + + +def get_partition_dev(dev, pnum): + """ + get the device name for a partition + + assume that partitions are named like the base dev, with a number, and optionally + some intervening characters (like 'p'). e.g., + + sda 1 -> sda1 + cciss/c0d1 1 -> cciss!c0d1p1 + """ + name = get_dev_name(os.path.realpath(dev)) + partname = None + for f in os.listdir(os.path.join('/sys/block', name)): + if f.startswith(name) and f.endswith(str(pnum)): + # we want the shortest name that starts with the base name and ends with the partition number + if not partname or len(f) < len(partname): + partname = f + if partname: + return get_dev_path(partname) + else: + raise Error('partition %d for %s does not appear to exist' % (pnum, dev)) + + +def list_all_partitions(): + """ + Return a list of devices and partitions + """ + dev_part_list = {} + for name in os.listdir('/sys/block'): + # /dev/fd0 may hang http://tracker.ceph.com/issues/6827 + if re.match(r'^fd\d$', name): + continue + if not os.path.exists(os.path.join('/sys/block', name, 'device')): + continue + dev_part_list[name] = list_partitions(name) + return dev_part_list + + +def list_partitions(basename): + """ + Return a list of partitions on the given device name + """ + partitions = [] + for name in os.listdir(os.path.join('/sys/block', basename)): + if name.startswith(basename): + partitions.append(name) + return partitions + +def get_partition_base(dev): + """ + Get the base device for a partition + """ + dev = os.path.realpath(dev) + if not stat.S_ISBLK(os.lstat(dev).st_mode): + raise Error('not a block device', dev) + + name = get_dev_name(dev) + if os.path.exists(os.path.join('/sys/block', name)): + raise Error('not a partition', dev) + + # find the base + for basename in os.listdir('/sys/block'): + if os.path.exists(os.path.join('/sys/block', basename, name)): + return '/dev/' + basename + raise Error('no parent device for partition', dev) + +def is_partition(dev): + """ + Check whether a given device path is a partition or a full disk. + """ + dev = os.path.realpath(dev) + if not stat.S_ISBLK(os.lstat(dev).st_mode): + raise Error('not a block device', dev) + + name = get_dev_name(dev) + if os.path.exists(os.path.join('/sys/block', name)): + return False + + # make sure it is a partition of something else + for basename in os.listdir('/sys/block'): + if os.path.exists(os.path.join('/sys/block', basename, name)): + return True + + raise Error('not a disk or partition', dev) + + +def is_mounted(dev): + """ + Check if the given device is mounted. + """ + dev = os.path.realpath(dev) + with file('/proc/mounts', 'rb') as proc_mounts: + for line in proc_mounts: + fields = line.split() + if len(fields) < 3: + continue + mounts_dev = fields[0] + path = fields[1] + if mounts_dev.startswith('/') and os.path.exists(mounts_dev): + mounts_dev = os.path.realpath(mounts_dev) + if mounts_dev == dev: + return path + return None + + +def is_held(dev): + """ + Check if a device is held by another device (e.g., a dm-crypt mapping) + """ + assert os.path.exists(dev) + dev = os.path.realpath(dev) + base = get_dev_name(dev) + + # full disk? + directory = '/sys/block/{base}/holders'.format(base=base) + if os.path.exists(directory): + return os.listdir(directory) + + # partition? + part = base + while len(base): + directory = '/sys/block/{base}/{part}/holders'.format(part=part, base=base) + if os.path.exists(directory): + return os.listdir(directory) + base = base[:-1] + return [] + + +def verify_not_in_use(dev, check_partitions=False): + """ + Verify if a given device (path) is in use (e.g. mounted or + in use by device-mapper). + + :raises: Error if device is in use. + """ + assert os.path.exists(dev) + if is_mounted(dev): + raise Error('Device is mounted', dev) + holders = is_held(dev) + if holders: + raise Error('Device is in use by a device-mapper mapping (dm-crypt?)' % dev, ','.join(holders)) + + if check_partitions and not is_partition(dev): + basename = get_dev_name(os.path.realpath(dev)) + for partname in list_partitions(basename): + partition = get_dev_path(partname) + if is_mounted(partition): + raise Error('Device is mounted', partition) + holders = is_held(partition) + if holders: + raise Error('Device %s is in use by a device-mapper mapping (dm-crypt?)' % partition, ','.join(holders)) + + +def must_be_one_line(line): + """ + Checks if given line is really one single line. + + :raises: TruncatedLineError or TooManyLinesError + :return: Content of the line, or None if line isn't valid. + """ + if line[-1:] != '\n': + raise TruncatedLineError(line) + line = line[:-1] + if '\n' in line: + raise TooManyLinesError(line) + return line + + +def read_one_line(parent, name): + """ + Read a file whose sole contents are a single line. + + Strips the newline. + + :return: Contents of the line, or None if file did not exist. + """ + path = os.path.join(parent, name) + try: + line = file(path, 'rb').read() + except IOError as e: + if e.errno == errno.ENOENT: + return None + else: + raise + + try: + line = must_be_one_line(line) + except (TruncatedLineError, TooManyLinesError) as e: + raise Error( + 'File is corrupt: {path}: {msg}'.format( + path=path, + msg=e, + ) + ) + return line + + +def write_one_line(parent, name, text): + """ + Write a file whose sole contents are a single line. + + Adds a newline. + """ + path = os.path.join(parent, name) + tmp = '{path}.{pid}.tmp'.format(path=path, pid=os.getpid()) + with file(tmp, 'wb') as tmp_file: + tmp_file.write(text + '\n') + os.fsync(tmp_file.fileno()) + os.rename(tmp, path) + + +def check_osd_magic(path): + """ + Check that this path has the Ceph OSD magic. + + :raises: BadMagicError if this does not look like a Ceph OSD data + dir. + """ + magic = read_one_line(path, 'magic') + if magic is None: + # probably not mkfs'ed yet + raise BadMagicError(path) + if magic != CEPH_OSD_ONDISK_MAGIC: + raise BadMagicError(path) + + +def check_osd_id(osd_id): + """ + Ensures osd id is numeric. + """ + if not re.match(r'^[0-9]+$', osd_id): + raise Error('osd id is not numeric', osd_id) + + +def allocate_osd_id( + cluster, + fsid, + keyring, + ): + """ + Accocates an OSD id on the given cluster. + + :raises: Error if the call to allocate the OSD id fails. + :return: The allocated OSD id. + """ + + LOG.debug('Allocating OSD id...') + try: + osd_id = _check_output( + args=[ + 'ceph', + '--cluster', cluster, + '--name', 'client.bootstrap-osd', + '--keyring', keyring, + 'osd', 'create', '--concise', + fsid, + ], + ) + except subprocess.CalledProcessError as e: + raise Error('ceph osd create failed', e, e.output) + osd_id = must_be_one_line(osd_id) + check_osd_id(osd_id) + return osd_id + + +def get_osd_id(path): + """ + Gets the OSD id of the OSD at the given path. + """ + osd_id = read_one_line(path, 'whoami') + if osd_id is not None: + check_osd_id(osd_id) + return osd_id + + +def _check_output(args=None, **kwargs): + out, ret = command(args, **kwargs) + if ret: + cmd = args[0] + error = subprocess.CalledProcessError(ret, cmd) + error.output = out + raise error + return out + + +def get_conf(cluster, variable): + """ + Get the value of the given configuration variable from the + cluster. + + :raises: Error if call to ceph-conf fails. + :return: The variable value or None. + """ + try: + out, ret = command( + [ + 'ceph-conf', + '--cluster={cluster}'.format( + cluster=cluster, + ), + '--name=osd.', + '--lookup', + variable, + ], + close_fds=True, + ) + except OSError as e: + raise Error('error executing ceph-conf', e) + if ret == 1: + # config entry not found + return None + elif ret != 0: + raise Error('getting variable from configuration failed') + value = out.split('\n', 1)[0] + # don't differentiate between "var=" and no var set + if not value: + return None + return value + + +def get_conf_with_default(cluster, variable): + """ + Get a config value that is known to the C++ code. + + This will fail if called on variables that are not defined in + common config options. + """ + try: + out = _check_output( + args=[ + 'ceph-osd', + '--cluster={cluster}'.format( + cluster=cluster, + ), + '--show-config-value={variable}'.format( + variable=variable, + ), + ], + close_fds=True, + ) + except subprocess.CalledProcessError as e: + raise Error( + 'getting variable from configuration failed', + e, + ) + + value = str(out).split('\n', 1)[0] + return value + + +def get_fsid(cluster): + """ + Get the fsid of the cluster. + + :return: The fsid or raises Error. + """ + fsid = get_conf_with_default(cluster=cluster, variable='fsid') + if fsid is None: + raise Error('getting cluster uuid from configuration failed') + return fsid.lower() + + +def get_or_create_dmcrypt_key( + _uuid, + key_dir, + ): + """ + Get path to dmcrypt key or create a new key file. + + :return: Path to the dmcrypt key file. + """ + path = os.path.join(key_dir, _uuid) + + # already have it? + if os.path.exists(path): + return path + + # make a new key + try: + if not os.path.exists(key_dir): + os.makedirs(key_dir) + with file('/dev/urandom', 'rb') as i: + key = i.read(256) + with file(path, 'wb') as key_file: + key_file.write(key) + return path + except: + raise Error('unable to read or create dm-crypt key', path) + + +def dmcrypt_map( + rawdev, + keypath, + _uuid, + ): + """ + Maps a device to a dmcrypt device. + + :return: Path to the dmcrypt device. + """ + dev = '/dev/mapper/' + _uuid + args = [ + 'cryptsetup', + '--key-file', + keypath, + '--key-size', '256', + 'create', + _uuid, + rawdev, + ] + try: + command_check_call(args) + return dev + + except subprocess.CalledProcessError as e: + raise Error('unable to map device', rawdev, e) + + +def dmcrypt_unmap( + _uuid + ): + """ + Removes the dmcrypt device with the given UUID. + """ + args = [ + 'cryptsetup', + 'remove', + _uuid + ] + + try: + command_check_call(args) + + except subprocess.CalledProcessError as e: + raise Error('unable to unmap device', _uuid, e) + + +def mount( + dev, + fstype, + options, + ): + """ + Mounts a device with given filessystem type and + mount options to a tempfile path under /var/lib/ceph/tmp. + """ + # sanity check: none of the arguments are None + if dev is None: + raise ValueError('dev may not be None') + if fstype is None: + raise ValueError('fstype may not be None') + + # pick best-of-breed mount options based on fs type + if options is None: + options = MOUNT_OPTIONS.get(fstype, '') + + # mount + path = tempfile.mkdtemp( + prefix='mnt.', + dir=STATEDIR + '/tmp', + ) + try: + LOG.debug('Mounting %s on %s with options %s', dev, path, options) + command_check_call( + [ + 'mount', + '-t', fstype, + '-o', options, + '--', + dev, + path, + ], + ) + except subprocess.CalledProcessError as e: + try: + os.rmdir(path) + except (OSError, IOError): + pass + raise MountError(e) + + return path + + +def unmount( + path, + ): + """ + Unmount and removes the given mount point. + """ + try: + LOG.debug('Unmounting %s', path) + command_check_call( + [ + '/bin/umount', + '--', + path, + ], + ) + except subprocess.CalledProcessError as e: + raise UnmountError(e) + + os.rmdir(path) + + +########################################### + + +def get_free_partition_index(dev): + """ + Get the next free partition index on a given device. + + :return: Index number (> 1 if there is already a partition on the device) + or 1 if there is no partition table. + """ + try: + lines = _check_output( + args=[ + 'parted', + '--machine', + '--', + dev, + 'print', + ], + ) + except subprocess.CalledProcessError as e: + print 'cannot read partition index; assume it isn\'t present\n (Error: %s)' % e + return 1 + + if not lines: + raise Error('parted failed to output anything') + lines = str(lines).splitlines(True) + + # work around buggy libreadline(?) library in rhel/centos. + idiot_prefix = '\x1b\x5b\x3f\x31\x30\x33\x34\x68' + if lines[0].startswith(idiot_prefix): + lines[0] = lines[0][8:] + + if lines[0] not in ['CHS;\n', 'CYL;\n', 'BYT;\n']: + raise Error('weird parted units', lines[0]) + del lines[0] + + if not lines[0].startswith('/dev/'): + raise Error('weird parted disk entry', lines[0]) + del lines[0] + + seen = set() + for line in lines: + idx, _ = line.split(':', 1) + idx = int(idx) + seen.add(idx) + + num = 1 + while num in seen: + num += 1 + return num + + +def zap(dev): + """ + Destroy the partition table and content of a given disk. + """ + try: + LOG.debug('Zapping partition table on %s', dev) + + # try to wipe out any GPT partition table backups. sgdisk + # isn't too thorough. + lba_size = 4096 + size = 33 * lba_size + with file(dev, 'wb') as dev_file: + dev_file.seek(-size, os.SEEK_END) + dev_file.write(size*'\0') + + command_check_call( + [ + 'sgdisk', + '--zap-all', + '--clear', + '--mbrtogpt', + '--', + dev, + ], + ) + except subprocess.CalledProcessError as e: + raise Error(e) + + +def prepare_journal_dev( + data, + journal, + journal_size, + journal_uuid, + journal_dm_keypath, + ): + + if is_partition(journal): + LOG.debug('Journal %s is a partition', journal) + LOG.warning('OSD will not be hot-swappable if journal is not the same device as the osd data') + return (journal, None, None) + + ptype = JOURNAL_UUID + if journal_dm_keypath: + ptype = DMCRYPT_JOURNAL_UUID + + # it is a whole disk. create a partition! + num = None + if journal == data: + # we're sharing the disk between osd data and journal; + # make journal be partition number 2, so it's pretty + num = 2 + journal_part = '{num}:0:{size}M'.format( + num=num, + size=journal_size, + ) + else: + # sgdisk has no way for me to say "whatever is the next + # free index number" when setting type guids etc, so we + # need to awkwardly look up the next free number, and then + # fix that in the call -- and hope nobody races with us; + # then again nothing guards the partition table from races + # anyway + num = get_free_partition_index(dev=journal) + journal_part = '{num}:0:+{size}M'.format( + num=num, + size=journal_size, + ) + LOG.warning('OSD will not be hot-swappable if journal is not the same device as the osd data') + + dev_size = get_dev_size(journal) + + if journal_size > dev_size: + LOG.error('refusing to create journal on %s' % journal) + LOG.error('journal size (%sM) is bigger than device (%sM)' % (journal_size, dev_size)) + raise Error( + '%s device size (%sM) is not big enough for journal' % (journal, dev_size) + ) + + try: + LOG.debug('Creating journal partition num %d size %d on %s', num, journal_size, journal) + command_check_call( + [ + 'sgdisk', + '--new={part}'.format(part=journal_part), + '--change-name={num}:ceph journal'.format(num=num), + '--partition-guid={num}:{journal_uuid}'.format( + num=num, + journal_uuid=journal_uuid, + ), + '--typecode={num}:{uuid}'.format( + num=num, + uuid=ptype, + ), + '--mbrtogpt', + '--', + journal, + ], + ) + + # try to make sure the kernel refreshes the table. note + # that if this gets ebusy, we are probably racing with + # udev because it already updated it.. ignore failure here. + + # On RHEL and CentOS distros, calling partprobe forces a reboot of the + # server. Since we are not resizing partitons so we rely on calling + # partx + if platform_distro().startswith(('centos', 'red')): + LOG.info('calling partx on prepared device %s', journal) + LOG.info('re-reading known partitions will display errors') + command( + [ + 'partx', + '-a', + journal, + ], + ) + + else: + LOG.debug('Calling partprobe on prepared device %s', journal) + command( + [ + 'partprobe', + journal, + ], + ) + + # wait for udev event queue to clear + command( + [ + 'udevadm', + 'settle', + ], + ) + + journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format( + journal_uuid=journal_uuid, + ) + + journal_dmcrypt = None + if journal_dm_keypath: + journal_dmcrypt = journal_symlink + journal_symlink = '/dev/mapper/{uuid}'.format(uuid=journal_uuid) + + LOG.debug('Journal is GPT partition %s', journal_symlink) + return (journal_symlink, journal_dmcrypt, journal_uuid) + + except subprocess.CalledProcessError as e: + raise Error(e) + + +def prepare_journal_file( + journal): + + if not os.path.exists(journal): + LOG.debug('Creating journal file %s with size 0 (ceph-osd will resize and allocate)', journal) + with file(journal, 'wb') as journal_file: # noqa + pass + + LOG.debug('Journal is file %s', journal) + LOG.warning('OSD will not be hot-swappable if journal is not the same device as the osd data') + return (journal, None, None) + + +def prepare_journal( + data, + journal, + journal_size, + journal_uuid, + force_file, + force_dev, + journal_dm_keypath, + ): + + if journal is None: + if force_dev: + raise Error('Journal is unspecified; not a block device') + return (None, None, None) + + if not os.path.exists(journal): + if force_dev: + raise Error('Journal does not exist; not a block device', journal) + return prepare_journal_file(journal) + + jmode = os.stat(journal).st_mode + if stat.S_ISREG(jmode): + if force_dev: + raise Error('Journal is not a block device', journal) + return prepare_journal_file(journal) + + if stat.S_ISBLK(jmode): + if force_file: + raise Error('Journal is not a regular file', journal) + return prepare_journal_dev(data, journal, journal_size, journal_uuid, journal_dm_keypath) + + raise Error('Journal %s is neither a block device nor regular file' % journal) + + +def adjust_symlink(target, path): + create = True + if os.path.lexists(path): + try: + mode = os.lstat(path).st_mode + if stat.S_ISREG(mode): + LOG.debug('Removing old file %s', path) + os.unlink(path) + elif stat.S_ISLNK(mode): + old = os.readlink(path) + if old != target: + LOG.debug('Removing old symlink %s -> %s', path, old) + os.unlink(path) + else: + create = False + except: + raise Error('unable to remove (or adjust) old file (symlink)', path) + if create: + LOG.debug('Creating symlink %s -> %s', path, target) + try: + os.symlink(target, path) + except: + raise Error('unable to create symlink %s -> %s' % (path, target)) + + +def prepare_dir( + path, + journal, + cluster_uuid, + osd_uuid, + journal_uuid, + journal_dmcrypt=None, + ): + + if os.path.exists(os.path.join(path, 'magic')): + LOG.debug('Data dir %s already exists', path) + return + else: + LOG.debug('Preparing osd data dir %s', path) + + if osd_uuid is None: + osd_uuid = str(uuid.uuid4()) + + if journal is not None: + # we're using an external journal; point to it here + adjust_symlink(journal, os.path.join(path, 'journal')) + + if journal_dmcrypt is not None: + adjust_symlink(journal_dmcrypt, os.path.join(path, 'journal_dmcrypt')) + else: + try: + os.unlink(os.path.join(path, 'journal_dmcrypt')) + except OSError: + pass + + write_one_line(path, 'ceph_fsid', cluster_uuid) + write_one_line(path, 'fsid', osd_uuid) + + if journal_uuid is not None: + # i.e., journal is a tagged partition + write_one_line(path, 'journal_uuid', journal_uuid) + + write_one_line(path, 'magic', CEPH_OSD_ONDISK_MAGIC) + + +def prepare_dev( + data, + journal, + fstype, + mkfs_args, + mount_options, + cluster_uuid, + osd_uuid, + journal_uuid, + journal_dmcrypt, + osd_dm_keypath, + ): + """ + Prepare a data/journal combination to be used for an OSD. + + The ``magic`` file is written last, so it's presence is a reliable + indicator of the whole sequence having completed. + + WARNING: This will unconditionally overwrite anything given to + it. + """ + + ptype_tobe = TOBE_UUID + ptype_osd = OSD_UUID + if osd_dm_keypath: + ptype_tobe = DMCRYPT_TOBE_UUID + ptype_osd = DMCRYPT_OSD_UUID + + rawdev = None + if is_partition(data): + LOG.debug('OSD data device %s is a partition', data) + rawdev = data + else: + LOG.debug('Creating osd partition on %s', data) + try: + command_check_call( + [ + 'sgdisk', + '--largest-new=1', + '--change-name=1:ceph data', + '--partition-guid=1:{osd_uuid}'.format( + osd_uuid=osd_uuid, + ), + '--typecode=1:%s' % ptype_tobe, + '--', + data, + ], + ) + command( + [ + 'partprobe', + data, + ], + ) + command( + [ + # wait for udev event queue to clear + 'udevadm', + 'settle', + ], + ) + except subprocess.CalledProcessError as e: + raise Error(e) + + rawdev = get_partition_dev(data, 1) + + dev = None + if osd_dm_keypath: + dev = dmcrypt_map(rawdev, osd_dm_keypath, osd_uuid) + else: + dev = rawdev + + try: + args = [ + 'mkfs', + '-t', + fstype, + ] + if mkfs_args is not None: + args.extend(mkfs_args.split()) + if fstype == 'xfs': + args.extend(['-f']) # always force + else: + args.extend(MKFS_ARGS.get(fstype, [])) + args.extend([ + '--', + dev, + ]) + try: + LOG.debug('Creating %s fs on %s', fstype, dev) + command_check_call(args) + except subprocess.CalledProcessError as e: + raise Error(e) + + #remove whitespaces from mount_options + if mount_options is not None: + mount_options = "".join(mount_options.split()) + + path = mount(dev=dev, fstype=fstype, options=mount_options) + + try: + prepare_dir( + path=path, + journal=journal, + cluster_uuid=cluster_uuid, + osd_uuid=osd_uuid, + journal_uuid=journal_uuid, + journal_dmcrypt=journal_dmcrypt, + ) + finally: + unmount(path) + finally: + if rawdev != dev: + dmcrypt_unmap(osd_uuid) + + if not is_partition(data): + try: + command_check_call( + [ + 'sgdisk', + '--typecode=1:%s' % ptype_osd, + '--', + data, + ], + ) + except subprocess.CalledProcessError as e: + raise Error(e) + + +def main_prepare(args): + journal_dm_keypath = None + osd_dm_keypath = None + + try: + prepare_lock.acquire() # noqa + if not os.path.exists(args.data): + if args.data_dev: + raise Error('data path does not exist', args.data) + else: + os.mkdir(args.data) + + # in use? + dmode = os.stat(args.data).st_mode + if stat.S_ISBLK(dmode): + verify_not_in_use(args.data, True) + + if args.journal and os.path.exists(args.journal): + jmode = os.stat(args.journal).st_mode + if stat.S_ISBLK(jmode): + verify_not_in_use(args.journal, False) + + if args.zap_disk is not None: + if stat.S_ISBLK(dmode) and not is_partition(args.data): + zap(args.data) + else: + raise Error('not full block device; cannot zap', args.data) + + if args.cluster_uuid is None: + args.cluster_uuid = get_fsid(cluster=args.cluster) + if args.cluster_uuid is None: + raise Error( + 'must have fsid in config or pass --cluster-uuid=', + ) + + if args.fs_type is None: + args.fs_type = get_conf( + cluster=args.cluster, + variable='osd_mkfs_type', + ) + if args.fs_type is None: + args.fs_type = get_conf( + cluster=args.cluster, + variable='osd_fs_type', + ) + if args.fs_type is None: + args.fs_type = DEFAULT_FS_TYPE + + mkfs_args = get_conf( + cluster=args.cluster, + variable='osd_mkfs_options_{fstype}'.format( + fstype=args.fs_type, + ), + ) + if mkfs_args is None: + mkfs_args = get_conf( + cluster=args.cluster, + variable='osd_fs_mkfs_options_{fstype}'.format( + fstype=args.fs_type, + ), + ) + + mount_options = get_conf( + cluster=args.cluster, + variable='osd_mount_options_{fstype}'.format( + fstype=args.fs_type, + ), + ) + if mount_options is None: + mount_options = get_conf( + cluster=args.cluster, + variable='osd_fs_mount_options_{fstype}'.format( + fstype=args.fs_type, + ), + ) + + journal_size = get_conf_with_default( + cluster=args.cluster, + variable='osd_journal_size', + ) + journal_size = int(journal_size) + + # colocate journal with data? + if stat.S_ISBLK(dmode) and not is_partition(args.data) and args.journal is None and args.journal_file is None: + LOG.info('Will colocate journal with data on %s', args.data) + args.journal = args.data + + if args.journal_uuid is None: + args.journal_uuid = str(uuid.uuid4()) + if args.osd_uuid is None: + args.osd_uuid = str(uuid.uuid4()) + + # dm-crypt keys? + if args.dmcrypt: + journal_dm_keypath = get_or_create_dmcrypt_key(args.journal_uuid, args.dmcrypt_key_dir) + osd_dm_keypath = get_or_create_dmcrypt_key(args.osd_uuid, args.dmcrypt_key_dir) + + # prepare journal + (journal_symlink, journal_dmcrypt, journal_uuid) = prepare_journal( + data=args.data, + journal=args.journal, + journal_size=journal_size, + journal_uuid=args.journal_uuid, + force_file=args.journal_file, + force_dev=args.journal_dev, + journal_dm_keypath=journal_dm_keypath, + ) + + # prepare data + if stat.S_ISDIR(dmode): + if args.data_dev: + raise Error('data path is not a block device', args.data) + prepare_dir( + path=args.data, + journal=journal_symlink, + cluster_uuid=args.cluster_uuid, + osd_uuid=args.osd_uuid, + journal_uuid=journal_uuid, + journal_dmcrypt=journal_dmcrypt, + ) + elif stat.S_ISBLK(dmode): + if args.data_dir: + raise Error('data path is not a directory', args.data) + prepare_dev( + data=args.data, + journal=journal_symlink, + fstype=args.fs_type, + mkfs_args=mkfs_args, + mount_options=mount_options, + cluster_uuid=args.cluster_uuid, + osd_uuid=args.osd_uuid, + journal_uuid=journal_uuid, + journal_dmcrypt=journal_dmcrypt, + osd_dm_keypath=osd_dm_keypath, + ) + else: + raise Error('not a dir or block device', args.data) + prepare_lock.release() # noqa + + if stat.S_ISBLK(dmode): + # try to make sure the kernel refreshes the table. note + # that if this gets ebusy, we are probably racing with + # udev because it already updated it.. ignore failure here. + + # On RHEL and CentOS distros, calling partprobe forces a reboot of + # the server. Since we are not resizing partitons so we rely on + # calling partx + if platform_distro().startswith(('centos', 'red')): + LOG.info('calling partx on prepared device %s', args.data) + LOG.info('re-reading known partitions will display errors') + + command( + [ + 'partx', + '-a', + args.data, + ], + ) + + else: + LOG.debug('Calling partprobe on prepared device %s', args.data) + command( + [ + 'partprobe', + args.data, + ], + ) + + except Error as e: + if journal_dm_keypath: + os.unlink(journal_dm_keypath) + if osd_dm_keypath: + os.unlink(osd_dm_keypath) + prepare_lock.release() # noqa + raise e + + +########################### + + +def mkfs( + path, + cluster, + osd_id, + fsid, + keyring, + ): + monmap = os.path.join(path, 'activate.monmap') + command_check_call( + [ + 'ceph', + '--cluster', cluster, + '--name', 'client.bootstrap-osd', + '--keyring', keyring, + 'mon', 'getmap', '-o', monmap, + ], + ) + + command_check_call( + [ + 'ceph-osd', + '--cluster', cluster, + '--mkfs', + '--mkkey', + '-i', osd_id, + '--monmap', monmap, + '--osd-data', path, + '--osd-journal', os.path.join(path, 'journal'), + '--osd-uuid', fsid, + '--keyring', os.path.join(path, 'keyring'), + ], + ) + # TODO ceph-osd --mkfs removes the monmap file? + # os.unlink(monmap) + + +def auth_key( + path, + cluster, + osd_id, + keyring, + ): + try: + # try dumpling+ cap scheme + command_check_call( + [ + 'ceph', + '--cluster', cluster, + '--name', 'client.bootstrap-osd', + '--keyring', keyring, + 'auth', 'add', 'osd.{osd_id}'.format(osd_id=osd_id), + '-i', os.path.join(path, 'keyring'), + 'osd', 'allow *', + 'mon', 'allow profile osd', + ], + ) + except subprocess.CalledProcessError as err: + if err.returncode == errno.EACCES: + # try old cap scheme + command_check_call( + [ + 'ceph', + '--cluster', cluster, + '--name', 'client.bootstrap-osd', + '--keyring', keyring, + 'auth', 'add', 'osd.{osd_id}'.format(osd_id=osd_id), + '-i', os.path.join(path, 'keyring'), + 'osd', 'allow *', + 'mon', 'allow rwx', + ], + ) + else: + raise + + +def move_mount( + dev, + path, + cluster, + osd_id, + fstype, + mount_options, + ): + LOG.debug('Moving mount to final location...') + parent = STATEDIR + '/osd' + osd_data = os.path.join( + parent, + '{cluster}-{osd_id}'.format(cluster=cluster, osd_id=osd_id), + ) + maybe_mkdir(osd_data) + + # pick best-of-breed mount options based on fs type + if mount_options is None: + mount_options = MOUNT_OPTIONS.get(fstype, '') + + # we really want to mount --move, but that is not supported when + # the parent mount is shared, as it is by default on RH, Fedora, + # and probably others. Also, --bind doesn't properly manipulate + # /etc/mtab, which *still* isn't a symlink to /proc/mounts despite + # this being 2013. Instead, mount the original device at the final + # location. + command_check_call( + [ + '/bin/mount', + '-o', + mount_options, + '--', + dev, + osd_data, + ], + ) + command_check_call( + [ + '/bin/umount', + '-l', # lazy, in case someone else is peeking at the + # wrong moment + '--', + path, + ], + ) + + +def start_daemon( + cluster, + osd_id, + ): + LOG.debug('Starting %s osd.%s...', cluster, osd_id) + + path = (STATEDIR + '/osd/{cluster}-{osd_id}').format( + cluster=cluster, osd_id=osd_id) + + # upstart? + try: + if os.path.exists(os.path.join(path,'upstart')): + command_check_call( + [ + '/sbin/initctl', + # use emit, not start, because start would fail if the + # instance was already running + 'emit', + # since the daemon starting doesn't guarantee much about + # the service being operational anyway, don't bother + # waiting for it + '--no-wait', + '--', + 'ceph-osd', + 'cluster={cluster}'.format(cluster=cluster), + 'id={osd_id}'.format(osd_id=osd_id), + ], + ) + elif os.path.exists(os.path.join(path, 'sysvinit')): + if os.path.exists('/usr/sbin/service'): + svc = '/usr/sbin/service' + else: + svc = '/sbin/service' + command_check_call( + [ + svc, + 'ceph', + '--cluster', + '{cluster}'.format(cluster=cluster), + 'start', + 'osd.{osd_id}'.format(osd_id=osd_id), + ], + ) + else: + raise Error('{cluster} osd.{osd_id} is not tagged with an init system'.format( + cluster=cluster, + osd_id=osd_id, + )) + except subprocess.CalledProcessError as e: + raise Error('ceph osd start failed', e) + + +def detect_fstype( + dev, + ): + fstype = _check_output( + args=[ + '/sbin/blkid', + # we don't want stale cached results + '-p', + '-s', 'TYPE', + '-o' 'value', + '--', + dev, + ], + ) + fstype = must_be_one_line(fstype) + return fstype + + +def mount_activate( + dev, + activate_key_template, + init, + ): + + try: + fstype = detect_fstype(dev=dev) + except (subprocess.CalledProcessError, + TruncatedLineError, + TooManyLinesError) as e: + raise FilesystemTypeError( + 'device {dev}'.format(dev=dev), + e, + ) + + # TODO always using mount options from cluster=ceph for + # now; see http://tracker.newdream.net/issues/3253 + mount_options = get_conf( + cluster='ceph', + variable='osd_mount_options_{fstype}'.format( + fstype=fstype, + ), + ) + + if mount_options is None: + mount_options = get_conf( + cluster='ceph', + variable='osd_fs_mount_options_{fstype}'.format( + fstype=fstype, + ), + ) + + #remove whitespaces from mount_options + if mount_options is not None: + mount_options = "".join(mount_options.split()) + + path = mount(dev=dev, fstype=fstype, options=mount_options) + + osd_id = None + cluster = None + try: + (osd_id, cluster) = activate(path, activate_key_template, init) + + # check if the disk is already active, or if something else is already + # mounted there + active = False + other = False + src_dev = os.stat(path).st_dev + try: + dst_dev = os.stat((STATEDIR + '/osd/{cluster}-{osd_id}').format( + cluster=cluster, + osd_id=osd_id)).st_dev + if src_dev == dst_dev: + active = True + else: + parent_dev = os.stat(STATEDIR + '/osd').st_dev + if dst_dev != parent_dev: + other = True + elif os.listdir((STATEDIR + '/osd/{cluster}-{osd_id}').format( + cluster=cluster, + osd_id=osd_id, + )): + other = True + + except OSError: + pass + + if active: + LOG.info('%s osd.%s already mounted in position; unmounting ours.' % (cluster, osd_id)) + unmount(path) + elif other: + raise Error('another %s osd.%s already mounted in position (old/different cluster instance?); unmounting ours.' % (cluster, osd_id)) + else: + move_mount( + dev=dev, + path=path, + cluster=cluster, + osd_id=osd_id, + fstype=fstype, + mount_options=mount_options, + ) + return (cluster, osd_id) + + except: + LOG.error('Failed to activate') + unmount(path) + raise + finally: + # remove our temp dir + if os.path.exists(path): + os.rmdir(path) + + +def activate_dir( + path, + activate_key_template, + init, + ): + + if not os.path.exists(path): + raise Error( + 'directory %s does not exist' % path + ) + + (osd_id, cluster) = activate(path, activate_key_template, init) + + if init not in (None, 'none' ): + canonical = (STATEDIR + '/osd/{cluster}-{osd_id}').format( + cluster=cluster, + osd_id=osd_id) + if path != canonical: + # symlink it from the proper location + create = True + if os.path.lexists(canonical): + old = os.readlink(canonical) + if old != path: + LOG.debug('Removing old symlink %s -> %s', canonical, old) + try: + os.unlink(canonical) + except: + raise Error('unable to remove old symlink', canonical) + else: + create = False + if create: + LOG.debug('Creating symlink %s -> %s', canonical, path) + try: + os.symlink(path, canonical) + except: + raise Error('unable to create symlink %s -> %s' % (canonical, path)) + + return (cluster, osd_id) + + +def find_cluster_by_uuid(_uuid): + """ + Find a cluster name by searching /etc/ceph/*.conf for a conf file + with the right uuid. + """ + _uuid = _uuid.lower() + no_fsid = [] + if not os.path.exists(SYSCONFDIR): + return None + for conf_file in os.listdir(SYSCONFDIR): + if not conf_file.endswith('.conf'): + continue + cluster = conf_file[:-5] + try: + fsid = get_fsid(cluster) + except Error as e: + if e.message != 'getting cluster uuid from configuration failed': + raise e + no_fsid.append(cluster) + else: + if fsid == _uuid: + return cluster + # be tolerant of /etc/ceph/ceph.conf without an fsid defined. + if len(no_fsid) == 1 and no_fsid[0] == 'ceph': + LOG.warning('No fsid defined in ' + SYSCONFDIR + '/ceph.conf; using anyway') + return 'ceph' + return None + + +def activate( + path, + activate_key_template, + init, + ): + + check_osd_magic(path) + + ceph_fsid = read_one_line(path, 'ceph_fsid') + if ceph_fsid is None: + raise Error('No cluster uuid assigned.') + LOG.debug('Cluster uuid is %s', ceph_fsid) + + cluster = find_cluster_by_uuid(ceph_fsid) + if cluster is None: + raise Error('No cluster conf found in ' + SYSCONFDIR + ' with fsid %s' % ceph_fsid) + LOG.debug('Cluster name is %s', cluster) + + fsid = read_one_line(path, 'fsid') + if fsid is None: + raise Error('No OSD uuid assigned.') + LOG.debug('OSD uuid is %s', fsid) + + keyring = activate_key_template.format(cluster=cluster) + + osd_id = get_osd_id(path) + if osd_id is None: + osd_id = allocate_osd_id( + cluster=cluster, + fsid=fsid, + keyring=keyring, + ) + write_one_line(path, 'whoami', osd_id) + LOG.debug('OSD id is %s', osd_id) + + if not os.path.exists(os.path.join(path, 'ready')): + LOG.debug('Initializing OSD...') + # re-running mkfs is safe, so just run until it completes + mkfs( + path=path, + cluster=cluster, + osd_id=osd_id, + fsid=fsid, + keyring=keyring, + ) + + if init not in (None, 'none' ): + if init == 'auto': + conf_val = get_conf( + cluster=cluster, + variable='init' + ) + if conf_val is not None: + init = conf_val + else: + (distro, release, codename) = platform.dist() + if distro == 'Ubuntu': + init = 'upstart' + else: + init = 'sysvinit' + + LOG.debug('Marking with init system %s', init) + with file(os.path.join(path, init), 'w'): + pass + + # remove markers for others, just in case. + for other in INIT_SYSTEMS: + if other != init: + try: + os.unlink(os.path.join(path, other)) + except OSError: + pass + + if not os.path.exists(os.path.join(path, 'active')): + LOG.debug('Authorizing OSD key...') + auth_key( + path=path, + cluster=cluster, + osd_id=osd_id, + keyring=keyring, + ) + write_one_line(path, 'active', 'ok') + LOG.debug('%s osd.%s data dir is ready at %s', cluster, osd_id, path) + return (osd_id, cluster) + + +def main_activate(args): + cluster = None + osd_id = None + + if not os.path.exists(args.path): + raise Error('%s does not exist' % args.path) + + if is_suppressed(args.path): + LOG.info('suppressed activate request on %s', args.path) + return + + activate_lock.acquire() # noqa + try: + mode = os.stat(args.path).st_mode + if stat.S_ISBLK(mode): + (cluster, osd_id) = mount_activate( + dev=args.path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + + elif stat.S_ISDIR(mode): + (cluster, osd_id) = activate_dir( + path=args.path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + + if args.mark_init == 'none': + command_check_call( + [ + 'ceph-osd', + '--cluster={cluster}'.format(cluster=cluster), + '--id={osd_id}'.format(osd_id=osd_id), + '--osd-data={path}'.format(path=args.path), + '--osd-journal={path}/journal'.format(path=args.path), + ], + ) + + else: + raise Error('%s is not a directory or block device' % args.path) + + if args.mark_init not in (None, 'none' ): + + start_daemon( + cluster=cluster, + osd_id=osd_id, + ) + + finally: + activate_lock.release() # noqa + + +########################### + +def get_journal_osd_uuid(path): + if not os.path.exists(path): + raise Error('%s does not exist' % path) + + mode = os.stat(path).st_mode + if not stat.S_ISBLK(mode): + raise Error('%s is not a block device' % path) + + try: + out = _check_output( + args=[ + 'ceph-osd', + '-i', '0', # this is ignored + '--get-journal-uuid', + '--osd-journal', + path, + ], + close_fds=True, + ) + except subprocess.CalledProcessError as e: + raise Error( + 'failed to get osd uuid/fsid from journal', + e, + ) + value = str(out).split('\n', 1)[0] + LOG.debug('Journal %s has OSD UUID %s', path, value) + return value + + +def main_activate_journal(args): + if not os.path.exists(args.dev): + raise Error('%s does not exist' % args.dev) + + cluster = None + osd_id = None + osd_uuid = None + activate_lock.acquire() # noqa + try: + osd_uuid = get_journal_osd_uuid(args.dev) + path = os.path.join('/dev/disk/by-partuuid/', osd_uuid.lower()) + + (cluster, osd_id) = mount_activate( + dev=path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + + start_daemon( + cluster=cluster, + osd_id=osd_id, + ) + + finally: + activate_lock.release() # noqa + + +########################### + + +def main_activate_all(args): + dir = '/dev/disk/by-parttypeuuid' + LOG.debug('Scanning %s', dir) + if not os.path.exists(dir): + return + err = False + for name in os.listdir(dir): + if name.find('.') < 0: + continue + (tag, uuid) = name.split('.') + + if tag == OSD_UUID or tag == DMCRYPT_OSD_UUID: + + if tag == DMCRYPT_OSD_UUID: + path = os.path.join('/dev/mapper', uuid) + else: + path = os.path.join(dir, name) + + LOG.info('Activating %s', path) + activate_lock.acquire() # noqa + try: + (cluster, osd_id) = mount_activate( + dev=path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + start_daemon( + cluster=cluster, + osd_id=osd_id, + ) + + except Exception as e: + print >> sys.stderr, '{prog}: {msg}'.format( + prog=args.prog, + msg=e, + ) + err = True + + finally: + activate_lock.release() # noqa + if err: + raise Error('One or more partitions failed to activate') + + +########################### + +def is_swap(dev): + dev = os.path.realpath(dev) + with file('/proc/swaps', 'rb') as proc_swaps: + for line in proc_swaps.readlines()[1:]: + fields = line.split() + if len(fields) < 3: + continue + swaps_dev = fields[0] + if swaps_dev.startswith('/') and os.path.exists(swaps_dev): + swaps_dev = os.path.realpath(swaps_dev) + if swaps_dev == dev: + return True + return False + + +def get_oneliner(base, name): + path = os.path.join(base, name) + if os.path.isfile(path): + with open(path, 'r') as _file: + return _file.readline().rstrip() + return None + + +def get_dev_fs(dev): + fscheck, _ = command( + [ + 'blkid', + '-s', + 'TYPE', + dev, + ], + ) + if 'TYPE' in fscheck: + fstype = fscheck.split()[1].split('"')[1] + return fstype + else: + return None + + +def get_partition_type(part): + """ + Get the GPT partition type UUID. If we have an old blkid and can't + get it that way, use sgdisk and use the description instead (and hope + dmcrypt isn't being used). + """ + blkid, _ = command( + [ + 'blkid', + '-p', + '-o', 'udev', + part, + ] + ) + saw_part_entry = False + for line in blkid.splitlines(): + (key, value) = line.split('=') + if key == 'ID_PART_ENTRY_TYPE': + return value + if key == 'ID_PART_ENTRY_SCHEME': + table_type = value + if key.startswith('ID_PART_ENTRY_'): + saw_part_entry = True + + # hmm, is it in fact GPT? + table_type = None + base = get_partition_base(part) + blkid, _ = command( + [ + 'blkid', + '-p', + '-o', 'udev', + base + ] + ) + for line in blkid.splitlines(): + (key, value) = line.split('=') + if key == 'ID_PART_TABLE_TYPE': + table_type = value + if table_type != 'gpt': + return None # not even GPT + + if saw_part_entry: + return None # GPT, and blkid appears to be new, so we're done. + + # bah, fall back to sgdisk. + if 'blkid' not in warned_about: + LOG.warning('Old blkid does not support ID_PART_ENTRY_* fields, trying sgdisk; may not correctly identify ceph volumes with dmcrypt') + warned_about['blkid'] = True + (base, partnum) = re.match('(\D+)(\d+)', part).group(1, 2) + sgdisk, _ = command( + [ + 'sgdisk', + '-p', + base, + ] + ) + + for line in sgdisk.splitlines(): + m = re.search('\s+(\d+)\s+\d+\s+\d+\s+\S+ \S+B\s+\S+\s+(.*)', line) + if m is not None: + num = m.group(1) + if num != partnum: + continue + desc = m.group(2) + # assume unencrypted ... blkid has failed us :( + if desc == 'ceph data': + return OSD_UUID + if desc == 'ceph journal': + return JOURNAL_UUID + + return None + + +def get_partition_uuid(dev): + (base, partnum) = re.match('(\D+)(\d+)', dev).group(1, 2) + out, _ = command(['sgdisk', '-i', partnum, base]) + for line in out.splitlines(): + m = re.match('Partition unique GUID: (\S+)', line) + if m: + return m.group(1).lower() + return None + + +def more_osd_info(path, uuid_map): + desc = [] + ceph_fsid = get_oneliner(path, 'ceph_fsid') + if ceph_fsid: + cluster = find_cluster_by_uuid(ceph_fsid) + if cluster: + desc.append('cluster ' + cluster) + else: + desc.append('unknown cluster ' + ceph_fsid) + + who = get_oneliner(path, 'whoami') + if who: + desc.append('osd.%s' % who) + + journal_uuid = get_oneliner(path, 'journal_uuid') + if journal_uuid: + journal_uuid = journal_uuid.lower() + if journal_uuid in uuid_map: + desc.append('journal %s' % uuid_map[journal_uuid]) + + return desc + +def list_dev_osd(dev, uuid_map): + path = is_mounted(dev) + fs_type = get_dev_fs(dev) + desc = [] + if path: + desc.append('active') + desc.extend(more_osd_info(path, uuid_map)) + elif fs_type: + try: + tpath = mount(dev=dev, fstype=fs_type, options='') + if tpath: + try: + magic = get_oneliner(tpath, 'magic') + if magic is not None: + desc.append('prepared') + desc.extend(more_osd_info(tpath, uuid_map)) + finally: + unmount(tpath) + except MountError: + pass + return desc + +def list_dev(dev, uuid_map, journal_map): + ptype = 'unknown' + prefix = '' + if is_partition(dev): + ptype = get_partition_type(dev) + prefix = ' ' + + desc = [] + if ptype == OSD_UUID: + desc = list_dev_osd(dev, uuid_map) + if desc: + desc = ['ceph data'] + desc + else: + desc = ['ceph data', 'unprepared'] + elif ptype == DMCRYPT_OSD_UUID: + holders = is_held(dev) + if not holders: + desc = ['ceph data (dmcrypt)', 'not currently mapped'] + elif len(holders) == 1: + holder = '/dev/' + holders[0] + fs_desc = list_dev_osd(holder, uuid_map) + desc = ['ceph data (dmcrypt %s)' % holder] + fs_desc + else: + desc = ['ceph data (dmcrypt)', 'holders: ' + ','.join(holders)] + elif ptype == JOURNAL_UUID: + desc.append('ceph journal') + part_uuid = get_partition_uuid(dev) + if part_uuid and part_uuid in journal_map: + desc.append('for %s' % journal_map[part_uuid]) + elif ptype == DMCRYPT_JOURNAL_UUID: + holders = is_held(dev) + if len(holders) == 1: + desc = ['ceph journal (dmcrypt /dev/%s)' % holders[0]] + else: + desc = ['ceph journal (dmcrypt)'] + part_uuid = get_partition_uuid(dev) + if part_uuid and part_uuid in journal_map: + desc.append('for %s' % journal_map[part_uuid]) + else: + path = is_mounted(dev) + fs_type = get_dev_fs(dev) + if is_swap(dev): + desc.append('swap') + else: + desc.append('other') + if fs_type: + desc.append(fs_type) + elif ptype: + desc.append(ptype) + if path: + desc.append('mounted on %s' % path) + + print '%s%s %s' % (prefix, dev, ', '.join(desc)) + + +def main_list(args): + partmap = list_all_partitions() + + uuid_map = {} + journal_map = {} + for base, parts in sorted(partmap.iteritems()): + for p in parts: + dev = get_dev_path(p) + part_uuid = get_partition_uuid(dev) + if part_uuid: + uuid_map[part_uuid] = dev + ptype = get_partition_type(dev) + if ptype == OSD_UUID: + fs_type = get_dev_fs(dev) + if fs_type is not None: + try: + tpath = mount(dev=dev, fstype=fs_type, options='') + try: + journal_uuid = get_oneliner(tpath, 'journal_uuid') + if journal_uuid: + journal_map[journal_uuid.lower()] = dev + finally: + unmount(tpath) + except MountError: + pass + if ptype == DMCRYPT_OSD_UUID: + holders = is_held(dev) + if len(holders) == 1: + holder = '/dev/' + holders[0] + fs_type = get_dev_fs(holder) + if fs_type is not None: + try: + tpath = mount(dev=holder, fstype=fs_type, options='') + try: + journal_uuid = get_oneliner(tpath, 'journal_uuid') + if journal_uuid: + journal_map[journal_uuid.lower()] = dev + finally: + unmount(tpath) + except MountError: + pass + + for base, parts in sorted(partmap.iteritems()): + if parts: + print '%s :' % get_dev_path(base) + for p in sorted(parts): + list_dev(get_dev_path(p), uuid_map, journal_map) + else: + list_dev(get_dev_path(base), uuid_map, journal_map) + + +########################### +# +# Mark devices that we want to suppress activates on with a +# file like +# +# /var/lib/ceph/tmp/suppress-activate.sdb +# +# where the last bit is the sanitized device name (/dev/X without the +# /dev/ prefix) and the is_suppress() check matches a prefix. That +# means suppressing sdb will stop activate on sdb1, sdb2, etc. +# + +def is_suppressed(path): + disk = os.path.realpath(path) + try: + if not disk.startswith('/dev/') or not stat.S_ISBLK(os.lstat(path).st_mode): + return False + base = get_dev_name(disk) + while len(base): + if os.path.exists(SUPPRESS_PREFIX + base): # noqa + return True + base = base[:-1] + except: + return False + + +def set_suppress(path): + disk = os.path.realpath(path) + if not os.path.exists(disk): + raise Error('does not exist', path) + if not stat.S_ISBLK(os.lstat(path).st_mode): + raise Error('not a block device', path) + base = get_dev_name(disk) + + with file(SUPPRESS_PREFIX + base, 'w') as f: # noqa + pass + LOG.info('set suppress flag on %s', base) + + +def unset_suppress(path): + disk = os.path.realpath(path) + if not os.path.exists(disk): + raise Error('does not exist', path) + if not stat.S_ISBLK(os.lstat(path).st_mode): + raise Error('not a block device', path) + assert disk.startswith('/dev/') + base = get_dev_name(disk) + + fn = SUPPRESS_PREFIX + base # noqa + if not os.path.exists(fn): + raise Error('not marked as suppressed', path) + + try: + os.unlink(fn) + LOG.info('unset suppress flag on %s', base) + except OSError as e: + raise Error('failed to unsuppress', e) + + +def main_suppress(args): + set_suppress(args.path) + + +def main_unsuppress(args): + unset_suppress(args.path) + + +def main_zap(args): + for dev in args.dev: + zap(dev) + +########################### + + +def setup_statedir(dir): + # XXX The following use of globals makes linting + # really hard. Global state in Python is iffy and + # should be avoided. + global STATEDIR + STATEDIR = dir + + if not os.path.exists(STATEDIR): + os.mkdir(STATEDIR) + if not os.path.exists(STATEDIR + "/tmp"): + os.mkdir(STATEDIR + "/tmp") + + global prepare_lock + prepare_lock = filelock(STATEDIR + '/tmp/ceph-disk.prepare.lock') + + global activate_lock + activate_lock = filelock(STATEDIR + '/tmp/ceph-disk.activate.lock') + + global SUPPRESS_PREFIX + SUPPRESS_PREFIX = STATEDIR + '/tmp/suppress-activate.' + + +def setup_sysconfdir(dir): + global SYSCONFDIR + SYSCONFDIR = dir + + +def parse_args(): + parser = argparse.ArgumentParser( + 'ceph-disk', + ) + parser.add_argument( + '-v', '--verbose', + action='store_true', default=None, + help='be more verbose', + ) + parser.add_argument( + '--prepend-to-path', + metavar='PATH', + default='/usr/bin', + help='prepend PATH to $PATH for backward compatibility (default /usr/bin)', + ) + parser.add_argument( + '--statedir', + metavar='PATH', + default='/var/lib/ceph', + help='directory in which ceph state is preserved (default /var/lib/ceph)', + ) + parser.add_argument( + '--sysconfdir', + metavar='PATH', + default='/etc/ceph', + help='directory in which ceph configuration files are found (default /etc/ceph)', + ) + parser.set_defaults( + # we want to hold on to this, for later + prog=parser.prog, + cluster='ceph', + ) + + subparsers = parser.add_subparsers( + title='subcommands', + description='valid subcommands', + help='sub-command help', + ) + + prepare_parser = subparsers.add_parser('prepare', help='Prepare a directory or disk for a Ceph OSD') + prepare_parser.add_argument( + '--cluster', + metavar='NAME', + help='cluster name to assign this disk to', + ) + prepare_parser.add_argument( + '--cluster-uuid', + metavar='UUID', + help='cluster uuid to assign this disk to', + ) + prepare_parser.add_argument( + '--osd-uuid', + metavar='UUID', + help='unique OSD uuid to assign this disk to', + ) + prepare_parser.add_argument( + '--journal-uuid', + metavar='UUID', + help='unique uuid to assign to the journal', + ) + prepare_parser.add_argument( + '--fs-type', + help='file system type to use (e.g. "ext4")', + ) + prepare_parser.add_argument( + '--zap-disk', + action='store_true', default=None, + help='destroy the partition table (and content) of a disk', + ) + prepare_parser.add_argument( + '--data-dir', + action='store_true', default=None, + help='verify that DATA is a dir', + ) + prepare_parser.add_argument( + '--data-dev', + action='store_true', default=None, + help='verify that DATA is a block device', + ) + prepare_parser.add_argument( + '--journal-file', + action='store_true', default=None, + help='verify that JOURNAL is a file', + ) + prepare_parser.add_argument( + '--journal-dev', + action='store_true', default=None, + help='verify that JOURNAL is a block device', + ) + prepare_parser.add_argument( + '--dmcrypt', + action='store_true', default=None, + help='encrypt DATA and/or JOURNAL devices with dm-crypt', + ) + prepare_parser.add_argument( + '--dmcrypt-key-dir', + metavar='KEYDIR', + default='/etc/ceph/dmcrypt-keys', + help='directory where dm-crypt keys are stored', + ) + prepare_parser.add_argument( + 'data', + metavar='DATA', + help='path to OSD data (a disk block device or directory)', + ) + prepare_parser.add_argument( + 'journal', + metavar='JOURNAL', + nargs='?', + help=('path to OSD journal disk block device;' + + ' leave out to store journal in file'), + ) + prepare_parser.set_defaults( + func=main_prepare, + ) + + activate_parser = subparsers.add_parser('activate', help='Activate a Ceph OSD') + activate_parser.add_argument( + '--mount', + action='store_true', default=None, + help='mount a block device [deprecated, ignored]', + ) + activate_parser.add_argument( + '--activate-key', + metavar='PATH', + help='bootstrap-osd keyring path template (%(default)s)', + dest='activate_key_template', + ) + activate_parser.add_argument( + '--mark-init', + metavar='INITSYSTEM', + help='init system to manage this dir', + default='auto', + choices=INIT_SYSTEMS, + ) + activate_parser.add_argument( + 'path', + metavar='PATH', + nargs='?', + help='path to block device or directory', + ) + activate_parser.set_defaults( + activate_key_template=STATEDIR + '/bootstrap-osd/{cluster}.keyring', + func=main_activate, + ) + + activate_journal_parser = subparsers.add_parser('activate-journal', help='Activate an OSD via its journal device') + activate_journal_parser.add_argument( + 'dev', + metavar='DEV', + help='path to journal block device', + ) + activate_journal_parser.add_argument( + '--activate-key', + metavar='PATH', + help='bootstrap-osd keyring path template (%(default)s)', + dest='activate_key_template', + ) + activate_journal_parser.add_argument( + '--mark-init', + metavar='INITSYSTEM', + help='init system to manage this dir', + default='auto', + choices=INIT_SYSTEMS, + ) + activate_journal_parser.set_defaults( + activate_key_template=STATEDIR + '/bootstrap-osd/{cluster}.keyring', + func=main_activate_journal, + ) + + activate_all_parser = subparsers.add_parser('activate-all', help='Activate all tagged OSD partitions') + activate_all_parser.add_argument( + '--activate-key', + metavar='PATH', + help='bootstrap-osd keyring path template (%(default)s)', + dest='activate_key_template', + ) + activate_all_parser.add_argument( + '--mark-init', + metavar='INITSYSTEM', + help='init system to manage this dir', + default='auto', + choices=INIT_SYSTEMS, + ) + activate_all_parser.set_defaults( + activate_key_template=STATEDIR + '/bootstrap-osd/{cluster}.keyring', + func=main_activate_all, + ) + + list_parser = subparsers.add_parser('list', help='List disks, partitions, and Ceph OSDs') + list_parser.set_defaults( + func=main_list, + ) + + suppress_parser = subparsers.add_parser('suppress-activate', help='Suppress activate on a device (prefix)') + suppress_parser.add_argument( + 'path', + metavar='PATH', + nargs='?', + help='path to block device or directory', + ) + suppress_parser.set_defaults( + func=main_suppress, + ) + + unsuppress_parser = subparsers.add_parser('unsuppress-activate', help='Stop suppressing activate on a device (prefix)') + unsuppress_parser.add_argument( + 'path', + metavar='PATH', + nargs='?', + help='path to block device or directory', + ) + unsuppress_parser.set_defaults( + func=main_unsuppress, + ) + + zap_parser = subparsers.add_parser('zap', help='Zap/erase/destroy a device\'s partition table (and contents)') + zap_parser.add_argument( + 'dev', + metavar='DEV', + nargs='+', + help='path to block device', + ) + zap_parser.set_defaults( + func=main_zap, + ) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + loglevel = logging.WARNING + if args.verbose: + loglevel = logging.DEBUG + + logging.basicConfig( + level=loglevel, + ) + + if args.prepend_to_path != '': + path = os.environ.get('PATH', os.defpath) + os.environ['PATH'] = args.prepend_to_path + ":" + path + + setup_statedir(args.statedir) + setup_sysconfdir(args.sysconfdir) + + try: + args.func(args) + + except Error as e: + raise SystemExit( + '{prog}: {msg}'.format( + prog=args.prog, + msg=e, + ) + ) + + except CephDiskException as error: + exc_name = error.__class__.__name__ + raise SystemExit( + '{prog} {exc_name}: {msg}'.format( + prog=args.prog, + exc_name=exc_name, + msg=error, + ) + ) + + +if __name__ == '__main__': + main() + warned_about = {} diff --git a/ceph/src/ceph-disk-activate b/ceph/src/ceph-disk-activate new file mode 100755 index 00000000..72e89f9a --- /dev/null +++ b/ceph/src/ceph-disk-activate @@ -0,0 +1,3 @@ +#!/bin/sh +dir=`dirname $0` +$dir/ceph-disk activate $* diff --git a/ceph/src/ceph-disk-prepare b/ceph/src/ceph-disk-prepare new file mode 100755 index 00000000..f9255eb8 --- /dev/null +++ b/ceph/src/ceph-disk-prepare @@ -0,0 +1,3 @@ +#!/bin/sh +dir=`dirname $0` +$dir/ceph-disk prepare $* diff --git a/ceph/src/ceph-disk-udev b/ceph/src/ceph-disk-udev new file mode 100755 index 00000000..bdf524e6 --- /dev/null +++ b/ceph/src/ceph-disk-udev @@ -0,0 +1,70 @@ +#! /bin/sh + +# Wrapper for the ceph udev rules. Since older versions of udev+blkid +# do not support gpt label fields, this shell script is invoked from +# the udev rule to read the needed gpt label fields and call the +# appropriate ceph OSD functions. + +PARTNO=$1 +NAME=$2 +PARENT_NAME=$3 + +# Get GPT partition type guid +ID_PART_ENTRY_TYPE=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition GUID code" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') + +if [ -z "$ID_PART_ENTRY_TYPE" ]; then + exit +fi + +ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') + +# set up the symlinks +mkdir -p /dev/disk/by-partuuid +ln -sf ../../${NAME} /dev/disk/by-partuuid/$ID_PART_ENTRY_UUID +mkdir -p /dev/disk/by-parttypeuuid +ln -sf ../../${NAME} /dev/disk/by-parttypeuuid/${ID_PART_ENTRY_TYPE}.${ID_PART_ENTRY_UUID} + +case $ID_PART_ENTRY_TYPE in + +45b0969e-9b03-4f30-b4c6-b4b80ceff106) + # JOURNAL_UUID + # activate ceph-tagged journal partitions. + /usr/sbin/ceph-disk -v activate-journal /dev/${NAME} + ;; + +45b0969e-9b03-4f30-b4c6-5ec00ceff106) + # DMCRYPT_JOURNAL_UUID + # Map journal if using dm-crypt + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME} + ;; + +4fbd7e29-9d25-41b8-afd0-062c0ceff05d) + # OSD_UUID + # activate ceph-tagged partitions. + /usr/sbin/ceph-disk -v activate /dev/${NAME} + ;; + +4fbd7e29-9d25-41b8-afd0-5ec00ceff05d) + # DMCRYPT_OSD_UUID + # Map data device and activate ceph-tagged partitions + # for dm-crypted data devices + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME} + bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done' + /usr/sbin/ceph-disk-activate /dev/mapper/${ID_PART_ENTRY_UUID} + ;; + +89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be) + # TOBE_UUID + ;; + +89c57f98-2fe5-4dc0-89c1-5ec00ceff2be) + # DMCRYPT_TOBE_UUID + ;; + +*) + # Not a Ceph device + ;; + +esac + +exit diff --git a/ceph/src/ceph-post-file.in b/ceph/src/ceph-post-file.in new file mode 100755 index 00000000..9b922a6c --- /dev/null +++ b/ceph/src/ceph-post-file.in @@ -0,0 +1,168 @@ +#!/bin/bash -e + +# if we start up as ./$0, assume we are running from a source +# checkout. +if [ `dirname $0` = "." ] && [ $PWD != "/usr/bin" ]; then + known_hosts=../share/known_hosts_drop.ceph.com + ssh_key=../share/id_dsa_drop.ceph.com +else + known_hosts=@datadir@/known_hosts_drop.ceph.com + ssh_key=@datadir@/id_dsa_drop.ceph.com +fi + +usage() { + echo "Usage: $0 [options] file1 [dir2 ...] + +Easily upload files or directories to ceph.com for analysis by Ceph +developers. + +Each invocation uploads files or directories to a separate directory +with a unique tag. That tag can be passed to a developer or +referenced in a bug report (http://tracker.ceph.com/). Once the +upload completes, the directory is marked non-readable and +non-writeable to prevent access or modification by other users. + +WARNING: + Basic measures are taken to make posted data be visible only to + developers with access to ceph.com infrastructure. However, users + should think twice and/or take appropriate precautions before + posting potentially sensitive data (for example, logs or data + directories that contain Ceph secrets). + +Options: + -d|--description Description for this post + [Default: none] + -u|--user User identifier + [Default: \`whoami\`@\`hostname -f\`] + -r|--remote Remote to upload to + [Default: postfile@drop.ceph.com] + -k|--known_hosts known_hosts file + [Default: /usr/share/ceph/known_hosts_drop.ceph.com] + -i Ssh identity file + [Default: /usr/share/ceph/id_dsa_drop.ceph.com] + -h|--help Show this usage information +" +} + +if [ -z "$*" ]; then + usage + exit 1 +fi + +description="" +user="`whoami`@`hostname -f`" +remote="postfile@drop.ceph.com" + +ARGS=$(getopt -n "ceph-post-file" -o 'd:u:hk:i:r:' -l "description:,user:,help,known-hosts:,remote:" -- "$@") +eval set -- $ARGS + +while true; do + echo "args: $@" + case $1 in + -d | --description) + description="$2" + shift + shift + ;; + -u | --user) + user="$2" + shift + shift + ;; + -h | --help) + usage + exit 0 + ;; + -k | --known-hosts) + known_hosts="$2" + shift + shift + ;; + -i) + ssh_key="$2" + shift + shift + ;; + -r | --remote) + remote="$2" + shift + shift + ;; + --) + shift + break + ;; + esac +done + +# this id should be shared +id=`uuidgen` +echo "$0: upload tag $id" + +# this is secret goop we add to the directory so that $id is not +# enough to find the data using the shared user; only ceph developers +# who have access to the server and can read the post directory can +# find the uploaded data. +nonce=`uuidgen` + +# stick the user info in the dir too +dir="${id}_${user}_${nonce}" + +t1=$(mktemp) || exit +t2=$(mktemp) || exit +t3=$(mktemp) || exit +t4=$(mktemp) || exit +trap "rm -f -- '$t1' '$t2' '$t3' '$t4'" EXIT +cat > $t1 < $t3 <> $t1 + +if [ -n "$description" ]; then + echo "$0: description: $description" + cat > $t2 <> $t1 +fi + +while [ -n "$*" ]; do + if [ -d "$1" ]; then + echo $0: will upload directory $1 + bn=`basename "$1"` + cat >> $t1 <> $t1 <> sys.stderr, DEVMODEMSG + os.execvp('python', ['python'] + sys.argv) + else: + os.environ['LD_LIBRARY_PATH'] = MYLIBPATH + print >> sys.stderr, DEVMODEMSG + os.execvp('python', ['python'] + sys.argv) + sys.path.insert(0, os.path.join(MYDIR, 'pybind')) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Ceph REST API webapp") + parser.add_argument('-c', '--conf', help='Ceph configuration file') + parser.add_argument('--cluster', help='Ceph cluster name') + parser.add_argument('-n', '--name', help='Ceph client name') + parser.add_argument('-i', '--id', help='Ceph client id') + + return parser.parse_known_args() + +# main + +parsed_args, rest = parse_args() + +# import now that env vars are available to imported module + +try: + import ceph_rest_api +except EnvironmentError as e: + print >> sys.stderr, "Error importing ceph_rest_api: ", str(e) + sys.exit(1) + +# let other exceptions generate traceback + +app = ceph_rest_api.generate_app( + parsed_args.conf, + parsed_args.cluster, + parsed_args.name, + parsed_args.id, + rest, +) + +files = [os.path.split(fr[1])[-1] for fr in inspect.stack()] +if 'pdb.py' in files: + app.run(host=app.ceph_addr, port=app.ceph_port, + debug=True, use_reloader=False, use_debugger=False) +else: + app.run(host=app.ceph_addr, port=app.ceph_port) diff --git a/ceph/src/ceph-run b/ceph/src/ceph-run new file mode 100755 index 00000000..afc3d1a5 --- /dev/null +++ b/ceph/src/ceph-run @@ -0,0 +1,21 @@ +#!/bin/sh + +sleep=5 + +while [ true ]; do + "$@" + exit_code=$? + + if [ $exit_code -gt 128 ]; then + case $(($exit_code-128)) in + 3 | 4 | 5 | 6 | 8 | 11) + ;; + *) + exit; + esac + echo ceph-run: $1 dumped core, restarting in $sleep seconds... + sleep $sleep + else + exit + fi +done diff --git a/ceph/src/ceph.in b/ceph/src/ceph.in new file mode 100755 index 00000000..82c90859 --- /dev/null +++ b/ceph/src/ceph.in @@ -0,0 +1,844 @@ +# -*- mode:python -*- +# vim: ts=4 sw=4 smarttab expandtab +# +# Processed in Makefile to add python #! line and version variable +# +# + + +""" +ceph.in becomes ceph, the command-line management tool for Ceph clusters. +This is a replacement for tools/ceph.cc and tools/common.cc. + +Copyright (C) 2013 Inktank Storage, Inc. + +This is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public +License version 2, as published by the Free Software +Foundation. See file COPYING. +""" + +import os +import sys + +# Make life easier on developers: +# If in src/, and .libs and pybind exist here, assume we're running +# from a Ceph source dir and tweak PYTHONPATH and LD_LIBRARY_PATH +# to use local files + +MYPATH = os.path.abspath(__file__) +MYDIR = os.path.dirname(MYPATH) +DEVMODEMSG = '*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***' + +if MYDIR.endswith('src') and \ + os.path.exists(os.path.join(MYDIR, '.libs')) and \ + os.path.exists(os.path.join(MYDIR, 'pybind')): + MYLIBPATH = os.path.join(MYDIR, '.libs') + if 'LD_LIBRARY_PATH' in os.environ: + if MYLIBPATH not in os.environ['LD_LIBRARY_PATH']: + os.environ['LD_LIBRARY_PATH'] += ':' + MYLIBPATH + print >> sys.stderr, DEVMODEMSG + os.execvp('python', ['python'] + sys.argv) + else: + os.environ['LD_LIBRARY_PATH'] = MYLIBPATH + print >> sys.stderr, DEVMODEMSG + os.execvp('python', ['python'] + sys.argv) + sys.path.insert(0, os.path.join(MYDIR, 'pybind')) + if os.environ.has_key('PATH') and MYDIR not in os.environ['PATH']: + os.environ['PATH'] += ':' + MYDIR + +import argparse +import errno +import json +import rados +import signal +import socket +import string +import struct +import subprocess + +from ceph_argparse import \ + concise_sig, descsort, parse_json_funcsigs, \ + matchnum, validate_command, find_cmd_target, \ + send_command, json_command + +# just a couple of globals + +verbose = False +cluster_handle = None + +############################################################################ + +def osdids(): + ret, outbuf, outs = json_command(cluster_handle, prefix='osd ls') + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, cmd=['osd', 'ls']) + if ret: + raise RuntimeError('Can\'t contact mon for osd list') + return [i for i in outbuf.split('\n') if i != ''] + +def monids(): + ret, outbuf, outs = json_command(cluster_handle, prefix='mon dump', + argdict={'format':'json'}) + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, + cmd=['mon', 'dump', '--format=json']) + if ret: + raise RuntimeError('Can\'t contact mon for mon list') + d = json.loads(outbuf) + return [m['name'] for m in d['mons']] + +def mdsids(): + ret, outbuf, outs = json_command(cluster_handle, prefix='mds dump', + argdict={'format':'json'}) + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, + cmd=['mds', 'dump', '--format=json']) + if ret: + raise RuntimeError('Can\'t contact mon for mds list') + d = json.loads(outbuf) + l = [] + infodict = d['info'] + for mdsdict in infodict.values(): + l.append(mdsdict['name']) + return l + +# these args must be passed to all child programs +GLOBAL_ARGS = { + 'client_id': '--id', + 'client_name': '--name', + 'cluster': '--cluster', + 'cephconf': '--conf', +} + +def parse_cmdargs(args=None, target=''): + # alias: let the line-wrapping be sane + AP = argparse.ArgumentParser + + # format our own help + parser = AP(description='Ceph administration tool', add_help=False) + + parser.add_argument('--completion', action='store_true', + help=argparse.SUPPRESS) + + parser.add_argument('-h', '--help', help='request mon help', + action='store_true') + + parser.add_argument('-c', '--conf', dest='cephconf', + help='ceph configuration file') + parser.add_argument('-i', '--in-file', dest='input_file', + help='input file') + parser.add_argument('-o', '--out-file', dest='output_file', + help='output file') + + parser.add_argument('--id', '--user', dest='client_id', + help='client id for authentication') + parser.add_argument('--name', '-n', dest='client_name', + help='client name for authentication') + parser.add_argument('--cluster', help='cluster name') + + parser.add_argument('--admin-daemon', dest='admin_socket', + help='submit admin-socket commands (\"help\" for help') + parser.add_argument('--admin-socket', dest='admin_socket_nope', + help='you probably mean --admin-daemon') + + parser.add_argument('-s', '--status', action='store_true', + help='show cluster status') + + parser.add_argument('-w', '--watch', action='store_true', + help='watch live cluster changes') + parser.add_argument('--watch-debug', action='store_true', + help='watch debug events') + parser.add_argument('--watch-info', action='store_true', + help='watch info events') + parser.add_argument('--watch-sec', action='store_true', + help='watch security events') + parser.add_argument('--watch-warn', action='store_true', + help='watch warn events') + parser.add_argument('--watch-error', action='store_true', + help='watch error events') + + parser.add_argument('--version', '-v', action="store_true", help="display version") + parser.add_argument('--verbose', action="store_true", help="make verbose") + parser.add_argument('--concise', dest='verbose', action="store_false", + help="make less verbose") + + parser.add_argument('-f', '--format', choices=['json', 'json-pretty', + 'xml', 'xml-pretty', 'plain'], dest='output_format') + + parser.add_argument('--connect-timeout', dest='cluster_timeout', + type=int, + help='set a timeout for connecting to the cluster') + + # returns a Namespace with the parsed args, and a list of all extras + parsed_args, extras = parser.parse_known_args(args) + + return parser, parsed_args, extras + + +def hdr(s): + print '\n', s, '\n', '=' * len(s) + +def do_basic_help(parser, args): + """ + Print basic parser help + If the cluster is available, get and print monitor help + """ + hdr('General usage:') + parser.print_help() + +def do_extended_help(parser, args): + def help_for_sigs(sigs, partial=None): + sys.stdout.write(format_help(parse_json_funcsigs(sigs, 'cli'), + partial=partial)) + + def help_for_target(target, partial=None): + ret, outbuf, outs = json_command(cluster_handle, target=target, + prefix='get_command_descriptions', + timeout=10) + if ret: + print >> sys.stderr, \ + "couldn't get command descriptions for {0}: {1}".\ + format(target, outs) + else: + help_for_sigs(outbuf, partial) + + partial = ' '.join(args) + if (cluster_handle.state == "connected"): + help_for_target(target=('mon', ''), partial=partial) + return 0 + +DONTSPLIT = string.letters + '{[<>]}' + +def wrap(s, width, indent): + """ + generator to transform s into a sequence of strings width or shorter, + for wrapping text to a specific column width. + Attempt to break on anything but DONTSPLIT characters. + indent is amount to indent 2nd-through-nth lines. + + so "long string long string long string" width=11 indent=1 becomes + 'long string', ' long string', ' long string' so that it can be printed + as + long string + long string + long string + + Consumes s. + """ + result = '' + leader = '' + while len(s): + + if (len(s) <= width): + # no splitting; just possibly indent + result = leader + s + s = '' + yield result + + else: + splitpos = width + while (splitpos > 0) and (s[splitpos-1] in DONTSPLIT): + splitpos -= 1 + + if splitpos == 0: + splitpos = width + + if result: + # prior result means we're mid-iteration, indent + result = leader + else: + # first time, set leader and width for next + leader = ' ' * indent + width -= 1 # for subsequent space additions + + # remove any leading spaces in this chunk of s + result += s[:splitpos].lstrip() + s = s[splitpos:] + + yield result + + raise StopIteration + +def format_help(cmddict, partial=None): + """ + Formats all the cmdsigs and helptexts from cmddict into a sorted-by- + cmdsig 2-column display, with each column wrapped and indented to + fit into 40 characters. + """ + + fullusage = '' + for cmd in sorted(cmddict.itervalues(), cmp=descsort): + + if not cmd['help']: + continue + concise = concise_sig(cmd['sig']) + if partial and not concise.startswith(partial): + continue + siglines = [l for l in wrap(concise, 40, 1)] + helplines = [l for l in wrap(cmd['help'], 39, 1)] + + # make lists the same length + maxlen = max(len(siglines), len(helplines)) + siglines.extend([''] * (maxlen - len(siglines))) + helplines.extend([''] * (maxlen - len(helplines))) + + # so we can zip them for output + for (s, h) in zip(siglines, helplines): + fullusage += '{0:40s} {1}\n'.format(s, h) + + return fullusage + +def admin_socket(asok_path, cmd, format=''): + """ + Send a daemon (--admin-daemon) command 'cmd'. asok_path is the + path to the admin socket; cmd is a list of strings; format may be + set to one of the formatted forms to get output in that form + (daemon commands don't support 'plain' output). + """ + + def do_sockio(path, cmd): + """ helper: do all the actual low-level stream I/O """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + try: + sock.sendall(cmd + '\0') + len_str = sock.recv(4) + if len(len_str) < 4: + raise RuntimeError("no data returned from admin socket") + l, = struct.unpack(">I", len_str) + ret = '' + + got = 0 + while got < l: + bit = sock.recv(l - got) + ret += bit + got += len(bit) + + except Exception as e: + raise RuntimeError('exception: ' + str(e)) + return ret + + try: + cmd_json = do_sockio(asok_path, + json.dumps({"prefix":"get_command_descriptions"})) + except Exception as e: + raise RuntimeError('exception getting command descriptions: ' + str(e)) + + if cmd == 'get_command_descriptions': + return cmd_json + + sigdict = parse_json_funcsigs(cmd_json, 'cli') + valid_dict = validate_command(sigdict, cmd) + if not valid_dict: + raise RuntimeError('invalid command') + + if format: + valid_dict['format'] = format + + try: + ret = do_sockio(asok_path, json.dumps(valid_dict)) + except Exception as e: + raise RuntimeError('exception: ' + str(e)) + + return ret + + +def ceph_conf(parsed_args, field, name): + args=['ceph-conf'] + + if name: + args.extend(['--name', name]) + + # add any args in GLOBAL_ARGS + for key, val in GLOBAL_ARGS.iteritems(): + # ignore name in favor of argument name, if any + if name and key == 'client_name': + continue + if getattr(parsed_args, key): + args.extend([val, getattr(parsed_args, key)]) + + args.extend(['--show-config-value', field]) + p = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + outdata, errdata = p.communicate() + if (len(errdata)): + raise RuntimeError('unable to get conf option %s for %s: %s' % (field, name, errdata)) + return outdata.rstrip() + +def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): + """ + Do new-style command dance. + target: daemon to receive command: mon (any) or osd.N + sigdict - the parsed output from the new monitor describing commands + inbuf - any -i input file data + verbose - bool + """ + if verbose: + for cmdtag in sorted(sigdict.keys()): + cmd = sigdict[cmdtag] + sig = cmd['sig'] + print '{0}: {1}'.format(cmdtag, concise_sig(sig)) + + got_command = False + + if not got_command: + if cmdargs: + # Validate input args against list of sigs + valid_dict = validate_command(sigdict, cmdargs, verbose) + if valid_dict: + got_command = True + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format + else: + return -errno.EINVAL, '', 'invalid command' + else: + # do the command-interpreter looping + # for raw_input to do readline cmd editing + import readline + while True: + interactive_input = raw_input('ceph> ') + if interactive_input in ['q', 'quit', 'Q']: + return 0, '', '' + cmdargs = parse_cmdargs(interactive_input.split())[2] + target = find_cmd_target(cmdargs) + valid_dict = validate_command(sigdict, cmdargs, verbose) + if valid_dict: + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format + if verbose: + print >> sys.stderr, "Submitting command ", valid_dict + ret, outbuf, outs = json_command(cluster_handle, + target=target, + argdict=valid_dict) + if ret: + ret = abs(ret) + print >> sys.stderr, \ + 'Error: {0} {1}'.format(ret, errno.errorcode[ret]) + if outbuf: + print outbuf + if outs: + print >> sys.stderr, 'Status:\n', outs + else: + print >> sys.stderr, "Invalid command" + + if verbose: + print >> sys.stderr, "Submitting command ", valid_dict + return json_command(cluster_handle, target=target, argdict=valid_dict, + inbuf=inbuf) + +def complete(sigdict, args, target): + """ + Command completion. Match as much of [args] as possible, + and print every possible match separated by newlines. + Return exitcode. + """ + # XXX this looks a lot like the front of validate_command(). Refactor? + + complete_verbose = 'COMPVERBOSE' in os.environ + + # Repulsive hack to handle tell: lop off 'tell' and target + # and validate the rest of the command. 'target' is already + # determined in our callers, so it's ok to remove it here. + if len(args) and args[0] == 'tell': + args = args[2:] + # look for best match, accumulate possibles in bestcmds + # (so we can maybe give a more-useful error message) + best_match_cnt = 0 + bestcmds = [] + for cmdtag, cmd in sigdict.iteritems(): + sig = cmd['sig'] + matched = matchnum(args, sig, partial=True) + if (matched > best_match_cnt): + if complete_verbose: + print >> sys.stderr, \ + "better match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + best_match_cnt = matched + bestcmds = [{cmdtag:cmd}] + elif matched == best_match_cnt: + if complete_verbose: + print >> sys.stderr, \ + "equal match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + bestcmds.append({cmdtag:cmd}) + + # look through all matching sigs + comps = [] + for cmddict in bestcmds: + for cmd in cmddict.itervalues(): + sig = cmd['sig'] + # either: + # we match everything fully, so we want the next desc, or + # we match more partially, so we want the partial match + fullindex = matchnum(args, sig, partial=False) - 1 + partindex = matchnum(args, sig, partial=True) - 1 + if complete_verbose: + print >> sys.stderr, '{}: f {} p {} len {}'.format(sig, fullindex, partindex, len(sig)) + if fullindex == partindex and fullindex + 1 < len(sig): + d = sig[fullindex + 1] + else: + d = sig[partindex] + comps.append(str(d)) + if complete_verbose: + print >> sys.stderr, '\n'.join(comps) + print '\n'.join(comps) + + return 0 + +### +# ping a monitor +### +def ping_monitor(cluster_handle, name): + if 'mon.' not in name: + print >> sys.stderr, '"ping" expects a monitor to ping; try "ping mon."' + return 1 + + mon_id = name[len('mon.'):] + s = cluster_handle.ping_monitor(mon_id) + print s + return 0 + +### +# main +### + +def main(): + ceph_args = os.environ.get('CEPH_ARGS') + if ceph_args: + sys.argv.extend(ceph_args.split()) + + parser, parsed_args, childargs = parse_cmdargs() + + if parsed_args.version: + print 'ceph version {0} ({1})'.format(CEPH_GIT_NICE_VER, CEPH_GIT_VER) + return 0 + + global verbose + verbose = parsed_args.verbose + + if parsed_args.admin_socket_nope: + print >> sys.stderr, '--admin-socket is used by daemons; '\ + 'you probably mean --admin-daemon/daemon' + return 1 + + # pass on --id, --name, --conf + name = 'client.admin' + if parsed_args.client_id: + name = 'client.' + parsed_args.client_id + if parsed_args.client_name: + name = parsed_args.client_name + + # default '' means default conf search + conffile = '' + if parsed_args.cephconf: + conffile = parsed_args.cephconf + # For now, --admin-daemon is handled as usual. Try it + # first in case we can't connect() to the cluster + + format = parsed_args.output_format + + sockpath = None + if parsed_args.admin_socket: + sockpath = parsed_args.admin_socket + elif len(childargs) > 0 and childargs[0] == "daemon": + # Treat "daemon " or "daemon " like --admin_daemon + if len(childargs) > 2: + if childargs[1].find('/') >= 0: + sockpath = childargs[1] + else: + # try resolve daemon name + try: + sockpath = ceph_conf(parsed_args, 'admin_socket', + childargs[1]) + except Exception as e: + print >> sys.stderr, \ + 'Can\'t get admin socket path: ' + str(e) + return errno.EINVAL + # for both: + childargs = childargs[2:] + else: + print >> sys.stderr, 'daemon requires at least 3 arguments' + return errno.EINVAL + + if sockpath: + try: + print admin_socket(sockpath, childargs, format) + except Exception as e: + print >> sys.stderr, 'admin_socket: {0}'.format(e) + return errno.EINVAL + return 0 + + timeout = None + if parsed_args.cluster_timeout: + timeout = parsed_args.cluster_timeout + + # basic help + if parsed_args.help: + do_basic_help(parser, childargs) + + # handle any 'generic' ceph arguments that we didn't parse here + global cluster_handle + + # rados.Rados() will call rados_create2, and then read the conf file, + # and then set the keys from the dict. So we must do these + # "pre-file defaults" first (see common_preinit in librados) + conf_defaults = { + 'log_to_stderr':'true', + 'err_to_stderr':'true', + 'log_flush_on_exit':'true', + } + + clustername = 'ceph' + if parsed_args.cluster: + clustername = parsed_args.cluster + + try: + cluster_handle = rados.Rados(name=name, clustername=clustername, + conf_defaults=conf_defaults, + conffile=conffile) + retargs = cluster_handle.conf_parse_argv(childargs) + except rados.Error as e: + print >> sys.stderr, 'Error initializing cluster client: {0}'.\ + format(e.__class__.__name__) + return 1 + + #tmp = childargs + childargs = retargs + if not childargs: + childargs = [] + + # -- means "stop parsing args", but we don't want to see it either + if '--' in childargs: + childargs.remove('--') + + # special deprecation warning for 'ceph tell' + # someday 'mds' will be here too + if len(childargs) >= 2 and \ + childargs[0] in ['mon', 'osd'] and \ + childargs[1] == 'tell': + print >> sys.stderr, '"{0} tell" is deprecated; try "tell {0}." instead (id can be "*") '.format(childargs[0]) + return 1 + + if parsed_args.help: + # short default timeout for -h + if not timeout: + timeout = 5 + + hdr('Monitor commands:') + print '[Contacting monitor, timeout after %d seconds]' % timeout + + if childargs and childargs[0] == 'ping': + if len(childargs) < 2: + print >> sys.stderr, '"ping" requires a monitor name as argument: "ping mon."' + return 1 + + try: + if childargs and childargs[0] == 'ping': + return ping_monitor(cluster_handle, childargs[1]) + cluster_handle.connect(timeout=timeout) + except KeyboardInterrupt: + print >> sys.stderr, 'Cluster connection aborted' + return 1 + except Exception as e: + print >> sys.stderr, 'Error connecting to cluster: {0}'.\ + format(e.__class__.__name__) + return 1 + + if parsed_args.help: + return do_extended_help(parser, childargs) + + # implement -w/--watch_* + # This is ugly, but Namespace() isn't quite rich enough. + level = '' + for k, v in parsed_args._get_kwargs(): + if k.startswith('watch') and v: + if k == 'watch': + level = 'info' + else: + level = k.replace('watch_', '') + if level: + + # an awfully simple callback + def watch_cb(arg, line, who, stamp_sec, stamp_nsec, seq, level, msg): + print line + sys.stdout.flush() + + # first do a ceph status + ret, outbuf, outs = json_command(cluster_handle, prefix='status') + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, cmd=['status']) + # old mon returns status to outs...ick + if ret == 0: + outbuf += outs + if ret: + print >> sys.stderr, "status query failed: ", outs + return ret + print outbuf + + # this instance keeps the watch connection alive, but is + # otherwise unused + logwatch = rados.MonitorLog(cluster_handle, level, watch_cb, 0) + + # loop forever letting watch_cb print lines + try: + signal.pause() + except KeyboardInterrupt: + # or until ^C, at least + return 0 + + # read input file, if any + inbuf = '' + if parsed_args.input_file: + try: + with open(parsed_args.input_file, 'r') as f: + inbuf = f.read() + except Exception as e: + print >> sys.stderr, 'Can\'t open input file {0}: {1}'.format(parsed_args.input_file, e) + return 1 + + # prepare output file, if any + if parsed_args.output_file: + try: + outf = open(parsed_args.output_file, 'w') + except Exception as e: + print >> sys.stderr, \ + 'Can\'t open output file {0}: {1}'.\ + format(parsed_args.output_file, e) + return 1 + + # -s behaves like a command (ceph status). + if parsed_args.status: + childargs.insert(0, 'status') + + target = find_cmd_target(childargs) + + # Repulsive hack to handle tell: lop off 'tell' and target + # and validate the rest of the command. 'target' is already + # determined in our callers, so it's ok to remove it here. + is_tell = False + if len(childargs) and childargs[0] == 'tell': + childargs = childargs[2:] + is_tell = True + + if is_tell and not len(childargs): + print >> sys.stderr, \ + 'Cannot use \'tell\' with interactive mode' + return errno.EINVAL + + # fetch JSON sigs from command + # each line contains one command signature (a placeholder name + # of the form 'cmdNNN' followed by an array of argument descriptors) + # as part of the validated argument JSON object + + targets = [target] + + if target[1] == '*': + if target[0] == 'osd': + targets = [(target[0], o) for o in osdids()] + elif target[0] == 'mon': + targets = [(target[0], m) for m in monids()] + + final_ret = 0 + for target in targets: + # prettify? prefix output with target, if there was a wildcard used + prefix = '' + suffix = '' + if not parsed_args.output_file and len(targets) > 1: + prefix = '{0}.{1}: '.format(*target) + suffix = '\n' + + ret, outbuf, outs = json_command(cluster_handle, target=target, + prefix='get_command_descriptions') + compat = False + if ret == -errno.EINVAL: + # send command to old monitor or OSD + if verbose: + print prefix + '{0} to old {1}'.format(' '.join(childargs), target[0]) + compat = True + if parsed_args.output_format: + childargs.extend(['--format', parsed_args.output_format]) + ret, outbuf, outs = send_command(cluster_handle, target, childargs, + inbuf) + + if ret == -errno.EINVAL: + # did we race with a mon upgrade? try again! + ret, outbuf, outs = json_command(cluster_handle, target=target, + prefix='get_command_descriptions') + if ret == 0: + compat = False # yep, carry on + if not compat: + if ret: + if ret < 0: + outs = 'problem getting command descriptions from {0}.{1}'.format(*target) + else: + sigdict = parse_json_funcsigs(outbuf, 'cli') + + if parsed_args.completion: + return complete(sigdict, childargs, target) + + ret, outbuf, outs = new_style_command(parsed_args, childargs, target, + sigdict, inbuf, verbose) + + # debug tool: send any successful command *again* to + # verify that it is idempotent. + if not ret and 'CEPH_CLI_TEST_DUP_COMMAND' in os.environ: + ret, outbuf, outs = new_style_command(parsed_args, childargs, target, + sigdict, inbuf, verbose) + if ret < 0: + ret = -ret + print >> sys.stderr, prefix + 'Second attempt of previously successful command failed with {0}: {1}'.format(errno.errorcode[ret], outs) + + if ret < 0: + ret = -ret + print >> sys.stderr, prefix + 'Error {0}: {1}'.format(errno.errorcode[ret], outs) + if len(targets) > 1: + final_ret = ret + else: + return ret + + # this assumes outs never has useful command output, only status + if compat: + if ret == 0: + # old cli/mon would send status string to stdout on non-error + print outs + else: + if outs: + print >> sys.stderr, prefix + outs + + if (parsed_args.output_file): + outf.write(outbuf) + else: + # hack: old code printed status line before many json outputs + # (osd dump, etc.) that consumers know to ignore. Add blank line + # to satisfy consumers that skip the first line, but not annoy + # consumers that don't. + if parsed_args.output_format and \ + parsed_args.output_format.startswith('json') and \ + not compat: + sys.stdout.write('\n') + + # if we are prettifying things, normalize newlines. sigh. + if suffix != '': + outbuf = outbuf.rstrip() + if outbuf != '': + sys.stdout.write(prefix + outbuf + suffix) + + sys.stdout.flush() + + if (parsed_args.output_file): + outf.close() + + if final_ret: + return final_ret + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ceph/src/ceph_common.sh b/ceph/src/ceph_common.sh new file mode 100644 index 00000000..07faddc2 --- /dev/null +++ b/ceph/src/ceph_common.sh @@ -0,0 +1,239 @@ +#!/bin/sh + +CCONF="$BINDIR/ceph-conf" + +default_conf=$ETCDIR"/ceph.conf" +conf=$default_conf + +hostname=`hostname -s` + +verify_conf() { + # fetch conf? + if [ -x "$ETCDIR/fetch_config" ] && [ "$conf" = "$default_conf" ]; then + conf="/tmp/fetched.ceph.conf.$$" + echo "[$ETCDIR/fetch_config $conf]" + if $ETCDIR/fetch_config $conf && [ -e $conf ]; then true ; else + echo "$0: failed to fetch config with '$ETCDIR/fetch_config $conf'" + exit 1 + fi + # yay! + else + # make sure ceph.conf exists + if [ ! -e $conf ]; then + if [ "$conf" = "$default_conf" ]; then + echo "$0: ceph conf $conf not found; system is not configured." + exit 0 + fi + echo "$0: ceph conf $conf not found!" + usage_exit + fi + fi +} + + +check_host() { + # what host is this daemon assigned to? + host=`$CCONF -c $conf -n $type.$id host` + if [ "$host" = "localhost" ]; then + echo "$0: use a proper short hostname (hostname -s), not 'localhost', in $conf section $type.$id; skipping entry" + return 1 + fi + if expr match "$host" '.*\.' > /dev/null 2>&1; then + echo "$0: $conf section $type.$id" + echo "contains host=$host, which contains dots; this is probably wrong" + echo "It must match the result of hostname -s" + fi + ssh="" + rootssh="" + sshdir=$PWD + get_conf user "" "user" + + #echo host for $name is $host, i am $hostname + + cluster=$1 + if [ -e "/var/lib/ceph/$type/$cluster-$id/upstart" ]; then + return 1 + fi + + # sysvinit managed instance in standard location? + if [ -e "/var/lib/ceph/$type/$cluster-$id/sysvinit" ]; then + host="$hostname" + echo "=== $type.$id === " + return 0 + fi + + # ignore all sections without 'host' defined + if [ -z "$host" ]; then + return 1 + fi + + if [ "$host" != "$hostname" ]; then + # skip, unless we're starting remote daemons too + if [ $allhosts -eq 0 ]; then + return 1 + fi + + # we'll need to ssh into that host + if [ -z "$user" ]; then + ssh="ssh $host" + else + ssh="ssh $user@$host" + fi + rootssh="ssh root@$host" + get_conf sshdir "$sshdir" "ssh path" + fi + + echo "=== $type.$id === " + + return 0 +} + +do_cmd() { + if [ -z "$ssh" ]; then + [ $verbose -eq 1 ] && echo "--- $host# $1" + ulimit -c unlimited + whoami=`whoami` + if [ "$whoami" = "$user" ] || [ -z "$user" ]; then + bash -c "$1" || { [ -z "$3" ] && echo "failed: '$1'" && exit 1; } + else + sudo su $user -c "$1" || { [ -z "$3" ] && echo "failed: '$1'" && exit 1; } + fi + else + [ $verbose -eq 1 ] && echo "--- $ssh $2 \"if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi; cd $sshdir ; ulimit -c unlimited ; $1\"" + $ssh $2 "if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi; cd $sshdir ; ulimit -c unlimited ; $1" || { [ -z "$3" ] && echo "failed: '$ssh $1'" && exit 1; } + fi +} + +do_cmd_okfail() { + ERR=0 + if [ -z "$ssh" ]; then + [ $verbose -eq 1 ] && echo "--- $host# $1" + ulimit -c unlimited + whoami=`whoami` + if [ "$whoami" = "$user" ] || [ -z "$user" ]; then + bash -c "$1" || { [ -z "$3" ] && echo "failed: '$1'" && ERR=1 && return 1; } + else + sudo su $user -c "$1" || { [ -z "$3" ] && echo "failed: '$1'" && ERR=1 && return 1; } + fi + else + [ $verbose -eq 1 ] && echo "--- $ssh $2 \"if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi; cd $sshdir ; ulimit -c unlimited ; $1\"" + $ssh $2 "if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi; cd $sshdir ; ulimit -c unlimited ; $1" || { [ -z "$3" ] && echo "failed: '$ssh $1'" && ERR=1 && return 1; } + fi + return 0 +} + +do_root_cmd() { + if [ -z "$ssh" ]; then + [ $verbose -eq 1 ] && echo "--- $host# $1" + ulimit -c unlimited + whoami=`whoami` + if [ "$whoami" = "root" ]; then + bash -c "$1" || { echo "failed: '$1'" ; exit 1; } + else + sudo bash -c "$1" || { echo "failed: '$1'" ; exit 1; } + fi + else + [ $verbose -eq 1 ] && echo "--- $rootssh $2 \"if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi ; cd $sshdir ; ulimit -c unlimited ; $1\"" + $rootssh $2 "if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi ; cd $sshdir; ulimit -c unlimited ; $1" || { echo "failed: '$rootssh $1'" ; exit 1; } + fi +} + +do_root_cmd_okfail() { + ERR=0 + if [ -z "$ssh" ]; then + [ $verbose -eq 1 ] && echo "--- $host# $1" + ulimit -c unlimited + whoami=`whoami` + if [ "$whoami" = "root" ]; then + bash -c "$1" || { [ -z "$3" ] && echo "failed: '$1'" && ERR=1 && return 1; } + else + sudo bash -c "$1" || { [ -z "$3" ] && echo "failed: '$1'" && ERR=1 && return 1; } + fi + else + [ $verbose -eq 1 ] && echo "--- $rootssh $2 \"if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi; cd $sshdir ; ulimit -c unlimited ; $1\"" + $rootssh $2 "if [ ! -d $sshdir ]; then mkdir -p $sshdir; fi; cd $sshdir ; ulimit -c unlimited ; $1" || { [ -z "$3" ] && echo "failed: '$rootssh $1'" && ERR=1 && return 1; } + fi + return 0 +} + +get_local_daemon_list() { + type=$1 + if [ -d "/var/lib/ceph/$type" ]; then + for i in `find -L /var/lib/ceph/$type -mindepth 1 -maxdepth 1 -type d -printf '%f\n'`; do + if [ -e "/var/lib/ceph/$type/$i/sysvinit" ]; then + id=`echo $i | sed 's/[^-]*-//'` + local="$local $type.$id" + fi + done + fi +} + +get_local_name_list() { + # enumerate local directories + local="" + get_local_daemon_list "mon" + get_local_daemon_list "osd" + get_local_daemon_list "mds" +} + +get_name_list() { + orig="$*" + + # extract list of monitors, mdss, osds defined in startup.conf + allconf="$local "`$CCONF -c $conf -l mon | egrep -v '^mon$' || true ; \ + $CCONF -c $conf -l mds | egrep -v '^mds$' || true ; \ + $CCONF -c $conf -l osd | egrep -v '^osd$' || true` + + if [ -z "$orig" ]; then + what="$allconf" + return + fi + + what="" + for f in $orig; do + type=`echo $f | cut -c 1-3` # e.g. 'mon', if $item is 'mon1' + id=`echo $f | cut -c 4- | sed 's/\\.//'` + case $f in + mon | osd | mds) + for d in $allconf; do + if echo $d | grep -q ^$type; then + what="$what $d" + fi + done + ;; + *) + if ! echo " " $allconf $local " " | egrep -q "( $type$id | $type.$id )"; then + echo "$0: $type.$id not found ($conf defines" $allconf", /var/lib/ceph defines" $local")" + exit 1 + fi + what="$what $f" + ;; + esac + done +} + +get_conf() { + var=$1 + def=$2 + key=$3 + shift; shift; shift + + if [ -z "$1" ]; then + [ "$verbose" -eq 1 ] && echo "$CCONF -c $conf -n $type.$id \"$key\"" + eval "$var=\"`$CCONF -c $conf -n $type.$id \"$key\" || eval echo -n \"$def\"`\"" + else + [ "$verbose" -eq 1 ] && echo "$CCONF -c $conf -s $1 \"$key\"" + eval "$var=\"`$CCONF -c $conf -s $1 \"$key\" || eval echo -n \"$def\"`\"" + fi +} + +get_conf_bool() { + get_conf "$@" + + eval "val=$"$1 + [ "$val" = "0" ] && export $1=0 + [ "$val" = "false" ] && export $1=0 + [ "$val" = "1" ] && export $1=1 + [ "$val" = "true" ] && export $1=1 +} + diff --git a/ceph/src/ceph_fuse.cc b/ceph/src/ceph_fuse.cc new file mode 100644 index 00000000..54616f60 --- /dev/null +++ b/ceph/src/ceph_fuse.cc @@ -0,0 +1,218 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +using namespace std; + +#include "common/config.h" +#include "common/errno.h" + +#include "client/Client.h" +#include "client/fuse_ll.h" + +#include "msg/Messenger.h" + +#include "mon/MonClient.h" + +#include "common/Timer.h" +#include "common/ceph_argparse.h" +#include "global/global_init.h" +#include "common/safe_io.h" + +#ifndef DARWIN +#include +#endif // DARWIN + +#include +#include + +void usage() +{ + cerr << "usage: ceph-fuse [-m mon-ip-addr:mon-port] " << std::endl; +} + +int main(int argc, const char **argv, const char *envp[]) { + int filer_flags = 0; + //cerr << "ceph-fuse starting " << myrank << "/" << world << std::endl; + vector args; + argv_to_vec(argc, argv, args); + env_to_vec(args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_DAEMON, + CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS); + for (std::vector::iterator i = args.begin(); i != args.end(); ) { + if (ceph_argparse_double_dash(args, i)) { + break; + } else if (ceph_argparse_flag(args, i, "--localize-reads", (char*)NULL)) { + cerr << "setting CEPH_OSD_FLAG_LOCALIZE_READS" << std::endl; + filer_flags |= CEPH_OSD_FLAG_LOCALIZE_READS; + } else { + ++i; + } + } + + // args for fuse + const char **newargv; + int newargc; + vec_to_argv(argv[0], args, &newargc, &newargv); + + // FUSE will chdir("/"); be ready. + g_ceph_context->_conf->set_val("chdir", "/"); + g_ceph_context->_conf->apply_changes(NULL); + + // check for 32-bit arch + if (sizeof(long) == 4) { + cerr << std::endl; + cerr << "WARNING: Ceph inode numbers are 64 bits wide, and FUSE on 32-bit kernels does" << std::endl; + cerr << " not cope well with that situation. Expect to crash shortly." << std::endl; + cerr << std::endl; + } + + // we need to handle the forking ourselves. + int fd[2] = {0, 0}; // parent's, child's + pid_t childpid = 0; + bool restart_log = false; + if (g_conf->daemonize) { + int r = socketpair(AF_UNIX, SOCK_STREAM, 0, fd); + if (r < 0) { + cerr << "ceph-fuse[" << getpid() << "]: unable to create socketpair: " << cpp_strerror(errno) << std::endl; + exit(1); + } + + g_ceph_context->_log->stop(); + restart_log = true; + + childpid = fork(); + } + + if (childpid == 0) { + common_init_finish(g_ceph_context); + + //cout << "child, mounting" << std::endl; + ::close(fd[0]); + + if (restart_log) + g_ceph_context->_log->start(); + + // get monmap + Messenger *messenger = NULL; + Client *client; + CephFuse *cfuse; + + MonClient mc(g_ceph_context); + int r = mc.build_initial_monmap(); + if (r == -EINVAL) + usage(); + if (r < 0) + goto out_mc_start_failed; + + // start up network + messenger = Messenger::create(g_ceph_context, + entity_name_t::CLIENT(), "client", + getpid()); + messenger->set_default_policy(Messenger::Policy::lossy_client(0, 0)); + messenger->set_policy(entity_name_t::TYPE_MDS, + Messenger::Policy::lossless_client(0, 0)); + + client = new Client(messenger, &mc); + if (filer_flags) { + client->set_filer_flags(filer_flags); + } + + cfuse = new CephFuse(client, fd[1]); + + cout << "ceph-fuse[" << getpid() << "]: starting ceph client" << std::endl; + r = messenger->start(); + if (r < 0) { + cerr << "ceph-fuse[" << getpid() << "]: ceph mount failed with " << cpp_strerror(-r) << std::endl; + goto out_messenger_start_failed; + } + + // start client + r = client->init(); + if (r < 0) { + cerr << "ceph-fuse[" << getpid() << "]: ceph mount failed with " << cpp_strerror(-r) << std::endl; + goto out_init_failed; + } + + // start up fuse + // use my argc, argv (make sure you pass a mount point!) + r = client->mount(g_conf->client_mountpoint.c_str()); + if (r < 0) { + cerr << "ceph-fuse[" << getpid() << "]: ceph mount failed with " << cpp_strerror(-r) << std::endl; + goto out_shutdown; + } + + r = cfuse->init(newargc, newargv); + if (r != 0) { + cerr << "ceph-fuse[" << getpid() << "]: fuse failed to initialize" << std::endl; + goto out_client_unmount; + } + cerr << "ceph-fuse[" << getpid() << "]: starting fuse" << std::endl; + r = cfuse->loop(); + cerr << "ceph-fuse[" << getpid() << "]: fuse finished with error " << r << std::endl; + + out_client_unmount: + client->unmount(); + //cout << "unmounted" << std::endl; + + cfuse->finalize(); + delete cfuse; + + out_shutdown: + client->shutdown(); + out_init_failed: + // wait for messenger to finish + messenger->shutdown(); + messenger->wait(); + out_messenger_start_failed: + delete client; + out_mc_start_failed: + + if (g_conf->daemonize) { + //cout << "child signalling parent with " << r << std::endl; + static int foo = 0; + foo += ::write(fd[1], &r, sizeof(r)); + } + + delete messenger; + g_ceph_context->put(); + free(newargv); + + //cout << "child done" << std::endl; + return r; + } else { + // i am the parent + //cout << "parent, waiting for signal" << std::endl; + ::close(fd[1]); + + int r = -1; + int err = safe_read_exact(fd[0], &r, sizeof(r)); + if (err == 0 && r == 0) { + // close stdout, etc. + //cout << "success" << std::endl; + ::close(0); + ::close(1); + ::close(2); + } else if (err) { + cerr << "ceph-fuse[" << getpid() << "]: mount failed: " << cpp_strerror(-err) << std::endl; + } else { + cerr << "ceph-fuse[" << getpid() << "]: mount failed: " << cpp_strerror(-r) << std::endl; + } + return r; + } +} + diff --git a/ceph/src/ceph_mds.cc b/ceph/src/ceph_mds.cc new file mode 100644 index 00000000..1f1fbd31 --- /dev/null +++ b/ceph/src/ceph_mds.cc @@ -0,0 +1,332 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include + +#include +#include +using namespace std; + +#include "include/ceph_features.h" + +#include "common/config.h" +#include "common/strtol.h" + +#include "mon/MonMap.h" +#include "mds/MDS.h" +#include "mds/Dumper.h" +#include "mds/Resetter.h" + +#include "msg/Messenger.h" + +#include "common/Timer.h" +#include "common/ceph_argparse.h" +#include "common/pick_address.h" + +#include "global/global_init.h" +#include "global/signal_handler.h" +#include "global/pidfile.h" + +#include "mon/MonClient.h" + +#include "auth/KeyRing.h" + +#include "include/assert.h" + +#define dout_subsys ceph_subsys_mds + +void usage() +{ + derr << "usage: ceph-mds -i name [flags] [[--journal_check rank]|[--hot-standby][rank]]\n" + << " -m monitorip:port\n" + << " connect to monitor at given address\n" + << " --debug_mds n\n" + << " debug MDS level (e.g. 10)\n" + << " --dump-journal rank filename\n" + << " dump the MDS journal (binary) for rank.\n" + << " --dump-journal-entries rank filename\n" + << " dump the MDS journal (JSON) for rank.\n" + << " --journal-check rank\n" + << " replay the journal for rank, then exit\n" + << " --hot-standby rank\n" + << " start up as a hot standby for rank\n" + << " --reset-journal rank\n" + << " discard the MDS journal for rank, and replace it with a single\n" + << " event that updates/resets inotable and sessionmap on replay.\n" + << dendl; + generic_server_usage(); +} + +static int do_cmds_special_action(const std::string &action, + const std::string &dump_file, int rank) +{ + common_init_finish(g_ceph_context, CINIT_FLAG_NO_DAEMON_ACTIONS); + + if (action == "dump-journal") { + dout(0) << "dumping journal for mds." << rank << " to " << dump_file << dendl; + Dumper journal_dumper; + journal_dumper.init(rank); + journal_dumper.dump(dump_file.c_str()); + journal_dumper.shutdown(); + } else if (action == "dump-journal-entries") { + Dumper journal_dumper; + journal_dumper.init(rank); + journal_dumper.dump_entries(); + journal_dumper.shutdown(); + } else if (action == "undump-journal") { + dout(0) << "undumping journal for mds." << rank << " from " << dump_file << dendl; + Dumper journal_dumper; + journal_dumper.init(rank); + journal_dumper.undump(dump_file.c_str()); + journal_dumper.shutdown(); + } else if (action == "reset-journal") { + dout(0) << "resetting journal" << dendl; + Resetter resetter; + resetter.init(rank); + resetter.reset(); + resetter.shutdown(); + } else { + assert(0); + } + return 0; +} + +static void set_special_action(std::string &dest, const std::string &act) +{ + if (!dest.empty()) { + derr << "Parse error! Can't specify more than one action. You " + << "specified both " << act << " and " << dest << "\n" << dendl; + usage(); + exit(1); + } + dest = act; +} + +static int parse_rank(const char *opt_name, const std::string &val) +{ + std::string err; + int ret = strict_strtol(val.c_str(), 10, &err); + if (!err.empty()) { + derr << "error parsing " << opt_name << ": failed to parse rank. " + << "It must be an int." << "\n" << dendl; + usage(); + } + return ret; +} + + + +MDS *mds = NULL; + + +static void handle_mds_signal(int signum) +{ + if (mds) + mds->handle_signal(signum); +} + +int main(int argc, const char **argv) +{ + vector args; + argv_to_vec(argc, argv, args); + env_to_vec(args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_MDS, CODE_ENVIRONMENT_DAEMON, 0); + + // mds specific args + int shadow = 0; + int rank = -1; + std::string dump_file; + + std::string val, action; + for (std::vector::iterator i = args.begin(); i != args.end(); ) { + if (ceph_argparse_double_dash(args, i)) { + break; + } + else if (ceph_argparse_witharg(args, i, &val, "--dump-journal", (char*)NULL)) { + set_special_action(action, "dump-journal"); + rank = parse_rank("dump-journal", val); + if (i == args.end()) { + derr << "error parsing --dump-journal: you must give a second " + << "dump-journal argument: the filename to dump the journal to. " + << "\n" << dendl; + usage(); + } + dump_file = *i++; + } + else if (ceph_argparse_witharg(args, i, &val, "--undump-journal", (char*)NULL)) { + set_special_action(action, "undump-journal"); + rank = parse_rank("undump-journal", val); + if (i == args.end()) { + derr << "error parsing --undump-journal: you must give a second " + << "undump-journal argument: the filename to undump the journal from. " + << "\n" << dendl; + usage(); + } + dump_file = *i++; + } + else if (ceph_argparse_witharg(args, i, &val, "--dump-journal-entries", (char*)NULL)){ + set_special_action(action, "dump-journal-entries"); + rank = parse_rank("dump-journal-entries", val); + } + else if (ceph_argparse_witharg(args, i, &val, "--reset-journal", (char*)NULL)) { + set_special_action(action, "reset-journal"); + rank = parse_rank("reset-journal", val); + } + else if (ceph_argparse_witharg(args, i, &val, "--journal-check", (char*)NULL)) { + int r = parse_rank("journal-check", val); + if (shadow) { + dout(0) << "Error: can only select one standby state" << dendl; + return -1; + } + dout(0) << "requesting oneshot_replay for mds." << r << dendl; + shadow = MDSMap::STATE_ONESHOT_REPLAY; + char rb[32]; + snprintf(rb, sizeof(rb), "%d", r); + g_conf->set_val("mds_standby_for_rank", rb); + g_conf->apply_changes(NULL); + } + else if (ceph_argparse_witharg(args, i, &val, "--hot-standby", (char*)NULL)) { + int r = parse_rank("hot-standby", val); + if (shadow) { + dout(0) << "Error: can only select one standby state" << dendl; + return -1; + } + dout(0) << "requesting standby_replay for mds." << r << dendl; + shadow = MDSMap::STATE_STANDBY_REPLAY; + char rb[32]; + snprintf(rb, sizeof(rb), "%d", r); + g_conf->set_val("mds_standby_for_rank", rb); + g_conf->apply_changes(NULL); + } + else { + derr << "Error: can't understand argument: " << *i << "\n" << dendl; + usage(); + } + } + + pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC); + + // Check for special actions + if (!action.empty()) { + return do_cmds_special_action(action, dump_file, rank); + } + + // Normal startup + if (g_conf->name.has_default_id()) { + derr << "must specify '-i name' with the ceph-mds instance name" << dendl; + usage(); + } + + Messenger *messenger = Messenger::create(g_ceph_context, + entity_name_t::MDS(-1), "mds", + getpid()); + messenger->set_cluster_protocol(CEPH_MDS_PROTOCOL); + + cout << "starting " << g_conf->name << " at " << messenger->get_myaddr() + << std::endl; + uint64_t supported = + CEPH_FEATURE_UID | + CEPH_FEATURE_NOSRCADDR | + CEPH_FEATURE_DIRLAYOUTHASH | + CEPH_FEATURE_MDS_INLINE_DATA | + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_MSG_AUTH | + CEPH_FEATURE_EXPORT_PEER; + uint64_t required = + CEPH_FEATURE_OSDREPLYMUX; + messenger->set_default_policy(Messenger::Policy::lossy_client(supported, required)); + messenger->set_policy(entity_name_t::TYPE_MON, + Messenger::Policy::lossy_client(supported, + CEPH_FEATURE_UID | + CEPH_FEATURE_PGID64)); + messenger->set_policy(entity_name_t::TYPE_MDS, + Messenger::Policy::lossless_peer(supported, + CEPH_FEATURE_UID)); + messenger->set_policy(entity_name_t::TYPE_CLIENT, + Messenger::Policy::stateful_server(supported, 0)); + + int r = messenger->bind(g_conf->public_addr); + if (r < 0) + exit(1); + + if (shadow != MDSMap::STATE_ONESHOT_REPLAY) + global_init_daemonize(g_ceph_context, 0); + common_init_finish(g_ceph_context); + + // get monmap + MonClient mc(g_ceph_context); + if (mc.build_initial_monmap() < 0) + return -1; + global_init_chdir(g_ceph_context); + + messenger->start(); + + // start mds + mds = new MDS(g_conf->name.get_id().c_str(), messenger, &mc); + + // in case we have to respawn... + mds->orig_argc = argc; + mds->orig_argv = argv; + + if (shadow) + r = mds->init(shadow); + else + r = mds->init(); + if (r < 0) + goto shutdown; + + // set up signal handlers, now that we've daemonized/forked. + init_async_signal_handler(); + register_async_signal_handler(SIGHUP, sighup_handler); + register_async_signal_handler_oneshot(SIGINT, handle_mds_signal); + register_async_signal_handler_oneshot(SIGTERM, handle_mds_signal); + + if (g_conf->inject_early_sigterm) + kill(getpid(), SIGTERM); + + messenger->wait(); + + unregister_async_signal_handler(SIGHUP, sighup_handler); + unregister_async_signal_handler(SIGINT, handle_mds_signal); + unregister_async_signal_handler(SIGTERM, handle_mds_signal); + shutdown_async_signal_handler(); + + shutdown: + // yuck: grab the mds lock, so we can be sure that whoever in *mds + // called shutdown finishes what they were doing. + mds->mds_lock.Lock(); + mds->mds_lock.Unlock(); + + pidfile_remove(); + + // only delete if it was a clean shutdown (to aid memory leak + // detection, etc.). don't bother if it was a suicide. + if (mds->is_stopped()) + delete mds; + + g_ceph_context->put(); + + // cd on exit, so that gmon.out (if any) goes into a separate directory for each node. + char s[20]; + snprintf(s, sizeof(s), "gmon/%d", getpid()); + if ((mkdir(s, 0755) == 0) && (chdir(s) == 0)) { + cerr << "ceph-mds: gmon.out should be in " << s << std::endl; + } + + return 0; +} + diff --git a/ceph/src/ceph_mon.cc b/ceph/src/ceph_mon.cc new file mode 100644 index 00000000..80b17a14 --- /dev/null +++ b/ceph/src/ceph_mon.cc @@ -0,0 +1,725 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include + +#include +#include +using namespace std; + +#include "common/config.h" +#include "include/ceph_features.h" + +#include "mon/MonMap.h" +#include "mon/Monitor.h" +#include "mon/MonitorDBStore.h" +#include "mon/MonClient.h" + +#include "msg/Messenger.h" + +#include "include/CompatSet.h" + +#include "common/ceph_argparse.h" +#include "common/pick_address.h" +#include "common/Timer.h" +#include "common/errno.h" +#include "common/Preforker.h" + +#include "global/global_init.h" +#include "global/signal_handler.h" + +#include "include/assert.h" + +#include "erasure-code/ErasureCodePlugin.h" + +#define dout_subsys ceph_subsys_mon + +Monitor *mon = NULL; + +void handle_mon_signal(int signum) +{ + if (mon) + mon->handle_signal(signum); +} + + +int obtain_monmap(MonitorDBStore &store, bufferlist &bl) +{ + dout(10) << __func__ << dendl; + /* + * the monmap may be in one of three places: + * 'monmap:' - the monmap we'd really like to have + * 'mon_sync:latest_monmap' - last monmap backed up for the last sync + * 'mkfs:monmap' - a monmap resulting from mkfs + */ + + if (store.exists("monmap", "last_committed")) { + version_t latest_ver = store.get("monmap", "last_committed"); + if (store.exists("monmap", latest_ver)) { + int err = store.get("monmap", latest_ver, bl); + assert(err == 0); + assert(bl.length() > 0); + dout(10) << __func__ << " read last committed monmap ver " + << latest_ver << dendl; + return 0; + } + } + + if (store.exists("mon_sync", "in_sync") + || store.exists("mon_sync", "force_sync")) { + dout(10) << __func__ << " detected aborted sync" << dendl; + if (store.exists("mon_sync", "latest_monmap")) { + int err = store.get("mon_sync", "latest_monmap", bl); + assert(err == 0); + assert(bl.length() > 0); + dout(10) << __func__ << " read backup monmap" << dendl; + return 0; + } + } + + if (store.exists("mkfs", "monmap")) { + dout(10) << __func__ << " found mkfs monmap" << dendl; + int err = store.get("mkfs", "monmap", bl); + assert(err == 0); + assert(bl.length() > 0); + return 0; + } + + derr << __func__ << " unable to find a monmap" << dendl; + return -ENOENT; +} + +int mon_data_exists(bool *r) +{ + string mon_data = g_conf->mon_data; + struct stat buf; + if (::stat(mon_data.c_str(), &buf)) { + if (errno == ENOENT) { + *r = false; + } else { + cerr << "stat(" << mon_data << ") " << cpp_strerror(errno) << std::endl; + return -errno; + } + } else { + *r = true; + } + return 0; +} + +int mon_data_empty(bool *r) +{ + string mon_data = g_conf->mon_data; + + DIR *dir = ::opendir(mon_data.c_str()); + if (!dir) { + cerr << "opendir(" << mon_data << ") " << cpp_strerror(errno) << std::endl; + return -errno; + } + char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1]; + + *r = false; + int code = 0; + struct dirent *de; + errno = 0; + while (!::readdir_r(dir, reinterpret_cast(buf), &de)) { + if (!de) { + if (errno) { + cerr << "readdir(" << mon_data << ") " << cpp_strerror(errno) << std::endl; + code = -errno; + } + break; + } + if (string(".") != de->d_name && + string("..") != de->d_name) { + *r = true; + break; + } + } + + ::closedir(dir); + + return code; +} + +int mon_exists(bool *r) +{ + int code = mon_data_exists(r); + if (code || *r == false) + return code; + return mon_data_empty(r); +} + +void usage() +{ + cerr << "usage: ceph-mon -i monid [flags]" << std::endl; + cerr << " --debug_mon n\n"; + cerr << " debug monitor level (e.g. 10)\n"; + cerr << " --mkfs\n"; + cerr << " build fresh monitor fs\n"; + cerr << " --force-sync\n"; + cerr << " force a sync from another mon by wiping local data (BE CAREFUL)\n"; + cerr << " --yes-i-really-mean-it\n"; + cerr << " mandatory safeguard for --force-sync\n"; + cerr << " --compact\n"; + cerr << " compact the monitor store\n"; + cerr << " --osdmap \n"; + cerr << " only used when --mkfs is provided: load the osdmap from \n"; + cerr << " --inject-monmap \n"; + cerr << " write the monmap to the local monitor store and exit\n"; + cerr << " --extract-monmap \n"; + cerr << " extract the monmap from the local monitor store and exit\n"; + cerr << " --mon-data \n"; + cerr << " where the mon store and keyring are located\n"; + generic_server_usage(); +} + +int preload_erasure_code() +{ + string directory = g_conf->osd_pool_default_erasure_code_directory; + string plugins = g_conf->osd_erasure_code_plugins; + stringstream ss; + int r = ErasureCodePluginRegistry::instance().preload(plugins, + directory, + ss); + if (r) + derr << ss.str() << dendl; + else + dout(10) << ss.str() << dendl; + return r; +} + +int main(int argc, const char **argv) +{ + int err; + + bool mkfs = false; + bool compact = false; + bool force_sync = false; + bool yes_really = false; + std::string osdmapfn, inject_monmap, extract_monmap; + + vector args; + argv_to_vec(argc, argv, args); + env_to_vec(args); + + int flags = 0; + { + vector args_copy = args; + std::string val; + for (std::vector::iterator i = args_copy.begin(); + i != args_copy.end(); ) { + if (ceph_argparse_double_dash(args_copy, i)) { + break; + } else if (ceph_argparse_flag(args_copy, i, "--mkfs", (char*)NULL)) { + flags |= CINIT_FLAG_NO_DAEMON_ACTIONS; + } else if (ceph_argparse_witharg(args_copy, i, &val, "--inject_monmap", (char*)NULL)) { + flags |= CINIT_FLAG_NO_DAEMON_ACTIONS; + } else if (ceph_argparse_witharg(args_copy, i, &val, "--extract-monmap", (char*)NULL)) { + flags |= CINIT_FLAG_NO_DAEMON_ACTIONS; + } else { + ++i; + } + } + } + + global_init(NULL, args, CEPH_ENTITY_TYPE_MON, CODE_ENVIRONMENT_DAEMON, flags); + + uuid_d fsid; + std::string val; + for (std::vector::iterator i = args.begin(); i != args.end(); ) { + if (ceph_argparse_double_dash(args, i)) { + break; + } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { + usage(); + exit(0); + } else if (ceph_argparse_flag(args, i, "--mkfs", (char*)NULL)) { + mkfs = true; + } else if (ceph_argparse_flag(args, i, "--compact", (char*)NULL)) { + compact = true; + } else if (ceph_argparse_flag(args, i, "--force-sync", (char*)NULL)) { + force_sync = true; + } else if (ceph_argparse_flag(args, i, "--yes-i-really-mean-it", (char*)NULL)) { + yes_really = true; + } else if (ceph_argparse_witharg(args, i, &val, "--osdmap", (char*)NULL)) { + osdmapfn = val; + } else if (ceph_argparse_witharg(args, i, &val, "--inject_monmap", (char*)NULL)) { + inject_monmap = val; + } else if (ceph_argparse_witharg(args, i, &val, "--extract-monmap", (char*)NULL)) { + extract_monmap = val; + } else { + ++i; + } + } + if (!args.empty()) { + cerr << "too many arguments: " << args << std::endl; + usage(); + } + + if (force_sync && !yes_really) { + cerr << "are you SURE you want to force a sync? this will erase local data and may\n" + << "break your mon cluster. pass --yes-i-really-mean-it if you do." << std::endl; + exit(1); + } + + if (g_conf->mon_data.empty()) { + cerr << "must specify '--mon-data=foo' data path" << std::endl; + usage(); + } + + if (g_conf->name.get_id().empty()) { + cerr << "must specify id (--id or --name mon.)" << std::endl; + usage(); + } + + bool exists; + if (mon_exists(&exists)) + exit(1); + + if (mkfs && exists) { + cerr << g_conf->mon_data << " already exists" << std::endl; + exit(0); + } + + // -- mkfs -- + if (mkfs) { + + if (mon_data_exists(&exists)) + exit(1); + + if (!exists) { + if (::mkdir(g_conf->mon_data.c_str(), 0755)) { + cerr << "mkdir(" << g_conf->mon_data << ") : " + << cpp_strerror(errno) << std::endl; + exit(1); + } + } + + // resolve public_network -> public_addr + pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC); + + common_init_finish(g_ceph_context, flags); + + bufferlist monmapbl, osdmapbl; + std::string error; + MonMap monmap; + + // load or generate monmap + if (g_conf->monmap.length()) { + int err = monmapbl.read_file(g_conf->monmap.c_str(), &error); + if (err < 0) { + cerr << argv[0] << ": error reading " << g_conf->monmap << ": " << error << std::endl; + exit(1); + } + try { + monmap.decode(monmapbl); + + // always mark seed/mkfs monmap as epoch 0 + monmap.set_epoch(0); + } + catch (const buffer::error& e) { + cerr << argv[0] << ": error decoding monmap " << g_conf->monmap << ": " << e.what() << std::endl; + exit(1); + } + } else { + int err = monmap.build_initial(g_ceph_context, cerr); + if (err < 0) { + cerr << argv[0] << ": warning: no initial monitors; must use admin socket to feed hints" << std::endl; + } + + // am i part of the initial quorum? + if (monmap.contains(g_conf->name.get_id())) { + // hmm, make sure the ip listed exists on the current host? + // maybe later. + } else if (!g_conf->public_addr.is_blank_ip()) { + entity_addr_t a = g_conf->public_addr; + if (a.get_port() == 0) + a.set_port(CEPH_MON_PORT); + if (monmap.contains(a)) { + string name; + monmap.get_addr_name(a, name); + monmap.rename(name, g_conf->name.get_id()); + cout << argv[0] << ": renaming mon." << name << " " << a + << " to mon." << g_conf->name.get_id() << std::endl; + } + } else { + // is a local address listed without a name? if so, name myself. + list ls; + monmap.list_addrs(ls); + entity_addr_t local; + + if (have_local_addr(g_ceph_context, ls, &local)) { + string name; + monmap.get_addr_name(local, name); + + if (name.find("noname-") == 0) { + cout << argv[0] << ": mon." << name << " " << local + << " is local, renaming to mon." << g_conf->name.get_id() << std::endl; + monmap.rename(name, g_conf->name.get_id()); + } else { + cout << argv[0] << ": mon." << name << " " << local + << " is local, but not 'noname-' + something; not assuming it's me" << std::endl; + } + } + } + } + + if (!g_conf->fsid.is_zero()) { + monmap.fsid = g_conf->fsid; + cout << argv[0] << ": set fsid to " << g_conf->fsid << std::endl; + } + + if (monmap.fsid.is_zero()) { + cerr << argv[0] << ": generated monmap has no fsid; use '--fsid '" << std::endl; + exit(10); + } + + //monmap.print(cout); + + // osdmap + if (osdmapfn.length()) { + err = osdmapbl.read_file(osdmapfn.c_str(), &error); + if (err < 0) { + cerr << argv[0] << ": error reading " << osdmapfn << ": " + << error << std::endl; + exit(1); + } + } + + // go + MonitorDBStore store(g_conf->mon_data); + int r = store.create_and_open(cerr); + if (r < 0) { + cerr << argv[0] << ": error opening mon data directory at '" + << g_conf->mon_data << "': " << cpp_strerror(r) << std::endl; + exit(1); + } + assert(r == 0); + + Monitor mon(g_ceph_context, g_conf->name.get_id(), &store, 0, &monmap); + r = mon.mkfs(osdmapbl); + if (r < 0) { + cerr << argv[0] << ": error creating monfs: " << cpp_strerror(r) << std::endl; + exit(1); + } + cout << argv[0] << ": created monfs at " << g_conf->mon_data + << " for " << g_conf->name << std::endl; + return 0; + } + + // we fork early to prevent leveldb's environment static state from + // screwing us over + Preforker prefork; + if (!(flags & CINIT_FLAG_NO_DAEMON_ACTIONS)) { + if (global_init_prefork(g_ceph_context, 0) >= 0) { + prefork.prefork(); + if (prefork.is_parent()) { + return prefork.parent_wait(); + } + global_init_postfork_start(g_ceph_context); + } + common_init_finish(g_ceph_context); + global_init_chdir(g_ceph_context); + if (preload_erasure_code() < -1) + prefork.exit(1); + } + + MonitorDBStore *store = new MonitorDBStore(g_conf->mon_data); + + Monitor::StoreConverter converter(g_conf->mon_data, store); + if (store->open(std::cerr) < 0) { + int ret = store->create_and_open(std::cerr); + if (ret < 0) { + derr << "failed to create new leveldb store" << dendl; + prefork.exit(1); + } + + ret = converter.needs_conversion(); + if (ret < 0) { + derr << "found errors while validating legacy unconverted monitor store: " + << cpp_strerror(ret) << dendl; + prefork.exit(1); + } + if (ret > 0) { + dout(0) << "converting monitor store, please do not interrupt..." << dendl; + int r = converter.convert(); + if (r) { + derr << "failed to convert monitor store: " << cpp_strerror(r) << dendl; + prefork.exit(1); + } + } + } else if (converter.is_converting()) { + derr << "there is an on-going (maybe aborted?) conversion." << dendl; + derr << "you should check what happened" << dendl; + derr << "remove store.db to restart conversion" << dendl; + prefork.exit(1); + } + + bufferlist magicbl; + err = store->get(Monitor::MONITOR_NAME, "magic", magicbl); + if (!magicbl.length()) { + derr << "unable to read magic from mon data.. did you run mkcephfs?" << dendl; + prefork.exit(1); + } + string magic(magicbl.c_str(), magicbl.length()-1); // ignore trailing \n + if (strcmp(magic.c_str(), CEPH_MON_ONDISK_MAGIC)) { + derr << "mon fs magic '" << magic << "' != current '" << CEPH_MON_ONDISK_MAGIC << "'" << dendl; + prefork.exit(1); + } + + err = Monitor::check_features(store); + if (err < 0) { + derr << "error checking features: " << cpp_strerror(err) << dendl; + prefork.exit(1); + } + + // inject new monmap? + if (!inject_monmap.empty()) { + bufferlist bl; + std::string error; + int r = bl.read_file(inject_monmap.c_str(), &error); + if (r) { + derr << "unable to read monmap from " << inject_monmap << ": " + << error << dendl; + prefork.exit(1); + } + + // get next version + version_t v = store->get("monmap", "last_committed"); + dout(0) << "last committed monmap epoch is " << v << ", injected map will be " << (v+1) + << dendl; + v++; + + // set the version + MonMap tmp; + tmp.decode(bl); + if (tmp.get_epoch() != v) { + dout(0) << "changing monmap epoch from " << tmp.get_epoch() + << " to " << v << dendl; + tmp.set_epoch(v); + } + bufferlist mapbl; + tmp.encode(mapbl, CEPH_FEATURES_ALL); + bufferlist final; + ::encode(v, final); + ::encode(mapbl, final); + + MonitorDBStore::Transaction t; + // save it + t.put("monmap", v, mapbl); + t.put("monmap", "latest", final); + t.put("monmap", "last_committed", v); + store->apply_transaction(t); + + dout(0) << "done." << dendl; + prefork.exit(0); + } + + // monmap? + MonMap monmap; + { + // note that even if we don't find a viable monmap, we should go ahead + // and try to build it up in the next if-else block. + bufferlist mapbl; + int err = obtain_monmap(*store, mapbl); + if (err >= 0) { + try { + monmap.decode(mapbl); + } catch (const buffer::error& e) { + cerr << "can't decode monmap: " << e.what() << std::endl; + } + } else { + derr << "unable to obtain a monmap: " << cpp_strerror(err) << dendl; + } + if (!extract_monmap.empty()) { + int r = mapbl.write_file(extract_monmap.c_str()); + if (r < 0) { + r = -errno; + derr << "error writing monmap to " << extract_monmap << ": " << cpp_strerror(r) << dendl; + prefork.exit(1); + } + derr << "wrote monmap to " << extract_monmap << dendl; + prefork.exit(0); + } + } + + // this is what i will bind to + entity_addr_t ipaddr; + + if (monmap.contains(g_conf->name.get_id())) { + ipaddr = monmap.get_addr(g_conf->name.get_id()); + + // print helpful warning if the conf file doesn't match + entity_addr_t conf_addr; + std::vector my_sections; + g_conf->get_my_sections(my_sections); + std::string mon_addr_str; + if (g_conf->get_val_from_conf_file(my_sections, "mon addr", + mon_addr_str, true) == 0) { + if (conf_addr.parse(mon_addr_str.c_str()) && (ipaddr != conf_addr)) { + derr << "WARNING: 'mon addr' config option " << conf_addr + << " does not match monmap file" << std::endl + << " continuing with monmap configuration" << dendl; + } + } + } else { + dout(0) << g_conf->name << " does not exist in monmap, will attempt to join an existing cluster" << dendl; + + pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC); + if (!g_conf->public_addr.is_blank_ip()) { + ipaddr = g_conf->public_addr; + if (ipaddr.get_port() == 0) + ipaddr.set_port(CEPH_MON_PORT); + dout(0) << "using public_addr " << g_conf->public_addr << " -> " + << ipaddr << dendl; + } else { + MonMap tmpmap; + int err = tmpmap.build_initial(g_ceph_context, cerr); + if (err < 0) { + derr << argv[0] << ": error generating initial monmap: " + << cpp_strerror(err) << dendl; + usage(); + prefork.exit(1); + } + if (tmpmap.contains(g_conf->name.get_id())) { + ipaddr = tmpmap.get_addr(g_conf->name.get_id()); + } else { + derr << "no public_addr or public_network specified, and " << g_conf->name + << " not present in monmap or ceph.conf" << dendl; + prefork.exit(1); + } + } + } + + // bind + int rank = monmap.get_rank(g_conf->name.get_id()); + Messenger *messenger = Messenger::create(g_ceph_context, + entity_name_t::MON(rank), + "mon", + 0); + messenger->set_cluster_protocol(CEPH_MON_PROTOCOL); + messenger->set_default_send_priority(CEPH_MSG_PRIO_HIGH); + + uint64_t supported = + CEPH_FEATURE_UID | + CEPH_FEATURE_NOSRCADDR | + CEPH_FEATURE_MONCLOCKCHECK | + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_MSG_AUTH; + messenger->set_default_policy(Messenger::Policy::stateless_server(supported, 0)); + messenger->set_policy(entity_name_t::TYPE_MON, + Messenger::Policy::lossless_peer_reuse(supported, + CEPH_FEATURE_UID | + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_MON_SINGLE_PAXOS)); + messenger->set_policy(entity_name_t::TYPE_OSD, + Messenger::Policy::stateless_server(supported, + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_OSDENC)); + messenger->set_policy(entity_name_t::TYPE_CLIENT, + Messenger::Policy::stateless_server(supported, 0)); + messenger->set_policy(entity_name_t::TYPE_MDS, + Messenger::Policy::stateless_server(supported, 0)); + + + // throttle client traffic + Throttle *client_throttler = new Throttle(g_ceph_context, "mon_client_bytes", + g_conf->mon_client_bytes); + messenger->set_policy_throttlers(entity_name_t::TYPE_CLIENT, client_throttler, NULL); + + // throttle daemon traffic + // NOTE: actual usage on the leader may multiply by the number of + // monitors if they forward large update messages from daemons. + Throttle *daemon_throttler = new Throttle(g_ceph_context, "mon_daemon_bytes", + g_conf->mon_daemon_bytes); + messenger->set_policy_throttlers(entity_name_t::TYPE_OSD, daemon_throttler, NULL); + messenger->set_policy_throttlers(entity_name_t::TYPE_MDS, daemon_throttler, NULL); + + dout(0) << "starting " << g_conf->name << " rank " << rank + << " at " << ipaddr + << " mon_data " << g_conf->mon_data + << " fsid " << monmap.get_fsid() + << dendl; + + err = messenger->bind(ipaddr); + if (err < 0) { + derr << "unable to bind monitor to " << ipaddr << dendl; + prefork.exit(1); + } + + // start monitor + mon = new Monitor(g_ceph_context, g_conf->name.get_id(), store, + messenger, &monmap); + + if (force_sync) { + derr << "flagging a forced sync ..." << dendl; + mon->sync_force(NULL, cerr); + } + + err = mon->preinit(); + if (err < 0) { + derr << "failed to initialize" << dendl; + prefork.exit(1); + } + + if (compact || g_conf->mon_compact_on_start) { + derr << "compacting monitor store ..." << dendl; + mon->store->compact(); + derr << "done compacting" << dendl; + } + + if (g_conf->daemonize) { + global_init_postfork_finish(g_ceph_context, 0); + prefork.daemonize(); + } + + messenger->start(); + + mon->init(); + + // set up signal handlers, now that we've daemonized/forked. + init_async_signal_handler(); + register_async_signal_handler(SIGHUP, sighup_handler); + register_async_signal_handler_oneshot(SIGINT, handle_mon_signal); + register_async_signal_handler_oneshot(SIGTERM, handle_mon_signal); + + if (g_conf->inject_early_sigterm) + kill(getpid(), SIGTERM); + + messenger->wait(); + + unregister_async_signal_handler(SIGHUP, sighup_handler); + unregister_async_signal_handler(SIGINT, handle_mon_signal); + unregister_async_signal_handler(SIGTERM, handle_mon_signal); + shutdown_async_signal_handler(); + + delete mon; + delete store; + delete messenger; + delete client_throttler; + delete daemon_throttler; + g_ceph_context->put(); + + // cd on exit, so that gmon.out (if any) goes into a separate directory for each node. + char s[20]; + snprintf(s, sizeof(s), "gmon/%d", getpid()); + if ((mkdir(s, 0755) == 0) && (chdir(s) == 0)) { + dout(0) << "ceph-mon: gmon.out should be in " << s << dendl; + } + + prefork.signal_exit(0); + return 0; +} + diff --git a/ceph/src/ceph_osd.cc b/ceph/src/ceph_osd.cc new file mode 100644 index 00000000..a2f45420 --- /dev/null +++ b/ceph/src/ceph_osd.cc @@ -0,0 +1,554 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +#include +#include + +#include +#include +using namespace std; + +#include "osd/OSD.h" +#include "os/ObjectStore.h" +#include "mon/MonClient.h" +#include "include/ceph_features.h" + +#include "common/config.h" + +#include "mon/MonMap.h" + + +#include "msg/Messenger.h" + +#include "common/Timer.h" +#include "common/ceph_argparse.h" + +#include "global/global_init.h" +#include "global/signal_handler.h" + +#include "include/color.h" +#include "common/errno.h" +#include "common/pick_address.h" + +#include "perfglue/heap_profiler.h" + +#include "include/assert.h" + +#include "erasure-code/ErasureCodePlugin.h" + +#define dout_subsys ceph_subsys_osd + +OSD *osd = NULL; + +void handle_osd_signal(int signum) +{ + if (osd) + osd->handle_signal(signum); +} + +void usage() +{ + derr << "usage: ceph-osd -i osdid [--osd-data=path] [--osd-journal=path] " + << "[--mkfs] [--mkjournal] [--convert-filestore]" << dendl; + derr << " --debug_osd N set debug level (e.g. 10)" << dendl; + generic_server_usage(); +} + +int preload_erasure_code() +{ + string directory = g_conf->osd_pool_default_erasure_code_directory; + string plugins = g_conf->osd_erasure_code_plugins; + stringstream ss; + int r = ErasureCodePluginRegistry::instance().preload(plugins, + directory, + ss); + if (r) + derr << ss.str() << dendl; + else + dout(10) << ss.str() << dendl; + return r; +} + +int main(int argc, const char **argv) +{ + vector args; + argv_to_vec(argc, argv, args); + env_to_vec(args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_OSD, CODE_ENVIRONMENT_DAEMON, 0); + ceph_heap_profiler_init(); + + // osd specific args + bool mkfs = false; + bool mkjournal = false; + bool mkkey = false; + bool flushjournal = false; + bool dump_journal = false; + bool convertfilestore = false; + bool get_journal_fsid = false; + bool get_osd_fsid = false; + bool get_cluster_fsid = false; + std::string dump_pg_log; + + std::string val; + for (std::vector::iterator i = args.begin(); i != args.end(); ) { + if (ceph_argparse_double_dash(args, i)) { + break; + } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { + usage(); + exit(0); + } else if (ceph_argparse_flag(args, i, "--mkfs", (char*)NULL)) { + mkfs = true; + } else if (ceph_argparse_flag(args, i, "--mkjournal", (char*)NULL)) { + mkjournal = true; + } else if (ceph_argparse_flag(args, i, "--mkkey", (char*)NULL)) { + mkkey = true; + } else if (ceph_argparse_flag(args, i, "--flush-journal", (char*)NULL)) { + flushjournal = true; + } else if (ceph_argparse_flag(args, i, "--convert-filestore", (char*)NULL)) { + convertfilestore = true; + } else if (ceph_argparse_witharg(args, i, &val, "--dump-pg-log", (char*)NULL)) { + dump_pg_log = val; + } else if (ceph_argparse_flag(args, i, "--dump-journal", (char*)NULL)) { + dump_journal = true; + } else if (ceph_argparse_flag(args, i, "--get-cluster-fsid", (char*)NULL)) { + get_cluster_fsid = true; + } else if (ceph_argparse_flag(args, i, "--get-osd-fsid", "--get-osd-uuid", (char*)NULL)) { + get_osd_fsid = true; + } else if (ceph_argparse_flag(args, i, "--get-journal-fsid", "--get-journal-uuid", (char*)NULL)) { + get_journal_fsid = true; + } else { + ++i; + } + } + if (!args.empty()) { + derr << "unrecognized arg " << args[0] << dendl; + usage(); + } + + if (!dump_pg_log.empty()) { + common_init_finish(g_ceph_context); + bufferlist bl; + std::string error; + int r = bl.read_file(dump_pg_log.c_str(), &error); + if (r >= 0) { + pg_log_entry_t e; + bufferlist::iterator p = bl.begin(); + while (!p.end()) { + uint64_t pos = p.get_off(); + try { + ::decode(e, p); + } + catch (const buffer::error &e) { + derr << "failed to decode LogEntry at offset " << pos << dendl; + return 1; + } + derr << pos << ":\t" << e << dendl; + } + } else { + derr << "unable to open " << dump_pg_log << ": " << error << dendl; + } + return 0; + } + + // whoami + char *end; + const char *id = g_conf->name.get_id().c_str(); + int whoami = strtol(id, &end, 10); + if (*end || end == id || whoami < 0) { + derr << "must specify '-i #' where # is the osd number" << dendl; + usage(); + } + + if (g_conf->osd_data.empty()) { + derr << "must specify '--osd-data=foo' data path" << dendl; + usage(); + } + + // the store + ObjectStore *store = ObjectStore::create(g_ceph_context, + g_conf->osd_objectstore, + g_conf->osd_data, + g_conf->osd_journal); + if (!store) { + derr << "unable to create object store" << dendl; + return -ENODEV; + } + + if (mkfs) { + common_init_finish(g_ceph_context); + MonClient mc(g_ceph_context); + if (mc.build_initial_monmap() < 0) + return -1; + if (mc.get_monmap_privately() < 0) + return -1; + + int err = OSD::mkfs(g_ceph_context, store, g_conf->osd_data, + mc.monmap.fsid, whoami); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: error creating empty object store in " + << g_conf->osd_data << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; + exit(1); + } + derr << "created object store " << g_conf->osd_data; + if (!g_conf->osd_journal.empty()) + *_dout << " journal " << g_conf->osd_journal; + *_dout << " for osd." << whoami << " fsid " << mc.monmap.fsid << dendl; + } + if (mkkey) { + common_init_finish(g_ceph_context); + KeyRing *keyring = KeyRing::create_empty(); + if (!keyring) { + derr << "Unable to get a Ceph keyring." << dendl; + return 1; + } + + EntityName ename(g_conf->name); + EntityAuth eauth; + + int ret = keyring->load(g_ceph_context, g_conf->keyring); + if (ret == 0 && + keyring->get_auth(ename, eauth)) { + derr << "already have key in keyring " << g_conf->keyring << dendl; + } else { + eauth.key.create(g_ceph_context, CEPH_CRYPTO_AES); + keyring->add(ename, eauth); + bufferlist bl; + keyring->encode_plaintext(bl); + int r = bl.write_file(g_conf->keyring.c_str(), 0600); + if (r) + derr << TEXT_RED << " ** ERROR: writing new keyring to " << g_conf->keyring + << ": " << cpp_strerror(r) << TEXT_NORMAL << dendl; + else + derr << "created new key in keyring " << g_conf->keyring << dendl; + } + } + if (mkfs || mkkey) + exit(0); + if (mkjournal) { + common_init_finish(g_ceph_context); + int err = store->mkjournal(); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: error creating fresh journal " << g_conf->osd_journal + << " for object store " << g_conf->osd_data + << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; + exit(1); + } + derr << "created new journal " << g_conf->osd_journal + << " for object store " << g_conf->osd_data << dendl; + exit(0); + } + if (flushjournal) { + common_init_finish(g_ceph_context); + int err = store->mount(); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: error flushing journal " << g_conf->osd_journal + << " for object store " << g_conf->osd_data + << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; + exit(1); + } + store->sync_and_flush(); + store->umount(); + derr << "flushed journal " << g_conf->osd_journal + << " for object store " << g_conf->osd_data + << dendl; + exit(0); + } + if (dump_journal) { + common_init_finish(g_ceph_context); + int err = store->dump_journal(cout); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: error dumping journal " << g_conf->osd_journal + << " for object store " << g_conf->osd_data + << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; + exit(1); + } + derr << "dumped journal " << g_conf->osd_journal + << " for object store " << g_conf->osd_data + << dendl; + exit(0); + + } + + + if (convertfilestore) { + int err = OSD::do_convertfs(store); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data + << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; + exit(1); + } + exit(0); + } + + if (get_journal_fsid) { + uuid_d fsid; + int r = store->peek_journal_fsid(&fsid); + if (r == 0) + cout << fsid << std::endl; + exit(r); + } + + string magic; + uuid_d cluster_fsid, osd_fsid; + int w; + int r = OSD::peek_meta(store, magic, cluster_fsid, osd_fsid, w); + if (r < 0) { + derr << TEXT_RED << " ** ERROR: unable to open OSD superblock on " + << g_conf->osd_data << ": " << cpp_strerror(-r) + << TEXT_NORMAL << dendl; + if (r == -ENOTSUP) { + derr << TEXT_RED << " ** please verify that underlying storage " + << "supports xattrs" << TEXT_NORMAL << dendl; + } + exit(1); + } + if (w != whoami) { + derr << "OSD id " << w << " != my id " << whoami << dendl; + exit(1); + } + if (strcmp(magic.c_str(), CEPH_OSD_ONDISK_MAGIC)) { + derr << "OSD magic " << magic << " != my " << CEPH_OSD_ONDISK_MAGIC + << dendl; + exit(1); + } + + if (get_cluster_fsid) { + cout << cluster_fsid << std::endl; + exit(0); + } + if (get_osd_fsid) { + cout << osd_fsid << std::endl; + exit(0); + } + + pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC + |CEPH_PICK_ADDRESS_CLUSTER); + + if (g_conf->public_addr.is_blank_ip() && !g_conf->cluster_addr.is_blank_ip()) { + derr << TEXT_YELLOW + << " ** WARNING: specified cluster addr but not public addr; we recommend **\n" + << " ** you specify neither or both. **" + << TEXT_NORMAL << dendl; + } + + Messenger *ms_public = Messenger::create(g_ceph_context, + entity_name_t::OSD(whoami), "client", + getpid()); + Messenger *ms_cluster = Messenger::create(g_ceph_context, + entity_name_t::OSD(whoami), "cluster", + getpid()); + Messenger *ms_hbclient = Messenger::create(g_ceph_context, + entity_name_t::OSD(whoami), "hbclient", + getpid()); + Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, + entity_name_t::OSD(whoami), "hb_back_server", + getpid()); + Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, + entity_name_t::OSD(whoami), "hb_front_server", + getpid()); + Messenger *ms_objecter = Messenger::create(g_ceph_context, + entity_name_t::OSD(whoami), "ms_objecter", + getpid()); + ms_cluster->set_cluster_protocol(CEPH_OSD_PROTOCOL); + ms_hbclient->set_cluster_protocol(CEPH_OSD_PROTOCOL); + ms_hb_back_server->set_cluster_protocol(CEPH_OSD_PROTOCOL); + ms_hb_front_server->set_cluster_protocol(CEPH_OSD_PROTOCOL); + + cout << "starting osd." << whoami + << " at " << ms_public->get_myaddr() + << " osd_data " << g_conf->osd_data + << " " << ((g_conf->osd_journal.empty()) ? + "(no journal)" : g_conf->osd_journal) + << std::endl; + + boost::scoped_ptr client_byte_throttler( + new Throttle(g_ceph_context, "osd_client_bytes", + g_conf->osd_client_message_size_cap)); + boost::scoped_ptr client_msg_throttler( + new Throttle(g_ceph_context, "osd_client_messages", + g_conf->osd_client_message_cap)); + + uint64_t supported = + CEPH_FEATURE_UID | + CEPH_FEATURE_NOSRCADDR | + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_MSG_AUTH | + CEPH_FEATURE_OSD_ERASURE_CODES; + + ms_public->set_default_policy(Messenger::Policy::stateless_server(supported, 0)); + ms_public->set_policy_throttlers(entity_name_t::TYPE_CLIENT, + client_byte_throttler.get(), + client_msg_throttler.get()); + ms_public->set_policy(entity_name_t::TYPE_MON, + Messenger::Policy::lossy_client(supported, + CEPH_FEATURE_UID | + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_OSDENC)); + //try to poison pill any OSD connections on the wrong address + ms_public->set_policy(entity_name_t::TYPE_OSD, + Messenger::Policy::stateless_server(0,0)); + + ms_cluster->set_default_policy(Messenger::Policy::stateless_server(0, 0)); + ms_cluster->set_policy(entity_name_t::TYPE_MON, Messenger::Policy::lossy_client(0,0)); + ms_cluster->set_policy(entity_name_t::TYPE_OSD, + Messenger::Policy::lossless_peer(supported, + CEPH_FEATURE_UID | + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_OSDENC)); + ms_cluster->set_policy(entity_name_t::TYPE_CLIENT, + Messenger::Policy::stateless_server(0, 0)); + + ms_hbclient->set_policy(entity_name_t::TYPE_OSD, + Messenger::Policy::lossy_client(0, 0)); + ms_hb_back_server->set_policy(entity_name_t::TYPE_OSD, + Messenger::Policy::stateless_server(0, 0)); + ms_hb_front_server->set_policy(entity_name_t::TYPE_OSD, + Messenger::Policy::stateless_server(0, 0)); + + ms_objecter->set_default_policy(Messenger::Policy::lossy_client(0, CEPH_FEATURE_OSDREPLYMUX)); + + r = ms_public->bind(g_conf->public_addr); + if (r < 0) + exit(1); + r = ms_cluster->bind(g_conf->cluster_addr); + if (r < 0) + exit(1); + + // hb back should bind to same ip as cluster_addr (if specified) + entity_addr_t hb_back_addr = g_conf->osd_heartbeat_addr; + if (hb_back_addr.is_blank_ip()) { + hb_back_addr = g_conf->cluster_addr; + if (hb_back_addr.is_ip()) + hb_back_addr.set_port(0); + } + r = ms_hb_back_server->bind(hb_back_addr); + if (r < 0) + exit(1); + + // hb front should bind to same ip as public_addr + entity_addr_t hb_front_addr = g_conf->public_addr; + if (hb_front_addr.is_ip()) + hb_front_addr.set_port(0); + r = ms_hb_front_server->bind(hb_front_addr); + if (r < 0) + exit(1); + + ms_objecter->bind(g_conf->public_addr); + + // Set up crypto, daemonize, etc. + global_init_daemonize(g_ceph_context, 0); + common_init_finish(g_ceph_context); + + if (g_conf->filestore_update_to >= (int)store->get_target_version()) { + int err = OSD::do_convertfs(store); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data + << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; + exit(1); + } + } + + MonClient mc(g_ceph_context); + if (mc.build_initial_monmap() < 0) + return -1; + global_init_chdir(g_ceph_context); + + if (preload_erasure_code() < -1) + return -1; + + osd = new OSD(g_ceph_context, + store, + whoami, + ms_cluster, + ms_public, + ms_hbclient, + ms_hb_front_server, + ms_hb_back_server, + ms_objecter, + &mc, + g_conf->osd_data, g_conf->osd_journal); + + int err = osd->pre_init(); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: osd pre_init failed: " << cpp_strerror(-err) + << TEXT_NORMAL << dendl; + return 1; + } + + // Now close the standard file descriptors + global_init_shutdown_stderr(g_ceph_context); + + ms_public->start(); + ms_hbclient->start(); + ms_hb_front_server->start(); + ms_hb_back_server->start(); + ms_cluster->start(); + ms_objecter->start(); + + // start osd + err = osd->init(); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: osd init failed: " << cpp_strerror(-err) + << TEXT_NORMAL << dendl; + return 1; + } + + // install signal handlers + init_async_signal_handler(); + register_async_signal_handler(SIGHUP, sighup_handler); + register_async_signal_handler_oneshot(SIGINT, handle_osd_signal); + register_async_signal_handler_oneshot(SIGTERM, handle_osd_signal); + + osd->final_init(); + + if (g_conf->inject_early_sigterm) + kill(getpid(), SIGTERM); + + ms_public->wait(); + ms_hbclient->wait(); + ms_hb_front_server->wait(); + ms_hb_back_server->wait(); + ms_cluster->wait(); + ms_objecter->wait(); + + unregister_async_signal_handler(SIGHUP, sighup_handler); + unregister_async_signal_handler(SIGINT, handle_osd_signal); + unregister_async_signal_handler(SIGTERM, handle_osd_signal); + shutdown_async_signal_handler(); + + // done + delete osd; + delete ms_public; + delete ms_hbclient; + delete ms_hb_front_server; + delete ms_hb_back_server; + delete ms_cluster; + delete ms_objecter; + client_byte_throttler.reset(); + client_msg_throttler.reset(); + g_ceph_context->put(); + + // cd on exit, so that gmon.out (if any) goes into a separate directory for each node. + char s[20]; + snprintf(s, sizeof(s), "gmon/%d", getpid()); + if ((mkdir(s, 0755) == 0) && (chdir(s) == 0)) { + dout(0) << "ceph-osd: gmon.out should be in " << s << dendl; + } + + return 0; +} diff --git a/ceph/src/ceph_syn.cc b/ceph/src/ceph_syn.cc new file mode 100644 index 00000000..c3410aa6 --- /dev/null +++ b/ceph/src/ceph_syn.cc @@ -0,0 +1,107 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +using namespace std; + +#include "common/config.h" + +#include "client/SyntheticClient.h" +#include "client/Client.h" + +#include "msg/Messenger.h" + +#include "mon/MonClient.h" + +#include "common/Timer.h" +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "common/pick_address.h" + +#if !defined(DARWIN) && !defined(__FreeBSD__) +#include +#endif // DARWIN || __FreeBSD__ + +#include +#include + +extern int syn_filer_flags; + +int main(int argc, const char **argv, char *envp[]) +{ + //cerr << "ceph-syn starting" << std::endl; + vector args; + argv_to_vec(argc, argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + parse_syn_options(args); // for SyntheticClient + + pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC); + + // get monmap + MonClient mc(g_ceph_context); + if (mc.build_initial_monmap() < 0) + return -1; + + list clients; + list synclients; + Messenger* messengers[g_conf->num_client]; + MonClient* mclients[g_conf->num_client]; + + cout << "ceph-syn: starting " << g_conf->num_client << " syn client(s)" << std::endl; + for (int i=0; inum_client; i++) { + messengers[i] = Messenger::create(g_ceph_context, + entity_name_t(entity_name_t::TYPE_CLIENT,-1), "synclient", + i * 1000000 + getpid()); + messengers[i]->bind(g_conf->public_addr); + mclients[i] = new MonClient(g_ceph_context); + mclients[i]->build_initial_monmap(); + Client *client = new Client(messengers[i], mclients[i]); + client->set_filer_flags(syn_filer_flags); + SyntheticClient *syn = new SyntheticClient(client); + clients.push_back(client); + synclients.push_back(syn); + messengers[i]->start(); + } + + for (list::iterator p = synclients.begin(); + p != synclients.end(); + ++p) + (*p)->start_thread(); + + //cout << "waiting for client(s) to finish" << std::endl; + while (!clients.empty()) { + Client *client = clients.front(); + SyntheticClient *syn = synclients.front(); + clients.pop_front(); + synclients.pop_front(); + syn->join_thread(); + delete syn; + delete client; + } + + for (int i = 0; i < g_conf->num_client; ++i) { + // wait for messenger to finish + delete mclients[i]; + messengers[i]->shutdown(); + messengers[i]->wait(); + delete messengers[i]; + } + return 0; +} + diff --git a/ceph/src/ceph_ver.c b/ceph/src/ceph_ver.c new file mode 100644 index 00000000..b983b552 --- /dev/null +++ b/ceph/src/ceph_ver.c @@ -0,0 +1,12 @@ + +#include "acconfig.h" +#include "ceph_ver.h" + +#define CONCAT_VER_SYMBOL(x) ceph_ver__##x + +#define DEFINE_VER_SYMBOL(x) int CONCAT_VER_SYMBOL(x) + +DEFINE_VER_SYMBOL(CEPH_GIT_VER); + + + diff --git a/ceph/src/cephfs.cc b/ceph/src/cephfs.cc new file mode 100644 index 00000000..90aee32c --- /dev/null +++ b/ceph/src/cephfs.cc @@ -0,0 +1,292 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2010 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + * Simple little program to let you: + * 1) View the layout information on a file or directory, + * 2) Modify the layout information on an empty file, + * 3) Modify the default layout on a directory + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "client/ioctl.h" +#include "common/errno.h" + +using namespace std; + + +#define CMD_SHOW_LAYOUT 1 +#define CMD_SHOW_LOC 2 +#define CMD_SET_LAYOUT 3 +#define CMD_MAP 4 + +void usage(); +int init_options(int argc, char **argv, int *fd, char **path, int *cmd, + int *stripe_unit, int *stripe_count, + int *object_size, int64_t *pool, int *file_offset, bool *dir); + +int main (int argc, char **argv) { + int fd = 0; + int err = 0; + char *path = 0; + int cmd = 0; + int stripe_unit = 0; + int stripe_count = 0; + int object_size = 0; + int64_t pool = 0; + int file_offset = 0; + bool dir = false; + + if (init_options(argc, argv, &fd, &path, &cmd, &stripe_unit, &stripe_count, + &object_size, &pool, &file_offset, &dir)){ + usage(); + return 0; + } + + if (CMD_SHOW_LAYOUT == cmd) { + struct ceph_ioctl_layout layout; + memset(&layout, 0, sizeof(layout)); + err = ioctl(fd, CEPH_IOC_GET_LAYOUT, (unsigned long)&layout); + if (err) { + cerr << "Error getting layout: " << cpp_strerror(errno) << endl; + return 1; + } + if (layout.stripe_unit == 0) { + cerr << "layout not specified" << endl; + } else { + cout << "layout.data_pool: " << layout.data_pool << endl; + cout << "layout.object_size: " << layout.object_size << endl; + cout << "layout.stripe_unit: " << layout.stripe_unit << endl; + cout << "layout.stripe_count: " << layout.stripe_count << endl; + } + } else if (CMD_SHOW_LOC == cmd) { + struct ceph_ioctl_dataloc location; + location.file_offset = file_offset; + err = ioctl(fd, CEPH_IOC_GET_DATALOC, (unsigned long)&location); + if (err) { + cerr << "Error getting location: " << cpp_strerror(err) << endl; + return 1; + } + cout << "location.file_offset: " << location.file_offset << endl; + cout << "location.object_offset:" << location.object_offset << endl; + cout << "location.object_no: " << location.object_no << endl; + cout << "location.object_size: " << location.object_size << endl; + cout << "location.object_name: " << location.object_name << endl; + cout << "location.block_offset: " << location.block_offset << endl; + cout << "location.block_size: " << location.block_size << endl; + cout << "location.osd: " << location.osd << endl; +// cout << "osd address: " << location.osd_addr << endl; + } else if (CMD_SET_LAYOUT == cmd) { + struct ceph_ioctl_layout layout; + memset(&layout, 0, sizeof(layout)); + int ioctl_num = (dir ? CEPH_IOC_SET_LAYOUT_POLICY : CEPH_IOC_SET_LAYOUT); + layout.data_pool = pool; + layout.object_size = object_size; + layout.stripe_count = stripe_count; + layout.stripe_unit = stripe_unit; + layout.unused = -1; /* used to be preferred_osd */ + err = ioctl(fd, ioctl_num, (unsigned long)&layout); + if (err) { + cerr << "Error setting layout: " << cpp_strerror(errno) << endl; + return 1; + } + } else if (CMD_MAP == cmd) { + struct stat st; + err = ::fstat(fd, &st); + if (err < 0) { + cerr << "error statting file: " << cpp_strerror(errno) << endl; + return 1; + } + + struct ceph_ioctl_layout layout; + memset(&layout, 0, sizeof(layout)); + err = ioctl(fd, CEPH_IOC_GET_LAYOUT, (unsigned long)&layout); + if (err) { + cerr << "Error getting layout: " << cpp_strerror(errno) << endl; + return 1; + } + + printf("%15s %24s %12s %12s %s\n", + "FILE OFFSET", "OBJECT", "OFFSET", "LENGTH", "OSD"); + + for (long long off = 0; off < st.st_size; off += layout.stripe_unit) { + struct ceph_ioctl_dataloc location; + location.file_offset = off; + err = ioctl(fd, CEPH_IOC_GET_DATALOC, (unsigned long)&location); + if (err) { + cerr << "Error getting location: " << cpp_strerror(errno) << endl; + return 1; + } + printf("%15lld %24s %12lld %12lld %d\n", + off, location.object_name, (long long)location.object_offset, + (long long)location.block_size, (int)location.osd); + } + + } else { + cerr << "unknown cmd somehow set!" << endl; + usage(); + return 1; + } + + return 0; +} + + +void usage() { + cerr << "usage: cephfs path command [options]*" << endl; + cerr << "Commands:" << endl; + cerr << " show_layout -- view the layout information on a file or dir" << endl; + cerr << " set_layout -- set the layout on an empty file,\n" + << " or the default layout on a directory" << endl; + cerr << " show_location -- view the location information on a file" << endl; + cerr << " map -- display file objects, pgs, osds" << endl; + cerr << "Options:" << endl; + cerr << " Useful for setting layouts:" << endl; + cerr << " --stripe_unit, -u: set the size of each stripe" << endl; + cerr << " --stripe_count, -c: set the number of objects to stripe across" << endl; + cerr << " --object_size, -s: set the size of the objects to stripe across" << endl; + cerr << " --pool, -p: set the pool to use" << endl; + cerr << endl; + cerr << " Useful for getting location data:" << endl; + cerr << " --offset, -l: the offset to retrieve location data for" << endl; + cerr << endl; +} + +int init_options(int argc, char **argv, int *fd, char **path, int *cmd, + int *stripe_unit, int *stripe_count, + int *object_size, int64_t *pool, int *file_offset, + bool *dir) { + // look through the options, make sure they're valid, + // and set the variables from them + int i = 3; + struct stat stat_field; + + if (argc < 3) { + cerr << "not enough parameters!" << endl; + return 1; + } + + *path = argv[1]; + + *fd = open(argv[1], O_RDONLY); + if (*fd < 0) { + cerr << "error opening path: " << cpp_strerror(*fd) << endl; + return 1; + } + + if (!strcmp(argv[2], "show_layout")) { + *cmd = CMD_SHOW_LAYOUT; + if (argc > 4) + return 1; + } else if (!strcmp(argv[2], "set_layout")) { + *cmd = CMD_SET_LAYOUT; + } else if (!strcmp(argv[2], "show_location")){ + *cmd = CMD_SHOW_LOC; + } else if (!strcmp(argv[2], "map")) { + *cmd = CMD_MAP; + } else { + cerr << "invalid command" << endl; + return 1; + } + + // okay, fill in options + while (i < argc) { + if (i == argc-1) { + // there's an option without an associated value! + cerr << "not all options are paired with a value!" << endl; + return 1; + } + + if (!strcmp(argv[i], "--stripe_unit") || argv[i][1] == 'u') { + if (*cmd != CMD_SET_LAYOUT) { + cerr << "Invalid option for command!" << endl; + return 1; + } + *stripe_unit = strtol(argv[i+1], NULL, 0); + if (!*stripe_unit) { + cerr << "invalid value for stripe unit" << endl; + return 1; + } + } else if (!strcmp(argv[i], "--stripe_count") || argv[i][1] == 'c') { + if (*cmd != CMD_SET_LAYOUT) { + cerr << "Invalid option for command!" << endl; + return 1; + } + *stripe_count = strtol(argv[i+1], NULL, 0); + if (!*stripe_count) { + cerr << "invalid value for stripe count" << endl; + return 1; + } + } else if (!strcmp(argv[i], "--object_size") || argv[i][1] == 's') { + if (*cmd != CMD_SET_LAYOUT) { + cerr << "Invalid option for command!" << endl; + return 1; + } + *object_size = strtol(argv[i+1], NULL, 0); + if (!*object_size) { + cerr << "invalid value for object size" << endl; + return 1; + } + } else if (!strcmp(argv[i], "--pool") || argv[i][1] == 'p') { + if (*cmd != CMD_SET_LAYOUT) { + cerr << "Invalid option for command!" << endl; + return 1; + } + errno = 0; + *pool= strtol(argv[i+1], NULL, 0); + if (!*pool && errno) { + cerr << "invalid value for pool" << endl; + return 1; + } + } else if (!strcmp(argv[i], "--offset") || argv[i][1] == 'l') { + if (*cmd != CMD_SHOW_LOC) { + cerr << "Invalid option for command!" << endl; + return 1; + } + errno = 0; + *file_offset = strtol(argv[i+1], NULL, 0); + if (!*file_offset && errno) { + cerr << "invalid value for offset" << endl; + return 1; + } + } + i += 2; + } + + int r = fstat (*fd, &stat_field); + if (r < 0) { + int err = errno; + cerr << "error doing stat file: " << cpp_strerror(err) << endl; + return 1; + } + + if (S_ISREG(stat_field.st_mode)) { // open read-write to set layout + close(*fd); + *fd = open(argv[1], O_RDWR); + if (*fd < 0) { + cerr << "error opening file" << endl; + return 1; + } + } else { + *dir = true; + } + + return 0; +} diff --git a/ceph/src/check_version b/ceph/src/check_version new file mode 100755 index 00000000..8600c556 --- /dev/null +++ b/ceph/src/check_version @@ -0,0 +1,19 @@ +#!/bin/sh + +dname=`dirname $0` + +if [ ! -d $dname/../.git ]; then + echo "not updating .git_version (no $dname/../.git)" + exit 0 +fi + +cur=`cd $dname && git rev-parse HEAD 2>/dev/null; git describe 2>/dev/null` +[ -e $1 ] && old=`cat $1` + +if [ "$cur" != "$old" ]; then + echo regenerating $1 with $cur + echo "$cur" > $1 +else + echo $1 is up to date. +fi + diff --git a/ceph/src/civetweb/civetweb.h b/ceph/src/civetweb/civetweb.h new file mode 100644 index 00000000..a6ca3e72 --- /dev/null +++ b/ceph/src/civetweb/civetweb.h @@ -0,0 +1,550 @@ +/* Copyright (c) 2013-2014 the Civetweb developers + * Copyright (c) 2004-2013 Sergey Lyubka + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef CIVETWEB_HEADER_INCLUDED +#define CIVETWEB_HEADER_INCLUDED + +#ifndef CIVETWEB_VERSION +#define CIVETWEB_VERSION "1.6" +#endif + +#ifndef CIVETWEB_API + #if defined(_WIN32) + #if defined(CIVETWEB_DLL_EXPORTS) + #define CIVETWEB_API __declspec(dllexport) + #elif defined(CIVETWEB_DLL_IMPORTS) + #define CIVETWEB_API __declspec(dllimport) + #else + #define CIVETWEB_API + #endif + #else + #define CIVETWEB_API + #endif +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +struct mg_context; /* Handle for the HTTP service itself */ +struct mg_connection; /* Handle for the individual connection */ + + +/* This structure contains information about the HTTP request. */ +struct mg_request_info { + const char *request_method; /* "GET", "POST", etc */ + const char *uri; /* URL-decoded URI */ + const char *http_version; /* E.g. "1.0", "1.1" */ + const char *query_string; /* URL part after '?', not including '?', or + NULL */ + const char *remote_user; /* Authenticated user, or NULL if no auth + used */ + long remote_ip; /* Client's IP address */ + int remote_port; /* Client's port */ + int is_ssl; /* 1 if SSL-ed, 0 if not */ + void *user_data; /* User data pointer passed to mg_start() */ + void *conn_data; /* Connection-specific user data */ + + int num_headers; /* Number of HTTP headers */ + struct mg_header { + const char *name; /* HTTP header name */ + const char *value; /* HTTP header value */ + } http_headers[64]; /* Maximum 64 headers */ +}; + + +/* This structure needs to be passed to mg_start(), to let civetweb know + which callbacks to invoke. For detailed description, see + https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md */ +struct mg_callbacks { + /* Called when civetweb has received new HTTP request. + If callback returns non-zero, + callback must process the request by sending valid HTTP headers and + body, and civetweb will not do any further processing. + If callback returns 0, civetweb processes the request itself. In this + case, callback must not send any data to the client. */ + int (*begin_request)(struct mg_connection *); + + /* Called when civetweb has finished processing request. */ + void (*end_request)(const struct mg_connection *, int reply_status_code); + + /* Called when civetweb is about to log a message. If callback returns + non-zero, civetweb does not log anything. */ + int (*log_message)(const struct mg_connection *, const char *message); + + /* Called when civetweb initializes SSL library. */ + int (*init_ssl)(void *ssl_context, void *user_data); + + /* Called when websocket request is received, before websocket handshake. + If callback returns 0, civetweb proceeds with handshake, otherwise + cinnection is closed immediately. */ + int (*websocket_connect)(const struct mg_connection *); + + /* Called when websocket handshake is successfully completed, and + connection is ready for data exchange. */ + void (*websocket_ready)(struct mg_connection *); + + /* Called when data frame has been received from the client. + Parameters: + bits: first byte of the websocket frame, see websocket RFC at + http://tools.ietf.org/html/rfc6455, section 5.2 + data, data_len: payload, with mask (if any) already applied. + Return value: + non-0: keep this websocket connection opened. + 0: close this websocket connection. */ + int (*websocket_data)(struct mg_connection *, int bits, + char *data, size_t data_len); + + /* Called when civetweb is closing a connection. The per-context mutex is + locked when this is invoked. This is primarily useful for noting when + a websocket is closing and removing it from any application-maintained + list of clients. */ + void (*connection_close)(struct mg_connection *); + + /* Called when civetweb tries to open a file. Used to intercept file open + calls, and serve file data from memory instead. + Parameters: + path: Full path to the file to open. + data_len: Placeholder for the file size, if file is served from + memory. + Return value: + NULL: do not serve file from memory, proceed with normal file open. + non-NULL: pointer to the file contents in memory. data_len must be + initilized with the size of the memory block. */ + const char * (*open_file)(const struct mg_connection *, + const char *path, size_t *data_len); + + /* Called when civetweb is about to serve Lua server page (.lp file), if + Lua support is enabled. + Parameters: + lua_context: "lua_State *" pointer. */ + void (*init_lua)(struct mg_connection *, void *lua_context); + + /* Called when civetweb has uploaded a file to a temporary directory as a + result of mg_upload() call. + Parameters: + file_file: full path name to the uploaded file. */ + void (*upload)(struct mg_connection *, const char *file_name); + + /* Called when civetweb is about to send HTTP error to the client. + Implementing this callback allows to create custom error pages. + Parameters: + status: HTTP error status code. */ + int (*http_error)(struct mg_connection *, int status); +}; + + +/* Start web server. + + Parameters: + callbacks: mg_callbacks structure with user-defined callbacks. + options: NULL terminated list of option_name, option_value pairs that + specify Civetweb configuration parameters. + + Side-effects: on UNIX, ignores SIGCHLD and SIGPIPE signals. If custom + processing is required for these, signal handlers must be set up + after calling mg_start(). + + + Example: + const char *options[] = { + "document_root", "/var/www", + "listening_ports", "80,443s", + NULL + }; + struct mg_context *ctx = mg_start(&my_func, NULL, options); + + Refer to https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md + for the list of valid option and their possible values. + + Return: + web server context, or NULL on error. */ +CIVETWEB_API struct mg_context *mg_start(const struct mg_callbacks *callbacks, + void *user_data, + const char **configuration_options); + + +/* Stop the web server. + + Must be called last, when an application wants to stop the web server and + release all associated resources. This function blocks until all Civetweb + threads are stopped. Context pointer becomes invalid. */ +CIVETWEB_API void mg_stop(struct mg_context *); + + +/* mg_request_handler + + Called when a new request comes in. This callback is URI based + and configured with mg_set_request_handler(). + + Parameters: + conn: current connection information. + cbdata: the callback data configured with mg_set_request_handler(). + Returns: + 0: the handler could not handle the request, so fall through. + 1: the handler processed the request. */ +typedef int (* mg_request_handler)(struct mg_connection *conn, void *cbdata); + + +/* mg_set_request_handler + + Sets or removes a URI mapping for a request handler. + + URI's are ordered and prefixed URI's are supported. For example, + consider two URIs: /a/b and /a + /a matches /a + /a/b matches /a/b + /a/c matches /a + + Parameters: + ctx: server context + uri: the URI to configure + handler: the callback handler to use when the URI is requested. + If NULL, the URI will be removed. + cbdata: the callback data to give to the handler when it s requested. */ +CIVETWEB_API void mg_set_request_handler(struct mg_context *ctx, const char *uri, mg_request_handler handler, void *cbdata); + + +/* Get the value of particular configuration parameter. + The value returned is read-only. Civetweb does not allow changing + configuration at run time. + If given parameter name is not valid, NULL is returned. For valid + names, return value is guaranteed to be non-NULL. If parameter is not + set, zero-length string is returned. */ +CIVETWEB_API const char *mg_get_option(const struct mg_context *ctx, const char *name); + + +#if defined(MG_LEGACY_INTERFACE) +/* Return array of strings that represent valid configuration options. + For each option, option name and default value is returned, i.e. the + number of entries in the array equals to number_of_options x 2. + Array is NULL terminated. */ +/* Deprecated: Use mg_get_valid_options instead. */ +CIVETWEB_API const char **mg_get_valid_option_names(void); +#endif + + +struct mg_option { + const char * name; + int type; + const char * default_value; +}; + +enum { + CONFIG_TYPE_UNKNOWN = 0x0, + CONFIG_TYPE_NUMBER = 0x1, + CONFIG_TYPE_STRING = 0x2, + CONFIG_TYPE_FILE = 0x3, + CONFIG_TYPE_DIRECTORY = 0x4, + CONFIG_TYPE_BOOLEAN = 0x5, + CONFIG_TYPE_EXT_PATTERN = 0x6 +}; + + +/* Return array of struct mg_option, representing all valid configuration + options of civetweb.c. + The array is terminated by a NULL name option. */ +CIVETWEB_API const struct mg_option *mg_get_valid_options(void); + + +/* Get the list of ports that civetweb is listening on. + size is the size of the ports int array and ssl int array to fill. + It is the caller's responsibility to make sure ports and ssl each + contain at least size int elements worth of memory to write into. + Return value is the number of ports and ssl information filled in. + The value returned is read-only. Civetweb does not allow changing + configuration at run time. */ +CIVETWEB_API size_t mg_get_ports(const struct mg_context *ctx, size_t size, int* ports, int* ssl); + + +/* Add, edit or delete the entry in the passwords file. + + This function allows an application to manipulate .htpasswd files on the + fly by adding, deleting and changing user records. This is one of the + several ways of implementing authentication on the server side. For another, + cookie-based way please refer to the examples/chat in the source tree. + + If password is not NULL, entry is added (or modified if already exists). + If password is NULL, entry is deleted. + + Return: + 1 on success, 0 on error. */ +CIVETWEB_API int mg_modify_passwords_file(const char *passwords_file_name, + const char *domain, + const char *user, + const char *password); + + +/* Return information associated with the request. */ +CIVETWEB_API struct mg_request_info *mg_get_request_info(struct mg_connection *); + + +/* Send data to the client. + Return: + 0 when the connection has been closed + -1 on error + >0 number of bytes written on success */ +CIVETWEB_API int mg_write(struct mg_connection *, const void *buf, size_t len); + + +/* Send data to a websocket client wrapped in a websocket frame. Uses mg_lock + to ensure that the transmission is not interrupted, i.e., when the + application is proactively communicating and responding to a request + simultaneously. + + Send data to a websocket client wrapped in a websocket frame. + This function is available when civetweb is compiled with -DUSE_WEBSOCKET + + Return: + 0 when the connection has been closed + -1 on error + >0 number of bytes written on success */ +CIVETWEB_API int mg_websocket_write(struct mg_connection* conn, int opcode, + const char *data, size_t data_len); + + +/* Blocks until unique access is obtained to this connection. Intended for use + with websockets only. + Invoke this before mg_write or mg_printf when communicating with a + websocket if your code has server-initiated communication as well as + communication in direct response to a message. */ +CIVETWEB_API void mg_lock(struct mg_connection* conn); +CIVETWEB_API void mg_unlock(struct mg_connection* conn); + + +/* Opcodes, from http://tools.ietf.org/html/rfc6455 */ +enum { + WEBSOCKET_OPCODE_CONTINUATION = 0x0, + WEBSOCKET_OPCODE_TEXT = 0x1, + WEBSOCKET_OPCODE_BINARY = 0x2, + WEBSOCKET_OPCODE_CONNECTION_CLOSE = 0x8, + WEBSOCKET_OPCODE_PING = 0x9, + WEBSOCKET_OPCODE_PONG = 0xa +}; + + +/* Macros for enabling compiler-specific checks for printf-like arguments. */ +#undef PRINTF_FORMAT_STRING +#if defined(_MSC_VER) && _MSC_VER >= 1400 +#include +#if defined(_MSC_VER) && _MSC_VER > 1400 +#define PRINTF_FORMAT_STRING(s) _Printf_format_string_ s +#else +#define PRINTF_FORMAT_STRING(s) __format_string s +#endif +#else +#define PRINTF_FORMAT_STRING(s) s +#endif + +#ifdef __GNUC__ +#define PRINTF_ARGS(x, y) __attribute__((format(printf, x, y))) +#else +#define PRINTF_ARGS(x, y) +#endif + +/* Send data to the client using printf() semantics. + Works exactly like mg_write(), but allows to do message formatting. */ +CIVETWEB_API int mg_printf(struct mg_connection *, + PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3); + + +/* Send contents of the entire file together with HTTP headers. */ +CIVETWEB_API void mg_send_file(struct mg_connection *conn, const char *path); + + +/* Read data from the remote end, return number of bytes read. + Return: + 0 connection has been closed by peer. No more data could be read. + < 0 read error. No more data could be read from the connection. + > 0 number of bytes read into the buffer. */ +CIVETWEB_API int mg_read(struct mg_connection *, void *buf, size_t len); + + +/* Get the value of particular HTTP header. + + This is a helper function. It traverses request_info->http_headers array, + and if the header is present in the array, returns its value. If it is + not present, NULL is returned. */ +CIVETWEB_API const char *mg_get_header(const struct mg_connection *, const char *name); + + +/* Get a value of particular form variable. + + Parameters: + data: pointer to form-uri-encoded buffer. This could be either POST data, + or request_info.query_string. + data_len: length of the encoded data. + var_name: variable name to decode from the buffer + dst: destination buffer for the decoded variable + dst_len: length of the destination buffer + + Return: + On success, length of the decoded variable. + On error: + -1 (variable not found). + -2 (destination buffer is NULL, zero length or too small to hold the + decoded variable). + + Destination buffer is guaranteed to be '\0' - terminated if it is not + NULL or zero length. */ +CIVETWEB_API int mg_get_var(const char *data, size_t data_len, + const char *var_name, char *dst, size_t dst_len); + + +/* Get a value of particular form variable. + + Parameters: + data: pointer to form-uri-encoded buffer. This could be either POST data, + or request_info.query_string. + data_len: length of the encoded data. + var_name: variable name to decode from the buffer + dst: destination buffer for the decoded variable + dst_len: length of the destination buffer + occurrence: which occurrence of the variable, 0 is the first, 1 the + second... + this makes it possible to parse a query like + b=x&a=y&a=z which will have occurrence values b:0, a:0 and a:1 + + Return: + On success, length of the decoded variable. + On error: + -1 (variable not found). + -2 (destination buffer is NULL, zero length or too small to hold the + decoded variable). + + Destination buffer is guaranteed to be '\0' - terminated if it is not + NULL or zero length. */ +CIVETWEB_API int mg_get_var2(const char *data, size_t data_len, + const char *var_name, char *dst, size_t dst_len, size_t occurrence); + + +/* Fetch value of certain cookie variable into the destination buffer. + + Destination buffer is guaranteed to be '\0' - terminated. In case of + failure, dst[0] == '\0'. Note that RFC allows many occurrences of the same + parameter. This function returns only first occurrence. + + Return: + On success, value length. + On error: + -1 (either "Cookie:" header is not present at all or the requested + parameter is not found). + -2 (destination buffer is NULL, zero length or too small to hold the + value). */ +CIVETWEB_API int mg_get_cookie(const char *cookie, const char *var_name, + char *buf, size_t buf_len); + + +/* Download data from the remote web server. + host: host name to connect to, e.g. "foo.com", or "10.12.40.1". + port: port number, e.g. 80. + use_ssl: wether to use SSL connection. + error_buffer, error_buffer_size: error message placeholder. + request_fmt,...: HTTP request. + Return: + On success, valid pointer to the new connection, suitable for mg_read(). + On error, NULL. error_buffer contains error message. + Example: + char ebuf[100]; + struct mg_connection *conn; + conn = mg_download("google.com", 80, 0, ebuf, sizeof(ebuf), + "%s", "GET / HTTP/1.0\r\nHost: google.com\r\n\r\n"); + */ +CIVETWEB_API struct mg_connection *mg_download(const char *host, int port, int use_ssl, + char *error_buffer, size_t error_buffer_size, + PRINTF_FORMAT_STRING(const char *request_fmt), + ...) PRINTF_ARGS(6, 7); + + +/* Close the connection opened by mg_download(). */ +CIVETWEB_API void mg_close_connection(struct mg_connection *conn); + + +/* File upload functionality. Each uploaded file gets saved into a temporary + file and MG_UPLOAD event is sent. + Return number of uploaded files. */ +CIVETWEB_API int mg_upload(struct mg_connection *conn, const char *destination_dir); + + +/* Convenience function -- create detached thread. + Return: 0 on success, non-0 on error. */ +typedef void * (*mg_thread_func_t)(void *); +CIVETWEB_API int mg_start_thread(mg_thread_func_t f, void *p); + + +/* Return builtin mime type for the given file name. + For unrecognized extensions, "text/plain" is returned. */ +CIVETWEB_API const char *mg_get_builtin_mime_type(const char *file_name); + + +/* Return Civetweb version. */ +CIVETWEB_API const char *mg_version(void); + + +/* URL-decode input buffer into destination buffer. + 0-terminate the destination buffer. + form-url-encoded data differs from URI encoding in a way that it + uses '+' as character for space, see RFC 1866 section 8.2.1 + http://ftp.ics.uci.edu/pub/ietf/html/rfc1866.txt + Return: length of the decoded data, or -1 if dst buffer is too small. */ +CIVETWEB_API int mg_url_decode(const char *src, int src_len, char *dst, + int dst_len, int is_form_url_encoded); + + +/* URL-encode input buffer into destination buffer. + returns the length of the resulting buffer or -1 + is the buffer is too small. */ +CIVETWEB_API int mg_url_encode(const char *src, char *dst, size_t dst_len); + + +/* MD5 hash given strings. + Buffer 'buf' must be 33 bytes long. Varargs is a NULL terminated list of + ASCIIz strings. When function returns, buf will contain human-readable + MD5 hash. Example: + char buf[33]; + mg_md5(buf, "aa", "bb", NULL); */ +CIVETWEB_API char *mg_md5(char buf[33], ...); + + +/* Print error message to the opened error log stream. + This utilizes the provided logging configuration. + conn: connection + fmt: format string without the line return + ...: variable argument list + Example: + mg_cry(conn,"i like %s", "logging"); */ +CIVETWEB_API void mg_cry(struct mg_connection *conn, + PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3); + + +/* utility method to compare two buffers, case incensitive. */ +CIVETWEB_API int mg_strncasecmp(const char *s1, const char *s2, size_t len); + + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* CIVETWEB_HEADER_INCLUDED */ diff --git a/ceph/src/civetweb/include/civetweb.h b/ceph/src/civetweb/include/civetweb.h new file mode 100644 index 00000000..a6ca3e72 --- /dev/null +++ b/ceph/src/civetweb/include/civetweb.h @@ -0,0 +1,550 @@ +/* Copyright (c) 2013-2014 the Civetweb developers + * Copyright (c) 2004-2013 Sergey Lyubka + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef CIVETWEB_HEADER_INCLUDED +#define CIVETWEB_HEADER_INCLUDED + +#ifndef CIVETWEB_VERSION +#define CIVETWEB_VERSION "1.6" +#endif + +#ifndef CIVETWEB_API + #if defined(_WIN32) + #if defined(CIVETWEB_DLL_EXPORTS) + #define CIVETWEB_API __declspec(dllexport) + #elif defined(CIVETWEB_DLL_IMPORTS) + #define CIVETWEB_API __declspec(dllimport) + #else + #define CIVETWEB_API + #endif + #else + #define CIVETWEB_API + #endif +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +struct mg_context; /* Handle for the HTTP service itself */ +struct mg_connection; /* Handle for the individual connection */ + + +/* This structure contains information about the HTTP request. */ +struct mg_request_info { + const char *request_method; /* "GET", "POST", etc */ + const char *uri; /* URL-decoded URI */ + const char *http_version; /* E.g. "1.0", "1.1" */ + const char *query_string; /* URL part after '?', not including '?', or + NULL */ + const char *remote_user; /* Authenticated user, or NULL if no auth + used */ + long remote_ip; /* Client's IP address */ + int remote_port; /* Client's port */ + int is_ssl; /* 1 if SSL-ed, 0 if not */ + void *user_data; /* User data pointer passed to mg_start() */ + void *conn_data; /* Connection-specific user data */ + + int num_headers; /* Number of HTTP headers */ + struct mg_header { + const char *name; /* HTTP header name */ + const char *value; /* HTTP header value */ + } http_headers[64]; /* Maximum 64 headers */ +}; + + +/* This structure needs to be passed to mg_start(), to let civetweb know + which callbacks to invoke. For detailed description, see + https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md */ +struct mg_callbacks { + /* Called when civetweb has received new HTTP request. + If callback returns non-zero, + callback must process the request by sending valid HTTP headers and + body, and civetweb will not do any further processing. + If callback returns 0, civetweb processes the request itself. In this + case, callback must not send any data to the client. */ + int (*begin_request)(struct mg_connection *); + + /* Called when civetweb has finished processing request. */ + void (*end_request)(const struct mg_connection *, int reply_status_code); + + /* Called when civetweb is about to log a message. If callback returns + non-zero, civetweb does not log anything. */ + int (*log_message)(const struct mg_connection *, const char *message); + + /* Called when civetweb initializes SSL library. */ + int (*init_ssl)(void *ssl_context, void *user_data); + + /* Called when websocket request is received, before websocket handshake. + If callback returns 0, civetweb proceeds with handshake, otherwise + cinnection is closed immediately. */ + int (*websocket_connect)(const struct mg_connection *); + + /* Called when websocket handshake is successfully completed, and + connection is ready for data exchange. */ + void (*websocket_ready)(struct mg_connection *); + + /* Called when data frame has been received from the client. + Parameters: + bits: first byte of the websocket frame, see websocket RFC at + http://tools.ietf.org/html/rfc6455, section 5.2 + data, data_len: payload, with mask (if any) already applied. + Return value: + non-0: keep this websocket connection opened. + 0: close this websocket connection. */ + int (*websocket_data)(struct mg_connection *, int bits, + char *data, size_t data_len); + + /* Called when civetweb is closing a connection. The per-context mutex is + locked when this is invoked. This is primarily useful for noting when + a websocket is closing and removing it from any application-maintained + list of clients. */ + void (*connection_close)(struct mg_connection *); + + /* Called when civetweb tries to open a file. Used to intercept file open + calls, and serve file data from memory instead. + Parameters: + path: Full path to the file to open. + data_len: Placeholder for the file size, if file is served from + memory. + Return value: + NULL: do not serve file from memory, proceed with normal file open. + non-NULL: pointer to the file contents in memory. data_len must be + initilized with the size of the memory block. */ + const char * (*open_file)(const struct mg_connection *, + const char *path, size_t *data_len); + + /* Called when civetweb is about to serve Lua server page (.lp file), if + Lua support is enabled. + Parameters: + lua_context: "lua_State *" pointer. */ + void (*init_lua)(struct mg_connection *, void *lua_context); + + /* Called when civetweb has uploaded a file to a temporary directory as a + result of mg_upload() call. + Parameters: + file_file: full path name to the uploaded file. */ + void (*upload)(struct mg_connection *, const char *file_name); + + /* Called when civetweb is about to send HTTP error to the client. + Implementing this callback allows to create custom error pages. + Parameters: + status: HTTP error status code. */ + int (*http_error)(struct mg_connection *, int status); +}; + + +/* Start web server. + + Parameters: + callbacks: mg_callbacks structure with user-defined callbacks. + options: NULL terminated list of option_name, option_value pairs that + specify Civetweb configuration parameters. + + Side-effects: on UNIX, ignores SIGCHLD and SIGPIPE signals. If custom + processing is required for these, signal handlers must be set up + after calling mg_start(). + + + Example: + const char *options[] = { + "document_root", "/var/www", + "listening_ports", "80,443s", + NULL + }; + struct mg_context *ctx = mg_start(&my_func, NULL, options); + + Refer to https://github.com/sunsetbrew/civetweb/blob/master/docs/UserManual.md + for the list of valid option and their possible values. + + Return: + web server context, or NULL on error. */ +CIVETWEB_API struct mg_context *mg_start(const struct mg_callbacks *callbacks, + void *user_data, + const char **configuration_options); + + +/* Stop the web server. + + Must be called last, when an application wants to stop the web server and + release all associated resources. This function blocks until all Civetweb + threads are stopped. Context pointer becomes invalid. */ +CIVETWEB_API void mg_stop(struct mg_context *); + + +/* mg_request_handler + + Called when a new request comes in. This callback is URI based + and configured with mg_set_request_handler(). + + Parameters: + conn: current connection information. + cbdata: the callback data configured with mg_set_request_handler(). + Returns: + 0: the handler could not handle the request, so fall through. + 1: the handler processed the request. */ +typedef int (* mg_request_handler)(struct mg_connection *conn, void *cbdata); + + +/* mg_set_request_handler + + Sets or removes a URI mapping for a request handler. + + URI's are ordered and prefixed URI's are supported. For example, + consider two URIs: /a/b and /a + /a matches /a + /a/b matches /a/b + /a/c matches /a + + Parameters: + ctx: server context + uri: the URI to configure + handler: the callback handler to use when the URI is requested. + If NULL, the URI will be removed. + cbdata: the callback data to give to the handler when it s requested. */ +CIVETWEB_API void mg_set_request_handler(struct mg_context *ctx, const char *uri, mg_request_handler handler, void *cbdata); + + +/* Get the value of particular configuration parameter. + The value returned is read-only. Civetweb does not allow changing + configuration at run time. + If given parameter name is not valid, NULL is returned. For valid + names, return value is guaranteed to be non-NULL. If parameter is not + set, zero-length string is returned. */ +CIVETWEB_API const char *mg_get_option(const struct mg_context *ctx, const char *name); + + +#if defined(MG_LEGACY_INTERFACE) +/* Return array of strings that represent valid configuration options. + For each option, option name and default value is returned, i.e. the + number of entries in the array equals to number_of_options x 2. + Array is NULL terminated. */ +/* Deprecated: Use mg_get_valid_options instead. */ +CIVETWEB_API const char **mg_get_valid_option_names(void); +#endif + + +struct mg_option { + const char * name; + int type; + const char * default_value; +}; + +enum { + CONFIG_TYPE_UNKNOWN = 0x0, + CONFIG_TYPE_NUMBER = 0x1, + CONFIG_TYPE_STRING = 0x2, + CONFIG_TYPE_FILE = 0x3, + CONFIG_TYPE_DIRECTORY = 0x4, + CONFIG_TYPE_BOOLEAN = 0x5, + CONFIG_TYPE_EXT_PATTERN = 0x6 +}; + + +/* Return array of struct mg_option, representing all valid configuration + options of civetweb.c. + The array is terminated by a NULL name option. */ +CIVETWEB_API const struct mg_option *mg_get_valid_options(void); + + +/* Get the list of ports that civetweb is listening on. + size is the size of the ports int array and ssl int array to fill. + It is the caller's responsibility to make sure ports and ssl each + contain at least size int elements worth of memory to write into. + Return value is the number of ports and ssl information filled in. + The value returned is read-only. Civetweb does not allow changing + configuration at run time. */ +CIVETWEB_API size_t mg_get_ports(const struct mg_context *ctx, size_t size, int* ports, int* ssl); + + +/* Add, edit or delete the entry in the passwords file. + + This function allows an application to manipulate .htpasswd files on the + fly by adding, deleting and changing user records. This is one of the + several ways of implementing authentication on the server side. For another, + cookie-based way please refer to the examples/chat in the source tree. + + If password is not NULL, entry is added (or modified if already exists). + If password is NULL, entry is deleted. + + Return: + 1 on success, 0 on error. */ +CIVETWEB_API int mg_modify_passwords_file(const char *passwords_file_name, + const char *domain, + const char *user, + const char *password); + + +/* Return information associated with the request. */ +CIVETWEB_API struct mg_request_info *mg_get_request_info(struct mg_connection *); + + +/* Send data to the client. + Return: + 0 when the connection has been closed + -1 on error + >0 number of bytes written on success */ +CIVETWEB_API int mg_write(struct mg_connection *, const void *buf, size_t len); + + +/* Send data to a websocket client wrapped in a websocket frame. Uses mg_lock + to ensure that the transmission is not interrupted, i.e., when the + application is proactively communicating and responding to a request + simultaneously. + + Send data to a websocket client wrapped in a websocket frame. + This function is available when civetweb is compiled with -DUSE_WEBSOCKET + + Return: + 0 when the connection has been closed + -1 on error + >0 number of bytes written on success */ +CIVETWEB_API int mg_websocket_write(struct mg_connection* conn, int opcode, + const char *data, size_t data_len); + + +/* Blocks until unique access is obtained to this connection. Intended for use + with websockets only. + Invoke this before mg_write or mg_printf when communicating with a + websocket if your code has server-initiated communication as well as + communication in direct response to a message. */ +CIVETWEB_API void mg_lock(struct mg_connection* conn); +CIVETWEB_API void mg_unlock(struct mg_connection* conn); + + +/* Opcodes, from http://tools.ietf.org/html/rfc6455 */ +enum { + WEBSOCKET_OPCODE_CONTINUATION = 0x0, + WEBSOCKET_OPCODE_TEXT = 0x1, + WEBSOCKET_OPCODE_BINARY = 0x2, + WEBSOCKET_OPCODE_CONNECTION_CLOSE = 0x8, + WEBSOCKET_OPCODE_PING = 0x9, + WEBSOCKET_OPCODE_PONG = 0xa +}; + + +/* Macros for enabling compiler-specific checks for printf-like arguments. */ +#undef PRINTF_FORMAT_STRING +#if defined(_MSC_VER) && _MSC_VER >= 1400 +#include +#if defined(_MSC_VER) && _MSC_VER > 1400 +#define PRINTF_FORMAT_STRING(s) _Printf_format_string_ s +#else +#define PRINTF_FORMAT_STRING(s) __format_string s +#endif +#else +#define PRINTF_FORMAT_STRING(s) s +#endif + +#ifdef __GNUC__ +#define PRINTF_ARGS(x, y) __attribute__((format(printf, x, y))) +#else +#define PRINTF_ARGS(x, y) +#endif + +/* Send data to the client using printf() semantics. + Works exactly like mg_write(), but allows to do message formatting. */ +CIVETWEB_API int mg_printf(struct mg_connection *, + PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3); + + +/* Send contents of the entire file together with HTTP headers. */ +CIVETWEB_API void mg_send_file(struct mg_connection *conn, const char *path); + + +/* Read data from the remote end, return number of bytes read. + Return: + 0 connection has been closed by peer. No more data could be read. + < 0 read error. No more data could be read from the connection. + > 0 number of bytes read into the buffer. */ +CIVETWEB_API int mg_read(struct mg_connection *, void *buf, size_t len); + + +/* Get the value of particular HTTP header. + + This is a helper function. It traverses request_info->http_headers array, + and if the header is present in the array, returns its value. If it is + not present, NULL is returned. */ +CIVETWEB_API const char *mg_get_header(const struct mg_connection *, const char *name); + + +/* Get a value of particular form variable. + + Parameters: + data: pointer to form-uri-encoded buffer. This could be either POST data, + or request_info.query_string. + data_len: length of the encoded data. + var_name: variable name to decode from the buffer + dst: destination buffer for the decoded variable + dst_len: length of the destination buffer + + Return: + On success, length of the decoded variable. + On error: + -1 (variable not found). + -2 (destination buffer is NULL, zero length or too small to hold the + decoded variable). + + Destination buffer is guaranteed to be '\0' - terminated if it is not + NULL or zero length. */ +CIVETWEB_API int mg_get_var(const char *data, size_t data_len, + const char *var_name, char *dst, size_t dst_len); + + +/* Get a value of particular form variable. + + Parameters: + data: pointer to form-uri-encoded buffer. This could be either POST data, + or request_info.query_string. + data_len: length of the encoded data. + var_name: variable name to decode from the buffer + dst: destination buffer for the decoded variable + dst_len: length of the destination buffer + occurrence: which occurrence of the variable, 0 is the first, 1 the + second... + this makes it possible to parse a query like + b=x&a=y&a=z which will have occurrence values b:0, a:0 and a:1 + + Return: + On success, length of the decoded variable. + On error: + -1 (variable not found). + -2 (destination buffer is NULL, zero length or too small to hold the + decoded variable). + + Destination buffer is guaranteed to be '\0' - terminated if it is not + NULL or zero length. */ +CIVETWEB_API int mg_get_var2(const char *data, size_t data_len, + const char *var_name, char *dst, size_t dst_len, size_t occurrence); + + +/* Fetch value of certain cookie variable into the destination buffer. + + Destination buffer is guaranteed to be '\0' - terminated. In case of + failure, dst[0] == '\0'. Note that RFC allows many occurrences of the same + parameter. This function returns only first occurrence. + + Return: + On success, value length. + On error: + -1 (either "Cookie:" header is not present at all or the requested + parameter is not found). + -2 (destination buffer is NULL, zero length or too small to hold the + value). */ +CIVETWEB_API int mg_get_cookie(const char *cookie, const char *var_name, + char *buf, size_t buf_len); + + +/* Download data from the remote web server. + host: host name to connect to, e.g. "foo.com", or "10.12.40.1". + port: port number, e.g. 80. + use_ssl: wether to use SSL connection. + error_buffer, error_buffer_size: error message placeholder. + request_fmt,...: HTTP request. + Return: + On success, valid pointer to the new connection, suitable for mg_read(). + On error, NULL. error_buffer contains error message. + Example: + char ebuf[100]; + struct mg_connection *conn; + conn = mg_download("google.com", 80, 0, ebuf, sizeof(ebuf), + "%s", "GET / HTTP/1.0\r\nHost: google.com\r\n\r\n"); + */ +CIVETWEB_API struct mg_connection *mg_download(const char *host, int port, int use_ssl, + char *error_buffer, size_t error_buffer_size, + PRINTF_FORMAT_STRING(const char *request_fmt), + ...) PRINTF_ARGS(6, 7); + + +/* Close the connection opened by mg_download(). */ +CIVETWEB_API void mg_close_connection(struct mg_connection *conn); + + +/* File upload functionality. Each uploaded file gets saved into a temporary + file and MG_UPLOAD event is sent. + Return number of uploaded files. */ +CIVETWEB_API int mg_upload(struct mg_connection *conn, const char *destination_dir); + + +/* Convenience function -- create detached thread. + Return: 0 on success, non-0 on error. */ +typedef void * (*mg_thread_func_t)(void *); +CIVETWEB_API int mg_start_thread(mg_thread_func_t f, void *p); + + +/* Return builtin mime type for the given file name. + For unrecognized extensions, "text/plain" is returned. */ +CIVETWEB_API const char *mg_get_builtin_mime_type(const char *file_name); + + +/* Return Civetweb version. */ +CIVETWEB_API const char *mg_version(void); + + +/* URL-decode input buffer into destination buffer. + 0-terminate the destination buffer. + form-url-encoded data differs from URI encoding in a way that it + uses '+' as character for space, see RFC 1866 section 8.2.1 + http://ftp.ics.uci.edu/pub/ietf/html/rfc1866.txt + Return: length of the decoded data, or -1 if dst buffer is too small. */ +CIVETWEB_API int mg_url_decode(const char *src, int src_len, char *dst, + int dst_len, int is_form_url_encoded); + + +/* URL-encode input buffer into destination buffer. + returns the length of the resulting buffer or -1 + is the buffer is too small. */ +CIVETWEB_API int mg_url_encode(const char *src, char *dst, size_t dst_len); + + +/* MD5 hash given strings. + Buffer 'buf' must be 33 bytes long. Varargs is a NULL terminated list of + ASCIIz strings. When function returns, buf will contain human-readable + MD5 hash. Example: + char buf[33]; + mg_md5(buf, "aa", "bb", NULL); */ +CIVETWEB_API char *mg_md5(char buf[33], ...); + + +/* Print error message to the opened error log stream. + This utilizes the provided logging configuration. + conn: connection + fmt: format string without the line return + ...: variable argument list + Example: + mg_cry(conn,"i like %s", "logging"); */ +CIVETWEB_API void mg_cry(struct mg_connection *conn, + PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(2, 3); + + +/* utility method to compare two buffers, case incensitive. */ +CIVETWEB_API int mg_strncasecmp(const char *s1, const char *s2, size_t len); + + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* CIVETWEB_HEADER_INCLUDED */ diff --git a/ceph/src/civetweb/src/civetweb.c b/ceph/src/civetweb/src/civetweb.c new file mode 100644 index 00000000..4aa8a02b --- /dev/null +++ b/ceph/src/civetweb/src/civetweb.c @@ -0,0 +1,6877 @@ +/* Copyright (c) 2013-2014 the Civetweb developers + * Copyright (c) 2004-2013 Sergey Lyubka + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#if defined(_WIN32) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS /* Disable deprecation warning in VS2005 */ +#endif +#else +#ifdef __linux__ +#define _XOPEN_SOURCE 600 /* For flockfile() on Linux */ +#endif +#ifndef _LARGEFILE_SOURCE +#define _LARGEFILE_SOURCE /* For fseeko(), ftello() */ +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 64 /* Use 64-bit file offsets by default */ +#endif +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS /* wants this for C++ */ +#endif +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS /* C++ wants that for INT64_MAX */ +#endif +#endif + +#if defined (_MSC_VER) +/* 'type cast' : conversion from 'int' to 'HANDLE' of greater size */ +#pragma warning (disable : 4306 ) +/* conditional expression is constant: introduced by FD_SET(..) */ +#pragma warning (disable : 4127) +/* non-constant aggregate initializer: issued due to missing C99 support */ +#pragma warning (disable : 4204) +#endif + +/* Disable WIN32_LEAN_AND_MEAN. + This makes windows.h always include winsock2.h */ +#if defined(WIN32_LEAN_AND_MEAN) +#undef WIN32_LEAN_AND_MEAN +#endif + +#if defined USE_IPV6 && defined(_WIN32) +#include +#endif + +#if defined(__SYMBIAN32__) +#define NO_SSL /* SSL is not supported */ +#define NO_CGI /* CGI is not supported */ +#define PATH_MAX FILENAME_MAX +#endif /* __SYMBIAN32__ */ + +#ifndef IGNORE_UNUSED_RESULT +#define IGNORE_UNUSED_RESULT(a) (void)((a) && 1) +#endif + +#ifndef _WIN32_WCE /* Some ANSI #includes are not available on Windows CE */ +#include +#include +#include +#include +#include +#endif /* !_WIN32_WCE */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef MAX_WORKER_THREADS +#define MAX_WORKER_THREADS 1024 +#endif + +#if defined(_WIN32) && !defined(__SYMBIAN32__) /* Windows specific */ +#if defined(_MSC_VER) && _MSC_VER <= 1400 +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0400 /* To make it link in VS2005 */ +#endif +#include + +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +#ifndef _IN_PORT_T +#ifndef in_port_t +#define in_port_t u_short +#endif +#endif + +#ifndef _WIN32_WCE +#include +#include +#include +#else /* _WIN32_WCE */ +#define NO_CGI /* WinCE has no pipes */ + +typedef long off_t; + +#define errno GetLastError() +#define strerror(x) _ultoa(x, (char *) _alloca(sizeof(x) *3 ), 10) +#endif /* _WIN32_WCE */ + +#define MAKEUQUAD(lo, hi) ((uint64_t)(((uint32_t)(lo)) | \ + ((uint64_t)((uint32_t)(hi))) << 32)) +#define RATE_DIFF 10000000 /* 100 nsecs */ +#define EPOCH_DIFF MAKEUQUAD(0xd53e8000, 0x019db1de) +#define SYS2UNIX_TIME(lo, hi) \ + (time_t) ((MAKEUQUAD((lo), (hi)) - EPOCH_DIFF) / RATE_DIFF) + +/* Visual Studio 6 does not know __func__ or __FUNCTION__ + The rest of MS compilers use __FUNCTION__, not C99 __func__ + Also use _strtoui64 on modern M$ compilers */ +#if defined(_MSC_VER) && _MSC_VER < 1300 +#define STRX(x) #x +#define STR(x) STRX(x) +#define __func__ __FILE__ ":" STR(__LINE__) +#define strtoull(x, y, z) (unsigned __int64) _atoi64(x) +#define strtoll(x, y, z) _atoi64(x) +#else +#define __func__ __FUNCTION__ +#define strtoull(x, y, z) _strtoui64(x, y, z) +#define strtoll(x, y, z) _strtoi64(x, y, z) +#endif /* _MSC_VER */ + +#define ERRNO GetLastError() +#define NO_SOCKLEN_T +#define SSL_LIB "ssleay32.dll" +#define CRYPTO_LIB "libeay32.dll" +#define O_NONBLOCK 0 +#if !defined(EWOULDBLOCK) +#define EWOULDBLOCK WSAEWOULDBLOCK +#endif /* !EWOULDBLOCK */ +#define _POSIX_ +#define INT64_FMT "I64d" + +#define WINCDECL __cdecl +#define SHUT_WR 1 +#define snprintf _snprintf +#define vsnprintf _vsnprintf +#define mg_sleep(x) Sleep(x) + +#define pipe(x) _pipe(x, MG_BUF_LEN, _O_BINARY) +#ifndef popen +#define popen(x, y) _popen(x, y) +#endif +#ifndef pclose +#define pclose(x) _pclose(x) +#endif +#define close(x) _close(x) +#define dlsym(x,y) GetProcAddress((HINSTANCE) (x), (y)) +#define RTLD_LAZY 0 +#define fseeko(x, y, z) _lseeki64(_fileno(x), (y), (z)) +#define fdopen(x, y) _fdopen((x), (y)) +#define write(x, y, z) _write((x), (y), (unsigned) z) +#define read(x, y, z) _read((x), (y), (unsigned) z) +#define flockfile(x) EnterCriticalSection(&global_log_file_lock) +#define funlockfile(x) LeaveCriticalSection(&global_log_file_lock) +#define sleep(x) Sleep((x) * 1000) +#define rmdir(x) _rmdir(x) + +#if !defined(va_copy) +#define va_copy(x, y) x = y +#endif /* !va_copy MINGW #defines va_copy */ + +#if !defined(fileno) +#define fileno(x) _fileno(x) +#endif /* !fileno MINGW #defines fileno */ + +typedef HANDLE pthread_mutex_t; +typedef DWORD pthread_key_t; +typedef HANDLE pthread_t; +typedef struct { + CRITICAL_SECTION threadIdSec; + int waitingthreadcount; /* The number of threads queued. */ + pthread_t *waitingthreadhdls; /* The thread handles. */ +} pthread_cond_t; + +#ifndef __clockid_t_defined +typedef DWORD clockid_t; +#endif +#ifndef CLOCK_MONOTONIC +#define CLOCK_MONOTONIC (1) +#endif +#ifndef CLOCK_REALTIME +#define CLOCK_REALTIME (2) +#endif + +#ifndef _TIMESPEC_DEFINED +struct timespec { + time_t tv_sec; /* seconds */ + long tv_nsec; /* nanoseconds */ +}; +#endif + +#define pid_t HANDLE /* MINGW typedefs pid_t to int. Using #define here. */ + +static int pthread_mutex_lock(pthread_mutex_t *); +static int pthread_mutex_unlock(pthread_mutex_t *); +static void to_unicode(const char *path, wchar_t *wbuf, size_t wbuf_len); +struct file; +static char *mg_fgets(char *buf, size_t size, struct file *filep, char **p); + +#if defined(HAVE_STDINT) +#include +#else +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; +typedef unsigned __int64 uint64_t; +typedef __int64 int64_t; +#define INT64_MAX 9223372036854775807 +#endif /* HAVE_STDINT */ + +/* POSIX dirent interface */ +struct dirent { + char d_name[PATH_MAX]; +}; + +typedef struct DIR { + HANDLE handle; + WIN32_FIND_DATAW info; + struct dirent result; +} DIR; + +#if !defined(USE_IPV6) && defined(_WIN32) +#ifndef HAVE_POLL +struct pollfd { + SOCKET fd; + short events; + short revents; +}; +#define POLLIN 1 +#endif +#endif + +/* Mark required libraries */ +#ifdef _MSC_VER +#pragma comment(lib, "Ws2_32.lib") +#endif + +#else /* UNIX specific */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(ANDROID) +typedef unsigned short int in_port_t; +#endif + +#include +#include +#include +#if !defined(NO_SSL_DL) && !defined(NO_SSL) +#include +#endif +#include +#if defined(__MACH__) +#define SSL_LIB "libssl.dylib" +#define CRYPTO_LIB "libcrypto.dylib" +#else +#if !defined(SSL_LIB) +#define SSL_LIB "libssl.so" +#endif +#if !defined(CRYPTO_LIB) +#define CRYPTO_LIB "libcrypto.so" +#endif +#endif +#ifndef O_BINARY +#define O_BINARY 0 +#endif /* O_BINARY */ +#define closesocket(a) close(a) +#define mg_mkdir(x, y) mkdir(x, y) +#define mg_remove(x) remove(x) +#define mg_sleep(x) usleep((x) * 1000) +#define ERRNO errno +#define INVALID_SOCKET (-1) +#define INT64_FMT PRId64 +typedef int SOCKET; +#define WINCDECL + +#endif /* End of Windows and UNIX specific includes */ + +#include "civetweb.h" + +#define PASSWORDS_FILE_NAME ".htpasswd" +#define CGI_ENVIRONMENT_SIZE 4096 +#define MAX_CGI_ENVIR_VARS 64 +#define MG_BUF_LEN 8192 +#ifndef MAX_REQUEST_SIZE +#define MAX_REQUEST_SIZE 16384 +#endif +#define ARRAY_SIZE(array) (sizeof(array) / sizeof(array[0])) + +#ifdef DEBUG_TRACE +#undef DEBUG_TRACE +#define DEBUG_TRACE(x) +#else +#if defined(DEBUG) +#define DEBUG_TRACE(x) do { \ + flockfile(stdout); \ + printf("*** %lu.%p.%s.%d: ", \ + (unsigned long) time(NULL), (void *) pthread_self(), \ + __func__, __LINE__); \ + printf x; \ + putchar('\n'); \ + fflush(stdout); \ + funlockfile(stdout); \ +} while (0) +#else +#define DEBUG_TRACE(x) +#endif /* DEBUG */ +#endif /* DEBUG_TRACE */ + +#if defined(MEMORY_DEBUGGING) +static unsigned long blockCount = 0; +static unsigned long totalMemUsed = 0; + +static void * mg_malloc_ex(size_t size, const char * file, unsigned line) { + + void * data = malloc(size + sizeof(size_t)); + void * memory = 0; + char mallocStr[256]; + + if (data) { + *(size_t*)data = size; + totalMemUsed += size; + blockCount++; + memory = (void *)(((char*)data)+sizeof(size_t)); + } + + sprintf(mallocStr, "MEM: %p %5u alloc %7u %4u --- %s:%u\n", memory, size, totalMemUsed, blockCount, file, line); +#if defined(_WIN32) + OutputDebugStringA(mallocStr); +#else + DEBUG_TRACE("%s", mallocStr); +#endif + + return memory; +} + +static void * mg_calloc_ex(size_t count, size_t size, const char * file, unsigned line) { + + void * data = mg_malloc_ex(size*count, file, line); + if (data) memset(data, 0, size); + + return data; +} + +static void mg_free_ex(void * memory, const char * file, unsigned line) { + + char mallocStr[256]; + void * data = (void *)(((char*)memory)-sizeof(size_t)); + size_t size; + + if (memory) { + size = *(size_t*)data; + totalMemUsed -= size; + blockCount--; + sprintf(mallocStr, "MEM: %p %5u free %7u %4u --- %s:%u\n", memory, size, totalMemUsed, blockCount, file, line); +#if defined(_WIN32) + OutputDebugStringA(mallocStr); +#else + DEBUG_TRACE("%s", mallocStr); +#endif + + free(data); + } +} + +static void * mg_realloc_ex(void * memory, size_t newsize, const char * file, unsigned line) { + + char mallocStr[256]; + void * data; + size_t oldsize; + + if (newsize) { + if (memory) { + data = (void *)(((char*)memory)-sizeof(size_t)); + oldsize = *(size_t*)data; + data = realloc(data, newsize+sizeof(size_t)); + if (data) { + totalMemUsed -= oldsize; + sprintf(mallocStr, "MEM: %p %5u r-free %7u %4u --- %s:%u\n", memory, oldsize, totalMemUsed, blockCount, file, line); +#if defined(_WIN32) + OutputDebugStringA(mallocStr); +#else + DEBUG_TRACE("%s", mallocStr); +#endif + totalMemUsed += newsize; + sprintf(mallocStr, "MEM: %p %5u r-alloc %7u %4u --- %s:%u\n", memory, newsize, totalMemUsed, blockCount, file, line); +#if defined(_WIN32) + OutputDebugStringA(mallocStr); +#else + DEBUG_TRACE("%s", mallocStr); +#endif + *(size_t*)data = newsize; + data = (void *)(((char*)data)+sizeof(size_t)); + } else { +#if defined(_WIN32) + OutputDebugStringA("MEM: realloc failed\n"); +#else + DEBUG_TRACE("MEM: realloc failed\n"); +#endif + } + } else { + data = mg_malloc_ex(newsize, file, line); + } + } else { + data = 0; + mg_free_ex(memory, file, line); + } + + return data; +} + +#define mg_malloc(a) mg_malloc_ex(a, __FILE__, __LINE__) +#define mg_calloc(a,b) mg_calloc_ex(a, b, __FILE__, __LINE__) +#define mg_realloc(a, b) mg_realloc_ex(a, b, __FILE__, __LINE__) +#define mg_free(a) mg_free_ex(a, __FILE__, __LINE__) + +#else +static __inline void * mg_malloc(size_t a) {return malloc(a);} +static __inline void * mg_calloc(size_t a, size_t b) {return calloc(a, b);} +static __inline void * mg_realloc(void * a, size_t b) {return realloc(a, b);} +static __inline void mg_free(void * a) {free(a);} +#endif + +#define malloc DO_NOT_USE_THIS_FUNCTION__USE_mg_malloc +#define calloc DO_NOT_USE_THIS_FUNCTION__USE_mg_calloc +#define realloc DO_NOT_USE_THIS_FUNCTION__USE_mg_realloc +#define free DO_NOT_USE_THIS_FUNCTION__USE_mg_free + +#ifdef _WIN32 +static CRITICAL_SECTION global_log_file_lock; +static DWORD pthread_self(void) +{ + return GetCurrentThreadId(); +} + +int pthread_key_create(pthread_key_t *key, void (*_must_be_zero)(void*) /* destructor function not supported for windows */) +{ + assert(_must_be_zero == NULL); + if ((key!=0) && (_must_be_zero == NULL)) { + *key = TlsAlloc(); + return (*key != TLS_OUT_OF_INDEXES) ? 0 : -1; + } + return -2; +} + +int pthread_key_delete(pthread_key_t key) +{ + return TlsFree(key) ? 0 : 1; +} + +int pthread_setspecific(pthread_key_t key, void * value) +{ + return TlsSetValue(key, value) ? 0 : 1; +} + +void *pthread_getspecific(pthread_key_t key) +{ + return TlsGetValue(key); +} +#endif /* _WIN32 */ + +#define MD5_STATIC static +#include "md5.h" + +/* Darwin prior to 7.0 and Win32 do not have socklen_t */ +#ifdef NO_SOCKLEN_T +typedef int socklen_t; +#endif /* NO_SOCKLEN_T */ +#define _DARWIN_UNLIMITED_SELECT + +#define IP_ADDR_STR_LEN 50 /* IPv6 hex string is 46 chars */ + +#if !defined(MSG_NOSIGNAL) +#define MSG_NOSIGNAL 0 +#endif + +#if !defined(SOMAXCONN) +#define SOMAXCONN 100 +#endif + +#if !defined(PATH_MAX) +#define PATH_MAX 4096 +#endif + +/* Size of the accepted socket queue */ +#if !defined(MGSQLEN) +#define MGSQLEN 20 +#endif + +static const char *http_500_error = "Internal Server Error"; + +#if defined(NO_SSL_DL) +#include +#include +#else +/* SSL loaded dynamically from DLL. + I put the prototypes here to be independent from OpenSSL source + installation. */ + +typedef struct ssl_st SSL; +typedef struct ssl_method_st SSL_METHOD; +typedef struct ssl_ctx_st SSL_CTX; + +struct ssl_func { + const char *name; /* SSL function name */ + void (*ptr)(void); /* Function pointer */ +}; + +#define SSL_free (* (void (*)(SSL *)) ssl_sw[0].ptr) +#define SSL_accept (* (int (*)(SSL *)) ssl_sw[1].ptr) +#define SSL_connect (* (int (*)(SSL *)) ssl_sw[2].ptr) +#define SSL_read (* (int (*)(SSL *, void *, int)) ssl_sw[3].ptr) +#define SSL_write (* (int (*)(SSL *, const void *,int)) ssl_sw[4].ptr) +#define SSL_get_error (* (int (*)(SSL *, int)) ssl_sw[5].ptr) +#define SSL_set_fd (* (int (*)(SSL *, SOCKET)) ssl_sw[6].ptr) +#define SSL_new (* (SSL * (*)(SSL_CTX *)) ssl_sw[7].ptr) +#define SSL_CTX_new (* (SSL_CTX * (*)(SSL_METHOD *)) ssl_sw[8].ptr) +#define SSLv23_server_method (* (SSL_METHOD * (*)(void)) ssl_sw[9].ptr) +#define SSL_library_init (* (int (*)(void)) ssl_sw[10].ptr) +#define SSL_CTX_use_PrivateKey_file (* (int (*)(SSL_CTX *, \ + const char *, int)) ssl_sw[11].ptr) +#define SSL_CTX_use_certificate_file (* (int (*)(SSL_CTX *, \ + const char *, int)) ssl_sw[12].ptr) +#define SSL_CTX_set_default_passwd_cb \ + (* (void (*)(SSL_CTX *, mg_callback_t)) ssl_sw[13].ptr) +#define SSL_CTX_free (* (void (*)(SSL_CTX *)) ssl_sw[14].ptr) +#define SSL_load_error_strings (* (void (*)(void)) ssl_sw[15].ptr) +#define SSL_CTX_use_certificate_chain_file \ + (* (int (*)(SSL_CTX *, const char *)) ssl_sw[16].ptr) +#define SSLv23_client_method (* (SSL_METHOD * (*)(void)) ssl_sw[17].ptr) +#define SSL_pending (* (int (*)(SSL *)) ssl_sw[18].ptr) +#define SSL_CTX_set_verify (* (void (*)(SSL_CTX *, int, int)) ssl_sw[19].ptr) +#define SSL_shutdown (* (int (*)(SSL *)) ssl_sw[20].ptr) + +#define CRYPTO_num_locks (* (int (*)(void)) crypto_sw[0].ptr) +#define CRYPTO_set_locking_callback \ + (* (void (*)(void (*)(int, int, const char *, int))) crypto_sw[1].ptr) +#define CRYPTO_set_id_callback \ + (* (void (*)(unsigned long (*)(void))) crypto_sw[2].ptr) +#define ERR_get_error (* (unsigned long (*)(void)) crypto_sw[3].ptr) +#define ERR_error_string (* (char * (*)(unsigned long,char *)) crypto_sw[4].ptr) + +/* set_ssl_option() function updates this array. + It loads SSL library dynamically and changes NULLs to the actual addresses + of respective functions. The macros above (like SSL_connect()) are really + just calling these functions indirectly via the pointer. */ +static struct ssl_func ssl_sw[] = { + {"SSL_free", NULL}, + {"SSL_accept", NULL}, + {"SSL_connect", NULL}, + {"SSL_read", NULL}, + {"SSL_write", NULL}, + {"SSL_get_error", NULL}, + {"SSL_set_fd", NULL}, + {"SSL_new", NULL}, + {"SSL_CTX_new", NULL}, + {"SSLv23_server_method", NULL}, + {"SSL_library_init", NULL}, + {"SSL_CTX_use_PrivateKey_file", NULL}, + {"SSL_CTX_use_certificate_file",NULL}, + {"SSL_CTX_set_default_passwd_cb",NULL}, + {"SSL_CTX_free", NULL}, + {"SSL_load_error_strings", NULL}, + {"SSL_CTX_use_certificate_chain_file", NULL}, + {"SSLv23_client_method", NULL}, + {"SSL_pending", NULL}, + {"SSL_CTX_set_verify", NULL}, + {"SSL_shutdown", NULL}, + {NULL, NULL} +}; + +/* Similar array as ssl_sw. These functions could be located in different + lib. */ +#if !defined(NO_SSL) +static struct ssl_func crypto_sw[] = { + {"CRYPTO_num_locks", NULL}, + {"CRYPTO_set_locking_callback", NULL}, + {"CRYPTO_set_id_callback", NULL}, + {"ERR_get_error", NULL}, + {"ERR_error_string", NULL}, + {NULL, NULL} +}; +#endif /* NO_SSL */ +#endif /* NO_SSL_DL */ + +static const char *month_names[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" +}; + +/* Unified socket address. For IPv6 support, add IPv6 address structure + in the union u. */ +union usa { + struct sockaddr sa; + struct sockaddr_in sin; +#if defined(USE_IPV6) + struct sockaddr_in6 sin6; +#endif +}; + +/* Describes a string (chunk of memory). */ +struct vec { + const char *ptr; + size_t len; +}; + +struct file { + int is_directory; + time_t modification_time; + int64_t size; + FILE *fp; + const char *membuf; /* Non-NULL if file data is in memory */ + /* set to 1 if the content is gzipped + in which case we need a content-encoding: gzip header */ + int gzipped; +}; +#define STRUCT_FILE_INITIALIZER {0, 0, 0, NULL, NULL, 0} + +/* Describes listening socket, or socket which was accept()-ed by the master + thread and queued for future handling by the worker thread. */ +struct socket { + SOCKET sock; /* Listening socket */ + union usa lsa; /* Local socket address */ + union usa rsa; /* Remote socket address */ + unsigned is_ssl:1; /* Is port SSL-ed */ + unsigned ssl_redir:1; /* Is port supposed to redirect everything to SSL + port */ +}; + +/* NOTE(lsm): this enum shoulds be in sync with the config_options below. */ +enum { + CGI_EXTENSIONS, CGI_ENVIRONMENT, PUT_DELETE_PASSWORDS_FILE, CGI_INTERPRETER, + PROTECT_URI, AUTHENTICATION_DOMAIN, SSI_EXTENSIONS, THROTTLE, + ACCESS_LOG_FILE, ENABLE_DIRECTORY_LISTING, ERROR_LOG_FILE, + GLOBAL_PASSWORDS_FILE, INDEX_FILES, ENABLE_KEEP_ALIVE, ACCESS_CONTROL_LIST, + EXTRA_MIME_TYPES, LISTENING_PORTS, DOCUMENT_ROOT, SSL_CERTIFICATE, + NUM_THREADS, RUN_AS_USER, REWRITE, HIDE_FILES, REQUEST_TIMEOUT, + +#if defined(USE_LUA) + LUA_PRELOAD_FILE, LUA_SCRIPT_EXTENSIONS, LUA_SERVER_PAGE_EXTENSIONS, +#endif +#if defined(USE_WEBSOCKET) + WEBSOCKET_ROOT, +#endif +#if defined(USE_LUA) && defined(USE_WEBSOCKET) + LUA_WEBSOCKET_EXTENSIONS, +#endif + ACCESS_CONTROL_ALLOW_ORIGIN, ERROR_PAGES, + + NUM_OPTIONS +}; + +/* TODO: replace 12345 by proper config types */ +static struct mg_option config_options[] = { + {"cgi_pattern", CONFIG_TYPE_EXT_PATTERN, "**.cgi$|**.pl$|**.php$"}, + {"cgi_environment", CONFIG_TYPE_STRING, NULL}, + {"put_delete_auth_file", CONFIG_TYPE_FILE, NULL}, + {"cgi_interpreter", CONFIG_TYPE_FILE, NULL}, + {"protect_uri", 12345, NULL}, + {"authentication_domain", CONFIG_TYPE_STRING, "mydomain.com"}, + {"ssi_pattern", CONFIG_TYPE_EXT_PATTERN, "**.shtml$|**.shtm$"}, + {"throttle", 12345, NULL}, + {"access_log_file", CONFIG_TYPE_FILE, NULL}, + {"enable_directory_listing", CONFIG_TYPE_BOOLEAN, "yes"}, + {"error_log_file", CONFIG_TYPE_FILE, NULL}, + {"global_auth_file", CONFIG_TYPE_FILE, NULL}, + {"index_files", 12345, +#ifdef USE_LUA + "index.xhtml,index.html,index.htm,index.lp,index.lsp,index.lua,index.cgi,index.shtml,index.php"}, +#else + "index.xhtml,index.html,index.htm,index.cgi,index.shtml,index.php"}, +#endif + {"enable_keep_alive", CONFIG_TYPE_BOOLEAN, "no"}, + {"access_control_list", 12345, NULL}, + {"extra_mime_types", 12345, NULL}, + {"listening_ports", 12345, "8080"}, + {"document_root", CONFIG_TYPE_DIRECTORY, NULL}, + {"ssl_certificate", CONFIG_TYPE_FILE, NULL}, + {"num_threads", CONFIG_TYPE_NUMBER, "50"}, + {"run_as_user", CONFIG_TYPE_STRING, NULL}, + {"url_rewrite_patterns", 12345, NULL}, + {"hide_files_patterns", 12345, NULL}, + {"request_timeout_ms", CONFIG_TYPE_NUMBER, "30000"}, + +#if defined(USE_LUA) + {"lua_preload_file", CONFIG_TYPE_FILE, NULL}, + {"lua_script_pattern", CONFIG_TYPE_EXT_PATTERN, "**.lua$"}, + {"lua_server_page_pattern", CONFIG_TYPE_EXT_PATTERN, "**.lp$|**.lsp$"}, +#endif +#if defined(USE_WEBSOCKET) + {"websocket_root", CONFIG_TYPE_DIRECTORY, NULL}, +#endif +#if defined(USE_LUA) && defined(USE_WEBSOCKET) + {"lua_websocket_pattern", CONFIG_TYPE_EXT_PATTERN, "**.lua$"}, +#endif + {"access_control_allow_origin", CONFIG_TYPE_STRING, "*"}, + {"error_pages", CONFIG_TYPE_DIRECTORY, NULL}, + + {NULL, CONFIG_TYPE_UNKNOWN, NULL} +}; + +struct mg_request_handler_info { + char *uri; + size_t uri_len; + mg_request_handler handler; + void *cbdata; + struct mg_request_handler_info *next; +}; + +struct mg_context { + volatile int stop_flag; /* Should we stop event loop */ + void *ssllib_dll_handle; /* Store the ssl library handle. */ + void *cryptolib_dll_handle; /* Store the crypto library handle. */ + SSL_CTX *ssl_ctx; /* SSL context */ + char *config[NUM_OPTIONS]; /* Civetweb configuration parameters */ + struct mg_callbacks callbacks; /* User-defined callback function */ + void *user_data; /* User-defined data */ + + struct socket *listening_sockets; + in_port_t *listening_ports; + int num_listening_sockets; + + volatile int num_threads; /* Number of threads */ + pthread_mutex_t mutex; /* Protects (max|num)_threads */ + pthread_cond_t cond; /* Condvar for tracking workers terminations */ + + struct socket queue[MGSQLEN]; /* Accepted sockets */ + volatile int sq_head; /* Head of the socket queue */ + volatile int sq_tail; /* Tail of the socket queue */ + pthread_cond_t sq_full; /* Signaled when socket is produced */ + pthread_cond_t sq_empty; /* Signaled when socket is consumed */ + pthread_t masterthreadid; /* The master thread ID. */ + int workerthreadcount; /* The amount of worker threads. */ + pthread_t *workerthreadids;/* The worker thread IDs. */ + + unsigned long start_time; /* Server start time, used for authentication */ + unsigned long nonce_count; /* Used nonces, used for authentication */ + + char *systemName; /* What operating system is running */ + + /* linked list of uri handlers */ + struct mg_request_handler_info *request_handlers; + +#if defined(USE_LUA) && defined(USE_WEBSOCKET) + /* linked list of shared lua websockets */ + struct mg_shared_lua_websocket *shared_lua_websockets; +#endif +}; + +struct mg_connection { + struct mg_request_info request_info; + struct mg_context *ctx; + SSL *ssl; /* SSL descriptor */ + SSL_CTX *client_ssl_ctx; /* SSL context for client connections */ + struct socket client; /* Connected client */ + time_t birth_time; /* Time when request was received */ + int64_t num_bytes_sent; /* Total bytes sent to client */ + int64_t content_len; /* Content-Length header value */ + int64_t consumed_content; /* How many bytes of content have been read */ + char *buf; /* Buffer for received data */ + char *path_info; /* PATH_INFO part of the URL */ + int must_close; /* 1 if connection must be closed */ + int in_error_handler; /* 1 if in handler for user defined error pages */ + int buf_size; /* Buffer size */ + int request_len; /* Size of the request + headers in a buffer */ + int data_len; /* Total size of data in a buffer */ + int status_code; /* HTTP reply status code, e.g. 200 */ + int throttle; /* Throttling, bytes/sec. <= 0 means no + throttle */ + time_t last_throttle_time; /* Last time throttled data was sent */ + int64_t last_throttle_bytes;/* Bytes sent this second */ + pthread_mutex_t mutex; /* Used by mg_lock/mg_unlock to ensure atomic + transmissions for websockets */ +#if defined(USE_LUA) && defined(USE_WEBSOCKET) + void * lua_websocket_state; /* Lua_State for a websocket connection */ +#endif +}; + +static pthread_key_t sTlsKey; /* Thread local storage index */ +static int sTlsInit = 0; + +struct mg_workerTLS { + int is_master; +#if defined(_WIN32) && !defined(__SYMBIAN32__) + HANDLE pthread_cond_helper_mutex; +#endif +}; + +/* Directory entry */ +struct de { + struct mg_connection *conn; + char *file_name; + struct file file; +}; + +#if defined(USE_WEBSOCKET) +static int is_websocket_request(const struct mg_connection *conn); +#endif + +#if defined(MG_LEGACY_INTERFACE) +const char **mg_get_valid_option_names(void) +{ + static const char * data[2 * sizeof(config_options) / sizeof(config_options[0])] = {0}; + int i; + + for (i=0; config_options[i].name != NULL; i++) { + data[i * 2] = config_options[i].name; + data[i * 2 + 1] = config_options[i].default_value; + } + + return data; +} +#endif + +const struct mg_option *mg_get_valid_options(void) +{ + return config_options; +} + + +static int is_file_in_memory(struct mg_connection *conn, const char *path, + struct file *filep) +{ + size_t size = 0; + if ((filep->membuf = conn->ctx->callbacks.open_file == NULL ? NULL : + conn->ctx->callbacks.open_file(conn, path, &size)) != NULL) { + /* NOTE: override filep->size only on success. Otherwise, it might + break constructs like if (!mg_stat() || !mg_fopen()) ... */ + filep->size = size; + } + return filep->membuf != NULL; +} + +static int is_file_opened(const struct file *filep) +{ + return filep->membuf != NULL || filep->fp != NULL; +} + +static int mg_fopen(struct mg_connection *conn, const char *path, + const char *mode, struct file *filep) +{ + if (!is_file_in_memory(conn, path, filep)) { +#ifdef _WIN32 + wchar_t wbuf[PATH_MAX], wmode[20]; + to_unicode(path, wbuf, ARRAY_SIZE(wbuf)); + MultiByteToWideChar(CP_UTF8, 0, mode, -1, wmode, ARRAY_SIZE(wmode)); + filep->fp = _wfopen(wbuf, wmode); +#else + filep->fp = fopen(path, mode); +#endif + } + + return is_file_opened(filep); +} + +static void mg_fclose(struct file *filep) +{ + if (filep != NULL && filep->fp != NULL) { + fclose(filep->fp); + } +} + +static void mg_strlcpy(register char *dst, register const char *src, size_t n) +{ + for (; *src != '\0' && n > 1; n--) { + *dst++ = *src++; + } + *dst = '\0'; +} + +static int lowercase(const char *s) +{ + return tolower(* (const unsigned char *) s); +} + +int mg_strncasecmp(const char *s1, const char *s2, size_t len) +{ + int diff = 0; + + if (len > 0) + do { + diff = lowercase(s1++) - lowercase(s2++); + } while (diff == 0 && s1[-1] != '\0' && --len > 0); + + return diff; +} + +static int mg_strcasecmp(const char *s1, const char *s2) +{ + int diff; + + do { + diff = lowercase(s1++) - lowercase(s2++); + } while (diff == 0 && s1[-1] != '\0'); + + return diff; +} + +static char * mg_strndup(const char *ptr, size_t len) +{ + char *p; + + if ((p = (char *) mg_malloc(len + 1)) != NULL) { + mg_strlcpy(p, ptr, len + 1); + } + + return p; +} + +static char * mg_strdup(const char *str) +{ + return mg_strndup(str, strlen(str)); +} + +static const char *mg_strcasestr(const char *big_str, const char *small_str) +{ + int i, big_len = (int)strlen(big_str), small_len = (int)strlen(small_str); + + for (i = 0; i <= big_len - small_len; i++) { + if (mg_strncasecmp(big_str + i, small_str, small_len) == 0) { + return big_str + i; + } + } + + return NULL; +} + +/* Like snprintf(), but never returns negative value, or a value + that is larger than a supplied buffer. + Thanks to Adam Zeldis to pointing snprintf()-caused vulnerability + in his audit report. */ +static int mg_vsnprintf(struct mg_connection *conn, char *buf, size_t buflen, + const char *fmt, va_list ap) +{ + int n; + + if (buflen == 0) + return 0; + + n = vsnprintf(buf, buflen, fmt, ap); + + if (n < 0) { + mg_cry(conn, "vsnprintf error"); + n = 0; + } else if (n >= (int) buflen) { + mg_cry(conn, "truncating vsnprintf buffer: [%.*s]", + n > 200 ? 200 : n, buf); + n = (int) buflen - 1; + } + buf[n] = '\0'; + + return n; +} + +static int mg_snprintf(struct mg_connection *conn, char *buf, size_t buflen, + PRINTF_FORMAT_STRING(const char *fmt), ...) +PRINTF_ARGS(4, 5); + +static int mg_snprintf(struct mg_connection *conn, char *buf, size_t buflen, + const char *fmt, ...) +{ + va_list ap; + int n; + + va_start(ap, fmt); + n = mg_vsnprintf(conn, buf, buflen, fmt, ap); + va_end(ap); + + return n; +} + +static int get_option_index(const char *name) +{ + int i; + + for (i = 0; config_options[i].name != NULL; i++) { + if (strcmp(config_options[i].name, name) == 0) { + return i; + } + } + return -1; +} + +const char *mg_get_option(const struct mg_context *ctx, const char *name) +{ + int i; + if ((i = get_option_index(name)) == -1) { + return NULL; + } else if (ctx->config[i] == NULL) { + return ""; + } else { + return ctx->config[i]; + } +} + +size_t mg_get_ports(const struct mg_context *ctx, size_t size, int* ports, int* ssl) +{ + size_t i; + for (i = 0; i < size && i < (size_t)ctx->num_listening_sockets; i++) + { + ssl[i] = ctx->listening_sockets[i].is_ssl; + ports[i] = ctx->listening_ports[i]; + } + return i; +} + +static void sockaddr_to_string(char *buf, size_t len, + const union usa *usa) +{ + buf[0] = '\0'; +#if defined(USE_IPV6) + inet_ntop(usa->sa.sa_family, usa->sa.sa_family == AF_INET ? + (void *) &usa->sin.sin_addr : + (void *) &usa->sin6.sin6_addr, buf, len); +#elif defined(_WIN32) + /* Only Windows Vista (and newer) have inet_ntop() */ + mg_strlcpy(buf, inet_ntoa(usa->sin.sin_addr), len); +#else + inet_ntop(usa->sa.sa_family, (void *) &usa->sin.sin_addr, buf, len); +#endif +} + +/* Convert time_t to a string. According to RFC2616, Sec 14.18, this must be included in all responses other than 100, 101, 5xx. */ +static void gmt_time_string(char *buf, size_t buf_len, time_t *t) +{ + struct tm *tm; + + tm = gmtime(t); + if (tm != NULL) { + strftime(buf, buf_len, "%a, %d %b %Y %H:%M:%S GMT", tm); + } else { + mg_strlcpy(buf, "Thu, 01 Jan 1970 00:00:00 GMT", buf_len); + buf[buf_len - 1] = '\0'; + } +} + +/* Print error message to the opened error log stream. */ +void mg_cry(struct mg_connection *conn, const char *fmt, ...) +{ + char buf[MG_BUF_LEN], src_addr[IP_ADDR_STR_LEN]; + va_list ap; + FILE *fp; + time_t timestamp; + + va_start(ap, fmt); + IGNORE_UNUSED_RESULT(vsnprintf(buf, sizeof(buf), fmt, ap)); + va_end(ap); + + /* Do not lock when getting the callback value, here and below. + I suppose this is fine, since function cannot disappear in the + same way string option can. */ + if (conn->ctx->callbacks.log_message == NULL || + conn->ctx->callbacks.log_message(conn, buf) == 0) { + fp = conn->ctx->config[ERROR_LOG_FILE] == NULL ? NULL : + fopen(conn->ctx->config[ERROR_LOG_FILE], "a+"); + + if (fp != NULL) { + flockfile(fp); + timestamp = time(NULL); + + sockaddr_to_string(src_addr, sizeof(src_addr), &conn->client.rsa); + fprintf(fp, "[%010lu] [error] [client %s] ", (unsigned long) timestamp, + src_addr); + + if (conn->request_info.request_method != NULL) { + fprintf(fp, "%s %s: ", conn->request_info.request_method, + conn->request_info.uri); + } + + fprintf(fp, "%s", buf); + fputc('\n', fp); + funlockfile(fp); + fclose(fp); + } + } +} + +/* Return fake connection structure. Used for logging, if connection + is not applicable at the moment of logging. */ +static struct mg_connection *fc(struct mg_context *ctx) +{ + static struct mg_connection fake_connection; + fake_connection.ctx = ctx; + return &fake_connection; +} + +const char *mg_version(void) +{ + return CIVETWEB_VERSION; +} + +struct mg_request_info *mg_get_request_info(struct mg_connection *conn) +{ + return &conn->request_info; +} + +/* Skip the characters until one of the delimiters characters found. + 0-terminate resulting word. Skip the delimiter and following whitespaces. + Advance pointer to buffer to the next word. Return found 0-terminated word. + Delimiters can be quoted with quotechar. */ +static char *skip_quoted(char **buf, const char *delimiters, + const char *whitespace, char quotechar) +{ + char *p, *begin_word, *end_word, *end_whitespace; + + begin_word = *buf; + end_word = begin_word + strcspn(begin_word, delimiters); + + /* Check for quotechar */ + if (end_word > begin_word) { + p = end_word - 1; + while (*p == quotechar) { + /* If there is anything beyond end_word, copy it */ + if (*end_word == '\0') { + *p = '\0'; + break; + } else { + size_t end_off = strcspn(end_word + 1, delimiters); + memmove (p, end_word, end_off + 1); + p += end_off; /* p must correspond to end_word - 1 */ + end_word += end_off + 1; + } + } + for (p++; p < end_word; p++) { + *p = '\0'; + } + } + + if (*end_word == '\0') { + *buf = end_word; + } else { + end_whitespace = end_word + 1 + strspn(end_word + 1, whitespace); + + for (p = end_word; p < end_whitespace; p++) { + *p = '\0'; + } + + *buf = end_whitespace; + } + + return begin_word; +} + +/* Simplified version of skip_quoted without quote char + and whitespace == delimiters */ +static char *skip(char **buf, const char *delimiters) +{ + return skip_quoted(buf, delimiters, delimiters, 0); +} + + +/* Return HTTP header value, or NULL if not found. */ +static const char *get_header(const struct mg_request_info *ri, + const char *name) +{ + int i; + + for (i = 0; i < ri->num_headers; i++) + if (!mg_strcasecmp(name, ri->http_headers[i].name)) + return ri->http_headers[i].value; + + return NULL; +} + +const char *mg_get_header(const struct mg_connection *conn, const char *name) +{ + return get_header(&conn->request_info, name); +} + +/* A helper function for traversing a comma separated list of values. + It returns a list pointer shifted to the next value, or NULL if the end + of the list found. + Value is stored in val vector. If value has form "x=y", then eq_val + vector is initialized to point to the "y" part, and val vector length + is adjusted to point only to "x". */ +static const char *next_option(const char *list, struct vec *val, + struct vec *eq_val) +{ + if (list == NULL || *list == '\0') { + /* End of the list */ + list = NULL; + } else { + val->ptr = list; + if ((list = strchr(val->ptr, ',')) != NULL) { + /* Comma found. Store length and shift the list ptr */ + val->len = list - val->ptr; + list++; + } else { + /* This value is the last one */ + list = val->ptr + strlen(val->ptr); + val->len = list - val->ptr; + } + + if (eq_val != NULL) { + /* Value has form "x=y", adjust pointers and lengths + so that val points to "x", and eq_val points to "y". */ + eq_val->len = 0; + eq_val->ptr = (const char *) memchr(val->ptr, '=', val->len); + if (eq_val->ptr != NULL) { + eq_val->ptr++; /* Skip over '=' character */ + eq_val->len = val->ptr + val->len - eq_val->ptr; + val->len = (eq_val->ptr - val->ptr) - 1; + } + } + } + + return list; +} + +/* Perform case-insensitive match of string against pattern */ +static int match_prefix(const char *pattern, int pattern_len, const char *str) +{ + const char *or_str; + int i, j, len, res; + + if ((or_str = (const char *) memchr(pattern, '|', pattern_len)) != NULL) { + res = match_prefix(pattern, (int)(or_str - pattern), str); + return res > 0 ? res : + match_prefix(or_str + 1, (int)((pattern + pattern_len) - (or_str + 1)), str); + } + + i = j = 0; + res = -1; + for (; i < pattern_len; i++, j++) { + if (pattern[i] == '?' && str[j] != '\0') { + continue; + } else if (pattern[i] == '$') { + return str[j] == '\0' ? j : -1; + } else if (pattern[i] == '*') { + i++; + if (pattern[i] == '*') { + i++; + len = (int) strlen(str + j); + } else { + len = (int) strcspn(str + j, "/"); + } + if (i == pattern_len) { + return j + len; + } + do { + res = match_prefix(pattern + i, pattern_len - i, str + j + len); + } while (res == -1 && len-- > 0); + return res == -1 ? -1 : j + res + len; + } else if (lowercase(&pattern[i]) != lowercase(&str[j])) { + return -1; + } + } + return j; +} + +/* HTTP 1.1 assumes keep alive if "Connection:" header is not set + This function must tolerate situations when connection info is not + set up, for example if request parsing failed. */ +static int should_keep_alive(const struct mg_connection *conn) +{ + const char *http_version = conn->request_info.http_version; + const char *header = mg_get_header(conn, "Connection"); + if (conn->must_close || + conn->status_code == 401 || + mg_strcasecmp(conn->ctx->config[ENABLE_KEEP_ALIVE], "yes") != 0 || + (header != NULL && mg_strcasecmp(header, "keep-alive") != 0) || + (header == NULL && http_version && 0!=strcmp(http_version, "1.1"))) { + return 0; + } + return 1; +} + +static const char *suggest_connection_header(const struct mg_connection *conn) +{ + return should_keep_alive(conn) ? "keep-alive" : "close"; +} + +static void handle_file_based_request(struct mg_connection *conn, const char *path, struct file *filep); +static int mg_stat(struct mg_connection *conn, const char *path, struct file *filep); + +static void send_http_error(struct mg_connection *, int, const char *, + PRINTF_FORMAT_STRING(const char *fmt), ...) +PRINTF_ARGS(4, 5); + + +static void send_http_error(struct mg_connection *conn, int status, + const char *reason, const char *fmt, ...) +{ + char buf[MG_BUF_LEN]; + va_list ap; + int len = 0, i, page_handler_found, scope; + char date[64]; + time_t curtime = time(NULL); + const char *error_handler = NULL; + struct file error_page_file = STRUCT_FILE_INITIALIZER; + const char *error_page_file_ext, *tstr; + + conn->status_code = status; + if (conn->in_error_handler || + conn->ctx->callbacks.http_error == NULL || + conn->ctx->callbacks.http_error(conn, status)) { + + if (!conn->in_error_handler) { + /* Send user defined error pages, if defined */ + error_handler = conn->ctx->config[ERROR_PAGES]; + error_page_file_ext = conn->ctx->config[INDEX_FILES]; + page_handler_found = 0; + if (error_handler != NULL) { + for (scope=1; (scope<=3) && !page_handler_found; scope++) { + switch (scope) { + case 1: + len = mg_snprintf(conn, buf, sizeof(buf)-32, "%serror%03u.", error_handler, status); + break; + case 2: + len = mg_snprintf(conn, buf, sizeof(buf)-32, "%serror%01uxx.", error_handler, status/100); + break; + default: + len = mg_snprintf(conn, buf, sizeof(buf)-32, "%serror.", error_handler); + break; + } + tstr = strchr(error_page_file_ext, '.'); + while (tstr) { + for (i=1; i<32 && tstr[i]!=0 && tstr[i]!=','; i++) buf[len+i-1]=tstr[i]; + buf[len+i-1]=0; + if (mg_stat(conn, buf, &error_page_file)) { + page_handler_found = 1; + break; + } + tstr = strchr(tstr+i, '.'); + } + } + } + + if (page_handler_found) { + conn->in_error_handler = 1; + handle_file_based_request(conn, buf, &error_page_file); + conn->in_error_handler = 0; + return; + } + } + + buf[0] = '\0'; + gmt_time_string(date, sizeof(date), &curtime); + + /* Errors 1xx, 204 and 304 MUST NOT send a body */ + if (status > 199 && status != 204 && status != 304) { + len = mg_snprintf(conn, buf, sizeof(buf)-1, "Error %d: %s", status, reason); + buf[len] = '\n'; + len++; + buf[len] = 0; + + va_start(ap, fmt); + len += mg_vsnprintf(conn, buf + len, sizeof(buf) - len, fmt, ap); + va_end(ap); + } + DEBUG_TRACE(("[%s]", buf)); + + mg_printf(conn, "HTTP/1.1 %d %s\r\n" + "Content-Length: %d\r\n" + "Date: %s\r\n" + "Connection: %s\r\n\r\n", + status, reason, len, date, + suggest_connection_header(conn)); + conn->num_bytes_sent += mg_printf(conn, "%s", buf); + } +} + +#if defined(_WIN32) && !defined(__SYMBIAN32__) +static int pthread_mutex_init(pthread_mutex_t *mutex, void *unused) +{ + (void) unused; + *mutex = CreateMutex(NULL, FALSE, NULL); + return *mutex == NULL ? -1 : 0; +} + +static int pthread_mutex_destroy(pthread_mutex_t *mutex) +{ + return CloseHandle(*mutex) == 0 ? -1 : 0; +} + +static int pthread_mutex_lock(pthread_mutex_t *mutex) +{ + return WaitForSingleObject(*mutex, INFINITE) == WAIT_OBJECT_0? 0 : -1; +} + +static int pthread_mutex_unlock(pthread_mutex_t *mutex) +{ + return ReleaseMutex(*mutex) == 0 ? -1 : 0; +} + +#ifndef WIN_PTHREADS_TIME_H +static int clock_gettime(clockid_t clk_id, struct timespec *tp) +{ + FILETIME ft; + ULARGE_INTEGER li; + BOOL ok = FALSE; + double d; + static double perfcnt_per_sec = 0.0; + + if (tp) { + if (clk_id == CLOCK_REALTIME) { + GetSystemTimeAsFileTime(&ft); + li.LowPart = ft.dwLowDateTime; + li.HighPart = ft.dwHighDateTime; + li.QuadPart -= 116444736000000000; /* 1.1.1970 in filedate */ + tp->tv_sec = (time_t)(li.QuadPart / 10000000); + tp->tv_nsec = (long)(li.QuadPart % 10000000) * 100; + ok = TRUE; + } else if (clk_id == CLOCK_MONOTONIC) { + if (perfcnt_per_sec == 0.0) { + QueryPerformanceFrequency((LARGE_INTEGER *) &li); + perfcnt_per_sec = 1.0 / li.QuadPart; + } + if (perfcnt_per_sec != 0.0) { + QueryPerformanceCounter((LARGE_INTEGER *) &li); + d = li.QuadPart * perfcnt_per_sec; + tp->tv_sec = (time_t)d; + d -= tp->tv_sec; + tp->tv_nsec = (long)(d*1.0E9); + ok = TRUE; + } + } + } + + return ok ? 0 : -1; +} +#endif + +static int pthread_cond_init(pthread_cond_t *cv, const void *unused) +{ + (void) unused; + InitializeCriticalSection(&cv->threadIdSec); + cv->waitingthreadcount = 0; + cv->waitingthreadhdls = mg_calloc(MAX_WORKER_THREADS, sizeof(pthread_t)); + return (cv->waitingthreadhdls!=NULL) ? 0 : -1; +} + +static int pthread_cond_timedwait(pthread_cond_t *cv, pthread_mutex_t *mutex, const struct timespec * abstime) +{ + struct mg_workerTLS * tls = (struct mg_workerTLS *)TlsGetValue(sTlsKey); + int ok; + struct timespec tsnow; + int64_t nsnow, nswaitabs, nswaitrel; + DWORD mswaitrel; + + EnterCriticalSection(&cv->threadIdSec); + assert(cv->waitingthreadcount < MAX_WORKER_THREADS); + cv->waitingthreadhdls[cv->waitingthreadcount] = tls->pthread_cond_helper_mutex; + cv->waitingthreadcount++; + LeaveCriticalSection(&cv->threadIdSec); + + if (abstime) { + clock_gettime(CLOCK_REALTIME, &tsnow); + nsnow = (((uint64_t)tsnow.tv_sec)<<32) + tsnow.tv_nsec; + nswaitabs = (((uint64_t)abstime->tv_sec)<<32) + abstime->tv_nsec; + nswaitrel = nswaitabs - nsnow; + if (nswaitrel<0) nswaitrel=0; + mswaitrel = (DWORD)(nswaitrel / 1000000); + } else { + mswaitrel = INFINITE; + } + + pthread_mutex_unlock(mutex); + ok = (WAIT_OBJECT_0 == WaitForSingleObject(tls->pthread_cond_helper_mutex, mswaitrel)); + pthread_mutex_lock(mutex); + + return ok ? 0 : -1; +} + +static int pthread_cond_wait(pthread_cond_t *cv, pthread_mutex_t *mutex) +{ + return pthread_cond_timedwait(cv, mutex, NULL); +} + +static int pthread_cond_signal(pthread_cond_t *cv) +{ + int i; + HANDLE wkup = NULL; + BOOL ok = FALSE; + + EnterCriticalSection(&cv->threadIdSec); + if (cv->waitingthreadcount) { + wkup = cv->waitingthreadhdls[0]; + ok = SetEvent(wkup); + + for (i=1; iwaitingthreadcount; i++) { + cv->waitingthreadhdls[i-1] = cv->waitingthreadhdls[i]; + } + cv->waitingthreadcount--; + + assert(ok); + } + LeaveCriticalSection(&cv->threadIdSec); + + return ok ? 0 : 1; +} + +static int pthread_cond_broadcast(pthread_cond_t *cv) +{ + EnterCriticalSection(&cv->threadIdSec); + while (cv->waitingthreadcount) { + pthread_cond_signal(cv); + } + LeaveCriticalSection(&cv->threadIdSec); + + return 0; +} + +static int pthread_cond_destroy(pthread_cond_t *cv) +{ + EnterCriticalSection(&cv->threadIdSec); + assert(cv->waitingthreadcount==0); + mg_free(cv->waitingthreadhdls); + cv->waitingthreadhdls = 0; + LeaveCriticalSection(&cv->threadIdSec); + DeleteCriticalSection(&cv->threadIdSec); + + return 0; +} + +/* For Windows, change all slashes to backslashes in path names. */ +static void change_slashes_to_backslashes(char *path) +{ + int i; + + for (i = 0; path[i] != '\0'; i++) { + if (path[i] == '/') + path[i] = '\\'; + /* i > 0 check is to preserve UNC paths, like \\server\file.txt */ + if (path[i] == '\\' && i > 0) + while (path[i + 1] == '\\' || path[i + 1] == '/') + (void) memmove(path + i + 1, + path + i + 2, strlen(path + i + 1)); + } +} + +/* Encode 'path' which is assumed UTF-8 string, into UNICODE string. + wbuf and wbuf_len is a target buffer and its length. */ +static void to_unicode(const char *path, wchar_t *wbuf, size_t wbuf_len) +{ + char buf[PATH_MAX], buf2[PATH_MAX]; + + mg_strlcpy(buf, path, sizeof(buf)); + change_slashes_to_backslashes(buf); + + /* Convert to Unicode and back. If doubly-converted string does not + match the original, something is fishy, reject. */ + memset(wbuf, 0, wbuf_len * sizeof(wchar_t)); + MultiByteToWideChar(CP_UTF8, 0, buf, -1, wbuf, (int) wbuf_len); + WideCharToMultiByte(CP_UTF8, 0, wbuf, (int) wbuf_len, buf2, sizeof(buf2), + NULL, NULL); + if (strcmp(buf, buf2) != 0) { + wbuf[0] = L'\0'; + } +} + +#if defined(_WIN32_WCE) +static time_t time(time_t *ptime) +{ + time_t t; + SYSTEMTIME st; + FILETIME ft; + + GetSystemTime(&st); + SystemTimeToFileTime(&st, &ft); + t = SYS2UNIX_TIME(ft.dwLowDateTime, ft.dwHighDateTime); + + if (ptime != NULL) { + *ptime = t; + } + + return t; +} + +static struct tm *localtime(const time_t *ptime, struct tm *ptm) +{ + int64_t t = ((int64_t) *ptime) * RATE_DIFF + EPOCH_DIFF; + FILETIME ft, lft; + SYSTEMTIME st; + TIME_ZONE_INFORMATION tzinfo; + + if (ptm == NULL) { + return NULL; + } + + * (int64_t *) &ft = t; + FileTimeToLocalFileTime(&ft, &lft); + FileTimeToSystemTime(&lft, &st); + ptm->tm_year = st.wYear - 1900; + ptm->tm_mon = st.wMonth - 1; + ptm->tm_wday = st.wDayOfWeek; + ptm->tm_mday = st.wDay; + ptm->tm_hour = st.wHour; + ptm->tm_min = st.wMinute; + ptm->tm_sec = st.wSecond; + ptm->tm_yday = 0; /* hope nobody uses this */ + ptm->tm_isdst = + GetTimeZoneInformation(&tzinfo) == TIME_ZONE_ID_DAYLIGHT ? 1 : 0; + + return ptm; +} + +static struct tm *gmtime(const time_t *ptime, struct tm *ptm) +{ + /* FIXME(lsm): fix this. */ + return localtime(ptime, ptm); +} + +static size_t strftime(char *dst, size_t dst_size, const char *fmt, + const struct tm *tm) +{ + (void) snprintf(dst, dst_size, "implement strftime() for WinCE"); + return 0; +} +#endif + +/* Windows happily opens files with some garbage at the end of file name. + For example, fopen("a.cgi ", "r") on Windows successfully opens + "a.cgi", despite one would expect an error back. + This function returns non-0 if path ends with some garbage. */ +static int path_cannot_disclose_cgi(const char *path) +{ + static const char *allowed_last_characters = "_-"; + int last = path[strlen(path) - 1]; + return isalnum(last) || strchr(allowed_last_characters, last) != NULL; +} + +static int mg_stat(struct mg_connection *conn, const char *path, struct file *filep) +{ + wchar_t wbuf[PATH_MAX]; + WIN32_FILE_ATTRIBUTE_DATA info; + + if (!is_file_in_memory(conn, path, filep)) { + to_unicode(path, wbuf, ARRAY_SIZE(wbuf)); + if (GetFileAttributesExW(wbuf, GetFileExInfoStandard, &info) != 0) { + filep->size = MAKEUQUAD(info.nFileSizeLow, info.nFileSizeHigh); + filep->modification_time = SYS2UNIX_TIME( + info.ftLastWriteTime.dwLowDateTime, + info.ftLastWriteTime.dwHighDateTime); + filep->is_directory = info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY; + /* If file name is fishy, reset the file structure and return + error. + Note it is important to reset, not just return the error, cause + functions like is_file_opened() check the struct. */ + if (!filep->is_directory && !path_cannot_disclose_cgi(path)) { + memset(filep, 0, sizeof(*filep)); + } + } + } + + return filep->membuf != NULL || filep->modification_time != 0; +} + +static int mg_remove(const char *path) +{ + wchar_t wbuf[PATH_MAX]; + to_unicode(path, wbuf, ARRAY_SIZE(wbuf)); + return DeleteFileW(wbuf) ? 0 : -1; +} + +static int mg_mkdir(const char *path, int mode) +{ + char buf[PATH_MAX]; + wchar_t wbuf[PATH_MAX]; + + (void) mode; + mg_strlcpy(buf, path, sizeof(buf)); + change_slashes_to_backslashes(buf); + + (void) MultiByteToWideChar(CP_UTF8, 0, buf, -1, wbuf, ARRAY_SIZE(wbuf)); + + return CreateDirectoryW(wbuf, NULL) ? 0 : -1; +} + +/* Implementation of POSIX opendir/closedir/readdir for Windows. */ +static DIR * opendir(const char *name) +{ + DIR *dir = NULL; + wchar_t wpath[PATH_MAX]; + DWORD attrs; + + if (name == NULL) { + SetLastError(ERROR_BAD_ARGUMENTS); + } else if ((dir = (DIR *) mg_malloc(sizeof(*dir))) == NULL) { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + } else { + to_unicode(name, wpath, ARRAY_SIZE(wpath)); + attrs = GetFileAttributesW(wpath); + if (attrs != 0xFFFFFFFF && + ((attrs & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY)) { + (void) wcscat(wpath, L"\\*"); + dir->handle = FindFirstFileW(wpath, &dir->info); + dir->result.d_name[0] = '\0'; + } else { + mg_free(dir); + dir = NULL; + } + } + + return dir; +} + +static int closedir(DIR *dir) +{ + int result = 0; + + if (dir != NULL) { + if (dir->handle != INVALID_HANDLE_VALUE) + result = FindClose(dir->handle) ? 0 : -1; + + mg_free(dir); + } else { + result = -1; + SetLastError(ERROR_BAD_ARGUMENTS); + } + + return result; +} + +static struct dirent *readdir(DIR *dir) +{ + struct dirent *result = 0; + + if (dir) { + if (dir->handle != INVALID_HANDLE_VALUE) { + result = &dir->result; + (void) WideCharToMultiByte(CP_UTF8, 0, + dir->info.cFileName, -1, result->d_name, + sizeof(result->d_name), NULL, NULL); + + if (!FindNextFileW(dir->handle, &dir->info)) { + (void) FindClose(dir->handle); + dir->handle = INVALID_HANDLE_VALUE; + } + + } else { + SetLastError(ERROR_FILE_NOT_FOUND); + } + } else { + SetLastError(ERROR_BAD_ARGUMENTS); + } + + return result; +} + +#ifndef HAVE_POLL +static int poll(struct pollfd *pfd, int n, int milliseconds) +{ + struct timeval tv; + fd_set set; + int i, result; + SOCKET maxfd = 0; + + tv.tv_sec = milliseconds / 1000; + tv.tv_usec = (milliseconds % 1000) * 1000; + FD_ZERO(&set); + + for (i = 0; i < n; i++) { + FD_SET((SOCKET) pfd[i].fd, &set); + pfd[i].revents = 0; + + if (pfd[i].fd > maxfd) { + maxfd = pfd[i].fd; + } + } + + if ((result = select((int)maxfd + 1, &set, NULL, NULL, &tv)) > 0) { + for (i = 0; i < n; i++) { + if (FD_ISSET(pfd[i].fd, &set)) { + pfd[i].revents = POLLIN; + } + } + } + + return result; +} +#endif /* HAVE_POLL */ + +static void set_close_on_exec(SOCKET sock, struct mg_connection *conn /* may be null */) +{ + (void) conn; /* Unused. */ + (void) SetHandleInformation((HANDLE) sock, HANDLE_FLAG_INHERIT, 0); +} + +int mg_start_thread(mg_thread_func_t f, void *p) +{ +#if defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1) + /* Compile-time option to control stack size, e.g. -DUSE_STACK_SIZE=16384 */ + return ((_beginthread((void (__cdecl *)(void *)) f, USE_STACK_SIZE, p) == ((uintptr_t)(-1L))) ? -1 : 0); +#else + return ((_beginthread((void (__cdecl *)(void *)) f, 0, p) == ((uintptr_t)(-1L))) ? -1 : 0); +#endif /* defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1) */ +} + +/* Start a thread storing the thread context. */ + +static int mg_start_thread_with_id(unsigned (__stdcall *f)(void *), void *p, + pthread_t *threadidptr) +{ + uintptr_t uip; + HANDLE threadhandle; + int result = -1; + + uip = _beginthreadex(NULL, 0, (unsigned (__stdcall *)(void *)) f, p, 0, + NULL); + threadhandle = (HANDLE) uip; + if ((uip != (uintptr_t)(-1L)) && (threadidptr != NULL)) { + *threadidptr = threadhandle; + result = 0; + } + + return result; +} + +/* Wait for a thread to finish. */ + +static int mg_join_thread(pthread_t threadid) +{ + int result; + DWORD dwevent; + + result = -1; + dwevent = WaitForSingleObject(threadid, INFINITE); + if (dwevent == WAIT_FAILED) { + int err; + + err = GetLastError(); + DEBUG_TRACE(("WaitForSingleObject() failed, error %d", err)); + } else { + if (dwevent == WAIT_OBJECT_0) { + CloseHandle(threadid); + result = 0; + } + } + + return result; +} + +static HANDLE dlopen(const char *dll_name, int flags) +{ + wchar_t wbuf[PATH_MAX]; + (void) flags; + to_unicode(dll_name, wbuf, ARRAY_SIZE(wbuf)); + return LoadLibraryW(wbuf); +} + +static int dlclose(void *handle) +{ + int result; + + if (FreeLibrary(handle) != 0) { + result = 0; + } else { + result = -1; + } + + return result; +} + +#if !defined(NO_CGI) +#define SIGKILL 0 +static int kill(pid_t pid, int sig_num) +{ + (void) TerminateProcess(pid, sig_num); + (void) CloseHandle(pid); + return 0; +} + +static void trim_trailing_whitespaces(char *s) +{ + char *e = s + strlen(s) - 1; + while (e > s && isspace(* (unsigned char *) e)) { + *e-- = '\0'; + } +} + +static pid_t spawn_process(struct mg_connection *conn, const char *prog, + char *envblk, char *envp[], int fdin, + int fdout, const char *dir) +{ + HANDLE me; + char *p, *interp, full_interp[PATH_MAX], full_dir[PATH_MAX], + cmdline[PATH_MAX], buf[PATH_MAX]; + struct file file = STRUCT_FILE_INITIALIZER; + STARTUPINFOA si; + PROCESS_INFORMATION pi = { 0 }; + + (void) envp; + + memset(&si, 0, sizeof(si)); + si.cb = sizeof(si); + + /* TODO(lsm): redirect CGI errors to the error log file */ + si.dwFlags = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW; + si.wShowWindow = SW_HIDE; + + me = GetCurrentProcess(); + DuplicateHandle(me, (HANDLE) _get_osfhandle(fdin), me, + &si.hStdInput, 0, TRUE, DUPLICATE_SAME_ACCESS); + DuplicateHandle(me, (HANDLE) _get_osfhandle(fdout), me, + &si.hStdOutput, 0, TRUE, DUPLICATE_SAME_ACCESS); + + /* If CGI file is a script, try to read the interpreter line */ + interp = conn->ctx->config[CGI_INTERPRETER]; + if (interp == NULL) { + buf[0] = buf[1] = '\0'; + + /* Read the first line of the script into the buffer */ + snprintf(cmdline, sizeof(cmdline), "%s%c%s", dir, '/', prog); + if (mg_fopen(conn, cmdline, "r", &file)) { + p = (char *) file.membuf; + mg_fgets(buf, sizeof(buf), &file, &p); + mg_fclose(&file); + buf[sizeof(buf) - 1] = '\0'; + } + + if (buf[0] == '#' && buf[1] == '!') { + trim_trailing_whitespaces(buf + 2); + } else { + buf[2] = '\0'; + } + interp = buf + 2; + } + + if (interp[0] != '\0') { + GetFullPathNameA(interp, sizeof(full_interp), full_interp, NULL); + interp = full_interp; + } + GetFullPathNameA(dir, sizeof(full_dir), full_dir, NULL); + + mg_snprintf(conn, cmdline, sizeof(cmdline), "%s%s\"%s\\%s\"", + interp, interp[0] == '\0' ? "" : " ", full_dir, prog); + + DEBUG_TRACE(("Running [%s]", cmdline)); + if (CreateProcessA(NULL, cmdline, NULL, NULL, TRUE, + CREATE_NEW_PROCESS_GROUP, envblk, NULL, &si, &pi) == 0) { + mg_cry(conn, "%s: CreateProcess(%s): %ld", + __func__, cmdline, ERRNO); + pi.hProcess = (pid_t) -1; + } + + (void) CloseHandle(si.hStdOutput); + (void) CloseHandle(si.hStdInput); + if (pi.hThread != NULL) + (void) CloseHandle(pi.hThread); + + return (pid_t) pi.hProcess; +} +#endif /* !NO_CGI */ + +static int set_non_blocking_mode(SOCKET sock) +{ + unsigned long on = 1; + return ioctlsocket(sock, FIONBIO, &on); +} + +#else +static int mg_stat(struct mg_connection *conn, const char *path, + struct file *filep) +{ + struct stat st; + + if (!is_file_in_memory(conn, path, filep) && !stat(path, &st)) { + filep->size = st.st_size; + filep->modification_time = st.st_mtime; + filep->is_directory = S_ISDIR(st.st_mode); + } else { + filep->modification_time = (time_t) 0; + } + + return filep->membuf != NULL || filep->modification_time != (time_t) 0; +} + +static void set_close_on_exec(int fd, struct mg_connection *conn /* may be null */) +{ + if (fcntl(fd, F_SETFD, FD_CLOEXEC) != 0) { + if (conn) + mg_cry(conn, "%s: fcntl(F_SETFD FD_CLOEXEC) failed: %s", + __func__, strerror(ERRNO)); + } +} + +int mg_start_thread(mg_thread_func_t func, void *param) +{ + pthread_t thread_id; + pthread_attr_t attr; + int result; + + (void) pthread_attr_init(&attr); + (void) pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + +#if defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1) + /* Compile-time option to control stack size, + e.g. -DUSE_STACK_SIZE=16384 */ + (void) pthread_attr_setstacksize(&attr, USE_STACK_SIZE); +#endif /* defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1) */ + + result = pthread_create(&thread_id, &attr, func, param); + pthread_attr_destroy(&attr); + + return result; +} + +/* Start a thread storing the thread context. */ + +static int mg_start_thread_with_id(mg_thread_func_t func, void *param, + pthread_t *threadidptr) +{ + pthread_t thread_id; + pthread_attr_t attr; + int result; + + (void) pthread_attr_init(&attr); + +#if defined(USE_STACK_SIZE) && (USE_STACK_SIZE > 1) + /* Compile-time option to control stack size, + e.g. -DUSE_STACK_SIZE=16384 */ + (void) pthread_attr_setstacksize(&attr, USE_STACK_SIZE); +#endif /* defined(USE_STACK_SIZE) && USE_STACK_SIZE > 1 */ + + result = pthread_create(&thread_id, &attr, func, param); + pthread_attr_destroy(&attr); + if (threadidptr != NULL) { + *threadidptr = thread_id; + } + return result; +} + +/* Wait for a thread to finish. */ + +static int mg_join_thread(pthread_t threadid) +{ + int result; + + result = pthread_join(threadid, NULL); + return result; +} + +#ifndef NO_CGI +static pid_t spawn_process(struct mg_connection *conn, const char *prog, + char *envblk, char *envp[], int fdin, + int fdout, const char *dir) +{ + pid_t pid; + const char *interp; + + (void) envblk; + + if ((pid = fork()) == -1) { + /* Parent */ + send_http_error(conn, 500, http_500_error, "fork(): %s", strerror(ERRNO)); + } else if (pid == 0) { + /* Child */ + if (chdir(dir) != 0) { + mg_cry(conn, "%s: chdir(%s): %s", __func__, dir, strerror(ERRNO)); + } else if (dup2(fdin, 0) == -1) { + mg_cry(conn, "%s: dup2(%d, 0): %s", __func__, fdin, strerror(ERRNO)); + } else if (dup2(fdout, 1) == -1) { + mg_cry(conn, "%s: dup2(%d, 1): %s", __func__, fdout, strerror(ERRNO)); + } else { + /* Not redirecting stderr to stdout, to avoid output being littered + with the error messages. */ + (void) close(fdin); + (void) close(fdout); + + /* After exec, all signal handlers are restored to their default + values, with one exception of SIGCHLD. According to + POSIX.1-2001 and Linux's implementation, SIGCHLD's handler will + leave unchanged after exec if it was set to be ignored. Restore + it to default action. */ + signal(SIGCHLD, SIG_DFL); + + interp = conn->ctx->config[CGI_INTERPRETER]; + if (interp == NULL) { + (void) execle(prog, prog, NULL, envp); + mg_cry(conn, "%s: execle(%s): %s", __func__, prog, strerror(ERRNO)); + } else { + (void) execle(interp, interp, prog, NULL, envp); + mg_cry(conn, "%s: execle(%s %s): %s", __func__, interp, prog, + strerror(ERRNO)); + } + } + exit(EXIT_FAILURE); + } + + return pid; +} +#endif /* !NO_CGI */ + +static int set_non_blocking_mode(SOCKET sock) +{ + int flags; + + flags = fcntl(sock, F_GETFL, 0); + (void) fcntl(sock, F_SETFL, flags | O_NONBLOCK); + + return 0; +} +#endif /* _WIN32 */ + +/* Write data to the IO channel - opened file descriptor, socket or SSL + descriptor. Return number of bytes written. */ +static int64_t push(FILE *fp, SOCKET sock, SSL *ssl, const char *buf, + int64_t len) +{ + int64_t sent; + int n, k; + + (void) ssl; /* Get rid of warning */ + sent = 0; + while (sent < len) { + + /* How many bytes we send in this iteration */ + k = len - sent > INT_MAX ? INT_MAX : (int) (len - sent); + +#ifndef NO_SSL + if (ssl != NULL) { + n = SSL_write(ssl, buf + sent, k); + } else +#endif + if (fp != NULL) { + n = (int) fwrite(buf + sent, 1, (size_t) k, fp); + if (ferror(fp)) + n = -1; + } else { + n = send(sock, buf + sent, (size_t) k, MSG_NOSIGNAL); + } + + if (n <= 0) + break; + + sent += n; + } + + return sent; +} + +/* Read from IO channel - opened file descriptor, socket, or SSL descriptor. + Return negative value on error, or number of bytes read on success. */ +static int pull(FILE *fp, struct mg_connection *conn, char *buf, int len) +{ + int nread; + + if (fp != NULL) { + /* Use read() instead of fread(), because if we're reading from the + CGI pipe, fread() may block until IO buffer is filled up. We cannot + afford to block and must pass all read bytes immediately to the + client. */ + nread = read(fileno(fp), buf, (size_t) len); +#ifndef NO_SSL + } else if (conn->ssl != NULL) { + nread = SSL_read(conn->ssl, buf, len); +#endif + } else { + nread = recv(conn->client.sock, buf, (size_t) len, 0); + } + + return conn->ctx->stop_flag ? -1 : nread; +} + +static int pull_all(FILE *fp, struct mg_connection *conn, char *buf, int len) +{ + int n, nread = 0; + + while (len > 0 && conn->ctx->stop_flag == 0) { + n = pull(fp, conn, buf + nread, len); + if (n < 0) { + nread = n; /* Propagate the error */ + break; + } else if (n == 0) { + break; /* No more data to read */ + } else { + conn->consumed_content += n; + nread += n; + len -= n; + } + } + + return nread; +} + +int mg_read(struct mg_connection *conn, void *buf, size_t len) +{ + int64_t n, buffered_len, nread; + const char *body; + + /* If Content-Length is not set for a PUT or POST request, read until socket is closed */ + if (conn->consumed_content == 0 && conn->content_len == -1) { + conn->content_len = INT64_MAX; + conn->must_close = 1; + } + + nread = 0; + if (conn->consumed_content < conn->content_len) { + /* Adjust number of bytes to read. */ + int64_t to_read = conn->content_len - conn->consumed_content; + if (to_read < (int64_t) len) { + len = (size_t) to_read; + } + + /* Return buffered data */ + body = conn->buf + conn->request_len + conn->consumed_content; + buffered_len = (int64_t)(&conn->buf[conn->data_len] - body); + if (buffered_len > 0) { + if (len < (size_t) buffered_len) { + buffered_len = (int64_t) len; + } + memcpy(buf, body, (size_t) buffered_len); + len -= buffered_len; + conn->consumed_content += buffered_len; + nread += buffered_len; + buf = (char *) buf + buffered_len; + } + + /* We have returned all buffered data. Read new data from the remote + socket. */ + n = pull_all(NULL, conn, (char *) buf, (int64_t) len); + nread = n >= 0 ? nread + n : n; + } + return nread; +} + +int mg_write(struct mg_connection *conn, const void *buf, size_t len) +{ + time_t now; + int64_t n, total, allowed; + + if (conn->throttle > 0) { + if ((now = time(NULL)) != conn->last_throttle_time) { + conn->last_throttle_time = now; + conn->last_throttle_bytes = 0; + } + allowed = conn->throttle - conn->last_throttle_bytes; + if (allowed > (int64_t) len) { + allowed = len; + } + if ((total = push(NULL, conn->client.sock, conn->ssl, (const char *) buf, + (int64_t) allowed)) == allowed) { + buf = (char *) buf + total; + conn->last_throttle_bytes += total; + while (total < (int64_t) len && conn->ctx->stop_flag == 0) { + allowed = conn->throttle > (int64_t) len - total ? + (int64_t) len - total : conn->throttle; + if ((n = push(NULL, conn->client.sock, conn->ssl, (const char *) buf, + (int64_t) allowed)) != allowed) { + break; + } + sleep(1); + conn->last_throttle_bytes = allowed; + conn->last_throttle_time = time(NULL); + buf = (char *) buf + n; + total += n; + } + } + } else { + total = push(NULL, conn->client.sock, conn->ssl, (const char *) buf, + (int64_t) len); + } + return (int) total; +} + +/* Alternative alloc_vprintf() for non-compliant C runtimes */ +static int alloc_vprintf2(char **buf, const char *fmt, va_list ap) +{ + va_list ap_copy; + int size = MG_BUF_LEN; + int len = -1; + + *buf = NULL; + while (len == -1) { + if (*buf) mg_free(*buf); + *buf = (char *)mg_malloc(size *= 4); + if (!*buf) break; + va_copy(ap_copy, ap); + len = vsnprintf(*buf, size, fmt, ap_copy); + va_end(ap_copy); + } + + return len; +} + +/* Print message to buffer. If buffer is large enough to hold the message, + return buffer. If buffer is to small, allocate large enough buffer on heap, + and return allocated buffer. */ +static int alloc_vprintf(char **buf, size_t size, const char *fmt, va_list ap) +{ + va_list ap_copy; + int len; + + /* Windows is not standard-compliant, and vsnprintf() returns -1 if + buffer is too small. Also, older versions of msvcrt.dll do not have + _vscprintf(). However, if size is 0, vsnprintf() behaves correctly. + Therefore, we make two passes: on first pass, get required message + length. + On second pass, actually print the message. */ + va_copy(ap_copy, ap); + len = vsnprintf(NULL, 0, fmt, ap_copy); + va_end(ap_copy); + + if (len < 0) { + /* C runtime is not standard compliant, vsnprintf() returned -1. + Switch to alternative code path that uses incremental allocations. + */ + va_copy(ap_copy, ap); + len = alloc_vprintf2(buf, fmt, ap); + va_end(ap_copy); + } else if (len > (int) size && + (size = len + 1) > 0 && + (*buf = (char *) mg_malloc(size)) == NULL) { + len = -1; /* Allocation failed, mark failure */ + } else { + va_copy(ap_copy, ap); + IGNORE_UNUSED_RESULT(vsnprintf(*buf, size, fmt, ap_copy)); + va_end(ap_copy); + } + + return len; +} + +int mg_vprintf(struct mg_connection *conn, const char *fmt, va_list ap); + +int mg_vprintf(struct mg_connection *conn, const char *fmt, va_list ap) +{ + char mem[MG_BUF_LEN], *buf = mem; + int len; + + if ((len = alloc_vprintf(&buf, sizeof(mem), fmt, ap)) > 0) { + len = mg_write(conn, buf, (size_t) len); + } + if (buf != mem && buf != NULL) { + mg_free(buf); + } + + return len; +} + +int mg_printf(struct mg_connection *conn, const char *fmt, ...) +{ + va_list ap; + int result; + + va_start(ap, fmt); + result = mg_vprintf(conn, fmt, ap); + va_end(ap); + + return result; +} + +int mg_url_decode(const char *src, int src_len, char *dst, + int dst_len, int is_form_url_encoded) +{ + int i, j, a, b; +#define HEXTOI(x) (isdigit(x) ? x - '0' : x - 'W') + + for (i = j = 0; i < src_len && j < dst_len - 1; i++, j++) { + if (src[i] == '%' && i < src_len - 2 && + isxdigit(* (const unsigned char *) (src + i + 1)) && + isxdigit(* (const unsigned char *) (src + i + 2))) { + a = tolower(* (const unsigned char *) (src + i + 1)); + b = tolower(* (const unsigned char *) (src + i + 2)); + dst[j] = (char) ((HEXTOI(a) << 4) | HEXTOI(b)); + i += 2; + } else if (is_form_url_encoded && src[i] == '+') { + dst[j] = ' '; + } else { + dst[j] = src[i]; + } + } + + dst[j] = '\0'; /* Null-terminate the destination */ + + return i >= src_len ? j : -1; +} + +int mg_get_var(const char *data, size_t data_len, const char *name, + char *dst, size_t dst_len) +{ + return mg_get_var2(data,data_len,name,dst,dst_len,0); +} + +int mg_get_var2(const char *data, size_t data_len, const char *name, + char *dst, size_t dst_len, size_t occurrence) +{ + const char *p, *e, *s; + size_t name_len; + int len; + + if (dst == NULL || dst_len == 0) { + len = -2; + } else if (data == NULL || name == NULL || data_len == 0) { + len = -1; + dst[0] = '\0'; + } else { + name_len = strlen(name); + e = data + data_len; + len = -1; + dst[0] = '\0'; + + /* data is "var1=val1&var2=val2...". Find variable first */ + for (p = data; p + name_len < e; p++) { + if ((p == data || p[-1] == '&') && p[name_len] == '=' && + !mg_strncasecmp(name, p, name_len) && 0 == occurrence--) { + + /* Point p to variable value */ + p += name_len + 1; + + /* Point s to the end of the value */ + s = (const char *) memchr(p, '&', (size_t)(e - p)); + if (s == NULL) { + s = e; + } + assert(s >= p); + + /* Decode variable into destination buffer */ + len = mg_url_decode(p, (int)(s - p), dst, (int)dst_len, 1); + + /* Redirect error code from -1 to -2 (destination buffer too + small). */ + if (len == -1) { + len = -2; + } + break; + } + } + } + + return len; +} + +int mg_get_cookie(const char *cookie_header, const char *var_name, + char *dst, size_t dst_size) +{ + const char *s, *p, *end; + int name_len, len = -1; + + if (dst == NULL || dst_size == 0) { + len = -2; + } else if (var_name == NULL || (s = cookie_header) == NULL) { + len = -1; + dst[0] = '\0'; + } else { + name_len = (int) strlen(var_name); + end = s + strlen(s); + dst[0] = '\0'; + + for (; (s = mg_strcasestr(s, var_name)) != NULL; s += name_len) { + if (s[name_len] == '=') { + s += name_len + 1; + if ((p = strchr(s, ' ')) == NULL) + p = end; + if (p[-1] == ';') + p--; + if (*s == '"' && p[-1] == '"' && p > s + 1) { + s++; + p--; + } + if ((size_t) (p - s) < dst_size) { + len = (int)(p - s); + mg_strlcpy(dst, s, (size_t) len + 1); + } else { + len = -3; + } + break; + } + } + } + return len; +} + +#if defined(USE_WEBSOCKET) || defined(USE_LUA) +static void base64_encode(const unsigned char *src, int src_len, char *dst) +{ + static const char *b64 = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + int i, j, a, b, c; + + for (i = j = 0; i < src_len; i += 3) { + a = src[i]; + b = i + 1 >= src_len ? 0 : src[i + 1]; + c = i + 2 >= src_len ? 0 : src[i + 2]; + + dst[j++] = b64[a >> 2]; + dst[j++] = b64[((a & 3) << 4) | (b >> 4)]; + if (i + 1 < src_len) { + dst[j++] = b64[(b & 15) << 2 | (c >> 6)]; + } + if (i + 2 < src_len) { + dst[j++] = b64[c & 63]; + } + } + while (j % 4 != 0) { + dst[j++] = '='; + } + dst[j++] = '\0'; +} + +static unsigned char b64reverse(char letter) { + if (letter>='A' && letter<='Z') return letter-'A'; + if (letter>='a' && letter<='z') return letter-'a'+26; + if (letter>='0' && letter<='9') return letter-'0'+52; + if (letter=='+') return 62; + if (letter=='/') return 63; + if (letter=='=') return 255; /* normal end */ + return 254; /* error */ +} + +static int base64_decode(const unsigned char *src, int src_len, char *dst, size_t *dst_len) +{ + int i; + unsigned char a, b, c, d; + + *dst_len = 0; + + for (i = 0; i < src_len; i += 4) { + a = b64reverse(src[i]); + if (a>=254) return i; + + b = b64reverse(i + 1 >= src_len ? 0 : src[i + 1]); + if (b>=254) return i+1; + + c = b64reverse(i + 2 >= src_len ? 0 : src[i + 2]); + if (c==254) return i+2; + + d = b64reverse(i + 3 >= src_len ? 0 : src[i + 3]); + if (c==254) return i+3; + + dst[(*dst_len)++] = (a << 2) + (b >> 4); + if (c!=255) { + dst[(*dst_len)++] = (b << 4) + (c >> 2); + if (d!=255) { + dst[(*dst_len)++] = (c << 6) + d; + } + } + } + return -1; +} +#endif + +static void convert_uri_to_file_name(struct mg_connection *conn, char *buf, + size_t buf_len, struct file *filep, + int * is_script_ressource) +{ + struct vec a, b; + const char *rewrite, *uri = conn->request_info.uri, + *root = conn->ctx->config[DOCUMENT_ROOT]; + char *p; + int match_len; + char gz_path[PATH_MAX]; + char const* accept_encoding; + + *is_script_ressource = 0; + +#if defined(USE_WEBSOCKET) + if (is_websocket_request(conn) && conn->ctx->config[WEBSOCKET_ROOT]) { + root = conn->ctx->config[WEBSOCKET_ROOT]; + } +#endif + + /* Using buf_len - 1 because memmove() for PATH_INFO may shift part + of the path one byte on the right. + If document_root is NULL, leave the file empty. */ + mg_snprintf(conn, buf, buf_len - 1, "%s%s", + root == NULL ? "" : root, + root == NULL ? "" : uri); + + rewrite = conn->ctx->config[REWRITE]; + while ((rewrite = next_option(rewrite, &a, &b)) != NULL) { + if ((match_len = match_prefix(a.ptr, (int) a.len, uri)) > 0) { + mg_snprintf(conn, buf, buf_len - 1, "%.*s%s", (int) b.len, b.ptr, + uri + match_len); + break; + } + } + + if (mg_stat(conn, buf, filep)) return; + + /* if we can't find the actual file, look for the file + with the same name but a .gz extension. If we find it, + use that and set the gzipped flag in the file struct + to indicate that the response need to have the content- + encoding: gzip header + we can only do this if the browser declares support */ + if ((accept_encoding = mg_get_header(conn, "Accept-Encoding")) != NULL) { + if (strstr(accept_encoding,"gzip") != NULL) { + snprintf(gz_path, sizeof(gz_path), "%s.gz", buf); + if (mg_stat(conn, gz_path, filep)) { + filep->gzipped = 1; + return; + } + } + } + + /* Support PATH_INFO for CGI scripts. */ + for (p = buf + strlen(buf); p > buf + 1; p--) { + if (*p == '/') { + *p = '\0'; + if ((match_prefix(conn->ctx->config[CGI_EXTENSIONS], + (int)strlen(conn->ctx->config[CGI_EXTENSIONS]), buf) > 0 +#ifdef USE_LUA + || + match_prefix(conn->ctx->config[LUA_SCRIPT_EXTENSIONS], + (int)strlen(conn->ctx->config[LUA_SCRIPT_EXTENSIONS]), buf) > 0 +#endif + ) && mg_stat(conn, buf, filep)) { + /* Shift PATH_INFO block one character right, e.g. + "/x.cgi/foo/bar\x00" => "/x.cgi\x00/foo/bar\x00" + conn->path_info is pointing to the local variable "path" + declared in handle_request(), so PATH_INFO is not valid + after handle_request returns. */ + conn->path_info = p + 1; + memmove(p + 2, p + 1, strlen(p + 1) + 1); /* +1 is for + trailing \0 */ + p[1] = '/'; + *is_script_ressource = 1; + break; + } else { + *p = '/'; + } + } + } +} + +/* Check whether full request is buffered. Return: + -1 if request is malformed + 0 if request is not yet fully buffered + >0 actual request length, including last \r\n\r\n */ +static int get_request_len(const char *buf, int buflen) +{ + const char *s, *e; + int len = 0; + + for (s = buf, e = s + buflen - 1; len <= 0 && s < e; s++) + /* Control characters are not allowed but >=128 is. */ + if (!isprint(* (const unsigned char *) s) && *s != '\r' && + *s != '\n' && * (const unsigned char *) s < 128) { + len = -1; + break; /* [i_a] abort scan as soon as one malformed character is + found; */ + /* don't let subsequent \r\n\r\n win us over anyhow */ + } else if (s[0] == '\n' && s[1] == '\n') { + len = (int) (s - buf) + 2; + } else if (s[0] == '\n' && &s[1] < e && + s[1] == '\r' && s[2] == '\n') { + len = (int) (s - buf) + 3; + } + + return len; +} + +/* Convert month to the month number. Return -1 on error, or month number */ +static int get_month_index(const char *s) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(month_names); i++) + if (!strcmp(s, month_names[i])) + return (int) i; + + return -1; +} + +static int num_leap_years(int year) +{ + return year / 4 - year / 100 + year / 400; +} + +/* Parse UTC date-time string, and return the corresponding time_t value. */ +static time_t parse_date_string(const char *datetime) +{ + static const unsigned short days_before_month[] = { + 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 + }; + char month_str[32]={0}; + int second, minute, hour, day, month, year, leap_days, days; + time_t result = (time_t) 0; + + if ((sscanf(datetime, "%d/%3s/%d %d:%d:%d", + &day, month_str, &year, &hour, &minute, &second) == 6) || + (sscanf(datetime, "%d %3s %d %d:%d:%d", + &day, month_str, &year, &hour, &minute, &second) == 6) || + (sscanf(datetime, "%*3s, %d %3s %d %d:%d:%d", + &day, month_str, &year, &hour, &minute, &second) == 6) || + (sscanf(datetime, "%d-%3s-%d %d:%d:%d", + &day, month_str, &year, &hour, &minute, &second) == 6)) { + + month = get_month_index(month_str); + if ((month >= 0) && (year > 1970)) { + leap_days = num_leap_years(year) - num_leap_years(1970); + year -= 1970; + days = year * 365 + days_before_month[month] + (day - 1) + leap_days; + result = (time_t) days * 24 * 3600 + (time_t) hour * 3600 + + minute * 60 + second; + } + } + + return result; +} + +/* Protect against directory disclosure attack by removing '..', + excessive '/' and '\' characters */ +static void remove_double_dots_and_double_slashes(char *s) +{ + char *p = s; + + while (*s != '\0') { + *p++ = *s++; + if (s[-1] == '/' || s[-1] == '\\') { + /* Skip all following slashes, backslashes and double-dots */ + while (s[0] != '\0') { + if (s[0] == '/' || s[0] == '\\') { + s++; + } else if (s[0] == '.' && s[1] == '.') { + s += 2; + } else { + break; + } + } + } + } + *p = '\0'; +} + +static const struct { + const char *extension; + size_t ext_len; + const char *mime_type; +} builtin_mime_types[] = { + /* IANA registered MIME types (http://www.iana.org/assignments/media-types) + application types */ + {".doc", 4, "application/msword"}, + {".eps", 4, "application/postscript"}, + {".exe", 4, "application/octet-stream"}, + {".js", 3, "application/javascript"}, + {".json", 5, "application/json"}, + {".pdf", 4, "application/pdf"}, + {".ps", 3, "application/postscript"}, + {".rtf", 4, "application/rtf"}, + {".xhtml", 6, "application/xhtml+xml"}, + {".xsl", 4, "application/xml"}, + {".xslt", 5, "application/xml"}, + + /* audio */ + {".mp3", 4, "audio/mpeg"}, + {".oga", 4, "audio/ogg"}, + {".ogg", 4, "audio/ogg"}, + + /* image */ + {".gif", 4, "image/gif"}, + {".ief", 4, "image/ief"}, + {".jpeg", 5, "image/jpeg"}, + {".jpg", 4, "image/jpeg"}, + {".jpm", 4, "image/jpm"}, + {".jpx", 4, "image/jpx"}, + {".png", 4, "image/png"}, + {".svg", 4, "image/svg+xml"}, + {".tif", 4, "image/tiff"}, + {".tiff", 5, "image/tiff"}, + + /* model */ + {".wrl", 4, "model/vrml"}, + + /* text */ + {".css", 4, "text/css"}, + {".csv", 4, "text/csv"}, + {".htm", 4, "text/html"}, + {".html", 5, "text/html"}, + {".sgm", 4, "text/sgml"}, + {".shtm", 5, "text/html"}, + {".shtml", 6, "text/html"}, + {".txt", 4, "text/plain"}, + {".xml", 4, "text/xml"}, + + /* video */ + {".mov", 4, "video/quicktime"}, + {".mp4", 4, "video/mp4"}, + {".mpeg", 5, "video/mpeg"}, + {".mpg", 4, "video/mpeg"}, + {".ogv", 4, "video/ogg"}, + {".qt", 3, "video/quicktime"}, + + /* not registered types + (http://reference.sitepoint.com/html/mime-types-full, + http://www.hansenb.pdx.edu/DMKB/dict/tutorials/mime_typ.php, ..) */ + {".arj", 4, "application/x-arj-compressed"}, + {".gz", 3, "application/x-gunzip"}, + {".rar", 4, "application/x-arj-compressed"}, + {".swf", 4, "application/x-shockwave-flash"}, + {".tar", 4, "application/x-tar"}, + {".tgz", 4, "application/x-tar-gz"}, + {".torrent", 8, "application/x-bittorrent"}, + {".ppt", 4, "application/x-mspowerpoint"}, + {".xls", 4, "application/x-msexcel"}, + {".zip", 4, "application/x-zip-compressed"}, + {".aac", 4, "audio/aac"}, /* http://en.wikipedia.org/wiki/Advanced_Audio_Coding */ + {".aif", 4, "audio/x-aif"}, + {".m3u", 4, "audio/x-mpegurl"}, + {".mid", 4, "audio/x-midi"}, + {".ra", 3, "audio/x-pn-realaudio"}, + {".ram", 4, "audio/x-pn-realaudio"}, + {".wav", 4, "audio/x-wav"}, + {".bmp", 4, "image/bmp"}, + {".ico", 4, "image/x-icon"}, + {".pct", 4, "image/x-pct"}, + {".pict", 5, "image/pict"}, + {".rgb", 4, "image/x-rgb"}, + {".webm", 5, "video/webm"}, /* http://en.wikipedia.org/wiki/WebM */ + {".asf", 4, "video/x-ms-asf"}, + {".avi", 4, "video/x-msvideo"}, + {".m4v", 4, "video/x-m4v"}, + {NULL, 0, NULL} +}; + +const char *mg_get_builtin_mime_type(const char *path) +{ + const char *ext; + size_t i, path_len; + + path_len = strlen(path); + + for (i = 0; builtin_mime_types[i].extension != NULL; i++) { + ext = path + (path_len - builtin_mime_types[i].ext_len); + if (path_len > builtin_mime_types[i].ext_len && + mg_strcasecmp(ext, builtin_mime_types[i].extension) == 0) { + return builtin_mime_types[i].mime_type; + } + } + + return "text/plain"; +} + +/* Look at the "path" extension and figure what mime type it has. + Store mime type in the vector. */ +static void get_mime_type(struct mg_context *ctx, const char *path, + struct vec *vec) +{ + struct vec ext_vec, mime_vec; + const char *list, *ext; + size_t path_len; + + path_len = strlen(path); + + /* Scan user-defined mime types first, in case user wants to + override default mime types. */ + list = ctx->config[EXTRA_MIME_TYPES]; + while ((list = next_option(list, &ext_vec, &mime_vec)) != NULL) { + /* ext now points to the path suffix */ + ext = path + path_len - ext_vec.len; + if (mg_strncasecmp(ext, ext_vec.ptr, ext_vec.len) == 0) { + *vec = mime_vec; + return; + } + } + + vec->ptr = mg_get_builtin_mime_type(path); + vec->len = strlen(vec->ptr); +} + +/* Stringify binary data. Output buffer must be twice as big as input, + because each byte takes 2 bytes in string representation */ +static void bin2str(char *to, const unsigned char *p, size_t len) +{ + static const char *hex = "0123456789abcdef"; + + for (; len--; p++) { + *to++ = hex[p[0] >> 4]; + *to++ = hex[p[0] & 0x0f]; + } + *to = '\0'; +} + +/* Return stringified MD5 hash for list of strings. Buffer must be 33 bytes. */ +char *mg_md5(char buf[33], ...) +{ + md5_byte_t hash[16]; + const char *p; + va_list ap; + md5_state_t ctx; + + md5_init(&ctx); + + va_start(ap, buf); + while ((p = va_arg(ap, const char *)) != NULL) { + md5_append(&ctx, (const md5_byte_t *) p, (int) strlen(p)); + } + va_end(ap); + + md5_finish(&ctx, hash); + bin2str(buf, hash, sizeof(hash)); + return buf; +} + +/* Check the user's password, return 1 if OK */ +static int check_password(const char *method, const char *ha1, const char *uri, + const char *nonce, const char *nc, const char *cnonce, + const char *qop, const char *response) +{ + char ha2[32 + 1], expected_response[32 + 1]; + + /* Some of the parameters may be NULL */ + if (method == NULL || nonce == NULL || nc == NULL || cnonce == NULL || + qop == NULL || response == NULL) { + return 0; + } + + /* NOTE(lsm): due to a bug in MSIE, we do not compare the URI */ + /* TODO(lsm): check for authentication timeout */ + if (/* strcmp(dig->uri, c->ouri) != 0 || */ + strlen(response) != 32 + /* || now - strtoul(dig->nonce, NULL, 10) > 3600 */ + ) { + return 0; + } + + mg_md5(ha2, method, ":", uri, NULL); + mg_md5(expected_response, ha1, ":", nonce, ":", nc, + ":", cnonce, ":", qop, ":", ha2, NULL); + + return mg_strcasecmp(response, expected_response) == 0; +} + +/* Use the global passwords file, if specified by auth_gpass option, + or search for .htpasswd in the requested directory. */ +static void open_auth_file(struct mg_connection *conn, const char *path, + struct file *filep) +{ + char name[PATH_MAX]; + const char *p, *e, *gpass = conn->ctx->config[GLOBAL_PASSWORDS_FILE]; + struct file file = STRUCT_FILE_INITIALIZER; + + if (gpass != NULL) { + /* Use global passwords file */ + if (!mg_fopen(conn, gpass, "r", filep)) { +#ifdef DEBUG + mg_cry(conn, "fopen(%s): %s", gpass, strerror(ERRNO)); +#endif + } + /* Important: using local struct file to test path for is_directory + flag. + If filep is used, mg_stat() makes it appear as if auth file was + opened. */ + } else if (mg_stat(conn, path, &file) && file.is_directory) { + mg_snprintf(conn, name, sizeof(name), "%s%c%s", + path, '/', PASSWORDS_FILE_NAME); + if (!mg_fopen(conn, name, "r", filep)) { +#ifdef DEBUG + mg_cry(conn, "fopen(%s): %s", name, strerror(ERRNO)); +#endif + } + } else { + /* Try to find .htpasswd in requested directory. */ + for (p = path, e = p + strlen(p) - 1; e > p; e--) + if (e[0] == '/') + break; + mg_snprintf(conn, name, sizeof(name), "%.*s%c%s", + (int) (e - p), p, '/', PASSWORDS_FILE_NAME); + if (!mg_fopen(conn, name, "r", filep)) { +#ifdef DEBUG + mg_cry(conn, "fopen(%s): %s", name, strerror(ERRNO)); +#endif + } + } +} + +/* Parsed Authorization header */ +struct ah { + char *user, *uri, *cnonce, *response, *qop, *nc, *nonce; +}; + +/* Return 1 on success. Always initializes the ah structure. */ +static int parse_auth_header(struct mg_connection *conn, char *buf, + size_t buf_size, struct ah *ah) +{ + char *name, *value, *s; + const char *auth_header; + unsigned long nonce; + + (void) memset(ah, 0, sizeof(*ah)); + if ((auth_header = mg_get_header(conn, "Authorization")) == NULL || + mg_strncasecmp(auth_header, "Digest ", 7) != 0) { + return 0; + } + + /* Make modifiable copy of the auth header */ + (void) mg_strlcpy(buf, auth_header + 7, buf_size); + s = buf; + + /* Parse authorization header */ + for (;;) { + /* Gobble initial spaces */ + while (isspace(* (unsigned char *) s)) { + s++; + } + name = skip_quoted(&s, "=", " ", 0); + /* Value is either quote-delimited, or ends at first comma or space. */ + if (s[0] == '\"') { + s++; + value = skip_quoted(&s, "\"", " ", '\\'); + if (s[0] == ',') { + s++; + } + } else { + value = skip_quoted(&s, ", ", " ", 0); /* IE uses commas, FF uses + spaces */ + } + if (*name == '\0') { + break; + } + + if (!strcmp(name, "username")) { + ah->user = value; + } else if (!strcmp(name, "cnonce")) { + ah->cnonce = value; + } else if (!strcmp(name, "response")) { + ah->response = value; + } else if (!strcmp(name, "uri")) { + ah->uri = value; + } else if (!strcmp(name, "qop")) { + ah->qop = value; + } else if (!strcmp(name, "nc")) { + ah->nc = value; + } else if (!strcmp(name, "nonce")) { + ah->nonce = value; + } + } + +#ifndef NO_NONCE_CHECK + /* Convert the nonce from the client to a number and check it. */ + /* Server side nonce check is valuable in all situations but one: if the server restarts frequently, + but the client should not see that, so the server should accept nonces from previous starts. */ + if (ah->nonce == NULL) { + return 0; + } + nonce = strtoul(ah->nonce, &s, 10); + if ((s == NULL) || (*s != 0)) { + return 0; + } + nonce ^= (unsigned long)(conn->ctx); + if (noncectx->start_time) { + /* nonce is from a previous start of the server and no longer valid (replay attack?) */ + return 0; + } + if (nonce>=conn->ctx->start_time+conn->ctx->nonce_count) { + return 0; + } +#endif + + /* CGI needs it as REMOTE_USER */ + if (ah->user != NULL) { + conn->request_info.remote_user = mg_strdup(ah->user); + } else { + return 0; + } + + return 1; +} + +static char *mg_fgets(char *buf, size_t size, struct file *filep, char **p) +{ + char *eof; + size_t len; + char *memend; + + if (filep->membuf != NULL && *p != NULL) { + memend = (char *) &filep->membuf[filep->size]; + eof = (char *) memchr(*p, '\n', memend - *p); /* Search for \n from p till the end of stream */ + if (eof != NULL) { + eof += 1; /* Include \n */ + } else { + eof = memend; /* Copy remaining data */ + } + len = (size_t) (eof - *p) > size - 1 ? size - 1 : (size_t) (eof - *p); + memcpy(buf, *p, len); + buf[len] = '\0'; + *p += len; + return len ? eof : NULL; + } else if (filep->fp != NULL) { + return fgets(buf, (int)size, filep->fp); + } else { + return NULL; + } +} + +/* Authorize against the opened passwords file. Return 1 if authorized. */ +static int authorize(struct mg_connection *conn, struct file *filep) +{ + struct ah ah; + char line[256], f_user[256] = "", ha1[256] = "", f_domain[256] = "", buf[MG_BUF_LEN], *p; + + if (!parse_auth_header(conn, buf, sizeof(buf), &ah)) { + return 0; + } + + /* Loop over passwords file */ + p = (char *) filep->membuf; + while (mg_fgets(line, sizeof(line), filep, &p) != NULL) { + if (sscanf(line, "%255[^:]:%255[^:]:%255s", f_user, f_domain, ha1) != 3) { + continue; + } + f_user[255]=0; + f_domain[255]=0; + ha1[255]=0; + + if (!strcmp(ah.user, f_user) && + !strcmp(conn->ctx->config[AUTHENTICATION_DOMAIN], f_domain)) + return check_password(conn->request_info.request_method, ha1, ah.uri, + ah.nonce, ah.nc, ah.cnonce, ah.qop, ah.response); + } + + return 0; +} + +/* Return 1 if request is authorised, 0 otherwise. */ +static int check_authorization(struct mg_connection *conn, const char *path) +{ + char fname[PATH_MAX]; + struct vec uri_vec, filename_vec; + const char *list; + struct file file = STRUCT_FILE_INITIALIZER; + int authorized = 1; + + list = conn->ctx->config[PROTECT_URI]; + while ((list = next_option(list, &uri_vec, &filename_vec)) != NULL) { + if (!memcmp(conn->request_info.uri, uri_vec.ptr, uri_vec.len)) { + mg_snprintf(conn, fname, sizeof(fname), "%.*s", + (int) filename_vec.len, filename_vec.ptr); + if (!mg_fopen(conn, fname, "r", &file)) { + mg_cry(conn, "%s: cannot open %s: %s", __func__, fname, strerror(errno)); + } + break; + } + } + + if (!is_file_opened(&file)) { + open_auth_file(conn, path, &file); + } + + if (is_file_opened(&file)) { + authorized = authorize(conn, &file); + mg_fclose(&file); + } + + return authorized; +} + +static void send_authorization_request(struct mg_connection *conn) +{ + char date[64]; + time_t curtime = time(NULL); + unsigned long nonce = (unsigned long)(conn->ctx->start_time); + + (void)pthread_mutex_lock(&conn->ctx->mutex); + nonce += conn->ctx->nonce_count; + ++conn->ctx->nonce_count; + (void)pthread_mutex_unlock(&conn->ctx->mutex); + + nonce ^= (unsigned long)(conn->ctx); + conn->status_code = 401; + conn->must_close = 1; + + gmt_time_string(date, sizeof(date), &curtime); + + mg_printf(conn, + "HTTP/1.1 401 Unauthorized\r\n" + "Date: %s\r\n" + "Connection: %s\r\n" + "Content-Length: 0\r\n" + "WWW-Authenticate: Digest qop=\"auth\", realm=\"%s\", nonce=\"%lu\"\r\n\r\n", + date, suggest_connection_header(conn), + conn->ctx->config[AUTHENTICATION_DOMAIN], + nonce); +} + +static int is_authorized_for_put(struct mg_connection *conn) +{ + struct file file = STRUCT_FILE_INITIALIZER; + const char *passfile = conn->ctx->config[PUT_DELETE_PASSWORDS_FILE]; + int ret = 0; + + if (passfile != NULL && mg_fopen(conn, passfile, "r", &file)) { + ret = authorize(conn, &file); + mg_fclose(&file); + } + + return ret; +} + +int mg_modify_passwords_file(const char *fname, const char *domain, + const char *user, const char *pass) +{ + int found; + char line[512], u[512] = "", d[512] ="", ha1[33], tmp[PATH_MAX+1]; + FILE *fp, *fp2; + + found = 0; + fp = fp2 = NULL; + + /* Regard empty password as no password - remove user record. */ + if (pass != NULL && pass[0] == '\0') { + pass = NULL; + } + + (void) snprintf(tmp, sizeof(tmp) - 1, "%s.tmp", fname); + tmp[sizeof(tmp) - 1] = 0; + + /* Create the file if does not exist */ + if ((fp = fopen(fname, "a+")) != NULL) { + (void) fclose(fp); + } + + /* Open the given file and temporary file */ + if ((fp = fopen(fname, "r")) == NULL) { + return 0; + } else if ((fp2 = fopen(tmp, "w+")) == NULL) { + fclose(fp); + return 0; + } + + /* Copy the stuff to temporary file */ + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "%255[^:]:%255[^:]:%*s", u, d) != 2) { + continue; + } + u[255]=0; + d[255]=0; + + if (!strcmp(u, user) && !strcmp(d, domain)) { + found++; + if (pass != NULL) { + mg_md5(ha1, user, ":", domain, ":", pass, NULL); + fprintf(fp2, "%s:%s:%s\n", user, domain, ha1); + } + } else { + fprintf(fp2, "%s", line); + } + } + + /* If new user, just add it */ + if (!found && pass != NULL) { + mg_md5(ha1, user, ":", domain, ":", pass, NULL); + fprintf(fp2, "%s:%s:%s\n", user, domain, ha1); + } + + /* Close files */ + fclose(fp); + fclose(fp2); + + /* Put the temp file in place of real file */ + IGNORE_UNUSED_RESULT(remove(fname)); + IGNORE_UNUSED_RESULT(rename(tmp, fname)); + + return 1; +} + +static SOCKET conn2(struct mg_context *ctx /* may be null */, const char *host, int port, + int use_ssl, char *ebuf, size_t ebuf_len) +{ + struct sockaddr_in sain; + struct hostent *he; + SOCKET sock = INVALID_SOCKET; + + if (host == NULL) { + snprintf(ebuf, ebuf_len, "%s", "NULL host"); + } else if (use_ssl && SSLv23_client_method == NULL) { + snprintf(ebuf, ebuf_len, "%s", "SSL is not initialized"); + /* TODO(lsm): use something threadsafe instead of gethostbyname() */ + } else if ((he = gethostbyname(host)) == NULL) { + snprintf(ebuf, ebuf_len, "gethostbyname(%s): %s", host, strerror(ERRNO)); + } else if ((sock = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) { + snprintf(ebuf, ebuf_len, "socket(): %s", strerror(ERRNO)); + } else { + set_close_on_exec(sock, fc(ctx)); + memset(&sain, '\0', sizeof(sain)); + sain.sin_family = AF_INET; + sain.sin_port = htons((uint16_t) port); + sain.sin_addr = * (struct in_addr *) he->h_addr_list[0]; + if (connect(sock, (struct sockaddr *) &sain, sizeof(sain)) != 0) { + snprintf(ebuf, ebuf_len, "connect(%s:%d): %s", + host, port, strerror(ERRNO)); + closesocket(sock); + sock = INVALID_SOCKET; + } + } + return sock; +} + +int mg_url_encode(const char *src, char *dst, size_t dst_len) +{ + static const char *dont_escape = "._-$,;~()"; + static const char *hex = "0123456789abcdef"; + char *pos = dst; + const char *end = dst + dst_len - 1; + + for (; *src != '\0' && pos < end; src++, pos++) { + if (isalnum(*(const unsigned char *) src) || + strchr(dont_escape, * (const unsigned char *) src) != NULL) { + *pos = *src; + } else if (pos + 2 < end) { + pos[0] = '%'; + pos[1] = hex[(* (const unsigned char *) src) >> 4]; + pos[2] = hex[(* (const unsigned char *) src) & 0xf]; + pos += 2; + } else { + return -1; + } + } + + *pos = '\0'; + return (*src == '\0') ? (int)(pos - dst) : -1; +} + +static void print_dir_entry(struct de *de) +{ + char size[64], mod[64], href[PATH_MAX]; + struct tm *tm; + + if (de->file.is_directory) { + mg_snprintf(de->conn, size, sizeof(size), "%s", "[DIRECTORY]"); + } else { + /* We use (signed) cast below because MSVC 6 compiler cannot + convert unsigned __int64 to double. Sigh. */ + if (de->file.size < 1024) { + mg_snprintf(de->conn, size, sizeof(size), "%d", (int) de->file.size); + } else if (de->file.size < 0x100000) { + mg_snprintf(de->conn, size, sizeof(size), + "%.1fk", (double) de->file.size / 1024.0); + } else if (de->file.size < 0x40000000) { + mg_snprintf(de->conn, size, sizeof(size), + "%.1fM", (double) de->file.size / 1048576); + } else { + mg_snprintf(de->conn, size, sizeof(size), + "%.1fG", (double) de->file.size / 1073741824); + } + } + tm = localtime(&de->file.modification_time); + if (tm != NULL) { + strftime(mod, sizeof(mod), "%d-%b-%Y %H:%M", tm); + } else { + mg_strlcpy(mod, "01-Jan-1970 00:00", sizeof(mod)); + mod[sizeof(mod) - 1] = '\0'; + } + mg_url_encode(de->file_name, href, sizeof(href)); + de->conn->num_bytes_sent += mg_printf(de->conn, + "%s%s" + " %s  %s\n", + de->conn->request_info.uri, href, de->file.is_directory ? "/" : "", + de->file_name, de->file.is_directory ? "/" : "", mod, size); +} + +/* This function is called from send_directory() and used for + sorting directory entries by size, or name, or modification time. + On windows, __cdecl specification is needed in case if project is built + with __stdcall convention. qsort always requires __cdels callback. */ +static int WINCDECL compare_dir_entries(const void *p1, const void *p2) +{ + const struct de *a = (const struct de *) p1, *b = (const struct de *) p2; + const char *query_string = a->conn->request_info.query_string; + int cmp_result = 0; + + if (query_string == NULL) { + query_string = "na"; + } + + if (a->file.is_directory && !b->file.is_directory) { + return -1; /* Always put directories on top */ + } else if (!a->file.is_directory && b->file.is_directory) { + return 1; /* Always put directories on top */ + } else if (*query_string == 'n') { + cmp_result = strcmp(a->file_name, b->file_name); + } else if (*query_string == 's') { + cmp_result = a->file.size == b->file.size ? 0 : + a->file.size > b->file.size ? 1 : -1; + } else if (*query_string == 'd') { + cmp_result = a->file.modification_time == b->file.modification_time ? 0 : + a->file.modification_time > b->file.modification_time ? 1 : -1; + } + + return query_string[1] == 'd' ? -cmp_result : cmp_result; +} + +static int must_hide_file(struct mg_connection *conn, const char *path) +{ + const char *pw_pattern = "**" PASSWORDS_FILE_NAME "$"; + const char *pattern = conn->ctx->config[HIDE_FILES]; + return match_prefix(pw_pattern, (int)strlen(pw_pattern), path) > 0 || + (pattern != NULL && match_prefix(pattern, (int)strlen(pattern), path) > 0); +} + +static int scan_directory(struct mg_connection *conn, const char *dir, + void *data, void (*cb)(struct de *, void *)) +{ + char path[PATH_MAX]; + struct dirent *dp; + DIR *dirp; + struct de de; + + if ((dirp = opendir(dir)) == NULL) { + return 0; + } else { + de.conn = conn; + + while ((dp = readdir(dirp)) != NULL) { + /* Do not show current dir and hidden files */ + if (!strcmp(dp->d_name, ".") || + !strcmp(dp->d_name, "..") || + must_hide_file(conn, dp->d_name)) { + continue; + } + + mg_snprintf(conn, path, sizeof(path), "%s%c%s", dir, '/', dp->d_name); + + /* If we don't memset stat structure to zero, mtime will have + garbage and strftime() will segfault later on in + print_dir_entry(). memset is required only if mg_stat() + fails. For more details, see + http://code.google.com/p/mongoose/issues/detail?id=79 */ + memset(&de.file, 0, sizeof(de.file)); + if (!mg_stat(conn, path, &de.file)) { + mg_cry(conn, "%s: mg_stat(%s) failed: %s", + __func__, path, strerror(ERRNO)); + } + + de.file_name = dp->d_name; + cb(&de, data); + } + (void) closedir(dirp); + } + return 1; +} + +static int remove_directory(struct mg_connection *conn, const char *dir) +{ + char path[PATH_MAX]; + struct dirent *dp; + DIR *dirp; + struct de de; + + if ((dirp = opendir(dir)) == NULL) { + return 0; + } else { + de.conn = conn; + + while ((dp = readdir(dirp)) != NULL) { + /* Do not show current dir (but show hidden files as they will + also be removed) */ + if (!strcmp(dp->d_name, ".") || + !strcmp(dp->d_name, "..")) { + continue; + } + + mg_snprintf(conn, path, sizeof(path), "%s%c%s", dir, '/', dp->d_name); + + /* If we don't memset stat structure to zero, mtime will have + garbage and strftime() will segfault later on in + print_dir_entry(). memset is required only if mg_stat() + fails. For more details, see + http://code.google.com/p/mongoose/issues/detail?id=79 */ + memset(&de.file, 0, sizeof(de.file)); + if (!mg_stat(conn, path, &de.file)) { + mg_cry(conn, "%s: mg_stat(%s) failed: %s", + __func__, path, strerror(ERRNO)); + } + if(de.file.modification_time) { + if(de.file.is_directory) { + remove_directory(conn, path); + } else { + mg_remove(path); + } + } + + } + (void) closedir(dirp); + + IGNORE_UNUSED_RESULT(rmdir(dir)); + } + + return 1; +} + +struct dir_scan_data { + struct de *entries; + int num_entries; + int arr_size; +}; + +/* Behaves like realloc(), but frees original pointer on failure */ +static void *realloc2(void *ptr, size_t size) +{ + void *new_ptr = mg_realloc(ptr, size); + if (new_ptr == NULL) { + mg_free(ptr); + } + return new_ptr; +} + +static void dir_scan_callback(struct de *de, void *data) +{ + struct dir_scan_data *dsd = (struct dir_scan_data *) data; + + if (dsd->entries == NULL || dsd->num_entries >= dsd->arr_size) { + dsd->arr_size *= 2; + dsd->entries = (struct de *) realloc2(dsd->entries, dsd->arr_size * + sizeof(dsd->entries[0])); + } + if (dsd->entries == NULL) { + /* TODO(lsm): propagate an error to the caller */ + dsd->num_entries = 0; + } else { + dsd->entries[dsd->num_entries].file_name = mg_strdup(de->file_name); + dsd->entries[dsd->num_entries].file = de->file; + dsd->entries[dsd->num_entries].conn = de->conn; + dsd->num_entries++; + } +} + +static void handle_directory_request(struct mg_connection *conn, + const char *dir) +{ + int i, sort_direction; + struct dir_scan_data data = { NULL, 0, 128 }; + char date[64]; + time_t curtime = time(NULL); + + if (!scan_directory(conn, dir, &data, dir_scan_callback)) { + send_http_error(conn, 500, "Cannot open directory", + "Error: opendir(%s): %s", dir, strerror(ERRNO)); + return; + } + + gmt_time_string(date, sizeof(date), &curtime); + + sort_direction = conn->request_info.query_string != NULL && + conn->request_info.query_string[1] == 'd' ? 'a' : 'd'; + + conn->must_close = 1; + mg_printf(conn, "HTTP/1.1 200 OK\r\n" + "Date: %s\r\n" + "Connection: close\r\n" + "Content-Type: text/html; charset=utf-8\r\n\r\n", + date); + + conn->num_bytes_sent += mg_printf(conn, + "Index of %s" + "" + "

Index of %s

"
+                                      ""
+                                      ""
+                                      ""
+                                      "",
+                                      conn->request_info.uri, conn->request_info.uri,
+                                      sort_direction, sort_direction, sort_direction);
+
+    /* Print first entry - link to a parent directory */
+    conn->num_bytes_sent += mg_printf(conn,
+                                      ""
+                                      "\n",
+                                      conn->request_info.uri, "..", "Parent directory", "-", "-");
+
+    /* Sort and print directory entries */
+    if (data.entries != NULL) {
+        qsort(data.entries, (size_t) data.num_entries,
+              sizeof(data.entries[0]), compare_dir_entries);
+        for (i = 0; i < data.num_entries; i++) {
+            print_dir_entry(&data.entries[i]);
+            mg_free(data.entries[i].file_name);
+        }
+        mg_free(data.entries);
+    }
+
+    conn->num_bytes_sent += mg_printf(conn, "%s", "
NameModifiedSize
%s %s  %s
");
+    conn->status_code = 200;
+}
+
+/* Send len bytes from the opened file to the client. */
+static void send_file_data(struct mg_connection *conn, struct file *filep,
+                           int64_t offset, int64_t len)
+{
+    char buf[MG_BUF_LEN];
+    int to_read, num_read, num_written;
+
+    /* Sanity check the offset */
+    offset = offset < 0 ? 0 : offset > filep->size ? filep->size : offset;
+
+    if (len > 0 && filep->membuf != NULL && filep->size > 0) {
+        if (len > filep->size - offset) {
+            len = filep->size - offset;
+        }
+        mg_write(conn, filep->membuf + offset, (size_t) len);
+    } else if (len > 0 && filep->fp != NULL) {
+        if (offset > 0 && fseeko(filep->fp, offset, SEEK_SET) != 0) {
+            mg_cry(conn, "%s: fseeko() failed: %s",
+                   __func__, strerror(ERRNO));
+        }
+        while (len > 0) {
+            /* Calculate how much to read from the file in the buffer */
+            to_read = sizeof(buf);
+            if ((int64_t) to_read > len) {
+                to_read = (int) len;
+            }
+
+            /* Read from file, exit the loop on error */
+            if ((num_read = (int) fread(buf, 1, (size_t) to_read, filep->fp)) <= 0) {
+                break;
+            }
+
+            /* Send read bytes to the client, exit the loop on error */
+            if ((num_written = mg_write(conn, buf, (size_t) num_read)) != num_read) {
+                break;
+            }
+
+            /* Both read and were successful, adjust counters */
+            conn->num_bytes_sent += num_written;
+            len -= num_written;
+        }
+    }
+}
+
+static int parse_range_header(const char *header, int64_t *a, int64_t *b)
+{
+    return sscanf(header, "bytes=%" INT64_FMT "-%" INT64_FMT, a, b);
+}
+
+static void construct_etag(char *buf, size_t buf_len,
+                           const struct file *filep)
+{
+    snprintf(buf, buf_len, "\"%lx.%" INT64_FMT "\"",
+             (unsigned long) filep->modification_time, filep->size);
+}
+
+static void fclose_on_exec(struct file *filep, struct mg_connection *conn)
+{
+    if (filep != NULL && filep->fp != NULL) {
+#ifdef _WIN32
+        (void) conn; /* Unused. */
+#else
+        if (fcntl(fileno(filep->fp), F_SETFD, FD_CLOEXEC) != 0) {
+            mg_cry(conn, "%s: fcntl(F_SETFD FD_CLOEXEC) failed: %s",
+                   __func__, strerror(ERRNO));
+        }
+#endif
+    }
+}
+
+static void handle_static_file_request(struct mg_connection *conn, const char *path, struct file *filep)
+{
+    char date[64], lm[64], etag[64], range[64];
+    const char *msg = "OK", *hdr;
+    time_t curtime = time(NULL);
+    int64_t cl, r1, r2;
+    struct vec mime_vec;
+    int n;
+    char gz_path[PATH_MAX];
+    const char *encoding = "";
+    const char *cors1, *cors2, *cors3;
+
+    get_mime_type(conn->ctx, path, &mime_vec);
+    cl = filep->size;
+    conn->status_code = 200;
+    range[0] = '\0';
+
+    /* if this file is in fact a pre-gzipped file, rewrite its filename
+       it's important to rewrite the filename after resolving
+       the mime type from it, to preserve the actual file's type */
+    if (filep->gzipped) {
+        snprintf(gz_path, sizeof(gz_path), "%s.gz", path);
+        path = gz_path;
+        encoding = "Content-Encoding: gzip\r\n";
+    }
+
+    if (!mg_fopen(conn, path, "rb", filep)) {
+        send_http_error(conn, 500, http_500_error,
+                        "fopen(%s): %s", path, strerror(ERRNO));
+        return;
+    }
+
+    fclose_on_exec(filep, conn);
+
+    /* If Range: header specified, act accordingly */
+    r1 = r2 = 0;
+    hdr = mg_get_header(conn, "Range");
+    if (hdr != NULL && (n = parse_range_header(hdr, &r1, &r2)) > 0 &&
+        r1 >= 0 && r2 >= 0) {
+        /* actually, range requests don't play well with a pre-gzipped
+           file (since the range is specified in the uncompressed space) */
+        if (filep->gzipped) {
+            send_http_error(conn, 501, "Not Implemented", "range requests in gzipped files are not supported");
+            mg_fclose(filep);
+            return;
+        }
+        conn->status_code = 206;
+        cl = n == 2 ? (r2 > cl ? cl : r2) - r1 + 1: cl - r1;
+        mg_snprintf(conn, range, sizeof(range),
+                    "Content-Range: bytes "
+                    "%" INT64_FMT "-%"
+                    INT64_FMT "/%" INT64_FMT "\r\n",
+                    r1, r1 + cl - 1, filep->size);
+        msg = "Partial Content";
+    }
+
+    hdr = mg_get_header(conn, "Origin");
+    if (hdr) {
+        /* Cross-origin resource sharing (CORS), see http://www.html5rocks.com/en/tutorials/cors/,
+           http://www.html5rocks.com/static/images/cors_server_flowchart.png - preflight is not supported for files. */
+        cors1 = "Access-Control-Allow-Origin: ";
+        cors2 = conn->ctx->config[ACCESS_CONTROL_ALLOW_ORIGIN];
+        cors3 = "\r\n";
+    } else {
+        cors1 = cors2 = cors3 = "";
+    }
+
+    /* Prepare Etag, Date, Last-Modified headers. Must be in UTC, according to
+       http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3 */
+    gmt_time_string(date, sizeof(date), &curtime);
+    gmt_time_string(lm, sizeof(lm), &filep->modification_time);
+    construct_etag(etag, sizeof(etag), filep);
+
+    (void) mg_printf(conn,
+                     "HTTP/1.1 %d %s\r\n"
+                     "%s%s%s"
+                     "Date: %s\r\n"
+                     "Last-Modified: %s\r\n"
+                     "Etag: %s\r\n"
+                     "Content-Type: %.*s\r\n"
+                     "Content-Length: %" INT64_FMT "\r\n"
+                     "Connection: %s\r\n"
+                     "Accept-Ranges: bytes\r\n"
+                     "%s%s\r\n",
+                     conn->status_code, msg,
+                     cors1, cors2, cors3,
+                     date, lm, etag, (int) mime_vec.len,
+                     mime_vec.ptr, cl, suggest_connection_header(conn), range, encoding);
+
+    if (strcmp(conn->request_info.request_method, "HEAD") != 0) {
+        send_file_data(conn, filep, r1, cl);
+    }
+    mg_fclose(filep);
+}
+
+void mg_send_file(struct mg_connection *conn, const char *path)
+{
+    struct file file = STRUCT_FILE_INITIALIZER;
+    if (mg_stat(conn, path, &file)) {
+        handle_static_file_request(conn, path, &file);
+    } else {
+        send_http_error(conn, 404, "Not Found", "%s", "File not found");
+    }
+}
+
+
+/* Parse HTTP headers from the given buffer, advance buffer to the point
+   where parsing stopped. */
+static void parse_http_headers(char **buf, struct mg_request_info *ri)
+{
+    int i;
+
+    for (i = 0; i < (int) ARRAY_SIZE(ri->http_headers); i++) {
+        ri->http_headers[i].name = skip_quoted(buf, ":", " ", 0);
+        ri->http_headers[i].value = skip(buf, "\r\n");
+        if (ri->http_headers[i].name[0] == '\0')
+            break;
+        ri->num_headers = i + 1;
+    }
+}
+
+static int is_valid_http_method(const char *method)
+{
+    return !strcmp(method, "GET") || !strcmp(method, "POST") ||
+           !strcmp(method, "HEAD") || !strcmp(method, "CONNECT") ||
+           !strcmp(method, "PUT") || !strcmp(method, "DELETE") ||
+           !strcmp(method, "OPTIONS") || !strcmp(method, "PROPFIND")
+           || !strcmp(method, "MKCOL")
+           ;
+}
+
+/* Parse HTTP request, fill in mg_request_info structure.
+   This function modifies the buffer by NUL-terminating
+   HTTP request components, header names and header values. */
+static int parse_http_message(char *buf, int len, struct mg_request_info *ri)
+{
+    int is_request, request_length = get_request_len(buf, len);
+    if (request_length > 0) {
+        /* Reset attributes. DO NOT TOUCH is_ssl, remote_ip, remote_port */
+        ri->remote_user = ri->request_method = ri->uri = ri->http_version = NULL;
+        ri->num_headers = 0;
+
+        buf[request_length - 1] = '\0';
+
+        /* RFC says that all initial whitespaces should be ingored */
+        while (*buf != '\0' && isspace(* (unsigned char *) buf)) {
+            buf++;
+        }
+        ri->request_method = skip(&buf, " ");
+        ri->uri = skip(&buf, " ");
+        ri->http_version = skip(&buf, "\r\n");
+
+        /* HTTP message could be either HTTP request or HTTP response, e.g.
+           "GET / HTTP/1.0 ...." or  "HTTP/1.0 200 OK ..." */
+        is_request = is_valid_http_method(ri->request_method);
+        if ((is_request && memcmp(ri->http_version, "HTTP/", 5) != 0) ||
+            (!is_request && memcmp(ri->request_method, "HTTP/", 5) != 0)) {
+            request_length = -1;
+        } else {
+            if (is_request) {
+                ri->http_version += 5;
+            }
+            parse_http_headers(&buf, ri);
+        }
+    }
+    return request_length;
+}
+
+/* Keep reading the input (either opened file descriptor fd, or socket sock,
+   or SSL descriptor ssl) into buffer buf, until \r\n\r\n appears in the
+   buffer (which marks the end of HTTP request). Buffer buf may already
+   have some data. The length of the data is stored in nread.
+   Upon every read operation, increase nread by the number of bytes read. */
+static int read_request(FILE *fp, struct mg_connection *conn,
+                        char *buf, int bufsiz, int *nread)
+{
+    int request_len, n = 0;
+
+    request_len = get_request_len(buf, *nread);
+    while (conn->ctx->stop_flag == 0 &&
+           *nread < bufsiz && request_len == 0 &&
+           (n = pull(fp, conn, buf + *nread, bufsiz - *nread)) > 0) {
+        *nread += n;
+        assert(*nread <= bufsiz);
+        request_len = get_request_len(buf, *nread);
+    }
+
+    return request_len <= 0 && n <= 0 ? -1 : request_len;
+}
+
+/* For given directory path, substitute it to valid index file.
+   Return 1 if index file has been found, 0 if not found.
+   If the file is found, it's stats is returned in stp. */
+static int substitute_index_file(struct mg_connection *conn, char *path,
+                                 size_t path_len, struct file *filep)
+{
+    const char *list = conn->ctx->config[INDEX_FILES];
+    struct file file = STRUCT_FILE_INITIALIZER;
+    struct vec filename_vec;
+    size_t n = strlen(path);
+    int found = 0;
+
+    /* The 'path' given to us points to the directory. Remove all trailing
+       directory separator characters from the end of the path, and
+       then append single directory separator character. */
+    while (n > 0 && path[n - 1] == '/') {
+        n--;
+    }
+    path[n] = '/';
+
+    /* Traverse index files list. For each entry, append it to the given
+       path and see if the file exists. If it exists, break the loop */
+    while ((list = next_option(list, &filename_vec, NULL)) != NULL) {
+
+        /* Ignore too long entries that may overflow path buffer */
+        if (filename_vec.len > path_len - (n + 2))
+            continue;
+
+        /* Prepare full path to the index file */
+        mg_strlcpy(path + n + 1, filename_vec.ptr, filename_vec.len + 1);
+
+        /* Does it exist? */
+        if (mg_stat(conn, path, &file)) {
+            /* Yes it does, break the loop */
+            *filep = file;
+            found = 1;
+            break;
+        }
+    }
+
+    /* If no index file exists, restore directory path */
+    if (!found) {
+        path[n] = '\0';
+    }
+
+    return found;
+}
+
+/* Return True if we should reply 304 Not Modified. */
+static int is_not_modified(const struct mg_connection *conn,
+                           const struct file *filep)
+{
+    char etag[64];
+    const char *ims = mg_get_header(conn, "If-Modified-Since");
+    const char *inm = mg_get_header(conn, "If-None-Match");
+    construct_etag(etag, sizeof(etag), filep);
+    return (inm != NULL && !mg_strcasecmp(etag, inm)) ||
+           (ims != NULL && filep->modification_time <= parse_date_string(ims));
+}
+
+static int forward_body_data(struct mg_connection *conn, FILE *fp,
+                             SOCKET sock, SSL *ssl)
+{
+    const char *expect, *body;
+    char buf[MG_BUF_LEN];
+    int to_read, nread, buffered_len, success = 0;
+
+    expect = mg_get_header(conn, "Expect");
+    assert(fp != NULL);
+
+    if (conn->content_len == -1) {
+        send_http_error(conn, 411, "Length Required", "%s", "");
+    } else if (expect != NULL && mg_strcasecmp(expect, "100-continue")) {
+        send_http_error(conn, 417, "Expectation Failed", "%s", "");
+    } else {
+        if (expect != NULL) {
+            (void) mg_printf(conn, "%s", "HTTP/1.1 100 Continue\r\n\r\n");
+        }
+
+        body = conn->buf + conn->request_len + conn->consumed_content;
+        buffered_len = (int)(&conn->buf[conn->data_len] - body);
+        assert(buffered_len >= 0);
+        assert(conn->consumed_content == 0);
+
+        if (buffered_len > 0) {
+            if ((int64_t) buffered_len > conn->content_len) {
+                buffered_len = (int) conn->content_len;
+            }
+            push(fp, sock, ssl, body, (int64_t) buffered_len);
+            conn->consumed_content += buffered_len;
+        }
+
+        nread = 0;
+        while (conn->consumed_content < conn->content_len) {
+            to_read = sizeof(buf);
+            if ((int64_t) to_read > conn->content_len - conn->consumed_content) {
+                to_read = (int) (conn->content_len - conn->consumed_content);
+            }
+            nread = pull(NULL, conn, buf, to_read);
+            if (nread <= 0 || push(fp, sock, ssl, buf, nread) != nread) {
+                break;
+            }
+            conn->consumed_content += nread;
+        }
+
+        if (conn->consumed_content == conn->content_len) {
+            success = nread >= 0;
+        }
+
+        /* Each error code path in this function must send an error */
+        if (!success) {
+            send_http_error(conn, 577, http_500_error, "%s", "");
+        }
+    }
+
+    return success;
+}
+
+#if !defined(NO_CGI)
+/* This structure helps to create an environment for the spawned CGI program.
+   Environment is an array of "VARIABLE=VALUE\0" ASCIIZ strings,
+   last element must be NULL.
+   However, on Windows there is a requirement that all these VARIABLE=VALUE\0
+   strings must reside in a contiguous buffer. The end of the buffer is
+   marked by two '\0' characters.
+   We satisfy both worlds: we create an envp array (which is vars), all
+   entries are actually pointers inside buf. */
+struct cgi_env_block {
+    struct mg_connection *conn;
+    char buf[CGI_ENVIRONMENT_SIZE]; /* Environment buffer */
+    int len; /* Space taken */
+    char *vars[MAX_CGI_ENVIR_VARS]; /* char **envp */
+    int nvars; /* Number of variables */
+};
+
+static char *addenv(struct cgi_env_block *block,
+                    PRINTF_FORMAT_STRING(const char *fmt), ...)
+PRINTF_ARGS(2, 3);
+
+/* Append VARIABLE=VALUE\0 string to the buffer, and add a respective
+   pointer into the vars array. */
+static char *addenv(struct cgi_env_block *block, const char *fmt, ...)
+{
+    int n, space;
+    char *added;
+    va_list ap;
+
+    /* Calculate how much space is left in the buffer */
+    space = sizeof(block->buf) - block->len - 2;
+    assert(space >= 0);
+
+    /* Make a pointer to the free space int the buffer */
+    added = block->buf + block->len;
+
+    /* Copy VARIABLE=VALUE\0 string into the free space */
+    va_start(ap, fmt);
+    n = mg_vsnprintf(block->conn, added, (size_t) space, fmt, ap);
+    va_end(ap);
+
+    /* Make sure we do not overflow buffer and the envp array */
+    if (n > 0 && n + 1 < space &&
+        block->nvars < (int) ARRAY_SIZE(block->vars) - 2) {
+        /* Append a pointer to the added string into the envp array */
+        block->vars[block->nvars++] = added;
+        /* Bump up used length counter. Include \0 terminator */
+        block->len += n + 1;
+    } else {
+        mg_cry(block->conn, "%s: CGI env buffer truncated for [%s]", __func__, fmt);
+    }
+
+    return added;
+}
+
+static void prepare_cgi_environment(struct mg_connection *conn,
+                                    const char *prog,
+                                    struct cgi_env_block *blk)
+{
+    const char *s, *slash;
+    struct vec var_vec;
+    char *p, src_addr[IP_ADDR_STR_LEN];
+    int  i;
+
+    blk->len = blk->nvars = 0;
+    blk->conn = conn;
+    sockaddr_to_string(src_addr, sizeof(src_addr), &conn->client.rsa);
+
+    addenv(blk, "SERVER_NAME=%s", conn->ctx->config[AUTHENTICATION_DOMAIN]);
+    addenv(blk, "SERVER_ROOT=%s", conn->ctx->config[DOCUMENT_ROOT]);
+    addenv(blk, "DOCUMENT_ROOT=%s", conn->ctx->config[DOCUMENT_ROOT]);
+    addenv(blk, "SERVER_SOFTWARE=%s/%s", "Civetweb", mg_version());
+
+    /* Prepare the environment block */
+    addenv(blk, "%s", "GATEWAY_INTERFACE=CGI/1.1");
+    addenv(blk, "%s", "SERVER_PROTOCOL=HTTP/1.1");
+    addenv(blk, "%s", "REDIRECT_STATUS=200"); /* For PHP */
+
+    /* TODO(lsm): fix this for IPv6 case */
+    addenv(blk, "SERVER_PORT=%d", ntohs(conn->client.lsa.sin.sin_port));
+
+    addenv(blk, "REQUEST_METHOD=%s", conn->request_info.request_method);
+    addenv(blk, "REMOTE_ADDR=%s", src_addr);
+    addenv(blk, "REMOTE_PORT=%d", conn->request_info.remote_port);
+    addenv(blk, "REQUEST_URI=%s", conn->request_info.uri);
+
+    /* SCRIPT_NAME */
+    assert(conn->request_info.uri[0] == '/');
+    slash = strrchr(conn->request_info.uri, '/');
+    if ((s = strrchr(prog, '/')) == NULL)
+        s = prog;
+    addenv(blk, "SCRIPT_NAME=%.*s%s", (int) (slash - conn->request_info.uri),
+           conn->request_info.uri, s);
+
+    addenv(blk, "SCRIPT_FILENAME=%s", prog);
+    addenv(blk, "PATH_TRANSLATED=%s", prog);
+    addenv(blk, "HTTPS=%s", conn->ssl == NULL ? "off" : "on");
+
+    if ((s = mg_get_header(conn, "Content-Type")) != NULL)
+        addenv(blk, "CONTENT_TYPE=%s", s);
+
+    if (conn->request_info.query_string != NULL)
+        addenv(blk, "QUERY_STRING=%s", conn->request_info.query_string);
+
+    if ((s = mg_get_header(conn, "Content-Length")) != NULL)
+        addenv(blk, "CONTENT_LENGTH=%s", s);
+
+    if ((s = getenv("PATH")) != NULL)
+        addenv(blk, "PATH=%s", s);
+
+    if (conn->path_info != NULL) {
+        addenv(blk, "PATH_INFO=%s", conn->path_info);
+    }
+
+    if (conn->status_code > 0) {
+        /* CGI error handler should show the status code */
+        addenv(blk, "STATUS=%d", conn->status_code);
+    }
+
+#if defined(_WIN32)
+    if ((s = getenv("COMSPEC")) != NULL) {
+        addenv(blk, "COMSPEC=%s", s);
+    }
+    if ((s = getenv("SYSTEMROOT")) != NULL) {
+        addenv(blk, "SYSTEMROOT=%s", s);
+    }
+    if ((s = getenv("SystemDrive")) != NULL) {
+        addenv(blk, "SystemDrive=%s", s);
+    }
+    if ((s = getenv("ProgramFiles")) != NULL) {
+        addenv(blk, "ProgramFiles=%s", s);
+    }
+    if ((s = getenv("ProgramFiles(x86)")) != NULL) {
+        addenv(blk, "ProgramFiles(x86)=%s", s);
+    }
+#else
+    if ((s = getenv("LD_LIBRARY_PATH")) != NULL)
+        addenv(blk, "LD_LIBRARY_PATH=%s", s);
+#endif /* _WIN32 */
+
+    if ((s = getenv("PERLLIB")) != NULL)
+        addenv(blk, "PERLLIB=%s", s);
+
+    if (conn->request_info.remote_user != NULL) {
+        addenv(blk, "REMOTE_USER=%s", conn->request_info.remote_user);
+        addenv(blk, "%s", "AUTH_TYPE=Digest");
+    }
+
+    /* Add all headers as HTTP_* variables */
+    for (i = 0; i < conn->request_info.num_headers; i++) {
+        p = addenv(blk, "HTTP_%s=%s",
+                   conn->request_info.http_headers[i].name,
+                   conn->request_info.http_headers[i].value);
+
+        /* Convert variable name into uppercase, and change - to _ */
+        for (; *p != '=' && *p != '\0'; p++) {
+            if (*p == '-')
+                *p = '_';
+            *p = (char) toupper(* (unsigned char *) p);
+        }
+    }
+
+    /* Add user-specified variables */
+    s = conn->ctx->config[CGI_ENVIRONMENT];
+    while ((s = next_option(s, &var_vec, NULL)) != NULL) {
+        addenv(blk, "%.*s", (int) var_vec.len, var_vec.ptr);
+    }
+
+    blk->vars[blk->nvars++] = NULL;
+    blk->buf[blk->len++] = '\0';
+
+    assert(blk->nvars < (int) ARRAY_SIZE(blk->vars));
+    assert(blk->len > 0);
+    assert(blk->len < (int) sizeof(blk->buf));
+}
+
+static void handle_cgi_request(struct mg_connection *conn, const char *prog)
+{
+    char *buf;
+    size_t buflen;
+    int headers_len, data_len, i, fdin[2] = { 0, 0 }, fdout[2] = { 0, 0 };
+    const char *status, *status_text, *connection_state;
+    char *pbuf, dir[PATH_MAX], *p;
+    struct mg_request_info ri;
+    struct cgi_env_block blk;
+    FILE *in = NULL, *out = NULL;
+    struct file fout = STRUCT_FILE_INITIALIZER;
+    pid_t pid = (pid_t) -1;
+
+    buf = NULL;
+    buflen = 16384;
+    prepare_cgi_environment(conn, prog, &blk);
+
+    /* CGI must be executed in its own directory. 'dir' must point to the
+       directory containing executable program, 'p' must point to the
+       executable program name relative to 'dir'. */
+    (void) mg_snprintf(conn, dir, sizeof(dir), "%s", prog);
+    if ((p = strrchr(dir, '/')) != NULL) {
+        *p++ = '\0';
+    } else {
+        dir[0] = '.', dir[1] = '\0';
+        p = (char *) prog;
+    }
+
+    if (pipe(fdin) != 0 || pipe(fdout) != 0) {
+        send_http_error(conn, 500, http_500_error,
+                        "Cannot create CGI pipe: %s", strerror(ERRNO));
+        goto done;
+    }
+
+    pid = spawn_process(conn, p, blk.buf, blk.vars, fdin[0], fdout[1], dir);
+    if (pid == (pid_t) -1) {
+        send_http_error(conn, 500, http_500_error,
+                        "Cannot spawn CGI process [%s]: %s", prog, strerror(ERRNO));
+        goto done;
+    }
+
+    /* Make sure child closes all pipe descriptors. It must dup them to 0,1 */
+    set_close_on_exec(fdin[0], conn);
+    set_close_on_exec(fdin[1], conn);
+    set_close_on_exec(fdout[0], conn);
+    set_close_on_exec(fdout[1], conn);
+
+    /* Parent closes only one side of the pipes.
+       If we don't mark them as closed, close() attempt before
+       return from this function throws an exception on Windows.
+       Windows does not like when closed descriptor is closed again. */
+    (void) close(fdin[0]);
+    (void) close(fdout[1]);
+    fdin[0] = fdout[1] = -1;
+
+
+    if ((in = fdopen(fdin[1], "wb")) == NULL ||
+        (out = fdopen(fdout[0], "rb")) == NULL) {
+        send_http_error(conn, 500, http_500_error,
+                        "fopen: %s", strerror(ERRNO));
+        goto done;
+    }
+
+    setbuf(in, NULL);
+    setbuf(out, NULL);
+    fout.fp = out;
+
+    /* Send POST data to the CGI process if needed */
+    if (!strcmp(conn->request_info.request_method, "POST") &&
+        !forward_body_data(conn, in, INVALID_SOCKET, NULL)) {
+        goto done;
+    }
+
+    /* Close so child gets an EOF. */
+    fclose(in);
+    in = NULL;
+    fdin[1] = -1;
+
+    /* Now read CGI reply into a buffer. We need to set correct
+       status code, thus we need to see all HTTP headers first.
+       Do not send anything back to client, until we buffer in all
+       HTTP headers. */
+    data_len = 0;
+    buf = mg_malloc(buflen);
+    if (buf == NULL) {
+        send_http_error(conn, 500, http_500_error,
+                        "Not enough memory for buffer (%u bytes)",
+                        (unsigned int) buflen);
+        goto done;
+    }
+    headers_len = read_request(out, conn, buf, (int) buflen, &data_len);
+    if (headers_len <= 0) {
+        send_http_error(conn, 500, http_500_error,
+                        "CGI program sent malformed or too big (>%u bytes) "
+                        "HTTP headers: [%.*s]",
+                        (unsigned) buflen, data_len, buf);
+        goto done;
+    }
+    pbuf = buf;
+    buf[headers_len - 1] = '\0';
+    parse_http_headers(&pbuf, &ri);
+
+    /* Make up and send the status line */
+    status_text = "OK";
+    if ((status = get_header(&ri, "Status")) != NULL) {
+        conn->status_code = atoi(status);
+        status_text = status;
+        while (isdigit(* (unsigned char *) status_text) || *status_text == ' ') {
+            status_text++;
+        }
+    } else if (get_header(&ri, "Location") != NULL) {
+        conn->status_code = 302;
+    } else {
+        conn->status_code = 200;
+    }
+    connection_state = get_header(&ri, "Connection");
+    if (connection_state == NULL ||
+        mg_strcasecmp(connection_state, "keep-alive")) {
+        conn->must_close = 1;
+    }
+    (void) mg_printf(conn, "HTTP/1.1 %d %s\r\n", conn->status_code,
+                     status_text);
+
+    /* Send headers */
+    for (i = 0; i < ri.num_headers; i++) {
+        mg_printf(conn, "%s: %s\r\n",
+                  ri.http_headers[i].name, ri.http_headers[i].value);
+    }
+    mg_write(conn, "\r\n", 2);
+
+    /* Send chunk of data that may have been read after the headers */
+    conn->num_bytes_sent += mg_write(conn, buf + headers_len,
+                                     (size_t)(data_len - headers_len));
+
+    /* Read the rest of CGI output and send to the client */
+    send_file_data(conn, &fout, 0, INT64_MAX);
+
+done:
+    if (pid != (pid_t) -1) {
+        kill(pid, SIGKILL);
+#if !defined(_WIN32)
+        {
+            int st;
+            while (waitpid(pid, &st, 0) != -1);  /* clean zombies */
+        }
+#endif
+    }
+    if (fdin[0] != -1) {
+        close(fdin[0]);
+    }
+    if (fdout[1] != -1) {
+        close(fdout[1]);
+    }
+
+    if (in != NULL) {
+        fclose(in);
+    } else if (fdin[1] != -1) {
+        close(fdin[1]);
+    }
+
+    if (out != NULL) {
+        fclose(out);
+    } else if (fdout[0] != -1) {
+        close(fdout[0]);
+    }
+    if (buf != NULL) {
+        mg_free(buf);
+    }
+}
+#endif /* !NO_CGI */
+
+/* For a given PUT path, create all intermediate subdirectories
+   for given path. Return 0 if the path itself is a directory,
+   or -1 on error, 1 if OK. */
+static int put_dir(struct mg_connection *conn, const char *path)
+{
+    char buf[PATH_MAX];
+    const char *s, *p;
+    struct file file = STRUCT_FILE_INITIALIZER;
+    int len, res = 1;
+
+    for (s = p = path + 2; (p = strchr(s, '/')) != NULL; s = ++p) {
+        len = (int)(p - path);
+        if (len >= (int) sizeof(buf)) {
+            res = -1;
+            break;
+        }
+        memcpy(buf, path, len);
+        buf[len] = '\0';
+
+        /* Try to create intermediate directory */
+        DEBUG_TRACE(("mkdir(%s)", buf));
+        if (!mg_stat(conn, buf, &file) && mg_mkdir(buf, 0755) != 0) {
+            res = -1;
+            break;
+        }
+
+        /* Is path itself a directory? */
+        if (p[1] == '\0') {
+            res = 0;
+        }
+    }
+
+    return res;
+}
+
+static void mkcol(struct mg_connection *conn, const char *path)
+{
+    int rc, body_len;
+    struct de de;
+    char date[64];
+    time_t curtime = time(NULL);
+
+    memset(&de.file, 0, sizeof(de.file));
+    if (!mg_stat(conn, path, &de.file)) {
+        mg_cry(conn, "%s: mg_stat(%s) failed: %s",
+               __func__, path, strerror(ERRNO));
+    }
+
+    if(de.file.modification_time) {
+        send_http_error(conn, 405, "Method Not Allowed",
+                        "mkcol(%s): %s", path, strerror(ERRNO));
+        return;
+    }
+
+    body_len = conn->data_len - conn->request_len;
+    if(body_len > 0) {
+        send_http_error(conn, 415, "Unsupported media type",
+                        "mkcol(%s): %s", path, strerror(ERRNO));
+        return;
+    }
+
+    rc = mg_mkdir(path, 0755);
+
+    if (rc == 0) {
+        conn->status_code = 201;
+        gmt_time_string(date, sizeof(date), &curtime);
+        mg_printf(conn, "HTTP/1.1 %d Created\r\nDate: %s\r\nContent-Length: 0\r\nConnection: %s\r\n\r\n",
+                  conn->status_code, date, suggest_connection_header(conn));
+    } else if (rc == -1) {
+        if(errno == EEXIST)
+            send_http_error(conn, 405, "Method Not Allowed",
+                            "mkcol(%s): %s", path, strerror(ERRNO));
+        else if(errno == EACCES)
+            send_http_error(conn, 403, "Forbidden",
+                            "mkcol(%s): %s", path, strerror(ERRNO));
+        else if(errno == ENOENT)
+            send_http_error(conn, 409, "Conflict",
+                            "mkcol(%s): %s", path, strerror(ERRNO));
+        else
+            send_http_error(conn, 500, http_500_error,
+                            "fopen(%s): %s", path, strerror(ERRNO));
+    }
+}
+
+static void put_file(struct mg_connection *conn, const char *path)
+{
+    struct file file = STRUCT_FILE_INITIALIZER;
+    const char *range;
+    int64_t r1, r2;
+    int rc;
+    char date[64];
+    time_t curtime = time(NULL);
+
+    conn->status_code = mg_stat(conn, path, &file) ? 200 : 201;
+
+    if ((rc = put_dir(conn, path)) == 0) {
+        gmt_time_string(date, sizeof(date), &curtime);
+        mg_printf(conn, "HTTP/1.1 %d OK\r\nDate: %s\r\nContent-Length: 0\r\nConnection: %s\r\n\r\n",
+                  conn->status_code, date, suggest_connection_header(conn));
+    } else if (rc == -1) {
+        send_http_error(conn, 500, http_500_error,
+                        "put_dir(%s): %s", path, strerror(ERRNO));
+    } else if (!mg_fopen(conn, path, "wb+", &file) || file.fp == NULL) {
+        mg_fclose(&file);
+        send_http_error(conn, 500, http_500_error,
+                        "fopen(%s): %s", path, strerror(ERRNO));
+    } else {
+        fclose_on_exec(&file, conn);
+        range = mg_get_header(conn, "Content-Range");
+        r1 = r2 = 0;
+        if (range != NULL && parse_range_header(range, &r1, &r2) > 0) {
+            conn->status_code = 206;
+            fseeko(file.fp, r1, SEEK_SET);
+        }
+        if (!forward_body_data(conn, file.fp, INVALID_SOCKET, NULL)) {
+            conn->status_code = 500;
+        }
+        gmt_time_string(date, sizeof(date), &curtime);
+        mg_printf(conn, "HTTP/1.1 %d OK\r\nDate: %s\r\nContent-Length: 0\r\nConnection: %s\r\n\r\n",
+                  conn->status_code, date, suggest_connection_header(conn));
+        mg_fclose(&file);
+    }
+}
+
+static void send_ssi_file(struct mg_connection *, const char *,
+                          struct file *, int);
+
+static void do_ssi_include(struct mg_connection *conn, const char *ssi,
+                           char *tag, int include_level)
+{
+    char file_name[MG_BUF_LEN], path[512], *p;
+    struct file file = STRUCT_FILE_INITIALIZER;
+    size_t len;
+
+    /* sscanf() is safe here, since send_ssi_file() also uses buffer
+       of size MG_BUF_LEN to get the tag. So strlen(tag) is
+       always < MG_BUF_LEN. */
+    if (sscanf(tag, " virtual=\"%511[^\"]\"", file_name) == 1) {
+        /* File name is relative to the webserver root */
+        file_name[511]=0;
+        (void) mg_snprintf(conn, path, sizeof(path), "%s%c%s",
+                           conn->ctx->config[DOCUMENT_ROOT], '/', file_name);
+    } else if (sscanf(tag, " abspath=\"%511[^\"]\"", file_name) == 1) {
+        /* File name is relative to the webserver working directory
+           or it is absolute system path */
+        file_name[511]=0;
+        (void) mg_snprintf(conn, path, sizeof(path), "%s", file_name);
+    } else if (sscanf(tag, " file=\"%511[^\"]\"", file_name) == 1 ||
+               sscanf(tag, " \"%511[^\"]\"", file_name) == 1) {
+        /* File name is relative to the currect document */
+        file_name[511]=0;
+        (void) mg_snprintf(conn, path, sizeof(path), "%s", ssi);
+        if ((p = strrchr(path, '/')) != NULL) {
+            p[1] = '\0';
+        }
+        len = strlen(path);
+        (void) mg_snprintf(conn, path + len, sizeof(path) - len, "%s", file_name);
+    } else {
+        mg_cry(conn, "Bad SSI #include: [%s]", tag);
+        return;
+    }
+
+    if (!mg_fopen(conn, path, "rb", &file)) {
+        mg_cry(conn, "Cannot open SSI #include: [%s]: fopen(%s): %s",
+               tag, path, strerror(ERRNO));
+    } else {
+        fclose_on_exec(&file, conn);
+        if (match_prefix(conn->ctx->config[SSI_EXTENSIONS],
+                         (int)strlen(conn->ctx->config[SSI_EXTENSIONS]), path) > 0) {
+            send_ssi_file(conn, path, &file, include_level + 1);
+        } else {
+            send_file_data(conn, &file, 0, INT64_MAX);
+        }
+        mg_fclose(&file);
+    }
+}
+
+#if !defined(NO_POPEN)
+static void do_ssi_exec(struct mg_connection *conn, char *tag)
+{
+    char cmd[1024] = "";
+    struct file file = STRUCT_FILE_INITIALIZER;
+
+    if (sscanf(tag, " \"%1023[^\"]\"", cmd) != 1) {
+        mg_cry(conn, "Bad SSI #exec: [%s]", tag);
+    } else {
+        cmd[1023]=0;
+        if ((file.fp = popen(cmd, "r")) == NULL) {
+            mg_cry(conn, "Cannot SSI #exec: [%s]: %s", cmd, strerror(ERRNO));
+        } else {
+            send_file_data(conn, &file, 0, INT64_MAX);
+            pclose(file.fp);
+        }
+    }
+}
+#endif /* !NO_POPEN */
+
+static int mg_fgetc(struct file *filep, int offset)
+{
+    if (filep->membuf != NULL && offset >=0 && offset < filep->size) {
+        return ((unsigned char *) filep->membuf)[offset];
+    } else if (filep->fp != NULL) {
+        return fgetc(filep->fp);
+    } else {
+        return EOF;
+    }
+}
+
+static void send_ssi_file(struct mg_connection *conn, const char *path,
+                          struct file *filep, int include_level)
+{
+    char buf[MG_BUF_LEN];
+    int ch, offset, len, in_ssi_tag;
+
+    if (include_level > 10) {
+        mg_cry(conn, "SSI #include level is too deep (%s)", path);
+        return;
+    }
+
+    in_ssi_tag = len = offset = 0;
+    while ((ch = mg_fgetc(filep, offset)) != EOF) {
+        if (in_ssi_tag && ch == '>') {
+            in_ssi_tag = 0;
+            buf[len++] = (char) ch;
+            buf[len] = '\0';
+            assert(len <= (int) sizeof(buf));
+            if (len < 6 || memcmp(buf, " LIBRBD_AIO_WRITE_COPYUP
+     *           |        ^                              |
+     *           v        \------------------------------/
+     *         done
+     *           ^
+     *           |
+     * LIBRBD_AIO_WRITE_FLAT
+     *
+     * Writes start in LIBRBD_AIO_WRITE_GUARD or _FLAT, depending on whether
+     * there is a parent or not.
+     */
+    enum write_state_d {
+      LIBRBD_AIO_WRITE_GUARD,
+      LIBRBD_AIO_WRITE_COPYUP,
+      LIBRBD_AIO_WRITE_FLAT
+    };
+
+  protected:
+    virtual void add_copyup_ops() = 0;
+
+    write_state_d m_state;
+    vector > m_object_image_extents;
+    uint64_t m_parent_overlap;
+    librados::ObjectWriteOperation m_write;
+    librados::ObjectWriteOperation m_copyup;
+    uint64_t m_snap_seq;
+    std::vector m_snaps;
+
+  private:
+    void send_copyup();
+  };
+
+  class AioWrite : public AbstractWrite {
+  public:
+    AioWrite(ImageCtx *ictx, const std::string &oid,
+	     uint64_t object_no, uint64_t object_off,
+	     vector >& objectx, uint64_t object_overlap,
+	     const ceph::bufferlist &data, const ::SnapContext &snapc,
+	     librados::snap_t snap_id,
+	     Context *completion)
+      : AbstractWrite(ictx, oid,
+		      object_no, object_off, data.length(),
+		      objectx, object_overlap,
+		      snapc, snap_id,
+		      completion, false),
+	m_write_data(data) {
+      guard_write();
+      add_write_ops(m_write);
+    }
+    virtual ~AioWrite() {}
+
+  protected:
+    virtual void add_copyup_ops() {
+      add_write_ops(m_copyup);
+    }
+
+  private:
+    void add_write_ops(librados::ObjectWriteOperation &wr);
+    ceph::bufferlist m_write_data;
+  };
+
+  class AioRemove : public AbstractWrite {
+  public:
+    AioRemove(ImageCtx *ictx, const std::string &oid,
+	      uint64_t object_no,
+	      vector >& objectx, uint64_t object_overlap,
+	      const ::SnapContext &snapc, librados::snap_t snap_id,
+	      Context *completion)
+      : AbstractWrite(ictx, oid,
+		      object_no, 0, 0,
+		      objectx, object_overlap,
+		      snapc, snap_id, completion,
+		      true) {
+      if (has_parent())
+	m_write.truncate(0);
+      else
+	m_write.remove();
+    }
+    virtual ~AioRemove() {}
+
+  protected:
+    virtual void add_copyup_ops() {
+      // removing an object never needs to copyup
+      assert(0);
+    }
+  };
+
+  class AioTruncate : public AbstractWrite {
+  public:
+    AioTruncate(ImageCtx *ictx, const std::string &oid,
+		uint64_t object_no, uint64_t object_off,
+		vector >& objectx, uint64_t object_overlap,
+		const ::SnapContext &snapc, librados::snap_t snap_id,
+		Context *completion)
+      : AbstractWrite(ictx, oid,
+		      object_no, object_off, 0,
+		      objectx, object_overlap,
+		      snapc, snap_id, completion,
+		      true) {
+      guard_write();
+      m_write.truncate(object_off);
+    }
+    virtual ~AioTruncate() {}
+
+  protected:
+    virtual void add_copyup_ops() {
+      m_copyup.truncate(m_object_off);
+    }
+  };
+
+  class AioZero : public AbstractWrite {
+  public:
+    AioZero(ImageCtx *ictx, const std::string &oid,
+	    uint64_t object_no, uint64_t object_off, uint64_t object_len,
+	    vector >& objectx, uint64_t object_overlap,
+	    const ::SnapContext &snapc, librados::snap_t snap_id,
+	    Context *completion)
+      : AbstractWrite(ictx, oid,
+		      object_no, object_off, object_len,
+		      objectx, object_overlap,
+		      snapc, snap_id, completion,
+		      true) {
+      guard_write();
+      m_write.zero(object_off, object_len);
+    }
+    virtual ~AioZero() {}
+
+  protected:
+    virtual void add_copyup_ops() {
+      m_copyup.zero(m_object_off, m_object_len);
+    }
+  };
+
+}
+
+#endif
diff --git a/ceph/src/librbd/ImageCtx.cc b/ceph/src/librbd/ImageCtx.cc
new file mode 100644
index 00000000..b5c2db6e
--- /dev/null
+++ b/ceph/src/librbd/ImageCtx.cc
@@ -0,0 +1,659 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include 
+
+#include "common/ceph_context.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "common/perf_counters.h"
+
+#include "librbd/internal.h"
+#include "librbd/WatchCtx.h"
+
+#include "librbd/ImageCtx.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::ImageCtx: "
+
+using std::map;
+using std::pair;
+using std::set;
+using std::string;
+using std::vector;
+
+using ceph::bufferlist;
+using librados::snap_t;
+using librados::IoCtx;
+
+namespace librbd {
+  ImageCtx::ImageCtx(const string &image_name, const string &image_id,
+		     const char *snap, IoCtx& p, bool ro)
+    : cct((CephContext*)p.cct()),
+      perfcounter(NULL),
+      snap_id(CEPH_NOSNAP),
+      snap_exists(true),
+      read_only(ro),
+      flush_encountered(false),
+      exclusive_locked(false),
+      name(image_name),
+      wctx(NULL),
+      refresh_seq(0),
+      last_refresh(0),
+      md_lock("librbd::ImageCtx::md_lock"),
+      cache_lock("librbd::ImageCtx::cache_lock"),
+      snap_lock("librbd::ImageCtx::snap_lock"),
+      parent_lock("librbd::ImageCtx::parent_lock"),
+      refresh_lock("librbd::ImageCtx::refresh_lock"),
+      extra_read_flags(0),
+      old_format(true),
+      order(0), size(0), features(0),
+      format_string(NULL),
+      id(image_id), parent(NULL),
+      stripe_unit(0), stripe_count(0),
+      object_cacher(NULL), writeback_handler(NULL), object_set(NULL)
+  {
+    md_ctx.dup(p);
+    data_ctx.dup(p);
+
+    memset(&header, 0, sizeof(header));
+    memset(&layout, 0, sizeof(layout));
+
+    string pname = string("librbd-") + id + string("-") +
+      data_ctx.get_pool_name() + string("/") + name;
+    if (snap) {
+      snap_name = snap;
+      pname += "@";
+      pname += snap_name;
+    }
+    perf_start(pname);
+
+    if (cct->_conf->rbd_cache) {
+      Mutex::Locker l(cache_lock);
+      ldout(cct, 20) << "enabling caching..." << dendl;
+      writeback_handler = new LibrbdWriteback(this, cache_lock);
+
+      uint64_t init_max_dirty = cct->_conf->rbd_cache_max_dirty;
+      if (cct->_conf->rbd_cache_writethrough_until_flush)
+	init_max_dirty = 0;
+      ldout(cct, 20) << "Initial cache settings:"
+		     << " size=" << cct->_conf->rbd_cache_size
+		     << " num_objects=" << 10
+		     << " max_dirty=" << init_max_dirty
+		     << " target_dirty=" << cct->_conf->rbd_cache_target_dirty
+		     << " max_dirty_age="
+		     << cct->_conf->rbd_cache_max_dirty_age << dendl;
+
+      object_cacher = new ObjectCacher(cct, pname, *writeback_handler, cache_lock,
+				       NULL, NULL,
+				       cct->_conf->rbd_cache_size,
+				       10,  /* reset this in init */
+				       init_max_dirty,
+				       cct->_conf->rbd_cache_target_dirty,
+				       cct->_conf->rbd_cache_max_dirty_age,
+				       cct->_conf->rbd_cache_block_writes_upfront);
+      object_set = new ObjectCacher::ObjectSet(NULL, data_ctx.get_id(), 0);
+      object_set->return_enoent = true;
+      object_cacher->start();
+    }
+  }
+
+  ImageCtx::~ImageCtx() {
+    perf_stop();
+    if (object_cacher) {
+      delete object_cacher;
+      object_cacher = NULL;
+    }
+    if (writeback_handler) {
+      delete writeback_handler;
+      writeback_handler = NULL;
+    }
+    if (object_set) {
+      delete object_set;
+      object_set = NULL;
+    }
+    delete[] format_string;
+  }
+
+  int ImageCtx::init() {
+    int r;
+    if (id.length()) {
+      old_format = false;
+    } else {
+      r = detect_format(md_ctx, name, &old_format, NULL);
+      if (r < 0) {
+	lderr(cct) << "error finding header: " << cpp_strerror(r) << dendl;
+	return r;
+      }
+    }
+
+    if (!old_format) {
+      if (!id.length()) {
+	r = cls_client::get_id(&md_ctx, id_obj_name(name), &id);
+	if (r < 0) {
+	  lderr(cct) << "error reading image id: " << cpp_strerror(r)
+		     << dendl;
+	  return r;
+	}
+      }
+
+      header_oid = header_name(id);
+      r = cls_client::get_immutable_metadata(&md_ctx, header_oid,
+					     &object_prefix, &order);
+      if (r < 0) {
+	lderr(cct) << "error reading immutable metadata: "
+		   << cpp_strerror(r) << dendl;
+	return r;
+      }
+
+      r = cls_client::get_stripe_unit_count(&md_ctx, header_oid,
+					    &stripe_unit, &stripe_count);
+      if (r < 0 && r != -ENOEXEC && r != -EINVAL) {
+	lderr(cct) << "error reading striping metadata: "
+		   << cpp_strerror(r) << dendl;
+	return r;
+      }
+
+      init_layout();
+    } else {
+      header_oid = old_header_name(name);
+    }
+    return 0;
+  }
+  
+  void ImageCtx::init_layout()
+  {
+    if (stripe_unit == 0 || stripe_count == 0) {
+      stripe_unit = 1ull << order;
+      stripe_count = 1;
+    }
+
+    memset(&layout, 0, sizeof(layout));
+    layout.fl_stripe_unit = stripe_unit;
+    layout.fl_stripe_count = stripe_count;
+    layout.fl_object_size = 1ull << order;
+    layout.fl_pg_pool = data_ctx.get_id();  // FIXME: pool id overflow?
+
+    delete[] format_string;
+    size_t len = object_prefix.length() + 16;
+    format_string = new char[len];
+    if (old_format) {
+      snprintf(format_string, len, "%s.%%012llx", object_prefix.c_str());
+    } else {
+      snprintf(format_string, len, "%s.%%016llx", object_prefix.c_str());
+    }
+
+    // size object cache appropriately
+    if (object_cacher) {
+      uint64_t obj = cct->_conf->rbd_cache_max_dirty_object;
+      if (!obj) {
+        obj = cct->_conf->rbd_cache_size / (1ull << order);
+        obj = obj * 4 + 10;
+      }
+      ldout(cct, 10) << " cache bytes " << cct->_conf->rbd_cache_size << " order " << (int)order
+		     << " -> about " << obj << " objects" << dendl;
+      object_cacher->set_max_objects(obj);
+    }
+
+    ldout(cct, 10) << "init_layout stripe_unit " << stripe_unit
+		   << " stripe_count " << stripe_count
+		   << " object_size " << layout.fl_object_size
+		   << " prefix " << object_prefix
+		   << " format " << format_string
+		   << dendl;
+  }
+
+  void ImageCtx::perf_start(string name) {
+    PerfCountersBuilder plb(cct, name, l_librbd_first, l_librbd_last);
+
+    plb.add_u64_counter(l_librbd_rd, "rd");
+    plb.add_u64_counter(l_librbd_rd_bytes, "rd_bytes");
+    plb.add_time_avg(l_librbd_rd_latency, "rd_latency");
+    plb.add_u64_counter(l_librbd_wr, "wr");
+    plb.add_u64_counter(l_librbd_wr_bytes, "wr_bytes");
+    plb.add_time_avg(l_librbd_wr_latency, "wr_latency");
+    plb.add_u64_counter(l_librbd_discard, "discard");
+    plb.add_u64_counter(l_librbd_discard_bytes, "discard_bytes");
+    plb.add_time_avg(l_librbd_discard_latency, "discard_latency");
+    plb.add_u64_counter(l_librbd_flush, "flush");
+    plb.add_u64_counter(l_librbd_aio_rd, "aio_rd");
+    plb.add_u64_counter(l_librbd_aio_rd_bytes, "aio_rd_bytes");
+    plb.add_time_avg(l_librbd_aio_rd_latency, "aio_rd_latency");
+    plb.add_u64_counter(l_librbd_aio_wr, "aio_wr");
+    plb.add_u64_counter(l_librbd_aio_wr_bytes, "aio_wr_bytes");
+    plb.add_time_avg(l_librbd_aio_wr_latency, "aio_wr_latency");
+    plb.add_u64_counter(l_librbd_aio_discard, "aio_discard");
+    plb.add_u64_counter(l_librbd_aio_discard_bytes, "aio_discard_bytes");
+    plb.add_time_avg(l_librbd_aio_discard_latency, "aio_discard_latency");
+    plb.add_u64_counter(l_librbd_aio_flush, "aio_flush");
+    plb.add_time_avg(l_librbd_aio_flush_latency, "aio_flush_latency");
+    plb.add_u64_counter(l_librbd_snap_create, "snap_create");
+    plb.add_u64_counter(l_librbd_snap_remove, "snap_remove");
+    plb.add_u64_counter(l_librbd_snap_rollback, "snap_rollback");
+    plb.add_u64_counter(l_librbd_notify, "notify");
+    plb.add_u64_counter(l_librbd_resize, "resize");
+
+    perfcounter = plb.create_perf_counters();
+    cct->get_perfcounters_collection()->add(perfcounter);
+  }
+
+  void ImageCtx::perf_stop() {
+    assert(perfcounter);
+    cct->get_perfcounters_collection()->remove(perfcounter);
+    delete perfcounter;
+  }
+
+  void ImageCtx::set_read_flag(unsigned flag) {
+    extra_read_flags |= flag;
+  }
+
+  int ImageCtx::get_read_flags(snap_t snap_id) {
+    int flags = librados::OPERATION_NOFLAG | extra_read_flags;
+    if (snap_id == LIBRADOS_SNAP_HEAD)
+      return flags;
+
+    if (cct->_conf->rbd_balance_snap_reads)
+      flags |= librados::OPERATION_BALANCE_READS;
+    else if (cct->_conf->rbd_localize_snap_reads)
+      flags |= librados::OPERATION_LOCALIZE_READS;
+    return flags;
+  }
+
+  int ImageCtx::snap_set(string in_snap_name)
+  {
+    map::iterator it = snaps_by_name.find(in_snap_name);
+    if (it != snaps_by_name.end()) {
+      snap_name = in_snap_name;
+      snap_id = it->second.id;
+      snap_exists = true;
+      data_ctx.snap_set_read(snap_id);
+      return 0;
+    }
+    return -ENOENT;
+  }
+
+  void ImageCtx::snap_unset()
+  {
+    snap_id = CEPH_NOSNAP;
+    snap_name = "";
+    snap_exists = true;
+    data_ctx.snap_set_read(snap_id);
+  }
+
+  snap_t ImageCtx::get_snap_id(string in_snap_name) const
+  {
+    map::const_iterator it = snaps_by_name.find(in_snap_name);
+    if (it != snaps_by_name.end())
+      return it->second.id;
+    return CEPH_NOSNAP;
+  }
+
+  int ImageCtx::get_snap_name(snapid_t in_snap_id, string *out_snap_name) const
+  {
+    map::const_iterator it;
+
+    for (it = snaps_by_name.begin(); it != snaps_by_name.end(); ++it) {
+      if (it->second.id == in_snap_id) {
+	*out_snap_name = it->first;
+	return 0;
+      }
+    }
+    return -ENOENT;
+  }
+
+  int ImageCtx::get_parent_spec(snapid_t in_snap_id, parent_spec *out_pspec)
+  {
+    map::iterator it;
+
+    for (it = snaps_by_name.begin(); it != snaps_by_name.end(); ++it) {
+      if (it->second.id == in_snap_id) {
+	*out_pspec = it->second.parent.spec;
+	return 0;
+      }
+    }
+    return -ENOENT;
+  }
+
+  uint64_t ImageCtx::get_current_size() const
+  {
+    return size;
+  }
+
+  uint64_t ImageCtx::get_object_size() const
+  {
+    return 1ull << order;
+  }
+
+  string ImageCtx::get_object_name(uint64_t num) const {
+    char buf[object_prefix.length() + 32];
+    snprintf(buf, sizeof(buf), format_string, num);
+    return string(buf);
+  }
+
+  uint64_t ImageCtx::get_stripe_unit() const
+  {
+    return stripe_unit;
+  }
+
+  uint64_t ImageCtx::get_stripe_count() const
+  {
+    return stripe_count;
+  }
+
+  uint64_t ImageCtx::get_stripe_period() const
+  {
+    return stripe_count * (1ull << order);
+  }
+
+  uint64_t ImageCtx::get_num_objects() const
+  {
+    uint64_t period = get_stripe_period();
+    uint64_t num_periods = (size + period - 1) / period;
+    return num_periods * stripe_count;
+  }
+
+  int ImageCtx::is_snap_protected(string in_snap_name, bool *is_protected) const
+  {
+    map::const_iterator it = snaps_by_name.find(in_snap_name);
+    if (it != snaps_by_name.end()) {
+      *is_protected =
+	(it->second.protection_status == RBD_PROTECTION_STATUS_PROTECTED);
+      return 0;
+    }
+    return -ENOENT;
+  }
+
+  int ImageCtx::is_snap_unprotected(string in_snap_name,
+				    bool *is_unprotected) const
+  {
+    map::const_iterator it = snaps_by_name.find(in_snap_name);
+    if (it != snaps_by_name.end()) {
+      *is_unprotected =
+	(it->second.protection_status == RBD_PROTECTION_STATUS_UNPROTECTED);
+      return 0;
+    }
+    return -ENOENT;
+  }
+
+  void ImageCtx::add_snap(string in_snap_name, snap_t id, uint64_t in_size,
+			  uint64_t features,
+			  parent_info parent,
+			  uint8_t protection_status)
+  {
+    snaps.push_back(id);
+    SnapInfo info(id, in_size, features, parent, protection_status);
+    snaps_by_name.insert(pair(in_snap_name, info));
+  }
+
+  uint64_t ImageCtx::get_image_size(snap_t in_snap_id) const
+  {
+    if (in_snap_id == CEPH_NOSNAP) {
+      return size;
+    }
+    string in_snap_name;
+    int r = get_snap_name(in_snap_id, &in_snap_name);
+    if (r < 0)
+      return 0;
+    map::const_iterator p = snaps_by_name.find(in_snap_name);
+    if (p == snaps_by_name.end())
+      return 0;
+    return p->second.size;
+  }
+
+  int ImageCtx::get_features(snap_t in_snap_id, uint64_t *out_features) const
+  {
+    if (in_snap_id == CEPH_NOSNAP) {
+      *out_features = features;
+      return 0;
+    }
+    string in_snap_name;
+    int r = get_snap_name(in_snap_id, &in_snap_name);
+    if (r < 0)
+      return r;
+    map::const_iterator p = snaps_by_name.find(in_snap_name);
+    if (p == snaps_by_name.end())
+      return -ENOENT;
+    *out_features = p->second.features;
+    return 0;
+  }
+
+  int64_t ImageCtx::get_parent_pool_id(snap_t in_snap_id) const
+  {
+    if (in_snap_id == CEPH_NOSNAP) {
+      return parent_md.spec.pool_id;
+    }
+    string in_snap_name;
+    int r = get_snap_name(in_snap_id, &in_snap_name);
+    if (r < 0)
+      return -1;
+    map::const_iterator p = snaps_by_name.find(in_snap_name);
+    if (p == snaps_by_name.end())
+      return -1;
+    return p->second.parent.spec.pool_id;
+  }
+
+  string ImageCtx::get_parent_image_id(snap_t in_snap_id) const
+  {
+    if (in_snap_id == CEPH_NOSNAP) {
+      return parent_md.spec.image_id;
+    }
+    string in_snap_name;
+    int r = get_snap_name(in_snap_id, &in_snap_name);
+    if (r < 0)
+      return "";
+    map::const_iterator p = snaps_by_name.find(in_snap_name);
+    if (p == snaps_by_name.end())
+      return "";
+    return p->second.parent.spec.image_id;
+  }
+
+  uint64_t ImageCtx::get_parent_snap_id(snap_t in_snap_id) const
+  {
+    if (in_snap_id == CEPH_NOSNAP) {
+      return parent_md.spec.snap_id;
+    }
+    string in_snap_name;
+    int r = get_snap_name(in_snap_id, &in_snap_name);
+    if (r < 0)
+      return CEPH_NOSNAP;
+    map::const_iterator p = snaps_by_name.find(in_snap_name);
+    if (p == snaps_by_name.end())
+      return CEPH_NOSNAP;
+    return p->second.parent.spec.snap_id;
+  }
+
+  int ImageCtx::get_parent_overlap(snap_t in_snap_id, uint64_t *overlap) const
+  {
+    if (in_snap_id == CEPH_NOSNAP) {
+      *overlap = parent_md.overlap;
+      return 0;
+    }
+    string in_snap_name;
+    int r = get_snap_name(in_snap_id, &in_snap_name);
+    if (r < 0)
+      return r;
+    map::const_iterator p = snaps_by_name.find(in_snap_name);
+    if (p == snaps_by_name.end())
+      return -ENOENT;
+    *overlap = p->second.parent.overlap;
+    return 0;
+  }
+
+  void ImageCtx::aio_read_from_cache(object_t o, bufferlist *bl, size_t len,
+				     uint64_t off, Context *onfinish) {
+    snap_lock.get_read();
+    ObjectCacher::OSDRead *rd = object_cacher->prepare_read(snap_id, bl, 0);
+    snap_lock.put_read();
+    ObjectExtent extent(o, 0 /* a lie */, off, len, 0);
+    extent.oloc.pool = data_ctx.get_id();
+    extent.buffer_extents.push_back(make_pair(0, len));
+    rd->extents.push_back(extent);
+    cache_lock.Lock();
+    int r = object_cacher->readx(rd, object_set, onfinish);
+    cache_lock.Unlock();
+    if (r != 0)
+      onfinish->complete(r);
+  }
+
+  void ImageCtx::write_to_cache(object_t o, bufferlist& bl, size_t len,
+				uint64_t off, Context *onfinish) {
+    snap_lock.get_read();
+    ObjectCacher::OSDWrite *wr = object_cacher->prepare_write(snapc, bl,
+							      utime_t(), 0);
+    snap_lock.put_read();
+    ObjectExtent extent(o, 0, off, len, 0);
+    extent.oloc.pool = data_ctx.get_id();
+    // XXX: nspace is always default, io_ctx_impl field private
+    //extent.oloc.nspace = data_ctx.io_ctx_impl->oloc.nspace;
+    extent.buffer_extents.push_back(make_pair(0, len));
+    wr->extents.push_back(extent);
+    {
+      Mutex::Locker l(cache_lock);
+      object_cacher->writex(wr, object_set, cache_lock, onfinish);
+    }
+  }
+
+  int ImageCtx::read_from_cache(object_t o, bufferlist *bl, size_t len,
+				uint64_t off) {
+    int r;
+    Mutex mylock("librbd::ImageCtx::read_from_cache");
+    Cond cond;
+    bool done;
+    Context *onfinish = new C_SafeCond(&mylock, &cond, &done, &r);
+    aio_read_from_cache(o, bl, len, off, onfinish);
+    mylock.Lock();
+    while (!done)
+      cond.Wait(mylock);
+    mylock.Unlock();
+    return r;
+  }
+
+  void ImageCtx::user_flushed() {
+    if (object_cacher && cct->_conf->rbd_cache_writethrough_until_flush) {
+      md_lock.get_read();
+      bool flushed_before = flush_encountered;
+      md_lock.put_read();
+
+      uint64_t max_dirty = cct->_conf->rbd_cache_max_dirty;
+      if (!flushed_before && max_dirty > 0) {
+	md_lock.get_write();
+	flush_encountered = true;
+	md_lock.put_write();
+
+	ldout(cct, 10) << "saw first user flush, enabling writeback" << dendl;
+	Mutex::Locker l(cache_lock);
+	object_cacher->set_max_dirty(max_dirty);
+      }
+    }
+  }
+
+  void ImageCtx::flush_cache_aio(Context *onfinish) {
+    cache_lock.Lock();
+    object_cacher->flush_set(object_set, onfinish);
+    cache_lock.Unlock();
+  }
+
+  int ImageCtx::flush_cache() {
+    int r = 0;
+    Mutex mylock("librbd::ImageCtx::flush_cache");
+    Cond cond;
+    bool done;
+    Context *onfinish = new C_SafeCond(&mylock, &cond, &done, &r);
+    flush_cache_aio(onfinish);
+    mylock.Lock();
+    while (!done) {
+      ldout(cct, 20) << "waiting for cache to be flushed" << dendl;
+      cond.Wait(mylock);
+    }
+    mylock.Unlock();
+    ldout(cct, 20) << "finished flushing cache" << dendl;
+    return r;
+  }
+
+  void ImageCtx::shutdown_cache() {
+    md_lock.get_write();
+    invalidate_cache();
+    md_lock.put_write();
+    object_cacher->stop();
+  }
+
+  int ImageCtx::invalidate_cache() {
+    if (!object_cacher)
+      return 0;
+    cache_lock.Lock();
+    object_cacher->release_set(object_set);
+    cache_lock.Unlock();
+    int r = flush_cache();
+    if (r)
+      lderr(cct) << "flush_cache returned " << r << dendl;
+    cache_lock.Lock();
+    bool unclean = object_cacher->release_set(object_set);
+    cache_lock.Unlock();
+    if (unclean) {
+      lderr(cct) << "could not release all objects from cache: "
+                 << unclean << " bytes remain" << dendl;
+      return -EBUSY;
+    }
+    return r;
+  }
+
+  void ImageCtx::clear_nonexistence_cache() {
+    if (!object_cacher)
+      return;
+    cache_lock.Lock();
+    object_cacher->clear_nonexistence(object_set);
+    cache_lock.Unlock();
+  }
+
+  int ImageCtx::register_watch() {
+    assert(!wctx);
+    wctx = new WatchCtx(this);
+    return md_ctx.watch(header_oid, 0, &(wctx->cookie), wctx);
+  }
+
+  void ImageCtx::unregister_watch() {
+    assert(wctx);
+    wctx->invalidate();
+    md_ctx.unwatch(header_oid, wctx->cookie);
+    delete wctx;
+    wctx = NULL;
+  }
+
+  size_t ImageCtx::parent_io_len(uint64_t offset, size_t length,
+				 snap_t in_snap_id)
+  {
+    uint64_t overlap = 0;
+    get_parent_overlap(in_snap_id, &overlap);
+
+    size_t parent_len = 0;
+    if (get_parent_pool_id(in_snap_id) != -1 && offset <= overlap)
+      parent_len = min(overlap, offset + length) - offset;
+
+    ldout(cct, 20) << __func__ << " off = " << offset << " len = " << length
+		   << " overlap = " << overlap << " parent_io_len = "
+		   << parent_len << dendl;
+    return parent_len;
+  }
+
+  uint64_t ImageCtx::prune_parent_extents(vector >& objectx,
+					  uint64_t overlap)
+  {
+    // drop extents completely beyond the overlap
+    while (!objectx.empty() && objectx.back().first >= overlap)
+      objectx.pop_back();
+
+    // trim final overlapping extent
+    if (!objectx.empty() && objectx.back().first + objectx.back().second > overlap)
+      objectx.back().second = overlap - objectx.back().first;
+
+    uint64_t len = 0;
+    for (vector >::iterator p = objectx.begin();
+	 p != objectx.end();
+	 ++p)
+      len += p->second;
+    ldout(cct, 10) << "prune_parent_extents image overlap " << overlap
+		   << ", object overlap " << len
+		   << " from image extents " << objectx << dendl;
+    return len;
+ }
+}
diff --git a/ceph/src/librbd/ImageCtx.h b/ceph/src/librbd/ImageCtx.h
new file mode 100644
index 00000000..83ed0449
--- /dev/null
+++ b/ceph/src/librbd/ImageCtx.h
@@ -0,0 +1,154 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_LIBRBD_IMAGECTX_H
+#define CEPH_LIBRBD_IMAGECTX_H
+
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Mutex.h"
+#include "common/RWLock.h"
+#include "common/snap_types.h"
+#include "include/buffer.h"
+#include "include/rbd/librbd.hpp"
+#include "include/rbd_types.h"
+#include "include/types.h"
+#include "osdc/ObjectCacher.h"
+
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/LibrbdWriteback.h"
+#include "librbd/SnapInfo.h"
+#include "librbd/parent_types.h"
+
+class CephContext;
+class PerfCounters;
+
+namespace librbd {
+
+  class WatchCtx;
+
+  struct ImageCtx {
+    CephContext *cct;
+    PerfCounters *perfcounter;
+    struct rbd_obj_header_ondisk header;
+    ::SnapContext snapc;
+    std::vector snaps; // this mirrors snapc.snaps, but is in
+                                        // a format librados can understand
+    std::map snaps_by_name;
+    uint64_t snap_id;
+    bool snap_exists; // false if our snap_id was deleted
+    // whether the image was opened read-only. cannot be changed after opening
+    bool read_only;
+    bool flush_encountered;
+
+    std::map lockers;
+    bool exclusive_locked;
+    std::string lock_tag;
+
+    std::string name;
+    std::string snap_name;
+    IoCtx data_ctx, md_ctx;
+    WatchCtx *wctx;
+    int refresh_seq;    ///< sequence for refresh requests
+    int last_refresh;   ///< last completed refresh
+
+    /**
+     * Lock ordering:
+     * md_lock, cache_lock, snap_lock, parent_lock, refresh_lock
+     */
+    RWLock md_lock; // protects access to the mutable image metadata that
+                   // isn't guarded by other locks below
+                   // (size, features, image locks, etc)
+    Mutex cache_lock; // used as client_lock for the ObjectCacher
+    RWLock snap_lock; // protects snapshot-related member variables:
+    RWLock parent_lock; // protects parent_md and parent
+    Mutex refresh_lock; // protects refresh_seq and last_refresh
+
+    unsigned extra_read_flags;
+
+    bool old_format;
+    uint8_t order;
+    uint64_t size;
+    uint64_t features;
+    std::string object_prefix;
+    char *format_string;
+    std::string header_oid;
+    std::string id; // only used for new-format images
+    parent_info parent_md;
+    ImageCtx *parent;
+    uint64_t stripe_unit, stripe_count;
+
+    ceph_file_layout layout;
+
+    ObjectCacher *object_cacher;
+    LibrbdWriteback *writeback_handler;
+    ObjectCacher::ObjectSet *object_set;
+
+    /**
+     * Either image_name or image_id must be set.
+     * If id is not known, pass the empty std::string,
+     * and init() will look it up.
+     */
+    ImageCtx(const std::string &image_name, const std::string &image_id,
+	     const char *snap, IoCtx& p, bool read_only);
+    ~ImageCtx();
+    int init();
+    void init_layout();
+    void perf_start(std::string name);
+    void perf_stop();
+    void set_read_flag(unsigned flag);
+    int get_read_flags(librados::snap_t snap_id);
+    int snap_set(std::string in_snap_name);
+    void snap_unset();
+    librados::snap_t get_snap_id(std::string in_snap_name) const;
+    int get_snap_name(snapid_t snap_id, std::string *out_snap_name) const;
+    int get_parent_spec(snapid_t snap_id, parent_spec *pspec);
+    int is_snap_protected(string in_snap_name, bool *is_protected) const;
+    int is_snap_unprotected(string in_snap_name, bool *is_unprotected) const;
+
+    uint64_t get_current_size() const;
+    uint64_t get_object_size() const;
+    string get_object_name(uint64_t num) const;
+    uint64_t get_num_objects() const;
+    uint64_t get_stripe_unit() const;
+    uint64_t get_stripe_count() const;
+    uint64_t get_stripe_period() const;
+
+    void add_snap(std::string in_snap_name, librados::snap_t id,
+		  uint64_t in_size, uint64_t features,
+		  parent_info parent, uint8_t protection_status);
+    uint64_t get_image_size(librados::snap_t in_snap_id) const;
+    int get_features(librados::snap_t in_snap_id,
+		     uint64_t *out_features) const;
+    int64_t get_parent_pool_id(librados::snap_t in_snap_id) const;
+    std::string get_parent_image_id(librados::snap_t in_snap_id) const;
+    uint64_t get_parent_snap_id(librados::snap_t in_snap_id) const;
+    int get_parent_overlap(librados::snap_t in_snap_id,
+			   uint64_t *overlap) const;
+    void aio_read_from_cache(object_t o, bufferlist *bl, size_t len,
+			     uint64_t off, Context *onfinish);
+    void write_to_cache(object_t o, bufferlist& bl, size_t len, uint64_t off,
+			Context *onfinish);
+    int read_from_cache(object_t o, bufferlist *bl, size_t len, uint64_t off);
+    void user_flushed();
+    void flush_cache_aio(Context *onfinish);
+    int flush_cache();
+    void shutdown_cache();
+    int invalidate_cache();
+    void clear_nonexistence_cache();
+    int register_watch();
+    void unregister_watch();
+    size_t parent_io_len(uint64_t offset, size_t length,
+			 librados::snap_t in_snap_id);
+    uint64_t prune_parent_extents(vector >& objectx,
+				  uint64_t overlap);
+
+  };
+}
+
+#endif
diff --git a/ceph/src/librbd/LibrbdWriteback.cc b/ceph/src/librbd/LibrbdWriteback.cc
new file mode 100644
index 00000000..854ac9dc
--- /dev/null
+++ b/ceph/src/librbd/LibrbdWriteback.cc
@@ -0,0 +1,197 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include 
+
+#include "common/ceph_context.h"
+#include "common/dout.h"
+#include "common/Mutex.h"
+#include "include/Context.h"
+#include "include/rados/librados.hpp"
+#include "include/rbd/librbd.hpp"
+
+#include "librbd/AioRequest.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/internal.h"
+#include "librbd/LibrbdWriteback.h"
+
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbdwriteback: "
+
+namespace librbd {
+
+  /**
+   * callback to finish a rados completion as a Context
+   *
+   * @param c completion
+   * @param arg Context* recast as void*
+   */
+  void context_cb(rados_completion_t c, void *arg)
+  {
+    Context *con = reinterpret_cast(arg);
+    con->complete(rados_aio_get_return_value(c));
+  }
+
+  /**
+   * context to wrap another context in a Mutex
+   *
+   * @param cct cct
+   * @param c context to finish
+   * @param l mutex to lock
+   */
+  class C_Request : public Context {
+  public:
+    C_Request(CephContext *cct, Context *c, Mutex *l)
+      : m_cct(cct), m_ctx(c), m_lock(l) {}
+    virtual ~C_Request() {}
+    virtual void finish(int r) {
+      ldout(m_cct, 20) << "aio_cb completing " << dendl;
+      {
+	Mutex::Locker l(*m_lock);
+	m_ctx->complete(r);
+      }
+      ldout(m_cct, 20) << "aio_cb finished" << dendl;
+    }
+  private:
+    CephContext *m_cct;
+    Context *m_ctx;
+    Mutex *m_lock;
+  };
+
+  class C_OrderedWrite : public Context {
+  public:
+    C_OrderedWrite(CephContext *cct, LibrbdWriteback::write_result_d *result,
+		   LibrbdWriteback *wb)
+      : m_cct(cct), m_result(result), m_wb_handler(wb) {}
+    virtual ~C_OrderedWrite() {}
+    virtual void finish(int r) {
+      ldout(m_cct, 20) << "C_OrderedWrite completing " << m_result << dendl;
+      {
+	Mutex::Locker l(m_wb_handler->m_lock);
+	assert(!m_result->done);
+	m_result->done = true;
+	m_result->ret = r;
+	m_wb_handler->complete_writes(m_result->oid);
+      }
+      ldout(m_cct, 20) << "C_OrderedWrite finished " << m_result << dendl;
+    }
+  private:
+    CephContext *m_cct;
+    LibrbdWriteback::write_result_d *m_result;
+    LibrbdWriteback *m_wb_handler;
+  };
+
+  LibrbdWriteback::LibrbdWriteback(ImageCtx *ictx, Mutex& lock)
+    : m_tid(0), m_lock(lock), m_ictx(ictx)
+  {
+  }
+
+  void LibrbdWriteback::read(const object_t& oid,
+			     const object_locator_t& oloc,
+			     uint64_t off, uint64_t len, snapid_t snapid,
+			     bufferlist *pbl, uint64_t trunc_size,
+			     __u32 trunc_seq, Context *onfinish)
+  {
+    // on completion, take the mutex and then call onfinish.
+    Context *req = new C_Request(m_ictx->cct, onfinish, &m_lock);
+    librados::AioCompletion *rados_completion =
+      librados::Rados::aio_create_completion(req, context_cb, NULL);
+    librados::ObjectReadOperation op;
+    op.read(off, len, pbl, NULL);
+    int flags = m_ictx->get_read_flags(snapid);
+    int r = m_ictx->data_ctx.aio_operate(oid.name, rados_completion, &op,
+					 flags, NULL);
+    rados_completion->release();
+    assert(r >= 0);
+  }
+
+  bool LibrbdWriteback::may_copy_on_write(const object_t& oid, uint64_t read_off, uint64_t read_len, snapid_t snapid)
+  {
+    m_ictx->snap_lock.get_read();
+    librados::snap_t snap_id = m_ictx->snap_id;
+    m_ictx->parent_lock.get_read();
+    uint64_t overlap = 0;
+    m_ictx->get_parent_overlap(snap_id, &overlap);
+    m_ictx->parent_lock.put_read();
+    m_ictx->snap_lock.put_read();
+
+    uint64_t object_no = oid_to_object_no(oid.name, m_ictx->object_prefix);
+
+    // reverse map this object extent onto the parent
+    vector > objectx;
+    Striper::extent_to_file(m_ictx->cct, &m_ictx->layout,
+			  object_no, 0, m_ictx->layout.fl_object_size,
+			  objectx);
+    uint64_t object_overlap = m_ictx->prune_parent_extents(objectx, overlap);
+    bool may = object_overlap > 0;
+    ldout(m_ictx->cct, 10) << "may_copy_on_write " << oid << " " << read_off << "~" << read_len << " = " << may << dendl;
+    return may;
+  }
+
+  ceph_tid_t LibrbdWriteback::write(const object_t& oid,
+			       const object_locator_t& oloc,
+			       uint64_t off, uint64_t len,
+			       const SnapContext& snapc,
+			       const bufferlist &bl, utime_t mtime,
+			       uint64_t trunc_size, __u32 trunc_seq,
+			       Context *oncommit)
+  {
+    m_ictx->snap_lock.get_read();
+    librados::snap_t snap_id = m_ictx->snap_id;
+    m_ictx->parent_lock.get_read();
+    uint64_t overlap = 0;
+    m_ictx->get_parent_overlap(snap_id, &overlap);
+    m_ictx->parent_lock.put_read();
+    m_ictx->snap_lock.put_read();
+
+    uint64_t object_no = oid_to_object_no(oid.name, m_ictx->object_prefix);
+    
+    // reverse map this object extent onto the parent
+    vector > objectx;
+    Striper::extent_to_file(m_ictx->cct, &m_ictx->layout,
+			  object_no, 0, m_ictx->layout.fl_object_size,
+			  objectx);
+    uint64_t object_overlap = m_ictx->prune_parent_extents(objectx, overlap);
+    write_result_d *result = new write_result_d(oid.name, oncommit);
+    m_writes[oid.name].push(result);
+    ldout(m_ictx->cct, 20) << "write will wait for result " << result << dendl;
+    C_OrderedWrite *req_comp = new C_OrderedWrite(m_ictx->cct, result, this);
+    AioWrite *req = new AioWrite(m_ictx, oid.name,
+				 object_no, off, objectx, object_overlap,
+				 bl, snapc, snap_id,
+				 req_comp);
+    req->send();
+    return ++m_tid;
+  }
+
+  void LibrbdWriteback::complete_writes(const std::string& oid)
+  {
+    assert(m_lock.is_locked());
+    std::queue& results = m_writes[oid];
+    ldout(m_ictx->cct, 20) << "complete_writes() oid " << oid << dendl;
+    std::list finished;
+
+    while (!results.empty()) {
+      write_result_d *result = results.front();
+      if (!result->done)
+	break;
+      finished.push_back(result);
+      results.pop();
+    }
+
+    if (results.empty())
+      m_writes.erase(oid);
+
+    for (std::list::iterator it = finished.begin();
+	 it != finished.end(); ++it) {
+      write_result_d *result = *it;
+      ldout(m_ictx->cct, 20) << "complete_writes() completing " << result
+			     << dendl;
+      result->oncommit->complete(result->ret);
+      delete result;
+    }
+  }
+}
diff --git a/ceph/src/librbd/LibrbdWriteback.h b/ceph/src/librbd/LibrbdWriteback.h
new file mode 100644
index 00000000..a8bd9cb2
--- /dev/null
+++ b/ceph/src/librbd/LibrbdWriteback.h
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_LIBRBD_LIBRBDWRITEBACKHANDLER_H
+#define CEPH_LIBRBD_LIBRBDWRITEBACKHANDLER_H
+
+#include 
+
+#include "include/Context.h"
+#include "include/types.h"
+#include "include/rados/librados.hpp"
+#include "osd/osd_types.h"
+#include "osdc/WritebackHandler.h"
+
+class Mutex;
+
+namespace librbd {
+
+  struct ImageCtx;
+
+  class LibrbdWriteback : public WritebackHandler {
+  public:
+    LibrbdWriteback(ImageCtx *ictx, Mutex& lock);
+    virtual ~LibrbdWriteback() {}
+
+    // Note that oloc, trunc_size, and trunc_seq are ignored
+    virtual void read(const object_t& oid, const object_locator_t& oloc,
+		      uint64_t off, uint64_t len, snapid_t snapid,
+		      bufferlist *pbl, uint64_t trunc_size,  __u32 trunc_seq,
+		      Context *onfinish);
+
+    // Determine whether a read to this extent could be affected by a write-triggered copy-on-write
+    virtual bool may_copy_on_write(const object_t& oid, uint64_t read_off, uint64_t read_len, snapid_t snapid);
+
+    // Note that oloc, trunc_size, and trunc_seq are ignored
+    virtual ceph_tid_t write(const object_t& oid, const object_locator_t& oloc,
+			uint64_t off, uint64_t len, const SnapContext& snapc,
+			const bufferlist &bl, utime_t mtime, uint64_t trunc_size,
+			__u32 trunc_seq, Context *oncommit);
+
+    struct write_result_d {
+      bool done;
+      int ret;
+      std::string oid;
+      Context *oncommit;
+      write_result_d(const std::string& oid, Context *oncommit) :
+	done(false), ret(0), oid(oid), oncommit(oncommit) {}
+    private:
+      write_result_d(const write_result_d& rhs);
+      const write_result_d& operator=(const write_result_d& rhs);
+    };
+
+  private:
+    void complete_writes(const std::string& oid);
+
+    ceph_tid_t m_tid;
+    Mutex& m_lock;
+    librbd::ImageCtx *m_ictx;
+    ceph::unordered_map > m_writes;
+    friend class C_OrderedWrite;
+  };
+}
+
+#endif
diff --git a/ceph/src/librbd/Makefile.am b/ceph/src/librbd/Makefile.am
new file mode 100644
index 00000000..0a3600da
--- /dev/null
+++ b/ceph/src/librbd/Makefile.am
@@ -0,0 +1,27 @@
+librbd_la_SOURCES = \
+	librbd/librbd.cc \
+	librbd/AioCompletion.cc \
+	librbd/AioRequest.cc \
+	librbd/ImageCtx.cc \
+	librbd/internal.cc \
+	librbd/LibrbdWriteback.cc \
+	librbd/WatchCtx.cc
+librbd_la_LIBADD = \
+	$(LIBRADOS) $(LIBOSDC) \
+	libcls_rbd_client.la libcls_lock_client.la \
+	$(PTHREAD_LIBS) $(EXTRALIBS)
+librbd_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0
+if LINUX
+librbd_la_LDFLAGS += -export-symbols-regex '^rbd_.*'
+endif
+lib_LTLIBRARIES += librbd.la
+
+noinst_HEADERS += \
+	librbd/AioCompletion.h \
+	librbd/AioRequest.h \
+	librbd/ImageCtx.h \
+	librbd/internal.h \
+	librbd/LibrbdWriteback.h \
+	librbd/parent_types.h \
+	librbd/SnapInfo.h \
+	librbd/WatchCtx.h
diff --git a/ceph/src/librbd/SnapInfo.h b/ceph/src/librbd/SnapInfo.h
new file mode 100644
index 00000000..44dd4cf2
--- /dev/null
+++ b/ceph/src/librbd/SnapInfo.h
@@ -0,0 +1,28 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_LIBRBD_SNAPINFO_H
+#define CEPH_LIBRBD_SNAPINFO_H
+
+#include "include/int_types.h"
+
+#include "include/rados/librados.hpp"
+
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/parent_types.h"
+
+namespace librbd {
+
+  struct SnapInfo {
+    librados::snap_t id;
+    uint64_t size;
+    uint64_t features;
+    parent_info parent;
+    uint8_t protection_status;
+    SnapInfo(librados::snap_t _id, uint64_t _size, uint64_t _features,
+	     parent_info _parent, uint8_t _protection_status) :
+      id(_id), size(_size), features(_features), parent(_parent),
+      protection_status(_protection_status) {}
+  };
+}
+
+#endif
diff --git a/ceph/src/librbd/WatchCtx.cc b/ceph/src/librbd/WatchCtx.cc
new file mode 100644
index 00000000..b64e8ef1
--- /dev/null
+++ b/ceph/src/librbd/WatchCtx.cc
@@ -0,0 +1,36 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/ceph_context.h"
+#include "common/dout.h"
+#include "common/perf_counters.h"
+
+#include "librbd/ImageCtx.h"
+#include "librbd/internal.h"
+
+#include "librbd/WatchCtx.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::WatchCtx: "
+
+namespace librbd {
+
+  void WatchCtx::invalidate()
+  {
+    Mutex::Locker l(lock);
+    valid = false;
+  }
+
+  void WatchCtx::notify(uint8_t opcode, uint64_t ver, bufferlist& bl)
+  {
+    Mutex::Locker l(lock);
+    ldout(ictx->cct, 1) <<  " got notification opcode=" << (int)opcode
+			<< " ver=" << ver << " cookie=" << cookie << dendl;
+    if (valid) {
+      Mutex::Locker lictx(ictx->refresh_lock);
+      ++ictx->refresh_seq;
+      ictx->perfcounter->inc(l_librbd_notify);
+    }
+  }
+}
diff --git a/ceph/src/librbd/WatchCtx.h b/ceph/src/librbd/WatchCtx.h
new file mode 100644
index 00000000..9872c843
--- /dev/null
+++ b/ceph/src/librbd/WatchCtx.h
@@ -0,0 +1,32 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_LIBRBD_WATCHCTX_H
+#define CEPH_LIBRBD_WATCHCTX_H
+
+#include "include/int_types.h"
+
+#include "common/Mutex.h"
+#include "include/buffer.h"
+#include "include/rados/librados.hpp"
+
+class ImageCtx;
+
+namespace librbd {
+
+  class WatchCtx : public librados::WatchCtx {
+    ImageCtx *ictx;
+    bool valid;
+    Mutex lock;
+  public:
+    uint64_t cookie;
+    WatchCtx(ImageCtx *ctx) : ictx(ctx),
+			      valid(true),
+			      lock("librbd::WatchCtx"),
+			      cookie(0) {}
+    virtual ~WatchCtx() {}
+    void invalidate();
+    virtual void notify(uint8_t opcode, uint64_t ver, ceph::bufferlist& bl);
+  };
+}
+
+#endif
diff --git a/ceph/src/librbd/internal.cc b/ceph/src/librbd/internal.cc
new file mode 100644
index 00000000..afa46602
--- /dev/null
+++ b/ceph/src/librbd/internal.cc
@@ -0,0 +1,3191 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "include/int_types.h"
+
+#include 
+#include 
+
+#include "common/ceph_context.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "common/Throttle.h"
+#include "cls/lock/cls_lock_client.h"
+#include "include/stringify.h"
+
+#include "cls/rbd/cls_rbd.h"
+
+#include "librbd/AioCompletion.h"
+#include "librbd/AioRequest.h"
+#include "librbd/ImageCtx.h"
+
+#include "librbd/internal.h"
+#include "librbd/parent_types.h"
+#include "include/util.h"
+
+#include "librados/snap_set_diff.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd: "
+
+#define rbd_howmany(x, y)  (((x) + (y) - 1) / (y))
+
+using std::map;
+using std::pair;
+using std::set;
+using std::string;
+using std::vector;
+// list binds to list() here, so std::list is explicitly used below
+
+using ceph::bufferlist;
+using librados::snap_t;
+using librados::IoCtx;
+using librados::Rados;
+
+namespace librbd {
+  const string id_obj_name(const string &name)
+  {
+    return RBD_ID_PREFIX + name;
+  }
+
+  const string header_name(const string &image_id)
+  {
+    return RBD_HEADER_PREFIX + image_id;
+  }
+
+  const string old_header_name(const string &image_name)
+  {
+    return image_name + RBD_SUFFIX;
+  }
+
+  int detect_format(IoCtx &io_ctx, const string &name,
+		    bool *old_format, uint64_t *size)
+  {
+    CephContext *cct = (CephContext *)io_ctx.cct();
+    if (old_format)
+      *old_format = true;
+    int r = io_ctx.stat(old_header_name(name), size, NULL);
+    if (r < 0) {
+      if (old_format)
+	*old_format = false;
+      r = io_ctx.stat(id_obj_name(name), size, NULL);
+      if (r < 0)
+	return r;
+    }
+
+    ldout(cct, 20) << "detect format of " << name << " : "
+		   << (old_format ? (*old_format ? "old" : "new") :
+		       "don't care")  << dendl;
+    return 0;
+  }
+
+  bool has_parent(int64_t parent_pool_id, uint64_t off, uint64_t overlap)
+  {
+    return (parent_pool_id != -1 && off <= overlap);
+  }
+
+  void init_rbd_header(struct rbd_obj_header_ondisk& ondisk,
+		       uint64_t size, int order, uint64_t bid)
+  {
+    uint32_t hi = bid >> 32;
+    uint32_t lo = bid & 0xFFFFFFFF;
+    uint32_t extra = rand() % 0xFFFFFFFF;
+    memset(&ondisk, 0, sizeof(ondisk));
+
+    memcpy(&ondisk.text, RBD_HEADER_TEXT, sizeof(RBD_HEADER_TEXT));
+    memcpy(&ondisk.signature, RBD_HEADER_SIGNATURE,
+	   sizeof(RBD_HEADER_SIGNATURE));
+    memcpy(&ondisk.version, RBD_HEADER_VERSION, sizeof(RBD_HEADER_VERSION));
+
+    snprintf(ondisk.block_name, sizeof(ondisk.block_name), "rb.%x.%x.%x",
+	     hi, lo, extra);
+
+    ondisk.image_size = size;
+    ondisk.options.order = order;
+    ondisk.options.crypt_type = RBD_CRYPT_NONE;
+    ondisk.options.comp_type = RBD_COMP_NONE;
+    ondisk.snap_seq = 0;
+    ondisk.snap_count = 0;
+    ondisk.reserved = 0;
+    ondisk.snap_names_len = 0;
+  }
+
+  void image_info(ImageCtx *ictx, image_info_t& info, size_t infosize)
+  {
+    int obj_order = ictx->order;
+    ictx->md_lock.get_read();
+    ictx->snap_lock.get_read();
+    info.size = ictx->get_image_size(ictx->snap_id);
+    ictx->snap_lock.put_read();
+    ictx->md_lock.put_read();
+    info.obj_size = 1ULL << obj_order;
+    info.num_objs = rbd_howmany(info.size, ictx->get_object_size());
+    info.order = obj_order;
+    memcpy(&info.block_name_prefix, ictx->object_prefix.c_str(),
+	   min((size_t)RBD_MAX_BLOCK_NAME_SIZE,
+	       ictx->object_prefix.length() + 1));
+    // clear deprecated fields
+    info.parent_pool = -1L;
+    info.parent_name[0] = '\0';
+  }
+
+  uint64_t oid_to_object_no(const string& oid, const string& object_prefix)
+  {
+    istringstream iss(oid);
+    // skip object prefix and separator
+    iss.ignore(object_prefix.length() + 1);
+    uint64_t num;
+    iss >> std::hex >> num;
+    return num;
+  }
+
+  int init_rbd_info(struct rbd_info *info)
+  {
+    memset(info, 0, sizeof(*info));
+    return 0;
+  }
+
+  void trim_image(ImageCtx *ictx, uint64_t newsize, ProgressContext& prog_ctx)
+  {
+    CephContext *cct = (CephContext *)ictx->data_ctx.cct();
+
+    uint64_t size = ictx->get_current_size();
+    uint64_t period = ictx->get_stripe_period();
+    uint64_t num_period = ((newsize + period - 1) / period);
+    uint64_t delete_off = MIN(num_period * period, size);
+    // first object we can delete free and clear
+    uint64_t delete_start = num_period * ictx->get_stripe_count();
+    uint64_t num_objects = ictx->get_num_objects();
+    uint64_t object_size = ictx->get_object_size();
+
+    ldout(cct, 10) << "trim_image " << size << " -> " << newsize
+		   << " periods " << num_period
+		   << " discard to offset " << delete_off
+		   << " delete objects " << delete_start
+		   << " to " << (num_objects-1)
+		   << dendl;
+
+    SimpleThrottle throttle(cct->_conf->rbd_concurrent_management_ops, true);
+    if (delete_start < num_objects) {
+      ldout(cct, 2) << "trim_image objects " << delete_start << " to "
+		    << (num_objects - 1) << dendl;
+      for (uint64_t i = delete_start; i < num_objects; ++i) {
+	string oid = ictx->get_object_name(i);
+	Context *req_comp = new C_SimpleThrottle(&throttle);
+	librados::AioCompletion *rados_completion =
+	  librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
+	ictx->data_ctx.aio_remove(oid, rados_completion);
+	rados_completion->release();
+	prog_ctx.update_progress((i - delete_start) * object_size,
+				 (num_objects - delete_start) * object_size);
+      }
+    }
+
+    // discard the weird boundary, if any
+    if (delete_off > newsize) {
+      vector extents;
+      Striper::file_to_extents(ictx->cct, ictx->format_string, &ictx->layout,
+			       newsize, delete_off - newsize, 0, extents);
+
+      for (vector::iterator p = extents.begin();
+	   p != extents.end(); ++p) {
+	ldout(ictx->cct, 20) << " ex " << *p << dendl;
+	Context *req_comp = new C_SimpleThrottle(&throttle);
+	librados::AioCompletion *rados_completion =
+	  librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
+	if (p->offset == 0) {
+	  ictx->data_ctx.aio_remove(p->oid.name, rados_completion);
+	} else {
+	  librados::ObjectWriteOperation op;
+	  op.truncate(p->offset);
+	  ictx->data_ctx.aio_operate(p->oid.name, rados_completion, &op);
+	}
+	rados_completion->release();
+      }
+    }
+    int r = throttle.wait_for_ret();
+    if (r < 0) {
+      lderr(cct) << "warning: failed to remove some object(s): "
+		 << cpp_strerror(r) << dendl;
+    }
+  }
+
+  int read_rbd_info(IoCtx& io_ctx, const string& info_oid,
+		    struct rbd_info *info)
+  {
+    int r;
+    bufferlist bl;
+    r = io_ctx.read(info_oid, bl, sizeof(*info), 0);
+    if (r < 0)
+      return r;
+    if (r == 0) {
+      return init_rbd_info(info);
+    }
+
+    if (r < (int)sizeof(*info))
+      return -EIO;
+
+    memcpy(info, bl.c_str(), r);
+    return 0;
+  }
+
+  int read_header_bl(IoCtx& io_ctx, const string& header_oid,
+		     bufferlist& header, uint64_t *ver)
+  {
+    int r;
+    uint64_t off = 0;
+#define READ_SIZE 4096
+    do {
+      bufferlist bl;
+      r = io_ctx.read(header_oid, bl, READ_SIZE, off);
+      if (r < 0)
+	return r;
+      header.claim_append(bl);
+      off += r;
+    } while (r == READ_SIZE);
+
+    if (memcmp(RBD_HEADER_TEXT, header.c_str(), sizeof(RBD_HEADER_TEXT))) {
+      CephContext *cct = (CephContext *)io_ctx.cct();
+      lderr(cct) << "unrecognized header format" << dendl;
+      return -ENXIO;
+    }
+
+    if (ver)
+      *ver = io_ctx.get_last_version();
+
+    return 0;
+  }
+
+  int notify_change(IoCtx& io_ctx, const string& oid, uint64_t *pver,
+		    ImageCtx *ictx)
+  {
+    uint64_t ver;
+
+    if (ictx) {
+      ictx->refresh_lock.Lock();
+      ldout(ictx->cct, 20) << "notify_change refresh_seq = " << ictx->refresh_seq
+			   << " last_refresh = " << ictx->last_refresh << dendl;
+      ++ictx->refresh_seq;
+      ictx->refresh_lock.Unlock();
+    }
+
+    if (pver)
+      ver = *pver;
+    else
+      ver = io_ctx.get_last_version();
+    bufferlist bl;
+    io_ctx.notify(oid, ver, bl);
+    return 0;
+  }
+
+  int read_header(IoCtx& io_ctx, const string& header_oid,
+		  struct rbd_obj_header_ondisk *header, uint64_t *ver)
+  {
+    bufferlist header_bl;
+    int r = read_header_bl(io_ctx, header_oid, header_bl, ver);
+    if (r < 0)
+      return r;
+    if (header_bl.length() < (int)sizeof(*header))
+      return -EIO;
+    memcpy(header, header_bl.c_str(), sizeof(*header));
+
+    return 0;
+  }
+
+  int write_header(IoCtx& io_ctx, const string& header_oid, bufferlist& header)
+  {
+    bufferlist bl;
+    int r = io_ctx.write(header_oid, header, header.length(), 0);
+
+    notify_change(io_ctx, header_oid, NULL, NULL);
+
+    return r;
+  }
+
+  int tmap_set(IoCtx& io_ctx, const string& imgname)
+  {
+    bufferlist cmdbl, emptybl;
+    __u8 c = CEPH_OSD_TMAP_SET;
+    ::encode(c, cmdbl);
+    ::encode(imgname, cmdbl);
+    ::encode(emptybl, cmdbl);
+    return io_ctx.tmap_update(RBD_DIRECTORY, cmdbl);
+  }
+
+  int tmap_rm(IoCtx& io_ctx, const string& imgname)
+  {
+    bufferlist cmdbl;
+    __u8 c = CEPH_OSD_TMAP_RM;
+    ::encode(c, cmdbl);
+    ::encode(imgname, cmdbl);
+    return io_ctx.tmap_update(RBD_DIRECTORY, cmdbl);
+  }
+
+  int rollback_image(ImageCtx *ictx, uint64_t snap_id,
+		     ProgressContext& prog_ctx)
+  {
+    uint64_t numseg = ictx->get_num_objects();
+    uint64_t bsize = ictx->get_object_size();
+    int r;
+    CephContext *cct = ictx->cct;
+    SimpleThrottle throttle(cct->_conf->rbd_concurrent_management_ops, true);
+
+    for (uint64_t i = 0; i < numseg; i++) {
+      string oid = ictx->get_object_name(i);
+      Context *req_comp = new C_SimpleThrottle(&throttle);
+      librados::AioCompletion *rados_completion =
+	librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
+      librados::ObjectWriteOperation op;
+      op.selfmanaged_snap_rollback(snap_id);
+      ictx->data_ctx.aio_operate(oid, rados_completion, &op);
+      ldout(cct, 10) << "scheduling selfmanaged_snap_rollback on "
+		     << oid << " to " << snap_id << dendl;
+      rados_completion->release();
+      prog_ctx.update_progress(i * bsize, numseg * bsize);
+    }
+
+    r = throttle.wait_for_ret();
+    if (r < 0) {
+      ldout(cct, 10) << "failed to rollback at least one object: "
+		     << cpp_strerror(r) << dendl;
+      return r;
+    }
+    return 0;
+  }
+
+  int list(IoCtx& io_ctx, vector& names)
+  {
+    CephContext *cct = (CephContext *)io_ctx.cct();
+    ldout(cct, 20) << "list " << &io_ctx << dendl;
+
+    bufferlist bl;
+    int r = io_ctx.read(RBD_DIRECTORY, bl, 0, 0);
+    if (r < 0)
+      return r;
+
+    // old format images are in a tmap
+    if (bl.length()) {
+      bufferlist::iterator p = bl.begin();
+      bufferlist header;
+      map m;
+      ::decode(header, p);
+      ::decode(m, p);
+      for (map::iterator q = m.begin(); q != m.end(); ++q) {
+	names.push_back(q->first);
+      }
+    }
+
+    // new format images are accessed by class methods
+    int max_read = 1024;
+    string last_read = "";
+    do {
+      map images;
+      cls_client::dir_list(&io_ctx, RBD_DIRECTORY,
+			   last_read, max_read, &images);
+      for (map::const_iterator it = images.begin();
+	   it != images.end(); ++it) {
+	names.push_back(it->first);
+      }
+      if (!images.empty()) {
+	last_read = images.rbegin()->first;
+      }
+      r = images.size();
+    } while (r == max_read);
+
+    return 0;
+  }
+
+  int list_children(ImageCtx *ictx, set >& names)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "children list " << ictx->name << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    // no children for non-layered or old format image
+    if ((ictx->features & RBD_FEATURE_LAYERING) == 0)
+      return 0;
+
+    parent_spec parent_spec(ictx->md_ctx.get_id(), ictx->id, ictx->snap_id);
+    names.clear();
+
+    // search all pools for children depending on this snapshot
+    Rados rados(ictx->md_ctx);
+    std::list pools;
+    rados.pool_list(pools);
+
+    for (std::list::const_iterator it = pools.begin();
+	 it != pools.end(); ++it) {
+      IoCtx ioctx;
+      rados.ioctx_create(it->c_str(), ioctx);
+      set image_ids;
+      int r = cls_client::get_children(&ioctx, RBD_CHILDREN,
+				       parent_spec, image_ids);
+      if (r < 0 && r != -ENOENT) {
+	lderr(cct) << "Error reading list of children from pool " << *it
+		   << dendl;
+	return r;
+      }
+
+      for (set::const_iterator id_it = image_ids.begin();
+	   id_it != image_ids.end(); ++id_it) {
+	string name;
+	r = cls_client::dir_get_name(&ioctx, RBD_DIRECTORY,
+				     *id_it, &name);
+	if (r < 0) {
+	  lderr(cct) << "Error looking up name for image id " << *id_it
+		     << " in pool " << *it << dendl;
+	  return r;
+	}
+	names.insert(make_pair(*it, name));
+      }
+    }
+
+    return 0;
+  }
+
+  int snap_create(ImageCtx *ictx, const char *snap_name)
+  {
+    ldout(ictx->cct, 20) << "snap_create " << ictx << " " << snap_name << dendl;
+
+    if (ictx->read_only)
+      return -EROFS;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker l(ictx->md_lock);
+    do {
+      r = add_snap(ictx, snap_name);
+    } while (r == -ESTALE);
+
+    if (r < 0)
+      return r;
+
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+
+    ictx->perfcounter->inc(l_librbd_snap_create);
+    return 0;
+  }
+
+  static int scan_for_parents(ImageCtx *ictx, parent_spec &pspec,
+			      snapid_t oursnap_id)
+  {
+    if (pspec.pool_id != -1) {
+      map::iterator it;
+      for (it = ictx->snaps_by_name.begin();
+	   it != ictx->snaps_by_name.end(); ++it) {
+	// skip our snap id (if checking base image, CEPH_NOSNAP won't match)
+	if (it->second.id == oursnap_id)
+	  continue;
+	if (it->second.parent.spec == pspec)
+	  break;
+      }
+      if (it == ictx->snaps_by_name.end())
+	return -ENOENT;
+    }
+    return 0;
+  }
+
+  int snap_remove(ImageCtx *ictx, const char *snap_name)
+  {
+    ldout(ictx->cct, 20) << "snap_remove " << ictx << " " << snap_name << dendl;
+
+    if (ictx->read_only)
+      return -EROFS;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker l(ictx->md_lock);
+    snap_t snap_id;
+
+    {
+      // block for purposes of auto-destruction of l2 on early return
+      RWLock::RLocker l2(ictx->snap_lock);
+      snap_id = ictx->get_snap_id(snap_name);
+      if (snap_id == CEPH_NOSNAP)
+	return -ENOENT;
+
+      parent_spec our_pspec;
+      RWLock::RLocker l3(ictx->parent_lock);
+      r = ictx->get_parent_spec(snap_id, &our_pspec);
+      if (r < 0) {
+	lderr(ictx->cct) << "snap_remove: can't get parent spec" << dendl;
+	return r;
+      }
+
+      if (ictx->parent_md.spec != our_pspec &&
+	  (scan_for_parents(ictx, our_pspec, snap_id) == -ENOENT)) {
+	  r = cls_client::remove_child(&ictx->md_ctx, RBD_CHILDREN,
+				       our_pspec, ictx->id);
+	  if (r < 0)
+	    return r;
+      }
+    }
+
+    r = rm_snap(ictx, snap_name);
+    if (r < 0)
+      return r;
+
+    r = ictx->data_ctx.selfmanaged_snap_remove(snap_id);
+
+    if (r < 0)
+      return r;
+
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+
+    ictx->perfcounter->inc(l_librbd_snap_remove);
+    return 0;
+  }
+
+  int snap_protect(ImageCtx *ictx, const char *snap_name)
+  {
+    ldout(ictx->cct, 20) << "snap_protect " << ictx << " " << snap_name
+			 << dendl;
+
+    if (ictx->read_only)
+      return -EROFS;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker l(ictx->md_lock);
+    RWLock::RLocker l2(ictx->snap_lock);
+    uint64_t features;
+    ictx->get_features(ictx->snap_id, &features);
+    if ((features & RBD_FEATURE_LAYERING) == 0) {
+      lderr(ictx->cct) << "snap_protect: image must support layering"
+		       << dendl;
+      return -ENOSYS;
+    }
+    snap_t snap_id = ictx->get_snap_id(snap_name);
+    if (snap_id == CEPH_NOSNAP)
+      return -ENOENT;
+
+    bool is_protected;
+    r = ictx->is_snap_protected(snap_name, &is_protected);
+    if (r < 0)
+      return r;
+
+    if (is_protected)
+      return -EBUSY;
+
+    r = cls_client::set_protection_status(&ictx->md_ctx,
+					  ictx->header_oid,
+					  snap_id,
+					  RBD_PROTECTION_STATUS_PROTECTED);
+    if (r < 0)
+      return r;
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+    return 0;
+  }
+
+  int snap_unprotect(ImageCtx *ictx, const char *snap_name)
+  {
+    ldout(ictx->cct, 20) << "snap_unprotect " << ictx << " " << snap_name
+			 << dendl;
+
+    if (ictx->read_only)
+      return -EROFS;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker l(ictx->md_lock);
+    RWLock::RLocker l2(ictx->snap_lock);
+    uint64_t features;
+    ictx->get_features(ictx->snap_id, &features);
+    if ((features & RBD_FEATURE_LAYERING) == 0) {
+      lderr(ictx->cct) << "snap_unprotect: image must support layering"
+		       << dendl;
+      return -ENOSYS;
+    }
+    snap_t snap_id = ictx->get_snap_id(snap_name);
+    if (snap_id == CEPH_NOSNAP)
+      return -ENOENT;
+
+    bool is_unprotected;
+    r = ictx->is_snap_unprotected(snap_name, &is_unprotected);
+    if (r < 0)
+      return r;
+
+    if (is_unprotected) {
+      lderr(ictx->cct) << "snap_unprotect: snapshot is already unprotected"
+		       << dendl;
+      return -EINVAL;
+    }
+
+    r = cls_client::set_protection_status(&ictx->md_ctx,
+					  ictx->header_oid,
+					  snap_id,
+					  RBD_PROTECTION_STATUS_UNPROTECTING);
+    if (r < 0)
+      return r;
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+
+    parent_spec pspec(ictx->md_ctx.get_id(), ictx->id, snap_id);
+    // search all pools for children depending on this snapshot
+    Rados rados(ictx->md_ctx);
+    std::list pools;
+    rados.pool_list(pools);
+    std::set children;
+    for (std::list::const_iterator it = pools.begin(); it != pools.end(); ++it) {
+      IoCtx pool_ioctx;
+      r = rados.ioctx_create(it->c_str(), pool_ioctx);
+      if (r < 0) {
+	lderr(ictx->cct) << "snap_unprotect: can't create ioctx for pool "
+			 << *it << dendl;
+	goto reprotect_and_return_err;
+      }
+      r = cls_client::get_children(&pool_ioctx, RBD_CHILDREN, pspec, children);
+      // key should not exist for this parent if there is no entry
+      if (((r < 0) && (r != -ENOENT))) {
+	lderr(ictx->cct) << "can't get children for pool " << *it << dendl;
+	goto reprotect_and_return_err;
+      }
+      // if we found a child, can't unprotect
+      if (r == 0) {
+	lderr(ictx->cct) << "snap_unprotect: can't unprotect; at least " 
+	  << children.size() << " child(ren) in pool " << it->c_str() << dendl;
+	r = -EBUSY;
+	goto reprotect_and_return_err;
+      }
+      pool_ioctx.close();	// last one out will self-destruct
+    }
+    // didn't find any child in any pool, go ahead with unprotect
+    r = cls_client::set_protection_status(&ictx->md_ctx,
+					  ictx->header_oid,
+					  snap_id,
+					  RBD_PROTECTION_STATUS_UNPROTECTED);
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+    return 0;
+
+reprotect_and_return_err:
+    int proterr = cls_client::set_protection_status(&ictx->md_ctx,
+						    ictx->header_oid,
+						    snap_id,
+					      RBD_PROTECTION_STATUS_PROTECTED);
+    if (proterr < 0) {
+      lderr(ictx->cct) << "snap_unprotect: can't reprotect image" << dendl;
+    }
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+    return r;
+  }
+
+  int snap_is_protected(ImageCtx *ictx, const char *snap_name,
+			bool *is_protected)
+  {
+    ldout(ictx->cct, 20) << "snap_is_protected " << ictx << " " << snap_name
+			 << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker l(ictx->snap_lock);
+    bool is_unprotected;
+    r = ictx->is_snap_unprotected(snap_name, &is_unprotected);
+    // consider both PROTECTED or UNPROTECTING to be 'protected',
+    // since in either state they can't be deleted
+    *is_protected = !is_unprotected;
+    return r;
+  }
+
+  int create_v1(IoCtx& io_ctx, const char *imgname, uint64_t bid,
+		uint64_t size, int order)
+  {
+    CephContext *cct = (CephContext *)io_ctx.cct();
+    ldout(cct, 2) << "adding rbd image to directory..." << dendl;
+    int r = tmap_set(io_ctx, imgname);
+    if (r < 0) {
+      lderr(cct) << "error adding image to directory: " << cpp_strerror(r)
+		 << dendl;
+      return r;
+    }
+
+    ldout(cct, 2) << "creating rbd image..." << dendl;
+    struct rbd_obj_header_ondisk header;
+    init_rbd_header(header, size, order, bid);
+
+    bufferlist bl;
+    bl.append((const char *)&header, sizeof(header));
+
+    string header_oid = old_header_name(imgname);
+    r = io_ctx.write(header_oid, bl, bl.length(), 0);
+    if (r < 0) {
+      lderr(cct) << "Error writing image header: " << cpp_strerror(r)
+		 << dendl;
+      int remove_r = tmap_rm(io_ctx, imgname);
+      if (remove_r < 0) {
+	lderr(cct) << "Could not remove image from directory after "
+		   << "header creation failed: "
+		   << cpp_strerror(r) << dendl;
+      }
+      return r;
+    }
+
+    ldout(cct, 2) << "done." << dendl;
+    return 0;
+  }
+
+  int create_v2(IoCtx& io_ctx, const char *imgname, uint64_t bid, uint64_t size,
+		int order, uint64_t features, uint64_t stripe_unit,
+		uint64_t stripe_count)
+  {
+    ostringstream bid_ss;
+    uint32_t extra;
+    string id, id_obj, header_oid;
+    int remove_r;
+    ostringstream oss;
+    CephContext *cct = (CephContext *)io_ctx.cct();
+
+    id_obj = id_obj_name(imgname);
+
+    int r = io_ctx.create(id_obj, true);
+    if (r < 0) {
+      lderr(cct) << "error creating rbd id object: " << cpp_strerror(r)
+		 << dendl;
+      return r;
+    }
+
+    extra = rand() % 0xFFFFFFFF;
+    bid_ss << std::hex << bid << std::hex << extra;
+    id = bid_ss.str();
+    r = cls_client::set_id(&io_ctx, id_obj, id);
+    if (r < 0) {
+      lderr(cct) << "error setting image id: " << cpp_strerror(r) << dendl;
+      goto err_remove_id;
+    }
+
+    ldout(cct, 2) << "adding rbd image to directory..." << dendl;
+    r = cls_client::dir_add_image(&io_ctx, RBD_DIRECTORY, imgname, id);
+    if (r < 0) {
+      lderr(cct) << "error adding image to directory: " << cpp_strerror(r)
+		 << dendl;
+      goto err_remove_id;
+    }
+
+    oss << RBD_DATA_PREFIX << id;
+    header_oid = header_name(id);
+    r = cls_client::create_image(&io_ctx, header_oid, size, order,
+				 features, oss.str());
+    if (r < 0) {
+      lderr(cct) << "error writing header: " << cpp_strerror(r) << dendl;
+      goto err_remove_from_dir;
+    }
+
+    if ((stripe_unit || stripe_count) &&
+	(stripe_count != 1 || stripe_unit != (1ull << order))) {
+      r = cls_client::set_stripe_unit_count(&io_ctx, header_oid,
+					    stripe_unit, stripe_count);
+      if (r < 0) {
+	lderr(cct) << "error setting striping parameters: "
+		   << cpp_strerror(r) << dendl;
+	goto err_remove_header;
+      }
+    }
+
+    ldout(cct, 2) << "done." << dendl;
+    return 0;
+
+  err_remove_header:
+    remove_r = io_ctx.remove(header_oid);
+    if (remove_r < 0) {
+      lderr(cct) << "error cleaning up image header after creation failed: "
+		 << dendl;
+    }
+  err_remove_from_dir:
+    remove_r = cls_client::dir_remove_image(&io_ctx, RBD_DIRECTORY,
+					    imgname, id);
+    if (remove_r < 0) {
+      lderr(cct) << "error cleaning up image from rbd_directory object "
+		 << "after creation failed: " << cpp_strerror(remove_r)
+		 << dendl;
+    }
+  err_remove_id:
+    remove_r = io_ctx.remove(id_obj);
+    if (remove_r < 0) {
+      lderr(cct) << "error cleaning up id object after creation failed: "
+		 << cpp_strerror(remove_r) << dendl;
+    }
+
+    return r;
+  }
+
+  int create(librados::IoCtx& io_ctx, const char *imgname, uint64_t size,
+	     int *order)
+  {
+    CephContext *cct = (CephContext *)io_ctx.cct();
+    bool old_format = cct->_conf->rbd_default_format == 1;
+    uint64_t features = old_format ? 0 : cct->_conf->rbd_default_features;
+    return create(io_ctx, imgname, size, old_format, features, order, 0, 0);
+  }
+
+  int create(IoCtx& io_ctx, const char *imgname, uint64_t size,
+	     bool old_format, uint64_t features, int *order,
+	     uint64_t stripe_unit, uint64_t stripe_count)
+  {
+    if (!order)
+      return -EINVAL;
+
+    CephContext *cct = (CephContext *)io_ctx.cct();
+    ldout(cct, 20) << "create " << &io_ctx << " name = " << imgname
+		   << " size = " << size << " old_format = " << old_format
+		   << " features = " << features << " order = " << *order
+		   << " stripe_unit = " << stripe_unit
+		   << " stripe_count = " << stripe_count
+		   << dendl;
+
+
+    if (features & ~RBD_FEATURES_ALL) {
+      lderr(cct) << "librbd does not support requested features." << dendl;
+      return -ENOSYS;
+    }
+
+    // make sure it doesn't already exist, in either format
+    int r = detect_format(io_ctx, imgname, NULL, NULL);
+    if (r != -ENOENT) {
+      if (r) {
+	lderr(cct) << "Could not tell if " << imgname << " already exists" << dendl;
+	return r;
+      }
+      lderr(cct) << "rbd image " << imgname << " already exists" << dendl;
+      return -EEXIST;
+    }
+
+    if (!*order)
+      *order = cct->_conf->rbd_default_order;
+    if (!*order)
+      *order = RBD_DEFAULT_OBJ_ORDER;
+
+    if (*order && (*order > 64 || *order < 12)) {
+      lderr(cct) << "order must be in the range [12, 64]" << dendl;
+      return -EDOM;
+    }
+
+    Rados rados(io_ctx);
+    uint64_t bid = rados.get_instance_id();
+
+    // if striping is enabled, use possibly custom defaults
+    if (!old_format && (features & RBD_FEATURE_STRIPINGV2) &&
+	!stripe_unit && !stripe_count) {
+      stripe_unit = cct->_conf->rbd_default_stripe_unit;
+      stripe_count = cct->_conf->rbd_default_stripe_count;
+    }
+
+    // normalize for default striping
+    if (stripe_unit == (1ull << *order) && stripe_count == 1) {
+      stripe_unit = 0;
+      stripe_count = 0;
+    }
+    if ((stripe_unit || stripe_count) &&
+	(features & RBD_FEATURE_STRIPINGV2) == 0) {
+      lderr(cct) << "STRIPINGV2 and format 2 or later required for non-default striping" << dendl;
+      return -EINVAL;
+    }
+    if ((stripe_unit && !stripe_count) ||
+	(!stripe_unit && stripe_count))
+      return -EINVAL;
+
+    if (old_format) {
+      if (stripe_unit && stripe_unit != (1ull << *order))
+	return -EINVAL;
+      if (stripe_count && stripe_count != 1)
+	return -EINVAL;
+
+      return create_v1(io_ctx, imgname, bid, size, *order);
+    } else {
+      return create_v2(io_ctx, imgname, bid, size, *order, features,
+		       stripe_unit, stripe_count);
+    }
+  }
+
+  /*
+   * Parent may be in different pool, hence different IoCtx
+   */
+  int clone(IoCtx& p_ioctx, const char *p_name, const char *p_snap_name,
+	    IoCtx& c_ioctx, const char *c_name,
+	    uint64_t features, int *c_order,
+	    uint64_t stripe_unit, int stripe_count)
+  {
+    CephContext *cct = (CephContext *)p_ioctx.cct();
+    ldout(cct, 20) << "clone " << &p_ioctx << " name " << p_name << " snap "
+		   << p_snap_name << "to child " << &c_ioctx << " name "
+		   << c_name << " features = " << features << " order = "
+		   << *c_order
+		   << " stripe_unit = " << stripe_unit
+		   << " stripe_count = " << stripe_count
+		   << dendl;
+
+    if (features & ~RBD_FEATURES_ALL) {
+      lderr(cct) << "librbd does not support requested features" << dendl;
+      return -ENOSYS;
+    }
+
+    // make sure child doesn't already exist, in either format
+    int r = detect_format(c_ioctx, c_name, NULL, NULL);
+    if (r != -ENOENT) {
+      lderr(cct) << "rbd image " << c_name << " already exists" << dendl;
+      return -EEXIST;
+    }
+
+    if (p_snap_name == NULL) {
+      lderr(cct) << "image to be cloned must be a snapshot" << dendl;
+      return -EINVAL;
+    }
+
+    bool snap_protected;
+    int order;
+    uint64_t size;
+    uint64_t p_features;
+    int remove_r;
+    librbd::NoOpProgressContext no_op;
+    ImageCtx *c_imctx = NULL;
+    // make sure parent snapshot exists
+    ImageCtx *p_imctx = new ImageCtx(p_name, "", p_snap_name, p_ioctx, true);
+    r = open_image(p_imctx);
+    if (r < 0) {
+      lderr(cct) << "error opening parent image: "
+		 << cpp_strerror(-r) << dendl;
+      return r;
+    }
+
+    parent_spec pspec(p_ioctx.get_id(), p_imctx->id,
+				  p_imctx->snap_id);
+
+    if (p_imctx->old_format) {
+      lderr(cct) << "parent image must be in new format" << dendl;
+      r = -EINVAL;
+      goto err_close_parent;
+    }
+
+    p_imctx->md_lock.get_read();
+    p_imctx->snap_lock.get_read();
+    p_imctx->get_features(p_imctx->snap_id, &p_features);
+    size = p_imctx->get_image_size(p_imctx->snap_id);
+    p_imctx->is_snap_protected(p_imctx->snap_name, &snap_protected);
+    p_imctx->snap_lock.put_read();
+    p_imctx->md_lock.put_read();
+
+    if ((p_features & RBD_FEATURE_LAYERING) != RBD_FEATURE_LAYERING) {
+      lderr(cct) << "parent image must support layering" << dendl;
+      r = -ENOSYS;
+      goto err_close_parent;
+    }
+
+    if (!snap_protected) {
+      lderr(cct) << "parent snapshot must be protected" << dendl;
+      r = -EINVAL;
+      goto err_close_parent;
+    }
+
+    order = *c_order;
+    if (!order)
+      order = p_imctx->order;
+
+    r = create(c_ioctx, c_name, size, false, features, &order,
+	       stripe_unit, stripe_count);
+    if (r < 0) {
+      lderr(cct) << "error creating child: " << cpp_strerror(r) << dendl;
+      goto err_close_parent;
+    }
+
+    c_imctx = new ImageCtx(c_name, "", NULL, c_ioctx, false);
+    r = open_image(c_imctx);
+    if (r < 0) {
+      lderr(cct) << "Error opening new image: " << cpp_strerror(r) << dendl;
+      goto err_remove;
+    }
+
+    r = cls_client::set_parent(&c_ioctx, c_imctx->header_oid, pspec, size);
+    if (r < 0) {
+      lderr(cct) << "couldn't set parent: " << r << dendl;
+      goto err_close_child;
+    }
+
+    r = cls_client::add_child(&c_ioctx, RBD_CHILDREN, pspec, c_imctx->id);
+    if (r < 0) {
+      lderr(cct) << "couldn't add child: " << r << dendl;
+      goto err_close_child;
+    }
+
+    p_imctx->md_lock.get_write();
+    r = ictx_refresh(p_imctx);
+    p_imctx->md_lock.put_write();
+
+    if (r == 0) {
+      p_imctx->snap_lock.get_read();
+      r = p_imctx->is_snap_protected(p_imctx->snap_name, &snap_protected);
+      p_imctx->snap_lock.put_read();
+    }
+    if (r < 0 || !snap_protected) {
+      // we lost the race with unprotect
+      r = -EINVAL;
+      goto err_remove_child;
+    }
+
+    ldout(cct, 2) << "done." << dendl;
+    close_image(c_imctx);
+    close_image(p_imctx);
+    return 0;
+
+  err_remove_child:
+    remove_r = cls_client::remove_child(&c_ioctx, RBD_CHILDREN, pspec,
+					c_imctx->id);
+    if (remove_r < 0) {
+     lderr(cct) << "Error removing failed clone from list of children: "
+		 << cpp_strerror(remove_r) << dendl;
+    }
+  err_close_child:
+    close_image(c_imctx);
+  err_remove:
+    remove_r = remove(c_ioctx, c_name, no_op);
+    if (remove_r < 0) {
+      lderr(cct) << "Error removing failed clone: "
+		 << cpp_strerror(remove_r) << dendl;
+    }
+  err_close_parent:
+    close_image(p_imctx);
+    return r;
+  }
+
+  int rename(IoCtx& io_ctx, const char *srcname, const char *dstname)
+  {
+    CephContext *cct = (CephContext *)io_ctx.cct();
+    ldout(cct, 20) << "rename " << &io_ctx << " " << srcname << " -> "
+		   << dstname << dendl;
+
+    bool old_format;
+    uint64_t src_size;
+    int r = detect_format(io_ctx, srcname, &old_format, &src_size);
+    if (r < 0) {
+      lderr(cct) << "error finding source object: " << cpp_strerror(r) << dendl;
+      return r;
+    }
+
+    string src_oid =
+      old_format ? old_header_name(srcname) : id_obj_name(srcname);
+    string dst_oid =
+      old_format ? old_header_name(dstname) : id_obj_name(dstname);
+
+    string id;
+    if (!old_format) {
+      r = cls_client::get_id(&io_ctx, src_oid, &id);
+      if (r < 0) {
+	lderr(cct) << "error reading image id: " << cpp_strerror(r) << dendl;
+	return r;
+      }
+    }
+
+    bufferlist databl;
+    map omap_values;
+    r = io_ctx.read(src_oid, databl, src_size, 0);
+    if (r < 0) {
+      lderr(cct) << "error reading source object: " << src_oid << ": "
+		 << cpp_strerror(r) << dendl;
+      return r;
+    }
+
+    int MAX_READ = 1024;
+    string last_read = "";
+    do {
+      map outbl;
+      r = io_ctx.omap_get_vals(src_oid, last_read, MAX_READ, &outbl);
+      if (r < 0) {
+	lderr(cct) << "error reading source object omap values: "
+		   << cpp_strerror(r) << dendl;
+	return r;
+      }
+      omap_values.insert(outbl.begin(), outbl.end());
+      if (!outbl.empty())
+	last_read = outbl.rbegin()->first;
+    } while (r == MAX_READ);
+
+    r = detect_format(io_ctx, dstname, NULL, NULL);
+    if (r < 0 && r != -ENOENT) {
+      lderr(cct) << "error checking for existing image called "
+		 << dstname << ":" << cpp_strerror(r) << dendl;
+      return r;
+    }
+    if (r == 0) {
+      lderr(cct) << "rbd image " << dstname << " already exists" << dendl;
+      return -EEXIST;
+    }
+
+    librados::ObjectWriteOperation op;
+    op.create(true);
+    op.write_full(databl);
+    if (!omap_values.empty())
+      op.omap_set(omap_values);
+    r = io_ctx.operate(dst_oid, &op);
+    if (r < 0) {
+      lderr(cct) << "error writing destination object: " << dst_oid << ": "
+		 << cpp_strerror(r) << dendl;
+      return r;
+    }
+
+    if (old_format) {
+      r = tmap_set(io_ctx, dstname);
+      if (r < 0) {
+	io_ctx.remove(dst_oid);
+	lderr(cct) << "couldn't add " << dstname << " to directory: "
+		   << cpp_strerror(r) << dendl;
+	return r;
+      }
+      r = tmap_rm(io_ctx, srcname);
+      if (r < 0) {
+	lderr(cct) << "warning: couldn't remove old entry from directory ("
+		   << srcname << ")" << dendl;
+      }
+    } else {
+      r = cls_client::dir_rename_image(&io_ctx, RBD_DIRECTORY,
+				       srcname, dstname, id);
+      if (r < 0) {
+	lderr(cct) << "error updating directory: " << cpp_strerror(r) << dendl;
+	return r;
+      }
+    }
+
+    r = io_ctx.remove(src_oid);
+    if (r < 0 && r != -ENOENT) {
+      lderr(cct) << "warning: couldn't remove old source object ("
+		 << src_oid << ")" << dendl;
+    }
+
+    if (old_format) {
+      notify_change(io_ctx, old_header_name(srcname), NULL, NULL);
+    }
+
+    return 0;
+  }
+
+
+  int info(ImageCtx *ictx, image_info_t& info, size_t infosize)
+  {
+    ldout(ictx->cct, 20) << "info " << ictx << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    image_info(ictx, info, infosize);
+    return 0;
+  }
+
+  int get_old_format(ImageCtx *ictx, uint8_t *old)
+  {
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+    *old = ictx->old_format;
+    return 0;
+  }
+
+  int get_size(ImageCtx *ictx, uint64_t *size)
+  {
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+    RWLock::RLocker l(ictx->md_lock);
+    RWLock::RLocker l2(ictx->snap_lock);
+    *size = ictx->get_image_size(ictx->snap_id);
+    return 0;
+  }
+
+  int get_features(ImageCtx *ictx, uint64_t *features)
+  {
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+    RWLock::RLocker l(ictx->md_lock);
+    RWLock::RLocker l2(ictx->snap_lock);
+    return ictx->get_features(ictx->snap_id, features);
+  }
+
+  int get_overlap(ImageCtx *ictx, uint64_t *overlap)
+  {
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+    RWLock::RLocker l(ictx->snap_lock);
+    RWLock::RLocker l2(ictx->parent_lock);
+    return ictx->get_parent_overlap(ictx->snap_id, overlap);
+  }
+
+  int open_parent(ImageCtx *ictx)
+  {
+    string pool_name;
+    Rados rados(ictx->md_ctx);
+
+    int64_t pool_id = ictx->get_parent_pool_id(ictx->snap_id);
+    string parent_image_id = ictx->get_parent_image_id(ictx->snap_id);
+    snap_t parent_snap_id = ictx->get_parent_snap_id(ictx->snap_id);
+    assert(parent_snap_id != CEPH_NOSNAP);
+
+    if (pool_id < 0)
+      return -ENOENT;
+    int r = rados.pool_reverse_lookup(pool_id, &pool_name);
+    if (r < 0) {
+      lderr(ictx->cct) << "error looking up name for pool id " << pool_id
+		       << ": " << cpp_strerror(r) << dendl;
+      return r;
+    }
+
+    IoCtx p_ioctx;
+    r = rados.ioctx_create(pool_name.c_str(), p_ioctx);
+    if (r < 0) {
+      lderr(ictx->cct) << "error opening pool " << pool_name << ": "
+		       << cpp_strerror(r) << dendl;
+      return r;
+    }
+
+    // since we don't know the image and snapshot name, set their ids and
+    // reset the snap_name and snap_exists fields after we read the header
+    ictx->parent = new ImageCtx("", parent_image_id, NULL, p_ioctx, true);
+
+    // set rados flags for reading the parent image
+    if (ictx->cct->_conf->rbd_balance_parent_reads)
+      ictx->parent->set_read_flag(librados::OPERATION_BALANCE_READS);
+    else if (ictx->cct->_conf->rbd_localize_parent_reads)
+      ictx->parent->set_read_flag(librados::OPERATION_LOCALIZE_READS);
+
+    r = open_image(ictx->parent);
+    if (r < 0) {
+      lderr(ictx->cct) << "error opening parent image: " << cpp_strerror(r)
+		       << dendl;
+      close_image(ictx->parent);
+      ictx->parent = NULL;
+      return r;
+    }
+
+    ictx->parent->snap_lock.get_write();
+    r = ictx->parent->get_snap_name(parent_snap_id, &ictx->parent->snap_name);
+    if (r < 0) {
+      lderr(ictx->cct) << "parent snapshot does not exist" << dendl;
+      ictx->parent->snap_lock.put_write();
+      close_image(ictx->parent);
+      ictx->parent = NULL;
+      return r;
+    }
+    ictx->parent->snap_set(ictx->parent->snap_name);
+    ictx->parent->parent_lock.get_write();
+    r = refresh_parent(ictx->parent);
+    if (r < 0) {
+      lderr(ictx->cct) << "error refreshing parent snapshot "
+		       << ictx->parent->id << " "
+		       << ictx->parent->snap_name << dendl;
+      ictx->parent->parent_lock.put_write();
+      ictx->parent->snap_lock.put_write();
+      close_image(ictx->parent);
+      ictx->parent = NULL;
+      return r;
+    }
+    ictx->parent->parent_lock.put_write();
+    ictx->parent->snap_lock.put_write();
+
+    return 0;
+  }
+
+  int get_parent_info(ImageCtx *ictx, string *parent_pool_name,
+		      string *parent_name, string *parent_snap_name)
+  {
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker l(ictx->snap_lock);
+    RWLock::RLocker l2(ictx->parent_lock);
+
+    parent_spec parent_spec;
+
+    if (ictx->snap_id == CEPH_NOSNAP) {
+      if (!ictx->parent)
+	return -ENOENT;
+      parent_spec = ictx->parent_md.spec;
+    } else {
+      r = ictx->get_parent_spec(ictx->snap_id, &parent_spec);
+      if (r < 0) {
+	lderr(ictx->cct) << "Can't find snapshot id" << ictx->snap_id << dendl;
+	return r;
+      }
+      if (parent_spec.pool_id == -1)
+	return -ENOENT;
+    }
+    if (parent_pool_name) {
+      Rados rados(ictx->md_ctx);
+      r = rados.pool_reverse_lookup(parent_spec.pool_id,
+				    parent_pool_name);
+      if (r < 0) {
+	lderr(ictx->cct) << "error looking up pool name" << cpp_strerror(r)
+			 << dendl;
+	return r;
+      }
+    }
+
+    if (parent_snap_name) {
+      RWLock::RLocker l(ictx->parent->snap_lock);
+      r = ictx->parent->get_snap_name(parent_spec.snap_id,
+				      parent_snap_name);
+      if (r < 0) {
+	lderr(ictx->cct) << "error finding parent snap name: "
+			 << cpp_strerror(r) << dendl;
+	return r;
+      }
+    }
+
+    if (parent_name) {
+      r = cls_client::dir_get_name(&ictx->parent->md_ctx, RBD_DIRECTORY,
+				   parent_spec.image_id, parent_name);
+      if (r < 0) {
+	lderr(ictx->cct) << "error getting parent image name: "
+			 << cpp_strerror(r) << dendl;
+	return r;
+      }
+    }
+
+    return 0;
+  }
+
+  int remove(IoCtx& io_ctx, const char *imgname, ProgressContext& prog_ctx)
+  {
+    CephContext *cct((CephContext *)io_ctx.cct());
+    ldout(cct, 20) << "remove " << &io_ctx << " " << imgname << dendl;
+
+    string id;
+    bool old_format = false;
+    bool unknown_format = true;
+    ImageCtx *ictx = new ImageCtx(imgname, "", NULL, io_ctx, false);
+    int r = open_image(ictx);
+    if (r < 0) {
+      ldout(cct, 2) << "error opening image: " << cpp_strerror(-r) << dendl;
+    } else {
+      string header_oid = ictx->header_oid;
+      old_format = ictx->old_format;
+      unknown_format = false;
+      id = ictx->id;
+
+      if (ictx->snaps.size()) {
+	lderr(cct) << "image has snapshots - not removing" << dendl;
+	close_image(ictx);
+	return -ENOTEMPTY;
+      }
+
+      std::list watchers;
+      r = io_ctx.list_watchers(header_oid, &watchers);
+      if (r < 0) {
+        lderr(cct) << "error listing watchers" << dendl;
+        close_image(ictx);
+        return r;
+      }
+      if (watchers.size() > 1) {
+        lderr(cct) << "image has watchers - not removing" << dendl;
+        close_image(ictx);
+        return -EBUSY;
+      }
+      assert(watchers.size() == 1);
+
+      ictx->md_lock.get_read();
+      trim_image(ictx, 0, prog_ctx);
+      ictx->md_lock.put_read();
+
+      ictx->parent_lock.get_read();
+      // struct assignment
+      parent_info parent_info = ictx->parent_md;
+      ictx->parent_lock.put_read();
+
+      r = cls_client::remove_child(&ictx->md_ctx, RBD_CHILDREN,
+				   parent_info.spec, id);
+      if (r < 0 && r != -ENOENT) {
+	lderr(cct) << "error removing child from children list" << dendl;
+        close_image(ictx);
+	return r;
+      }
+      close_image(ictx);
+
+      ldout(cct, 2) << "removing header..." << dendl;
+      r = io_ctx.remove(header_oid);
+      if (r < 0 && r != -ENOENT) {
+	lderr(cct) << "error removing header: " << cpp_strerror(-r) << dendl;
+	return r;
+      }
+    }
+
+    if (old_format || unknown_format) {
+      ldout(cct, 2) << "removing rbd image from directory..." << dendl;
+      r = tmap_rm(io_ctx, imgname);
+      old_format = (r == 0);
+      if (r < 0 && !unknown_format) {
+	lderr(cct) << "error removing img from old-style directory: "
+		   << cpp_strerror(-r) << dendl;
+	return r;
+      }
+    }
+    if (!old_format) {
+      ldout(cct, 2) << "removing id object..." << dendl;
+      r = io_ctx.remove(id_obj_name(imgname));
+      if (r < 0 && r != -ENOENT) {
+	lderr(cct) << "error removing id object: " << cpp_strerror(r) << dendl;
+	return r;
+      }
+
+      r = cls_client::dir_get_id(&io_ctx, RBD_DIRECTORY, imgname, &id);
+      if (r < 0 && r != -ENOENT) {
+	lderr(cct) << "error getting id of image" << dendl;
+	return r;
+      }
+
+      ldout(cct, 2) << "removing rbd image from directory..." << dendl;
+      r = cls_client::dir_remove_image(&io_ctx, RBD_DIRECTORY, imgname, id);
+      if (r < 0) {
+	lderr(cct) << "error removing img from new-style directory: "
+		   << cpp_strerror(-r) << dendl;
+	return r;
+      }
+    } 
+
+    ldout(cct, 2) << "done." << dendl;
+    return 0;
+  }
+
+  int resize_helper(ImageCtx *ictx, uint64_t size, ProgressContext& prog_ctx)
+  {
+    CephContext *cct = ictx->cct;
+
+    if (size == ictx->size) {
+      ldout(cct, 2) << "no change in size (" << ictx->size << " -> " << size
+		    << ")" << dendl;
+      return 0;
+    }
+
+    if (size > ictx->size) {
+      ldout(cct, 2) << "expanding image " << ictx->size << " -> " << size
+		    << dendl;
+      // TODO: make ictx->set_size
+    } else {
+      ldout(cct, 2) << "shrinking image " << ictx->size << " -> " << size
+		    << dendl;
+      trim_image(ictx, size, prog_ctx);
+    }
+    ictx->size = size;
+
+    int r;
+    if (ictx->old_format) {
+      // rewrite header
+      bufferlist bl;
+      ictx->header.image_size = size;
+      bl.append((const char *)&(ictx->header), sizeof(ictx->header));
+      r = ictx->md_ctx.write(ictx->header_oid, bl, bl.length(), 0);
+    } else {
+      r = cls_client::set_size(&(ictx->md_ctx), ictx->header_oid, size);
+    }
+
+    // TODO: remove this useless check
+    if (r == -ERANGE)
+      lderr(cct) << "operation might have conflicted with another client!"
+		 << dendl;
+    if (r < 0) {
+      lderr(cct) << "error writing header: " << cpp_strerror(-r) << dendl;
+      return r;
+    } else {
+      notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+    }
+
+    return 0;
+  }
+
+  int resize(ImageCtx *ictx, uint64_t size, ProgressContext& prog_ctx)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "resize " << ictx << " " << ictx->size << " -> "
+		   << size << dendl;
+
+    if (ictx->read_only)
+      return -EROFS;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::WLocker l(ictx->md_lock);
+    if (size < ictx->size && ictx->object_cacher) {
+      // need to invalidate since we're deleting objects, and
+      // ObjectCacher doesn't track non-existent objects
+      r = ictx->invalidate_cache();
+      if (r < 0)
+	return r;
+    }
+    resize_helper(ictx, size, prog_ctx);
+
+    ldout(cct, 2) << "done." << dendl;
+
+    ictx->perfcounter->inc(l_librbd_resize);
+    return 0;
+  }
+
+  int snap_list(ImageCtx *ictx, vector& snaps)
+  {
+    ldout(ictx->cct, 20) << "snap_list " << ictx << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+    bufferlist bl, bl2;
+
+    RWLock::RLocker l(ictx->snap_lock);
+    for (map::iterator it = ictx->snaps_by_name.begin();
+	 it != ictx->snaps_by_name.end(); ++it) {
+      snap_info_t info;
+      info.name = it->first;
+      info.id = it->second.id;
+      info.size = it->second.size;
+      snaps.push_back(info);
+    }
+
+    return 0;
+  }
+
+  bool snap_exists(ImageCtx *ictx, const char *snap_name)
+  {
+    ldout(ictx->cct, 20) << "snap_exists " << ictx << " " << snap_name << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker l(ictx->snap_lock);
+    return ictx->snaps_by_name.count(snap_name);
+  }
+
+
+  int add_snap(ImageCtx *ictx, const char *snap_name)
+  {
+    uint64_t snap_id;
+
+    int r = ictx->md_ctx.selfmanaged_snap_create(&snap_id);
+    if (r < 0) {
+      lderr(ictx->cct) << "failed to create snap id: " << cpp_strerror(-r)
+		       << dendl;
+      return r;
+    }
+
+    if (ictx->old_format) {
+      r = cls_client::old_snapshot_add(&ictx->md_ctx, ictx->header_oid,
+				       snap_id, snap_name);
+    } else {
+      r = cls_client::snapshot_add(&ictx->md_ctx, ictx->header_oid,
+				   snap_id, snap_name);
+    }
+
+    if (r < 0) {
+      lderr(ictx->cct) << "adding snapshot to header failed: "
+		       << cpp_strerror(r) << dendl;
+      return r;
+    }
+
+    return 0;
+  }
+
+  int rm_snap(ImageCtx *ictx, const char *snap_name)
+  {
+    int r;
+    if (ictx->old_format) {
+      r = cls_client::old_snapshot_remove(&ictx->md_ctx,
+					  ictx->header_oid, snap_name);
+    } else {
+      RWLock::RLocker l(ictx->snap_lock);
+      r = cls_client::snapshot_remove(&ictx->md_ctx,
+				      ictx->header_oid,
+				      ictx->get_snap_id(snap_name));
+    }
+
+    if (r < 0) {
+      lderr(ictx->cct) << "removing snapshot from header failed: "
+		       << cpp_strerror(r) << dendl;
+      return r;
+    }
+
+    return 0;
+  }
+
+  int ictx_check(ImageCtx *ictx)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "ictx_check " << ictx << dendl;
+    ictx->refresh_lock.Lock();
+    bool needs_refresh = ictx->last_refresh != ictx->refresh_seq;
+    ictx->refresh_lock.Unlock();
+
+    if (needs_refresh) {
+      RWLock::WLocker l(ictx->md_lock);
+
+      int r = ictx_refresh(ictx);
+      if (r < 0) {
+	lderr(cct) << "Error re-reading rbd header: " << cpp_strerror(-r)
+		   << dendl;
+	return r;
+      }
+    }
+    return 0;
+  }
+
+  int refresh_parent(ImageCtx *ictx) {
+    // close the parent if it changed or this image no longer needs
+    // to read from it
+    int r;
+    if (ictx->parent) {
+      uint64_t overlap;
+      r = ictx->get_parent_overlap(ictx->snap_id, &overlap);
+      if (r < 0)
+	return r;
+      if (!overlap ||
+	  ictx->parent->md_ctx.get_id() !=
+	  ictx->get_parent_pool_id(ictx->snap_id) ||
+	  ictx->parent->id != ictx->get_parent_image_id(ictx->snap_id) ||
+	  ictx->parent->snap_id != ictx->get_parent_snap_id(ictx->snap_id)) {
+	ictx->clear_nonexistence_cache();
+	close_image(ictx->parent);
+	ictx->parent = NULL;
+      }
+    }
+
+    if (ictx->get_parent_pool_id(ictx->snap_id) > -1 && !ictx->parent) {
+      r = open_parent(ictx);
+      if (r < 0) {
+	lderr(ictx->cct) << "error opening parent snapshot: "
+			 << cpp_strerror(r) << dendl;
+	return r;
+      }
+    }
+
+    return 0;
+  }
+
+  int ictx_refresh(ImageCtx *ictx)
+  {
+    CephContext *cct = ictx->cct;
+    bufferlist bl, bl2;
+
+    ldout(cct, 20) << "ictx_refresh " << ictx << dendl;
+
+    ictx->refresh_lock.Lock();
+    int refresh_seq = ictx->refresh_seq;
+    ictx->refresh_lock.Unlock();
+
+    ::SnapContext new_snapc;
+    bool new_snap = false;
+    vector snap_names;
+    vector snap_sizes;
+    vector snap_features;
+    vector snap_parents;
+    vector snap_protection;
+    {
+      RWLock::WLocker l(ictx->snap_lock);
+      {
+        int r;
+	RWLock::WLocker l2(ictx->parent_lock);
+	ictx->lockers.clear();
+	if (ictx->old_format) {
+	  r = read_header(ictx->md_ctx, ictx->header_oid, &ictx->header, NULL);
+	  if (r < 0) {
+	    lderr(cct) << "Error reading header: " << cpp_strerror(r) << dendl;
+	    return r;
+	  }
+	  r = cls_client::old_snapshot_list(&ictx->md_ctx, ictx->header_oid,
+					    &snap_names, &snap_sizes, &new_snapc);
+	  if (r < 0) {
+	    lderr(cct) << "Error listing snapshots: " << cpp_strerror(r)
+		       << dendl;
+	    return r;
+	  }
+	  ClsLockType lock_type = LOCK_NONE;
+	  r = rados::cls::lock::get_lock_info(&ictx->md_ctx, ictx->header_oid,
+					      RBD_LOCK_NAME, &ictx->lockers,
+					      &lock_type, &ictx->lock_tag);
+
+	  // If EOPNOTSUPP, treat image as if there are no locks (we can't
+	  // query them).
+
+	  // Ugly: OSDs prior to eed28daaf8927339c2ecae1b1b06c1b63678ab03
+	  // return EIO when the class isn't present; should be EOPNOTSUPP.
+	  // Treat EIO or EOPNOTSUPP the same for now, as LOCK_NONE.  Blech.
+
+	  if (r < 0 && ((r != -EOPNOTSUPP) && (r != -EIO))) {
+	    lderr(cct) << "Error getting lock info: " << cpp_strerror(r)
+		       << dendl;
+	    return r;
+	  }
+	  ictx->exclusive_locked = (lock_type == LOCK_EXCLUSIVE);
+	  ictx->order = ictx->header.options.order;
+	  ictx->size = ictx->header.image_size;
+	  ictx->object_prefix = ictx->header.block_name;
+	  ictx->init_layout();
+	} else {
+	  do {
+	    uint64_t incompatible_features;
+	    r = cls_client::get_mutable_metadata(&ictx->md_ctx, ictx->header_oid,
+						 &ictx->size, &ictx->features,
+						 &incompatible_features,
+						 &ictx->lockers,
+						 &ictx->exclusive_locked,
+						 &ictx->lock_tag,
+						 &new_snapc,
+						 &ictx->parent_md);
+	    if (r < 0) {
+	      lderr(cct) << "Error reading mutable metadata: " << cpp_strerror(r)
+			 << dendl;
+	      return r;
+	    }
+
+	    uint64_t unsupported = incompatible_features & ~RBD_FEATURES_ALL;
+	    if (unsupported) {
+	      lderr(ictx->cct) << "Image uses unsupported features: "
+			       << unsupported << dendl;
+	      return -ENOSYS;
+	    }
+
+	    r = cls_client::snapshot_list(&(ictx->md_ctx), ictx->header_oid,
+					  new_snapc.snaps, &snap_names,
+					  &snap_sizes, &snap_features,
+					  &snap_parents,
+					  &snap_protection);
+	    // -ENOENT here means we raced with snapshot deletion
+	    if (r < 0 && r != -ENOENT) {
+	      lderr(ictx->cct) << "snapc = " << new_snapc << dendl;
+	      lderr(ictx->cct) << "Error listing snapshots: " << cpp_strerror(r)
+			       << dendl;
+	      return r;
+	    }
+	  } while (r == -ENOENT);
+	}
+
+	for (size_t i = 0; i < new_snapc.snaps.size(); ++i) {
+	  uint64_t features = ictx->old_format ? 0 : snap_features[i];
+	  parent_info parent;
+	  if (!ictx->old_format)
+	    parent = snap_parents[i];
+	  vector::const_iterator it =
+	    find(ictx->snaps.begin(), ictx->snaps.end(), new_snapc.snaps[i].val);
+	  if (it == ictx->snaps.end()) {
+	    new_snap = true;
+	    ldout(cct, 20) << "new snapshot id=" << new_snapc.snaps[i].val
+			   << " name=" << snap_names[i]
+			   << " size=" << snap_sizes[i]
+			   << " features=" << features
+			   << dendl;
+	  }
+	}
+
+	ictx->snaps.clear();
+	ictx->snaps_by_name.clear();
+	for (size_t i = 0; i < new_snapc.snaps.size(); ++i) {
+	  uint64_t features = ictx->old_format ? 0 : snap_features[i];
+	  uint8_t protection_status = ictx->old_format ?
+	    (uint8_t)RBD_PROTECTION_STATUS_UNPROTECTED : snap_protection[i];
+	  parent_info parent;
+	  if (!ictx->old_format)
+	    parent = snap_parents[i];
+	  ictx->add_snap(snap_names[i], new_snapc.snaps[i].val,
+			 snap_sizes[i], features, parent, protection_status);
+	}
+
+	r = refresh_parent(ictx);
+	if (r < 0)
+	  return r;
+      } // release parent_lock
+
+      if (!new_snapc.is_valid()) {
+	lderr(cct) << "image snap context is invalid!" << dendl;
+	return -EIO;
+      }
+
+      ictx->snapc = new_snapc;
+
+      if (ictx->snap_id != CEPH_NOSNAP &&
+	  ictx->get_snap_id(ictx->snap_name) != ictx->snap_id) {
+	lderr(cct) << "tried to read from a snapshot that no longer exists: "
+		   << ictx->snap_name << dendl;
+	ictx->snap_exists = false;
+      }
+
+      ictx->data_ctx.selfmanaged_snap_set_write_ctx(ictx->snapc.seq, ictx->snaps);
+    } // release snap_lock
+
+    if (new_snap) {
+      _flush(ictx);
+    }
+
+    ictx->refresh_lock.Lock();
+    ictx->last_refresh = refresh_seq;
+    ictx->refresh_lock.Unlock();
+
+    return 0;
+  }
+
+  int snap_rollback(ImageCtx *ictx, const char *snap_name,
+		    ProgressContext& prog_ctx)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "snap_rollback " << ictx << " snap = " << snap_name
+		   << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::WLocker l(ictx->md_lock);
+    snap_t snap_id;
+    uint64_t new_size;
+    {
+      // need to drop snap_lock before invalidating cache
+      RWLock::RLocker l2(ictx->snap_lock);
+      if (!ictx->snap_exists)
+	return -ENOENT;
+
+      if (ictx->snap_id != CEPH_NOSNAP || ictx->read_only)
+	return -EROFS;
+
+      snap_id = ictx->get_snap_id(snap_name);
+      if (snap_id == CEPH_NOSNAP) {
+	lderr(cct) << "No such snapshot found." << dendl;
+	return -ENOENT;
+      }
+      new_size = ictx->get_image_size(snap_id);
+    }
+
+    // need to flush any pending writes before resizing and rolling back -
+    // writes might create new snapshots. Rolling back will replace
+    // the current version, so we have to invalidate that too.
+    r = ictx->invalidate_cache();
+    if (r < 0)
+      return r;
+
+    ldout(cct, 2) << "resizing to snapshot size..." << dendl;
+    NoOpProgressContext no_op;
+    r = resize_helper(ictx, new_size, no_op);
+    if (r < 0) {
+      lderr(cct) << "Error resizing to snapshot size: "
+		 << cpp_strerror(-r) << dendl;
+      return r;
+    }
+
+    r = rollback_image(ictx, snap_id, prog_ctx);
+    if (r < 0) {
+      lderr(cct) << "Error rolling back image: " << cpp_strerror(-r) << dendl;
+      return r;
+    }
+
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+
+    ictx->perfcounter->inc(l_librbd_snap_rollback);
+    return r;
+  }
+
+  struct CopyProgressCtx {
+    CopyProgressCtx(ProgressContext &p)
+      : destictx(NULL), src_size(0), prog_ctx(p)
+    { }
+
+    ImageCtx *destictx;
+    uint64_t src_size;
+    ProgressContext &prog_ctx;
+  };
+
+  int do_copy_extent(uint64_t offset, size_t len, const char *buf, void *data)
+  {
+    CopyProgressCtx *cp = reinterpret_cast(data);
+    cp->prog_ctx.update_progress(offset, cp->src_size);
+    int ret = 0;
+    if (buf) {
+      ret = write(cp->destictx, offset, len, buf);
+    }
+    return ret;
+  }
+
+  int copy(ImageCtx *src, IoCtx& dest_md_ctx, const char *destname,
+	   ProgressContext &prog_ctx)
+  {
+    CephContext *cct = (CephContext *)dest_md_ctx.cct();
+    ldout(cct, 20) << "copy " << src->name
+		   << (src->snap_name.length() ? "@" + src->snap_name : "")
+		   << " -> " << destname << dendl;
+    int order = src->order;
+
+    src->md_lock.get_read();
+    src->snap_lock.get_read();
+    uint64_t src_size = src->get_image_size(src->snap_id);
+    src->snap_lock.put_read();
+    src->md_lock.put_read();
+
+    int r = create(dest_md_ctx, destname, src_size, src->old_format,
+		   src->features, &order, src->stripe_unit, src->stripe_count);
+    if (r < 0) {
+      lderr(cct) << "header creation failed" << dendl;
+      return r;
+    }
+
+    ImageCtx *dest = new librbd::ImageCtx(destname, "", NULL,
+					  dest_md_ctx, false);
+    r = open_image(dest);
+    if (r < 0) {
+      lderr(cct) << "failed to read newly created header" << dendl;
+      return r;
+    }
+
+    r = copy(src, dest, prog_ctx);
+    close_image(dest);
+    return r;
+  }
+
+  class C_CopyWrite : public Context {
+  public:
+    C_CopyWrite(SimpleThrottle *throttle, bufferlist *bl)
+      : m_throttle(throttle), m_bl(bl) {}
+    virtual void finish(int r) {
+      delete m_bl;
+      m_throttle->end_op(r);
+    }
+  private:
+    SimpleThrottle *m_throttle;
+    bufferlist *m_bl;
+  };
+
+  class C_CopyRead : public Context {
+  public:
+    C_CopyRead(SimpleThrottle *throttle, ImageCtx *dest, uint64_t offset,
+	       bufferlist *bl)
+      : m_throttle(throttle), m_dest(dest), m_offset(offset), m_bl(bl) {
+      m_throttle->start_op();
+    }
+    virtual void finish(int r) {
+      if (r < 0) {
+	lderr(m_dest->cct) << "error reading from source image at offset "
+			   << m_offset << ": " << cpp_strerror(r) << dendl;
+	delete m_bl;
+	m_throttle->end_op(r);
+	return;
+      }
+      assert(m_bl->length() == (size_t)r);
+
+      if (m_bl->is_zero()) {
+	delete m_bl;
+	m_throttle->end_op(r);
+	return;
+      }
+
+      Context *ctx = new C_CopyWrite(m_throttle, m_bl);
+      AioCompletion *comp = aio_create_completion_internal(ctx, rbd_ctx_cb);
+      r = aio_write(m_dest, m_offset, m_bl->length(), m_bl->c_str(), comp);
+      if (r < 0) {
+	ctx->complete(r);
+	comp->release();
+	lderr(m_dest->cct) << "error writing to destination image at offset "
+			   << m_offset << ": " << cpp_strerror(r) << dendl;
+      }
+    }
+  private:
+    SimpleThrottle *m_throttle;
+    ImageCtx *m_dest;
+    uint64_t m_offset;
+    bufferlist *m_bl;
+  };
+
+  int copy(ImageCtx *src, ImageCtx *dest, ProgressContext &prog_ctx)
+  {
+    src->md_lock.get_read();
+    src->snap_lock.get_read();
+    uint64_t src_size = src->get_image_size(src->snap_id);
+    src->snap_lock.put_read();
+    src->md_lock.put_read();
+
+    dest->md_lock.get_read();
+    dest->snap_lock.get_read();
+    uint64_t dest_size = dest->get_image_size(dest->snap_id);
+    dest->snap_lock.put_read();
+    dest->md_lock.put_read();
+
+    CephContext *cct = src->cct;
+    if (dest_size < src_size) {
+      lderr(cct) << " src size " << src_size << " >= dest size "
+		 << dest_size << dendl;
+      return -EINVAL;
+    }
+    int r;
+    SimpleThrottle throttle(cct->_conf->rbd_concurrent_management_ops, false);
+    uint64_t period = src->get_stripe_period();
+    for (uint64_t offset = 0; offset < src_size; offset += period) {
+      uint64_t len = min(period, src_size - offset);
+      bufferlist *bl = new bufferlist();
+      Context *ctx = new C_CopyRead(&throttle, dest, offset, bl);
+      AioCompletion *comp = aio_create_completion_internal(ctx, rbd_ctx_cb);
+      r = aio_read(src, offset, len, NULL, bl, comp);
+      if (r < 0) {
+	ctx->complete(r);
+	comp->release();
+	throttle.wait_for_ret();
+	lderr(cct) << "could not read from source image from "
+		   << offset << " to " << offset + len << ": "
+		   << cpp_strerror(r) << dendl;
+	return r;
+      }
+      prog_ctx.update_progress(offset, src_size);
+    }
+
+    r = throttle.wait_for_ret();
+    if (r >= 0)
+      prog_ctx.update_progress(src_size, src_size);
+    return r;
+  }
+
+  // common snap_set functionality for snap_set and open_image
+
+  int _snap_set(ImageCtx *ictx, const char *snap_name)
+  {
+    RWLock::WLocker l1(ictx->snap_lock);
+    RWLock::WLocker l2(ictx->parent_lock);
+    int r;
+    if ((snap_name != NULL) && (strlen(snap_name) != 0)) {
+      r = ictx->snap_set(snap_name);
+    } else {
+      ictx->snap_unset();
+      r = 0;
+    }
+    if (r < 0) {
+      return r;
+    }
+    refresh_parent(ictx);
+    return 0;
+  }
+
+  int snap_set(ImageCtx *ictx, const char *snap_name)
+  {
+    ldout(ictx->cct, 20) << "snap_set " << ictx << " snap = "
+			 << (snap_name ? snap_name : "NULL") << dendl;
+    // ignore return value, since we may be set to a non-existent
+    // snapshot and the user is trying to fix that
+    ictx_check(ictx);
+    if (ictx->object_cacher) {
+      // complete pending writes before we're set to a snapshot and
+      // get -EROFS for writes
+      RWLock::WLocker l(ictx->md_lock);
+      ictx->flush_cache();
+    }
+    return _snap_set(ictx, snap_name);
+  }
+
+  int open_image(ImageCtx *ictx)
+  {
+    ldout(ictx->cct, 20) << "open_image: ictx = " << ictx
+			 << " name = '" << ictx->name
+			 << "' id = '" << ictx->id
+			 << "' snap_name = '"
+			 << ictx->snap_name << "'" << dendl;
+    int r = ictx->init();
+    if (r < 0)
+      goto err_close;
+
+    if (!ictx->read_only) {
+      r = ictx->register_watch();
+      if (r < 0) {
+	lderr(ictx->cct) << "error registering a watch: " << cpp_strerror(r)
+			 << dendl;
+	goto err_close;
+      }
+    }
+
+    ictx->md_lock.get_write();
+    r = ictx_refresh(ictx);
+    ictx->md_lock.put_write();
+    if (r < 0)
+      goto err_close;
+
+    if ((r = _snap_set(ictx, ictx->snap_name.c_str())) < 0)
+      goto err_close;
+
+    return 0;
+
+  err_close:
+    close_image(ictx);
+    return r;
+  }
+
+  void close_image(ImageCtx *ictx)
+  {
+    ldout(ictx->cct, 20) << "close_image " << ictx << dendl;
+    if (ictx->object_cacher)
+      ictx->shutdown_cache(); // implicitly flushes
+    else
+      flush(ictx);
+
+    if (ictx->parent) {
+      close_image(ictx->parent);
+      ictx->parent = NULL;
+    }
+
+    if (ictx->wctx)
+      ictx->unregister_watch();
+
+    delete ictx;
+  }
+
+  // 'flatten' child image by copying all parent's blocks
+  int flatten(ImageCtx *ictx, ProgressContext &prog_ctx)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "flatten" << dendl;
+
+    if (ictx->read_only)
+      return -EROFS;
+
+    int r;
+    // ictx_check also updates parent data
+    if ((r = ictx_check(ictx)) < 0) {
+      lderr(cct) << "ictx_check failed" << dendl;
+      return r;
+    }
+
+    uint64_t object_size;
+    uint64_t period;
+    uint64_t overlap;
+    uint64_t overlap_periods;
+    uint64_t overlap_objects;
+    ::SnapContext snapc;
+
+    {
+      RWLock::RLocker l(ictx->md_lock);
+      RWLock::RLocker l2(ictx->snap_lock);
+      RWLock::RLocker l3(ictx->parent_lock);
+
+      // can't flatten a non-clone
+      if (ictx->parent_md.spec.pool_id == -1) {
+	lderr(cct) << "image has no parent" << dendl;
+	return -EINVAL;
+      }
+      if (ictx->snap_id != CEPH_NOSNAP || ictx->read_only) {
+	lderr(cct) << "snapshots cannot be flattened" << dendl;
+	return -EROFS;
+      }
+
+      snapc = ictx->snapc;
+      assert(ictx->parent != NULL);
+      assert(ictx->parent_md.overlap <= ictx->size);
+
+      object_size = ictx->get_object_size();
+      period = ictx->get_stripe_period();
+      overlap = ictx->parent_md.overlap;
+      overlap_periods = (overlap + period - 1) / period;
+      overlap_objects = overlap_periods * ictx->get_stripe_count();
+    }
+
+    SimpleThrottle throttle(cct->_conf->rbd_concurrent_management_ops, false);
+
+    for (uint64_t ono = 0; ono < overlap_objects; ono++) {
+      {
+	RWLock::RLocker l(ictx->parent_lock);
+	// stop early if the parent went away - it just means
+	// another flatten finished first, so this one is useless.
+	if (!ictx->parent) {
+	  r = 0;
+	  goto err;
+	}
+      }
+
+      // map child object onto the parent
+      vector > objectx;
+      Striper::extent_to_file(cct, &ictx->layout,
+			    ono, 0, object_size,
+			    objectx);
+      uint64_t object_overlap = ictx->prune_parent_extents(objectx, overlap);
+      assert(object_overlap <= object_size);
+
+      bufferlist bl;
+      string oid = ictx->get_object_name(ono);
+      Context *comp = new C_SimpleThrottle(&throttle);
+      AioWrite *req = new AioWrite(ictx, oid, ono, 0, objectx, object_overlap,
+				   bl, snapc, CEPH_NOSNAP, comp);
+      r = req->send();
+      if (r < 0) {
+	lderr(cct) << "failed to flatten object " << oid << dendl;
+	goto err;
+      }
+
+      prog_ctx.update_progress(ono, overlap_objects);
+    }
+
+    r = throttle.wait_for_ret();
+    if (r < 0) {
+      lderr(cct) << "failed to flatten at least one object: "
+		 << cpp_strerror(r) << dendl;
+      goto err;
+    }
+
+    // remove parent from this (base) image
+    r = cls_client::remove_parent(&ictx->md_ctx, ictx->header_oid);
+    if (r < 0) {
+      lderr(cct) << "error removing parent" << dendl;
+      return r;
+    }
+
+    // and if there are no snaps, remove from the children object as well
+    // (if snapshots remain, they have their own parent info, and the child
+    // will be removed when the last snap goes away)
+    ictx->snap_lock.get_read();
+    if (ictx->snaps.empty()) {
+      ldout(cct, 2) << "removing child from children list..." << dendl;
+      int r = cls_client::remove_child(&ictx->md_ctx, RBD_CHILDREN,
+				       ictx->parent_md.spec, ictx->id);
+      if (r < 0) {
+	lderr(cct) << "error removing child from children list" << dendl;
+	ictx->snap_lock.put_read();
+	return r;
+      }
+    }
+    ictx->snap_lock.put_read();
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+
+    ldout(cct, 20) << "finished flattening" << dendl;
+
+    return 0;
+
+  err:
+    throttle.wait_for_ret();
+    return r;
+  }
+
+  int list_lockers(ImageCtx *ictx,
+		   std::list *lockers,
+		   bool *exclusive,
+		   string *tag)
+  {
+    ldout(ictx->cct, 20) << "list_locks on image " << ictx << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker locker(ictx->md_lock);
+    if (exclusive)
+      *exclusive = ictx->exclusive_locked;
+    if (tag)
+      *tag = ictx->lock_tag;
+    if (lockers) {
+      lockers->clear();
+      map::const_iterator it;
+      for (it = ictx->lockers.begin(); it != ictx->lockers.end(); ++it) {
+	locker_t locker;
+	locker.client = stringify(it->first.locker);
+	locker.cookie = it->first.cookie;
+	locker.address = stringify(it->second.addr);
+	lockers->push_back(locker);
+      }
+    }
+
+    return 0;
+  }
+
+  int lock(ImageCtx *ictx, bool exclusive, const string& cookie,
+	   const string& tag)
+  {
+    ldout(ictx->cct, 20) << "lock image " << ictx << " exclusive=" << exclusive
+			 << " cookie='" << cookie << "' tag='" << tag << "'"
+			 << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    /**
+     * If we wanted we could do something more intelligent, like local
+     * checks that we think we will succeed. But for now, let's not
+     * duplicate that code.
+     */
+    RWLock::RLocker locker(ictx->md_lock);
+    r = rados::cls::lock::lock(&ictx->md_ctx, ictx->header_oid, RBD_LOCK_NAME,
+			       exclusive ? LOCK_EXCLUSIVE : LOCK_SHARED,
+			       cookie, tag, "", utime_t(), 0);
+    if (r < 0)
+      return r;
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+    return 0;
+  }
+
+  int unlock(ImageCtx *ictx, const string& cookie)
+  {
+    ldout(ictx->cct, 20) << "unlock image " << ictx
+			 << " cookie='" << cookie << "'" << dendl;
+
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::RLocker locker(ictx->md_lock);
+    r = rados::cls::lock::unlock(&ictx->md_ctx, ictx->header_oid,
+				 RBD_LOCK_NAME, cookie);
+    if (r < 0)
+      return r;
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+    return 0;
+  }
+
+  int break_lock(ImageCtx *ictx, const string& client,
+		 const string& cookie)
+  {
+    ldout(ictx->cct, 20) << "break_lock image " << ictx << " client='" << client
+			 << "' cookie='" << cookie << "'" << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    entity_name_t lock_client;
+    if (!lock_client.parse(client)) {
+      lderr(ictx->cct) << "Unable to parse client '" << client
+		       << "'" << dendl;
+      return -EINVAL;
+    }
+    RWLock::RLocker locker(ictx->md_lock);
+    r = rados::cls::lock::break_lock(&ictx->md_ctx, ictx->header_oid,
+				     RBD_LOCK_NAME, cookie, lock_client);
+    if (r < 0)
+      return r;
+    notify_change(ictx->md_ctx, ictx->header_oid, NULL, ictx);
+    return 0;
+  }
+
+  void rbd_ctx_cb(completion_t cb, void *arg)
+  {
+    Context *ctx = reinterpret_cast(arg);
+    AioCompletion *comp = reinterpret_cast(cb);
+    ctx->complete(comp->get_return_value());
+    comp->release();
+  }
+
+  int64_t read_iterate(ImageCtx *ictx, uint64_t off, uint64_t len,
+		       int (*cb)(uint64_t, size_t, const char *, void *),
+		       void *arg)
+  {
+    utime_t start_time, elapsed;
+
+    ldout(ictx->cct, 20) << "read_iterate " << ictx << " off = " << off
+			 << " len = " << len << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    uint64_t mylen = len;
+    r = clip_io(ictx, off, &mylen);
+    if (r < 0)
+      return r;
+
+    int64_t total_read = 0;
+    uint64_t period = ictx->get_stripe_period();
+    uint64_t left = mylen;
+
+    start_time = ceph_clock_now(ictx->cct);
+    while (left > 0) {
+      uint64_t period_off = off - (off % period);
+      uint64_t read_len = min(period_off + period - off, left);
+
+      bufferlist bl;
+
+      Mutex mylock("IoCtxImpl::write::mylock");
+      Cond cond;
+      bool done;
+      int ret;
+
+      Context *ctx = new C_SafeCond(&mylock, &cond, &done, &ret);
+      AioCompletion *c = aio_create_completion_internal(ctx, rbd_ctx_cb);
+      r = aio_read(ictx, off, read_len, NULL, &bl, c);
+      if (r < 0) {
+	c->release();
+	delete ctx;
+	return r;
+      }
+
+      mylock.Lock();
+      while (!done)
+	cond.Wait(mylock);
+      mylock.Unlock();
+
+      if (ret < 0)
+	return ret;
+
+      r = cb(total_read, ret, bl.c_str(), arg);
+      if (r < 0)
+	return r;
+
+      total_read += ret;
+      left -= ret;
+      off += ret;
+    }
+
+    elapsed = ceph_clock_now(ictx->cct) - start_time;
+    ictx->perfcounter->tinc(l_librbd_rd_latency, elapsed);
+    ictx->perfcounter->inc(l_librbd_rd);
+    ictx->perfcounter->inc(l_librbd_rd_bytes, mylen);
+    return total_read;
+  }
+
+  int simple_diff_cb(uint64_t off, size_t len, int exists, void *arg)
+  {
+    // This reads the existing extents in a parent from the beginning
+    // of time.  Since images are thin-provisioned, the extents will
+    // always represent data, not holes.
+    assert(exists);
+    interval_set *diff = static_cast *>(arg);
+    diff->insert(off, len);
+    return 0;
+  }
+
+
+  int diff_iterate(ImageCtx *ictx, const char *fromsnapname,
+		   uint64_t off, uint64_t len,
+		   int (*cb)(uint64_t, size_t, int, void *),
+		   void *arg)
+  {
+    utime_t start_time, elapsed;
+
+    ldout(ictx->cct, 20) << "diff_iterate " << ictx << " off = " << off
+			 << " len = " << len << dendl;
+
+    // ensure previous writes are visible to listsnaps
+    _flush(ictx);
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    r = clip_io(ictx, off, &len);
+    if (r < 0)
+      return r;
+
+    librados::IoCtx head_ctx;
+
+    ictx->md_lock.get_read();
+    ictx->snap_lock.get_read();
+    head_ctx.dup(ictx->data_ctx);
+    snap_t from_snap_id = 0;
+    uint64_t from_size = 0;
+    if (fromsnapname) {
+      from_snap_id = ictx->get_snap_id(fromsnapname);
+      from_size = ictx->get_image_size(from_snap_id);
+    }
+    snap_t end_snap_id = ictx->snap_id;
+    uint64_t end_size = ictx->get_image_size(end_snap_id);
+    ictx->snap_lock.put_read();
+    ictx->md_lock.put_read();
+    if (from_snap_id == CEPH_NOSNAP) {
+      return -ENOENT;
+    }
+    if (from_snap_id == end_snap_id) {
+      // no diff.
+      return 0;
+    }
+    if (from_snap_id >= end_snap_id) {
+      return -EINVAL;
+    }
+
+    // we must list snaps via the head, not end snap
+    head_ctx.snap_set_read(CEPH_SNAPDIR);
+
+    ldout(ictx->cct, 20) << "diff_iterate from " << from_snap_id << " to " << end_snap_id
+			 << " size from " << from_size << " to " << end_size << dendl;
+
+    // FIXME: if end_size > from_size, we could read_iterate for the
+    // final part, and skip the listsnaps op.
+
+    // check parent overlap only if we are comparing to the beginning of time
+    interval_set parent_diff;
+    if (from_snap_id == 0) {
+      ictx->parent_lock.get_read();
+      uint64_t overlap = end_size;
+      ictx->get_parent_overlap(from_snap_id, &overlap);
+      r = 0;
+      if (ictx->parent && overlap > 0) {
+	ldout(ictx->cct, 10) << " first getting parent diff" << dendl;
+	r = diff_iterate(ictx->parent, NULL, 0, overlap, simple_diff_cb, &parent_diff);
+      }
+      ictx->parent_lock.put_read();
+      if (r < 0)
+	return r;
+    }
+
+    uint64_t period = ictx->get_stripe_period();
+    uint64_t left = len;
+
+    while (left > 0) {
+      uint64_t period_off = off - (off % period);
+      uint64_t read_len = min(period_off + period - off, left);
+
+      // map to extents
+      map > object_extents;
+      Striper::file_to_extents(ictx->cct, ictx->format_string, &ictx->layout,
+			       off, read_len, 0, object_extents, 0);
+
+      // get snap info for each object
+      for (map >::iterator p = object_extents.begin();
+	   p != object_extents.end();
+	   ++p) {
+	ldout(ictx->cct, 20) << "diff_iterate object " << p->first << dendl;
+
+	librados::snap_set_t snap_set;
+	int r = head_ctx.list_snaps(p->first.name, &snap_set);
+	if (r == -ENOENT) {
+	  if (from_snap_id == 0 && !parent_diff.empty()) {
+	    // report parent diff instead
+	    for (vector::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+	      for (vector >::iterator r = q->buffer_extents.begin();
+		   r != q->buffer_extents.end();
+		   ++r) {
+		interval_set o;
+		o.insert(off + r->first, r->second);
+		o.intersection_of(parent_diff);
+		ldout(ictx->cct, 20) << " reporting parent overlap " << o << dendl;
+		for (interval_set::iterator s = o.begin(); s != o.end(); ++s) {
+		  cb(s.get_start(), s.get_len(), true, arg);
+		}
+	      }
+	    }
+	  }
+	  continue;
+	}
+	if (r < 0)
+	  return r;
+
+	// calc diff from from_snap_id -> to_snap_id
+	interval_set diff;
+	bool end_exists;
+	calc_snap_set_diff(ictx->cct, snap_set,
+			   from_snap_id,
+			   end_snap_id,
+			   &diff, &end_exists);
+	ldout(ictx->cct, 20) << "  diff " << diff << " end_exists=" << end_exists << dendl;
+	if (diff.empty())
+	  continue;
+
+	for (vector::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+	  ldout(ictx->cct, 20) << "diff_iterate object " << p->first
+			       << " extent " << q->offset << "~" << q->length
+			       << " from " << q->buffer_extents
+			       << dendl;
+	  uint64_t opos = q->offset;
+	  for (vector >::iterator r = q->buffer_extents.begin();
+	       r != q->buffer_extents.end();
+	       ++r) {
+	    interval_set overlap;  // object extents
+	    overlap.insert(opos, r->second);
+	    overlap.intersection_of(diff);
+	    ldout(ictx->cct, 20) << " opos " << opos
+				 << " buf " << r->first << "~" << r->second
+				 << " overlap " << overlap
+				 << dendl;
+	    for (interval_set::iterator s = overlap.begin();
+		 s != overlap.end();
+		 ++s) {
+	      uint64_t su_off = s.get_start() - opos;
+	      uint64_t logical_off = off + r->first + su_off;
+	      ldout(ictx->cct, 20) << "   overlap extent " << s.get_start() << "~" << s.get_len()
+				   << " logical "
+				   << logical_off << "~" << s.get_len()
+				   << dendl;
+	      cb(logical_off, s.get_len(), end_exists, arg);
+	    }
+	    opos += r->second;
+	  }
+	  assert(opos == q->offset + q->length);
+	}
+      }
+
+      left -= read_len;
+      off += read_len;
+    }
+
+    return 0;
+  }
+
+  int simple_read_cb(uint64_t ofs, size_t len, const char *buf, void *arg)
+  {
+    char *dest_buf = (char *)arg;
+    if (buf)
+      memcpy(dest_buf + ofs, buf, len);
+    else
+      memset(dest_buf + ofs, 0, len);
+
+    return 0;
+  }
+
+  ssize_t read(ImageCtx *ictx, uint64_t ofs, size_t len, char *buf)
+  {
+    vector > extents;
+    extents.push_back(make_pair(ofs, len));
+    return read(ictx, extents, buf, NULL);
+  }
+
+  ssize_t read(ImageCtx *ictx, const vector >& image_extents, char *buf, bufferlist *pbl)
+  {
+    Mutex mylock("IoCtxImpl::write::mylock");
+    Cond cond;
+    bool done;
+    int ret;
+
+    Context *ctx = new C_SafeCond(&mylock, &cond, &done, &ret);
+    AioCompletion *c = aio_create_completion_internal(ctx, rbd_ctx_cb);
+    int r = aio_read(ictx, image_extents, buf, pbl, c);
+    if (r < 0) {
+      c->release();
+      delete ctx;
+      return r;
+    }
+
+    mylock.Lock();
+    while (!done)
+      cond.Wait(mylock);
+    mylock.Unlock();
+
+    return ret;
+  }
+
+  ssize_t write(ImageCtx *ictx, uint64_t off, size_t len, const char *buf)
+  {
+    utime_t start_time, elapsed;
+    ldout(ictx->cct, 20) << "write " << ictx << " off = " << off << " len = "
+			 << len << dendl;
+
+    start_time = ceph_clock_now(ictx->cct);
+    Mutex mylock("librbd::write::mylock");
+    Cond cond;
+    bool done;
+    int ret;
+
+    uint64_t mylen = len;
+    int r = clip_io(ictx, off, &mylen);
+    if (r < 0)
+      return r;
+
+    Context *ctx = new C_SafeCond(&mylock, &cond, &done, &ret);
+    AioCompletion *c = aio_create_completion_internal(ctx, rbd_ctx_cb);
+    r = aio_write(ictx, off, mylen, buf, c);
+    if (r < 0) {
+      c->release();
+      delete ctx;
+      return r;
+    }
+
+    mylock.Lock();
+    while (!done)
+      cond.Wait(mylock);
+    mylock.Unlock();
+
+    if (ret < 0)
+      return ret;
+
+    elapsed = ceph_clock_now(ictx->cct) - start_time;
+    ictx->perfcounter->tinc(l_librbd_wr_latency, elapsed);
+    ictx->perfcounter->inc(l_librbd_wr);
+    ictx->perfcounter->inc(l_librbd_wr_bytes, mylen);
+    return mylen;
+  }
+
+  int discard(ImageCtx *ictx, uint64_t off, uint64_t len)
+  {
+    utime_t start_time, elapsed;
+    ldout(ictx->cct, 20) << "discard " << ictx << " off = " << off << " len = "
+			 << len << dendl;
+
+    start_time = ceph_clock_now(ictx->cct);
+    Mutex mylock("librbd::discard::mylock");
+    Cond cond;
+    bool done;
+    int ret;
+
+    Context *ctx = new C_SafeCond(&mylock, &cond, &done, &ret);
+    AioCompletion *c = aio_create_completion_internal(ctx, rbd_ctx_cb);
+    int r = aio_discard(ictx, off, len, c);
+    if (r < 0) {
+      c->release();
+      delete ctx;
+      return r;
+    }
+
+    mylock.Lock();
+    while (!done)
+      cond.Wait(mylock);
+    mylock.Unlock();
+
+    if (ret < 0)
+      return ret;
+
+    elapsed = ceph_clock_now(ictx->cct) - start_time;
+    ictx->perfcounter->inc(l_librbd_discard_latency, elapsed);
+    ictx->perfcounter->inc(l_librbd_discard);
+    ictx->perfcounter->inc(l_librbd_discard_bytes, len);
+    return len;
+  }
+
+  ssize_t handle_sparse_read(CephContext *cct,
+			     bufferlist data_bl,
+			     uint64_t block_ofs,
+			     const map &data_map,
+			     uint64_t buf_ofs,   // offset into buffer
+			     size_t buf_len,     // length in buffer (not size of buffer!)
+			     char *dest_buf)
+  {
+    uint64_t bl_ofs = 0;
+    size_t buf_left = buf_len;
+
+    for (map::const_iterator iter = data_map.begin();
+	 iter != data_map.end();
+	 ++iter) {
+      uint64_t extent_ofs = iter->first;
+      size_t extent_len = iter->second;
+
+      ldout(cct, 10) << "extent_ofs=" << extent_ofs
+		     << " extent_len=" << extent_len << dendl;
+      ldout(cct, 10) << "block_ofs=" << block_ofs << dendl;
+
+      /* a hole? */
+      if (extent_ofs > block_ofs) {
+	uint64_t gap = extent_ofs - block_ofs;
+	ldout(cct, 10) << "<1>zeroing " << buf_ofs << "~" << gap << dendl;
+	memset(dest_buf + buf_ofs, 0, gap);
+
+	buf_ofs += gap;
+	buf_left -= gap;
+	block_ofs = extent_ofs;
+      } else if (extent_ofs < block_ofs) {
+	assert(0 == "osd returned data prior to what we asked for");
+	return -EIO;
+      }
+
+      if (bl_ofs + extent_len > (buf_ofs + buf_left)) {
+	assert(0 == "osd returned more data than we asked for");
+	return -EIO;
+      }
+
+      /* data */
+      ldout(cct, 10) << "<2>copying " << buf_ofs << "~" << extent_len
+		     << " from ofs=" << bl_ofs << dendl;
+      memcpy(dest_buf + buf_ofs, data_bl.c_str() + bl_ofs, extent_len);
+
+      bl_ofs += extent_len;
+      buf_ofs += extent_len;
+      assert(buf_left >= extent_len);
+      buf_left -= extent_len;
+      block_ofs += extent_len;
+    }
+
+    /* last hole */
+    if (buf_left > 0) {
+      ldout(cct, 10) << "<3>zeroing " << buf_ofs << "~" << buf_left << dendl;
+      memset(dest_buf + buf_ofs, 0, buf_left);
+    }
+
+    return buf_len;
+  }
+
+  void rados_req_cb(rados_completion_t c, void *arg)
+  {
+    AioRequest *req = reinterpret_cast(arg);
+    req->complete(rados_aio_get_return_value(c));
+  }
+
+  void rados_ctx_cb(rados_completion_t c, void *arg)
+  {
+    Context *comp = reinterpret_cast(arg);
+    comp->complete(rados_aio_get_return_value(c));
+  }
+
+  // validate extent against image size; clip to image size if necessary
+  int clip_io(ImageCtx *ictx, uint64_t off, uint64_t *len)
+  {
+    ictx->md_lock.get_read();
+    ictx->snap_lock.get_read();
+    uint64_t image_size = ictx->get_image_size(ictx->snap_id);
+    bool snap_exists = ictx->snap_exists;
+    ictx->snap_lock.put_read();
+    ictx->md_lock.put_read();
+
+    if (!snap_exists)
+      return -ENOENT;
+
+    // special-case "len == 0" requests: always valid
+    if (*len == 0)
+      return 0;
+
+    // can't start past end
+    if (off >= image_size)
+      return -EINVAL;
+
+    // clip requests that extend past end to just end
+    if ((off + *len) > image_size)
+      *len = (size_t)(image_size - off);
+
+    return 0;
+  }
+
+  int aio_flush(ImageCtx *ictx, AioCompletion *c)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "aio_flush " << ictx << " completion " << c <<  dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    ictx->user_flushed();
+
+    c->get();
+    c->add_request();
+    c->init_time(ictx, AIO_TYPE_FLUSH);
+    C_AioWrite *req_comp = new C_AioWrite(cct, c);
+    if (ictx->object_cacher) {
+      ictx->flush_cache_aio(req_comp);
+    } else {
+      librados::AioCompletion *rados_completion =
+	librados::Rados::aio_create_completion(req_comp, NULL, rados_ctx_cb);
+      ictx->data_ctx.aio_flush_async(rados_completion);
+      rados_completion->release();
+    }
+    c->finish_adding_requests(cct);
+    c->put();
+    ictx->perfcounter->inc(l_librbd_aio_flush);
+
+    return 0;
+  }
+
+  int flush(ImageCtx *ictx)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "flush " << ictx << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    ictx->user_flushed();
+    r = _flush(ictx);
+    ictx->perfcounter->inc(l_librbd_flush);
+    return r;
+  }
+
+  int _flush(ImageCtx *ictx)
+  {
+    CephContext *cct = ictx->cct;
+    int r;
+    // flush any outstanding writes
+    if (ictx->object_cacher) {
+      r = ictx->flush_cache();
+    } else {
+      r = ictx->data_ctx.aio_flush();
+    }
+
+    if (r)
+      lderr(cct) << "_flush " << ictx << " r = " << r << dendl;
+
+    return r;
+  }
+
+  int invalidate_cache(ImageCtx *ictx)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "invalidate_cache " << ictx << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    RWLock::WLocker l(ictx->md_lock);
+    return ictx->invalidate_cache();
+  }
+
+  int aio_write(ImageCtx *ictx, uint64_t off, size_t len, const char *buf,
+		AioCompletion *c)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "aio_write " << ictx << " off = " << off << " len = "
+		   << len << " buf = " << (void*)buf << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    uint64_t mylen = len;
+    r = clip_io(ictx, off, &mylen);
+    if (r < 0)
+      return r;
+
+    ictx->snap_lock.get_read();
+    snapid_t snap_id = ictx->snap_id;
+    ::SnapContext snapc = ictx->snapc;
+    ictx->parent_lock.get_read();
+    uint64_t overlap = 0;
+    ictx->get_parent_overlap(ictx->snap_id, &overlap);
+    ictx->parent_lock.put_read();
+    ictx->snap_lock.put_read();
+
+    if (snap_id != CEPH_NOSNAP || ictx->read_only)
+      return -EROFS;
+
+    ldout(cct, 20) << "  parent overlap " << overlap << dendl;
+
+    // map
+    vector extents;
+    if (len > 0) {
+      Striper::file_to_extents(ictx->cct, ictx->format_string,
+			       &ictx->layout, off, mylen, 0, extents);
+    }
+
+    c->get();
+    c->init_time(ictx, AIO_TYPE_WRITE);
+    for (vector::iterator p = extents.begin(); p != extents.end(); ++p) {
+      ldout(cct, 20) << " oid " << p->oid << " " << p->offset << "~" << p->length
+		     << " from " << p->buffer_extents << dendl;
+      // assemble extent
+      bufferlist bl;
+      for (vector >::iterator q = p->buffer_extents.begin();
+	   q != p->buffer_extents.end();
+	   ++q) {
+	bl.append(buf + q->first, q->second);
+      }
+
+      C_AioWrite *req_comp = new C_AioWrite(cct, c);
+      if (ictx->object_cacher) {
+	c->add_request();
+	ictx->write_to_cache(p->oid, bl, p->length, p->offset, req_comp);
+      } else {
+	// reverse map this object extent onto the parent
+	vector > objectx;
+	Striper::extent_to_file(ictx->cct, &ictx->layout,
+			      p->objectno, 0, ictx->layout.fl_object_size,
+			      objectx);
+	uint64_t object_overlap = ictx->prune_parent_extents(objectx, overlap);
+
+	AioWrite *req = new AioWrite(ictx, p->oid.name, p->objectno, p->offset,
+				     objectx, object_overlap,
+				     bl, snapc, snap_id, req_comp);
+	c->add_request();
+	r = req->send();
+	if (r < 0)
+	  goto done;
+      }
+    }
+  done:
+    c->finish_adding_requests(ictx->cct);
+    c->put();
+
+    ictx->perfcounter->inc(l_librbd_aio_wr);
+    ictx->perfcounter->inc(l_librbd_aio_wr_bytes, mylen);
+
+    /* FIXME: cleanup all the allocated stuff */
+    return r;
+  }
+
+  int aio_discard(ImageCtx *ictx, uint64_t off, uint64_t len, AioCompletion *c)
+  {
+    CephContext *cct = ictx->cct;
+    ldout(cct, 20) << "aio_discard " << ictx << " off = " << off << " len = "
+		   << len << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    r = clip_io(ictx, off, &len);
+    if (r < 0)
+      return r;
+
+    // TODO: check for snap
+    ictx->snap_lock.get_read();
+    snapid_t snap_id = ictx->snap_id;
+    ::SnapContext snapc = ictx->snapc;
+    ictx->parent_lock.get_read();
+    uint64_t overlap = 0;
+    ictx->get_parent_overlap(ictx->snap_id, &overlap);
+    ictx->parent_lock.put_read();
+    ictx->snap_lock.put_read();
+
+    if (snap_id != CEPH_NOSNAP || ictx->read_only)
+      return -EROFS;
+
+    // map
+    vector extents;
+    if (len > 0) {
+      Striper::file_to_extents(ictx->cct, ictx->format_string,
+			       &ictx->layout, off, len, 0, extents);
+    }
+
+    c->get();
+    c->init_time(ictx, AIO_TYPE_DISCARD);
+    for (vector::iterator p = extents.begin(); p != extents.end(); ++p) {
+      ldout(cct, 20) << " oid " << p->oid << " " << p->offset << "~" << p->length
+		     << " from " << p->buffer_extents << dendl;
+      C_AioWrite *req_comp = new C_AioWrite(cct, c);
+      AbstractWrite *req;
+      c->add_request();
+
+      // reverse map this object extent onto the parent
+      vector > objectx;
+      uint64_t object_overlap = 0;
+      if (off < overlap) {   // we might overlap...
+	Striper::extent_to_file(ictx->cct, &ictx->layout,
+			      p->objectno, 0, ictx->layout.fl_object_size,
+			      objectx);
+	object_overlap = ictx->prune_parent_extents(objectx, overlap);
+      }
+
+      if (p->offset == 0 && p->length == ictx->layout.fl_object_size) {
+	req = new AioRemove(ictx, p->oid.name, p->objectno, objectx, object_overlap,
+			    snapc, snap_id, req_comp);
+      } else if (p->offset + p->length == ictx->layout.fl_object_size) {
+	req = new AioTruncate(ictx, p->oid.name, p->objectno, p->offset, objectx, object_overlap,
+			      snapc, snap_id, req_comp);
+      } else {
+	req = new AioZero(ictx, p->oid.name, p->objectno, p->offset, p->length,
+			  objectx, object_overlap,
+			  snapc, snap_id, req_comp);
+      }
+
+      r = req->send();
+      if (r < 0)
+	goto done;
+    }
+    r = 0;
+  done:
+    if (ictx->object_cacher) {
+      Mutex::Locker l(ictx->cache_lock);
+      ictx->object_cacher->discard_set(ictx->object_set, extents);
+    }
+
+    c->finish_adding_requests(ictx->cct);
+    c->put();
+
+    ictx->perfcounter->inc(l_librbd_aio_discard);
+    ictx->perfcounter->inc(l_librbd_aio_discard_bytes, len);
+
+    /* FIXME: cleanup all the allocated stuff */
+    return r;
+  }
+
+  void rbd_req_cb(completion_t cb, void *arg)
+  {
+    AioRequest *req = reinterpret_cast(arg);
+    AioCompletion *comp = reinterpret_cast(cb);
+    req->complete(comp->get_return_value());
+  }
+
+  int aio_read(ImageCtx *ictx, uint64_t off, size_t len,
+	       char *buf, bufferlist *bl,
+	       AioCompletion *c)
+  {
+    vector > image_extents(1);
+    image_extents[0] = make_pair(off, len);
+    return aio_read(ictx, image_extents, buf, bl, c);
+  }
+
+  int aio_read(ImageCtx *ictx, const vector >& image_extents,
+	       char *buf, bufferlist *pbl, AioCompletion *c)
+  {
+    ldout(ictx->cct, 20) << "aio_read " << ictx << " completion " << c << " " << image_extents << dendl;
+
+    int r = ictx_check(ictx);
+    if (r < 0)
+      return r;
+
+    ictx->snap_lock.get_read();
+    snap_t snap_id = ictx->snap_id;
+    ictx->snap_lock.put_read();
+
+    // map
+    map > object_extents;
+
+    uint64_t buffer_ofs = 0;
+    for (vector >::const_iterator p = image_extents.begin();
+	 p != image_extents.end();
+	 ++p) {
+      uint64_t len = p->second;
+      r = clip_io(ictx, p->first, &len);
+      if (r < 0)
+	return r;
+      if (len == 0)
+	continue;
+
+      Striper::file_to_extents(ictx->cct, ictx->format_string, &ictx->layout,
+			       p->first, len, 0, object_extents, buffer_ofs);
+      buffer_ofs += len;
+    }
+
+    int64_t ret;
+
+    c->read_buf = buf;
+    c->read_buf_len = buffer_ofs;
+    c->read_bl = pbl;
+
+    c->get();
+    c->init_time(ictx, AIO_TYPE_READ);
+    for (map >::iterator p = object_extents.begin(); p != object_extents.end(); ++p) {
+      for (vector::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+	ldout(ictx->cct, 20) << " oid " << q->oid << " " << q->offset << "~" << q->length
+			     << " from " << q->buffer_extents << dendl;
+
+	C_AioRead *req_comp = new C_AioRead(ictx->cct, c);
+	AioRead *req = new AioRead(ictx, q->oid.name, 
+				   q->objectno, q->offset, q->length,
+				   q->buffer_extents,
+				   snap_id, true, req_comp);
+	req_comp->set_req(req);
+	c->add_request();
+
+	if (ictx->object_cacher) {
+	  C_CacheRead *cache_comp = new C_CacheRead(req);
+	  ictx->aio_read_from_cache(q->oid, &req->data(),
+				    q->length, q->offset,
+				    cache_comp);
+	} else {
+	  r = req->send();
+	  if (r < 0 && r == -ENOENT)
+	    r = 0;
+	  if (r < 0) {
+	    ret = r;
+	    goto done;
+	  }
+	}
+      }
+    }
+    ret = buffer_ofs;
+  done:
+    c->finish_adding_requests(ictx->cct);
+    c->put();
+
+    ictx->perfcounter->inc(l_librbd_aio_rd);
+    ictx->perfcounter->inc(l_librbd_aio_rd_bytes, buffer_ofs);
+
+    return ret;
+  }
+
+  AioCompletion *aio_create_completion() {
+    AioCompletion *c = new AioCompletion();
+    return c;
+  }
+
+  AioCompletion *aio_create_completion(void *cb_arg, callback_t cb_complete) {
+    AioCompletion *c = new AioCompletion();
+    c->set_complete_cb(cb_arg, cb_complete);
+    return c;
+  }
+
+  AioCompletion *aio_create_completion_internal(void *cb_arg,
+						callback_t cb_complete) {
+    AioCompletion *c = aio_create_completion(cb_arg, cb_complete);
+    c->rbd_comp = c;
+    return c;
+  }
+}
diff --git a/ceph/src/librbd/internal.h b/ceph/src/librbd/internal.h
new file mode 100644
index 00000000..1e9fd9a6
--- /dev/null
+++ b/ceph/src/librbd/internal.h
@@ -0,0 +1,215 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_LIBRBD_INTERNAL_H
+#define CEPH_LIBRBD_INTERNAL_H
+
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "include/buffer.h"
+#include "include/rbd/librbd.hpp"
+#include "include/rbd_types.h"
+
+enum {
+  l_librbd_first = 26000,
+
+  l_librbd_rd,               // read ops
+  l_librbd_rd_bytes,         // bytes read
+  l_librbd_rd_latency,       // average latency
+  l_librbd_wr,
+  l_librbd_wr_bytes,
+  l_librbd_wr_latency,
+  l_librbd_discard,
+  l_librbd_discard_bytes,
+  l_librbd_discard_latency,
+  l_librbd_flush,
+
+  l_librbd_aio_rd,               // read ops
+  l_librbd_aio_rd_bytes,         // bytes read
+  l_librbd_aio_rd_latency,
+  l_librbd_aio_wr,
+  l_librbd_aio_wr_bytes,
+  l_librbd_aio_wr_latency,
+  l_librbd_aio_discard,
+  l_librbd_aio_discard_bytes,
+  l_librbd_aio_discard_latency,
+  l_librbd_aio_flush,
+  l_librbd_aio_flush_latency,
+
+  l_librbd_snap_create,
+  l_librbd_snap_remove,
+  l_librbd_snap_rollback,
+
+  l_librbd_notify,
+  l_librbd_resize,
+
+  l_librbd_last,
+};
+
+namespace librbd {
+
+  struct AioCompletion;
+  struct ImageCtx;
+
+  class NoOpProgressContext : public ProgressContext
+  {
+  public:
+    NoOpProgressContext()
+    {
+    }
+    int update_progress(uint64_t offset, uint64_t src_size)
+    {
+      return 0;
+    }
+  };
+
+  const std::string id_obj_name(const std::string &name);
+  const std::string header_name(const std::string &image_id);
+  const std::string old_header_name(const std::string &image_name);
+
+  int detect_format(librados::IoCtx &io_ctx, const std::string &name,
+		    bool *old_format, uint64_t *size);
+
+  bool has_parent(int64_t parent_pool_id, uint64_t off, uint64_t overlap);
+
+  int snap_set(ImageCtx *ictx, const char *snap_name);
+  int list(librados::IoCtx& io_ctx, std::vector& names);
+  int list_children(ImageCtx *ictx,
+		    std::set > & names);
+  int create(librados::IoCtx& io_ctx, const char *imgname, uint64_t size,
+	     int *order);
+  int create(librados::IoCtx& io_ctx, const char *imgname, uint64_t size,
+	     bool old_format, uint64_t features, int *order,
+	     uint64_t stripe_unit, uint64_t stripe_count);
+  int clone(IoCtx& p_ioctx, const char *p_name, const char *p_snap_name,
+	    IoCtx& c_ioctx, const char *c_name,
+	    uint64_t features, int *c_order,
+	    uint64_t stripe_unit, int stripe_count);
+  int rename(librados::IoCtx& io_ctx, const char *srcname, const char *dstname);
+  int info(ImageCtx *ictx, image_info_t& info, size_t image_size);
+  int get_old_format(ImageCtx *ictx, uint8_t *old);
+  int get_size(ImageCtx *ictx, uint64_t *size);
+  int get_features(ImageCtx *ictx, uint64_t *features);
+  int get_overlap(ImageCtx *ictx, uint64_t *overlap);
+  int get_parent_info(ImageCtx *ictx, string *parent_pool_name,
+		      string *parent_name, string *parent_snap_name);
+
+  int remove(librados::IoCtx& io_ctx, const char *imgname,
+	     ProgressContext& prog_ctx);
+  int resize(ImageCtx *ictx, uint64_t size, ProgressContext& prog_ctx);
+  int resize_helper(ImageCtx *ictx, uint64_t size, ProgressContext& prog_ctx);
+  int snap_create(ImageCtx *ictx, const char *snap_name);
+  int snap_list(ImageCtx *ictx, std::vector& snaps);
+  bool snap_exists(ImageCtx *ictx, const char *snap_name);
+  int snap_rollback(ImageCtx *ictx, const char *snap_name,
+		    ProgressContext& prog_ctx);
+  int snap_remove(ImageCtx *ictx, const char *snap_name);
+  int snap_protect(ImageCtx *ictx, const char *snap_name);
+  int snap_unprotect(ImageCtx *ictx, const char *snap_name);
+  int snap_is_protected(ImageCtx *ictx, const char *snap_name,
+			bool *is_protected);
+  int add_snap(ImageCtx *ictx, const char *snap_name);
+  int rm_snap(ImageCtx *ictx, const char *snap_name);
+  int refresh_parent(ImageCtx *ictx);
+  int ictx_check(ImageCtx *ictx);
+  int ictx_refresh(ImageCtx *ictx);
+  int copy(ImageCtx *ictx, IoCtx& dest_md_ctx, const char *destname,
+	   ProgressContext &prog_ctx);
+  int copy(ImageCtx *src, ImageCtx *dest, ProgressContext &prog_ctx);
+
+  int open_parent(ImageCtx *ictx);
+  int open_image(ImageCtx *ictx);
+  void close_image(ImageCtx *ictx);
+
+  int copyup_block(ImageCtx *ictx, uint64_t offset, size_t len,
+		   const char *buf);
+  int flatten(ImageCtx *ictx, ProgressContext &prog_ctx);
+
+  /* cooperative locking */
+  int list_lockers(ImageCtx *ictx,
+		   std::list *locks,
+		   bool *exclusive,
+		   std::string *tag);
+  int lock(ImageCtx *ictx, bool exclusive, const std::string& cookie,
+	   const std::string& tag);
+  int lock_shared(ImageCtx *ictx, const std::string& cookie,
+		  const std::string& tag);
+  int unlock(ImageCtx *ictx, const std::string& cookie);
+  int break_lock(ImageCtx *ictx, const std::string& client,
+		 const std::string& cookie);
+
+  void trim_image(ImageCtx *ictx, uint64_t newsize, ProgressContext& prog_ctx);
+  int read_rbd_info(librados::IoCtx& io_ctx, const std::string& info_oid,
+		    struct rbd_info *info);
+
+  int read_header_bl(librados::IoCtx& io_ctx, const std::string& md_oid,
+		     ceph::bufferlist& header, uint64_t *ver);
+  int notify_change(librados::IoCtx& io_ctx, const std::string& oid,
+		    uint64_t *pver, ImageCtx *ictx);
+  int read_header(librados::IoCtx& io_ctx, const std::string& md_oid,
+		  struct rbd_obj_header_ondisk *header, uint64_t *ver);
+  int write_header(librados::IoCtx& io_ctx, const std::string& md_oid,
+		   ceph::bufferlist& header);
+  int tmap_set(librados::IoCtx& io_ctx, const std::string& imgname);
+  int tmap_rm(librados::IoCtx& io_ctx, const std::string& imgname);
+  int rollback_image(ImageCtx *ictx, uint64_t snap_id,
+		     ProgressContext& prog_ctx);
+  void image_info(const ImageCtx *ictx, image_info_t& info, size_t info_size);
+  std::string get_block_oid(const std::string &object_prefix, uint64_t num,
+			    bool old_format);
+  uint64_t oid_to_object_no(const string& oid, const string& object_prefix);
+  int clip_io(ImageCtx *ictx, uint64_t off, uint64_t *len);
+  int init_rbd_info(struct rbd_info *info);
+  void init_rbd_header(struct rbd_obj_header_ondisk& ondisk,
+		       uint64_t size, int order, uint64_t bid);
+
+  int64_t read_iterate(ImageCtx *ictx, uint64_t off, uint64_t len,
+		       int (*cb)(uint64_t, size_t, const char *, void *),
+		       void *arg);
+  int diff_iterate(ImageCtx *ictx, const char *fromsnapname,
+		   uint64_t off, uint64_t len,
+		   int (*cb)(uint64_t, size_t, int, void *),
+		   void *arg);
+  ssize_t read(ImageCtx *ictx, uint64_t off, size_t len, char *buf);
+  ssize_t read(ImageCtx *ictx, const vector >& image_extents,
+	       char *buf, bufferlist *pbl);
+  ssize_t write(ImageCtx *ictx, uint64_t off, size_t len, const char *buf);
+  int discard(ImageCtx *ictx, uint64_t off, uint64_t len);
+  int aio_write(ImageCtx *ictx, uint64_t off, size_t len, const char *buf,
+		AioCompletion *c);
+  int aio_discard(ImageCtx *ictx, uint64_t off, uint64_t len, AioCompletion *c);
+  int aio_read(ImageCtx *ictx, uint64_t off, size_t len,
+	       char *buf, bufferlist *pbl, AioCompletion *c);
+  int aio_read(ImageCtx *ictx, const vector >& image_extents,
+	       char *buf, bufferlist *pbl, AioCompletion *c);
+  int aio_flush(ImageCtx *ictx, AioCompletion *c);
+  int flush(ImageCtx *ictx);
+  int _flush(ImageCtx *ictx);
+  int invalidate_cache(ImageCtx *ictx);
+
+  ssize_t handle_sparse_read(CephContext *cct,
+			     ceph::bufferlist data_bl,
+			     uint64_t block_ofs,
+			     const std::map &data_map,
+			     uint64_t buf_ofs,
+			     size_t buf_len,
+			     char *dest_buf);
+
+  AioCompletion *aio_create_completion();
+  AioCompletion *aio_create_completion(void *cb_arg, callback_t cb_complete);
+  AioCompletion *aio_create_completion_internal(void *cb_arg,
+						callback_t cb_complete);
+
+  // raw callbacks
+  int simple_read_cb(uint64_t ofs, size_t len, const char *buf, void *arg);
+  void rados_req_cb(rados_completion_t cb, void *arg);
+  void rados_ctx_cb(rados_completion_t cb, void *arg);
+  void rbd_req_cb(completion_t cb, void *arg);
+  void rbd_ctx_cb(completion_t cb, void *arg);
+}
+
+#endif
diff --git a/ceph/src/librbd/librbd.cc b/ceph/src/librbd/librbd.cc
new file mode 100644
index 00000000..658f24b8
--- /dev/null
+++ b/ceph/src/librbd/librbd.cc
@@ -0,0 +1,1167 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.	See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+
+#include 
+
+#include "common/Cond.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "common/snap_types.h"
+#include "common/perf_counters.h"
+#include "include/Context.h"
+#include "include/rbd/librbd.hpp"
+#include "osdc/ObjectCacher.h"
+
+#include "librbd/AioCompletion.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/internal.h"
+#include "librbd/LibrbdWriteback.h"
+
+#include 
+#include 
+#include 
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd: "
+
+using std::string;
+using std::vector;
+
+using ceph::bufferlist;
+using librados::snap_t;
+using librados::IoCtx;
+
+namespace librbd {
+  ProgressContext::~ProgressContext()
+  {
+  }
+
+  class CProgressContext : public ProgressContext
+  {
+  public:
+    CProgressContext(librbd_progress_fn_t fn, void *data)
+      : m_fn(fn), m_data(data)
+    {
+    }
+    int update_progress(uint64_t offset, uint64_t src_size)
+    {
+      return m_fn(offset, src_size, m_data);
+    }
+  private:
+    librbd_progress_fn_t m_fn;
+    void *m_data;
+  };
+
+  /*
+    RBD
+  */
+  RBD::RBD()
+  {
+  }
+
+  RBD::~RBD()
+  {
+  }
+
+  void RBD::version(int *major, int *minor, int *extra)
+  {
+    rbd_version(major, minor, extra);
+  }
+
+  int RBD::open(IoCtx& io_ctx, Image& image, const char *name)
+  {
+    return open(io_ctx, image, name, NULL);
+  }
+
+  int RBD::open(IoCtx& io_ctx, Image& image, const char *name,
+		const char *snap_name)
+  {
+    ImageCtx *ictx = new ImageCtx(name, "", snap_name, io_ctx, false);
+
+    int r = librbd::open_image(ictx);
+    if (r < 0)
+      return r;
+
+    image.ctx = (image_ctx_t) ictx;
+    return 0;
+  }
+
+  int RBD::open_read_only(IoCtx& io_ctx, Image& image, const char *name,
+			  const char *snap_name)
+  {
+    ImageCtx *ictx = new ImageCtx(name, "", snap_name, io_ctx, true);
+
+    int r = librbd::open_image(ictx);
+    if (r < 0)
+      return r;
+
+    image.ctx = (image_ctx_t) ictx;
+    return 0;
+  }
+
+  int RBD::create(IoCtx& io_ctx, const char *name, uint64_t size, int *order)
+  {
+    return librbd::create(io_ctx, name, size, order);
+  }
+
+  int RBD::create2(IoCtx& io_ctx, const char *name, uint64_t size,
+		   uint64_t features, int *order)
+  {
+    return librbd::create(io_ctx, name, size, false, features, order, 0, 0);
+  }
+
+  int RBD::create3(IoCtx& io_ctx, const char *name, uint64_t size,
+		   uint64_t features, int *order, uint64_t stripe_unit,
+		   uint64_t stripe_count)
+  {
+    return librbd::create(io_ctx, name, size, false, features, order,
+			  stripe_unit, stripe_count);
+  }
+
+  int RBD::clone(IoCtx& p_ioctx, const char *p_name, const char *p_snap_name,
+		 IoCtx& c_ioctx, const char *c_name, uint64_t features,
+		 int *c_order)
+  {
+    return librbd::clone(p_ioctx, p_name, p_snap_name, c_ioctx, c_name,
+			 features, c_order, 0, 0);
+  }
+
+  int RBD::clone2(IoCtx& p_ioctx, const char *p_name, const char *p_snap_name,
+		  IoCtx& c_ioctx, const char *c_name, uint64_t features,
+		  int *c_order, uint64_t stripe_unit, int stripe_count)
+  {
+    return librbd::clone(p_ioctx, p_name, p_snap_name, c_ioctx, c_name,
+			 features, c_order, stripe_unit, stripe_count);
+  }
+
+  int RBD::remove(IoCtx& io_ctx, const char *name)
+  {
+    librbd::NoOpProgressContext prog_ctx;
+    int r = librbd::remove(io_ctx, name, prog_ctx);
+    return r;
+  }
+
+  int RBD::remove_with_progress(IoCtx& io_ctx, const char *name,
+				ProgressContext& pctx)
+  {
+    int r = librbd::remove(io_ctx, name, pctx);
+    return r;
+  }
+
+  int RBD::list(IoCtx& io_ctx, vector& names)
+  {
+    int r = librbd::list(io_ctx, names);
+    return r;
+  }
+
+  int RBD::rename(IoCtx& src_io_ctx, const char *srcname, const char *destname)
+  {
+    int r = librbd::rename(src_io_ctx, srcname, destname);
+    return r;
+  }
+
+  RBD::AioCompletion::AioCompletion(void *cb_arg, callback_t complete_cb)
+  {
+    librbd::AioCompletion *c = librbd::aio_create_completion(cb_arg,
+							     complete_cb);
+    pc = (void *)c;
+    c->rbd_comp = this;
+  }
+
+  bool RBD::AioCompletion::is_complete()
+  {
+    librbd::AioCompletion *c = (librbd::AioCompletion *)pc;
+    return c->is_complete();
+  }
+
+  int RBD::AioCompletion::wait_for_complete()
+  {
+    librbd::AioCompletion *c = (librbd::AioCompletion *)pc;
+    return c->wait_for_complete();
+  }
+
+  ssize_t RBD::AioCompletion::get_return_value()
+  {
+    librbd::AioCompletion *c = (librbd::AioCompletion *)pc;
+    return c->get_return_value();
+  }
+
+  void RBD::AioCompletion::release()
+  {
+    librbd::AioCompletion *c = (librbd::AioCompletion *)pc;
+    c->release();
+    delete this;
+  }
+
+  /*
+    Image
+  */
+
+  Image::Image() : ctx(NULL)
+  {
+  }
+
+  Image::~Image()
+  {
+    if (ctx) {
+      ImageCtx *ictx = (ImageCtx *)ctx;
+      close_image(ictx);
+    }
+  }
+
+  int Image::resize(uint64_t size)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    librbd::NoOpProgressContext prog_ctx;
+    return librbd::resize(ictx, size, prog_ctx);
+  }
+
+  int Image::resize_with_progress(uint64_t size, librbd::ProgressContext& pctx)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::resize(ictx, size, pctx);
+  }
+
+  int Image::stat(image_info_t& info, size_t infosize)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::info(ictx, info, infosize);
+  }
+
+  int Image::old_format(uint8_t *old)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::get_old_format(ictx, old);
+  }
+
+  int Image::size(uint64_t *size)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::get_size(ictx, size);
+  }
+
+  int Image::features(uint64_t *features)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::get_features(ictx, features);
+  }
+
+  uint64_t Image::get_stripe_unit() const
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return ictx->get_stripe_unit();
+  }
+
+  uint64_t Image::get_stripe_count() const
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return ictx->get_stripe_count();
+  }
+
+  int Image::overlap(uint64_t *overlap)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::get_overlap(ictx, overlap);
+  }
+
+  int Image::parent_info(string *parent_pool_name, string *parent_name,
+			 string *parent_snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::get_parent_info(ictx, parent_pool_name, parent_name,
+				   parent_snap_name);
+  }
+
+  int Image::copy(IoCtx& dest_io_ctx, const char *destname)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    librbd::NoOpProgressContext prog_ctx;
+    return librbd::copy(ictx, dest_io_ctx, destname, prog_ctx);
+  }
+
+  int Image::copy2(Image& dest)
+  {
+    ImageCtx *srcctx = (ImageCtx *)ctx;
+    ImageCtx *destctx = (ImageCtx *)dest.ctx;
+    librbd::NoOpProgressContext prog_ctx;
+    return librbd::copy(srcctx, destctx, prog_ctx);
+  }
+
+  int Image::copy_with_progress(IoCtx& dest_io_ctx, const char *destname,
+				librbd::ProgressContext &pctx)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::copy(ictx, dest_io_ctx, destname, pctx);
+  }
+
+  int Image::copy_with_progress2(Image& dest, librbd::ProgressContext &pctx)
+  {
+    ImageCtx *srcctx = (ImageCtx *)ctx;
+    ImageCtx *destctx = (ImageCtx *)dest.ctx;
+    return librbd::copy(srcctx, destctx, pctx);
+  }
+
+  int Image::flatten()
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    librbd::NoOpProgressContext prog_ctx;
+    return librbd::flatten(ictx, prog_ctx);
+  }
+
+  int Image::flatten_with_progress(librbd::ProgressContext& prog_ctx)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::flatten(ictx, prog_ctx);
+  }
+
+  int Image::list_children(set > *children)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::list_children(ictx, *children);
+  }
+
+  int Image::list_lockers(std::list *lockers,
+			  bool *exclusive, string *tag)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::list_lockers(ictx, lockers, exclusive, tag);
+  }
+
+  int Image::lock_exclusive(const string& cookie)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::lock(ictx, true, cookie, "");
+  }
+
+  int Image::lock_shared(const string& cookie, const std::string& tag)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::lock(ictx, false, cookie, tag);
+  }
+
+  int Image::unlock(const string& cookie)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::unlock(ictx, cookie);
+  }
+
+  int Image::break_lock(const string& client, const string& cookie)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::break_lock(ictx, client, cookie);
+  }
+
+  int Image::snap_create(const char *snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_create(ictx, snap_name);
+  }
+
+  int Image::snap_remove(const char *snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_remove(ictx, snap_name);
+  }
+
+  int Image::snap_rollback(const char *snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    librbd::NoOpProgressContext prog_ctx;
+    return librbd::snap_rollback(ictx, snap_name, prog_ctx);
+  }
+
+  int Image::snap_rollback_with_progress(const char *snap_name,
+					 ProgressContext& prog_ctx)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_rollback(ictx, snap_name, prog_ctx);
+  }
+
+  int Image::snap_protect(const char *snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_protect(ictx, snap_name);
+  }
+
+  int Image::snap_unprotect(const char *snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_unprotect(ictx, snap_name);
+  }
+
+  int Image::snap_is_protected(const char *snap_name, bool *is_protected)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_is_protected(ictx, snap_name, is_protected);
+  }
+
+  int Image::snap_list(vector& snaps)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_list(ictx, snaps);
+  }
+
+  bool Image::snap_exists(const char *snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_exists(ictx, snap_name);
+  }
+
+  int Image::snap_set(const char *snap_name)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::snap_set(ictx, snap_name);
+  }
+
+  ssize_t Image::read(uint64_t ofs, size_t len, bufferlist& bl)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    bufferptr ptr(len);
+    bl.push_back(ptr);
+    return librbd::read(ictx, ofs, len, bl.c_str());
+  }
+
+  int64_t Image::read_iterate(uint64_t ofs, size_t len,
+			      int (*cb)(uint64_t, size_t, const char *, void *),
+			      void *arg)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::read_iterate(ictx, ofs, len, cb, arg);
+  }
+
+  int Image::read_iterate2(uint64_t ofs, uint64_t len,
+			      int (*cb)(uint64_t, size_t, const char *, void *),
+			      void *arg)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    int64_t r = librbd::read_iterate(ictx, ofs, len, cb, arg);
+    if (r > 0)
+      r = 0;
+    return (int)r;
+  }
+
+  int Image::diff_iterate(const char *fromsnapname,
+			  uint64_t ofs, uint64_t len,
+			  int (*cb)(uint64_t, size_t, int, void *),
+			  void *arg)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::diff_iterate(ictx, fromsnapname, ofs, len, cb, arg);
+  }
+
+  ssize_t Image::write(uint64_t ofs, size_t len, bufferlist& bl)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    if (bl.length() < len)
+      return -EINVAL;
+    return librbd::write(ictx, ofs, len, bl.c_str());
+  }
+
+  int Image::discard(uint64_t ofs, uint64_t len)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::discard(ictx, ofs, len);
+  }
+
+  int Image::aio_write(uint64_t off, size_t len, bufferlist& bl,
+		       RBD::AioCompletion *c)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    if (bl.length() < len)
+      return -EINVAL;
+    return librbd::aio_write(ictx, off, len, bl.c_str(),
+			     (librbd::AioCompletion *)c->pc);
+  }
+
+  int Image::aio_discard(uint64_t off, uint64_t len, RBD::AioCompletion *c)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::aio_discard(ictx, off, len, (librbd::AioCompletion *)c->pc);
+  }
+
+  int Image::aio_read(uint64_t off, size_t len, bufferlist& bl,
+		      RBD::AioCompletion *c)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    ldout(ictx->cct, 10) << "Image::aio_read() buf=" << (void *)bl.c_str() << "~"
+			 << (void *)(bl.c_str() + len - 1) << dendl;
+    return librbd::aio_read(ictx, off, len, NULL, &bl, (librbd::AioCompletion *)c->pc);
+  }
+
+  int Image::flush()
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::flush(ictx);
+  }
+
+  int Image::aio_flush(RBD::AioCompletion *c)
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::aio_flush(ictx, (librbd::AioCompletion *)c->pc);
+  }
+
+  int Image::invalidate_cache()
+  {
+    ImageCtx *ictx = (ImageCtx *)ctx;
+    return librbd::invalidate_cache(ictx);
+  }
+
+} // namespace librbd
+
+extern "C" void rbd_version(int *major, int *minor, int *extra)
+{
+  if (major)
+    *major = LIBRBD_VER_MAJOR;
+  if (minor)
+    *minor = LIBRBD_VER_MINOR;
+  if (extra)
+    *extra = LIBRBD_VER_EXTRA;
+}
+
+/* images */
+extern "C" int rbd_list(rados_ioctx_t p, char *names, size_t *size)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  vector cpp_names;
+  int r = librbd::list(io_ctx, cpp_names);
+  if (r == -ENOENT)
+    return 0;
+
+  if (r < 0)
+    return r;
+
+  size_t expected_size = 0;
+
+  for (size_t i = 0; i < cpp_names.size(); i++) {
+    expected_size += cpp_names[i].size() + 1;
+  }
+  if (*size < expected_size) {
+    *size = expected_size;
+    return -ERANGE;
+  }
+
+  for (int i = 0; i < (int)cpp_names.size(); i++) {
+    strcpy(names, cpp_names[i].c_str());
+    names += strlen(names) + 1;
+  }
+  return (int)expected_size;
+}
+
+extern "C" int rbd_create(rados_ioctx_t p, const char *name, uint64_t size, int *order)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  return librbd::create(io_ctx, name, size, order);
+}
+
+extern "C" int rbd_create2(rados_ioctx_t p, const char *name,
+			   uint64_t size, uint64_t features,
+			   int *order)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  return librbd::create(io_ctx, name, size, false, features, order, 0, 0);
+}
+
+extern "C" int rbd_create3(rados_ioctx_t p, const char *name,
+			   uint64_t size, uint64_t features,
+			   int *order,
+			   uint64_t stripe_unit, uint64_t stripe_count)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  return librbd::create(io_ctx, name, size, false, features, order,
+			stripe_unit, stripe_count);
+}
+
+extern "C" int rbd_clone(rados_ioctx_t p_ioctx, const char *p_name,
+			 const char *p_snap_name, rados_ioctx_t c_ioctx,
+			 const char *c_name, uint64_t features, int *c_order)
+{
+  librados::IoCtx p_ioc, c_ioc;
+  librados::IoCtx::from_rados_ioctx_t(p_ioctx, p_ioc);
+  librados::IoCtx::from_rados_ioctx_t(c_ioctx, c_ioc);
+  return librbd::clone(p_ioc, p_name, p_snap_name, c_ioc, c_name,
+		       features, c_order, 0, 0);
+}
+
+extern "C" int rbd_clone2(rados_ioctx_t p_ioctx, const char *p_name,
+			  const char *p_snap_name, rados_ioctx_t c_ioctx,
+			  const char *c_name, uint64_t features, int *c_order,
+			  uint64_t stripe_unit, int stripe_count)
+{
+  librados::IoCtx p_ioc, c_ioc;
+  librados::IoCtx::from_rados_ioctx_t(p_ioctx, p_ioc);
+  librados::IoCtx::from_rados_ioctx_t(c_ioctx, c_ioc);
+  return librbd::clone(p_ioc, p_name, p_snap_name, c_ioc, c_name,
+		       features, c_order, stripe_unit, stripe_count);
+}
+
+extern "C" int rbd_remove(rados_ioctx_t p, const char *name)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  librbd::NoOpProgressContext prog_ctx;
+  return librbd::remove(io_ctx, name, prog_ctx);
+}
+
+extern "C" int rbd_remove_with_progress(rados_ioctx_t p, const char *name,
+					librbd_progress_fn_t cb, void *cbdata)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  librbd::CProgressContext prog_ctx(cb, cbdata);
+  return librbd::remove(io_ctx, name, prog_ctx);
+}
+
+extern "C" int rbd_copy(rbd_image_t image, rados_ioctx_t dest_p,
+			const char *destname)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librados::IoCtx dest_io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(dest_p, dest_io_ctx);
+  librbd::NoOpProgressContext prog_ctx;
+  return librbd::copy(ictx, dest_io_ctx, destname, prog_ctx);
+}
+
+extern "C" int rbd_copy2(rbd_image_t srcp, rbd_image_t destp)
+{
+  librbd::ImageCtx *src = (librbd::ImageCtx *)srcp;
+  librbd::ImageCtx *dest = (librbd::ImageCtx *)destp;
+  librbd::NoOpProgressContext prog_ctx;
+  return librbd::copy(src, dest, prog_ctx);
+}
+
+extern "C" int rbd_copy_with_progress(rbd_image_t image, rados_ioctx_t dest_p,
+				      const char *destname,
+				      librbd_progress_fn_t fn, void *data)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librados::IoCtx dest_io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(dest_p, dest_io_ctx);
+  librbd::CProgressContext prog_ctx(fn, data);
+  int ret = librbd::copy(ictx, dest_io_ctx, destname, prog_ctx);
+  return ret;
+}
+
+extern "C" int rbd_copy_with_progress2(rbd_image_t srcp, rbd_image_t destp,
+				      librbd_progress_fn_t fn, void *data)
+{
+  librbd::ImageCtx *src = (librbd::ImageCtx *)srcp;
+  librbd::ImageCtx *dest = (librbd::ImageCtx *)destp;
+  librbd::CProgressContext prog_ctx(fn, data);
+  int ret = librbd::copy(src, dest, prog_ctx);
+  return ret;
+}
+
+extern "C" int rbd_flatten(rbd_image_t image)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::NoOpProgressContext prog_ctx;
+  return librbd::flatten(ictx, prog_ctx);
+}
+
+extern "C" int rbd_flatten_with_progress(rbd_image_t image,
+					 librbd_progress_fn_t cb, void *cbdata)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::CProgressContext prog_ctx(cb, cbdata);
+  return librbd::flatten(ictx, prog_ctx);
+}
+
+extern "C" int rbd_rename(rados_ioctx_t src_p, const char *srcname,
+			  const char *destname)
+{
+  librados::IoCtx src_io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(src_p, src_io_ctx);
+  return librbd::rename(src_io_ctx, srcname, destname);
+}
+
+extern "C" int rbd_open(rados_ioctx_t p, const char *name, rbd_image_t *image,
+			const char *snap_name)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  librbd::ImageCtx *ictx = new librbd::ImageCtx(name, "", snap_name, io_ctx,
+						false);
+  int r = librbd::open_image(ictx);
+  if (r >= 0)
+    *image = (rbd_image_t)ictx;
+  return r;
+}
+
+extern "C" int rbd_open_read_only(rados_ioctx_t p, const char *name,
+				  rbd_image_t *image, const char *snap_name)
+{
+  librados::IoCtx io_ctx;
+  librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
+  librbd::ImageCtx *ictx = new librbd::ImageCtx(name, "", snap_name, io_ctx,
+						true);
+  int r = librbd::open_image(ictx);
+  if (r >= 0)
+    *image = (rbd_image_t)ictx;
+  return r;
+}
+
+extern "C" int rbd_close(rbd_image_t image)
+{
+  librbd::ImageCtx *ctx = (librbd::ImageCtx *)image;
+  librbd::close_image(ctx);
+  return 0;
+}
+
+extern "C" int rbd_resize(rbd_image_t image, uint64_t size)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::NoOpProgressContext prog_ctx;
+  return librbd::resize(ictx, size, prog_ctx);
+}
+
+extern "C" int rbd_resize_with_progress(rbd_image_t image, uint64_t size,
+					librbd_progress_fn_t cb, void *cbdata)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::CProgressContext prog_ctx(cb, cbdata);
+  return librbd::resize(ictx, size, prog_ctx);
+}
+
+extern "C" int rbd_stat(rbd_image_t image, rbd_image_info_t *info,
+			size_t infosize)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::info(ictx, *info, infosize);
+}
+
+extern "C" int rbd_get_old_format(rbd_image_t image, uint8_t *old)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::get_old_format(ictx, old);
+}
+
+extern "C" int rbd_get_size(rbd_image_t image, uint64_t *size)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::get_size(ictx, size);
+}
+
+extern "C" int rbd_get_features(rbd_image_t image, uint64_t *features)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::get_features(ictx, features);
+}
+
+extern "C" int rbd_get_stripe_unit(rbd_image_t image, uint64_t *stripe_unit)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  *stripe_unit = ictx->get_stripe_unit();
+  return 0;
+}
+
+extern "C" int rbd_get_stripe_count(rbd_image_t image, uint64_t *stripe_count)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  *stripe_count = ictx->get_stripe_count();
+  return 0;
+}
+
+extern "C" int rbd_get_overlap(rbd_image_t image, uint64_t *overlap)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::get_overlap(ictx, overlap);
+}
+
+extern "C" int rbd_get_parent_info(rbd_image_t image,
+  char *parent_pool_name, size_t ppool_namelen, char *parent_name,
+  size_t pnamelen, char *parent_snap_name, size_t psnap_namelen)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  string p_pool_name, p_name, p_snap_name;
+
+  int r = librbd::get_parent_info(ictx, &p_pool_name, &p_name, &p_snap_name);
+  if (r < 0)
+    return r;
+
+  // compare against input bufferlen, leaving room for \0
+  if (p_pool_name.length() + 1 > ppool_namelen ||
+      p_name.length() + 1 > pnamelen ||
+      p_snap_name.length() + 1 > psnap_namelen) {
+    return -ERANGE;
+  }
+
+  strcpy(parent_pool_name, p_pool_name.c_str());
+  strcpy(parent_name, p_name.c_str());
+  strcpy(parent_snap_name, p_snap_name.c_str());
+  return 0;
+}
+
+/* snapshots */
+extern "C" int rbd_snap_create(rbd_image_t image, const char *snap_name)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::snap_create(ictx, snap_name);
+}
+
+extern "C" int rbd_snap_remove(rbd_image_t image, const char *snap_name)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::snap_remove(ictx, snap_name);
+}
+
+extern "C" int rbd_snap_rollback(rbd_image_t image, const char *snap_name)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::NoOpProgressContext prog_ctx;
+  return librbd::snap_rollback(ictx, snap_name, prog_ctx);
+}
+
+extern "C" int rbd_snap_rollback_with_progress(rbd_image_t image,
+					       const char *snap_name,
+					       librbd_progress_fn_t cb,
+					       void *cbdata)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::CProgressContext prog_ctx(cb, cbdata);
+  return librbd::snap_rollback(ictx, snap_name, prog_ctx);
+}
+
+extern "C" int rbd_snap_list(rbd_image_t image, rbd_snap_info_t *snaps,
+			     int *max_snaps)
+{
+  vector cpp_snaps;
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  int r = librbd::snap_list(ictx, cpp_snaps);
+  if (r == -ENOENT)
+    return 0;
+  if (r < 0)
+    return r;
+  if (!max_snaps)
+    return -EINVAL;
+  if (*max_snaps < (int)cpp_snaps.size() + 1) {
+    *max_snaps = (int)cpp_snaps.size() + 1;
+    return -ERANGE;
+  }
+
+  int i;
+
+  for (i = 0; i < (int)cpp_snaps.size(); i++) {
+    snaps[i].id = cpp_snaps[i].id;
+    snaps[i].size = cpp_snaps[i].size;
+    snaps[i].name = strdup(cpp_snaps[i].name.c_str());
+    if (!snaps[i].name) {
+      for (int j = 0; j < i; j++)
+	free((void *)snaps[j].name);
+      return -ENOMEM;
+    }
+  }
+  snaps[i].id = 0;
+  snaps[i].size = 0;
+  snaps[i].name = NULL;
+
+  return (int)cpp_snaps.size();
+}
+
+extern "C" void rbd_snap_list_end(rbd_snap_info_t *snaps)
+{
+  while (snaps->name) {
+    free((void *)snaps->name);
+    snaps++;
+  }
+}
+
+extern "C" int rbd_snap_protect(rbd_image_t image, const char *snap_name)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::snap_protect(ictx, snap_name);
+}
+
+extern "C" int rbd_snap_unprotect(rbd_image_t image, const char *snap_name)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::snap_unprotect(ictx, snap_name);
+}
+
+extern "C" int rbd_snap_is_protected(rbd_image_t image, const char *snap_name,
+				     int *is_protected)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  bool protected_snap;
+  int r = librbd::snap_is_protected(ictx, snap_name, &protected_snap);
+  if (r < 0)
+    return r;
+  *is_protected = protected_snap ? 1 : 0;
+  return 0;
+}
+
+extern "C" int rbd_snap_set(rbd_image_t image, const char *snap_name)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::snap_set(ictx, snap_name);
+}
+
+extern "C" ssize_t rbd_list_children(rbd_image_t image, char *pools,
+				     size_t *pools_len, char *images,
+				     size_t *images_len)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  set > image_set;
+
+  int r = librbd::list_children(ictx, image_set);
+  if (r < 0)
+    return r;
+
+  size_t pools_total = 0;
+  size_t images_total = 0;
+  for (set >::const_iterator it = image_set.begin();
+       it != image_set.end(); ++it) {
+    pools_total += it->first.length() + 1;
+    images_total += it->second.length() + 1;
+  }
+
+  bool too_short = false;
+  if (pools_total > *pools_len)
+    too_short = true;
+  if (images_total > *images_len)
+    too_short = true;
+  *pools_len = pools_total;
+  *images_len = images_total;
+  if (too_short)
+    return -ERANGE;
+
+  char *pools_p = pools;
+  char *images_p = images;
+  for (set >::const_iterator it = image_set.begin();
+       it != image_set.end(); ++it) {
+    strcpy(pools_p, it->first.c_str());
+    pools_p += it->first.length() + 1;
+    strcpy(images_p, it->second.c_str());
+    images_p += it->second.length() + 1;
+  }
+
+  return image_set.size();
+}
+
+extern "C" ssize_t rbd_list_lockers(rbd_image_t image, int *exclusive,
+				    char *tag, size_t *tag_len,
+				    char *clients, size_t *clients_len,
+				    char *cookies, size_t *cookies_len,
+				    char *addrs, size_t *addrs_len)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  std::list lockers;
+  bool exclusive_bool;
+  string tag_str;
+
+  int r = list_lockers(ictx, &lockers, &exclusive_bool, &tag_str);
+  if (r < 0)
+    return r;
+
+  ldout(ictx->cct, 20) << "list_lockers r = " << r << " lockers.size() = " << lockers.size() << dendl;
+
+  *exclusive = (int)exclusive_bool;
+  size_t clients_total = 0;
+  size_t cookies_total = 0;
+  size_t addrs_total = 0;
+  for (list::const_iterator it = lockers.begin();
+       it != lockers.end(); ++it) {
+    clients_total += it->client.length() + 1;
+    cookies_total += it->cookie.length() + 1;
+    addrs_total += it->address.length() + 1;
+  }
+
+  bool too_short = ((clients_total > *clients_len) ||
+		    (cookies_total > *cookies_len) ||
+		    (addrs_total > *addrs_len) ||
+		    (tag_str.length() + 1 > *tag_len));
+  *clients_len = clients_total;
+  *cookies_len = cookies_total;
+  *addrs_len = addrs_total;
+  *tag_len = tag_str.length() + 1;
+  if (too_short)
+    return -ERANGE;
+
+  strcpy(tag, tag_str.c_str());
+  char *clients_p = clients;
+  char *cookies_p = cookies;
+  char *addrs_p = addrs;
+  for (list::const_iterator it = lockers.begin();
+       it != lockers.end(); ++it) {
+    strcpy(clients_p, it->client.c_str());
+    clients_p += it->client.length() + 1;
+    strcpy(cookies_p, it->cookie.c_str());
+    cookies_p += it->cookie.length() + 1;
+    strcpy(addrs_p, it->address.c_str());
+    addrs_p += it->address.length() + 1;
+  }
+
+  return lockers.size();
+}
+
+extern "C" int rbd_lock_exclusive(rbd_image_t image, const char *cookie)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::lock(ictx, true, cookie ? cookie : "", "");
+}
+
+extern "C" int rbd_lock_shared(rbd_image_t image, const char *cookie,
+			       const char *tag)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::lock(ictx, false, cookie ? cookie : "", tag ? tag : "");
+}
+
+extern "C" int rbd_unlock(rbd_image_t image, const char *cookie)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::unlock(ictx, cookie ? cookie : "");
+}
+
+extern "C" int rbd_break_lock(rbd_image_t image, const char *client,
+			      const char *cookie)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::break_lock(ictx, client, cookie ? cookie : "");
+}
+
+/* I/O */
+extern "C" ssize_t rbd_read(rbd_image_t image, uint64_t ofs, size_t len,
+			    char *buf)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::read(ictx, ofs, len, buf);
+}
+
+extern "C" int64_t rbd_read_iterate(rbd_image_t image, uint64_t ofs, size_t len,
+				    int (*cb)(uint64_t, size_t, const char *, void *),
+				    void *arg)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::read_iterate(ictx, ofs, len, cb, arg);
+}
+
+extern "C" int rbd_read_iterate2(rbd_image_t image, uint64_t ofs, uint64_t len,
+				 int (*cb)(uint64_t, size_t, const char *, void *),
+				 void *arg)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  int64_t r = librbd::read_iterate(ictx, ofs, len, cb, arg);
+  if (r > 0)
+    r = 0;
+  return (int)r;
+}
+
+extern "C" int rbd_diff_iterate(rbd_image_t image,
+				const char *fromsnapname,
+				uint64_t ofs, uint64_t len,
+				int (*cb)(uint64_t, size_t, int, void *),
+				void *arg)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::diff_iterate(ictx, fromsnapname, ofs, len, cb, arg);
+}
+
+extern "C" ssize_t rbd_write(rbd_image_t image, uint64_t ofs, size_t len,
+			     const char *buf)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::write(ictx, ofs, len, buf);
+}
+
+extern "C" int rbd_discard(rbd_image_t image, uint64_t ofs, uint64_t len)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::discard(ictx, ofs, len);
+}
+
+extern "C" int rbd_aio_create_completion(void *cb_arg,
+					 rbd_callback_t complete_cb,
+					 rbd_completion_t *c)
+{
+  librbd::RBD::AioCompletion *rbd_comp =
+    new librbd::RBD::AioCompletion(cb_arg, complete_cb);
+  *c = (rbd_completion_t) rbd_comp;
+  return 0;
+}
+
+extern "C" int rbd_aio_write(rbd_image_t image, uint64_t off, size_t len,
+			     const char *buf, rbd_completion_t c)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  return librbd::aio_write(ictx, off, len, buf,
+			   (librbd::AioCompletion *)comp->pc);
+}
+
+extern "C" int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len,
+			       rbd_completion_t c)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  return librbd::aio_discard(ictx, off, len, (librbd::AioCompletion *)comp->pc);
+}
+
+extern "C" int rbd_aio_read(rbd_image_t image, uint64_t off, size_t len,
+			    char *buf, rbd_completion_t c)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  return librbd::aio_read(ictx, off, len, buf, NULL,
+			  (librbd::AioCompletion *)comp->pc);
+}
+
+extern "C" int rbd_flush(rbd_image_t image)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::flush(ictx);
+}
+
+extern "C" int rbd_aio_flush(rbd_image_t image, rbd_completion_t c)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  return librbd::aio_flush(ictx, (librbd::AioCompletion *)comp->pc);
+}
+
+extern "C" int rbd_invalidate_cache(rbd_image_t image)
+{
+  librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;
+  return librbd::invalidate_cache(ictx);
+}
+
+extern "C" int rbd_aio_is_complete(rbd_completion_t c)
+{
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  return comp->is_complete();
+}
+
+extern "C" int rbd_aio_wait_for_complete(rbd_completion_t c)
+{
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  return comp->wait_for_complete();
+}
+
+extern "C" ssize_t rbd_aio_get_return_value(rbd_completion_t c)
+{
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  return comp->get_return_value();
+}
+
+extern "C" void rbd_aio_release(rbd_completion_t c)
+{
+  librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;
+  comp->release();
+}
diff --git a/ceph/src/librbd/parent_types.h b/ceph/src/librbd/parent_types.h
new file mode 100644
index 00000000..4dcc4529
--- /dev/null
+++ b/ceph/src/librbd/parent_types.h
@@ -0,0 +1,41 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_LIBRBD_PARENT_TYPES_H
+#define CEPH_LIBRBD_PARENT_TYPES_H
+
+// parent_spec uniquely identifies a parent in the clone relationship
+// (clone(parent) creates child, then parent_spec <-> child_imageid)
+
+namespace librbd {
+  struct parent_spec {
+    int64_t pool_id;
+    string image_id;
+    snapid_t snap_id;
+    parent_spec() : pool_id(-1), snap_id(CEPH_NOSNAP) {}
+    parent_spec(uint64_t pool_id, string image_id, snapid_t snap_id) :
+      pool_id(pool_id), image_id(image_id), snap_id(snap_id) {}
+    bool operator==(const parent_spec &other) {
+      return ((this->pool_id == other.pool_id) &&
+	      (this->image_id == other.image_id) &&
+	      (this->snap_id == other.snap_id));
+    }
+    bool operator!=(const parent_spec &other) {
+      return !(*this == other);
+    }
+  };
+
+  struct parent_info {
+    parent_spec spec;
+    uint64_t overlap;
+    parent_info() : overlap(0) {}
+  };
+}
+
+enum {
+  RBD_PROTECTION_STATUS_UNPROTECTED  = 0,
+  RBD_PROTECTION_STATUS_UNPROTECTING = 1,
+  RBD_PROTECTION_STATUS_PROTECTED    = 2,
+  RBD_PROTECTION_STATUS_LAST         = 3
+};
+
+#endif
diff --git a/ceph/src/libs3/COPYING b/ceph/src/libs3/COPYING
new file mode 100644
index 00000000..94a9ed02
--- /dev/null
+++ b/ceph/src/libs3/COPYING
@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. 
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    
+    Copyright (C)   
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+      Copyright (C)   
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+.
diff --git a/ceph/src/libs3/ChangeLog b/ceph/src/libs3/ChangeLog
new file mode 100644
index 00000000..fc8797ac
--- /dev/null
+++ b/ceph/src/libs3/ChangeLog
@@ -0,0 +1,16 @@
+Thu Sep 18 10:03:02 NZST 2008   bryan@ischo.com
+	* This file is no longer maintained, sorry
+
+Sat Aug  9 13:44:21 NZST 2008   bryan@ischo.com
+	* Fixed bug wherein keys with non-URI-safe characters did not work
+	  correctly because they were not being URI-encoded in the request UR
+	* Split RPM and DEB packages into normal and devel packages
+
+Fri Aug  8 22:40:19 NZST 2008	bryan@ischo.com
+	* Branched 0.4
+	* Created RPM and Debian packaging
+
+Tue Aug  5 08:52:33 NZST 2008	bryan@ischo.com
+	* Bumped version number to 0.3
+	* Moved Makefile to GNUmakefile, added shared library build
+	* Added a bunch of GNU standard files (README, INSTALL, ChangeLog, etc)
diff --git a/ceph/src/libs3/GNUmakefile b/ceph/src/libs3/GNUmakefile
new file mode 100644
index 00000000..7a74ec37
--- /dev/null
+++ b/ceph/src/libs3/GNUmakefile
@@ -0,0 +1,414 @@
+# GNUmakefile
+# 
+# Copyright 2008 Bryan Ischo 
+# 
+# This file is part of libs3.
+# 
+# libs3 is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, version 3 of the License.
+#
+# In addition, as a special exception, the copyright holders give
+# permission to link the code of this library and its programs with the
+# OpenSSL library, and distribute linked combinations including the two.
+#
+# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License version 3
+# along with libs3, in a file named COPYING.  If not, see
+# .
+
+# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools
+# but I just couldn't stomach them.  Since this is a Makefile for POSIX
+# systems, I will simply do away with autohell completely and use a GNU
+# Makefile.  GNU make ought to be available pretty much everywhere, so I
+# don't see this being a significant issue for portability.
+
+# All commands assume a GNU compiler.  For systems which do not use a GNU
+# compiler, write scripts with the same names as these commands, and taking
+# the same arguments, and translate the arguments and commands into the
+# appropriate non-POSIX ones as needed.  libs3 assumes a GNU toolchain as
+# the most portable way to build software possible.  Non-POSIX, non-GNU
+# systems can do the work of supporting this build infrastructure.
+
+
+# --------------------------------------------------------------------------
+# Set libs3 version number, unless it is already set.
+# This is trunk0.trunk0 on the libs3 git master branch; release branches
+# are created with this set to specific version numbers when releases are
+# made.
+
+LIBS3_VER_MAJOR ?= trunk0
+LIBS3_VER_MINOR ?= trunk0
+LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR)
+
+
+# -----------------------------------------------------------------------------
+# Determine verbosity.  VERBOSE_SHOW should be prepended to every command which
+# should only be displayed if VERBOSE is set.  QUIET_ECHO may be used to
+# echo text only if VERBOSE is not set.  Typically, a VERBOSE_SHOW command will
+# be paired with a QUIET_ECHO command, to provide a command which is displayed
+# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to
+# describe the command.
+#
+# No matter what VERBOSE is defined to, it ends up as true if it's defined.
+# This will be weird if you defined VERBOSE=false in the environment, and we
+# switch it to true here; but the meaning of VERBOSE is, "if it's defined to
+# any value, then verbosity is turned on".  So don't define VERBOSE if you
+# don't want verbosity in the build process.
+# -----------------------------------------------------------------------------
+
+ifdef VERBOSE
+        VERBOSE = true
+        VERBOSE_ECHO = @ echo
+        VERBOSE_SHOW =
+        QUIET_ECHO = @ echo > /dev/null
+else
+        VERBOSE = false
+        VERBOSE_ECHO = @ echo > /dev/null
+        VERBOSE_SHOW = @
+        QUIET_ECHO = @ echo
+endif
+
+
+# --------------------------------------------------------------------------
+# BUILD directory
+ifndef BUILD
+    ifdef DEBUG
+        BUILD := build-debug
+    else
+        BUILD := build
+    endif
+endif
+
+
+# --------------------------------------------------------------------------
+# DESTDIR directory
+ifndef DESTDIR
+    DESTDIR := /usr
+endif
+
+
+# --------------------------------------------------------------------------
+# Acquire configuration information for libraries that libs3 depends upon
+
+ifndef CURL_LIBS
+    CURL_LIBS := $(shell curl-config --libs)
+endif
+
+ifndef CURL_CFLAGS
+    CURL_CFLAGS := $(shell curl-config --cflags)
+endif
+
+ifndef LIBXML2_LIBS
+    LIBXML2_LIBS := $(shell xml2-config --libs)
+endif
+
+ifndef LIBXML2_CFLAGS
+    LIBXML2_CFLAGS := $(shell xml2-config --cflags)
+endif
+
+
+# --------------------------------------------------------------------------
+# These CFLAGS assume a GNU compiler.  For other compilers, write a script
+# which converts these arguments into their equivalent for that particular
+# compiler.
+
+ifndef CFLAGS
+    ifdef DEBUG
+        CFLAGS := -g
+    else
+        CFLAGS := -O3
+    endif
+endif
+
+CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \
+          $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \
+          -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \
+          -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \
+          -DLIBS3_VER=\"$(LIBS3_VER)\" \
+          -D__STRICT_ANSI__ \
+          -D_ISOC99_SOURCE \
+          -D_POSIX_C_SOURCE=200112L
+
+LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) -lpthread
+
+
+# --------------------------------------------------------------------------
+# Default targets are everything
+
+.PHONY: all
+all: exported test
+
+
+# --------------------------------------------------------------------------
+# Exported targets are the library and driver program
+
+.PHONY: exported
+exported: libs3 s3 headers
+
+
+# --------------------------------------------------------------------------
+# Install target
+
+# adding empty install target, don't want to install anything when integrated
+# with ceph
+.PHONY: install
+install:
+
+# this is the original install target
+.PHONY: install-all
+install-all: exported
+	$(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable
+	$(VERBOSE_SHOW) install -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \
+                    $(DESTDIR)/bin/s3
+	$(QUIET_ECHO) \
+        $(DESTDIR)/lib/libs3.so.$(LIBS3_VER): Installing shared library
+	$(VERBOSE_SHOW) install -Dps -m u+rw,go+r \
+               $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \
+               $(DESTDIR)/lib/libs3.so.$(LIBS3_VER)
+	$(QUIET_ECHO) \
+        $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR): Linking shared library
+	$(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER) \
+               $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR)
+	$(QUIET_ECHO) $(DESTDIR)/lib/libs3.so: Linking shared library
+	$(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(DESTDIR)/lib/libs3.so
+	$(QUIET_ECHO) $(DESTDIR)/lib/libs3.a: Installing static library
+	$(VERBOSE_SHOW) install -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \
+                    $(DESTDIR)/lib/libs3.a
+	$(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header
+	$(VERBOSE_SHOW) install -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \
+                    $(DESTDIR)/include/libs3.h
+
+
+# --------------------------------------------------------------------------
+# Uninstall target
+
+.PHONY: uninstall
+uninstall:
+	$(QUIET_ECHO) Installed files: Uninstalling
+	$(VERBOSE_SHOW) \
+	    rm -f $(DESTDIR)/bin/s3 \
+              $(DESTDIR)/include/libs3.h \
+              $(DESTDIR)/lib/libs3.a \
+              $(DESTDIR)/lib/libs3.so \
+              $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR) \
+              $(DESTDIR)/lib/libs3.so.$(LIBS3_VER)
+
+
+# --------------------------------------------------------------------------
+# Compile target patterns
+
+$(BUILD)/obj/%.o: src/%.c
+	$(QUIET_ECHO) $@: Compiling object
+	@ mkdir -p $(dir $(BUILD)/dep/$<)
+	@ gcc $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
+        -o $(BUILD)/dep/$(<:%.c=%.d) -c $<
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc $(CFLAGS) -o $@ -c $<
+
+$(BUILD)/obj/%.do: src/%.c
+	$(QUIET_ECHO) $@: Compiling dynamic object
+	@ mkdir -p $(dir $(BUILD)/dep/$<)
+	@ gcc $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
+        -o $(BUILD)/dep/$(<:%.c=%.dd) -c $<
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc $(CFLAGS) -fpic -fPIC -o $@ -c $< 
+
+
+# --------------------------------------------------------------------------
+# libs3 library targets
+
+LIBS3_SHARED = $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR)
+LIBS3_STATIC = $(BUILD)/lib/libs3.a
+
+.PHONY: libs3
+libs3: $(LIBS3_SHARED) $(LIBS3_STATIC)
+
+LIBS3_SOURCES := acl.c bucket.c error_parser.c general.c \
+                 object.c request.c request_context.c \
+                 response_headers_handler.c service_access_logging.c \
+                 service.c simplexml.c util.c
+
+$(LIBS3_SHARED): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.do)
+	$(QUIET_ECHO) $@: Building shared library
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc -shared -Wl,-soname,libs3.so.$(LIBS3_VER_MAJOR) \
+        -o $@ $^ $(LDFLAGS)
+
+$(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o)
+	$(QUIET_ECHO) $@: Building static library
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(AR) cr $@ $^
+
+
+# --------------------------------------------------------------------------
+# Driver program targets
+
+.PHONY: s3
+s3: $(BUILD)/bin/s3
+
+$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED)
+	$(QUIET_ECHO) $@: Building executable
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc -o $@ $^ $(LDFLAGS)
+
+
+# --------------------------------------------------------------------------
+# libs3 header targets
+
+.PHONY: headers
+headers: $(BUILD)/include/libs3.h
+
+$(BUILD)/include/libs3.h: inc/libs3.h
+	$(QUIET_ECHO) $@: Linking header
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) ln -sf $(abspath $<) $@
+
+
+# --------------------------------------------------------------------------
+# Test targets
+
+.PHONY: test
+test: $(BUILD)/bin/testsimplexml
+
+$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC)
+	$(QUIET_ECHO) $@: Building executable
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc -o $@ $^ $(LIBXML2_LIBS)
+
+
+# --------------------------------------------------------------------------
+# Check target
+
+check:
+distdir:
+dist:
+
+# --------------------------------------------------------------------------
+# Clean target
+
+.PHONY: clean
+clean:
+	$(QUIET_ECHO) $(BUILD): Cleaning
+	$(VERBOSE_SHOW) rm -rf $(BUILD)
+
+.PHONY: distclean
+distclean:
+	$(QUIET_ECHO) $(BUILD): Cleaning
+	$(VERBOSE_SHOW) rm -rf $(BUILD)
+
+
+# --------------------------------------------------------------------------
+# Clean dependencies target
+
+.PHONY: cleandeps
+cleandeps:
+	$(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies
+	$(VERBOSE_SHOW) rm -rf $(BUILD)/dep
+
+
+# --------------------------------------------------------------------------
+# Dependencies
+
+ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c
+
+$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d)))
+$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd)))
+
+
+# --------------------------------------------------------------------------
+# Debian package target
+
+DEBPKG = $(BUILD)/pkg/libs3_$(LIBS3_VER).deb
+DEBDEVPKG = $(BUILD)/pkg/libs3-dev_$(LIBS3_VER).deb
+
+.PHONY: deb
+deb: $(DEBPKG) $(DEBDEVPKG)
+
+$(DEBPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \
+                       cut -d '=' -f 2)
+$(DEBPKG): exported $(BUILD)/deb/DEBIAN/control $(BUILD)/deb/DEBIAN/shlibs \
+           $(BUILD)/deb/DEBIAN/postinst \
+           $(BUILD)/deb/usr/share/doc/libs3/changelog.gz \
+           $(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz \
+           $(BUILD)/deb/usr/share/doc/libs3/copyright
+	DESTDIR=$(BUILD)/deb/usr $(MAKE) install
+	rm -rf $(BUILD)/deb/usr/include
+	rm -f $(BUILD)/deb/usr/lib/libs3.a
+	@mkdir -p $(dir $@)
+	fakeroot dpkg-deb -b $(BUILD)/deb $@
+	mv $@ $(BUILD)/pkg/libs3_$(LIBS3_VER)_$(DEBARCH).deb
+
+$(DEBDEVPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \
+                          cut -d '=' -f 2)
+$(DEBDEVPKG): exported $(BUILD)/deb-dev/DEBIAN/control \
+           $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz \
+           $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz \
+           $(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright
+	DESTDIR=$(BUILD)/deb-dev/usr $(MAKE) install
+	rm -rf $(BUILD)/deb-dev/usr/bin
+	rm -f $(BUILD)/deb-dev/usr/lib/libs3.so*
+	@mkdir -p $(dir $@)
+	fakeroot dpkg-deb -b $(BUILD)/deb-dev $@
+	mv $@ $(BUILD)/pkg/libs3-dev_$(LIBS3_VER)_$(DEBARCH).deb
+
+$(BUILD)/deb/DEBIAN/control: debian/control
+	@mkdir -p $(dir $@)
+	echo -n "Depends: " > $@
+	dpkg-shlibdeps -Sbuild -O $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) | \
+            cut -d '=' -f 2- >> $@
+	sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \
+            < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' | \
+            grep -v ^Source: >> $@
+
+$(BUILD)/deb-dev/DEBIAN/control: debian/control.dev
+	@mkdir -p $(dir $@)
+	sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \
+            < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' > $@
+
+$(BUILD)/deb/DEBIAN/shlibs:
+	echo -n "libs3 $(LIBS3_VER_MAJOR) libs3 " > $@
+	echo "(>= $(LIBS3_VER))" >> $@
+
+$(BUILD)/deb/DEBIAN/postinst: debian/postinst
+	@mkdir -p $(dir $@)
+	cp $< $@
+
+$(BUILD)/deb/usr/share/doc/libs3/copyright: LICENSE
+	@mkdir -p $(dir $@)
+	cp $< $@
+	@echo >> $@
+	@echo -n "An alternate location for the GNU General Public " >> $@
+	@echo "License version 3 on Debian" >> $@
+	@echo "systems is /usr/share/common-licenses/GPL-3." >> $@
+
+$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright: LICENSE
+	@mkdir -p $(dir $@)
+	cp $< $@
+	@echo >> $@
+	@echo -n "An alternate location for the GNU General Public " >> $@
+	@echo "License version 3 on Debian" >> $@
+	@echo "systems is /usr/share/common-licenses/GPL-3." >> $@
+
+$(BUILD)/deb/usr/share/doc/libs3/changelog.gz: debian/changelog
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz: debian/changelog
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz: debian/changelog.Debian
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz: \
+    debian/changelog.Debian
+	@mkdir -p $(dir $@)
+	gzip --best -c $< > $@
+
+
diff --git a/ceph/src/libs3/GNUmakefile.mingw b/ceph/src/libs3/GNUmakefile.mingw
new file mode 100644
index 00000000..175d1e92
--- /dev/null
+++ b/ceph/src/libs3/GNUmakefile.mingw
@@ -0,0 +1,296 @@
+# GNUmakefile.mingw
+# 
+# Copyright 2008 Bryan Ischo 
+# 
+# This file is part of libs3.
+# 
+# libs3 is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, version 3 of the License.
+#
+# In addition, as a special exception, the copyright holders give
+# permission to link the code of this library and its programs with the
+# OpenSSL library, and distribute linked combinations including the two.
+#
+# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License version 3
+# along with libs3, in a file named COPYING.  If not, see
+# .
+
+# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools
+# but I just couldn't stomach them.  Since this is a Makefile for POSIX
+# systems, I will simply do away with autohell completely and use a GNU
+# Makefile.  GNU make ought to be available pretty much everywhere, so I
+# don't see this being a significant issue for portability.
+
+# All commands assume a GNU compiler.  For systems which do not use a GNU
+# compiler, write scripts with the same names as these commands, and taking
+# the same arguments, and translate the arguments and commands into the
+# appropriate non-POSIX ones as needed.  libs3 assumes a GNU toolchain as
+# the most portable way to build software possible.  Non-POSIX, non-GNU
+# systems can do the work of supporting this build infrastructure.
+
+
+# --------------------------------------------------------------------------
+# Set libs3 version number, unless it is already set.
+# This is trunk0.trunk0 on the libs3 git master branch; release branches
+# are created with this set to specific version numbers when releases are
+# made.
+
+LIBS3_VER_MAJOR ?= trunk0
+LIBS3_VER_MINOR ?= trunk0
+LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR)
+
+
+# -----------------------------------------------------------------------------
+# Determine verbosity.  VERBOSE_SHOW should be prepended to every command which
+# should only be displayed if VERBOSE is set.  QUIET_ECHO may be used to
+# echo text only if VERBOSE is not set.  Typically, a VERBOSE_SHOW command will
+# be paired with a QUIET_ECHO command, to provide a command which is displayed
+# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to
+# describe the command.
+#
+# No matter what VERBOSE is defined to, it ends up as true if it's defined.
+# This will be weird if you defined VERBOSE=false in the environment, and we
+# switch it to true here; but the meaning of VERBOSE is, "if it's defined to
+# any value, then verbosity is turned on".  So don't define VERBOSE if you
+# don't want verbosity in the build process.
+# -----------------------------------------------------------------------------
+
+ifdef VERBOSE
+        VERBOSE = true
+        VERBOSE_ECHO = @ echo
+        VERBOSE_SHOW =
+        QUIET_ECHO = @ echo >nul
+else
+        VERBOSE = false
+        VERBOSE_ECHO = @ echo >nul
+        VERBOSE_SHOW = @
+        QUIET_ECHO = @ echo
+endif
+
+
+# --------------------------------------------------------------------------
+# BUILD directory
+ifndef BUILD
+    ifdef DEBUG
+        BUILD := build-debug
+    else
+        BUILD := build
+    endif
+endif
+
+
+# --------------------------------------------------------------------------
+# DESTDIR directory
+ifndef DESTDIR
+    DESTDIR := libs3-$(LIBS3_VER)
+endif
+
+
+# --------------------------------------------------------------------------
+# Acquire configuration information for libraries that libs3 depends upon
+
+ifndef CURL_LIBS
+    CURL_LIBS := -Lc:\libs3-libs\bin -lcurl
+endif
+
+ifndef CURL_CFLAGS
+    CURL_CFLAGS := -Ic:\libs3-libs\include
+endif
+
+ifndef LIBXML2_LIBS
+    LIBXML2_LIBS := -Lc:\libs3-libs\bin -lxml2
+endif
+
+ifndef LIBXML2_CFLAGS
+    LIBXML2_CFLAGS := -Ic:\libs3-libs\include
+endif
+
+
+# --------------------------------------------------------------------------
+# These CFLAGS assume a GNU compiler.  For other compilers, write a script
+# which converts these arguments into their equivalent for that particular
+# compiler.
+
+ifndef CFLAGS
+    ifdef DEBUG
+        CFLAGS := -g
+    else
+        CFLAGS := -O3
+    endif
+endif
+
+CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \
+          $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \
+          -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \
+          -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \
+          -DLIBS3_VER=\"$(LIBS3_VER)\" \
+          -D__STRICT_ANSI__ \
+          -D_ISOC99_SOURCE \
+          -D_POSIX_C_SOURCE=200112L \
+          -Dsleep=Sleep -DSLEEP_UNITS_PER_SECOND=1000 \
+          -DFOPEN_EXTRA_FLAGS=\"b\" \
+          -Iinc/mingw -include windows.h
+
+LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS)
+
+# --------------------------------------------------------------------------
+# Default targets are everything
+
+.PHONY: all
+all: exported test
+
+
+# --------------------------------------------------------------------------
+# Exported targets are the library and driver program
+
+.PHONY: exported
+exported: libs3 s3 headers
+
+
+# --------------------------------------------------------------------------
+# Install target
+
+.PHONY: install
+install: exported
+	$(QUIET_ECHO) $(DESTDIR)/bin/s3.exe: Installing executable
+	- @ mkdir $(DESTDIR)\bin 2>&1 | echo >nul
+	$(VERBOSE_SHOW) copy $(BUILD)\bin\s3.exe $(DESTDIR)\bin\s3.exe >nul
+	$(QUIET_ECHO) $(DESTDIR)/bin/libs3/dll: Installing dynamic library
+	$(VERBOSE_SHOW) copy $(BUILD)\bin\libs3.dll $(DESTDIR)\bin\libs3.dll >nul
+	$(QUIET_ECHO) $(DESTDIR)/lib/libs3.a: Installing static library
+	- @ mkdir $(DESTDIR)\lib 2>&1 | echo >nul
+	$(VERBOSE_SHOW) copy $(BUILD)\lib\libs3.a $(DESTDIR)\lib\libs3.a >nul
+	$(QUIET_ECHO) $(DESTDIR)/lib/libs3.def: Installing def file
+	$(VERBOSE_SHOW) copy mswin\libs3.def $(DESTDIR)\lib\libs3.def >nul
+	- @ mkdir $(DESTDIR)\include 2>&1 | echo >nul
+	$(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Copying header
+	$(VERBOSE_SHOW) copy $(BUILD)\include\libs3.h \
+                    $(DESTDIR)\include\libs3.h >nul
+	$(QUIET_ECHO) $(DESTDIR)/LICENSE: Copying license
+	$(VERBOSE_SHOW) copy LICENSE $(DESTDIR)\LICENSE >nul
+	$(QUIET_ECHO) $(DESTDIR)/COPYING: Copying license
+	$(VERBOSE_SHOW) copy COPYING $(DESTDIR)\COPYING >nul
+
+
+# --------------------------------------------------------------------------
+# Uninstall target
+
+.PHONY: uninstall
+uninstall:
+	$(QUIET_ECHO) Installed files: Uninstalling
+	$(VERBOSE_SHOW) \
+	    del $(DESTDIR)\bin\s3.exe \
+            $(DESTDIR)\bin\libs3.dll \
+            $(DESTDIR)\lib\libs3.a \
+            $(DESTDIR)\lib\libs3.def \
+            $(DESTDIR)\include\libs3.h \
+            $(DESTDIR)\LICENSE \
+            $(DESTDIR)\COPYING
+
+
+# --------------------------------------------------------------------------
+# Compile target patterns
+
+$(BUILD)/obj/%.o: src/%.c
+	$(QUIET_ECHO) $@: Compiling object
+	- @ mkdir $(subst /,\,$(dir $(BUILD)/dep/$<)) 2>&1 | echo >nul
+	@ gcc $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
+        -o $(BUILD)/dep/$(<:%.c=%.d) -c $<
+	- @ mkdir $(subst /,\,$(dir $@)) 2>&1 | echo >nul
+	$(VERBOSE_SHOW) gcc $(CFLAGS) -o $@ -c $<
+
+
+# --------------------------------------------------------------------------
+# libs3 library targets
+
+LIBS3_SHARED = $(BUILD)/bin/libs3.dll
+LIBS3_STATIC = $(BUILD)/lib/libs3.a
+
+.PHONY: libs3
+libs3: $(LIBS3_SHARED) $(BUILD)/lib/libs3.a
+
+LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \
+                 src/object.c src/request.c src/request_context.c \
+                 src/response_headers_handler.c src/service_access_logging.c \
+                 src/service.c src/simplexml.c src/util.c src/mingw_functions.c
+
+$(LIBS3_SHARED): $(LIBS3_SOURCES:src/%.c=$(BUILD)/obj/%.o)
+	$(QUIET_ECHO) $@: Building dynamic library
+	- @ mkdir $(subst /,\,$(dir $@)) 2>&1 | echo >nul
+	$(VERBOSE_SHOW) gcc -shared -o $@ $^ $(LDFLAGS) -lws2_32
+
+$(LIBS3_STATIC): $(LIBS3_SHARED)
+	$(QUIET_ECHO) $@: Building static library
+	- @ mkdir $(subst /,\,$(dir $@)) 2>&1 | echo >nul
+	$(VERBOSE_SHOW) dlltool --def mswin\libs3.def --dllname $(subst /,\,$<) \
+            --output-lib $(subst /,\,$@)
+
+
+# --------------------------------------------------------------------------
+# Driver program targets
+
+.PHONY: s3
+s3: $(BUILD)/bin/s3.exe
+
+$(BUILD)/bin/s3.exe: $(BUILD)/obj/s3.o $(BUILD)/obj/mingw_s3_functions.o \
+                     $(BUILD)/lib/libs3.a
+	$(QUIET_ECHO) $@: Building executable
+	- @ mkdir $(subst /,\,$(dir $@)) 2>&1 | echo >nul
+	$(VERBOSE_SHOW) gcc -o $@ $^ $(LDFLAGS) -lws2_32
+
+
+# --------------------------------------------------------------------------
+# libs3 header targets
+
+.PHONY: headers
+headers: $(BUILD)\include\libs3.h
+
+$(BUILD)\include\libs3.h: inc\libs3.h
+	$(QUIET_ECHO) $@: Copying header
+	- @ mkdir $(subst /,\,$(dir $@)) 2>&1 | echo >nul
+	$(VERBOSE_SHOW) copy $< $@
+
+
+# --------------------------------------------------------------------------
+# Test targets
+
+.PHONY: test
+test: $(BUILD)/bin/testsimplexml
+
+$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o \
+                            $(BUILD)/obj/simplexml.o
+	$(QUIET_ECHO) $@: Building executable
+	- @ mkdir $(subst /,\,$(dir $@)) 2>&1 | echo >nul
+	$(VERBOSE_SHOW) gcc -o $@ $^ $(LIBXML2_LIBS)
+
+
+# --------------------------------------------------------------------------
+# Clean target
+
+.PHONY: clean
+clean:
+	$(QUIET_ECHO) $(BUILD): Cleaning
+	$(VERBOSE_SHOW) mswin\rmrf.bat $(BUILD)
+
+
+# --------------------------------------------------------------------------
+# Clean dependencies target
+
+.PHONY: cleandeps
+cleandeps:
+	$(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies
+	$(VERBOSE_SHOW) mswin\rmrf.bat $(BUILD)\dep
+
+
+# --------------------------------------------------------------------------
+# Dependencies
+
+ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c
+
+$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d)))
diff --git a/ceph/src/libs3/GNUmakefile.osx b/ceph/src/libs3/GNUmakefile.osx
new file mode 100644
index 00000000..d4de5620
--- /dev/null
+++ b/ceph/src/libs3/GNUmakefile.osx
@@ -0,0 +1,305 @@
+# GNUmakefile.osx
+# 
+# Copyright 2008 Bryan Ischo 
+# 
+# This file is part of libs3.
+# 
+# libs3 is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, version 3 of the License.
+#
+# In addition, as a special exception, the copyright holders give
+# permission to link the code of this library and its programs with the
+# OpenSSL library, and distribute linked combinations including the two.
+#
+# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License version 3
+# along with libs3, in a file named COPYING.  If not, see
+# .
+
+# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools
+# but I just couldn't stomach them.  Since this is a Makefile for POSIX
+# systems, I will simply do away with autohell completely and use a GNU
+# Makefile.  GNU make ought to be available pretty much everywhere, so I
+# don't see this being a significant issue for portability.
+
+# All commands assume a GNU compiler.  For systems which do not use a GNU
+# compiler, write scripts with the same names as these commands, and taking
+# the same arguments, and translate the arguments and commands into the
+# appropriate non-POSIX ones as needed.  libs3 assumes a GNU toolchain as
+# the most portable way to build software possible.  Non-POSIX, non-GNU
+# systems can do the work of supporting this build infrastructure.
+
+
+# --------------------------------------------------------------------------
+# Set libs3 version number, unless it is already set.
+# This is trunk0.trunk0 on the libs3 git master branch; release branches
+# are created with this set to specific version numbers when releases are
+# made.
+
+LIBS3_VER_MAJOR ?= trunk0
+LIBS3_VER_MINOR ?= trunk0
+LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR)
+
+
+# -----------------------------------------------------------------------------
+# Determine verbosity.  VERBOSE_SHOW should be prepended to every command which
+# should only be displayed if VERBOSE is set.  QUIET_ECHO may be used to
+# echo text only if VERBOSE is not set.  Typically, a VERBOSE_SHOW command will
+# be paired with a QUIET_ECHO command, to provide a command which is displayed
+# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to
+# describe the command.
+#
+# No matter what VERBOSE is defined to, it ends up as true if it's defined.
+# This will be weird if you defined VERBOSE=false in the environment, and we
+# switch it to true here; but the meaning of VERBOSE is, "if it's defined to
+# any value, then verbosity is turned on".  So don't define VERBOSE if you
+# don't want verbosity in the build process.
+# -----------------------------------------------------------------------------
+
+ifdef VERBOSE
+        VERBOSE = true
+        VERBOSE_ECHO = @ echo
+        VERBOSE_SHOW =
+        QUIET_ECHO = @ echo > /dev/null
+else
+        VERBOSE = false
+        VERBOSE_ECHO = @ echo > /dev/null
+        VERBOSE_SHOW = @
+        QUIET_ECHO = @ echo
+endif
+
+
+# --------------------------------------------------------------------------
+# BUILD directory
+ifndef BUILD
+    ifdef DEBUG
+        BUILD := build-debug
+    else
+        BUILD := build
+    endif
+endif
+
+
+# --------------------------------------------------------------------------
+# DESTDIR directory
+ifndef DESTDIR
+    DESTDIR := /usr
+endif
+
+
+# --------------------------------------------------------------------------
+# Acquire configuration information for libraries that libs3 depends upon
+
+ifndef CURL_LIBS
+    CURL_LIBS := $(shell curl-config --libs)
+endif
+
+ifndef CURL_CFLAGS
+    CURL_CFLAGS := $(shell curl-config --cflags)
+endif
+
+ifndef LIBXML2_LIBS
+    LIBXML2_LIBS := $(shell xml2-config --libs)
+endif
+
+ifndef LIBXML2_CFLAGS
+    LIBXML2_CFLAGS := $(shell xml2-config --cflags)
+endif
+
+
+# --------------------------------------------------------------------------
+# These CFLAGS assume a GNU compiler.  For other compilers, write a script
+# which converts these arguments into their equivalent for that particular
+# compiler.
+
+ifndef CFLAGS
+    ifdef DEBUG
+        CFLAGS := -g
+    else
+        CFLAGS := -O3
+    endif
+endif
+
+CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \
+          $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \
+          -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \
+          -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \
+          -DLIBS3_VER=\"$(LIBS3_VER)\" \
+          -D__STRICT_ANSI__ \
+          -D_ISOC99_SOURCE \
+          -fno-common
+
+LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) -lpthread
+
+
+# --------------------------------------------------------------------------
+# Default targets are everything
+
+.PHONY: all
+all: exported test
+
+
+# --------------------------------------------------------------------------
+# Exported targets are the library and driver program
+
+.PHONY: exported
+exported: libs3 s3 headers
+
+
+# --------------------------------------------------------------------------
+# Install target
+
+.PHONY: install
+install: exported
+	$(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable
+	$(VERBOSE_SHOW) install -ps -m u+rwx,go+rx $(BUILD)/bin/s3 \
+                    $(DESTDIR)/bin/s3
+	$(QUIET_ECHO) \
+        $(DESTDIR)/lib/libs3.$(LIBS3_VER).dylib: Installing dynamic library
+	$(VERBOSE_SHOW) install -p -m u+rw,go+r \
+                    $(BUILD)/lib/libs3.$(LIBS3_VER_MAJOR).dylib \
+                    $(DESTDIR)/lib/libs3.$(LIBS3_VER).dylib
+	$(QUIET_ECHO) \
+        $(DESTDIR)/lib/libs3.$(LIBS3_VER_MAJOR).dylib: Linking dynamic library
+	$(VERBOSE_SHOW) ln -sf libs3.$(LIBS3_VER).dylib \
+                    $(DESTDIR)/lib/libs3.$(LIBS3_VER_MAJOR).dylib
+	$(QUIET_ECHO) $(DESTDIR)/lib/libs3.dylib: Linking dynamic library
+	$(VERBOSE_SHOW) ln -sf libs3.$(LIBS3_VER_MAJOR).dylib \
+                    $(DESTDIR)/lib/libs3.dylib
+	$(QUIET_ECHO) $(DESTDIR)/lib/libs3.a: Installing static library
+	$(VERBOSE_SHOW) install -p -m u+rw,go+r $(BUILD)/lib/libs3.a \
+                    $(DESTDIR)/lib/libs3.a
+	$(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header
+	$(VERBOSE_SHOW) install -p -m u+rw,go+r $(BUILD)/include/libs3.h \
+                    $(DESTDIR)/include/libs3.h
+
+
+# --------------------------------------------------------------------------
+# Uninstall target
+
+.PHONY: uninstall
+uninstall:
+	$(QUIET_ECHO) Installed files: Uninstalling
+	$(VERBOSE_SHOW) \
+        rm -f $(DESTDIR)/bin/s3 \
+              $(DESTDIR)/lib/libs3.dylib \
+              $(DESTDIR)/lib/libs3.$(LIBS3_VER_MAJOR).dylib \
+              $(DESTDIR)/lib/libs3.$(LIBS3_VER).dylib \
+              $(DESTDIR)/lib/libs3.a \
+              $(DESTDIR)/include/libs3.h
+
+
+# --------------------------------------------------------------------------
+# Compile target patterns
+
+$(BUILD)/obj/%.o: src/%.c
+	$(QUIET_ECHO) $@: Compiling object
+	@ mkdir -p $(dir $(BUILD)/dep/$<)
+	@ gcc $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
+        -o $(BUILD)/dep/$(<:%.c=%.d) -c $<
+	@ mkdir -p $(dir $@)
+	@(VERBOSE_SHOW) gcc $(CFLAGS) -o $@ -c $<
+
+$(BUILD)/obj/%.do: src/%.c
+	$(QUIET_ECHO) $@: Compiling dynamic object
+	@ mkdir -p $(dir $(BUILD)/dep/$<)
+	@ gcc $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \
+        -o $(BUILD)/dep/$(<:%.c=%.dd) -c $<
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc $(CFLAGS) -fpic -fPIC -o $@ -c $< 
+
+
+# --------------------------------------------------------------------------
+# libs3 library targets
+
+LIBS3_SHARED = $(BUILD)/lib/libs3.$(LIBS3_VER_MAJOR).dylib
+LIBS3_STATIC = $(BUILD)/lib/libs3.a
+
+.PHONY: libs3
+libs3: $(LIBS3_SHARED) $(LIBS3_SHARED_MAJOR) $(BUILD)/lib/libs3.a
+
+LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \
+                 src/object.c src/request.c src/request_context.c \
+                 src/response_headers_handler.c src/service_access_logging.c \
+                 src/service.c src/simplexml.c src/util.c
+
+$(LIBS3_SHARED): $(LIBS3_SOURCES:src/%.c=$(BUILD)/obj/%.do)
+	$(QUIET_ECHO) $@: Building shared library
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc -dynamiclib -install_name \
+        libs3.$(LIBS3_VER_MAJOR).dylib \
+        -compatibility_version $(LIBS3_VER_MAJOR) \
+        -current_version $(LIBS3_VER) -o $@ $^ $(LDFLAGS)
+
+$(LIBS3_STATIC): $(LIBS3_SOURCES:src/%.c=$(BUILD)/obj/%.o)
+	$(QUIET_ECHO) $@: Building static library
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) $(AR) cr $@ $^
+
+
+# --------------------------------------------------------------------------
+# Driver program targets
+
+.PHONY: s3
+s3: $(BUILD)/bin/s3
+
+$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED)
+	$(QUIET_ECHO) $@: Building executable
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc -o $@ $^ $(LDFLAGS)
+
+
+# --------------------------------------------------------------------------
+# libs3 header targets
+
+.PHONY: headers
+headers: $(BUILD)/include/libs3.h
+
+$(BUILD)/include/libs3.h: inc/libs3.h
+	$(QUIET_ECHO) $@: Linking header
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) ln -sf $(abspath $<) $@
+
+
+# --------------------------------------------------------------------------
+# Test targets
+
+.PHONY: test
+test: $(BUILD)/bin/testsimplexml
+
+$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC)
+	$(QUIET_ECHO) $@: Building executable
+	@ mkdir -p $(dir $@)
+	$(VERBOSE_SHOW) gcc -o $@ $^ $(LIBXML2_LIBS)
+
+
+# --------------------------------------------------------------------------
+# Clean target
+
+.PHONY: clean
+clean:
+	$(QUIET_ECHO) $(BUILD): Cleaning
+	$(VERBOSE_SHOW) rm -rf $(BUILD)
+
+
+# --------------------------------------------------------------------------
+# Clean dependencies target
+
+.PHONY: cleandeps
+cleandeps:
+	$(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies
+	$(VERBOSE_SHOW) rm -rf $(BUILD)/dep
+
+
+# --------------------------------------------------------------------------
+# Dependencies
+
+ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c
+
+$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d)))
+$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd)))
diff --git a/ceph/src/libs3/INSTALL b/ceph/src/libs3/INSTALL
new file mode 100644
index 00000000..54431fc4
--- /dev/null
+++ b/ceph/src/libs3/INSTALL
@@ -0,0 +1,73 @@
+
+To install libs3 on a POSIX system (except Microsoft Windows):
+--------------------------------------------------------------
+
+Note that all POSIX builds have prerequisites, such as development libraries
+that libs3 requires and that must be installed at the time that libs3 is
+built.  The easiest way to find out what those are, is to run the build
+command and then observe the results.
+
+*** For RPM-based systems (Fedora Core, Mandrake, etc) ***
+
+* rpmbuild -ta 
+
+for example:
+
+rpmbuild -ta libs3-0.3.tar.gz
+
+
+*** For dpkg-based systems (Debian, Ubuntu, etc) ***
+
+* make deb
+
+This will produce a Debian package in the build/pkg directory.
+
+
+*** For all other systems ***
+
+* make [DESTDIR=destination root] install
+
+DESTDIR defaults to /usr
+
+
+To install libs3 on a Microsoft Windows system:
+-----------------------------------------------
+
+*** Using MingW ***
+
+* libs3 can be built on Windows using the MingW compiler.  No other tool
+  is needed.  However, the following libraries are needed to build libs3:
+
+  - curl development libraries
+  - libxml2 development libraries, and the libraries that it requires:
+    - iconv
+    - zlib
+
+  These projects are independent of libs3, and their release schedule and
+  means of distribution would make it very difficult to provide links to
+  the files to download and keep them up-to-date in this file, so no attempt
+  is made here.
+
+  Development libraries and other files can be placed in:
+  c:\libs3-libs\bin
+  c:\libs3-libs\include
+
+  If the above locations are used, then the GNUmakefile.mingw will work with
+  no special caveats.  If the above locations are not used, then the following
+  environment variables should be set:
+  CURL_LIBS should be set to the MingW compiler flags needed to locate and
+      link in the curl libraries
+  CURL_CFLAGS should be set to the MingW compiler flags needed to locate and
+      include the curl headers
+  LIBXML2_LIBS should be set to the MingW compiler flags needed to locate and
+      link in the libxml2 libraries
+  LIBXML2_CFLAGS should be set to the MingW compiler flags needed to locate and
+      include the libxml2 headers
+
+* mingw32-make [DESTDIR=destination] -f GNUmakefile.mingw install
+
+DESTDIR defaults to libs3-
+
+* DESTDIR can be zipped up into a .zip file for distribution.  For best
+  results, the dependent libraries (curl, openssl, etc) should be included,
+  along with their licenses.
diff --git a/ceph/src/libs3/LICENSE b/ceph/src/libs3/LICENSE
new file mode 100644
index 00000000..db90987b
--- /dev/null
+++ b/ceph/src/libs3/LICENSE
@@ -0,0 +1,20 @@
+Copyright 2008 Bryan Ischo 
+
+libs3 is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, version 3 of the License.
+
+In addition, as a special exception, the copyright holders give
+permission to link the code of this library and its programs with the
+OpenSSL library, and distribute linked combinations including the two.
+
+libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+details.
+
+You should have received a copy of the GNU General Public License version 3
+along with libs3, in a file named COPYING.  If not, see
+.
+
+
diff --git a/ceph/src/libs3/README b/ceph/src/libs3/README
new file mode 100644
index 00000000..c881a778
--- /dev/null
+++ b/ceph/src/libs3/README
@@ -0,0 +1,4 @@
+This directory contains the libs3 library.
+
+The libs3 library is free software.  See the file LICENSE for copying
+permission.
diff --git a/ceph/src/libs3/TODO b/ceph/src/libs3/TODO
new file mode 100644
index 00000000..d8821f45
--- /dev/null
+++ b/ceph/src/libs3/TODO
@@ -0,0 +1,3 @@
+* Implement functions for generating form stuff for posting to s3
+
+* Write s3 man page
diff --git a/ceph/src/libs3/archlinux/PKGBUILD b/ceph/src/libs3/archlinux/PKGBUILD
new file mode 100644
index 00000000..9256deea
--- /dev/null
+++ b/ceph/src/libs3/archlinux/PKGBUILD
@@ -0,0 +1,28 @@
+# Contributor: Bryan Ischo 
+pkgname=libs3
+pkgver=trunk
+pkgrel=1
+pkgdesc="C Library and Tools for Amazon S3 Access"
+arch=('i686' 'x86_64')
+url="http://libs3.ischo.com/index.html"
+license=('GPL')
+groups=()
+depends=('libxml2' 'openssl' 'curl')
+makedepends=('make' 'libxml2' 'openssl' 'curl')
+provides=()
+conflicts=()
+replaces=()
+backup=()
+options=()
+install=
+source=(http://libs3.ischo.com/$pkgname-$pkgver.tar.gz)
+noextract=()
+md5sums=('source md5') #generate with 'makepkg -g'
+
+build() {
+  cd "$srcdir/$pkgname-$pkgver"
+
+  DESTDIR=$pkgdir/usr make install || return 1
+}
+
+# vim:set ts=2 sw=2 et:
diff --git a/ceph/src/libs3/debian/changelog b/ceph/src/libs3/debian/changelog
new file mode 100644
index 00000000..520b2b9f
--- /dev/null
+++ b/ceph/src/libs3/debian/changelog
@@ -0,0 +1,5 @@
+libs3 (all) unstable; urgency=low
+
+  * This file is not maintained.  See project source code for changes.
+
+ -- Bryan Ischo   Wed, 06 Aug 2008 09:36:43 -0400
diff --git a/ceph/src/libs3/debian/changelog.Debian b/ceph/src/libs3/debian/changelog.Debian
new file mode 100644
index 00000000..81072be8
--- /dev/null
+++ b/ceph/src/libs3/debian/changelog.Debian
@@ -0,0 +1,6 @@
+libs3 (all) unstable; urgency=low
+
+      * libs3 Debian maintainer and upstream author are identical.
+        Therefore see normal changelog file for Debian changes.
+
+ -- Bryan Ischo   Wed, 06 Aug 2008 09:36:43 -0400
diff --git a/ceph/src/libs3/debian/control b/ceph/src/libs3/debian/control
new file mode 100644
index 00000000..28ddc6b7
--- /dev/null
+++ b/ceph/src/libs3/debian/control
@@ -0,0 +1,12 @@
+Package: libs3
+Source: THIS LINE WILL BE REMOVED, dpkg-shlibdepends NEEDS IT
+Version: LIBS3_VERSION
+Architecture: DEBIAN_ARCHITECTURE
+Section: net
+Priority: extra
+Maintainer: Bryan Ischo 
+Homepage: http://libs3.ischo.com/index.html
+Description: C Library and Tools for Amazon S3 Access
+ This package includes the libs3 shared object library, needed to run
+ applications compiled against libs3, and additionally contains the s3
+ utility for accessing Amazon S3.
diff --git a/ceph/src/libs3/debian/control.dev b/ceph/src/libs3/debian/control.dev
new file mode 100644
index 00000000..5ee5ae78
--- /dev/null
+++ b/ceph/src/libs3/debian/control.dev
@@ -0,0 +1,26 @@
+Package: libs3-dev
+Version: LIBS3_VERSION
+Architecture: DEBIAN_ARCHITECTURE
+Section: libdevel
+Priority: extra
+Depends: libs3 (>= LIBS3_VERSION)
+Maintainer: Bryan Ischo 
+Homepage: http://libs3.ischo.com/index.html
+Description: C Development Library for Amazon S3 Access
+ This library provides an API for using Amazon's S3 service (see
+ http://s3.amazonaws.com).  Its design goals are:
+ .
+  - To provide a simple and straightforward API for accessing all of S3's
+    functionality
+  - To not require the developer using libs3 to need to know anything about:
+      - HTTP
+      - XML
+      - SSL
+    In other words, this API is meant to stand on its own, without requiring
+    any implicit knowledge of how S3 services are accessed using HTTP
+    protocols.
+  - To be usable from multithreaded code
+  - To be usable by code which wants to process multiple S3 requests
+    simultaneously from a single thread
+  - To be usable in the simple, straightforward way using sequentialized
+    blocking requests
diff --git a/ceph/src/libs3/debian/postinst b/ceph/src/libs3/debian/postinst
new file mode 100755
index 00000000..2d1871b1
--- /dev/null
+++ b/ceph/src/libs3/debian/postinst
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+ldconfig
diff --git a/ceph/src/libs3/doxyfile b/ceph/src/libs3/doxyfile
new file mode 100644
index 00000000..0c7aedce
--- /dev/null
+++ b/ceph/src/libs3/doxyfile
@@ -0,0 +1,886 @@
+# Doxyfile 1.2.14
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project
+#
+# All text after a hash (#) is considered a comment and will be ignored
+# The format is:
+#       TAG = value [value, ...]
+# For lists items can also be appended using:
+#       TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (" ")
+
+#---------------------------------------------------------------------------
+# General configuration options
+#---------------------------------------------------------------------------
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded 
+# by quotes) that should identify the project.
+
+PROJECT_NAME           = libs3
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. 
+# This could be handy for archiving the generated documentation or 
+# if some version control system is used.
+
+PROJECT_NUMBER         = trunk
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) 
+# base path where the generated documentation will be put. 
+# If a relative path is entered, it will be relative to the location 
+# where doxygen was started. If left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       = dox
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all 
+# documentation generated by doxygen is written. Doxygen will use this 
+# information to generate all constant output in the proper language. 
+# The default language is English, other supported languages are: 
+# Brazilian, Chinese, Croatian, Czech, Danish, Dutch, Finnish, French, 
+# German, Greek, Hungarian, Italian, Japanese, Korean, Norwegian, Polish, 
+# Portuguese, Romanian, Russian, Slovak, Slovene, Spanish and Swedish.
+
+OUTPUT_LANGUAGE        = English
+
+# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in 
+# documentation are documented, even if no documentation was available. 
+# Private class members and static file members will be hidden unless 
+# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
+
+EXTRACT_ALL            = YES
+
+# If the EXTRACT_PRIVATE tag is set to YES all private members of a class 
+# will be included in the documentation.
+
+EXTRACT_PRIVATE        = YES
+
+# If the EXTRACT_STATIC tag is set to YES all static members of a file 
+# will be included in the documentation.
+
+EXTRACT_STATIC         = YES
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) 
+# defined locally in source files will be included in the documentation. 
+# If set to NO only classes defined in header files are included.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all 
+# undocumented members of documented classes, files or namespaces. 
+# If set to NO (the default) these members will be included in the 
+# various overviews, but no documentation section is generated. 
+# This option has no effect if EXTRACT_ALL is enabled.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all 
+# undocumented classes that are normally visible in the class hierarchy. 
+# If set to NO (the default) these class will be included in the various 
+# overviews. This option has no effect if EXTRACT_ALL is enabled.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will 
+# include brief member descriptions after the members that are listed in 
+# the file and class documentation (similar to JavaDoc). 
+# Set to NO to disable this.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend 
+# the brief description of a member or function before the detailed description. 
+# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the 
+# brief descriptions will be completely suppressed.
+
+REPEAT_BRIEF           = YES
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then 
+# Doxygen will generate a detailed section even if there is only a brief 
+# description.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited 
+# members of a class in the documentation of that class as if those members were 
+# ordinary class members. Constructors, destructors and assignment operators of 
+# the base classes will not be shown.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full 
+# path before files name in the file list and in the header files. If set 
+# to NO the shortest path that makes the file name unique will be used.
+
+FULL_PATH_NAMES        = NO
+
+# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag 
+# can be used to strip a user defined part of the path. Stripping is 
+# only done if one of the specified strings matches the left-hand part of 
+# the path. It is allowed to use relative paths in the argument list.
+
+STRIP_FROM_PATH        = 
+
+# The INTERNAL_DOCS tag determines if documentation 
+# that is typed after a \internal command is included. If the tag is set 
+# to NO (the default) then the documentation will be excluded. 
+# Set it to YES to include the internal documentation.
+
+INTERNAL_DOCS          = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct 
+# doxygen to hide any special comment blocks from generated source code 
+# fragments. Normal C and C++ comments will always remain visible.
+
+STRIP_CODE_COMMENTS    = YES
+
+# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate 
+# file names in lower case letters. If set to YES upper case letters are also 
+# allowed. This is useful if you have classes or files whose names only differ 
+# in case and if your file system supports case sensitive file names. Windows 
+# users are adviced to set this option to NO.
+
+CASE_SENSE_NAMES       = YES
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter 
+# (but less readable) file names. This can be useful is your file systems 
+# doesn't support long names like on DOS, Mac, or CD-ROM.
+
+SHORT_NAMES            = NO
+
+# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen 
+# will show members with their full class and namespace scopes in the 
+# documentation. If set to YES the scope will be hidden.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen 
+# will generate a verbatim copy of the header file for each class for 
+# which an include is specified. Set to NO to disable this.
+
+VERBATIM_HEADERS       = YES
+
+# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen 
+# will put list of the files that are included by a file in the documentation 
+# of that file.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen 
+# will interpret the first line (until the first dot) of a JavaDoc-style 
+# comment as the brief description. If set to NO, the JavaDoc 
+# comments  will behave just like the Qt-style comments (thus requiring an 
+# explict @brief command for a brief description.
+
+JAVADOC_AUTOBRIEF      = NO
+
+# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented 
+# member inherits the documentation from any documented member that it 
+# reimplements.
+
+INHERIT_DOCS           = YES
+
+# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] 
+# is inserted in the documentation for inline members.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen 
+# will sort the (detailed) documentation of file and class members 
+# alphabetically by member name. If set to NO the members will appear in 
+# declaration order.
+
+SORT_MEMBER_DOCS       = NO
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC 
+# tag is set to YES, then doxygen will reuse the documentation of the first 
+# member in the group (if any) for the other members of the group. By default 
+# all members of a group must be documented explicitly.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. 
+# Doxygen uses this value to replace tabs by spaces in code fragments.
+
+TAB_SIZE               = 8
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or 
+# disable (NO) the todo list. This list is created by putting \todo 
+# commands in the documentation.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or 
+# disable (NO) the test list. This list is created by putting \test 
+# commands in the documentation.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or 
+# disable (NO) the bug list. This list is created by putting \bug 
+# commands in the documentation.
+
+GENERATE_BUGLIST       = YES
+
+# This tag can be used to specify a number of aliases that acts 
+# as commands in the documentation. An alias has the form "name=value". 
+# For example adding "sideeffect=\par Side Effects:\n" will allow you to 
+# put the command \sideeffect (or @sideeffect) in the documentation, which 
+# will result in a user defined paragraph with heading "Side Effects:". 
+# You can put \n's in the value part of an alias to insert newlines.
+
+ALIASES                = 
+
+# The ENABLED_SECTIONS tag can be used to enable conditional 
+# documentation sections, marked by \if sectionname ... \endif.
+
+ENABLED_SECTIONS       = 
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines 
+# the initial value of a variable or define consist of for it to appear in 
+# the documentation. If the initializer consists of more lines than specified 
+# here it will be hidden. Use a value of 0 to hide initializers completely. 
+# The appearance of the initializer of individual variables and defines in the 
+# documentation can be controlled using \showinitializer or \hideinitializer 
+# command in the documentation regardless of this setting.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources 
+# only. Doxygen will then generate output that is more tailored for C. 
+# For instance some of the names that are used will be different. The list 
+# of all members will be omitted, etc.
+
+OPTIMIZE_OUTPUT_FOR_C  = NO
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated 
+# at the bottom of the documentation of classes and structs. If set to YES the 
+# list will mention the files that were used to generate the documentation.
+
+SHOW_USED_FILES        = YES
+
+#---------------------------------------------------------------------------
+# configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated 
+# by doxygen. Possible values are YES and NO. If left blank NO is used.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are 
+# generated by doxygen. Possible values are YES and NO. If left blank 
+# NO is used.
+
+WARNINGS               = YES
+
+# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings 
+# for undocumented members. If EXTRACT_ALL is set to YES then this flag will 
+# automatically be disabled.
+
+WARN_IF_UNDOCUMENTED   = YES
+
+# The WARN_FORMAT tag determines the format of the warning messages that 
+# doxygen can produce. The string should contain the $file, $line, and $text 
+# tags, which will be replaced by the file and line number from which the 
+# warning originated and the warning text.
+
+WARN_FORMAT            = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning 
+# and error messages should be written. If left blank the output is written 
+# to stderr.
+
+WARN_LOGFILE           = 
+
+#---------------------------------------------------------------------------
+# configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag can be used to specify the files and/or directories that contain 
+# documented source files. You may enter file names like "myfile.cpp" or 
+# directories like "/usr/src/myproject". Separate the files or directories 
+# with spaces.
+
+INPUT                  = inc/libs3.h
+
+# If the value of the INPUT tag contains directories, you can use the 
+# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
+# and *.h) to filter out the source-files in the directories. If left 
+# blank the following patterns are tested: 
+# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp 
+# *.h++ *.idl *.odl
+
+FILE_PATTERNS          =
+
+# The RECURSIVE tag can be used to turn specify whether or not subdirectories 
+# should be searched for input files as well. Possible values are YES and NO. 
+# If left blank NO is used.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should 
+# excluded from the INPUT source files. This way you can easily exclude a 
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories 
+# that are symbolic links (a Unix filesystem feature) are excluded from the input.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the 
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude 
+# certain files from those directories.
+
+EXCLUDE_PATTERNS       = 
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or 
+# directories that contain example code fragments that are included (see 
+# the \include command).
+
+EXAMPLE_PATH           = 
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the 
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
+# and *.h) to filter out the source-files in the directories. If left 
+# blank all files are included.
+
+EXAMPLE_PATTERNS       = 
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be 
+# searched for input files to be used with the \include or \dontinclude 
+# commands irrespective of the value of the RECURSIVE tag. 
+# Possible values are YES and NO. If left blank NO is used.
+
+EXAMPLE_RECURSIVE      = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or 
+# directories that contain image that are included in the documentation (see 
+# the \image command).
+
+IMAGE_PATH             = 
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should 
+# invoke to filter for each input file. Doxygen will invoke the filter program 
+# by executing (via popen()) the command  , where  
+# is the value of the INPUT_FILTER tag, and  is the name of an 
+# input file. Doxygen will then use the output that the filter program writes 
+# to standard output.
+
+INPUT_FILTER           = 
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using 
+# INPUT_FILTER) will be used to filter the input files when producing source 
+# files to browse.
+
+FILTER_SOURCE_FILES    = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will 
+# be generated. Documented entities will be cross-referenced with these sources.
+
+SOURCE_BROWSER         = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body 
+# of functions and classes directly in the documentation.
+
+INLINE_SOURCES         = NO
+
+# If the REFERENCED_BY_RELATION tag is set to YES (the default) 
+# then for each documented function all documented 
+# functions referencing it will be listed.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES (the default) 
+# then for each documented function all documented entities 
+# called/used by that function will be listed.
+
+REFERENCES_RELATION    = YES
+
+#---------------------------------------------------------------------------
+# configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index 
+# of all compounds will be generated. Enable this if the project 
+# contains a lot of classes, structs, unions or interfaces.
+
+ALPHABETICAL_INDEX     = NO
+
+# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then 
+# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns 
+# in which this list will be split (can be a number in the range [1..20])
+
+COLS_IN_ALPHA_INDEX    = 5
+
+# In case all classes in a project start with a common prefix, all 
+# classes will be put under the same header in the alphabetical index. 
+# The IGNORE_PREFIX tag can be used to specify one or more prefixes that 
+# should be ignored while generating the index headers.
+
+IGNORE_PREFIX          = 
+
+#---------------------------------------------------------------------------
+# configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES (the default) Doxygen will 
+# generate HTML output.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `html' will be used as the default path.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for 
+# each generated HTML page (for example: .htm,.php,.asp). If it is left blank 
+# doxygen will generate files with .html extension.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a personal HTML header for 
+# each generated HTML page. If it is left blank doxygen will generate a 
+# standard header.
+
+HTML_HEADER            = 
+
+# The HTML_FOOTER tag can be used to specify a personal HTML footer for 
+# each generated HTML page. If it is left blank doxygen will generate a 
+# standard footer.
+
+HTML_FOOTER            = 
+
+# The HTML_STYLESHEET tag can be used to specify a user defined cascading 
+# style sheet that is used by each HTML page. It can be used to 
+# fine-tune the look of the HTML output. If the tag is left blank doxygen 
+# will generate a default style sheet
+
+HTML_STYLESHEET        = 
+
+# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, 
+# files or namespaces will be aligned in HTML using tables. If set to 
+# NO a bullet list will be used.
+
+HTML_ALIGN_MEMBERS     = YES
+
+# If the GENERATE_HTMLHELP tag is set to YES, additional index files 
+# will be generated that can be used as input for tools like the 
+# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) 
+# of the generated HTML documentation.
+
+GENERATE_HTMLHELP      = NO
+
+# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag 
+# controls if a separate .chi index file is generated (YES) or that 
+# it should be included in the master .chm file (NO).
+
+GENERATE_CHI           = NO
+
+# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag 
+# controls whether a binary table of contents is generated (YES) or a 
+# normal table of contents (NO) in the .chm file.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members 
+# to the contents of the Html help documentation and to the tree view.
+
+TOC_EXPAND             = NO
+
+# The DISABLE_INDEX tag can be used to turn on/off the condensed index at 
+# top of each HTML page. The value NO (the default) enables the index and 
+# the value YES disables it.
+
+DISABLE_INDEX          = NO
+
+# This tag can be used to set the number of enum values (range [1..20]) 
+# that doxygen will group on one line in the generated HTML documentation.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be
+# generated containing a tree-like index structure (just like the one that 
+# is generated for HTML Help). For this to work a browser that supports 
+# JavaScript and frames is required (for instance Mozilla, Netscape 4.0+, 
+# or Internet explorer 4.0+). Note that for large projects the tree generation 
+# can take a very long time. In such cases it is better to disable this feature. 
+# Windows users are probably better off using the HTML help feature.
+
+GENERATE_TREEVIEW      = YES
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be 
+# used to set the initial width (in pixels) of the frame in which the tree 
+# is shown.
+
+TREEVIEW_WIDTH         = 250
+
+#---------------------------------------------------------------------------
+# configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will 
+# generate Latex output.
+
+GENERATE_LATEX         = NO
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `latex' will be used as the default path.
+
+LATEX_OUTPUT           = latex
+
+# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact 
+# LaTeX documents. This may be useful for small projects and may help to 
+# save some trees in general.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used 
+# by the printer. Possible values are: a4, a4wide, letter, legal and 
+# executive. If left blank a4wide will be used.
+
+PAPER_TYPE             = a4wide
+
+# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX 
+# packages that should be included in the LaTeX output.
+
+EXTRA_PACKAGES         = 
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for 
+# the generated latex document. The header should contain everything until 
+# the first chapter. If it is left blank doxygen will generate a 
+# standard header. Notice: only use this tag if you know what you are doing!
+
+LATEX_HEADER           = 
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated 
+# is prepared for conversion to pdf (using ps2pdf). The pdf file will 
+# contain links (just like the HTML output) instead of page references 
+# This makes the output suitable for online browsing using a pdf viewer.
+
+PDF_HYPERLINKS         = NO
+
+# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of 
+# plain latex in the generated Makefile. Set this option to YES to get a 
+# higher quality PDF documentation.
+
+USE_PDFLATEX           = NO
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. 
+# command to the generated LaTeX files. This will instruct LaTeX to keep 
+# running if errors occur, instead of asking the user for help. 
+# This option is also used when generating formulas in HTML.
+
+LATEX_BATCHMODE        = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output 
+# The RTF output is optimised for Word 97 and may not look very pretty with 
+# other RTF readers or editors.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `rtf' will be used as the default path.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES Doxygen generates more compact 
+# RTF documents. This may be useful for small projects and may help to 
+# save some trees in general.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated 
+# will contain hyperlink fields. The RTF file will 
+# contain links (just like the HTML output) instead of page references. 
+# This makes the output suitable for online browsing using WORD or other 
+# programs which support those fields. 
+# Note: wordpad (write) and others do not support links.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's 
+# config file, i.e. a series of assigments. You only have to provide 
+# replacements, missing definitions are set to their default value.
+
+RTF_STYLESHEET_FILE    = 
+
+# Set optional variables used in the generation of an rtf document. 
+# Syntax is similar to doxygen's config file.
+
+RTF_EXTENSIONS_FILE    = 
+
+#---------------------------------------------------------------------------
+# configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES (the default) Doxygen will 
+# generate man pages
+
+GENERATE_MAN           = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `man' will be used as the default path.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to 
+# the generated man pages (default is the subroutine's section .3)
+
+MAN_EXTENSION          = .3
+
+# If the MAN_LINKS tag is set to YES and Doxygen generates man output, 
+# then it will generate one additional man file for each entity 
+# documented in the real man page(s). These additional files 
+# only source the real man page, but without them the man command 
+# would be unable to find the correct page. The default is NO.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES Doxygen will 
+# generate an XML file that captures the structure of 
+# the code including all documentation. Note that this 
+# feature is still experimental and incomplete at the 
+# moment.
+
+GENERATE_XML           = NO
+
+#---------------------------------------------------------------------------
+# configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will 
+# generate an AutoGen Definitions (see autogen.sf.net) file 
+# that captures the structure of the code including all 
+# documentation. Note that this feature is still experimental 
+# and incomplete at the moment.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor   
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will 
+# evaluate all C-preprocessor directives found in the sources and include 
+# files.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro 
+# names in the source code. If set to NO (the default) only conditional 
+# compilation will be performed. Macro expansion can be done in a controlled 
+# way by setting EXPAND_ONLY_PREDEF to YES.
+
+MACRO_EXPANSION        = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES 
+# then the macro expansion is limited to the macros specified with the 
+# PREDEFINED and EXPAND_AS_PREDEFINED tags.
+
+EXPAND_ONLY_PREDEF     = NO
+
+# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files 
+# in the INCLUDE_PATH (see below) will be search if a #include is found.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that 
+# contain include files that are not input files but should be processed by 
+# the preprocessor.
+
+INCLUDE_PATH           = 
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard 
+# patterns (like *.h and *.hpp) to filter out the header-files in the 
+# directories. If left blank, the patterns specified with FILE_PATTERNS will 
+# be used.
+
+INCLUDE_FILE_PATTERNS  = 
+
+# The PREDEFINED tag can be used to specify one or more macro names that 
+# are defined before the preprocessor is started (similar to the -D option of 
+# gcc). The argument of the tag is a list of macros of the form: name 
+# or name=definition (no spaces). If the definition and the = are 
+# omitted =1 is assumed.
+
+PREDEFINED             =  DOXYGEN
+
+# If the MACRO_EXPANSION and EXPAND_PREDEF_ONLY tags are set to YES then 
+# this tag can be used to specify a list of macro names that should be expanded. 
+# The macro definition that is found in the sources will be used. 
+# Use the PREDEFINED tag if you want to use a different macro definition.
+
+EXPAND_AS_DEFINED      = 
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then 
+# doxygen's preprocessor will remove all function-like macros that are alone 
+# on a line and do not end with a semicolon. Such function macros are typically 
+# used for boiler-plate code, and will confuse the parser if not removed.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration::addtions related to external references   
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tagfiles.
+
+TAGFILES               = 
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create 
+# a tag file that is based on the input files it reads.
+
+GENERATE_TAGFILE       = 
+
+# If the ALLEXTERNALS tag is set to YES all external classes will be listed 
+# in the class index. If set to NO only the inherited external classes 
+# will be listed.
+
+ALLEXTERNALS           = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed 
+# in the modules index. If set to NO, only the current project's groups will 
+# be listed.
+
+EXTERNAL_GROUPS        = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script 
+# interpreter (i.e. the result of `which perl').
+
+PERL_PATH              = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool   
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will 
+# generate a inheritance diagram (in Html, RTF and LaTeX) for classes with base or 
+# super classes. Setting the tag to NO turns the diagrams off. Note that this 
+# option is superceded by the HAVE_DOT option below. This is only a fallback. It is 
+# recommended to install and use dot, since it yield more powerful graphs.
+
+CLASS_DIAGRAMS         = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is 
+# available from the path. This tool is part of Graphviz, a graph visualization 
+# toolkit from AT&T and Lucent Bell Labs. The other options in this section 
+# have no effect if this option is set to NO (the default)
+
+HAVE_DOT               = NO
+
+# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen 
+# will generate a graph for each documented class showing the direct and 
+# indirect inheritance relations. Setting this tag to YES will force the 
+# the CLASS_DIAGRAMS tag to NO.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen 
+# will generate a graph for each documented class showing the direct and 
+# indirect implementation dependencies (inheritance, containment, and 
+# class references variables) of the class with other documented classes.
+
+COLLABORATION_GRAPH    = YES
+
+# If set to YES, the inheritance and collaboration graphs will show the 
+# relations between templates and their instances.
+
+TEMPLATE_RELATIONS     = YES
+
+# If set to YES, the inheritance and collaboration graphs will hide 
+# inheritance and usage relations if the target is undocumented 
+# or is not a class.
+
+HIDE_UNDOC_RELATIONS   = YES
+
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT 
+# tags are set to YES then doxygen will generate a graph for each documented 
+# file showing the direct and indirect include dependencies of the file with 
+# other documented files.
+
+INCLUDE_GRAPH          = YES
+
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and 
+# HAVE_DOT tags are set to YES then doxygen will generate a graph for each 
+# documented header file showing the documented files that directly or 
+# indirectly include this file.
+
+INCLUDED_BY_GRAPH      = YES
+
+# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen 
+# will graphical hierarchy of all classes instead of a textual one.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images 
+# generated by dot. Possible values are gif, jpg, and png
+# If left blank gif will be used.
+
+DOT_IMAGE_FORMAT       = gif
+
+# The tag DOT_PATH can be used to specify the path where the dot tool can be 
+# found. If left blank, it is assumed the dot tool can be found on the path.
+
+DOT_PATH               = 
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that 
+# contain dot files that are included in the documentation (see the 
+# \dotfile command).
+
+DOTFILE_DIRS           = 
+
+# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width 
+# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
+# this value, doxygen will try to truncate the graph, so that it fits within 
+# the specified constraint. Beware that most browsers cannot cope with very 
+# large images.
+
+MAX_DOT_GRAPH_WIDTH    = 1024
+
+# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height 
+# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
+# this value, doxygen will try to truncate the graph, so that it fits within 
+# the specified constraint. Beware that most browsers cannot cope with very 
+# large images.
+
+MAX_DOT_GRAPH_HEIGHT   = 1024
+
+# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will 
+# generate a legend page explaining the meaning of the various boxes and 
+# arrows in the dot generated graphs.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will 
+# remove the intermedate dot files that are used to generate 
+# the various graphs.
+
+DOT_CLEANUP            = YES
+
+#---------------------------------------------------------------------------
+# Configuration::addtions related to the search engine   
+#---------------------------------------------------------------------------
+
+# The SEARCHENGINE tag specifies whether or not a search engine should be 
+# used. If set to NO the values of all tags below this one will be ignored.
+
+SEARCHENGINE           = NO
diff --git a/ceph/src/libs3/inc/error_parser.h b/ceph/src/libs3/inc/error_parser.h
new file mode 100644
index 00000000..87852018
--- /dev/null
+++ b/ceph/src/libs3/inc/error_parser.h
@@ -0,0 +1,82 @@
+/** **************************************************************************
+ * error_parser.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef ERROR_PARSER_H
+#define ERROR_PARSER_H
+
+#include "libs3.h"
+#include "simplexml.h"
+#include "string_buffer.h"
+
+
+#define EXTRA_DETAILS_SIZE 8
+
+typedef struct ErrorParser
+{
+    // This is the S3ErrorDetails that this ErrorParser fills in from the
+    // data that it parses
+    S3ErrorDetails s3ErrorDetails;
+
+    // This is the error XML parser
+    SimpleXml errorXmlParser;
+
+    // Set to 1 after the first call to add
+    int errorXmlParserInitialized;
+
+    // Used to buffer the S3 Error Code as it is read in
+    string_buffer(code, 1024);
+
+    // Used to buffer the S3 Error Message as it is read in
+    string_buffer(message, 1024);
+
+    // Used to buffer the S3 Error Resource as it is read in
+    string_buffer(resource, 1024);
+
+    // Used to buffer the S3 Error Further Details as it is read in
+    string_buffer(furtherDetails, 1024);
+    
+    // The extra details; we support up to EXTRA_DETAILS_SIZE of them
+    S3NameValue extraDetails[EXTRA_DETAILS_SIZE];
+
+    // This is the buffer from which the names and values used in extraDetails
+    // are allocated
+    string_multibuffer(extraDetailsNamesValues, EXTRA_DETAILS_SIZE * 1024);
+} ErrorParser;
+
+
+// Always call this
+void error_parser_initialize(ErrorParser *errorParser);
+
+S3Status error_parser_add(ErrorParser *errorParser, char *buffer,
+                          int bufferSize);
+
+void error_parser_convert_status(ErrorParser *errorParser, S3Status *status);
+
+// Always call this
+void error_parser_deinitialize(ErrorParser *errorParser);
+
+
+#endif /* ERROR_PARSER_H */
diff --git a/ceph/src/libs3/inc/libs3.h b/ceph/src/libs3/inc/libs3.h
new file mode 100644
index 00000000..baa200c6
--- /dev/null
+++ b/ceph/src/libs3/inc/libs3.h
@@ -0,0 +1,1892 @@
+/** **************************************************************************
+ * libs3.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef LIBS3_H
+#define LIBS3_H
+
+#include 
+#include 
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/** **************************************************************************
+ * Overview
+ * --------
+ *
+ * This library provides an API for using Amazon's S3 service (see
+ * http://s3.amazonaws.com).  Its design goals are:
+ *
+ * - To provide a simple and straightforward API for accessing all of S3's
+ *   functionality
+ * - To not require the developer using libs3 to need to know anything about:
+ *     - HTTP
+ *     - XML
+ *     - SSL
+ *   In other words, this API is meant to stand on its own, without requiring
+ *   any implicit knowledge of how S3 services are accessed using HTTP
+ *   protocols.
+ * - To be usable from multithreaded code
+ * - To be usable by code which wants to process multiple S3 requests
+ *   simultaneously from a single thread
+ * - To be usable in the simple, straightforward way using sequentialized
+ *   blocking requests
+ *
+ * The general usage pattern of libs3 is:
+ *
+ * - Initialize libs3 once per program by calling S3_initialize() at program
+ *   start up time
+ * - Make any number of requests to S3 for getting, putting, or listing
+ *   S3 buckets or objects, or modifying the ACLs associated with buckets
+ *   or objects, using one of three general approaches:
+ *   1. Simple blocking requests, one at a time
+ *   2. Multiple threads each making simple blocking requests
+ *   3. From a single thread, managing multiple S3 requests simultaneously
+ *      using file descriptors and a select()/poll() loop
+ * - Shut down libs3 at program exit time by calling S3_deinitialize()
+ *
+ * All functions which send requests to S3 return their results via a set of
+ * callback functions which must be supplied to libs3 at the time that the
+ * request is initiated.  libs3 will call these functions back in the thread
+ * calling the libs3 function if blocking requests are made (i.e., if the
+ * S3RequestContext for the function invocation is passed in as NULL).
+ * If an S3RequestContext is used to drive multiple S3 requests
+ * simultaneously, then the callbacks will be made from the thread which
+ * calls S3_runall_request_context() or S3_runonce_request_context(), or
+ * possibly from the thread which calls S3_destroy_request_context(), if
+ * S3 requests are in progress at the time that this function is called.
+ *
+ * NOTE: Response headers from Amazon S3 are limited to 4K (2K of metas is all
+ * that Amazon supports, and libs3 allows Amazon an additional 2K of headers).
+ *
+ * NOTE: Because HTTP and the S3 REST protocol are highly under-specified,
+ * libs3 must make some assumptions about the maximum length of certain HTTP
+ * elements (such as headers) that it will accept.  While efforts have been
+ * made to enforce maximums which are beyond that expected to be needed by any
+ * user of S3, it is always possible that these maximums may be too low in
+ * some rare circumstances.  Bug reports should this unlikely situation occur
+ * would be most appreciated.
+ * 
+ * Threading Rules
+ * ---------------
+ * 
+ * 1. All arguments passed to any function must not be modified directly until
+ *    the function returns.
+ * 2. All S3RequestContext and S3Request arguments passed to all functions may
+ *    not be passed to any other libs3 function by any other thread until the
+ *    function returns.
+ * 3. All functions may be called simultaneously by multiple threads as long
+ *    as (1) and (2) are observed, EXCEPT for S3_initialize(), which must be
+ *    called from one thread at a time only.
+ * 4. All callbacks will be made in the thread of the caller of the function
+ *    which invoked them, so the caller of all libs3 functions should not hold
+ *    locks that it would try to re-acquire in a callback, as this may
+ *    deadlock.
+ ************************************************************************** **/
+
+
+/** **************************************************************************
+ * Constants
+ ************************************************************************** **/
+
+/**
+ * S3_MAX_HOSTNAME_SIZE is the maximum size we allow for a host name
+ **/
+#define S3_MAX_HOSTNAME_SIZE               255
+
+/**
+ * This is the default hostname that is being used for the S3 requests
+ **/
+#define S3_DEFAULT_HOSTNAME                "s3.amazonaws.com"
+
+
+/**
+ * S3_MAX_BUCKET_NAME_SIZE is the maximum size of a bucket name.
+ **/
+
+#define S3_MAX_BUCKET_NAME_SIZE            255
+
+/**
+ * S3_MAX_KEY_SIZE is the maximum size of keys that Amazon S3 supports.
+ **/
+#define S3_MAX_KEY_SIZE                    1024
+
+
+/**
+ * S3_MAX_METADATA_SIZE is the maximum number of bytes allowed for
+ * x-amz-meta header names and values in any request passed to Amazon S3
+ **/
+#define S3_MAX_METADATA_SIZE               2048
+
+
+/**
+ * S3_METADATA_HEADER_NAME_PREFIX is the prefix of an S3 "meta header"
+ **/
+#define S3_METADATA_HEADER_NAME_PREFIX     "x-amz-meta-"
+
+
+/**
+ * S3_MAX_METADATA_COUNT is the maximum number of x-amz-meta- headers that
+ * could be included in a request to S3.  The smallest meta header is
+ * "x-amz-meta-n: v".  Since S3 doesn't count the ": " against the total, the
+ * smallest amount of data to count for a header would be the length of
+ * "x-amz-meta-nv".
+ **/
+#define S3_MAX_METADATA_COUNT \
+    (S3_MAX_METADATA_SIZE / (sizeof(S3_METADATA_HEADER_NAME_PREFIX "nv") - 1))
+
+
+/**
+ * S3_MAX_ACL_GRANT_COUNT is the maximum number of ACL grants that may be
+ * set on a bucket or object at one time.  It is also the maximum number of
+ * ACL grants that the XML ACL parsing routine will parse.
+ **/
+#define S3_MAX_ACL_GRANT_COUNT             100
+
+
+/**
+ * This is the maximum number of characters (including terminating \0) that
+ * libs3 supports in an ACL grantee email address.
+ **/
+#define S3_MAX_GRANTEE_EMAIL_ADDRESS_SIZE  128
+
+
+/**
+ * This is the maximum number of characters (including terminating \0) that
+ * libs3 supports in an ACL grantee user id.
+ **/
+#define S3_MAX_GRANTEE_USER_ID_SIZE        128
+
+
+/**
+ * This is the maximum number of characters (including terminating \0) that
+ * libs3 supports in an ACL grantee user display name.
+ **/
+#define S3_MAX_GRANTEE_DISPLAY_NAME_SIZE   128
+
+
+/**
+ * This is the maximum number of characters that will be stored in the
+ * return buffer for the utility function which computes an HTTP authenticated
+ * query string
+ **/
+#define S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE \
+    (sizeof("https:///") + S3_MAX_HOSTNAME_SIZE + (S3_MAX_KEY_SIZE * 3) + \
+     sizeof("?AWSAccessKeyId=") + 32 + sizeof("&Expires=") + 32 + \
+     sizeof("&Signature=") + 28 + 1)
+
+
+/**
+ * This constant is used by the S3_initialize() function, to specify that
+ * the winsock library should be initialized by libs3; only relevent on 
+ * Microsoft Windows platforms.
+ **/
+#define S3_INIT_WINSOCK                    1
+
+
+/**
+ * This convenience constant is used by the S3_initialize() function to
+ * indicate that all libraries required by libs3 should be initialized.
+ **/
+#define S3_INIT_ALL                        (S3_INIT_WINSOCK)
+
+
+/** **************************************************************************
+ * Enumerations
+ ************************************************************************** **/
+
+/**
+ * S3Status is a status code as returned by a libs3 function.  The meaning of
+ * each status code is defined in the comments for each function which returns
+ * that status.
+ **/
+typedef enum
+{
+    S3StatusOK                                              ,
+
+    /**
+     * Errors that prevent the S3 request from being issued or response from
+     * being read
+     **/
+    S3StatusInternalError                                   ,
+    S3StatusOutOfMemory                                     ,
+    S3StatusInterrupted                                     ,
+    S3StatusInvalidBucketNameTooLong                        ,
+    S3StatusInvalidBucketNameFirstCharacter                 ,
+    S3StatusInvalidBucketNameCharacter                      ,
+    S3StatusInvalidBucketNameCharacterSequence              ,
+    S3StatusInvalidBucketNameTooShort                       ,
+    S3StatusInvalidBucketNameDotQuadNotation                ,
+    S3StatusQueryParamsTooLong                              ,
+    S3StatusFailedToInitializeRequest                       ,
+    S3StatusMetaDataHeadersTooLong                          ,
+    S3StatusBadMetaData                                     ,
+    S3StatusBadContentType                                  ,
+    S3StatusContentTypeTooLong                              ,
+    S3StatusBadMD5                                          ,
+    S3StatusMD5TooLong                                      ,
+    S3StatusBadCacheControl                                 ,
+    S3StatusCacheControlTooLong                             ,
+    S3StatusBadContentDispositionFilename                   ,
+    S3StatusContentDispositionFilenameTooLong               ,
+    S3StatusBadContentEncoding                              ,
+    S3StatusContentEncodingTooLong                          ,
+    S3StatusBadIfMatchETag                                  ,
+    S3StatusIfMatchETagTooLong                              ,
+    S3StatusBadIfNotMatchETag                               ,
+    S3StatusIfNotMatchETagTooLong                           ,
+    S3StatusHeadersTooLong                                  ,
+    S3StatusKeyTooLong                                      ,
+    S3StatusUriTooLong                                      ,
+    S3StatusXmlParseFailure                                 ,
+    S3StatusEmailAddressTooLong                             ,
+    S3StatusUserIdTooLong                                   ,
+    S3StatusUserDisplayNameTooLong                          ,
+    S3StatusGroupUriTooLong                                 ,
+    S3StatusPermissionTooLong                               ,
+    S3StatusTargetBucketTooLong                             ,
+    S3StatusTargetPrefixTooLong                             ,
+    S3StatusTooManyGrants                                   ,
+    S3StatusBadGrantee                                      ,
+    S3StatusBadPermission                                   ,
+    S3StatusXmlDocumentTooLarge                             ,
+    S3StatusNameLookupError                                 ,
+    S3StatusFailedToConnect                                 ,
+    S3StatusServerFailedVerification                        ,
+    S3StatusConnectionFailed                                ,
+    S3StatusAbortedByCallback                               ,
+    
+    /**
+     * Errors from the S3 service
+     **/
+    S3StatusErrorAccessDenied                               ,
+    S3StatusErrorAccountProblem                             ,
+    S3StatusErrorAmbiguousGrantByEmailAddress               ,
+    S3StatusErrorBadDigest                                  ,
+    S3StatusErrorBucketAlreadyExists                        ,
+    S3StatusErrorBucketAlreadyOwnedByYou                    ,
+    S3StatusErrorBucketNotEmpty                             ,
+    S3StatusErrorCredentialsNotSupported                    ,
+    S3StatusErrorCrossLocationLoggingProhibited             ,
+    S3StatusErrorEntityTooSmall                             ,
+    S3StatusErrorEntityTooLarge                             ,
+    S3StatusErrorExpiredToken                               ,
+    S3StatusErrorIncompleteBody                             ,
+    S3StatusErrorIncorrectNumberOfFilesInPostRequest        ,
+    S3StatusErrorInlineDataTooLarge                         ,
+    S3StatusErrorInternalError                              ,
+    S3StatusErrorInvalidAccessKeyId                         ,
+    S3StatusErrorInvalidAddressingHeader                    ,
+    S3StatusErrorInvalidArgument                            ,
+    S3StatusErrorInvalidBucketName                          ,
+    S3StatusErrorInvalidDigest                              ,
+    S3StatusErrorInvalidLocationConstraint                  ,
+    S3StatusErrorInvalidPayer                               ,
+    S3StatusErrorInvalidPolicyDocument                      ,
+    S3StatusErrorInvalidRange                               ,
+    S3StatusErrorInvalidSecurity                            ,
+    S3StatusErrorInvalidSOAPRequest                         ,
+    S3StatusErrorInvalidStorageClass                        ,
+    S3StatusErrorInvalidTargetBucketForLogging              ,
+    S3StatusErrorInvalidToken                               ,
+    S3StatusErrorInvalidURI                                 ,
+    S3StatusErrorKeyTooLong                                 ,
+    S3StatusErrorMalformedACLError                          ,
+    S3StatusErrorMalformedXML                               ,
+    S3StatusErrorMaxMessageLengthExceeded                   ,
+    S3StatusErrorMaxPostPreDataLengthExceededError          ,
+    S3StatusErrorMetadataTooLarge                           ,
+    S3StatusErrorMethodNotAllowed                           ,
+    S3StatusErrorMissingAttachment                          ,
+    S3StatusErrorMissingContentLength                       ,
+    S3StatusErrorMissingSecurityElement                     ,
+    S3StatusErrorMissingSecurityHeader                      ,
+    S3StatusErrorNoLoggingStatusForKey                      ,
+    S3StatusErrorNoSuchBucket                               ,
+    S3StatusErrorNoSuchKey                                  ,
+    S3StatusErrorNotImplemented                             ,
+    S3StatusErrorNotSignedUp                                ,
+    S3StatusErrorOperationAborted                           ,
+    S3StatusErrorPermanentRedirect                          ,
+    S3StatusErrorPreconditionFailed                         ,
+    S3StatusErrorRedirect                                   ,
+    S3StatusErrorRequestIsNotMultiPartContent               ,
+    S3StatusErrorRequestTimeout                             ,
+    S3StatusErrorRequestTimeTooSkewed                       ,
+    S3StatusErrorRequestTorrentOfBucketError                ,
+    S3StatusErrorSignatureDoesNotMatch                      ,
+    S3StatusErrorSlowDown                                   ,
+    S3StatusErrorTemporaryRedirect                          ,
+    S3StatusErrorTokenRefreshRequired                       ,
+    S3StatusErrorTooManyBuckets                             ,
+    S3StatusErrorUnexpectedContent                          ,
+    S3StatusErrorUnresolvableGrantByEmailAddress            ,
+    S3StatusErrorUserKeyMustBeSpecified                     ,
+    S3StatusErrorUnknown                                    ,
+
+    /**
+     * The following are HTTP errors returned by S3 without enough detail to
+     * distinguish any of the above S3StatusError conditions
+     **/
+    S3StatusHttpErrorMovedTemporarily                       ,
+    S3StatusHttpErrorBadRequest                             ,
+    S3StatusHttpErrorForbidden                              ,
+    S3StatusHttpErrorNotFound                               ,
+    S3StatusHttpErrorConflict                               ,
+    S3StatusHttpErrorUnknown
+} S3Status;
+
+
+/**
+ * S3Protocol represents a protocol that may be used for communicating a
+ * request to the Amazon S3 service.
+ *
+ * In general, HTTPS is greatly preferred (and should be the default of any
+ * application using libs3) because it protects any data being sent to or
+ * from S3 using strong encryption.  However, HTTPS is much more CPU intensive
+ * than HTTP, and if the caller is absolutely certain that it is OK for the
+ * data to be viewable by anyone in transit, then HTTP can be used.
+ **/
+typedef enum
+{
+    S3ProtocolHTTPS                     = 0,
+    S3ProtocolHTTP                      = 1
+} S3Protocol;
+
+
+/**
+ * S3UriStyle defines the form that an Amazon S3 URI identifying a bucket or
+ * object can take.  They are of these forms:
+ *
+ * Virtual Host: ${protocol}://${bucket}.s3.amazonaws.com/[${key}]
+ * Path: ${protocol}://s3.amazonaws.com/${bucket}/[${key}]
+ *
+ * It is generally better to use the Virual Host URI form, because it ensures
+ * that the bucket name used is compatible with normal HTTP GETs and POSTs of
+ * data to/from the bucket.  However, if DNS lookups for the bucket are too
+ * slow or unreliable for some reason, Path URI form may be used.
+ **/
+typedef enum
+{
+    S3UriStyleVirtualHost               = 0,
+    S3UriStylePath                      = 1
+} S3UriStyle;
+
+
+/**
+ * S3GranteeType defines the type of Grantee used in an S3 ACL Grant.
+ * Amazon Customer By Email - identifies the Grantee using their Amazon S3
+ *     account email address
+ * Canonical User - identifies the Grantee by S3 User ID and Display Name,
+ *     which can only be obtained by making requests to S3, for example, by
+ *     listing owned buckets
+ * All AWS Users - identifies all authenticated AWS users
+ * All Users - identifies all users
+ * Log Delivery - identifies the Amazon group responsible for writing
+ *                server access logs into buckets
+ **/
+typedef enum
+{
+    S3GranteeTypeAmazonCustomerByEmail  = 0,
+    S3GranteeTypeCanonicalUser          = 1,
+    S3GranteeTypeAllAwsUsers            = 2,
+    S3GranteeTypeAllUsers               = 3,
+    S3GranteeTypeLogDelivery            = 4
+} S3GranteeType;
+
+
+/**
+ * This is an individual permission granted to a grantee in an S3 ACL Grant.
+ * Read permission gives the Grantee the permission to list the bucket, or
+ *     read the object or its metadata
+ * Write permission gives the Grantee the permission to create, overwrite, or
+ *     delete any object in the bucket, and is not supported for objects
+ * ReadACP permission gives the Grantee the permission to read the ACP for
+ *     the bucket or object; the owner of the bucket or object always has
+ *     this permission implicitly
+ * WriteACP permission gives the Grantee the permission to overwrite the ACP
+ *     for the bucket or object; the owner of the bucket or object always has
+ *     this permission implicitly
+ * FullControl permission gives the Grantee all permissions specified by the
+ *     Read, Write, ReadACP, and WriteACP permissions
+ **/
+typedef enum
+{
+    S3PermissionRead                    = 0,
+    S3PermissionWrite                   = 1,
+    S3PermissionReadACP                 = 2,
+    S3PermissionWriteACP                = 3,
+    S3PermissionFullControl             = 4
+} S3Permission;
+
+
+/**
+ * S3CannedAcl is an ACL that can be specified when an object is created or
+ * updated.  Each canned ACL has a predefined value when expanded to a full
+ * set of S3 ACL Grants.
+ * Private canned ACL gives the owner FULL_CONTROL and no other permissions
+ *     are issued
+ * Public Read canned ACL gives the owner FULL_CONTROL and all users Read
+ *     permission 
+ * Public Read Write canned ACL gives the owner FULL_CONTROL and all users
+ *     Read and Write permission
+ * AuthenticatedRead canned ACL gives the owner FULL_CONTROL and authenticated
+ *     S3 users Read permission
+ **/
+typedef enum
+{
+    S3CannedAclPrivate                  = 0, /* private */
+    S3CannedAclPublicRead               = 1, /* public-read */
+    S3CannedAclPublicReadWrite          = 2, /* public-read-write */
+    S3CannedAclAuthenticatedRead        = 3  /* authenticated-read */
+} S3CannedAcl;
+
+
+/** **************************************************************************
+ * Data Types
+ ************************************************************************** **/
+
+/**
+ * An S3RequestContext manages multiple S3 requests simultaneously; see the
+ * S3_XXX_request_context functions below for details
+ **/
+typedef struct S3RequestContext S3RequestContext;
+
+
+/**
+ * S3NameValue represents a single Name - Value pair, used to represent either
+ * S3 metadata associated with a key, or S3 error details.
+ **/
+typedef struct S3NameValue
+{
+    /**
+     * The name part of the Name - Value pair
+     **/
+    const char *name;
+
+    /**
+     * The value part of the Name - Value pair
+     **/
+    const char *value;
+} S3NameValue;
+
+
+/**
+ * S3ResponseProperties is passed to the properties callback function which is
+ * called when the complete response properties have been received.  Some of
+ * the fields of this structure are optional and may not be provided in the
+ * response, and some will always be provided in the response.
+ **/
+typedef struct S3ResponseProperties
+{
+    /**
+     * This optional field identifies the request ID and may be used when
+     * reporting problems to Amazon.
+     **/
+    const char *requestId;
+
+    /**
+     * This optional field identifies the request ID and may be used when
+     * reporting problems to Amazon.
+     **/
+    const char *requestId2;
+
+    /**
+     * This optional field is the content type of the data which is returned
+     * by the request.  If not provided, the default can be assumed to be
+     * "binary/octet-stream".
+     **/
+    const char *contentType;
+
+    /**
+     * This optional field is the content length of the data which is returned
+     * in the response.  A negative value means that this value was not
+     * provided in the response.  A value of 0 means that there is no content
+     * provided.  A positive value gives the number of bytes in the content of
+     * the response.
+     **/
+    uint64_t contentLength;
+
+    /**
+     * This optional field names the server which serviced the request.
+     **/
+    const char *server;
+
+    /**
+     * This optional field provides a string identifying the unique contents
+     * of the resource identified by the request, such that the contents can
+     * be assumed not to be changed if the same eTag is returned at a later
+     * time decribing the same resource.  This is an MD5 sum of the contents.
+     **/
+    const char *eTag;
+
+    /**
+     * This optional field provides the last modified time, relative to the
+     * Unix epoch, of the contents.  If this value is < 0, then the last
+     * modified time was not provided in the response.  If this value is >= 0,
+     * then the last modified date of the contents are available as a number
+     * of seconds since the UNIX epoch.
+     * 
+     **/
+    int64_t lastModified;
+
+    /**
+     * This is the number of user-provided meta data associated with the
+     * resource.
+     **/
+    int metaDataCount;
+
+    /**
+     * These are the meta data associated with the resource.  In each case,
+     * the name will not include any S3-specific header prefixes
+     * (i.e. x-amz-meta- will have been removed from the beginning), and
+     * leading and trailing whitespace will have been stripped from the value.
+     **/
+    const S3NameValue *metaData;
+} S3ResponseProperties;
+
+
+/**
+ * S3AclGrant identifies a single grant in the ACL for a bucket or object.  An
+ * ACL is composed of any number of grants, which specify a grantee and the
+ * permissions given to that grantee.  S3 does not normalize ACLs in any way,
+ * so a redundant ACL specification will lead to a redundant ACL stored in S3.
+ **/
+typedef struct S3AclGrant
+{
+    /**
+     * The granteeType gives the type of grantee specified by this grant.
+     **/
+    S3GranteeType granteeType;
+    /**
+     * The identifier of the grantee that is set is determined by the
+     * granteeType:
+     *
+     * S3GranteeTypeAmazonCustomerByEmail - amazonCustomerByEmail.emailAddress
+     * S3GranteeTypeCanonicalUser - canonicalUser.id, canonicalUser.displayName
+     * S3GranteeTypeAllAwsUsers - none
+     * S3GranteeTypeAllUsers - none
+     **/
+    union
+    {
+        /**
+         * This structure is used iff the granteeType is 
+         * S3GranteeTypeAmazonCustomerByEmail.
+         **/
+        struct
+        {
+            /**
+             * This is the email address of the Amazon Customer being granted
+             * permissions by this S3AclGrant.
+             **/
+            char emailAddress[S3_MAX_GRANTEE_EMAIL_ADDRESS_SIZE];
+        } amazonCustomerByEmail;
+        /**
+         * This structure is used iff the granteeType is
+         * S3GranteeTypeCanonicalUser.
+         **/
+        struct
+        {
+            /**
+             * This is the CanonicalUser ID of the grantee
+             **/
+            char id[S3_MAX_GRANTEE_USER_ID_SIZE];
+            /**
+             * This is the display name of the grantee
+             **/
+            char displayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE];
+        } canonicalUser;
+    } grantee;
+    /**
+     * This is the S3Permission to be granted to the grantee
+     **/
+    S3Permission permission;
+} S3AclGrant;
+
+
+/**
+ * A context for working with objects within a bucket.  A bucket context holds
+ * all information necessary for working with a bucket, and may be used
+ * repeatedly over many consecutive (or simultaneous) calls into libs3 bucket
+ * operation functions.
+ **/
+typedef struct S3BucketContext
+{
+    /**
+     * The name of the host to connect to when making S3 requests.  If set to
+     * NULL, the default S3 hostname passed in to S3_initialize will be used.
+     **/
+    const char *hostName;
+
+    /**
+     * The name of the bucket to use in the bucket context
+     **/
+    const char *bucketName;
+
+    /**
+     * The protocol to use when accessing the bucket
+     **/
+    S3Protocol protocol;
+
+    /**
+     * The URI style to use for all URIs sent to Amazon S3 while working with
+     * this bucket context
+     **/
+    S3UriStyle uriStyle;
+
+    /**
+     * The Amazon Access Key ID to use for access to the bucket
+     **/
+    const char *accessKeyId;
+
+    /**
+     *  The Amazon Secret Access Key to use for access to the bucket
+     **/
+    const char *secretAccessKey;
+} S3BucketContext;
+
+
+/**
+ * This is a single entry supplied to the list bucket callback by a call to
+ * S3_list_bucket.  It identifies a single matching key from the list
+ * operation.
+ **/
+typedef struct S3ListBucketContent
+{
+    /**
+     * This is the next key in the list bucket results.
+     **/
+    const char *key;
+
+    /**
+     * This is the number of seconds since UNIX epoch of the last modified
+     * date of the object identified by the key. 
+     **/
+    int64_t lastModified;
+
+    /**
+     * This gives a tag which gives a signature of the contents of the object,
+     * which is the MD5 of the contents of the object.
+     **/
+    const char *eTag;
+
+    /**
+     * This is the size of the object in bytes.
+     **/
+    uint64_t size;
+
+    /**
+     * This is the ID of the owner of the key; it is present only if access
+     * permissions allow it to be viewed.
+     **/
+    const char *ownerId;
+
+    /**
+     * This is the display name of the owner of the key; it is present only if
+     * access permissions allow it to be viewed.
+     **/
+    const char *ownerDisplayName;
+} S3ListBucketContent;
+
+
+/**
+ * S3PutProperties is the set of properties that may optionally be set by the
+ * user when putting objects to S3.  Each field of this structure is optional
+ * and may or may not be present.
+ **/
+typedef struct S3PutProperties
+{
+    /**
+     * If present, this is the Content-Type that should be associated with the
+     * object.  If not provided, S3 defaults to "binary/octet-stream".
+     **/
+    const char *contentType;
+
+    /**
+     * If present, this provides the MD5 signature of the contents, and is
+     * used to validate the contents.  This is highly recommended by Amazon
+     * but not required.  Its format is as a base64-encoded MD5 sum.
+     **/
+    const char *md5;
+
+    /**
+     * If present, this gives a Cache-Control header string to be supplied to
+     * HTTP clients which download this
+     **/
+    const char *cacheControl;
+
+    /**
+     * If present, this gives the filename to save the downloaded file to,
+     * whenever the object is downloaded via a web browser.  This is only
+     * relevent for objects which are intended to be shared to users via web
+     * browsers and which is additionally intended to be downloaded rather
+     * than viewed.
+     **/
+    const char *contentDispositionFilename;
+
+    /**
+     * If present, this identifies the content encoding of the object.  This
+     * is only applicable to encoded (usually, compressed) content, and only
+     * relevent if the object is intended to be downloaded via a browser.
+     **/
+    const char *contentEncoding;
+
+    /**
+     * If >= 0, this gives an expiration date for the content.  This
+     * information is typically only delivered to users who download the
+     * content via a web browser.
+     **/
+    int64_t expires;
+
+    /**
+     * This identifies the "canned ACL" that should be used for this object.
+     * The default (0) gives only the owner of the object access to it.
+     **/
+    S3CannedAcl cannedAcl;
+
+    /**
+     * This is the number of values in the metaData field.
+     **/
+    int metaDataCount;
+
+    /**
+     * These are the meta data to pass to S3.  In each case, the name part of
+     * the Name - Value pair should not include any special S3 HTTP header
+     * prefix (i.e., should be of the form 'foo', NOT 'x-amz-meta-foo').
+     **/
+    const S3NameValue *metaData;
+} S3PutProperties;
+
+
+/**
+ * S3GetConditions is used for the get_object operation, and specifies
+ * conditions which the object must meet in order to be successfully returned.
+ **/
+typedef struct S3GetConditions
+{
+    /**
+     * The request will be processed if the Last-Modification header of the
+     * object is greater than or equal to this value, specified as a number of
+     * seconds since Unix epoch.  If this value is less than zero, it will not
+     * be used in the conditional.
+     **/
+    int64_t ifModifiedSince;
+
+    /**
+     * The request will be processed if the Last-Modification header of the
+     * object is less than this value, specified as a number of seconds since
+     * Unix epoch.  If this value is less than zero, it will not be used in
+     * the conditional.
+     **/
+    int64_t ifNotModifiedSince;
+
+    /**
+     * If non-NULL, this gives an eTag header value which the object must
+     * match in order to be returned.  Note that altough the eTag is simply an
+     * MD5, this must be presented in the S3 eTag form, which typically
+     * includes double-quotes.
+     **/
+    const char *ifMatchETag;
+
+    /**
+     * If non-NULL, this gives an eTag header value which the object must not
+     * match in order to be returned.  Note that altough the eTag is simply an
+     * MD5, this must be presented in the S3 eTag form, which typically
+     * includes double-quotes.
+     **/
+    const char *ifNotMatchETag;
+} S3GetConditions;
+
+
+/**
+ * S3ErrorDetails provides detailed information describing an S3 error.  This
+ * is only presented when the error is an S3-generated error (i.e. one of the
+ * S3StatusErrorXXX values).
+ **/
+typedef struct S3ErrorDetails
+{
+    /**
+     * This is the human-readable message that Amazon supplied describing the
+     * error
+     **/
+    const char *message;
+
+    /**
+     * This identifies the resource for which the error occurred
+     **/
+    const char *resource;
+
+    /**
+     * This gives human-readable further details describing the specifics of
+     * this error
+     **/
+    const char *furtherDetails;
+
+    /**
+     * This gives the number of S3NameValue pairs present in the extraDetails
+     * array
+     **/
+    int extraDetailsCount;
+
+    /**
+     * S3 can provide extra details in a freeform Name - Value pair format.
+     * Each error can have any number of these, and this array provides these
+     * additional extra details.
+     **/
+    S3NameValue *extraDetails;
+} S3ErrorDetails;
+
+
+/** **************************************************************************
+ * Callback Signatures
+ ************************************************************************** **/
+
+/**
+ * This callback is made whenever the response properties become available for
+ * any request.
+ *
+ * @param properties are the properties that are available from the response
+ * @param callbackData is the callback data as specified when the request
+ *        was issued.
+ * @return S3StatusOK to continue processing the request, anything else to
+ *         immediately abort the request with a status which will be
+ *         passed to the S3ResponseCompleteCallback for this request.
+ *         Typically, this will return either S3StatusOK or
+ *         S3StatusAbortedByCallback.
+ **/
+typedef S3Status (S3ResponsePropertiesCallback)
+    (const S3ResponseProperties *properties, void *callbackData);
+
+
+/**
+ * This callback is made when the response has been completely received, or an
+ * error has occurred which has prematurely aborted the request, or one of the
+ * other user-supplied callbacks returned a value intended to abort the
+ * request.  This callback is always made for every request, as the very last
+ * callback made for that request.
+ *
+ * @param status gives the overall status of the response, indicating success
+ *        or failure; use S3_status_is_retryable() as a simple way to detect
+ *        whether or not the status indicates that the request failed but may
+ *        be retried.
+ * @param errorDetails if non-NULL, gives details as returned by the S3
+ *        service, describing the error
+ * @param callbackData is the callback data as specified when the request
+ *        was issued.
+ **/
+typedef void (S3ResponseCompleteCallback)(S3Status status,
+                                          const S3ErrorDetails *errorDetails,
+                                          void *callbackData);
+
+                                    
+/**
+ * This callback is made for each bucket resulting from a list service
+ * operation.
+ *
+ * @param ownerId is the ID of the owner of the bucket
+ * @param ownerDisplayName is the owner display name of the owner of the bucket
+ * @param bucketName is the name of the bucket
+ * @param creationDateSeconds if < 0 indicates that no creation date was
+ *        supplied for the bucket; if >= 0 indicates the number of seconds
+ *        since UNIX Epoch of the creation date of the bucket
+ * @param callbackData is the callback data as specified when the request
+ *        was issued.
+ * @return S3StatusOK to continue processing the request, anything else to
+ *         immediately abort the request with a status which will be
+ *         passed to the S3ResponseCompleteCallback for this request.
+ *         Typically, this will return either S3StatusOK or
+ *         S3StatusAbortedByCallback.
+ **/
+typedef S3Status (S3ListServiceCallback)(const char *ownerId, 
+                                         const char *ownerDisplayName,
+                                         const char *bucketName,
+                                         int64_t creationDateSeconds,
+                                         void *callbackData);
+
+
+/**
+ * This callback is made repeatedly as a list bucket operation progresses.
+ * The contents reported via this callback are only reported once per list
+ * bucket operation, but multiple calls to this callback may be necessary to
+ * report all items resulting from the list bucket operation.
+ *
+ * @param isTruncated is true if the list bucket request was truncated by the
+ *        S3 service, in which case the remainder of the list may be obtained
+ *        by querying again using the Marker parameter to start the query
+ *        after this set of results
+ * @param nextMarker if present, gives the largest (alphabetically) key
+ *        returned in the response, which, if isTruncated is true, may be used
+ *        as the marker in a subsequent list buckets operation to continue
+ *        listing
+ * @param contentsCount is the number of ListBucketContent structures in the
+ *        contents parameter
+ * @param contents is an array of ListBucketContent structures, each one
+ *        describing an object in the bucket
+ * @param commonPrefixesCount is the number of common prefixes strings in the
+ *        commonPrefixes parameter
+ * @param commonPrefixes is an array of strings, each specifing one of the
+ *        common prefixes as returned by S3
+ * @param callbackData is the callback data as specified when the request
+ *        was issued.
+ * @return S3StatusOK to continue processing the request, anything else to
+ *         immediately abort the request with a status which will be
+ *         passed to the S3ResponseCompleteCallback for this request.
+ *         Typically, this will return either S3StatusOK or
+ *         S3StatusAbortedByCallback.
+ **/
+typedef S3Status (S3ListBucketCallback)(int isTruncated,
+                                        const char *nextMarker,
+                                        int contentsCount, 
+                                        const S3ListBucketContent *contents,
+                                        int commonPrefixesCount,
+                                        const char **commonPrefixes,
+                                        void *callbackData);
+                                       
+
+/**
+ * This callback is made during a put object operation, to obtain the next
+ * chunk of data to put to the S3 service as the contents of the object.  This
+ * callback is made repeatedly, each time acquiring the next chunk of data to
+ * write to the service, until a negative or 0 value is returned.
+ *
+ * @param bufferSize gives the maximum number of bytes that may be written
+ *        into the buffer parameter by this callback
+ * @param buffer gives the buffer to fill with at most bufferSize bytes of
+ *        data as the next chunk of data to send to S3 as the contents of this
+ *        object
+ * @param callbackData is the callback data as specified when the request
+ *        was issued.
+ * @return < 0 to abort the request with the S3StatusAbortedByCallback, which
+ *        will be pased to the response complete callback for this request, or
+ *        0 to indicate the end of data, or > 0 to identify the number of
+ *        bytes that were written into the buffer by this callback
+ **/
+typedef int (S3PutObjectDataCallback)(int bufferSize, char *buffer,
+                                      void *callbackData);
+
+
+/**
+ * This callback is made during a get object operation, to provide the next
+ * chunk of data available from the S3 service constituting the contents of
+ * the object being fetched.  This callback is made repeatedly, each time
+ * providing the next chunk of data read, until the complete object contents
+ * have been passed through the callback in this way, or the callback
+ * returns an error status.
+ *
+ * @param bufferSize gives the number of bytes in buffer
+ * @param buffer is the data being passed into the callback
+ * @param callbackData is the callback data as specified when the request
+ *        was issued.
+ * @return S3StatusOK to continue processing the request, anything else to
+ *         immediately abort the request with a status which will be
+ *         passed to the S3ResponseCompleteCallback for this request.
+ *         Typically, this will return either S3StatusOK or
+ *         S3StatusAbortedByCallback.
+ **/
+typedef S3Status (S3GetObjectDataCallback)(int bufferSize, const char *buffer,
+                                           void *callbackData);
+                                       
+
+/** **************************************************************************
+ * Callback Structures
+ ************************************************************************** **/
+
+
+/**
+ * An S3ResponseHandler defines the callbacks which are made for any
+ * request.
+ **/
+typedef struct S3ResponseHandler
+{
+    /**
+     * The propertiesCallback is made when the response properties have
+     * successfully been returned from S3.  This function may not be called
+     * if the response properties were not successfully returned from S3.
+     **/
+    S3ResponsePropertiesCallback *propertiesCallback;
+    
+    /**
+     * The completeCallback is always called for every request made to S3,
+     * regardless of the outcome of the request.  It provides the status of
+     * the request upon its completion, as well as extra error details in the
+     * event of an S3 error.
+     **/
+    S3ResponseCompleteCallback *completeCallback;
+} S3ResponseHandler;
+
+
+/**
+ * An S3ListServiceHandler defines the callbacks which are made for
+ * list_service requests.
+ **/
+typedef struct S3ListServiceHandler
+{
+    /**
+     * responseHandler provides the properties and complete callback
+     **/
+    S3ResponseHandler responseHandler;
+
+    /**
+     * The listServiceCallback is called as items are reported back from S3 as
+     * responses to the request
+     **/
+    S3ListServiceCallback *listServiceCallback;
+} S3ListServiceHandler;
+
+
+/**
+ * An S3ListBucketHandler defines the callbacks which are made for
+ * list_bucket requests.
+ **/
+typedef struct S3ListBucketHandler
+{
+    /**
+     * responseHandler provides the properties and complete callback
+     **/
+    S3ResponseHandler responseHandler;
+
+    /**
+     * The listBucketCallback is called as items are reported back from S3 as
+     * responses to the request.  This may be called more than one time per
+     * list bucket request, each time providing more items from the list
+     * operation.
+     **/
+    S3ListBucketCallback *listBucketCallback;
+} S3ListBucketHandler;
+
+
+/**
+ * An S3PutObjectHandler defines the callbacks which are made for
+ * put_object requests.
+ **/
+typedef struct S3PutObjectHandler
+{
+    /**
+     * responseHandler provides the properties and complete callback
+     **/
+    S3ResponseHandler responseHandler;
+
+    /**
+     * The putObjectDataCallback is called to acquire data to send to S3 as
+     * the contents of the put_object request.  It is made repeatedly until it
+     * returns a negative number (indicating that the request should be
+     * aborted), or 0 (indicating that all data has been supplied).
+     **/
+    S3PutObjectDataCallback *putObjectDataCallback;
+} S3PutObjectHandler;
+
+
+/**
+ * An S3GetObjectHandler defines the callbacks which are made for
+ * get_object requests.
+ **/
+typedef struct S3GetObjectHandler
+{
+    /**
+     * responseHandler provides the properties and complete callback
+     **/
+    S3ResponseHandler responseHandler;
+
+    /**
+     * The getObjectDataCallback is called as data is read from S3 as the
+     * contents of the object being read in the get_object request.  It is
+     * called repeatedly until there is no more data provided in the request,
+     * or until the callback returns an error status indicating that the
+     * request should be aborted.
+     **/
+    S3GetObjectDataCallback *getObjectDataCallback;
+} S3GetObjectHandler;
+
+
+/** **************************************************************************
+ * General Library Functions
+ ************************************************************************** **/
+
+/**
+ * Initializes libs3 for use.  This function must be called before any other
+ * libs3 function is called.  It may be called multiple times, with the same
+ * effect as calling it once, as long as S3_deinitialize() is called an
+ * equal number of times when the program has finished.  This function is NOT
+ * thread-safe and must only be called by one thread at a time.
+ *
+ * @param userAgentInfo is a string that will be included in the User-Agent
+ *        header of every request made to the S3 service.  You may provide
+ *        NULL or the empty string if you don't care about this.  The value
+ *        will not be copied by this function and must remain unaltered by the
+ *        caller until S3_deinitialize() is called.
+ * @param flags is a bitmask of some combination of S3_INIT_XXX flag, or
+ *        S3_INIT_ALL, indicating which of the libraries that libs3 depends
+ *        upon should be initialized by S3_initialize().  Only if your program
+ *        initializes one of these dependency libraries itself should anything
+ *        other than S3_INIT_ALL be passed in for this bitmask.
+ *
+ *        You should pass S3_INIT_WINSOCK if and only if your application does
+ *        not initialize winsock elsewhere.  On non-Microsoft Windows
+ *        platforms it has no effect.
+ *
+ *        As a convenience, the macro S3_INIT_ALL is provided, which will do
+ *        all necessary initialization; however, be warned that things may
+ *        break if your application re-initializes the dependent libraries
+ *        later.
+ * @param defaultS3Hostname is a string the specifies the default S3 server
+ *        hostname to use when making S3 requests; this value is used
+ *        whenever the hostName of an S3BucketContext is NULL.  If NULL is
+ *        passed here then the default of S3_DEFAULT_HOSTNAME will be used.
+ * @return One of:
+ *         S3StatusOK on success
+ *         S3StatusUriTooLong if the defaultS3HostName is longer than
+ *             S3_MAX_HOSTNAME_SIZE
+ *         S3StatusInternalError if dependent libraries could not be
+ *             initialized
+ *         S3StatusOutOfMemory on failure due to out of memory
+ **/
+S3Status S3_initialize(const char *userAgentInfo, int flags,
+                       const char *defaultS3HostName);
+
+
+/**
+ * Must be called once per program for each call to libs3_initialize().  After
+ * this call is complete, no libs3 function may be called except
+ * S3_initialize().
+ **/
+void S3_deinitialize();
+
+
+/**
+ * Returns a string with the textual name of an S3Status code
+ *
+ * @param status is S3Status code for which the textual name will be returned
+ * @return a string with the textual name of an S3Status code
+ **/
+const char *S3_get_status_name(S3Status status);
+
+
+/**
+ * This function may be used to validate an S3 bucket name as being in the
+ * correct form for use with the S3 service.  Amazon S3 limits the allowed
+ * characters in S3 bucket names, as well as imposing some additional rules on
+ * the length of bucket names and their structure.  There are actually two
+ * limits; one for bucket names used only in path-style URIs, and a more
+ * strict limit used for bucket names used in virtual-host-style URIs.  It is
+ * advisable to use only bucket names which meet the more strict requirements
+ * regardless of how the bucket expected to be used.
+ *
+ * This method does NOT validate that the bucket is available for use in the
+ * S3 service, so the return value of this function cannot be used to decide
+ * whether or not a bucket with the give name already exists in Amazon S3 or
+ * is accessible by the caller.  It merely validates that the bucket name is
+ * valid for use with S3.
+ *
+ * @param bucketName is the bucket name to validate
+ * @param uriStyle gives the URI style to validate the bucket name against.
+ *        It is advisable to always use S3UriStyleVirtuallHost.
+ * @return One of:
+ *         S3StatusOK if the bucket name was validates successfully
+ *         S3StatusInvalidBucketNameTooLong if the bucket name exceeded the
+ *             length limitation for the URI style, which is 255 bytes for
+ *             path style URIs and 63 bytes for virtual host type URIs
+ *         S3StatusInvalidBucketNameTooShort if the bucket name is less than
+ *             3 characters
+ *         S3StatusInvalidBucketNameFirstCharacter if the bucket name as an
+ *             invalid first character, which is anything other than
+ *             an alphanumeric character
+ *         S3StatusInvalidBucketNameCharacterSequence if the bucket name
+ *             includes an invalid character sequence, which for virtual host
+ *             style buckets is ".-" or "-."
+ *         S3StatusInvalidBucketNameCharacter if the bucket name includes an
+ *             invalid character, which is anything other than alphanumeric,
+ *             '-', '.', or for path style URIs only, '_'.
+ *         S3StatusInvalidBucketNameDotQuadNotation if the bucket name is in
+ *             dot-quad notation, i.e. the form of an IP address, which is
+ *             not allowed by Amazon S3.
+ **/
+S3Status S3_validate_bucket_name(const char *bucketName, S3UriStyle uriStyle);
+
+
+/**
+ * Converts an XML representation of an ACL to a libs3 structured
+ * representation.  This method is not strictly necessary for working with
+ * ACLs using libs3, but may be convenient for users of the library who read
+ * ACLs from elsewhere in XML format and need to use these ACLs with libs3.
+ *
+ * @param aclXml is the XML representation of the ACL.  This must be a
+ *        zero-terminated character string.
+ * @param ownerId will be filled in with the Owner ID specified in the XML.
+ *        At most MAX_GRANTEE_USER_ID_SIZE bytes will be stored at this
+ *        location.
+ * @param ownerDisplayName will be filled in with the Owner Display Name
+ *        specified in the XML.  At most MAX_GRANTEE_DISPLAY_NAME_SIZE bytes
+ *        will be stored at this location.
+ * @param aclGrantCountReturn returns the number of S3AclGrant structures
+ *        returned in the aclGrantsReturned array
+ * @param aclGrants must be passed in as an array of at least S3_ACL_MAXCOUNT
+ *        structures, and on return from this function, the first
+ *        aclGrantCountReturn structures will be filled in with the ACLs
+ *        represented by the input XML.
+ * @return One of:
+ *         S3StatusOK on successful conversion of the ACL
+ *         S3StatusInternalError on internal error representing a bug in the
+ *             libs3 library
+ *         S3StatusXmlParseFailure if the XML document was malformed
+ **/
+S3Status S3_convert_acl(char *aclXml, char *ownerId, char *ownerDisplayName,
+                        int *aclGrantCountReturn, S3AclGrant *aclGrants);
+                        
+
+/**
+ * Returns nonzero if the status indicates that the request should be
+ * immediately retried, because the status indicates an error of a nature that
+ * is likely due to transient conditions on the local system or S3, such as
+ * network failures, or internal retryable errors reported by S3.  Returns
+ * zero otherwise.
+ *
+ * @param status is the status to evaluate
+ * @return nonzero if the status indicates a retryable error, 0 otherwise
+ **/
+int S3_status_is_retryable(S3Status status);
+
+
+/** **************************************************************************
+ * Request Context Management Functions
+ ************************************************************************** **/
+
+/**
+ * An S3RequestContext allows muliple requests to be serviced by the same
+ * thread simultaneously.  It is an optional parameter to all libs3 request
+ * functions, and if provided, the request is managed by the S3RequestContext;
+ * if not, the request is handled synchronously and is complete when the libs3
+ * request function has returned.
+ *
+ * @param requestContextReturn returns the newly-created S3RequestContext
+ *        structure, which if successfully returned, must be destroyed via a
+ *        call to S3_destroy_request_context when it is no longer needed.  If
+ *        an error status is returned from this function, then
+ *        requestContextReturn will not have been filled in, and
+ *        S3_destroy_request_context should not be called on it
+ * @return One of:
+ *         S3StatusOK if the request context was successfully created
+ *         S3StatusOutOfMemory if the request context could not be created due
+ *             to an out of memory error
+ **/
+S3Status S3_create_request_context(S3RequestContext **requestContextReturn);
+
+
+/**
+ * Destroys an S3RequestContext which was created with
+ * S3_create_request_context.  Any requests which are currently being
+ * processed by the S3RequestContext will immediately be aborted and their
+ * request completed callbacks made with the status S3StatusInterrupted.
+ *
+ * @param requestContext is the S3RequestContext to destroy
+ **/
+void S3_destroy_request_context(S3RequestContext *requestContext);
+
+
+/**
+ * Runs the S3RequestContext until all requests within it have completed,
+ * or until an error occurs.
+ *
+ * @param requestContext is the S3RequestContext to run until all requests
+ *            within it have completed or until an error occurs
+ * @return One of:
+ *         S3Status if all requests were successfully run to completion
+ *         S3StatusInternalError if an internal error prevented the
+ *             S3RequestContext from running one or more requests
+ *         S3StatusOutOfMemory if requests could not be run to completion
+ *             due to an out of memory error
+ **/
+S3Status S3_runall_request_context(S3RequestContext *requestContext);
+
+
+/**
+ * Does some processing of requests within the S3RequestContext.  One or more
+ * requests may have callbacks made on them and may complete.  This function
+ * processes any requests which have immediately available I/O, and will not
+ * block waiting for I/O on any request.  This function would normally be used
+ * with S3_get_request_context_fdsets.
+ *
+ * @param requestContext is the S3RequestContext to process
+ * @param requestsRemainingReturn returns the number of requests remaining
+ *            and not yet completed within the S3RequestContext after this
+ *            function returns.
+ * @return One of:
+ *         S3StatusOK if request processing proceeded without error
+ *         S3StatusInternalError if an internal error prevented the
+ *             S3RequestContext from running one or more requests
+ *         S3StatusOutOfMemory if requests could not be processed due to
+ *             an out of memory error
+ **/
+S3Status S3_runonce_request_context(S3RequestContext *requestContext, 
+                                    int *requestsRemainingReturn);
+
+
+/**
+ * This function, in conjunction allows callers to manually manage a set of
+ * requests using an S3RequestContext.  This function returns the set of file
+ * descriptors which the caller can watch (typically using select()), along
+ * with any other file descriptors of interest to the caller, and using
+ * whatever timeout (if any) the caller wishes, until one or more file
+ * descriptors in the returned sets become ready for I/O, at which point
+ * S3_runonce_request_context can be called to process requests with available
+ * I/O.
+ *
+ * @param requestContext is the S3RequestContext to get fd_sets from
+ * @param readFdSet is a pointer to an fd_set which will have all file
+ *        descriptors to watch for read events for the requests in the
+ *        S3RequestContext set into it upon return.  Should be zero'd out
+ *        (using FD_ZERO) before being passed into this function.
+ * @param writeFdSet is a pointer to an fd_set which will have all file
+ *        descriptors to watch for write events for the requests in the
+ *        S3RequestContext set into it upon return.  Should be zero'd out
+ *        (using FD_ZERO) before being passed into this function.
+ * @param exceptFdSet is a pointer to an fd_set which will have all file
+ *        descriptors to watch for exception events for the requests in the
+ *        S3RequestContext set into it upon return.  Should be zero'd out
+ *        (using FD_ZERO) before being passed into this function.
+ * @param maxFd returns the highest file descriptor set into any of the
+ *        fd_sets, or -1 if no file descriptors were set
+ * @return One of:
+ *         S3StatusOK if all fd_sets were successfully set
+ *         S3StatusInternalError if an internal error prevented this function
+ *             from completing successfully
+ **/
+S3Status S3_get_request_context_fdsets(S3RequestContext *requestContext,
+                                       fd_set *readFdSet, fd_set *writeFdSet,
+                                       fd_set *exceptFdSet, int *maxFd);
+
+
+/**
+ * This function returns the maximum number of milliseconds that the caller of
+ * S3_runonce_request_context should wait on the fdsets obtained via a call to
+ * S3_get_request_context_fdsets.  In other words, this is essentially the
+ * select() timeout that needs to be used (shorter values are OK, but no
+ * longer than this) to ensure that internal timeout code of libs3 can work
+ * properly.  This function should be called right before select() each time
+ * select() on the request_context fdsets are to be performed by the libs3
+ * user.
+ *
+ * @param requestContext is the S3RequestContext to get the timeout from
+ * @return the maximum number of milliseconds to select() on fdsets.  Callers
+ *         could wait a shorter time if they wish, but not longer.
+ **/
+int64_t S3_get_request_context_timeout(S3RequestContext *requestContext);
+
+
+/** **************************************************************************
+ * S3 Utility Functions
+ ************************************************************************** **/
+
+/**
+ * Generates an HTTP authenticated query string, which may then be used by
+ * a browser (or other web client) to issue the request.  The request is
+ * implicitly a GET request; Amazon S3 is documented to only support this type
+ * of authenticated query string request.
+ *
+ * @param buffer is the output buffer for the authenticated query string.
+ *        It must be at least S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in 
+ *        length.
+ * @param bucketContext gives the bucket and associated parameters for the
+ *        request to generate.
+ * @param key gives the key which the authenticated request will GET.
+ * @param expires gives the number of seconds since Unix epoch for the
+ *        expiration date of the request; after this time, the request will
+ *        no longer be valid.  If this value is negative, the largest
+ *        expiration date possible is used (currently, Jan 19, 2038).
+ * @param resource gives a sub-resource to be fetched for the request, or NULL
+ *        for none.  This should be of the form "?", i.e. 
+ *        "?torrent".
+ * @return One of:
+ *         S3StatusUriTooLong if, due to an internal error, the generated URI
+ *             is longer than S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in
+ *             length and thus will not fit into the supplied buffer
+ *         S3StatusOK on success
+ **/
+S3Status S3_generate_authenticated_query_string
+    (char *buffer, const S3BucketContext *bucketContext,
+     const char *key, int64_t expires, const char *resource);
+
+
+/** **************************************************************************
+ * Service Functions
+ ************************************************************************** **/
+
+/**
+ * Lists all S3 buckets belonging to the access key id.
+ *
+ * @param protocol gives the protocol to use for this request
+ * @param accessKeyId gives the Amazon Access Key ID for which to list owned
+ *        buckets
+ * @param secretAccessKey gives the Amazon Secret Access Key for which to list
+ *        owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ *        default S3 host as provided to S3_initialize() will be used.
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_list_service(S3Protocol protocol, const char *accessKeyId,
+                     const char *secretAccessKey, const char *hostName,
+                     S3RequestContext *requestContext,
+                     const S3ListServiceHandler *handler,
+                     void *callbackData);
+                         
+
+/** **************************************************************************
+ * Bucket Functions
+ ************************************************************************** **/
+
+/**
+ * Tests the existence of an S3 bucket, additionally returning the bucket's
+ * location if it exists and is accessible.
+ *
+ * @param protocol gives the protocol to use for this request
+ * @param uriStyle gives the URI style to use for this request
+ * @param accessKeyId gives the Amazon Access Key ID for which to list owned
+ *        buckets
+ * @param secretAccessKey gives the Amazon Secret Access Key for which to list
+ *        owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ *        default S3 host as provided to S3_initialize() will be used.
+ * @param bucketName is the bucket name to test
+ * @param locationConstraintReturnSize gives the number of bytes in the
+ *        locationConstraintReturn parameter
+ * @param locationConstraintReturn provides the location into which to write
+ *        the name of the location constraint naming the geographic location
+ *        of the S3 bucket.  This must have at least as many characters in it
+ *        as specified by locationConstraintReturn, and should start out
+ *        NULL-terminated.  On successful completion of this request, this
+ *        will be set to the name of the geographic location of S3 bucket, or
+ *        will be left as a zero-length string if no location was available.
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle,
+                    const char *accessKeyId, const char *secretAccessKey,
+                    const char *hostName, const char *bucketName,
+                    int locationConstraintReturnSize,
+                    char *locationConstraintReturn,
+                    S3RequestContext *requestContext,
+                    const S3ResponseHandler *handler, void *callbackData);
+
+                           
+/**
+ * Creates a new bucket.
+ *
+ * @param protocol gives the protocol to use for this request
+ * @param accessKeyId gives the Amazon Access Key ID for which to list owned
+ *        buckets
+ * @param secretAccessKey gives the Amazon Secret Access Key for which to list
+ *        owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ *        default S3 host as provided to S3_initialize() will be used.
+ * @param bucketName is the name of the bucket to be created
+ * @param cannedAcl gives the "REST canned ACL" to use for the created bucket
+ * @param locationConstraint if non-NULL, gives the geographic location for
+ *        the bucket to create.
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_create_bucket(S3Protocol protocol, const char *accessKeyId,
+                      const char *secretAccessKey, const char *hostName,
+                      const char *bucketName, S3CannedAcl cannedAcl,
+                      const char *locationConstraint,
+                      S3RequestContext *requestContext,
+                      const S3ResponseHandler *handler, void *callbackData);
+
+
+/**
+ * Deletes a bucket.  The bucket must be empty, or the status
+ * S3StatusErrorBucketNotEmpty will result.
+ *
+ * @param protocol gives the protocol to use for this request
+ * @param uriStyle gives the URI style to use for this request
+ * @param accessKeyId gives the Amazon Access Key ID for which to list owned
+ *        buckets
+ * @param secretAccessKey gives the Amazon Secret Access Key for which to list
+ *        owned buckets
+ * @param hostName is the S3 host name to use; if NULL is passed in, the
+ *        default S3 host as provided to S3_initialize() will be used.
+ * @param bucketName is the name of the bucket to be deleted
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle,
+                      const char *accessKeyId, const char *secretAccessKey,
+                      const char *hostName, const char *bucketName,
+                      S3RequestContext *requestContext,
+                      const S3ResponseHandler *handler, void *callbackData);
+
+
+/**
+ * Lists keys within a bucket.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request
+ * @param prefix if present, gives a prefix for matching keys
+ * @param marker if present, only keys occuring after this value will be
+ *        listed
+ * @param delimiter if present, causes keys that contain the same string
+ *        between the prefix and the first occurrence of the delimiter to be
+ *        rolled up into a single result element
+ * @param maxkeys is the maximum number of keys to return
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_list_bucket(const S3BucketContext *bucketContext,
+                    const char *prefix, const char *marker, 
+                    const char *delimiter, int maxkeys,
+                    S3RequestContext *requestContext,
+                    const S3ListBucketHandler *handler, void *callbackData);
+
+
+/** **************************************************************************
+ * Object Functions
+ ************************************************************************** **/
+
+/**
+ * Puts object data to S3.  This overwrites any existing object at that key;
+ * note that S3 currently only supports full-object upload.  The data to
+ * upload will be acquired by calling the handler's putObjectDataCallback.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request
+ * @param key is the key of the object to put to
+ * @param contentLength is required and gives the total number of bytes that
+ *        will be put
+ * @param putProperties optionally provides additional properties to apply to
+ *        the object that is being put to
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_put_object(const S3BucketContext *bucketContext, const char *key,
+                   uint64_t contentLength,
+                   const S3PutProperties *putProperties,
+                   S3RequestContext *requestContext,
+                   const S3PutObjectHandler *handler, void *callbackData);
+                        
+
+/**
+ * Copies an object from one location to another.  The object may be copied
+ * back to itself, which is useful for replacing metadata without changing
+ * the object.
+ *
+ * @param bucketContext gives the source bucket and associated parameters for
+ *        this request
+ * @param key is the source key
+ * @param destinationBucket gives the destination bucket into which to copy
+ *        the object.  If NULL, the source bucket will be used.
+ * @param destinationKey gives the destination key into which to copy the
+ *        object.  If NULL, the source key will be used.
+ * @param putProperties optionally provides properties to apply to the object
+ *        that is being put to.  If not supplied (i.e. NULL is passed in),
+ *        then the copied object will retain the metadata of the copied
+ *        object.
+ * @param lastModifiedReturn returns the last modified date of the copied
+ *        object
+ * @param eTagReturnSize specifies the number of bytes provided in the
+ *        eTagReturn buffer
+ * @param eTagReturn is a buffer into which the resulting eTag of the copied
+ *        object will be written
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_copy_object(const S3BucketContext *bucketContext,
+                    const char *key, const char *destinationBucket,
+                    const char *destinationKey,
+                    const S3PutProperties *putProperties,
+                    int64_t *lastModifiedReturn, int eTagReturnSize,
+                    char *eTagReturn, S3RequestContext *requestContext,
+                    const S3ResponseHandler *handler, void *callbackData);
+
+
+/**
+ * Gets an object from S3.  The contents of the object are returned in the
+ * handler's getObjectDataCallback.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request
+ * @param key is the key of the object to get
+ * @param getConditions if non-NULL, gives a set of conditions which must be
+ *        met in order for the request to succeed
+ * @param startByte gives the start byte for the byte range of the contents
+ *        to be returned
+ * @param byteCount gives the number of bytes to return; a value of 0
+ *        indicates that the contents up to the end should be returned
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_get_object(const S3BucketContext *bucketContext, const char *key,
+                   const S3GetConditions *getConditions,
+                   uint64_t startByte, uint64_t byteCount,
+                   S3RequestContext *requestContext,
+                   const S3GetObjectHandler *handler, void *callbackData);
+
+
+/**
+ * Gets the response properties for the object, but not the object contents.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request
+ * @param key is the key of the object to get the properties of
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_head_object(const S3BucketContext *bucketContext, const char *key,
+                    S3RequestContext *requestContext,
+                    const S3ResponseHandler *handler, void *callbackData);
+                         
+/**
+ * Deletes an object from S3.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request
+ * @param key is the key of the object to delete
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_delete_object(const S3BucketContext *bucketContext, const char *key,
+                      S3RequestContext *requestContext,
+                      const S3ResponseHandler *handler, void *callbackData);
+
+
+/** **************************************************************************
+ * Access Control List Functions
+ ************************************************************************** **/
+
+/**
+ * Gets the ACL for the given bucket or object.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request
+ * @param key is the key of the object to get the ACL of; or NULL to get the
+ *        ACL of the bucket
+ * @param ownerId must be supplied as a buffer of at least
+ *        S3_MAX_GRANTEE_USER_ID_SIZE bytes, and will be filled in with the
+ *        owner ID of the object/bucket
+ * @param ownerDisplayName must be supplied as a buffer of at least
+ *        S3_MAX_GRANTEE_DISPLAY_NAME_SIZE bytes, and will be filled in with
+ *        the display name of the object/bucket
+ * @param aclGrantCountReturn returns the number of S3AclGrant structures
+ *        returned in the aclGrants parameter
+ * @param aclGrants must be passed in as an array of at least
+ *        S3_MAX_ACL_GRANT_COUNT S3AclGrant structures, which will be filled
+ *        in with the grant information for the ACL
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_get_acl(const S3BucketContext *bucketContext, const char *key, 
+                char *ownerId, char *ownerDisplayName,
+                int *aclGrantCountReturn, S3AclGrant *aclGrants, 
+                S3RequestContext *requestContext,
+                const S3ResponseHandler *handler, void *callbackData);
+
+
+/**
+ * Sets the ACL for the given bucket or object.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request
+ * @param key is the key of the object to set the ACL for; or NULL to set the
+ *        ACL for the bucket
+ * @param ownerId is the owner ID of the object/bucket.  Unfortunately, S3
+ *        requires this to be valid and thus it must have been fetched by a
+ *        previous S3 request, such as a list_buckets request.
+ * @param ownerDisplayName is the owner display name of the object/bucket.
+ *        Unfortunately, S3 requires this to be valid and thus it must have
+ *        been fetched by a previous S3 request, such as a list_buckets
+ *        request.
+ * @param aclGrantCount is the number of ACL grants to set for the
+ *        object/bucket
+ * @param aclGrants are the ACL grants to set for the object/bucket
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_set_acl(const S3BucketContext *bucketContext, const char *key, 
+                const char *ownerId, const char *ownerDisplayName,
+                int aclGrantCount, const S3AclGrant *aclGrants, 
+                S3RequestContext *requestContext,
+                const S3ResponseHandler *handler, void *callbackData);
+
+
+/** **************************************************************************
+ * Server Access Log Functions
+ ************************************************************************** **/
+
+/**
+ * Gets the service access logging settings for a bucket.  The service access
+ * logging settings specify whether or not the S3 service will write service
+ * access logs for requests made for the given bucket, and if so, several
+ * settings controlling how these logs will be written.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request; this is the bucket for which service access logging is
+ *        being requested
+ * @param targetBucketReturn must be passed in as a buffer of at least
+ *        (S3_MAX_BUCKET_NAME_SIZE + 1) bytes in length, and will be filled
+ *        in with the target bucket name for access logging for the given
+ *        bucket, which is the bucket into which access logs for the specified
+ *        bucket will be written.  This is returned as an empty string if
+ *        service access logging is not enabled for the given bucket.
+ * @param targetPrefixReturn must be passed in as a buffer of at least
+ *        (S3_MAX_KEY_SIZE + 1) bytes in length, and will be filled in
+ *        with the key prefix for server access logs for the given bucket,
+ *        or the empty string if no such prefix is specified.
+ * @param aclGrantCountReturn returns the number of ACL grants that are
+ *        associated with the server access logging for the given bucket.
+ * @param aclGrants must be passed in as an array of at least
+ *        S3_MAX_ACL_GRANT_COUNT S3AclGrant structures, and these will be
+ *        filled in with the target grants associated with the server access
+ *        logging for the given bucket, whose number is returned in the
+ *        aclGrantCountReturn parameter.  These grants will be applied to the
+ *        ACL of any server access logging log files generated by the S3
+ *        service for the given bucket.
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_get_server_access_logging(const S3BucketContext *bucketContext,
+                                  char *targetBucketReturn,
+                                  char *targetPrefixReturn,
+                                  int *aclGrantCountReturn, 
+                                  S3AclGrant *aclGrants,
+                                  S3RequestContext *requestContext,
+                                  const S3ResponseHandler *handler,
+                                  void *callbackData);
+                                  
+
+/**
+ * Sets the service access logging settings for a bucket.  The service access
+ * logging settings specify whether or not the S3 service will write service
+ * access logs for requests made for the given bucket, and if so, several
+ * settings controlling how these logs will be written.
+ *
+ * @param bucketContext gives the bucket and associated parameters for this
+ *        request; this is the bucket for which service access logging is
+ *        being set
+ * @param targetBucket gives the target bucket name for access logging for the
+ *        given bucket, which is the bucket into which access logs for the
+ *        specified bucket will be written.
+ * @param targetPrefix is an option parameter which specifies the key prefix
+ *        for server access logs for the given bucket, or NULL if no such
+ *        prefix is to be used.
+ * @param aclGrantCount specifies the number of ACL grants that are to be
+ *        associated with the server access logging for the given bucket.
+ * @param aclGrants is as an array of S3AclGrant structures, whose number is
+ *        given by the aclGrantCount parameter.  These grants will be applied
+ *        to the ACL of any server access logging log files generated by the
+ *        S3 service for the given bucket.
+ * @param requestContext if non-NULL, gives the S3RequestContext to add this
+ *        request to, and does not perform the request immediately.  If NULL,
+ *        performs the request immediately and synchronously.
+ * @param handler gives the callbacks to call as the request is processed and
+ *        completed 
+ * @param callbackData will be passed in as the callbackData parameter to
+ *        all callbacks for this request
+ **/
+void S3_set_server_access_logging(const S3BucketContext *bucketContext,
+                                  const char *targetBucket, 
+                                  const char *targetPrefix, int aclGrantCount, 
+                                  const S3AclGrant *aclGrants, 
+                                  S3RequestContext *requestContext,
+                                  const S3ResponseHandler *handler,
+                                  void *callbackData);
+                                  
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LIBS3_H */
diff --git a/ceph/src/libs3/inc/mingw/pthread.h b/ceph/src/libs3/inc/mingw/pthread.h
new file mode 100644
index 00000000..674a62a4
--- /dev/null
+++ b/ceph/src/libs3/inc/mingw/pthread.h
@@ -0,0 +1,45 @@
+/** **************************************************************************
+ * pthread.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef PTHREAD_H
+#define PTHREAD_H
+
+// This is a minimal implementation of pthreads on Windows, implementing just
+// the APIs needed by libs3
+
+unsigned long pthread_self();
+
+typedef struct
+{
+    CRITICAL_SECTION criticalSection;
+} pthread_mutex_t;
+
+int pthread_mutex_init(pthread_mutex_t *mutex, void *);
+int pthread_mutex_lock(pthread_mutex_t *mutex);
+int pthread_mutex_unlock(pthread_mutex_t *mutex);
+int pthread_mutex_destroy(pthread_mutex_t *mutex);
+
+#endif /* PTHREAD_H */
diff --git a/ceph/src/libs3/inc/mingw/sys/select.h b/ceph/src/libs3/inc/mingw/sys/select.h
new file mode 100644
index 00000000..0981da2e
--- /dev/null
+++ b/ceph/src/libs3/inc/mingw/sys/select.h
@@ -0,0 +1,30 @@
+/** **************************************************************************
+ * select.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+// This file is used only on a MingW build, and converts an include of
+// sys/select.h to its Windows equivalent
+
+#include 
diff --git a/ceph/src/libs3/inc/mingw/sys/utsname.h b/ceph/src/libs3/inc/mingw/sys/utsname.h
new file mode 100644
index 00000000..1e6b4701
--- /dev/null
+++ b/ceph/src/libs3/inc/mingw/sys/utsname.h
@@ -0,0 +1,41 @@
+/** **************************************************************************
+ * utsname.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+// This file is used only on a MingW build, and provides an implementation
+// of POSIX sys/utsname.h
+
+#ifndef UTSNAME_H
+#define UTSNAME_H
+
+struct utsname
+{
+    const char *sysname;
+    const char *machine;
+};
+
+int uname(struct utsname *);
+
+#endif /* UTSNAME_H */
diff --git a/ceph/src/libs3/inc/request.h b/ceph/src/libs3/inc/request.h
new file mode 100644
index 00000000..9e3a4770
--- /dev/null
+++ b/ceph/src/libs3/inc/request.h
@@ -0,0 +1,186 @@
+/** **************************************************************************
+ * request.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef REQUEST_H
+#define REQUEST_H
+
+#include "libs3.h"
+#include "error_parser.h"
+#include "response_headers_handler.h"
+#include "util.h"
+
+// Describes a type of HTTP request (these are our supported HTTP "verbs")
+typedef enum
+{
+    HttpRequestTypeGET,
+    HttpRequestTypeHEAD,
+    HttpRequestTypePUT,
+    HttpRequestTypeCOPY,
+    HttpRequestTypeDELETE
+} HttpRequestType;
+
+
+// This completely describes a request.  A RequestParams is not required to be
+// allocated from the heap and its lifetime is not assumed to extend beyond
+// the lifetime of the function to which it has been passed.
+typedef struct RequestParams
+{
+    // Request type, affects the HTTP verb used
+    HttpRequestType httpRequestType;
+
+    // Bucket context for request
+    S3BucketContext bucketContext;
+
+    // Key, if any
+    const char *key;
+
+    // Query params - ready to append to URI (i.e. ?p1=v1?p2=v2)
+    const char *queryParams;
+
+    // sub resource, like ?acl, ?location, ?torrent, ?logging
+    const char *subResource;
+
+    // If this is a copy operation, this gives the source bucket
+    const char *copySourceBucketName;
+
+    // If this is a copy operation, this gives the source key
+    const char *copySourceKey;
+
+    // Get conditions
+    const S3GetConditions *getConditions;
+
+    // Start byte
+    uint64_t startByte;
+
+    // Byte count
+    uint64_t byteCount;
+
+    // Put properties
+    const S3PutProperties *putProperties;
+
+    // Callback to be made when headers are available.  Might not be called.
+    S3ResponsePropertiesCallback *propertiesCallback;
+
+    // Callback to be made to supply data to send to S3.  Might not be called.
+    S3PutObjectDataCallback *toS3Callback;
+
+    // Number of bytes total that readCallback will supply
+    int64_t toS3CallbackTotalSize;
+
+    // Callback to be made that supplies data read from S3.
+    // Might not be called.
+    S3GetObjectDataCallback *fromS3Callback;
+
+    // Callback to be made when request is complete.  This will *always* be
+    // called.
+    S3ResponseCompleteCallback *completeCallback;
+
+    // Data passed to the callbacks
+    void *callbackData;
+} RequestParams;
+
+
+// This is the stuff associated with a request that needs to be on the heap
+// (and thus live while a curl_multi is in use).
+typedef struct Request
+{
+    // These put the request on a doubly-linked list of requests in a
+    // request context, *if* the request is in a request context (else these
+    // will both be 0)
+    struct Request *prev, *next;
+
+    // The status of this Request, as will be reported to the user via the
+    // complete callback
+    S3Status status;
+
+    // The HTTP code returned by the S3 server, if it is known.  Would rather
+    // not have to keep track of this but S3 doesn't always indicate its
+    // errors the same way
+    int httpResponseCode;
+
+    // The HTTP headers to use for the curl request
+    struct curl_slist *headers;
+
+    // The CURL structure driving the request
+    CURL *curl;
+
+    // libcurl requires that the uri be stored outside of the curl handle
+    char uri[MAX_URI_SIZE + 1];
+
+    // Callback to be made when headers are available.  Might not be called.
+    S3ResponsePropertiesCallback *propertiesCallback;
+
+    // Callback to be made to supply data to send to S3.  Might not be called.
+    S3PutObjectDataCallback *toS3Callback;
+
+    // Number of bytes total that readCallback has left to supply
+    int64_t toS3CallbackBytesRemaining;
+
+    // Callback to be made that supplies data read from S3.
+    // Might not be called.
+    S3GetObjectDataCallback *fromS3Callback;
+
+    // Callback to be made when request is complete.  This will *always* be
+    // called.
+    S3ResponseCompleteCallback *completeCallback;
+
+    // Data passed to the callbacks
+    void *callbackData;
+
+    // Handler of response headers
+    ResponseHeadersHandler responseHeadersHandler;
+
+    // This is set to nonzero after the properties callback has been made
+    int propertiesCallbackMade;
+
+    // Parser of errors
+    ErrorParser errorParser;
+} Request;
+
+
+// Request functions
+// ----------------------------------------------------------------------------
+
+// Initialize the API
+S3Status request_api_initialize(const char *userAgentInfo, int flags,
+                                const char *hostName);
+
+// Deinitialize the API
+void request_api_deinitialize();
+
+// Perform a request; if context is 0, performs the request immediately;
+// otherwise, sets it up to be performed by context.
+void request_perform(const RequestParams *params, S3RequestContext *context);
+
+// Called by the internal request code or internal request context code when a
+// curl has finished the request
+void request_finish(Request *request);
+
+// Convert a CURLE code to an S3Status
+S3Status request_curl_code_to_status(CURLcode code);
+
+
+#endif /* REQUEST_H */
diff --git a/ceph/src/libs3/inc/request_context.h b/ceph/src/libs3/inc/request_context.h
new file mode 100644
index 00000000..8074c503
--- /dev/null
+++ b/ceph/src/libs3/inc/request_context.h
@@ -0,0 +1,40 @@
+/** **************************************************************************
+ * request_context.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef REQUEST_CONTEXT_H
+#define REQUEST_CONTEXT_H
+
+#include "libs3.h"
+
+struct S3RequestContext
+{
+    CURLM *curlm;
+
+    struct Request *requests;
+};
+
+
+#endif /* REQUEST_CONTEXT_H */
diff --git a/ceph/src/libs3/inc/response_headers_handler.h b/ceph/src/libs3/inc/response_headers_handler.h
new file mode 100644
index 00000000..2813e9aa
--- /dev/null
+++ b/ceph/src/libs3/inc/response_headers_handler.h
@@ -0,0 +1,64 @@
+/** **************************************************************************
+ * response_headers_handler.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef RESPONSE_HEADERS_HANDLER_H
+#define RESPONSE_HEADERS_HANDLER_H
+
+#include "libs3.h"
+#include "string_buffer.h"
+#include "util.h"
+
+
+typedef struct ResponseHeadersHandler
+{
+    // The structure to pass to the headers callback.  This is filled in by
+    // the ResponseHeadersHandler from the headers added to it.
+    S3ResponseProperties responseProperties;
+
+    // Set to 1 after the done call has been made
+    int done;
+
+    // copied into here.  We allow 128 bytes for each header, plus \0 term.
+    string_multibuffer(responsePropertyStrings, 5 * 129);
+
+    // responseproperties.metaHeaders strings get copied into here
+    string_multibuffer(responseMetaDataStrings, 
+                       COMPACTED_METADATA_BUFFER_SIZE);
+
+    // Response meta data
+    S3NameValue responseMetaData[S3_MAX_METADATA_COUNT];
+} ResponseHeadersHandler;
+
+
+void response_headers_handler_initialize(ResponseHeadersHandler *handler);
+
+void response_headers_handler_add(ResponseHeadersHandler *handler,
+                                  char *data, int dataLen);
+
+void response_headers_handler_done(ResponseHeadersHandler *handler, 
+                                   CURL *curl);
+
+#endif /* RESPONSE_HEADERS_HANDLER_H */
diff --git a/ceph/src/libs3/inc/simplexml.h b/ceph/src/libs3/inc/simplexml.h
new file mode 100644
index 00000000..704db070
--- /dev/null
+++ b/ceph/src/libs3/inc/simplexml.h
@@ -0,0 +1,76 @@
+/** **************************************************************************
+ * simplexml.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef SIMPLEXML_H
+#define SIMPLEXML_H
+
+#include "libs3.h"
+
+
+// Simple XML callback.
+//
+// elementPath: is the full "path" of the element; i.e.
+// data would have 'data' in the element
+// foo/bar/baz.
+// 
+// Return of anything other than S3StatusOK causes the calling
+// simplexml_add() function to immediately stop and return the status.
+//
+// data is passed in as 0 on end of element
+typedef S3Status (SimpleXmlCallback)(const char *elementPath, const char *data,
+                                     int dataLen, void *callbackData);
+
+typedef struct SimpleXml
+{
+    void *xmlParser;
+
+    SimpleXmlCallback *callback;
+
+    void *callbackData;
+
+    char elementPath[512];
+
+    int elementPathLen;
+
+    S3Status status;
+} SimpleXml;
+
+
+// Simple XML parsing
+// ----------------------------------------------------------------------------
+
+// Always call this, even if the simplexml doesn't end up being used
+void simplexml_initialize(SimpleXml *simpleXml, SimpleXmlCallback *callback,
+                          void *callbackData);
+
+S3Status simplexml_add(SimpleXml *simpleXml, const char *data, int dataLen);
+
+
+// Always call this
+void simplexml_deinitialize(SimpleXml *simpleXml);
+
+
+#endif /* SIMPLEXML_H */
diff --git a/ceph/src/libs3/inc/string_buffer.h b/ceph/src/libs3/inc/string_buffer.h
new file mode 100644
index 00000000..eed9bd42
--- /dev/null
+++ b/ceph/src/libs3/inc/string_buffer.h
@@ -0,0 +1,107 @@
+/** **************************************************************************
+ * string_buffer.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef STRING_BUFFER_H
+#define STRING_BUFFER_H
+
+#include 
+
+
+// Declare a string_buffer with the given name of the given maximum length
+#define string_buffer(name, len)                                        \
+    char name[len + 1];                                                 \
+    int name##Len
+
+
+// Initialize a string_buffer
+#define string_buffer_initialize(sb)                                    \
+    do {                                                                \
+        sb[0] = 0;                                                      \
+        sb##Len = 0;                                                    \
+    } while (0)
+
+
+// Append [len] bytes of [str] to [sb], setting [all_fit] to 1 if it fit, and
+// 0 if it did not
+#define string_buffer_append(sb, str, len, all_fit)                     \
+    do {                                                                \
+        sb##Len += snprintf(&(sb[sb##Len]), sizeof(sb) - sb##Len - 1,   \
+                            "%.*s", (int) (len), str);                  \
+        if (sb##Len > (int) (sizeof(sb) - 1)) {                         \
+            sb##Len = sizeof(sb) - 1;                                   \
+            all_fit = 0;                                                \
+        }                                                               \
+        else {                                                          \
+            all_fit = 1;                                                \
+        }                                                               \
+    } while (0)
+
+
+// Declare a string multibuffer with the given name of the given maximum size
+#define string_multibuffer(name, size)                                  \
+    char name[size];                                                    \
+    int name##Size
+
+
+// Initialize a string_multibuffer
+#define string_multibuffer_initialize(smb)                              \
+    do {                                                                \
+        smb##Size = 0;                                                  \
+    } while (0)
+
+
+// Evaluates to the current string within the string_multibuffer
+#define string_multibuffer_current(smb)                                  \
+    &(smb[smb##Size])
+
+
+// Adds a new string to the string_multibuffer
+#define string_multibuffer_add(smb, str, len, all_fit)                  \
+    do {                                                                \
+        smb##Size += (snprintf(&(smb[smb##Size]),                       \
+                               sizeof(smb) - smb##Size,                 \
+                               "%.*s", (int) (len), str) + 1);          \
+        if (smb##Size > (int) sizeof(smb)) {                            \
+            smb##Size = sizeof(smb);                                    \
+            all_fit = 0;                                                \
+        }                                                               \
+        else {                                                          \
+            all_fit = 1;                                                \
+        }                                                               \
+    } while (0)
+
+
+// Appends to the current string in the string_multibuffer.  There must be a
+// current string, meaning that string_multibuffer_add must have been called
+// at least once for this string_multibuffer.
+#define string_multibuffer_append(smb, str, len, all_fit)               \
+    do {                                                                \
+        smb##Size--;                                                    \
+        string_multibuffer_add(smb, str, len, all_fit);                 \
+    } while (0)
+
+
+#endif /* STRING_BUFFER_H */
diff --git a/ceph/src/libs3/inc/util.h b/ceph/src/libs3/inc/util.h
new file mode 100644
index 00000000..4138ca78
--- /dev/null
+++ b/ceph/src/libs3/inc/util.h
@@ -0,0 +1,98 @@
+/** **************************************************************************
+ * util.h
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#ifndef UTIL_H
+#define UTIL_H
+
+#include 
+#include 
+#include 
+#include "libs3.h"
+
+// acl groups
+#define ACS_URL "http://acs.amazonaws.com/groups/"
+
+#define ACS_GROUP_ALL_USERS     ACS_URL "global/AllUsers"
+#define ACS_GROUP_AWS_USERS     ACS_URL "global/AuthenticatedUsers"
+#define ACS_GROUP_LOG_DELIVERY  ACS_URL "s3/LogDelivery"
+
+
+
+// Derived from S3 documentation
+
+// This is the maximum number of bytes needed in a "compacted meta header"
+// buffer, which is a buffer storing all of the compacted meta headers.
+#define COMPACTED_METADATA_BUFFER_SIZE \
+    (S3_MAX_METADATA_COUNT * sizeof(S3_METADATA_HEADER_NAME_PREFIX "n: v"))
+
+// Maximum url encoded key size; since every single character could require
+// URL encoding, it's 3 times the size of a key (since each url encoded
+// character takes 3 characters: %NN)
+#define MAX_URLENCODED_KEY_SIZE (3 * S3_MAX_KEY_SIZE)
+
+// This is the maximum size of a URI that could be passed to S3:
+// https://s3.amazonaws.com/${BUCKET}/${KEY}?acl
+// 255 is the maximum bucket length
+#define MAX_URI_SIZE \
+    ((sizeof("https:///") - 1) + S3_MAX_HOSTNAME_SIZE + 255 + 1 +       \
+     MAX_URLENCODED_KEY_SIZE + (sizeof("?torrent" - 1)) + 1)
+
+// Maximum size of a canonicalized resource
+#define MAX_CANONICALIZED_RESOURCE_SIZE \
+    (1 + 255 + 1 + MAX_URLENCODED_KEY_SIZE + (sizeof("?torrent") - 1) + 1)
+
+
+// Utilities -----------------------------------------------------------------
+
+// URL-encodes a string from [src] into [dest].  [dest] must have at least
+// 3x the number of characters that [source] has.   At most [maxSrcSize] bytes
+// from [src] are encoded; if more are present in [src], 0 is returned from
+// urlEncode, else nonzero is returned.
+int urlEncode(char *dest, const char *src, int maxSrcSize);
+
+// Returns < 0 on failure >= 0 on success
+int64_t parseIso8601Time(const char *str);
+
+uint64_t parseUnsignedInt(const char *str);
+
+// base64 encode bytes.  The output buffer must have at least
+// ((4 * (inLen + 1)) / 3) bytes in it.  Returns the number of bytes written
+// to [out].
+int base64Encode(const unsigned char *in, int inLen, char *out);
+
+// Compute HMAC-SHA-1 with key [key] and message [message], storing result
+// in [hmac]
+void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len,
+               const unsigned char *message, int message_len);
+
+// Compute a 64-bit hash values given a set of bytes
+uint64_t hash(const unsigned char *k, int length);
+
+// Because Windows seems to be missing isblank(), use our own; it's a very
+// easy function to write in any case
+int is_blank(char c);
+
+#endif /* UTIL_H */
diff --git a/ceph/src/libs3/libs3.spec b/ceph/src/libs3/libs3.spec
new file mode 100644
index 00000000..95c36785
--- /dev/null
+++ b/ceph/src/libs3/libs3.spec
@@ -0,0 +1,81 @@
+Summary: C Library and Tools for Amazon S3 Access
+Name: libs3
+Version: trunk
+Release: 1
+License: GPL
+Group: Networking/Utilities
+URL: http://sourceforge.net/projects/reallibs3
+Source0: libs3-trunk.tar.gz
+Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root
+# Want to include curl dependencies, but older Fedora Core uses curl-devel,
+# and newer Fedora Core uses libcurl-devel ... have to figure out how to
+# handle this problem, but for now, just don't check for any curl libraries
+# Buildrequires: curl-devel
+Buildrequires: libxml2-devel
+Buildrequires: openssl-devel
+Buildrequires: make
+# Requires: libcurl
+Requires: libxml2
+Requires: openssl
+
+%define debug_package %{nil}
+
+%description
+This package includes the libs3 shared object library, needed to run
+applications compiled against libs3, and additionally contains the s3
+utility for accessing Amazon S3.
+
+%package devel
+Summary: Headers and documentation for libs3
+Group: Development/Libraries
+Requires: %{name} = %{version}-%{release}
+
+%description devel
+This library provides an API for using Amazon's S3 service (see
+http://s3.amazonaws.com).  Its design goals are:
+
+ - To provide a simple and straightforward API for accessing all of S3's
+   functionality
+ - To not require the developer using libs3 to need to know anything about:
+     - HTTP
+     - XML
+     - SSL
+   In other words, this API is meant to stand on its own, without requiring
+   any implicit knowledge of how S3 services are accessed using HTTP
+   protocols.
+ - To be usable from multithreaded code
+ - To be usable by code which wants to process multiple S3 requests
+   simultaneously from a single thread
+ - To be usable in the simple, straightforward way using sequentialized
+   blocking requests
+
+
+%prep
+%setup -q
+
+%build
+BUILD=$RPM_BUILD_ROOT/build make exported
+
+%install
+BUILD=$RPM_BUILD_ROOT/build DESTDIR=$RPM_BUILD_ROOT/usr make install
+rm -rf $RPM_BUILD_ROOT/build
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%defattr(-,root,root,-)
+/usr/bin/s3
+/usr/lib/libs3.so*
+
+%files devel
+%defattr(-,root,root,-)
+/usr/include/libs3.h
+/usr/lib/libs3.a
+
+%changelog
+* Sat Aug 09 2008   Bryan Ischo
+- Split into regular and devel packages.
+
+* Tue Aug 05 2008   Bryan Ischo
+- Initial build.
diff --git a/ceph/src/libs3/mswin/libs3.def b/ceph/src/libs3/mswin/libs3.def
new file mode 100644
index 00000000..c5bd6d86
--- /dev/null
+++ b/ceph/src/libs3/mswin/libs3.def
@@ -0,0 +1,27 @@
+EXPORTS
+S3_convert_acl
+S3_copy_object
+S3_create_bucket
+S3_create_request_context
+S3_deinitialize
+S3_delete_bucket
+S3_delete_object
+S3_destroy_request_context
+S3_generate_authenticated_query_string
+S3_get_acl
+S3_get_object
+S3_get_request_context_fdsets
+S3_get_server_access_logging
+S3_get_status_name
+S3_head_object
+S3_initialize
+S3_list_bucket
+S3_list_service
+S3_put_object
+S3_runall_request_context
+S3_runonce_request_context
+S3_set_acl
+S3_set_server_access_logging
+S3_status_is_retryable
+S3_test_bucket
+S3_validate_bucket_name
diff --git a/ceph/src/libs3/mswin/rmrf.bat b/ceph/src/libs3/mswin/rmrf.bat
new file mode 100644
index 00000000..204efd9e
--- /dev/null
+++ b/ceph/src/libs3/mswin/rmrf.bat
@@ -0,0 +1,9 @@
+@echo off
+
+if exist "%1". (
+   rmdir /S /Q "%1"
+)
+
+if exist "%1". (
+   del /Q "%1"
+)
diff --git a/ceph/src/libs3/src/acl.c b/ceph/src/libs3/src/acl.c
new file mode 100644
index 00000000..25e4058b
--- /dev/null
+++ b/ceph/src/libs3/src/acl.c
@@ -0,0 +1,348 @@
+/** **************************************************************************
+ * acl.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "libs3.h"
+#include "request.h"
+
+// Use a rather arbitrary max size for the document of 64K
+#define ACL_XML_DOC_MAXSIZE (64 * 1024)
+
+
+// get acl -------------------------------------------------------------------
+
+typedef struct GetAclData
+{
+    SimpleXml simpleXml;
+
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    int *aclGrantCountReturn;
+    S3AclGrant *aclGrants;
+    char *ownerId;
+    char *ownerDisplayName;
+    string_buffer(aclXmlDocument, ACL_XML_DOC_MAXSIZE);
+} GetAclData;
+
+
+static S3Status getAclPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    GetAclData *gaData = (GetAclData *) callbackData;
+    
+    return (*(gaData->responsePropertiesCallback))
+        (responseProperties, gaData->callbackData);
+}
+
+
+static S3Status getAclDataCallback(int bufferSize, const char *buffer,
+                                   void *callbackData)
+{
+    GetAclData *gaData = (GetAclData *) callbackData;
+
+    int fit;
+
+    string_buffer_append(gaData->aclXmlDocument, buffer, bufferSize, fit);
+    
+    return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge;
+}
+
+
+static void getAclCompleteCallback(S3Status requestStatus, 
+                                   const S3ErrorDetails *s3ErrorDetails,
+                                   void *callbackData)
+{
+    GetAclData *gaData = (GetAclData *) callbackData;
+
+    if (requestStatus == S3StatusOK) {
+        // Parse the document
+        requestStatus = S3_convert_acl
+            (gaData->aclXmlDocument, gaData->ownerId, gaData->ownerDisplayName,
+             gaData->aclGrantCountReturn, gaData->aclGrants);
+    }
+
+    (*(gaData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, gaData->callbackData);
+
+    free(gaData);
+}
+
+
+void S3_get_acl(const S3BucketContext *bucketContext, const char *key, 
+                char *ownerId, char *ownerDisplayName,
+                int *aclGrantCountReturn, S3AclGrant *aclGrants, 
+                S3RequestContext *requestContext,
+                const S3ResponseHandler *handler, void *callbackData)
+{
+    // Create the callback data
+    GetAclData *gaData = (GetAclData *) malloc(sizeof(GetAclData));
+    if (!gaData) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    gaData->responsePropertiesCallback = handler->propertiesCallback;
+    gaData->responseCompleteCallback = handler->completeCallback;
+    gaData->callbackData = callbackData;
+
+    gaData->aclGrantCountReturn = aclGrantCountReturn;
+    gaData->aclGrants = aclGrants;
+    gaData->ownerId = ownerId;
+    gaData->ownerDisplayName = ownerDisplayName;
+    string_buffer_initialize(gaData->aclXmlDocument);
+    *aclGrantCountReturn = 0;
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeGET,                           // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        key,                                          // key
+        0,                                            // queryParams
+        "acl",                                        // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        &getAclPropertiesCallback,                    // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        &getAclDataCallback,                          // fromS3Callback
+        &getAclCompleteCallback,                      // completeCallback
+        gaData                                        // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
+// set acl -------------------------------------------------------------------
+
+static S3Status generateAclXmlDocument(const char *ownerId, 
+                                       const char *ownerDisplayName,
+                                       int aclGrantCount, 
+                                       const S3AclGrant *aclGrants,
+                                       int *xmlDocumentLenReturn,
+                                       char *xmlDocument,
+                                       int xmlDocumentBufferSize)
+{
+    *xmlDocumentLenReturn = 0;
+
+#define append(fmt, ...)                                        \
+    do {                                                        \
+        *xmlDocumentLenReturn += snprintf                       \
+            (&(xmlDocument[*xmlDocumentLenReturn]),             \
+             xmlDocumentBufferSize - *xmlDocumentLenReturn - 1, \
+             fmt, __VA_ARGS__);                                 \
+        if (*xmlDocumentLenReturn >= xmlDocumentBufferSize) {   \
+            return S3StatusXmlDocumentTooLarge;                 \
+        } \
+    } while (0)
+
+    append("%s%s"
+           "", ownerId,
+           ownerDisplayName);
+
+    int i;
+    for (i = 0; i < aclGrantCount; i++) {
+        append("%s", "granteeType) {
+        case S3GranteeTypeAmazonCustomerByEmail:
+            append("AmazonCustomerByEmail\">%s",
+                   grant->grantee.amazonCustomerByEmail.emailAddress);
+            break;
+        case S3GranteeTypeCanonicalUser:
+            append("CanonicalUser\">%s%s",
+                   grant->grantee.canonicalUser.id, 
+                   grant->grantee.canonicalUser.displayName);
+            break;
+        default: { // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers:
+            const char *grantee;
+            switch (grant->granteeType) {
+            case S3GranteeTypeAllAwsUsers:
+                grantee = ACS_GROUP_AWS_USERS;
+                break;
+            case S3GranteeTypeAllUsers:
+                grantee = ACS_GROUP_ALL_USERS;
+                break;
+            default:
+                grantee = ACS_GROUP_LOG_DELIVERY;
+                break;
+            }
+            append("Group\">%s", grantee);
+        }
+            break;
+        }
+        append("%s",
+               ((grant->permission == S3PermissionRead) ? "READ" :
+                (grant->permission == S3PermissionWrite) ? "WRITE" :
+                (grant->permission == S3PermissionReadACP) ? "READ_ACP" :
+                (grant->permission == S3PermissionWriteACP) ? "WRITE_ACP" :
+                "FULL_CONTROL"));
+    }
+
+    append("%s", "");
+
+    return S3StatusOK;
+}
+
+
+typedef struct SetAclData
+{
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    int aclXmlDocumentLen;
+    char aclXmlDocument[ACL_XML_DOC_MAXSIZE];
+    int aclXmlDocumentBytesWritten;
+
+} SetAclData;
+
+
+static S3Status setAclPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    SetAclData *paData = (SetAclData *) callbackData;
+    
+    return (*(paData->responsePropertiesCallback))
+        (responseProperties, paData->callbackData);
+}
+
+
+static int setAclDataCallback(int bufferSize, char *buffer, void *callbackData)
+{
+    SetAclData *paData = (SetAclData *) callbackData;
+
+    int remaining = (paData->aclXmlDocumentLen - 
+                     paData->aclXmlDocumentBytesWritten);
+
+    int toCopy = bufferSize > remaining ? remaining : bufferSize;
+    
+    if (!toCopy) {
+        return 0;
+    }
+
+    memcpy(buffer, &(paData->aclXmlDocument
+                     [paData->aclXmlDocumentBytesWritten]), toCopy);
+
+    paData->aclXmlDocumentBytesWritten += toCopy;
+
+    return toCopy;
+}
+
+
+static void setAclCompleteCallback(S3Status requestStatus, 
+                                   const S3ErrorDetails *s3ErrorDetails,
+                                   void *callbackData)
+{
+    SetAclData *paData = (SetAclData *) callbackData;
+
+    (*(paData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, paData->callbackData);
+
+    free(paData);
+}
+
+
+void S3_set_acl(const S3BucketContext *bucketContext, const char *key,
+                const char *ownerId, const char *ownerDisplayName,
+                int aclGrantCount, const S3AclGrant *aclGrants,
+                S3RequestContext *requestContext,
+                const S3ResponseHandler *handler, void *callbackData)
+{
+    if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) {
+        (*(handler->completeCallback))
+            (S3StatusTooManyGrants, 0, callbackData);
+        return;
+    }
+
+    SetAclData *data = (SetAclData *) malloc(sizeof(SetAclData));
+    if (!data) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+    
+    // Convert aclGrants to XML document
+    S3Status status = generateAclXmlDocument
+        (ownerId, ownerDisplayName, aclGrantCount, aclGrants,
+         &(data->aclXmlDocumentLen), data->aclXmlDocument, 
+         sizeof(data->aclXmlDocument));
+    if (status != S3StatusOK) {
+        free(data);
+        (*(handler->completeCallback))(status, 0, callbackData);
+        return;
+    }
+
+    data->responsePropertiesCallback = handler->propertiesCallback;
+    data->responseCompleteCallback = handler->completeCallback;
+    data->callbackData = callbackData;
+
+    data->aclXmlDocumentBytesWritten = 0;
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypePUT,                           // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        key,                                          // key
+        0,                                            // queryParams
+        "acl",                                        // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        &setAclPropertiesCallback,                    // propertiesCallback
+        &setAclDataCallback,                          // toS3Callback
+        data->aclXmlDocumentLen,                      // toS3CallbackTotalSize
+        0,                                            // fromS3Callback
+        &setAclCompleteCallback,                      // completeCallback
+        data                                          // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
diff --git a/ceph/src/libs3/src/bucket.c b/ceph/src/libs3/src/bucket.c
new file mode 100644
index 00000000..f4f19877
--- /dev/null
+++ b/ceph/src/libs3/src/bucket.c
@@ -0,0 +1,743 @@
+/** **************************************************************************
+ * bucket.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "libs3.h"
+#include "request.h"
+#include "simplexml.h"
+
+// test bucket ---------------------------------------------------------------
+
+typedef struct TestBucketData
+{
+    SimpleXml simpleXml;
+
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    int locationConstraintReturnSize;
+    char *locationConstraintReturn;
+
+    string_buffer(locationConstraint, 256);
+} TestBucketData;
+
+
+static S3Status testBucketXmlCallback(const char *elementPath,
+                                      const char *data, int dataLen,
+                                      void *callbackData)
+{
+    TestBucketData *tbData = (TestBucketData *) callbackData;
+
+    int fit;
+
+    if (data && !strcmp(elementPath, "LocationConstraint")) {
+        string_buffer_append(tbData->locationConstraint, data, dataLen, fit);
+    }
+
+    /* Avoid compiler error about variable set but not used */
+    (void) fit;
+
+    return S3StatusOK;
+}
+
+
+static S3Status testBucketPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    TestBucketData *tbData = (TestBucketData *) callbackData;
+    
+    return (*(tbData->responsePropertiesCallback))
+        (responseProperties, tbData->callbackData);
+}
+
+
+static S3Status testBucketDataCallback(int bufferSize, const char *buffer,
+                                       void *callbackData)
+{
+    TestBucketData *tbData = (TestBucketData *) callbackData;
+
+    return simplexml_add(&(tbData->simpleXml), buffer, bufferSize);
+}
+
+
+static void testBucketCompleteCallback(S3Status requestStatus, 
+                                       const S3ErrorDetails *s3ErrorDetails,
+                                       void *callbackData)
+{
+    TestBucketData *tbData = (TestBucketData *) callbackData;
+
+    // Copy the location constraint into the return buffer
+    snprintf(tbData->locationConstraintReturn, 
+             tbData->locationConstraintReturnSize, "%s", 
+             tbData->locationConstraint);
+
+    (*(tbData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, tbData->callbackData);
+
+    simplexml_deinitialize(&(tbData->simpleXml));
+
+    free(tbData);
+}
+
+
+void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle,
+                    const char *accessKeyId, const char *secretAccessKey,
+                    const char *hostName, const char *bucketName,
+                    int locationConstraintReturnSize,
+                    char *locationConstraintReturn,
+                    S3RequestContext *requestContext,
+                    const S3ResponseHandler *handler, void *callbackData)
+{
+    // Create the callback data
+    TestBucketData *tbData = 
+        (TestBucketData *) malloc(sizeof(TestBucketData));
+    if (!tbData) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    simplexml_initialize(&(tbData->simpleXml), &testBucketXmlCallback, tbData);
+
+    tbData->responsePropertiesCallback = handler->propertiesCallback;
+    tbData->responseCompleteCallback = handler->completeCallback;
+    tbData->callbackData = callbackData;
+
+    tbData->locationConstraintReturnSize = locationConstraintReturnSize;
+    tbData->locationConstraintReturn = locationConstraintReturn;
+    string_buffer_initialize(tbData->locationConstraint);
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeGET,                           // httpRequestType
+        { hostName,                                   // hostName
+          bucketName,                                 // bucketName
+          protocol,                                   // protocol
+          uriStyle,                                   // uriStyle
+          accessKeyId,                                // accessKeyId
+          secretAccessKey },                          // secretAccessKey
+        0,                                            // key
+        0,                                            // queryParams
+        "location",                                   // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        &testBucketPropertiesCallback,                // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        &testBucketDataCallback,                      // fromS3Callback
+        &testBucketCompleteCallback,                  // completeCallback
+        tbData                                        // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
+// create bucket -------------------------------------------------------------
+
+typedef struct CreateBucketData
+{
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    char doc[1024];
+    int docLen, docBytesWritten;
+} CreateBucketData;                         
+                            
+
+static S3Status createBucketPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    CreateBucketData *cbData = (CreateBucketData *) callbackData;
+    
+    return (*(cbData->responsePropertiesCallback))
+        (responseProperties, cbData->callbackData);
+}
+
+
+static int createBucketDataCallback(int bufferSize, char *buffer, 
+                                    void *callbackData)
+{
+    CreateBucketData *cbData = (CreateBucketData *) callbackData;
+
+    if (!cbData->docLen) {
+        return 0;
+    }
+
+    int remaining = (cbData->docLen - cbData->docBytesWritten);
+
+    int toCopy = bufferSize > remaining ? remaining : bufferSize;
+    
+    if (!toCopy) {
+        return 0;
+    }
+
+    memcpy(buffer, &(cbData->doc[cbData->docBytesWritten]), toCopy);
+
+    cbData->docBytesWritten += toCopy;
+
+    return toCopy;
+}
+
+
+static void createBucketCompleteCallback(S3Status requestStatus, 
+                                         const S3ErrorDetails *s3ErrorDetails,
+                                         void *callbackData)
+{
+    CreateBucketData *cbData = (CreateBucketData *) callbackData;
+
+    (*(cbData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, cbData->callbackData);
+
+    free(cbData);
+}
+
+
+void S3_create_bucket(S3Protocol protocol, const char *accessKeyId,
+                      const char *secretAccessKey, const char *hostName,
+                      const char *bucketName, S3CannedAcl cannedAcl,
+                      const char *locationConstraint,
+                      S3RequestContext *requestContext,
+                      const S3ResponseHandler *handler, void *callbackData)
+{
+    // Create the callback data
+    CreateBucketData *cbData = 
+        (CreateBucketData *) malloc(sizeof(CreateBucketData));
+    if (!cbData) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    cbData->responsePropertiesCallback = handler->propertiesCallback;
+    cbData->responseCompleteCallback = handler->completeCallback;
+    cbData->callbackData = callbackData;
+
+    if (locationConstraint) {
+        cbData->docLen =
+            snprintf(cbData->doc, sizeof(cbData->doc),
+                     ""
+                     "%s",
+                     locationConstraint);
+        cbData->docBytesWritten = 0;
+    }
+    else {
+        cbData->docLen = 0;
+    }
+    
+    // Set up S3PutProperties
+    S3PutProperties properties =
+    {
+        0,                                       // contentType
+        0,                                       // md5
+        0,                                       // cacheControl
+        0,                                       // contentDispositionFilename
+        0,                                       // contentEncoding
+        0,                                       // expires
+        cannedAcl,                               // cannedAcl
+        0,                                       // metaDataCount
+        0                                        // metaData
+    };
+    
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypePUT,                           // httpRequestType
+        { hostName,                                   // hostName
+          bucketName,                                 // bucketName
+          protocol,                                   // protocol
+          S3UriStylePath,                             // uriStyle
+          accessKeyId,                                // accessKeyId
+          secretAccessKey },                          // secretAccessKey
+        0,                                            // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        &properties,                                  // putProperties
+        &createBucketPropertiesCallback,              // propertiesCallback
+        &createBucketDataCallback,                    // toS3Callback
+        cbData->docLen,                               // toS3CallbackTotalSize
+        0,                                            // fromS3Callback
+        &createBucketCompleteCallback,                // completeCallback
+        cbData                                        // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+                           
+// delete bucket -------------------------------------------------------------
+
+typedef struct DeleteBucketData
+{
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+} DeleteBucketData;
+
+
+static S3Status deleteBucketPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    DeleteBucketData *dbData = (DeleteBucketData *) callbackData;
+    
+    return (*(dbData->responsePropertiesCallback))
+        (responseProperties, dbData->callbackData);
+}
+
+
+static void deleteBucketCompleteCallback(S3Status requestStatus, 
+                                         const S3ErrorDetails *s3ErrorDetails,
+                                         void *callbackData)
+{
+    DeleteBucketData *dbData = (DeleteBucketData *) callbackData;
+
+    (*(dbData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, dbData->callbackData);
+
+    free(dbData);
+}
+
+
+void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle,
+                      const char *accessKeyId, const char *secretAccessKey,
+                      const char *hostName, const char *bucketName,
+                      S3RequestContext *requestContext,
+                      const S3ResponseHandler *handler, void *callbackData)
+{
+    // Create the callback data
+    DeleteBucketData *dbData = 
+        (DeleteBucketData *) malloc(sizeof(DeleteBucketData));
+    if (!dbData) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    dbData->responsePropertiesCallback = handler->propertiesCallback;
+    dbData->responseCompleteCallback = handler->completeCallback;
+    dbData->callbackData = callbackData;
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeDELETE,                        // httpRequestType
+        { hostName,                                   // hostName
+          bucketName,                                 // bucketName
+          protocol,                                   // protocol
+          uriStyle,                                   // uriStyle
+          accessKeyId,                                // accessKeyId
+          secretAccessKey },                          // secretAccessKey
+        0,                                            // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        &deleteBucketPropertiesCallback,              // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        0,                                            // fromS3Callback
+        &deleteBucketCompleteCallback,                // completeCallback
+        dbData                                        // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
+// list bucket ----------------------------------------------------------------
+
+typedef struct ListBucketContents
+{
+    string_buffer(key, 1024);
+    string_buffer(lastModified, 256);
+    string_buffer(eTag, 256);
+    string_buffer(size, 24);
+    string_buffer(ownerId, 256);
+    string_buffer(ownerDisplayName, 256);
+} ListBucketContents;
+
+
+static void initialize_list_bucket_contents(ListBucketContents *contents)
+{
+    string_buffer_initialize(contents->key);
+    string_buffer_initialize(contents->lastModified);
+    string_buffer_initialize(contents->eTag);
+    string_buffer_initialize(contents->size);
+    string_buffer_initialize(contents->ownerId);
+    string_buffer_initialize(contents->ownerDisplayName);
+}
+
+// We read up to 32 Contents at a time
+#define MAX_CONTENTS 32
+// We read up to 8 CommonPrefixes at a time
+#define MAX_COMMON_PREFIXES 8
+
+typedef struct ListBucketData
+{
+    SimpleXml simpleXml;
+
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ListBucketCallback *listBucketCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    string_buffer(isTruncated, 64);
+    string_buffer(nextMarker, 1024);
+
+    int contentsCount;
+    ListBucketContents contents[MAX_CONTENTS];
+
+    int commonPrefixesCount;
+    char commonPrefixes[MAX_COMMON_PREFIXES][1024];
+    int commonPrefixLens[MAX_COMMON_PREFIXES];
+} ListBucketData;
+
+
+static void initialize_list_bucket_data(ListBucketData *lbData)
+{
+    lbData->contentsCount = 0;
+    initialize_list_bucket_contents(lbData->contents);
+    lbData->commonPrefixesCount = 0;
+    lbData->commonPrefixes[0][0] = 0;
+    lbData->commonPrefixLens[0] = 0;
+}
+
+
+static S3Status make_list_bucket_callback(ListBucketData *lbData)
+{
+    int i;
+
+    // Convert IsTruncated
+    int isTruncated = (!strcmp(lbData->isTruncated, "true") ||
+                       !strcmp(lbData->isTruncated, "1")) ? 1 : 0;
+
+    // Convert the contents
+    S3ListBucketContent contents[lbData->contentsCount];
+
+    int contentsCount = lbData->contentsCount;
+    for (i = 0; i < contentsCount; i++) {
+        S3ListBucketContent *contentDest = &(contents[i]);
+        ListBucketContents *contentSrc = &(lbData->contents[i]);
+        contentDest->key = contentSrc->key;
+        contentDest->lastModified = 
+            parseIso8601Time(contentSrc->lastModified);
+        contentDest->eTag = contentSrc->eTag;
+        contentDest->size = parseUnsignedInt(contentSrc->size);
+        contentDest->ownerId =
+            contentSrc->ownerId[0] ?contentSrc->ownerId : 0;
+        contentDest->ownerDisplayName = (contentSrc->ownerDisplayName[0] ?
+                                         contentSrc->ownerDisplayName : 0);
+    }
+
+    // Make the common prefixes array
+    int commonPrefixesCount = lbData->commonPrefixesCount;
+    char *commonPrefixes[commonPrefixesCount];
+    for (i = 0; i < commonPrefixesCount; i++) {
+        commonPrefixes[i] = lbData->commonPrefixes[i];
+    }
+
+    return (*(lbData->listBucketCallback))
+        (isTruncated, lbData->nextMarker,
+         contentsCount, contents, commonPrefixesCount, 
+         (const char **) commonPrefixes, lbData->callbackData);
+}
+
+
+static S3Status listBucketXmlCallback(const char *elementPath,
+                                      const char *data, int dataLen,
+                                      void *callbackData)
+{
+    ListBucketData *lbData = (ListBucketData *) callbackData;
+
+    int fit;
+
+    if (data) {
+        if (!strcmp(elementPath, "ListBucketResult/IsTruncated")) {
+            string_buffer_append(lbData->isTruncated, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, "ListBucketResult/NextMarker")) {
+            string_buffer_append(lbData->nextMarker, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, "ListBucketResult/Contents/Key")) {
+            ListBucketContents *contents = 
+                &(lbData->contents[lbData->contentsCount]);
+            string_buffer_append(contents->key, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, 
+                         "ListBucketResult/Contents/LastModified")) {
+            ListBucketContents *contents = 
+                &(lbData->contents[lbData->contentsCount]);
+            string_buffer_append(contents->lastModified, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, "ListBucketResult/Contents/ETag")) {
+            ListBucketContents *contents = 
+                &(lbData->contents[lbData->contentsCount]);
+            string_buffer_append(contents->eTag, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, "ListBucketResult/Contents/Size")) {
+            ListBucketContents *contents = 
+                &(lbData->contents[lbData->contentsCount]);
+            string_buffer_append(contents->size, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, "ListBucketResult/Contents/Owner/ID")) {
+            ListBucketContents *contents = 
+                &(lbData->contents[lbData->contentsCount]);
+            string_buffer_append(contents->ownerId, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, 
+                         "ListBucketResult/Contents/Owner/DisplayName")) {
+            ListBucketContents *contents = 
+                &(lbData->contents[lbData->contentsCount]);
+            string_buffer_append
+                (contents->ownerDisplayName, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, 
+                         "ListBucketResult/CommonPrefixes/Prefix")) {
+            int which = lbData->commonPrefixesCount;
+            lbData->commonPrefixLens[which] +=
+                snprintf(lbData->commonPrefixes[which],
+                         sizeof(lbData->commonPrefixes[which]) -
+                         lbData->commonPrefixLens[which] - 1,
+                         "%.*s", dataLen, data);
+            if (lbData->commonPrefixLens[which] >=
+                (int) sizeof(lbData->commonPrefixes[which])) {
+                return S3StatusXmlParseFailure;
+            }
+        }
+    }
+    else {
+        if (!strcmp(elementPath, "ListBucketResult/Contents")) {
+            // Finished a Contents
+            lbData->contentsCount++;
+            if (lbData->contentsCount == MAX_CONTENTS) {
+                // Make the callback
+                S3Status status = make_list_bucket_callback(lbData);
+                if (status != S3StatusOK) {
+                    return status;
+                }
+                initialize_list_bucket_data(lbData);
+            }
+            else {
+                // Initialize the next one
+                initialize_list_bucket_contents
+                    (&(lbData->contents[lbData->contentsCount]));
+            }
+        }
+        else if (!strcmp(elementPath,
+                         "ListBucketResult/CommonPrefixes/Prefix")) {
+            // Finished a Prefix
+            lbData->commonPrefixesCount++;
+            if (lbData->commonPrefixesCount == MAX_COMMON_PREFIXES) {
+                // Make the callback
+                S3Status status = make_list_bucket_callback(lbData);
+                if (status != S3StatusOK) {
+                    return status;
+                }
+                initialize_list_bucket_data(lbData);
+            }
+            else {
+                // Initialize the next one
+                lbData->commonPrefixes[lbData->commonPrefixesCount][0] = 0;
+                lbData->commonPrefixLens[lbData->commonPrefixesCount] = 0;
+            }
+        }
+    }
+
+    /* Avoid compiler error about variable set but not used */
+    (void) fit;
+
+    return S3StatusOK;
+}
+
+
+static S3Status listBucketPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    ListBucketData *lbData = (ListBucketData *) callbackData;
+    
+    return (*(lbData->responsePropertiesCallback))
+        (responseProperties, lbData->callbackData);
+}
+
+
+static S3Status listBucketDataCallback(int bufferSize, const char *buffer, 
+                                       void *callbackData)
+{
+    ListBucketData *lbData = (ListBucketData *) callbackData;
+    
+    return simplexml_add(&(lbData->simpleXml), buffer, bufferSize);
+}
+
+
+static void listBucketCompleteCallback(S3Status requestStatus, 
+                                       const S3ErrorDetails *s3ErrorDetails,
+                                       void *callbackData)
+{
+    ListBucketData *lbData = (ListBucketData *) callbackData;
+
+    // Make the callback if there is anything
+    if (lbData->contentsCount || lbData->commonPrefixesCount) {
+        make_list_bucket_callback(lbData);
+    }
+
+    (*(lbData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, lbData->callbackData);
+
+    simplexml_deinitialize(&(lbData->simpleXml));
+
+    free(lbData);
+}
+
+
+void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix,
+                    const char *marker, const char *delimiter, int maxkeys,
+                    S3RequestContext *requestContext,
+                    const S3ListBucketHandler *handler, void *callbackData)
+{
+    // Compose the query params
+    string_buffer(queryParams, 4096);
+    string_buffer_initialize(queryParams);
+    
+#define safe_append(name, value)                                        \
+    do {                                                                \
+        int fit;                                                        \
+        if (amp) {                                                      \
+            string_buffer_append(queryParams, "&", 1, fit);             \
+            if (!fit) {                                                 \
+                (*(handler->responseHandler.completeCallback))          \
+                    (S3StatusQueryParamsTooLong, 0, callbackData);      \
+                return;                                                 \
+            }                                                           \
+        }                                                               \
+        string_buffer_append(queryParams, name "=",                     \
+                             sizeof(name "=") - 1, fit);                \
+        if (!fit) {                                                     \
+            (*(handler->responseHandler.completeCallback))              \
+                (S3StatusQueryParamsTooLong, 0, callbackData);          \
+            return;                                                     \
+        }                                                               \
+        amp = 1;                                                        \
+        char encoded[3 * 1024];                                         \
+        if (!urlEncode(encoded, value, 1024)) {                         \
+            (*(handler->responseHandler.completeCallback))              \
+                (S3StatusQueryParamsTooLong, 0, callbackData);          \
+            return;                                                     \
+        }                                                               \
+        string_buffer_append(queryParams, encoded, strlen(encoded),     \
+                             fit);                                      \
+        if (!fit) {                                                     \
+            (*(handler->responseHandler.completeCallback))              \
+                (S3StatusQueryParamsTooLong, 0, callbackData);          \
+            return;                                                     \
+        }                                                               \
+    } while (0)
+
+
+    int amp = 0;
+    if (prefix) {
+        safe_append("prefix", prefix);
+    }
+    if (marker) {
+        safe_append("marker", marker);
+    }
+    if (delimiter) {
+        safe_append("delimiter", delimiter);
+    }
+    if (maxkeys) {
+        char maxKeysString[64];
+        snprintf(maxKeysString, sizeof(maxKeysString), "%d", maxkeys);
+        safe_append("max-keys", maxKeysString);
+    }
+
+    ListBucketData *lbData =
+        (ListBucketData *) malloc(sizeof(ListBucketData));
+
+    if (!lbData) {
+        (*(handler->responseHandler.completeCallback))
+            (S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    simplexml_initialize(&(lbData->simpleXml), &listBucketXmlCallback, lbData);
+    
+    lbData->responsePropertiesCallback = 
+        handler->responseHandler.propertiesCallback;
+    lbData->listBucketCallback = handler->listBucketCallback;
+    lbData->responseCompleteCallback = 
+        handler->responseHandler.completeCallback;
+    lbData->callbackData = callbackData;
+
+    string_buffer_initialize(lbData->isTruncated);
+    string_buffer_initialize(lbData->nextMarker);
+    initialize_list_bucket_data(lbData);
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeGET,                           // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        0,                                            // key
+        queryParams[0] ? queryParams : 0,             // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        &listBucketPropertiesCallback,                // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        &listBucketDataCallback,                      // fromS3Callback
+        &listBucketCompleteCallback,                  // completeCallback
+        lbData                                        // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
diff --git a/ceph/src/libs3/src/error_parser.c b/ceph/src/libs3/src/error_parser.c
new file mode 100644
index 00000000..baa206ea
--- /dev/null
+++ b/ceph/src/libs3/src/error_parser.c
@@ -0,0 +1,239 @@
+/** **************************************************************************
+ * error_parser.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include "error_parser.h"
+
+
+static S3Status errorXmlCallback(const char *elementPath, const char *data,
+                                 int dataLen, void *callbackData)
+{
+    // We ignore end of element callbacks because we don't care about them
+    if (!data) {
+        return S3StatusOK;
+    }
+
+    ErrorParser *errorParser = (ErrorParser *) callbackData;
+
+    int fit;
+
+    if (!strcmp(elementPath, "Error")) {
+        // Ignore, this is the Error element itself, we only care about subs
+    }
+    else if (!strcmp(elementPath, "Error/Code")) {
+        string_buffer_append(errorParser->code, data, dataLen, fit);
+    }
+    else if (!strcmp(elementPath, "Error/Message")) {
+        string_buffer_append(errorParser->message, data, dataLen, fit);
+        errorParser->s3ErrorDetails.message = errorParser->message;
+    }
+    else if (!strcmp(elementPath, "Error/Resource")) {
+        string_buffer_append(errorParser->resource, data, dataLen, fit);
+        errorParser->s3ErrorDetails.resource = errorParser->resource;
+    }
+    else if (!strcmp(elementPath, "Error/FurtherDetails")) {
+        string_buffer_append(errorParser->furtherDetails, data, dataLen, fit);
+        errorParser->s3ErrorDetails.furtherDetails = 
+            errorParser->furtherDetails;
+    }
+    else {
+        if (strncmp(elementPath, "Error/", sizeof("Error/") - 1)) {
+            // If for some weird reason it's not within the Error element,
+            // ignore it
+            return S3StatusOK;
+        }
+        // It's an unknown error element.  See if it matches the most
+        // recent error element.
+        const char *elementName = &(elementPath[sizeof("Error/") - 1]);
+        if (errorParser->s3ErrorDetails.extraDetailsCount && 
+            !strcmp(elementName, errorParser->s3ErrorDetails.extraDetails
+                    [errorParser->s3ErrorDetails.extraDetailsCount - 1].name)) {
+            // Append the value
+            string_multibuffer_append(errorParser->extraDetailsNamesValues,
+                                      data, dataLen, fit);
+            // If it didn't fit, remove this extra
+            if (!fit) {
+                errorParser->s3ErrorDetails.extraDetailsCount--;
+            }
+            return S3StatusOK;
+        }
+        // OK, must add another unknown error element, if it will fit.
+        if (errorParser->s3ErrorDetails.extraDetailsCount ==
+            sizeof(errorParser->extraDetails)) {
+            // Won't fit.  Ignore this one.
+            return S3StatusOK;
+        }
+        // Copy in the name and value
+        char *name = string_multibuffer_current
+            (errorParser->extraDetailsNamesValues);
+        int nameLen = strlen(elementName);
+        string_multibuffer_add(errorParser->extraDetailsNamesValues,
+                               elementName, nameLen, fit);
+        if (!fit) {
+            // Name didn't fit; ignore this one.
+            return S3StatusOK;
+        }
+        char *value = string_multibuffer_current
+            (errorParser->extraDetailsNamesValues);
+        string_multibuffer_add(errorParser->extraDetailsNamesValues,
+                               data, dataLen, fit);
+        if (!fit) {
+            // Value didn't fit; ignore this one.
+            return S3StatusOK;
+        }
+        S3NameValue *nv = 
+            &(errorParser->extraDetails
+              [errorParser->s3ErrorDetails.extraDetailsCount++]);
+        nv->name = name;
+        nv->value = value;
+    }
+
+    return S3StatusOK;
+}
+
+
+void error_parser_initialize(ErrorParser *errorParser)
+{
+    errorParser->s3ErrorDetails.message = 0;
+    errorParser->s3ErrorDetails.resource = 0;
+    errorParser->s3ErrorDetails.furtherDetails = 0;
+    errorParser->s3ErrorDetails.extraDetailsCount = 0;
+    errorParser->s3ErrorDetails.extraDetails = errorParser->extraDetails;
+    errorParser->errorXmlParserInitialized = 0;
+    string_buffer_initialize(errorParser->code);
+    string_buffer_initialize(errorParser->message);
+    string_buffer_initialize(errorParser->resource);
+    string_buffer_initialize(errorParser->furtherDetails);
+    string_multibuffer_initialize(errorParser->extraDetailsNamesValues);
+}
+
+
+S3Status error_parser_add(ErrorParser *errorParser, char *buffer,
+                          int bufferSize)
+{
+    if (!errorParser->errorXmlParserInitialized) {
+        simplexml_initialize(&(errorParser->errorXmlParser), &errorXmlCallback,
+                             errorParser);
+        errorParser->errorXmlParserInitialized = 1;
+    }
+
+    return simplexml_add(&(errorParser->errorXmlParser), buffer, bufferSize);
+}
+
+
+void error_parser_convert_status(ErrorParser *errorParser, S3Status *status)
+{
+    // Convert the error status string into a code
+    if (!errorParser->codeLen) {
+        return;
+    }
+
+#define HANDLE_CODE(name)                                       \
+    do {                                                        \
+        if (!strcmp(errorParser->code, #name)) {                \
+            *status = S3StatusError##name;                      \
+            goto code_set;                                      \
+        }                                                       \
+    } while (0)
+    
+    HANDLE_CODE(AccessDenied);
+    HANDLE_CODE(AccountProblem);
+    HANDLE_CODE(AmbiguousGrantByEmailAddress);
+    HANDLE_CODE(BadDigest);
+    HANDLE_CODE(BucketAlreadyExists);
+    HANDLE_CODE(BucketAlreadyOwnedByYou);
+    HANDLE_CODE(BucketNotEmpty);
+    HANDLE_CODE(CredentialsNotSupported);
+    HANDLE_CODE(CrossLocationLoggingProhibited);
+    HANDLE_CODE(EntityTooSmall);
+    HANDLE_CODE(EntityTooLarge);
+    HANDLE_CODE(ExpiredToken);
+    HANDLE_CODE(IncompleteBody);
+    HANDLE_CODE(IncorrectNumberOfFilesInPostRequest);
+    HANDLE_CODE(InlineDataTooLarge);
+    HANDLE_CODE(InternalError);
+    HANDLE_CODE(InvalidAccessKeyId);
+    HANDLE_CODE(InvalidAddressingHeader);
+    HANDLE_CODE(InvalidArgument);
+    HANDLE_CODE(InvalidBucketName);
+    HANDLE_CODE(InvalidDigest);
+    HANDLE_CODE(InvalidLocationConstraint);
+    HANDLE_CODE(InvalidPayer);
+    HANDLE_CODE(InvalidPolicyDocument);
+    HANDLE_CODE(InvalidRange);
+    HANDLE_CODE(InvalidSecurity);
+    HANDLE_CODE(InvalidSOAPRequest);
+    HANDLE_CODE(InvalidStorageClass);
+    HANDLE_CODE(InvalidTargetBucketForLogging);
+    HANDLE_CODE(InvalidToken);
+    HANDLE_CODE(InvalidURI);
+    HANDLE_CODE(KeyTooLong);
+    HANDLE_CODE(MalformedACLError);
+    HANDLE_CODE(MalformedXML);
+    HANDLE_CODE(MaxMessageLengthExceeded);
+    HANDLE_CODE(MaxPostPreDataLengthExceededError);
+    HANDLE_CODE(MetadataTooLarge);
+    HANDLE_CODE(MethodNotAllowed);
+    HANDLE_CODE(MissingAttachment);
+    HANDLE_CODE(MissingContentLength);
+    HANDLE_CODE(MissingSecurityElement);
+    HANDLE_CODE(MissingSecurityHeader);
+    HANDLE_CODE(NoLoggingStatusForKey);
+    HANDLE_CODE(NoSuchBucket);
+    HANDLE_CODE(NoSuchKey);
+    HANDLE_CODE(NotImplemented);
+    HANDLE_CODE(NotSignedUp);
+    HANDLE_CODE(OperationAborted);
+    HANDLE_CODE(PermanentRedirect);
+    HANDLE_CODE(PreconditionFailed);
+    HANDLE_CODE(Redirect);
+    HANDLE_CODE(RequestIsNotMultiPartContent);
+    HANDLE_CODE(RequestTimeout);
+    HANDLE_CODE(RequestTimeTooSkewed);
+    HANDLE_CODE(RequestTorrentOfBucketError);
+    HANDLE_CODE(SignatureDoesNotMatch);
+    HANDLE_CODE(SlowDown);
+    HANDLE_CODE(TemporaryRedirect);
+    HANDLE_CODE(TokenRefreshRequired);
+    HANDLE_CODE(TooManyBuckets);
+    HANDLE_CODE(UnexpectedContent);
+    HANDLE_CODE(UnresolvableGrantByEmailAddress);
+    HANDLE_CODE(UserKeyMustBeSpecified);
+    *status = S3StatusErrorUnknown;
+
+ code_set:
+
+    return;
+}
+
+
+// Always call this
+void error_parser_deinitialize(ErrorParser *errorParser)
+{
+    if (errorParser->errorXmlParserInitialized) {
+        simplexml_deinitialize(&(errorParser->errorXmlParser));
+    }
+}
diff --git a/ceph/src/libs3/src/general.c b/ceph/src/libs3/src/general.c
new file mode 100644
index 00000000..fb30c37b
--- /dev/null
+++ b/ceph/src/libs3/src/general.c
@@ -0,0 +1,473 @@
+/** **************************************************************************
+ * general.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "request.h"
+#include "simplexml.h"
+#include "util.h"
+
+static int initializeCountG = 0;
+
+S3Status S3_initialize(const char *userAgentInfo, int flags,
+                       const char *defaultS3HostName)
+{
+    if (initializeCountG++) {
+        return S3StatusOK;
+    }
+
+    return request_api_initialize(userAgentInfo, flags, defaultS3HostName);
+}
+
+
+void S3_deinitialize()
+{
+    if (--initializeCountG) {
+        return;
+    }
+
+    request_api_deinitialize();
+}
+
+const char *S3_get_status_name(S3Status status)
+{
+    switch (status) {
+#define handlecase(s)                           \
+        case S3Status##s:                       \
+            return #s
+
+        handlecase(OK);
+        handlecase(InternalError);
+        handlecase(OutOfMemory);
+        handlecase(Interrupted);
+        handlecase(InvalidBucketNameTooLong);
+        handlecase(InvalidBucketNameFirstCharacter);
+        handlecase(InvalidBucketNameCharacter);
+        handlecase(InvalidBucketNameCharacterSequence);
+        handlecase(InvalidBucketNameTooShort);
+        handlecase(InvalidBucketNameDotQuadNotation);
+        handlecase(QueryParamsTooLong);
+        handlecase(FailedToInitializeRequest);
+        handlecase(MetaDataHeadersTooLong);
+        handlecase(BadMetaData);
+        handlecase(BadContentType);
+        handlecase(ContentTypeTooLong);
+        handlecase(BadMD5);
+        handlecase(MD5TooLong);
+        handlecase(BadCacheControl);
+        handlecase(CacheControlTooLong);
+        handlecase(BadContentDispositionFilename);
+        handlecase(ContentDispositionFilenameTooLong);
+        handlecase(BadContentEncoding);
+        handlecase(ContentEncodingTooLong);
+        handlecase(BadIfMatchETag);
+        handlecase(IfMatchETagTooLong);
+        handlecase(BadIfNotMatchETag);
+        handlecase(IfNotMatchETagTooLong);
+        handlecase(HeadersTooLong);
+        handlecase(KeyTooLong);
+        handlecase(UriTooLong);
+        handlecase(XmlParseFailure);
+        handlecase(EmailAddressTooLong);
+        handlecase(UserIdTooLong);
+        handlecase(UserDisplayNameTooLong);
+        handlecase(GroupUriTooLong);
+        handlecase(PermissionTooLong);
+        handlecase(TargetBucketTooLong);
+        handlecase(TargetPrefixTooLong);
+        handlecase(TooManyGrants);
+        handlecase(BadGrantee);
+        handlecase(BadPermission);
+        handlecase(XmlDocumentTooLarge);
+        handlecase(NameLookupError);
+        handlecase(FailedToConnect);
+        handlecase(ServerFailedVerification);
+        handlecase(ConnectionFailed);
+        handlecase(AbortedByCallback);
+        handlecase(ErrorAccessDenied);
+        handlecase(ErrorAccountProblem);
+        handlecase(ErrorAmbiguousGrantByEmailAddress);
+        handlecase(ErrorBadDigest);
+        handlecase(ErrorBucketAlreadyExists);
+        handlecase(ErrorBucketAlreadyOwnedByYou);
+        handlecase(ErrorBucketNotEmpty);
+        handlecase(ErrorCredentialsNotSupported);
+        handlecase(ErrorCrossLocationLoggingProhibited);
+        handlecase(ErrorEntityTooSmall);
+        handlecase(ErrorEntityTooLarge);
+        handlecase(ErrorExpiredToken);
+        handlecase(ErrorIncompleteBody);
+        handlecase(ErrorIncorrectNumberOfFilesInPostRequest);
+        handlecase(ErrorInlineDataTooLarge);
+        handlecase(ErrorInternalError);
+        handlecase(ErrorInvalidAccessKeyId);
+        handlecase(ErrorInvalidAddressingHeader);
+        handlecase(ErrorInvalidArgument);
+        handlecase(ErrorInvalidBucketName);
+        handlecase(ErrorInvalidDigest);
+        handlecase(ErrorInvalidLocationConstraint);
+        handlecase(ErrorInvalidPayer);
+        handlecase(ErrorInvalidPolicyDocument);
+        handlecase(ErrorInvalidRange);
+        handlecase(ErrorInvalidSecurity);
+        handlecase(ErrorInvalidSOAPRequest);
+        handlecase(ErrorInvalidStorageClass);
+        handlecase(ErrorInvalidTargetBucketForLogging);
+        handlecase(ErrorInvalidToken);
+        handlecase(ErrorInvalidURI);
+        handlecase(ErrorKeyTooLong);
+        handlecase(ErrorMalformedACLError);
+        handlecase(ErrorMalformedXML);
+        handlecase(ErrorMaxMessageLengthExceeded);
+        handlecase(ErrorMaxPostPreDataLengthExceededError);
+        handlecase(ErrorMetadataTooLarge);
+        handlecase(ErrorMethodNotAllowed);
+        handlecase(ErrorMissingAttachment);
+        handlecase(ErrorMissingContentLength);
+        handlecase(ErrorMissingSecurityElement);
+        handlecase(ErrorMissingSecurityHeader);
+        handlecase(ErrorNoLoggingStatusForKey);
+        handlecase(ErrorNoSuchBucket);
+        handlecase(ErrorNoSuchKey);
+        handlecase(ErrorNotImplemented);
+        handlecase(ErrorNotSignedUp);
+        handlecase(ErrorOperationAborted);
+        handlecase(ErrorPermanentRedirect);
+        handlecase(ErrorPreconditionFailed);
+        handlecase(ErrorRedirect);
+        handlecase(ErrorRequestIsNotMultiPartContent);
+        handlecase(ErrorRequestTimeout);
+        handlecase(ErrorRequestTimeTooSkewed);
+        handlecase(ErrorRequestTorrentOfBucketError);
+        handlecase(ErrorSignatureDoesNotMatch);
+        handlecase(ErrorSlowDown);
+        handlecase(ErrorTemporaryRedirect);
+        handlecase(ErrorTokenRefreshRequired);
+        handlecase(ErrorTooManyBuckets);
+        handlecase(ErrorUnexpectedContent);
+        handlecase(ErrorUnresolvableGrantByEmailAddress);
+        handlecase(ErrorUserKeyMustBeSpecified);
+        handlecase(ErrorUnknown);    
+        handlecase(HttpErrorMovedTemporarily);
+        handlecase(HttpErrorBadRequest);
+        handlecase(HttpErrorForbidden);
+        handlecase(HttpErrorNotFound);
+        handlecase(HttpErrorConflict);
+        handlecase(HttpErrorUnknown);
+    }
+
+    return "Unknown";
+}
+
+
+S3Status S3_validate_bucket_name(const char *bucketName, S3UriStyle uriStyle)
+{
+    int virtualHostStyle = (uriStyle == S3UriStyleVirtualHost);
+    int len = 0, maxlen = virtualHostStyle ? 63 : 255;
+    const char *b = bucketName;
+
+    int hasDot = 0;
+    int hasNonDigit = 0;
+
+    while (*b) {
+        if (len == maxlen) {
+            return S3StatusInvalidBucketNameTooLong;
+        }
+        else if (isalpha(*b)) {
+            len++, b++;
+            hasNonDigit = 1;
+        }
+        else if (isdigit(*b)) {
+            len++, b++;
+        }
+        else if (len == 0) {
+            return S3StatusInvalidBucketNameFirstCharacter;
+        }
+        else if (*b == '_') {
+            /* Virtual host style bucket names cannot have underscores */
+            if (virtualHostStyle) {
+                return S3StatusInvalidBucketNameCharacter;
+            }
+            len++, b++;
+            hasNonDigit = 1;
+        }
+        else if (*b == '-') {
+            /* Virtual host style bucket names cannot have .- */
+            if (virtualHostStyle && (b > bucketName) && (*(b - 1) == '.')) {
+                return S3StatusInvalidBucketNameCharacterSequence;
+            }
+            len++, b++;
+            hasNonDigit = 1;
+        }
+        else if (*b == '.') {
+            /* Virtual host style bucket names cannot have -. */
+            if (virtualHostStyle && (b > bucketName) && (*(b - 1) == '-')) {
+                return S3StatusInvalidBucketNameCharacterSequence;
+            }
+            len++, b++;
+            hasDot = 1;
+        }
+        else {
+            return S3StatusInvalidBucketNameCharacter;
+        }
+    }
+
+    if (len < 3) {
+        return S3StatusInvalidBucketNameTooShort;
+    }
+
+    /* It's not clear from Amazon's documentation exactly what 'IP address
+       style' means.  In its strictest sense, it could mean 'could be a valid
+       IP address', which would mean that 255.255.255.255 would be invalid,
+       wherase 256.256.256.256 would be valid.  Or it could mean 'has 4 sets
+       of digits separated by dots'.  Who knows.  Let's just be really
+       conservative here: if it has any dots, and no non-digit characters,
+       then we reject it */
+    if (hasDot && !hasNonDigit) {
+        return S3StatusInvalidBucketNameDotQuadNotation;
+    }
+
+    return S3StatusOK;
+}
+
+
+typedef struct ConvertAclData
+{
+    char *ownerId;
+    int ownerIdLen;
+    char *ownerDisplayName;
+    int ownerDisplayNameLen;
+    int *aclGrantCountReturn;
+    S3AclGrant *aclGrants;
+
+    string_buffer(emailAddress, S3_MAX_GRANTEE_EMAIL_ADDRESS_SIZE);
+    string_buffer(userId, S3_MAX_GRANTEE_USER_ID_SIZE);
+    string_buffer(userDisplayName, S3_MAX_GRANTEE_DISPLAY_NAME_SIZE);
+    string_buffer(groupUri, 128);
+    string_buffer(permission, 32);
+} ConvertAclData;
+
+
+static S3Status convertAclXmlCallback(const char *elementPath,
+                                      const char *data, int dataLen,
+                                      void *callbackData)
+{
+    ConvertAclData *caData = (ConvertAclData *) callbackData;
+
+    int fit;
+
+    if (data) {
+        if (!strcmp(elementPath, "AccessControlPolicy/Owner/ID")) {
+            caData->ownerIdLen += 
+                snprintf(&(caData->ownerId[caData->ownerIdLen]),
+                         S3_MAX_GRANTEE_USER_ID_SIZE - caData->ownerIdLen - 1,
+                         "%.*s", dataLen, data);
+            if (caData->ownerIdLen >= S3_MAX_GRANTEE_USER_ID_SIZE) {
+                return S3StatusUserIdTooLong;
+            }
+        }
+        else if (!strcmp(elementPath, "AccessControlPolicy/Owner/"
+                         "DisplayName")) {
+            caData->ownerDisplayNameLen += 
+                snprintf(&(caData->ownerDisplayName
+                           [caData->ownerDisplayNameLen]),
+                         S3_MAX_GRANTEE_DISPLAY_NAME_SIZE -
+                         caData->ownerDisplayNameLen - 1, 
+                         "%.*s", dataLen, data);
+            if (caData->ownerDisplayNameLen >= 
+                S3_MAX_GRANTEE_DISPLAY_NAME_SIZE) {
+                return S3StatusUserDisplayNameTooLong;
+            }
+        }
+        else if (!strcmp(elementPath, 
+                    "AccessControlPolicy/AccessControlList/Grant/"
+                    "Grantee/EmailAddress")) {
+            // AmazonCustomerByEmail
+            string_buffer_append(caData->emailAddress, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusEmailAddressTooLong;
+            }
+        }
+        else if (!strcmp(elementPath,
+                         "AccessControlPolicy/AccessControlList/Grant/"
+                         "Grantee/ID")) {
+            // CanonicalUser
+            string_buffer_append(caData->userId, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusUserIdTooLong;
+            }
+        }
+        else if (!strcmp(elementPath,
+                         "AccessControlPolicy/AccessControlList/Grant/"
+                         "Grantee/DisplayName")) {
+            // CanonicalUser
+            string_buffer_append(caData->userDisplayName, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusUserDisplayNameTooLong;
+            }
+        }
+        else if (!strcmp(elementPath,
+                         "AccessControlPolicy/AccessControlList/Grant/"
+                         "Grantee/URI")) {
+            // Group
+            string_buffer_append(caData->groupUri, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusGroupUriTooLong;
+            }
+        }
+        else if (!strcmp(elementPath,
+                         "AccessControlPolicy/AccessControlList/Grant/"
+                         "Permission")) {
+            // Permission
+            string_buffer_append(caData->permission, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusPermissionTooLong;
+            }
+        }
+    }
+    else {
+        if (!strcmp(elementPath, "AccessControlPolicy/AccessControlList/"
+                    "Grant")) {
+            // A grant has just been completed; so add the next S3AclGrant
+            // based on the values read
+            if (*(caData->aclGrantCountReturn) == S3_MAX_ACL_GRANT_COUNT) {
+                return S3StatusTooManyGrants;
+            }
+
+            S3AclGrant *grant = &(caData->aclGrants
+                                  [*(caData->aclGrantCountReturn)]);
+
+            if (caData->emailAddress[0]) {
+                grant->granteeType = S3GranteeTypeAmazonCustomerByEmail;
+                strcpy(grant->grantee.amazonCustomerByEmail.emailAddress,
+                       caData->emailAddress);
+            }
+            else if (caData->userId[0] && caData->userDisplayName[0]) {
+                grant->granteeType = S3GranteeTypeCanonicalUser;
+                strcpy(grant->grantee.canonicalUser.id, caData->userId);
+                strcpy(grant->grantee.canonicalUser.displayName, 
+                       caData->userDisplayName);
+            }
+            else if (caData->groupUri[0]) {
+                if (!strcmp(caData->groupUri,
+                            ACS_GROUP_AWS_USERS)) {
+                    grant->granteeType = S3GranteeTypeAllAwsUsers;
+                }
+                else if (!strcmp(caData->groupUri,
+                            ACS_GROUP_ALL_USERS)) {
+                    grant->granteeType = S3GranteeTypeAllUsers;
+                }
+                else if (!strcmp(caData->groupUri,
+                                 ACS_GROUP_LOG_DELIVERY)) {
+                    grant->granteeType = S3GranteeTypeLogDelivery;
+                }
+                else {
+                    return S3StatusBadGrantee;
+                }
+            }
+            else {
+                return S3StatusBadGrantee;
+            }
+
+            if (!strcmp(caData->permission, "READ")) {
+                grant->permission = S3PermissionRead;
+            }
+            else if (!strcmp(caData->permission, "WRITE")) {
+                grant->permission = S3PermissionWrite;
+            }
+            else if (!strcmp(caData->permission, "READ_ACP")) {
+                grant->permission = S3PermissionReadACP;
+            }
+            else if (!strcmp(caData->permission, "WRITE_ACP")) {
+                grant->permission = S3PermissionWriteACP;
+            }
+            else if (!strcmp(caData->permission, "FULL_CONTROL")) {
+                grant->permission = S3PermissionFullControl;
+            }
+            else {
+                return S3StatusBadPermission;
+            }
+
+            (*(caData->aclGrantCountReturn))++;
+
+            string_buffer_initialize(caData->emailAddress);
+            string_buffer_initialize(caData->userId);
+            string_buffer_initialize(caData->userDisplayName);
+            string_buffer_initialize(caData->groupUri);
+            string_buffer_initialize(caData->permission);
+        }
+    }
+
+    return S3StatusOK;
+}
+
+
+S3Status S3_convert_acl(char *aclXml, char *ownerId, char *ownerDisplayName,
+                        int *aclGrantCountReturn, S3AclGrant *aclGrants)
+{
+    ConvertAclData data;
+
+    data.ownerId = ownerId;
+    data.ownerIdLen = 0;
+    data.ownerId[0] = 0;
+    data.ownerDisplayName = ownerDisplayName;
+    data.ownerDisplayNameLen = 0;
+    data.ownerDisplayName[0] = 0;
+    data.aclGrantCountReturn = aclGrantCountReturn;
+    data.aclGrants = aclGrants;
+    *aclGrantCountReturn = 0;
+    string_buffer_initialize(data.emailAddress);
+    string_buffer_initialize(data.userId);
+    string_buffer_initialize(data.userDisplayName);
+    string_buffer_initialize(data.groupUri);
+    string_buffer_initialize(data.permission);
+
+    // Use a simplexml parser
+    SimpleXml simpleXml;
+    simplexml_initialize(&simpleXml, &convertAclXmlCallback, &data);
+
+    S3Status status = simplexml_add(&simpleXml, aclXml, strlen(aclXml));
+
+    simplexml_deinitialize(&simpleXml);
+                                          
+    return status;
+}
+
+
+int S3_status_is_retryable(S3Status status)
+{
+    switch (status) {
+    case S3StatusNameLookupError:
+    case S3StatusFailedToConnect:
+    case S3StatusConnectionFailed:
+    case S3StatusErrorInternalError:
+    case S3StatusErrorOperationAborted:
+    case S3StatusErrorRequestTimeout:
+        return 1;
+    default:
+        return 0;
+    }
+}
diff --git a/ceph/src/libs3/src/mingw_functions.c b/ceph/src/libs3/src/mingw_functions.c
new file mode 100644
index 00000000..0e2b7b23
--- /dev/null
+++ b/ceph/src/libs3/src/mingw_functions.c
@@ -0,0 +1,119 @@
+/** **************************************************************************
+ * mingw_functions.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+
+unsigned long pthread_self()
+{
+    return (unsigned long) GetCurrentThreadId();
+}
+
+
+int pthread_mutex_init(pthread_mutex_t *mutex, void *v)
+{
+    (void) v;
+
+    InitializeCriticalSection(&(mutex->criticalSection));
+
+    return 0;
+}
+
+
+int pthread_mutex_lock(pthread_mutex_t *mutex)
+{
+    EnterCriticalSection(&(mutex->criticalSection));
+
+    return 0;
+}
+
+
+int pthread_mutex_unlock(pthread_mutex_t *mutex)
+{
+    LeaveCriticalSection(&(mutex->criticalSection));
+
+    return 0;
+}
+
+
+int pthread_mutex_destroy(pthread_mutex_t *mutex)
+{
+    DeleteCriticalSection(&(mutex->criticalSection));
+
+    return 0;
+}
+
+
+int uname(struct utsname *u)
+{
+    OSVERSIONINFO info;
+    info.dwOSVersionInfoSize = sizeof(info);
+
+    if (!GetVersionEx(&info)) {
+        return -1;
+    }
+
+    u->machine = "";
+
+    switch (info.dwMajorVersion) {
+    case 4:
+        switch (info.dwMinorVersion) {
+        case 0:
+            u->sysname = "Microsoft Windows NT 4.0";
+            break;
+        case 10:
+            u->sysname = "Microsoft Windows 98";
+            break;
+        case 90:
+            u->sysname = "Microsoft Windows Me";
+            break;
+        default:
+            return -1;
+        }
+        break;
+
+    case 5:
+        switch (info.dwMinorVersion) {
+        case 0:
+            u->sysname = "Microsoft Windows 2000";
+            break;
+        case 1:
+            u->sysname = "Microsoft Windows XP";
+            break;
+        case 2:
+            u->sysname = "Microsoft Server 2003";
+            break;
+        default:
+            return -1;
+        }
+        break;
+
+    default:
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/ceph/src/libs3/src/mingw_s3_functions.c b/ceph/src/libs3/src/mingw_s3_functions.c
new file mode 100644
index 00000000..142569df
--- /dev/null
+++ b/ceph/src/libs3/src/mingw_s3_functions.c
@@ -0,0 +1,37 @@
+/** **************************************************************************
+ * mingw_s3_functions.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+int setenv(const char *a, const char *b, int c)
+{
+    (void) c;
+
+    return SetEnvironmentVariable(a, b);
+}
+
+int unsetenv(const char *a)
+{
+    return SetEnvironmentVariable(a, 0);
+}
diff --git a/ceph/src/libs3/src/object.c b/ceph/src/libs3/src/object.c
new file mode 100644
index 00000000..d7c7f802
--- /dev/null
+++ b/ceph/src/libs3/src/object.c
@@ -0,0 +1,345 @@
+/** **************************************************************************
+ * object.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "libs3.h"
+#include "request.h"
+
+
+// put object ----------------------------------------------------------------
+
+void S3_put_object(const S3BucketContext *bucketContext, const char *key,
+                   uint64_t contentLength,
+                   const S3PutProperties *putProperties,
+                   S3RequestContext *requestContext,
+                   const S3PutObjectHandler *handler, void *callbackData)
+{
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypePUT,                           // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        key,                                          // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        putProperties,                                // putProperties
+        handler->responseHandler.propertiesCallback,  // propertiesCallback
+        handler->putObjectDataCallback,               // toS3Callback
+        contentLength,                                // toS3CallbackTotalSize
+        0,                                            // fromS3Callback
+        handler->responseHandler.completeCallback,    // completeCallback
+        callbackData                                  // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
+// copy object ---------------------------------------------------------------
+
+
+typedef struct CopyObjectData
+{
+    SimpleXml simpleXml;
+
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    int64_t *lastModifiedReturn;
+    int eTagReturnSize;
+    char *eTagReturn;
+    int eTagReturnLen;
+    
+    string_buffer(lastModified, 256);
+} CopyObjectData;
+
+
+static S3Status copyObjectXmlCallback(const char *elementPath,
+                                      const char *data, int dataLen,
+                                      void *callbackData)
+{
+    CopyObjectData *coData = (CopyObjectData *) callbackData;
+
+    int fit;
+
+    if (data) {
+        if (!strcmp(elementPath, "CopyObjectResult/LastModified")) {
+            string_buffer_append(coData->lastModified, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, "CopyObjectResult/ETag")) {
+            if (coData->eTagReturnSize && coData->eTagReturn) {
+                coData->eTagReturnLen +=
+                    snprintf(&(coData->eTagReturn[coData->eTagReturnLen]),
+                             coData->eTagReturnSize - 
+                             coData->eTagReturnLen - 1,
+                             "%.*s", dataLen, data);
+                if (coData->eTagReturnLen >= coData->eTagReturnSize) {
+                    return S3StatusXmlParseFailure;
+                }
+            }
+        }
+    }
+
+    /* Avoid compiler error about variable set but not used */
+    (void) fit;
+
+    return S3StatusOK;
+}
+
+
+static S3Status copyObjectPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    CopyObjectData *coData = (CopyObjectData *) callbackData;
+    
+    return (*(coData->responsePropertiesCallback))
+        (responseProperties, coData->callbackData);
+}
+
+
+static S3Status copyObjectDataCallback(int bufferSize, const char *buffer,
+                                       void *callbackData)
+{
+    CopyObjectData *coData = (CopyObjectData *) callbackData;
+
+    return simplexml_add(&(coData->simpleXml), buffer, bufferSize);
+}
+
+
+static void copyObjectCompleteCallback(S3Status requestStatus, 
+                                       const S3ErrorDetails *s3ErrorDetails,
+                                       void *callbackData)
+{
+    CopyObjectData *coData = (CopyObjectData *) callbackData;
+
+    if (coData->lastModifiedReturn) {
+        time_t lastModified = -1;
+        if (coData->lastModifiedLen) {
+            lastModified = parseIso8601Time(coData->lastModified);
+        }
+
+        *(coData->lastModifiedReturn) = lastModified;
+    }
+
+    (*(coData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, coData->callbackData);
+
+    simplexml_deinitialize(&(coData->simpleXml));
+
+    free(coData);
+}
+
+
+void S3_copy_object(const S3BucketContext *bucketContext, const char *key,
+                    const char *destinationBucket, const char *destinationKey,
+                    const S3PutProperties *putProperties,
+                    int64_t *lastModifiedReturn, int eTagReturnSize,
+                    char *eTagReturn, S3RequestContext *requestContext,
+                    const S3ResponseHandler *handler, void *callbackData)
+{
+    // Create the callback data
+    CopyObjectData *data = 
+        (CopyObjectData *) malloc(sizeof(CopyObjectData));
+    if (!data) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    simplexml_initialize(&(data->simpleXml), ©ObjectXmlCallback, data);
+
+    data->responsePropertiesCallback = handler->propertiesCallback;
+    data->responseCompleteCallback = handler->completeCallback;
+    data->callbackData = callbackData;
+
+    data->lastModifiedReturn = lastModifiedReturn;
+    data->eTagReturnSize = eTagReturnSize;
+    data->eTagReturn = eTagReturn;
+    if (data->eTagReturnSize && data->eTagReturn) {
+        data->eTagReturn[0] = 0;
+    }
+    data->eTagReturnLen = 0;
+    string_buffer_initialize(data->lastModified);
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeCOPY,                          // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          destinationBucket ? destinationBucket : 
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        destinationKey ? destinationKey : key,        // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        bucketContext->bucketName,                    // copySourceBucketName
+        key,                                          // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        putProperties,                                // putProperties
+        ©ObjectPropertiesCallback,                // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        ©ObjectDataCallback,                      // fromS3Callback
+        ©ObjectCompleteCallback,                  // completeCallback
+        data                                          // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
+// get object ----------------------------------------------------------------
+
+void S3_get_object(const S3BucketContext *bucketContext, const char *key,
+                   const S3GetConditions *getConditions,
+                   uint64_t startByte, uint64_t byteCount,
+                   S3RequestContext *requestContext,
+                   const S3GetObjectHandler *handler, void *callbackData)
+{
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeGET,                           // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        key,                                          // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        getConditions,                                // getConditions
+        startByte,                                    // startByte
+        byteCount,                                    // byteCount
+        0,                                            // putProperties
+        handler->responseHandler.propertiesCallback,  // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        handler->getObjectDataCallback,               // fromS3Callback
+        handler->responseHandler.completeCallback,    // completeCallback
+        callbackData                                  // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
+// head object ---------------------------------------------------------------
+
+void S3_head_object(const S3BucketContext *bucketContext, const char *key,
+                    S3RequestContext *requestContext,
+                    const S3ResponseHandler *handler, void *callbackData)
+{
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeHEAD,                          // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        key,                                          // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        handler->propertiesCallback,                  // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        0,                                            // fromS3Callback
+        handler->completeCallback,                    // completeCallback
+        callbackData                                  // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+                         
+
+// delete object --------------------------------------------------------------
+
+void S3_delete_object(const S3BucketContext *bucketContext, const char *key,
+                      S3RequestContext *requestContext,
+                      const S3ResponseHandler *handler, void *callbackData)
+{
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeDELETE,                        // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        key,                                          // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        handler->propertiesCallback,                  // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        0,                                            // fromS3Callback
+        handler->completeCallback,                    // completeCallback
+        callbackData                                  // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
diff --git a/ceph/src/libs3/src/request.c b/ceph/src/libs3/src/request.c
new file mode 100644
index 00000000..b267a5e9
--- /dev/null
+++ b/ceph/src/libs3/src/request.c
@@ -0,0 +1,1392 @@
+/** **************************************************************************
+ * request.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "request.h"
+#include "request_context.h"
+#include "response_headers_handler.h"
+#include "util.h"
+
+
+#define USER_AGENT_SIZE 256
+#define REQUEST_STACK_SIZE 32
+
+static char userAgentG[USER_AGENT_SIZE];
+
+static pthread_mutex_t requestStackMutexG;
+
+static Request *requestStackG[REQUEST_STACK_SIZE];
+
+static int requestStackCountG;
+
+char defaultHostNameG[S3_MAX_HOSTNAME_SIZE];
+
+
+typedef struct RequestComputedValues
+{
+    // All x-amz- headers, in normalized form (i.e. NAME: VALUE, no other ws)
+    char *amzHeaders[S3_MAX_METADATA_COUNT + 2]; // + 2 for acl and date
+
+    // The number of x-amz- headers
+    int amzHeadersCount;
+
+    // Storage for amzHeaders (the +256 is for x-amz-acl and x-amz-date)
+    char amzHeadersRaw[COMPACTED_METADATA_BUFFER_SIZE + 256 + 1];
+
+    // Canonicalized x-amz- headers
+    string_multibuffer(canonicalizedAmzHeaders,
+                       COMPACTED_METADATA_BUFFER_SIZE + 256 + 1);
+
+    // URL-Encoded key
+    char urlEncodedKey[MAX_URLENCODED_KEY_SIZE + 1];
+
+    // Canonicalized resource
+    char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE + 1];
+
+    // Cache-Control header (or empty)
+    char cacheControlHeader[128];
+
+    // Content-Type header (or empty)
+    char contentTypeHeader[128];
+
+    // Content-MD5 header (or empty)
+    char md5Header[128];
+
+    // Content-Disposition header (or empty)
+    char contentDispositionHeader[128];
+
+    // Content-Encoding header (or empty)
+    char contentEncodingHeader[128];
+
+    // Expires header (or empty)
+    char expiresHeader[128];
+
+    // If-Modified-Since header
+    char ifModifiedSinceHeader[128];
+
+    // If-Unmodified-Since header
+    char ifUnmodifiedSinceHeader[128];
+
+    // If-Match header
+    char ifMatchHeader[128];
+
+    // If-None-Match header
+    char ifNoneMatchHeader[128];
+
+    // Range header
+    char rangeHeader[128];
+
+    // Authorization header
+    char authorizationHeader[128];
+} RequestComputedValues;
+
+
+// Called whenever we detect that the request headers have been completely
+// processed; which happens either when we get our first read/write callback,
+// or the request is finished being procesed.  Returns nonzero on success,
+// zero on failure.
+static void request_headers_done(Request *request)
+{
+    if (request->propertiesCallbackMade) {
+        return;
+    }
+
+    request->propertiesCallbackMade = 1;
+
+    // Get the http response code
+    long httpResponseCode;
+    request->httpResponseCode = 0;
+    if (curl_easy_getinfo(request->curl, CURLINFO_RESPONSE_CODE, 
+                          &httpResponseCode) != CURLE_OK) {
+        // Not able to get the HTTP response code - error
+        request->status = S3StatusInternalError;
+        return;
+    }
+    else {
+        request->httpResponseCode = httpResponseCode;
+    }
+
+    response_headers_handler_done(&(request->responseHeadersHandler), 
+                                  request->curl);
+
+    // Only make the callback if it was a successful request; otherwise we're
+    // returning information about the error response itself
+    if (request->propertiesCallback &&
+        (request->httpResponseCode >= 200) &&
+        (request->httpResponseCode <= 299)) {
+        request->status = (*(request->propertiesCallback))
+            (&(request->responseHeadersHandler.responseProperties), 
+             request->callbackData);
+    }
+}
+
+
+static size_t curl_header_func(void *ptr, size_t size, size_t nmemb,
+                               void *data)
+{
+    Request *request = (Request *) data;
+
+    int len = size * nmemb;
+
+    response_headers_handler_add
+        (&(request->responseHeadersHandler), (char *) ptr, len);
+
+    return len;
+}
+
+
+static size_t curl_read_func(void *ptr, size_t size, size_t nmemb, void *data)
+{
+    Request *request = (Request *) data;
+
+    int len = size * nmemb;
+
+    request_headers_done(request);
+
+    if (request->status != S3StatusOK) {
+        return CURL_READFUNC_ABORT;
+    }
+
+    // If there is no data callback, or the data callback has already returned
+    // contentLength bytes, return 0;
+    if (!request->toS3Callback || !request->toS3CallbackBytesRemaining) {
+        return 0;
+    }
+    
+    // Don't tell the callback that we are willing to accept more data than we
+    // really are
+    if (len > request->toS3CallbackBytesRemaining) {
+        len = request->toS3CallbackBytesRemaining;
+    }
+
+    // Otherwise, make the data callback
+    int ret = (*(request->toS3Callback))
+        (len, (char *) ptr, request->callbackData);
+    if (ret < 0) {
+        request->status = S3StatusAbortedByCallback;
+        return CURL_READFUNC_ABORT;
+    }
+    else {
+        if (ret > request->toS3CallbackBytesRemaining) {
+            ret = request->toS3CallbackBytesRemaining;
+        }
+        request->toS3CallbackBytesRemaining -= ret;
+        return ret;
+    }
+}
+
+
+static size_t curl_write_func(void *ptr, size_t size, size_t nmemb,
+                              void *data)
+{
+    Request *request = (Request *) data;
+
+    int len = size * nmemb;
+
+    request_headers_done(request);
+
+    if (request->status != S3StatusOK) {
+        return 0;
+    }
+
+    // On HTTP error, we expect to parse an HTTP error response
+    if ((request->httpResponseCode < 200) || 
+        (request->httpResponseCode > 299)) {
+        request->status = error_parser_add
+            (&(request->errorParser), (char *) ptr, len);
+    }
+    // If there was a callback registered, make it
+    else if (request->fromS3Callback) {
+        request->status = (*(request->fromS3Callback))
+            (len, (char *) ptr, request->callbackData);
+    }
+    // Else, consider this an error - S3 has sent back data when it was not
+    // expected
+    else {
+        request->status = S3StatusInternalError;
+    }
+
+    return ((request->status == S3StatusOK) ? len : 0);
+}
+
+
+// This function 'normalizes' all x-amz-meta headers provided in
+// params->requestHeaders, which means it removes all whitespace from
+// them such that they all look exactly like this:
+// x-amz-meta-${NAME}: ${VALUE}
+// It also adds the x-amz-acl, x-amz-copy-source, and x-amz-metadata-directive
+// headers if necessary, and always adds the x-amz-date header.  It copies the
+// raw string values into params->amzHeadersRaw, and creates an array of
+// string pointers representing these headers in params->amzHeaders (and also
+// sets params->amzHeadersCount to be the count of the total number of x-amz-
+// headers thus created).
+static S3Status compose_amz_headers(const RequestParams *params,
+                                    RequestComputedValues *values)
+{
+    const S3PutProperties *properties = params->putProperties;
+
+    values->amzHeadersCount = 0;
+    values->amzHeadersRaw[0] = 0;
+    int len = 0;
+
+    // Append a header to amzHeaders, trimming whitespace from the end.
+    // Does NOT trim whitespace from the beginning.
+#define headers_append(isNewHeader, format, ...)                        \
+    do {                                                                \
+        if (isNewHeader) {                                              \
+            values->amzHeaders[values->amzHeadersCount++] =             \
+                &(values->amzHeadersRaw[len]);                          \
+        }                                                               \
+        len += snprintf(&(values->amzHeadersRaw[len]),                  \
+                        sizeof(values->amzHeadersRaw) - len,            \
+                        format, __VA_ARGS__);                           \
+        if (len >= (int) sizeof(values->amzHeadersRaw)) {               \
+            return S3StatusMetaDataHeadersTooLong;                      \
+        }                                                               \
+        while ((len > 0) && (values->amzHeadersRaw[len - 1] == ' ')) {  \
+            len--;                                                      \
+        }                                                               \
+        values->amzHeadersRaw[len++] = 0;                               \
+    } while (0)
+
+#define header_name_tolower_copy(str, l)                                \
+    do {                                                                \
+        values->amzHeaders[values->amzHeadersCount++] =                 \
+            &(values->amzHeadersRaw[len]);                              \
+        if ((len + l) >= (int) sizeof(values->amzHeadersRaw)) {         \
+            return S3StatusMetaDataHeadersTooLong;                      \
+        }                                                               \
+        int todo = l;                                                   \
+        while (todo--) {                                                \
+            if ((*(str) >= 'A') && (*(str) <= 'Z')) {                   \
+                values->amzHeadersRaw[len++] = 'a' + (*(str) - 'A');    \
+            }                                                           \
+            else {                                                      \
+                values->amzHeadersRaw[len++] = *(str);                  \
+            }                                                           \
+            (str)++;                                                    \
+        }                                                               \
+    } while (0)
+
+    // Check and copy in the x-amz-meta headers
+    if (properties) {
+        int i;
+        for (i = 0; i < properties->metaDataCount; i++) {
+            const S3NameValue *property = &(properties->metaData[i]);
+            char headerName[S3_MAX_METADATA_SIZE - sizeof(": v")];
+            int l = snprintf(headerName, sizeof(headerName),
+                             S3_METADATA_HEADER_NAME_PREFIX "%s",
+                             property->name);
+            char *hn = headerName;
+            header_name_tolower_copy(hn, l);
+            // Copy in the value
+            headers_append(0, ": %s", property->value);
+        }
+
+        // Add the x-amz-acl header, if necessary
+        const char *cannedAclString;
+        switch (params->putProperties->cannedAcl) {
+        case S3CannedAclPrivate:
+            cannedAclString = 0;
+            break;
+        case S3CannedAclPublicRead:
+            cannedAclString = "public-read";
+            break;
+        case S3CannedAclPublicReadWrite:
+            cannedAclString = "public-read-write";
+            break;
+        default: // S3CannedAclAuthenticatedRead
+            cannedAclString = "authenticated-read";
+            break;
+        }
+        if (cannedAclString) {
+            headers_append(1, "x-amz-acl: %s", cannedAclString);
+        }
+    }
+
+    // Add the x-amz-date header
+    time_t now = time(NULL);
+    char date[64];
+    strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&now));
+    headers_append(1, "x-amz-date: %s", date);
+
+    if (params->httpRequestType == HttpRequestTypeCOPY) {
+        // Add the x-amz-copy-source header
+        if (params->copySourceBucketName && params->copySourceBucketName[0] &&
+            params->copySourceKey && params->copySourceKey[0]) {
+            headers_append(1, "x-amz-copy-source: /%s/%s",
+                           params->copySourceBucketName,
+                           params->copySourceKey);
+        }
+        // And the x-amz-metadata-directive header
+        if (params->putProperties) {
+            headers_append(1, "%s", "x-amz-metadata-directive: REPLACE");
+        }
+    }
+
+    return S3StatusOK;
+}
+
+
+// Composes the other headers
+static S3Status compose_standard_headers(const RequestParams *params,
+                                         RequestComputedValues *values)
+{
+
+#define do_put_header(fmt, sourceField, destField, badError, tooLongError)  \
+    do {                                                                    \
+        if (params->putProperties &&                                        \
+            params->putProperties-> sourceField &&                          \
+            params->putProperties-> sourceField[0]) {                       \
+            /* Skip whitespace at beginning of val */                       \
+            const char *val = params->putProperties-> sourceField;          \
+            while (*val && is_blank(*val)) {                                \
+                val++;                                                      \
+            }                                                               \
+            if (!*val) {                                                    \
+                return badError;                                            \
+            }                                                               \
+            /* Compose header, make sure it all fit */                      \
+            int len = snprintf(values-> destField,                          \
+                               sizeof(values-> destField), fmt, val);       \
+            if (len >= (int) sizeof(values-> destField)) {                  \
+                return tooLongError;                                        \
+            }                                                               \
+            /* Now remove the whitespace at the end */                      \
+            while (is_blank(values-> destField[len])) {                     \
+                len--;                                                      \
+            }                                                               \
+            values-> destField[len] = 0;                                    \
+        }                                                                   \
+        else {                                                              \
+            values-> destField[0] = 0;                                      \
+        }                                                                   \
+    } while (0)
+
+#define do_get_header(fmt, sourceField, destField, badError, tooLongError)  \
+    do {                                                                    \
+        if (params->getConditions &&                                        \
+            params->getConditions-> sourceField &&                          \
+            params->getConditions-> sourceField[0]) {                       \
+            /* Skip whitespace at beginning of val */                       \
+            const char *val = params->getConditions-> sourceField;          \
+            while (*val && is_blank(*val)) {                                \
+                val++;                                                      \
+            }                                                               \
+            if (!*val) {                                                    \
+                return badError;                                            \
+            }                                                               \
+            /* Compose header, make sure it all fit */                      \
+            int len = snprintf(values-> destField,                          \
+                               sizeof(values-> destField), fmt, val);       \
+            if (len >= (int) sizeof(values-> destField)) {                  \
+                return tooLongError;                                        \
+            }                                                               \
+            /* Now remove the whitespace at the end */                      \
+            while (is_blank(values-> destField[len])) {                     \
+                len--;                                                      \
+            }                                                               \
+            values-> destField[len] = 0;                                    \
+        }                                                                   \
+        else {                                                              \
+            values-> destField[0] = 0;                                      \
+        }                                                                   \
+    } while (0)
+
+    // Cache-Control
+    do_put_header("Cache-Control: %s", cacheControl, cacheControlHeader,
+                  S3StatusBadCacheControl, S3StatusCacheControlTooLong);
+    
+    // ContentType
+    do_put_header("Content-Type: %s", contentType, contentTypeHeader,
+                  S3StatusBadContentType, S3StatusContentTypeTooLong);
+
+    // MD5
+    do_put_header("Content-MD5: %s", md5, md5Header, S3StatusBadMD5,
+                  S3StatusMD5TooLong);
+
+    // Content-Disposition
+    do_put_header("Content-Disposition: attachment; filename=\"%s\"",
+                  contentDispositionFilename, contentDispositionHeader,
+                  S3StatusBadContentDispositionFilename,
+                  S3StatusContentDispositionFilenameTooLong);
+    
+    // ContentEncoding
+    do_put_header("Content-Encoding: %s", contentEncoding, 
+                  contentEncodingHeader, S3StatusBadContentEncoding,
+                  S3StatusContentEncodingTooLong);
+    
+    // Expires
+    if (params->putProperties && (params->putProperties->expires >= 0)) {
+        time_t t = (time_t) params->putProperties->expires;
+        strftime(values->expiresHeader, sizeof(values->expiresHeader),
+                 "Expires: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
+    }
+    else {
+        values->expiresHeader[0] = 0;
+    }
+
+    // If-Modified-Since
+    if (params->getConditions &&
+        (params->getConditions->ifModifiedSince >= 0)) {
+        time_t t = (time_t) params->getConditions->ifModifiedSince;
+        strftime(values->ifModifiedSinceHeader,
+                 sizeof(values->ifModifiedSinceHeader),
+                 "If-Modified-Since: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
+    }
+    else {
+        values->ifModifiedSinceHeader[0] = 0;
+    }
+
+    // If-Unmodified-Since header
+    if (params->getConditions &&
+        (params->getConditions->ifNotModifiedSince >= 0)) {
+        time_t t = (time_t) params->getConditions->ifNotModifiedSince;
+        strftime(values->ifUnmodifiedSinceHeader,
+                 sizeof(values->ifUnmodifiedSinceHeader),
+                 "If-Unmodified-Since: %a, %d %b %Y %H:%M:%S UTC", gmtime(&t));
+    }
+    else {
+        values->ifUnmodifiedSinceHeader[0] = 0;
+    }
+    
+    // If-Match header
+    do_get_header("If-Match: %s", ifMatchETag, ifMatchHeader,
+                  S3StatusBadIfMatchETag, S3StatusIfMatchETagTooLong);
+    
+    // If-None-Match header
+    do_get_header("If-None-Match: %s", ifNotMatchETag, ifNoneMatchHeader,
+                  S3StatusBadIfNotMatchETag, 
+                  S3StatusIfNotMatchETagTooLong);
+    
+    // Range header
+    if (params->startByte || params->byteCount) {
+        if (params->byteCount) {
+            snprintf(values->rangeHeader, sizeof(values->rangeHeader),
+                     "Range: bytes=%llu-%llu", 
+                     (unsigned long long) params->startByte,
+                     (unsigned long long) (params->startByte + 
+                                           params->byteCount - 1));
+        }
+        else {
+            snprintf(values->rangeHeader, sizeof(values->rangeHeader),
+                     "Range: bytes=%llu-", 
+                     (unsigned long long) params->startByte);
+        }
+    }
+    else {
+        values->rangeHeader[0] = 0;
+    }
+
+    return S3StatusOK;
+}
+
+
+// URL encodes the params->key value into params->urlEncodedKey
+static S3Status encode_key(const RequestParams *params,
+                           RequestComputedValues *values)
+{
+    return (urlEncode(values->urlEncodedKey, params->key, S3_MAX_KEY_SIZE) ?
+            S3StatusOK : S3StatusUriTooLong);
+}
+
+
+// Simple comparison function for comparing two HTTP header names that are
+// embedded within an HTTP header line, returning true if header1 comes
+// before header2 alphabetically, false if not
+static int headerle(const char *header1, const char *header2)
+{
+    while (1) {
+        if (*header1 == ':') {
+            return (*header2 != ':');
+        }
+        else if (*header2 == ':') {
+            return 0;
+        }
+        else if (*header2 < *header1) {
+            return 0;
+        }
+        else if (*header2 > *header1) {
+            return 1;
+        }
+        header1++, header2++;
+    }
+}
+
+
+// Replace this with merge sort eventually, it's the best stable sort.  But
+// since typically the number of elements being sorted is small, it doesn't
+// matter that much which sort is used, and gnome sort is the world's simplest
+// stable sort.  Added a slight twist to the standard gnome_sort - don't go
+// forward +1, go forward to the last highest index considered.  This saves
+// all the string comparisons that would be done "going forward", and thus
+// only does the necessary string comparisons to move values back into their
+// sorted position.
+static void header_gnome_sort(const char **headers, int size)
+{
+    int i = 0, last_highest = 0;
+
+    while (i < size) {
+        if ((i == 0) || headerle(headers[i - 1], headers[i])) {
+            i = ++last_highest;
+        }
+        else {
+            const char *tmp = headers[i];
+            headers[i] = headers[i - 1];
+            headers[--i] = tmp;
+        }
+    }
+}
+
+
+// Canonicalizes the x-amz- headers into the canonicalizedAmzHeaders buffer
+static void canonicalize_amz_headers(RequestComputedValues *values)
+{
+    // Make a copy of the headers that will be sorted
+    const char *sortedHeaders[S3_MAX_METADATA_COUNT];
+
+    memcpy(sortedHeaders, values->amzHeaders,
+           (values->amzHeadersCount * sizeof(sortedHeaders[0])));
+
+    // Now sort these
+    header_gnome_sort(sortedHeaders, values->amzHeadersCount);
+
+    // Now copy this sorted list into the buffer, all the while:
+    // - folding repeated headers into single lines, and
+    // - folding multiple lines
+    // - removing the space after the colon
+    int lastHeaderLen = 0, i;
+    char *buffer = values->canonicalizedAmzHeaders;
+    for (i = 0; i < values->amzHeadersCount; i++) {
+        const char *header = sortedHeaders[i];
+        const char *c = header;
+        // If the header names are the same, append the next value
+        if ((i > 0) && 
+            !strncmp(header, sortedHeaders[i - 1], lastHeaderLen)) {
+            // Replacing the previous newline with a comma
+            *(buffer - 1) = ',';
+            // Skip the header name and space
+            c += (lastHeaderLen + 1);
+        }
+        // Else this is a new header
+        else {
+            // Copy in everything up to the space in the ": "
+            while (*c != ' ') {
+                *buffer++ = *c++;
+            }
+            // Save the header len since it's a new header
+            lastHeaderLen = c - header;
+            // Skip the space
+            c++;
+        }
+        // Now copy in the value, folding the lines
+        while (*c) {
+            // If c points to a \r\n[whitespace] sequence, then fold
+            // this newline out
+            if ((*c == '\r') && (*(c + 1) == '\n') && is_blank(*(c + 2))) {
+                c += 3;
+                while (is_blank(*c)) {
+                    c++;
+                }
+                // Also, what has most recently been copied into buffer amy
+                // have been whitespace, and since we're folding whitespace
+                // out around this newline sequence, back buffer up over
+                // any whitespace it contains
+                while (is_blank(*(buffer - 1))) {
+                    buffer--;
+                }
+                continue;
+            }
+            *buffer++ = *c++;
+        }
+        // Finally, add the newline
+        *buffer++ = '\n';
+    }
+
+    // Terminate the buffer
+    *buffer = 0;
+}
+
+
+// Canonicalizes the resource into params->canonicalizedResource
+static void canonicalize_resource(const char *bucketName,
+                                  const char *subResource,
+                                  const char *urlEncodedKey,
+                                  char *buffer)
+{
+    int len = 0;
+
+    *buffer = 0;
+
+#define append(str) len += sprintf(&(buffer[len]), "%s", str)
+
+    if (bucketName && bucketName[0]) {
+        buffer[len++] = '/';
+        append(bucketName);
+    }
+
+    append("/");
+
+    if (urlEncodedKey && urlEncodedKey[0]) {
+        append(urlEncodedKey);
+    }
+
+    if (subResource && subResource[0]) {
+        append("?");
+        append(subResource);
+    }
+}
+
+
+// Convert an HttpRequestType to an HTTP Verb string
+static const char *http_request_type_to_verb(HttpRequestType requestType)
+{
+    switch (requestType) {
+    case HttpRequestTypeGET:
+        return "GET";
+    case HttpRequestTypeHEAD:
+        return "HEAD";
+    case HttpRequestTypePUT:
+    case HttpRequestTypeCOPY:
+        return "PUT";
+    default: // HttpRequestTypeDELETE
+        return "DELETE";
+    }
+}
+
+
+// Composes the Authorization header for the request
+static S3Status compose_auth_header(const RequestParams *params,
+                                    RequestComputedValues *values)
+{
+    // We allow for:
+    // 17 bytes for HTTP-Verb + \n
+    // 129 bytes for Content-MD5 + \n
+    // 129 bytes for Content-Type + \n
+    // 1 byte for empty Date + \n
+    // CanonicalizedAmzHeaders & CanonicalizedResource
+    char signbuf[17 + 129 + 129 + 1 + 
+                 (sizeof(values->canonicalizedAmzHeaders) - 1) +
+                 (sizeof(values->canonicalizedResource) - 1) + 1];
+    int len = 0;
+
+#define signbuf_append(format, ...)                             \
+    len += snprintf(&(signbuf[len]), sizeof(signbuf) - len,     \
+                    format, __VA_ARGS__)
+
+    signbuf_append
+        ("%s\n", http_request_type_to_verb(params->httpRequestType));
+
+    // For MD5 and Content-Type, use the value in the actual header, because
+    // it's already been trimmed
+    signbuf_append("%s\n", values->md5Header[0] ? 
+                   &(values->md5Header[sizeof("Content-MD5: ") - 1]) : "");
+
+    signbuf_append
+        ("%s\n", values->contentTypeHeader[0] ? 
+         &(values->contentTypeHeader[sizeof("Content-Type: ") - 1]) : "");
+
+    signbuf_append("%s", "\n"); // Date - we always use x-amz-date
+
+    signbuf_append("%s", values->canonicalizedAmzHeaders);
+
+    signbuf_append("%s", values->canonicalizedResource);
+
+    // Generate an HMAC-SHA-1 of the signbuf
+    unsigned char hmac[20];
+
+    HMAC_SHA1(hmac, (unsigned char *) params->bucketContext.secretAccessKey,
+              strlen(params->bucketContext.secretAccessKey),
+              (unsigned char *) signbuf, len);
+
+    // Now base-64 encode the results
+    char b64[((20 + 1) * 4) / 3];
+    int b64Len = base64Encode(hmac, 20, b64);
+    
+    snprintf(values->authorizationHeader, sizeof(values->authorizationHeader),
+             "Authorization: AWS %s:%.*s", params->bucketContext.accessKeyId,
+             b64Len, b64);
+
+    return S3StatusOK;
+}
+
+
+// Compose the URI to use for the request given the request parameters
+static S3Status compose_uri(char *buffer, int bufferSize,
+                            const S3BucketContext *bucketContext,
+                            const char *urlEncodedKey,
+                            const char *subResource, const char *queryParams)
+{
+    int len = 0;
+    
+#define uri_append(fmt, ...)                                                 \
+    do {                                                                     \
+        len += snprintf(&(buffer[len]), bufferSize - len, fmt, __VA_ARGS__); \
+        if (len >= bufferSize) {                                             \
+            return S3StatusUriTooLong;                                       \
+        }                                                                    \
+    } while (0)
+
+    uri_append("http%s://", 
+               (bucketContext->protocol == S3ProtocolHTTP) ? "" : "s");
+
+    const char *hostName = 
+        bucketContext->hostName ? bucketContext->hostName : defaultHostNameG;
+
+    if (bucketContext->bucketName && 
+        bucketContext->bucketName[0]) {
+        if (bucketContext->uriStyle == S3UriStyleVirtualHost) {
+            uri_append("%s.%s", bucketContext->bucketName, hostName);
+        }
+        else {
+            uri_append("%s/%s", hostName, bucketContext->bucketName);
+        }
+    }
+    else {
+        uri_append("%s", hostName);
+    }
+
+    uri_append("%s", "/");
+
+    uri_append("%s", urlEncodedKey);
+    
+    if (subResource && subResource[0]) {
+        uri_append("?%s", subResource);
+    }
+    
+    if (queryParams) {
+        uri_append("%s%s", (subResource && subResource[0]) ? "&" : "?",
+                   queryParams);
+    }
+    
+    return S3StatusOK;
+}
+
+
+// Sets up the curl handle given the completely computed RequestParams
+static S3Status setup_curl(Request *request,
+                           const RequestParams *params,
+                           const RequestComputedValues *values)
+{
+    CURLcode status;
+
+#define curl_easy_setopt_safe(opt, val)                                 \
+    if ((status = curl_easy_setopt                                      \
+         (request->curl, opt, val)) != CURLE_OK) {                      \
+        return S3StatusFailedToInitializeRequest;                       \
+    }
+
+    // Debugging only
+    // curl_easy_setopt_safe(CURLOPT_VERBOSE, 1);
+    
+    // Set private data to request for the benefit of S3RequestContext
+    curl_easy_setopt_safe(CURLOPT_PRIVATE, request);
+    
+    // Set header callback and data
+    curl_easy_setopt_safe(CURLOPT_HEADERDATA, request);
+    curl_easy_setopt_safe(CURLOPT_HEADERFUNCTION, &curl_header_func);
+    
+    // Set read callback, data, and readSize
+    curl_easy_setopt_safe(CURLOPT_READFUNCTION, &curl_read_func);
+    curl_easy_setopt_safe(CURLOPT_READDATA, request);
+    
+    // Set write callback and data
+    curl_easy_setopt_safe(CURLOPT_WRITEFUNCTION, &curl_write_func);
+    curl_easy_setopt_safe(CURLOPT_WRITEDATA, request);
+
+    // Ask curl to parse the Last-Modified header.  This is easier than
+    // parsing it ourselves.
+    curl_easy_setopt_safe(CURLOPT_FILETIME, 1);
+
+    // Curl docs suggest that this is necessary for multithreaded code.
+    // However, it also points out that DNS timeouts will not be honored
+    // during DNS lookup, which can be worked around by using the c-ares
+    // library, which we do not do yet.
+    curl_easy_setopt_safe(CURLOPT_NOSIGNAL, 1);
+
+    // Turn off Curl's built-in progress meter
+    curl_easy_setopt_safe(CURLOPT_NOPROGRESS, 1);
+
+    // xxx todo - support setting the proxy for Curl to use (can't use https
+    // for proxies though)
+
+    // xxx todo - support setting the network interface for Curl to use
+
+    // I think this is useful - we don't need interactive performance, we need
+    // to complete large operations quickly
+    curl_easy_setopt_safe(CURLOPT_TCP_NODELAY, 1);
+    
+    // Don't use Curl's 'netrc' feature
+    curl_easy_setopt_safe(CURLOPT_NETRC, CURL_NETRC_IGNORED);
+
+    // Don't verify S3's certificate, there are known to be issues with
+    // them sometimes
+    // xxx todo - support an option for verifying the S3 CA (default false)
+    curl_easy_setopt_safe(CURLOPT_SSL_VERIFYPEER, 0);
+
+    // Follow any redirection directives that S3 sends
+    curl_easy_setopt_safe(CURLOPT_FOLLOWLOCATION, 1);
+
+    // A safety valve in case S3 goes bananas with redirects
+    curl_easy_setopt_safe(CURLOPT_MAXREDIRS, 10);
+
+    // Set the User-Agent; maybe Amazon will track these?
+    curl_easy_setopt_safe(CURLOPT_USERAGENT, userAgentG);
+
+    // Set the low speed limit and time; we abort transfers that stay at
+    // less than 1K per second for more than 15 seconds.
+    // xxx todo - make these configurable
+    // xxx todo - allow configurable max send and receive speed
+    curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, 1024);
+    curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, 15);
+
+    // Append standard headers
+#define append_standard_header(fieldName)                               \
+    if (values-> fieldName [0]) {                                       \
+        request->headers = curl_slist_append(request->headers,          \
+                                             values-> fieldName);       \
+    }
+
+    // Would use CURLOPT_INFILESIZE_LARGE, but it is buggy in libcurl
+    if (params->httpRequestType == HttpRequestTypePUT) {
+        char header[256];
+        snprintf(header, sizeof(header), "Content-Length: %llu",
+                 (unsigned long long) params->toS3CallbackTotalSize);
+        request->headers = curl_slist_append(request->headers, header);
+        request->headers = curl_slist_append(request->headers, 
+                                             "Transfer-Encoding:");
+    }
+    else if (params->httpRequestType == HttpRequestTypeCOPY) {
+        request->headers = curl_slist_append(request->headers, 
+                                             "Transfer-Encoding:");
+    }
+    
+    append_standard_header(cacheControlHeader);
+    append_standard_header(contentTypeHeader);
+    append_standard_header(md5Header);
+    append_standard_header(contentDispositionHeader);
+    append_standard_header(contentEncodingHeader);
+    append_standard_header(expiresHeader);
+    append_standard_header(ifModifiedSinceHeader);
+    append_standard_header(ifUnmodifiedSinceHeader);
+    append_standard_header(ifMatchHeader);
+    append_standard_header(ifNoneMatchHeader);
+    append_standard_header(rangeHeader);
+    append_standard_header(authorizationHeader);
+
+    // Append x-amz- headers
+    int i;
+    for (i = 0; i < values->amzHeadersCount; i++) {
+        request->headers = 
+            curl_slist_append(request->headers, values->amzHeaders[i]);
+    }
+
+    // Set the HTTP headers
+    curl_easy_setopt_safe(CURLOPT_HTTPHEADER, request->headers);
+
+    // Set URI
+    curl_easy_setopt_safe(CURLOPT_URL, request->uri);
+
+    // Set request type.
+    switch (params->httpRequestType) {
+    case HttpRequestTypeHEAD:
+    curl_easy_setopt_safe(CURLOPT_NOBODY, 1);
+        break;
+    case HttpRequestTypePUT:
+    case HttpRequestTypeCOPY:
+        curl_easy_setopt_safe(CURLOPT_UPLOAD, 1);
+        break;
+    case HttpRequestTypeDELETE:
+    curl_easy_setopt_safe(CURLOPT_CUSTOMREQUEST, "DELETE");
+        break;
+    default: // HttpRequestTypeGET
+        break;
+    }
+    
+    return S3StatusOK;
+}
+
+
+static void request_deinitialize(Request *request)
+{
+    if (request->headers) {
+        curl_slist_free_all(request->headers);
+    }
+    
+    error_parser_deinitialize(&(request->errorParser));
+
+    // curl_easy_reset prevents connections from being re-used for some
+    // reason.  This makes HTTP Keep-Alive meaningless and is very bad for
+    // performance.  But it is necessary to allow curl to work properly.
+    // xxx todo figure out why
+    curl_easy_reset(request->curl);
+}
+
+
+static S3Status request_get(const RequestParams *params, 
+                            const RequestComputedValues *values,
+                            Request **reqReturn)
+{
+    Request *request = 0;
+    
+    // Try to get one from the request stack.  We hold the lock for the
+    // shortest time possible here.
+    pthread_mutex_lock(&requestStackMutexG);
+
+    if (requestStackCountG) {
+        request = requestStackG[--requestStackCountG];
+    }
+    
+    pthread_mutex_unlock(&requestStackMutexG);
+
+    // If we got one, deinitialize it for re-use
+    if (request) {
+        request_deinitialize(request);
+    }
+    // Else there wasn't one available in the request stack, so create one
+    else {
+        if (!(request = (Request *) malloc(sizeof(Request)))) {
+            return S3StatusOutOfMemory;
+        }
+        if (!(request->curl = curl_easy_init())) {
+            free(request);
+            return S3StatusFailedToInitializeRequest;
+        }
+    }
+
+    // Initialize the request
+    request->prev = 0;
+    request->next = 0;
+
+    // Request status is initialized to no error, will be updated whenever
+    // an error occurs
+    request->status = S3StatusOK;
+
+    S3Status status;
+                        
+    // Start out with no headers
+    request->headers = 0;
+
+    // Compute the URL
+    if ((status = compose_uri
+         (request->uri, sizeof(request->uri), 
+          &(params->bucketContext), values->urlEncodedKey,
+          params->subResource, params->queryParams)) != S3StatusOK) {
+        curl_easy_cleanup(request->curl);
+        free(request);
+        return status;
+    }
+
+    // Set all of the curl handle options
+    if ((status = setup_curl(request, params, values)) != S3StatusOK) {
+        curl_easy_cleanup(request->curl);
+        free(request);
+        return status;
+    }
+
+    request->propertiesCallback = params->propertiesCallback;
+
+    request->toS3Callback = params->toS3Callback;
+
+    request->toS3CallbackBytesRemaining = params->toS3CallbackTotalSize;
+
+    request->fromS3Callback = params->fromS3Callback;
+
+    request->completeCallback = params->completeCallback;
+
+    request->callbackData = params->callbackData;
+
+    response_headers_handler_initialize(&(request->responseHeadersHandler));
+
+    request->propertiesCallbackMade = 0;
+    
+    error_parser_initialize(&(request->errorParser));
+
+    *reqReturn = request;
+    
+    return S3StatusOK;
+}
+
+
+static void request_destroy(Request *request)
+{
+    request_deinitialize(request);
+    curl_easy_cleanup(request->curl);
+    free(request);
+}
+
+
+static void request_release(Request *request)
+{
+    pthread_mutex_lock(&requestStackMutexG);
+
+    // If the request stack is full, destroy this one
+    if (requestStackCountG == REQUEST_STACK_SIZE) {
+        pthread_mutex_unlock(&requestStackMutexG);
+        request_destroy(request);
+    }
+    // Else put this one at the front of the request stack; we do this because
+    // we want the most-recently-used curl handle to be re-used on the next
+    // request, to maximize our chances of re-using a TCP connection before it
+    // times out
+    else {
+        requestStackG[requestStackCountG++] = request;
+        pthread_mutex_unlock(&requestStackMutexG);
+    }
+}
+
+
+S3Status request_api_initialize(const char *userAgentInfo, int flags,
+                                const char *defaultHostName)
+{
+    if (curl_global_init(CURL_GLOBAL_ALL & 
+                         ~((flags & S3_INIT_WINSOCK) ? 0 : CURL_GLOBAL_WIN32))
+        != CURLE_OK) {
+        return S3StatusInternalError;
+    }
+
+    if (!defaultHostName) {
+        defaultHostName = S3_DEFAULT_HOSTNAME;
+    }
+
+    if (snprintf(defaultHostNameG, S3_MAX_HOSTNAME_SIZE, 
+                 "%s", defaultHostName) >= S3_MAX_HOSTNAME_SIZE) {
+        return S3StatusUriTooLong;
+    }
+
+    pthread_mutex_init(&requestStackMutexG, 0);
+
+    requestStackCountG = 0;
+
+    if (!userAgentInfo || !*userAgentInfo) {
+        userAgentInfo = "Unknown";
+    }
+
+    char platform[96];
+    struct utsname utsn;
+    if (uname(&utsn)) {
+        strncpy(platform, "Unknown", sizeof(platform));
+        // Because strncpy doesn't always zero terminate
+        platform[sizeof(platform) - 1] = 0;
+    }
+    else {
+        snprintf(platform, sizeof(platform), "%s%s%s", utsn.sysname, 
+                 utsn.machine[0] ? " " : "", utsn.machine);
+    }
+
+    snprintf(userAgentG, sizeof(userAgentG), 
+             "Mozilla/4.0 (Compatible; %s; libs3 %s.%s; %s)",
+             userAgentInfo, LIBS3_VER_MAJOR, LIBS3_VER_MINOR, platform);
+    
+    return S3StatusOK;
+}
+
+
+void request_api_deinitialize()
+{
+    pthread_mutex_destroy(&requestStackMutexG);
+
+    while (requestStackCountG--) {
+        request_destroy(requestStackG[requestStackCountG]);
+    }
+}
+
+
+void request_perform(const RequestParams *params, S3RequestContext *context)
+{
+    Request *request;
+    S3Status status;
+
+#define return_status(status)                                           \
+    (*(params->completeCallback))(status, 0, params->callbackData);     \
+    return
+
+    // These will hold the computed values
+    RequestComputedValues computed;
+
+    // Validate the bucket name
+    if (params->bucketContext.bucketName && 
+        ((status = S3_validate_bucket_name
+          (params->bucketContext.bucketName, 
+           params->bucketContext.uriStyle)) != S3StatusOK)) {
+        return_status(status);
+    }
+
+    // Compose the amz headers
+    if ((status = compose_amz_headers(params, &computed)) != S3StatusOK) {
+        return_status(status);
+    }
+
+    // Compose standard headers
+    if ((status = compose_standard_headers
+         (params, &computed)) != S3StatusOK) {
+        return_status(status);
+    }
+
+    // URL encode the key
+    if ((status = encode_key(params, &computed)) != S3StatusOK) {
+        return_status(status);
+    }
+
+    // Compute the canonicalized amz headers
+    canonicalize_amz_headers(&computed);
+
+    // Compute the canonicalized resource
+    canonicalize_resource(params->bucketContext.bucketName,
+                          params->subResource, computed.urlEncodedKey,
+                          computed.canonicalizedResource);
+
+    // Compose Authorization header
+    if ((status = compose_auth_header(params, &computed)) != S3StatusOK) {
+        return_status(status);
+    }
+    
+    // Get an initialized Request structure now
+    if ((status = request_get(params, &computed, &request)) != S3StatusOK) {
+        return_status(status);
+    }
+
+    // If a RequestContext was provided, add the request to the curl multi
+    if (context) {
+        CURLMcode code = curl_multi_add_handle(context->curlm, request->curl);
+        if (code == CURLM_OK) {
+            if (context->requests) {
+                request->prev = context->requests->prev;
+                request->next = context->requests;
+                context->requests->prev->next = request;
+                context->requests->prev = request;
+            }
+            else {
+                context->requests = request->next = request->prev = request;
+            }
+        }
+        else {
+            if (request->status == S3StatusOK) {
+                request->status = (code == CURLM_OUT_OF_MEMORY) ?
+                    S3StatusOutOfMemory : S3StatusInternalError;
+            }
+            request_finish(request);
+        }
+    }
+    // Else, perform the request immediately
+    else {
+        CURLcode code = curl_easy_perform(request->curl);
+        if ((code != CURLE_OK) && (request->status == S3StatusOK)) {
+            request->status = request_curl_code_to_status(code);
+        }
+        // Finish the request, ensuring that all callbacks have been made, and
+        // also releases the request
+        request_finish(request);
+    }
+}
+
+
+void request_finish(Request *request)
+{
+    // If we haven't detected this already, we now know that the headers are
+    // definitely done being read in
+    request_headers_done(request);
+    
+    // If there was no error processing the request, then possibly there was
+    // an S3 error parsed, which should be converted into the request status
+    if (request->status == S3StatusOK) {
+        error_parser_convert_status(&(request->errorParser), 
+                                    &(request->status));
+        // If there still was no error recorded, then it is possible that
+        // there was in fact an error but that there was no error XML
+        // detailing the error
+        if ((request->status == S3StatusOK) &&
+            ((request->httpResponseCode < 200) ||
+             (request->httpResponseCode > 299))) {
+            switch (request->httpResponseCode) {
+            case 0:
+                // This happens if the request never got any HTTP response
+                // headers at all, we call this a ConnectionFailed error
+                request->status = S3StatusConnectionFailed;
+                break;
+            case 100: // Some versions of libcurl erroneously set HTTP
+                      // status to this
+                break;
+            case 301:
+                request->status = S3StatusErrorPermanentRedirect;
+                break;
+            case 307:
+                request->status = S3StatusHttpErrorMovedTemporarily;
+                break;
+            case 400:
+                request->status = S3StatusHttpErrorBadRequest;
+                break;
+            case 403: 
+                request->status = S3StatusHttpErrorForbidden;
+                break;
+            case 404:
+                request->status = S3StatusHttpErrorNotFound;
+                break;
+            case 405:
+                request->status = S3StatusErrorMethodNotAllowed;
+                break;
+            case 409:
+                request->status = S3StatusHttpErrorConflict;
+                break;
+            case 411:
+                request->status = S3StatusErrorMissingContentLength;
+                break;
+            case 412:
+                request->status = S3StatusErrorPreconditionFailed;
+                break;
+            case 416:
+                request->status = S3StatusErrorInvalidRange;
+                break;
+            case 500:
+                request->status = S3StatusErrorInternalError;
+                break;
+            case 501:
+                request->status = S3StatusErrorNotImplemented;
+                break;
+            case 503:
+                request->status = S3StatusErrorSlowDown;
+                break;
+            default:
+                request->status = S3StatusHttpErrorUnknown;
+                break;
+            }
+        }
+    }
+
+    (*(request->completeCallback))
+        (request->status, &(request->errorParser.s3ErrorDetails),
+         request->callbackData);
+
+    request_release(request);
+}
+
+
+S3Status request_curl_code_to_status(CURLcode code)
+{
+    switch (code) {
+    case CURLE_OUT_OF_MEMORY:
+        return S3StatusOutOfMemory;
+    case CURLE_COULDNT_RESOLVE_PROXY:
+    case CURLE_COULDNT_RESOLVE_HOST:
+        return S3StatusNameLookupError;
+    case CURLE_COULDNT_CONNECT:
+        return S3StatusFailedToConnect;
+    case CURLE_WRITE_ERROR:
+    case CURLE_OPERATION_TIMEDOUT:
+        return S3StatusConnectionFailed;
+    case CURLE_PARTIAL_FILE:
+        return S3StatusOK;
+    case CURLE_SSL_CACERT:
+        return S3StatusServerFailedVerification;
+    default:
+        return S3StatusInternalError;
+    }
+}
+
+
+S3Status S3_generate_authenticated_query_string
+    (char *buffer, const S3BucketContext *bucketContext,
+     const char *key, int64_t expires, const char *resource)
+{
+#define MAX_EXPIRES (((int64_t) 1 << 31) - 1)
+    // S3 seems to only accept expiration dates up to the number of seconds
+    // representably by a signed 32-bit integer
+    if (expires < 0) {
+        expires = MAX_EXPIRES;
+    }
+    else if (expires > MAX_EXPIRES) {
+        expires = MAX_EXPIRES;
+    }
+
+    // xxx todo: rework this so that it can be incorporated into shared code
+    // with request_perform().  It's really unfortunate that this code is not
+    // shared with request_perform().
+
+    // URL encode the key
+    char urlEncodedKey[S3_MAX_KEY_SIZE * 3];
+    if (key) {
+        urlEncode(urlEncodedKey, key, strlen(key));
+    }
+    else {
+        urlEncodedKey[0] = 0;
+    }
+
+    // Compute canonicalized resource
+    char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE];
+    canonicalize_resource(bucketContext->bucketName, resource, urlEncodedKey,
+                          canonicalizedResource);
+                          
+    // We allow for:
+    // 17 bytes for HTTP-Verb + \n
+    // 1 byte for empty Content-MD5 + \n
+    // 1 byte for empty Content-Type + \n
+    // 20 bytes for Expires + \n
+    // 0 bytes for CanonicalizedAmzHeaders
+    // CanonicalizedResource
+    char signbuf[17 + 1 + 1 + 1 + 20 + sizeof(canonicalizedResource) + 1];
+    int len = 0;
+
+#define signbuf_append(format, ...)                             \
+    len += snprintf(&(signbuf[len]), sizeof(signbuf) - len,     \
+                    format, __VA_ARGS__)
+
+    signbuf_append("%s\n", "GET"); // HTTP-Verb
+    signbuf_append("%s\n", ""); // Content-MD5
+    signbuf_append("%s\n", ""); // Content-Type
+    signbuf_append("%llu\n", (unsigned long long) expires);
+    signbuf_append("%s", canonicalizedResource);
+
+    // Generate an HMAC-SHA-1 of the signbuf
+    unsigned char hmac[20];
+
+    HMAC_SHA1(hmac, (unsigned char *) bucketContext->secretAccessKey,
+              strlen(bucketContext->secretAccessKey),
+              (unsigned char *) signbuf, len);
+
+    // Now base-64 encode the results
+    char b64[((20 + 1) * 4) / 3];
+    int b64Len = base64Encode(hmac, 20, b64);
+
+    // Now urlEncode that
+    char signature[sizeof(b64) * 3];
+    urlEncode(signature, b64, b64Len);
+
+    // Finally, compose the uri, with params:
+    // ?AWSAccessKeyId=xxx[&Expires=]&Signature=xxx
+    char queryParams[sizeof("AWSAccessKeyId=") + 20 + 
+                     sizeof("&Expires=") + 20 + 
+                     sizeof("&Signature=") + sizeof(signature) + 1];
+
+    sprintf(queryParams, "AWSAccessKeyId=%s&Expires=%ld&Signature=%s",
+            bucketContext->accessKeyId, (long) expires, signature);
+
+    return compose_uri(buffer, S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE,
+                       bucketContext, urlEncodedKey, resource, queryParams);
+}
diff --git a/ceph/src/libs3/src/request_context.c b/ceph/src/libs3/src/request_context.c
new file mode 100644
index 00000000..ae48e55d
--- /dev/null
+++ b/ceph/src/libs3/src/request_context.c
@@ -0,0 +1,190 @@
+/** **************************************************************************
+ * request_context.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include 
+#include "request.h"
+#include "request_context.h"
+
+
+S3Status S3_create_request_context(S3RequestContext **requestContextReturn)
+{
+    *requestContextReturn = 
+        (S3RequestContext *) malloc(sizeof(S3RequestContext));
+    
+    if (!*requestContextReturn) {
+        return S3StatusOutOfMemory;
+    }
+    
+    if (!((*requestContextReturn)->curlm = curl_multi_init())) {
+        free(*requestContextReturn);
+        return S3StatusOutOfMemory;
+    }
+
+    (*requestContextReturn)->requests = 0;
+
+    return S3StatusOK;
+}
+
+
+void S3_destroy_request_context(S3RequestContext *requestContext)
+{
+    curl_multi_cleanup(requestContext->curlm);
+
+    // For each request in the context, call back its done method with
+    // 'interrupted' status
+    Request *r = requestContext->requests, *rFirst = r;
+    
+    if (r) do {
+        r->status = S3StatusInterrupted;
+        Request *rNext = r->next;
+        request_finish(r);
+        r = rNext;
+    } while (r != rFirst);
+
+    free(requestContext);
+}
+
+
+S3Status S3_runall_request_context(S3RequestContext *requestContext)
+{
+    int requestsRemaining;
+    do {
+        fd_set readfds, writefds, exceptfds;
+        FD_ZERO(&readfds);
+        FD_ZERO(&writefds);
+        FD_ZERO(&exceptfds);
+        int maxfd;
+        S3Status status = S3_get_request_context_fdsets
+            (requestContext, &readfds, &writefds, &exceptfds, &maxfd);
+        if (status != S3StatusOK) {
+            return status;
+        }
+        // curl will return -1 if it hasn't even created any fds yet because
+        // none of the connections have started yet.  In this case, don't
+        // do the select at all, because it will wait forever; instead, just
+        // skip it and go straight to running the underlying CURL handles
+        if (maxfd != -1) {
+            int64_t timeout = S3_get_request_context_timeout(requestContext);
+            struct timeval tv = { timeout / 1000, (timeout % 1000) * 1000 };
+            select(maxfd + 1, &readfds, &writefds, &exceptfds,
+                   (timeout == -1) ? 0 : &tv);
+        }
+        status = S3_runonce_request_context(requestContext,
+                                            &requestsRemaining);
+        if (status != S3StatusOK) {
+            return status;
+        }
+    } while (requestsRemaining);
+    
+    return S3StatusOK;
+}
+
+
+S3Status S3_runonce_request_context(S3RequestContext *requestContext, 
+                                    int *requestsRemainingReturn)
+{
+    CURLMcode status;
+
+    do {
+        status = curl_multi_perform(requestContext->curlm,
+                                    requestsRemainingReturn);
+
+        switch (status) {
+        case CURLM_OK:
+        case CURLM_CALL_MULTI_PERFORM:
+            break;
+        case CURLM_OUT_OF_MEMORY:
+            return S3StatusOutOfMemory;
+        default:
+            return S3StatusInternalError;
+        }
+
+        CURLMsg *msg;
+        int junk;
+        while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) {
+            if (msg->msg != CURLMSG_DONE) {
+                return S3StatusInternalError;
+            }
+            Request *request;
+            if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, 
+                                  (char **) (char *) &request) != CURLE_OK) {
+                return S3StatusInternalError;
+            }
+            // Remove the request from the list of requests
+            if (request->prev == request->next) {
+                // It was the only one on the list
+                requestContext->requests = 0;
+            }
+            else {
+                // It doesn't matter what the order of them are, so just in
+                // case request was at the head of the list, put the one after
+                // request to the head of the list
+                requestContext->requests = request->next;
+                request->prev->next = request->next;
+                request->next->prev = request->prev;
+            }
+            if ((msg->data.result != CURLE_OK) &&
+                (request->status == S3StatusOK)) {
+                request->status = request_curl_code_to_status
+                    (msg->data.result);
+            }
+            if (curl_multi_remove_handle(requestContext->curlm, 
+                                         msg->easy_handle) != CURLM_OK) {
+                return S3StatusInternalError;
+            }
+            // Finish the request, ensuring that all callbacks have been made,
+            // and also releases the request
+            request_finish(request);
+            // Now, since a callback was made, there may be new requests 
+            // queued up to be performed immediately, so do so
+            status = CURLM_CALL_MULTI_PERFORM;
+        }
+    } while (status == CURLM_CALL_MULTI_PERFORM);
+
+    return S3StatusOK;
+}
+
+S3Status S3_get_request_context_fdsets(S3RequestContext *requestContext,
+                                       fd_set *readFdSet, fd_set *writeFdSet,
+                                       fd_set *exceptFdSet, int *maxFd)
+{
+    return ((curl_multi_fdset(requestContext->curlm, readFdSet, writeFdSet,
+                              exceptFdSet, maxFd) == CURLM_OK) ?
+            S3StatusOK : S3StatusInternalError);
+}
+
+int64_t S3_get_request_context_timeout(S3RequestContext *requestContext)
+{
+    long timeout;
+
+    if (curl_multi_timeout(requestContext->curlm, &timeout) != CURLM_OK) {
+        timeout = 0;
+    }
+    
+    return timeout;
+}
diff --git a/ceph/src/libs3/src/response_headers_handler.c b/ceph/src/libs3/src/response_headers_handler.c
new file mode 100644
index 00000000..e506ea4f
--- /dev/null
+++ b/ceph/src/libs3/src/response_headers_handler.c
@@ -0,0 +1,205 @@
+/** **************************************************************************
+ * response_headers_handler.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "response_headers_handler.h"
+
+
+void response_headers_handler_initialize(ResponseHeadersHandler *handler)
+{
+    handler->responseProperties.requestId = 0;
+    handler->responseProperties.requestId2 = 0;
+    handler->responseProperties.contentType = 0;
+    handler->responseProperties.contentLength = 0;
+    handler->responseProperties.server = 0;
+    handler->responseProperties.eTag = 0;
+    handler->responseProperties.lastModified = -1;
+    handler->responseProperties.metaDataCount = 0;
+    handler->responseProperties.metaData = 0;
+    handler->done = 0;
+    string_multibuffer_initialize(handler->responsePropertyStrings);
+    string_multibuffer_initialize(handler->responseMetaDataStrings);
+}
+
+
+void response_headers_handler_add(ResponseHeadersHandler *handler,
+                                  char *header, int len)
+{
+    S3ResponseProperties *responseProperties = &(handler->responseProperties);
+    char *end = &(header[len]);
+    
+    // Curl might call back the header function after the body has been
+    // received, for 'chunked encoded' contents.  We don't handle this as of
+    // yet, and it's not clear that it would ever be useful.
+    if (handler->done) {
+        return;
+    }
+
+    // If we've already filled up the response headers, ignore this data.
+    // This sucks, but it shouldn't happen - S3 should not be sending back
+    // really long headers.
+    if (handler->responsePropertyStringsSize == 
+        (sizeof(handler->responsePropertyStrings) - 1)) {
+        return;
+    }
+
+    // It should not be possible to have a header line less than 3 long
+    if (len < 3) {
+        return;
+    }
+
+    // Skip whitespace at beginning of header; there never should be any,
+    // but just to be safe
+    while (is_blank(*header)) {
+        header++;
+    }
+
+    // The header must end in \r\n, so skip back over it, and also over any
+    // trailing whitespace
+    end -= 3;
+    while ((end > header) && is_blank(*end)) {
+        end--;
+    }
+    if (!is_blank(*end)) {
+        end++;
+    }
+
+    if (end == header) {
+        // totally bogus
+        return;
+    }
+
+    *end = 0;
+    
+    // Find the colon to split the header up
+    char *c = header;
+    while (*c && (*c != ':')) {
+        c++;
+    }
+    
+    int namelen = c - header;
+
+    // Now walk c past the colon
+    c++;
+    // Now skip whitespace to the beginning of the value
+    while (is_blank(*c)) {
+        c++;
+    }
+
+    int valuelen = (end - c) + 1, fit;
+
+    if (!strncmp(header, "x-amz-request-id", namelen)) {
+        responseProperties->requestId = 
+            string_multibuffer_current(handler->responsePropertyStrings);
+        string_multibuffer_add(handler->responsePropertyStrings, c, 
+                               valuelen, fit);
+    }
+    else if (!strncmp(header, "x-amz-id-2", namelen)) {
+        responseProperties->requestId2 = 
+            string_multibuffer_current(handler->responsePropertyStrings);
+        string_multibuffer_add(handler->responsePropertyStrings, c, 
+                               valuelen, fit);
+    }
+    else if (!strncmp(header, "Content-Type", namelen)) {
+        responseProperties->contentType = 
+            string_multibuffer_current(handler->responsePropertyStrings);
+        string_multibuffer_add(handler->responsePropertyStrings, c, 
+                               valuelen, fit);
+    }
+    else if (!strncmp(header, "Content-Length", namelen)) {
+        handler->responseProperties.contentLength = 0;
+        while (*c) {
+            handler->responseProperties.contentLength *= 10;
+            handler->responseProperties.contentLength += (*c++ - '0');
+        }
+    }
+    else if (!strncmp(header, "Server", namelen)) {
+        responseProperties->server = 
+            string_multibuffer_current(handler->responsePropertyStrings);
+        string_multibuffer_add(handler->responsePropertyStrings, c, 
+                               valuelen, fit);
+    }
+    else if (!strncmp(header, "ETag", namelen)) {
+        responseProperties->eTag = 
+            string_multibuffer_current(handler->responsePropertyStrings);
+        string_multibuffer_add(handler->responsePropertyStrings, c, 
+                               valuelen, fit);
+    }
+    else if (!strncmp(header, S3_METADATA_HEADER_NAME_PREFIX, 
+                      sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1)) {
+        // Make sure there is room for another x-amz-meta header
+        if (handler->responseProperties.metaDataCount ==
+            sizeof(handler->responseMetaData)) {
+            return;
+        }
+        // Copy the name in
+        char *metaName = &(header[sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1]);
+        int metaNameLen = 
+            (namelen - (sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1));
+        char *copiedName = 
+            string_multibuffer_current(handler->responseMetaDataStrings);
+        string_multibuffer_add(handler->responseMetaDataStrings, metaName,
+                               metaNameLen, fit);
+        if (!fit) {
+            return;
+        }
+
+        // Copy the value in
+        char *copiedValue = 
+            string_multibuffer_current(handler->responseMetaDataStrings);
+        string_multibuffer_add(handler->responseMetaDataStrings,
+                               c, valuelen, fit);
+        if (!fit) {
+            return;
+        }
+
+        if (!handler->responseProperties.metaDataCount) {
+            handler->responseProperties.metaData = 
+                handler->responseMetaData;
+        }
+
+        S3NameValue *metaHeader = 
+            &(handler->responseMetaData
+              [handler->responseProperties.metaDataCount++]);
+        metaHeader->name = copiedName;
+        metaHeader->value = copiedValue;
+    }
+}
+
+
+void response_headers_handler_done(ResponseHeadersHandler *handler, CURL *curl)
+{
+    // Now get the last modification time from curl, since it's easiest to let
+    // curl parse it
+    time_t lastModified;
+    if (curl_easy_getinfo
+        (curl, CURLINFO_FILETIME, &lastModified) == CURLE_OK) {
+        handler->responseProperties.lastModified = lastModified;
+    }
+    
+    handler->done = 1;
+}
diff --git a/ceph/src/libs3/src/s3.c b/ceph/src/libs3/src/s3.c
new file mode 100644
index 00000000..65acc521
--- /dev/null
+++ b/ceph/src/libs3/src/s3.c
@@ -0,0 +1,2787 @@
+/** **************************************************************************
+ * s3.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+/**
+ * This is a 'driver' program that simply converts command-line input into
+ * calls to libs3 functions, and prints the results.
+ **/
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "libs3.h"
+
+// Some Windows stuff
+#ifndef FOPEN_EXTRA_FLAGS
+#define FOPEN_EXTRA_FLAGS ""
+#endif
+
+// Some Unix stuff (to work around Windows issues)
+#ifndef SLEEP_UNITS_PER_SECOND
+#define SLEEP_UNITS_PER_SECOND 1
+#endif
+
+// Also needed for Windows, because somehow MinGW doesn't define this
+extern int putenv(char *);
+
+
+// Command-line options, saved as globals ------------------------------------
+
+static int forceG = 0;
+static int showResponsePropertiesG = 0;
+static S3Protocol protocolG = S3ProtocolHTTPS;
+static S3UriStyle uriStyleG = S3UriStylePath;
+static int retriesG = 5;
+
+
+// Environment variables, saved as globals ----------------------------------
+
+static const char *accessKeyIdG = 0;
+static const char *secretAccessKeyG = 0;
+
+
+// Request results, saved as globals -----------------------------------------
+
+static int statusG = 0;
+static char errorDetailsG[4096] = { 0 };
+
+
+// Other globals -------------------------------------------------------------
+
+static char putenvBufG[256];
+
+
+// Option prefixes -----------------------------------------------------------
+
+#define LOCATION_PREFIX "location="
+#define LOCATION_PREFIX_LEN (sizeof(LOCATION_PREFIX) - 1)
+#define CANNED_ACL_PREFIX "cannedAcl="
+#define CANNED_ACL_PREFIX_LEN (sizeof(CANNED_ACL_PREFIX) - 1)
+#define PREFIX_PREFIX "prefix="
+#define PREFIX_PREFIX_LEN (sizeof(PREFIX_PREFIX) - 1)
+#define MARKER_PREFIX "marker="
+#define MARKER_PREFIX_LEN (sizeof(MARKER_PREFIX) - 1)
+#define DELIMITER_PREFIX "delimiter="
+#define DELIMITER_PREFIX_LEN (sizeof(DELIMITER_PREFIX) - 1)
+#define MAXKEYS_PREFIX "maxkeys="
+#define MAXKEYS_PREFIX_LEN (sizeof(MAXKEYS_PREFIX) - 1)
+#define FILENAME_PREFIX "filename="
+#define FILENAME_PREFIX_LEN (sizeof(FILENAME_PREFIX) - 1)
+#define CONTENT_LENGTH_PREFIX "contentLength="
+#define CONTENT_LENGTH_PREFIX_LEN (sizeof(CONTENT_LENGTH_PREFIX) - 1)
+#define CACHE_CONTROL_PREFIX "cacheControl="
+#define CACHE_CONTROL_PREFIX_LEN (sizeof(CACHE_CONTROL_PREFIX) - 1)
+#define CONTENT_TYPE_PREFIX "contentType="
+#define CONTENT_TYPE_PREFIX_LEN (sizeof(CONTENT_TYPE_PREFIX) - 1)
+#define MD5_PREFIX "md5="
+#define MD5_PREFIX_LEN (sizeof(MD5_PREFIX) - 1)
+#define CONTENT_DISPOSITION_FILENAME_PREFIX "contentDispositionFilename="
+#define CONTENT_DISPOSITION_FILENAME_PREFIX_LEN \
+    (sizeof(CONTENT_DISPOSITION_FILENAME_PREFIX) - 1)
+#define CONTENT_ENCODING_PREFIX "contentEncoding="
+#define CONTENT_ENCODING_PREFIX_LEN (sizeof(CONTENT_ENCODING_PREFIX) - 1)
+#define EXPIRES_PREFIX "expires="
+#define EXPIRES_PREFIX_LEN (sizeof(EXPIRES_PREFIX) - 1)
+#define X_AMZ_META_PREFIX "x-amz-meta-"
+#define X_AMZ_META_PREFIX_LEN (sizeof(X_AMZ_META_PREFIX) - 1)
+#define IF_MODIFIED_SINCE_PREFIX "ifModifiedSince="
+#define IF_MODIFIED_SINCE_PREFIX_LEN (sizeof(IF_MODIFIED_SINCE_PREFIX) - 1)
+#define IF_NOT_MODIFIED_SINCE_PREFIX "ifNotmodifiedSince="
+#define IF_NOT_MODIFIED_SINCE_PREFIX_LEN \
+    (sizeof(IF_NOT_MODIFIED_SINCE_PREFIX) - 1)
+#define IF_MATCH_PREFIX "ifMatch="
+#define IF_MATCH_PREFIX_LEN (sizeof(IF_MATCH_PREFIX) - 1)
+#define IF_NOT_MATCH_PREFIX "ifNotMatch="
+#define IF_NOT_MATCH_PREFIX_LEN (sizeof(IF_NOT_MATCH_PREFIX) - 1)
+#define START_BYTE_PREFIX "startByte="
+#define START_BYTE_PREFIX_LEN (sizeof(START_BYTE_PREFIX) - 1)
+#define BYTE_COUNT_PREFIX "byteCount="
+#define BYTE_COUNT_PREFIX_LEN (sizeof(BYTE_COUNT_PREFIX) - 1)
+#define ALL_DETAILS_PREFIX "allDetails="
+#define ALL_DETAILS_PREFIX_LEN (sizeof(ALL_DETAILS_PREFIX) - 1)
+#define NO_STATUS_PREFIX "noStatus="
+#define NO_STATUS_PREFIX_LEN (sizeof(NO_STATUS_PREFIX) - 1)
+#define RESOURCE_PREFIX "resource="
+#define RESOURCE_PREFIX_LEN (sizeof(RESOURCE_PREFIX) - 1)
+#define TARGET_BUCKET_PREFIX "targetBucket="
+#define TARGET_BUCKET_PREFIX_LEN (sizeof(TARGET_BUCKET_PREFIX) - 1)
+#define TARGET_PREFIX_PREFIX "targetPrefix="
+#define TARGET_PREFIX_PREFIX_LEN (sizeof(TARGET_PREFIX_PREFIX) - 1)
+
+
+// util ----------------------------------------------------------------------
+
+static void S3_init()
+{
+    S3Status status;
+    const char *hostname = getenv("S3_HOSTNAME");
+    
+    if ((status = S3_initialize("s3", S3_INIT_ALL, hostname))
+        != S3StatusOK) {
+        fprintf(stderr, "Failed to initialize libs3: %s\n", 
+                S3_get_status_name(status));
+        exit(-1);
+    }
+}
+
+
+static void printError()
+{
+    if (statusG < S3StatusErrorAccessDenied) {
+        fprintf(stderr, "\nERROR: %s\n", S3_get_status_name(statusG));
+    }
+    else {
+        fprintf(stderr, "\nERROR: %s\n", S3_get_status_name(statusG));
+        fprintf(stderr, "%s\n", errorDetailsG);
+    }
+}
+
+
+static void usageExit(FILE *out)
+{
+    fprintf(out,
+"\n Options:\n"
+"\n"
+"   Command Line:\n"
+"\n"
+"   -f/--force           : force operation despite warnings\n"
+"   -h/--vhost-style     : use virtual-host-style URIs (default is "
+                          "path-style)\n"
+"   -u/--unencrypted     : unencrypted (use HTTP instead of HTTPS)\n"
+"   -s/--show-properties : show response properties on stdout\n"
+"   -r/--retries         : retry retryable failures this number of times\n"
+"                          (default is 5)\n"
+"\n"
+"   Environment:\n"
+"\n"
+"   S3_ACCESS_KEY_ID     : S3 access key ID (required)\n"
+"   S3_SECRET_ACCESS_KEY : S3 secret access key (required)\n"
+"   S3_HOSTNAME          : specify alternative S3 host (optional)\n"
+"\n" 
+" Commands (with  and [optional parameters]) :\n"
+"\n"
+"   (NOTE: all command parameters take a value and are specified using the\n"
+"          pattern parameter=value)\n"
+"\n"
+"   help                 : Prints this help text\n"
+"\n"
+"   list                 : Lists owned buckets\n"
+"     [allDetails]       : Show full details\n"
+"\n"
+"   test                 : Tests a bucket for existence and accessibility\n"
+"                : Bucket to test\n"
+"\n"
+"   create               : Create a new bucket\n"
+"                : Bucket to create\n"
+"     [cannedAcl]        : Canned ACL for the bucket (see Canned ACLs)\n"
+"     [location]         : Location for bucket (for example, EU)\n"
+"\n"
+"   delete               : Delete a bucket or key\n"
+"     [/]   : Bucket or bucket/key to delete\n"
+"\n"
+"   list                 : List bucket contents\n"
+"                : Bucket to list\n"
+"     [prefix]           : Prefix for results set\n"
+"     [marker]           : Where in results set to start listing\n"
+"     [delimiter]        : Delimiter for rolling up results set\n"
+"     [maxkeys]          : Maximum number of keys to return in results set\n"
+"     [allDetails]       : Show full details for each key\n"
+"\n"
+"   getacl               : Get the ACL of a bucket or key\n"
+"     [/]   : Bucket or bucket/key to get the ACL of\n"
+"     [filename]         : Output filename for ACL (default is stdout)\n"
+"\n"
+"   setacl               : Set the ACL of a bucket or key\n"
+"     [/]   : Bucket or bucket/key to set the ACL of\n"
+"     [filename]         : Input filename for ACL (default is stdin)\n"
+"\n"
+"   getlogging           : Get the logging status of a bucket\n"
+"                : Bucket to get the logging status of\n"
+"     [filename]         : Output filename for ACL (default is stdout)\n"
+"\n"
+"   setlogging           : Set the logging status of a bucket\n"
+"                : Bucket to set the logging status of\n"
+"     [targetBucket]     : Target bucket to log to; if not present, disables\n"
+"                          logging\n"
+"     [targetPrefix]     : Key prefix to use for logs\n"
+"     [filename]         : Input filename for ACL (default is stdin)\n"
+"\n"
+"   put                  : Puts an object\n"
+"     /     : Bucket/key to put object to\n"
+"     [filename]         : Filename to read source data from "
+                          "(default is stdin)\n"
+"     [contentLength]    : How many bytes of source data to put (required if\n"
+"                          source file is stdin)\n"
+"     [cacheControl]     : Cache-Control HTTP header string to associate with\n"
+"                          object\n"
+"     [contentType]      : Content-Type HTTP header string to associate with\n"
+"                          object\n"
+"     [md5]              : MD5 for validating source data\n"
+"     [contentDispositionFilename] : Content-Disposition filename string to\n"
+"                          associate with object\n"
+"     [contentEncoding]  : Content-Encoding HTTP header string to associate\n"
+"                          with object\n"
+"     [expires]          : Expiration date to associate with object\n"
+"     [cannedAcl]        : Canned ACL for the object (see Canned ACLs)\n"
+"     [x-amz-meta-...]]  : Metadata headers to associate with the object\n"
+"\n"
+"   copy                 : Copies an object; if any options are set, the "
+                          "entire\n"
+"                          metadata of the object is replaced\n"
+"     / : Source bucket/key\n"
+"     / : Destination bucket/key\n"
+"     [cacheControl]     : Cache-Control HTTP header string to associate with\n"
+"                          object\n"
+"     [contentType]      : Content-Type HTTP header string to associate with\n"
+"                          object\n"
+"     [contentDispositionFilename] : Content-Disposition filename string to\n"
+"                          associate with object\n"
+"     [contentEncoding]  : Content-Encoding HTTP header string to associate\n"
+"                          with object\n"
+"     [expires]          : Expiration date to associate with object\n"
+"     [cannedAcl]        : Canned ACL for the object (see Canned ACLs)\n"
+"     [x-amz-meta-...]]  : Metadata headers to associate with the object\n"
+"\n"
+"   get                  : Gets an object\n"
+"     /     : Bucket/key of object to get\n"
+"     [filename]         : Filename to write object data to (required if -s\n"
+"                          command line parameter was used)\n"
+"     [ifModifiedSince]  : Only return the object if it has been modified "
+                          "since\n"
+"                          this date\n"
+"     [ifNotmodifiedSince] : Only return the object if it has not been "
+                          "modified\n"
+"                          since this date\n"
+"     [ifMatch]          : Only return the object if its ETag header matches\n"
+"                          this string\n"
+"     [ifNotMatch]       : Only return the object if its ETag header does "
+                          "not\n"
+"                          match this string\n"
+"     [startByte]        : First byte of byte range to return\n"
+"     [byteCount]        : Number of bytes of byte range to return\n"
+"\n"
+"   head                 : Gets only the headers of an object, implies -s\n"
+"     /     : Bucket/key of object to get headers of\n"
+"\n"
+"   gqs                  : Generates an authenticated query string\n"
+"     [/]   : Bucket or bucket/key to generate query string for\n"
+"     [expires]          : Expiration date for query string\n"
+"     [resource]         : Sub-resource of key for query string, without a\n"
+"                          leading '?', for example, \"torrent\"\n"
+"\n"
+" Canned ACLs:\n"
+"\n"
+"  The following canned ACLs are supported:\n"
+"    private (default), public-read, public-read-write, authenticated-read\n"
+"\n"
+" ACL Format:\n"
+"\n"
+"  For the getacl and setacl commands, the format of the ACL list is:\n"
+"  1) An initial line giving the owner id in this format:\n"
+"       OwnerID  \n"
+"  2) Optional header lines, giving column headers, starting with the\n"
+"     word \"Type\", or with some number of dashes\n"
+"  3) Grant lines, of the form:\n"
+"        (whitespace)  (whitespace) \n"
+"     where Grant Type is one of: Email, UserID, or Group, and\n"
+"     Grantee is the identification of the grantee based on this type,\n"
+"     and Permission is one of: READ, WRITE, READ_ACP, or FULL_CONTROL.\n"
+"\n"
+"  Note that the easiest way to modify an ACL is to first get it, saving it\n"
+"  into a file, then modifying the file, and then setting the modified file\n"
+"  back as the new ACL for the bucket/object.\n"
+"\n"
+" Date Format:\n"
+"\n"
+"  The format for dates used in parameters is as ISO 8601 dates, i.e.\n"
+"  YYYY-MM-DDTHH:MM:SS[+/-dd:dd].  Examples:\n"
+"      2008-07-29T20:36:14\n"
+"      2008-07-29T20:36:14-06:00\n"
+"      2008-07-29T20:36:14+11:30\n"
+"\n");
+
+    exit(-1);
+}
+
+
+static uint64_t convertInt(const char *str, const char *paramName)
+{
+    uint64_t ret = 0;
+
+    while (*str) {
+        if (!isdigit(*str)) {
+            fprintf(stderr, "\nERROR: Nondigit in %s parameter: %c\n", 
+                    paramName, *str);
+            usageExit(stderr);
+        }
+        ret *= 10;
+        ret += (*str++ - '0');
+    }
+
+    return ret;
+}
+
+
+typedef struct growbuffer
+{
+    // The total number of bytes, and the start byte
+    int size;
+    // The start byte
+    int start;
+    // The blocks
+    char data[64 * 1024];
+    struct growbuffer *prev, *next;
+} growbuffer;
+
+
+// returns nonzero on success, zero on out of memory
+static int growbuffer_append(growbuffer **gb, const char *data, int dataLen)
+{
+    while (dataLen) {
+        growbuffer *buf = *gb ? (*gb)->prev : 0;
+        if (!buf || (buf->size == sizeof(buf->data))) {
+            buf = (growbuffer *) malloc(sizeof(growbuffer));
+            if (!buf) {
+                return 0;
+            }
+            buf->size = 0;
+            buf->start = 0;
+            if (*gb) {
+                buf->prev = (*gb)->prev;
+                buf->next = *gb;
+                (*gb)->prev->next = buf;
+                (*gb)->prev = buf;
+            }
+            else {
+                buf->prev = buf->next = buf;
+                *gb = buf;
+            }
+        }
+
+        int toCopy = (sizeof(buf->data) - buf->size);
+        if (toCopy > dataLen) {
+            toCopy = dataLen;
+        }
+
+        memcpy(&(buf->data[buf->size]), data, toCopy);
+        
+        buf->size += toCopy, data += toCopy, dataLen -= toCopy;
+    }
+
+    return 1;
+}
+
+
+static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, 
+                            char *buffer)
+{
+    *amtReturn = 0;
+
+    growbuffer *buf = *gb;
+
+    if (!buf) {
+        return;
+    }
+
+    *amtReturn = (buf->size > amt) ? amt : buf->size;
+
+    memcpy(buffer, &(buf->data[buf->start]), *amtReturn);
+    
+    buf->start += *amtReturn, buf->size -= *amtReturn;
+
+    if (buf->size == 0) {
+        if (buf->next == buf) {
+            *gb = 0;
+        }
+        else {
+            *gb = buf->next;
+            buf->prev->next = buf->next;
+            buf->next->prev = buf->prev;
+        }
+        free(buf);
+    }
+}
+
+
+static void growbuffer_destroy(growbuffer *gb)
+{
+    growbuffer *start = gb;
+
+    while (gb) {
+        growbuffer *next = gb->next;
+        free(gb);
+        gb = (next == start) ? 0 : next;
+    }
+}
+
+
+// Convenience utility for making the code look nicer.  Tests a string
+// against a format; only the characters specified in the format are
+// checked (i.e. if the string is longer than the format, the string still
+// checks out ok).  Format characters are:
+// d - is a digit
+// anything else - is that character
+// Returns nonzero the string checks out, zero if it does not.
+static int checkString(const char *str, const char *format)
+{
+    while (*format) {
+        if (*format == 'd') {
+            if (!isdigit(*str)) {
+                return 0;
+            }
+        }
+        else if (*str != *format) {
+            return 0;
+        }
+        str++, format++;
+    }
+
+    return 1;
+}
+
+
+static int64_t parseIso8601Time(const char *str)
+{
+    // Check to make sure that it has a valid format
+    if (!checkString(str, "dddd-dd-ddTdd:dd:dd")) {
+        return -1;
+    }
+
+#define nextnum() (((*str - '0') * 10) + (*(str + 1) - '0'))
+
+    // Convert it
+    struct tm stm;
+    memset(&stm, 0, sizeof(stm));
+
+    stm.tm_year = (nextnum() - 19) * 100;
+    str += 2;
+    stm.tm_year += nextnum();
+    str += 3;
+
+    stm.tm_mon = nextnum() - 1;
+    str += 3;
+
+    stm.tm_mday = nextnum();
+    str += 3;
+
+    stm.tm_hour = nextnum();
+    str += 3;
+
+    stm.tm_min = nextnum();
+    str += 3;
+
+    stm.tm_sec = nextnum();
+    str += 2;
+
+    stm.tm_isdst = -1;
+
+    // This is hokey but it's the recommended way ...
+    char *tz = getenv("TZ");
+    snprintf(putenvBufG, sizeof(putenvBufG), "TZ=UTC");
+    putenv(putenvBufG);
+
+    int64_t ret = mktime(&stm);
+
+    snprintf(putenvBufG, sizeof(putenvBufG), "TZ=%s", tz ? tz : "");
+    putenv(putenvBufG);
+
+    // Skip the millis
+
+    if (*str == '.') {
+        str++;
+        while (isdigit(*str)) {
+            str++;
+        }
+    }
+    
+    if (checkString(str, "-dd:dd") || checkString(str, "+dd:dd")) {
+        int sign = (*str++ == '-') ? -1 : 1;
+        int hours = nextnum();
+        str += 3;
+        int minutes = nextnum();
+        ret += (-sign * (((hours * 60) + minutes) * 60));
+    }
+    // Else it should be Z to be a conformant time string, but we just assume
+    // that it is rather than enforcing that
+
+    return ret;
+}
+
+
+// Simple ACL format:  Lines of this format:
+// Type - ignored
+// Starting with a dash - ignored
+// Email email_address permission
+// UserID user_id (display_name) permission
+// Group Authenticated AWS Users permission
+// Group All Users  permission
+// permission is one of READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL
+static int convert_simple_acl(char *aclXml, char *ownerId,
+                              char *ownerDisplayName,
+                              int *aclGrantCountReturn,
+                              S3AclGrant *aclGrants)
+{
+    *aclGrantCountReturn = 0;
+    *ownerId = 0;
+    *ownerDisplayName = 0;
+
+#define SKIP_SPACE(require_more)                \
+    do {                                        \
+        while (isspace(*aclXml)) {              \
+            aclXml++;                           \
+        }                                       \
+        if (require_more && !*aclXml) {         \
+            return 0;                           \
+        }                                       \
+    } while (0)
+    
+#define COPY_STRING_MAXLEN(field, maxlen)               \
+    do {                                                \
+        SKIP_SPACE(1);                                  \
+        int len = 0;                                    \
+        while ((len < maxlen) && !isspace(*aclXml)) {   \
+            field[len++] = *aclXml++;                   \
+        }                                               \
+        field[len] = 0;                                 \
+    } while (0)
+
+#define COPY_STRING(field)                              \
+    COPY_STRING_MAXLEN(field, (int) (sizeof(field) - 1))
+
+    while (1) {
+        SKIP_SPACE(0);
+
+        if (!*aclXml) {
+            break;
+        }
+        
+        // Skip Type lines and dash lines
+        if (!strncmp(aclXml, "Type", sizeof("Type") - 1) ||
+            (*aclXml == '-')) {
+            while (*aclXml && ((*aclXml != '\n') && (*aclXml != '\r'))) {
+                aclXml++;
+            }
+            continue;
+        }
+        
+        if (!strncmp(aclXml, "OwnerID", sizeof("OwnerID") - 1)) {
+            aclXml += sizeof("OwnerID") - 1;
+            COPY_STRING_MAXLEN(ownerId, S3_MAX_GRANTEE_USER_ID_SIZE);
+            SKIP_SPACE(1);
+            COPY_STRING_MAXLEN(ownerDisplayName,
+                               S3_MAX_GRANTEE_DISPLAY_NAME_SIZE);
+            continue;
+        }
+
+        if (*aclGrantCountReturn == S3_MAX_ACL_GRANT_COUNT) {
+            return 0;
+        }
+
+        S3AclGrant *grant = &(aclGrants[(*aclGrantCountReturn)++]);
+
+        if (!strncmp(aclXml, "Email", sizeof("Email") - 1)) {
+            grant->granteeType = S3GranteeTypeAmazonCustomerByEmail;
+            aclXml += sizeof("Email") - 1;
+            COPY_STRING(grant->grantee.amazonCustomerByEmail.emailAddress);
+        }
+        else if (!strncmp(aclXml, "UserID", sizeof("UserID") - 1)) {
+            grant->granteeType = S3GranteeTypeCanonicalUser;
+            aclXml += sizeof("UserID") - 1;
+            COPY_STRING(grant->grantee.canonicalUser.id);
+            SKIP_SPACE(1);
+            // Now do display name
+            COPY_STRING(grant->grantee.canonicalUser.displayName);
+        }
+        else if (!strncmp(aclXml, "Group", sizeof("Group") - 1)) {
+            aclXml += sizeof("Group") - 1;
+            SKIP_SPACE(1);
+            if (!strncmp(aclXml, "Authenticated AWS Users",
+                         sizeof("Authenticated AWS Users") - 1)) {
+                grant->granteeType = S3GranteeTypeAllAwsUsers;
+                aclXml += (sizeof("Authenticated AWS Users") - 1);
+            }
+            else if (!strncmp(aclXml, "All Users", sizeof("All Users") - 1)) {
+                grant->granteeType = S3GranteeTypeAllUsers;
+                aclXml += (sizeof("All Users") - 1);
+            }
+            else if (!strncmp(aclXml, "Log Delivery", 
+                              sizeof("Log Delivery") - 1)) {
+                grant->granteeType = S3GranteeTypeLogDelivery;
+                aclXml += (sizeof("Log Delivery") - 1);
+            }
+            else {
+                return 0;
+            }
+        }
+        else {
+            return 0;
+        }
+
+        SKIP_SPACE(1);
+        
+        if (!strncmp(aclXml, "READ_ACP", sizeof("READ_ACP") - 1)) {
+            grant->permission = S3PermissionReadACP;
+            aclXml += (sizeof("READ_ACP") - 1);
+        }
+        else if (!strncmp(aclXml, "READ", sizeof("READ") - 1)) {
+            grant->permission = S3PermissionRead;
+            aclXml += (sizeof("READ") - 1);
+        }
+        else if (!strncmp(aclXml, "WRITE_ACP", sizeof("WRITE_ACP") - 1)) {
+            grant->permission = S3PermissionWriteACP;
+            aclXml += (sizeof("WRITE_ACP") - 1);
+        }
+        else if (!strncmp(aclXml, "WRITE", sizeof("WRITE") - 1)) {
+            grant->permission = S3PermissionWrite;
+            aclXml += (sizeof("WRITE") - 1);
+        }
+        else if (!strncmp(aclXml, "FULL_CONTROL", 
+                          sizeof("FULL_CONTROL") - 1)) {
+            grant->permission = S3PermissionFullControl;
+            aclXml += (sizeof("FULL_CONTROL") - 1);
+        }
+    }
+
+    return 1;
+}
+
+static int should_retry()
+{
+    if (retriesG--) {
+        // Sleep before next retry; start out with a 1 second sleep
+        static int retrySleepInterval = 1 * SLEEP_UNITS_PER_SECOND;
+        sleep(retrySleepInterval);
+        // Next sleep 1 second longer
+        retrySleepInterval++;
+        return 1;
+    }
+
+    return 0;
+}
+
+
+static struct option longOptionsG[] =
+{
+    { "force",                no_argument,        0,  'f' },
+    { "vhost-style",          no_argument,        0,  'h' },
+    { "unencrypted",          no_argument,        0,  'u' },
+    { "show-properties",      no_argument,        0,  's' },
+    { "retries",              required_argument,  0,  'r' },
+    { 0,                      0,                  0,   0  }
+};
+
+
+// response properties callback ----------------------------------------------
+
+// This callback does the same thing for every request type: prints out the
+// properties if the user has requested them to be so
+static S3Status responsePropertiesCallback
+    (const S3ResponseProperties *properties, void *callbackData)
+{
+    (void) callbackData;
+
+    if (!showResponsePropertiesG) {
+        return S3StatusOK;
+    }
+
+#define print_nonnull(name, field)                                 \
+    do {                                                           \
+        if (properties-> field) {                                  \
+            printf("%s: %s\n", name, properties-> field);          \
+        }                                                          \
+    } while (0)
+    
+    print_nonnull("Content-Type", contentType);
+    print_nonnull("Request-Id", requestId);
+    print_nonnull("Request-Id-2", requestId2);
+    if (properties->contentLength > 0) {
+        printf("Content-Length: %lld\n", 
+               (unsigned long long) properties->contentLength);
+    }
+    print_nonnull("Server", server);
+    print_nonnull("ETag", eTag);
+    if (properties->lastModified > 0) {
+        char timebuf[256];
+        time_t t = (time_t) properties->lastModified;
+        // gmtime is not thread-safe but we don't care here.
+        strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t));
+        printf("Last-Modified: %s\n", timebuf);
+    }
+    int i;
+    for (i = 0; i < properties->metaDataCount; i++) {
+        printf("x-amz-meta-%s: %s\n", properties->metaData[i].name,
+               properties->metaData[i].value);
+    }
+
+    return S3StatusOK;
+}
+
+
+// response complete callback ------------------------------------------------
+
+// This callback does the same thing for every request type: saves the status
+// and error stuff in global variables
+static void responseCompleteCallback(S3Status status,
+                                     const S3ErrorDetails *error, 
+                                     void *callbackData)
+{
+    (void) callbackData;
+
+    statusG = status;
+    // Compose the error details message now, although we might not use it.
+    // Can't just save a pointer to [error] since it's not guaranteed to last
+    // beyond this callback
+    int len = 0;
+    if (error && error->message) {
+        len += snprintf(&(errorDetailsG[len]), sizeof(errorDetailsG) - len,
+                        "  Message: %s\n", error->message);
+    }
+    if (error && error->resource) {
+        len += snprintf(&(errorDetailsG[len]), sizeof(errorDetailsG) - len,
+                        "  Resource: %s\n", error->resource);
+    }
+    if (error && error->furtherDetails) {
+        len += snprintf(&(errorDetailsG[len]), sizeof(errorDetailsG) - len,
+                        "  Further Details: %s\n", error->furtherDetails);
+    }
+    if (error && error->extraDetailsCount) {
+        len += snprintf(&(errorDetailsG[len]), sizeof(errorDetailsG) - len,
+                        "%s", "  Extra Details:\n");
+        int i;
+        for (i = 0; i < error->extraDetailsCount; i++) {
+            len += snprintf(&(errorDetailsG[len]), 
+                            sizeof(errorDetailsG) - len, "    %s: %s\n", 
+                            error->extraDetails[i].name,
+                            error->extraDetails[i].value);
+        }
+    }
+}
+
+
+// list service --------------------------------------------------------------
+
+typedef struct list_service_data
+{
+    int headerPrinted;
+    int allDetails;
+} list_service_data;
+
+
+static void printListServiceHeader(int allDetails)
+{
+    printf("%-56s  %-20s", "                         Bucket",
+           "      Created");
+    if (allDetails) {
+        printf("  %-64s  %-12s", 
+               "                            Owner ID",
+               "Display Name");
+    }
+    printf("\n");
+    printf("--------------------------------------------------------  "
+           "--------------------");
+    if (allDetails) {
+        printf("  -------------------------------------------------"
+               "---------------  ------------");
+    }
+    printf("\n");
+}
+
+
+static S3Status listServiceCallback(const char *ownerId, 
+                                    const char *ownerDisplayName,
+                                    const char *bucketName,
+                                    int64_t creationDate, void *callbackData)
+{
+    list_service_data *data = (list_service_data *) callbackData;
+
+    if (!data->headerPrinted) {
+        data->headerPrinted = 1;
+        printListServiceHeader(data->allDetails);
+    }
+
+    char timebuf[256];
+    if (creationDate >= 0) {
+        time_t t = (time_t) creationDate;
+        strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t));
+    }
+    else {
+        timebuf[0] = 0;
+    }
+
+    printf("%-56s  %-20s", bucketName, timebuf);
+    if (data->allDetails) {
+        printf("  %-64s  %-12s", ownerId ? ownerId : "", 
+               ownerDisplayName ? ownerDisplayName : "");
+    }
+    printf("\n");
+
+    return S3StatusOK;
+}
+
+
+static void list_service(int allDetails)
+{
+    list_service_data data;
+
+    data.headerPrinted = 0;
+    data.allDetails = allDetails;
+
+    S3_init();
+
+    S3ListServiceHandler listServiceHandler =
+    {
+        { &responsePropertiesCallback, &responseCompleteCallback },
+        &listServiceCallback
+    };
+
+    do {
+        S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, 
+                        &listServiceHandler, &data);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (statusG == S3StatusOK) {
+        if (!data.headerPrinted) {
+            printListServiceHeader(allDetails);
+        }
+    }
+    else {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+// test bucket ---------------------------------------------------------------
+
+static void test_bucket(int argc, char **argv, int optindex)
+{
+    // test bucket
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex++];
+
+    if (optindex != argc) {
+        fprintf(stderr, "\nERROR: Extraneous parameter: %s\n", argv[optindex]);
+        usageExit(stderr);
+    }
+
+    S3_init();
+
+    S3ResponseHandler responseHandler =
+    {
+        &responsePropertiesCallback, &responseCompleteCallback
+    };
+
+    char locationConstraint[64];
+    do {
+        S3_test_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG,
+                       0, bucketName, sizeof(locationConstraint),
+                       locationConstraint, 0, &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    const char *result;
+
+    switch (statusG) {
+    case S3StatusOK:
+        // bucket exists
+        result = locationConstraint[0] ? locationConstraint : "USA";
+        break;
+    case S3StatusErrorNoSuchBucket:
+        result = "Does Not Exist";
+        break;
+    case S3StatusErrorAccessDenied:
+        result = "Access Denied";
+        break;
+    default:
+        result = 0;
+        break;
+    }
+
+    if (result) {
+        printf("%-56s  %-20s\n", "                         Bucket",
+               "       Status");
+        printf("--------------------------------------------------------  "
+               "--------------------\n");
+        printf("%-56s  %-20s\n", bucketName, result);
+    }
+    else {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+// create bucket -------------------------------------------------------------
+
+static void create_bucket(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex++];
+
+    if (!forceG && (S3_validate_bucket_name
+                    (bucketName, S3UriStyleVirtualHost) != S3StatusOK)) {
+        fprintf(stderr, "\nWARNING: Bucket name is not valid for "
+                "virtual-host style URI access.\n");
+        fprintf(stderr, "Bucket not created.  Use -f option to force the "
+                "bucket to be created despite\n");
+        fprintf(stderr, "this warning.\n\n");
+        exit(-1);
+    }
+
+    const char *locationConstraint = 0;
+    S3CannedAcl cannedAcl = S3CannedAclPrivate;
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, LOCATION_PREFIX, LOCATION_PREFIX_LEN)) {
+            locationConstraint = &(param[LOCATION_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, CANNED_ACL_PREFIX, CANNED_ACL_PREFIX_LEN)) {
+            char *val = &(param[CANNED_ACL_PREFIX_LEN]);
+            if (!strcmp(val, "private")) {
+                cannedAcl = S3CannedAclPrivate;
+            }
+            else if (!strcmp(val, "public-read")) {
+                cannedAcl = S3CannedAclPublicRead;
+            }
+            else if (!strcmp(val, "public-read-write")) {
+                cannedAcl = S3CannedAclPublicReadWrite;
+            }
+            else if (!strcmp(val, "authenticated-read")) {
+                cannedAcl = S3CannedAclAuthenticatedRead;
+            }
+            else {
+                fprintf(stderr, "\nERROR: Unknown canned ACL: %s\n", val);
+                usageExit(stderr);
+            }
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    S3_init();
+
+    S3ResponseHandler responseHandler =
+    {
+        &responsePropertiesCallback, &responseCompleteCallback
+    };
+
+    do {
+        S3_create_bucket(protocolG, accessKeyIdG, secretAccessKeyG,
+                         0, bucketName, cannedAcl, locationConstraint, 0,
+                         &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (statusG == S3StatusOK) {
+        printf("Bucket successfully created.\n");
+    }
+    else {
+        printError();
+    }
+    
+    S3_deinitialize();
+}
+
+
+// delete bucket -------------------------------------------------------------
+
+static void delete_bucket(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex++];
+
+    if (optindex != argc) {
+        fprintf(stderr, "\nERROR: Extraneous parameter: %s\n", argv[optindex]);
+        usageExit(stderr);
+    }
+
+    S3_init();
+
+    S3ResponseHandler responseHandler =
+    {
+        &responsePropertiesCallback, &responseCompleteCallback
+    };
+
+    do {
+        S3_delete_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG,
+                         0, bucketName, 0, &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (statusG != S3StatusOK) {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+// list bucket ---------------------------------------------------------------
+
+typedef struct list_bucket_callback_data
+{
+    int isTruncated;
+    char nextMarker[1024];
+    int keyCount;
+    int allDetails;
+} list_bucket_callback_data;
+
+
+static void printListBucketHeader(int allDetails)
+{
+    printf("%-50s  %-20s  %-5s", 
+           "                       Key", 
+           "   Last Modified", "Size");
+    if (allDetails) {
+        printf("  %-34s  %-64s  %-12s", 
+               "               ETag", 
+               "                            Owner ID",
+               "Display Name");
+    }
+    printf("\n");
+    printf("--------------------------------------------------  "
+           "--------------------  -----");
+    if (allDetails) {
+        printf("  ----------------------------------  "
+               "-------------------------------------------------"
+               "---------------  ------------");
+    }
+    printf("\n");
+}
+
+
+static S3Status listBucketCallback(int isTruncated, const char *nextMarker,
+                                   int contentsCount, 
+                                   const S3ListBucketContent *contents,
+                                   int commonPrefixesCount,
+                                   const char **commonPrefixes,
+                                   void *callbackData)
+{
+    list_bucket_callback_data *data = 
+        (list_bucket_callback_data *) callbackData;
+
+    data->isTruncated = isTruncated;
+    // This is tricky.  S3 doesn't return the NextMarker if there is no
+    // delimiter.  Why, I don't know, since it's still useful for paging
+    // through results.  We want NextMarker to be the last content in the
+    // list, so set it to that if necessary.
+    if ((!nextMarker || !nextMarker[0]) && contentsCount) {
+        nextMarker = contents[contentsCount - 1].key;
+    }
+    if (nextMarker) {
+        snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", 
+                 nextMarker);
+    }
+    else {
+        data->nextMarker[0] = 0;
+    }
+    
+    if (contentsCount && !data->keyCount) {
+        printListBucketHeader(data->allDetails);
+    }
+
+    int i;
+    for (i = 0; i < contentsCount; i++) {
+        const S3ListBucketContent *content = &(contents[i]);
+        char timebuf[256];
+        if (0) {
+            time_t t = (time_t) content->lastModified;
+            strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ",
+                     gmtime(&t));
+            printf("\nKey: %s\n", content->key);
+            printf("Last Modified: %s\n", timebuf);
+            printf("ETag: %s\n", content->eTag);
+            printf("Size: %llu\n", (unsigned long long) content->size);
+            if (content->ownerId) {
+                printf("Owner ID: %s\n", content->ownerId);
+            }
+            if (content->ownerDisplayName) {
+                printf("Owner Display Name: %s\n", content->ownerDisplayName);
+            }
+        }
+        else {
+            time_t t = (time_t) content->lastModified;
+            strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", 
+                     gmtime(&t));
+            char sizebuf[16];
+            if (content->size < 100000) {
+                sprintf(sizebuf, "%5llu", (unsigned long long) content->size);
+            }
+            else if (content->size < (1024 * 1024)) {
+                sprintf(sizebuf, "%4lluK", 
+                        ((unsigned long long) content->size) / 1024ULL);
+            }
+            else if (content->size < (10 * 1024 * 1024)) {
+                float f = content->size;
+                f /= (1024 * 1024);
+                sprintf(sizebuf, "%1.2fM", f);
+            }
+            else if (content->size < (1024 * 1024 * 1024)) {
+                sprintf(sizebuf, "%4lluM", 
+                        ((unsigned long long) content->size) / 
+                        (1024ULL * 1024ULL));
+            }
+            else {
+                float f = (content->size / 1024);
+                f /= (1024 * 1024);
+                sprintf(sizebuf, "%1.2fG", f);
+            }
+            printf("%-50s  %s  %s", content->key, timebuf, sizebuf);
+            if (data->allDetails) {
+                printf("  %-34s  %-64s  %-12s",
+                       content->eTag, 
+                       content->ownerId ? content->ownerId : "",
+                       content->ownerDisplayName ? 
+                       content->ownerDisplayName : "");
+            }
+            printf("\n");
+        }
+    }
+
+    data->keyCount += contentsCount;
+
+    for (i = 0; i < commonPrefixesCount; i++) {
+        printf("\nCommon Prefix: %s\n", commonPrefixes[i]);
+    }
+
+    return S3StatusOK;
+}
+
+
+static void list_bucket(const char *bucketName, const char *prefix,
+                        const char *marker, const char *delimiter,
+                        int maxkeys, int allDetails)
+{
+    S3_init();
+    
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3ListBucketHandler listBucketHandler =
+    {
+        { &responsePropertiesCallback, &responseCompleteCallback },
+        &listBucketCallback
+    };
+
+    list_bucket_callback_data data;
+
+    snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
+    data.keyCount = 0;
+    data.allDetails = allDetails;
+
+    do {
+        data.isTruncated = 0;
+        do {
+            S3_list_bucket(&bucketContext, prefix, data.nextMarker,
+                           delimiter, maxkeys, 0, &listBucketHandler, &data);
+        } while (S3_status_is_retryable(statusG) && should_retry());
+        if (statusG != S3StatusOK) {
+            break;
+        }
+    } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys)));
+
+    if (statusG == S3StatusOK) {
+        if (!data.keyCount) {
+            printListBucketHeader(allDetails);
+        }
+    }
+    else {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+static void list(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        list_service(0);
+        return;
+    }
+
+    const char *bucketName = 0;
+
+    const char *prefix = 0, *marker = 0, *delimiter = 0;
+    int maxkeys = 0, allDetails = 0;
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, PREFIX_PREFIX, PREFIX_PREFIX_LEN)) {
+            prefix = &(param[PREFIX_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, MARKER_PREFIX, MARKER_PREFIX_LEN)) {
+            marker = &(param[MARKER_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, DELIMITER_PREFIX, DELIMITER_PREFIX_LEN)) {
+            delimiter = &(param[DELIMITER_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, MAXKEYS_PREFIX, MAXKEYS_PREFIX_LEN)) {
+            maxkeys = convertInt(&(param[MAXKEYS_PREFIX_LEN]), "maxkeys");
+        }
+        else if (!strncmp(param, ALL_DETAILS_PREFIX,
+                          ALL_DETAILS_PREFIX_LEN)) {
+            const char *ad = &(param[ALL_DETAILS_PREFIX_LEN]);
+            if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || 
+                !strcmp(ad, "yes") || !strcmp(ad, "YES") ||
+                !strcmp(ad, "1")) {
+                allDetails = 1;
+            }
+        }
+        else if (!bucketName) {
+            bucketName = param;
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    if (bucketName) {
+        list_bucket(bucketName, prefix, marker, delimiter, maxkeys, 
+                    allDetails);
+    }
+    else {
+        list_service(allDetails);
+    }
+}
+
+    
+
+// delete object -------------------------------------------------------------
+
+static void delete_object(int argc, char **argv, int optindex)
+{
+    (void) argc;
+
+    // Split bucket/key
+    char *slash = argv[optindex];
+
+    // We know there is a slash in there, put_object is only called if so
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    *slash++ = 0;
+
+    const char *bucketName = argv[optindex++];
+    const char *key = slash;
+
+    S3_init();
+    
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3ResponseHandler responseHandler =
+    { 
+        0,
+        &responseCompleteCallback
+    };
+
+    do {
+        S3_delete_object(&bucketContext, key, 0, &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if ((statusG != S3StatusOK) &&
+        (statusG != S3StatusErrorPreconditionFailed)) {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+// put object ----------------------------------------------------------------
+
+typedef struct put_object_callback_data
+{
+    FILE *infile;
+    growbuffer *gb;
+    uint64_t contentLength, originalContentLength;
+    int noStatus;
+} put_object_callback_data;
+
+
+static int putObjectDataCallback(int bufferSize, char *buffer,
+                                 void *callbackData)
+{
+    put_object_callback_data *data = 
+        (put_object_callback_data *) callbackData;
+    
+    int ret = 0;
+
+    if (data->contentLength) {
+        int toRead = ((data->contentLength > (unsigned) bufferSize) ?
+                      (unsigned) bufferSize : data->contentLength);
+        if (data->gb) {
+            growbuffer_read(&(data->gb), toRead, &ret, buffer);
+        }
+        else if (data->infile) {
+            ret = fread(buffer, 1, toRead, data->infile);
+        }
+    }
+
+    data->contentLength -= ret;
+
+    if (data->contentLength && !data->noStatus) {
+        // Avoid a weird bug in MingW, which won't print the second integer
+        // value properly when it's in the same call, so print separately
+        printf("%llu bytes remaining ", 
+               (unsigned long long) data->contentLength);
+        printf("(%d%% complete) ...\n",
+               (int) (((data->originalContentLength - 
+                        data->contentLength) * 100) /
+                      data->originalContentLength));
+    }
+
+    return ret;
+}
+
+
+static void put_object(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket/key\n");
+        usageExit(stderr);
+    }
+
+    // Split bucket/key
+    char *slash = argv[optindex];
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (!*slash || !*(slash + 1)) {
+        fprintf(stderr, "\nERROR: Invalid bucket/key name: %s\n",
+                argv[optindex]);
+        usageExit(stderr);
+    }
+    *slash++ = 0;
+
+    const char *bucketName = argv[optindex++];
+    const char *key = slash;
+
+    const char *filename = 0;
+    uint64_t contentLength = 0;
+    const char *cacheControl = 0, *contentType = 0, *md5 = 0;
+    const char *contentDispositionFilename = 0, *contentEncoding = 0;
+    int64_t expires = -1;
+    S3CannedAcl cannedAcl = S3CannedAclPrivate;
+    int metaPropertiesCount = 0;
+    S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
+    int noStatus = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) {
+            filename = &(param[FILENAME_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, CONTENT_LENGTH_PREFIX, 
+                          CONTENT_LENGTH_PREFIX_LEN)) {
+            contentLength = convertInt(&(param[CONTENT_LENGTH_PREFIX_LEN]),
+                                       "contentLength");
+            if (contentLength > (5LL * 1024 * 1024 * 1024)) {
+                fprintf(stderr, "\nERROR: contentLength must be no greater "
+                        "than 5 GB\n");
+                usageExit(stderr);
+            }
+        }
+        else if (!strncmp(param, CACHE_CONTROL_PREFIX, 
+                          CACHE_CONTROL_PREFIX_LEN)) {
+            cacheControl = &(param[CACHE_CONTROL_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, CONTENT_TYPE_PREFIX, 
+                          CONTENT_TYPE_PREFIX_LEN)) {
+            contentType = &(param[CONTENT_TYPE_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, MD5_PREFIX, MD5_PREFIX_LEN)) {
+            md5 = &(param[MD5_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, 
+                          CONTENT_DISPOSITION_FILENAME_PREFIX_LEN)) {
+            contentDispositionFilename = 
+                &(param[CONTENT_DISPOSITION_FILENAME_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, CONTENT_ENCODING_PREFIX, 
+                          CONTENT_ENCODING_PREFIX_LEN)) {
+            contentEncoding = &(param[CONTENT_ENCODING_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, EXPIRES_PREFIX, EXPIRES_PREFIX_LEN)) {
+            expires = parseIso8601Time(&(param[EXPIRES_PREFIX_LEN]));
+            if (expires < 0) {
+                fprintf(stderr, "\nERROR: Invalid expires time "
+                        "value; ISO 8601 time format required\n");
+                usageExit(stderr);
+            }
+        }
+        else if (!strncmp(param, X_AMZ_META_PREFIX, X_AMZ_META_PREFIX_LEN)) {
+            if (metaPropertiesCount == S3_MAX_METADATA_COUNT) {
+                fprintf(stderr, "\nERROR: Too many x-amz-meta- properties, "
+                        "limit %lu: %s\n", 
+                        (unsigned long) S3_MAX_METADATA_COUNT, param);
+                usageExit(stderr);
+            }
+            char *name = &(param[X_AMZ_META_PREFIX_LEN]);
+            char *value = name;
+            while (*value && (*value != '=')) {
+                value++;
+            }
+            if (!*value || !*(value + 1)) {
+                fprintf(stderr, "\nERROR: Invalid parameter: %s\n", param);
+                usageExit(stderr);
+            }
+            *value++ = 0;
+            metaProperties[metaPropertiesCount].name = name;
+            metaProperties[metaPropertiesCount++].value = value;
+        }
+        else if (!strncmp(param, CANNED_ACL_PREFIX, CANNED_ACL_PREFIX_LEN)) {
+            char *val = &(param[CANNED_ACL_PREFIX_LEN]);
+            if (!strcmp(val, "private")) {
+                cannedAcl = S3CannedAclPrivate;
+            }
+            else if (!strcmp(val, "public-read")) {
+                cannedAcl = S3CannedAclPublicRead;
+            }
+            else if (!strcmp(val, "public-read-write")) {
+                cannedAcl = S3CannedAclPublicReadWrite;
+            }
+            else if (!strcmp(val, "authenticated-read")) {
+                cannedAcl = S3CannedAclAuthenticatedRead;
+            }
+            else {
+                fprintf(stderr, "\nERROR: Unknown canned ACL: %s\n", val);
+                usageExit(stderr);
+            }
+        }
+        else if (!strncmp(param, NO_STATUS_PREFIX, NO_STATUS_PREFIX_LEN)) {
+            const char *ns = &(param[NO_STATUS_PREFIX_LEN]);
+            if (!strcmp(ns, "true") || !strcmp(ns, "TRUE") || 
+                !strcmp(ns, "yes") || !strcmp(ns, "YES") ||
+                !strcmp(ns, "1")) {
+                noStatus = 1;
+            }
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    put_object_callback_data data;
+
+    data.infile = 0;
+    data.gb = 0;
+    data.noStatus = noStatus;
+
+    if (filename) {
+        if (!contentLength) {
+            struct stat statbuf;
+            // Stat the file to get its length
+            if (stat(filename, &statbuf) == -1) {
+                fprintf(stderr, "\nERROR: Failed to stat file %s: ",
+                        filename);
+                perror(0);
+                exit(-1);
+            }
+            contentLength = statbuf.st_size;
+        }
+        // Open the file
+        if (!(data.infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) {
+            fprintf(stderr, "\nERROR: Failed to open input file %s: ",
+                    filename);
+            perror(0);
+            exit(-1);
+        }
+    }
+    else {
+        // Read from stdin.  If contentLength is not provided, we have
+        // to read it all in to get contentLength.
+        if (!contentLength) {
+            // Read all if stdin to get the data
+            char buffer[64 * 1024];
+            while (1) {
+                int amtRead = fread(buffer, 1, sizeof(buffer), stdin);
+                if (amtRead == 0) {
+                    break;
+                }
+                if (!growbuffer_append(&(data.gb), buffer, amtRead)) {
+                    fprintf(stderr, "\nERROR: Out of memory while reading "
+                            "stdin\n");
+                    exit(-1);
+                }
+                contentLength += amtRead;
+                if (amtRead < (int) sizeof(buffer)) {
+                    break;
+                }
+            }
+        }
+        else {
+            data.infile = stdin;
+        }
+    }
+
+    data.contentLength = data.originalContentLength = contentLength;
+
+    S3_init();
+    
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3PutProperties putProperties =
+    {
+        contentType,
+        md5,
+        cacheControl,
+        contentDispositionFilename,
+        contentEncoding,
+        expires,
+        cannedAcl,
+        metaPropertiesCount,
+        metaProperties
+    };
+
+    S3PutObjectHandler putObjectHandler =
+    {
+        { &responsePropertiesCallback, &responseCompleteCallback },
+        &putObjectDataCallback
+    };
+
+    do {
+        S3_put_object(&bucketContext, key, contentLength, &putProperties, 0,
+                      &putObjectHandler, &data);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (data.infile) {
+        fclose(data.infile);
+    }
+    else if (data.gb) {
+        growbuffer_destroy(data.gb);
+    }
+
+    if (statusG != S3StatusOK) {
+        printError();
+    }
+    else if (data.contentLength) {
+        fprintf(stderr, "\nERROR: Failed to read remaining %llu bytes from "
+                "input\n", (unsigned long long) data.contentLength);
+    }
+
+    S3_deinitialize();
+}
+
+
+// copy object ---------------------------------------------------------------
+
+static void copy_object(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: source bucket/key\n");
+        usageExit(stderr);
+    }
+
+    // Split bucket/key
+    char *slash = argv[optindex];
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (!*slash || !*(slash + 1)) {
+        fprintf(stderr, "\nERROR: Invalid source bucket/key name: %s\n",
+                argv[optindex]);
+        usageExit(stderr);
+    }
+    *slash++ = 0;
+
+    const char *sourceBucketName = argv[optindex++];
+    const char *sourceKey = slash;
+
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: "
+                "destination bucket/key\n");
+        usageExit(stderr);
+    }
+
+    // Split bucket/key
+    slash = argv[optindex];
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (!*slash || !*(slash + 1)) {
+        fprintf(stderr, "\nERROR: Invalid destination bucket/key name: %s\n",
+                argv[optindex]);
+        usageExit(stderr);
+    }
+    *slash++ = 0;
+
+    const char *destinationBucketName = argv[optindex++];
+    const char *destinationKey = slash;
+
+    const char *cacheControl = 0, *contentType = 0;
+    const char *contentDispositionFilename = 0, *contentEncoding = 0;
+    int64_t expires = -1;
+    S3CannedAcl cannedAcl = S3CannedAclPrivate;
+    int metaPropertiesCount = 0;
+    S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
+    int anyPropertiesSet = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, CACHE_CONTROL_PREFIX, 
+                          CACHE_CONTROL_PREFIX_LEN)) {
+            cacheControl = &(param[CACHE_CONTROL_PREFIX_LEN]);
+            anyPropertiesSet = 1;
+        }
+        else if (!strncmp(param, CONTENT_TYPE_PREFIX, 
+                          CONTENT_TYPE_PREFIX_LEN)) {
+            contentType = &(param[CONTENT_TYPE_PREFIX_LEN]);
+            anyPropertiesSet = 1;
+        }
+        else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, 
+                          CONTENT_DISPOSITION_FILENAME_PREFIX_LEN)) {
+            contentDispositionFilename = 
+                &(param[CONTENT_DISPOSITION_FILENAME_PREFIX_LEN]);
+            anyPropertiesSet = 1;
+        }
+        else if (!strncmp(param, CONTENT_ENCODING_PREFIX, 
+                          CONTENT_ENCODING_PREFIX_LEN)) {
+            contentEncoding = &(param[CONTENT_ENCODING_PREFIX_LEN]);
+            anyPropertiesSet = 1;
+        }
+        else if (!strncmp(param, EXPIRES_PREFIX, EXPIRES_PREFIX_LEN)) {
+            expires = parseIso8601Time(&(param[EXPIRES_PREFIX_LEN]));
+            if (expires < 0) {
+                fprintf(stderr, "\nERROR: Invalid expires time "
+                        "value; ISO 8601 time format required\n");
+                usageExit(stderr);
+            }
+            anyPropertiesSet = 1;
+        }
+        else if (!strncmp(param, X_AMZ_META_PREFIX, X_AMZ_META_PREFIX_LEN)) {
+            if (metaPropertiesCount == S3_MAX_METADATA_COUNT) {
+                fprintf(stderr, "\nERROR: Too many x-amz-meta- properties, "
+                        "limit %lu: %s\n", 
+                        (unsigned long) S3_MAX_METADATA_COUNT, param);
+                usageExit(stderr);
+            }
+            char *name = &(param[X_AMZ_META_PREFIX_LEN]);
+            char *value = name;
+            while (*value && (*value != '=')) {
+                value++;
+            }
+            if (!*value || !*(value + 1)) {
+                fprintf(stderr, "\nERROR: Invalid parameter: %s\n", param);
+                usageExit(stderr);
+            }
+            *value++ = 0;
+            metaProperties[metaPropertiesCount].name = name;
+            metaProperties[metaPropertiesCount++].value = value;
+            anyPropertiesSet = 1;
+        }
+        else if (!strncmp(param, CANNED_ACL_PREFIX, CANNED_ACL_PREFIX_LEN)) {
+            char *val = &(param[CANNED_ACL_PREFIX_LEN]);
+            if (!strcmp(val, "private")) {
+                cannedAcl = S3CannedAclPrivate;
+            }
+            else if (!strcmp(val, "public-read")) {
+                cannedAcl = S3CannedAclPublicRead;
+            }
+            else if (!strcmp(val, "public-read-write")) {
+                cannedAcl = S3CannedAclPublicReadWrite;
+            }
+            else if (!strcmp(val, "authenticated-read")) {
+                cannedAcl = S3CannedAclAuthenticatedRead;
+            }
+            else {
+                fprintf(stderr, "\nERROR: Unknown canned ACL: %s\n", val);
+                usageExit(stderr);
+            }
+            anyPropertiesSet = 1;
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    S3_init();
+    
+    S3BucketContext bucketContext =
+    {
+        0,
+        sourceBucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3PutProperties putProperties =
+    {
+        contentType,
+        0,
+        cacheControl,
+        contentDispositionFilename,
+        contentEncoding,
+        expires,
+        cannedAcl,
+        metaPropertiesCount,
+        metaProperties
+    };
+
+    S3ResponseHandler responseHandler =
+    { 
+        &responsePropertiesCallback,
+        &responseCompleteCallback
+    };
+
+    int64_t lastModified;
+    char eTag[256];
+
+    do {
+        S3_copy_object(&bucketContext, sourceKey, destinationBucketName,
+                       destinationKey, anyPropertiesSet ? &putProperties : 0,
+                       &lastModified, sizeof(eTag), eTag, 0,
+                       &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (statusG == S3StatusOK) {
+        if (lastModified >= 0) {
+            char timebuf[256];
+            time_t t = (time_t) lastModified;
+            strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ",
+                     gmtime(&t));
+            printf("Last-Modified: %s\n", timebuf);
+        }
+        if (eTag[0]) {
+            printf("ETag: %s\n", eTag);
+        }
+    }
+    else {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+// get object ----------------------------------------------------------------
+
+static S3Status getObjectDataCallback(int bufferSize, const char *buffer,
+                                      void *callbackData)
+{
+    FILE *outfile = (FILE *) callbackData;
+
+    size_t wrote = fwrite(buffer, 1, bufferSize, outfile);
+    
+    return ((wrote < (size_t) bufferSize) ? 
+            S3StatusAbortedByCallback : S3StatusOK);
+}
+
+
+static void get_object(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket/key\n");
+        usageExit(stderr);
+    }
+
+    // Split bucket/key
+    char *slash = argv[optindex];
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (!*slash || !*(slash + 1)) {
+        fprintf(stderr, "\nERROR: Invalid bucket/key name: %s\n",
+                argv[optindex]);
+        usageExit(stderr);
+    }
+    *slash++ = 0;
+
+    const char *bucketName = argv[optindex++];
+    const char *key = slash;
+
+    const char *filename = 0;
+    int64_t ifModifiedSince = -1, ifNotModifiedSince = -1;
+    const char *ifMatch = 0, *ifNotMatch = 0;
+    uint64_t startByte = 0, byteCount = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) {
+            filename = &(param[FILENAME_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, IF_MODIFIED_SINCE_PREFIX, 
+                     IF_MODIFIED_SINCE_PREFIX_LEN)) {
+            // Parse ifModifiedSince
+            ifModifiedSince = parseIso8601Time
+                (&(param[IF_MODIFIED_SINCE_PREFIX_LEN]));
+            if (ifModifiedSince < 0) {
+                fprintf(stderr, "\nERROR: Invalid ifModifiedSince time "
+                        "value; ISO 8601 time format required\n");
+                usageExit(stderr);
+            }
+        }
+        else if (!strncmp(param, IF_NOT_MODIFIED_SINCE_PREFIX, 
+                          IF_NOT_MODIFIED_SINCE_PREFIX_LEN)) {
+            // Parse ifModifiedSince
+            ifNotModifiedSince = parseIso8601Time
+                (&(param[IF_NOT_MODIFIED_SINCE_PREFIX_LEN]));
+            if (ifNotModifiedSince < 0) {
+                fprintf(stderr, "\nERROR: Invalid ifNotModifiedSince time "
+                        "value; ISO 8601 time format required\n");
+                usageExit(stderr);
+            }
+        }
+        else if (!strncmp(param, IF_MATCH_PREFIX, IF_MATCH_PREFIX_LEN)) {
+            ifMatch = &(param[IF_MATCH_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, IF_NOT_MATCH_PREFIX,
+                          IF_NOT_MATCH_PREFIX_LEN)) {
+            ifNotMatch = &(param[IF_NOT_MATCH_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, START_BYTE_PREFIX, START_BYTE_PREFIX_LEN)) {
+            startByte = convertInt
+                (&(param[START_BYTE_PREFIX_LEN]), "startByte");
+        }
+        else if (!strncmp(param, BYTE_COUNT_PREFIX, BYTE_COUNT_PREFIX_LEN)) {
+            byteCount = convertInt
+                (&(param[BYTE_COUNT_PREFIX_LEN]), "byteCount");
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    FILE *outfile = 0;
+
+    if (filename) {
+        // Stat the file, and if it doesn't exist, open it in w mode
+        struct stat buf;
+        if (stat(filename, &buf) == -1) {
+            outfile = fopen(filename, "w" FOPEN_EXTRA_FLAGS);
+        }
+        else {
+            // Open in r+ so that we don't truncate the file, just in case
+            // there is an error and we write no bytes, we leave the file
+            // unmodified
+            outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS);
+        }
+        
+        if (!outfile) {
+            fprintf(stderr, "\nERROR: Failed to open output file %s: ",
+                    filename);
+            perror(0);
+            exit(-1);
+        }
+    }
+    else if (showResponsePropertiesG) {
+        fprintf(stderr, "\nERROR: get -s requires a filename parameter\n");
+        usageExit(stderr);
+    }
+    else {
+        outfile = stdout;
+    }
+
+    S3_init();
+    
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3GetConditions getConditions =
+    {
+        ifModifiedSince,
+        ifNotModifiedSince,
+        ifMatch,
+        ifNotMatch
+    };
+
+    S3GetObjectHandler getObjectHandler =
+    {
+        { &responsePropertiesCallback, &responseCompleteCallback },
+        &getObjectDataCallback
+    };
+
+    do {
+        S3_get_object(&bucketContext, key, &getConditions, startByte,
+                      byteCount, 0, &getObjectHandler, outfile);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (statusG != S3StatusOK) {
+        printError();
+    }
+
+    fclose(outfile);
+
+    S3_deinitialize();
+}
+
+
+// head object ---------------------------------------------------------------
+
+static void head_object(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket/key\n");
+        usageExit(stderr);
+    }
+    
+    // Head implies showing response properties
+    showResponsePropertiesG = 1;
+
+    // Split bucket/key
+    char *slash = argv[optindex];
+
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (!*slash || !*(slash + 1)) {
+        fprintf(stderr, "\nERROR: Invalid bucket/key name: %s\n",
+                argv[optindex]);
+        usageExit(stderr);
+    }
+    *slash++ = 0;
+
+    const char *bucketName = argv[optindex++];
+    const char *key = slash;
+
+    if (optindex != argc) {
+        fprintf(stderr, "\nERROR: Extraneous parameter: %s\n", argv[optindex]);
+        usageExit(stderr);
+    }
+
+    S3_init();
+    
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3ResponseHandler responseHandler =
+    { 
+        &responsePropertiesCallback,
+        &responseCompleteCallback
+    };
+
+    do {
+        S3_head_object(&bucketContext, key, 0, &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if ((statusG != S3StatusOK) &&
+        (statusG != S3StatusErrorPreconditionFailed)) {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+// generate query string ------------------------------------------------------
+
+static void generate_query_string(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket[/key]\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex];
+    const char *key = 0;
+
+    // Split bucket/key
+    char *slash = argv[optindex++];
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (*slash) {
+        *slash++ = 0;
+        key = slash;
+    }
+    else {
+        key = 0;
+    }
+
+    int64_t expires = -1;
+
+    const char *resource = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, EXPIRES_PREFIX, EXPIRES_PREFIX_LEN)) {
+            expires = parseIso8601Time(&(param[EXPIRES_PREFIX_LEN]));
+            if (expires < 0) {
+                fprintf(stderr, "\nERROR: Invalid expires time "
+                        "value; ISO 8601 time format required\n");
+                usageExit(stderr);
+            }
+        }
+        else if (!strncmp(param, RESOURCE_PREFIX, RESOURCE_PREFIX_LEN)) {
+            resource = &(param[RESOURCE_PREFIX_LEN]);
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    S3_init();
+    
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    char buffer[S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE];
+
+    S3Status status = S3_generate_authenticated_query_string
+        (buffer, &bucketContext, key, expires, resource);
+    
+    if (status != S3StatusOK) {
+        printf("Failed to generate authenticated query string: %s\n",
+               S3_get_status_name(status));
+    }
+    else {
+        printf("%s\n", buffer);
+    }
+
+    S3_deinitialize();
+}
+
+
+// get acl -------------------------------------------------------------------
+
+void get_acl(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket[/key]\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex];
+    const char *key = 0;
+
+    // Split bucket/key
+    char *slash = argv[optindex++];
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (*slash) {
+        *slash++ = 0;
+        key = slash;
+    }
+    else {
+        key = 0;
+    }
+
+    const char *filename = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) {
+            filename = &(param[FILENAME_PREFIX_LEN]);
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    FILE *outfile = 0;
+
+    if (filename) {
+        // Stat the file, and if it doesn't exist, open it in w mode
+        struct stat buf;
+        if (stat(filename, &buf) == -1) {
+            outfile = fopen(filename, "w" FOPEN_EXTRA_FLAGS);
+        }
+        else {
+            // Open in r+ so that we don't truncate the file, just in case
+            // there is an error and we write no bytes, we leave the file
+            // unmodified
+            outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS);
+        }
+        
+        if (!outfile) {
+            fprintf(stderr, "\nERROR: Failed to open output file %s: ",
+                    filename);
+            perror(0);
+            exit(-1);
+        }
+    }
+    else if (showResponsePropertiesG) {
+        fprintf(stderr, "\nERROR: getacl -s requires a filename parameter\n");
+        usageExit(stderr);
+    }
+    else {
+        outfile = stdout;
+    }
+
+    int aclGrantCount;
+    S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT];
+    char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE];
+    char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE];
+
+    S3_init();
+
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3ResponseHandler responseHandler =
+    {
+        &responsePropertiesCallback,
+        &responseCompleteCallback
+    };
+
+    do {
+        S3_get_acl(&bucketContext, key, ownerId, ownerDisplayName, 
+                   &aclGrantCount, aclGrants, 0, &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (statusG == S3StatusOK) {
+        fprintf(outfile, "OwnerID %s %s\n", ownerId, ownerDisplayName);
+        fprintf(outfile, "%-6s  %-90s  %-12s\n", " Type", 
+                "                                   User Identifier",
+                " Permission");
+        fprintf(outfile, "------  "
+                "------------------------------------------------------------"
+                "------------------------------  ------------\n");
+        int i;
+        for (i = 0; i < aclGrantCount; i++) {
+            S3AclGrant *grant = &(aclGrants[i]);
+            const char *type;
+            char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + 
+                            S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16];
+            const char *id;
+
+            switch (grant->granteeType) {
+            case S3GranteeTypeAmazonCustomerByEmail:
+                type = "Email";
+                id = grant->grantee.amazonCustomerByEmail.emailAddress;
+                break;
+            case S3GranteeTypeCanonicalUser:
+                type = "UserID";
+                snprintf(composedId, sizeof(composedId),
+                         "%s (%s)", grant->grantee.canonicalUser.id,
+                         grant->grantee.canonicalUser.displayName);
+                id = composedId;
+                break;
+            case S3GranteeTypeAllAwsUsers:
+                type = "Group";
+                id = "Authenticated AWS Users";
+                break;
+            case S3GranteeTypeAllUsers:
+                type = "Group";
+                id = "All Users";
+                break;
+            default:
+                type = "Group";
+                id = "Log Delivery";
+                break;
+            }
+            const char *perm;
+            switch (grant->permission) {
+            case S3PermissionRead:
+                perm = "READ";
+                break;
+            case S3PermissionWrite:
+                perm = "WRITE";
+                break;
+            case S3PermissionReadACP:
+                perm = "READ_ACP";
+                break;
+            case S3PermissionWriteACP:
+                perm = "WRITE_ACP";
+                break;
+            default:
+                perm = "FULL_CONTROL";
+                break;
+            }
+            fprintf(outfile, "%-6s  %-90s  %-12s\n", type, id, perm);
+        }
+    }
+    else {
+        printError();
+    }
+
+    fclose(outfile);
+
+    S3_deinitialize();
+}
+
+
+// set acl -------------------------------------------------------------------
+
+void set_acl(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket[/key]\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex];
+    const char *key = 0;
+
+    // Split bucket/key
+    char *slash = argv[optindex++];
+    while (*slash && (*slash != '/')) {
+        slash++;
+    }
+    if (*slash) {
+        *slash++ = 0;
+        key = slash;
+    }
+    else {
+        key = 0;
+    }
+
+    const char *filename = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) {
+            filename = &(param[FILENAME_PREFIX_LEN]);
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    FILE *infile;
+
+    if (filename) {
+        if (!(infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) {
+            fprintf(stderr, "\nERROR: Failed to open input file %s: ",
+                    filename);
+            perror(0);
+            exit(-1);
+        }
+    }
+    else {
+        infile = stdin;
+    }
+
+    // Read in the complete ACL
+    char aclBuf[65536];
+    aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0;
+    char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE];
+    char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE];
+    
+    // Parse it
+    int aclGrantCount;
+    S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT];
+    if (!convert_simple_acl(aclBuf, ownerId, ownerDisplayName,
+                            &aclGrantCount, aclGrants)) {
+        fprintf(stderr, "\nERROR: Failed to parse ACLs\n");
+        fclose(infile);
+        exit(-1);
+    }
+
+    S3_init();
+
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3ResponseHandler responseHandler =
+    {
+        &responsePropertiesCallback,
+        &responseCompleteCallback
+    };
+
+    do {
+        S3_set_acl(&bucketContext, key, ownerId, ownerDisplayName,
+                   aclGrantCount, aclGrants, 0, &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+    
+    if (statusG != S3StatusOK) {
+        printError();
+    }
+
+    fclose(infile);
+
+    S3_deinitialize();
+}
+
+
+// get logging ----------------------------------------------------------------
+
+void get_logging(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex++];
+    const char *filename = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) {
+            filename = &(param[FILENAME_PREFIX_LEN]);
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    FILE *outfile = 0;
+
+    if (filename) {
+        // Stat the file, and if it doesn't exist, open it in w mode
+        struct stat buf;
+        if (stat(filename, &buf) == -1) {
+            outfile = fopen(filename, "w" FOPEN_EXTRA_FLAGS);
+        }
+        else {
+            // Open in r+ so that we don't truncate the file, just in case
+            // there is an error and we write no bytes, we leave the file
+            // unmodified
+            outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS);
+        }
+        
+        if (!outfile) {
+            fprintf(stderr, "\nERROR: Failed to open output file %s: ",
+                    filename);
+            perror(0);
+            exit(-1);
+        }
+    }
+    else if (showResponsePropertiesG) {
+        fprintf(stderr, "\nERROR: getlogging -s requires a filename "
+                "parameter\n");
+        usageExit(stderr);
+    }
+    else {
+        outfile = stdout;
+    }
+
+    int aclGrantCount;
+    S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT];
+    char targetBucket[S3_MAX_BUCKET_NAME_SIZE];
+    char targetPrefix[S3_MAX_KEY_SIZE];
+
+    S3_init();
+
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3ResponseHandler responseHandler =
+    {
+        &responsePropertiesCallback,
+        &responseCompleteCallback
+    };
+
+    do {
+        S3_get_server_access_logging(&bucketContext, targetBucket, targetPrefix,
+                                     &aclGrantCount, aclGrants, 0, 
+                                     &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+
+    if (statusG == S3StatusOK) {
+        if (targetBucket[0]) {
+            printf("Target Bucket: %s\n", targetBucket);
+            if (targetPrefix[0]) {
+                printf("Target Prefix: %s\n", targetPrefix);
+            }
+            fprintf(outfile, "%-6s  %-90s  %-12s\n", " Type", 
+                    "                                   User Identifier",
+                    " Permission");
+            fprintf(outfile, "------  "
+                    "---------------------------------------------------------"
+                    "---------------------------------  ------------\n");
+            int i;
+            for (i = 0; i < aclGrantCount; i++) {
+                S3AclGrant *grant = &(aclGrants[i]);
+                const char *type;
+                char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + 
+                                S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16];
+                const char *id;
+                
+                switch (grant->granteeType) {
+                case S3GranteeTypeAmazonCustomerByEmail:
+                    type = "Email";
+                    id = grant->grantee.amazonCustomerByEmail.emailAddress;
+                    break;
+                case S3GranteeTypeCanonicalUser:
+                    type = "UserID";
+                    snprintf(composedId, sizeof(composedId),
+                             "%s (%s)", grant->grantee.canonicalUser.id,
+                             grant->grantee.canonicalUser.displayName);
+                    id = composedId;
+                    break;
+                case S3GranteeTypeAllAwsUsers:
+                    type = "Group";
+                    id = "Authenticated AWS Users";
+                    break;
+                default:
+                    type = "Group";
+                    id = "All Users";
+                    break;
+                }
+                const char *perm;
+                switch (grant->permission) {
+                case S3PermissionRead:
+                    perm = "READ";
+                    break;
+                case S3PermissionWrite:
+                    perm = "WRITE";
+                    break;
+                case S3PermissionReadACP:
+                    perm = "READ_ACP";
+                    break;
+                case S3PermissionWriteACP:
+                    perm = "WRITE_ACP";
+                    break;
+                default:
+                    perm = "FULL_CONTROL";
+                    break;
+                }
+                fprintf(outfile, "%-6s  %-90s  %-12s\n", type, id, perm);
+            }
+        }
+        else {
+            printf("Service logging is not enabled for this bucket.\n");
+        }
+    }
+    else {
+        printError();
+    }
+
+    fclose(outfile);
+
+    S3_deinitialize();
+}
+
+
+// set logging ----------------------------------------------------------------
+
+void set_logging(int argc, char **argv, int optindex)
+{
+    if (optindex == argc) {
+        fprintf(stderr, "\nERROR: Missing parameter: bucket\n");
+        usageExit(stderr);
+    }
+
+    const char *bucketName = argv[optindex++];
+
+    const char *targetBucket = 0, *targetPrefix = 0, *filename = 0;
+
+    while (optindex < argc) {
+        char *param = argv[optindex++];
+        if (!strncmp(param, TARGET_BUCKET_PREFIX, TARGET_BUCKET_PREFIX_LEN)) {
+            targetBucket = &(param[TARGET_BUCKET_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, TARGET_PREFIX_PREFIX, 
+                          TARGET_PREFIX_PREFIX_LEN)) {
+            targetPrefix = &(param[TARGET_PREFIX_PREFIX_LEN]);
+        }
+        else if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) {
+            filename = &(param[FILENAME_PREFIX_LEN]);
+        }
+        else {
+            fprintf(stderr, "\nERROR: Unknown param: %s\n", param);
+            usageExit(stderr);
+        }
+    }
+
+    int aclGrantCount = 0;
+    S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT];
+
+    if (targetBucket) {
+        FILE *infile;
+        
+        if (filename) {
+            if (!(infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) {
+                fprintf(stderr, "\nERROR: Failed to open input file %s: ",
+                        filename);
+                perror(0);
+                exit(-1);
+            }
+        }
+        else {
+            infile = stdin;
+        }
+
+        // Read in the complete ACL
+        char aclBuf[65536];
+        aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0;
+        char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE];
+        char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE];
+        
+        // Parse it
+        if (!convert_simple_acl(aclBuf, ownerId, ownerDisplayName,
+                                &aclGrantCount, aclGrants)) {
+            fprintf(stderr, "\nERROR: Failed to parse ACLs\n");
+            fclose(infile);
+            exit(-1);
+        }
+
+        fclose(infile);
+    }
+
+    S3_init();
+
+    S3BucketContext bucketContext =
+    {
+        0,
+        bucketName,
+        protocolG,
+        uriStyleG,
+        accessKeyIdG,
+        secretAccessKeyG
+    };
+
+    S3ResponseHandler responseHandler =
+    {
+        &responsePropertiesCallback,
+        &responseCompleteCallback
+    };
+
+    do {
+        S3_set_server_access_logging(&bucketContext, targetBucket, 
+                                     targetPrefix, aclGrantCount, aclGrants, 
+                                     0, &responseHandler, 0);
+    } while (S3_status_is_retryable(statusG) && should_retry());
+    
+    if (statusG != S3StatusOK) {
+        printError();
+    }
+
+    S3_deinitialize();
+}
+
+
+// main ----------------------------------------------------------------------
+
+int main(int argc, char **argv)
+{
+    // Parse args
+    while (1) {
+        int idx = 0;
+        int c = getopt_long(argc, argv, "fhusr:", longOptionsG, &idx);
+
+        if (c == -1) {
+            // End of options
+            break;
+        }
+
+        switch (c) {
+        case 'f':
+            forceG = 1;
+            break;
+        case 'h':
+            uriStyleG = S3UriStyleVirtualHost;
+            break;
+        case 'u':
+            protocolG = S3ProtocolHTTP;
+            break;
+        case 's':
+            showResponsePropertiesG = 1;
+            break;
+        case 'r': {
+            const char *v = optarg;
+            retriesG = 0;
+            while (*v) {
+                retriesG *= 10;
+                retriesG += *v - '0';
+                v++;
+            }
+            break;
+        }
+        default:
+            fprintf(stderr, "\nERROR: Unknown option: -%c\n", c);
+            // Usage exit
+            usageExit(stderr);
+        }
+    }
+
+    // The first non-option argument gives the operation to perform
+    if (optind == argc) {
+        fprintf(stderr, "\n\nERROR: Missing argument: command\n\n");
+        usageExit(stderr);
+    }
+
+    const char *command = argv[optind++];
+    
+    if (!strcmp(command, "help")) {
+        fprintf(stdout, "\ns3 is a program for performing single requests "
+                "to Amazon S3.\n");
+        usageExit(stdout);
+    }
+
+    accessKeyIdG = getenv("S3_ACCESS_KEY_ID");
+    if (!accessKeyIdG) {
+        fprintf(stderr, "Missing environment variable: S3_ACCESS_KEY_ID\n");
+        return -1;
+    }
+    secretAccessKeyG = getenv("S3_SECRET_ACCESS_KEY");
+    if (!secretAccessKeyG) {
+        fprintf(stderr, 
+                "Missing environment variable: S3_SECRET_ACCESS_KEY\n");
+        return -1;
+    }
+
+    if (!strcmp(command, "list")) {
+        list(argc, argv, optind);
+    }
+    else if (!strcmp(command, "test")) {
+        test_bucket(argc, argv, optind);
+    }
+    else if (!strcmp(command, "create")) {
+        create_bucket(argc, argv, optind);
+    }
+    else if (!strcmp(command, "delete")) {
+        if (optind == argc) {
+            fprintf(stderr, 
+                    "\nERROR: Missing parameter: bucket or bucket/key\n");
+            usageExit(stderr);
+        }
+        char *val = argv[optind];
+        int hasSlash = 0;
+        while (*val) {
+            if (*val++ == '/') {
+                hasSlash = 1;
+                break;
+            }
+        }
+        if (hasSlash) {
+            delete_object(argc, argv, optind);
+        }
+        else {
+            delete_bucket(argc, argv, optind);
+        }
+    }
+    else if (!strcmp(command, "put")) {
+        put_object(argc, argv, optind);
+    }
+    else if (!strcmp(command, "copy")) {
+        copy_object(argc, argv, optind);
+    }
+    else if (!strcmp(command, "get")) {
+        get_object(argc, argv, optind);
+    }
+    else if (!strcmp(command, "head")) {
+        head_object(argc, argv, optind);
+    }
+    else if (!strcmp(command, "gqs")) {
+        generate_query_string(argc, argv, optind);
+    }
+    else if (!strcmp(command, "getacl")) {
+        get_acl(argc, argv, optind);
+    }
+    else if (!strcmp(command, "setacl")) {
+        set_acl(argc, argv, optind);
+    }
+    else if (!strcmp(command, "getlogging")) {
+        get_logging(argc, argv, optind);
+    }
+    else if (!strcmp(command, "setlogging")) {
+        set_logging(argc, argv, optind);
+    }
+    else {
+        fprintf(stderr, "Unknown command: %s\n", command);
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/ceph/src/libs3/src/service.c b/ceph/src/libs3/src/service.c
new file mode 100644
index 00000000..2d1e0388
--- /dev/null
+++ b/ceph/src/libs3/src/service.c
@@ -0,0 +1,191 @@
+/** **************************************************************************
+ * service.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include 
+#include 
+#include "request.h"
+
+
+typedef struct XmlCallbackData
+{
+    SimpleXml simpleXml;
+    
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ListServiceCallback *listServiceCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    string_buffer(ownerId, 256);
+    string_buffer(ownerDisplayName, 256);
+    string_buffer(bucketName, 256);
+    string_buffer(creationDate, 128);
+} XmlCallbackData;
+
+
+static S3Status xmlCallback(const char *elementPath, const char *data,
+                            int dataLen, void *callbackData)
+{
+    XmlCallbackData *cbData = (XmlCallbackData *) callbackData;
+
+    int fit;
+
+    if (data) {
+        if (!strcmp(elementPath, "ListAllMyBucketsResult/Owner/ID")) {
+            string_buffer_append(cbData->ownerId, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, 
+                         "ListAllMyBucketsResult/Owner/DisplayName")) {
+            string_buffer_append(cbData->ownerDisplayName, data, dataLen, fit);
+        }
+        else if (!strcmp(elementPath, 
+                         "ListAllMyBucketsResult/Buckets/Bucket/Name")) {
+            string_buffer_append(cbData->bucketName, data, dataLen, fit);
+        }
+        else if (!strcmp
+                 (elementPath, 
+                  "ListAllMyBucketsResult/Buckets/Bucket/CreationDate")) {
+            string_buffer_append(cbData->creationDate, data, dataLen, fit);
+        }
+    }
+    else {
+        if (!strcmp(elementPath, "ListAllMyBucketsResult/Buckets/Bucket")) {
+            // Parse date.  Assume ISO-8601 date format.
+            time_t creationDate = parseIso8601Time(cbData->creationDate);
+
+            // Make the callback - a bucket just finished
+            S3Status status = (*(cbData->listServiceCallback))
+                (cbData->ownerId, cbData->ownerDisplayName,
+                 cbData->bucketName, creationDate, cbData->callbackData);
+
+            string_buffer_initialize(cbData->bucketName);
+            string_buffer_initialize(cbData->creationDate);
+
+            return status;
+        }
+    }
+
+    /* Avoid compiler error about variable set but not used */
+    (void) fit;
+
+    return S3StatusOK;
+}
+
+
+static S3Status propertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    XmlCallbackData *cbData = (XmlCallbackData *) callbackData;
+    
+    return (*(cbData->responsePropertiesCallback))
+        (responseProperties, cbData->callbackData);
+}
+
+
+static S3Status dataCallback(int bufferSize, const char *buffer,
+                             void *callbackData)
+{
+    XmlCallbackData *cbData = (XmlCallbackData *) callbackData;
+
+    return simplexml_add(&(cbData->simpleXml), buffer, bufferSize);
+}
+
+
+static void completeCallback(S3Status requestStatus,
+                             const S3ErrorDetails *s3ErrorDetails,
+                             void *callbackData)
+{
+    XmlCallbackData *cbData = (XmlCallbackData *) callbackData;
+
+    (*(cbData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, cbData->callbackData);
+
+    simplexml_deinitialize(&(cbData->simpleXml));
+
+    free(cbData);
+}
+
+
+void S3_list_service(S3Protocol protocol, const char *accessKeyId,
+                     const char *secretAccessKey, const char *hostName,
+                     S3RequestContext *requestContext,
+                     const S3ListServiceHandler *handler, void *callbackData)
+{
+    // Create and set up the callback data
+    XmlCallbackData *data = 
+        (XmlCallbackData *) malloc(sizeof(XmlCallbackData));
+    if (!data) {
+        (*(handler->responseHandler.completeCallback))
+            (S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    simplexml_initialize(&(data->simpleXml), &xmlCallback, data);
+
+    data->responsePropertiesCallback =
+        handler->responseHandler.propertiesCallback;
+    data->listServiceCallback = handler->listServiceCallback;
+    data->responseCompleteCallback = handler->responseHandler.completeCallback;
+    data->callbackData = callbackData;
+
+    string_buffer_initialize(data->ownerId);
+    string_buffer_initialize(data->ownerDisplayName);
+    string_buffer_initialize(data->bucketName);
+    string_buffer_initialize(data->creationDate);
+    
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeGET,                           // httpRequestType
+        { hostName,                                   // hostName
+          0,                                          // bucketName
+          protocol,                                   // protocol
+          S3UriStylePath,                             // uriStyle
+          accessKeyId,                                // accessKeyId
+          secretAccessKey },                          // secretAccessKey
+        0,                                            // key
+        0,                                            // queryParams
+        0,                                            // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // requestProperties
+        &propertiesCallback,                          // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        &dataCallback,                                // fromS3Callback
+        &completeCallback,                            // completeCallback
+        data                                          // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
diff --git a/ceph/src/libs3/src/service_access_logging.c b/ceph/src/libs3/src/service_access_logging.c
new file mode 100644
index 00000000..22c6e404
--- /dev/null
+++ b/ceph/src/libs3/src/service_access_logging.c
@@ -0,0 +1,555 @@
+/** **************************************************************************
+ * server_access_logging.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "libs3.h"
+#include "request.h"
+
+
+// get server access logging---------------------------------------------------
+
+typedef struct ConvertBlsData
+{
+    char *targetBucketReturn;
+    int targetBucketReturnLen;
+    char *targetPrefixReturn;
+    int targetPrefixReturnLen;
+    int *aclGrantCountReturn;
+    S3AclGrant *aclGrants;
+
+    string_buffer(emailAddress, S3_MAX_GRANTEE_EMAIL_ADDRESS_SIZE);
+    string_buffer(userId, S3_MAX_GRANTEE_USER_ID_SIZE);
+    string_buffer(userDisplayName, S3_MAX_GRANTEE_DISPLAY_NAME_SIZE);
+    string_buffer(groupUri, 128);
+    string_buffer(permission, 32);
+} ConvertBlsData;
+
+
+static S3Status convertBlsXmlCallback(const char *elementPath,
+                                      const char *data, int dataLen,
+                                      void *callbackData)
+{
+    ConvertBlsData *caData = (ConvertBlsData *) callbackData;
+
+    int fit;
+
+    if (data) {
+        if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/"
+                    "TargetBucket")) {
+            caData->targetBucketReturnLen += 
+                snprintf(&(caData->targetBucketReturn
+                           [caData->targetBucketReturnLen]),
+                         255 - caData->targetBucketReturnLen - 1, 
+                         "%.*s", dataLen, data);
+            if (caData->targetBucketReturnLen >= 255) {
+                return S3StatusTargetBucketTooLong;
+            }
+        }
+        else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/"
+                    "TargetPrefix")) {
+            caData->targetPrefixReturnLen += 
+                snprintf(&(caData->targetPrefixReturn
+                           [caData->targetPrefixReturnLen]),
+                         255 - caData->targetPrefixReturnLen - 1, 
+                         "%.*s", dataLen, data);
+            if (caData->targetPrefixReturnLen >= 255) {
+                return S3StatusTargetPrefixTooLong;
+            }
+        }
+        else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/"
+                         "TargetGrants/Grant/Grantee/EmailAddress")) {
+            // AmazonCustomerByEmail
+            string_buffer_append(caData->emailAddress, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusEmailAddressTooLong;
+            }
+        }
+        else if (!strcmp(elementPath,
+                         "AccessControlPolicy/AccessControlList/Grant/"
+                         "Grantee/ID")) {
+            // CanonicalUser
+            string_buffer_append(caData->userId, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusUserIdTooLong;
+            }
+        }
+        else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/"
+                         "TargetGrants/Grant/Grantee/DisplayName")) {
+            // CanonicalUser
+            string_buffer_append(caData->userDisplayName, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusUserDisplayNameTooLong;
+            }
+        }
+        else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/"
+                         "TargetGrants/Grant/Grantee/URI")) {
+            // Group
+            string_buffer_append(caData->groupUri, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusGroupUriTooLong;
+            }
+        }
+        else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/"
+                         "TargetGrants/Grant/Permission")) {
+            // Permission
+            string_buffer_append(caData->permission, data, dataLen, fit);
+            if (!fit) {
+                return S3StatusPermissionTooLong;
+            }
+        }
+    }
+    else {
+        if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/"
+                    "TargetGrants/Grant")) {
+            // A grant has just been completed; so add the next S3AclGrant
+            // based on the values read
+            if (*(caData->aclGrantCountReturn) == S3_MAX_ACL_GRANT_COUNT) {
+                return S3StatusTooManyGrants;
+            }
+
+            S3AclGrant *grant = &(caData->aclGrants
+                                  [*(caData->aclGrantCountReturn)]);
+
+            if (caData->emailAddress[0]) {
+                grant->granteeType = S3GranteeTypeAmazonCustomerByEmail;
+                strcpy(grant->grantee.amazonCustomerByEmail.emailAddress,
+                       caData->emailAddress);
+            }
+            else if (caData->userId[0] && caData->userDisplayName[0]) {
+                grant->granteeType = S3GranteeTypeCanonicalUser;
+                strcpy(grant->grantee.canonicalUser.id, caData->userId);
+                strcpy(grant->grantee.canonicalUser.displayName, 
+                       caData->userDisplayName);
+            }
+            else if (caData->groupUri[0]) {
+                if (!strcmp(caData->groupUri,
+                            ACS_GROUP_AWS_USERS)) {
+                    grant->granteeType = S3GranteeTypeAllAwsUsers;
+                }
+                else if (!strcmp(caData->groupUri,
+                                 ACS_GROUP_ALL_USERS)) {
+                    grant->granteeType = S3GranteeTypeAllUsers;
+                }
+                else {
+                    return S3StatusBadGrantee;
+                }
+            }
+            else {
+                return S3StatusBadGrantee;
+            }
+
+            if (!strcmp(caData->permission, "READ")) {
+                grant->permission = S3PermissionRead;
+            }
+            else if (!strcmp(caData->permission, "WRITE")) {
+                grant->permission = S3PermissionWrite;
+            }
+            else if (!strcmp(caData->permission, "READ_ACP")) {
+                grant->permission = S3PermissionReadACP;
+            }
+            else if (!strcmp(caData->permission, "WRITE_ACP")) {
+                grant->permission = S3PermissionWriteACP;
+            }
+            else if (!strcmp(caData->permission, "FULL_CONTROL")) {
+                grant->permission = S3PermissionFullControl;
+            }
+            else {
+                return S3StatusBadPermission;
+            }
+
+            (*(caData->aclGrantCountReturn))++;
+
+            string_buffer_initialize(caData->emailAddress);
+            string_buffer_initialize(caData->userId);
+            string_buffer_initialize(caData->userDisplayName);
+            string_buffer_initialize(caData->groupUri);
+            string_buffer_initialize(caData->permission);
+        }
+    }
+
+    return S3StatusOK;
+}
+
+
+static S3Status convert_bls(char *blsXml, char *targetBucketReturn,
+                            char *targetPrefixReturn, int *aclGrantCountReturn,
+                            S3AclGrant *aclGrants)
+{
+    ConvertBlsData data;
+
+    data.targetBucketReturn = targetBucketReturn;
+    data.targetBucketReturn[0] = 0;
+    data.targetBucketReturnLen = 0;
+    data.targetPrefixReturn = targetPrefixReturn;
+    data.targetPrefixReturn[0] = 0;
+    data.targetPrefixReturnLen = 0;
+    data.aclGrantCountReturn = aclGrantCountReturn;
+    data.aclGrants = aclGrants;
+    *aclGrantCountReturn = 0;
+    string_buffer_initialize(data.emailAddress);
+    string_buffer_initialize(data.userId);
+    string_buffer_initialize(data.userDisplayName);
+    string_buffer_initialize(data.groupUri);
+    string_buffer_initialize(data.permission);
+
+    // Use a simplexml parser
+    SimpleXml simpleXml;
+    simplexml_initialize(&simpleXml, &convertBlsXmlCallback, &data);
+
+    S3Status status = simplexml_add(&simpleXml, blsXml, strlen(blsXml));
+
+    simplexml_deinitialize(&simpleXml);
+                                          
+    return status;
+}
+
+
+// Use a rather arbitrary max size for the document of 64K
+#define BLS_XML_DOC_MAXSIZE (64 * 1024)
+
+
+typedef struct GetBlsData
+{
+    SimpleXml simpleXml;
+
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    char *targetBucketReturn;
+    char *targetPrefixReturn;
+    int *aclGrantCountReturn;
+    S3AclGrant *aclGrants;
+    string_buffer(blsXmlDocument, BLS_XML_DOC_MAXSIZE);
+} GetBlsData;
+
+
+static S3Status getBlsPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    GetBlsData *gsData = (GetBlsData *) callbackData;
+    
+    return (*(gsData->responsePropertiesCallback))
+        (responseProperties, gsData->callbackData);
+}
+
+
+static S3Status getBlsDataCallback(int bufferSize, const char *buffer,
+                                   void *callbackData)
+{
+    GetBlsData *gsData = (GetBlsData *) callbackData;
+
+    int fit;
+
+    string_buffer_append(gsData->blsXmlDocument, buffer, bufferSize, fit);
+    
+    return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge;
+}
+
+
+static void getBlsCompleteCallback(S3Status requestStatus, 
+                                   const S3ErrorDetails *s3ErrorDetails,
+                                   void *callbackData)
+{
+    GetBlsData *gsData = (GetBlsData *) callbackData;
+
+    if (requestStatus == S3StatusOK) {
+        // Parse the document
+        requestStatus = convert_bls
+            (gsData->blsXmlDocument, gsData->targetBucketReturn,
+             gsData->targetPrefixReturn, gsData->aclGrantCountReturn, 
+             gsData->aclGrants);
+    }
+
+    (*(gsData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, gsData->callbackData);
+
+    free(gsData);
+}
+
+
+void S3_get_server_access_logging(const S3BucketContext *bucketContext,
+                                  char *targetBucketReturn,
+                                  char *targetPrefixReturn,
+                                  int *aclGrantCountReturn, 
+                                  S3AclGrant *aclGrants,
+                                  S3RequestContext *requestContext,
+                                  const S3ResponseHandler *handler,
+                                  void *callbackData)
+{
+    // Create the callback data
+    GetBlsData *gsData = (GetBlsData *) malloc(sizeof(GetBlsData));
+    if (!gsData) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+
+    gsData->responsePropertiesCallback = handler->propertiesCallback;
+    gsData->responseCompleteCallback = handler->completeCallback;
+    gsData->callbackData = callbackData;
+
+    gsData->targetBucketReturn = targetBucketReturn;
+    gsData->targetPrefixReturn = targetPrefixReturn;
+    gsData->aclGrantCountReturn = aclGrantCountReturn;
+    gsData->aclGrants = aclGrants;
+    string_buffer_initialize(gsData->blsXmlDocument);
+    *aclGrantCountReturn = 0;
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypeGET,                           // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        0,                                            // key
+        0,                                            // queryParams
+        "logging",                                    // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        &getBlsPropertiesCallback,                    // propertiesCallback
+        0,                                            // toS3Callback
+        0,                                            // toS3CallbackTotalSize
+        &getBlsDataCallback,                          // fromS3Callback
+        &getBlsCompleteCallback,                      // completeCallback
+        gsData                                        // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
+
+
+
+// set server access logging---------------------------------------------------
+
+static S3Status generateSalXmlDocument(const char *targetBucket,
+                                       const char *targetPrefix,
+                                       int aclGrantCount, 
+                                       const S3AclGrant *aclGrants,
+                                       int *xmlDocumentLenReturn,
+                                       char *xmlDocument,
+                                       int xmlDocumentBufferSize)
+{
+    *xmlDocumentLenReturn = 0;
+
+#define append(fmt, ...)                                        \
+    do {                                                        \
+        *xmlDocumentLenReturn += snprintf                       \
+            (&(xmlDocument[*xmlDocumentLenReturn]),             \
+             xmlDocumentBufferSize - *xmlDocumentLenReturn - 1, \
+             fmt, __VA_ARGS__);                                 \
+        if (*xmlDocumentLenReturn >= xmlDocumentBufferSize) {   \
+            return S3StatusXmlDocumentTooLarge;                 \
+        } \
+    } while (0)
+
+    append("%s", "");
+
+    if (targetBucket && targetBucket[0]) {
+        append("%s", targetBucket);
+        append("%s", 
+               targetPrefix ? targetPrefix : "");
+
+        if (aclGrantCount) {
+            append("%s", "");
+            int i;
+            for (i = 0; i < aclGrantCount; i++) {
+                append("%s", "granteeType) {
+                case S3GranteeTypeAmazonCustomerByEmail:
+                    append("AmazonCustomerByEmail\">%s"
+                           "",
+                           grant->grantee.amazonCustomerByEmail.emailAddress);
+                    break;
+                case S3GranteeTypeCanonicalUser:
+                    append("CanonicalUser\">%s%s"
+                           "",
+                           grant->grantee.canonicalUser.id, 
+                           grant->grantee.canonicalUser.displayName);
+                    break;
+                default: // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers:
+                    append("Group\">%s",
+                           (grant->granteeType == S3GranteeTypeAllAwsUsers) ?
+                           ACS_GROUP_AWS_USERS : ACS_GROUP_ALL_USERS);
+                    break;
+                }
+                append("%s",
+                       ((grant->permission == S3PermissionRead) ? "READ" :
+                        (grant->permission == S3PermissionWrite) ? "WRITE" :
+                        (grant->permission == 
+                         S3PermissionReadACP) ? "READ_ACP" :
+                        (grant->permission == 
+                         S3PermissionWriteACP) ? "WRITE_ACP" : "FULL_CONTROL"));
+            }
+            append("%s", "");
+        }
+        append("%s", "");
+    }
+
+    append("%s", "");
+
+    return S3StatusOK;
+}
+
+
+typedef struct SetSalData
+{
+    S3ResponsePropertiesCallback *responsePropertiesCallback;
+    S3ResponseCompleteCallback *responseCompleteCallback;
+    void *callbackData;
+
+    int salXmlDocumentLen;
+    char salXmlDocument[BLS_XML_DOC_MAXSIZE];
+    int salXmlDocumentBytesWritten;
+
+} SetSalData;
+
+
+static S3Status setSalPropertiesCallback
+    (const S3ResponseProperties *responseProperties, void *callbackData)
+{
+    SetSalData *paData = (SetSalData *) callbackData;
+    
+    return (*(paData->responsePropertiesCallback))
+        (responseProperties, paData->callbackData);
+}
+
+
+static int setSalDataCallback(int bufferSize, char *buffer, void *callbackData)
+{
+    SetSalData *paData = (SetSalData *) callbackData;
+
+    int remaining = (paData->salXmlDocumentLen - 
+                     paData->salXmlDocumentBytesWritten);
+
+    int toCopy = bufferSize > remaining ? remaining : bufferSize;
+    
+    if (!toCopy) {
+        return 0;
+    }
+
+    memcpy(buffer, &(paData->salXmlDocument
+                     [paData->salXmlDocumentBytesWritten]), toCopy);
+
+    paData->salXmlDocumentBytesWritten += toCopy;
+
+    return toCopy;
+}
+
+
+static void setSalCompleteCallback(S3Status requestStatus, 
+                                   const S3ErrorDetails *s3ErrorDetails,
+                                   void *callbackData)
+{
+    SetSalData *paData = (SetSalData *) callbackData;
+
+    (*(paData->responseCompleteCallback))
+        (requestStatus, s3ErrorDetails, paData->callbackData);
+
+    free(paData);
+}
+
+
+void S3_set_server_access_logging(const S3BucketContext *bucketContext,
+                                  const char *targetBucket, 
+                                  const char *targetPrefix, int aclGrantCount, 
+                                  const S3AclGrant *aclGrants, 
+                                  S3RequestContext *requestContext,
+                                  const S3ResponseHandler *handler,
+                                  void *callbackData)
+{
+    if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) {
+        (*(handler->completeCallback))
+            (S3StatusTooManyGrants, 0, callbackData);
+        return;
+    }
+
+    SetSalData *data = (SetSalData *) malloc(sizeof(SetSalData));
+    if (!data) {
+        (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData);
+        return;
+    }
+    
+    // Convert aclGrants to XML document
+    S3Status status = generateSalXmlDocument
+        (targetBucket, targetPrefix, aclGrantCount, aclGrants,
+         &(data->salXmlDocumentLen), data->salXmlDocument, 
+         sizeof(data->salXmlDocument));
+    if (status != S3StatusOK) {
+        free(data);
+        (*(handler->completeCallback))(status, 0, callbackData);
+        return;
+    }
+
+    data->responsePropertiesCallback = handler->propertiesCallback;
+    data->responseCompleteCallback = handler->completeCallback;
+    data->callbackData = callbackData;
+
+    data->salXmlDocumentBytesWritten = 0;
+
+    // Set up the RequestParams
+    RequestParams params =
+    {
+        HttpRequestTypePUT,                           // httpRequestType
+        { bucketContext->hostName,                    // hostName
+          bucketContext->bucketName,                  // bucketName
+          bucketContext->protocol,                    // protocol
+          bucketContext->uriStyle,                    // uriStyle
+          bucketContext->accessKeyId,                 // accessKeyId
+          bucketContext->secretAccessKey },           // secretAccessKey
+        0,                                            // key
+        0,                                            // queryParams
+        "logging",                                    // subResource
+        0,                                            // copySourceBucketName
+        0,                                            // copySourceKey
+        0,                                            // getConditions
+        0,                                            // startByte
+        0,                                            // byteCount
+        0,                                            // putProperties
+        &setSalPropertiesCallback,                    // propertiesCallback
+        &setSalDataCallback,                          // toS3Callback
+        data->salXmlDocumentLen,                      // toS3CallbackTotalSize
+        0,                                            // fromS3Callback
+        &setSalCompleteCallback,                      // completeCallback
+        data                                          // callbackData
+    };
+
+    // Perform the request
+    request_perform(¶ms, requestContext);
+}
diff --git a/ceph/src/libs3/src/simplexml.c b/ceph/src/libs3/src/simplexml.c
new file mode 100644
index 00000000..bd8616b0
--- /dev/null
+++ b/ceph/src/libs3/src/simplexml.c
@@ -0,0 +1,207 @@
+/** **************************************************************************
+ * simplexml.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "simplexml.h"
+
+// Use libxml2 for parsing XML.  XML is severely overused in modern
+// computing.  It is useful for only a very small subset of tasks, but
+// software developers who don't know better and are afraid to go against the
+// grain use it for everything, and in most cases, it is completely
+// inappropriate.  Usually, the document structure is severely under-specified
+// as well, as is the case with S3.  We do our best by just caring about the
+// most important aspects of the S3 "XML document" responses: the elements and
+// their values.  The SAX API (just about the lamest API ever devised and
+// proof that XML sucks - well, the real proof is how crappy all of the XML
+// parsing libraries are, including libxml2 - but I digress) is used here
+// because we don't need much from the parser and SAX is fast and low memory.
+//
+// Note that for simplicity we assume all ASCII here.  No attempts are made to
+// detect non-ASCII sequences in utf-8 and convert them into ASCII in any way.
+// S3 appears to only use ASCII anyway.
+
+
+static xmlEntityPtr saxGetEntity(void *user_data, const xmlChar *name)
+{
+    (void) user_data;
+
+    return xmlGetPredefinedEntity(name);
+}
+
+
+static void saxStartElement(void *user_data, const xmlChar *nameUtf8,
+                            const xmlChar **attr)
+{
+    (void) attr;
+
+    SimpleXml *simpleXml = (SimpleXml *) user_data;
+
+    if (simpleXml->status != S3StatusOK) {
+        return;
+    }
+    
+    // Assume that name has no non-ASCII in it
+    char *name = (char *) nameUtf8;
+
+    // Append the element to the element path
+    int len = strlen(name);
+
+    if ((simpleXml->elementPathLen + len + 1) >= 
+        (int) sizeof(simpleXml->elementPath)) {
+        // Cannot handle this element, stop!
+        simpleXml->status = S3StatusXmlParseFailure;
+        return;
+    }
+
+    if (simpleXml->elementPathLen) {
+        simpleXml->elementPath[simpleXml->elementPathLen++] = '/';
+    }
+    strcpy(&(simpleXml->elementPath[simpleXml->elementPathLen]), name);
+    simpleXml->elementPathLen += len;
+}
+
+
+static void saxEndElement(void *user_data, const xmlChar *name)
+{
+    (void) name;
+
+    SimpleXml *simpleXml = (SimpleXml *) user_data;
+
+    if (simpleXml->status != S3StatusOK) {
+        return;
+    }
+
+    // Call back with 0 data
+    simpleXml->status = (*(simpleXml->callback))
+        (simpleXml->elementPath, 0, 0, simpleXml->callbackData);
+
+    while ((simpleXml->elementPathLen > 0) &&
+           (simpleXml->elementPath[simpleXml->elementPathLen] != '/')) {
+        simpleXml->elementPathLen--;
+    }
+
+    simpleXml->elementPath[simpleXml->elementPathLen] = 0;
+}
+
+
+static void saxCharacters(void *user_data, const xmlChar *ch, int len)
+{
+    SimpleXml *simpleXml = (SimpleXml *) user_data;
+
+    if (simpleXml->status != S3StatusOK) {
+        return;
+    }
+
+    simpleXml->status = (*(simpleXml->callback))
+        (simpleXml->elementPath, (char *) ch, len, simpleXml->callbackData);
+}
+
+
+static void saxError(void *user_data, const char *msg, ...)
+{
+    (void) msg;
+
+    SimpleXml *simpleXml = (SimpleXml *) user_data;
+
+    if (simpleXml->status != S3StatusOK) {
+        return;
+    }
+
+    simpleXml->status = S3StatusXmlParseFailure;
+}
+
+
+static struct _xmlSAXHandler saxHandlerG =
+{
+    0, // internalSubsetSAXFunc
+    0, // isStandaloneSAXFunc
+    0, // hasInternalSubsetSAXFunc
+    0, // hasExternalSubsetSAXFunc
+    0, // resolveEntitySAXFunc
+    &saxGetEntity, // getEntitySAXFunc
+    0, // entityDeclSAXFunc
+    0, // notationDeclSAXFunc
+    0, // attributeDeclSAXFunc
+    0, // elementDeclSAXFunc
+    0, // unparsedEntityDeclSAXFunc
+    0, // setDocumentLocatorSAXFunc
+    0, // startDocumentSAXFunc
+    0, // endDocumentSAXFunc
+    &saxStartElement, // startElementSAXFunc
+    &saxEndElement, // endElementSAXFunc
+    0, // referenceSAXFunc
+    &saxCharacters, // charactersSAXFunc
+    0, // ignorableWhitespaceSAXFunc
+    0, // processingInstructionSAXFunc
+    0, // commentSAXFunc
+    0, // warningSAXFunc
+    &saxError, // errorSAXFunc
+    &saxError, // fatalErrorSAXFunc
+    0, // getParameterEntitySAXFunc
+    &saxCharacters, // cdataBlockSAXFunc
+    0, // externalSubsetSAXFunc
+    0, // initialized
+    0, // _private
+    0, // startElementNsSAX2Func
+    0, // endElementNsSAX2Func
+    0 // xmlStructuredErrorFunc serror;
+};
+
+void simplexml_initialize(SimpleXml *simpleXml, 
+                          SimpleXmlCallback *callback, void *callbackData)
+{
+    simpleXml->callback = callback;
+    simpleXml->callbackData = callbackData;
+    simpleXml->elementPathLen = 0;
+    simpleXml->status = S3StatusOK;
+    simpleXml->xmlParser = 0;
+}
+
+
+void simplexml_deinitialize(SimpleXml *simpleXml)
+{
+    if (simpleXml->xmlParser) {
+        xmlFreeParserCtxt(simpleXml->xmlParser);
+    }
+}
+
+
+S3Status simplexml_add(SimpleXml *simpleXml, const char *data, int dataLen)
+{
+    if (!simpleXml->xmlParser &&
+        (!(simpleXml->xmlParser = xmlCreatePushParserCtxt
+           (&saxHandlerG, simpleXml, 0, 0, 0)))) {
+        return S3StatusInternalError;
+    }
+
+    if (xmlParseChunk((xmlParserCtxtPtr) simpleXml->xmlParser, 
+                      data, dataLen, 0)) {
+        return S3StatusXmlParseFailure;
+    }
+
+    return simpleXml->status;
+}
diff --git a/ceph/src/libs3/src/testsimplexml.c b/ceph/src/libs3/src/testsimplexml.c
new file mode 100644
index 00000000..57fba7d7
--- /dev/null
+++ b/ceph/src/libs3/src/testsimplexml.c
@@ -0,0 +1,87 @@
+/** **************************************************************************
+ * testsimplexml.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include 
+#include 
+#include "simplexml.h"
+
+static S3Status simpleXmlCallback(const char *elementPath, const char *data,
+                                  int dataLen, void *callbackData)
+{
+    (void) callbackData;
+
+    printf("[%s]: [%.*s]\n", elementPath, dataLen, data);
+
+    return S3StatusOK;
+}
+
+
+// The only argument allowed is a specification of the random seed to use
+int main(int argc, char **argv)
+{
+    if (argc > 1) {
+        char *arg = argv[1];
+        int seed = 0;
+        while (*arg) {
+            seed *= 10;
+            seed += (*arg++ - '0');
+        }
+        
+        srand(seed);
+    }
+    else {
+        srand(time(0));
+    }
+
+    SimpleXml simpleXml;
+
+    simplexml_initialize(&simpleXml, &simpleXmlCallback, 0);
+
+    // Read chunks of 10K from stdin, and then feed them in random chunks
+    // to simplexml_add
+    char inbuf[10000];
+
+    int amt_read;
+    while ((amt_read = fread(inbuf, 1, sizeof(inbuf), stdin)) > 0) {
+        char *buf = inbuf;
+        while (amt_read) {
+            int amt = (rand() % amt_read) + 1;
+            S3Status status = simplexml_add(&simpleXml, buf, amt);
+            if (status != S3StatusOK) {
+                fprintf(stderr, "ERROR: Parse failure: %d\n", status);
+                simplexml_deinitialize(&simpleXml);
+                return -1;
+            }
+            buf += amt, amt_read -= amt;
+        }
+    }
+
+    simplexml_deinitialize(&simpleXml);
+
+    return 0;
+}
diff --git a/ceph/src/libs3/src/util.c b/ceph/src/libs3/src/util.c
new file mode 100644
index 00000000..0737084a
--- /dev/null
+++ b/ceph/src/libs3/src/util.c
@@ -0,0 +1,560 @@
+/** **************************************************************************
+ * util.c
+ * 
+ * Copyright 2008 Bryan Ischo 
+ * 
+ * This file is part of libs3.
+ * 
+ * libs3 is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3 of the License.
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of this library and its programs with the
+ * OpenSSL library, and distribute linked combinations including the two.
+ *
+ * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License version 3
+ * along with libs3, in a file named COPYING.  If not, see
+ * .
+ *
+ ************************************************************************** **/
+
+#include 
+#include 
+#include "util.h"
+
+
+// Convenience utility for making the code look nicer.  Tests a string
+// against a format; only the characters specified in the format are
+// checked (i.e. if the string is longer than the format, the string still
+// checks out ok).  Format characters are:
+// d - is a digit
+// anything else - is that character
+// Returns nonzero the string checks out, zero if it does not.
+static int checkString(const char *str, const char *format)
+{
+    while (*format) {
+        if (*format == 'd') {
+            if (!isdigit(*str)) {
+                return 0;
+            }
+        }
+        else if (*str != *format) {
+            return 0;
+        }
+        str++, format++;
+    }
+
+    return 1;
+}
+
+
+int urlEncode(char *dest, const char *src, int maxSrcSize)
+{
+    static const char *hex = "0123456789ABCDEF";
+
+    int len = 0;
+
+    if (src) while (*src) {
+        if (++len > maxSrcSize) {
+            *dest = 0;
+            return 0;
+        }
+        unsigned char c = *src;
+        if (isalnum(c) ||
+            (c == '-') || (c == '_') || (c == '.') || (c == '!') || 
+            (c == '~') || (c == '*') || (c == '\'') || (c == '(') ||
+            (c == ')') || (c == '/')) {
+            *dest++ = c;
+        }
+        else if (*src == ' ') {
+            *dest++ = '+';
+        }
+        else {
+            *dest++ = '%';
+            *dest++ = hex[c >> 4];
+            *dest++ = hex[c & 15];
+        }
+        src++;
+    }
+
+    *dest = 0;
+
+    return 1;
+}
+
+
+int64_t parseIso8601Time(const char *str)
+{
+    // Check to make sure that it has a valid format
+    if (!checkString(str, "dddd-dd-ddTdd:dd:dd")) {
+        return -1;
+    }
+
+#define nextnum() (((*str - '0') * 10) + (*(str + 1) - '0'))
+
+    // Convert it
+    struct tm stm;
+    memset(&stm, 0, sizeof(stm));
+
+    stm.tm_year = (nextnum() - 19) * 100;
+    str += 2;
+    stm.tm_year += nextnum();
+    str += 3;
+
+    stm.tm_mon = nextnum() - 1;
+    str += 3;
+
+    stm.tm_mday = nextnum();
+    str += 3;
+
+    stm.tm_hour = nextnum();
+    str += 3;
+
+    stm.tm_min = nextnum();
+    str += 3;
+
+    stm.tm_sec = nextnum();
+    str += 2;
+
+    stm.tm_isdst = -1;
+    
+    int64_t ret = mktime(&stm);
+
+    // Skip the millis
+
+    if (*str == '.') {
+        str++;
+        while (isdigit(*str)) {
+            str++;
+        }
+    }
+    
+    if (checkString(str, "-dd:dd") || checkString(str, "+dd:dd")) {
+        int sign = (*str++ == '-') ? -1 : 1;
+        int hours = nextnum();
+        str += 3;
+        int minutes = nextnum();
+        ret += (-sign * (((hours * 60) + minutes) * 60));
+    }
+    // Else it should be Z to be a conformant time string, but we just assume
+    // that it is rather than enforcing that
+
+    return ret;
+}
+
+
+uint64_t parseUnsignedInt(const char *str)
+{
+    // Skip whitespace
+    while (is_blank(*str)) {
+        str++;
+    }
+
+    uint64_t ret = 0;
+
+    while (isdigit(*str)) {
+        ret *= 10;
+        ret += (*str++ - '0');
+    }
+
+    return ret;
+}
+
+
+int base64Encode(const unsigned char *in, int inLen, char *out)
+{
+    static const char *ENC = 
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+    char *original_out = out;
+
+    while (inLen) {
+        // first 6 bits of char 1
+        *out++ = ENC[*in >> 2];
+        if (!--inLen) {
+            // last 2 bits of char 1, 4 bits of 0
+            *out++ = ENC[(*in & 0x3) << 4];
+            *out++ = '=';
+            *out++ = '=';
+            break;
+        }
+        // last 2 bits of char 1, first 4 bits of char 2
+        *out++ = ENC[((*in & 0x3) << 4) | (*(in + 1) >> 4)];
+        in++;
+        if (!--inLen) {
+            // last 4 bits of char 2, 2 bits of 0
+            *out++ = ENC[(*in & 0xF) << 2];
+            *out++ = '=';
+            break;
+        }
+        // last 4 bits of char 2, first 2 bits of char 3
+        *out++ = ENC[((*in & 0xF) << 2) | (*(in + 1) >> 6)];
+        in++;
+        // last 6 bits of char 3
+        *out++ = ENC[*in & 0x3F];
+        in++, inLen--;
+    }
+
+    return (out - original_out);
+}
+
+
+#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits))))
+
+#define blk0L(i) (block->l[i] = (rol(block->l[i], 24) & 0xFF00FF00)     \
+                  | (rol(block->l[i], 8) & 0x00FF00FF))
+
+#define blk0B(i) (block->l[i])
+
+#define blk(i) (block->l[i & 15] = rol(block->l[(i + 13) & 15] ^        \
+                                       block->l[(i + 8) & 15] ^         \
+                                       block->l[(i + 2) & 15] ^         \
+                                       block->l[i & 15], 1))
+
+#define R0_L(v, w, x, y, z, i)                                          \
+    z += ((w & (x ^ y)) ^ y) + blk0L(i) + 0x5A827999 + rol(v, 5);       \
+    w = rol(w, 30);
+#define R0_B(v, w, x, y, z, i)                                          \
+    z += ((w & (x ^ y)) ^ y) + blk0B(i) + 0x5A827999 + rol(v, 5);       \
+    w = rol(w, 30);
+#define R1(v, w, x, y, z, i)                                            \
+    z += ((w & (x ^ y)) ^ y) + blk(i) + 0x5A827999 + rol(v, 5);         \
+    w = rol(w, 30);
+#define R2(v, w, x, y, z, i)                                            \
+    z += (w ^ x ^ y) + blk(i) + 0x6ED9EBA1 + rol(v, 5);                 \
+    w = rol(w, 30);
+#define R3(v, w, x, y, z, i)                                            \
+    z += (((w | x) & y) | (w & x)) + blk(i) + 0x8F1BBCDC + rol(v, 5);   \
+    w = rol(w, 30);
+#define R4(v, w, x, y, z, i)                                            \
+    z += (w ^ x ^ y) + blk(i) + 0xCA62C1D6 + rol(v, 5);                 \
+    w = rol(w, 30);
+
+#define R0A_L(i) R0_L(a, b, c, d, e, i)
+#define R0B_L(i) R0_L(b, c, d, e, a, i)
+#define R0C_L(i) R0_L(c, d, e, a, b, i)
+#define R0D_L(i) R0_L(d, e, a, b, c, i)
+#define R0E_L(i) R0_L(e, a, b, c, d, i)
+
+#define R0A_B(i) R0_B(a, b, c, d, e, i)
+#define R0B_B(i) R0_B(b, c, d, e, a, i)
+#define R0C_B(i) R0_B(c, d, e, a, b, i)
+#define R0D_B(i) R0_B(d, e, a, b, c, i)
+#define R0E_B(i) R0_B(e, a, b, c, d, i)
+
+#define R1A(i) R1(a, b, c, d, e, i)
+#define R1B(i) R1(b, c, d, e, a, i)
+#define R1C(i) R1(c, d, e, a, b, i)
+#define R1D(i) R1(d, e, a, b, c, i)
+#define R1E(i) R1(e, a, b, c, d, i)
+
+#define R2A(i) R2(a, b, c, d, e, i)
+#define R2B(i) R2(b, c, d, e, a, i)
+#define R2C(i) R2(c, d, e, a, b, i)
+#define R2D(i) R2(d, e, a, b, c, i)
+#define R2E(i) R2(e, a, b, c, d, i)
+
+#define R3A(i) R3(a, b, c, d, e, i)
+#define R3B(i) R3(b, c, d, e, a, i)
+#define R3C(i) R3(c, d, e, a, b, i)
+#define R3D(i) R3(d, e, a, b, c, i)
+#define R3E(i) R3(e, a, b, c, d, i)
+
+#define R4A(i) R4(a, b, c, d, e, i)
+#define R4B(i) R4(b, c, d, e, a, i)
+#define R4C(i) R4(c, d, e, a, b, i)
+#define R4D(i) R4(d, e, a, b, c, i)
+#define R4E(i) R4(e, a, b, c, d, i)
+
+
+static void SHA1_transform(uint32_t state[5], const unsigned char buffer[64])
+{
+    uint32_t a, b, c, d, e;
+
+    typedef union {
+        unsigned char c[64];
+        uint32_t l[16];
+    } u;
+
+    unsigned char w[64];
+    u *block = (u *) w;
+
+    memcpy(block, buffer, 64);
+
+    a = state[0];
+    b = state[1];
+    c = state[2];
+    d = state[3];
+    e = state[4];
+
+    static uint32_t endianness_indicator = 0x1;
+    if (((unsigned char *) &endianness_indicator)[0]) {
+        R0A_L( 0);
+        R0E_L( 1); R0D_L( 2); R0C_L( 3); R0B_L( 4); R0A_L( 5);
+        R0E_L( 6); R0D_L( 7); R0C_L( 8); R0B_L( 9); R0A_L(10);
+        R0E_L(11); R0D_L(12); R0C_L(13); R0B_L(14); R0A_L(15);
+    }
+    else {
+        R0A_B( 0);
+        R0E_B( 1); R0D_B( 2); R0C_B( 3); R0B_B( 4); R0A_B( 5);
+        R0E_B( 6); R0D_B( 7); R0C_B( 8); R0B_B( 9); R0A_B(10);
+        R0E_B(11); R0D_B(12); R0C_B(13); R0B_B(14); R0A_B(15);
+    }
+    R1E(16); R1D(17); R1C(18); R1B(19); R2A(20);
+    R2E(21); R2D(22); R2C(23); R2B(24); R2A(25);
+    R2E(26); R2D(27); R2C(28); R2B(29); R2A(30);
+    R2E(31); R2D(32); R2C(33); R2B(34); R2A(35);
+    R2E(36); R2D(37); R2C(38); R2B(39); R3A(40);
+    R3E(41); R3D(42); R3C(43); R3B(44); R3A(45);
+    R3E(46); R3D(47); R3C(48); R3B(49); R3A(50);
+    R3E(51); R3D(52); R3C(53); R3B(54); R3A(55);
+    R3E(56); R3D(57); R3C(58); R3B(59); R4A(60);
+    R4E(61); R4D(62); R4C(63); R4B(64); R4A(65);
+    R4E(66); R4D(67); R4C(68); R4B(69); R4A(70);
+    R4E(71); R4D(72); R4C(73); R4B(74); R4A(75);
+    R4E(76); R4D(77); R4C(78); R4B(79);
+
+    state[0] += a;
+    state[1] += b;
+    state[2] += c;
+    state[3] += d;
+    state[4] += e;
+}
+
+
+typedef struct
+{
+    uint32_t state[5];
+    uint32_t count[2];
+    unsigned char buffer[64];
+} SHA1Context;
+
+
+static void SHA1_init(SHA1Context *context)
+{
+    context->state[0] = 0x67452301;
+    context->state[1] = 0xEFCDAB89;
+    context->state[2] = 0x98BADCFE;
+    context->state[3] = 0x10325476;
+    context->state[4] = 0xC3D2E1F0;
+    context->count[0] = context->count[1] = 0;
+}
+
+
+static void SHA1_update(SHA1Context *context, const unsigned char *data,
+                        unsigned int len)
+{
+    uint32_t i, j;
+
+    j = (context->count[0] >> 3) & 63;
+
+    if ((context->count[0] += len << 3) < (len << 3)) {
+        context->count[1]++;
+    }
+
+    context->count[1] += (len >> 29);
+
+    if ((j + len) > 63) {
+        memcpy(&(context->buffer[j]), data, (i = 64 - j));
+        SHA1_transform(context->state, context->buffer);
+        for ( ; (i + 63) < len; i += 64) {
+            SHA1_transform(context->state, &(data[i]));
+        }
+        j = 0;
+    }
+    else {
+        i = 0;
+    }
+
+    memcpy(&(context->buffer[j]), &(data[i]), len - i);
+}
+
+
+static void SHA1_final(unsigned char digest[20], SHA1Context *context)
+{
+    uint32_t i;
+    unsigned char finalcount[8];
+
+    for (i = 0; i < 8; i++) {
+        finalcount[i] = (unsigned char)
+            ((context->count[(i >= 4 ? 0 : 1)] >>
+              ((3 - (i & 3)) * 8)) & 255);
+    }
+
+    SHA1_update(context, (unsigned char *) "\200", 1);
+
+    while ((context->count[0] & 504) != 448) {
+        SHA1_update(context, (unsigned char *) "\0", 1);
+    }
+
+    SHA1_update(context, finalcount, 8);
+
+    for (i = 0; i < 20; i++) {
+        digest[i] = (unsigned char)
+            ((context->state[i >> 2] >> ((3 - (i & 3)) * 8)) & 255);
+    }
+
+    memset(context->buffer, 0, 64);
+    memset(context->state, 0, 20);
+    memset(context->count, 0, 8);
+    memset(&finalcount, 0, 8);
+
+    SHA1_transform(context->state, context->buffer);
+}
+
+
+// HMAC-SHA-1:
+//
+// K - is key padded with zeros to 512 bits
+// m - is message
+// OPAD - 0x5c5c5c...
+// IPAD - 0x363636...
+//
+// HMAC(K,m) = SHA1((K ^ OPAD) . SHA1((K ^ IPAD) . m))
+void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len,
+               const unsigned char *message, int message_len)
+{
+    unsigned char kopad[64], kipad[64];
+    int i;
+    
+    if (key_len > 64) {
+        key_len = 64;
+    }
+
+    for (i = 0; i < key_len; i++) {
+        kopad[i] = key[i] ^ 0x5c;
+        kipad[i] = key[i] ^ 0x36;
+    }
+
+    for ( ; i < 64; i++) {
+        kopad[i] = 0 ^ 0x5c;
+        kipad[i] = 0 ^ 0x36;
+    }
+
+    unsigned char digest[20];
+
+    SHA1Context context;
+    
+    SHA1_init(&context);
+    SHA1_update(&context, kipad, 64);
+    SHA1_update(&context, message, message_len);
+    SHA1_final(digest, &context);
+
+    SHA1_init(&context);
+    SHA1_update(&context, kopad, 64);
+    SHA1_update(&context, digest, 20);
+    SHA1_final(hmac, &context);
+}
+
+#define rot(x,k) (((x) << (k)) | ((x) >> (32 - (k))))
+
+uint64_t hash(const unsigned char *k, int length)
+{
+    uint32_t a, b, c;
+
+    a = b = c = 0xdeadbeef + ((uint32_t) length);
+
+    static uint32_t endianness_indicator = 0x1;
+    if (((unsigned char *) &endianness_indicator)[0]) {
+        while (length > 12) {
+            a += k[0];
+            a += ((uint32_t) k[1]) << 8;
+            a += ((uint32_t) k[2]) << 16;
+            a += ((uint32_t) k[3]) << 24;
+            b += k[4];
+            b += ((uint32_t) k[5]) << 8;
+            b += ((uint32_t) k[6]) << 16;
+            b += ((uint32_t) k[7]) << 24;
+            c += k[8];
+            c += ((uint32_t) k[9]) << 8;
+            c += ((uint32_t) k[10]) << 16;
+            c += ((uint32_t) k[11]) << 24;
+            a -= c; a ^= rot(c, 4);  c += b;
+            b -= a; b ^= rot(a, 6);  a += c;
+            c -= b; c ^= rot(b, 8);  b += a;
+            a -= c; a ^= rot(c, 16);  c += b;
+            b -= a; b ^= rot(a, 19);  a += c;
+            c -= b; c ^= rot(b, 4);  b += a;
+            length -= 12;
+            k += 12;
+        }
+        
+        switch(length) {
+        case 12: c += ((uint32_t) k[11]) << 24;
+        case 11: c += ((uint32_t) k[10]) << 16;
+        case 10: c += ((uint32_t) k[9]) << 8;
+        case 9 : c += k[8];
+        case 8 : b += ((uint32_t) k[7]) << 24;
+        case 7 : b += ((uint32_t) k[6]) << 16;
+        case 6 : b += ((uint32_t) k[5]) << 8;
+        case 5 : b += k[4];
+        case 4 : a += ((uint32_t) k[3]) << 24;
+        case 3 : a += ((uint32_t) k[2]) << 16;
+        case 2 : a += ((uint32_t) k[1]) << 8;
+        case 1 : a += k[0]; break;
+        case 0 : goto end;
+        }
+    }
+    else {
+        while (length > 12) {
+            a += ((uint32_t) k[0]) << 24;
+            a += ((uint32_t) k[1]) << 16;
+            a += ((uint32_t) k[2]) << 8;
+            a += ((uint32_t) k[3]);
+            b += ((uint32_t) k[4]) << 24;
+            b += ((uint32_t) k[5]) << 16;
+            b += ((uint32_t) k[6]) << 8;
+            b += ((uint32_t) k[7]);
+            c += ((uint32_t) k[8]) << 24;
+            c += ((uint32_t) k[9]) << 16;
+            c += ((uint32_t) k[10]) << 8;
+            c += ((uint32_t) k[11]);
+            a -= c; a ^= rot(c, 4);  c += b;
+            b -= a; b ^= rot(a, 6);  a += c;
+            c -= b; c ^= rot(b, 8);  b += a;
+            a -= c; a ^= rot(c, 16);  c += b;
+            b -= a; b ^= rot(a, 19);  a += c;
+            c -= b; c ^= rot(b, 4);  b += a;
+            length -= 12;
+            k += 12;
+        }
+
+        switch(length) {
+        case 12: c += k[11];
+        case 11: c += ((uint32_t) k[10]) << 8;
+        case 10: c += ((uint32_t) k[9]) << 16;
+        case 9 : c += ((uint32_t) k[8]) << 24;
+        case 8 : b += k[7];
+        case 7 : b += ((uint32_t) k[6]) << 8;
+        case 6 : b += ((uint32_t) k[5]) << 16;
+        case 5 : b += ((uint32_t) k[4]) << 24;
+        case 4 : a += k[3];
+        case 3 : a += ((uint32_t) k[2]) << 8;
+        case 2 : a += ((uint32_t) k[1]) << 16;
+        case 1 : a += ((uint32_t) k[0]) << 24; break;
+        case 0 : goto end;
+        }
+    }
+    
+    c ^= b; c -= rot(b, 14);
+    a ^= c; a -= rot(c, 11);
+    b ^= a; b -= rot(a, 25);
+    c ^= b; c -= rot(b, 16);
+    a ^= c; a -= rot(c, 4);
+    b ^= a; b -= rot(a, 14);
+    c ^= b; c -= rot(b, 24);
+
+ end:
+    return ((((uint64_t) c) << 32) | b);
+}
+
+int is_blank(char c)
+{
+    return ((c == ' ') || (c == '\t'));
+}
diff --git a/ceph/src/libs3/test/badxml_01.xml b/ceph/src/libs3/test/badxml_01.xml
new file mode 100644
index 00000000..e9917023
--- /dev/null
+++ b/ceph/src/libs3/test/badxml_01.xml
@@ -0,0 +1,105 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Data
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ceph/src/libs3/test/goodxml_01.xml b/ceph/src/libs3/test/goodxml_01.xml
new file mode 100644
index 00000000..687214ee
--- /dev/null
+++ b/ceph/src/libs3/test/goodxml_01.xml
@@ -0,0 +1,7 @@
+
+
+  NoSuchKey
+   The resource  & then]]> you requested does not exist & so there  
+  /mybucket/myfoto.jpg 
+  4442587FB7D0A2F9
+
diff --git a/ceph/src/libs3/test/goodxml_02.xml b/ceph/src/libs3/test/goodxml_02.xml
new file mode 100644
index 00000000..d713bc68
--- /dev/null
+++ b/ceph/src/libs3/test/goodxml_02.xml
@@ -0,0 +1,105 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Data
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ceph/src/libs3/test/goodxml_03.xml b/ceph/src/libs3/test/goodxml_03.xml
new file mode 100644
index 00000000..9a851668
--- /dev/null
+++ b/ceph/src/libs3/test/goodxml_03.xml
@@ -0,0 +1,13 @@
+
+
+1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+
+
diff --git a/ceph/src/libs3/test/test.sh b/ceph/src/libs3/test/test.sh
new file mode 100755
index 00000000..3acfc86c
--- /dev/null
+++ b/ceph/src/libs3/test/test.sh
@@ -0,0 +1,173 @@
+#!/bin/sh
+
+# Environment:
+# S3_ACCESS_KEY_ID - must be set to S3 Access Key ID
+# S3_SECRET_ACCESS_KEY - must be set to S3 Secret Access Key
+# TEST_BUCKET_PREFIX - must be set to the test bucket prefix to use
+# S3_COMMAND - may be set to s3 command to use (i.e. valgrind s3); defaults
+#              to "s3"
+
+if [ -z "$S3_ACCESS_KEY_ID" ]; then
+    echo "S3_ACCESS_KEY_ID required"
+    exit -1;
+fi
+
+if [ -z "$S3_SECRET_ACCESS_KEY" ]; then
+    echo "S3_SECRET_ACCESS_KEY required"
+    exit -1;
+fi
+
+if [ -z "$TEST_BUCKET_PREFIX" ]; then
+    echo "TEST_BUCKET_PREFIX required"
+    exit -1;
+fi
+
+if [ -z "$S3_COMMAND" ]; then
+    S3_COMMAND=s3
+fi
+
+TEST_BUCKET=${TEST_BUCKET_PREFIX}.testbucket
+
+# Create the test bucket in EU
+echo "$S3_COMMAND create $TEST_BUCKET locationConstraint=EU"
+$S3_COMMAND create $TEST_BUCKET
+
+# List to find it
+echo "$S3_COMMAND list | grep $TEST_BUCKET"
+$S3_COMMAND list | grep $TEST_BUCKET
+
+# Test it
+echo "$S3_COMMAND test $TEST_BUCKET"
+$S3_COMMAND test $TEST_BUCKET
+
+# List to ensure that it is empty
+echo "$S3_COMMAND list $TEST_BUCKET"
+$S3_COMMAND list $TEST_BUCKET
+
+# Put some data
+rm -f seqdata
+seq 1 10000 > seqdata
+echo "$S3_COMMAND put $TEST_BUCKET/testkey filename=seqdata noStatus=1"
+$S3_COMMAND put $TEST_BUCKET/testkey filename=seqdata noStatus=1
+
+rm -f testkey
+# Get the data and make sure that it matches
+echo "$S3_COMMAND get $TEST_BUCKET/testkey filename=testkey"
+$S3_COMMAND get $TEST_BUCKET/testkey filename=testkey
+diff seqdata testkey
+rm -f seqdata testkey
+
+# Delete the file
+echo "$S3_COMMAND delete $TEST_BUCKET/testkey"
+$S3_COMMAND delete $TEST_BUCKET/testkey
+
+# Remove the test bucket
+echo "$S3_COMMAND delete $TEST_BUCKET"
+$S3_COMMAND delete $TEST_BUCKET
+
+# Make sure it's not there
+echo "$S3_COMMAND list | grep $TEST_BUCKET"
+$S3_COMMAND list | grep $TEST_BUCKET
+
+# Now create it again
+echo "$S3_COMMAND create $TEST_BUCKET"
+$S3_COMMAND create $TEST_BUCKET
+
+# Put 10 files in it
+for i in `seq 0 9`; do
+    echo "echo \"Hello\" | $S3_COMMAND put $TEST_BUCKET/key_$i"
+    echo "Hello" | $S3_COMMAND put $TEST_BUCKET/key_$i
+done
+
+# List with all details
+echo "$S3_COMMAND list $TEST_BUCKET allDetails=1"
+$S3_COMMAND list $TEST_BUCKET allDetails=1
+
+COPY_BUCKET=${TEST_BUCKET_PREFIX}.copybucket
+
+# Create another test bucket and copy a file into it
+echo "$S3_COMMAND create $COPY_BUCKET"
+$S3_COMMAND create $COPY_BUCKET
+echo <> acl
+Group   Authenticated AWS Users                                   READ
+EOF
+echo <> acl
+Group   All Users                                                 READ_ACP
+EOF
+echo "$S3_COMMAND setacl $TEST_BUCKET filename=acl"
+$S3_COMMAND setacl $TEST_BUCKET filename=acl
+
+# Test to make sure that it worked
+rm -f acl_new
+echo "$S3_COMMAND getacl $TEST_BUCKET filename=acl_new allDetails=1"
+$S3_COMMAND getacl $TEST_BUCKET filename=acl_new allDetails=1
+diff acl acl_new
+rm -f acl acl_new
+
+# Get the key acl
+rm -f acl
+echo "$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl allDetails=1"
+$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl allDetails=1
+
+# Add READ for all AWS users, and READ_ACP for everyone
+echo <> acl
+Group   Authenticated AWS Users                                   READ
+EOF
+echo <> acl
+Group   All Users                                                 READ_ACP
+EOF
+echo "$S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl"
+$S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl
+
+# Test to make sure that it worked
+rm -f acl_new
+echo "$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl_new allDetails=1"
+$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl_new allDetails=1
+diff acl acl_new
+rm -f acl acl_new
+
+# Remove the test file
+echo "$S3_COMMAND delete $TEST_BUCKET/aclkey"
+$S3_COMMAND delete $TEST_BUCKET/aclkey
+echo "$S3_COMMAND delete $TEST_BUCKET"
+$S3_COMMAND delete $TEST_BUCKET
diff --git a/ceph/src/log/Entry.h b/ceph/src/log/Entry.h
new file mode 100644
index 00000000..7cdf1161
--- /dev/null
+++ b/ceph/src/log/Entry.h
@@ -0,0 +1,56 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef __CEPH_LOG_ENTRY_H
+#define __CEPH_LOG_ENTRY_H
+
+#include "include/utime.h"
+#include "common/PrebufferedStreambuf.h"
+#include 
+#include 
+
+#define CEPH_LOG_ENTRY_PREALLOC 80
+
+namespace ceph {
+namespace log {
+
+struct Entry {
+  utime_t m_stamp;
+  pthread_t m_thread;
+  short m_prio, m_subsys;
+  Entry *m_next;
+
+  char m_static_buf[CEPH_LOG_ENTRY_PREALLOC];
+  PrebufferedStreambuf m_streambuf;
+
+  Entry()
+    : m_thread(0), m_prio(0), m_subsys(0),
+      m_next(NULL),
+      m_streambuf(m_static_buf, sizeof(m_static_buf))
+  {}
+  Entry(utime_t s, pthread_t t, short pr, short sub,
+	const char *msg = NULL)
+    : m_stamp(s), m_thread(t), m_prio(pr), m_subsys(sub),
+      m_next(NULL),
+      m_streambuf(m_static_buf, sizeof(m_static_buf))
+  {
+    if (msg) {
+      ostream os(&m_streambuf);
+      os << msg;
+    }
+  }
+
+  void set_str(const std::string &s) {
+    ostream os(&m_streambuf);
+    os << s;
+  }
+
+  std::string get_str() const {
+    return m_streambuf.get_str();
+  }
+};
+
+}
+}
+
+#endif
diff --git a/ceph/src/log/EntryQueue.h b/ceph/src/log/EntryQueue.h
new file mode 100644
index 00000000..d125540c
--- /dev/null
+++ b/ceph/src/log/EntryQueue.h
@@ -0,0 +1,71 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef __CEPH_LOG_ENTRYQUEUE_H
+#define __CEPH_LOG_ENTRYQUEUE_H
+
+#include "Entry.h"
+
+namespace ceph {
+namespace log {
+
+struct EntryQueue {
+  int m_len;
+  struct Entry *m_head, *m_tail;
+
+  bool empty() const {
+    return m_len == 0;
+  }
+
+  void swap(EntryQueue& other) {
+    int len = m_len;
+    struct Entry *h = m_head, *t = m_tail;
+    m_len = other.m_len;
+    m_head = other.m_head;
+    m_tail = other.m_tail;
+    other.m_len = len;
+    other.m_head = h;
+    other.m_tail = t;
+  }
+
+  void enqueue(Entry *e) {
+    if (m_tail) {
+      m_tail->m_next = e;
+      m_tail = e;
+    } else {
+      m_head = m_tail = e;
+    }
+    m_len++;
+  }
+
+  Entry *dequeue() {
+    if (!m_head)
+      return NULL;
+    Entry *e = m_head;
+    m_head = m_head->m_next;
+    if (!m_head)
+      m_tail = NULL;
+    m_len--;
+    e->m_next = NULL;
+    return e;
+  }
+
+  EntryQueue()
+    : m_len(0),
+      m_head(NULL),
+      m_tail(NULL)
+  {}
+  ~EntryQueue() {
+    Entry *t;
+    while (m_head) {
+      t = m_head->m_next;
+      delete m_head;
+      m_head = t;
+    }      
+  }
+};
+
+}
+}
+
+#endif
diff --git a/ceph/src/log/Log.cc b/ceph/src/log/Log.cc
new file mode 100644
index 00000000..37bb4ef6
--- /dev/null
+++ b/ceph/src/log/Log.cc
@@ -0,0 +1,331 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "Log.h"
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "common/Clock.h"
+#include "include/assert.h"
+#include "include/compat.h"
+#include "include/on_exit.h"
+
+#define DEFAULT_MAX_NEW    100
+#define DEFAULT_MAX_RECENT 10000
+
+#define PREALLOC 1000000
+
+namespace ceph {
+namespace log {
+
+static OnExitManager exit_callbacks;
+
+static void log_on_exit(void *p)
+{
+  Log *l = *(Log **)p;
+  if (l)
+    l->flush();
+}
+
+Log::Log(SubsystemMap *s)
+  : m_indirect_this(NULL),
+    m_subs(s),
+    m_new(), m_recent(),
+    m_fd(-1),
+    m_syslog_log(-2), m_syslog_crash(-2),
+    m_stderr_log(1), m_stderr_crash(-1),
+    m_stop(false),
+    m_max_new(DEFAULT_MAX_NEW),
+    m_max_recent(DEFAULT_MAX_RECENT)
+{
+  int ret;
+
+  ret = pthread_mutex_init(&m_flush_mutex, NULL);
+  assert(ret == 0);
+
+  ret = pthread_mutex_init(&m_queue_mutex, NULL);
+  assert(ret == 0);
+
+  ret = pthread_cond_init(&m_cond_loggers, NULL);
+  assert(ret == 0);
+
+  ret = pthread_cond_init(&m_cond_flusher, NULL);
+  assert(ret == 0);
+
+  // kludge for prealloc testing
+  if (false)
+    for (int i=0; i < PREALLOC; i++)
+      m_recent.enqueue(new Entry);
+}
+
+Log::~Log()
+{
+  if (m_indirect_this) {
+    *m_indirect_this = NULL;
+  }
+
+  assert(!is_started());
+  if (m_fd >= 0)
+    VOID_TEMP_FAILURE_RETRY(::close(m_fd));
+
+  pthread_mutex_destroy(&m_queue_mutex);
+  pthread_mutex_destroy(&m_flush_mutex);
+  pthread_cond_destroy(&m_cond_loggers);
+  pthread_cond_destroy(&m_cond_flusher);
+}
+
+
+///
+
+void Log::set_flush_on_exit()
+{
+  // Make sure we flush on shutdown.  We do this by deliberately
+  // leaking an indirect pointer to ourselves (on_exit() can't
+  // unregister a callback).  This is not racy only becuase we
+  // assume that exit() won't race with ~Log().
+  if (m_indirect_this == NULL) {
+    m_indirect_this = new (Log*)(this);
+    exit_callbacks.add_callback(log_on_exit, m_indirect_this);
+  }
+}
+
+void Log::set_max_new(int n)
+{
+  m_max_new = n;
+}
+
+void Log::set_max_recent(int n)
+{
+  m_max_recent = n;
+}
+
+void Log::set_log_file(string fn)
+{
+  m_log_file = fn;
+}
+
+void Log::reopen_log_file()
+{
+  if (m_fd >= 0)
+    VOID_TEMP_FAILURE_RETRY(::close(m_fd));
+  if (m_log_file.length()) {
+    m_fd = ::open(m_log_file.c_str(), O_CREAT|O_WRONLY|O_APPEND, 0644);
+  } else {
+    m_fd = -1;
+  }
+}
+
+void Log::set_syslog_level(int log, int crash)
+{
+  pthread_mutex_lock(&m_flush_mutex);
+  m_syslog_log = log;
+  m_syslog_crash = crash;
+  pthread_mutex_unlock(&m_flush_mutex);
+}
+
+void Log::set_stderr_level(int log, int crash)
+{
+  pthread_mutex_lock(&m_flush_mutex);
+  m_stderr_log = log;
+  m_stderr_crash = crash;
+  pthread_mutex_unlock(&m_flush_mutex);
+}
+
+void Log::submit_entry(Entry *e)
+{
+  pthread_mutex_lock(&m_queue_mutex);
+
+  // wait for flush to catch up
+  while (m_new.m_len > m_max_new)
+    pthread_cond_wait(&m_cond_loggers, &m_queue_mutex);
+
+  m_new.enqueue(e);
+  pthread_cond_signal(&m_cond_flusher);
+  pthread_mutex_unlock(&m_queue_mutex);
+}
+
+Entry *Log::create_entry(int level, int subsys)
+{
+  if (true) {
+    return new Entry(ceph_clock_now(NULL),
+		   pthread_self(),
+		   level, subsys);
+  } else {
+    // kludge for perf testing
+    Entry *e = m_recent.dequeue();
+    e->m_stamp = ceph_clock_now(NULL);
+    e->m_thread = pthread_self();
+    e->m_prio = level;
+    e->m_subsys = subsys;
+    return e;
+  }
+}
+
+void Log::flush()
+{
+  pthread_mutex_lock(&m_flush_mutex);
+  pthread_mutex_lock(&m_queue_mutex);
+  EntryQueue t;
+  t.swap(m_new);
+  pthread_cond_broadcast(&m_cond_loggers);
+  pthread_mutex_unlock(&m_queue_mutex);
+  _flush(&t, &m_recent, false);
+
+  // trim
+  while (m_recent.m_len > m_max_recent) {
+    delete m_recent.dequeue();
+  }
+
+  pthread_mutex_unlock(&m_flush_mutex);
+}
+
+void Log::_flush(EntryQueue *t, EntryQueue *requeue, bool crash)
+{
+  Entry *e;
+  char buf[80];
+  while ((e = t->dequeue()) != NULL) {
+    unsigned sub = e->m_subsys;
+
+    bool should_log = crash || m_subs->get_log_level(sub) >= e->m_prio;
+    bool do_fd = m_fd >= 0 && should_log;
+    bool do_syslog = m_syslog_crash >= e->m_prio && should_log;
+    bool do_stderr = m_stderr_crash >= e->m_prio && should_log;
+
+    if (do_fd || do_syslog || do_stderr) {
+      int buflen = 0;
+
+      if (crash)
+	buflen += snprintf(buf, sizeof(buf), "%6d> ", -t->m_len);
+      buflen += e->m_stamp.sprintf(buf + buflen, sizeof(buf)-buflen);
+      buflen += snprintf(buf + buflen, sizeof(buf)-buflen, " %lx %2d ",
+			(unsigned long)e->m_thread, e->m_prio);
+
+      // FIXME: this is slow
+      string s = e->get_str();
+
+      if (do_fd) {
+	int r = safe_write(m_fd, buf, buflen);
+	if (r >= 0)
+	  r = safe_write(m_fd, s.data(), s.size());
+	if (r >= 0)
+	  r = write(m_fd, "\n", 1);
+	if (r < 0)
+	  cerr << "problem writing to " << m_log_file << ": " << cpp_strerror(r) << std::endl;
+      }
+
+      if (do_syslog) {
+	syslog(LOG_USER, "%s%s", buf, s.c_str());
+      }
+
+      if (do_stderr) {
+	cerr << buf << s << std::endl;
+      }
+    }
+
+    requeue->enqueue(e);
+  }
+}
+
+void Log::_log_message(const char *s, bool crash)
+{
+  if (m_fd >= 0) {
+    int r = safe_write(m_fd, s, strlen(s));
+    if (r >= 0)
+      r = safe_write(m_fd, "\n", 1);
+    if (r < 0)
+      cerr << "problem writing to " << m_log_file << ": " << cpp_strerror(r) << std::endl;
+  }
+  if ((crash ? m_syslog_crash : m_syslog_log) >= 0) {
+    syslog(LOG_USER, "%s", s);
+  }
+  
+  if ((crash ? m_stderr_crash : m_stderr_log) >= 0) {
+    cerr << s << std::endl;
+  }
+}
+
+void Log::dump_recent()
+{
+  pthread_mutex_lock(&m_flush_mutex);
+
+  pthread_mutex_lock(&m_queue_mutex);
+  EntryQueue t;
+  t.swap(m_new);
+  pthread_mutex_unlock(&m_queue_mutex);
+  _flush(&t, &m_recent, false);
+
+  EntryQueue old;
+  _log_message("--- begin dump of recent events ---", true);
+  _flush(&m_recent, &old, true);  
+
+  char buf[4096];
+  _log_message("--- logging levels ---", true);
+  for (vector::iterator p = m_subs->m_subsys.begin();
+       p != m_subs->m_subsys.end();
+       ++p) {
+    snprintf(buf, sizeof(buf), "  %2d/%2d %s", p->log_level, p->gather_level, p->name.c_str());
+    _log_message(buf, true);
+  }
+
+  sprintf(buf, "  %2d/%2d (syslog threshold)", m_syslog_log, m_syslog_crash);
+  _log_message(buf, true);
+  sprintf(buf, "  %2d/%2d (stderr threshold)", m_stderr_log, m_stderr_crash);
+  _log_message(buf, true);
+  sprintf(buf, "  max_recent %9d", m_max_recent);
+  _log_message(buf, true);
+  sprintf(buf, "  max_new    %9d", m_max_new);
+  _log_message(buf, true);
+  sprintf(buf, "  log_file %s", m_log_file.c_str());
+  _log_message(buf, true);
+
+  _log_message("--- end dump of recent events ---", true);
+
+  pthread_mutex_unlock(&m_flush_mutex);
+}
+
+void Log::start()
+{
+  assert(!is_started());
+  pthread_mutex_lock(&m_queue_mutex);
+  m_stop = false;
+  pthread_mutex_unlock(&m_queue_mutex);
+  create();
+}
+
+void Log::stop()
+{
+  assert(is_started());
+  pthread_mutex_lock(&m_queue_mutex);
+  m_stop = true;
+  pthread_cond_signal(&m_cond_flusher);
+  pthread_cond_broadcast(&m_cond_loggers);
+  pthread_mutex_unlock(&m_queue_mutex);
+  join();
+}
+
+void *Log::entry()
+{
+  pthread_mutex_lock(&m_queue_mutex);
+  while (!m_stop) {
+    if (!m_new.empty()) {
+      pthread_mutex_unlock(&m_queue_mutex);
+      flush();
+      pthread_mutex_lock(&m_queue_mutex);
+      continue;
+    }
+
+    pthread_cond_wait(&m_cond_flusher, &m_queue_mutex);
+  }
+  pthread_mutex_unlock(&m_queue_mutex);
+  flush();
+  return NULL;
+}
+
+} // ceph::log::
+} // ceph::
diff --git a/ceph/src/log/Log.h b/ceph/src/log/Log.h
new file mode 100644
index 00000000..b5e16fdd
--- /dev/null
+++ b/ceph/src/log/Log.h
@@ -0,0 +1,76 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef __CEPH_LOG_LOG_H
+#define __CEPH_LOG_LOG_H
+
+#include "common/Thread.h"
+
+#include 
+
+#include "Entry.h"
+#include "EntryQueue.h"
+#include "SubsystemMap.h"
+
+namespace ceph {
+namespace log {
+
+class Log : private Thread
+{
+  Log **m_indirect_this;
+
+  SubsystemMap *m_subs;
+  
+  pthread_mutex_t m_queue_mutex;
+  pthread_mutex_t m_flush_mutex;
+  pthread_cond_t m_cond_loggers;
+  pthread_cond_t m_cond_flusher;
+
+  EntryQueue m_new;    ///< new entries
+  EntryQueue m_recent; ///< recent (less new) entries we've already written at low detail
+
+  std::string m_log_file;
+  int m_fd;
+
+  int m_syslog_log, m_syslog_crash;
+  int m_stderr_log, m_stderr_crash;
+
+  bool m_stop;
+
+  int m_max_new, m_max_recent;
+
+  void *entry();
+
+  void _flush(EntryQueue *q, EntryQueue *requeue, bool crash);
+
+  void _log_message(const char *s, bool crash);
+
+public:
+  Log(SubsystemMap *s);
+  virtual ~Log();
+
+  void set_flush_on_exit();
+
+  void set_max_new(int n);
+  void set_max_recent(int n);
+  void set_log_file(std::string fn);
+  void reopen_log_file();
+
+  void flush(); 
+
+  void dump_recent();
+
+  void set_syslog_level(int log, int crash);
+  void set_stderr_level(int log, int crash);
+
+  Entry *create_entry(int level, int subsys);
+  void submit_entry(Entry *e);
+
+  void start();
+  void stop();
+};
+
+}
+}
+
+#endif
diff --git a/ceph/src/log/Makefile.am b/ceph/src/log/Makefile.am
new file mode 100644
index 00000000..b66e6cf8
--- /dev/null
+++ b/ceph/src/log/Makefile.am
@@ -0,0 +1,11 @@
+liblog_la_SOURCES = \
+	log/Log.cc \
+	log/SubsystemMap.cc
+noinst_LTLIBRARIES += liblog.la
+
+noinst_HEADERS += \
+	log/Entry.h \
+	log/EntryQueue.h \
+	log/Log.h \
+	log/SubsystemMap.h
+
diff --git a/ceph/src/log/SubsystemMap.cc b/ceph/src/log/SubsystemMap.cc
new file mode 100644
index 00000000..06173e6e
--- /dev/null
+++ b/ceph/src/log/SubsystemMap.cc
@@ -0,0 +1,31 @@
+
+#include "SubsystemMap.h"
+
+namespace ceph {
+namespace log {
+
+void SubsystemMap::add(unsigned subsys, std::string name, int log, int gather)
+{
+  if (subsys >= m_subsys.size())
+    m_subsys.resize(subsys + 1);
+  m_subsys[subsys].name = name;
+  m_subsys[subsys].log_level = log;
+  m_subsys[subsys].gather_level = gather;
+  if (name.length() > m_max_name_len)
+    m_max_name_len = name.length();
+}
+
+void SubsystemMap::set_log_level(unsigned subsys, int log)
+{
+  assert(subsys < m_subsys.size());
+  m_subsys[subsys].log_level = log;
+}
+
+void SubsystemMap::set_gather_level(unsigned subsys, int gather)
+{
+  assert(subsys < m_subsys.size());
+  m_subsys[subsys].gather_level = gather;
+}
+
+}
+}
diff --git a/ceph/src/log/SubsystemMap.h b/ceph/src/log/SubsystemMap.h
new file mode 100644
index 00000000..16b6b72c
--- /dev/null
+++ b/ceph/src/log/SubsystemMap.h
@@ -0,0 +1,71 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LOG_SUBSYSTEMS
+#define CEPH_LOG_SUBSYSTEMS
+
+#include 
+#include 
+
+#include "include/assert.h"
+
+namespace ceph {
+namespace log {
+
+struct Subsystem {
+  int log_level, gather_level;
+  std::string name;
+  
+  Subsystem() : log_level(0), gather_level(0) {}     
+};
+
+class SubsystemMap {
+  std::vector m_subsys;
+  unsigned m_max_name_len;
+
+  friend class Log;
+
+public:
+  SubsystemMap() : m_max_name_len(0) {}
+
+  int get_num() const {
+    return m_subsys.size();
+  }
+
+  int get_max_subsys_len() const {
+    return m_max_name_len;
+  }
+
+  void add(unsigned subsys, std::string name, int log, int gather);  
+  void set_log_level(unsigned subsys, int log);
+  void set_gather_level(unsigned subsys, int gather);
+
+  int get_log_level(unsigned subsys) const {
+    if (subsys >= m_subsys.size())
+      subsys = 0;
+    return m_subsys[subsys].log_level;
+  }
+
+  int get_gather_level(unsigned subsys) const {
+    if (subsys >= m_subsys.size())
+      subsys = 0;
+    return m_subsys[subsys].gather_level;
+  }
+
+  const std::string& get_name(unsigned subsys) const {
+    if (subsys >= m_subsys.size())
+      subsys = 0;
+    return m_subsys[subsys].name;
+  }
+
+  bool should_gather(unsigned sub, int level) {
+    assert(sub < m_subsys.size());
+    return level <= m_subsys[sub].gather_level ||
+      level <= m_subsys[sub].log_level;
+  }
+};
+
+}
+}
+
+#endif
diff --git a/ceph/src/log/test.cc b/ceph/src/log/test.cc
new file mode 100644
index 00000000..85e9ebc0
--- /dev/null
+++ b/ceph/src/log/test.cc
@@ -0,0 +1,189 @@
+#include 
+
+#include "log/Log.h"
+#include "common/Clock.h"
+#include "common/PrebufferedStreambuf.h"
+
+using namespace ceph::log;
+
+TEST(Log, Simple)
+{
+  SubsystemMap subs;
+  subs.add(0, "none", 10, 10);
+  subs.add(1, "foosys", 20, 1);
+  subs.add(2, "bar", 20, 2);
+  subs.add(3, "baz", 10, 3);
+
+  Log log(&subs);
+  log.start();
+ 
+  log.set_log_file("/tmp/foo");
+  log.reopen_log_file();
+
+  log.set_stderr_level(5, -1);
+
+
+  for (int i=0; i<100; i++) {
+    int sys = i % 4;
+    int l = 5 + (i%4);
+    if (subs.should_gather(sys, l)) {
+      Entry *e = new Entry(ceph_clock_now(NULL),
+			   pthread_self(),
+			   l,
+			   sys,
+			   "hello world");
+      log.submit_entry(e);
+    }
+  }
+  
+  log.flush();
+
+  log.dump_recent();
+
+  log.stop();
+}
+
+int many = 10000;
+
+TEST(Log, ManyNoGather)
+{
+  SubsystemMap subs;
+  subs.add(1, "foo", 1, 1);
+  Log log(&subs);
+  log.start();
+  log.set_log_file("/tmp/big");
+  log.reopen_log_file();
+  for (int i=0; iset_str(oss.str());
+      log.submit_entry(e);
+    }
+  }
+  log.flush();
+  log.stop();
+}
+TEST(Log, ManyGatherLogStringAssignWithReserve)
+{
+  SubsystemMap subs;
+  subs.add(1, "foo", 20, 10);
+  Log log(&subs);
+  log.start();
+  log.set_log_file("/tmp/big");
+  log.reopen_log_file();
+  for (int i=0; iset_str(oss.str());
+      log.submit_entry(e);
+    }
+  }
+  log.flush();
+  log.stop();
+}
+
+TEST(Log, ManyGatherLogPrebuf)
+{
+  SubsystemMap subs;
+  subs.add(1, "foo", 20, 10);
+  Log log(&subs);
+  log.start();
+  log.set_log_file("/tmp/big");
+  log.reopen_log_file();
+  for (int i=0; im_static_buf, sizeof(e->m_static_buf));
+      ostream oss(&psb);
+      oss << "this i a long stream asdf asdf asdf asdf asdf asdf asdf asdf asdf as fd";
+      //e->m_str = oss.str();
+      log.submit_entry(e);
+    }
+  }
+  log.flush();
+  log.stop();
+}
+
+TEST(Log, ManyGatherLogPrebufOverflow)
+{
+  SubsystemMap subs;
+  subs.add(1, "foo", 20, 10);
+  Log log(&subs);
+  log.start();
+  log.set_log_file("/tmp/big");
+  log.reopen_log_file();
+  for (int i=0; im_static_buf, 20);
+      ostream oss(&psb);
+      oss << "this i a long stream asdf asdf asdf asdf asdf asdf asdf asdf asdf as fd";
+      //e->m_str = oss.str();
+      log.submit_entry(e);
+    }
+  }
+  log.flush();
+  log.stop();
+}
+
+TEST(Log, ManyGather)
+{
+  SubsystemMap subs;
+  subs.add(1, "foo", 20, 1);
+  Log log(&subs);
+  log.start();
+  log.set_log_file("/tmp/big");
+  log.reopen_log_file();
+  for (int i=0; i /dev/null 2>&1 && [ -x `which invoke-rc.d` ]; then
+            invoke-rc.d ceph reload >/dev/null
+        elif which service > /dev/null 2>&1 && [ -x `which service` ]; then
+            service ceph reload >/dev/null
+        fi
+        # Possibly reload twice, but depending on ceph.conf the reload above may be a no-op
+        if which initctl > /dev/null 2>&1 && [ -x `which initctl` ]; then
+            for daemon in osd mon mds ; do
+              find -L /var/lib/ceph/$daemon/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+                | while read f; do
+                    if [ -e "/var/lib/ceph/$daemon/$f/done" -o -e "/var/lib/ceph/$daemon/$f/ready" ] && [ -e "/var/lib/ceph/$daemon/$f/upstart" ] && [ ! -e "/var/lib/ceph/$daemon/$f/sysvinit" ]; then
+                      cluster="${f%%-*}"
+                      id="${f#*-}"
+
+                      initctl reload ceph-$daemon cluster="$cluster" id="$id" 2>/dev/null || :
+                    fi
+                  done
+            done
+        fi
+    endscript
+    missingok
+    notifempty
+}
diff --git a/ceph/src/make_version b/ceph/src/make_version
new file mode 100755
index 00000000..3ed2bac6
--- /dev/null
+++ b/ceph/src/make_version
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+echo '$1: '$1
+
+if [ "$1" =  "-n" ] ; then
+   cur="no_version"
+   v="Development"
+else
+   cur=`head -1 $1`
+   v=`tail -1 $1 | cut -c 2-`
+fi
+
+print_all() {
+	echo "#ifndef CEPH_VERSION_H"
+	echo "#define CEPH_VERSION_H"
+	echo
+	echo "#define CEPH_GIT_VER $cur"
+	echo "#define CEPH_GIT_NICE_VER \"$v\""
+	echo
+	echo "#endif"
+}
+
+print_all > $2
diff --git a/ceph/src/mds/Anchor.cc b/ceph/src/mds/Anchor.cc
new file mode 100644
index 00000000..e24c5f1e
--- /dev/null
+++ b/ceph/src/mds/Anchor.cc
@@ -0,0 +1,64 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "mds/Anchor.h"
+
+#include "common/Formatter.h"
+
+void Anchor::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(ino, bl);
+  ::encode(dirino, bl);
+  ::encode(dn_hash, bl);
+  ::encode(nref, bl);
+  ::encode(updated, bl);
+  ENCODE_FINISH(bl);
+}
+
+void Anchor::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(ino, bl);
+  ::decode(dirino, bl);
+  ::decode(dn_hash, bl);
+  ::decode(nref, bl);
+  ::decode(updated, bl);
+  DECODE_FINISH(bl);
+}
+
+void Anchor::dump(Formatter *f) const
+{
+  f->dump_unsigned("ino", ino);
+  f->dump_unsigned("dirino", dirino);
+  f->dump_unsigned("dn_hash", dn_hash);
+  f->dump_unsigned("num_ref", nref);
+  f->dump_unsigned("updated", updated);
+}
+
+void Anchor::generate_test_instances(list& ls)
+{
+  ls.push_back(new Anchor);
+  ls.push_back(new Anchor);
+  ls.back()->ino = 1;
+  ls.back()->dirino = 2;
+  ls.back()->dn_hash = 3;
+  ls.back()->nref = 4;
+  ls.back()->updated = 5;
+}
+
+ostream& operator<<(ostream& out, const Anchor &a)
+{
+  return out << "a(" << a.ino << " " << a.dirino << "/" << a.dn_hash << " " << a.nref << " v" << a.updated << ")";
+}
diff --git a/ceph/src/mds/Anchor.h b/ceph/src/mds/Anchor.h
new file mode 100644
index 00000000..e8a6a645
--- /dev/null
+++ b/ceph/src/mds/Anchor.h
@@ -0,0 +1,55 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_ANCHOR_H
+#define CEPH_ANCHOR_H
+
+#include 
+using std::string;
+
+#include "include/types.h"
+#include "mdstypes.h"
+#include "include/buffer.h"
+
+
+// identifies a anchor table mutation
+
+namespace ceph {
+  class Formatter;
+}
+
+// anchor type
+
+class Anchor {
+public:
+  inodeno_t ino;      // anchored ino
+  inodeno_t dirino;
+  __u32     dn_hash;
+  int       nref;     // reference count
+  version_t updated;
+
+  Anchor() : dn_hash(0), nref(0), updated(0) {}
+  Anchor(inodeno_t i, inodeno_t di, __u32 hash, int nr, version_t u) :
+    ino(i), dirino(di), dn_hash(hash), nref(nr), updated(u) { }
+  
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(Anchor)
+
+ostream& operator<<(ostream& out, const Anchor &a);
+
+#endif
diff --git a/ceph/src/mds/AnchorClient.cc b/ceph/src/mds/AnchorClient.cc
new file mode 100644
index 00000000..30cbfd34
--- /dev/null
+++ b/ceph/src/mds/AnchorClient.cc
@@ -0,0 +1,129 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "AnchorClient.h"
+#include "MDSMap.h"
+#include "LogSegment.h"
+#include "MDS.h"
+#include "msg/Messenger.h"
+
+#include "messages/MMDSTableRequest.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".anchorclient "
+
+
+
+// LOOKUPS
+/* This function DOES NOT put the passed message before returning */
+void AnchorClient::handle_query_result(class MMDSTableRequest *m)
+{
+  dout(10) << "handle_anchor_reply " << *m << dendl;
+
+  inodeno_t ino;
+  vector trace;
+
+  bufferlist::iterator p = m->bl.begin();
+  ::decode(ino, p);
+  ::decode(trace, p);
+
+  if (!pending_lookup.count(ino))
+    return;
+
+  list<_pending_lookup> ls;
+  ls.swap(pending_lookup[ino]);
+  pending_lookup.erase(ino);
+
+  for (list<_pending_lookup>::iterator q = ls.begin(); q != ls.end(); ++q) {
+    *q->trace = trace;
+    if (q->onfinish) {
+      q->onfinish->complete(0);
+    }
+  }
+}
+
+void AnchorClient::resend_queries()
+{
+  // resend any pending lookups.
+  for (map >::iterator p = pending_lookup.begin();
+       p != pending_lookup.end();
+       ++p) {
+    dout(10) << "resending lookup on " << p->first << dendl;
+    _lookup(p->first);
+  }
+}
+
+void AnchorClient::lookup(inodeno_t ino, vector& trace, Context *onfinish)
+{
+  _pending_lookup l;
+  l.onfinish = onfinish;
+  l.trace = &trace;
+
+  bool isnew = (pending_lookup.count(ino) == 0);
+  pending_lookup[ino].push_back(l);
+  if (isnew)
+    _lookup(ino);
+}
+
+void AnchorClient::_lookup(inodeno_t ino)
+{
+  int ts = mds->mdsmap->get_tableserver();
+  if (mds->mdsmap->get_state(ts) < MDSMap::STATE_REJOIN)
+    return;
+  MMDSTableRequest *req = new MMDSTableRequest(table, TABLESERVER_OP_QUERY, 0, 0);
+  ::encode(ino, req->bl);
+  mds->send_message_mds(req, ts);
+}
+
+
+// FRIENDLY PREPARE
+
+void AnchorClient::prepare_create(inodeno_t ino, vector& trace, 
+				  version_t *patid, Context *onfinish)
+{
+  dout(10) << "prepare_create " << ino << " " << trace << dendl;
+  bufferlist bl;
+  __u32 op = TABLE_OP_CREATE;
+  ::encode(op, bl);
+  ::encode(ino, bl);
+  ::encode(trace, bl);
+  _prepare(bl, patid, 0, onfinish);
+}
+
+void AnchorClient::prepare_destroy(inodeno_t ino, 
+				   version_t *patid, Context *onfinish)
+{
+  dout(10) << "prepare_destroy " << ino << dendl;
+  bufferlist bl;
+  __u32 op = TABLE_OP_DESTROY;
+  ::encode(op, bl);
+  ::encode(ino, bl);
+  _prepare(bl, patid, 0, onfinish);
+}
+
+
+void AnchorClient::prepare_update(inodeno_t ino, vector& trace, 
+				  version_t *patid, Context *onfinish)
+{
+  dout(10) << "prepare_update " << ino << " " << trace << dendl;
+  bufferlist bl;
+  __u32 op = TABLE_OP_UPDATE;
+  ::encode(op, bl);
+  ::encode(ino, bl);
+  ::encode(trace, bl);
+  _prepare(bl, patid, 0, onfinish);
+}
diff --git a/ceph/src/mds/AnchorClient.h b/ceph/src/mds/AnchorClient.h
new file mode 100644
index 00000000..049cbcb0
--- /dev/null
+++ b/ceph/src/mds/AnchorClient.h
@@ -0,0 +1,46 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_ANCHORCLIENT_H
+#define CEPH_ANCHORCLIENT_H
+
+#include "MDSTableClient.h"
+#include "Anchor.h"
+
+class Context;
+class MDS;
+class LogSegment;
+
+class AnchorClient : public MDSTableClient {
+  // lookups
+  struct _pending_lookup {
+    vector *trace;
+    Context *onfinish;
+  };
+  map > pending_lookup;
+
+public:
+  AnchorClient(MDS *m) : MDSTableClient(m, TABLE_ANCHOR) {}
+  
+  void handle_query_result(MMDSTableRequest *m);
+  void lookup(inodeno_t ino, vector& trace, Context *onfinish);
+  void _lookup(inodeno_t ino);
+  void resend_queries();
+
+  void prepare_create(inodeno_t ino, vector& trace, version_t *atid, Context *onfinish);
+  void prepare_destroy(inodeno_t ino, version_t *atid, Context *onfinish);
+  void prepare_update(inodeno_t ino, vector& trace, version_t *atid, Context *onfinish);
+};
+
+#endif
diff --git a/ceph/src/mds/AnchorServer.cc b/ceph/src/mds/AnchorServer.cc
new file mode 100644
index 00000000..6f37e537
--- /dev/null
+++ b/ceph/src/mds/AnchorServer.cc
@@ -0,0 +1,351 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "AnchorServer.h"
+#include "MDS.h"
+#include "msg/Messenger.h"
+#include "messages/MMDSTableRequest.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".anchorserver "
+
+// table
+
+void AnchorServer::reset_state()
+{
+  anchor_map.clear();
+  pending_create.clear();
+  pending_destroy.clear();
+  pending_update.clear();
+  pending_for_mds.clear();
+}
+
+void AnchorServer::dump()
+{
+  dout(7) << "dump v " << version << dendl;
+  for (map::iterator it = anchor_map.begin();
+       it != anchor_map.end();
+       ++it)
+    dout(15) << "dump " << it->second << dendl;
+}
+
+void AnchorServer::dump(Formatter *f) const
+{
+  f->open_array_section("anchor map");
+  for (map::const_iterator i = anchor_map.begin();
+       i != anchor_map.end(); ++i) {
+    f->open_object_section("entry");
+    f->dump_int("ino", i->first);
+    f->open_object_section("Anchor");
+    i->second.dump(f);
+    f->close_section(); // Anchor
+    f->close_section(); // entry
+  }
+  f->close_section(); // anchor map
+}
+
+void AnchorServer::generate_test_instances(list& ls)
+{
+  AnchorServer *sample = new AnchorServer();
+  sample->pending_create[0] = 0;
+  sample->pending_destroy[0] = 1;
+  sample->anchor_map[0] = Anchor();
+  ls.push_back(sample);
+}
+
+
+
+/*
+ * basic updates
+ * Returns true if it changed the anchor_map contents.
+ */
+
+bool AnchorServer::add(inodeno_t ino, inodeno_t dirino, __u32 dn_hash,
+                       bool replace)
+{
+  //dout(17) << "add " << ino << " dirfrag " << dirfrag << dendl;
+  
+  // parent should be there
+  assert(MDS_INO_IS_BASE(dirino) ||     // base case,
+         anchor_map.count(dirino));   // or have it
+  
+  if (anchor_map.count(ino) == 0) {
+    // new item
+    anchor_map[ino] = Anchor(ino, dirino, dn_hash, 0, version);
+    dout(7) << "add added " << anchor_map[ino] << dendl;
+  } else if (replace) {
+    anchor_map[ino].dirino = dirino;
+    anchor_map[ino].dn_hash = dn_hash;
+    dout(7) << "add had old Anchor, updated it to "
+            << anchor_map[ino] << dendl;
+  } else {
+    dout(7) << "add had " << anchor_map[ino] << dendl;
+    return false;
+  }
+  return true;
+}
+
+void AnchorServer::inc(inodeno_t ino, int ref)
+{
+  dout(7) << "inc " << ino << dendl;
+
+  assert(anchor_map.count(ino));
+
+  while (1) {
+    Anchor &anchor = anchor_map[ino];
+    anchor.nref += ref;
+    anchor.updated = version;
+      
+    dout(10) << "inc now " << anchor << dendl;
+    ino = anchor.dirino;
+    
+    if (ino == 0 || MDS_INO_IS_BASE(ino)) break;
+    if (anchor_map.count(ino) == 0) break;
+  }
+}
+
+void AnchorServer::dec(inodeno_t ino, int ref)
+{
+  dout(7) << "dec " << ino << dendl;
+  assert(anchor_map.count(ino));
+
+  while (true) {
+    Anchor &anchor = anchor_map[ino];
+    anchor.nref -= ref;
+    anchor.updated = version;
+
+    if (anchor.nref == 0) {
+      dout(10) << "dec removing " << anchor << dendl;
+      inodeno_t dirino = anchor.dirino;
+      anchor_map.erase(ino);
+      ino = dirino;
+    } else {
+      dout(10) << "dec now " << anchor << dendl;
+      ino = anchor.dirino;
+    }
+    
+    if (ino == 0) break;
+    if (anchor_map.count(ino) == 0) break;
+  }
+}
+
+
+// server
+
+void AnchorServer::_prepare(bufferlist &bl, uint64_t reqid, int bymds)
+{
+  bufferlist::iterator p = bl.begin();
+  __u32 what;
+  inodeno_t ino;
+  vector trace;
+  ::decode(what, p);
+  ::decode(ino, p);
+
+  switch (what) {
+  case TABLE_OP_CREATE:
+    ::decode(trace, p);
+    version++;
+
+    // make sure trace is in table
+    dout(25) << "trace.size=" << trace.size() << dendl;
+    for (unsigned i=0; i(version, NULL));
+    break;
+
+  case TABLE_OP_DESTROY:
+    version++;
+    pending_destroy[version] = ino;
+    pending_ops[ino].push_back(pair(version, NULL));
+    break;
+    
+  case TABLE_OP_UPDATE:
+    ::decode(trace, p);
+    version++;
+    pending_update[version].first = ino;
+    pending_update[version].second = trace;
+    pending_ops[ino].push_back(pair(version, NULL));
+    break;
+
+  default:
+    assert(0);
+  }
+  //dump();
+}
+
+bool AnchorServer::check_pending(version_t tid, MMDSTableRequest *req, list& finished)
+{
+  inodeno_t ino;
+  if (pending_create.count(tid))
+    ino = pending_create[tid];
+  else if (pending_destroy.count(tid))
+    ino = pending_destroy[tid];
+  else if (pending_update.count(tid))
+    ino = pending_update[tid].first;
+  else
+    assert(0);
+
+  assert(pending_ops.count(ino));
+  list< pair >& pending = pending_ops[ino];
+  list< pair >::iterator p = pending.begin();
+  if (p->first == tid) {
+    assert(p->second == NULL);
+  } else {
+    while (p != pending.end()) {
+      if (p->first == tid)
+	break;
+      ++p;
+    }
+    assert(p != pending.end());
+    assert(p->second == NULL);
+    // not the earliest pending operation, wait if it's a commit
+    if (req) {
+      p->second = new C_MDS_RetryMessage(mds, req);
+      return false;
+    }
+  }
+
+  pending.erase(p);
+  if (pending.empty()) {
+    pending_ops.erase(ino);
+  } else {
+    for (p = pending.begin(); p != pending.end() && p->second; ++p) {
+      finished.push_back(p->second);
+      p->second = NULL;
+    }
+  }
+  return true;
+}
+
+bool AnchorServer::_commit(version_t tid, MMDSTableRequest *req)
+{
+  list finished;
+  if (!check_pending(tid, req, finished))
+    return false;
+
+  if (pending_create.count(tid)) {
+    dout(7) << "commit " << tid << " create " << pending_create[tid] << dendl;
+    pending_create.erase(tid);
+  } 
+
+  else if (pending_destroy.count(tid)) {
+    inodeno_t ino = pending_destroy[tid];
+    dout(7) << "commit " << tid << " destroy " << ino << dendl;
+    
+    dec(ino);  // destroy
+    
+    pending_destroy.erase(tid);
+  }
+
+  else if (pending_update.count(tid)) {
+    inodeno_t ino = pending_update[tid].first;
+    vector &trace = pending_update[tid].second;
+    
+    if (anchor_map.count(ino)) {
+      dout(7) << "commit " << tid << " update " << ino << dendl;
+
+      int ref = anchor_map[ino].nref;
+      // remove old
+      dec(ino, ref);
+      
+      // add new
+      for (unsigned i=0; iqueue_waiters(finished);
+  return true;
+}
+
+void AnchorServer::_rollback(version_t tid) 
+{
+  list finished;
+  check_pending(tid, NULL, finished);
+
+  if (pending_create.count(tid)) {
+    inodeno_t ino = pending_create[tid];
+    dout(7) << "rollback " << tid << " create " << ino << dendl;
+    dec(ino);
+    pending_create.erase(tid);
+  } 
+
+  else if (pending_destroy.count(tid)) {
+    inodeno_t ino = pending_destroy[tid];
+    dout(7) << "rollback " << tid << " destroy " << ino << dendl;
+    pending_destroy.erase(tid);
+  }
+
+  else if (pending_update.count(tid)) {
+    inodeno_t ino = pending_update[tid].first;
+    dout(7) << "rollback " << tid << " update " << ino << dendl;
+    pending_update.erase(tid);
+  }
+  else
+    assert(0);
+
+  // bump version.
+  version++;
+  //dump();
+  mds->queue_waiters(finished);
+}
+
+/* This function DOES put the passed message before returning */
+void AnchorServer::handle_query(MMDSTableRequest *req)
+{
+  bufferlist::iterator p = req->bl.begin();
+  inodeno_t ino;
+  ::decode(ino, p);
+  dout(7) << "handle_lookup " << *req << " ino " << ino << dendl;
+
+  vector trace;
+  inodeno_t curino = ino;
+  while (true) {
+    assert(anchor_map.count(curino) == 1);
+    Anchor &anchor = anchor_map[curino];
+    
+    dout(10) << "handle_lookup  adding " << anchor << dendl;
+    trace.insert(trace.begin(), anchor);  // lame FIXME
+    
+    if (MDS_INO_IS_BASE(anchor.dirino))
+      break;
+    curino = anchor.dirino;
+  }
+
+  // reply
+  MMDSTableRequest *reply = new MMDSTableRequest(table, TABLESERVER_OP_QUERY_REPLY, req->reqid, version);
+  ::encode(ino, reply->bl);
+  ::encode(trace, reply->bl);
+  mds->send_message(reply, req->get_connection());
+
+  req->put();
+}
+
+
diff --git a/ceph/src/mds/AnchorServer.h b/ceph/src/mds/AnchorServer.h
new file mode 100644
index 00000000..d6f1d840
--- /dev/null
+++ b/ceph/src/mds/AnchorServer.h
@@ -0,0 +1,87 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_ANCHORSERVER_H
+#define CEPH_ANCHORSERVER_H
+
+#include "MDSTableServer.h"
+#include "Anchor.h"
+
+class AnchorServer : public MDSTableServer {
+ public:
+  AnchorServer(MDS *mds) :
+    MDSTableServer(mds, TABLE_ANCHOR) {}
+
+  // table bits
+  map  anchor_map;
+
+  // uncommitted operations
+  map pending_create;
+  map pending_destroy;
+  map > > pending_update;
+  map > > pending_ops;
+
+  void reset_state();
+  void encode_server_state(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(anchor_map, bl);
+    ::encode(pending_create, bl);
+    ::encode(pending_destroy, bl);
+    ::encode(pending_update, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode_server_state(bufferlist::iterator& p) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, p);
+    ::decode(anchor_map, p);
+    ::decode(pending_create, p);
+    ::decode(pending_destroy, p);
+    ::decode(pending_update, p);
+    DECODE_FINISH(p);
+
+    map sort;
+    sort.insert(pending_create.begin(), pending_create.end());
+    sort.insert(pending_destroy.begin(), pending_destroy.end());
+    for (map > >::iterator p = pending_update.begin();
+	 p != pending_update.end(); ++p)
+      sort[p->first] = p->second.first;
+    for (map::iterator p = sort.begin(); p != sort.end(); ++p)
+      pending_ops[p->second].push_back(pair(p->first, NULL));
+  }
+
+  bool add(inodeno_t ino, inodeno_t dirino, __u32 dn_hash, bool replace);
+  void inc(inodeno_t ino, int ref=1);
+  void dec(inodeno_t ino, int ref=1);
+  bool check_pending(version_t tid, MMDSTableRequest *req, list& finished);
+
+  void dump();
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  // for the dencoder
+  AnchorServer() : MDSTableServer(NULL, TABLE_ANCHOR) {}
+  void encode(bufferlist& bl) const {
+    encode_server_state(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    decode_server_state(bl);
+  }
+
+  // server bits
+  void _prepare(bufferlist &bl, uint64_t reqid, int bymds);
+  bool _commit(version_t tid, MMDSTableRequest *req=NULL);
+  void _rollback(version_t tid);
+  void handle_query(MMDSTableRequest *m);
+};
+
+
+#endif
diff --git a/ceph/src/mds/CDentry.cc b/ceph/src/mds/CDentry.cc
new file mode 100644
index 00000000..05766587
--- /dev/null
+++ b/ceph/src/mds/CDentry.cc
@@ -0,0 +1,580 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#include "CDentry.h"
+#include "CInode.h"
+#include "CDir.h"
+#include "Anchor.h"
+
+#include "MDS.h"
+#include "MDCache.h"
+#include "Locker.h"
+#include "LogSegment.h"
+
+#include "messages/MLock.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << dir->cache->mds->get_nodeid() << ".cache.den(" << dir->dirfrag() << " " << name << ") "
+
+
+ostream& CDentry::print_db_line_prefix(ostream& out) 
+{
+  return out << ceph_clock_now(g_ceph_context) << " mds." << dir->cache->mds->get_nodeid() << ".cache.den(" << dir->ino() << " " << name << ") ";
+}
+
+boost::pool<> CDentry::pool(sizeof(CDentry));
+
+LockType CDentry::lock_type(CEPH_LOCK_DN);
+LockType CDentry::versionlock_type(CEPH_LOCK_DVERSION);
+
+
+// CDentry
+
+ostream& operator<<(ostream& out, CDentry& dn)
+{
+  filepath path;
+  dn.make_path(path);
+  
+  out << "[dentry " << path;
+  
+  if (true || dn.first != 0 || dn.last != CEPH_NOSNAP) {
+    out << " [" << dn.first << ",";
+    if (dn.last == CEPH_NOSNAP) 
+      out << "head";
+    else
+      out << dn.last;
+    out << ']';
+  }
+
+  if (dn.is_auth()) {
+    out << " auth";
+    if (dn.is_replicated()) 
+      out << dn.get_replicas();
+  } else {
+    out << " rep@" << dn.authority();
+    out << "." << dn.get_replica_nonce();
+  }
+
+  if (dn.get_linkage()->is_null()) out << " NULL";
+  if (dn.get_linkage()->is_remote()) {
+    out << " REMOTE(";
+    switch (DTTOIF(dn.get_linkage()->get_remote_d_type())) {
+    case S_IFSOCK: out << "sock"; break;
+    case S_IFLNK: out << "lnk"; break;
+    case S_IFREG: out << "reg"; break;
+    case S_IFBLK: out << "blk"; break;
+    case S_IFDIR: out << "dir"; break;
+    case S_IFCHR: out << "chr"; break;
+    case S_IFIFO: out << "fifo"; break;
+    default: assert(0);
+    }
+    out << ")";
+  }
+
+  if (!dn.lock.is_sync_and_unlocked())
+    out << " " << dn.lock;
+  if (!dn.versionlock.is_sync_and_unlocked())
+    out << " " << dn.versionlock;
+
+  if (dn.get_projected_version() != dn.get_version())
+    out << " pv=" << dn.get_projected_version();
+  out << " v=" << dn.get_version();
+
+  if (dn.is_auth_pinned())
+    out << " ap=" << dn.get_num_auth_pins() << "+" << dn.get_num_nested_auth_pins();
+
+  out << " inode=" << dn.get_linkage()->get_inode();
+
+  if (dn.is_new()) out << " state=new";
+
+  if (dn.get_num_ref()) {
+    out << " |";
+    dn.print_pin_set(out);
+  }
+
+  out << " " << &dn;
+  out << "]";
+  return out;
+}
+
+
+bool operator<(const CDentry& l, const CDentry& r)
+{
+  if ((l.get_dir()->ino() < r.get_dir()->ino()) ||
+      (l.get_dir()->ino() == r.get_dir()->ino() &&
+       (l.get_name() < r.get_name() ||
+	(l.get_name() == r.get_name() && l.last < r.last))))
+    return true;
+  return false;
+}
+
+
+void CDentry::print(ostream& out)
+{
+  out << *this;
+}
+
+
+/*
+inodeno_t CDentry::get_ino()
+{
+  if (get_inode()) 
+    return get_inode()->ino();
+  return inodeno_t();
+}
+*/
+
+pair CDentry::authority()
+{
+  return dir->authority();
+}
+
+
+void CDentry::add_waiter(uint64_t tag, Context *c)
+{
+  // wait on the directory?
+  if (tag & (WAIT_UNFREEZE|WAIT_SINGLEAUTH)) {
+    dir->add_waiter(tag, c);
+    return;
+  }
+  MDSCacheObject::add_waiter(tag, c);
+}
+
+
+version_t CDentry::pre_dirty(version_t min)
+{
+  projected_version = dir->pre_dirty(min);
+  dout(10) << " pre_dirty " << *this << dendl;
+  return projected_version;
+}
+
+
+void CDentry::_mark_dirty(LogSegment *ls)
+{
+  // state+pin
+  if (!state_test(STATE_DIRTY)) {
+    state_set(STATE_DIRTY);
+    dir->inc_num_dirty();
+    get(PIN_DIRTY);
+    assert(ls);
+  }
+  if (ls) 
+    ls->dirty_dentries.push_back(&item_dirty);
+}
+
+void CDentry::mark_dirty(version_t pv, LogSegment *ls) 
+{
+  dout(10) << " mark_dirty " << *this << dendl;
+
+  // i now live in this new dir version
+  assert(pv <= projected_version);
+  version = pv;
+  _mark_dirty(ls);
+
+  // mark dir too
+  dir->mark_dirty(pv, ls);
+}
+
+
+void CDentry::mark_clean() 
+{
+  dout(10) << " mark_clean " << *this << dendl;
+  assert(is_dirty());
+
+  // not always true for recalc_auth_bits during resolve finish
+  //assert(dir->get_version() == 0 || version <= dir->get_version());  // hmm?
+
+  // state+pin
+  state_clear(STATE_DIRTY);
+  dir->dec_num_dirty();
+  put(PIN_DIRTY);
+  
+  item_dirty.remove_myself();
+
+  clear_new();
+}    
+
+void CDentry::mark_new() 
+{
+  dout(10) << " mark_new " << *this << dendl;
+  state_set(STATE_NEW);
+}
+
+void CDentry::make_path_string(string& s)
+{
+  if (dir) {
+    dir->inode->make_path_string(s);
+  } else {
+    s = "???";
+  }
+  s += "/";
+  s.append(name.data(), name.length());
+}
+
+void CDentry::make_path(filepath& fp)
+{
+  assert(dir);
+  if (dir->inode->is_base())
+    fp = filepath(dir->inode->ino());               // base case
+  else if (dir->inode->get_parent_dn())
+    dir->inode->get_parent_dn()->make_path(fp);  // recurse
+  else
+    fp = filepath(dir->inode->ino());               // relative but not base?  hrm!
+  fp.push_dentry(name);
+}
+
+/*
+void CDentry::make_path(string& s, inodeno_t tobase)
+{
+  assert(dir);
+  
+  if (dir->inode->is_root()) {
+    s += "/";  // make it an absolute path (no matter what) if we hit the root.
+  } 
+  else if (dir->inode->get_parent_dn() &&
+	   dir->inode->ino() != tobase) {
+    dir->inode->get_parent_dn()->make_path(s, tobase);
+    s += "/";
+  }
+  s += name;
+}
+*/
+
+/** make_anchor_trace
+ * construct an anchor trace for this dentry, as if it were linked to *in.
+ */
+void CDentry::make_anchor_trace(vector& trace, CInode *in)
+{
+  // start with parent dir inode
+  dir->inode->make_anchor_trace(trace);
+
+  // add this inode (in my dirfrag) to the end
+  trace.push_back(Anchor(in->ino(), dir->ino(), get_hash(), 0, 0));
+  dout(10) << "make_anchor_trace added " << trace.back() << dendl;
+}
+
+
+/*
+ * we only add ourselves to remote_parents when the linkage is
+ * active (no longer projected).  if the passed dnl is projected,
+ * don't link in, and do that work later in pop_projected_linkage().
+ */
+void CDentry::link_remote(CDentry::linkage_t *dnl, CInode *in)
+{
+  assert(dnl->is_remote());
+  assert(in->ino() == dnl->get_remote_ino());
+  dnl->inode = in;
+
+  if (dnl == &linkage)
+    in->add_remote_parent(this);
+}
+
+void CDentry::unlink_remote(CDentry::linkage_t *dnl)
+{
+  assert(dnl->is_remote());
+  assert(dnl->inode);
+  
+  if (dnl == &linkage)
+    dnl->inode->remove_remote_parent(this);
+
+  dnl->inode = 0;
+}
+
+void CDentry::push_projected_linkage(CInode *inode)
+{
+  // dirty rstat tracking is in the projected plane
+  bool dirty_rstat = inode->is_dirty_rstat();
+  if (dirty_rstat)
+    inode->clear_dirty_rstat();
+
+  _project_linkage()->inode = inode;
+  inode->push_projected_parent(this);
+
+  if (dirty_rstat)
+    inode->mark_dirty_rstat();
+}
+
+CDentry::linkage_t *CDentry::pop_projected_linkage()
+{
+  assert(projected.size());
+  
+  linkage_t& n = projected.front();
+
+  /*
+   * the idea here is that the link_remote_inode(), link_primary_inode(), 
+   * etc. calls should make linkage identical to &n (and we assert as
+   * much).
+   */
+
+  if (n.remote_ino) {
+    dir->link_remote_inode(this, n.remote_ino, n.remote_d_type);
+    if (n.inode) {
+      linkage.inode = n.inode;
+      linkage.inode->add_remote_parent(this);
+    }
+  } else if (n.inode) {
+    dir->link_primary_inode(this, n.inode);
+    n.inode->pop_projected_parent();
+  }
+
+  assert(n.inode == linkage.inode);
+  assert(n.remote_ino == linkage.remote_ino);
+  assert(n.remote_d_type == linkage.remote_d_type);
+
+  projected.pop_front();
+
+  return &linkage;
+}
+
+
+
+// ----------------------------
+// auth pins
+
+int CDentry::get_num_dir_auth_pins()
+{
+  assert(!is_projected());
+  if (get_linkage()->is_primary())
+    return auth_pins + get_linkage()->get_inode()->get_num_auth_pins();
+  return auth_pins;
+}
+
+bool CDentry::can_auth_pin()
+{
+  assert(dir);
+  return dir->can_auth_pin();
+}
+
+void CDentry::auth_pin(void *by)
+{
+  if (auth_pins == 0)
+    get(PIN_AUTHPIN);
+  auth_pins++;
+
+#ifdef MDS_AUTHPIN_SET
+  auth_pin_set.insert(by);
+#endif
+
+  dout(10) << "auth_pin by " << by << " on " << *this 
+	   << " now " << auth_pins << "+" << nested_auth_pins
+	   << dendl;
+
+  dir->adjust_nested_auth_pins(1, 1, by);
+}
+
+void CDentry::auth_unpin(void *by)
+{
+  auth_pins--;
+
+#ifdef MDS_AUTHPIN_SET
+  assert(auth_pin_set.count(by));
+  auth_pin_set.erase(auth_pin_set.find(by));
+#endif
+
+  if (auth_pins == 0)
+    put(PIN_AUTHPIN);
+
+  dout(10) << "auth_unpin by " << by << " on " << *this
+	   << " now " << auth_pins << "+" << nested_auth_pins
+	   << dendl;
+  assert(auth_pins >= 0);
+
+  dir->adjust_nested_auth_pins(-1, -1, by);
+}
+
+void CDentry::adjust_nested_auth_pins(int adjustment, int diradj, void *by)
+{
+  nested_auth_pins += adjustment;
+
+  dout(35) << "adjust_nested_auth_pins by " << by 
+	   << ", change " << adjustment << " yields "
+	   << auth_pins << "+" << nested_auth_pins
+	   << dendl;
+  assert(nested_auth_pins >= 0);
+
+  dir->adjust_nested_auth_pins(adjustment, diradj, by);
+}
+
+bool CDentry::is_frozen()
+{
+  return dir->is_frozen();
+}
+
+bool CDentry::is_freezing()
+{
+  return dir->is_freezing();
+}
+
+
+void CDentry::adjust_nested_anchors(int by)
+{
+  nested_anchors += by;
+  dout(20) << "adjust_nested_anchors by " << by << " -> " << nested_anchors << dendl;
+  assert(nested_anchors >= 0);
+  dir->adjust_nested_anchors(by);
+}
+
+
+void CDentry::decode_replica(bufferlist::iterator& p, bool is_new)
+{
+  __u32 nonce;
+  ::decode(nonce, p);
+  replica_nonce = nonce;
+  
+  ::decode(first, p);
+
+  inodeno_t rino;
+  unsigned char rdtype;
+  __s32 ls;
+  ::decode(rino, p);
+  ::decode(rdtype, p);
+  ::decode(ls, p);
+
+  if (is_new) {
+    if (rino)
+      dir->link_remote_inode(this, rino, rdtype);
+    lock.set_state(ls);
+  }
+}
+
+// ----------------------------
+// locking
+
+void CDentry::set_object_info(MDSCacheObjectInfo &info) 
+{
+  info.dirfrag = dir->dirfrag();
+  info.dname = name;
+  info.snapid = last;
+}
+
+void CDentry::encode_lock_state(int type, bufferlist& bl)
+{
+  ::encode(first, bl);
+
+  // null, ino, or remote_ino?
+  char c;
+  if (linkage.is_primary()) {
+    c = 1;
+    ::encode(c, bl);
+    ::encode(linkage.get_inode()->inode.ino, bl);
+  }
+  else if (linkage.is_remote()) {
+    c = 2;
+    ::encode(c, bl);
+    ::encode(linkage.get_remote_ino(), bl);
+  }
+  else if (linkage.is_null()) {
+    // encode nothing.
+  }
+  else assert(0);  
+}
+
+void CDentry::decode_lock_state(int type, bufferlist& bl)
+{  
+  bufferlist::iterator p = bl.begin();
+
+  snapid_t newfirst;
+  ::decode(newfirst, p);
+
+  if (!is_auth() && newfirst != first) {
+    dout(10) << "decode_lock_state first " << first << " -> " << newfirst << dendl;
+    assert(newfirst > first);
+    first = newfirst;
+  }
+
+  if (p.end()) {
+    // null
+    assert(linkage.is_null());
+    return;
+  }
+
+  char c;
+  inodeno_t ino;
+  ::decode(c, p);
+
+  switch (c) {
+  case 1:
+  case 2:
+    ::decode(ino, p);
+    // newly linked?
+    if (linkage.is_null() && !is_auth()) {
+      // force trim from cache!
+      dout(10) << "decode_lock_state replica dentry null -> non-null, must trim" << dendl;
+      //assert(get_num_ref() == 0);
+    } else {
+      // verify?
+      
+    }
+    break;
+  default: 
+    assert(0);
+  }
+}
+
+
+ClientLease *CDentry::add_client_lease(client_t c, Session *session) 
+{
+  ClientLease *l;
+  if (client_lease_map.count(c))
+    l = client_lease_map[c];
+  else {
+    dout(20) << "add_client_lease client." << c << " on " << lock << dendl;
+    if (client_lease_map.empty())
+      get(PIN_CLIENTLEASE);
+    l = client_lease_map[c] = new ClientLease(c, this);
+    l->seq = ++session->lease_seq;
+  
+    lock.get_client_lease();
+  }
+  
+  return l;
+}
+
+void CDentry::remove_client_lease(ClientLease *l, Locker *locker) 
+{
+  assert(l->parent == this);
+
+  bool gather = false;
+
+  dout(20) << "remove_client_lease client." << l->client << " on " << lock << dendl;
+  lock.put_client_lease();
+  if (lock.get_num_client_lease() == 0 && !lock.is_stable())
+    gather = true;
+
+  client_lease_map.erase(l->client);
+  l->item_lease.remove_myself();
+  l->item_session_lease.remove_myself();
+  delete l;
+
+  if (client_lease_map.empty())
+    put(PIN_CLIENTLEASE);
+
+  if (gather)
+    locker->eval_gather(&lock);
+}
+
+void CDentry::_put()
+{
+  if (get_num_ref() <= (int)is_dirty() + 1) {
+    CDentry::linkage_t *dnl = get_projected_linkage();
+    if (dnl->is_primary()) {
+      CInode *in = dnl->get_inode();
+      if (get_num_ref() == (int)is_dirty() + !!in->get_num_ref())
+	in->mdcache->maybe_eval_stray(in, true);
+    }
+  }
+}
diff --git a/ceph/src/mds/CDentry.h b/ceph/src/mds/CDentry.h
new file mode 100644
index 00000000..41ed83d6
--- /dev/null
+++ b/ceph/src/mds/CDentry.h
@@ -0,0 +1,408 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_CDENTRY_H
+#define CEPH_CDENTRY_H
+
+#include 
+#include 
+using namespace std;
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "include/lru.h"
+#include "include/elist.h"
+#include "include/filepath.h"
+#include "mdstypes.h"
+
+#include "SimpleLock.h"
+#include "LocalLock.h"
+
+class CInode;
+class CDir;
+struct MDRequest;
+
+class Message;
+class Anchor;
+
+class CDentry;
+class LogSegment;
+
+class Session;
+
+
+
+// define an ordering
+bool operator<(const CDentry& l, const CDentry& r);
+
+// dentry
+class CDentry : public MDSCacheObject, public LRUObject {
+  /*
+   * This class uses a boost::pool to handle allocation. This is *not*
+   * thread-safe, so don't do allocations from multiple threads!
+   *
+   * Alternatively, switch the pool to use a boost::singleton_pool.
+   */
+
+private:
+  static boost::pool<> pool;
+public:
+  static void *operator new(size_t num_bytes) { 
+    void *n = pool.malloc();
+    if (!n)
+      throw std::bad_alloc();
+    return n;
+  }
+  void operator delete(void *p) {
+    pool.free(p);
+  }
+
+public:
+  // -- state --
+  static const int STATE_NEW =          (1<<0);
+  static const int STATE_FRAGMENTING =  (1<<1);
+  static const int STATE_PURGING =      (1<<2);
+  static const int STATE_BADREMOTEINO = (1<<3);
+  // stray dentry needs notification of releasing reference
+  static const int STATE_STRAY =	STATE_NOTIFYREF;
+
+  // -- pins --
+  static const int PIN_INODEPIN =     1;  // linked inode is pinned
+  static const int PIN_FRAGMENTING = -2;  // containing dir is refragmenting
+  static const int PIN_PURGING =      3;
+  const char *pin_name(int p) {
+    switch (p) {
+    case PIN_INODEPIN: return "inodepin";
+    case PIN_FRAGMENTING: return "fragmenting";
+    case PIN_PURGING: return "purging";
+    default: return generic_pin_name(p);
+    }
+  };
+
+  // -- wait --
+  //static const int WAIT_LOCK_OFFSET = 8;
+
+  void add_waiter(uint64_t tag, Context *c);
+
+  static const unsigned EXPORT_NONCE = 1;
+
+  bool is_lt(const MDSCacheObject *r) const {
+    return *this < *static_cast(r);
+  }
+
+public:
+  string name;
+  __u32 hash;
+  snapid_t first, last;
+
+  dentry_key_t key() { 
+    return dentry_key_t(last, name.c_str()); 
+  }
+
+public:
+  struct linkage_t {
+    CInode *inode;
+    inodeno_t remote_ino;
+    unsigned char remote_d_type;
+    
+    linkage_t() : inode(0), remote_ino(0), remote_d_type(0) {}
+
+    // dentry type is primary || remote || null
+    // inode ptr is required for primary, optional for remote, undefined for null
+    bool is_primary() { return remote_ino == 0 && inode != 0; }
+    bool is_remote() { return remote_ino > 0; }
+    bool is_null() { return remote_ino == 0 && inode == 0; }
+
+    CInode *get_inode() { return inode; }
+    inodeno_t get_remote_ino() { return remote_ino; }
+    unsigned char get_remote_d_type() { return remote_d_type; }
+
+    void set_remote(inodeno_t ino, unsigned char d_type) { 
+      remote_ino = ino;
+      remote_d_type = d_type;
+      inode = 0;
+    }
+    void link_remote(CInode *in);
+  };
+  
+protected:
+  CDir *dir;     // containing dirfrag
+  linkage_t linkage;
+  list projected;
+  
+  version_t version;  // dir version when last touched.
+  version_t projected_version;  // what it will be when i unlock/commit.
+
+public:
+  elist::item item_dirty;
+  elist::item item_stray;
+
+protected:
+  int auth_pins, nested_auth_pins;
+#ifdef MDS_AUTHPIN_SET
+  multiset auth_pin_set;
+#endif
+  int nested_anchors;
+
+  friend class Migrator;
+  friend class Locker;
+  friend class MDCache;
+  friend class CInode;
+  friend class C_MDC_XlockRequest;
+
+
+public:
+  // lock
+  static LockType lock_type;
+  static LockType versionlock_type;
+
+  SimpleLock lock;
+  LocalLock versionlock;
+
+ public:
+  // cons
+  CDentry(const string& n, __u32 h,
+	  snapid_t f, snapid_t l) :
+    name(n), hash(h),
+    first(f), last(l),
+    dir(0),
+    version(0), projected_version(0),
+    item_dirty(this),
+    auth_pins(0), nested_auth_pins(0), nested_anchors(0),
+    lock(this, &lock_type),
+    versionlock(this, &versionlock_type) {
+    g_num_dn++;
+    g_num_dna++;
+  }
+  CDentry(const string& n, __u32 h, inodeno_t ino, unsigned char dt,
+	  snapid_t f, snapid_t l) :
+    name(n), hash(h),
+    first(f), last(l),
+    dir(0),
+    version(0), projected_version(0),
+    item_dirty(this),
+    auth_pins(0), nested_auth_pins(0), nested_anchors(0),
+    lock(this, &lock_type),
+    versionlock(this, &versionlock_type) {
+    g_num_dn++;
+    g_num_dna++;
+    linkage.remote_ino = ino;
+    linkage.remote_d_type = dt;
+  }
+  ~CDentry() {
+    g_num_dn--;
+    g_num_dns++;
+  }
+
+
+  CDir *get_dir() const { return dir; }
+  const string& get_name() const { return name; }
+
+  __u32 get_hash() const { return hash; }
+
+  // linkage
+  linkage_t *get_linkage() { return &linkage; }
+
+  linkage_t *_project_linkage() {
+    projected.push_back(linkage_t());
+    return &projected.back();
+  }
+  void push_projected_linkage() {
+    _project_linkage();
+  }
+  void push_projected_linkage(inodeno_t ino, char d_type) {
+    linkage_t *p = _project_linkage();
+    p->remote_ino = ino;
+    p->remote_d_type = d_type;
+  }
+  void push_projected_linkage(CInode *inode); 
+  linkage_t *pop_projected_linkage();
+
+  bool is_projected() { return projected.size(); }
+
+  linkage_t *get_projected_linkage() {
+    if (!projected.empty())
+      return &projected.back();
+    return &linkage;
+  }
+  CInode *get_projected_inode() {
+    return get_projected_linkage()->inode;
+  }
+
+  bool use_projected(client_t client, const MutationRef& mut) const {
+    return lock.can_read_projected(client) || 
+      lock.get_xlock_by() == mut;
+  }
+  linkage_t *get_linkage(client_t client, const MutationRef& mut) {
+    return use_projected(client, mut) ? get_projected_linkage() : get_linkage();
+  }
+
+  // ref counts: pin ourselves in the LRU when we're pinned.
+  void first_get() {
+    lru_pin();
+  }
+  void last_put() {
+    lru_unpin();
+  }
+  void _put();
+
+  // auth pins
+  bool can_auth_pin();
+  void auth_pin(void *by);
+  void auth_unpin(void *by);
+  void adjust_nested_auth_pins(int adjustment, int diradj, void *by);
+  bool is_frozen();
+  bool is_freezing();
+  bool is_auth_pinned() { return auth_pins || nested_auth_pins; }
+  int get_num_auth_pins() { return auth_pins; }
+  int get_num_dir_auth_pins();
+  int get_num_nested_auth_pins() { return nested_auth_pins; }
+  
+  void adjust_nested_anchors(int by);
+
+  // remote links
+  void link_remote(linkage_t *dnl, CInode *in);
+  void unlink_remote(linkage_t *dnl);
+  
+  // copy cons
+  CDentry(const CDentry& m);
+  const CDentry& operator= (const CDentry& right);
+
+  // misc
+  void make_path_string(string& s);
+  void make_path(filepath& fp);
+  void make_anchor_trace(vector& trace, CInode *in);
+
+  // -- version --
+  version_t get_version() { return version; }
+  void set_version(version_t v) { projected_version = version = v; }
+  version_t get_projected_version() { return projected_version; }
+  void set_projected_version(version_t v) { projected_version = v; }
+  
+  pair authority();
+
+  version_t pre_dirty(version_t min=0);
+  void _mark_dirty(LogSegment *ls);
+  void mark_dirty(version_t projected_dirv, LogSegment *ls);
+  void mark_clean();
+
+  void mark_new();
+  bool is_new() { return state_test(STATE_NEW); }
+  void clear_new() { state_clear(STATE_NEW); }
+  
+  // -- replication
+  void encode_replica(int mds, bufferlist& bl) {
+    if (!is_replicated())
+      lock.replicate_relax();
+
+    __u32 nonce = add_replica(mds);
+    ::encode(nonce, bl);
+    ::encode(first, bl);
+    ::encode(linkage.remote_ino, bl);
+    ::encode(linkage.remote_d_type, bl);
+    __s32 ls = lock.get_replica_state();
+    ::encode(ls, bl);
+  }
+  void decode_replica(bufferlist::iterator& p, bool is_new);
+
+  // -- exporting
+  // note: this assumes the dentry already exists.  
+  // i.e., the name is already extracted... so we just need the other state.
+  void encode_export(bufferlist& bl) {
+    ::encode(first, bl);
+    ::encode(state, bl);
+    ::encode(version, bl);
+    ::encode(projected_version, bl);
+    ::encode(lock, bl);
+    ::encode(replica_map, bl);
+    get(PIN_TEMPEXPORTING);
+  }
+  void finish_export() {
+    // twiddle
+    clear_replica_map();
+    replica_nonce = EXPORT_NONCE;
+    state_clear(CDentry::STATE_AUTH);
+    if (is_dirty())
+      mark_clean();
+    put(PIN_TEMPEXPORTING);
+  }
+  void abort_export() {
+    put(PIN_TEMPEXPORTING);
+  }
+  void decode_import(bufferlist::iterator& blp, LogSegment *ls) {
+    ::decode(first, blp);
+    __u32 nstate;
+    ::decode(nstate, blp);
+    ::decode(version, blp);
+    ::decode(projected_version, blp);
+    ::decode(lock, blp);
+    ::decode(replica_map, blp);
+
+    // twiddle
+    state = 0;
+    state_set(CDentry::STATE_AUTH);
+    if (nstate & STATE_DIRTY)
+      _mark_dirty(ls);
+    if (!replica_map.empty())
+      get(PIN_REPLICATED);
+  }
+
+  // -- locking --
+  SimpleLock* get_lock(int type) {
+    assert(type == CEPH_LOCK_DN);
+    return &lock;
+  }
+  void set_object_info(MDSCacheObjectInfo &info);
+  void encode_lock_state(int type, bufferlist& bl);
+  void decode_lock_state(int type, bufferlist& bl);
+
+
+  // ---------------------------------------------
+  // replicas (on clients)
+ public:
+  map client_lease_map;
+
+  bool is_any_leases() {
+    return !client_lease_map.empty();
+  }
+  ClientLease *get_client_lease(client_t c) {
+    if (client_lease_map.count(c))
+      return client_lease_map[c];
+    return 0;
+  }
+  bool have_client_lease(client_t c) {
+    ClientLease *l = get_client_lease(c);
+    if (l) 
+      return true;
+    else
+      return false;
+  }
+
+  ClientLease *add_client_lease(client_t c, Session *session);
+  void remove_client_lease(ClientLease *r, class Locker *locker);  // returns remaining mask (if any), and kicks locker eval_gathers
+  
+
+  
+  ostream& print_db_line_prefix(ostream& out);
+  void print(ostream& out);
+
+  friend class CDir;
+};
+
+ostream& operator<<(ostream& out, CDentry& dn);
+
+
+#endif
diff --git a/ceph/src/mds/CDir.cc b/ceph/src/mds/CDir.cc
new file mode 100644
index 00000000..a423ef75
--- /dev/null
+++ b/ceph/src/mds/CDir.cc
@@ -0,0 +1,2659 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include "include/types.h"
+
+#include "CDir.h"
+#include "CDentry.h"
+#include "CInode.h"
+#include "Mutation.h"
+
+#include "MDSMap.h"
+#include "MDS.h"
+#include "MDCache.h"
+#include "Locker.h"
+#include "MDLog.h"
+#include "LogSegment.h"
+
+#include "common/bloom_filter.hpp"
+#include "include/Context.h"
+#include "common/Clock.h"
+
+#include "osdc/Objecter.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << cache->mds->get_nodeid() << ".cache.dir(" << this->dirfrag() << ") "
+
+
+
+// PINS
+//int cdir_pins[CDIR_NUM_PINS] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
+
+boost::pool<> CDir::pool(sizeof(CDir));
+
+
+ostream& operator<<(ostream& out, CDir& dir)
+{
+  string path;
+  dir.get_inode()->make_path_string_projected(path);
+  out << "[dir " << dir.dirfrag() << " " << path << "/"
+      << " [" << dir.first << ",head]";
+  if (dir.is_auth()) {
+    out << " auth";
+    if (dir.is_replicated())
+      out << dir.get_replicas();
+
+    if (dir.is_projected())
+      out << " pv=" << dir.get_projected_version();
+    out << " v=" << dir.get_version();
+    out << " cv=" << dir.get_committing_version();
+    out << "/" << dir.get_committed_version();
+  } else {
+    pair a = dir.authority();
+    out << " rep@" << a.first;
+    if (a.second != CDIR_AUTH_UNKNOWN)
+      out << "," << a.second;
+    out << "." << dir.get_replica_nonce();
+  }
+
+  if (dir.is_rep()) out << " REP";
+
+  if (dir.get_dir_auth() != CDIR_AUTH_DEFAULT) {
+    if (dir.get_dir_auth().second == CDIR_AUTH_UNKNOWN)
+      out << " dir_auth=" << dir.get_dir_auth().first;
+    else
+      out << " dir_auth=" << dir.get_dir_auth();
+  }
+  
+  if (dir.get_cum_auth_pins())
+    out << " ap=" << dir.get_auth_pins() 
+	<< "+" << dir.get_dir_auth_pins()
+	<< "+" << dir.get_nested_auth_pins();
+  if (dir.get_nested_anchors())
+    out << " na=" << dir.get_nested_anchors();
+
+  out << " state=" << dir.get_state();
+  if (dir.state_test(CDir::STATE_COMPLETE)) out << "|complete";
+  if (dir.state_test(CDir::STATE_FREEZINGTREE)) out << "|freezingtree";
+  if (dir.state_test(CDir::STATE_FROZENTREE)) out << "|frozentree";
+  //if (dir.state_test(CDir::STATE_FROZENTREELEAF)) out << "|frozentreeleaf";
+  if (dir.state_test(CDir::STATE_FROZENDIR)) out << "|frozendir";
+  if (dir.state_test(CDir::STATE_FREEZINGDIR)) out << "|freezingdir";
+  if (dir.state_test(CDir::STATE_EXPORTBOUND)) out << "|exportbound";
+  if (dir.state_test(CDir::STATE_IMPORTBOUND)) out << "|importbound";
+
+  // fragstat
+  out << " " << dir.fnode.fragstat;
+  if (!(dir.fnode.fragstat == dir.fnode.accounted_fragstat))
+    out << "/" << dir.fnode.accounted_fragstat;
+  if (g_conf->mds_debug_scatterstat && dir.is_projected()) {
+    fnode_t *pf = dir.get_projected_fnode();
+    out << "->" << pf->fragstat;
+    if (!(pf->fragstat == pf->accounted_fragstat))
+      out << "/" << pf->accounted_fragstat;
+  }
+  
+  // rstat
+  out << " " << dir.fnode.rstat;
+  if (!(dir.fnode.rstat == dir.fnode.accounted_rstat))
+    out << "/" << dir.fnode.accounted_rstat;
+  if (g_conf->mds_debug_scatterstat && dir.is_projected()) {
+    fnode_t *pf = dir.get_projected_fnode();
+    out << "->" << pf->rstat;
+    if (!(pf->rstat == pf->accounted_rstat))
+      out << "/" << pf->accounted_rstat;
+ }
+
+  out << " hs=" << dir.get_num_head_items() << "+" << dir.get_num_head_null();
+  out << ",ss=" << dir.get_num_snap_items() << "+" << dir.get_num_snap_null();
+  if (dir.get_num_dirty())
+    out << " dirty=" << dir.get_num_dirty();
+  
+  if (dir.get_num_ref()) {
+    out << " |";
+    dir.print_pin_set(out);
+  }
+
+  out << " " << &dir;
+  return out << "]";
+}
+
+
+void CDir::print(ostream& out) 
+{
+  out << *this;
+}
+
+
+
+
+ostream& CDir::print_db_line_prefix(ostream& out) 
+{
+  return out << ceph_clock_now(g_ceph_context) << " mds." << cache->mds->get_nodeid() << ".cache.dir(" << this->dirfrag() << ") ";
+}
+
+
+
+// -------------------------------------------------------------------
+// CDir
+
+CDir::CDir(CInode *in, frag_t fg, MDCache *mdcache, bool auth) :
+  dirty_rstat_inodes(member_offset(CInode, dirty_rstat_item)),
+  item_dirty(this), item_new(this),
+  pop_me(ceph_clock_now(g_ceph_context)),
+  pop_nested(ceph_clock_now(g_ceph_context)),
+  pop_auth_subtree(ceph_clock_now(g_ceph_context)),
+  pop_auth_subtree_nested(ceph_clock_now(g_ceph_context)),
+  bloom(NULL)
+{
+  g_num_dir++;
+  g_num_dira++;
+
+  inode = in;
+  frag = fg;
+  this->cache = mdcache;
+
+  first = 2;
+  
+  num_head_items = num_head_null = 0;
+  num_snap_items = num_snap_null = 0;
+  num_dirty = 0;
+
+  num_dentries_nested = 0;
+  num_dentries_auth_subtree = 0;
+  num_dentries_auth_subtree_nested = 0;
+
+  state = STATE_INITIAL;
+
+  memset(&fnode, 0, sizeof(fnode));
+  projected_version = 0;
+
+  committing_version = 0;
+  committed_version = 0;
+
+  // dir_auth
+  dir_auth = CDIR_AUTH_DEFAULT;
+
+  // auth
+  assert(in->is_dir());
+  if (auth) 
+    state |= STATE_AUTH;
+ 
+  auth_pins = 0;
+  nested_auth_pins = 0;
+  dir_auth_pins = 0;
+  request_pins = 0;
+
+  nested_anchors = 0;
+  
+  dir_rep = REP_NONE;
+}
+
+/**
+ * Check the recursive statistics on size for consistency.
+ * If mds_debug_scatterstat is enabled, assert for correctness,
+ * otherwise just print out the mismatch and continue.
+ */
+bool CDir::check_rstats()
+{
+  if (!g_conf->mds_debug_scatterstat)
+    return true;
+
+  dout(25) << "check_rstats on " << this << dendl;
+  if (!is_complete() || !is_auth() || is_frozen()) {
+    dout(10) << "check_rstats bailing out -- incomplete or non-auth or frozen dir!" << dendl;
+    return true;
+  }
+
+  // fragstat
+  if(!(get_num_head_items()==
+      (fnode.fragstat.nfiles + fnode.fragstat.nsubdirs))) {
+    dout(1) << "mismatch between head items and fnode.fragstat! printing dentries" << dendl;
+    dout(1) << "get_num_head_items() = " << get_num_head_items()
+             << "; fnode.fragstat.nfiles=" << fnode.fragstat.nfiles
+             << " fnode.fragstat.nsubdirs=" << fnode.fragstat.nsubdirs << dendl;
+    for (map_t::iterator i = items.begin(); i != items.end(); ++i) {
+      //if (i->second->get_linkage()->is_primary())
+        dout(1) << *(i->second) << dendl;
+    }
+    assert(get_num_head_items() == (fnode.fragstat.nfiles + fnode.fragstat.nsubdirs));
+  } else {
+    dout(20) << "get_num_head_items() = " << get_num_head_items()
+             << "; fnode.fragstat.nfiles=" << fnode.fragstat.nfiles
+             << " fnode.fragstat.nsubdirs=" << fnode.fragstat.nsubdirs << dendl;
+  }
+
+  // rstat
+  nest_info_t sub_info;
+  for (map_t::iterator i = items.begin(); i != items.end(); ++i) {
+    if (i->second->get_linkage()->is_primary() &&
+	i->second->last == CEPH_NOSNAP) {
+      sub_info.add(i->second->get_linkage()->inode->inode.accounted_rstat);
+    }
+  }
+
+  if ((!(sub_info.rbytes == fnode.rstat.rbytes)) ||
+      (!(sub_info.rfiles == fnode.rstat.rfiles)) ||
+      (!(sub_info.rsubdirs == fnode.rstat.rsubdirs))) {
+    dout(1) << "mismatch between child accounted_rstats and my rstats!" << dendl;
+    dout(1) << "total of child dentrys: " << sub_info << dendl;
+    dout(1) << "my rstats:              " << fnode.rstat << dendl;
+    for (map_t::iterator i = items.begin(); i != items.end(); ++i) {
+      if (i->second->get_linkage()->is_primary()) {
+        dout(1) << *(i->second) << " "
+                << i->second->get_linkage()->inode->inode.accounted_rstat
+                << dendl;
+      }
+    }
+  } else {
+    dout(25) << "total of child dentrys: " << sub_info << dendl;
+    dout(25) << "my rstats:              " << fnode.rstat << dendl;
+  }
+
+  assert(sub_info.rbytes == fnode.rstat.rbytes);
+  assert(sub_info.rfiles == fnode.rstat.rfiles);
+  assert(sub_info.rsubdirs == fnode.rstat.rsubdirs);
+  dout(10) << "check_rstats complete on " << this << dendl;
+  return true;
+}
+
+
+CDentry *CDir::lookup(const char *name, snapid_t snap)
+{ 
+  dout(20) << "lookup (" << snap << ", '" << name << "')" << dendl;
+  map_t::iterator iter = items.lower_bound(dentry_key_t(snap, name));
+  if (iter == items.end())
+    return 0;
+  if (iter->second->name == name &&
+      iter->second->first <= snap) {
+    dout(20) << "  hit -> " << iter->first << dendl;
+    return iter->second;
+  }
+  dout(20) << "  miss -> " << iter->first << dendl;
+  return 0;
+}
+
+
+
+
+
+/***
+ * linking fun
+ */
+
+CDentry* CDir::add_null_dentry(const string& dname,
+			       snapid_t first, snapid_t last)
+{
+  // foreign
+  assert(lookup_exact_snap(dname, last) == 0);
+   
+  // create dentry
+  CDentry* dn = new CDentry(dname, inode->hash_dentry_name(dname), first, last);
+  if (is_auth()) 
+    dn->state_set(CDentry::STATE_AUTH);
+  cache->lru.lru_insert_mid(dn);
+
+  dn->dir = this;
+  dn->version = get_projected_version();
+  
+  // add to dir
+  assert(items.count(dn->key()) == 0);
+  //assert(null_items.count(dn->name) == 0);
+
+  items[dn->key()] = dn;
+  if (last == CEPH_NOSNAP)
+    num_head_null++;
+  else
+    num_snap_null++;
+
+  if (state_test(CDir::STATE_DNPINNEDFRAG)) {
+    dn->get(CDentry::PIN_FRAGMENTING);
+    dn->state_set(CDentry::STATE_FRAGMENTING);
+  }    
+
+  dout(12) << "add_null_dentry " << *dn << dendl;
+
+  // pin?
+  if (get_num_any() == 1)
+    get(PIN_CHILD);
+  
+  assert(get_num_any() == items.size());
+  return dn;
+}
+
+
+CDentry* CDir::add_primary_dentry(const string& dname, CInode *in,
+				  snapid_t first, snapid_t last) 
+{
+  // primary
+  assert(lookup_exact_snap(dname, last) == 0);
+  
+  // create dentry
+  CDentry* dn = new CDentry(dname, inode->hash_dentry_name(dname), first, last);
+  if (is_auth()) 
+    dn->state_set(CDentry::STATE_AUTH);
+  cache->lru.lru_insert_mid(dn);
+
+  dn->dir = this;
+  dn->version = get_projected_version();
+  
+  // add to dir
+  assert(items.count(dn->key()) == 0);
+  //assert(null_items.count(dn->name) == 0);
+
+  items[dn->key()] = dn;
+
+  dn->get_linkage()->inode = in;
+  in->set_primary_parent(dn);
+
+  link_inode_work(dn, in);
+
+  if (dn->last == CEPH_NOSNAP)
+    num_head_items++;
+  else
+    num_snap_items++;
+  
+  if (state_test(CDir::STATE_DNPINNEDFRAG)) {
+    dn->get(CDentry::PIN_FRAGMENTING);
+    dn->state_set(CDentry::STATE_FRAGMENTING);
+  }    
+
+  dout(12) << "add_primary_dentry " << *dn << dendl;
+
+  // pin?
+  if (get_num_any() == 1)
+    get(PIN_CHILD);
+  assert(get_num_any() == items.size());
+  return dn;
+}
+
+CDentry* CDir::add_remote_dentry(const string& dname, inodeno_t ino, unsigned char d_type,
+				 snapid_t first, snapid_t last) 
+{
+  // foreign
+  assert(lookup_exact_snap(dname, last) == 0);
+
+  // create dentry
+  CDentry* dn = new CDentry(dname, inode->hash_dentry_name(dname), ino, d_type, first, last);
+  if (is_auth()) 
+    dn->state_set(CDentry::STATE_AUTH);
+  cache->lru.lru_insert_mid(dn);
+
+  dn->dir = this;
+  dn->version = get_projected_version();
+  
+  // add to dir
+  assert(items.count(dn->key()) == 0);
+  //assert(null_items.count(dn->name) == 0);
+
+  items[dn->key()] = dn;
+  if (last == CEPH_NOSNAP)
+    num_head_items++;
+  else
+    num_snap_items++;
+
+  if (state_test(CDir::STATE_DNPINNEDFRAG)) {
+    dn->get(CDentry::PIN_FRAGMENTING);
+    dn->state_set(CDentry::STATE_FRAGMENTING);
+  }    
+
+  dout(12) << "add_remote_dentry " << *dn << dendl;
+
+  // pin?
+  if (get_num_any() == 1)
+    get(PIN_CHILD);
+  
+  assert(get_num_any() == items.size());
+  return dn;
+}
+
+
+
+void CDir::remove_dentry(CDentry *dn) 
+{
+  dout(12) << "remove_dentry " << *dn << dendl;
+
+  // there should be no client leases at this point!
+  assert(dn->client_lease_map.empty());
+
+  if (state_test(CDir::STATE_DNPINNEDFRAG)) {
+    dn->put(CDentry::PIN_FRAGMENTING);
+    dn->state_clear(CDentry::STATE_FRAGMENTING);
+  }    
+
+  if (dn->get_linkage()->is_null()) {
+    if (dn->last == CEPH_NOSNAP)
+      num_head_null--;
+    else
+      num_snap_null--;
+  } else {
+    if (dn->last == CEPH_NOSNAP)
+      num_head_items--;
+    else
+      num_snap_items--;
+  }
+
+  if (!dn->get_linkage()->is_null())
+    // detach inode and dentry
+    unlink_inode_work(dn);
+  
+  // remove from list
+  assert(items.count(dn->key()) == 1);
+  items.erase(dn->key());
+
+  // clean?
+  if (dn->is_dirty())
+    dn->mark_clean();
+
+  cache->lru.lru_remove(dn);
+  delete dn;
+
+  // unpin?
+  if (get_num_any() == 0)
+    put(PIN_CHILD);
+  assert(get_num_any() == items.size());
+}
+
+void CDir::link_remote_inode(CDentry *dn, CInode *in)
+{
+  link_remote_inode(dn, in->ino(), IFTODT(in->get_projected_inode()->mode));
+}
+
+void CDir::link_remote_inode(CDentry *dn, inodeno_t ino, unsigned char d_type)
+{
+  dout(12) << "link_remote_inode " << *dn << " remote " << ino << dendl;
+  assert(dn->get_linkage()->is_null());
+
+  dn->get_linkage()->set_remote(ino, d_type);
+
+  if (dn->last == CEPH_NOSNAP) {
+    num_head_items++;
+    num_head_null--;
+  } else {
+    num_snap_items++;
+    num_snap_null--;
+  }
+  assert(get_num_any() == items.size());
+}
+
+void CDir::link_primary_inode(CDentry *dn, CInode *in)
+{
+  dout(12) << "link_primary_inode " << *dn << " " << *in << dendl;
+  assert(dn->get_linkage()->is_null());
+
+  dn->get_linkage()->inode = in;
+  in->set_primary_parent(dn);
+
+  link_inode_work(dn, in);
+  
+  if (dn->last == CEPH_NOSNAP) {
+    num_head_items++;
+    num_head_null--;
+  } else {
+    num_snap_items++;
+    num_snap_null--;
+  }
+
+  assert(get_num_any() == items.size());
+}
+
+void CDir::link_inode_work( CDentry *dn, CInode *in)
+{
+  assert(dn->get_linkage()->get_inode() == in);
+  assert(in->get_parent_dn() == dn);
+
+  // set inode version
+  //in->inode.version = dn->get_version();
+  
+  // pin dentry?
+  if (in->get_num_ref())
+    dn->get(CDentry::PIN_INODEPIN);
+  
+  // adjust auth pin count
+  if (in->auth_pins + in->nested_auth_pins)
+    dn->adjust_nested_auth_pins(in->auth_pins + in->nested_auth_pins, in->auth_pins, NULL);
+
+  if (in->inode.anchored + in->nested_anchors)
+    dn->adjust_nested_anchors(in->nested_anchors + in->inode.anchored);
+
+  // verify open snaprealm parent
+  if (in->snaprealm)
+    in->snaprealm->adjust_parent();
+}
+
+void CDir::unlink_inode(CDentry *dn)
+{
+  if (dn->get_linkage()->is_remote()) {
+    dout(12) << "unlink_inode " << *dn << dendl;
+  } else {
+    dout(12) << "unlink_inode " << *dn << " " << *dn->get_linkage()->get_inode() << dendl;
+  }
+
+  unlink_inode_work(dn);
+
+  if (dn->last == CEPH_NOSNAP) {
+    num_head_items--;
+    num_head_null++;
+  } else {
+    num_snap_items--;
+    num_snap_null++;
+  }
+  assert(get_num_any() == items.size());
+}
+
+
+void CDir::try_remove_unlinked_dn(CDentry *dn)
+{
+  assert(dn->dir == this);
+  assert(dn->get_linkage()->is_null());
+  
+  // no pins (besides dirty)?
+  if (dn->get_num_ref() != dn->is_dirty()) 
+    return;
+
+  // was the dn new?
+  if (dn->is_new()) {
+    dout(10) << "try_remove_unlinked_dn " << *dn << " in " << *this << dendl;
+    if (dn->is_dirty())
+      dn->mark_clean();
+    remove_dentry(dn);
+
+    // NOTE: we may not have any more dirty dentries, but the fnode
+    // still changed, so the directory must remain dirty.
+  }
+}
+
+
+void CDir::unlink_inode_work( CDentry *dn )
+{
+  CInode *in = dn->get_linkage()->get_inode();
+
+  if (dn->get_linkage()->is_remote()) {
+    // remote
+    if (in) 
+      dn->unlink_remote(dn->get_linkage());
+
+    dn->get_linkage()->set_remote(0, 0);
+  } else {
+    // primary
+    assert(dn->get_linkage()->is_primary());
+ 
+    // unpin dentry?
+    if (in->get_num_ref())
+      dn->put(CDentry::PIN_INODEPIN);
+    
+    // unlink auth_pin count
+    if (in->auth_pins + in->nested_auth_pins)
+      dn->adjust_nested_auth_pins(0 - (in->auth_pins + in->nested_auth_pins), 0 - in->auth_pins, NULL);
+    
+    if (in->inode.anchored + in->nested_anchors)
+      dn->adjust_nested_anchors(0 - (in->nested_anchors + in->inode.anchored));
+
+    // detach inode
+    in->remove_primary_parent(dn);
+    dn->get_linkage()->inode = 0;
+  }
+}
+
+void CDir::add_to_bloom(CDentry *dn)
+{
+  if (!bloom) {
+    /* not create bloom filter for incomplete dir that was added by log replay */
+    if (!is_complete())
+      return;
+    unsigned size = get_num_head_items() + get_num_snap_items();
+    if (size < 100) size = 100;
+    bloom = new bloom_filter(size, 1.0 / size, 0);
+  }
+  /* This size and false positive probability is completely random.*/
+  bloom->insert(dn->name.c_str(), dn->name.size());
+}
+
+bool CDir::is_in_bloom(const string& name)
+{
+  if (!bloom)
+    return false;
+  return bloom->contains(name.c_str(), name.size());
+}
+
+void CDir::remove_bloom()
+{
+  delete bloom;
+  bloom = NULL;
+}
+
+void CDir::remove_null_dentries() {
+  dout(12) << "remove_null_dentries " << *this << dendl;
+
+  CDir::map_t::iterator p = items.begin();
+  while (p != items.end()) {
+    CDentry *dn = p->second;
+    ++p;
+    if (dn->get_linkage()->is_null() && !dn->is_projected())
+      remove_dentry(dn);
+  }
+
+  assert(num_snap_null == 0);
+  assert(num_head_null == 0);
+  assert(get_num_any() == items.size());
+}
+
+void CDir::touch_dentries_bottom() {
+  dout(12) << "touch_dentries_bottom " << *this << dendl;
+
+  for (CDir::map_t::iterator p = items.begin();
+       p != items.end();
+       ++p)
+    inode->mdcache->touch_dentry_bottom(p->second);
+}
+
+bool CDir::try_trim_snap_dentry(CDentry *dn, const set& snaps)
+{
+  assert(dn->last != CEPH_NOSNAP);
+  set::const_iterator p = snaps.lower_bound(dn->first);
+  CDentry::linkage_t *dnl= dn->get_linkage();
+  CInode *in = 0;
+  if (dnl->is_primary())
+    in = dnl->get_inode();
+  if ((p == snaps.end() || *p > dn->last) &&
+      (dn->get_num_ref() == dn->is_dirty()) &&
+      (!in || in->get_num_ref() == in->is_dirty())) {
+    dout(10) << " purging snapped " << *dn << dendl;
+    if (in && in->is_dirty())
+      in->mark_clean();
+    remove_dentry(dn);
+    if (in) {
+      dout(10) << " purging snapped " << *in << dendl;
+      cache->remove_inode(in);
+    }
+    return true;
+  }
+  return false;
+}
+
+
+void CDir::purge_stale_snap_data(const set& snaps)
+{
+  dout(10) << "purge_stale_snap_data " << snaps << dendl;
+
+  CDir::map_t::iterator p = items.begin();
+  while (p != items.end()) {
+    CDentry *dn = p->second;
+    ++p;
+
+    if (dn->last == CEPH_NOSNAP)
+      continue;
+
+    try_trim_snap_dentry(dn, snaps);
+  }
+}
+
+
+/**
+ * steal_dentry -- semi-violently move a dentry from one CDir to another
+ * (*) violently, in that nitems, most pins, etc. are not correctly maintained 
+ * on the old CDir corpse; must call finish_old_fragment() when finished.
+ */
+void CDir::steal_dentry(CDentry *dn)
+{
+  dout(15) << "steal_dentry " << *dn << dendl;
+
+  items[dn->key()] = dn;
+
+  dn->dir->items.erase(dn->key());
+  if (dn->dir->items.empty())
+    dn->dir->put(PIN_CHILD);
+
+  if (get_num_any() == 0)
+    get(PIN_CHILD);
+  if (dn->get_linkage()->is_null()) {
+    if (dn->last == CEPH_NOSNAP)
+      num_head_null++;
+    else
+      num_snap_null++;
+  } else {
+    if (dn->last == CEPH_NOSNAP)
+      num_head_items++;
+    else
+      num_snap_items++;
+
+    if (dn->get_linkage()->is_primary()) {
+      CInode *in = dn->get_linkage()->get_inode();
+      inode_t *pi = in->get_projected_inode();
+      if (dn->get_linkage()->get_inode()->is_dir())
+	fnode.fragstat.nsubdirs++;
+      else
+	fnode.fragstat.nfiles++;
+      fnode.rstat.rbytes += pi->accounted_rstat.rbytes;
+      fnode.rstat.rfiles += pi->accounted_rstat.rfiles;
+      fnode.rstat.rsubdirs += pi->accounted_rstat.rsubdirs;
+      fnode.rstat.ranchors += pi->accounted_rstat.ranchors;
+      fnode.rstat.rsnaprealms += pi->accounted_rstat.ranchors;
+      if (pi->accounted_rstat.rctime > fnode.rstat.rctime)
+	fnode.rstat.rctime = pi->accounted_rstat.rctime;
+
+      // move dirty inode rstat to new dirfrag
+      if (in->is_dirty_rstat())
+	dirty_rstat_inodes.push_back(&in->dirty_rstat_item);
+    } else if (dn->get_linkage()->is_remote()) {
+      if (dn->get_linkage()->get_remote_d_type() == DT_DIR)
+	fnode.fragstat.nsubdirs++;
+      else
+	fnode.fragstat.nfiles++;
+    }
+  }
+
+  if (dn->auth_pins || dn->nested_auth_pins) {
+    // use the helpers here to maintain the auth_pin invariants on the dir inode
+    int ap = dn->get_num_auth_pins() + dn->get_num_nested_auth_pins();
+    int dap = dn->get_num_dir_auth_pins();
+    assert(dap <= ap);
+    adjust_nested_auth_pins(ap, dap, NULL);
+    dn->dir->adjust_nested_auth_pins(-ap, -dap, NULL);
+  }
+
+  nested_anchors += dn->nested_anchors;
+  if (dn->is_dirty()) 
+    num_dirty++;
+
+  dn->dir = this;
+}
+
+void CDir::prepare_old_fragment(bool replay)
+{
+  // auth_pin old fragment for duration so that any auth_pinning
+  // during the dentry migration doesn't trigger side effects
+  if (!replay && is_auth())
+    auth_pin(this);
+}
+
+void CDir::prepare_new_fragment(bool replay)
+{
+  if (!replay && is_auth()) {
+    _freeze_dir();
+    mark_complete();
+  }
+}
+
+void CDir::finish_old_fragment(list& waiters, bool replay)
+{
+  // take waiters _before_ unfreeze...
+  if (!replay) {
+    take_waiting(WAIT_ANY_MASK, waiters);
+    if (is_auth()) {
+      auth_unpin(this);  // pinned in prepare_old_fragment
+      assert(is_frozen_dir());
+      unfreeze_dir();
+    }
+  }
+
+  assert(nested_auth_pins == 0);
+  assert(dir_auth_pins == 0);
+  assert(auth_pins == 0);
+
+  num_head_items = num_head_null = 0;
+  num_snap_items = num_snap_null = 0;
+
+  // this mirrors init_fragment_pins()
+  if (is_auth()) 
+    clear_replica_map();
+  if (is_dirty())
+    mark_clean();
+  if (state_test(STATE_IMPORTBOUND))
+    put(PIN_IMPORTBOUND);
+  if (state_test(STATE_EXPORTBOUND))
+    put(PIN_EXPORTBOUND);
+  if (is_subtree_root())
+    put(PIN_SUBTREE);
+
+  if (auth_pins > 0)
+    put(PIN_AUTHPIN);
+
+  assert(get_num_ref() == (state_test(STATE_STICKY) ? 1:0));
+}
+
+void CDir::init_fragment_pins()
+{
+  if (!replica_map.empty())
+    get(PIN_REPLICATED);
+  if (state_test(STATE_DIRTY))
+    get(PIN_DIRTY);
+  if (state_test(STATE_EXPORTBOUND))
+    get(PIN_EXPORTBOUND);
+  if (state_test(STATE_IMPORTBOUND))
+    get(PIN_IMPORTBOUND);
+  if (is_subtree_root())
+    get(PIN_SUBTREE);
+}
+
+void CDir::split(int bits, list& subs, list& waiters, bool replay)
+{
+  dout(10) << "split by " << bits << " bits on " << *this << dendl;
+
+  if (cache->mds->logger) cache->mds->logger->inc(l_mds_dir_sp);
+
+  assert(replay || is_complete() || !is_auth());
+
+  list frags;
+  frag.split(bits, frags);
+
+  vector subfrags(1 << bits);
+  
+  double fac = 1.0 / (double)(1 << bits);  // for scaling load vecs
+
+  dout(15) << "           rstat " << fnode.rstat << dendl;
+  dout(15) << " accounted_rstat " << fnode.accounted_rstat << dendl;
+  nest_info_t rstatdiff;
+  rstatdiff.add_delta(fnode.accounted_rstat, fnode.rstat);
+  dout(15) << "           fragstat " << fnode.fragstat << dendl;
+  dout(15) << " accounted_fragstat " << fnode.accounted_fragstat << dendl;
+  frag_info_t fragstatdiff;
+  bool touched_mtime;
+  fragstatdiff.add_delta(fnode.accounted_fragstat, fnode.fragstat, touched_mtime);
+  dout(10) << " rstatdiff " << rstatdiff << " fragstatdiff " << fragstatdiff << dendl;
+
+  prepare_old_fragment(replay);
+
+  // create subfrag dirs
+  int n = 0;
+  for (list::iterator p = frags.begin(); p != frags.end(); ++p) {
+    CDir *f = new CDir(inode, *p, cache, is_auth());
+    f->state_set(state & (MASK_STATE_FRAGMENT_KEPT | STATE_COMPLETE));
+    f->replica_map = replica_map;
+    f->dir_auth = dir_auth;
+    f->init_fragment_pins();
+    f->set_version(get_version());
+
+    f->pop_me = pop_me;
+    f->pop_me.scale(fac);
+
+    // FIXME; this is an approximation
+    f->pop_nested = pop_nested;
+    f->pop_nested.scale(fac);
+    f->pop_auth_subtree = pop_auth_subtree;
+    f->pop_auth_subtree.scale(fac);
+    f->pop_auth_subtree_nested = pop_auth_subtree_nested;
+    f->pop_auth_subtree_nested.scale(fac);
+
+    dout(10) << " subfrag " << *p << " " << *f << dendl;
+    subfrags[n++] = f;
+    subs.push_back(f);
+    inode->add_dirfrag(f);
+
+    f->set_dir_auth(get_dir_auth());
+    f->prepare_new_fragment(replay);
+  }
+  
+  // repartition dentries
+  while (!items.empty()) {
+    CDir::map_t::iterator p = items.begin();
+    
+    CDentry *dn = p->second;
+    frag_t subfrag = inode->pick_dirfrag(dn->name);
+    int n = (subfrag.value() & (subfrag.mask() ^ frag.mask())) >> subfrag.mask_shift();
+    dout(15) << " subfrag " << subfrag << " n=" << n << " for " << p->first << dendl;
+    CDir *f = subfrags[n];
+    f->steal_dentry(dn);
+  }
+
+  // FIXME: handle dirty old rstat
+
+  // fix up new frag fragstats
+  for (int i=0; ifnode.rstat.version = fnode.rstat.version;
+    f->fnode.accounted_rstat = f->fnode.rstat;
+    f->fnode.fragstat.version = fnode.fragstat.version;
+    f->fnode.accounted_fragstat = f->fnode.fragstat;
+    dout(10) << " rstat " << f->fnode.rstat << " fragstat " << f->fnode.fragstat
+	     << " on " << *f << dendl;
+  }
+
+  // give any outstanding frag stat differential to first frag
+  dout(10) << " giving rstatdiff " << rstatdiff << " fragstatdiff" << fragstatdiff
+           << " to " << *subfrags[0] << dendl;
+  subfrags[0]->fnode.accounted_rstat.add(rstatdiff);
+  subfrags[0]->fnode.accounted_fragstat.add(fragstatdiff);
+
+  finish_old_fragment(waiters, replay);
+}
+
+void CDir::merge(list& subs, list& waiters, bool replay)
+{
+  dout(10) << "merge " << subs << dendl;
+
+  set_dir_auth(subs.front()->get_dir_auth());
+  prepare_new_fragment(replay);
+
+  nest_info_t rstatdiff;
+  frag_info_t fragstatdiff;
+  bool touched_mtime;
+  version_t rstat_version = inode->get_projected_inode()->rstat.version;
+  version_t dirstat_version = inode->get_projected_inode()->dirstat.version;
+
+  for (list::iterator p = subs.begin(); p != subs.end(); ++p) {
+    CDir *dir = *p;
+    dout(10) << " subfrag " << dir->get_frag() << " " << *dir << dendl;
+    assert(!dir->is_auth() || dir->is_complete() || replay);
+
+    if (dir->fnode.accounted_rstat.version == rstat_version)
+      rstatdiff.add_delta(dir->fnode.accounted_rstat, dir->fnode.rstat);
+    if (dir->fnode.accounted_fragstat.version == dirstat_version)
+      fragstatdiff.add_delta(dir->fnode.accounted_fragstat, dir->fnode.fragstat,
+			     touched_mtime);
+
+    dir->prepare_old_fragment(replay);
+
+    // steal dentries
+    while (!dir->items.empty()) 
+      steal_dentry(dir->items.begin()->second);
+    
+    // merge replica map
+    for (map::iterator p = dir->replicas_begin();
+	 p != dir->replica_map.end();
+	 ++p) {
+      unsigned cur = replica_map[p->first];
+      if (p->second > cur)
+	replica_map[p->first] = p->second;
+    }
+
+    // merge version
+    if (dir->get_version() > get_version())
+      set_version(dir->get_version());
+
+    // merge state
+    state_set(dir->get_state() & MASK_STATE_FRAGMENT_KEPT);
+    dir_auth = dir->dir_auth;
+
+    dir->finish_old_fragment(waiters, replay);
+    inode->close_dirfrag(dir->get_frag());
+  }
+
+  if (is_auth() && !replay)
+    mark_complete();
+
+  // FIXME: merge dirty old rstat
+  fnode.rstat.version = rstat_version;
+  fnode.accounted_rstat = fnode.rstat;
+  fnode.accounted_rstat.add(rstatdiff);
+
+  fnode.fragstat.version = dirstat_version;
+  fnode.accounted_fragstat = fnode.fragstat;
+  fnode.accounted_fragstat.add(fragstatdiff);
+
+  init_fragment_pins();
+}
+
+
+
+
+void CDir::resync_accounted_fragstat()
+{
+  fnode_t *pf = get_projected_fnode();
+  inode_t *pi = inode->get_projected_inode();
+
+  if (pf->accounted_fragstat.version != pi->dirstat.version) {
+    pf->fragstat.version = pi->dirstat.version;
+    dout(10) << "resync_accounted_fragstat " << pf->accounted_fragstat << " -> " << pf->fragstat << dendl;
+    pf->accounted_fragstat = pf->fragstat;
+  }
+}
+
+/*
+ * resync rstat and accounted_rstat with inode
+ */
+void CDir::resync_accounted_rstat()
+{
+  fnode_t *pf = get_projected_fnode();
+  inode_t *pi = inode->get_projected_inode();
+  
+  if (pf->accounted_rstat.version != pi->rstat.version) {
+    pf->rstat.version = pi->rstat.version;
+    dout(10) << "resync_accounted_rstat " << pf->accounted_rstat << " -> " << pf->rstat << dendl;
+    pf->accounted_rstat = pf->rstat;
+    dirty_old_rstat.clear();
+  }
+}
+
+void CDir::assimilate_dirty_rstat_inodes()
+{
+  dout(10) << "assimilate_dirty_rstat_inodes" << dendl;
+  for (elist::iterator p = dirty_rstat_inodes.begin_use_current();
+       !p.end(); ++p) {
+    CInode *in = *p;
+    assert(in->is_auth());
+    if (in->is_frozen())
+      continue;
+
+    inode_t *pi = in->project_inode();
+    pi->version = in->pre_dirty();
+
+    inode->mdcache->project_rstat_inode_to_frag(in, this, 0, 0);
+  }
+  state_set(STATE_ASSIMRSTAT);
+  dout(10) << "assimilate_dirty_rstat_inodes done" << dendl;
+}
+
+void CDir::assimilate_dirty_rstat_inodes_finish(MutationRef& mut, EMetaBlob *blob)
+{
+  if (!state_test(STATE_ASSIMRSTAT))
+    return;
+  state_clear(STATE_ASSIMRSTAT);
+  dout(10) << "assimilate_dirty_rstat_inodes_finish" << dendl;
+  elist::iterator p = dirty_rstat_inodes.begin_use_current();
+  while (!p.end()) {
+    CInode *in = *p;
+    ++p;
+
+    if (in->is_frozen())
+      continue;
+
+    CDentry *dn = in->get_projected_parent_dn();
+
+    mut->auth_pin(in);
+    mut->add_projected_inode(in);
+
+    in->clear_dirty_rstat();
+    blob->add_primary_dentry(dn, in, true);
+  }
+
+  if (!dirty_rstat_inodes.empty())
+    inode->mdcache->mds->locker->mark_updated_scatterlock(&inode->nestlock);
+}
+
+
+
+
+/****************************************
+ * WAITING
+ */
+
+void CDir::add_dentry_waiter(const string& dname, snapid_t snapid, Context *c) 
+{
+  if (waiting_on_dentry.empty())
+    get(PIN_DNWAITER);
+  waiting_on_dentry[string_snap_t(dname, snapid)].push_back(c);
+  dout(10) << "add_dentry_waiter dentry " << dname
+	   << " snap " << snapid
+	   << " " << c << " on " << *this << dendl;
+}
+
+void CDir::take_dentry_waiting(const string& dname, snapid_t first, snapid_t last,
+			       list& ls)
+{
+  if (waiting_on_dentry.empty())
+    return;
+  
+  string_snap_t lb(dname, first);
+  string_snap_t ub(dname, last);
+  map >::iterator p = waiting_on_dentry.lower_bound(lb);
+  while (p != waiting_on_dentry.end() &&
+	 !(ub < p->first)) {
+    dout(10) << "take_dentry_waiting dentry " << dname
+	     << " [" << first << "," << last << "] found waiter on snap "
+	     << p->first.snapid
+	     << " on " << *this << dendl;
+    ls.splice(ls.end(), p->second);
+    waiting_on_dentry.erase(p++);
+  }
+
+  if (waiting_on_dentry.empty())
+    put(PIN_DNWAITER);
+}
+
+void CDir::add_ino_waiter(inodeno_t ino, Context *c) 
+{
+  if (waiting_on_ino.empty())
+    get(PIN_INOWAITER);
+  waiting_on_ino[ino].push_back(c);
+  dout(10) << "add_ino_waiter ino " << ino << " " << c << " on " << *this << dendl;
+}
+
+void CDir::take_ino_waiting(inodeno_t ino, list& ls)
+{
+  if (waiting_on_ino.empty()) return;
+  if (waiting_on_ino.count(ino) == 0) return;
+  dout(10) << "take_ino_waiting ino " << ino
+	   << " x " << waiting_on_ino[ino].size() 
+	   << " on " << *this << dendl;
+  ls.splice(ls.end(), waiting_on_ino[ino]);
+  waiting_on_ino.erase(ino);
+  if (waiting_on_ino.empty())
+    put(PIN_INOWAITER);
+}
+
+void CDir::take_sub_waiting(list& ls)
+{
+  dout(10) << "take_sub_waiting" << dendl;
+  if (!waiting_on_dentry.empty()) {
+    for (map >::iterator p = waiting_on_dentry.begin(); 
+	 p != waiting_on_dentry.end();
+	 ++p) 
+      ls.splice(ls.end(), p->second);
+    waiting_on_dentry.clear();
+    put(PIN_DNWAITER);
+  }
+  if (!waiting_on_ino.empty()) {
+    for (map >::iterator p = waiting_on_ino.begin(); 
+	 p != waiting_on_ino.end();
+	 ++p) 
+      ls.splice(ls.end(), p->second);
+    waiting_on_ino.clear();
+    put(PIN_INOWAITER);
+  }
+}
+
+
+
+void CDir::add_waiter(uint64_t tag, Context *c) 
+{
+  // hierarchical?
+
+  // at free root?
+  if (tag & WAIT_ATFREEZEROOT) {
+    if (!(is_freezing_tree_root() || is_frozen_tree_root() ||
+	  is_freezing_dir() || is_frozen_dir())) {
+      // try parent
+      dout(10) << "add_waiter " << std::hex << tag << std::dec << " " << c << " should be ATFREEZEROOT, " << *this << " is not root, trying parent" << dendl;
+      inode->parent->dir->add_waiter(tag, c);
+      return;
+    }
+  }
+  
+  // at subtree root?
+  if (tag & WAIT_ATSUBTREEROOT) {
+    if (!is_subtree_root()) {
+      // try parent
+      dout(10) << "add_waiter " << std::hex << tag << std::dec << " " << c << " should be ATSUBTREEROOT, " << *this << " is not root, trying parent" << dendl;
+      inode->parent->dir->add_waiter(tag, c);
+      return;
+    }
+  }
+
+  MDSCacheObject::add_waiter(tag, c);
+}
+
+
+
+/* NOTE: this checks dentry waiters too */
+void CDir::take_waiting(uint64_t mask, list& ls)
+{
+  if ((mask & WAIT_DENTRY) && !waiting_on_dentry.empty()) {
+    // take all dentry waiters
+    while (!waiting_on_dentry.empty()) {
+      map >::iterator p = waiting_on_dentry.begin(); 
+      dout(10) << "take_waiting dentry " << p->first.name
+	       << " snap " << p->first.snapid << " on " << *this << dendl;
+      ls.splice(ls.end(), p->second);
+      waiting_on_dentry.erase(p);
+    }
+    put(PIN_DNWAITER);
+  }
+  
+  // waiting
+  MDSCacheObject::take_waiting(mask, ls);
+}
+
+
+void CDir::finish_waiting(uint64_t mask, int result) 
+{
+  dout(11) << "finish_waiting mask " << hex << mask << dec << " result " << result << " on " << *this << dendl;
+
+  list finished;
+  take_waiting(mask, finished);
+  if (result < 0)
+    finish_contexts(g_ceph_context, finished, result);
+  else
+    cache->mds->queue_waiters(finished);
+}
+
+
+
+// dirty/clean
+
+fnode_t *CDir::project_fnode()
+{
+  assert(get_version() != 0);
+  fnode_t *p = new fnode_t;
+  *p = *get_projected_fnode();
+  projected_fnode.push_back(p);
+  dout(10) << "project_fnode " << p << dendl;
+  return p;
+}
+
+void CDir::pop_and_dirty_projected_fnode(LogSegment *ls)
+{
+  assert(!projected_fnode.empty());
+  dout(15) << "pop_and_dirty_projected_fnode " << projected_fnode.front()
+	   << " v" << projected_fnode.front()->version << dendl;
+  fnode = *projected_fnode.front();
+  _mark_dirty(ls);
+  delete projected_fnode.front();
+  projected_fnode.pop_front();
+}
+
+
+version_t CDir::pre_dirty(version_t min)
+{
+  if (min > projected_version)
+    projected_version = min;
+  ++projected_version;
+  dout(10) << "pre_dirty " << projected_version << dendl;
+  return projected_version;
+}
+
+void CDir::mark_dirty(version_t pv, LogSegment *ls)
+{
+  assert(get_version() < pv);
+  assert(pv <= projected_version);
+  fnode.version = pv;
+  _mark_dirty(ls);
+}
+
+void CDir::_mark_dirty(LogSegment *ls)
+{
+  if (!state_test(STATE_DIRTY)) {
+    dout(10) << "mark_dirty (was clean) " << *this << " version " << get_version() << dendl;
+    _set_dirty_flag();
+    assert(ls);
+  } else {
+    dout(10) << "mark_dirty (already dirty) " << *this << " version " << get_version() << dendl;
+  }
+  if (ls) {
+    ls->dirty_dirfrags.push_back(&item_dirty);
+
+    // if i've never committed, i need to be before _any_ mention of me is trimmed from the journal.
+    if (committed_version == 0 && !item_new.is_on_list())
+      ls->new_dirfrags.push_back(&item_new);
+  }
+}
+
+void CDir::mark_new(LogSegment *ls)
+{
+  ls->new_dirfrags.push_back(&item_new);
+}
+
+void CDir::mark_clean()
+{
+  dout(10) << "mark_clean " << *this << " version " << get_version() << dendl;
+  if (state_test(STATE_DIRTY)) {
+    state_clear(STATE_DIRTY);
+    put(PIN_DIRTY);
+
+    item_dirty.remove_myself();
+    item_new.remove_myself();
+  }
+}
+
+
+struct C_Dir_Dirty : public Context {
+  CDir *dir;
+  version_t pv;
+  LogSegment *ls;
+  C_Dir_Dirty(CDir *d, version_t p, LogSegment *l) : dir(d), pv(p), ls(l) {}
+  void finish(int r) {
+    dir->mark_dirty(pv, ls);
+  }
+};
+
+void CDir::log_mark_dirty()
+{
+  MDLog *mdlog = inode->mdcache->mds->mdlog;
+  version_t pv = pre_dirty();
+  mdlog->flush();
+  mdlog->wait_for_safe(new C_Dir_Dirty(this, pv, mdlog->get_current_segment()));
+}
+
+void CDir::mark_complete() {
+  state_set(STATE_COMPLETE);
+  remove_bloom();
+}
+
+void CDir::first_get()
+{
+  inode->get(CInode::PIN_DIRFRAG);
+}
+
+void CDir::last_put()
+{
+  inode->put(CInode::PIN_DIRFRAG);
+}
+
+
+
+/******************************************************************************
+ * FETCH and COMMIT
+ */
+
+// -----------------------
+// FETCH
+void CDir::fetch(Context *c, bool ignore_authpinnability)
+{
+  string want;
+  return fetch(c, want, ignore_authpinnability);
+}
+
+void CDir::fetch(Context *c, const string& want_dn, bool ignore_authpinnability)
+{
+  dout(10) << "fetch on " << *this << dendl;
+  
+  assert(is_auth());
+  assert(!is_complete());
+
+  if (!can_auth_pin() && !ignore_authpinnability) {
+    if (c) {
+      dout(7) << "fetch waiting for authpinnable" << dendl;
+      add_waiter(WAIT_UNFREEZE, c);
+    } else
+      dout(7) << "fetch not authpinnable and no context" << dendl;
+    return;
+  }
+
+  if (c) add_waiter(WAIT_COMPLETE, c);
+  
+  // already fetching?
+  if (state_test(CDir::STATE_FETCHING)) {
+    dout(7) << "already fetching; waiting" << dendl;
+    return;
+  }
+
+  auth_pin(this);
+  state_set(CDir::STATE_FETCHING);
+
+  if (cache->mds->logger) cache->mds->logger->inc(l_mds_dir_f);
+
+  _omap_fetch(want_dn);
+}
+
+class C_Dir_TMAP_Fetched : public Context {
+ protected:
+  CDir *dir;
+  string want_dn;
+ public:
+  bufferlist bl;
+
+  C_Dir_TMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
+  void finish(int r) {
+    dir->_tmap_fetched(bl, want_dn, r);
+  }
+};
+
+void CDir::_tmap_fetch(const string& want_dn)
+{
+  // start by reading the first hunk of it
+  C_Dir_TMAP_Fetched *fin = new C_Dir_TMAP_Fetched(this, want_dn);
+  object_t oid = get_ondisk_object();
+  object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
+  ObjectOperation rd;
+  rd.tmap_get(&fin->bl, NULL);
+  cache->mds->objecter->read(oid, oloc, rd, CEPH_NOSNAP, NULL, 0, fin);
+}
+
+void CDir::_tmap_fetched(bufferlist& bl, const string& want_dn, int r)
+{
+  LogClient &clog = cache->mds->clog;
+  dout(10) << "_tmap_fetched " << bl.length()  << " bytes for " << *this
+	   << " want_dn=" << want_dn << dendl;
+
+  assert(r == 0 || r == -ENOENT);
+  assert(is_auth());
+  assert(!is_frozen());
+
+  bufferlist header;
+  map omap;
+
+  if (bl.length() == 0) {
+    r = -ENODATA;
+  } else {
+    bufferlist::iterator p = bl.begin();
+    ::decode(header, p);
+    ::decode(omap, p);
+
+    if (!p.end()) {
+      clog.warn() << "tmap buffer of dir " << dirfrag() << " has "
+		  << bl.length() - p.get_off() << " extra bytes\n";
+    }
+    bl.clear();
+  }
+
+  _omap_fetched(header, omap, want_dn, r);
+}
+
+class C_Dir_OMAP_Fetched : public Context {
+ protected:
+  CDir *dir;
+  string want_dn;
+ public:
+  bufferlist hdrbl;
+  map omap;
+  int ret1, ret2;
+
+  C_Dir_OMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
+  void finish(int r) {
+    if (r >= 0) r = ret1;
+    if (r >= 0) r = ret2;
+    dir->_omap_fetched(hdrbl, omap, want_dn, r);
+  }
+};
+
+void CDir::_omap_fetch(const string& want_dn)
+{
+  C_Dir_OMAP_Fetched *fin = new C_Dir_OMAP_Fetched(this, want_dn);
+  object_t oid = get_ondisk_object();
+  object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
+  ObjectOperation rd;
+  rd.omap_get_header(&fin->hdrbl, &fin->ret1);
+  rd.omap_get_vals("", "", (uint64_t)-1, &fin->omap, &fin->ret2);
+  cache->mds->objecter->read(oid, oloc, rd, CEPH_NOSNAP, NULL, 0, fin);
+}
+
+void CDir::_omap_fetched(bufferlist& hdrbl, map& omap,
+			 const string& want_dn, int r)
+{
+  LogClient &clog = cache->mds->clog;
+  dout(10) << "_fetched header " << hdrbl.length() << " bytes "
+	   << omap.size() << " keys for " << *this
+	   << " want_dn=" << want_dn << dendl;
+
+  assert(r == 0 || r == -ENOENT || r == -ENODATA);
+  assert(is_auth());
+  assert(!is_frozen());
+
+  if (hdrbl.length() == 0) {
+    if (r != -ENODATA) { // called by _tmap_fetched() ?
+      dout(10) << "_fetched 0 byte from omap, retry tmap" << dendl;
+      _tmap_fetch(want_dn);
+      return;
+    }
+
+    dout(0) << "_fetched missing object for " << *this << dendl;
+    clog.error() << "dir " << dirfrag() << " object missing on disk; some files may be lost\n";
+
+    log_mark_dirty();
+
+    // mark complete, !fetching
+    mark_complete();
+    state_clear(STATE_FETCHING);
+    auth_unpin(this);
+    
+    // kick waiters
+    finish_waiting(WAIT_COMPLETE, 0);
+    return;
+  }
+
+  fnode_t got_fnode;
+  {
+    bufferlist::iterator p = hdrbl.begin();
+    ::decode(got_fnode, p);
+    if (!p.end()) {
+      clog.warn() << "header buffer of dir " << dirfrag() << " has "
+		  << hdrbl.length() - p.get_off() << " extra bytes\n";
+    }
+  }
+
+  dout(10) << "_fetched version " << got_fnode.version << dendl;
+  
+  // take the loaded fnode?
+  // only if we are a fresh CDir* with no prior state.
+  if (get_version() == 0) {
+    assert(!is_projected());
+    assert(!state_test(STATE_COMMITTING));
+    fnode = got_fnode;
+    projected_version = committing_version = committed_version = got_fnode.version;
+
+    if (state_test(STATE_REJOINUNDEF)) {
+      assert(cache->mds->is_rejoin());
+      state_clear(STATE_REJOINUNDEF);
+      cache->opened_undef_dirfrag(this);
+    }
+  }
+
+  list undef_inodes;
+
+  // purge stale snaps?
+  // only if we have past_parents open!
+  bool purged_any = false;
+  const set *snaps = NULL;
+  SnapRealm *realm = inode->find_snaprealm();
+  if (!realm->have_past_parents_open()) {
+    dout(10) << " no snap purge, one or more past parents NOT open" << dendl;
+  } else if (fnode.snap_purged_thru < realm->get_last_destroyed()) {
+    snaps = &realm->get_snaps();
+    dout(10) << " snap_purged_thru " << fnode.snap_purged_thru
+	     << " < " << realm->get_last_destroyed()
+	     << ", snap purge based on " << *snaps << dendl;
+    fnode.snap_purged_thru = realm->get_last_destroyed();
+  }
+
+  bool stray = inode->is_stray();
+
+  unsigned pos = omap.size() - 1;
+  for (map::reverse_iterator p = omap.rbegin();
+       p != omap.rend();
+       ++p, --pos) {
+    // dname
+    string dname;
+    snapid_t first, last;
+    dentry_key_t::decode_helper(p->first, dname, last);
+    
+    bufferlist::iterator q = p->second.begin();
+    ::decode(first, q);
+
+    // marker
+    char type;
+    ::decode(type, q);
+
+    dout(20) << "_fetched pos " << pos << " marker '" << type << "' dname '" << dname
+	     << " [" << first << "," << last << "]"
+	     << dendl;
+
+    bool stale = false;
+    if (snaps && last != CEPH_NOSNAP) {
+      set::const_iterator p = snaps->lower_bound(first);
+      if (p == snaps->end() || *p > last) {
+	dout(10) << " skipping stale dentry on [" << first << "," << last << "]" << dendl;
+	stale = true;
+	purged_any = true;
+      }
+    }
+    
+    /*
+     * look for existing dentry for _last_ snap, because unlink +
+     * create may leave a "hole" (epochs during which the dentry
+     * doesn't exist) but for which no explicit negative dentry is in
+     * the cache.
+     */
+    CDentry *dn = NULL;
+    if (!stale)
+      dn = lookup(dname, last);
+
+    if (type == 'L') {
+      // hard link
+      inodeno_t ino;
+      unsigned char d_type;
+      ::decode(ino, q);
+      ::decode(d_type, q);
+
+      if (stale)
+	continue;
+
+      if (dn) {
+        if (dn->get_linkage()->get_inode() == 0) {
+          dout(12) << "_fetched  had NEG dentry " << *dn << dendl;
+        } else {
+          dout(12) << "_fetched  had dentry " << *dn << dendl;
+        }
+      } else {
+	// (remote) link
+	dn = add_remote_dentry(dname, ino, d_type, first, last);
+	
+	// link to inode?
+	CInode *in = cache->get_inode(ino);   // we may or may not have it.
+	if (in) {
+	  dn->link_remote(dn->get_linkage(), in);
+	  dout(12) << "_fetched  got remote link " << ino << " which we have " << *in << dendl;
+	} else {
+	  dout(12) << "_fetched  got remote link " << ino << " (dont' have it)" << dendl;
+	}
+      }
+    } 
+    else if (type == 'I') {
+      // inode
+      
+      // parse out inode
+      inode_t inode;
+      string symlink;
+      fragtree_t fragtree;
+      map xattrs;
+      bufferlist snapbl;
+      map old_inodes;
+      ::decode(inode, q);
+      if (inode.is_symlink())
+        ::decode(symlink, q);
+      ::decode(fragtree, q);
+      ::decode(xattrs, q);
+      ::decode(snapbl, q);
+      ::decode(old_inodes, q);
+      
+      if (stale)
+	continue;
+
+      bool undef_inode = false;
+      if (dn) {
+	CInode *in = dn->get_linkage()->get_inode();
+	if (in) {
+	  dout(12) << "_fetched  had dentry " << *dn << dendl;
+	  if (in->state_test(CInode::STATE_REJOINUNDEF)) {
+	    undef_inodes.push_back(in);
+	    undef_inode = true;
+	  }
+	} else
+	  dout(12) << "_fetched  had NEG dentry " << *dn << dendl;
+      }
+
+      if (!dn || undef_inode) {
+	// add inode
+	CInode *in = cache->get_inode(inode.ino, last);
+	if (!in || undef_inode) {
+	  if (undef_inode && in)
+	    in->first = first;
+	  else
+	    in = new CInode(cache, true, first, last);
+	  
+	  in->inode = inode;
+	  // symlink?
+	  if (in->is_symlink()) 
+	    in->symlink = symlink;
+	  
+	  in->dirfragtree.swap(fragtree);
+	  in->xattrs.swap(xattrs);
+	  in->decode_snap_blob(snapbl);
+	  in->old_inodes.swap(old_inodes);
+	  if (snaps)
+	    in->purge_stale_snap_data(*snaps);
+
+	  if (undef_inode) {
+	    if (inode.anchored)
+	      dn->adjust_nested_anchors(1);
+	  } else {
+	    cache->add_inode( in ); // add
+	    dn = add_primary_dentry(dname, in, first, last); // link
+	  }
+	  dout(12) << "_fetched  got " << *dn << " " << *in << dendl;
+
+	  if (in->inode.is_dirty_rstat())
+	    in->mark_dirty_rstat();
+
+	  if (stray) {
+	    dn->state_set(CDentry::STATE_STRAY);
+	    if (in->inode.nlink == 0)
+	      in->state_set(CInode::STATE_ORPHAN);
+	  }
+
+	  //in->hack_accessed = false;
+	  //in->hack_load_stamp = ceph_clock_now(g_ceph_context);
+	  //num_new_inodes_loaded++;
+	} else {
+	  dout(0) << "_fetched  badness: got (but i already had) " << *in
+		  << " mode " << in->inode.mode
+		  << " mtime " << in->inode.mtime << dendl;
+	  string dirpath, inopath;
+	  this->inode->make_path_string(dirpath);
+	  in->make_path_string(inopath);
+	  clog.error() << "loaded dup inode " << inode.ino
+	    << " [" << first << "," << last << "] v" << inode.version
+	    << " at " << dirpath << "/" << dname
+	    << ", but inode " << in->vino() << " v" << in->inode.version
+	    << " already exists at " << inopath << "\n";
+	  continue;
+	}
+      }
+    } else {
+      dout(1) << "corrupt directory, i got tag char '" << type << "' pos " << pos << dendl;
+      assert(0);
+    }
+    
+    if (dn && want_dn.length() && want_dn == dname) {
+      dout(10) << " touching wanted dn " << *dn << dendl;
+      inode->mdcache->touch_dentry(dn);
+    }
+
+    /** clean underwater item?
+     * Underwater item is something that is dirty in our cache from
+     * journal replay, but was previously flushed to disk before the
+     * mds failed.
+     *
+     * We only do this is committed_version == 0. that implies either
+     * - this is a fetch after from a clean/empty CDir is created
+     *   (and has no effect, since the dn won't exist); or
+     * - this is a fetch after _recovery_, which is what we're worried 
+     *   about.  Items that are marked dirty from the journal should be
+     *   marked clean if they appear on disk.
+     */
+    if (committed_version == 0 &&     
+	dn &&
+	dn->get_version() <= got_fnode.version &&
+	dn->is_dirty()) {
+      dout(10) << "_fetched  had underwater dentry " << *dn << ", marking clean" << dendl;
+      dn->mark_clean();
+
+      if (dn->get_linkage()->is_primary()) {
+	assert(dn->get_linkage()->get_inode()->get_version() <= got_fnode.version);
+	dout(10) << "_fetched  had underwater inode " << *dn->get_linkage()->get_inode() << ", marking clean" << dendl;
+	dn->get_linkage()->get_inode()->mark_clean();
+      }
+    }
+  }
+
+  //cache->mds->logger->inc("newin", num_new_inodes_loaded);
+
+  if (purged_any)
+    log_mark_dirty();
+
+  // mark complete, !fetching
+  mark_complete();
+  state_clear(STATE_FETCHING);
+
+  // open & force frags
+  while (!undef_inodes.empty()) {
+    CInode *in = undef_inodes.front();
+    undef_inodes.pop_front();
+    in->state_clear(CInode::STATE_REJOINUNDEF);
+    cache->opened_undef_inode(in);
+  }
+
+  auth_unpin(this);
+
+  // kick waiters
+  finish_waiting(WAIT_COMPLETE, 0);
+}
+
+
+
+// -----------------------
+// COMMIT
+
+/**
+ * commit
+ *
+ * @param want - min version i want committed
+ * @param c - callback for completion
+ */
+void CDir::commit(version_t want, Context *c, bool ignore_authpinnability, int op_prio)
+{
+  dout(10) << "commit want " << want << " on " << *this << dendl;
+  if (want == 0) want = get_version();
+
+  // preconditions
+  assert(want <= get_version() || get_version() == 0);    // can't commit the future
+  assert(want > committed_version); // the caller is stupid
+  assert(is_auth());
+  assert(ignore_authpinnability || can_auth_pin());
+
+  // note: queue up a noop if necessary, so that we always
+  // get an auth_pin.
+  if (!c)
+    c = new C_NoopContext;
+
+  // auth_pin on first waiter
+  if (waiting_for_commit.empty())
+    auth_pin(this);
+  waiting_for_commit[want].push_back(c);
+  
+  // ok.
+  _commit(want, op_prio);
+}
+
+class C_Dir_Committed : public Context {
+  CDir *dir;
+  version_t version;
+public:
+  C_Dir_Committed(CDir *d, version_t v) : dir(d), version(v) { }
+  void finish(int r) {
+    assert(r == 0);
+    dir->_committed(version);
+  }
+};
+
+/**
+ * Flush out the modified dentries in this dir. Keep the bufferlist
+ * below max_write_size;
+ */
+void CDir::_omap_commit(int op_prio)
+{
+  dout(10) << "_omap_commit" << dendl;
+
+  unsigned max_write_size = cache->max_dir_commit_size;
+  unsigned write_size = 0;
+
+  if (op_prio < 0)
+    op_prio = CEPH_MSG_PRIO_DEFAULT;
+
+  // snap purge?
+  const set *snaps = NULL;
+  SnapRealm *realm = inode->find_snaprealm();
+  if (!realm->have_past_parents_open()) {
+    dout(10) << " no snap purge, one or more past parents NOT open" << dendl;
+  } else if (fnode.snap_purged_thru < realm->get_last_destroyed()) {
+    snaps = &realm->get_snaps();
+    dout(10) << " snap_purged_thru " << fnode.snap_purged_thru
+	     << " < " << realm->get_last_destroyed()
+	     << ", snap purge based on " << *snaps << dendl;
+  }
+
+  set to_remove;
+  map to_set;
+
+  C_GatherBuilder gather(g_ceph_context, new C_Dir_Committed(this, get_version()));
+
+  SnapContext snapc;
+  object_t oid = get_ondisk_object();
+  object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
+
+  for (map_t::iterator p = items.begin();
+      p != items.end(); ) {
+    CDentry *dn = p->second;
+    ++p;
+
+    string key;
+    dn->key().encode(key);
+
+    if (dn->last != CEPH_NOSNAP &&
+	snaps && try_trim_snap_dentry(dn, *snaps)) {
+      dout(10) << " rm " << dn->name << " " << *dn << dendl;
+      write_size += key.length();
+      to_remove.insert(key);
+      continue;
+    }
+
+    if (!dn->is_dirty() &&
+	(!dn->state_test(CDentry::STATE_FRAGMENTING) || dn->get_linkage()->is_null()))
+      continue;  // skip clean dentries
+
+    if (dn->get_linkage()->is_null()) {
+      dout(10) << " rm " << dn->name << " " << *dn << dendl;
+      write_size += key.length();
+      to_remove.insert(key);
+    } else {
+      dout(10) << " set " << dn->name << " " << *dn << dendl;
+      bufferlist dnbl;
+      _encode_dentry(dn, dnbl, snaps);
+      write_size += key.length() + dnbl.length();
+      to_set[key].swap(dnbl);
+    }
+
+    if (write_size >= max_write_size) {
+      ObjectOperation op;
+      op.priority = op_prio;
+      op.tmap_to_omap(true); // convert tmap to omap
+
+      if (!to_set.empty())
+	op.omap_set(to_set);
+      if (!to_remove.empty())
+	op.omap_rm_keys(to_remove);
+
+      cache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
+				   0, NULL, gather.new_sub());
+
+      write_size = 0;
+      to_set.clear();
+      to_remove.clear();
+    }
+  }
+
+  ObjectOperation op;
+  op.priority = op_prio;
+  op.tmap_to_omap(true); // convert tmap to omap
+
+  /*
+   * save the header at the last moment.. If we were to send it off before other
+   * updates, but die before sending them all, we'd think that the on-disk state
+   * was fully committed even though it wasn't! However, since the messages are
+   * strictly ordered between the MDS and the OSD, and since messages to a given
+   * PG are strictly ordered, if we simply send the message containing the header
+   * off last, we cannot get our header into an incorrect state.
+   */
+  bufferlist header;
+  ::encode(fnode, header);
+  op.omap_set_header(header);
+
+  if (!to_set.empty())
+    op.omap_set(to_set);
+  if (!to_remove.empty())
+    op.omap_rm_keys(to_remove);
+
+  cache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
+			       0, NULL, gather.new_sub());
+
+  gather.activate();
+}
+
+void CDir::_encode_dentry(CDentry *dn, bufferlist& bl,
+			  const set *snaps)
+{
+  // clear dentry NEW flag, if any.  we can no longer silently drop it.
+  dn->clear_new();
+
+  ::encode(dn->first, bl);
+
+  // primary or remote?
+  if (dn->linkage.is_remote()) {
+    inodeno_t ino = dn->linkage.get_remote_ino();
+    unsigned char d_type = dn->linkage.get_remote_d_type();
+    dout(14) << " pos " << bl.length() << " dn '" << dn->name << "' remote ino " << ino << dendl;
+    
+    // marker, name, ino
+    bl.append('L');         // remote link
+    ::encode(ino, bl);
+    ::encode(d_type, bl);
+  } else {
+    // primary link
+    CInode *in = dn->linkage.get_inode();
+    assert(in);
+    
+    dout(14) << " pos " << bl.length() << " dn '" << dn->name << "' inode " << *in << dendl;
+    
+    // marker, name, inode, [symlink string]
+    bl.append('I');         // inode
+    ::encode(in->inode, bl);
+    
+    if (in->is_symlink()) {
+      // include symlink destination!
+      dout(18) << "    including symlink ptr " << in->symlink << dendl;
+      ::encode(in->symlink, bl);
+    }
+    
+    ::encode(in->dirfragtree, bl);
+    ::encode(in->xattrs, bl);
+    bufferlist snapbl;
+    in->encode_snap_blob(snapbl);
+    ::encode(snapbl, bl);
+    
+    if (in->is_multiversion() && snaps)
+      in->purge_stale_snap_data(*snaps);
+    ::encode(in->old_inodes, bl);
+  }
+}
+
+void CDir::_commit(version_t want, int op_prio)
+{
+  dout(10) << "_commit want " << want << " on " << *this << dendl;
+
+  // we can't commit things in the future.
+  // (even the projected future.)
+  assert(want <= get_version() || get_version() == 0);
+
+  // check pre+postconditions.
+  assert(is_auth());
+
+  // already committed?
+  if (committed_version >= want) {
+    dout(10) << "already committed " << committed_version << " >= " << want << dendl;
+    return;
+  }
+  // already committing >= want?
+  if (committing_version >= want) {
+    dout(10) << "already committing " << committing_version << " >= " << want << dendl;
+    assert(state_test(STATE_COMMITTING));
+    return;
+  }
+
+  // alrady committed an older version?
+  if (committing_version > committed_version) {
+    dout(10) << "already committing older " << committing_version << ", waiting for that to finish" << dendl;
+    return;
+  }
+  
+  // commit.
+  committing_version = get_version();
+
+  // mark committing (if not already)
+  if (!state_test(STATE_COMMITTING)) {
+    dout(10) << "marking committing" << dendl;
+    state_set(STATE_COMMITTING);
+  }
+  
+  if (cache->mds->logger) cache->mds->logger->inc(l_mds_dir_c);
+
+   _omap_commit(op_prio);
+}
+
+
+/**
+ * _committed
+ *
+ * @param v version i just committed
+ */
+void CDir::_committed(version_t v)
+{
+  dout(10) << "_committed v " << v << " on " << *this << dendl;
+  assert(is_auth());
+
+  bool stray = inode->is_stray();
+
+  // take note.
+  assert(v > committed_version);
+  assert(v <= committing_version);
+  committed_version = v;
+
+  // _all_ commits done?
+  if (committing_version == committed_version) 
+    state_clear(CDir::STATE_COMMITTING);
+
+  // _any_ commit, even if we've been redirtied, means we're no longer new.
+  item_new.remove_myself();
+  
+  // dir clean?
+  if (committed_version == get_version()) 
+    mark_clean();
+
+  // dentries clean?
+  for (map_t::iterator it = items.begin();
+       it != items.end(); ) {
+    CDentry *dn = it->second;
+    ++it;
+    
+    // inode?
+    if (dn->linkage.is_primary()) {
+      CInode *in = dn->linkage.get_inode();
+      assert(in);
+      assert(in->is_auth());
+      
+      if (committed_version >= in->get_version()) {
+	if (in->is_dirty()) {
+	  dout(15) << " dir " << committed_version << " >= inode " << in->get_version() << " now clean " << *in << dendl;
+	  in->mark_clean();
+	}
+      } else {
+	dout(15) << " dir " << committed_version << " < inode " << in->get_version() << " still dirty " << *in << dendl;
+	assert(in->is_dirty() || in->last < CEPH_NOSNAP);  // special case for cow snap items (not predirtied)
+      }
+    }
+
+    // dentry
+    if (committed_version >= dn->get_version()) {
+      if (dn->is_dirty()) {
+	dout(15) << " dir " << committed_version << " >= dn " << dn->get_version() << " now clean " << *dn << dendl;
+	dn->mark_clean();
+
+	// drop clean null stray dentries immediately
+	if (stray && 
+	    dn->get_num_ref() == 0 &&
+	    !dn->is_projected() &&
+	    dn->get_linkage()->is_null())
+	  remove_dentry(dn);
+      } 
+    } else {
+      dout(15) << " dir " << committed_version << " < dn " << dn->get_version() << " still dirty " << *dn << dendl;
+    }
+  }
+
+  // finishers?
+  bool were_waiters = !waiting_for_commit.empty();
+  
+  map >::iterator p = waiting_for_commit.begin();
+  while (p != waiting_for_commit.end()) {
+    map >::iterator n = p;
+    ++n;
+    if (p->first > committed_version) {
+      dout(10) << " there are waiters for " << p->first << ", committing again" << dendl;
+      _commit(p->first, -1);
+      break;
+    }
+    cache->mds->queue_waiters(p->second);
+    waiting_for_commit.erase(p);
+    p = n;
+  } 
+
+  // unpin if we kicked the last waiter.
+  if (were_waiters &&
+      waiting_for_commit.empty())
+    auth_unpin(this);
+}
+
+
+
+
+
+// IMPORT/EXPORT
+
+void CDir::encode_export(bufferlist& bl)
+{
+  assert(!is_projected());
+  ::encode(first, bl);
+  ::encode(fnode, bl);
+  ::encode(dirty_old_rstat, bl);
+  ::encode(committed_version, bl);
+
+  ::encode(state, bl);
+  ::encode(dir_rep, bl);
+
+  ::encode(pop_me, bl);
+  ::encode(pop_auth_subtree, bl);
+
+  ::encode(dir_rep_by, bl);  
+  ::encode(replica_map, bl);
+
+  get(PIN_TEMPEXPORTING);
+}
+
+void CDir::finish_export(utime_t now)
+{
+  state &= MASK_STATE_EXPORT_KEPT;
+  pop_auth_subtree_nested.sub(now, cache->decayrate, pop_auth_subtree);
+  pop_me.zero(now);
+  pop_auth_subtree.zero(now);
+  put(PIN_TEMPEXPORTING);
+  dirty_old_rstat.clear();
+}
+
+void CDir::decode_import(bufferlist::iterator& blp, utime_t now, LogSegment *ls)
+{
+  ::decode(first, blp);
+  ::decode(fnode, blp);
+  ::decode(dirty_old_rstat, blp);
+  projected_version = fnode.version;
+  ::decode(committed_version, blp);
+  committing_version = committed_version;
+
+  unsigned s;
+  ::decode(s, blp);
+  state &= MASK_STATE_IMPORT_KEPT;
+  state |= (s & MASK_STATE_EXPORTED);
+  if (is_dirty()) {
+    get(PIN_DIRTY);
+    _mark_dirty(ls);
+  }
+
+  ::decode(dir_rep, blp);
+
+  ::decode(pop_me, now, blp);
+  ::decode(pop_auth_subtree, now, blp);
+  pop_auth_subtree_nested.add(now, cache->decayrate, pop_auth_subtree);
+
+  ::decode(dir_rep_by, blp);
+  ::decode(replica_map, blp);
+  if (!replica_map.empty()) get(PIN_REPLICATED);
+
+  replica_nonce = 0;  // no longer defined
+
+  // did we import some dirty scatterlock data?
+  if (dirty_old_rstat.size() ||
+      !(fnode.rstat == fnode.accounted_rstat)) {
+    cache->mds->locker->mark_updated_scatterlock(&inode->nestlock);
+    ls->dirty_dirfrag_nest.push_back(&inode->item_dirty_dirfrag_nest);
+  }
+  if (!(fnode.fragstat == fnode.accounted_fragstat)) {
+    cache->mds->locker->mark_updated_scatterlock(&inode->filelock);
+    ls->dirty_dirfrag_dir.push_back(&inode->item_dirty_dirfrag_dir);
+  }
+  if (is_dirty_dft()) {
+    if (inode->dirfragtreelock.get_state() != LOCK_MIX &&
+	inode->dirfragtreelock.is_stable()) {
+      // clear stale dirtydft
+      state_clear(STATE_DIRTYDFT);
+    } else {
+      cache->mds->locker->mark_updated_scatterlock(&inode->dirfragtreelock);
+      ls->dirty_dirfrag_dirfragtree.push_back(&inode->item_dirty_dirfrag_dirfragtree);
+    }
+  }
+}
+
+
+
+
+/********************************
+ * AUTHORITY
+ */
+
+/*
+ * if dir_auth.first == parent, auth is same as inode.
+ * unless .second != unknown, in which case that sticks.
+ */
+pair CDir::authority() 
+{
+  if (is_subtree_root()) 
+    return dir_auth;
+  else
+    return inode->authority();
+}
+
+/** is_subtree_root()
+ * true if this is an auth delegation point.  
+ * that is, dir_auth != default (parent,unknown)
+ *
+ * some key observations:
+ *  if i am auth:
+ *    - any region bound will be an export, or frozen.
+ *
+ * note that this DOES heed dir_auth.pending
+ */
+/*
+bool CDir::is_subtree_root()
+{
+  if (dir_auth == CDIR_AUTH_DEFAULT) {
+    //dout(10) << "is_subtree_root false " << dir_auth << " != " << CDIR_AUTH_DEFAULT
+    //<< " on " << ino() << dendl;
+    return false;
+  } else {
+    //dout(10) << "is_subtree_root true " << dir_auth << " != " << CDIR_AUTH_DEFAULT
+    //<< " on " << ino() << dendl;
+    return true;
+  }
+}
+*/
+
+/** contains(x)
+ * true if we are x, or an ancestor of x
+ */
+bool CDir::contains(CDir *x)
+{
+  while (1) {
+    if (x == this)
+      return true;
+    x = x->get_inode()->get_projected_parent_dir();
+    if (x == 0)
+      return false;    
+  }
+}
+
+
+
+/** set_dir_auth
+ */
+void CDir::set_dir_auth(pair a)
+{ 
+  dout(10) << "setting dir_auth=" << a
+	   << " from " << dir_auth
+	   << " on " << *this << dendl;
+  
+  bool was_subtree = is_subtree_root();
+  bool was_ambiguous = dir_auth.second >= 0;
+
+  // set it.
+  dir_auth = a;
+
+  // new subtree root?
+  if (!was_subtree && is_subtree_root()) {
+    dout(10) << " new subtree root, adjusting auth_pins" << dendl;
+    
+    // adjust nested auth pins
+    if (get_cum_auth_pins())
+      inode->adjust_nested_auth_pins(-1, NULL);
+    
+    // unpin parent of frozen dir/tree?
+    if (inode->is_auth() && (is_frozen_tree_root() || is_frozen_dir()))
+      inode->auth_unpin(this);
+  } 
+  if (was_subtree && !is_subtree_root()) {
+    dout(10) << " old subtree root, adjusting auth_pins" << dendl;
+    
+    // adjust nested auth pins
+    if (get_cum_auth_pins())
+      inode->adjust_nested_auth_pins(1, NULL);
+
+    // pin parent of frozen dir/tree?
+    if (inode->is_auth() && (is_frozen_tree_root() || is_frozen_dir()))
+      inode->auth_pin(this);
+  }
+
+  // newly single auth?
+  if (was_ambiguous && dir_auth.second == CDIR_AUTH_UNKNOWN) {
+    list ls;
+    take_waiting(WAIT_SINGLEAUTH, ls);
+    cache->mds->queue_waiters(ls);
+  }
+}
+
+
+/*****************************************
+ * AUTH PINS and FREEZING
+ *
+ * the basic plan is that auth_pins only exist in auth regions, and they
+ * prevent a freeze (and subsequent auth change).  
+ *
+ * however, we also need to prevent a parent from freezing if a child is frozen.
+ * for that reason, the parent inode of a frozen directory is auth_pinned.
+ *
+ * the oddity is when the frozen directory is a subtree root.  if that's the case,
+ * the parent inode isn't frozen.  which means that when subtree authority is adjusted
+ * at the bounds, inodes for any frozen bound directories need to get auth_pins at that
+ * time.
+ *
+ */
+
+void CDir::auth_pin(void *by) 
+{
+  if (auth_pins == 0)
+    get(PIN_AUTHPIN);
+  auth_pins++;
+
+#ifdef MDS_AUTHPIN_SET
+  auth_pin_set.insert(by);
+#endif
+
+  dout(10) << "auth_pin by " << by
+	   << " on " << *this
+	   << " count now " << auth_pins << " + " << nested_auth_pins << dendl;
+
+  // nest pins?
+  if (!is_subtree_root() &&
+      get_cum_auth_pins() == 1)
+    inode->adjust_nested_auth_pins(1, by);
+}
+
+void CDir::auth_unpin(void *by) 
+{
+  auth_pins--;
+
+#ifdef MDS_AUTHPIN_SET
+  assert(auth_pin_set.count(by));
+  auth_pin_set.erase(auth_pin_set.find(by));
+#endif
+  if (auth_pins == 0)
+    put(PIN_AUTHPIN);
+
+  dout(10) << "auth_unpin by " << by
+	   << " on " << *this
+	   << " count now " << auth_pins << " + " << nested_auth_pins << dendl;
+  assert(auth_pins >= 0);
+  
+  int newcum = get_cum_auth_pins();
+
+  maybe_finish_freeze();  // pending freeze?
+  
+  // nest?
+  if (!is_subtree_root() &&
+      newcum == 0)
+    inode->adjust_nested_auth_pins(-1, by);
+}
+
+void CDir::adjust_nested_auth_pins(int inc, int dirinc, void *by)
+{
+  assert(inc);
+  nested_auth_pins += inc;
+  dir_auth_pins += dirinc;
+  
+  dout(15) << "adjust_nested_auth_pins " << inc << "/" << dirinc << " on " << *this
+	   << " by " << by << " count now "
+	   << auth_pins << " + " << nested_auth_pins << dendl;
+  assert(nested_auth_pins >= 0);
+  assert(dir_auth_pins >= 0);
+
+  int newcum = get_cum_auth_pins();
+
+  maybe_finish_freeze();  // pending freeze?
+  
+  // nest?
+  if (!is_subtree_root()) {
+    if (newcum == 0)
+      inode->adjust_nested_auth_pins(-1, by);
+    else if (newcum == inc)      
+      inode->adjust_nested_auth_pins(1, by);
+  }
+}
+
+void CDir::adjust_nested_anchors(int by)
+{
+  assert(by);
+  nested_anchors += by;
+  dout(20) << "adjust_nested_anchors by " << by << " -> " << nested_anchors << dendl;
+  assert(nested_anchors >= 0);
+  inode->adjust_nested_anchors(by);
+}
+
+#ifdef MDS_VERIFY_FRAGSTAT
+void CDir::verify_fragstat()
+{
+  assert(is_complete());
+  if (inode->is_stray())
+    return;
+
+  frag_info_t c;
+  memset(&c, 0, sizeof(c));
+
+  for (map_t::iterator it = items.begin();
+       it != items.end();
+       ++it) {
+    CDentry *dn = it->second;
+    if (dn->is_null())
+      continue;
+    
+    dout(10) << " " << *dn << dendl;
+    if (dn->is_primary())
+      dout(10) << "     " << *dn->inode << dendl;
+
+    if (dn->is_primary()) {
+      if (dn->inode->is_dir())
+	c.nsubdirs++;
+      else
+	c.nfiles++;
+    }
+    if (dn->is_remote()) {
+      if (dn->get_remote_d_type() == DT_DIR)
+	c.nsubdirs++;
+      else
+	c.nfiles++;
+    }
+  }
+
+  if (c.nsubdirs != fnode.fragstat.nsubdirs ||
+      c.nfiles != fnode.fragstat.nfiles) {
+    dout(0) << "verify_fragstat failed " << fnode.fragstat << " on " << *this << dendl;
+    dout(0) << "               i count " << c << dendl;
+    assert(0);
+  } else {
+    dout(0) << "verify_fragstat ok " << fnode.fragstat << " on " << *this << dendl;
+  }
+}
+#endif
+
+/*****************************************************************************
+ * FREEZING
+ */
+
+// FREEZE TREE
+
+bool CDir::freeze_tree()
+{
+  assert(!is_frozen());
+  assert(!is_freezing());
+
+  auth_pin(this);
+  if (is_freezeable(true)) {
+    _freeze_tree();
+    auth_unpin(this);
+    return true;
+  } else {
+    state_set(STATE_FREEZINGTREE);
+    dout(10) << "freeze_tree waiting " << *this << dendl;
+    return false;
+  }
+}
+
+void CDir::_freeze_tree()
+{
+  dout(10) << "_freeze_tree " << *this << dendl;
+  assert(is_freezeable(true));
+
+  // twiddle state
+  state_clear(STATE_FREEZINGTREE);   // actually, this may get set again by next context?
+  state_set(STATE_FROZENTREE);
+  get(PIN_FROZEN);
+
+  // auth_pin inode for duration of freeze, if we are not a subtree root.
+  if (is_auth() && !is_subtree_root())
+    inode->auth_pin(this);
+}
+
+void CDir::unfreeze_tree()
+{
+  dout(10) << "unfreeze_tree " << *this << dendl;
+
+  if (state_test(STATE_FROZENTREE)) {
+    // frozen.  unfreeze.
+    state_clear(STATE_FROZENTREE);
+    put(PIN_FROZEN);
+
+    // unpin  (may => FREEZEABLE)   FIXME: is this order good?
+    if (is_auth() && !is_subtree_root())
+      inode->auth_unpin(this);
+
+    // waiters?
+    finish_waiting(WAIT_UNFREEZE);
+  } else {
+    finish_waiting(WAIT_FROZEN, -1);
+
+    // freezing.  stop it.
+    assert(state_test(STATE_FREEZINGTREE));
+    state_clear(STATE_FREEZINGTREE);
+    auth_unpin(this);
+    
+    finish_waiting(WAIT_UNFREEZE);
+  }
+}
+
+bool CDir::is_freezing_tree()
+{
+  CDir *dir = this;
+  while (1) {
+    if (dir->is_freezing_tree_root()) return true;
+    if (dir->is_subtree_root()) return false;
+    if (dir->inode->parent)
+      dir = dir->inode->parent->dir;
+    else
+      return false; // root on replica
+  }
+}
+
+bool CDir::is_frozen_tree()
+{
+  CDir *dir = this;
+  while (1) {
+    if (dir->is_frozen_tree_root()) return true;
+    if (dir->is_subtree_root()) return false;
+    if (dir->inode->parent)
+      dir = dir->inode->parent->dir;
+    else
+      return false;  // root on replica
+  }
+}
+
+CDir *CDir::get_frozen_tree_root() 
+{
+  assert(is_frozen());
+  CDir *dir = this;
+  while (1) {
+    if (dir->is_frozen_tree_root()) 
+      return dir;
+    if (dir->inode->parent)
+      dir = dir->inode->parent->dir;
+    else
+      assert(0);
+  }
+}
+
+struct C_Dir_AuthUnpin : public Context {
+  CDir *dir;
+  C_Dir_AuthUnpin(CDir *d) : dir(d) {}
+  void finish(int r) {
+    dir->auth_unpin(dir->get_inode());
+  }
+};
+
+void CDir::maybe_finish_freeze()
+{
+  if (auth_pins != 1 || dir_auth_pins != 0)
+    return;
+
+  // we can freeze the _dir_ even with nested pins...
+  if (state_test(STATE_FREEZINGDIR)) {
+    _freeze_dir();
+    auth_unpin(this);
+    finish_waiting(WAIT_FROZEN);
+  }
+
+  if (nested_auth_pins != 0)
+    return;
+
+  if (state_test(STATE_FREEZINGTREE)) {
+    if (!is_subtree_root() && inode->is_frozen()) {
+      dout(10) << "maybe_finish_freeze !subtree root and frozen inode, waiting for unfreeze on " << inode << dendl;
+      // retake an auth_pin...
+      auth_pin(inode);
+      // and release it when the parent inode unfreezes
+      inode->add_waiter(WAIT_UNFREEZE, new C_Dir_AuthUnpin(this));
+      return;
+    }
+
+    _freeze_tree();
+    auth_unpin(this);
+    finish_waiting(WAIT_FROZEN);
+  }
+}
+
+
+
+// FREEZE DIR
+
+bool CDir::freeze_dir()
+{
+  assert(!is_frozen());
+  assert(!is_freezing());
+  
+  auth_pin(this);
+  if (is_freezeable_dir(true)) {
+    _freeze_dir();
+    auth_unpin(this);
+    return true;
+  } else {
+    state_set(STATE_FREEZINGDIR);
+    dout(10) << "freeze_dir + wait " << *this << dendl;
+    return false;
+  } 
+}
+
+void CDir::_freeze_dir()
+{
+  dout(10) << "_freeze_dir " << *this << dendl;
+  //assert(is_freezeable_dir(true));
+  // not always true during split because the original fragment may have frozen a while
+  // ago and we're just now getting around to breaking it up.
+
+  state_clear(STATE_FREEZINGDIR);
+  state_set(STATE_FROZENDIR);
+  get(PIN_FROZEN);
+
+  if (is_auth() && !is_subtree_root())
+    inode->auth_pin(this);  // auth_pin for duration of freeze
+}
+
+
+void CDir::unfreeze_dir()
+{
+  dout(10) << "unfreeze_dir " << *this << dendl;
+
+  if (state_test(STATE_FROZENDIR)) {
+    state_clear(STATE_FROZENDIR);
+    put(PIN_FROZEN);
+
+    // unpin  (may => FREEZEABLE)   FIXME: is this order good?
+    if (is_auth() && !is_subtree_root())
+      inode->auth_unpin(this);
+
+    finish_waiting(WAIT_UNFREEZE);
+  } else {
+    finish_waiting(WAIT_FROZEN, -1);
+
+    // still freezing. stop.
+    assert(state_test(STATE_FREEZINGDIR));
+    state_clear(STATE_FREEZINGDIR);
+    auth_unpin(this);
+    
+    finish_waiting(WAIT_UNFREEZE);
+  }
+}
+
+
+
+
+
+
+
+
diff --git a/ceph/src/mds/CDir.h b/ceph/src/mds/CDir.h
new file mode 100644
index 00000000..f5762e21
--- /dev/null
+++ b/ceph/src/mds/CDir.h
@@ -0,0 +1,632 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_CDIR_H
+#define CEPH_CDIR_H
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "mdstypes.h"
+#include "common/config.h"
+#include "common/DecayCounter.h"
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+using namespace std;
+
+
+#include "CInode.h"
+
+class CDentry;
+class MDCache;
+class MDCluster;
+class Context;
+class bloom_filter;
+
+struct ObjectOperation;
+
+ostream& operator<<(ostream& out, class CDir& dir);
+class CDir : public MDSCacheObject {
+  /*
+   * This class uses a boost::pool to handle allocation. This is *not*
+   * thread-safe, so don't do allocations from multiple threads!
+   *
+   * Alternatively, switch the pool to use a boost::singleton_pool.
+   */
+private:
+  static boost::pool<> pool;
+public:
+  static void *operator new(size_t num_bytes) { 
+    void *n = pool.malloc();
+    if (!n)
+      throw std::bad_alloc();
+    return n;
+  }
+  void operator delete(void *p) {
+    pool.free(p);
+  }
+
+public:
+  // -- pins --
+  static const int PIN_DNWAITER =     1;
+  static const int PIN_INOWAITER =    2;
+  static const int PIN_CHILD =        3;
+  static const int PIN_FROZEN =       4;
+  static const int PIN_SUBTREE =      5;
+  static const int PIN_IMPORTING =    7;
+  static const int PIN_IMPORTBOUND =  9;
+  static const int PIN_EXPORTBOUND = 10;
+  static const int PIN_STICKY =      11;
+  static const int PIN_SUBTREETEMP = 12;  // used by MDCache::trim_non_auth()
+  const char *pin_name(int p) {
+    switch (p) {
+    case PIN_DNWAITER: return "dnwaiter";
+    case PIN_INOWAITER: return "inowaiter";
+    case PIN_CHILD: return "child";
+    case PIN_FROZEN: return "frozen";
+    case PIN_SUBTREE: return "subtree";
+    case PIN_IMPORTING: return "importing";
+    case PIN_IMPORTBOUND: return "importbound";
+    case PIN_EXPORTBOUND: return "exportbound";
+    case PIN_STICKY: return "sticky";
+    case PIN_SUBTREETEMP: return "subtreetemp";
+    default: return generic_pin_name(p);
+    }
+  }
+
+  // -- state --
+  static const unsigned STATE_COMPLETE =      (1<< 1);   // the complete contents are in cache
+  static const unsigned STATE_FROZENTREE =    (1<< 2);   // root of tree (bounded by exports)
+  static const unsigned STATE_FREEZINGTREE =  (1<< 3);   // in process of freezing 
+  static const unsigned STATE_FROZENDIR =     (1<< 4);
+  static const unsigned STATE_FREEZINGDIR =   (1<< 5);
+  static const unsigned STATE_COMMITTING =    (1<< 6);   // mid-commit
+  static const unsigned STATE_FETCHING =      (1<< 7);   // currenting fetching
+  static const unsigned STATE_IMPORTBOUND =   (1<<10);
+  static const unsigned STATE_EXPORTBOUND =   (1<<11);
+  static const unsigned STATE_EXPORTING =     (1<<12);
+  static const unsigned STATE_IMPORTING =     (1<<13);
+  static const unsigned STATE_FRAGMENTING =   (1<<14);
+  static const unsigned STATE_STICKY =        (1<<15);  // sticky pin due to inode stickydirs
+  static const unsigned STATE_DNPINNEDFRAG =  (1<<16);  // dir is refragmenting
+  static const unsigned STATE_ASSIMRSTAT =    (1<<17);  // assimilating inode->frag rstats
+  static const unsigned STATE_DIRTYDFT =      (1<<18);  // dirty dirfragtree
+
+  // common states
+  static const unsigned STATE_CLEAN =  0;
+  static const unsigned STATE_INITIAL = 0;
+
+  // these state bits are preserved by an import/export
+  // ...except if the directory is hashed, in which case none of them are!
+  static const unsigned MASK_STATE_EXPORTED = 
+  (STATE_COMPLETE|STATE_DIRTY|STATE_DIRTYDFT);
+  static const unsigned MASK_STATE_IMPORT_KEPT = 
+  (						  
+   STATE_IMPORTING
+   |STATE_IMPORTBOUND|STATE_EXPORTBOUND
+   |STATE_FROZENTREE
+   |STATE_STICKY);
+  static const unsigned MASK_STATE_EXPORT_KEPT = 
+  (STATE_EXPORTING
+   |STATE_IMPORTBOUND|STATE_EXPORTBOUND
+   |STATE_FROZENTREE
+   |STATE_FROZENDIR
+   |STATE_STICKY);
+  static const unsigned MASK_STATE_FRAGMENT_KEPT = 
+  (STATE_DIRTY|
+   STATE_EXPORTBOUND |
+   STATE_IMPORTBOUND |
+   STATE_REJOINUNDEF);
+
+  // -- rep spec --
+  static const int REP_NONE =     0;
+  static const int REP_ALL =      1;
+  static const int REP_LIST =     2;
+
+
+  static const unsigned EXPORT_NONCE  = 1;
+
+
+  // -- wait masks --
+  static const uint64_t WAIT_DENTRY       = (1<<0);  // wait for item to be in cache
+  static const uint64_t WAIT_COMPLETE     = (1<<1);  // wait for complete dir contents
+  static const uint64_t WAIT_FROZEN       = (1<<2);  // auth pins removed
+
+  static const int WAIT_DNLOCK_OFFSET = 4;
+
+  static const uint64_t WAIT_ANY_MASK = (uint64_t)(-1);
+  static const uint64_t WAIT_ATFREEZEROOT = (WAIT_UNFREEZE);
+  static const uint64_t WAIT_ATSUBTREEROOT = (WAIT_SINGLEAUTH);
+
+
+
+
+ public:
+  // context
+  MDCache  *cache;
+
+  CInode          *inode;  // my inode
+  frag_t           frag;   // my frag
+
+  bool is_lt(const MDSCacheObject *r) const {
+    return dirfrag() < (static_cast(r))->dirfrag();
+  }
+
+  fnode_t fnode;
+  snapid_t first;
+  map dirty_old_rstat;  // [value.first,key]
+
+  // my inodes with dirty rstat data
+  elist dirty_rstat_inodes;     
+
+  void resync_accounted_fragstat();
+  void resync_accounted_rstat();
+  void assimilate_dirty_rstat_inodes();
+  void assimilate_dirty_rstat_inodes_finish(MutationRef& mut, EMetaBlob *blob);
+
+protected:
+  version_t projected_version;
+  list projected_fnode;
+
+public:
+  elist::item item_dirty, item_new;
+
+
+public:
+  version_t get_version() { return fnode.version; }
+  void set_version(version_t v) { 
+    assert(projected_fnode.empty());
+    projected_version = fnode.version = v; 
+  }
+  version_t get_projected_version() { return projected_version; }
+
+  fnode_t *get_projected_fnode() {
+    if (projected_fnode.empty())
+      return &fnode;
+    else
+      return projected_fnode.back();
+  }
+  fnode_t *project_fnode();
+
+  void pop_and_dirty_projected_fnode(LogSegment *ls);
+  bool is_projected() { return !projected_fnode.empty(); }
+  version_t pre_dirty(version_t min=0);
+  void _mark_dirty(LogSegment *ls);
+  void _set_dirty_flag() {
+    if (!state_test(STATE_DIRTY)) {
+      state_set(STATE_DIRTY);
+      get(PIN_DIRTY);
+    }
+  }
+  void mark_dirty(version_t pv, LogSegment *ls);
+  void log_mark_dirty();
+  void mark_clean();
+
+  bool is_new() { return item_new.is_on_list(); }
+  void mark_new(LogSegment *ls);
+
+public:
+  typedef map map_t;
+protected:
+
+  // contents of this directory
+  map_t items;       // non-null AND null
+  unsigned num_head_items;
+  unsigned num_head_null;
+  unsigned num_snap_items;
+  unsigned num_snap_null;
+
+  int num_dirty;
+
+  // state
+  version_t committing_version;
+  version_t committed_version;
+
+
+  // lock nesting, freeze
+  int auth_pins;
+#ifdef MDS_AUTHPIN_SET
+  multiset auth_pin_set;
+#endif
+  int nested_auth_pins, dir_auth_pins;
+  int request_pins;
+
+  int nested_anchors;
+
+  // cache control  (defined for authority; hints for replicas)
+  __s32      dir_rep;
+  set<__s32> dir_rep_by;      // if dir_rep == REP_LIST
+
+  // popularity
+  dirfrag_load_vec_t pop_me;
+  dirfrag_load_vec_t pop_nested;
+  dirfrag_load_vec_t pop_auth_subtree;
+  dirfrag_load_vec_t pop_auth_subtree_nested;
+ 
+  utime_t last_popularity_sample;
+
+  load_spread_t pop_spread;
+
+  // and to provide density
+  int num_dentries_nested;
+  int num_dentries_auth_subtree;
+  int num_dentries_auth_subtree_nested;
+
+
+  // friends
+  friend class Migrator;
+  friend class CInode;
+  friend class MDCache;
+  friend class MDiscover;
+  friend class MDBalancer;
+
+  friend class CDirDiscover;
+  friend class CDirExport;
+  friend class C_Dir_TMAP_Fetched;
+  friend class C_Dir_OMAP_Fetched;
+  friend class C_Dir_Committed;
+
+  bloom_filter *bloom;
+  /* If you set up the bloom filter, you must keep it accurate!
+   * It's deleted when you mark_complete() and is deliberately not serialized.*/
+
+ public:
+  CDir(CInode *in, frag_t fg, MDCache *mdcache, bool auth);
+  ~CDir() {
+    remove_bloom();
+    g_num_dir--;
+    g_num_dirs++;
+  }
+
+
+
+  // -- accessors --
+  inodeno_t ino()     const { return inode->ino(); }          // deprecate me?
+  frag_t    get_frag()    const { return frag; }
+  dirfrag_t dirfrag() const { return dirfrag_t(inode->ino(), frag); }
+
+  CInode *get_inode()    { return inode; }
+  CDir *get_parent_dir() { return inode->get_parent_dir(); }
+
+  map_t::iterator begin() { return items.begin(); }
+  map_t::iterator end() { return items.end(); }
+
+  unsigned get_num_head_items() { return num_head_items; }
+  unsigned get_num_head_null() { return num_head_null; }
+  unsigned get_num_snap_items() { return num_snap_items; }
+  unsigned get_num_snap_null() { return num_snap_null; }
+  unsigned get_num_any() { return num_head_items + num_head_null + num_snap_items + num_snap_null; }
+  
+  bool check_rstats();
+
+  void inc_num_dirty() { num_dirty++; }
+  void dec_num_dirty() { 
+    assert(num_dirty > 0);
+    num_dirty--; 
+  }
+  int get_num_dirty() {
+    return num_dirty;
+  }
+
+  int64_t get_frag_size() { return get_projected_fnode()->fragstat.size(); }
+
+  // -- dentries and inodes --
+ public:
+  CDentry* lookup_exact_snap(const string& dname, snapid_t last) {
+    map_t::iterator p = items.find(dentry_key_t(last, dname.c_str()));
+    if (p == items.end())
+      return NULL;
+    return p->second;
+  }
+  CDentry* lookup(const string& n, snapid_t snap=CEPH_NOSNAP) {
+    return lookup(n.c_str(), snap);
+  }
+  CDentry* lookup(const char *n, snapid_t snap=CEPH_NOSNAP);
+
+  CDentry* add_null_dentry(const string& dname, 
+			   snapid_t first=2, snapid_t last=CEPH_NOSNAP);
+  CDentry* add_primary_dentry(const string& dname, CInode *in, 
+			      snapid_t first=2, snapid_t last=CEPH_NOSNAP);
+  CDentry* add_remote_dentry(const string& dname, inodeno_t ino, unsigned char d_type, 
+			     snapid_t first=2, snapid_t last=CEPH_NOSNAP);
+  void remove_dentry( CDentry *dn );         // delete dentry
+  void link_remote_inode( CDentry *dn, inodeno_t ino, unsigned char d_type);
+  void link_remote_inode( CDentry *dn, CInode *in );
+  void link_primary_inode( CDentry *dn, CInode *in );
+  void unlink_inode( CDentry *dn );
+  void try_remove_unlinked_dn(CDentry *dn);
+
+  void add_to_bloom(CDentry *dn);
+  bool is_in_bloom(const string& name);
+  bool has_bloom() { return (bloom ? true : false); }
+  void remove_bloom();
+private:
+  void link_inode_work( CDentry *dn, CInode *in );
+  void unlink_inode_work( CDentry *dn );
+  void remove_null_dentries();
+  void purge_stale_snap_data(const set& snaps);
+public:
+  void touch_dentries_bottom();
+  bool try_trim_snap_dentry(CDentry *dn, const set& snaps);
+
+
+public:
+  void split(int bits, list& subs, list& waiters, bool replay);
+  void merge(list& subs, list& waiters, bool replay);
+
+  bool should_split() {
+    return (int)get_frag_size() > g_conf->mds_bal_split_size;
+  }
+  bool should_merge() {
+    return (int)get_frag_size() < g_conf->mds_bal_merge_size;
+  }
+
+private:
+  void prepare_new_fragment(bool replay);
+  void prepare_old_fragment(bool replay);
+  void steal_dentry(CDentry *dn);  // from another dir.  used by merge/split.
+  void finish_old_fragment(list& waiters, bool replay);
+  void init_fragment_pins();
+
+
+  // -- authority --
+  /*
+   *     normal:    !subtree_root
+   * delegation:        subtree_root
+   *  ambiguous:          subtree_root
+   *                    subtree_root     
+   */
+  pair dir_auth;
+
+ public:
+  pair authority();
+  pair get_dir_auth() { return dir_auth; }
+  void set_dir_auth(pair a);
+  void set_dir_auth(int a) { set_dir_auth(pair(a, CDIR_AUTH_UNKNOWN)); }
+  bool is_ambiguous_dir_auth() {
+    return dir_auth.second != CDIR_AUTH_UNKNOWN;
+  }
+  bool is_full_dir_auth() {
+    return is_auth() && !is_ambiguous_dir_auth();
+  }
+  bool is_full_dir_nonauth() {
+    return !is_auth() && !is_ambiguous_dir_auth();
+  }
+  
+  bool is_subtree_root() {
+    return dir_auth != CDIR_AUTH_DEFAULT;
+  }
+
+  bool contains(CDir *x);  // true if we are x or an ancestor of x 
+
+
+  // for giving to clients
+  void get_dist_spec(set& ls, int auth) {
+    if (is_rep()) {
+      for (map::iterator p = replicas_begin();
+	   p != replicas_end(); 
+	   ++p)
+	ls.insert(p->first);
+      if (!ls.empty()) 
+	ls.insert(auth);
+    }
+  }
+  void encode_dirstat(bufferlist& bl, int whoami) {
+    /*
+     * note: encoding matches struct ceph_client_reply_dirfrag
+     */
+    frag_t frag = get_frag();
+    __s32 auth;
+    set<__s32> dist;
+    
+    auth = dir_auth.first;
+    if (is_auth()) 
+      get_dist_spec(dist, whoami);
+
+    ::encode(frag, bl);
+    ::encode(auth, bl);
+    ::encode(dist, bl);
+  }
+
+  void _encode_base(bufferlist& bl) {
+    ::encode(first, bl);
+    ::encode(fnode, bl);
+    ::encode(dir_rep, bl);
+    ::encode(dir_rep_by, bl);
+  }
+  void _decode_base(bufferlist::iterator& p) {
+    ::decode(first, p);
+    ::decode(fnode, p);
+    ::decode(dir_rep, p);
+    ::decode(dir_rep_by, p);
+  }
+  void encode_replica(int who, bufferlist& bl) {
+    __u32 nonce = add_replica(who);
+    ::encode(nonce, bl);
+    _encode_base(bl);
+  }
+  void decode_replica(bufferlist::iterator& p) {
+    __u32 nonce;
+    ::decode(nonce, p);
+    replica_nonce = nonce;
+    _decode_base(p);
+  }
+
+
+
+  // -- state --
+  bool is_complete() { return state & STATE_COMPLETE; }
+  bool is_exporting() { return state & STATE_EXPORTING; }
+  bool is_importing() { return state & STATE_IMPORTING; }
+  bool is_dirty_dft() { return state & STATE_DIRTYDFT; }
+
+  int get_dir_rep() { return dir_rep; }
+  bool is_rep() { 
+    if (dir_rep == REP_NONE) return false;
+    return true;
+  }
+ 
+  // -- fetch --
+  object_t get_ondisk_object() { 
+    return file_object_t(ino(), frag);
+  }
+  void fetch(Context *c, bool ignore_authpinnability=false);
+  void fetch(Context *c, const string& want_dn, bool ignore_authpinnability=false);
+protected:
+  void _omap_fetch(const string& want_dn);
+  void _omap_fetched(bufferlist& hdrbl, map& omap,
+		     const string& want_dn, int r);
+  void _tmap_fetch(const string& want_dn);
+  void _tmap_fetched(bufferlist &bl, const string& want_dn, int r);
+
+  // -- commit --
+  map > waiting_for_commit;
+  void _commit(version_t want, int op_prio);
+  void _omap_commit(int op_prio);
+  void _encode_dentry(CDentry *dn, bufferlist& bl, const set *snaps);
+  void _committed(version_t v);
+public:
+  void wait_for_commit(Context *c, version_t v=0);
+  void commit_to(version_t want);
+  void commit(version_t want, Context *c,
+	      bool ignore_authpinnability=false, int op_prio=-1);
+
+  // -- dirtyness --
+  version_t get_committing_version() { return committing_version; }
+  version_t get_committed_version() { return committed_version; }
+  void set_committed_version(version_t v) { committed_version = v; }
+
+  void mark_complete();
+
+
+  // -- reference counting --
+  void first_get();
+  void last_put();
+
+  void request_pin_get() {
+    if (request_pins == 0) get(PIN_REQUEST);
+    request_pins++;
+  }
+  void request_pin_put() {
+    request_pins--;
+    if (request_pins == 0) put(PIN_REQUEST);
+  }
+
+    
+  // -- waiters --
+protected:
+  map< string_snap_t, list > waiting_on_dentry;
+  map< inodeno_t, list > waiting_on_ino;
+
+public:
+  bool is_waiting_for_dentry(const string& dname, snapid_t snap) {
+    return waiting_on_dentry.count(string_snap_t(dname, snap));
+  }
+  void add_dentry_waiter(const string& dentry, snapid_t snap, Context *c);
+  void take_dentry_waiting(const string& dentry, snapid_t first, snapid_t last, list& ls);
+
+  bool is_waiting_for_ino(inodeno_t ino) {
+    return waiting_on_ino.count(ino);
+  }
+  void add_ino_waiter(inodeno_t ino, Context *c);
+  void take_ino_waiting(inodeno_t ino, list& ls);
+
+  void take_sub_waiting(list& ls);  // dentry or ino
+
+  void add_waiter(uint64_t mask, Context *c);
+  void take_waiting(uint64_t mask, list& ls);  // may include dentry waiters
+  void finish_waiting(uint64_t mask, int result = 0);    // ditto
+  
+
+  // -- import/export --
+  void encode_export(bufferlist& bl);
+  void finish_export(utime_t now);
+  void abort_export() {
+    put(PIN_TEMPEXPORTING);
+  }
+  void decode_import(bufferlist::iterator& blp, utime_t now, LogSegment *ls);
+
+
+  // -- auth pins --
+  bool can_auth_pin() { return is_auth() && !(is_frozen() || is_freezing()); }
+  int get_cum_auth_pins() { return auth_pins + nested_auth_pins; }
+  int get_auth_pins() { return auth_pins; }
+  int get_nested_auth_pins() { return nested_auth_pins; }
+  int get_dir_auth_pins() { return dir_auth_pins; }
+  void auth_pin(void *who);
+  void auth_unpin(void *who);
+
+  void adjust_nested_auth_pins(int inc, int dirinc, void *by);
+  void verify_fragstat();
+
+  int get_nested_anchors() { return nested_anchors; }
+  void adjust_nested_anchors(int by);
+
+  // -- freezing --
+  bool freeze_tree();
+  void _freeze_tree();
+  void unfreeze_tree();
+
+  bool freeze_dir();
+  void _freeze_dir();
+  void unfreeze_dir();
+
+  void maybe_finish_freeze();
+
+  bool is_freezing() { return is_freezing_tree() || is_freezing_dir(); }
+  bool is_freezing_tree();
+  bool is_freezing_tree_root() { return state & STATE_FREEZINGTREE; }
+  bool is_freezing_dir() { return state & STATE_FREEZINGDIR; }
+
+  bool is_frozen() { return is_frozen_dir() || is_frozen_tree(); }
+  bool is_frozen_tree();
+  bool is_frozen_tree_root() { return state & STATE_FROZENTREE; }
+  bool is_frozen_dir() { return state & STATE_FROZENDIR; }
+  
+  bool is_freezeable(bool freezing=false) {
+    // no nested auth pins.
+    if ((auth_pins-freezing) > 0 || nested_auth_pins > 0) 
+      return false;
+
+    // inode must not be frozen.
+    if (!is_subtree_root() && inode->is_frozen())
+      return false;
+
+    return true;
+  }
+  bool is_freezeable_dir(bool freezing=false) {
+    if ((auth_pins-freezing) > 0 || dir_auth_pins > 0) 
+      return false;
+
+    // if not subtree root, inode must not be frozen (tree--frozen_dir is okay).
+    if (!is_subtree_root() && inode->is_frozen() && !inode->is_frozen_dir())
+      return false;
+
+    return true;
+  }
+
+  CDir *get_frozen_tree_root();
+
+
+  ostream& print_db_line_prefix(ostream& out);
+  void print(ostream& out);
+};
+
+#endif
diff --git a/ceph/src/mds/CInode.cc b/ceph/src/mds/CInode.cc
new file mode 100644
index 00000000..53a0f443
--- /dev/null
+++ b/ceph/src/mds/CInode.cc
@@ -0,0 +1,3374 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "include/int_types.h"
+
+#include 
+#include 
+
+#include "CInode.h"
+#include "CDir.h"
+#include "CDentry.h"
+
+#include "MDS.h"
+#include "MDCache.h"
+#include "MDLog.h"
+#include "Locker.h"
+#include "Mutation.h"
+
+#include "events/EUpdate.h"
+
+#include "osdc/Objecter.h"
+
+#include "snap.h"
+
+#include "LogSegment.h"
+
+#include "common/Clock.h"
+
+#include "messages/MLock.h"
+#include "messages/MClientCaps.h"
+
+#include "common/config.h"
+#include "global/global_context.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mdcache->mds->get_nodeid() << ".cache.ino(" << inode.ino << ") "
+
+
+boost::pool<> CInode::pool(sizeof(CInode));
+boost::pool<> Capability::pool(sizeof(Capability));
+
+LockType CInode::versionlock_type(CEPH_LOCK_IVERSION);
+LockType CInode::authlock_type(CEPH_LOCK_IAUTH);
+LockType CInode::linklock_type(CEPH_LOCK_ILINK);
+LockType CInode::dirfragtreelock_type(CEPH_LOCK_IDFT);
+LockType CInode::filelock_type(CEPH_LOCK_IFILE);
+LockType CInode::xattrlock_type(CEPH_LOCK_IXATTR);
+LockType CInode::snaplock_type(CEPH_LOCK_ISNAP);
+LockType CInode::nestlock_type(CEPH_LOCK_INEST);
+LockType CInode::flocklock_type(CEPH_LOCK_IFLOCK);
+LockType CInode::policylock_type(CEPH_LOCK_IPOLICY);
+
+//int cinode_pins[CINODE_NUM_PINS];  // counts
+ostream& CInode::print_db_line_prefix(ostream& out)
+{
+  return out << ceph_clock_now(g_ceph_context) << " mds." << mdcache->mds->get_nodeid() << ".cache.ino(" << inode.ino << ") ";
+}
+
+/*
+ * write caps and lock ids
+ */
+struct cinode_lock_info_t cinode_lock_info[] = {
+  { CEPH_LOCK_IFILE, CEPH_CAP_ANY_FILE_WR },
+  { CEPH_LOCK_IAUTH, CEPH_CAP_AUTH_EXCL },
+  { CEPH_LOCK_ILINK, CEPH_CAP_LINK_EXCL },
+  { CEPH_LOCK_IXATTR, CEPH_CAP_XATTR_EXCL },
+  { CEPH_LOCK_IFLOCK, CEPH_CAP_FLOCK_EXCL }  
+};
+int num_cinode_locks = 5;
+
+
+
+ostream& operator<<(ostream& out, CInode& in)
+{
+  string path;
+  in.make_path_string_projected(path);
+
+  out << "[inode " << in.inode.ino;
+  out << " [" 
+      << (in.is_multiversion() ? "...":"")
+      << in.first << "," << in.last << "]";
+  out << " " << path << (in.is_dir() ? "/":"");
+
+  if (in.is_auth()) {
+    out << " auth";
+    if (in.is_replicated()) 
+      out << in.get_replicas();
+  } else {
+    pair a = in.authority();
+    out << " rep@" << a.first;
+    if (a.second != CDIR_AUTH_UNKNOWN)
+      out << "," << a.second;
+    out << "." << in.get_replica_nonce();
+  }
+
+  if (in.is_symlink())
+    out << " symlink='" << in.symlink << "'";
+  if (in.is_dir() && !in.dirfragtree.empty())
+    out << " " << in.dirfragtree;
+  
+  out << " v" << in.get_version();
+  if (in.get_projected_version() > in.get_version())
+    out << " pv" << in.get_projected_version();
+
+  if (in.is_auth_pinned()) {
+    out << " ap=" << in.get_num_auth_pins() << "+" << in.get_num_nested_auth_pins();
+#ifdef MDS_AUTHPIN_SET
+    out << "(" << in.auth_pin_set << ")";
+#endif
+  }
+
+  if (in.snaprealm)
+    out << " snaprealm=" << in.snaprealm;
+
+  if (in.state_test(CInode::STATE_AMBIGUOUSAUTH)) out << " AMBIGAUTH";
+  if (in.state_test(CInode::STATE_NEEDSRECOVER)) out << " needsrecover";
+  if (in.state_test(CInode::STATE_RECOVERING)) out << " recovering";
+  if (in.state_test(CInode::STATE_DIRTYPARENT)) out << " dirtyparent";
+  if (in.is_freezing_inode()) out << " FREEZING=" << in.auth_pin_freeze_allowance;
+  if (in.is_frozen_inode()) out << " FROZEN";
+  if (in.is_frozen_auth_pin()) out << " FROZEN_AUTHPIN";
+
+  inode_t *pi = in.get_projected_inode();
+  if (pi->is_truncating())
+    out << " truncating(" << pi->truncate_from << " to " << pi->truncate_size << ")";
+
+  // anchors
+  if (in.is_anchored())
+    out << " anc";
+  if (in.get_nested_anchors())
+    out << " na=" << in.get_nested_anchors();
+
+  if (in.inode.is_dir()) {
+    out << " " << in.inode.dirstat;
+    if (g_conf->mds_debug_scatterstat && in.is_projected()) {
+      inode_t *pi = in.get_projected_inode();
+      out << "->" << pi->dirstat;
+    }
+  } else {
+    out << " s=" << in.inode.size;
+    if (in.inode.nlink != 1)
+      out << " nl=" << in.inode.nlink;
+  }
+
+  // rstat
+  out << " " << in.inode.rstat;
+  if (!(in.inode.rstat == in.inode.accounted_rstat))
+    out << "/" << in.inode.accounted_rstat;
+  if (g_conf->mds_debug_scatterstat && in.is_projected()) {
+    inode_t *pi = in.get_projected_inode();
+    out << "->" << pi->rstat;
+    if (!(pi->rstat == pi->accounted_rstat))
+      out << "/" << pi->accounted_rstat;
+  }
+
+  if (!in.client_need_snapflush.empty())
+    out << " need_snapflush=" << in.client_need_snapflush;
+
+
+  // locks
+  if (!in.authlock.is_sync_and_unlocked())
+    out << " " << in.authlock;
+  if (!in.linklock.is_sync_and_unlocked())
+    out << " " << in.linklock;
+  if (in.inode.is_dir()) {
+    if (!in.dirfragtreelock.is_sync_and_unlocked())
+      out << " " << in.dirfragtreelock;
+    if (!in.snaplock.is_sync_and_unlocked())
+      out << " " << in.snaplock;
+    if (!in.nestlock.is_sync_and_unlocked())
+      out << " " << in.nestlock;
+    if (!in.policylock.is_sync_and_unlocked())
+      out << " " << in.policylock;
+  } else  {
+    if (!in.flocklock.is_sync_and_unlocked())
+      out << " " << in.flocklock;
+  }
+  if (!in.filelock.is_sync_and_unlocked())
+    out << " " << in.filelock;
+  if (!in.xattrlock.is_sync_and_unlocked())
+    out << " " << in.xattrlock;
+  if (!in.versionlock.is_sync_and_unlocked())  
+    out << " " << in.versionlock;
+
+  // hack: spit out crap on which clients have caps
+  if (in.inode.client_ranges.size())
+    out << " cr=" << in.inode.client_ranges;
+
+  if (!in.get_client_caps().empty()) {
+    out << " caps={";
+    for (map::iterator it = in.get_client_caps().begin();
+         it != in.get_client_caps().end();
+         ++it) {
+      if (it != in.get_client_caps().begin()) out << ",";
+      out << it->first << "="
+	  << ccap_string(it->second->pending());
+      if (it->second->issued() != it->second->pending())
+	out << "/" << ccap_string(it->second->issued());
+      out << "/" << ccap_string(it->second->wanted())
+	  << "@" << it->second->get_last_sent();
+    }
+    out << "}";
+    if (in.get_loner() >= 0 || in.get_wanted_loner() >= 0) {
+      out << ",l=" << in.get_loner();
+      if (in.get_loner() != in.get_wanted_loner())
+	out << "(" << in.get_wanted_loner() << ")";
+    }
+  }
+  if (!in.get_mds_caps_wanted().empty()) {
+    out << " mcw={";
+    for (map::iterator p = in.get_mds_caps_wanted().begin();
+	 p != in.get_mds_caps_wanted().end(); ++p) {
+      if (p != in.get_mds_caps_wanted().begin())
+	out << ',';
+      out << p->first << '=' << ccap_string(p->second);
+    }
+    out << '}';
+  }
+
+  if (in.get_num_ref()) {
+    out << " |";
+    in.print_pin_set(out);
+  }
+
+  out << " " << ∈
+  out << "]";
+  return out;
+}
+
+
+void CInode::print(ostream& out)
+{
+  out << *this;
+}
+
+
+
+void CInode::add_need_snapflush(CInode *snapin, snapid_t snapid, client_t client)
+{
+  dout(10) << "add_need_snapflush client." << client << " snapid " << snapid << " on " << snapin << dendl;
+
+  if (client_need_snapflush.empty()) {
+    get(CInode::PIN_NEEDSNAPFLUSH);
+
+    // FIXME: this is non-optimal, as we'll block freezes/migrations for potentially
+    // long periods waiting for clients to flush their snaps.
+    auth_pin(this);   // pin head inode...
+  }
+
+  set& clients = client_need_snapflush[snapid];
+  if (clients.empty())
+    snapin->auth_pin(this);  // ...and pin snapped/old inode!
+  
+  clients.insert(client);
+}
+
+void CInode::remove_need_snapflush(CInode *snapin, snapid_t snapid, client_t client)
+{
+  dout(10) << "remove_need_snapflush client." << client << " snapid " << snapid << " on " << snapin << dendl;
+  set& clients = client_need_snapflush[snapid];
+  clients.erase(client);
+  if (clients.empty()) {
+    client_need_snapflush.erase(snapid);
+    snapin->auth_unpin(this);
+
+    if (client_need_snapflush.empty()) {
+      put(CInode::PIN_NEEDSNAPFLUSH);
+      auth_unpin(this);
+    }
+  }
+}
+
+
+
+void CInode::mark_dirty_rstat()
+{
+  if (!state_test(STATE_DIRTYRSTAT)) {
+    dout(10) << "mark_dirty_rstat" << dendl;
+    state_set(STATE_DIRTYRSTAT);
+    get(PIN_DIRTYRSTAT);
+    CDentry *dn = get_projected_parent_dn();
+    CDir *pdir = dn->dir;
+    pdir->dirty_rstat_inodes.push_back(&dirty_rstat_item);
+
+    mdcache->mds->locker->mark_updated_scatterlock(&pdir->inode->nestlock);
+  }
+}
+void CInode::clear_dirty_rstat()
+{
+  if (state_test(STATE_DIRTYRSTAT)) {
+    dout(10) << "clear_dirty_rstat" << dendl;
+    state_clear(STATE_DIRTYRSTAT);
+    put(PIN_DIRTYRSTAT);
+    dirty_rstat_item.remove_myself();
+  }
+}
+
+inode_t *CInode::project_inode(map *px) 
+{
+  if (projected_nodes.empty()) {
+    projected_nodes.push_back(new projected_inode_t(new inode_t(inode)));
+    if (px)
+      *px = xattrs;
+  } else {
+    projected_nodes.push_back(new projected_inode_t(
+        new inode_t(*projected_nodes.back()->inode)));
+    if (px)
+      *px = *get_projected_xattrs();
+  }
+  projected_nodes.back()->xattrs = px;
+  dout(15) << "project_inode " << projected_nodes.back()->inode << dendl;
+  return projected_nodes.back()->inode;
+}
+
+void CInode::pop_and_dirty_projected_inode(LogSegment *ls) 
+{
+  assert(!projected_nodes.empty());
+  dout(15) << "pop_and_dirty_projected_inode " << projected_nodes.front()->inode
+	   << " v" << projected_nodes.front()->inode->version << dendl;
+  int64_t old_pool = inode.layout.fl_pg_pool;
+
+  mark_dirty(projected_nodes.front()->inode->version, ls);
+  inode = *projected_nodes.front()->inode;
+
+  if (inode.is_backtrace_updated())
+    _mark_dirty_parent(ls, old_pool != inode.layout.fl_pg_pool);
+
+  map *px = projected_nodes.front()->xattrs;
+  if (px) {
+    xattrs = *px;
+    delete px;
+  }
+
+  if (projected_nodes.front()->snapnode)
+    pop_projected_snaprealm(projected_nodes.front()->snapnode);
+
+  delete projected_nodes.front()->inode;
+  delete projected_nodes.front();
+
+  projected_nodes.pop_front();
+}
+
+sr_t *CInode::project_snaprealm(snapid_t snapid)
+{
+  sr_t *cur_srnode = get_projected_srnode();
+  sr_t *new_srnode;
+
+  if (cur_srnode) {
+    new_srnode = new sr_t(*cur_srnode);
+  } else {
+    new_srnode = new sr_t();
+    new_srnode->created = snapid;
+    new_srnode->current_parent_since = snapid;
+  }
+  dout(10) << "project_snaprealm " << new_srnode << dendl;
+  projected_nodes.back()->snapnode = new_srnode;
+  return new_srnode;
+}
+
+/* if newparent != parent, add parent to past_parents
+ if parent DNE, we need to find what the parent actually is and fill that in */
+void CInode::project_past_snaprealm_parent(SnapRealm *newparent)
+{
+  sr_t *new_snap = project_snaprealm();
+  SnapRealm *oldparent;
+  if (!snaprealm) {
+    oldparent = find_snaprealm();
+    new_snap->seq = oldparent->get_newest_seq();
+  }
+  else
+    oldparent = snaprealm->parent;
+
+  if (newparent != oldparent) {
+    snapid_t oldparentseq = oldparent->get_newest_seq();
+    if (oldparentseq + 1 > new_snap->current_parent_since) {
+      new_snap->past_parents[oldparentseq].ino = oldparent->inode->ino();
+      new_snap->past_parents[oldparentseq].first = new_snap->current_parent_since;
+    }
+    new_snap->current_parent_since = MAX(oldparentseq, newparent->get_last_created()) + 1;
+  }
+}
+
+void CInode::pop_projected_snaprealm(sr_t *next_snaprealm)
+{
+  assert(next_snaprealm);
+  dout(10) << "pop_projected_snaprealm " << next_snaprealm
+          << " seq" << next_snaprealm->seq << dendl;
+  bool invalidate_cached_snaps = false;
+  if (!snaprealm) {
+    open_snaprealm();
+  } else if (next_snaprealm->past_parents.size() !=
+	     snaprealm->srnode.past_parents.size()) {
+    invalidate_cached_snaps = true;
+
+    // update parent pointer
+    assert(snaprealm->open);
+    assert(snaprealm->parent);   // had a parent before
+    SnapRealm *new_parent = get_parent_inode()->find_snaprealm();
+    assert(new_parent);
+    CInode *parenti = new_parent->inode;
+    assert(parenti);
+    assert(parenti->snaprealm);
+    snaprealm->parent = new_parent;
+    snaprealm->add_open_past_parent(new_parent);
+    dout(10) << " realm " << *snaprealm << " past_parents " << snaprealm->srnode.past_parents
+	     << " -> " << next_snaprealm->past_parents << dendl;
+    dout(10) << " pinning new parent " << *parenti << dendl;
+  }
+  snaprealm->srnode = *next_snaprealm;
+  delete next_snaprealm;
+
+  // we should be able to open these up (or have them already be open).
+  bool ok = snaprealm->_open_parents(NULL);
+  assert(ok);
+
+  if (invalidate_cached_snaps)
+    snaprealm->invalidate_cached_snaps();
+
+  if (snaprealm->parent)
+    dout(10) << " realm " << *snaprealm << " parent " << *snaprealm->parent << dendl;
+}
+
+
+// ====== CInode =======
+
+// dirfrags
+
+__u32 CInode::hash_dentry_name(const string &dn)
+{
+  int which = inode.dir_layout.dl_dir_hash;
+  if (!which)
+    which = CEPH_STR_HASH_LINUX;
+  return ceph_str_hash(which, dn.data(), dn.length());
+}
+
+frag_t CInode::pick_dirfrag(const string& dn)
+{
+  if (dirfragtree.empty())
+    return frag_t();          // avoid the string hash if we can.
+
+  __u32 h = hash_dentry_name(dn);
+  return dirfragtree[h];
+}
+
+bool CInode::get_dirfrags_under(frag_t fg, list& ls)
+{
+  bool all = true;
+  list fglist;
+  dirfragtree.get_leaves_under(fg, fglist);
+  for (list::iterator p = fglist.begin(); p != fglist.end(); ++p)
+    if (dirfrags.count(*p))
+      ls.push_back(dirfrags[*p]);
+    else 
+      all = false;
+
+  if (all)
+    return all;
+
+  fragtree_t tmpdft;
+  tmpdft.force_to_leaf(g_ceph_context, fg);
+  for (map::iterator p = dirfrags.begin(); p != dirfrags.end(); ++p) {
+    tmpdft.force_to_leaf(g_ceph_context, p->first);
+    if (fg.contains(p->first) && !dirfragtree.is_leaf(p->first))
+      ls.push_back(p->second);
+  }
+
+  all = true;
+  tmpdft.get_leaves_under(fg, fglist);
+  for (list::iterator p = fglist.begin(); p != fglist.end(); ++p)
+    if (!dirfrags.count(*p)) {
+      all = false;
+      break;
+    }
+
+  return all;
+}
+
+void CInode::verify_dirfrags()
+{
+  bool bad = false;
+  for (map::iterator p = dirfrags.begin(); p != dirfrags.end(); ++p) {
+    if (!dirfragtree.is_leaf(p->first)) {
+      dout(0) << "have open dirfrag " << p->first << " but not leaf in " << dirfragtree
+	      << ": " << *p->second << dendl;
+      bad = true;
+    }
+  }
+  assert(!bad);
+}
+
+void CInode::force_dirfrags()
+{
+  bool bad = false;
+  for (map::iterator p = dirfrags.begin(); p != dirfrags.end(); ++p) {
+    if (!dirfragtree.is_leaf(p->first)) {
+      dout(0) << "have open dirfrag " << p->first << " but not leaf in " << dirfragtree
+	      << ": " << *p->second << dendl;
+      bad = true;
+    }
+  }
+
+  if (bad) {
+    list leaves;
+    dirfragtree.get_leaves(leaves);
+    for (list::iterator p = leaves.begin(); p != leaves.end(); ++p)
+      mdcache->get_force_dirfrag(dirfrag_t(ino(),*p));
+  }
+
+  verify_dirfrags();
+}
+
+CDir *CInode::get_approx_dirfrag(frag_t fg)
+{
+  CDir *dir = get_dirfrag(fg);
+  if (dir) return dir;
+
+  // find a child?
+  list ls;
+  get_dirfrags_under(fg, ls);
+  if (!ls.empty()) 
+    return ls.front();
+
+  // try parents?
+  while (fg.bits() > 0) {
+    fg = fg.parent();
+    dir = get_dirfrag(fg);
+    if (dir) return dir;
+  }
+  return NULL;
+}	
+
+void CInode::get_dirfrags(list& ls) 
+{
+  // all dirfrags
+  for (map::iterator p = dirfrags.begin();
+       p != dirfrags.end();
+       ++p)
+    ls.push_back(p->second);
+}
+void CInode::get_nested_dirfrags(list& ls) 
+{  
+  // dirfrags in same subtree
+  for (map::iterator p = dirfrags.begin();
+       p != dirfrags.end();
+       ++p)
+    if (!p->second->is_subtree_root())
+      ls.push_back(p->second);
+}
+void CInode::get_subtree_dirfrags(list& ls) 
+{ 
+  // dirfrags that are roots of new subtrees
+  for (map::iterator p = dirfrags.begin();
+       p != dirfrags.end();
+       ++p)
+    if (p->second->is_subtree_root())
+      ls.push_back(p->second);
+}
+
+
+CDir *CInode::get_or_open_dirfrag(MDCache *mdcache, frag_t fg)
+{
+  assert(is_dir());
+
+  // have it?
+  CDir *dir = get_dirfrag(fg);
+  if (!dir) {
+    // create it.
+    assert(is_auth());
+    dir = new CDir(this, fg, mdcache, true);
+    add_dirfrag(dir);
+  }
+  return dir;
+}
+
+CDir *CInode::add_dirfrag(CDir *dir)
+{
+  assert(dirfrags.count(dir->dirfrag().frag) == 0);
+  dirfrags[dir->dirfrag().frag] = dir;
+
+  if (stickydir_ref > 0) {
+    dir->state_set(CDir::STATE_STICKY);
+    dir->get(CDir::PIN_STICKY);
+  }
+
+  return dir;
+}
+
+void CInode::close_dirfrag(frag_t fg)
+{
+  dout(14) << "close_dirfrag " << fg << dendl;
+  assert(dirfrags.count(fg));
+  
+  CDir *dir = dirfrags[fg];
+  dir->remove_null_dentries();
+  
+  // clear dirty flag
+  if (dir->is_dirty())
+    dir->mark_clean();
+  
+  if (stickydir_ref > 0) {
+    dir->state_clear(CDir::STATE_STICKY);
+    dir->put(CDir::PIN_STICKY);
+  }
+  
+  // dump any remaining dentries, for debugging purposes
+  for (CDir::map_t::iterator p = dir->items.begin();
+       p != dir->items.end();
+       ++p) 
+    dout(14) << "close_dirfrag LEFTOVER dn " << *p->second << dendl;
+
+  assert(dir->get_num_ref() == 0);
+  delete dir;
+  dirfrags.erase(fg);
+}
+
+void CInode::close_dirfrags()
+{
+  while (!dirfrags.empty()) 
+    close_dirfrag(dirfrags.begin()->first);
+}
+
+bool CInode::has_subtree_root_dirfrag(int auth)
+{
+  for (map::iterator p = dirfrags.begin();
+       p != dirfrags.end();
+       ++p)
+    if (p->second->is_subtree_root() &&
+	(auth == -1 || p->second->dir_auth.first == auth))
+      return true;
+  return false;
+}
+
+bool CInode::has_subtree_or_exporting_dirfrag()
+{
+  for (map::iterator p = dirfrags.begin();
+       p != dirfrags.end();
+       ++p)
+    if (p->second->is_subtree_root() ||
+	p->second->state_test(CDir::STATE_EXPORTING))
+      return true;
+  return false;
+}
+
+void CInode::get_stickydirs()
+{
+  if (stickydir_ref == 0) {
+    get(PIN_STICKYDIRS);
+    for (map::iterator p = dirfrags.begin();
+	 p != dirfrags.end();
+	 ++p) {
+      p->second->state_set(CDir::STATE_STICKY);
+      p->second->get(CDir::PIN_STICKY);
+    }
+  }
+  stickydir_ref++;
+}
+
+void CInode::put_stickydirs()
+{
+  assert(stickydir_ref > 0);
+  stickydir_ref--;
+  if (stickydir_ref == 0) {
+    put(PIN_STICKYDIRS);
+    for (map::iterator p = dirfrags.begin();
+	 p != dirfrags.end();
+	 ++p) {
+      p->second->state_clear(CDir::STATE_STICKY);
+      p->second->put(CDir::PIN_STICKY);
+    }
+  }
+}
+
+
+
+
+
+// pins
+
+void CInode::first_get()
+{
+  // pin my dentry?
+  if (parent) 
+    parent->get(CDentry::PIN_INODEPIN);
+}
+
+void CInode::last_put() 
+{
+  // unpin my dentry?
+  if (parent) 
+    parent->put(CDentry::PIN_INODEPIN);
+}
+
+void CInode::_put()
+{
+  if (get_num_ref() == (int)is_dirty() + (int)is_dirty_parent())
+    mdcache->maybe_eval_stray(this, true);
+}
+
+void CInode::add_remote_parent(CDentry *p) 
+{
+  if (remote_parents.empty())
+    get(PIN_REMOTEPARENT);
+  remote_parents.insert(p);
+}
+void CInode::remove_remote_parent(CDentry *p) 
+{
+  remote_parents.erase(p);
+  if (remote_parents.empty())
+    put(PIN_REMOTEPARENT);
+}
+
+
+
+
+CDir *CInode::get_parent_dir()
+{
+  if (parent)
+    return parent->dir;
+  return NULL;
+}
+CDir *CInode::get_projected_parent_dir()
+{
+  CDentry *p = get_projected_parent_dn();
+  if (p)
+    return p->dir;
+  return NULL;
+}
+CInode *CInode::get_parent_inode() 
+{
+  if (parent) 
+    return parent->dir->inode;
+  return NULL;
+}
+
+bool CInode::is_projected_ancestor_of(CInode *other)
+{
+  while (other) {
+    if (other == this)
+      return true;
+    if (!other->get_projected_parent_dn())
+      break;
+    other = other->get_projected_parent_dn()->get_dir()->get_inode();
+  }
+  return false;
+}
+
+void CInode::make_path_string(string& s, bool force, CDentry *use_parent)
+{
+  if (!force)
+    use_parent = parent;
+
+  if (use_parent) {
+    use_parent->make_path_string(s);
+  } 
+  else if (is_root()) {
+    s = "";  // root
+  } 
+  else if (is_mdsdir()) {
+    char t[40];
+    uint64_t eino(ino());
+    eino -= MDS_INO_MDSDIR_OFFSET;
+    snprintf(t, sizeof(t), "~mds%" PRId64, eino);
+    s = t;
+  }
+  else {
+    char n[40];
+    uint64_t eino(ino());
+    snprintf(n, sizeof(n), "#%" PRIx64, eino);
+    s += n;
+  }
+}
+void CInode::make_path_string_projected(string& s)
+{
+  make_path_string(s);
+  
+  if (!projected_parent.empty()) {
+    string q;
+    q.swap(s);
+    s = "{" + q;
+    for (list::iterator p = projected_parent.begin();
+	 p != projected_parent.end();
+	 ++p) {
+      string q;
+      make_path_string(q, true, *p);
+      s += " ";
+      s += q;
+    }
+    s += "}";
+  }
+}
+
+void CInode::make_path(filepath& fp)
+{
+  if (parent) 
+    parent->make_path(fp);
+  else
+    fp = filepath(ino());
+}
+
+void CInode::make_anchor_trace(vector& trace)
+{
+  if (get_projected_parent_dn())
+    get_projected_parent_dn()->make_anchor_trace(trace, this);
+  else 
+    assert(is_base());
+}
+
+void CInode::name_stray_dentry(string& dname)
+{
+  char s[20];
+  snprintf(s, sizeof(s), "%llx", (unsigned long long)inode.ino.val);
+  dname = s;
+}
+
+
+
+
+version_t CInode::pre_dirty()
+{
+  version_t pv; 
+  if (parent || !projected_parent.empty()) {
+    pv = get_projected_parent_dn()->pre_dirty(get_projected_version());
+    dout(10) << "pre_dirty " << pv << " (current v " << inode.version << ")" << dendl;
+  } else {
+    assert(is_base());
+    pv = get_projected_version() + 1;
+  }
+  // force update backtrace for old format inode (see inode_t::decode)
+  if (inode.backtrace_version == 0 && !projected_nodes.empty()) {
+    inode_t *pi = projected_nodes.back()->inode;
+    if (pi->backtrace_version == 0)
+      pi->update_backtrace(pv);
+  }
+  return pv;
+}
+
+void CInode::_mark_dirty(LogSegment *ls)
+{
+  if (!state_test(STATE_DIRTY)) {
+    state_set(STATE_DIRTY);
+    get(PIN_DIRTY);
+    assert(ls);
+  }
+  
+  // move myself to this segment's dirty list
+  if (ls) 
+    ls->dirty_inodes.push_back(&item_dirty);
+}
+
+void CInode::mark_dirty(version_t pv, LogSegment *ls) {
+  
+  dout(10) << "mark_dirty " << *this << dendl;
+
+  /*
+    NOTE: I may already be dirty, but this fn _still_ needs to be called so that
+    the directory is (perhaps newly) dirtied, and so that parent_dir_version is 
+    updated below.
+  */
+  
+  // only auth can get dirty.  "dirty" async data in replicas is relative to
+  // filelock state, not the dirty flag.
+  assert(is_auth());
+  
+  // touch my private version
+  assert(inode.version < pv);
+  inode.version = pv;
+  _mark_dirty(ls);
+
+  // mark dentry too
+  if (parent)
+    parent->mark_dirty(pv, ls);
+}
+
+
+void CInode::mark_clean()
+{
+  dout(10) << " mark_clean " << *this << dendl;
+  if (state_test(STATE_DIRTY)) {
+    state_clear(STATE_DIRTY);
+    put(PIN_DIRTY);
+    
+    // remove myself from ls dirty list
+    item_dirty.remove_myself();
+  }
+}    
+
+
+// --------------
+// per-inode storage
+// (currently for root inode only)
+
+struct C_Inode_Stored : public Context {
+  CInode *in;
+  version_t version;
+  Context *fin;
+  C_Inode_Stored(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
+  void finish(int r) {
+    assert(r == 0);
+    in->_stored(version, fin);
+  }
+};
+
+object_t CInode::get_object_name(inodeno_t ino, frag_t fg, const char *suffix)
+{
+  char n[60];
+  snprintf(n, sizeof(n), "%llx.%08llx%s", (long long unsigned)ino, (long long unsigned)fg, suffix ? suffix : "");
+  return object_t(n);
+}
+
+void CInode::store(Context *fin)
+{
+  dout(10) << "store " << get_version() << dendl;
+  assert(is_base());
+
+  // encode
+  bufferlist bl;
+  string magic = CEPH_FS_ONDISK_MAGIC;
+  ::encode(magic, bl);
+  encode_store(bl);
+
+  // write it.
+  SnapContext snapc;
+  ObjectOperation m;
+  m.write_full(bl);
+
+  object_t oid = CInode::get_object_name(ino(), frag_t(), ".inode");
+  object_locator_t oloc(mdcache->mds->mdsmap->get_metadata_pool());
+
+  mdcache->mds->objecter->mutate(oid, oloc, m, snapc, ceph_clock_now(g_ceph_context), 0,
+				 NULL, new C_Inode_Stored(this, get_version(), fin) );
+}
+
+void CInode::_stored(version_t v, Context *fin)
+{
+  dout(10) << "_stored " << v << " " << *this << dendl;
+  if (v == get_projected_version())
+    mark_clean();
+
+  fin->complete(0);
+}
+
+struct C_Inode_Fetched : public Context {
+  CInode *in;
+  bufferlist bl, bl2;
+  Context *fin;
+  C_Inode_Fetched(CInode *i, Context *f) : in(i), fin(f) {}
+  void finish(int r) {
+    in->_fetched(bl, bl2, fin);
+  }
+};
+
+void CInode::fetch(Context *fin)
+{
+  dout(10) << "fetch" << dendl;
+
+  C_Inode_Fetched *c = new C_Inode_Fetched(this, fin);
+  C_GatherBuilder gather(g_ceph_context, c);
+
+  object_t oid = CInode::get_object_name(ino(), frag_t(), "");
+  object_locator_t oloc(mdcache->mds->mdsmap->get_metadata_pool());
+
+  ObjectOperation rd;
+  rd.getxattr("inode", &c->bl, NULL);
+
+  mdcache->mds->objecter->read(oid, oloc, rd, CEPH_NOSNAP, (bufferlist*)NULL, 0, gather.new_sub());
+
+  // read from separate object too
+  object_t oid2 = CInode::get_object_name(ino(), frag_t(), ".inode");
+  mdcache->mds->objecter->read(oid2, oloc, 0, 0, CEPH_NOSNAP, &c->bl2, 0, gather.new_sub());
+
+  gather.activate();
+}
+
+void CInode::_fetched(bufferlist& bl, bufferlist& bl2, Context *fin)
+{
+  dout(10) << "_fetched got " << bl.length() << " and " << bl2.length() << dendl;
+  bufferlist::iterator p;
+  if (bl2.length())
+    p = bl2.begin();
+  else
+    p = bl.begin();
+  string magic;
+  ::decode(magic, p);
+  dout(10) << " magic is '" << magic << "' (expecting '" << CEPH_FS_ONDISK_MAGIC << "')" << dendl;
+  if (magic != CEPH_FS_ONDISK_MAGIC) {
+    dout(0) << "on disk magic '" << magic << "' != my magic '" << CEPH_FS_ONDISK_MAGIC
+	    << "'" << dendl;
+    fin->complete(-EINVAL);
+  } else {
+    decode_store(p);
+    dout(10) << "_fetched " << *this << dendl;
+    fin->complete(0);
+  }
+}
+
+void CInode::build_backtrace(int64_t pool, inode_backtrace_t& bt)
+{
+  bt.ino = inode.ino;
+  bt.ancestors.clear();
+  bt.pool = pool;
+
+  CInode *in = this;
+  CDentry *pdn = get_parent_dn();
+  while (pdn) {
+    CInode *diri = pdn->get_dir()->get_inode();
+    bt.ancestors.push_back(inode_backpointer_t(diri->ino(), pdn->name, in->inode.version));
+    in = diri;
+    pdn = in->get_parent_dn();
+  }
+  vector::iterator i = inode.old_pools.begin();
+  while(i != inode.old_pools.end()) {
+    // don't add our own pool id to old_pools to avoid looping (e.g. setlayout 0, 1, 0)
+    if (*i == pool) {
+      ++i;
+      continue;
+    }
+    bt.old_pools.insert(*i);
+    ++i;
+  }
+}
+
+struct C_Inode_StoredBacktrace : public Context {
+  CInode *in;
+  version_t version;
+  Context *fin;
+  C_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
+  void finish(int r) {
+    assert(r == 0);
+    in->_stored_backtrace(version, fin);
+  }
+};
+
+void CInode::store_backtrace(Context *fin)
+{
+  dout(10) << "store_backtrace on " << *this << dendl;
+  assert(is_dirty_parent());
+
+  auth_pin(this);
+
+  int64_t pool;
+  if (is_dir())
+    pool = mdcache->mds->mdsmap->get_metadata_pool();
+  else
+    pool = inode.layout.fl_pg_pool;
+
+  inode_backtrace_t bt;
+  build_backtrace(pool, bt);
+  bufferlist bl;
+  ::encode(bt, bl);
+
+  ObjectOperation op;
+  op.create(false);
+  op.setxattr("parent", bl);
+
+  SnapContext snapc;
+  object_t oid = get_object_name(ino(), frag_t(), "");
+  object_locator_t oloc(pool);
+  Context *fin2 = new C_Inode_StoredBacktrace(this, inode.backtrace_version, fin);
+
+  if (!state_test(STATE_DIRTYPOOL) || inode.old_pools.empty()) {
+    mdcache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
+				   0, NULL, fin2);
+    return;
+  }
+
+  C_GatherBuilder gather(g_ceph_context, fin2);
+  mdcache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
+				 0, NULL, gather.new_sub());
+
+  set old_pools;
+  for (vector::iterator p = inode.old_pools.begin();
+      p != inode.old_pools.end();
+      ++p) {
+    if (*p == pool || old_pools.count(*p))
+      continue;
+
+    ObjectOperation op;
+    op.create(false);
+    op.setxattr("parent", bl);
+
+    object_locator_t oloc(*p);
+    mdcache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),
+				   0, NULL, gather.new_sub());
+    old_pools.insert(*p);
+  }
+  gather.activate();
+}
+
+void CInode::_stored_backtrace(version_t v, Context *fin)
+{
+  dout(10) << "_stored_backtrace" << dendl;
+
+  auth_unpin(this);
+  if (v == inode.backtrace_version)
+    clear_dirty_parent();
+  if (fin)
+    fin->complete(0);
+}
+
+void CInode::_mark_dirty_parent(LogSegment *ls, bool dirty_pool)
+{
+  if (!state_test(STATE_DIRTYPARENT)) {
+    dout(10) << "mark_dirty_parent" << dendl;
+    state_set(STATE_DIRTYPARENT);
+    get(PIN_DIRTYPARENT);
+    assert(ls);
+  }
+  if (dirty_pool)
+    state_set(STATE_DIRTYPOOL);
+  if (ls)
+    ls->dirty_parent_inodes.push_back(&item_dirty_parent);
+}
+
+void CInode::clear_dirty_parent()
+{
+  if (state_test(STATE_DIRTYPARENT)) {
+    dout(10) << "clear_dirty_parent" << dendl;
+    state_clear(STATE_DIRTYPARENT);
+    state_clear(STATE_DIRTYPOOL);
+    put(PIN_DIRTYPARENT);
+    item_dirty_parent.remove_myself();
+  }
+}
+
+// ------------------
+// parent dir
+
+void CInode::encode_store(bufferlist& bl)
+{
+  ENCODE_START(4, 4, bl);
+  ::encode(inode, bl);
+  if (is_symlink())
+    ::encode(symlink, bl);
+  ::encode(dirfragtree, bl);
+  ::encode(xattrs, bl);
+  bufferlist snapbl;
+  encode_snap_blob(snapbl);
+  ::encode(snapbl, bl);
+  ::encode(old_inodes, bl);
+  ENCODE_FINISH(bl);
+}
+
+void CInode::decode_store(bufferlist::iterator& bl) {
+  DECODE_START_LEGACY_COMPAT_LEN(4, 4, 4, bl);
+  ::decode(inode, bl);
+  if (is_symlink())
+    ::decode(symlink, bl);
+  ::decode(dirfragtree, bl);
+  ::decode(xattrs, bl);
+  bufferlist snapbl;
+  ::decode(snapbl, bl);
+  decode_snap_blob(snapbl);
+  ::decode(old_inodes, bl);
+  if (struct_v == 2 && inode.is_dir()) {
+    bool default_layout_exists;
+    ::decode(default_layout_exists, bl);
+    if (default_layout_exists) {
+      ::decode(struct_v, bl); // this was a default_file_layout
+      ::decode(inode.layout, bl); // but we only care about the layout portion
+    }
+  }
+  DECODE_FINISH(bl);
+}
+
+// ------------------
+// locking
+
+void CInode::set_object_info(MDSCacheObjectInfo &info)
+{
+  info.ino = ino();
+  info.snapid = last;
+}
+
+void CInode::encode_lock_state(int type, bufferlist& bl)
+{
+  ::encode(first, bl);
+
+  switch (type) {
+  case CEPH_LOCK_IAUTH:
+    ::encode(inode.version, bl);
+    ::encode(inode.ctime, bl);
+    ::encode(inode.mode, bl);
+    ::encode(inode.uid, bl);
+    ::encode(inode.gid, bl);  
+    break;
+    
+  case CEPH_LOCK_ILINK:
+    ::encode(inode.version, bl);
+    ::encode(inode.ctime, bl);
+    ::encode(inode.nlink, bl);
+    ::encode(inode.anchored, bl);
+    break;
+    
+  case CEPH_LOCK_IDFT:
+    if (is_auth()) {
+      ::encode(inode.version, bl);
+    } else {
+      // treat flushing as dirty when rejoining cache
+      bool dirty = dirfragtreelock.is_dirty_or_flushing();
+      ::encode(dirty, bl);
+    }
+    {
+      // encode the raw tree
+      ::encode(dirfragtree, bl);
+
+      // also specify which frags are mine
+      set myfrags;
+      list dfls;
+      get_dirfrags(dfls);
+      for (list::iterator p = dfls.begin(); p != dfls.end(); ++p) 
+	if ((*p)->is_auth()) {
+	  frag_t fg = (*p)->get_frag();
+	  myfrags.insert(fg);
+	}
+      ::encode(myfrags, bl);
+    }
+    break;
+    
+  case CEPH_LOCK_IFILE:
+    if (is_auth()) {
+      ::encode(inode.version, bl);
+      ::encode(inode.mtime, bl);
+      ::encode(inode.atime, bl);
+      ::encode(inode.time_warp_seq, bl);
+      if (!is_dir()) {
+	::encode(inode.layout, bl);
+	::encode(inode.size, bl);
+	::encode(inode.truncate_seq, bl);
+	::encode(inode.truncate_size, bl);
+	::encode(inode.client_ranges, bl);
+	::encode(inode.inline_data, bl);
+	::encode(inode.inline_version, bl);
+      }
+    } else {
+      // treat flushing as dirty when rejoining cache
+      bool dirty = filelock.is_dirty_or_flushing();
+      ::encode(dirty, bl);
+    }
+
+    {
+      dout(15) << "encode_lock_state inode.dirstat is " << inode.dirstat << dendl;
+      ::encode(inode.dirstat, bl);  // only meaningful if i am auth.
+      bufferlist tmp;
+      __u32 n = 0;
+      for (map::iterator p = dirfrags.begin();
+	   p != dirfrags.end();
+	   ++p) {
+	frag_t fg = p->first;
+	CDir *dir = p->second;
+	if (is_auth() || dir->is_auth()) {
+	  fnode_t *pf = dir->get_projected_fnode();
+	  dout(15) << fg << " " << *dir << dendl;
+	  dout(20) << fg << "           fragstat " << pf->fragstat << dendl;
+	  dout(20) << fg << " accounted_fragstat " << pf->accounted_fragstat << dendl;
+	  ::encode(fg, tmp);
+	  ::encode(dir->first, tmp);
+	  ::encode(pf->fragstat, tmp);
+	  ::encode(pf->accounted_fragstat, tmp);
+	  n++;
+	}
+      }
+      ::encode(n, bl);
+      bl.claim_append(tmp);
+    }
+    break;
+
+  case CEPH_LOCK_INEST:
+    if (is_auth()) {
+      ::encode(inode.version, bl);
+    } else {
+      // treat flushing as dirty when rejoining cache
+      bool dirty = nestlock.is_dirty_or_flushing();
+      ::encode(dirty, bl);
+    }
+    {
+      dout(15) << "encode_lock_state inode.rstat is " << inode.rstat << dendl;
+      ::encode(inode.rstat, bl);  // only meaningful if i am auth.
+      bufferlist tmp;
+      __u32 n = 0;
+      for (map::iterator p = dirfrags.begin();
+	   p != dirfrags.end();
+	   ++p) {
+	frag_t fg = p->first;
+	CDir *dir = p->second;
+	if (is_auth() || dir->is_auth()) {
+	  fnode_t *pf = dir->get_projected_fnode();
+	  dout(10) << fg << " " << *dir << dendl;
+	  dout(10) << fg << " " << pf->rstat << dendl;
+	  dout(10) << fg << " " << pf->rstat << dendl;
+	  dout(10) << fg << " " << dir->dirty_old_rstat << dendl;
+	  ::encode(fg, tmp);
+	  ::encode(dir->first, tmp);
+	  ::encode(pf->rstat, tmp);
+	  ::encode(pf->accounted_rstat, tmp);
+	  ::encode(dir->dirty_old_rstat, tmp);
+	  n++;
+	}
+      }
+      ::encode(n, bl);
+      bl.claim_append(tmp);
+    }
+    break;
+    
+  case CEPH_LOCK_IXATTR:
+    ::encode(inode.version, bl);
+    ::encode(xattrs, bl);
+    break;
+
+  case CEPH_LOCK_ISNAP:
+    ::encode(inode.version, bl);
+    encode_snap(bl);
+    break;
+
+  case CEPH_LOCK_IFLOCK:
+    ::encode(inode.version, bl);
+    ::encode(fcntl_locks, bl);
+    ::encode(flock_locks, bl);
+    break;
+
+  case CEPH_LOCK_IPOLICY:
+    if (inode.is_dir()) {
+      ::encode(inode.version, bl);
+      ::encode(inode.layout, bl);
+    }
+    break;
+  
+  default:
+    assert(0);
+  }
+}
+
+
+/* for more info on scatterlocks, see comments by Locker::scatter_writebehind */
+
+void CInode::decode_lock_state(int type, bufferlist& bl)
+{
+  bufferlist::iterator p = bl.begin();
+  utime_t tm;
+
+  snapid_t newfirst;
+  ::decode(newfirst, p);
+
+  if (!is_auth() && newfirst != first) {
+    dout(10) << "decode_lock_state first " << first << " -> " << newfirst << dendl;
+    assert(newfirst > first);
+    if (!is_multiversion() && parent) {
+      assert(parent->first == first);
+      parent->first = newfirst;
+    }
+    first = newfirst;
+  }
+
+  switch (type) {
+  case CEPH_LOCK_IAUTH:
+    ::decode(inode.version, p);
+    ::decode(tm, p);
+    if (inode.ctime < tm) inode.ctime = tm;
+    ::decode(inode.mode, p);
+    ::decode(inode.uid, p);
+    ::decode(inode.gid, p);
+    break;
+
+  case CEPH_LOCK_ILINK:
+    ::decode(inode.version, p);
+    ::decode(tm, p);
+    if (inode.ctime < tm) inode.ctime = tm;
+    ::decode(inode.nlink, p);
+    {
+      bool was_anchored = inode.anchored;
+      ::decode(inode.anchored, p);
+      if (parent && was_anchored != inode.anchored)
+	parent->adjust_nested_anchors((int)inode.anchored - (int)was_anchored);
+    }
+    break;
+
+  case CEPH_LOCK_IDFT:
+    if (is_auth()) {
+      bool replica_dirty;
+      ::decode(replica_dirty, p);
+      if (replica_dirty) {
+	dout(10) << "decode_lock_state setting dftlock dirty flag" << dendl;
+	dirfragtreelock.mark_dirty();  // ok bc we're auth and caller will handle
+      }
+    } else {
+      ::decode(inode.version, p);
+    }
+    {
+      fragtree_t temp;
+      ::decode(temp, p);
+      set authfrags;
+      ::decode(authfrags, p);
+      if (is_auth()) {
+	// auth.  believe replica's auth frags only.
+	for (set::iterator p = authfrags.begin(); p != authfrags.end(); ++p)
+	  if (!dirfragtree.is_leaf(*p)) {
+	    dout(10) << " forcing frag " << *p << " to leaf (split|merge)" << dendl;
+	    dirfragtree.force_to_leaf(g_ceph_context, *p);
+	    dirfragtreelock.mark_dirty();  // ok bc we're auth and caller will handle
+	  }
+      } else {
+	// replica.  take the new tree, BUT make sure any open
+	//  dirfrags remain leaves (they may have split _after_ this
+	//  dft was scattered, or we may still be be waiting on the
+	//  notify from the auth)
+	dirfragtree.swap(temp);
+	for (map::iterator p = dirfrags.begin();
+	     p != dirfrags.end();
+	     ++p) {
+	  if (!dirfragtree.is_leaf(p->first)) {
+	    dout(10) << " forcing open dirfrag " << p->first << " to leaf (racing with split|merge)" << dendl;
+	    dirfragtree.force_to_leaf(g_ceph_context, p->first);
+	  }
+	  if (p->second->is_auth())
+	    p->second->state_clear(CDir::STATE_DIRTYDFT);
+	}
+      }
+      if (g_conf->mds_debug_frag)
+	verify_dirfrags();
+    }
+    break;
+
+  case CEPH_LOCK_IFILE:
+    if (!is_auth()) {
+      ::decode(inode.version, p);
+      ::decode(inode.mtime, p);
+      ::decode(inode.atime, p);
+      ::decode(inode.time_warp_seq, p);
+      if (!is_dir()) {
+	::decode(inode.layout, p);
+	::decode(inode.size, p);
+	::decode(inode.truncate_seq, p);
+	::decode(inode.truncate_size, p);
+	::decode(inode.client_ranges, p);
+	::decode(inode.inline_data, p);
+	::decode(inode.inline_version, p);
+      }
+    } else {
+      bool replica_dirty;
+      ::decode(replica_dirty, p);
+      if (replica_dirty) {
+	dout(10) << "decode_lock_state setting filelock dirty flag" << dendl;
+	filelock.mark_dirty();  // ok bc we're auth and caller will handle
+      }
+    }
+    {
+      frag_info_t dirstat;
+      ::decode(dirstat, p);
+      if (!is_auth()) {
+	dout(10) << " taking inode dirstat " << dirstat << " for " << *this << dendl;
+	inode.dirstat = dirstat;    // take inode summation if replica
+      }
+      __u32 n;
+      ::decode(n, p);
+      dout(10) << " ...got " << n << " fragstats on " << *this << dendl;
+      while (n--) {
+	frag_t fg;
+	snapid_t fgfirst;
+	frag_info_t fragstat;
+	frag_info_t accounted_fragstat;
+	::decode(fg, p);
+	::decode(fgfirst, p);
+	::decode(fragstat, p);
+	::decode(accounted_fragstat, p);
+	dout(10) << fg << " [" << fgfirst << ",head] " << dendl;
+	dout(10) << fg << "           fragstat " << fragstat << dendl;
+	dout(20) << fg << " accounted_fragstat " << accounted_fragstat << dendl;
+
+	CDir *dir = get_dirfrag(fg);
+	if (is_auth()) {
+	  assert(dir);                // i am auth; i had better have this dir open
+	  dout(10) << fg << " first " << dir->first << " -> " << fgfirst
+		   << " on " << *dir << dendl;
+	  dir->first = fgfirst;
+	  dir->fnode.fragstat = fragstat;
+	  dir->fnode.accounted_fragstat = accounted_fragstat;
+	  dir->first = fgfirst;
+	  if (!(fragstat == accounted_fragstat)) {
+	    dout(10) << fg << " setting filelock updated flag" << dendl;
+	    filelock.mark_dirty();  // ok bc we're auth and caller will handle
+	  }
+	} else {
+	  if (dir && dir->is_auth()) {
+	    dout(10) << fg << " first " << dir->first << " -> " << fgfirst
+		     << " on " << *dir << dendl;
+	    dir->first = fgfirst;
+	    fnode_t *pf = dir->get_projected_fnode();
+	    finish_scatter_update(&filelock, dir,
+				  inode.dirstat.version, pf->accounted_fragstat.version);
+	  }
+	}
+      }
+    }
+    break;
+
+  case CEPH_LOCK_INEST:
+    if (is_auth()) {
+      bool replica_dirty;
+      ::decode(replica_dirty, p);
+      if (replica_dirty) {
+	dout(10) << "decode_lock_state setting nestlock dirty flag" << dendl;
+	nestlock.mark_dirty();  // ok bc we're auth and caller will handle
+      }
+    } else {
+      ::decode(inode.version, p);
+    }
+    {
+      nest_info_t rstat;
+      ::decode(rstat, p);
+      if (!is_auth()) {
+	dout(10) << " taking inode rstat " << rstat << " for " << *this << dendl;
+	inode.rstat = rstat;    // take inode summation if replica
+      }
+      __u32 n;
+      ::decode(n, p);
+      while (n--) {
+	frag_t fg;
+	snapid_t fgfirst;
+	nest_info_t rstat;
+	nest_info_t accounted_rstat;
+	map dirty_old_rstat;
+	::decode(fg, p);
+	::decode(fgfirst, p);
+	::decode(rstat, p);
+	::decode(accounted_rstat, p);
+	::decode(dirty_old_rstat, p);
+	dout(10) << fg << " [" << fgfirst << ",head]" << dendl;
+	dout(10) << fg << "               rstat " << rstat << dendl;
+	dout(10) << fg << "     accounted_rstat " << accounted_rstat << dendl;
+	dout(10) << fg << "     dirty_old_rstat " << dirty_old_rstat << dendl;
+
+	CDir *dir = get_dirfrag(fg);
+	if (is_auth()) {
+	  assert(dir);                // i am auth; i had better have this dir open
+	  dout(10) << fg << " first " << dir->first << " -> " << fgfirst
+		   << " on " << *dir << dendl;
+	  dir->first = fgfirst;
+	  dir->fnode.rstat = rstat;
+	  dir->fnode.accounted_rstat = accounted_rstat;
+	  dir->dirty_old_rstat.swap(dirty_old_rstat);
+	  if (!(rstat == accounted_rstat) || !dir->dirty_old_rstat.empty()) {
+	    dout(10) << fg << " setting nestlock updated flag" << dendl;
+	    nestlock.mark_dirty();  // ok bc we're auth and caller will handle
+	  }
+	} else {
+	  if (dir && dir->is_auth()) {
+	    dout(10) << fg << " first " << dir->first << " -> " << fgfirst
+		     << " on " << *dir << dendl;
+	    dir->first = fgfirst;
+	    fnode_t *pf = dir->get_projected_fnode();
+	    finish_scatter_update(&nestlock, dir,
+				  inode.rstat.version, pf->accounted_rstat.version);
+	  }
+	}
+      }
+    }
+    break;
+
+  case CEPH_LOCK_IXATTR:
+    ::decode(inode.version, p);
+    ::decode(xattrs, p);
+    break;
+
+  case CEPH_LOCK_ISNAP:
+    {
+      ::decode(inode.version, p);
+      snapid_t seq = 0;
+      if (snaprealm)
+	seq = snaprealm->srnode.seq;
+      decode_snap(p);
+      if (snaprealm && snaprealm->srnode.seq != seq)
+	mdcache->do_realm_invalidate_and_update_notify(this, seq ? CEPH_SNAP_OP_UPDATE:CEPH_SNAP_OP_SPLIT);
+    }
+    break;
+
+  case CEPH_LOCK_IFLOCK:
+    ::decode(inode.version, p);
+    ::decode(fcntl_locks, p);
+    ::decode(flock_locks, p);
+    break;
+
+  case CEPH_LOCK_IPOLICY:
+    if (inode.is_dir()) {
+      ::decode(inode.version, p);
+      ::decode(inode.layout, p);
+    }
+    break;
+
+  default:
+    assert(0);
+  }
+}
+
+
+bool CInode::is_dirty_scattered()
+{
+  return
+    filelock.is_dirty_or_flushing() ||
+    nestlock.is_dirty_or_flushing() ||
+    dirfragtreelock.is_dirty_or_flushing();
+}
+
+void CInode::clear_scatter_dirty()
+{
+  filelock.remove_dirty();
+  nestlock.remove_dirty();
+  dirfragtreelock.remove_dirty();
+}
+
+void CInode::clear_dirty_scattered(int type)
+{
+  dout(10) << "clear_dirty_scattered " << type << " on " << *this << dendl;
+  switch (type) {
+  case CEPH_LOCK_IFILE:
+    item_dirty_dirfrag_dir.remove_myself();
+    break;
+
+  case CEPH_LOCK_INEST:
+    item_dirty_dirfrag_nest.remove_myself();
+    break;
+
+  case CEPH_LOCK_IDFT:
+    item_dirty_dirfrag_dirfragtree.remove_myself();
+    break;
+
+  default:
+    assert(0);
+  }
+}
+
+
+/*
+ * when we initially scatter a lock, we need to check if any of the dirfrags
+ * have out of date accounted_rstat/fragstat.  if so, mark the lock stale.
+ */
+/* for more info on scatterlocks, see comments by Locker::scatter_writebehind */
+void CInode::start_scatter(ScatterLock *lock)
+{
+  dout(10) << "start_scatter " << *lock << " on " << *this << dendl;
+  assert(is_auth());
+  inode_t *pi = get_projected_inode();
+
+  for (map::iterator p = dirfrags.begin();
+       p != dirfrags.end();
+       ++p) {
+    frag_t fg = p->first;
+    CDir *dir = p->second;
+    fnode_t *pf = dir->get_projected_fnode();
+    dout(20) << fg << " " << *dir << dendl;
+
+    if (!dir->is_auth())
+      continue;
+
+    switch (lock->get_type()) {
+    case CEPH_LOCK_IFILE:
+      finish_scatter_update(lock, dir, pi->dirstat.version, pf->accounted_fragstat.version);
+      break;
+
+    case CEPH_LOCK_INEST:
+      finish_scatter_update(lock, dir, pi->rstat.version, pf->accounted_rstat.version);
+      break;
+
+    case CEPH_LOCK_IDFT:
+      dir->state_clear(CDir::STATE_DIRTYDFT);
+      break;
+    }
+  }
+}
+
+struct C_Inode_FragUpdate : public Context {
+  CInode *in;
+  CDir *dir;
+  MutationRef mut;
+
+  C_Inode_FragUpdate(CInode *i, CDir *d, MutationRef& m) : in(i), dir(d), mut(m) {}
+  void finish(int r) {
+    in->_finish_frag_update(dir, mut);
+  }    
+};
+
+void CInode::finish_scatter_update(ScatterLock *lock, CDir *dir,
+				   version_t inode_version, version_t dir_accounted_version)
+{
+  frag_t fg = dir->get_frag();
+  assert(dir->is_auth());
+
+  if (dir->is_frozen()) {
+    dout(10) << "finish_scatter_update " << fg << " frozen, marking " << *lock << " stale " << *dir << dendl;
+  } else if (dir->get_version() == 0) {
+    dout(10) << "finish_scatter_update " << fg << " not loaded, marking " << *lock << " stale " << *dir << dendl;
+  } else {
+    if (dir_accounted_version != inode_version) {
+      dout(10) << "finish_scatter_update " << fg << " journaling accounted scatterstat update v" << inode_version << dendl;
+
+      MDLog *mdlog = mdcache->mds->mdlog;
+      MutationRef mut(new MutationImpl);
+      mut->ls = mdlog->get_current_segment();
+
+      inode_t *pi = get_projected_inode();
+      fnode_t *pf = dir->project_fnode();
+      pf->version = dir->pre_dirty();
+
+      const char *ename = 0;
+      switch (lock->get_type()) {
+      case CEPH_LOCK_IFILE:
+	pf->fragstat.version = pi->dirstat.version;
+	pf->accounted_fragstat = pf->fragstat;
+	ename = "lock ifile accounted scatter stat update";
+	break;
+      case CEPH_LOCK_INEST:
+	pf->rstat.version = pi->rstat.version;
+	pf->accounted_rstat = pf->rstat;
+	ename = "lock inest accounted scatter stat update";
+	break;
+      default:
+	assert(0);
+      }
+	
+      mut->add_projected_fnode(dir);
+
+      EUpdate *le = new EUpdate(mdlog, ename);
+      mdlog->start_entry(le);
+      le->metablob.add_dir_context(dir);
+      le->metablob.add_dir(dir, true);
+      
+      assert(!dir->is_frozen());
+      mut->auth_pin(dir);
+      
+      mdlog->submit_entry(le, new C_Inode_FragUpdate(this, dir, mut));
+    } else {
+      dout(10) << "finish_scatter_update " << fg << " accounted " << *lock
+	       << " scatter stat unchanged at v" << dir_accounted_version << dendl;
+    }
+  }
+}
+
+void CInode::_finish_frag_update(CDir *dir, MutationRef& mut)
+{
+  dout(10) << "_finish_frag_update on " << *dir << dendl;
+  mut->apply();
+  mut->cleanup();
+}
+
+
+/*
+ * when we gather a lock, we need to assimilate dirfrag changes into the inode
+ * state.  it's possible we can't update the dirfrag accounted_rstat/fragstat
+ * because the frag is auth and frozen, or that the replica couldn't for the same
+ * reason.  hopefully it will get updated the next time the lock cycles.
+ *
+ * we have two dimensions of behavior:
+ *  - we may be (auth and !frozen), and able to update, or not.
+ *  - the frag may be stale, or not.
+ *
+ * if the frag is non-stale, we want to assimilate the diff into the
+ * inode, regardless of whether it's auth or updateable.
+ *
+ * if we update the frag, we want to set accounted_fragstat = frag,
+ * both if we took the diff or it was stale and we are making it
+ * un-stale.
+ */
+/* for more info on scatterlocks, see comments by Locker::scatter_writebehind */
+void CInode::finish_scatter_gather_update(int type)
+{
+  LogClient &clog = mdcache->mds->clog;
+
+  dout(10) << "finish_scatter_gather_update " << type << " on " << *this << dendl;
+  assert(is_auth());
+
+  switch (type) {
+  case CEPH_LOCK_IFILE:
+    {
+      // adjust summation
+      assert(is_auth());
+      inode_t *pi = get_projected_inode();
+
+      bool touched_mtime = false;
+      dout(20) << "  orig dirstat " << pi->dirstat << dendl;
+      pi->dirstat.version++;
+      for (map::iterator p = dirfrags.begin();
+	   p != dirfrags.end();
+	   ++p) {
+	frag_t fg = p->first;
+	CDir *dir = p->second;
+	dout(20) << fg << " " << *dir << dendl;
+
+	bool update = dir->is_auth() && dir->get_version() != 0 &&  !dir->is_frozen();
+
+	fnode_t *pf = dir->get_projected_fnode();
+	if (update)
+	  pf = dir->project_fnode();
+
+	if (pf->accounted_fragstat.version == pi->dirstat.version - 1) {
+	  dout(20) << fg << "           fragstat " << pf->fragstat << dendl;
+	  dout(20) << fg << " accounted_fragstat " << pf->accounted_fragstat << dendl;
+	  pi->dirstat.add_delta(pf->fragstat, pf->accounted_fragstat, touched_mtime);
+	} else {
+	  dout(20) << fg << " skipping STALE accounted_fragstat " << pf->accounted_fragstat << dendl;
+	}
+
+	if (pf->fragstat.nfiles < 0 ||
+	    pf->fragstat.nsubdirs < 0) {
+	  clog.error() << "bad/negative dir size on "
+	      << dir->dirfrag() << " " << pf->fragstat << "\n";
+	  
+	  if (pf->fragstat.nfiles < 0)
+	    pf->fragstat.nfiles = 0;
+	  if (pf->fragstat.nsubdirs < 0)
+	    pf->fragstat.nsubdirs = 0;
+
+	  assert(!"bad/negative frag size" == g_conf->mds_verify_scatter);
+	}
+
+	if (update) {
+	  pf->accounted_fragstat = pf->fragstat;
+	  pf->fragstat.version = pf->accounted_fragstat.version = pi->dirstat.version;
+	  dout(10) << fg << " updated accounted_fragstat " << pf->fragstat << " on " << *dir << dendl;
+	}
+
+	if (fg == frag_t()) { // i.e., we are the only frag
+	  if (pi->dirstat.size() != pf->fragstat.size()) {
+	    clog.error() << "unmatched fragstat size on single "
+	       << "dirfrag " << dir->dirfrag() << ", inode has " 
+	       << pi->dirstat << ", dirfrag has " << pf->fragstat << "\n";
+	    
+	    // trust the dirfrag for now
+	    version_t v = pi->dirstat.version;
+	    pi->dirstat = pf->fragstat;
+	    pi->dirstat.version = v;
+
+	    assert(!"unmatched fragstat size" == g_conf->mds_verify_scatter);
+	  }
+	}
+      }
+      if (touched_mtime)
+	pi->mtime = pi->ctime = pi->dirstat.mtime;
+      dout(20) << " final dirstat " << pi->dirstat << dendl;
+
+      if (pi->dirstat.nfiles < 0 ||
+	  pi->dirstat.nsubdirs < 0) {
+	clog.error() << "bad/negative dir size on " << ino()
+	    << ", inode has " << pi->dirstat << "\n";
+
+	if (pi->dirstat.nfiles < 0)
+	  pi->dirstat.nfiles = 0;
+	if (pi->dirstat.nsubdirs < 0)
+	  pi->dirstat.nsubdirs = 0;
+
+	assert(!"bad/negative dir size" == g_conf->mds_verify_scatter);
+      }
+    }
+    break;
+
+  case CEPH_LOCK_INEST:
+    {
+      // adjust summation
+      assert(is_auth());
+      inode_t *pi = get_projected_inode();
+      dout(20) << "  orig rstat " << pi->rstat << dendl;
+      pi->rstat.version++;
+      for (map::iterator p = dirfrags.begin();
+	   p != dirfrags.end();
+	   ++p) {
+	frag_t fg = p->first;
+	CDir *dir = p->second;
+	dout(20) << fg << " " << *dir << dendl;
+
+	bool update = dir->is_auth() && dir->get_version() != 0 && !dir->is_frozen();
+
+	fnode_t *pf = dir->get_projected_fnode();
+	if (update)
+	  pf = dir->project_fnode();
+
+	if (pf->accounted_rstat.version == pi->rstat.version-1) {
+	  // only pull this frag's dirty rstat inodes into the frag if
+	  // the frag is non-stale and updateable.  if it's stale,
+	  // that info will just get thrown out!
+	  if (update)
+	    dir->assimilate_dirty_rstat_inodes();
+
+	  dout(20) << fg << "           rstat " << pf->rstat << dendl;
+	  dout(20) << fg << " accounted_rstat " << pf->accounted_rstat << dendl;
+	  dout(20) << fg << " dirty_old_rstat " << dir->dirty_old_rstat << dendl;
+	  mdcache->project_rstat_frag_to_inode(pf->rstat, pf->accounted_rstat,
+					       dir->first, CEPH_NOSNAP, this, true);
+	  for (map::iterator q = dir->dirty_old_rstat.begin();
+	       q != dir->dirty_old_rstat.end();
+	       ++q)
+	    mdcache->project_rstat_frag_to_inode(q->second.rstat, q->second.accounted_rstat,
+						 q->second.first, q->first, this, true);
+	  if (update)  // dir contents not valid if frozen or non-auth
+	    dir->check_rstats();
+	} else {
+	  dout(20) << fg << " skipping STALE accounted_rstat " << pf->accounted_rstat << dendl;
+	}
+	if (update) {
+	  pf->accounted_rstat = pf->rstat;
+	  dir->dirty_old_rstat.clear();
+	  pf->rstat.version = pf->accounted_rstat.version = pi->rstat.version;
+	  dout(10) << fg << " updated accounted_rstat " << pf->rstat << " on " << *dir << dendl;
+	}
+
+	if (fg == frag_t()) { // i.e., we are the only frag
+	  if (pi->rstat.rbytes != pf->rstat.rbytes) { 
+	    clog.error() << "unmatched rstat rbytes on single dirfrag "
+		<< dir->dirfrag() << ", inode has " << pi->rstat
+		<< ", dirfrag has " << pf->rstat << "\n";
+	    
+	    // trust the dirfrag for now
+	    version_t v = pi->rstat.version;
+	    pi->rstat = pf->rstat;
+	    pi->rstat.version = v;
+	    
+	    assert(!"unmatched rstat rbytes" == g_conf->mds_verify_scatter);
+	  }
+	}
+	if (update)
+	  dir->check_rstats();
+      }
+      dout(20) << " final rstat " << pi->rstat << dendl;
+
+      //assert(pi->rstat.rfiles >= 0);
+      if (pi->rstat.rfiles < 0) {
+	clog.error() << "rfiles underflow " << pi->rstat.rfiles
+		     << " on " << *this << "\n";
+	pi->rstat.rfiles = 0;
+      }
+
+      //assert(pi->rstat.rsubdirs >= 0);
+      if (pi->rstat.rsubdirs < 0) {
+	clog.error() << "rsubdirs underflow " << pi->rstat.rsubdirs
+		     << " on " << *this << "\n";
+	pi->rstat.rsubdirs = 0;
+      }
+    }
+    break;
+
+  case CEPH_LOCK_IDFT:
+    break;
+
+  default:
+    assert(0);
+  }
+}
+
+void CInode::finish_scatter_gather_update_accounted(int type, MutationRef& mut, EMetaBlob *metablob)
+{
+  dout(10) << "finish_scatter_gather_update_accounted " << type << " on " << *this << dendl;
+  assert(is_auth());
+
+  for (map::iterator p = dirfrags.begin();
+       p != dirfrags.end();
+       ++p) {
+    CDir *dir = p->second;
+    if (!dir->is_auth() || dir->get_version() == 0 || dir->is_frozen())
+      continue;
+    
+    if (type == CEPH_LOCK_IDFT)
+      continue;  // nothing to do.
+
+    dout(10) << " journaling updated frag accounted_ on " << *dir << dendl;
+    assert(dir->is_projected());
+    fnode_t *pf = dir->get_projected_fnode();
+    pf->version = dir->pre_dirty();
+    mut->add_projected_fnode(dir);
+    metablob->add_dir(dir, true);
+    mut->auth_pin(dir);
+
+    if (type == CEPH_LOCK_INEST)
+      dir->assimilate_dirty_rstat_inodes_finish(mut, metablob);
+  }
+}
+
+// waiting
+
+bool CInode::is_frozen()
+{
+  if (is_frozen_inode()) return true;
+  if (parent && parent->dir->is_frozen()) return true;
+  return false;
+}
+
+bool CInode::is_frozen_dir()
+{
+  if (parent && parent->dir->is_frozen_dir()) return true;
+  return false;
+}
+
+bool CInode::is_freezing()
+{
+  if (is_freezing_inode()) return true;
+  if (parent && parent->dir->is_freezing()) return true;
+  return false;
+}
+
+void CInode::add_dir_waiter(frag_t fg, Context *c)
+{
+  if (waiting_on_dir.empty())
+    get(PIN_DIRWAITER);
+  waiting_on_dir[fg].push_back(c);
+  dout(10) << "add_dir_waiter frag " << fg << " " << c << " on " << *this << dendl;
+}
+
+void CInode::take_dir_waiting(frag_t fg, list& ls)
+{
+  if (waiting_on_dir.empty())
+    return;
+
+  map >::iterator p = waiting_on_dir.find(fg);
+  if (p != waiting_on_dir.end()) {
+    dout(10) << "take_dir_waiting frag " << fg << " on " << *this << dendl;
+    ls.splice(ls.end(), p->second);
+    waiting_on_dir.erase(p);
+
+    if (waiting_on_dir.empty())
+      put(PIN_DIRWAITER);
+  }
+}
+
+void CInode::add_waiter(uint64_t tag, Context *c) 
+{
+  dout(10) << "add_waiter tag " << std::hex << tag << std::dec << " " << c
+	   << " !ambig " << !state_test(STATE_AMBIGUOUSAUTH)
+	   << " !frozen " << !is_frozen_inode()
+	   << " !freezing " << !is_freezing_inode()
+	   << dendl;
+  // wait on the directory?
+  //  make sure its not the inode that is explicitly ambiguous|freezing|frozen
+  if (((tag & WAIT_SINGLEAUTH) && !state_test(STATE_AMBIGUOUSAUTH)) ||
+      ((tag & WAIT_UNFREEZE) &&
+       !is_frozen_inode() && !is_freezing_inode() && !is_frozen_auth_pin())) {
+    dout(15) << "passing waiter up tree" << dendl;
+    parent->dir->add_waiter(tag, c);
+    return;
+  }
+  dout(15) << "taking waiter here" << dendl;
+  MDSCacheObject::add_waiter(tag, c);
+}
+
+void CInode::take_waiting(uint64_t mask, list& ls)
+{
+  if ((mask & WAIT_DIR) && !waiting_on_dir.empty()) {
+    // take all dentry waiters
+    while (!waiting_on_dir.empty()) {
+      map >::iterator p = waiting_on_dir.begin();
+      dout(10) << "take_waiting dirfrag " << p->first << " on " << *this << dendl;
+      ls.splice(ls.end(), p->second);
+      waiting_on_dir.erase(p);
+    }
+    put(PIN_DIRWAITER);
+  }
+
+  // waiting
+  MDSCacheObject::take_waiting(mask, ls);
+}
+
+bool CInode::freeze_inode(int auth_pin_allowance)
+{
+  assert(auth_pin_allowance > 0);  // otherwise we need to adjust parent's nested_auth_pins
+  assert(auth_pins >= auth_pin_allowance);
+  if (auth_pins > auth_pin_allowance) {
+    dout(10) << "freeze_inode - waiting for auth_pins to drop to " << auth_pin_allowance << dendl;
+    auth_pin_freeze_allowance = auth_pin_allowance;
+    get(PIN_FREEZING);
+    state_set(STATE_FREEZING);
+    return false;
+  }
+
+  dout(10) << "freeze_inode - frozen" << dendl;
+  assert(auth_pins == auth_pin_allowance);
+  if (!state_test(STATE_FROZEN)) {
+    get(PIN_FROZEN);
+    state_set(STATE_FROZEN);
+  }
+  return true;
+}
+
+void CInode::unfreeze_inode(list& finished) 
+{
+  dout(10) << "unfreeze_inode" << dendl;
+  if (state_test(STATE_FREEZING)) {
+    state_clear(STATE_FREEZING);
+    put(PIN_FREEZING);
+  } else if (state_test(STATE_FROZEN)) {
+    state_clear(STATE_FROZEN);
+    put(PIN_FROZEN);
+  } else 
+    assert(0);
+  take_waiting(WAIT_UNFREEZE, finished);
+}
+
+void CInode::unfreeze_inode()
+{
+    list finished;
+    unfreeze_inode(finished);
+    mdcache->mds->queue_waiters(finished);
+}
+
+void CInode::freeze_auth_pin()
+{
+  assert(state_test(CInode::STATE_FROZEN));
+  state_set(CInode::STATE_FROZENAUTHPIN);
+}
+
+void CInode::unfreeze_auth_pin()
+{
+  assert(state_test(CInode::STATE_FROZENAUTHPIN));
+  state_clear(CInode::STATE_FROZENAUTHPIN);
+  if (!state_test(STATE_FREEZING|STATE_FROZEN)) {
+    list finished;
+    take_waiting(WAIT_UNFREEZE, finished);
+    mdcache->mds->queue_waiters(finished);
+  }
+}
+
+void CInode::clear_ambiguous_auth(list& finished)
+{
+  assert(state_test(CInode::STATE_AMBIGUOUSAUTH));
+  state_clear(CInode::STATE_AMBIGUOUSAUTH);
+  take_waiting(CInode::WAIT_SINGLEAUTH, finished);
+}
+
+void CInode::clear_ambiguous_auth()
+{
+  list finished;
+  clear_ambiguous_auth(finished);
+  mdcache->mds->queue_waiters(finished);
+}
+
+// auth_pins
+bool CInode::can_auth_pin() {
+  if (!is_auth() || is_freezing_inode() || is_frozen_inode() || is_frozen_auth_pin())
+    return false;
+  if (parent)
+    return parent->can_auth_pin();
+  return true;
+}
+
+void CInode::auth_pin(void *by) 
+{
+  if (auth_pins == 0)
+    get(PIN_AUTHPIN);
+  auth_pins++;
+
+#ifdef MDS_AUTHPIN_SET
+  auth_pin_set.insert(by);
+#endif
+
+  dout(10) << "auth_pin by " << by << " on " << *this
+	   << " now " << auth_pins << "+" << nested_auth_pins
+	   << dendl;
+  
+  if (parent)
+    parent->adjust_nested_auth_pins(1, 1, this);
+}
+
+void CInode::auth_unpin(void *by) 
+{
+  auth_pins--;
+
+#ifdef MDS_AUTHPIN_SET
+  assert(auth_pin_set.count(by));
+  auth_pin_set.erase(auth_pin_set.find(by));
+#endif
+
+  if (auth_pins == 0)
+    put(PIN_AUTHPIN);
+  
+  dout(10) << "auth_unpin by " << by << " on " << *this
+	   << " now " << auth_pins << "+" << nested_auth_pins
+	   << dendl;
+  
+  assert(auth_pins >= 0);
+
+  if (parent)
+    parent->adjust_nested_auth_pins(-1, -1, by);
+
+  if (is_freezing_inode() &&
+      auth_pins == auth_pin_freeze_allowance) {
+    dout(10) << "auth_unpin freezing!" << dendl;
+    get(PIN_FROZEN);
+    put(PIN_FREEZING);
+    state_clear(STATE_FREEZING);
+    state_set(STATE_FROZEN);
+    finish_waiting(WAIT_FROZEN);
+  }  
+}
+
+void CInode::adjust_nested_auth_pins(int a, void *by)
+{
+  assert(a);
+  nested_auth_pins += a;
+  dout(35) << "adjust_nested_auth_pins by " << by
+	   << " change " << a << " yields "
+	   << auth_pins << "+" << nested_auth_pins << dendl;
+  assert(nested_auth_pins >= 0);
+
+  if (g_conf->mds_debug_auth_pins) {
+    // audit
+    int s = 0;
+    for (map::iterator p = dirfrags.begin();
+	 p != dirfrags.end();
+	 ++p) {
+      CDir *dir = p->second;
+      if (!dir->is_subtree_root() && dir->get_cum_auth_pins())
+	s++;
+    } 
+    assert(s == nested_auth_pins);
+  } 
+
+  if (parent)
+    parent->adjust_nested_auth_pins(a, 0, by);
+}
+
+void CInode::adjust_nested_anchors(int by)
+{
+  assert(by);
+  nested_anchors += by;
+  dout(20) << "adjust_nested_anchors by " << by << " -> " << nested_anchors << dendl;
+  assert(nested_anchors >= 0);
+  if (parent)
+    parent->adjust_nested_anchors(by);
+}
+
+// authority
+
+pair CInode::authority() 
+{
+  if (inode_auth.first >= 0) 
+    return inode_auth;
+
+  if (parent)
+    return parent->dir->authority();
+
+  // new items that are not yet linked in (in the committed plane) belong
+  // to their first parent.
+  if (!projected_parent.empty())
+    return projected_parent.front()->dir->authority();
+
+  return CDIR_AUTH_UNDEF;
+}
+
+
+// SNAP
+
+snapid_t CInode::get_oldest_snap()
+{
+  snapid_t t = first;
+  if (!old_inodes.empty())
+    t = old_inodes.begin()->second.first;
+  return MIN(t, first);
+}
+
+old_inode_t& CInode::cow_old_inode(snapid_t follows, bool cow_head)
+{
+  assert(follows >= first);
+
+  inode_t *pi = cow_head ? get_projected_inode() : get_previous_projected_inode();
+  map *px = cow_head ? get_projected_xattrs() : get_previous_projected_xattrs();
+
+  old_inode_t &old = old_inodes[follows];
+  old.first = first;
+  old.inode = *pi;
+  old.xattrs = *px;
+  
+  dout(10) << " " << px->size() << " xattrs cowed, " << *px << dendl;
+
+  old.inode.trim_client_ranges(follows);
+
+  if (!(old.inode.rstat == old.inode.accounted_rstat))
+    dirty_old_rstats.insert(follows);
+  
+  first = follows+1;
+
+  dout(10) << "cow_old_inode " << (cow_head ? "head" : "previous_head" )
+	   << " to [" << old.first << "," << follows << "] on "
+	   << *this << dendl;
+
+  return old;
+}
+
+void CInode::pre_cow_old_inode()
+{
+  snapid_t follows = find_snaprealm()->get_newest_seq();
+  if (first <= follows)
+    cow_old_inode(follows, true);
+}
+
+void CInode::purge_stale_snap_data(const set& snaps)
+{
+  dout(10) << "purge_stale_snap_data " << snaps << dendl;
+
+  if (old_inodes.empty())
+    return;
+
+  map::iterator p = old_inodes.begin();
+  while (p != old_inodes.end()) {
+    set::const_iterator q = snaps.lower_bound(p->second.first);
+    if (q == snaps.end() || *q > p->first) {
+      dout(10) << " purging old_inode [" << p->second.first << "," << p->first << "]" << dendl;
+      old_inodes.erase(p++);
+    } else
+      ++p;
+  }
+}
+
+/*
+ * pick/create an old_inode
+ */
+old_inode_t * CInode::pick_old_inode(snapid_t snap)
+{
+  map::iterator p = old_inodes.lower_bound(snap);  // p is first key >= to snap
+  if (p != old_inodes.end() && p->second.first <= snap) {
+    dout(10) << "pick_old_inode snap " << snap << " -> [" << p->second.first << "," << p->first << "]" << dendl;
+    return &p->second;
+  }
+  dout(10) << "pick_old_inode snap " << snap << " -> nothing" << dendl;
+  return NULL;
+}
+
+void CInode::open_snaprealm(bool nosplit)
+{
+  if (!snaprealm) {
+    SnapRealm *parent = find_snaprealm();
+    snaprealm = new SnapRealm(mdcache, this);
+    if (parent) {
+      dout(10) << "open_snaprealm " << snaprealm
+	       << " parent is " << parent
+	       << dendl;
+      dout(30) << " siblings are " << parent->open_children << dendl;
+      snaprealm->parent = parent;
+      if (!nosplit)
+	parent->split_at(snaprealm);
+      parent->open_children.insert(snaprealm);
+    }
+  }
+}
+void CInode::close_snaprealm(bool nojoin)
+{
+  if (snaprealm) {
+    dout(15) << "close_snaprealm " << *snaprealm << dendl;
+    snaprealm->close_parents();
+    if (snaprealm->parent) {
+      snaprealm->parent->open_children.erase(snaprealm);
+      //if (!nojoin)
+      //snaprealm->parent->join(snaprealm);
+    }
+    delete snaprealm;
+    snaprealm = 0;
+  }
+}
+
+SnapRealm *CInode::find_snaprealm()
+{
+  CInode *cur = this;
+  while (!cur->snaprealm) {
+    if (cur->get_parent_dn())
+      cur = cur->get_parent_dn()->get_dir()->get_inode();
+    else if (get_projected_parent_dn())
+      cur = cur->get_projected_parent_dn()->get_dir()->get_inode();
+    else
+      break;
+  }
+  return cur->snaprealm;
+}
+
+void CInode::encode_snap_blob(bufferlist &snapbl)
+{
+  if (snaprealm) {
+    ::encode(snaprealm->srnode, snapbl);
+    dout(20) << "encode_snap_blob " << *snaprealm << dendl;
+  }
+}
+void CInode::decode_snap_blob(bufferlist& snapbl) 
+{
+  if (snapbl.length()) {
+    open_snaprealm();
+    bufferlist::iterator p = snapbl.begin();
+    ::decode(snaprealm->srnode, p);
+    dout(20) << "decode_snap_blob " << *snaprealm << dendl;
+  }
+}
+
+void CInode::encode_snap(bufferlist& bl)
+{
+  bufferlist snapbl;
+  encode_snap_blob(snapbl);
+  ::encode(snapbl, bl);
+}    
+
+void CInode::decode_snap(bufferlist::iterator& p)
+{
+  bufferlist snapbl;
+  ::decode(snapbl, p);
+  decode_snap_blob(snapbl);
+}
+
+// =============================================
+
+client_t CInode::calc_ideal_loner()
+{
+  if (!mds_caps_wanted.empty())
+    return -1;
+  
+  int n = 0;
+  client_t loner = -1;
+  for (map::iterator it = client_caps.begin();
+       it != client_caps.end();
+       ++it) 
+    if (!it->second->is_stale() &&
+	((it->second->wanted() & (CEPH_CAP_ANY_WR|CEPH_CAP_FILE_WR|CEPH_CAP_FILE_RD)) ||
+	 (inode.is_dir() && !has_subtree_root_dirfrag()))) {
+      if (n)
+	return -1;
+      n++;
+      loner = it->first;
+    }
+  return loner;
+}
+
+client_t CInode::choose_ideal_loner()
+{
+  want_loner_cap = calc_ideal_loner();
+  return want_loner_cap;
+}
+
+bool CInode::try_set_loner()
+{
+  assert(want_loner_cap >= 0);
+  if (loner_cap >= 0 && loner_cap != want_loner_cap)
+    return false;
+  set_loner_cap(want_loner_cap);
+  return true;
+}
+
+void CInode::set_loner_cap(client_t l)
+{
+  loner_cap = l;
+  authlock.set_excl_client(loner_cap);
+  filelock.set_excl_client(loner_cap);
+  linklock.set_excl_client(loner_cap);
+  xattrlock.set_excl_client(loner_cap);
+}
+
+bool CInode::try_drop_loner()
+{
+  if (loner_cap < 0)
+    return true;
+
+  int other_allowed = get_caps_allowed_by_type(CAP_ANY);
+  Capability *cap = get_client_cap(loner_cap);
+  if (!cap ||
+      (cap->issued() & ~other_allowed) == 0) {
+    set_loner_cap(-1);
+    return true;
+  }
+  return false;
+}
+
+
+// choose new lock state during recovery, based on issued caps
+void CInode::choose_lock_state(SimpleLock *lock, int allissued)
+{
+  int shift = lock->get_cap_shift();
+  int issued = (allissued >> shift) & lock->get_cap_mask();
+  if (is_auth()) {
+    if (lock->is_xlocked()) {
+      // do nothing here
+    } else {
+      if (issued & CEPH_CAP_GEXCL)
+	lock->set_state(LOCK_EXCL);
+      else if (issued & CEPH_CAP_GWR)
+	lock->set_state(LOCK_MIX);
+      else if (lock->is_dirty()) {
+	if (is_replicated())
+	  lock->set_state(LOCK_MIX);
+	else
+	  lock->set_state(LOCK_LOCK);
+      } else
+	lock->set_state(LOCK_SYNC);
+    }
+  } else {
+    // our states have already been chosen during rejoin.
+    if (lock->is_xlocked())
+      assert(lock->get_state() == LOCK_LOCK);
+  }
+}
+ 
+void CInode::choose_lock_states()
+{
+  int issued = get_caps_issued();
+  if (is_auth() && (issued & (CEPH_CAP_ANY_EXCL|CEPH_CAP_ANY_WR)) &&
+      choose_ideal_loner() >= 0)
+    try_set_loner();
+  choose_lock_state(&filelock, issued);
+  choose_lock_state(&nestlock, issued);
+  choose_lock_state(&dirfragtreelock, issued);
+  choose_lock_state(&authlock, issued);
+  choose_lock_state(&xattrlock, issued);
+  choose_lock_state(&linklock, issued);
+}
+
+Capability *CInode::add_client_cap(client_t client, Session *session, SnapRealm *conrealm)
+{
+  if (client_caps.empty()) {
+    get(PIN_CAPS);
+    if (conrealm)
+      containing_realm = conrealm;
+    else
+      containing_realm = find_snaprealm();
+    containing_realm->inodes_with_caps.push_back(&item_caps);
+    dout(10) << "add_client_cap first cap, joining realm " << *containing_realm << dendl;
+  }
+
+  mdcache->num_caps++;
+  if (client_caps.empty())
+    mdcache->num_inodes_with_caps++;
+  
+  Capability *cap = new Capability(this, ++mdcache->last_cap_id, client);
+  assert(client_caps.count(client) == 0);
+  client_caps[client] = cap;
+
+  session->add_cap(cap);
+  if (session->is_stale())
+    cap->mark_stale();
+  
+  cap->client_follows = first-1;
+  
+  containing_realm->add_cap(client, cap);
+  
+  return cap;
+}
+
+void CInode::remove_client_cap(client_t client)
+{
+  assert(client_caps.count(client) == 1);
+  Capability *cap = client_caps[client];
+  
+  cap->item_session_caps.remove_myself();
+  containing_realm->remove_cap(client, cap);
+  
+  if (client == loner_cap)
+    loner_cap = -1;
+
+  delete cap;
+  client_caps.erase(client);
+  if (client_caps.empty()) {
+    dout(10) << "remove_client_cap last cap, leaving realm " << *containing_realm << dendl;
+    put(PIN_CAPS);
+    item_caps.remove_myself();
+    containing_realm = NULL;
+    item_open_file.remove_myself();  // unpin logsegment
+    mdcache->num_inodes_with_caps--;
+  }
+  mdcache->num_caps--;
+
+  //clean up advisory locks
+  bool fcntl_removed = fcntl_locks.remove_all_from(client);
+  bool flock_removed = flock_locks.remove_all_from(client);
+  if (fcntl_removed || flock_removed) {
+    list waiters;
+    take_waiting(CInode::WAIT_FLOCK, waiters);
+    mdcache->mds->queue_waiters(waiters);
+  }
+}
+
+void CInode::move_to_realm(SnapRealm *realm)
+{
+  dout(10) << "move_to_realm joining realm " << *realm
+	   << ", leaving realm " << *containing_realm << dendl;
+  for (map::iterator q = client_caps.begin();
+       q != client_caps.end();
+       ++q) {
+    containing_realm->remove_cap(q->first, q->second);
+    realm->add_cap(q->first, q->second);
+  }
+  item_caps.remove_myself();
+  realm->inodes_with_caps.push_back(&item_caps);
+  containing_realm = realm;
+}
+
+Capability *CInode::reconnect_cap(client_t client, ceph_mds_cap_reconnect& icr, Session *session)
+{
+  Capability *cap = get_client_cap(client);
+  if (cap) {
+    // FIXME?
+    cap->merge(icr.wanted, icr.issued);
+  } else {
+    cap = add_client_cap(client, session);
+    cap->set_wanted(icr.wanted);
+    cap->issue_norevoke(icr.issued);
+    cap->reset_seq();
+    cap->set_cap_id(icr.cap_id);
+  }
+  cap->set_last_issue_stamp(ceph_clock_now(g_ceph_context));
+  return cap;
+}
+
+void CInode::clear_client_caps_after_export()
+{
+  while (!client_caps.empty())
+    remove_client_cap(client_caps.begin()->first);
+  loner_cap = -1;
+  want_loner_cap = -1;
+  mds_caps_wanted.clear();
+}
+
+void CInode::export_client_caps(map& cl)
+{
+  for (map::iterator it = client_caps.begin();
+       it != client_caps.end();
+       ++it) {
+    cl[it->first] = it->second->make_export();
+  }
+}
+
+  // caps allowed
+int CInode::get_caps_liked()
+{
+  if (is_dir())
+    return CEPH_CAP_PIN | CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_SHARED;  // but not, say, FILE_RD|WR|WRBUFFER
+  else
+    return CEPH_CAP_ANY & ~CEPH_CAP_FILE_LAZYIO;
+}
+
+int CInode::get_caps_allowed_ever()
+{
+  int allowed;
+  if (is_dir())
+    allowed = CEPH_CAP_PIN | CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_SHARED;
+  else
+    allowed = CEPH_CAP_ANY;
+  return allowed & 
+    (CEPH_CAP_PIN |
+     (filelock.gcaps_allowed_ever() << filelock.get_cap_shift()) |
+     (authlock.gcaps_allowed_ever() << authlock.get_cap_shift()) |
+     (xattrlock.gcaps_allowed_ever() << xattrlock.get_cap_shift()) |
+     (linklock.gcaps_allowed_ever() << linklock.get_cap_shift()));
+}
+
+int CInode::get_caps_allowed_by_type(int type)
+{
+  return 
+    CEPH_CAP_PIN |
+    (filelock.gcaps_allowed(type) << filelock.get_cap_shift()) |
+    (authlock.gcaps_allowed(type) << authlock.get_cap_shift()) |
+    (xattrlock.gcaps_allowed(type) << xattrlock.get_cap_shift()) |
+    (linklock.gcaps_allowed(type) << linklock.get_cap_shift());
+}
+
+int CInode::get_caps_careful()
+{
+  return 
+    (filelock.gcaps_careful() << filelock.get_cap_shift()) |
+    (authlock.gcaps_careful() << authlock.get_cap_shift()) |
+    (xattrlock.gcaps_careful() << xattrlock.get_cap_shift()) |
+    (linklock.gcaps_careful() << linklock.get_cap_shift());
+}
+
+int CInode::get_xlocker_mask(client_t client)
+{
+  return 
+    (filelock.gcaps_xlocker_mask(client) << filelock.get_cap_shift()) |
+    (authlock.gcaps_xlocker_mask(client) << authlock.get_cap_shift()) |
+    (xattrlock.gcaps_xlocker_mask(client) << xattrlock.get_cap_shift()) |
+    (linklock.gcaps_xlocker_mask(client) << linklock.get_cap_shift());
+}
+
+int CInode::get_caps_allowed_for_client(client_t client)
+{
+  int allowed;
+  if (client == get_loner()) {
+    // as the loner, we get the loner_caps AND any xlocker_caps for things we have xlocked
+    allowed =
+      get_caps_allowed_by_type(CAP_LONER) |
+      (get_caps_allowed_by_type(CAP_XLOCKER) & get_xlocker_mask(client));
+  } else {
+    allowed = get_caps_allowed_by_type(CAP_ANY);
+  }
+  if (inode.inline_version != CEPH_INLINE_NONE &&
+      !mdcache->mds->get_session(client)->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA))
+    allowed &= ~(CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR);
+  return allowed;
+}
+
+// caps issued, wanted
+int CInode::get_caps_issued(int *ploner, int *pother, int *pxlocker,
+			    int shift, int mask)
+{
+  int c = 0;
+  int loner = 0, other = 0, xlocker = 0;
+  if (!is_auth())
+    loner_cap = -1;
+  for (map::iterator it = client_caps.begin();
+       it != client_caps.end();
+       ++it) {
+    int i = it->second->issued();
+    c |= i;
+    if (it->first == loner_cap)
+      loner |= i;
+    else
+      other |= i;
+    xlocker |= get_xlocker_mask(it->first) & i;
+  }
+  if (ploner) *ploner = (loner >> shift) & mask;
+  if (pother) *pother = (other >> shift) & mask;
+  if (pxlocker) *pxlocker = (xlocker >> shift) & mask;
+  return (c >> shift) & mask;
+}
+
+bool CInode::is_any_caps_wanted()
+{
+  for (map::iterator it = client_caps.begin();
+       it != client_caps.end();
+       ++it)
+    if (it->second->wanted())
+      return true;
+  return false;
+}
+
+int CInode::get_caps_wanted(int *ploner, int *pother, int shift, int mask)
+{
+  int w = 0;
+  int loner = 0, other = 0;
+  for (map::iterator it = client_caps.begin();
+       it != client_caps.end();
+       ++it) {
+    if (!it->second->is_stale()) {
+      int t = it->second->wanted();
+      w |= t;
+      if (it->first == loner_cap)
+	loner |= t;
+      else
+	other |= t;	
+    }
+    //cout << " get_caps_wanted client " << it->first << " " << cap_string(it->second.wanted()) << endl;
+  }
+  if (is_auth())
+    for (map::iterator it = mds_caps_wanted.begin();
+	 it != mds_caps_wanted.end();
+	 ++it) {
+      w |= it->second;
+      other |= it->second;
+      //cout << " get_caps_wanted mds " << it->first << " " << cap_string(it->second) << endl;
+    }
+  if (ploner) *ploner = (loner >> shift) & mask;
+  if (pother) *pother = (other >> shift) & mask;
+  return (w >> shift) & mask;
+}
+
+bool CInode::issued_caps_need_gather(SimpleLock *lock)
+{
+  int loner_issued, other_issued, xlocker_issued;
+  get_caps_issued(&loner_issued, &other_issued, &xlocker_issued,
+		  lock->get_cap_shift(), lock->get_cap_mask());
+  if ((loner_issued & ~lock->gcaps_allowed(CAP_LONER)) ||
+      (other_issued & ~lock->gcaps_allowed(CAP_ANY)) ||
+      (xlocker_issued & ~lock->gcaps_allowed(CAP_XLOCKER)))
+    return true;
+  return false;
+}
+
+void CInode::replicate_relax_locks()
+{
+  //dout(10) << " relaxing locks on " << *this << dendl;
+  assert(is_auth());
+  assert(!is_replicated());
+  
+  authlock.replicate_relax();
+  linklock.replicate_relax();
+  dirfragtreelock.replicate_relax();
+  filelock.replicate_relax();
+  xattrlock.replicate_relax();
+  snaplock.replicate_relax();
+  nestlock.replicate_relax();
+  flocklock.replicate_relax();
+  policylock.replicate_relax();
+}
+
+
+
+// =============================================
+
+int CInode::encode_inodestat(bufferlist& bl, Session *session,
+			     SnapRealm *dir_realm,
+			     snapid_t snapid,
+			     unsigned max_bytes,
+			     int getattr_caps)
+{
+  int client = session->info.inst.name.num();
+  assert(snapid);
+  assert(session->connection);
+  
+  bool valid = true;
+
+  // do not issue caps if inode differs from readdir snaprealm
+  SnapRealm *realm = find_snaprealm();
+  bool no_caps = session->is_stale() ||
+		 (realm && dir_realm && realm != dir_realm) ||
+		 is_frozen() || state_test(CInode::STATE_EXPORTINGCAPS);
+  if (no_caps)
+    dout(20) << "encode_inodestat no caps"
+	     << (session->is_stale()?", session stale ":"")
+	     << ((realm && dir_realm && realm != dir_realm)?", snaprealm differs ":"")
+	     << (state_test(CInode::STATE_EXPORTINGCAPS)?", exporting caps":"")
+	     << (is_frozen()?", frozen inode":"") << dendl;
+
+  // pick a version!
+  inode_t *oi = &inode;
+  inode_t *pi = get_projected_inode();
+
+  map *pxattrs = 0;
+
+  if (snapid != CEPH_NOSNAP && is_multiversion()) {
+
+    // for now at least, old_inodes is only defined/valid on the auth
+    if (!is_auth())
+      valid = false;
+
+    map::iterator p = old_inodes.lower_bound(snapid);
+    if (p != old_inodes.end()) {
+      if (p->second.first > snapid) {
+        if  (p != old_inodes.begin())
+          --p;
+        else dout(0) << "old_inode lower_bound starts after snapid!" << dendl;
+      }
+      dout(15) << "encode_inodestat snapid " << snapid
+	       << " to old_inode [" << p->second.first << "," << p->first << "]" 
+	       << " " << p->second.inode.rstat
+	       << dendl;
+      assert(p->second.first <= snapid && snapid <= p->first);
+      pi = oi = &p->second.inode;
+      pxattrs = &p->second.xattrs;
+    }
+  }
+  
+  /*
+   * note: encoding matches struct ceph_client_reply_inode
+   */
+  struct ceph_mds_reply_inode e;
+  memset(&e, 0, sizeof(e));
+  e.ino = oi->ino;
+  e.snapid = snapid;  // 0 -> NOSNAP
+  e.rdev = oi->rdev;
+
+  // "fake" a version that is old (stable) version, +1 if projected.
+  e.version = (oi->version * 2) + is_projected();
+
+
+  Capability *cap = get_client_cap(client);
+  bool pfile = filelock.is_xlocked_by_client(client) || get_loner() == client;
+  //(cap && (cap->issued() & CEPH_CAP_FILE_EXCL));
+  bool pauth = authlock.is_xlocked_by_client(client) || get_loner() == client;
+  bool plink = linklock.is_xlocked_by_client(client) || get_loner() == client;
+  bool pxattr = xattrlock.is_xlocked_by_client(client) || get_loner() == client;
+
+  bool plocal = versionlock.get_last_wrlock_client() == client;
+  bool ppolicy = policylock.is_xlocked_by_client(client) || get_loner()==client;
+  
+  inode_t *i = (pfile|pauth|plink|pxattr|plocal) ? pi : oi;
+  i->ctime.encode_timeval(&e.ctime);
+  
+  dout(20) << " pfile " << pfile << " pauth " << pauth << " plink " << plink << " pxattr " << pxattr
+	   << " plocal " << plocal
+	   << " ctime " << i->ctime
+	   << " valid=" << valid << dendl;
+
+  // file
+  i = pfile ? pi:oi;
+  if (is_dir()) {
+    e.layout = (ppolicy ? pi : oi)->layout;
+  } else {
+    e.layout = i->layout;
+  }
+  e.size = i->size;
+  e.truncate_seq = i->truncate_seq;
+  e.truncate_size = i->truncate_size;
+  i->mtime.encode_timeval(&e.mtime);
+  i->atime.encode_timeval(&e.atime);
+  e.time_warp_seq = i->time_warp_seq;
+
+  // max_size is min of projected, actual
+  e.max_size = MIN(oi->client_ranges.count(client) ? oi->client_ranges[client].range.last : 0,
+		   pi->client_ranges.count(client) ? pi->client_ranges[client].range.last : 0);
+
+  e.files = i->dirstat.nfiles;
+  e.subdirs = i->dirstat.nsubdirs;
+
+  // inline data
+  version_t inline_version = 0;
+  bufferlist inline_data;
+  if (i->inline_version == CEPH_INLINE_NONE) {
+    inline_version = CEPH_INLINE_NONE;
+  } else if ((!cap && !no_caps) ||
+	     (cap && cap->client_inline_version < i->inline_version) ||
+	     (getattr_caps & CEPH_CAP_FILE_RD)) { // client requests inline data
+    inline_version = i->inline_version;
+    inline_data = i->inline_data;
+  }
+
+  // nest (do same as file... :/)
+  i->rstat.rctime.encode_timeval(&e.rctime);
+  e.rbytes = i->rstat.rbytes;
+  e.rfiles = i->rstat.rfiles;
+  e.rsubdirs = i->rstat.rsubdirs;
+
+  // auth
+  i = pauth ? pi:oi;
+  e.mode = i->mode;
+  e.uid = i->uid;
+  e.gid = i->gid;
+
+  // link
+  i = plink ? pi:oi;
+  e.nlink = i->nlink;
+  
+  // xattr
+  i = pxattr ? pi:oi;
+  bool had_latest_xattrs = cap && (cap->issued() & CEPH_CAP_XATTR_SHARED) &&
+    cap->client_xattr_version == i->xattr_version &&
+    snapid == CEPH_NOSNAP;
+
+  // xattr
+  bufferlist xbl;
+  e.xattr_version = i->xattr_version;
+  if (!had_latest_xattrs) {
+    if (!pxattrs)
+      pxattrs = pxattr ? get_projected_xattrs() : &xattrs;
+    ::encode(*pxattrs, xbl);
+  }
+  
+  // do we have room?
+  if (max_bytes) {
+    unsigned bytes = sizeof(e);
+    bytes += sizeof(__u32);
+    bytes += (sizeof(__u32) + sizeof(__u32)) * dirfragtree._splits.size();
+    bytes += sizeof(__u32) + symlink.length();
+    bytes += sizeof(__u32) + xbl.length();
+    bytes += sizeof(version_t) + sizeof(__u32) + inline_data.length();
+    if (bytes > max_bytes)
+      return -ENOSPC;
+  }
+
+
+  // encode caps
+  if (snapid != CEPH_NOSNAP) {
+    /*
+     * snapped inodes (files or dirs) only get read-only caps.  always
+     * issue everything possible, since it is read only.
+     *
+     * if a snapped inode has caps, limit issued caps based on the
+     * lock state.
+     *
+     * if it is a live inode, limit issued caps based on the lock
+     * state.
+     *
+     * do NOT adjust cap issued state, because the client always
+     * tracks caps per-snap and the mds does either per-interval or
+     * multiversion.
+     */
+    e.cap.caps = valid ? get_caps_allowed_by_type(CAP_ANY) : CEPH_STAT_CAP_INODE;
+    if (last == CEPH_NOSNAP || is_any_caps())
+      e.cap.caps = e.cap.caps & get_caps_allowed_for_client(client);
+    e.cap.seq = 0;
+    e.cap.mseq = 0;
+    e.cap.realm = 0;
+  } else {
+    if (!no_caps && valid && !cap) {
+      // add a new cap
+      cap = add_client_cap(client, session, realm);
+      if (is_auth()) {
+	if (choose_ideal_loner() >= 0)
+	  try_set_loner();
+	else if (get_wanted_loner() < 0)
+	  try_drop_loner();
+      }
+    }
+
+    if (!no_caps && valid && cap) {
+      int likes = get_caps_liked();
+      int allowed = get_caps_allowed_for_client(client);
+      int issue = (cap->wanted() | likes) & allowed;
+      cap->issue_norevoke(issue);
+      issue = cap->pending();
+      cap->set_last_issue();
+      cap->set_last_issue_stamp(ceph_clock_now(g_ceph_context));
+      cap->clear_new();
+      e.cap.caps = issue;
+      e.cap.wanted = cap->wanted();
+      e.cap.cap_id = cap->get_cap_id();
+      e.cap.seq = cap->get_last_seq();
+      dout(10) << "encode_inodestat issueing " << ccap_string(issue) << " seq " << cap->get_last_seq() << dendl;
+      e.cap.mseq = cap->get_mseq();
+      e.cap.realm = realm->inode->ino();
+    } else {
+      e.cap.cap_id = 0;
+      e.cap.caps = 0;
+      e.cap.seq = 0;
+      e.cap.mseq = 0;
+      e.cap.realm = 0;
+      e.cap.wanted = 0;
+    }
+  }
+  e.cap.flags = is_auth() ? CEPH_CAP_FLAG_AUTH:0;
+  dout(10) << "encode_inodestat caps " << ccap_string(e.cap.caps)
+	   << " seq " << e.cap.seq << " mseq " << e.cap.mseq
+	   << " xattrv " << e.xattr_version << " len " << xbl.length()
+	   << dendl;
+
+  if (inline_data.length() && cap) {
+    if ((cap->pending() | getattr_caps) & CEPH_CAP_FILE_SHARED) {
+      dout(10) << "including inline version " << inline_version << dendl;
+      cap->client_inline_version = inline_version;
+    } else {
+      dout(10) << "dropping inline version " << inline_version << dendl;
+      inline_version = 0;
+      inline_data.clear();
+    }
+  }
+
+  // include those xattrs?
+  if (xbl.length() && cap) {
+    if ((cap->pending() | getattr_caps) & CEPH_CAP_XATTR_SHARED) {
+      dout(10) << "including xattrs version " << i->xattr_version << dendl;
+      cap->client_xattr_version = i->xattr_version;
+    } else {
+      dout(10) << "dropping xattrs version " << i->xattr_version << dendl;
+      xbl.clear(); // no xattrs .. XXX what's this about?!?
+    }
+  }
+
+  // encode
+  e.fragtree.nsplits = dirfragtree._splits.size();
+  ::encode(e, bl);
+  for (map::iterator p = dirfragtree._splits.begin();
+       p != dirfragtree._splits.end();
+       ++p) {
+    ::encode(p->first, bl);
+    ::encode(p->second, bl);
+  }
+  ::encode(symlink, bl);
+  if (session->connection->has_feature(CEPH_FEATURE_DIRLAYOUTHASH)) {
+    i = pfile ? pi : oi;
+    ::encode(i->dir_layout, bl);
+  }
+  ::encode(xbl, bl);
+  if (session->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA)) {
+    ::encode(inline_version, bl);
+    ::encode(inline_data, bl);
+  }
+
+  return valid;
+}
+
+void CInode::encode_cap_message(MClientCaps *m, Capability *cap)
+{
+  assert(cap);
+
+  client_t client = cap->get_client();
+
+  bool pfile = filelock.is_xlocked_by_client(client) || (cap->issued() & CEPH_CAP_FILE_EXCL);
+  bool pauth = authlock.is_xlocked_by_client(client);
+  bool plink = linklock.is_xlocked_by_client(client);
+  bool pxattr = xattrlock.is_xlocked_by_client(client);
+ 
+  inode_t *oi = &inode;
+  inode_t *pi = get_projected_inode();
+  inode_t *i = (pfile|pauth|plink|pxattr) ? pi : oi;
+  i->ctime.encode_timeval(&m->head.ctime);
+  
+  dout(20) << "encode_cap_message pfile " << pfile
+	   << " pauth " << pauth << " plink " << plink << " pxattr " << pxattr
+	   << " ctime " << i->ctime << dendl;
+
+  i = pfile ? pi:oi;
+  m->head.layout = i->layout;
+  m->head.size = i->size;
+  m->head.truncate_seq = i->truncate_seq;
+  m->head.truncate_size = i->truncate_size;
+  i->mtime.encode_timeval(&m->head.mtime);
+  i->atime.encode_timeval(&m->head.atime);
+  m->head.time_warp_seq = i->time_warp_seq;
+
+  if (cap->client_inline_version < i->inline_version) {
+    m->inline_version = cap->client_inline_version = i->inline_version;
+    m->inline_data = i->inline_data;
+  } else {
+    m->inline_version = 0;
+  }
+
+  // max_size is min of projected, actual.
+  uint64_t oldms = oi->client_ranges.count(client) ? oi->client_ranges[client].range.last : 0;
+  uint64_t newms = pi->client_ranges.count(client) ? pi->client_ranges[client].range.last : 0;
+  m->head.max_size = MIN(oldms, newms);
+
+  i = pauth ? pi:oi;
+  m->head.mode = i->mode;
+  m->head.uid = i->uid;
+  m->head.gid = i->gid;
+
+  i = plink ? pi:oi;
+  m->head.nlink = i->nlink;
+
+  i = pxattr ? pi:oi;
+  map *ix = pxattr ? get_projected_xattrs() : &xattrs;
+  if ((cap->pending() & CEPH_CAP_XATTR_SHARED) &&
+      i->xattr_version > cap->client_xattr_version) {
+    dout(10) << "    including xattrs v " << i->xattr_version << dendl;
+    ::encode(*ix, m->xattrbl);
+    m->head.xattr_version = i->xattr_version;
+    cap->client_xattr_version = i->xattr_version;
+  }
+}
+
+
+
+void CInode::_encode_base(bufferlist& bl)
+{
+  ::encode(first, bl);
+  ::encode(inode, bl);
+  ::encode(symlink, bl);
+  ::encode(dirfragtree, bl);
+  ::encode(xattrs, bl);
+  ::encode(old_inodes, bl);
+  encode_snap(bl);
+}
+void CInode::_decode_base(bufferlist::iterator& p)
+{
+  ::decode(first, p);
+  bool was_anchored = inode.anchored;
+  ::decode(inode, p);
+  if (parent && was_anchored != inode.anchored)
+    parent->adjust_nested_anchors((int)inode.anchored - (int)was_anchored);
+
+  ::decode(symlink, p);
+  ::decode(dirfragtree, p);
+  ::decode(xattrs, p);
+  ::decode(old_inodes, p);
+  decode_snap(p);
+}
+
+void CInode::_encode_locks_full(bufferlist& bl)
+{
+  ::encode(authlock, bl);
+  ::encode(linklock, bl);
+  ::encode(dirfragtreelock, bl);
+  ::encode(filelock, bl);
+  ::encode(xattrlock, bl);
+  ::encode(snaplock, bl);
+  ::encode(nestlock, bl);
+  ::encode(flocklock, bl);
+  ::encode(policylock, bl);
+
+  ::encode(loner_cap, bl);
+}
+void CInode::_decode_locks_full(bufferlist::iterator& p)
+{
+  ::decode(authlock, p);
+  ::decode(linklock, p);
+  ::decode(dirfragtreelock, p);
+  ::decode(filelock, p);
+  ::decode(xattrlock, p);
+  ::decode(snaplock, p);
+  ::decode(nestlock, p);
+  ::decode(flocklock, p);
+  ::decode(policylock, p);
+
+  ::decode(loner_cap, p);
+  set_loner_cap(loner_cap);
+  want_loner_cap = loner_cap;  // for now, we'll eval() shortly.
+}
+
+void CInode::_encode_locks_state_for_replica(bufferlist& bl)
+{
+  authlock.encode_state_for_replica(bl);
+  linklock.encode_state_for_replica(bl);
+  dirfragtreelock.encode_state_for_replica(bl);
+  filelock.encode_state_for_replica(bl);
+  nestlock.encode_state_for_replica(bl);
+  xattrlock.encode_state_for_replica(bl);
+  snaplock.encode_state_for_replica(bl);
+  flocklock.encode_state_for_replica(bl);
+  policylock.encode_state_for_replica(bl);
+}
+void CInode::_encode_locks_state_for_rejoin(bufferlist& bl, int rep)
+{
+  authlock.encode_state_for_replica(bl);
+  linklock.encode_state_for_replica(bl);
+  dirfragtreelock.encode_state_for_rejoin(bl, rep);
+  filelock.encode_state_for_rejoin(bl, rep);
+  nestlock.encode_state_for_rejoin(bl, rep);
+  xattrlock.encode_state_for_replica(bl);
+  snaplock.encode_state_for_replica(bl);
+  flocklock.encode_state_for_replica(bl);
+  policylock.encode_state_for_replica(bl);
+}
+void CInode::_decode_locks_state(bufferlist::iterator& p, bool is_new)
+{
+  authlock.decode_state(p, is_new);
+  linklock.decode_state(p, is_new);
+  dirfragtreelock.decode_state(p, is_new);
+  filelock.decode_state(p, is_new);
+  nestlock.decode_state(p, is_new);
+  xattrlock.decode_state(p, is_new);
+  snaplock.decode_state(p, is_new);
+  flocklock.decode_state(p, is_new);
+  policylock.decode_state(p, is_new);
+}
+void CInode::_decode_locks_rejoin(bufferlist::iterator& p, list& waiters,
+				  list& eval_locks)
+{
+  authlock.decode_state_rejoin(p, waiters);
+  linklock.decode_state_rejoin(p, waiters);
+  dirfragtreelock.decode_state_rejoin(p, waiters);
+  filelock.decode_state_rejoin(p, waiters);
+  nestlock.decode_state_rejoin(p, waiters);
+  xattrlock.decode_state_rejoin(p, waiters);
+  snaplock.decode_state_rejoin(p, waiters);
+  flocklock.decode_state_rejoin(p, waiters);
+  policylock.decode_state_rejoin(p, waiters);
+
+  if (!dirfragtreelock.is_stable() && !dirfragtreelock.is_wrlocked())
+    eval_locks.push_back(&dirfragtreelock);
+  if (!filelock.is_stable() && !filelock.is_wrlocked())
+    eval_locks.push_back(&filelock);
+  if (!nestlock.is_stable() && !nestlock.is_wrlocked())
+    eval_locks.push_back(&nestlock);
+}
+
+
+// IMPORT/EXPORT
+
+void CInode::encode_export(bufferlist& bl)
+{
+  ENCODE_START(5, 4, bl)
+  _encode_base(bl);
+
+  ::encode(state, bl);
+
+  ::encode(pop, bl);
+
+  ::encode(replica_map, bl);
+
+  // include scatterlock info for any bounding CDirs
+  bufferlist bounding;
+  if (inode.is_dir())
+    for (map::iterator p = dirfrags.begin();
+	 p != dirfrags.end();
+	 ++p) {
+      CDir *dir = p->second;
+      if (dir->state_test(CDir::STATE_EXPORTBOUND)) {
+	::encode(p->first, bounding);
+	::encode(dir->fnode.fragstat, bounding);
+	::encode(dir->fnode.accounted_fragstat, bounding);
+	::encode(dir->fnode.rstat, bounding);
+	::encode(dir->fnode.accounted_rstat, bounding);
+	dout(10) << " encoded fragstat/rstat info for " << *dir << dendl;
+      }
+    }
+  ::encode(bounding, bl);
+
+  _encode_locks_full(bl);
+
+  ::encode(fcntl_locks, bl);
+  ::encode(flock_locks, bl);
+  ENCODE_FINISH(bl);
+
+  get(PIN_TEMPEXPORTING);
+}
+
+void CInode::finish_export(utime_t now)
+{
+  state &= MASK_STATE_EXPORT_KEPT;
+
+  pop.zero(now);
+
+  // just in case!
+  //dirlock.clear_updated();
+
+  loner_cap = -1;
+
+  put(PIN_TEMPEXPORTING);
+}
+
+void CInode::decode_import(bufferlist::iterator& p,
+			   LogSegment *ls)
+{
+  DECODE_START(5, p);
+
+  _decode_base(p);
+
+  unsigned s;
+  ::decode(s, p);
+  state |= (s & MASK_STATE_EXPORTED);
+  if (is_dirty()) {
+    get(PIN_DIRTY);
+    _mark_dirty(ls);
+  }
+  if (is_dirty_parent()) {
+    get(PIN_DIRTYPARENT);
+    _mark_dirty_parent(ls);
+  }
+
+  ::decode(pop, ceph_clock_now(g_ceph_context), p);
+
+  ::decode(replica_map, p);
+  if (!replica_map.empty())
+    get(PIN_REPLICATED);
+
+  // decode fragstat info on bounding cdirs
+  bufferlist bounding;
+  ::decode(bounding, p);
+  bufferlist::iterator q = bounding.begin();
+  while (!q.end()) {
+    frag_t fg;
+    ::decode(fg, q);
+    CDir *dir = get_dirfrag(fg);
+    assert(dir);  // we should have all bounds open
+
+    // Only take the remote's fragstat/rstat if we are non-auth for
+    // this dirfrag AND the lock is NOT in a scattered (MIX) state.
+    // We know lock is stable, and MIX is the only state in which
+    // the inode auth (who sent us this data) may not have the best
+    // info.
+
+    // HMM: Are there cases where dir->is_auth() is an insufficient
+    // check because the dirfrag is under migration?  That implies
+    // it is frozen (and in a SYNC or LOCK state).  FIXME.
+
+    if (dir->is_auth() ||
+        filelock.get_state() == LOCK_MIX) {
+      dout(10) << " skipped fragstat info for " << *dir << dendl;
+      frag_info_t f;
+      ::decode(f, q);
+      ::decode(f, q);
+    } else {
+      ::decode(dir->fnode.fragstat, q);
+      ::decode(dir->fnode.accounted_fragstat, q);
+      dout(10) << " took fragstat info for " << *dir << dendl;
+    }
+    if (dir->is_auth() ||
+        nestlock.get_state() == LOCK_MIX) {
+      dout(10) << " skipped rstat info for " << *dir << dendl;
+      nest_info_t n;
+      ::decode(n, q);
+      ::decode(n, q);
+    } else {
+      ::decode(dir->fnode.rstat, q);
+      ::decode(dir->fnode.accounted_rstat, q);
+      dout(10) << " took rstat info for " << *dir << dendl;
+    }
+  }
+
+  _decode_locks_full(p);
+
+  if (struct_v >= 5) {
+    ::decode(fcntl_locks, p);
+    ::decode(flock_locks, p);
+  }
+
+  DECODE_FINISH(p);
+}
diff --git a/ceph/src/mds/CInode.h b/ceph/src/mds/CInode.h
new file mode 100644
index 00000000..cb1add37
--- /dev/null
+++ b/ceph/src/mds/CInode.h
@@ -0,0 +1,865 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_CINODE_H
+#define CEPH_CINODE_H
+
+#include "common/config.h"
+#include "include/dlist.h"
+#include "include/elist.h"
+#include "include/types.h"
+#include "include/lru.h"
+
+#include "mdstypes.h"
+#include "flock.h"
+
+#include "CDentry.h"
+#include "SimpleLock.h"
+#include "ScatterLock.h"
+#include "LocalLock.h"
+#include "Capability.h"
+#include "SnapRealm.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+using namespace std;
+
+class Context;
+class CDentry;
+class CDir;
+class Message;
+class CInode;
+class MDCache;
+class LogSegment;
+struct SnapRealm;
+class Session;
+class MClientCaps;
+struct ObjectOperation;
+class EMetaBlob;
+
+ostream& operator<<(ostream& out, CInode& in);
+
+struct cinode_lock_info_t {
+  int lock;
+  int wr_caps;
+};
+
+extern cinode_lock_info_t cinode_lock_info[];
+extern int num_cinode_locks;
+
+// cached inode wrapper
+class CInode : public MDSCacheObject {
+  /*
+   * This class uses a boost::pool to handle allocation. This is *not*
+   * thread-safe, so don't do allocations from multiple threads!
+   *
+   * Alternatively, switch the pool to use a boost::singleton_pool.
+   */
+
+private:
+  static boost::pool<> pool;
+public:
+  static void *operator new(size_t num_bytes) { 
+    void *n = pool.malloc();
+    if (!n)
+      throw std::bad_alloc();
+    return n;
+  }
+  void operator delete(void *p) {
+    pool.free(p);
+  }
+
+
+ public:
+  // -- pins --
+  static const int PIN_DIRFRAG =         -1; 
+  static const int PIN_CAPS =             2;  // client caps
+  static const int PIN_IMPORTING =       -4;  // importing
+  static const int PIN_ANCHORING =        5;
+  static const int PIN_UNANCHORING =      6;
+  static const int PIN_OPENINGDIR =       7;
+  static const int PIN_REMOTEPARENT =     8;
+  static const int PIN_BATCHOPENJOURNAL = 9;
+  static const int PIN_SCATTERED =        10;
+  static const int PIN_STICKYDIRS =       11;
+  //static const int PIN_PURGING =         -12;	
+  static const int PIN_FREEZING =         13;
+  static const int PIN_FROZEN =           14;
+  static const int PIN_IMPORTINGCAPS =   -15;
+  static const int PIN_PASTSNAPPARENT =  -16;
+  static const int PIN_OPENINGSNAPPARENTS = 17;
+  static const int PIN_TRUNCATING =       18;
+  static const int PIN_STRAY =            19;  // we pin our stray inode while active
+  static const int PIN_NEEDSNAPFLUSH =    20;
+  static const int PIN_DIRTYRSTAT =       21;
+  static const int PIN_EXPORTINGCAPS =    22;
+  static const int PIN_DIRTYPARENT =      23;
+  static const int PIN_DIRWAITER =        24;
+
+  const char *pin_name(int p) {
+    switch (p) {
+    case PIN_DIRFRAG: return "dirfrag";
+    case PIN_CAPS: return "caps";
+    case PIN_IMPORTING: return "importing";
+    case PIN_ANCHORING: return "anchoring";
+    case PIN_UNANCHORING: return "unanchoring";
+    case PIN_OPENINGDIR: return "openingdir";
+    case PIN_REMOTEPARENT: return "remoteparent";
+    case PIN_BATCHOPENJOURNAL: return "batchopenjournal";
+    case PIN_SCATTERED: return "scattered";
+    case PIN_STICKYDIRS: return "stickydirs";
+      //case PIN_PURGING: return "purging";
+    case PIN_FREEZING: return "freezing";
+    case PIN_FROZEN: return "frozen";
+    case PIN_IMPORTINGCAPS: return "importingcaps";
+    case PIN_EXPORTINGCAPS: return "exportingcaps";
+    case PIN_PASTSNAPPARENT: return "pastsnapparent";
+    case PIN_OPENINGSNAPPARENTS: return "openingsnapparents";
+    case PIN_TRUNCATING: return "truncating";
+    case PIN_STRAY: return "stray";
+    case PIN_NEEDSNAPFLUSH: return "needsnapflush";
+    case PIN_DIRTYRSTAT: return "dirtyrstat";
+    case PIN_DIRTYPARENT: return "dirtyparent";
+    case PIN_DIRWAITER: return "dirwaiter";
+    default: return generic_pin_name(p);
+    }
+  }
+
+  // -- state --
+  static const int STATE_EXPORTING =   (1<<2);   // on nonauth bystander.
+  static const int STATE_ANCHORING =   (1<<3);
+  static const int STATE_UNANCHORING = (1<<4);
+  static const int STATE_OPENINGDIR =  (1<<5);
+  static const int STATE_FREEZING =    (1<<7);
+  static const int STATE_FROZEN =      (1<<8);
+  static const int STATE_AMBIGUOUSAUTH = (1<<9);
+  static const int STATE_EXPORTINGCAPS = (1<<10);
+  static const int STATE_NEEDSRECOVER = (1<<11);
+  static const int STATE_RECOVERING =   (1<<12);
+  static const int STATE_PURGING =     (1<<13);
+  static const int STATE_DIRTYPARENT =  (1<<14);
+  static const int STATE_DIRTYRSTAT =  (1<<15);
+  static const int STATE_STRAYPINNED = (1<<16);
+  static const int STATE_FROZENAUTHPIN = (1<<17);
+  static const int STATE_DIRTYPOOL =   (1<<18);
+  // orphan inode needs notification of releasing reference
+  static const int STATE_ORPHAN =	STATE_NOTIFYREF;
+
+  static const int MASK_STATE_EXPORTED =
+    (STATE_DIRTY|STATE_NEEDSRECOVER|STATE_DIRTYPARENT|STATE_DIRTYPOOL);
+  static const int MASK_STATE_EXPORT_KEPT =
+    (STATE_FROZEN|STATE_AMBIGUOUSAUTH|STATE_EXPORTINGCAPS);
+
+  // -- waiters --
+  static const uint64_t WAIT_DIR         = (1<<0);
+  static const uint64_t WAIT_ANCHORED    = (1<<1);
+  static const uint64_t WAIT_UNANCHORED  = (1<<2);
+  static const uint64_t WAIT_FROZEN      = (1<<3);
+  static const uint64_t WAIT_TRUNC       = (1<<4);
+  static const uint64_t WAIT_FLOCK       = (1<<5);
+  
+  static const uint64_t WAIT_ANY_MASK	= (uint64_t)(-1);
+
+  // misc
+  static const unsigned EXPORT_NONCE = 1; // nonce given to replicas created by export
+
+  ostream& print_db_line_prefix(ostream& out);
+
+ public:
+  MDCache *mdcache;
+
+  // inode contents proper
+  inode_t          inode;        // the inode itself
+  string           symlink;      // symlink dest, if symlink
+  map xattrs;
+  fragtree_t       dirfragtree;  // dir frag tree, if any.  always consistent with our dirfrag map.
+  SnapRealm        *snaprealm;
+
+  SnapRealm        *containing_realm;
+  snapid_t          first, last;
+  map old_inodes;  // key = last, value.first = first
+  set dirty_old_rstats;
+
+  bool is_multiversion() {
+    return snaprealm ||  // other snaprealms will link to me
+      inode.is_dir() ||  // links to me in other snaps
+      inode.nlink > 1 || // there are remote links, possibly snapped, that will need to find me
+      !old_inodes.empty(); // once multiversion, always multiversion.  until old_inodes gets cleaned out.
+  }
+  snapid_t get_oldest_snap();
+
+  uint64_t last_journaled;       // log offset for the last time i was journaled
+  //loff_t last_open_journaled;  // log offset for the last journaled EOpen
+  utime_t last_dirstat_prop;
+
+
+  // list item node for when we have unpropagated rstat data
+  elist::item dirty_rstat_item;
+
+  bool is_dirty_rstat() {
+    return state_test(STATE_DIRTYRSTAT);
+  }
+  void mark_dirty_rstat();
+  void clear_dirty_rstat();
+
+  //bool hack_accessed;
+  //utime_t hack_load_stamp;
+
+  /**
+   * Projection methods, used to store inode changes until they have been journaled,
+   * at which point they are popped.
+   * Usage:
+   * project_inode as needed. If you're also projecting xattrs, pass
+   * in an xattr map (by pointer), then edit the map.
+   * If you're also projecting the snaprealm, call project_snaprealm after
+   * calling project_inode, and modify the snaprealm as necessary.
+   *
+   * Then, journal. Once journaling is done, pop_and_dirty_projected_inode.
+   * This function will take care of the inode itself, the xattrs, and the snaprealm.
+   */
+
+  struct projected_inode_t {
+    inode_t *inode;
+    map *xattrs;
+    sr_t *snapnode;
+
+    projected_inode_t()
+      : inode(NULL), xattrs(NULL), snapnode(NULL) {}
+    projected_inode_t(inode_t *in, sr_t *sn)
+      : inode(in), xattrs(NULL), snapnode(sn) {}
+    projected_inode_t(inode_t *in, map *xp = NULL, sr_t *sn = NULL)
+      : inode(in), xattrs(xp), snapnode(sn) {}
+  };
+  list projected_nodes;   // projected values (only defined while dirty)
+  
+  inode_t *project_inode(map *px=0);
+  void pop_and_dirty_projected_inode(LogSegment *ls);
+
+  projected_inode_t *get_projected_node() {
+    if (projected_nodes.empty())
+      return NULL;
+    else
+      return projected_nodes.back();
+  }
+
+  version_t get_projected_version() {
+    if (projected_nodes.empty())
+      return inode.version;
+    else
+      return projected_nodes.back()->inode->version;
+  }
+  bool is_projected() {
+    return !projected_nodes.empty();
+  }
+
+  inode_t *get_projected_inode() { 
+    if (projected_nodes.empty())
+      return &inode;
+    else
+      return projected_nodes.back()->inode;
+  }
+  inode_t *get_previous_projected_inode() {
+    assert(!projected_nodes.empty());
+    list::reverse_iterator p = projected_nodes.rbegin();
+    ++p;
+    if (p != projected_nodes.rend())
+      return (*p)->inode;
+    else
+      return &inode;
+  }
+
+  map *get_projected_xattrs() {
+    for (list::reverse_iterator p = projected_nodes.rbegin();
+	 p != projected_nodes.rend();
+	 ++p)
+      if ((*p)->xattrs)
+	return (*p)->xattrs;
+    return &xattrs;
+  }
+  map *get_previous_projected_xattrs() {
+    list::reverse_iterator p = projected_nodes.rbegin();
+    for (p++;  // skip the most recent projected value
+	 p != projected_nodes.rend();
+	 ++p)
+      if ((*p)->xattrs)
+	return (*p)->xattrs;
+    return &xattrs;
+  }
+
+  sr_t *project_snaprealm(snapid_t snapid=0);
+  sr_t *get_projected_srnode() {
+    if (projected_nodes.empty()) {
+      if (snaprealm)
+	return &snaprealm->srnode;
+      else
+	return NULL;
+    } else {
+      for (list::reverse_iterator p = projected_nodes.rbegin();
+          p != projected_nodes.rend();
+          ++p)
+        if ((*p)->snapnode)
+          return (*p)->snapnode;
+    }
+    return &snaprealm->srnode;
+  }
+  void project_past_snaprealm_parent(SnapRealm *newparent);
+
+private:
+  void pop_projected_snaprealm(sr_t *next_snaprealm);
+
+public:
+  old_inode_t& cow_old_inode(snapid_t follows, bool cow_head);
+  old_inode_t *pick_old_inode(snapid_t last);
+  void pre_cow_old_inode();
+  void purge_stale_snap_data(const set& snaps);
+
+  // -- cache infrastructure --
+private:
+  map dirfrags; // cached dir fragments under this Inode
+  int stickydir_ref;
+
+public:
+  __u32 hash_dentry_name(const string &dn);
+  frag_t pick_dirfrag(const string &dn);
+  bool has_dirfrags() { return !dirfrags.empty(); }
+  CDir* get_dirfrag(frag_t fg) {
+    if (dirfrags.count(fg)) {
+      //assert(g_conf->debug_mds < 2 || dirfragtree.is_leaf(fg)); // performance hack FIXME
+      return dirfrags[fg];
+    } else
+      return NULL;
+  }
+  bool get_dirfrags_under(frag_t fg, list& ls);
+  CDir* get_approx_dirfrag(frag_t fg);
+  void get_dirfrags(list& ls);
+  void get_nested_dirfrags(list& ls);
+  void get_subtree_dirfrags(list& ls);
+  CDir *get_or_open_dirfrag(MDCache *mdcache, frag_t fg);
+  CDir *add_dirfrag(CDir *dir);
+  void close_dirfrag(frag_t fg);
+  void close_dirfrags();
+  bool has_subtree_root_dirfrag(int auth=-1);
+  bool has_subtree_or_exporting_dirfrag();
+
+  void force_dirfrags();
+  void verify_dirfrags();
+
+  void get_stickydirs();
+  void put_stickydirs();  
+
+ protected:
+  // parent dentries in cache
+  CDentry         *parent;             // primary link
+  set    remote_parents;     // if hard linked
+
+  list   projected_parent;   // for in-progress rename, (un)link, etc.
+
+  pair inode_auth;
+
+  // -- distributed state --
+protected:
+  // file capabilities
+  map client_caps;         // client -> caps
+  map      mds_caps_wanted;     // [auth] mds -> caps wanted
+  int                   replica_caps_wanted; // [replica] what i've requested from auth
+
+  map > client_snap_caps;     // [auth] [snap] dirty metadata we still need from the head
+public:
+  map > client_need_snapflush;
+
+  void add_need_snapflush(CInode *snapin, snapid_t snapid, client_t client);
+  void remove_need_snapflush(CInode *snapin, snapid_t snapid, client_t client);
+
+protected:
+
+  ceph_lock_state_t fcntl_locks;
+  ceph_lock_state_t flock_locks;
+
+  void clear_file_locks() {
+    fcntl_locks.clear();
+    flock_locks.clear();
+  }
+
+  // LogSegment dlists i (may) belong to
+public:
+  elist::item item_dirty;
+  elist::item item_caps;
+  elist::item item_open_file;
+  elist::item item_dirty_parent;
+  elist::item item_dirty_dirfrag_dir;
+  elist::item item_dirty_dirfrag_nest;
+  elist::item item_dirty_dirfrag_dirfragtree;
+
+private:
+  // auth pin
+  int auth_pins;
+  int nested_auth_pins;
+public:
+#ifdef MDS_AUTHPIN_SET
+  multiset auth_pin_set;
+#endif
+  int auth_pin_freeze_allowance;
+
+private:
+  int nested_anchors;   // _NOT_ including me!
+
+ public:
+  inode_load_vec_t pop;
+
+  // friends
+  friend class Server;
+  friend class Locker;
+  friend class Migrator;
+  friend class MDCache;
+  friend class CDir;
+  friend class CInodeExport;
+
+ public:
+  // ---------------------------
+  CInode(MDCache *c, bool auth=true, snapid_t f=2, snapid_t l=CEPH_NOSNAP) : 
+    mdcache(c),
+    snaprealm(0), containing_realm(0),
+    first(f), last(l),
+    last_journaled(0), //last_open_journaled(0), 
+    //hack_accessed(true),
+    stickydir_ref(0),
+    parent(0),
+    inode_auth(CDIR_AUTH_DEFAULT),
+    replica_caps_wanted(0),
+    item_dirty(this), item_caps(this), item_open_file(this), item_dirty_parent(this),
+    item_dirty_dirfrag_dir(this), 
+    item_dirty_dirfrag_nest(this), 
+    item_dirty_dirfrag_dirfragtree(this), 
+    auth_pins(0), nested_auth_pins(0),
+    auth_pin_freeze_allowance(0),
+    nested_anchors(0),
+    pop(ceph_clock_now(g_ceph_context)),
+    versionlock(this, &versionlock_type),
+    authlock(this, &authlock_type),
+    linklock(this, &linklock_type),
+    dirfragtreelock(this, &dirfragtreelock_type),
+    filelock(this, &filelock_type),
+    xattrlock(this, &xattrlock_type),
+    snaplock(this, &snaplock_type),
+    nestlock(this, &nestlock_type),
+    flocklock(this, &flocklock_type),
+    policylock(this, &policylock_type),
+    loner_cap(-1), want_loner_cap(-1)
+  {
+    g_num_ino++;
+    g_num_inoa++;
+    state = 0;  
+    if (auth) state_set(STATE_AUTH);
+  };
+  ~CInode() {
+    g_num_ino--;
+    g_num_inos++;
+    close_dirfrags();
+    close_snaprealm();
+  }
+  
+
+  // -- accessors --
+  bool is_file()    { return inode.is_file(); }
+  bool is_symlink() { return inode.is_symlink(); }
+  bool is_dir()     { return inode.is_dir(); }
+
+  bool is_anchored() { return inode.anchored; }
+  bool is_anchoring() { return state_test(STATE_ANCHORING); }
+  bool is_unanchoring() { return state_test(STATE_UNANCHORING); }
+  
+  bool is_root() { return inode.ino == MDS_INO_ROOT; }
+  bool is_stray() { return MDS_INO_IS_STRAY(inode.ino); }
+  bool is_mdsdir() { return MDS_INO_IS_MDSDIR(inode.ino); }
+  bool is_base() { return is_root() || is_mdsdir(); }
+  bool is_system() { return inode.ino < MDS_INO_SYSTEM_BASE; }
+
+  bool is_head() { return last == CEPH_NOSNAP; }
+
+  // note: this overloads MDSCacheObject
+  bool is_ambiguous_auth() {
+    return state_test(STATE_AMBIGUOUSAUTH) ||
+      MDSCacheObject::is_ambiguous_auth();
+  }
+  void set_ambiguous_auth() {
+    state_set(STATE_AMBIGUOUSAUTH);
+  }
+  void clear_ambiguous_auth(list& finished);
+  void clear_ambiguous_auth();
+
+  inodeno_t ino() const { return inode.ino; }
+  vinodeno_t vino() const { return vinodeno_t(inode.ino, last); }
+  int d_type() const { return IFTODT(inode.mode); }
+
+  inode_t& get_inode() { return inode; }
+  CDentry* get_parent_dn() { return parent; }
+  CDentry* get_projected_parent_dn() { return !projected_parent.empty() ? projected_parent.back() : parent; }
+  CDir *get_parent_dir();
+  CDir *get_projected_parent_dir();
+  CInode *get_parent_inode();
+  
+  bool is_lt(const MDSCacheObject *r) const {
+    const CInode *o = static_cast(r);
+    return ino() < o->ino() ||
+      (ino() == o->ino() && last < o->last);
+  }
+
+  // -- misc -- 
+  bool is_projected_ancestor_of(CInode *other);
+  void make_path_string(string& s, bool force=false, CDentry *use_parent=NULL);
+  void make_path_string_projected(string& s);  
+  void make_path(filepath& s);
+  void make_anchor_trace(vector& trace);
+  void name_stray_dentry(string& dname);
+
+
+  static object_t get_object_name(inodeno_t ino, frag_t fg, const char *suffix);
+
+  
+  // -- dirtyness --
+  version_t get_version() { return inode.version; }
+
+  version_t pre_dirty();
+  void _mark_dirty(LogSegment *ls);
+  void mark_dirty(version_t projected_dirv, LogSegment *ls);
+  void mark_clean();
+
+  void store(Context *fin);
+  void _stored(version_t cv, Context *fin);
+  void fetch(Context *fin);
+  void _fetched(bufferlist& bl, bufferlist& bl2, Context *fin);  
+
+  void build_backtrace(int64_t pool, inode_backtrace_t& bt);
+  void store_backtrace(Context *fin);
+  void _stored_backtrace(version_t v, Context *fin);
+  void _mark_dirty_parent(LogSegment *ls, bool dirty_pool=false);
+  void clear_dirty_parent();
+  bool is_dirty_parent() { return state_test(STATE_DIRTYPARENT); }
+  bool is_dirty_pool() { return state_test(STATE_DIRTYPOOL); }
+
+  void encode_store(bufferlist& bl);
+  void decode_store(bufferlist::iterator& bl);
+
+  void encode_replica(int rep, bufferlist& bl) {
+    assert(is_auth());
+    
+    // relax locks?
+    if (!is_replicated())
+      replicate_relax_locks();
+    
+    __u32 nonce = add_replica(rep);
+    ::encode(nonce, bl);
+    
+    _encode_base(bl);
+    _encode_locks_state_for_replica(bl);
+  }
+  void decode_replica(bufferlist::iterator& p, bool is_new) {
+    __u32 nonce;
+    ::decode(nonce, p);
+    replica_nonce = nonce;
+    
+    _decode_base(p);
+    _decode_locks_state(p, is_new);
+  }
+
+  // -- waiting --
+protected:
+  map > waiting_on_dir;
+public:
+  void add_dir_waiter(frag_t fg, Context *c);
+  void take_dir_waiting(frag_t fg, list& ls);
+  bool is_waiting_for_dir(frag_t fg) {
+    return waiting_on_dir.count(fg);
+  }
+  void add_waiter(uint64_t tag, Context *c);
+  void take_waiting(uint64_t tag, list& ls);
+
+  // -- encode/decode helpers --
+  void _encode_base(bufferlist& bl);
+  void _decode_base(bufferlist::iterator& p);
+  void _encode_locks_full(bufferlist& bl);
+  void _decode_locks_full(bufferlist::iterator& p);
+  void _encode_locks_state_for_replica(bufferlist& bl);
+  void _encode_locks_state_for_rejoin(bufferlist& bl, int rep);
+  void _decode_locks_state(bufferlist::iterator& p, bool is_new);
+  void _decode_locks_rejoin(bufferlist::iterator& p, list& waiters,
+			    list& eval_locks);
+
+  // -- import/export --
+  void encode_export(bufferlist& bl);
+  void finish_export(utime_t now);
+  void abort_export() {
+    put(PIN_TEMPEXPORTING);
+    assert(state_test(STATE_EXPORTINGCAPS));
+    state_clear(STATE_EXPORTINGCAPS);
+    put(PIN_EXPORTINGCAPS);
+  }
+  void decode_import(bufferlist::iterator& p, LogSegment *ls);
+  
+
+  // for giving to clients
+  int encode_inodestat(bufferlist& bl, Session *session, SnapRealm *realm,
+		       snapid_t snapid=CEPH_NOSNAP, unsigned max_bytes=0,
+		       int getattr_wants=0);
+  void encode_cap_message(MClientCaps *m, Capability *cap);
+
+
+  // -- locks --
+public:
+  static LockType versionlock_type;
+  static LockType authlock_type;
+  static LockType linklock_type;
+  static LockType dirfragtreelock_type;
+  static LockType filelock_type;
+  static LockType xattrlock_type;
+  static LockType snaplock_type;
+  static LockType nestlock_type;
+  static LockType flocklock_type;
+  static LockType policylock_type;
+
+  LocalLock  versionlock;
+  SimpleLock authlock;
+  SimpleLock linklock;
+  ScatterLock dirfragtreelock;
+  ScatterLock filelock;
+  SimpleLock xattrlock;
+  SimpleLock snaplock;
+  ScatterLock nestlock;
+  SimpleLock flocklock;
+  SimpleLock policylock;
+
+  SimpleLock* get_lock(int type) {
+    switch (type) {
+    case CEPH_LOCK_IFILE: return &filelock;
+    case CEPH_LOCK_IAUTH: return &authlock;
+    case CEPH_LOCK_ILINK: return &linklock;
+    case CEPH_LOCK_IDFT: return &dirfragtreelock;
+    case CEPH_LOCK_IXATTR: return &xattrlock;
+    case CEPH_LOCK_ISNAP: return &snaplock;
+    case CEPH_LOCK_INEST: return &nestlock;
+    case CEPH_LOCK_IFLOCK: return &flocklock;
+    case CEPH_LOCK_IPOLICY: return &policylock;
+    }
+    return 0;
+  }
+
+  void set_object_info(MDSCacheObjectInfo &info);
+  void encode_lock_state(int type, bufferlist& bl);
+  void decode_lock_state(int type, bufferlist& bl);
+
+  void _finish_frag_update(CDir *dir, MutationRef& mut);
+
+  void clear_dirty_scattered(int type);
+  bool is_dirty_scattered();
+  void clear_scatter_dirty();  // on rejoin ack
+
+  void start_scatter(ScatterLock *lock);
+  void finish_scatter_update(ScatterLock *lock, CDir *dir,
+			     version_t inode_version, version_t dir_accounted_version);
+  void finish_scatter_gather_update(int type);
+  void finish_scatter_gather_update_accounted(int type, MutationRef& mut, EMetaBlob *metablob);
+
+  // -- snap --
+  void open_snaprealm(bool no_split=false);
+  void close_snaprealm(bool no_join=false);
+  SnapRealm *find_snaprealm();
+  void encode_snap_blob(bufferlist &bl);
+  void decode_snap_blob(bufferlist &bl);
+  void encode_snap(bufferlist& bl);
+  void decode_snap(bufferlist::iterator& p);
+
+  // -- caps -- (new)
+  // client caps
+  client_t loner_cap, want_loner_cap;
+
+  client_t get_loner() { return loner_cap; }
+  client_t get_wanted_loner() { return want_loner_cap; }
+
+  // this is the loner state our locks should aim for
+  client_t get_target_loner() {
+    if (loner_cap == want_loner_cap)
+      return loner_cap;
+    else
+      return -1;
+  }
+
+  client_t calc_ideal_loner();
+  client_t choose_ideal_loner();
+  bool try_set_loner();
+  void set_loner_cap(client_t l);
+  bool try_drop_loner();
+
+  // choose new lock state during recovery, based on issued caps
+  void choose_lock_state(SimpleLock *lock, int allissued);
+  void choose_lock_states();
+
+  int count_nonstale_caps() {
+    int n = 0;
+    for (map::iterator it = client_caps.begin();
+         it != client_caps.end();
+         ++it) 
+      if (!it->second->is_stale())
+	n++;
+    return n;
+  }
+  bool multiple_nonstale_caps() {
+    int n = 0;
+    for (map::iterator it = client_caps.begin();
+         it != client_caps.end();
+         ++it) 
+      if (!it->second->is_stale()) {
+	if (n)
+	  return true;
+	n++;
+      }
+    return false;
+  }
+
+  bool is_any_caps() { return !client_caps.empty(); }
+  bool is_any_nonstale_caps() { return count_nonstale_caps(); }
+
+  map& get_mds_caps_wanted() { return mds_caps_wanted; }
+
+  map& get_client_caps() { return client_caps; }
+  Capability *get_client_cap(client_t client) {
+    if (client_caps.count(client))
+      return client_caps[client];
+    return 0;
+  }
+  int get_client_cap_pending(client_t client) {
+    Capability *c = get_client_cap(client);
+    if (c) return c->pending();
+    return 0;
+  }
+
+  Capability *add_client_cap(client_t client, Session *session, SnapRealm *conrealm=0);
+  void remove_client_cap(client_t client);
+  void move_to_realm(SnapRealm *realm);
+
+  Capability *reconnect_cap(client_t client, ceph_mds_cap_reconnect& icr, Session *session);
+  void clear_client_caps_after_export();
+  void export_client_caps(map& cl);
+
+  // caps allowed
+  int get_caps_liked();
+  int get_caps_allowed_ever();
+  int get_caps_allowed_by_type(int type);
+  int get_caps_careful();
+  int get_xlocker_mask(client_t client);
+  int get_caps_allowed_for_client(client_t client);
+
+  // caps issued, wanted
+  int get_caps_issued(int *ploner = 0, int *pother = 0, int *pxlocker = 0,
+		      int shift = 0, int mask = -1);
+  bool is_any_caps_wanted();
+  int get_caps_wanted(int *ploner = 0, int *pother = 0, int shift = 0, int mask = -1);
+  bool issued_caps_need_gather(SimpleLock *lock);
+  void replicate_relax_locks();
+
+
+  // -- authority --
+  pair authority();
+
+
+  // -- auth pins --
+  bool is_auth_pinned() { return auth_pins || nested_auth_pins; }
+  int get_num_auth_pins() { return auth_pins; }
+  int get_num_nested_auth_pins() { return nested_auth_pins; }
+  void adjust_nested_auth_pins(int a, void *by);
+  bool can_auth_pin();
+  void auth_pin(void *by);
+  void auth_unpin(void *by);
+
+  void adjust_nested_anchors(int by);
+  int get_nested_anchors() { return nested_anchors; }
+
+  // -- freeze --
+  bool is_freezing_inode() { return state_test(STATE_FREEZING); }
+  bool is_frozen_inode() { return state_test(STATE_FROZEN); }
+  bool is_frozen_auth_pin() { return state_test(STATE_FROZENAUTHPIN); }
+  bool is_frozen();
+  bool is_frozen_dir();
+  bool is_freezing();
+
+  /* Freeze the inode. auth_pin_allowance lets the caller account for any
+   * auth_pins it is itself holding/responsible for. */
+  bool freeze_inode(int auth_pin_allowance=0);
+  void unfreeze_inode(list& finished);
+  void unfreeze_inode();
+
+  void freeze_auth_pin();
+  void unfreeze_auth_pin();
+
+  // -- reference counting --
+  void bad_put(int by) {
+    generic_dout(0) << " bad put " << *this << " by " << by << " " << pin_name(by) << " was " << ref
+#ifdef MDS_REF_SET
+		    << " (" << ref_map << ")"
+#endif
+		    << dendl;
+#ifdef MDS_REF_SET
+    assert(ref_map[by] > 0);
+#endif
+    assert(ref > 0);
+  }
+  void bad_get(int by) {
+    generic_dout(0) << " bad get " << *this << " by " << by << " " << pin_name(by) << " was " << ref
+#ifdef MDS_REF_SET
+		    << " (" << ref_map << ")"
+#endif
+		    << dendl;
+#ifdef MDS_REF_SET
+    assert(ref_map[by] >= 0);
+#endif
+  }
+  void first_get();
+  void last_put();
+  void _put();
+
+
+  // -- hierarchy stuff --
+public:
+  void set_primary_parent(CDentry *p) {
+    assert(parent == 0);
+    parent = p;
+  }
+  void remove_primary_parent(CDentry *dn) {
+    assert(dn == parent);
+    parent = 0;
+  }
+  void add_remote_parent(CDentry *p);
+  void remove_remote_parent(CDentry *p);
+  int num_remote_parents() {
+    return remote_parents.size(); 
+  }
+
+  void push_projected_parent(CDentry *dn) {
+    projected_parent.push_back(dn);
+  }
+  void pop_projected_parent() {
+    assert(projected_parent.size());
+    parent = projected_parent.front();
+    projected_parent.pop_front();
+  }
+
+  void print(ostream& out);
+
+};
+
+#endif
diff --git a/ceph/src/mds/Capability.cc b/ceph/src/mds/Capability.cc
new file mode 100644
index 00000000..6a68fd76
--- /dev/null
+++ b/ceph/src/mds/Capability.cc
@@ -0,0 +1,202 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "Capability.h"
+
+#include "common/Formatter.h"
+
+
+/*
+ * Capability::Export
+ */
+
+void Capability::Export::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(cap_id, bl);
+  ::encode(wanted, bl);
+  ::encode(issued, bl);
+  ::encode(pending, bl);
+  ::encode(client_follows, bl);
+  ::encode(seq, bl);
+  ::encode(mseq, bl);
+  ::encode(last_issue_stamp, bl);
+  ENCODE_FINISH(bl);
+}
+
+void Capability::Export::decode(bufferlist::iterator &p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, p);
+  ::decode(cap_id, p);
+  ::decode(wanted, p);
+  ::decode(issued, p);
+  ::decode(pending, p);
+  ::decode(client_follows, p);
+  ::decode(seq, p);
+  ::decode(mseq, p);
+  ::decode(last_issue_stamp, p);
+  DECODE_FINISH(p);
+}
+
+void Capability::Export::dump(Formatter *f) const
+{
+  f->dump_unsigned("cap_id", cap_id);
+  f->dump_unsigned("wanted", wanted);
+  f->dump_unsigned("issued", issued);
+  f->dump_unsigned("pending", pending);
+  f->dump_unsigned("client_follows", client_follows);
+  f->dump_unsigned("seq", seq);
+  f->dump_unsigned("migrate_seq", mseq);
+  f->dump_stream("last_issue_stamp") << last_issue_stamp;
+}
+
+void Capability::Export::generate_test_instances(list& ls)
+{
+  ls.push_back(new Export);
+  ls.push_back(new Export);
+  ls.back()->wanted = 1;
+  ls.back()->issued = 2;
+  ls.back()->pending = 3;
+  ls.back()->client_follows = 4;
+  ls.back()->mseq = 5;
+  ls.back()->last_issue_stamp = utime_t(6, 7);
+}
+
+void Capability::Import::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(cap_id, bl);
+  ::encode(issue_seq, bl);
+  ::encode(mseq, bl);
+  ENCODE_FINISH(bl);
+}
+
+void Capability::Import::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(cap_id, bl);
+  ::decode(issue_seq, bl);
+  ::decode(mseq, bl);
+  DECODE_FINISH(bl);
+}
+
+void Capability::Import::dump(Formatter *f) const
+{
+  f->dump_unsigned("cap_id", cap_id);
+  f->dump_unsigned("issue_seq", issue_seq);
+  f->dump_unsigned("migrate_seq", mseq);
+}
+
+/*
+ * Capability::revoke_info
+ */
+
+void Capability::revoke_info::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl)
+  ::encode(before, bl);
+  ::encode(seq, bl);
+  ::encode(last_issue, bl);
+  ENCODE_FINISH(bl);
+}
+
+void Capability::revoke_info::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(before, bl);
+  ::decode(seq, bl);
+  ::decode(last_issue, bl);
+  DECODE_FINISH(bl);
+}
+
+void Capability::revoke_info::dump(Formatter *f) const
+{
+  f->dump_unsigned("before", before);
+  f->dump_unsigned("seq", seq);
+  f->dump_unsigned("last_issue", last_issue);
+}
+
+void Capability::revoke_info::generate_test_instances(list& ls)
+{
+  ls.push_back(new revoke_info);
+  ls.push_back(new revoke_info);
+  ls.back()->before = 1;
+  ls.back()->seq = 2;
+  ls.back()->last_issue = 3;
+}
+
+
+/*
+ * Capability
+ */
+
+void Capability::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl)
+  ::encode(last_sent, bl);
+  ::encode(last_issue_stamp, bl);
+
+  ::encode(_wanted, bl);
+  ::encode(_pending, bl);
+  ::encode(_revokes, bl);
+  ENCODE_FINISH(bl);
+}
+
+void Capability::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl)
+  ::decode(last_sent, bl);
+  ::decode(last_issue_stamp, bl);
+
+  ::decode(_wanted, bl);
+  ::decode(_pending, bl);
+  ::decode(_revokes, bl);
+  DECODE_FINISH(bl);
+  
+  _calc_issued();
+}
+
+void Capability::dump(Formatter *f) const
+{
+  f->dump_unsigned("last_sent", last_sent);
+  f->dump_unsigned("last_issue_stamp", last_issue_stamp);
+  f->dump_unsigned("wanted", _wanted);
+  f->dump_unsigned("pending", _pending);
+
+  f->open_array_section("revokes");
+  for (list::const_iterator p = _revokes.begin(); p != _revokes.end(); ++p) {
+    f->open_object_section("revoke");
+    p->dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void Capability::generate_test_instances(list& ls)
+{
+  ls.push_back(new Capability);
+  ls.push_back(new Capability);
+  ls.back()->last_sent = 11;
+  ls.back()->last_issue_stamp = utime_t(12, 13);
+  ls.back()->_wanted = 14;
+  ls.back()->_pending = 15;
+  ls.back()->_revokes.push_back(revoke_info());
+  ls.back()->_revokes.back().before = 16;
+  ls.back()->_revokes.back().seq = 17;
+  ls.back()->_revokes.back().last_issue = 18;
+  ls.back()->_revokes.push_back(revoke_info());
+  ls.back()->_revokes.back().before = 19;
+  ls.back()->_revokes.back().seq = 20;
+  ls.back()->_revokes.back().last_issue = 21;
+}
diff --git a/ceph/src/mds/Capability.h b/ceph/src/mds/Capability.h
new file mode 100644
index 00000000..3d976487
--- /dev/null
+++ b/ceph/src/mds/Capability.h
@@ -0,0 +1,358 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_CAPABILITY_H
+#define CEPH_CAPABILITY_H
+
+#include "include/buffer.h"
+#include "include/xlist.h"
+
+#include "common/config.h"
+
+#include "mdstypes.h"
+
+/*
+
+  Capability protocol notes.
+
+- two types of cap events from mds -> client:
+  - cap "issue" in a MClientReply, or an MClientCaps IMPORT op.
+  - cap "update" (revocation or grant) .. an MClientCaps message.
+- if client has cap, the mds should have it too.
+
+- if client has no dirty data, it can release it without waiting for an mds ack.
+  - client may thus get a cap _update_ and not have the cap.  ignore it.
+
+- mds should track seq of last issue.  any release
+  attempt will only succeed if the client has seen the latest.
+
+- a UPDATE updates the clients issued caps, wanted, etc.  it may also flush dirty metadata.
+  - 'caps' are which caps the client retains.
+    - if 0, client wishes to release the cap
+  - 'wanted' is which caps the client wants.
+  - 'dirty' is which metadata is to be written.
+    - client gets a FLUSH_ACK with matching dirty flags indicating which caps were written.
+
+- a FLUSH_ACK acks a FLUSH.
+  - 'dirty' is the _original_ FLUSH's dirty (i.e., which metadata was written back)
+  - 'seq' is the _original_ FLUSH's seq.
+  - 'caps' is the _original_ FLUSH's caps (not actually important)
+  - client can conclude that (dirty & ~caps) bits were successfully cleaned.
+
+- a FLUSHSNAP flushes snapshot metadata.
+  - 'dirty' indicates which caps, were dirty, if any.
+  - mds writes metadata.  if dirty!=0, replies with FLUSHSNAP_ACK.
+
+ */
+
+class CInode;
+
+namespace ceph {
+  class Formatter;
+}
+
+class Capability {
+private:
+  static boost::pool<> pool;
+public:
+  static void *operator new(size_t num_bytes) { 
+    void *n = pool.malloc();
+    if (!n)
+      throw std::bad_alloc();
+    return n;
+  }
+  void operator delete(void *p) {
+    pool.free(p);
+  }
+public:
+  struct Export {
+    int64_t cap_id;
+    int32_t wanted;
+    int32_t issued;
+    int32_t pending;
+    snapid_t client_follows;
+    ceph_seq_t seq;
+    ceph_seq_t mseq;
+    utime_t last_issue_stamp;
+    Export() {}
+    Export(int64_t id, int w, int i, int p, snapid_t cf, ceph_seq_t s, ceph_seq_t m, utime_t lis) :
+      cap_id(id), wanted(w), issued(i), pending(p), client_follows(cf),
+      seq(s), mseq(m), last_issue_stamp(lis) {}
+    void encode(bufferlist &bl) const;
+    void decode(bufferlist::iterator &p);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+  };
+  struct Import {
+    int64_t cap_id;
+    ceph_seq_t issue_seq;
+    ceph_seq_t mseq;
+    Import() {}
+    Import(int64_t i, ceph_seq_t s, ceph_seq_t m) : cap_id(i), issue_seq(s), mseq(m) {}
+    void encode(bufferlist &bl) const;
+    void decode(bufferlist::iterator &p);
+    void dump(Formatter *f) const;
+  };
+
+private:
+  CInode *inode;
+  client_t client;
+
+  uint64_t cap_id;
+
+  __u32 _wanted;     // what the client wants (ideally)
+
+  utime_t last_issue_stamp;
+
+
+  // track in-flight caps --------------
+  //  - add new caps to _pending
+  //  - track revocations in _revokes list
+public:
+  struct revoke_info {
+    __u32 before;
+    ceph_seq_t seq, last_issue;
+    revoke_info() {}
+    revoke_info(__u32 b, ceph_seq_t s, ceph_seq_t li) : before(b), seq(s), last_issue(li) {}
+    void encode(bufferlist& bl) const;
+    void decode(bufferlist::iterator& bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+  };
+private:
+  __u32 _pending, _issued;
+  list _revokes;
+
+public:
+  int pending() { return _pending; }
+  int issued() { return _issued; }
+  bool is_null() { return !_pending && _revokes.empty(); }
+
+  ceph_seq_t issue(unsigned c) {
+    if (_pending & ~c) {
+      // revoking (and maybe adding) bits.  note caps prior to this revocation
+      _revokes.push_back(revoke_info(_pending, last_sent, last_issue));
+      _pending = c;
+      _issued |= c;
+    } else if (~_pending & c) {
+      // adding bits only.  remove obsolete revocations?
+      _pending |= c;
+      _issued |= c;
+      // drop old _revokes with no bits we don't have
+      while (!_revokes.empty() &&
+	     (_revokes.back().before & ~_pending) == 0)
+	_revokes.pop_back();
+    } else {
+      // no change.
+      assert(_pending == c);
+    }
+    //last_issue = 
+    ++last_sent;
+    return last_sent;
+  }
+  ceph_seq_t issue_norevoke(unsigned c) {
+    _pending |= c;
+    _issued |= c;
+    //check_rdcaps_list();
+    ++last_sent;
+    return last_sent;
+  }
+  void _calc_issued() {
+    _issued = _pending;
+    for (list::iterator p = _revokes.begin(); p != _revokes.end(); ++p)
+      _issued |= p->before;
+  }
+  void confirm_receipt(ceph_seq_t seq, unsigned caps) {
+    if (seq == last_sent) {
+      _revokes.clear();
+      _issued = caps;
+      // don't add bits
+      _pending &= caps;
+    } else {
+      // can i forget any revocations?
+      while (!_revokes.empty() && _revokes.front().seq < seq)
+	_revokes.pop_front();
+      if (!_revokes.empty()) {
+	if (_revokes.front().seq == seq)
+	  _revokes.begin()->before = caps;
+	_calc_issued();
+      } else {
+	// seq < last_sent
+	_issued = caps | _pending;
+      }
+    }
+    //check_rdcaps_list();
+  }
+  // we may get a release racing with revocations, which means our revokes will be ignored
+  // by the client.  clean them out of our _revokes history so we don't wait on them.
+  void clean_revoke_from(ceph_seq_t li) {
+    bool changed = false;
+    while (!_revokes.empty() && _revokes.front().last_issue <= li) {
+      _revokes.pop_front();
+      changed = true;
+    }
+    if (changed)
+      _calc_issued();
+  }
+
+
+private:
+  ceph_seq_t last_sent;
+  ceph_seq_t last_issue;
+  ceph_seq_t mseq;
+
+  int suppress;
+  unsigned state;
+
+  const static unsigned STATE_STALE		= (1<<0);
+  const static unsigned STATE_NEW		= (1<<1);
+
+public:
+  snapid_t client_follows;
+  version_t client_xattr_version;
+  version_t client_inline_version;
+  
+  xlist::item item_session_caps;
+  xlist::item item_snaprealm_caps;
+
+  Capability(CInode *i = NULL, uint64_t id = 0, client_t c = 0) : 
+    inode(i), client(c),
+    cap_id(id),
+    _wanted(0),
+    _pending(0), _issued(0),
+    last_sent(0),
+    last_issue(0),
+    mseq(0),
+    suppress(0), state(0),
+    client_follows(0), client_xattr_version(0),
+    client_inline_version(0),
+    item_session_caps(this), item_snaprealm_caps(this) {
+    g_num_cap++;
+    g_num_capa++;
+  }
+  ~Capability() {
+    g_num_cap--;
+    g_num_caps++;
+  }
+
+  Capability(const Capability& other);  // no copying
+  const Capability& operator=(const Capability& other);  // no copying
+  
+  ceph_seq_t get_mseq() { return mseq; }
+  void inc_mseq() { mseq++; }
+
+  ceph_seq_t get_last_sent() { return last_sent; }
+  utime_t get_last_issue_stamp() { return last_issue_stamp; }
+
+  void set_last_issue() { last_issue = last_sent; }
+  void set_last_issue_stamp(utime_t t) { last_issue_stamp = t; }
+
+  void set_cap_id(uint64_t i) { cap_id = i; }
+  uint64_t get_cap_id() { return cap_id; }
+
+  //ceph_seq_t get_last_issue() { return last_issue; }
+
+  bool is_suppress() { return suppress > 0; }
+  void inc_suppress() { suppress++; }
+  void dec_suppress() { suppress--; }
+
+  bool is_stale() { return state & STATE_STALE; }
+  void mark_stale() { state |= STATE_STALE; }
+  void clear_stale() { state &= ~STATE_STALE; }
+  bool is_new() { return state & STATE_NEW; }
+  void mark_new() { state |= STATE_NEW; }
+  void clear_new() { state &= ~STATE_NEW; }
+
+  CInode *get_inode() { return inode; }
+  client_t get_client() { return client; }
+
+  // caps this client wants to hold
+  int wanted() { return _wanted; }
+  void set_wanted(int w) {
+    _wanted = w;
+    //check_rdcaps_list();
+  }
+
+  void inc_last_seq() { last_sent++; };
+  ceph_seq_t get_last_seq() { return last_sent; }
+  ceph_seq_t get_last_issue() { return last_issue; }
+
+  void reset_seq() {
+    last_sent = 0;
+    last_issue = 0;
+  }
+  
+  // -- exports --
+  Export make_export() {
+    return Export(cap_id, _wanted, issued(), pending(), client_follows, last_sent, mseq+1, last_issue_stamp);
+  }
+  void merge(Export& other, bool auth_cap) {
+    if (!is_stale()) {
+      // issued + pending
+      int newpending = other.pending | pending();
+      if (other.issued & ~newpending)
+	issue(other.issued | newpending);
+      else
+	issue(newpending);
+      last_issue_stamp = other.last_issue_stamp;
+    } else {
+      issue(CEPH_CAP_PIN);
+    }
+
+    client_follows = other.client_follows;
+
+    // wanted
+    _wanted = _wanted | other.wanted;
+    if (auth_cap)
+      mseq = other.mseq;
+  }
+  void merge(int otherwanted, int otherissued) {
+    if (!is_stale()) {
+      // issued + pending
+      int newpending = pending();
+      if (otherissued & ~newpending)
+	issue(otherissued | newpending);
+      else
+	issue(newpending);
+    } else {
+      issue(CEPH_CAP_PIN);
+    }
+
+    // wanted
+    _wanted = _wanted | otherwanted;
+  }
+
+  void revoke() {
+    if (pending() & ~CEPH_CAP_PIN)
+      issue(CEPH_CAP_PIN);
+    confirm_receipt(last_sent, CEPH_CAP_PIN);
+  }
+
+  // serializers
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  
+};
+
+WRITE_CLASS_ENCODER(Capability::Export)
+WRITE_CLASS_ENCODER(Capability::Import)
+WRITE_CLASS_ENCODER(Capability::revoke_info)
+WRITE_CLASS_ENCODER(Capability)
+
+
+
+#endif
diff --git a/ceph/src/mds/Dumper.cc b/ceph/src/mds/Dumper.cc
new file mode 100644
index 00000000..f7f18c9b
--- /dev/null
+++ b/ceph/src/mds/Dumper.cc
@@ -0,0 +1,274 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Greg Farnum 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef _BACKWARD_BACKWARD_WARNING_H
+#define _BACKWARD_BACKWARD_WARNING_H   // make gcc 4.3 shut up about hash_*
+#endif
+
+#include "include/compat.h"
+#include "common/entity_name.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "mds/Dumper.h"
+#include "mds/mdstypes.h"
+#include "mds/LogEvent.h"
+#include "osdc/Journaler.h"
+
+#define dout_subsys ceph_subsys_mds
+
+
+int Dumper::init(int rank_)
+{
+  rank = rank_;
+
+  int r = MDSUtility::init();
+  if (r < 0) {
+    return r;
+  }
+
+  inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
+  journaler = new Journaler(ino, mdsmap->get_metadata_pool(), CEPH_FS_ONDISK_MAGIC,
+                                       objecter, 0, 0, &timer);
+  return 0;
+}
+
+
+int Dumper::recover_journal()
+{
+  bool done = false;
+  Cond cond;
+  Mutex localLock("dump:recover_journal");
+  int r;
+
+  lock.Lock();
+  journaler->recover(new C_SafeCond(&localLock, &cond, &done, &r));
+  lock.Unlock();
+  localLock.Lock();
+  while (!done)
+    cond.Wait(localLock);
+  localLock.Unlock();
+
+  if (r < 0) { // Error
+    derr << "error on recovery: " << cpp_strerror(r) << dendl;
+    return r;
+  } else {
+    dout(10) << "completed journal recovery" << dendl;
+    return 0;
+  }
+}
+
+
+void Dumper::dump(const char *dump_file)
+{
+  bool done = false;
+  int r = 0;
+  Cond cond;
+  Mutex localLock("dump:lock");
+
+  r = recover_journal();
+  if (r) {
+    return;
+  }
+  uint64_t start = journaler->get_read_pos();
+  uint64_t end = journaler->get_write_pos();
+  uint64_t len = end-start;
+  inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
+
+  cout << "journal is " << start << "~" << len << std::endl;
+
+  Filer filer(objecter);
+  bufferlist bl;
+
+  lock.Lock();
+  filer.read(ino, &journaler->get_layout(), CEPH_NOSNAP,
+             start, len, &bl, 0, new C_SafeCond(&localLock, &cond, &done));
+  lock.Unlock();
+  localLock.Lock();
+  while (!done)
+    cond.Wait(localLock);
+  localLock.Unlock();
+
+  cout << "read " << bl.length() << " bytes at offset " << start << std::endl;
+
+  int fd = ::open(dump_file, O_WRONLY|O_CREAT|O_TRUNC, 0644);
+  if (fd >= 0) {
+    // include an informative header
+    char buf[200];
+    memset(buf, 0, sizeof(buf));
+    sprintf(buf, "Ceph mds%d journal dump\n start offset %llu (0x%llx)\n       length %llu (0x%llx)\n%c",
+	    rank, 
+	    (unsigned long long)start, (unsigned long long)start,
+	    (unsigned long long)bl.length(), (unsigned long long)bl.length(),
+	    4);
+    int r = safe_write(fd, buf, sizeof(buf));
+    if (r)
+      ceph_abort();
+
+    // write the data
+    ::lseek64(fd, start, SEEK_SET);
+    bl.write_fd(fd);
+    ::close(fd);
+
+    cout << "wrote " << bl.length() << " bytes at offset " << start << " to " << dump_file << "\n"
+	 << "NOTE: this is a _sparse_ file; you can\n"
+	 << "\t$ tar cSzf " << dump_file << ".tgz " << dump_file << "\n"
+	 << "      to efficiently compress it while preserving sparseness." << std::endl;
+  } else {
+    int err = errno;
+    derr << "unable to open " << dump_file << ": " << cpp_strerror(err) << dendl;
+  }
+}
+
+void Dumper::undump(const char *dump_file)
+{
+  cout << "undump " << dump_file << std::endl;
+  
+  int fd = ::open(dump_file, O_RDONLY);
+  if (fd < 0) {
+    derr << "couldn't open " << dump_file << ": " << cpp_strerror(errno) << dendl;
+    return;
+  }
+
+  // Ceph mds0 journal dump
+  //  start offset 232401996 (0xdda2c4c)
+  //        length 1097504 (0x10bf20)
+
+  char buf[200];
+  int r = safe_read(fd, buf, sizeof(buf));
+  if (r < 0) {
+    VOID_TEMP_FAILURE_RETRY(::close(fd));
+    return;
+  }
+
+  long long unsigned start, len;
+  sscanf(strstr(buf, "start offset"), "start offset %llu", &start);
+  sscanf(strstr(buf, "length"), "length %llu", &len);
+
+  cout << "start " << start << " len " << len << std::endl;
+  
+  inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
+
+  Journaler::Header h;
+  h.trimmed_pos = start;
+  h.expire_pos = start;
+  h.write_pos = start+len;
+  h.magic = CEPH_FS_ONDISK_MAGIC;
+
+  h.layout = g_default_file_layout;
+  h.layout.fl_pg_pool = mdsmap->get_metadata_pool();
+  
+  bufferlist hbl;
+  ::encode(h, hbl);
+
+  object_t oid = file_object_t(ino, 0);
+  object_locator_t oloc(mdsmap->get_metadata_pool());
+  SnapContext snapc;
+
+  bool done = false;
+  Cond cond;
+  
+  cout << "writing header " << oid << std::endl;
+  objecter->write_full(oid, oloc, snapc, hbl, ceph_clock_now(g_ceph_context), 0, 
+		       NULL, 
+		       new C_SafeCond(&lock, &cond, &done));
+
+  lock.Lock();
+  while (!done)
+    cond.Wait(lock);
+  lock.Unlock();
+  
+  // read
+  Filer filer(objecter);
+  uint64_t pos = start;
+  uint64_t left = len;
+  while (left > 0) {
+    bufferlist j;
+    lseek64(fd, pos, SEEK_SET);
+    uint64_t l = MIN(left, 1024*1024);
+    j.read_fd(fd, l);
+    cout << " writing " << pos << "~" << l << std::endl;
+    filer.write(ino, &h.layout, snapc, pos, l, j, ceph_clock_now(g_ceph_context), 0, NULL, new C_SafeCond(&lock, &cond, &done));
+
+    lock.Lock();
+    while (!done)
+      cond.Wait(lock);
+    lock.Unlock();
+    
+    pos += l;
+    left -= l;
+  }
+
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+  cout << "done." << std::endl;
+}
+
+
+/**
+ * Write JSON-formatted log entries to standard out.
+ */
+void Dumper::dump_entries()
+{
+  Mutex localLock("dump_entries");
+  JSONFormatter jf(true);
+
+  int r = recover_journal();
+  if (r) {
+    return;
+  }
+
+  jf.open_array_section("log");
+  bool got_data = true;
+  lock.Lock();
+  // Until the journal is empty, pop an event or wait for one to
+  // be available.
+  dout(10) << "Journaler read/write/size: "
+      << journaler->get_read_pos() << "/" << journaler->get_write_pos()
+      << "/" << journaler->get_write_pos() - journaler->get_read_pos() << dendl;
+  while (journaler->get_read_pos() != journaler->get_write_pos()) {
+    bufferlist entry_bl;
+    got_data = journaler->try_read_entry(entry_bl);
+    dout(10) << "try_read_entry: " << got_data << dendl;
+    if (got_data) {
+      LogEvent *le = LogEvent::decode(entry_bl);
+      if (!le) {
+	dout(0) << "Error decoding LogEvent" << dendl;
+	break;
+      } else {
+	jf.open_object_section("log_event");
+	jf.dump_unsigned("type", le->get_type());
+	jf.dump_unsigned("start_off", le->get_start_off());
+	jf.dump_unsigned("stamp_sec", le->get_stamp().tv.tv_sec);
+	jf.dump_unsigned("stamp_nsec", le->get_stamp().tv.tv_nsec);
+	le->dump(&jf);
+	jf.close_section();
+	delete le;
+      }
+    } else {
+      bool done = false;
+      Cond cond;
+
+      journaler->wait_for_readable(new C_SafeCond(&localLock, &cond, &done));
+      lock.Unlock();
+      localLock.Lock();
+      while (!done)
+        cond.Wait(localLock);
+      localLock.Unlock();
+      lock.Lock();
+    }
+  }
+  lock.Unlock();
+  jf.close_section();
+  jf.flush(cout);
+  return;
+}
diff --git a/ceph/src/mds/Dumper.h b/ceph/src/mds/Dumper.h
new file mode 100644
index 00000000..6218ef42
--- /dev/null
+++ b/ceph/src/mds/Dumper.h
@@ -0,0 +1,47 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Greg Farnum 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef JOURNAL_DUMPER_H_
+#define JOURNAL_DUMPER_H_
+
+
+#include "mds/MDSUtility.h"
+#include "osdc/Journaler.h"
+
+/**
+ * This class lets you dump out an mds journal for troubleshooting or whatever.
+ *
+ * It was built to work with cmds so some of the design choices are random.
+ * To use, create a Dumper, call init(), and then call dump() with the name
+ * of the file to dump to.
+ */
+
+class Dumper : public MDSUtility {
+private:
+  Journaler *journaler;
+  int rank;
+
+public:
+  Dumper() : journaler(NULL), rank(-1)
+  {}
+
+  void handle_mds_map(MMDSMap* m);
+
+  int init(int rank);
+  int recover_journal();
+  void dump(const char *dumpfile);
+  void undump(const char *dumpfile);
+  void dump_entries();
+};
+
+#endif /* JOURNAL_DUMPER_H_ */
diff --git a/ceph/src/mds/InoTable.cc b/ceph/src/mds/InoTable.cc
new file mode 100644
index 00000000..f0bf3bd5
--- /dev/null
+++ b/ceph/src/mds/InoTable.cc
@@ -0,0 +1,189 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "InoTable.h"
+#include "MDS.h"
+
+#include "include/types.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << "." << table_name << ": "
+
+void InoTable::reset_state()
+{
+  // use generic range. FIXME THIS IS CRAP
+  free.clear();
+  //#ifdef __LP64__
+  uint64_t start = (uint64_t)(mds->get_nodeid()+1) << 40;
+  uint64_t end = ((uint64_t)(mds->get_nodeid()+2) << 40) - 1;
+  //#else
+  //# warning this looks like a 32-bit system, using small inode numbers.
+  //  uint64_t start = (uint64_t)(mds->get_nodeid()+1) << 25;
+  //  uint64_t end = ((uint64_t)(mds->get_nodeid()+2) << 25) - 1;
+  //#endif
+  free.insert(start, end);
+
+  projected_free = free;
+}
+
+inodeno_t InoTable::project_alloc_id(inodeno_t id) 
+{
+  dout(10) << "project_alloc_id " << id << " to " << projected_free << "/" << free << dendl;
+  assert(is_active());
+  if (!id)
+    id = projected_free.range_start();
+  projected_free.erase(id);
+  ++projected_version;
+  return id;
+}
+void InoTable::apply_alloc_id(inodeno_t id)
+{
+  dout(10) << "apply_alloc_id " << id << " to " << projected_free << "/" << free << dendl;
+  free.erase(id);
+  ++version;
+}
+
+void InoTable::project_alloc_ids(interval_set& ids, int want) 
+{
+  assert(is_active());
+  while (want > 0) {
+    inodeno_t start = projected_free.range_start();
+    inodeno_t end = projected_free.end_after(start);
+    inodeno_t num = end - start;
+    if (num > (inodeno_t)want)
+      num = want;
+    projected_free.erase(start, num);
+    ids.insert(start, num);
+    want -= num;
+  }
+  dout(10) << "project_alloc_ids " << ids << " to " << projected_free << "/" << free << dendl;
+  ++projected_version;
+}
+void InoTable::apply_alloc_ids(interval_set& ids)
+{
+  dout(10) << "apply_alloc_ids " << ids << " to " << projected_free << "/" << free << dendl;
+  free.subtract(ids);
+  ++version;
+}
+
+
+void InoTable::project_release_ids(interval_set& ids) 
+{
+  dout(10) << "project_release_ids " << ids << " to " << projected_free << "/" << free << dendl;
+  projected_free.insert(ids);
+  ++projected_version;
+}
+void InoTable::apply_release_ids(interval_set& ids) 
+{
+  dout(10) << "apply_release_ids " << ids << " to " << projected_free << "/" << free << dendl;
+  free.insert(ids);
+  ++version;
+}
+
+
+//
+
+void InoTable::replay_alloc_id(inodeno_t id) 
+{
+  dout(10) << "replay_alloc_id " << id << dendl;
+  if (free.contains(id)) {
+    free.erase(id);
+    projected_free.erase(id);
+  } else {
+    mds->clog.error() << "journal replay alloc " << id
+      << " not in free " << free << "\n";
+  }
+  projected_version = ++version;
+}
+void InoTable::replay_alloc_ids(interval_set& ids) 
+{
+  dout(10) << "replay_alloc_ids " << ids << dendl;
+  interval_set is;
+  is.intersection_of(free, ids);
+  if (is == ids) {
+    free.subtract(ids);
+    projected_free.subtract(ids);
+  } else {
+    mds->clog.error() << "journal replay alloc " << ids << ", only "
+	<< is << " is in free " << free << "\n";
+    free.subtract(is);
+    projected_free.subtract(is);
+  }
+  projected_version = ++version;
+}
+void InoTable::replay_release_ids(interval_set& ids) 
+{
+  dout(10) << "replay_release_ids " << ids << dendl;
+  free.insert(ids);
+  projected_free.insert(ids);
+  projected_version = ++version;
+}
+
+
+void InoTable::replay_reset()
+{
+  dout(10) << "replay_reset " << free << dendl;
+  skip_inos(inodeno_t(10000000));  // a lot!
+  projected_free = free;
+  projected_version = ++version;
+}
+
+
+void InoTable::skip_inos(inodeno_t i)
+{
+  dout(10) << "skip_inos was " << free << dendl;
+  inodeno_t first = free.range_start();
+  inodeno_t last = first + i;
+  interval_set s;
+  s.insert(first, last);
+  s.intersection_of(free);
+  free.subtract(s);
+  projected_free = free;
+  projected_version = ++version;
+  dout(10) << "skip_inos now " << free << dendl;
+}
+
+void InoTable::dump(Formatter *f) const
+{
+  f->open_object_section("inotable");
+
+  f->open_array_section("projected_free");
+  for (interval_set::const_iterator i = projected_free.begin(); i != projected_free.end(); ++i) {
+    f->open_object_section("range");
+    f->dump_int("start", (*i).first);
+    f->dump_int("len", (*i).second);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("free");
+  for (interval_set::const_iterator i = free.begin(); i != free.end(); ++i) {
+    f->open_object_section("range");
+    f->dump_int("start", (*i).first);
+    f->dump_int("len", (*i).second);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->close_section();
+}
+
+
+void InoTable::generate_test_instances(list& ls)
+{
+  ls.push_back(new InoTable());
+}
diff --git a/ceph/src/mds/InoTable.h b/ceph/src/mds/InoTable.h
new file mode 100644
index 00000000..100db956
--- /dev/null
+++ b/ceph/src/mds/InoTable.h
@@ -0,0 +1,72 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_INOTABLE_H
+#define CEPH_INOTABLE_H
+
+#include "MDSTable.h"
+#include "include/interval_set.h"
+
+class MDS;
+
+class InoTable : public MDSTable {
+  interval_set free;   // unused ids
+  interval_set projected_free;
+
+ public:
+  InoTable(MDS *m) : MDSTable(m, "inotable", true) { }
+
+  inodeno_t project_alloc_id(inodeno_t id=0);
+  void apply_alloc_id(inodeno_t id);
+
+  void project_alloc_ids(interval_set& inos, int want);
+  void apply_alloc_ids(interval_set& inos);
+
+  void project_release_ids(interval_set& inos);
+  void apply_release_ids(interval_set& inos);
+
+  void replay_alloc_id(inodeno_t ino);
+  void replay_alloc_ids(interval_set& inos);
+  void replay_release_ids(interval_set& inos);
+  void replay_reset();
+
+  void reset_state();
+  void encode_state(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(free, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode_state(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    ::decode(free, bl);
+    projected_free = free;
+    DECODE_FINISH(bl);
+  }
+
+  // To permit enc/decoding in isolation in dencoder
+  InoTable() : MDSTable(NULL, "inotable", true) {}
+  void encode(bufferlist& bl) const {
+    encode_state(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    decode_state(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void skip_inos(inodeno_t i);
+};
+
+#endif
diff --git a/ceph/src/mds/LocalLock.h b/ceph/src/mds/LocalLock.h
new file mode 100644
index 00000000..c797ea7d
--- /dev/null
+++ b/ceph/src/mds/LocalLock.h
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_LOCALLOCK_H
+#define CEPH_LOCALLOCK_H
+
+#include "SimpleLock.h"
+
+class LocalLock : public SimpleLock {
+public:
+  client_t last_wrlock_client;
+  
+  LocalLock(MDSCacheObject *o, LockType *t) : 
+    SimpleLock(o, t) {
+    set_state(LOCK_LOCK); // always.
+  }
+
+  bool is_locallock() const {
+    return true;
+  }
+
+  bool can_xlock_local() const {
+    return !is_wrlocked() && (get_xlock_by() == 0);
+  }
+
+  bool can_wrlock() const {
+    return !is_xlocked();
+  }
+  void get_wrlock(client_t client) {
+    assert(can_wrlock());
+    SimpleLock::get_wrlock();
+    last_wrlock_client = client;
+  }
+  void put_wrlock() {
+    SimpleLock::put_wrlock();
+    if (get_num_wrlocks() == 0)
+      last_wrlock_client = client_t();
+  }
+  client_t get_last_wrlock_client() const {
+    return last_wrlock_client;
+  }
+  
+  virtual void print(ostream& out) const {
+    out << "(";
+    _print(out);
+    if (last_wrlock_client >= 0)
+      out << " last_client=" << last_wrlock_client;
+    out << ")";
+  }
+};
+
+
+#endif
diff --git a/ceph/src/mds/Locker.cc b/ceph/src/mds/Locker.cc
new file mode 100644
index 00000000..74305b9c
--- /dev/null
+++ b/ceph/src/mds/Locker.cc
@@ -0,0 +1,4796 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include "MDS.h"
+#include "MDCache.h"
+#include "Locker.h"
+#include "CInode.h"
+#include "CDir.h"
+#include "CDentry.h"
+#include "Mutation.h"
+
+#include "MDLog.h"
+#include "MDSMap.h"
+
+#include "include/filepath.h"
+
+#include "events/EUpdate.h"
+#include "events/EOpen.h"
+
+#include "msg/Messenger.h"
+
+#include "messages/MGenericMessage.h"
+#include "messages/MDiscover.h"
+#include "messages/MDiscoverReply.h"
+
+#include "messages/MDirUpdate.h"
+
+#include "messages/MInodeFileCaps.h"
+
+#include "messages/MLock.h"
+#include "messages/MClientLease.h"
+#include "messages/MDentryUnlink.h"
+
+#include "messages/MClientRequest.h"
+#include "messages/MClientReply.h"
+#include "messages/MClientCaps.h"
+#include "messages/MClientCapRelease.h"
+
+#include "messages/MMDSSlaveRequest.h"
+
+#include 
+
+#include "common/config.h"
+
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#undef DOUT_COND
+#define DOUT_COND(cct, l) l<=cct->_conf->debug_mds || l <= cct->_conf->debug_mds_locker
+#define dout_prefix _prefix(_dout, mds)
+static ostream& _prefix(std::ostream *_dout, MDS *mds) {
+  return *_dout << "mds." << mds->get_nodeid() << ".locker ";
+}
+
+/* This function DOES put the passed message before returning */
+void Locker::dispatch(Message *m)
+{
+
+  switch (m->get_type()) {
+
+    // inter-mds locking
+  case MSG_MDS_LOCK:
+    handle_lock(static_cast(m));
+    break;
+    // inter-mds caps
+  case MSG_MDS_INODEFILECAPS:
+    handle_inode_file_caps(static_cast(m));
+    break;
+
+    // client sync
+  case CEPH_MSG_CLIENT_CAPS:
+    handle_client_caps(static_cast(m));
+    break;
+  case CEPH_MSG_CLIENT_CAPRELEASE:
+    handle_client_cap_release(static_cast(m));
+    break;
+  case CEPH_MSG_CLIENT_LEASE:
+    handle_client_lease(static_cast(m));
+    break;
+    
+  default:
+    assert(0);
+  }
+}
+
+
+/*
+ * locks vs rejoin
+ *
+ * 
+ *
+ */
+
+void Locker::send_lock_message(SimpleLock *lock, int msg)
+{
+  for (map::iterator it = lock->get_parent()->replicas_begin();
+       it != lock->get_parent()->replicas_end(); 
+       ++it) {
+    if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN) 
+      continue;
+    MLock *m = new MLock(lock, msg, mds->get_nodeid());
+    mds->send_message_mds(m, it->first);
+  }
+}
+
+void Locker::send_lock_message(SimpleLock *lock, int msg, const bufferlist &data)
+{
+  for (map::iterator it = lock->get_parent()->replicas_begin();
+       it != lock->get_parent()->replicas_end(); 
+       ++it) {
+    if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN) 
+      continue;
+    MLock *m = new MLock(lock, msg, mds->get_nodeid());
+    m->set_data(data);
+    mds->send_message_mds(m, it->first);
+  }
+}
+
+
+
+
+void Locker::include_snap_rdlocks(set& rdlocks, CInode *in)
+{
+  // rdlock ancestor snaps
+  CInode *t = in;
+  rdlocks.insert(&in->snaplock);
+  while (t->get_projected_parent_dn()) {
+    t = t->get_projected_parent_dn()->get_dir()->get_inode();
+    rdlocks.insert(&t->snaplock);
+  }
+}
+
+void Locker::include_snap_rdlocks_wlayout(set& rdlocks, CInode *in,
+                                  ceph_file_layout **layout)
+{
+  //rdlock ancestor snaps
+  CInode *t = in;
+  rdlocks.insert(&in->snaplock);
+  rdlocks.insert(&in->policylock);
+  bool found_layout = false;
+  while (t) {
+    rdlocks.insert(&t->snaplock);
+    if (!found_layout) {
+      rdlocks.insert(&t->policylock);
+      if (t->get_projected_inode()->has_layout()) {
+        *layout = &t->get_projected_inode()->layout;
+        found_layout = true;
+      }
+    }
+    if (t->get_projected_parent_dn() &&
+        t->get_projected_parent_dn()->get_dir())
+      t = t->get_projected_parent_dn()->get_dir()->get_inode();
+    else t = NULL;
+  }
+}
+
+
+/* If this function returns false, the mdr has been placed
+ * on the appropriate wait list */
+bool Locker::acquire_locks(MDRequestRef& mdr,
+			   set &rdlocks,
+			   set &wrlocks,
+			   set &xlocks,
+			   map *remote_wrlocks,
+			   CInode *auth_pin_freeze,
+			   bool auth_pin_nonblock)
+{
+  if (mdr->done_locking &&
+      !mdr->is_slave()) {  // not on slaves!  master requests locks piecemeal.
+    dout(10) << "acquire_locks " << *mdr << " - done locking" << dendl;    
+    return true;  // at least we had better be!
+  }
+  dout(10) << "acquire_locks " << *mdr << dendl;
+
+  client_t client = mdr->get_client();
+
+  set sorted;  // sort everything we will lock
+  set mustpin = xlocks;            // items to authpin
+
+  // xlocks
+  for (set::iterator p = xlocks.begin(); p != xlocks.end(); ++p) {
+    dout(20) << " must xlock " << **p << " " << *(*p)->get_parent() << dendl;
+    sorted.insert(*p);
+
+    // augment xlock with a versionlock?
+    if ((*p)->get_type() == CEPH_LOCK_DN) {
+      CDentry *dn = (CDentry*)(*p)->get_parent();
+      if (!dn->is_auth())
+	continue;
+
+      if (xlocks.count(&dn->versionlock))
+	continue;  // we're xlocking the versionlock too; don't wrlock it!
+
+      if (mdr->is_master()) {
+	// master.  wrlock versionlock so we can pipeline dentry updates to journal.
+	wrlocks.insert(&dn->versionlock);
+      } else {
+	// slave.  exclusively lock the dentry version (i.e. block other journal updates).
+	// this makes rollback safe.
+	xlocks.insert(&dn->versionlock);
+	sorted.insert(&dn->versionlock);
+      }
+    }
+    if ((*p)->get_type() > CEPH_LOCK_IVERSION) {
+      // inode version lock?
+      CInode *in = (CInode*)(*p)->get_parent();
+      if (!in->is_auth())
+	continue;
+      if (mdr->is_master()) {
+	// master.  wrlock versionlock so we can pipeline inode updates to journal.
+	wrlocks.insert(&in->versionlock);
+      } else {
+	// slave.  exclusively lock the inode version (i.e. block other journal updates).
+	// this makes rollback safe.
+	xlocks.insert(&in->versionlock);
+	sorted.insert(&in->versionlock);
+      }
+    }
+  }
+
+  // wrlocks
+  for (set::iterator p = wrlocks.begin(); p != wrlocks.end(); ++p) {
+    dout(20) << " must wrlock " << **p << " " << *(*p)->get_parent() << dendl;
+    sorted.insert(*p);
+    if ((*p)->get_parent()->is_auth())
+      mustpin.insert(*p);
+    else if (!(*p)->get_parent()->is_auth() &&
+	     !(*p)->can_wrlock(client) &&  // we might have to request a scatter
+	     !mdr->is_slave()) {           // if we are slave (remote_wrlock), the master already authpinned
+      dout(15) << " will also auth_pin " << *(*p)->get_parent()
+	       << " in case we need to request a scatter" << dendl;
+      mustpin.insert(*p);
+    }
+  }
+
+  // remote_wrlocks
+  if (remote_wrlocks) {
+    for (map::iterator p = remote_wrlocks->begin(); p != remote_wrlocks->end(); ++p) {
+      dout(20) << " must remote_wrlock on mds." << p->second << " "
+	       << *p->first << " " << *(p->first)->get_parent() << dendl;
+      sorted.insert(p->first);
+      mustpin.insert(p->first);
+    }
+  }
+
+  // rdlocks
+  for (set::iterator p = rdlocks.begin();
+	 p != rdlocks.end();
+       ++p) {
+    dout(20) << " must rdlock " << **p << " " << *(*p)->get_parent() << dendl;
+    sorted.insert(*p);
+    if ((*p)->get_parent()->is_auth())
+      mustpin.insert(*p);
+    else if (!(*p)->get_parent()->is_auth() &&
+	     !(*p)->can_rdlock(client)) {      // we might have to request an rdlock
+      dout(15) << " will also auth_pin " << *(*p)->get_parent()
+	       << " in case we need to request a rdlock" << dendl;
+      mustpin.insert(*p);
+    }
+  }
+
+ 
+  // AUTH PINS
+  map > mustpin_remote;  // mds -> (object set)
+  
+  // can i auth pin them all now?
+  for (set::iterator p = mustpin.begin();
+       p != mustpin.end();
+       ++p) {
+    MDSCacheObject *object = (*p)->get_parent();
+
+    dout(10) << " must authpin " << *object << dendl;
+
+    if (mdr->is_auth_pinned(object)) 
+      continue;
+    
+    if (!object->is_auth()) {
+      if (!mdr->locks.empty())
+	mds->locker->drop_locks(mdr.get());
+      if (object->is_ambiguous_auth()) {
+	// wait
+	dout(10) << " ambiguous auth, waiting to authpin " << *object << dendl;
+	object->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, new C_MDS_RetryRequest(mdcache, mdr));
+	mdr->drop_local_auth_pins();
+	return false;
+      }
+      mustpin_remote[object->authority().first].insert(object);
+      continue;
+    }
+    if (!object->can_auth_pin()) {
+      // wait
+      mds->locker->drop_locks(mdr.get());
+      mdr->drop_local_auth_pins();
+      if (auth_pin_nonblock) {
+	dout(10) << " can't auth_pin (freezing?) " << *object << ", nonblocking" << dendl;
+	mdr->aborted = true;
+	return false;
+      }
+      dout(10) << " can't auth_pin (freezing?), waiting to authpin " << *object << dendl;
+      object->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
+      return false;
+    }
+  }
+
+  // ok, grab local auth pins
+  for (set::iterator p = mustpin.begin();
+       p != mustpin.end();
+       ++p) {
+    MDSCacheObject *object = (*p)->get_parent();
+    if (mdr->is_auth_pinned(object)) {
+      dout(10) << " already auth_pinned " << *object << dendl;
+    } else if (object->is_auth()) {
+      dout(10) << " auth_pinning " << *object << dendl;
+      mdr->auth_pin(object);
+    }
+  }
+
+  // request remote auth_pins
+  if (!mustpin_remote.empty()) {
+    for (map >::iterator p = mustpin_remote.begin();
+	 p != mustpin_remote.end();
+	 ++p) {
+      dout(10) << "requesting remote auth_pins from mds." << p->first << dendl;
+
+      // wait for active auth
+      if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(p->first)) {
+	dout(10) << " mds." << p->first << " is not active" << dendl;
+	if (mdr->more()->waiting_on_slave.empty())
+	  mds->wait_for_active_peer(p->first, new C_MDS_RetryRequest(mdcache, mdr));
+	return false;
+      }
+      
+      MMDSSlaveRequest *req = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+						   MMDSSlaveRequest::OP_AUTHPIN);
+      for (set::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q) {
+	dout(10) << " req remote auth_pin of " << **q << dendl;
+	MDSCacheObjectInfo info;
+	(*q)->set_object_info(info);
+	req->get_authpins().push_back(info);
+	if (*q == auth_pin_freeze)
+	  (*q)->set_object_info(req->get_authpin_freeze());
+	mdr->pin(*q);
+      }
+      if (auth_pin_nonblock)
+	req->mark_nonblock();
+      mds->send_message_mds(req, p->first);
+
+      // put in waiting list
+      assert(mdr->more()->waiting_on_slave.count(p->first) == 0);
+      mdr->more()->waiting_on_slave.insert(p->first);
+    }
+    return false;
+  }
+
+  // caps i'll need to issue
+  set issue_set;
+  bool result = false;
+
+  // acquire locks.
+  // make sure they match currently acquired locks.
+  set::iterator existing = mdr->locks.begin();
+  for (set::iterator p = sorted.begin();
+       p != sorted.end();
+       ++p) {
+    bool need_wrlock = !!wrlocks.count(*p);
+    bool need_remote_wrlock = !!(remote_wrlocks && remote_wrlocks->count(*p));
+
+    // already locked?
+    if (existing != mdr->locks.end() && *existing == *p) {
+      // right kind?
+      SimpleLock *have = *existing;
+      ++existing;
+      if (xlocks.count(have) && mdr->xlocks.count(have)) {
+	dout(10) << " already xlocked " << *have << " " << *have->get_parent() << dendl;
+	continue;
+      }
+      if (mdr->remote_wrlocks.count(have)) {
+	if (!need_remote_wrlock ||
+	    mdr->remote_wrlocks[have] != (*remote_wrlocks)[have]) {
+	  dout(10) << " unlocking remote_wrlock on wrong mds." << mdr->remote_wrlocks[have]
+		   << " " << *have << " " << *have->get_parent() << dendl;
+	  remote_wrlock_finish(have, mdr->remote_wrlocks[have], mdr.get());
+	}
+      }
+      if (need_wrlock || need_remote_wrlock) {
+	if (need_wrlock == !!mdr->wrlocks.count(have) &&
+	    need_remote_wrlock == !!mdr->remote_wrlocks.count(have)) {
+	  if (need_wrlock)
+	    dout(10) << " already wrlocked " << *have << " " << *have->get_parent() << dendl;
+	  if (need_remote_wrlock)
+	    dout(10) << " already remote_wrlocked " << *have << " " << *have->get_parent() << dendl;
+	  continue;
+	}
+      }
+      if (rdlocks.count(have) && mdr->rdlocks.count(have)) {
+	dout(10) << " already rdlocked " << *have << " " << *have->get_parent() << dendl;
+	continue;
+      }
+    }
+    
+    // hose any stray locks
+    if (*existing == *p) {
+      assert(need_wrlock || need_remote_wrlock);
+      SimpleLock *lock = *existing;
+      if (mdr->wrlocks.count(lock)) {
+	if (!need_wrlock)
+	  dout(10) << " unlocking extra " << *lock << " " << *lock->get_parent() << dendl;
+	else if (need_remote_wrlock) // acquire remote_wrlock first
+	  dout(10) << " unlocking out-of-order " << *lock << " " << *lock->get_parent() << dendl;
+	bool need_issue = false;
+	wrlock_finish(lock, mdr.get(), &need_issue);
+	if (need_issue)
+	  issue_set.insert(static_cast(lock->get_parent()));
+      }
+      ++existing;
+    }
+    while (existing != mdr->locks.end()) {
+      SimpleLock *stray = *existing;
+      ++existing;
+      dout(10) << " unlocking out-of-order " << *stray << " " << *stray->get_parent() << dendl;
+      bool need_issue = false;
+      if (mdr->xlocks.count(stray)) {
+	xlock_finish(stray, mdr.get(), &need_issue);
+      } else if (mdr->rdlocks.count(stray)) {
+	rdlock_finish(stray, mdr.get(), &need_issue);
+      } else {
+	// may have acquired both wrlock and remore wrlock
+	if (mdr->wrlocks.count(stray))
+	  wrlock_finish(stray, mdr.get(), &need_issue);
+	if (mdr->remote_wrlocks.count(stray))
+	  remote_wrlock_finish(stray, mdr->remote_wrlocks[stray], mdr.get());
+      }
+      if (need_issue)
+	issue_set.insert(static_cast(stray->get_parent()));
+    }
+
+    // lock
+    if (mdr->locking && *p != mdr->locking) {
+      cancel_locking(mdr.get(), &issue_set);
+    }
+    if (xlocks.count(*p)) {
+      if (!xlock_start(*p, mdr)) 
+	goto out;
+      dout(10) << " got xlock on " << **p << " " << *(*p)->get_parent() << dendl;
+    } else if (need_wrlock || need_remote_wrlock) {
+      if (need_remote_wrlock && !mdr->remote_wrlocks.count(*p)) {
+	remote_wrlock_start(*p, (*remote_wrlocks)[*p], mdr);
+	goto out;
+      }
+      if (need_wrlock && !mdr->wrlocks.count(*p)) {
+	if (need_remote_wrlock && !(*p)->can_wrlock(mdr->get_client())) {
+	  // can't take the wrlock because the scatter lock is gathering. need to
+	  // release the remote wrlock, so that the gathering process can finish.
+	  remote_wrlock_finish(*p, mdr->remote_wrlocks[*p], mdr.get());
+	  remote_wrlock_start(*p, (*remote_wrlocks)[*p], mdr);
+	  goto out;
+	}
+	if (!wrlock_start(*p, mdr))
+	  goto out;
+	dout(10) << " got wrlock on " << **p << " " << *(*p)->get_parent() << dendl;
+      }
+    } else {
+      if (!rdlock_start(*p, mdr)) 
+	goto out;
+      dout(10) << " got rdlock on " << **p << " " << *(*p)->get_parent() << dendl;
+    }
+  }
+    
+  // any extra unneeded locks?
+  while (existing != mdr->locks.end()) {
+    SimpleLock *stray = *existing;
+    ++existing;
+    dout(10) << " unlocking extra " << *stray << " " << *stray->get_parent() << dendl;
+    bool need_issue = false;
+    if (mdr->xlocks.count(stray)) {
+      xlock_finish(stray, mdr.get(), &need_issue);
+    } else if (mdr->rdlocks.count(stray)) {
+      rdlock_finish(stray, mdr.get(), &need_issue);
+    } else {
+      // may have acquired both wrlock and remore wrlock
+      if (mdr->wrlocks.count(stray))
+	wrlock_finish(stray, mdr.get(), &need_issue);
+      if (mdr->remote_wrlocks.count(stray))
+	remote_wrlock_finish(stray, mdr->remote_wrlocks[stray], mdr.get());
+    }
+    if (need_issue)
+      issue_set.insert(static_cast(stray->get_parent()));
+  }
+
+  mdr->done_locking = true;
+  result = true;
+
+ out:
+  issue_caps_set(issue_set);
+  return result;
+}
+
+
+void Locker::set_xlocks_done(MutationImpl *mut, bool skip_dentry)
+{
+  for (set::iterator p = mut->xlocks.begin();
+       p != mut->xlocks.end();
+       ++p) {
+    if (skip_dentry &&
+	((*p)->get_type() == CEPH_LOCK_DN || (*p)->get_type() == CEPH_LOCK_DVERSION))
+      continue;
+    dout(10) << "set_xlocks_done on " << **p << " " << *(*p)->get_parent() << dendl;
+    (*p)->set_xlock_done();
+  }
+}
+
+void Locker::_drop_rdlocks(MutationImpl *mut, set *pneed_issue)
+{
+  while (!mut->rdlocks.empty()) {
+    bool ni = false;
+    MDSCacheObject *p = (*mut->rdlocks.begin())->get_parent();
+    rdlock_finish(*mut->rdlocks.begin(), mut, &ni);
+    if (ni)
+      pneed_issue->insert(static_cast(p));
+  }
+}
+
+void Locker::_drop_non_rdlocks(MutationImpl *mut, set *pneed_issue)
+{
+  set slaves;
+
+  while (!mut->xlocks.empty()) {
+    SimpleLock *lock = *mut->xlocks.begin();
+    MDSCacheObject *p = lock->get_parent();
+    if (!p->is_auth()) {
+      assert(lock->get_sm()->can_remote_xlock);
+      slaves.insert(p->authority().first);
+      lock->put_xlock();
+      mut->locks.erase(lock);
+      mut->xlocks.erase(lock);
+      continue;
+    }
+    bool ni = false;
+    xlock_finish(lock, mut, &ni);
+    if (ni)
+      pneed_issue->insert(static_cast(p));
+  }
+
+  while (!mut->remote_wrlocks.empty()) {
+    map::iterator p = mut->remote_wrlocks.begin();
+    slaves.insert(p->second);
+    if (mut->wrlocks.count(p->first) == 0)
+      mut->locks.erase(p->first);
+    mut->remote_wrlocks.erase(p);
+  }
+
+  while (!mut->wrlocks.empty()) {
+    bool ni = false;
+    MDSCacheObject *p = (*mut->wrlocks.begin())->get_parent();
+    wrlock_finish(*mut->wrlocks.begin(), mut, &ni);
+    if (ni)
+      pneed_issue->insert(static_cast(p));
+  }
+
+  for (set::iterator p = slaves.begin(); p != slaves.end(); ++p) {
+    if (mds->mdsmap->get_state(*p) >= MDSMap::STATE_REJOIN) {
+      dout(10) << "_drop_non_rdlocks dropping remote locks on mds." << *p << dendl;
+      MMDSSlaveRequest *slavereq = new MMDSSlaveRequest(mut->reqid, mut->attempt,
+							MMDSSlaveRequest::OP_DROPLOCKS);
+      mds->send_message_mds(slavereq, *p);
+    }
+  }
+}
+
+void Locker::cancel_locking(MutationImpl *mut, set *pneed_issue)
+{
+  SimpleLock *lock = mut->locking;
+  assert(lock);
+  dout(10) << "cancel_locking " << *lock << " on " << *mut << dendl;
+
+  if (lock->get_parent()->is_auth()) {
+    bool need_issue = false;
+    if (lock->get_state() == LOCK_PREXLOCK) {
+      _finish_xlock(lock, -1, &need_issue);
+    } else if (lock->get_state() == LOCK_LOCK_XLOCK &&
+	       lock->get_num_xlocks() == 0) {
+      lock->set_state(LOCK_XLOCKDONE);
+      eval_gather(lock, true, &need_issue);
+    }
+    if (need_issue)
+      pneed_issue->insert(static_cast(lock->get_parent()));
+  }
+  mut->finish_locking(lock);
+}
+
+void Locker::drop_locks(MutationImpl *mut, set *pneed_issue)
+{
+  // leftover locks
+  set my_need_issue;
+  if (!pneed_issue)
+    pneed_issue = &my_need_issue;
+
+  if (mut->locking)
+    cancel_locking(mut, pneed_issue);
+  _drop_non_rdlocks(mut, pneed_issue);
+  _drop_rdlocks(mut, pneed_issue);
+
+  if (pneed_issue == &my_need_issue)
+    issue_caps_set(*pneed_issue);
+  mut->done_locking = false;
+}
+
+void Locker::drop_non_rdlocks(MutationImpl *mut, set *pneed_issue)
+{
+  set my_need_issue;
+  if (!pneed_issue)
+    pneed_issue = &my_need_issue;
+
+  _drop_non_rdlocks(mut, pneed_issue);
+
+  if (pneed_issue == &my_need_issue)
+    issue_caps_set(*pneed_issue);
+}
+
+void Locker::drop_rdlocks(MutationImpl *mut, set *pneed_issue)
+{
+  set my_need_issue;
+  if (!pneed_issue)
+    pneed_issue = &my_need_issue;
+
+  _drop_rdlocks(mut, pneed_issue);
+
+  if (pneed_issue == &my_need_issue)
+    issue_caps_set(*pneed_issue);
+}
+
+
+// generics
+
+void Locker::eval_gather(SimpleLock *lock, bool first, bool *pneed_issue, list *pfinishers)
+{
+  dout(10) << "eval_gather " << *lock << " on " << *lock->get_parent() << dendl;
+  assert(!lock->is_stable());
+
+  int next = lock->get_next_state();
+
+  CInode *in = 0;
+  bool caps = lock->get_cap_shift();
+  if (lock->get_type() != CEPH_LOCK_DN)
+    in = static_cast(lock->get_parent());
+
+  bool need_issue = false;
+
+  int loner_issued = 0, other_issued = 0, xlocker_issued = 0;
+  assert(!caps || in != NULL);
+  if (caps && in->is_head()) {
+    in->get_caps_issued(&loner_issued, &other_issued, &xlocker_issued,
+			lock->get_cap_shift(), lock->get_cap_mask());
+    dout(10) << " next state is " << lock->get_state_name(next) 
+	     << " issued/allows loner " << gcap_string(loner_issued)
+	     << "/" << gcap_string(lock->gcaps_allowed(CAP_LONER, next))
+	     << " xlocker " << gcap_string(xlocker_issued)
+	     << "/" << gcap_string(lock->gcaps_allowed(CAP_XLOCKER, next))
+	     << " other " << gcap_string(other_issued)
+	     << "/" << gcap_string(lock->gcaps_allowed(CAP_ANY, next))
+	     << dendl;
+
+    if (first && ((~lock->gcaps_allowed(CAP_ANY, next) & other_issued) ||
+		  (~lock->gcaps_allowed(CAP_LONER, next) & loner_issued) ||
+		  (~lock->gcaps_allowed(CAP_XLOCKER, next) & xlocker_issued)))
+      need_issue = true;
+  }
+
+#define IS_TRUE_AND_LT_AUTH(x, auth) (x && ((auth && x <= AUTH) || (!auth && x < AUTH)))
+  bool auth = lock->get_parent()->is_auth();
+  if (!lock->is_gathering() &&
+      (IS_TRUE_AND_LT_AUTH(lock->get_sm()->states[next].can_rdlock, auth) || !lock->is_rdlocked()) &&
+      (IS_TRUE_AND_LT_AUTH(lock->get_sm()->states[next].can_wrlock, auth) || !lock->is_wrlocked()) &&
+      (IS_TRUE_AND_LT_AUTH(lock->get_sm()->states[next].can_xlock, auth) || !lock->is_xlocked()) &&
+      (IS_TRUE_AND_LT_AUTH(lock->get_sm()->states[next].can_lease, auth) || !lock->is_leased()) &&
+      !(lock->get_parent()->is_auth() && lock->is_flushing()) &&  // i.e. wait for scatter_writebehind!
+      (!caps || ((~lock->gcaps_allowed(CAP_ANY, next) & other_issued) == 0 &&
+		 (~lock->gcaps_allowed(CAP_LONER, next) & loner_issued) == 0 &&
+		 (~lock->gcaps_allowed(CAP_XLOCKER, next) & xlocker_issued) == 0)) &&
+      lock->get_state() != LOCK_SYNC_MIX2 &&  // these states need an explicit trigger from the auth mds
+      lock->get_state() != LOCK_MIX_SYNC2
+      ) {
+    dout(7) << "eval_gather finished gather on " << *lock
+	    << " on " << *lock->get_parent() << dendl;
+
+    if (lock->get_sm() == &sm_filelock) {
+      assert(in);
+      if (in->state_test(CInode::STATE_RECOVERING)) {
+	dout(7) << "eval_gather finished gather, but still recovering" << dendl;
+	return;
+      } else if (in->state_test(CInode::STATE_NEEDSRECOVER)) {
+	dout(7) << "eval_gather finished gather, but need to recover" << dendl;
+	mds->mdcache->queue_file_recover(in);
+	mds->mdcache->do_file_recover();
+	return;
+      }
+    }
+
+    if (!lock->get_parent()->is_auth()) {
+      // replica: tell auth
+      int auth = lock->get_parent()->authority().first;
+
+      if (lock->get_parent()->is_rejoining() &&
+	  mds->mdsmap->get_state(auth) == MDSMap::STATE_REJOIN) {
+	dout(7) << "eval_gather finished gather, but still rejoining "
+		<< *lock->get_parent() << dendl;
+	return;
+      }
+
+      if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN) {
+	switch (lock->get_state()) {
+	case LOCK_SYNC_LOCK:
+	  mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid()),
+				auth);
+	  break;
+
+	case LOCK_MIX_SYNC:
+	  {
+	    MLock *reply = new MLock(lock, LOCK_AC_SYNCACK, mds->get_nodeid());
+	    lock->encode_locked_state(reply->get_data());
+	    mds->send_message_mds(reply, auth);
+	    next = LOCK_MIX_SYNC2;
+	    (static_cast(lock))->start_flush();
+	  }
+	  break;
+
+	case LOCK_MIX_SYNC2:
+	  (static_cast(lock))->finish_flush();
+	  (static_cast(lock))->clear_flushed();
+
+	case LOCK_SYNC_MIX2:
+	  // do nothing, we already acked
+	  break;
+	  
+	case LOCK_SYNC_MIX:
+	  { 
+	    MLock *reply = new MLock(lock, LOCK_AC_MIXACK, mds->get_nodeid());
+	    mds->send_message_mds(reply, auth);
+	    next = LOCK_SYNC_MIX2;
+	  }
+	  break;
+
+	case LOCK_MIX_LOCK:
+	  {
+	    bufferlist data;
+	    lock->encode_locked_state(data);
+	    mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid(), data), auth);
+	    (static_cast(lock))->start_flush();
+	    // we'll get an AC_LOCKFLUSHED to complete
+	  }
+	  break;
+
+	default:
+	  assert(0);
+	}
+      }
+    } else {
+      // auth
+
+      // once the first (local) stage of mix->lock gather complete we can
+      // gather from replicas
+      if (lock->get_state() == LOCK_MIX_LOCK &&
+	  lock->get_parent()->is_replicated()) {
+	dout(10) << " finished (local) gather for mix->lock, now gathering from replicas" << dendl;
+	send_lock_message(lock, LOCK_AC_LOCK);
+	lock->init_gather();
+	lock->set_state(LOCK_MIX_LOCK2);
+	return;
+      }
+
+      if (lock->is_dirty() && !lock->is_flushed()) {
+	scatter_writebehind(static_cast(lock));
+	mds->mdlog->flush();
+	return;
+      }
+      lock->clear_flushed();
+      
+      switch (lock->get_state()) {
+	// to mixed
+      case LOCK_TSYN_MIX:
+      case LOCK_SYNC_MIX:
+      case LOCK_EXCL_MIX:
+	in->start_scatter(static_cast(lock));
+	if (lock->get_parent()->is_replicated()) {
+	  bufferlist softdata;
+	  lock->encode_locked_state(softdata);
+	  send_lock_message(lock, LOCK_AC_MIX, softdata);
+	}
+	(static_cast(lock))->clear_scatter_wanted();
+	break;
+
+      case LOCK_XLOCK:
+      case LOCK_XLOCKDONE:
+	if (next != LOCK_SYNC)
+	  break;
+	// fall-thru
+
+	// to sync
+      case LOCK_EXCL_SYNC:
+      case LOCK_LOCK_SYNC:
+      case LOCK_MIX_SYNC:
+      case LOCK_XSYN_SYNC:
+	if (lock->get_parent()->is_replicated()) {
+	  bufferlist softdata;
+	  lock->encode_locked_state(softdata);
+	  send_lock_message(lock, LOCK_AC_SYNC, softdata);
+	}
+	break;
+      }
+
+    }
+
+    lock->set_state(next);
+    
+    if (lock->get_parent()->is_auth() &&
+	lock->is_stable())
+      lock->get_parent()->auth_unpin(lock);
+
+    // drop loner before doing waiters
+    if (caps &&
+	in->is_head() &&
+	in->is_auth() &&
+	in->get_wanted_loner() != in->get_loner()) {
+      dout(10) << "  trying to drop loner" << dendl;
+      if (in->try_drop_loner()) {
+	dout(10) << "  dropped loner" << dendl;
+	need_issue = true;
+      }
+    }
+
+    if (pfinishers)
+      lock->take_waiting(SimpleLock::WAIT_STABLE|SimpleLock::WAIT_WR|SimpleLock::WAIT_RD|SimpleLock::WAIT_XLOCK,
+			 *pfinishers);
+    else
+      lock->finish_waiters(SimpleLock::WAIT_STABLE|SimpleLock::WAIT_WR|SimpleLock::WAIT_RD|SimpleLock::WAIT_XLOCK);
+    
+    if (caps && in->is_head())
+      need_issue = true;
+
+    if (lock->get_parent()->is_auth() &&
+	lock->is_stable())
+      try_eval(lock, &need_issue);
+  }
+
+  if (need_issue) {
+    if (pneed_issue)
+      *pneed_issue = true;
+    else if (in->is_head())
+      issue_caps(in);
+  }
+
+}
+
+bool Locker::eval(CInode *in, int mask, bool caps_imported)
+{
+  bool need_issue = caps_imported;
+  list finishers;
+  
+  dout(10) << "eval " << mask << " " << *in << dendl;
+
+  // choose loner?
+  if (in->is_auth() && in->is_head()) {
+    if (in->choose_ideal_loner() >= 0) {
+      if (in->try_set_loner()) {
+	dout(10) << "eval set loner to client." << in->get_loner() << dendl;
+	need_issue = true;
+	mask = -1;
+      } else
+	dout(10) << "eval want loner client." << in->get_wanted_loner() << " but failed to set it" << dendl;
+    } else
+      dout(10) << "eval doesn't want loner" << dendl;
+  }
+
+ retry:
+  if (mask & CEPH_LOCK_IFILE)
+    eval_any(&in->filelock, &need_issue, &finishers, caps_imported);
+  if (mask & CEPH_LOCK_IAUTH)
+    eval_any(&in->authlock, &need_issue, &finishers, caps_imported);
+  if (mask & CEPH_LOCK_ILINK)
+    eval_any(&in->linklock, &need_issue, &finishers, caps_imported);
+  if (mask & CEPH_LOCK_IXATTR)
+    eval_any(&in->xattrlock, &need_issue, &finishers, caps_imported);
+  if (mask & CEPH_LOCK_INEST)
+    eval_any(&in->nestlock, &need_issue, &finishers, caps_imported);
+  if (mask & CEPH_LOCK_IFLOCK)
+    eval_any(&in->flocklock, &need_issue, &finishers, caps_imported);
+  if (mask & CEPH_LOCK_IPOLICY)
+    eval_any(&in->policylock, &need_issue, &finishers, caps_imported);
+
+  // drop loner?
+  if (in->is_auth() && in->is_head() && in->get_wanted_loner() != in->get_loner()) {
+    dout(10) << "  trying to drop loner" << dendl;
+    if (in->try_drop_loner()) {
+      dout(10) << "  dropped loner" << dendl;
+      need_issue = true;
+
+      if (in->get_wanted_loner() >= 0) {
+	if (in->try_set_loner()) {
+	  dout(10) << "eval end set loner to client." << in->get_loner() << dendl;
+	  mask = -1;
+	  goto retry;
+	} else {
+	  dout(10) << "eval want loner client." << in->get_wanted_loner() << " but failed to set it" << dendl;
+	}
+      }
+    }
+  }
+
+  finish_contexts(g_ceph_context, finishers);
+
+  if (need_issue && in->is_head())
+    issue_caps(in);
+
+  dout(10) << "eval done" << dendl;
+  return need_issue;
+}
+
+class C_Locker_Eval : public Context {
+  Locker *locker;
+  MDSCacheObject *p;
+  int mask;
+public:
+  C_Locker_Eval(Locker *l, MDSCacheObject *pp, int m) : locker(l), p(pp), mask(m) {
+    p->get(MDSCacheObject::PIN_PTRWAITER);    
+  }
+  void finish(int r) {
+    p->put(MDSCacheObject::PIN_PTRWAITER);
+    locker->try_eval(p, mask);
+  }
+};
+
+void Locker::try_eval(MDSCacheObject *p, int mask)
+{
+  // unstable and ambiguous auth?
+  if (p->is_ambiguous_auth()) {
+    dout(7) << "try_eval ambiguous auth, waiting on " << *p << dendl;
+    p->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, new C_Locker_Eval(this, p, mask));
+    return;
+  }
+
+  if (p->is_auth() && p->is_frozen()) {
+    dout(7) << "try_eval frozen, waiting on " << *p << dendl;
+    p->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_Locker_Eval(this, p, mask));
+    return;
+  }
+
+  if (mask & CEPH_LOCK_DN) {
+    assert(mask == CEPH_LOCK_DN);
+    bool need_issue = false;  // ignore this, no caps on dentries
+    CDentry *dn = static_cast(p);
+    eval_any(&dn->lock, &need_issue);
+  } else {
+    CInode *in = static_cast(p);
+    eval(in, mask);
+  }
+}
+
+void Locker::try_eval(SimpleLock *lock, bool *pneed_issue)
+{
+  MDSCacheObject *p = lock->get_parent();
+
+  // unstable and ambiguous auth?
+  if (p->is_ambiguous_auth()) {
+    dout(7) << "try_eval " << *lock << " ambiguousauth, waiting on " << *p << dendl;
+    p->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, new C_Locker_Eval(this, p, lock->get_type()));
+    return;
+  }
+  
+  if (!p->is_auth()) {
+    dout(7) << "try_eval " << *lock << " not auth for " << *p << dendl;
+    return;
+  }
+
+  if (p->is_frozen()) {
+    dout(7) << "try_eval " << *lock << " frozen, waiting on " << *p << dendl;
+    p->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_Locker_Eval(this, p, lock->get_type()));
+    return;
+  }
+
+  /*
+   * We could have a situation like:
+   *
+   * - mds A authpins item on mds B
+   * - mds B starts to freeze tree containing item
+   * - mds A tries wrlock_start on A, sends REQSCATTER to B
+   * - mds B lock is unstable, sets scatter_wanted
+   * - mds B lock stabilizes, calls try_eval.
+   *
+   * We can defer while freezing without causing a deadlock.  Honor
+   * scatter_wanted flag here.  This will never get deferred by the
+   * checks above due to the auth_pin held by the master.
+   */
+  if (lock->is_scatterlock()) {
+    ScatterLock *slock = static_cast(lock);
+    if (slock->get_scatter_wanted() &&
+	slock->get_state() != LOCK_MIX) {
+      scatter_mix(slock, pneed_issue);
+      if (!lock->is_stable())
+	return;
+    } else if (slock->get_unscatter_wanted() &&
+        slock->get_state() != LOCK_LOCK) {
+      simple_lock(slock, pneed_issue);
+      if (!lock->is_stable()) {
+        return;
+      }
+    }
+  }
+
+  if (lock->get_type() != CEPH_LOCK_DN && p->is_freezing()) {
+    dout(7) << "try_eval " << *lock << " freezing, waiting on " << *p << dendl;
+    p->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_Locker_Eval(this, p, lock->get_type()));
+    return;
+  }
+
+  eval(lock, pneed_issue);
+}
+
+void Locker::eval_cap_gather(CInode *in, set *issue_set)
+{
+  bool need_issue = false;
+  list finishers;
+
+  // kick locks now
+  if (!in->filelock.is_stable())
+    eval_gather(&in->filelock, false, &need_issue, &finishers);
+  if (!in->authlock.is_stable())
+    eval_gather(&in->authlock, false, &need_issue, &finishers);
+  if (!in->linklock.is_stable())
+    eval_gather(&in->linklock, false, &need_issue, &finishers);
+  if (!in->xattrlock.is_stable())
+    eval_gather(&in->xattrlock, false, &need_issue, &finishers);
+
+  if (need_issue && in->is_head()) {
+    if (issue_set)
+      issue_set->insert(in);
+    else
+      issue_caps(in);
+  }
+
+  finish_contexts(g_ceph_context, finishers);
+}
+
+void Locker::eval_scatter_gathers(CInode *in)
+{
+  bool need_issue = false;
+  list finishers;
+
+  dout(10) << "eval_scatter_gathers " << *in << dendl;
+
+  // kick locks now
+  if (!in->filelock.is_stable())
+    eval_gather(&in->filelock, false, &need_issue, &finishers);
+  if (!in->nestlock.is_stable())
+    eval_gather(&in->nestlock, false, &need_issue, &finishers);
+  if (!in->dirfragtreelock.is_stable())
+    eval_gather(&in->dirfragtreelock, false, &need_issue, &finishers);
+  
+  if (need_issue && in->is_head())
+    issue_caps(in);
+  
+  finish_contexts(g_ceph_context, finishers);
+}
+
+void Locker::eval(SimpleLock *lock, bool *need_issue)
+{
+  switch (lock->get_type()) {
+  case CEPH_LOCK_IFILE:
+    return file_eval(static_cast(lock), need_issue);
+  case CEPH_LOCK_IDFT:
+  case CEPH_LOCK_INEST:
+    return scatter_eval(static_cast(lock), need_issue);
+  default:
+    return simple_eval(lock, need_issue);
+  }
+}
+
+
+// ------------------
+// rdlock
+
+bool Locker::_rdlock_kick(SimpleLock *lock, bool as_anon)
+{
+  // kick the lock
+  if (lock->is_stable()) {
+    if (lock->get_parent()->is_auth()) {
+      if (lock->get_sm() == &sm_scatterlock) {
+	// not until tempsync is fully implemented
+	//if (lock->get_parent()->is_replicated())
+	//scatter_tempsync((ScatterLock*)lock);
+	//else
+	simple_sync(lock);
+      } else if (lock->get_sm() == &sm_filelock) {
+	CInode *in = static_cast(lock->get_parent());
+	if (lock->get_state() == LOCK_EXCL &&
+	    in->get_target_loner() >= 0 &&
+	    !as_anon)   // as_anon => caller wants SYNC, not XSYN
+	  file_xsyn(lock);
+	else
+	  simple_sync(lock);
+      } else
+	simple_sync(lock);
+      return true;
+    } else {
+      // request rdlock state change from auth
+      int auth = lock->get_parent()->authority().first;
+      if (mds->mdsmap->is_clientreplay_or_active_or_stopping(auth)) {
+	dout(10) << "requesting rdlock from auth on "
+		 << *lock << " on " << *lock->get_parent() << dendl;
+	mds->send_message_mds(new MLock(lock, LOCK_AC_REQRDLOCK, mds->get_nodeid()), auth);
+      }
+      return false;
+    }
+  }
+  return false;
+}
+
+bool Locker::rdlock_try(SimpleLock *lock, client_t client, Context *con)
+{
+  dout(7) << "rdlock_try on " << *lock << " on " << *lock->get_parent() << dendl;  
+
+  // can read?  grab ref.
+  if (lock->can_rdlock(client)) 
+    return true;
+  
+  _rdlock_kick(lock, false);
+
+  if (lock->can_rdlock(client)) 
+    return true;
+
+  // wait!
+  if (con) {
+    dout(7) << "rdlock_try waiting on " << *lock << " on " << *lock->get_parent() << dendl;
+    lock->add_waiter(SimpleLock::WAIT_STABLE|SimpleLock::WAIT_RD, con);
+  }
+  return false;
+}
+
+bool Locker::rdlock_start(SimpleLock *lock, MDRequestRef& mut, bool as_anon)
+{
+  dout(7) << "rdlock_start  on " << *lock << " on " << *lock->get_parent() << dendl;  
+
+  // client may be allowed to rdlock the same item it has xlocked.
+  //  UNLESS someone passes in as_anon, or we're reading snapped version here.
+  if (mut->snapid != CEPH_NOSNAP)
+    as_anon = true;
+  client_t client = as_anon ? -1 : mut->get_client();
+
+  CInode *in = 0;
+  if (lock->get_type() != CEPH_LOCK_DN)
+    in = static_cast(lock->get_parent());
+
+  /*
+  if (!lock->get_parent()->is_auth() &&
+      lock->fw_rdlock_to_auth()) {
+    mdcache->request_forward(mut, lock->get_parent()->authority().first);
+    return false;
+  }
+  */
+
+  while (1) {
+    // can read?  grab ref.
+    if (lock->can_rdlock(client)) {
+      lock->get_rdlock();
+      mut->rdlocks.insert(lock);
+      mut->locks.insert(lock);
+      return true;
+    }
+
+    if (!_rdlock_kick(lock, as_anon))
+      break;
+
+    // hmm, wait a second.
+    if (in && !in->is_head() && in->is_auth() &&
+	lock->get_state() == LOCK_SNAP_SYNC) {
+      // okay, we actually need to kick the head's lock to get ourselves synced up.
+      CInode *head = mdcache->get_inode(in->ino());
+      assert(head);
+      SimpleLock *hlock = head->get_lock(lock->get_type());
+      if (hlock->get_state() != LOCK_SYNC) {
+	dout(10) << "rdlock_start trying head inode " << *head << dendl;
+	if (!rdlock_start(head->get_lock(lock->get_type()), mut, true)) // ** as_anon, no rdlock on EXCL **
+	  return false;
+	// oh, check our lock again then
+      }
+    }
+  }
+
+  // wait!
+  int wait_on;
+  if (lock->get_parent()->is_auth() && lock->is_stable())
+    wait_on = SimpleLock::WAIT_RD;
+  else
+    wait_on = SimpleLock::WAIT_STABLE;  // REQRDLOCK is ignored if lock is unstable, so we need to retry.
+  dout(7) << "rdlock_start waiting on " << *lock << " on " << *lock->get_parent() << dendl;
+  lock->add_waiter(wait_on, new C_MDS_RetryRequest(mdcache, mut));
+  nudge_log(lock);
+  return false;
+}
+
+void Locker::nudge_log(SimpleLock *lock)
+{
+  dout(10) << "nudge_log " << *lock << " on " << *lock->get_parent() << dendl;
+  if (lock->get_parent()->is_auth() && !lock->is_stable())    // as with xlockdone, or cap flush
+    mds->mdlog->flush();
+}
+
+void Locker::rdlock_finish(SimpleLock *lock, MutationImpl *mut, bool *pneed_issue)
+{
+  // drop ref
+  lock->put_rdlock();
+  if (mut) {
+    mut->rdlocks.erase(lock);
+    mut->locks.erase(lock);
+  }
+
+  dout(7) << "rdlock_finish on " << *lock << " on " << *lock->get_parent() << dendl;
+  
+  // last one?
+  if (!lock->is_rdlocked()) {
+    if (!lock->is_stable())
+      eval_gather(lock, false, pneed_issue);
+    else if (lock->get_parent()->is_auth())
+      try_eval(lock, pneed_issue);
+  }
+}
+
+
+bool Locker::can_rdlock_set(set& locks)
+{
+  dout(10) << "can_rdlock_set " << locks << dendl;
+  for (set::iterator p = locks.begin(); p != locks.end(); ++p)
+    if (!(*p)->can_rdlock(-1)) {
+      dout(10) << "can_rdlock_set can't rdlock " << *p << " on " << *(*p)->get_parent() << dendl;
+      return false;
+    }
+  return true;
+}
+
+bool Locker::rdlock_try_set(set& locks)
+{
+  dout(10) << "rdlock_try_set " << locks << dendl;
+  for (set::iterator p = locks.begin(); p != locks.end(); ++p)
+    if (!rdlock_try(*p, -1, NULL)) {
+      dout(10) << "rdlock_try_set can't rdlock " << *p << " on " << *(*p)->get_parent() << dendl;
+      return false;
+    }
+  return true;
+}
+
+void Locker::rdlock_take_set(set& locks, MutationRef& mut)
+{
+  dout(10) << "rdlock_take_set " << locks << dendl;
+  for (set::iterator p = locks.begin(); p != locks.end(); ++p) {
+    (*p)->get_rdlock();
+    mut->rdlocks.insert(*p);
+    mut->locks.insert(*p);
+  }
+}
+
+// ------------------
+// wrlock
+
+void Locker::wrlock_force(SimpleLock *lock, MutationRef& mut)
+{
+  if (lock->get_type() == CEPH_LOCK_IVERSION ||
+      lock->get_type() == CEPH_LOCK_DVERSION)
+    return local_wrlock_grab(static_cast(lock), mut);
+
+  dout(7) << "wrlock_force  on " << *lock
+	  << " on " << *lock->get_parent() << dendl;  
+  lock->get_wrlock(true);
+  mut->wrlocks.insert(lock);
+  mut->locks.insert(lock);
+}
+
+bool Locker::wrlock_start(SimpleLock *lock, MDRequestRef& mut, bool nowait)
+{
+  if (lock->get_type() == CEPH_LOCK_IVERSION ||
+      lock->get_type() == CEPH_LOCK_DVERSION)
+    return local_wrlock_start(static_cast(lock), mut);
+
+  dout(10) << "wrlock_start " << *lock << " on " << *lock->get_parent() << dendl;
+
+  CInode *in = static_cast(lock->get_parent());
+  client_t client = mut->get_client();
+  bool want_scatter = !nowait && lock->get_parent()->is_auth() &&
+		      (in->has_subtree_or_exporting_dirfrag() ||
+		       static_cast(lock)->get_scatter_wanted());
+
+  while (1) {
+    // wrlock?
+    if (lock->can_wrlock(client) &&
+	(!want_scatter || lock->get_state() == LOCK_MIX)) {
+      lock->get_wrlock();
+      mut->wrlocks.insert(lock);
+      mut->locks.insert(lock);
+      return true;
+    }
+
+    if (!lock->is_stable())
+      break;
+
+    if (in->is_auth()) {
+      // don't do nested lock state change if we have dirty scatterdata and
+      // may scatter_writebehind or start_scatter, because nowait==true implies
+      // that the caller already has a log entry open!
+      if (nowait && lock->is_dirty())
+	return false;
+
+      if (want_scatter)
+	scatter_mix(static_cast(lock));
+      else
+	simple_lock(lock);
+
+      if (nowait && !lock->can_wrlock(client))
+	return false;
+      
+    } else {
+      // replica.
+      // auth should be auth_pinned (see acquire_locks wrlock weird mustpin case).
+      int auth = lock->get_parent()->authority().first;
+      if (mds->mdsmap->is_clientreplay_or_active_or_stopping(auth)) {
+	dout(10) << "requesting scatter from auth on "
+		 << *lock << " on " << *lock->get_parent() << dendl;
+	mds->send_message_mds(new MLock(lock, LOCK_AC_REQSCATTER, mds->get_nodeid()), auth);
+      }
+      break;
+    }
+  }
+
+  if (!nowait) {
+    dout(7) << "wrlock_start waiting on " << *lock << " on " << *lock->get_parent() << dendl;
+    lock->add_waiter(SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mut));
+    nudge_log(lock);
+  }
+    
+  return false;
+}
+
+void Locker::wrlock_finish(SimpleLock *lock, MutationImpl *mut, bool *pneed_issue)
+{
+  if (lock->get_type() == CEPH_LOCK_IVERSION ||
+      lock->get_type() == CEPH_LOCK_DVERSION)
+    return local_wrlock_finish(static_cast(lock), mut);
+
+  dout(7) << "wrlock_finish on " << *lock << " on " << *lock->get_parent() << dendl;
+  lock->put_wrlock();
+  if (mut) {
+    mut->wrlocks.erase(lock);
+    if (mut->remote_wrlocks.count(lock) == 0)
+      mut->locks.erase(lock);
+  }
+
+  if (!lock->is_wrlocked()) {
+    if (!lock->is_stable())
+      eval_gather(lock, false, pneed_issue);
+    else if (lock->get_parent()->is_auth())
+      try_eval(lock, pneed_issue);
+  }
+}
+
+
+// remote wrlock
+
+void Locker::remote_wrlock_start(SimpleLock *lock, int target, MDRequestRef& mut)
+{
+  dout(7) << "remote_wrlock_start mds." << target << " on " << *lock << " on " << *lock->get_parent() << dendl;
+
+  // wait for active target
+  if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(target)) {
+    dout(7) << " mds." << target << " is not active" << dendl;
+    if (mut->more()->waiting_on_slave.empty())
+      mds->wait_for_active_peer(target, new C_MDS_RetryRequest(mdcache, mut));
+    return;
+  }
+
+  // send lock request
+  mut->start_locking(lock, target);
+  mut->more()->slaves.insert(target);
+  MMDSSlaveRequest *r = new MMDSSlaveRequest(mut->reqid, mut->attempt,
+					     MMDSSlaveRequest::OP_WRLOCK);
+  r->set_lock_type(lock->get_type());
+  lock->get_parent()->set_object_info(r->get_object_info());
+  mds->send_message_mds(r, target);
+
+  assert(mut->more()->waiting_on_slave.count(target) == 0);
+  mut->more()->waiting_on_slave.insert(target);
+}
+
+void Locker::remote_wrlock_finish(SimpleLock *lock, int target,
+                                  MutationImpl *mut)
+{
+  // drop ref
+  mut->remote_wrlocks.erase(lock);
+  if (mut->wrlocks.count(lock) == 0)
+    mut->locks.erase(lock);
+  
+  dout(7) << "remote_wrlock_finish releasing remote wrlock on mds." << target
+	  << " " << *lock->get_parent()  << dendl;
+  if (mds->mdsmap->get_state(target) >= MDSMap::STATE_REJOIN) {
+    MMDSSlaveRequest *slavereq = new MMDSSlaveRequest(mut->reqid, mut->attempt,
+						      MMDSSlaveRequest::OP_UNWRLOCK);
+    slavereq->set_lock_type(lock->get_type());
+    lock->get_parent()->set_object_info(slavereq->get_object_info());
+    mds->send_message_mds(slavereq, target);
+  }
+}
+
+
+// ------------------
+// xlock
+
+bool Locker::xlock_start(SimpleLock *lock, MDRequestRef& mut)
+{
+  if (lock->get_type() == CEPH_LOCK_IVERSION ||
+      lock->get_type() == CEPH_LOCK_DVERSION)
+    return local_xlock_start(static_cast(lock), mut);
+
+  dout(7) << "xlock_start on " << *lock << " on " << *lock->get_parent() << dendl;
+  client_t client = mut->get_client();
+
+  // auth?
+  if (lock->get_parent()->is_auth()) {
+    // auth
+    while (1) {
+      if (lock->can_xlock(client)) {
+	lock->set_state(LOCK_XLOCK);
+	lock->get_xlock(mut, client);
+	mut->xlocks.insert(lock);
+	mut->locks.insert(lock);
+	mut->finish_locking(lock);
+	return true;
+      }
+      
+      if (!lock->is_stable() && !(lock->get_state() == LOCK_XLOCKDONE &&
+				  lock->get_xlock_by_client() == client))
+	break;
+
+      if (lock->get_state() == LOCK_LOCK || lock->get_state() == LOCK_XLOCKDONE) {
+	mut->start_locking(lock);
+	simple_xlock(lock);
+      } else {
+	simple_lock(lock);
+      }
+    }
+    
+    lock->add_waiter(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mut));
+    nudge_log(lock);
+    return false;
+  } else {
+    // replica
+    assert(lock->get_sm()->can_remote_xlock);
+    assert(!mut->slave_request);
+    
+    // wait for single auth
+    if (lock->get_parent()->is_ambiguous_auth()) {
+      lock->get_parent()->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, 
+				     new C_MDS_RetryRequest(mdcache, mut));
+      return false;
+    }
+    
+    // wait for active auth
+    int auth = lock->get_parent()->authority().first;
+    if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(auth)) {
+      dout(7) << " mds." << auth << " is not active" << dendl;
+      if (mut->more()->waiting_on_slave.empty())
+	mds->wait_for_active_peer(auth, new C_MDS_RetryRequest(mdcache, mut));
+      return false;
+    }
+
+    // send lock request
+    mut->more()->slaves.insert(auth);
+    mut->start_locking(lock, auth);
+    MMDSSlaveRequest *r = new MMDSSlaveRequest(mut->reqid, mut->attempt,
+					       MMDSSlaveRequest::OP_XLOCK);
+    r->set_lock_type(lock->get_type());
+    lock->get_parent()->set_object_info(r->get_object_info());
+    mds->send_message_mds(r, auth);
+
+    assert(mut->more()->waiting_on_slave.count(auth) == 0);
+    mut->more()->waiting_on_slave.insert(auth);
+
+    return false;
+  }
+}
+
+void Locker::_finish_xlock(SimpleLock *lock, client_t xlocker, bool *pneed_issue)
+{
+  assert(!lock->is_stable());
+  if (lock->get_num_rdlocks() == 0 &&
+      lock->get_num_wrlocks() == 0 &&
+      lock->get_num_client_lease() == 0 &&
+      lock->get_type() != CEPH_LOCK_DN) {
+    CInode *in = static_cast(lock->get_parent());
+    client_t loner = in->get_target_loner();
+    if (loner >= 0 && (xlocker < 0 || xlocker == loner)) {
+      lock->set_state(LOCK_EXCL);
+      lock->get_parent()->auth_unpin(lock);
+      lock->finish_waiters(SimpleLock::WAIT_STABLE|SimpleLock::WAIT_WR|SimpleLock::WAIT_RD);
+      if (lock->get_cap_shift())
+	*pneed_issue = true;
+      if (lock->get_parent()->is_auth() &&
+	  lock->is_stable())
+	try_eval(lock, pneed_issue);
+      return;
+    }
+  }
+  // the xlocker may have CEPH_CAP_GSHARED, need to revoke it if next state is LOCK_LOCK
+  eval_gather(lock, true, pneed_issue);
+}
+
+void Locker::xlock_finish(SimpleLock *lock, MutationImpl *mut, bool *pneed_issue)
+{
+  if (lock->get_type() == CEPH_LOCK_IVERSION ||
+      lock->get_type() == CEPH_LOCK_DVERSION)
+    return local_xlock_finish(static_cast(lock), mut);
+
+  dout(10) << "xlock_finish on " << *lock << " " << *lock->get_parent() << dendl;
+
+  client_t xlocker = lock->get_xlock_by_client();
+
+  // drop ref
+  lock->put_xlock();
+  assert(mut);
+  mut->xlocks.erase(lock);
+  mut->locks.erase(lock);
+  
+  bool do_issue = false;
+
+  // remote xlock?
+  if (!lock->get_parent()->is_auth()) {
+    assert(lock->get_sm()->can_remote_xlock);
+
+    // tell auth
+    dout(7) << "xlock_finish releasing remote xlock on " << *lock->get_parent()  << dendl;
+    int auth = lock->get_parent()->authority().first;
+    if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN) {
+      MMDSSlaveRequest *slavereq = new MMDSSlaveRequest(mut->reqid, mut->attempt,
+							MMDSSlaveRequest::OP_UNXLOCK);
+      slavereq->set_lock_type(lock->get_type());
+      lock->get_parent()->set_object_info(slavereq->get_object_info());
+      mds->send_message_mds(slavereq, auth);
+    }
+    // others waiting?
+    lock->finish_waiters(SimpleLock::WAIT_STABLE |
+			 SimpleLock::WAIT_WR | 
+			 SimpleLock::WAIT_RD, 0); 
+  } else {
+    if (lock->get_num_xlocks() == 0) {
+      if (lock->get_state() == LOCK_LOCK_XLOCK)
+	lock->set_state(LOCK_XLOCKDONE);
+      _finish_xlock(lock, xlocker, &do_issue);
+    }
+  }
+  
+  if (do_issue) {
+    CInode *in = static_cast(lock->get_parent());
+    if (in->is_head()) {
+      if (pneed_issue)
+	*pneed_issue = true;
+      else
+	issue_caps(in);
+    }
+  }
+}
+
+void Locker::xlock_export(SimpleLock *lock, MutationImpl *mut)
+{
+  dout(10) << "xlock_export on " << *lock << " " << *lock->get_parent() << dendl;
+
+  lock->put_xlock();
+  mut->xlocks.erase(lock);
+  mut->locks.erase(lock);
+
+  MDSCacheObject *p = lock->get_parent();
+  assert(p->state_test(CInode::STATE_AMBIGUOUSAUTH));  // we are exporting this (inode)
+
+  if (!lock->is_stable())
+    lock->get_parent()->auth_unpin(lock);
+
+  lock->set_state(LOCK_LOCK);
+}
+
+void Locker::xlock_import(SimpleLock *lock)
+{
+  dout(10) << "xlock_import on " << *lock << " " << *lock->get_parent() << dendl;
+  lock->get_parent()->auth_pin(lock);
+}
+
+
+
+// file i/o -----------------------------------------
+
+version_t Locker::issue_file_data_version(CInode *in)
+{
+  dout(7) << "issue_file_data_version on " << *in << dendl;
+  return in->inode.file_data_version;
+}
+
+struct C_Locker_FileUpdate_finish : public Context {
+  Locker *locker;
+  CInode *in;
+  MutationRef mut;
+  bool share;
+  client_t client;
+  Capability *cap;
+  MClientCaps *ack;
+  C_Locker_FileUpdate_finish(Locker *l, CInode *i, MutationRef& m,
+                             bool e=false, client_t c=-1,
+			     Capability *cp = 0,
+			     MClientCaps *ac = 0) : 
+    locker(l), in(i), mut(m), share(e), client(c), cap(cp),
+    ack(ac) {
+    in->get(CInode::PIN_PTRWAITER);
+  }
+  void finish(int r) {
+    locker->file_update_finish(in, mut, share, client, cap, ack);
+  }
+};
+
+void Locker::file_update_finish(CInode *in, MutationRef& mut, bool share, client_t client,
+				Capability *cap, MClientCaps *ack)
+{
+  dout(10) << "file_update_finish on " << *in << dendl;
+  in->pop_and_dirty_projected_inode(mut->ls);
+  in->put(CInode::PIN_PTRWAITER);
+
+  mut->apply();
+  
+  if (ack)
+    mds->send_message_client_counted(ack, client);
+
+  set need_issue;
+  drop_locks(mut.get(), &need_issue);
+
+  if (!in->is_head() && !in->client_snap_caps.empty()) {
+    dout(10) << " client_snap_caps " << in->client_snap_caps << dendl;
+    // check for snap writeback completion
+    bool gather = false;
+    map >::iterator p = in->client_snap_caps.begin();
+    while (p != in->client_snap_caps.end()) {
+      SimpleLock *lock = in->get_lock(p->first);
+      assert(lock);
+      dout(10) << " completing client_snap_caps for " << ccap_string(p->first)
+	       << " lock " << *lock << " on " << *in << dendl;
+      lock->put_wrlock();
+
+      p->second.erase(client);
+      if (p->second.empty()) {
+	gather = true;
+	in->client_snap_caps.erase(p++);
+      } else
+	++p;
+    }
+    if (gather)
+      eval_cap_gather(in, &need_issue);
+  } else {
+    if (cap && (cap->wanted() & ~cap->pending()) &&
+	need_issue.count(in) == 0) {  // if we won't issue below anyway
+      issue_caps(in, cap);
+    }
+  
+    if (share && in->is_auth() &&
+	(in->filelock.gcaps_allowed(CAP_LONER) & (CEPH_CAP_GWR|CEPH_CAP_GBUFFER)))
+      share_inode_max_size(in);
+  }
+  issue_caps_set(need_issue);
+
+  // auth unpin after issuing caps
+  mut->cleanup();
+}
+
+Capability* Locker::issue_new_caps(CInode *in,
+				   int mode,
+				   Session *session,
+				   SnapRealm *realm,
+				   bool is_replay)
+{
+  dout(7) << "issue_new_caps for mode " << mode << " on " << *in << dendl;
+  bool is_new;
+
+  // if replay, try to reconnect cap, and otherwise do nothing.
+  if (is_replay) {
+    mds->mdcache->try_reconnect_cap(in, session);
+    return 0;
+  }
+
+  // my needs
+  assert(session->info.inst.name.is_client());
+  int my_client = session->info.inst.name.num();
+  int my_want = ceph_caps_for_mode(mode);
+
+  // register a capability
+  Capability *cap = in->get_client_cap(my_client);
+  if (!cap) {
+    // new cap
+    cap = in->add_client_cap(my_client, session, realm);
+    cap->set_wanted(my_want);
+    cap->mark_new();
+    cap->inc_suppress(); // suppress file cap messages for new cap (we'll bundle with the open() reply)
+    is_new = true;
+  } else {
+    is_new = false;
+    // make sure it wants sufficient caps
+    if (my_want & ~cap->wanted()) {
+      // augment wanted caps for this client
+      cap->set_wanted(cap->wanted() | my_want);
+    }
+  }
+
+  if (in->is_auth()) {
+    // [auth] twiddle mode?
+    eval(in, CEPH_CAP_LOCKS);
+
+    if (!in->filelock.is_stable() ||
+	!in->authlock.is_stable() ||
+	!in->linklock.is_stable() ||
+	!in->xattrlock.is_stable())
+      mds->mdlog->flush();
+
+  } else {
+    // [replica] tell auth about any new caps wanted
+    request_inode_file_caps(in);
+  }
+
+  // issue caps (pot. incl new one)
+  //issue_caps(in);  // note: _eval above may have done this already...
+
+  // re-issue whatever we can
+  //cap->issue(cap->pending());
+
+  if (is_new)
+    cap->dec_suppress();
+
+  return cap;
+}
+
+
+void Locker::issue_caps_set(set& inset)
+{
+  for (set::iterator p = inset.begin(); p != inset.end(); ++p)
+    issue_caps(*p);
+}
+
+bool Locker::issue_caps(CInode *in, Capability *only_cap)
+{
+  // allowed caps are determined by the lock mode.
+  int all_allowed = in->get_caps_allowed_by_type(CAP_ANY);
+  int loner_allowed = in->get_caps_allowed_by_type(CAP_LONER);
+  int xlocker_allowed = in->get_caps_allowed_by_type(CAP_XLOCKER);
+
+  client_t loner = in->get_loner();
+  if (loner >= 0) {
+    dout(7) << "issue_caps loner client." << loner
+	    << " allowed=" << ccap_string(loner_allowed) 
+	    << ", xlocker allowed=" << ccap_string(xlocker_allowed)
+	    << ", others allowed=" << ccap_string(all_allowed)
+	    << " on " << *in << dendl;
+  } else {
+    dout(7) << "issue_caps allowed=" << ccap_string(all_allowed) 
+	    << ", xlocker allowed=" << ccap_string(xlocker_allowed)
+	    << " on " << *in << dendl;
+  }
+
+  assert(in->is_head());
+
+  // count conflicts with
+  int nissued = 0;        
+
+  // client caps
+  map::iterator it;
+  if (only_cap)
+    it = in->client_caps.find(only_cap->get_client());
+  else
+    it = in->client_caps.begin();
+  for (; it != in->client_caps.end(); ++it) {
+    Capability *cap = it->second;
+    if (cap->is_stale())
+      continue;
+
+    // do not issue _new_ bits when size|mtime is projected
+    int allowed;
+    if (loner == it->first)
+      allowed = loner_allowed;
+    else
+      allowed = all_allowed;
+
+    // add in any xlocker-only caps (for locks this client is the xlocker for)
+    allowed |= xlocker_allowed & in->get_xlocker_mask(it->first);
+
+    Session *session = mds->get_session(it->first);
+    if (in->inode.inline_version != CEPH_INLINE_NONE &&
+	!(session && session->connection &&
+	  session->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA)))
+      allowed &= ~(CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR);
+
+    int pending = cap->pending();
+    int wanted = cap->wanted();
+
+    dout(20) << " client." << it->first
+	     << " pending " << ccap_string(pending) 
+	     << " allowed " << ccap_string(allowed) 
+	     << " wanted " << ccap_string(wanted)
+	     << dendl;
+
+    // skip if suppress, and not revocation
+    if (cap->is_suppress() && !(pending & ~allowed)) {
+      dout(20) << "  suppressed and !revoke, skipping client." << it->first << dendl;
+      continue;
+    }
+
+    // notify clients about deleted inode, to make sure they release caps ASAP.
+    if (in->inode.nlink == 0)
+      wanted |= CEPH_CAP_LINK_SHARED;
+
+    // are there caps that the client _wants_ and can have, but aren't pending?
+    // or do we need to revoke?
+    if (((wanted & allowed) & ~pending) ||  // missing wanted+allowed caps
+	(pending & ~allowed)) {             // need to revoke ~allowed caps.
+      // issue
+      nissued++;
+
+      // include caps that clients generally like, while we're at it.
+      int likes = in->get_caps_liked();      
+      int before = pending;
+      long seq;
+      if (pending & ~allowed)
+	seq = cap->issue((wanted|likes) & allowed & pending);  // if revoking, don't issue anything new.
+      else
+	seq = cap->issue((wanted|likes) & allowed);
+      int after = cap->pending();
+
+      if (cap->is_new()) {
+	// haven't send caps to client yet
+	if (before & ~after)
+	  cap->confirm_receipt(seq, after);
+      } else {
+        dout(7) << "   sending MClientCaps to client." << it->first
+		<< " seq " << cap->get_last_seq()
+		<< " new pending " << ccap_string(after) << " was " << ccap_string(before) 
+		<< dendl;
+
+	MClientCaps *m = new MClientCaps((before & ~after) ? CEPH_CAP_OP_REVOKE:CEPH_CAP_OP_GRANT,
+					 in->ino(),
+					 in->find_snaprealm()->inode->ino(),
+					 cap->get_cap_id(), cap->get_last_seq(),
+					 after, wanted, 0,
+					 cap->get_mseq());
+	in->encode_cap_message(m, cap);
+
+	mds->send_message_client_counted(m, it->first);
+      }
+    }
+
+    if (only_cap)
+      break;
+  }
+
+  return (nissued == 0);  // true if no re-issued, no callbacks
+}
+
+void Locker::issue_truncate(CInode *in)
+{
+  dout(7) << "issue_truncate on " << *in << dendl;
+  
+  for (map::iterator it = in->client_caps.begin();
+       it != in->client_caps.end();
+       ++it) {
+    Capability *cap = it->second;
+    MClientCaps *m = new MClientCaps(CEPH_CAP_OP_TRUNC,
+				     in->ino(),
+				     in->find_snaprealm()->inode->ino(),
+				     cap->get_cap_id(), cap->get_last_seq(),
+				     cap->pending(), cap->wanted(), 0,
+				     cap->get_mseq());
+    in->encode_cap_message(m, cap);			     
+    mds->send_message_client_counted(m, it->first);
+  }
+
+  // should we increase max_size?
+  if (in->is_auth() && in->is_file())
+    check_inode_max_size(in);
+}
+
+void Locker::revoke_stale_caps(Session *session)
+{
+  dout(10) << "revoke_stale_caps for " << session->info.inst.name << dendl;
+  client_t client = session->get_client();
+
+  for (xlist::iterator p = session->caps.begin(); !p.end(); ++p) {
+    Capability *cap = *p;
+    cap->mark_stale();
+    CInode *in = cap->get_inode();
+    int issued = cap->issued();
+    if (issued & ~CEPH_CAP_PIN) {
+      dout(10) << " revoking " << ccap_string(issued) << " on " << *in << dendl;      
+      cap->revoke();
+
+      if (in->is_auth() &&
+	  in->inode.client_ranges.count(client))
+	in->state_set(CInode::STATE_NEEDSRECOVER);
+
+      if (!in->filelock.is_stable()) eval_gather(&in->filelock);
+      if (!in->linklock.is_stable()) eval_gather(&in->linklock);
+      if (!in->authlock.is_stable()) eval_gather(&in->authlock);
+      if (!in->xattrlock.is_stable()) eval_gather(&in->xattrlock);
+
+      if (in->is_auth()) {
+	try_eval(in, CEPH_CAP_LOCKS);
+      } else {
+	request_inode_file_caps(in);
+      }
+    } else {
+      dout(10) << " nothing issued on " << *in << dendl;
+    }
+  }
+}
+
+void Locker::resume_stale_caps(Session *session)
+{
+  dout(10) << "resume_stale_caps for " << session->info.inst.name << dendl;
+
+  for (xlist::iterator p = session->caps.begin(); !p.end(); ++p) {
+    Capability *cap = *p;
+    CInode *in = cap->get_inode();
+    assert(in->is_head());
+    if (cap->is_stale()) {
+      dout(10) << " clearing stale flag on " << *in << dendl;
+      cap->clear_stale();
+      if (!in->is_auth() || !eval(in, CEPH_CAP_LOCKS))
+	issue_caps(in, cap);
+    }
+  }
+}
+
+void Locker::remove_stale_leases(Session *session)
+{
+  dout(10) << "remove_stale_leases for " << session->info.inst.name << dendl;
+  xlist::iterator p = session->leases.begin();
+  while (!p.end()) {
+    ClientLease *l = *p;
+    ++p;
+    CDentry *parent = static_cast(l->parent);
+    dout(15) << " removing lease on " << *parent << dendl;
+    parent->remove_client_lease(l, this);
+  }
+}
+
+
+class C_MDL_RequestInodeFileCaps : public Context {
+  Locker *locker;
+  CInode *in;
+public:
+  C_MDL_RequestInodeFileCaps(Locker *l, CInode *i) : locker(l), in(i) {
+    in->get(CInode::PIN_PTRWAITER);
+  }
+  void finish(int r) {
+    in->put(CInode::PIN_PTRWAITER);
+    if (!in->is_auth())
+      locker->request_inode_file_caps(in);
+  }
+};
+
+void Locker::request_inode_file_caps(CInode *in)
+{
+  assert(!in->is_auth());
+
+  int wanted = in->get_caps_wanted() & ~CEPH_CAP_PIN;
+  if (wanted != in->replica_caps_wanted) {
+    // wait for single auth
+    if (in->is_ambiguous_auth()) {
+      in->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, 
+                     new C_MDL_RequestInodeFileCaps(this, in));
+      return;
+    }
+
+    int auth = in->authority().first;
+    if (mds->mdsmap->get_state(auth) == MDSMap::STATE_REJOIN) {
+      mds->wait_for_active_peer(auth, new C_MDL_RequestInodeFileCaps(this, in));
+      return;
+    }
+
+    dout(7) << "request_inode_file_caps " << ccap_string(wanted)
+            << " was " << ccap_string(in->replica_caps_wanted) 
+            << " on " << *in << " to mds." << auth << dendl;
+
+    in->replica_caps_wanted = wanted;
+
+    if (mds->mdsmap->is_clientreplay_or_active_or_stopping(auth))
+      mds->send_message_mds(new MInodeFileCaps(in->ino(), in->replica_caps_wanted),
+			    auth);
+  }
+}
+
+/* This function DOES put the passed message before returning */
+void Locker::handle_inode_file_caps(MInodeFileCaps *m)
+{
+  // nobody should be talking to us during recovery.
+  assert(mds->is_clientreplay() || mds->is_active() || mds->is_stopping());
+
+  // ok
+  CInode *in = mdcache->get_inode(m->get_ino());
+  int from = m->get_source().num();
+
+  assert(in);
+  assert(in->is_auth());
+
+  dout(7) << "handle_inode_file_caps replica mds." << from << " wants caps " << ccap_string(m->get_caps()) << " on " << *in << dendl;
+
+  if (m->get_caps())
+    in->mds_caps_wanted[from] = m->get_caps();
+  else
+    in->mds_caps_wanted.erase(from);
+
+  try_eval(in, CEPH_CAP_LOCKS);
+  m->put();
+}
+
+
+class C_MDL_CheckMaxSize : public Context {
+  Locker *locker;
+  CInode *in;
+  bool update_size;
+  uint64_t newsize;
+  bool update_max;
+  uint64_t new_max_size;
+  utime_t mtime;
+
+public:
+  C_MDL_CheckMaxSize(Locker *l, CInode *i, bool _update_size, uint64_t _newsize,
+                     bool _update_max, uint64_t _new_max_size, utime_t _mtime) :
+    locker(l), in(i),
+    update_size(_update_size), newsize(_newsize),
+    update_max(_update_max), new_max_size(_new_max_size),
+    mtime(_mtime)
+  {
+    in->get(CInode::PIN_PTRWAITER);
+  }
+  void finish(int r) {
+    in->put(CInode::PIN_PTRWAITER);
+    if (in->is_auth())
+      locker->check_inode_max_size(in, false, update_size, newsize,
+                                   update_max, new_max_size, mtime);
+  }
+};
+
+
+void Locker::calc_new_client_ranges(CInode *in, uint64_t size, map& new_ranges)
+{
+  inode_t *latest = in->get_projected_inode();
+  uint64_t ms;
+  if(latest->has_layout()) {
+    ms = ROUND_UP_TO((size+1)<<1, latest->get_layout_size_increment());
+  } else {
+    // Layout-less directories like ~mds0/, have zero size
+    ms = 0;
+  }
+
+  // increase ranges as appropriate.
+  // shrink to 0 if no WR|BUFFER caps issued.
+  for (map::iterator p = in->client_caps.begin();
+       p != in->client_caps.end();
+       ++p) {
+    if ((p->second->issued() | p->second->wanted()) & (CEPH_CAP_FILE_WR|CEPH_CAP_FILE_BUFFER)) {
+      client_writeable_range_t& nr = new_ranges[p->first];
+      nr.range.first = 0;
+      if (latest->client_ranges.count(p->first)) {
+	client_writeable_range_t& oldr = latest->client_ranges[p->first];
+	nr.range.last = MAX(ms, oldr.range.last);
+	nr.follows = oldr.follows;
+      } else {
+	nr.range.last = ms;
+	nr.follows = in->first - 1;
+      }
+    }
+  }
+}
+
+bool Locker::check_inode_max_size(CInode *in, bool force_wrlock,
+				  bool update_size, uint64_t new_size,
+				  bool update_max, uint64_t new_max_size,
+				  utime_t new_mtime)
+{
+  assert(in->is_auth());
+
+  inode_t *latest = in->get_projected_inode();
+  map new_ranges;
+  uint64_t size = latest->size;
+  bool new_max = update_max;
+
+  if (update_size) {
+    new_size = size = MAX(size, new_size);
+    new_mtime = MAX(new_mtime, latest->mtime);
+    if (latest->size == new_size && latest->mtime == new_mtime)
+      update_size = false;
+  }
+
+  uint64_t client_range_size = update_max ? new_max_size : size;
+
+  calc_new_client_ranges(in, client_range_size, new_ranges);
+
+  if (latest->client_ranges != new_ranges)
+    new_max = true;
+
+  if (!update_size && !new_max) {
+    dout(20) << "check_inode_max_size no-op on " << *in << dendl;
+    return false;
+  }
+
+  dout(10) << "check_inode_max_size new_ranges " << new_ranges
+	   << " update_size " << update_size
+	   << " on " << *in << dendl;
+
+  if (in->is_frozen()) {
+    dout(10) << "check_inode_max_size frozen, waiting on " << *in << dendl;
+    C_MDL_CheckMaxSize *cms = new C_MDL_CheckMaxSize(this, in,
+                                                     update_size, new_size,
+                                                     update_max, new_max_size,
+                                                     new_mtime);
+    in->add_waiter(CInode::WAIT_UNFREEZE, cms);
+    return false;
+  }
+  if (!force_wrlock && !in->filelock.can_wrlock(in->get_loner())) {
+    // lock?
+    if (in->filelock.is_stable()) {
+      if (in->get_target_loner() >= 0)
+	file_excl(&in->filelock);
+      else
+	simple_lock(&in->filelock);
+    }
+    if (!in->filelock.can_wrlock(in->get_loner())) {
+      // try again later
+      C_MDL_CheckMaxSize *cms = new C_MDL_CheckMaxSize(this, in,
+                                                       update_size, new_size,
+                                                       update_max, new_max_size,
+                                                       new_mtime);
+
+      in->filelock.add_waiter(SimpleLock::WAIT_STABLE, cms);
+      dout(10) << "check_inode_max_size can't wrlock, waiting on " << *in << dendl;
+      return false;    
+    }
+  }
+
+  MutationRef mut(new MutationImpl);
+  mut->ls = mds->mdlog->get_current_segment();
+    
+  inode_t *pi = in->project_inode();
+  pi->version = in->pre_dirty();
+
+  if (new_max) {
+    dout(10) << "check_inode_max_size client_ranges " << pi->client_ranges << " -> " << new_ranges << dendl;
+    pi->client_ranges = new_ranges;
+  }
+
+  if (update_size) {
+    dout(10) << "check_inode_max_size size " << pi->size << " -> " << new_size << dendl;
+    pi->size = new_size;
+    pi->rstat.rbytes = new_size;
+    dout(10) << "check_inode_max_size mtime " << pi->mtime << " -> " << new_mtime << dendl;
+    pi->mtime = new_mtime;
+  }
+
+  // use EOpen if the file is still open; otherwise, use EUpdate.
+  // this is just an optimization to push open files forward into
+  // newer log segments.
+  LogEvent *le;
+  EMetaBlob *metablob;
+  if (in->is_any_caps_wanted() && in->last == CEPH_NOSNAP) {   
+    EOpen *eo = new EOpen(mds->mdlog);
+    eo->add_ino(in->ino());
+    metablob = &eo->metablob;
+    le = eo;
+    mut->ls->open_files.push_back(&in->item_open_file);
+  } else {
+    EUpdate *eu = new EUpdate(mds->mdlog, "check_inode_max_size");
+    metablob = &eu->metablob;
+    le = eu;
+  }
+  mds->mdlog->start_entry(le);
+  if (update_size) {  // FIXME if/when we do max_size nested accounting
+    mdcache->predirty_journal_parents(mut, metablob, in, 0, PREDIRTY_PRIMARY);
+    // no cow, here!
+    CDentry *parent = in->get_projected_parent_dn();
+    metablob->add_primary_dentry(parent, in, true);
+  } else {
+    metablob->add_dir_context(in->get_projected_parent_dn()->get_dir());
+    mdcache->journal_dirty_inode(mut.get(), metablob, in);
+  }
+  mds->mdlog->submit_entry(le, new C_Locker_FileUpdate_finish(this, in, mut, true));
+  wrlock_force(&in->filelock, mut);  // wrlock for duration of journal
+  mut->auth_pin(in);
+
+  // make max_size _increase_ timely
+  if (new_max)
+    mds->mdlog->flush();
+
+  return true;
+}
+
+
+void Locker::share_inode_max_size(CInode *in, Capability *only_cap)
+{
+  /*
+   * only share if currently issued a WR cap.  if client doesn't have it,
+   * file_max doesn't matter, and the client will get it if/when they get
+   * the cap later.
+   */
+  dout(10) << "share_inode_max_size on " << *in << dendl;
+  map::iterator it;
+  if (only_cap)
+    it = in->client_caps.find(only_cap->get_client());
+  else
+    it = in->client_caps.begin();
+  for (; it != in->client_caps.end(); ++it) {
+    const client_t client = it->first;
+    Capability *cap = it->second;
+    if (cap->is_suppress())
+      continue;
+    if (cap->pending() & (CEPH_CAP_FILE_WR|CEPH_CAP_FILE_BUFFER)) {
+      dout(10) << "share_inode_max_size with client." << client << dendl;
+      cap->inc_last_seq();
+      MClientCaps *m = new MClientCaps(CEPH_CAP_OP_GRANT,
+				       in->ino(),
+				       in->find_snaprealm()->inode->ino(),
+				       cap->get_cap_id(), cap->get_last_seq(),
+				       cap->pending(), cap->wanted(), 0,
+				       cap->get_mseq());
+      in->encode_cap_message(m, cap);
+      mds->send_message_client_counted(m, client);
+    }
+    if (only_cap)
+      break;
+  }
+}
+
+void Locker::adjust_cap_wanted(Capability *cap, int wanted, int issue_seq)
+{
+  if (ceph_seq_cmp(issue_seq, cap->get_last_issue()) == 0) {
+    dout(10) << " wanted " << ccap_string(cap->wanted())
+	     << " -> " << ccap_string(wanted) << dendl;
+    cap->set_wanted(wanted);
+  } else if (wanted & ~cap->wanted()) {
+    dout(10) << " wanted " << ccap_string(cap->wanted())
+	     << " -> " << ccap_string(wanted)
+	     << " (added caps even though we had seq mismatch!)" << dendl;
+    cap->set_wanted(wanted | cap->wanted());
+  } else {
+    dout(10) << " NOT changing wanted " << ccap_string(cap->wanted())
+	     << " -> " << ccap_string(wanted)
+	     << " (issue_seq " << issue_seq << " != last_issue "
+	     << cap->get_last_issue() << ")" << dendl;
+    return;
+  }
+
+  CInode *cur = cap->get_inode();
+  if (!cur->is_auth()) {
+    request_inode_file_caps(cur);
+    return;
+  }
+
+  if (cap->wanted() == 0) {
+    if (cur->item_open_file.is_on_list() &&
+	!cur->is_any_caps_wanted()) {
+      dout(10) << " removing unwanted file from open file list " << *cur << dendl;
+      cur->item_open_file.remove_myself();
+    }
+  } else {
+    if (!cur->item_open_file.is_on_list()) {
+      dout(10) << " adding to open file list " << *cur << dendl;
+      assert(cur->last == CEPH_NOSNAP);
+      LogSegment *ls = mds->mdlog->get_current_segment();
+      EOpen *le = new EOpen(mds->mdlog);
+      mds->mdlog->start_entry(le);
+      le->add_clean_inode(cur);
+      ls->open_files.push_back(&cur->item_open_file);
+      mds->mdlog->submit_entry(le);
+    }
+  }
+}
+
+
+
+void Locker::_do_null_snapflush(CInode *head_in, client_t client, snapid_t follows)
+{
+  dout(10) << "_do_null_snapflish client." << client << " follows " << follows << " on " << *head_in << dendl;
+  map >::iterator p = head_in->client_need_snapflush.begin();
+  while (p != head_in->client_need_snapflush.end()) {
+    snapid_t snapid = p->first;
+    set& clients = p->second;
+    ++p;  // be careful, q loop below depends on this
+
+    // snapid is the snap inode's ->last
+    if (follows > snapid)
+      break;
+    if (clients.count(client)) {
+      dout(10) << " doing async NULL snapflush on " << snapid << " from client." << client << dendl;
+      CInode *sin = mdcache->get_inode(head_in->ino(), snapid);
+      if (!sin) {
+	// hrm, look forward until we find the inode. 
+	//  (we can only look it up by the last snapid it is valid for)
+	dout(10) << " didn't have " << head_in->ino() << " snapid " << snapid << dendl;
+	for (map >::iterator q = p;  // p is already at next entry
+	     q != head_in->client_need_snapflush.end();
+	     ++q) {
+	  dout(10) << " trying snapid " << q->first << dendl;
+	  sin = mdcache->get_inode(head_in->ino(), q->first);
+	  if (sin) {
+	    assert(sin->first <= snapid);
+	    break;
+	  }
+	  dout(10) << " didn't have " << head_in->ino() << " snapid " << q->first << dendl;
+	}
+	if (!sin && head_in->is_multiversion())
+	  sin = head_in;
+	assert(sin);
+      }
+      _do_snap_update(sin, snapid, 0, sin->first - 1, client, NULL, NULL);
+      head_in->remove_need_snapflush(sin, snapid, client);
+    }
+  }
+}
+
+
+bool Locker::should_defer_client_cap_frozen(CInode *in)
+{
+  /*
+   * This policy needs to be AT LEAST as permissive as allowing a client request
+   * to go forward, or else a client request can release something, the release
+   * gets deferred, but the request gets processed and deadlocks because when the
+   * caps can't get revoked.
+   *
+   * Currently, a request wait if anything locked is freezing (can't
+   * auth_pin), which would avoid any deadlock with cap release.  Thus @in
+   * _MUST_ be in the lock/auth_pin set.
+   *
+   * auth_pins==0 implies no unstable lock and not auth pinnned by
+   * client request, otherwise continue even it's freezing.
+   */
+  return (in->is_freezing() && in->get_num_auth_pins() == 0) || in->is_frozen();
+}
+
+/*
+ * This function DOES put the passed message before returning
+ */
+void Locker::handle_client_caps(MClientCaps *m)
+{
+  client_t client = m->get_source().num();
+
+  snapid_t follows = m->get_snap_follows();
+  dout(7) << "handle_client_caps on " << m->get_ino()
+	  << " follows " << follows 
+	  << " op " << ceph_cap_op_name(m->get_op()) << dendl;
+
+  if (!mds->is_clientreplay() && !mds->is_active() && !mds->is_stopping()) {
+    mds->wait_for_replay(new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+
+  CInode *head_in = mdcache->get_inode(m->get_ino());
+  if (!head_in) {
+    dout(7) << "handle_client_caps on unknown ino " << m->get_ino() << ", dropping" << dendl;
+    m->put();
+    return;
+  }
+
+  CInode *in = mdcache->pick_inode_snap(head_in, follows);
+  if (in != head_in)
+    dout(10) << " head inode " << *head_in << dendl;
+  dout(10) << "  cap inode " << *in << dendl;
+
+  Capability *cap = 0;
+  cap = in->get_client_cap(client);
+  if (!cap && in != head_in)
+    cap = head_in->get_client_cap(client);
+  if (!cap) {
+    dout(7) << "handle_client_caps no cap for client." << client << " on " << *in << dendl;
+    m->put();
+    return;
+  }  
+  assert(cap);
+
+  // freezing|frozen?
+  if (should_defer_client_cap_frozen(in)) {
+    dout(7) << "handle_client_caps freezing|frozen on " << *in << dendl;
+    in->add_waiter(CInode::WAIT_UNFREEZE, new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+  if (ceph_seq_cmp(m->get_mseq(), cap->get_mseq()) < 0) {
+    dout(7) << "handle_client_caps mseq " << m->get_mseq() << " < " << cap->get_mseq()
+	    << ", dropping" << dendl;
+    m->put();
+    return;
+  }
+
+  int op = m->get_op();
+
+  // flushsnap?
+  if (op == CEPH_CAP_OP_FLUSHSNAP) {
+    if (!in->is_auth()) {
+      dout(7) << " not auth, ignoring flushsnap on " << *in << dendl;
+      goto out;
+    }
+
+    SnapRealm *realm = in->find_snaprealm();
+    snapid_t snap = realm->get_snap_following(follows);
+    dout(10) << "  flushsnap follows " << follows << " -> snap " << snap << dendl;
+
+    if (in == head_in ||
+	(head_in->client_need_snapflush.count(snap) &&
+	 head_in->client_need_snapflush[snap].count(client))) {
+      dout(7) << " flushsnap snap " << snap
+	      << " client." << client << " on " << *in << dendl;
+
+      // this cap now follows a later snap (i.e. the one initiating this flush, or later)
+      cap->client_follows = MAX(follows, in->first) + 1;
+   
+      // we can prepare the ack now, since this FLUSHEDSNAP is independent of any
+      // other cap ops.  (except possibly duplicate FLUSHSNAP requests, but worst
+      // case we get a dup response, so whatever.)
+      MClientCaps *ack = 0;
+      if (m->get_dirty()) {
+	ack = new MClientCaps(CEPH_CAP_OP_FLUSHSNAP_ACK, in->ino(), 0, 0, 0, 0, 0, m->get_dirty(), 0);
+	ack->set_snap_follows(follows);
+	ack->set_client_tid(m->get_client_tid());
+      }
+
+      _do_snap_update(in, snap, m->get_dirty(), follows, client, m, ack);
+
+      if (in != head_in)
+	head_in->remove_need_snapflush(in, snap, client);
+      
+    } else
+      dout(7) << " not expecting flushsnap " << snap << " from client." << client << " on " << *in << dendl;
+    goto out;
+  }
+
+  if (cap->get_cap_id() != m->get_cap_id()) {
+    dout(7) << " ignoring client capid " << m->get_cap_id() << " != my " << cap->get_cap_id() << dendl;
+  } else {
+    // intermediate snap inodes
+    while (in != head_in) {
+      assert(in->last != CEPH_NOSNAP);
+      if (in->is_auth() && m->get_dirty()) {
+	dout(10) << " updating intermediate snapped inode " << *in << dendl;
+	_do_cap_update(in, NULL, m->get_dirty(), follows, m, NULL);
+      }
+      in = mdcache->pick_inode_snap(head_in, in->last);
+    }
+ 
+    // head inode, and cap
+    MClientCaps *ack = 0;
+
+    int caps = m->get_caps();
+    if (caps & ~cap->issued()) {
+      dout(10) << " confirming not issued caps " << ccap_string(caps & ~cap->issued()) << dendl;
+      caps &= cap->issued();
+    }
+    
+    cap->confirm_receipt(m->get_seq(), caps);
+    dout(10) << " follows " << follows
+	     << " retains " << ccap_string(m->get_caps())
+	     << " dirty " << ccap_string(m->get_dirty())
+	     << " on " << *in << dendl;
+
+
+    // missing/skipped snapflush?
+    //  The client MAY send a snapflush if it is issued WR/EXCL caps, but
+    //  presently only does so when it has actual dirty metadata.  But, we
+    //  set up the need_snapflush stuff based on the issued caps.
+    //  We can infer that the client WONT send a FLUSHSNAP once they have
+    //  released all WR/EXCL caps (the FLUSHSNAP always comes before the cap
+    //  update/release).
+    if (!head_in->client_need_snapflush.empty()) {
+      if ((cap->issued() & CEPH_CAP_ANY_FILE_WR) == 0) {
+	_do_null_snapflush(head_in, client, follows);
+      } else {
+	dout(10) << " revocation in progress, not making any conclusions about null snapflushes" << dendl;
+      }
+    }
+    
+    if (m->get_dirty() && in->is_auth()) {
+      dout(7) << " flush client." << client << " dirty " << ccap_string(m->get_dirty()) 
+	      << " seq " << m->get_seq() << " on " << *in << dendl;
+      ack = new MClientCaps(CEPH_CAP_OP_FLUSH_ACK, in->ino(), 0, cap->get_cap_id(), m->get_seq(),
+			    m->get_caps(), 0, m->get_dirty(), 0);
+      ack->set_client_tid(m->get_client_tid());
+    }
+
+    // filter wanted based on what we could ever give out (given auth/replica status)
+    int new_wanted = m->get_wanted() & head_in->get_caps_allowed_ever();
+    if (new_wanted != cap->wanted()) {
+      if (new_wanted & ~cap->wanted()) {
+	// exapnding caps.  make sure we aren't waiting for a log flush
+	if (!in->filelock.is_stable() ||
+	    !in->authlock.is_stable() ||
+	    !in->xattrlock.is_stable())
+	  mds->mdlog->flush();
+      }
+
+      adjust_cap_wanted(cap, new_wanted, m->get_issue_seq());
+    }
+      
+    if (in->is_auth() &&
+	_do_cap_update(in, cap, m->get_dirty(), follows, m, ack)) {
+      // updated
+      eval(in, CEPH_CAP_LOCKS);
+      
+      if (cap->wanted() & ~cap->pending())
+	mds->mdlog->flush();
+    } else {
+      // no update, ack now.
+      if (ack)
+	mds->send_message_client_counted(ack, m->get_connection());
+      
+      bool did_issue = eval(in, CEPH_CAP_LOCKS);
+      if (!did_issue && (cap->wanted() & ~cap->pending()))
+	issue_caps(in, cap);
+      if (cap->get_last_seq() == 0 &&
+	  (cap->pending() & (CEPH_CAP_FILE_WR|CEPH_CAP_FILE_BUFFER))) {
+	cap->issue_norevoke(cap->issued());
+	share_inode_max_size(in, cap);
+      }
+    }
+  }
+
+ out:
+  m->put();
+}
+
+class C_Locker_RetryRequestCapRelease : public Context {
+  Locker *locker;
+  client_t client;
+  ceph_mds_request_release item;
+public:
+  C_Locker_RetryRequestCapRelease(Locker *l, client_t c, const ceph_mds_request_release& it) :
+    locker(l), client(c), item(it) { }
+  void finish(int r) {
+    string dname;
+    MDRequestRef null_ref;
+    locker->process_request_cap_release(null_ref, client, item, dname);
+  }
+};
+
+void Locker::process_request_cap_release(MDRequestRef& mdr, client_t client, const ceph_mds_request_release& item,
+					 const string &dname)
+{
+  inodeno_t ino = (uint64_t)item.ino;
+  uint64_t cap_id = item.cap_id;
+  int caps = item.caps;
+  int wanted = item.wanted;
+  int seq = item.seq;
+  int issue_seq = item.issue_seq;
+  int mseq = item.mseq;
+
+  CInode *in = mdcache->get_inode(ino);
+  if (!in)
+    return;
+
+  if (dname.length()) {
+    frag_t fg = in->pick_dirfrag(dname);
+    CDir *dir = in->get_dirfrag(fg);
+    if (dir) {
+      CDentry *dn = dir->lookup(dname);
+      if (dn) {
+	ClientLease *l = dn->get_client_lease(client);
+	if (l) {
+	  dout(10) << "process_cap_release removing lease on " << *dn << dendl;
+	  dn->remove_client_lease(l, this);
+	} else {
+	  dout(7) << "process_cap_release client." << client
+		  << " doesn't have lease on " << *dn << dendl;
+	}
+      } else {
+	dout(7) << "process_cap_release client." << client << " released lease on dn "
+		<< dir->dirfrag() << "/" << dname << " which dne" << dendl;
+      }
+    }
+  }
+
+  Capability *cap = in->get_client_cap(client);
+  if (!cap)
+    return;
+
+  dout(10) << "process_cap_release client." << client << " " << ccap_string(caps) << " on " << *in
+	   << (mdr ? "" : " (DEFERRED, no mdr)")
+	   << dendl;
+    
+  if (ceph_seq_cmp(mseq, cap->get_mseq()) < 0) {
+    dout(7) << " mseq " << mseq << " < " << cap->get_mseq() << ", dropping" << dendl;
+    return;
+  }
+
+  if (cap->get_cap_id() != cap_id) {
+    dout(7) << " cap_id " << cap_id << " != " << cap->get_cap_id() << ", dropping" << dendl;
+    return;
+  }
+
+  if (should_defer_client_cap_frozen(in)) {
+    dout(7) << " frozen, deferring" << dendl;
+    in->add_waiter(CInode::WAIT_UNFREEZE, new C_Locker_RetryRequestCapRelease(this, client, item));
+    return;
+  }
+    
+  if (caps & ~cap->issued()) {
+    dout(10) << " confirming not issued caps " << ccap_string(caps & ~cap->issued()) << dendl;
+    caps &= cap->issued();
+  }
+  cap->confirm_receipt(seq, caps);
+  adjust_cap_wanted(cap, wanted, issue_seq);
+  
+  if (mdr)
+    cap->inc_suppress();
+  eval(in, CEPH_CAP_LOCKS);
+  if (mdr)
+    cap->dec_suppress();
+  
+  // take note; we may need to reissue on this cap later
+  if (mdr)
+    mdr->cap_releases[in->vino()] = cap->get_last_seq();
+}
+
+class C_Locker_RetryKickIssueCaps : public Context {
+  Locker *locker;
+  CInode *in;
+  client_t client;
+  ceph_seq_t seq;
+public:
+  C_Locker_RetryKickIssueCaps(Locker *l, CInode *i, client_t c, ceph_seq_t s) :
+    locker(l), in(i), client(c), seq(s) {
+    in->get(CInode::PIN_PTRWAITER);
+  }
+  void finish(int r) {
+    locker->kick_issue_caps(in, client, seq);
+    in->put(CInode::PIN_PTRWAITER);
+  }
+};
+
+void Locker::kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq)
+{
+  Capability *cap = in->get_client_cap(client);
+  if (!cap || cap->get_last_sent() != seq)
+    return;
+  if (in->is_frozen()) {
+    dout(10) << "kick_issue_caps waiting for unfreeze on " << *in << dendl;
+    in->add_waiter(CInode::WAIT_UNFREEZE,
+	new C_Locker_RetryKickIssueCaps(this, in, client, seq));
+    return;
+  }
+  dout(10) << "kick_issue_caps released at current seq " << seq
+    << ", reissuing" << dendl;
+  issue_caps(in, cap);
+}
+
+void Locker::kick_cap_releases(MDRequestRef& mdr)
+{
+  client_t client = mdr->get_client();
+  for (map::iterator p = mdr->cap_releases.begin();
+       p != mdr->cap_releases.end();
+       ++p) {
+    CInode *in = mdcache->get_inode(p->first);
+    if (!in)
+      continue;
+    kick_issue_caps(in, client, p->second);
+  }
+}
+
+static uint64_t calc_bounding(uint64_t t)
+{
+  t |= t >> 1;
+  t |= t >> 2;
+  t |= t >> 4;
+  t |= t >> 8;
+  t |= t >> 16;
+  t |= t >> 32;
+  return t + 1;
+}
+
+void Locker::_do_snap_update(CInode *in, snapid_t snap, int dirty, snapid_t follows, client_t client, MClientCaps *m, MClientCaps *ack)
+{
+  dout(10) << "_do_snap_update dirty " << ccap_string(dirty)
+	   << " follows " << follows << " snap " << snap
+	   << " on " << *in << dendl;
+
+  if (snap == CEPH_NOSNAP) {
+    // hmm, i guess snap was already deleted?  just ack!
+    dout(10) << " wow, the snap following " << follows
+	     << " was already deleted.  nothing to record, just ack." << dendl;
+    if (ack)
+      mds->send_message_client_counted(ack, m->get_connection());
+    return;
+  }
+
+  EUpdate *le = new EUpdate(mds->mdlog, "snap flush");
+  mds->mdlog->start_entry(le);
+  MutationRef mut(new MutationImpl);
+  mut->ls = mds->mdlog->get_current_segment();
+
+  // normal metadata updates that we can apply to the head as well.
+
+  // update xattrs?
+  bool xattrs = false;
+  map *px = 0;
+  if ((dirty & CEPH_CAP_XATTR_EXCL) && 
+      m->xattrbl.length() &&
+      m->head.xattr_version > in->get_projected_inode()->xattr_version)
+    xattrs = true;
+
+  old_inode_t *oi = 0;
+  if (in->is_multiversion()) {
+    oi = in->pick_old_inode(snap);
+    if (oi) {
+      dout(10) << " writing into old inode" << dendl;
+      if (xattrs)
+	px = &oi->xattrs;
+    }
+  }
+  if (xattrs && !px)
+    px = new map;
+
+  inode_t *pi = in->project_inode(px);
+  pi->version = in->pre_dirty();
+  if (oi)
+    pi = &oi->inode;
+
+  _update_cap_fields(in, dirty, m, pi);
+
+  // xattr
+  if (px) {
+    dout(7) << " xattrs v" << pi->xattr_version << " -> " << m->head.xattr_version
+	    << " len " << m->xattrbl.length() << dendl;
+    pi->xattr_version = m->head.xattr_version;
+    bufferlist::iterator p = m->xattrbl.begin();
+    ::decode(*px, p);
+  }
+
+  if (pi->client_ranges.count(client)) {
+    if (in->last == follows+1) {
+      dout(10) << "  removing client_range entirely" << dendl;
+      pi->client_ranges.erase(client);
+    } else {
+      dout(10) << "  client_range now follows " << snap << dendl;
+      pi->client_ranges[client].follows = snap;
+    }
+  }
+
+  mut->auth_pin(in);
+  mdcache->predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY, 0, follows);
+  mdcache->journal_dirty_inode(mut.get(), &le->metablob, in, follows);
+
+  mds->mdlog->submit_entry(le);
+  mds->mdlog->wait_for_safe(new C_Locker_FileUpdate_finish(this, in, mut, false,
+							   client, NULL, ack));
+}
+
+
+void Locker::_update_cap_fields(CInode *in, int dirty, MClientCaps *m, inode_t *pi)
+{
+  // file
+  if (dirty & (CEPH_CAP_FILE_EXCL|CEPH_CAP_FILE_WR)) {
+    utime_t atime = m->get_atime();
+    utime_t mtime = m->get_mtime();
+    utime_t ctime = m->get_ctime();
+    uint64_t size = m->get_size();
+    version_t inline_version = m->inline_version;
+    
+    if (((dirty & CEPH_CAP_FILE_WR) && mtime > pi->mtime) ||
+	((dirty & CEPH_CAP_FILE_EXCL) && mtime != pi->mtime)) {
+      dout(7) << "  mtime " << pi->mtime << " -> " << mtime
+	      << " for " << *in << dendl;
+      pi->mtime = mtime;
+    }
+    if (ctime > pi->ctime) {
+      dout(7) << "  ctime " << pi->ctime << " -> " << ctime
+	      << " for " << *in << dendl;
+      pi->ctime = ctime;
+    }
+    if (in->inode.is_file() &&   // ONLY if regular file
+	size > pi->size) {
+      dout(7) << "  size " << pi->size << " -> " << size
+	      << " for " << *in << dendl;
+      pi->size = size;
+      pi->rstat.rbytes = size;
+    }
+    if (in->inode.is_file() &&
+        (dirty & CEPH_CAP_FILE_WR) &&
+        inline_version > pi->inline_version) {
+      pi->inline_version = inline_version;
+      pi->inline_data = m->inline_data;
+    }
+    if ((dirty & CEPH_CAP_FILE_EXCL) && atime != pi->atime) {
+      dout(7) << "  atime " << pi->atime << " -> " << atime
+	      << " for " << *in << dendl;
+      pi->atime = atime;
+    }
+    if ((dirty & CEPH_CAP_FILE_EXCL) &&
+	ceph_seq_cmp(pi->time_warp_seq, m->get_time_warp_seq()) < 0) {
+      dout(7) << "  time_warp_seq " << pi->time_warp_seq << " -> " << m->get_time_warp_seq()
+	      << " for " << *in << dendl;
+      pi->time_warp_seq = m->get_time_warp_seq();
+    }
+  }
+  // auth
+  if (dirty & CEPH_CAP_AUTH_EXCL) {
+    if (m->head.uid != pi->uid) {
+      dout(7) << "  uid " << pi->uid
+	      << " -> " << m->head.uid
+	      << " for " << *in << dendl;
+      pi->uid = m->head.uid;
+    }
+    if (m->head.gid != pi->gid) {
+      dout(7) << "  gid " << pi->gid
+	      << " -> " << m->head.gid
+	      << " for " << *in << dendl;
+      pi->gid = m->head.gid;
+    }
+    if (m->head.mode != pi->mode) {
+      dout(7) << "  mode " << oct << pi->mode
+	      << " -> " << m->head.mode << dec
+	      << " for " << *in << dendl;
+      pi->mode = m->head.mode;
+    }
+  }
+
+}
+
+/*
+ * update inode based on cap flush|flushsnap|wanted.
+ *  adjust max_size, if needed.
+ * if we update, return true; otherwise, false (no updated needed).
+ */
+bool Locker::_do_cap_update(CInode *in, Capability *cap,
+			    int dirty, snapid_t follows, MClientCaps *m,
+			    MClientCaps *ack)
+{
+  dout(10) << "_do_cap_update dirty " << ccap_string(dirty)
+	   << " issued " << ccap_string(cap ? cap->issued() : 0)
+	   << " wanted " << ccap_string(cap ? cap->wanted() : 0)
+	   << " on " << *in << dendl;
+  assert(in->is_auth());
+  client_t client = m->get_source().num();
+  inode_t *latest = in->get_projected_inode();
+
+  // increase or zero max_size?
+  uint64_t size = m->get_size();
+  bool change_max = false;
+  uint64_t old_max = latest->client_ranges.count(client) ? latest->client_ranges[client].range.last : 0;
+  uint64_t new_max = old_max;
+  
+  if (in->is_file()) {
+    bool forced_change_max = false;
+    dout(20) << "inode is file" << dendl;
+    if (cap && ((cap->issued() | cap->wanted()) & CEPH_CAP_ANY_FILE_WR)) {
+      dout(20) << "client has write caps; m->get_max_size="
+               << m->get_max_size() << "; old_max=" << old_max << dendl;
+      if (m->get_max_size() > new_max) {
+	dout(10) << "client requests file_max " << m->get_max_size()
+		 << " > max " << old_max << dendl;
+	change_max = true;
+	forced_change_max = true;
+	new_max = ROUND_UP_TO((m->get_max_size()+1) << 1, latest->get_layout_size_increment());
+      } else {
+	new_max = calc_bounding(size * 2);
+	if (new_max < latest->get_layout_size_increment())
+	  new_max = latest->get_layout_size_increment();
+
+	if (new_max > old_max)
+	  change_max = true;
+	else
+	  new_max = old_max;
+      }
+    } else {
+      if (old_max) {
+	change_max = true;
+	new_max = 0;
+      }
+    }
+
+    if (in->last == CEPH_NOSNAP &&
+	change_max &&
+	!in->filelock.can_wrlock(client) &&
+	!in->filelock.can_force_wrlock(client)) {
+      dout(10) << " i want to change file_max, but lock won't allow it (yet)" << dendl;
+      if (in->filelock.is_stable()) {
+	bool need_issue = false;
+	if (cap)
+	  cap->inc_suppress();
+	if (in->mds_caps_wanted.empty() &&
+	    (in->get_loner() >= 0 || (in->get_wanted_loner() >= 0 && in->try_set_loner()))) {
+	  if (in->filelock.get_state() != LOCK_EXCL)
+	    file_excl(&in->filelock, &need_issue);
+	} else
+	  simple_lock(&in->filelock, &need_issue);
+	if (need_issue)
+	  issue_caps(in);
+	if (cap)
+	  cap->dec_suppress();
+      }
+      if (!in->filelock.can_wrlock(client) &&
+	  !in->filelock.can_force_wrlock(client)) {
+	C_MDL_CheckMaxSize *cms = new C_MDL_CheckMaxSize(this, in,
+	                                                 false, 0,
+	                                                 forced_change_max,
+	                                                 new_max,
+	                                                 utime_t());
+
+	in->filelock.add_waiter(SimpleLock::WAIT_STABLE, cms);
+	change_max = false;
+      }
+    }
+  }
+
+  if (m->flockbl.length()) {
+    int32_t num_locks;
+    bufferlist::iterator bli = m->flockbl.begin();
+    ::decode(num_locks, bli);
+    for ( int i=0; i < num_locks; ++i) {
+      ceph_filelock decoded_lock;
+      ::decode(decoded_lock, bli);
+      in->fcntl_locks.held_locks.
+	insert(pair(decoded_lock.start, decoded_lock));
+      ++in->fcntl_locks.client_held_lock_counts[(client_t)(decoded_lock.client)];
+    }
+    ::decode(num_locks, bli);
+    for ( int i=0; i < num_locks; ++i) {
+      ceph_filelock decoded_lock;
+      ::decode(decoded_lock, bli);
+      in->flock_locks.held_locks.
+	insert(pair(decoded_lock.start, decoded_lock));
+      ++in->flock_locks.client_held_lock_counts[(client_t)(decoded_lock.client)];
+    }
+  }
+
+  if (!dirty && !change_max)
+    return false;
+
+
+  // do the update.
+  EUpdate *le = new EUpdate(mds->mdlog, "cap update");
+  mds->mdlog->start_entry(le);
+
+  // xattrs update?
+  map *px = 0;
+  if ((dirty & CEPH_CAP_XATTR_EXCL) && 
+      m->xattrbl.length() &&
+      m->head.xattr_version > in->get_projected_inode()->xattr_version)
+    px = new map;
+
+  inode_t *pi = in->project_inode(px);
+  pi->version = in->pre_dirty();
+
+  MutationRef mut(new MutationImpl);
+  mut->ls = mds->mdlog->get_current_segment();
+
+  _update_cap_fields(in, dirty, m, pi);
+
+  if (change_max) {
+    dout(7) << "  max_size " << old_max << " -> " << new_max
+	    << " for " << *in << dendl;
+    if (new_max) {
+      pi->client_ranges[client].range.first = 0;
+      pi->client_ranges[client].range.last = new_max;
+      pi->client_ranges[client].follows = in->first - 1;
+    } else 
+      pi->client_ranges.erase(client);
+  }
+    
+  if (change_max || (dirty & (CEPH_CAP_FILE_EXCL|CEPH_CAP_FILE_WR))) 
+    wrlock_force(&in->filelock, mut);  // wrlock for duration of journal
+
+  // auth
+  if (dirty & CEPH_CAP_AUTH_EXCL)
+    wrlock_force(&in->authlock, mut);
+
+  // xattr
+  if (px) {
+    dout(7) << " xattrs v" << pi->xattr_version << " -> " << m->head.xattr_version << dendl;
+    pi->xattr_version = m->head.xattr_version;
+    bufferlist::iterator p = m->xattrbl.begin();
+    ::decode(*px, p);
+
+    wrlock_force(&in->xattrlock, mut);
+  }
+  
+  mut->auth_pin(in);
+  mdcache->predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY, 0, follows);
+  mdcache->journal_dirty_inode(mut.get(), &le->metablob, in, follows);
+
+  mds->mdlog->submit_entry(le);
+  mds->mdlog->wait_for_safe(new C_Locker_FileUpdate_finish(this, in, mut, change_max, 
+							   client, cap, ack));
+  // only flush immediately if the lock is unstable, or unissued caps are wanted, or max_size is 
+  // changing
+  if (((dirty & (CEPH_CAP_FILE_EXCL|CEPH_CAP_FILE_WR)) && !in->filelock.is_stable()) ||
+      ((dirty & CEPH_CAP_AUTH_EXCL) && !in->authlock.is_stable()) ||
+      ((dirty & CEPH_CAP_XATTR_EXCL) && !in->xattrlock.is_stable()) ||
+      (!dirty && (!in->filelock.is_stable() || !in->authlock.is_stable() || !in->xattrlock.is_stable())) ||  // nothing dirty + unstable lock -> probably a revoke?
+      (change_max && new_max) ||         // max INCREASE
+      (cap && (cap->wanted() & ~cap->pending())))
+    mds->mdlog->flush();
+
+  return true;
+}
+
+/* This function DOES put the passed message before returning */
+void Locker::handle_client_cap_release(MClientCapRelease *m)
+{
+  client_t client = m->get_source().num();
+  dout(10) << "handle_client_cap_release " << *m << dendl;
+
+  if (!mds->is_clientreplay() && !mds->is_active() && !mds->is_stopping()) {
+    mds->wait_for_replay(new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+
+  for (vector::iterator p = m->caps.begin(); p != m->caps.end(); ++p)
+    _do_cap_release(client, inodeno_t((uint64_t)p->ino) , p->cap_id, p->migrate_seq, p->seq);
+
+  m->put();
+}
+
+class C_Locker_RetryCapRelease : public Context {
+  Locker *locker;
+  client_t client;
+  inodeno_t ino;
+  uint64_t cap_id;
+  ceph_seq_t migrate_seq;
+  ceph_seq_t issue_seq;
+public:
+  C_Locker_RetryCapRelease(Locker *l, client_t c, inodeno_t i, uint64_t id,
+			   ceph_seq_t mseq, ceph_seq_t seq) :
+    locker(l), client(c), ino(i), cap_id(id), migrate_seq(mseq), issue_seq(seq) {}
+  void finish(int r) {
+    locker->_do_cap_release(client, ino, cap_id, migrate_seq, issue_seq);
+  }
+};
+
+void Locker::_do_cap_release(client_t client, inodeno_t ino, uint64_t cap_id,
+			     ceph_seq_t mseq, ceph_seq_t seq)
+{
+  CInode *in = mdcache->get_inode(ino);
+  if (!in) {
+    dout(7) << "_do_cap_release missing ino " << ino << dendl;
+    return;
+  }
+  Capability *cap = in->get_client_cap(client);
+  if (!cap) {
+    dout(7) << "_do_cap_release no cap for client" << client << " on "<< *in << dendl;
+    return;
+  }
+
+  dout(7) << "_do_cap_release for client." << client << " on "<< *in << dendl;
+  if (cap->get_cap_id() != cap_id) {
+    dout(7) << " capid " << cap_id << " != " << cap->get_cap_id() << ", ignore" << dendl;
+    return;
+  }
+  if (ceph_seq_cmp(mseq, cap->get_mseq()) < 0) {
+    dout(7) << " mseq " << mseq << " < " << cap->get_mseq() << ", ignore" << dendl;
+    return;
+  }
+  if (should_defer_client_cap_frozen(in)) {
+    dout(7) << " freezing|frozen, deferring" << dendl;
+    in->add_waiter(CInode::WAIT_UNFREEZE,
+                  new C_Locker_RetryCapRelease(this, client, ino, cap_id, mseq, seq));
+    return;
+  }
+  if (seq != cap->get_last_issue()) {
+    dout(7) << " issue_seq " << seq << " != " << cap->get_last_issue() << dendl;
+    // clean out any old revoke history
+    cap->clean_revoke_from(seq);
+    eval_cap_gather(in);
+    return;
+  }
+  remove_client_cap(in, client);
+}
+
+/* This function DOES put the passed message before returning */
+
+void Locker::remove_client_cap(CInode *in, client_t client)
+{
+  // clean out any pending snapflush state
+  if (!in->client_need_snapflush.empty())
+    _do_null_snapflush(in, client, 0);
+
+  in->remove_client_cap(client);
+
+  if (in->is_auth()) {
+    // make sure we clear out the client byte range
+    if (in->get_projected_inode()->client_ranges.count(client) &&
+	!(in->inode.nlink == 0 && !in->is_any_caps()))    // unless it's unlink + stray
+      check_inode_max_size(in);
+  } else {
+    request_inode_file_caps(in);
+  }
+  
+  try_eval(in, CEPH_CAP_LOCKS);
+}
+
+
+void Locker::handle_client_lease(MClientLease *m)
+{
+  dout(10) << "handle_client_lease " << *m << dendl;
+
+  assert(m->get_source().is_client());
+  client_t client = m->get_source().num();
+
+  CInode *in = mdcache->get_inode(m->get_ino(), m->get_last());
+  if (!in) {
+    dout(7) << "handle_client_lease don't have ino " << m->get_ino() << "." << m->get_last() << dendl;
+    m->put();
+    return;
+  }
+  CDentry *dn = 0;
+
+  frag_t fg = in->pick_dirfrag(m->dname);
+  CDir *dir = in->get_dirfrag(fg);
+  if (dir) 
+    dn = dir->lookup(m->dname);
+  if (!dn) {
+    dout(7) << "handle_client_lease don't have dn " << m->get_ino() << " " << m->dname << dendl;
+    m->put();
+    return;
+  }
+  dout(10) << " on " << *dn << dendl;
+
+  // replica and lock
+  ClientLease *l = dn->get_client_lease(client);
+  if (!l) {
+    dout(7) << "handle_client_lease didn't have lease for client." << client << " of " << *dn << dendl;
+    m->put();
+    return;
+  } 
+
+  switch (m->get_action()) {
+  case CEPH_MDS_LEASE_REVOKE_ACK:
+  case CEPH_MDS_LEASE_RELEASE:
+    if (l->seq != m->get_seq()) {
+      dout(7) << "handle_client_lease release - seq " << l->seq << " != provided " << m->get_seq() << dendl;
+    } else {
+      dout(7) << "handle_client_lease client." << client
+	      << " on " << *dn << dendl;
+      dn->remove_client_lease(l, this);
+    }
+    m->put();
+    break;
+
+  case CEPH_MDS_LEASE_RENEW:
+    {
+      dout(7) << "handle_client_lease client." << client << " renew on " << *dn
+	      << (!dn->lock.can_lease(client)?", revoking lease":"") << dendl;
+      if (dn->lock.can_lease(client)) {
+	int pool = 1;   // fixme.. do something smart!
+	m->h.duration_ms = (int)(1000 * mdcache->client_lease_durations[pool]);
+	m->h.seq = ++l->seq;
+	m->clear_payload();
+
+	utime_t now = ceph_clock_now(g_ceph_context);
+	now += mdcache->client_lease_durations[pool];
+	mdcache->touch_client_lease(l, pool, now);
+
+	mds->send_message_client_counted(m, m->get_connection());
+      }
+    }
+    break;
+
+  default:
+    assert(0); // implement me
+    break;
+  }
+}
+
+
+void Locker::issue_client_lease(CDentry *dn, client_t client,
+			       bufferlist &bl, utime_t now, Session *session)
+{
+  CInode *diri = dn->get_dir()->get_inode();
+  if (!diri->is_stray() &&  // do not issue dn leases in stray dir!
+      ((!diri->filelock.can_lease(client) &&
+	(diri->get_client_cap_pending(client) & (CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_EXCL)) == 0)) &&
+      dn->lock.can_lease(client)) {
+    int pool = 1;   // fixme.. do something smart!
+    // issue a dentry lease
+    ClientLease *l = dn->add_client_lease(client, session);
+    session->touch_lease(l);
+    
+    now += mdcache->client_lease_durations[pool];
+    mdcache->touch_client_lease(l, pool, now);
+
+    LeaseStat e;
+    e.mask = 1 | CEPH_LOCK_DN;  // old and new bit values
+    e.seq = ++l->seq;
+    e.duration_ms = (int)(1000 * mdcache->client_lease_durations[pool]);
+    ::encode(e, bl);
+    dout(20) << "issue_client_lease seq " << e.seq << " dur " << e.duration_ms << "ms "
+	     << " on " << *dn << dendl;
+  } else {
+    // null lease
+    LeaseStat e;
+    e.mask = 0;
+    e.seq = 0;
+    e.duration_ms = 0;
+    ::encode(e, bl);
+    dout(20) << "issue_client_lease no/null lease on " << *dn << dendl;
+  }
+}
+
+
+void Locker::revoke_client_leases(SimpleLock *lock)
+{
+  int n = 0;
+  CDentry *dn = static_cast(lock->get_parent());
+  for (map::iterator p = dn->client_lease_map.begin();
+       p != dn->client_lease_map.end();
+       ++p) {
+    ClientLease *l = p->second;
+    
+    n++;
+    assert(lock->get_type() == CEPH_LOCK_DN);
+
+    CDentry *dn = static_cast(lock->get_parent());
+    int mask = 1 | CEPH_LOCK_DN; // old and new bits
+    
+    // i should also revoke the dir ICONTENT lease, if they have it!
+    CInode *diri = dn->get_dir()->get_inode();
+    mds->send_message_client_counted(new MClientLease(CEPH_MDS_LEASE_REVOKE, l->seq,
+					      mask,
+					      diri->ino(),
+					      diri->first, CEPH_NOSNAP,
+					      dn->get_name()),
+			     l->client);
+  }
+  assert(n == lock->get_num_client_lease());
+}
+
+
+
+// locks ----------------------------------------------------------------
+
+SimpleLock *Locker::get_lock(int lock_type, MDSCacheObjectInfo &info) 
+{
+  switch (lock_type) {
+  case CEPH_LOCK_DN:
+    {
+      // be careful; info.dirfrag may have incorrect frag; recalculate based on dname.
+      CInode *diri = mdcache->get_inode(info.dirfrag.ino);
+      frag_t fg;
+      CDir *dir = 0;
+      CDentry *dn = 0;
+      if (diri) {
+	fg = diri->pick_dirfrag(info.dname);
+	dir = diri->get_dirfrag(fg);
+	if (dir) 
+	  dn = dir->lookup(info.dname, info.snapid);
+      }
+      if (!dn) {
+	dout(7) << "get_lock don't have dn " << info.dirfrag.ino << " " << info.dname << dendl;
+	return 0;
+      }
+      return &dn->lock;
+    }
+
+  case CEPH_LOCK_IAUTH:
+  case CEPH_LOCK_ILINK:
+  case CEPH_LOCK_IDFT:
+  case CEPH_LOCK_IFILE:
+  case CEPH_LOCK_INEST:
+  case CEPH_LOCK_IXATTR:
+  case CEPH_LOCK_ISNAP:
+  case CEPH_LOCK_IFLOCK:
+  case CEPH_LOCK_IPOLICY:
+    {
+      CInode *in = mdcache->get_inode(info.ino, info.snapid);
+      if (!in) {
+	dout(7) << "get_lock don't have ino " << info.ino << dendl;
+	return 0;
+      }
+      switch (lock_type) {
+      case CEPH_LOCK_IAUTH: return &in->authlock;
+      case CEPH_LOCK_ILINK: return &in->linklock;
+      case CEPH_LOCK_IDFT: return &in->dirfragtreelock;
+      case CEPH_LOCK_IFILE: return &in->filelock;
+      case CEPH_LOCK_INEST: return &in->nestlock;
+      case CEPH_LOCK_IXATTR: return &in->xattrlock;
+      case CEPH_LOCK_ISNAP: return &in->snaplock;
+      case CEPH_LOCK_IFLOCK: return &in->flocklock;
+      case CEPH_LOCK_IPOLICY: return &in->policylock;
+      }
+    }
+
+  default:
+    dout(7) << "get_lock don't know lock_type " << lock_type << dendl;
+    assert(0);
+    break;
+  }
+
+  return 0;  
+}
+
+/* This function DOES put the passed message before returning */
+void Locker::handle_lock(MLock *m)
+{
+  // nobody should be talking to us during recovery.
+  assert(mds->is_rejoin() || mds->is_clientreplay() || mds->is_active() || mds->is_stopping());
+
+  SimpleLock *lock = get_lock(m->get_lock_type(), m->get_object_info());
+  if (!lock) {
+    dout(10) << "don't have object " << m->get_object_info() << ", must have trimmed, dropping" << dendl;
+    m->put();
+    return;
+  }
+
+  switch (lock->get_type()) {
+  case CEPH_LOCK_DN:
+  case CEPH_LOCK_IAUTH:
+  case CEPH_LOCK_ILINK:
+  case CEPH_LOCK_ISNAP:
+  case CEPH_LOCK_IXATTR:
+  case CEPH_LOCK_IFLOCK:
+  case CEPH_LOCK_IPOLICY:
+    handle_simple_lock(lock, m);
+    break;
+    
+  case CEPH_LOCK_IDFT:
+  case CEPH_LOCK_INEST:
+    //handle_scatter_lock((ScatterLock*)lock, m);
+    //break;
+
+  case CEPH_LOCK_IFILE:
+    handle_file_lock(static_cast(lock), m);
+    break;
+    
+  default:
+    dout(7) << "handle_lock got otype " << m->get_lock_type() << dendl;
+    assert(0);
+    break;
+  }
+}
+ 
+
+
+
+
+// ==========================================================================
+// simple lock
+
+/** This function may take a reference to m if it needs one, but does
+ * not put references. */
+void Locker::handle_reqrdlock(SimpleLock *lock, MLock *m)
+{
+  MDSCacheObject *parent = lock->get_parent();
+  if (parent->is_auth() &&
+      lock->get_state() != LOCK_SYNC &&
+      !parent->is_frozen()) {
+    dout(7) << "handle_reqrdlock got rdlock request on " << *lock
+	    << " on " << *parent << dendl;
+    assert(parent->is_auth()); // replica auth pinned if they're doing this!
+    if (lock->is_stable()) {
+      simple_sync(lock);
+    } else {
+      dout(7) << "handle_reqrdlock delaying request until lock is stable" << dendl;
+      lock->add_waiter(SimpleLock::WAIT_STABLE | MDSCacheObject::WAIT_UNFREEZE,
+                       new C_MDS_RetryMessage(mds, m->get()));
+    }
+  } else {
+    dout(7) << "handle_reqrdlock dropping rdlock request on " << *lock
+	    << " on " << *parent << dendl;
+    // replica should retry
+  }
+}
+
+/* This function DOES put the passed message before returning */
+void Locker::handle_simple_lock(SimpleLock *lock, MLock *m)
+{
+  int from = m->get_asker();
+  
+  dout(10) << "handle_simple_lock " << *m
+	   << " on " << *lock << " " << *lock->get_parent() << dendl;
+
+  if (mds->is_rejoin()) {
+    if (lock->get_parent()->is_rejoining()) {
+      dout(7) << "handle_simple_lock still rejoining " << *lock->get_parent()
+	      << ", dropping " << *m << dendl;
+      m->put();
+      return;
+    }
+  }
+
+  switch (m->get_action()) {
+    // -- replica --
+  case LOCK_AC_SYNC:
+    assert(lock->get_state() == LOCK_LOCK);
+    lock->decode_locked_state(m->get_data());
+    lock->set_state(LOCK_SYNC);
+    lock->finish_waiters(SimpleLock::WAIT_RD|SimpleLock::WAIT_STABLE);
+    break;
+    
+  case LOCK_AC_LOCK:
+    assert(lock->get_state() == LOCK_SYNC);
+    lock->set_state(LOCK_SYNC_LOCK);
+    if (lock->is_leased())
+      revoke_client_leases(lock);
+    eval_gather(lock, true);
+    break;
+
+
+    // -- auth --
+  case LOCK_AC_LOCKACK:
+    assert(lock->get_state() == LOCK_SYNC_LOCK ||
+	   lock->get_state() == LOCK_SYNC_EXCL);
+    assert(lock->is_gathering(from));
+    lock->remove_gather(from);
+    
+    if (lock->is_gathering()) {
+      dout(7) << "handle_simple_lock " << *lock << " on " << *lock->get_parent() << " from " << from
+	      << ", still gathering " << lock->get_gather_set() << dendl;
+    } else {
+      dout(7) << "handle_simple_lock " << *lock << " on " << *lock->get_parent() << " from " << from
+	      << ", last one" << dendl;
+      eval_gather(lock);
+    }
+    break;
+
+  case LOCK_AC_REQRDLOCK:
+    handle_reqrdlock(lock, m);
+    break;
+
+  }
+
+  m->put();
+}
+
+/* unused, currently.
+
+class C_Locker_SimpleEval : public Context {
+  Locker *locker;
+  SimpleLock *lock;
+public:
+  C_Locker_SimpleEval(Locker *l, SimpleLock *lk) : locker(l), lock(lk) {}
+  void finish(int r) {
+    locker->try_simple_eval(lock);
+  }
+};
+
+void Locker::try_simple_eval(SimpleLock *lock)
+{
+  // unstable and ambiguous auth?
+  if (!lock->is_stable() &&
+      lock->get_parent()->is_ambiguous_auth()) {
+    dout(7) << "simple_eval not stable and ambiguous auth, waiting on " << *lock->get_parent() << dendl;
+    //if (!lock->get_parent()->is_waiter(MDSCacheObject::WAIT_SINGLEAUTH))
+    lock->get_parent()->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, new C_Locker_SimpleEval(this, lock));
+    return;
+  }
+
+  if (!lock->get_parent()->is_auth()) {
+    dout(7) << "try_simple_eval not auth for " << *lock->get_parent() << dendl;
+    return;
+  }
+
+  if (!lock->get_parent()->can_auth_pin()) {
+    dout(7) << "try_simple_eval can't auth_pin, waiting on " << *lock->get_parent() << dendl;
+    //if (!lock->get_parent()->is_waiter(MDSCacheObject::WAIT_SINGLEAUTH))
+    lock->get_parent()->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_Locker_SimpleEval(this, lock));
+    return;
+  }
+
+  if (lock->is_stable())
+    simple_eval(lock);
+}
+*/
+
+
+void Locker::simple_eval(SimpleLock *lock, bool *need_issue)
+{
+  dout(10) << "simple_eval " << *lock << " on " << *lock->get_parent() << dendl;
+
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  if (lock->get_parent()->is_freezing_or_frozen()) {
+    // dentry lock in unreadable state can block path traverse
+    if ((lock->get_type() != CEPH_LOCK_DN ||
+	 lock->get_state() == LOCK_SYNC ||
+	 lock->get_parent()->is_frozen()))
+      return;
+  }
+
+  CInode *in = 0;
+  int wanted = 0;
+  if (lock->get_type() != CEPH_LOCK_DN) {
+    in = static_cast(lock->get_parent());
+    in->get_caps_wanted(&wanted, NULL, lock->get_cap_shift());
+  }
+  
+  // -> excl?
+  if (lock->get_state() != LOCK_EXCL &&
+      in && in->get_target_loner() >= 0 &&
+      (wanted & CEPH_CAP_GEXCL)) {
+    dout(7) << "simple_eval stable, going to excl " << *lock 
+	    << " on " << *lock->get_parent() << dendl;
+    simple_excl(lock, need_issue);
+  }
+
+  // stable -> sync?
+  else if (lock->get_state() != LOCK_SYNC &&
+	   !lock->is_wrlocked() &&
+	   ((!(wanted & CEPH_CAP_GEXCL) && !lock->is_waiter_for(SimpleLock::WAIT_WR)) ||
+	    (lock->get_state() == LOCK_EXCL && in && in->get_target_loner() < 0))) {
+    dout(7) << "simple_eval stable, syncing " << *lock 
+	    << " on " << *lock->get_parent() << dendl;
+    simple_sync(lock, need_issue);
+  }
+}
+
+
+// mid
+
+bool Locker::simple_sync(SimpleLock *lock, bool *need_issue)
+{
+  dout(7) << "simple_sync on " << *lock << " on " << *lock->get_parent() << dendl;
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  CInode *in = 0;
+  if (lock->get_cap_shift())
+    in = static_cast(lock->get_parent());
+
+  int old_state = lock->get_state();
+
+  if (old_state != LOCK_TSYN) {
+
+    switch (lock->get_state()) {
+    case LOCK_MIX: lock->set_state(LOCK_MIX_SYNC); break;
+    case LOCK_LOCK: lock->set_state(LOCK_LOCK_SYNC); break;
+    case LOCK_XSYN: lock->set_state(LOCK_XSYN_SYNC); break;
+    case LOCK_EXCL: lock->set_state(LOCK_EXCL_SYNC); break;
+    default: assert(0);
+    }
+
+    int gather = 0;
+    if (lock->is_wrlocked())
+      gather++;
+    
+    if (lock->get_parent()->is_replicated() && old_state == LOCK_MIX) {
+      send_lock_message(lock, LOCK_AC_SYNC);
+      lock->init_gather();
+      gather++;
+    }
+    
+    if (in && in->is_head()) {
+      if (in->issued_caps_need_gather(lock)) {
+	if (need_issue)
+	  *need_issue = true;
+	else
+	  issue_caps(in);
+	gather++;
+      }
+    }
+    
+    bool need_recover = false;
+    if (lock->get_type() == CEPH_LOCK_IFILE) {
+      assert(in);
+      if (in->state_test(CInode::STATE_NEEDSRECOVER)) {
+        mds->mdcache->queue_file_recover(in);
+	need_recover = true;
+        gather++;
+      }
+    }
+    
+    if (!gather && lock->is_dirty()) {
+      lock->get_parent()->auth_pin(lock);
+      scatter_writebehind(static_cast(lock));
+      mds->mdlog->flush();
+      return false;
+    }
+
+    if (gather) {
+      lock->get_parent()->auth_pin(lock);
+      if (need_recover)
+	mds->mdcache->do_file_recover();
+      return false;
+    }
+  }
+
+  if (lock->get_parent()->is_replicated()) {    // FIXME
+    bufferlist data;
+    lock->encode_locked_state(data);
+    send_lock_message(lock, LOCK_AC_SYNC, data);
+  }
+  lock->set_state(LOCK_SYNC);
+  lock->finish_waiters(SimpleLock::WAIT_RD|SimpleLock::WAIT_STABLE);
+  if (in && in->is_head()) {
+    if (need_issue)
+      *need_issue = true;
+    else
+      issue_caps(in);
+  }
+  return true;
+}
+
+void Locker::simple_excl(SimpleLock *lock, bool *need_issue)
+{
+  dout(7) << "simple_excl on " << *lock << " on " << *lock->get_parent() << dendl;
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  CInode *in = 0;
+  if (lock->get_cap_shift())
+    in = static_cast(lock->get_parent());
+
+  switch (lock->get_state()) {
+  case LOCK_LOCK: lock->set_state(LOCK_LOCK_EXCL); break;
+  case LOCK_SYNC: lock->set_state(LOCK_SYNC_EXCL); break;
+  case LOCK_XSYN: lock->set_state(LOCK_XSYN_EXCL); break;
+  default: assert(0);
+  }
+  
+  int gather = 0;
+  if (lock->is_rdlocked())
+    gather++;
+  if (lock->is_wrlocked())
+    gather++;
+
+  if (lock->get_parent()->is_replicated() && 
+      lock->get_state() != LOCK_LOCK_EXCL &&
+      lock->get_state() != LOCK_XSYN_EXCL) {
+    send_lock_message(lock, LOCK_AC_LOCK);
+    lock->init_gather();
+    gather++;
+  }
+  
+  if (in && in->is_head()) {
+    if (in->issued_caps_need_gather(lock)) {
+      if (need_issue)
+	*need_issue = true;
+      else
+	issue_caps(in);
+      gather++;
+    }
+  }
+  
+  if (gather) {
+    lock->get_parent()->auth_pin(lock);
+  } else {
+    lock->set_state(LOCK_EXCL);
+    lock->finish_waiters(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE);
+    if (in) {
+      if (need_issue)
+	*need_issue = true;
+      else
+	issue_caps(in);
+    }
+  }
+}
+
+void Locker::simple_lock(SimpleLock *lock, bool *need_issue)
+{
+  dout(7) << "simple_lock on " << *lock << " on " << *lock->get_parent() << dendl;
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+  assert(lock->get_state() != LOCK_LOCK);
+  
+  CInode *in = 0;
+  if (lock->get_cap_shift())
+    in = static_cast(lock->get_parent());
+
+  int old_state = lock->get_state();
+
+  switch (lock->get_state()) {
+  case LOCK_SYNC: lock->set_state(LOCK_SYNC_LOCK); break;
+  case LOCK_XSYN:
+    file_excl(static_cast(lock), need_issue);
+    if (lock->get_state() != LOCK_EXCL)
+      return;
+    // fall-thru
+  case LOCK_EXCL: lock->set_state(LOCK_EXCL_LOCK); break;
+  case LOCK_MIX: lock->set_state(LOCK_MIX_LOCK);
+    (static_cast(lock))->clear_unscatter_wanted();
+    break;
+  case LOCK_TSYN: lock->set_state(LOCK_TSYN_LOCK); break;
+  default: assert(0);
+  }
+
+  int gather = 0;
+  if (lock->is_leased()) {
+    gather++;
+    revoke_client_leases(lock);
+  }
+  if (lock->is_rdlocked())
+    gather++;
+  if (in && in->is_head()) {
+    if (in->issued_caps_need_gather(lock)) {
+      if (need_issue)
+	*need_issue = true;
+      else
+	issue_caps(in);
+      gather++;
+    }
+  }
+
+  bool need_recover = false;
+  if (lock->get_type() == CEPH_LOCK_IFILE) {
+    assert(in);
+    if(in->state_test(CInode::STATE_NEEDSRECOVER)) {
+      mds->mdcache->queue_file_recover(in);
+      need_recover = true;
+      gather++;
+    }
+  }
+
+  if (lock->get_parent()->is_replicated() &&
+      lock->get_state() == LOCK_MIX_LOCK &&
+      gather) {
+    dout(10) << " doing local stage of mix->lock gather before gathering from replicas" << dendl;
+  } else {
+    // move to second stage of gather now, so we don't send the lock action later.
+    if (lock->get_state() == LOCK_MIX_LOCK)
+      lock->set_state(LOCK_MIX_LOCK2);
+
+    if (lock->get_parent()->is_replicated() &&
+	lock->get_sm()->states[old_state].replica_state != LOCK_LOCK) {  // replica may already be LOCK
+      gather++;
+      send_lock_message(lock, LOCK_AC_LOCK);
+      lock->init_gather();
+    }
+  }
+
+  if (!gather && lock->is_dirty()) {
+    lock->get_parent()->auth_pin(lock);
+    scatter_writebehind(static_cast(lock));
+    mds->mdlog->flush();
+    return;
+  }
+
+  if (gather) {
+    lock->get_parent()->auth_pin(lock);
+    if (need_recover)
+      mds->mdcache->do_file_recover();
+  } else {
+    lock->set_state(LOCK_LOCK);
+    lock->finish_waiters(ScatterLock::WAIT_XLOCK|ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
+  }
+}
+
+
+void Locker::simple_xlock(SimpleLock *lock)
+{
+  dout(7) << "simple_xlock on " << *lock << " on " << *lock->get_parent() << dendl;
+  assert(lock->get_parent()->is_auth());
+  //assert(lock->is_stable());
+  assert(lock->get_state() != LOCK_XLOCK);
+  
+  CInode *in = 0;
+  if (lock->get_cap_shift())
+    in = static_cast(lock->get_parent());
+
+  if (lock->is_stable())
+    lock->get_parent()->auth_pin(lock);
+
+  switch (lock->get_state()) {
+  case LOCK_LOCK: 
+  case LOCK_XLOCKDONE: lock->set_state(LOCK_LOCK_XLOCK); break;
+  default: assert(0);
+  }
+
+  int gather = 0;
+  if (lock->is_rdlocked())
+    gather++;
+  if (lock->is_wrlocked())
+    gather++;
+  
+  if (in && in->is_head()) {
+    if (in->issued_caps_need_gather(lock)) {
+      issue_caps(in);
+      gather++;
+    }
+  }
+
+  if (!gather) {
+    lock->set_state(LOCK_PREXLOCK);
+    //assert("shouldn't be called if we are already xlockable" == 0);
+  }
+}
+
+
+
+
+
+// ==========================================================================
+// scatter lock
+
+/*
+
+Some notes on scatterlocks.
+
+ - The scatter/gather is driven by the inode lock.  The scatter always
+   brings in the latest metadata from the fragments.
+
+ - When in a scattered/MIX state, fragments are only allowed to
+   update/be written to if the accounted stat matches the inode's
+   current version.
+
+ - That means, on gather, we _only_ assimilate diffs for frag metadata
+   that match the current version, because those are the only ones
+   written during this scatter/gather cycle.  (Others didn't permit
+   it.)  We increment the version and journal this to disk.
+
+ - When possible, we also simultaneously update our local frag
+   accounted stats to match.
+
+ - On scatter, the new inode info is broadcast to frags, both local
+   and remote.  If possible (auth and !frozen), the dirfrag auth
+   should update the accounted state (if it isn't already up to date).
+   Note that this may occur on both the local inode auth node and
+   inode replicas, so there are two potential paths. If it is NOT
+   possible, they need to mark_stale to prevent any possible writes.
+
+ - A scatter can be to MIX (potentially writeable) or to SYNC (read
+   only).  Both are opportunities to update the frag accounted stats,
+   even though only the MIX case is affected by a stale dirfrag.
+
+ - Because many scatter/gather cycles can potentially go by without a
+   frag being able to update its accounted stats (due to being frozen
+   by exports/refragments in progress), the frag may have (even very)
+   old stat versions.  That's fine.  If when we do want to update it,
+   we can update accounted_* and the version first.
+
+*/
+
+void Locker::scatter_writebehind(ScatterLock *lock)
+{
+  CInode *in = static_cast(lock->get_parent());
+  dout(10) << "scatter_writebehind " << in->inode.mtime << " on " << *lock << " on " << *in << dendl;
+
+  // journal
+  MutationRef mut(new MutationImpl);
+  mut->ls = mds->mdlog->get_current_segment();
+
+  // forcefully take a wrlock
+  lock->get_wrlock(true);
+  mut->wrlocks.insert(lock);
+  mut->locks.insert(lock);
+
+  in->pre_cow_old_inode();  // avoid cow mayhem
+
+  inode_t *pi = in->project_inode();
+  pi->version = in->pre_dirty();
+
+  in->finish_scatter_gather_update(lock->get_type());
+  lock->start_flush();
+
+  EUpdate *le = new EUpdate(mds->mdlog, "scatter_writebehind");
+  mds->mdlog->start_entry(le);
+
+  mdcache->predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mut.get(), &le->metablob, in);
+  
+  in->finish_scatter_gather_update_accounted(lock->get_type(), mut, &le->metablob);
+
+  mds->mdlog->submit_entry(le);
+  mds->mdlog->wait_for_safe(new C_Locker_ScatterWB(this, lock, mut));
+}
+
+void Locker::scatter_writebehind_finish(ScatterLock *lock, MutationRef& mut)
+{
+  CInode *in = static_cast(lock->get_parent());
+  dout(10) << "scatter_writebehind_finish on " << *lock << " on " << *in << dendl;
+  in->pop_and_dirty_projected_inode(mut->ls);
+
+  lock->finish_flush();
+
+  // if replicas may have flushed in a mix->lock state, send another
+  // message so they can finish_flush().
+  if (in->is_replicated()) {
+    switch (lock->get_state()) {
+    case LOCK_MIX_LOCK:
+    case LOCK_MIX_LOCK2:
+    case LOCK_MIX_EXCL:
+    case LOCK_MIX_TSYN:
+      send_lock_message(lock, LOCK_AC_LOCKFLUSHED);
+    }
+  }
+
+  mut->apply();
+  drop_locks(mut.get());
+  mut->cleanup();
+
+  if (lock->is_stable())
+    lock->finish_waiters(ScatterLock::WAIT_STABLE);
+
+  //scatter_eval_gather(lock);
+}
+
+void Locker::scatter_eval(ScatterLock *lock, bool *need_issue)
+{
+  dout(10) << "scatter_eval " << *lock << " on " << *lock->get_parent() << dendl;
+
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  if (lock->get_parent()->is_freezing_or_frozen()) {
+    dout(20) << "  freezing|frozen" << dendl;
+    return;
+  }
+  
+  if (!lock->is_rdlocked() &&
+      lock->get_state() != LOCK_MIX &&
+      lock->get_scatter_wanted()) {
+    dout(10) << "scatter_eval scatter_wanted, bump to mix " << *lock
+	     << " on " << *lock->get_parent() << dendl;
+    scatter_mix(lock, need_issue);
+    return;
+  }
+
+  if (lock->get_type() == CEPH_LOCK_INEST) {
+    // in general, we want to keep INEST writable at all times.
+    if (!lock->is_rdlocked()) {
+      if (lock->get_parent()->is_replicated()) {
+	if (lock->get_state() != LOCK_MIX)
+	  scatter_mix(lock, need_issue);
+      } else {
+	if (lock->get_state() != LOCK_LOCK)
+	  simple_lock(lock, need_issue);
+      }
+    }
+    return;
+  }
+
+  CInode *in = static_cast(lock->get_parent());
+  if (!in->has_subtree_or_exporting_dirfrag() || in->is_base()) {
+    // i _should_ be sync.
+    if (!lock->is_wrlocked() &&
+	lock->get_state() != LOCK_SYNC) {
+      dout(10) << "scatter_eval no wrlocks|xlocks, not subtree root inode, syncing" << dendl;
+      simple_sync(lock, need_issue);
+    }
+  }
+}
+
+
+/*
+ * mark a scatterlock to indicate that the dir fnode has some dirty data
+ */
+void Locker::mark_updated_scatterlock(ScatterLock *lock)
+{
+  lock->mark_dirty();
+  if (lock->get_updated_item()->is_on_list()) {
+    dout(10) << "mark_updated_scatterlock " << *lock
+	     << " - already on list since " << lock->get_update_stamp() << dendl;
+  } else {
+    updated_scatterlocks.push_back(lock->get_updated_item());
+    utime_t now = ceph_clock_now(g_ceph_context);
+    lock->set_update_stamp(now);
+    dout(10) << "mark_updated_scatterlock " << *lock
+	     << " - added at " << now << dendl;
+  }
+}
+
+/*
+ * this is called by scatter_tick and LogSegment::try_to_trim() when
+ * trying to flush dirty scattered data (i.e. updated fnode) back to
+ * the inode.
+ *
+ * we need to lock|scatter in order to push fnode changes into the
+ * inode.dirstat.
+ */
+void Locker::scatter_nudge(ScatterLock *lock, Context *c, bool forcelockchange)
+{
+  CInode *p = static_cast(lock->get_parent());
+
+  if (p->is_frozen() || p->is_freezing()) {
+    dout(10) << "scatter_nudge waiting for unfreeze on " << *p << dendl;
+    if (c) 
+      p->add_waiter(MDSCacheObject::WAIT_UNFREEZE, c);
+    else
+      // just requeue.  not ideal.. starvation prone..
+      updated_scatterlocks.push_back(lock->get_updated_item());
+    return;
+  }
+
+  if (p->is_ambiguous_auth()) {
+    dout(10) << "scatter_nudge waiting for single auth on " << *p << dendl;
+    if (c) 
+      p->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, c);
+    else
+      // just requeue.  not ideal.. starvation prone..
+      updated_scatterlocks.push_back(lock->get_updated_item());
+    return;
+  }
+
+  if (p->is_auth()) {
+    int count = 0;
+    while (true) {
+      if (lock->is_stable()) {
+	// can we do it now?
+	//  (only if we're not replicated.. if we are, we really do need
+	//   to nudge the lock state!)
+	/*
+	  actually, even if we're not replicated, we can't stay in MIX, because another mds
+	  could discover and replicate us at any time.  if that happens while we're flushing,
+	  they end up in MIX but their inode has the old scatterstat version.
+
+	if (!forcelockchange && !lock->get_parent()->is_replicated() && lock->can_wrlock(-1)) {
+	  dout(10) << "scatter_nudge auth, propagating " << *lock << " on " << *p << dendl;
+	  scatter_writebehind(lock);
+	  if (c)
+	    lock->add_waiter(SimpleLock::WAIT_STABLE, c);
+	  return;
+	}
+	*/
+
+	// adjust lock state
+	dout(10) << "scatter_nudge auth, scatter/unscattering " << *lock << " on " << *p << dendl;
+	switch (lock->get_type()) {
+	case CEPH_LOCK_IFILE:
+	  if (p->is_replicated() && lock->get_state() != LOCK_MIX)
+	    scatter_mix(static_cast(lock));
+	  else if (lock->get_state() != LOCK_LOCK)
+	    simple_lock(static_cast(lock));
+	  else
+	    simple_sync(static_cast(lock));
+	  break;
+	  
+	case CEPH_LOCK_IDFT:
+	case CEPH_LOCK_INEST:
+	  if (p->is_replicated() && lock->get_state() != LOCK_MIX)
+	    scatter_mix(lock);
+	  else if (lock->get_state() != LOCK_LOCK)
+	    simple_lock(lock);
+	  else
+	    simple_sync(lock);
+	  break;
+	default:
+	  assert(0);
+	}
+	++count;
+	if (lock->is_stable() && count == 2) {
+	  dout(10) << "scatter_nudge oh, stable after two cycles." << dendl;
+	  // this should only realy happen when called via
+	  // handle_file_lock due to AC_NUDGE, because the rest of the
+	  // time we are replicated or have dirty data and won't get
+	  // called.  bailing here avoids an infinite loop.
+	  assert(!c); 
+	  break;
+	}
+      } else {
+	dout(10) << "scatter_nudge auth, waiting for stable " << *lock << " on " << *p << dendl;
+	if (c)
+	  lock->add_waiter(SimpleLock::WAIT_STABLE, c);
+	return;
+      }
+    }
+  } else {
+    dout(10) << "scatter_nudge replica, requesting scatter/unscatter of " 
+	     << *lock << " on " << *p << dendl;
+    // request unscatter?
+    int auth = lock->get_parent()->authority().first;
+    if (mds->mdsmap->is_clientreplay_or_active_or_stopping(auth))
+      mds->send_message_mds(new MLock(lock, LOCK_AC_NUDGE, mds->get_nodeid()), auth);
+
+    // wait...
+    if (c)
+      lock->add_waiter(SimpleLock::WAIT_STABLE, c);
+
+    // also, requeue, in case we had wrong auth or something
+    updated_scatterlocks.push_back(lock->get_updated_item());
+  }
+}
+
+void Locker::scatter_tick()
+{
+  dout(10) << "scatter_tick" << dendl;
+  
+  // updated
+  utime_t now = ceph_clock_now(g_ceph_context);
+  int n = updated_scatterlocks.size();
+  while (!updated_scatterlocks.empty()) {
+    ScatterLock *lock = updated_scatterlocks.front();
+
+    if (n-- == 0) break;  // scatter_nudge() may requeue; avoid looping
+    
+    if (!lock->is_dirty()) {
+      updated_scatterlocks.pop_front();
+      dout(10) << " removing from updated_scatterlocks " 
+	       << *lock << " " << *lock->get_parent() << dendl;
+      continue;
+    }
+    if (now - lock->get_update_stamp() < g_conf->mds_scatter_nudge_interval)
+      break;
+    updated_scatterlocks.pop_front();
+    scatter_nudge(lock, 0);
+  }
+  mds->mdlog->flush();
+}
+
+
+void Locker::scatter_tempsync(ScatterLock *lock, bool *need_issue)
+{
+  dout(10) << "scatter_tempsync " << *lock
+	   << " on " << *lock->get_parent() << dendl;
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  assert(0 == "not fully implemented, at least not for filelock");
+
+  CInode *in = static_cast(lock->get_parent());
+
+  switch (lock->get_state()) {
+  case LOCK_SYNC: assert(0);   // this shouldn't happen
+  case LOCK_LOCK: lock->set_state(LOCK_LOCK_TSYN); break;
+  case LOCK_MIX: lock->set_state(LOCK_MIX_TSYN); break;
+  default: assert(0);
+  }
+
+  int gather = 0;
+  if (lock->is_wrlocked())
+    gather++;
+
+  if (lock->get_cap_shift() &&
+      in->is_head() &&
+      in->issued_caps_need_gather(lock)) {
+    if (need_issue)
+      *need_issue = true;
+    else
+      issue_caps(in);
+    gather++;
+  }
+
+  if (lock->get_state() == LOCK_MIX_TSYN &&
+      in->is_replicated()) {
+    lock->init_gather();
+    send_lock_message(lock, LOCK_AC_LOCK);
+    gather++;
+  }
+
+  if (gather) {
+    in->auth_pin(lock);
+  } else {
+    // do tempsync
+    lock->set_state(LOCK_TSYN);
+    lock->finish_waiters(ScatterLock::WAIT_RD|ScatterLock::WAIT_STABLE);
+    if (lock->get_cap_shift()) {
+      if (need_issue)
+	*need_issue = true;
+      else
+	issue_caps(in);
+    }
+  }
+}
+
+
+
+// ==========================================================================
+// local lock
+
+void Locker::local_wrlock_grab(LocalLock *lock, MutationRef& mut)
+{
+  dout(7) << "local_wrlock_grab  on " << *lock
+	  << " on " << *lock->get_parent() << dendl;  
+  
+  assert(lock->get_parent()->is_auth());
+  assert(lock->can_wrlock());
+  assert(!mut->wrlocks.count(lock));
+  lock->get_wrlock(mut->get_client());
+  mut->wrlocks.insert(lock);
+  mut->locks.insert(lock);
+}
+
+bool Locker::local_wrlock_start(LocalLock *lock, MDRequestRef& mut)
+{
+  dout(7) << "local_wrlock_start  on " << *lock
+	  << " on " << *lock->get_parent() << dendl;  
+  
+  assert(lock->get_parent()->is_auth());
+  if (lock->can_wrlock()) {
+    assert(!mut->wrlocks.count(lock));
+    lock->get_wrlock(mut->get_client());
+    mut->wrlocks.insert(lock);
+    mut->locks.insert(lock);
+    return true;
+  } else {
+    lock->add_waiter(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mut));
+    return false;
+  }
+}
+
+void Locker::local_wrlock_finish(LocalLock *lock, MutationImpl *mut)
+{
+  dout(7) << "local_wrlock_finish  on " << *lock
+	  << " on " << *lock->get_parent() << dendl;  
+  lock->put_wrlock();
+  mut->wrlocks.erase(lock);
+  mut->locks.erase(lock);
+  if (lock->get_num_wrlocks() == 0) {
+    lock->finish_waiters(SimpleLock::WAIT_STABLE |
+                         SimpleLock::WAIT_WR |
+                         SimpleLock::WAIT_RD);
+  }
+}
+
+bool Locker::local_xlock_start(LocalLock *lock, MDRequestRef& mut)
+{
+  dout(7) << "local_xlock_start  on " << *lock
+	  << " on " << *lock->get_parent() << dendl;  
+  
+  assert(lock->get_parent()->is_auth());
+  if (!lock->can_xlock_local()) {
+    lock->add_waiter(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mut));
+    return false;
+  }
+
+  lock->get_xlock(mut, mut->get_client());
+  mut->xlocks.insert(lock);
+  mut->locks.insert(lock);
+  return true;
+}
+
+void Locker::local_xlock_finish(LocalLock *lock, MutationImpl *mut)
+{
+  dout(7) << "local_xlock_finish  on " << *lock
+	  << " on " << *lock->get_parent() << dendl;  
+  lock->put_xlock();
+  mut->xlocks.erase(lock);
+  mut->locks.erase(lock);
+
+  lock->finish_waiters(SimpleLock::WAIT_STABLE | 
+		       SimpleLock::WAIT_WR | 
+		       SimpleLock::WAIT_RD);
+}
+
+
+
+// ==========================================================================
+// file lock
+
+
+void Locker::file_eval(ScatterLock *lock, bool *need_issue)
+{
+  CInode *in = static_cast(lock->get_parent());
+  int loner_wanted, other_wanted;
+  int wanted = in->get_caps_wanted(&loner_wanted, &other_wanted, CEPH_CAP_SFILE);
+  dout(7) << "file_eval wanted=" << gcap_string(wanted)
+	  << " loner_wanted=" << gcap_string(loner_wanted)
+	  << " other_wanted=" << gcap_string(other_wanted)
+	  << "  filelock=" << *lock << " on " << *lock->get_parent()
+	  << dendl;
+
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  if (lock->get_parent()->is_freezing_or_frozen())
+    return;
+
+  // excl -> *?
+  if (lock->get_state() == LOCK_EXCL) {
+    dout(20) << " is excl" << dendl;
+    int loner_issued, other_issued, xlocker_issued;
+    in->get_caps_issued(&loner_issued, &other_issued, &xlocker_issued, CEPH_CAP_SFILE);
+    dout(7) << "file_eval loner_issued=" << gcap_string(loner_issued)
+            << " other_issued=" << gcap_string(other_issued)
+	    << " xlocker_issued=" << gcap_string(xlocker_issued)
+	    << dendl;
+    if (!((loner_wanted|loner_issued) & (CEPH_CAP_GEXCL|CEPH_CAP_GWR|CEPH_CAP_GBUFFER)) ||
+	 (other_wanted & (CEPH_CAP_GEXCL|CEPH_CAP_GWR|CEPH_CAP_GRD)) ||
+	(in->inode.is_dir() && in->multiple_nonstale_caps())) {  // FIXME.. :/
+      dout(20) << " should lose it" << dendl;
+      // we should lose it.
+      //  loner  other   want
+      //  R      R       SYNC
+      //  R      R|W     MIX
+      //  R      W       MIX
+      //  R|W    R       MIX
+      //  R|W    R|W     MIX
+      //  R|W    W       MIX
+      //  W      R       MIX
+      //  W      R|W     MIX
+      //  W      W       MIX
+      // -> any writer means MIX; RD doesn't matter.
+      if (((other_wanted|loner_wanted) & CEPH_CAP_GWR) ||
+	  lock->is_waiter_for(SimpleLock::WAIT_WR))
+	scatter_mix(lock, need_issue);
+      else if (!lock->is_wrlocked())   // let excl wrlocks drain first
+	simple_sync(lock, need_issue);
+      else
+	dout(10) << " waiting for wrlock to drain" << dendl;
+    }    
+  }
+
+  // * -> excl?
+  else if (lock->get_state() != LOCK_EXCL &&
+	   !lock->is_rdlocked() &&
+	   //!lock->is_waiter_for(SimpleLock::WAIT_WR) &&
+	   ((wanted & (CEPH_CAP_GWR|CEPH_CAP_GBUFFER)) ||
+	    (in->inode.is_dir() && !in->has_subtree_or_exporting_dirfrag())) &&
+	   in->get_target_loner() >= 0) {
+    dout(7) << "file_eval stable, bump to loner " << *lock
+	    << " on " << *lock->get_parent() << dendl;
+    file_excl(lock, need_issue);
+  }
+
+  // * -> mixed?
+  else if (lock->get_state() != LOCK_MIX &&
+	   !lock->is_rdlocked() &&
+	   //!lock->is_waiter_for(SimpleLock::WAIT_WR) &&
+	   (lock->get_scatter_wanted() ||
+	    (in->get_wanted_loner() < 0 && (wanted & CEPH_CAP_GWR)))) {
+    dout(7) << "file_eval stable, bump to mixed " << *lock
+	    << " on " << *lock->get_parent() << dendl;
+    scatter_mix(lock, need_issue);
+  }
+  
+  // * -> sync?
+  else if (lock->get_state() != LOCK_SYNC &&
+	   !lock->is_wrlocked() &&   // drain wrlocks first!
+	   !lock->is_waiter_for(SimpleLock::WAIT_WR) &&
+	   !(wanted & (CEPH_CAP_GWR|CEPH_CAP_GBUFFER)) &&
+	   !((lock->get_state() == LOCK_MIX) &&
+	     in->is_dir() && in->has_subtree_or_exporting_dirfrag())  // if we are a delegation point, stay where we are
+	   //((wanted & CEPH_CAP_RD) || 
+	   //in->is_replicated() || 
+	   //lock->get_num_client_lease() || 
+	   //(!loner && lock->get_state() == LOCK_EXCL)) &&
+	   ) {
+    dout(7) << "file_eval stable, bump to sync " << *lock 
+	    << " on " << *lock->get_parent() << dendl;
+    simple_sync(lock, need_issue);
+  }
+}
+
+
+
+void Locker::scatter_mix(ScatterLock *lock, bool *need_issue)
+{
+  dout(7) << "scatter_mix " << *lock << " on " << *lock->get_parent() << dendl;
+
+  CInode *in = static_cast(lock->get_parent());
+  assert(in->is_auth());
+  assert(lock->is_stable());
+
+  if (lock->get_state() == LOCK_LOCK) {
+    in->start_scatter(lock);
+    if (in->is_replicated()) {
+      // data
+      bufferlist softdata;
+      lock->encode_locked_state(softdata);
+
+      // bcast to replicas
+      send_lock_message(lock, LOCK_AC_MIX, softdata);
+    }
+
+    // change lock
+    lock->set_state(LOCK_MIX);
+    lock->clear_scatter_wanted();
+    if (lock->get_cap_shift()) {
+      if (need_issue)
+	*need_issue = true;
+      else
+	issue_caps(in);
+    }
+  } else {
+    // gather?
+    switch (lock->get_state()) {
+    case LOCK_SYNC: lock->set_state(LOCK_SYNC_MIX); break;
+    case LOCK_XSYN:
+      file_excl(lock, need_issue);
+      if (lock->get_state() != LOCK_EXCL)
+	return;
+      // fall-thru
+    case LOCK_EXCL: lock->set_state(LOCK_EXCL_MIX); break;
+    case LOCK_TSYN: lock->set_state(LOCK_TSYN_MIX); break;
+    default: assert(0);
+    }
+
+    int gather = 0;
+    if (lock->is_rdlocked())
+      gather++;
+    if (in->is_replicated()) {
+      if (lock->get_state() != LOCK_EXCL_MIX &&   // EXCL replica is already LOCK
+	  lock->get_state() != LOCK_XSYN_EXCL) {  // XSYN replica is already LOCK;  ** FIXME here too!
+	send_lock_message(lock, LOCK_AC_MIX);
+	lock->init_gather();
+	gather++;
+      }
+    }
+    if (lock->is_leased()) {
+      revoke_client_leases(lock);
+      gather++;
+    }
+    if (lock->get_cap_shift() &&
+	in->is_head() &&
+	in->issued_caps_need_gather(lock)) {
+      if (need_issue)
+	*need_issue = true;
+      else
+	issue_caps(in);
+      gather++;
+    }
+    bool need_recover = false;
+    if (in->state_test(CInode::STATE_NEEDSRECOVER)) {
+      mds->mdcache->queue_file_recover(in);
+      need_recover = true;
+      gather++;
+    }
+
+    if (gather) {
+      lock->get_parent()->auth_pin(lock);
+      if (need_recover)
+	mds->mdcache->do_file_recover();
+    } else {
+      in->start_scatter(lock);
+      lock->set_state(LOCK_MIX);
+      lock->clear_scatter_wanted();
+      if (in->is_replicated()) {
+	bufferlist softdata;
+	lock->encode_locked_state(softdata);
+	send_lock_message(lock, LOCK_AC_MIX, softdata);
+      }
+      if (lock->get_cap_shift()) {
+	if (need_issue)
+	  *need_issue = true;
+	else
+	  issue_caps(in);
+      }
+    }
+  }
+}
+
+
+void Locker::file_excl(ScatterLock *lock, bool *need_issue)
+{
+  CInode *in = static_cast(lock->get_parent());
+  dout(7) << "file_excl " << *lock << " on " << *lock->get_parent() << dendl;  
+
+  assert(in->is_auth());
+  assert(lock->is_stable());
+
+  assert((in->get_loner() >= 0 && in->mds_caps_wanted.empty()) ||
+	 (lock->get_state() == LOCK_XSYN));  // must do xsyn -> excl -> 
+  
+  switch (lock->get_state()) {
+  case LOCK_SYNC: lock->set_state(LOCK_SYNC_EXCL); break;
+  case LOCK_MIX: lock->set_state(LOCK_MIX_EXCL); break;
+  case LOCK_LOCK: lock->set_state(LOCK_LOCK_EXCL); break;
+  case LOCK_XSYN: lock->set_state(LOCK_XSYN_EXCL); break;
+  default: assert(0);
+  }
+  int gather = 0;
+  
+  if (lock->is_rdlocked())
+    gather++;
+  if (lock->is_wrlocked())
+    gather++;
+
+  if (in->is_replicated() &&
+      lock->get_state() != LOCK_LOCK_EXCL &&
+      lock->get_state() != LOCK_XSYN_EXCL) {  // if we were lock, replicas are already lock.
+    send_lock_message(lock, LOCK_AC_LOCK);
+    lock->init_gather();
+    gather++;
+  }
+  if (lock->is_leased()) {
+    revoke_client_leases(lock);
+    gather++;
+  }
+  if (in->is_head() &&
+      in->issued_caps_need_gather(lock)) {
+    if (need_issue)
+      *need_issue = true;
+    else
+      issue_caps(in);
+    gather++;
+  }
+  bool need_recover = false;
+  if (in->state_test(CInode::STATE_NEEDSRECOVER)) {
+    mds->mdcache->queue_file_recover(in);
+    need_recover = true;
+    gather++;
+  }
+  
+  if (gather) {
+    lock->get_parent()->auth_pin(lock);
+    if (need_recover)
+      mds->mdcache->do_file_recover();
+  } else {
+    lock->set_state(LOCK_EXCL);
+    if (need_issue)
+      *need_issue = true;
+    else
+      issue_caps(in);
+  }
+}
+
+void Locker::file_xsyn(SimpleLock *lock, bool *need_issue)
+{
+  dout(7) << "file_xsyn on " << *lock << " on " << *lock->get_parent() << dendl;
+  CInode *in = static_cast(lock->get_parent());
+  assert(in->is_auth());
+  assert(in->get_loner() >= 0 && in->mds_caps_wanted.empty());
+
+  switch (lock->get_state()) {
+  case LOCK_EXCL: lock->set_state(LOCK_EXCL_XSYN); break;
+  default: assert(0);
+  }
+  
+  int gather = 0;
+  if (lock->is_wrlocked())
+    gather++;
+
+  if (in->is_head() &&
+      in->issued_caps_need_gather(lock)) {
+    if (need_issue)
+      *need_issue = true;
+    else
+      issue_caps(in);
+    gather++;
+  }
+  
+  if (gather) {
+    lock->get_parent()->auth_pin(lock);
+  } else {
+    lock->set_state(LOCK_XSYN);
+    lock->finish_waiters(SimpleLock::WAIT_RD|SimpleLock::WAIT_STABLE);
+    if (need_issue)
+      *need_issue = true;
+    else
+      issue_caps(in);
+  }
+}
+
+void Locker::file_recover(ScatterLock *lock)
+{
+  CInode *in = static_cast(lock->get_parent());
+  dout(7) << "file_recover " << *lock << " on " << *in << dendl;
+
+  assert(in->is_auth());
+  //assert(lock->is_stable());
+  assert(lock->get_state() == LOCK_PRE_SCAN); // only called from MDCache::start_files_to_recover()
+
+  int gather = 0;
+  
+  /*
+  if (in->is_replicated()
+      lock->get_sm()->states[oldstate].replica_state != LOCK_LOCK) {
+    send_lock_message(lock, LOCK_AC_LOCK);
+    lock->init_gather();
+    gather++;
+  }
+  */
+  if (in->is_head() &&
+      in->issued_caps_need_gather(lock)) {
+    issue_caps(in);
+    gather++;
+  }
+
+  lock->set_state(LOCK_SCAN);
+  if (gather)
+    in->state_set(CInode::STATE_NEEDSRECOVER);
+  else
+    mds->mdcache->queue_file_recover(in);
+}
+
+
+// messenger
+/* This function DOES put the passed message before returning */
+void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
+{
+  CInode *in = static_cast(lock->get_parent());
+  int from = m->get_asker();
+
+  if (mds->is_rejoin()) {
+    if (in->is_rejoining()) {
+      dout(7) << "handle_file_lock still rejoining " << *in
+	      << ", dropping " << *m << dendl;
+      m->put();
+      return;
+    }
+  }
+
+  dout(7) << "handle_file_lock a=" << get_lock_action_name(m->get_action())
+	  << " on " << *lock
+	  << " from mds." << from << " " 
+	  << *in << dendl;
+
+  bool caps = lock->get_cap_shift();
+  
+  switch (m->get_action()) {
+    // -- replica --
+  case LOCK_AC_SYNC:
+    assert(lock->get_state() == LOCK_LOCK ||
+	   lock->get_state() == LOCK_MIX ||
+	   lock->get_state() == LOCK_MIX_SYNC2);
+    
+    if (lock->get_state() == LOCK_MIX) {
+      lock->set_state(LOCK_MIX_SYNC);
+      eval_gather(lock, true);
+      break;
+    }
+
+    (static_cast(lock))->finish_flush();
+    (static_cast(lock))->clear_flushed();
+
+    // ok
+    lock->decode_locked_state(m->get_data());
+    lock->set_state(LOCK_SYNC);
+
+    lock->get_rdlock();
+    if (caps)
+      issue_caps(in);
+    lock->finish_waiters(SimpleLock::WAIT_RD|SimpleLock::WAIT_STABLE);
+    lock->put_rdlock();
+    break;
+    
+  case LOCK_AC_LOCK:
+    switch (lock->get_state()) {
+    case LOCK_SYNC: lock->set_state(LOCK_SYNC_LOCK); break;
+    case LOCK_MIX: lock->set_state(LOCK_MIX_LOCK); break;
+    default: assert(0);
+    }
+
+    eval_gather(lock, true);
+    break;
+
+  case LOCK_AC_LOCKFLUSHED:
+    (static_cast(lock))->finish_flush();
+    (static_cast(lock))->clear_flushed();
+    break;
+    
+  case LOCK_AC_MIX:
+    assert(lock->get_state() == LOCK_SYNC ||
+           lock->get_state() == LOCK_LOCK ||
+	   lock->get_state() == LOCK_SYNC_MIX2);
+    
+    if (lock->get_state() == LOCK_SYNC) {
+      // MIXED
+      lock->set_state(LOCK_SYNC_MIX);
+      eval_gather(lock, true);
+      break;
+    } 
+    
+    // ok
+    lock->decode_locked_state(m->get_data());
+    lock->set_state(LOCK_MIX);
+
+    if (caps)
+      issue_caps(in);
+    
+    lock->finish_waiters(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE);
+    break;
+
+
+    // -- auth --
+  case LOCK_AC_LOCKACK:
+    assert(lock->get_state() == LOCK_SYNC_LOCK ||
+           lock->get_state() == LOCK_MIX_LOCK ||
+           lock->get_state() == LOCK_MIX_LOCK2 ||
+           lock->get_state() == LOCK_MIX_EXCL ||
+           lock->get_state() == LOCK_SYNC_EXCL ||
+           lock->get_state() == LOCK_SYNC_MIX ||
+	   lock->get_state() == LOCK_MIX_TSYN);
+    assert(lock->is_gathering(from));
+    lock->remove_gather(from);
+    
+    if (lock->get_state() == LOCK_MIX_LOCK ||
+	lock->get_state() == LOCK_MIX_LOCK2 ||
+	lock->get_state() == LOCK_MIX_EXCL ||
+	lock->get_state() == LOCK_MIX_TSYN) {
+      lock->decode_locked_state(m->get_data());
+      // replica is waiting for AC_LOCKFLUSHED, eval_gather() should not
+      // delay calling scatter_writebehind().
+      lock->clear_flushed();
+    }
+
+    if (lock->is_gathering()) {
+      dout(7) << "handle_file_lock " << *in << " from " << from
+	      << ", still gathering " << lock->get_gather_set() << dendl;
+    } else {
+      dout(7) << "handle_file_lock " << *in << " from " << from
+	      << ", last one" << dendl;
+      eval_gather(lock);
+    }
+    break;
+    
+  case LOCK_AC_SYNCACK:
+    assert(lock->get_state() == LOCK_MIX_SYNC);
+    assert(lock->is_gathering(from));
+    lock->remove_gather(from);
+    
+    lock->decode_locked_state(m->get_data());
+
+    if (lock->is_gathering()) {
+      dout(7) << "handle_file_lock " << *in << " from " << from
+	      << ", still gathering " << lock->get_gather_set() << dendl;
+    } else {
+      dout(7) << "handle_file_lock " << *in << " from " << from
+	      << ", last one" << dendl;
+      eval_gather(lock);
+    }
+    break;
+
+  case LOCK_AC_MIXACK:
+    assert(lock->get_state() == LOCK_SYNC_MIX);
+    assert(lock->is_gathering(from));
+    lock->remove_gather(from);
+    
+    if (lock->is_gathering()) {
+      dout(7) << "handle_file_lock " << *in << " from " << from
+	      << ", still gathering " << lock->get_gather_set() << dendl;
+    } else {
+      dout(7) << "handle_file_lock " << *in << " from " << from
+	      << ", last one" << dendl;
+      eval_gather(lock);
+    }
+    break;
+
+
+    // requests....
+  case LOCK_AC_REQSCATTER:
+    if (lock->is_stable()) {
+      /* NOTE: we can do this _even_ if !can_auth_pin (i.e. freezing)
+       *  because the replica should be holding an auth_pin if they're
+       *  doing this (and thus, we are freezing, not frozen, and indefinite
+       *  starvation isn't an issue).
+       */
+      dout(7) << "handle_file_lock got scatter request on " << *lock
+	      << " on " << *lock->get_parent() << dendl;
+      if (lock->get_state() != LOCK_MIX)  // i.e., the reqscatter didn't race with an actual mix/scatter
+	scatter_mix(lock);
+    } else {
+      dout(7) << "handle_file_lock got scatter request, !stable, marking scatter_wanted on " << *lock
+	      << " on " << *lock->get_parent() << dendl;
+      lock->set_scatter_wanted();
+    }
+    break;
+
+  case LOCK_AC_REQUNSCATTER:
+    if (lock->is_stable()) {
+      /* NOTE: we can do this _even_ if !can_auth_pin (i.e. freezing)
+       *  because the replica should be holding an auth_pin if they're
+       *  doing this (and thus, we are freezing, not frozen, and indefinite
+       *  starvation isn't an issue).
+       */
+      dout(7) << "handle_file_lock got unscatter request on " << *lock
+	      << " on " << *lock->get_parent() << dendl;
+      if (lock->get_state() == LOCK_MIX)  // i.e., the reqscatter didn't race with an actual mix/scatter
+	simple_lock(lock);  // FIXME tempsync?
+    } else {
+      dout(7) << "handle_file_lock ignoring unscatter request on " << *lock
+	      << " on " << *lock->get_parent() << dendl;
+      lock->set_unscatter_wanted();
+    }
+    break;
+
+  case LOCK_AC_REQRDLOCK:
+    handle_reqrdlock(lock, m);
+    break;
+
+  case LOCK_AC_NUDGE:
+    if (!lock->get_parent()->is_auth()) {
+      dout(7) << "handle_file_lock IGNORING nudge on non-auth " << *lock
+	      << " on " << *lock->get_parent() << dendl;
+    } else if (!lock->get_parent()->is_replicated()) {
+      dout(7) << "handle_file_lock IGNORING nudge on non-replicated " << *lock
+	      << " on " << *lock->get_parent() << dendl;
+    } else {
+      dout(7) << "handle_file_lock trying nudge on " << *lock
+	      << " on " << *lock->get_parent() << dendl;
+      scatter_nudge(lock, 0, true);
+      mds->mdlog->flush();
+    }    
+    break;
+
+  default:
+    assert(0);
+  }  
+  
+  m->put();
+}
+
+
+
+
+
+
diff --git a/ceph/src/mds/Locker.h b/ceph/src/mds/Locker.h
new file mode 100644
index 00000000..17b3c470
--- /dev/null
+++ b/ceph/src/mds/Locker.h
@@ -0,0 +1,300 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_LOCKER_H
+#define CEPH_MDS_LOCKER_H
+
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+using std::map;
+using std::list;
+using std::set;
+
+class MDS;
+class Session;
+class CDir;
+class CInode;
+class CDentry;
+class EMetaBlob;
+struct SnapRealm;
+
+class Message;
+
+class MDiscover;
+class MDiscoverReply;
+class MCacheExpire;
+class MDirUpdate;
+class MDentryUnlink;
+class MLock;
+
+class MClientRequest;
+
+class Anchor;
+class Capability;
+class LogSegment;
+
+class SimpleLock;
+class ScatterLock;
+class LocalLock;
+class MDCache;
+
+#include "SimpleLock.h"
+
+class Locker {
+private:
+  MDS *mds;
+  MDCache *mdcache;
+ 
+ public:
+  Locker(MDS *m, MDCache *c) : mds(m), mdcache(c) {}  
+
+  SimpleLock *get_lock(int lock_type, MDSCacheObjectInfo &info);
+  
+  void dispatch(Message *m);
+  void handle_lock(MLock *m);
+
+
+  void nudge_log(SimpleLock *lock);
+
+protected:
+  void send_lock_message(SimpleLock *lock, int msg);
+  void send_lock_message(SimpleLock *lock, int msg, const bufferlist &data);
+
+  // -- locks --
+  void _drop_rdlocks(MutationImpl *mut, set *pneed_issue);
+  void _drop_non_rdlocks(MutationImpl *mut, set *pneed_issue);
+public:
+  void include_snap_rdlocks(set& rdlocks, CInode *in);
+  void include_snap_rdlocks_wlayout(set& rdlocks, CInode *in,
+                                    ceph_file_layout **layout);
+
+  bool acquire_locks(MDRequestRef& mdr,
+		     set &rdlocks,
+		     set &wrlocks,
+		     set &xlocks,
+		     map *remote_wrlocks=NULL,
+		     CInode *auth_pin_freeze=NULL,
+		     bool auth_pin_nonblock=false);
+
+  void cancel_locking(MutationImpl *mut, set *pneed_issue);
+  void drop_locks(MutationImpl *mut, set *pneed_issue=0);
+  void set_xlocks_done(MutationImpl *mut, bool skip_dentry=false);
+  void drop_non_rdlocks(MutationImpl *mut, set *pneed_issue=0);
+  void drop_rdlocks(MutationImpl *mut, set *pneed_issue=0);
+
+  void eval_gather(SimpleLock *lock, bool first=false, bool *need_issue=0, list *pfinishers=0);
+  void eval(SimpleLock *lock, bool *need_issue);
+  void eval_any(SimpleLock *lock, bool *need_issue, list *pfinishers=0, bool first=false) {
+    if (!lock->is_stable())
+      eval_gather(lock, first, need_issue, pfinishers);
+    else if (lock->get_parent()->is_auth())
+      eval(lock, need_issue);
+  }
+
+  class C_EvalScatterGathers : public Context {
+    Locker *locker;
+    CInode *in;
+  public:
+    C_EvalScatterGathers(Locker *l, CInode *i) : locker(l), in(i) {
+      in->get(CInode::PIN_PTRWAITER);    
+    }
+    void finish(int r) {
+      in->put(CInode::PIN_PTRWAITER);
+      locker->eval_scatter_gathers(in);
+    }
+  };
+  void eval_scatter_gathers(CInode *in);
+
+  void eval_cap_gather(CInode *in, set *issue_set=0);
+
+  bool eval(CInode *in, int mask, bool caps_imported=false);
+  void try_eval(MDSCacheObject *p, int mask);
+  void try_eval(SimpleLock *lock, bool *pneed_issue);
+
+  bool _rdlock_kick(SimpleLock *lock, bool as_anon);
+  bool rdlock_try(SimpleLock *lock, client_t client, Context *c);
+  bool rdlock_start(SimpleLock *lock, MDRequestRef& mut, bool as_anon=false);
+  void rdlock_finish(SimpleLock *lock, MutationImpl *mut, bool *pneed_issue);
+  bool can_rdlock_set(set& locks);
+  bool rdlock_try_set(set& locks);
+  void rdlock_take_set(set& locks, MutationRef& mut);
+
+  void wrlock_force(SimpleLock *lock, MutationRef& mut);
+  bool wrlock_start(SimpleLock *lock, MDRequestRef& mut, bool nowait=false);
+  void wrlock_finish(SimpleLock *lock, MutationImpl *mut, bool *pneed_issue);
+
+  void remote_wrlock_start(SimpleLock *lock, int target, MDRequestRef& mut);
+  void remote_wrlock_finish(SimpleLock *lock, int target, MutationImpl *mut);
+
+  bool xlock_start(SimpleLock *lock, MDRequestRef& mut);
+  void _finish_xlock(SimpleLock *lock, client_t xlocker, bool *pneed_issue);
+  void xlock_finish(SimpleLock *lock, MutationImpl *mut, bool *pneed_issue);
+
+  void xlock_export(SimpleLock *lock, MutationImpl *mut);
+  void xlock_import(SimpleLock *lock);
+
+
+  // simple
+public:
+  void try_simple_eval(SimpleLock *lock);
+  bool simple_rdlock_try(SimpleLock *lock, Context *con);
+protected:
+  void simple_eval(SimpleLock *lock, bool *need_issue);
+  void handle_simple_lock(SimpleLock *lock, MLock *m);
+
+public:
+  bool simple_sync(SimpleLock *lock, bool *need_issue=0);
+protected:
+  void simple_lock(SimpleLock *lock, bool *need_issue=0);
+  void simple_excl(SimpleLock *lock, bool *need_issue=0);
+  void simple_xlock(SimpleLock *lock);
+
+
+  // scatter
+public:
+  void scatter_eval(ScatterLock *lock, bool *need_issue);        // public for MDCache::adjust_subtree_auth()
+
+  void scatter_tick();
+  void scatter_nudge(ScatterLock *lock, Context *c, bool forcelockchange=false);
+
+protected:
+  void handle_scatter_lock(ScatterLock *lock, MLock *m);
+  void _scatter_replica_lock(ScatterLock *lock, int auth);
+  bool scatter_scatter_fastpath(ScatterLock *lock);
+  void scatter_scatter(ScatterLock *lock, bool nowait=false);
+  void scatter_tempsync(ScatterLock *lock, bool *need_issue=0);
+
+  void scatter_writebehind(ScatterLock *lock);
+  class C_Locker_ScatterWB : public Context {
+    Locker *locker;
+    ScatterLock *lock;
+    MutationRef mut;
+  public:
+    C_Locker_ScatterWB(Locker *l, ScatterLock *sl, MutationRef& m) :
+      locker(l), lock(sl), mut(m) {}
+    void finish(int r) { 
+      locker->scatter_writebehind_finish(lock, mut); 
+    }
+  };
+  void scatter_writebehind_finish(ScatterLock *lock, MutationRef& mut);
+
+  xlist updated_scatterlocks;
+public:
+  void mark_updated_scatterlock(ScatterLock *lock);
+
+
+  void handle_reqrdlock(SimpleLock *lock, MLock *m);
+
+
+
+  // caps
+
+  // when to defer processing client cap release or writeback due to being
+  // frozen.  the condition must be consistent across handle_client_caps and
+  // process_request_cap_release to preserve ordering.
+  bool should_defer_client_cap_frozen(CInode *in);
+
+  void process_request_cap_release(MDRequestRef& mdr, client_t client, const ceph_mds_request_release& r,
+				   const string &dname);
+
+  void kick_cap_releases(MDRequestRef& mdr);
+  void kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq);
+
+  void remove_client_cap(CInode *in, client_t client);
+
+ protected:
+  void adjust_cap_wanted(Capability *cap, int wanted, int issue_seq);
+  void handle_client_caps(class MClientCaps *m);
+  void _update_cap_fields(CInode *in, int dirty, MClientCaps *m, inode_t *pi);
+  void _do_snap_update(CInode *in, snapid_t snap, int dirty, snapid_t follows, client_t client, MClientCaps *m, MClientCaps *ack);
+  void _do_null_snapflush(CInode *head_in, client_t client, snapid_t follows);
+  bool _do_cap_update(CInode *in, Capability *cap, int dirty, snapid_t follows, MClientCaps *m,
+		      MClientCaps *ack=0);
+  void handle_client_cap_release(class MClientCapRelease *m);
+  void _do_cap_release(client_t client, inodeno_t ino, uint64_t cap_id, ceph_seq_t mseq, ceph_seq_t seq);
+
+
+  // local
+public:
+  void local_wrlock_grab(LocalLock *lock, MutationRef& mut);
+protected:
+  bool local_wrlock_start(LocalLock *lock, MDRequestRef& mut);
+  void local_wrlock_finish(LocalLock *lock, MutationImpl *mut);
+  bool local_xlock_start(LocalLock *lock, MDRequestRef& mut);
+  void local_xlock_finish(LocalLock *lock, MutationImpl *mut);
+
+
+  // file
+public:
+  void file_eval(ScatterLock *lock, bool *need_issue);
+protected:
+  void handle_file_lock(ScatterLock *lock, MLock *m);
+  void scatter_mix(ScatterLock *lock, bool *need_issue=0);
+  void file_excl(ScatterLock *lock, bool *need_issue=0);
+  void file_xsyn(SimpleLock *lock, bool *need_issue=0);
+
+public:
+  void file_recover(ScatterLock *lock);
+
+private:
+  xlist updated_filelocks;
+public:
+  void mark_updated_Filelock(ScatterLock *lock);
+
+  // -- file i/o --
+ public:
+  version_t issue_file_data_version(CInode *in);
+  Capability* issue_new_caps(CInode *in, int mode, Session *session, SnapRealm *conrealm, bool is_replay);
+  bool issue_caps(CInode *in, Capability *only_cap=0);
+  void issue_caps_set(set& inset);
+  void issue_truncate(CInode *in);
+  void revoke_stale_caps(Session *session);
+  void resume_stale_caps(Session *session);
+  void remove_stale_leases(Session *session);
+
+public:
+  void request_inode_file_caps(CInode *in);
+protected:
+  void handle_inode_file_caps(class MInodeFileCaps *m);
+
+  void file_update_finish(CInode *in, MutationRef& mut, bool share, client_t client, Capability *cap,
+			  MClientCaps *ack);
+public:
+  void calc_new_client_ranges(CInode *in, uint64_t size, map& new_ranges);
+  bool check_inode_max_size(CInode *in, bool force_wrlock=false,
+                            bool update_size=false, uint64_t newsize=0,
+                            bool update_max=false, uint64_t newmax=0,
+			    utime_t mtime=utime_t());
+  void share_inode_max_size(CInode *in, Capability *only_cap=0);
+
+private:
+  friend class C_MDL_CheckMaxSize;
+  friend class C_MDL_RequestInodeFileCaps;
+  friend struct C_Locker_FileUpdate_finish;
+  friend class C_Locker_RetryCapRelease;
+
+  
+  // -- client leases --
+public:
+  void handle_client_lease(struct MClientLease *m);
+
+  void issue_client_lease(CDentry *dn, client_t client, bufferlist &bl, utime_t now, Session *session);
+  void revoke_client_leases(SimpleLock *lock);
+};
+
+
+#endif
diff --git a/ceph/src/mds/LogEvent.cc b/ceph/src/mds/LogEvent.cc
new file mode 100644
index 00000000..16e7f803
--- /dev/null
+++ b/ceph/src/mds/LogEvent.cc
@@ -0,0 +1,115 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "common/config.h"
+#include "LogEvent.h"
+
+#include "MDS.h"
+
+// events i know of
+#include "events/ESubtreeMap.h"
+#include "events/EExport.h"
+#include "events/EImportStart.h"
+#include "events/EImportFinish.h"
+#include "events/EFragment.h"
+
+#include "events/EResetJournal.h"
+#include "events/ESession.h"
+#include "events/ESessions.h"
+
+#include "events/EUpdate.h"
+#include "events/ESlaveUpdate.h"
+#include "events/EOpen.h"
+#include "events/ECommitted.h"
+
+#include "events/ETableClient.h"
+#include "events/ETableServer.h"
+
+
+LogEvent *LogEvent::decode(bufferlist& bl)
+{
+  // parse type, length
+  bufferlist::iterator p = bl.begin();
+  __u32 type;
+  LogEvent *event = NULL;
+  ::decode(type, p);
+
+  if (EVENT_NEW_ENCODING == type) {
+    try {
+      DECODE_START(1, p);
+      ::decode(type, p);
+      event = decode_event(bl, p, type);
+      DECODE_FINISH(p);
+    }
+    catch (const buffer::error &e) {
+      generic_dout(0) << "failed to decode LogEvent (type maybe " << type << ")" << dendl;
+      return NULL;
+    }
+  } else { // we are using classic encoding
+    event = decode_event(bl, p, type);
+  }
+  return event;
+}
+
+LogEvent *LogEvent::decode_event(bufferlist& bl, bufferlist::iterator& p, __u32 type)
+{
+  int length = bl.length() - p.get_off();
+  generic_dout(15) << "decode_log_event type " << type << ", size " << length << dendl;
+  
+  // create event
+  LogEvent *le;
+  switch (type) {
+  case EVENT_SUBTREEMAP: le = new ESubtreeMap; break;
+  case EVENT_SUBTREEMAP_TEST: 
+    le = new ESubtreeMap;
+    le->set_type(type);
+    break;
+  case EVENT_EXPORT: le = new EExport; break;
+  case EVENT_IMPORTSTART: le = new EImportStart; break;
+  case EVENT_IMPORTFINISH: le = new EImportFinish; break;
+  case EVENT_FRAGMENT: le = new EFragment; break;
+
+  case EVENT_RESETJOURNAL: le = new EResetJournal; break;
+
+  case EVENT_SESSION: le = new ESession; break;
+  case EVENT_SESSIONS_OLD: le = new ESessions; (static_cast(le))->mark_old_encoding(); break;
+  case EVENT_SESSIONS: le = new ESessions; break;
+
+  case EVENT_UPDATE: le = new EUpdate; break;
+  case EVENT_SLAVEUPDATE: le = new ESlaveUpdate; break;
+  case EVENT_OPEN: le = new EOpen; break;
+  case EVENT_COMMITTED: le = new ECommitted; break;
+
+  case EVENT_TABLECLIENT: le = new ETableClient; break;
+  case EVENT_TABLESERVER: le = new ETableServer; break;
+
+  default:
+    generic_dout(0) << "uh oh, unknown log event type " << type << " length " << length << dendl;
+    return NULL;
+  }
+
+  // decode
+  try {
+    le->decode(p);
+  }
+  catch (const buffer::error &e) {
+    generic_dout(0) << "failed to decode LogEvent type " << type << dendl;
+    delete le;
+    return NULL;
+  }
+
+  assert(p.end());
+  return le;
+}
+
diff --git a/ceph/src/mds/LogEvent.h b/ceph/src/mds/LogEvent.h
new file mode 100644
index 00000000..e7e8a62e
--- /dev/null
+++ b/ceph/src/mds/LogEvent.h
@@ -0,0 +1,118 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_LOGEVENT_H
+#define CEPH_LOGEVENT_H
+
+#define EVENT_NEW_ENCODING 0 // indicates that the encoding is versioned
+#define EVENT_UNUSED       1 // was previously EVENT_STRING
+
+#define EVENT_SUBTREEMAP   2
+#define EVENT_EXPORT       3
+#define EVENT_IMPORTSTART  4
+#define EVENT_IMPORTFINISH 5
+#define EVENT_FRAGMENT     6
+
+#define EVENT_RESETJOURNAL 9
+
+#define EVENT_SESSION      10
+#define EVENT_SESSIONS_OLD 11
+#define EVENT_SESSIONS     12
+
+#define EVENT_UPDATE       20
+#define EVENT_SLAVEUPDATE  21
+#define EVENT_OPEN         22
+#define EVENT_COMMITTED    23
+
+#define EVENT_TABLECLIENT  42
+#define EVENT_TABLESERVER  43
+
+#define EVENT_SUBTREEMAP_TEST   50
+
+
+#include 
+using namespace std;
+
+#include "include/buffer.h"
+#include "include/Context.h"
+#include "include/utime.h"
+
+class MDS;
+class LogSegment;
+
+// generic log event
+class LogEvent {
+ private:
+  __u32 _type;
+  uint64_t _start_off;
+  static LogEvent *decode_event(bufferlist& bl, bufferlist::iterator& p, __u32 type);
+
+protected:
+  utime_t stamp;
+
+  friend class MDLog;
+
+ public:
+  LogSegment *_segment;
+
+  LogEvent(int t)
+    : _type(t), _start_off(0), _segment(0) { }
+  virtual ~LogEvent() { }
+
+  int get_type() const { return _type; }
+  void set_type(int t) { _type = t; }
+
+  uint64_t get_start_off() const { return _start_off; }
+  void set_start_off(uint64_t o) { _start_off = o; }
+
+  utime_t get_stamp() const { return stamp; }
+  void set_stamp(utime_t t) { stamp = t; }
+
+  // encoding
+  virtual void encode(bufferlist& bl) const = 0;
+  virtual void decode(bufferlist::iterator &bl) = 0;
+  static LogEvent *decode(bufferlist &bl);
+  virtual void dump(Formatter *f) const = 0;
+
+  void encode_with_header(bufferlist& bl) {
+    ::encode(EVENT_NEW_ENCODING, bl);
+    ENCODE_START(1, 1, bl)
+    ::encode(_type, bl);
+    encode(bl);
+    ENCODE_FINISH(bl);
+  }
+
+  virtual void print(ostream& out) const { 
+    out << "event(" << _type << ")";
+  }
+
+  /*** live journal ***/
+  /* update_segment() - adjust any state we need to in the LogSegment 
+   */
+  virtual void update_segment() { }
+
+  /*** recovery ***/
+  /* replay() - replay given event.  this is idempotent.
+   */
+  virtual void replay(MDS *m) { assert(0); }
+
+
+};
+
+inline ostream& operator<<(ostream& out, LogEvent& le) {
+  le.print(out);
+  return out;
+}
+
+#endif
diff --git a/ceph/src/mds/LogSegment.h b/ceph/src/mds/LogSegment.h
new file mode 100644
index 00000000..a2b8ace3
--- /dev/null
+++ b/ceph/src/mds/LogSegment.h
@@ -0,0 +1,89 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_LOGSEGMENT_H
+#define CEPH_LOGSEGMENT_H
+
+#include "include/dlist.h"
+#include "include/elist.h"
+#include "include/interval_set.h"
+#include "include/Context.h"
+#include "mdstypes.h"
+#include "CInode.h"
+#include "CDentry.h"
+#include "CDir.h"
+
+#include "include/unordered_set.h"
+using ceph::unordered_set;
+
+class CDir;
+class CInode;
+class CDentry;
+class MDS;
+struct MDSlaveUpdate;
+
+class LogSegment {
+ public:
+  uint64_t offset, end;
+  int num_events;
+  uint64_t trimmable_at;
+
+  // dirty items
+  elist    dirty_dirfrags, new_dirfrags;
+  elist  dirty_inodes;
+  elist dirty_dentries;
+
+  elist  open_files;
+  elist  dirty_parent_inodes;
+  elist  dirty_dirfrag_dir;
+  elist  dirty_dirfrag_nest;
+  elist  dirty_dirfrag_dirfragtree;
+
+  elist slave_updates;
+  
+  set truncating_inodes;
+
+  map > pending_commit_tids;  // mdstable
+  set uncommitted_masters;
+  set uncommitted_fragments;
+
+  // client request ids
+  map last_client_tids;
+
+  // table version
+  version_t inotablev;
+  version_t sessionmapv;
+  map tablev;
+
+  // try to expire
+  void try_to_expire(MDS *mds, C_GatherBuilder &gather_bld, int op_prio);
+
+  // cons
+  LogSegment(loff_t off) :
+    offset(off), end(off), num_events(0), trimmable_at(0),
+    dirty_dirfrags(member_offset(CDir, item_dirty)),
+    new_dirfrags(member_offset(CDir, item_new)),
+    dirty_inodes(member_offset(CInode, item_dirty)),
+    dirty_dentries(member_offset(CDentry, item_dirty)),
+    open_files(member_offset(CInode, item_open_file)),
+    dirty_parent_inodes(member_offset(CInode, item_dirty_parent)),
+    dirty_dirfrag_dir(member_offset(CInode, item_dirty_dirfrag_dir)),
+    dirty_dirfrag_nest(member_offset(CInode, item_dirty_dirfrag_nest)),
+    dirty_dirfrag_dirfragtree(member_offset(CInode, item_dirty_dirfrag_dirfragtree)),
+    slave_updates(0), // passed to begin() manually
+    inotablev(0), sessionmapv(0)
+  { }
+};
+
+#endif
diff --git a/ceph/src/mds/MDBalancer.cc b/ceph/src/mds/MDBalancer.cc
new file mode 100644
index 00000000..fbf24c94
--- /dev/null
+++ b/ceph/src/mds/MDBalancer.cc
@@ -0,0 +1,1149 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "mdstypes.h"
+
+#include "MDBalancer.h"
+#include "MDS.h"
+#include "mon/MonClient.h"
+#include "MDSMap.h"
+#include "CInode.h"
+#include "CDir.h"
+#include "MDCache.h"
+#include "Migrator.h"
+
+#include "include/Context.h"
+#include "msg/Messenger.h"
+#include "messages/MHeartbeat.h"
+#include "messages/MMDSLoadTargets.h"
+
+#include 
+#include 
+#include 
+#include 
+using std::map;
+using std::vector;
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef DOUT_COND
+#define DOUT_COND(cct, l) l<=cct->_conf->debug_mds || l <= cct->_conf->debug_mds_balancer
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".bal "
+
+#define MIN_LOAD    50   //  ??
+#define MIN_REEXPORT 5  // will automatically reexport
+#define MIN_OFFLOAD 10   // point at which i stop trying, close enough
+
+
+/* This function DOES put the passed message before returning */
+int MDBalancer::proc_message(Message *m)
+{
+  switch (m->get_type()) {
+
+  case MSG_MDS_HEARTBEAT:
+    handle_heartbeat(static_cast(m));
+    break;
+
+  default:
+    dout(1) << " balancer unknown message " << m->get_type() << dendl;
+    assert(0);
+    m->put();
+    break;
+  }
+
+  return 0;
+}
+
+
+
+
+void MDBalancer::tick()
+{
+  static int num_bal_times = g_conf->mds_bal_max;
+  static utime_t first = ceph_clock_now(g_ceph_context);
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t elapsed = now;
+  elapsed -= first;
+
+  // sample?
+  if ((double)now - (double)last_sample > g_conf->mds_bal_sample_interval) {
+    dout(15) << "tick last_sample now " << now << dendl;
+    last_sample = now;
+  }
+
+  // balance?
+  if (last_heartbeat == utime_t())
+    last_heartbeat = now;
+  if (mds->get_nodeid() == 0 &&
+      g_conf->mds_bal_interval > 0 &&
+      (num_bal_times ||
+       (g_conf->mds_bal_max_until >= 0 &&
+	elapsed.sec() > g_conf->mds_bal_max_until)) &&
+      mds->is_active() &&
+      now.sec() - last_heartbeat.sec() >= g_conf->mds_bal_interval) {
+    last_heartbeat = now;
+    send_heartbeat();
+    num_bal_times--;
+  }
+
+  // hash?
+  if ((g_conf->mds_bal_frag || g_conf->mds_thrash_fragments) &&
+      g_conf->mds_bal_fragment_interval > 0 &&
+      now.sec() - last_fragment.sec() > g_conf->mds_bal_fragment_interval) {
+    last_fragment = now;
+    do_fragmenting();
+  }
+}
+
+
+
+
+class C_Bal_SendHeartbeat : public Context {
+public:
+  MDS *mds;
+  C_Bal_SendHeartbeat(MDS *mds) {
+    this->mds = mds;
+  }
+  virtual void finish(int f) {
+    mds->balancer->send_heartbeat();
+  }
+};
+
+
+double mds_load_t::mds_load()
+{
+  switch(g_conf->mds_bal_mode) {
+  case 0:
+    return
+      .8 * auth.meta_load() +
+      .2 * all.meta_load() +
+      req_rate +
+      10.0 * queue_len;
+
+  case 1:
+    return req_rate + 10.0*queue_len;
+
+  case 2:
+    return cpu_load_avg;
+
+  }
+  assert(0);
+  return 0;
+}
+
+mds_load_t MDBalancer::get_load(utime_t now)
+{
+  mds_load_t load(now);
+
+  if (mds->mdcache->get_root()) {
+    list ls;
+    mds->mdcache->get_root()->get_dirfrags(ls);
+    for (list::iterator p = ls.begin();
+	 p != ls.end();
+	 ++p) {
+      load.auth.add(now, mds->mdcache->decayrate, (*p)->pop_auth_subtree_nested);
+      load.all.add(now, mds->mdcache->decayrate, (*p)->pop_nested);
+    }
+  } else {
+    dout(20) << "get_load no root, no load" << dendl;
+  }
+
+  load.req_rate = mds->get_req_rate();
+  load.queue_len = mds->messenger->get_dispatch_queue_len();
+
+  ifstream cpu("/proc/loadavg");
+  if (cpu.is_open())
+    cpu >> load.cpu_load_avg;
+
+  dout(15) << "get_load " << load << dendl;
+  return load;
+}
+
+void MDBalancer::send_heartbeat()
+{
+  utime_t now = ceph_clock_now(g_ceph_context);
+  
+  if (mds->mdsmap->is_degraded()) {
+    dout(10) << "send_heartbeat degraded" << dendl;
+    return;
+  }
+
+  if (!mds->mdcache->is_open()) {
+    dout(5) << "not open" << dendl;
+    mds->mdcache->wait_for_open(new C_Bal_SendHeartbeat(mds));
+    return;
+  }
+
+  mds_load.clear();
+  if (mds->get_nodeid() == 0)
+    beat_epoch++;
+
+  // my load
+  mds_load_t load = get_load(now);
+  map::value_type val(mds->get_nodeid(), load);
+  mds_load.insert(val);
+
+  // import_map -- how much do i import from whom
+  map import_map;
+  set authsubs;
+  mds->mdcache->get_auth_subtrees(authsubs);
+  for (set::iterator it = authsubs.begin();
+       it != authsubs.end();
+       ++it) {
+    CDir *im = *it;
+    int from = im->inode->authority().first;
+    if (from == mds->get_nodeid()) continue;
+    if (im->get_inode()->is_stray()) continue;
+    import_map[from] += im->pop_auth_subtree.meta_load(now, mds->mdcache->decayrate);
+  }
+  mds_import_map[ mds->get_nodeid() ] = import_map;
+
+
+  dout(5) << "mds." << mds->get_nodeid() << " epoch " << beat_epoch << " load " << load << dendl;
+  for (map::iterator it = import_map.begin();
+       it != import_map.end();
+       ++it) {
+    dout(5) << "  import_map from " << it->first << " -> " << it->second << dendl;
+  }
+
+
+  set up;
+  mds->get_mds_map()->get_mds_set(up);
+  for (set::iterator p = up.begin(); p != up.end(); ++p) {
+    if (*p == mds->get_nodeid())
+      continue;
+    MHeartbeat *hb = new MHeartbeat(load, beat_epoch);
+    hb->get_import_map() = import_map;
+    mds->messenger->send_message(hb,
+                                 mds->mdsmap->get_inst(*p));
+  }
+}
+
+/* This function DOES put the passed message before returning */
+void MDBalancer::handle_heartbeat(MHeartbeat *m)
+{
+  typedef map mds_load_map_t;
+
+  int who = m->get_source().num();
+  dout(25) << "=== got heartbeat " << m->get_beat() << " from " << m->get_source().num() << " " << m->get_load() << dendl;
+
+  if (!mds->is_active())
+    goto out;
+
+  if (!mds->mdcache->is_open()) {
+    dout(10) << "opening root on handle_heartbeat" << dendl;
+    mds->mdcache->wait_for_open(new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+
+  if (mds->mdsmap->is_degraded()) {
+    dout(10) << " degraded, ignoring" << dendl;
+    goto out;
+  }
+
+  if (who == 0) {
+    dout(20) << " from mds0, new epoch" << dendl;
+    beat_epoch = m->get_beat();
+    send_heartbeat();
+
+    show_imports();
+  }
+
+  {
+    // set mds_load[who]
+    mds_load_map_t::value_type val(who, m->get_load());
+    pair < mds_load_map_t::iterator, bool > rval (mds_load.insert(val));
+    if (!rval.second) {
+      rval.first->second = val.second;
+    }
+  }
+  mds_import_map[ who ] = m->get_import_map();
+
+  //dout(0) << "  load is " << load << " have " << mds_load.size() << dendl;
+
+  {
+    unsigned cluster_size = mds->get_mds_map()->get_num_in_mds();
+    if (mds_load.size() == cluster_size) {
+      // let's go!
+      //export_empties();  // no!
+      prep_rebalance(m->get_beat());
+    }
+  }
+
+  // done
+ out:
+  m->put();
+}
+
+
+void MDBalancer::export_empties()
+{
+  dout(5) << "export_empties checking for empty imports" << dendl;
+
+  for (map >::iterator it = mds->mdcache->subtrees.begin();
+       it != mds->mdcache->subtrees.end();
+       ++it) {
+    CDir *dir = it->first;
+    if (!dir->is_auth() ||
+	dir->is_ambiguous_auth() ||
+	dir->is_freezing() ||
+	dir->is_frozen())
+      continue;
+
+    if (!dir->inode->is_base() &&
+	!dir->inode->is_stray() &&
+	dir->get_num_head_items() == 0)
+      mds->mdcache->migrator->export_empty_import(dir);
+  }
+}
+
+
+
+double MDBalancer::try_match(int ex, double& maxex,
+                             int im, double& maxim)
+{
+  if (maxex <= 0 || maxim <= 0) return 0.0;
+
+  double howmuch = MIN(maxex, maxim);
+  if (howmuch <= 0) return 0.0;
+
+  dout(5) << "   - mds." << ex << " exports " << howmuch << " to mds." << im << dendl;
+
+  if (ex == mds->get_nodeid())
+    my_targets[im] += howmuch;
+
+  exported[ex] += howmuch;
+  imported[im] += howmuch;
+
+  maxex -= howmuch;
+  maxim -= howmuch;
+
+  return howmuch;
+}
+
+void MDBalancer::queue_split(CDir *dir)
+{
+  split_queue.insert(dir->dirfrag());
+}
+
+void MDBalancer::queue_merge(CDir *dir)
+{
+  merge_queue.insert(dir->dirfrag());
+}
+
+void MDBalancer::do_fragmenting()
+{
+  if (split_queue.empty() && merge_queue.empty()) {
+    dout(20) << "do_fragmenting has nothing to do" << dendl;
+    return;
+  }
+
+  if (!split_queue.empty()) {
+    dout(10) << "do_fragmenting " << split_queue.size() << " dirs marked for possible splitting" << dendl;
+
+    set q;
+    q.swap(split_queue);
+
+    for (set::iterator i = q.begin();
+	 i != q.end();
+	 ++i) {
+      CDir *dir = mds->mdcache->get_dirfrag(*i);
+      if (!dir ||
+	  !dir->is_auth())
+	continue;
+
+      dout(10) << "do_fragmenting splitting " << *dir << dendl;
+      mds->mdcache->split_dir(dir, g_conf->mds_bal_split_bits);
+    }
+  }
+
+  if (!merge_queue.empty()) {
+    dout(10) << "do_fragmenting " << merge_queue.size() << " dirs marked for possible merging" << dendl;
+
+    set q;
+    q.swap(merge_queue);
+
+    for (set::iterator i = q.begin();
+	 i != q.end();
+	 ++i) {
+      CDir *dir = mds->mdcache->get_dirfrag(*i);
+      if (!dir ||
+	  !dir->is_auth() ||
+	  dir->get_frag() == frag_t())  // ok who's the joker?
+	continue;
+
+      dout(10) << "do_fragmenting merging " << *dir << dendl;
+
+      CInode *diri = dir->get_inode();
+
+      frag_t fg = dir->get_frag();
+      while (fg != frag_t()) {
+	frag_t sibfg = fg.get_sibling();
+	list sibs;
+	bool complete = diri->get_dirfrags_under(sibfg, sibs);
+	if (!complete) {
+	  dout(10) << "  not all sibs under " << sibfg << " in cache (have " << sibs << ")" << dendl;
+	  break;
+	}
+	bool all = true;
+	for (list::iterator p = sibs.begin(); p != sibs.end(); ++p) {
+	  CDir *sib = *p;
+	  if (!sib->is_auth() || !sib->should_merge()) {
+	    all = false;
+	    break;
+	  }
+	}
+	if (!all) {
+	  dout(10) << "  not all sibs under " << sibfg << " " << sibs << " should_merge" << dendl;
+	  break;
+	}
+	dout(10) << "  all sibs under " << sibfg << " " << sibs << " should merge" << dendl;
+	fg = fg.parent();
+      }
+
+      if (fg != dir->get_frag())
+	mds->mdcache->merge_dir(diri, fg);
+    }
+  }
+}
+
+
+
+void MDBalancer::prep_rebalance(int beat)
+{
+  if (g_conf->mds_thrash_exports) {
+    //we're going to randomly export to all the mds in the cluster
+    my_targets.clear();
+    set up_mds;
+    mds->get_mds_map()->get_up_mds_set(up_mds);
+    for (set::iterator i = up_mds.begin();
+	 i != up_mds.end();
+	 ++i)
+      my_targets[*i] = 0.0;
+  } else {
+    int cluster_size = mds->get_mds_map()->get_num_in_mds();
+    int whoami = mds->get_nodeid();
+    rebalance_time = ceph_clock_now(g_ceph_context);
+
+    // reset
+    my_targets.clear();
+    imported.clear();
+    exported.clear();
+
+    dout(5) << " prep_rebalance: cluster loads are" << dendl;
+
+    mds->mdcache->migrator->clear_export_queue();
+
+    // rescale!  turn my mds_load back into meta_load units
+    double load_fac = 1.0;
+    map::iterator m = mds_load.find(whoami);
+    if ((m != mds_load.end()) && (m->second.mds_load() > 0)) {
+      double metald = m->second.auth.meta_load(rebalance_time, mds->mdcache->decayrate);
+      double mdsld = m->second.mds_load();
+      load_fac = metald / mdsld;
+      dout(7) << " load_fac is " << load_fac
+	      << " <- " << m->second.auth << " " << metald
+	      << " / " << mdsld
+	      << dendl;
+    }
+
+    double total_load = 0;
+    multimap load_map;
+    for (int i=0; i::value_type val(i, mds_load_t(ceph_clock_now(g_ceph_context)));
+      std::pair < map::iterator, bool > r(mds_load.insert(val));
+      mds_load_t &load(r.first->second);
+
+      double l = load.mds_load() * load_fac;
+      mds_meta_load[i] = l;
+
+      if (whoami == 0)
+	dout(0) << "  mds." << i
+		<< " " << load
+		<< " = " << load.mds_load()
+		<< " ~ " << l << dendl;
+
+      if (whoami == i) my_load = l;
+      total_load += l;
+
+      load_map.insert(pair( l, i ));
+    }
+
+    // target load
+    target_load = total_load / (double)cluster_size;
+    dout(5) << "prep_rebalance:  my load " << my_load
+	    << "   target " << target_load
+	    << "   total " << total_load
+	    << dendl;
+
+    // under or over?
+    if (my_load < target_load * (1.0 + g_conf->mds_bal_min_rebalance)) {
+      dout(5) << "  i am underloaded or barely overloaded, doing nothing." << dendl;
+      last_epoch_under = beat_epoch;
+      show_imports();
+      return;
+    }
+
+    last_epoch_over = beat_epoch;
+
+    // am i over long enough?
+    if (last_epoch_under && beat_epoch - last_epoch_under < 2) {
+      dout(5) << "  i am overloaded, but only for " << (beat_epoch - last_epoch_under) << " epochs" << dendl;
+      return;
+    }
+
+    dout(5) << "  i am sufficiently overloaded" << dendl;
+
+
+    // first separate exporters and importers
+    multimap importers;
+    multimap exporters;
+    set             importer_set;
+    set             exporter_set;
+
+    for (multimap::iterator it = load_map.begin();
+	 it != load_map.end();
+	 ++it) {
+      if (it->first < target_load) {
+	dout(15) << "   mds." << it->second << " is importer" << dendl;
+	importers.insert(pair(it->first,it->second));
+	importer_set.insert(it->second);
+      } else {
+	dout(15) << "   mds." << it->second << " is exporter" << dendl;
+	exporters.insert(pair(it->first,it->second));
+	exporter_set.insert(it->second);
+      }
+    }
+
+
+    // determine load transfer mapping
+
+    if (true) {
+      // analyze import_map; do any matches i can
+
+      dout(15) << "  matching exporters to import sources" << dendl;
+
+      // big -> small exporters
+      for (multimap::reverse_iterator ex = exporters.rbegin();
+	   ex != exporters.rend();
+	   ++ex) {
+	double maxex = get_maxex(ex->second);
+	if (maxex <= .001) continue;
+
+	// check importers. for now, just in arbitrary order (no intelligent matching).
+	for (map::iterator im = mds_import_map[ex->second].begin();
+	     im != mds_import_map[ex->second].end();
+	     ++im) {
+	  double maxim = get_maxim(im->first);
+	  if (maxim <= .001) continue;
+	  try_match(ex->second, maxex,
+		    im->first, maxim);
+	  if (maxex <= .001) break;;
+	}
+      }
+    }
+
+
+    if (1) {
+      if (beat % 2 == 1) {
+	// old way
+	dout(15) << "  matching big exporters to big importers" << dendl;
+	// big exporters to big importers
+	multimap::reverse_iterator ex = exporters.rbegin();
+	multimap::iterator im = importers.begin();
+	while (ex != exporters.rend() &&
+	       im != importers.end()) {
+	  double maxex = get_maxex(ex->second);
+	  double maxim = get_maxim(im->second);
+	  if (maxex < .001 || maxim < .001) break;
+	  try_match(ex->second, maxex,
+		    im->second, maxim);
+	  if (maxex <= .001) ++ex;
+	  if (maxim <= .001) ++im;
+	}
+      } else {
+	// new way
+	dout(15) << "  matching small exporters to big importers" << dendl;
+	// small exporters to big importers
+	multimap::iterator ex = exporters.begin();
+	multimap::iterator im = importers.begin();
+	while (ex != exporters.end() &&
+	       im != importers.end()) {
+	  double maxex = get_maxex(ex->second);
+	  double maxim = get_maxim(im->second);
+	  if (maxex < .001 || maxim < .001) break;
+	  try_match(ex->second, maxex,
+		    im->second, maxim);
+	  if (maxex <= .001) ++ex;
+	  if (maxim <= .001) ++im;
+	}
+      }
+    }
+  }
+  try_rebalance();
+}
+
+
+
+void MDBalancer::try_rebalance()
+{
+  if (!check_targets())
+    return;
+
+  if (g_conf->mds_thrash_exports) {
+    dout(5) << "mds_thrash is on; not performing standard rebalance operation!"
+	    << dendl;
+    return;
+  }
+
+  // make a sorted list of my imports
+  map    import_pop_map;
+  multimap  import_from_map;
+  set fullauthsubs;
+
+  mds->mdcache->get_fullauth_subtrees(fullauthsubs);
+  for (set::iterator it = fullauthsubs.begin();
+       it != fullauthsubs.end();
+       ++it) {
+    CDir *im = *it;
+    if (im->get_inode()->is_stray()) continue;
+
+    double pop = im->pop_auth_subtree.meta_load(rebalance_time, mds->mdcache->decayrate);
+    if (g_conf->mds_bal_idle_threshold > 0 &&
+	pop < g_conf->mds_bal_idle_threshold &&
+	im->inode != mds->mdcache->get_root() &&
+	im->inode->authority().first != mds->get_nodeid()) {
+      dout(0) << " exporting idle (" << pop << ") import " << *im
+	      << " back to mds." << im->inode->authority().first
+	      << dendl;
+      mds->mdcache->migrator->export_dir_nicely(im, im->inode->authority().first);
+      continue;
+    }
+
+    import_pop_map[ pop ] = im;
+    int from = im->inode->authority().first;
+    dout(15) << "  map: i imported " << *im << " from " << from << dendl;
+    import_from_map.insert(pair(from, im));
+  }
+
+
+
+  // do my exports!
+  set already_exporting;
+  double total_sent = 0;
+  double total_goal = 0;
+
+  for (map::iterator it = my_targets.begin();
+       it != my_targets.end();
+       ++it) {
+
+      /*
+	double fac = 1.0;
+	if (false && total_goal > 0 && total_sent > 0) {
+	fac = total_goal / total_sent;
+	dout(0) << " total sent is " << total_sent << " / " << total_goal << " -> fac 1/ " << fac << dendl;
+	if (fac > 1.0) fac = 1.0;
+	}
+	fac = .9 - .4 * ((float)g_conf->num_mds / 128.0);  // hack magic fixme
+      */
+
+    int target = (*it).first;
+    double amount = (*it).second;
+    total_goal += amount;
+
+    if (amount < MIN_OFFLOAD) continue;
+    if (amount / target_load < .2) continue;
+
+    dout(5) << "want to send " << amount << " to mds." << target
+      //<< " .. " << (*it).second << " * " << load_fac
+	    << " -> " << amount
+	    << dendl;//" .. fudge is " << fudge << dendl;
+    double have = 0;
+
+
+    show_imports();
+
+    // search imports from target
+    if (import_from_map.count(target)) {
+      dout(5) << " aha, looking through imports from target mds." << target << dendl;
+      pair::iterator, multimap::iterator> p =
+	import_from_map.equal_range(target);
+      while (p.first != p.second) {
+	CDir *dir = (*p.first).second;
+	dout(5) << "considering " << *dir << " from " << (*p.first).first << dendl;
+	multimap::iterator plast = p.first++;
+
+	if (dir->inode->is_base() ||
+	    dir->inode->is_stray())
+	  continue;
+	if (dir->is_freezing() || dir->is_frozen()) continue;  // export pbly already in progress
+	double pop = dir->pop_auth_subtree.meta_load(rebalance_time, mds->mdcache->decayrate);
+	assert(dir->inode->authority().first == target);  // cuz that's how i put it in the map, dummy
+
+	if (pop <= amount-have) {
+	  dout(0) << "reexporting " << *dir
+		  << " pop " << pop
+		  << " back to mds." << target << dendl;
+	  mds->mdcache->migrator->export_dir_nicely(dir, target);
+	  have += pop;
+	  import_from_map.erase(plast);
+	  import_pop_map.erase(pop);
+	} else {
+	  dout(5) << "can't reexport " << *dir << ", too big " << pop << dendl;
+	}
+	if (amount-have < MIN_OFFLOAD) break;
+      }
+    }
+    if (amount-have < MIN_OFFLOAD) {
+      total_sent += have;
+      continue;
+    }
+
+    // any other imports
+    if (false)
+      for (map::iterator import = import_pop_map.begin();
+	   import != import_pop_map.end();
+	   import++) {
+	CDir *imp = (*import).second;
+	if (imp->inode->is_base() ||
+	    imp->inode->is_stray())
+	  continue;
+
+	double pop = (*import).first;
+	if (pop < amount-have || pop < MIN_REEXPORT) {
+	  dout(0) << "reexporting " << *imp
+		  << " pop " << pop
+		  << " back to mds." << imp->inode->authority()
+		  << dendl;
+	  have += pop;
+	  mds->mdcache->migrator->export_dir_nicely(imp, imp->inode->authority().first);
+	}
+	if (amount-have < MIN_OFFLOAD) break;
+      }
+    if (amount-have < MIN_OFFLOAD) {
+      //fudge = amount-have;
+      total_sent += have;
+      continue;
+    }
+
+    // okay, search for fragments of my workload
+    set candidates;
+    mds->mdcache->get_fullauth_subtrees(candidates);
+
+    list exports;
+
+    for (set::iterator pot = candidates.begin();
+	 pot != candidates.end();
+	 ++pot) {
+      if ((*pot)->get_inode()->is_stray()) continue;
+      find_exports(*pot, amount, exports, have, already_exporting);
+      if (have > amount-MIN_OFFLOAD)
+	break;
+    }
+    //fudge = amount - have;
+    total_sent += have;
+
+    for (list::iterator it = exports.begin(); it != exports.end(); ++it) {
+      dout(0) << "   - exporting "
+	       << (*it)->pop_auth_subtree
+	       << " "
+	       << (*it)->pop_auth_subtree.meta_load(rebalance_time, mds->mdcache->decayrate)
+	       << " to mds." << target
+	       << " " << **it
+	       << dendl;
+      mds->mdcache->migrator->export_dir_nicely(*it, target);
+    }
+  }
+
+  dout(5) << "rebalance done" << dendl;
+  show_imports();
+}
+
+
+/* returns true if all my_target MDS are in the MDSMap.
+ */
+bool MDBalancer::check_targets()
+{
+  // get MonMap's idea of my_targets
+  const set& map_targets = mds->mdsmap->get_mds_info(mds->whoami).export_targets;
+
+  bool send = false;
+  bool ok = true;
+
+  // make sure map targets are in the old_prev_targets map
+  for (set::iterator p = map_targets.begin(); p != map_targets.end(); ++p) {
+    if (old_prev_targets.count(*p) == 0)
+      old_prev_targets[*p] = 0;
+    if (my_targets.count(*p) == 0)
+      old_prev_targets[*p]++;
+  }
+
+  // check if the current MonMap has all our targets
+  set need_targets;
+  for (map::iterator i = my_targets.begin();
+       i != my_targets.end();
+       ++i) {
+    need_targets.insert(i->first);
+    old_prev_targets[i->first] = 0;
+
+    if (!map_targets.count(i->first)) {
+      dout(20) << " target mds." << i->first << " not in map's export_targets" << dendl;
+      send = true;
+      ok = false;
+    }
+  }
+
+  set want_targets = need_targets;
+  map::iterator p = old_prev_targets.begin();
+  while (p != old_prev_targets.end()) {
+    if (map_targets.count(p->first) == 0 &&
+	need_targets.count(p->first) == 0) {
+      old_prev_targets.erase(p++);
+      continue;
+    }
+    dout(20) << " target mds." << p->first << " has been non-target for " << p->second << dendl;
+    if (p->second < g_conf->mds_bal_target_removal_min)
+      want_targets.insert(p->first);
+    if (p->second >= g_conf->mds_bal_target_removal_max)
+      send = true;
+    ++p;
+  }
+
+  dout(10) << "check_targets have " << map_targets << " need " << need_targets << " want " << want_targets << dendl;
+
+  if (send) {
+    MMDSLoadTargets* m = new MMDSLoadTargets(mds->monc->get_global_id(), want_targets);
+    mds->monc->send_mon_message(m);
+  }
+  return ok;
+}
+
+void MDBalancer::find_exports(CDir *dir,
+                              double amount,
+                              list& exports,
+                              double& have,
+                              set& already_exporting)
+{
+  double need = amount - have;
+  if (need < amount * g_conf->mds_bal_min_start)
+    return;   // good enough!
+  double needmax = need * g_conf->mds_bal_need_max;
+  double needmin = need * g_conf->mds_bal_need_min;
+  double midchunk = need * g_conf->mds_bal_midchunk;
+  double minchunk = need * g_conf->mds_bal_minchunk;
+
+  list bigger_rep, bigger_unrep;
+  multimap smaller;
+
+  double dir_pop = dir->pop_auth_subtree.meta_load(rebalance_time, mds->mdcache->decayrate);
+  dout(7) << " find_exports in " << dir_pop << " " << *dir << " need " << need << " (" << needmin << " - " << needmax << ")" << dendl;
+
+  double subdir_sum = 0;
+  for (CDir::map_t::iterator it = dir->begin();
+       it != dir->end();
+       ++it) {
+    CInode *in = it->second->get_linkage()->get_inode();
+    if (!in) continue;
+    if (!in->is_dir()) continue;
+
+    list dfls;
+    in->get_dirfrags(dfls);
+    for (list::iterator p = dfls.begin();
+	 p != dfls.end();
+	 ++p) {
+      CDir *subdir = *p;
+      if (!subdir->is_auth()) continue;
+      if (already_exporting.count(subdir)) continue;
+
+      if (subdir->is_frozen()) continue;  // can't export this right now!
+
+      // how popular?
+      double pop = subdir->pop_auth_subtree.meta_load(rebalance_time, mds->mdcache->decayrate);
+      subdir_sum += pop;
+      dout(15) << "   subdir pop " << pop << " " << *subdir << dendl;
+
+      if (pop < minchunk) continue;
+
+      // lucky find?
+      if (pop > needmin && pop < needmax) {
+	exports.push_back(subdir);
+	already_exporting.insert(subdir);
+	have += pop;
+	return;
+      }
+
+      if (pop > need) {
+	if (subdir->is_rep())
+	  bigger_rep.push_back(subdir);
+	else
+	  bigger_unrep.push_back(subdir);
+      } else
+	smaller.insert(pair(pop, subdir));
+    }
+  }
+  dout(15) << "   sum " << subdir_sum << " / " << dir_pop << dendl;
+
+  // grab some sufficiently big small items
+  multimap::reverse_iterator it;
+  for (it = smaller.rbegin();
+       it != smaller.rend();
+       ++it) {
+
+    if ((*it).first < midchunk)
+      break;  // try later
+
+    dout(7) << "   taking smaller " << *(*it).second << dendl;
+
+    exports.push_back((*it).second);
+    already_exporting.insert((*it).second);
+    have += (*it).first;
+    if (have > needmin)
+      return;
+  }
+
+  // apprently not enough; drill deeper into the hierarchy (if non-replicated)
+  for (list::iterator it = bigger_unrep.begin();
+       it != bigger_unrep.end();
+       ++it) {
+    dout(15) << "   descending into " << **it << dendl;
+    find_exports(*it, amount, exports, have, already_exporting);
+    if (have > needmin)
+      return;
+  }
+
+  // ok fine, use smaller bits
+  for (;
+       it != smaller.rend();
+       ++it) {
+    dout(7) << "   taking (much) smaller " << it->first << " " << *(*it).second << dendl;
+
+    exports.push_back((*it).second);
+    already_exporting.insert((*it).second);
+    have += (*it).first;
+    if (have > needmin)
+      return;
+  }
+
+  // ok fine, drill into replicated dirs
+  for (list::iterator it = bigger_rep.begin();
+       it != bigger_rep.end();
+       ++it) {
+    dout(7) << "   descending into replicated " << **it << dendl;
+    find_exports(*it, amount, exports, have, already_exporting);
+    if (have > needmin)
+      return;
+  }
+
+}
+
+void MDBalancer::hit_inode(utime_t now, CInode *in, int type, int who)
+{
+  // hit inode
+  in->pop.get(type).hit(now, mds->mdcache->decayrate);
+
+  if (in->get_parent_dn())
+    hit_dir(now, in->get_parent_dn()->get_dir(), type, who);
+}
+/*
+  // hit me
+  in->popularity[MDS_POP_JUSTME].pop[type].hit(now);
+  in->popularity[MDS_POP_NESTED].pop[type].hit(now);
+  if (in->is_auth()) {
+    in->popularity[MDS_POP_CURDOM].pop[type].hit(now);
+    in->popularity[MDS_POP_ANYDOM].pop[type].hit(now);
+
+    dout(20) << "hit_inode " << type << " pop "
+	     << in->popularity[MDS_POP_JUSTME].pop[type].get(now) << " me, "
+	     << in->popularity[MDS_POP_NESTED].pop[type].get(now) << " nested, "
+	     << in->popularity[MDS_POP_CURDOM].pop[type].get(now) << " curdom, "
+	     << in->popularity[MDS_POP_CURDOM].pop[type].get(now) << " anydom"
+	     << " on " << *in
+	     << dendl;
+  } else {
+    dout(20) << "hit_inode " << type << " pop "
+	     << in->popularity[MDS_POP_JUSTME].pop[type].get(now) << " me, "
+	     << in->popularity[MDS_POP_NESTED].pop[type].get(now) << " nested, "
+      	     << " on " << *in
+	     << dendl;
+  }
+
+  // hit auth up to import
+  CDir *dir = in->get_parent_dir();
+  if (dir) hit_dir(now, dir, type);
+*/
+
+
+void MDBalancer::hit_dir(utime_t now, CDir *dir, int type, int who, double amount)
+{
+  // hit me
+  double v = dir->pop_me.get(type).hit(now, amount);
+
+  //if (dir->ino() == inodeno_t(0x10000000000))
+  //dout(0) << "hit_dir " << type << " pop " << v << " in " << *dir << dendl;
+
+  // split/merge
+  if (g_conf->mds_bal_frag && g_conf->mds_bal_fragment_interval > 0 &&
+      !dir->inode->is_base() &&        // not root/base (for now at least)
+      dir->is_auth()) {
+
+    dout(20) << "hit_dir " << type << " pop is " << v << ", frag " << dir->get_frag()
+	     << " size " << dir->get_frag_size() << dendl;
+
+    // split
+    if (g_conf->mds_bal_split_size > 0 &&
+	(dir->should_split() ||
+	 (v > g_conf->mds_bal_split_rd && type == META_POP_IRD) ||
+	 (v > g_conf->mds_bal_split_wr && type == META_POP_IWR)) &&
+	split_queue.count(dir->dirfrag()) == 0) {
+      dout(10) << "hit_dir " << type << " pop is " << v << ", putting in split_queue: " << *dir << dendl;
+      split_queue.insert(dir->dirfrag());
+    }
+
+    // merge?
+    if (dir->get_frag() != frag_t() && dir->should_merge() &&
+	merge_queue.count(dir->dirfrag()) == 0) {
+      dout(10) << "hit_dir " << type << " pop is " << v << ", putting in merge_queue: " << *dir << dendl;
+      merge_queue.insert(dir->dirfrag());
+    }
+  }
+
+  // replicate?
+  if (type == META_POP_IRD && who >= 0) {
+    dir->pop_spread.hit(now, mds->mdcache->decayrate, who);
+  }
+
+  double rd_adj = 0;
+  if (type == META_POP_IRD &&
+      dir->last_popularity_sample < last_sample) {
+    float dir_pop = dir->pop_auth_subtree.get(type).get(now, mds->mdcache->decayrate);    // hmm??
+    dir->last_popularity_sample = last_sample;
+    float pop_sp = dir->pop_spread.get(now, mds->mdcache->decayrate);
+    dir_pop += pop_sp * 10;
+
+    //if (dir->ino() == inodeno_t(0x10000000002))
+    if (pop_sp > 0) {
+      dout(20) << "hit_dir " << type << " pop " << dir_pop << " spread " << pop_sp
+	      << " " << dir->pop_spread.last[0]
+	      << " " << dir->pop_spread.last[1]
+	      << " " << dir->pop_spread.last[2]
+	      << " " << dir->pop_spread.last[3]
+	      << " in " << *dir << dendl;
+    }
+
+    if (dir->is_auth() && !dir->is_ambiguous_auth()) {
+      if (!dir->is_rep() &&
+	  dir_pop >= g_conf->mds_bal_replicate_threshold) {
+	// replicate
+	float rdp = dir->pop_me.get(META_POP_IRD).get(now, mds->mdcache->decayrate);
+	rd_adj = rdp / mds->get_mds_map()->get_num_in_mds() - rdp;
+	rd_adj /= 2.0;  // temper somewhat
+
+	dout(0) << "replicating dir " << *dir << " pop " << dir_pop << " .. rdp " << rdp << " adj " << rd_adj << dendl;
+
+	dir->dir_rep = CDir::REP_ALL;
+	mds->mdcache->send_dir_updates(dir, true);
+
+	// fixme this should adjust the whole pop hierarchy
+	dir->pop_me.get(META_POP_IRD).adjust(rd_adj);
+	dir->pop_auth_subtree.get(META_POP_IRD).adjust(rd_adj);
+      }
+
+      if (dir->ino() != 1 &&
+	  dir->is_rep() &&
+	  dir_pop < g_conf->mds_bal_unreplicate_threshold) {
+	// unreplicate
+	dout(0) << "unreplicating dir " << *dir << " pop " << dir_pop << dendl;
+
+	dir->dir_rep = CDir::REP_NONE;
+	mds->mdcache->send_dir_updates(dir);
+      }
+    }
+  }
+
+  // adjust ancestors
+  bool hit_subtree = dir->is_auth();         // current auth subtree (if any)
+  bool hit_subtree_nested = dir->is_auth();  // all nested auth subtrees
+
+  while (1) {
+    dir->pop_nested.get(type).hit(now, amount);
+    if (rd_adj != 0.0)
+      dir->pop_nested.get(META_POP_IRD).adjust(now, mds->mdcache->decayrate, rd_adj);
+
+    if (hit_subtree) {
+      dir->pop_auth_subtree.get(type).hit(now, amount);
+      if (rd_adj != 0.0)
+	dir->pop_auth_subtree.get(META_POP_IRD).adjust(now, mds->mdcache->decayrate, rd_adj);
+    }
+
+    if (hit_subtree_nested) {
+      dir->pop_auth_subtree_nested.get(type).hit(now, mds->mdcache->decayrate, amount);
+      if (rd_adj != 0.0)
+	dir->pop_auth_subtree_nested.get(META_POP_IRD).adjust(now, mds->mdcache->decayrate, rd_adj);
+    }
+
+    if (dir->is_subtree_root())
+      hit_subtree = false;                // end of auth domain, stop hitting auth counters.
+
+    if (dir->inode->get_parent_dn() == 0) break;
+    dir = dir->inode->get_parent_dn()->get_dir();
+  }
+}
+
+
+/*
+ * subtract off an exported chunk.
+ *  this excludes *dir itself (encode_export_dir should have take care of that)
+ *  we _just_ do the parents' nested counters.
+ *
+ * NOTE: call me _after_ forcing *dir into a subtree root,
+ *       but _before_ doing the encode_export_dirs.
+ */
+void MDBalancer::subtract_export(CDir *dir, utime_t now)
+{
+  dirfrag_load_vec_t subload = dir->pop_auth_subtree;
+
+  while (true) {
+    dir = dir->inode->get_parent_dir();
+    if (!dir) break;
+
+    dir->pop_nested.sub(now, mds->mdcache->decayrate, subload);
+    dir->pop_auth_subtree_nested.sub(now, mds->mdcache->decayrate, subload);
+  }
+}
+
+
+void MDBalancer::add_import(CDir *dir, utime_t now)
+{
+  dirfrag_load_vec_t subload = dir->pop_auth_subtree;
+
+  while (true) {
+    dir = dir->inode->get_parent_dir();
+    if (!dir) break;
+
+    dir->pop_nested.add(now, mds->mdcache->decayrate, subload);
+    dir->pop_auth_subtree_nested.add(now, mds->mdcache->decayrate, subload);
+  }
+}
+
+
+
+
+
+
+void MDBalancer::show_imports(bool external)
+{
+  mds->mdcache->show_subtrees();
+}
diff --git a/ceph/src/mds/MDBalancer.h b/ceph/src/mds/MDBalancer.h
new file mode 100644
index 00000000..e73a0882
--- /dev/null
+++ b/ceph/src/mds/MDBalancer.h
@@ -0,0 +1,125 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MDBALANCER_H
+#define CEPH_MDBALANCER_H
+
+#include 
+#include 
+using std::list;
+using std::map;
+
+#include "include/types.h"
+#include "common/Clock.h"
+#include "CInode.h"
+
+
+class MDS;
+class Message;
+class MHeartbeat;
+class CInode;
+class Context;
+class CDir;
+
+class MDBalancer {
+ protected:
+  MDS *mds;
+  int beat_epoch;
+
+  int last_epoch_under;  
+  int last_epoch_over; 
+
+  utime_t last_heartbeat;
+  utime_t last_fragment;
+  utime_t last_sample;    
+  utime_t rebalance_time; //ensure a consistent view of load for rebalance
+
+  // todo
+  set   split_queue, merge_queue;
+
+  // per-epoch scatter/gathered info
+  map  mds_load;
+  map       mds_meta_load;
+  map > mds_import_map;
+
+  // per-epoch state
+  double          my_load, target_load;
+  map my_targets;
+  map imported;
+  map exported;
+
+  map old_prev_targets;  // # iterations they _haven't_ been targets
+  bool check_targets();
+
+  double try_match(int ex, double& maxex,
+                   int im, double& maxim);
+  double get_maxim(int im) {
+    return target_load - mds_meta_load[im] - imported[im];
+  }
+  double get_maxex(int ex) {
+    return mds_meta_load[ex] - target_load - exported[ex];    
+  }
+
+public:
+  MDBalancer(MDS *m) : 
+    mds(m),
+    beat_epoch(0),
+    last_epoch_under(0), last_epoch_over(0), my_load(0.0), target_load(0.0) { }
+  
+  mds_load_t get_load(utime_t);
+
+  int proc_message(Message *m);
+  
+  void send_heartbeat();
+  void handle_heartbeat(MHeartbeat *m);
+
+  void tick();
+
+  void do_fragmenting();
+
+  void export_empties();
+  //set up the rebalancing targets for export and do one if the
+  //MDSMap is up to date
+  void prep_rebalance(int beat);
+  /*check if the monitor has recorded the current export targets;
+    if it has then do the actual export. Otherwise send off our
+    export targets message again*/
+  void try_rebalance();
+  void find_exports(CDir *dir, 
+                    double amount, 
+                    list& exports, 
+                    double& have,
+                    set& already_exporting);
+
+
+  void subtract_export(class CDir *ex, utime_t now);
+  void add_import(class CDir *im, utime_t now);
+
+  void hit_inode(utime_t now, class CInode *in, int type, int who=-1);
+  void hit_dir(utime_t now, class CDir *dir, int type, int who=-1, double amount=1.0);
+  void hit_recursive(utime_t now, class CDir *dir, int type, double amount, double rd_adj);
+
+
+  void show_imports(bool external=false);
+
+  void queue_split(CDir *dir);
+  void queue_merge(CDir *dir);
+
+};
+
+
+
+#endif
diff --git a/ceph/src/mds/MDCache.cc b/ceph/src/mds/MDCache.cc
new file mode 100644
index 00000000..d6cfebdd
--- /dev/null
+++ b/ceph/src/mds/MDCache.cc
@@ -0,0 +1,12223 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "MDCache.h"
+#include "MDS.h"
+#include "Server.h"
+#include "Locker.h"
+#include "MDLog.h"
+#include "MDBalancer.h"
+#include "Migrator.h"
+
+#include "AnchorClient.h"
+#include "SnapClient.h"
+
+#include "MDSMap.h"
+
+#include "CInode.h"
+#include "CDir.h"
+
+#include "Mutation.h"
+
+#include "include/ceph_fs.h"
+#include "include/filepath.h"
+
+#include "msg/Message.h"
+#include "msg/Messenger.h"
+
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "common/perf_counters.h"
+#include "common/MemoryModel.h"
+#include "osdc/Journaler.h"
+#include "osdc/Filer.h"
+
+#include "events/ESubtreeMap.h"
+#include "events/EUpdate.h"
+#include "events/ESlaveUpdate.h"
+#include "events/EImportFinish.h"
+#include "events/EFragment.h"
+#include "events/ECommitted.h"
+#include "events/ESessions.h"
+
+#include "messages/MGenericMessage.h"
+
+#include "messages/MMDSResolve.h"
+#include "messages/MMDSResolveAck.h"
+#include "messages/MMDSCacheRejoin.h"
+
+#include "messages/MDiscover.h"
+#include "messages/MDiscoverReply.h"
+
+//#include "messages/MInodeUpdate.h"
+#include "messages/MDirUpdate.h"
+#include "messages/MCacheExpire.h"
+
+#include "messages/MInodeFileCaps.h"
+
+#include "messages/MLock.h"
+#include "messages/MDentryLink.h"
+#include "messages/MDentryUnlink.h"
+
+#include "messages/MMDSFindIno.h"
+#include "messages/MMDSFindInoReply.h"
+
+#include "messages/MMDSOpenIno.h"
+#include "messages/MMDSOpenInoReply.h"
+
+#include "messages/MClientRequest.h"
+#include "messages/MClientCaps.h"
+#include "messages/MClientSnap.h"
+
+#include "messages/MMDSSlaveRequest.h"
+
+#include "messages/MMDSFragmentNotify.h"
+
+
+#include "InoTable.h"
+
+#include "common/Timer.h"
+
+using namespace std;
+
+extern struct ceph_file_layout g_default_file_layout;
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mds)
+static ostream& _prefix(std::ostream *_dout, MDS *mds) {
+  return *_dout << "mds." << mds->get_nodeid() << ".cache ";
+}
+
+long g_num_ino = 0;
+long g_num_dir = 0;
+long g_num_dn = 0;
+long g_num_cap = 0;
+
+long g_num_inoa = 0;
+long g_num_dira = 0;
+long g_num_dna = 0;
+long g_num_capa = 0;
+
+long g_num_inos = 0;
+long g_num_dirs = 0;
+long g_num_dns = 0;
+long g_num_caps = 0;
+
+set SimpleLock::empty_gather_set;
+
+
+MDCache::MDCache(MDS *m) :
+  delayed_eval_stray(member_offset(CDentry, item_stray))
+{
+  mds = m;
+  migrator = new Migrator(mds, this);
+  root = NULL;
+  myin = NULL;
+
+  stray_index = 0;
+  for (int i = 0; i < NUM_STRAY; ++i) {
+    strays[i] = NULL;
+  }
+
+  num_inodes_with_caps = 0;
+  num_caps = 0;
+
+  max_dir_commit_size = g_conf->mds_dir_max_commit_size ?
+                        (g_conf->mds_dir_max_commit_size << 20) :
+                        (0.9 *(g_conf->osd_max_write_size << 20));
+
+  discover_last_tid = 0;
+  open_ino_last_tid = 0;
+  find_ino_peer_last_tid = 0;
+
+  last_cap_id = 0;
+
+  client_lease_durations[0] = 5.0;
+  client_lease_durations[1] = 30.0;
+  client_lease_durations[2] = 300.0;
+
+  resolves_pending = false;
+  rejoins_pending = false;
+  cap_imports_num_opening = 0;
+
+  opening_root = open = false;
+  lru.lru_set_max(g_conf->mds_cache_size);
+  lru.lru_set_midpoint(g_conf->mds_cache_mid);
+
+  decayrate.set_halflife(g_conf->mds_decay_halflife);
+
+  memset(&default_log_layout, 0, sizeof(default_log_layout));
+
+  did_shutdown_log_cap = false;
+}
+
+MDCache::~MDCache() 
+{
+  delete migrator;
+  //delete renamer;
+}
+
+
+
+void MDCache::log_stat()
+{
+  mds->logger->set(l_mds_imax, g_conf->mds_cache_size);
+  mds->logger->set(l_mds_i, lru.lru_get_size());
+  mds->logger->set(l_mds_ipin, lru.lru_get_num_pinned());
+  mds->logger->set(l_mds_itop, lru.lru_get_top());
+  mds->logger->set(l_mds_ibot, lru.lru_get_bot());
+  mds->logger->set(l_mds_iptail, lru.lru_get_pintail());
+  mds->logger->set(l_mds_icap, num_inodes_with_caps);
+  mds->logger->set(l_mds_cap, num_caps);
+}
+
+
+// 
+
+bool MDCache::shutdown()
+{
+  if (lru.lru_get_size() > 0) {
+    dout(7) << "WARNING: mdcache shutdown with non-empty cache" << dendl;
+    //show_cache();
+    show_subtrees();
+    //dump();
+  }
+  return true;
+}
+
+
+// ====================================================================
+// some inode functions
+
+void MDCache::add_inode(CInode *in) 
+{
+  // add to lru, inode map
+  assert(inode_map.count(in->vino()) == 0);  // should be no dup inos!
+  inode_map[ in->vino() ] = in;
+
+  if (in->ino() < MDS_INO_SYSTEM_BASE) {
+    if (in->ino() == MDS_INO_ROOT)
+      root = in;
+    else if (in->ino() == MDS_INO_MDSDIR(mds->get_nodeid()))
+      myin = in;
+    else if (in->is_stray()) {
+      if (MDS_INO_STRAY_OWNER(in->ino()) == mds->get_nodeid()) {
+	strays[MDS_INO_STRAY_INDEX(in->ino())] = in;
+      }
+    }
+    if (in->is_base())
+      base_inodes.insert(in);
+  }
+}
+
+void MDCache::remove_inode(CInode *o) 
+{ 
+  dout(14) << "remove_inode " << *o << dendl;
+
+  if (o->get_parent_dn()) {
+    // FIXME: multiple parents?
+    CDentry *dn = o->get_parent_dn();
+    assert(!dn->is_dirty());
+    dn->dir->unlink_inode(dn);   // leave dentry ... FIXME?
+  }
+
+  if (o->is_dirty())
+    o->mark_clean();
+  if (o->is_dirty_parent())
+    o->clear_dirty_parent();
+
+  o->filelock.remove_dirty();
+  o->nestlock.remove_dirty();
+  o->dirfragtreelock.remove_dirty();
+
+  o->item_open_file.remove_myself();
+
+  // remove from inode map
+  inode_map.erase(o->vino());    
+
+  if (o->ino() < MDS_INO_SYSTEM_BASE) {
+    if (o == root) root = 0;
+    if (o == myin) myin = 0;
+    if (o->is_stray()) {
+      if (MDS_INO_STRAY_OWNER(o->ino()) == mds->get_nodeid()) {
+	strays[MDS_INO_STRAY_INDEX(o->ino())] = 0;
+      }
+    }
+    if (o->is_base())
+      base_inodes.erase(o);
+    }
+
+  // delete it
+  assert(o->get_num_ref() == 0);
+  delete o; 
+}
+
+
+
+void MDCache::init_layouts()
+{
+  default_file_layout = g_default_file_layout;
+  default_file_layout.fl_pg_pool = mds->mdsmap->get_first_data_pool();
+
+  default_log_layout = g_default_file_layout;
+  default_log_layout.fl_pg_pool = mds->mdsmap->get_metadata_pool();
+  if (g_conf->mds_log_segment_size > 0) {
+    default_log_layout.fl_object_size = g_conf->mds_log_segment_size;
+    default_log_layout.fl_stripe_unit = g_conf->mds_log_segment_size;
+  }
+}
+
+CInode *MDCache::create_system_inode(inodeno_t ino, int mode)
+{
+  dout(0) << "creating system inode with ino:" << ino << dendl;
+  CInode *in = new CInode(this);
+  in->inode.ino = ino;
+  in->inode.version = 1;
+  in->inode.mode = 0500 | mode;
+  in->inode.size = 0;
+  in->inode.ctime = 
+    in->inode.mtime = ceph_clock_now(g_ceph_context);
+  in->inode.nlink = 1;
+  in->inode.truncate_size = -1ull;
+
+  memset(&in->inode.dir_layout, 0, sizeof(in->inode.dir_layout));
+  if (in->inode.is_dir()) {
+    memset(&in->inode.layout, 0, sizeof(in->inode.layout));
+    in->inode.dir_layout.dl_dir_hash = g_conf->mds_default_dir_hash;
+    ++in->inode.rstat.rsubdirs;
+  } else {
+    in->inode.layout = default_file_layout;
+    ++in->inode.rstat.rfiles;
+  }
+  in->inode.accounted_rstat = in->inode.rstat;
+
+  if (in->is_base()) {
+    if (in->is_root())
+      in->inode_auth = pair(mds->whoami, CDIR_AUTH_UNKNOWN);
+    else
+      in->inode_auth = pair(in->ino() - MDS_INO_MDSDIR_OFFSET, CDIR_AUTH_UNKNOWN);
+    in->open_snaprealm();  // empty snaprealm
+    in->snaprealm->srnode.seq = 1;
+  }
+  
+  add_inode(in);
+  return in;
+}
+
+CInode *MDCache::create_root_inode()
+{
+  CInode *i = create_system_inode(MDS_INO_ROOT, S_IFDIR|0755);
+  i->inode.layout = default_file_layout;
+  i->inode.layout.fl_pg_pool = mds->mdsmap->get_first_data_pool();
+  return i;
+}
+
+void MDCache::create_empty_hierarchy(C_Gather *gather)
+{
+  // create root dir
+  CInode *root = create_root_inode();
+
+  // force empty root dir
+  CDir *rootdir = root->get_or_open_dirfrag(this, frag_t());
+  adjust_subtree_auth(rootdir, mds->whoami);   
+  rootdir->dir_rep = CDir::REP_ALL;   //NONE;
+
+  rootdir->fnode.accounted_fragstat = rootdir->fnode.fragstat;
+  rootdir->fnode.accounted_rstat = rootdir->fnode.rstat;
+
+  root->inode.dirstat = rootdir->fnode.fragstat;
+  root->inode.rstat = rootdir->fnode.rstat;
+  ++root->inode.rstat.rsubdirs;
+  root->inode.accounted_rstat = root->inode.rstat;
+
+  rootdir->mark_complete();
+  rootdir->mark_dirty(rootdir->pre_dirty(), mds->mdlog->get_current_segment());
+  rootdir->commit(0, gather->new_sub());
+
+  root->store(gather->new_sub());
+}
+
+void MDCache::create_mydir_hierarchy(C_Gather *gather)
+{
+  // create mds dir
+  char myname[10];
+  snprintf(myname, sizeof(myname), "mds%d", mds->whoami);
+  CInode *my = create_system_inode(MDS_INO_MDSDIR(mds->whoami), S_IFDIR);
+
+  CDir *mydir = my->get_or_open_dirfrag(this, frag_t());
+  adjust_subtree_auth(mydir, mds->whoami);   
+
+  // stray dir
+  for (int i = 0; i < NUM_STRAY; ++i) {
+    CInode *stray = create_system_inode(MDS_INO_STRAY(mds->whoami, i), S_IFDIR);
+    CDir *straydir = stray->get_or_open_dirfrag(this, frag_t());
+    stringstream name;
+    name << "stray" << i;
+    CDentry *sdn = mydir->add_primary_dentry(name.str(), stray);
+    sdn->_mark_dirty(mds->mdlog->get_current_segment());
+
+    stray->inode.dirstat = straydir->fnode.fragstat;
+
+    mydir->fnode.rstat.add(stray->inode.rstat);
+    mydir->fnode.fragstat.nsubdirs++;
+    // save them
+    straydir->mark_complete();
+    straydir->mark_dirty(straydir->pre_dirty(), mds->mdlog->get_current_segment());
+    straydir->commit(0, gather->new_sub());
+  }
+
+  CInode *journal = create_system_inode(MDS_INO_LOG_OFFSET + mds->whoami, S_IFREG);
+  string name = "journal";
+  CDentry *jdn = mydir->add_primary_dentry(name, journal);
+  jdn->_mark_dirty(mds->mdlog->get_current_segment());
+
+  mydir->fnode.fragstat.nfiles++;
+  mydir->fnode.rstat.rfiles++;
+  mydir->fnode.accounted_fragstat = mydir->fnode.fragstat;
+  mydir->fnode.accounted_rstat = mydir->fnode.rstat;
+
+  myin->inode.dirstat = mydir->fnode.fragstat;
+  myin->inode.rstat = mydir->fnode.rstat;
+  ++myin->inode.rstat.rsubdirs;
+  myin->inode.accounted_rstat = myin->inode.rstat;
+
+
+  mydir->mark_complete();
+  mydir->mark_dirty(mydir->pre_dirty(), mds->mdlog->get_current_segment());
+  mydir->commit(0, gather->new_sub());
+
+  myin->store(gather->new_sub());
+}
+
+struct C_MDC_CreateSystemFile : public Context {
+  MDCache *cache;
+  MutationRef mut;
+  CDentry *dn;
+  version_t dpv;
+  Context *fin;
+  C_MDC_CreateSystemFile(MDCache *c, MutationRef& mu, CDentry *d, version_t v, Context *f) :
+    cache(c), mut(mu), dn(d), dpv(v), fin(f) {}
+  void finish(int r) {
+    cache->_create_system_file_finish(mut, dn, dpv, fin);
+  }
+};
+
+void MDCache::_create_system_file(CDir *dir, const char *name, CInode *in, Context *fin)
+{
+  dout(10) << "_create_system_file " << name << " in " << *dir << dendl;
+  CDentry *dn = dir->add_null_dentry(name);
+
+  dn->push_projected_linkage(in);
+  version_t dpv = dn->pre_dirty();
+  
+  CDir *mdir = 0;
+  if (in->inode.is_dir()) {
+    in->inode.rstat.rsubdirs = 1;
+
+    mdir = in->get_or_open_dirfrag(this, frag_t());
+    mdir->mark_complete();
+    mdir->pre_dirty();
+  } else
+    in->inode.rstat.rfiles = 1;
+  in->inode.version = dn->pre_dirty();
+  
+  SnapRealm *realm = dir->get_inode()->find_snaprealm();
+  dn->first = in->first = realm->get_newest_seq() + 1;
+
+  MutationRef mut(new MutationImpl);
+
+  // force some locks.  hacky.
+  mds->locker->wrlock_force(&dir->inode->filelock, mut);
+  mds->locker->wrlock_force(&dir->inode->nestlock, mut);
+
+  mut->ls = mds->mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mds->mdlog, "create system file");
+  mds->mdlog->start_entry(le);
+
+  if (!in->is_mdsdir()) {
+    predirty_journal_parents(mut, &le->metablob, in, dir, PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
+    le->metablob.add_primary_dentry(dn, in, true);
+  } else {
+    predirty_journal_parents(mut, &le->metablob, in, dir, PREDIRTY_DIR, 1);
+    journal_dirty_inode(mut.get(), &le->metablob, in);
+    dn->push_projected_linkage(in->ino(), in->d_type());
+    le->metablob.add_remote_dentry(dn, true, in->ino(), in->d_type());
+    le->metablob.add_root(true, in);
+  }
+  if (mdir)
+    le->metablob.add_new_dir(mdir); // dirty AND complete AND new
+
+  mds->mdlog->submit_entry(le);
+  mds->mdlog->wait_for_safe(new C_MDC_CreateSystemFile(this, mut, dn, dpv, fin));
+  mds->mdlog->flush();
+}
+
+void MDCache::_create_system_file_finish(MutationRef& mut, CDentry *dn, version_t dpv, Context *fin)
+{
+  dout(10) << "_create_system_file_finish " << *dn << dendl;
+  
+  dn->pop_projected_linkage();
+  dn->mark_dirty(dpv, mut->ls);
+
+  CInode *in = dn->get_linkage()->get_inode();
+  in->inode.version--;
+  in->mark_dirty(in->inode.version + 1, mut->ls);
+
+  if (in->inode.is_dir()) {
+    CDir *dir = in->get_dirfrag(frag_t());
+    assert(dir);
+    dir->mark_dirty(1, mut->ls);
+    dir->mark_new(mut->ls);
+  }
+
+  mut->apply();
+  mds->locker->drop_locks(mut.get());
+  mut->cleanup();
+
+  fin->complete(0);
+
+  //if (dir && MDS_INO_IS_MDSDIR(in->ino()))
+  //migrator->export_dir(dir, (int)in->ino() - MDS_INO_MDSDIR_OFFSET);
+}
+
+
+
+struct C_MDS_RetryOpenRoot : public Context {
+  MDCache *cache;
+  C_MDS_RetryOpenRoot(MDCache *c) : cache(c) {}
+  void finish(int r) {
+    if (r < 0)
+      cache->mds->suicide();
+    else
+      cache->open_root();
+  }
+};
+
+void MDCache::open_root_inode(Context *c)
+{
+  if (mds->whoami == mds->mdsmap->get_root()) {
+    CInode *in;
+    in = create_system_inode(MDS_INO_ROOT, S_IFDIR|0755);  // initially inaccurate!
+    in->fetch(c);
+  } else {
+    discover_base_ino(MDS_INO_ROOT, c, mds->mdsmap->get_root());
+  }
+}
+
+void MDCache::open_mydir_inode(Context *c)
+{
+  CInode *in = create_system_inode(MDS_INO_MDSDIR(mds->whoami), S_IFDIR|0755);  // initially inaccurate!
+  in->fetch(c);
+}
+
+void MDCache::open_root()
+{
+  dout(10) << "open_root" << dendl;
+
+  if (!root) {
+    open_root_inode(new C_MDS_RetryOpenRoot(this));
+    return;
+  }
+  if (mds->whoami == mds->mdsmap->get_root()) {
+    assert(root->is_auth());  
+    CDir *rootdir = root->get_or_open_dirfrag(this, frag_t());
+    assert(rootdir);
+    if (!rootdir->is_subtree_root())
+      adjust_subtree_auth(rootdir, mds->whoami);   
+    if (!rootdir->is_complete()) {
+      rootdir->fetch(new C_MDS_RetryOpenRoot(this));
+      return;
+    }
+  } else {
+    assert(!root->is_auth());
+    CDir *rootdir = root->get_dirfrag(frag_t());
+    if (!rootdir) {
+      discover_dir_frag(root, frag_t(), new C_MDS_RetryOpenRoot(this));
+      return;
+    }    
+  }
+
+  if (!myin) {
+    CInode *in = create_system_inode(MDS_INO_MDSDIR(mds->whoami), S_IFDIR|0755);  // initially inaccurate!
+    in->fetch(new C_MDS_RetryOpenRoot(this));
+    return;
+  }
+  CDir *mydir = myin->get_or_open_dirfrag(this, frag_t());
+  assert(mydir);
+  adjust_subtree_auth(mydir, mds->whoami);
+
+  populate_mydir();
+}
+
+void MDCache::populate_mydir()
+{
+  assert(myin);
+  CDir *mydir = myin->get_or_open_dirfrag(this, frag_t());
+  assert(mydir);
+
+  dout(10) << "populate_mydir " << *mydir << dendl;
+
+  if (!mydir->is_complete()) {
+    mydir->fetch(new C_MDS_RetryOpenRoot(this));
+    return;
+  }    
+
+  // open or create stray
+  for (int i = 0; i < NUM_STRAY; ++i) {
+    stringstream name;
+    name << "stray" << i;
+    CDentry *straydn = mydir->lookup(name.str());
+
+    // allow for older fs's with stray instead of stray0
+    if (straydn == NULL && i == 0)
+      straydn = mydir->lookup("stray");
+
+    if (!straydn || !straydn->get_linkage()->get_inode()) {
+      _create_system_file(mydir, name.str().c_str(), create_system_inode(MDS_INO_STRAY(mds->whoami, i), S_IFDIR),
+			  new C_MDS_RetryOpenRoot(this));
+      return;
+    }
+    assert(straydn);
+    assert(strays[i]);
+    // we make multiple passes through this method; make sure we only pin each stray once.
+    if (!strays[i]->state_test(CInode::STATE_STRAYPINNED)) {
+      strays[i]->get(CInode::PIN_STRAY);
+      strays[i]->state_set(CInode::STATE_STRAYPINNED);
+      strays[i]->get_stickydirs();
+    }
+    dout(20) << " stray num " << i << " is " << *strays[i] << dendl;
+
+    // open all frags
+    list ls;
+    strays[i]->dirfragtree.get_leaves(ls);
+    for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+      frag_t fg = *p;
+      CDir *dir = strays[i]->get_dirfrag(fg);
+      if (!dir)
+	dir = strays[i]->get_or_open_dirfrag(this, fg);
+      if (dir->get_version() == 0) {
+	dir->fetch(new C_MDS_RetryOpenRoot(this));
+	return;
+      }
+    }
+  }
+
+  // open or create journal file
+  string jname("journal");
+  CDentry *jdn = mydir->lookup(jname);
+  if (!jdn || !jdn->get_linkage()->get_inode()) {
+    _create_system_file(mydir, jname.c_str(), create_system_inode(MDS_INO_LOG_OFFSET + mds->whoami, S_IFREG),
+			new C_MDS_RetryOpenRoot(this));
+    return;
+  }
+
+  // okay!
+  dout(10) << "populate_mydir done" << dendl;
+  assert(!open);    
+  open = true;
+  mds->queue_waiters(waiting_for_open);
+
+  scan_stray_dir();
+}
+
+void MDCache::open_foreign_mdsdir(inodeno_t ino, Context *fin)
+{
+  discover_base_ino(ino, fin, ino & (MAX_MDS-1));
+}
+
+CDentry *MDCache::get_or_create_stray_dentry(CInode *in)
+{
+  string straydname;
+  in->name_stray_dentry(straydname);
+
+  CInode *strayi = get_stray();
+  assert(strayi);
+  frag_t fg = strayi->pick_dirfrag(straydname);
+  CDir *straydir = strayi->get_dirfrag(fg);
+  assert(straydir);
+  CDentry *straydn = straydir->lookup(straydname);
+  if (!straydn) {
+    straydn = straydir->add_null_dentry(straydname);
+    straydn->mark_new();
+  } else 
+    assert(straydn->get_projected_linkage()->is_null());
+
+  straydn->state_set(CDentry::STATE_STRAY);
+  return straydn;
+}
+
+
+
+MDSCacheObject *MDCache::get_object(MDSCacheObjectInfo &info) 
+{
+  // inode?
+  if (info.ino) 
+    return get_inode(info.ino, info.snapid);
+
+  // dir or dentry.
+  CDir *dir = get_dirfrag(info.dirfrag);
+  if (!dir) return 0;
+    
+  if (info.dname.length()) 
+    return dir->lookup(info.dname, info.snapid);
+  else
+    return dir;
+}
+
+
+
+
+// ====================================================================
+// subtree management
+
+void MDCache::list_subtrees(list& ls)
+{
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p)
+    ls.push_back(p->first);
+}
+
+/*
+ * adjust the dir_auth of a subtree.
+ * merge with parent and/or child subtrees, if is it appropriate.
+ * merge can ONLY happen if both parent and child have unambiguous auth.
+ */
+void MDCache::adjust_subtree_auth(CDir *dir, pair auth, bool do_eval)
+{
+  dout(7) << "adjust_subtree_auth " << dir->get_dir_auth() << " -> " << auth
+	  << " on " << *dir << dendl;
+
+  if (mds->is_any_replay() || mds->is_resolve())
+    do_eval = false;
+
+  show_subtrees();
+
+  CDir *root;
+  if (dir->inode->is_base()) {
+    root = dir;  // bootstrap hack.
+    if (subtrees.count(root) == 0) {
+      subtrees[root].clear();
+      root->get(CDir::PIN_SUBTREE);
+    }
+  } else {
+    root = get_subtree_root(dir);  // subtree root
+  }
+  assert(root);
+  assert(subtrees.count(root));
+  dout(7) << " current root is " << *root << dendl;
+
+  if (root == dir) {
+    // i am already a subtree.
+    dir->set_dir_auth(auth);
+  } else {
+    // i am a new subtree.
+    dout(10) << "  new subtree at " << *dir << dendl;
+    assert(subtrees.count(dir) == 0);
+    subtrees[dir].clear();      // create empty subtree bounds list for me.
+    dir->get(CDir::PIN_SUBTREE);
+
+    // set dir_auth
+    dir->set_dir_auth(auth);
+    
+    // move items nested beneath me, under me.
+    set::iterator p = subtrees[root].begin();
+    while (p != subtrees[root].end()) {
+      set::iterator next = p;
+      ++next;
+      if (get_subtree_root((*p)->get_parent_dir()) == dir) {
+	// move under me
+	dout(10) << "  claiming child bound " << **p << dendl;
+	subtrees[dir].insert(*p); 
+	subtrees[root].erase(p);
+      }
+      p = next;
+    }
+    
+    // i am a bound of the parent subtree.
+    subtrees[root].insert(dir); 
+
+    // i am now the subtree root.
+    root = dir;
+
+    // adjust recursive pop counters
+    if (dir->is_auth()) {
+      utime_t now = ceph_clock_now(g_ceph_context);
+      CDir *p = dir->get_parent_dir();
+      while (p) {
+	p->pop_auth_subtree.sub(now, decayrate, dir->pop_auth_subtree);
+	if (p->is_subtree_root()) break;
+	p = p->inode->get_parent_dir();
+      }
+    }
+
+    if (do_eval)
+      eval_subtree_root(dir->get_inode());
+  }
+
+  show_subtrees();
+}
+
+
+void MDCache::try_subtree_merge(CDir *dir)
+{
+  dout(7) << "try_subtree_merge " << *dir << dendl;
+  assert(subtrees.count(dir));
+  set oldbounds = subtrees[dir];
+
+  // try merge at my root
+  try_subtree_merge_at(dir);
+
+  // try merge at my old bounds
+  for (set::iterator p = oldbounds.begin();
+       p != oldbounds.end();
+       ++p) 
+    try_subtree_merge_at(*p);
+}
+
+class C_MDC_SubtreeMergeWB : public Context {
+  MDCache *mdcache;
+  CInode *in;
+  MutationRef mut;
+public:
+  C_MDC_SubtreeMergeWB(MDCache *mdc, CInode *i, MutationRef& m) : mdcache(mdc), in(i), mut(m) {}
+  void finish(int r) { 
+    mdcache->subtree_merge_writebehind_finish(in, mut);
+  }
+};
+
+void MDCache::try_subtree_merge_at(CDir *dir, bool do_eval)
+{
+  dout(10) << "try_subtree_merge_at " << *dir << dendl;
+  assert(subtrees.count(dir));
+
+  if (mds->is_any_replay() || mds->is_resolve())
+    do_eval = false;
+
+  // merge with parent?
+  CDir *parent = dir;  
+  if (!dir->inode->is_base())
+    parent = get_subtree_root(dir->get_parent_dir());
+  
+  if (parent != dir &&                              // we have a parent,
+      parent->dir_auth == dir->dir_auth &&          // auth matches,
+      dir->dir_auth.second == CDIR_AUTH_UNKNOWN &&  // auth is unambiguous,
+      !dir->state_test(CDir::STATE_EXPORTBOUND)) {  // not an exportbound,
+    // merge with parent.
+    dout(10) << "  subtree merge at " << *dir << dendl;
+    dir->set_dir_auth(CDIR_AUTH_DEFAULT);
+    
+    // move our bounds under the parent
+    for (set::iterator p = subtrees[dir].begin();
+	 p != subtrees[dir].end();
+	 ++p) 
+      subtrees[parent].insert(*p);
+    
+    // we are no longer a subtree or bound
+    dir->put(CDir::PIN_SUBTREE);
+    subtrees.erase(dir);
+    subtrees[parent].erase(dir);
+
+    // adjust popularity?
+    if (dir->is_auth()) {
+      utime_t now = ceph_clock_now(g_ceph_context);
+      CDir *p = dir->get_parent_dir();
+      while (p) {
+	p->pop_auth_subtree.add(now, decayrate, dir->pop_auth_subtree);
+	if (p->is_subtree_root()) break;
+	p = p->inode->get_parent_dir();
+      }
+    }
+
+    if (do_eval)
+      eval_subtree_root(dir->get_inode());
+
+    // journal inode? 
+    //  (this is a large hammer to ensure that dirfragtree updates will
+    //   hit the disk before the relevant dirfrags ever close)
+    if (dir->inode->is_auth() &&
+	dir->inode->can_auth_pin() &&
+	(mds->is_clientreplay() || mds->is_active() || mds->is_stopping())) {
+      CInode *in = dir->inode;
+      dout(10) << "try_subtree_merge_at journaling merged bound " << *in << dendl;
+      
+      in->auth_pin(this);
+
+      // journal write-behind.
+      inode_t *pi = in->project_inode();
+      pi->version = in->pre_dirty();
+      
+      MutationRef mut(new MutationImpl);
+      mut->ls = mds->mdlog->get_current_segment();
+      EUpdate *le = new EUpdate(mds->mdlog, "subtree merge writebehind");
+      mds->mdlog->start_entry(le);
+
+      le->metablob.add_dir_context(in->get_parent_dn()->get_dir());
+      journal_dirty_inode(mut.get(), &le->metablob, in);
+      
+      mds->mdlog->submit_entry(le);
+      mds->mdlog->wait_for_safe(new C_MDC_SubtreeMergeWB(this, in, mut));
+      mds->mdlog->flush();
+    }
+  } 
+
+  show_subtrees(15);
+}
+
+void MDCache::subtree_merge_writebehind_finish(CInode *in, MutationRef& mut)
+{
+  dout(10) << "subtree_merge_writebehind_finish on " << in << dendl;
+  in->pop_and_dirty_projected_inode(mut->ls);
+
+  mut->apply();
+  mds->locker->drop_locks(mut.get());
+  mut->cleanup();
+
+  in->auth_unpin(this);
+}
+
+void MDCache::eval_subtree_root(CInode *diri)
+{
+  // evaluate subtree inode filelock?
+  //  (we should scatter the filelock on subtree bounds)
+  if (diri->is_auth())
+    mds->locker->try_eval(diri, CEPH_LOCK_IFILE | CEPH_LOCK_INEST);
+}
+
+
+void MDCache::adjust_bounded_subtree_auth(CDir *dir, set& bounds, pair auth)
+{
+  dout(7) << "adjust_bounded_subtree_auth " << dir->get_dir_auth() << " -> " << auth
+	  << " on " << *dir
+	  << " bounds " << bounds
+	  << dendl;
+
+  show_subtrees();
+
+  CDir *root;
+  if (dir->ino() == MDS_INO_ROOT) {
+    root = dir;  // bootstrap hack.
+    if (subtrees.count(root) == 0) {
+      subtrees[root].clear();
+      root->get(CDir::PIN_SUBTREE);
+    }
+  } else {
+    root = get_subtree_root(dir);  // subtree root
+  }
+  assert(root);
+  assert(subtrees.count(root));
+  dout(7) << " current root is " << *root << dendl;
+
+  pair oldauth = dir->authority();
+
+  if (root == dir) {
+    // i am already a subtree.
+    dir->set_dir_auth(auth);
+  } else {
+    // i am a new subtree.
+    dout(10) << "  new subtree at " << *dir << dendl;
+    assert(subtrees.count(dir) == 0);
+    subtrees[dir].clear();      // create empty subtree bounds list for me.
+    dir->get(CDir::PIN_SUBTREE);
+    
+    // set dir_auth
+    dir->set_dir_auth(auth);
+    
+    // move items nested beneath me, under me.
+    set::iterator p = subtrees[root].begin();
+    while (p != subtrees[root].end()) {
+      set::iterator next = p;
+      ++next;
+      if (get_subtree_root((*p)->get_parent_dir()) == dir) {
+	// move under me
+	dout(10) << "  claiming child bound " << **p << dendl;
+	subtrees[dir].insert(*p); 
+	subtrees[root].erase(p);
+      }
+      p = next;
+    }
+    
+    // i am a bound of the parent subtree.
+    subtrees[root].insert(dir); 
+
+    // i am now the subtree root.
+    root = dir;
+  }
+
+  // verify/adjust bounds.
+  // - these may be new, or
+  // - beneath existing ambiguous bounds (which will be collapsed),
+  // - but NOT beneath unambiguous bounds.
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) {
+    CDir *bound = *p;
+    
+    // new bound?
+    if (subtrees[dir].count(bound) == 0) {
+      if (get_subtree_root(bound) == dir) {
+	dout(10) << "  new bound " << *bound << ", adjusting auth back to old " << oldauth << dendl;
+	adjust_subtree_auth(bound, oldauth);       // otherwise, adjust at bound.
+      }
+      else {
+	dout(10) << "  want bound " << *bound << dendl;
+	CDir *t = get_subtree_root(bound->get_parent_dir());
+	if (subtrees[t].count(bound) == 0) {
+	  assert(t != dir);
+	  dout(10) << "  new bound " << *bound << dendl;
+	  adjust_subtree_auth(bound, t->authority());
+	}
+	// make sure it's nested beneath ambiguous subtree(s)
+	while (1) {
+	  while (subtrees[dir].count(t) == 0)
+	    t = get_subtree_root(t->get_parent_dir());
+	  dout(10) << "  swallowing intervening subtree at " << *t << dendl;
+	  adjust_subtree_auth(t, auth);
+	  try_subtree_merge_at(t);
+	  t = get_subtree_root(bound->get_parent_dir());
+	  if (t == dir) break;
+	}
+      }
+    }
+    else {
+      dout(10) << "  already have bound " << *bound << dendl;
+    }
+  }
+  // merge stray bounds?
+  while (!subtrees[dir].empty()) {
+    set copy = subtrees[dir];
+    for (set::iterator p = copy.begin(); p != copy.end(); ++p) {
+      if (bounds.count(*p) == 0) {
+	CDir *stray = *p;
+	dout(10) << "  swallowing extra subtree at " << *stray << dendl;
+	adjust_subtree_auth(stray, auth);
+	try_subtree_merge_at(stray);
+      }
+    }
+    // swallowing subtree may add new subtree bounds
+    if (copy == subtrees[dir])
+      break;
+  }
+
+  // bound should now match.
+  verify_subtree_bounds(dir, bounds);
+
+  show_subtrees();
+}
+
+
+/*
+ * return a set of CDir*'s that correspond to the given bound set.  Only adjust
+ * fragmentation as necessary to get an equivalent bounding set.  That is, only
+ * split if one of our frags spans the provided bounding set.  Never merge.
+ */
+void MDCache::get_force_dirfrag_bound_set(vector& dfs, set& bounds)
+{
+  dout(10) << "get_force_dirfrag_bound_set " << dfs << dendl;
+
+  // sort by ino
+  map byino;
+  for (vector::iterator p = dfs.begin(); p != dfs.end(); ++p)
+    byino[p->ino].insert(p->frag);
+  dout(10) << " by ino: " << byino << dendl;
+
+  for (map::iterator p = byino.begin(); p != byino.end(); ++p) {
+    CInode *diri = get_inode(p->first);
+    if (!diri)
+      continue;
+    dout(10) << " checking fragset " << p->second.get() << " on " << *diri << dendl;
+
+    fragtree_t tmpdft;
+    for (set::iterator q = p->second.begin(); q != p->second.end(); ++q)
+      tmpdft.force_to_leaf(g_ceph_context, *q);
+
+    for (set::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+      frag_t fg = *q;
+      list fgls;
+      diri->dirfragtree.get_leaves_under(fg, fgls);
+      if (fgls.empty()) {
+	bool all = true;
+	frag_t approx_fg = diri->dirfragtree[fg.value()];
+	list ls;
+	tmpdft.get_leaves_under(approx_fg, ls);
+	for (list::iterator r = ls.begin(); r != ls.end(); ++r) {
+	  if (p->second.get().count(*r) == 0) {
+	    // not bound, so the resolve message is from auth MDS of the dirfrag
+	    force_dir_fragment(diri, *r);
+	    all = false;
+	  }
+	}
+	if (all)
+	  fgls.push_back(approx_fg);
+	else
+	  diri->dirfragtree.get_leaves_under(fg, fgls);
+      }
+      dout(10) << "  frag " << fg << " contains " << fgls << dendl;
+      for (list::iterator r = fgls.begin(); r != fgls.end(); ++r) {
+	CDir *dir = diri->get_dirfrag(*r);
+	if (dir)
+	  bounds.insert(dir);
+      }
+    }
+  }
+}
+
+void MDCache::adjust_bounded_subtree_auth(CDir *dir, vector& bound_dfs, pair auth)
+{
+  dout(7) << "adjust_bounded_subtree_auth " << dir->get_dir_auth() << " -> " << auth
+	  << " on " << *dir << " bound_dfs " << bound_dfs << dendl;
+
+  set bounds;
+  get_force_dirfrag_bound_set(bound_dfs, bounds);
+  adjust_bounded_subtree_auth(dir, bounds, auth);
+}
+
+void MDCache::map_dirfrag_set(list& dfs, set& result)
+{
+  dout(10) << "map_dirfrag_set " << dfs << dendl;
+
+  // group by inode
+  map ino_fragset;
+  for (list::iterator p = dfs.begin(); p != dfs.end(); ++p)
+    ino_fragset[p->ino].insert(p->frag);
+
+  // get frags
+  for (map::iterator p = ino_fragset.begin();
+       p != ino_fragset.end();
+       ++p) {
+    CInode *in = get_inode(p->first);
+    if (!in)
+      continue;
+
+    list fglist;
+    for (set::iterator q = p->second.begin(); q != p->second.end(); ++q)
+      in->dirfragtree.get_leaves_under(*q, fglist);
+
+    dout(15) << "map_dirfrag_set " << p->second << " -> " << fglist
+	     << " on " << *in << dendl;
+
+    for (list::iterator q = fglist.begin(); q != fglist.end(); ++q) {
+      CDir *dir = in->get_dirfrag(*q);
+      if (dir)
+	result.insert(dir);
+    }
+  }
+}
+
+
+
+CDir *MDCache::get_subtree_root(CDir *dir)
+{
+  // find the underlying dir that delegates (or is about to delegate) auth
+  while (true) {
+    if (dir->is_subtree_root()) 
+      return dir;
+    dir = dir->get_inode()->get_parent_dir();
+    if (!dir) 
+      return 0;             // none
+  }
+}
+
+CDir *MDCache::get_projected_subtree_root(CDir *dir)
+{
+  // find the underlying dir that delegates (or is about to delegate) auth
+  while (true) {
+    if (dir->is_subtree_root()) 
+      return dir;
+    dir = dir->get_inode()->get_projected_parent_dir();
+    if (!dir) 
+      return 0;             // none
+  }
+}
+
+void MDCache::remove_subtree(CDir *dir)
+{
+  dout(10) << "remove_subtree " << *dir << dendl;
+  assert(subtrees.count(dir));
+  assert(subtrees[dir].empty());
+  subtrees.erase(dir);
+  dir->put(CDir::PIN_SUBTREE);
+  if (dir->get_parent_dir()) {
+    CDir *p = get_subtree_root(dir->get_parent_dir());
+    assert(subtrees[p].count(dir));
+    subtrees[p].erase(dir);
+  }
+}
+
+void MDCache::get_subtree_bounds(CDir *dir, set& bounds)
+{
+  assert(subtrees.count(dir));
+  bounds = subtrees[dir];
+}
+
+void MDCache::get_wouldbe_subtree_bounds(CDir *dir, set& bounds)
+{
+  if (subtrees.count(dir)) {
+    // just copy them, dir is a subtree.
+    get_subtree_bounds(dir, bounds);
+  } else {
+    // find them
+    CDir *root = get_subtree_root(dir);
+    for (set::iterator p = subtrees[root].begin();
+	 p != subtrees[root].end();
+	 ++p) {
+      CDir *t = *p;
+      while (t != root) {
+	t = t->get_parent_dir();
+	assert(t);
+	if (t == dir) {
+	  bounds.insert(*p);
+	  continue;
+	}
+      }
+    }
+  }
+}
+
+void MDCache::verify_subtree_bounds(CDir *dir, const set& bounds)
+{
+  // for debugging only.
+  assert(subtrees.count(dir));
+  if (bounds != subtrees[dir]) {
+    dout(0) << "verify_subtree_bounds failed" << dendl;
+    set b = bounds;
+    for (set::iterator p = subtrees[dir].begin();
+	 p != subtrees[dir].end();
+	 ++p) {
+      if (bounds.count(*p)) {
+	b.erase(*p);
+	continue;
+      }
+      dout(0) << "  missing bound " << **p << dendl;
+    }
+    for (set::iterator p = b.begin();
+	 p != b.end();
+	 ++p) 
+      dout(0) << "    extra bound " << **p << dendl;
+  }
+  assert(bounds == subtrees[dir]);
+}
+
+void MDCache::verify_subtree_bounds(CDir *dir, const list& bounds)
+{
+  // for debugging only.
+  assert(subtrees.count(dir));
+
+  // make sure that any bounds i do have are properly noted as such.
+  int failed = 0;
+  for (list::const_iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) {
+    CDir *bd = get_dirfrag(*p);
+    if (!bd) continue;
+    if (subtrees[dir].count(bd) == 0) {
+      dout(0) << "verify_subtree_bounds failed: extra bound " << *bd << dendl;
+      failed++;
+    }
+  }
+  assert(failed == 0);
+}
+
+void MDCache::project_subtree_rename(CInode *diri, CDir *olddir, CDir *newdir)
+{
+  dout(10) << "project_subtree_rename " << *diri << " from " << *olddir
+	   << " to " << *newdir << dendl;
+  projected_subtree_renames[diri].push_back(pair(olddir, newdir));
+}
+
+void MDCache::adjust_subtree_after_rename(CInode *diri, CDir *olddir,
+                                          bool pop, bool imported)
+{
+  dout(10) << "adjust_subtree_after_rename " << *diri << " from " << *olddir << dendl;
+
+  //show_subtrees();
+
+  CDir *newdir = diri->get_parent_dir();
+
+  if (pop) {
+    map > >::iterator p = projected_subtree_renames.find(diri);
+    assert(p != projected_subtree_renames.end());
+    assert(!p->second.empty());
+    assert(p->second.front().first == olddir);
+    assert(p->second.front().second == newdir);
+    p->second.pop_front();
+    if (p->second.empty())
+      projected_subtree_renames.erase(p);
+  }
+
+  // adjust subtree
+  list dfls;
+  // make sure subtree dirfrags are at the front of the list
+  diri->get_subtree_dirfrags(dfls);
+  diri->get_nested_dirfrags(dfls);
+  for (list::iterator p = dfls.begin(); p != dfls.end(); ++p) {
+    CDir *dir = *p;
+
+    dout(10) << "dirfrag " << *dir << dendl;
+    CDir *oldparent = get_subtree_root(olddir);
+    dout(10) << " old parent " << *oldparent << dendl;
+    CDir *newparent = get_subtree_root(newdir);
+    dout(10) << " new parent " << *newparent << dendl;
+
+    if (oldparent == newparent) {
+      dout(10) << "parent unchanged for " << *dir << " at " << *oldparent << dendl;
+      continue;
+    }
+
+    if (dir->is_subtree_root()) {
+      // children are fine.  change parent.
+      dout(10) << "moving " << *dir << " from " << *oldparent << " to " << *newparent << dendl;
+      assert(subtrees[oldparent].count(dir));
+      subtrees[oldparent].erase(dir);
+      assert(subtrees.count(newparent));
+      subtrees[newparent].insert(dir);
+      try_subtree_merge_at(dir, !imported);
+    } else {
+      // mid-subtree.
+
+      // see if any old bounds move to the new parent.
+      list tomove;
+      for (set::iterator p = subtrees[oldparent].begin();
+	   p != subtrees[oldparent].end();
+	   ++p) {
+	CDir *bound = *p;
+	CDir *broot = get_subtree_root(bound->get_parent_dir());
+	if (broot != oldparent) {
+	  assert(broot == newparent);
+	  tomove.push_back(bound);
+	}
+      }
+      for (list::iterator p = tomove.begin(); p != tomove.end(); ++p) {
+	CDir *bound = *p;
+	dout(10) << "moving bound " << *bound << " from " << *oldparent << " to " << *newparent << dendl;
+	subtrees[oldparent].erase(bound);
+	subtrees[newparent].insert(bound);
+      }	   
+
+      // did auth change?
+      if (oldparent->authority() != newparent->authority()) {
+	adjust_subtree_auth(dir, oldparent->authority(), !imported);  // caller is responsible for *diri.
+	try_subtree_merge_at(dir, !imported);
+      }
+    }
+  }
+
+  show_subtrees();
+}
+
+
+void MDCache::get_fullauth_subtrees(set& s)
+{
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *root = p->first;
+    if (root->is_full_dir_auth())
+      s.insert(root);
+  }
+}
+void MDCache::get_auth_subtrees(set& s)
+{
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *root = p->first;
+    if (root->is_auth())
+      s.insert(root);
+  }
+}
+
+
+// count.
+
+int MDCache::num_subtrees()
+{
+  return subtrees.size();
+}
+
+int MDCache::num_subtrees_fullauth()
+{
+  int n = 0;
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *root = p->first;
+    if (root->is_full_dir_auth())
+      n++;
+  }
+  return n;
+}
+
+int MDCache::num_subtrees_fullnonauth()
+{
+  int n = 0;
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *root = p->first;
+    if (root->is_full_dir_nonauth())
+      n++;
+  }
+  return n;
+}
+
+
+
+// ===================================
+// journal and snap/cow helpers
+
+
+/*
+ * find first inode in cache that follows given snapid.  otherwise, return current.
+ */
+CInode *MDCache::pick_inode_snap(CInode *in, snapid_t follows)
+{
+  dout(10) << "pick_inode_snap follows " << follows << " on " << *in << dendl;
+  assert(in->last == CEPH_NOSNAP);
+
+  SnapRealm *realm = in->find_snaprealm();
+  const set& snaps = realm->get_snaps();
+  dout(10) << " realm " << *realm << " " << *realm->inode << dendl;
+  dout(10) << " snaps " << snaps << dendl;
+
+  if (snaps.empty())
+    return in;
+
+  for (set::const_iterator p = snaps.upper_bound(follows);  // first item > follows
+       p != snaps.end();
+       ++p) {
+    CInode *t = get_inode(in->ino(), *p);
+    if (t) {
+      in = t;
+      dout(10) << "pick_inode_snap snap " << *p << " found " << *in << dendl;
+      break;
+    }
+  }
+  return in;
+}
+
+
+/*
+ * note: i'm currently cheating wrt dirty and inode.version on cow
+ * items.  instead of doing a full dir predirty, i just take the
+ * original item's version, and set the dirty flag (via
+ * mutation::add_cow_{inode,dentry}() and mutation::apply().  that
+ * means a special case in the dir commit clean sweep assertions.
+ * bah.
+ */
+CInode *MDCache::cow_inode(CInode *in, snapid_t last)
+{
+  assert(last >= in->first);
+
+  CInode *oldin = new CInode(this, true, in->first, last);
+  oldin->inode = *in->get_previous_projected_inode();
+  oldin->symlink = in->symlink;
+  oldin->xattrs = *in->get_previous_projected_xattrs();
+
+  oldin->inode.trim_client_ranges(last);
+
+  in->first = last+1;
+
+  dout(10) << "cow_inode " << *in << " to " << *oldin << dendl;
+  add_inode(oldin);
+  
+  SnapRealm *realm = in->find_snaprealm();
+  const set& snaps = realm->get_snaps();
+
+  // clone caps?
+  for (map::iterator p = in->client_caps.begin();
+      p != in->client_caps.end();
+      ++p) {
+    client_t client = p->first;
+    Capability *cap = p->second;
+    int issued = cap->issued();
+    if ((issued & CEPH_CAP_ANY_WR) &&
+	cap->client_follows < last) {
+      // note in oldin
+      for (int i = 0; i < num_cinode_locks; i++) {
+	if (issued & cinode_lock_info[i].wr_caps) {
+	  int lockid = cinode_lock_info[i].lock;
+	  SimpleLock *lock = oldin->get_lock(lockid);
+	  assert(lock);
+	  oldin->client_snap_caps[lockid].insert(client);
+	  oldin->auth_pin(lock);
+	  lock->set_state(LOCK_SNAP_SYNC);  // gathering
+	  lock->get_wrlock(true);
+	  dout(10) << " client." << client << " cap " << ccap_string(issued & cinode_lock_info[i].wr_caps)
+		   << " wrlock lock " << *lock << " on " << *oldin << dendl;
+	}
+      }
+      cap->client_follows = last;
+      
+      // we need snapflushes for any intervening snaps
+      dout(10) << "  snaps " << snaps << dendl;
+      for (set::const_iterator q = snaps.lower_bound(oldin->first);
+	   q != snaps.end() && *q <= last;
+	   ++q) {
+	in->add_need_snapflush(oldin, *q, client);
+      }
+    } else {
+      dout(10) << " ignoring client." << client << " cap follows " << cap->client_follows << dendl;
+    }
+  }
+
+  return oldin;
+}
+
+void MDCache::journal_cow_dentry(MutationImpl *mut, EMetaBlob *metablob,
+                                 CDentry *dn, snapid_t follows,
+				 CInode **pcow_inode, CDentry::linkage_t *dnl)
+{
+  if (!dn) {
+    dout(10) << "journal_cow_dentry got null CDentry, returning" << dendl;
+    return;
+  }
+  dout(10) << "journal_cow_dentry follows " << follows << " on " << *dn << dendl;
+  assert(dn->is_auth());
+
+  // nothing to cow on a null dentry, fix caller
+  if (!dnl)
+    dnl = dn->get_projected_linkage();
+  assert(!dnl->is_null());
+
+  if (dnl->is_primary() && dnl->get_inode()->is_multiversion()) {
+    // multiversion inode.
+    CInode *in = dnl->get_inode();
+
+    if (follows == CEPH_NOSNAP)
+      follows = in->find_snaprealm()->get_newest_seq();
+
+    if (in->get_projected_parent_dn() != dn &&
+	follows+1 > dn->first) {
+      snapid_t oldfirst = dn->first;
+      dn->first = follows+1;
+      CDentry *olddn = dn->dir->add_remote_dentry(dn->name, in->ino(),  in->d_type(),
+						  oldfirst, follows);
+      olddn->pre_dirty();
+      dout(10) << " olddn " << *olddn << dendl;
+      metablob->add_remote_dentry(olddn, true);
+      mut->add_cow_dentry(olddn);
+
+      // FIXME: adjust link count here?  hmm.
+    }
+
+    // already cloned?
+    if (follows < in->first) {
+      dout(10) << "journal_cow_dentry follows " << follows << " < first on " << *in << dendl;
+      return;
+    }
+
+    in->cow_old_inode(follows, false);
+
+  } else {
+    if (follows == CEPH_NOSNAP)
+      follows = dn->dir->inode->find_snaprealm()->get_newest_seq();
+    
+    // already cloned?
+    if (follows < dn->first) {
+      dout(10) << "journal_cow_dentry follows " << follows << " < first on " << *dn << dendl;
+      return;
+    }
+       
+    // update dn.first before adding old dentry to cdir's map
+    snapid_t oldfirst = dn->first;
+    dn->first = follows+1;
+    
+    dout(10) << "    dn " << *dn << dendl;
+    if (dnl->is_primary()) {
+      assert(oldfirst == dnl->get_inode()->first);
+      CInode *oldin = cow_inode(dnl->get_inode(), follows);
+      mut->add_cow_inode(oldin);
+      if (pcow_inode)
+	*pcow_inode = oldin;
+      CDentry *olddn = dn->dir->add_primary_dentry(dn->name, oldin, oldfirst, follows);
+      oldin->inode.version = olddn->pre_dirty();
+      dout(10) << " olddn " << *olddn << dendl;
+      metablob->add_primary_dentry(olddn, 0, true);
+      mut->add_cow_dentry(olddn);
+    } else {
+      assert(dnl->is_remote());
+      CDentry *olddn = dn->dir->add_remote_dentry(dn->name, dnl->get_remote_ino(), dnl->get_remote_d_type(),
+						  oldfirst, follows);
+      olddn->pre_dirty();
+      dout(10) << " olddn " << *olddn << dendl;
+      metablob->add_remote_dentry(olddn, true);
+      mut->add_cow_dentry(olddn);
+    }
+  }
+}
+
+
+void MDCache::journal_cow_inode(MutationRef& mut, EMetaBlob *metablob,
+                                CInode *in, snapid_t follows,
+				CInode **pcow_inode)
+{
+  dout(10) << "journal_cow_inode follows " << follows << " on " << *in << dendl;
+  CDentry *dn = in->get_projected_parent_dn();
+  journal_cow_dentry(mut.get(), metablob, dn, follows, pcow_inode);
+}
+
+void MDCache::journal_dirty_inode(MutationImpl *mut, EMetaBlob *metablob, CInode *in, snapid_t follows)
+{
+  if (in->is_base()) {
+    metablob->add_root(true, in, in->get_projected_inode());
+  } else {
+    if (follows == CEPH_NOSNAP && in->last != CEPH_NOSNAP)
+      follows = in->first - 1;
+    CDentry *dn = in->get_projected_parent_dn();
+    if (!dn->get_projected_linkage()->is_null())  // no need to cow a null dentry
+      journal_cow_dentry(mut, metablob, dn, follows);
+    if (in->get_projected_inode()->is_backtrace_updated()) {
+      bool dirty_pool = in->get_projected_inode()->layout.fl_pg_pool !=
+			in->get_previous_projected_inode()->layout.fl_pg_pool;
+      metablob->add_primary_dentry(dn, in, true, true, dirty_pool);
+    } else {
+      metablob->add_primary_dentry(dn, in, true);
+    }
+  }
+}
+
+
+
+// nested ---------------------------------------------------------------
+
+void MDCache::project_rstat_inode_to_frag(CInode *cur, CDir *parent, snapid_t first, int linkunlink)
+{
+  CDentry *parentdn = cur->get_projected_parent_dn();
+  inode_t *curi = cur->get_projected_inode();
+
+  if (cur->first > first)
+    first = cur->first;
+
+  dout(10) << "projected_rstat_inode_to_frag first " << first << " linkunlink " << linkunlink
+	   << " " << *cur << dendl;
+  dout(20) << "    frag head is [" << parent->first << ",head] " << dendl;
+  dout(20) << " inode update is [" << first << "," << cur->last << "]" << dendl;
+
+  /*
+   * FIXME.  this incompletely propagates rstats to _old_ parents
+   * (i.e. shortly after a directory rename).  but we need full
+   * blown hard link backpointers to make this work properly...
+   */
+  snapid_t floor = parentdn->first;
+  dout(20) << " floor of " << floor << " from parent dn " << *parentdn << dendl;
+
+  if (cur->last >= floor)
+    _project_rstat_inode_to_frag(*curi, MAX(first, floor), cur->last, parent, linkunlink);
+      
+  for (set::iterator p = cur->dirty_old_rstats.begin();
+       p != cur->dirty_old_rstats.end();
+       ++p) {
+    old_inode_t& old = cur->old_inodes[*p];
+    if (*p >= floor)
+      _project_rstat_inode_to_frag(old.inode, MAX(old.first, floor), *p, parent);
+  }
+  cur->dirty_old_rstats.clear();
+}
+
+
+void MDCache::_project_rstat_inode_to_frag(inode_t& inode, snapid_t ofirst, snapid_t last,
+					  CDir *parent, int linkunlink)
+{
+  dout(10) << "_project_rstat_inode_to_frag [" << ofirst << "," << last << "]" << dendl;
+  dout(20) << "  inode           rstat " << inode.rstat << dendl;
+  dout(20) << "  inode accounted_rstat " << inode.accounted_rstat << dendl;
+  nest_info_t delta;
+  if (linkunlink == 0) {
+    delta.add(inode.rstat);
+    delta.sub(inode.accounted_rstat);
+  } else if (linkunlink < 0) {
+    delta.sub(inode.accounted_rstat);
+  } else {
+    delta.add(inode.rstat);
+  }
+  dout(20) << "                  delta " << delta << dendl;
+
+  inode.accounted_rstat = inode.rstat;
+
+  while (last >= ofirst) {
+    /*
+     * pick fnode version to update.  at each iteration, we want to
+     * pick a segment ending in 'last' to update.  split as necessary
+     * to make that work.  then, adjust first up so that we only
+     * update one segment at a time.  then loop to cover the whole
+     * [ofirst,last] interval.
+     */    
+    nest_info_t *prstat;
+    snapid_t first;
+    fnode_t *pf = parent->get_projected_fnode();
+    if (last == CEPH_NOSNAP) {
+      first = MAX(ofirst, parent->first);
+      prstat = &pf->rstat;
+      dout(20) << " projecting to head [" << first << "," << last << "] " << *prstat << dendl;
+
+      if (first > parent->first &&
+	  !(pf->rstat == pf->accounted_rstat)) {
+	dout(10) << "  target snapped and not fully accounted, cow to dirty_old_rstat ["
+		 << parent->first << "," << (first-1) << "] "
+		 << " " << *prstat << "/" << pf->accounted_rstat
+		 << dendl;
+	parent->dirty_old_rstat[first-1].first = parent->first;
+	parent->dirty_old_rstat[first-1].rstat = pf->rstat;
+	parent->dirty_old_rstat[first-1].accounted_rstat = pf->accounted_rstat;
+      }
+      parent->first = first;
+    } else if (last >= parent->first) {
+      first = parent->first;
+      parent->dirty_old_rstat[last].first = first;
+      parent->dirty_old_rstat[last].rstat = pf->rstat;
+      parent->dirty_old_rstat[last].accounted_rstat = pf->accounted_rstat;
+      prstat = &parent->dirty_old_rstat[last].rstat;
+      dout(10) << " projecting to newly split dirty_old_fnode [" << first << "," << last << "] "
+	       << " " << *prstat << "/" << pf->accounted_rstat << dendl;
+    } else {
+      // be careful, dirty_old_rstat is a _sparse_ map.
+      // sorry, this is ugly.
+      first = ofirst;
+
+      // find any intersection with last
+      map::iterator p = parent->dirty_old_rstat.lower_bound(last);
+      if (p == parent->dirty_old_rstat.end()) {
+	dout(20) << "  no dirty_old_rstat with last >= last " << last << dendl;
+	if (!parent->dirty_old_rstat.empty() && parent->dirty_old_rstat.rbegin()->first >= first) {
+	  dout(20) << "  last dirty_old_rstat ends at " << parent->dirty_old_rstat.rbegin()->first << dendl;
+	  first = parent->dirty_old_rstat.rbegin()->first+1;
+	}
+      } else {
+	// *p last is >= last
+	if (p->second.first <= last) {
+	  // *p intersects [first,last]
+	  if (p->second.first < first) {
+	    dout(10) << " splitting off left bit [" << p->second.first << "," << first-1 << "]" << dendl;
+	    parent->dirty_old_rstat[first-1] = p->second;
+	    p->second.first = first;
+	  }
+	  if (p->second.first > first)
+	    first = p->second.first;
+	  if (last < p->first) {
+	    dout(10) << " splitting off right bit [" << last+1 << "," << p->first << "]" << dendl;
+	    parent->dirty_old_rstat[last] = p->second;
+	    p->second.first = last+1;
+	  }
+	} else {
+	  // *p is to the _right_ of [first,last]
+	  p = parent->dirty_old_rstat.lower_bound(first);
+	  // new *p last is >= first
+	  if (p->second.first <= last &&  // new *p isn't also to the right, and
+	      p->first >= first) {        // it intersects our first bit,
+	    dout(10) << " staying to the right of [" << p->second.first << "," << p->first << "]..." << dendl;
+	    first = p->first+1;
+	  }
+	  dout(10) << " projecting to new dirty_old_rstat [" << first << "," << last << "]" << dendl;
+	}
+      }
+      dout(20) << " projecting to dirty_old_rstat [" << first << "," << last << "]" << dendl;
+      parent->dirty_old_rstat[last].first = first;
+      prstat = &parent->dirty_old_rstat[last].rstat;
+    }
+    
+    // apply
+    dout(20) << "  project to [" << first << "," << last << "] " << *prstat << dendl;
+    assert(last >= first);
+    prstat->add(delta);
+    inode.accounted_rstat = inode.rstat;
+    dout(20) << "      result [" << first << "," << last << "] " << *prstat << " " << *parent << dendl;
+
+    last = first-1;
+  }
+}
+
+void MDCache::project_rstat_frag_to_inode(nest_info_t& rstat, nest_info_t& accounted_rstat,
+					  snapid_t ofirst, snapid_t last, 
+					  CInode *pin, bool cow_head)
+{
+  dout(10) << "project_rstat_frag_to_inode [" << ofirst << "," << last << "]" << dendl;
+  dout(20) << "  frag           rstat " << rstat << dendl;
+  dout(20) << "  frag accounted_rstat " << accounted_rstat << dendl;
+  nest_info_t delta = rstat;
+  delta.sub(accounted_rstat);
+  dout(20) << "                 delta " << delta << dendl;
+
+  while (last >= ofirst) {
+    inode_t *pi;
+    snapid_t first;
+    if (last == pin->last) {
+      pi = pin->get_projected_inode();
+      first = MAX(ofirst, pin->first);
+      if (first > pin->first) {
+	old_inode_t& old = pin->cow_old_inode(first-1, cow_head);
+	dout(20) << "   cloned old_inode rstat is " << old.inode.rstat << dendl;
+      }
+    } else {
+      if (last >= pin->first) {
+	first = pin->first;
+	pin->cow_old_inode(last, cow_head);
+      } else {
+	// our life is easier here because old_inodes is not sparse
+	// (although it may not begin at snapid 1)
+	map::iterator p = pin->old_inodes.lower_bound(last);
+	if (p == pin->old_inodes.end()) {
+	  dout(10) << " no old_inode <= " << last << ", done." << dendl;
+	  break;
+	}
+	first = p->second.first;
+	if (first > last) {
+	  dout(10) << " oldest old_inode is [" << first << "," << p->first << "], done." << dendl;
+	  assert(p == pin->old_inodes.begin());
+	  break;
+	}
+	if (p->first > last) {
+	  dout(10) << " splitting right old_inode [" << first << "," << p->first << "] to ["
+		   << (last+1) << "," << p->first << "]" << dendl;
+	  pin->old_inodes[last] = p->second;
+	  p->second.first = last+1;
+	  pin->dirty_old_rstats.insert(p->first);
+	}
+      }
+      if (first < ofirst) {
+	dout(10) << " splitting left old_inode [" << first << "," << last << "] to ["
+		 << first << "," << ofirst-1 << "]" << dendl;
+	pin->old_inodes[ofirst-1] = pin->old_inodes[last];
+	pin->dirty_old_rstats.insert(ofirst-1);
+	pin->old_inodes[last].first = first = ofirst;
+      }
+      pi = &pin->old_inodes[last].inode;
+      pin->dirty_old_rstats.insert(last);
+    }
+    dout(20) << " projecting to [" << first << "," << last << "] " << pi->rstat << dendl;
+    pi->rstat.add(delta);
+    dout(20) << "        result [" << first << "," << last << "] " << pi->rstat << dendl;
+
+    if (pi->rstat.rbytes < 0 && pin->dirfragtree.is_leaf(frag_t()))
+      assert(!"negative rstat rbytes" == g_conf->mds_verify_scatter);
+
+    last = first-1;
+  }
+}
+
+
+/*
+ * NOTE: we _have_ to delay the scatter if we are called during a
+ * rejoin, because we can't twiddle locks between when the
+ * rejoin_(weak|strong) is received and when we send the rejoin_ack.
+ * normally, this isn't a problem: a recover mds doesn't twiddle locks
+ * (no requests), and a survivor acks immediately.  _except_ that
+ * during rejoin_(weak|strong) processing, we may complete a lock
+ * gather, and do a scatter_writebehind.. and we _can't_ twiddle the
+ * scatterlock state in that case or the lock states will get out of
+ * sync between the auth and replica.
+ *
+ * the simple solution is to never do the scatter here.  instead, put
+ * the scatterlock on a list if it isn't already wrlockable.  this is
+ * probably the best plan anyway, since we avoid too many
+ * scatters/locks under normal usage.
+ */
+/*
+ * some notes on dirlock/nestlock scatterlock semantics:
+ *
+ * the fragstat (dirlock) will never be updated without
+ * dirlock+nestlock wrlock held by the caller.
+ *
+ * the rstat (nestlock) _may_ get updated without a wrlock when nested
+ * data is pushed up the tree.  this could be changed with some
+ * restructuring here, but in its current form we ensure that the
+ * fragstat+rstat _always_ reflect an accurrate summation over the dir
+ * frag, which is nice.  and, we only need to track frags that need to
+ * be nudged (and not inodes with pending rstat changes that need to
+ * be pushed into the frag).  a consequence of this is that the
+ * accounted_rstat on scatterlock sync may not match our current
+ * rstat.  this is normal and expected.
+ */
+void MDCache::predirty_journal_parents(MutationRef mut, EMetaBlob *blob,
+				       CInode *in, CDir *parent,
+				       int flags, int linkunlink,
+				       snapid_t cfollows)
+{
+  bool primary_dn = flags & PREDIRTY_PRIMARY;
+  bool do_parent_mtime = flags & PREDIRTY_DIR;
+  bool shallow = flags & PREDIRTY_SHALLOW;
+
+  assert(mds->mdlog->entry_is_open());
+
+  // declare now?
+  if (mut->now == utime_t())
+    mut->now = ceph_clock_now(g_ceph_context);
+
+  if (in->is_base())
+    return;
+
+  dout(10) << "predirty_journal_parents"
+	   << (do_parent_mtime ? " do_parent_mtime":"")
+	   << " linkunlink=" <<  linkunlink
+	   << (primary_dn ? " primary_dn":" remote_dn")
+	   << (shallow ? " SHALLOW":"")
+	   << " follows " << cfollows
+	   << " " << *in << dendl;
+
+  if (!parent) {
+    assert(primary_dn);
+    parent = in->get_projected_parent_dn()->get_dir();
+  }
+
+  if (flags == 0 && linkunlink == 0) {
+    dout(10) << " no flags/linkunlink, just adding dir context to blob(s)" << dendl;
+    blob->add_dir_context(parent);
+    return;
+  }
+
+  // build list of inodes to wrlock, dirty, and update
+  list lsi;
+  CInode *cur = in;
+  CDentry *parentdn = NULL;
+  bool first = true;
+  while (parent) {
+    //assert(cur->is_auth() || !primary_dn);  // this breaks the rename auth twiddle hack
+    assert(parent->is_auth());
+    
+    // opportunistically adjust parent dirfrag
+    CInode *pin = parent->get_inode();
+
+    // inode -> dirfrag
+    mut->auth_pin(parent);
+    mut->add_projected_fnode(parent);
+
+    fnode_t *pf = parent->project_fnode();
+    pf->version = parent->pre_dirty();
+
+    if (do_parent_mtime || linkunlink) {
+      assert(mut->wrlocks.count(&pin->filelock));
+      assert(cfollows == CEPH_NOSNAP);
+      
+      // update stale fragstat?
+      parent->resync_accounted_fragstat();
+
+      if (do_parent_mtime) {
+	pf->fragstat.mtime = mut->now;
+	if (mut->now > pf->rstat.rctime) {
+	  dout(10) << "predirty_journal_parents updating mtime on " << *parent << dendl;
+	  pf->rstat.rctime = mut->now;
+	} else {
+	  dout(10) << "predirty_journal_parents updating mtime UNDERWATER on " << *parent << dendl;
+	}
+      }
+      if (linkunlink) {
+	dout(10) << "predirty_journal_parents updating size on " << *parent << dendl;
+	if (in->is_dir()) {
+	  pf->fragstat.nsubdirs += linkunlink;
+	  //pf->rstat.rsubdirs += linkunlink;
+	} else {
+ 	  pf->fragstat.nfiles += linkunlink;
+ 	  //pf->rstat.rfiles += linkunlink;
+	}
+      }
+    }
+
+    
+    // rstat
+    if (!primary_dn) {
+      // don't update parent this pass
+    } else if (!linkunlink && !(parent->inode->nestlock.can_wrlock(-1) &&
+			        parent->inode->versionlock.can_wrlock())) {
+      dout(20) << " unwritable parent nestlock " << parent->inode->nestlock
+	       << ", marking dirty rstat on " << *cur << dendl;
+      cur->mark_dirty_rstat();
+   } else {
+      // if we don't hold a wrlock reference on this nestlock, take one,
+      // because we are about to write into the dirfrag fnode and that needs
+      // to commit before the lock can cycle.
+     if (linkunlink) {
+       assert(parent->inode->nestlock.get_num_wrlocks() || mut->is_slave());
+     }
+
+      if (mut->wrlocks.count(&parent->inode->nestlock) == 0) {
+	dout(10) << " taking wrlock on " << parent->inode->nestlock << " on " << *parent->inode << dendl;
+	mds->locker->wrlock_force(&parent->inode->nestlock, mut);
+      }
+
+      // now we can project the inode rstat diff the dirfrag
+      SnapRealm *prealm = parent->inode->find_snaprealm();
+      
+      snapid_t follows = cfollows;
+      if (follows == CEPH_NOSNAP)
+	follows = prealm->get_newest_seq();
+      
+      snapid_t first = follows+1;
+
+      // first, if the frag is stale, bring it back in sync.
+      parent->resync_accounted_rstat();
+
+      // now push inode rstats into frag
+      project_rstat_inode_to_frag(cur, parent, first, linkunlink);
+      cur->clear_dirty_rstat();
+    }
+
+    bool stop = false;
+    if (!pin->is_auth() || (!mut->is_auth_pinned(pin) && !pin->can_auth_pin())) {
+      dout(10) << "predirty_journal_parents !auth or ambig or can't authpin on " << *pin << dendl;
+      stop = true;
+    }
+
+    // delay propagating until later?
+    if (!stop && !first &&
+	g_conf->mds_dirstat_min_interval > 0) {
+      if (pin->last_dirstat_prop.sec() > 0) {
+	double since_last_prop = mut->now - pin->last_dirstat_prop;
+	if (since_last_prop < g_conf->mds_dirstat_min_interval) {
+	  dout(10) << "predirty_journal_parents last prop " << since_last_prop
+		   << " < " << g_conf->mds_dirstat_min_interval
+		   << ", stopping" << dendl;
+	  stop = true;
+	} else {
+	  dout(10) << "predirty_journal_parents last prop " << since_last_prop << " ago, continuing" << dendl;
+	}
+      } else {
+	dout(10) << "predirty_journal_parents last prop never, stopping" << dendl;
+	stop = true;
+      }
+    }
+
+    // can cast only because i'm passing nowait=true in the sole user
+    MDRequestRef mdmut =
+      ceph::static_pointer_cast(mut);
+    if (!stop &&
+	mut->wrlocks.count(&pin->nestlock) == 0 &&
+	(!pin->versionlock.can_wrlock() ||                   // make sure we can take versionlock, too
+	 //true
+	 !mds->locker->wrlock_start(&pin->nestlock, mdmut, true)
+	 )) {  // ** do not initiate.. see above comment **
+      dout(10) << "predirty_journal_parents can't wrlock one of " << pin->versionlock << " or " << pin->nestlock
+	       << " on " << *pin << dendl;
+      stop = true;
+    }
+    if (stop) {
+      dout(10) << "predirty_journal_parents stop.  marking nestlock on " << *pin << dendl;
+      mds->locker->mark_updated_scatterlock(&pin->nestlock);
+      mut->ls->dirty_dirfrag_nest.push_back(&pin->item_dirty_dirfrag_nest);
+      mut->add_updated_lock(&pin->nestlock);
+      if (do_parent_mtime || linkunlink) {
+	mds->locker->mark_updated_scatterlock(&pin->filelock);
+	mut->ls->dirty_dirfrag_dir.push_back(&pin->item_dirty_dirfrag_dir);
+	mut->add_updated_lock(&pin->filelock);
+      }
+      break;
+    }
+    if (!mut->wrlocks.count(&pin->versionlock))
+      mds->locker->local_wrlock_grab(&pin->versionlock, mut);
+
+    assert(mut->wrlocks.count(&pin->nestlock) ||
+	   mut->is_slave());
+    
+    pin->last_dirstat_prop = mut->now;
+
+    // dirfrag -> diri
+    mut->auth_pin(pin);
+    mut->add_projected_inode(pin);
+    lsi.push_front(pin);
+
+    pin->pre_cow_old_inode();  // avoid cow mayhem!
+
+    inode_t *pi = pin->project_inode();
+    pi->version = pin->pre_dirty();
+
+    // dirstat
+    if (do_parent_mtime || linkunlink) {
+      dout(20) << "predirty_journal_parents add_delta " << pf->fragstat << dendl;
+      dout(20) << "predirty_journal_parents         - " << pf->accounted_fragstat << dendl;
+      bool touched_mtime = false;
+      pi->dirstat.add_delta(pf->fragstat, pf->accounted_fragstat, touched_mtime);
+      pf->accounted_fragstat = pf->fragstat;
+      if (touched_mtime)
+	pi->mtime = pi->ctime = pi->dirstat.mtime;
+      dout(20) << "predirty_journal_parents     gives " << pi->dirstat << " on " << *pin << dendl;
+
+      if (parent->get_frag() == frag_t()) { // i.e., we are the only frag
+	if (pi->dirstat.size() < 0)
+	  assert(!"negative dirstat size" == g_conf->mds_verify_scatter);
+	if (pi->dirstat.size() != pf->fragstat.size()) {
+	  mds->clog.error() << "unmatched fragstat size on single dirfrag "
+	     << parent->dirfrag() << ", inode has " << pi->dirstat
+	     << ", dirfrag has " << pf->fragstat << "\n";
+	  
+	  // trust the dirfrag for now
+	  pi->dirstat = pf->fragstat;
+
+	  assert(!"unmatched fragstat size" == g_conf->mds_verify_scatter);
+	}
+      }
+    }
+
+    /* 
+     * the rule here is to follow the _oldest_ parent with dirty rstat
+     * data.  if we don't propagate all data, we add ourselves to the
+     * nudge list.  that way all rstat data will (eventually) get
+     * pushed up the tree.
+     *
+     * actually, no.  for now, silently drop rstats for old parents.  we need 
+     * hard link backpointers to do the above properly.
+     */
+
+    // stop?
+    if (pin->is_base())
+      break;
+    parentdn = pin->get_projected_parent_dn();
+    assert(parentdn);
+
+    // rstat
+    if (primary_dn) {
+
+      dout(10) << "predirty_journal_parents frag->inode on " << *parent << dendl;
+
+      // first, if the frag is stale, bring it back in sync.
+      parent->resync_accounted_rstat();
+
+      for (map::iterator p = parent->dirty_old_rstat.begin();
+	   p != parent->dirty_old_rstat.end();
+	   ++p)
+	project_rstat_frag_to_inode(p->second.rstat, p->second.accounted_rstat, p->second.first, p->first, pin, true);//false);
+      parent->dirty_old_rstat.clear();
+      project_rstat_frag_to_inode(pf->rstat, pf->accounted_rstat, parent->first, CEPH_NOSNAP, pin, true);//false);
+
+      pf->accounted_rstat = pf->rstat;
+
+      if (parent->get_frag() == frag_t()) { // i.e., we are the only frag
+	if (pi->rstat.rbytes != pf->rstat.rbytes) { 
+	  mds->clog.error() << "unmatched rstat rbytes on single dirfrag "
+	      << parent->dirfrag() << ", inode has " << pi->rstat
+	      << ", dirfrag has " << pf->rstat << "\n";
+	  
+	  // trust the dirfrag for now
+	  pi->rstat = pf->rstat;
+
+	  assert(!"unmatched rstat rbytes" == g_conf->mds_verify_scatter);
+	}
+      }
+    }
+
+    parent->check_rstats();
+    // next parent!
+    cur = pin;
+    parent = parentdn->get_dir();
+    linkunlink = 0;
+    do_parent_mtime = false;
+    primary_dn = true;
+    first = false;
+  }
+
+  // now, stick it in the blob
+  assert(parent);
+  assert(parent->is_auth());
+  blob->add_dir_context(parent);
+  blob->add_dir(parent, true);
+  for (list::iterator p = lsi.begin();
+       p != lsi.end();
+       ++p) {
+    CInode *cur = *p;
+    journal_dirty_inode(mut.get(), blob, cur);
+  }
+ 
+}
+
+
+
+
+
+// ===================================
+// slave requests
+
+
+/*
+ * some handlers for master requests with slaves.  we need to make 
+ * sure slaves journal commits before we forget we mastered them and
+ * remove them from the uncommitted_masters map (used during recovery
+ * to commit|abort slaves).
+ */
+struct C_MDC_CommittedMaster : public Context {
+  MDCache *cache;
+  metareqid_t reqid;
+  C_MDC_CommittedMaster(MDCache *s, metareqid_t r) : cache(s), reqid(r) {}
+  void finish(int r) {
+    cache->_logged_master_commit(reqid);
+  }
+};
+
+void MDCache::log_master_commit(metareqid_t reqid)
+{
+  dout(10) << "log_master_commit " << reqid << dendl;
+  uncommitted_masters[reqid].committing = true;
+  mds->mdlog->start_submit_entry(new ECommitted(reqid), 
+				 new C_MDC_CommittedMaster(this, reqid));
+}
+
+void MDCache::_logged_master_commit(metareqid_t reqid)
+{
+  dout(10) << "_logged_master_commit " << reqid << dendl;
+  assert(uncommitted_masters.count(reqid));
+  uncommitted_masters[reqid].ls->uncommitted_masters.erase(reqid);
+  mds->queue_waiters(uncommitted_masters[reqid].waiters);
+  uncommitted_masters.erase(reqid);
+}
+
+// while active...
+
+void MDCache::committed_master_slave(metareqid_t r, int from)
+{
+  dout(10) << "committed_master_slave mds." << from << " on " << r << dendl;
+  assert(uncommitted_masters.count(r));
+  uncommitted_masters[r].slaves.erase(from);
+  if (!uncommitted_masters[r].recovering && uncommitted_masters[r].slaves.empty())
+    log_master_commit(r);
+}
+
+void MDCache::logged_master_update(metareqid_t reqid)
+{
+  dout(10) << "logged_master_update " << reqid << dendl;
+  assert(uncommitted_masters.count(reqid));
+  uncommitted_masters[reqid].safe = true;
+  if (pending_masters.count(reqid)) {
+    pending_masters.erase(reqid);
+    if (pending_masters.empty())
+      process_delayed_resolve();
+  }
+}
+
+/*
+ * Master may crash after receiving all slaves' commit acks, but before journalling
+ * the final commit. Slaves may crash after journalling the slave commit, but before
+ * sending commit ack to the master. Commit masters with no uncommitted slave when
+ * resolve finishes.
+ */
+void MDCache::finish_committed_masters()
+{
+  for (map::iterator p = uncommitted_masters.begin();
+       p != uncommitted_masters.end();
+       ++p) {
+    p->second.recovering = false;
+    if (!p->second.committing && p->second.slaves.empty()) {
+      dout(10) << "finish_committed_masters " << p->first << dendl;
+      log_master_commit(p->first);
+    }
+  }
+}
+
+/*
+ * at end of resolve... we must journal a commit|abort for all slave
+ * updates, before moving on.
+ * 
+ * this is so that the master can safely journal ECommitted on ops it
+ * masters when it reaches up:active (all other recovering nodes must
+ * complete resolve before that happens).
+ */
+struct C_MDC_SlaveCommit : public Context {
+  MDCache *cache;
+  int from;
+  metareqid_t reqid;
+  C_MDC_SlaveCommit(MDCache *c, int f, metareqid_t r) : cache(c), from(f), reqid(r) {}
+  void finish(int r) {
+    cache->_logged_slave_commit(from, reqid);
+  }
+};
+
+void MDCache::_logged_slave_commit(int from, metareqid_t reqid)
+{
+  dout(10) << "_logged_slave_commit from mds." << from << " " << reqid << dendl;
+  
+  // send a message
+  MMDSSlaveRequest *req = new MMDSSlaveRequest(reqid, 0, MMDSSlaveRequest::OP_COMMITTED);
+  mds->send_message_mds(req, from);
+}
+
+
+
+
+
+
+// ====================================================================
+// import map, recovery
+
+void MDCache::_move_subtree_map_bound(dirfrag_t df, dirfrag_t oldparent, dirfrag_t newparent,
+				      map >& subtrees)
+{
+  if (subtrees.count(oldparent)) {
+      vector& v = subtrees[oldparent];
+      dout(10) << " removing " << df << " from " << oldparent << " bounds " << v << dendl;
+      for (vector::iterator it = v.begin(); it != v.end(); ++it)
+	if (*it == df) {
+	  v.erase(it);
+	  break;
+	}
+    }
+  if (subtrees.count(newparent)) {
+    vector& v = subtrees[newparent];
+    dout(10) << " adding " << df << " to " << newparent << " bounds " << v << dendl;
+    v.push_back(df);
+  }
+}
+
+ESubtreeMap *MDCache::create_subtree_map() 
+{
+  dout(10) << "create_subtree_map " << num_subtrees() << " subtrees, " 
+	   << num_subtrees_fullauth() << " fullauth"
+	   << dendl;
+
+  show_subtrees();
+
+  ESubtreeMap *le = new ESubtreeMap();
+  mds->mdlog->start_entry(le);
+  
+  CDir *mydir = 0;
+  if (myin) {
+    mydir = myin->get_dirfrag(frag_t());
+  }
+
+  // include all auth subtrees, and their bounds.
+  // and a spanning tree to tie it to the root.
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *dir = p->first;
+
+    // journal subtree as "ours" if we are
+    //   me, -2
+    //   me, me
+    //   me, !me (may be importing and ambiguous!)
+
+    // so not
+    //   !me, *
+    if (dir->get_dir_auth().first != mds->whoami)
+      continue;
+
+    if (migrator->is_ambiguous_import(dir->dirfrag()) ||
+	my_ambiguous_imports.count(dir->dirfrag())) {
+      dout(15) << " ambig subtree " << *dir << dendl;
+      le->ambiguous_subtrees.insert(dir->dirfrag());
+    } else {
+      dout(15) << " subtree " << *dir << dendl;
+    }
+
+    le->subtrees[dir->dirfrag()].clear();
+    le->metablob.add_dir_context(dir, EMetaBlob::TO_ROOT);
+    le->metablob.add_dir(dir, false);
+
+    if (mydir == dir)
+      mydir = NULL;
+
+    // bounds
+    for (set::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      CDir *bound = *q;
+      dout(15) << " subtree bound " << *bound << dendl;
+      le->subtrees[dir->dirfrag()].push_back(bound->dirfrag());
+      le->metablob.add_dir_context(bound, EMetaBlob::TO_ROOT);
+      le->metablob.add_dir(bound, false);
+    }
+  }
+
+  // apply projected renames
+  for (map > >::iterator p = projected_subtree_renames.begin();
+       p != projected_subtree_renames.end();
+       ++p) {
+    for (list >::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+      CInode *diri = p->first;
+      CDir *olddir = q->first;
+      CDir *newdir = q->second;
+      dout(10) << " adjusting for projected rename of " << *diri << " to " << *newdir << dendl;
+
+      list dfls;
+      diri->get_dirfrags(dfls);
+      for (list::iterator p = dfls.begin(); p != dfls.end(); ++p) {
+	CDir *dir = *p;
+	dout(10) << "dirfrag " << dir->dirfrag() << " " << *dir << dendl;
+	CDir *oldparent = get_projected_subtree_root(olddir);
+	dout(10) << " old parent " << oldparent->dirfrag() << " " << *oldparent << dendl;
+	CDir *newparent = get_projected_subtree_root(newdir);
+	dout(10) << " new parent " << newparent->dirfrag() << " " << *newparent << dendl;
+
+	if (oldparent == newparent) {
+	  dout(10) << "parent unchanged for " << dir->dirfrag() << " at "
+		   << oldparent->dirfrag() << dendl;
+	  continue;
+	}
+
+	bool journal_dir = false;
+	if (dir->is_subtree_root()) {
+	  if (le->subtrees.count(newparent->dirfrag()) &&
+	      oldparent->get_dir_auth() != newparent->get_dir_auth())
+	    journal_dir = true;
+	  // children are fine.  change parent.
+	  _move_subtree_map_bound(dir->dirfrag(), oldparent->dirfrag(), newparent->dirfrag(),
+				  le->subtrees);
+	} else {
+	  // mid-subtree.
+
+	  if (oldparent->get_dir_auth() != newparent->get_dir_auth()) {
+	    dout(10) << " creating subtree for " << dir->dirfrag() << dendl;
+	    // if oldparent is auth, subtree is mine; include it.
+	    if (le->subtrees.count(oldparent->dirfrag())) {
+	      le->subtrees[dir->dirfrag()].clear();
+	      journal_dir = true;
+	    }
+	    // if newparent is auth, subtree is a new bound
+	    if (le->subtrees.count(newparent->dirfrag())) {
+	      le->subtrees[newparent->dirfrag()].push_back(dir->dirfrag());  // newparent is auth; new bound
+	      journal_dir = true;
+	    }
+	    newparent = dir;
+	  }
+	  
+	  // see if any old bounds move to the new parent.
+	  for (set::iterator p = subtrees[oldparent].begin();
+	       p != subtrees[oldparent].end();
+	       ++p) {
+	    CDir *bound = *p;
+	    if (dir->contains(bound->get_parent_dir()))
+	      _move_subtree_map_bound(bound->dirfrag(), oldparent->dirfrag(), newparent->dirfrag(),
+				      le->subtrees);
+	  }
+	}
+	if (journal_dir) {
+	  le->metablob.add_dir_context(dir, EMetaBlob::TO_ROOT);
+	  le->metablob.add_dir(dir, false);
+	}
+      }
+    }
+  }
+
+  // simplify the journaled map.  our in memory map may have more
+  // subtrees than needed due to migrations that are just getting
+  // started or just completing.  but on replay, the "live" map will
+  // be simple and we can do a straight comparison.
+  for (map >::iterator p = le->subtrees.begin(); p != le->subtrees.end(); ++p) {
+    if (le->ambiguous_subtrees.count(p->first))
+      continue;
+    unsigned i = 0;
+    while (i < p->second.size()) {
+      dirfrag_t b = p->second[i];
+      if (le->subtrees.count(b) &&
+	  le->ambiguous_subtrees.count(b) == 0) {
+	vector& bb = le->subtrees[b];
+	dout(10) << "simplify: " << p->first << " swallowing " << b << " with bounds " << bb << dendl;
+	for (vector::iterator r = bb.begin(); r != bb.end(); ++r)
+	  p->second.push_back(*r);
+	le->subtrees.erase(b);
+	p->second.erase(p->second.begin() + i);
+      } else {
+	++i;
+      }
+    }
+  }
+  dout(15) << " subtrees " << le->subtrees << dendl;
+  dout(15) << " ambiguous_subtrees " << le->ambiguous_subtrees << dendl;
+
+  if (mydir) {
+    // include my dir
+    le->metablob.add_dir_context(mydir, EMetaBlob::TO_ROOT);
+    le->metablob.add_dir(mydir, false);
+  }
+
+  //le->metablob.print(cout);
+  le->expire_pos = mds->mdlog->journaler->get_expire_pos();
+  return le;
+}
+
+
+void MDCache::resolve_start()
+{
+  dout(10) << "resolve_start" << dendl;
+
+  if (mds->mdsmap->get_root() != mds->whoami) {
+    // if we don't have the root dir, adjust it to UNKNOWN.  during
+    // resolve we want mds0 to explicit claim the portion of it that
+    // it owns, so that anything beyond its bounds get left as
+    // unknown.
+    CDir *rootdir = root->get_dirfrag(frag_t());
+    if (rootdir)
+      adjust_subtree_auth(rootdir, CDIR_AUTH_UNKNOWN);
+  }
+  resolve_gather = recovery_set;
+}
+
+void MDCache::send_resolves()
+{
+  send_slave_resolves();
+  if (!resolve_ack_gather.empty()) {
+    dout(10) << "send_resolves still waiting for resolve ack from ("
+	     << resolve_ack_gather << ")" << dendl;
+    return;
+  }
+  if (!need_resolve_rollback.empty()) {
+    dout(10) << "send_resolves still waiting for rollback to commit on ("
+	     << need_resolve_rollback << ")" << dendl;
+    return;
+  }
+  send_subtree_resolves();
+}
+
+void MDCache::send_slave_resolves()
+{
+  dout(10) << "send_slave_resolves" << dendl;
+
+  map resolves;
+
+  if (mds->is_resolve()) {
+    for (map >::iterator p = uncommitted_slave_updates.begin();
+	 p != uncommitted_slave_updates.end();
+	 ++p) {
+      resolves[p->first] = new MMDSResolve;
+      for (map::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q) {
+	dout(10) << " including uncommitted " << q->first << dendl;
+	resolves[p->first]->add_slave_request(q->first);
+      }
+    }
+  } else {
+    set resolve_set;
+    mds->mdsmap->get_mds_set(resolve_set, MDSMap::STATE_RESOLVE);
+    for (ceph::unordered_map::iterator p = active_requests.begin();
+	 p != active_requests.end();
+	 ++p) {
+      MDRequestRef& mdr = p->second;
+      if (!mdr->is_slave() || !mdr->slave_did_prepare())
+	continue;
+      int master = mdr->slave_to_mds;
+      if (resolve_set.count(master) || is_ambiguous_slave_update(p->first, master)) {
+	dout(10) << " including uncommitted " << *mdr << dendl;
+	if (!resolves.count(master))
+	  resolves[master] = new MMDSResolve;
+	if (mdr->has_more() && mdr->more()->is_inode_exporter) {
+	  // re-send cap exports
+	  CInode *in = mdr->more()->rename_inode;
+	  map cap_map;
+	  in->export_client_caps(cap_map);
+	  bufferlist bl;
+	  ::encode(in->ino(), bl);
+	  ::encode(cap_map, bl);
+	  resolves[master]->add_slave_request(p->first, bl);
+	} else {
+	  resolves[master]->add_slave_request(p->first);
+	}
+      }
+    }
+  }
+
+  for (map::iterator p = resolves.begin();
+       p != resolves.end();
+       ++p) {
+    dout(10) << "sending slave resolve to mds." << p->first << dendl;
+    mds->send_message_mds(p->second, p->first);
+    resolve_ack_gather.insert(p->first);
+  }
+}
+
+void MDCache::send_subtree_resolves()
+{
+  dout(10) << "send_subtree_resolves" << dendl;
+
+  if (migrator->is_exporting() || migrator->is_importing()) {
+    dout(7) << "send_subtree_resolves waiting, imports/exports still in progress" << dendl;
+    migrator->show_importing();
+    migrator->show_exporting();
+    resolves_pending = true;
+    return;  // not now
+  }
+
+  map resolves;
+  for (set::iterator p = recovery_set.begin();
+       p != recovery_set.end();
+       ++p) {
+    if (*p == mds->whoami)
+      continue;
+    if (mds->is_resolve() || mds->mdsmap->is_resolve(*p))
+      resolves[*p] = new MMDSResolve;
+  }
+
+  // known
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *dir = p->first;
+
+    // only our subtrees
+    if (dir->authority().first != mds->get_nodeid()) 
+      continue;
+
+    if (mds->is_resolve() && my_ambiguous_imports.count(dir->dirfrag()))
+      continue;  // we'll add it below
+    
+    if (migrator->is_ambiguous_import(dir->dirfrag())) {
+      // ambiguous (mid-import)
+      set bounds;
+      get_subtree_bounds(dir, bounds);
+      vector dfls;
+      for (set::iterator q = bounds.begin(); q != bounds.end(); ++q)
+	dfls.push_back((*q)->dirfrag());
+      for (map::iterator q = resolves.begin();
+	   q != resolves.end();
+	   ++q)
+	resolves[q->first]->add_ambiguous_import(dir->dirfrag(), dfls);
+      dout(10) << " ambig " << dir->dirfrag() << " " << dfls << dendl;
+    } else {
+      // not ambiguous.
+      for (map::iterator q = resolves.begin();
+	   q != resolves.end();
+	   ++q)
+	resolves[q->first]->add_subtree(dir->dirfrag());
+      // bounds too
+      vector dfls;
+      for (set::iterator q = subtrees[dir].begin();
+	   q != subtrees[dir].end();
+	   ++q) {
+	CDir *bound = *q;
+	dfls.push_back(bound->dirfrag());
+	for (map::iterator r = resolves.begin();
+	     r != resolves.end();
+	     ++r)
+	  resolves[r->first]->add_subtree_bound(dir->dirfrag(), bound->dirfrag());
+      }
+      dout(10) << " claim " << dir->dirfrag() << " " << dfls << dendl;
+    }
+  }
+
+  // ambiguous
+  for (map >::iterator p = my_ambiguous_imports.begin();
+       p != my_ambiguous_imports.end();
+       ++p) {
+    for (map::iterator q = resolves.begin();
+	 q != resolves.end();
+	 ++q)
+      resolves[q->first]->add_ambiguous_import(p->first, p->second);
+    dout(10) << " ambig " << p->first << " " << p->second << dendl;
+  }
+
+  // send
+  for (map::iterator p = resolves.begin();
+       p != resolves.end();
+       ++p) {
+    dout(10) << "sending subtee resolve to mds." << p->first << dendl;
+    mds->send_message_mds(p->second, p->first);
+  }
+  resolves_pending = false;
+}
+
+void MDCache::handle_mds_failure(int who)
+{
+  dout(7) << "handle_mds_failure mds." << who << dendl;
+  
+  // make note of recovery set
+  mds->mdsmap->get_recovery_mds_set(recovery_set);
+  recovery_set.erase(mds->get_nodeid());
+  dout(1) << "handle_mds_failure mds." << who << " : recovery peers are " << recovery_set << dendl;
+
+  resolve_gather.insert(who);
+  discard_delayed_resolve(who);
+  ambiguous_slave_updates.erase(who);
+
+  rejoin_gather.insert(who);
+  rejoin_sent.erase(who);        // i need to send another
+  rejoin_ack_gather.erase(who);  // i'll need/get another.
+
+  dout(10) << " resolve_gather " << resolve_gather << dendl;
+  dout(10) << " resolve_ack_gather " << resolve_ack_gather << dendl;
+  dout(10) << " rejoin_sent " << rejoin_sent << dendl;
+  dout(10) << " rejoin_gather " << rejoin_gather << dendl;
+  dout(10) << " rejoin_ack_gather " << rejoin_ack_gather << dendl;
+
+ 
+  // tell the migrator too.
+  migrator->handle_mds_failure_or_stop(who);
+
+  // clean up any requests slave to/from this node
+  list finish;
+  for (ceph::unordered_map::iterator p = active_requests.begin();
+       p != active_requests.end();
+       ++p) {
+    MDRequestRef& mdr = p->second;;
+    // slave to the failed node?
+    if (mdr->slave_to_mds == who) {
+      if (mdr->slave_did_prepare()) {
+	dout(10) << " slave request " << *mdr << " uncommitted, will resolve shortly" << dendl;
+	if (!mdr->more()->waiting_on_slave.empty()) {
+	  assert(mdr->more()->srcdn_auth_mds == mds->get_nodeid());
+	  // will rollback, no need to wait
+	  if (mdr->slave_request) {
+	    mdr->slave_request->put();
+	    mdr->slave_request = 0;
+	  }
+	  mdr->more()->waiting_on_slave.clear();
+	}
+      } else {
+	dout(10) << " slave request " << *mdr << " has no prepare, finishing up" << dendl;
+	if (mdr->slave_request)
+	  mdr->aborted = true;
+	else
+	  finish.push_back(mdr);
+      }
+    }
+
+    if (mdr->is_slave() && mdr->slave_did_prepare()) {
+      if (mdr->more()->waiting_on_slave.count(who)) {
+	assert(mdr->more()->srcdn_auth_mds == mds->get_nodeid());
+	dout(10) << " slave request " << *mdr << " no longer need rename notity ack from mds."
+		 << who << dendl;
+	mdr->more()->waiting_on_slave.erase(who);
+	if (mdr->more()->waiting_on_slave.empty() && mdr->slave_request)
+	  mds->queue_waiter(new C_MDS_RetryRequest(this, mdr));
+      }
+
+      if (mdr->more()->srcdn_auth_mds == who &&
+	  mds->mdsmap->is_clientreplay_or_active_or_stopping(mdr->slave_to_mds)) {
+	// rename srcdn's auth mds failed, resolve even I'm a survivor.
+	dout(10) << " slave request " << *mdr << " uncommitted, will resolve shortly" << dendl;
+	add_ambiguous_slave_update(p->first, mdr->slave_to_mds);
+      }
+    }
+    
+    // failed node is slave?
+    if (mdr->is_master() && !mdr->committing) {
+      if (mdr->more()->srcdn_auth_mds == who) {
+	dout(10) << " master request " << *mdr << " waiting for rename srcdn's auth mds."
+		 << who << " to recover" << dendl;
+	assert(mdr->more()->witnessed.count(who) == 0);
+	if (mdr->more()->is_ambiguous_auth)
+	  mdr->clear_ambiguous_auth();
+	// rename srcdn's auth mds failed, all witnesses will rollback
+	mdr->more()->witnessed.clear();
+	pending_masters.erase(p->first);
+      }
+
+      if (mdr->more()->witnessed.count(who)) {
+	int srcdn_auth = mdr->more()->srcdn_auth_mds;
+	if (srcdn_auth >= 0 && mdr->more()->waiting_on_slave.count(srcdn_auth)) {
+	  dout(10) << " master request " << *mdr << " waiting for rename srcdn's auth mds."
+		   << mdr->more()->srcdn_auth_mds << " to reply" << dendl;
+	  // waiting for the slave (rename srcdn's auth mds), delay sending resolve ack
+	  // until either the request is committing or the slave also fails.
+	  assert(mdr->more()->waiting_on_slave.size() == 1);
+	  pending_masters.insert(p->first);
+	} else {
+	  dout(10) << " master request " << *mdr << " no longer witnessed by slave mds."
+		   << who << " to recover" << dendl;
+	  if (srcdn_auth >= 0)
+	    assert(mdr->more()->witnessed.count(srcdn_auth) == 0);
+
+	  // discard this peer's prepare (if any)
+	  mdr->more()->witnessed.erase(who);
+	}
+      }
+      
+      if (mdr->more()->waiting_on_slave.count(who)) {
+	dout(10) << " master request " << *mdr << " waiting for slave mds." << who
+		 << " to recover" << dendl;
+	// retry request when peer recovers
+	mdr->more()->waiting_on_slave.erase(who);
+	if (mdr->more()->waiting_on_slave.empty())
+	  mds->wait_for_active_peer(who, new C_MDS_RetryRequest(this, mdr));
+      }
+
+      if (mdr->locking && mdr->locking_target_mds == who)
+	mdr->finish_locking(mdr->locking);
+    }
+  }
+
+  for (map::iterator p = uncommitted_masters.begin();
+       p != uncommitted_masters.end();
+       ++p) {
+    // The failed MDS may have already committed the slave update
+    if (p->second.slaves.count(who)) {
+      p->second.recovering = true;
+      p->second.slaves.erase(who);
+    }
+  }
+
+  while (!finish.empty()) {
+    dout(10) << "cleaning up slave request " << *finish.front() << dendl;
+    request_finish(finish.front());
+    finish.pop_front();
+  }
+
+  kick_find_ino_peers(who);
+  kick_open_ino_peers(who);
+
+  show_subtrees();  
+}
+
+/*
+ * handle_mds_recovery - called on another node's transition 
+ * from resolve -> active.
+ */
+void MDCache::handle_mds_recovery(int who)
+{
+  dout(7) << "handle_mds_recovery mds." << who << dendl;
+
+  // exclude all discover waiters. kick_discovers() will do the job
+  static const uint64_t i_mask = CInode::WAIT_ANY_MASK & ~CInode::WAIT_DIR;
+  static const uint64_t d_mask = CDir::WAIT_ANY_MASK & ~CDir::WAIT_DENTRY;
+
+  list waiters;
+
+  // wake up any waiters in their subtrees
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *dir = p->first;
+
+    if (dir->authority().first != who ||
+	dir->authority().second == mds->whoami)
+      continue;
+    assert(!dir->is_auth());
+   
+    // wake any waiters
+    list q;
+    q.push_back(dir);
+
+    while (!q.empty()) {
+      CDir *d = q.front();
+      q.pop_front();
+      d->take_waiting(d_mask, waiters);
+
+      // inode waiters too
+      for (CDir::map_t::iterator p = d->items.begin();
+	   p != d->items.end();
+	   ++p) {
+	CDentry *dn = p->second;
+	CDentry::linkage_t *dnl = dn->get_linkage();
+	if (dnl->is_primary()) {
+	  dnl->get_inode()->take_waiting(i_mask, waiters);
+	  
+	  // recurse?
+	  list ls;
+	  dnl->get_inode()->get_dirfrags(ls);
+	  for (list::iterator p = ls.begin();
+	       p != ls.end();
+	       ++p) {
+	    CDir *subdir = *p;
+	    if (!subdir->is_subtree_root())
+	      q.push_back(subdir);
+	  }
+	}
+      }
+    }
+  }
+
+  kick_discovers(who);
+  kick_open_ino_peers(who);
+  kick_find_ino_peers(who);
+
+  // queue them up.
+  mds->queue_waiters(waiters);
+}
+
+void MDCache::set_recovery_set(set& s) 
+{
+  dout(7) << "set_recovery_set " << s << dendl;
+  recovery_set = s;
+}
+
+
+/*
+ * during resolve state, we share resolves to determine who
+ * is authoritative for which trees.  we expect to get an resolve
+ * from _everyone_ in the recovery_set (the mds cluster at the time of
+ * the first failure).
+ *
+ * This functions puts the passed message before returning
+ */
+void MDCache::handle_resolve(MMDSResolve *m)
+{
+  dout(7) << "handle_resolve from " << m->get_source() << dendl;
+  int from = m->get_source().num();
+
+  if (mds->get_state() < MDSMap::STATE_RESOLVE) {
+    if (mds->get_want_state() == CEPH_MDS_STATE_RESOLVE) {
+      mds->wait_for_resolve(new C_MDS_RetryMessage(mds, m));
+      return;
+    }
+    // wait until we reach the resolve stage!
+    m->put();
+    return;
+  }
+
+  discard_delayed_resolve(from);
+
+  // ambiguous slave requests?
+  if (!m->slave_requests.empty()) {
+    for (map::iterator p = m->slave_requests.begin();
+	 p != m->slave_requests.end();
+	 ++p) {
+      if (uncommitted_masters.count(p->first) && !uncommitted_masters[p->first].safe)
+	pending_masters.insert(p->first);
+    }
+
+    if (!pending_masters.empty()) {
+      dout(10) << " still have pending updates, delay processing slave resolve" << dendl;
+      delayed_resolve[from] = m;
+      return;
+    }
+
+    MMDSResolveAck *ack = new MMDSResolveAck;
+    for (map::iterator p = m->slave_requests.begin();
+	 p != m->slave_requests.end();
+	 ++p) {
+      if (uncommitted_masters.count(p->first)) {  //mds->sessionmap.have_completed_request(p->first)) {
+	// COMMIT
+	dout(10) << " ambiguous slave request " << *p << " will COMMIT" << dendl;
+	ack->add_commit(p->first);
+	uncommitted_masters[p->first].slaves.insert(from);   // wait for slave OP_COMMITTED before we log ECommitted
+
+	if (p->second.length() > 0) {
+	  // slave wants to export caps (rename)
+	  assert(mds->is_resolve());
+
+	  inodeno_t ino;
+	  map cap_exports;
+	  bufferlist::iterator q = p->second.begin();
+	  ::decode(ino, q);
+	  ::decode(cap_exports, q);
+
+	  assert(get_inode(ino));
+
+	  for (map::iterator q = cap_exports.begin();
+	      q != cap_exports.end();
+	      ++q) {
+	    Capability::Import& im = rejoin_imported_caps[from][ino][q->first];
+	    im.cap_id = ++last_cap_id; // assign a new cap ID
+	    im.issue_seq = 1;
+	    im.mseq = q->second.mseq;
+	  }
+
+	  // will process these caps in rejoin stage
+	  rejoin_slave_exports[ino].first = from;
+	  rejoin_slave_exports[ino].second.swap(cap_exports);
+
+	  // send information of imported caps back to slave
+	  ::encode(rejoin_imported_caps[from][ino], ack->commit[p->first]);
+	}
+      } else {
+	// ABORT
+	dout(10) << " ambiguous slave request " << *p << " will ABORT" << dendl;
+	ack->add_abort(p->first);
+      }
+    }
+    mds->send_message(ack, m->get_connection());
+    m->put();
+    return;
+  }
+
+  if (!resolve_ack_gather.empty() || !need_resolve_rollback.empty()) {
+    dout(10) << "delay processing subtree resolve" << dendl;
+    delayed_resolve[from] = m;
+    return;
+  }
+
+  // am i a surviving ambiguous importer?
+  if (mds->is_clientreplay() || mds->is_active() || mds->is_stopping()) {
+    // check for any import success/failure (from this node)
+    map >::iterator p = my_ambiguous_imports.begin();
+    while (p != my_ambiguous_imports.end()) {
+      map >::iterator next = p;
+      ++next;
+      CDir *dir = get_dirfrag(p->first);
+      assert(dir);
+      dout(10) << "checking ambiguous import " << *dir << dendl;
+      if (migrator->is_importing(dir->dirfrag()) &&
+	  migrator->get_import_peer(dir->dirfrag()) == from) {
+	assert(migrator->get_import_state(dir->dirfrag()) == Migrator::IMPORT_ACKING);
+	
+	// check if sender claims the subtree
+	bool claimed_by_sender = false;
+	for (map >::iterator q = m->subtrees.begin();
+	     q != m->subtrees.end();
+	     ++q) {
+	  // an ambiguous import won't race with a refragmentation; it's appropriate to force here.
+	  CDir *base = get_force_dirfrag(q->first);
+	  if (!base || !base->contains(dir)) 
+	    continue;  // base not dir or an ancestor of dir, clearly doesn't claim dir.
+
+	  bool inside = true;
+	  set bounds;
+	  get_force_dirfrag_bound_set(q->second, bounds);
+	  for (set::iterator p = bounds.begin(); p != bounds.end(); ++p) {
+	    CDir *bound = *p;
+	    if (bound->contains(dir)) {
+	      inside = false;  // nope, bound is dir or parent of dir, not inside.
+	      break;
+	    }
+	  }
+	  if (inside)
+	    claimed_by_sender = true;
+	}
+
+	if (claimed_by_sender) {
+	  dout(7) << "ambiguous import failed on " << *dir << dendl;
+	  migrator->import_reverse(dir);
+	} else {
+	  dout(7) << "ambiguous import succeeded on " << *dir << dendl;
+	  migrator->import_finish(dir, true);
+	}
+	my_ambiguous_imports.erase(p);  // no longer ambiguous.
+      }
+      p = next;
+    }
+  }    
+
+  // update my dir_auth values
+  //   need to do this on recoverying nodes _and_ bystanders (to resolve ambiguous
+  //   migrations between other nodes)
+  for (map >::iterator pi = m->subtrees.begin();
+       pi != m->subtrees.end();
+       ++pi) {
+    dout(10) << "peer claims " << pi->first << " bounds " << pi->second << dendl;
+    CDir *dir = get_force_dirfrag(pi->first);
+    if (!dir)
+      continue;
+    adjust_bounded_subtree_auth(dir, pi->second, from);
+    try_subtree_merge(dir);
+  }
+
+  show_subtrees();
+
+  // note ambiguous imports too
+  for (map >::iterator pi = m->ambiguous_imports.begin();
+       pi != m->ambiguous_imports.end();
+       ++pi) {
+    dout(10) << "noting ambiguous import on " << pi->first << " bounds " << pi->second << dendl;
+    other_ambiguous_imports[from][pi->first].swap( pi->second );
+  }
+  
+  // did i get them all?
+  resolve_gather.erase(from);
+  
+  maybe_resolve_finish();
+
+  m->put();
+}
+
+void MDCache::process_delayed_resolve()
+{
+  dout(10) << "process_delayed_resolve" << dendl;
+  map tmp;
+  tmp.swap(delayed_resolve);
+  for (map::iterator p = tmp.begin(); p != tmp.end(); ++p)
+    handle_resolve(p->second);
+}
+
+void MDCache::discard_delayed_resolve(int who)
+{
+  if (delayed_resolve.count(who)) {
+      delayed_resolve[who]->put();
+      delayed_resolve.erase(who);
+  }
+}
+
+void MDCache::maybe_resolve_finish()
+{
+  assert(resolve_ack_gather.empty());
+  assert(need_resolve_rollback.empty());
+
+  if (!resolve_gather.empty()) {
+    dout(10) << "maybe_resolve_finish still waiting for resolves ("
+	     << resolve_gather << ")" << dendl;
+    return;
+  }
+
+  dout(10) << "maybe_resolve_finish got all resolves+resolve_acks, done." << dendl;
+  disambiguate_imports();
+  finish_committed_masters();
+  if (mds->is_resolve()) {
+    trim_unlinked_inodes();
+    recalc_auth_bits();
+    mds->resolve_done();
+  } else {
+    maybe_send_pending_rejoins();
+  }
+}
+
+/* This functions puts the passed message before returning */
+void MDCache::handle_resolve_ack(MMDSResolveAck *ack)
+{
+  dout(10) << "handle_resolve_ack " << *ack << " from " << ack->get_source() << dendl;
+  int from = ack->get_source().num();
+
+  if (!resolve_ack_gather.count(from) ||
+      mds->mdsmap->get_state(from) < MDSMap::STATE_RESOLVE) {
+    ack->put();
+    return;
+  }
+
+  if (ambiguous_slave_updates.count(from)) {
+    assert(mds->mdsmap->is_clientreplay_or_active_or_stopping(from));
+    assert(mds->is_clientreplay() || mds->is_active() || mds->is_stopping());
+  }
+
+  for (map::iterator p = ack->commit.begin();
+       p != ack->commit.end();
+       ++p) {
+    dout(10) << " commit on slave " << p->first << dendl;
+    
+    if (ambiguous_slave_updates.count(from)) {
+      remove_ambiguous_slave_update(p->first, from);
+      continue;
+    }
+
+    if (mds->is_resolve()) {
+      // replay
+      MDSlaveUpdate *su = get_uncommitted_slave_update(p->first, from);
+      assert(su);
+
+      // log commit
+      mds->mdlog->start_submit_entry(new ESlaveUpdate(mds->mdlog, "unknown", p->first, from,
+						      ESlaveUpdate::OP_COMMIT, su->origop));
+      mds->mdlog->wait_for_safe(new C_MDC_SlaveCommit(this, from, p->first));
+      mds->mdlog->flush();
+
+      finish_uncommitted_slave_update(p->first, from);
+    } else {
+      MDRequestRef mdr = request_get(p->first);
+      // information about master imported caps
+      if (p->second.length() > 0)
+	mdr->more()->inode_import.claim(p->second);
+
+      assert(mdr->slave_request == 0);  // shouldn't be doing anything!
+      request_finish(mdr);
+    }
+  }
+
+  for (vector::iterator p = ack->abort.begin();
+       p != ack->abort.end();
+       ++p) {
+    dout(10) << " abort on slave " << *p << dendl;
+
+    if (mds->is_resolve()) {
+      MDSlaveUpdate *su = get_uncommitted_slave_update(*p, from);
+      assert(su);
+
+      // perform rollback (and journal a rollback entry)
+      // note: this will hold up the resolve a bit, until the rollback entries journal.
+      MDRequestRef null_ref;
+      switch (su->origop) {
+      case ESlaveUpdate::LINK:
+	mds->server->do_link_rollback(su->rollback, from, null_ref);
+	break;
+      case ESlaveUpdate::RENAME:
+	mds->server->do_rename_rollback(su->rollback, from, null_ref);
+	break;
+      case ESlaveUpdate::RMDIR:
+	mds->server->do_rmdir_rollback(su->rollback, from, null_ref);
+	break;
+      default:
+	assert(0);
+      }
+    } else {
+      MDRequestRef mdr = request_get(*p);
+      mdr->aborted = true;
+      if (mdr->slave_request) {
+	if (mdr->more()->slave_commit) // journaling slave prepare ?
+	  add_rollback(*p, from);
+      } else {
+	request_finish(mdr);
+      }
+    }
+  }
+
+  if (!ambiguous_slave_updates.count(from))
+    resolve_ack_gather.erase(from);
+  if (resolve_ack_gather.empty() && need_resolve_rollback.empty()) {
+    send_subtree_resolves();
+    process_delayed_resolve();
+  }
+
+  ack->put();
+}
+
+void MDCache::add_uncommitted_slave_update(metareqid_t reqid, int master, MDSlaveUpdate *su)
+{
+  assert(uncommitted_slave_updates[master].count(reqid) == 0);
+  uncommitted_slave_updates[master][reqid] = su;
+  for(set::iterator p = su->olddirs.begin(); p != su->olddirs.end(); ++p)
+    uncommitted_slave_rename_olddir[*p]++;
+  for(set::iterator p = su->unlinked.begin(); p != su->unlinked.end(); ++p)
+    uncommitted_slave_unlink[*p]++;
+}
+
+void MDCache::finish_uncommitted_slave_update(metareqid_t reqid, int master)
+{
+  assert(uncommitted_slave_updates[master].count(reqid));
+  MDSlaveUpdate* su = uncommitted_slave_updates[master][reqid];
+
+  uncommitted_slave_updates[master].erase(reqid);
+  if (uncommitted_slave_updates[master].empty())
+    uncommitted_slave_updates.erase(master);
+  // discard the non-auth subtree we renamed out of
+  for(set::iterator p = su->olddirs.begin(); p != su->olddirs.end(); ++p) {
+    CInode *diri = *p;
+    map::iterator it = uncommitted_slave_rename_olddir.find(diri);
+    assert(it != uncommitted_slave_rename_olddir.end());
+    it->second--;
+    if (it->second == 0) {
+      uncommitted_slave_rename_olddir.erase(it);
+      list ls;
+      diri->get_dirfrags(ls);
+      for (list::iterator q = ls.begin(); q != ls.end(); ++q) {
+	CDir *root = get_subtree_root(*q);
+	if (root->get_dir_auth() == CDIR_AUTH_UNDEF) {
+	  try_trim_non_auth_subtree(root);
+	  if (*q != root)
+	    break;
+	}
+      }
+    } else
+      assert(it->second > 0);
+  }
+  // removed the inodes that were unlinked by slave update
+  for(set::iterator p = su->unlinked.begin(); p != su->unlinked.end(); ++p) {
+    CInode *in = *p;
+    map::iterator it = uncommitted_slave_unlink.find(in);
+    assert(it != uncommitted_slave_unlink.end());
+    it->second--;
+    if (it->second == 0) {
+      uncommitted_slave_unlink.erase(it);
+      if (!in->get_projected_parent_dn())
+	mds->mdcache->remove_inode_recursive(in);
+    } else
+      assert(it->second > 0);
+  }
+  delete su;
+}
+
+MDSlaveUpdate* MDCache::get_uncommitted_slave_update(metareqid_t reqid, int master)
+{
+
+  MDSlaveUpdate* su = NULL;
+  if (uncommitted_slave_updates.count(master) &&
+      uncommitted_slave_updates[master].count(reqid)) {
+    su = uncommitted_slave_updates[master][reqid];
+    assert(su);
+  }
+  return su;
+}
+
+void MDCache::finish_rollback(metareqid_t reqid) {
+  assert(need_resolve_rollback.count(reqid));
+  if (mds->is_resolve())
+    finish_uncommitted_slave_update(reqid, need_resolve_rollback[reqid]);
+  need_resolve_rollback.erase(reqid);
+  if (resolve_ack_gather.empty() && need_resolve_rollback.empty()) {
+    send_subtree_resolves();
+    process_delayed_resolve();
+  }
+}
+
+void MDCache::disambiguate_imports()
+{
+  dout(10) << "disambiguate_imports" << dendl;
+
+  // other nodes' ambiguous imports
+  for (map > >::iterator p = other_ambiguous_imports.begin();
+       p != other_ambiguous_imports.end();
+       ++p) {
+    int who = p->first;
+    dout(10) << "ambiguous imports for mds." << who << dendl;
+
+    for (map >::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      dout(10) << " ambiguous import " << q->first << " bounds " << q->second << dendl;
+      // an ambiguous import will not race with a refragmentation; it's appropriate to force here.
+      CDir *dir = get_force_dirfrag(q->first);
+      if (!dir) continue;
+      
+      if (dir->is_ambiguous_auth() ||	// works for me_ambig or if i am a surviving bystander
+	  dir->authority() == CDIR_AUTH_UNDEF) { // resolving
+	dout(10) << "  mds." << who << " did import " << *dir << dendl;
+	adjust_bounded_subtree_auth(dir, q->second, who);
+	try_subtree_merge(dir);
+      } else {
+	dout(10) << "  mds." << who << " did not import " << *dir << dendl;
+      }
+    }
+  }
+  other_ambiguous_imports.clear();
+
+  // my ambiguous imports
+  pair me_ambig(mds->whoami, mds->whoami);
+  while (!my_ambiguous_imports.empty()) {
+    map >::iterator q = my_ambiguous_imports.begin();
+
+    CDir *dir = get_dirfrag(q->first);
+    if (!dir) continue;
+    
+    if (dir->authority() != me_ambig) {
+      dout(10) << "ambiguous import auth known, must not be me " << *dir << dendl;
+      cancel_ambiguous_import(dir);
+
+      // subtree may have been swallowed by another node claiming dir
+      // as their own.
+      CDir *root = get_subtree_root(dir);
+      if (root != dir)
+	dout(10) << "  subtree root is " << *root << dendl;
+      assert(root->dir_auth.first != mds->whoami);  // no us!
+      try_trim_non_auth_subtree(root);
+
+      mds->mdlog->start_submit_entry(new EImportFinish(dir, false));
+    } else {
+      dout(10) << "ambiguous import auth unclaimed, must be me " << *dir << dendl;
+      finish_ambiguous_import(q->first);
+      mds->mdlog->start_submit_entry(new EImportFinish(dir, true));
+    }
+  }
+  assert(my_ambiguous_imports.empty());
+  mds->mdlog->flush();
+
+  if (mds->is_resolve()) {
+    // verify all my subtrees are unambiguous!
+    for (map >::iterator p = subtrees.begin();
+	 p != subtrees.end();
+	 ++p) {
+      CDir *dir = p->first;
+      if (dir->is_ambiguous_dir_auth()) {
+	dout(0) << "disambiguate_imports uh oh, dir_auth is still ambiguous for " << *dir << dendl;
+	show_subtrees();
+      }
+      assert(!dir->is_ambiguous_dir_auth());
+    }
+  }
+
+  show_subtrees();
+}
+
+
+void MDCache::add_ambiguous_import(dirfrag_t base, const vector& bounds) 
+{
+  assert(my_ambiguous_imports.count(base) == 0);
+  my_ambiguous_imports[base] = bounds;
+}
+
+
+void MDCache::add_ambiguous_import(CDir *base, const set& bounds)
+{
+  // make a list
+  vector binos;
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) 
+    binos.push_back((*p)->dirfrag());
+  
+  // note: this can get called twice if the exporter fails during recovery
+  if (my_ambiguous_imports.count(base->dirfrag()))
+    my_ambiguous_imports.erase(base->dirfrag());
+
+  add_ambiguous_import(base->dirfrag(), binos);
+}
+
+void MDCache::cancel_ambiguous_import(CDir *dir)
+{
+  dirfrag_t df = dir->dirfrag();
+  assert(my_ambiguous_imports.count(df));
+  dout(10) << "cancel_ambiguous_import " << df
+	   << " bounds " << my_ambiguous_imports[df]
+	   << " " << *dir
+	   << dendl;
+  my_ambiguous_imports.erase(df);
+}
+
+void MDCache::finish_ambiguous_import(dirfrag_t df)
+{
+  assert(my_ambiguous_imports.count(df));
+  vector bounds;
+  bounds.swap(my_ambiguous_imports[df]);
+  my_ambiguous_imports.erase(df);
+  
+  dout(10) << "finish_ambiguous_import " << df
+	   << " bounds " << bounds
+	   << dendl;
+  CDir *dir = get_dirfrag(df);
+  assert(dir);
+  
+  // adjust dir_auth, import maps
+  adjust_bounded_subtree_auth(dir, bounds, mds->get_nodeid());
+  try_subtree_merge(dir);
+}
+
+void MDCache::remove_inode_recursive(CInode *in)
+{
+  dout(10) << "remove_inode_recursive " << *in << dendl;
+  list ls;
+  in->get_dirfrags(ls);
+  list::iterator p = ls.begin();
+  while (p != ls.end()) {
+    CDir *subdir = *p++;
+
+    dout(10) << " removing dirfrag " << subdir << dendl;
+    CDir::map_t::iterator q = subdir->items.begin();
+    while (q != subdir->items.end()) {
+      CDentry *dn = q->second;
+      ++q;
+      CDentry::linkage_t *dnl = dn->get_linkage();
+      if (dnl->is_primary()) {
+	CInode *tin = dnl->get_inode();
+	subdir->unlink_inode(dn);
+	remove_inode_recursive(tin);
+      }
+      subdir->remove_dentry(dn);
+    }
+    
+    if (subdir->is_subtree_root()) 
+      remove_subtree(subdir);
+    in->close_dirfrag(subdir->dirfrag().frag);
+  }
+  remove_inode(in);
+}
+
+void MDCache::trim_unlinked_inodes()
+{
+  dout(7) << "trim_unlinked_inodes" << dendl;
+  list q;
+  for (ceph::unordered_map::iterator p = inode_map.begin();
+       p != inode_map.end();
+       ++p) {
+    CInode *in = p->second;
+    if (in->get_parent_dn() == NULL && !in->is_base()) {
+      dout(7) << " will trim from " << *in << dendl;
+      q.push_back(in);
+    }
+  }
+  for (list::iterator p = q.begin(); p != q.end(); ++p)
+    remove_inode_recursive(*p);
+}
+
+/** recalc_auth_bits()
+ * once subtree auth is disambiguated, we need to adjust all the 
+ * auth and dirty bits in our cache before moving on.
+ */
+void MDCache::recalc_auth_bits()
+{
+  dout(7) << "recalc_auth_bits" << dendl;
+
+  if (root) {
+    root->inode_auth.first = mds->mdsmap->get_root();
+    if (mds->whoami != root->inode_auth.first) {
+      root->state_clear(CInode::STATE_AUTH);
+      root->state_set(CInode::STATE_REJOINING);
+    }
+  }
+
+  set subtree_inodes;
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    if (p->first->dir_auth.first == mds->get_nodeid())
+      subtree_inodes.insert(p->first->inode);
+  }
+
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+
+    CInode *inode = p->first->get_inode();
+    if (inode->is_mdsdir() && inode->ino() != MDS_INO_MDSDIR(mds->get_nodeid())) {
+      inode->state_clear(CInode::STATE_AUTH);
+      inode->state_set(CInode::STATE_REJOINING);
+    }
+
+    list dfq;  // dirfrag queue
+    dfq.push_back(p->first);
+
+    bool auth = p->first->authority().first == mds->get_nodeid();
+    dout(10) << " subtree auth=" << auth << " for " << *p->first << dendl;
+
+    while (!dfq.empty()) {
+      CDir *dir = dfq.front();
+      dfq.pop_front();
+
+      // dir
+      if (auth) {
+	dir->state_set(CDir::STATE_AUTH);
+      } else {
+	// close empty non-auth dirfrag
+	if (!dir->is_subtree_root() && dir->get_num_any() == 0) {
+	  dir->inode->close_dirfrag(dir->get_frag());
+	  continue;
+	}
+	dir->state_set(CDir::STATE_REJOINING);
+	dir->state_clear(CDir::STATE_AUTH);
+	dir->state_clear(CDir::STATE_COMPLETE);
+	if (dir->is_dirty()) 
+	  dir->mark_clean();
+      }
+
+      // dentries in this dir
+      for (CDir::map_t::iterator q = dir->items.begin();
+	   q != dir->items.end();
+	   ++q) {
+	// dn
+	CDentry *dn = q->second;
+	CDentry::linkage_t *dnl = dn->get_linkage();
+	if (auth)
+	  dn->state_set(CDentry::STATE_AUTH);
+	else {
+	  dn->state_set(CDentry::STATE_REJOINING);
+	  dn->state_clear(CDentry::STATE_AUTH);
+	  if (dn->is_dirty()) 
+	    dn->mark_clean();
+	}
+
+	if (dnl->is_primary()) {
+	  // inode
+	  if (auth) 
+	    dnl->get_inode()->state_set(CInode::STATE_AUTH);
+	  else {
+	    dnl->get_inode()->state_set(CInode::STATE_REJOINING);
+	    dnl->get_inode()->state_clear(CInode::STATE_AUTH);
+	    if (dnl->get_inode()->is_dirty())
+	      dnl->get_inode()->mark_clean();
+	    if (dnl->get_inode()->is_dirty_parent())
+	      dnl->get_inode()->clear_dirty_parent();
+	    // avoid touching scatterlocks for our subtree roots!
+	    if (subtree_inodes.count(dnl->get_inode()) == 0)
+	      dnl->get_inode()->clear_scatter_dirty();
+	  }
+
+	  // recurse?
+	  if (dnl->get_inode()->is_dir()) 
+	    dnl->get_inode()->get_nested_dirfrags(dfq);
+	}
+      }
+    }
+  }
+  
+  show_subtrees();
+  show_cache();
+}
+
+
+
+// ===========================================================================
+// REJOIN
+
+/*
+ * notes on scatterlock recovery:
+ *
+ * - recovering inode replica sends scatterlock data for any subtree
+ *   roots (the only ones that are possibly dirty).
+ *
+ * - surviving auth incorporates any provided scatterlock data.  any
+ *   pending gathers are then finished, as with the other lock types.
+ *
+ * that takes care of surviving auth + (recovering replica)*.
+ *
+ * - surviving replica sends strong_inode, which includes current
+ *   scatterlock state, AND any dirty scatterlock data.  this
+ *   provides the recovering auth with everything it might need.
+ * 
+ * - recovering auth must pick initial scatterlock state based on
+ *   (weak|strong) rejoins.
+ *   - always assimilate scatterlock data (it can't hurt)
+ *   - any surviving replica in SCATTER state -> SCATTER.  otherwise, SYNC.
+ *   - include base inode in ack for all inodes that saw scatterlock content
+ *
+ * also, for scatter gather,
+ *
+ * - auth increments {frag,r}stat.version on completion of any gather.
+ *
+ * - auth incorporates changes in a gather _only_ if the version
+ *   matches.
+ *
+ * - replica discards changes any time the scatterlock syncs, and
+ *   after recovery.
+ */
+
+void MDCache::rejoin_start()
+{
+  dout(10) << "rejoin_start" << dendl;
+
+  rejoin_gather = recovery_set;
+  // need finish opening cap inodes before sending cache rejoins
+  rejoin_gather.insert(mds->get_nodeid());
+  process_imported_caps();
+}
+
+/*
+ * rejoin phase!
+ *
+ * this initiates rejoin.  it shoudl be called before we get any
+ * rejoin or rejoin_ack messages (or else mdsmap distribution is broken).
+ *
+ * we start out by sending rejoins to everyone in the recovery set.
+ *
+ * if we are rejoin, send for all regions in our cache.
+ * if we are active|stopping, send only to nodes that are are rejoining.
+ */
+void MDCache::rejoin_send_rejoins()
+{
+  dout(10) << "rejoin_send_rejoins with recovery_set " << recovery_set << dendl;
+
+  if (rejoin_gather.count(mds->get_nodeid())) {
+    dout(7) << "rejoin_send_rejoins still processing imported caps, delaying" << dendl;
+    rejoins_pending = true;
+    return;
+  }
+  if (!resolve_gather.empty()) {
+    dout(7) << "rejoin_send_rejoins still waiting for resolves ("
+	    << resolve_gather << ")" << dendl;
+    rejoins_pending = true;
+    return;
+  }
+
+  map rejoins;
+
+
+  // if i am rejoining, send a rejoin to everyone.
+  // otherwise, just send to others who are rejoining.
+  for (set::iterator p = recovery_set.begin();
+       p != recovery_set.end();
+       ++p) {
+    if (*p == mds->get_nodeid())  continue;  // nothing to myself!
+    if (rejoin_sent.count(*p)) continue;     // already sent a rejoin to this node!
+    if (mds->is_rejoin())
+      rejoins[*p] = new MMDSCacheRejoin(MMDSCacheRejoin::OP_WEAK);
+    else if (mds->mdsmap->is_rejoin(*p))
+      rejoins[*p] = new MMDSCacheRejoin(MMDSCacheRejoin::OP_STRONG);
+  }
+
+  if (mds->is_rejoin()) {
+    map > client_exports;
+    for (map >::iterator p = cap_exports.begin();
+         p != cap_exports.end();
+	 ++p) {
+      assert(cap_export_targets.count(p->first));
+      int target = cap_export_targets[p->first];
+      if (rejoins.count(target) == 0)
+	continue;
+      rejoins[target]->cap_exports[p->first] = p->second;
+      for (map::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q)
+	client_exports[q->first].insert(target);
+    }
+    for (map >::iterator p = client_exports.begin();
+	 p != client_exports.end();
+	 ++p) {
+      entity_inst_t inst = mds->sessionmap.get_inst(entity_name_t::CLIENT(p->first.v));
+      for (set::iterator q = p->second.begin(); q != p->second.end(); ++q)
+	rejoins[*q]->client_map[p->first] = inst;
+    }
+  }
+  
+  assert(!migrator->is_importing());
+  assert(!migrator->is_exporting());
+  
+  // check all subtrees
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *dir = p->first;
+    assert(dir->is_subtree_root());
+    if (dir->is_ambiguous_dir_auth()) {
+      // exporter is recovering, importer is survivor.
+      assert(rejoins.count(dir->authority().first));
+      assert(!rejoins.count(dir->authority().second));
+      continue;
+    }
+
+    // my subtree?
+    if (dir->is_auth())
+      continue;  // skip my own regions!
+
+    int auth = dir->get_dir_auth().first;
+    assert(auth >= 0);
+    if (rejoins.count(auth) == 0)
+      continue;   // don't care about this node's subtrees
+
+    rejoin_walk(dir, rejoins[auth]);
+  }
+  
+  // rejoin root inodes, too
+  for (map::iterator p = rejoins.begin();
+       p != rejoins.end();
+       ++p) {
+    if (mds->is_rejoin()) {
+      // weak
+      if (p->first == 0 && root) {
+	p->second->add_weak_inode(root->vino());
+	if (root->is_dirty_scattered()) {
+	  dout(10) << " sending scatterlock state on root " << *root << dendl;
+	  p->second->add_scatterlock_state(root);
+	}
+      }
+      if (CInode *in = get_inode(MDS_INO_MDSDIR(p->first))) { 
+	if (in)
+	  p->second->add_weak_inode(in->vino());
+      }
+    } else {
+      // strong
+      if (p->first == 0 && root) {
+	p->second->add_strong_inode(root->vino(),
+				    root->get_replica_nonce(),
+				    root->get_caps_wanted(),
+				    root->filelock.get_state(),
+				    root->nestlock.get_state(),
+				    root->dirfragtreelock.get_state());
+	root->state_set(CInode::STATE_REJOINING);
+	if (root->is_dirty_scattered()) {
+	  dout(10) << " sending scatterlock state on root " << *root << dendl;
+	  p->second->add_scatterlock_state(root);
+	}
+      }
+
+      if (CInode *in = get_inode(MDS_INO_MDSDIR(p->first))) {
+	p->second->add_strong_inode(in->vino(),
+				    in->get_replica_nonce(),
+				    in->get_caps_wanted(),
+				    in->filelock.get_state(),
+				    in->nestlock.get_state(),
+				    in->dirfragtreelock.get_state());
+	in->state_set(CInode::STATE_REJOINING);
+      }
+    }
+  }  
+
+  if (!mds->is_rejoin()) {
+    // i am survivor.  send strong rejoin.
+    // note request remote_auth_pins, xlocks
+    for (ceph::unordered_map::iterator p = active_requests.begin();
+	 p != active_requests.end();
+	 ++p) {
+      MDRequestRef& mdr = p->second;
+      if (mdr->is_slave())
+	continue;
+      // auth pins
+      for (set::iterator q = mdr->remote_auth_pins.begin();
+	   q != mdr->remote_auth_pins.end();
+	   ++q) {
+	if (!(*q)->is_auth()) {
+	  int who = (*q)->authority().first;
+	  if (rejoins.count(who) == 0) continue;
+	  MMDSCacheRejoin *rejoin = rejoins[who];
+	  
+	  dout(15) << " " << *mdr << " authpin on " << **q << dendl;
+	  MDSCacheObjectInfo i;
+	  (*q)->set_object_info(i);
+	  if (i.ino)
+	    rejoin->add_inode_authpin(vinodeno_t(i.ino, i.snapid), mdr->reqid, mdr->attempt);
+	  else
+	    rejoin->add_dentry_authpin(i.dirfrag, i.dname, i.snapid, mdr->reqid, mdr->attempt);
+
+	  if (mdr->has_more() && mdr->more()->is_remote_frozen_authpin &&
+	      mdr->more()->rename_inode == (*q))
+	    rejoin->add_inode_frozen_authpin(vinodeno_t(i.ino, i.snapid),
+					     mdr->reqid, mdr->attempt);
+	}
+      }
+      // xlocks
+      for (set::iterator q = mdr->xlocks.begin();
+	   q != mdr->xlocks.end();
+	   ++q) {
+	if (!(*q)->get_parent()->is_auth()) {
+	  int who = (*q)->get_parent()->authority().first;
+	  if (rejoins.count(who) == 0) continue;
+	  MMDSCacheRejoin *rejoin = rejoins[who];
+	  
+	  dout(15) << " " << *mdr << " xlock on " << **q << " " << *(*q)->get_parent() << dendl;
+	  MDSCacheObjectInfo i;
+	  (*q)->get_parent()->set_object_info(i);
+	  if (i.ino)
+	    rejoin->add_inode_xlock(vinodeno_t(i.ino, i.snapid), (*q)->get_type(),
+				    mdr->reqid, mdr->attempt);
+	  else
+	    rejoin->add_dentry_xlock(i.dirfrag, i.dname, i.snapid,
+				     mdr->reqid, mdr->attempt);
+	}
+      }
+      // remote wrlocks
+      for (map::iterator q = mdr->remote_wrlocks.begin();
+	   q != mdr->remote_wrlocks.end();
+	   ++q) {
+	int who = q->second;
+	if (rejoins.count(who) == 0) continue;
+	MMDSCacheRejoin *rejoin = rejoins[who];
+
+	dout(15) << " " << *mdr << " wrlock on " << q->second
+		 << " " << q->first->get_parent() << dendl;
+	MDSCacheObjectInfo i;
+	q->first->get_parent()->set_object_info(i);
+	assert(i.ino);
+	rejoin->add_inode_wrlock(vinodeno_t(i.ino, i.snapid), q->first->get_type(),
+				 mdr->reqid, mdr->attempt);
+      }
+    }
+  }
+
+  // send the messages
+  for (map::iterator p = rejoins.begin();
+       p != rejoins.end();
+       ++p) {
+    assert(rejoin_sent.count(p->first) == 0);
+    assert(rejoin_ack_gather.count(p->first) == 0);
+    rejoin_sent.insert(p->first);
+    rejoin_ack_gather.insert(p->first);
+    mds->send_message_mds(p->second, p->first);
+  }
+  rejoin_ack_gather.insert(mds->whoami);   // we need to complete rejoin_gather_finish, too
+  rejoins_pending = false;
+
+  // nothing?
+  if (mds->is_rejoin() && rejoins.empty()) {
+    dout(10) << "nothing to rejoin" << dendl;
+    rejoin_gather_finish();
+  }
+}
+
+
+/** 
+ * rejoin_walk - build rejoin declarations for a subtree
+ * 
+ * @param dir subtree root
+ * @param rejoin rejoin message
+ *
+ * from a rejoining node:
+ *  weak dirfrag
+ *  weak dentries (w/ connectivity)
+ *
+ * from a surviving node:
+ *  strong dirfrag
+ *  strong dentries (no connectivity!)
+ *  strong inodes
+ */
+void MDCache::rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin)
+{
+  dout(10) << "rejoin_walk " << *dir << dendl;
+
+  list nested;  // finish this dir, then do nested items
+  
+  if (mds->is_rejoin()) {
+    // WEAK
+    rejoin->add_weak_dirfrag(dir->dirfrag());
+    for (CDir::map_t::iterator p = dir->items.begin();
+	 p != dir->items.end();
+	 ++p) {
+      CDentry *dn = p->second;
+      CDentry::linkage_t *dnl = dn->get_linkage();
+      dout(15) << " add_weak_primary_dentry " << *dn << dendl;
+      assert(dnl->is_primary());
+      CInode *in = dnl->get_inode();
+      assert(dnl->get_inode()->is_dir());
+      rejoin->add_weak_primary_dentry(dir->ino(), dn->name.c_str(), dn->first, dn->last, in->ino());
+      in->get_nested_dirfrags(nested);
+      if (in->is_dirty_scattered()) {
+	dout(10) << " sending scatterlock state on " << *in << dendl;
+	rejoin->add_scatterlock_state(in);
+      }
+    }
+  } else {
+    // STRONG
+    dout(15) << " add_strong_dirfrag " << *dir << dendl;
+    rejoin->add_strong_dirfrag(dir->dirfrag(), dir->get_replica_nonce(), dir->get_dir_rep());
+    dir->state_set(CDir::STATE_REJOINING);
+
+    for (CDir::map_t::iterator p = dir->items.begin();
+	 p != dir->items.end();
+	 ++p) {
+      CDentry *dn = p->second;
+      CDentry::linkage_t *dnl = dn->get_linkage();
+      dout(15) << " add_strong_dentry " << *dn << dendl;
+      rejoin->add_strong_dentry(dir->dirfrag(), dn->name, dn->first, dn->last,
+				dnl->is_primary() ? dnl->get_inode()->ino():inodeno_t(0),
+				dnl->is_remote() ? dnl->get_remote_ino():inodeno_t(0),
+				dnl->is_remote() ? dnl->get_remote_d_type():0, 
+				dn->get_replica_nonce(),
+				dn->lock.get_state());
+      dn->state_set(CDentry::STATE_REJOINING);
+      if (dnl->is_primary()) {
+	CInode *in = dnl->get_inode();
+	dout(15) << " add_strong_inode " << *in << dendl;
+	rejoin->add_strong_inode(in->vino(),
+				 in->get_replica_nonce(),
+				 in->get_caps_wanted(),
+				 in->filelock.get_state(),
+				 in->nestlock.get_state(),
+				 in->dirfragtreelock.get_state());
+	in->state_set(CInode::STATE_REJOINING);
+	in->get_nested_dirfrags(nested);
+	if (in->is_dirty_scattered()) {
+	  dout(10) << " sending scatterlock state on " << *in << dendl;
+	  rejoin->add_scatterlock_state(in);
+	}
+      }
+    }
+  }
+
+  // recurse into nested dirs
+  for (list::iterator p = nested.begin(); 
+       p != nested.end();
+       ++p)
+    rejoin_walk(*p, rejoin);
+}
+
+
+/*
+ * i got a rejoin.
+ *  - reply with the lockstate
+ *
+ * if i am active|stopping, 
+ *  - remove source from replica list for everything not referenced here.
+ * This function puts the passed message before returning.
+ */
+void MDCache::handle_cache_rejoin(MMDSCacheRejoin *m)
+{
+  dout(7) << "handle_cache_rejoin " << *m << " from " << m->get_source() 
+	  << " (" << m->get_payload().length() << " bytes)"
+	  << dendl;
+
+  switch (m->op) {
+  case MMDSCacheRejoin::OP_WEAK:
+    handle_cache_rejoin_weak(m);
+    break;
+  case MMDSCacheRejoin::OP_STRONG:
+    handle_cache_rejoin_strong(m);
+    break;
+
+  case MMDSCacheRejoin::OP_ACK:
+    handle_cache_rejoin_ack(m);
+    break;
+  case MMDSCacheRejoin::OP_MISSING:
+    handle_cache_rejoin_missing(m);
+    break;
+
+  case MMDSCacheRejoin::OP_FULL:
+    handle_cache_rejoin_full(m);
+    break;
+
+  default: 
+    assert(0);
+  }
+  m->put();
+}
+
+
+/*
+ * handle_cache_rejoin_weak
+ *
+ * the sender 
+ *  - is recovering from their journal.
+ *  - may have incorrect (out of date) inode contents
+ *  - will include weak dirfrag if sender is dirfrag auth and parent inode auth is recipient
+ *
+ * if the sender didn't trim_non_auth(), they
+ *  - may have incorrect (out of date) dentry/inode linkage
+ *  - may have deleted/purged inodes
+ * and i may have to go to disk to get accurate inode contents.  yuck.
+ * This functions DOES NOT put the passed message before returning
+ */
+void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
+{
+  int from = weak->get_source().num();
+
+  // possible response(s)
+  MMDSCacheRejoin *ack = 0;      // if survivor
+  set acked_inodes;  // if survivor
+  set gather_locks;  // if survivor
+  bool survivor = false;  // am i a survivor?
+
+  if (mds->is_clientreplay() || mds->is_active() || mds->is_stopping()) {
+    survivor = true;
+    dout(10) << "i am a surivivor, and will ack immediately" << dendl;
+    ack = new MMDSCacheRejoin(MMDSCacheRejoin::OP_ACK);
+
+    map > imported_caps;
+
+    // check cap exports
+    for (map >::iterator p = weak->cap_exports.begin();
+	 p != weak->cap_exports.end();
+	 ++p) {
+      CInode *in = get_inode(p->first);
+      assert(!in || in->is_auth());
+      for (map::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q) {
+	dout(10) << " claiming cap import " << p->first << " client." << q->first << " on " << *in << dendl;
+	Capability *cap = rejoin_import_cap(in, q->first, q->second, from);
+	Capability::Import& im = imported_caps[p->first][q->first];
+	im.cap_id = cap->get_cap_id();
+	im.issue_seq = cap->get_last_seq();
+	im.mseq = cap->get_mseq();
+      }
+      mds->locker->eval(in, CEPH_CAP_LOCKS, true);
+    }
+
+    ::encode(imported_caps, ack->imported_caps);
+  } else {
+    assert(mds->is_rejoin());
+
+    // we may have already received a strong rejoin from the sender.
+    rejoin_scour_survivor_replicas(from, NULL, acked_inodes, gather_locks);
+    assert(gather_locks.empty());
+
+    // check cap exports.
+    rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end());
+
+    for (map >::iterator p = weak->cap_exports.begin();
+	 p != weak->cap_exports.end();
+	 ++p) {
+      CInode *in = get_inode(p->first);
+      assert(in && in->is_auth());
+      // note
+      for (map::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q) {
+	dout(10) << " claiming cap import " << p->first << " client." << q->first << dendl;
+	cap_imports[p->first][q->first][from] = q->second;
+      }
+    }
+  }
+
+  // assimilate any potentially dirty scatterlock state
+  for (map::iterator p = weak->inode_scatterlocks.begin();
+       p != weak->inode_scatterlocks.end();
+       ++p) {
+    CInode *in = get_inode(p->first);
+    assert(in);
+    in->decode_lock_state(CEPH_LOCK_IFILE, p->second.file);
+    in->decode_lock_state(CEPH_LOCK_INEST, p->second.nest);
+    in->decode_lock_state(CEPH_LOCK_IDFT, p->second.dft);
+    if (!survivor)
+      rejoin_potential_updated_scatterlocks.insert(in);
+  }
+
+  // recovering peer may send incorrect dirfrags here.  we need to
+  // infer which dirfrag they meant.  the ack will include a
+  // strong_dirfrag that will set them straight on the fragmentation.
+  
+  // walk weak map
+  set dirs_to_share;
+  for (set::iterator p = weak->weak_dirfrags.begin();
+       p != weak->weak_dirfrags.end();
+       ++p) {
+    CInode *diri = get_inode(p->ino);
+    if (!diri)
+      dout(0) << " missing dir ino " << p->ino << dendl;
+    assert(diri);
+
+    list ls;
+    if (diri->dirfragtree.is_leaf(p->frag)) {
+      ls.push_back(p->frag);
+    } else {
+      diri->dirfragtree.get_leaves_under(p->frag, ls);
+      if (ls.empty())
+	ls.push_back(diri->dirfragtree[p->frag.value()]);
+    }
+    for (list::iterator q = ls.begin(); q != ls.end(); ++q) {
+      frag_t fg = *q;
+      CDir *dir = diri->get_dirfrag(fg);
+      if (!dir) {
+	dout(0) << " missing dir for " << p->frag << " (which maps to " << fg << ") on " << *diri << dendl;
+	continue;
+      }
+      assert(dir);
+      if (dirs_to_share.count(dir)) {
+	dout(10) << " already have " << p->frag << " -> " << fg << " " << *dir << dendl;
+      } else {
+	dirs_to_share.insert(dir);
+	unsigned nonce = dir->add_replica(from);
+	dout(10) << " have " << p->frag << " -> " << fg << " " << *dir << dendl;
+	if (ack)
+	  ack->add_strong_dirfrag(dir->dirfrag(), nonce, dir->dir_rep);
+      }
+    }
+  }
+
+  for (map >::iterator p = weak->weak.begin();
+       p != weak->weak.end();
+       ++p) {
+    CInode *diri = get_inode(p->first);
+    if (!diri)
+      dout(0) << " missing dir ino " << p->first << dendl;
+    assert(diri);
+
+    // weak dentries
+    CDir *dir = 0;
+    for (map::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      // locate proper dirfrag.
+      //  optimize for common case (one dirfrag) to avoid dirs_to_share set check
+      frag_t fg = diri->pick_dirfrag(q->first.name);
+      if (!dir || dir->get_frag() != fg) {
+	dir = diri->get_dirfrag(fg);
+	if (!dir)
+	  dout(0) << " missing dir frag " << fg << " on " << *diri << dendl;
+	assert(dir);
+	assert(dirs_to_share.count(dir));
+      }
+
+      // and dentry
+      CDentry *dn = dir->lookup(q->first.name, q->first.snapid);
+      assert(dn);
+      CDentry::linkage_t *dnl = dn->get_linkage();
+      assert(dnl->is_primary());
+      
+      if (survivor && dn->is_replica(from)) 
+	dentry_remove_replica(dn, from, gather_locks);
+      unsigned dnonce = dn->add_replica(from);
+      dout(10) << " have " << *dn << dendl;
+      if (ack) 
+	ack->add_strong_dentry(dir->dirfrag(), dn->name, dn->first, dn->last,
+			       dnl->get_inode()->ino(), inodeno_t(0), 0, 
+			       dnonce, dn->lock.get_replica_state());
+
+      // inode
+      CInode *in = dnl->get_inode();
+      assert(in);
+
+      if (survivor && in->is_replica(from)) 
+	inode_remove_replica(in, from, true, gather_locks);
+      unsigned inonce = in->add_replica(from);
+      dout(10) << " have " << *in << dendl;
+
+      // scatter the dirlock, just in case?
+      if (!survivor && in->is_dir() && in->has_subtree_root_dirfrag())
+	in->filelock.set_state(LOCK_MIX);
+
+      if (ack) {
+	acked_inodes.insert(in->vino());
+	ack->add_inode_base(in);
+	bufferlist bl;
+	in->_encode_locks_state_for_rejoin(bl, from);
+	ack->add_inode_locks(in, inonce, bl);
+      }
+    }
+  }
+  
+  // weak base inodes?  (root, stray, etc.)
+  for (set::iterator p = weak->weak_inodes.begin();
+       p != weak->weak_inodes.end();
+       ++p) {
+    CInode *in = get_inode(*p);
+    assert(in);   // hmm fixme wrt stray?
+    if (survivor && in->is_replica(from)) 
+      inode_remove_replica(in, from, true, gather_locks);
+    unsigned inonce = in->add_replica(from);
+    dout(10) << " have base " << *in << dendl;
+    
+    if (ack) {
+      acked_inodes.insert(in->vino());
+      ack->add_inode_base(in);
+      bufferlist bl;
+      in->_encode_locks_state_for_rejoin(bl, from);
+      ack->add_inode_locks(in, inonce, bl);
+    }
+  }
+
+  assert(rejoin_gather.count(from));
+  rejoin_gather.erase(from);
+  if (survivor) {
+    // survivor.  do everything now.
+    for (map::iterator p = weak->inode_scatterlocks.begin();
+	 p != weak->inode_scatterlocks.end();
+	 ++p) {
+      CInode *in = get_inode(p->first);
+      assert(in);
+      dout(10) << " including base inode (due to potential scatterlock update) " << *in << dendl;
+      acked_inodes.insert(in->vino());
+      ack->add_inode_base(in);
+    }
+
+    rejoin_scour_survivor_replicas(from, ack, acked_inodes, gather_locks);
+    mds->send_message(ack, weak->get_connection());
+
+    for (set::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p) {
+      if (!(*p)->is_stable())
+	mds->locker->eval_gather(*p);
+    }
+  } else {
+    // done?
+    if (rejoin_gather.empty()) {
+      rejoin_gather_finish();
+    } else {
+      dout(7) << "still need rejoin from (" << rejoin_gather << ")" << dendl;
+    }
+  }
+}
+
+class C_MDC_RejoinGatherFinish : public Context {
+  MDCache *cache;
+public:
+  C_MDC_RejoinGatherFinish(MDCache *c) : cache(c) {}
+  void finish(int r) {
+    cache->rejoin_gather_finish();
+  }
+};
+
+#if 0
+/**
+ * parallel_fetch -- make a pass at fetching a bunch of paths in parallel
+ *
+ * @param pathmap map of inodeno to full pathnames.  we remove items
+ *            from this map as we discover we have them.
+ *
+ *	      returns true if there is work to do, false otherwise.
+ */
+
+bool MDCache::parallel_fetch(map& pathmap, set& missing)
+{
+  dout(10) << "parallel_fetch on " << pathmap.size() << " paths" << dendl;
+
+  C_GatherBuilder gather_bld(g_ceph_context, new C_MDC_RejoinGatherFinish(this));
+
+  // scan list
+  set fetch_queue;
+  map::iterator p = pathmap.begin();
+  while (p != pathmap.end()) {
+    // do we have the target already?
+    CInode *cur = get_inode(p->first);
+    if (cur) {
+      dout(15) << " have " << *cur << dendl;
+      pathmap.erase(p++);
+      continue;
+    }
+
+    // traverse
+    dout(17) << " missing " << p->first << " at " << p->second << dendl;
+    if (parallel_fetch_traverse_dir(p->first, p->second, fetch_queue,
+				    missing, gather_bld))
+      pathmap.erase(p++);
+    else
+      ++p;
+  }
+
+  if (pathmap.empty() && (!gather_bld.has_subs())) {
+    dout(10) << "parallel_fetch done" << dendl;
+    assert(fetch_queue.empty());
+    return false;
+  }
+
+  // do a parallel fetch
+  for (set::iterator p = fetch_queue.begin();
+       p != fetch_queue.end();
+       ++p) {
+    dout(10) << "parallel_fetch fetching " << **p << dendl;
+    (*p)->fetch(gather_bld.new_sub());
+  }
+  
+  if (gather_bld.get()) {
+    gather_bld.activate();
+    return true;
+  }
+  return false;
+}
+
+// true if we're done with this path
+bool MDCache::parallel_fetch_traverse_dir(inodeno_t ino, filepath& path,
+					  set& fetch_queue, set& missing,
+					  C_GatherBuilder &gather_bld)
+{
+  CInode *cur = get_inode(path.get_ino());
+  if (!cur) {
+    dout(5) << " missing " << path << " base ino " << path.get_ino() << dendl;
+    missing.insert(ino);
+    return true;
+  }
+
+  for (unsigned i=0; iis_dir()) {
+      dout(5) << " bad path " << path << " ENOTDIR at " << path[i] << dendl;
+      missing.insert(ino);
+      return true;
+    }
+      
+    frag_t fg = cur->pick_dirfrag(path[i]);
+    CDir *dir = cur->get_or_open_dirfrag(this, fg);
+    CDentry *dn = dir->lookup(path[i]);
+    CDentry::linkage_t *dnl = dn ? dn->get_linkage() : NULL;
+
+    if (!dnl || dnl->is_null()) {
+      if (!dir->is_auth()) {
+	dout(10) << " not dirfrag auth " << *dir << dendl;
+	return true;
+      }
+      if (dnl || dir->is_complete()) {
+	// probably because the client created it and held a cap but it never committed
+	// to the journal, and the op hasn't replayed yet.
+	dout(5) << " dne (not created yet?) " << ino << " at " << path << dendl;
+	missing.insert(ino);
+	return true;
+      }
+      // fetch dir
+      fetch_queue.insert(dir);
+      return false;
+    }
+
+    cur = dnl->get_inode();
+    if (!cur) {
+      assert(dnl->is_remote());
+      cur = get_inode(dnl->get_remote_ino());
+      if (cur) {
+	dn->link_remote(dnl, cur);
+      } else {
+	// open remote ino
+	open_remote_ino(dnl->get_remote_ino(), gather_bld.new_sub());
+	return false;
+      }
+    }
+  }
+
+  dout(5) << " ino not found " << ino << " at " << path << dendl;
+  missing.insert(ino);
+  return true;
+}
+#endif
+
+/*
+ * rejoin_scour_survivor_replica - remove source from replica list on unmentioned objects
+ *
+ * all validated replicas are acked with a strong nonce, etc.  if that isn't in the
+ * ack, the replica dne, and we can remove it from our replica maps.
+ */
+void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
+					     set& acked_inodes,
+					     set& gather_locks)
+{
+  dout(10) << "rejoin_scour_survivor_replicas from mds." << from << dendl;
+
+  for (ceph::unordered_map::iterator p = inode_map.begin();
+       p != inode_map.end();
+       ++p) {
+    CInode *in = p->second;
+    
+    // inode?
+    if (in->is_auth() &&
+	in->is_replica(from) &&
+	(ack == NULL || acked_inodes.count(p->second->vino()) == 0)) {
+      inode_remove_replica(in, from, false, gather_locks);
+      dout(10) << " rem " << *in << dendl;
+    }
+
+    if (!in->is_dir()) continue;
+    
+    list dfs;
+    in->get_dirfrags(dfs);
+    for (list::iterator p = dfs.begin();
+	 p != dfs.end();
+	 ++p) {
+      CDir *dir = *p;
+      
+      if (dir->is_auth() &&
+	  dir->is_replica(from) &&
+	  (ack == NULL || ack->strong_dirfrags.count(dir->dirfrag()) == 0)) {
+	dir->remove_replica(from);
+	dout(10) << " rem " << *dir << dendl;
+      } 
+      
+      // dentries
+      for (CDir::map_t::iterator p = dir->items.begin();
+	   p != dir->items.end();
+	   ++p) {
+	CDentry *dn = p->second;
+	
+	if (dn->is_replica(from) &&
+	    (ack == NULL ||
+	     ack->strong_dentries.count(dir->dirfrag()) == 0 ||
+	     ack->strong_dentries[dir->dirfrag()].count(string_snap_t(dn->name, dn->last)) == 0)) {
+	  dentry_remove_replica(dn, from, gather_locks);
+	  dout(10) << " rem " << *dn << dendl;
+	}
+      }
+    }
+  }
+}
+
+
+CInode *MDCache::rejoin_invent_inode(inodeno_t ino, snapid_t last)
+{
+  CInode *in = new CInode(this, true, 1, last);
+  in->inode.ino = ino;
+  in->state_set(CInode::STATE_REJOINUNDEF);
+  add_inode(in);
+  rejoin_undef_inodes.insert(in);
+  dout(10) << " invented " << *in << dendl;
+  return in;
+}
+
+CDir *MDCache::rejoin_invent_dirfrag(dirfrag_t df)
+{
+  CInode *in = get_inode(df.ino);
+  if (!in)
+    in = rejoin_invent_inode(df.ino, CEPH_NOSNAP);
+  if (!in->is_dir()) {
+    assert(in->state_test(CInode::STATE_REJOINUNDEF));
+    in->inode.mode = S_IFDIR;
+  }
+  CDir *dir = in->get_or_open_dirfrag(this, df.frag);
+  dir->state_set(CDir::STATE_REJOINUNDEF);
+  rejoin_undef_dirfrags.insert(dir);
+  dout(10) << " invented " << *dir << dendl;
+  return dir;
+}
+
+/* This functions DOES NOT put the passed message before returning */
+void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
+{
+  int from = strong->get_source().num();
+
+  // only a recovering node will get a strong rejoin.
+  assert(mds->is_rejoin());
+
+  // assimilate any potentially dirty scatterlock state
+  for (map::iterator p = strong->inode_scatterlocks.begin();
+       p != strong->inode_scatterlocks.end();
+       ++p) {
+    CInode *in = get_inode(p->first);
+    assert(in);
+    in->decode_lock_state(CEPH_LOCK_IFILE, p->second.file);
+    in->decode_lock_state(CEPH_LOCK_INEST, p->second.nest);
+    in->decode_lock_state(CEPH_LOCK_IDFT, p->second.dft);
+    rejoin_potential_updated_scatterlocks.insert(in);
+  }
+
+  rejoin_unlinked_inodes[from].clear();
+
+  // surviving peer may send incorrect dirfrag here (maybe they didn't
+  // get the fragment notify, or maybe we rolled back?).  we need to
+  // infer the right frag and get them with the program.  somehow.
+  // we don't normally send ACK.. so we'll need to bundle this with
+  // MISSING or something.
+
+  // strong dirfrags/dentries.
+  //  also process auth_pins, xlocks.
+  for (map::iterator p = strong->strong_dirfrags.begin();
+       p != strong->strong_dirfrags.end();
+       ++p) {
+    CInode *diri = get_inode(p->first.ino);
+    if (!diri)
+      diri = rejoin_invent_inode(p->first.ino, CEPH_NOSNAP);
+    CDir *dir = diri->get_dirfrag(p->first.frag);
+    bool refragged = false;
+    if (dir) {
+      dout(10) << " have " << *dir << dendl;
+    } else {
+      if (diri->state_test(CInode::STATE_REJOINUNDEF))
+	dir = rejoin_invent_dirfrag(dirfrag_t(diri->ino(), frag_t()));
+      else if (diri->dirfragtree.is_leaf(p->first.frag))
+	dir = rejoin_invent_dirfrag(p->first);
+    }
+    if (dir) {
+      dir->add_replica(from, p->second.nonce);
+      dir->dir_rep = p->second.dir_rep;
+    } else {
+      dout(10) << " frag " << p->first << " doesn't match dirfragtree " << *diri << dendl;
+      list ls;
+      diri->dirfragtree.get_leaves_under(p->first.frag, ls);
+      if (ls.empty())
+	ls.push_back(diri->dirfragtree[p->first.frag.value()]);
+      dout(10) << " maps to frag(s) " << ls << dendl;
+      for (list::iterator q = ls.begin(); q != ls.end(); ++q) {
+	CDir *dir = diri->get_dirfrag(*q);
+	if (!dir)
+	  dir = rejoin_invent_dirfrag(dirfrag_t(diri->ino(), *q));
+	else
+	  dout(10) << " have(approx) " << *dir << dendl;
+	dir->add_replica(from, p->second.nonce);
+	dir->dir_rep = p->second.dir_rep;
+      }
+      refragged = true;
+    }
+    
+    map& dmap = strong->strong_dentries[p->first];
+    for (map::iterator q = dmap.begin();
+	 q != dmap.end();
+	 ++q) {
+      CDentry *dn;
+      if (!refragged)
+	dn = dir->lookup(q->first.name, q->first.snapid);
+      else {
+	frag_t fg = diri->pick_dirfrag(q->first.name);
+	dir = diri->get_dirfrag(fg);
+	assert(dir);
+	dn = dir->lookup(q->first.name, q->first.snapid);
+      }
+      if (!dn) {
+	if (q->second.is_remote()) {
+	  dn = dir->add_remote_dentry(q->first.name, q->second.remote_ino, q->second.remote_d_type,
+				      q->second.first, q->first.snapid);
+	} else if (q->second.is_null()) {
+	  dn = dir->add_null_dentry(q->first.name, q->second.first, q->first.snapid);
+	} else {
+	  CInode *in = get_inode(q->second.ino, q->first.snapid);
+	  if (!in) in = rejoin_invent_inode(q->second.ino, q->first.snapid);
+	  dn = dir->add_primary_dentry(q->first.name, in, q->second.first, q->first.snapid);
+	}
+	dout(10) << " invented " << *dn << dendl;
+      }
+      CDentry::linkage_t *dnl = dn->get_linkage();
+
+      // dn auth_pin?
+      if (strong->authpinned_dentries.count(p->first) &&
+	  strong->authpinned_dentries[p->first].count(q->first)) {
+	for (list::iterator r = strong->authpinned_dentries[p->first][q->first].begin();
+	     r != strong->authpinned_dentries[p->first][q->first].end();
+	     ++r) {
+	  dout(10) << " dn authpin by " << *r << " on " << *dn << dendl;
+
+	  // get/create slave mdrequest
+	  MDRequestRef mdr;
+	  if (have_request(r->reqid))
+	    mdr = request_get(r->reqid);
+	  else
+	    mdr = request_start_slave(r->reqid, r->attempt, from);
+	  mdr->auth_pin(dn);
+	}
+      }
+
+      // dn xlock?
+      if (strong->xlocked_dentries.count(p->first) &&
+	  strong->xlocked_dentries[p->first].count(q->first)) {
+	MMDSCacheRejoin::slave_reqid r = strong->xlocked_dentries[p->first][q->first];
+	dout(10) << " dn xlock by " << r << " on " << *dn << dendl;
+	MDRequestRef mdr = request_get(r.reqid);  // should have this from auth_pin above.
+	assert(mdr->is_auth_pinned(dn));
+	if (!mdr->xlocks.count(&dn->versionlock)) {
+	  assert(dn->versionlock.can_xlock_local());
+	  dn->versionlock.get_xlock(mdr, mdr->get_client());
+	  mdr->xlocks.insert(&dn->versionlock);
+	  mdr->locks.insert(&dn->versionlock);
+	}
+	if (dn->lock.is_stable())
+	  dn->auth_pin(&dn->lock);
+	dn->lock.set_state(LOCK_XLOCK);
+	dn->lock.get_xlock(mdr, mdr->get_client());
+	mdr->xlocks.insert(&dn->lock);
+	mdr->locks.insert(&dn->lock);
+      }
+
+      dn->add_replica(from, q->second.nonce);
+      dout(10) << " have " << *dn << dendl;
+
+      if (dnl->is_primary()) {
+	if (q->second.is_primary()) {
+	  if (vinodeno_t(q->second.ino, q->first.snapid) != dnl->get_inode()->vino()) {
+	    // the survivor missed MDentryUnlink+MDentryLink messages ?
+	    assert(strong->strong_inodes.count(dnl->get_inode()->vino()) == 0);
+	    CInode *in = get_inode(q->second.ino, q->first.snapid);
+	    assert(in);
+	    assert(in->get_parent_dn());
+	    rejoin_unlinked_inodes[from].insert(in);
+	    dout(7) << " sender has primary dentry but wrong inode" << dendl;
+	  }
+	} else {
+	  // the survivor missed MDentryLink message ?
+	  assert(strong->strong_inodes.count(dnl->get_inode()->vino()) == 0);
+	  dout(7) << " sender doesn't have primay dentry" << dendl;
+	}
+      } else {
+	if (q->second.is_primary()) {
+	  // the survivor missed MDentryUnlink message ?
+	  CInode *in = get_inode(q->second.ino, q->first.snapid);
+	  assert(in);
+	  assert(in->get_parent_dn());
+	  rejoin_unlinked_inodes[from].insert(in);
+	  dout(7) << " sender has primary dentry but we don't" << dendl;
+	}
+      }
+    }
+  }
+
+  for (map::iterator p = strong->strong_inodes.begin();
+       p != strong->strong_inodes.end();
+       ++p) {
+    CInode *in = get_inode(p->first);
+    assert(in);
+    in->add_replica(from, p->second.nonce);
+    dout(10) << " have " << *in << dendl;
+
+    MMDSCacheRejoin::inode_strong &is = p->second;
+
+    // caps_wanted
+    if (is.caps_wanted) {
+      in->mds_caps_wanted[from] = is.caps_wanted;
+      dout(15) << " inode caps_wanted " << ccap_string(is.caps_wanted)
+	       << " on " << *in << dendl;
+    }
+
+    // scatterlocks?
+    //  infer state from replica state:
+    //   * go to MIX if they might have wrlocks
+    //   * go to LOCK if they are LOCK (just bc identify_files_to_recover might start twiddling filelock)
+    in->filelock.infer_state_from_strong_rejoin(is.filelock, true);  // maybe also go to LOCK
+    in->nestlock.infer_state_from_strong_rejoin(is.nestlock, false);
+    in->dirfragtreelock.infer_state_from_strong_rejoin(is.dftlock, false);
+
+    // auth pin?
+    if (strong->authpinned_inodes.count(in->vino())) {
+      for (list::iterator r = strong->authpinned_inodes[in->vino()].begin();
+	   r != strong->authpinned_inodes[in->vino()].end();
+	   ++r) {
+	dout(10) << " inode authpin by " << *r << " on " << *in << dendl;
+
+	// get/create slave mdrequest
+	MDRequestRef mdr;
+	if (have_request(r->reqid))
+	  mdr = request_get(r->reqid);
+	else
+	  mdr = request_start_slave(r->reqid, r->attempt, from);
+	if (strong->frozen_authpin_inodes.count(in->vino())) {
+	  assert(!in->get_num_auth_pins());
+	  mdr->freeze_auth_pin(in);
+	} else {
+	  assert(!in->is_frozen_auth_pin());
+	}
+	mdr->auth_pin(in);
+      }
+    }
+    // xlock(s)?
+    if (strong->xlocked_inodes.count(in->vino())) {
+      for (map::iterator q = strong->xlocked_inodes[in->vino()].begin();
+	   q != strong->xlocked_inodes[in->vino()].end();
+	   ++q) {
+	SimpleLock *lock = in->get_lock(q->first);
+	dout(10) << " inode xlock by " << q->second << " on " << *lock << " on " << *in << dendl;
+	MDRequestRef mdr = request_get(q->second.reqid);  // should have this from auth_pin above.
+	assert(mdr->is_auth_pinned(in));
+	if (!mdr->xlocks.count(&in->versionlock)) {
+	  assert(in->versionlock.can_xlock_local());
+	  in->versionlock.get_xlock(mdr, mdr->get_client());
+	  mdr->xlocks.insert(&in->versionlock);
+	  mdr->locks.insert(&in->versionlock);
+	}
+	if (lock->is_stable())
+	  in->auth_pin(lock);
+	lock->set_state(LOCK_XLOCK);
+	if (lock == &in->filelock)
+	  in->loner_cap = -1;
+	lock->get_xlock(mdr, mdr->get_client());
+	mdr->xlocks.insert(lock);
+	mdr->locks.insert(lock);
+      }
+    }
+  }
+  // wrlock(s)?
+  for (map > >::iterator p = strong->wrlocked_inodes.begin();
+       p != strong->wrlocked_inodes.end();
+       ++p) {
+    CInode *in = get_inode(p->first);
+    for (map >::iterator q = p->second.begin();
+	 q != p->second.end();
+	++q) {
+      SimpleLock *lock = in->get_lock(q->first);
+      for (list::iterator r = q->second.begin();
+	  r != q->second.end();
+	  ++r) {
+	dout(10) << " inode wrlock by " << *r << " on " << *lock << " on " << *in << dendl;
+	MDRequestRef mdr = request_get(r->reqid);  // should have this from auth_pin above.
+	if (in->is_auth())
+	  assert(mdr->is_auth_pinned(in));
+	lock->set_state(LOCK_MIX);
+	if (lock == &in->filelock)
+	  in->loner_cap = -1;
+	lock->get_wrlock(true);
+	mdr->wrlocks.insert(lock);
+	mdr->locks.insert(lock);
+      }
+    }
+  }
+
+  // done?
+  assert(rejoin_gather.count(from));
+  rejoin_gather.erase(from);
+  if (rejoin_gather.empty()) {
+    rejoin_gather_finish();
+  } else {
+    dout(7) << "still need rejoin from (" << rejoin_gather << ")" << dendl;
+  }
+}
+
+/* This functions DOES NOT put the passed message before returning */
+void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
+{
+  dout(7) << "handle_cache_rejoin_ack from " << ack->get_source() << dendl;
+  int from = ack->get_source().num();
+
+  // for sending cache expire message
+  set isolated_inodes;
+  set refragged_inodes;
+
+  // dirs
+  for (map::iterator p = ack->strong_dirfrags.begin();
+       p != ack->strong_dirfrags.end();
+       ++p) {
+    // we may have had incorrect dir fragmentation; refragment based
+    // on what they auth tells us.
+    CInode *diri = get_inode(p->first.ino);
+    CDir *dir = NULL;
+    if (diri) {
+      dir = diri->get_dirfrag(p->first.frag);
+      if (!dir) {
+	dir = force_dir_fragment(diri, p->first.frag, false);
+	if (dir)
+	  refragged_inodes.insert(dir->get_inode());
+      }
+    }
+    if (!dir) {
+      if (!diri) {
+	// barebones inode; the full inode loop below will clean up.
+	diri = new CInode(this, false);
+	diri->inode.ino = p->first.ino;
+	diri->inode.mode = S_IFDIR;
+	add_inode(diri);
+	if (MDS_INO_MDSDIR(from) == p->first.ino) {
+	  diri->inode_auth = pair(from, CDIR_AUTH_UNKNOWN);
+	  dout(10) << " add inode " << *diri << dendl;
+	} else {
+	  diri->inode_auth = CDIR_AUTH_DEFAULT;
+	  isolated_inodes.insert(diri);
+	  dout(10) << " unconnected dirfrag " << p->first << dendl;
+	}
+      }
+      // barebones dirfrag; the full dirfrag loop below will clean up.
+      dir = diri->add_dirfrag(new CDir(diri, p->first.frag, this, false));
+      if (MDS_INO_MDSDIR(from) == p->first.ino ||
+	  (dir->authority() != CDIR_AUTH_UNDEF &&
+	   dir->authority().first != from))
+	adjust_subtree_auth(dir, from);
+      dout(10) << " add dirfrag " << *dir << dendl;
+    }
+
+    dir->set_replica_nonce(p->second.nonce);
+    dir->state_clear(CDir::STATE_REJOINING);
+    dout(10) << " got " << *dir << dendl;
+
+    // dentries
+    map& dmap = ack->strong_dentries[p->first];
+    for (map::iterator q = dmap.begin();
+	 q != dmap.end();
+	 ++q) {
+      CDentry *dn = dir->lookup(q->first.name, q->first.snapid);
+      if(!dn)
+	dn = dir->add_null_dentry(q->first.name, q->second.first, q->first.snapid);
+
+      CDentry::linkage_t *dnl = dn->get_linkage();
+
+      assert(dn->last == q->first.snapid);
+      if (dn->first != q->second.first) {
+	dout(10) << " adjust dn.first " << dn->first << " -> " << q->second.first << " on " << *dn << dendl;
+	dn->first = q->second.first;
+      }
+
+      // may have bad linkage if we missed dentry link/unlink messages
+      if (dnl->is_primary()) {
+	CInode *in = dnl->get_inode();
+	if (!q->second.is_primary() ||
+	    vinodeno_t(q->second.ino, q->first.snapid) != in->vino()) {
+	  dout(10) << " had bad linkage for " << *dn << ", unlinking " << *in << dendl;
+	  dir->unlink_inode(dn);
+	}
+      } else if (dnl->is_remote()) {
+	if (!q->second.is_remote() ||
+	    q->second.remote_ino != dnl->get_remote_ino() ||
+	    q->second.remote_d_type != dnl->get_remote_d_type()) {
+	  dout(10) << " had bad linkage for " << *dn <<  dendl;
+	  dir->unlink_inode(dn);
+	}
+      } else {
+	if (!q->second.is_null())
+	  dout(10) << " had bad linkage for " << *dn <<  dendl;
+      }
+
+      // hmm, did we have the proper linkage here?
+      if (dnl->is_null() && !q->second.is_null()) {
+	if (q->second.is_remote()) {
+	  dn->dir->link_remote_inode(dn, q->second.remote_ino, q->second.remote_d_type);
+	} else {
+	  CInode *in = get_inode(q->second.ino, q->first.snapid);
+	  if (!in) {
+	    // barebones inode; assume it's dir, the full inode loop below will clean up.
+	    in = new CInode(this, false, q->second.first, q->first.snapid);
+	    in->inode.ino = q->second.ino;
+	    in->inode.mode = S_IFDIR;
+	    add_inode(in);
+	    dout(10) << " add inode " << *in << dendl;
+	  } else if (in->get_parent_dn()) {
+	    dout(10) << " had bad linkage for " << *(in->get_parent_dn())
+		     << ", unlinking " << *in << dendl;
+	    in->get_parent_dir()->unlink_inode(in->get_parent_dn());
+	  }
+	  dn->dir->link_primary_inode(dn, in);
+	  isolated_inodes.erase(in);
+	}
+      }
+
+      dn->set_replica_nonce(q->second.nonce);
+      dn->lock.set_state_rejoin(q->second.lock, rejoin_waiters);
+      dn->state_clear(CDentry::STATE_REJOINING);
+      dout(10) << " got " << *dn << dendl;
+    }
+  }
+
+  for (set::iterator p = refragged_inodes.begin();
+       p != refragged_inodes.end();
+       ++p) {
+    list ls;
+    (*p)->get_nested_dirfrags(ls);
+    for (list::iterator q = ls.begin(); q != ls.end(); ++q) {
+      if ((*q)->is_auth() || ack->strong_dirfrags.count((*q)->dirfrag()))
+	continue;
+      assert((*q)->get_num_any() == 0);
+      (*p)->close_dirfrag((*q)->get_frag());
+    }
+  }
+
+  // full dirfrags
+  for (map::iterator p = ack->dirfrag_bases.begin();
+       p != ack->dirfrag_bases.end();
+       ++p) {
+    CDir *dir = get_dirfrag(p->first);
+    assert(dir);
+    bufferlist::iterator q = p->second.begin();
+    dir->_decode_base(q);
+    dout(10) << " got dir replica " << *dir << dendl;
+  }
+
+  // full inodes
+  bufferlist::iterator p = ack->inode_base.begin();
+  while (!p.end()) {
+    inodeno_t ino;
+    snapid_t last;
+    bufferlist basebl;
+    ::decode(ino, p);
+    ::decode(last, p);
+    ::decode(basebl, p);
+    CInode *in = get_inode(ino, last);
+    assert(in);
+    bufferlist::iterator q = basebl.begin();
+    in->_decode_base(q);
+    dout(10) << " got inode base " << *in << dendl;
+  }
+
+  // inodes
+  p = ack->inode_locks.begin();
+  //dout(10) << "inode_locks len " << ack->inode_locks.length() << " is " << ack->inode_locks << dendl;
+  while (!p.end()) {
+    inodeno_t ino;
+    snapid_t last;
+    __u32 nonce;
+    bufferlist lockbl;
+    ::decode(ino, p);
+    ::decode(last, p);
+    ::decode(nonce, p);
+    ::decode(lockbl, p);
+    
+    CInode *in = get_inode(ino, last);
+    assert(in);
+    in->set_replica_nonce(nonce);
+    bufferlist::iterator q = lockbl.begin();
+    in->_decode_locks_rejoin(q, rejoin_waiters, rejoin_eval_locks);
+    in->state_clear(CInode::STATE_REJOINING);
+    dout(10) << " got inode locks " << *in << dendl;
+  }
+
+  // FIXME: This can happen if entire subtree, together with the inode subtree root
+  // belongs to, were trimmed between sending cache rejoin and receiving rejoin ack.
+  assert(isolated_inodes.empty());
+
+  map > peer_imported;
+  bufferlist::iterator bp = ack->imported_caps.begin();
+  ::decode(peer_imported, bp);
+
+  for (map >::iterator p = peer_imported.begin();
+       p != peer_imported.end();
+       ++p) {
+    assert(cap_exports.count(p->first));
+    assert(cap_export_targets.count(p->first));
+    assert(cap_export_targets[p->first] == from);
+    for (map::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      assert(cap_exports[p->first].count(q->first));
+
+      dout(10) << " exporting caps for client." << q->first << " ino " << p->first << dendl;
+      Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+      assert(session);
+
+      // mark client caps stale.
+      MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, p->first, 0,
+				       cap_exports[p->first][q->first].cap_id, 0);
+      m->set_cap_peer(q->second.cap_id, q->second.issue_seq, q->second.mseq, from, 0);
+      mds->send_message_client_counted(m, session);
+
+      cap_exports[p->first].erase(q->first);
+    }
+    assert(cap_exports[p->first].empty());
+  }
+
+  // done?
+  assert(rejoin_ack_gather.count(from));
+  rejoin_ack_gather.erase(from);
+  if (mds->is_rejoin()) {
+
+    if (rejoin_gather.empty()) {
+      // eval unstable scatter locks after all wrlocks are rejoined.
+      while (!rejoin_eval_locks.empty()) {
+	SimpleLock *lock = rejoin_eval_locks.front();
+	rejoin_eval_locks.pop_front();
+	if (!lock->is_stable())
+	  mds->locker->eval_gather(lock);
+      }
+    }
+
+    if (rejoin_gather.empty() &&     // make sure we've gotten our FULL inodes, too.
+	rejoin_ack_gather.empty()) {
+      // finally, kickstart past snap parent opens
+      open_snap_parents();
+    } else {
+      dout(7) << "still need rejoin from (" << rejoin_gather << ")"
+	      << ", rejoin_ack from (" << rejoin_ack_gather << ")" << dendl;
+    }
+  } else {
+    // survivor.
+    mds->queue_waiters(rejoin_waiters);
+  }
+}
+
+
+
+/* This functions DOES NOT put the passed message before returning */
+void MDCache::handle_cache_rejoin_missing(MMDSCacheRejoin *missing)
+{
+  dout(7) << "handle_cache_rejoin_missing from " << missing->get_source() << dendl;
+
+  MMDSCacheRejoin *full = new MMDSCacheRejoin(MMDSCacheRejoin::OP_FULL);
+
+  // dirs
+  for (map::iterator p = missing->strong_dirfrags.begin();
+       p != missing->strong_dirfrags.end();
+       ++p) {
+    // we may have had incorrect dir fragmentation; refragment based
+    // on what they auth tells us.
+    CDir *dir = get_force_dirfrag(p->first);
+    assert(dir);
+
+    dir->set_replica_nonce(p->second.nonce);
+    dir->state_clear(CDir::STATE_REJOINING);
+    dout(10) << " adjusted frag on " << *dir << dendl;
+  }
+
+  // inodes
+  for (set::iterator p = missing->weak_inodes.begin();
+       p != missing->weak_inodes.end();
+       ++p) {
+    CInode *in = get_inode(*p);
+    if (!in) {
+      dout(10) << " don't have inode " << *p << dendl;
+      continue; // we must have trimmed it after the originalo rejoin
+    }
+    
+    dout(10) << " sending " << *in << dendl;
+    full->add_inode_base(in);
+  }
+
+  mds->send_message(full, missing->get_connection());
+}
+
+/* This function DOES NOT put the passed message before returning */
+void MDCache::handle_cache_rejoin_full(MMDSCacheRejoin *full)
+{
+  dout(7) << "handle_cache_rejoin_full from " << full->get_source() << dendl;
+  int from = full->get_source().num();
+  
+  // integrate full inodes
+  bufferlist::iterator p = full->inode_base.begin();
+  while (!p.end()) {
+    inodeno_t ino;
+    snapid_t last;
+    bufferlist basebl;
+    ::decode(ino, p);
+    ::decode(last, p);
+    ::decode(basebl, p);
+
+    CInode *in = get_inode(ino);
+    assert(in);
+    bufferlist::iterator pp = basebl.begin();
+    in->_decode_base(pp);
+
+    set::iterator q = rejoin_undef_inodes.find(in);
+    if (q != rejoin_undef_inodes.end()) {
+      CInode *in = *q;
+      in->state_clear(CInode::STATE_REJOINUNDEF);
+      dout(10) << " got full " << *in << dendl;
+      rejoin_undef_inodes.erase(q);
+    } else {
+      dout(10) << " had full " << *in << dendl;
+    }
+  }
+
+  // done?
+  assert(rejoin_gather.count(from));
+  rejoin_gather.erase(from);
+  if (rejoin_gather.empty()) {
+    rejoin_gather_finish();
+  } else {
+    dout(7) << "still need rejoin from (" << rejoin_gather << ")" << dendl;
+  }
+}
+
+
+
+/**
+ * rejoin_trim_undef_inodes() -- remove REJOINUNDEF flagged inodes
+ *
+ * FIXME: wait, can this actually happen?  a survivor should generate cache trim
+ * messages that clean these guys up...
+ */
+void MDCache::rejoin_trim_undef_inodes()
+{
+  dout(10) << "rejoin_trim_undef_inodes" << dendl;
+
+  while (!rejoin_undef_inodes.empty()) {
+    set::iterator p = rejoin_undef_inodes.begin();
+    CInode *in = *p;
+    rejoin_undef_inodes.erase(p);
+
+    in->clear_replica_map();
+    
+    // close out dirfrags
+    if (in->is_dir()) {
+      list dfls;
+      in->get_dirfrags(dfls);
+      for (list::iterator p = dfls.begin();
+	   p != dfls.end();
+	   ++p) {
+	CDir *dir = *p;
+	dir->clear_replica_map();
+
+	for (CDir::map_t::iterator p = dir->items.begin();
+	     p != dir->items.end();
+	     ++p) {
+	  CDentry *dn = p->second;
+	  dn->clear_replica_map();
+
+	  dout(10) << " trimming " << *dn << dendl;
+	  dir->remove_dentry(dn);
+	}
+
+	dout(10) << " trimming " << *dir << dendl;
+	in->close_dirfrag(dir->dirfrag().frag);
+      }
+    }
+    
+    CDentry *dn = in->get_parent_dn();
+    if (dn) {
+      dn->clear_replica_map();
+      dout(10) << " trimming " << *dn << dendl;
+      dn->dir->remove_dentry(dn);
+    } else {
+      dout(10) << " trimming " << *in << dendl;
+      remove_inode(in);
+    }
+  }
+
+  assert(rejoin_undef_inodes.empty());
+}
+
+void MDCache::rejoin_gather_finish() 
+{
+  dout(10) << "rejoin_gather_finish" << dendl;
+  assert(mds->is_rejoin());
+
+  if (open_undef_inodes_dirfrags())
+    return;
+
+  if (process_imported_caps())
+    return;
+
+  choose_lock_states_and_reconnect_caps();
+
+  identify_files_to_recover(rejoin_recover_q, rejoin_check_q);
+  rejoin_send_acks();
+  
+  // signal completion of fetches, rejoin_gather_finish, etc.
+  assert(rejoin_ack_gather.count(mds->whoami));
+  rejoin_ack_gather.erase(mds->whoami);
+
+  // did we already get our acks too?
+  // this happens when the rejoin_gather has to wait on a MISSING/FULL exchange.
+  if (rejoin_ack_gather.empty()) {
+    // finally, kickstart past snap parent opens
+    open_snap_parents();
+  }
+}
+
+class C_MDC_RejoinOpenInoFinish: public Context {
+  MDCache *cache;
+  inodeno_t ino;
+public:
+  C_MDC_RejoinOpenInoFinish(MDCache *c, inodeno_t i) : cache(c), ino(i) {}
+  void finish(int r) {
+    cache->rejoin_open_ino_finish(ino, r);
+  }
+};
+
+void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret)
+{
+  dout(10) << "open_caps_inode_finish ino " << ino << " ret " << ret << dendl;
+
+  if (ret < 0) {
+    cap_imports_missing.insert(ino);
+  } else if (ret == mds->get_nodeid()) {
+    assert(get_inode(ino));
+  } else {
+    map > >::iterator p;
+    p = cap_imports.find(ino);
+    assert(p != cap_imports.end());
+    for (map >::iterator q = p->second.begin();
+	q != p->second.end();
+	++q) {
+      assert(q->second.count(-1));
+      assert(q->second.size() == 1);
+      rejoin_export_caps(p->first, q->first, q->second[-1], ret);
+    }
+    cap_imports.erase(p);
+  }
+
+  assert(cap_imports_num_opening > 0);
+  cap_imports_num_opening--;
+
+  if (cap_imports_num_opening == 0) {
+    if (rejoin_gather.empty())
+      rejoin_gather_finish();
+    else if (rejoin_gather.count(mds->get_nodeid()))
+      process_imported_caps();
+  }
+}
+
+class C_MDC_RejoinSessionsOpened : public Context {
+  MDCache *cache;
+public:
+  map client_map;
+  map sseqmap;
+
+  C_MDC_RejoinSessionsOpened(MDCache *c, map& cm) :
+    cache(c), client_map(cm) {}
+  void finish(int r) {
+    assert(r == 0);
+    cache->rejoin_open_sessions_finish(client_map, sseqmap);
+  }
+};
+
+void MDCache::rejoin_open_sessions_finish(map client_map,
+					  map& sseqmap)
+{
+  dout(10) << "rejoin_open_sessions_finish" << dendl;
+  mds->server->finish_force_open_sessions(client_map, sseqmap);
+  if (rejoin_gather.empty())
+    rejoin_gather_finish();
+}
+
+bool MDCache::process_imported_caps()
+{
+  dout(10) << "process_imported_caps" << dendl;
+
+  for (map > >::iterator p = cap_imports.begin();
+       p != cap_imports.end();
+       ++p) {
+    CInode *in = get_inode(p->first);
+    if (in) {
+      assert(in->is_auth());
+      cap_imports_missing.erase(p->first);
+      continue;
+    }
+    if (cap_imports_missing.count(p->first) > 0)
+      continue;
+
+    cap_imports_num_opening++;
+    dout(10) << "  opening missing ino " << p->first << dendl;
+    open_ino(p->first, (int64_t)-1, new C_MDC_RejoinOpenInoFinish(this, p->first), false);
+  }
+
+  if (cap_imports_num_opening > 0)
+    return true;
+
+  // called by rejoin_gather_finish() ?
+  if (rejoin_gather.count(mds->get_nodeid()) == 0) {
+    // if sessions for imported caps are all open ?
+    for (map::iterator p = rejoin_client_map.begin();
+	 p != rejoin_client_map.end();
+	 ++p) {
+      if (!mds->sessionmap.have_session(entity_name_t::CLIENT(p->first.v))) {
+	C_MDC_RejoinSessionsOpened *finish = new C_MDC_RejoinSessionsOpened(this, rejoin_client_map);
+	version_t pv = mds->server->prepare_force_open_sessions(rejoin_client_map, finish->sseqmap);
+	ESessions *le = new ESessions(pv, rejoin_client_map);
+	mds->mdlog->start_submit_entry(le, finish);
+	mds->mdlog->flush();
+	rejoin_client_map.clear();
+	return true;
+      }
+    }
+    rejoin_client_map.clear();
+
+    // process caps that were exported by slave rename
+    for (map > >::iterator p = rejoin_slave_exports.begin();
+	 p != rejoin_slave_exports.end();
+	 ++p) {
+      CInode *in = get_inode(p->first);
+      assert(in);
+      for (map::iterator q = p->second.second.begin();
+	   q != p->second.second.end();
+	   ++q) {
+	Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+	assert(session);
+
+	Capability *cap = in->get_client_cap(q->first);
+	if (!cap)
+	  cap = in->add_client_cap(q->first, session);
+	cap->merge(q->second, true);
+
+	Capability::Import& im = rejoin_imported_caps[p->second.first][p->first][q->first];
+	assert(cap->get_last_seq() == im.issue_seq);
+	assert(cap->get_mseq() == im.mseq);
+	cap->set_cap_id(im.cap_id);
+	// send cap import because we assigned a new cap ID
+	do_cap_import(session, in, cap, q->second.cap_id, q->second.seq, q->second.mseq - 1,
+		      p->second.first, CEPH_CAP_FLAG_AUTH);
+      }
+    }
+    rejoin_slave_exports.clear();
+    rejoin_imported_caps.clear();
+
+    // process cap imports
+    //  ino -> client -> frommds -> capex
+    for (map > >::iterator p = cap_imports.begin();
+	 p != cap_imports.end(); ) {
+      CInode *in = get_inode(p->first);
+      if (!in) {
+	dout(10) << " still missing ino " << p->first
+	         << ", will try again after replayed client requests" << dendl;
+	++p;
+	continue;
+      }
+      assert(in->is_auth());
+      for (map >::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q) {
+	Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+	assert(session);
+	for (map::iterator r = q->second.begin();
+	     r != q->second.end();
+	     ++r) {
+	  add_reconnected_cap(in, q->first, inodeno_t(r->second.snaprealm));
+	  Capability *cap = in->reconnect_cap(q->first, r->second, session);
+	  if (r->first >= 0) {
+	    if (cap->get_last_seq() == 0) // don't increase mseq if cap already exists
+	      cap->inc_mseq();
+	    do_cap_import(session, in, cap, r->second.cap_id, 0, 0, r->first, 0);
+
+	    Capability::Import& im = rejoin_imported_caps[r->first][p->first][q->first];
+	    im.cap_id = cap->get_cap_id();
+	    im.issue_seq = cap->get_last_seq();
+	    im.mseq = cap->get_mseq();
+	  }
+	}
+      }
+      cap_imports.erase(p++);  // remove and move on
+    }
+  } else {
+    trim_non_auth();
+
+    rejoin_gather.erase(mds->get_nodeid());
+    maybe_send_pending_rejoins();
+
+    if (rejoin_gather.empty() && rejoin_ack_gather.count(mds->get_nodeid()))
+      rejoin_gather_finish();
+  }
+  return false;
+}
+
+void MDCache::check_realm_past_parents(SnapRealm *realm)
+{
+  // are this realm's parents fully open?
+  if (realm->have_past_parents_open()) {
+    dout(10) << " have past snap parents for realm " << *realm 
+	     << " on " << *realm->inode << dendl;
+  } else {
+    if (!missing_snap_parents.count(realm->inode)) {
+      dout(10) << " MISSING past snap parents for realm " << *realm
+	       << " on " << *realm->inode << dendl;
+      realm->inode->get(CInode::PIN_OPENINGSNAPPARENTS);
+      missing_snap_parents[realm->inode].size();   // just to get it into the map!
+    } else {
+      dout(10) << " (already) MISSING past snap parents for realm " << *realm 
+	       << " on " << *realm->inode << dendl;
+    }
+  }
+}
+
+/*
+ * choose lock states based on reconnected caps
+ */
+void MDCache::choose_lock_states_and_reconnect_caps()
+{
+  dout(10) << "choose_lock_states_and_reconnect_caps" << dendl;
+
+  map splits;
+
+  for (ceph::unordered_map::iterator i = inode_map.begin();
+       i != inode_map.end();
+       ++i) {
+    CInode *in = i->second;
+ 
+    if (in->is_auth() && !in->is_base() && in->inode.is_dirty_rstat())
+      in->mark_dirty_rstat();
+
+    in->choose_lock_states();
+    dout(15) << " chose lock states on " << *in << dendl;
+
+    SnapRealm *realm = in->find_snaprealm();
+
+    check_realm_past_parents(realm);
+
+    map >::iterator p = reconnected_caps.find(in);
+    if (p != reconnected_caps.end()) {
+
+      // also, make sure client's cap is in the correct snaprealm.
+      for (map::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q) {
+	if (q->second == realm->inode->ino()) {
+	  dout(15) << "  client." << q->first << " has correct realm " << q->second << dendl;
+	} else {
+	  dout(15) << "  client." << q->first << " has wrong realm " << q->second
+		   << " != " << realm->inode->ino() << dendl;
+	  if (realm->have_past_parents_open()) {
+	    // ok, include in a split message _now_.
+	    prepare_realm_split(realm, q->first, in->ino(), splits);
+	  } else {
+	    // send the split later.
+	    missing_snap_parents[realm->inode][q->first].insert(in->ino());
+	  }
+	}
+      }
+    }
+  }    
+  reconnected_caps.clear();
+
+  send_snaps(splits);
+}
+
+void MDCache::prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino,
+				  map& splits)
+{
+  MClientSnap *snap;
+  if (splits.count(client) == 0) {
+    splits[client] = snap = new MClientSnap(CEPH_SNAP_OP_SPLIT);
+    snap->head.split = realm->inode->ino();
+    realm->build_snap_trace(snap->bl);
+
+    for (set::iterator p = realm->open_children.begin();
+	 p != realm->open_children.end();
+	 ++p)
+      snap->split_realms.push_back((*p)->inode->ino());
+    
+  } else 
+    snap = splits[client]; 
+  snap->split_inos.push_back(ino);	
+}
+
+void MDCache::send_snaps(map& splits)
+{
+  dout(10) << "send_snaps" << dendl;
+  
+  for (map::iterator p = splits.begin();
+       p != splits.end();
+       ++p) {
+    Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->first.v));
+    if (session) {
+      dout(10) << " client." << p->first
+	       << " split " << p->second->head.split
+	       << " inos " << p->second->split_inos
+	       << dendl;
+      mds->send_message_client_counted(p->second, session);
+    } else {
+      dout(10) << " no session for client." << p->first << dendl;
+      p->second->put();
+    }
+  }
+  splits.clear();
+}
+
+
+/*
+ * remove any items from logsegment open_file lists that don't have
+ * any caps
+ */
+void MDCache::clean_open_file_lists()
+{
+  dout(10) << "clean_open_file_lists" << dendl;
+  
+  for (map::iterator p = mds->mdlog->segments.begin();
+       p != mds->mdlog->segments.end();
+       ++p) {
+    LogSegment *ls = p->second;
+    
+    elist::iterator q = ls->open_files.begin(member_offset(CInode, item_open_file));
+    while (!q.end()) {
+      CInode *in = *q;
+      ++q;
+      if (!in->is_any_caps_wanted()) {
+	dout(10) << " unlisting unwanted/capless inode " << *in << dendl;
+	in->item_open_file.remove_myself();
+      }
+    }
+  }
+}
+
+
+
+Capability* MDCache::rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, int frommds)
+{
+  dout(10) << "rejoin_import_cap for client." << client << " from mds." << frommds
+	   << " on " << *in << dendl;
+  Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(client.v));
+  assert(session);
+
+  Capability *cap = in->reconnect_cap(client, icr, session);
+
+  if (frommds >= 0) {
+    if (cap->get_last_seq() == 0) // don't increase mseq if cap already exists
+      cap->inc_mseq();
+    do_cap_import(session, in, cap, icr.cap_id, 0, 0, frommds, 0);
+  }
+
+  return cap;
+}
+
+void MDCache::export_remaining_imported_caps()
+{
+  dout(10) << "export_remaining_imported_caps" << dendl;
+
+  stringstream warn_str;
+
+  for (map > >::iterator p = cap_imports.begin();
+       p != cap_imports.end();
+       ++p) {
+    warn_str << " ino " << p->first << "\n";
+    for (map >::iterator q = p->second.begin();
+	q != p->second.end();
+	++q) {
+      Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+      if (session) {
+	// mark client caps stale.
+	MClientCaps *stale = new MClientCaps(CEPH_CAP_OP_EXPORT, p->first, 0, 0, 0);
+	stale->set_cap_peer(0, 0, 0, -1, 0);
+	mds->send_message_client_counted(stale, q->first);
+      }
+    }
+  }
+
+  cap_imports.clear();
+
+  if (warn_str.peek() != EOF) {
+    mds->clog.warn() << "failed to reconnect caps for missing inodes:" << "\n";
+    mds->clog.warn(warn_str);
+  }
+}
+
+void MDCache::try_reconnect_cap(CInode *in, Session *session)
+{
+  client_t client = session->info.get_client();
+  ceph_mds_cap_reconnect *rc = get_replay_cap_reconnect(in->ino(), client);
+  if (rc) {
+    in->reconnect_cap(client, *rc, session);
+    dout(10) << "try_reconnect_cap client." << client
+	     << " reconnect wanted " << ccap_string(rc->wanted)
+	     << " issue " << ccap_string(rc->issued)
+	     << " on " << *in << dendl;
+    remove_replay_cap_reconnect(in->ino(), client);
+
+    if (in->is_replicated()) {
+      mds->locker->try_eval(in, CEPH_CAP_LOCKS);
+    } else {
+      in->choose_lock_states();
+      dout(15) << " chose lock states on " << *in << dendl;
+    }
+  }
+}
+
+
+
+// -------
+// cap imports and delayed snap parent opens
+
+void MDCache::do_cap_import(Session *session, CInode *in, Capability *cap,
+			    uint64_t p_cap_id, ceph_seq_t p_seq, ceph_seq_t p_mseq,
+			    int peer, int p_flags)
+{
+  client_t client = session->info.inst.name.num();
+  SnapRealm *realm = in->find_snaprealm();
+  if (realm->have_past_parents_open()) {
+    dout(10) << "do_cap_import " << session->info.inst.name << " mseq " << cap->get_mseq() << " on " << *in << dendl;
+    if (cap->get_last_seq() == 0) // reconnected cap
+      cap->inc_last_seq();
+    cap->set_last_issue();
+    cap->set_last_issue_stamp(ceph_clock_now(g_ceph_context));
+    cap->clear_new();
+    MClientCaps *reap = new MClientCaps(CEPH_CAP_OP_IMPORT,
+					in->ino(),
+					realm->inode->ino(),
+					cap->get_cap_id(), cap->get_last_seq(),
+					cap->pending(), cap->wanted(), 0,
+					cap->get_mseq());
+    in->encode_cap_message(reap, cap);
+    realm->build_snap_trace(reap->snapbl);
+    reap->set_cap_peer(p_cap_id, p_seq, p_mseq, peer, p_flags);
+    mds->send_message_client_counted(reap, session);
+  } else {
+    dout(10) << "do_cap_import missing past snap parents, delaying " << session->info.inst.name << " mseq "
+	     << cap->get_mseq() << " on " << *in << dendl;
+    in->auth_pin(this);
+    cap->inc_suppress();
+    delayed_imported_caps[client].insert(in);
+    missing_snap_parents[in].size();
+  }
+}
+
+void MDCache::do_delayed_cap_imports()
+{
+  dout(10) << "do_delayed_cap_imports" << dendl;
+
+  assert(delayed_imported_caps.empty());
+#if 0
+  map > d;
+  d.swap(delayed_imported_caps);
+
+  for (map >::iterator p = d.begin();
+       p != d.end();
+       ++p) {
+    for (set::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      CInode *in = *q;
+      Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->first.v));
+      if (session) {
+	Capability *cap = in->get_client_cap(p->first);
+	if (cap) {
+	  do_cap_import(session, in, cap);  // note: this may fail and requeue!
+	  cap->dec_suppress();
+	}
+      }
+      in->auth_unpin(this);
+
+      if (in->is_head())
+	mds->locker->issue_caps(in);
+    }
+  }    
+#endif
+}
+
+struct C_MDC_OpenSnapParents : public Context {
+  MDCache *mdcache;
+  C_MDC_OpenSnapParents(MDCache *c) : mdcache(c) {}
+  void finish(int r) {
+    mdcache->open_snap_parents();
+  }
+};
+
+void MDCache::open_snap_parents()
+{
+  dout(10) << "open_snap_parents" << dendl;
+  
+  map splits;
+  C_GatherBuilder gather(g_ceph_context);
+
+  map > >::iterator p = missing_snap_parents.begin();
+  while (p != missing_snap_parents.end()) {
+    CInode *in = p->first;
+    assert(in->snaprealm);
+    if (in->snaprealm->open_parents(gather.new_sub())) {
+      dout(10) << " past parents now open on " << *in << dendl;
+      
+      // include in a (now safe) snap split?
+      for (map >::iterator q = p->second.begin();
+	   q != p->second.end();
+	   ++q)
+	for (set::iterator r = q->second.begin();
+	     r != q->second.end();
+	     ++r) 
+	  prepare_realm_split(in->snaprealm, q->first, *r, splits);
+
+      missing_snap_parents.erase(p++);
+
+      in->put(CInode::PIN_OPENINGSNAPPARENTS);
+
+      // finish off client snaprealm reconnects?
+      map >::iterator q = reconnected_snaprealms.find(in->ino());
+      if (q != reconnected_snaprealms.end()) {
+	for (map::iterator r = q->second.begin();
+	     r != q->second.end();
+	     ++r)
+	  finish_snaprealm_reconnect(r->first, in->snaprealm, r->second);
+      	reconnected_snaprealms.erase(q);
+      }
+    } else {
+      dout(10) << " opening past parents on " << *in << dendl;
+      ++p;
+    }
+  }
+
+  send_snaps(splits);
+
+  if (gather.has_subs()) {
+    dout(10) << "open_snap_parents - waiting for "
+	     << gather.num_subs_remaining() << dendl;
+    gather.set_finisher(new C_MDC_OpenSnapParents(this));
+    gather.activate();
+  } else {
+    if (!reconnected_snaprealms.empty()) {
+      stringstream warn_str;
+      for (map >::iterator p = reconnected_snaprealms.begin();
+	   p != reconnected_snaprealms.end();
+	   ++p) {
+	warn_str << " unconnected snaprealm " << p->first << "\n";
+	for (map::iterator q = p->second.begin();
+	     q != p->second.end();
+	     ++q)
+	  warn_str << "  client." << q->first << " snapid " << q->second << "\n";
+      }
+      mds->clog.warn() << "open_snap_parents has:" << "\n";
+      mds->clog.warn(warn_str);
+    }
+    assert(rejoin_waiters.empty());
+    assert(missing_snap_parents.empty());
+    dout(10) << "open_snap_parents - all open" << dendl;
+    do_delayed_cap_imports();
+
+    start_files_to_recover(rejoin_recover_q, rejoin_check_q);
+    mds->rejoin_done();
+  }
+}
+
+bool MDCache::open_undef_inodes_dirfrags()
+{
+  dout(10) << "open_undef_inodes_dirfrags "
+	   << rejoin_undef_inodes.size() << " inodes "
+	   << rejoin_undef_dirfrags.size() << " dirfrags" << dendl;
+
+  set fetch_queue = rejoin_undef_dirfrags;
+
+  for (set::iterator p = rejoin_undef_inodes.begin();
+       p != rejoin_undef_inodes.end();
+       ++p) {
+    CInode *in = *p;
+    assert(!in->is_base());
+    fetch_queue.insert(in->get_parent_dir());
+  }
+
+  if (fetch_queue.empty())
+    return false;
+
+  C_GatherBuilder gather(g_ceph_context, new C_MDC_RejoinGatherFinish(this));
+  for (set::iterator p = fetch_queue.begin();
+       p != fetch_queue.end();
+       ++p) {
+    CDir *dir = *p;
+    CInode *diri = dir->get_inode();
+    if (diri->state_test(CInode::STATE_REJOINUNDEF))
+      continue;
+    if (dir->state_test(CDir::STATE_REJOINUNDEF))
+      assert(diri->dirfragtree.is_leaf(dir->get_frag()));
+    dir->fetch(gather.new_sub());
+  }
+  assert(gather.has_subs());
+  gather.activate();
+  return true;
+}
+
+void MDCache::opened_undef_inode(CInode *in) {
+  dout(10) << "opened_undef_inode " << *in << dendl;
+  rejoin_undef_inodes.erase(in);
+  if (in->is_dir()) {
+    if (in->has_dirfrags() && !in->dirfragtree.is_leaf(frag_t())) {
+      CDir *dir = in->get_dirfrag(frag_t());
+      assert(dir);
+      rejoin_undef_dirfrags.erase(dir);
+      in->force_dirfrags();
+      list ls;
+      in->get_dirfrags(ls);
+      for (list::iterator p = ls.begin(); p != ls.end(); ++p)
+	rejoin_undef_dirfrags.insert(*p);
+    }
+  }
+}
+
+void MDCache::finish_snaprealm_reconnect(client_t client, SnapRealm *realm, snapid_t seq)
+{
+  if (seq < realm->get_newest_seq()) {
+    dout(10) << "finish_snaprealm_reconnect client." << client << " has old seq " << seq << " < " 
+	     << realm->get_newest_seq()
+    	     << " on " << *realm << dendl;
+    // send an update
+    Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(client.v));
+    if (session) {
+      MClientSnap *snap = new MClientSnap(CEPH_SNAP_OP_UPDATE);
+      realm->build_snap_trace(snap->bl);
+      mds->send_message_client_counted(snap, session);
+    } else {
+      dout(10) << " ...or not, no session for this client!" << dendl;
+    }
+  } else {
+    dout(10) << "finish_snaprealm_reconnect client." << client << " up to date"
+	     << " on " << *realm << dendl;
+  }
+}
+
+
+
+void MDCache::rejoin_send_acks()
+{
+  dout(7) << "rejoin_send_acks" << dendl;
+
+  // replicate stray
+  for (map >::iterator p = rejoin_unlinked_inodes.begin();
+       p != rejoin_unlinked_inodes.end();
+       ++p) {
+    for (set::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      CInode *in = *q;
+      dout(7) << " unlinked inode " << *in << dendl;
+      // inode expired
+      if (!in->is_replica(p->first))
+	continue;
+      while (1) {
+	CDentry *dn = in->get_parent_dn();
+	if (dn->is_replica(p->first))
+	  break;
+	dn->add_replica(p->first);
+	CDir *dir = dn->get_dir();
+	if (dir->is_replica(p->first))
+	  break;
+	dir->add_replica(p->first);
+	in = dir->get_inode();
+	if (in->is_replica(p->first))
+	  break;
+	if (in->is_base())
+	  break;
+      }
+    }
+  }
+  rejoin_unlinked_inodes.clear();
+  
+  // send acks to everyone in the recovery set
+  map ack;
+  for (set::iterator p = recovery_set.begin();
+       p != recovery_set.end();
+       ++p) 
+    ack[*p] = new MMDSCacheRejoin(MMDSCacheRejoin::OP_ACK);
+  
+  // walk subtrees
+  for (map >::iterator p = subtrees.begin(); 
+       p != subtrees.end();
+       ++p) {
+    CDir *dir = p->first;
+    if (!dir->is_auth())
+      continue;
+    dout(10) << "subtree " << *dir << dendl;
+    
+    // auth items in this subtree
+    list dq;
+    dq.push_back(dir);
+
+    while (!dq.empty()) {
+      CDir *dir = dq.front();
+      dq.pop_front();
+      
+      // dir
+      for (map::iterator r = dir->replicas_begin();
+	   r != dir->replicas_end();
+	   ++r) {
+	ack[r->first]->add_strong_dirfrag(dir->dirfrag(), ++r->second, dir->dir_rep);
+	ack[r->first]->add_dirfrag_base(dir);
+      }
+	   
+      for (CDir::map_t::iterator q = dir->items.begin();
+	   q != dir->items.end();
+	   ++q) {
+	CDentry *dn = q->second;
+	CDentry::linkage_t *dnl = dn->get_linkage();
+
+	// inode
+	CInode *in = NULL;
+	if (dnl->is_primary())
+	  in = dnl->get_inode();
+
+	// dentry
+	for (map::iterator r = dn->replicas_begin();
+	     r != dn->replicas_end();
+	     ++r) {
+	  ack[r->first]->add_strong_dentry(dir->dirfrag(), dn->name, dn->first, dn->last,
+					   dnl->is_primary() ? dnl->get_inode()->ino():inodeno_t(0),
+					   dnl->is_remote() ? dnl->get_remote_ino():inodeno_t(0),
+					   dnl->is_remote() ? dnl->get_remote_d_type():0,
+					   ++r->second,
+					   dn->lock.get_replica_state());
+	  // peer missed MDentrylink message ?
+	  if (in && !in->is_replica(r->first))
+	    in->add_replica(r->first);
+	}
+	
+	if (!in)
+	  continue;
+
+	for (map::iterator r = in->replicas_begin();
+	     r != in->replicas_end();
+	     ++r) {
+	  ack[r->first]->add_inode_base(in);
+	  bufferlist bl;
+	  in->_encode_locks_state_for_rejoin(bl, r->first);
+	  ack[r->first]->add_inode_locks(in, ++r->second, bl);
+	}
+	
+	// subdirs in this subtree?
+	in->get_nested_dirfrags(dq);
+      }
+    }
+  }
+
+  // base inodes too
+  if (root && root->is_auth()) 
+    for (map::iterator r = root->replicas_begin();
+	 r != root->replicas_end();
+	 ++r) {
+      ack[r->first]->add_inode_base(root);
+      bufferlist bl;
+      root->_encode_locks_state_for_rejoin(bl, r->first);
+      ack[r->first]->add_inode_locks(root, ++r->second, bl);
+    }
+  if (myin)
+    for (map::iterator r = myin->replicas_begin();
+	 r != myin->replicas_end();
+	 ++r) {
+      ack[r->first]->add_inode_base(myin);
+      bufferlist bl;
+      myin->_encode_locks_state_for_rejoin(bl, r->first);
+      ack[r->first]->add_inode_locks(myin, ++r->second, bl);
+    }
+
+  // include inode base for any inodes whose scatterlocks may have updated
+  for (set::iterator p = rejoin_potential_updated_scatterlocks.begin();
+       p != rejoin_potential_updated_scatterlocks.end();
+       ++p) {
+    CInode *in = *p;
+    for (map::iterator r = in->replicas_begin();
+	 r != in->replicas_end();
+	 ++r)
+      ack[r->first]->add_inode_base(in);
+  }
+
+  // send acks
+  for (map::iterator p = ack.begin();
+       p != ack.end();
+       ++p) {
+    ::encode(rejoin_imported_caps[p->first], p->second->imported_caps);
+    mds->send_message_mds(p->second, p->first);
+  }
+
+  rejoin_imported_caps.clear();
+}
+
+
+void MDCache::reissue_all_caps()
+{
+  dout(10) << "reissue_all_caps" << dendl;
+
+  for (ceph::unordered_map::iterator p = inode_map.begin();
+       p != inode_map.end();
+       ++p) {
+    CInode *in = p->second;
+    if (in->is_head() && in->is_any_caps()) {
+      if (!mds->locker->eval(in, CEPH_CAP_LOCKS))
+	mds->locker->issue_caps(in);
+    }
+  }
+}
+
+
+// ===============================================================================
+
+struct C_MDC_QueuedCow : public Context {
+  MDCache *mdcache;
+  CInode *in;
+  MutationRef mut;
+  C_MDC_QueuedCow(MDCache *mdc, CInode *i, MutationRef& m) :
+    mdcache(mdc), in(i), mut(m) {}
+  void finish(int r) {
+    mdcache->_queued_file_recover_cow(in, mut);
+  }
+};
+
+void MDCache::queue_file_recover(CInode *in)
+{
+  dout(10) << "queue_file_recover " << *in << dendl;
+  assert(in->is_auth());
+
+  // cow?
+  SnapRealm *realm = in->find_snaprealm();
+  set s = realm->get_snaps();
+  while (!s.empty() && *s.begin() < in->first)
+    s.erase(s.begin());
+  while (!s.empty() && *s.rbegin() > in->last)
+    s.erase(*s.rbegin());
+  dout(10) << " snaps in [" << in->first << "," << in->last << "] are " << s << dendl;
+  if (s.size() > 1) {
+    inode_t *pi = in->project_inode();
+    pi->version = in->pre_dirty();
+
+    MutationRef mut(new MutationImpl);
+    mut->ls = mds->mdlog->get_current_segment();
+    EUpdate *le = new EUpdate(mds->mdlog, "queue_file_recover cow");
+    mds->mdlog->start_entry(le);
+    predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY);
+
+    s.erase(*s.begin());
+    while (!s.empty()) {
+      snapid_t snapid = *s.begin();
+      CInode *cow_inode = 0;
+      journal_cow_inode(mut, &le->metablob, in, snapid-1, &cow_inode);
+      assert(cow_inode);
+      _queue_file_recover(cow_inode);
+      s.erase(*s.begin());
+    }
+    
+    in->parent->first = in->first;
+    le->metablob.add_primary_dentry(in->parent, in, true);
+    mds->mdlog->submit_entry(le, new C_MDC_QueuedCow(this, in, mut));
+    mds->mdlog->flush();
+  }
+
+  _queue_file_recover(in);
+}
+
+void MDCache::_queued_file_recover_cow(CInode *in, MutationRef& mut)
+{
+  in->pop_and_dirty_projected_inode(mut->ls);
+  mut->apply();
+  mds->locker->drop_locks(mut.get());
+  mut->cleanup();
+}
+
+void MDCache::_queue_file_recover(CInode *in)
+{
+  dout(15) << "_queue_file_recover " << *in << dendl;
+  assert(in->is_auth());
+  in->state_clear(CInode::STATE_NEEDSRECOVER);
+  if (!in->state_test(CInode::STATE_RECOVERING)) {
+    in->state_set(CInode::STATE_RECOVERING);
+    in->auth_pin(this);
+  }
+  file_recover_queue.insert(in);
+}
+
+void MDCache::unqueue_file_recover(CInode *in)
+{
+  dout(15) << "unqueue_file_recover " << *in << dendl;
+  in->state_clear(CInode::STATE_RECOVERING);
+  in->auth_unpin(this);
+  file_recover_queue.erase(in);
+}
+
+/*
+ * called after recovery to recover file sizes for previously opened (for write)
+ * files.  that is, those where max_size > size.
+ */
+void MDCache::identify_files_to_recover(vector& recover_q, vector& check_q)
+{
+  dout(10) << "identify_files_to_recover" << dendl;
+  for (ceph::unordered_map::iterator p = inode_map.begin();
+       p != inode_map.end();
+       ++p) {
+    CInode *in = p->second;
+    if (!in->is_auth())
+      continue;
+    
+    bool recover = false;
+    for (map::iterator p = in->inode.client_ranges.begin();
+	 p != in->inode.client_ranges.end();
+	 ++p) {
+      Capability *cap = in->get_client_cap(p->first);
+      if (!cap) {
+	dout(10) << " client." << p->first << " has range " << p->second << " but no cap on " << *in << dendl;
+	recover = true;
+	break;
+      }
+    }
+
+    if (recover) {
+      in->auth_pin(&in->filelock);
+      in->filelock.set_state(LOCK_PRE_SCAN);
+      recover_q.push_back(in);
+      
+      // make sure past parents are open/get opened
+      SnapRealm *realm = in->find_snaprealm();
+      check_realm_past_parents(realm);
+    } else {
+      check_q.push_back(in);
+    }
+  }
+}
+
+void MDCache::start_files_to_recover(vector& recover_q, vector& check_q)
+{
+  for (vector::iterator p = check_q.begin(); p != check_q.end(); ++p) {
+    CInode *in = *p;
+    mds->locker->check_inode_max_size(in);
+  }
+  for (vector::iterator p = recover_q.begin(); p != recover_q.end(); ++p) {
+    CInode *in = *p;
+    mds->locker->file_recover(&in->filelock);
+  }
+}
+
+struct C_MDC_Recover : public Context {
+  MDCache *mdc;
+  CInode *in;
+  uint64_t size;
+  utime_t mtime;
+  C_MDC_Recover(MDCache *m, CInode *i) : mdc(m), in(i), size(0) {}
+  void finish(int r) {
+    mdc->_recovered(in, r, size, mtime);
+  }
+};
+
+void MDCache::do_file_recover()
+{
+  dout(10) << "do_file_recover " << file_recover_queue.size() << " queued, "
+	   << file_recovering.size() << " recovering" << dendl;
+
+  while (file_recovering.size() < 5 &&
+	 !file_recover_queue.empty()) {
+    CInode *in = *file_recover_queue.begin();
+    file_recover_queue.erase(in);
+
+    inode_t *pi = in->get_projected_inode();
+
+    // blech
+    if (pi->client_ranges.size() && !pi->get_max_size()) {
+      mds->clog.warn() << "bad client_range " << pi->client_ranges
+	  << " on ino " << pi->ino << "\n";
+    }
+
+    if (pi->client_ranges.size() && pi->get_max_size()) {
+      dout(10) << "do_file_recover starting " << in->inode.size << " " << pi->client_ranges
+	       << " " << *in << dendl;
+      file_recovering.insert(in);
+      
+      C_MDC_Recover *fin = new C_MDC_Recover(this, in);
+      mds->filer->probe(in->inode.ino, &in->inode.layout, in->last,
+			pi->get_max_size(), &fin->size, &fin->mtime, false,
+			0, fin);    
+    } else {
+      dout(10) << "do_file_recover skipping " << in->inode.size
+	       << " " << *in << dendl;
+      in->state_clear(CInode::STATE_RECOVERING);
+      mds->locker->eval(in, CEPH_LOCK_IFILE);
+      in->auth_unpin(this);
+    }
+  }
+}
+
+void MDCache::_recovered(CInode *in, int r, uint64_t size, utime_t mtime)
+{
+  dout(10) << "_recovered r=" << r << " size=" << size << " mtime=" << mtime
+	   << " for " << *in << dendl;
+
+  if (r != 0) {
+    dout(0) << "recovery error! " << r << dendl;
+    if (r == -EBLACKLISTED) {
+      mds->suicide();
+      return;
+    }
+    assert(0 == "unexpected error from osd during recovery");
+  }
+
+  file_recovering.erase(in);
+  in->state_clear(CInode::STATE_RECOVERING);
+
+  if (!in->get_parent_dn() && !in->get_projected_parent_dn()) {
+    dout(10) << " inode has no parents, killing it off" << dendl;
+    in->auth_unpin(this);
+    remove_inode(in);
+  } else {
+    // journal
+    mds->locker->check_inode_max_size(in, true, true, size, false, 0, mtime);
+    mds->locker->eval(in, CEPH_LOCK_IFILE);
+    in->auth_unpin(this);
+  }
+
+  do_file_recover();
+}
+
+void MDCache::purge_prealloc_ino(inodeno_t ino, Context *fin)
+{
+  object_t oid = CInode::get_object_name(ino, frag_t(), "");
+  object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+
+  dout(10) << "purge_prealloc_ino " << ino << " oid " << oid << dendl;
+  SnapContext snapc;
+  mds->objecter->remove(oid, oloc, snapc, ceph_clock_now(g_ceph_context), 0, 0, fin);
+}  
+
+
+
+
+// ===============================================================================
+
+
+
+// ----------------------------
+// truncate
+
+void MDCache::truncate_inode(CInode *in, LogSegment *ls)
+{
+  inode_t *pi = in->get_projected_inode();
+  dout(10) << "truncate_inode "
+	   << pi->truncate_from << " -> " << pi->truncate_size
+	   << " on " << *in
+	   << dendl;
+
+  ls->truncating_inodes.insert(in);
+  in->get(CInode::PIN_TRUNCATING);
+
+  _truncate_inode(in, ls);
+}
+
+struct C_MDC_TruncateFinish : public Context {
+  MDCache *mdc;
+  CInode *in;
+  LogSegment *ls;
+  C_MDC_TruncateFinish(MDCache *c, CInode *i, LogSegment *l) :
+    mdc(c), in(i), ls(l) {}
+  void finish(int r) {
+    assert(r == 0 || r == -ENOENT);
+    mdc->truncate_inode_finish(in, ls);
+  }
+};
+
+void MDCache::_truncate_inode(CInode *in, LogSegment *ls)
+{
+  inode_t *pi = &in->inode;
+  dout(10) << "_truncate_inode "
+	   << pi->truncate_from << " -> " << pi->truncate_size
+	   << " on " << *in << dendl;
+
+  assert(pi->is_truncating());
+  assert(pi->truncate_size < (1ULL << 63));
+  assert(pi->truncate_from < (1ULL << 63));
+  assert(pi->truncate_size < pi->truncate_from);
+
+  in->auth_pin(this);
+
+  SnapRealm *realm = in->find_snaprealm();
+  SnapContext nullsnap;
+  const SnapContext *snapc;
+  if (realm) {
+    dout(10) << " realm " << *realm << dendl;
+    snapc = &realm->get_snap_context();
+  } else {
+    dout(10) << " NO realm, using null context" << dendl;
+    snapc = &nullsnap;
+    assert(in->last == CEPH_NOSNAP);
+  }
+  dout(10) << "_truncate_inode  snapc " << snapc << " on " << *in << dendl;
+  mds->filer->truncate(in->inode.ino, &in->inode.layout, *snapc,
+		       pi->truncate_size, pi->truncate_from-pi->truncate_size, pi->truncate_seq, utime_t(), 0,
+		       0, new C_MDC_TruncateFinish(this, in, ls));
+}
+
+struct C_MDC_TruncateLogged : public Context {
+  MDCache *mdc;
+  CInode *in;
+  MutationRef mut;
+  C_MDC_TruncateLogged(MDCache *m, CInode *i, MutationRef& mu) :
+    mdc(m), in(i), mut(mu) {}
+  void finish(int r) {
+    mdc->truncate_inode_logged(in, mut);
+  }
+};
+
+void MDCache::truncate_inode_finish(CInode *in, LogSegment *ls)
+{
+  dout(10) << "truncate_inode_finish " << *in << dendl;
+  
+  set::iterator p = ls->truncating_inodes.find(in);
+  assert(p != ls->truncating_inodes.end());
+  ls->truncating_inodes.erase(p);
+
+  // update
+  inode_t *pi = in->project_inode();
+  pi->version = in->pre_dirty();
+  pi->truncate_from = 0;
+  pi->truncate_pending--;
+
+  MutationRef mut(new MutationImpl);
+  mut->ls = mds->mdlog->get_current_segment();
+  mut->add_projected_inode(in);
+
+  EUpdate *le = new EUpdate(mds->mdlog, "truncate finish");
+  mds->mdlog->start_entry(le);
+  CDentry *dn = in->get_projected_parent_dn();
+  le->metablob.add_dir_context(dn->get_dir());
+  le->metablob.add_primary_dentry(dn, in, true);
+  le->metablob.add_truncate_finish(in->ino(), ls->offset);
+
+  journal_dirty_inode(mut.get(), &le->metablob, in);
+  mds->mdlog->submit_entry(le, new C_MDC_TruncateLogged(this, in, mut));
+
+  // flush immediately if there are readers/writers waiting
+  if (in->get_caps_wanted() & (CEPH_CAP_FILE_RD|CEPH_CAP_FILE_WR))
+    mds->mdlog->flush();
+}
+
+void MDCache::truncate_inode_logged(CInode *in, MutationRef& mut)
+{
+  dout(10) << "truncate_inode_logged " << *in << dendl;
+  mut->apply();
+  mds->locker->drop_locks(mut.get());
+  mut->cleanup();
+
+  in->put(CInode::PIN_TRUNCATING);
+  in->auth_unpin(this);
+
+  list waiters;
+  in->take_waiting(CInode::WAIT_TRUNC, waiters);
+  mds->queue_waiters(waiters);
+}
+
+
+void MDCache::add_recovered_truncate(CInode *in, LogSegment *ls)
+{
+  dout(20) << "add_recovered_truncate " << *in << " in " << ls << " offset " << ls->offset << dendl;
+  ls->truncating_inodes.insert(in);
+  in->get(CInode::PIN_TRUNCATING);
+}
+
+void MDCache::remove_recovered_truncate(CInode *in, LogSegment *ls)
+{
+  dout(20) << "remove_recovered_truncate " << *in << " in " << ls << " offset " << ls->offset << dendl;
+  // if we have the logseg the truncate started in, it must be in our list.
+  set::iterator p = ls->truncating_inodes.find(in);
+  assert(p != ls->truncating_inodes.end());
+  ls->truncating_inodes.erase(p);
+  in->put(CInode::PIN_TRUNCATING);
+}
+
+void MDCache::start_recovered_truncates()
+{
+  dout(10) << "start_recovered_truncates" << dendl;
+  for (map::iterator p = mds->mdlog->segments.begin();
+       p != mds->mdlog->segments.end();
+       ++p) {
+    LogSegment *ls = p->second;
+    for (set::iterator q = ls->truncating_inodes.begin();
+	 q != ls->truncating_inodes.end();
+	 ++q)
+      _truncate_inode(*q, ls);
+  }
+}
+
+
+
+
+
+
+// ================================================================================
+// cache trimming
+
+
+/*
+ * note: only called while MDS is active or stopping... NOT during recovery.
+ * however, we may expire a replica whose authority is recovering.
+ * 
+ */
+bool MDCache::trim(int max) 
+{
+  // trim LRU
+  if (max < 0) {
+    max = g_conf->mds_cache_size;
+    if (!max) return false;
+  }
+  dout(7) << "trim max=" << max << "  cur=" << lru.lru_get_size() << dendl;
+
+  // process delayed eval_stray()
+  for (elist::iterator p = delayed_eval_stray.begin(); !p.end(); ) {
+    CDentry *dn = *p;
+    ++p;
+    dn->item_stray.remove_myself();
+    eval_stray(dn);
+  }
+
+  map expiremap;
+  bool is_standby_replay = mds->is_standby_replay();
+  int unexpirable = 0;
+  list unexpirables;
+  // trim dentries from the LRU
+  while (lru.lru_get_size() + unexpirable > (unsigned)max) {
+    CDentry *dn = static_cast(lru.lru_expire());
+    if (!dn) break;
+    if ((is_standby_replay && dn->get_linkage() &&
+        dn->get_linkage()->inode->item_open_file.is_on_list()) ||
+	trim_dentry(dn, expiremap)) {
+      unexpirables.push_back(dn);
+      ++unexpirable;
+    }
+  }
+  for(list::iterator i = unexpirables.begin();
+      i != unexpirables.end();
+      ++i)
+    lru.lru_insert_mid(*i);
+
+  // trim non-auth, non-bound subtrees
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();) {
+    CDir *dir = p->first;
+    ++p;
+    if (!dir->is_auth() && !dir->get_inode()->is_auth()) {
+      // don't trim subtree root if its auth MDS is recovering.
+      // This simplify the cache rejoin code.
+      if (dir->is_subtree_root() &&
+	  rejoin_ack_gather.count(dir->get_dir_auth().first))
+	continue;
+      if (dir->get_num_ref() == 1)  // subtree pin
+	trim_dirfrag(dir, 0, expiremap);
+    }
+  }
+
+  // trim root?
+  if (max == 0 && root) {
+    list ls;
+    root->get_dirfrags(ls);
+    for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+      CDir *dir = *p;
+      if (dir->get_num_ref() == 1)  // subtree pin
+	trim_dirfrag(dir, 0, expiremap);
+    }
+    if (root->get_num_ref() == 0)
+      trim_inode(0, root, 0, expiremap);
+  }
+
+  // send any expire messages
+  send_expire_messages(expiremap);
+
+  return true;
+}
+
+void MDCache::send_expire_messages(map& expiremap)
+{
+  // send expires
+  for (map::iterator it = expiremap.begin();
+       it != expiremap.end();
+       ++it) {
+    if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN ||
+	(mds->mdsmap->get_state(it->first) == MDSMap::STATE_REJOIN &&
+	 rejoin_sent.count(it->first) == 0)) {
+      it->second->put();
+      continue;
+    }
+    dout(7) << "sending cache_expire to " << it->first << dendl;
+    mds->send_message_mds(it->second, it->first);
+  }
+}
+
+
+bool MDCache::trim_dentry(CDentry *dn, map& expiremap)
+{
+  dout(12) << "trim_dentry " << *dn << dendl;
+  
+  CDentry::linkage_t *dnl = dn->get_linkage();
+
+  CDir *dir = dn->get_dir();
+  assert(dir);
+  
+  CDir *con = get_subtree_root(dir);
+  if (con)
+    dout(12) << " in container " << *con << dendl;
+  else {
+    dout(12) << " no container; under a not-yet-linked dir" << dendl;
+    assert(dn->is_auth());
+  }
+
+  // adjust the dir state
+  // NOTE: we can safely remove a clean, null dentry without effecting
+  //       directory completeness.
+  // (check this _before_ we unlink the inode, below!)
+  bool null_dentry = false;
+  bool clear_complete = false;
+  if (!(dnl->is_null() && dn->is_clean()))
+    clear_complete = true;
+
+  // unlink the dentry
+  if (dnl->is_remote()) {
+    // just unlink.
+    dir->unlink_inode(dn);
+  } else if (dnl->is_primary()) {
+    // expire the inode, too.
+    CInode *in = dnl->get_inode();
+    assert(in);
+    if (trim_inode(dn, in, con, expiremap))
+      return true; // purging stray instead of trimming
+  } else {
+    assert(dnl->is_null());
+    null_dentry = true;
+  }
+
+  // notify dentry authority?
+  if (!dn->is_auth()) {
+    // If null replica dentry is not readable, it's likely we will
+    // receive a MDentryLink message soon. MDentryLink message only
+    // replicates an inode, so we should avoid trimming the inode's
+    // parent dentry. This is because that unconnected replicas are
+    // problematic for subtree migration.
+    if (null_dentry && !dn->lock.can_read(-1) &&
+	!dn->get_dir()->get_inode()->is_stray())
+      return true;
+
+    pair auth = dn->authority();
+    
+    for (int p=0; p<2; p++) {
+      int a = auth.first;
+      if (p) a = auth.second;
+      if (a < 0 || (p == 1 && auth.second == auth.first)) break;
+      if (mds->get_nodeid() == auth.second &&
+	  con->is_importing()) break;                // don't send any expire while importing.
+      if (a == mds->get_nodeid()) continue;          // on export, ignore myself.
+      
+      dout(12) << "  sending expire to mds." << a << " on " << *dn << dendl;
+      assert(a != mds->get_nodeid());
+      if (expiremap.count(a) == 0) 
+	expiremap[a] = new MCacheExpire(mds->get_nodeid());
+      expiremap[a]->add_dentry(con->dirfrag(), dir->dirfrag(), dn->name, dn->last, dn->get_replica_nonce());
+    }
+  }
+
+  // remove dentry
+  if (dir->is_auth())
+    dir->add_to_bloom(dn);
+  dir->remove_dentry(dn);
+
+  if (clear_complete)
+    dir->state_clear(CDir::STATE_COMPLETE);
+  
+  // reexport?
+  if (dir->get_num_head_items() == 0 && dir->is_subtree_root())
+    migrator->export_empty_import(dir);
+  
+  if (mds->logger) mds->logger->inc(l_mds_iex);
+  return false;
+}
+
+
+void MDCache::trim_dirfrag(CDir *dir, CDir *con, map& expiremap)
+{
+  dout(15) << "trim_dirfrag " << *dir << dendl;
+
+  if (dir->is_subtree_root()) {
+    assert(!dir->is_auth() ||
+	   (!dir->is_replicated() && dir->inode->is_base()));
+    remove_subtree(dir);	// remove from subtree map
+  }
+  assert(dir->get_num_ref() == 0);
+
+  CInode *in = dir->get_inode();
+
+  if (!dir->is_auth()) {
+    pair auth = dir->authority();
+    
+    // was this an auth delegation?  (if so, slightly modified container)
+    dirfrag_t condf;
+    if (dir->is_subtree_root()) {
+      dout(12) << " subtree root, container is " << *dir << dendl;
+      con = dir;
+      condf = dir->dirfrag();
+    } else {
+      condf = con->dirfrag();
+    }
+      
+    for (int p=0; p<2; p++) {
+      int a = auth.first;
+      if (p) a = auth.second;
+      if (a < 0 || (p == 1 && auth.second == auth.first)) break;
+      if (mds->get_nodeid() == auth.second &&
+	  con->is_importing()) break;                // don't send any expire while importing.
+      if (a == mds->get_nodeid()) continue;          // on export, ignore myself.
+
+      dout(12) << "  sending expire to mds." << a << " on   " << *dir << dendl;
+      assert(a != mds->get_nodeid());
+      if (expiremap.count(a) == 0) 
+	expiremap[a] = new MCacheExpire(mds->get_nodeid());
+      expiremap[a]->add_dir(condf, dir->dirfrag(), dir->replica_nonce);
+    }
+  }
+  
+  in->close_dirfrag(dir->dirfrag().frag);
+}
+
+bool MDCache::trim_inode(CDentry *dn, CInode *in, CDir *con, map& expiremap)
+{
+  dout(15) << "trim_inode " << *in << dendl;
+  assert(in->get_num_ref() == 0);
+
+  if (in->is_dir()) {
+    // If replica inode's dirfragtreelock is not readable, it's likely
+    // some dirfrags of the inode are being fragmented and we will receive
+    // MMDSFragmentNotify soon. MMDSFragmentNotify only replicates the new
+    // dirfrags, so we should avoid trimming these dirfrags' parent inode.
+    // This is because that unconnected replicas are problematic for
+    // subtree migration.
+    //
+    if (!in->is_auth() && !in->dirfragtreelock.can_read(-1))
+      return true;
+
+    // DIR
+    list dfls;
+    in->get_dirfrags(dfls);
+    for (list::iterator p = dfls.begin(); p != dfls.end(); ++p) {
+      CDir *dir = *p;
+      assert(!dir->is_subtree_root());
+      trim_dirfrag(dir, con ? con:dir, expiremap);  // if no container (e.g. root dirfrag), use *p
+    }
+  }
+  
+  // INODE
+  if (in->is_auth()) {
+    // eval stray after closing dirfrags
+    if (dn) {
+      maybe_eval_stray(in);
+      if (dn->get_num_ref() > 0)
+	return true;
+    }
+  } else {
+    pair auth = in->authority();
+    
+    dirfrag_t df;
+    if (con)
+      df = con->dirfrag();
+    else
+      df = dirfrag_t(0,frag_t());   // must be a root or stray inode.
+
+    for (int p=0; p<2; p++) {
+      int a = auth.first;
+      if (p) a = auth.second;
+      if (a < 0 || (p == 1 && auth.second == auth.first)) break;
+      if (con && mds->get_nodeid() == auth.second &&
+	  con->is_importing()) break;                // don't send any expire while importing.
+      if (a == mds->get_nodeid()) continue;          // on export, ignore myself.
+
+      dout(12) << "  sending expire to mds." << a << " on " << *in << dendl;
+      assert(a != mds->get_nodeid());
+      if (expiremap.count(a) == 0) 
+	expiremap[a] = new MCacheExpire(mds->get_nodeid());
+      expiremap[a]->add_inode(df, in->vino(), in->get_replica_nonce());
+    }
+  }
+
+  /*
+  if (in->is_auth()) {
+    if (in->hack_accessed)
+      mds->logger->inc("outt");
+    else {
+      mds->logger->inc("outut");
+      mds->logger->fset("oututl", ceph_clock_now(g_ceph_context) - in->hack_load_stamp);
+    }
+  }
+  */
+    
+  // unlink
+  if (dn)
+    dn->get_dir()->unlink_inode(dn);
+  remove_inode(in);
+  return false;
+}
+
+
+/**
+ * trim_non_auth - remove any non-auth items from our cache
+ *
+ * this reduces the amount of non-auth metadata in our cache, reducing the 
+ * load incurred by the rejoin phase.
+ *
+ * the only non-auth items that remain are those that are needed to 
+ * attach our own subtrees to the root.  
+ *
+ * when we are done, all dentries will be in the top bit of the lru.
+ *
+ * why we have to do this:
+ *  we may not have accurate linkage for non-auth items.  which means we will 
+ *  know which subtree it falls into, and can not be sure to declare it to the
+ *  correct authority.  
+ */
+void MDCache::trim_non_auth()
+{
+  dout(7) << "trim_non_auth" << dendl;
+  
+  // temporarily pin all subtree roots
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) 
+    p->first->get(CDir::PIN_SUBTREETEMP);
+
+  // note first auth item we see.  
+  // when we see it the second time, stop.
+  CDentry *first_auth = 0;
+  
+  // trim non-auth items from the lru
+  while (lru.lru_get_size() > 0) {
+    CDentry *dn = static_cast(lru.lru_expire());
+    if (!dn) break;
+    CDentry::linkage_t *dnl = dn->get_linkage();
+
+    if (dn->is_auth()) {
+      // add back into lru (at the top)
+      lru.lru_insert_top(dn);
+
+      if (dnl->is_remote() && dnl->get_inode() && !dnl->get_inode()->is_auth())
+	dn->unlink_remote(dnl);
+
+      if (dn->get_dir()->get_inode()->is_stray()) {
+	dn->state_set(CDentry::STATE_STRAY);
+	if (dnl->is_primary() && dnl->get_inode()->inode.nlink == 0)
+	  dnl->get_inode()->state_set(CInode::STATE_ORPHAN);
+      }
+
+      if (!first_auth) {
+	first_auth = dn;
+      } else {
+	if (first_auth == dn) 
+	  break;
+      }
+    } else {
+      // non-auth.  expire.
+      CDir *dir = dn->get_dir();
+      assert(dir);
+
+      // unlink the dentry
+      dout(10) << " removing " << *dn << dendl;
+      if (dnl->is_remote()) {
+	dir->unlink_inode(dn);
+      } 
+      else if (dnl->is_primary()) {
+	CInode *in = dnl->get_inode();
+	dout(10) << " removing " << *in << dendl;
+	list ls;
+	in->get_dirfrags(ls);
+	for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+	  CDir *subdir = *p;
+	  assert(!subdir->is_subtree_root());
+	  in->close_dirfrag(subdir->dirfrag().frag);
+	}
+	dir->unlink_inode(dn);
+	remove_inode(in);
+      } 
+      else {
+	assert(dnl->is_null());
+      }
+
+      assert(!dir->has_bloom());
+      dir->remove_dentry(dn);
+      // adjust the dir state
+      dir->state_clear(CDir::STATE_COMPLETE);  // dir incomplete!
+      // close empty non-auth dirfrag
+      if (!dir->is_subtree_root() && dir->get_num_any() == 0)
+	dir->inode->close_dirfrag(dir->get_frag());
+    }
+  }
+
+  // move everything in the pintail to the top bit of the lru.
+  lru.lru_touch_entire_pintail();
+
+  // unpin all subtrees
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) 
+    p->first->put(CDir::PIN_SUBTREETEMP);
+
+  if (lru.lru_get_size() == 0) {
+    // root, stray, etc.?
+    ceph::unordered_map::iterator p = inode_map.begin();
+    while (p != inode_map.end()) {
+      ceph::unordered_map::iterator next = p;
+      ++next;
+      CInode *in = p->second;
+      if (!in->is_auth()) {
+	list ls;
+	in->get_dirfrags(ls);
+	for (list::iterator p = ls.begin();
+	     p != ls.end();
+	     ++p) {
+	  dout(10) << " removing " << **p << dendl;
+	  assert((*p)->get_num_ref() == 1);  // SUBTREE
+	  remove_subtree((*p));
+	  in->close_dirfrag((*p)->dirfrag().frag);
+	}
+	dout(10) << " removing " << *in << dendl;
+	assert(!in->get_parent_dn());
+	assert(in->get_num_ref() == 0);
+	remove_inode(in);
+      }
+      p = next;
+    }
+  }
+
+  show_subtrees();
+}
+
+/**
+ * Recursively trim the subtree rooted at directory to remove all
+ * CInodes/CDentrys/CDirs that aren't links to remote MDSes, or ancestors
+ * of those links. This is used to clear invalid data out of the cache.
+ * Note that it doesn't clear the passed-in directory, since that's not
+ * always safe.
+ */
+bool MDCache::trim_non_auth_subtree(CDir *dir)
+{
+  dout(10) << "trim_non_auth_subtree(" << dir << ") " << *dir << dendl;
+
+  bool keep_dir = !can_trim_non_auth_dirfrag(dir);
+
+  CDir::map_t::iterator j = dir->begin();
+  CDir::map_t::iterator i = j;
+  while (j != dir->end()) {
+    i = j++;
+    CDentry *dn = i->second;
+    dout(10) << "trim_non_auth_subtree(" << dir << ") Checking dentry " << dn << dendl;
+    CDentry::linkage_t *dnl = dn->get_linkage();
+    if (dnl->is_primary()) { // check for subdirectories, etc
+      CInode *in = dnl->get_inode();
+      bool keep_inode = false;
+      if (in->is_dir()) {
+        list subdirs;
+        in->get_dirfrags(subdirs);
+        for (list::iterator subdir = subdirs.begin();
+            subdir != subdirs.end();
+            ++subdir) {
+          if ((*subdir)->is_subtree_root()) {
+            keep_inode = true;
+            dout(10) << "trim_non_auth_subtree(" << dir << ") keeping " << **subdir << dendl;
+          } else {
+            if (trim_non_auth_subtree(*subdir))
+              keep_inode = true;
+            else {
+              in->close_dirfrag((*subdir)->get_frag());
+              dir->state_clear(CDir::STATE_COMPLETE);  // now incomplete!
+            }
+          }
+        }
+
+      }
+      if (!keep_inode) { // remove it!
+        dout(20) << "trim_non_auth_subtree(" << dir << ") removing inode " << in << " with dentry" << dn << dendl;
+        dir->unlink_inode(dn);
+        remove_inode(in);
+	assert(!dir->has_bloom());
+        dir->remove_dentry(dn);
+      } else {
+        dout(20) << "trim_non_auth_subtree(" << dir << ") keeping inode " << in << " with dentry " << dn <is_null()) { // keep null dentry for slave rollback
+      dout(20) << "trim_non_auth_subtree(" << dir << ") keeping dentry " << dn <is_remote())
+        dir->unlink_inode(dn);
+      dir->remove_dentry(dn);
+    }
+  }
+  /**
+   * We've now checked all our children and deleted those that need it.
+   * Now return to caller, and tell them if *we're* a keeper.
+   */
+  return keep_dir || dir->get_num_any();
+}
+
+/*
+ * during replay, when we determine a subtree is no longer ours, we
+ * try to trim it from our cache.  because subtrees must be connected
+ * to the root, the fact that we can trim this tree may mean that our
+ * children or parents can also be trimmed.
+ */
+void MDCache::try_trim_non_auth_subtree(CDir *dir)
+{
+  dout(10) << "try_trim_nonauth_subtree " << *dir << dendl;
+
+  // can we now trim child subtrees?
+  set bounds;
+  get_subtree_bounds(dir, bounds);
+  for (set::iterator p = bounds.begin(); p != bounds.end(); ++p) {
+    CDir *bd = *p;
+    if (bd->get_dir_auth().first != mds->whoami &&  // we are not auth
+	bd->get_num_any() == 0 && // and empty
+	can_trim_non_auth_dirfrag(bd)) {
+      CInode *bi = bd->get_inode();
+      dout(10) << " closing empty non-auth child subtree " << *bd << dendl;
+      remove_subtree(bd);
+      bd->mark_clean();
+      bi->close_dirfrag(bd->get_frag());
+    }
+  }
+
+  if (trim_non_auth_subtree(dir)) {
+    // keep
+    try_subtree_merge(dir);
+  } else {
+    // can we trim this subtree (and possibly our ancestors) too?
+    while (true) {
+      CInode *diri = dir->get_inode();
+      if (diri->is_base()) {
+	if (!diri->is_root() && diri->authority().first != mds->whoami) {
+	  dout(10) << " closing empty non-auth subtree " << *dir << dendl;
+	  remove_subtree(dir);
+	  dir->mark_clean();
+	  diri->close_dirfrag(dir->get_frag());
+
+	  dout(10) << " removing " << *diri << dendl;
+	  assert(!diri->get_parent_dn());
+	  assert(diri->get_num_ref() == 0);
+	  remove_inode(diri);
+	}
+	break;
+      }
+
+      CDir *psub = get_subtree_root(diri->get_parent_dir());
+      dout(10) << " parent subtree is " << *psub << dendl;
+      if (psub->get_dir_auth().first == mds->whoami)
+	break;  // we are auth, keep.
+
+      dout(10) << " closing empty non-auth subtree " << *dir << dendl;
+      remove_subtree(dir);
+      dir->mark_clean();
+      diri->close_dirfrag(dir->get_frag());
+
+      dout(10) << " parent subtree also non-auth: " << *psub << dendl;
+      if (trim_non_auth_subtree(psub))
+	break;
+      dir = psub;
+    }
+  }
+
+  show_subtrees();
+}
+
+
+/* This function DOES put the passed message before returning */
+void MDCache::handle_cache_expire(MCacheExpire *m)
+{
+  int from = m->get_from();
+  
+  dout(7) << "cache_expire from mds." << from << dendl;
+
+  if (mds->get_state() < MDSMap::STATE_REJOIN) {
+    m->put();
+    return;
+  }
+
+  set gather_locks;
+  // loop over realms
+  for (map::iterator p = m->realms.begin();
+       p != m->realms.end();
+       ++p) {
+    // check container?
+    if (p->first.ino > 0) {
+      CInode *expired_inode = get_inode(p->first.ino);
+      assert(expired_inode);  // we had better have this.
+      CDir *parent_dir = expired_inode->get_approx_dirfrag(p->first.frag);
+      assert(parent_dir);
+
+      int export_state = -1;
+      if (parent_dir->is_auth() && parent_dir->is_exporting()) {
+	export_state = migrator->get_export_state(parent_dir);
+	assert(export_state >= 0);
+      }
+
+      if (!parent_dir->is_auth() ||
+	  (export_state != -1 &&
+	   ((export_state == Migrator::EXPORT_WARNING &&
+	     migrator->export_has_warned(parent_dir,from)) ||
+	    export_state == Migrator::EXPORT_EXPORTING ||
+	    export_state == Migrator::EXPORT_LOGGINGFINISH ||
+	    (export_state == Migrator::EXPORT_NOTIFYING &&
+	     !migrator->export_has_notified(parent_dir,from))))) {
+
+	// not auth.
+	dout(7) << "delaying nonauth|warned expires for " << *parent_dir << dendl;
+	assert(parent_dir->is_frozen_tree_root());
+	
+	// make a message container
+	if (delayed_expire[parent_dir].count(from) == 0)
+	  delayed_expire[parent_dir][from] = new MCacheExpire(from);
+	
+	// merge these expires into it
+	delayed_expire[parent_dir][from]->add_realm(p->first, p->second);
+	continue;
+      }
+      assert(export_state <= Migrator::EXPORT_PREPPING ||
+             (export_state == Migrator::EXPORT_WARNING &&
+              !migrator->export_has_warned(parent_dir, from)));
+
+      dout(7) << "expires for " << *parent_dir << dendl;
+    } else {
+      dout(7) << "containerless expires (root, stray inodes)" << dendl;
+    }
+
+    // INODES
+    for (map::iterator it = p->second.inodes.begin();
+	 it != p->second.inodes.end();
+	 ++it) {
+      CInode *in = get_inode(it->first);
+      unsigned nonce = it->second;
+      
+      if (!in) {
+	dout(0) << " inode expire on " << it->first << " from " << from 
+		<< ", don't have it" << dendl;
+	assert(in);
+      }        
+      assert(in->is_auth());
+      
+      // check nonce
+      if (nonce == in->get_replica_nonce(from)) {
+	// remove from our cached_by
+	dout(7) << " inode expire on " << *in << " from mds." << from 
+		<< " cached_by was " << in->get_replicas() << dendl;
+	inode_remove_replica(in, from, false, gather_locks);
+      } 
+      else {
+	// this is an old nonce, ignore expire.
+	dout(7) << " inode expire on " << *in << " from mds." << from
+		<< " with old nonce " << nonce
+		<< " (current " << in->get_replica_nonce(from) << "), dropping" 
+		<< dendl;
+      }
+    }
+    
+    // DIRS
+    for (map::iterator it = p->second.dirs.begin();
+	 it != p->second.dirs.end();
+	 ++it) {
+      CDir *dir = get_dirfrag(it->first);
+      unsigned nonce = it->second;
+      
+      if (!dir) {
+	CInode *diri = get_inode(it->first.ino);
+	if (diri) {
+	  if (mds->is_rejoin() &&
+	      rejoin_ack_gather.count(mds->whoami) && // haven't sent rejoin ack yet
+	      !diri->is_replica(from)) {
+	    list ls;
+	    diri->get_nested_dirfrags(ls);
+	    dout(7) << " dir expire on dirfrag " << it->first << " from mds." << from
+		    << " while rejoining, inode isn't replicated" << dendl;
+	    for (list::iterator q = ls.begin(); q != ls.end(); ++q) {
+	      dir = *q;
+	      if (dir->is_replica(from)) {
+		dout(7) << " dir expire on " << *dir << " from mds." << from << dendl;
+		dir->remove_replica(from);
+	      }
+	    }
+	    continue;
+	  }
+	  CDir *other = diri->get_approx_dirfrag(it->first.frag);
+	  if (other) {
+	    dout(7) << " dir expire on dirfrag " << it->first << " from mds." << from
+		    << " have " << *other << ", mismatched frags, dropping" << dendl;
+	    continue;
+	  }
+	}
+	dout(0) << " dir expire on " << it->first << " from " << from
+		<< ", don't have it" << dendl;
+	assert(dir);
+      }
+      assert(dir->is_auth());
+      
+      // check nonce
+      if (nonce == dir->get_replica_nonce(from)) {
+	// remove from our cached_by
+	dout(7) << " dir expire on " << *dir << " from mds." << from
+		<< " replicas was " << dir->replica_map << dendl;
+	dir->remove_replica(from);
+      } 
+      else {
+	// this is an old nonce, ignore expire.
+	dout(7) << " dir expire on " << *dir << " from mds." << from 
+		<< " with old nonce " << nonce << " (current " << dir->get_replica_nonce(from)
+		<< "), dropping" << dendl;
+      }
+    }
+    
+    // DENTRIES
+    for (map,uint32_t> >::iterator pd = p->second.dentries.begin();
+	 pd != p->second.dentries.end();
+	 ++pd) {
+      dout(10) << " dn expires in dir " << pd->first << dendl;
+      CInode *diri = get_inode(pd->first.ino);
+      assert(diri);
+      CDir *dir = diri->get_dirfrag(pd->first.frag);
+      
+      if (!dir) {
+	dout(0) << " dn expires on " << pd->first << " from " << from
+		<< ", must have refragmented" << dendl;
+      } else {
+	assert(dir->is_auth());
+      }
+      
+      for (map,uint32_t>::iterator p = pd->second.begin();
+	   p != pd->second.end();
+	   ++p) {
+	unsigned nonce = p->second;
+	CDentry *dn;
+	
+	if (dir) {
+	  dn = dir->lookup(p->first.first, p->first.second);
+	} else {
+	  // which dirfrag for this dentry?
+	  CDir *dir = diri->get_dirfrag(diri->pick_dirfrag(p->first.first));
+	  assert(dir); 
+	  assert(dir->is_auth());
+	  dn = dir->lookup(p->first.first, p->first.second);
+	}
+
+	if (!dn) { 
+	  if (dir)
+	    dout(0) << "  missing dentry for " << p->first.first << " snap " << p->first.second << " in " << *dir << dendl;
+	  else
+	    dout(0) << "  missing dentry for " << p->first.first << " snap " << p->first.second << dendl;
+	}
+	assert(dn);
+	
+	if (nonce == dn->get_replica_nonce(from)) {
+	  dout(7) << "  dentry_expire on " << *dn << " from mds." << from << dendl;
+	  dentry_remove_replica(dn, from, gather_locks);
+	} 
+	else {
+	  dout(7) << "  dentry_expire on " << *dn << " from mds." << from
+		  << " with old nonce " << nonce << " (current " << dn->get_replica_nonce(from)
+		  << "), dropping" << dendl;
+	}
+      }
+    }
+  }
+
+  // done
+  m->put();
+
+  for (set::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p) {
+    if (!(*p)->is_stable())
+      mds->locker->eval_gather(*p);
+  }
+}
+
+void MDCache::process_delayed_expire(CDir *dir)
+{
+  dout(7) << "process_delayed_expire on " << *dir << dendl;
+  for (map::iterator p = delayed_expire[dir].begin();
+       p != delayed_expire[dir].end();
+       ++p) 
+    handle_cache_expire(p->second);
+  delayed_expire.erase(dir);  
+}
+
+void MDCache::discard_delayed_expire(CDir *dir)
+{
+  dout(7) << "discard_delayed_expire on " << *dir << dendl;
+  for (map::iterator p = delayed_expire[dir].begin();
+       p != delayed_expire[dir].end();
+       ++p) 
+    p->second->put();
+  delayed_expire.erase(dir);  
+}
+
+void MDCache::inode_remove_replica(CInode *in, int from, bool rejoin,
+				   set& gather_locks)
+{
+  in->remove_replica(from);
+  in->mds_caps_wanted.erase(from);
+  
+  // note: this code calls _eval more often than it needs to!
+  // fix lock
+  if (in->authlock.remove_replica(from)) gather_locks.insert(&in->authlock);
+  if (in->linklock.remove_replica(from)) gather_locks.insert(&in->linklock);
+  if (in->snaplock.remove_replica(from)) gather_locks.insert(&in->snaplock);
+  if (in->xattrlock.remove_replica(from)) gather_locks.insert(&in->xattrlock);
+  if (in->flocklock.remove_replica(from)) gather_locks.insert(&in->flocklock);
+  if (in->policylock.remove_replica(from)) gather_locks.insert(&in->policylock);
+
+  // If 'rejoin' is true and the scatter lock is in LOCK_MIX_* state.
+  // Don't remove the recovering mds from lock's gathering list because
+  // it may hold rejoined wrlocks.
+  if (in->dirfragtreelock.remove_replica(from, rejoin)) gather_locks.insert(&in->dirfragtreelock);
+  if (in->filelock.remove_replica(from, rejoin)) gather_locks.insert(&in->filelock);
+  if (in->nestlock.remove_replica(from, rejoin)) gather_locks.insert(&in->nestlock);
+}
+
+void MDCache::dentry_remove_replica(CDentry *dn, int from, set& gather_locks)
+{
+  dn->remove_replica(from);
+
+  // fix lock
+  if (dn->lock.remove_replica(from))
+    gather_locks.insert(&dn->lock);
+}
+
+void MDCache::trim_client_leases()
+{
+  utime_t now = ceph_clock_now(g_ceph_context);
+  
+  dout(10) << "trim_client_leases" << dendl;
+
+  for (int pool=0; poolttl > now) break;
+      CDentry *dn = static_cast(r->parent);
+      dout(10) << " expiring client." << r->client << " lease of " << *dn << dendl;
+      dn->remove_client_lease(r, mds->locker);
+    }
+    int after = client_leases[pool].size();
+    dout(10) << "trim_client_leases pool " << pool << " trimmed "
+	     << (before-after) << " leases, " << after << " left" << dendl;
+  }
+}
+
+
+void MDCache::check_memory_usage()
+{
+  static MemoryModel mm(g_ceph_context);
+  static MemoryModel::snap last;
+  mm.sample(&last);
+  static MemoryModel::snap baseline = last;
+
+  // check client caps
+  int num_inodes = inode_map.size();
+  float caps_per_inode = (float)num_caps / (float)num_inodes;
+  //float cap_rate = (float)num_inodes_with_caps / (float)inode_map.size();
+
+  dout(2) << "check_memory_usage"
+	   << " total " << last.get_total()
+	   << ", rss " << last.get_rss()
+	   << ", heap " << last.get_heap()
+	   << ", malloc " << last.malloc << " mmap " << last.mmap
+	   << ", baseline " << baseline.get_heap()
+	   << ", buffers " << (buffer::get_total_alloc() >> 10)
+	   << ", max " << g_conf->mds_mem_max
+	   << ", " << num_inodes_with_caps << " / " << inode_map.size() << " inodes have caps"
+	   << ", " << num_caps << " caps, " << caps_per_inode << " caps per inode"
+	   << dendl;
+
+  mds->mlogger->set(l_mdm_rss, last.get_rss());
+  mds->mlogger->set(l_mdm_heap, last.get_heap());
+  mds->mlogger->set(l_mdm_malloc, last.malloc);
+
+  /*int size = last.get_total();
+  if (size > g_conf->mds_mem_max * .9) {
+    float ratio = (float)g_conf->mds_mem_max * .9 / (float)size;
+    if (ratio < 1.0)
+      mds->server->recall_client_state(ratio);
+  } else 
+    */
+  if (num_inodes_with_caps > g_conf->mds_cache_size) {
+    float ratio = (float)g_conf->mds_cache_size * .9 / (float)num_inodes_with_caps;
+    if (ratio < 1.0)
+      mds->server->recall_client_state(ratio);
+  }
+
+}
+
+
+
+// =========================================================================================
+// shutdown
+
+class C_MDC_ShutdownCheck : public Context {
+  MDCache *mdc;
+public:
+  C_MDC_ShutdownCheck(MDCache *m) : mdc(m) {}
+  void finish(int) {
+    mdc->shutdown_check();
+  }
+};
+
+void MDCache::shutdown_check()
+{
+  dout(0) << "shutdown_check at " << ceph_clock_now(g_ceph_context) << dendl;
+
+  // cache
+  char old_val[32] = { 0 };
+  char *o = old_val;
+  g_conf->get_val("debug_mds", &o, sizeof(old_val));
+  g_conf->set_val("debug_mds", "10");
+  g_conf->apply_changes(NULL);
+  show_cache();
+  g_conf->set_val("debug_mds", old_val);
+  g_conf->apply_changes(NULL);
+  mds->timer.add_event_after(g_conf->mds_shutdown_check, new C_MDC_ShutdownCheck(this));
+
+  // this
+  dout(0) << "lru size now " << lru.lru_get_size() << dendl;
+  dout(0) << "log len " << mds->mdlog->get_num_events() << dendl;
+
+
+  if (mds->objecter->is_active()) {
+    dout(0) << "objecter still active" << dendl;
+    mds->objecter->dump_active();
+  }
+}
+
+
+void MDCache::shutdown_start()
+{
+  dout(2) << "shutdown_start" << dendl;
+
+  if (g_conf->mds_shutdown_check)
+    mds->timer.add_event_after(g_conf->mds_shutdown_check, new C_MDC_ShutdownCheck(this));
+
+  //  g_conf->debug_mds = 10;
+}
+
+
+
+bool MDCache::shutdown_pass()
+{
+  dout(7) << "shutdown_pass" << dendl;
+
+  if (mds->is_stopped()) {
+    dout(7) << " already shut down" << dendl;
+    show_cache();
+    show_subtrees();
+    return true;
+  }
+
+  // close out any sessions (and open files!) before we try to trim the log, etc.
+  if (!mds->server->terminating_sessions &&
+      mds->sessionmap.have_unclosed_sessions()) {
+    mds->server->terminate_sessions();
+    return false;
+  }
+
+
+  // flush what we can from the log
+  mds->mdlog->trim(0);
+
+  if (mds->mdlog->get_num_segments() > 1) {
+    dout(7) << "still >1 segments, waiting for log to trim" << dendl;
+    return false;
+  }
+
+  // empty stray dir
+  if (!shutdown_export_strays()) {
+    dout(7) << "waiting for strays to migrate" << dendl;
+    return false;
+  }
+  
+  // drop our reference to our stray dir inode
+  for (int i = 0; i < NUM_STRAY; ++i) {
+    if (strays[i] &&
+	strays[i]->state_test(CInode::STATE_STRAYPINNED)) {
+      strays[i]->state_clear(CInode::STATE_STRAYPINNED);
+      strays[i]->put(CInode::PIN_STRAY);
+      strays[i]->put_stickydirs();
+    }
+  }
+
+  // trim cache
+  trim(0);
+  dout(5) << "lru size now " << lru.lru_get_size() << dendl;
+
+  // SUBTREES
+  if (!subtrees.empty() &&
+      mds->get_nodeid() != 0 && 
+      !migrator->is_exporting() //&&
+      //!migrator->is_importing()
+      ) {
+    dout(7) << "looking for subtrees to export to mds0" << dendl;
+    list ls;
+    for (map >::iterator it = subtrees.begin();
+         it != subtrees.end();
+         ++it) {
+      CDir *dir = it->first;
+      if (dir->get_inode()->is_mdsdir())
+	continue;
+      if (dir->is_frozen() || dir->is_freezing())
+	continue;
+      if (!dir->is_full_dir_auth())
+	continue;
+      ls.push_back(dir);
+    }
+    int max = 5; // throttle shutdown exports.. hack!
+    for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+      CDir *dir = *p;
+      int dest = dir->get_inode()->authority().first;
+      if (dest > 0 && !mds->mdsmap->is_active(dest))
+	dest = 0;
+      dout(7) << "sending " << *dir << " back to mds." << dest << dendl;
+      migrator->export_dir(dir, dest);
+      if (--max == 0)
+	break;
+    }
+  }
+
+  if (!shutdown_export_caps()) {
+    dout(7) << "waiting for residual caps to export" << dendl;
+    return false;
+  }
+
+  CDir *mydir = myin ? myin->get_dirfrag(frag_t()) : NULL;
+  if (mydir && !mydir->is_subtree_root())
+    mydir = NULL;
+
+  // subtrees map not empty yet?
+  if (subtrees.size() > (mydir ? 1 : 0)) {
+    dout(7) << "still have " << num_subtrees() << " subtrees" << dendl;
+    show_subtrees();
+    migrator->show_importing();
+    migrator->show_exporting();
+    if (!migrator->is_importing() && !migrator->is_exporting())
+      show_cache();
+    return false;
+  }
+  assert(!migrator->is_exporting());
+  assert(!migrator->is_importing());
+
+  // make mydir subtree go away
+  if (mydir) {
+    adjust_subtree_auth(mydir, CDIR_AUTH_UNKNOWN);
+    remove_subtree(mydir);
+  }
+  assert(subtrees.empty());
+
+  // (only do this once!)
+  if (!mds->mdlog->is_capped()) {
+    dout(7) << "capping the log" << dendl;
+    mds->mdlog->cap();
+    mds->mdlog->trim();
+  }
+  
+  if (!mds->mdlog->empty()) {
+    dout(7) << "waiting for log to flush.. " << mds->mdlog->get_num_events() 
+	    << " in " << mds->mdlog->get_num_segments() << " segments" << dendl;
+    return false;
+  }
+  
+  if (!did_shutdown_log_cap) {
+    // flush journal header
+    dout(7) << "writing header for (now-empty) journal" << dendl;
+    assert(mds->mdlog->empty());
+    mds->mdlog->write_head(0);  
+    // NOTE: filer active checker below will block us until this completes.
+    did_shutdown_log_cap = true;
+    return false;
+  }
+
+  // filer active?
+  if (mds->objecter->is_active()) {
+    dout(7) << "objecter still active" << dendl;
+    mds->objecter->dump_active();
+    return false;
+  }
+
+  // trim what we can from the cache
+  if (lru.lru_get_size() > 0) {
+    dout(7) << "there's still stuff in the cache: " << lru.lru_get_size() << dendl;
+    show_cache();
+    //dump();
+    return false;
+  } 
+  
+  // done!
+  dout(2) << "shutdown done." << dendl;
+  return true;
+}
+
+bool MDCache::shutdown_export_strays()
+{
+  if (mds->get_nodeid() == 0) return true;
+  
+  dout(10) << "shutdown_export_strays" << dendl;
+
+  bool done = true;
+  static set exported_strays;
+
+  list dfs;
+  for (int i = 0; i < NUM_STRAY; ++i) {
+    if (!strays[i]) {
+      continue;
+    }
+    strays[i]->get_dirfrags(dfs);
+  }
+  
+  while (!dfs.empty()) {
+    CDir *dir = dfs.front();
+    dfs.pop_front();
+
+    if (!dir->is_complete()) {
+      dir->fetch(0);
+      done = false;
+    }
+    
+    for (CDir::map_t::iterator p = dir->items.begin();
+	 p != dir->items.end();
+	 ++p) {
+      CDentry *dn = p->second;
+      CDentry::linkage_t *dnl = dn->get_linkage();
+      if (dnl->is_null()) continue;
+      done = false;
+      
+      // FIXME: we'll deadlock if a rename fails.
+      if (exported_strays.count(dnl->get_inode()->ino()) == 0) {
+	exported_strays.insert(dnl->get_inode()->ino());
+	migrate_stray(dn, 0);  // send to root!
+      } else {
+	dout(10) << "already exporting " << *dn << dendl;
+      }
+    }
+  }
+
+  return done;
+}
+
+bool MDCache::shutdown_export_caps()
+{
+  // export caps?
+  //  note: this runs more often than it should.
+  static bool exported_caps = false;
+  static set exported_caps_in;
+  if (!exported_caps) {
+    dout(7) << "searching for caps to export" << dendl;
+    exported_caps = true;
+
+    list dirq;
+    for (map >::iterator p = subtrees.begin();
+	 p != subtrees.end();
+	 ++p) {
+      if (exported_caps_in.count(p->first)) continue;
+      if (p->first->is_auth() ||
+	  p->first->is_ambiguous_auth())
+	exported_caps = false; // we'll have to try again
+      else {
+	dirq.push_back(p->first);
+	exported_caps_in.insert(p->first);
+      }
+    }
+    while (!dirq.empty()) {
+      CDir *dir = dirq.front();
+      dirq.pop_front();
+      for (CDir::map_t::iterator p = dir->items.begin();
+	   p != dir->items.end();
+	   ++p) {
+	CDentry *dn = p->second;
+	CDentry::linkage_t *dnl = dn->get_linkage();
+	if (!dnl->is_primary()) continue;
+	CInode *in = dnl->get_inode();
+	if (in->is_dir())
+	  in->get_nested_dirfrags(dirq);
+	if (in->is_any_caps() && !in->state_test(CInode::STATE_EXPORTINGCAPS))
+	  migrator->export_caps(in);
+      }
+    }
+  }
+
+  return true;
+}
+
+
+
+
+
+// ========= messaging ==============
+
+/* This function DOES put the passed message before returning */
+void MDCache::dispatch(Message *m)
+{
+  switch (m->get_type()) {
+
+    // RESOLVE
+  case MSG_MDS_RESOLVE:
+    handle_resolve(static_cast(m));
+    break;
+  case MSG_MDS_RESOLVEACK:
+    handle_resolve_ack(static_cast(m));
+    break;
+
+    // REJOIN
+  case MSG_MDS_CACHEREJOIN:
+    handle_cache_rejoin(static_cast(m));
+    break;
+
+  case MSG_MDS_DISCOVER:
+    handle_discover(static_cast(m));
+    break;
+  case MSG_MDS_DISCOVERREPLY:
+    handle_discover_reply(static_cast(m));
+    break;
+
+  case MSG_MDS_DIRUPDATE:
+    handle_dir_update(static_cast(m));
+    break;
+
+  case MSG_MDS_CACHEEXPIRE:
+    handle_cache_expire(static_cast(m));
+    break;
+
+  case MSG_MDS_DENTRYLINK:
+    handle_dentry_link(static_cast(m));
+    break;
+  case MSG_MDS_DENTRYUNLINK:
+    handle_dentry_unlink(static_cast(m));
+    break;
+
+  case MSG_MDS_FRAGMENTNOTIFY:
+    handle_fragment_notify(static_cast(m));
+    break;
+
+  case MSG_MDS_FINDINO:
+    handle_find_ino(static_cast(m));
+    break;
+  case MSG_MDS_FINDINOREPLY:
+    handle_find_ino_reply(static_cast(m));
+    break;
+
+  case MSG_MDS_OPENINO:
+    handle_open_ino(static_cast(m));
+    break;
+  case MSG_MDS_OPENINOREPLY:
+    handle_open_ino_reply(static_cast(m));
+    break;
+    
+  default:
+    dout(7) << "cache unknown message " << m->get_type() << dendl;
+    assert(0);
+    m->put();
+    break;
+  }
+}
+
+Context *MDCache::_get_waiter(MDRequestRef& mdr, Message *req, Context *fin)
+{
+  if (mdr) {
+    dout(20) << "_get_waiter retryrequest" << dendl;
+    return new C_MDS_RetryRequest(this, mdr);
+  } else if (req) {
+    dout(20) << "_get_waiter retrymessage" << dendl;
+    return new C_MDS_RetryMessage(mds, req);
+  } else {
+    return fin;
+  }
+}
+
+int MDCache::path_traverse(MDRequestRef& mdr, Message *req, Context *fin,     // who
+			   const filepath& path,                   // what
+                           vector *pdnvec,         // result
+			   CInode **pin,
+                           int onfail)
+{
+  bool discover = (onfail == MDS_TRAVERSE_DISCOVER);
+  bool null_okay = (onfail == MDS_TRAVERSE_DISCOVERXLOCK);
+  bool forward = (onfail == MDS_TRAVERSE_FORWARD);
+
+  assert(mdr || req || fin);
+  assert(!forward || mdr || req);  // forward requires a request
+
+  snapid_t snapid = CEPH_NOSNAP;
+  if (mdr)
+    mdr->snapid = snapid;
+
+  client_t client = (mdr && mdr->reqid.name.is_client()) ? mdr->reqid.name.num() : -1;
+
+  if (mds->logger) mds->logger->inc(l_mds_t);
+
+  dout(7) << "traverse: opening base ino " << path.get_ino() << " snap " << snapid << dendl;
+  CInode *cur = get_inode(path.get_ino());
+  if (cur == NULL) {
+    if (MDS_INO_IS_MDSDIR(path.get_ino())) 
+      open_foreign_mdsdir(path.get_ino(), _get_waiter(mdr, req, fin));
+    else {
+      //assert(0);  // hrm.. broken
+      return -ESTALE;
+    }
+    return 1;
+  }
+  if (cur->state_test(CInode::STATE_PURGING))
+    return -ESTALE;
+
+  // start trace
+  if (pdnvec)
+    pdnvec->clear();
+  if (pin)
+    *pin = cur;
+
+  unsigned depth = 0;
+  while (depth < path.depth()) {
+    dout(12) << "traverse: path seg depth " << depth << " '" << path[depth]
+	     << "' snapid " << snapid << dendl;
+    
+    if (!cur->is_dir()) {
+      dout(7) << "traverse: " << *cur << " not a dir " << dendl;
+      return -ENOTDIR;
+    }
+
+    // walk into snapdir?
+    if (path[depth].length() == 0) {
+      dout(10) << "traverse: snapdir" << dendl;
+      if (!mdr)
+	return -EINVAL;
+      snapid = CEPH_SNAPDIR;
+      mdr->snapid = snapid;
+      depth++;
+      continue;
+    }
+    // walk thru snapdir?
+    if (snapid == CEPH_SNAPDIR) {
+      if (!mdr)
+	return -EINVAL;
+      SnapRealm *realm = cur->find_snaprealm();
+      snapid = realm->resolve_snapname(path[depth], cur->ino());
+      dout(10) << "traverse: snap " << path[depth] << " -> " << snapid << dendl;
+      if (!snapid)
+	return -ENOENT;
+      mdr->snapid = snapid;
+      depth++;
+      continue;
+    }
+
+    // open dir
+    frag_t fg = cur->pick_dirfrag(path[depth]);
+    CDir *curdir = cur->get_dirfrag(fg);
+    if (!curdir) {
+      if (cur->is_auth()) {
+        // parent dir frozen_dir?
+        if (cur->is_frozen()) {
+          dout(7) << "traverse: " << *cur << " is frozen, waiting" << dendl;
+          cur->add_waiter(CDir::WAIT_UNFREEZE, _get_waiter(mdr, req, fin));
+          return 1;
+        }
+        curdir = cur->get_or_open_dirfrag(this, fg);
+      } else {
+        // discover?
+	dout(10) << "traverse: need dirfrag " << fg << ", doing discover from " << *cur << dendl;
+	discover_path(cur, snapid, path.postfixpath(depth), _get_waiter(mdr, req, fin),
+		      null_okay);
+	if (mds->logger) mds->logger->inc(l_mds_tdis);
+        return 1;
+      }
+    }
+    assert(curdir);
+
+#ifdef MDS_VERIFY_FRAGSTAT
+    if (curdir->is_complete())
+      curdir->verify_fragstat();
+#endif
+
+    // frozen?
+    /*
+    if (curdir->is_frozen()) {
+    // doh!
+      // FIXME: traverse is allowed?
+      dout(7) << "traverse: " << *curdir << " is frozen, waiting" << dendl;
+      curdir->add_waiter(CDir::WAIT_UNFREEZE, _get_waiter(mdr, req, fin));
+      if (onfinish) delete onfinish;
+      return 1;
+    }
+    */
+
+    // must read directory hard data (permissions, x bit) to traverse
+#if 0    
+    if (!noperm && 
+	!mds->locker->rdlock_try(&cur->authlock, client, 0)) {
+      dout(7) << "traverse: waiting on authlock rdlock on " << *cur << dendl;
+      cur->authlock.add_waiter(SimpleLock::WAIT_RD, _get_waiter(mdr, req, fin));
+      return 1;
+    }
+#endif
+    
+
+    // make sure snaprealm parents are open...
+    if (cur->snaprealm && !cur->snaprealm->open && mdr &&
+	!cur->snaprealm->open_parents(_get_waiter(mdr, req, fin))) {
+      return 1;
+    }
+
+
+    // dentry
+    CDentry *dn = curdir->lookup(path[depth], snapid);
+    CDentry::linkage_t *dnl = dn ? dn->get_projected_linkage() : 0;
+
+    // null and last_bit and xlocked by me?
+    if (dnl && dnl->is_null() && null_okay) {
+      dout(10) << "traverse: hit null dentry at tail of traverse, succeeding" << dendl;
+      if (pdnvec)
+	pdnvec->push_back(dn);
+      if (pin)
+	*pin = 0;
+      break; // done!
+    }
+
+    if (dnl &&
+	dn->lock.is_xlocked() &&
+	dn->lock.get_xlock_by() != mdr &&
+	!dn->lock.can_read(client) &&
+	(dnl->is_null() || forward)) {
+      dout(10) << "traverse: xlocked dentry at " << *dn << dendl;
+      dn->lock.add_waiter(SimpleLock::WAIT_RD, _get_waiter(mdr, req, fin));
+      if (mds->logger) mds->logger->inc(l_mds_tlock);
+      mds->mdlog->flush();
+      return 1;
+    }
+    
+    // can we conclude ENOENT?
+    if (dnl && dnl->is_null()) {
+      if (dn->lock.can_read(client) ||
+	  (dn->lock.is_xlocked() && dn->lock.get_xlock_by() == mdr)) {
+        dout(10) << "traverse: miss on null+readable dentry " << path[depth] << " " << *dn << dendl;
+        return -ENOENT;
+      } else {
+        dout(10) << "miss on dentry " << *dn << ", can't read due to lock" << dendl;
+        dn->lock.add_waiter(SimpleLock::WAIT_RD, _get_waiter(mdr, req, fin));
+        return 1;
+      }
+    }
+
+    if (dnl && !dnl->is_null()) {
+      CInode *in = dnl->get_inode();
+      
+      // do we have inode?
+      if (!in) {
+        assert(dnl->is_remote());
+        // do i have it?
+        in = get_inode(dnl->get_remote_ino());
+        if (in) {
+	  dout(7) << "linking in remote in " << *in << dendl;
+	  dn->link_remote(dnl, in);
+	} else {
+          dout(7) << "remote link to " << dnl->get_remote_ino() << ", which i don't have" << dendl;
+	  assert(mdr);  // we shouldn't hit non-primary dentries doing a non-mdr traversal!
+          open_remote_dentry(dn, true, _get_waiter(mdr, req, fin),
+			     (null_okay && depth == path.depth() - 1));
+	  if (mds->logger) mds->logger->inc(l_mds_trino);
+          return 1;
+        }        
+      }
+
+      // forwarder wants replicas?
+#if 0
+      if (mdr && mdr->client_request && 
+	  mdr->client_request->get_mds_wants_replica_in_dirino()) {
+	dout(30) << "traverse: REP is here, " 
+		 << mdr->client_request->get_mds_wants_replica_in_dirino() 
+		 << " vs " << curdir->dirfrag() << dendl;
+	
+	if (mdr->client_request->get_mds_wants_replica_in_dirino() == curdir->ino() &&
+	    curdir->is_auth() && 
+	    curdir->is_rep() &&
+	    curdir->is_replica(req->get_source().num()) &&
+	    dn->is_auth()
+	    ) {
+	  assert(req->get_source().is_mds());
+	  int from = req->get_source().num();
+	  
+	  if (dn->is_replica(from)) {
+	    dout(15) << "traverse: REP would replicate to mds." << from << ", but already cached_by " 
+		     << req->get_source() << " dn " << *dn << dendl; 
+	  } else {
+	    dout(10) << "traverse: REP replicating to " << req->get_source() << " dn " << *dn << dendl;
+	    MDiscoverReply *reply = new MDiscoverReply(curdir->dirfrag());
+	    reply->mark_unsolicited();
+	    reply->starts_with = MDiscoverReply::DENTRY;
+	    replicate_dentry(dn, from, reply->trace);
+	    if (dnl->is_primary())
+	      replicate_inode(in, from, reply->trace);
+	    if (req->get_source() != req->get_orig_source())
+	      mds->send_message_mds(reply, req->get_source().num());
+	    else mds->send_message(reply->req->get_connnection());
+	  }
+	}
+      }
+#endif
+      
+      // add to trace, continue.
+      cur = in;
+      touch_inode(cur);
+      if (pdnvec)
+	pdnvec->push_back(dn);
+      if (pin)
+	*pin = cur;
+      depth++;
+      continue;
+    }
+    
+
+    // MISS.  dentry doesn't exist.
+    dout(12) << "traverse: miss on dentry " << path[depth] << " in " << *curdir << dendl;
+
+    if (curdir->is_auth()) {
+      // dentry is mine.
+      if (curdir->is_complete() || (curdir->has_bloom() &&
+          !curdir->is_in_bloom(path[depth]))){
+        // file not found
+	if (pdnvec) {
+	  // instantiate a null dn?
+	  if (depth < path.depth()-1){
+	    dout(20) << " didn't traverse full path; not returning pdnvec" << dendl;
+	    dn = NULL;
+	  } else if (dn) {
+	    assert(0); // should have fallen out in ->is_null() check above
+	  } else if (curdir->is_frozen()) {
+	    dout(20) << " not adding null to frozen dir " << dendl;
+	  } else if (snapid < CEPH_MAXSNAP) {
+	    dout(20) << " not adding null for snapid " << snapid << dendl;
+	  } else {
+	    // create a null dentry
+	    dn = curdir->add_null_dentry(path[depth]);
+	    dout(20) << " added null " << *dn << dendl;
+	  }
+	  if (dn)
+	    pdnvec->push_back(dn);
+	  else
+	    pdnvec->clear();   // do not confuse likes of rdlock_path_pin_ref();
+	}
+        return -ENOENT;
+      } else {
+	// directory isn't complete; reload
+        dout(7) << "traverse: incomplete dir contents for " << *cur << ", fetching" << dendl;
+        touch_inode(cur);
+        curdir->fetch(_get_waiter(mdr, req, fin), path[depth]);
+	if (mds->logger) mds->logger->inc(l_mds_tdirf);
+        return 1;
+      }
+    } else {
+      // dirfrag/dentry is not mine.
+      pair dauth = curdir->authority();
+
+      if (forward &&
+	  snapid && mdr && mdr->client_request &&
+	  (int)depth < mdr->client_request->get_num_fwd()) {
+	dout(7) << "traverse: snap " << snapid << " and depth " << depth
+		<< " < fwd " << mdr->client_request->get_num_fwd()
+		<< ", discovering instead of forwarding" << dendl;
+	discover = true;
+      }
+
+      if ((discover || null_okay)) {
+	dout(7) << "traverse: discover from " << path[depth] << " from " << *curdir << dendl;
+	discover_path(curdir, snapid, path.postfixpath(depth), _get_waiter(mdr, req, fin),
+		      null_okay);
+	if (mds->logger) mds->logger->inc(l_mds_tdis);
+        return 1;
+      } 
+      if (forward) {
+        // forward
+        dout(7) << "traverse: not auth for " << path << " in " << *curdir << dendl;
+	
+	if (curdir->is_ambiguous_auth()) {
+	  // wait
+	  dout(7) << "traverse: waiting for single auth in " << *curdir << dendl;
+	  curdir->add_waiter(CDir::WAIT_SINGLEAUTH, _get_waiter(mdr, req, fin));
+	  return 1;
+	} 
+
+	dout(7) << "traverse: forwarding, not auth for " << *curdir << dendl;
+	
+#if 0
+	// request replication?
+	if (mdr && mdr->client_request && curdir->is_rep()) {
+	  dout(15) << "traverse: REP fw to mds." << dauth << ", requesting rep under "
+		   << *curdir << " req " << *(MClientRequest*)req << dendl;
+	  mdr->client_request->set_mds_wants_replica_in_dirino(curdir->ino());
+	  req->clear_payload();  // reencode!
+	}
+#endif
+	
+	if (mdr) 
+	  request_forward(mdr, dauth.first);
+	else
+	  mds->forward_message_mds(req, dauth.first);
+	
+	if (mds->logger) mds->logger->inc(l_mds_tfw);
+	assert(fin == NULL);
+	return 2;
+      }    
+    }
+    
+    assert(0);  // i shouldn't get here
+  }
+  
+  // success.
+  if (mds->logger) mds->logger->inc(l_mds_thit);
+  dout(10) << "path_traverse finish on snapid " << snapid << dendl;
+  if (mdr) 
+    assert(mdr->snapid == snapid);
+  return 0;
+}
+
+#if 0
+/**
+ * Find out if the MDS is auth for a given path.
+ *
+ * Returns true if:
+ * 1) The full path DNE and we are auth for the deepest existing piece
+ * 2) We are auth for the inode linked to by the last dentry.
+ */
+bool MDCache::path_is_mine(filepath& path)
+{
+  dout(15) << "path_is_mine " << path.get_ino() << " " << path << dendl;
+  
+  CInode *cur = get_inode(path.get_ino());
+  if (!cur)
+    return false;  // who knows!
+
+  for (unsigned i=0; ipick_dirfrag(path[i]);
+    CDir *dir = cur->get_dirfrag(fg);
+    if (!dir)
+      return cur->is_auth();
+    CDentry *dn = dir->lookup(path[i]);
+    CDentry::linkage_t *dnl = dn->get_linkage();
+    if (!dn || dnl->is_null())
+      return dir->is_auth();
+    if (!dnl->is_primary())
+      return false;
+    cur = dnl->get_inode();
+  }
+
+  return cur->is_auth();
+}
+#endif
+
+CInode *MDCache::cache_traverse(const filepath& fp)
+{
+  dout(10) << "cache_traverse " << fp << dendl;
+
+  CInode *in;
+  if (fp.get_ino())
+    in = get_inode(fp.get_ino());
+  else
+    in = root;
+  if (!in)
+    return NULL;
+
+  for (unsigned i = 0; i < fp.depth(); i++) {
+    const string& dname = fp[i];
+    frag_t fg = in->pick_dirfrag(dname);
+    dout(20) << " " << i << " " << dname << " frag " << fg << " from " << *in << dendl;
+    CDir *curdir = in->get_dirfrag(fg);
+    if (!curdir)
+      return NULL;
+    CDentry *dn = curdir->lookup(dname, CEPH_NOSNAP);
+    if (!dn)
+      return NULL;
+    in = dn->get_linkage()->get_inode();
+    if (!in)
+      return NULL;
+  }
+  dout(10) << " got " << *in << dendl;
+  return in;
+}
+
+
+/**
+ * open_remote_dir -- open up a remote dirfrag
+ *
+ * @param diri base inode
+ * @param approxfg approximate fragment.
+ * @param fin completion callback
+ */
+void MDCache::open_remote_dirfrag(CInode *diri, frag_t approxfg, Context *fin) 
+{
+  dout(10) << "open_remote_dir on " << *diri << dendl;
+  
+  assert(diri->is_dir());
+  assert(!diri->is_auth());
+  assert(diri->get_dirfrag(approxfg) == 0);
+
+  int auth = diri->authority().first;
+
+  if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN) {
+    discover_dir_frag(diri, approxfg, fin);
+  } else {
+    // mds is down or recovering.  forge a replica!
+    forge_replica_dir(diri, approxfg, auth);
+    if (fin)
+      mds->queue_waiter(fin);
+  }
+}
+
+
+/** 
+ * get_dentry_inode - get or open inode
+ *
+ * @param dn the dentry
+ * @param mdr current request
+ *
+ * will return inode for primary, or link up/open up remote link's inode as necessary.
+ * If it's not available right now, puts mdr on wait list and returns null.
+ */
+CInode *MDCache::get_dentry_inode(CDentry *dn, MDRequestRef& mdr, bool projected)
+{
+  CDentry::linkage_t *dnl;
+  if (projected)
+    dnl = dn->get_projected_linkage();
+  else
+    dnl = dn->get_linkage();
+
+  assert(!dnl->is_null());
+  
+  if (dnl->is_primary())
+    return dnl->inode;
+
+  assert(dnl->is_remote());
+  CInode *in = get_inode(dnl->get_remote_ino());
+  if (in) {
+    dout(7) << "get_dentry_inode linking in remote in " << *in << dendl;
+    dn->link_remote(dnl, in);
+    return in;
+  } else {
+    dout(10) << "get_dentry_inode on remote dn, opening inode for " << *dn << dendl;
+    open_remote_ino(dnl->remote_ino, new C_MDS_RetryRequest(this, mdr));
+    return 0;
+  }
+}
+
+class C_MDC_RetryOpenRemoteIno : public Context {
+  MDCache *mdcache;
+  inodeno_t ino;
+  bool want_xlocked;
+  Context *onfinish;
+public:
+  C_MDC_RetryOpenRemoteIno(MDCache *mdc, inodeno_t i, Context *c, bool wx) :
+    mdcache(mdc), ino(i), want_xlocked(wx), onfinish(c) {}
+  void finish(int r) {
+    if (mdcache->get_inode(ino)) {
+      onfinish->complete(0);
+    } else
+      mdcache->open_remote_ino(ino, onfinish, want_xlocked);
+  }
+};
+
+
+class C_MDC_OpenRemoteIno : public Context {
+  MDCache *mdcache;
+  inodeno_t ino;
+  inodeno_t hadino;
+  version_t hadv;
+  bool want_xlocked;
+  Context *onfinish;
+public:
+  vector anchortrace;
+
+  C_MDC_OpenRemoteIno(MDCache *mdc, inodeno_t i, bool wx, inodeno_t hi, version_t hv, Context *c) :
+    mdcache(mdc), ino(i), hadino(hi), hadv(hv), want_xlocked(wx), onfinish(c) {}
+  C_MDC_OpenRemoteIno(MDCache *mdc, inodeno_t i, bool wx, vector& at, Context *c) :
+    mdcache(mdc), ino(i), hadino(0), hadv(0), want_xlocked(wx), onfinish(c), anchortrace(at) {}
+
+  void finish(int r) {
+    assert(r == 0);
+    if (r == 0)
+      mdcache->open_remote_ino_2(ino, anchortrace, want_xlocked, hadino, hadv, onfinish);
+    else {
+      onfinish->complete(r);
+    }
+  }
+};
+
+void MDCache::open_remote_ino(inodeno_t ino, Context *onfinish, bool want_xlocked,
+			      inodeno_t hadino, version_t hadv)
+{
+  dout(7) << "open_remote_ino on " << ino << (want_xlocked ? " want_xlocked":"") << dendl;
+  
+  C_MDC_OpenRemoteIno *c = new C_MDC_OpenRemoteIno(this, ino, want_xlocked,
+						   hadino, hadv, onfinish);
+  mds->anchorclient->lookup(ino, c->anchortrace, c);
+}
+
+void MDCache::open_remote_ino_2(inodeno_t ino, vector& anchortrace, bool want_xlocked,
+				inodeno_t hadino, version_t hadv, Context *onfinish)
+{
+  dout(7) << "open_remote_ino_2 on " << ino
+	  << ", trace depth is " << anchortrace.size() << dendl;
+  
+  // find deepest cached inode in prefix
+  unsigned i = anchortrace.size();  // i := array index + 1
+  CInode *in = 0;
+  while (1) {
+    // inode?
+    dout(10) << " " << i << ": " << anchortrace[i-1] << dendl;
+    in = get_inode(anchortrace[i-1].ino);
+    if (in)
+      break;
+    i--;
+    if (!i) {
+      in = get_inode(anchortrace[i].dirino);
+      if (!in) {
+	dout(0) << "open_remote_ino_2 don't have dir inode " << anchortrace[i].dirino << dendl;
+	if (MDS_INO_IS_MDSDIR(anchortrace[i].dirino)) {
+	  open_foreign_mdsdir(anchortrace[i].dirino, onfinish);
+	  return;
+	}
+	assert(in);  // hrm!
+      }
+      break;
+    }
+  }
+  dout(10) << "deepest cached inode at " << i << " is " << *in << dendl;
+
+  if (in->ino() == ino) {
+    // success
+    dout(10) << "open_remote_ino_2 have " << *in << dendl;
+    onfinish->complete(0);
+    return;
+  } 
+
+  // open dirfrag beneath *in
+  frag_t frag = in->dirfragtree[anchortrace[i].dn_hash];
+
+  if (!in->dirfragtree.contains(frag)) {
+    dout(10) << "frag " << frag << " not valid, requerying anchortable" << dendl;
+    open_remote_ino(ino, onfinish, want_xlocked);
+    return;
+  }
+
+  CDir *dir = in->get_dirfrag(frag);
+
+  if (!dir && !in->is_auth()) {
+    dout(10) << "opening remote dirfrag " << frag << " under " << *in << dendl;
+    /* we re-query the anchortable just to avoid a fragtree update race */
+    open_remote_dirfrag(in, frag,
+			new C_MDC_RetryOpenRemoteIno(this, ino, onfinish, want_xlocked));
+    return;
+  }
+
+  if (!dir && in->is_auth()) {
+    if (in->is_frozen_dir()) {
+      dout(7) << "traverse: " << *in << " is frozen_dir, waiting" << dendl;
+      in->parent->dir->add_waiter(CDir::WAIT_UNFREEZE,
+				  new C_MDC_RetryOpenRemoteIno(this, ino, onfinish, want_xlocked));
+      return;
+    }
+    dir = in->get_or_open_dirfrag(this, frag);
+  }
+  assert(dir);
+
+  if (dir->is_auth()) {
+    if (dir->is_complete()) {
+      // make sure we didn't get to the same version anchor 2x in a row
+      if (hadv && hadino == anchortrace[i].ino && hadv == anchortrace[i].updated) {
+	dout(10) << "expected ino " << anchortrace[i].ino
+		 << " in complete dir " << *dir
+		 << ", got same anchor " << anchortrace[i] << " 2x in a row" << dendl;
+	onfinish->complete(-ENOENT);
+      } else {
+	// hrm.  requery anchor table.
+	dout(10) << "expected ino " << anchortrace[i].ino
+		 << " in complete dir " << *dir
+		 << ", requerying anchortable"
+		 << dendl;
+	open_remote_ino(ino, onfinish, want_xlocked,
+		        anchortrace[i].ino, anchortrace[i].updated);
+      }
+    } else {
+      dout(10) << "need ino " << anchortrace[i].ino
+	       << ", fetching incomplete dir " << *dir
+	       << dendl;
+      dir->fetch(new C_MDC_OpenRemoteIno(this, ino, want_xlocked, anchortrace, onfinish));
+    }
+  } else {
+    // hmm, discover.
+    dout(10) << "have remote dirfrag " << *dir << ", discovering " 
+	     << anchortrace[i].ino << dendl;
+    discover_ino(dir, anchortrace[i].ino,
+	         new C_MDC_RetryOpenRemoteIno(this, ino, onfinish, want_xlocked),
+		 (want_xlocked && i == anchortrace.size() - 1));
+  }
+}
+
+
+struct C_MDC_OpenRemoteDentry : public Context {
+  MDCache *mdc;
+  CDentry *dn;
+  inodeno_t ino;
+  Context *onfinish;
+  bool want_xlocked;
+  int mode;
+  C_MDC_OpenRemoteDentry(MDCache *m, CDentry *d, inodeno_t i, Context *f,
+			 bool wx, int md) :
+    mdc(m), dn(d), ino(i), onfinish(f), want_xlocked(wx), mode(md) {}
+  void finish(int r) {
+    mdc->_open_remote_dentry_finish(dn, ino, onfinish, want_xlocked, mode, r);
+  }
+};
+
+void MDCache::open_remote_dentry(CDentry *dn, bool projected, Context *fin, bool want_xlocked)
+{
+  dout(10) << "open_remote_dentry " << *dn << dendl;
+  CDentry::linkage_t *dnl = projected ? dn->get_projected_linkage() : dn->get_linkage();
+  inodeno_t ino = dnl->get_remote_ino();
+  int mode = g_conf->mds_open_remote_link_mode;
+  Context *fin2 =  new C_MDC_OpenRemoteDentry(this, dn, ino, fin, want_xlocked, mode);
+  if (mode == 0)
+    open_remote_ino(ino, fin2, want_xlocked); // anchor
+  else
+    open_ino(ino, -1, fin2, true, want_xlocked); // backtrace
+}
+
+void MDCache::_open_remote_dentry_finish(CDentry *dn, inodeno_t ino, Context *fin,
+					 bool want_xlocked, int mode, int r)
+{
+  if (r < 0) {
+    if (mode == 0) {
+      dout(0) << "open_remote_dentry_finish bad remote dentry " << *dn << dendl;
+      dn->state_set(CDentry::STATE_BADREMOTEINO);
+    } else {
+      dout(7) << "open_remote_dentry_finish failed to open ino " << ino
+	      << " for " << *dn << ", retry using anchortable" << dendl;
+      assert(mode == 1);
+      Context *fin2 =  new C_MDC_OpenRemoteDentry(this, dn, ino, fin, want_xlocked, 0);
+      open_remote_ino(ino, fin2, want_xlocked);
+      return;
+    }
+  }
+  fin->complete(r < 0 ? r : 0);
+}
+
+
+void MDCache::make_trace(vector& trace, CInode *in)
+{
+  // empty trace if we're a base inode
+  if (in->is_base())
+    return;
+
+  CInode *parent = in->get_parent_inode();
+  assert(parent);
+  make_trace(trace, parent);
+
+  CDentry *dn = in->get_parent_dn();
+  dout(15) << "make_trace adding " << *dn << dendl;
+  trace.push_back(dn);
+}
+
+
+// -------------------------------------------------------------------------------
+// Open inode by inode number
+
+class C_MDC_OpenInoBacktraceFetched : public Context {
+  MDCache *cache;
+  inodeno_t ino;
+  public:
+  bufferlist bl;
+  C_MDC_OpenInoBacktraceFetched(MDCache *c, inodeno_t i) :
+    cache(c), ino(i) {}
+  void finish(int r) {
+    cache->_open_ino_backtrace_fetched(ino, bl, r);
+  }
+};
+
+struct C_MDC_OpenInoTraverseDir : public Context {
+  MDCache *cache;
+  inodeno_t ino;
+  public:
+  C_MDC_OpenInoTraverseDir(MDCache *c, inodeno_t i) : cache(c), ino(i) {}
+  void finish(int r) {
+    assert(cache->opening_inodes.count(ino));
+    cache->_open_ino_traverse_dir(ino, cache->opening_inodes[ino], r);
+  }
+};
+
+struct C_MDC_OpenInoParentOpened : public Context {
+  MDCache *cache;
+  inodeno_t ino;
+  public:
+  C_MDC_OpenInoParentOpened(MDCache *c, inodeno_t i) : cache(c), ino(i) {}
+  void finish(int r) {
+    cache->_open_ino_parent_opened(ino, r);
+  }
+};
+
+void MDCache::_open_ino_backtrace_fetched(inodeno_t ino, bufferlist& bl, int err)
+{
+  dout(10) << "_open_ino_backtrace_fetched ino " << ino << " errno " << err << dendl;
+
+  assert(opening_inodes.count(ino));
+  open_ino_info_t& info = opening_inodes[ino];
+
+  CInode *in = get_inode(ino);
+  if (in) {
+    dout(10) << " found cached " << *in << dendl;
+    open_ino_finish(ino, info, in->authority().first);
+    return;
+  }
+
+  inode_backtrace_t backtrace;
+  if (err == 0) {
+    ::decode(backtrace, bl);
+    if (backtrace.pool != info.pool && backtrace.pool != -1) {
+      dout(10) << " old object in pool " << info.pool
+	       << ", retrying pool " << backtrace.pool << dendl;
+      info.pool = backtrace.pool;
+      C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino);
+      fetch_backtrace(ino, info.pool, fin->bl, fin);
+      return;
+    }
+  } else if (err == -ENOENT) {
+    int64_t meta_pool = mds->mdsmap->get_metadata_pool();
+    if (info.pool != meta_pool) {
+      dout(10) << " no object in pool " << info.pool
+	       << ", retrying pool " << meta_pool << dendl;
+      info.pool = meta_pool;
+      C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino);
+      fetch_backtrace(ino, info.pool, fin->bl, fin);
+      return;
+    }
+  }
+
+  if (err == 0) {
+    if (backtrace.ancestors.empty()) {
+      dout(10) << " got empty backtrace " << dendl;
+      err = -EIO;
+    } else if (!info.ancestors.empty()) {
+      if (info.ancestors[0] == backtrace.ancestors[0]) {
+	dout(10) << " got same parents " << info.ancestors[0] << " 2 times" << dendl;
+	err = -EINVAL;
+      }
+    }
+  }
+  if (err) {
+    dout(10) << " failed to open ino " << ino << dendl;
+    open_ino_finish(ino, info, err);
+    return;
+  }
+
+  dout(10) << " got backtrace " << backtrace << dendl;
+  info.ancestors = backtrace.ancestors;
+
+  _open_ino_traverse_dir(ino, info, 0);
+}
+
+void MDCache::_open_ino_parent_opened(inodeno_t ino, int ret)
+{
+  dout(10) << "_open_ino_parent_opened ino " << ino << " ret " << ret << dendl;
+
+  assert(opening_inodes.count(ino));
+  open_ino_info_t& info = opening_inodes[ino];
+
+  CInode *in = get_inode(ino);
+  if (in) {
+    dout(10) << " found cached " << *in << dendl;
+    open_ino_finish(ino, info, in->authority().first);
+    return;
+  }
+
+  if (ret == mds->get_nodeid()) {
+    _open_ino_traverse_dir(ino, info, 0);
+  } else {
+    if (ret >= 0) {
+      info.check_peers = true;
+      info.auth_hint = ret;
+      info.checked.erase(ret);
+    }
+    do_open_ino(ino, info, ret);
+  }
+}
+
+Context* MDCache::_open_ino_get_waiter(inodeno_t ino, MMDSOpenIno *m)
+{
+  if (m)
+    return new C_MDS_RetryMessage(mds, m);
+  else
+    return new C_MDC_OpenInoTraverseDir(this, ino);
+}
+
+void MDCache::_open_ino_traverse_dir(inodeno_t ino, open_ino_info_t& info, int ret)
+{
+  dout(10) << "_open_ino_trvserse_dir ino " << ino << " ret " << ret << dendl;
+
+  CInode *in = get_inode(ino);
+  if (in) {
+    dout(10) << " found cached " << *in << dendl;
+    open_ino_finish(ino, info, in->authority().first);
+    return;
+  }
+
+  if (ret) {
+    do_open_ino(ino, info, ret);
+    return;
+  }
+
+  int hint = info.auth_hint;
+  ret = open_ino_traverse_dir(ino, NULL, info.ancestors,
+			      info.discover, info.want_xlocked, &hint);
+  if (ret > 0)
+    return;
+  if (hint != mds->get_nodeid())
+    info.auth_hint = hint;
+  do_open_ino(ino, info, ret);
+}
+
+void MDCache::_open_ino_fetch_dir(inodeno_t ino, MMDSOpenIno *m, CDir *dir)
+{
+  if (dir->state_test(CDir::STATE_REJOINUNDEF))
+    assert(dir->get_inode()->dirfragtree.is_leaf(dir->get_frag()));
+  dir->fetch(_open_ino_get_waiter(ino, m));
+}
+
+int MDCache::open_ino_traverse_dir(inodeno_t ino, MMDSOpenIno *m,
+				   vector& ancestors,
+				   bool discover, bool want_xlocked, int *hint)
+{
+  dout(10) << "open_ino_traverse_dir ino " << ino << " " << ancestors << dendl;
+  int err = 0;
+  for (unsigned i = 0; i < ancestors.size(); i++) {
+    CInode *diri = get_inode(ancestors[i].dirino);
+
+    if (!diri) {
+      if (discover && MDS_INO_IS_MDSDIR(ancestors[i].dirino)) {
+	open_foreign_mdsdir(ancestors[i].dirino, _open_ino_get_waiter(ino, m));
+	return 1;
+      }
+      continue;
+    }
+
+    if (diri->state_test(CInode::STATE_REJOINUNDEF)) {
+      CDir *dir = diri->get_parent_dir();
+      while (dir->state_test(CDir::STATE_REJOINUNDEF) &&
+	     dir->get_inode()->state_test(CInode::STATE_REJOINUNDEF))
+	dir = dir->get_inode()->get_parent_dir();
+      _open_ino_fetch_dir(ino, m, dir);
+      return 1;
+    }
+
+    if (!diri->is_dir()) {
+      dout(10) << " " << *diri << " is not dir" << dendl;
+      if (i == 0)
+	err = -ENOTDIR;
+      break;
+    }
+
+    string &name = ancestors[i].dname;
+    frag_t fg = diri->pick_dirfrag(name);
+    CDir *dir = diri->get_dirfrag(fg);
+    if (!dir) {
+      if (diri->is_auth()) {
+	if (diri->is_frozen()) {
+	  dout(10) << " " << *diri << " is frozen, waiting " << dendl;
+	  diri->add_waiter(CDir::WAIT_UNFREEZE, _open_ino_get_waiter(ino, m));
+	  return 1;
+	}
+	dir = diri->get_or_open_dirfrag(this, fg);
+      } else if (discover) {
+	open_remote_dirfrag(diri, fg, _open_ino_get_waiter(ino, m));
+	return 1;
+      }
+    }
+    if (dir) {
+      inodeno_t next_ino = i > 0 ? ancestors[i - 1].dirino : ino;
+      CDentry *dn = dir->lookup(name);
+      CDentry::linkage_t *dnl = dn ? dn->get_linkage() : NULL;
+      if (dir->is_auth()) {
+	if (dnl && dnl->is_primary() &&
+	    dnl->get_inode()->state_test(CInode::STATE_REJOINUNDEF)) {
+	  dout(10) << " fetching undef " << *dnl->get_inode() << dendl;
+	  _open_ino_fetch_dir(ino, m, dir);
+	  return 1;
+	}
+
+	if (!dnl && !dir->is_complete() &&
+	    (!dir->has_bloom() || dir->is_in_bloom(name))) {
+	  dout(10) << " fetching incomplete " << *dir << dendl;
+	  _open_ino_fetch_dir(ino, m, dir);
+	  return 1;
+	}
+
+	dout(10) << " no ino " << next_ino << " in " << *dir << dendl;
+	if (i == 0)
+	  err = -ENOENT;
+      } else if (discover) {
+	if (!dnl) {
+	  filepath path(name, 0);
+	  discover_path(dir, CEPH_NOSNAP, path, _open_ino_get_waiter(ino, m),
+			(i == 0 && want_xlocked));
+	  return 1;
+	}
+	if (dnl->is_null() && !dn->lock.can_read(-1)) {
+	  dout(10) << " null " << *dn << " is not readable, waiting" << dendl;
+	  dn->lock.add_waiter(SimpleLock::WAIT_RD, _open_ino_get_waiter(ino, m));
+	  return 1;
+	}
+	dout(10) << " no ino " << next_ino << " in " << *dir << dendl;
+	if (i == 0)
+	  err = -ENOENT;
+      }
+    }
+    if (hint && i == 0)
+      *hint = dir ? dir->authority().first : diri->authority().first;
+    break;
+  }
+  return err;
+}
+
+void MDCache::open_ino_finish(inodeno_t ino, open_ino_info_t& info, int ret)
+{
+  dout(10) << "open_ino_finish ino " << ino << " ret " << ret << dendl;
+
+  list waiters;
+  waiters.swap(info.waiters);
+  opening_inodes.erase(ino);
+  finish_contexts(g_ceph_context, waiters, ret);
+}
+
+void MDCache::do_open_ino(inodeno_t ino, open_ino_info_t& info, int err)
+{
+  if (err < 0) {
+    info.checked.clear();
+    info.checked.insert(mds->get_nodeid());
+    info.checking = -1;
+    info.check_peers = true;
+    info.fetch_backtrace = true;
+    if (info.discover) {
+      info.discover = false;
+      info.ancestors.clear();
+    }
+  }
+
+  if (info.check_peers) {
+    info.check_peers = false;
+    info.checking = -1;
+    do_open_ino_peer(ino, info);
+  } else if (info.fetch_backtrace) {
+    info.check_peers = true;
+    info.fetch_backtrace = false;
+    info.checking = mds->get_nodeid();
+    info.checked.clear();
+    info.checked.insert(mds->get_nodeid());
+    C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino);
+    fetch_backtrace(ino, info.pool, fin->bl, fin);
+  } else {
+    assert(!info.ancestors.empty());
+    info.checking = mds->get_nodeid();
+    open_ino(info.ancestors[0].dirino, mds->mdsmap->get_metadata_pool(),
+	     new C_MDC_OpenInoParentOpened(this, ino), info.want_replica);
+  }
+}
+
+void MDCache::do_open_ino_peer(inodeno_t ino, open_ino_info_t& info)
+{
+  set all, active;
+  mds->mdsmap->get_mds_set(all);
+  mds->mdsmap->get_clientreplay_or_active_or_stopping_mds_set(active);
+  if (mds->get_state() == MDSMap::STATE_REJOIN)
+    mds->mdsmap->get_mds_set(active, MDSMap::STATE_REJOIN);
+
+  dout(10) << "do_open_ino_peer " << ino << " active " << active
+	   << " all " << all << " checked " << info.checked << dendl;
+
+  int peer = -1;
+  if (info.auth_hint >= 0) {
+    if (active.count(info.auth_hint)) {
+      peer = info.auth_hint;
+      info.auth_hint = -1;
+    }
+  } else {
+    for (set::iterator p = active.begin(); p != active.end(); ++p)
+      if (*p != mds->get_nodeid() && info.checked.count(*p) == 0) {
+	peer = *p;
+	break;
+      }
+  }
+  if (peer < 0) {
+    if (all.size() > active.size() && all != info.checked) {
+      dout(10) << " waiting for more peers to be active" << dendl;
+    } else {
+      dout(10) << " all MDS peers have been checked " << dendl;
+      do_open_ino(ino, info, 0);
+    }
+  } else {
+    info.checking = peer;
+    mds->send_message_mds(new MMDSOpenIno(info.tid, ino, info.ancestors), peer);
+  }
+}
+
+void MDCache::handle_open_ino(MMDSOpenIno *m)
+{
+  dout(10) << "handle_open_ino " << *m << dendl;
+
+  inodeno_t ino = m->ino;
+  MMDSOpenInoReply *reply;
+  CInode *in = get_inode(ino);
+  if (in) {
+    dout(10) << " have " << *in << dendl;
+    reply = new MMDSOpenInoReply(m->get_tid(), ino, 0);
+    if (in->is_auth()) {
+      touch_inode(in);
+      while (1) {
+	CDentry *pdn = in->get_parent_dn();
+	if (!pdn)
+	  break;
+	CInode *diri = pdn->get_dir()->get_inode();
+	reply->ancestors.push_back(inode_backpointer_t(diri->ino(), pdn->name,
+						       in->inode.version));
+	in = diri;
+      }
+    } else {
+      reply->hint = in->authority().first;
+    }
+  } else {
+    int hint = -1;
+    int ret = open_ino_traverse_dir(ino, m, m->ancestors, false, false, &hint);
+    if (ret > 0)
+      return;
+    reply = new MMDSOpenInoReply(m->get_tid(), ino, hint, ret);
+  }
+  mds->messenger->send_message(reply, m->get_connection());
+  m->put();
+}
+
+void MDCache::handle_open_ino_reply(MMDSOpenInoReply *m)
+{
+  dout(10) << "handle_open_ino_reply " << *m << dendl;
+
+  inodeno_t ino = m->ino;
+  int from = m->get_source().num();
+  if (opening_inodes.count(ino)) {
+    open_ino_info_t& info = opening_inodes[ino];
+
+    if (info.checking == from)
+	info.checking = -1;
+    info.checked.insert(from);
+
+    CInode *in = get_inode(ino);
+    if (in) {
+      dout(10) << " found cached " << *in << dendl;
+      open_ino_finish(ino, info, in->authority().first);
+    } else if (!m->ancestors.empty()) {
+      dout(10) << " found ino " << ino << " on mds." << from << dendl;
+      if (!info.want_replica) {
+	open_ino_finish(ino, info, from);
+	return;
+      }
+
+      info.ancestors = m->ancestors;
+      info.auth_hint = from;
+      info.checking = mds->get_nodeid();
+      info.discover = true;
+      _open_ino_traverse_dir(ino, info, 0);
+    } else if (m->error) {
+      dout(10) << " error " << m->error << " from mds." << from << dendl;
+      do_open_ino(ino, info, m->error);
+    } else {
+      if (m->hint >= 0 && m->hint != mds->get_nodeid()) {
+	info.auth_hint = m->hint;
+	info.checked.erase(m->hint);
+      }
+      do_open_ino_peer(ino, info);
+    }
+  }
+  m->put();
+}
+
+void MDCache::kick_open_ino_peers(int who)
+{
+  dout(10) << "kick_open_ino_peers mds." << who << dendl;
+
+  for (map::iterator p = opening_inodes.begin();
+       p != opening_inodes.end();
+       ++p) {
+    open_ino_info_t& info = p->second;
+    if (info.checking == who) {
+      dout(10) << "  kicking ino " << p->first << " who was checking mds." << who << dendl;
+      info.checking = -1;
+      do_open_ino_peer(p->first, info);
+    } else if (info.checking == -1) {
+      dout(10) << "  kicking ino " << p->first << " who was waiting" << dendl;
+      do_open_ino_peer(p->first, info);
+    }
+  }
+}
+
+void MDCache::open_ino(inodeno_t ino, int64_t pool, Context* fin,
+		       bool want_replica, bool want_xlocked)
+{
+  dout(10) << "open_ino " << ino << " pool " << pool << " want_replica "
+	   << want_replica << dendl;
+
+  if (opening_inodes.count(ino)) {
+    open_ino_info_t& info = opening_inodes[ino];
+    if (want_replica) {
+      info.want_replica = true;
+      if (want_xlocked && !info.want_xlocked) {
+	if (!info.ancestors.empty()) {
+	  CInode *diri = get_inode(info.ancestors[0].dirino);
+	  if (diri) {
+	    frag_t fg = diri->pick_dirfrag(info.ancestors[0].dname);
+	    CDir *dir = diri->get_dirfrag(fg);
+	    if (dir && !dir->is_auth()) {
+	      filepath path(info.ancestors[0].dname, 0);
+	      discover_path(dir, CEPH_NOSNAP, path, NULL, true);
+	    }
+	  }
+	}
+	info.want_xlocked = true;
+      }
+    }
+    info.waiters.push_back(fin);
+  } else {
+    open_ino_info_t& info = opening_inodes[ino];
+    info.checked.insert(mds->get_nodeid());
+    info.want_replica = want_replica;
+    info.want_xlocked = want_xlocked;
+    info.tid = ++open_ino_last_tid;
+    info.pool = pool >= 0 ? pool : default_file_layout.fl_pg_pool;
+    info.waiters.push_back(fin);
+    do_open_ino(ino, info, 0);
+  }
+}
+
+/* ---------------------------- */
+
+/*
+ * search for a given inode on MDS peers.  optionally start with the given node.
+
+
+ TODO 
+  - recover from mds node failure, recovery
+  - traverse path
+
+ */
+void MDCache::find_ino_peers(inodeno_t ino, Context *c, int hint)
+{
+  dout(5) << "find_ino_peers " << ino << " hint " << hint << dendl;
+  assert(!have_inode(ino));
+  
+  ceph_tid_t tid = ++find_ino_peer_last_tid;
+  find_ino_peer_info_t& fip = find_ino_peer[tid];
+  fip.ino = ino;
+  fip.tid = tid;
+  fip.fin = c;
+  fip.hint = hint;
+  fip.checked.insert(mds->whoami);
+  _do_find_ino_peer(fip);
+}
+
+void MDCache::_do_find_ino_peer(find_ino_peer_info_t& fip)
+{
+  set all, active;
+  mds->mdsmap->get_mds_set(all);
+  mds->mdsmap->get_clientreplay_or_active_or_stopping_mds_set(active);
+
+  dout(10) << "_do_find_ino_peer " << fip.tid << " " << fip.ino
+	   << " active " << active << " all " << all
+	   << " checked " << fip.checked
+	   << dendl;
+    
+  int m = -1;
+  if (fip.hint >= 0) {
+    m = fip.hint;
+    fip.hint = -1;
+  } else {
+    for (set::iterator p = active.begin(); p != active.end(); ++p)
+      if (*p != mds->whoami &&
+	  fip.checked.count(*p) == 0) {
+	m = *p;
+	break;
+      }
+  }
+  if (m < 0) {
+    if (all.size() > active.size()) {
+      dout(10) << "_do_find_ino_peer waiting for more peers to be active" << dendl;
+    } else {
+      dout(10) << "_do_find_ino_peer failed on " << fip.ino << dendl;
+      fip.fin->complete(-ESTALE);
+      find_ino_peer.erase(fip.tid);
+    }
+  } else {
+    fip.checking = m;
+    mds->send_message_mds(new MMDSFindIno(fip.tid, fip.ino), m);
+  }
+}
+
+void MDCache::handle_find_ino(MMDSFindIno *m)
+{
+  dout(10) << "handle_find_ino " << *m << dendl;
+  MMDSFindInoReply *r = new MMDSFindInoReply(m->tid);
+  CInode *in = get_inode(m->ino);
+  if (in) {
+    in->make_path(r->path);
+    dout(10) << " have " << r->path << " " << *in << dendl;
+  }
+  mds->messenger->send_message(r, m->get_connection());
+  m->put();
+}
+
+
+void MDCache::handle_find_ino_reply(MMDSFindInoReply *m)
+{
+  map::iterator p = find_ino_peer.find(m->tid);
+  if (p != find_ino_peer.end()) {
+    dout(10) << "handle_find_ino_reply " << *m << dendl;
+    find_ino_peer_info_t& fip = p->second;
+
+    // success?
+    if (get_inode(fip.ino)) {
+      dout(10) << "handle_find_ino_reply successfully found " << fip.ino << dendl;
+      mds->queue_waiter(fip.fin);
+      find_ino_peer.erase(p);
+      m->put();
+      return;
+    }
+
+    int from = m->get_source().num();
+    if (fip.checking == from)
+      fip.checking = -1;
+    fip.checked.insert(from);
+
+    if (!m->path.empty()) {
+      // we got a path!
+      vector trace;
+      MDRequestRef null_ref;
+      int r = path_traverse(null_ref, m, NULL, m->path, &trace, NULL, MDS_TRAVERSE_DISCOVER);
+      if (r > 0)
+	return; 
+      dout(0) << "handle_find_ino_reply failed with " << r << " on " << m->path 
+	      << ", retrying" << dendl;
+      fip.checked.clear();
+      _do_find_ino_peer(fip);
+    } else {
+      // nope, continue.
+      _do_find_ino_peer(fip);
+    }      
+  } else {
+    dout(10) << "handle_find_ino_reply tid " << m->tid << " dne" << dendl;
+  }  
+  m->put();
+}
+
+void MDCache::kick_find_ino_peers(int who)
+{
+  // find_ino_peers requests we should move on from
+  for (map::iterator p = find_ino_peer.begin();
+       p != find_ino_peer.end();
+       ++p) {
+    find_ino_peer_info_t& fip = p->second;
+    if (fip.checking == who) {
+      dout(10) << "kicking find_ino_peer " << fip.tid << " who was checking mds." << who << dendl;
+      fip.checking = -1;
+      _do_find_ino_peer(fip);
+    } else if (fip.checking == -1) {
+      dout(10) << "kicking find_ino_peer " << fip.tid << " who was waiting" << dendl;
+      _do_find_ino_peer(fip);
+    }
+  }
+}
+
+/* ---------------------------- */
+
+int MDCache::get_num_client_requests()
+{
+  int count = 0;
+  for (ceph::unordered_map::iterator p = active_requests.begin();
+      p != active_requests.end();
+      ++p) {
+    MDRequestRef& mdr = p->second;
+    if (mdr->reqid.name.is_client() && !mdr->is_slave())
+      count++;
+  }
+  return count;
+}
+
+/* This function takes over the reference to the passed Message */
+MDRequestRef MDCache::request_start(MClientRequest *req)
+{
+  // did we win a forward race against a slave?
+  if (active_requests.count(req->get_reqid())) {
+    MDRequestRef& mdr = active_requests[req->get_reqid()];
+    assert(mdr);
+    if (mdr->is_slave()) {
+      dout(10) << "request_start already had " << *mdr << ", waiting for finish" << dendl;
+      mdr->more()->waiting_for_finish.push_back(new C_MDS_RetryMessage(mds, req));
+    } else {
+      dout(10) << "request_start already processing " << *mdr << ", dropping new msg" << dendl;
+      req->put();
+    }
+    return MDRequestRef();
+  }
+
+  // register new client request
+  MDRequestRef mdr(new MDRequestImpl(req->get_reqid(),
+                                     req->get_num_fwd(), req));
+  active_requests[req->get_reqid()] = mdr;
+  dout(7) << "request_start " << *mdr << dendl;
+  return mdr;
+}
+
+MDRequestRef MDCache::request_start_slave(metareqid_t ri, __u32 attempt, int by)
+{
+  MDRequestRef mdr(new MDRequestImpl(ri, attempt, by));
+  assert(active_requests.count(mdr->reqid) == 0);
+  active_requests[mdr->reqid] = mdr;
+  dout(7) << "request_start_slave " << *mdr << " by mds." << by << dendl;
+  return mdr;
+}
+
+MDRequestRef MDCache::request_start_internal(int op)
+{
+  MDRequestRef mdr(new MDRequestImpl);
+  mdr->reqid.name = entity_name_t::MDS(mds->get_nodeid());
+  mdr->reqid.tid = mds->issue_tid();
+  mdr->internal_op = op;
+
+  assert(active_requests.count(mdr->reqid) == 0);
+  active_requests[mdr->reqid] = mdr;
+  dout(7) << "request_start_internal " << *mdr << " op " << op << dendl;
+  return mdr;
+}
+
+MDRequestRef MDCache::request_get(metareqid_t rid)
+{
+  ceph::unordered_map::iterator p = active_requests.find(rid);
+  assert(p != active_requests.end());
+  dout(7) << "request_get " << rid << " " << *p->second << dendl;
+  return p->second;
+}
+
+void MDCache::request_finish(MDRequestRef& mdr)
+{
+  dout(7) << "request_finish " << *mdr << dendl;
+
+  // slave finisher?
+  if (mdr->has_more() && mdr->more()->slave_commit) {
+    Context *fin = mdr->more()->slave_commit;
+    mdr->more()->slave_commit = 0;
+    fin->complete(mdr->aborted ? -1 : 0);   // this must re-call request_finish.
+    return; 
+  }
+
+  request_cleanup(mdr);
+}
+
+
+void MDCache::request_forward(MDRequestRef& mdr, int who, int port)
+{
+  if (mdr->client_request->get_source().is_client()) {
+    dout(7) << "request_forward " << *mdr << " to mds." << who << " req "
+            << *mdr->client_request << dendl;
+    mds->forward_message_mds(mdr->client_request, who);
+    mdr->client_request = 0;
+    if (mds->logger) mds->logger->inc(l_mds_fw);
+  } else {
+    dout(7) << "request_forward drop " << *mdr << " req " << *mdr->client_request
+            << " was from mds" << dendl;
+  }
+  request_cleanup(mdr);
+}
+
+
+void MDCache::dispatch_request(MDRequestRef& mdr)
+{
+  if (mdr->killed) {
+    dout(10) << "request " << *mdr << " was killed" << dendl;
+    return;
+  }
+  if (mdr->client_request) {
+    mds->server->dispatch_client_request(mdr);
+  } else if (mdr->slave_request) {
+    mds->server->dispatch_slave_request(mdr);
+  } else {
+    switch (mdr->internal_op) {
+    case CEPH_MDS_OP_FRAGMENTDIR:
+      dispatch_fragment_dir(mdr);
+      break;
+    case CEPH_MDS_OP_EXPORTDIR:
+      migrator->dispatch_export_dir(mdr);
+      break;
+    default:
+      assert(0);
+    }
+  }
+}
+
+
+void MDCache::request_drop_foreign_locks(MDRequestRef& mdr)
+{
+  if (!mdr->has_more())
+    return;
+
+  // clean up slaves
+  //  (will implicitly drop remote dn pins)
+  for (set::iterator p = mdr->more()->slaves.begin();
+       p != mdr->more()->slaves.end();
+       ++p) {
+    MMDSSlaveRequest *r = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+					       MMDSSlaveRequest::OP_FINISH);
+
+    // information about rename imported caps
+    if (mdr->more()->srcdn_auth_mds == *p &&
+	mdr->more()->inode_import.length() > 0)
+      r->inode_export.claim(mdr->more()->inode_import);
+
+    mds->send_message_mds(r, *p);
+  }
+
+  /* strip foreign xlocks out of lock lists, since the OP_FINISH drops them
+   * implicitly. Note that we don't call the finishers -- there shouldn't
+   * be any on a remote lock and the request finish wakes up all
+   * the waiters anyway! */
+  set::iterator p = mdr->xlocks.begin();
+  while (p != mdr->xlocks.end()) {
+    if ((*p)->get_parent()->is_auth()) 
+      ++p;
+    else {
+      dout(10) << "request_drop_foreign_locks forgetting lock " << **p
+	       << " on " << *(*p)->get_parent() << dendl;
+      (*p)->put_xlock();
+      mdr->locks.erase(*p);
+      mdr->xlocks.erase(p++);
+    }
+  }
+
+  map::iterator q = mdr->remote_wrlocks.begin();
+  while (q != mdr->remote_wrlocks.end()) {
+    dout(10) << "request_drop_foreign_locks forgetting remote_wrlock " << *q->first
+	     << " on mds." << q->second
+	     << " on " << *(q->first)->get_parent() << dendl;
+    mdr->locks.erase(q->first);
+    mdr->remote_wrlocks.erase(q++);
+  }
+
+  mdr->more()->slaves.clear(); /* we no longer have requests out to them, and
+                                * leaving them in can cause double-notifies as
+                                * this function can get called more than once */
+}
+
+void MDCache::request_drop_non_rdlocks(MDRequestRef& mdr)
+{
+  request_drop_foreign_locks(mdr);
+  mds->locker->drop_non_rdlocks(mdr.get());
+}
+
+void MDCache::request_drop_locks(MDRequestRef& mdr)
+{
+  request_drop_foreign_locks(mdr);
+  mds->locker->drop_locks(mdr.get());
+}
+
+void MDCache::request_cleanup(MDRequestRef& mdr)
+{
+  dout(15) << "request_cleanup " << *mdr << dendl;
+
+  if (mdr->has_more()) {
+    if (mdr->more()->is_ambiguous_auth)
+      mdr->clear_ambiguous_auth();
+    if (!mdr->more()->waiting_for_finish.empty())
+      mds->queue_waiters(mdr->more()->waiting_for_finish);
+  }
+
+  request_drop_locks(mdr);
+
+  // drop (local) auth pins
+  mdr->drop_local_auth_pins();
+
+  // drop stickydirs
+  for (set::iterator p = mdr->stickydirs.begin();
+       p != mdr->stickydirs.end();
+       ++p) 
+    (*p)->put_stickydirs();
+
+  mds->locker->kick_cap_releases(mdr);
+
+  // drop cache pins
+  mdr->drop_pins();
+
+  // remove from session
+  mdr->item_session_request.remove_myself();
+
+  bool was_replay = mdr->client_request && mdr->client_request->is_replay();
+
+  // remove from map
+  active_requests.erase(mdr->reqid);
+
+  // fail-safe!
+  if (was_replay && active_requests.empty()) {
+    dout(10) << " fail-safe queueing next replay op" << dendl;
+    mds->queue_one_replay();
+  }
+
+  if (mds->logger)
+    log_stat();
+}
+
+void MDCache::request_kill(MDRequestRef& mdr)
+{
+  mdr->killed = true;
+  if (!mdr->committing) {
+    dout(10) << "request_kill " << *mdr << dendl;
+    request_cleanup(mdr);
+  } else {
+    dout(10) << "request_kill " << *mdr << " -- already committing, no-op" << dendl;
+  }
+}
+
+
+// --------------------------------------------------------------------
+// ANCHORS
+
+class C_MDC_AnchorPrepared : public Context {
+  MDCache *cache;
+  CInode *in;
+  bool add;
+public:
+  version_t atid;
+  C_MDC_AnchorPrepared(MDCache *c, CInode *i, bool a) : cache(c), in(i), add(a), atid(0) {}
+  void finish(int r) {
+    cache->_anchor_prepared(in, atid, add);
+  }
+};
+
+void MDCache::anchor_create_prep_locks(MDRequestRef& mdr, CInode *in,
+				       set& rdlocks, set& xlocks)
+{
+  dout(10) << "anchor_create_prep_locks " << *in << dendl;
+
+  if (in->is_anchored()) {
+    // caller may have already xlocked it.. if so, that will suffice!
+    if (xlocks.count(&in->linklock) == 0)
+      rdlocks.insert(&in->linklock);
+  } else {
+    xlocks.insert(&in->linklock);
+
+    // path components too!
+    CDentry *dn = in->get_projected_parent_dn();
+    while (dn) {
+      rdlocks.insert(&dn->lock);
+      dn = dn->get_dir()->get_inode()->get_parent_dn();
+    }
+  }
+}
+
+void MDCache::anchor_create(MDRequestRef& mdr, CInode *in, Context *onfinish)
+{
+  assert(in->is_auth());
+  dout(10) << "anchor_create " << *in << dendl;
+
+  // auth pin
+  if (!in->can_auth_pin() &&
+      !mdr->is_auth_pinned(in)) {
+    dout(7) << "anchor_create not authpinnable, waiting on " << *in << dendl;
+    in->add_waiter(CInode::WAIT_UNFREEZE, onfinish);
+    return;
+  }
+
+  // wait
+  in->add_waiter(CInode::WAIT_ANCHORED, onfinish);
+
+  // already anchoring?
+  if (in->state_test(CInode::STATE_ANCHORING)) {
+    dout(7) << "anchor_create already anchoring " << *in << dendl;
+    return;
+  }
+
+  dout(7) << "anchor_create " << *in << dendl;
+
+  // auth: do it
+  in->state_set(CInode::STATE_ANCHORING);
+  in->get(CInode::PIN_ANCHORING);
+  in->auth_pin(this);
+  
+  // make trace
+  vector trace;
+  in->make_anchor_trace(trace);
+  if (trace.empty()) {
+    assert(MDS_INO_IS_BASE(in->ino()));
+    trace.push_back(Anchor(in->ino(), in->ino(), 0, 0, 0));
+  }
+  
+  // do it
+  C_MDC_AnchorPrepared *fin = new C_MDC_AnchorPrepared(this, in, true);
+  mds->anchorclient->prepare_create(in->ino(), trace, &fin->atid, fin);
+}
+
+void MDCache::anchor_destroy(CInode *in, Context *onfinish)
+{
+  assert(in->is_auth());
+
+  // auth pin
+  if (!in->can_auth_pin()/* &&
+			    !mdr->is_auth_pinned(in)*/) {
+    dout(7) << "anchor_destroy not authpinnable, waiting on " << *in << dendl;
+    in->add_waiter(CInode::WAIT_UNFREEZE, onfinish);
+    return;
+  }
+
+  dout(7) << "anchor_destroy " << *in << dendl;
+
+  // wait
+  if (onfinish)
+    in->add_waiter(CInode::WAIT_UNANCHORED, onfinish);
+
+  // already anchoring?
+  if (in->state_test(CInode::STATE_UNANCHORING)) {
+    dout(7) << "anchor_destroy already unanchoring " << *in << dendl;
+    return;
+  }
+
+  // auth: do it
+  in->state_set(CInode::STATE_UNANCHORING);
+  in->get(CInode::PIN_UNANCHORING);
+  in->auth_pin(this);
+  
+  // do it
+  C_MDC_AnchorPrepared *fin = new C_MDC_AnchorPrepared(this, in, false);
+  mds->anchorclient->prepare_destroy(in->ino(), &fin->atid, fin);
+}
+
+class C_MDC_AnchorLogged : public Context {
+  MDCache *cache;
+  CInode *in;
+  version_t atid;
+  MutationRef mut;
+public:
+  C_MDC_AnchorLogged(MDCache *c, CInode *i, version_t t, MutationRef& m) :
+    cache(c), in(i), atid(t), mut(m) {}
+  void finish(int r) {
+    cache->_anchor_logged(in, atid, mut);
+  }
+};
+
+void MDCache::_anchor_prepared(CInode *in, version_t atid, bool add)
+{
+  dout(10) << "_anchor_prepared " << *in << " atid " << atid 
+	   << " " << (add ? "create":"destroy") << dendl;
+  assert(in->inode.anchored == !add);
+
+  // update the logged inode copy
+  inode_t *pi = in->project_inode();
+  if (add) {
+    pi->anchored = true;
+    pi->rstat.ranchors++;
+  } else {
+    pi->anchored = false;
+    pi->rstat.ranchors--;
+  }
+  pi->version = in->pre_dirty();
+
+  MutationRef mut(new MutationImpl);
+  mut->ls = mds->mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mds->mdlog, add ? "anchor_create":"anchor_destroy");
+  mds->mdlog->start_entry(le);
+  predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY);
+  journal_dirty_inode(mut.get(), &le->metablob, in);
+  le->metablob.add_table_transaction(TABLE_ANCHOR, atid);
+  mds->mdlog->submit_entry(le, new C_MDC_AnchorLogged(this, in, atid, mut));
+  mds->mdlog->flush();
+}
+
+
+void MDCache::_anchor_logged(CInode *in, version_t atid, MutationRef& mut)
+{
+  dout(10) << "_anchor_logged on " << *in << dendl;
+
+  // unpin
+  if (in->state_test(CInode::STATE_ANCHORING)) {
+    in->state_clear(CInode::STATE_ANCHORING);
+    in->put(CInode::PIN_ANCHORING);
+    if (in->parent)
+      in->parent->adjust_nested_anchors(1);
+  } else if (in->state_test(CInode::STATE_UNANCHORING)) {
+    in->state_clear(CInode::STATE_UNANCHORING);
+    in->put(CInode::PIN_UNANCHORING);
+    if (in->parent)
+      in->parent->adjust_nested_anchors(-1);
+  }
+  in->auth_unpin(this);
+  
+  // apply update to cache
+  in->pop_and_dirty_projected_inode(mut->ls);
+  mut->apply();
+  
+  // tell the anchortable we've committed
+  mds->anchorclient->commit(atid, mut->ls);
+
+  // drop locks and finish
+  mds->locker->drop_locks(mut.get());
+  mut->cleanup();
+
+  // trigger waiters
+  in->finish_waiting(CInode::WAIT_ANCHORED|CInode::WAIT_UNANCHORED, 0);
+}
+
+
+// -------------------------------------------------------------------------------
+// SNAPREALMS
+
+struct C_MDC_snaprealm_create_finish : public Context {
+  MDCache *cache;
+  MDRequestRef mdr;
+  MutationRef mut;
+  CInode *in;
+  C_MDC_snaprealm_create_finish(MDCache *c, MDRequestRef& m,
+                                MutationRef& mu, CInode *i) :
+    cache(c), mdr(m), mut(mu), in(i) {}
+  void finish(int r) {
+    cache->_snaprealm_create_finish(mdr, mut, in);
+  }
+};
+
+void MDCache::snaprealm_create(MDRequestRef& mdr, CInode *in)
+{
+  dout(10) << "snaprealm_create " << *in << dendl;
+  assert(!in->snaprealm);
+
+  if (!in->inode.anchored) {
+    mds->mdcache->anchor_create(mdr, in, new C_MDS_RetryRequest(mds->mdcache, mdr));
+    return;
+  }
+
+  // allocate an id..
+  if (!mdr->more()->stid) {
+    mds->snapclient->prepare_create_realm(in->ino(), &mdr->more()->stid, &mdr->more()->snapidbl,
+					  new C_MDS_RetryRequest(this, mdr));
+    return;
+  }
+
+  MutationRef mut(new MutationImpl);
+  mut->ls = mds->mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mds->mdlog, "snaprealm_create");
+  mds->mdlog->start_entry(le);
+
+  le->metablob.add_table_transaction(TABLE_SNAP, mdr->more()->stid);
+
+  inode_t *pi = in->project_inode();
+  pi->version = in->pre_dirty();
+  pi->rstat.rsnaprealms++;
+
+  bufferlist::iterator p = mdr->more()->snapidbl.begin();
+  snapid_t seq;
+  ::decode(seq, p);
+
+  sr_t *newsnap = in->project_snaprealm(seq);
+  newsnap->seq = seq;
+  newsnap->last_created = seq;
+  
+  predirty_journal_parents(mut, &le->metablob, in, 0, PREDIRTY_PRIMARY);
+  journal_cow_inode(mut, &le->metablob, in);
+  le->metablob.add_primary_dentry(in->get_projected_parent_dn(), in, true);
+
+  mds->mdlog->submit_entry(le, new C_MDC_snaprealm_create_finish(this, mdr, mut, in));
+  mds->mdlog->flush();
+}
+
+
+void MDCache::do_realm_invalidate_and_update_notify(CInode *in, int snapop, bool nosend)
+{
+  dout(10) << "do_realm_invalidate_and_update_notify " << *in->snaprealm << " " << *in << dendl;
+
+  vector split_inos;
+  vector split_realms;
+
+  if (snapop == CEPH_SNAP_OP_SPLIT) {
+    // notify clients of update|split
+    for (elist::iterator p = in->snaprealm->inodes_with_caps.begin(member_offset(CInode, item_caps));
+	 !p.end(); ++p)
+      split_inos.push_back((*p)->ino());
+    
+    for (set::iterator p = in->snaprealm->open_children.begin();
+	 p != in->snaprealm->open_children.end();
+	 ++p)
+      split_realms.push_back((*p)->inode->ino());
+  }
+
+  bufferlist snapbl;
+  in->snaprealm->build_snap_trace(snapbl);
+
+  map updates;
+  list q;
+  q.push_back(in->snaprealm);
+  while (!q.empty()) {
+    SnapRealm *realm = q.front();
+    q.pop_front();
+
+    dout(10) << " realm " << *realm << " on " << *realm->inode << dendl;
+    realm->invalidate_cached_snaps();
+
+    for (map* >::iterator p = realm->client_caps.begin();
+	 p != realm->client_caps.end();
+	 ++p) {
+      assert(!p->second->empty());
+      if (!nosend && updates.count(p->first) == 0) {
+	MClientSnap *update = new MClientSnap(snapop);
+	update->head.split = in->ino();
+	update->split_inos = split_inos;
+	update->split_realms = split_realms;
+	update->bl = snapbl;
+	updates[p->first] = update;
+      }
+    }
+
+    // notify for active children, too.
+    dout(10) << " " << realm << " open_children are " << realm->open_children << dendl;
+    for (set::iterator p = realm->open_children.begin();
+	 p != realm->open_children.end();
+	 ++p)
+      q.push_back(*p);
+  }
+
+  if (!nosend)
+    send_snaps(updates);
+}
+
+void MDCache::_snaprealm_create_finish(MDRequestRef& mdr, MutationRef& mut, CInode *in)
+{
+  dout(10) << "_snaprealm_create_finish " << *in << dendl;
+
+  // apply
+  in->pop_and_dirty_projected_inode(mut->ls);
+  mut->apply();
+  mds->locker->drop_locks(mut.get());
+  mut->cleanup();
+
+  // tell table we've committed
+  mds->snapclient->commit(mdr->more()->stid, mut->ls);
+
+  // create
+  bufferlist::iterator p = mdr->more()->snapidbl.begin();
+  snapid_t seq;
+  ::decode(seq, p);
+
+  in->open_snaprealm();
+  in->snaprealm->open = true;
+  in->snaprealm->srnode.seq = seq;
+  in->snaprealm->srnode.created = seq;
+  in->snaprealm->srnode.current_parent_since = seq;
+
+  do_realm_invalidate_and_update_notify(in, CEPH_SNAP_OP_SPLIT);
+
+  /*
+  static int count = 5;
+  if (--count == 0)
+    assert(0);  // hack test test **********
+  */
+
+  // done.
+  mdr->more()->stid = 0;  // caller will likely need to reuse this
+  dispatch_request(mdr);
+}
+
+
+// -------------------------------------------------------------------------------
+// STRAYS
+
+struct C_MDC_RetryScanStray : public Context {
+  MDCache *cache;
+  dirfrag_t next;
+  C_MDC_RetryScanStray(MDCache *c,  dirfrag_t n) : cache(c), next(n) { }
+  void finish(int r) {
+    cache->scan_stray_dir(next);
+  }
+};
+
+void MDCache::scan_stray_dir(dirfrag_t next)
+{
+  dout(10) << "scan_stray_dir " << next << dendl;
+
+  list ls;
+  for (int i = 0; i < NUM_STRAY; ++i) {
+    if (strays[i]->ino() < next.ino)
+      continue;
+    strays[i]->get_dirfrags(ls);
+  }
+
+  for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+    CDir *dir = *p;
+    if (dir->dirfrag() < next)
+      continue;
+    if (!dir->is_complete()) {
+      dir->fetch(new C_MDC_RetryScanStray(this, dir->dirfrag()));
+      return;
+    }
+    for (CDir::map_t::iterator q = dir->items.begin(); q != dir->items.end(); ++q) {
+      CDentry *dn = q->second;
+      CDentry::linkage_t *dnl = dn->get_projected_linkage();
+      if (dnl->is_primary())
+	maybe_eval_stray(dnl->get_inode());
+    }
+  }
+}
+
+struct C_MDC_EvalStray : public Context {
+  MDCache *mdcache;
+  CDentry *dn;
+  C_MDC_EvalStray(MDCache *c, CDentry *d) : mdcache(c), dn(d) {}
+  void finish(int r) {
+    mdcache->eval_stray(dn);
+  }
+};
+
+void MDCache::eval_stray(CDentry *dn, bool delay)
+{
+  dout(10) << "eval_stray " << *dn << dendl;
+  CDentry::linkage_t *dnl = dn->get_projected_linkage();
+  dout(10) << " inode is " << *dnl->get_inode() << dendl;
+  assert(dnl->is_primary());
+  CInode *in = dnl->get_inode();
+  assert(in);
+
+  assert(dn->get_dir()->get_inode()->is_stray());
+
+  if (!dn->is_auth()) {
+    // has to be mine
+    // move to bottom of lru so that we trim quickly!
+    touch_dentry_bottom(dn);
+    return;
+  }
+
+  // purge?
+  if (in->inode.nlink == 0) {
+    if (in->is_dir()) {
+      // past snaprealm parents imply snapped dentry remote links.
+      // only important for directories.  normal file data snaps are handled
+      // by the object store.
+      if (in->snaprealm && in->snaprealm->has_past_parents()) {
+	if (!in->snaprealm->have_past_parents_open() &&
+	    !in->snaprealm->open_parents(new C_MDC_EvalStray(this, dn)))
+	  return;
+	in->snaprealm->prune_past_parents();
+	if (in->snaprealm->has_past_parents()) {
+	  dout(20) << "  has past parents " << in->snaprealm->srnode.past_parents << dendl;
+	  return;  // not until some snaps are deleted.
+	}
+      }
+    }
+    if (dn->is_replicated()) {
+      dout(20) << " replicated" << dendl;
+      return;
+    }
+    if (dn->is_any_leases() || in->is_any_caps()) {
+      dout(20) << " caps | leases" << dendl;
+      return;  // wait
+    }
+    if (!in->dirfrags.empty()) {
+      dout(20) << " open dirfrags" << dendl;
+      return;  // wait for dirs to close/trim
+    }
+    if (dn->state_test(CDentry::STATE_PURGING)) {
+      dout(20) << " already purging" << dendl;
+      return;  // already purging
+    }
+    if (in->state_test(CInode::STATE_NEEDSRECOVER) ||
+	in->state_test(CInode::STATE_RECOVERING)) {
+      dout(20) << " pending recovery" << dendl;
+      return;  // don't mess with file size probing
+    }
+    if (in->get_num_ref() > (int)in->is_dirty() + (int)in->is_dirty_parent()) {
+      dout(20) << " too many inode refs" << dendl;
+      return;
+    }
+    if (dn->get_num_ref() > (int)dn->is_dirty() + !!in->get_num_ref()) {
+      dout(20) << " too many dn refs" << dendl;
+      return;
+    }
+    if (delay) {
+      if (!dn->item_stray.is_on_list())
+	delayed_eval_stray.push_back(&dn->item_stray);
+    } else
+      purge_stray(dn);
+  }
+  else if (in->inode.nlink >= 1) {
+    // trivial reintegrate?
+    if (!in->remote_parents.empty()) {
+      CDentry *rlink = *in->remote_parents.begin();
+      
+      // don't do anything if the remote parent is projected, or we may
+      // break user-visible semantics!
+      // NOTE: we repeat this check in _rename(), since our submission path is racey.
+      if (!rlink->is_projected()) {
+	if (rlink->is_auth() && rlink->dir->can_auth_pin())
+	  reintegrate_stray(dn, rlink);
+	
+	if (!rlink->is_auth() && dn->is_auth())
+	  migrate_stray(dn, rlink->authority().first);
+      }
+    }
+  } else {
+    // wait for next use.
+  }
+}
+
+void MDCache::eval_remote(CDentry *dn)
+{
+  dout(10) << "eval_remote " << *dn << dendl;
+  CDentry::linkage_t *dnl = dn->get_projected_linkage();
+  assert(dnl->is_remote());
+  CInode *in = dnl->get_inode();
+  if (!in) return;
+
+  // refers to stray?
+  if (in->get_parent_dn()->get_dir()->get_inode()->is_stray()) {
+    if (in->is_auth())
+      eval_stray(in->get_parent_dn());
+    else
+      migrate_stray(in->get_parent_dn(), mds->get_nodeid());
+  }
+}
+
+void MDCache::fetch_backtrace(inodeno_t ino, int64_t pool, bufferlist& bl, Context *fin)
+{
+  object_t oid = CInode::get_object_name(ino, frag_t(), "");
+  mds->objecter->getxattr(oid, object_locator_t(pool), "parent", CEPH_NOSNAP, &bl, 0, fin);
+}
+
+class C_MDC_PurgeStrayPurged : public Context {
+  MDCache *cache;
+  CDentry *dn;
+public:
+  C_MDC_PurgeStrayPurged(MDCache *c, CDentry *d) : 
+    cache(c), dn(d) { }
+  void finish(int r) {
+    assert(r == 0 || r == -ENOENT);
+    cache->_purge_stray_purged(dn, r);
+  }
+};
+
+void MDCache::purge_stray(CDentry *dn)
+{
+  CDentry::linkage_t *dnl = dn->get_projected_linkage();
+  CInode *in = dnl->get_inode();
+  dout(10) << "purge_stray " << *dn << " " << *in << dendl;
+  assert(!dn->is_replicated());
+
+  // anchored?
+  if (in->inode.anchored) {
+    anchor_destroy(in, new C_MDC_EvalStray(this, dn));
+    return;
+  }
+
+  dn->state_set(CDentry::STATE_PURGING);
+  dn->get(CDentry::PIN_PURGING);
+  in->state_set(CInode::STATE_PURGING);
+
+  if (dn->item_stray.is_on_list())
+    dn->item_stray.remove_myself();
+
+  if (in->is_dirty_parent())
+    in->clear_dirty_parent();
+
+  // CHEAT.  there's no real need to journal our intent to purge, since
+  // that is implicit in the dentry's presence and non-use in the stray
+  // dir.  on recovery, we'll need to re-eval all strays anyway.
+  
+  SnapContext nullsnapc;
+  C_GatherBuilder gather(g_ceph_context, new C_MDC_PurgeStrayPurged(this, dn));
+
+  if (in->is_dir()) {
+    object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+    list ls;
+    if (!in->dirfragtree.is_leaf(frag_t()))
+      in->dirfragtree.get_leaves(ls);
+    ls.push_back(frag_t());
+    for (list::iterator p = ls.begin();
+         p != ls.end();
+         ++p) {
+      object_t oid = CInode::get_object_name(in->inode.ino, *p, "");
+      dout(10) << "purge_stray remove dirfrag " << oid << dendl;
+      mds->objecter->remove(oid, oloc, nullsnapc, ceph_clock_now(g_ceph_context),
+                            0, NULL, gather.new_sub());
+    }
+    assert(gather.has_subs());
+    gather.activate();
+    return;
+  }
+
+  const SnapContext *snapc;
+  SnapRealm *realm = in->find_snaprealm();
+  if (realm) {
+    dout(10) << " realm " << *realm << dendl;
+    snapc = &realm->get_snap_context();
+  } else {
+    dout(10) << " NO realm, using null context" << dendl;
+    snapc = &nullsnapc;
+    assert(in->last == CEPH_NOSNAP);
+  }
+
+  if (in->is_file()) {
+    uint64_t period = (uint64_t)in->inode.layout.fl_object_size *
+		      (uint64_t)in->inode.layout.fl_stripe_count;
+    uint64_t to = in->inode.get_max_size();
+    to = MAX(in->inode.size, to);
+    // when truncating a file, the filer does not delete stripe objects that are
+    // truncated to zero. so we need to purge stripe objects up to the max size
+    // the file has ever been.
+    to = MAX(in->inode.max_size_ever, to);
+    if (to && period) {
+      uint64_t num = (to + period - 1) / period;
+      dout(10) << "purge_stray 0~" << to << " objects 0~" << num
+	       << " snapc " << snapc << " on " << *in << dendl;
+      mds->filer->purge_range(in->inode.ino, &in->inode.layout, *snapc,
+			      0, num, ceph_clock_now(g_ceph_context), 0,
+			      gather.new_sub());
+    }
+  }
+
+  inode_t *pi = in->get_projected_inode();
+  object_t oid = CInode::get_object_name(pi->ino, frag_t(), "");
+  // remove the backtrace object if it was not purged
+  if (!gather.has_subs()) {
+    object_locator_t oloc(pi->layout.fl_pg_pool);
+    dout(10) << "purge_stray remove backtrace object " << oid
+	     << " pool " << oloc.pool << " snapc " << snapc << dendl;
+    mds->objecter->remove(oid, oloc, *snapc, ceph_clock_now(g_ceph_context), 0,
+			  NULL, gather.new_sub());
+  }
+  // remove old backtrace objects
+  for (vector::iterator p = pi->old_pools.begin();
+       p != pi->old_pools.end();
+       ++p) {
+    object_locator_t oloc(*p);
+    dout(10) << "purge_stray remove backtrace object " << oid
+	     << " old pool " << *p << " snapc " << snapc << dendl;
+    mds->objecter->remove(oid, oloc, *snapc, ceph_clock_now(g_ceph_context), 0,
+			  NULL, gather.new_sub());
+  }
+  assert(gather.has_subs());
+  gather.activate();
+}
+
+class C_MDC_PurgeStrayLogged : public Context {
+  MDCache *cache;
+  CDentry *dn;
+  version_t pdv;
+  LogSegment *ls;
+public:
+  C_MDC_PurgeStrayLogged(MDCache *c, CDentry *d, version_t v, LogSegment *s) : 
+    cache(c), dn(d), pdv(v), ls(s) { }
+  void finish(int r) {
+    cache->_purge_stray_logged(dn, pdv, ls);
+  }
+};
+class C_MDC_PurgeStrayLoggedTruncate : public Context {
+  MDCache *cache;
+  CDentry *dn;
+  LogSegment *ls;
+public:
+  C_MDC_PurgeStrayLoggedTruncate(MDCache *c, CDentry *d, LogSegment *s) : 
+    cache(c), dn(d), ls(s) { }
+  void finish(int r) {
+    cache->_purge_stray_logged_truncate(dn, ls);
+  }
+};
+
+void MDCache::_purge_stray_purged(CDentry *dn, int r)
+{
+  assert (r == 0 || r == -ENOENT);
+  CInode *in = dn->get_projected_linkage()->get_inode();
+  dout(10) << "_purge_stray_purged " << *dn << " " << *in << dendl;
+
+  if (in->get_num_ref() == (int)in->is_dirty() &&
+      dn->get_num_ref() == (int)dn->is_dirty() + !!in->get_num_ref() + 1/*PIN_PURGING*/) {
+    // kill dentry.
+    version_t pdv = dn->pre_dirty();
+    dn->push_projected_linkage(); // NULL
+
+    EUpdate *le = new EUpdate(mds->mdlog, "purge_stray");
+    mds->mdlog->start_entry(le);
+
+    // update dirfrag fragstat, rstat
+    CDir *dir = dn->get_dir();
+    fnode_t *pf = dir->project_fnode();
+    pf->version = dir->pre_dirty();
+    if (in->is_dir())
+      pf->fragstat.nsubdirs--;
+    else
+      pf->fragstat.nfiles--;
+    pf->rstat.sub(in->inode.accounted_rstat);
+
+    le->metablob.add_dir_context(dn->dir);
+    EMetaBlob::dirlump& dl = le->metablob.add_dir(dn->dir, true);
+    le->metablob.add_null_dentry(dl, dn, true);
+    le->metablob.add_destroyed_inode(in->ino());
+
+    mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLogged(this, dn, pdv, mds->mdlog->get_current_segment()));
+  } else {
+    // new refs.. just truncate to 0
+    EUpdate *le = new EUpdate(mds->mdlog, "purge_stray truncate");
+    mds->mdlog->start_entry(le);
+    
+    inode_t *pi = in->project_inode();
+    pi->size = 0;
+    pi->client_ranges.clear();
+    pi->truncate_size = 0;
+    pi->truncate_from = 0;
+    pi->version = in->pre_dirty();
+
+    le->metablob.add_dir_context(dn->dir);
+    le->metablob.add_primary_dentry(dn, in, true);
+
+    mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLoggedTruncate(this, dn, mds->mdlog->get_current_segment()));
+  }
+}
+
+void MDCache::_purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls)
+{
+  CInode *in = dn->get_linkage()->get_inode();
+  dout(10) << "_purge_stray_logged " << *dn << " " << *in << dendl;
+
+  assert(!in->state_test(CInode::STATE_RECOVERING));
+
+  // unlink
+  assert(dn->get_projected_linkage()->is_null());
+  dn->dir->unlink_inode(dn);
+  dn->pop_projected_linkage();
+  dn->mark_dirty(pdv, ls);
+
+  dn->dir->pop_and_dirty_projected_fnode(ls);
+
+  in->state_clear(CInode::STATE_ORPHAN);
+  dn->state_clear(CDentry::STATE_PURGING);
+  dn->put(CDentry::PIN_PURGING);
+
+  // drop inode
+  if (in->is_dirty())
+    in->mark_clean();
+
+  remove_inode(in);
+
+  // drop dentry?
+  if (dn->is_new()) {
+    dout(20) << " dn is new, removing" << dendl;
+    dn->mark_clean();
+    dn->dir->remove_dentry(dn);
+  } else
+    touch_dentry_bottom(dn);  // drop dn as quickly as possible.
+}
+
+void MDCache::_purge_stray_logged_truncate(CDentry *dn, LogSegment *ls)
+{
+  CInode *in = dn->get_projected_linkage()->get_inode();
+  dout(10) << "_purge_stray_logged_truncate " << *dn << " " << *in << dendl;
+
+  dn->state_clear(CDentry::STATE_PURGING);
+  dn->put(CDentry::PIN_PURGING);
+
+  in->pop_and_dirty_projected_inode(ls);
+
+  eval_stray(dn);
+}
+
+
+
+void MDCache::reintegrate_stray(CDentry *straydn, CDentry *rdn)
+{
+  dout(10) << "reintegrate_stray " << *straydn << " into " << *rdn << dendl;
+  
+  // rename it to another mds.
+  filepath src;
+  straydn->make_path(src);
+  filepath dst;
+  rdn->make_path(dst);
+
+  MClientRequest *req = new MClientRequest(CEPH_MDS_OP_RENAME);
+  req->set_filepath(dst);
+  req->set_filepath2(src);
+  req->set_tid(mds->issue_tid());
+
+  mds->send_message_mds(req, rdn->authority().first);
+}
+ 
+
+void MDCache::migrate_stray(CDentry *dn, int to)
+{
+  CInode *in = dn->get_linkage()->get_inode();
+  assert(in);
+  CInode *diri = dn->dir->get_inode();
+  assert(diri->is_stray());
+  dout(10) << "migrate_stray from mds." << MDS_INO_STRAY_OWNER(diri->inode.ino)
+	   << " to mds." << to
+	   << " " << *dn << " " << *in << dendl;
+
+  // rename it to another mds.
+  filepath src;
+  dn->make_path(src);
+
+  string dname;
+  in->name_stray_dentry(dname);
+  filepath dst(dname, MDS_INO_STRAY(to, 0));
+
+  MClientRequest *req = new MClientRequest(CEPH_MDS_OP_RENAME);
+  req->set_filepath(dst);
+  req->set_filepath2(src);
+  req->set_tid(mds->issue_tid());
+
+  mds->send_message_mds(req, to);
+}
+
+
+
+
+// ========================================================================================
+// DISCOVER
+/*
+
+  - for all discovers (except base_inos, e.g. root, stray), waiters are attached
+  to the parent metadata object in the cache (pinning it).
+
+  - all discovers are tracked by tid, so that we can ignore potentially dup replies.
+
+*/
+
+void MDCache::_send_discover(discover_info_t& d)
+{
+  MDiscover *dis = new MDiscover(d.ino, d.frag, d.snap,
+				 d.want_path, d.want_ino, d.want_base_dir, d.want_xlocked);
+  dis->set_tid(d.tid);
+  mds->send_message_mds(dis, d.mds);
+}
+
+void MDCache::discover_base_ino(inodeno_t want_ino,
+				Context *onfinish,
+				int from) 
+{
+  dout(7) << "discover_base_ino " << want_ino << " from mds." << from << dendl;
+  if (waiting_for_base_ino[from].count(want_ino) == 0) {
+    discover_info_t& d = _create_discover(from);
+    d.ino = want_ino;
+    _send_discover(d);
+  }
+  waiting_for_base_ino[from][want_ino].push_back(onfinish);
+}
+
+
+void MDCache::discover_dir_frag(CInode *base,
+				frag_t approx_fg,
+				Context *onfinish,
+				int from)
+{
+  if (from < 0)
+    from = base->authority().first;
+
+  dirfrag_t df(base->ino(), approx_fg);
+  dout(7) << "discover_dir_frag " << df
+	  << " from mds." << from << dendl;
+
+  if (!base->is_waiting_for_dir(approx_fg) || !onfinish) {
+    discover_info_t& d = _create_discover(from);
+    d.ino = base->ino();
+    d.frag = approx_fg;
+    d.want_base_dir = true;
+    _send_discover(d);
+  }
+
+  if (onfinish) 
+    base->add_dir_waiter(approx_fg, onfinish);
+}
+
+struct C_MDC_RetryDiscoverPath : public Context {
+  MDCache *mdc;
+  CInode *base;
+  snapid_t snapid;
+  filepath path;
+  int from;
+  C_MDC_RetryDiscoverPath(MDCache *c, CInode *b, snapid_t s, filepath &p, int f) :
+    mdc(c), base(b), snapid(s), path(p), from(f)  {}
+  void finish(int r) {
+    mdc->discover_path(base, snapid, path, 0, from);
+  }
+};
+
+void MDCache::discover_path(CInode *base,
+			    snapid_t snap,
+			    filepath want_path,
+			    Context *onfinish,
+			    bool want_xlocked,
+			    int from)
+{
+  if (from < 0)
+    from = base->authority().first;
+
+  dout(7) << "discover_path " << base->ino() << " " << want_path << " snap " << snap << " from mds." << from
+	  << (want_xlocked ? " want_xlocked":"")
+	  << dendl;
+
+  if (base->is_ambiguous_auth()) {
+    dout(10) << " waiting for single auth on " << *base << dendl;
+    if (!onfinish)
+      onfinish = new C_MDC_RetryDiscoverPath(this, base, snap, want_path, from);
+    base->add_waiter(CInode::WAIT_SINGLEAUTH, onfinish);
+    return;
+  } else if (from == mds->get_nodeid()) {
+    list finished;
+    base->take_waiting(CInode::WAIT_DIR, finished);
+    mds->queue_waiters(finished);
+    return;
+  }
+
+  frag_t fg = base->pick_dirfrag(want_path[0]);
+  if ((want_xlocked && want_path.depth() == 1) ||
+      !base->is_waiting_for_dir(fg) || !onfinish) {
+    discover_info_t& d = _create_discover(from);
+    d.ino = base->ino();
+    d.frag = fg;
+    d.snap = snap;
+    d.want_path = want_path;
+    d.want_base_dir = true;
+    d.want_xlocked = want_xlocked;
+    _send_discover(d);
+  }
+
+  // register + wait
+  if (onfinish)
+    base->add_dir_waiter(fg, onfinish);
+}
+
+struct C_MDC_RetryDiscoverPath2 : public Context {
+  MDCache *mdc;
+  CDir *base;
+  snapid_t snapid;
+  filepath path;
+  C_MDC_RetryDiscoverPath2(MDCache *c, CDir *b, snapid_t s, filepath &p) :
+    mdc(c), base(b), snapid(s), path(p) {}
+  void finish(int r) {
+    mdc->discover_path(base, snapid, path, 0);
+  }
+};
+
+void MDCache::discover_path(CDir *base,
+			    snapid_t snap,
+			    filepath want_path,
+			    Context *onfinish,
+			    bool want_xlocked)
+{
+  int from = base->authority().first;
+
+  dout(7) << "discover_path " << base->dirfrag() << " " << want_path << " snap " << snap << " from mds." << from
+	  << (want_xlocked ? " want_xlocked":"")
+	  << dendl;
+
+  if (base->is_ambiguous_auth()) {
+    dout(7) << " waiting for single auth on " << *base << dendl;
+    if (!onfinish)
+      onfinish = new C_MDC_RetryDiscoverPath2(this, base, snap, want_path);
+    base->add_waiter(CDir::WAIT_SINGLEAUTH, onfinish);
+    return;
+  } else if (from == mds->get_nodeid()) {
+    list finished;
+    base->take_sub_waiting(finished);
+    mds->queue_waiters(finished);
+    return;
+  }
+
+  if ((want_xlocked && want_path.depth() == 1) ||
+      !base->is_waiting_for_dentry(want_path[0].c_str(), snap) || !onfinish) {
+    discover_info_t& d = _create_discover(from);
+    d.ino = base->ino();
+    d.frag = base->get_frag();
+    d.snap = snap;
+    d.want_path = want_path;
+    d.want_base_dir = false;
+    d.want_xlocked = want_xlocked;
+    _send_discover(d);
+  }
+
+  // register + wait
+  if (onfinish)
+    base->add_dentry_waiter(want_path[0], snap, onfinish);
+}
+
+struct C_MDC_RetryDiscoverIno : public Context {
+  MDCache *mdc;
+  CDir *base;
+  inodeno_t want_ino;
+  C_MDC_RetryDiscoverIno(MDCache *c, CDir *b, inodeno_t i) :
+    mdc(c), base(b), want_ino(i) {}
+  void finish(int r) {
+    mdc->discover_ino(base, want_ino, 0);
+  }
+};
+
+void MDCache::discover_ino(CDir *base,
+			   inodeno_t want_ino,
+			   Context *onfinish,
+			   bool want_xlocked)
+{
+  int from = base->authority().first;
+
+  dout(7) << "discover_ino " << base->dirfrag() << " " << want_ino << " from mds." << from
+	  << (want_xlocked ? " want_xlocked":"")
+	  << dendl;
+  
+  if (base->is_ambiguous_auth()) {
+    dout(10) << " waiting for single auth on " << *base << dendl;
+    if (!onfinish)
+      onfinish = new C_MDC_RetryDiscoverIno(this, base, want_ino);
+    base->add_waiter(CDir::WAIT_SINGLEAUTH, onfinish);
+    return;
+  } else if (from == mds->get_nodeid()) {
+    list finished;
+    base->take_sub_waiting(finished);
+    mds->queue_waiters(finished);
+    return;
+  }
+
+  if (want_xlocked || !base->is_waiting_for_ino(want_ino) || !onfinish) {
+    discover_info_t& d = _create_discover(from);
+    d.ino = base->ino();
+    d.frag = base->get_frag();
+    d.want_ino = want_ino;
+    d.want_base_dir = false;
+    d.want_xlocked = want_xlocked;
+    _send_discover(d);
+  }
+
+  // register + wait
+  if (onfinish)
+    base->add_ino_waiter(want_ino, onfinish);
+}
+
+
+
+void MDCache::kick_discovers(int who)
+{
+  for (map::iterator p = discovers.begin();
+       p != discovers.end();
+       ++p) {
+    if (p->second.mds != who)
+      continue;
+    _send_discover(p->second);
+  }
+}
+
+
+/* This function DOES put the passed message before returning */
+void MDCache::handle_discover(MDiscover *dis) 
+{
+  int whoami = mds->get_nodeid();
+  int from = dis->get_source_inst().name._num;
+
+  assert(from != whoami);
+
+  if (mds->get_state() <= MDSMap::STATE_REJOIN) {
+    int from = dis->get_source().num();
+    // proceed if requester is in the REJOIN stage, the request is from parallel_fetch().
+    // delay processing request from survivor because we may not yet choose lock states.
+    if (mds->get_state() < MDSMap::STATE_REJOIN ||
+	!mds->mdsmap->is_rejoin(from)) {
+      dout(0) << "discover_reply not yet active(|still rejoining), delaying" << dendl;
+      mds->wait_for_active(new C_MDS_RetryMessage(mds, dis));
+      return;
+    }
+  }
+
+
+  CInode *cur = 0;
+  MDiscoverReply *reply = new MDiscoverReply(dis);
+
+  snapid_t snapid = dis->get_snapid();
+
+  // get started.
+  if (MDS_INO_IS_BASE(dis->get_base_ino()) &&
+      !dis->wants_base_dir() && dis->get_want().depth() == 0) {
+    // wants root
+    dout(7) << "handle_discover from mds." << from
+	    << " wants base + " << dis->get_want().get_path()
+	    << " snap " << snapid
+	    << dendl;
+
+    cur = get_inode(dis->get_base_ino());
+    assert(cur);
+
+    // add root
+    reply->starts_with = MDiscoverReply::INODE;
+    replicate_inode(cur, from, reply->trace);
+    dout(10) << "added base " << *cur << dendl;
+  }
+  else {
+    // there's a base inode
+    cur = get_inode(dis->get_base_ino(), snapid);
+    if (!cur && snapid != CEPH_NOSNAP) {
+      cur = get_inode(dis->get_base_ino());
+      if (cur && !cur->is_multiversion())
+	cur = NULL;  // nope!
+    }
+    
+    if (!cur) {
+      dout(7) << "handle_discover mds." << from 
+	      << " don't have base ino " << dis->get_base_ino() << "." << snapid
+	      << dendl;
+      if (!dis->wants_base_dir() && dis->get_want().depth() > 0)
+	reply->set_error_dentry(dis->get_dentry(0));
+      reply->set_flag_error_dir();
+    } else if (dis->wants_base_dir()) {
+      dout(7) << "handle_discover mds." << from 
+	      << " wants basedir+" << dis->get_want().get_path() 
+	      << " has " << *cur 
+	      << dendl;
+    } else {
+      dout(7) << "handle_discover mds." << from 
+	      << " wants " << dis->get_want().get_path()
+	      << " has " << *cur
+	      << dendl;
+    }
+  }
+
+  assert(reply);
+  
+  // add content
+  // do some fidgeting to include a dir if they asked for the base dir, or just root.
+  for (unsigned i = 0; 
+       cur && (i < dis->get_want().depth() || dis->get_want().depth() == 0); 
+       i++) {
+
+    // -- figure out the dir
+
+    // is *cur even a dir at all?
+    if (!cur->is_dir()) {
+      dout(7) << *cur << " not a dir" << dendl;
+      reply->set_flag_error_dir();
+      break;
+    }
+
+    // pick frag
+    frag_t fg;
+    if (dis->get_want().depth()) {
+      // dentry specifies
+      fg = cur->pick_dirfrag(dis->get_dentry(i));
+    } else {
+      // requester explicity specified the frag
+      assert(dis->wants_base_dir() || dis->get_want_ino() || MDS_INO_IS_BASE(dis->get_base_ino()));
+      fg = dis->get_base_dir_frag();
+      if (!cur->dirfragtree.is_leaf(fg))
+	fg = cur->dirfragtree[fg.value()];
+    }
+    CDir *curdir = cur->get_dirfrag(fg);
+
+    if ((!curdir && !cur->is_auth()) ||
+	(curdir && !curdir->is_auth())) {
+
+	/* before:
+	 * ONLY set flag if empty!!
+	 * otherwise requester will wake up waiter(s) _and_ continue with discover,
+	 * resulting in duplicate discovers in flight,
+	 * which can wreak havoc when discovering rename srcdn (which may move)
+	 */
+
+      if (reply->is_empty()) {
+	// only hint if empty.
+	//  someday this could be better, but right now the waiter logic isn't smart enough.
+	
+	// hint
+	if (curdir) {
+	  dout(7) << " not dirfrag auth, setting dir_auth_hint for " << *curdir << dendl;
+	  reply->set_dir_auth_hint(curdir->authority().first);
+	} else {
+	  dout(7) << " dirfrag not open, not inode auth, setting dir_auth_hint for " 
+		  << *cur << dendl;
+	  reply->set_dir_auth_hint(cur->authority().first);
+	}
+	
+	// note error dentry, if any
+	//  NOTE: important, as it allows requester to issue an equivalent discover
+	//        to whomever we hint at.
+	if (dis->get_want().depth() > i)
+	  reply->set_error_dentry(dis->get_dentry(i));
+      }
+
+      break;
+    }
+
+    if (!curdir) { // open dir?
+      if (cur->is_frozen()) {
+	if (!reply->is_empty()) {
+	  dout(7) << *cur << " is frozen, non-empty reply, stopping" << dendl;
+	  break;
+	}
+	dout(7) << *cur << " is frozen, empty reply, waiting" << dendl;
+	cur->add_waiter(CInode::WAIT_UNFREEZE, new C_MDS_RetryMessage(mds, dis));
+	reply->put();
+	return;
+      }
+      curdir = cur->get_or_open_dirfrag(this, fg);
+    } else if (curdir->is_frozen_tree() ||
+	       (curdir->is_frozen_dir() && fragment_are_all_frozen(curdir))) {
+      if (dis->wants_base_dir() && dis->get_base_dir_frag() != curdir->get_frag()) {
+	dout(7) << *curdir << " is frozen, dirfrag mismatch, stopping" << dendl;
+	reply->set_flag_error_dir();
+	break;
+      }
+      if (!reply->is_empty()) {
+	dout(7) << *curdir << " is frozen, non-empty reply, stopping" << dendl;
+	break;
+      }
+      dout(7) << *curdir << " is frozen, empty reply, waiting" << dendl;
+      curdir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryMessage(mds, dis));
+      reply->put();
+      return;
+    }
+    
+    // add dir
+    if (curdir->get_version() == 0) {
+      // fetch newly opened dir
+    } else if (reply->is_empty() && !dis->wants_base_dir()) {
+      dout(7) << "handle_discover not adding unwanted base dir " << *curdir << dendl;
+      // make sure the base frag is correct, though, in there was a refragment since the
+      // original request was sent.
+      reply->set_base_dir_frag(curdir->get_frag());
+    } else {
+      assert(!curdir->is_ambiguous_auth()); // would be frozen.
+      if (!reply->trace.length())
+	reply->starts_with = MDiscoverReply::DIR;
+      replicate_dir(curdir, from, reply->trace);
+      dout(7) << "handle_discover added dir " << *curdir << dendl;
+    }
+
+    // lookup
+    CDentry *dn = 0;
+    if (curdir->get_version() == 0) {
+      // fetch newly opened dir
+    } else if  (dis->get_want_ino()) {
+      // lookup by ino
+      CInode *in = get_inode(dis->get_want_ino(), snapid);
+      if (in && in->is_auth() && in->get_parent_dn()->get_dir() == curdir) {
+	dn = in->get_parent_dn();
+	if (dn->state_test(CDentry::STATE_PURGING)) {
+	  // set error flag in reply
+	  dout(7) << "dentry " << *dn << " is purging, flagging error ino" << dendl;
+	  reply->set_flag_error_ino();
+	  break;
+	}
+      }
+    } else if (dis->get_want().depth() > 0) {
+      // lookup dentry
+      dn = curdir->lookup(dis->get_dentry(i), snapid);
+    } else 
+      break; // done!
+          
+    // incomplete dir?
+    if (!dn) {
+      if (!curdir->is_complete()) {
+	// readdir
+	dout(7) << "incomplete dir contents for " << *curdir << ", fetching" << dendl;
+	if (reply->is_empty()) {
+	  // fetch and wait
+	  curdir->fetch(new C_MDS_RetryMessage(mds, dis),
+			dis->wants_base_dir() && curdir->get_version() == 0);
+	  reply->put();
+	  return;
+	} else {
+	  // initiate fetch, but send what we have so far
+	  curdir->fetch(0);
+	  break;
+	}
+      }
+      
+      // don't have wanted ino in this dir?
+      if (dis->get_want_ino()) {
+	// set error flag in reply
+	dout(7) << "no ino " << dis->get_want_ino() << " in this dir, flagging error in "
+		<< *curdir << dendl;
+	reply->set_flag_error_ino();
+	break;
+      }
+
+      // send null dentry
+      dout(7) << "dentry " << dis->get_dentry(i) << " dne, returning null in "
+	      << *curdir << dendl;
+      dn = curdir->add_null_dentry(dis->get_dentry(i));
+    }
+    assert(dn);
+
+    CDentry::linkage_t *dnl = dn->get_linkage();
+
+    // xlocked dentry?
+    //  ...always block on non-tail items (they are unrelated)
+    //  ...allow xlocked tail disocvery _only_ if explicitly requested
+    bool tailitem = (dis->get_want().depth() == 0) || (i == dis->get_want().depth() - 1);
+    if (dn->lock.is_xlocked()) {
+      // is this the last (tail) item in the discover traversal?
+      if (tailitem && dis->wants_xlocked()) {
+	dout(7) << "handle_discover allowing discovery of xlocked tail " << *dn << dendl;
+      } else if (reply->is_empty()) {
+	dout(7) << "handle_discover blocking on xlocked " << *dn << dendl;
+	dn->lock.add_waiter(SimpleLock::WAIT_RD, new C_MDS_RetryMessage(mds, dis));
+	reply->put();
+	return;
+      } else {
+	dout(7) << "handle_discover non-empty reply, xlocked tail " << *dn << dendl;
+	break;
+      }
+    }
+
+    // frozen inode?
+    if (dnl->is_primary() && dnl->get_inode()->is_frozen()) {
+      if (tailitem && dis->wants_xlocked()) {
+	dout(7) << "handle_discover allowing discovery of frozen tail " << *dnl->get_inode() << dendl;
+      } else if (reply->is_empty()) {
+	dout(7) << *dnl->get_inode() << " is frozen, empty reply, waiting" << dendl;
+	dnl->get_inode()->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryMessage(mds, dis));
+	reply->put();
+	return;
+      } else {
+	dout(7) << *dnl->get_inode() << " is frozen, non-empty reply, stopping" << dendl;
+	break;
+      }
+    }
+
+    // add dentry
+    if (!reply->trace.length())
+      reply->starts_with = MDiscoverReply::DENTRY;
+    replicate_dentry(dn, from, reply->trace);
+    dout(7) << "handle_discover added dentry " << *dn << dendl;
+    
+    if (!dnl->is_primary()) break;  // stop on null or remote link.
+    
+    // add inode
+    CInode *next = dnl->get_inode();
+    assert(next->is_auth());
+    
+    replicate_inode(next, from, reply->trace);
+    dout(7) << "handle_discover added inode " << *next << dendl;
+    
+    // descend, keep going.
+    cur = next;
+    continue;
+  }
+
+  // how did we do?
+  assert(!reply->is_empty());
+  dout(7) << "handle_discover sending result back to asker mds." << from << dendl;
+  mds->send_message(reply, dis->get_connection());
+
+  dis->put();
+}
+
+/* This function DOES put the passed message before returning */
+void MDCache::handle_discover_reply(MDiscoverReply *m) 
+{
+  /*
+  if (mds->get_state() < MDSMap::STATE_ACTIVE) {
+    dout(0) << "discover_reply NOT ACTIVE YET" << dendl;
+    m->put();
+    return;
+  }
+  */
+  dout(7) << "discover_reply " << *m << dendl;
+  if (m->is_flag_error_dir()) 
+    dout(7) << " flag error, dir" << dendl;
+  if (m->is_flag_error_dn()) 
+    dout(7) << " flag error, dentry = " << m->get_error_dentry() << dendl;
+  if (m->is_flag_error_ino()) 
+    dout(7) << " flag error, ino = " << m->get_wanted_ino() << dendl;
+
+  list finished, error;
+  int from = m->get_source().num();
+
+  // starting point
+  CInode *cur = get_inode(m->get_base_ino());
+  bufferlist::iterator p = m->trace.begin();
+
+  int next = m->starts_with;
+
+  // decrement discover counters
+  if (m->get_tid()) {
+    map::iterator p = discovers.find(m->get_tid());
+    if (p != discovers.end()) {
+      dout(10) << " found tid " << m->get_tid() << dendl;
+      discovers.erase(p);
+    } else {
+      dout(10) << " tid " << m->get_tid() << " not found, must be dup reply" << dendl;
+    }
+  }
+
+  // discover ino error
+  if (p.end() && m->is_flag_error_ino()) {
+    assert(cur);
+    assert(cur->is_dir());
+    CDir *dir = cur->get_dirfrag(m->get_base_dir_frag());
+    if (dir) {
+      dout(7) << " flag_error on ino " << m->get_wanted_ino()
+	      << ", triggering ino" << dendl;
+      dir->take_ino_waiting(m->get_wanted_ino(), error);
+    } else
+      assert(0);
+  }
+
+  // discover may start with an inode
+  if (!p.end() && next == MDiscoverReply::INODE) {
+    cur = add_replica_inode(p, NULL, finished);
+    dout(7) << "discover_reply got base inode " << *cur << dendl;
+    assert(cur->is_base());
+    
+    next = MDiscoverReply::DIR;
+    
+    // take waiters?
+    if (cur->is_base() &&
+	waiting_for_base_ino[from].count(cur->ino())) {
+      finished.swap(waiting_for_base_ino[from][cur->ino()]);
+      waiting_for_base_ino[from].erase(cur->ino());
+    }
+  }
+  assert(cur);
+  
+  // loop over discover results.
+  // indexes follow each ([[dir] dentry] inode) 
+  // can start, end with any type.
+  while (!p.end()) {
+    // dir
+    frag_t fg;
+    CDir *curdir = 0;
+    if (next == MDiscoverReply::DIR) {
+      curdir = add_replica_dir(p, cur, m->get_source().num(), finished);
+      if (cur->ino() == m->get_base_ino() && curdir->get_frag() != m->get_base_dir_frag()) {
+	assert(m->get_wanted_base_dir());
+	cur->take_dir_waiting(m->get_base_dir_frag(), finished);
+      }
+    } else {
+      // note: this can only happen our first way around this loop.
+      if (p.end() && m->is_flag_error_dn()) {
+	fg = cur->pick_dirfrag(m->get_error_dentry());
+	curdir = cur->get_dirfrag(fg);
+      } else
+	curdir = cur->get_dirfrag(m->get_base_dir_frag());
+    }
+
+    if (p.end())
+      break;
+    
+    // dentry
+    CDentry *dn = add_replica_dentry(p, curdir, finished);
+    
+    if (p.end())
+      break;
+
+    // inode
+    cur = add_replica_inode(p, dn, finished);
+
+    next = MDiscoverReply::DIR;
+  }
+
+  // dir error?
+  // or dir_auth hint?
+  if (m->is_flag_error_dir() && !cur->is_dir()) {
+    // not a dir.
+    cur->take_waiting(CInode::WAIT_DIR, error);
+  } else if (m->is_flag_error_dir() || m->get_dir_auth_hint() != CDIR_AUTH_UNKNOWN) {
+    int who = m->get_dir_auth_hint();
+    if (who == mds->get_nodeid()) who = -1;
+    if (who >= 0)
+      dout(7) << " dir_auth_hint is " << m->get_dir_auth_hint() << dendl;
+
+    frag_t fg = m->get_base_dir_frag();
+    CDir *dir = cur->get_dirfrag(fg);
+
+    if (m->get_wanted_base_dir()) {
+      if (cur->is_waiting_for_dir(fg)) {
+	if (cur->is_auth())
+	  cur->take_waiting(CInode::WAIT_DIR, finished);
+	else if (dir || !cur->dirfragtree.is_leaf(fg))
+	  cur->take_dir_waiting(fg, finished);
+	else
+	  discover_dir_frag(cur, fg, 0, who);
+      } else
+	dout(7) << " doing nothing, nobody is waiting for dir" << dendl;
+    }
+
+    // try again?
+    if (m->get_error_dentry().length()) {
+      // wanted a dentry
+      if (dir && dir->is_waiting_for_dentry(m->get_error_dentry(), m->get_wanted_snapid())) {
+	if (dir->is_auth() || dir->lookup(m->get_error_dentry())) {
+	  dir->take_dentry_waiting(m->get_error_dentry(), m->get_wanted_snapid(),
+				   m->get_wanted_snapid(), finished);
+	} else {
+	  filepath relpath(m->get_error_dentry(), 0);
+	  discover_path(dir, m->get_wanted_snapid(), relpath, 0, m->get_wanted_xlocked());
+	}
+      } else
+	dout(7) << " doing nothing, have dir but nobody is waiting on dentry "
+		<< m->get_error_dentry() << dendl;
+    } else {
+      if (dir && m->get_wanted_ino() && dir->is_waiting_for_ino(m->get_wanted_ino())) {
+	if (dir->is_auth() || get_inode(m->get_wanted_ino()))
+	  dir->take_ino_waiting(m->get_wanted_ino(), finished);
+	else
+	  discover_ino(dir, m->get_wanted_ino(), 0, m->get_wanted_xlocked());
+      } else
+	dout(7) << " doing nothing, nobody is waiting for ino" << dendl;
+    }
+  }
+
+  // waiters
+  finish_contexts(g_ceph_context, error, -ENOENT);  // finish errors directly
+  mds->queue_waiters(finished);
+
+  // done
+  m->put();
+}
+
+
+
+// ----------------------------
+// REPLICAS
+
+CDir *MDCache::add_replica_dir(bufferlist::iterator& p, CInode *diri, int from,
+			       list& finished)
+{
+  dirfrag_t df;
+  ::decode(df, p);
+
+  assert(diri->ino() == df.ino);
+
+  // add it (_replica_)
+  CDir *dir = diri->get_dirfrag(df.frag);
+
+  if (dir) {
+    // had replica. update w/ new nonce.
+    dir->decode_replica(p);
+    dout(7) << "add_replica_dir had " << *dir << " nonce " << dir->replica_nonce << dendl;
+  } else {
+    // force frag to leaf in the diri tree
+    if (!diri->dirfragtree.is_leaf(df.frag)) {
+      dout(7) << "add_replica_dir forcing frag " << df.frag << " to leaf in the fragtree "
+	      << diri->dirfragtree << dendl;
+      diri->dirfragtree.force_to_leaf(g_ceph_context, df.frag);
+    }
+
+    // add replica.
+    dir = diri->add_dirfrag( new CDir(diri, df.frag, this, false) );
+    dir->decode_replica(p);
+
+    // is this a dir_auth delegation boundary?
+    if (from != diri->authority().first ||
+	diri->is_ambiguous_auth() ||
+	diri->is_base())
+      adjust_subtree_auth(dir, from);
+    
+    dout(7) << "add_replica_dir added " << *dir << " nonce " << dir->replica_nonce << dendl;
+    
+    // get waiters
+    diri->take_dir_waiting(df.frag, finished);
+  }
+
+  return dir;
+}
+
+CDir *MDCache::forge_replica_dir(CInode *diri, frag_t fg, int from)
+{
+  assert(mds->mdsmap->get_state(from) < MDSMap::STATE_REJOIN);
+  
+  // forge a replica.
+  CDir *dir = diri->add_dirfrag( new CDir(diri, fg, this, false) );
+  
+  // i'm assuming this is a subtree root. 
+  adjust_subtree_auth(dir, from);
+
+  dout(7) << "forge_replica_dir added " << *dir << " while mds." << from << " is down" << dendl;
+
+  return dir;
+}
+
+CDentry *MDCache::add_replica_dentry(bufferlist::iterator& p, CDir *dir, list& finished)
+{
+  string name;
+  snapid_t last;
+  ::decode(name, p);
+  ::decode(last, p);
+
+  CDentry *dn = dir->lookup(name, last);
+  
+  // have it?
+  if (dn) {
+    dn->decode_replica(p, false);
+    dout(7) << "add_replica_dentry had " << *dn << dendl;
+  } else {
+    dn = dir->add_null_dentry(name, 1 /* this will get updated below */, last);
+    dn->decode_replica(p, true);
+    dout(7) << "add_replica_dentry added " << *dn << dendl;
+  }
+
+  dir->take_dentry_waiting(name, dn->first, dn->last, finished);
+
+  return dn;
+}
+
+CInode *MDCache::add_replica_inode(bufferlist::iterator& p, CDentry *dn, list& finished)
+{
+  inodeno_t ino;
+  snapid_t last;
+  ::decode(ino, p);
+  ::decode(last, p);
+  CInode *in = get_inode(ino, last);
+  if (!in) {
+    in = new CInode(this, false, 1, last);
+    in->decode_replica(p, true);
+    add_inode(in);
+    if (in->ino() == MDS_INO_ROOT)
+      in->inode_auth.first = 0;
+    else if (in->is_mdsdir())
+      in->inode_auth.first = in->ino() - MDS_INO_MDSDIR_OFFSET;
+    dout(10) << "add_replica_inode added " << *in << dendl;
+    if (dn) {
+      assert(dn->get_linkage()->is_null());
+      dn->dir->link_primary_inode(dn, in);
+    }
+  } else {
+    in->decode_replica(p, false);
+    dout(10) << "add_replica_inode had " << *in << dendl;
+  }
+
+  if (dn) {
+    if (!dn->get_linkage()->is_primary() || dn->get_linkage()->get_inode() != in)
+      dout(10) << "add_replica_inode different linkage in dentry " << *dn << dendl;
+    
+    dn->get_dir()->take_ino_waiting(in->ino(), finished);
+  }
+  
+  return in;
+}
+
+ 
+void MDCache::replicate_stray(CDentry *straydn, int who, bufferlist& bl)
+{
+  replicate_inode(get_myin(), who, bl);
+  replicate_dir(straydn->get_dir()->inode->get_parent_dn()->get_dir(), who, bl);
+  replicate_dentry(straydn->get_dir()->inode->get_parent_dn(), who, bl);
+  replicate_inode(straydn->get_dir()->inode, who, bl);
+  replicate_dir(straydn->get_dir(), who, bl);
+  replicate_dentry(straydn, who, bl);
+}
+   
+CDentry *MDCache::add_replica_stray(bufferlist &bl, int from)
+{
+  list finished;
+  bufferlist::iterator p = bl.begin();
+
+  CInode *mdsin = add_replica_inode(p, NULL, finished);
+  CDir *mdsdir = add_replica_dir(p, mdsin, from, finished);
+  CDentry *straydirdn = add_replica_dentry(p, mdsdir, finished);
+  CInode *strayin = add_replica_inode(p, straydirdn, finished);
+  CDir *straydir = add_replica_dir(p, strayin, from, finished);
+  CDentry *straydn = add_replica_dentry(p, straydir, finished);
+  if (!finished.empty())
+    mds->queue_waiters(finished);
+
+  return straydn;
+}
+
+
+int MDCache::send_dir_updates(CDir *dir, bool bcast)
+{
+  // this is an FYI, re: replication
+
+  set who;
+  if (bcast) {
+    mds->get_mds_map()->get_active_mds_set(who);
+  } else {
+    for (map::iterator p = dir->replicas_begin();
+	 p != dir->replicas_end();
+	 ++p)
+      who.insert(p->first);
+  }
+  
+  dout(7) << "sending dir_update on " << *dir << " bcast " << bcast << " to " << who << dendl;
+
+  filepath path;
+  dir->inode->make_path(path);
+
+  int whoami = mds->get_nodeid();
+  for (set::iterator it = who.begin();
+       it != who.end();
+       ++it) {
+    if (*it == whoami) continue;
+    //if (*it == except) continue;
+    dout(7) << "sending dir_update on " << *dir << " to " << *it << dendl;
+
+    mds->send_message_mds(new MDirUpdate(mds->get_nodeid(),
+					 dir->dirfrag(),
+					 dir->dir_rep,
+					 dir->dir_rep_by,
+					 path,
+					 bcast),
+			  *it);
+  }
+
+  return 0;
+}
+
+/* This function DOES put the passed message before returning */
+void MDCache::handle_dir_update(MDirUpdate *m)
+{
+  CDir *dir = get_dirfrag(m->get_dirfrag());
+  if (!dir) {
+    dout(5) << "dir_update on " << m->get_dirfrag() << ", don't have it" << dendl;
+
+    // discover it?
+    if (m->should_discover()) {
+      // only try once! 
+      // this is key to avoid a fragtree update race, among other things.
+      m->tried_discover();        
+      vector trace;
+      CInode *in;
+      filepath path = m->get_path();
+      dout(5) << "trying discover on dir_update for " << path << dendl;
+      MDRequestRef null_ref;
+      int r = path_traverse(null_ref, m, NULL, path, &trace, &in, MDS_TRAVERSE_DISCOVER);
+      if (r > 0)
+        return;
+      assert(r == 0);
+      open_remote_dirfrag(in, m->get_dirfrag().frag, 
+			  new C_MDS_RetryMessage(mds, m));
+      return;
+    }
+
+    m->put();
+    return;
+  }
+
+  // update
+  dout(5) << "dir_update on " << *dir << dendl;
+  dir->dir_rep = m->get_dir_rep();
+  dir->dir_rep_by = m->get_dir_rep_by();
+  
+  // done
+  m->put();
+}
+
+
+
+
+
+// LINK
+
+void MDCache::send_dentry_link(CDentry *dn)
+{
+  dout(7) << "send_dentry_link " << *dn << dendl;
+
+  CDir *subtree = get_subtree_root(dn->get_dir());
+  for (map::iterator p = dn->replicas_begin();
+       p != dn->replicas_end(); 
+       ++p) {
+    if (mds->mdsmap->get_state(p->first) < MDSMap::STATE_REJOIN ||
+	(mds->mdsmap->get_state(p->first) == MDSMap::STATE_REJOIN &&
+	 rejoin_gather.count(p->first)))
+      continue;
+    CDentry::linkage_t *dnl = dn->get_linkage();
+    MDentryLink *m = new MDentryLink(subtree->dirfrag(), dn->get_dir()->dirfrag(),
+				     dn->name, dnl->is_primary());
+    if (dnl->is_primary()) {
+      dout(10) << "  primary " << *dnl->get_inode() << dendl;
+      replicate_inode(dnl->get_inode(), p->first, m->bl);
+    } else if (dnl->is_remote()) {
+      inodeno_t ino = dnl->get_remote_ino();
+      __u8 d_type = dnl->get_remote_d_type();
+      dout(10) << "  remote " << ino << " " << d_type << dendl;
+      ::encode(ino, m->bl);
+      ::encode(d_type, m->bl);
+    } else
+      assert(0);   // aie, bad caller!
+    mds->send_message_mds(m, p->first);
+  }
+}
+
+/* This function DOES put the passed message before returning */
+void MDCache::handle_dentry_link(MDentryLink *m)
+{
+
+  CDentry *dn = NULL;
+  CDir *dir = get_dirfrag(m->get_dirfrag());
+  if (!dir) {
+    dout(7) << "handle_dentry_link don't have dirfrag " << m->get_dirfrag() << dendl;
+  } else {
+    dn = dir->lookup(m->get_dn());
+    if (!dn) {
+      dout(7) << "handle_dentry_link don't have dentry " << *dir << " dn " << m->get_dn() << dendl;
+    } else {
+      dout(7) << "handle_dentry_link on " << *dn << dendl;
+      CDentry::linkage_t *dnl = dn->get_linkage();
+
+      assert(!dn->is_auth());
+      assert(dnl->is_null());
+    }
+  }
+
+  bufferlist::iterator p = m->bl.begin();
+  list finished;
+  if (dn) {
+    if (m->get_is_primary()) {
+      // primary link.
+      add_replica_inode(p, dn, finished);
+    } else {
+      // remote link, easy enough.
+      inodeno_t ino;
+      __u8 d_type;
+      ::decode(ino, p);
+      ::decode(d_type, p);
+      dir->link_remote_inode(dn, ino, d_type);
+    }
+  } else {
+    assert(0);
+  }
+
+  if (!finished.empty())
+    mds->queue_waiters(finished);
+
+  m->put();
+  return;
+}
+
+
+// UNLINK
+
+void MDCache::send_dentry_unlink(CDentry *dn, CDentry *straydn, MDRequestRef& mdr)
+{
+  dout(10) << "send_dentry_unlink " << *dn << dendl;
+  // share unlink news with replicas
+  for (map::iterator it = dn->replicas_begin();
+       it != dn->replicas_end();
+       ++it) {
+    // don't tell (rmdir) witnesses; they already know
+    if (mdr && mdr->more()->witnessed.count(it->first))
+      continue;
+
+    if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN ||
+	(mds->mdsmap->get_state(it->first) == MDSMap::STATE_REJOIN &&
+	 rejoin_gather.count(it->first)))
+      continue;
+
+    MDentryUnlink *unlink = new MDentryUnlink(dn->get_dir()->dirfrag(), dn->name);
+    if (straydn)
+      replicate_stray(straydn, it->first, unlink->straybl);
+    mds->send_message_mds(unlink, it->first);
+  }
+}
+
+/* This function DOES put the passed message before returning */
+void MDCache::handle_dentry_unlink(MDentryUnlink *m)
+{
+  // straydn
+  CDentry *straydn = NULL;
+  if (m->straybl.length())
+    straydn = add_replica_stray(m->straybl, m->get_source().num());
+
+  CDir *dir = get_dirfrag(m->get_dirfrag());
+  if (!dir) {
+    dout(7) << "handle_dentry_unlink don't have dirfrag " << m->get_dirfrag() << dendl;
+  } else {
+    CDentry *dn = dir->lookup(m->get_dn());
+    if (!dn) {
+      dout(7) << "handle_dentry_unlink don't have dentry " << *dir << " dn " << m->get_dn() << dendl;
+    } else {
+      dout(7) << "handle_dentry_unlink on " << *dn << dendl;
+      CDentry::linkage_t *dnl = dn->get_linkage();
+
+      // open inode?
+      if (dnl->is_primary()) {
+	CInode *in = dnl->get_inode();
+	dn->dir->unlink_inode(dn);
+	assert(straydn);
+	straydn->dir->link_primary_inode(straydn, in);
+
+	// in->first is lazily updated on replica; drag it forward so
+	// that we always keep it in sync with the dnq
+	assert(straydn->first >= in->first);
+	in->first = straydn->first;
+
+	// update subtree map?
+	if (in->is_dir()) 
+	  adjust_subtree_after_rename(in, dir, false);
+
+	// send caps to auth (if we're not already)
+	if (in->is_any_caps() &&
+	    !in->state_test(CInode::STATE_EXPORTINGCAPS))
+	  migrator->export_caps(in);
+	
+	touch_dentry_bottom(straydn); // move stray to end of lru
+	straydn = NULL;
+      } else {
+	assert(!straydn);
+	assert(dnl->is_remote());
+	dn->dir->unlink_inode(dn);
+      }
+      assert(dnl->is_null());
+      
+      // move to bottom of lru
+      touch_dentry_bottom(dn);
+    }
+  }
+
+  // race with trim_dentry()
+  if (straydn) {
+    assert(straydn->get_num_ref() == 0);
+    assert(straydn->get_linkage()->is_null());
+    map expiremap;
+    trim_dentry(straydn, expiremap);
+    send_expire_messages(expiremap);
+  }
+
+  m->put();
+  return;
+}
+
+
+
+
+
+
+// ===================================================================
+
+
+
+// ===================================================================
+// FRAGMENT
+
+
+/** 
+ * adjust_dir_fragments -- adjust fragmentation for a directory
+ *
+ * @param diri directory inode
+ * @param basefrag base fragment
+ * @param bits bit adjustment.  positive for split, negative for merge.
+ */
+void MDCache::adjust_dir_fragments(CInode *diri, frag_t basefrag, int bits,
+				   list& resultfrags, 
+				   list& waiters,
+				   bool replay)
+{
+  dout(10) << "adjust_dir_fragments " << basefrag << " " << bits 
+	   << " on " << *diri << dendl;
+
+  list srcfrags;
+  diri->get_dirfrags_under(basefrag, srcfrags);
+
+  adjust_dir_fragments(diri, srcfrags, basefrag, bits, resultfrags, waiters, replay);
+}
+
+CDir *MDCache::force_dir_fragment(CInode *diri, frag_t fg, bool replay)
+{
+  CDir *dir = diri->get_dirfrag(fg);
+  if (dir)
+    return dir;
+
+  dout(10) << "force_dir_fragment " << fg << " on " << *diri << dendl;
+
+  list src, result;
+  list waiters;
+
+  // split a parent?
+  frag_t parent = diri->dirfragtree.get_branch_or_leaf(fg);
+  while (1) {
+    CDir *pdir = diri->get_dirfrag(parent);
+    if (pdir) {
+      int split = fg.bits() - parent.bits();
+      dout(10) << " splitting parent by " << split << " " << *pdir << dendl;
+      src.push_back(pdir);
+      adjust_dir_fragments(diri, src, parent, split, result, waiters, replay);
+      dir = diri->get_dirfrag(fg);
+      if (dir)
+        dout(10) << "force_dir_fragment result " << *dir << dendl;
+      return dir;
+    }
+    if (parent == frag_t())
+      break;
+    frag_t last = parent;
+    parent = parent.parent();
+    dout(10) << " " << last << " parent is " << parent << dendl;
+  }
+
+  // hoover up things under fg?
+  diri->get_dirfrags_under(fg, src);
+  if (src.empty()) {
+    dout(10) << "force_dir_fragment no frags under " << fg << dendl;
+    return NULL;
+  }
+  dout(10) << " will combine frags under " << fg << ": " << src << dendl;
+  adjust_dir_fragments(diri, src, fg, 0, result, waiters, replay);
+  dir = result.front();
+  dout(10) << "force_dir_fragment result " << *dir << dendl;
+  return dir;
+}
+
+void MDCache::adjust_dir_fragments(CInode *diri,
+				   list& srcfrags,
+				   frag_t basefrag, int bits,
+				   list& resultfrags, 
+				   list& waiters,
+				   bool replay)
+{
+  dout(10) << "adjust_dir_fragments " << basefrag << " bits " << bits
+	   << " srcfrags " << srcfrags
+	   << " on " << *diri << dendl;
+
+  // adjust fragtree
+  // yuck.  we may have discovered the inode while it was being fragmented.
+  if (!diri->dirfragtree.is_leaf(basefrag))
+    diri->dirfragtree.force_to_leaf(g_ceph_context, basefrag);
+
+  if (bits > 0)
+    diri->dirfragtree.split(basefrag, bits);
+  dout(10) << " new fragtree is " << diri->dirfragtree << dendl;
+
+  if (srcfrags.empty())
+    return;
+
+  // split
+  CDir *parent_dir = diri->get_parent_dir();
+  CDir *parent_subtree = 0;
+  if (parent_dir)
+    parent_subtree = get_subtree_root(parent_dir);
+
+  if (bits > 0) {
+    // SPLIT
+    assert(srcfrags.size() == 1);
+    CDir *dir = srcfrags.front();
+
+    dir->split(bits, resultfrags, waiters, replay);
+
+    // did i change the subtree map?
+    if (dir->is_subtree_root()) {
+      // new frags are now separate subtrees
+      for (list::iterator p = resultfrags.begin();
+	   p != resultfrags.end();
+	   ++p)
+	subtrees[*p].clear();   // new frag is now its own subtree
+      
+      // was i a bound?
+      if (parent_subtree) {
+	assert(subtrees[parent_subtree].count(dir));
+	subtrees[parent_subtree].erase(dir);
+	for (list::iterator p = resultfrags.begin();
+	     p != resultfrags.end();
+	     ++p)
+	  subtrees[parent_subtree].insert(*p);
+      }
+      
+      // adjust my bounds.
+      set bounds;
+      bounds.swap(subtrees[dir]);
+      subtrees.erase(dir);
+      for (set::iterator p = bounds.begin();
+	   p != bounds.end();
+	   ++p) {
+	CDir *frag = get_subtree_root((*p)->get_parent_dir());
+	subtrees[frag].insert(*p);
+      }
+
+      show_subtrees(10);
+
+      // dir has no PIN_SUBTREE; CDir::purge_stolen() drops it.
+      dir->dir_auth = CDIR_AUTH_DEFAULT;
+    }
+    
+    diri->close_dirfrag(dir->get_frag());
+    
+  } else {
+    // MERGE
+
+    // are my constituent bits subtrees?  if so, i will be too.
+    // (it's all or none, actually.)
+    bool was_subtree = false;
+    set new_bounds;
+    for (list::iterator p = srcfrags.begin(); p != srcfrags.end(); ++p) {
+      CDir *dir = *p;
+      if (dir->is_subtree_root()) {
+	dout(10) << " taking srcfrag subtree bounds from " << *dir << dendl;
+	was_subtree = true;
+	map >::iterator q = subtrees.find(dir);
+	set::iterator r = q->second.begin();
+	while (r != subtrees[dir].end()) {
+	  new_bounds.insert(*r);
+	  subtrees[dir].erase(r++);
+	}
+	subtrees.erase(q);
+	
+	// remove myself as my parent's bound
+	if (parent_subtree)
+	  subtrees[parent_subtree].erase(dir);
+      }
+    }
+    
+    // merge
+    CDir *f = new CDir(diri, basefrag, this, srcfrags.front()->is_auth());
+    f->merge(srcfrags, waiters, replay);
+    diri->add_dirfrag(f);
+
+    if (was_subtree) {
+      subtrees[f].swap(new_bounds);
+      if (parent_subtree)
+	subtrees[parent_subtree].insert(f);
+      
+      show_subtrees(10);
+    }
+
+    resultfrags.push_back(f);
+  }
+}
+
+
+class C_MDC_FragmentFrozen : public Context {
+  MDCache *mdcache;
+  dirfrag_t basedirfrag;
+public:
+  C_MDC_FragmentFrozen(MDCache *m, dirfrag_t df) :
+    mdcache(m), basedirfrag(df) {}
+  virtual void finish(int r) {
+    mdcache->fragment_frozen(basedirfrag, r);
+  }
+};
+
+bool MDCache::can_fragment(CInode *diri, list& dirs)
+{
+  if (mds->mdsmap->is_degraded()) {
+    dout(7) << "can_fragment: cluster degraded, no fragmenting for now" << dendl;
+    return false;
+  }
+  if (diri->get_parent_dir() &&
+      diri->get_parent_dir()->get_inode()->is_stray()) {
+    dout(7) << "can_fragment: i won't merge|split anything in stray" << dendl;
+    return false;
+  }
+  if (diri->is_mdsdir() || diri->is_stray() || diri->ino() == MDS_INO_CEPH) {
+    dout(7) << "can_fragment: i won't fragment the mdsdir or straydir or .ceph" << dendl;
+    return false;
+  }
+
+  for (list::iterator p = dirs.begin(); p != dirs.end(); ++p) {
+    CDir *dir = *p;
+    if (dir->state_test(CDir::STATE_FRAGMENTING)) {
+      dout(7) << "can_fragment: already fragmenting " << *dir << dendl;
+      return false;
+    }
+    if (!dir->is_auth()) {
+      dout(7) << "can_fragment: not auth on " << *dir << dendl;
+      return false;
+    }
+    if (dir->is_frozen() ||
+	dir->is_freezing()) {
+      dout(7) << "can_fragment: can't merge, freezing|frozen.  wait for other exports to finish first." << dendl;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+void MDCache::split_dir(CDir *dir, int bits)
+{
+  dout(7) << "split_dir " << *dir << " bits " << bits << dendl;
+  assert(dir->is_auth());
+  CInode *diri = dir->inode;
+
+  list dirs;
+  dirs.push_back(dir);
+
+  if (!can_fragment(diri, dirs))
+    return;
+
+  assert(fragments.count(dir->dirfrag()) == 0);
+  fragment_info_t& info = fragments[dir->dirfrag()];
+  info.dirs.push_back(dir);
+  info.bits = bits;
+  info.last_cum_auth_pins_change = ceph_clock_now(g_ceph_context);
+
+  C_GatherBuilder gather(g_ceph_context, new C_MDC_FragmentFrozen(this, dir->dirfrag()));
+  fragment_freeze_dirs(dirs, gather);
+  gather.activate();
+
+  // initial mark+complete pass
+  fragment_mark_and_complete(dirs);
+}
+
+void MDCache::merge_dir(CInode *diri, frag_t frag)
+{
+  dout(7) << "merge_dir to " << frag << " on " << *diri << dendl;
+
+  list dirs;
+  if (!diri->get_dirfrags_under(frag, dirs)) {
+    dout(7) << "don't have all frags under " << frag << " for " << *diri << dendl;
+    return;
+  }
+
+  if (diri->dirfragtree.is_leaf(frag)) {
+    dout(10) << " " << frag << " already a leaf for " << *diri << dendl;
+    return;
+  }
+
+  if (!can_fragment(diri, dirs))
+    return;
+
+  CDir *first = dirs.front();
+  int bits = first->get_frag().bits() - frag.bits();
+  dout(10) << " we are merginb by " << bits << " bits" << dendl;
+
+  dirfrag_t df(diri->ino(), frag);
+  assert(fragments.count(df) == 0);
+  fragment_info_t& info = fragments[df];
+  info.dirs = dirs;
+  info.bits = -bits;
+  info.last_cum_auth_pins_change = ceph_clock_now(g_ceph_context);
+
+  C_GatherBuilder gather(g_ceph_context,
+			 new C_MDC_FragmentFrozen(this, dirfrag_t(diri->ino(), frag)));
+  fragment_freeze_dirs(dirs, gather);
+  gather.activate();
+
+  // initial mark+complete pass
+  fragment_mark_and_complete(dirs);
+}
+
+void MDCache::fragment_freeze_dirs(list& dirs, C_GatherBuilder &gather)
+{
+  for (list::iterator p = dirs.begin(); p != dirs.end(); ++p) {
+    CDir *dir = *p;
+    dir->auth_pin(dir);  // until we mark and complete them
+    dir->state_set(CDir::STATE_FRAGMENTING);
+    dir->freeze_dir();
+    assert(dir->is_freezing_dir());
+    dir->add_waiter(CDir::WAIT_FROZEN, gather.new_sub());
+  }
+}
+
+class C_MDC_FragmentMarking : public Context {
+  MDCache *mdcache;
+  list dirs;
+public:
+  C_MDC_FragmentMarking(MDCache *m, list& d) : mdcache(m), dirs(d) {}
+  virtual void finish(int r) {
+    mdcache->fragment_mark_and_complete(dirs);
+  }
+};
+
+void MDCache::fragment_mark_and_complete(list& dirs)
+{
+  CInode *diri = dirs.front()->get_inode();
+  dout(10) << "fragment_mark_and_complete " << dirs << " on " << *diri << dendl;
+
+  C_GatherBuilder gather(g_ceph_context);
+  
+  for (list::iterator p = dirs.begin();
+       p != dirs.end();
+       ++p) {
+    CDir *dir = *p;
+
+    bool ready = true;
+    if (!dir->is_complete()) {
+      dout(15) << " fetching incomplete " << *dir << dendl;
+      dir->fetch(gather.new_sub(), true);  // ignore authpinnability
+      ready = false;
+    }
+    if (dir->get_frag() == frag_t() && dir->is_new()) {
+      // The COMPLETE flag gets lost if we fragment a new dirfrag, then rollback
+      // the operation. To avoid CDir::fetch() complaining about missing object,
+      // we commit new dirfrag first.
+      dout(15) << " committing new " << *dir << dendl;
+      assert(dir->is_dirty());
+      dir->commit(0, gather.new_sub(), true);
+      ready = false;
+    }
+    if (!ready)
+      continue;
+
+    if (!dir->state_test(CDir::STATE_DNPINNEDFRAG)) {
+      dout(15) << " marking " << *dir << dendl;
+      for (CDir::map_t::iterator p = dir->items.begin();
+	   p != dir->items.end();
+	   ++p) {
+	CDentry *dn = p->second;
+	dn->get(CDentry::PIN_FRAGMENTING);
+	assert(!dn->state_test(CDentry::STATE_FRAGMENTING));
+	dn->state_set(CDentry::STATE_FRAGMENTING);
+      }
+      dir->state_set(CDir::STATE_DNPINNEDFRAG);
+      dir->auth_unpin(dir);
+    } else {
+      dout(15) << " already marked " << *dir << dendl;
+    }
+  }
+  if (gather.has_subs()) {
+    gather.set_finisher(new C_MDC_FragmentMarking(this, dirs));
+    gather.activate();
+  }
+
+  // flush log so that request auth_pins are retired
+  mds->mdlog->flush();
+}
+
+void MDCache::fragment_unmark_unfreeze_dirs(list& dirs)
+{
+  dout(10) << "fragment_unmark_unfreeze_dirs " << dirs << dendl;
+  for (list::iterator p = dirs.begin(); p != dirs.end(); ++p) {
+    CDir *dir = *p;
+    dout(10) << " frag " << *dir << dendl;
+
+    assert(dir->state_test(CDir::STATE_DNPINNEDFRAG));
+    dir->state_clear(CDir::STATE_DNPINNEDFRAG);
+
+    assert(dir->state_test(CDir::STATE_FRAGMENTING));
+    dir->state_clear(CDir::STATE_FRAGMENTING);
+
+    for (CDir::map_t::iterator p = dir->items.begin();
+	 p != dir->items.end();
+	 ++p) {
+      CDentry *dn = p->second;
+      assert(dn->state_test(CDentry::STATE_FRAGMENTING));
+      dn->state_clear(CDentry::STATE_FRAGMENTING);
+      dn->put(CDentry::PIN_FRAGMENTING);
+    }
+
+    dir->unfreeze_dir();
+  }
+}
+
+bool MDCache::fragment_are_all_frozen(CDir *dir)
+{
+  assert(dir->is_frozen_dir());
+  map::iterator p;
+  for (p = fragments.lower_bound(dirfrag_t(dir->ino(), 0));
+       p != fragments.end() && p->first.ino == dir->ino();
+       ++p) {
+    if (p->first.frag.contains(dir->get_frag()))
+      return p->second.has_frozen;
+  }
+  assert(0);
+  return false;
+}
+
+void MDCache::fragment_freeze_inc_num_waiters(CDir *dir)
+{
+  map::iterator p;
+  for (p = fragments.lower_bound(dirfrag_t(dir->ino(), 0));
+       p != fragments.end() && p->first.ino == dir->ino();
+       ++p) {
+    if (p->first.frag.contains(dir->get_frag())) {
+      p->second.num_remote_waiters++;
+      return;
+    }
+  }
+  assert(0);
+}
+
+void MDCache::find_stale_fragment_freeze()
+{
+  dout(10) << "find_stale_fragment_freeze" << dendl;
+  // see comment in Migrator::find_stale_export_freeze()
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t cutoff = now;
+  cutoff -= g_conf->mds_freeze_tree_timeout;
+
+  for (map::iterator p = fragments.begin();
+       p != fragments.end(); ) {
+    dirfrag_t df = p->first;
+    fragment_info_t& info = p->second;
+    ++p;
+    if (info.has_frozen)
+      continue;
+    CDir *dir;
+    int total_auth_pins = 0;
+    for (list::iterator q = info.dirs.begin();
+	 q != info.dirs.end();
+	 ++q) {
+      dir = *q;
+      if (!dir->state_test(CDir::STATE_DNPINNEDFRAG)) {
+	total_auth_pins = -1;
+	break;
+      }
+      if (dir->is_frozen_dir())
+	continue;
+      total_auth_pins += dir->get_auth_pins() + dir->get_dir_auth_pins();
+    }
+    if (total_auth_pins < 0)
+      continue;
+    if (info.last_cum_auth_pins != total_auth_pins) {
+      info.last_cum_auth_pins = total_auth_pins;
+      info.last_cum_auth_pins_change = now;
+      continue;
+    }
+    if (info.last_cum_auth_pins_change >= cutoff)
+      continue;
+    dir = info.dirs.front();
+    if (info.num_remote_waiters > 0 ||
+	(!dir->inode->is_root() && dir->get_parent_dir()->is_freezing())) {
+      dout(10) << " cancel fragmenting " << df << " bit " << info.bits << dendl;
+      list dirs;
+      dirs.swap(info.dirs);
+      fragments.erase(df);
+      fragment_unmark_unfreeze_dirs(dirs);
+    }
+  }
+}
+
+class C_MDC_FragmentPrep : public Context {
+  MDCache *mdcache;
+  MDRequestRef mdr;
+public:
+  C_MDC_FragmentPrep(MDCache *m, MDRequestRef& r) : mdcache(m), mdr(r) {}
+  virtual void finish(int r) {
+    mdcache->_fragment_logged(mdr);
+  }
+};
+
+class C_MDC_FragmentStore : public Context {
+  MDCache *mdcache;
+  MDRequestRef mdr;
+public:
+  C_MDC_FragmentStore(MDCache *m, MDRequestRef& r) : mdcache(m), mdr(r) {}
+  virtual void finish(int r) {
+    mdcache->_fragment_stored(mdr);
+  }
+};
+
+class C_MDC_FragmentCommit : public Context {
+  MDCache *mdcache;
+  dirfrag_t basedirfrag;
+  list resultfrags;
+public:
+  C_MDC_FragmentCommit(MDCache *m, dirfrag_t df, list& l) :
+    mdcache(m), basedirfrag(df), resultfrags(l) {}
+  virtual void finish(int r) {
+    mdcache->_fragment_committed(basedirfrag, resultfrags);
+  }
+};
+
+class C_MDC_FragmentFinish : public Context {
+  MDCache *mdcache;
+  dirfrag_t basedirfrag;
+  list resultfrags;
+public:
+  C_MDC_FragmentFinish(MDCache *m, dirfrag_t f, list& l) :
+    mdcache(m), basedirfrag(f) {
+    resultfrags.swap(l);
+  }
+  virtual void finish(int r) {
+    assert(r == 0 || r == -ENOENT);
+    mdcache->_fragment_finish(basedirfrag, resultfrags);
+  }
+};
+
+void MDCache::fragment_frozen(dirfrag_t basedirfrag, int r)
+{
+  map::iterator it = fragments.find(basedirfrag);
+  if (r < 0) {
+    dout(7) << "fragment_frozen " << basedirfrag << " must have aborted" << dendl;
+    assert(it == fragments.end());
+    return;
+  }
+  assert(it != fragments.end());
+  fragment_info_t& info = it->second;
+
+  dout(10) << "fragment_frozen " << basedirfrag.frag << " by " << info.bits
+	   << " on " << info.dirs.front()->get_inode() << dendl;
+
+  info.has_frozen = true;
+
+  MDRequestRef mdr = request_start_internal(CEPH_MDS_OP_FRAGMENTDIR);
+  mdr->more()->fragment_base = basedirfrag;
+  dispatch_fragment_dir(mdr);
+}
+
+void MDCache::dispatch_fragment_dir(MDRequestRef& mdr)
+{
+  dirfrag_t basedirfrag = mdr->more()->fragment_base;
+  map::iterator it = fragments.find(basedirfrag);
+  assert(it != fragments.end());
+  fragment_info_t& info = it->second;
+  CInode *diri = info.dirs.front()->get_inode();
+
+  dout(10) << "dispatch_fragment_dir " << basedirfrag << " bits " << info.bits
+	   << " on " << *diri << dendl;
+  if (!mdr->aborted) {
+    set rdlocks, wrlocks, xlocks;
+    wrlocks.insert(&diri->dirfragtreelock);
+    // prevent a racing gather on any other scatterlocks too
+    wrlocks.insert(&diri->nestlock);
+    wrlocks.insert(&diri->filelock);
+    if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks, NULL, NULL, true))
+      if (!mdr->aborted)
+	return;
+  }
+
+  if (mdr->aborted) {
+    dout(10) << " can't auth_pin " << *diri << ", requeuing dir "
+	     << info.dirs.front()->dirfrag() << dendl;
+    if (info.bits > 0)
+      mds->balancer->queue_split(info.dirs.front());
+    else
+      mds->balancer->queue_merge(info.dirs.front());
+    fragment_unmark_unfreeze_dirs(info.dirs);
+    fragments.erase(it);
+    request_finish(mdr);
+    return;
+  }
+
+  mdr->ls = mds->mdlog->get_current_segment();
+  EFragment *le = new EFragment(mds->mdlog, EFragment::OP_PREPARE, basedirfrag, info.bits);
+  mds->mdlog->start_entry(le);
+
+  for (list::iterator p = info.dirs.begin(); p != info.dirs.end(); ++p) {
+    CDir *dir = *p;
+    dirfrag_rollback rollback;
+    rollback.fnode = dir->fnode;
+    le->add_orig_frag(dir->get_frag(), &rollback);
+  }
+
+  // refragment
+  list waiters;
+  adjust_dir_fragments(diri, info.dirs, basedirfrag.frag, info.bits,
+		       info.resultfrags, waiters, false);
+  if (g_conf->mds_debug_frag)
+    diri->verify_dirfrags();
+  mds->queue_waiters(waiters);
+
+  for (list::iterator p = le->orig_frags.begin(); p != le->orig_frags.end(); ++p)
+    assert(!diri->dirfragtree.is_leaf(*p));
+
+  le->metablob.add_dir_context(*info.resultfrags.begin());
+  for (list::iterator p = info.resultfrags.begin();
+       p != info.resultfrags.end();
+       ++p) {
+    if (diri->is_auth()) {
+      le->metablob.add_fragmented_dir(*p, false, false);
+    } else {
+      (*p)->state_set(CDir::STATE_DIRTYDFT);
+      le->metablob.add_fragmented_dir(*p, false, true);
+    }
+  }
+
+  // dft lock
+  if (diri->is_auth()) {
+    // journal dirfragtree
+    inode_t *pi = diri->project_inode();
+    pi->version = diri->pre_dirty();
+    journal_dirty_inode(mdr.get(), &le->metablob, diri);
+  } else {
+    mds->locker->mark_updated_scatterlock(&diri->dirfragtreelock);
+    mdr->ls->dirty_dirfrag_dirfragtree.push_back(&diri->item_dirty_dirfrag_dirfragtree);
+    mdr->add_updated_lock(&diri->dirfragtreelock);
+  }
+
+  /*
+  // filelock
+  mds->locker->mark_updated_scatterlock(&diri->filelock);
+  mut->ls->dirty_dirfrag_dir.push_back(&diri->item_dirty_dirfrag_dir);
+  mut->add_updated_lock(&diri->filelock);
+
+  // dirlock
+  mds->locker->mark_updated_scatterlock(&diri->nestlock);
+  mut->ls->dirty_dirfrag_nest.push_back(&diri->item_dirty_dirfrag_nest);
+  mut->add_updated_lock(&diri->nestlock);
+  */
+
+  add_uncommitted_fragment(basedirfrag, info.bits, le->orig_frags, mdr->ls);
+  mds->mdlog->submit_entry(le, new C_MDC_FragmentPrep(this, mdr));
+  mds->mdlog->flush();
+}
+
+void MDCache::_fragment_logged(MDRequestRef& mdr)
+{
+  dirfrag_t basedirfrag = mdr->more()->fragment_base;
+  map::iterator it = fragments.find(basedirfrag);
+  assert(it != fragments.end());
+  fragment_info_t &info = it->second;
+  CInode *diri = info.resultfrags.front()->get_inode();
+
+  dout(10) << "fragment_logged " << basedirfrag << " bits " << info.bits
+	   << " on " << *diri << dendl;
+
+  if (diri->is_auth())
+    diri->pop_and_dirty_projected_inode(mdr->ls);
+
+  mdr->apply();  // mark scatterlock
+
+  // store resulting frags
+  C_GatherBuilder gather(g_ceph_context, new C_MDC_FragmentStore(this, mdr));
+
+  for (list::iterator p = info.resultfrags.begin();
+       p != info.resultfrags.end();
+       ++p) {
+    CDir *dir = *p;
+    dout(10) << " storing result frag " << *dir << dendl;
+
+    // freeze and store them too
+    dir->auth_pin(this);
+    dir->state_set(CDir::STATE_FRAGMENTING);
+    dir->commit(0, gather.new_sub(), true);  // ignore authpinnability
+  }
+
+  gather.activate();
+}
+
+void MDCache::_fragment_stored(MDRequestRef& mdr)
+{
+  dirfrag_t basedirfrag = mdr->more()->fragment_base;
+  map::iterator it = fragments.find(basedirfrag);
+  assert(it != fragments.end());
+  fragment_info_t &info = it->second;
+  CInode *diri = info.resultfrags.front()->get_inode();
+
+  dout(10) << "fragment_stored " << basedirfrag << " bits " << info.bits
+	   << " on " << *diri << dendl;
+
+  // tell peers
+  CDir *first = *info.resultfrags.begin();
+  for (map::iterator p = first->replicas_begin();
+       p != first->replica_map.end();
+       ++p) {
+    if (mds->mdsmap->get_state(p->first) < MDSMap::STATE_REJOIN ||
+	(mds->mdsmap->get_state(p->first) == MDSMap::STATE_REJOIN &&
+	 rejoin_gather.count(p->first)))
+      continue;
+
+    MMDSFragmentNotify *notify = new MMDSFragmentNotify(basedirfrag, info.bits);
+
+    // freshly replicate new dirs to peers
+    for (list::iterator q = info.resultfrags.begin();
+	 q != info.resultfrags.end();
+	 ++q)
+      replicate_dir(*q, p->first, notify->basebl);
+
+    mds->send_message_mds(notify, p->first);
+  }
+
+  // journal commit
+  EFragment *le = new EFragment(mds->mdlog, EFragment::OP_COMMIT, basedirfrag, info.bits);
+  mds->mdlog->start_submit_entry(le, new C_MDC_FragmentCommit(this, basedirfrag,
+							      info.resultfrags));
+
+  mds->locker->drop_locks(mdr.get());
+
+  // unfreeze resulting frags
+  for (list::iterator p = info.resultfrags.begin();
+       p != info.resultfrags.end();
+       ++p) {
+    CDir *dir = *p;
+    dout(10) << " result frag " << *dir << dendl;
+
+    for (CDir::map_t::iterator p = dir->items.begin();
+	 p != dir->items.end();
+	 ++p) { 
+      CDentry *dn = p->second;
+      assert(dn->state_test(CDentry::STATE_FRAGMENTING));
+      dn->state_clear(CDentry::STATE_FRAGMENTING);
+      dn->put(CDentry::PIN_FRAGMENTING);
+    }
+
+    // unfreeze
+    dir->unfreeze_dir();
+  }
+
+  fragments.erase(it);
+  request_finish(mdr);
+}
+
+void MDCache::_fragment_committed(dirfrag_t basedirfrag, list& resultfrags)
+{
+  dout(10) << "fragment_committed " << basedirfrag << dendl;
+  map::iterator it = uncommitted_fragments.find(basedirfrag);
+  assert(it != uncommitted_fragments.end());
+  ufragment &uf = it->second;
+
+  // remove old frags
+  C_GatherBuilder gather(g_ceph_context, new C_MDC_FragmentFinish(this, basedirfrag, resultfrags));
+
+  SnapContext nullsnapc;
+  object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+  for (list::iterator p = uf.old_frags.begin();
+       p != uf.old_frags.end();
+       ++p) {
+    object_t oid = CInode::get_object_name(basedirfrag.ino, *p, "");
+    ObjectOperation op;
+    if (*p == frag_t()) {
+      // backtrace object
+      dout(10) << " truncate orphan dirfrag " << oid << dendl;
+      op.truncate(0);
+      op.omap_clear();
+    } else {
+      dout(10) << " removing orphan dirfrag " << oid << dendl;
+      op.remove();
+    }
+    mds->objecter->mutate(oid, oloc, op, nullsnapc, ceph_clock_now(g_ceph_context),
+			  0, NULL, gather.new_sub());
+  }
+
+  assert(gather.has_subs());
+  gather.activate();
+}
+
+void MDCache::_fragment_finish(dirfrag_t basedirfrag, list& resultfrags)
+{
+  dout(10) << "fragment_finish " << basedirfrag << dendl;
+  map::iterator it = uncommitted_fragments.find(basedirfrag);
+  assert(it != uncommitted_fragments.end());
+  ufragment &uf = it->second;
+
+  // unmark & auth_unpin
+  for (list::iterator p = resultfrags.begin(); p != resultfrags.end(); ++p) {
+    (*p)->state_clear(CDir::STATE_FRAGMENTING);
+    (*p)->auth_unpin(this);
+  }
+
+  EFragment *le = new EFragment(mds->mdlog, EFragment::OP_FINISH, basedirfrag, uf.bits);
+  mds->mdlog->start_submit_entry(le);
+
+  finish_uncommitted_fragment(basedirfrag, EFragment::OP_FINISH);
+}
+
+/* This function DOES put the passed message before returning */
+void MDCache::handle_fragment_notify(MMDSFragmentNotify *notify)
+{
+  dout(10) << "handle_fragment_notify " << *notify << " from " << notify->get_source() << dendl;
+
+  if (mds->get_state() < MDSMap::STATE_REJOIN) {
+    notify->put();
+    return;
+  }
+
+  CInode *diri = get_inode(notify->get_ino());
+  if (diri) {
+    frag_t base = notify->get_basefrag();
+    int bits = notify->get_bits();
+
+/*
+    if ((bits < 0 && diri->dirfragtree.is_leaf(base)) ||
+	(bits > 0 && !diri->dirfragtree.is_leaf(base))) {
+      dout(10) << " dft " << diri->dirfragtree << " state doesn't match " << base << " by " << bits
+	       << ", must have found out during resolve/rejoin?  ignoring. " << *diri << dendl;
+      notify->put();
+      return;
+    }
+*/
+
+    // refragment
+    list waiters;
+    list resultfrags;
+    adjust_dir_fragments(diri, base, bits, resultfrags, waiters, false);
+    if (g_conf->mds_debug_frag)
+      diri->verify_dirfrags();
+    
+    for (list::iterator p = resultfrags.begin(); p != resultfrags.end(); ++p)
+      diri->take_dir_waiting((*p)->get_frag(), waiters);
+
+    // add new replica dirs values
+    bufferlist::iterator p = notify->basebl.begin();
+    while (!p.end())
+      add_replica_dir(p, diri, notify->get_source().num(), waiters);
+
+    mds->queue_waiters(waiters);
+  } else {
+    assert(0);
+  }
+
+  notify->put();
+}
+
+void MDCache::add_uncommitted_fragment(dirfrag_t basedirfrag, int bits, list& old_frags,
+				       LogSegment *ls, bufferlist *rollback)
+{
+  dout(10) << "add_uncommitted_fragment: base dirfrag " << basedirfrag << " bits " << bits << dendl;
+  assert(!uncommitted_fragments.count(basedirfrag));
+  ufragment& uf = uncommitted_fragments[basedirfrag];
+  uf.old_frags = old_frags;
+  uf.bits = bits;
+  uf.ls = ls;
+  ls->uncommitted_fragments.insert(basedirfrag);
+  if (rollback)
+    uf.rollback.swap(*rollback);
+}
+
+void MDCache::finish_uncommitted_fragment(dirfrag_t basedirfrag, int op)
+{
+  dout(10) << "finish_uncommitted_fragments: base dirfrag " << basedirfrag
+	   << " op " << EFragment::op_name(op) << dendl;
+  map::iterator it = uncommitted_fragments.find(basedirfrag);
+  if (it != uncommitted_fragments.end()) {
+    ufragment& uf = it->second;
+    if (op != EFragment::OP_FINISH && !uf.old_frags.empty()) {
+      uf.committed = true;
+    } else {
+      uf.ls->uncommitted_fragments.erase(basedirfrag);
+      mds->queue_waiters(uf.waiters);
+      uncommitted_fragments.erase(it);
+    }
+  }
+}
+
+void MDCache::rollback_uncommitted_fragment(dirfrag_t basedirfrag, list& old_frags)
+{
+  dout(10) << "rollback_uncommitted_fragment: base dirfrag " << basedirfrag
+           << " old_frags (" << old_frags << ")" << dendl;
+  map::iterator it = uncommitted_fragments.find(basedirfrag);
+  if (it != uncommitted_fragments.end()) {
+    ufragment& uf = it->second;
+    if (!uf.old_frags.empty()) {
+      uf.old_frags.swap(old_frags);
+      uf.committed = true;
+    } else {
+      uf.ls->uncommitted_fragments.erase(basedirfrag);
+      uncommitted_fragments.erase(it);
+    }
+  }
+}
+
+void MDCache::rollback_uncommitted_fragments()
+{
+  dout(10) << "rollback_uncommitted_fragments: " << uncommitted_fragments.size() << " pending" << dendl;
+  for (map::iterator p = uncommitted_fragments.begin();
+       p != uncommitted_fragments.end();
+       ++p) {
+    ufragment &uf = p->second;
+    CInode *diri = get_inode(p->first.ino);
+    assert(diri);
+
+    if (uf.committed) {
+      list frags;
+      diri->get_dirfrags_under(p->first.frag, frags);
+      for (list::iterator q = frags.begin(); q != frags.end(); ++q) {
+	CDir *dir = *q;
+	dir->auth_pin(this);
+	dir->state_set(CDir::STATE_FRAGMENTING);
+      }
+      _fragment_committed(p->first, frags);
+      continue;
+    }
+
+    dout(10) << " rolling back " << p->first << " refragment by " << uf.bits << " bits" << dendl;
+
+    LogSegment *ls = mds->mdlog->get_current_segment();
+    EFragment *le = new EFragment(mds->mdlog, EFragment::OP_ROLLBACK, p->first, uf.bits);
+    mds->mdlog->start_entry(le);
+    bool diri_auth = (diri->authority() != CDIR_AUTH_UNDEF);
+
+    list old_frags;
+    diri->dirfragtree.get_leaves_under(p->first.frag, old_frags);
+
+    list resultfrags;
+    if (uf.old_frags.empty()) {
+      // created by old format EFragment
+      list waiters;
+      adjust_dir_fragments(diri, p->first.frag, -uf.bits, resultfrags, waiters, true);
+    } else {
+      bufferlist::iterator bp = uf.rollback.begin();
+      for (list::iterator q = uf.old_frags.begin(); q != uf.old_frags.end(); ++q) {
+	CDir *dir = force_dir_fragment(diri, *q);
+	resultfrags.push_back(dir);
+
+	dirfrag_rollback rollback;
+	::decode(rollback, bp);
+
+	dir->set_version(rollback.fnode.version);
+	dir->fnode = rollback.fnode;
+
+	dir->_mark_dirty(ls);
+
+	if (!(dir->fnode.rstat == dir->fnode.accounted_rstat)) {
+	  dout(10) << "    dirty nestinfo on " << *dir << dendl;
+	  mds->locker->mark_updated_scatterlock(&dir->inode->nestlock);
+	  ls->dirty_dirfrag_nest.push_back(&dir->inode->item_dirty_dirfrag_nest);
+	}
+	if (!(dir->fnode.fragstat == dir->fnode.accounted_fragstat)) {
+	  dout(10) << "    dirty fragstat on " << *dir << dendl;
+	  mds->locker->mark_updated_scatterlock(&dir->inode->filelock);
+	  ls->dirty_dirfrag_dir.push_back(&dir->inode->item_dirty_dirfrag_dir);
+	}
+
+	le->add_orig_frag(dir->get_frag());
+	le->metablob.add_dir_context(dir);
+	if (diri_auth) {
+	  le->metablob.add_fragmented_dir(dir, true, false);
+	} else {
+	  dout(10) << "    dirty dirfragtree on " << *dir << dendl;
+	  dir->state_set(CDir::STATE_DIRTYDFT);
+	  le->metablob.add_fragmented_dir(dir, true, true);
+	}
+      }
+    }
+
+    if (diri_auth) {
+      diri->project_inode()->version = diri->pre_dirty();
+      diri->pop_and_dirty_projected_inode(ls); // hacky
+      le->metablob.add_primary_dentry(diri->get_projected_parent_dn(), diri, true);
+    } else {
+      mds->locker->mark_updated_scatterlock(&diri->dirfragtreelock);
+      ls->dirty_dirfrag_dirfragtree.push_back(&diri->item_dirty_dirfrag_dirfragtree);
+    }
+
+    if (g_conf->mds_debug_frag)
+      diri->verify_dirfrags();
+
+    for (list::iterator q = old_frags.begin(); q != old_frags.end(); ++q)
+      assert(!diri->dirfragtree.is_leaf(*q));
+
+    for (list::iterator q = resultfrags.begin(); q != resultfrags.end(); ++q) {
+      CDir *dir = *q;
+      dir->auth_pin(this);
+      dir->state_set(CDir::STATE_FRAGMENTING);
+    }
+
+    mds->mdlog->submit_entry(le);
+
+    uf.old_frags.swap(old_frags);
+    _fragment_committed(p->first, resultfrags);
+  }
+}
+
+
+
+// ==============================================================
+// debug crap
+
+void MDCache::show_subtrees(int dbl)
+{
+  //dout(10) << "show_subtrees" << dendl;
+
+  if (!g_conf->subsys.should_gather(ceph_subsys_mds, dbl))
+    return;  // i won't print anything.
+
+  if (subtrees.empty()) {
+    dout(dbl) << "show_subtrees - no subtrees" << dendl;
+    return;
+  }
+
+  // root frags
+  list basefrags;
+  for (set::iterator p = base_inodes.begin();
+       p != base_inodes.end();
+       ++p) 
+    (*p)->get_dirfrags(basefrags);
+  //dout(15) << "show_subtrees, base dirfrags " << basefrags << dendl;
+  dout(15) << "show_subtrees" << dendl;
+
+  // queue stuff
+  list > q;
+  string indent;
+  set seen;
+
+  // calc max depth
+  for (list::iterator p = basefrags.begin(); p != basefrags.end(); ++p) 
+    q.push_back(pair(*p, 0));
+
+  set subtrees_seen;
+
+  int depth = 0;
+  while (!q.empty()) {
+    CDir *dir = q.front().first;
+    int d = q.front().second;
+    q.pop_front();
+
+    if (subtrees.count(dir) == 0) continue;
+
+    subtrees_seen.insert(dir);
+
+    if (d > depth) depth = d;
+
+    // sanity check
+    //dout(25) << "saw depth " << d << " " << *dir << dendl;
+    if (seen.count(dir)) dout(0) << "aah, already seen " << *dir << dendl;
+    assert(seen.count(dir) == 0);
+    seen.insert(dir);
+
+    // nested items?
+    if (!subtrees[dir].empty()) {
+      for (set::iterator p = subtrees[dir].begin();
+	   p != subtrees[dir].end();
+	   ++p) {
+	//dout(25) << " saw sub " << **p << dendl;
+	q.push_front(pair(*p, d+1));
+      }
+    }
+  }
+
+
+  // print tree
+  for (list::iterator p = basefrags.begin(); p != basefrags.end(); ++p) 
+    q.push_back(pair(*p, 0));
+
+  while (!q.empty()) {
+    CDir *dir = q.front().first;
+    int d = q.front().second;
+    q.pop_front();
+
+    if (subtrees.count(dir) == 0) continue;
+
+    // adjust indenter
+    while ((unsigned)d < indent.size()) 
+      indent.resize(d);
+    
+    // pad
+    string pad = "______________________________________";
+    pad.resize(depth*2+1-indent.size());
+    if (!subtrees[dir].empty()) 
+      pad[0] = '.'; // parent
+
+
+    string auth;
+    if (dir->is_auth())
+      auth = "auth ";
+    else
+      auth = " rep ";
+
+    char s[10];
+    if (dir->get_dir_auth().second == CDIR_AUTH_UNKNOWN)
+      snprintf(s, sizeof(s), "%2d   ", dir->get_dir_auth().first);
+    else
+      snprintf(s, sizeof(s), "%2d,%2d", dir->get_dir_auth().first, dir->get_dir_auth().second);
+    
+    // print
+    dout(dbl) << indent << "|_" << pad << s << " " << auth << *dir << dendl;
+
+    if (dir->ino() == MDS_INO_ROOT)
+      assert(dir->inode == root);
+    if (dir->ino() == MDS_INO_MDSDIR(mds->get_nodeid()))
+      assert(dir->inode == myin);
+    if (dir->inode->is_stray() && (MDS_INO_STRAY_OWNER(dir->ino()) == mds->get_nodeid()))
+      assert(strays[MDS_INO_STRAY_INDEX(dir->ino())] == dir->inode);
+
+    // nested items?
+    if (!subtrees[dir].empty()) {
+      // more at my level?
+      if (!q.empty() && q.front().second == d)
+	indent += "| ";
+      else
+	indent += "  ";
+
+      for (set::iterator p = subtrees[dir].begin();
+	   p != subtrees[dir].end();
+	   ++p) 
+	q.push_front(pair(*p, d+2));
+    }
+  }
+
+  // verify there isn't stray crap in subtree map
+  int lost = 0;
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    if (subtrees_seen.count(p->first)) continue;
+    dout(10) << "*** stray/lost entry in subtree map: " << *p->first << dendl;
+    lost++;
+  }
+  assert(lost == 0);
+}
+
+
+void MDCache::show_cache()
+{
+  dout(7) << "show_cache" << dendl;
+  
+  for (ceph::unordered_map::iterator it = inode_map.begin();
+       it != inode_map.end();
+       ++it) {
+    // unlinked?
+    if (!it->second->parent)
+      dout(7) << " unlinked " << *it->second << dendl;
+    
+    // dirfrags?
+    list dfs;
+    it->second->get_dirfrags(dfs);
+    for (list::iterator p = dfs.begin(); p != dfs.end(); ++p) {
+      CDir *dir = *p;
+      dout(7) << "  dirfrag " << *dir << dendl;
+	    
+      for (CDir::map_t::iterator p = dir->items.begin();
+	   p != dir->items.end();
+	   ++p) {
+	CDentry *dn = p->second;
+	dout(7) << "   dentry " << *dn << dendl;
+	CDentry::linkage_t *dnl = dn->get_linkage();
+	if (dnl->is_primary() && dnl->get_inode()) 
+	  dout(7) << "    inode " << *dnl->get_inode() << dendl;
+      }
+    }
+  }
+}
+
+
+void MDCache::dump_cache(const char *fn)
+{
+  int r;
+  char deffn[200];
+  if (!fn) {
+    snprintf(deffn, sizeof(deffn), "cachedump.%d.mds%d", (int)mds->mdsmap->get_epoch(), mds->get_nodeid());
+    fn = deffn;
+  }
+
+  dout(1) << "dump_cache to " << fn << dendl;
+
+  int fd = ::open(fn, O_WRONLY|O_CREAT|O_EXCL, 0600);
+  if (fd < 0) {
+    derr << "failed to open " << fn << ": " << cpp_strerror(errno) << dendl;
+    return;
+  }
+  
+  for (ceph::unordered_map::iterator it = inode_map.begin();
+       it != inode_map.end();
+       ++it) {
+    CInode *in = it->second;
+    ostringstream ss;
+    ss << *in << std::endl;
+    std::string s = ss.str();
+    r = safe_write(fd, s.c_str(), s.length());
+    if (r < 0)
+      goto out;
+
+    list dfs;
+    in->get_dirfrags(dfs);
+    for (list::iterator p = dfs.begin(); p != dfs.end(); ++p) {
+      CDir *dir = *p;
+      ostringstream tt;
+      tt << " " << *dir << std::endl;
+      string t = tt.str();
+      r = safe_write(fd, t.c_str(), t.length());
+      if (r < 0)
+	goto out;
+      
+      for (CDir::map_t::iterator q = dir->items.begin();
+	   q != dir->items.end();
+	   ++q) {
+	CDentry *dn = q->second;
+	ostringstream uu;
+	uu << "  " << *dn << std::endl;
+	string u = uu.str();
+	r = safe_write(fd, u.c_str(), u.length());
+	if (r < 0)
+	  goto out;
+      }
+      dir->check_rstats();
+    }
+  }
+
+ out:
+  ::close(fd);
+}
+
+
+
+C_MDS_RetryRequest::C_MDS_RetryRequest(MDCache *c, MDRequestRef& r)
+  : cache(c), mdr(r)
+{}
+
+void C_MDS_RetryRequest::finish(int r)
+{
+  mdr->retry++;
+  cache->dispatch_request(mdr);
+}
diff --git a/ceph/src/mds/MDCache.h b/ceph/src/mds/MDCache.h
new file mode 100644
index 00000000..c536d32f
--- /dev/null
+++ b/ceph/src/mds/MDCache.h
@@ -0,0 +1,1071 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MDCACHE_H
+#define CEPH_MDCACHE_H
+
+#include "include/types.h"
+#include "include/filepath.h"
+#include "include/elist.h"
+
+#include "CInode.h"
+#include "CDentry.h"
+#include "CDir.h"
+#include "include/Context.h"
+#include "events/EMetaBlob.h"
+
+#include "messages/MClientRequest.h"
+#include "messages/MMDSSlaveRequest.h"
+
+class PerfCounters;
+
+class MDS;
+class Session;
+class Migrator;
+
+class Message;
+class Session;
+
+class MMDSResolve;
+class MMDSResolveAck;
+class MMDSCacheRejoin;
+class MMDSCacheRejoinAck;
+class MJoin;
+class MJoinAck;
+class MDiscover;
+class MDiscoverReply;
+class MCacheExpire;
+class MDirUpdate;
+class MDentryLink;
+class MDentryUnlink;
+class MLock;
+struct MMDSFindIno;
+struct MMDSFindInoReply;
+struct MMDSOpenIno;
+struct MMDSOpenInoReply;
+
+class Message;
+class MClientRequest;
+class MMDSSlaveRequest;
+struct MClientSnap;
+
+class MMDSFragmentNotify;
+
+class ESubtreeMap;
+
+struct MDRequestImpl;
+typedef ceph::shared_ptr MDRequestRef;
+struct MDSlaveUpdate;
+
+
+// flags for predirty_journal_parents()
+static const int PREDIRTY_PRIMARY = 1; // primary dn, adjust nested accounting
+static const int PREDIRTY_DIR = 2;     // update parent dir mtime/size
+static const int PREDIRTY_SHALLOW = 4; // only go to immediate parent (for easier rollback)
+
+class MDCache {
+ public:
+  // my master
+  MDS *mds;
+
+  // -- my cache --
+  LRU lru;   // dentry lru for expiring items from cache
+ protected:
+  ceph::unordered_map inode_map;  // map of inodes by ino
+  CInode *root;                            // root inode
+  CInode *myin;                            // .ceph/mds%d dir
+
+  CInode *strays[NUM_STRAY];         // my stray dir
+  int stray_index;
+
+  CInode *get_stray() {
+    return strays[stray_index];
+  }
+
+  set base_inodes;
+
+public:
+  void advance_stray() {
+    stray_index = (stray_index+1)%NUM_STRAY;
+  }
+
+  DecayRate decayrate;
+
+  int num_inodes_with_caps;
+  int num_caps;
+
+  unsigned max_dir_commit_size;
+
+  ceph_file_layout default_file_layout;
+  ceph_file_layout default_log_layout;
+
+  // -- client leases --
+public:
+  static const int client_lease_pools = 3;
+  float client_lease_durations[client_lease_pools];
+protected:
+  xlist client_leases[client_lease_pools];
+public:
+  void touch_client_lease(ClientLease *r, int pool, utime_t ttl) {
+    client_leases[pool].push_back(&r->item_lease);
+    r->ttl = ttl;
+  }
+
+  // -- client caps --
+  uint64_t              last_cap_id;
+  
+
+
+  // -- discover --
+  struct discover_info_t {
+    ceph_tid_t tid;
+    int mds;
+    inodeno_t ino;
+    frag_t frag;
+    snapid_t snap;
+    filepath want_path;
+    inodeno_t want_ino;
+    bool want_base_dir;
+    bool want_xlocked;
+
+    discover_info_t() : tid(0), mds(-1), snap(CEPH_NOSNAP), want_base_dir(false), want_xlocked(false) {}
+  };
+
+  map discovers;
+  ceph_tid_t discover_last_tid;
+
+  void _send_discover(discover_info_t& dis);
+  discover_info_t& _create_discover(int mds) {
+    ceph_tid_t t = ++discover_last_tid;
+    discover_info_t& d = discovers[t];
+    d.tid = t;
+    d.mds = mds;
+    return d;
+  }
+
+  // waiters
+  map > > waiting_for_base_ino;
+
+  void discover_base_ino(inodeno_t want_ino, Context *onfinish, int from=-1);
+  void discover_dir_frag(CInode *base, frag_t approx_fg, Context *onfinish,
+			 int from=-1);
+  void discover_path(CInode *base, snapid_t snap, filepath want_path, Context *onfinish,
+		     bool want_xlocked=false, int from=-1);
+  void discover_path(CDir *base, snapid_t snap, filepath want_path, Context *onfinish,
+		     bool want_xlocked=false);
+  void discover_ino(CDir *base, inodeno_t want_ino, Context *onfinish,
+		    bool want_xlocked=false);
+
+  void kick_discovers(int who);  // after a failure.
+
+
+public:
+  int get_num_inodes() { return inode_map.size(); }
+  int get_num_dentries() { return lru.lru_get_size(); }
+
+
+  // -- subtrees --
+protected:
+  map > subtrees;   // nested bounds on subtrees.
+  map > > projected_subtree_renames;  // renamed ino -> target dir
+  
+  // adjust subtree auth specification
+  //  dir->dir_auth
+  //  imports/exports/nested_exports
+  //  join/split subtrees as appropriate
+public:
+  bool is_subtrees() { return !subtrees.empty(); }
+  void list_subtrees(list& ls);
+  void adjust_subtree_auth(CDir *root, pair auth, bool do_eval=true);
+  void adjust_subtree_auth(CDir *root, int a, int b=CDIR_AUTH_UNKNOWN, bool do_eval=true) {
+    adjust_subtree_auth(root, pair(a,b), do_eval); 
+  }
+  void adjust_bounded_subtree_auth(CDir *dir, set& bounds, pair auth);
+  void adjust_bounded_subtree_auth(CDir *dir, set& bounds, int a) {
+    adjust_bounded_subtree_auth(dir, bounds, pair(a, CDIR_AUTH_UNKNOWN));
+  }
+  void adjust_bounded_subtree_auth(CDir *dir, vector& bounds, pair auth);
+  void adjust_bounded_subtree_auth(CDir *dir, vector& bounds, int a) {
+    adjust_bounded_subtree_auth(dir, bounds, pair(a, CDIR_AUTH_UNKNOWN));
+  }
+  void map_dirfrag_set(list& dfs, set& result);
+  void try_subtree_merge(CDir *root);
+  void try_subtree_merge_at(CDir *root, bool do_eval=true);
+  void subtree_merge_writebehind_finish(CInode *in, MutationRef& mut);
+  void eval_subtree_root(CInode *diri);
+  CDir *get_subtree_root(CDir *dir);
+  CDir *get_projected_subtree_root(CDir *dir);
+  bool is_leaf_subtree(CDir *dir) {
+    assert(subtrees.count(dir));
+    return subtrees[dir].empty();
+  }
+  void remove_subtree(CDir *dir);
+  bool is_subtree(CDir *root) {
+    return subtrees.count(root);
+  }
+  void get_subtree_bounds(CDir *root, set& bounds);
+  void get_wouldbe_subtree_bounds(CDir *root, set& bounds);
+  void verify_subtree_bounds(CDir *root, const set& bounds);
+  void verify_subtree_bounds(CDir *root, const list& bounds);
+
+  void project_subtree_rename(CInode *diri, CDir *olddir, CDir *newdir);
+  void adjust_subtree_after_rename(CInode *diri, CDir *olddir,
+                                   bool pop, bool imported = false);
+
+  void get_auth_subtrees(set& s);
+  void get_fullauth_subtrees(set& s);
+
+  int num_subtrees();
+  int num_subtrees_fullauth();
+  int num_subtrees_fullnonauth();
+
+  
+protected:
+  // delayed cache expire
+  map > delayed_expire; // subtree root -> expire msg
+
+
+  // -- requests --
+protected:
+  ceph::unordered_map active_requests;
+
+public:
+  int get_num_client_requests();
+
+  MDRequestRef request_start(MClientRequest *req);
+  MDRequestRef request_start_slave(metareqid_t rid, __u32 attempt, int by);
+  MDRequestRef request_start_internal(int op);
+  bool have_request(metareqid_t rid) {
+    return active_requests.count(rid);
+  }
+  MDRequestRef request_get(metareqid_t rid);
+  void request_pin_ref(MDRequestRef& r, CInode *ref, vector& trace);
+  void request_finish(MDRequestRef& mdr);
+  void request_forward(MDRequestRef& mdr, int mds, int port=0);
+  void dispatch_request(MDRequestRef& mdr);
+  void request_drop_foreign_locks(MDRequestRef& mdr);
+  void request_drop_non_rdlocks(MDRequestRef& r);
+  void request_drop_locks(MDRequestRef& r);
+  void request_cleanup(MDRequestRef& r);
+  
+  void request_kill(MDRequestRef& r);  // called when session closes
+
+  // journal/snap helpers
+  CInode *pick_inode_snap(CInode *in, snapid_t follows);
+  CInode *cow_inode(CInode *in, snapid_t last);
+  void journal_cow_dentry(MutationImpl *mut, EMetaBlob *metablob, CDentry *dn,
+                          snapid_t follows=CEPH_NOSNAP,
+			  CInode **pcow_inode=0, CDentry::linkage_t *dnl=0);
+  void journal_cow_inode(MutationRef& mut, EMetaBlob *metablob, CInode *in, snapid_t follows=CEPH_NOSNAP,
+			  CInode **pcow_inode=0);
+  void journal_dirty_inode(MutationImpl *mut, EMetaBlob *metablob, CInode *in, snapid_t follows=CEPH_NOSNAP);
+
+  void project_rstat_inode_to_frag(CInode *cur, CDir *parent, snapid_t first, int linkunlink);
+  void _project_rstat_inode_to_frag(inode_t& inode, snapid_t ofirst, snapid_t last,
+				    CDir *parent, int linkunlink=0);
+  void project_rstat_frag_to_inode(nest_info_t& rstat, nest_info_t& accounted_rstat,
+				   snapid_t ofirst, snapid_t last, 
+				   CInode *pin, bool cow_head);
+  void predirty_journal_parents(MutationRef mut, EMetaBlob *blob,
+				CInode *in, CDir *parent,
+				int flags, int linkunlink=0,
+				snapid_t follows=CEPH_NOSNAP);
+
+  // slaves
+  void add_uncommitted_master(metareqid_t reqid, LogSegment *ls, set &slaves, bool safe=false) {
+    uncommitted_masters[reqid].ls = ls;
+    uncommitted_masters[reqid].slaves = slaves;
+    uncommitted_masters[reqid].safe = safe;
+  }
+  void wait_for_uncommitted_master(metareqid_t reqid, Context *c) {
+    uncommitted_masters[reqid].waiters.push_back(c);
+  }
+  void log_master_commit(metareqid_t reqid);
+  void logged_master_update(metareqid_t reqid);
+  void _logged_master_commit(metareqid_t reqid);
+  void committed_master_slave(metareqid_t r, int from);
+  void finish_committed_masters();
+
+  void _logged_slave_commit(int from, metareqid_t reqid);
+
+  // -- recovery --
+protected:
+  set recovery_set;
+
+public:
+  void set_recovery_set(set& s);
+  void handle_mds_failure(int who);
+  void handle_mds_recovery(int who);
+
+protected:
+  // [resolve]
+  // from EImportStart w/o EImportFinish during journal replay
+  map >            my_ambiguous_imports;  
+  // from MMDSResolves
+  map > > other_ambiguous_imports;  
+
+  map > uncommitted_slave_updates;  // slave: for replay.
+  map uncommitted_slave_rename_olddir;  // slave: preserve the non-auth dir until seeing commit.
+  map uncommitted_slave_unlink;  // slave: preserve the unlinked inode until seeing commit.
+
+  // track master requests whose slaves haven't acknowledged commit
+  struct umaster {
+    set slaves;
+    LogSegment *ls;
+    list waiters;
+    bool safe;
+    bool committing;
+    bool recovering;
+    umaster() : committing(false), recovering(false) {}
+  };
+  map                 uncommitted_masters;         // master: req -> slave set
+
+  set		pending_masters;
+  map >	ambiguous_slave_updates;
+
+  friend class ESlaveUpdate;
+  friend class ECommitted;
+
+  bool resolves_pending;
+  set resolve_gather;	// nodes i need resolves from
+  set resolve_ack_gather;	// nodes i need a resolve_ack from
+  map need_resolve_rollback;  // rollbacks i'm writing to the journal
+  map delayed_resolve;
+  
+  void handle_resolve(MMDSResolve *m);
+  void handle_resolve_ack(MMDSResolveAck *m);
+  void process_delayed_resolve();
+  void discard_delayed_resolve(int who);
+  void maybe_resolve_finish();
+  void disambiguate_imports();
+  void recalc_auth_bits();
+  void trim_unlinked_inodes();
+  void add_uncommitted_slave_update(metareqid_t reqid, int master, MDSlaveUpdate*);
+  void finish_uncommitted_slave_update(metareqid_t reqid, int master);
+  MDSlaveUpdate* get_uncommitted_slave_update(metareqid_t reqid, int master);
+public:
+  void remove_inode_recursive(CInode *in);
+
+  bool is_ambiguous_slave_update(metareqid_t reqid, int master) {
+    return ambiguous_slave_updates.count(master) &&
+	   ambiguous_slave_updates[master].count(reqid);
+  }
+  void add_ambiguous_slave_update(metareqid_t reqid, int master) {
+    ambiguous_slave_updates[master].insert(reqid);
+  }
+  void remove_ambiguous_slave_update(metareqid_t reqid, int master) {
+    assert(ambiguous_slave_updates[master].count(reqid));
+    ambiguous_slave_updates[master].erase(reqid);
+    if (ambiguous_slave_updates[master].empty())
+      ambiguous_slave_updates.erase(master);
+  }
+
+  void add_rollback(metareqid_t reqid, int master) {
+    need_resolve_rollback[reqid] = master;
+  }
+  void finish_rollback(metareqid_t reqid);
+
+  // ambiguous imports
+  void add_ambiguous_import(dirfrag_t base, const vector& bounds);
+  void add_ambiguous_import(CDir *base, const set& bounds);
+  bool have_ambiguous_import(dirfrag_t base) {
+    return my_ambiguous_imports.count(base);
+  }
+  void get_ambiguous_import_bounds(dirfrag_t base, vector& bounds) {
+    assert(my_ambiguous_imports.count(base));
+    bounds = my_ambiguous_imports[base];
+  }
+  void cancel_ambiguous_import(CDir *);
+  void finish_ambiguous_import(dirfrag_t dirino);
+  void resolve_start();
+  void send_resolves();
+  void send_slave_resolves();
+  void send_subtree_resolves();
+  void maybe_send_pending_resolves() {
+    if (resolves_pending)
+      send_subtree_resolves();
+  }
+  
+  void _move_subtree_map_bound(dirfrag_t df, dirfrag_t oldparent, dirfrag_t newparent,
+			       map >& subtrees);
+  ESubtreeMap *create_subtree_map();
+
+
+  void clean_open_file_lists();
+
+protected:
+  // [rejoin]
+  bool rejoins_pending;
+  set rejoin_gather;      // nodes from whom i need a rejoin
+  set rejoin_sent;        // nodes i sent a rejoin to
+  set rejoin_ack_gather;  // nodes from whom i need a rejoin ack
+  map > > rejoin_imported_caps;
+  map > > rejoin_slave_exports;
+  map rejoin_client_map;
+
+  map > cap_exports; // ino -> client -> capex
+  map cap_export_targets; // ino -> auth mds
+
+  map > > cap_imports;  // ino -> client -> frommds -> capex
+  map cap_import_paths;
+  set cap_imports_missing;
+  int cap_imports_num_opening;
+  
+  set rejoin_undef_inodes;
+  set rejoin_potential_updated_scatterlocks;
+  set   rejoin_undef_dirfrags;
+  map > rejoin_unlinked_inodes;
+
+  vector rejoin_recover_q, rejoin_check_q;
+  list rejoin_eval_locks;
+  list rejoin_waiters;
+
+  void rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin);
+  void handle_cache_rejoin(MMDSCacheRejoin *m);
+  void handle_cache_rejoin_weak(MMDSCacheRejoin *m);
+  CInode* rejoin_invent_inode(inodeno_t ino, snapid_t last);
+  CDir* rejoin_invent_dirfrag(dirfrag_t df);
+  void handle_cache_rejoin_strong(MMDSCacheRejoin *m);
+  void rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
+				      set& acked_inodes,
+				      set& gather_locks);
+  void handle_cache_rejoin_ack(MMDSCacheRejoin *m);
+  void handle_cache_rejoin_purge(MMDSCacheRejoin *m);
+  void handle_cache_rejoin_missing(MMDSCacheRejoin *m);
+  void handle_cache_rejoin_full(MMDSCacheRejoin *m);
+  void rejoin_send_acks();
+  void rejoin_trim_undef_inodes();
+  void maybe_send_pending_rejoins() {
+    if (rejoins_pending)
+      rejoin_send_rejoins();
+  }
+public:
+  void rejoin_start();
+  void rejoin_gather_finish();
+  void rejoin_send_rejoins();
+  void rejoin_export_caps(inodeno_t ino, client_t client, ceph_mds_cap_reconnect& capinfo,
+			  int target=-1) {
+    cap_exports[ino][client] = capinfo;
+    cap_export_targets[ino] = target;
+  }
+  void rejoin_recovered_caps(inodeno_t ino, client_t client, cap_reconnect_t& icr, 
+			     int frommds=-1) {
+    cap_imports[ino][client][frommds] = icr.capinfo;
+    cap_import_paths[ino] = filepath(icr.path, (uint64_t)icr.capinfo.pathbase);
+  }
+  ceph_mds_cap_reconnect *get_replay_cap_reconnect(inodeno_t ino, client_t client) {
+    if (cap_imports.count(ino) &&
+	cap_imports[ino].count(client) &&
+	cap_imports[ino][client].count(-1)) {
+      return &cap_imports[ino][client][-1];
+    }
+    return NULL;
+  }
+  void remove_replay_cap_reconnect(inodeno_t ino, client_t client) {
+    assert(cap_imports[ino].size() == 1);
+    assert(cap_imports[ino][client].size() == 1);
+    cap_imports.erase(ino);
+  }
+
+  // [reconnect/rejoin caps]
+  map >  reconnected_caps;   // inode -> client -> realmino
+  map > reconnected_snaprealms;  // realmino -> client -> realmseq
+
+  void add_reconnected_cap(CInode *in, client_t client, inodeno_t realm) {
+    reconnected_caps[in][client] = realm;
+  }
+  void add_reconnected_snaprealm(client_t client, inodeno_t ino, snapid_t seq) {
+    reconnected_snaprealms[ino][client] = seq;
+  }
+
+  friend class C_MDC_RejoinOpenInoFinish;
+  friend class C_MDC_RejoinSessionsOpened;
+  void rejoin_open_ino_finish(inodeno_t ino, int ret);
+  void rejoin_open_sessions_finish(map client_map,
+				   map& sseqmap);
+  bool process_imported_caps();
+  void choose_lock_states_and_reconnect_caps();
+  void prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino,
+			   map& splits);
+  void do_realm_invalidate_and_update_notify(CInode *in, int snapop, bool nosend=false);
+  void send_snaps(map& splits);
+  Capability* rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, int frommds);
+  void finish_snaprealm_reconnect(client_t client, SnapRealm *realm, snapid_t seq);
+  void try_reconnect_cap(CInode *in, Session *session);
+  void export_remaining_imported_caps();
+
+  // cap imports.  delayed snap parent opens.
+  //  realm inode -> client -> cap inodes needing to split to this realm
+  map > > missing_snap_parents; 
+  map > delayed_imported_caps;
+
+  void do_cap_import(Session *session, CInode *in, Capability *cap,
+		     uint64_t p_cap_id, ceph_seq_t p_seq, ceph_seq_t p_mseq,
+		     int peer, int p_flags);
+  void do_delayed_cap_imports();
+  void check_realm_past_parents(SnapRealm *realm);
+  void open_snap_parents();
+
+  bool open_undef_inodes_dirfrags();
+  void opened_undef_inode(CInode *in);
+  void opened_undef_dirfrag(CDir *dir) {
+    rejoin_undef_dirfrags.erase(dir);
+  }
+
+  void reissue_all_caps();
+  
+
+  friend class Locker;
+  friend class Migrator;
+  friend class MDBalancer;
+
+
+  // file size recovery
+  set file_recover_queue;
+  set file_recovering;
+
+  void queue_file_recover(CInode *in);
+  void unqueue_file_recover(CInode *in);
+  void _queued_file_recover_cow(CInode *in, MutationRef& mut);
+  void _queue_file_recover(CInode *in);
+  void identify_files_to_recover(vector& recover_q, vector& check_q);
+  void start_files_to_recover(vector& recover_q, vector& check_q);
+
+  void do_file_recover();
+  void _recovered(CInode *in, int r, uint64_t size, utime_t mtime);
+
+  void purge_prealloc_ino(inodeno_t ino, Context *fin);
+
+
+
+ public:
+
+  // subsystems
+  Migrator *migrator;
+
+ public:
+  MDCache(MDS *m);
+  ~MDCache();
+  
+  // debug
+  void log_stat();
+
+  // root inode
+  CInode *get_root() { return root; }
+  CInode *get_myin() { return myin; }
+
+  // cache
+  void set_cache_size(size_t max) { lru.lru_set_max(max); }
+  size_t get_cache_size() { return lru.lru_get_size(); }
+
+  // trimming
+  bool trim(int max = -1);   // trim cache
+  bool trim_dentry(CDentry *dn, map& expiremap);
+  void trim_dirfrag(CDir *dir, CDir *con,
+		    map& expiremap);
+  bool trim_inode(CDentry *dn, CInode *in, CDir *con,
+		  map& expiremap);
+  void send_expire_messages(map& expiremap);
+  void trim_non_auth();      // trim out trimmable non-auth items
+  bool trim_non_auth_subtree(CDir *directory);
+  void try_trim_non_auth_subtree(CDir *dir);
+  bool can_trim_non_auth_dirfrag(CDir *dir) {
+    return my_ambiguous_imports.count((dir)->dirfrag()) == 0 &&
+	   uncommitted_slave_rename_olddir.count(dir->inode) == 0;
+  }
+
+  void trim_client_leases();
+  void check_memory_usage();
+
+  // shutdown
+  void shutdown_start();
+  void shutdown_check();
+  bool shutdown_pass();
+  bool shutdown_export_strays();
+  bool shutdown_export_caps();
+  bool shutdown();                    // clear cache (ie at shutodwn)
+
+  bool did_shutdown_log_cap;
+
+  // inode_map
+  bool have_inode(vinodeno_t vino) {
+    return inode_map.count(vino) ? true:false;
+  }
+  bool have_inode(inodeno_t ino, snapid_t snap=CEPH_NOSNAP) {
+    return have_inode(vinodeno_t(ino, snap));
+  }
+  CInode* get_inode(vinodeno_t vino) {
+    if (have_inode(vino))
+      return inode_map[vino];
+    return NULL;
+  }
+  CInode* get_inode(inodeno_t ino, snapid_t s=CEPH_NOSNAP) {
+    return get_inode(vinodeno_t(ino, s));
+  }
+
+  CDir* get_dirfrag(dirfrag_t df) {
+    CInode *in = get_inode(df.ino);
+    if (!in)
+      return NULL;
+    return in->get_dirfrag(df.frag);
+  }
+  CDir* get_dirfrag(inodeno_t ino, const string& dn) {
+    CInode *in = get_inode(ino);
+    if (!in)
+      return NULL;
+    frag_t fg = in->pick_dirfrag(dn);
+    return in->get_dirfrag(fg);
+  }
+  CDir* get_force_dirfrag(dirfrag_t df) {
+    CInode *diri = get_inode(df.ino);
+    if (!diri)
+      return NULL;
+    CDir *dir = force_dir_fragment(diri, df.frag);
+    if (!dir)
+      dir = diri->get_dirfrag(df.frag);
+    return dir;
+  }
+
+  MDSCacheObject *get_object(MDSCacheObjectInfo &info);
+
+  
+
+ public:
+  void add_inode(CInode *in);
+
+  void remove_inode(CInode *in);
+ protected:
+  void touch_inode(CInode *in) {
+    if (in->get_parent_dn())
+      touch_dentry(in->get_projected_parent_dn());
+  }
+public:
+  void touch_dentry(CDentry *dn) {
+    // touch ancestors
+    if (dn->get_dir()->get_inode()->get_projected_parent_dn())
+      touch_dentry(dn->get_dir()->get_inode()->get_projected_parent_dn());
+    
+    // touch me
+    if (dn->is_auth())
+      lru.lru_touch(dn);
+    else
+      lru.lru_midtouch(dn);
+  }
+  void touch_dentry_bottom(CDentry *dn) {
+    lru.lru_bottouch(dn);
+    if (dn->get_projected_linkage()->is_primary()) {
+      CInode *in = dn->get_projected_linkage()->get_inode();
+      if (in->has_dirfrags()) {
+	list ls;
+	in->get_dirfrags(ls);
+	for (list::iterator p = ls.begin(); p != ls.end(); ++p)
+	  (*p)->touch_dentries_bottom();
+      }
+    }
+  }
+protected:
+
+  void inode_remove_replica(CInode *in, int rep, bool rejoin,
+			    set& gather_locks);
+  void dentry_remove_replica(CDentry *dn, int rep, set& gather_locks);
+
+  void rename_file(CDentry *srcdn, CDentry *destdn);
+
+ public:
+  // truncate
+  void truncate_inode(CInode *in, LogSegment *ls);
+  void _truncate_inode(CInode *in, LogSegment *ls);
+  void truncate_inode_finish(CInode *in, LogSegment *ls);
+  void truncate_inode_logged(CInode *in, MutationRef& mut);
+
+  void add_recovered_truncate(CInode *in, LogSegment *ls);
+  void remove_recovered_truncate(CInode *in, LogSegment *ls);
+  void start_recovered_truncates();
+
+
+ public:
+  CDir *get_auth_container(CDir *in);
+  CDir *get_export_container(CDir *dir);
+  void find_nested_exports(CDir *dir, set& s);
+  void find_nested_exports_under(CDir *import, CDir *dir, set& s);
+
+
+private:
+  bool opening_root, open;
+  list waiting_for_open;
+
+public:
+  void init_layouts();
+  CInode *create_system_inode(inodeno_t ino, int mode);
+  CInode *create_root_inode();
+
+  void create_empty_hierarchy(C_Gather *gather);
+  void create_mydir_hierarchy(C_Gather *gather);
+
+  bool is_open() { return open; }
+  void wait_for_open(Context *c) {
+    waiting_for_open.push_back(c);
+  }
+
+  void open_root_inode(Context *c);
+  void open_root();
+  void open_mydir_inode(Context *c);
+  void populate_mydir();
+
+  void _create_system_file(CDir *dir, const char *name, CInode *in, Context *fin);
+  void _create_system_file_finish(MutationRef& mut, CDentry *dn,
+                                  version_t dpv, Context *fin);
+
+  void open_foreign_mdsdir(inodeno_t ino, Context *c);
+  CDentry *get_or_create_stray_dentry(CInode *in);
+
+  Context *_get_waiter(MDRequestRef& mdr, Message *req, Context *fin);
+
+  /**
+   * Find the given dentry (and whether it exists or not), its ancestors,
+   * and get them all into memory and usable on this MDS. This function
+   * makes a best-effort attempt to load everything; if it needs to
+   * go away and do something then it will put the request on a waitlist.
+   * It prefers the mdr, then the req, then the fin. (At least one of these
+   * must be non-null.)
+   *
+   * At least one of the params mdr, req, and fin must be non-null.
+   *
+   * @param mdr The MDRequest associated with the path. Can be null.
+   * @param req The Message associated with the path. Can be null.
+   * @param fin The Context associated with the path. Can be null.
+   * @param path The path to traverse to.
+   * @param pdnvec Data return parameter -- on success, contains a
+   * vector of dentries. On failure, is either empty or contains the
+   * full trace of traversable dentries.
+   * @param pin Data return parameter -- if successful, points to the inode
+   * associated with filepath. If unsuccessful, is null.
+   * @param onfail Specifies different lookup failure behaviors. If set to
+   * MDS_TRAVERSE_DISCOVERXLOCK, path_traverse will succeed on null
+   * dentries (instead of returning -ENOENT). If set to
+   * MDS_TRAVERSE_FORWARD, it will forward the request to the auth
+   * MDS if that becomes appropriate (ie, if it doesn't know the contents
+   * of a directory). If set to MDS_TRAVERSE_DISCOVER, it
+   * will attempt to look up the path from a different MDS (and bring them
+   * into its cache as replicas).
+   *
+   * @returns 0 on success, 1 on "not done yet", 2 on "forwarding", -errno otherwise.
+   * If it returns 1, the requester associated with this call has been placed
+   * on the appropriate waitlist, and it should unwind itself and back out.
+   * If it returns 2 the request has been forwarded, and again the requester
+   * should unwind itself and back out.
+   */
+  int path_traverse(MDRequestRef& mdr, Message *req, Context *fin, const filepath& path,
+		    vector *pdnvec, CInode **pin, int onfail);
+  bool path_is_mine(filepath& path);
+  bool path_is_mine(string& p) {
+    filepath path(p, 1);
+    return path_is_mine(path);
+  }
+
+  CInode *cache_traverse(const filepath& path);
+
+  void open_remote_dirfrag(CInode *diri, frag_t fg, Context *fin);
+  CInode *get_dentry_inode(CDentry *dn, MDRequestRef& mdr, bool projected=false);
+  void open_remote_ino(inodeno_t ino, Context *fin, bool want_xlocked=false,
+		       inodeno_t hadino=0, version_t hadv=0);
+  void open_remote_ino_2(inodeno_t ino,
+			 vector& anchortrace, bool want_xlocked,
+			 inodeno_t hadino, version_t hadv, Context *onfinish);
+
+  bool parallel_fetch(map& pathmap, set& missing);
+  bool parallel_fetch_traverse_dir(inodeno_t ino, filepath& path, 
+				   set& fetch_queue, set& missing,
+				   C_GatherBuilder &gather_bld);
+
+  void open_remote_dentry(CDentry *dn, bool projected, Context *fin,
+			  bool want_xlocked=false);
+  void _open_remote_dentry_finish(CDentry *dn, inodeno_t ino, Context *fin,
+				  bool want_xlocked, int mode, int r);
+
+  void make_trace(vector& trace, CInode *in);
+
+protected:
+  struct open_ino_info_t {
+    vector ancestors;
+    set checked;
+    int checking;
+    int auth_hint;
+    bool check_peers;
+    bool fetch_backtrace;
+    bool discover;
+    bool want_replica;
+    bool want_xlocked;
+    version_t tid;
+    int64_t pool;
+    list waiters;
+    open_ino_info_t() : checking(-1), auth_hint(-1),
+      check_peers(true), fetch_backtrace(true), discover(false) {}
+  };
+  ceph_tid_t open_ino_last_tid;
+  map opening_inodes;
+
+  void _open_ino_backtrace_fetched(inodeno_t ino, bufferlist& bl, int err);
+  void _open_ino_parent_opened(inodeno_t ino, int ret);
+  void _open_ino_traverse_dir(inodeno_t ino, open_ino_info_t& info, int err);
+  void _open_ino_fetch_dir(inodeno_t ino, MMDSOpenIno *m, CDir *dir);
+  Context* _open_ino_get_waiter(inodeno_t ino, MMDSOpenIno *m);
+  int open_ino_traverse_dir(inodeno_t ino, MMDSOpenIno *m,
+			    vector& ancestors,
+			    bool discover, bool want_xlocked, int *hint);
+  void open_ino_finish(inodeno_t ino, open_ino_info_t& info, int err);
+  void do_open_ino(inodeno_t ino, open_ino_info_t& info, int err);
+  void do_open_ino_peer(inodeno_t ino, open_ino_info_t& info);
+  void handle_open_ino(MMDSOpenIno *m);
+  void handle_open_ino_reply(MMDSOpenInoReply *m);
+  friend class C_MDC_OpenInoBacktraceFetched;
+  friend struct C_MDC_OpenInoTraverseDir;
+  friend struct C_MDC_OpenInoParentOpened;
+
+public:
+  void kick_open_ino_peers(int who);
+  void open_ino(inodeno_t ino, int64_t pool, Context *fin,
+		bool want_replica=true, bool want_xlocked=false);
+  
+  // -- find_ino_peer --
+  struct find_ino_peer_info_t {
+    inodeno_t ino;
+    ceph_tid_t tid;
+    Context *fin;
+    int hint;
+    int checking;
+    set checked;
+
+    find_ino_peer_info_t() : tid(0), fin(NULL), hint(-1), checking(-1) {}
+  };
+
+  map find_ino_peer;
+  ceph_tid_t find_ino_peer_last_tid;
+
+  void find_ino_peers(inodeno_t ino, Context *c, int hint=-1);
+  void _do_find_ino_peer(find_ino_peer_info_t& fip);
+  void handle_find_ino(MMDSFindIno *m);
+  void handle_find_ino_reply(MMDSFindInoReply *m);
+  void kick_find_ino_peers(int who);
+
+  // -- anchors --
+public:
+  void anchor_create_prep_locks(MDRequestRef& mdr, CInode *in, set& rdlocks,
+				set& xlocks);
+  void anchor_create(MDRequestRef& mdr, CInode *in, Context *onfinish);
+  void anchor_destroy(CInode *in, Context *onfinish);
+protected:
+  void _anchor_prepared(CInode *in, version_t atid, bool add);
+  void _anchor_logged(CInode *in, version_t atid, MutationRef& mut);
+  friend class C_MDC_AnchorPrepared;
+  friend class C_MDC_AnchorLogged;
+
+  // -- snaprealms --
+public:
+  void snaprealm_create(MDRequestRef& mdr, CInode *in);
+  void _snaprealm_create_finish(MDRequestRef& mdr, MutationRef& mut, CInode *in);
+
+  // -- stray --
+public:
+  elist delayed_eval_stray;
+
+  void eval_stray(CDentry *dn, bool delay=false);
+  void eval_remote(CDentry *dn);
+
+  void maybe_eval_stray(CInode *in, bool delay=false) {
+    if (in->inode.nlink > 0 || in->is_base())
+      return;
+    CDentry *dn = in->get_projected_parent_dn();
+    if (!dn->state_test(CDentry::STATE_PURGING) &&
+	dn->get_projected_linkage()->is_primary() &&
+	dn->get_dir()->get_inode()->is_stray())
+      eval_stray(dn, delay);
+  }
+protected:
+  void scan_stray_dir(dirfrag_t next=dirfrag_t());
+  void fetch_backtrace(inodeno_t ino, int64_t pool, bufferlist& bl, Context *fin);
+  void purge_stray(CDentry *dn);
+  void _purge_stray_purged(CDentry *dn, int r=0);
+  void _purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls);
+  void _purge_stray_logged_truncate(CDentry *dn, LogSegment *ls);
+  friend struct C_MDC_RetryScanStray;
+  friend class C_MDC_FetchedBacktrace;
+  friend class C_MDC_PurgeStrayLogged;
+  friend class C_MDC_PurgeStrayLoggedTruncate;
+  friend class C_MDC_PurgeStrayPurged;
+  void reintegrate_stray(CDentry *dn, CDentry *rlink);
+  void migrate_stray(CDentry *dn, int dest);
+
+
+  // == messages ==
+ public:
+  void dispatch(Message *m);
+
+ protected:
+  // -- replicas --
+  void handle_discover(MDiscover *dis);
+  void handle_discover_reply(MDiscoverReply *m);
+  friend class C_MDC_Join;
+
+public:
+  void replicate_dir(CDir *dir, int to, bufferlist& bl) {
+    dirfrag_t df = dir->dirfrag();
+    ::encode(df, bl);
+    dir->encode_replica(to, bl);
+  }
+  void replicate_dentry(CDentry *dn, int to, bufferlist& bl) {
+    ::encode(dn->name, bl);
+    ::encode(dn->last, bl);
+    dn->encode_replica(to, bl);
+  }
+  void replicate_inode(CInode *in, int to, bufferlist& bl) {
+    ::encode(in->inode.ino, bl);  // bleh, minor assymetry here
+    ::encode(in->last, bl);
+    in->encode_replica(to, bl);
+  }
+  
+  CDir* add_replica_dir(bufferlist::iterator& p, CInode *diri, int from, list& finished);
+  CDir* forge_replica_dir(CInode *diri, frag_t fg, int from);
+  CDentry *add_replica_dentry(bufferlist::iterator& p, CDir *dir, list& finished);
+  CInode *add_replica_inode(bufferlist::iterator& p, CDentry *dn, list& finished);
+
+  void replicate_stray(CDentry *straydn, int who, bufferlist& bl);
+  CDentry *add_replica_stray(bufferlist &bl, int from);
+
+  // -- namespace --
+public:
+  void send_dentry_link(CDentry *dn);
+  void send_dentry_unlink(CDentry *dn, CDentry *straydn, MDRequestRef& mdr);
+protected:
+  void handle_dentry_link(MDentryLink *m);
+  void handle_dentry_unlink(MDentryUnlink *m);
+
+
+  // -- fragmenting --
+private:
+  struct ufragment {
+    int bits;
+    bool committed;
+    LogSegment *ls;
+    list waiters;
+    list old_frags;
+    bufferlist rollback;
+    ufragment() : bits(0), committed(false), ls(NULL) {}
+  };
+  map uncommitted_fragments;
+
+  struct fragment_info_t {
+    int bits;
+    list dirs;
+    list resultfrags;
+    MDRequestRef mdr;
+    // for deadlock detection
+    bool has_frozen;
+    utime_t last_cum_auth_pins_change;
+    int last_cum_auth_pins;
+    int num_remote_waiters;	// number of remote authpin waiters
+    fragment_info_t() : has_frozen(false), last_cum_auth_pins(0), num_remote_waiters(0) {}
+  };
+  map fragments;
+
+  void adjust_dir_fragments(CInode *diri, frag_t basefrag, int bits,
+			    list& frags, list& waiters, bool replay);
+  void adjust_dir_fragments(CInode *diri,
+			    list& srcfrags,
+			    frag_t basefrag, int bits,
+			    list& resultfrags, 
+			    list& waiters,
+			    bool replay);
+  CDir *force_dir_fragment(CInode *diri, frag_t fg, bool replay=true);
+  void get_force_dirfrag_bound_set(vector& dfs, set& bounds);
+
+  bool can_fragment(CInode *diri, list& dirs);
+  void fragment_freeze_dirs(list& dirs, C_GatherBuilder &gather);
+  void fragment_mark_and_complete(list& dirs);
+  void fragment_frozen(dirfrag_t basedirfrag, int r);
+  void fragment_unmark_unfreeze_dirs(list& dirs);
+  void dispatch_fragment_dir(MDRequestRef& mdr);
+  void _fragment_logged(MDRequestRef& mdr);
+  void _fragment_stored(MDRequestRef& mdr);
+  void _fragment_committed(dirfrag_t f, list& resultfrags);
+  void _fragment_finish(dirfrag_t f, list& resultfrags);
+
+  friend class EFragment;
+  friend class C_MDC_FragmentFrozen;
+  friend class C_MDC_FragmentMarking;
+  friend class C_MDC_FragmentPrep;
+  friend class C_MDC_FragmentStore;
+  friend class C_MDC_FragmentCommit;
+  friend class C_MDC_FragmentFinish;
+
+  void handle_fragment_notify(MMDSFragmentNotify *m);
+
+  void add_uncommitted_fragment(dirfrag_t basedirfrag, int bits, list& old_frag,
+				LogSegment *ls, bufferlist *rollback=NULL);
+  void finish_uncommitted_fragment(dirfrag_t basedirfrag, int op);
+  void rollback_uncommitted_fragment(dirfrag_t basedirfrag, list& old_frags);
+public:
+  void wait_for_uncommitted_fragment(dirfrag_t dirfrag, Context *c) {
+    assert(uncommitted_fragments.count(dirfrag));
+    uncommitted_fragments[dirfrag].waiters.push_back(c);
+  }
+  void split_dir(CDir *dir, int byn);
+  void merge_dir(CInode *diri, frag_t fg);
+  void rollback_uncommitted_fragments();
+
+  void find_stale_fragment_freeze();
+  void fragment_freeze_inc_num_waiters(CDir *dir);
+  bool fragment_are_all_frozen(CDir *dir);
+  int get_num_fragmenting_dirs() { return fragments.size(); }
+
+  // -- updates --
+  //int send_inode_updates(CInode *in);
+  //void handle_inode_update(MInodeUpdate *m);
+
+  int send_dir_updates(CDir *in, bool bcast=false);
+  void handle_dir_update(MDirUpdate *m);
+
+  // -- cache expiration --
+  void handle_cache_expire(MCacheExpire *m);
+  void process_delayed_expire(CDir *dir);
+  void discard_delayed_expire(CDir *dir);
+
+
+  // == crap fns ==
+ public:
+  void show_cache();
+  void dump_cache(const char *fn=0);
+  void show_subtrees(int dbl=10);
+
+  CInode *hack_pick_random_inode() {
+    assert(!inode_map.empty());
+    int n = rand() % inode_map.size();
+    ceph::unordered_map::iterator p = inode_map.begin();
+    while (n--) ++p;
+    return p->second;
+  }
+
+};
+
+class C_MDS_RetryRequest : public Context {
+  MDCache *cache;
+  MDRequestRef mdr;
+ public:
+  C_MDS_RetryRequest(MDCache *c, MDRequestRef& r);
+  virtual void finish(int r);
+};
+
+#endif
diff --git a/ceph/src/mds/MDLog.cc b/ceph/src/mds/MDLog.cc
new file mode 100644
index 00000000..c2246956
--- /dev/null
+++ b/ceph/src/mds/MDLog.cc
@@ -0,0 +1,657 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MDLog.h"
+#include "MDS.h"
+#include "MDCache.h"
+#include "LogEvent.h"
+
+#include "osdc/Journaler.h"
+
+#include "common/entity_name.h"
+#include "common/perf_counters.h"
+
+#include "events/ESubtreeMap.h"
+
+#include "common/config.h"
+#include "common/errno.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef DOUT_COND
+#define DOUT_COND(cct, l) l<=cct->_conf->debug_mds || l <= cct->_conf->debug_mds_log
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".log "
+
+// cons/des
+MDLog::~MDLog()
+{
+  if (journaler) { delete journaler; journaler = 0; }
+  if (logger) {
+    g_ceph_context->get_perfcounters_collection()->remove(logger);
+    delete logger;
+    logger = 0;
+  }
+}
+
+
+void MDLog::create_logger()
+{
+  PerfCountersBuilder plb(g_ceph_context, "mds_log", l_mdl_first, l_mdl_last);
+
+  plb.add_u64_counter(l_mdl_evadd, "evadd");
+  plb.add_u64_counter(l_mdl_evex, "evex");
+  plb.add_u64_counter(l_mdl_evtrm, "evtrm");
+  plb.add_u64(l_mdl_ev, "ev");
+  plb.add_u64(l_mdl_evexg, "evexg");
+  plb.add_u64(l_mdl_evexd, "evexd");
+
+  plb.add_u64_counter(l_mdl_segadd, "segadd");
+  plb.add_u64_counter(l_mdl_segex, "segex");
+  plb.add_u64_counter(l_mdl_segtrm, "segtrm");
+  plb.add_u64(l_mdl_seg, "seg");
+  plb.add_u64(l_mdl_segexg, "segexg");
+  plb.add_u64(l_mdl_segexd, "segexd");
+
+  plb.add_u64(l_mdl_expos, "expos");
+  plb.add_u64(l_mdl_wrpos, "wrpos");
+  plb.add_u64(l_mdl_rdpos, "rdpos");
+  plb.add_u64(l_mdl_jlat, "jlat");
+
+  // logger
+  logger = plb.create_perf_counters();
+  g_ceph_context->get_perfcounters_collection()->add(logger);
+}
+
+void MDLog::init_journaler()
+{
+  // inode
+  ino = MDS_INO_LOG_OFFSET + mds->get_nodeid();
+  
+  // log streamer
+  if (journaler) delete journaler;
+  journaler = new Journaler(ino, mds->mdsmap->get_metadata_pool(), CEPH_FS_ONDISK_MAGIC, mds->objecter,
+			    logger, l_mdl_jlat,
+			    &mds->timer);
+  assert(journaler->is_readonly());
+  journaler->set_write_error_handler(new C_MDL_WriteError(this));
+}
+
+void MDLog::handle_journaler_write_error(int r)
+{
+  if (r == -EBLACKLISTED) {
+    derr << "we have been blacklisted (fenced), respawning..." << dendl;
+    mds->respawn();
+  } else {
+    derr << "unhandled error " << cpp_strerror(r) << ", shutting down..." << dendl;
+    mds->suicide();
+  }
+}
+
+void MDLog::write_head(Context *c) 
+{
+  journaler->write_head(c);
+}
+
+uint64_t MDLog::get_read_pos()
+{
+  return journaler->get_read_pos(); 
+}
+
+uint64_t MDLog::get_write_pos()
+{
+  return journaler->get_write_pos(); 
+}
+
+uint64_t MDLog::get_safe_pos()
+{
+  return journaler->get_write_safe_pos(); 
+}
+
+
+
+void MDLog::create(Context *c)
+{
+  dout(5) << "create empty log" << dendl;
+  init_journaler();
+  journaler->set_writeable();
+  journaler->create(&mds->mdcache->default_log_layout);
+  journaler->write_head(c);
+
+  logger->set(l_mdl_expos, journaler->get_expire_pos());
+  logger->set(l_mdl_wrpos, journaler->get_write_pos());
+}
+
+void MDLog::open(Context *c)
+{
+  dout(5) << "open discovering log bounds" << dendl;
+  init_journaler();
+  journaler->recover(c);
+
+  // either append() or replay() will follow.
+}
+
+void MDLog::append()
+{
+  dout(5) << "append positioning at end and marking writeable" << dendl;
+  journaler->set_read_pos(journaler->get_write_pos());
+  journaler->set_expire_pos(journaler->get_write_pos());
+  
+  journaler->set_writeable();
+
+  logger->set(l_mdl_expos, journaler->get_write_pos());
+}
+
+
+
+// -------------------------------------------------
+
+void MDLog::start_entry(LogEvent *e)
+{
+  assert(cur_event == NULL);
+  cur_event = e;
+  e->set_start_off(get_write_pos());
+}
+
+void MDLog::submit_entry(LogEvent *le, Context *c) 
+{
+  assert(!mds->is_any_replay());
+  assert(le == cur_event);
+  cur_event = NULL;
+
+  if (!g_conf->mds_log) {
+    // hack: log is disabled.
+    if (c) {
+      c->complete(0);
+    }
+    return;
+  }
+
+  // let the event register itself in the segment
+  assert(!segments.empty());
+  le->_segment = segments.rbegin()->second;
+  le->_segment->num_events++;
+  le->update_segment();
+
+  le->set_stamp(ceph_clock_now(g_ceph_context));
+  
+  num_events++;
+  assert(!capped);
+  
+  // encode it, with event type
+  {
+    bufferlist bl;
+    le->encode_with_header(bl);
+
+    dout(5) << "submit_entry " << journaler->get_write_pos() << "~" << bl.length()
+	    << " : " << *le << dendl;
+      
+    // journal it.
+    journaler->append_entry(bl);  // bl is destroyed.
+  }
+
+  le->_segment->end = journaler->get_write_pos();
+
+  if (logger) {
+    logger->inc(l_mdl_evadd);
+    logger->set(l_mdl_ev, num_events);
+    logger->set(l_mdl_wrpos, journaler->get_write_pos());
+  }
+
+  unflushed++;
+
+  if (c)
+    journaler->wait_for_flush(c);
+  
+  // start a new segment?
+  //  FIXME: should this go elsewhere?
+  uint64_t last_seg = get_last_segment_offset();
+  uint64_t period = journaler->get_layout_period();
+  // start a new segment if there are none or if we reach end of last segment
+  if (le->get_type() == EVENT_SUBTREEMAP ||
+      (le->get_type() == EVENT_IMPORTFINISH && mds->is_resolve())) {
+    // avoid infinite loop when ESubtreeMap is very large.
+    // don not insert ESubtreeMap among EImportFinish events that finish
+    // disambiguate imports. Because the ESubtreeMap reflects the subtree
+    // state when all EImportFinish events are replayed.
+  } else if (journaler->get_write_pos()/period != last_seg/period) {
+    dout(10) << "submit_entry also starting new segment: last = " << last_seg
+	     << ", cur pos = " << journaler->get_write_pos() << dendl;
+    start_new_segment();
+  } else if (g_conf->mds_debug_subtrees &&
+	     le->get_type() != EVENT_SUBTREEMAP_TEST) {
+    // debug: journal this every time to catch subtree replay bugs.
+    // use a different event id so it doesn't get interpreted as a
+    // LogSegment boundary on replay.
+    LogEvent *sle = mds->mdcache->create_subtree_map();
+    sle->set_type(EVENT_SUBTREEMAP_TEST);
+    submit_entry(sle);
+  }
+
+  delete le;
+}
+
+void MDLog::wait_for_safe(Context *c)
+{
+  if (g_conf->mds_log) {
+    // wait
+    journaler->wait_for_flush(c);
+  } else {
+    // hack: bypass.
+    c->complete(0);
+  }
+}
+
+void MDLog::flush()
+{
+  if (unflushed)
+    journaler->flush();
+  unflushed = 0;
+}
+
+void MDLog::cap()
+{ 
+  dout(5) << "cap" << dendl;
+  capped = true;
+}
+
+
+// -----------------------------
+// segments
+
+void MDLog::start_new_segment(Context *onsync)
+{
+  prepare_new_segment();
+  journal_segment_subtree_map();
+  if (onsync) {
+    wait_for_safe(onsync);
+    flush();
+  }
+}
+
+void MDLog::prepare_new_segment()
+{
+  dout(7) << __func__ << " at " << journaler->get_write_pos() << dendl;
+
+  segments[journaler->get_write_pos()] = new LogSegment(journaler->get_write_pos());
+
+  logger->inc(l_mdl_segadd);
+  logger->set(l_mdl_seg, segments.size());
+
+  // Adjust to next stray dir
+  dout(10) << "Advancing to next stray directory on mds " << mds->get_nodeid() 
+	   << dendl;
+  mds->mdcache->advance_stray();
+}
+
+void MDLog::journal_segment_subtree_map()
+{
+  dout(7) << __func__ << dendl;
+  submit_entry(mds->mdcache->create_subtree_map());
+}
+
+void MDLog::trim(int m)
+{
+  int max_segments = g_conf->mds_log_max_segments;
+  int max_events = g_conf->mds_log_max_events;
+  if (m >= 0)
+    max_events = m;
+
+  // trim!
+  dout(10) << "trim " 
+	   << segments.size() << " / " << max_segments << " segments, " 
+	   << num_events << " / " << max_events << " events"
+	   << ", " << expiring_segments.size() << " (" << expiring_events << ") expiring"
+	   << ", " << expired_segments.size() << " (" << expired_events << ") expired"
+	   << dendl;
+
+  if (segments.empty())
+    return;
+
+  // hack: only trim for a few seconds at a time
+  utime_t stop = ceph_clock_now(g_ceph_context);
+  stop += 2.0;
+
+  map::iterator p = segments.begin();
+  while (p != segments.end() && 
+	 ((max_events >= 0 &&
+	   num_events - expiring_events - expired_events > max_events) ||
+	  (max_segments >= 0 &&
+	   segments.size() - expiring_segments.size() - expired_segments.size() > (unsigned)max_segments))) {
+    
+    if (stop < ceph_clock_now(g_ceph_context))
+      break;
+
+    int num_expiring_segments = (int)expiring_segments.size();
+    if (num_expiring_segments >= g_conf->mds_log_max_expiring)
+      break;
+
+    int op_prio = CEPH_MSG_PRIO_LOW +
+		  (CEPH_MSG_PRIO_HIGH - CEPH_MSG_PRIO_LOW) *
+		  num_expiring_segments / g_conf->mds_log_max_expiring;
+
+    // look at first segment
+    LogSegment *ls = p->second;
+    assert(ls);
+    ++p;
+    
+    if (ls->end > journaler->get_write_safe_pos()) {
+      dout(5) << "trim segment " << ls->offset << ", not fully flushed yet, safe "
+	      << journaler->get_write_safe_pos() << " < end " << ls->end << dendl;
+      break;
+    }
+    if (expiring_segments.count(ls)) {
+      dout(5) << "trim already expiring segment " << ls->offset << ", " << ls->num_events << " events" << dendl;
+    } else if (expired_segments.count(ls)) {
+      dout(5) << "trim already expired segment " << ls->offset << ", " << ls->num_events << " events" << dendl;
+    } else {
+      try_expire(ls, op_prio);
+    }
+  }
+
+  // discard expired segments
+  _trim_expired_segments();
+}
+
+
+void MDLog::try_expire(LogSegment *ls, int op_prio)
+{
+  C_GatherBuilder gather_bld(g_ceph_context);
+  ls->try_to_expire(mds, gather_bld, op_prio);
+  if (gather_bld.has_subs()) {
+    assert(expiring_segments.count(ls) == 0);
+    expiring_segments.insert(ls);
+    expiring_events += ls->num_events;
+    dout(5) << "try_expire expiring segment " << ls->offset << dendl;
+    gather_bld.set_finisher(new C_MaybeExpiredSegment(this, ls, op_prio));
+    gather_bld.activate();
+  } else {
+    dout(10) << "try_expire expired segment " << ls->offset << dendl;
+    _expired(ls);
+  }
+  
+  logger->set(l_mdl_segexg, expiring_segments.size());
+  logger->set(l_mdl_evexg, expiring_events);
+}
+
+void MDLog::_maybe_expired(LogSegment *ls, int op_prio)
+{
+  dout(10) << "_maybe_expired segment " << ls->offset << " " << ls->num_events << " events" << dendl;
+  assert(expiring_segments.count(ls));
+  expiring_segments.erase(ls);
+  expiring_events -= ls->num_events;
+  try_expire(ls, op_prio);
+}
+
+void MDLog::_trim_expired_segments()
+{
+  // trim expired segments?
+  bool trimmed = false;
+  while (!segments.empty()) {
+    LogSegment *ls = segments.begin()->second;
+    if (!expired_segments.count(ls)) {
+      dout(10) << "_trim_expired_segments waiting for " << ls->offset << " to expire" << dendl;
+      break;
+    }
+    
+    dout(10) << "_trim_expired_segments trimming expired " << ls->offset << dendl;
+    expired_events -= ls->num_events;
+    expired_segments.erase(ls);
+    num_events -= ls->num_events;
+      
+    // this was the oldest segment, adjust expire pos
+    if (journaler->get_expire_pos() < ls->offset)
+      journaler->set_expire_pos(ls->offset);
+    
+    logger->set(l_mdl_expos, ls->offset);
+    logger->inc(l_mdl_segtrm);
+    logger->inc(l_mdl_evtrm, ls->num_events);
+    
+    segments.erase(ls->offset);
+    delete ls;
+    trimmed = true;
+  }
+  
+  if (trimmed)
+    journaler->write_head(0);
+}
+
+void MDLog::_expired(LogSegment *ls)
+{
+  dout(5) << "_expired segment " << ls->offset << " " << ls->num_events << " events" << dendl;
+
+  if (!capped && ls == peek_current_segment()) {
+    dout(5) << "_expired not expiring " << ls->offset << ", last one and !capped" << dendl;
+  } else {
+    // expired.
+    expired_segments.insert(ls);
+    expired_events += ls->num_events;
+    
+    logger->inc(l_mdl_evex, ls->num_events);
+    logger->inc(l_mdl_segex);
+  }
+
+  logger->set(l_mdl_ev, num_events);
+  logger->set(l_mdl_evexd, expired_events);
+  logger->set(l_mdl_seg, segments.size());
+  logger->set(l_mdl_segexd, expired_segments.size());
+}
+
+
+
+void MDLog::replay(Context *c)
+{
+  assert(journaler->is_active());
+  assert(journaler->is_readonly());
+
+  // empty?
+  if (journaler->get_read_pos() == journaler->get_write_pos()) {
+    dout(10) << "replay - journal empty, done." << dendl;
+    if (c) {
+      c->complete(0);
+    }
+    return;
+  }
+
+  // add waiter
+  if (c)
+    waitfor_replay.push_back(c);
+
+  // go!
+  dout(10) << "replay start, from " << journaler->get_read_pos()
+	   << " to " << journaler->get_write_pos() << dendl;
+
+  assert(num_events == 0 || already_replayed);
+  already_replayed = true;
+
+  replay_thread.create();
+  replay_thread.detach();
+}
+
+class C_MDL_Replay : public Context {
+  MDLog *mdlog;
+public:
+  C_MDL_Replay(MDLog *l) : mdlog(l) {}
+  void finish(int r) { 
+    mdlog->replay_cond.Signal();
+  }
+};
+
+
+
+// i am a separate thread
+void MDLog::_replay_thread()
+{
+  mds->mds_lock.Lock();
+  dout(10) << "_replay_thread start" << dendl;
+
+  // loop
+  int r = 0;
+  while (1) {
+    // wait for read?
+    while (!journaler->is_readable() &&
+	   journaler->get_read_pos() < journaler->get_write_pos() &&
+	   !journaler->get_error()) {
+      journaler->wait_for_readable(new C_MDL_Replay(this));
+      replay_cond.Wait(mds->mds_lock);
+    }
+    if (journaler->get_error()) {
+      r = journaler->get_error();
+      dout(0) << "_replay journaler got error " << r << ", aborting" << dendl;
+      if (r == -ENOENT) {
+	// journal has been trimmed by somebody else?
+	assert(journaler->is_readonly());
+	r = -EAGAIN;
+      } else if (r == -EINVAL) {
+        if (journaler->get_read_pos() < journaler->get_expire_pos()) {
+          // this should only happen if you're following somebody else
+          assert(journaler->is_readonly());
+          dout(0) << "expire_pos is higher than read_pos, returning EAGAIN" << dendl;
+          r = -EAGAIN;
+        } else {
+          /* re-read head and check it
+           * Given that replay happens in a separate thread and
+           * the MDS is going to either shut down or restart when
+           * we return this error, doing it synchronously is fine
+           * -- as long as we drop the main mds lock--. */
+          Mutex mylock("MDLog::_replay_thread lock");
+          Cond cond;
+          bool done = false;
+          int err = 0;
+          journaler->reread_head(new C_SafeCond(&mylock, &cond, &done, &err));
+          mds->mds_lock.Unlock();
+	  mylock.Lock();
+          while (!done)
+            cond.Wait(mylock);
+	  mylock.Unlock();
+          if (err) { // well, crap
+            dout(0) << "got error while reading head: " << cpp_strerror(err)
+                    << dendl;
+            mds->suicide();
+          }
+          mds->mds_lock.Lock();
+	  standby_trim_segments();
+          if (journaler->get_read_pos() < journaler->get_expire_pos()) {
+            dout(0) << "expire_pos is higher than read_pos, returning EAGAIN" << dendl;
+            r = -EAGAIN;
+          }
+        }
+      }
+      break;
+    }
+    
+    if (!journaler->is_readable() &&
+	journaler->get_read_pos() == journaler->get_write_pos())
+      break;
+    
+    assert(journaler->is_readable());
+    
+    // read it
+    uint64_t pos = journaler->get_read_pos();
+    bufferlist bl;
+    bool r = journaler->try_read_entry(bl);
+    if (!r && journaler->get_error())
+      continue;
+    assert(r);
+    
+    // unpack event
+    LogEvent *le = LogEvent::decode(bl);
+    if (!le) {
+      dout(0) << "_replay " << pos << "~" << bl.length() << " / " << journaler->get_write_pos() 
+	      << " -- unable to decode event" << dendl;
+      dout(0) << "dump of unknown or corrupt event:\n";
+      bl.hexdump(*_dout);
+      *_dout << dendl;
+
+      assert(!!"corrupt log event" == g_conf->mds_log_skip_corrupt_events);
+      continue;
+    }
+    le->set_start_off(pos);
+
+    // new segment?
+    if (le->get_type() == EVENT_SUBTREEMAP ||
+	le->get_type() == EVENT_RESETJOURNAL) {
+      segments[pos] = new LogSegment(pos);
+      logger->set(l_mdl_seg, segments.size());
+    }
+
+    // have we seen an import map yet?
+    if (segments.empty()) {
+      dout(10) << "_replay " << pos << "~" << bl.length() << " / " << journaler->get_write_pos() 
+	       << " " << le->get_stamp() << " -- waiting for subtree_map.  (skipping " << *le << ")" << dendl;
+    } else {
+      dout(10) << "_replay " << pos << "~" << bl.length() << " / " << journaler->get_write_pos() 
+	       << " " << le->get_stamp() << ": " << *le << dendl;
+      le->_segment = get_current_segment();    // replay may need this
+      le->_segment->num_events++;
+      le->_segment->end = journaler->get_read_pos();
+      num_events++;
+
+      le->replay(mds);
+    }
+    delete le;
+
+    logger->set(l_mdl_rdpos, pos);
+
+    // drop lock for a second, so other events/messages (e.g. beacon timer!) can go off
+    mds->mds_lock.Unlock();
+    mds->mds_lock.Lock();
+  }
+
+  // done!
+  if (r == 0) {
+    assert(journaler->get_read_pos() == journaler->get_write_pos());
+    dout(10) << "_replay - complete, " << num_events
+	     << " events" << dendl;
+
+    logger->set(l_mdl_expos, journaler->get_expire_pos());
+  }
+
+  dout(10) << "_replay_thread kicking waiters" << dendl;
+  finish_contexts(g_ceph_context, waitfor_replay, r);  
+
+  dout(10) << "_replay_thread finish" << dendl;
+  mds->mds_lock.Unlock();
+}
+
+void MDLog::standby_trim_segments()
+{
+  dout(10) << "standby_trim_segments" << dendl;
+  uint64_t expire_pos = journaler->get_expire_pos();
+  dout(10) << " expire_pos=" << expire_pos << dendl;
+  bool removed_segment = false;
+  while (have_any_segments()) {
+    LogSegment *seg = get_oldest_segment();
+    if (seg->end > expire_pos)
+      break;
+    dout(10) << " removing segment " << seg->offset << dendl;
+    seg->dirty_dirfrags.clear_list();
+    seg->new_dirfrags.clear_list();
+    seg->dirty_inodes.clear_list();
+    seg->dirty_dentries.clear_list();
+    seg->open_files.clear_list();
+    seg->dirty_parent_inodes.clear_list();
+    seg->dirty_dirfrag_dir.clear_list();
+    seg->dirty_dirfrag_nest.clear_list();
+    seg->dirty_dirfrag_dirfragtree.clear_list();
+    remove_oldest_segment();
+    removed_segment = true;
+  }
+
+  if (removed_segment) {
+    dout(20) << " calling mdcache->trim!" << dendl;
+    mds->mdcache->trim(-1);
+  } else
+    dout(20) << " removed no segments!" << dendl;
+}
diff --git a/ceph/src/mds/MDLog.h b/ceph/src/mds/MDLog.h
new file mode 100644
index 00000000..6e8e980c
--- /dev/null
+++ b/ceph/src/mds/MDLog.h
@@ -0,0 +1,250 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDLOG_H
+#define CEPH_MDLOG_H
+
+enum {
+  l_mdl_first = 5000,
+  l_mdl_evadd,
+  l_mdl_evex,
+  l_mdl_evtrm,
+  l_mdl_ev,
+  l_mdl_evexg,
+  l_mdl_evexd,
+  l_mdl_segadd,
+  l_mdl_segex,
+  l_mdl_segtrm,
+  l_mdl_seg,
+  l_mdl_segexg,
+  l_mdl_segexd,
+  l_mdl_expos,
+  l_mdl_wrpos,
+  l_mdl_rdpos,
+  l_mdl_jlat,
+  l_mdl_last,
+};
+
+#include "include/types.h"
+#include "include/Context.h"
+
+#include "common/Thread.h"
+#include "common/Cond.h"
+
+#include "LogSegment.h"
+
+#include 
+
+class Journaler;
+class LogEvent;
+class MDS;
+class LogSegment;
+class ESubtreeMap;
+
+class PerfCounters;
+
+#include 
+using std::map;
+
+
+class MDLog {
+public:
+  MDS *mds;
+protected:
+  int num_events; // in events
+
+  int unflushed;
+
+  bool capped;
+
+  inodeno_t ino;
+  Journaler *journaler;
+
+  PerfCounters *logger;
+
+
+  // -- replay --
+  Cond replay_cond;
+
+  class ReplayThread : public Thread {
+    MDLog *log;
+  public:
+    ReplayThread(MDLog *l) : log(l) {}
+    void* entry() {
+      log->_replay_thread();
+      return 0;
+    }
+  } replay_thread;
+  bool already_replayed;
+
+  friend class ReplayThread;
+  friend class C_MDL_Replay;
+
+  list waitfor_replay;
+
+  void _replay();         // old way
+  void _replay_thread();  // new way
+
+
+  // -- segments --
+  map segments;
+  set expiring_segments;
+  set expired_segments;
+  int expiring_events;
+  int expired_events;
+
+  // -- subtreemaps --
+  friend class ESubtreeMap;
+  friend class C_MDS_WroteImportMap;
+  friend class MDCache;
+
+public:
+  uint64_t get_last_segment_offset() {
+    assert(!segments.empty());
+    return segments.rbegin()->first;
+  }
+  LogSegment *get_oldest_segment() {
+    return segments.begin()->second;
+  }
+  void remove_oldest_segment() {
+    map::iterator p = segments.begin();
+    delete p->second;
+    segments.erase(p);
+  }
+
+
+private:
+  void init_journaler();
+
+  struct C_MDL_WriteError : public Context {
+    MDLog *mdlog;
+    C_MDL_WriteError(MDLog *m) : mdlog(m) {}
+    void finish(int r) {
+      mdlog->handle_journaler_write_error(r);
+    }
+  };
+  void handle_journaler_write_error(int r);
+ 
+public:
+  void create_logger();
+  
+  // replay state
+  map >   pending_exports;
+
+
+
+public:
+  MDLog(MDS *m) : mds(m),
+		  num_events(0), 
+		  unflushed(0),
+		  capped(false),
+		  journaler(0),
+		  logger(0),
+		  replay_thread(this),
+		  already_replayed(false),
+		  expiring_events(0), expired_events(0),
+		  cur_event(NULL) { }		  
+  ~MDLog();
+
+
+  // -- segments --
+  void start_new_segment(Context *onsync=0);
+  void prepare_new_segment();
+  void journal_segment_subtree_map();
+
+  LogSegment *peek_current_segment() {
+    return segments.empty() ? NULL : segments.rbegin()->second;
+  }
+
+  LogSegment *get_current_segment() { 
+    assert(!segments.empty());
+    return segments.rbegin()->second;
+  }
+
+  LogSegment *get_segment(uint64_t off) {
+    if (segments.count(off))
+      return segments[off];
+    return NULL;
+  }
+
+  bool have_any_segments() {
+    return !segments.empty();
+  }
+
+  void flush_logger();
+
+  size_t get_num_events() { return num_events; }
+  size_t get_num_segments() { return segments.size(); }  
+
+  uint64_t get_read_pos();
+  uint64_t get_write_pos();
+  uint64_t get_safe_pos();
+  Journaler *get_journaler() { return journaler; }
+  bool empty() { return segments.empty(); }
+
+  bool is_capped() { return capped; }
+  void cap();
+
+  // -- events --
+private:
+  LogEvent *cur_event;
+public:
+  void start_entry(LogEvent *e);
+  void submit_entry(LogEvent *e, Context *c = 0);
+  void start_submit_entry(LogEvent *e, Context *c = 0) {
+    start_entry(e);
+    submit_entry(e, c);
+  }
+  bool entry_is_open() { return cur_event != NULL; }
+
+  void wait_for_safe( Context *c );
+  void flush();
+  bool is_flushed() {
+    return unflushed == 0;
+  }
+
+private:
+  class C_MaybeExpiredSegment : public Context {
+    MDLog *mdlog;
+    LogSegment *ls;
+    int op_prio;
+  public:
+    C_MaybeExpiredSegment(MDLog *mdl, LogSegment *s, int p) : mdlog(mdl), ls(s), op_prio(p) {}
+    void finish(int res) {
+      mdlog->_maybe_expired(ls, op_prio);
+    }
+  };
+
+  void try_expire(LogSegment *ls, int op_prio);
+  void _maybe_expired(LogSegment *ls, int op_prio);
+  void _expired(LogSegment *ls);
+  void _trim_expired_segments();
+
+public:
+  void trim(int max=-1);
+
+private:
+  void write_head(Context *onfinish);
+
+public:
+  void create(Context *onfinish);  // fresh, empty log! 
+  void open(Context *onopen);      // append() or replay() to follow!
+  void append();
+  void replay(Context *onfinish);
+
+  void standby_trim_segments();
+};
+
+#endif
diff --git a/ceph/src/mds/MDS.cc b/ceph/src/mds/MDS.cc
new file mode 100644
index 00000000..4509cea2
--- /dev/null
+++ b/ceph/src/mds/MDS.cc
@@ -0,0 +1,2263 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+
+#include "global/signal_handler.h"
+
+#include "include/types.h"
+#include "common/entity_name.h"
+#include "common/Clock.h"
+#include "common/signal.h"
+#include "common/ceph_argparse.h"
+#include "common/errno.h"
+
+
+#include "msg/Messenger.h"
+#include "mon/MonClient.h"
+
+#include "osd/OSDMap.h"
+#include "osdc/Objecter.h"
+#include "osdc/Filer.h"
+#include "osdc/Journaler.h"
+
+#include "MDSMap.h"
+
+#include "MDS.h"
+#include "Server.h"
+#include "Locker.h"
+#include "MDCache.h"
+#include "MDLog.h"
+#include "MDBalancer.h"
+#include "Migrator.h"
+
+#include "AnchorServer.h"
+#include "AnchorClient.h"
+#include "SnapServer.h"
+#include "SnapClient.h"
+
+#include "InoTable.h"
+
+#include "common/perf_counters.h"
+
+#include "common/Timer.h"
+
+#include "events/ESession.h"
+
+#include "messages/MMDSMap.h"
+#include "messages/MMDSBeacon.h"
+
+#include "messages/MGenericMessage.h"
+
+#include "messages/MOSDMap.h"
+
+#include "messages/MClientRequest.h"
+#include "messages/MClientRequestForward.h"
+
+#include "messages/MMDSTableRequest.h"
+
+#include "messages/MMonCommand.h"
+
+#include "auth/AuthAuthorizeHandler.h"
+#include "auth/KeyRing.h"
+
+#include "common/config.h"
+
+#include "perfglue/cpu_profiler.h"
+#include "perfglue/heap_profiler.h"
+
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << whoami << '.' << incarnation << ' '
+
+
+
+// cons/des
+MDS::MDS(const std::string &n, Messenger *m, MonClient *mc) : 
+  Dispatcher(m->cct),
+  mds_lock("MDS::mds_lock"),
+  timer(m->cct, mds_lock),
+  authorize_handler_cluster_registry(new AuthAuthorizeHandlerRegistry(m->cct,
+								      m->cct->_conf->auth_supported.length() ?
+								      m->cct->_conf->auth_supported :
+								      m->cct->_conf->auth_cluster_required)),
+  authorize_handler_service_registry(new AuthAuthorizeHandlerRegistry(m->cct,
+								      m->cct->_conf->auth_supported.length() ?
+								      m->cct->_conf->auth_supported :
+								      m->cct->_conf->auth_service_required)),
+  name(n),
+  whoami(-1), incarnation(0),
+  standby_for_rank(MDSMap::MDS_NO_STANDBY_PREF),
+  standby_type(0),
+  standby_replaying(false),
+  messenger(m),
+  monc(mc),
+  clog(m->cct, messenger, &mc->monmap, LogClient::NO_FLAGS),
+  sessionmap(this) {
+
+  orig_argc = 0;
+  orig_argv = NULL;
+
+  last_tid = 0;
+
+  monc->set_messenger(messenger);
+
+  mdsmap = new MDSMap;
+  osdmap = new OSDMap;
+
+  objecter = new Objecter(m->cct, messenger, monc, osdmap, mds_lock, timer,
+			  0, 0);
+  objecter->unset_honor_osdmap_full();
+
+  filer = new Filer(objecter);
+
+  mdcache = new MDCache(this);
+  mdlog = new MDLog(this);
+  balancer = new MDBalancer(this);
+
+  inotable = new InoTable(this);
+  snapserver = new SnapServer(this);
+  snapclient = new SnapClient(this);
+  anchorserver = new AnchorServer(this);
+  anchorclient = new AnchorClient(this);
+
+  server = new Server(this);
+  locker = new Locker(this, mdcache);
+
+  dispatch_depth = 0;
+
+  // clients
+  last_client_mdsmap_bcast = 0;
+  
+  // beacon
+  beacon_last_seq = 0;
+  beacon_sender = 0;
+  was_laggy = false;
+
+  // tick
+  tick_event = 0;
+
+  req_rate = 0;
+
+  last_state = want_state = state = MDSMap::STATE_BOOT;
+
+  logger = 0;
+  mlogger = 0;
+}
+
+MDS::~MDS() {
+  Mutex::Locker lock(mds_lock);
+
+  delete authorize_handler_service_registry;
+  delete authorize_handler_cluster_registry;
+
+  if (mdcache) { delete mdcache; mdcache = NULL; }
+  if (mdlog) { delete mdlog; mdlog = NULL; }
+  if (balancer) { delete balancer; balancer = NULL; }
+  if (inotable) { delete inotable; inotable = NULL; }
+  if (anchorserver) { delete anchorserver; anchorserver = NULL; }
+  if (snapserver) { delete snapserver; snapserver = NULL; }
+  if (snapclient) { delete snapclient; snapclient = NULL; }
+  if (anchorclient) { delete anchorclient; anchorclient = NULL; }
+  if (osdmap) { delete osdmap; osdmap = 0; }
+  if (mdsmap) { delete mdsmap; mdsmap = 0; }
+
+  if (server) { delete server; server = 0; }
+  if (locker) { delete locker; locker = 0; }
+
+  if (filer) { delete filer; filer = 0; }
+  if (objecter) { delete objecter; objecter = 0; }
+
+  if (logger) {
+    g_ceph_context->get_perfcounters_collection()->remove(logger);
+    delete logger;
+    logger = 0;
+  }
+  if (mlogger) {
+    g_ceph_context->get_perfcounters_collection()->remove(mlogger);
+    delete mlogger;
+    mlogger = 0;
+  }
+  
+  if (messenger)
+    delete messenger;
+}
+
+void MDS::create_logger()
+{
+  dout(10) << "create_logger" << dendl;
+  {
+    PerfCountersBuilder mds_plb(g_ceph_context, "mds", l_mds_first, l_mds_last);
+
+    mds_plb.add_u64_counter(l_mds_req, "req"); // FIXME: nobody is actually setting this
+    mds_plb.add_u64_counter(l_mds_reply, "reply");
+    mds_plb.add_time_avg(l_mds_replyl, "replyl");
+    mds_plb.add_u64_counter(l_mds_fw, "fw");
+    
+    mds_plb.add_u64_counter(l_mds_dir_f, "dir_f");
+    mds_plb.add_u64_counter(l_mds_dir_c, "dir_c");
+    mds_plb.add_u64_counter(l_mds_dir_sp, "dir_sp");
+    mds_plb.add_u64_counter(l_mds_dir_ffc, "dir_ffc");
+    //mds_plb.add_u64_counter("mkdir");
+
+    /*
+    mds_plb.add_u64_counter("newin"); // new inodes (pre)loaded
+    mds_plb.add_u64_counter("newt");  // inodes first touched/used
+    mds_plb.add_u64_counter("outt");  // trimmed touched
+    mds_plb.add_u64_counter("outut"); // trimmed untouched (wasted effort)
+    mds_plb.add_fl_avg("oututl"); // avg trim latency for untouched
+
+    mds_plb.add_u64_counter("dirt1");
+    mds_plb.add_u64_counter("dirt2");
+    mds_plb.add_u64_counter("dirt3");
+    mds_plb.add_u64_counter("dirt4");
+    mds_plb.add_u64_counter("dirt5");
+    */
+
+    mds_plb.add_u64(l_mds_imax, "imax");
+    mds_plb.add_u64(l_mds_i, "i");
+    mds_plb.add_u64(l_mds_itop, "itop");
+    mds_plb.add_u64(l_mds_ibot, "ibot");
+    mds_plb.add_u64(l_mds_iptail, "iptail");  
+    mds_plb.add_u64(l_mds_ipin, "ipin");
+    mds_plb.add_u64_counter(l_mds_iex, "iex");
+    mds_plb.add_u64_counter(l_mds_icap, "icap");
+    mds_plb.add_u64_counter(l_mds_cap, "cap");
+    
+    mds_plb.add_u64_counter(l_mds_dis, "dis"); // FIXME: unused
+
+    mds_plb.add_u64_counter(l_mds_t, "t"); 
+    mds_plb.add_u64_counter(l_mds_thit, "thit");
+    mds_plb.add_u64_counter(l_mds_tfw, "tfw");
+    mds_plb.add_u64_counter(l_mds_tdis, "tdis");
+    mds_plb.add_u64_counter(l_mds_tdirf, "tdirf");
+    mds_plb.add_u64_counter(l_mds_trino, "trino");
+    mds_plb.add_u64_counter(l_mds_tlock, "tlock");
+    
+    mds_plb.add_u64(l_mds_l, "l");
+    mds_plb.add_u64(l_mds_q, "q");
+    mds_plb.add_u64(l_mds_popanyd, "popanyd"); // FIXME: unused
+    mds_plb.add_u64(l_mds_popnest, "popnest");
+    
+    mds_plb.add_u64(l_mds_sm, "sm");
+    mds_plb.add_u64_counter(l_mds_ex, "ex");
+    mds_plb.add_u64_counter(l_mds_iexp, "iexp");
+    mds_plb.add_u64_counter(l_mds_im, "im");
+    mds_plb.add_u64_counter(l_mds_iim, "iim");
+    logger = mds_plb.create_perf_counters();
+    g_ceph_context->get_perfcounters_collection()->add(logger);
+  }
+
+  {
+    PerfCountersBuilder mdm_plb(g_ceph_context, "mds_mem", l_mdm_first, l_mdm_last);
+    mdm_plb.add_u64(l_mdm_ino, "ino");
+    mdm_plb.add_u64_counter(l_mdm_inoa, "ino+");
+    mdm_plb.add_u64_counter(l_mdm_inos, "ino-");
+    mdm_plb.add_u64(l_mdm_dir, "dir");
+    mdm_plb.add_u64_counter(l_mdm_dira, "dir+");
+    mdm_plb.add_u64_counter(l_mdm_dirs, "dir-");
+    mdm_plb.add_u64(l_mdm_dn, "dn");
+    mdm_plb.add_u64_counter(l_mdm_dna, "dn+");
+    mdm_plb.add_u64_counter(l_mdm_dns, "dn-");
+    mdm_plb.add_u64(l_mdm_cap, "cap");
+    mdm_plb.add_u64_counter(l_mdm_capa, "cap+");
+    mdm_plb.add_u64_counter(l_mdm_caps, "cap-");
+    mdm_plb.add_u64(l_mdm_rss, "rss");
+    mdm_plb.add_u64(l_mdm_heap, "heap");
+    mdm_plb.add_u64(l_mdm_malloc, "malloc");
+    mdm_plb.add_u64(l_mdm_buf, "buf");
+    mlogger = mdm_plb.create_perf_counters();
+    g_ceph_context->get_perfcounters_collection()->add(mlogger);
+  }
+
+  mdlog->create_logger();
+  server->create_logger();
+}
+
+
+
+MDSTableClient *MDS::get_table_client(int t)
+{
+  switch (t) {
+  case TABLE_ANCHOR: return anchorclient;
+  case TABLE_SNAP: return snapclient;
+  default: assert(0);
+  }
+}
+
+MDSTableServer *MDS::get_table_server(int t)
+{
+  switch (t) {
+  case TABLE_ANCHOR: return anchorserver;
+  case TABLE_SNAP: return snapserver;
+  default: assert(0);
+  }
+}
+
+
+
+
+
+
+
+
+void MDS::send_message(Message *m, Connection *c)
+{ 
+  assert(c);
+  messenger->send_message(m, c);
+}
+
+
+void MDS::send_message_mds(Message *m, int mds)
+{
+  if (!mdsmap->is_up(mds)) {
+    dout(10) << "send_message_mds mds." << mds << " not up, dropping " << *m << dendl;
+    m->put();
+    return;
+  }
+
+  // send mdsmap first?
+  if (mds != whoami && peer_mdsmap_epoch[mds] < mdsmap->get_epoch()) {
+    messenger->send_message(new MMDSMap(monc->get_fsid(), mdsmap), 
+			    mdsmap->get_inst(mds));
+    peer_mdsmap_epoch[mds] = mdsmap->get_epoch();
+  }
+
+  // send message
+  messenger->send_message(m, mdsmap->get_inst(mds));
+}
+
+void MDS::forward_message_mds(Message *m, int mds)
+{
+  assert(mds != whoami);
+
+  // client request?
+  if (m->get_type() == CEPH_MSG_CLIENT_REQUEST &&
+      (static_cast(m))->get_source().is_client()) {
+    MClientRequest *creq = static_cast(m);
+    creq->inc_num_fwd();    // inc forward counter
+
+    /*
+     * don't actually forward if non-idempotent!
+     * client has to do it.  although the MDS will ignore duplicate requests,
+     * the affected metadata may migrate, in which case the new authority
+     * won't have the metareq_id in the completed request map.
+     */
+    // NEW: always make the client resend!  
+    bool client_must_resend = true;  //!creq->can_forward();
+
+    // tell the client where it should go
+    messenger->send_message(new MClientRequestForward(creq->get_tid(), mds, creq->get_num_fwd(),
+						      client_must_resend),
+			    creq->get_source_inst());
+    
+    if (client_must_resend) {
+      m->put();
+      return; 
+    }
+  }
+
+  // these are the only types of messages we should be 'forwarding'; they
+  // explicitly encode their source mds, which gets clobbered when we resend
+  // them here.
+  assert(m->get_type() == MSG_MDS_DIRUPDATE ||
+	 m->get_type() == MSG_MDS_EXPORTDIRDISCOVER);
+
+  // send mdsmap first?
+  if (peer_mdsmap_epoch[mds] < mdsmap->get_epoch()) {
+    messenger->send_message(new MMDSMap(monc->get_fsid(), mdsmap), 
+			    mdsmap->get_inst(mds));
+    peer_mdsmap_epoch[mds] = mdsmap->get_epoch();
+  }
+
+  messenger->send_message(m, mdsmap->get_inst(mds));
+}
+
+
+
+void MDS::send_message_client_counted(Message *m, client_t client)
+{
+  Session *session =  sessionmap.get_session(entity_name_t::CLIENT(client.v));
+  if (session) {
+    send_message_client_counted(m, session);
+  } else {
+    dout(10) << "send_message_client_counted no session for client." << client << " " << *m << dendl;
+  }
+}
+
+void MDS::send_message_client_counted(Message *m, Connection *connection)
+{
+  Session *session = static_cast(connection->get_priv());
+  if (session) {
+    session->put();  // do not carry ref
+    send_message_client_counted(m, session);
+  } else {
+    dout(10) << "send_message_client_counted has no session for " << m->get_source_inst() << dendl;
+    // another Connection took over the Session
+  }
+}
+
+void MDS::send_message_client_counted(Message *m, Session *session)
+{
+  version_t seq = session->inc_push_seq();
+  dout(10) << "send_message_client_counted " << session->info.inst.name << " seq "
+	   << seq << " " << *m << dendl;
+  if (session->connection) {
+    messenger->send_message(m, session->connection);
+  } else {
+    session->preopen_out_queue.push_back(m);
+  }
+}
+
+void MDS::send_message_client(Message *m, Session *session)
+{
+  dout(10) << "send_message_client " << session->info.inst << " " << *m << dendl;
+ if (session->connection) {
+    messenger->send_message(m, session->connection);
+  } else {
+    session->preopen_out_queue.push_back(m);
+  }
+}
+
+int MDS::init(int wanted_state)
+{
+  dout(10) << sizeof(MDSCacheObject) << "\tMDSCacheObject" << dendl;
+  dout(10) << sizeof(CInode) << "\tCInode" << dendl;
+  dout(10) << sizeof(elist::item) << "\t elist<>::item   *7=" << 7*sizeof(elist::item) << dendl;
+  dout(10) << sizeof(inode_t) << "\t inode_t " << dendl;
+  dout(10) << sizeof(nest_info_t) << "\t  nest_info_t " << dendl;
+  dout(10) << sizeof(frag_info_t) << "\t  frag_info_t " << dendl;
+  dout(10) << sizeof(SimpleLock) << "\t SimpleLock   *5=" << 5*sizeof(SimpleLock) << dendl;
+  dout(10) << sizeof(ScatterLock) << "\t ScatterLock  *3=" << 3*sizeof(ScatterLock) << dendl;
+  dout(10) << sizeof(CDentry) << "\tCDentry" << dendl;
+  dout(10) << sizeof(elist::item) << "\t elist<>::item" << dendl;
+  dout(10) << sizeof(SimpleLock) << "\t SimpleLock" << dendl;
+  dout(10) << sizeof(CDir) << "\tCDir " << dendl;
+  dout(10) << sizeof(elist::item) << "\t elist<>::item   *2=" << 2*sizeof(elist::item) << dendl;
+  dout(10) << sizeof(fnode_t) << "\t fnode_t " << dendl;
+  dout(10) << sizeof(nest_info_t) << "\t  nest_info_t *2" << dendl;
+  dout(10) << sizeof(frag_info_t) << "\t  frag_info_t *2" << dendl;
+  dout(10) << sizeof(Capability) << "\tCapability " << dendl;
+  dout(10) << sizeof(xlist::item) << "\t xlist<>::item   *2=" << 2*sizeof(xlist::item) << dendl;
+
+  messenger->add_dispatcher_tail(this);
+
+  // get monmap
+  monc->set_messenger(messenger);
+
+  monc->set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD | CEPH_ENTITY_TYPE_MDS);
+  monc->init();
+
+  // tell monc about log_client so it will know about mon session resets
+  monc->set_log_client(&clog);
+  
+  int r = monc->authenticate();
+  if (r < 0) {
+    derr << "ERROR: failed to authenticate: " << cpp_strerror(-r) << dendl;
+    mds_lock.Lock();
+    suicide();
+    mds_lock.Unlock();
+    return r;
+  }
+  while (monc->wait_auth_rotating(30.0) < 0) {
+    derr << "unable to obtain rotating service keys; retrying" << dendl;
+  }
+  objecter->init_unlocked();
+
+  mds_lock.Lock();
+  if (want_state == CEPH_MDS_STATE_DNE) {
+    mds_lock.Unlock();
+    return 0;
+  }
+
+  objecter->init_locked();
+
+  monc->sub_want("mdsmap", 0, 0);
+  monc->renew_subs();
+
+  mds_lock.Unlock();
+
+  // verify that osds support tmap2omap
+  while (true) {
+    objecter->maybe_request_map();
+    objecter->wait_for_osd_map();
+    uint64_t osd_features = objecter->osdmap->get_up_osd_features();
+    if (osd_features & CEPH_FEATURE_OSD_TMAP2OMAP)
+      break;
+    if (objecter->osdmap->get_num_up_osds() > 0) {
+        derr << "*** one or more OSDs do not support TMAP2OMAP; upgrade OSDs before starting MDS (or downgrade MDS) ***" << dendl;
+    } else {
+        derr << "*** no OSDs are up as of epoch " << objecter->osdmap->get_epoch() << ", waiting" << dendl;
+    }
+    sleep(10);
+  }
+
+  mds_lock.Lock();
+  if (want_state == CEPH_MDS_STATE_DNE) {
+    suicide();  // we could do something more graceful here
+  }
+
+  timer.init();
+
+  if (wanted_state==MDSMap::STATE_BOOT && g_conf->mds_standby_replay)
+    wanted_state = MDSMap::STATE_STANDBY_REPLAY;
+  // starting beacon.  this will induce an MDSMap from the monitor
+  want_state = wanted_state;
+  if (wanted_state==MDSMap::STATE_STANDBY_REPLAY ||
+      wanted_state==MDSMap::STATE_ONESHOT_REPLAY) {
+    g_conf->set_val_or_die("mds_standby_replay", "true");
+    g_conf->apply_changes(NULL);
+    if ( wanted_state == MDSMap::STATE_ONESHOT_REPLAY &&
+        (g_conf->mds_standby_for_rank == -1) &&
+        g_conf->mds_standby_for_name.empty()) {
+      // uh-oh, must specify one or the other!
+      dout(0) << "Specified oneshot replay mode but not an MDS!" << dendl;
+      suicide();
+    }
+    want_state = MDSMap::STATE_BOOT;
+    standby_type = wanted_state;
+  }
+
+  standby_for_rank = g_conf->mds_standby_for_rank;
+  standby_for_name.assign(g_conf->mds_standby_for_name);
+
+  if (wanted_state == MDSMap::STATE_STANDBY_REPLAY &&
+      standby_for_rank == -1) {
+    if (standby_for_name.empty())
+      standby_for_rank = MDSMap::MDS_STANDBY_ANY;
+    else
+      standby_for_rank = MDSMap::MDS_STANDBY_NAME;
+  } else if (!standby_type && !standby_for_name.empty())
+    standby_for_rank = MDSMap::MDS_MATCHED_ACTIVE;
+
+  beacon_start();
+  whoami = -1;
+  messenger->set_myname(entity_name_t::MDS(whoami));
+  
+  // schedule tick
+  reset_tick();
+
+  create_logger();
+
+  mds_lock.Unlock();
+
+  return 0;
+}
+
+void MDS::reset_tick()
+{
+  // cancel old
+  if (tick_event) timer.cancel_event(tick_event);
+
+  // schedule
+  tick_event = new C_MDS_Tick(this);
+  timer.add_event_after(g_conf->mds_tick_interval, tick_event);
+}
+
+void MDS::tick()
+{
+  tick_event = 0;
+
+  // reschedule
+  reset_tick();
+
+  if (is_laggy()) {
+    dout(5) << "tick bailing out since we seem laggy" << dendl;
+    return;
+  }
+
+  // make sure mds log flushes, trims periodically
+  mdlog->flush();
+
+  if (is_active() || is_stopping()) {
+    mdcache->trim();
+    mdcache->trim_client_leases();
+    mdcache->check_memory_usage();
+    mdlog->trim();  // NOT during recovery!
+  }
+
+  // log
+  utime_t now = ceph_clock_now(g_ceph_context);
+  mds_load_t load = balancer->get_load(now);
+  
+  if (logger) {
+    req_rate = logger->get(l_mds_req);
+    
+    logger->set(l_mds_l, 100 * load.mds_load());
+    logger->set(l_mds_q, messenger->get_dispatch_queue_len());
+    logger->set(l_mds_sm, mdcache->num_subtrees());
+
+    mdcache->log_stat();
+  }
+
+  // ...
+  if (is_clientreplay() || is_active() || is_stopping()) {
+    locker->scatter_tick();
+    server->find_idle_sessions();
+  }
+  
+  if (is_reconnect())
+    server->reconnect_tick();
+  
+  if (is_active()) {
+    balancer->tick();
+    mdcache->find_stale_fragment_freeze();
+    mdcache->migrator->find_stale_export_freeze();
+    if (snapserver)
+      snapserver->check_osd_map(false);
+  }
+}
+
+
+
+
+// -----------------------
+// beacons
+
+void MDS::beacon_start()
+{
+  beacon_send();         // send first beacon
+}
+  
+
+
+void MDS::beacon_send()
+{
+  ++beacon_last_seq;
+  dout(10) << "beacon_send " << ceph_mds_state_name(want_state)
+	   << " seq " << beacon_last_seq
+	   << " (currently " << ceph_mds_state_name(state) << ")"
+	   << dendl;
+
+  beacon_seq_stamp[beacon_last_seq] = ceph_clock_now(g_ceph_context);
+  
+  MMDSBeacon *beacon = new MMDSBeacon(monc->get_fsid(), monc->get_global_id(), name, mdsmap->get_epoch(), 
+				      want_state, beacon_last_seq);
+  beacon->set_standby_for_rank(standby_for_rank);
+  beacon->set_standby_for_name(standby_for_name);
+
+  // include _my_ feature set
+  CompatSet mdsmap_compat(get_mdsmap_compat_set_default());
+  mdsmap_compat.merge(mdsmap->compat);
+  beacon->set_compat(mdsmap_compat);
+
+  monc->send_mon_message(beacon);
+
+  // schedule next sender
+  if (beacon_sender) timer.cancel_event(beacon_sender);
+  beacon_sender = new C_MDS_BeaconSender(this);
+  timer.add_event_after(g_conf->mds_beacon_interval, beacon_sender);
+}
+
+
+bool MDS::is_laggy()
+{
+  if (beacon_last_acked_stamp == utime_t())
+    return false;
+
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t since = now - beacon_last_acked_stamp;
+  if (since > g_conf->mds_beacon_grace) {
+    dout(5) << "is_laggy " << since << " > " << g_conf->mds_beacon_grace
+	    << " since last acked beacon" << dendl;
+    was_laggy = true;
+    if (since > (g_conf->mds_beacon_grace*2)) {
+      // maybe it's not us?
+      dout(5) << "initiating monitor reconnect; maybe we're not the slow one"
+              << dendl;
+      monc->reopen_session();
+    }
+    return true;
+  }
+  return false;
+}
+
+
+/* This fuction puts the passed message before returning */
+void MDS::handle_mds_beacon(MMDSBeacon *m)
+{
+  version_t seq = m->get_seq();
+
+  // update lab
+  if (beacon_seq_stamp.count(seq)) {
+    assert(beacon_seq_stamp[seq] > beacon_last_acked_stamp);
+    beacon_last_acked_stamp = beacon_seq_stamp[seq];
+    utime_t now = ceph_clock_now(g_ceph_context);
+    utime_t rtt = now - beacon_last_acked_stamp;
+
+    dout(10) << "handle_mds_beacon " << ceph_mds_state_name(m->get_state())
+	     << " seq " << m->get_seq() 
+	     << " rtt " << rtt << dendl;
+
+    if (was_laggy && rtt < g_conf->mds_beacon_grace) {
+      dout(0) << "handle_mds_beacon no longer laggy" << dendl;
+      was_laggy = false;
+      laggy_until = now;
+    }
+
+    // clean up seq_stamp map
+    while (!beacon_seq_stamp.empty() &&
+	   beacon_seq_stamp.begin()->first <= seq)
+      beacon_seq_stamp.erase(beacon_seq_stamp.begin());
+  } else {
+    dout(10) << "handle_mds_beacon " << ceph_mds_state_name(m->get_state())
+	     << " seq " << m->get_seq() << " dne" << dendl;
+  }
+
+  m->put();
+}
+
+void MDS::request_osdmap(Context *c) {
+  objecter->wait_for_new_map(c, osdmap->get_epoch());
+}
+
+/* This function DOES put the passed message before returning*/
+void MDS::handle_command(MMonCommand *m)
+{
+  dout(10) << "handle_command args: " << m->cmd << dendl;
+  if (m->cmd[0] == "injectargs") {
+    if (m->cmd.size() < 2) {
+      derr << "Ignoring empty injectargs!" << dendl;
+    }
+    else {
+      std::ostringstream oss;
+      mds_lock.Unlock();
+      g_conf->injectargs(m->cmd[1], &oss);
+      mds_lock.Lock();
+      derr << "injectargs:" << dendl;
+      derr << oss.str() << dendl;
+    }
+  }
+  else if (m->cmd[0] == "dumpcache") {
+    if (m->cmd.size() > 1)
+      mdcache->dump_cache(m->cmd[1].c_str());
+    else
+      mdcache->dump_cache();
+  }
+  else if (m->cmd[0] == "exit") {
+    suicide();
+  }
+  else if (m->cmd[0] == "respawn") {
+    respawn();
+  }
+  else if (m->cmd[0] == "session" && m->cmd[1] == "kill") {
+    Session *session = sessionmap.get_session(entity_name_t(CEPH_ENTITY_TYPE_CLIENT,
+							    strtol(m->cmd[2].c_str(), 0, 10)));
+    if (session)
+      server->kill_session(session);
+    else
+      dout(15) << "session " << session << " not in sessionmap!" << dendl;
+  } else if (m->cmd[0] == "issue_caps") {
+    long inum = strtol(m->cmd[1].c_str(), 0, 10);
+    CInode *in = mdcache->get_inode(inodeno_t(inum));
+    if (in) {
+      bool r = locker->issue_caps(in);
+      dout(20) << "called issue_caps on inode "  << inum
+	       << " with result " << r << dendl;
+    } else dout(15) << "inode " << inum << " not in mdcache!" << dendl;
+  } else if (m->cmd[0] == "try_eval") {
+    long inum = strtol(m->cmd[1].c_str(), 0, 10);
+    int mask = strtol(m->cmd[2].c_str(), 0, 10);
+    CInode * ino = mdcache->get_inode(inodeno_t(inum));
+    if (ino) {
+      locker->try_eval(ino, mask);
+      dout(20) << "try_eval(" << inum << ", " << mask << ")" << dendl;
+    } else dout(15) << "inode " << inum << " not in mdcache!" << dendl;
+  } else if (m->cmd[0] == "fragment_dir") {
+    if (m->cmd.size() == 4) {
+      filepath fp(m->cmd[1].c_str());
+      CInode *in = mdcache->cache_traverse(fp);
+      if (in) {
+	frag_t fg;
+	if (fg.parse(m->cmd[2].c_str())) {
+	  CDir *dir = in->get_dirfrag(fg);
+	  if (dir) {
+	    if (dir->is_auth()) {
+	      int by = atoi(m->cmd[3].c_str());
+	      if (by)
+		mdcache->split_dir(dir, by);
+	      else
+		dout(0) << "need to split by >0 bits" << dendl;
+	    } else dout(0) << "dir " << dir->dirfrag() << " not auth" << dendl;
+	  } else dout(0) << "dir " << in->ino() << " " << fg << " dne" << dendl;
+	} else dout(0) << " frag " << m->cmd[2] << " does not parse" << dendl;
+      } else dout(0) << "path " << fp << " not found" << dendl;
+    } else dout(0) << "bad syntax" << dendl;
+  } else if (m->cmd[0] == "merge_dir") {
+    if (m->cmd.size() == 3) {
+      filepath fp(m->cmd[1].c_str());
+      CInode *in = mdcache->cache_traverse(fp);
+      if (in) {
+	frag_t fg;
+	if (fg.parse(m->cmd[2].c_str())) {
+	  mdcache->merge_dir(in, fg);
+	} else dout(0) << " frag " << m->cmd[2] << " does not parse" << dendl;
+      } else dout(0) << "path " << fp << " not found" << dendl;
+    } else dout(0) << "bad syntax" << dendl;
+  } else if (m->cmd[0] == "export_dir") {
+    if (m->cmd.size() == 3) {
+      filepath fp(m->cmd[1].c_str());
+      int target = atoi(m->cmd[2].c_str());
+      if (target != whoami && mdsmap->is_up(target) && mdsmap->is_in(target)) {
+	CInode *in = mdcache->cache_traverse(fp);
+	if (in) {
+	  CDir *dir = in->get_dirfrag(frag_t());
+	  if (dir && dir->is_auth()) {
+	    mdcache->migrator->export_dir(dir, target);
+	  } else dout(0) << "bad export_dir path dirfrag frag_t() or dir not auth" << dendl;
+	} else dout(0) << "bad export_dir path" << dendl;
+      } else dout(0) << "bad export_dir target syntax" << dendl;
+    } else dout(0) << "bad export_dir syntax" << dendl;
+  } 
+  else if (m->cmd[0] == "cpu_profiler") {
+    ostringstream ss;
+    cpu_profiler_handle_command(m->cmd, ss);
+    clog.info() << ss.str();
+  }
+ else if (m->cmd[0] == "heap") {
+   if (!ceph_using_tcmalloc())
+     clog.info() << "tcmalloc not enabled, can't use heap profiler commands\n";
+   else {
+     ostringstream ss;
+     vector cmdargs;
+     cmdargs.insert(cmdargs.begin(), m->cmd.begin()+1, m->cmd.end());
+     ceph_heap_profiler_handle_command(cmdargs, ss);
+     clog.info() << ss.str();
+   }
+ } else dout(0) << "unrecognized command! " << m->cmd << dendl;
+  m->put();
+}
+
+/* This function deletes the passed message before returning. */
+void MDS::handle_mds_map(MMDSMap *m)
+{
+  version_t epoch = m->get_epoch();
+  dout(5) << "handle_mds_map epoch " << epoch << " from " << m->get_source() << dendl;
+
+  // note source's map version
+  if (m->get_source().is_mds() && 
+      peer_mdsmap_epoch[m->get_source().num()] < epoch) {
+    dout(15) << " peer " << m->get_source()
+	     << " has mdsmap epoch >= " << epoch
+	     << dendl;
+    peer_mdsmap_epoch[m->get_source().num()] = epoch;
+  }
+
+  // is it new?
+  if (epoch <= mdsmap->get_epoch()) {
+    dout(5) << " old map epoch " << epoch << " <= " << mdsmap->get_epoch() 
+	    << ", discarding" << dendl;
+    m->put();
+    return;
+  }
+
+  // keep old map, for a moment
+  MDSMap *oldmap = mdsmap;
+  int oldwhoami = whoami;
+  int oldstate = state;
+  entity_addr_t addr;
+
+  // decode and process
+  mdsmap = new MDSMap;
+  mdsmap->decode(m->get_encoded());
+
+  monc->sub_got("mdsmap", mdsmap->get_epoch());
+
+  // verify compatset
+  CompatSet mdsmap_compat(get_mdsmap_compat_set_all());
+  dout(10) << "     my compat " << mdsmap_compat << dendl;
+  dout(10) << " mdsmap compat " << mdsmap->compat << dendl;
+  if (!mdsmap_compat.writeable(mdsmap->compat)) {
+    dout(0) << "handle_mds_map mdsmap compatset " << mdsmap->compat
+	    << " not writeable with daemon features " << mdsmap_compat
+	    << ", killing myself" << dendl;
+    suicide();
+    goto out;
+  }
+
+  // see who i am
+  addr = messenger->get_myaddr();
+  whoami = mdsmap->get_rank_gid(monc->get_global_id());
+  state = mdsmap->get_state_gid(monc->get_global_id());
+  incarnation = mdsmap->get_inc_gid(monc->get_global_id());
+  dout(10) << "map says i am " << addr << " mds." << whoami << "." << incarnation
+	   << " state " << ceph_mds_state_name(state) << dendl;
+
+  // mark down any failed peers
+  for (map::const_iterator p = oldmap->get_mds_info().begin();
+       p != oldmap->get_mds_info().end();
+       ++p) {
+    if (mdsmap->get_mds_info().count(p->first) == 0) {
+      dout(10) << " peer mds gid " << p->first << " removed from map" << dendl;
+      messenger->mark_down(p->second.addr);
+    }
+  }
+
+  if (state != oldstate)
+    last_state = oldstate;
+
+  if (state == MDSMap::STATE_STANDBY) {
+    want_state = state = MDSMap::STATE_STANDBY;
+    dout(1) << "handle_mds_map standby" << dendl;
+
+    if (standby_type) // we want to be in standby_replay or oneshot_replay!
+      request_state(standby_type);
+
+    goto out;
+  } else if (state == MDSMap::STATE_STANDBY_REPLAY) {
+    if (standby_type && standby_type != MDSMap::STATE_STANDBY_REPLAY) {
+      want_state = standby_type;
+      beacon_send();
+      state = oldstate;
+      goto out;
+    }
+  }
+
+  if (whoami < 0) {
+    if (state == MDSMap::STATE_STANDBY_REPLAY ||
+        state == MDSMap::STATE_ONESHOT_REPLAY) {
+      // fill in whoami from standby-for-rank. If we let this be changed
+      // the logic used to set it here will need to be adjusted.
+      whoami = mdsmap->get_mds_info_gid(monc->get_global_id()).standby_for_rank;
+    } else {
+      if (want_state == MDSMap::STATE_STANDBY) {
+        dout(10) << "dropped out of mdsmap, try to re-add myself" << dendl;
+        want_state = state = MDSMap::STATE_BOOT;
+        goto out;
+      }
+      if (want_state == MDSMap::STATE_BOOT) {
+        dout(10) << "not in map yet" << dendl;
+      } else {
+	// did i get kicked by someone else?
+	if (g_conf->mds_enforce_unique_name) {
+	  if (uint64_t existing = mdsmap->find_mds_gid_by_name(name)) {
+	    MDSMap::mds_info_t& i = mdsmap->get_info_gid(existing);
+	    if (i.global_id > monc->get_global_id()) {
+	      dout(1) << "handle_mds_map i (" << addr
+		      << ") dne in the mdsmap, new instance has larger gid " << i.global_id
+		      << ", suicide" << dendl;
+	      suicide();
+	      goto out;
+	    }
+	  }
+	}
+
+        dout(1) << "handle_mds_map i (" << addr
+            << ") dne in the mdsmap, respawning myself" << dendl;
+        respawn();
+      }
+      goto out;
+    }
+  }
+
+  // ??
+
+  if (oldwhoami != whoami || oldstate != state) {
+    // update messenger.
+    if (state == MDSMap::STATE_STANDBY_REPLAY || state == MDSMap::STATE_ONESHOT_REPLAY) {
+      dout(1) << "handle_mds_map i am now mds." << monc->get_global_id() << "." << incarnation
+	      << "replaying mds." << whoami << "." << incarnation << dendl;
+      messenger->set_myname(entity_name_t::MDS(monc->get_global_id()));
+    } else {
+      dout(1) << "handle_mds_map i am now mds." << whoami << "." << incarnation << dendl;
+      messenger->set_myname(entity_name_t::MDS(whoami));
+    }
+  }
+
+  // tell objecter my incarnation
+  if (objecter->get_client_incarnation() != incarnation)
+    objecter->set_client_incarnation(incarnation);
+
+  // for debug
+  if (g_conf->mds_dump_cache_on_map)
+    mdcache->dump_cache();
+
+  // did it change?
+  if (oldstate != state) {
+    dout(1) << "handle_mds_map state change "
+	    << ceph_mds_state_name(oldstate) << " --> "
+	    << ceph_mds_state_name(state) << dendl;
+    want_state = state;
+
+    if (oldstate == MDSMap::STATE_STANDBY_REPLAY) {
+        dout(10) << "Monitor activated us! Deactivating replay loop" << dendl;
+        assert (state == MDSMap::STATE_REPLAY);
+    } else {
+      // did i just recover?
+      if ((is_active() || is_clientreplay()) &&
+          (oldstate == MDSMap::STATE_CREATING ||
+	   oldstate == MDSMap::STATE_REJOIN ||
+	   oldstate == MDSMap::STATE_RECONNECT))
+        recovery_done(oldstate);
+
+      if (is_active()) {
+        active_start();
+      } else if (is_any_replay()) {
+        replay_start();
+      } else if (is_resolve()) {
+        resolve_start();
+      } else if (is_reconnect()) {
+        reconnect_start();
+      } else if (is_rejoin()) {
+	rejoin_start();
+      } else if (is_clientreplay()) {
+        clientreplay_start();
+      } else if (is_creating()) {
+        boot_create();
+      } else if (is_starting()) {
+        boot_start();
+      } else if (is_stopping()) {
+        assert(oldstate == MDSMap::STATE_ACTIVE);
+        stopping_start();
+      }
+    }
+  }
+  
+  // RESOLVE
+  // is someone else newly resolving?
+  if (is_resolve() || is_reconnect() || is_rejoin() ||
+      is_clientreplay() || is_active() || is_stopping()) {
+    if (!oldmap->is_resolving() && mdsmap->is_resolving()) {
+      set resolve;
+      mdsmap->get_mds_set(resolve, MDSMap::STATE_RESOLVE);
+      dout(10) << " resolve set is " << resolve << dendl;
+      calc_recovery_set();
+      mdcache->send_resolves();
+    }
+  }
+  
+  // REJOIN
+  // is everybody finally rejoining?
+  if (is_rejoin() || is_clientreplay() || is_active() || is_stopping()) {
+    // did we start?
+    if (!oldmap->is_rejoining() && mdsmap->is_rejoining())
+      rejoin_joint_start();
+
+    // did we finish?
+    if (g_conf->mds_dump_cache_after_rejoin &&
+	oldmap->is_rejoining() && !mdsmap->is_rejoining()) 
+      mdcache->dump_cache();      // for DEBUG only
+
+    if (oldstate >= MDSMap::STATE_REJOIN) {
+      // ACTIVE|CLIENTREPLAY|REJOIN => we can discover from them.
+      set olddis, dis;
+      oldmap->get_mds_set(olddis, MDSMap::STATE_ACTIVE);
+      oldmap->get_mds_set(olddis, MDSMap::STATE_CLIENTREPLAY);
+      oldmap->get_mds_set(olddis, MDSMap::STATE_REJOIN);
+      mdsmap->get_mds_set(dis, MDSMap::STATE_ACTIVE);
+      mdsmap->get_mds_set(dis, MDSMap::STATE_CLIENTREPLAY);
+      mdsmap->get_mds_set(dis, MDSMap::STATE_REJOIN);
+      for (set::iterator p = dis.begin(); p != dis.end(); ++p)
+	if (*p != whoami &&            // not me
+	    olddis.count(*p) == 0) {  // newly so?
+	  mdcache->kick_discovers(*p);
+	  mdcache->kick_open_ino_peers(*p);
+	}
+    }
+  }
+
+  if (oldmap->is_degraded() && !mdsmap->is_degraded() && state >= MDSMap::STATE_ACTIVE)
+    dout(1) << "cluster recovered." << dendl;
+
+  // did someone go active?
+  if (oldstate >= MDSMap::STATE_CLIENTREPLAY &&
+      (is_clientreplay() || is_active() || is_stopping())) {
+    set oldactive, active;
+    oldmap->get_mds_set(oldactive, MDSMap::STATE_ACTIVE);
+    oldmap->get_mds_set(oldactive, MDSMap::STATE_CLIENTREPLAY);
+    mdsmap->get_mds_set(active, MDSMap::STATE_ACTIVE);
+    mdsmap->get_mds_set(active, MDSMap::STATE_CLIENTREPLAY);
+    for (set::iterator p = active.begin(); p != active.end(); ++p) 
+      if (*p != whoami &&            // not me
+	  oldactive.count(*p) == 0)  // newly so?
+	handle_mds_recovery(*p);
+  }
+
+  // did someone fail?
+  if (true) {
+    // new failed?
+    set oldfailed, failed;
+    oldmap->get_failed_mds_set(oldfailed);
+    mdsmap->get_failed_mds_set(failed);
+    for (set::iterator p = failed.begin(); p != failed.end(); ++p)
+      if (oldfailed.count(*p) == 0) {
+	messenger->mark_down(oldmap->get_inst(*p).addr);
+	handle_mds_failure(*p);
+      }
+    
+    // or down then up?
+    //  did their addr/inst change?
+    set up;
+    mdsmap->get_up_mds_set(up);
+    for (set::iterator p = up.begin(); p != up.end(); ++p) 
+      if (oldmap->have_inst(*p) &&
+	  oldmap->get_inst(*p) != mdsmap->get_inst(*p)) {
+	messenger->mark_down(oldmap->get_inst(*p).addr);
+	handle_mds_failure(*p);
+      }
+  }
+  if (is_clientreplay() || is_active() || is_stopping()) {
+    // did anyone stop?
+    set oldstopped, stopped;
+    oldmap->get_stopped_mds_set(oldstopped);
+    mdsmap->get_stopped_mds_set(stopped);
+    for (set::iterator p = stopped.begin(); p != stopped.end(); ++p) 
+      if (oldstopped.count(*p) == 0)      // newly so?
+	mdcache->migrator->handle_mds_failure_or_stop(*p);
+  }
+
+  if (!is_any_replay())
+    balancer->try_rebalance();
+
+  {
+    map >::iterator p = waiting_for_mdsmap.begin();
+    while (p != waiting_for_mdsmap.end() && p->first <= mdsmap->get_epoch()) {
+      list ls;
+      ls.swap(p->second);
+      waiting_for_mdsmap.erase(p++);
+      finish_contexts(g_ceph_context, ls);
+    }
+  }
+
+ out:
+  m->put();
+  delete oldmap;
+}
+
+void MDS::bcast_mds_map()
+{
+  dout(7) << "bcast_mds_map " << mdsmap->get_epoch() << dendl;
+
+  // share the map with mounted clients
+  set clients;
+  sessionmap.get_client_session_set(clients);
+  for (set::const_iterator p = clients.begin();
+       p != clients.end();
+       ++p) 
+    messenger->send_message(new MMDSMap(monc->get_fsid(), mdsmap),
+			    (*p)->connection);
+  last_client_mdsmap_bcast = mdsmap->get_epoch();
+}
+
+
+void MDS::request_state(int s)
+{
+  dout(3) << "request_state " << ceph_mds_state_name(s) << dendl;
+  want_state = s;
+  beacon_send();
+}
+
+
+class C_MDS_CreateFinish : public Context {
+  MDS *mds;
+public:
+  C_MDS_CreateFinish(MDS *m) : mds(m) {}
+  void finish(int r) { mds->creating_done(); }
+};
+
+void MDS::boot_create()
+{
+  dout(3) << "boot_create" << dendl;
+
+  C_GatherBuilder fin(g_ceph_context, new C_MDS_CreateFinish(this));
+
+  mdcache->init_layouts();
+
+  // start with a fresh journal
+  dout(10) << "boot_create creating fresh journal" << dendl;
+  mdlog->create(fin.new_sub());
+
+  // open new journal segment, but do not journal subtree map (yet)
+  mdlog->prepare_new_segment();
+
+  if (whoami == mdsmap->get_root()) {
+    dout(3) << "boot_create creating fresh hierarchy" << dendl;
+    mdcache->create_empty_hierarchy(fin.get());
+  }
+
+  dout(3) << "boot_create creating mydir hierarchy" << dendl;
+  mdcache->create_mydir_hierarchy(fin.get());
+
+  // fixme: fake out inotable (reset, pretend loaded)
+  dout(10) << "boot_create creating fresh inotable table" << dendl;
+  inotable->reset();
+  inotable->save(fin.new_sub());
+
+  // write empty sessionmap
+  sessionmap.save(fin.new_sub());
+
+  // initialize tables
+  if (mdsmap->get_tableserver() == whoami) {
+    dout(10) << "boot_create creating fresh anchortable" << dendl;
+    anchorserver->reset();
+    anchorserver->save(fin.new_sub());
+
+    dout(10) << "boot_create creating fresh snaptable" << dendl;
+    snapserver->reset();
+    snapserver->save(fin.new_sub());
+  }
+
+  assert(g_conf->mds_kill_create_at != 1);
+
+  // ok now journal it
+  mdlog->journal_segment_subtree_map();
+  mdlog->wait_for_safe(fin.new_sub());
+  mdlog->flush();
+
+  fin.activate();
+}
+
+void MDS::creating_done()
+{
+  dout(1)<< "creating_done" << dendl;
+  request_state(MDSMap::STATE_ACTIVE);
+}
+
+
+class C_MDS_BootStart : public Context {
+  MDS *mds;
+  int nextstep;
+public:
+  C_MDS_BootStart(MDS *m, int n) : mds(m), nextstep(n) {}
+  void finish(int r) { mds->boot_start(nextstep, r); }
+};
+
+void MDS::boot_start(int step, int r)
+{
+  if (r < 0) {
+    if (is_standby_replay() && (r == -EAGAIN)) {
+      dout(0) << "boot_start encountered an error EAGAIN"
+              << ", respawning since we fell behind journal" << dendl;
+      respawn();
+    } else {
+      dout(0) << "boot_start encountered an error, failing" << dendl;
+      suicide();
+      return;
+    }
+  }
+
+  switch (step) {
+  case 0:
+    mdcache->init_layouts();
+    step = 1;  // fall-thru.
+
+  case 1:
+    {
+      C_GatherBuilder gather(g_ceph_context, new C_MDS_BootStart(this, 2));
+      dout(2) << "boot_start " << step << ": opening inotable" << dendl;
+      inotable->load(gather.new_sub());
+
+      dout(2) << "boot_start " << step << ": opening sessionmap" << dendl;
+      sessionmap.load(gather.new_sub());
+
+      if (mdsmap->get_tableserver() == whoami) {
+	dout(2) << "boot_start " << step << ": opening anchor table" << dendl;
+	anchorserver->load(gather.new_sub());
+
+	dout(2) << "boot_start " << step << ": opening snap table" << dendl;	
+	snapserver->load(gather.new_sub());
+      }
+      
+      dout(2) << "boot_start " << step << ": opening mds log" << dendl;
+      mdlog->open(gather.new_sub());
+      gather.activate();
+    }
+    break;
+
+  case 2:
+    {
+      dout(2) << "boot_start " << step << ": loading/discovering base inodes" << dendl;
+
+      C_GatherBuilder gather(g_ceph_context, new C_MDS_BootStart(this, 3));
+
+      mdcache->open_mydir_inode(gather.new_sub());
+
+      if (is_starting() ||
+	  whoami == mdsmap->get_root()) {  // load root inode off disk if we are auth
+	mdcache->open_root_inode(gather.new_sub());
+      } else {
+	// replay.  make up fake root inode to start with
+	mdcache->create_root_inode();
+      }
+      gather.activate();
+    }
+    break;
+
+  case 3:
+    if (is_any_replay()) {
+      dout(2) << "boot_start " << step << ": replaying mds log" << dendl;
+      mdlog->replay(new C_MDS_BootStart(this, 4));
+      break;
+    } else {
+      dout(2) << "boot_start " << step << ": positioning at end of old mds log" << dendl;
+      mdlog->append();
+      step++;
+    }
+
+  case 4:
+    if (is_any_replay()) {
+      replay_done();
+      break;
+    }
+    step++;
+    
+    starting_done();
+    break;
+  }
+}
+
+void MDS::starting_done()
+{
+  dout(3) << "starting_done" << dendl;
+  assert(is_starting());
+  request_state(MDSMap::STATE_ACTIVE);
+
+  mdcache->open_root();
+
+  // start new segment
+  mdlog->start_new_segment(0);
+}
+
+
+void MDS::calc_recovery_set()
+{
+  // initialize gather sets
+  set rs;
+  mdsmap->get_recovery_mds_set(rs);
+  rs.erase(whoami);
+  mdcache->set_recovery_set(rs);
+
+  dout(1) << " recovery set is " << rs << dendl;
+}
+
+
+void MDS::replay_start()
+{
+  dout(1) << "replay_start" << dendl;
+
+  if (is_standby_replay())
+    standby_replaying = true;
+  
+  standby_type = 0;
+
+  calc_recovery_set();
+
+  dout(1) << " need osdmap epoch " << mdsmap->get_last_failure_osd_epoch()
+	  <<", have " << osdmap->get_epoch()
+	  << dendl;
+
+  // start?
+  if (osdmap->get_epoch() >= mdsmap->get_last_failure_osd_epoch()) {
+    boot_start();
+  } else {
+    dout(1) << " waiting for osdmap " << mdsmap->get_last_failure_osd_epoch() 
+	    << " (which blacklists prior instance)" << dendl;
+    objecter->wait_for_new_map(new C_MDS_BootStart(this, 0),
+			       mdsmap->get_last_failure_osd_epoch());
+  }
+}
+
+
+class MDS::C_MDS_StandbyReplayRestartFinish : public Context {
+  MDS *mds;
+  uint64_t old_read_pos;
+public:
+  C_MDS_StandbyReplayRestartFinish(MDS *mds_, uint64_t old_read_pos_) :
+    mds(mds_), old_read_pos(old_read_pos_) {}
+  void finish(int r) {
+    mds->_standby_replay_restart_finish(r, old_read_pos);
+  }
+};
+
+void MDS::_standby_replay_restart_finish(int r, uint64_t old_read_pos)
+{
+  if (old_read_pos < mdlog->get_journaler()->get_trimmed_pos()) {
+    dout(0) << "standby MDS fell behind active MDS journal's expire_pos, restarting" << dendl;
+    respawn(); /* we're too far back, and this is easier than
+		  trying to reset everything in the cache, etc */
+  } else {
+    mdlog->standby_trim_segments();
+    boot_start(3, r);
+  }
+}
+
+inline void MDS::standby_replay_restart()
+{
+  dout(1) << "standby_replay_restart" << (standby_replaying ? " (as standby)":" (final takeover pass)") << dendl;
+
+  if (!standby_replaying && osdmap->get_epoch() < mdsmap->get_last_failure_osd_epoch()) {
+    dout(1) << " waiting for osdmap " << mdsmap->get_last_failure_osd_epoch() 
+	    << " (which blacklists prior instance)" << dendl;
+    objecter->wait_for_new_map(new C_MDS_BootStart(this, 3),
+			       mdsmap->get_last_failure_osd_epoch());
+  } else {
+    mdlog->get_journaler()->reread_head_and_probe(
+      new C_MDS_StandbyReplayRestartFinish(this, mdlog->get_journaler()->get_read_pos()));
+  }
+}
+
+class MDS::C_MDS_StandbyReplayRestart : public Context {
+  MDS *mds;
+public:
+  C_MDS_StandbyReplayRestart(MDS *m) : mds(m) {}
+  void finish(int r) {
+    assert(!r);
+    mds->standby_replay_restart();
+  }
+};
+
+void MDS::replay_done()
+{
+  dout(1) << "replay_done" << (standby_replaying ? " (as standby)" : "") << dendl;
+
+  if (is_oneshot_replay()) {
+    dout(2) << "hack.  journal looks ok.  shutting down." << dendl;
+    suicide();
+    return;
+  }
+
+  if (is_standby_replay()) {
+    dout(10) << "setting replay timer" << dendl;
+    timer.add_event_after(g_conf->mds_replay_interval,
+                          new C_MDS_StandbyReplayRestart(this));
+    return;
+  }
+
+  if (standby_replaying) {
+    dout(10) << " last replay pass was as a standby; making final pass" << dendl;
+    standby_replaying = false;
+    standby_replay_restart();
+    return;
+  }
+
+  dout(1) << "making mds journal writeable" << dendl;
+  mdlog->get_journaler()->set_writeable();
+  mdlog->get_journaler()->trim_tail();
+
+  if (g_conf->mds_wipe_sessions) {
+    dout(1) << "wiping out client sessions" << dendl;
+    sessionmap.wipe();
+    sessionmap.save(new C_NoopContext);
+  }
+  if (g_conf->mds_wipe_ino_prealloc) {
+    dout(1) << "wiping out ino prealloc from sessions" << dendl;
+    sessionmap.wipe_ino_prealloc();
+    sessionmap.save(new C_NoopContext);
+  }
+  if (g_conf->mds_skip_ino) {
+    inodeno_t i = g_conf->mds_skip_ino;
+    dout(1) << "skipping " << i << " inodes" << dendl;
+    inotable->skip_inos(i);
+    inotable->save(new C_NoopContext);
+  }
+
+  if (mdsmap->get_num_in_mds() == 1 &&
+      mdsmap->get_num_failed_mds() == 0) { // just me!
+    dout(2) << "i am alone, moving to state reconnect" << dendl;      
+    request_state(MDSMap::STATE_RECONNECT);
+  } else {
+    dout(2) << "i am not alone, moving to state resolve" << dendl;
+    request_state(MDSMap::STATE_RESOLVE);
+  }
+}
+
+void MDS::reopen_log()
+{
+  dout(1) << "reopen_log" << dendl;
+  mdcache->rollback_uncommitted_fragments();
+}
+
+
+void MDS::resolve_start()
+{
+  dout(1) << "resolve_start" << dendl;
+
+  reopen_log();
+
+  mdcache->resolve_start();
+  finish_contexts(g_ceph_context, waiting_for_resolve);
+}
+void MDS::resolve_done()
+{
+  dout(1) << "resolve_done" << dendl;
+  request_state(MDSMap::STATE_RECONNECT);
+}
+
+void MDS::reconnect_start()
+{
+  dout(1) << "reconnect_start" << dendl;
+
+  if (last_state == MDSMap::STATE_REPLAY)
+    reopen_log();
+
+  server->reconnect_clients();
+  finish_contexts(g_ceph_context, waiting_for_reconnect);
+}
+void MDS::reconnect_done()
+{
+  dout(1) << "reconnect_done" << dendl;
+  request_state(MDSMap::STATE_REJOIN);    // move to rejoin state
+}
+
+void MDS::rejoin_joint_start()
+{
+  dout(1) << "rejoin_joint_start" << dendl;
+  mdcache->rejoin_send_rejoins();
+}
+void MDS::rejoin_start()
+{
+  dout(1) << "rejoin_start" << dendl;
+  mdcache->rejoin_start();
+}
+void MDS::rejoin_done()
+{
+  dout(1) << "rejoin_done" << dendl;
+  mdcache->show_subtrees();
+  mdcache->show_cache();
+
+  // funny case: is our cache empty?  no subtrees?
+  if (!mdcache->is_subtrees()) {
+    dout(1) << " empty cache, no subtrees, leaving cluster" << dendl;
+    request_state(MDSMap::STATE_STOPPED);
+    return;
+  }
+
+  if (replay_queue.empty())
+    request_state(MDSMap::STATE_ACTIVE);
+  else
+    request_state(MDSMap::STATE_CLIENTREPLAY);
+}
+
+void MDS::clientreplay_start()
+{
+  dout(1) << "clientreplay_start" << dendl;
+  finish_contexts(g_ceph_context, waiting_for_replay);  // kick waiters
+  queue_one_replay();
+}
+
+void MDS::clientreplay_done()
+{
+  dout(1) << "clientreplay_done" << dendl;
+  request_state(MDSMap::STATE_ACTIVE);
+}
+
+void MDS::active_start()
+{
+  dout(1) << "active_start" << dendl;
+
+  if (last_state == MDSMap::STATE_CREATING)
+    mdcache->open_root();
+
+  mdcache->clean_open_file_lists();
+  mdcache->export_remaining_imported_caps();
+  finish_contexts(g_ceph_context, waiting_for_replay);  // kick waiters
+  finish_contexts(g_ceph_context, waiting_for_active);  // kick waiters
+}
+
+void MDS::recovery_done(int oldstate)
+{
+  dout(1) << "recovery_done -- successful recovery!" << dendl;
+  assert(is_clientreplay() || is_active());
+  
+  // kick anchortable (resent AGREEs)
+  if (mdsmap->get_tableserver() == whoami) {
+    set active;
+    mdsmap->get_mds_set(active, MDSMap::STATE_CLIENTREPLAY);
+    mdsmap->get_mds_set(active, MDSMap::STATE_ACTIVE);
+    mdsmap->get_mds_set(active, MDSMap::STATE_STOPPING);
+    anchorserver->finish_recovery(active);
+    snapserver->finish_recovery(active);
+  }
+
+  if (oldstate == MDSMap::STATE_CREATING)
+    return;
+
+  mdcache->start_recovered_truncates();
+  mdcache->do_file_recover();
+
+  mdcache->reissue_all_caps();
+  
+  // tell connected clients
+  //bcast_mds_map();     // not anymore, they get this from the monitor
+
+  mdcache->populate_mydir();
+}
+
+void MDS::handle_mds_recovery(int who) 
+{
+  dout(5) << "handle_mds_recovery mds." << who << dendl;
+  
+  mdcache->handle_mds_recovery(who);
+
+  if (mdsmap->get_tableserver() == whoami) {
+    anchorserver->handle_mds_recovery(who);
+    snapserver->handle_mds_recovery(who);
+  }
+
+  queue_waiters(waiting_for_active_peer[who]);
+  waiting_for_active_peer.erase(who);
+}
+
+void MDS::handle_mds_failure(int who)
+{
+  if (who == whoami) {
+    dout(5) << "handle_mds_failure for myself; not doing anything" << dendl;
+    return;
+  }
+  dout(5) << "handle_mds_failure mds." << who << dendl;
+
+  mdcache->handle_mds_failure(who);
+
+  anchorclient->handle_mds_failure(who);
+  snapclient->handle_mds_failure(who);
+}
+
+void MDS::stopping_start()
+{
+  dout(2) << "stopping_start" << dendl;
+
+  if (mdsmap->get_num_in_mds() == 1 && !sessionmap.empty()) {
+    // we're the only mds up!
+    dout(0) << "we are the last MDS, and have mounted clients: we cannot flush our journal.  suicide!" << dendl;
+    suicide();
+  }
+
+  mdcache->shutdown_start();
+}
+
+void MDS::stopping_done()
+{
+  dout(2) << "stopping_done" << dendl;
+
+  // tell monitor we shut down cleanly.
+  request_state(MDSMap::STATE_STOPPED);
+}
+
+void MDS::handle_signal(int signum)
+{
+  assert(signum == SIGINT || signum == SIGTERM);
+  derr << "*** got signal " << sys_siglist[signum] << " ***" << dendl;
+  mds_lock.Lock();
+  suicide();
+  mds_lock.Unlock();
+}
+
+void MDS::suicide()
+{
+  assert(mds_lock.is_locked());
+  want_state = CEPH_MDS_STATE_DNE; // whatever.
+
+  dout(1) << "suicide.  wanted " << ceph_mds_state_name(want_state)
+	  << ", now " << ceph_mds_state_name(state) << dendl;
+
+  // stop timers
+  if (beacon_sender) {
+    timer.cancel_event(beacon_sender);
+    beacon_sender = 0;
+  }
+  if (tick_event) {
+    timer.cancel_event(tick_event);
+    tick_event = 0;
+  }
+  timer.cancel_all_events();
+  //timer.join();
+  timer.shutdown();
+  
+  // shut down cache
+  mdcache->shutdown();
+
+  if (objecter->initialized)
+    objecter->shutdown_locked();
+
+  monc->shutdown();
+
+  // shut down messenger
+  messenger->shutdown();
+}
+
+void MDS::respawn()
+{
+  dout(1) << "respawn" << dendl;
+
+  char *new_argv[orig_argc+1];
+  dout(1) << " e: '" << orig_argv[0] << "'" << dendl;
+  for (int i=0; iput();
+    ret = true;
+  } else {
+    inc_dispatch_depth();
+    ret = _dispatch(m);
+    dec_dispatch_depth();
+  }
+  mds_lock.Unlock();
+  return ret;
+}
+
+bool MDS::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new)
+{
+  dout(10) << "MDS::ms_get_authorizer type=" << ceph_entity_type_name(dest_type) << dendl;
+
+  /* monitor authorization is being handled on different layer */
+  if (dest_type == CEPH_ENTITY_TYPE_MON)
+    return true;
+
+  if (force_new) {
+    if (monc->wait_auth_rotating(10) < 0)
+      return false;
+  }
+
+  *authorizer = monc->auth->build_authorizer(dest_type);
+  return *authorizer != NULL;
+}
+
+
+#define ALLOW_MESSAGES_FROM(peers) \
+do { \
+  if (m->get_connection() && (m->get_connection()->get_peer_type() & (peers)) == 0) { \
+    dout(0) << __FILE__ << "." << __LINE__ << ": filtered out request, peer=" << m->get_connection()->get_peer_type() \
+           << " allowing=" << #peers << " message=" << *m << dendl; \
+    m->put();							    \
+    return true; \
+  } \
+} while (0)
+
+
+/*
+ * high priority messages we always process
+ */
+bool MDS::handle_core_message(Message *m)
+{
+  switch (m->get_type()) {
+  case CEPH_MSG_MON_MAP:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MON);
+    m->put();
+    break;
+
+    // MDS
+  case CEPH_MSG_MDS_MAP:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_MDS);
+    handle_mds_map(static_cast(m));
+    break;
+  case MSG_MDS_BEACON:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MON);
+    handle_mds_beacon(static_cast(m));
+    break;
+    
+    // misc
+  case MSG_MON_COMMAND:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MON);
+    handle_command(static_cast(m));
+    break;    
+
+    // OSD
+  case CEPH_MSG_OSD_OPREPLY:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_OSD);
+    objecter->handle_osd_op_reply((class MOSDOpReply*)m);
+    break;
+  case CEPH_MSG_OSD_MAP:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD);
+    objecter->handle_osd_map((MOSDMap*)m);
+    if (is_active() && snapserver)
+      snapserver->check_osd_map(true);
+    break;
+
+  default:
+    return false;
+  }
+  return true;
+}
+
+/*
+ * lower priority messages we defer if we seem laggy
+ */
+bool MDS::handle_deferrable_message(Message *m)
+{
+  int port = m->get_type() & 0xff00;
+
+  switch (port) {
+  case MDS_PORT_CACHE:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
+    mdcache->dispatch(m);
+    break;
+    
+  case MDS_PORT_MIGRATOR:
+    ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
+    mdcache->migrator->dispatch(m);
+    break;
+    
+  default:
+    switch (m->get_type()) {
+      // SERVER
+    case CEPH_MSG_CLIENT_SESSION:
+    case CEPH_MSG_CLIENT_RECONNECT:
+      ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_CLIENT);
+      // fall-thru
+    case CEPH_MSG_CLIENT_REQUEST:
+      server->dispatch(m);
+      break;
+    case MSG_MDS_SLAVE_REQUEST:
+      ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
+      server->dispatch(m);
+      break;
+      
+    case MSG_MDS_HEARTBEAT:
+      ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
+      balancer->proc_message(m);
+      break;
+	  
+    case MSG_MDS_TABLE_REQUEST:
+      ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
+      {
+	MMDSTableRequest *req = static_cast(m);
+	if (req->op < 0) {
+	  MDSTableClient *client = get_table_client(req->table);
+	      client->handle_request(req);
+	} else {
+	  MDSTableServer *server = get_table_server(req->table);
+	  server->handle_request(req);
+	}
+      }
+      break;
+
+    case MSG_MDS_LOCK:
+    case MSG_MDS_INODEFILECAPS:
+      ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
+      locker->dispatch(m);
+      break;
+      
+    case CEPH_MSG_CLIENT_CAPS:
+    case CEPH_MSG_CLIENT_CAPRELEASE:
+    case CEPH_MSG_CLIENT_LEASE:
+      ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_CLIENT);
+      locker->dispatch(m);
+      break;
+      
+    default:
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool MDS::is_stale_message(Message *m)
+{
+  // from bad mds?
+  if (m->get_source().is_mds()) {
+    int from = m->get_source().num();
+    if (!mdsmap->have_inst(from) ||
+	mdsmap->get_inst(from) != m->get_source_inst() ||
+	mdsmap->is_down(from)) {
+      // bogus mds?
+      if (m->get_type() == CEPH_MSG_MDS_MAP) {
+	dout(5) << "got " << *m << " from old/bad/imposter mds " << m->get_source()
+		<< ", but it's an mdsmap, looking at it" << dendl;
+      } else if (m->get_type() == MSG_MDS_CACHEEXPIRE &&
+		 mdsmap->get_inst(from) == m->get_source_inst()) {
+	dout(5) << "got " << *m << " from down mds " << m->get_source()
+		<< ", but it's a cache_expire, looking at it" << dendl;
+      } else {
+	dout(5) << "got " << *m << " from down/old/bad/imposter mds " << m->get_source()
+		<< ", dropping" << dendl;
+	return true;
+      }
+    }
+  }
+  return false;
+}
+
+/* If this function returns true, it has put the message. If it returns false,
+ * it has not put the message. */
+bool MDS::_dispatch(Message *m)
+{
+  if (is_stale_message(m)) {
+    m->put();
+    return true;
+  }
+
+  // core
+  if (!handle_core_message(m)) {
+    if (is_laggy()) {
+      dout(10) << " laggy, deferring " << *m << dendl;
+      waiting_for_nolaggy.push_back(m);
+    } else {
+      if (!handle_deferrable_message(m)) {
+	dout(0) << "unrecognized message " << *m << dendl;
+	m->put();
+	return false;
+      }
+    }
+  }
+
+  if (dispatch_depth > 1)
+    return true;
+
+  // finish any triggered contexts
+  while (!finished_queue.empty()) {
+    dout(7) << "mds has " << finished_queue.size() << " queued contexts" << dendl;
+    dout(10) << finished_queue << dendl;
+    list ls;
+    ls.swap(finished_queue);
+    while (!ls.empty()) {
+      dout(10) << " finish " << ls.front() << dendl;
+      ls.front()->complete(0);
+      ls.pop_front();
+      
+      // give other threads (beacon!) a chance
+      mds_lock.Unlock();
+      mds_lock.Lock();
+    }
+  }
+
+  while (!waiting_for_nolaggy.empty()) {
+
+    // stop if we're laggy now!
+    if (is_laggy())
+      return true;
+
+    Message *old = waiting_for_nolaggy.front();
+    waiting_for_nolaggy.pop_front();
+
+    if (is_stale_message(old)) {
+      old->put();
+    } else {
+      dout(7) << " processing laggy deferred " << *old << dendl;
+      handle_deferrable_message(old);
+    }
+
+    // give other threads (beacon!) a chance
+    mds_lock.Unlock();
+    mds_lock.Lock();
+  }
+
+  // done with all client replayed requests?
+  if (is_clientreplay() &&
+      mdcache->is_open() &&
+      replay_queue.empty() &&
+      want_state == MDSMap::STATE_CLIENTREPLAY) {
+    int num_requests = mdcache->get_num_client_requests();
+    dout(10) << " still have " << num_requests << " active replay requests" << dendl;
+    if (num_requests == 0)
+      clientreplay_done();
+  }
+
+  // hack: thrash exports
+  static utime_t start;
+  utime_t now = ceph_clock_now(g_ceph_context);
+  if (start == utime_t()) 
+    start = now;
+  /*double el = now - start;
+  if (el > 30.0 &&
+    el < 60.0)*/
+  for (int i=0; imds_thrash_exports; i++) {
+    set s;
+    if (!is_active()) break;
+    mdsmap->get_mds_set(s, MDSMap::STATE_ACTIVE);
+    if (s.size() < 2 || mdcache->get_num_inodes() < 10) 
+      break;  // need peers for this to work.
+
+    dout(7) << "mds thrashing exports pass " << (i+1) << "/" << g_conf->mds_thrash_exports << dendl;
+    
+    // pick a random dir inode
+    CInode *in = mdcache->hack_pick_random_inode();
+
+    list ls;
+    in->get_dirfrags(ls);
+    if (ls.empty())
+      continue;                // must be an open dir.
+    list::iterator p = ls.begin();
+    int n = rand() % ls.size();
+    while (n--)
+      ++p;
+    CDir *dir = *p;
+    if (!dir->get_parent_dir()) continue;    // must be linked.
+    if (!dir->is_auth()) continue;           // must be auth.
+
+    int dest;
+    do {
+      int k = rand() % s.size();
+      set::iterator p = s.begin();
+      while (k--) ++p;
+      dest = *p;
+    } while (dest == whoami);
+    mdcache->migrator->export_dir_nicely(dir,dest);
+  }
+  // hack: thrash fragments
+  for (int i=0; imds_thrash_fragments; i++) {
+    if (!is_active()) break;
+    if (mdcache->get_num_fragmenting_dirs() > 5) break;
+    dout(7) << "mds thrashing fragments pass " << (i+1) << "/" << g_conf->mds_thrash_fragments << dendl;
+    
+    // pick a random dir inode
+    CInode *in = mdcache->hack_pick_random_inode();
+
+    list ls;
+    in->get_dirfrags(ls);
+    if (ls.empty()) continue;                // must be an open dir.
+    CDir *dir = ls.front();
+    if (!dir->get_parent_dir()) continue;    // must be linked.
+    if (!dir->is_auth()) continue;           // must be auth.
+    frag_t fg = dir->get_frag();
+    if (fg == frag_t() || (rand() % (1 << fg.bits()) == 0))
+      mdcache->split_dir(dir, 1);
+    else
+      balancer->queue_merge(dir);
+  }
+
+  // hack: force hash root?
+  /*
+  if (false &&
+      mdcache->get_root() &&
+      mdcache->get_root()->dir &&
+      !(mdcache->get_root()->dir->is_hashed() || 
+        mdcache->get_root()->dir->is_hashing())) {
+    dout(0) << "hashing root" << dendl;
+    mdcache->migrator->hash_dir(mdcache->get_root()->dir);
+  }
+  */
+
+  if (mlogger) {
+    mlogger->set(l_mdm_ino, g_num_ino);
+    mlogger->set(l_mdm_dir, g_num_dir);
+    mlogger->set(l_mdm_dn, g_num_dn);
+    mlogger->set(l_mdm_cap, g_num_cap);
+
+    mlogger->inc(l_mdm_inoa, g_num_inoa);  g_num_inoa = 0;
+    mlogger->inc(l_mdm_inos, g_num_inos);  g_num_inos = 0;
+    mlogger->inc(l_mdm_dira, g_num_dira);  g_num_dira = 0;
+    mlogger->inc(l_mdm_dirs, g_num_dirs);  g_num_dirs = 0;
+    mlogger->inc(l_mdm_dna, g_num_dna);  g_num_dna = 0;
+    mlogger->inc(l_mdm_dns, g_num_dns);  g_num_dns = 0;
+    mlogger->inc(l_mdm_capa, g_num_capa);  g_num_capa = 0;
+    mlogger->inc(l_mdm_caps, g_num_caps);  g_num_caps = 0;
+
+    mlogger->set(l_mdm_buf, buffer::get_total_alloc());
+
+  }
+
+  // shut down?
+  if (is_stopping()) {
+    mdlog->trim();
+    if (mdcache->shutdown_pass()) {
+      dout(7) << "shutdown_pass=true, finished w/ shutdown, moving to down:stopped" << dendl;
+      stopping_done();
+    }
+    else {
+      dout(7) << "shutdown_pass=false" << dendl;
+    }
+  }
+  return true;
+}
+
+
+
+
+void MDS::ms_handle_connect(Connection *con) 
+{
+  Mutex::Locker l(mds_lock);
+  dout(5) << "ms_handle_connect on " << con->get_peer_addr() << dendl;
+  if (want_state == CEPH_MDS_STATE_DNE)
+    return;
+  objecter->ms_handle_connect(con);
+}
+
+bool MDS::ms_handle_reset(Connection *con) 
+{
+  Mutex::Locker l(mds_lock);
+  dout(5) << "ms_handle_reset on " << con->get_peer_addr() << dendl;
+  if (want_state == CEPH_MDS_STATE_DNE)
+    return false;
+
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_OSD) {
+    objecter->ms_handle_reset(con);
+  } else if (con->get_peer_type() == CEPH_ENTITY_TYPE_CLIENT) {
+    Session *session = static_cast(con->get_priv());
+    if (session) {
+      if (session->is_closed()) {
+	dout(3) << "ms_handle_reset closing connection for session " << session->info.inst << dendl;
+	messenger->mark_down(con);
+	con->set_priv(NULL);
+	sessionmap.remove_session(session);
+      }
+      session->put();
+    } else {
+      messenger->mark_down(con);
+    }
+  }
+  return false;
+}
+
+
+void MDS::ms_handle_remote_reset(Connection *con) 
+{
+  Mutex::Locker l(mds_lock);
+  dout(5) << "ms_handle_remote_reset on " << con->get_peer_addr() << dendl;
+  if (want_state == CEPH_MDS_STATE_DNE)
+    return;
+  switch (con->get_peer_type()) {
+  case CEPH_ENTITY_TYPE_OSD:
+    objecter->ms_handle_remote_reset(con);
+    break;
+
+  case CEPH_ENTITY_TYPE_CLIENT:
+    Session *session = static_cast(con->get_priv());
+    if (session) {
+      if (session->is_closed()) {
+	dout(3) << "ms_handle_remote_reset closing connection for session " << session->info.inst << dendl;
+	messenger->mark_down(con);
+	con->set_priv(NULL);
+	sessionmap.remove_session(session);
+      }
+      session->put();
+    }
+    break;
+  }
+}
+
+bool MDS::ms_verify_authorizer(Connection *con, int peer_type,
+			       int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply,
+			       bool& is_valid, CryptoKey& session_key)
+{
+  Mutex::Locker l(mds_lock);
+  if (want_state == CEPH_MDS_STATE_DNE)
+    return false;
+
+  AuthAuthorizeHandler *authorize_handler = 0;
+  switch (peer_type) {
+  case CEPH_ENTITY_TYPE_MDS:
+    authorize_handler = authorize_handler_cluster_registry->get_handler(protocol);
+    break;
+  default:
+    authorize_handler = authorize_handler_service_registry->get_handler(protocol);
+  }
+  if (!authorize_handler) {
+    dout(0) << "No AuthAuthorizeHandler found for protocol " << protocol << dendl;
+    is_valid = false;
+    return true;
+  }
+
+  AuthCapsInfo caps_info;
+  EntityName name;
+  uint64_t global_id;
+
+  is_valid = authorize_handler->verify_authorizer(cct, monc->rotating_secrets,
+						  authorizer_data, authorizer_reply, name, global_id, caps_info, session_key);
+
+  if (is_valid) {
+    // wire up a Session* to this connection, and add it to the session map
+    entity_name_t n(con->get_peer_type(), global_id);
+    Session *s = sessionmap.get_session(n);
+    if (!s) {
+      s = new Session;
+      s->info.inst.addr = con->get_peer_addr();
+      s->info.inst.name = n;
+      dout(10) << " new session " << s << " for " << s->info.inst << " con " << con << dendl;
+      con->set_priv(s);
+      s->connection = con;
+      sessionmap.add_session(s);
+    } else {
+      dout(10) << " existing session " << s << " for " << s->info.inst << " existing con " << s->connection
+	       << ", new/authorizing con " << con << dendl;
+      con->set_priv(s->get());
+
+      // Wait until we fully accept the connection before setting
+      // s->connection.  In particular, if there are multiple incoming
+      // connection attempts, they will all get their authorizer
+      // validated, but some of them may "lose the race" and get
+      // dropped.  We only want to consider the winner(s).  See
+      // ms_handle_accept().  This is important for Sessions we replay
+      // from the journal on recovery that don't have established
+      // messenger state; we want the con from only the winning
+      // connect attempt(s).  (Normal reconnects that don't follow MDS
+      // recovery are reconnected to the existing con by the
+      // messenger.)
+    }
+
+    /*
+    s->caps.set_allow_all(caps_info.allow_all);
+ 
+    if (caps_info.caps.length() > 0) {
+      bufferlist::iterator iter = caps_info.caps.begin();
+      s->caps.parse(iter);
+      dout(10) << " session " << s << " has caps " << s->caps << dendl;
+    }
+    */
+  }
+
+  return true;  // we made a decision (see is_valid)
+};
+
+
+void MDS::ms_handle_accept(Connection *con)
+{
+  Mutex::Locker l(mds_lock);
+  Session *s = static_cast(con->get_priv());
+  dout(10) << "ms_handle_accept " << con->get_peer_addr() << " con " << con << " session " << s << dendl;
+  if (s) {
+    if (s->connection != con) {
+      dout(10) << " session connection " << s->connection << " -> " << con << dendl;
+      s->connection = con;
+
+      // send out any queued messages
+      while (!s->preopen_out_queue.empty()) {
+	messenger->send_message(s->preopen_out_queue.front(), con);
+	s->preopen_out_queue.pop_front();
+      }
+    }
+    s->put();
+  }
+}
diff --git a/ceph/src/mds/MDS.h b/ceph/src/mds/MDS.h
new file mode 100644
index 00000000..1cd6096b
--- /dev/null
+++ b/ceph/src/mds/MDS.h
@@ -0,0 +1,449 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MDS_H
+#define CEPH_MDS_H
+
+#include "mdstypes.h"
+
+#include "msg/Dispatcher.h"
+#include "include/CompatSet.h"
+#include "include/types.h"
+#include "include/Context.h"
+#include "common/DecayCounter.h"
+#include "common/perf_counters.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/Timer.h"
+#include "common/LogClient.h"
+
+#include "MDSMap.h"
+
+#include "SessionMap.h"
+
+
+#define CEPH_MDS_PROTOCOL    23 /* cluster internal */
+
+
+enum {
+  l_mds_first = 2000,
+  l_mds_req,
+  l_mds_reply,
+  l_mds_replyl,
+  l_mds_fw,
+  l_mds_dir_f,
+  l_mds_dir_c,
+  l_mds_dir_sp,
+  l_mds_dir_ffc,
+  l_mds_imax,
+  l_mds_i,
+  l_mds_itop,
+  l_mds_ibot,
+  l_mds_iptail,
+  l_mds_ipin,
+  l_mds_iex,
+  l_mds_icap,
+  l_mds_cap,
+  l_mds_dis,
+  l_mds_t,
+  l_mds_thit,
+  l_mds_tfw,
+  l_mds_tdis,
+  l_mds_tdirf,
+  l_mds_trino,
+  l_mds_tlock,
+  l_mds_l,
+  l_mds_q,
+  l_mds_popanyd,
+  l_mds_popnest,
+  l_mds_sm,
+  l_mds_ex,
+  l_mds_iexp,
+  l_mds_im,
+  l_mds_iim,
+  l_mds_last,
+};
+
+enum {
+  l_mdc_first = 3000,
+  l_mdc_last,
+};
+
+enum {
+  l_mdm_first = 2500,
+  l_mdm_ino,
+  l_mdm_inoa,
+  l_mdm_inos,
+  l_mdm_dir,
+  l_mdm_dira,
+  l_mdm_dirs,
+  l_mdm_dn,
+  l_mdm_dna,
+  l_mdm_dns,
+  l_mdm_cap,
+  l_mdm_capa,
+  l_mdm_caps,
+  l_mdm_rss,
+  l_mdm_heap,
+  l_mdm_malloc,
+  l_mdm_buf,
+  l_mdm_last,
+};
+
+
+
+class filepath;
+
+class MonClient;
+
+class OSDMap;
+class Objecter;
+class Filer;
+
+class Server;
+class Locker;
+class MDCache;
+class MDLog;
+class MDBalancer;
+
+class CInode;
+class CDir;
+class CDentry;
+
+class Messenger;
+class Message;
+
+class MClientRequest;
+class MClientReply;
+
+class MMDSBeacon;
+
+class InoTable;
+class SnapServer;
+class SnapClient;
+class AnchorServer;
+class AnchorClient;
+
+class MDSTableServer;
+class MDSTableClient;
+
+class AuthAuthorizeHandlerRegistry;
+
+class MDS : public Dispatcher {
+ public:
+  Mutex        mds_lock;
+  SafeTimer    timer;
+
+  AuthAuthorizeHandlerRegistry *authorize_handler_cluster_registry;
+  AuthAuthorizeHandlerRegistry *authorize_handler_service_registry;
+
+  string name;
+  int whoami;
+  int incarnation;
+
+  int standby_for_rank;
+  int standby_type;
+  string standby_for_name;
+  bool standby_replaying;  // true if current replay pass is in standby-replay mode
+
+  Messenger    *messenger;
+  MonClient    *monc;
+  MDSMap       *mdsmap;
+  OSDMap       *osdmap;
+  Objecter     *objecter;
+  Filer        *filer;       // for reading/writing to/from osds
+  LogClient    clog;
+
+  // sub systems
+  Server       *server;
+  MDCache      *mdcache;
+  Locker       *locker;
+  MDLog        *mdlog;
+  MDBalancer   *balancer;
+
+  InoTable     *inotable;
+
+  AnchorServer *anchorserver;
+  AnchorClient *anchorclient;
+
+  SnapServer   *snapserver;
+  SnapClient   *snapclient;
+
+  MDSTableClient *get_table_client(int t);
+  MDSTableServer *get_table_server(int t);
+
+  PerfCounters       *logger, *mlogger;
+
+  int orig_argc;
+  const char **orig_argv;
+
+ protected:
+  // -- MDS state --
+  int last_state;
+  int state;         // my confirmed state
+  int want_state;    // the state i want
+
+  list waiting_for_active, waiting_for_replay, waiting_for_reconnect, waiting_for_resolve;
+  list replay_queue;
+  map > waiting_for_active_peer;
+  list waiting_for_nolaggy;
+  map > waiting_for_mdsmap;
+
+  map peer_mdsmap_epoch;
+
+  ceph_tid_t last_tid;    // for mds-initiated requests (e.g. stray rename)
+
+ public:
+  void wait_for_active(Context *c) { 
+    waiting_for_active.push_back(c); 
+  }
+  void wait_for_active_peer(int who, Context *c) { 
+    waiting_for_active_peer[who].push_back(c);
+  }
+  void wait_for_replay(Context *c) { 
+    waiting_for_replay.push_back(c); 
+  }
+  void wait_for_reconnect(Context *c) {
+    waiting_for_reconnect.push_back(c);
+  }
+  void wait_for_resolve(Context *c) {
+    waiting_for_resolve.push_back(c);
+  }
+  void wait_for_mdsmap(epoch_t e, Context *c) {
+    waiting_for_mdsmap[e].push_back(c);
+  }
+  void enqueue_replay(Context *c) {
+    replay_queue.push_back(c);
+  }
+
+  int get_state() { return state; } 
+  int get_want_state() { return want_state; } 
+  bool is_creating() { return state == MDSMap::STATE_CREATING; }
+  bool is_starting() { return state == MDSMap::STATE_STARTING; }
+  bool is_standby()  { return state == MDSMap::STATE_STANDBY; }
+  bool is_replay()   { return state == MDSMap::STATE_REPLAY; }
+  bool is_standby_replay() { return state == MDSMap::STATE_STANDBY_REPLAY; }
+  bool is_resolve()  { return state == MDSMap::STATE_RESOLVE; }
+  bool is_reconnect() { return state == MDSMap::STATE_RECONNECT; }
+  bool is_rejoin()   { return state == MDSMap::STATE_REJOIN; }
+  bool is_clientreplay()   { return state == MDSMap::STATE_CLIENTREPLAY; }
+  bool is_active()   { return state == MDSMap::STATE_ACTIVE; }
+  bool is_stopping() { return state == MDSMap::STATE_STOPPING; }
+
+  bool is_oneshot_replay()   { return state == MDSMap::STATE_ONESHOT_REPLAY; }
+  bool is_any_replay() { return (is_replay() || is_standby_replay() ||
+                                 is_oneshot_replay()); }
+
+  bool is_stopped()  { return mdsmap->is_stopped(whoami); }
+
+  void request_state(int s);
+
+  ceph_tid_t issue_tid() { return ++last_tid; }
+    
+
+  // -- waiters --
+  list finished_queue;
+
+  void queue_waiter(Context *c) {
+    finished_queue.push_back(c);
+  }
+  void queue_waiters(list& ls) {
+    finished_queue.splice( finished_queue.end(), ls );
+  }
+  bool queue_one_replay() {
+    if (replay_queue.empty())
+      return false;
+    queue_waiter(replay_queue.front());
+    replay_queue.pop_front();
+    return true;
+  }
+  
+  // -- keepalive beacon --
+  version_t               beacon_last_seq;          // last seq sent to monitor
+  map  beacon_seq_stamp;         // seq # -> time sent
+  utime_t                 beacon_last_acked_stamp;  // last time we sent a beacon that got acked
+  bool was_laggy;
+  utime_t laggy_until;
+
+  bool is_laggy();
+  utime_t get_laggy_until() { return laggy_until; }
+
+  class C_MDS_BeaconSender : public Context {
+    MDS *mds;
+  public:
+    C_MDS_BeaconSender(MDS *m) : mds(m) {}
+    void finish(int r) {
+      mds->beacon_sender = 0;
+      mds->beacon_send();
+    }
+  } *beacon_sender;
+
+  // tick and other timer fun
+  class C_MDS_Tick : public Context {
+    MDS *mds;
+  public:
+    C_MDS_Tick(MDS *m) : mds(m) {}
+    void finish(int r) {
+      mds->tick_event = 0;
+      mds->tick();
+    }
+  } *tick_event;
+  void     reset_tick();
+
+  // -- client map --
+  SessionMap   sessionmap;
+  epoch_t      last_client_mdsmap_bcast;
+  //void log_clientmap(Context *c);
+
+
+  // shutdown crap
+  int req_rate;
+
+  // ino's and fh's
+ public:
+
+  int get_req_rate() { return req_rate; }
+  Session *get_session(client_t client) {
+    return sessionmap.get_session(entity_name_t::CLIENT(client.v));
+  }
+
+ private:
+  int dispatch_depth;
+  bool ms_dispatch(Message *m);
+  bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new);
+  bool ms_verify_authorizer(Connection *con, int peer_type,
+			       int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply,
+			       bool& isvalid, CryptoKey& session_key);
+  void ms_handle_accept(Connection *con);
+  void ms_handle_connect(Connection *con);
+  bool ms_handle_reset(Connection *con);
+  void ms_handle_remote_reset(Connection *con);
+
+ public:
+  MDS(const std::string &n, Messenger *m, MonClient *mc);
+  ~MDS();
+
+  // handle a signal (e.g., SIGTERM)
+  void handle_signal(int signum);
+
+  // who am i etc
+  int get_nodeid() { return whoami; }
+  MDSMap *get_mds_map() { return mdsmap; }
+  OSDMap *get_osd_map() { return osdmap; }
+
+  void send_message_mds(Message *m, int mds);
+  void forward_message_mds(Message *req, int mds);
+
+  void send_message_client_counted(Message *m, client_t client);
+  void send_message_client_counted(Message *m, Session *session);
+  void send_message_client_counted(Message *m, Connection *connection);
+  void send_message_client_counted(Message *m, const ConnectionRef& con) {
+    send_message_client_counted(m, con.get());
+  }
+  void send_message_client(Message *m, Session *session);
+  void send_message(Message *m, Connection *c);
+  void send_message(Message *m, const ConnectionRef& c) {
+    send_message(m, c.get());
+  }
+
+  // start up, shutdown
+  int init(int wanted_state=MDSMap::STATE_BOOT);
+
+  void create_logger();
+
+  void bcast_mds_map();  // to mounted clients
+
+  void boot_create();             // i am new mds.
+  void boot_start(int step=0, int r=0);    // starting|replay
+
+  void calc_recovery_set();
+
+  void replay_start();
+  void creating_done();
+  void starting_done();
+  void replay_done();
+  void standby_replay_restart();
+  void _standby_replay_restart_finish(int r, uint64_t old_read_pos);
+  class C_MDS_StandbyReplayRestart;
+  class C_MDS_StandbyReplayRestartFinish;
+
+  void reopen_log();
+
+  void resolve_start();
+  void resolve_done();
+  void reconnect_start();
+  void reconnect_done();
+  void rejoin_joint_start();
+  void rejoin_start();
+  void rejoin_done();
+  void recovery_done(int oldstate);
+  void clientreplay_start();
+  void clientreplay_done();
+  void active_start();
+  void stopping_start();
+  void stopping_done();
+
+  void handle_mds_recovery(int who);
+  void handle_mds_failure(int who);
+
+  void suicide();
+  void respawn();
+
+  void tick();
+  
+  void beacon_start();
+  void beacon_send();
+  void handle_mds_beacon(MMDSBeacon *m);
+
+  void request_osdmap(Context *c);
+
+  void inc_dispatch_depth() { ++dispatch_depth; }
+  void dec_dispatch_depth() { --dispatch_depth; }
+
+  // messages
+  bool _dispatch(Message *m);
+
+  bool is_stale_message(Message *m);
+
+  bool handle_core_message(Message *m);
+  bool handle_deferrable_message(Message *m);
+  
+  // special message types
+  void handle_command(class MMonCommand *m);
+  void handle_mds_map(class MMDSMap *m);
+};
+
+
+/* This expects to be given a reference which it is responsible for.
+ * The finish function calls functions which
+ * will put the Message exactly once.*/
+class C_MDS_RetryMessage : public Context {
+  Message *m;
+  MDS *mds;
+public:
+  C_MDS_RetryMessage(MDS *mds, Message *m) {
+    assert(m);
+    this->m = m;
+    this->mds = mds;
+  }
+  virtual void finish(int r) {
+    mds->inc_dispatch_depth();
+    mds->_dispatch(m);
+    mds->dec_dispatch_depth();
+  }
+};
+
+#endif
diff --git a/ceph/src/mds/MDSMap.cc b/ceph/src/mds/MDSMap.cc
new file mode 100644
index 00000000..c641358c
--- /dev/null
+++ b/ceph/src/mds/MDSMap.cc
@@ -0,0 +1,574 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include "MDSMap.h"
+
+#include 
+using std::stringstream;
+
+
+// features
+CompatSet get_mdsmap_compat_set_all() {
+  CompatSet::FeatureSet feature_compat;
+  CompatSet::FeatureSet feature_ro_compat;
+  CompatSet::FeatureSet feature_incompat;
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_BASE);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_CLIENTRANGES);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_FILELAYOUT);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_DIRINODE);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_ENCODING);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_OMAPDIRFRAG);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_INLINE);
+
+  return CompatSet(feature_compat, feature_ro_compat, feature_incompat);
+}
+
+CompatSet get_mdsmap_compat_set_default() {
+  CompatSet::FeatureSet feature_compat;
+  CompatSet::FeatureSet feature_ro_compat;
+  CompatSet::FeatureSet feature_incompat;
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_BASE);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_CLIENTRANGES);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_FILELAYOUT);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_DIRINODE);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_ENCODING);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_OMAPDIRFRAG);
+
+  return CompatSet(feature_compat, feature_ro_compat, feature_incompat);
+}
+
+// base (pre v0.20)
+CompatSet get_mdsmap_compat_set_base() {
+  CompatSet::FeatureSet feature_compat_base;
+  CompatSet::FeatureSet feature_incompat_base;
+  feature_incompat_base.insert(MDS_FEATURE_INCOMPAT_BASE);
+  CompatSet::FeatureSet feature_ro_compat_base;
+
+  return CompatSet(feature_compat_base, feature_incompat_base, feature_ro_compat_base);
+}
+
+void MDSMap::mds_info_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("gid", global_id);
+  f->dump_string("name", name);
+  f->dump_int("rank", rank);
+  f->dump_int("incarnation", inc);
+  f->dump_stream("state") << ceph_mds_state_name(state);
+  f->dump_int("state_seq", state_seq);
+  f->dump_stream("addr") << addr;
+  if (laggy_since != utime_t())
+    f->dump_stream("laggy_since") << laggy_since;
+  
+  f->dump_int("standby_for_rank", standby_for_rank);
+  f->dump_string("standby_for_name", standby_for_name);
+  f->open_array_section("export_targets");
+  for (set::iterator p = export_targets.begin();
+       p != export_targets.end(); ++p) {
+    f->dump_int("mds", *p);
+  }
+  f->close_section();
+}
+
+void MDSMap::mds_info_t::generate_test_instances(list& ls)
+{
+  mds_info_t *sample = new mds_info_t();
+  ls.push_back(sample);
+  sample = new mds_info_t();
+  sample->global_id = 1;
+  sample->name = "test_instance";
+  sample->rank = 0;
+  ls.push_back(sample);
+}
+
+void MDSMap::dump(Formatter *f) const
+{
+  f->dump_int("epoch", epoch);
+  f->dump_unsigned("flags", flags);
+  f->dump_stream("created") << created;
+  f->dump_stream("modified") << modified;
+  f->dump_int("tableserver", tableserver);
+  f->dump_int("root", root);
+  f->dump_int("session_timeout", session_timeout);
+  f->dump_int("session_autoclose", session_autoclose);
+  f->dump_int("max_file_size", max_file_size);
+  f->dump_int("last_failure", last_failure);
+  f->dump_int("last_failure_osd_epoch", last_failure_osd_epoch);
+  f->open_object_section("compat");
+  compat.dump(f);
+  f->close_section();
+  f->dump_int("max_mds", max_mds);
+  f->open_array_section("in");
+  for (set::const_iterator p = in.begin(); p != in.end(); ++p)
+    f->dump_int("mds", *p);
+  f->close_section();
+  f->open_object_section("up");
+  for (map::const_iterator p = up.begin(); p != up.end(); ++p) {
+    char s[10];
+    sprintf(s, "mds_%d", p->first);
+    f->dump_int(s, p->second);
+  }
+  f->close_section();
+  f->open_array_section("failed");
+  for (set::const_iterator p = failed.begin(); p != failed.end(); ++p)
+    f->dump_int("mds", *p);
+  f->close_section();
+  f->open_array_section("stopped");
+  for (set::const_iterator p = stopped.begin(); p != stopped.end(); ++p)
+    f->dump_int("mds", *p);
+  f->close_section();
+  f->open_object_section("info");
+  for (map::const_iterator p = mds_info.begin(); p != mds_info.end(); ++p) {
+    char s[25]; // 'gid_' + len(str(ULLONG_MAX)) + '\0'
+    sprintf(s, "gid_%llu", (long long unsigned)p->first);
+    f->open_object_section(s);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("data_pools");
+  for (set::const_iterator p = data_pools.begin(); p != data_pools.end(); ++p)
+    f->dump_int("pool", *p);
+  f->close_section();
+  f->dump_int("metadata_pool", metadata_pool);
+}
+
+void MDSMap::generate_test_instances(list& ls)
+{
+  MDSMap *m = new MDSMap();
+  m->max_mds = 1;
+  m->data_pools.insert(0);
+  m->metadata_pool = 1;
+  m->cas_pool = 2;
+  m->compat = get_mdsmap_compat_set_all();
+
+  // these aren't the defaults, just in case anybody gets confused
+  m->session_timeout = 61;
+  m->session_autoclose = 301;
+  m->max_file_size = 1<<24;
+  ls.push_back(m);
+}
+
+void MDSMap::print(ostream& out) 
+{
+  out << "epoch\t" << epoch << "\n";
+  out << "flags\t" << hex << flags << dec << "\n";
+  out << "created\t" << created << "\n";
+  out << "modified\t" << modified << "\n";
+  out << "tableserver\t" << tableserver << "\n";
+  out << "root\t" << root << "\n";
+  out << "session_timeout\t" << session_timeout << "\n"
+      << "session_autoclose\t" << session_autoclose << "\n";
+  out << "max_file_size\t" << max_file_size << "\n";
+  out << "last_failure\t" << last_failure << "\n"
+      << "last_failure_osd_epoch\t" << last_failure_osd_epoch << "\n";
+  out << "compat\t" << compat << "\n";
+  out << "max_mds\t" << max_mds << "\n";
+  out << "in\t" << in << "\n"
+      << "up\t" << up << "\n"
+      << "failed\t" << failed << "\n"
+      << "stopped\t" << stopped << "\n";
+  out << "data_pools\t" << data_pools << "\n";
+  out << "metadata_pool\t" << metadata_pool << "\n";
+  out << "inline_data\t" << (inline_data_enabled ? "enabled" : "disabled") << "\n";
+
+  multimap< pair, uint64_t > foo;
+  for (map::iterator p = mds_info.begin();
+       p != mds_info.end();
+       ++p)
+    foo.insert(pair,uint64_t>(pair(p->second.rank, p->second.inc-1), p->first));
+
+  for (multimap< pair, uint64_t >::iterator p = foo.begin();
+       p != foo.end();
+       ++p) {
+    mds_info_t& info = mds_info[p->second];
+    
+    out << p->second << ":\t"
+	<< info.addr
+	<< " '" << info.name << "'"
+	<< " mds." << info.rank
+	<< "." << info.inc
+	<< " " << ceph_mds_state_name(info.state)
+	<< " seq " << info.state_seq;
+    if (info.laggy())
+      out << " laggy since " << info.laggy_since;
+    if (info.standby_for_rank != -1 ||
+	!info.standby_for_name.empty()) {
+      out << " (standby for";
+      //if (info.standby_for_rank >= 0)
+	out << " rank " << info.standby_for_rank;
+      if (!info.standby_for_name.empty())
+	out << " '" << info.standby_for_name << "'";
+      out << ")";
+    }
+    if (!info.export_targets.empty())
+      out << " export_targets=" << info.export_targets;
+    out << "\n";    
+  }
+}
+
+
+
+void MDSMap::print_summary(Formatter *f, ostream *out)
+{
+  map by_rank;
+  map by_state;
+
+  if (f) {
+    f->dump_unsigned("epoch", get_epoch());
+    f->dump_unsigned("up", up.size());
+    f->dump_unsigned("in", in.size());
+    f->dump_unsigned("max", max_mds);
+  } else {
+    *out << "e" << get_epoch() << ": " << up.size() << "/" << in.size() << "/" << max_mds << " up";
+  }
+
+  if (f)
+    f->open_array_section("by_rank");
+  for (map::iterator p = mds_info.begin();
+       p != mds_info.end();
+       ++p) {
+    string s = ceph_mds_state_name(p->second.state);
+    if (p->second.laggy())
+      s += "(laggy or crashed)";
+
+    if (p->second.rank >= 0) {
+      if (f) {
+	f->open_object_section("mds");
+	f->dump_unsigned("rank", p->second.rank);
+	f->dump_string("name", p->second.name);
+	f->dump_string("status", s);
+	f->close_section();
+      } else {
+	by_rank[p->second.rank] = p->second.name + "=" + s;
+      }
+    } else {
+      by_state[s]++;
+    }
+  }
+  if (f) {
+    f->close_section();
+  } else {
+    if (!by_rank.empty())
+      *out << " " << by_rank;
+  }
+
+  for (map::reverse_iterator p = by_state.rbegin(); p != by_state.rend(); ++p) {
+    if (f) {
+      f->dump_unsigned(p->first.c_str(), p->second);
+    } else {
+      *out << ", " << p->second << " " << p->first;
+    }
+  }
+
+  if (!failed.empty()) {
+    if (f) {
+      f->dump_unsigned("failed", failed.size());
+    } else {
+      *out << ", " << failed.size() << " failed";
+    }
+  }
+  //if (stopped.size())
+  //out << ", " << stopped.size() << " stopped";
+}
+
+void MDSMap::get_health(list >& summary,
+			list > *detail) const
+{
+  if (!failed.empty()) {
+    std::ostringstream oss;
+    oss << "mds rank"
+	<< ((failed.size() > 1) ? "s ":" ")
+	<< failed
+	<< ((failed.size() > 1) ? " have":" has")
+	<< " failed";
+    summary.push_back(make_pair(HEALTH_ERR, oss.str()));
+    if (detail) {
+      for (set::const_iterator p = failed.begin(); p != failed.end(); ++p) {
+	std::ostringstream oss;
+	oss << "mds." << *p << " has failed";
+	detail->push_back(make_pair(HEALTH_ERR, oss.str()));
+      }
+    }
+  }
+
+  if (is_degraded()) {
+    summary.push_back(make_pair(HEALTH_WARN, "mds cluster is degraded"));
+    if (detail) {
+      detail->push_back(make_pair(HEALTH_WARN, "mds cluster is degraded"));
+      for (unsigned i=0; i< get_max_mds(); i++) {
+	if (!is_up(i))
+	  continue;
+	uint64_t gid = up.find(i)->second;
+	map::const_iterator info = mds_info.find(gid);
+	stringstream ss;
+	if (is_resolve(i))
+	  ss << "mds." << info->second.name << " at " << info->second.addr << " rank " << i << " is resolving";
+	if (is_replay(i))
+	  ss << "mds." << info->second.name << " at " << info->second.addr << " rank " << i << " is replaying journal";
+	if (is_rejoin(i))
+	  ss << "mds." << info->second.name << " at " << info->second.addr << " rank " << i << " is rejoining";
+	if (is_reconnect(i))
+	  ss << "mds." << info->second.name << " at " << info->second.addr << " rank " << i << " is reconnecting to clients";
+	if (ss.str().length())
+	  detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+  }
+
+  map::const_iterator u = up.begin();
+  map::const_iterator u_end = up.end();
+  map::const_iterator m_end = mds_info.end();
+  set laggy;
+  for (; u != u_end; ++u) {
+    map::const_iterator m = mds_info.find(u->second);
+    assert(m != m_end);
+    const mds_info_t &mds_info(m->second);
+    if (mds_info.laggy()) {
+      laggy.insert(mds_info.name);
+      if (detail) {
+	std::ostringstream oss;
+	oss << "mds." << mds_info.name << " at " << mds_info.addr << " is laggy/unresponsive";
+	detail->push_back(make_pair(HEALTH_WARN, oss.str()));
+      }
+    }
+  }
+
+  if (!laggy.empty()) {
+    std::ostringstream oss;
+    oss << "mds " << laggy
+	<< ((laggy.size() > 1) ? " are":" is")
+	<< " laggy";
+    summary.push_back(make_pair(HEALTH_WARN, oss.str()));
+  }
+}
+
+void MDSMap::mds_info_t::encode_versioned(bufferlist& bl, uint64_t features) const
+{
+  ENCODE_START(4, 4, bl);
+  ::encode(global_id, bl);
+  ::encode(name, bl);
+  ::encode(rank, bl);
+  ::encode(inc, bl);
+  ::encode(state, bl);
+  ::encode(state_seq, bl);
+  ::encode(addr, bl);
+  ::encode(laggy_since, bl);
+  ::encode(standby_for_rank, bl);
+  ::encode(standby_for_name, bl);
+  ::encode(export_targets, bl);
+  ENCODE_FINISH(bl);
+}
+
+void MDSMap::mds_info_t::encode_unversioned(bufferlist& bl) const
+{
+  __u8 struct_v = 3;
+  ::encode(struct_v, bl);
+  ::encode(global_id, bl);
+  ::encode(name, bl);
+  ::encode(rank, bl);
+  ::encode(inc, bl);
+  ::encode(state, bl);
+  ::encode(state_seq, bl);
+  ::encode(addr, bl);
+  ::encode(laggy_since, bl);
+  ::encode(standby_for_rank, bl);
+  ::encode(standby_for_name, bl);
+  ::encode(export_targets, bl);
+}
+
+void MDSMap::mds_info_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(4, 4, 4, bl);
+  ::decode(global_id, bl);
+  ::decode(name, bl);
+  ::decode(rank, bl);
+  ::decode(inc, bl);
+  ::decode(state, bl);
+  ::decode(state_seq, bl);
+  ::decode(addr, bl);
+  ::decode(laggy_since, bl);
+  ::decode(standby_for_rank, bl);
+  ::decode(standby_for_name, bl);
+  if (struct_v >= 2)
+    ::decode(export_targets, bl);
+  DECODE_FINISH(bl);
+}
+
+
+
+void MDSMap::encode(bufferlist& bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_PGID64) == 0) {
+    __u16 v = 2;
+    ::encode(v, bl);
+    ::encode(epoch, bl);
+    ::encode(flags, bl);
+    ::encode(last_failure, bl);
+    ::encode(root, bl);
+    ::encode(session_timeout, bl);
+    ::encode(session_autoclose, bl);
+    ::encode(max_file_size, bl);
+    ::encode(max_mds, bl);
+    __u32 n = mds_info.size();
+    ::encode(n, bl);
+    for (map::const_iterator i = mds_info.begin();
+	i != mds_info.end(); ++i) {
+      ::encode(i->first, bl);
+      ::encode(i->second, bl, features);
+    }
+    n = data_pools.size();
+    ::encode(n, bl);
+    for (set::const_iterator p = data_pools.begin(); p != data_pools.end(); ++p) {
+      n = *p;
+      ::encode(n, bl);
+    }
+
+    int32_t m = cas_pool;
+    ::encode(m, bl);
+    return;
+  } else if ((features & CEPH_FEATURE_MDSENC) == 0) {
+    __u16 v = 3;
+    ::encode(v, bl);
+    ::encode(epoch, bl);
+    ::encode(flags, bl);
+    ::encode(last_failure, bl);
+    ::encode(root, bl);
+    ::encode(session_timeout, bl);
+    ::encode(session_autoclose, bl);
+    ::encode(max_file_size, bl);
+    ::encode(max_mds, bl);
+    __u32 n = mds_info.size();
+    ::encode(n, bl);
+    for (map::const_iterator i = mds_info.begin();
+	i != mds_info.end(); ++i) {
+      ::encode(i->first, bl);
+      ::encode(i->second, bl, features);
+    }
+    ::encode(data_pools, bl);
+    ::encode(cas_pool, bl);
+
+    // kclient ignores everything from here
+    __u16 ev = 5;
+    ::encode(ev, bl);
+    ::encode(compat, bl);
+    ::encode(metadata_pool, bl);
+    ::encode(created, bl);
+    ::encode(modified, bl);
+    ::encode(tableserver, bl);
+    ::encode(in, bl);
+    ::encode(inc, bl);
+    ::encode(up, bl);
+    ::encode(failed, bl);
+    ::encode(stopped, bl);
+    ::encode(last_failure_osd_epoch, bl);
+  } else {// have MDS encoding feature!
+    ENCODE_START(4, 4, bl);
+    ::encode(epoch, bl);
+    ::encode(flags, bl);
+    ::encode(last_failure, bl);
+    ::encode(root, bl);
+    ::encode(session_timeout, bl);
+    ::encode(session_autoclose, bl);
+    ::encode(max_file_size, bl);
+    ::encode(max_mds, bl);
+    ::encode(mds_info, bl, features);
+    ::encode(data_pools, bl);
+    ::encode(cas_pool, bl);
+
+    // kclient ignores everything from here
+    __u16 ev = 7;
+    ::encode(ev, bl);
+    ::encode(compat, bl);
+    ::encode(metadata_pool, bl);
+    ::encode(created, bl);
+    ::encode(modified, bl);
+    ::encode(tableserver, bl);
+    ::encode(in, bl);
+    ::encode(inc, bl);
+    ::encode(up, bl);
+    ::encode(failed, bl);
+    ::encode(stopped, bl);
+    ::encode(last_failure_osd_epoch, bl);
+    ::encode(ever_allowed_snaps, bl);
+    ::encode(explicitly_allowed_snaps, bl);
+    ::encode(inline_data_enabled, bl);
+    ENCODE_FINISH(bl);
+  }
+}
+
+void MDSMap::decode(bufferlist::iterator& p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN_16(4, 4, 4, p);
+  ::decode(epoch, p);
+  ::decode(flags, p);
+  ::decode(last_failure, p);
+  ::decode(root, p);
+  ::decode(session_timeout, p);
+  ::decode(session_autoclose, p);
+  ::decode(max_file_size, p);
+  ::decode(max_mds, p);
+  ::decode(mds_info, p);
+  if (struct_v < 3) {
+    __u32 n;
+    ::decode(n, p);
+    while (n--) {
+      __u32 m;
+      ::decode(m, p);
+      data_pools.insert(m);
+    }
+    __s32 s;
+    ::decode(s, p);
+    cas_pool = s;
+  } else {
+    ::decode(data_pools, p);
+    ::decode(cas_pool, p);
+  }
+
+  // kclient ignores everything from here
+  __u16 ev = 1;
+  if (struct_v >= 2)
+    ::decode(ev, p);
+  if (ev >= 3)
+    ::decode(compat, p);
+  else
+    compat = get_mdsmap_compat_set_base();
+  if (ev < 5) {
+    __u32 n;
+    ::decode(n, p);
+    metadata_pool = n;
+  } else {
+    ::decode(metadata_pool, p);
+  }
+  ::decode(created, p);
+  ::decode(modified, p);
+  ::decode(tableserver, p);
+  ::decode(in, p);
+  ::decode(inc, p);
+  ::decode(up, p);
+  ::decode(failed, p);
+  ::decode(stopped, p);
+  if (ev >= 4)
+    ::decode(last_failure_osd_epoch, p);
+  if (ev >= 6) {
+    ::decode(ever_allowed_snaps, p);
+    ::decode(explicitly_allowed_snaps, p);
+  } else {
+    ever_allowed_snaps = true;
+    explicitly_allowed_snaps = false;
+  }
+  if (ev >= 7)
+    ::decode(inline_data_enabled, p);
+  DECODE_FINISH(p);
+}
diff --git a/ceph/src/mds/MDSMap.h b/ceph/src/mds/MDSMap.h
new file mode 100644
index 00000000..8046a7c4
--- /dev/null
+++ b/ceph/src/mds/MDSMap.h
@@ -0,0 +1,569 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDSMAP_H
+#define CEPH_MDSMAP_H
+
+#include 
+
+#include "include/types.h"
+#include "common/Clock.h"
+#include "msg/Message.h"
+
+#include 
+#include 
+#include 
+using namespace std;
+
+#include "common/config.h"
+
+#include "include/CompatSet.h"
+#include "include/ceph_features.h"
+#include "common/Formatter.h"
+
+/*
+
+ boot  --> standby, creating, or starting.
+
+
+ dne  ---->   creating  ----->   active*
+ ^ ^___________/                /  ^ ^
+ |                             /  /  |
+ destroying                   /  /   |
+   ^                         /  /    |
+   |                        /  /     |
+ stopped <---- stopping* <-/  /      |
+      \                      /       |
+        ----- starting* ----/        |
+                                     |
+ failed                              |
+    \                                |
+     \--> replay*  --> reconnect* --> rejoin*
+
+     * = can fail
+
+*/
+
+class CephContext;
+
+extern CompatSet get_mdsmap_compat_set_all();
+extern CompatSet get_mdsmap_compat_set_default();
+extern CompatSet get_mdsmap_compat_set_base(); // pre v0.20
+
+#define MDS_FEATURE_INCOMPAT_BASE CompatSet::Feature(1, "base v0.20")
+#define MDS_FEATURE_INCOMPAT_CLIENTRANGES CompatSet::Feature(2, "client writeable ranges")
+#define MDS_FEATURE_INCOMPAT_FILELAYOUT CompatSet::Feature(3, "default file layouts on dirs")
+#define MDS_FEATURE_INCOMPAT_DIRINODE CompatSet::Feature(4, "dir inode in separate object")
+#define MDS_FEATURE_INCOMPAT_ENCODING CompatSet::Feature(5, "mds uses versioned encoding")
+#define MDS_FEATURE_INCOMPAT_OMAPDIRFRAG CompatSet::Feature(6, "dirfrag is stored in omap")
+#define MDS_FEATURE_INCOMPAT_INLINE CompatSet::Feature(7, "mds uses inline data")
+
+class MDSMap {
+public:
+  // mds states
+  /*
+  static const int STATE_DNE =        CEPH_MDS_STATE_DNE;  // down, never existed.
+  static const int STATE_DESTROYING = CEPH_MDS_STATE_DESTROYING;  // down, existing, semi-destroyed.
+  static const int STATE_FAILED =     CEPH_MDS_STATE_FAILED;  // down, active subtrees; needs to be recovered.
+  */
+  static const int STATE_STOPPED =    CEPH_MDS_STATE_STOPPED;  // down, once existed, but no subtrees. empty log.
+  static const int STATE_BOOT     =   CEPH_MDS_STATE_BOOT;  // up, boot announcement.  destiny unknown.
+
+  static const int STATE_STANDBY  =   CEPH_MDS_STATE_STANDBY;  // up, idle.  waiting for assignment by monitor.
+  static const int STATE_STANDBY_REPLAY = CEPH_MDS_STATE_STANDBY_REPLAY;  // up, replaying active node; ready to take over.
+  static const int STATE_ONESHOT_REPLAY = CEPH_MDS_STATE_REPLAYONCE; //up, replaying active node journal to verify it, then shutting down
+
+  static const int STATE_CREATING  =  CEPH_MDS_STATE_CREATING;  // up, creating MDS instance (new journal, idalloc..).
+  static const int STATE_STARTING  =  CEPH_MDS_STATE_STARTING;  // up, starting prior stopped MDS instance.
+
+  static const int STATE_REPLAY    =  CEPH_MDS_STATE_REPLAY;  // up, starting prior failed instance. scanning journal.
+  static const int STATE_RESOLVE   =  CEPH_MDS_STATE_RESOLVE;  // up, disambiguating distributed operations (import, rename, etc.)
+  static const int STATE_RECONNECT =  CEPH_MDS_STATE_RECONNECT;  // up, reconnect to clients
+  static const int STATE_REJOIN    =  CEPH_MDS_STATE_REJOIN; // up, replayed journal, rejoining distributed cache
+  static const int STATE_CLIENTREPLAY = CEPH_MDS_STATE_CLIENTREPLAY; // up, active
+  static const int STATE_ACTIVE =     CEPH_MDS_STATE_ACTIVE; // up, active
+  static const int STATE_STOPPING  =  CEPH_MDS_STATE_STOPPING; // up, exporting metadata (-> standby or out)
+
+  // indicate startup standby preferences for MDS
+  // of course, if they have a specific rank to follow, they just set that!
+  static const int MDS_NO_STANDBY_PREF = -1; // doesn't have instructions to do anything
+  static const int MDS_STANDBY_ANY = -2; // is instructed to be standby-replay, may
+                                     // or may not have specific name to follow
+  static const int MDS_STANDBY_NAME = -3; // standby for a named MDS
+  static const int MDS_MATCHED_ACTIVE = -4; // has a matched standby, which if up
+                                            // it should follow, but otherwise should
+                                            // be assigned a rank
+
+  struct mds_info_t {
+    uint64_t global_id;
+    string name;
+    int32_t rank;
+    int32_t inc;
+    int32_t state;
+    version_t state_seq;
+    entity_addr_t addr;
+    utime_t laggy_since;
+    int32_t standby_for_rank;
+    string standby_for_name;
+    set export_targets;
+
+    mds_info_t() : global_id(0), rank(-1), inc(0), state(STATE_STANDBY), state_seq(0),
+		   standby_for_rank(MDS_NO_STANDBY_PREF) { }
+
+    bool laggy() const { return !(laggy_since == utime_t()); }
+    void clear_laggy() { laggy_since = utime_t(); }
+
+    entity_inst_t get_inst() const { return entity_inst_t(entity_name_t::MDS(rank), addr); }
+
+    void encode(bufferlist& bl, uint64_t features) const {
+      if ((features & CEPH_FEATURE_MDSENC) == 0 ) encode_unversioned(bl);
+      else encode_versioned(bl, features);
+    }
+    void decode(bufferlist::iterator& p);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+  private:
+    void encode_versioned(bufferlist& bl, uint64_t features) const;
+    void encode_unversioned(bufferlist& bl) const;
+  };
+
+
+protected:
+  // base map
+  epoch_t epoch;
+  uint32_t flags;        // flags
+  epoch_t last_failure;  // mds epoch of last failure
+  epoch_t last_failure_osd_epoch; // osd epoch of last failure; any mds entering replay needs
+                                  // at least this osdmap to ensure the blacklist propagates.
+  utime_t created, modified;
+
+  int32_t tableserver;   // which MDS has anchortable, snaptable
+  int32_t root;          // which MDS has root directory
+
+  __u32 session_timeout;
+  __u32 session_autoclose;
+  uint64_t max_file_size;
+
+  set data_pools;  // file data pools available to clients (via an ioctl).  first is the default.
+  int64_t cas_pool;            // where CAS objects go
+  int64_t metadata_pool;       // where fs metadata objects go
+  
+  /*
+   * in: the set of logical mds #'s that define the cluster.  this is the set
+   *     of mds's the metadata may be distributed over.
+   * up: map from logical mds #'s to the addrs filling those roles.
+   * failed: subset of @in that are failed.
+   * stopped: set of nodes that have been initialized, but are not active.
+   *
+   *    @up + @failed = @in.  @in * @stopped = {}.
+   */
+
+  uint32_t max_mds; /* The maximum number of active MDSes. Also, the maximum rank. */
+
+  set in;              // currently defined cluster
+  map inc;     // most recent incarnation.
+  set failed, stopped; // which roles are failed or stopped
+  map up;        // who is in those roles
+  map mds_info;
+
+  bool ever_allowed_snaps; //< the cluster has ever allowed snap creation
+  bool explicitly_allowed_snaps; //< the user has explicitly enabled snap creation
+
+  bool inline_data_enabled;
+
+public:
+  CompatSet compat;
+
+  friend class MDSMonitor;
+
+public:
+  MDSMap() 
+    : epoch(0), flags(0), last_failure(0), last_failure_osd_epoch(0), tableserver(0), root(0),
+      session_timeout(0),
+      session_autoclose(0),
+      max_file_size(0),
+      cas_pool(-1),
+      metadata_pool(0),
+      max_mds(0),
+      ever_allowed_snaps(false),
+      explicitly_allowed_snaps(false),
+      inline_data_enabled(false)
+  { }
+
+  bool get_inline_data_enabled() { return inline_data_enabled; }
+  void set_inline_data_enabled(bool enabled) { inline_data_enabled = enabled; }
+
+  utime_t get_session_timeout() {
+    return utime_t(session_timeout,0);
+  }
+  uint64_t get_max_filesize() { return max_file_size; }
+  
+  int get_flags() const { return flags; }
+  int test_flag(int f) const { return flags & f; }
+  void set_flag(int f) { flags |= f; }
+  void clear_flag(int f) { flags &= ~f; }
+
+  void set_snaps_allowed() {
+    set_flag(CEPH_MDSMAP_ALLOW_SNAPS);
+    ever_allowed_snaps = true;
+    explicitly_allowed_snaps = true;
+  }
+  bool allows_snaps() { return test_flag(CEPH_MDSMAP_ALLOW_SNAPS); }
+  void clear_snaps_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_SNAPS); }
+
+  epoch_t get_epoch() const { return epoch; }
+  void inc_epoch() { epoch++; }
+
+  const utime_t& get_created() const { return created; }
+  void set_created(utime_t ct) { modified = created = ct; }
+  const utime_t& get_modified() const { return modified; }
+  void set_modified(utime_t mt) { modified = mt; }
+
+  epoch_t get_last_failure() const { return last_failure; }
+  epoch_t get_last_failure_osd_epoch() const { return last_failure_osd_epoch; }
+
+  unsigned get_max_mds() const { return max_mds; }
+  void set_max_mds(int m) { max_mds = m; }
+
+  int get_tableserver() const { return tableserver; }
+  int get_root() const { return root; }
+
+  const set &get_data_pools() const { return data_pools; }
+  int64_t get_first_data_pool() const { return *data_pools.begin(); }
+  int64_t get_cas_pool() const { return cas_pool; }
+  int64_t get_metadata_pool() const { return metadata_pool; }
+  bool is_data_pool(int64_t poolid) const {
+    return data_pools.count(poolid);
+  }
+
+  const map& get_mds_info() { return mds_info; }
+  const mds_info_t& get_mds_info_gid(uint64_t gid) {
+    assert(mds_info.count(gid));
+    return mds_info[gid];
+  }
+  const mds_info_t& get_mds_info(int m) {
+    assert(up.count(m) && mds_info.count(up[m]));
+    return mds_info[up[m]];
+  }
+  uint64_t find_mds_gid_by_name(const string& s) {
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p) {
+      if (p->second.name == s) {
+	return p->first;
+      }
+    }
+    return 0;
+  }
+
+  // counts
+  unsigned get_num_in_mds() {
+    return in.size();
+  }
+  unsigned get_num_up_mds() {
+    return up.size();
+  }
+  int get_num_failed_mds() {
+    return failed.size();
+  }
+  unsigned get_num_mds(int state) const {
+    unsigned n = 0;
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p)
+      if (p->second.state == state) ++n;
+    return n;
+  }
+
+  // data pools
+  void add_data_pool(int64_t poolid) {
+    data_pools.insert(poolid);
+  }
+  int remove_data_pool(int64_t poolid) {
+    set::iterator p = data_pools.find(poolid);
+    if (p == data_pools.end())
+      return -ENOENT;
+    data_pools.erase(p);
+    return 0;
+  }
+
+  // sets
+  void get_mds_set(set& s) {
+    s = in;
+  }
+  void get_up_mds_set(set& s) {
+    for (map::const_iterator p = up.begin();
+	 p != up.end();
+	 ++p)
+      s.insert(p->first);
+  }
+  void get_active_mds_set(set& s) {
+    get_mds_set(s, MDSMap::STATE_ACTIVE);
+  }
+  void get_failed_mds_set(set& s) {
+    s = failed;
+  }
+  int get_failed() {
+    if (!failed.empty()) return *failed.begin();
+    return -1;
+  }
+  void get_stopped_mds_set(set& s) {
+    s = stopped;
+  }
+  void get_recovery_mds_set(set& s) {
+    s = failed;
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p)
+      if (p->second.state >= STATE_REPLAY && p->second.state <= STATE_STOPPING)
+	s.insert(p->second.rank);
+  }
+  void get_clientreplay_or_active_or_stopping_mds_set(set& s) {
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p)
+      if (p->second.state >= STATE_CLIENTREPLAY && p->second.state <= STATE_STOPPING)
+	s.insert(p->second.rank);
+  }
+  void get_mds_set(set& s, int state) {
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p)
+      if (p->second.state == state)
+	s.insert(p->second.rank);
+  } 
+
+  int get_random_up_mds() {
+    if (up.empty())
+      return -1;
+    map::iterator p = up.begin();
+    for (int n = rand() % up.size(); n; n--)
+      ++p;
+    return p->first;
+  }
+
+  const mds_info_t* find_by_name(const string& name) const {
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p) {
+      if (p->second.name == name)
+	return &p->second;
+    }
+    return NULL;
+  }
+
+  uint64_t find_standby_for(int mds, string& name) {
+    map::const_iterator generic_standby
+      = mds_info.end();
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p) {
+      if ((p->second.state != MDSMap::STATE_STANDBY && p->second.state != MDSMap::STATE_STANDBY_REPLAY) ||
+	  p->second.laggy() ||
+	  p->second.rank >= 0)
+	continue;
+      if (p->second.standby_for_rank == mds || (name.length() && p->second.standby_for_name == name))
+	return p->first;
+      if (p->second.standby_for_rank < 0 && p->second.standby_for_name.length() == 0)
+	generic_standby = p;
+    }
+    if (generic_standby != mds_info.end())
+      return generic_standby->first;
+    return 0;
+  }
+  uint64_t find_unused_for(int mds, string& name) {
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p) {
+      if (p->second.state != MDSMap::STATE_STANDBY ||
+	  p->second.laggy() ||
+	  p->second.rank >= 0)
+	continue;
+      if ((p->second.standby_for_rank == MDS_NO_STANDBY_PREF ||
+	   p->second.standby_for_rank == MDS_MATCHED_ACTIVE ||
+	   (p->second.standby_for_rank == MDS_STANDBY_ANY && g_conf->mon_force_standby_active))) {
+	return p->first;
+      }
+    }
+    return 0;
+  }
+  uint64_t find_replacement_for(int mds, string& name) {
+    uint64_t standby = find_standby_for(mds, name);
+    if (standby)
+      return standby;
+    else
+      return find_unused_for(mds, name);
+  }
+
+  void get_health(list >& summary,
+		  list > *detail) const;
+
+  // mds states
+  bool is_down(int m) const { return up.count(m) == 0; }
+  bool is_up(int m) const { return up.count(m); }
+  bool is_in(int m) const { return up.count(m) || failed.count(m); }
+  bool is_out(int m) const { return !is_in(m); }
+
+  bool is_failed(int m) const   { return failed.count(m); }
+  bool is_stopped(int m) const    { return stopped.count(m); }
+
+  bool is_dne(int m) const      { return in.count(m) == 0; }
+  bool is_dne_gid(uint64_t gid) const     { return mds_info.count(gid) == 0; }
+
+  int get_state(int m) const {
+    map::const_iterator u = up.find(m);
+    if (u == up.end())
+      return 0;
+    return get_state_gid(u->second);
+  }
+  int get_state_gid(uint64_t gid) const {
+    map::const_iterator i = mds_info.find(gid);
+    if (i == mds_info.end())
+      return 0;
+    return i->second.state;
+  }
+
+  mds_info_t& get_info(int m) { assert(up.count(m)); return mds_info[up[m]]; }
+  mds_info_t& get_info_gid(uint64_t gid) { assert(mds_info.count(gid)); return mds_info[gid]; }
+
+  bool is_boot(int m) const { return get_state(m) == STATE_BOOT; }
+  bool is_creating(int m) const { return get_state(m) == STATE_CREATING; }
+  bool is_starting(int m) const { return get_state(m) == STATE_STARTING; }
+  bool is_replay(int m) const   { return get_state(m) == STATE_REPLAY; }
+  bool is_resolve(int m) const  { return get_state(m) == STATE_RESOLVE; }
+  bool is_reconnect(int m) const { return get_state(m) == STATE_RECONNECT; }
+  bool is_rejoin(int m) const   { return get_state(m) == STATE_REJOIN; }
+  bool is_clientreplay(int m) const { return get_state(m) == STATE_CLIENTREPLAY; }
+  bool is_active(int m) const  { return get_state(m) == STATE_ACTIVE; }
+  bool is_stopping(int m) const { return get_state(m) == STATE_STOPPING; }
+  bool is_active_or_stopping(int m) const {
+    return is_active(m) || is_stopping(m);
+  }
+  bool is_clientreplay_or_active_or_stopping(int m) const {
+    return is_clientreplay(m) || is_active(m) || is_stopping(m);
+  }
+
+  bool is_followable(int m) const {
+    return (is_resolve(m) ||
+	    is_replay(m) ||
+	    is_rejoin(m) ||
+	    is_clientreplay(m) ||
+	    is_active(m) ||
+	    is_stopping(m));
+  }
+
+  bool is_laggy_gid(uint64_t gid) const {
+    if (!mds_info.count(gid))
+      return false;
+    map::const_iterator p = mds_info.find(gid);
+    return p->second.laggy();
+  }
+
+
+  // cluster states
+  bool is_full() const {
+    return in.size() >= max_mds;
+  }
+  bool is_degraded() const {   // degraded = some recovery in process.  fixes active membership and recovery_set.
+    if (!failed.empty())
+      return true;
+    for (map::const_iterator p = mds_info.begin();
+	 p != mds_info.end();
+	 ++p)
+      if (p->second.state >= STATE_REPLAY && p->second.state <= STATE_CLIENTREPLAY)
+	return true;
+    return false;
+  }
+  bool is_any_failed() {
+    return failed.size();
+  }
+  bool is_resolving() {
+    return
+      get_num_mds(STATE_RESOLVE) > 0 &&
+      get_num_mds(STATE_REPLAY) == 0 &&
+      failed.empty();
+  }
+  bool is_rejoining() {  
+    // nodes are rejoining cache state
+    return 
+      get_num_mds(STATE_REJOIN) > 0 &&
+      get_num_mds(STATE_REPLAY) == 0 &&
+      get_num_mds(STATE_RECONNECT) == 0 &&
+      get_num_mds(STATE_RESOLVE) == 0 &&
+      failed.empty();
+  }
+  bool is_stopped() {
+    return up.empty();
+  }
+
+  // inst
+  bool have_inst(int m) {
+    return up.count(m);
+  }
+  const entity_inst_t get_inst(int m) {
+    assert(up.count(m));
+    return mds_info[up[m]].get_inst();
+  }
+  const entity_addr_t get_addr(int m) {
+    assert(up.count(m));
+    return mds_info[up[m]].addr;
+  }
+  bool get_inst(int m, entity_inst_t& inst) { 
+    if (up.count(m)) {
+      inst = get_inst(m);
+      return true;
+    } 
+    return false;
+  }
+  
+  int get_rank_gid(uint64_t gid) {
+    if (mds_info.count(gid))
+      return mds_info[gid].rank;
+    return -1;
+  }
+
+  int get_inc(int m) {
+    if (up.count(m)) 
+      return mds_info[up[m]].inc;
+    return 0;
+  }
+  int get_inc_gid(uint64_t gid) {
+    if (mds_info.count(gid))
+      return mds_info[gid].inc;
+    return -1;
+  }
+  void encode(bufferlist& bl, uint64_t features) const;
+  void decode(bufferlist::iterator& p);
+  void decode(bufferlist& bl) {
+    bufferlist::iterator p = bl.begin();
+    decode(p);
+  }
+
+
+  void print(ostream& out);
+  void print_summary(Formatter *f, ostream *out);
+
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER_FEATURES(MDSMap::mds_info_t)
+WRITE_CLASS_ENCODER_FEATURES(MDSMap)
+
+inline ostream& operator<<(ostream& out, MDSMap& m) {
+  m.print_summary(NULL, &out);
+  return out;
+}
+
+#endif
diff --git a/ceph/src/mds/MDSTable.cc b/ceph/src/mds/MDSTable.cc
new file mode 100644
index 00000000..967c9ee5
--- /dev/null
+++ b/ceph/src/mds/MDSTable.cc
@@ -0,0 +1,165 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MDSTable.h"
+
+#include "MDS.h"
+#include "MDLog.h"
+
+#include "osdc/Filer.h"
+
+#include "include/types.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << (mds ? mds->get_nodeid() : -1) << "." << table_name << ": "
+
+
+class C_MT_Save : public Context {
+  MDSTable *ida;
+  version_t version;
+public:
+  C_MT_Save(MDSTable *i, version_t v) : ida(i), version(v) {}
+  void finish(int r) {
+    ida->save_2(r, version);
+  }
+};
+
+void MDSTable::save(Context *onfinish, version_t v)
+{
+  if (v > 0 && v <= committing_version) {
+    dout(10) << "save v " << version << " - already saving "
+	     << committing_version << " >= needed " << v << dendl;
+    waitfor_save[v].push_back(onfinish);
+    return;
+  }
+  
+  dout(10) << "save v " << version << dendl;
+  assert(is_active());
+  
+  bufferlist bl;
+  ::encode(version, bl);
+  encode_state(bl);
+
+  committing_version = version;
+
+  if (onfinish)
+    waitfor_save[version].push_back(onfinish);
+
+  // write (async)
+  SnapContext snapc;
+  object_t oid = get_object_name();
+  object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+  mds->objecter->write_full(oid, oloc,
+			    snapc,
+			    bl, ceph_clock_now(g_ceph_context), 0,
+			    NULL, new C_MT_Save(this, version));
+}
+
+void MDSTable::save_2(int r, version_t v)
+{
+  dout(10) << "save_2 v " << v << dendl;
+  if (r == -EBLACKLISTED) {
+    mds->suicide();
+    return;
+  }
+  if (r < 0) {
+    dout(10) << "save_2 could not write table: " << r << dendl;
+    assert(r >= 0);
+  }
+  assert(r >= 0);
+  committed_version = v;
+  
+  list ls;
+  while (!waitfor_save.empty()) {
+    if (waitfor_save.begin()->first > v) break;
+    ls.splice(ls.end(), waitfor_save.begin()->second);
+    waitfor_save.erase(waitfor_save.begin());
+  }
+  finish_contexts(g_ceph_context, ls,0);
+}
+
+
+void MDSTable::reset()
+{
+  reset_state();
+  state = STATE_ACTIVE;
+}
+
+
+
+// -----------------------
+
+class C_MT_Load : public Context {
+public:
+  MDSTable *ida;
+  Context *onfinish;
+  bufferlist bl;
+  C_MT_Load(MDSTable *i, Context *o) : ida(i), onfinish(o) {}
+  void finish(int r) {
+    ida->load_2(r, bl, onfinish);
+  }
+};
+
+object_t MDSTable::get_object_name()
+{
+  char n[50];
+  if (per_mds)
+    snprintf(n, sizeof(n), "mds%d_%s", mds->whoami, table_name);
+  else
+    snprintf(n, sizeof(n), "mds_%s", table_name);
+  return object_t(n);
+}
+
+void MDSTable::load(Context *onfinish)
+{ 
+  dout(10) << "load" << dendl;
+
+  assert(is_undef());
+  state = STATE_OPENING;
+
+  C_MT_Load *c = new C_MT_Load(this, onfinish);
+  object_t oid = get_object_name();
+  object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+  mds->objecter->read_full(oid, oloc, CEPH_NOSNAP, &c->bl, 0, c);
+}
+
+void MDSTable::load_2(int r, bufferlist& bl, Context *onfinish)
+{
+  assert(is_opening());
+  state = STATE_ACTIVE;
+  if (r == -EBLACKLISTED) {
+    mds->suicide();
+    return;
+  }
+  if (r < 0) {
+    derr << "load_2 could not read table: " << r << dendl;
+    assert(r >= 0);
+  }
+
+  dout(10) << "load_2 got " << bl.length() << " bytes" << dendl;
+  bufferlist::iterator p = bl.begin();
+  ::decode(version, p);
+  projected_version = committed_version = version;
+  dout(10) << "load_2 loaded v" << version << dendl;
+  decode_state(p);
+
+  if (onfinish) {
+    onfinish->complete(0);
+  }
+}
diff --git a/ceph/src/mds/MDSTable.h b/ceph/src/mds/MDSTable.h
new file mode 100644
index 00000000..edcb2032
--- /dev/null
+++ b/ceph/src/mds/MDSTable.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDSTABLE_H
+#define CEPH_MDSTABLE_H
+
+#include "mdstypes.h"
+#include "mds_table_types.h"
+#include "include/buffer.h"
+#include "include/Context.h"
+
+class MDS;
+
+class MDSTable {
+ protected:
+  MDS *mds;
+
+  const char *table_name;
+  bool per_mds;
+
+  object_t get_object_name();
+  
+  static const int STATE_UNDEF   = 0;
+  static const int STATE_OPENING = 1;
+  static const int STATE_ACTIVE  = 2;
+  //static const int STATE_COMMITTING = 3;
+  int state;
+  
+  version_t version, committing_version, committed_version, projected_version;
+  
+  map > waitfor_save;
+  
+public:
+  MDSTable(MDS *m, const char *n, bool is_per_mds) :
+    mds(m), table_name(n), per_mds(is_per_mds),
+    state(STATE_UNDEF),
+    version(0), committing_version(0), committed_version(0), projected_version(0) {}
+  virtual ~MDSTable() {}
+
+  version_t get_version() { return version; }
+  version_t get_committed_version() { return committed_version; }
+  version_t get_committing_version() { return committing_version; }
+  version_t get_projected_version() { return projected_version; }
+  
+  void force_replay_version(version_t v) {
+    version = projected_version = v;
+  }
+
+  //version_t project_version() { return ++projected_version; }
+  //version_t inc_version() { return ++version; }
+
+  // load/save from disk (hack)
+  bool is_undef() { return state == STATE_UNDEF; }
+  bool is_active() { return state == STATE_ACTIVE; }
+  bool is_opening() { return state == STATE_OPENING; }
+
+  void reset();
+  void save(Context *onfinish=0, version_t need=0);
+  void save_2(int r, version_t v);
+
+  void shutdown() {
+    if (is_active()) save(0);
+  }
+
+  void load(Context *onfinish);
+  void load_2(int, bufferlist&, Context *onfinish);
+
+  // child must overload these
+  virtual void reset_state() = 0;
+  virtual void decode_state(bufferlist::iterator& p) = 0;
+  virtual void encode_state(bufferlist& bl) const = 0;
+};
+
+#endif
diff --git a/ceph/src/mds/MDSTableClient.cc b/ceph/src/mds/MDSTableClient.cc
new file mode 100644
index 00000000..cc3152f1
--- /dev/null
+++ b/ceph/src/mds/MDSTableClient.cc
@@ -0,0 +1,242 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+
+#include "MDSMap.h"
+
+#include "include/Context.h"
+#include "msg/Messenger.h"
+
+#include "MDS.h"
+#include "MDLog.h"
+#include "LogSegment.h"
+
+#include "MDSTableClient.h"
+#include "events/ETableClient.h"
+
+#include "messages/MMDSTableRequest.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".tableclient(" << get_mdstable_name(table) << ") "
+
+
+void MDSTableClient::handle_request(class MMDSTableRequest *m)
+{
+  dout(10) << "handle_request " << *m << dendl;
+  assert(m->table == table);
+
+  version_t tid = m->get_tid();
+  uint64_t reqid = m->reqid;
+
+  switch (m->op) {
+  case TABLESERVER_OP_QUERY_REPLY:
+    handle_query_result(m);
+    break;
+    
+  case TABLESERVER_OP_AGREE:
+    if (pending_prepare.count(reqid)) {
+      dout(10) << "got agree on " << reqid << " atid " << tid << dendl;
+
+      assert(g_conf->mds_kill_mdstable_at != 3);
+
+      Context *onfinish = pending_prepare[reqid].onfinish;
+      *pending_prepare[reqid].ptid = tid;
+      if (pending_prepare[reqid].pbl)
+	*pending_prepare[reqid].pbl = m->bl;
+      pending_prepare.erase(reqid);
+      prepared_update[tid] = reqid;
+      if (onfinish) {
+        onfinish->complete(0);
+      }
+    }
+    else if (prepared_update.count(tid)) {
+      dout(10) << "got duplicated agree on " << reqid << " atid " << tid << dendl;
+      assert(prepared_update[tid] == reqid);
+      assert(!server_ready);
+    }
+    else if (pending_commit.count(tid)) {
+      dout(10) << "stray agree on " << reqid << " tid " << tid
+	       << ", already committing, will resend COMMIT" << dendl;
+      assert(!server_ready);
+      // will re-send commit when receiving the server ready message
+    }
+    else {
+      dout(10) << "stray agree on " << reqid << " tid " << tid
+	       << ", sending ROLLBACK" << dendl;
+      assert(!server_ready);
+      MMDSTableRequest *req = new MMDSTableRequest(table, TABLESERVER_OP_ROLLBACK, 0, tid);
+      mds->send_message_mds(req, mds->mdsmap->get_tableserver());
+    }
+    break;
+
+  case TABLESERVER_OP_ACK:
+    if (pending_commit.count(tid) &&
+	pending_commit[tid]->pending_commit_tids[table].count(tid)) {
+      dout(10) << "got ack on tid " << tid << ", logging" << dendl;
+      
+      assert(g_conf->mds_kill_mdstable_at != 7);
+      
+      // remove from committing list
+      pending_commit[tid]->pending_commit_tids[table].erase(tid);
+      pending_commit.erase(tid);
+
+      // log ACK.
+      mds->mdlog->start_submit_entry(new ETableClient(table, TABLESERVER_OP_ACK, tid),
+				     new C_LoggedAck(this, tid));
+    } else {
+      dout(10) << "got stray ack on tid " << tid << ", ignoring" << dendl;
+    }
+    break;
+
+  case TABLESERVER_OP_SERVER_READY:
+    assert(!server_ready);
+    server_ready = true;
+
+    if (last_reqid == ~0ULL)
+      last_reqid = reqid;
+
+    resend_queries();
+    resend_prepares();
+    resend_commits();
+    break;
+
+  default:
+    assert(0);
+  }
+
+  m->put();
+}
+
+
+void MDSTableClient::_logged_ack(version_t tid)
+{
+  dout(10) << "_logged_ack " << tid << dendl;
+
+  assert(g_conf->mds_kill_mdstable_at != 8);
+
+  // kick any waiters (LogSegment trim)
+  if (ack_waiters.count(tid)) {
+    dout(15) << "kicking ack waiters on tid " << tid << dendl;
+    mds->queue_waiters(ack_waiters[tid]);
+    ack_waiters.erase(tid);
+  }
+}
+
+void MDSTableClient::_prepare(bufferlist& mutation, version_t *ptid, bufferlist *pbl,
+			      Context *onfinish)
+{
+  if (last_reqid == ~0ULL) {
+    dout(10) << "tableserver is not ready yet, waiting for request id" << dendl;
+    waiting_for_reqid.push_back(_pending_prepare(onfinish, ptid, pbl, mutation));
+    return;
+  }
+
+  uint64_t reqid = ++last_reqid;
+  dout(10) << "_prepare " << reqid << dendl;
+
+  pending_prepare[reqid].mutation = mutation;
+  pending_prepare[reqid].ptid = ptid;
+  pending_prepare[reqid].pbl = pbl;
+  pending_prepare[reqid].onfinish = onfinish;
+
+  if (server_ready) {
+    // send message
+    MMDSTableRequest *req = new MMDSTableRequest(table, TABLESERVER_OP_PREPARE, reqid);
+    req->bl = mutation;
+    mds->send_message_mds(req, mds->mdsmap->get_tableserver());
+  } else
+    dout(10) << "tableserver is not ready yet, deferring request" << dendl;
+}
+
+void MDSTableClient::commit(version_t tid, LogSegment *ls)
+{
+  dout(10) << "commit " << tid << dendl;
+
+  assert(prepared_update.count(tid));
+  prepared_update.erase(tid);
+
+  assert(pending_commit.count(tid) == 0);
+  pending_commit[tid] = ls;
+  ls->pending_commit_tids[table].insert(tid);
+
+  assert(g_conf->mds_kill_mdstable_at != 4);
+
+  if (server_ready) {
+    // send message
+    MMDSTableRequest *req = new MMDSTableRequest(table, TABLESERVER_OP_COMMIT, 0, tid);
+    mds->send_message_mds(req, mds->mdsmap->get_tableserver());
+  } else
+    dout(10) << "tableserver is not ready yet, deferring request" << dendl;
+}
+
+
+
+// recovery
+
+void MDSTableClient::got_journaled_agree(version_t tid, LogSegment *ls)
+{
+  dout(10) << "got_journaled_agree " << tid << dendl;
+  ls->pending_commit_tids[table].insert(tid);
+  pending_commit[tid] = ls;
+}
+
+void MDSTableClient::got_journaled_ack(version_t tid)
+{
+  dout(10) << "got_journaled_ack " << tid << dendl;
+  if (pending_commit.count(tid)) {
+    pending_commit[tid]->pending_commit_tids[table].erase(tid);
+    pending_commit.erase(tid);
+  }
+}
+
+void MDSTableClient::resend_commits()
+{
+  for (map::iterator p = pending_commit.begin();
+       p != pending_commit.end();
+       ++p) {
+    dout(10) << "resending commit on " << p->first << dendl;
+    MMDSTableRequest *req = new MMDSTableRequest(table, TABLESERVER_OP_COMMIT, 0, p->first);
+    mds->send_message_mds(req, mds->mdsmap->get_tableserver());
+  }
+}
+
+void MDSTableClient::resend_prepares()
+{
+  while (!waiting_for_reqid.empty()) {
+    pending_prepare[++last_reqid] = waiting_for_reqid.front();
+    waiting_for_reqid.pop_front();
+  }
+
+  for (map::iterator p = pending_prepare.begin();
+       p != pending_prepare.end();
+       ++p) {
+    dout(10) << "resending prepare on " << p->first << dendl;
+    MMDSTableRequest *req = new MMDSTableRequest(table, TABLESERVER_OP_PREPARE, p->first);
+    req->bl = p->second.mutation;
+    mds->send_message_mds(req, mds->mdsmap->get_tableserver());
+  }
+}
+
+void MDSTableClient::handle_mds_failure(int who)
+{
+  if (who != mds->mdsmap->get_tableserver())
+    return; // do nothing.
+
+  dout(7) << "tableserver mds." << who << " fails" << dendl;
+  server_ready = false;
+}
diff --git a/ceph/src/mds/MDSTableClient.h b/ceph/src/mds/MDSTableClient.h
new file mode 100644
index 00000000..16b14c4e
--- /dev/null
+++ b/ceph/src/mds/MDSTableClient.h
@@ -0,0 +1,102 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDSTABLECLIENT_H
+#define CEPH_MDSTABLECLIENT_H
+
+#include "include/types.h"
+#include "include/Context.h"
+#include "mds_table_types.h"
+
+class MDS;
+class LogSegment;
+class MMDSTableRequest;
+
+class MDSTableClient {
+protected:
+  MDS *mds;
+  int table;
+
+  uint64_t last_reqid;
+
+  bool server_ready;
+
+  // prepares
+  struct _pending_prepare {
+    Context *onfinish;
+    version_t *ptid;
+    bufferlist *pbl; 
+    bufferlist mutation;
+
+    _pending_prepare() : onfinish(0), ptid(0), pbl(0) {}
+    _pending_prepare(Context *c, version_t *pt, bufferlist *pb, bufferlist& m) :
+      onfinish(c), ptid(pt), pbl(pb), mutation(m) {}
+  };
+
+  map pending_prepare;
+  map prepared_update;
+  list<_pending_prepare> waiting_for_reqid;
+
+  // pending commits
+  map pending_commit;
+  map > ack_waiters;
+
+  void handle_reply(class MMDSTableQuery *m);  
+
+  class C_LoggedAck : public Context {
+    MDSTableClient *tc;
+    version_t tid;
+  public:
+    C_LoggedAck(MDSTableClient *a, version_t t) : tc(a), tid(t) {}
+    void finish(int r) {
+      tc->_logged_ack(tid);
+    }
+  };
+  void _logged_ack(version_t tid);
+
+public:
+  MDSTableClient(MDS *m, int tab) :
+    mds(m), table(tab), last_reqid(~0ULL), server_ready(false) {}
+  virtual ~MDSTableClient() {}
+
+  void handle_request(MMDSTableRequest *m);
+
+  void _prepare(bufferlist& mutation, version_t *ptid, bufferlist *pbl, Context *onfinish);
+  void commit(version_t tid, LogSegment *ls);
+
+  void resend_commits();
+  void resend_prepares();
+
+  // for recovery (by me)
+  void got_journaled_agree(version_t tid, LogSegment *ls);
+  void got_journaled_ack(version_t tid);
+
+  bool has_committed(version_t tid) {
+    return pending_commit.count(tid) == 0;
+  }
+  void wait_for_ack(version_t tid, Context *c) {
+    ack_waiters[tid].push_back(c);
+  }
+
+  void handle_mds_failure(int mds);
+
+  // child must implement
+  virtual void resend_queries() = 0;
+  virtual void handle_query_result(MMDSTableRequest *m) = 0;
+
+  // and friendly front-end for _prepare.
+
+};
+
+#endif
diff --git a/ceph/src/mds/MDSTableServer.cc b/ceph/src/mds/MDSTableServer.cc
new file mode 100644
index 00000000..b7752468
--- /dev/null
+++ b/ceph/src/mds/MDSTableServer.cc
@@ -0,0 +1,176 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MDSTableServer.h"
+#include "MDS.h"
+#include "MDLog.h"
+#include "msg/Messenger.h"
+
+#include "messages/MMDSTableRequest.h"
+#include "events/ETableServer.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".tableserver(" << get_mdstable_name(table) << ") "
+
+/* This function DOES put the passed message before returning */
+void MDSTableServer::handle_request(MMDSTableRequest *req)
+{
+  assert(req->op >= 0);
+  switch (req->op) {
+  case TABLESERVER_OP_QUERY: return handle_query(req);
+  case TABLESERVER_OP_PREPARE: return handle_prepare(req);
+  case TABLESERVER_OP_COMMIT: return handle_commit(req);
+  case TABLESERVER_OP_ROLLBACK: return handle_rollback(req);
+  default: assert(0);
+  }
+}
+
+// prepare
+/* This function DOES put the passed message before returning */
+void MDSTableServer::handle_prepare(MMDSTableRequest *req)
+{
+  dout(7) << "handle_prepare " << *req << dendl;
+  int from = req->get_source().num();
+  bufferlist bl = req->bl;
+
+  _prepare(req->bl, req->reqid, from);
+  _note_prepare(from, req->reqid);
+
+  assert(g_conf->mds_kill_mdstable_at != 1);
+
+  ETableServer *le = new ETableServer(table, TABLESERVER_OP_PREPARE, req->reqid, from, version, version);
+  mds->mdlog->start_entry(le);
+  le->mutation = bl;  // original request, NOT modified return value coming out of _prepare!
+  mds->mdlog->submit_entry(le, new C_Prepare(this, req, version));
+  mds->mdlog->flush();
+}
+
+void MDSTableServer::_prepare_logged(MMDSTableRequest *req, version_t tid)
+{
+  dout(7) << "_create_logged " << *req << " tid " << tid << dendl;
+
+  assert(g_conf->mds_kill_mdstable_at != 2);
+
+  MMDSTableRequest *reply = new MMDSTableRequest(table, TABLESERVER_OP_AGREE, req->reqid, tid);
+  reply->bl = req->bl;
+  mds->send_message_mds(reply, req->get_source().num());
+  req->put();
+}
+
+
+// commit
+/* This function DOES put the passed message before returning */
+void MDSTableServer::handle_commit(MMDSTableRequest *req)
+{
+  dout(7) << "handle_commit " << *req << dendl;
+
+  version_t tid = req->get_tid();
+
+  if (pending_for_mds.count(tid)) {
+
+    assert(g_conf->mds_kill_mdstable_at != 5);
+
+    if (!_commit(tid, req))
+      return;
+
+    _note_commit(tid);
+    mds->mdlog->start_submit_entry(new ETableServer(table, TABLESERVER_OP_COMMIT, 0, -1, 
+						    tid, version));
+    mds->mdlog->wait_for_safe(new C_Commit(this, req));
+  }
+  else if (tid <= version) {
+    dout(0) << "got commit for tid " << tid << " <= " << version 
+	    << ", already committed, sending ack." 
+	    << dendl;
+    _commit_logged(req);
+  } 
+  else {
+    // wtf.
+    dout(0) << "got commit for tid " << tid << " > " << version << dendl;
+    assert(tid <= version);
+  }
+}
+
+/* This function DOES put the passed message before returning */
+void MDSTableServer::_commit_logged(MMDSTableRequest *req)
+{
+  dout(7) << "_commit_logged, sending ACK" << dendl;
+
+  assert(g_conf->mds_kill_mdstable_at != 6);
+
+  MMDSTableRequest *reply = new MMDSTableRequest(table, TABLESERVER_OP_ACK, req->reqid, req->get_tid());
+  mds->send_message_mds(reply, req->get_source().num());
+  req->put();
+}
+
+// ROLLBACK
+/* This function DOES put the passed message before returning */
+void MDSTableServer::handle_rollback(MMDSTableRequest *req)
+{
+  dout(7) << "handle_rollback " << *req << dendl;
+
+  version_t tid = req->get_tid();
+  assert(pending_for_mds.count(tid));
+  _rollback(tid);
+  _note_rollback(tid);
+  mds->mdlog->start_submit_entry(new ETableServer(table, TABLESERVER_OP_ROLLBACK, 0, -1, 
+						  tid, version));
+  req->put();
+}
+
+
+
+// SERVER UPDATE
+
+void MDSTableServer::do_server_update(bufferlist& bl)
+{
+  dout(10) << "do_server_update len " << bl.length() << dendl;
+  _server_update(bl);
+  ETableServer *le = new ETableServer(table, TABLESERVER_OP_SERVER_UPDATE, 0, -1, 0, version);
+  mds->mdlog->start_entry(le);
+  le->mutation = bl;
+  mds->mdlog->submit_entry(le);
+}
+
+
+// recovery
+
+void MDSTableServer::finish_recovery(set& active)
+{
+  dout(7) << "finish_recovery" << dendl;
+  for (set::iterator p = active.begin(); p != active.end(); ++p)
+    handle_mds_recovery(*p);  // resend agrees for everyone.
+}
+
+void MDSTableServer::handle_mds_recovery(int who)
+{
+  dout(7) << "handle_mds_recovery mds." << who << dendl;
+
+  uint64_t next_reqid = 0;
+  // resend agrees for recovered mds
+  for (map::iterator p = pending_for_mds.begin();
+       p != pending_for_mds.end();
+       ++p) {
+    if (p->second.mds != who)
+      continue;
+    if (p->second.reqid >= next_reqid)
+      next_reqid = p->second.reqid + 1;
+    MMDSTableRequest *reply = new MMDSTableRequest(table, TABLESERVER_OP_AGREE, p->second.reqid, p->second.tid);
+    mds->send_message_mds(reply, who);
+  }
+
+  MMDSTableRequest *reply = new MMDSTableRequest(table, TABLESERVER_OP_SERVER_READY, next_reqid);
+  mds->send_message_mds(reply, who);
+}
diff --git a/ceph/src/mds/MDSTableServer.h b/ceph/src/mds/MDSTableServer.h
new file mode 100644
index 00000000..223fc91f
--- /dev/null
+++ b/ceph/src/mds/MDSTableServer.h
@@ -0,0 +1,97 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDSTABLESERVER_H
+#define CEPH_MDSTABLESERVER_H
+
+#include "MDSTable.h"
+
+class MMDSTableRequest;
+
+class MDSTableServer : public MDSTable {
+public:
+  int table;
+  map pending_for_mds;  // ** child should encode this! **
+
+
+private:
+  void handle_prepare(MMDSTableRequest *m);
+  void _prepare_logged(MMDSTableRequest *m, version_t tid);
+  struct C_Prepare : public Context {
+    MDSTableServer *server;
+    MMDSTableRequest *req;
+    version_t tid;
+    C_Prepare(MDSTableServer *s, MMDSTableRequest *r, version_t v) : server(s), req(r), tid(v) {}
+    void finish(int r) {
+      server->_prepare_logged(req, tid);
+    }
+  };
+
+  void handle_commit(MMDSTableRequest *m);
+  void _commit_logged(MMDSTableRequest *m);
+  struct C_Commit : public Context {
+    MDSTableServer *server;
+    MMDSTableRequest *req;
+    C_Commit(MDSTableServer *s, MMDSTableRequest *r) : server(s), req(r) {}
+    void finish(int r) {
+      server->_commit_logged(req);
+    }
+  };
+
+  void handle_rollback(MMDSTableRequest *m);
+
+ public:
+  virtual void handle_query(MMDSTableRequest *m) = 0;
+  virtual void _prepare(bufferlist &bl, uint64_t reqid, int bymds) = 0;
+  virtual bool _commit(version_t tid, MMDSTableRequest *req=NULL) = 0;
+  virtual void _rollback(version_t tid) = 0;
+  virtual void _server_update(bufferlist& bl) { assert(0); }
+
+  void _note_prepare(int mds, uint64_t reqid) {
+    pending_for_mds[version].mds = mds;
+    pending_for_mds[version].reqid = reqid;
+    pending_for_mds[version].tid = version;
+  }
+  void _note_commit(uint64_t tid) {
+    pending_for_mds.erase(tid);
+  }
+  void _note_rollback(uint64_t tid) {
+    pending_for_mds.erase(tid);
+  }
+  
+
+  MDSTableServer(MDS *m, int tab) : MDSTable(m, get_mdstable_name(tab), false), table(tab) {}
+  virtual ~MDSTableServer() {}
+
+  void handle_request(MMDSTableRequest *m);
+  void do_server_update(bufferlist& bl);
+
+  virtual void encode_server_state(bufferlist& bl) const = 0;
+  virtual void decode_server_state(bufferlist::iterator& bl) = 0;
+
+  void encode_state(bufferlist& bl) const {
+    encode_server_state(bl);
+    ::encode(pending_for_mds, bl);
+  }
+  void decode_state(bufferlist::iterator& bl) {
+    decode_server_state(bl);
+    ::decode(pending_for_mds, bl);
+  }
+
+  // recovery
+  void finish_recovery(set& active);
+  void handle_mds_recovery(int who);
+};
+
+#endif
diff --git a/ceph/src/mds/MDSUtility.cc b/ceph/src/mds/MDSUtility.cc
new file mode 100644
index 00000000..09be280f
--- /dev/null
+++ b/ceph/src/mds/MDSUtility.cc
@@ -0,0 +1,160 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 John Spray 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include "mds/MDSUtility.h"
+#include "mon/MonClient.h"
+
+#define dout_subsys ceph_subsys_mds
+
+
+MDSUtility::MDSUtility() :
+  Dispatcher(g_ceph_context),
+  objecter(NULL),
+  lock("MDSUtility::lock"),
+  timer(g_ceph_context, lock),
+  waiting_for_mds_map(NULL)
+{
+  monc = new MonClient(g_ceph_context);
+  messenger = Messenger::create(g_ceph_context, entity_name_t::CLIENT(), "mds", getpid());
+  mdsmap = new MDSMap();
+  osdmap = new OSDMap();
+  objecter = new Objecter(g_ceph_context, messenger, monc, osdmap, lock, timer, 0, 0);
+}
+
+
+MDSUtility::~MDSUtility()
+{
+  delete objecter;
+  delete monc;
+  delete messenger;
+  delete osdmap;
+  delete mdsmap;
+  assert(waiting_for_mds_map == NULL);
+}
+
+
+int MDSUtility::init()
+{
+  // Initialize Messenger
+  int r = messenger->bind(g_conf->public_addr);
+  if (r < 0)
+    return r;
+
+  messenger->add_dispatcher_head(this);
+  messenger->start();
+
+  // Initialize MonClient
+  if (monc->build_initial_monmap() < 0)
+    return -1;
+
+  monc->set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD|CEPH_ENTITY_TYPE_MDS);
+  monc->set_messenger(messenger);
+  monc->init();
+  r = monc->authenticate();
+  if (r < 0) {
+    derr << "Authentication failed, did you specify an MDS ID with a valid keyring?" << dendl;
+    return r;
+  }
+
+  client_t whoami = monc->get_global_id();
+  messenger->set_myname(entity_name_t::CLIENT(whoami.v));
+
+  // Initialize Objecter and wait for OSD map
+  objecter->set_client_incarnation(0);
+  objecter->init_unlocked();
+  lock.Lock();
+  objecter->init_locked();
+  lock.Unlock();
+  objecter->wait_for_osd_map();
+  timer.init();
+
+  // Prepare to receive MDS map and request it
+  Mutex init_lock("MDSUtility:init");
+  Cond cond;
+  bool done = false;
+  assert(!mdsmap->get_epoch());
+  lock.Lock();
+  waiting_for_mds_map = new C_SafeCond(&init_lock, &cond, &done, NULL);
+  lock.Unlock();
+  monc->sub_want("mdsmap", 0, CEPH_SUBSCRIBE_ONETIME);
+  monc->renew_subs();
+
+  // Wait for MDS map
+  dout(4) << "waiting for MDS map..." << dendl;
+  init_lock.Lock();
+  while (!done)
+    cond.Wait(init_lock);
+  init_lock.Unlock();
+  dout(4) << "Got MDS map " << mdsmap->get_epoch() << dendl;
+
+  return 0;
+}
+
+
+void MDSUtility::shutdown()
+{
+  lock.Lock();
+  timer.shutdown();
+  objecter->shutdown_locked();
+  lock.Unlock();
+  objecter->shutdown_unlocked();
+  monc->shutdown();
+  messenger->shutdown();
+  messenger->wait();
+}
+
+
+bool MDSUtility::ms_dispatch(Message *m)
+{
+   Mutex::Locker locker(lock);
+   switch (m->get_type()) {
+   case CEPH_MSG_OSD_OPREPLY:
+     objecter->handle_osd_op_reply((MOSDOpReply *)m);
+     break;
+   case CEPH_MSG_OSD_MAP:
+     objecter->handle_osd_map((MOSDMap*)m);
+     break;
+   case CEPH_MSG_MDS_MAP:
+     handle_mds_map((MMDSMap*)m);
+     break;
+   default:
+     return false;
+   }
+   return true;
+}
+
+
+void MDSUtility::handle_mds_map(MMDSMap* m)
+{
+  mdsmap->decode(m->get_encoded());
+  if (waiting_for_mds_map) {
+    waiting_for_mds_map->complete(0);
+    waiting_for_mds_map = NULL;
+  }
+}
+
+
+bool MDSUtility::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer,
+                         bool force_new)
+{
+  if (dest_type == CEPH_ENTITY_TYPE_MON)
+    return true;
+
+  if (force_new) {
+    if (monc->wait_auth_rotating(10) < 0)
+      return false;
+  }
+
+  *authorizer = monc->auth->build_authorizer(dest_type);
+  return *authorizer != NULL;
+}
diff --git a/ceph/src/mds/MDSUtility.h b/ceph/src/mds/MDSUtility.h
new file mode 100644
index 00000000..d3f938e8
--- /dev/null
+++ b/ceph/src/mds/MDSUtility.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 John Spray 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef MDS_UTILITY_H_
+#define MDS_UTILITY_H_
+
+#include "osd/OSDMap.h"
+#include "osdc/Objecter.h"
+#include "mds/MDSMap.h"
+#include "messages/MMDSMap.h"
+#include "msg/Dispatcher.h"
+#include "msg/Messenger.h"
+#include "auth/Auth.h"
+
+/// MDS Utility
+/**
+ * This class is the parent for MDS utilities, i.e. classes that
+ * need access the objects belonging to the MDS without actually
+ * acting as an MDS daemon themselves.
+ */
+class MDSUtility : public Dispatcher {
+protected:
+  Objecter *objecter;
+  OSDMap *osdmap;
+  MDSMap *mdsmap;
+  Messenger *messenger;
+  MonClient *monc;
+
+  Mutex lock;
+  SafeTimer timer;
+
+  Context *waiting_for_mds_map;
+
+public:
+  MDSUtility();
+  ~MDSUtility();
+
+  void handle_mds_map(MMDSMap* m);
+  bool ms_dispatch(Message *m);
+  bool ms_handle_reset(Connection *con) { return false; }
+  void ms_handle_remote_reset(Connection *con) {}
+  bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer,
+                         bool force_new);
+  int init();
+  void shutdown();
+};
+
+#endif /* MDS_UTILITY_H_ */
diff --git a/ceph/src/mds/Makefile.am b/ceph/src/mds/Makefile.am
new file mode 100644
index 00000000..4ee35003
--- /dev/null
+++ b/ceph/src/mds/Makefile.am
@@ -0,0 +1,94 @@
+libmds_la_SOURCES = \
+	mds/Anchor.cc \
+	mds/Capability.cc \
+	mds/Dumper.cc \
+	mds/Resetter.cc \
+	mds/MDS.cc \
+	mds/flock.cc \
+	mds/locks.c \
+	mds/journal.cc \
+	mds/Server.cc \
+	mds/Mutation.cc \
+	mds/MDCache.cc \
+	mds/Locker.cc \
+	mds/Migrator.cc \
+	mds/MDBalancer.cc \
+	mds/CDentry.cc \
+	mds/CDir.cc \
+	mds/CInode.cc \
+	mds/LogEvent.cc \
+	mds/MDSTable.cc \
+	mds/InoTable.cc \
+	mds/MDSTableClient.cc \
+	mds/MDSTableServer.cc \
+	mds/AnchorServer.cc \
+	mds/AnchorClient.cc \
+	mds/SnapRealm.cc \
+	mds/SnapServer.cc \
+	mds/snap.cc \
+	mds/SessionMap.cc \
+	mds/MDLog.cc \
+	mds/MDSUtility.cc
+libmds_la_LIBADD = $(LIBOSDC)
+noinst_LTLIBRARIES += libmds.la
+
+noinst_HEADERS += \
+	mds/inode_backtrace.h \
+	mds/flock.h \
+	mds/locks.c \
+	mds/locks.h \
+	mds/Anchor.h \
+	mds/AnchorClient.h \
+	mds/AnchorServer.h \
+	mds/CDentry.h \
+	mds/CDir.h \
+	mds/CInode.h \
+	mds/Capability.h \
+	mds/Dumper.h \
+	mds/InoTable.h \
+	mds/LocalLock.h \
+	mds/Locker.h \
+	mds/LogEvent.h \
+	mds/LogSegment.h \
+	mds/MDBalancer.h \
+	mds/MDCache.h \
+	mds/MDLog.h \
+	mds/MDS.h \
+	mds/MDSMap.h \
+	mds/MDSTable.h \
+	mds/MDSTableServer.h \
+	mds/MDSTableClient.h \
+	mds/MDSUtility.h \
+	mds/Mutation.h \
+	mds/Migrator.h \
+	mds/Resetter.h \
+	mds/ScatterLock.h \
+	mds/Server.h \
+	mds/SessionMap.h \
+	mds/SimpleLock.h \
+	mds/SnapClient.h \
+	mds/SnapRealm.h \
+	mds/SnapServer.h \
+	mds/inode_backtrace.h \
+	mds/mds_table_types.h \
+	mds/mdstypes.h \
+	mds/snap.h
+
+noinst_HEADERS += \
+	mds/events/ECommitted.h \
+	mds/events/EExport.h \
+	mds/events/EFragment.h \
+	mds/events/EImportFinish.h \
+	mds/events/EImportStart.h \
+	mds/events/EMetaBlob.h \
+	mds/events/EOpen.h \
+	mds/events/EResetJournal.h \
+	mds/events/ESession.h \
+	mds/events/ESessions.h \
+	mds/events/ESlaveUpdate.h \
+	mds/events/ESubtreeMap.h \
+	mds/events/ETableClient.h \
+	mds/events/ETableServer.h \
+	mds/events/EUpdate.h
+
+
diff --git a/ceph/src/mds/Migrator.cc b/ceph/src/mds/Migrator.cc
new file mode 100644
index 00000000..cde0f704
--- /dev/null
+++ b/ceph/src/mds/Migrator.cc
@@ -0,0 +1,3040 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MDS.h"
+#include "MDCache.h"
+#include "CInode.h"
+#include "CDir.h"
+#include "CDentry.h"
+#include "Migrator.h"
+#include "Locker.h"
+#include "Server.h"
+
+#include "MDBalancer.h"
+#include "MDLog.h"
+#include "MDSMap.h"
+#include "Mutation.h"
+
+#include "include/filepath.h"
+
+#include "events/EExport.h"
+#include "events/EImportStart.h"
+#include "events/EImportFinish.h"
+#include "events/ESessions.h"
+
+#include "msg/Messenger.h"
+
+#include "messages/MClientCaps.h"
+
+#include "messages/MExportDirDiscover.h"
+#include "messages/MExportDirDiscoverAck.h"
+#include "messages/MExportDirCancel.h"
+#include "messages/MExportDirPrep.h"
+#include "messages/MExportDirPrepAck.h"
+#include "messages/MExportDir.h"
+#include "messages/MExportDirAck.h"
+#include "messages/MExportDirNotify.h"
+#include "messages/MExportDirNotifyAck.h"
+#include "messages/MExportDirFinish.h"
+
+#include "messages/MExportCaps.h"
+#include "messages/MExportCapsAck.h"
+
+
+/*
+ * this is what the dir->dir_auth values look like
+ *
+ *   dir_auth  authbits  
+ * export
+ *   me         me      - before
+ *   me, me     me      - still me, but preparing for export
+ *   me, them   me      - send MExportDir (peer is preparing)
+ *   them, me   me      - journaled EExport
+ *   them       them    - done
+ *
+ * import:
+ *   them       them    - before
+ *   me, them   me      - journaled EImportStart
+ *   me         me      - done
+ *
+ * which implies:
+ *  - auth bit is set if i am listed as first _or_ second dir_auth.
+ */
+
+#include "common/config.h"
+
+
+#define dout_subsys ceph_subsys_mds
+#undef DOUT_COND
+#define DOUT_COND(cct, l) (l <= cct->_conf->debug_mds || l <= cct->_conf->debug_mds_migrator)
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".migrator "
+
+/* This function DOES put the passed message before returning*/
+void Migrator::dispatch(Message *m)
+{
+  switch (m->get_type()) {
+    // import
+  case MSG_MDS_EXPORTDIRDISCOVER:
+    handle_export_discover(static_cast(m));
+    break;
+  case MSG_MDS_EXPORTDIRPREP:
+    handle_export_prep(static_cast(m));
+    break;
+  case MSG_MDS_EXPORTDIR:
+    handle_export_dir(static_cast(m));
+    break;
+  case MSG_MDS_EXPORTDIRFINISH:
+    handle_export_finish(static_cast(m));
+    break;
+  case MSG_MDS_EXPORTDIRCANCEL:
+    handle_export_cancel(static_cast(m));
+    break;
+
+    // export 
+  case MSG_MDS_EXPORTDIRDISCOVERACK:
+    handle_export_discover_ack(static_cast(m));
+    break;
+  case MSG_MDS_EXPORTDIRPREPACK:
+    handle_export_prep_ack(static_cast(m));
+    break;
+  case MSG_MDS_EXPORTDIRACK:
+    handle_export_ack(static_cast(m));
+    break;
+  case MSG_MDS_EXPORTDIRNOTIFYACK:
+    handle_export_notify_ack(static_cast(m));
+    break;    
+
+    // export 3rd party (dir_auth adjustments)
+  case MSG_MDS_EXPORTDIRNOTIFY:
+    handle_export_notify(static_cast(m));
+    break;
+
+    // caps
+  case MSG_MDS_EXPORTCAPS:
+    handle_export_caps(static_cast(m));
+    break;
+
+  default:
+    assert(0);
+  }
+}
+
+
+class C_MDC_EmptyImport : public Context {
+  Migrator *mig;
+  CDir *dir;
+public:
+  C_MDC_EmptyImport(Migrator *m, CDir *d) : mig(m), dir(d) {}
+  void finish(int r) {
+    mig->export_empty_import(dir);
+  }
+};
+
+
+void Migrator::export_empty_import(CDir *dir)
+{
+  dout(7) << "export_empty_import " << *dir << dendl;
+  assert(dir->is_subtree_root());
+
+  if (dir->inode->is_auth()) {
+    dout(7) << " inode is auth" << dendl;
+    return;
+  }
+  if (!dir->is_auth()) {
+    dout(7) << " not auth" << dendl;
+    return;
+  }
+  if (dir->is_freezing() || dir->is_frozen()) {
+    dout(7) << " freezing or frozen" << dendl;
+    return;
+  }
+  if (dir->get_num_head_items() > 0) {
+    dout(7) << " not actually empty" << dendl;
+    return;
+  }
+  if (dir->inode->is_root()) {
+    dout(7) << " root" << dendl;
+    return;
+  }
+  
+  int dest = dir->inode->authority().first;
+  //if (mds->is_shutting_down()) dest = 0;  // this is more efficient.
+  
+  dout(7) << " really empty, exporting to " << dest << dendl;
+  assert (dest != mds->get_nodeid());
+  
+  dout(7) << "exporting to mds." << dest 
+           << " empty import " << *dir << dendl;
+  export_dir( dir, dest );
+}
+
+void Migrator::find_stale_export_freeze()
+{
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t cutoff = now;
+  cutoff -= g_conf->mds_freeze_tree_timeout;
+
+
+  /*
+   * We could have situations like:
+   *
+   * - mds.0 authpins an item in subtree A
+   * - mds.0 sends request to mds.1 to authpin an item in subtree B
+   * - mds.0 freezes subtree A
+   * - mds.1 authpins an item in subtree B
+   * - mds.1 sends request to mds.0 to authpin an item in subtree A
+   * - mds.1 freezes subtree B
+   * - mds.1 receives the remote authpin request from mds.0
+   *   (wait because subtree B is freezing)
+   * - mds.0 receives the remote authpin request from mds.1
+   *   (wait because subtree A is freezing)
+   *
+   *
+   * - client request authpins items in subtree B
+   * - freeze subtree B
+   * - import subtree A which is parent of subtree B
+   *   (authpins parent inode of subtree B, see CDir::set_dir_auth())
+   * - freeze subtree A
+   * - client request tries authpinning items in subtree A
+   *   (wait because subtree A is freezing)
+   */
+  for (map::iterator p = export_state.begin();
+       p != export_state.end(); ) {
+    CDir* dir = p->first;
+    export_state_t& stat = p->second;
+    ++p;
+    if (p->second.state != EXPORT_DISCOVERING &&
+	p->second.state != EXPORT_FREEZING)
+      continue;
+    if (stat.last_cum_auth_pins != dir->get_cum_auth_pins()) {
+      stat.last_cum_auth_pins = dir->get_cum_auth_pins();
+      stat.last_cum_auth_pins_change = now;
+      continue;
+    }
+    if (stat.last_cum_auth_pins_change >= cutoff)
+      continue;
+    if (stat.num_remote_waiters > 0 ||
+	(!dir->inode->is_root() && dir->get_parent_dir()->is_freezing())) {
+      export_try_cancel(dir);
+    }
+  }
+}
+
+void Migrator::export_try_cancel(CDir *dir, bool notify_peer)
+{
+  dout(10) << "export_try_cancel " << *dir << dendl;
+
+  map::iterator it = export_state.find(dir);
+  assert(it != export_state.end());
+
+  int state = it->second.state;
+  switch (state) {
+  case EXPORT_LOCKING:
+    dout(10) << "export state=locking : dropping locks and removing auth_pin" << dendl;
+    it->second.state = EXPORT_CANCELLED;
+    dir->auth_unpin(this);
+    dir->state_clear(CDir::STATE_EXPORTING);
+    break;
+  case EXPORT_DISCOVERING:
+    dout(10) << "export state=discovering : canceling freeze and removing auth_pin" << dendl;
+    it->second.state = EXPORT_CANCELLED;
+    dir->unfreeze_tree();  // cancel the freeze
+    dir->auth_unpin(this);
+    dir->state_clear(CDir::STATE_EXPORTING);
+    if (notify_peer && mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+      mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
+    break;
+
+  case EXPORT_FREEZING:
+    dout(10) << "export state=freezing : canceling freeze" << dendl;
+    it->second.state = EXPORT_CANCELLED;
+    dir->unfreeze_tree();  // cancel the freeze
+    dir->state_clear(CDir::STATE_EXPORTING);
+    if (notify_peer && mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+      mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
+    break;
+
+    // NOTE: state order reversal, warning comes after prepping
+  case EXPORT_WARNING:
+    dout(10) << "export state=warning : unpinning bounds, unfreezing, notifying" << dendl;
+    // fall-thru
+
+  case EXPORT_PREPPING:
+    if (state != EXPORT_WARNING)
+      dout(10) << "export state=prepping : unpinning bounds, unfreezing" << dendl;
+
+    it->second.state = EXPORT_CANCELLED;
+    {
+      // unpin bounds
+      set bounds;
+      cache->get_subtree_bounds(dir, bounds);
+      for (set::iterator q = bounds.begin();
+          q != bounds.end();
+          ++q) {
+        CDir *bd = *q;
+        bd->put(CDir::PIN_EXPORTBOUND);
+        bd->state_clear(CDir::STATE_EXPORTBOUND);
+      }
+      if (state == EXPORT_WARNING) {
+	// notify bystanders
+	export_notify_abort(dir, bounds);
+	// process delayed expires
+	cache->process_delayed_expire(dir);
+      }
+    }
+    dir->unfreeze_tree();
+    cache->adjust_subtree_auth(dir, mds->get_nodeid());
+    cache->try_subtree_merge(dir);  // NOTE: this may journal subtree_map as side effect
+    dir->state_clear(CDir::STATE_EXPORTING);
+    if (notify_peer && mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) // tell them.
+      mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
+    break;
+
+  case EXPORT_EXPORTING:
+    dout(10) << "export state=exporting : reversing, and unfreezing" << dendl;
+    it->second.state = EXPORT_CANCELLED;
+    export_reverse(dir);
+    dir->state_clear(CDir::STATE_EXPORTING);
+    break;
+
+  case EXPORT_LOGGINGFINISH:
+  case EXPORT_NOTIFYING:
+    dout(10) << "export state=loggingfinish|notifying : ignoring dest failure, we were successful." << dendl;
+    // leave export_state, don't clean up now.
+    break;
+
+  default:
+    assert(0);
+  }
+
+  // finish clean-up?
+  if (it->second.state == EXPORT_CANCELLED) {
+    // wake up any waiters
+    mds->queue_waiters(it->second.waiting_for_finish);
+    // drop locks
+    if (state == EXPORT_LOCKING || state == EXPORT_DISCOVERING) {
+      MDRequestRef mdr = ceph::static_pointer_cast(it->second.mut);
+      assert(mdr);
+      if (mdr->more()->waiting_on_slave.empty())
+	mds->mdcache->request_finish(mdr);
+    } else if (it->second.mut) {
+      MutationRef& mut = it->second.mut;
+      mds->locker->drop_locks(mut.get());
+      mut->cleanup();
+    }
+
+    export_state.erase(it);
+    // send pending import_maps?  (these need to go out when all exports have finished.)
+    cache->maybe_send_pending_resolves();
+
+    cache->show_subtrees();
+
+    maybe_do_queued_export();
+  }
+}
+
+// ==========================================================
+// mds failure handling
+
+void Migrator::handle_mds_failure_or_stop(int who)
+{
+  dout(5) << "handle_mds_failure_or_stop mds." << who << dendl;
+
+  // check my exports
+
+  // first add an extra auth_pin on any freezes, so that canceling a
+  // nested freeze doesn't complete one further up the hierarchy and
+  // confuse the shit out of us.  we'll remove it after canceling the
+  // freeze.  this way no freeze completions run before we want them
+  // to.
+  list pinned_dirs;
+  for (map::iterator p = export_state.begin();
+       p != export_state.end();
+       ++p) {
+    if (p->second.state == EXPORT_FREEZING) {
+      CDir *dir = p->first;
+      dout(10) << "adding temp auth_pin on freezing " << *dir << dendl;
+      dir->auth_pin(this);
+      pinned_dirs.push_back(dir);
+    }
+  }
+
+  map::iterator p = export_state.begin();
+  while (p != export_state.end()) {
+    map::iterator next = p;
+    ++next;
+    CDir *dir = p->first;
+    
+    // abort exports:
+    //  - that are going to the failed node
+    //  - that aren't frozen yet (to avoid auth_pin deadlock)
+    //  - they havne't prepped yet (they may need to discover bounds to do that)
+    if (p->second.peer == who ||
+	p->second.state == EXPORT_LOCKING ||
+	p->second.state == EXPORT_DISCOVERING ||
+	p->second.state == EXPORT_FREEZING ||
+	p->second.state == EXPORT_PREPPING) {
+      // the guy i'm exporting to failed, or we're just freezing.
+      dout(10) << "cleaning up export state (" << p->second.state << ")"
+	       << get_export_statename(p->second.state) << " of " << *dir << dendl;
+      export_try_cancel(dir);
+    } else {
+      // bystander failed.
+      if (p->second.warning_ack_waiting.count(who)) {
+	p->second.warning_ack_waiting.erase(who);
+	p->second.notify_ack_waiting.erase(who);   // they won't get a notify either.
+	if (p->second.state == EXPORT_WARNING) {
+	  // exporter waiting for warning acks, let's fake theirs.
+	  dout(10) << "faking export_warning_ack from mds." << who
+		   << " on " << *dir << " to mds." << p->second.peer
+		   << dendl;
+	  if (p->second.warning_ack_waiting.empty())
+	    export_go(dir);
+	}
+      }
+      if (p->second.notify_ack_waiting.count(who)) {
+	p->second.notify_ack_waiting.erase(who);
+	if (p->second.state == EXPORT_NOTIFYING) {
+	  // exporter is waiting for notify acks, fake it
+	  dout(10) << "faking export_notify_ack from mds." << who
+		   << " on " << *dir << " to mds." << p->second.peer
+		   << dendl;
+	  if (p->second.notify_ack_waiting.empty())
+	    export_finish(dir);
+	}
+      }
+    }
+    
+    // next!
+    p = next;
+  }
+
+
+  // check my imports
+  map::iterator q = import_state.begin();
+  while (q != import_state.end()) {
+    map::iterator next = q;
+    ++next;
+    dirfrag_t df = q->first;
+    CInode *diri = mds->mdcache->get_inode(df.ino);
+    CDir *dir = mds->mdcache->get_dirfrag(df);
+
+    if (q->second.peer == who) {
+      if (dir)
+	dout(10) << "cleaning up import state (" << q->second.state << ")"
+		 << get_import_statename(q->second.state) << " of " << *dir << dendl;
+      else
+	dout(10) << "cleaning up import state (" << q->second.state << ")"
+		 << get_import_statename(q->second.state) << " of " << df << dendl;
+
+      switch (q->second.state) {
+      case IMPORT_DISCOVERING:
+	dout(10) << "import state=discovering : clearing state" << dendl;
+	import_reverse_discovering(df);
+	break;
+
+      case IMPORT_DISCOVERED:
+	assert(diri);
+	dout(10) << "import state=discovered : unpinning inode " << *diri << dendl;
+	import_reverse_discovered(df, diri);
+	break;
+
+      case IMPORT_PREPPING:
+	assert(dir);
+	dout(10) << "import state=prepping : unpinning base+bounds " << *dir << dendl;
+	import_reverse_prepping(dir);
+	break;
+
+      case IMPORT_PREPPED:
+	assert(dir);
+	dout(10) << "import state=prepped : unpinning base+bounds, unfreezing " << *dir << dendl;
+	{
+	  set bounds;
+	  cache->get_subtree_bounds(dir, bounds);
+	  import_remove_pins(dir, bounds);
+	  
+	  // adjust auth back to the exporter
+	  cache->adjust_subtree_auth(dir, q->second.peer);
+	  cache->try_subtree_merge(dir);   // NOTE: may journal subtree_map as side-effect
+
+	  // bystanders?
+	  if (q->second.bystanders.empty()) {
+	    import_reverse_unfreeze(dir);
+	  } else {
+	    // notify them; wait in aborting state
+	    import_notify_abort(dir, bounds);
+	    import_state[df].state = IMPORT_ABORTING;
+	    assert(g_conf->mds_kill_import_at != 10);
+	  }
+	}
+	break;
+
+      case IMPORT_LOGGINGSTART:
+	assert(dir);
+	dout(10) << "import state=loggingstart : reversing import on " << *dir << dendl;
+	import_reverse(dir);
+	break;
+
+      case IMPORT_ACKING:
+	assert(dir);
+	// hrm.  make this an ambiguous import, and wait for exporter recovery to disambiguate
+	dout(10) << "import state=acking : noting ambiguous import " << *dir << dendl;
+	{
+	  set bounds;
+	  cache->get_subtree_bounds(dir, bounds);
+	  cache->add_ambiguous_import(dir, bounds);
+	}
+	break;
+	
+      case IMPORT_FINISHING:
+	assert(dir);
+	dout(10) << "import state=finishing : finishing import on " << *dir << dendl;
+	import_finish(dir, true);
+	break;
+
+      case IMPORT_ABORTING:
+	assert(dir);
+	dout(10) << "import state=aborting : ignoring repeat failure " << *dir << dendl;
+	break;
+      }
+    } else {
+      if (q->second.bystanders.count(who)) {
+	q->second.bystanders.erase(who);
+	if (q->second.state == IMPORT_ABORTING) {
+	  assert(dir);
+	  dout(10) << "faking export_notify_ack from mds." << who
+		   << " on aborting import " << *dir << " from mds." << q->second.peer
+		   << dendl;
+	  if (q->second.bystanders.empty()) {
+	    import_reverse_unfreeze(dir);
+	  }
+	}
+      }
+    }
+
+    // next!
+    q = next;
+  }
+
+  while (!pinned_dirs.empty()) {
+    CDir *dir = pinned_dirs.front();
+    dout(10) << "removing temp auth_pin on " << *dir << dendl;
+    dir->auth_unpin(this);
+    pinned_dirs.pop_front();
+  }  
+}
+
+
+
+void Migrator::show_importing()
+{  
+  dout(10) << "show_importing" << dendl;
+  for (map::iterator p = import_state.begin();
+       p != import_state.end();
+       ++p) {
+    CDir *dir = mds->mdcache->get_dirfrag(p->first);
+    if (dir) {
+      dout(10) << " importing from " << p->second.peer
+	       << ": (" << p->second.state << ") " << get_import_statename(p->second.state)
+	       << " " << p->first << " " << *dir << dendl;
+    } else {
+      dout(10) << " importing from " << p->second.peer
+	       << ": (" << p->second.state << ") " << get_import_statename(p->second.state)
+	       << " " << p->first << dendl;
+    }
+  }
+}
+
+void Migrator::show_exporting() 
+{
+  dout(10) << "show_exporting" << dendl;
+  for (map::iterator p = export_state.begin();
+       p != export_state.end();
+       ++p) 
+    dout(10) << " exporting to " << p->second.peer
+	     << ": (" << p->second.state << ") " << get_export_statename(p->second.state)
+	     << " " << p->first->dirfrag() << " " << *p->first << dendl;
+}
+
+
+
+void Migrator::audit()
+{
+  if (!g_conf->subsys.should_gather(ceph_subsys_mds, 5))
+    return;  // hrm.
+
+  // import_state
+  show_importing();
+  for (map::iterator p = import_state.begin();
+       p != import_state.end();
+       ++p) {
+    if (p->second.state == IMPORT_DISCOVERING)
+      continue;
+    if (p->second.state == IMPORT_DISCOVERED) {
+      CInode *in = cache->get_inode(p->first.ino);
+      assert(in);
+      continue;
+    }
+    CDir *dir = cache->get_dirfrag(p->first);
+    assert(dir);
+    if (p->second.state == IMPORT_PREPPING)
+      continue;
+    if (p->second.state == IMPORT_ABORTING) {
+      assert(!dir->is_ambiguous_dir_auth());
+      assert(dir->get_dir_auth().first != mds->get_nodeid());
+      continue;
+    }
+    assert(dir->is_ambiguous_dir_auth());
+    assert(dir->authority().first  == mds->get_nodeid() ||
+	   dir->authority().second == mds->get_nodeid());
+  }
+
+  // export_state
+  show_exporting();
+  for (map::iterator p = export_state.begin();
+       p != export_state.end();
+       ++p) {
+    CDir *dir = p->first;
+    if (p->second.state == EXPORT_LOCKING ||
+	p->second.state == EXPORT_DISCOVERING ||
+	p->second.state == EXPORT_FREEZING) continue;
+    assert(dir->is_ambiguous_dir_auth());
+    assert(dir->authority().first  == mds->get_nodeid() ||
+	   dir->authority().second == mds->get_nodeid());
+  }
+
+  // ambiguous+me subtrees should be importing|exporting
+
+  // write me
+}
+
+
+
+
+
+// ==========================================================
+// EXPORT
+
+void Migrator::export_dir_nicely(CDir *dir, int dest)
+{
+  // enqueue
+  dout(7) << "export_dir_nicely " << *dir << " to " << dest << dendl;
+  export_queue.push_back(pair(dir->dirfrag(), dest));
+
+  maybe_do_queued_export();
+}
+
+void Migrator::maybe_do_queued_export()
+{
+  static bool running;
+  if (running)
+    return;
+  running = true;
+  while (!export_queue.empty() &&
+	 export_state.size() <= 4) {
+    dirfrag_t df = export_queue.front().first;
+    int dest = export_queue.front().second;
+    export_queue.pop_front();
+    
+    CDir *dir = mds->mdcache->get_dirfrag(df);
+    if (!dir) continue;
+    if (!dir->is_auth()) continue;
+
+    dout(0) << "nicely exporting to mds." << dest << " " << *dir << dendl;
+
+    export_dir(dir, dest);
+  }
+  running = false;
+}
+
+
+
+
+class C_MDC_ExportFreeze : public Context {
+  Migrator *mig;
+  CDir *ex;   // dir i'm exporting
+
+public:
+  C_MDC_ExportFreeze(Migrator *m, CDir *e) :
+	mig(m), ex(e) {}
+  virtual void finish(int r) {
+    if (r >= 0)
+      mig->export_frozen(ex);
+  }
+};
+
+
+void Migrator::get_export_lock_set(CDir *dir, set& locks)
+{
+  // path
+  vector trace;
+  cache->make_trace(trace, dir->inode);
+  for (vector::iterator it = trace.begin();
+       it != trace.end();
+       ++it)
+    locks.insert(&(*it)->lock);
+
+  // prevent scatter gather race
+  locks.insert(&dir->get_inode()->dirfragtreelock);
+
+  // bound dftlocks:
+  // NOTE: We need to take an rdlock on bounding dirfrags during
+  //  migration for a rather irritating reason: when we export the
+  //  bound inode, we need to send scatterlock state for the dirfrags
+  //  as well, so that the new auth also gets the correct info.  If we
+  //  race with a refragment, this info is useless, as we can't
+  //  redivvy it up.  And it's needed for the scatterlocks to work
+  //  properly: when the auth is in a sync/lock state it keeps each
+  //  dirfrag's portion in the local (auth OR replica) dirfrag.
+  set wouldbe_bounds;
+  cache->get_wouldbe_subtree_bounds(dir, wouldbe_bounds);
+  for (set::iterator p = wouldbe_bounds.begin(); p != wouldbe_bounds.end(); ++p)
+    locks.insert(&(*p)->get_inode()->dirfragtreelock);
+}
+
+
+/** export_dir(dir, dest)
+ * public method to initiate an export.
+ * will fail if the directory is freezing, frozen, unpinnable, or root. 
+ */
+void Migrator::export_dir(CDir *dir, int dest)
+{
+  dout(7) << "export_dir " << *dir << " to " << dest << dendl;
+  assert(dir->is_auth());
+  assert(dest != mds->get_nodeid());
+   
+  if (mds->mdsmap->is_degraded()) {
+    dout(7) << "cluster degraded, no exports for now" << dendl;
+    return;
+  }
+  if (dir->inode->is_system()) {
+    dout(7) << "i won't export system dirs (root, mdsdirs, stray, /.ceph, etc.)" << dendl;
+    //assert(0);
+    return;
+  }
+
+  if (!dir->inode->is_base() && dir->get_parent_dir()->get_inode()->is_stray() &&
+      dir->get_parent_dir()->get_parent_dir()->ino() != MDS_INO_MDSDIR(dest)) {
+    dout(7) << "i won't export anything in stray" << dendl;
+    return;
+  }
+
+  if (dir->is_frozen() ||
+      dir->is_freezing()) {
+    dout(7) << " can't export, freezing|frozen.  wait for other exports to finish first." << dendl;
+    return;
+  }
+  if (dir->state_test(CDir::STATE_EXPORTING)) {
+    dout(7) << "already exporting" << dendl;
+    return;
+  }
+
+  dir->auth_pin(this);
+  dir->state_set(CDir::STATE_EXPORTING);
+
+  MDRequestRef mdr = mds->mdcache->request_start_internal(CEPH_MDS_OP_EXPORTDIR);
+  mdr->more()->export_dir = dir;
+
+  assert(export_state.count(dir) == 0);
+  export_state_t& stat = export_state[dir];
+  stat.state = EXPORT_LOCKING;
+  stat.peer = dest;
+  stat.tid = mdr->reqid.tid;
+  stat.mut = mdr;
+
+  dispatch_export_dir(mdr);
+}
+
+void Migrator::dispatch_export_dir(MDRequestRef& mdr)
+{
+  dout(7) << "dispatch_export_dir " << *mdr << dendl;
+  CDir *dir = mdr->more()->export_dir;
+
+  map::iterator it = export_state.find(dir);
+  if (it == export_state.end() || it->second.tid != mdr->reqid.tid) {
+    // export must have aborted.
+    dout(7) << "export must have aborted " << *mdr << dendl;
+    mds->mdcache->request_finish(mdr);
+    return;
+  }
+  assert(it->second.state == EXPORT_LOCKING);
+
+  if (mdr->aborted || dir->is_frozen() || dir->is_freezing()) {
+    dout(7) << "wouldblock|freezing|frozen, canceling export" << dendl;
+    export_try_cancel(dir);
+    return;
+  }
+
+  // locks?
+  set rdlocks;
+  set xlocks;
+  set wrlocks;
+  get_export_lock_set(dir, rdlocks);
+  // If auth MDS of the subtree root inode is neither the exporter MDS
+  // nor the importer MDS and it gathers subtree root's fragstat/neststat
+  // while the subtree is exporting. It's possible that the exporter MDS
+  // and the importer MDS both are auth MDS of the subtree root or both
+  // are not auth MDS of the subtree root at the time they receive the
+  // lock messages. So the auth MDS of the subtree root inode may get no
+  // or duplicated fragstat/neststat for the subtree root dirfrag.
+  wrlocks.insert(&dir->get_inode()->filelock);
+  wrlocks.insert(&dir->get_inode()->nestlock);
+  if (dir->get_inode()->is_auth()) {
+    dir->get_inode()->filelock.set_scatter_wanted();
+    dir->get_inode()->nestlock.set_scatter_wanted();
+  }
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks, NULL, NULL, true)) {
+    if (mdr->aborted)
+      export_try_cancel(dir);
+    return;
+  }
+
+  assert(g_conf->mds_kill_export_at != 1);
+  it->second.state = EXPORT_DISCOVERING;
+
+  // send ExportDirDiscover (ask target)
+  filepath path;
+  dir->inode->make_path(path);
+  MExportDirDiscover *discover = new MExportDirDiscover(dir->dirfrag(), path,
+							mds->get_nodeid(),
+							it->second.tid);
+  mds->send_message_mds(discover, it->second.peer);
+  assert(g_conf->mds_kill_export_at != 2);
+
+  it->second.last_cum_auth_pins_change = ceph_clock_now(g_ceph_context);
+
+  // start the freeze, but hold it up with an auth_pin.
+  dir->freeze_tree();
+  assert(dir->is_freezing_tree());
+  dir->add_waiter(CDir::WAIT_FROZEN, new C_MDC_ExportFreeze(this, dir));
+}
+
+/*
+ * called on receipt of MExportDirDiscoverAck
+ * the importer now has the directory's _inode_ in memory, and pinned.
+ *
+ * This function DOES put the passed message before returning
+ */
+void Migrator::handle_export_discover_ack(MExportDirDiscoverAck *m)
+{
+  CDir *dir = cache->get_dirfrag(m->get_dirfrag());
+  assert(dir);
+  
+  dout(7) << "export_discover_ack from " << m->get_source()
+	  << " on " << *dir << dendl;
+
+  map::iterator it = export_state.find(dir);
+  if (it == export_state.end() ||
+      it->second.tid != m->get_tid() ||
+      it->second.peer != m->get_source().num()) {
+    dout(7) << "must have aborted" << dendl;
+  } else {
+    assert(it->second.state == EXPORT_DISCOVERING);
+    // release locks to avoid deadlock
+    MDRequestRef mdr = ceph::static_pointer_cast(it->second.mut);
+    assert(mdr);
+    mds->mdcache->request_finish(mdr);
+    it->second.mut.reset();
+    // freeze the subtree
+    it->second.state = EXPORT_FREEZING;
+    dir->auth_unpin(this);
+    assert(g_conf->mds_kill_export_at != 3);
+  }
+  
+  m->put();  // done
+}
+
+class C_M_ExportSessionsFlushed : public Context {
+  Migrator *migrator;
+  CDir *dir;
+  uint64_t tid;
+public:
+  C_M_ExportSessionsFlushed(Migrator *m, CDir *d, uint64_t t) :
+    migrator(m), dir(d), tid(t) {}
+  void finish(int r) {
+    migrator->export_sessions_flushed(dir, tid);
+  }
+};
+
+void Migrator::export_sessions_flushed(CDir *dir, uint64_t tid)
+{
+  dout(7) << "export_sessions_flushed " << *dir << dendl;
+
+  map::iterator it = export_state.find(dir);
+  if (it == export_state.end() || it->second.tid != tid) {
+    // export must have aborted.
+    dout(7) << "export must have aborted on " << dir << dendl;
+    return;
+  }
+
+  assert(it->second.state == EXPORT_PREPPING || it->second.state == EXPORT_WARNING);
+  assert(it->second.warning_ack_waiting.count(-1) > 0);
+  it->second.warning_ack_waiting.erase(-1);
+  if (it->second.state == EXPORT_WARNING && it->second.warning_ack_waiting.empty())
+    export_go(dir);     // start export.
+}
+
+void Migrator::export_frozen(CDir *dir)
+{
+  dout(7) << "export_frozen on " << *dir << dendl;
+  assert(dir->is_frozen());
+  assert(dir->get_cum_auth_pins() == 0);
+
+  map::iterator it = export_state.find(dir);
+  assert(it != export_state.end());
+  assert(it->second.state == EXPORT_FREEZING);
+
+  CInode *diri = dir->get_inode();
+
+  // ok, try to grab all my locks.
+  set rdlocks;
+  get_export_lock_set(dir, rdlocks);
+  if ((diri->is_auth() && diri->is_frozen()) ||
+      !mds->locker->can_rdlock_set(rdlocks) ||
+      !diri->filelock.can_wrlock(-1) ||
+      !diri->nestlock.can_wrlock(-1)) {
+    dout(7) << "export_dir couldn't acquire all needed locks, failing. "
+	    << *dir << dendl;
+
+    // .. unwind ..
+    dir->unfreeze_tree();
+    dir->state_clear(CDir::STATE_EXPORTING);
+    mds->queue_waiters(it->second.waiting_for_finish);
+
+    mds->send_message_mds(new MExportDirCancel(dir->dirfrag(), it->second.tid), it->second.peer);
+
+    export_state.erase(it);
+    return;
+  }
+
+  it->second.mut = MutationRef(new MutationImpl);
+  if (diri->is_auth())
+    it->second.mut->auth_pin(diri);
+  mds->locker->rdlock_take_set(rdlocks, it->second.mut);
+  mds->locker->wrlock_force(&diri->filelock, it->second.mut);
+  mds->locker->wrlock_force(&diri->nestlock, it->second.mut);
+
+  cache->show_subtrees();
+
+  // note the bounds.
+  //  force it into a subtree by listing auth as .
+  cache->adjust_subtree_auth(dir, mds->get_nodeid(), mds->get_nodeid());
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+
+  // generate prep message, log entry.
+  MExportDirPrep *prep = new MExportDirPrep(dir->dirfrag(), it->second.tid);
+
+  // include list of bystanders
+  for (map::iterator p = dir->replicas_begin();
+       p != dir->replicas_end();
+       ++p) {
+    if (p->first != it->second.peer) {
+      dout(10) << "bystander mds." << p->first << dendl;
+      prep->add_bystander(p->first);
+    }
+  }
+
+  // include base dirfrag
+  cache->replicate_dir(dir, it->second.peer, prep->basedir);
+  
+  /*
+   * include spanning tree for all nested exports.
+   * these need to be on the destination _before_ the final export so that
+   * dir_auth updates on any nested exports are properly absorbed.
+   * this includes inodes and dirfrags included in the subtree, but
+   * only the inodes at the bounds.
+   *
+   * each trace is: df ('-' | ('f' dir | 'd') dentry inode (dir dentry inode)*)
+   */
+  set inodes_added;
+  set dirfrags_added;
+
+  // check bounds
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) {
+    CDir *bound = *p;
+
+    // pin it.
+    bound->get(CDir::PIN_EXPORTBOUND);
+    bound->state_set(CDir::STATE_EXPORTBOUND);
+    
+    dout(7) << "  export bound " << *bound << dendl;
+    prep->add_bound( bound->dirfrag() );
+
+    // trace to bound
+    bufferlist tracebl;
+    CDir *cur = bound;
+    
+    char start = '-';
+    while (1) {
+      // don't repeat inodes
+      if (inodes_added.count(cur->inode->ino()))
+	break;
+      inodes_added.insert(cur->inode->ino());
+
+      // prepend dentry + inode
+      assert(cur->inode->is_auth());
+      bufferlist bl;
+      cache->replicate_dentry(cur->inode->parent, it->second.peer, bl);
+      dout(7) << "  added " << *cur->inode->parent << dendl;
+      cache->replicate_inode(cur->inode, it->second.peer, bl);
+      dout(7) << "  added " << *cur->inode << dendl;
+      bl.claim_append(tracebl);
+      tracebl.claim(bl);
+
+      cur = cur->get_parent_dir();
+
+      // don't repeat dirfrags
+      if (dirfrags_added.count(cur->dirfrag()) ||
+	  cur == dir) {
+	start = 'd';  // start with dentry
+	break;
+      }
+      dirfrags_added.insert(cur->dirfrag());
+
+      // prepend dir
+      cache->replicate_dir(cur, it->second.peer, bl);
+      dout(7) << "  added " << *cur << dendl;
+      bl.claim_append(tracebl);
+      tracebl.claim(bl);
+
+      start = 'f';  // start with dirfrag
+    }
+    bufferlist final;
+    dirfrag_t df = cur->dirfrag();
+    ::encode(df, final);
+    ::encode(start, final);
+    final.claim_append(tracebl);
+    prep->add_trace(final);
+  }
+
+  // send.
+  it->second.state = EXPORT_PREPPING;
+  mds->send_message_mds(prep, it->second.peer);
+  assert (g_conf->mds_kill_export_at != 4);
+
+  // make sure any new instantiations of caps are flushed out
+  assert(it->second.warning_ack_waiting.empty());
+
+  set export_client_set;
+  get_export_client_set(dir, export_client_set);
+
+  C_GatherBuilder gather(g_ceph_context);
+  mds->server->flush_client_sessions(export_client_set, gather);
+  if (gather.has_subs()) {
+    it->second.warning_ack_waiting.insert(-1);
+    gather.set_finisher(new C_M_ExportSessionsFlushed(this, dir, it->second.tid));
+    gather.activate();
+  }
+}
+
+void Migrator::get_export_client_set(CDir *dir, set& client_set)
+{
+  list dfs;
+  dfs.push_back(dir);
+  while (!dfs.empty()) {
+    CDir *dir = dfs.front();
+    dfs.pop_front();
+    for (CDir::map_t::iterator p = dir->begin(); p != dir->end(); ++p) {
+      CDentry *dn = p->second;
+      if (!dn->get_linkage()->is_primary())
+	continue;
+      CInode *in = dn->get_linkage()->get_inode();
+      if (in->is_dir()) {
+	// directory?
+	list ls;
+	in->get_dirfrags(ls);
+	for (list::iterator q = ls.begin(); q != ls.end(); ++q) {
+	  if (!(*q)->state_test(CDir::STATE_EXPORTBOUND)) {
+	    // include nested dirfrag
+	    assert((*q)->get_dir_auth().first == CDIR_AUTH_PARENT);
+	    dfs.push_back(*q); // it's ours, recurse (later)
+	  }
+	}
+      }
+      for (map::iterator q = in->client_caps.begin();
+	   q != in->client_caps.end();
+	   ++q)
+	client_set.insert(q->first);
+    }
+  }
+}
+
+void Migrator::get_export_client_set(CInode *in, set& client_set)
+{
+  for (map::iterator q = in->client_caps.begin();
+      q != in->client_caps.end();
+      ++q)
+    client_set.insert(q->first);
+}
+
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_prep_ack(MExportDirPrepAck *m)
+{
+  CDir *dir = cache->get_dirfrag(m->get_dirfrag());
+  assert(dir);
+
+  dout(7) << "export_prep_ack " << *dir << dendl;
+
+  map::iterator it = export_state.find(dir);
+  if (it == export_state.end() ||
+      it->second.tid != m->get_tid() ||
+      it->second.peer != m->get_source().num()) {
+    // export must have aborted.  
+    dout(7) << "export must have aborted" << dendl;
+    m->put();
+    return;
+  }
+  assert(it->second.state == EXPORT_PREPPING);
+
+  if (!m->is_success()) {
+    dout(7) << "peer couldn't acquire all needed locks, canceling" << dendl;
+    export_try_cancel(dir, false);
+    m->put();
+    return;
+  }
+
+  assert (g_conf->mds_kill_export_at != 5);
+  // send warnings
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+
+  assert(it->second.warning_ack_waiting.empty() ||
+         (it->second.warning_ack_waiting.size() == 1 &&
+	  it->second.warning_ack_waiting.count(-1) > 0));
+  assert(it->second.notify_ack_waiting.empty());
+
+  for (map::iterator p = dir->replicas_begin();
+       p != dir->replicas_end();
+       ++p) {
+    if (p->first == it->second.peer) continue;
+    if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(p->first))
+      continue;  // only if active
+    it->second.warning_ack_waiting.insert(p->first);
+    it->second.notify_ack_waiting.insert(p->first);  // we'll eventually get a notifyack, too!
+
+    MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(), it->second.tid, true,
+						    pair(mds->get_nodeid(),CDIR_AUTH_UNKNOWN),
+						    pair(mds->get_nodeid(),it->second.peer));
+    for (set::iterator q = bounds.begin(); q != bounds.end(); ++q)
+      notify->get_bounds().push_back((*q)->dirfrag());
+    mds->send_message_mds(notify, p->first);
+    
+  }
+
+  it->second.state = EXPORT_WARNING;
+
+  assert(g_conf->mds_kill_export_at != 6);
+  // nobody to warn?
+  if (it->second.warning_ack_waiting.empty())
+    export_go(dir);  // start export.
+    
+  // done.
+  m->put();
+}
+
+
+class C_M_ExportGo : public Context {
+  Migrator *migrator;
+  CDir *dir;
+  uint64_t tid;
+public:
+  C_M_ExportGo(Migrator *m, CDir *d, uint64_t t) :
+    migrator(m), dir(d), tid(t) {}
+  void finish(int r) {
+    migrator->export_go_synced(dir, tid);
+  }
+};
+
+void Migrator::export_go(CDir *dir)
+{
+  assert(export_state.count(dir));
+  dout(7) << "export_go " << *dir << " to " << export_state[dir].peer << dendl;
+
+  // first sync log to flush out e.g. any cap imports
+  mds->mdlog->wait_for_safe(new C_M_ExportGo(this, dir, export_state[dir].tid));
+  mds->mdlog->flush();
+}
+
+void Migrator::export_go_synced(CDir *dir, uint64_t tid)
+{
+
+  map::iterator it = export_state.find(dir);
+  if (it == export_state.end() ||
+      it->second.tid != tid) {
+    // export must have aborted.  
+    dout(7) << "export must have aborted on " << dir << dendl;
+    return;
+  }
+  assert(it->second.state == EXPORT_WARNING);
+
+  dout(7) << "export_go_synced " << *dir << " to " << it->second.peer << dendl;
+
+  cache->show_subtrees();
+  
+  it->second.state = EXPORT_EXPORTING;
+  assert(g_conf->mds_kill_export_at != 7);
+
+  assert(dir->get_cum_auth_pins() == 0);
+
+  // set ambiguous auth
+  cache->adjust_subtree_auth(dir, mds->get_nodeid(), it->second.peer);
+
+  // take away the popularity we're sending.
+  utime_t now = ceph_clock_now(g_ceph_context);
+  mds->balancer->subtract_export(dir, now);
+  
+  // fill export message with cache data
+  MExportDir *req = new MExportDir(dir->dirfrag(), it->second.tid);
+  map exported_client_map;
+  int num_exported_inodes = encode_export_dir(req->export_data,
+					      dir,   // recur start point
+					      exported_client_map,
+					      now);
+  ::encode(exported_client_map, req->client_map);
+
+  // add bounds to message
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p)
+    req->add_export((*p)->dirfrag());
+
+  // send
+  mds->send_message_mds(req, it->second.peer);
+  assert(g_conf->mds_kill_export_at != 8);
+
+  // stats
+  if (mds->logger) mds->logger->inc(l_mds_ex);
+  if (mds->logger) mds->logger->inc(l_mds_iexp, num_exported_inodes);
+
+  cache->show_subtrees();
+}
+
+
+/** encode_export_inode
+ * update our local state for this inode to export.
+ * encode relevant state to be sent over the wire.
+ * used by: encode_export_dir, file_rename (if foreign)
+ *
+ * FIXME: the separation between CInode.encode_export and these methods 
+ * is pretty arbitrary and dumb.
+ */
+void Migrator::encode_export_inode(CInode *in, bufferlist& enc_state, 
+				   map& exported_client_map)
+{
+  dout(7) << "encode_export_inode " << *in << dendl;
+  assert(!in->is_replica(mds->get_nodeid()));
+
+  // relax locks?
+  if (!in->is_replicated()) {
+    in->replicate_relax_locks();
+    dout(20) << " did replicate_relax_locks, now " << *in << dendl;
+  }
+
+  ::encode(in->inode.ino, enc_state);
+  ::encode(in->last, enc_state);
+  in->encode_export(enc_state);
+
+  // caps 
+  encode_export_inode_caps(in, true, enc_state, exported_client_map);
+}
+
+void Migrator::encode_export_inode_caps(CInode *in, bool auth_cap, bufferlist& bl,
+					map& exported_client_map)
+{
+  dout(20) << "encode_export_inode_caps " << *in << dendl;
+
+  // encode caps
+  map cap_map;
+  in->export_client_caps(cap_map);
+  ::encode(cap_map, bl);
+  if (auth_cap) {
+    ::encode(in->get_mds_caps_wanted(), bl);
+
+    in->state_set(CInode::STATE_EXPORTINGCAPS);
+    in->get(CInode::PIN_EXPORTINGCAPS);
+  }
+
+  // make note of clients named by exported capabilities
+  for (map::iterator it = in->client_caps.begin();
+       it != in->client_caps.end();
+       ++it) 
+    exported_client_map[it->first] = mds->sessionmap.get_inst(entity_name_t::CLIENT(it->first.v));
+}
+
+void Migrator::finish_export_inode_caps(CInode *in, int peer,
+					map& peer_imported)
+{
+  dout(20) << "finish_export_inode_caps " << *in << dendl;
+
+  in->state_clear(CInode::STATE_EXPORTINGCAPS);
+  in->put(CInode::PIN_EXPORTINGCAPS);
+
+  // tell (all) clients about migrating caps.. 
+  for (map::iterator it = in->client_caps.begin();
+       it != in->client_caps.end();
+       ++it) {
+    Capability *cap = it->second;
+    dout(7) << "finish_export_inode_caps telling client." << it->first
+	    << " exported caps on " << *in << dendl;
+    MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, in->ino(), 0,
+				     cap->get_cap_id(), cap->get_mseq());
+
+    map::iterator q = peer_imported.find(it->first);
+    assert(q != peer_imported.end());
+    m->set_cap_peer(q->second.cap_id, q->second.issue_seq, q->second.mseq, peer, 0);
+    mds->send_message_client_counted(m, it->first);
+  }
+  in->clear_client_caps_after_export();
+  mds->locker->eval(in, CEPH_CAP_LOCKS);
+}
+
+void Migrator::finish_export_inode(CInode *in, utime_t now, int peer,
+				   map& peer_imported,
+				   list& finished)
+{
+  dout(12) << "finish_export_inode " << *in << dendl;
+
+  // clean
+  if (in->is_dirty())
+    in->mark_clean();
+  
+  // clear/unpin cached_by (we're no longer the authority)
+  in->clear_replica_map();
+  
+  // twiddle lock states for auth -> replica transition
+  in->authlock.export_twiddle();
+  in->linklock.export_twiddle();
+  in->dirfragtreelock.export_twiddle();
+  in->filelock.export_twiddle();
+  in->nestlock.export_twiddle();
+  in->xattrlock.export_twiddle();
+  in->snaplock.export_twiddle();
+  in->flocklock.export_twiddle();
+  in->policylock.export_twiddle();
+  
+  // mark auth
+  assert(in->is_auth());
+  in->state_clear(CInode::STATE_AUTH);
+  in->replica_nonce = CInode::EXPORT_NONCE;
+  
+  in->clear_dirty_rstat();
+
+  // no more auth subtree? clear scatter dirty
+  if (!in->has_subtree_root_dirfrag(mds->get_nodeid()))
+    in->clear_scatter_dirty();
+
+  in->item_open_file.remove_myself();
+
+  in->clear_dirty_parent();
+
+  in->clear_file_locks();
+
+  // waiters
+  in->take_waiting(CInode::WAIT_ANY_MASK, finished);
+
+  in->finish_export(now);
+  
+  finish_export_inode_caps(in, peer, peer_imported);
+
+  // *** other state too?
+
+  // move to end of LRU so we drop out of cache quickly!
+  if (in->get_parent_dn()) 
+    cache->lru.lru_bottouch(in->get_parent_dn());
+
+}
+
+int Migrator::encode_export_dir(bufferlist& exportbl,
+				CDir *dir,
+				map& exported_client_map,
+				utime_t now)
+{
+  int num_exported = 0;
+
+  dout(7) << "encode_export_dir " << *dir << " " << dir->get_num_head_items() << " head items" << dendl;
+  
+  assert(dir->get_projected_version() == dir->get_version());
+
+#ifdef MDS_VERIFY_FRAGSTAT
+  if (dir->is_complete())
+    dir->verify_fragstat();
+#endif
+
+  // dir 
+  dirfrag_t df = dir->dirfrag();
+  ::encode(df, exportbl);
+  dir->encode_export(exportbl);
+  
+  __u32 nden = dir->items.size();
+  ::encode(nden, exportbl);
+  
+  // dentries
+  list subdirs;
+  CDir::map_t::iterator it;
+  for (it = dir->begin(); it != dir->end(); ++it) {
+    CDentry *dn = it->second;
+    CInode *in = dn->get_linkage()->get_inode();
+    
+    if (!dn->is_replicated())
+      dn->lock.replicate_relax();
+
+    num_exported++;
+    
+    // -- dentry
+    dout(7) << "encode_export_dir exporting " << *dn << dendl;
+    
+    // dn name
+    ::encode(dn->name, exportbl);
+    ::encode(dn->last, exportbl);
+    
+    // state
+    dn->encode_export(exportbl);
+    
+    // points to...
+    
+    // null dentry?
+    if (dn->get_linkage()->is_null()) {
+      exportbl.append("N", 1);  // null dentry
+      continue;
+    }
+    
+    if (dn->get_linkage()->is_remote()) {
+      // remote link
+      exportbl.append("L", 1);  // remote link
+      
+      inodeno_t ino = dn->get_linkage()->get_remote_ino();
+      unsigned char d_type = dn->get_linkage()->get_remote_d_type();
+      ::encode(ino, exportbl);
+      ::encode(d_type, exportbl);
+      continue;
+    }
+    
+    // primary link
+    // -- inode
+    exportbl.append("I", 1);    // inode dentry
+    
+    encode_export_inode(in, exportbl, exported_client_map);  // encode, and (update state for) export
+    
+    // directory?
+    list dfs;
+    in->get_dirfrags(dfs);
+    for (list::iterator p = dfs.begin(); p != dfs.end(); ++p) {
+      CDir *t = *p;
+      if (!t->state_test(CDir::STATE_EXPORTBOUND)) {
+	// include nested dirfrag
+	assert(t->get_dir_auth().first == CDIR_AUTH_PARENT);
+	subdirs.push_back(t);  // it's ours, recurse (later)
+      }
+    }
+  }
+
+  // subdirs
+  for (list::iterator it = subdirs.begin(); it != subdirs.end(); ++it)
+    num_exported += encode_export_dir(exportbl, *it, exported_client_map, now);
+
+  return num_exported;
+}
+
+void Migrator::finish_export_dir(CDir *dir, utime_t now, int peer,
+				 map >& peer_imported,
+				 list& finished)
+{
+  dout(10) << "finish_export_dir " << *dir << dendl;
+
+  // release open_by 
+  dir->clear_replica_map();
+
+  // mark
+  assert(dir->is_auth());
+  dir->state_clear(CDir::STATE_AUTH);
+  dir->remove_bloom();
+  dir->replica_nonce = CDir::EXPORT_NONCE;
+
+  if (dir->is_dirty())
+    dir->mark_clean();
+
+  // suck up all waiters
+  dir->take_waiting(CDir::WAIT_ANY_MASK, finished);    // all dir waiters
+  
+  // pop
+  dir->finish_export(now);
+
+  // dentries
+  list subdirs;
+  CDir::map_t::iterator it;
+  for (it = dir->begin(); it != dir->end(); ++it) {
+    CDentry *dn = it->second;
+    CInode *in = dn->get_linkage()->get_inode();
+
+    // dentry
+    dn->finish_export();
+
+    // inode?
+    if (dn->get_linkage()->is_primary()) {
+      finish_export_inode(in, now, peer, peer_imported[in->ino()], finished);
+
+      // subdirs?
+      in->get_nested_dirfrags(subdirs);
+    }
+  }
+
+  // subdirs
+  for (list::iterator it = subdirs.begin(); it != subdirs.end(); ++it) 
+    finish_export_dir(*it, now, peer, peer_imported, finished);
+}
+
+class C_MDS_ExportFinishLogged : public Context {
+  Migrator *migrator;
+  CDir *dir;
+public:
+  C_MDS_ExportFinishLogged(Migrator *m, CDir *d) : migrator(m), dir(d) {}
+  void finish(int r) {
+    migrator->export_logged_finish(dir);
+  }
+};
+
+
+/*
+ * i should get an export_ack from the export target.
+ *
+ * This function DOES put the passed message before returning
+ */
+void Migrator::handle_export_ack(MExportDirAck *m)
+{
+  CDir *dir = cache->get_dirfrag(m->get_dirfrag());
+  assert(dir);
+  assert(dir->is_frozen_tree_root());  // i'm exporting!
+
+  // yay!
+  dout(7) << "handle_export_ack " << *dir << dendl;
+
+  map::iterator it = export_state.find(dir);
+  assert(it != export_state.end());
+  assert(it->second.state == EXPORT_EXPORTING);
+  assert(it->second.tid == m->get_tid());
+
+  bufferlist::iterator bp = m->imported_caps.begin();
+  ::decode(it->second.peer_imported, bp);
+
+  it->second.state = EXPORT_LOGGINGFINISH;
+  assert (g_conf->mds_kill_export_at != 9);
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+
+  // list us second, them first.
+  // this keeps authority().first in sync with subtree auth state in the journal.
+  cache->adjust_subtree_auth(dir, it->second.peer, mds->get_nodeid());
+
+  // log completion. 
+  //  include export bounds, to ensure they're in the journal.
+  EExport *le = new EExport(mds->mdlog, dir);
+  mds->mdlog->start_entry(le);
+
+  le->metablob.add_dir_context(dir, EMetaBlob::TO_ROOT);
+  le->metablob.add_dir( dir, false );
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) {
+    CDir *bound = *p;
+    le->get_bounds().insert(bound->dirfrag());
+    le->metablob.add_dir_context(bound);
+    le->metablob.add_dir(bound, false);
+  }
+
+  // log export completion, then finish (unfreeze, trigger finish context, etc.)
+  mds->mdlog->submit_entry(le);
+  mds->mdlog->wait_for_safe(new C_MDS_ExportFinishLogged(this, dir));
+  mds->mdlog->flush();
+  assert (g_conf->mds_kill_export_at != 10);
+  
+  m->put();
+}
+
+void Migrator::export_notify_abort(CDir *dir, set& bounds)
+{
+  dout(7) << "export_notify_abort " << *dir << dendl;
+
+  export_state_t& stat = export_state[dir];
+
+  for (set::iterator p = stat.notify_ack_waiting.begin();
+       p != stat.notify_ack_waiting.end();
+       ++p) {
+    MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(),stat.tid,false,
+						    pair(mds->get_nodeid(),stat.peer),
+						    pair(mds->get_nodeid(),CDIR_AUTH_UNKNOWN));
+    for (set::iterator i = bounds.begin(); i != bounds.end(); ++i)
+      notify->get_bounds().push_back((*i)->dirfrag());
+    mds->send_message_mds(notify, *p);
+  }
+}
+
+/*
+ * this happens if hte dest failes after i send teh export data but before it is acked
+ * that is, we don't know they safely received and logged it, so we reverse our changes
+ * and go on.
+ */
+void Migrator::export_reverse(CDir *dir)
+{
+  dout(7) << "export_reverse " << *dir << dendl;
+
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+
+  // remove exporting pins
+  list rq;
+  rq.push_back(dir);
+  while (!rq.empty()) {
+    CDir *t = rq.front(); 
+    rq.pop_front();
+    t->abort_export();
+    for (CDir::map_t::iterator p = t->items.begin(); p != t->items.end(); ++p) {
+      p->second->abort_export();
+      if (!p->second->get_linkage()->is_primary())
+	continue;
+      CInode *in = p->second->get_linkage()->get_inode();
+      in->abort_export();
+      if (in->is_dir())
+	in->get_nested_dirfrags(rq);
+    }
+  }
+  
+  // unpin bounds
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) {
+    CDir *bd = *p;
+    bd->put(CDir::PIN_EXPORTBOUND);
+    bd->state_clear(CDir::STATE_EXPORTBOUND);
+  }
+
+  // adjust auth, with possible subtree merge.
+  cache->adjust_subtree_auth(dir, mds->get_nodeid());
+  cache->try_subtree_merge(dir);  // NOTE: may journal subtree_map as side-effect
+
+  // notify bystanders
+  export_notify_abort(dir, bounds);
+
+  // process delayed expires
+  cache->process_delayed_expire(dir);
+  
+  // unfreeze
+  dir->unfreeze_tree();
+
+  cache->show_cache();
+}
+
+
+/*
+ * once i get the ack, and logged the EExportFinish(true),
+ * send notifies (if any), otherwise go straight to finish.
+ * 
+ */
+void Migrator::export_logged_finish(CDir *dir)
+{
+  dout(7) << "export_logged_finish " << *dir << dendl;
+
+  export_state_t& stat = export_state[dir];
+
+  // send notifies
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+
+  for (set::iterator p = stat.notify_ack_waiting.begin();
+       p != stat.notify_ack_waiting.end();
+       ++p) {
+    MExportDirNotify *notify = new MExportDirNotify(dir->dirfrag(), stat.tid, true,
+						    pair(mds->get_nodeid(), stat.peer),
+						    pair(stat.peer, CDIR_AUTH_UNKNOWN));
+
+    for (set::iterator i = bounds.begin(); i != bounds.end(); ++i)
+      notify->get_bounds().push_back((*i)->dirfrag());
+    
+    mds->send_message_mds(notify, *p);
+  }
+
+  // wait for notifyacks
+  stat.state = EXPORT_NOTIFYING;
+  assert (g_conf->mds_kill_export_at != 11);
+  
+  // no notifies to wait for?
+  if (stat.notify_ack_waiting.empty()) {
+    export_finish(dir);  // skip notify/notify_ack stage.
+  } else {
+    // notify peer to send cap import messages to clients
+    if (mds->mdsmap->is_clientreplay_or_active_or_stopping(stat.peer)) {
+      mds->send_message_mds(new MExportDirFinish(dir->dirfrag(), false, stat.tid), stat.peer);
+    } else {
+      dout(7) << "not sending MExportDirFinish, dest has failed" << dendl;
+    }
+  }
+}
+
+/*
+ * warning:
+ *  i'll get an ack from each bystander.
+ *  when i get them all, do the export.
+ * notify:
+ *  i'll get an ack from each bystander.
+ *  when i get them all, unfreeze and send the finish.
+ *
+ * This function DOES put the passed message before returning
+ */
+void Migrator::handle_export_notify_ack(MExportDirNotifyAck *m)
+{
+  CDir *dir = cache->get_dirfrag(m->get_dirfrag());
+  assert(dir);
+  int from = m->get_source().num();
+
+  if (export_state.count(dir)) {
+    export_state_t& stat = export_state[dir];
+    if (stat.state == EXPORT_WARNING) {
+      // exporting. process warning.
+      dout(7) << "handle_export_notify_ack from " << m->get_source()
+	      << ": exporting, processing warning on " << *dir << dendl;
+      stat.warning_ack_waiting.erase(from);
+
+      if (stat.warning_ack_waiting.empty())
+	export_go(dir);     // start export.
+    } else if (stat.state == EXPORT_NOTIFYING) {
+      // exporting. process notify.
+      dout(7) << "handle_export_notify_ack from " << m->get_source()
+	      << ": exporting, processing notify on " << *dir << dendl;
+      stat.notify_ack_waiting.erase(from);
+
+      if (stat.notify_ack_waiting.empty())
+	export_finish(dir);
+    }
+  }
+  else if (import_state.count(dir->dirfrag())) {
+    import_state_t& stat = import_state[dir->dirfrag()];
+    if (stat.state == IMPORT_ABORTING) {
+      // reversing import
+      dout(7) << "handle_export_notify_ack from " << m->get_source()
+	      << ": aborting import on " << *dir << dendl;
+      assert(stat.bystanders.count(from));
+      stat.bystanders.erase(from);
+      if (stat.bystanders.empty())
+	import_reverse_unfreeze(dir);
+    }
+  }
+
+  m->put();
+}
+
+void Migrator::export_finish(CDir *dir)
+{
+  dout(5) << "export_finish " << *dir << dendl;
+
+  assert (g_conf->mds_kill_export_at != 12);
+  map::iterator it = export_state.find(dir);
+  if (it == export_state.end()) {
+    dout(7) << "target must have failed, not sending final commit message.  export succeeded anyway." << dendl;
+    return;
+  }
+
+  // send finish/commit to new auth
+  if (mds->mdsmap->is_clientreplay_or_active_or_stopping(it->second.peer)) {
+    mds->send_message_mds(new MExportDirFinish(dir->dirfrag(), true, it->second.tid), it->second.peer);
+  } else {
+    dout(7) << "not sending MExportDirFinish last, dest has failed" << dendl;
+  }
+  assert(g_conf->mds_kill_export_at != 13);
+  
+  // finish export (adjust local cache state)
+  C_Contexts *fin = new C_Contexts(g_ceph_context);
+  finish_export_dir(dir, ceph_clock_now(g_ceph_context),
+		    it->second.peer, it->second.peer_imported, fin->contexts);
+  dir->add_waiter(CDir::WAIT_UNFREEZE, fin);
+
+  // unfreeze
+  dout(7) << "export_finish unfreezing" << dendl;
+  dir->unfreeze_tree();
+  
+  // unpin bounds
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) {
+    CDir *bd = *p;
+    bd->put(CDir::PIN_EXPORTBOUND);
+    bd->state_clear(CDir::STATE_EXPORTBOUND);
+  }
+
+  // adjust auth, with possible subtree merge.
+  //  (we do this _after_ removing EXPORTBOUND pins, to allow merges)
+  cache->adjust_subtree_auth(dir, it->second.peer);
+  cache->try_subtree_merge(dir);  // NOTE: may journal subtree_map as sideeffect
+
+  // no more auth subtree? clear scatter dirty
+  if (!dir->get_inode()->is_auth() &&
+      !dir->get_inode()->has_subtree_root_dirfrag(mds->get_nodeid()))
+    dir->get_inode()->clear_scatter_dirty();
+
+  // discard delayed expires
+  cache->discard_delayed_expire(dir);
+
+  // remove from exporting list, clean up state
+  dir->state_clear(CDir::STATE_EXPORTING);
+
+  // queue finishers
+  mds->queue_waiters(it->second.waiting_for_finish);
+
+  // unpin path
+  MutationRef& mut = it->second.mut;
+  if (mut) {
+    mds->locker->drop_locks(mut.get());
+    mut->cleanup();
+  }
+
+  export_state.erase(it);
+
+  cache->show_subtrees();
+  audit();
+
+  // send pending import_maps?
+  mds->mdcache->maybe_send_pending_resolves();
+  
+  maybe_do_queued_export();
+}
+
+
+
+
+
+
+
+
+// ==========================================================
+// IMPORT
+
+void Migrator::handle_export_discover(MExportDirDiscover *m)
+{
+  int from = m->get_source_mds();
+  assert(from != mds->get_nodeid());
+
+  dout(7) << "handle_export_discover on " << m->get_path() << dendl;
+
+  // note import state
+  dirfrag_t df = m->get_dirfrag();
+  // only start discovering on this message once.
+  map::iterator it = import_state.find(df);
+  if (!m->started) {
+    assert(it == import_state.end());
+    m->started = true;
+    import_state[df].state = IMPORT_DISCOVERING;
+    import_state[df].peer = from;
+    import_state[df].tid = m->get_tid();
+  } else {
+    // am i retrying after ancient path_traverse results?
+    if (it == import_state.end() ||
+	it->second.peer != from ||
+	it->second.tid != m->get_tid()) {
+      dout(7) << " dropping obsolete message" << dendl;
+      m->put();
+      return;
+    }
+    assert(it->second.state == IMPORT_DISCOVERING);
+  }
+
+  if (!mds->mdcache->is_open()) {
+    dout(5) << " waiting for root" << dendl;
+    mds->mdcache->wait_for_open(new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+
+  assert (g_conf->mds_kill_import_at != 1);
+
+  // do we have it?
+  CInode *in = cache->get_inode(m->get_dirfrag().ino);
+  if (!in) {
+    // must discover it!
+    filepath fpath(m->get_path());
+    vector trace;
+    MDRequestRef null_ref;
+    int r = cache->path_traverse(null_ref, m, NULL, fpath, &trace, NULL, MDS_TRAVERSE_DISCOVER);
+    if (r > 0) return;
+    if (r < 0) {
+      dout(7) << "handle_export_discover_2 failed to discover or not dir " << m->get_path() << ", NAK" << dendl;
+      assert(0);    // this shouldn't happen if the auth pins his path properly!!!! 
+    }
+
+    assert(0); // this shouldn't happen; the get_inode above would have succeeded.
+  }
+
+  // yay
+  dout(7) << "handle_export_discover have " << df << " inode " << *in << dendl;
+  
+  import_state[df].state = IMPORT_DISCOVERED;
+
+  // pin inode in the cache (for now)
+  assert(in->is_dir());
+  in->get(CInode::PIN_IMPORTING);
+
+  // reply
+  dout(7) << " sending export_discover_ack on " << *in << dendl;
+  mds->send_message_mds(new MExportDirDiscoverAck(df, m->get_tid()), import_state[df].peer);
+  m->put();
+  assert (g_conf->mds_kill_import_at != 2);  
+}
+
+void Migrator::import_reverse_discovering(dirfrag_t df)
+{
+  import_state.erase(df);
+}
+
+void Migrator::import_reverse_discovered(dirfrag_t df, CInode *diri)
+{
+  // unpin base
+  diri->put(CInode::PIN_IMPORTING);
+  import_state.erase(df);
+}
+
+void Migrator::import_reverse_prepping(CDir *dir)
+{
+  set bounds;
+  cache->map_dirfrag_set(import_state[dir->dirfrag()].bound_ls, bounds);
+  import_remove_pins(dir, bounds);
+  import_reverse_final(dir);
+}
+
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_cancel(MExportDirCancel *m)
+{
+  dout(7) << "handle_export_cancel on " << m->get_dirfrag() << dendl;
+  dirfrag_t df = m->get_dirfrag();
+  map::iterator it = import_state.find(df);
+  if (it == import_state.end()) {
+    assert(0 == "got export_cancel in weird state");
+  } else if (it->second.state == IMPORT_DISCOVERING) {
+    import_reverse_discovering(df);
+  } else if (it->second.state == IMPORT_DISCOVERED) {
+    CInode *in = cache->get_inode(df.ino);
+    assert(in);
+    import_reverse_discovered(df, in);
+  } else if (it->second.state == IMPORT_PREPPING) {
+    CDir *dir = mds->mdcache->get_dirfrag(df);
+    assert(dir);
+    import_reverse_prepping(dir);
+  } else if (it->second.state == IMPORT_PREPPED) {
+    CDir *dir = mds->mdcache->get_dirfrag(df);
+    assert(dir);
+    set bounds;
+    cache->get_subtree_bounds(dir, bounds);
+    import_remove_pins(dir, bounds);
+    // adjust auth back to the exportor
+    cache->adjust_subtree_auth(dir, it->second.peer);
+    cache->try_subtree_merge(dir);
+    import_reverse_unfreeze(dir);
+  } else {
+    assert(0 == "got export_cancel in weird state");
+  }
+  m->put();
+}
+
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_prep(MExportDirPrep *m)
+{
+  int oldauth = m->get_source().num();
+  assert(oldauth != mds->get_nodeid());
+
+  CDir *dir;
+  CInode *diri;
+  list finished;
+
+  // assimilate root dir.
+  map::iterator it = import_state.find(m->get_dirfrag());
+  if (!m->did_assim()) {
+    assert(it != import_state.end());
+    assert(it->second.state == IMPORT_DISCOVERED);
+    diri = cache->get_inode(m->get_dirfrag().ino);
+    assert(diri);
+    bufferlist::iterator p = m->basedir.begin();
+    dir = cache->add_replica_dir(p, diri, oldauth, finished);
+    dout(7) << "handle_export_prep on " << *dir << " (first pass)" << dendl;
+  } else {
+    if (it == import_state.end() ||
+	it->second.peer != oldauth ||
+	it->second.tid != m->get_tid()) {
+      dout(7) << "handle_export_prep obsolete message, dropping" << dendl;
+      m->put();
+      return;
+    }
+    assert(it->second.state == IMPORT_PREPPING);
+
+    dir = cache->get_dirfrag(m->get_dirfrag());
+    assert(dir);
+    dout(7) << "handle_export_prep on " << *dir << " (subsequent pass)" << dendl;
+    diri = dir->get_inode();
+  }
+  assert(dir->is_auth() == false);
+
+  cache->show_subtrees();
+
+  // build import bound map
+  map import_bound_fragset;
+  for (list::iterator p = m->get_bounds().begin();
+       p != m->get_bounds().end();
+       ++p) {
+    dout(10) << " bound " << *p << dendl;
+    import_bound_fragset[p->ino].insert(p->frag);
+  }
+
+  // assimilate contents?
+  if (!m->did_assim()) {
+    dout(7) << "doing assim on " << *dir << dendl;
+    m->mark_assim();  // only do this the first time!
+
+    // change import state
+    it->second.state = IMPORT_PREPPING;
+    it->second.bound_ls = m->get_bounds();
+    it->second.bystanders = m->get_bystanders();
+    assert(g_conf->mds_kill_import_at != 3);
+
+    // bystander list
+    dout(7) << "bystanders are " << it->second.bystanders << dendl;
+
+    // move pin to dir
+    diri->put(CInode::PIN_IMPORTING);
+    dir->get(CDir::PIN_IMPORTING);  
+    dir->state_set(CDir::STATE_IMPORTING);
+
+    // assimilate traces to exports
+    // each trace is: df ('-' | ('f' dir | 'd') dentry inode (dir dentry inode)*)
+    for (list::iterator p = m->traces.begin();
+	 p != m->traces.end();
+	 ++p) {
+      bufferlist::iterator q = p->begin();
+      dirfrag_t df;
+      ::decode(df, q);
+      char start;
+      ::decode(start, q);
+      dout(10) << " trace from " << df << " start " << start << " len " << p->length() << dendl;
+
+      CDir *cur = 0;
+      if (start == 'd') {
+	cur = cache->get_dirfrag(df);
+	assert(cur);
+	dout(10) << "  had " << *cur << dendl;
+      } else if (start == 'f') {
+	CInode *in = cache->get_inode(df.ino);
+	assert(in);
+	dout(10) << "  had " << *in << dendl;
+	cur = cache->add_replica_dir(q, in, oldauth, finished);
+ 	dout(10) << "  added " << *cur << dendl;
+      } else if (start == '-') {
+	// nothing
+      } else
+	assert(0 == "unrecognized start char");
+
+      while (start != '-') {
+	CDentry *dn = cache->add_replica_dentry(q, cur, finished);
+	dout(10) << "  added " << *dn << dendl;
+	CInode *in = cache->add_replica_inode(q, dn, finished);
+	dout(10) << "  added " << *in << dendl;
+	if (q.end())
+	  break;
+	cur = cache->add_replica_dir(q, in, oldauth, finished);
+	dout(10) << "  added " << *cur << dendl;
+      }
+    }
+
+    // make bound sticky
+    for (map::iterator p = import_bound_fragset.begin();
+	 p != import_bound_fragset.end();
+	 ++p) {
+      CInode *in = cache->get_inode(p->first);
+      assert(in);
+      in->get_stickydirs();
+      dout(7) << " set stickydirs on bound inode " << *in << dendl;
+    }
+
+  } else {
+    dout(7) << " not doing assim on " << *dir << dendl;
+  }
+
+  if (!finished.empty())
+    mds->queue_waiters(finished);
+
+
+  // open all bounds
+  set import_bounds;
+  for (map::iterator p = import_bound_fragset.begin();
+       p != import_bound_fragset.end();
+       ++p) {
+    CInode *in = cache->get_inode(p->first);
+    assert(in);
+
+    // map fragset into a frag_t list, based on the inode fragtree
+    list fglist;
+    for (set::iterator q = p->second.begin(); q != p->second.end(); ++q)
+      in->dirfragtree.get_leaves_under(*q, fglist);
+    dout(10) << " bound inode " << p->first << " fragset " << p->second << " maps to " << fglist << dendl;
+    
+    for (list::iterator q = fglist.begin();
+	 q != fglist.end();
+	 ++q) {
+      CDir *bound = cache->get_dirfrag(dirfrag_t(p->first, *q));
+      if (!bound) {
+	dout(7) << "  opening bounding dirfrag " << *q << " on " << *in << dendl;
+	cache->open_remote_dirfrag(in, *q,
+				   new C_MDS_RetryMessage(mds, m));
+	return;
+      }
+
+      if (!bound->state_test(CDir::STATE_IMPORTBOUND)) {
+	dout(7) << "  pinning import bound " << *bound << dendl;
+	bound->get(CDir::PIN_IMPORTBOUND);
+	bound->state_set(CDir::STATE_IMPORTBOUND);
+      } else {
+	dout(7) << "  already pinned import bound " << *bound << dendl;
+      }
+      import_bounds.insert(bound);
+    }
+  }
+
+  dout(7) << " all ready, noting auth and freezing import region" << dendl;
+
+  bool success = true;
+  if (dir->get_inode()->filelock.can_wrlock(-1) &&
+      dir->get_inode()->nestlock.can_wrlock(-1)) {
+    it->second.mut = MutationRef(new MutationImpl);
+    // force some locks.  hacky.
+    mds->locker->wrlock_force(&dir->inode->filelock, it->second.mut);
+    mds->locker->wrlock_force(&dir->inode->nestlock, it->second.mut);
+
+    // note that i am an ambiguous auth for this subtree.
+    // specify bounds, since the exporter explicitly defines the region.
+    cache->adjust_bounded_subtree_auth(dir, import_bounds,
+				       pair(oldauth, mds->get_nodeid()));
+    cache->verify_subtree_bounds(dir, import_bounds);
+    // freeze.
+    dir->_freeze_tree();
+    // note new state
+    it->second.state = IMPORT_PREPPED;
+  } else {
+    dout(7) << " couldn't acquire all needed locks, failing. " << *dir << dendl;
+    success = false;
+    import_reverse_prepping(dir);
+  }
+
+  // ok!
+  dout(7) << " sending export_prep_ack on " << *dir << dendl;
+  mds->send_message(new MExportDirPrepAck(dir->dirfrag(), success, m->get_tid()), m->get_connection());
+
+  assert(g_conf->mds_kill_import_at != 4);
+  // done 
+  m->put();
+}
+
+
+
+
+class C_MDS_ImportDirLoggedStart : public Context {
+  Migrator *migrator;
+  dirfrag_t df;
+  CDir *dir;
+  int from;
+public:
+  map imported_client_map;
+  map sseqmap;
+
+  C_MDS_ImportDirLoggedStart(Migrator *m, CDir *d, int f) :
+    migrator(m), df(d->dirfrag()), dir(d), from(f) {
+  }
+  void finish(int r) {
+    migrator->import_logged_start(df, dir, from, imported_client_map, sseqmap);
+  }
+};
+
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_dir(MExportDir *m)
+{
+  assert (g_conf->mds_kill_import_at != 5);
+  CDir *dir = cache->get_dirfrag(m->dirfrag);
+  assert(dir);
+  
+  map::iterator it = import_state.find(m->dirfrag);
+  assert(it != import_state.end());
+  assert(it->second.state == IMPORT_PREPPED);
+  assert(it->second.tid == m->get_tid());
+
+  utime_t now = ceph_clock_now(g_ceph_context);
+  int oldauth = m->get_source().num();
+  dout(7) << "handle_export_dir importing " << *dir << " from " << oldauth << dendl;
+  assert(dir->is_auth() == false);
+
+  if (!dir->get_inode()->dirfragtree.is_leaf(dir->get_frag()))
+    dir->get_inode()->dirfragtree.force_to_leaf(g_ceph_context, dir->get_frag());
+
+  cache->show_subtrees();
+
+  C_MDS_ImportDirLoggedStart *onlogged = new C_MDS_ImportDirLoggedStart(this, dir, m->get_source().num());
+
+  // start the journal entry
+  EImportStart *le = new EImportStart(mds->mdlog, dir->dirfrag(), m->bounds);
+  mds->mdlog->start_entry(le);
+
+  le->metablob.add_dir_context(dir);
+  
+  // adjust auth (list us _first_)
+  cache->adjust_subtree_auth(dir, mds->get_nodeid(), oldauth);
+
+  // new client sessions, open these after we journal
+  // include imported sessions in EImportStart
+  bufferlist::iterator cmp = m->client_map.begin();
+  ::decode(onlogged->imported_client_map, cmp);
+  assert(cmp.end());
+  le->cmapv = mds->server->prepare_force_open_sessions(onlogged->imported_client_map, onlogged->sseqmap);
+  le->client_map.claim(m->client_map);
+
+  bufferlist::iterator blp = m->export_data.begin();
+  int num_imported_inodes = 0;
+  while (!blp.end()) {
+    num_imported_inodes += 
+      decode_import_dir(blp,
+			oldauth, 
+			dir,                 // import root
+			le,
+			mds->mdlog->get_current_segment(),
+			it->second.peer_exports,
+			it->second.updated_scatterlocks,
+			now);
+  }
+  dout(10) << " " << m->bounds.size() << " imported bounds" << dendl;
+  
+  // include bounds in EImportStart
+  set import_bounds;
+  cache->get_subtree_bounds(dir, import_bounds);
+  for (set::iterator it = import_bounds.begin();
+       it != import_bounds.end();
+       ++it) 
+    le->metablob.add_dir(*it, false);  // note that parent metadata is already in the event
+
+  // adjust popularity
+  mds->balancer->add_import(dir, now);
+
+  dout(7) << "handle_export_dir did " << *dir << dendl;
+
+  // note state
+  it->second.state = IMPORT_LOGGINGSTART;
+  assert (g_conf->mds_kill_import_at != 6);
+
+  // log it
+  mds->mdlog->submit_entry(le);
+  mds->mdlog->wait_for_safe(onlogged);
+  mds->mdlog->flush();
+
+  // some stats
+  if (mds->logger) {
+    mds->logger->inc(l_mds_im);
+    mds->logger->inc(l_mds_iim, num_imported_inodes);
+  }
+
+  m->put();
+}
+
+
+/*
+ * this is an import helper
+ *  called by import_finish, and import_reverse and friends.
+ */
+void Migrator::import_remove_pins(CDir *dir, set& bounds)
+{
+  import_state_t& stat = import_state[dir->dirfrag()];
+  // root
+  dir->put(CDir::PIN_IMPORTING);
+  dir->state_clear(CDir::STATE_IMPORTING);
+
+  // bounding inodes
+  set did;
+  for (list::iterator p = stat.bound_ls.begin();
+       p != stat.bound_ls.end();
+       ++p) {
+    if (did.count(p->ino))
+      continue;
+    did.insert(p->ino);
+    CInode *in = cache->get_inode(p->ino);
+    assert(in);
+    in->put_stickydirs();
+  }
+
+  if (stat.state >= IMPORT_PREPPED) {
+    // bounding dirfrags
+    for (set::iterator it = bounds.begin();
+	 it != bounds.end();
+	 ++it) {
+      CDir *bd = *it;
+      bd->put(CDir::PIN_IMPORTBOUND);
+      bd->state_clear(CDir::STATE_IMPORTBOUND);
+    }
+  }
+}
+
+
+/*
+ * note: this does teh full work of reversing and import and cleaning up
+ *  state.  
+ * called by both handle_mds_failure and by handle_resolve (if we are
+ *  a survivor coping with an exporter failure+recovery).
+ */
+void Migrator::import_reverse(CDir *dir)
+{
+  dout(7) << "import_reverse " << *dir << dendl;
+
+  import_state_t& stat = import_state[dir->dirfrag()];
+
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+
+  // remove pins
+  import_remove_pins(dir, bounds);
+
+  // update auth, with possible subtree merge.
+  assert(dir->is_subtree_root());
+  if (mds->is_resolve())
+    cache->trim_non_auth_subtree(dir);
+
+  cache->adjust_subtree_auth(dir, stat.peer);
+
+  if (!dir->get_inode()->is_auth() &&
+      !dir->get_inode()->has_subtree_root_dirfrag(mds->get_nodeid()))
+    dir->get_inode()->clear_scatter_dirty();
+
+  // adjust auth bits.
+  list q;
+  q.push_back(dir);
+  while (!q.empty()) {
+    CDir *cur = q.front();
+    q.pop_front();
+    
+    // dir
+    assert(cur->is_auth());
+    cur->state_clear(CDir::STATE_AUTH);
+    cur->remove_bloom();
+    cur->clear_replica_map();
+    if (cur->is_dirty())
+      cur->mark_clean();
+
+    CDir::map_t::iterator it;
+    for (it = cur->begin(); it != cur->end(); ++it) {
+      CDentry *dn = it->second;
+
+      // dentry
+      dn->state_clear(CDentry::STATE_AUTH);
+      dn->clear_replica_map();
+      if (dn->is_dirty()) 
+	dn->mark_clean();
+
+      // inode?
+      if (dn->get_linkage()->is_primary()) {
+	CInode *in = dn->get_linkage()->get_inode();
+	in->state_clear(CDentry::STATE_AUTH);
+	in->clear_replica_map();
+	if (in->is_dirty()) 
+	  in->mark_clean();
+	in->clear_dirty_rstat();
+	if (!in->has_subtree_root_dirfrag(mds->get_nodeid()))
+	  in->clear_scatter_dirty();
+
+	in->clear_dirty_parent();
+
+	in->authlock.clear_gather();
+	in->linklock.clear_gather();
+	in->dirfragtreelock.clear_gather();
+	in->filelock.clear_gather();
+
+	in->clear_file_locks();
+
+	// non-bounding dir?
+	list dfs;
+	in->get_dirfrags(dfs);
+	for (list::iterator p = dfs.begin(); p != dfs.end(); ++p)
+	  if (bounds.count(*p) == 0)
+	    q.push_back(*p);
+      }
+    }
+  }
+
+  if (stat.state == IMPORT_ACKING) {
+    // remove imported caps
+    for (map >::iterator p = stat.peer_exports.begin();
+	p != stat.peer_exports.end();
+	++p) {
+      CInode *in = p->first;
+      for (map::iterator q = p->second.begin();
+	  q != p->second.end();
+	  ++q) {
+	Capability *cap = in->get_client_cap(q->first);
+	assert(cap);
+	if (cap->is_new())
+	  in->remove_client_cap(q->first);
+      }
+      in->put(CInode::PIN_IMPORTINGCAPS);
+    }
+    for (map::iterator p = stat.client_map.begin();
+	 p != stat.client_map.end();
+	 ++p) {
+      Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->first.v));
+      assert(session);
+      session->dec_importing();
+    }
+  }
+	 
+  // log our failure
+  mds->mdlog->start_submit_entry(new EImportFinish(dir, false));	// log failure
+
+  cache->try_subtree_merge(dir);  // NOTE: this may journal subtree map as side effect
+
+  // bystanders?
+  if (stat.bystanders.empty()) {
+    dout(7) << "no bystanders, finishing reverse now" << dendl;
+    import_reverse_unfreeze(dir);
+  } else {
+    // notify them; wait in aborting state
+    dout(7) << "notifying bystanders of abort" << dendl;
+    import_notify_abort(dir, bounds);
+    stat.state = IMPORT_ABORTING;
+    assert (g_conf->mds_kill_import_at != 10);
+  }
+}
+
+void Migrator::import_notify_finish(CDir *dir, set& bounds)
+{
+  dout(7) << "import_notify_finish " << *dir << dendl;
+
+  import_state_t& stat = import_state[dir->dirfrag()];
+  for (set::iterator p = stat.bystanders.begin();
+       p != stat.bystanders.end();
+       ++p) {
+    MExportDirNotify *notify =
+      new MExportDirNotify(dir->dirfrag(), stat.tid, false,
+			   pair(stat.peer, mds->get_nodeid()),
+			   pair(mds->get_nodeid(), CDIR_AUTH_UNKNOWN));
+    for (set::iterator i = bounds.begin(); i != bounds.end(); ++i)
+      notify->get_bounds().push_back((*i)->dirfrag());
+    mds->send_message_mds(notify, *p);
+  }
+}
+
+void Migrator::import_notify_abort(CDir *dir, set& bounds)
+{
+  dout(7) << "import_notify_abort " << *dir << dendl;
+  
+  import_state_t& stat = import_state[dir->dirfrag()];
+  for (set::iterator p = stat.bystanders.begin();
+       p != stat.bystanders.end();
+       ++p) {
+    MExportDirNotify *notify =
+      new MExportDirNotify(dir->dirfrag(), stat.tid, true,
+			   pair(stat.peer, mds->get_nodeid()),
+			   pair(stat.peer, CDIR_AUTH_UNKNOWN));
+    for (set::iterator i = bounds.begin(); i != bounds.end(); ++i)
+      notify->get_bounds().push_back((*i)->dirfrag());
+    mds->send_message_mds(notify, *p);
+  }
+}
+
+void Migrator::import_reverse_unfreeze(CDir *dir)
+{
+  assert(dir);
+  dout(7) << "import_reverse_unfreeze " << *dir << dendl;
+  dir->unfreeze_tree();
+  list ls;
+  mds->queue_waiters(ls);
+  cache->discard_delayed_expire(dir);
+  import_reverse_final(dir);
+}
+
+void Migrator::import_reverse_final(CDir *dir) 
+{
+  dout(7) << "import_reverse_final " << *dir << dendl;
+
+  // clean up
+  map::iterator it = import_state.find(dir->dirfrag());
+  if (it->second.mut) {
+    mds->locker->drop_locks(it->second.mut.get());
+    it->second.mut->cleanup();
+  }
+  import_state.erase(it);
+
+  // send pending import_maps?
+  mds->mdcache->maybe_send_pending_resolves();
+
+  cache->show_subtrees();
+  //audit();  // this fails, bc we munge up the subtree map during handle_import_map (resolve phase)
+}
+
+
+
+
+void Migrator::import_logged_start(dirfrag_t df, CDir *dir, int from,
+				   map& imported_client_map,
+				   map& sseqmap)
+{
+  map::iterator it = import_state.find(dir->dirfrag());
+  if (it == import_state.end() ||
+      it->second.state != IMPORT_LOGGINGSTART) {
+    dout(7) << "import " << df << " must have aborted" << dendl;
+    mds->server->finish_force_open_sessions(imported_client_map, sseqmap);
+    return;
+  }
+
+  dout(7) << "import_logged " << *dir << dendl;
+
+  // note state
+  it->second.state = IMPORT_ACKING;
+
+  assert (g_conf->mds_kill_import_at != 7);
+
+  // force open client sessions and finish cap import
+  mds->server->finish_force_open_sessions(imported_client_map, sseqmap, false);
+  it->second.client_map.swap(imported_client_map);
+  
+  map > imported_caps;
+  for (map >::iterator p = it->second.peer_exports.begin();
+       p != it->second.peer_exports.end();
+       ++p) {
+    // parameter 'peer' is -1, delay sending cap import messages to client
+    finish_import_inode_caps(p->first, -1, true, p->second, imported_caps[p->first->ino()]);
+  }
+  
+  // send notify's etc.
+  dout(7) << "sending ack for " << *dir << " to old auth mds." << from << dendl;
+
+  // test surviving observer of a failed migration that did not complete
+  //assert(dir->replica_map.size() < 2 || mds->whoami != 0);
+
+  MExportDirAck *ack = new MExportDirAck(dir->dirfrag(), it->second.tid);
+  ::encode(imported_caps, ack->imported_caps);
+
+  mds->send_message_mds(ack, from);
+  assert (g_conf->mds_kill_import_at != 8);
+
+  cache->show_subtrees();
+}
+
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_finish(MExportDirFinish *m)
+{
+  CDir *dir = cache->get_dirfrag(m->get_dirfrag());
+  assert(dir);
+  dout(7) << "handle_export_finish on " << *dir << (m->is_last() ? " last" : "") << dendl;
+
+  map::iterator it = import_state.find(m->get_dirfrag());
+  assert(it != import_state.end());
+  assert(it->second.tid == m->get_tid());
+
+  import_finish(dir, false, m->is_last());
+
+  m->put();
+}
+
+void Migrator::import_finish(CDir *dir, bool notify, bool last)
+{
+  dout(7) << "import_finish on " << *dir << dendl;
+
+  map::iterator it = import_state.find(dir->dirfrag());
+  assert(it != import_state.end());
+  assert(it->second.state == IMPORT_ACKING || it->second.state == IMPORT_FINISHING);
+
+  // log finish
+  assert(g_conf->mds_kill_import_at != 9);
+
+  if (it->second.state == IMPORT_ACKING) {
+    for (map >::iterator p = it->second.peer_exports.begin();
+	p != it->second.peer_exports.end();
+	++p) {
+      CInode *in = p->first;
+      assert(in->is_auth());
+      for (map::iterator q = p->second.begin();
+	  q != p->second.end();
+	  ++q) {
+	Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v));
+	assert(session);
+	Capability *cap = in->get_client_cap(q->first);
+	assert(cap);
+	cap->merge(q->second, true);
+	mds->mdcache->do_cap_import(session, in, cap, q->second.cap_id, q->second.seq,
+				    q->second.mseq - 1, it->second.peer, CEPH_CAP_FLAG_AUTH);
+      }
+      p->second.clear();
+      in->replica_caps_wanted = 0;
+    }
+    for (map::iterator p = it->second.client_map.begin();
+	 p != it->second.client_map.end();
+	 ++p) {
+      Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->first.v));
+      assert(session);
+      session->dec_importing();
+    }
+  }
+
+  if (!last) {
+    assert(it->second.state == IMPORT_ACKING);
+    it->second.state = IMPORT_FINISHING;
+    return;
+  }
+
+  // clear updated scatterlocks
+  /*
+  for (list::iterator p = import_updated_scatterlocks[dir].begin();
+       p != import_updated_scatterlocks[dir].end();
+       ++p) 
+    (*p)->clear_updated();
+  */
+
+  // remove pins
+  set bounds;
+  cache->get_subtree_bounds(dir, bounds);
+
+  if (notify)
+    import_notify_finish(dir, bounds);
+
+  import_remove_pins(dir, bounds);
+
+  map > peer_exports;
+  it->second.peer_exports.swap(peer_exports);
+
+  // clear import state (we're done!)
+  MutationRef mut = it->second.mut;
+  import_state.erase(it);
+
+  mds->mdlog->start_submit_entry(new EImportFinish(dir, true));
+
+  // adjust auth, with possible subtree merge.
+  cache->adjust_subtree_auth(dir, mds->get_nodeid());
+  cache->try_subtree_merge(dir);   // NOTE: this may journal subtree_map as sideffect
+
+  // process delayed expires
+  cache->process_delayed_expire(dir);
+
+  // ok now unfreeze (and thus kick waiters)
+  dir->unfreeze_tree();
+  cache->show_subtrees();
+  //audit();  // this fails, bc we munge up the subtree map during handle_import_map (resolve phase)
+
+  if (mut) {
+    mds->locker->drop_locks(mut.get());
+    mut->cleanup();
+  }
+
+  // re-eval imported caps
+  for (map >::iterator p = peer_exports.begin();
+       p != peer_exports.end();
+       ++p) {
+    if (p->first->is_auth())
+      mds->locker->eval(p->first, CEPH_CAP_LOCKS, true);
+    p->first->put(CInode::PIN_IMPORTINGCAPS);
+  }
+
+  // send pending import_maps?
+  mds->mdcache->maybe_send_pending_resolves();
+
+  // did i just import mydir?
+  if (dir->ino() == MDS_INO_MDSDIR(mds->whoami))
+    cache->populate_mydir();
+
+  // is it empty?
+  if (dir->get_num_head_items() == 0 &&
+      !dir->inode->is_auth()) {
+    // reexport!
+    export_empty_import(dir);
+  }
+}
+
+
+void Migrator::decode_import_inode(CDentry *dn, bufferlist::iterator& blp, int oldauth,
+				   LogSegment *ls, uint64_t log_offset,
+				   map >& peer_exports,
+				   list& updated_scatterlocks)
+{  
+  dout(15) << "decode_import_inode on " << *dn << dendl;
+
+  inodeno_t ino;
+  snapid_t last;
+  ::decode(ino, blp);
+  ::decode(last, blp);
+
+  bool added = false;
+  CInode *in = cache->get_inode(ino, last);
+  if (!in) {
+    in = new CInode(mds->mdcache, true, 1, last);
+    added = true;
+  } else {
+    in->state_set(CInode::STATE_AUTH);
+  }
+
+  // state after link  -- or not!  -sage
+  in->decode_import(blp, ls);  // cap imports are noted for later action
+
+  // note that we are journaled at this log offset
+  in->last_journaled = log_offset;
+
+  // caps
+  decode_import_inode_caps(in, true, blp, peer_exports);
+
+  // link before state  -- or not!  -sage
+  if (dn->get_linkage()->get_inode() != in) {
+    assert(!dn->get_linkage()->get_inode());
+    dn->dir->link_primary_inode(dn, in);
+  }
+ 
+  // add inode?
+  if (added) {
+    cache->add_inode(in);
+    dout(10) << "added " << *in << dendl;
+  } else {
+    dout(10) << "  had " << *in << dendl;
+  }
+
+  if (in->inode.is_dirty_rstat())
+    in->mark_dirty_rstat();
+  
+  // clear if dirtyscattered, since we're going to journal this
+  //  but not until we _actually_ finish the import...
+  if (in->filelock.is_dirty()) {
+    updated_scatterlocks.push_back(&in->filelock);
+    mds->locker->mark_updated_scatterlock(&in->filelock);
+  }
+
+  if (in->dirfragtreelock.is_dirty()) {
+    updated_scatterlocks.push_back(&in->dirfragtreelock);
+    mds->locker->mark_updated_scatterlock(&in->dirfragtreelock);
+  }
+
+  // adjust replica list
+  //assert(!in->is_replica(oldauth));  // not true on failed export
+  in->add_replica(oldauth, CInode::EXPORT_NONCE);
+  if (in->is_replica(mds->get_nodeid()))
+    in->remove_replica(mds->get_nodeid());
+  
+}
+
+void Migrator::decode_import_inode_caps(CInode *in, bool auth_cap,
+					bufferlist::iterator &blp,
+					map >& peer_exports)
+{
+  map cap_map;
+  ::decode(cap_map, blp);
+  if (auth_cap)
+    ::decode(in->get_mds_caps_wanted(), blp);
+  if (!cap_map.empty() ||
+      (auth_cap && !in->get_mds_caps_wanted().empty())) {
+    peer_exports[in].swap(cap_map);
+    in->get(CInode::PIN_IMPORTINGCAPS);
+  }
+}
+
+void Migrator::finish_import_inode_caps(CInode *in, int peer, bool auth_cap,
+					map &export_map,
+					map &import_map)
+{
+  for (map::iterator it = export_map.begin();
+       it != export_map.end();
+       ++it) {
+    dout(10) << "finish_import_inode_caps for client." << it->first << " on " << *in << dendl;
+    Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(it->first.v));
+    assert(session);
+
+    Capability *cap = in->get_client_cap(it->first);
+    if (!cap) {
+      cap = in->add_client_cap(it->first, session);
+      if (peer < 0)
+	cap->mark_new();
+    }
+
+    Capability::Import& im = import_map[it->first];
+    im.cap_id = cap->get_cap_id();
+    im.mseq = auth_cap ? it->second.mseq : cap->get_mseq();
+    im.issue_seq = cap->get_last_seq() + 1;
+
+    if (peer >= 0) {
+      cap->merge(it->second, auth_cap);
+      mds->mdcache->do_cap_import(session, in, cap, it->second.cap_id,
+				  it->second.seq, it->second.mseq - 1, peer,
+				  auth_cap ? CEPH_CAP_FLAG_AUTH : CEPH_CAP_FLAG_RELEASE);
+    }
+  }
+
+  if (peer >= 0) {
+    in->replica_caps_wanted = 0;
+    in->put(CInode::PIN_IMPORTINGCAPS);
+  }
+}
+
+int Migrator::decode_import_dir(bufferlist::iterator& blp,
+				int oldauth,
+				CDir *import_root,
+				EImportStart *le,
+				LogSegment *ls,
+				map >& peer_exports,
+				list& updated_scatterlocks, utime_t now)
+{
+  // set up dir
+  dirfrag_t df;
+  ::decode(df, blp);
+
+  CInode *diri = cache->get_inode(df.ino);
+  assert(diri);
+  CDir *dir = diri->get_or_open_dirfrag(mds->mdcache, df.frag);
+  assert(dir);
+  
+  dout(7) << "decode_import_dir " << *dir << dendl;
+
+  // assimilate state
+  dir->decode_import(blp, now, ls);
+
+  // mark  (may already be marked from get_or_open_dir() above)
+  if (!dir->is_auth())
+    dir->state_set(CDir::STATE_AUTH);
+
+  // adjust replica list
+  //assert(!dir->is_replica(oldauth));    // not true on failed export
+  dir->add_replica(oldauth, CDir::EXPORT_NONCE);
+  if (dir->is_replica(mds->get_nodeid()))
+    dir->remove_replica(mds->get_nodeid());
+
+  // add to journal entry
+  if (le) 
+    le->metablob.add_import_dir(dir);
+
+  int num_imported = 0;
+
+  // take all waiters on this dir
+  // NOTE: a pass of imported data is guaranteed to get all of my waiters because
+  // a replica's presense in my cache implies/forces it's presense in authority's.
+  list waiters;
+  
+  dir->take_waiting(CDir::WAIT_ANY_MASK, waiters);
+  for (list::iterator it = waiters.begin();
+       it != waiters.end();
+       ++it) 
+    import_root->add_waiter(CDir::WAIT_UNFREEZE, *it);  // UNFREEZE will get kicked both on success or failure
+  
+  dout(15) << "doing contents" << dendl;
+  
+  // contents
+  __u32 nden;
+  ::decode(nden, blp);
+  
+  for (; nden>0; nden--) {
+    num_imported++;
+    
+    // dentry
+    string dname;
+    snapid_t last;
+    ::decode(dname, blp);
+    ::decode(last, blp);
+    
+    CDentry *dn = dir->lookup_exact_snap(dname, last);
+    if (!dn)
+      dn = dir->add_null_dentry(dname, 1, last);
+    
+    dn->decode_import(blp, ls);
+
+    dn->add_replica(oldauth, CDentry::EXPORT_NONCE);
+    if (dn->is_replica(mds->get_nodeid()))
+      dn->remove_replica(mds->get_nodeid());
+
+    // dentry lock in unreadable state can block path traverse
+    if (dn->lock.get_state() != LOCK_SYNC)
+      mds->locker->try_eval(&dn->lock, NULL);
+
+    dout(15) << "decode_import_dir got " << *dn << dendl;
+    
+    // points to...
+    char icode;
+    ::decode(icode, blp);
+    
+    if (icode == 'N') {
+      // null dentry
+      assert(dn->get_linkage()->is_null());  
+      
+      // fall thru
+    }
+    else if (icode == 'L') {
+      // remote link
+      inodeno_t ino;
+      unsigned char d_type;
+      ::decode(ino, blp);
+      ::decode(d_type, blp);
+      if (dn->get_linkage()->is_remote()) {
+	assert(dn->get_linkage()->get_remote_ino() == ino);
+      } else {
+	dir->link_remote_inode(dn, ino, d_type);
+      }
+    }
+    else if (icode == 'I') {
+      // inode
+      assert(le);
+      decode_import_inode(dn, blp, oldauth, ls, le->get_start_off(), peer_exports, updated_scatterlocks);
+    }
+    
+    // add dentry to journal entry
+    if (le)
+      le->metablob.add_import_dentry(dn);
+  }
+  
+#ifdef MDS_VERIFY_FRAGSTAT
+  if (dir->is_complete())
+    dir->verify_fragstat();
+#endif
+
+  dout(7) << "decode_import_dir done " << *dir << dendl;
+  return num_imported;
+}
+
+
+
+
+
+// authority bystander
+
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_notify(MExportDirNotify *m)
+{
+  CDir *dir = cache->get_dirfrag(m->get_dirfrag());
+
+  int from = m->get_source().num();
+  pair old_auth = m->get_old_auth();
+  pair new_auth = m->get_new_auth();
+  
+  if (!dir) {
+    dout(7) << "handle_export_notify " << old_auth << " -> " << new_auth
+	    << " on missing dir " << m->get_dirfrag() << dendl;
+  } else if (dir->authority() != old_auth) {
+    dout(7) << "handle_export_notify old_auth was " << dir->authority() 
+	    << " != " << old_auth << " -> " << new_auth
+	    << " on " << *dir << dendl;
+  } else {
+    dout(7) << "handle_export_notify " << old_auth << " -> " << new_auth
+	    << " on " << *dir << dendl;
+    // adjust auth
+    set have;
+    cache->map_dirfrag_set(m->get_bounds(), have);
+    cache->adjust_bounded_subtree_auth(dir, have, new_auth);
+    
+    // induce a merge?
+    cache->try_subtree_merge(dir);
+  }
+  
+  // send ack
+  if (m->wants_ack()) {
+    mds->send_message_mds(new MExportDirNotifyAck(m->get_dirfrag(), m->get_tid()), from);
+  } else {
+    // aborted.  no ack.
+    dout(7) << "handle_export_notify no ack requested" << dendl;
+  }
+  
+  m->put();
+}
+
+/** cap exports **/
+void Migrator::export_caps(CInode *in)
+{
+  int dest = in->authority().first;
+  dout(7) << "export_caps to mds." << dest << " " << *in << dendl;
+
+  assert(in->is_any_caps());
+  assert(!in->is_auth());
+  assert(!in->is_ambiguous_auth());
+  assert(!in->state_test(CInode::STATE_EXPORTINGCAPS));
+
+  MExportCaps *ex = new MExportCaps;
+  ex->ino = in->ino();
+
+  encode_export_inode_caps(in, false, ex->cap_bl, ex->client_map);
+
+  mds->send_message_mds(ex, dest);
+}
+
+class C_M_LoggedImportCaps : public Context {
+  Migrator *migrator;
+  CInode *in;
+  int from;
+public:
+  map > peer_exports;
+  map client_map;
+  map sseqmap;
+
+  C_M_LoggedImportCaps(Migrator *m, CInode *i, int f) : migrator(m), in(i), from(f) {}
+  void finish(int r) {
+    migrator->logged_import_caps(in, from, peer_exports, client_map, sseqmap);
+  }  
+};
+
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_caps(MExportCaps *ex)
+{
+  dout(10) << "handle_export_caps " << *ex << " from " << ex->get_source() << dendl;
+  CInode *in = cache->get_inode(ex->ino);
+  
+  assert(in);
+  assert(in->is_auth());
+
+  // FIXME
+  if (in->is_frozen())
+    return;
+
+  C_M_LoggedImportCaps *finish = new C_M_LoggedImportCaps(this, in, ex->get_source().num());
+  finish->client_map = ex->client_map;
+
+  // decode new caps
+  bufferlist::iterator blp = ex->cap_bl.begin();
+  decode_import_inode_caps(in, false, blp, finish->peer_exports);
+  assert(!finish->peer_exports.empty());   // thus, inode is pinned.
+
+  // journal open client sessions
+  version_t pv = mds->server->prepare_force_open_sessions(finish->client_map, finish->sseqmap);
+  
+  ESessions *le = new ESessions(pv, ex->client_map);
+  mds->mdlog->start_entry(le);
+  mds->mdlog->submit_entry(le);
+  mds->mdlog->wait_for_safe(finish);
+  mds->mdlog->flush();
+
+  ex->put();
+}
+
+
+void Migrator::logged_import_caps(CInode *in, 
+				  int from,
+				  map >& peer_exports,
+				  map& client_map,
+				  map& sseqmap) 
+{
+  dout(10) << "logged_import_caps on " << *in << dendl;
+  // see export_go() vs export_go_synced()
+  assert(in->is_auth());
+
+  // force open client sessions and finish cap import
+  mds->server->finish_force_open_sessions(client_map, sseqmap);
+
+  map imported_caps;
+
+  assert(peer_exports.count(in));
+  // clients will release caps from the exporter when they receive the cap import message.
+  finish_import_inode_caps(in, from, false, peer_exports[in], imported_caps);
+  mds->locker->eval(in, CEPH_CAP_LOCKS, true);
+}
diff --git a/ceph/src/mds/Migrator.h b/ceph/src/mds/Migrator.h
new file mode 100644
index 00000000..0819c821
--- /dev/null
+++ b/ceph/src/mds/Migrator.h
@@ -0,0 +1,352 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ * Handles the import and export of  mds authorities and actual cache data.
+ * See src/doc/exports.txt for a description.
+ */
+
+#ifndef CEPH_MDS_MIGRATOR_H
+#define CEPH_MDS_MIGRATOR_H
+
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+using std::map;
+using std::list;
+using std::set;
+
+
+class MDS;
+class CDir;
+class CInode;
+class CDentry;
+
+class MExportDirDiscover;
+class MExportDirDiscoverAck;
+class MExportDirCancel;
+class MExportDirPrep;
+class MExportDirPrepAck;
+class MExportDir;
+class MExportDirAck;
+class MExportDirNotify;
+class MExportDirNotifyAck;
+class MExportDirFinish;
+
+class MExportCaps;
+class MExportCapsAck;
+
+class EImportStart;
+
+class Migrator {
+private:
+  MDS *mds;
+  MDCache *cache;
+
+  // -- exports --
+public:
+  // export stages.  used to clean up intelligently if there's a failure.
+  const static int EXPORT_CANCELLED	= 0;  // cancelled
+  const static int EXPORT_LOCKING	= 1;  // acquiring locks
+  const static int EXPORT_DISCOVERING	= 2;  // dest is disovering export dir
+  const static int EXPORT_FREEZING	= 3;  // we're freezing the dir tree
+  const static int EXPORT_PREPPING	= 4;  // sending dest spanning tree to export bounds
+  const static int EXPORT_WARNING	= 5;  // warning bystanders of dir_auth_pending
+  const static int EXPORT_EXPORTING	= 6;  // sent actual export, waiting for ack
+  const static int EXPORT_LOGGINGFINISH	= 7;  // logging EExportFinish
+  const static int EXPORT_NOTIFYING	= 8;  // waiting for notifyacks
+  static const char *get_export_statename(int s) {
+    switch (s) {
+    case EXPORT_LOCKING: return "locking";
+    case EXPORT_DISCOVERING: return "discovering";
+    case EXPORT_FREEZING: return "freezing";
+    case EXPORT_PREPPING: return "prepping";
+    case EXPORT_WARNING: return "warning";
+    case EXPORT_EXPORTING: return "exporting";
+    case EXPORT_LOGGINGFINISH: return "loggingfinish";
+    case EXPORT_NOTIFYING: return "notifying";
+    default: assert(0); return 0;
+    }
+  }
+
+protected:
+  // export fun
+  struct export_state_t {
+    int state;
+    int peer;
+    uint64_t tid;
+    set warning_ack_waiting;
+    set notify_ack_waiting;
+    map > peer_imported;
+    list waiting_for_finish;
+    MutationRef mut;
+    // for freeze tree deadlock detection
+    utime_t last_cum_auth_pins_change;
+    int last_cum_auth_pins;
+    int num_remote_waiters; // number of remote authpin waiters
+    export_state_t() : state(0), peer(0), tid(0), mut(), 
+		       last_cum_auth_pins(0), num_remote_waiters(0) {}
+  };
+
+  map  export_state;
+  
+  list >  export_queue;
+
+
+  // -- imports --
+public:
+  const static int IMPORT_DISCOVERING   = 1; // waiting for prep
+  const static int IMPORT_DISCOVERED    = 2; // waiting for prep
+  const static int IMPORT_PREPPING      = 3; // opening dirs on bounds
+  const static int IMPORT_PREPPED       = 4; // opened bounds, waiting for import
+  const static int IMPORT_LOGGINGSTART  = 5; // got import, logging EImportStart
+  const static int IMPORT_ACKING        = 6; // logged EImportStart, sent ack, waiting for finish
+  const static int IMPORT_FINISHING     = 7; // sent cap imports, waiting for finish
+  const static int IMPORT_ABORTING      = 8; // notifying bystanders of an abort before unfreezing
+
+  static const char *get_import_statename(int s) {
+    switch (s) {
+    case IMPORT_DISCOVERING: return "discovering";
+    case IMPORT_DISCOVERED: return "discovered";
+    case IMPORT_PREPPING: return "prepping";
+    case IMPORT_PREPPED: return "prepped";
+    case IMPORT_LOGGINGSTART: return "loggingstart";
+    case IMPORT_ACKING: return "acking";
+    case IMPORT_FINISHING: return "finishing";
+    case IMPORT_ABORTING: return "aborting";
+    default: assert(0); return 0;
+    }
+  }
+
+protected:
+  struct import_state_t {
+    int state;
+    int peer;
+    uint64_t tid;
+    set bystanders;
+    list bound_ls;
+    list updated_scatterlocks;
+    map client_map;
+    map > peer_exports;
+    MutationRef mut;
+    import_state_t() : state(0), peer(0), tid(0), mut() {}
+  };
+
+  map  import_state;
+
+public:
+  // -- cons --
+  Migrator(MDS *m, MDCache *c) : mds(m), cache(c) {}
+
+  void dispatch(Message*);
+
+  void show_importing();
+  void show_exporting();
+  
+  // -- status --
+  int is_exporting(CDir *dir) {
+    map::iterator it = export_state.find(dir);
+    if (it != export_state.end()) return it->second.state;
+    return 0;
+  }
+  bool is_exporting() { return !export_state.empty(); }
+  int is_importing(dirfrag_t df) {
+    map::iterator it = import_state.find(df);
+    if (it != import_state.end()) return it->second.state;
+    return 0;
+  }
+  bool is_importing() { return !import_state.empty(); }
+
+  bool is_ambiguous_import(dirfrag_t df) {
+    map::iterator p = import_state.find(df);
+    if (p == import_state.end())
+      return false;
+    if (p->second.state >= IMPORT_LOGGINGSTART &&
+	p->second.state < IMPORT_ABORTING)
+      return true;
+    return false;
+  }
+
+  int get_import_state(dirfrag_t df) {
+    map::iterator it = import_state.find(df);
+    assert(it != import_state.end());
+    return it->second.state;
+  }
+  int get_import_peer(dirfrag_t df) {
+    map::iterator it = import_state.find(df);
+    assert(it != import_state.end());
+    return it->second.peer;
+  }
+
+  int get_export_state(CDir *dir) {
+    map::iterator it = export_state.find(dir);
+    assert(it != export_state.end());
+    return it->second.state;
+  }
+  // this returns true if we are export @dir,
+  // and are not waiting for @who to be
+  // be warned of ambiguous auth.
+  // only returns meaningful results during EXPORT_WARNING state.
+  bool export_has_warned(CDir *dir, int who) {
+    map::iterator it = export_state.find(dir);
+    assert(it != export_state.end());
+    assert(it->second.state == EXPORT_WARNING);
+    return (it->second.warning_ack_waiting.count(who) == 0);
+  }
+
+  bool export_has_notified(CDir *dir, int who) {
+    map::iterator it = export_state.find(dir);
+    assert(it != export_state.end());
+    assert(it->second.state == EXPORT_NOTIFYING);
+    return (it->second.notify_ack_waiting.count(who) == 0);
+  }
+
+  void export_freeze_inc_num_waiters(CDir *dir) {
+    map::iterator it = export_state.find(dir);
+    assert(it != export_state.end());
+    it->second.num_remote_waiters++;
+  }
+  void find_stale_export_freeze();
+
+  // -- misc --
+  void handle_mds_failure_or_stop(int who);
+
+  void audit();
+
+  // -- import/export --
+  // exporter
+ public:
+  void dispatch_export_dir(MDRequestRef& mdr);
+  void export_dir(CDir *dir, int dest);
+  void export_empty_import(CDir *dir);
+
+  void export_dir_nicely(CDir *dir, int dest);
+  void maybe_do_queued_export();
+  void clear_export_queue() {
+    export_queue.clear();
+  }
+  
+  void get_export_lock_set(CDir *dir, set& locks);
+  void get_export_client_set(CDir *dir, set &client_set);
+  void get_export_client_set(CInode *in, set &client_set);
+
+  void encode_export_inode(CInode *in, bufferlist& bl, 
+			   map& exported_client_map);
+  void encode_export_inode_caps(CInode *in, bool auth_cap, bufferlist& bl,
+				map& exported_client_map);
+  void finish_export_inode(CInode *in, utime_t now, int target,
+			   map& peer_imported,
+			   list& finished);
+  void finish_export_inode_caps(CInode *in, int target,
+			        map& peer_imported);
+
+
+  int encode_export_dir(bufferlist& exportbl,
+			CDir *dir,
+			map& exported_client_map,
+			utime_t now);
+  void finish_export_dir(CDir *dir, utime_t now, int target,
+			 map >& peer_imported,
+			 list& finished);
+
+  void add_export_finish_waiter(CDir *dir, Context *c) {
+    map::iterator it = export_state.find(dir);
+    assert(it != export_state.end());
+    it->second.waiting_for_finish.push_back(c);
+  }
+  void clear_export_proxy_pins(CDir *dir);
+
+  void export_caps(CInode *in);
+
+ protected:
+  void handle_export_discover_ack(MExportDirDiscoverAck *m);
+  void export_frozen(CDir *dir);
+  void handle_export_prep_ack(MExportDirPrepAck *m);
+  void export_sessions_flushed(CDir *dir, uint64_t tid);
+  void export_go(CDir *dir);
+  void export_go_synced(CDir *dir, uint64_t tid);
+  void export_try_cancel(CDir *dir, bool notify_peer=true);
+  void export_reverse(CDir *dir);
+  void export_notify_abort(CDir *dir, set& bounds);
+  void handle_export_ack(MExportDirAck *m);
+  void export_logged_finish(CDir *dir);
+  void handle_export_notify_ack(MExportDirNotifyAck *m);
+  void export_finish(CDir *dir);
+
+  friend class C_MDC_ExportFreeze;
+  friend class C_MDS_ExportFinishLogged;
+  friend class C_M_ExportGo;
+  friend class C_M_ExportSessionsFlushed;
+
+  // importer
+  void handle_export_discover(MExportDirDiscover *m);
+  void handle_export_cancel(MExportDirCancel *m);
+  void handle_export_prep(MExportDirPrep *m);
+  void handle_export_dir(MExportDir *m);
+
+public:
+  void decode_import_inode(CDentry *dn, bufferlist::iterator& blp, int oldauth, 
+			   LogSegment *ls, uint64_t log_offset,
+			   map >& cap_imports,
+			   list& updated_scatterlocks);
+  void decode_import_inode_caps(CInode *in, bool auth_cap, bufferlist::iterator &blp,
+				map >& cap_imports);
+  void finish_import_inode_caps(CInode *in, int from, bool auth_cap,
+				map &export_map,
+				map &import_map);
+  int decode_import_dir(bufferlist::iterator& blp,
+			int oldauth,
+			CDir *import_root,
+			EImportStart *le, 
+			LogSegment *ls,
+			map >& cap_imports,
+			list& updated_scatterlocks, utime_t now);
+
+public:
+  void import_reverse(CDir *dir);
+protected:
+  void import_reverse_discovering(dirfrag_t df);
+  void import_reverse_discovered(dirfrag_t df, CInode *diri);
+  void import_reverse_prepping(CDir *dir);
+  void import_remove_pins(CDir *dir, set& bounds);
+  void import_reverse_unfreeze(CDir *dir);
+  void import_reverse_final(CDir *dir);
+  void import_notify_abort(CDir *dir, set& bounds);
+  void import_notify_finish(CDir *dir, set& bounds);
+  void import_logged_start(dirfrag_t df, CDir *dir, int from,
+			   map &imported_client_map,
+			   map& sseqmap);
+  void handle_export_finish(MExportDirFinish *m);
+public:
+  void import_finish(CDir *dir, bool notify, bool last=true);
+protected:
+
+  void handle_export_caps(MExportCaps *m);
+  void logged_import_caps(CInode *in, 
+			  int from,
+			  map >& cap_imports,
+			  map& client_map,
+			  map& sseqmap);
+
+
+  friend class C_MDS_ImportDirLoggedStart;
+  friend class C_MDS_ImportDirLoggedFinish;
+  friend class C_M_LoggedImportCaps;
+
+  // bystander
+  void handle_export_notify(MExportDirNotify *m);
+
+};
+
+
+#endif
diff --git a/ceph/src/mds/Mutation.cc b/ceph/src/mds/Mutation.cc
new file mode 100644
index 00000000..ffeb066b
--- /dev/null
+++ b/ceph/src/mds/Mutation.cc
@@ -0,0 +1,290 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "Mutation.h"
+#include "ScatterLock.h"
+#include "CDir.h"
+
+#include "messages/MClientRequest.h"
+#include "messages/MMDSSlaveRequest.h"
+
+
+// MutationImpl
+
+void MutationImpl::pin(MDSCacheObject *o)
+{
+  if (pins.count(o) == 0) {
+    o->get(MDSCacheObject::PIN_REQUEST);
+    pins.insert(o);
+  }      
+}
+
+void MutationImpl::unpin(MDSCacheObject *o)
+{
+  assert(pins.count(o));
+  o->put(MDSCacheObject::PIN_REQUEST);
+  pins.erase(o);
+}
+
+void MutationImpl::set_stickydirs(CInode *in)
+{
+  if (stickydirs.count(in) == 0) {
+    in->get_stickydirs();
+    stickydirs.insert(in);
+  }
+}
+
+void MutationImpl::drop_pins()
+{
+  for (set::iterator it = pins.begin();
+       it != pins.end();
+       ++it) 
+    (*it)->put(MDSCacheObject::PIN_REQUEST);
+  pins.clear();
+}
+
+void MutationImpl::start_locking(SimpleLock *lock, int target)
+{
+  assert(locking == NULL);
+  pin(lock->get_parent());
+  locking = lock;
+  locking_target_mds = target;
+}
+
+void MutationImpl::finish_locking(SimpleLock *lock)
+{
+  assert(locking == lock);
+  locking = NULL;
+  locking_target_mds = -1;
+}
+
+
+// auth pins
+bool MutationImpl::is_auth_pinned(MDSCacheObject *object)
+{ 
+  return auth_pins.count(object) || remote_auth_pins.count(object); 
+}
+
+void MutationImpl::auth_pin(MDSCacheObject *object)
+{
+  if (!is_auth_pinned(object)) {
+    object->auth_pin(this);
+    auth_pins.insert(object);
+  }
+}
+
+void MutationImpl::auth_unpin(MDSCacheObject *object)
+{
+  assert(auth_pins.count(object));
+  object->auth_unpin(this);
+  auth_pins.erase(object);
+}
+
+void MutationImpl::drop_local_auth_pins()
+{
+  for (set::iterator it = auth_pins.begin();
+       it != auth_pins.end();
+       ++it) {
+    assert((*it)->is_auth());
+    (*it)->auth_unpin(this);
+  }
+  auth_pins.clear();
+}
+
+void MutationImpl::add_projected_inode(CInode *in)
+{
+  projected_inodes.push_back(in);
+}
+
+void MutationImpl::pop_and_dirty_projected_inodes()
+{
+  while (!projected_inodes.empty()) {
+    CInode *in = projected_inodes.front();
+    projected_inodes.pop_front();
+    in->pop_and_dirty_projected_inode(ls);
+  }
+}
+
+void MutationImpl::add_projected_fnode(CDir *dir)
+{
+  projected_fnodes.push_back(dir);
+}
+
+void MutationImpl::pop_and_dirty_projected_fnodes()
+{
+  while (!projected_fnodes.empty()) {
+    CDir *dir = projected_fnodes.front();
+    projected_fnodes.pop_front();
+    dir->pop_and_dirty_projected_fnode(ls);
+  }
+}
+
+void MutationImpl::add_updated_lock(ScatterLock *lock)
+{
+  updated_locks.push_back(lock);
+}
+
+void MutationImpl::add_cow_inode(CInode *in)
+{
+  pin(in);
+  dirty_cow_inodes.push_back(in);
+}
+
+void MutationImpl::add_cow_dentry(CDentry *dn)
+{
+  pin(dn);
+  dirty_cow_dentries.push_back(pair(dn, dn->get_projected_version()));
+}
+
+void MutationImpl::apply()
+{
+  pop_and_dirty_projected_inodes();
+  pop_and_dirty_projected_fnodes();
+  
+  for (list::iterator p = dirty_cow_inodes.begin();
+       p != dirty_cow_inodes.end();
+       ++p) 
+    (*p)->_mark_dirty(ls);
+  for (list >::iterator p = dirty_cow_dentries.begin();
+       p != dirty_cow_dentries.end();
+       ++p)
+    p->first->mark_dirty(p->second, ls);
+  
+  for (list::iterator p = updated_locks.begin();
+       p != updated_locks.end();
+       ++p)
+    (*p)->mark_dirty();
+}
+
+void MutationImpl::cleanup()
+{
+  drop_local_auth_pins();
+  drop_pins();
+}
+
+
+// MDRequestImpl
+
+MDRequestImpl::~MDRequestImpl()
+{
+  if (client_request)
+    client_request->put();
+  if (slave_request)
+    slave_request->put();
+  delete _more;
+}
+
+MDRequestImpl::More* MDRequestImpl::more()
+{ 
+  if (!_more)
+    _more = new More();
+  return _more;
+}
+
+bool MDRequestImpl::has_more()
+{
+  return _more;
+}
+
+bool MDRequestImpl::are_slaves()
+{
+  return _more && !_more->slaves.empty();
+}
+
+bool MDRequestImpl::slave_did_prepare()
+{
+  return more()->slave_commit;
+}
+
+bool MDRequestImpl::did_ino_allocation()
+{
+  return alloc_ino || used_prealloc_ino || prealloc_inos.size();
+}      
+
+bool MDRequestImpl::freeze_auth_pin(CInode *inode)
+{
+  assert(!more()->rename_inode || more()->rename_inode == inode);
+  more()->rename_inode = inode;
+  more()->is_freeze_authpin = true;
+  auth_pin(inode);
+  if (!inode->freeze_inode(1)) {
+    return false;
+  }
+  inode->freeze_auth_pin();
+  inode->unfreeze_inode();
+  return true;
+}
+
+void MDRequestImpl::unfreeze_auth_pin(bool clear_inode)
+{
+  assert(more()->is_freeze_authpin);
+  CInode *inode = more()->rename_inode;
+  if (inode->is_frozen_auth_pin())
+    inode->unfreeze_auth_pin();
+  else
+    inode->unfreeze_inode();
+  more()->is_freeze_authpin = false;
+  if (clear_inode)
+    more()->rename_inode = NULL;
+}
+
+void MDRequestImpl::set_remote_frozen_auth_pin(CInode *inode)
+{
+  more()->rename_inode = inode;
+  more()->is_remote_frozen_authpin = true;
+}
+
+void MDRequestImpl::set_ambiguous_auth(CInode *inode)
+{
+  assert(!more()->rename_inode || more()->rename_inode == inode);
+  assert(!more()->is_ambiguous_auth);
+
+  inode->set_ambiguous_auth();
+  more()->rename_inode = inode;
+  more()->is_ambiguous_auth = true;
+}
+
+void MDRequestImpl::clear_ambiguous_auth()
+{
+  CInode *inode = more()->rename_inode;
+  assert(inode && more()->is_ambiguous_auth);
+  inode->clear_ambiguous_auth();
+  more()->is_ambiguous_auth = false;
+}
+
+bool MDRequestImpl::can_auth_pin(MDSCacheObject *object)
+{
+  return object->can_auth_pin() ||
+         (is_auth_pinned(object) && has_more() &&
+	  more()->is_freeze_authpin &&
+	  more()->rename_inode == object);
+}
+
+void MDRequestImpl::drop_local_auth_pins()
+{
+  if (has_more() && more()->is_freeze_authpin)
+    unfreeze_auth_pin(true);
+  MutationImpl::drop_local_auth_pins();
+}
+
+void MDRequestImpl::print(ostream &out)
+{
+  out << "request(" << reqid;
+  //if (request) out << " " << *request;
+  if (is_slave()) out << " slave_to mds." << slave_to_mds;
+  if (client_request) out << " cr=" << client_request;
+  if (slave_request) out << " sr=" << slave_request;
+  out << ")";
+}
+
diff --git a/ceph/src/mds/Mutation.h b/ceph/src/mds/Mutation.h
new file mode 100644
index 00000000..dbfbe759
--- /dev/null
+++ b/ceph/src/mds/Mutation.h
@@ -0,0 +1,337 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_MUTATION_H
+#define CEPH_MDS_MUTATION_H
+
+#include "include/interval_set.h"
+#include "include/elist.h"
+
+#include "mdstypes.h"
+
+#include "SimpleLock.h"
+#include "Capability.h"
+
+class LogSegment;
+class Capability;
+class CInode;
+class CDir;
+class CDentry;
+class Session;
+class ScatterLock;
+class MClientRequest;
+class MMDSSlaveRequest;
+
+struct MutationImpl {
+  metareqid_t reqid;
+  __u32 attempt;      // which attempt for this request
+  LogSegment *ls;  // the log segment i'm committing to
+  utime_t now;
+
+  // flag mutation as slave
+  int slave_to_mds;                // this is a slave request if >= 0.
+
+  // -- my pins and locks --
+  // cache pins (so things don't expire)
+  set< MDSCacheObject* > pins;
+  set stickydirs;
+
+  // auth pins
+  set< MDSCacheObject* > remote_auth_pins;
+  set< MDSCacheObject* > auth_pins;
+  
+  // held locks
+  set< SimpleLock* > rdlocks;  // always local.
+  set< SimpleLock* > wrlocks;  // always local.
+  map< SimpleLock*, int > remote_wrlocks;
+  set< SimpleLock* > xlocks;   // local or remote.
+  set< SimpleLock*, SimpleLock::ptr_lt > locks;  // full ordering
+
+  // lock we are currently trying to acquire.  if we give up for some reason,
+  // be sure to eval() this.
+  SimpleLock *locking;
+  int locking_target_mds;
+
+  // if this flag is set, do not attempt to acquire further locks.
+  //  (useful for wrlock, which may be a moving auth target)
+  bool done_locking; 
+  bool committing;
+  bool aborted;
+  bool killed;
+
+  // for applying projected inode changes
+  list projected_inodes;
+  list projected_fnodes;
+  list updated_locks;
+
+  list dirty_cow_inodes;
+  list > dirty_cow_dentries;
+
+  MutationImpl()
+    : attempt(0),
+      ls(0),
+      slave_to_mds(-1),
+      locking(NULL),
+      locking_target_mds(-1),
+      done_locking(false), committing(false), aborted(false), killed(false) { }
+  MutationImpl(metareqid_t ri, __u32 att=0, int slave_to=-1)
+    : reqid(ri), attempt(att),
+      ls(0),
+      slave_to_mds(slave_to), 
+      locking(NULL),
+      locking_target_mds(-1),
+      done_locking(false), committing(false), aborted(false), killed(false) { }
+  virtual ~MutationImpl() {
+    assert(locking == NULL);
+    assert(pins.empty());
+    assert(auth_pins.empty());
+    assert(xlocks.empty());
+    assert(rdlocks.empty());
+    assert(wrlocks.empty());
+    assert(remote_wrlocks.empty());
+  }
+
+  bool is_master() { return slave_to_mds < 0; }
+  bool is_slave() { return slave_to_mds >= 0; }
+
+  client_t get_client() {
+    if (reqid.name.is_client())
+      return client_t(reqid.name.num());
+    return -1;
+  }
+
+  // pin items in cache
+  void pin(MDSCacheObject *o);
+  void unpin(MDSCacheObject *o);
+  void set_stickydirs(CInode *in);
+  void drop_pins();
+
+  void start_locking(SimpleLock *lock, int target=-1);
+  void finish_locking(SimpleLock *lock);
+
+  // auth pins
+  bool is_auth_pinned(MDSCacheObject *object);
+  void auth_pin(MDSCacheObject *object);
+  void auth_unpin(MDSCacheObject *object);
+  void drop_local_auth_pins();
+  void add_projected_inode(CInode *in);
+  void pop_and_dirty_projected_inodes();
+  void add_projected_fnode(CDir *dir);
+  void pop_and_dirty_projected_fnodes();
+  void add_updated_lock(ScatterLock *lock);
+  void add_cow_inode(CInode *in);
+  void add_cow_dentry(CDentry *dn);
+  void apply();
+  void cleanup();
+
+  virtual void print(ostream &out) {
+    out << "mutation(" << this << ")";
+  }
+};
+
+inline ostream& operator<<(ostream& out, MutationImpl &mut)
+{
+  mut.print(out);
+  return out;
+}
+
+typedef ceph::shared_ptr MutationRef;
+
+
+
+/** active_request_t
+ * state we track for requests we are currently processing.
+ * mostly information about locks held, so that we can drop them all
+ * the request is finished or forwarded.  see request_*().
+ */
+struct MDRequestImpl : public MutationImpl {
+  Session *session;
+  elist::item item_session_request;  // if not on list, op is aborted.
+
+  // -- i am a client (master) request
+  MClientRequest *client_request; // client request (if any)
+
+  // store up to two sets of dn vectors, inode pointers, for request path1 and path2.
+  vector dn[2];
+  CDentry *straydn;
+  CInode *in[2];
+  snapid_t snapid;
+
+  CInode *tracei;
+  CDentry *tracedn;
+
+  inodeno_t alloc_ino, used_prealloc_ino;  
+  interval_set prealloc_inos;
+
+  int snap_caps;
+  bool did_early_reply;
+  bool o_trunc;           ///< request is an O_TRUNC mutation
+  int getattr_caps;       ///< caps requested by getattr
+
+  bufferlist reply_extra_bl;
+
+  // inos we did a embedded cap release on, and may need to eval if we haven't since reissued
+  map cap_releases;  
+
+  // -- i am a slave request
+  MMDSSlaveRequest *slave_request; // slave request (if one is pending; implies slave == true)
+
+  // -- i am an internal op
+  int internal_op;
+
+  // indicates how may retries of request have been made
+  int retry;
+
+  // indicator for vxattr osdmap update
+  bool waited_for_osdmap;
+
+  // break rarely-used fields into a separately allocated structure 
+  // to save memory for most ops
+  struct More {
+    set slaves;           // mds nodes that have slave requests to me (implies client_request)
+    set waiting_on_slave; // peers i'm waiting for slavereq replies from. 
+
+    // for rename/link/unlink
+    set witnessed;       // nodes who have journaled a RenamePrepare
+    map pvmap;
+    
+    // for rename
+    set extra_witnesses; // replica list from srcdn auth (rename)
+    int srcdn_auth_mds;
+    version_t src_reanchor_atid;  // src->dst
+    version_t dst_reanchor_atid;  // dst->stray
+    bufferlist inode_import;
+    version_t inode_import_v;
+    CInode* rename_inode;
+    bool is_freeze_authpin;
+    bool is_ambiguous_auth;
+    bool is_remote_frozen_authpin;
+    bool is_inode_exporter;
+
+    map imported_client_map;
+    map sseq_map;
+    map > cap_imports;
+    
+    // for lock/flock
+    bool flock_was_waiting;
+
+    // for snaps
+    version_t stid;
+    bufferlist snapidbl;
+
+    // called when slave commits or aborts
+    Context *slave_commit;
+    bufferlist rollback_bl;
+
+    list waiting_for_finish;
+
+    // export & fragment
+    CDir* export_dir;
+    dirfrag_t fragment_base;
+
+    More() : 
+      srcdn_auth_mds(-1),
+      src_reanchor_atid(0), dst_reanchor_atid(0), inode_import_v(0),
+      rename_inode(0), is_freeze_authpin(false), is_ambiguous_auth(false),
+      is_remote_frozen_authpin(false), is_inode_exporter(false),
+      flock_was_waiting(false), stid(0), slave_commit(0), export_dir(NULL)  { }
+  } *_more;
+
+
+  // ---------------------------------------------------
+  MDRequestImpl() :
+    session(0), item_session_request(this),
+    client_request(0), straydn(NULL), snapid(CEPH_NOSNAP), tracei(0), tracedn(0),
+    alloc_ino(0), used_prealloc_ino(0), snap_caps(0), did_early_reply(false),
+    o_trunc(false),
+    getattr_caps(0),
+    slave_request(0),
+    internal_op(-1),
+    retry(0),
+    waited_for_osdmap(false),
+    _more(0) {
+    in[0] = in[1] = 0; 
+  }
+  MDRequestImpl(metareqid_t ri, __u32 attempt, MClientRequest *req) :
+    MutationImpl(ri, attempt),
+    session(0), item_session_request(this),
+    client_request(req), straydn(NULL), snapid(CEPH_NOSNAP), tracei(0), tracedn(0),
+    alloc_ino(0), used_prealloc_ino(0), snap_caps(0), did_early_reply(false),
+    o_trunc(false),
+    getattr_caps(0),
+    slave_request(0),
+    internal_op(-1),
+    retry(0),
+    waited_for_osdmap(false),
+    _more(0) {
+    in[0] = in[1] = 0; 
+  }
+  MDRequestImpl(metareqid_t ri, __u32 attempt, int by) :
+    MutationImpl(ri, attempt, by),
+    session(0), item_session_request(this),
+    client_request(0), straydn(NULL), snapid(CEPH_NOSNAP), tracei(0), tracedn(0),
+    alloc_ino(0), used_prealloc_ino(0), snap_caps(0), did_early_reply(false),
+    o_trunc(false),
+    getattr_caps(0),
+    slave_request(0),
+    internal_op(-1),
+    retry(0),
+    waited_for_osdmap(false),
+    _more(0) {
+    in[0] = in[1] = 0; 
+  }
+  ~MDRequestImpl();
+  
+  More* more();
+  bool has_more();
+  bool are_slaves();
+  bool slave_did_prepare();
+  bool did_ino_allocation();
+  bool freeze_auth_pin(CInode *inode);
+  void unfreeze_auth_pin(bool clear_inode=false);
+  void set_remote_frozen_auth_pin(CInode *inode);
+  bool can_auth_pin(MDSCacheObject *object);
+  void drop_local_auth_pins();
+  void set_ambiguous_auth(CInode *inode);
+  void clear_ambiguous_auth();
+
+  void print(ostream &out);
+};
+
+typedef ceph::shared_ptr MDRequestRef;
+
+
+struct MDSlaveUpdate {
+  int origop;
+  bufferlist rollback;
+  elist::item item;
+  Context *waiter;
+  set olddirs;
+  set unlinked;
+  MDSlaveUpdate(int oo, bufferlist &rbl, elist &list) :
+    origop(oo),
+    item(this),
+    waiter(0) {
+    rollback.claim(rbl);
+    list.push_back(&item);
+  }
+  ~MDSlaveUpdate() {
+    item.remove_myself();
+    if (waiter)
+      waiter->complete(0);
+  }
+};
+
+
+#endif
diff --git a/ceph/src/mds/Resetter.cc b/ceph/src/mds/Resetter.cc
new file mode 100644
index 00000000..22839ecc
--- /dev/null
+++ b/ceph/src/mds/Resetter.cc
@@ -0,0 +1,114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Greg Farnum 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "mds/Resetter.h"
+#include "osdc/Journaler.h"
+#include "mds/mdstypes.h"
+#include "mon/MonClient.h"
+#include "mds/events/EResetJournal.h"
+
+
+int Resetter::init(int rank)
+{
+  int r = MDSUtility::init();
+  if (r < 0) {
+    return r;
+  }
+
+  inodeno_t ino = MDS_INO_LOG_OFFSET + rank;
+  journaler = new Journaler(ino,
+      mdsmap->get_metadata_pool(),
+      CEPH_FS_ONDISK_MAGIC,
+      objecter, 0, 0, &timer);
+
+  return 0;
+}
+
+void Resetter::reset()
+{
+  Mutex mylock("Resetter::reset::lock");
+  Cond cond;
+  bool done;
+  int r;
+
+  lock.Lock();
+  journaler->recover(new C_SafeCond(&mylock, &cond, &done, &r));
+  lock.Unlock();
+
+  mylock.Lock();
+  while (!done)
+    cond.Wait(mylock);
+  mylock.Unlock();
+
+  if (r != 0) {
+    if (r == -ENOENT) {
+      cerr << "journal does not exist on-disk. Did you set a bad rank?"
+	   << std::endl;
+      shutdown();
+      return;
+    } else {
+      cerr << "got error " << r << "from Journaler, failling" << std::endl;
+      shutdown();
+      return;
+    }
+  }
+
+  lock.Lock();
+  uint64_t old_start = journaler->get_read_pos();
+  uint64_t old_end = journaler->get_write_pos();
+  uint64_t old_len = old_end - old_start;
+  cout << "old journal was " << old_start << "~" << old_len << std::endl;
+
+  uint64_t new_start = ROUND_UP_TO(old_end+1, journaler->get_layout_period());
+  cout << "new journal start will be " << new_start
+       << " (" << (new_start - old_end) << " bytes past old end)" << std::endl;
+
+  journaler->set_read_pos(new_start);
+  journaler->set_write_pos(new_start);
+  journaler->set_expire_pos(new_start);
+  journaler->set_trimmed_pos(new_start);
+  journaler->set_writeable();
+
+  cout << "writing journal head" << std::endl;
+  journaler->write_head(new C_SafeCond(&mylock, &cond, &done, &r));
+  lock.Unlock();
+
+  mylock.Lock();
+  while (!done)
+    cond.Wait(mylock);
+  mylock.Unlock();
+    
+  lock.Lock();
+  assert(r == 0);
+
+  LogEvent *le = new EResetJournal;
+
+  bufferlist bl;
+  le->encode_with_header(bl);
+  
+  cout << "writing EResetJournal entry" << std::endl;
+  journaler->append_entry(bl);
+  journaler->flush(new C_SafeCond(&mylock, &cond, &done,&r));
+
+  lock.Unlock();
+
+  mylock.Lock();
+  while (!done)
+    cond.Wait(mylock);
+  mylock.Unlock();
+
+  assert(r == 0);
+
+  cout << "done" << std::endl;
+}
diff --git a/ceph/src/mds/Resetter.h b/ceph/src/mds/Resetter.h
new file mode 100644
index 00000000..e967b246
--- /dev/null
+++ b/ceph/src/mds/Resetter.h
@@ -0,0 +1,37 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Greg Farnum 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef JOURNAL_RESETTER_H_
+#define JOURNAL_RESETTER_H_
+
+
+#include "osdc/Journaler.h"
+#include "mds/MDSUtility.h"
+
+/**
+ * This class lets you reset an mds journal for troubleshooting or whatever.
+ *
+ * To use, create a Resetter, call init(), and then call reset() with the name
+ * of the file to dump to.
+ */
+class Resetter : public MDSUtility {
+public:
+  Journaler *journaler;
+
+  Resetter() : journaler(NULL) {}
+
+  int init(int rank);
+  void reset();
+};
+
+#endif /* JOURNAL_RESETTER_H_ */
diff --git a/ceph/src/mds/ScatterLock.h b/ceph/src/mds/ScatterLock.h
new file mode 100644
index 00000000..a85caa82
--- /dev/null
+++ b/ceph/src/mds/ScatterLock.h
@@ -0,0 +1,251 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_SCATTERLOCK_H
+#define CEPH_SCATTERLOCK_H
+
+#include "SimpleLock.h"
+
+class ScatterLock : public SimpleLock {
+
+  struct more_bits_t {
+    int state_flags;
+    utime_t last_scatter;
+    xlist::item item_updated;
+    utime_t update_stamp;
+
+    more_bits_t(ScatterLock *lock) :
+      state_flags(0),
+      item_updated(lock)
+    {}
+
+    bool empty() const {
+      return
+	!state_flags &&
+	!item_updated.is_on_list();
+    }
+  };
+  more_bits_t *_more;
+
+  bool have_more() const { return _more ? true : false; }
+  void try_clear_more() {
+    if (_more && _more->empty()) {
+      delete _more;
+      _more = NULL;
+    }
+  }
+  more_bits_t *more() {
+    if (!_more)
+      _more = new more_bits_t(this);
+    return _more;
+  }
+
+  enum flag_values {  // flag values for more_bits_t state
+    SCATTER_WANTED   = 1 << 0,
+    UNSCATTER_WANTED = 1 << 1,
+    DIRTY            = 1 << 2,
+    FLUSHING         = 1 << 3,
+    FLUSHED          = 1 << 4,
+  };
+
+public:
+  ScatterLock(MDSCacheObject *o, LockType *lt) : 
+    SimpleLock(o, lt), _more(NULL)
+  {}
+  ~ScatterLock() {
+    if (_more) {
+      _more->item_updated.remove_myself();   // FIXME this should happen sooner, i think...
+      delete _more;
+    }
+  }
+
+  bool is_scatterlock() const {
+    return true;
+  }
+
+  bool is_sync_and_unlocked() const {
+    return
+      SimpleLock::is_sync_and_unlocked() && 
+      !is_dirty() &&
+      !is_flushing();
+  }
+
+  bool can_scatter_pin(client_t loner) {
+    /*
+      LOCK : NOT okay because it can MIX and force replicas to journal something
+      TSYN : also not okay for same reason
+      EXCL : also not okay
+
+      MIX  : okay, replica can stall before sending AC_SYNCACK
+      SYNC : okay, replica can stall before sending AC_MIXACK or AC_LOCKACK
+    */   
+    return
+      get_state() == LOCK_SYNC ||
+      get_state() == LOCK_MIX;
+  }
+
+  xlist::item *get_updated_item() { return &more()->item_updated; }
+
+  utime_t get_update_stamp() {
+    return more()->update_stamp;
+  }
+
+  void set_update_stamp(utime_t t) { more()->update_stamp = t; }
+
+  void set_scatter_wanted() {
+    more()->state_flags |= SCATTER_WANTED;
+  }
+  void set_unscatter_wanted() {
+    more()->state_flags |= UNSCATTER_WANTED;
+  }
+  void clear_scatter_wanted() {
+    if (have_more())
+      _more->state_flags &= ~SCATTER_WANTED;
+    try_clear_more();
+  }
+  void clear_unscatter_wanted() {
+    if (have_more())
+      _more->state_flags &= ~UNSCATTER_WANTED;
+    try_clear_more();
+  }
+  bool get_scatter_wanted() const {
+    return have_more() ? _more->state_flags & SCATTER_WANTED : false;
+  }
+  bool get_unscatter_wanted() const {
+    return have_more() ? _more->state_flags & UNSCATTER_WANTED : false;
+  }
+
+  bool is_dirty() const {
+    return have_more() ? _more->state_flags & DIRTY : false;
+  }
+  bool is_flushing() const {
+    return have_more() ? _more->state_flags & FLUSHING: false;
+  }
+  bool is_flushed() const {
+    return have_more() ? _more->state_flags & FLUSHED: false;
+  }
+  bool is_dirty_or_flushing() const {
+    return have_more() ? (is_dirty() || is_flushing()) : false;
+  }
+
+  void mark_dirty() { 
+    if (!is_dirty()) {
+      if (!is_flushing())
+	parent->get(MDSCacheObject::PIN_DIRTYSCATTERED);
+      set_dirty();
+    }
+  }
+  void start_flush() {
+    if (is_dirty()) {
+      set_flushing();
+      clear_dirty();
+    }
+  }
+  void finish_flush() {
+    if (is_flushing()) {
+      clear_flushing();
+      set_flushed();
+      if (!is_dirty()) {
+	parent->put(MDSCacheObject::PIN_DIRTYSCATTERED);
+	parent->clear_dirty_scattered(get_type());
+      }
+    }
+  }
+  void remove_dirty() {
+    start_flush();
+    finish_flush();
+  }
+  void clear_flushed() {
+    if (have_more()) {
+      _more->state_flags &= ~FLUSHED;
+      try_clear_more();
+    }
+  }
+
+  void set_last_scatter(utime_t t) { more()->last_scatter = t; }
+  utime_t get_last_scatter() {
+    return more()->last_scatter;
+  }
+
+  void infer_state_from_strong_rejoin(int rstate, bool locktoo) {
+    if (rstate == LOCK_MIX || 
+	rstate == LOCK_MIX_LOCK || // replica still has wrlocks?
+	rstate == LOCK_MIX_SYNC || // "
+	rstate == LOCK_MIX_TSYN)  // "
+      state = LOCK_MIX;
+    else if (locktoo && rstate == LOCK_LOCK)
+      state = LOCK_LOCK;
+  }
+
+  void encode_state_for_rejoin(bufferlist& bl, int rep) const {
+    __s16 s = get_replica_state();
+    if (is_gathering(rep)) {
+      // the recovering mds may hold rejoined wrlocks
+      if (state == LOCK_MIX_SYNC)
+	s = LOCK_MIX_SYNC;
+      else
+	s = LOCK_MIX_LOCK;
+    }
+    ::encode(s, bl);
+  }
+
+  bool remove_replica(int from, bool rejoin) {
+    if (rejoin &&
+	(state == LOCK_MIX ||
+	 state == LOCK_MIX_SYNC ||
+	 state == LOCK_MIX_LOCK2 ||
+	 state == LOCK_MIX_TSYN ||
+	 state == LOCK_MIX_EXCL))
+      return false;
+    return SimpleLock::remove_replica(from);
+  }
+
+  virtual void print(ostream& out) const {
+    out << "(";
+    _print(out);
+    if (is_dirty())
+      out << " dirty";
+    if (is_flushing())
+      out << " flushing";
+    if (is_flushed())
+      out << " flushed";
+    if (get_scatter_wanted())
+      out << " scatter_wanted";
+    out << ")";
+  }
+
+private:
+  void set_flushing() {
+    more()->state_flags |= FLUSHING;
+  }
+  void clear_flushing() {
+    if (have_more()) {
+      _more->state_flags &= ~FLUSHING;
+    }
+  }
+  void set_flushed() {
+    more()->state_flags |= FLUSHED;
+  }
+  void set_dirty() {
+    more()->state_flags |= DIRTY;
+  }
+  void clear_dirty() {
+    if (have_more()) {
+      _more->state_flags &= ~DIRTY;
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/mds/Server.cc b/ceph/src/mds/Server.cc
new file mode 100644
index 00000000..64004b22
--- /dev/null
+++ b/ceph/src/mds/Server.cc
@@ -0,0 +1,7607 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include "include/assert.h"  // lexical_cast includes system assert.h
+
+#include 
+#include 
+#include 
+
+#include "MDS.h"
+#include "Server.h"
+#include "Locker.h"
+#include "MDCache.h"
+#include "MDLog.h"
+#include "Migrator.h"
+#include "MDBalancer.h"
+#include "AnchorClient.h"
+#include "InoTable.h"
+#include "SnapClient.h"
+#include "Mutation.h"
+
+#include "msg/Messenger.h"
+
+#include "messages/MClientSession.h"
+#include "messages/MClientRequest.h"
+#include "messages/MClientReply.h"
+#include "messages/MClientReconnect.h"
+#include "messages/MClientCaps.h"
+#include "messages/MClientSnap.h"
+
+#include "messages/MMDSSlaveRequest.h"
+
+#include "messages/MLock.h"
+
+#include "messages/MDentryUnlink.h"
+
+#include "events/EUpdate.h"
+#include "events/ESlaveUpdate.h"
+#include "events/ESession.h"
+#include "events/EOpen.h"
+#include "events/ECommitted.h"
+
+#include "include/filepath.h"
+#include "common/errno.h"
+#include "common/Timer.h"
+#include "common/perf_counters.h"
+#include "include/compat.h"
+#include "osd/OSDMap.h"
+
+#include 
+#include 
+
+#include 
+#include 
+using namespace std;
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".server "
+
+void Server::create_logger()
+{
+  PerfCountersBuilder plb(g_ceph_context, "mds_server", l_mdss_first, l_mdss_last);
+  plb.add_u64_counter(l_mdss_hcreq,"hcreq"); // handle client req
+  plb.add_u64_counter(l_mdss_hsreq, "hsreq"); // slave
+  plb.add_u64_counter(l_mdss_hcsess, "hcsess");    // client session
+  plb.add_u64_counter(l_mdss_dcreq, "dcreq"); // dispatch client req
+  plb.add_u64_counter(l_mdss_dsreq, "dsreq"); // slave
+  logger = plb.create_perf_counters();
+  g_ceph_context->get_perfcounters_collection()->add(logger);
+}
+
+
+/* This function DOES put the passed message before returning*/
+void Server::dispatch(Message *m) 
+{
+  switch (m->get_type()) {
+  case CEPH_MSG_CLIENT_RECONNECT:
+    handle_client_reconnect(static_cast(m));
+    return;
+  }
+
+  // active?
+  if (!mds->is_active() && 
+      !(mds->is_stopping() && m->get_source().is_mds())) {
+    if ((mds->is_reconnect() || mds->get_want_state() == CEPH_MDS_STATE_RECONNECT) &&
+	m->get_type() == CEPH_MSG_CLIENT_REQUEST &&
+	(static_cast(m))->is_replay()) {
+      dout(3) << "queuing replayed op" << dendl;
+      mds->enqueue_replay(new C_MDS_RetryMessage(mds, m));
+      return;
+    } else if (mds->is_clientreplay() &&
+	       // session open requests need to be handled during replay,
+	       // close requests need to be delayed
+	       ((m->get_type() == CEPH_MSG_CLIENT_SESSION &&
+		 (static_cast(m))->get_op() != CEPH_SESSION_REQUEST_CLOSE) ||
+		(m->get_type() == CEPH_MSG_CLIENT_REQUEST &&
+		 (static_cast(m))->is_replay()))) {
+      // replaying!
+    } else if (m->get_type() == MSG_MDS_SLAVE_REQUEST) {
+      // handle_slave_request() will wait if necessary
+    } else {
+      dout(3) << "not active yet, waiting" << dendl;
+      mds->wait_for_active(new C_MDS_RetryMessage(mds, m));
+      return;
+    }
+  }
+
+  switch (m->get_type()) {
+  case CEPH_MSG_CLIENT_SESSION:
+    handle_client_session(static_cast(m));
+    return;
+  case CEPH_MSG_CLIENT_REQUEST:
+    handle_client_request(static_cast(m));
+    return;
+  case MSG_MDS_SLAVE_REQUEST:
+    handle_slave_request(static_cast(m));
+    return;
+  }
+
+  dout(1) << "server unknown message " << m->get_type() << dendl;
+  assert(0);
+}
+
+
+
+// ----------------------------------------------------------
+// SESSION management
+
+class C_MDS_session_finish : public Context {
+  MDS *mds;
+  Session *session;
+  uint64_t state_seq;
+  bool open;
+  version_t cmapv;
+  interval_set inos;
+  version_t inotablev;
+public:
+  C_MDS_session_finish(MDS *m, Session *se, uint64_t sseq, bool s, version_t mv) :
+    mds(m), session(se), state_seq(sseq), open(s), cmapv(mv), inotablev(0) { }
+  C_MDS_session_finish(MDS *m, Session *se, uint64_t sseq, bool s, version_t mv, interval_set& i, version_t iv) :
+    mds(m), session(se), state_seq(sseq), open(s), cmapv(mv), inos(i), inotablev(iv) { }
+  void finish(int r) {
+    assert(r == 0);
+    mds->server->_session_logged(session, state_seq, open, cmapv, inos, inotablev);
+  }
+};
+
+Session *Server::get_session(Message *m)
+{
+  Session *session = static_cast(m->get_connection()->get_priv());
+  if (session) {
+    dout(20) << "get_session have " << session << " " << session->info.inst
+	     << " state " << session->get_state_name() << dendl;
+    session->put();  // not carry ref
+  } else {
+    dout(20) << "get_session dne for " << m->get_source_inst() << dendl;
+  }
+  return session;
+}
+
+/* This function DOES put the passed message before returning*/
+void Server::handle_client_session(MClientSession *m)
+{
+  version_t pv;
+  Session *session = get_session(m);
+
+  dout(3) << "handle_client_session " << *m << " from " << m->get_source() << dendl;
+  assert(m->get_source().is_client()); // should _not_ come from an mds!
+
+  if (!session) {
+    dout(0) << " ignoring sessionless msg " << *m << dendl;
+    m->put();
+    return;
+  }
+
+  uint64_t sseq = 0;
+  switch (m->get_op()) {
+  case CEPH_SESSION_REQUEST_OPEN:
+    if (session->is_opening() ||
+	session->is_open() ||
+	session->is_stale() ||
+	session->is_killing()) {
+      dout(10) << "currently open|opening|stale|killing, dropping this req" << dendl;
+      m->put();
+      return;
+    }
+    assert(session->is_closed() ||
+	   session->is_closing());
+    sseq = mds->sessionmap.set_state(session, Session::STATE_OPENING);
+    mds->sessionmap.touch_session(session);
+    pv = ++mds->sessionmap.projected;
+    mdlog->start_submit_entry(new ESession(m->get_source_inst(), true, pv),
+			      new C_MDS_session_finish(mds, session, sseq, true, pv));
+    mdlog->flush();
+    break;
+
+  case CEPH_SESSION_REQUEST_RENEWCAPS:
+    if (session->is_open() ||
+	session->is_stale()) {
+      mds->sessionmap.touch_session(session);
+      if (session->is_stale()) {
+	mds->sessionmap.set_state(session, Session::STATE_OPEN);
+	mds->locker->resume_stale_caps(session);
+	mds->sessionmap.touch_session(session);
+      }
+      mds->messenger->send_message(new MClientSession(CEPH_SESSION_RENEWCAPS, m->get_seq()), 
+				   m->get_connection());
+    } else {
+      dout(10) << "ignoring renewcaps on non open|stale session (" << session->get_state_name() << ")" << dendl;
+    }
+    break;
+    
+  case CEPH_SESSION_REQUEST_CLOSE:
+    {
+      if (session->is_closed() || 
+	  session->is_closing() ||
+	  session->is_killing()) {
+	dout(10) << "already closed|closing|killing, dropping this req" << dendl;
+	m->put();
+	return;
+      }
+      if (session->is_importing()) {
+	dout(10) << "ignoring close req on importing session" << dendl;
+	m->put();
+	return;
+      }
+      assert(session->is_open() || 
+	     session->is_stale() || 
+	     session->is_opening());
+      if (m->get_seq() < session->get_push_seq()) {
+	dout(10) << "old push seq " << m->get_seq() << " < " << session->get_push_seq() 
+		 << ", dropping" << dendl;
+	m->put();
+	return;
+      }
+      if (m->get_seq() != session->get_push_seq()) {
+	dout(0) << "old push seq " << m->get_seq() << " != " << session->get_push_seq() 
+		<< ", BUGGY!" << dendl;
+	assert(0);
+      }
+      journal_close_session(session, Session::STATE_CLOSING);
+    }
+    break;
+
+  case CEPH_SESSION_FLUSHMSG_ACK:
+    finish_flush_session(session, m->get_seq());
+    break;
+
+  default:
+    assert(0);
+  }
+  m->put();
+}
+
+void Server::flush_client_sessions(set& client_set, C_GatherBuilder& gather)
+{
+  for (set::iterator p = client_set.begin(); p != client_set.end(); ++p) {
+    Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->v));
+    assert(session);
+    if (!session->is_open() ||
+	!session->connection.get() ||
+	!session->connection->has_feature(CEPH_FEATURE_EXPORT_PEER))
+      continue;
+    version_t seq = session->wait_for_flush(gather.new_sub());
+    mds->send_message_client(new MClientSession(CEPH_SESSION_FLUSHMSG, seq), session);
+  }
+}
+
+void Server::finish_flush_session(Session *session, version_t seq)
+{
+  list finished;
+  session->finish_flush(seq, finished);
+  mds->queue_waiters(finished);
+}
+
+void Server::_session_logged(Session *session, uint64_t state_seq, bool open, version_t pv,
+			     interval_set& inos, version_t piv)
+{
+  dout(10) << "_session_logged " << session->info.inst << " state_seq " << state_seq << " " << (open ? "open":"close")
+	   << " " << pv << dendl;
+
+  if (piv) {
+    mds->inotable->apply_release_ids(inos);
+    assert(mds->inotable->get_version() == piv);
+  }
+
+  // apply
+  if (session->get_state_seq() != state_seq) {
+    dout(10) << " journaled state_seq " << state_seq << " != current " << session->get_state_seq()
+	     << ", noop" << dendl;
+    // close must have been canceled (by an import?), or any number of other things..
+  } else if (open) {
+    assert(session->is_opening());
+    mds->sessionmap.set_state(session, Session::STATE_OPEN);
+    mds->sessionmap.touch_session(session);
+    mds->messenger->send_message(new MClientSession(CEPH_SESSION_OPEN), session->connection);
+  } else if (session->is_closing() ||
+	     session->is_killing()) {
+    // kill any lingering capabilities, leases, requests
+    while (!session->caps.empty()) {
+      Capability *cap = session->caps.front();
+      CInode *in = cap->get_inode();
+      dout(20) << " killing capability " << ccap_string(cap->issued()) << " on " << *in << dendl;
+      mds->locker->remove_client_cap(in, session->info.inst.name.num());
+    }
+    while (!session->leases.empty()) {
+      ClientLease *r = session->leases.front();
+      CDentry *dn = static_cast(r->parent);
+      dout(20) << " killing client lease of " << *dn << dendl;
+      dn->remove_client_lease(r, mds->locker);
+    }
+    
+    if (session->is_closing()) {
+      // mark con disposable.  if there is a fault, we will get a
+      // reset and clean it up.  if the client hasn't received the
+      // CLOSE message yet, they will reconnect and get an
+      // ms_handle_remote_reset() and realize they had in fact closed.
+      // do this *before* sending the message to avoid a possible
+      // race.
+      mds->messenger->mark_disposable(session->connection.get());
+
+      // reset session
+      mds->send_message_client(new MClientSession(CEPH_SESSION_CLOSE), session);
+      mds->sessionmap.set_state(session, Session::STATE_CLOSED);
+      session->clear();
+    } else if (session->is_killing()) {
+      // destroy session, close connection
+      mds->messenger->mark_down(session->connection);
+      mds->sessionmap.remove_session(session);
+    } else {
+      assert(0);
+    }
+  } else {
+    assert(0);
+  }
+  mds->sessionmap.version++;  // noop
+}
+
+version_t Server::prepare_force_open_sessions(map& cm,
+					      map& sseqmap)
+{
+  version_t pv = ++mds->sessionmap.projected;
+  dout(10) << "prepare_force_open_sessions " << pv 
+	   << " on " << cm.size() << " clients"
+	   << dendl;
+  for (map::iterator p = cm.begin(); p != cm.end(); ++p) {
+    Session *session = mds->sessionmap.get_or_add_session(p->second);
+    if (session->is_closed() || 
+	session->is_closing() ||
+	session->is_killing())
+      sseqmap[p->first] = mds->sessionmap.set_state(session, Session::STATE_OPENING);
+    else
+      assert(session->is_open() ||
+	     session->is_opening() ||
+	     session->is_stale());
+    session->inc_importing();
+//  mds->sessionmap.touch_session(session);
+  }
+  return pv;
+}
+
+void Server::finish_force_open_sessions(map& cm,
+					map& sseqmap,
+					bool dec_import)
+{
+  /*
+   * FIXME: need to carefully consider the race conditions between a
+   * client trying to close a session and an MDS doing an import
+   * trying to force open a session...  
+   */
+  dout(10) << "finish_force_open_sessions on " << cm.size() << " clients,"
+	   << " v " << mds->sessionmap.version << " -> " << (mds->sessionmap.version+1) << dendl;
+  for (map::iterator p = cm.begin(); p != cm.end(); ++p) {
+    Session *session = mds->sessionmap.get_session(p->second.name);
+    assert(session);
+    
+    if (sseqmap.count(p->first)) {
+      uint64_t sseq = sseqmap[p->first];
+      if (session->get_state_seq() != sseq) {
+	dout(10) << "force_open_sessions skipping changed " << session->info.inst << dendl;
+      } else {
+	dout(10) << "force_open_sessions opened " << session->info.inst << dendl;
+	mds->sessionmap.set_state(session, Session::STATE_OPEN);
+	mds->sessionmap.touch_session(session);
+	Message *m = new MClientSession(CEPH_SESSION_OPEN);
+	if (session->connection)
+	  messenger->send_message(m, session->connection);
+	else
+	  session->preopen_out_queue.push_back(m);
+      }
+    } else {
+      dout(10) << "force_open_sessions skipping already-open " << session->info.inst << dendl;
+      assert(session->is_open() || session->is_stale());
+    }
+    if (dec_import)
+      session->dec_importing();
+  }
+  mds->sessionmap.version++;
+}
+
+struct C_MDS_TerminatedSessions : public Context {
+  Server *server;
+  C_MDS_TerminatedSessions(Server *s) : server(s) {}
+  void finish(int r) {
+    server->terminating_sessions = false;
+  }
+};
+
+void Server::terminate_sessions()
+{
+  dout(2) << "terminate_sessions" << dendl;
+
+  // kill them off.  clients will retry etc.
+  set sessions;
+  mds->sessionmap.get_client_session_set(sessions);
+  for (set::const_iterator p = sessions.begin();
+       p != sessions.end();
+       ++p) {
+    Session *session = *p;
+    if (session->is_closing() ||
+	session->is_killing() ||
+	session->is_closed())
+      continue;
+    journal_close_session(session, Session::STATE_CLOSING);
+  }
+
+  mdlog->wait_for_safe(new C_MDS_TerminatedSessions(this));
+}
+
+
+void Server::find_idle_sessions()
+{
+  dout(10) << "find_idle_sessions.  laggy until " << mds->laggy_until << dendl;
+  
+  // timeout/stale
+  //  (caps go stale, lease die)
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t cutoff = now;
+  cutoff -= g_conf->mds_session_timeout;  
+  while (1) {
+    Session *session = mds->sessionmap.get_oldest_session(Session::STATE_OPEN);
+    if (!session) break;
+    dout(20) << "laggiest active session is " << session->info.inst << dendl;
+    if (session->last_cap_renew >= cutoff) {
+      dout(20) << "laggiest active session is " << session->info.inst << " and sufficiently new (" 
+	       << session->last_cap_renew << ")" << dendl;
+      break;
+    }
+
+    dout(10) << "new stale session " << session->info.inst << " last " << session->last_cap_renew << dendl;
+    mds->sessionmap.set_state(session, Session::STATE_STALE);
+    mds->locker->revoke_stale_caps(session);
+    mds->locker->remove_stale_leases(session);
+    mds->send_message_client(new MClientSession(CEPH_SESSION_STALE, session->get_push_seq()), session);
+    finish_flush_session(session, session->get_push_seq());
+  }
+
+  // autoclose
+  cutoff = now;
+  cutoff -= g_conf->mds_session_autoclose;
+
+  // don't kick clients if we've been laggy
+  if (mds->laggy_until > cutoff) {
+    dout(10) << " laggy_until " << mds->laggy_until << " > cutoff " << cutoff
+	     << ", not kicking any clients to be safe" << dendl;
+    return;
+  }
+
+  while (1) {
+    Session *session = mds->sessionmap.get_oldest_session(Session::STATE_STALE);
+    if (!session)
+      break;
+    if (session->is_importing()) {
+      dout(10) << "stopping at importing session " << session->info.inst << dendl;
+      break;
+    }
+    assert(session->is_stale());
+    if (session->last_cap_renew >= cutoff) {
+      dout(20) << "oldest stale session is " << session->info.inst << " and sufficiently new (" 
+	       << session->last_cap_renew << ")" << dendl;
+      break;
+    }
+    
+    utime_t age = now;
+    age -= session->last_cap_renew;
+    mds->clog.info() << "closing stale session " << session->info.inst
+	<< " after " << age << "\n";
+    dout(10) << "autoclosing stale session " << session->info.inst << " last " << session->last_cap_renew << dendl;
+    kill_session(session);
+  }
+}
+
+void Server::kill_session(Session *session)
+{
+  if ((session->is_opening() ||
+       session->is_open() ||
+       session->is_stale()) &&
+      !session->is_importing()) {
+    dout(10) << "kill_session " << session << dendl;
+    journal_close_session(session, Session::STATE_KILLING);
+  } else {
+    dout(10) << "kill_session importing or already closing/killing " << session << dendl;
+    assert(session->is_closing() || 
+	   session->is_closed() || 
+	   session->is_killing() ||
+	   session->is_importing());
+  }
+}
+
+void Server::journal_close_session(Session *session, int state)
+{
+  uint64_t sseq = mds->sessionmap.set_state(session, state);
+  version_t pv = ++mds->sessionmap.projected;
+  version_t piv = 0;
+
+  // release alloc and pending-alloc inos for this session
+  // and wipe out session state, in case the session close aborts for some reason
+  interval_set both;
+  both.swap(session->info.prealloc_inos);
+  both.insert(session->pending_prealloc_inos);
+  session->pending_prealloc_inos.clear();
+  if (both.size()) {
+    mds->inotable->project_release_ids(both);
+    piv = mds->inotable->get_projected_version();
+  } else
+    piv = 0;
+
+  mdlog->start_submit_entry(new ESession(session->info.inst, false, pv, both, piv),
+			    new C_MDS_session_finish(mds, session, sseq, false, pv, both, piv));
+  mdlog->flush();
+
+  // clean up requests, too
+  elist::iterator p =
+    session->requests.begin(member_offset(MDRequestImpl,
+					  item_session_request));
+  while (!p.end()) {
+    MDRequestRef mdr = mdcache->request_get((*p)->reqid);
+    ++p;
+    mdcache->request_kill(mdr);
+  }
+
+  finish_flush_session(session, session->get_push_seq());
+}
+
+void Server::reconnect_clients()
+{
+  mds->sessionmap.get_client_set(client_reconnect_gather);
+
+  if (client_reconnect_gather.empty()) {
+    dout(7) << "reconnect_clients -- no sessions, doing nothing." << dendl;
+    reconnect_gather_finish();
+    return;
+  }
+
+  // clients will get the mdsmap and discover we're reconnecting via the monitor.
+  
+  reconnect_start = ceph_clock_now(g_ceph_context);
+  dout(1) << "reconnect_clients -- " << client_reconnect_gather.size() << " sessions" << dendl;
+  mds->sessionmap.dump();
+}
+
+/* This function DOES put the passed message before returning*/
+void Server::handle_client_reconnect(MClientReconnect *m)
+{
+  dout(7) << "handle_client_reconnect " << m->get_source() << dendl;
+  int from = m->get_source().num();
+  Session *session = get_session(m);
+  assert(session);
+
+  if (!mds->is_reconnect() && mds->get_want_state() == CEPH_MDS_STATE_RECONNECT) {
+    dout(10) << " we're almost in reconnect state (mdsmap delivery race?); waiting" << dendl;
+    mds->wait_for_reconnect(new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+
+  utime_t delay = ceph_clock_now(g_ceph_context);
+  delay -= reconnect_start;
+  dout(10) << " reconnect_start " << reconnect_start << " delay " << delay << dendl;
+
+  if (!mds->is_reconnect()) {
+    // XXX maybe in the future we can do better than this?
+    dout(1) << " no longer in reconnect state, ignoring reconnect, sending close" << dendl;
+    mds->clog.info() << "denied reconnect attempt (mds is "
+       << ceph_mds_state_name(mds->get_state())
+       << ") from " << m->get_source_inst()
+       << " after " << delay << " (allowed interval " << g_conf->mds_reconnect_timeout << ")\n";
+    mds->messenger->send_message(new MClientSession(CEPH_SESSION_CLOSE), m->get_connection());
+    m->put();
+    return;
+  }
+
+  // notify client of success with an OPEN
+  mds->messenger->send_message(new MClientSession(CEPH_SESSION_OPEN), m->get_connection());
+
+  if (session->is_closed()) {
+    dout(10) << " session is closed, will make best effort to reconnect " 
+	     << m->get_source_inst() << dendl;
+    mds->sessionmap.set_state(session, Session::STATE_OPENING);
+    version_t pv = ++mds->sessionmap.projected;
+    uint64_t sseq = session->get_state_seq();
+    mdlog->start_submit_entry(new ESession(session->info.inst, true, pv),
+			      new C_MDS_session_finish(mds, session, sseq, true, pv));
+    mdlog->flush();
+    mds->clog.debug() << "reconnect by new " << session->info.inst
+	<< " after " << delay << "\n";
+  } else {
+    mds->clog.debug() << "reconnect by " << session->info.inst
+	<< " after " << delay << "\n";
+  }
+  
+  // snaprealms
+  for (vector::iterator p = m->realms.begin();
+       p != m->realms.end();
+       ++p) {
+    CInode *in = mdcache->get_inode(inodeno_t(p->ino));
+    if (in && in->state_test(CInode::STATE_PURGING))
+      continue;
+    if (in) {
+      assert(in->snaprealm);
+      if (in->snaprealm->have_past_parents_open()) {
+	dout(15) << "open snaprealm (w/ past parents) on " << *in << dendl;
+	mdcache->finish_snaprealm_reconnect(from, in->snaprealm, snapid_t(p->seq));
+      } else {
+	dout(15) << "open snaprealm (w/o past parents) on " << *in << dendl;
+	mdcache->add_reconnected_snaprealm(from, inodeno_t(p->ino), snapid_t(p->seq));
+      }
+    } else {
+      dout(15) << "open snaprealm (w/o inode) on " << inodeno_t(p->ino)
+	       << " seq " << p->seq << dendl;
+      mdcache->add_reconnected_snaprealm(from, inodeno_t(p->ino), snapid_t(p->seq));
+    }
+  }
+
+  // caps
+  for (map::iterator p = m->caps.begin();
+       p != m->caps.end();
+       ++p) {
+    // make sure our last_cap_id is MAX over all issued caps
+    if (p->second.capinfo.cap_id > mdcache->last_cap_id)
+      mdcache->last_cap_id = p->second.capinfo.cap_id;
+    
+    CInode *in = mdcache->get_inode(p->first);
+    if (in && in->state_test(CInode::STATE_PURGING))
+      continue;
+    if (in && in->is_auth()) {
+      // we recovered it, and it's ours.  take note.
+      dout(15) << "open cap realm " << inodeno_t(p->second.capinfo.snaprealm)
+	       << " on " << *in << dendl;
+      in->reconnect_cap(from, p->second.capinfo, session);
+      mds->mdcache->add_reconnected_cap(in, from, inodeno_t(p->second.capinfo.snaprealm));
+      recover_filelocks(in, p->second.flockbl, m->get_orig_source().num());
+      continue;
+    }
+      
+    if (in && !in->is_auth()) {
+      // not mine.
+      dout(10) << "non-auth " << *in << ", will pass off to authority" << dendl;
+      // add to cap export list.
+      mdcache->rejoin_export_caps(p->first, from, p->second.capinfo,
+				  in->authority().first);
+    } else {
+      // don't know if the inode is mine
+      dout(10) << "missing ino " << p->first << ", will load later" << dendl;
+      mdcache->rejoin_recovered_caps(p->first, from, p->second, -1);
+    }
+  }
+
+  // remove from gather set
+  client_reconnect_gather.erase(from);
+  if (client_reconnect_gather.empty())
+    reconnect_gather_finish();
+
+  m->put();
+}
+
+
+
+void Server::reconnect_gather_finish()
+{
+  dout(7) << "reconnect_gather_finish.  failed on " << failed_reconnects << " clients" << dendl;
+  mds->reconnect_done();
+}
+
+void Server::reconnect_tick()
+{
+  utime_t reconnect_end = reconnect_start;
+  reconnect_end += g_conf->mds_reconnect_timeout;
+  if (ceph_clock_now(g_ceph_context) >= reconnect_end &&
+      !client_reconnect_gather.empty()) {
+    dout(10) << "reconnect timed out" << dendl;
+    for (set::iterator p = client_reconnect_gather.begin();
+	 p != client_reconnect_gather.end();
+	 ++p) {
+      Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(p->v));
+      assert(session);
+      dout(1) << "reconnect gave up on " << session->info.inst << dendl;
+      kill_session(session);
+      failed_reconnects++;
+    }
+    client_reconnect_gather.clear();
+    reconnect_gather_finish();
+  }
+}
+
+void Server::recover_filelocks(CInode *in, bufferlist locks, int64_t client)
+{
+  if (!locks.length()) return;
+  int numlocks;
+  ceph_filelock lock;
+  bufferlist::iterator p = locks.begin();
+  ::decode(numlocks, p);
+  for (int i = 0; i < numlocks; ++i) {
+    ::decode(lock, p);
+    lock.client = client;
+    in->fcntl_locks.held_locks.insert(pair
+				      (lock.start, lock));
+    ++in->fcntl_locks.client_held_lock_counts[client];
+  }
+  ::decode(numlocks, p);
+  for (int i = 0; i < numlocks; ++i) {
+    ::decode(lock, p);
+    lock.client = client;
+    in->flock_locks.held_locks.insert(pair
+				      (lock.start, lock));
+    ++in->flock_locks.client_held_lock_counts[client];
+  }
+}
+
+void Server::recall_client_state(float ratio)
+{
+  int max_caps_per_client = (int)(g_conf->mds_cache_size * .8);
+  int min_caps_per_client = 100;
+
+  dout(10) << "recall_client_state " << ratio
+	   << ", caps per client " << min_caps_per_client << "-" << max_caps_per_client
+	   << dendl;
+
+  set sessions;
+  mds->sessionmap.get_client_session_set(sessions);
+  for (set::const_iterator p = sessions.begin();
+       p != sessions.end();
+       ++p) {
+    Session *session = *p;
+    if (!session->is_open() ||
+	!session->info.inst.name.is_client())
+      continue;
+
+    dout(10) << " session " << session->info.inst
+	     << " caps " << session->caps.size()
+	     << ", leases " << session->leases.size()
+	     << dendl;
+
+    if (session->caps.size() > min_caps_per_client) {	
+      int newlim = (int)(session->caps.size() * ratio);
+      if (newlim > max_caps_per_client)
+	newlim = max_caps_per_client;
+      MClientSession *m = new MClientSession(CEPH_SESSION_RECALL_STATE);
+      m->head.max_caps = newlim;
+      mds->send_message_client(m, session);
+    }
+  }
+ 
+}
+
+
+/*******
+ * some generic stuff for finishing off requests
+ */
+void Server::journal_and_reply(MDRequestRef& mdr, CInode *in, CDentry *dn, LogEvent *le, Context *fin)
+{
+  dout(10) << "journal_and_reply tracei " << in << " tracedn " << dn << dendl;
+
+  // note trace items for eventual reply.
+  mdr->tracei = in;
+  if (in)
+    mdr->pin(in);
+
+  mdr->tracedn = dn;
+  if (dn)
+    mdr->pin(dn);
+
+  early_reply(mdr, in, dn);
+  
+  mdr->committing = true;
+  mdlog->submit_entry(le, fin);
+  
+  if (mdr->client_request && mdr->client_request->is_replay()) {
+    if (mds->queue_one_replay()) {
+      dout(10) << " queued next replay op" << dendl;
+    } else {
+      dout(10) << " journaled last replay op, flushing" << dendl;
+      mdlog->flush();
+    }
+  } else if (mdr->did_early_reply)
+    mds->locker->drop_rdlocks(mdr.get());
+  else
+    mdlog->flush();
+}
+
+/*
+ * send generic response (just an error code), clean up mdr
+ */
+void Server::reply_request(MDRequestRef& mdr, int r, CInode *tracei, CDentry *tracedn)
+{
+  reply_request(mdr, new MClientReply(mdr->client_request, r), tracei, tracedn);
+}
+
+void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
+{
+  if (!g_conf->mds_early_reply)
+    return;
+
+  if (mdr->are_slaves()) {
+    dout(10) << "early_reply - there are slaves, not allowed." << dendl;
+    mds->mdlog->flush();
+    return; 
+  }
+
+  if (mdr->alloc_ino) {
+    dout(10) << "early_reply - allocated ino, not allowed" << dendl;
+    return;
+  }
+
+  MClientRequest *req = mdr->client_request;
+  entity_inst_t client_inst = req->get_source_inst();
+  if (client_inst.name.is_mds())
+    return;
+
+  if (req->is_replay()) {
+    dout(10) << " no early reply on replay op" << dendl;
+    mds->mdlog->flush();
+    return;
+  }
+
+
+  MClientReply *reply = new MClientReply(mdr->client_request, 0);
+  reply->set_unsafe();
+
+  // mark xlocks "done", indicating that we are exposing uncommitted changes.
+  //
+  //_rename_finish() does not send dentry link/unlink message to replicas.
+  // so do not set xlocks on dentries "done", the xlocks prevent dentries
+  // that have projected linkages from getting new replica.
+  mds->locker->set_xlocks_done(mdr.get(), mdr->client_request->get_op() == CEPH_MDS_OP_RENAME);
+
+  dout(10) << "early_reply " << reply->get_result() 
+	   << " (" << cpp_strerror(reply->get_result())
+	   << ") " << *req << dendl;
+
+  if (tracei || tracedn) {
+    if (tracei)
+      mdr->cap_releases.erase(tracei->vino());
+    if (tracedn)
+      mdr->cap_releases.erase(tracedn->get_dir()->get_inode()->vino());
+
+    set_trace_dist(mdr->session, reply, tracei, tracedn, mdr->snapid,
+		   mdr->client_request->get_dentry_wanted(),
+		   mdr);
+  }
+
+  reply->set_extra_bl(mdr->reply_extra_bl);
+  messenger->send_message(reply, req->get_connection());
+
+  mdr->did_early_reply = true;
+
+  mds->logger->inc(l_mds_reply);
+  utime_t lat = ceph_clock_now(g_ceph_context) - mdr->client_request->get_recv_stamp();
+  mds->logger->tinc(l_mds_replyl, lat);
+  dout(20) << "lat " << lat << dendl;
+}
+
+/*
+ * send given reply
+ * include a trace to tracei
+ * Clean up mdr
+ */
+void Server::reply_request(MDRequestRef& mdr, MClientReply *reply, CInode *tracei, CDentry *tracedn)
+{
+  assert(mdr.get());
+  MClientRequest *req = mdr->client_request;
+  
+  dout(10) << "reply_request " << reply->get_result() 
+	   << " (" << cpp_strerror(reply->get_result())
+	   << ") " << *req << dendl;
+
+  // note successful request in session map?
+  if (req->may_write() && mdr->session && reply->get_result() == 0)
+    mdr->session->add_completed_request(mdr->reqid.tid, mdr->alloc_ino);
+
+  // give any preallocated inos to the session
+  apply_allocated_inos(mdr);
+
+  // get tracei/tracedn from mdr?
+  snapid_t snapid = mdr->snapid;
+  if (!tracei)
+    tracei = mdr->tracei;
+  if (!tracedn)
+    tracedn = mdr->tracedn;
+
+  bool is_replay = mdr->client_request->is_replay();
+  bool did_early_reply = mdr->did_early_reply;
+  Session *session = mdr->session;
+  entity_inst_t client_inst = req->get_source_inst();
+  int dentry_wanted = req->get_dentry_wanted();
+
+  if (!did_early_reply && !is_replay) {
+
+    mds->logger->inc(l_mds_reply);
+    utime_t lat = ceph_clock_now(g_ceph_context) - mdr->client_request->get_recv_stamp();
+    mds->logger->tinc(l_mds_replyl, lat);
+    dout(20) << "lat " << lat << dendl;
+    
+    if (tracei)
+      mdr->cap_releases.erase(tracei->vino());
+    if (tracedn)
+      mdr->cap_releases.erase(tracedn->get_dir()->get_inode()->vino());
+  }
+
+  // note client connection to direct my reply
+  ConnectionRef client_con = req->get_connection();
+
+  // drop non-rdlocks before replying, so that we can issue leases
+  mdcache->request_drop_non_rdlocks(mdr);
+
+  // reply at all?
+  if (client_inst.name.is_mds()) {
+    reply->put();   // mds doesn't need a reply
+    reply = 0;
+  } else {
+    // send reply.
+    if (!did_early_reply &&   // don't issue leases if we sent an earlier reply already
+	(tracei || tracedn)) {
+      if (is_replay) {
+	if (tracei)
+	  mdcache->try_reconnect_cap(tracei, session);
+      } else {
+	// include metadata in reply
+	set_trace_dist(session, reply, tracei, tracedn,
+	               snapid, dentry_wanted,
+		       mdr);
+      }
+    }
+
+    reply->set_mdsmap_epoch(mds->mdsmap->get_epoch());
+    messenger->send_message(reply, client_con);
+  }
+  
+  // clean up request
+  mdcache->request_finish(mdr);
+
+  // take a closer look at tracei, if it happens to be a remote link
+  if (tracei && 
+      tracei->get_parent_dn() &&
+      tracei->get_parent_dn()->get_projected_linkage()->is_remote())
+    mdcache->eval_remote(tracei->get_parent_dn());
+}
+
+
+void Server::encode_empty_dirstat(bufferlist& bl)
+{
+  static DirStat empty;
+  empty.encode(bl);
+}
+
+void Server::encode_infinite_lease(bufferlist& bl)
+{
+  LeaseStat e;
+  e.seq = 0;
+  e.mask = -1;
+  e.duration_ms = -1;
+  ::encode(e, bl);
+  dout(20) << "encode_infinite_lease " << e << dendl;
+}
+
+void Server::encode_null_lease(bufferlist& bl)
+{
+  LeaseStat e;
+  e.seq = 0;
+  e.mask = 0;
+  e.duration_ms = 0;
+  ::encode(e, bl);
+  dout(20) << "encode_null_lease " << e << dendl;
+}
+
+
+/*
+ * pass inode OR dentry (not both, or we may get confused)
+ *
+ * trace is in reverse order (i.e. root inode comes last)
+ */
+void Server::set_trace_dist(Session *session, MClientReply *reply,
+			    CInode *in, CDentry *dn,
+			    snapid_t snapid,
+			    int dentry_wanted,
+			    MDRequestRef& mdr)
+{
+  // skip doing this for debugging purposes?
+  if (g_conf->mds_inject_traceless_reply_probability &&
+      mdr->ls && !mdr->o_trunc &&
+      (rand() % 10000 < g_conf->mds_inject_traceless_reply_probability * 10000.0)) {
+    dout(5) << "deliberately skipping trace for " << *reply << dendl;
+    return;
+  }
+
+  // inode, dentry, dir, ..., inode
+  bufferlist bl;
+  int whoami = mds->get_nodeid();
+  client_t client = session->get_client();
+  utime_t now = ceph_clock_now(g_ceph_context);
+
+  dout(20) << "set_trace_dist snapid " << snapid << dendl;
+
+  //assert((bool)dn == (bool)dentry_wanted);  // not true for snapshot lookups
+
+  // realm
+  SnapRealm *realm = 0;
+  if (in)
+    realm = in->find_snaprealm();
+  else
+    realm = dn->get_dir()->get_inode()->find_snaprealm();
+  reply->snapbl = realm->get_snap_trace();
+  dout(10) << "set_trace_dist snaprealm " << *realm << " len=" << reply->snapbl.length() << dendl;
+
+  // dir + dentry?
+  if (dn) {
+    reply->head.is_dentry = 1;
+    CDir *dir = dn->get_dir();
+    CInode *diri = dir->get_inode();
+
+    diri->encode_inodestat(bl, session, NULL, snapid);
+    dout(20) << "set_trace_dist added diri " << *diri << dendl;
+
+#ifdef MDS_VERIFY_FRAGSTAT
+    if (dir->is_complete())
+      dir->verify_fragstat();
+#endif
+    dir->encode_dirstat(bl, whoami);
+    dout(20) << "set_trace_dist added dir  " << *dir << dendl;
+
+    ::encode(dn->get_name(), bl);
+    if (snapid == CEPH_NOSNAP)
+      mds->locker->issue_client_lease(dn, client, bl, now, session);
+    else
+      encode_null_lease(bl);
+    dout(20) << "set_trace_dist added dn   " << snapid << " " << *dn << dendl;
+  } else
+    reply->head.is_dentry = 0;
+
+  // inode
+  if (in) {
+    in->encode_inodestat(bl, session, NULL, snapid, 0, mdr->getattr_caps);
+    dout(20) << "set_trace_dist added in   " << *in << dendl;
+    reply->head.is_target = 1;
+  } else
+    reply->head.is_target = 0;
+
+  reply->set_trace(bl);
+}
+
+
+
+
+/***
+ * process a client request
+ * This function DOES put the passed message before returning
+ */
+void Server::handle_client_request(MClientRequest *req)
+{
+  dout(4) << "handle_client_request " << *req << dendl;
+
+  if (logger) logger->inc(l_mdss_hcreq);
+
+  if (!mdcache->is_open()) {
+    dout(5) << "waiting for root" << dendl;
+    mdcache->wait_for_open(new C_MDS_RetryMessage(mds, req));
+    return;
+  }
+
+  // active session?
+  Session *session = 0;
+  if (req->get_source().is_client()) {
+    session = get_session(req);
+    if (!session) {
+      dout(5) << "no session for " << req->get_source() << ", dropping" << dendl;
+      req->put();
+      return;
+    }
+    if (session->is_closed() ||
+	session->is_closing() ||
+	session->is_killing()) {
+      dout(5) << "session closed|closing|killing, dropping" << dendl;
+      req->put();
+      return;
+    }
+  }
+
+  // old mdsmap?
+  if (req->get_mdsmap_epoch() < mds->mdsmap->get_epoch()) {
+    // send it?  hrm, this isn't ideal; they may get a lot of copies if
+    // they have a high request rate.
+  }
+
+  // completed request?
+  if (req->is_replay() ||
+      (req->get_retry_attempt() &&
+       req->get_op() != CEPH_MDS_OP_OPEN && 
+       req->get_op() != CEPH_MDS_OP_CREATE)) {
+    assert(session);
+    inodeno_t created;
+    if (session->have_completed_request(req->get_reqid().tid, &created)) {
+      dout(5) << "already completed " << req->get_reqid() << dendl;
+      MClientReply *reply = new MClientReply(req, 0);
+      if (created != inodeno_t()) {
+	bufferlist extra;
+	::encode(created, extra);
+	reply->set_extra_bl(extra);
+      }
+      mds->messenger->send_message(reply, req->get_connection());
+
+      if (req->is_replay())
+	mds->queue_one_replay();
+
+      req->put();
+      return;
+    }
+  }
+
+  // trim completed_request list
+  if (req->get_oldest_client_tid() > 0) {
+    dout(15) << " oldest_client_tid=" << req->get_oldest_client_tid() << dendl;
+    assert(session);
+    session->trim_completed_requests(req->get_oldest_client_tid());
+  }
+
+  // register + dispatch
+  MDRequestRef mdr = mdcache->request_start(req);
+  if (mdr.get()) {
+    if (session) {
+      mdr->session = session;
+      session->requests.push_back(&mdr->item_session_request);
+    }
+  }
+
+  // process embedded cap releases?
+  //  (only if NOT replay!)
+  if (!req->releases.empty() && req->get_source().is_client() && !req->is_replay()) {
+    client_t client = req->get_source().num();
+    for (vector::iterator p = req->releases.begin();
+	 p != req->releases.end();
+	 ++p)
+      mds->locker->process_request_cap_release(mdr, client, p->item, p->dname);
+    req->releases.clear();
+  }
+
+  if (mdr.get())
+    dispatch_client_request(mdr);
+  return;
+}
+
+void Server::dispatch_client_request(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+
+  if (logger) logger->inc(l_mdss_dcreq);
+
+  dout(7) << "dispatch_client_request " << *req << dendl;
+
+  // we shouldn't be waiting on anyone.
+  assert(mdr->more()->waiting_on_slave.empty());
+  
+  switch (req->get_op()) {
+  case CEPH_MDS_OP_LOOKUPHASH:
+  case CEPH_MDS_OP_LOOKUPINO:
+    handle_client_lookup_ino(mdr, false, false);
+    break;
+  case CEPH_MDS_OP_LOOKUPPARENT:
+    handle_client_lookup_ino(mdr, true, false);
+    break;
+  case CEPH_MDS_OP_LOOKUPNAME:
+    handle_client_lookup_ino(mdr, false, true);
+    break;
+
+    // inodes ops.
+  case CEPH_MDS_OP_LOOKUP:
+    handle_client_getattr(mdr, true);
+    break;
+
+  case CEPH_MDS_OP_LOOKUPSNAP:
+    // lookupsnap does not reference a CDentry; treat it as a getattr
+  case CEPH_MDS_OP_GETATTR:
+    handle_client_getattr(mdr, false);
+    break;
+
+  case CEPH_MDS_OP_SETATTR:
+    handle_client_setattr(mdr);
+    break;
+  case CEPH_MDS_OP_SETLAYOUT:
+    handle_client_setlayout(mdr);
+    break;
+  case CEPH_MDS_OP_SETDIRLAYOUT:
+    handle_client_setdirlayout(mdr);
+    break;
+  case CEPH_MDS_OP_SETXATTR:
+    handle_client_setxattr(mdr);
+    break;
+  case CEPH_MDS_OP_RMXATTR:
+    handle_client_removexattr(mdr);
+    break;
+
+  case CEPH_MDS_OP_READDIR:
+    handle_client_readdir(mdr);
+    break;
+
+  case CEPH_MDS_OP_SETFILELOCK:
+    handle_client_file_setlock(mdr);
+    break;
+
+  case CEPH_MDS_OP_GETFILELOCK:
+    handle_client_file_readlock(mdr);
+    break;
+
+    // funky.
+  case CEPH_MDS_OP_CREATE:
+    if (req->get_retry_attempt() &&
+	mdr->session->have_completed_request(req->get_reqid().tid, NULL))
+      handle_client_open(mdr);  // already created.. just open
+    else
+      handle_client_openc(mdr);
+    break;
+
+  case CEPH_MDS_OP_OPEN:
+    handle_client_open(mdr);
+    break;
+
+    // namespace.
+    // no prior locks.
+  case CEPH_MDS_OP_MKNOD:
+    handle_client_mknod(mdr);
+    break;
+  case CEPH_MDS_OP_LINK:
+    handle_client_link(mdr);
+    break;
+  case CEPH_MDS_OP_UNLINK:
+  case CEPH_MDS_OP_RMDIR:
+    handle_client_unlink(mdr);
+    break;
+  case CEPH_MDS_OP_RENAME:
+    handle_client_rename(mdr);
+    break;
+  case CEPH_MDS_OP_MKDIR:
+    handle_client_mkdir(mdr);
+    break;
+  case CEPH_MDS_OP_SYMLINK:
+    handle_client_symlink(mdr);
+    break;
+
+
+    // snaps
+  case CEPH_MDS_OP_LSSNAP:
+    handle_client_lssnap(mdr);
+    break;
+  case CEPH_MDS_OP_MKSNAP:
+    handle_client_mksnap(mdr);
+    break;
+  case CEPH_MDS_OP_RMSNAP:
+    handle_client_rmsnap(mdr);
+    break;
+
+
+  default:
+    dout(1) << " unknown client op " << req->get_op() << dendl;
+    reply_request(mdr, -EOPNOTSUPP);
+  }
+}
+
+
+// ---------------------------------------
+// SLAVE REQUESTS
+
+/* This function DOES put the passed message before returning*/
+void Server::handle_slave_request(MMDSSlaveRequest *m)
+{
+  dout(4) << "handle_slave_request " << m->get_reqid() << " from " << m->get_source() << dendl;
+  int from = m->get_source().num();
+
+  if (logger) logger->inc(l_mdss_hsreq);
+
+  // reply?
+  if (m->is_reply())
+    return handle_slave_request_reply(m);
+
+  // the purpose of rename notify is enforcing causal message ordering. making sure
+  // bystanders have received all messages from rename srcdn's auth MDS.
+  if (m->get_op() == MMDSSlaveRequest::OP_RENAMENOTIFY) {
+    MMDSSlaveRequest *reply = new MMDSSlaveRequest(m->get_reqid(), m->get_attempt(),
+						   MMDSSlaveRequest::OP_RENAMENOTIFYACK);
+    mds->send_message(reply, m->get_connection());
+    m->put();
+    return;
+  }
+
+  CDentry *straydn = NULL;
+  if (m->stray.length() > 0) {
+    straydn = mdcache->add_replica_stray(m->stray, from);
+    assert(straydn);
+    m->stray.clear();
+  }
+
+  // am i a new slave?
+  MDRequestRef mdr;
+  if (mdcache->have_request(m->get_reqid())) {
+    // existing?
+    mdr = mdcache->request_get(m->get_reqid());
+
+    // is my request newer?
+    if (mdr->attempt > m->get_attempt()) {
+      dout(10) << "local request " << *mdr << " attempt " << mdr->attempt << " > " << m->get_attempt()
+	       << ", dropping " << *m << dendl;
+      m->put();
+      return;
+    }
+
+
+    if (mdr->attempt < m->get_attempt()) {
+      // mine is old, close it out
+      dout(10) << "local request " << *mdr << " attempt " << mdr->attempt << " < " << m->get_attempt()
+	       << ", closing out" << dendl;
+      mdcache->request_finish(mdr);
+      mdr.reset();
+    } else if (mdr->slave_to_mds != from) {
+      dout(10) << "local request " << *mdr << " not slave to mds." << from << dendl;
+      m->put();
+      return;
+    }
+  }
+  if (!mdr.get()) {
+    // new?
+    if (m->get_op() == MMDSSlaveRequest::OP_FINISH) {
+      dout(10) << "missing slave request for " << m->get_reqid() 
+	       << " OP_FINISH, must have lost race with a forward" << dendl;
+      m->put();
+      return;
+    }
+    mdr = mdcache->request_start_slave(m->get_reqid(), m->get_attempt(), from);
+  }
+  assert(mdr->slave_request == 0);     // only one at a time, please!  
+
+  if (straydn) {
+    mdr->pin(straydn);
+    mdr->straydn = straydn;
+  }
+
+  if (!mds->is_clientreplay() && !mds->is_active() && !mds->is_stopping()) {
+    dout(3) << "not clientreplay|active yet, waiting" << dendl;
+    mds->wait_for_replay(new C_MDS_RetryMessage(mds, m));
+    return;
+  } else if (mds->is_clientreplay() && !mds->mdsmap->is_clientreplay(from) &&
+	     mdr->locks.empty()) {
+    dout(3) << "not active yet, waiting" << dendl;
+    mds->wait_for_active(new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+
+  mdr->slave_request = m;
+  
+  dispatch_slave_request(mdr);
+}
+
+/* This function DOES put the passed message before returning*/
+void Server::handle_slave_request_reply(MMDSSlaveRequest *m)
+{
+  int from = m->get_source().num();
+  
+  if (!mds->is_clientreplay() && !mds->is_active() && !mds->is_stopping()) {
+    dout(3) << "not clientreplay|active yet, waiting" << dendl;
+    mds->wait_for_replay(new C_MDS_RetryMessage(mds, m));
+    return;
+  }
+
+  if (m->get_op() == MMDSSlaveRequest::OP_COMMITTED) {
+    metareqid_t r = m->get_reqid();
+    mds->mdcache->committed_master_slave(r, from);
+    m->put();
+    return;
+  }
+
+  MDRequestRef mdr = mdcache->request_get(m->get_reqid());
+  if (!mdr.get()) {
+    dout(10) << "handle_slave_request_reply ignoring reply from unknown reqid " << m->get_reqid() << dendl;
+    m->put();
+    return;
+  }
+  if (m->get_attempt() != mdr->attempt) {
+    dout(10) << "handle_slave_request_reply " << *mdr << " ignoring reply from other attempt "
+	     << m->get_attempt() << dendl;
+    m->put();
+    return;
+  }
+
+  switch (m->get_op()) {
+  case MMDSSlaveRequest::OP_XLOCKACK:
+    {
+      // identify lock, master request
+      SimpleLock *lock = mds->locker->get_lock(m->get_lock_type(),
+					       m->get_object_info());
+      mdr->more()->slaves.insert(from);
+      dout(10) << "got remote xlock on " << *lock << " on " << *lock->get_parent() << dendl;
+      mdr->xlocks.insert(lock);
+      mdr->locks.insert(lock);
+      mdr->finish_locking(lock);
+      lock->get_xlock(mdr, mdr->get_client());
+
+      assert(mdr->more()->waiting_on_slave.count(from));
+      mdr->more()->waiting_on_slave.erase(from);
+      assert(mdr->more()->waiting_on_slave.empty());
+      mdcache->dispatch_request(mdr);
+    }
+    break;
+    
+  case MMDSSlaveRequest::OP_WRLOCKACK:
+    {
+      // identify lock, master request
+      SimpleLock *lock = mds->locker->get_lock(m->get_lock_type(),
+					       m->get_object_info());
+      mdr->more()->slaves.insert(from);
+      dout(10) << "got remote wrlock on " << *lock << " on " << *lock->get_parent() << dendl;
+      mdr->remote_wrlocks[lock] = from;
+      mdr->locks.insert(lock);
+      mdr->finish_locking(lock);
+
+      assert(mdr->more()->waiting_on_slave.count(from));
+      mdr->more()->waiting_on_slave.erase(from);
+      assert(mdr->more()->waiting_on_slave.empty());
+      mdcache->dispatch_request(mdr);
+    }
+    break;
+
+  case MMDSSlaveRequest::OP_AUTHPINACK:
+    handle_slave_auth_pin_ack(mdr, m);
+    break;
+
+  case MMDSSlaveRequest::OP_LINKPREPACK:
+    handle_slave_link_prep_ack(mdr, m);
+    break;
+
+  case MMDSSlaveRequest::OP_RMDIRPREPACK:
+    handle_slave_rmdir_prep_ack(mdr, m);
+    break;
+
+  case MMDSSlaveRequest::OP_RENAMEPREPACK:
+    handle_slave_rename_prep_ack(mdr, m);
+    break;
+
+  case MMDSSlaveRequest::OP_RENAMENOTIFYACK:
+    handle_slave_rename_notify_ack(mdr, m);
+    break;
+
+  default:
+    assert(0);
+  }
+  
+  // done with reply.
+  m->put();
+}
+
+/* This function DOES put the mdr->slave_request before returning*/
+void Server::dispatch_slave_request(MDRequestRef& mdr)
+{
+  dout(7) << "dispatch_slave_request " << *mdr << " " << *mdr->slave_request << dendl;
+
+  if (mdr->aborted) {
+    dout(7) << " abort flag set, finishing" << dendl;
+    mdcache->request_finish(mdr);
+    return;
+  }
+
+  if (logger) logger->inc(l_mdss_dsreq);
+
+  int op = mdr->slave_request->get_op();
+  switch (op) {
+  case MMDSSlaveRequest::OP_XLOCK:
+  case MMDSSlaveRequest::OP_WRLOCK:
+    {
+      // identify object
+      SimpleLock *lock = mds->locker->get_lock(mdr->slave_request->get_lock_type(),
+					       mdr->slave_request->get_object_info());
+
+      if (!lock) {
+	dout(10) << "don't have object, dropping" << dendl;
+	assert(0); // can this happen, if we auth pinned properly.
+      }
+      if (op == MMDSSlaveRequest::OP_XLOCK && !lock->get_parent()->is_auth()) {
+	dout(10) << "not auth for remote xlock attempt, dropping on " 
+		 << *lock << " on " << *lock->get_parent() << dendl;
+      } else {
+	// use acquire_locks so that we get auth_pinning.
+	set rdlocks;
+	set wrlocks = mdr->wrlocks;
+	set xlocks = mdr->xlocks;
+
+	int replycode = 0;
+	switch (op) {
+	case MMDSSlaveRequest::OP_XLOCK:
+	  xlocks.insert(lock);
+	  replycode = MMDSSlaveRequest::OP_XLOCKACK;
+	  break;
+	case MMDSSlaveRequest::OP_WRLOCK:
+	  wrlocks.insert(lock);
+	  replycode = MMDSSlaveRequest::OP_WRLOCKACK;
+	  break;
+	}
+	
+	if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+	  return;
+	
+	// ack
+	MMDSSlaveRequest *r = new MMDSSlaveRequest(mdr->reqid, mdr->attempt, replycode);
+	r->set_lock_type(lock->get_type());
+	lock->get_parent()->set_object_info(r->get_object_info());
+	mds->send_message(r, mdr->slave_request->get_connection());
+      }
+
+      // done.
+      mdr->slave_request->put();
+      mdr->slave_request = 0;
+    }
+    break;
+
+  case MMDSSlaveRequest::OP_UNXLOCK:
+  case MMDSSlaveRequest::OP_UNWRLOCK:
+    {  
+      SimpleLock *lock = mds->locker->get_lock(mdr->slave_request->get_lock_type(),
+					       mdr->slave_request->get_object_info());
+      assert(lock);
+      bool need_issue = false;
+      switch (op) {
+      case MMDSSlaveRequest::OP_UNXLOCK:
+	mds->locker->xlock_finish(lock, mdr.get(), &need_issue);
+	break;
+      case MMDSSlaveRequest::OP_UNWRLOCK:
+	mds->locker->wrlock_finish(lock, mdr.get(), &need_issue);
+	break;
+      }
+      if (need_issue)
+	mds->locker->issue_caps(static_cast(lock->get_parent()));
+
+      // done.  no ack necessary.
+      mdr->slave_request->put();
+      mdr->slave_request = 0;
+    }
+    break;
+
+  case MMDSSlaveRequest::OP_DROPLOCKS:
+    mds->locker->drop_locks(mdr.get());
+    mdr->slave_request->put();
+    mdr->slave_request = 0;
+    break;
+
+  case MMDSSlaveRequest::OP_AUTHPIN:
+    handle_slave_auth_pin(mdr);
+    break;
+
+  case MMDSSlaveRequest::OP_LINKPREP:
+  case MMDSSlaveRequest::OP_UNLINKPREP:
+    handle_slave_link_prep(mdr);
+    break;
+
+  case MMDSSlaveRequest::OP_RMDIRPREP:
+    handle_slave_rmdir_prep(mdr);
+    break;
+
+  case MMDSSlaveRequest::OP_RENAMEPREP:
+    handle_slave_rename_prep(mdr);
+    break;
+
+  case MMDSSlaveRequest::OP_FINISH:
+    // information about rename imported caps
+    if (mdr->slave_request->inode_export.length() > 0)
+      mdr->more()->inode_import.claim(mdr->slave_request->inode_export);
+    // finish off request.
+    mdcache->request_finish(mdr);
+    break;
+
+  default: 
+    assert(0);
+  }
+}
+
+/* This function DOES put the mdr->slave_request before returning*/
+void Server::handle_slave_auth_pin(MDRequestRef& mdr)
+{
+  dout(10) << "handle_slave_auth_pin " << *mdr << dendl;
+
+  // build list of objects
+  list objects;
+  CInode *auth_pin_freeze = NULL;
+  bool fail = false, wouldblock = false;
+
+  for (vector::iterator p = mdr->slave_request->get_authpins().begin();
+       p != mdr->slave_request->get_authpins().end();
+       ++p) {
+    MDSCacheObject *object = mdcache->get_object(*p);
+    if (!object) {
+      dout(10) << " don't have " << *p << dendl;
+      fail = true;
+      break;
+    }
+
+    objects.push_back(object);
+    if (*p == mdr->slave_request->get_authpin_freeze())
+      auth_pin_freeze = static_cast(object);
+  }
+  
+  // can we auth pin them?
+  if (!fail) {
+    for (list::iterator p = objects.begin();
+	 p != objects.end();
+	 ++p) {
+      if (!(*p)->is_auth()) {
+	dout(10) << " not auth for " << **p << dendl;
+	fail = true;
+	break;
+      }
+      if (!mdr->can_auth_pin(*p)) {
+	if (mdr->slave_request->is_nonblock()) {
+	  dout(10) << " can't auth_pin (freezing?) " << **p << " nonblocking" << dendl;
+	  fail = true;
+	  wouldblock = true;
+	  break;
+	}
+	// wait
+	dout(10) << " waiting for authpinnable on " << **p << dendl;
+	(*p)->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
+	mdr->drop_local_auth_pins();
+
+	CDir *dir = NULL;
+	if (CInode *in = dynamic_cast(*p)) {
+	  if (!in->is_root())
+	    dir = in->get_parent_dir();
+	} else if (CDentry *dn = dynamic_cast(*p)) {
+	  dir = dn->get_dir();
+	} else {
+	  assert(0);
+	}
+	if (dir) {
+	  if (dir->is_freezing_dir())
+	    mdcache->fragment_freeze_inc_num_waiters(dir);
+	  if (dir->is_freezing_tree()) {
+	    while (!dir->is_freezing_tree_root())
+	      dir = dir->get_parent_dir();
+	    mdcache->migrator->export_freeze_inc_num_waiters(dir);
+	  }
+	}
+	return;
+      }
+    }
+  }
+
+  // auth pin!
+  if (fail) {
+    mdr->drop_local_auth_pins();  // just in case
+  } else {
+    /* freeze authpin wrong inode */
+    if (mdr->has_more() && mdr->more()->is_freeze_authpin &&
+	mdr->more()->rename_inode != auth_pin_freeze)
+      mdr->unfreeze_auth_pin(true);
+
+    /* handle_slave_rename_prep() call freeze_inode() to wait for all other operations
+     * on the source inode to complete. This happens after all locks for the rename
+     * operation are acquired. But to acquire locks, we need auth pin locks' parent
+     * objects first. So there is an ABBA deadlock if someone auth pins the source inode
+     * after locks are acquired and before Server::handle_slave_rename_prep() is called.
+     * The solution is freeze the inode and prevent other MDRequests from getting new
+     * auth pins.
+     */
+    if (auth_pin_freeze) {
+      dout(10) << " freezing auth pin on " << *auth_pin_freeze << dendl;
+      if (!mdr->freeze_auth_pin(auth_pin_freeze)) {
+	auth_pin_freeze->add_waiter(CInode::WAIT_FROZEN, new C_MDS_RetryRequest(mdcache, mdr));
+	mds->mdlog->flush();
+	return;
+      }
+    }
+    for (list::iterator p = objects.begin();
+	 p != objects.end();
+	 ++p) {
+      dout(10) << "auth_pinning " << **p << dendl;
+      mdr->auth_pin(*p);
+    }
+  }
+
+  // ack!
+  MMDSSlaveRequest *reply = new MMDSSlaveRequest(mdr->reqid, mdr->attempt, MMDSSlaveRequest::OP_AUTHPINACK);
+  
+  // return list of my auth_pins (if any)
+  for (set::iterator p = mdr->auth_pins.begin();
+       p != mdr->auth_pins.end();
+       ++p) {
+    MDSCacheObjectInfo info;
+    (*p)->set_object_info(info);
+    reply->get_authpins().push_back(info);
+  }
+
+  if (auth_pin_freeze)
+    auth_pin_freeze->set_object_info(reply->get_authpin_freeze());
+
+  if (wouldblock)
+    reply->mark_error_wouldblock();
+
+  mds->send_message_mds(reply, mdr->slave_to_mds);
+  
+  // clean up this request
+  mdr->slave_request->put();
+  mdr->slave_request = 0;
+  return;
+}
+
+/* This function DOES NOT put the passed ack before returning*/
+void Server::handle_slave_auth_pin_ack(MDRequestRef& mdr, MMDSSlaveRequest *ack)
+{
+  dout(10) << "handle_slave_auth_pin_ack on " << *mdr << " " << *ack << dendl;
+  int from = ack->get_source().num();
+
+  // added auth pins?
+  set pinned;
+  for (vector::iterator p = ack->get_authpins().begin();
+       p != ack->get_authpins().end();
+       ++p) {
+    MDSCacheObject *object = mdcache->get_object(*p);
+    assert(object);  // we pinned it
+    dout(10) << " remote has pinned " << *object << dendl;
+    if (!mdr->is_auth_pinned(object))
+      mdr->remote_auth_pins.insert(object);
+    if (*p == ack->get_authpin_freeze())
+      mdr->set_remote_frozen_auth_pin(static_cast(object));
+    pinned.insert(object);
+  }
+
+  // removed auth pins?
+  set::iterator p = mdr->remote_auth_pins.begin();
+  while (p != mdr->remote_auth_pins.end()) {
+    if ((*p)->authority().first == from &&
+	pinned.count(*p) == 0) {
+      dout(10) << " remote has unpinned " << **p << dendl;
+      set::iterator o = p;
+      ++p;
+      mdr->remote_auth_pins.erase(o);
+    } else {
+      ++p;
+    }
+  }
+
+  if (ack->is_error_wouldblock())
+    mdr->aborted = true;
+  
+  // note slave
+  mdr->more()->slaves.insert(from);
+
+  // clear from waiting list
+  assert(mdr->more()->waiting_on_slave.count(from));
+  mdr->more()->waiting_on_slave.erase(from);
+
+  // go again?
+  if (mdr->more()->waiting_on_slave.empty())
+    mdcache->dispatch_request(mdr);
+  else 
+    dout(10) << "still waiting on slaves " << mdr->more()->waiting_on_slave << dendl;
+}
+
+
+// ---------------------------------------
+// HELPERS
+
+
+/** validate_dentry_dir
+ *
+ * verify that the dir exists and would own the dname.
+ * do not check if the dentry exists.
+ */
+CDir *Server::validate_dentry_dir(MDRequestRef& mdr, CInode *diri, const string& dname)
+{
+  // make sure parent is a dir?
+  if (!diri->is_dir()) {
+    dout(7) << "validate_dentry_dir: not a dir" << dendl;
+    reply_request(mdr, -ENOTDIR);
+    return NULL;
+  }
+
+  // which dirfrag?
+  frag_t fg = diri->pick_dirfrag(dname);
+  CDir *dir = try_open_auth_dirfrag(diri, fg, mdr);
+  if (!dir)
+    return 0;
+
+  // frozen?
+  if (dir->is_frozen()) {
+    dout(7) << "dir is frozen " << *dir << dendl;
+    dir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
+    return NULL;
+  }
+
+  return dir;
+}
+
+
+/** prepare_null_dentry
+ * prepare a null (or existing) dentry in given dir. 
+ * wait for any dn lock.
+ */
+CDentry* Server::prepare_null_dentry(MDRequestRef& mdr, CDir *dir, const string& dname, bool okexist)
+{
+  dout(10) << "prepare_null_dentry " << dname << " in " << *dir << dendl;
+  assert(dir->is_auth());
+
+  client_t client = mdr->get_client();
+
+  // does it already exist?
+  CDentry *dn = dir->lookup(dname);
+  if (dn) {
+    /*
+    if (dn->lock.is_xlocked_by_other(mdr)) {
+      dout(10) << "waiting on xlocked dentry " << *dn << dendl;
+      dn->lock.add_waiter(SimpleLock::WAIT_RD, new C_MDS_RetryRequest(mdcache, mdr));
+      return 0;
+    }
+    */
+    if (!dn->get_linkage(client, mdr)->is_null()) {
+      // name already exists
+      dout(10) << "dentry " << dname << " exists in " << *dir << dendl;
+      if (!okexist) {
+        reply_request(mdr, -EEXIST);
+        return 0;
+      }
+    }
+
+    return dn;
+  }
+
+  // make sure dir is complete
+  if (!dir->is_complete() && (!dir->has_bloom() || dir->is_in_bloom(dname))) {
+    dout(7) << " incomplete dir contents for " << *dir << ", fetching" << dendl;
+    dir->fetch(new C_MDS_RetryRequest(mdcache, mdr));
+    return 0;
+  }
+  
+  // create
+  dn = dir->add_null_dentry(dname);
+  dn->mark_new();
+  dout(10) << "prepare_null_dentry added " << *dn << dendl;
+  return dn;
+}
+
+CDentry* Server::prepare_stray_dentry(MDRequestRef& mdr, CInode *in)
+{
+  CDentry *straydn = mdr->straydn;
+  if (straydn) {
+    string name;
+    in->name_stray_dentry(name);
+    if (straydn->get_name() == name)
+      return straydn;
+
+    assert(!mdr->done_locking);
+    mdr->unpin(straydn);
+  }
+
+  straydn = mdcache->get_or_create_stray_dentry(in);
+  mdr->straydn = straydn;
+  mdr->pin(straydn);
+  return straydn;
+}
+
+/** prepare_new_inode
+ *
+ * create a new inode.  set c/m/atime.  hit dir pop.
+ */
+CInode* Server::prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino, unsigned mode,
+				  ceph_file_layout *layout) 
+{
+  CInode *in = new CInode(mdcache);
+  
+  // assign ino
+  if (mdr->session->info.prealloc_inos.size()) {
+    mdr->used_prealloc_ino = 
+      in->inode.ino = mdr->session->take_ino(useino);  // prealloc -> used
+    mds->sessionmap.projected++;
+    dout(10) << "prepare_new_inode used_prealloc " << mdr->used_prealloc_ino
+	     << " (" << mdr->session->info.prealloc_inos
+	     << ", " << mdr->session->info.prealloc_inos.size() << " left)"
+	     << dendl;
+  } else {
+    mdr->alloc_ino = 
+      in->inode.ino = mds->inotable->project_alloc_id();
+    dout(10) << "prepare_new_inode alloc " << mdr->alloc_ino << dendl;
+  }
+
+  if (useino && useino != in->inode.ino) {
+    dout(0) << "WARNING: client specified " << useino << " and i allocated " << in->inode.ino << dendl;
+    mds->clog.error() << mdr->client_request->get_source()
+       << " specified ino " << useino
+       << " but mds." << mds->whoami << " allocated " << in->inode.ino << "\n";
+    //assert(0); // just for now.
+  }
+    
+  int got = g_conf->mds_client_prealloc_inos - mdr->session->get_num_projected_prealloc_inos();
+  if (got > 0) {
+    mds->inotable->project_alloc_ids(mdr->prealloc_inos, got);
+    assert(mdr->prealloc_inos.size());  // or else fix projected increment semantics
+    mdr->session->pending_prealloc_inos.insert(mdr->prealloc_inos);
+    mds->sessionmap.projected++;
+    dout(10) << "prepare_new_inode prealloc " << mdr->prealloc_inos << dendl;
+  }
+
+  in->inode.version = 1;
+  in->inode.nlink = 1;   // FIXME
+
+  in->inode.mode = mode;
+
+  memset(&in->inode.dir_layout, 0, sizeof(in->inode.dir_layout));
+  if (in->inode.is_dir()) {
+    in->inode.dir_layout.dl_dir_hash = g_conf->mds_default_dir_hash;
+    memset(&in->inode.layout, 0, sizeof(in->inode.layout));
+  } else if (layout) {
+    in->inode.layout = *layout;
+  } else {
+    in->inode.layout = mds->mdcache->default_file_layout;
+  }
+
+  in->inode.truncate_size = -1ull;  // not truncated, yet!
+  in->inode.truncate_seq = 1; /* starting with 1, 0 is kept for no-truncation logic */
+
+  CInode *diri = dir->get_inode();
+
+  dout(10) << oct << " dir mode 0" << diri->inode.mode << " new mode 0" << mode << dec << dendl;
+
+  if (diri->inode.mode & S_ISGID) {
+    dout(10) << " dir is sticky" << dendl;
+    in->inode.gid = diri->inode.gid;
+    if (S_ISDIR(mode)) {
+      dout(10) << " new dir also sticky" << dendl;      
+      in->inode.mode |= S_ISGID;
+    }
+  } else 
+    in->inode.gid = mdr->client_request->get_caller_gid();
+
+  in->inode.uid = mdr->client_request->get_caller_uid();
+
+  in->inode.ctime = in->inode.mtime = in->inode.atime = mdr->now;   // now
+
+  MClientRequest *req = mdr->client_request;
+  if (req->get_data().length()) {
+    bufferlist::iterator p = req->get_data().begin();
+
+    // xattrs on new inode?
+    map xattrs;
+    ::decode(xattrs, p);
+    for (map::iterator p = xattrs.begin(); p != xattrs.end(); ++p) {
+      dout(10) << "prepare_new_inode setting xattr " << p->first << dendl;
+      in->xattrs[p->first] = p->second;
+    }
+  }
+
+  if (!mds->mdsmap->get_inline_data_enabled() ||
+      !mdr->session->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA))
+    in->inode.inline_version = CEPH_INLINE_NONE;
+
+  mdcache->add_inode(in);  // add
+  dout(10) << "prepare_new_inode " << *in << dendl;
+  return in;
+}
+
+void Server::journal_allocated_inos(MDRequestRef& mdr, EMetaBlob *blob)
+{
+  dout(20) << "journal_allocated_inos sessionmapv " << mds->sessionmap.projected
+	   << " inotablev " << mds->inotable->get_projected_version()
+	   << dendl;
+  blob->set_ino_alloc(mdr->alloc_ino,
+		      mdr->used_prealloc_ino,
+		      mdr->prealloc_inos,
+		      mdr->client_request->get_source(),
+		      mds->sessionmap.projected,
+		      mds->inotable->get_projected_version());
+}
+
+void Server::apply_allocated_inos(MDRequestRef& mdr)
+{
+  Session *session = mdr->session;
+  dout(10) << "apply_allocated_inos " << mdr->alloc_ino
+	   << " / " << mdr->prealloc_inos
+	   << " / " << mdr->used_prealloc_ino << dendl;
+
+  if (mdr->alloc_ino) {
+    mds->inotable->apply_alloc_id(mdr->alloc_ino);
+  }
+  if (mdr->prealloc_inos.size()) {
+    assert(session);
+    session->pending_prealloc_inos.subtract(mdr->prealloc_inos);
+    session->info.prealloc_inos.insert(mdr->prealloc_inos);
+    mds->sessionmap.version++;
+    mds->inotable->apply_alloc_ids(mdr->prealloc_inos);
+  }
+  if (mdr->used_prealloc_ino) {
+    assert(session);
+    session->info.used_inos.erase(mdr->used_prealloc_ino);
+    mds->sessionmap.version++;
+  }
+}
+
+
+
+CDir *Server::traverse_to_auth_dir(MDRequestRef& mdr, vector &trace, filepath refpath)
+{
+  // figure parent dir vs dname
+  if (refpath.depth() == 0) {
+    dout(7) << "can't do that to root" << dendl;
+    reply_request(mdr, -EINVAL);
+    return 0;
+  }
+  string dname = refpath.last_dentry();
+  refpath.pop_dentry();
+  
+  dout(10) << "traverse_to_auth_dir dirpath " << refpath << " dname " << dname << dendl;
+
+  // traverse to parent dir
+  CInode *diri;
+  int r = mdcache->path_traverse(mdr, NULL, NULL, refpath, &trace, &diri, MDS_TRAVERSE_FORWARD);
+  if (r > 0) return 0; // delayed
+  if (r < 0) {
+    reply_request(mdr, r);
+    return 0;
+  }
+
+  // is it an auth dir?
+  CDir *dir = validate_dentry_dir(mdr, diri, dname);
+  if (!dir)
+    return 0; // forwarded or waiting for freeze
+
+  dout(10) << "traverse_to_auth_dir " << *dir << dendl;
+  return dir;
+}
+
+class C_MDS_TryFindInode : public Context {
+  Server *server;
+  MDRequestRef mdr;
+public:
+  C_MDS_TryFindInode(Server *s, MDRequestRef& r) : server(s), mdr(r) {}
+  virtual void finish(int r) {
+    if (r == -ESTALE) // :( find_ino_peers failed
+      server->reply_request(mdr, r);
+    else
+      server->dispatch_client_request(mdr);
+  }
+};
+
+/* If this returns null, the request has been handled
+ * as appropriate: forwarded on, or the client's been replied to */
+CInode* Server::rdlock_path_pin_ref(MDRequestRef& mdr, int n,
+				    set &rdlocks,
+				    bool want_auth,
+				    bool no_want_auth, /* for readdir, who doesn't want auth _even_if_ it's
+							  a snapped dir */
+				    ceph_file_layout **layout,
+				    bool no_lookup)    // true if we cannot return a null dentry lease
+{
+  MClientRequest *req = mdr->client_request;
+  const filepath& refpath = n ? req->get_filepath2() : req->get_filepath();
+  dout(10) << "rdlock_path_pin_ref " << *mdr << " " << refpath << dendl;
+
+  if (mdr->done_locking)
+    return mdr->in[n];
+
+  // traverse
+  int r = mdcache->path_traverse(mdr, NULL, NULL, refpath, &mdr->dn[n], &mdr->in[n], MDS_TRAVERSE_FORWARD);
+  if (r > 0)
+    return NULL; // delayed
+  if (r < 0) {  // error
+    if (r == -ENOENT && n == 0 && mdr->dn[n].size()) {
+      reply_request(mdr, r, NULL, no_lookup ? NULL : mdr->dn[n][mdr->dn[n].size()-1]);
+    } else if (r == -ESTALE) {
+      dout(10) << "FAIL on ESTALE but attempting recovery" << dendl;
+      Context *c = new C_MDS_TryFindInode(this, mdr);
+      mdcache->find_ino_peers(refpath.get_ino(), c);
+    } else {
+      dout(10) << "FAIL on error " << r << dendl;
+      reply_request(mdr, r);
+    }
+    return 0;
+  }
+  CInode *ref = mdr->in[n];
+  dout(10) << "ref is " << *ref << dendl;
+
+  // fw to inode auth?
+  if (mdr->snapid != CEPH_NOSNAP && !no_want_auth)
+    want_auth = true;
+
+  if (want_auth) {
+    if (ref->is_ambiguous_auth()) {
+      dout(10) << "waiting for single auth on " << *ref << dendl;
+      ref->add_waiter(CInode::WAIT_SINGLEAUTH, new C_MDS_RetryRequest(mdcache, mdr));
+      return 0;
+    }
+    if (!ref->is_auth()) {
+      dout(10) << "fw to auth for " << *ref << dendl;
+      mdcache->request_forward(mdr, ref->authority().first);
+      return 0;
+    }
+
+    // auth_pin?
+    //   do NOT proceed if freezing, as cap release may defer in that case, and
+    //   we could deadlock when we try to lock @ref.
+    // if we're already auth_pinned, continue; the release has already been processed.
+    if (ref->is_frozen() || ref->is_frozen_auth_pin() ||
+	(ref->is_freezing() && !mdr->is_auth_pinned(ref))) {
+      dout(7) << "waiting for !frozen/authpinnable on " << *ref << dendl;
+      ref->add_waiter(CInode::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
+      /* If we have any auth pins, this will deadlock.
+       * But the only way to get here if we've already got auth pins
+       * is because we're on an inode with snapshots that got updated
+       * between dispatches of this request. So we're going to drop
+       * our locks and our auth pins and reacquire them later.
+       *
+       * This is safe since we're only in this function when working on
+       * a single MDS request; otherwise we'd be in
+       * rdlock_path_xlock_dentry.
+       */
+      mds->locker->drop_locks(mdr.get(), NULL);
+      mdr->drop_local_auth_pins();
+      return 0;
+    }
+
+    mdr->auth_pin(ref);
+  }
+
+  for (int i=0; i<(int)mdr->dn[n].size(); i++) 
+    rdlocks.insert(&mdr->dn[n][i]->lock);
+  if (layout)
+    mds->locker->include_snap_rdlocks_wlayout(rdlocks, ref, layout);
+  else
+    mds->locker->include_snap_rdlocks(rdlocks, ref);
+
+  // set and pin ref
+  mdr->pin(ref);
+  return ref;
+}
+
+
+/** rdlock_path_xlock_dentry
+ * traverse path to the directory that could/would contain dentry.
+ * make sure i am auth for that dentry, forward as necessary.
+ * create null dentry in place (or use existing if okexist).
+ * get rdlocks on traversed dentries, xlock on new dentry.
+ */
+CDentry* Server::rdlock_path_xlock_dentry(MDRequestRef& mdr, int n,
+					  set& rdlocks, set& wrlocks, set& xlocks,
+					  bool okexist, bool mustexist, bool alwaysxlock,
+					  ceph_file_layout **layout)
+{
+  MClientRequest *req = mdr->client_request;
+  const filepath& refpath = n ? req->get_filepath2() : req->get_filepath();
+
+  dout(10) << "rdlock_path_xlock_dentry " << *mdr << " " << refpath << dendl;
+
+  client_t client = mdr->get_client();
+
+  if (mdr->done_locking)
+    return mdr->dn[n].back();
+
+  CDir *dir = traverse_to_auth_dir(mdr, mdr->dn[n], refpath);
+  if (!dir) return 0;
+  dout(10) << "rdlock_path_xlock_dentry dir " << *dir << dendl;
+
+  // make sure we can auth_pin (or have already authpinned) dir
+  if (dir->is_frozen()) {
+    dout(7) << "waiting for !frozen/authpinnable on " << *dir << dendl;
+    dir->add_waiter(CInode::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
+    return 0;
+  }
+
+  CInode *diri = dir->get_inode();
+  if (!mdr->reqid.name.is_mds()) {
+    if (diri->is_system() && !diri->is_root()) {
+      reply_request(mdr, -EROFS);
+      return 0;
+    }
+    if (!diri->is_base() && diri->get_projected_parent_dir()->inode->is_stray()) {
+      reply_request(mdr, -ENOENT);
+      return 0;
+    }
+  }
+
+  // make a null dentry?
+  const string &dname = refpath.last_dentry();
+  CDentry *dn;
+  if (mustexist) {
+    dn = dir->lookup(dname);
+
+    // make sure dir is complete
+    if (!dn && !dir->is_complete() &&
+        (!dir->has_bloom() || dir->is_in_bloom(dname))) {
+      dout(7) << " incomplete dir contents for " << *dir << ", fetching" << dendl;
+      dir->fetch(new C_MDS_RetryRequest(mdcache, mdr));
+      return 0;
+    }
+
+    // readable?
+    if (dn && !dn->lock.can_read(client) && dn->lock.get_xlock_by() != mdr) {
+      dout(10) << "waiting on xlocked dentry " << *dn << dendl;
+      dn->lock.add_waiter(SimpleLock::WAIT_RD, new C_MDS_RetryRequest(mdcache, mdr));
+      return 0;
+    }
+      
+    // exists?
+    if (!dn || dn->get_linkage(client, mdr)->is_null()) {
+      dout(7) << "dentry " << dname << " dne in " << *dir << dendl;
+      reply_request(mdr, -ENOENT);
+      return 0;
+    }    
+  } else {
+    dn = prepare_null_dentry(mdr, dir, dname, okexist);
+    if (!dn) 
+      return 0;
+  }
+
+  mdr->dn[n].push_back(dn);
+  mdr->in[n] = dn->get_projected_linkage()->get_inode();
+
+  // -- lock --
+  // NOTE: rename takes the same set of locks for srcdn
+  for (int i=0; i<(int)mdr->dn[n].size(); i++) 
+    rdlocks.insert(&mdr->dn[n][i]->lock);
+  if (alwaysxlock || dn->get_linkage(client, mdr)->is_null())
+    xlocks.insert(&dn->lock);                 // new dn, xlock
+  else
+    rdlocks.insert(&dn->lock);  // existing dn, rdlock
+  wrlocks.insert(&dn->get_dir()->inode->filelock); // also, wrlock on dir mtime
+  wrlocks.insert(&dn->get_dir()->inode->nestlock); // also, wrlock on dir mtime
+  if (layout)
+    mds->locker->include_snap_rdlocks_wlayout(rdlocks, dn->get_dir()->inode, layout);
+  else
+    mds->locker->include_snap_rdlocks(rdlocks, dn->get_dir()->inode);
+
+  return dn;
+}
+
+
+
+
+
+/**
+ * try_open_auth_dirfrag -- open dirfrag, or forward to dirfrag auth
+ *
+ * @param diri base inode
+ * @param fg the exact frag we want
+ * @param mdr request
+ * @returns the pointer, or NULL if it had to be delayed (but mdr is taken care of)
+ */
+CDir* Server::try_open_auth_dirfrag(CInode *diri, frag_t fg, MDRequestRef& mdr)
+{
+  CDir *dir = diri->get_dirfrag(fg);
+
+  // not open and inode not mine?
+  if (!dir && !diri->is_auth()) {
+    int inauth = diri->authority().first;
+    dout(7) << "try_open_auth_dirfrag: not open, not inode auth, fw to mds." << inauth << dendl;
+    mdcache->request_forward(mdr, inauth);
+    return 0;
+  }
+
+  // not open and inode frozen?
+  if (!dir && diri->is_frozen()) {
+    dout(10) << "try_open_auth_dirfrag: dir inode is frozen, waiting " << *diri << dendl;
+    assert(diri->get_parent_dir());
+    diri->get_parent_dir()->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
+    return 0;
+  }
+
+  // invent?
+  if (!dir) 
+    dir = diri->get_or_open_dirfrag(mds->mdcache, fg);
+ 
+  // am i auth for the dirfrag?
+  if (!dir->is_auth()) {
+    int auth = dir->authority().first;
+    dout(7) << "try_open_auth_dirfrag: not auth for " << *dir
+	    << ", fw to mds." << auth << dendl;
+    mdcache->request_forward(mdr, auth);
+    return 0;
+  }
+
+  return dir;
+}
+
+
+// ===============================================================================
+// STAT
+
+void Server::handle_client_getattr(MDRequestRef& mdr, bool is_lookup)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+
+  if (req->get_filepath().depth() == 0 && is_lookup) {
+    // refpath can't be empty for lookup but it can for
+    // getattr (we do getattr with empty refpath for mount of '/')
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+
+  CInode *ref = rdlock_path_pin_ref(mdr, 0, rdlocks, false, false, NULL, !is_lookup);
+  if (!ref) return;
+
+  /*
+   * if client currently holds the EXCL cap on a field, do not rdlock
+   * it; client's stat() will result in valid info if _either_ EXCL
+   * cap is held or MDS rdlocks and reads the value here.
+   *
+   * handling this case here is easier than weakening rdlock
+   * semantics... that would cause problems elsewhere.
+   */
+  client_t client = mdr->get_client();
+  int issued = 0;
+  Capability *cap = ref->get_client_cap(client);
+  if (cap && (mdr->snapid == CEPH_NOSNAP ||
+	      mdr->snapid <= cap->client_follows))
+    issued = cap->issued();
+
+  int mask = req->head.args.getattr.mask;
+  if ((mask & CEPH_CAP_LINK_SHARED) && (issued & CEPH_CAP_LINK_EXCL) == 0) rdlocks.insert(&ref->linklock);
+  if ((mask & CEPH_CAP_AUTH_SHARED) && (issued & CEPH_CAP_AUTH_EXCL) == 0) rdlocks.insert(&ref->authlock);
+  if ((mask & CEPH_CAP_FILE_SHARED) && (issued & CEPH_CAP_FILE_EXCL) == 0) rdlocks.insert(&ref->filelock);
+  if ((mask & CEPH_CAP_XATTR_SHARED) && (issued & CEPH_CAP_XATTR_EXCL) == 0) rdlocks.insert(&ref->xattrlock);
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // note which caps are requested, so we return at least a snapshot
+  // value for them.  (currently this matters for xattrs and inline data)
+  mdr->getattr_caps = mask;
+
+  mds->balancer->hit_inode(ceph_clock_now(g_ceph_context), ref, META_POP_IRD,
+			   mdr->client_request->get_source().num());
+
+  // reply
+  dout(10) << "reply to stat on " << *req << dendl;
+  reply_request(mdr, 0, ref,
+		is_lookup ? mdr->dn[0].back() : 0);
+}
+
+struct C_MDS_LookupIno2 : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  C_MDS_LookupIno2(Server *s, MDRequestRef& r) : server(s), mdr(r) {}
+  void finish(int r) {
+    server->_lookup_ino_2(mdr, r);
+  }
+};
+
+/* This function DOES clean up the mdr before returning*/
+/*
+ * filepath:  ino
+ */
+void Server::handle_client_lookup_ino(MDRequestRef& mdr,
+				      bool want_parent, bool want_dentry)
+{
+  MClientRequest *req = mdr->client_request;
+
+  inodeno_t ino = req->get_filepath().get_ino();
+  CInode *in = mdcache->get_inode(ino);
+  if (in && in->state_test(CInode::STATE_PURGING)) {
+    reply_request(mdr, -ESTALE);
+    return;
+  }
+  if (!in) {
+    mdcache->open_ino(ino, (int64_t)-1, new C_MDS_LookupIno2(this, mdr), false);
+    return;
+  }
+
+  CDentry *dn = in->get_projected_parent_dn();
+  CInode *diri = dn ? dn->get_dir()->inode : NULL;
+  if (dn && (want_parent || want_dentry)) {
+    mdr->pin(dn);
+    set rdlocks, wrlocks, xlocks;
+    rdlocks.insert(&dn->lock);
+    if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+      return;
+  }
+
+  if (want_parent) {
+    if (in->is_base()) {
+      reply_request(mdr, -EINVAL);
+      return;
+    }
+    if (!diri || diri->is_stray()) {
+      reply_request(mdr, -ESTALE);
+      return;
+    }
+    dout(10) << "reply to lookup_parent " << *in << dendl;
+    reply_request(mdr, 0, diri);
+  } else {
+    if (want_dentry) {
+      inodeno_t dirino = req->get_filepath2().get_ino();
+      if (!diri || (dirino != inodeno_t() && diri->ino() != dirino)) {
+	reply_request(mdr, -ENOENT);
+	return;
+      }
+      dout(10) << "reply to lookup_name " << *in << dendl;
+    } else
+      dout(10) << "reply to lookup_ino " << *in << dendl;
+
+    reply_request(mdr, 0, in, want_dentry ? dn : NULL);
+  }
+}
+
+void Server::_lookup_ino_2(MDRequestRef& mdr, int r)
+{
+  inodeno_t ino = mdr->client_request->get_filepath().get_ino();
+  dout(10) << "_lookup_ino_2 " << mdr.get() << " ino " << ino << " r=" << r << dendl;
+  if (r >= 0) {
+    if (r == mds->get_nodeid())
+      dispatch_client_request(mdr);
+    else
+      mdcache->request_forward(mdr, r);
+    return;
+  }
+
+  // give up
+  if (r == -ENOENT || r == -ENODATA)
+    r = -ESTALE;
+  reply_request(mdr, r);
+}
+
+
+/* This function takes responsibility for the passed mdr*/
+void Server::handle_client_open(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+
+  int flags = req->head.args.open.flags;
+  int cmode = ceph_flags_to_mode(req->head.args.open.flags);
+
+  bool need_auth = !file_mode_is_readonly(cmode) || (flags & O_TRUNC);
+
+  dout(7) << "open on " << req->get_filepath() << dendl;
+
+  if (cmode < 0) {
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+  
+  set rdlocks, wrlocks, xlocks;
+  CInode *cur = rdlock_path_pin_ref(mdr, 0, rdlocks, need_auth);
+  if (!cur)
+    return;
+
+  if (cur->is_frozen() || cur->state_test(CInode::STATE_EXPORTINGCAPS)) {
+    assert(!need_auth);
+    mdr->done_locking = false;
+    CInode *cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true);
+    if (!cur)
+      return;
+  }
+
+  if (mdr->snapid != CEPH_NOSNAP && mdr->client_request->may_write()) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+
+  // can only open a dir with mode FILE_MODE_PIN, at least for now.
+  if (cur->inode.is_dir())
+    cmode = CEPH_FILE_MODE_PIN;
+
+  dout(10) << "open flags = " << flags
+	   << ", filemode = " << cmode
+	   << ", need_auth = " << need_auth
+	   << dendl;
+  
+  // regular file?
+  /*if (!cur->inode.is_file() && !cur->inode.is_dir()) {
+    dout(7) << "not a file or dir " << *cur << dendl;
+    reply_request(mdr, -ENXIO);                 // FIXME what error do we want?
+    return;
+    }*/
+  if ((req->head.args.open.flags & O_DIRECTORY) && !cur->inode.is_dir()) {
+    dout(7) << "specified O_DIRECTORY on non-directory " << *cur << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+
+  if (cur->inode.inline_version != CEPH_INLINE_NONE &&
+      !mdr->session->connection->has_feature(CEPH_FEATURE_MDS_INLINE_DATA)) {
+    dout(7) << "old client cannot open inline data file " << *cur << dendl;
+    reply_request(mdr, -EPERM);
+    return;
+  }
+  
+  // snapped data is read only
+  if (mdr->snapid != CEPH_NOSNAP &&
+      (cmode & CEPH_FILE_MODE_WR)) {
+    dout(7) << "snap " << mdr->snapid << " is read-only " << *cur << dendl;
+    reply_request(mdr, -EPERM);
+    return;
+  }
+
+  // O_TRUNC
+  if ((flags & O_TRUNC) &&
+      !(req->get_retry_attempt() &&
+	mdr->session->have_completed_request(req->get_reqid().tid, NULL))) {
+    assert(cur->is_auth());
+
+    xlocks.insert(&cur->filelock);
+    if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+      return;
+
+    // wait for pending truncate?
+    inode_t *pi = cur->get_projected_inode();
+    if (pi->is_truncating()) {
+      dout(10) << " waiting for pending truncate from " << pi->truncate_from
+	       << " to " << pi->truncate_size << " to complete on " << *cur << dendl;
+      cur->add_waiter(CInode::WAIT_TRUNC, new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+    
+    do_open_truncate(mdr, cmode);
+    return;
+  }
+
+  // sync filelock if snapped.
+  //  this makes us wait for writers to flushsnaps, ensuring we get accurate metadata,
+  //  and that data itself is flushed so that we can read the snapped data off disk.
+  if (mdr->snapid != CEPH_NOSNAP && !cur->is_dir()) {
+    rdlocks.insert(&cur->filelock);
+    if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+      return;
+  }
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  if (cur->is_file() || cur->is_dir()) {
+    if (mdr->snapid == CEPH_NOSNAP) {
+      // register new cap
+      Capability *cap = mds->locker->issue_new_caps(cur, cmode, mdr->session, 0, req->is_replay());
+      if (cap)
+	dout(12) << "open issued caps " << ccap_string(cap->pending())
+		 << " for " << req->get_source()
+		 << " on " << *cur << dendl;
+    } else {
+      int caps = ceph_caps_for_mode(cmode);
+      dout(12) << "open issued IMMUTABLE SNAP caps " << ccap_string(caps)
+	       << " for " << req->get_source()
+	       << " snapid " << mdr->snapid
+	       << " on " << *cur << dendl;
+      mdr->snap_caps = caps;
+    }
+  }
+
+  // increase max_size?
+  if (cmode & CEPH_FILE_MODE_WR)
+    mds->locker->check_inode_max_size(cur);
+
+  // make sure this inode gets into the journal
+  if (cur->is_auth() && cur->last == CEPH_NOSNAP &&
+      !cur->item_open_file.is_on_list()) {
+    LogSegment *ls = mds->mdlog->get_current_segment();
+    EOpen *le = new EOpen(mds->mdlog);
+    mdlog->start_entry(le);
+    le->add_clean_inode(cur);
+    ls->open_files.push_back(&cur->item_open_file);
+    mds->mdlog->submit_entry(le);
+  }
+  
+  // hit pop
+  mdr->now = ceph_clock_now(g_ceph_context);
+  if (cmode == CEPH_FILE_MODE_RDWR ||
+      cmode == CEPH_FILE_MODE_WR) 
+    mds->balancer->hit_inode(mdr->now, cur, META_POP_IWR);
+  else
+    mds->balancer->hit_inode(mdr->now, cur, META_POP_IRD, 
+			     mdr->client_request->get_source().num());
+
+  CDentry *dn = 0;
+  if (req->get_dentry_wanted()) {
+    assert(mdr->dn[0].size());
+    dn = mdr->dn[0].back();
+  }
+
+  reply_request(mdr, 0, cur, dn);
+}
+
+class C_MDS_openc_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CDentry *dn;
+  CInode *newi;
+  snapid_t follows;
+public:
+  C_MDS_openc_finish(MDS *m, MDRequestRef& r, CDentry *d, CInode *ni, snapid_t f) :
+    mds(m), mdr(r), dn(d), newi(ni), follows(f) {}
+  void finish(int r) {
+    assert(r == 0);
+
+    dn->pop_projected_linkage();
+
+    // dirty inode, dn, dir
+    newi->inode.version--;   // a bit hacky, see C_MDS_mknod_finish
+    newi->mark_dirty(newi->inode.version+1, mdr->ls);
+    newi->_mark_dirty_parent(mdr->ls, true);
+
+    mdr->apply();
+
+    mds->locker->share_inode_max_size(newi);
+
+    mds->mdcache->send_dentry_link(dn);
+
+    mds->balancer->hit_inode(mdr->now, newi, META_POP_IWR);
+
+    MClientReply *reply = new MClientReply(mdr->client_request, 0);
+    reply->set_extra_bl(mdr->reply_extra_bl);
+    mds->server->reply_request(mdr, reply);
+
+    assert(g_conf->mds_kill_openc_at != 1);
+  }
+};
+
+/* This function takes responsibility for the passed mdr*/
+void Server::handle_client_openc(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  client_t client = mdr->get_client();
+
+  dout(7) << "open w/ O_CREAT on " << req->get_filepath() << dendl;
+
+  int cmode = ceph_flags_to_mode(req->head.args.open.flags);
+  if (cmode < 0) {
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+
+  if (!(req->head.args.open.flags & O_EXCL)) {
+    int r = mdcache->path_traverse(mdr, NULL, NULL, req->get_filepath(),
+				   &mdr->dn[0], NULL, MDS_TRAVERSE_FORWARD);
+    if (r > 0) return;
+    if (r == 0) {
+      // it existed.
+      handle_client_open(mdr);
+      return;
+    }
+    if (r < 0 && r != -ENOENT) {
+      if (r == -ESTALE) {
+	dout(10) << "FAIL on ESTALE but attempting recovery" << dendl;
+	Context *c = new C_MDS_TryFindInode(this, mdr);
+	mdcache->find_ino_peers(req->get_filepath().get_ino(), c);
+      } else {
+	dout(10) << "FAIL on error " << r << dendl;
+	reply_request(mdr, r);
+      }
+      return;
+    }
+    // r == -ENOENT
+  }
+
+  bool excl = (req->head.args.open.flags & O_EXCL);
+  set rdlocks, wrlocks, xlocks;
+  ceph_file_layout *dir_layout = NULL;
+  CDentry *dn = rdlock_path_xlock_dentry(mdr, 0, rdlocks, wrlocks, xlocks,
+                                         !excl, false, false, &dir_layout);
+  if (!dn) return;
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  // set layout
+  ceph_file_layout layout;
+  if (dir_layout)
+    layout = *dir_layout;
+  else
+    layout = mds->mdcache->default_file_layout;
+
+  // fill in any special params from client
+  if (req->head.args.open.stripe_unit)
+    layout.fl_stripe_unit = req->head.args.open.stripe_unit;
+  if (req->head.args.open.stripe_count)
+    layout.fl_stripe_count = req->head.args.open.stripe_count;
+  if (req->head.args.open.object_size)
+    layout.fl_object_size = req->head.args.open.object_size;
+  if (req->get_connection()->has_feature(CEPH_FEATURE_CREATEPOOLID) &&
+      (__s32)req->head.args.open.pool >= 0) {
+    layout.fl_pg_pool = req->head.args.open.pool;
+
+    // make sure we have as new a map as the client
+    if (req->get_mdsmap_epoch() > mds->mdsmap->get_epoch()) {
+      mds->wait_for_mdsmap(req->get_mdsmap_epoch(), new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+  }
+
+  if (!ceph_file_layout_is_valid(&layout)) {
+    dout(10) << " invalid initial file layout" << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+  if (!mds->mdsmap->is_data_pool(layout.fl_pg_pool)) {
+    dout(10) << " invalid data pool " << layout.fl_pg_pool << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+
+  CInode *diri = dn->get_dir()->get_inode();
+  rdlocks.insert(&diri->authlock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  CDentry::linkage_t *dnl = dn->get_projected_linkage();
+
+  if (!dnl->is_null()) {
+    // it existed.  
+    assert(req->head.args.open.flags & O_EXCL);
+    dout(10) << "O_EXCL, target exists, failing with -EEXIST" << dendl;
+    reply_request(mdr, -EEXIST, dnl->get_inode(), dn);
+    return;
+  }
+
+  // created null dn.
+    
+  // create inode.
+  mdr->now = ceph_clock_now(g_ceph_context);
+
+  SnapRealm *realm = diri->find_snaprealm();   // use directory's realm; inode isn't attached yet.
+  snapid_t follows = realm->get_newest_seq();
+
+  CInode *in = prepare_new_inode(mdr, dn->get_dir(), inodeno_t(req->head.ino),
+				 req->head.args.open.mode | S_IFREG, &layout);
+  assert(in);
+
+  // it's a file.
+  dn->push_projected_linkage(in);
+
+  in->inode.version = dn->pre_dirty();
+  if (layout.fl_pg_pool != mdcache->default_file_layout.fl_pg_pool)
+    in->inode.add_old_pool(mdcache->default_file_layout.fl_pg_pool);
+  in->inode.update_backtrace();
+  if (cmode & CEPH_FILE_MODE_WR) {
+    in->inode.client_ranges[client].range.first = 0;
+    in->inode.client_ranges[client].range.last = in->inode.get_layout_size_increment();
+    in->inode.client_ranges[client].follows = follows;
+  }
+  in->inode.rstat.rfiles = 1;
+
+  if (follows >= dn->first)
+    dn->first = follows+1;
+  in->first = dn->first;
+  
+  // prepare finisher
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "openc");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  journal_allocated_inos(mdr, &le->metablob);
+  mdcache->predirty_journal_parents(mdr, &le->metablob, in, dn->get_dir(), PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
+  le->metablob.add_primary_dentry(dn, in, true, true, true);
+
+  // do the open
+  mds->locker->issue_new_caps(in, cmode, mdr->session, realm, req->is_replay());
+  in->authlock.set_state(LOCK_EXCL);
+  in->xattrlock.set_state(LOCK_EXCL);
+
+  // make sure this inode gets into the journal
+  le->metablob.add_opened_ino(in->ino());
+  LogSegment *ls = mds->mdlog->get_current_segment();
+  ls->open_files.push_back(&in->item_open_file);
+
+  C_MDS_openc_finish *fin = new C_MDS_openc_finish(mds, mdr, dn, in, follows);
+
+  if (mdr->client_request->get_connection()->has_feature(CEPH_FEATURE_REPLY_CREATE_INODE)) {
+    dout(10) << "adding ino to reply to indicate inode was created" << dendl;
+    // add the file created flag onto the reply if create_flags features is supported
+    ::encode(in->inode.ino, mdr->reply_extra_bl);
+  }
+
+  journal_and_reply(mdr, in, dn, le, fin);
+}
+
+
+
+void Server::handle_client_readdir(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  client_t client = req->get_source().num();
+  set rdlocks, wrlocks, xlocks;
+  CInode *diri = rdlock_path_pin_ref(mdr, 0, rdlocks, false, true);
+  if (!diri) return;
+
+  // it's a directory, right?
+  if (!diri->is_dir()) {
+    // not a dir
+    dout(10) << "reply to " << *req << " readdir -ENOTDIR" << dendl;
+    reply_request(mdr, -ENOTDIR);
+    return;
+  }
+
+  rdlocks.insert(&diri->filelock);
+  rdlocks.insert(&diri->dirfragtreelock);
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // which frag?
+  frag_t fg = (__u32)req->head.args.readdir.frag;
+  string offset_str = req->get_path2();
+  dout(10) << " frag " << fg << " offset '" << offset_str << "'" << dendl;
+
+  // does the frag exist?
+  if (diri->dirfragtree[fg.value()] != fg) {
+    frag_t newfg = diri->dirfragtree[fg.value()];
+    dout(10) << " adjust frag " << fg << " -> " << newfg << " " << diri->dirfragtree << dendl;
+    fg = newfg;
+    offset_str.clear();
+  }
+  
+  CDir *dir = try_open_auth_dirfrag(diri, fg, mdr);
+  if (!dir) return;
+
+  // ok!
+  dout(10) << "handle_client_readdir on " << *dir << dendl;
+  assert(dir->is_auth());
+
+  if (!dir->is_complete()) {
+    if (dir->is_frozen()) {
+      dout(7) << "dir is frozen " << *dir << dendl;
+      mds->locker->drop_locks(mdr.get());
+      mdr->drop_local_auth_pins();
+      dir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+    // fetch
+    dout(10) << " incomplete dir contents for readdir on " << *dir << ", fetching" << dendl;
+    dir->fetch(new C_MDS_RetryRequest(mdcache, mdr), true);
+    return;
+  }
+
+#ifdef MDS_VERIFY_FRAGSTAT
+  dir->verify_fragstat();
+#endif
+
+  mdr->now = ceph_clock_now(g_ceph_context);
+
+  snapid_t snapid = mdr->snapid;
+  dout(10) << "snapid " << snapid << dendl;
+
+  // purge stale snap data?
+  const set *snaps = 0;
+  SnapRealm *realm = diri->find_snaprealm();
+  if (realm->get_last_destroyed() > dir->fnode.snap_purged_thru) {
+    snaps = &realm->get_snaps();
+    dout(10) << " last_destroyed " << realm->get_last_destroyed() << " > " << dir->fnode.snap_purged_thru
+	     << ", doing snap purge with " << *snaps << dendl;
+    dir->fnode.snap_purged_thru = realm->get_last_destroyed();
+    assert(snapid == CEPH_NOSNAP || snaps->count(snapid));  // just checkin'! 
+  }
+
+  unsigned max = req->head.args.readdir.max_entries;
+  if (!max)
+    max = dir->get_num_any();  // whatever, something big.
+  unsigned max_bytes = req->head.args.readdir.max_bytes;
+  if (!max_bytes)
+    max_bytes = 512 << 10;  // 512 KB?
+
+  // start final blob
+  bufferlist dirbl;
+  dir->encode_dirstat(dirbl, mds->get_nodeid());
+
+  // count bytes available.
+  //  this isn't perfect, but we should capture the main variable/unbounded size items!
+  int front_bytes = dirbl.length() + sizeof(__u32) + sizeof(__u8)*2;
+  int bytes_left = max_bytes - front_bytes;
+  bytes_left -= realm->get_snap_trace().length();
+
+  // build dir contents
+  bufferlist dnbl;
+  __u32 numfiles = 0;
+  __u8 end = (dir->begin() == dir->end());
+  for (CDir::map_t::iterator it = dir->begin();
+       !end && numfiles < max;
+       end = (it == dir->end())) {
+    CDentry *dn = it->second;
+    ++it;
+
+    if (dn->state_test(CDentry::STATE_PURGING))
+      continue;
+
+    bool dnp = dn->use_projected(client, mdr);
+    CDentry::linkage_t *dnl = dnp ? dn->get_projected_linkage() : dn->get_linkage();
+
+    if (dnl->is_null())
+      continue;
+    if (snaps && dn->last != CEPH_NOSNAP)
+      if (dir->try_trim_snap_dentry(dn, *snaps))
+	continue;
+    if (dn->last < snapid || dn->first > snapid) {
+      dout(20) << "skipping non-overlapping snap " << *dn << dendl;
+      continue;
+    }
+
+    if (!offset_str.empty() && dn->get_name().compare(offset_str) <= 0)
+      continue;
+
+    CInode *in = dnl->get_inode();
+
+    if (in && in->ino() == CEPH_INO_CEPH)
+      continue;
+
+    // remote link?
+    // better for the MDS to do the work, if we think the client will stat any of these files.
+    if (dnl->is_remote() && !in) {
+      in = mdcache->get_inode(dnl->get_remote_ino());
+      if (in) {
+	dn->link_remote(dnl, in);
+      } else if (dn->state_test(CDentry::STATE_BADREMOTEINO)) {
+	dout(10) << "skipping bad remote ino on " << *dn << dendl;
+	continue;
+      } else {
+	// touch everything i _do_ have
+	for (CDir::map_t::iterator p = dir->begin(); p != dir->end(); ++p)
+	  if (!p->second->get_linkage()->is_null())
+	    mdcache->lru.lru_touch(p->second);
+
+	// already issued caps and leases, reply immediately.
+	if (dnbl.length() > 0) {
+	  mdcache->open_remote_dentry(dn, dnp, new C_NoopContext);
+	  dout(10) << " open remote dentry after caps were issued, stopping at "
+		   << dnbl.length() << " < " << bytes_left << dendl;
+	  break;
+	}
+
+	mds->locker->drop_locks(mdr.get());
+	mdr->drop_local_auth_pins();
+	mdcache->open_remote_dentry(dn, dnp, new C_MDS_RetryRequest(mdcache, mdr));
+	return;
+      }
+    }
+    assert(in);
+
+    if ((int)(dnbl.length() + dn->name.length() + sizeof(__u32) + sizeof(LeaseStat)) > bytes_left) {
+      dout(10) << " ran out of room, stopping at " << dnbl.length() << " < " << bytes_left << dendl;
+      break;
+    }
+    
+    unsigned start_len = dnbl.length();
+
+    // dentry
+    dout(12) << "including    dn " << *dn << dendl;
+    ::encode(dn->name, dnbl);
+    mds->locker->issue_client_lease(dn, client, dnbl, mdr->now, mdr->session);
+
+    // inode
+    dout(12) << "including inode " << *in << dendl;
+    int r = in->encode_inodestat(dnbl, mdr->session, realm, snapid, bytes_left - (int)dnbl.length());
+    if (r < 0) {
+      // chop off dn->name, lease
+      dout(10) << " ran out of room, stopping at " << start_len << " < " << bytes_left << dendl;
+      bufferlist keep;
+      keep.substr_of(dnbl, 0, start_len);
+      dnbl.swap(keep);
+      break;
+    }
+    assert(r >= 0);
+    numfiles++;
+
+    // touch dn
+    mdcache->lru.lru_touch(dn);
+  }
+  
+  __u8 complete = (end && offset_str.empty());  // FIXME: what purpose does this serve
+  
+  // finish final blob
+  ::encode(numfiles, dirbl);
+  ::encode(end, dirbl);
+  ::encode(complete, dirbl);
+  dirbl.claim_append(dnbl);
+  
+  if (snaps)
+    dir->log_mark_dirty();
+
+  // yay, reply
+  dout(10) << "reply to " << *req << " readdir num=" << numfiles
+	   << " bytes=" << dirbl.length()
+	   << " end=" << (int)end
+	   << " complete=" << (int)complete
+	   << dendl;
+  MClientReply *reply = new MClientReply(req, 0);
+  reply->set_extra_bl(dirbl);
+  dout(10) << "reply to " << *req << " readdir num=" << numfiles << " end=" << (int)end
+	   << " complete=" << (int)complete << dendl;
+
+  // bump popularity.  NOTE: this doesn't quite capture it.
+  mds->balancer->hit_dir(ceph_clock_now(g_ceph_context), dir, META_POP_IRD, -1, numfiles);
+  
+  // reply
+  reply_request(mdr, reply, diri);
+}
+
+
+
+// ===============================================================================
+// INODE UPDATES
+
+
+/* 
+ * finisher for basic inode updates
+ */
+class C_MDS_inode_update_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CInode *in;
+  bool truncating_smaller, changed_ranges;
+public:
+  C_MDS_inode_update_finish(MDS *m, MDRequestRef& r, CInode *i,
+			    bool sm=false, bool cr=false) :
+    mds(m), mdr(r), in(i), truncating_smaller(sm), changed_ranges(cr) { }
+  void finish(int r) {
+    assert(r == 0);
+
+    // apply
+    in->pop_and_dirty_projected_inode(mdr->ls);
+    mdr->apply();
+
+    // notify any clients
+    if (truncating_smaller && in->inode.is_truncating()) {
+      mds->locker->issue_truncate(in);
+      mds->mdcache->truncate_inode(in, mdr->ls);
+    }
+
+    mds->balancer->hit_inode(mdr->now, in, META_POP_IWR);   
+
+    mds->server->reply_request(mdr, 0);
+
+    if (changed_ranges)
+      mds->locker->share_inode_max_size(in);
+  }
+};
+
+void Server::handle_client_file_setlock(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+
+  // get the inode to operate on, and set up any locks needed for that
+  CInode *cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true);
+  if (!cur)
+    return;
+
+  xlocks.insert(&cur->flocklock);
+  /* acquire_locks will return true if it gets the locks. If it fails,
+     it will redeliver this request at a later date, so drop the request.
+   */
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks)) {
+    dout(0) << "handle_client_file_setlock could not get locks!" << dendl;
+    return;
+  }
+
+  // copy the lock change into a ceph_filelock so we can store/apply it
+  ceph_filelock set_lock;
+  set_lock.start = req->head.args.filelock_change.start;
+  set_lock.length = req->head.args.filelock_change.length;
+  set_lock.client = req->get_orig_source().num();
+  set_lock.owner = req->head.args.filelock_change.owner;
+  set_lock.pid = req->head.args.filelock_change.pid;
+  set_lock.type = req->head.args.filelock_change.type;
+  bool will_wait = req->head.args.filelock_change.wait;
+
+  dout(0) << "handle_client_file_setlock: " << set_lock << dendl;
+
+  ceph_lock_state_t *lock_state = NULL;
+
+  // get the appropriate lock state
+  switch (req->head.args.filelock_change.rule) {
+  case CEPH_LOCK_FLOCK:
+    lock_state = &cur->flock_locks;
+    break;
+
+  case CEPH_LOCK_FCNTL:
+    lock_state = &cur->fcntl_locks;
+    break;
+
+  default:
+    dout(0) << "got unknown lock type " << set_lock.type
+	    << ", dropping request!" << dendl;
+    return;
+  }
+
+  dout(10) << " state prior to lock change: " << *lock_state << dendl;;
+  if (CEPH_LOCK_UNLOCK == set_lock.type) {
+    list activated_locks;
+    list waiters;
+    if (lock_state->is_waiting(set_lock)) {
+      dout(10) << " unlock removing waiting lock " << set_lock << dendl;
+      lock_state->remove_waiting(set_lock);
+    } else {
+      dout(10) << " unlock attempt on " << set_lock << dendl;
+      lock_state->remove_lock(set_lock, activated_locks);
+      cur->take_waiting(CInode::WAIT_FLOCK, waiters);
+    }
+    reply_request(mdr, 0);
+    /* For now we're ignoring the activated locks because their responses
+     * will be sent when the lock comes up again in rotation by the MDS.
+     * It's a cheap hack, but it's easy to code. */
+    mds->queue_waiters(waiters);
+  } else {
+    dout(10) << " lock attempt on " << set_lock << dendl;
+    if (mdr->more()->flock_was_waiting &&
+	!lock_state->is_waiting(set_lock)) {
+      dout(10) << " was waiting for lock but not anymore, must have been canceled " << set_lock << dendl;
+      reply_request(mdr, -EINTR);
+    } else if (!lock_state->add_lock(set_lock, will_wait, mdr->more()->flock_was_waiting)) {
+      dout(10) << " it failed on this attempt" << dendl;
+      // couldn't set lock right now
+      if (!will_wait) {
+	reply_request(mdr, -EWOULDBLOCK);
+      } else {
+	dout(10) << " added to waiting list" << dendl;
+	assert(lock_state->is_waiting(set_lock));
+	mdr->more()->flock_was_waiting = true;
+	mds->locker->drop_locks(mdr.get());
+	mdr->drop_local_auth_pins();
+	cur->add_waiter(CInode::WAIT_FLOCK, new C_MDS_RetryRequest(mdcache, mdr));
+      }
+    } else
+      reply_request(mdr, 0);
+  }
+  dout(10) << " state after lock change: " << *lock_state << dendl;
+}
+
+void Server::handle_client_file_readlock(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+
+  // get the inode to operate on, and set up any locks needed for that
+  CInode *cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true);
+  if (!cur)
+    return;
+
+  /* acquire_locks will return true if it gets the locks. If it fails,
+     it will redeliver this request at a later date, so drop the request.
+  */
+  rdlocks.insert(&cur->flocklock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks)) {
+    dout(0) << "handle_client_file_readlock could not get locks!" << dendl;
+    return;
+  }
+  
+  // copy the lock change into a ceph_filelock so we can store/apply it
+  ceph_filelock checking_lock;
+  checking_lock.start = req->head.args.filelock_change.start;
+  checking_lock.length = req->head.args.filelock_change.length;
+  checking_lock.client = req->get_orig_source().num();
+  checking_lock.owner = req->head.args.filelock_change.owner;
+  checking_lock.pid = req->head.args.filelock_change.pid;
+  checking_lock.type = req->head.args.filelock_change.type;
+
+  // get the appropriate lock state
+  ceph_lock_state_t *lock_state = NULL;
+  switch (req->head.args.filelock_change.rule) {
+  case CEPH_LOCK_FLOCK:
+    lock_state = &cur->flock_locks;
+    break;
+
+  case CEPH_LOCK_FCNTL:
+    lock_state = &cur->fcntl_locks;
+    break;
+
+  default:
+    dout(0) << "got unknown lock type " << checking_lock.type
+	    << ", dropping request!" << dendl;
+    return;
+  }
+  lock_state->look_for_lock(checking_lock);
+
+  bufferlist lock_bl;
+  ::encode(checking_lock, lock_bl);
+
+  MClientReply *reply = new MClientReply(req);
+  reply->set_extra_bl(lock_bl);
+  reply_request(mdr, reply);
+}
+
+void Server::handle_client_setattr(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+  CInode *cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true);
+  if (!cur) return;
+
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  if (cur->ino() < MDS_INO_SYSTEM_BASE && !cur->is_base()) {
+    reply_request(mdr, -EPERM);
+    return;
+  }
+
+  __u32 mask = req->head.args.setattr.mask;
+
+  // xlock inode
+  if (mask & (CEPH_SETATTR_MODE|CEPH_SETATTR_UID|CEPH_SETATTR_GID))
+    xlocks.insert(&cur->authlock);
+  if (mask & (CEPH_SETATTR_MTIME|CEPH_SETATTR_ATIME|CEPH_SETATTR_SIZE))
+    xlocks.insert(&cur->filelock);
+  if (mask & CEPH_SETATTR_CTIME)
+    wrlocks.insert(&cur->versionlock);
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // trunc from bigger -> smaller?
+  inode_t *pi = cur->get_projected_inode();
+
+  uint64_t old_size = MAX(pi->size, req->head.args.setattr.old_size);
+  bool truncating_smaller = false;
+  if (mask & CEPH_SETATTR_SIZE) {
+    truncating_smaller = req->head.args.setattr.size < old_size;
+    if (truncating_smaller && pi->is_truncating()) {
+      dout(10) << " waiting for pending truncate from " << pi->truncate_from
+	       << " to " << pi->truncate_size << " to complete on " << *cur << dendl;
+      cur->add_waiter(CInode::WAIT_TRUNC, new C_MDS_RetryRequest(mdcache, mdr));
+      mds->mdlog->flush();
+      return;
+    }
+  }
+
+  bool changed_ranges = false;
+
+  // project update
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "setattr");
+  mdlog->start_entry(le);
+
+  pi = cur->project_inode();
+
+  utime_t now = ceph_clock_now(g_ceph_context);
+
+  if (mask & CEPH_SETATTR_MODE)
+    pi->mode = (pi->mode & ~07777) | (req->head.args.setattr.mode & 07777);
+  if (mask & CEPH_SETATTR_UID)
+    pi->uid = req->head.args.setattr.uid;
+  if (mask & CEPH_SETATTR_GID)
+    pi->gid = req->head.args.setattr.gid;
+
+  if (mask & CEPH_SETATTR_MTIME)
+    pi->mtime = req->head.args.setattr.mtime;
+  if (mask & CEPH_SETATTR_ATIME)
+    pi->atime = req->head.args.setattr.atime;
+  if (mask & (CEPH_SETATTR_ATIME | CEPH_SETATTR_MTIME))
+    pi->time_warp_seq++;   // maybe not a timewarp, but still a serialization point.
+  if (mask & CEPH_SETATTR_SIZE) {
+    if (truncating_smaller) {
+      pi->truncate(old_size, req->head.args.setattr.size);
+      le->metablob.add_truncate_start(cur->ino());
+    } else {
+      pi->size = req->head.args.setattr.size;
+      pi->rstat.rbytes = pi->size;
+    }
+    pi->mtime = now;
+
+    // adjust client's max_size?
+    map new_ranges;
+    mds->locker->calc_new_client_ranges(cur, pi->size, new_ranges);
+    if (pi->client_ranges != new_ranges) {
+      dout(10) << " client_ranges " << pi->client_ranges << " -> " << new_ranges << dendl;
+      pi->client_ranges = new_ranges;
+      changed_ranges = true;
+    }
+  }
+
+  pi->version = cur->pre_dirty();
+  pi->ctime = now;
+
+  // log + wait
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, cur);
+  
+  journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(mds, mdr, cur,
+								   truncating_smaller, changed_ranges));
+
+  // flush immediately if there are readers/writers waiting
+  if (cur->get_caps_wanted() & (CEPH_CAP_FILE_RD|CEPH_CAP_FILE_WR))
+    mds->mdlog->flush();
+}
+
+/* Takes responsibility for mdr */
+void Server::do_open_truncate(MDRequestRef& mdr, int cmode)
+{
+  CInode *in = mdr->in[0];
+  client_t client = mdr->get_client();
+  assert(in);
+
+  dout(10) << "do_open_truncate " << *in << dendl;
+
+  SnapRealm *realm = in->find_snaprealm();
+  mds->locker->issue_new_caps(in, cmode, mdr->session, realm, mdr->client_request->is_replay());
+
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "open_truncate");
+  mdlog->start_entry(le);
+
+  // prepare
+  inode_t *pi = in->project_inode();
+  pi->mtime = pi->ctime = ceph_clock_now(g_ceph_context);
+  pi->version = in->pre_dirty();
+
+  uint64_t old_size = MAX(pi->size, mdr->client_request->head.args.open.old_size);
+  if (old_size > 0) {
+    pi->truncate(old_size, 0);
+    le->metablob.add_truncate_start(in->ino());
+  }
+
+  bool changed_ranges = false;
+  if (cmode & CEPH_FILE_MODE_WR) {
+    pi->client_ranges[client].range.first = 0;
+    pi->client_ranges[client].range.last = pi->get_layout_size_increment();
+    pi->client_ranges[client].follows = in->find_snaprealm()->get_newest_seq();
+    changed_ranges = true;
+  }
+  
+  le->metablob.add_client_req(mdr->reqid, mdr->client_request->get_oldest_client_tid());
+
+  mdcache->predirty_journal_parents(mdr, &le->metablob, in, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, in);
+  
+  // make sure ino gets into the journal
+  le->metablob.add_opened_ino(in->ino());
+  LogSegment *ls = mds->mdlog->get_current_segment();
+  ls->open_files.push_back(&in->item_open_file);
+  
+  mdr->o_trunc = true;
+
+  CDentry *dn = 0;
+  if (mdr->client_request->get_dentry_wanted()) {
+    assert(mdr->dn[0].size());
+    dn = mdr->dn[0].back();
+  }
+
+  journal_and_reply(mdr, in, dn, le, new C_MDS_inode_update_finish(mds, mdr, in, old_size > 0,
+								   changed_ranges));
+}
+
+
+/* This function cleans up the passed mdr */
+void Server::handle_client_setlayout(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+  CInode *cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true);
+  if (!cur) return;
+
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  if (!cur->is_file()) {
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+  if (cur->get_projected_inode()->size ||
+      cur->get_projected_inode()->truncate_seq > 1) {
+    reply_request(mdr, -ENOTEMPTY);
+    return;
+  }
+
+  // validate layout
+  ceph_file_layout layout = cur->get_projected_inode()->layout;
+  // save existing layout for later
+  int64_t old_pool = layout.fl_pg_pool;
+
+  if (req->head.args.setlayout.layout.fl_object_size > 0)
+    layout.fl_object_size = req->head.args.setlayout.layout.fl_object_size;
+  if (req->head.args.setlayout.layout.fl_stripe_unit > 0)
+    layout.fl_stripe_unit = req->head.args.setlayout.layout.fl_stripe_unit;
+  if (req->head.args.setlayout.layout.fl_stripe_count > 0)
+    layout.fl_stripe_count=req->head.args.setlayout.layout.fl_stripe_count;
+  if (req->head.args.setlayout.layout.fl_cas_hash > 0)
+    layout.fl_cas_hash = req->head.args.setlayout.layout.fl_cas_hash;
+  if (req->head.args.setlayout.layout.fl_object_stripe_unit > 0)
+    layout.fl_object_stripe_unit = req->head.args.setlayout.layout.fl_object_stripe_unit;
+  if (req->head.args.setlayout.layout.fl_pg_pool > 0) {
+    layout.fl_pg_pool = req->head.args.setlayout.layout.fl_pg_pool;
+
+    // make sure we have as new a map as the client
+    if (req->get_mdsmap_epoch() > mds->mdsmap->get_epoch()) {
+      mds->wait_for_mdsmap(req->get_mdsmap_epoch(), new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+  }
+  if (!ceph_file_layout_is_valid(&layout)) {
+    dout(10) << "bad layout" << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+  if (!mds->mdsmap->is_data_pool(layout.fl_pg_pool)) {
+    dout(10) << " invalid data pool " << layout.fl_pg_pool << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+
+  xlocks.insert(&cur->filelock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // project update
+  inode_t *pi = cur->project_inode();
+  pi->layout = layout;
+  // add the old pool to the inode
+  pi->add_old_pool(old_pool);
+  pi->version = cur->pre_dirty();
+  pi->ctime = ceph_clock_now(g_ceph_context);
+  
+  // log + wait
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "setlayout");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, cur);
+  
+  journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(mds, mdr, cur));
+}
+
+void Server::handle_client_setdirlayout(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+  ceph_file_layout *dir_layout = NULL;
+  CInode *cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true, false, &dir_layout);
+  if (!cur) return;
+
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+
+  if (!cur->is_dir()) {
+    reply_request(mdr, -ENOTDIR);
+    return;
+  }
+
+  xlocks.insert(&cur->policylock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // validate layout
+  inode_t *pi = cur->get_projected_inode();
+  ceph_file_layout layout;
+  if (pi->has_layout())
+    layout = pi->layout;
+  else if (dir_layout)
+    layout = *dir_layout;
+  else
+    layout = mds->mdcache->default_file_layout;
+
+  if (req->head.args.setlayout.layout.fl_object_size > 0)
+    layout.fl_object_size = req->head.args.setlayout.layout.fl_object_size;
+  if (req->head.args.setlayout.layout.fl_stripe_unit > 0)
+    layout.fl_stripe_unit = req->head.args.setlayout.layout.fl_stripe_unit;
+  if (req->head.args.setlayout.layout.fl_stripe_count > 0)
+    layout.fl_stripe_count=req->head.args.setlayout.layout.fl_stripe_count;
+  if (req->head.args.setlayout.layout.fl_cas_hash > 0)
+    layout.fl_cas_hash = req->head.args.setlayout.layout.fl_cas_hash;
+  if (req->head.args.setlayout.layout.fl_object_stripe_unit > 0)
+    layout.fl_object_stripe_unit = req->head.args.setlayout.layout.fl_object_stripe_unit;
+  if (req->head.args.setlayout.layout.fl_pg_pool > 0) {
+    layout.fl_pg_pool = req->head.args.setlayout.layout.fl_pg_pool;
+    // make sure we have as new a map as the client
+    if (req->get_mdsmap_epoch() > mds->mdsmap->get_epoch()) {
+      mds->wait_for_mdsmap(req->get_mdsmap_epoch(), new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }  
+  }
+  if (!ceph_file_layout_is_valid(&layout)) {
+    dout(10) << "bad layout" << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+  if (!mds->mdsmap->is_data_pool(layout.fl_pg_pool)) {
+    dout(10) << " invalid data pool " << layout.fl_pg_pool << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+
+  pi = cur->project_inode();
+  pi->layout = layout;
+  pi->version = cur->pre_dirty();
+
+  // log + wait
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "setlayout");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, cur);
+
+  journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(mds, mdr, cur));
+}
+
+
+
+
+// XATTRS
+
+// parse a map of keys/values.
+namespace qi = boost::spirit::qi;
+
+template 
+struct keys_and_values
+  : qi::grammar()>
+{
+    keys_and_values()
+      : keys_and_values::base_type(query)
+    {
+      query =  pair >> *(qi::lit(' ') >> pair);
+      pair  =  key >> '=' >> value;
+      key   =  qi::char_("a-zA-Z_") >> *qi::char_("a-zA-Z_0-9");
+      value = +qi::char_("a-zA-Z_0-9");
+    }
+    qi::rule()> query;
+    qi::rule()> pair;
+    qi::rule key, value;
+};
+
+int Server::parse_layout_vxattr(string name, string value, ceph_file_layout *layout)
+{
+  dout(20) << "parse_layout_vxattr name " << name << " value '" << value << "'" << dendl;
+  try {
+    if (name == "layout") {
+      string::iterator begin = value.begin();
+      string::iterator end = value.end();
+      keys_and_values p;    // create instance of parser
+      std::map m;             // map to receive results
+      if (!qi::parse(begin, end, p, m)) {     // returns true if successful
+	return -EINVAL;
+      }
+      string left(begin, end);
+      dout(10) << " parsed " << m << " left '" << left << "'" << dendl;
+      if (begin != end)
+	return -EINVAL;
+      for (map::iterator q = m.begin(); q != m.end(); ++q) {
+	int r = parse_layout_vxattr(string("layout.") + q->first, q->second, layout);
+	if (r < 0)
+	  return r;
+      }
+    } else if (name == "layout.object_size") {
+      layout->fl_object_size = boost::lexical_cast(value);
+    } else if (name == "layout.stripe_unit") {
+      layout->fl_stripe_unit = boost::lexical_cast(value);
+    } else if (name == "layout.stripe_count") {
+      layout->fl_stripe_count = boost::lexical_cast(value);
+    } else if (name == "layout.pool") {
+      try {
+	layout->fl_pg_pool = boost::lexical_cast(value);
+      } catch (boost::bad_lexical_cast const&) {
+	int64_t pool = mds->osdmap->lookup_pg_pool_name(value);
+	if (pool < 0) {
+	  dout(10) << " unknown pool " << value << dendl;
+	  return -ENOENT;
+	}
+	layout->fl_pg_pool = pool;
+      }
+    } else {
+      dout(10) << " unknown layout vxattr " << name << dendl;
+      return -EINVAL;
+    }
+  } catch (boost::bad_lexical_cast const&) {
+    dout(10) << "bad vxattr value, unable to parse int for " << name << dendl;
+    return -EINVAL;
+  }
+
+  if (!ceph_file_layout_is_valid(layout)) {
+    dout(10) << "bad layout" << dendl;
+    return -EINVAL;
+  }
+  if (!mds->mdsmap->is_data_pool(layout->fl_pg_pool)) {
+    dout(10) << " invalid data pool " << layout->fl_pg_pool << dendl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
+void Server::handle_set_vxattr(MDRequestRef& mdr, CInode *cur,
+			       ceph_file_layout *dir_layout,
+			       set rdlocks,
+			       set wrlocks,
+			       set xlocks)
+{
+  MClientRequest *req = mdr->client_request;
+  string name(req->get_path2());
+  bufferlist bl = req->get_data();
+  string value (bl.c_str(), bl.length());
+  dout(10) << "handle_set_vxattr " << name << " val " << value.length() << " bytes on " << *cur << dendl;
+
+  // layout?
+  if (name.find("ceph.file.layout") == 0 ||
+      name.find("ceph.dir.layout") == 0) {
+    inode_t *pi;
+    string rest;
+    int64_t old_pool = -1;
+    if (name.find("ceph.dir.layout") == 0) {
+      if (!cur->is_dir()) {
+	reply_request(mdr, -EINVAL);
+	return;
+      }
+
+      ceph_file_layout layout;
+      if (cur->get_projected_inode()->has_layout())
+	layout = cur->get_projected_inode()->layout;
+      else if (dir_layout)
+	layout = *dir_layout;
+      else
+	layout = mds->mdcache->default_file_layout;
+
+      rest = name.substr(name.find("layout"));
+      int r = parse_layout_vxattr(rest, value, &layout);
+      if (r < 0) {
+	if (r == -ENOENT) {
+	  if (!mdr->waited_for_osdmap) {
+	    // send request to get latest map, but don't wait if
+	    // we don't get anything newer than what we have
+	    mdr->waited_for_osdmap = true;
+	    mds->request_osdmap(
+		  new C_MDS_RetryRequest(mdcache, mdr));
+	    return;
+	  }
+	  r = -EINVAL;
+	}
+	reply_request(mdr, r);
+	return;
+      }
+
+      xlocks.insert(&cur->policylock);
+      if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+	return;
+
+      pi = cur->project_inode();
+      cur->get_projected_inode()->layout = layout;
+    } else {
+      if (!cur->is_file()) {
+	reply_request(mdr, -EINVAL);
+	return;
+      }
+      ceph_file_layout layout = cur->get_projected_inode()->layout;
+      rest = name.substr(name.find("layout"));
+      int r = parse_layout_vxattr(rest, value, &layout);
+      if (r < 0) {
+	if (r == -ENOENT) {
+	  if (!mdr->waited_for_osdmap) {
+	    // send request to get latest map, but don't wait if
+	    // we don't get anything newer than what we have
+	    mdr->waited_for_osdmap = true;
+	    mds->request_osdmap(
+		  new C_MDS_RetryRequest(mdcache, mdr));
+	    return;
+	  }
+	  r = -EINVAL;
+	}
+	reply_request(mdr, r);
+	return;
+      }
+
+      xlocks.insert(&cur->filelock);
+      if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+	return;
+
+      pi = cur->project_inode();
+      old_pool = pi->layout.fl_pg_pool;
+      pi->add_old_pool(old_pool);
+      pi->layout = layout;
+      pi->ctime = ceph_clock_now(g_ceph_context);
+    }
+
+    pi->version = cur->pre_dirty();
+    if (cur->is_file())
+      pi->update_backtrace();
+
+    // log + wait
+    mdr->ls = mdlog->get_current_segment();
+    EUpdate *le = new EUpdate(mdlog, "set vxattr layout");
+    mdlog->start_entry(le);
+    le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+    mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
+    mdcache->journal_dirty_inode(mdr.get(), &le->metablob, cur);
+
+    journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(mds, mdr, cur));
+    return;
+  }
+
+  dout(10) << " unknown vxattr " << name << dendl;
+  reply_request(mdr, -EINVAL);
+}
+
+void Server::handle_remove_vxattr(MDRequestRef& mdr, CInode *cur,
+				  set rdlocks,
+				  set wrlocks,
+				  set xlocks)
+{
+  MClientRequest *req = mdr->client_request;
+  string name(req->get_path2());
+  if (name == "ceph.dir.layout") {
+    if (!cur->is_dir()) {
+      reply_request(mdr, -ENODATA);
+      return;
+    }
+    if (cur->is_root()) {
+      dout(10) << "can't remove layout policy on the root directory" << dendl;
+      reply_request(mdr, -EINVAL);
+      return;
+    }
+
+    if (!cur->get_projected_inode()->has_layout()) {
+      reply_request(mdr, -ENODATA);
+      return;
+    }
+
+    xlocks.insert(&cur->policylock);
+    if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+      return;
+
+    cur->project_inode();
+    cur->get_projected_inode()->clear_layout();
+    cur->get_projected_inode()->version = cur->pre_dirty();
+
+    // log + wait
+    mdr->ls = mdlog->get_current_segment();
+    EUpdate *le = new EUpdate(mdlog, "remove dir layout vxattr");
+    mdlog->start_entry(le);
+    le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+    mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
+    mdcache->journal_dirty_inode(mdr.get(), &le->metablob, cur);
+
+    journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(mds, mdr, cur));
+    return;
+  }
+
+  reply_request(mdr, -ENODATA);
+}
+
+class C_MDS_inode_xattr_update_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CInode *in;
+public:
+
+  C_MDS_inode_xattr_update_finish(MDS *m, MDRequestRef& r, CInode *i) :
+    mds(m), mdr(r), in(i) { }
+  void finish(int r) {
+    assert(r == 0);
+
+    // apply
+    in->pop_and_dirty_projected_inode(mdr->ls);
+    
+    mdr->apply();
+
+    mds->balancer->hit_inode(mdr->now, in, META_POP_IWR);   
+
+    mds->server->reply_request(mdr, 0);
+  }
+};
+
+void Server::handle_client_setxattr(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  string name(req->get_path2());
+  set rdlocks, wrlocks, xlocks;
+  CInode *cur;
+
+  ceph_file_layout *dir_layout = NULL;
+  if (name.find("ceph.dir.layout") == 0)
+    cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true, false, &dir_layout);
+  else
+    cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true);
+  if (!cur)
+    return;
+
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+
+  int flags = req->head.args.setxattr.flags;
+
+  // magic ceph.* namespace?
+  if (name.find("ceph.") == 0) {
+    handle_set_vxattr(mdr, cur, dir_layout, rdlocks, wrlocks, xlocks);
+    return;
+  }
+
+  xlocks.insert(&cur->xattrlock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  map *pxattrs = cur->get_projected_xattrs();
+  if ((flags & CEPH_XATTR_CREATE) && pxattrs->count(name)) {
+    dout(10) << "setxattr '" << name << "' XATTR_CREATE and EEXIST on " << *cur << dendl;
+    reply_request(mdr, -EEXIST);
+    return;
+  }
+  if ((flags & CEPH_XATTR_REPLACE) && !pxattrs->count(name)) {
+    dout(10) << "setxattr '" << name << "' XATTR_REPLACE and ENODATA on " << *cur << dendl;
+    reply_request(mdr, -ENODATA);
+    return;
+  }
+
+  int len = req->get_data().length();
+  dout(10) << "setxattr '" << name << "' len " << len << " on " << *cur << dendl;
+
+  // project update
+  map *px = new map;
+  inode_t *pi = cur->project_inode(px);
+  pi->version = cur->pre_dirty();
+  pi->ctime = ceph_clock_now(g_ceph_context);
+  pi->xattr_version++;
+  px->erase(name);
+  if (!(flags & CEPH_XATTR_REMOVE)) {
+    (*px)[name] = buffer::create(len);
+    if (len)
+      req->get_data().copy(0, len, (*px)[name].c_str());
+  }
+
+  // log + wait
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "setxattr");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, cur);
+
+  journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(mds, mdr, cur));
+}
+
+void Server::handle_client_removexattr(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  string name(req->get_path2());
+  set rdlocks, wrlocks, xlocks;
+  ceph_file_layout *dir_layout = NULL;
+  CInode *cur;
+  if (name == "ceph.dir.layout")
+    cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true, false, &dir_layout);
+  else
+    cur = rdlock_path_pin_ref(mdr, 0, rdlocks, true);
+  if (!cur)
+    return;
+
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+
+  if (name.find("ceph.") == 0) {
+    handle_remove_vxattr(mdr, cur, rdlocks, wrlocks, xlocks);
+    return;
+  }
+
+  xlocks.insert(&cur->xattrlock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  map *pxattrs = cur->get_projected_xattrs();
+  if (pxattrs->count(name) == 0) {
+    dout(10) << "removexattr '" << name << "' and ENODATA on " << *cur << dendl;
+    reply_request(mdr, -ENODATA);
+    return;
+  }
+
+  dout(10) << "removexattr '" << name << "' on " << *cur << dendl;
+
+  // project update
+  map *px = new map;
+  inode_t *pi = cur->project_inode(px);
+  pi->version = cur->pre_dirty();
+  pi->ctime = ceph_clock_now(g_ceph_context);
+  pi->xattr_version++;
+  px->erase(name);
+
+  // log + wait
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "removexattr");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, cur);
+
+  journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(mds, mdr, cur));
+}
+
+
+// =================================================================
+// DIRECTORY and NAMESPACE OPS
+
+
+// ------------------------------------------------
+
+// MKNOD
+
+class C_MDS_mknod_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CDentry *dn;
+  CInode *newi;
+  snapid_t follows;
+public:
+  C_MDS_mknod_finish(MDS *m, MDRequestRef& r, CDentry *d, CInode *ni, snapid_t f) :
+    mds(m), mdr(r), dn(d), newi(ni), follows(f) {}
+  void finish(int r) {
+    assert(r == 0);
+
+    // link the inode
+    dn->pop_projected_linkage();
+    
+    // be a bit hacky with the inode version, here.. we decrement it
+    // just to keep mark_dirty() happen. (we didn't bother projecting
+    // a new version of hte inode since it's just been created)
+    newi->inode.version--; 
+    newi->mark_dirty(newi->inode.version + 1, mdr->ls);
+    newi->_mark_dirty_parent(mdr->ls, true);
+
+    // mkdir?
+    if (newi->inode.is_dir()) { 
+      CDir *dir = newi->get_dirfrag(frag_t());
+      assert(dir);
+      dir->fnode.version--;
+      dir->mark_dirty(dir->fnode.version + 1, mdr->ls);
+      dir->mark_new(mdr->ls);
+    }
+
+    mdr->apply();
+
+    mds->mdcache->send_dentry_link(dn);
+
+    if (newi->inode.is_file())
+      mds->locker->share_inode_max_size(newi);
+
+    // hit pop
+    mds->balancer->hit_inode(mdr->now, newi, META_POP_IWR);
+
+    // reply
+    MClientReply *reply = new MClientReply(mdr->client_request, 0);
+    reply->set_result(0);
+    mds->server->reply_request(mdr, reply);
+  }
+};
+
+
+void Server::handle_client_mknod(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  client_t client = mdr->get_client();
+  set rdlocks, wrlocks, xlocks;
+  ceph_file_layout *dir_layout = NULL;
+  CDentry *dn = rdlock_path_xlock_dentry(mdr, 0, rdlocks, wrlocks, xlocks, false, false, false,
+					 &dir_layout);
+  if (!dn) return;
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  CInode *diri = dn->get_dir()->get_inode();
+  rdlocks.insert(&diri->authlock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  unsigned mode = req->head.args.mknod.mode;
+  if ((mode & S_IFMT) == 0)
+    mode |= S_IFREG;
+
+  // set layout
+  ceph_file_layout layout;
+  if (dir_layout && S_ISREG(mode))
+    layout = *dir_layout;
+  else
+    layout = mds->mdcache->default_file_layout;
+
+  SnapRealm *realm = dn->get_dir()->inode->find_snaprealm();
+  snapid_t follows = realm->get_newest_seq();
+  mdr->now = ceph_clock_now(g_ceph_context);
+
+  CInode *newi = prepare_new_inode(mdr, dn->get_dir(), inodeno_t(req->head.ino),
+				   mode, &layout);
+  assert(newi);
+
+  dn->push_projected_linkage(newi);
+
+  newi->inode.rdev = req->head.args.mknod.rdev;
+  newi->inode.version = dn->pre_dirty();
+  newi->inode.rstat.rfiles = 1;
+  if (layout.fl_pg_pool != mdcache->default_file_layout.fl_pg_pool)
+    newi->inode.add_old_pool(mdcache->default_file_layout.fl_pg_pool);
+  newi->inode.update_backtrace();
+
+  // if the client created a _regular_ file via MKNOD, it's highly likely they'll
+  // want to write to it (e.g., if they are reexporting NFS)
+  if (S_ISREG(newi->inode.mode)) {
+    dout(15) << " setting a client_range too, since this is a regular file" << dendl;
+    newi->inode.client_ranges[client].range.first = 0;
+    newi->inode.client_ranges[client].range.last = newi->inode.get_layout_size_increment();
+    newi->inode.client_ranges[client].follows = follows;
+
+    // issue a cap on the file
+    int cmode = CEPH_FILE_MODE_RDWR;
+    Capability *cap = mds->locker->issue_new_caps(newi, cmode, mdr->session, realm, req->is_replay());
+    if (cap) {
+      cap->set_wanted(0);
+
+      // put locks in excl mode
+      newi->filelock.set_state(LOCK_EXCL);
+      newi->authlock.set_state(LOCK_EXCL);
+      newi->xattrlock.set_state(LOCK_EXCL);
+      cap->issue_norevoke(CEPH_CAP_AUTH_EXCL|CEPH_CAP_AUTH_SHARED|
+			  CEPH_CAP_XATTR_EXCL|CEPH_CAP_XATTR_SHARED|
+			  CEPH_CAP_ANY_FILE_WR);
+    }
+  }
+
+  if (follows >= dn->first)
+    dn->first = follows + 1;
+  newi->first = dn->first;
+    
+  dout(10) << "mknod mode " << newi->inode.mode << " rdev " << newi->inode.rdev << dendl;
+
+  // prepare finisher
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "mknod");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  journal_allocated_inos(mdr, &le->metablob);
+  
+  mdcache->predirty_journal_parents(mdr, &le->metablob, newi, dn->get_dir(),
+				    PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
+  le->metablob.add_primary_dentry(dn, newi, true, true, true);
+
+  journal_and_reply(mdr, newi, dn, le, new C_MDS_mknod_finish(mds, mdr, dn, newi, follows));
+}
+
+
+
+// MKDIR
+/* This function takes responsibility for the passed mdr*/
+void Server::handle_client_mkdir(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+  CDentry *dn = rdlock_path_xlock_dentry(mdr, 0, rdlocks, wrlocks, xlocks, false, false, false);
+  if (!dn) return;
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  CInode *diri = dn->get_dir()->get_inode();
+  rdlocks.insert(&diri->authlock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // new inode
+  SnapRealm *realm = dn->get_dir()->inode->find_snaprealm();
+  snapid_t follows = realm->get_newest_seq();
+  mdr->now = ceph_clock_now(g_ceph_context);
+
+  unsigned mode = req->head.args.mkdir.mode;
+  mode &= ~S_IFMT;
+  mode |= S_IFDIR;
+  CInode *newi = prepare_new_inode(mdr, dn->get_dir(), inodeno_t(req->head.ino), mode);  
+  assert(newi);
+
+  // it's a directory.
+  dn->push_projected_linkage(newi);
+
+  newi->inode.version = dn->pre_dirty();
+  newi->inode.rstat.rsubdirs = 1;
+  newi->inode.update_backtrace();
+
+  dout(12) << " follows " << follows << dendl;
+  if (follows >= dn->first)
+    dn->first = follows + 1;
+  newi->first = dn->first;
+
+  // ...and that new dir is empty.
+  CDir *newdir = newi->get_or_open_dirfrag(mds->mdcache, frag_t());
+  newdir->mark_complete();
+  newdir->fnode.version = newdir->pre_dirty();
+
+  // prepare finisher
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "mkdir");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  journal_allocated_inos(mdr, &le->metablob);
+  mdcache->predirty_journal_parents(mdr, &le->metablob, newi, dn->get_dir(), PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
+  le->metablob.add_primary_dentry(dn, newi, true, true);
+  le->metablob.add_new_dir(newdir); // dirty AND complete AND new
+  
+  // issue a cap on the directory
+  int cmode = CEPH_FILE_MODE_RDWR;
+  Capability *cap = mds->locker->issue_new_caps(newi, cmode, mdr->session, realm, req->is_replay());
+  if (cap) {
+    cap->set_wanted(0);
+
+    // put locks in excl mode
+    newi->filelock.set_state(LOCK_EXCL);
+    newi->authlock.set_state(LOCK_EXCL);
+    newi->xattrlock.set_state(LOCK_EXCL);
+    cap->issue_norevoke(CEPH_CAP_AUTH_EXCL|CEPH_CAP_AUTH_SHARED|
+			CEPH_CAP_XATTR_EXCL|CEPH_CAP_XATTR_SHARED);
+  }
+
+  // make sure this inode gets into the journal
+  le->metablob.add_opened_ino(newi->ino());
+  LogSegment *ls = mds->mdlog->get_current_segment();
+  ls->open_files.push_back(&newi->item_open_file);
+
+  journal_and_reply(mdr, newi, dn, le, new C_MDS_mknod_finish(mds, mdr, dn, newi, follows));
+}
+
+
+// SYMLINK
+
+void Server::handle_client_symlink(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  set rdlocks, wrlocks, xlocks;
+  CDentry *dn = rdlock_path_xlock_dentry(mdr, 0, rdlocks, wrlocks, xlocks, false, false, false);
+  if (!dn) return;
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  CInode *diri = dn->get_dir()->get_inode();
+  rdlocks.insert(&diri->authlock);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  mdr->now = ceph_clock_now(g_ceph_context);
+  snapid_t follows = dn->get_dir()->inode->find_snaprealm()->get_newest_seq();
+
+  unsigned mode = S_IFLNK | 0777;
+  CInode *newi = prepare_new_inode(mdr, dn->get_dir(), inodeno_t(req->head.ino), mode);
+  assert(newi);
+
+  // it's a symlink
+  dn->push_projected_linkage(newi);
+
+  newi->symlink = req->get_path2();
+  newi->inode.size = newi->symlink.length();
+  newi->inode.rstat.rbytes = newi->inode.size;
+  newi->inode.rstat.rfiles = 1;
+  newi->inode.version = dn->pre_dirty();
+  newi->inode.update_backtrace();
+
+  if (follows >= dn->first)
+    dn->first = follows + 1;
+  newi->first = dn->first;
+
+  // prepare finisher
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "symlink");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  journal_allocated_inos(mdr, &le->metablob);
+  mdcache->predirty_journal_parents(mdr, &le->metablob, newi, dn->get_dir(), PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
+  le->metablob.add_primary_dentry(dn, newi, true, true);
+
+  journal_and_reply(mdr, newi, dn, le, new C_MDS_mknod_finish(mds, mdr, dn, newi, follows));
+}
+
+
+
+
+
+// LINK
+
+void Server::handle_client_link(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+
+  dout(7) << "handle_client_link " << req->get_filepath()
+	  << " to " << req->get_filepath2()
+	  << dendl;
+
+  set rdlocks, wrlocks, xlocks;
+
+  CDentry *dn = rdlock_path_xlock_dentry(mdr, 0, rdlocks, wrlocks, xlocks, false, false, false);
+  if (!dn) return;
+  CInode *targeti = rdlock_path_pin_ref(mdr, 1, rdlocks, false);
+  if (!targeti) return;
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+
+  CDir *dir = dn->get_dir();
+  dout(7) << "handle_client_link link " << dn->get_name() << " in " << *dir << dendl;
+  dout(7) << "target is " << *targeti << dendl;
+  if (targeti->is_dir()) {
+    dout(7) << "target is a dir, failing..." << dendl;
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+  
+  xlocks.insert(&targeti->linklock);
+
+  // take any locks needed for anchor creation/verification
+  // NOTE: we do this on the master even if the anchor/link update may happen
+  // on the slave.  That means we may have out of date anchor state on our
+  // end.  That's fine:  either, we xlock when we don't need to (slow but
+  // not a problem), or we rdlock when we need to xlock, but then discover we
+  // need to xlock and on our next pass through we adjust the locks (this works
+  // as long as the linklock rdlock isn't the very last lock we take).
+  mds->mdcache->anchor_create_prep_locks(mdr, targeti, rdlocks, xlocks);
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // pick mtime
+  if (mdr->now == utime_t())
+    mdr->now = ceph_clock_now(g_ceph_context);
+
+  // does the target need an anchor?
+  if (targeti->is_auth()) {
+    if (targeti->is_anchored()) {
+      dout(7) << "target anchored already (nlink=" << targeti->inode.nlink << "), sweet" << dendl;
+    } 
+    else {
+      dout(7) << "target needs anchor, nlink=" << targeti->inode.nlink << ", creating anchor" << dendl;
+      
+      mdcache->anchor_create(mdr, targeti,
+			     new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+  }
+
+  // go!
+  assert(g_conf->mds_kill_link_at != 1);
+
+  // local or remote?
+  if (targeti->is_auth()) 
+    _link_local(mdr, dn, targeti);
+  else 
+    _link_remote(mdr, true, dn, targeti);
+}
+
+
+class C_MDS_link_local_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CDentry *dn;
+  CInode *targeti;
+  version_t dnpv;
+  version_t tipv;
+public:
+  C_MDS_link_local_finish(MDS *m, MDRequestRef& r, CDentry *d, CInode *ti,
+			  version_t dnpv_, version_t tipv_) :
+    mds(m), mdr(r), dn(d), targeti(ti),
+    dnpv(dnpv_), tipv(tipv_) { }
+  void finish(int r) {
+    assert(r == 0);
+    mds->server->_link_local_finish(mdr, dn, targeti, dnpv, tipv);
+  }
+};
+
+
+void Server::_link_local(MDRequestRef& mdr, CDentry *dn, CInode *targeti)
+{
+  dout(10) << "_link_local " << *dn << " to " << *targeti << dendl;
+
+  mdr->ls = mdlog->get_current_segment();
+
+  // predirty NEW dentry
+  version_t dnpv = dn->pre_dirty();
+  version_t tipv = targeti->pre_dirty();
+  
+  // project inode update
+  inode_t *pi = targeti->project_inode();
+  pi->nlink++;
+  pi->ctime = mdr->now;
+  pi->version = tipv;
+
+  snapid_t follows = dn->get_dir()->inode->find_snaprealm()->get_newest_seq();
+  if (follows >= dn->first)
+    dn->first = follows;
+
+  // log + wait
+  EUpdate *le = new EUpdate(mdlog, "link_local");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(mdr->reqid, mdr->client_request->get_oldest_client_tid());
+  mdcache->predirty_journal_parents(mdr, &le->metablob, targeti, dn->get_dir(), PREDIRTY_DIR, 1);      // new dn
+  mdcache->predirty_journal_parents(mdr, &le->metablob, targeti, 0, PREDIRTY_PRIMARY);           // targeti
+  le->metablob.add_remote_dentry(dn, true, targeti->ino(), targeti->d_type());  // new remote
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, targeti);
+
+  // do this after predirty_*, to avoid funky extra dnl arg
+  dn->push_projected_linkage(targeti->ino(), targeti->d_type());
+
+  journal_and_reply(mdr, targeti, dn, le, new C_MDS_link_local_finish(mds, mdr, dn, targeti, dnpv, tipv));
+}
+
+void Server::_link_local_finish(MDRequestRef& mdr, CDentry *dn, CInode *targeti,
+				version_t dnpv, version_t tipv)
+{
+  dout(10) << "_link_local_finish " << *dn << " to " << *targeti << dendl;
+
+  // link and unlock the NEW dentry
+  dn->pop_projected_linkage();
+  dn->mark_dirty(dnpv, mdr->ls);
+
+  // target inode
+  targeti->pop_and_dirty_projected_inode(mdr->ls);
+
+  mdr->apply();
+  
+  mds->mdcache->send_dentry_link(dn);
+
+  // bump target popularity
+  mds->balancer->hit_inode(mdr->now, targeti, META_POP_IWR);
+  mds->balancer->hit_dir(mdr->now, dn->get_dir(), META_POP_IWR);
+
+  // reply
+  MClientReply *reply = new MClientReply(mdr->client_request, 0);
+  reply_request(mdr, reply);
+}
+
+
+// link / unlink remote
+
+class C_MDS_link_remote_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  bool inc;
+  CDentry *dn;
+  CInode *targeti;
+  version_t dpv;
+public:
+  C_MDS_link_remote_finish(MDS *m, MDRequestRef& r, bool i, CDentry *d, CInode *ti) :
+    mds(m), mdr(r), inc(i), dn(d), targeti(ti),
+    dpv(d->get_projected_version()) {}
+  void finish(int r) {
+    assert(r == 0);
+    mds->server->_link_remote_finish(mdr, inc, dn, targeti, dpv);
+  }
+};
+
+void Server::_link_remote(MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti)
+{
+  dout(10) << "_link_remote " 
+	   << (inc ? "link ":"unlink ")
+	   << *dn << " to " << *targeti << dendl;
+
+  // 1. send LinkPrepare to dest (journal nlink++ prepare)
+  int linkauth = targeti->authority().first;
+  if (mdr->more()->witnessed.count(linkauth) == 0) {
+    if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(linkauth)) {
+      dout(10) << " targeti auth mds." << linkauth << " is not active" << dendl;
+      if (mdr->more()->waiting_on_slave.empty())
+	mds->wait_for_active_peer(linkauth, new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+
+    dout(10) << " targeti auth must prepare nlink++/--" << dendl;
+    int op;
+    if (inc)
+      op = MMDSSlaveRequest::OP_LINKPREP;
+    else 
+      op = MMDSSlaveRequest::OP_UNLINKPREP;
+    MMDSSlaveRequest *req = new MMDSSlaveRequest(mdr->reqid, mdr->attempt, op);
+    targeti->set_object_info(req->get_object_info());
+    req->now = mdr->now;
+    mds->send_message_mds(req, linkauth);
+
+    assert(mdr->more()->waiting_on_slave.count(linkauth) == 0);
+    mdr->more()->waiting_on_slave.insert(linkauth);
+    return;
+  }
+  dout(10) << " targeti auth has prepared nlink++/--" << dendl;
+
+  assert(g_conf->mds_kill_link_at != 2);
+
+  // add to event
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, inc ? "link_remote":"unlink_remote");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(mdr->reqid, mdr->client_request->get_oldest_client_tid());
+  if (!mdr->more()->witnessed.empty()) {
+    dout(20) << " noting uncommitted_slaves " << mdr->more()->witnessed << dendl;
+    le->reqid = mdr->reqid;
+    le->had_slaves = true;
+    mds->mdcache->add_uncommitted_master(mdr->reqid, mdr->ls, mdr->more()->witnessed);
+  }
+
+  if (inc) {
+    dn->pre_dirty();
+    mdcache->predirty_journal_parents(mdr, &le->metablob, targeti, dn->get_dir(), PREDIRTY_DIR, 1);
+    le->metablob.add_remote_dentry(dn, true, targeti->ino(), targeti->d_type()); // new remote
+    dn->push_projected_linkage(targeti->ino(), targeti->d_type());
+  } else {
+    dn->pre_dirty();
+    mdcache->predirty_journal_parents(mdr, &le->metablob, targeti, dn->get_dir(), PREDIRTY_DIR, -1);
+    mdcache->journal_cow_dentry(mdr.get(), &le->metablob, dn);
+    le->metablob.add_null_dentry(dn, true);
+  }
+
+  if (mdr->more()->dst_reanchor_atid)
+    le->metablob.add_table_transaction(TABLE_ANCHOR, mdr->more()->dst_reanchor_atid);
+
+  journal_and_reply(mdr, targeti, dn, le, new C_MDS_link_remote_finish(mds, mdr, inc, dn, targeti));
+}
+
+void Server::_link_remote_finish(MDRequestRef& mdr, bool inc,
+				 CDentry *dn, CInode *targeti,
+				 version_t dpv)
+{
+  dout(10) << "_link_remote_finish "
+	   << (inc ? "link ":"unlink ")
+	   << *dn << " to " << *targeti << dendl;
+
+  assert(g_conf->mds_kill_link_at != 3);
+
+  if (!mdr->more()->witnessed.empty())
+    mdcache->logged_master_update(mdr->reqid);
+
+  if (inc) {
+    // link the new dentry
+    dn->pop_projected_linkage();
+    dn->mark_dirty(dpv, mdr->ls);
+  } else {
+    // unlink main dentry
+    dn->get_dir()->unlink_inode(dn);
+    dn->mark_dirty(dn->get_projected_version(), mdr->ls);  // dirty old dentry
+  }
+
+  mdr->apply();
+
+  if (inc)
+    mds->mdcache->send_dentry_link(dn);
+  else {
+    MDRequestRef null_ref;
+    mds->mdcache->send_dentry_unlink(dn, NULL, null_ref);
+  }
+  
+  // commit anchor update?
+  if (mdr->more()->dst_reanchor_atid) 
+    mds->anchorclient->commit(mdr->more()->dst_reanchor_atid, mdr->ls);
+
+  // bump target popularity
+  mds->balancer->hit_inode(mdr->now, targeti, META_POP_IWR);
+  mds->balancer->hit_dir(mdr->now, dn->get_dir(), META_POP_IWR);
+
+  // reply
+  MClientReply *reply = new MClientReply(mdr->client_request, 0);
+  reply_request(mdr, reply);
+
+  if (!inc)
+    // removing a new dn?
+    dn->get_dir()->try_remove_unlinked_dn(dn);
+}
+
+
+// remote linking/unlinking
+
+class C_MDS_SlaveLinkPrep : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  CInode *targeti;
+public:
+  C_MDS_SlaveLinkPrep(Server *s, MDRequestRef& r, CInode *t) :
+    server(s), mdr(r), targeti(t) { }
+  void finish(int r) {
+    assert(r == 0);
+    server->_logged_slave_link(mdr, targeti);
+  }
+};
+
+class C_MDS_SlaveLinkCommit : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  CInode *targeti;
+public:
+  C_MDS_SlaveLinkCommit(Server *s, MDRequestRef& r, CInode *t) :
+    server(s), mdr(r), targeti(t) { }
+  void finish(int r) {
+    server->_commit_slave_link(mdr, r, targeti);
+  }
+};
+
+/* This function DOES put the mdr->slave_request before returning*/
+void Server::handle_slave_link_prep(MDRequestRef& mdr)
+{
+  dout(10) << "handle_slave_link_prep " << *mdr 
+	   << " on " << mdr->slave_request->get_object_info() 
+	   << dendl;
+
+  assert(g_conf->mds_kill_link_at != 4);
+
+  CInode *targeti = mdcache->get_inode(mdr->slave_request->get_object_info().ino);
+  assert(targeti);
+  dout(10) << "targeti " << *targeti << dendl;
+  CDentry *dn = targeti->get_parent_dn();
+  CDentry::linkage_t *dnl = dn->get_linkage();
+  assert(dnl->is_primary());
+
+  mdr->now = mdr->slave_request->now;
+
+  mdr->auth_pin(targeti);
+
+  //assert(0);  // test hack: make sure master can handle a slave that fails to prepare...
+
+  // anchor?
+  if (mdr->slave_request->get_op() == MMDSSlaveRequest::OP_LINKPREP) {
+    
+    // NOTE: the master took any locks needed for anchor creation/verification.
+
+    if (targeti->is_anchored()) {
+      dout(7) << "target anchored already (nlink=" << targeti->inode.nlink << "), sweet" << dendl;
+    } 
+    else {
+      dout(7) << "target needs anchor, nlink=" << targeti->inode.nlink << ", creating anchor" << dendl;
+      mdcache->anchor_create(mdr, targeti,
+			     new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+  }
+
+  assert(g_conf->mds_kill_link_at != 5);
+
+  // journal it
+  mdr->ls = mdlog->get_current_segment();
+  ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_link_prep", mdr->reqid, mdr->slave_to_mds,
+				      ESlaveUpdate::OP_PREPARE, ESlaveUpdate::LINK);
+  mdlog->start_entry(le);
+
+  inode_t *pi = dnl->get_inode()->project_inode();
+
+  // update journaled target inode
+  bool inc;
+  if (mdr->slave_request->get_op() == MMDSSlaveRequest::OP_LINKPREP) {
+    inc = true;
+    pi->nlink++;
+  } else {
+    inc = false;
+    pi->nlink--;
+  }
+
+  link_rollback rollback;
+  rollback.reqid = mdr->reqid;
+  rollback.ino = targeti->ino();
+  rollback.old_ctime = targeti->inode.ctime;   // we hold versionlock xlock; no concorrent projections
+  fnode_t *pf = targeti->get_parent_dn()->get_dir()->get_projected_fnode();
+  rollback.old_dir_mtime = pf->fragstat.mtime;
+  rollback.old_dir_rctime = pf->rstat.rctime;
+  rollback.was_inc = inc;
+  ::encode(rollback, le->rollback);
+  mdr->more()->rollback_bl = le->rollback;
+
+  pi->ctime = mdr->now;
+  pi->version = targeti->pre_dirty();
+
+  dout(10) << " projected inode " << pi << " v " << pi->version << dendl;
+
+  // commit case
+  mdcache->predirty_journal_parents(mdr, &le->commit, dnl->get_inode(), 0, PREDIRTY_SHALLOW|PREDIRTY_PRIMARY, 0);
+  mdcache->journal_dirty_inode(mdr.get(), &le->commit, targeti);
+
+  // set up commit waiter
+  mdr->more()->slave_commit = new C_MDS_SlaveLinkCommit(this, mdr, targeti);
+
+  mdlog->submit_entry(le, new C_MDS_SlaveLinkPrep(this, mdr, targeti));
+  mdlog->flush();
+}
+
+void Server::_logged_slave_link(MDRequestRef& mdr, CInode *targeti)
+{
+  dout(10) << "_logged_slave_link " << *mdr
+	   << " " << *targeti << dendl;
+
+  assert(g_conf->mds_kill_link_at != 6);
+
+  // update the target
+  targeti->pop_and_dirty_projected_inode(mdr->ls);
+  mdr->apply();
+
+  // hit pop
+  mds->balancer->hit_inode(mdr->now, targeti, META_POP_IWR);
+
+  // done.
+  mdr->slave_request->put();
+  mdr->slave_request = 0;
+
+  // ack
+  if (!mdr->aborted) {
+    MMDSSlaveRequest *reply = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+						   MMDSSlaveRequest::OP_LINKPREPACK);
+    mds->send_message_mds(reply, mdr->slave_to_mds);
+  } else {
+    dout(10) << " abort flag set, finishing" << dendl;
+    mdcache->request_finish(mdr);
+  }
+}
+
+
+struct C_MDS_CommittedSlave : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  C_MDS_CommittedSlave(Server *s, MDRequestRef& m) : server(s), mdr(m) {}
+  void finish(int r) {
+    server->_committed_slave(mdr);
+  }
+};
+
+void Server::_commit_slave_link(MDRequestRef& mdr, int r, CInode *targeti)
+{  
+  dout(10) << "_commit_slave_link " << *mdr
+	   << " r=" << r
+	   << " " << *targeti << dendl;
+
+  assert(g_conf->mds_kill_link_at != 7);
+
+  if (r == 0) {
+    // drop our pins, etc.
+    mdr->cleanup();
+
+    // write a commit to the journal
+    ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_link_commit", mdr->reqid, mdr->slave_to_mds,
+					ESlaveUpdate::OP_COMMIT, ESlaveUpdate::LINK);
+    mdlog->start_submit_entry(le, new C_MDS_CommittedSlave(this, mdr));
+    mdlog->flush();
+  } else {
+    do_link_rollback(mdr->more()->rollback_bl, mdr->slave_to_mds, mdr);
+  }
+}
+
+void Server::_committed_slave(MDRequestRef& mdr)
+{
+  dout(10) << "_committed_slave " << *mdr << dendl;
+
+  assert(g_conf->mds_kill_link_at != 8);
+
+  MMDSSlaveRequest *req = new MMDSSlaveRequest(mdr->reqid, mdr->attempt, 
+					       MMDSSlaveRequest::OP_COMMITTED);
+  mds->send_message_mds(req, mdr->slave_to_mds);
+  mds->mdcache->request_finish(mdr);
+}
+
+struct C_MDS_LoggedLinkRollback : public Context {
+  Server *server;
+  MutationRef mut;
+  MDRequestRef mdr;
+  C_MDS_LoggedLinkRollback(Server *s, MutationRef& m, MDRequestRef& r) : server(s), mut(m), mdr(r) {}
+  void finish(int r) {
+    server->_link_rollback_finish(mut, mdr);
+  }
+};
+
+void Server::do_link_rollback(bufferlist &rbl, int master, MDRequestRef& mdr)
+{
+  link_rollback rollback;
+  bufferlist::iterator p = rbl.begin();
+  ::decode(rollback, p);
+
+  dout(10) << "do_link_rollback on " << rollback.reqid 
+	   << (rollback.was_inc ? " inc":" dec") 
+	   << " ino " << rollback.ino
+	   << dendl;
+
+  assert(g_conf->mds_kill_link_at != 9);
+
+  mds->mdcache->add_rollback(rollback.reqid, master); // need to finish this update before resolve finishes
+  assert(mdr || mds->is_resolve());
+
+  MutationRef mut(new MutationImpl(rollback.reqid));
+  mut->ls = mds->mdlog->get_current_segment();
+
+  CInode *in = mds->mdcache->get_inode(rollback.ino);
+  assert(in);
+  dout(10) << " target is " << *in << dendl;
+  assert(!in->is_projected());  // live slave request hold versionlock xlock.
+  
+  inode_t *pi = in->project_inode();
+  pi->version = in->pre_dirty();
+  mut->add_projected_inode(in);
+
+  // parent dir rctime
+  CDir *parent = in->get_projected_parent_dn()->get_dir();
+  fnode_t *pf = parent->project_fnode();
+  mut->add_projected_fnode(parent);
+  pf->version = parent->pre_dirty();
+  if (pf->fragstat.mtime == pi->ctime) {
+    pf->fragstat.mtime = rollback.old_dir_mtime;
+    if (pf->rstat.rctime == pi->ctime)
+      pf->rstat.rctime = rollback.old_dir_rctime;
+    mut->add_updated_lock(&parent->get_inode()->filelock);
+    mut->add_updated_lock(&parent->get_inode()->nestlock);
+  }
+
+  // inode
+  pi->ctime = rollback.old_ctime;
+  if (rollback.was_inc)
+    pi->nlink--;
+  else
+    pi->nlink++;
+
+  // journal it
+  ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_link_rollback", rollback.reqid, master,
+				      ESlaveUpdate::OP_ROLLBACK, ESlaveUpdate::LINK);
+  mdlog->start_entry(le);
+  le->commit.add_dir_context(parent);
+  le->commit.add_dir(parent, true);
+  le->commit.add_primary_dentry(in->get_projected_parent_dn(), 0, true);
+  
+  mdlog->submit_entry(le, new C_MDS_LoggedLinkRollback(this, mut, mdr));
+  mdlog->flush();
+}
+
+void Server::_link_rollback_finish(MutationRef& mut, MDRequestRef& mdr)
+{
+  dout(10) << "_link_rollback_finish" << dendl;
+
+  assert(g_conf->mds_kill_link_at != 10);
+
+  mut->apply();
+  if (mdr)
+    mds->mdcache->request_finish(mdr);
+
+  mds->mdcache->finish_rollback(mut->reqid);
+
+  mut->cleanup();
+}
+
+
+/* This function DOES NOT put the passed message before returning*/
+void Server::handle_slave_link_prep_ack(MDRequestRef& mdr, MMDSSlaveRequest *m)
+{
+  dout(10) << "handle_slave_link_prep_ack " << *mdr 
+	   << " " << *m << dendl;
+  int from = m->get_source().num();
+
+  assert(g_conf->mds_kill_link_at != 11);
+
+  // note slave
+  mdr->more()->slaves.insert(from);
+  
+  // witnessed!
+  assert(mdr->more()->witnessed.count(from) == 0);
+  mdr->more()->witnessed.insert(from);
+  
+  // remove from waiting list
+  assert(mdr->more()->waiting_on_slave.count(from));
+  mdr->more()->waiting_on_slave.erase(from);
+
+  assert(mdr->more()->waiting_on_slave.empty());
+
+  dispatch_client_request(mdr);  // go again!
+}
+
+
+
+
+
+// UNLINK
+
+void Server::handle_client_unlink(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  client_t client = mdr->get_client();
+
+  // rmdir or unlink?
+  bool rmdir = false;
+  if (req->get_op() == CEPH_MDS_OP_RMDIR) rmdir = true;
+
+  if (req->get_filepath().depth() == 0) {
+    reply_request(mdr, -EINVAL);
+    return;
+  }    
+
+  // traverse to path
+  vector trace;
+  CInode *in;
+  int r = mdcache->path_traverse(mdr, NULL, NULL, req->get_filepath(), &trace, &in, MDS_TRAVERSE_FORWARD);
+  if (r > 0) return;
+  if (r < 0) {
+    reply_request(mdr, r);
+    return;
+  }
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+
+  CDentry *dn = trace[trace.size()-1];
+  assert(dn);
+  if (!dn->is_auth()) {
+    mdcache->request_forward(mdr, dn->authority().first);
+    return;
+  }
+
+  CDentry::linkage_t *dnl = dn->get_linkage(client, mdr);
+  assert(!dnl->is_null());
+
+  if (rmdir) {
+    dout(7) << "handle_client_rmdir on " << *dn << dendl;
+  } else {
+    dout(7) << "handle_client_unlink on " << *dn << dendl;
+  }
+  dout(7) << "dn links to " << *in << dendl;
+
+  // rmdir vs is_dir 
+  if (in->is_dir()) {
+    if (rmdir) {
+      // do empty directory checks
+      if (_dir_is_nonempty_unlocked(mdr, in)) {
+	reply_request(mdr, -ENOTEMPTY);
+	return;
+      }
+    } else {
+      dout(7) << "handle_client_unlink on dir " << *in << ", returning error" << dendl;
+      reply_request(mdr, -EISDIR);
+      return;
+    }
+  } else {
+    if (rmdir) {
+      // unlink
+      dout(7) << "handle_client_rmdir on non-dir " << *in << ", returning error" << dendl;
+      reply_request(mdr, -ENOTDIR);
+      return;
+    }
+  }
+
+  // -- create stray dentry? --
+  CDentry *straydn = NULL;
+  if (dnl->is_primary()) {
+    straydn = prepare_stray_dentry(mdr, dnl->get_inode());
+    dout(10) << " straydn is " << *straydn << dendl;
+  } else if (mdr->straydn) {
+    mdr->unpin(mdr->straydn);
+    mdr->straydn = NULL;
+  }
+
+  // lock
+  set rdlocks, wrlocks, xlocks;
+
+  for (int i=0; i<(int)trace.size()-1; i++)
+    rdlocks.insert(&trace[i]->lock);
+  xlocks.insert(&dn->lock);
+  wrlocks.insert(&dn->get_dir()->inode->filelock);
+  wrlocks.insert(&dn->get_dir()->inode->nestlock);
+  xlocks.insert(&in->linklock);
+  if (straydn) {
+    wrlocks.insert(&straydn->get_dir()->inode->filelock);
+    wrlocks.insert(&straydn->get_dir()->inode->nestlock);
+    xlocks.insert(&straydn->lock);
+  }
+  if (in->is_dir())
+    rdlocks.insert(&in->filelock);   // to verify it's empty
+  mds->locker->include_snap_rdlocks(rdlocks, dnl->get_inode());
+
+  // if we unlink a snapped multiversion inode and are creating a
+  // remote link to it, it must be anchored.  this mirrors the logic
+  // in MDCache::journal_cow_dentry().
+  bool need_snap_dentry = 
+    dnl->is_primary() &&
+    in->is_multiversion() &&
+    in->find_snaprealm()->get_newest_seq() + 1 > dn->first;
+  if (need_snap_dentry) {
+    dout(10) << " i need to be anchored because i am multiversion and will get a remote cow dentry" << dendl;
+    mds->mdcache->anchor_create_prep_locks(mdr, in, rdlocks, xlocks);
+  }
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  if (in->is_dir() &&
+      _dir_is_nonempty(mdr, in)) {
+    reply_request(mdr, -ENOTEMPTY);
+    return;
+  }
+
+  // yay!
+  if (mdr->now == utime_t())
+    mdr->now = ceph_clock_now(g_ceph_context);
+
+  // NOTE: this is non-optimal.  we create an anchor at the old
+  // location, and then change it.  we can do better, but it's more
+  // complicated.  this is fine for now.
+  if (need_snap_dentry && !in->is_anchored()) {
+    mdcache->anchor_create(mdr, in, new C_MDS_RetryRequest(mdcache, mdr));
+    return;
+  }
+
+  // get stray dn ready?
+  if (dnl->is_primary()) {
+    if (!mdr->more()->dst_reanchor_atid && in->is_anchored()) {
+      dout(10) << "reanchoring to stray " << *dnl->get_inode() << dendl;
+      vector trace;
+      straydn->make_anchor_trace(trace, dnl->get_inode());
+      mds->anchorclient->prepare_update(dnl->get_inode()->ino(), trace, &mdr->more()->dst_reanchor_atid, 
+					new C_MDS_RetryRequest(mdcache, mdr));
+      return;
+    }
+  }
+
+  if (in->is_dir() && in->has_subtree_root_dirfrag()) {
+    // subtree root auths need to be witnesses
+    set witnesses;
+    list ls;
+    in->get_subtree_dirfrags(ls);
+    for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+      CDir *dir = *p;
+      int auth = dir->authority().first;
+      witnesses.insert(auth);
+      dout(10) << " need mds." << auth << " to witness for dirfrag " << *dir << dendl;      
+    } 
+    dout(10) << " witnesses " << witnesses << ", have " << mdr->more()->witnessed << dendl;
+
+    for (set::iterator p = witnesses.begin();
+	 p != witnesses.end();
+	 ++p) {
+      if (mdr->more()->witnessed.count(*p)) {
+	dout(10) << " already witnessed by mds." << *p << dendl;
+      } else if (mdr->more()->waiting_on_slave.count(*p)) {
+	dout(10) << " already waiting on witness mds." << *p << dendl;      
+      } else {
+	if (!_rmdir_prepare_witness(mdr, *p, dn, straydn))
+	  return;
+      }
+    }
+    if (!mdr->more()->waiting_on_slave.empty())
+      return;  // we're waiting for a witness.
+  }
+  
+  // ok!
+  if (dnl->is_remote() && !dnl->get_inode()->is_auth()) 
+    _link_remote(mdr, false, dn, dnl->get_inode());
+  else
+    _unlink_local(mdr, dn, straydn);
+}
+
+class C_MDS_unlink_local_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CDentry *dn;
+  CDentry *straydn;
+  version_t dnpv;  // deleted dentry
+public:
+  C_MDS_unlink_local_finish(MDS *m, MDRequestRef& r, CDentry *d, CDentry *sd) :
+    mds(m), mdr(r), dn(d), straydn(sd),
+    dnpv(d->get_projected_version()) {}
+  void finish(int r) {
+    assert(r == 0);
+    mds->server->_unlink_local_finish(mdr, dn, straydn, dnpv);
+  }
+};
+
+void Server::_unlink_local(MDRequestRef& mdr, CDentry *dn, CDentry *straydn)
+{
+  dout(10) << "_unlink_local " << *dn << dendl;
+
+  CDentry::linkage_t *dnl = dn->get_projected_linkage();
+  CInode *in = dnl->get_inode();
+
+  SnapRealm *realm = in->find_snaprealm();
+  snapid_t follows = realm->get_newest_seq();
+
+  // ok, let's do it.
+  mdr->ls = mdlog->get_current_segment();
+
+  // prepare log entry
+  EUpdate *le = new EUpdate(mdlog, "unlink_local");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(mdr->reqid, mdr->client_request->get_oldest_client_tid());
+  if (!mdr->more()->witnessed.empty()) {
+    dout(20) << " noting uncommitted_slaves " << mdr->more()->witnessed << dendl;
+    le->reqid = mdr->reqid;
+    le->had_slaves = true;
+    mds->mdcache->add_uncommitted_master(mdr->reqid, mdr->ls, mdr->more()->witnessed);
+  }
+
+  if (straydn) {
+    assert(dnl->is_primary());
+    straydn->push_projected_linkage(in);
+    straydn->first = follows + 1;
+  }
+
+  // the unlinked dentry
+  dn->pre_dirty();
+
+  inode_t *pi = in->project_inode();
+  mdr->add_projected_inode(in); // do this _after_ my dn->pre_dirty().. we apply that one manually.
+  pi->version = in->pre_dirty();
+  pi->ctime = mdr->now;
+  pi->nlink--;
+  if (pi->nlink == 0)
+    in->state_set(CInode::STATE_ORPHAN);
+
+  if (dnl->is_primary()) {
+    // primary link.  add stray dentry.
+    assert(straydn);
+    mdcache->predirty_journal_parents(mdr, &le->metablob, in, dn->get_dir(), PREDIRTY_PRIMARY|PREDIRTY_DIR, -1);
+    mdcache->predirty_journal_parents(mdr, &le->metablob, in, straydn->get_dir(), PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
+
+    // project snaprealm, too
+    if (in->snaprealm || follows + 1 > dn->first)
+      in->project_past_snaprealm_parent(straydn->get_dir()->inode->find_snaprealm());
+
+    pi->update_backtrace();
+    le->metablob.add_primary_dentry(straydn, in, true, true);
+  } else {
+    // remote link.  update remote inode.
+    mdcache->predirty_journal_parents(mdr, &le->metablob, in, dn->get_dir(), PREDIRTY_DIR, -1);
+    mdcache->predirty_journal_parents(mdr, &le->metablob, in, 0, PREDIRTY_PRIMARY);
+    mdcache->journal_dirty_inode(mdr.get(), &le->metablob, in);
+  }
+
+  mdcache->journal_cow_dentry(mdr.get(), &le->metablob, dn);
+  le->metablob.add_null_dentry(dn, true);
+
+  if (in->is_dir()) {
+    dout(10) << " noting renamed (unlinked) dir ino " << in->ino() << " in metablob" << dendl;
+    le->metablob.renamed_dirino = in->ino();
+  }
+
+  if (mdr->more()->dst_reanchor_atid)
+    le->metablob.add_table_transaction(TABLE_ANCHOR, mdr->more()->dst_reanchor_atid);
+
+  dn->push_projected_linkage();
+
+  if (in->is_dir()) {
+    assert(straydn);
+    mds->mdcache->project_subtree_rename(in, dn->get_dir(), straydn->get_dir());
+  }
+
+  journal_and_reply(mdr, 0, dn, le, new C_MDS_unlink_local_finish(mds, mdr, dn, straydn));
+}
+
+void Server::_unlink_local_finish(MDRequestRef& mdr,
+				  CDentry *dn, CDentry *straydn,
+				  version_t dnpv) 
+{
+  dout(10) << "_unlink_local_finish " << *dn << dendl;
+
+  if (!mdr->more()->witnessed.empty())
+    mdcache->logged_master_update(mdr->reqid);
+
+  // unlink main dentry
+  dn->get_dir()->unlink_inode(dn);
+  dn->pop_projected_linkage();
+
+  // relink as stray?  (i.e. was primary link?)
+  CDentry::linkage_t *straydnl = 0;
+
+  bool snap_is_new = false;
+  if (straydn) {
+    dout(20) << " straydn is " << *straydn << dendl;
+    straydnl = straydn->pop_projected_linkage();
+    
+    snap_is_new = straydnl->get_inode()->snaprealm ? true : false;
+    mdcache->touch_dentry_bottom(straydn);
+  }
+
+  dn->mark_dirty(dnpv, mdr->ls);
+  mdr->apply();
+
+  if (snap_is_new) //only new if straydnl exists
+    mdcache->do_realm_invalidate_and_update_notify(straydnl->get_inode(), CEPH_SNAP_OP_SPLIT, true);
+  
+  mds->mdcache->send_dentry_unlink(dn, straydn, mdr);
+  
+  // update subtree map?
+  if (straydn && straydnl->get_inode()->is_dir()) 
+    mdcache->adjust_subtree_after_rename(straydnl->get_inode(), dn->get_dir(), true);
+
+  // commit anchor update?
+  if (mdr->more()->dst_reanchor_atid) 
+    mds->anchorclient->commit(mdr->more()->dst_reanchor_atid, mdr->ls);
+
+  // bump pop
+  mds->balancer->hit_dir(mdr->now, dn->get_dir(), META_POP_IWR);
+
+  // reply
+  MClientReply *reply = new MClientReply(mdr->client_request, 0);
+  reply_request(mdr, reply);
+  
+  // clean up?
+  if (straydn)
+    mdcache->eval_stray(straydn);
+
+  // removing a new dn?
+  dn->get_dir()->try_remove_unlinked_dn(dn);
+}
+
+bool Server::_rmdir_prepare_witness(MDRequestRef& mdr, int who, CDentry *dn, CDentry *straydn)
+{
+  if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(who)) {
+    dout(10) << "_rmdir_prepare_witness mds." << who << " is not active" << dendl;
+    if (mdr->more()->waiting_on_slave.empty())
+      mds->wait_for_active_peer(who, new C_MDS_RetryRequest(mdcache, mdr));
+    return false;
+  }
+  
+  dout(10) << "_rmdir_prepare_witness mds." << who << dendl;
+  MMDSSlaveRequest *req = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+					       MMDSSlaveRequest::OP_RMDIRPREP);
+  dn->make_path(req->srcdnpath);
+  straydn->make_path(req->destdnpath);
+  req->now = mdr->now;
+  
+  mdcache->replicate_stray(straydn, who, req->stray);
+  
+  mds->send_message_mds(req, who);
+  
+  assert(mdr->more()->waiting_on_slave.count(who) == 0);
+  mdr->more()->waiting_on_slave.insert(who);
+  return true;
+}
+
+struct C_MDS_SlaveRmdirPrep : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  CDentry *dn, *straydn;
+  C_MDS_SlaveRmdirPrep(Server *s, MDRequestRef& r, CDentry *d, CDentry *st)
+    : server(s), mdr(r), dn(d), straydn(st) {}
+  void finish(int r) {
+    server->_logged_slave_rmdir(mdr, dn, straydn);
+  }
+};
+
+struct C_MDS_SlaveRmdirCommit : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  C_MDS_SlaveRmdirCommit(Server *s, MDRequestRef& r)
+    : server(s), mdr(r) { }
+  void finish(int r) {
+    server->_commit_slave_rmdir(mdr, r);
+  }
+};
+
+void Server::handle_slave_rmdir_prep(MDRequestRef& mdr)
+{
+  dout(10) << "handle_slave_rmdir_prep " << *mdr 
+	   << " " << mdr->slave_request->srcdnpath 
+	   << " to " << mdr->slave_request->destdnpath
+	   << dendl;
+
+  vector trace;
+  filepath srcpath(mdr->slave_request->srcdnpath);
+  dout(10) << " src " << srcpath << dendl;
+  CInode *in;
+  int r = mdcache->path_traverse(mdr, NULL, NULL, srcpath, &trace, &in, MDS_TRAVERSE_DISCOVERXLOCK);
+  assert(r == 0);
+  CDentry *dn = trace[trace.size()-1];
+  dout(10) << " dn " << *dn << dendl;
+  mdr->pin(dn);
+
+  assert(mdr->straydn);
+  CDentry *straydn = mdr->straydn;
+  dout(10) << " straydn " << *straydn << dendl;
+  
+  mdr->now = mdr->slave_request->now;
+
+  rmdir_rollback rollback;
+  rollback.reqid = mdr->reqid;
+  rollback.src_dir = dn->get_dir()->dirfrag();
+  rollback.src_dname = dn->name;
+  rollback.dest_dir = straydn->get_dir()->dirfrag();
+  rollback.dest_dname = straydn->name;
+  ::encode(rollback, mdr->more()->rollback_bl);
+  dout(20) << " rollback is " << mdr->more()->rollback_bl.length() << " bytes" << dendl;
+
+  straydn->push_projected_linkage(in);
+  dn->push_projected_linkage();
+
+  ESlaveUpdate *le =  new ESlaveUpdate(mdlog, "slave_rmdir", mdr->reqid, mdr->slave_to_mds,
+				       ESlaveUpdate::OP_PREPARE, ESlaveUpdate::RMDIR);
+  mdlog->start_entry(le);
+  le->rollback = mdr->more()->rollback_bl;
+
+  le->commit.add_dir_context(straydn->get_dir());
+  le->commit.add_primary_dentry(straydn, in, true);
+  // slave: no need to journal original dentry
+
+  dout(10) << " noting renamed (unlinked) dir ino " << in->ino() << " in metablob" << dendl;
+  le->commit.renamed_dirino = in->ino();
+
+  mds->mdcache->project_subtree_rename(in, dn->get_dir(), straydn->get_dir());
+
+  // set up commit waiter
+  mdr->more()->slave_commit = new C_MDS_SlaveRmdirCommit(this, mdr);
+
+  mdlog->submit_entry(le, new C_MDS_SlaveRmdirPrep(this, mdr, dn, straydn));
+  mdlog->flush();
+}
+
+void Server::_logged_slave_rmdir(MDRequestRef& mdr, CDentry *dn, CDentry *straydn)
+{
+  dout(10) << "_logged_slave_rmdir " << *mdr << " on " << *dn << dendl;
+
+  // update our cache now, so we are consistent with what is in the journal
+  // when we journal a subtree map
+  CInode *in = dn->get_linkage()->get_inode();
+  dn->get_dir()->unlink_inode(dn);
+  straydn->pop_projected_linkage();
+  dn->pop_projected_linkage();
+  mdcache->adjust_subtree_after_rename(in, dn->get_dir(), true);
+
+  // done.
+  mdr->slave_request->put();
+  mdr->slave_request = 0;
+  mdr->straydn = 0;
+
+  if (!mdr->aborted) {
+    MMDSSlaveRequest *reply = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+						   MMDSSlaveRequest::OP_RMDIRPREPACK);
+    mds->send_message_mds(reply, mdr->slave_to_mds);
+  } else {
+    dout(10) << " abort flag set, finishing" << dendl;
+    mdcache->request_finish(mdr);
+  }
+}
+
+void Server::handle_slave_rmdir_prep_ack(MDRequestRef& mdr, MMDSSlaveRequest *ack)
+{
+  dout(10) << "handle_slave_rmdir_prep_ack " << *mdr 
+	   << " " << *ack << dendl;
+
+  int from = ack->get_source().num();
+
+  mdr->more()->slaves.insert(from);
+  mdr->more()->witnessed.insert(from);
+
+  // remove from waiting list
+  assert(mdr->more()->waiting_on_slave.count(from));
+  mdr->more()->waiting_on_slave.erase(from);
+
+  if (mdr->more()->waiting_on_slave.empty())
+    dispatch_client_request(mdr);  // go again!
+  else 
+    dout(10) << "still waiting on slaves " << mdr->more()->waiting_on_slave << dendl;
+}
+
+void Server::_commit_slave_rmdir(MDRequestRef& mdr, int r)
+{
+  dout(10) << "_commit_slave_rmdir " << *mdr << " r=" << r << dendl;
+  
+  if (r == 0) {
+    // write a commit to the journal
+    ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_rmdir_commit", mdr->reqid, mdr->slave_to_mds,
+					ESlaveUpdate::OP_COMMIT, ESlaveUpdate::RMDIR);
+    mdlog->start_entry(le);
+    mdr->cleanup();
+
+    mdlog->submit_entry(le, new C_MDS_CommittedSlave(this, mdr));
+    mdlog->flush();
+  } else {
+    // abort
+    do_rmdir_rollback(mdr->more()->rollback_bl, mdr->slave_to_mds, mdr);
+  }
+}
+
+struct C_MDS_LoggedRmdirRollback : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  metareqid_t reqid;
+  CDentry *dn;
+  CDentry *straydn;
+  C_MDS_LoggedRmdirRollback(Server *s, MDRequestRef& m, metareqid_t mr, CDentry *d, CDentry *st)
+    : server(s), mdr(m), reqid(mr), dn(d), straydn(st) {}
+  void finish(int r) {
+    server->_rmdir_rollback_finish(mdr, reqid, dn, straydn);
+  }
+};
+
+void Server::do_rmdir_rollback(bufferlist &rbl, int master, MDRequestRef& mdr)
+{
+  // unlink the other rollback methods, the rmdir rollback is only
+  // needed to record the subtree changes in the journal for inode
+  // replicas who are auth for empty dirfrags.  no actual changes to
+  // the file system are taking place here, so there is no Mutation.
+
+  rmdir_rollback rollback;
+  bufferlist::iterator p = rbl.begin();
+  ::decode(rollback, p);
+  
+  dout(10) << "do_rmdir_rollback on " << rollback.reqid << dendl;
+  mds->mdcache->add_rollback(rollback.reqid, master); // need to finish this update before resolve finishes
+  assert(mdr || mds->is_resolve());
+
+  CDir *dir = mds->mdcache->get_dirfrag(rollback.src_dir);
+  if (!dir)
+    dir = mds->mdcache->get_dirfrag(rollback.src_dir.ino, rollback.src_dname);
+  assert(dir);
+  CDentry *dn = dir->lookup(rollback.src_dname);
+  assert(dn);
+  dout(10) << " dn " << *dn << dendl;
+  dir = mds->mdcache->get_dirfrag(rollback.dest_dir);
+  assert(dir);
+  CDentry *straydn = dir->lookup(rollback.dest_dname);
+  assert(straydn);
+  dout(10) << " straydn " << *dn << dendl;
+  CInode *in = straydn->get_linkage()->get_inode();
+
+  dn->push_projected_linkage(in);
+  straydn->push_projected_linkage();
+
+  ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_rmdir_rollback", rollback.reqid, master,
+				      ESlaveUpdate::OP_ROLLBACK, ESlaveUpdate::RMDIR);
+  mdlog->start_entry(le);
+  
+  le->commit.add_dir_context(dn->get_dir());
+  le->commit.add_primary_dentry(dn, in, true);
+  // slave: no need to journal straydn
+  
+  dout(10) << " noting renamed (unlinked) dir ino " << in->ino() << " in metablob" << dendl;
+  le->commit.renamed_dirino = in->ino();
+
+  mdcache->project_subtree_rename(in, straydn->get_dir(), dn->get_dir());
+
+  mdlog->submit_entry(le, new C_MDS_LoggedRmdirRollback(this, mdr, rollback.reqid, dn, straydn));
+  mdlog->flush();
+}
+
+void Server::_rmdir_rollback_finish(MDRequestRef& mdr, metareqid_t reqid, CDentry *dn, CDentry *straydn)
+{
+  dout(10) << "_rmdir_rollback_finish " << reqid << dendl;
+
+  straydn->get_dir()->unlink_inode(straydn);
+  dn->pop_projected_linkage();
+  straydn->pop_projected_linkage();
+
+  CInode *in = dn->get_linkage()->get_inode();
+  mdcache->adjust_subtree_after_rename(in, straydn->get_dir(), true);
+  if (mds->is_resolve()) {
+    CDir *root = mdcache->get_subtree_root(straydn->get_dir());
+    mdcache->try_trim_non_auth_subtree(root);
+  }
+
+  if (mdr)
+    mds->mdcache->request_finish(mdr);
+
+  mds->mdcache->finish_rollback(reqid);
+}
+
+
+/** _dir_is_nonempty[_unlocked]
+ *
+ * check if a directory is non-empty (i.e. we can rmdir it).
+ *
+ * the unlocked varient this is a fastpath check.  we can't really be
+ * sure until we rdlock the filelock.
+ */
+bool Server::_dir_is_nonempty_unlocked(MDRequestRef& mdr, CInode *in)
+{
+  dout(10) << "dir_is_nonempty_unlocked " << *in << dendl;
+  assert(in->is_auth());
+
+  if (in->snaprealm && in->snaprealm->srnode.snaps.size())
+    return true; // in a snapshot!
+
+  list ls;
+  in->get_dirfrags(ls);
+  for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+    CDir *dir = *p;
+    // is the frag obviously non-empty?
+    if (dir->is_auth()) {
+      if (dir->get_projected_fnode()->fragstat.size()) {
+	dout(10) << "dir_is_nonempty_unlocked dirstat has " 
+		 << dir->get_projected_fnode()->fragstat.size() << " items " << *dir << dendl;
+	return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+bool Server::_dir_is_nonempty(MDRequestRef& mdr, CInode *in)
+{
+  dout(10) << "dir_is_nonempty " << *in << dendl;
+  assert(in->is_auth());
+  assert(in->filelock.can_read(-1));
+
+  frag_info_t dirstat;
+  version_t dirstat_version = in->get_projected_inode()->dirstat.version;
+
+  list ls;
+  in->get_dirfrags(ls);
+  for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+    CDir *dir = *p;
+    fnode_t *pf = dir->get_projected_fnode();
+    if (pf->fragstat.size()) {
+      dout(10) << "dir_is_nonempty_unlocked dirstat has "
+	       << pf->fragstat.size() << " items " << *dir << dendl;
+      return true;
+    }
+
+    if (pf->accounted_fragstat.version == dirstat_version)
+      dirstat.add(pf->accounted_fragstat);
+    else
+      dirstat.add(pf->fragstat);
+  }
+
+  return dirstat.size() != in->get_projected_inode()->dirstat.size();
+}
+
+
+// ======================================================
+
+
+class C_MDS_rename_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CDentry *srcdn;
+  CDentry *destdn;
+  CDentry *straydn;
+public:
+  C_MDS_rename_finish(MDS *m, MDRequestRef& r,
+		      CDentry *sdn, CDentry *ddn, CDentry *stdn) :
+    mds(m), mdr(r),
+    srcdn(sdn), destdn(ddn), straydn(stdn) { }
+  void finish(int r) {
+    assert(r == 0);
+    mds->server->_rename_finish(mdr, srcdn, destdn, straydn);
+  }
+};
+
+
+/** handle_client_rename
+ *
+ * rename master is the destdn auth.  this is because cached inodes
+ * must remain connected.  thus, any replica of srci, must also
+ * replicate destdn, and possibly straydn, so that srci (and
+ * destdn->inode) remain connected during the rename.
+ *
+ * to do this, we freeze srci, then master (destdn auth) verifies that
+ * all other nodes have also replciated destdn and straydn.  note that
+ * destdn replicas need not also replicate srci.  this only works when 
+ * destdn is master.
+ *
+ * This function takes responsibility for the passed mdr.
+ */
+void Server::handle_client_rename(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+  dout(7) << "handle_client_rename " << *req << dendl;
+
+  filepath destpath = req->get_filepath();
+  filepath srcpath = req->get_filepath2();
+  if (destpath.depth() == 0 || srcpath.depth() == 0) {
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+  const string &destname = destpath.last_dentry();
+
+  vector& srctrace = mdr->dn[1];
+  vector& desttrace = mdr->dn[0];
+
+  set rdlocks, wrlocks, xlocks;
+
+  CDentry *destdn = rdlock_path_xlock_dentry(mdr, 0, rdlocks, wrlocks, xlocks, true, false, true);
+  if (!destdn) return;
+  dout(10) << " destdn " << *destdn << dendl;
+  if (mdr->snapid != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  CDentry::linkage_t *destdnl = destdn->get_projected_linkage();
+  CDir *destdir = destdn->get_dir();
+  assert(destdir->is_auth());
+
+  int r = mdcache->path_traverse(mdr, NULL, NULL, srcpath, &srctrace, NULL, MDS_TRAVERSE_DISCOVER);
+  if (r > 0)
+    return; // delayed
+  if (r < 0) {
+    if (r == -ESTALE) {
+      dout(10) << "FAIL on ESTALE but attempting recovery" << dendl;
+      Context *c = new C_MDS_TryFindInode(this, mdr);
+      mdcache->find_ino_peers(srcpath.get_ino(), c);
+    } else {
+      dout(10) << "FAIL on error " << r << dendl;
+      reply_request(mdr, r);
+    }
+    return;
+
+  }
+  assert(!srctrace.empty());
+  CDentry *srcdn = srctrace[srctrace.size()-1];
+  dout(10) << " srcdn " << *srcdn << dendl;
+  if (srcdn->last != CEPH_NOSNAP) {
+    reply_request(mdr, -EROFS);
+    return;
+  }
+  CDentry::linkage_t *srcdnl = srcdn->get_projected_linkage();
+  CInode *srci = srcdnl->get_inode();
+  dout(10) << " srci " << *srci << dendl;
+
+  CInode *oldin = 0;
+  if (!destdnl->is_null()) {
+    //dout(10) << "dest dn exists " << *destdn << dendl;
+    oldin = mdcache->get_dentry_inode(destdn, mdr, true);
+    if (!oldin) return;
+    dout(10) << " oldin " << *oldin << dendl;
+    
+    // mv /some/thing /to/some/existing_other_thing
+    if (oldin->is_dir() && !srci->is_dir()) {
+      reply_request(mdr, -EISDIR);
+      return;
+    }
+    if (!oldin->is_dir() && srci->is_dir()) {
+      reply_request(mdr, -ENOTDIR);
+      return;
+    }
+
+    // non-empty dir?
+    if (oldin->is_dir() && _dir_is_nonempty_unlocked(mdr, oldin)) {
+      reply_request(mdr, -ENOTEMPTY);
+      return;
+    }
+    if (srci == oldin && !srcdn->get_dir()->inode->is_stray()) {
+      reply_request(mdr, 0);  // no-op.  POSIX makes no sense.
+      return;
+    }
+  }
+
+  // -- some sanity checks --
+
+  // src+dest traces _must_ share a common ancestor for locking to prevent orphans
+  if (destpath.get_ino() != srcpath.get_ino() &&
+      !(req->get_source().is_mds() &&
+	MDS_INO_IS_MDSDIR(srcpath.get_ino()))) {  // <-- mds 'rename' out of stray dir is ok!
+    // do traces share a dentry?
+    CDentry *common = 0;
+    for (unsigned i=0; i < srctrace.size(); i++) {
+      for (unsigned j=0; j < desttrace.size(); j++) {
+	if (srctrace[i] == desttrace[j]) {
+	  common = srctrace[i];
+	  break;
+	}
+      }
+      if (common)
+	break;
+    }
+
+    if (common) {
+      dout(10) << "rename src and dest traces share common dentry " << *common << dendl;
+    } else {
+      CInode *srcbase = srctrace[0]->get_dir()->get_inode();
+      CInode *destbase = destdir->get_inode();
+      if (!desttrace.empty())
+	destbase = desttrace[0]->get_dir()->get_inode();
+
+      // ok, extend srctrace toward root until it is an ancestor of desttrace.
+      while (srcbase != destbase &&
+	     !srcbase->is_projected_ancestor_of(destbase)) {
+	srctrace.insert(srctrace.begin(),
+			srcbase->get_projected_parent_dn());
+	dout(10) << "rename prepending srctrace with " << *srctrace[0] << dendl;
+	srcbase = srcbase->get_projected_parent_dn()->get_dir()->get_inode();
+      }
+
+      // then, extend destpath until it shares the same parent inode as srcpath.
+      while (destbase != srcbase) {
+	desttrace.insert(desttrace.begin(),
+			 destbase->get_projected_parent_dn());
+	rdlocks.insert(&desttrace[0]->lock);
+	dout(10) << "rename prepending desttrace with " << *desttrace[0] << dendl;
+	destbase = destbase->get_projected_parent_dn()->get_dir()->get_inode();
+      }
+      dout(10) << "rename src and dest traces now share common ancestor " << *destbase << dendl;
+    }
+  }
+
+  // src == dest?
+  if (srcdn->get_dir() == destdir && srcdn->name == destname) {
+    dout(7) << "rename src=dest, noop" << dendl;
+    reply_request(mdr, 0);
+    return;
+  }
+
+  // dest a child of src?
+  // e.g. mv /usr /usr/foo
+  CDentry *pdn = destdir->inode->parent;
+  while (pdn) {
+    if (pdn == srcdn) {
+      dout(7) << "cannot rename item to be a child of itself" << dendl;
+      reply_request(mdr, -EINVAL);
+      return;
+    }
+    pdn = pdn->get_dir()->inode->parent;
+  }
+
+  // is this a stray migration, reintegration or merge? (sanity checks!)
+  if (mdr->reqid.name.is_mds() &&
+      !(MDS_INO_IS_MDSDIR(srcpath.get_ino()) &&
+	MDS_INO_IS_STRAY(destpath.get_ino())) &&
+      !(destdnl->is_remote() &&
+	destdnl->get_remote_ino() == srci->ino())) {
+    reply_request(mdr, -EINVAL);  // actually, this won't reply, but whatev.
+    return;
+  }
+
+  bool linkmerge = (srcdnl->get_inode() == destdnl->get_inode() &&
+		    (srcdnl->is_primary() || destdnl->is_primary()));
+  if (linkmerge)
+    dout(10) << " this is a link merge" << dendl;
+
+  // -- create stray dentry? --
+  CDentry *straydn = NULL;
+  if (destdnl->is_primary() && !linkmerge) {
+    straydn = prepare_stray_dentry(mdr, destdnl->get_inode());
+    dout(10) << " straydn is " << *straydn << dendl;
+  } else if (mdr->straydn) {
+    mdr->unpin(mdr->straydn);
+    mdr->straydn = NULL;
+  }
+
+  // -- prepare witness list --
+  /*
+   * NOTE: we use _all_ replicas as witnesses.
+   * this probably isn't totally necessary (esp for file renames),
+   * but if/when we change that, we have to make sure rejoin is
+   * sufficiently robust to handle strong rejoins from survivors
+   * with totally wrong dentry->inode linkage.
+   * (currently, it can ignore rename effects, because the resolve
+   * stage will sort them out.)
+   */
+  set witnesses = mdr->more()->extra_witnesses;
+  if (srcdn->is_auth())
+    srcdn->list_replicas(witnesses);
+  else
+    witnesses.insert(srcdn->authority().first);
+  destdn->list_replicas(witnesses);
+  if (destdnl->is_remote() && !oldin->is_auth())
+    witnesses.insert(oldin->authority().first);
+  dout(10) << " witnesses " << witnesses << ", have " << mdr->more()->witnessed << dendl;
+
+
+  // -- locks --
+  map remote_wrlocks;
+
+  // srctrace items.  this mirrors locks taken in rdlock_path_xlock_dentry
+  for (int i=0; i<(int)srctrace.size(); i++) 
+    rdlocks.insert(&srctrace[i]->lock);
+  xlocks.insert(&srcdn->lock);
+  int srcdirauth = srcdn->get_dir()->authority().first;
+  if (srcdirauth != mds->whoami) {
+    dout(10) << " will remote_wrlock srcdir scatterlocks on mds." << srcdirauth << dendl;
+    remote_wrlocks[&srcdn->get_dir()->inode->filelock] = srcdirauth;
+    remote_wrlocks[&srcdn->get_dir()->inode->nestlock] = srcdirauth;
+    if (srci->is_dir())
+      rdlocks.insert(&srci->dirfragtreelock);
+  } else {
+    wrlocks.insert(&srcdn->get_dir()->inode->filelock);
+    wrlocks.insert(&srcdn->get_dir()->inode->nestlock);
+  }
+  mds->locker->include_snap_rdlocks(rdlocks, srcdn->get_dir()->inode);
+
+  // straydn?
+  if (straydn) {
+    wrlocks.insert(&straydn->get_dir()->inode->filelock);
+    wrlocks.insert(&straydn->get_dir()->inode->nestlock);
+    xlocks.insert(&straydn->lock);
+  }
+
+  // xlock versionlock on dentries if there are witnesses.
+  //  replicas can't see projected dentry linkages, and will get
+  //  confused if we try to pipeline things.
+  if (!witnesses.empty()) {
+    // take xlock on all projected ancestor dentries for srcdn and destdn.
+    // this ensures the srcdn and destdn can be traversed to by the witnesses.
+    for (int i= 0; i<(int)srctrace.size(); i++) {
+      if (srctrace[i]->is_auth() && srctrace[i]->is_projected())
+	  xlocks.insert(&srctrace[i]->versionlock);
+    }
+    for (int i=0; i<(int)desttrace.size(); i++) {
+      if (desttrace[i]->is_auth() && desttrace[i]->is_projected())
+	  xlocks.insert(&desttrace[i]->versionlock);
+    }
+    // xlock srci and oldin's primary dentries, so witnesses can call
+    // open_remote_ino() with 'want_locked=true' when the srcdn or destdn
+    // is traversed.
+    if (srcdnl->is_remote())
+      xlocks.insert(&srci->get_projected_parent_dn()->lock);
+    if (destdnl->is_remote())
+      xlocks.insert(&oldin->get_projected_parent_dn()->lock);
+  }
+
+  // we need to update srci's ctime.  xlock its least contended lock to do that...
+  xlocks.insert(&srci->linklock);
+
+  // xlock oldin (for nlink--)
+  if (oldin) {
+    xlocks.insert(&oldin->linklock);
+    if (oldin->is_dir())
+      rdlocks.insert(&oldin->filelock);
+  }
+  if (srcdnl->is_primary() && srci->is_dir())  
+    // FIXME: this should happen whenever we are renamning between
+    // realms, regardless of the file type
+    // FIXME: If/when this changes, make sure to update the
+    // "allowance" in handle_slave_rename_prep
+    xlocks.insert(&srci->snaplock);  // FIXME: an auth bcast could be sufficient?
+  else
+    rdlocks.insert(&srci->snaplock);
+
+  // take any locks needed for anchor creation/verification
+  mds->mdcache->anchor_create_prep_locks(mdr, srci, rdlocks, xlocks);
+
+  CInode *auth_pin_freeze = !srcdn->is_auth() && srcdnl->is_primary() ? srci : NULL;
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks,
+				  &remote_wrlocks, auth_pin_freeze))
+    return;
+
+  if (oldin &&
+      oldin->is_dir() &&
+      _dir_is_nonempty(mdr, oldin)) {
+    reply_request(mdr, -ENOTEMPTY);
+    return;
+  }
+
+  // moving between snaprealms?
+  if (srcdnl->is_primary() && srci->is_multiversion() && !srci->snaprealm) {
+    SnapRealm *srcrealm = srci->find_snaprealm();
+    SnapRealm *destrealm = destdn->get_dir()->inode->find_snaprealm();
+    if (srcrealm != destrealm &&
+	(srcrealm->get_newest_seq() + 1 > srcdn->first ||
+	 destrealm->get_newest_seq() + 1 > srcdn->first)) {
+      dout(10) << " renaming between snaprealms, creating snaprealm for " << *srci << dendl;
+      mds->mdcache->snaprealm_create(mdr, srci);
+      return;
+    }
+  }
+
+  assert(g_conf->mds_kill_rename_at != 1);
+
+  // -- open all srcdn inode frags, if any --
+  // we need these open so that auth can properly delegate from inode to dirfrags
+  // after the inode is _ours_.
+  if (srcdnl->is_primary() && 
+      !srcdn->is_auth() && 
+      srci->is_dir()) {
+    dout(10) << "srci is remote dir, setting stickydirs and opening all frags" << dendl;
+    mdr->set_stickydirs(srci);
+
+    list frags;
+    srci->dirfragtree.get_leaves(frags);
+    for (list::iterator p = frags.begin();
+	 p != frags.end();
+	 ++p) {
+      CDir *dir = srci->get_dirfrag(*p);
+      if (!dir) {
+	dout(10) << " opening " << *p << " under " << *srci << dendl;
+	mdcache->open_remote_dirfrag(srci, *p, new C_MDS_RetryRequest(mdcache, mdr));
+	return;
+      }
+    }
+  }
+
+  // -- declare now --
+  if (mdr->now == utime_t())
+    mdr->now = ceph_clock_now(g_ceph_context);
+
+  // -- prepare anchor updates --
+  if (!linkmerge || srcdnl->is_primary()) {
+    C_GatherBuilder anchorgather(g_ceph_context);
+
+    if (srcdnl->is_primary() &&
+      (srcdnl->get_inode()->is_anchored() ||
+       (srcdnl->get_inode()->is_dir() && (srcdnl->get_inode()->inode.rstat.ranchors ||
+                                          srcdnl->get_inode()->nested_anchors ||
+                                          !mdcache->is_leaf_subtree(mdcache->get_projected_subtree_root(srcdn->get_dir()))))) &&
+      !mdr->more()->src_reanchor_atid) {
+      dout(10) << "reanchoring src->dst " << *srcdnl->get_inode() << dendl;
+      vector trace;
+      destdn->make_anchor_trace(trace, srcdnl->get_inode());
+      mds->anchorclient->prepare_update(srcdnl->get_inode()->ino(),
+					trace, &mdr->more()->src_reanchor_atid,
+					anchorgather.new_sub());
+    }
+    if (destdnl->is_primary() &&
+	destdnl->get_inode()->is_anchored() &&
+	!mdr->more()->dst_reanchor_atid) {
+      dout(10) << "reanchoring dst->stray " << *destdnl->get_inode() << dendl;
+
+      assert(straydn);
+      vector trace;
+      straydn->make_anchor_trace(trace, destdnl->get_inode());
+
+      mds->anchorclient->prepare_update(destdnl->get_inode()->ino(), trace,
+		  &mdr->more()->dst_reanchor_atid, anchorgather.new_sub());
+    }
+
+    if (anchorgather.has_subs())  {
+      anchorgather.set_finisher(new C_MDS_RetryRequest(mdcache, mdr));
+      anchorgather.activate();
+      return;  // waiting for anchor prepares
+    }
+
+    assert(g_conf->mds_kill_rename_at != 2);
+  }
+
+  // -- prepare witnesses --
+
+  // do srcdn auth last
+  int last = -1;
+  if (!srcdn->is_auth()) {
+    last = srcdn->authority().first;
+    mdr->more()->srcdn_auth_mds = last;
+    // ask auth of srci to mark srci as ambiguous auth if more than two MDS
+    // are involved in the rename operation.
+    if (srcdnl->is_primary() && !mdr->more()->is_ambiguous_auth) {
+      dout(10) << " preparing ambiguous auth for srci" << dendl;
+      mdr->set_ambiguous_auth(srci);
+      _rename_prepare_witness(mdr, last, witnesses, srcdn, destdn, straydn);
+      return;
+    }
+  }
+  
+  for (set::iterator p = witnesses.begin();
+       p != witnesses.end();
+       ++p) {
+    if (*p == last) continue;  // do it last!
+    if (mdr->more()->witnessed.count(*p)) {
+      dout(10) << " already witnessed by mds." << *p << dendl;
+    } else if (mdr->more()->waiting_on_slave.count(*p)) {
+      dout(10) << " already waiting on witness mds." << *p << dendl;      
+    } else {
+      if (!_rename_prepare_witness(mdr, *p, witnesses, srcdn, destdn, straydn))
+	return;
+    }
+  }
+  if (!mdr->more()->waiting_on_slave.empty())
+    return;  // we're waiting for a witness.
+
+  if (last >= 0 && mdr->more()->witnessed.count(last) == 0) {
+    dout(10) << " preparing last witness (srcdn auth)" << dendl;
+    assert(mdr->more()->waiting_on_slave.count(last) == 0);
+    _rename_prepare_witness(mdr, last, witnesses, srcdn, destdn, straydn);
+    return;
+  }
+
+  // test hack: bail after slave does prepare, so we can verify it's _live_ rollback.
+  if (!mdr->more()->slaves.empty() && !srci->is_dir())
+    assert(g_conf->mds_kill_rename_at != 3);
+  if (!mdr->more()->slaves.empty() && srci->is_dir())
+    assert(g_conf->mds_kill_rename_at != 4);
+
+  // -- prepare journal entry --
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "rename");
+  mdlog->start_entry(le);
+  le->metablob.add_client_req(mdr->reqid, mdr->client_request->get_oldest_client_tid());
+  if (!mdr->more()->witnessed.empty()) {
+    dout(20) << " noting uncommitted_slaves " << mdr->more()->witnessed << dendl;
+    
+    le->reqid = mdr->reqid;
+    le->had_slaves = true;
+    
+    mds->mdcache->add_uncommitted_master(mdr->reqid, mdr->ls, mdr->more()->witnessed);
+    // no need to send frozen auth pin to recovring auth MDS of srci
+    mdr->more()->is_remote_frozen_authpin = false;
+  }
+  
+  _rename_prepare(mdr, &le->metablob, &le->client_map, srcdn, destdn, straydn);
+  if (le->client_map.length())
+    le->cmapv = mds->sessionmap.projected;
+
+  // -- commit locally --
+  C_MDS_rename_finish *fin = new C_MDS_rename_finish(mds, mdr, srcdn, destdn, straydn);
+
+  journal_and_reply(mdr, srci, destdn, le, fin);
+}
+
+
+void Server::_rename_finish(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+{
+  dout(10) << "_rename_finish " << *mdr << dendl;
+
+  if (!mdr->more()->witnessed.empty())
+    mdcache->logged_master_update(mdr->reqid);
+
+  // apply
+  _rename_apply(mdr, srcdn, destdn, straydn);
+
+  CDentry::linkage_t *destdnl = destdn->get_linkage();
+  CInode *in = destdnl->get_inode();
+  bool need_eval = mdr->more()->cap_imports.count(in);
+
+  // test hack: test slave commit
+  if (!mdr->more()->slaves.empty() && !in->is_dir())
+    assert(g_conf->mds_kill_rename_at != 5);
+  if (!mdr->more()->slaves.empty() && in->is_dir())
+    assert(g_conf->mds_kill_rename_at != 6);
+  
+  // commit anchor updates?
+  if (mdr->more()->src_reanchor_atid) 
+    mds->anchorclient->commit(mdr->more()->src_reanchor_atid, mdr->ls);
+  if (mdr->more()->dst_reanchor_atid) 
+    mds->anchorclient->commit(mdr->more()->dst_reanchor_atid, mdr->ls);
+
+  // bump popularity
+  mds->balancer->hit_dir(mdr->now, srcdn->get_dir(), META_POP_IWR);
+  if (destdnl->is_remote() && in->is_auth())
+    mds->balancer->hit_inode(mdr->now, in, META_POP_IWR);
+
+  // did we import srci?  if so, explicitly ack that import that, before we unlock and reply.
+
+  assert(g_conf->mds_kill_rename_at != 7);
+
+  // reply
+  MClientReply *reply = new MClientReply(mdr->client_request, 0);
+  reply_request(mdr, reply);
+
+  if (need_eval)
+    mds->locker->eval(in, CEPH_CAP_LOCKS, true);
+
+  // clean up?
+  if (straydn) 
+    mdcache->eval_stray(straydn);
+}
+
+
+
+// helpers
+
+bool Server::_rename_prepare_witness(MDRequestRef& mdr, int who, set &witnesse,
+				     CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+{
+  if (!mds->mdsmap->is_clientreplay_or_active_or_stopping(who)) {
+    dout(10) << "_rename_prepare_witness mds." << who << " is not active" << dendl;
+    if (mdr->more()->waiting_on_slave.empty())
+      mds->wait_for_active_peer(who, new C_MDS_RetryRequest(mdcache, mdr));
+    return false;
+  }
+
+  dout(10) << "_rename_prepare_witness mds." << who << dendl;
+  MMDSSlaveRequest *req = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+					       MMDSSlaveRequest::OP_RENAMEPREP);
+  srcdn->make_path(req->srcdnpath);
+  destdn->make_path(req->destdnpath);
+  req->now = mdr->now;
+  
+  if (straydn)
+    mdcache->replicate_stray(straydn, who, req->stray);
+  
+  // srcdn auth will verify our current witness list is sufficient
+  req->witnesses = witnesse;
+
+  mds->send_message_mds(req, who);
+  
+  assert(mdr->more()->waiting_on_slave.count(who) == 0);
+  mdr->more()->waiting_on_slave.insert(who);
+  return true;
+}
+
+version_t Server::_rename_prepare_import(MDRequestRef& mdr, CDentry *srcdn, bufferlist *client_map_bl)
+{
+  version_t oldpv = mdr->more()->inode_import_v;
+
+  CDentry::linkage_t *srcdnl = srcdn->get_linkage();
+
+  /* import node */
+  bufferlist::iterator blp = mdr->more()->inode_import.begin();
+	  
+  // imported caps
+  ::decode(mdr->more()->imported_client_map, blp);
+  ::encode(mdr->more()->imported_client_map, *client_map_bl);
+  prepare_force_open_sessions(mdr->more()->imported_client_map, mdr->more()->sseq_map);
+
+  list updated_scatterlocks;  // we clear_updated explicitly below
+  mdcache->migrator->decode_import_inode(srcdn, blp, 
+					 srcdn->authority().first,
+					 mdr->ls, 0,
+					 mdr->more()->cap_imports, updated_scatterlocks);
+  srcdnl->get_inode()->filelock.remove_dirty();  
+  srcdnl->get_inode()->nestlock.remove_dirty();  
+
+  // hack: force back to !auth and clean, temporarily
+  srcdnl->get_inode()->state_clear(CInode::STATE_AUTH);
+  srcdnl->get_inode()->mark_clean();
+
+  return oldpv;
+}
+
+bool Server::_need_force_journal(CInode *diri, bool empty)
+{
+  list ls;
+  diri->get_dirfrags(ls);
+
+  bool force_journal = false;
+  if (empty) {
+    for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+      if ((*p)->is_subtree_root() && (*p)->get_dir_auth().first == mds->whoami) {
+	dout(10) << " frag " << (*p)->get_frag() << " is auth subtree dirfrag, will force journal" << dendl;
+	force_journal = true;
+	break;
+      } else
+	dout(20) << " frag " << (*p)->get_frag() << " is not auth subtree dirfrag" << dendl;
+    }
+  } else {
+    // see if any children of our frags are auth subtrees.
+    list subtrees;
+    mds->mdcache->list_subtrees(subtrees);
+    dout(10) << " subtrees " << subtrees << " frags " << ls << dendl;
+    for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+      CDir *dir = *p;
+      for (list::iterator q = subtrees.begin(); q != subtrees.end(); ++q) {
+	if (dir->contains(*q)) {
+	  if ((*q)->get_dir_auth().first == mds->whoami) {
+	    dout(10) << " frag " << (*p)->get_frag() << " contains (maybe) auth subtree, will force journal "
+		     << **q << dendl;
+	    force_journal = true;
+	    break;
+	  } else
+	    dout(20) << " frag " << (*p)->get_frag() << " contains but isn't auth for " << **q << dendl;
+	} else
+	  dout(20) << " frag " << (*p)->get_frag() << " does not contain " << **q << dendl;
+      }
+      if (force_journal)
+	break;
+    }
+  }
+  return force_journal;
+}
+
+void Server::_rename_prepare(MDRequestRef& mdr,
+			     EMetaBlob *metablob, bufferlist *client_map_bl,
+			     CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+{
+  dout(10) << "_rename_prepare " << *mdr << " " << *srcdn << " " << *destdn << dendl;
+  if (straydn)
+    dout(10) << " straydn " << *straydn << dendl;
+
+  CDentry::linkage_t *srcdnl = srcdn->get_projected_linkage();
+  CDentry::linkage_t *destdnl = destdn->get_projected_linkage();
+  CInode *srci = srcdnl->get_inode();
+  CInode *oldin = destdnl->get_inode();
+
+  // primary+remote link merge?
+  bool linkmerge = (srci == destdnl->get_inode() &&
+		    (srcdnl->is_primary() || destdnl->is_primary()));
+  bool silent = srcdn->get_dir()->inode->is_stray();
+
+  bool force_journal_dest = false;
+  if (srci->is_dir() && !destdn->is_auth()) {
+    if (srci->is_auth()) {
+      // if we are auth for srci and exporting it, force journal because journal replay needs
+      // the source inode to create auth subtrees.
+      dout(10) << " we are exporting srci, will force journal destdn" << dendl;
+      force_journal_dest = true;
+    } else
+      force_journal_dest = _need_force_journal(srci, false);
+  }
+
+  bool force_journal_stray = false;
+  if (oldin && oldin->is_dir() && straydn && !straydn->is_auth())
+    force_journal_stray = _need_force_journal(oldin, true);
+
+  if (linkmerge)
+    dout(10) << " merging remote and primary links to the same inode" << dendl;
+  if (silent)
+    dout(10) << " reintegrating stray; will avoid changing nlink or dir mtime" << dendl;
+  if (force_journal_dest)
+    dout(10) << " forcing journal destdn because we (will) have auth subtrees nested beneath it" << dendl;
+  if (force_journal_stray)
+    dout(10) << " forcing journal straydn because we (will) have auth subtrees nested beneath it" << dendl;
+
+  if (srci->is_dir() && (destdn->is_auth() || force_journal_dest)) {
+    dout(10) << " noting renamed dir ino " << srci->ino() << " in metablob" << dendl;
+    metablob->renamed_dirino = srci->ino();
+  } else if (oldin && oldin->is_dir() && force_journal_stray) {
+    dout(10) << " noting rename target dir " << oldin->ino() << " in metablob" << dendl;
+    metablob->renamed_dirino = oldin->ino();
+  }
+
+  // prepare
+  inode_t *pi = 0;    // renamed inode
+  inode_t *tpi = 0;  // target/overwritten inode
+  
+  // target inode
+  if (!linkmerge) {
+    if (destdnl->is_primary()) {
+      assert(straydn);  // moving to straydn.
+      // link--, and move.
+      if (destdn->is_auth()) {
+	tpi = oldin->project_inode(); //project_snaprealm
+	tpi->version = straydn->pre_dirty(tpi->version);
+	tpi->update_backtrace();
+      }
+      straydn->push_projected_linkage(oldin);
+    } else if (destdnl->is_remote()) {
+      // nlink-- targeti
+      if (oldin->is_auth()) {
+	tpi = oldin->project_inode();
+	tpi->version = oldin->pre_dirty();
+      }
+    }
+  }
+
+  // dest
+  if (srcdnl->is_remote()) {
+    if (!linkmerge) {
+      // destdn
+      if (destdn->is_auth())
+	mdr->more()->pvmap[destdn] = destdn->pre_dirty();
+      destdn->push_projected_linkage(srcdnl->get_remote_ino(), srcdnl->get_remote_d_type());
+      // srci
+      if (srci->is_auth()) {
+	pi = srci->project_inode();
+	pi->version = srci->pre_dirty();
+      }
+    } else {
+      dout(10) << " will merge remote onto primary link" << dendl;
+      if (destdn->is_auth()) {
+	pi = oldin->project_inode();
+	pi->version = mdr->more()->pvmap[destdn] = destdn->pre_dirty(oldin->inode.version);
+      }
+    }
+  } else { // primary
+    if (destdn->is_auth()) {
+      version_t oldpv;
+      if (srcdn->is_auth())
+	oldpv = srci->get_projected_version();
+      else {
+	oldpv = _rename_prepare_import(mdr, srcdn, client_map_bl);
+
+	// note which dirfrags have child subtrees in the journal
+	// event, so that we can open those (as bounds) during replay.
+	if (srci->is_dir()) {
+	  list ls;
+	  srci->get_dirfrags(ls);
+	  for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+	    CDir *dir = *p;
+	    if (!dir->is_auth())
+	      metablob->renamed_dir_frags.push_back(dir->get_frag());
+	  }
+	  dout(10) << " noting renamed dir open frags " << metablob->renamed_dir_frags << dendl;
+	}
+      }
+      pi = srci->project_inode(); // project snaprealm if srcdnl->is_primary
+                                                 // & srcdnl->snaprealm
+      pi->version = mdr->more()->pvmap[destdn] = destdn->pre_dirty(oldpv);
+      pi->update_backtrace();
+    }
+    destdn->push_projected_linkage(srci);
+  }
+
+  // src
+  if (srcdn->is_auth())
+    mdr->more()->pvmap[srcdn] = srcdn->pre_dirty();
+  srcdn->push_projected_linkage();  // push null linkage
+
+  if (!silent) {
+    if (pi) {
+      pi->ctime = mdr->now;
+      if (linkmerge)
+	pi->nlink--;
+    }
+    if (tpi) {
+      tpi->ctime = mdr->now;
+      tpi->nlink--;
+      if (tpi->nlink == 0)
+	oldin->state_set(CInode::STATE_ORPHAN);
+    }
+  }
+
+  // prepare nesting, mtime updates
+  int predirty_dir = silent ? 0:PREDIRTY_DIR;
+  
+  // guarantee stray dir is processed first during journal replay. unlink the old inode,
+  // then link the source inode to destdn
+  if (destdnl->is_primary()) {
+    assert(straydn);
+    if (straydn->is_auth()) {
+      metablob->add_dir_context(straydn->get_dir());
+      metablob->add_dir(straydn->get_dir(), true);
+    }
+  }
+
+  // sub off target
+  if (destdn->is_auth() && !destdnl->is_null()) {
+    mdcache->predirty_journal_parents(mdr, metablob, oldin, destdn->get_dir(),
+				      (destdnl->is_primary() ? PREDIRTY_PRIMARY:0)|predirty_dir, -1);
+    if (destdnl->is_primary())
+      mdcache->predirty_journal_parents(mdr, metablob, oldin, straydn->get_dir(),
+					PREDIRTY_PRIMARY|PREDIRTY_DIR, 1);
+  }
+  
+  // move srcdn
+  int predirty_primary = (srcdnl->is_primary() && srcdn->get_dir() != destdn->get_dir()) ? PREDIRTY_PRIMARY:0;
+  int flags = predirty_dir | predirty_primary;
+  if (srcdn->is_auth())
+    mdcache->predirty_journal_parents(mdr, metablob, srci, srcdn->get_dir(), PREDIRTY_SHALLOW|flags, -1);
+  if (destdn->is_auth())
+    mdcache->predirty_journal_parents(mdr, metablob, srci, destdn->get_dir(), flags, 1);
+
+  SnapRealm *src_realm = srci->find_snaprealm();
+  SnapRealm *dest_realm = destdn->get_dir()->inode->find_snaprealm();
+  snapid_t next_dest_snap = dest_realm->get_newest_seq() + 1;
+
+  // add it all to the metablob
+  // target inode
+  if (!linkmerge) {
+    if (destdnl->is_primary()) {
+      if (destdn->is_auth()) {
+	// project snaprealm, too
+	if (oldin->snaprealm || src_realm->get_newest_seq() + 1 > srcdn->first)
+	  oldin->project_past_snaprealm_parent(straydn->get_dir()->inode->find_snaprealm());
+	straydn->first = MAX(oldin->first, next_dest_snap);
+	metablob->add_primary_dentry(straydn, oldin, true, true);
+      } else if (force_journal_stray) {
+	dout(10) << " forced journaling straydn " << *straydn << dendl;
+	metablob->add_dir_context(straydn->get_dir());
+	metablob->add_primary_dentry(straydn, oldin, true);
+      }
+    } else if (destdnl->is_remote()) {
+      if (oldin->is_auth()) {
+	// auth for targeti
+	metablob->add_dir_context(oldin->get_projected_parent_dir());
+	mdcache->journal_cow_dentry(mdr.get(), metablob, oldin->get_projected_parent_dn(),
+				    CEPH_NOSNAP, 0, destdnl);
+	metablob->add_primary_dentry(oldin->get_projected_parent_dn(), oldin, true);
+      }
+    }
+  }
+
+  // dest
+  if (srcdnl->is_remote()) {
+    if (!linkmerge) {
+      if (destdn->is_auth() && !destdnl->is_null())
+	mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl);
+      else
+	destdn->first = MAX(destdn->first, next_dest_snap);
+
+      if (destdn->is_auth())
+        metablob->add_remote_dentry(destdn, true, srcdnl->get_remote_ino(), srcdnl->get_remote_d_type());
+      if (srci->get_projected_parent_dn()->is_auth()) { // it's remote
+	metablob->add_dir_context(srci->get_projected_parent_dir());
+        mdcache->journal_cow_dentry(mdr.get(), metablob, srci->get_projected_parent_dn(), CEPH_NOSNAP, 0, srcdnl);
+	metablob->add_primary_dentry(srci->get_projected_parent_dn(), srci, true);
+      }
+    } else {
+      if (destdn->is_auth() && !destdnl->is_null())
+	mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl);
+      else
+	destdn->first = MAX(destdn->first, next_dest_snap);
+
+      if (destdn->is_auth())
+        metablob->add_primary_dentry(destdn, destdnl->get_inode(), true, true);
+    }
+  } else if (srcdnl->is_primary()) {
+    // project snap parent update?
+    if (destdn->is_auth() &&
+        (srci->snaprealm || src_realm->get_newest_seq() + 1 > srcdn->first))
+      srci->project_past_snaprealm_parent(dest_realm);
+    
+    if (destdn->is_auth() && !destdnl->is_null())
+      mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl);
+    else
+      destdn->first = MAX(destdn->first, next_dest_snap);
+
+    if (destdn->is_auth())
+      metablob->add_primary_dentry(destdn, srci, true, true);
+    else if (force_journal_dest) {
+      dout(10) << " forced journaling destdn " << *destdn << dendl;
+      metablob->add_dir_context(destdn->get_dir());
+      metablob->add_primary_dentry(destdn, srci, true);
+      if (srcdn->is_auth() && srci->is_dir()) {
+	// journal new subtrees root dirfrags
+	list ls;
+	srci->get_dirfrags(ls);
+	for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+	  CDir *dir = *p;
+	  if (dir->is_auth())
+	    metablob->add_dir(dir, true);
+	}
+      }
+    }
+  }
+    
+  // src
+  if (srcdn->is_auth()) {
+    dout(10) << " journaling srcdn " << *srcdn << dendl;
+    mdcache->journal_cow_dentry(mdr.get(), metablob, srcdn, CEPH_NOSNAP, 0, srcdnl);
+    // also journal the inode in case we need do slave rename rollback. It is Ok to add
+    // both primary and NULL dentries. Because during journal replay, null dentry is
+    // processed after primary dentry.
+    if (srcdnl->is_primary() && !srci->is_dir() && !destdn->is_auth())
+      metablob->add_primary_dentry(srcdn, srci, true);
+    metablob->add_null_dentry(srcdn, true);
+  } else
+    dout(10) << " NOT journaling srcdn " << *srcdn << dendl;
+
+  // make renamed inode first track the dn
+  if (srcdnl->is_primary() && destdn->is_auth())
+    srci->first = destdn->first;  
+
+  // anchor updates?
+  if (mdr->more()->src_reanchor_atid)
+    metablob->add_table_transaction(TABLE_ANCHOR, mdr->more()->src_reanchor_atid);
+  if (mdr->more()->dst_reanchor_atid)
+    metablob->add_table_transaction(TABLE_ANCHOR, mdr->more()->dst_reanchor_atid);
+
+  if (oldin && oldin->is_dir())
+    mdcache->project_subtree_rename(oldin, destdn->get_dir(), straydn->get_dir());
+  if (srci->is_dir())
+    mdcache->project_subtree_rename(srci, srcdn->get_dir(), destdn->get_dir());
+
+}
+
+
+void Server::_rename_apply(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+{
+  dout(10) << "_rename_apply " << *mdr << " " << *srcdn << " " << *destdn << dendl;
+  dout(10) << " pvs " << mdr->more()->pvmap << dendl;
+
+  CDentry::linkage_t *srcdnl = srcdn->get_linkage();
+  CDentry::linkage_t *destdnl = destdn->get_linkage();
+
+  CInode *oldin = destdnl->get_inode();
+  
+  bool imported_inode = false;
+
+  // primary+remote link merge?
+  bool linkmerge = (srcdnl->get_inode() == destdnl->get_inode() &&
+		    (srcdnl->is_primary() || destdnl->is_primary()));
+
+  // target inode
+  if (!linkmerge) {
+    if (destdnl->is_primary()) {
+      assert(straydn);
+      dout(10) << "straydn is " << *straydn << dendl;
+      destdn->get_dir()->unlink_inode(destdn);
+
+      straydn->pop_projected_linkage();
+      mdcache->touch_dentry_bottom(straydn);  // drop dn as quickly as possible.
+
+      // nlink-- targeti
+      if (destdn->is_auth()) {
+	bool hadrealm = (oldin->snaprealm ? true : false);
+	oldin->pop_and_dirty_projected_inode(mdr->ls);
+	if (oldin->snaprealm && !hadrealm)
+	  mdcache->do_realm_invalidate_and_update_notify(oldin, CEPH_SNAP_OP_SPLIT);
+      } else {
+	// FIXME this snaprealm is not filled out correctly
+	//oldin->open_snaprealm();  might be sufficient..	
+      }
+    } else if (destdnl->is_remote()) {
+      destdn->get_dir()->unlink_inode(destdn);
+      if (oldin->is_auth())
+  	oldin->pop_and_dirty_projected_inode(mdr->ls);
+    }
+  }
+
+  // unlink src before we relink it at dest
+  CInode *in = srcdnl->get_inode();
+  assert(in);
+
+  bool srcdn_was_remote = srcdnl->is_remote();
+  srcdn->get_dir()->unlink_inode(srcdn);
+
+  // dest
+  if (srcdn_was_remote) {
+    if (!linkmerge) {
+      // destdn
+      destdnl = destdn->pop_projected_linkage();
+
+      destdn->link_remote(destdnl, in);
+      if (destdn->is_auth())
+	destdn->mark_dirty(mdr->more()->pvmap[destdn], mdr->ls);
+      // in
+      if (in->is_auth())
+	in->pop_and_dirty_projected_inode(mdr->ls);
+    } else {
+      dout(10) << "merging remote onto primary link" << dendl;
+      oldin->pop_and_dirty_projected_inode(mdr->ls);
+    }
+  } else { // primary
+    if (linkmerge) {
+      dout(10) << "merging primary onto remote link" << dendl;
+      destdn->get_dir()->unlink_inode(destdn);
+    }
+    destdnl = destdn->pop_projected_linkage();
+
+    // srcdn inode import?
+    if (!srcdn->is_auth() && destdn->is_auth()) {
+      assert(mdr->more()->inode_import.length() > 0);
+
+      map imported_caps;
+      
+      // finish cap imports
+      finish_force_open_sessions(mdr->more()->imported_client_map, mdr->more()->sseq_map);
+      if (mdr->more()->cap_imports.count(destdnl->get_inode())) {
+	mds->mdcache->migrator->finish_import_inode_caps(destdnl->get_inode(),
+							 mdr->more()->srcdn_auth_mds, true,
+							 mdr->more()->cap_imports[destdnl->get_inode()],
+							 imported_caps);
+      }
+
+      mdr->more()->inode_import.clear();
+      ::encode(imported_caps, mdr->more()->inode_import);
+
+      /* hack: add an auth pin for each xlock we hold. These were
+       * remote xlocks previously but now they're local and
+       * we're going to try and unpin when we xlock_finish. */
+      for (set::iterator i = mdr->xlocks.begin();
+	  i !=  mdr->xlocks.end();
+	  ++i)
+	if ((*i)->get_parent() == destdnl->get_inode() &&
+	    !(*i)->is_locallock())
+	  mds->locker->xlock_import(*i);
+      
+      // hack: fix auth bit
+      in->state_set(CInode::STATE_AUTH);
+      imported_inode = true;
+
+      mdr->clear_ambiguous_auth();
+    }
+
+    if (destdn->is_auth()) {
+      in->pop_and_dirty_projected_inode(mdr->ls);
+
+    } else {
+      // FIXME: fix up snaprealm!
+    }
+  }
+
+  // src
+  if (srcdn->is_auth())
+    srcdn->mark_dirty(mdr->more()->pvmap[srcdn], mdr->ls);
+  srcdn->pop_projected_linkage();
+  
+  // apply remaining projected inodes (nested)
+  mdr->apply();
+
+  // update subtree map?
+  if (destdnl->is_primary() && in->is_dir()) 
+    mdcache->adjust_subtree_after_rename(in, srcdn->get_dir(), true, imported_inode);
+
+  if (straydn && oldin->is_dir())
+    mdcache->adjust_subtree_after_rename(oldin, destdn->get_dir(), true);
+
+  // removing a new dn?
+  if (srcdn->is_auth())
+    srcdn->get_dir()->try_remove_unlinked_dn(srcdn);
+}
+
+
+
+
+
+// ------------
+// SLAVE
+
+class C_MDS_SlaveRenamePrep : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  CDentry *srcdn, *destdn, *straydn;
+public:
+  C_MDS_SlaveRenamePrep(Server *s, MDRequestRef& m, CDentry *sr, CDentry *de, CDentry *st) :
+    server(s), mdr(m), srcdn(sr), destdn(de), straydn(st) {}
+  void finish(int r) {
+    server->_logged_slave_rename(mdr, srcdn, destdn, straydn);
+  }
+};
+
+class C_MDS_SlaveRenameCommit : public Context {
+  Server *server;
+  MDRequestRef mdr;
+  CDentry *srcdn, *destdn, *straydn;
+public:
+  C_MDS_SlaveRenameCommit(Server *s, MDRequestRef& m, CDentry *sr, CDentry *de, CDentry *st) :
+    server(s), mdr(m), srcdn(sr), destdn(de), straydn(st) {}
+  void finish(int r) {
+    server->_commit_slave_rename(mdr, r, srcdn, destdn, straydn);
+  }
+};
+
+class C_MDS_SlaveRenameSessionsFlushed : public Context {
+  Server *server;
+  MDRequestRef mdr;
+public:
+  C_MDS_SlaveRenameSessionsFlushed(Server *s, MDRequestRef& r) :
+    server(s), mdr(r) {}
+  void finish(int r) {
+    server->_slave_rename_sessions_flushed(mdr);
+  }
+};
+
+/* This function DOES put the mdr->slave_request before returning*/
+void Server::handle_slave_rename_prep(MDRequestRef& mdr)
+{
+  dout(10) << "handle_slave_rename_prep " << *mdr 
+	   << " " << mdr->slave_request->srcdnpath 
+	   << " to " << mdr->slave_request->destdnpath
+	   << dendl;
+  
+  // discover destdn
+  filepath destpath(mdr->slave_request->destdnpath);
+  dout(10) << " dest " << destpath << dendl;
+  vector trace;
+  int r = mdcache->path_traverse(mdr, NULL, NULL, destpath, &trace, NULL, MDS_TRAVERSE_DISCOVERXLOCK);
+  if (r > 0) return;
+  assert(r == 0);  // we shouldn't get an error here!
+      
+  CDentry *destdn = trace[trace.size()-1];
+  CDentry::linkage_t *destdnl = destdn->get_projected_linkage();
+  dout(10) << " destdn " << *destdn << dendl;
+  mdr->pin(destdn);
+  
+  // discover srcdn
+  filepath srcpath(mdr->slave_request->srcdnpath);
+  dout(10) << " src " << srcpath << dendl;
+  CInode *srci;
+  r = mdcache->path_traverse(mdr, NULL, NULL, srcpath, &trace, &srci, MDS_TRAVERSE_DISCOVERXLOCK);
+  if (r > 0) return;
+  assert(r == 0);
+      
+  CDentry *srcdn = trace[trace.size()-1];
+  CDentry::linkage_t *srcdnl = srcdn->get_projected_linkage();
+  dout(10) << " srcdn " << *srcdn << dendl;
+  mdr->pin(srcdn);
+  mdr->pin(srci);
+
+  // stray?
+  bool linkmerge = (srcdnl->get_inode() == destdnl->get_inode() &&
+		    (srcdnl->is_primary() || destdnl->is_primary()));
+  CDentry *straydn = mdr->straydn;
+  if (destdnl->is_primary() && !linkmerge)
+    assert(straydn);
+
+  mdr->now = mdr->slave_request->now;
+  mdr->more()->srcdn_auth_mds = srcdn->authority().first;
+
+  // set up commit waiter (early, to clean up any freezing etc we do)
+  if (!mdr->more()->slave_commit)
+    mdr->more()->slave_commit = new C_MDS_SlaveRenameCommit(this, mdr, srcdn, destdn, straydn);
+
+  // am i srcdn auth?
+  if (srcdn->is_auth()) {
+    set srcdnrep;
+    srcdn->list_replicas(srcdnrep);
+
+    bool reply_witness = false;
+    if (srcdnl->is_primary() && !srcdnl->get_inode()->state_test(CInode::STATE_AMBIGUOUSAUTH)) {
+      // freeze?
+      // we need this to
+      //  - avoid conflicting lock state changes
+      //  - avoid concurrent updates to the inode
+      //     (this could also be accomplished with the versionlock)
+      int allowance = 2; // 1 for the mdr auth_pin, 1 for the link lock
+      allowance += srcdnl->get_inode()->is_dir(); // for the snap lock
+      dout(10) << " freezing srci " << *srcdnl->get_inode() << " with allowance " << allowance << dendl;
+      bool frozen_inode = srcdnl->get_inode()->freeze_inode(allowance);
+
+      // unfreeze auth pin after freezing the inode to avoid queueing waiters
+      if (srcdnl->get_inode()->is_frozen_auth_pin())
+	mdr->unfreeze_auth_pin();
+
+      if (!frozen_inode) {
+	srcdnl->get_inode()->add_waiter(CInode::WAIT_FROZEN, new C_MDS_RetryRequest(mdcache, mdr));
+	return;
+      }
+
+      /*
+       * set ambiguous auth for srci
+       * NOTE: we don't worry about ambiguous cache expire as we do
+       * with subtree migrations because all slaves will pin
+       * srcdn->get_inode() for duration of this rename.
+       */
+      mdr->set_ambiguous_auth(srcdnl->get_inode());
+
+      // just mark the source inode as ambiguous auth if more than two MDS are involved.
+      // the master will send another OP_RENAMEPREP slave request later.
+      if (mdr->slave_request->witnesses.size() > 1) {
+	dout(10) << " set srci ambiguous auth; providing srcdn replica list" << dendl;
+	reply_witness = true;
+      }
+
+      // make sure bystanders have received all lock related messages
+      for (set::iterator p = srcdnrep.begin(); p != srcdnrep.end(); ++p) {
+	if (*p == mdr->slave_to_mds ||
+	    !mds->mdsmap->is_clientreplay_or_active_or_stopping(*p))
+	  continue;
+	MMDSSlaveRequest *notify = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+	    MMDSSlaveRequest::OP_RENAMENOTIFY);
+	mds->send_message_mds(notify, *p);
+	mdr->more()->waiting_on_slave.insert(*p);
+      }
+
+      // make sure clients have received all cap related messages
+      set export_client_set;
+      mdcache->migrator->get_export_client_set(srcdnl->get_inode(), export_client_set);
+
+      C_GatherBuilder gather(g_ceph_context);
+      flush_client_sessions(export_client_set, gather);
+      if (gather.has_subs()) {
+	mdr->more()->waiting_on_slave.insert(-1);
+	gather.set_finisher(new C_MDS_SlaveRenameSessionsFlushed(this, mdr));
+	gather.activate();
+      }
+    }
+
+    // is witness list sufficient?
+    for (set::iterator p = srcdnrep.begin(); p != srcdnrep.end(); ++p) {
+      if (*p == mdr->slave_to_mds ||
+	  mdr->slave_request->witnesses.count(*p)) continue;
+      dout(10) << " witness list insufficient; providing srcdn replica list" << dendl;
+      reply_witness = true;
+      break;
+    }
+
+    if (reply_witness) {
+      assert(srcdnrep.size());
+      MMDSSlaveRequest *reply = new MMDSSlaveRequest(mdr->reqid, mdr->attempt,
+						     MMDSSlaveRequest::OP_RENAMEPREPACK);
+      reply->witnesses.swap(srcdnrep);
+      mds->send_message_mds(reply, mdr->slave_to_mds);
+      mdr->slave_request->put();
+      mdr->slave_request = 0;
+      return;	
+    }
+    dout(10) << " witness list sufficient: includes all srcdn replicas" << dendl;
+    if (!mdr->more()->waiting_on_slave.empty()) {
+      dout(10) << " still waiting for rename notify acks from "
+	       << mdr->more()->waiting_on_slave << dendl;
+      return;
+    }
+  } else if (srcdnl->is_primary() && srcdn->authority() != destdn->authority()) {
+    // set ambiguous auth for srci on witnesses
+    mdr->set_ambiguous_auth(srcdnl->get_inode());
+  }
+
+  // encode everything we'd need to roll this back... basically, just the original state.
+  rename_rollback rollback;
+  
+  rollback.reqid = mdr->reqid;
+  
+  rollback.orig_src.dirfrag = srcdn->get_dir()->dirfrag();
+  rollback.orig_src.dirfrag_old_mtime = srcdn->get_dir()->get_projected_fnode()->fragstat.mtime;
+  rollback.orig_src.dirfrag_old_rctime = srcdn->get_dir()->get_projected_fnode()->rstat.rctime;
+  rollback.orig_src.dname = srcdn->name;
+  if (srcdnl->is_primary())
+    rollback.orig_src.ino = srcdnl->get_inode()->ino();
+  else {
+    assert(srcdnl->is_remote());
+    rollback.orig_src.remote_ino = srcdnl->get_remote_ino();
+    rollback.orig_src.remote_d_type = srcdnl->get_remote_d_type();
+  }
+  
+  rollback.orig_dest.dirfrag = destdn->get_dir()->dirfrag();
+  rollback.orig_dest.dirfrag_old_mtime = destdn->get_dir()->get_projected_fnode()->fragstat.mtime;
+  rollback.orig_dest.dirfrag_old_rctime = destdn->get_dir()->get_projected_fnode()->rstat.rctime;
+  rollback.orig_dest.dname = destdn->name;
+  if (destdnl->is_primary())
+    rollback.orig_dest.ino = destdnl->get_inode()->ino();
+  else if (destdnl->is_remote()) {
+    rollback.orig_dest.remote_ino = destdnl->get_remote_ino();
+    rollback.orig_dest.remote_d_type = destdnl->get_remote_d_type();
+  }
+  
+  if (straydn) {
+    rollback.stray.dirfrag = straydn->get_dir()->dirfrag();
+    rollback.stray.dirfrag_old_mtime = straydn->get_dir()->get_projected_fnode()->fragstat.mtime;
+    rollback.stray.dirfrag_old_rctime = straydn->get_dir()->get_projected_fnode()->rstat.rctime;
+    rollback.stray.dname = straydn->name;
+  }
+  ::encode(rollback, mdr->more()->rollback_bl);
+  dout(20) << " rollback is " << mdr->more()->rollback_bl.length() << " bytes" << dendl;
+
+  // journal.
+  mdr->ls = mdlog->get_current_segment();
+  ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_rename_prep", mdr->reqid, mdr->slave_to_mds,
+				      ESlaveUpdate::OP_PREPARE, ESlaveUpdate::RENAME);
+  mdlog->start_entry(le);
+  le->rollback = mdr->more()->rollback_bl;
+  
+  bufferlist blah;  // inode import data... obviously not used if we're the slave
+  _rename_prepare(mdr, &le->commit, &blah, srcdn, destdn, straydn);
+  
+  mdlog->submit_entry(le, new C_MDS_SlaveRenamePrep(this, mdr, srcdn, destdn, straydn));
+  mdlog->flush();
+}
+
+void Server::_logged_slave_rename(MDRequestRef& mdr,
+				  CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+{
+  dout(10) << "_logged_slave_rename " << *mdr << dendl;
+
+  // prepare ack
+  MMDSSlaveRequest *reply = NULL;
+  if (!mdr->aborted)
+    reply= new MMDSSlaveRequest(mdr->reqid, mdr->attempt, MMDSSlaveRequest::OP_RENAMEPREPACK);
+
+  CDentry::linkage_t *srcdnl = srcdn->get_linkage();
+  CDentry::linkage_t *destdnl = destdn->get_linkage();
+  //CDentry::linkage_t *straydnl = straydn ? straydn->get_linkage() : 0;
+
+  // export srci?
+  if (srcdn->is_auth() && srcdnl->is_primary()) {
+    // set export bounds for CInode::encode_export()
+    list bounds;
+    if (srcdnl->get_inode()->is_dir()) {
+      srcdnl->get_inode()->get_dirfrags(bounds);
+      for (list::iterator p = bounds.begin(); p != bounds.end(); ++p)
+	(*p)->state_set(CDir::STATE_EXPORTBOUND);
+    }
+
+    map exported_client_map;
+    bufferlist inodebl;
+    mdcache->migrator->encode_export_inode(srcdnl->get_inode(), inodebl, 
+					   exported_client_map);
+
+    for (list::iterator p = bounds.begin(); p != bounds.end(); ++p)
+      (*p)->state_clear(CDir::STATE_EXPORTBOUND);
+
+    if (reply) {
+      ::encode(exported_client_map, reply->inode_export);
+      reply->inode_export.claim_append(inodebl);
+      reply->inode_export_v = srcdnl->get_inode()->inode.version;
+    }
+
+    // remove mdr auth pin
+    mdr->auth_unpin(srcdnl->get_inode());
+    mdr->more()->is_inode_exporter = true;
+
+    if (srcdnl->get_inode()->is_dirty())
+      srcdnl->get_inode()->mark_clean();
+
+    dout(10) << " exported srci " << *srcdnl->get_inode() << dendl;
+  }
+
+  // apply
+  _rename_apply(mdr, srcdn, destdn, straydn);   
+  
+  destdnl = destdn->get_linkage();
+
+  // bump popularity
+  mds->balancer->hit_dir(mdr->now, srcdn->get_dir(), META_POP_IWR);
+  if (destdnl->get_inode() && destdnl->get_inode()->is_auth())
+    mds->balancer->hit_inode(mdr->now, destdnl->get_inode(), META_POP_IWR);
+
+  // done.
+  mdr->slave_request->put();
+  mdr->slave_request = 0;
+  mdr->straydn = 0;
+
+  if (reply) {
+    mds->send_message_mds(reply, mdr->slave_to_mds);
+  } else {
+    assert(mdr->aborted);
+    dout(10) << " abort flag set, finishing" << dendl;
+    mdcache->request_finish(mdr);
+  }
+}
+
+void Server::_commit_slave_rename(MDRequestRef& mdr, int r,
+				  CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+{
+  dout(10) << "_commit_slave_rename " << *mdr << " r=" << r << dendl;
+
+  CDentry::linkage_t *destdnl = destdn->get_linkage();
+
+  list finished;
+  if (r == 0) {
+    // write a commit to the journal
+    ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_rename_commit", mdr->reqid, 
+					mdr->slave_to_mds, ESlaveUpdate::OP_COMMIT, 
+					ESlaveUpdate::RENAME);
+    mdlog->start_entry(le);
+
+    // unfreeze+singleauth inode
+    //  hmm, do i really need to delay this?
+    if (mdr->more()->is_inode_exporter) {
+
+      CInode *in = destdnl->get_inode();
+
+      // drop our pins
+      // we exported, clear out any xlocks that we moved to another MDS
+      set::iterator i = mdr->xlocks.begin();
+      while (i != mdr->xlocks.end()) {
+        SimpleLock *lock = *i++;
+
+	// we only care about xlocks on the exported inode
+	if (lock->get_parent() == in &&
+	    !lock->is_locallock())
+	  mds->locker->xlock_export(lock, mdr.get());
+      }
+
+      map peer_imported;
+      bufferlist::iterator bp = mdr->more()->inode_import.begin();
+      ::decode(peer_imported, bp);
+
+      dout(10) << " finishing inode export on " << *destdnl->get_inode() << dendl;
+      mdcache->migrator->finish_export_inode(destdnl->get_inode(), mdr->now,
+					     mdr->slave_to_mds, peer_imported, finished);
+      mds->queue_waiters(finished);   // this includes SINGLEAUTH waiters.
+
+      // unfreeze
+      assert(destdnl->get_inode()->is_frozen_inode());
+      destdnl->get_inode()->unfreeze_inode(finished);
+    }
+
+    // singleauth
+    if (mdr->more()->is_ambiguous_auth) {
+      mdr->more()->rename_inode->clear_ambiguous_auth(finished);
+      mdr->more()->is_ambiguous_auth = false;
+    }
+
+    mds->queue_waiters(finished);
+    mdr->cleanup();
+
+    mdlog->submit_entry(le, new C_MDS_CommittedSlave(this, mdr));
+    mdlog->flush();
+  } else {
+
+    // abort
+    //  rollback_bl may be empty if we froze the inode but had to provide an expanded
+    // witness list from the master, and they failed before we tried prep again.
+    if (mdr->more()->rollback_bl.length()) {
+      if (mdr->more()->is_inode_exporter) {
+	dout(10) << " reversing inode export of " << *destdnl->get_inode() << dendl;
+	destdnl->get_inode()->abort_export();
+      }
+      if (mdcache->is_ambiguous_slave_update(mdr->reqid, mdr->slave_to_mds)) {
+	mdcache->remove_ambiguous_slave_update(mdr->reqid, mdr->slave_to_mds);
+	// rollback but preserve the slave request
+	do_rename_rollback(mdr->more()->rollback_bl, mdr->slave_to_mds, mdr, false);
+	mdr->more()->rollback_bl.clear();
+      } else
+	do_rename_rollback(mdr->more()->rollback_bl, mdr->slave_to_mds, mdr, true);
+    } else {
+      dout(10) << " rollback_bl empty, not rollback back rename (master failed after getting extra witnesses?)" << dendl;
+      // singleauth
+      if (mdr->more()->is_ambiguous_auth) {
+	if (srcdn->is_auth())
+	  mdr->more()->rename_inode->unfreeze_inode(finished);
+
+	mdr->more()->rename_inode->clear_ambiguous_auth(finished);
+	mdr->more()->is_ambiguous_auth = false;
+      }
+      mds->queue_waiters(finished);
+      mds->mdcache->request_finish(mdr);
+    }
+  }
+}
+
+void _rollback_repair_dir(MutationRef& mut, CDir *dir, rename_rollback::drec &r, utime_t ctime,
+			  bool isdir, int linkunlink, nest_info_t &rstat)
+{
+  fnode_t *pf;
+  pf = dir->project_fnode();
+  mut->add_projected_fnode(dir);
+  pf->version = dir->pre_dirty();
+
+  if (isdir) {
+    pf->fragstat.nsubdirs += linkunlink;
+    pf->rstat.rsubdirs += linkunlink;
+  } else {
+    pf->fragstat.nfiles += linkunlink;
+    pf->rstat.rfiles += linkunlink;
+  }    
+  if (r.ino) {
+    pf->rstat.rbytes += linkunlink * rstat.rbytes;
+    pf->rstat.rfiles += linkunlink * rstat.rfiles;
+    pf->rstat.rsubdirs += linkunlink * rstat.rsubdirs;
+    pf->rstat.ranchors += linkunlink * rstat.ranchors;
+    pf->rstat.rsnaprealms += linkunlink * rstat.rsnaprealms;
+  }
+  if (pf->fragstat.mtime == ctime) {
+    pf->fragstat.mtime = r.dirfrag_old_mtime;
+    if (pf->rstat.rctime == ctime)
+      pf->rstat.rctime = r.dirfrag_old_rctime;
+  }
+  mut->add_updated_lock(&dir->get_inode()->filelock);
+  mut->add_updated_lock(&dir->get_inode()->nestlock);
+}
+
+struct C_MDS_LoggedRenameRollback : public Context {
+  Server *server;
+  MutationRef mut;
+  MDRequestRef mdr;
+  CDentry *srcdn;
+  version_t srcdnpv;
+  CDentry *destdn;
+  CDentry *straydn;
+  bool finish_mdr;
+  C_MDS_LoggedRenameRollback(Server *s, MutationRef& m, MDRequestRef& r,
+			     CDentry *sd, version_t pv, CDentry *dd,
+			    CDentry *st, bool f) :
+    server(s), mut(m), mdr(r), srcdn(sd), srcdnpv(pv), destdn(dd),
+    straydn(st), finish_mdr(f) {}
+  void finish(int r) {
+    server->_rename_rollback_finish(mut, mdr, srcdn, srcdnpv,
+				    destdn, straydn, finish_mdr);
+  }
+};
+
+void Server::do_rename_rollback(bufferlist &rbl, int master, MDRequestRef& mdr,
+				bool finish_mdr)
+{
+  rename_rollback rollback;
+  bufferlist::iterator p = rbl.begin();
+  ::decode(rollback, p);
+
+  dout(10) << "do_rename_rollback on " << rollback.reqid << dendl;
+  // need to finish this update before sending resolve to claim the subtree
+  mds->mdcache->add_rollback(rollback.reqid, master);
+
+  MutationRef mut(new MutationImpl(rollback.reqid));
+  mut->ls = mds->mdlog->get_current_segment();
+
+  CDentry *srcdn = NULL;
+  CDir *srcdir = mds->mdcache->get_dirfrag(rollback.orig_src.dirfrag);
+  if (!srcdir)
+    srcdir = mds->mdcache->get_dirfrag(rollback.orig_src.dirfrag.ino, rollback.orig_src.dname);
+  if (srcdir) {
+    dout(10) << "  srcdir " << *srcdir << dendl;
+    srcdn = srcdir->lookup(rollback.orig_src.dname);
+    if (srcdn) {
+      dout(10) << "   srcdn " << *srcdn << dendl;
+      assert(srcdn->get_linkage()->is_null());
+    } else
+      dout(10) << "   srcdn not found" << dendl;
+  } else
+    dout(10) << "  srcdir not found" << dendl;
+
+  CDentry *destdn = NULL;
+  CDir *destdir = mds->mdcache->get_dirfrag(rollback.orig_dest.dirfrag);
+  if (!destdir)
+    destdir = mds->mdcache->get_dirfrag(rollback.orig_dest.dirfrag.ino, rollback.orig_dest.dname);
+  if (destdir) {
+    dout(10) << " destdir " << *destdir << dendl;
+    destdn = destdir->lookup(rollback.orig_dest.dname);
+    if (destdn)
+      dout(10) << "  destdn " << *destdn << dendl;
+    else
+      dout(10) << "  destdn not found" << dendl;
+  } else
+    dout(10) << " destdir not found" << dendl;
+
+  CInode *in = NULL;
+  if (rollback.orig_src.ino) {
+    in = mds->mdcache->get_inode(rollback.orig_src.ino);
+    if (in && in->is_dir())
+      assert(srcdn && destdn);
+  } else
+    in = mds->mdcache->get_inode(rollback.orig_src.remote_ino);
+
+  CDir *straydir = NULL;
+  CDentry *straydn = NULL;
+  if (rollback.stray.dirfrag.ino) {
+    straydir = mds->mdcache->get_dirfrag(rollback.stray.dirfrag);
+    if (straydir) {
+      dout(10) << "straydir " << *straydir << dendl;
+      straydn = straydir->lookup(rollback.stray.dname);
+      if (straydn) {
+	dout(10) << " straydn " << *straydn << dendl;
+	assert(straydn->get_linkage()->is_primary());
+      } else
+	dout(10) << " straydn not found" << dendl;
+    } else
+      dout(10) << "straydir not found" << dendl;
+  }
+
+  CInode *target = NULL;
+  if (rollback.orig_dest.ino) {
+    target = mds->mdcache->get_inode(rollback.orig_dest.ino);
+    if (target)
+      assert(destdn && straydn);
+  } else if (rollback.orig_dest.remote_ino)
+    target = mds->mdcache->get_inode(rollback.orig_dest.remote_ino);
+
+  // can't use is_auth() in the resolve stage
+  int whoami = mds->get_nodeid();
+  // slave
+  assert(!destdn || destdn->authority().first != whoami);
+  assert(!straydn || straydn->authority().first != whoami);
+
+  bool force_journal_src = false;
+  bool force_journal_dest = false;
+  if (in && in->is_dir() && srcdn->authority().first != whoami)
+    force_journal_src = _need_force_journal(in, false);
+  if (in && target && target->is_dir())
+    force_journal_dest = _need_force_journal(in, true);
+  
+  version_t srcdnpv = 0;
+  // repair src
+  if (srcdn) {
+    if (srcdn->authority().first == whoami)
+      srcdnpv = srcdn->pre_dirty();
+    if (rollback.orig_src.ino) {
+      assert(in);
+      srcdn->push_projected_linkage(in);
+    } else
+      srcdn->push_projected_linkage(rollback.orig_src.remote_ino,
+				    rollback.orig_src.remote_d_type);
+  }
+
+  inode_t *pi = 0;
+  if (in) {
+    if (in->authority().first == whoami) {
+      pi = in->project_inode();
+      mut->add_projected_inode(in);
+      pi->version = in->pre_dirty();
+    } else
+      pi = in->get_projected_inode();
+    if (pi->ctime == rollback.ctime)
+      pi->ctime = rollback.orig_src.old_ctime;
+  }
+
+  if (srcdn && srcdn->authority().first == whoami) {
+    nest_info_t blah;
+    _rollback_repair_dir(mut, srcdir, rollback.orig_src, rollback.ctime,
+			 in ? in->is_dir() : false, 1, pi ? pi->rstat : blah);
+  }
+
+  // repair dest
+  if (destdn) {
+    if (rollback.orig_dest.ino && target) {
+      destdn->push_projected_linkage(target);
+    } else if (rollback.orig_dest.remote_ino) {
+      destdn->push_projected_linkage(rollback.orig_dest.remote_ino,
+				     rollback.orig_dest.remote_d_type);
+    } else {
+      // the dentry will be trimmed soon, it's ok to have wrong linkage
+      if (rollback.orig_dest.ino)
+	assert(mds->is_resolve());
+      destdn->push_projected_linkage();
+    }
+  }
+
+  if (straydn)
+    straydn->push_projected_linkage();
+
+  if (target) {
+    inode_t *ti = NULL;
+    if (target->authority().first == whoami) {
+      ti = target->project_inode();
+      mut->add_projected_inode(target);
+      ti->version = target->pre_dirty();
+    } else 
+      ti = target->get_projected_inode();
+    if (ti->ctime == rollback.ctime)
+      ti->ctime = rollback.orig_dest.old_ctime;
+    if (MDS_INO_IS_STRAY(rollback.orig_src.dirfrag.ino)) {
+      if (MDS_INO_IS_STRAY(rollback.orig_dest.dirfrag.ino))
+	assert(!rollback.orig_dest.ino && !rollback.orig_dest.remote_ino);
+      else
+	assert(rollback.orig_dest.remote_ino &&
+	       rollback.orig_dest.remote_ino == rollback.orig_src.ino);
+    } else
+      ti->nlink++;
+  }
+
+  if (srcdn)
+    dout(0) << " srcdn back to " << *srcdn << dendl;
+  if (in)
+    dout(0) << "  srci back to " << *in << dendl;
+  if (destdn)
+    dout(0) << " destdn back to " << *destdn << dendl;
+  if (target)
+    dout(0) << "  desti back to " << *target << dendl;
+  
+  // journal it
+  ESlaveUpdate *le = new ESlaveUpdate(mdlog, "slave_rename_rollback", rollback.reqid, master,
+				      ESlaveUpdate::OP_ROLLBACK, ESlaveUpdate::RENAME);
+  mdlog->start_entry(le);
+
+  if (srcdn && (srcdn->authority().first == whoami || force_journal_src)) {
+    le->commit.add_dir_context(srcdir);
+    if (rollback.orig_src.ino)
+      le->commit.add_primary_dentry(srcdn, 0, true);
+    else
+      le->commit.add_remote_dentry(srcdn, true);
+  }
+
+  if (force_journal_dest) {
+    assert(rollback.orig_dest.ino);
+    le->commit.add_dir_context(destdir);
+    le->commit.add_primary_dentry(destdn, 0, true);
+  }
+
+  // slave: no need to journal straydn
+
+  if (target && target->authority().first == whoami) {
+    assert(rollback.orig_dest.remote_ino);
+    le->commit.add_dir_context(target->get_projected_parent_dir());
+    le->commit.add_primary_dentry(target->get_projected_parent_dn(), target, true);
+  }
+
+  if (force_journal_dest) {
+    dout(10) << " noting rename target ino " << target->ino() << " in metablob" << dendl;
+    le->commit.renamed_dirino = target->ino();
+  } else if (force_journal_src || (in && in->is_dir() && srcdn->authority().first == whoami)) {
+    dout(10) << " noting renamed dir ino " << in->ino() << " in metablob" << dendl;
+    le->commit.renamed_dirino = in->ino();
+  }
+  
+  if (target && target->is_dir()) {
+    assert(destdn);
+    mdcache->project_subtree_rename(target, straydir, destdir);
+  }
+
+  if (in && in->is_dir()) {
+    assert(srcdn);
+    mdcache->project_subtree_rename(in, destdir, srcdir);
+  }
+
+  mdlog->submit_entry(le, new C_MDS_LoggedRenameRollback(this, mut, mdr, srcdn, srcdnpv,
+							 destdn, straydn, finish_mdr));
+  mdlog->flush();
+}
+
+void Server::_rename_rollback_finish(MutationRef& mut, MDRequestRef& mdr, CDentry *srcdn,
+				     version_t srcdnpv, CDentry *destdn,
+				     CDentry *straydn, bool finish_mdr)
+{
+  dout(10) << "_rename_rollback_finish " << mut->reqid << dendl;
+
+  if (straydn) {
+    straydn->get_dir()->unlink_inode(straydn);
+    straydn->pop_projected_linkage();
+  }
+  if (destdn) {
+    destdn->get_dir()->unlink_inode(destdn);
+    destdn->pop_projected_linkage();
+  }
+  if (srcdn) {
+    srcdn->pop_projected_linkage();
+    if (srcdn->authority().first == mds->get_nodeid())
+      srcdn->mark_dirty(srcdnpv, mut->ls);
+  }
+
+  mut->apply();
+
+  if (srcdn) {
+    CInode *in = srcdn->get_linkage()->get_inode();
+    // update subtree map?
+    if (in && in->is_dir()) {
+      assert(destdn);
+      mdcache->adjust_subtree_after_rename(in, destdn->get_dir(), true);
+    }
+  }
+
+  if (destdn) {
+    CInode *oldin = destdn->get_linkage()->get_inode();
+    // update subtree map?
+    if (oldin && oldin->is_dir()) {
+      assert(straydn);
+      mdcache->adjust_subtree_after_rename(oldin, straydn->get_dir(), true);
+    }
+  }
+
+  if (mds->is_resolve()) {
+    CDir *root = NULL;
+    if (straydn)
+      root = mdcache->get_subtree_root(straydn->get_dir());
+    else if (destdn)
+      root = mdcache->get_subtree_root(destdn->get_dir());
+    if (root)
+      mdcache->try_trim_non_auth_subtree(root);
+  }
+
+  if (mdr) {
+    list finished;
+    if (mdr->more()->is_ambiguous_auth) {
+      if (srcdn->is_auth())
+	mdr->more()->rename_inode->unfreeze_inode(finished);
+
+      mdr->more()->rename_inode->clear_ambiguous_auth(finished);
+      mdr->more()->is_ambiguous_auth = false;
+    }
+    mds->queue_waiters(finished);
+    if (finish_mdr)
+      mds->mdcache->request_finish(mdr);
+  }
+
+  mds->mdcache->finish_rollback(mut->reqid);
+
+  mut->cleanup();
+}
+
+/* This function DOES put the passed message before returning*/
+void Server::handle_slave_rename_prep_ack(MDRequestRef& mdr, MMDSSlaveRequest *ack)
+{
+  dout(10) << "handle_slave_rename_prep_ack " << *mdr 
+	   << " witnessed by " << ack->get_source()
+	   << " " << *ack << dendl;
+  int from = ack->get_source().num();
+
+  // note slave
+  mdr->more()->slaves.insert(from);
+
+  // witnessed?  or add extra witnesses?
+  assert(mdr->more()->witnessed.count(from) == 0);
+  if (ack->witnesses.empty()) {
+    mdr->more()->witnessed.insert(from);
+  } else {
+    dout(10) << " extra witnesses (srcdn replicas) are " << ack->witnesses << dendl;
+    mdr->more()->extra_witnesses.swap(ack->witnesses);
+    mdr->more()->extra_witnesses.erase(mds->get_nodeid());  // not me!
+  }
+
+  // srci import?
+  if (ack->inode_export.length()) {
+    dout(10) << " got srci import" << dendl;
+    mdr->more()->inode_import.claim(ack->inode_export);
+    mdr->more()->inode_import_v = ack->inode_export_v;
+  }
+
+  // remove from waiting list
+  assert(mdr->more()->waiting_on_slave.count(from));
+  mdr->more()->waiting_on_slave.erase(from);
+
+  if (mdr->more()->waiting_on_slave.empty())
+    dispatch_client_request(mdr);  // go again!
+  else 
+    dout(10) << "still waiting on slaves " << mdr->more()->waiting_on_slave << dendl;
+}
+
+void Server::handle_slave_rename_notify_ack(MDRequestRef& mdr, MMDSSlaveRequest *ack)
+{
+  dout(10) << "handle_slave_rename_notify_ack " << *mdr << " from mds."
+	   << ack->get_source() << dendl;
+  assert(mdr->is_slave());
+  int from = ack->get_source().num();
+
+  if (mdr->more()->waiting_on_slave.count(from)) {
+    mdr->more()->waiting_on_slave.erase(from);
+
+    if (mdr->more()->waiting_on_slave.empty()) {
+      if (mdr->slave_request)
+	dispatch_slave_request(mdr);
+    } else 
+      dout(10) << " still waiting for rename notify acks from "
+	       << mdr->more()->waiting_on_slave << dendl;
+  }
+}
+
+void Server::_slave_rename_sessions_flushed(MDRequestRef& mdr)
+{
+  dout(10) << "_slave_rename_sessions_flushed " << *mdr << dendl;
+
+  if (mdr->more()->waiting_on_slave.count(-1)) {
+    mdr->more()->waiting_on_slave.erase(-1);
+
+    if (mdr->more()->waiting_on_slave.empty()) {
+      if (mdr->slave_request)
+	dispatch_slave_request(mdr);
+    } else
+      dout(10) << " still waiting for rename notify acks from "
+	<< mdr->more()->waiting_on_slave << dendl;
+  }
+}
+
+// snaps
+/* This function takes responsibility for the passed mdr*/
+void Server::handle_client_lssnap(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+
+  // traverse to path
+  CInode *diri = mdcache->get_inode(req->get_filepath().get_ino());
+  if (!diri || diri->state_test(CInode::STATE_PURGING)) {
+     reply_request(mdr, -ESTALE);
+     return;
+  }
+  if (!diri->is_auth()) {
+    mdcache->request_forward(mdr, diri->authority().first);
+    return;
+  }
+  if (!diri->is_dir()) {
+    reply_request(mdr, -ENOTDIR);
+    return;
+  }
+  dout(10) << "lssnap on " << *diri << dendl;
+
+  // lock snap
+  set rdlocks, wrlocks, xlocks;
+  mds->locker->include_snap_rdlocks(rdlocks, diri);
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  SnapRealm *realm = diri->find_snaprealm();
+  map infomap;
+  realm->get_snap_info(infomap, diri->get_oldest_snap());
+
+  __u32 num = 0;
+  bufferlist dnbl;
+  for (map::iterator p = infomap.begin();
+       p != infomap.end();
+       ++p) {
+    dout(10) << p->first << " -> " << *p->second << dendl;
+
+    // actual
+    if (p->second->ino == diri->ino())
+      ::encode(p->second->name, dnbl);
+    else
+      ::encode(p->second->get_long_name(), dnbl);
+    encode_infinite_lease(dnbl);
+    diri->encode_inodestat(dnbl, mdr->session, realm, p->first);
+    num++;
+  }
+
+  bufferlist dirbl;
+  encode_empty_dirstat(dirbl);
+  ::encode(num, dirbl);
+  __u8 t = 1;
+  ::encode(t, dirbl);  // end
+  ::encode(t, dirbl);  // complete
+  dirbl.claim_append(dnbl);
+  
+  MClientReply *reply = new MClientReply(req);
+  reply->set_extra_bl(dirbl);
+  reply_request(mdr, reply, diri);
+}
+
+
+// MKSNAP
+
+struct C_MDS_mksnap_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CInode *diri;
+  SnapInfo info;
+  C_MDS_mksnap_finish(MDS *m, MDRequestRef& r, CInode *di, SnapInfo &i) :
+    mds(m), mdr(r), diri(di), info(i) {}
+  void finish(int r) {
+    mds->server->_mksnap_finish(mdr, diri, info);
+  }
+};
+
+/* This function takes responsibility for the passed mdr*/
+void Server::handle_client_mksnap(MDRequestRef& mdr)
+{
+  if (!mds->mdsmap->allows_snaps()) {
+    // you can't make snapshots until you set an option right now
+    reply_request(mdr, -EPERM);
+    return;
+  }
+
+  MClientRequest *req = mdr->client_request;
+  CInode *diri = mdcache->get_inode(req->get_filepath().get_ino());
+  if (!diri || diri->state_test(CInode::STATE_PURGING)) {
+    reply_request(mdr, -ESTALE);
+    return;
+  }
+
+  if (!diri->is_auth()) {    // fw to auth?
+    mdcache->request_forward(mdr, diri->authority().first);
+    return;
+  }
+
+  // dir only
+  if (!diri->is_dir()) {
+    reply_request(mdr, -ENOTDIR);
+    return;
+  }
+  if (diri->is_system() && !diri->is_root()) {
+    // no snaps in system dirs (root is ok)
+    reply_request(mdr, -EPERM);
+    return;
+  }
+
+  const string &snapname = req->get_filepath().last_dentry();
+  dout(10) << "mksnap " << snapname << " on " << *diri << dendl;
+
+  // lock snap
+  set rdlocks, wrlocks, xlocks;
+
+  mds->locker->include_snap_rdlocks(rdlocks, diri);
+  rdlocks.erase(&diri->snaplock);
+  xlocks.insert(&diri->snaplock);
+
+  // we need to anchor... get these locks up front!
+  mds->mdcache->anchor_create_prep_locks(mdr, diri, rdlocks, xlocks);
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // make sure name is unique
+  if (diri->snaprealm &&
+      diri->snaprealm->exists(snapname)) {
+    reply_request(mdr, -EEXIST);
+    return;
+  }
+  if (snapname.length() == 0 ||
+      snapname[0] == '_') {
+    reply_request(mdr, -EINVAL);
+    return;
+  }
+
+  if (mdr->now == utime_t())
+    mdr->now = ceph_clock_now(g_ceph_context);
+
+  // anchor
+  if (!diri->is_anchored()) {
+    mdcache->anchor_create(mdr, diri, new C_MDS_RetryRequest(mdcache, mdr));
+    return;
+  }
+
+  // allocate a snapid
+  if (!mdr->more()->stid) {
+    // prepare an stid
+    mds->snapclient->prepare_create(diri->ino(), snapname, mdr->now, 
+				    &mdr->more()->stid, &mdr->more()->snapidbl,
+				    new C_MDS_RetryRequest(mds->mdcache, mdr));
+    return;
+  }
+
+  version_t stid = mdr->more()->stid;
+  snapid_t snapid;
+  bufferlist::iterator p = mdr->more()->snapidbl.begin();
+  ::decode(snapid, p);
+  dout(10) << " stid " << stid << " snapid " << snapid << dendl;
+
+  // journal
+  SnapInfo info;
+  info.ino = diri->ino();
+  info.snapid = snapid;
+  info.name = snapname;
+  info.stamp = mdr->now;
+
+  inode_t *pi = diri->project_inode();
+  pi->ctime = info.stamp;
+  pi->version = diri->pre_dirty();
+
+  // project the snaprealm
+  sr_t *newsnap = diri->project_snaprealm(snapid);
+  newsnap->snaps[snapid] = info;
+  newsnap->seq = snapid;
+  newsnap->last_created = snapid;
+
+  // journal the inode changes
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "mksnap");
+  mdlog->start_entry(le);
+
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  le->metablob.add_table_transaction(TABLE_SNAP, stid);
+  mdcache->predirty_journal_parents(mdr, &le->metablob, diri, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, diri);
+
+  // journal the snaprealm changes
+  mdlog->submit_entry(le, new C_MDS_mksnap_finish(mds, mdr, diri, info));
+  mdlog->flush();
+}
+
+void Server::_mksnap_finish(MDRequestRef& mdr, CInode *diri, SnapInfo &info)
+{
+  dout(10) << "_mksnap_finish " << *mdr << " " << info << dendl;
+
+  int op = (diri->snaprealm? CEPH_SNAP_OP_CREATE : CEPH_SNAP_OP_SPLIT);
+
+  diri->pop_and_dirty_projected_inode(mdr->ls);
+  mdr->apply();
+
+  mds->snapclient->commit(mdr->more()->stid, mdr->ls);
+
+  // create snap
+  dout(10) << "snaprealm now " << *diri->snaprealm << dendl;
+
+  mdcache->do_realm_invalidate_and_update_notify(diri, op);
+
+  // yay
+  mdr->in[0] = diri;
+  mdr->snapid = info.snapid;
+  MClientReply *reply = new MClientReply(mdr->client_request, 0);
+  reply->snapbl = diri->snaprealm->get_snap_trace();
+  reply_request(mdr, reply, diri);
+}
+
+
+// RMSNAP
+
+struct C_MDS_rmsnap_finish : public Context {
+  MDS *mds;
+  MDRequestRef mdr;
+  CInode *diri;
+  snapid_t snapid;
+  C_MDS_rmsnap_finish(MDS *m, MDRequestRef& r, CInode *di, snapid_t sn) :
+    mds(m), mdr(r), diri(di), snapid(sn) {}
+  void finish(int r) {
+    mds->server->_rmsnap_finish(mdr, diri, snapid);
+  }
+};
+
+/* This function takes responsibility for the passed mdr*/
+void Server::handle_client_rmsnap(MDRequestRef& mdr)
+{
+  MClientRequest *req = mdr->client_request;
+
+  CInode *diri = mdcache->get_inode(req->get_filepath().get_ino());
+  if (!diri || diri->state_test(CInode::STATE_PURGING)) {
+    reply_request(mdr, -ESTALE);
+    return;
+  }
+  if (!diri->is_auth()) {    // fw to auth?
+    mdcache->request_forward(mdr, diri->authority().first);
+    return;
+  }
+  if (!diri->is_dir()) {
+    reply_request(mdr, -ENOTDIR);
+    return;
+  }
+
+  const string &snapname = req->get_filepath().last_dentry();
+  dout(10) << "rmsnap " << snapname << " on " << *diri << dendl;
+
+  // does snap exist?
+  if (snapname.length() == 0 || snapname[0] == '_') {
+    reply_request(mdr, -EINVAL);   // can't prune a parent snap, currently.
+    return;
+  }
+  if (!diri->snaprealm || !diri->snaprealm->exists(snapname)) {
+    reply_request(mdr, -ENOENT);
+    return;
+  }
+  snapid_t snapid = diri->snaprealm->resolve_snapname(snapname, diri->ino());
+  dout(10) << " snapname " << snapname << " is " << snapid << dendl;
+
+  set rdlocks, wrlocks, xlocks;
+  mds->locker->include_snap_rdlocks(rdlocks, diri);
+  rdlocks.erase(&diri->snaplock);
+  xlocks.insert(&diri->snaplock);
+
+  if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks))
+    return;
+
+  // prepare
+  if (!mdr->more()->stid) {
+    mds->snapclient->prepare_destroy(diri->ino(), snapid,
+				     &mdr->more()->stid, &mdr->more()->snapidbl,
+				     new C_MDS_RetryRequest(mds->mdcache, mdr));
+    return;
+  }
+  version_t stid = mdr->more()->stid;
+  bufferlist::iterator p = mdr->more()->snapidbl.begin();
+  snapid_t seq;
+  ::decode(seq, p);  
+  dout(10) << " stid is " << stid << ", seq is " << seq << dendl;
+
+  // journal
+  inode_t *pi = diri->project_inode();
+  pi->ctime = ceph_clock_now(g_ceph_context);
+  pi->version = diri->pre_dirty();
+  
+  mdr->ls = mdlog->get_current_segment();
+  EUpdate *le = new EUpdate(mdlog, "rmsnap");
+  mdlog->start_entry(le);
+  
+  // project the snaprealm
+  sr_t *newnode = diri->project_snaprealm();
+  newnode->snaps.erase(snapid);
+  newnode->seq = seq;
+  newnode->last_destroyed = seq;
+
+  le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  le->metablob.add_table_transaction(TABLE_SNAP, stid);
+  mdcache->predirty_journal_parents(mdr, &le->metablob, diri, 0, PREDIRTY_PRIMARY, false);
+  mdcache->journal_dirty_inode(mdr.get(), &le->metablob, diri);
+
+  mdlog->submit_entry(le, new C_MDS_rmsnap_finish(mds, mdr, diri, snapid));
+  mdlog->flush();
+}
+
+void Server::_rmsnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid)
+{
+  dout(10) << "_rmsnap_finish " << *mdr << " " << snapid << dendl;
+  snapid_t stid = mdr->more()->stid;
+  bufferlist::iterator p = mdr->more()->snapidbl.begin();
+  snapid_t seq;
+  ::decode(seq, p);  
+
+  diri->pop_and_dirty_projected_inode(mdr->ls);
+  mdr->apply();
+
+  mds->snapclient->commit(stid, mdr->ls);
+
+  dout(10) << "snaprealm now " << *diri->snaprealm << dendl;
+
+  mdcache->do_realm_invalidate_and_update_notify(diri, CEPH_SNAP_OP_DESTROY);
+
+  // yay
+  mdr->in[0] = diri;
+  MClientReply *reply = new MClientReply(mdr->client_request, 0);
+  reply_request(mdr, reply);
+}
+
+
diff --git a/ceph/src/mds/Server.h b/ceph/src/mds/Server.h
new file mode 100644
index 00000000..95a5ae42
--- /dev/null
+++ b/ceph/src/mds/Server.h
@@ -0,0 +1,259 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_SERVER_H
+#define CEPH_MDS_SERVER_H
+
+#include "MDS.h"
+
+class PerfCounters;
+class LogEvent;
+class EMetaBlob;
+class EUpdate;
+class MMDSSlaveRequest;
+struct SnapInfo;
+
+struct MutationImpl;
+struct MDRequestImpl;
+typedef ceph::shared_ptr MutationRef;
+typedef ceph::shared_ptr MDRequestRef;
+
+enum {
+  l_mdss_first = 1000,
+  l_mdss_hcreq,
+  l_mdss_hsreq,
+  l_mdss_hcsess,
+  l_mdss_dcreq,
+  l_mdss_dsreq,
+  l_mdss_last,
+};
+
+class Server {
+  MDS *mds;
+  MDCache *mdcache;
+  MDLog *mdlog;
+  Messenger *messenger;
+  PerfCounters *logger;
+
+public:
+  int failed_reconnects;
+
+  bool terminating_sessions;
+
+  Server(MDS *m) : 
+    mds(m), 
+    mdcache(mds->mdcache), mdlog(mds->mdlog),
+    messenger(mds->messenger),
+    logger(0),
+    failed_reconnects(0),
+    terminating_sessions(false) {
+  }
+  ~Server() {
+    g_ceph_context->get_perfcounters_collection()->remove(logger);
+    delete logger;
+  }
+
+  void create_logger();
+
+  // message handler
+  void dispatch(Message *m);
+
+
+  // -- sessions and recovery --
+  utime_t  reconnect_start;
+  set client_reconnect_gather;  // clients i need a reconnect msg from.
+
+  Session *get_session(Message *m);
+  void handle_client_session(class MClientSession *m);
+  void _session_logged(Session *session, uint64_t state_seq, 
+		       bool open, version_t pv, interval_set& inos,version_t piv);
+  version_t prepare_force_open_sessions(map &cm,
+					map& sseqmap);
+  void finish_force_open_sessions(map &cm,
+				  map& sseqmap,
+				  bool dec_import=true);
+  void flush_client_sessions(set& client_set, C_GatherBuilder& gather);
+  void finish_flush_session(Session *session, version_t seq);
+  void terminate_sessions();
+  void find_idle_sessions();
+  void kill_session(Session *session);
+  void journal_close_session(Session *session, int state);
+  void reconnect_clients();
+  void handle_client_reconnect(class MClientReconnect *m);
+  //void process_reconnect_cap(CInode *in, int from, ceph_mds_cap_reconnect& capinfo);
+  void reconnect_gather_finish();
+  void reconnect_tick();
+  void recover_filelocks(CInode *in, bufferlist locks, int64_t client);
+
+  void recall_client_state(float ratio);
+
+  // -- requests --
+  void handle_client_request(MClientRequest *m);
+
+  void journal_and_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn,
+			 LogEvent *le, Context *fin);
+  void dispatch_client_request(MDRequestRef& mdr);
+  void early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn);
+  void reply_request(MDRequestRef& mdr, int r = 0, CInode *tracei = 0, CDentry *tracedn = 0);
+  void reply_request(MDRequestRef& mdr, MClientReply *reply, CInode *tracei = 0, CDentry *tracedn = 0);
+  void set_trace_dist(Session *session, MClientReply *reply, CInode *in, CDentry *dn,
+		      snapid_t snapid,
+		      int num_dentries_wanted,
+		      MDRequestRef& mdr);
+
+  void encode_empty_dirstat(bufferlist& bl);
+  void encode_infinite_lease(bufferlist& bl);
+  void encode_null_lease(bufferlist& bl);
+
+  void handle_slave_request(MMDSSlaveRequest *m);
+  void handle_slave_request_reply(MMDSSlaveRequest *m);
+  void dispatch_slave_request(MDRequestRef& mdr);
+  void handle_slave_auth_pin(MDRequestRef& mdr);
+  void handle_slave_auth_pin_ack(MDRequestRef& mdr, MMDSSlaveRequest *ack);
+
+  // some helpers
+  CDir *validate_dentry_dir(MDRequestRef& mdr, CInode *diri, const string& dname);
+  CDir *traverse_to_auth_dir(MDRequestRef& mdr, vector &trace, filepath refpath);
+  CDentry *prepare_null_dentry(MDRequestRef& mdr, CDir *dir, const string& dname, bool okexist=false);
+  CDentry *prepare_stray_dentry(MDRequestRef& mdr, CInode *in);
+  CInode* prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino, unsigned mode,
+			    ceph_file_layout *layout=NULL);
+  void journal_allocated_inos(MDRequestRef& mdr, EMetaBlob *blob);
+  void apply_allocated_inos(MDRequestRef& mdr);
+
+  CInode* rdlock_path_pin_ref(MDRequestRef& mdr, int n, set& rdlocks, bool want_auth,
+			      bool no_want_auth=false,
+			      ceph_file_layout **layout=NULL,
+			      bool no_lookup=false);
+  CDentry* rdlock_path_xlock_dentry(MDRequestRef& mdr, int n,
+                                    set& rdlocks,
+                                    set& wrlocks,
+				    set& xlocks, bool okexist,
+				    bool mustexist, bool alwaysxlock,
+				    ceph_file_layout **layout=NULL);
+
+  CDir* try_open_auth_dirfrag(CInode *diri, frag_t fg, MDRequestRef& mdr);
+
+
+  // requests on existing inodes.
+  void handle_client_getattr(MDRequestRef& mdr, bool is_lookup);
+  void handle_client_lookup_ino(MDRequestRef& mdr,
+				bool want_parent, bool want_dentry);
+  void _lookup_ino_2(MDRequestRef& mdr, int r);
+  void handle_client_readdir(MDRequestRef& mdr);
+  void handle_client_file_setlock(MDRequestRef& mdr);
+  void handle_client_file_readlock(MDRequestRef& mdr);
+
+  void handle_client_setattr(MDRequestRef& mdr);
+  void handle_client_setlayout(MDRequestRef& mdr);
+  void handle_client_setdirlayout(MDRequestRef& mdr);
+
+  int parse_layout_vxattr(string name, string value, ceph_file_layout *layout);
+  void handle_set_vxattr(MDRequestRef& mdr, CInode *cur,
+			 ceph_file_layout *dir_layout,
+			 set rdlocks,
+			 set wrlocks,
+			 set xlocks);
+  void handle_remove_vxattr(MDRequestRef& mdr, CInode *cur,
+			    set rdlocks,
+			    set wrlocks,
+			    set xlocks);
+  void handle_client_setxattr(MDRequestRef& mdr);
+  void handle_client_removexattr(MDRequestRef& mdr);
+
+  void handle_client_fsync(MDRequestRef& mdr);
+
+  // open
+  void handle_client_open(MDRequestRef& mdr);
+  void handle_client_openc(MDRequestRef& mdr);  // O_CREAT variant.
+  void do_open_truncate(MDRequestRef& mdr, int cmode);  // O_TRUNC variant.
+
+  // namespace changes
+  void handle_client_mknod(MDRequestRef& mdr);
+  void handle_client_mkdir(MDRequestRef& mdr);
+  void handle_client_symlink(MDRequestRef& mdr);
+
+  // link
+  void handle_client_link(MDRequestRef& mdr);
+  void _link_local(MDRequestRef& mdr, CDentry *dn, CInode *targeti);
+  void _link_local_finish(MDRequestRef& mdr,
+			  CDentry *dn, CInode *targeti,
+			  version_t, version_t);
+
+  void _link_remote(MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti);
+  void _link_remote_finish(MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti,
+			   version_t);
+
+  void handle_slave_link_prep(MDRequestRef& mdr);
+  void _logged_slave_link(MDRequestRef& mdr, CInode *targeti);
+  void _commit_slave_link(MDRequestRef& mdr, int r, CInode *targeti);
+  void _committed_slave(MDRequestRef& mdr);  // use for rename, too
+  void handle_slave_link_prep_ack(MDRequestRef& mdr, MMDSSlaveRequest *m);
+  void do_link_rollback(bufferlist &rbl, int master, MDRequestRef& mdr);
+  void _link_rollback_finish(MutationRef& mut, MDRequestRef& mdr);
+
+  // unlink
+  void handle_client_unlink(MDRequestRef& mdr);
+  bool _dir_is_nonempty_unlocked(MDRequestRef& mdr, CInode *rmdiri);
+  bool _dir_is_nonempty(MDRequestRef& mdr, CInode *rmdiri);
+  void _unlink_local(MDRequestRef& mdr, CDentry *dn, CDentry *straydn);
+  void _unlink_local_finish(MDRequestRef& mdr,
+			    CDentry *dn, CDentry *straydn,
+			    version_t);
+  bool _rmdir_prepare_witness(MDRequestRef& mdr, int who, CDentry *dn, CDentry *straydn);
+  void handle_slave_rmdir_prep(MDRequestRef& mdr);
+  void _logged_slave_rmdir(MDRequestRef& mdr, CDentry *srcdn, CDentry *straydn);
+  void _commit_slave_rmdir(MDRequestRef& mdr, int r);
+  void handle_slave_rmdir_prep_ack(MDRequestRef& mdr, MMDSSlaveRequest *ack);
+  void do_rmdir_rollback(bufferlist &rbl, int master, MDRequestRef& mdr);
+  void _rmdir_rollback_finish(MDRequestRef& mdr, metareqid_t reqid, CDentry *dn, CDentry *straydn);
+
+  // rename
+  void handle_client_rename(MDRequestRef& mdr);
+  void _rename_finish(MDRequestRef& mdr,
+		      CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+
+  void handle_client_lssnap(MDRequestRef& mdr);
+  void handle_client_mksnap(MDRequestRef& mdr);
+  void _mksnap_finish(MDRequestRef& mdr, CInode *diri, SnapInfo &info);
+  void handle_client_rmsnap(MDRequestRef& mdr);
+  void _rmsnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid);
+
+  // helpers
+  bool _rename_prepare_witness(MDRequestRef& mdr, int who, set &witnesse,
+			       CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+  version_t _rename_prepare_import(MDRequestRef& mdr, CDentry *srcdn, bufferlist *client_map_bl);
+  bool _need_force_journal(CInode *diri, bool empty);
+  void _rename_prepare(MDRequestRef& mdr,
+		       EMetaBlob *metablob, bufferlist *client_map_bl,
+		       CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+  /* set not_journaling=true if you're going to discard the results --
+   * this bypasses the asserts to make sure we're journaling the right
+   * things on the right nodes */
+  void _rename_apply(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+
+  // slaving
+  void handle_slave_rename_prep(MDRequestRef& mdr);
+  void handle_slave_rename_prep_ack(MDRequestRef& mdr, MMDSSlaveRequest *m);
+  void handle_slave_rename_notify_ack(MDRequestRef& mdr, MMDSSlaveRequest *m);
+  void _slave_rename_sessions_flushed(MDRequestRef& mdr);
+  void _logged_slave_rename(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+  void _commit_slave_rename(MDRequestRef& mdr, int r, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+  void do_rename_rollback(bufferlist &rbl, int master, MDRequestRef& mdr, bool finish_mdr=false);
+  void _rename_rollback_finish(MutationRef& mut, MDRequestRef& mdr, CDentry *srcdn, version_t srcdnpv,
+			       CDentry *destdn, CDentry *staydn, bool finish_mdr);
+
+};
+
+#endif
diff --git a/ceph/src/mds/SessionMap.cc b/ceph/src/mds/SessionMap.cc
new file mode 100644
index 00000000..1b270290
--- /dev/null
+++ b/ceph/src/mds/SessionMap.cc
@@ -0,0 +1,269 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MDS.h"
+#include "MDCache.h"
+#include "SessionMap.h"
+#include "osdc/Filer.h"
+
+#include "common/config.h"
+#include "common/errno.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".sessionmap "
+
+
+void SessionMap::dump()
+{
+  dout(10) << "dump" << dendl;
+  for (ceph::unordered_map::iterator p = session_map.begin();
+       p != session_map.end();
+       ++p) 
+    dout(10) << p->first << " " << p->second
+	     << " state " << p->second->get_state_name()
+	     << " completed " << p->second->info.completed_requests
+	     << " prealloc_inos " << p->second->info.prealloc_inos
+	     << " used_ions " << p->second->info.used_inos
+	     << dendl;
+}
+
+
+// ----------------
+// LOAD
+
+
+object_t SessionMap::get_object_name()
+{
+  char s[30];
+  snprintf(s, sizeof(s), "mds%d_sessionmap", mds->whoami);
+  return object_t(s);
+}
+
+class C_SM_Load : public Context {
+  SessionMap *sessionmap;
+public:
+  bufferlist bl;
+  C_SM_Load(SessionMap *cm) : sessionmap(cm) {}
+  void finish(int r) {
+    sessionmap->_load_finish(r, bl);
+  }
+};
+
+void SessionMap::load(Context *onload)
+{
+  dout(10) << "load" << dendl;
+
+  if (onload)
+    waiting_for_load.push_back(onload);
+  
+  C_SM_Load *c = new C_SM_Load(this);
+  object_t oid = get_object_name();
+  object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+  mds->objecter->read_full(oid, oloc, CEPH_NOSNAP, &c->bl, 0, c);
+}
+
+void SessionMap::_load_finish(int r, bufferlist &bl)
+{ 
+  bufferlist::iterator blp = bl.begin();
+  if (r < 0) {
+    derr << "_load_finish got " << cpp_strerror(r) << dendl;
+    assert(0 == "failed to load sessionmap");
+  }
+  dump();
+  decode(blp);  // note: this sets last_cap_renew = now()
+  dout(10) << "_load_finish v " << version 
+	   << ", " << session_map.size() << " sessions, "
+	   << bl.length() << " bytes"
+	   << dendl;
+  projected = committing = committed = version;
+  dump();
+  finish_contexts(g_ceph_context, waiting_for_load);
+}
+
+
+// ----------------
+// SAVE
+
+class C_SM_Save : public Context {
+  SessionMap *sessionmap;
+  version_t version;
+public:
+  C_SM_Save(SessionMap *cm, version_t v) : sessionmap(cm), version(v) {}
+  void finish(int r) {
+    assert(r == 0);
+    sessionmap->_save_finish(version);
+  }
+};
+
+void SessionMap::save(Context *onsave, version_t needv)
+{
+  dout(10) << "save needv " << needv << ", v " << version << dendl;
+ 
+  if (needv && committing >= needv) {
+    assert(committing > committed);
+    commit_waiters[committing].push_back(onsave);
+    return;
+  }
+
+  commit_waiters[version].push_back(onsave);
+  
+  bufferlist bl;
+  
+  encode(bl);
+  committing = version;
+  SnapContext snapc;
+  object_t oid = get_object_name();
+  object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+
+  mds->objecter->write_full(oid, oloc,
+			    snapc,
+			    bl, ceph_clock_now(g_ceph_context), 0,
+			    NULL, new C_SM_Save(this, version));
+}
+
+void SessionMap::_save_finish(version_t v)
+{
+  dout(10) << "_save_finish v" << v << dendl;
+  committed = v;
+
+  finish_contexts(g_ceph_context, commit_waiters[v]);
+  commit_waiters.erase(v);
+}
+
+
+// -------------------
+
+void SessionMap::encode(bufferlist& bl) const
+{
+  uint64_t pre = -1;     // for 0.19 compatibility; we forgot an encoding prefix.
+  ::encode(pre, bl);
+
+  ENCODE_START(3, 3, bl);
+  ::encode(version, bl);
+
+  for (ceph::unordered_map::const_iterator p = session_map.begin(); 
+       p != session_map.end(); 
+       ++p) {
+    if (p->second->is_open() ||
+	p->second->is_closing() ||
+	p->second->is_stale() ||
+	p->second->is_killing()) {
+      ::encode(p->first, bl);
+      p->second->info.encode(bl);
+    }
+  }
+  ENCODE_FINISH(bl);
+}
+
+void SessionMap::decode(bufferlist::iterator& p)
+{
+  utime_t now = ceph_clock_now(g_ceph_context);
+  uint64_t pre;
+  ::decode(pre, p);
+  if (pre == (uint64_t)-1) {
+    DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, p);
+    assert(struct_v >= 2);
+    
+    ::decode(version, p);
+    
+    while (!p.end()) {
+      entity_inst_t inst;
+      ::decode(inst.name, p);
+      Session *s = get_or_add_session(inst);
+      if (s->is_closed())
+	set_state(s, Session::STATE_OPEN);
+      s->info.decode(p);
+    }
+
+    DECODE_FINISH(p);
+  } else {
+    // --- old format ----
+    version = pre;
+
+    // this is a meaningless upper bound.  can be ignored.
+    __u32 n;
+    ::decode(n, p);
+    
+    while (n-- && !p.end()) {
+      bufferlist::iterator p2 = p;
+      Session *s = new Session;
+      s->info.decode(p);
+      if (session_map.count(s->info.inst.name)) {
+	// eager client connected too fast!  aie.
+	dout(10) << " already had session for " << s->info.inst.name << ", recovering" << dendl;
+	entity_name_t n = s->info.inst.name;
+	delete s;
+	s = session_map[n];
+	p = p2;
+	s->info.decode(p);
+      } else {
+	session_map[s->info.inst.name] = s;
+      }
+      set_state(s, Session::STATE_OPEN);
+      s->last_cap_renew = now;
+    }
+  }
+}
+
+void SessionMap::dump(Formatter *f) const
+{
+  f->open_array_section("Sessions");
+  for (ceph::unordered_map::const_iterator p = session_map.begin();
+       p != session_map.end();
+       ++p)  {
+    f->open_object_section("Session");
+    f->open_object_section("entity name");
+    p->first.dump(f);
+    f->close_section(); // entity name
+    f->open_object_section("Session info");
+    p->second->info.dump(f);
+    f->close_section(); // Session info
+    f->close_section(); // Session
+  }
+  f->close_section(); // Sessions
+}
+
+void SessionMap::generate_test_instances(list& ls)
+{
+  // pretty boring for now
+  ls.push_back(new SessionMap(NULL));
+}
+
+void SessionMap::wipe()
+{
+  dout(1) << "wipe start" << dendl;
+  dump();
+  while (!session_map.empty()) {
+    Session *s = session_map.begin()->second;
+    remove_session(s);
+  }
+  version = ++projected;
+  dout(1) << "wipe result" << dendl;
+  dump();
+  dout(1) << "wipe done" << dendl;
+}
+
+void SessionMap::wipe_ino_prealloc()
+{
+  for (ceph::unordered_map::iterator p = session_map.begin(); 
+       p != session_map.end(); 
+       ++p) {
+    p->second->pending_prealloc_inos.clear();
+    p->second->info.prealloc_inos.clear();
+    p->second->info.used_inos.clear();
+  }
+  projected = ++version;
+}
diff --git a/ceph/src/mds/SessionMap.h b/ceph/src/mds/SessionMap.h
new file mode 100644
index 00000000..ac7fd46d
--- /dev/null
+++ b/ceph/src/mds/SessionMap.h
@@ -0,0 +1,401 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_SESSIONMAP_H
+#define CEPH_MDS_SESSIONMAP_H
+
+#include 
+using std::set;
+
+#include "include/unordered_map.h"
+
+#include "include/Context.h"
+#include "include/xlist.h"
+#include "include/elist.h"
+#include "include/interval_set.h"
+#include "mdstypes.h"
+
+class CInode;
+struct MDRequestImpl;
+
+#include "CInode.h"
+#include "Capability.h"
+#include "msg/Message.h"
+
+
+/* 
+ * session
+ */
+
+class Session : public RefCountedObject {
+  // -- state etc --
+public:
+  /*
+                    
+         <-- closed <------------+
+             ^         |                  |
+             |         v                  |
+          killing <-- opening <----+      |
+             ^         |           |      |
+             |         v           |      |
+           stale <--> open --> closing ---+
+
+    + additional dimension of 'importing' (with counter)
+
+  */
+  enum {
+    STATE_CLOSED = 0,
+    STATE_OPENING = 1,   // journaling open
+    STATE_OPEN = 2,
+    STATE_CLOSING = 3,   // journaling close
+    STATE_STALE = 4,
+    STATE_KILLING = 5
+  };
+
+  const char *get_state_name(int s) {
+    switch (s) {
+    case STATE_CLOSED: return "closed";
+    case STATE_OPENING: return "opening";
+    case STATE_OPEN: return "open";
+    case STATE_CLOSING: return "closing";
+    case STATE_STALE: return "stale";
+    case STATE_KILLING: return "killing";
+    default: return "???";
+    }
+  }
+
+private:
+  int state;
+  uint64_t state_seq;
+  int importing_count;
+  friend class SessionMap;
+public:
+  session_info_t info;                         ///< durable bits
+
+  ConnectionRef connection;
+  xlist::item item_session_list;
+
+  list preopen_out_queue;  ///< messages for client, queued before they connect
+
+  elist requests;
+
+  interval_set pending_prealloc_inos; // journaling prealloc, will be added to prealloc_inos
+
+  inodeno_t next_ino() {
+    if (info.prealloc_inos.empty())
+      return 0;
+    return info.prealloc_inos.range_start();
+  }
+  inodeno_t take_ino(inodeno_t ino = 0) {
+    assert(!info.prealloc_inos.empty());
+
+    if (ino) {
+      if (info.prealloc_inos.contains(ino))
+	info.prealloc_inos.erase(ino);
+      else
+	ino = 0;
+    }
+    if (!ino) {
+      ino = info.prealloc_inos.range_start();
+      info.prealloc_inos.erase(ino);
+    }
+    info.used_inos.insert(ino, 1);
+    return ino;
+  }
+  int get_num_projected_prealloc_inos() {
+    return info.prealloc_inos.size() + pending_prealloc_inos.size();
+  }
+
+  client_t get_client() {
+    return info.get_client();
+  }
+
+  int get_state() { return state; }
+  const char *get_state_name() { return get_state_name(state); }
+  uint64_t get_state_seq() { return state_seq; }
+  bool is_closed() { return state == STATE_CLOSED; }
+  bool is_opening() { return state == STATE_OPENING; }
+  bool is_open() { return state == STATE_OPEN; }
+  bool is_closing() { return state == STATE_CLOSING; }
+  bool is_stale() { return state == STATE_STALE; }
+  bool is_killing() { return state == STATE_KILLING; }
+
+  void inc_importing() {
+    ++importing_count;
+  }
+  void dec_importing() {
+    assert(importing_count);
+    --importing_count;
+  }
+  bool is_importing() { return importing_count > 0; }
+
+  // -- caps --
+private:
+  version_t cap_push_seq;        // cap push seq #
+  map > waitfor_flush; // flush session messages
+public:
+  xlist caps;     // inodes with caps; front=most recently used
+  xlist leases;  // metadata leases to clients
+  utime_t last_cap_renew;
+
+public:
+  version_t inc_push_seq() { return ++cap_push_seq; }
+  version_t get_push_seq() const { return cap_push_seq; }
+
+  version_t wait_for_flush(Context* c) {
+    waitfor_flush[get_push_seq()].push_back(c);
+    return get_push_seq();
+  }
+  void finish_flush(version_t seq, list& ls) {
+    while (!waitfor_flush.empty()) {
+      if (waitfor_flush.begin()->first > seq)
+	break;
+      ls.splice(ls.end(), waitfor_flush.begin()->second);
+      waitfor_flush.erase(waitfor_flush.begin());
+    }
+  }
+
+  void add_cap(Capability *cap) {
+    caps.push_back(&cap->item_session_caps);
+  }
+  void touch_lease(ClientLease *r) {
+    leases.push_back(&r->item_session_lease);
+  }
+
+  // -- leases --
+  uint32_t lease_seq;
+
+  // -- completed requests --
+private:
+
+
+public:
+  void add_completed_request(ceph_tid_t t, inodeno_t created) {
+    info.completed_requests[t] = created;
+  }
+  void trim_completed_requests(ceph_tid_t mintid) {
+    // trim
+    while (!info.completed_requests.empty() && 
+	   (mintid == 0 || info.completed_requests.begin()->first < mintid))
+      info.completed_requests.erase(info.completed_requests.begin());
+  }
+  bool have_completed_request(ceph_tid_t tid, inodeno_t *pcreated) const {
+    map::const_iterator p = info.completed_requests.find(tid);
+    if (p == info.completed_requests.end())
+      return false;
+    if (pcreated)
+      *pcreated = p->second;
+    return true;
+  }
+
+
+  Session() : 
+    state(STATE_CLOSED), state_seq(0), importing_count(0),
+    connection(NULL), item_session_list(this),
+    requests(0),  // member_offset passed to front() manually
+    cap_push_seq(0),
+    lease_seq(0) { }
+  ~Session() {
+    assert(!item_session_list.is_on_list());
+    while (!preopen_out_queue.empty()) {
+      preopen_out_queue.front()->put();
+      preopen_out_queue.pop_front();
+    }
+  }
+
+  void clear() {
+    pending_prealloc_inos.clear();
+    info.clear_meta();
+
+    cap_push_seq = 0;
+    last_cap_renew = utime_t();
+
+  }
+
+};
+
+/*
+ * session map
+ */
+
+class MDS;
+
+class SessionMap {
+private:
+  MDS *mds;
+  ceph::unordered_map session_map;
+public:
+  map* > by_state;
+  
+public:  // i am lazy
+  version_t version, projected, committing, committed;
+  map > commit_waiters;
+
+public:
+  SessionMap(MDS *m) : mds(m), 
+		       version(0), projected(0), committing(0), committed(0) 
+  { }
+  
+  //for the dencoder
+  SessionMap() : mds(NULL), version(0), projected(0),
+		 committing(0), committed(0) {}
+    
+  // sessions
+  bool empty() { return session_map.empty(); }
+
+  bool is_any_state(int state) {
+    map* >::iterator p = by_state.find(state);
+    if (p == by_state.end() || p->second->empty())
+      return false;
+    return true;
+  }
+
+  bool have_unclosed_sessions() {
+    return
+      is_any_state(Session::STATE_OPENING) ||
+      is_any_state(Session::STATE_OPENING) ||
+      is_any_state(Session::STATE_OPEN) ||
+      is_any_state(Session::STATE_CLOSING) ||
+      is_any_state(Session::STATE_STALE) ||
+      is_any_state(Session::STATE_KILLING);
+  }
+  bool have_session(entity_name_t w) {
+    return session_map.count(w);
+  }
+  Session* get_session(entity_name_t w) {
+    if (session_map.count(w))
+      return session_map[w];
+    return 0;
+  }
+  Session* get_or_add_session(const entity_inst_t& i) {
+    Session *s;
+    if (session_map.count(i.name)) {
+      s = session_map[i.name];
+    } else {
+      s = session_map[i.name] = new Session;
+      s->info.inst = i;
+      s->last_cap_renew = ceph_clock_now(g_ceph_context);
+    }
+    return s;
+  }
+  void add_session(Session *s) {
+    assert(session_map.count(s->info.inst.name) == 0);
+    session_map[s->info.inst.name] = s;
+    if (by_state.count(s->state) == 0)
+      by_state[s->state] = new xlist;
+    by_state[s->state]->push_back(&s->item_session_list);
+    s->get();
+  }
+  void remove_session(Session *s) {
+    s->trim_completed_requests(0);
+    s->item_session_list.remove_myself();
+    session_map.erase(s->info.inst.name);
+    s->put();
+  }
+  void touch_session(Session *session) {
+    if (session->item_session_list.is_on_list()) {
+      if (by_state.count(session->state) == 0)
+	by_state[session->state] = new xlist;
+      by_state[session->state]->push_back(&session->item_session_list);
+      session->last_cap_renew = ceph_clock_now(g_ceph_context);
+    } else {
+      assert(0);  // hrm, should happen?
+    }
+  }
+  Session *get_oldest_session(int state) {
+    if (by_state.count(state) == 0 || by_state[state]->empty())
+      return 0;
+    return by_state[state]->front();
+  }
+  uint64_t set_state(Session *session, int s) {
+    if (session->state != s) {
+      session->state = s;
+      session->state_seq++;
+      if (by_state.count(s) == 0)
+	by_state[s] = new xlist;
+      by_state[s]->push_back(&session->item_session_list);
+    }
+    return session->state_seq;
+  }
+  void dump();
+
+  void get_client_set(set& s) {
+    for (ceph::unordered_map::iterator p = session_map.begin();
+	 p != session_map.end();
+	 ++p)
+      if (p->second->info.inst.name.is_client())
+	s.insert(p->second->info.inst.name.num());
+  }
+  void get_client_session_set(set& s) {
+    for (ceph::unordered_map::iterator p = session_map.begin();
+	 p != session_map.end();
+	 ++p)
+      if (p->second->info.inst.name.is_client())
+	s.insert(p->second);
+  }
+
+  void open_sessions(map& client_map) {
+    for (map::iterator p = client_map.begin(); 
+	 p != client_map.end(); 
+	 ++p) {
+      Session *s = get_or_add_session(p->second);
+      set_state(s, Session::STATE_OPEN);
+    }
+    version++;
+  }
+
+  // helpers
+  entity_inst_t& get_inst(entity_name_t w) {
+    assert(session_map.count(w));
+    return session_map[w]->info.inst;
+  }
+  version_t inc_push_seq(client_t client) {
+    return get_session(entity_name_t::CLIENT(client.v))->inc_push_seq();
+  }
+  version_t get_push_seq(client_t client) {
+    return get_session(entity_name_t::CLIENT(client.v))->get_push_seq();
+  }
+  bool have_completed_request(metareqid_t rid) {
+    Session *session = get_session(rid.name);
+    return session && session->have_completed_request(rid.tid, NULL);
+  }
+  void trim_completed_requests(entity_name_t c, ceph_tid_t tid) {
+    Session *session = get_session(c);
+    assert(session);
+    session->trim_completed_requests(tid);
+  }
+
+  void wipe();
+  void wipe_ino_prealloc();
+
+  // -- loading, saving --
+  inodeno_t ino;
+  list waiting_for_load;
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& blp);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  object_t get_object_name();
+
+  void load(Context *onload);
+  void _load_finish(int r, bufferlist &bl);
+  void save(Context *onsave, version_t needv=0);
+  void _save_finish(version_t v);
+ 
+};
+
+
+#endif
diff --git a/ceph/src/mds/SimpleLock.h b/ceph/src/mds/SimpleLock.h
new file mode 100644
index 00000000..cbaa9964
--- /dev/null
+++ b/ceph/src/mds/SimpleLock.h
@@ -0,0 +1,691 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_SIMPLELOCK_H
+#define CEPH_SIMPLELOCK_H
+
+// -- lock types --
+// see CEPH_LOCK_*
+
+inline const char *get_lock_type_name(int t) {
+  switch (t) {
+  case CEPH_LOCK_DN: return "dn";
+  case CEPH_LOCK_DVERSION: return "dversion";
+  case CEPH_LOCK_IVERSION: return "iversion";
+  case CEPH_LOCK_IFILE: return "ifile";
+  case CEPH_LOCK_IAUTH: return "iauth";
+  case CEPH_LOCK_ILINK: return "ilink";
+  case CEPH_LOCK_IDFT: return "idft";
+  case CEPH_LOCK_INEST: return "inest";
+  case CEPH_LOCK_IXATTR: return "ixattr";
+  case CEPH_LOCK_ISNAP: return "isnap";
+  case CEPH_LOCK_INO: return "ino";
+  case CEPH_LOCK_IFLOCK: return "iflock";
+  case CEPH_LOCK_IPOLICY: return "ipolicy";
+  default: assert(0); return 0;
+  }
+}
+
+#include "include/memory.h"
+struct MutationImpl;
+typedef ceph::shared_ptr MutationRef;
+
+extern "C" {
+#include "locks.h"
+}
+
+
+#define CAP_ANY     0
+#define CAP_LONER   1
+#define CAP_XLOCKER 2
+
+struct LockType {
+  int type;
+  const sm_t *sm;
+
+  LockType(int t) : type(t) {
+    switch (type) {
+    case CEPH_LOCK_DN:
+    case CEPH_LOCK_IAUTH:
+    case CEPH_LOCK_ILINK:
+    case CEPH_LOCK_IXATTR:
+    case CEPH_LOCK_ISNAP:
+    case CEPH_LOCK_IFLOCK:
+    case CEPH_LOCK_IPOLICY:
+      sm = &sm_simplelock;
+      break;
+    case CEPH_LOCK_IDFT:
+    case CEPH_LOCK_INEST:
+      sm = &sm_scatterlock;
+      break;
+    case CEPH_LOCK_IFILE:
+      sm = &sm_filelock;
+      break;
+    case CEPH_LOCK_DVERSION:
+    case CEPH_LOCK_IVERSION:
+      sm = &sm_locallock;
+      break;
+    default:
+      sm = 0;
+    }
+  }
+
+};
+
+
+class SimpleLock {
+public:
+  LockType *type;
+  
+  const char *get_state_name(int n) const {
+    switch (n) {
+    case LOCK_UNDEF: return "UNDEF";
+    case LOCK_SYNC: return "sync";
+    case LOCK_LOCK: return "lock";
+
+    case LOCK_PREXLOCK: return "prexlock";
+    case LOCK_XLOCK: return "xlock";
+    case LOCK_XLOCKDONE: return "xlockdone";
+    case LOCK_LOCK_XLOCK: return "lock->xlock";
+
+    case LOCK_SYNC_LOCK: return "sync->lock";
+    case LOCK_LOCK_SYNC: return "lock->sync";
+    case LOCK_REMOTEXLOCK: return "remote_xlock";
+    case LOCK_EXCL: return "excl";
+    case LOCK_EXCL_SYNC: return "excl->sync";
+    case LOCK_EXCL_LOCK: return "excl->lock";
+    case LOCK_SYNC_EXCL: return "sync->excl";
+    case LOCK_LOCK_EXCL: return "lock->excl";      
+
+    case LOCK_XSYN: return "xsyn";
+    case LOCK_XSYN_EXCL: return "xsyn->excl";
+    case LOCK_EXCL_XSYN: return "excl->xsyn";
+    case LOCK_XSYN_SYNC: return "xsyn->sync";
+
+    case LOCK_SYNC_MIX: return "sync->mix";
+    case LOCK_SYNC_MIX2: return "sync->mix(2)";
+    case LOCK_LOCK_TSYN: return "lock->tsyn";
+      
+    case LOCK_MIX_LOCK: return "mix->lock";
+    case LOCK_MIX_LOCK2: return "mix->lock(2)";
+    case LOCK_MIX: return "mix";
+    case LOCK_MIX_TSYN: return "mix->tsyn";
+      
+    case LOCK_TSYN_MIX: return "tsyn->mix";
+    case LOCK_TSYN_LOCK: return "tsyn->lock";
+    case LOCK_TSYN: return "tsyn";
+
+    case LOCK_MIX_SYNC: return "mix->sync";
+    case LOCK_MIX_SYNC2: return "mix->sync(2)";
+    case LOCK_EXCL_MIX: return "excl->mix";
+    case LOCK_MIX_EXCL: return "mix->excl";
+
+    case LOCK_PRE_SCAN: return "*->scan";
+    case LOCK_SCAN: return "scan";
+
+    case LOCK_SNAP_SYNC: return "snap->sync";
+
+    default: assert(0); return 0;
+    }
+  }
+
+
+  // waiting
+  static const uint64_t WAIT_RD          = (1<<0);  // to read
+  static const uint64_t WAIT_WR          = (1<<1);  // to write
+  static const uint64_t WAIT_XLOCK       = (1<<2);  // to xlock   (** dup)
+  static const uint64_t WAIT_STABLE      = (1<<2);  // for a stable state
+  static const uint64_t WAIT_REMOTEXLOCK = (1<<3);  // for a remote xlock
+  static const int WAIT_BITS        = 4;
+  static const uint64_t WAIT_ALL         = ((1< gather_set;  // auth+rep.  >= 0 is mds, < 0 is client
+
+    // local state
+    int num_wrlock, num_xlock;
+    MutationRef xlock_by;
+    client_t xlock_by_client;
+    client_t excl_client;
+
+    bool empty() {
+      return
+	gather_set.empty() &&
+	num_wrlock == 0 &&
+	num_xlock == 0 &&
+	xlock_by.get() == NULL &&
+	xlock_by_client == -1 &&
+	excl_client == -1;
+    }
+
+    unstable_bits_t() : num_wrlock(0),
+			num_xlock(0),
+			xlock_by(),
+			xlock_by_client(-1),
+			excl_client(-1) {}
+  };
+
+  mutable unstable_bits_t *_unstable;
+
+  bool have_more() const { return _unstable ? true : false; }
+  unstable_bits_t *more() const {
+    if (!_unstable)
+      _unstable = new unstable_bits_t;
+    return _unstable;
+  }
+  void clear_more() {
+    if (_unstable) {
+      assert(_unstable->empty());
+      delete _unstable;
+      _unstable = NULL;
+    }
+  }
+  void try_clear_more() {
+    if (_unstable && _unstable->empty()) {
+      delete _unstable;
+      _unstable = NULL;
+    }
+  }
+
+public:
+
+  client_t get_excl_client() const {
+    return have_more() ? more()->excl_client : -1;
+  }
+  void set_excl_client(client_t c) {
+    if (c < 0 && !have_more())
+      return;  // default is -1
+    more()->excl_client = c;
+  }
+
+  SimpleLock(MDSCacheObject *o, LockType *lt) :
+    type(lt),
+    parent(o), 
+    state(LOCK_SYNC),
+    num_rdlock(0),
+    num_client_lease(0),
+    _unstable(NULL)
+  {}
+  virtual ~SimpleLock() {
+    delete _unstable;
+  }
+
+  virtual bool is_scatterlock() const {
+    return false;
+  }
+  virtual bool is_locallock() const {
+    return false;
+  }
+
+  // parent
+  MDSCacheObject *get_parent() { return parent; }
+  int get_type() const { return type->type; }
+  const sm_t* get_sm() const { return type->sm; }
+
+  int get_wait_shift() const {
+    switch (get_type()) {
+    case CEPH_LOCK_DN:       return 8;
+    case CEPH_LOCK_DVERSION: return 8 + 1*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IAUTH:    return 8 + 2*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_ILINK:    return 8 + 3*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IDFT:     return 8 + 4*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IFILE:    return 8 + 5*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IVERSION: return 8 + 6*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IXATTR:   return 8 + 7*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_ISNAP:    return 8 + 8*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_INEST:    return 8 + 9*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IFLOCK:   return 8 +10*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IPOLICY:  return 8 +11*SimpleLock::WAIT_BITS;
+    default:
+      assert(0);
+    }
+  }
+
+  int get_cap_shift() const {
+    switch (get_type()) {
+    case CEPH_LOCK_IAUTH: return CEPH_CAP_SAUTH;
+    case CEPH_LOCK_ILINK: return CEPH_CAP_SLINK;
+    case CEPH_LOCK_IFILE: return CEPH_CAP_SFILE;
+    case CEPH_LOCK_IXATTR: return CEPH_CAP_SXATTR;
+    default: return 0;
+    }
+  }
+  int get_cap_mask() const {
+    switch (get_type()) {
+    case CEPH_LOCK_IFILE: return (1 << CEPH_CAP_FILE_BITS) - 1;
+    default: return (1 << CEPH_CAP_SIMPLE_BITS) - 1;
+    }
+  }
+
+  struct ptr_lt {
+    bool operator()(const SimpleLock* l, const SimpleLock* r) const {
+      // first sort by object type (dn < inode)
+      if ((l->type->type>CEPH_LOCK_DN) <  (r->type->type>CEPH_LOCK_DN)) return true;
+      if ((l->type->type>CEPH_LOCK_DN) == (r->type->type>CEPH_LOCK_DN)) {
+	// then sort by object
+	if (l->parent->is_lt(r->parent)) return true;
+	if (l->parent == r->parent) {
+	  // then sort by (inode) lock type
+	  if (l->type->type < r->type->type) return true;
+	}
+      }
+      return false;
+    }
+  };
+
+  void decode_locked_state(bufferlist& bl) {
+    parent->decode_lock_state(type->type, bl);
+  }
+  void encode_locked_state(bufferlist& bl) {
+    parent->encode_lock_state(type->type, bl);
+  }
+  void finish_waiters(uint64_t mask, int r=0) {
+    parent->finish_waiting(mask << get_wait_shift(), r);
+  }
+  void take_waiting(uint64_t mask, list& ls) {
+    parent->take_waiting(mask << get_wait_shift(), ls);
+  }
+  void add_waiter(uint64_t mask, Context *c) {
+    parent->add_waiter(mask << get_wait_shift(), c);
+  }
+  bool is_waiter_for(uint64_t mask) const {
+    return parent->is_waiter_for(mask << get_wait_shift());
+  }
+  
+  
+
+  // state
+  int get_state() const { return state; }
+  int set_state(int s) { 
+    state = s; 
+    //assert(!is_stable() || gather_set.size() == 0);  // gather should be empty in stable states.
+    return s;
+  }
+  void set_state_rejoin(int s, list& waiters) {
+    if (!is_stable() && get_parent()->is_auth()) {
+      state = s;
+      get_parent()->auth_unpin(this);
+    } else {
+      state = s;
+    }
+    if (is_stable())
+      take_waiting(SimpleLock::WAIT_ALL, waiters);
+  }
+
+  bool is_stable() const {
+    return get_sm()->states[state].next == 0;
+  }
+  int get_next_state() {
+    return get_sm()->states[state].next;
+  }
+
+
+  bool is_sync_and_unlocked() const {
+    return
+      get_state() == LOCK_SYNC &&
+      !is_rdlocked() &&
+      !is_leased() &&
+      !is_wrlocked() &&
+      !is_xlocked();
+  }
+
+
+  /*
+  bool fw_rdlock_to_auth() {
+    return get_sm()->states[state].can_rdlock == FW;
+  }
+  */
+  bool req_rdlock_from_auth() {
+    return get_sm()->states[state].can_rdlock == REQ;
+  }
+
+  // gather set
+  static set empty_gather_set;
+
+  const set& get_gather_set() const {
+    return have_more() ? more()->gather_set : empty_gather_set;
+  }
+
+  void init_gather() {
+    for (map::const_iterator p = parent->replicas_begin();
+	 p != parent->replicas_end(); 
+	 ++p)
+      more()->gather_set.insert(p->first);
+  }
+  bool is_gathering() const {
+    return have_more() && !more()->gather_set.empty();
+  }
+  bool is_gathering(int i) const {
+    return have_more() && more()->gather_set.count(i);
+  }
+  void clear_gather() {
+    if (have_more())
+      more()->gather_set.clear();
+  }
+  void remove_gather(int i) {
+    if (have_more())
+      more()->gather_set.erase(i);
+  }
+
+
+
+  virtual bool is_dirty() const { return false; }
+  virtual bool is_stale() const { return false; }
+  virtual bool is_flushing() const { return false; }
+  virtual bool is_flushed() const { return false; }
+  virtual void clear_flushed() { }
+
+  // can_*
+  bool can_lease(client_t client) const {
+    return get_sm()->states[state].can_lease == ANY ||
+      (get_sm()->states[state].can_lease == AUTH && parent->is_auth()) ||
+      (get_sm()->states[state].can_lease == XCL && client >= 0 && get_xlock_by_client() == client);
+  }
+  bool can_read(client_t client) const {
+    return get_sm()->states[state].can_read == ANY ||
+      (get_sm()->states[state].can_read == AUTH && parent->is_auth()) ||
+      (get_sm()->states[state].can_read == XCL && client >= 0 && get_xlock_by_client() == client);
+  }
+  bool can_read_projected(client_t client) const {
+    return get_sm()->states[state].can_read_projected == ANY ||
+      (get_sm()->states[state].can_read_projected == AUTH && parent->is_auth()) ||
+      (get_sm()->states[state].can_read_projected == XCL && client >= 0 && get_xlock_by_client() == client);
+  }
+  bool can_rdlock(client_t client) const {
+    return get_sm()->states[state].can_rdlock == ANY ||
+      (get_sm()->states[state].can_rdlock == AUTH && parent->is_auth()) ||
+      (get_sm()->states[state].can_rdlock == XCL && client >= 0 && get_xlock_by_client() == client);
+  }
+  bool can_wrlock(client_t client) const {
+    return get_sm()->states[state].can_wrlock == ANY ||
+      (get_sm()->states[state].can_wrlock == AUTH && parent->is_auth()) ||
+      (get_sm()->states[state].can_wrlock == XCL && client >= 0 && (get_xlock_by_client() == client ||
+								    get_excl_client() == client));
+  }
+  bool can_force_wrlock(client_t client) const {
+    return get_sm()->states[state].can_force_wrlock == ANY ||
+      (get_sm()->states[state].can_force_wrlock == AUTH && parent->is_auth()) ||
+      (get_sm()->states[state].can_force_wrlock == XCL && client >= 0 && (get_xlock_by_client() == client ||
+									  get_excl_client() == client));
+  }
+  bool can_xlock(client_t client) const {
+    return get_sm()->states[state].can_xlock == ANY ||
+      (get_sm()->states[state].can_xlock == AUTH && parent->is_auth()) ||
+      (get_sm()->states[state].can_xlock == XCL && client >= 0 && get_xlock_by_client() == client);
+  }
+
+  // rdlock
+  bool is_rdlocked() const { return num_rdlock > 0; }
+  int get_rdlock() { 
+    if (!num_rdlock)
+      parent->get(MDSCacheObject::PIN_LOCK);
+    return ++num_rdlock; 
+  }
+  int put_rdlock() {
+    assert(num_rdlock>0);
+    --num_rdlock;
+    if (num_rdlock == 0)
+      parent->put(MDSCacheObject::PIN_LOCK);
+    return num_rdlock;
+  }
+  int get_num_rdlocks() const {
+    return num_rdlock;
+  }
+
+  // wrlock
+  void get_wrlock(bool force=false) {
+    //assert(can_wrlock() || force);
+    if (more()->num_wrlock == 0)
+      parent->get(MDSCacheObject::PIN_LOCK);
+    ++more()->num_wrlock;
+  }
+  void put_wrlock() {
+    --more()->num_wrlock;
+    if (more()->num_wrlock == 0) {
+      parent->put(MDSCacheObject::PIN_LOCK);
+      try_clear_more();
+    }
+  }
+  bool is_wrlocked() const {
+    return have_more() && more()->num_wrlock > 0;
+  }
+  int get_num_wrlocks() const {
+    return have_more() ? more()->num_wrlock : 0;
+  }
+
+  // xlock
+  void get_xlock(MutationRef who, client_t client) { 
+    assert(get_xlock_by() == 0);
+    assert(state == LOCK_XLOCK || is_locallock() ||
+	   state == LOCK_LOCK /* if we are a slave */);
+    parent->get(MDSCacheObject::PIN_LOCK);
+    more()->num_xlock++;
+    more()->xlock_by = who; 
+    more()->xlock_by_client = client;
+  }
+  void set_xlock_done() {
+    assert(more()->xlock_by);
+    assert(state == LOCK_XLOCK || is_locallock() ||
+	   state == LOCK_LOCK /* if we are a slave */);
+    if (!is_locallock())
+      state = LOCK_XLOCKDONE;
+    more()->xlock_by.reset();
+  }
+  void put_xlock() {
+    assert(state == LOCK_XLOCK || state == LOCK_XLOCKDONE || is_locallock() ||
+	   state == LOCK_LOCK /* if we are a master of a slave */);
+    --more()->num_xlock;
+    parent->put(MDSCacheObject::PIN_LOCK);
+    if (more()->num_xlock == 0) {
+      more()->xlock_by.reset();
+      more()->xlock_by_client = -1;
+      try_clear_more();
+    }
+  }
+  bool is_xlocked() const {
+    return have_more() && more()->num_xlock > 0;
+  }
+  int get_num_xlocks() const {
+    return have_more() ? more()->num_xlock : 0;
+  }
+  client_t get_xlock_by_client() const {
+    return have_more() ? more()->xlock_by_client : -1;
+  }
+  bool is_xlocked_by_client(client_t c) const {
+    return have_more() ? more()->xlock_by_client == c : false;
+  }
+  MutationRef get_xlock_by() const {
+    return have_more() ? more()->xlock_by : MutationRef();
+  }
+  
+  // lease
+  void get_client_lease() {
+    num_client_lease++;
+  }
+  void put_client_lease() {
+    assert(num_client_lease > 0);
+    num_client_lease--;
+    if (num_client_lease == 0) {
+      try_clear_more();
+    }
+  }
+  bool is_leased() const {
+    return num_client_lease > 0;
+  }
+  int get_num_client_lease() const {
+    return num_client_lease;
+  }
+
+  bool is_used() const {
+    return is_xlocked() || is_rdlocked() || is_wrlocked() || num_client_lease;
+  }
+
+  // encode/decode
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(state, bl);
+    if (have_more())
+      ::encode(more()->gather_set, bl);
+    else
+      ::encode(empty_gather_set, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& p) {
+    DECODE_START(2, p);
+    ::decode(state, p);
+    set g;
+    ::decode(g, p);
+    if (!g.empty())
+      more()->gather_set.swap(g);
+    DECODE_FINISH(p);
+  }
+  void encode_state_for_replica(bufferlist& bl) const {
+    __s16 s = get_replica_state();
+    ::encode(s, bl);
+  }
+  void decode_state(bufferlist::iterator& p, bool is_new=true) {
+    __s16 s;
+    ::decode(s, p);
+    if (is_new)
+      state = s;
+  }
+  void decode_state_rejoin(bufferlist::iterator& p, list& waiters) {
+    __s16 s;
+    ::decode(s, p);
+    set_state_rejoin(s, waiters);
+  }
+
+
+  // caps
+  bool is_loner_mode() {
+    return get_sm()->states[state].loner;
+  }
+  int gcaps_allowed_ever() {
+    return parent->is_auth() ? get_sm()->allowed_ever_auth : get_sm()->allowed_ever_replica;
+  }
+  int gcaps_allowed(int who, int s=-1) {
+    if (s < 0) s = state;
+    if (parent->is_auth()) {
+      if (get_xlock_by_client() >= 0 && who == CAP_XLOCKER)
+	return get_sm()->states[s].xlocker_caps | get_sm()->states[s].caps; // xlocker always gets more
+      else if (is_loner_mode() && who == CAP_ANY)
+	return get_sm()->states[s].caps;
+      else 
+	return get_sm()->states[s].loner_caps | get_sm()->states[s].caps;  // loner always gets more
+    } else 
+      return get_sm()->states[s].replica_caps;
+  }
+  int gcaps_careful() {
+    if (get_num_wrlocks())
+      return get_sm()->careful;
+    return 0;
+  }
+
+
+  int gcaps_xlocker_mask(client_t client) {
+    if (client == get_xlock_by_client())
+      return type->type == CEPH_LOCK_IFILE ? 0xf : (CEPH_CAP_GSHARED|CEPH_CAP_GEXCL);
+    return 0;
+  }
+
+  // simplelock specifics
+  int get_replica_state() const {
+    return get_sm()->states[state].replica_state;
+  }
+  void export_twiddle() {
+    clear_gather();
+    state = get_replica_state();
+  }
+
+  /** replicate_relax
+   * called on first replica creation.
+   */
+  void replicate_relax() {
+    assert(parent->is_auth());
+    assert(!parent->is_replicated());
+    if (state == LOCK_LOCK && !is_used())
+      state = LOCK_SYNC;
+  }
+  bool remove_replica(int from) {
+    if (is_gathering(from)) {
+      remove_gather(from);
+      if (!is_gathering())
+	return true;
+    }
+    return false;
+  }
+  bool do_import(int from, int to) {
+    if (!is_stable()) {
+      remove_gather(from);
+      remove_gather(to);
+      if (!is_gathering())
+	return true;
+    }
+    if (!is_stable() && !is_gathering())
+      return true;
+    return false;
+  }
+
+  void _print(ostream& out) const {
+    out << get_lock_type_name(get_type()) << " ";
+    out << get_state_name(get_state());
+    if (!get_gather_set().empty())
+      out << " g=" << get_gather_set();
+    if (num_client_lease)
+      out << " l=" << num_client_lease;
+    if (is_rdlocked()) 
+      out << " r=" << get_num_rdlocks();
+    if (is_wrlocked()) 
+      out << " w=" << get_num_wrlocks();
+    if (is_xlocked()) {
+      out << " x=" << get_num_xlocks();
+      if (get_xlock_by())
+	out << " by " << get_xlock_by();
+    }
+    /*if (is_stable())
+      out << " stable";
+    else
+      out << " unstable";
+    */
+  }
+
+  virtual void print(ostream& out) const {
+    out << "(";
+    _print(out);
+    out << ")";
+  }
+};
+WRITE_CLASS_ENCODER(SimpleLock)
+
+inline ostream& operator<<(ostream& out, const SimpleLock& l) 
+{
+  l.print(out);
+  return out;
+}
+
+
+#endif
diff --git a/ceph/src/mds/SnapClient.h b/ceph/src/mds/SnapClient.h
new file mode 100644
index 00000000..fd52d2bf
--- /dev/null
+++ b/ceph/src/mds/SnapClient.h
@@ -0,0 +1,61 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_SNAPCLIENT_H
+#define CEPH_SNAPCLIENT_H
+
+#include "MDSTableClient.h"
+#include "snap.h"
+
+class Context;
+class MDS;
+class LogSegment;
+
+class SnapClient : public MDSTableClient {
+public:
+  SnapClient(MDS *m) : MDSTableClient(m, TABLE_SNAP) {}
+
+  void resend_queries() {}
+  void handle_query_result(MMDSTableRequest *m) {}
+
+  void prepare_create(inodeno_t dirino, const string& name, utime_t stamp,
+		      version_t *pstid, bufferlist *pbl, Context *onfinish) {
+    bufferlist bl;
+    __u32 op = TABLE_OP_CREATE;
+    ::encode(op, bl);
+    ::encode(dirino, bl);
+    ::encode(name, bl);
+    ::encode(stamp, bl);
+    _prepare(bl, pstid, pbl, onfinish);
+  }
+
+  void prepare_create_realm(inodeno_t ino, version_t *pstid, bufferlist *pbl, Context *onfinish) {
+    bufferlist bl;
+    __u32 op = TABLE_OP_CREATE;
+    ::encode(op, bl);
+    ::encode(ino, bl);
+    _prepare(bl, pstid, pbl, onfinish);
+  }
+
+  void prepare_destroy(inodeno_t ino, snapid_t snapid, version_t *pstid, bufferlist *pbl, Context *onfinish) {
+    bufferlist bl;
+    __u32 op = TABLE_OP_DESTROY;
+    ::encode(op, bl);
+    ::encode(ino, bl);
+    ::encode(snapid, bl);
+    _prepare(bl, pstid, pbl, onfinish);
+  }
+};
+
+#endif
diff --git a/ceph/src/mds/SnapRealm.cc b/ceph/src/mds/SnapRealm.cc
new file mode 100644
index 00000000..a87b7487
--- /dev/null
+++ b/ceph/src/mds/SnapRealm.cc
@@ -0,0 +1,488 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "SnapRealm.h"
+#include "MDCache.h"
+#include "MDS.h"
+
+#include "messages/MClientSnap.h"
+
+
+/*
+ * SnapRealm
+ */
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mdcache->mds->get_nodeid(), inode, srnode.seq, this)
+static ostream& _prefix(std::ostream *_dout, int whoami, CInode *inode,
+			uint64_t seq, SnapRealm *realm) {
+  return *_dout << " mds." << whoami
+		<< ".cache.snaprealm(" << inode->ino()
+		<< " seq " << seq << " " << realm << ") ";
+}
+
+ostream& operator<<(ostream& out, const SnapRealm& realm) 
+{
+  out << "snaprealm(" << realm.inode->ino()
+      << " seq " << realm.srnode.seq
+      << " lc " << realm.srnode.last_created
+      << " cr " << realm.srnode.created;
+  if (realm.srnode.created != realm.srnode.current_parent_since)
+    out << " cps " << realm.srnode.current_parent_since;
+  out << " snaps=" << realm.srnode.snaps;
+  if (realm.srnode.past_parents.size()) {
+    out << " past_parents=(";
+    for (map::const_iterator p = realm.srnode.past_parents.begin(); 
+	 p != realm.srnode.past_parents.end(); 
+	 ++p) {
+      if (p != realm.srnode.past_parents.begin()) out << ",";
+      out << p->second.first << "-" << p->first
+	  << "=" << p->second.ino;
+    }
+    out << ")";
+  }
+  out << " " << &realm << ")";
+  return out;
+}
+
+
+
+
+void SnapRealm::add_open_past_parent(SnapRealm *parent)
+{
+  open_past_parents[parent->inode->ino()] = parent;
+  parent->inode->get(CInode::PIN_PASTSNAPPARENT);
+}
+
+bool SnapRealm::_open_parents(Context *finish, snapid_t first, snapid_t last)
+{
+  dout(10) << "open_parents [" << first << "," << last << "]" << dendl;
+  if (open) 
+    return true;
+
+  // make sure my current parents' parents are open...
+  if (parent) {
+    dout(10) << " current parent [" << srnode.current_parent_since << ",head] is " << *parent
+	     << " on " << *parent->inode << dendl;
+    if (last >= srnode.current_parent_since &&
+	!parent->_open_parents(finish, MAX(first, srnode.current_parent_since), last))
+      return false;
+  }
+
+  // and my past parents too!
+  assert(srnode.past_parents.size() >= open_past_parents.size());
+  if (srnode.past_parents.size() > open_past_parents.size()) {
+    for (map::iterator p = srnode.past_parents.begin();
+	 p != srnode.past_parents.end();
+	 ++p) {    
+      dout(10) << " past_parent [" << p->second.first << "," << p->first << "] is "
+	       << p->second.ino << dendl;
+      CInode *parent = mdcache->get_inode(p->second.ino);
+      if (!parent) {
+	mdcache->open_remote_ino(p->second.ino, finish);
+	return false;
+      }
+      assert(parent->snaprealm);  // hmm!
+      if (!open_past_parents.count(p->second.ino)) {
+	add_open_past_parent(parent->snaprealm);
+      }
+      if (!parent->snaprealm->_open_parents(finish, p->second.first, p->first))
+	return false;
+    }
+  }
+
+  open = true;
+  return true;
+}
+
+bool SnapRealm::have_past_parents_open(snapid_t first, snapid_t last)
+{
+  dout(10) << "have_past_parents_open [" << first << "," << last << "]" << dendl;
+  if (open)
+    return true;
+
+  for (map::iterator p = srnode.past_parents.lower_bound(first);
+       p != srnode.past_parents.end();
+       ++p) {
+    if (p->second.first > last)
+      break;
+    dout(10) << " past parent [" << p->second.first << "," << p->first << "] was "
+	     << p->second.ino << dendl;
+    if (open_past_parents.count(p->second.ino) == 0) {
+      dout(10) << " past parent " << p->second.ino << " is not open" << dendl;
+      return false;
+    }
+    if (!open_past_parents[p->second.ino]->have_past_parents_open(MAX(first, p->second.first),
+								  MIN(last, p->first)))
+      return false;
+  }
+
+  open = true;
+  return true;
+}
+
+void SnapRealm::close_parents()
+{
+  for (map::iterator p = open_past_parents.begin();
+       p != open_past_parents.end();
+       ++p)
+    p->second->inode->put(CInode::PIN_PASTSNAPPARENT);
+  open_past_parents.clear();
+}
+
+
+/*
+ * get list of snaps for this realm.  we must include parents' snaps
+ * for the intervals during which they were our parent.
+ */
+void SnapRealm::build_snap_set(set &s,
+			       snapid_t& max_seq, snapid_t& max_last_created, snapid_t& max_last_destroyed,
+			       snapid_t first, snapid_t last)
+{
+  dout(10) << "build_snap_set [" << first << "," << last << "] on " << *this << dendl;
+
+  if (srnode.seq > max_seq)
+    max_seq = srnode.seq;
+  if (srnode.last_created > max_last_created)
+    max_last_created = srnode.last_created;
+  if (srnode.last_destroyed > max_last_destroyed)
+    max_last_destroyed = srnode.last_destroyed;
+
+  // include my snaps within interval [first,last]
+  for (map::iterator p = srnode.snaps.lower_bound(first); // first element >= first
+       p != srnode.snaps.end() && p->first <= last;
+       ++p)
+    s.insert(p->first);
+
+  // include snaps for parents during intervals that intersect [first,last]
+  for (map::iterator p = srnode.past_parents.lower_bound(first);
+       p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
+       ++p) {
+    CInode *oldparent = mdcache->get_inode(p->second.ino);
+    assert(oldparent);  // call open_parents first!
+    assert(oldparent->snaprealm);
+    oldparent->snaprealm->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
+					 MAX(first, p->second.first),
+					 MIN(last, p->first));
+  }
+  if (srnode.current_parent_since <= last && parent)
+    parent->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
+			   MAX(first, srnode.current_parent_since), last);
+}
+
+
+void SnapRealm::check_cache()
+{
+  if (cached_seq >= srnode.seq)
+    return;
+
+  cached_snaps.clear();
+  cached_snap_context.clear();
+
+  cached_last_created = srnode.last_created;
+  cached_last_destroyed = srnode.last_destroyed;
+  cached_seq = srnode.seq;
+  build_snap_set(cached_snaps, cached_seq, cached_last_created, cached_last_destroyed,
+		 0, CEPH_NOSNAP);
+
+  cached_snap_trace.clear();
+  build_snap_trace(cached_snap_trace);
+  
+  dout(10) << "check_cache rebuilt " << cached_snaps
+	   << " seq " << srnode.seq
+	   << " cached_seq " << cached_seq
+	   << " cached_last_created " << cached_last_created
+	   << " cached_last_destroyed " << cached_last_destroyed
+	   << ")" << dendl;
+}
+
+const set& SnapRealm::get_snaps()
+{
+  check_cache();
+  dout(10) << "get_snaps " << cached_snaps
+	   << " (seq " << srnode.seq << " cached_seq " << cached_seq << ")"
+	   << dendl;
+  return cached_snaps;
+}
+
+/*
+ * build vector in reverse sorted order
+ */
+const SnapContext& SnapRealm::get_snap_context()
+{
+  check_cache();
+
+  if (!cached_snap_context.seq) {
+    cached_snap_context.seq = cached_seq;
+    cached_snap_context.snaps.resize(cached_snaps.size());
+    unsigned i = 0;
+    for (set::reverse_iterator p = cached_snaps.rbegin();
+	 p != cached_snaps.rend();
+	 ++p)
+      cached_snap_context.snaps[i++] = *p;
+  }
+
+  return cached_snap_context;
+}
+
+void SnapRealm::get_snap_info(map& infomap, snapid_t first, snapid_t last)
+{
+  const set& snaps = get_snaps();
+  dout(10) << "get_snap_info snaps " << snaps << dendl;
+
+  // include my snaps within interval [first,last]
+  for (map::iterator p = srnode.snaps.lower_bound(first); // first element >= first
+       p != srnode.snaps.end() && p->first <= last;
+       ++p)
+    infomap[p->first] = &p->second;
+
+  // include snaps for parents during intervals that intersect [first,last]
+  for (map::iterator p = srnode.past_parents.lower_bound(first);
+       p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
+       ++p) {
+    CInode *oldparent = mdcache->get_inode(p->second.ino);
+    assert(oldparent);  // call open_parents first!
+    assert(oldparent->snaprealm);
+    oldparent->snaprealm->get_snap_info(infomap,
+					MAX(first, p->second.first),
+					MIN(last, p->first));
+  }
+  if (srnode.current_parent_since <= last && parent)
+    parent->get_snap_info(infomap, MAX(first, srnode.current_parent_since), last);
+}
+
+const string& SnapRealm::get_snapname(snapid_t snapid, inodeno_t atino)
+{
+  if (srnode.snaps.count(snapid)) {
+    if (atino == inode->ino())
+      return srnode.snaps[snapid].name;
+    else
+      return srnode.snaps[snapid].get_long_name();
+  }
+
+  map::iterator p = srnode.past_parents.lower_bound(snapid);
+  if (p != srnode.past_parents.end() && p->second.first <= snapid) {
+    CInode *oldparent = mdcache->get_inode(p->second.ino);
+    assert(oldparent);  // call open_parents first!
+    assert(oldparent->snaprealm);    
+    return oldparent->snaprealm->get_snapname(snapid, atino);
+  }
+
+  assert(srnode.current_parent_since <= snapid);
+  assert(parent);
+  return parent->get_snapname(snapid, atino);
+}
+
+snapid_t SnapRealm::resolve_snapname(const string& n, inodeno_t atino, snapid_t first, snapid_t last)
+{
+  // first try me
+  dout(10) << "resolve_snapname '" << n << "' in [" << first << "," << last << "]" << dendl;
+
+  //snapid_t num;
+  //if (n[0] == '~') num = atoll(n.c_str()+1);
+
+  bool actual = (atino == inode->ino());
+  string pname;
+  inodeno_t pino;
+  if (!actual) {
+    if (!n.length() ||
+	n[0] != '_') return 0;
+    int next_ = n.find('_', 1);
+    if (next_ < 0) return 0;
+    pname = n.substr(1, next_ - 1);
+    pino = atoll(n.c_str() + next_ + 1);
+    dout(10) << " " << n << " parses to name '" << pname << "' dirino " << pino << dendl;
+  }
+
+  for (map::iterator p = srnode.snaps.lower_bound(first); // first element >= first
+       p != srnode.snaps.end() && p->first <= last;
+       ++p) {
+    dout(15) << " ? " << p->second << dendl;
+    //if (num && p->second.snapid == num)
+    //return p->first;
+    if (actual && p->second.name == n)
+	return p->first;
+    if (!actual && p->second.name == pname && p->second.ino == pino)
+      return p->first;
+  }
+
+    // include snaps for parents during intervals that intersect [first,last]
+  for (map::iterator p = srnode.past_parents.lower_bound(first);
+       p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
+       ++p) {
+    CInode *oldparent = mdcache->get_inode(p->second.ino);
+    assert(oldparent);  // call open_parents first!
+    assert(oldparent->snaprealm);
+    snapid_t r = oldparent->snaprealm->resolve_snapname(n, atino,
+							MAX(first, p->second.first),
+							MIN(last, p->first));
+    if (r)
+      return r;
+  }
+  if (parent && srnode.current_parent_since <= last)
+    return parent->resolve_snapname(n, atino, MAX(first, srnode.current_parent_since), last);
+  return 0;
+}
+
+
+void SnapRealm::adjust_parent()
+{
+  SnapRealm *newparent = inode->get_parent_dn()->get_dir()->get_inode()->find_snaprealm();
+  if (newparent != parent) {
+    dout(10) << "adjust_parent " << parent << " -> " << newparent << dendl;
+    if (parent)
+      parent->open_children.erase(this);
+    parent = newparent;
+    if (parent)
+      parent->open_children.insert(this);
+    
+    invalidate_cached_snaps();
+  }
+}
+
+void SnapRealm::split_at(SnapRealm *child)
+{
+  dout(10) << "split_at " << *child 
+	   << " on " << *child->inode << dendl;
+
+  if (!child->inode->is_dir()) {
+    // it's not a dir.
+    if (child->inode->containing_realm) {
+      //  - no open children.
+      //  - only need to move this child's inode's caps.
+      child->inode->move_to_realm(child);
+    } else {
+      // no caps, nothing to move/split.
+      dout(20) << " split no-op, no caps to move on file " << *child->inode << dendl;
+      assert(!child->inode->is_any_caps());
+    }
+    return;
+  }
+
+  // it's a dir.
+
+  // split open_children
+  dout(10) << " open_children are " << open_children << dendl;
+  for (set::iterator p = open_children.begin();
+       p != open_children.end(); ) {
+    SnapRealm *realm = *p;
+    if (realm != child &&
+	child->inode->is_projected_ancestor_of(realm->inode)) {
+      dout(20) << " child gets child realm " << *realm << " on " << *realm->inode << dendl;
+      realm->parent = child;
+      child->open_children.insert(realm);
+      open_children.erase(p++);
+    } else {
+      dout(20) << "    keeping child realm " << *realm << " on " << *realm->inode << dendl;
+      ++p;
+    }
+  }
+
+  // split inodes_with_caps
+  elist::iterator p = inodes_with_caps.begin(member_offset(CInode, item_caps));
+  while (!p.end()) {
+    CInode *in = *p;
+    ++p;
+
+    // does inode fall within the child realm?
+    bool under_child = false;
+
+    if (in == child->inode) {
+      under_child = true;
+    } else {
+      CInode *t = in;
+      while (t->get_parent_dn()) {
+	t = t->get_parent_dn()->get_dir()->get_inode();
+	if (t == child->inode) {
+	  under_child = true;
+	  break;
+	}
+	if (t == in)
+	  break;
+      }
+    }
+    if (under_child) {
+      dout(20) << " child gets " << *in << dendl;
+      in->move_to_realm(child);
+    } else {
+      dout(20) << "    keeping " << *in << dendl;
+    }
+  }
+
+}
+
+const bufferlist& SnapRealm::get_snap_trace()
+{
+  check_cache();
+  return cached_snap_trace;
+}
+
+void SnapRealm::build_snap_trace(bufferlist& snapbl)
+{
+  SnapRealmInfo info(inode->ino(), srnode.created, srnode.seq, srnode.current_parent_since);
+
+  if (parent) {
+    info.h.parent = parent->inode->ino();
+    if (!srnode.past_parents.empty()) {
+      snapid_t last = srnode.past_parents.rbegin()->first;
+      set past;
+      snapid_t max_seq, max_last_created, max_last_destroyed;
+      build_snap_set(past, max_seq, max_last_created, max_last_destroyed, 0, last);
+      info.prior_parent_snaps.reserve(past.size());
+      for (set::reverse_iterator p = past.rbegin(); p != past.rend(); ++p)
+	info.prior_parent_snaps.push_back(*p);
+      dout(10) << "build_snap_trace prior_parent_snaps from [1," << last << "] "
+	       << info.prior_parent_snaps << dendl;
+    }
+  } else 
+    info.h.parent = 0;
+
+  info.my_snaps.reserve(srnode.snaps.size());
+  for (map::reverse_iterator p = srnode.snaps.rbegin();
+       p != srnode.snaps.rend();
+       ++p)
+    info.my_snaps.push_back(p->first);
+  dout(10) << "build_snap_trace my_snaps " << info.my_snaps << dendl;
+
+  ::encode(info, snapbl);
+
+  if (parent)
+    parent->build_snap_trace(snapbl);
+}
+
+
+
+void SnapRealm::prune_past_parents()
+{
+  dout(10) << "prune_past_parents" << dendl;
+  check_cache();
+  assert(open);
+
+  map::iterator p = srnode.past_parents.begin();
+  while (p != srnode.past_parents.end()) {
+    set::iterator q = cached_snaps.lower_bound(p->second.first);
+    if (q == cached_snaps.end() ||
+	*q > p->first) {
+      dout(10) << "prune_past_parents pruning [" << p->second.first << "," << p->first 
+	       << "] " << p->second.ino << dendl;
+      srnode.past_parents.erase(p++);
+    } else {
+      dout(10) << "prune_past_parents keeping [" << p->second.first << "," << p->first 
+	       << "] " << p->second.ino << dendl;
+      ++p;
+    }
+  }
+}
+
diff --git a/ceph/src/mds/SnapRealm.h b/ceph/src/mds/SnapRealm.h
new file mode 100644
index 00000000..d8381323
--- /dev/null
+++ b/ceph/src/mds/SnapRealm.h
@@ -0,0 +1,149 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_SNAPREALM_H
+#define CEPH_MDS_SNAPREALM_H
+
+#include "mdstypes.h"
+#include "snap.h"
+#include "include/xlist.h"
+#include "include/elist.h"
+#include "common/snap_types.h"
+
+struct SnapRealm {
+  // realm state
+
+  sr_t srnode;
+
+  // in-memory state
+  MDCache *mdcache;
+  CInode *inode;
+
+  bool open;                        // set to true once all past_parents are opened
+  SnapRealm *parent;
+  set open_children;    // active children that are currently open
+  map open_past_parents;  // these are explicitly pinned.
+
+  // cache
+  snapid_t cached_seq;           // max seq over self and all past+present parents.
+  snapid_t cached_last_created;  // max last_created over all past+present parents
+  snapid_t cached_last_destroyed;
+  set cached_snaps;
+  SnapContext cached_snap_context;
+
+  bufferlist cached_snap_trace;
+
+  elist inodes_with_caps;             // for efficient realm splits
+  map* > client_caps;   // to identify clients who need snap notifications
+
+  SnapRealm(MDCache *c, CInode *in) : 
+    srnode(),
+    mdcache(c), inode(in),
+    open(false), parent(0),
+    inodes_with_caps(0) 
+  { }
+
+  bool exists(const string &name) {
+    for (map::iterator p = srnode.snaps.begin();
+	 p != srnode.snaps.end();
+	 ++p) {
+      if (p->second.name == name)
+	return true;
+    }
+    return false;
+  }
+
+  bool _open_parents(Context *retryorfinish, snapid_t first=1, snapid_t last=CEPH_NOSNAP);
+  bool open_parents(Context *retryorfinish) {
+    if (!_open_parents(retryorfinish))
+      return false;
+    delete retryorfinish;
+    return true;
+  }
+  bool have_past_parents_open(snapid_t first=1, snapid_t last=CEPH_NOSNAP);
+  void add_open_past_parent(SnapRealm *parent);
+  void close_parents();
+
+  void prune_past_parents();
+  bool has_past_parents() { return !srnode.past_parents.empty(); }
+
+  void build_snap_set(set& s, 
+		      snapid_t& max_seq, snapid_t& max_last_created, snapid_t& max_last_destroyed,
+		      snapid_t first, snapid_t last);
+  void get_snap_info(map& infomap, snapid_t first=0, snapid_t last=CEPH_NOSNAP);
+
+  const bufferlist& get_snap_trace();
+  void build_snap_trace(bufferlist& snapbl);
+
+  const string& get_snapname(snapid_t snapid, inodeno_t atino);
+  snapid_t resolve_snapname(const string &name, inodeno_t atino, snapid_t first=0, snapid_t last=CEPH_NOSNAP);
+
+  void check_cache();
+  const set& get_snaps();
+  const SnapContext& get_snap_context();
+  void invalidate_cached_snaps() {
+    cached_seq = 0;
+  }
+  snapid_t get_last_created() {
+    check_cache();
+    return cached_last_created;
+  }
+  snapid_t get_last_destroyed() {
+    check_cache();
+    return cached_last_destroyed;
+  }
+  snapid_t get_newest_snap() {
+    check_cache();
+    if (cached_snaps.empty())
+      return 0;
+    else
+      return *cached_snaps.rbegin();
+  }
+  snapid_t get_newest_seq() {
+    check_cache();
+    return cached_seq;
+  }
+
+  snapid_t get_snap_following(snapid_t follows) {
+    check_cache();
+    set s = get_snaps();
+    set::iterator p = s.upper_bound(follows);
+    if (p != s.end())
+      return *p;
+    return CEPH_NOSNAP;
+  }
+
+  void adjust_parent();
+
+  void split_at(SnapRealm *child);
+  void join(SnapRealm *child);
+
+  void add_cap(client_t client, Capability *cap) {
+    if (client_caps.count(client) == 0)
+      client_caps[client] = new xlist;
+    client_caps[client]->push_back(&cap->item_snaprealm_caps);
+  }
+  void remove_cap(client_t client, Capability *cap) {
+    cap->item_snaprealm_caps.remove_myself();
+    if (client_caps[client]->empty()) {
+      delete client_caps[client];
+      client_caps.erase(client);
+    }
+  }
+
+};
+
+ostream& operator<<(ostream& out, const SnapRealm &realm);
+
+#endif
diff --git a/ceph/src/mds/SnapServer.cc b/ceph/src/mds/SnapServer.cc
new file mode 100644
index 00000000..3d2b4176
--- /dev/null
+++ b/ceph/src/mds/SnapServer.cc
@@ -0,0 +1,340 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "SnapServer.h"
+#include "MDS.h"
+#include "osd/OSDMap.h"
+#include "mon/MonClient.h"
+
+#include "include/types.h"
+#include "messages/MMDSTableRequest.h"
+#include "messages/MRemoveSnaps.h"
+
+#include "msg/Messenger.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".snap "
+
+
+void SnapServer::reset_state()
+{
+  last_snap = 1;  /* snapid 1 reserved for initial root snaprealm */
+  snaps.clear();
+  need_to_purge.clear();
+}
+
+
+// SERVER
+
+void SnapServer::_prepare(bufferlist &bl, uint64_t reqid, int bymds)
+{
+  bufferlist::iterator p = bl.begin();
+  __u32 op;
+  ::decode(op, p);
+
+  switch (op) {
+  case TABLE_OP_CREATE:
+    {
+      version++;
+
+      SnapInfo info;
+      ::decode(info.ino, p);
+      if (!p.end()) {
+	::decode(info.name, p);
+	::decode(info.stamp, p);
+	info.snapid = ++last_snap;
+	pending_create[version] = info;
+	dout(10) << "prepare v" << version << " create " << info << dendl;
+      } else {
+	pending_noop.insert(version);
+	dout(10) << "prepare v" << version << " noop" << dendl;
+      }
+      bl.clear();
+      ::encode(last_snap, bl);
+    }
+    break;
+
+  case TABLE_OP_DESTROY:
+    {
+      inodeno_t ino;
+      snapid_t snapid;
+      ::decode(ino, p);    // not used, currently.
+      ::decode(snapid, p);
+      version++;
+
+      // bump last_snap... we use it as a version value on the snaprealm.
+      ++last_snap;
+
+      pending_destroy[version] = pair(snapid, last_snap);
+      dout(10) << "prepare v" << version << " destroy " << snapid << " seq " << last_snap << dendl;
+
+      bl.clear();
+      ::encode(last_snap, bl);
+    }
+    break;
+
+  default:
+    assert(0);
+  }
+  //dump();
+}
+
+bool SnapServer::_is_prepared(version_t tid)
+{
+  return 
+    pending_create.count(tid) ||
+    pending_destroy.count(tid);
+}
+
+bool SnapServer::_commit(version_t tid, MMDSTableRequest *req)
+{
+  if (pending_create.count(tid)) {
+    dout(7) << "commit " << tid << " create " << pending_create[tid] << dendl;
+    snaps[pending_create[tid].snapid] = pending_create[tid];
+    pending_create.erase(tid);
+  } 
+
+  else if (pending_destroy.count(tid)) {
+    snapid_t sn = pending_destroy[tid].first;
+    snapid_t seq = pending_destroy[tid].second;
+    dout(7) << "commit " << tid << " destroy " << sn << " seq " << seq << dendl;
+    snaps.erase(sn);
+
+    for (set::const_iterator p = mds->mdsmap->get_data_pools().begin();
+	 p != mds->mdsmap->get_data_pools().end();
+	 ++p) {
+      need_to_purge[*p].insert(sn);
+      need_to_purge[*p].insert(seq);
+    }
+
+    pending_destroy.erase(tid);
+  }
+  else if (pending_noop.count(tid)) {
+    dout(7) << "commit " << tid << " noop" << dendl;
+    pending_noop.erase(tid);
+  }
+  else
+    assert(0);
+
+  // bump version.
+  version++;
+  //dump();
+  return true;
+}
+
+void SnapServer::_rollback(version_t tid) 
+{
+  if (pending_create.count(tid)) {
+    dout(7) << "rollback " << tid << " create " << pending_create[tid] << dendl;
+    pending_create.erase(tid);
+  } 
+
+  else if (pending_destroy.count(tid)) {
+    dout(7) << "rollback " << tid << " destroy " << pending_destroy[tid] << dendl;
+    pending_destroy.erase(tid);
+  }
+  
+  else if (pending_noop.count(tid)) {
+    dout(7) << "rollback " << tid << " noop" << dendl;
+    pending_noop.erase(tid);
+  }    
+
+  else
+    assert(0);
+
+  // bump version.
+  version++;
+  //dump();
+}
+
+void SnapServer::_server_update(bufferlist& bl)
+{
+  bufferlist::iterator p = bl.begin();
+  map > purge;
+  ::decode(purge, p);
+
+  dout(7) << "_server_update purged " << purge << dendl;
+  for (map >::iterator p = purge.begin();
+       p != purge.end();
+       ++p) {
+    for (vector::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q)
+      need_to_purge[p->first].erase(*q);
+    if (need_to_purge[p->first].empty())
+      need_to_purge.erase(p->first);
+  }
+
+  version++;
+}
+
+void SnapServer::handle_query(MMDSTableRequest *req)
+{
+  /*  bufferlist::iterator p = req->bl.begin();
+  inodeno_t curino;
+  ::decode(curino, p);
+  dout(7) << "handle_lookup " << *req << " ino " << curino << dendl;
+
+  vector trace;
+  while (true) {
+    assert(anchor_map.count(curino) == 1);
+    Anchor &anchor = anchor_map[curino];
+    
+    dout(10) << "handle_lookup  adding " << anchor << dendl;
+    trace.insert(trace.begin(), anchor);  // lame FIXME
+    
+    if (anchor.dirino < MDS_INO_BASE) break;
+    curino = anchor.dirino;
+  }
+
+  // reply
+  MMDSTableRequest *reply = new MMDSTableRequest(table, TABLE_OP_QUERY_REPLY, req->reqid, version);
+  ::encode(curino, req->bl);
+  ::encode(trace, req->bl);
+  mds->send_message_mds(reply, req->get_source().num());
+
+  */
+  req->put();
+}
+
+
+
+void SnapServer::check_osd_map(bool force)
+{
+  if (!force && version == last_checked_osdmap) {
+    dout(10) << "check_osd_map - version unchanged" << dendl;
+    return;
+  }
+  dout(10) << "check_osd_map need_to_purge=" << need_to_purge << dendl;
+
+  map > all_purge;
+  map > all_purged;
+
+  for (map >::iterator p = need_to_purge.begin();
+       p != need_to_purge.end();
+       ++p) {
+    int id = p->first;
+    const pg_pool_t *pi = mds->osdmap->get_pg_pool(id);
+    for (set::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      if (pi->is_removed_snap(*q)) {
+	dout(10) << " osdmap marks " << *q << " as removed" << dendl;
+	all_purged[id].push_back(*q);
+      } else {
+	all_purge[id].push_back(*q);
+      }
+    }
+  }
+
+  if (!all_purged.empty()) {
+    // prepare to remove from need_to_purge list
+    bufferlist bl;
+    ::encode(all_purged, bl);
+    do_server_update(bl);
+  }
+
+  if (!all_purge.empty()) {
+    dout(10) << "requesting removal of " << all_purge << dendl;
+    MRemoveSnaps *m = new MRemoveSnaps(all_purge);
+    mds->monc->send_mon_message(m);
+  }
+
+  last_checked_osdmap = version;
+}
+
+
+void SnapServer::dump(Formatter *f) const
+{
+  f->open_object_section("snapserver");
+
+  f->dump_int("last_snap", last_snap.val);
+
+  f->open_array_section("pending_noop");
+  for(set::const_iterator i = pending_noop.begin(); i != pending_noop.end(); ++i) {
+    f->dump_unsigned("version", *i);
+  }
+  f->close_section();
+
+  f->open_array_section("snaps");
+  for (map::const_iterator i = snaps.begin(); i != snaps.end(); ++i) {
+    f->open_object_section("snap");
+    i->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_object_section("need_to_purge");
+  for (map >::const_iterator i = need_to_purge.begin(); i != need_to_purge.end(); ++i) {
+    stringstream pool_id;
+    pool_id << i->first;
+    f->open_array_section(pool_id.str().c_str());
+    for (set::const_iterator s = i->second.begin(); s != i->second.end(); ++s) {
+      f->dump_unsigned("snapid", s->val);
+    }
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("pending_create");
+  for(map::const_iterator i = pending_create.begin(); i != pending_create.end(); ++i) {
+    f->open_object_section("snap");
+    f->dump_unsigned("version", i->first);
+    f->open_object_section("snapinfo");
+    i->second.dump(f);
+    f->close_section();
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("pending_destroy");
+  for(map >::const_iterator i = pending_destroy.begin(); i != pending_destroy.end(); ++i) {
+    f->open_object_section("snap");
+    f->dump_unsigned("version", i->first);
+    f->dump_unsigned("removed_snap", i->second.first);
+    f->dump_unsigned("seq", i->second.second);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->close_section();
+}
+
+void SnapServer::generate_test_instances(list& ls)
+{
+  list snapinfo_instances;
+  SnapInfo::generate_test_instances(snapinfo_instances);
+  SnapInfo populated_snapinfo = *(snapinfo_instances.back());
+  for (list::iterator i = snapinfo_instances.begin(); i != snapinfo_instances.end(); ++i) {
+    delete *i;
+  }
+
+  SnapServer *blank = new SnapServer();
+  ls.push_back(blank);
+  SnapServer *populated = new SnapServer();
+  populated->last_snap = 123;
+  populated->snaps[456] = populated_snapinfo;
+  populated->need_to_purge[2].insert(012);
+  populated->pending_create[234] = populated_snapinfo;
+  populated->pending_destroy[345].first = 567;
+  populated->pending_destroy[345].second = 768;
+  populated->pending_noop.insert(890);
+
+  ls.push_back(populated);
+
+}
diff --git a/ceph/src/mds/SnapServer.h b/ceph/src/mds/SnapServer.h
new file mode 100644
index 00000000..95e15080
--- /dev/null
+++ b/ceph/src/mds/SnapServer.h
@@ -0,0 +1,92 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_SNAPSERVER_H
+#define CEPH_SNAPSERVER_H
+
+#include "MDSTableServer.h"
+#include "snap.h"
+
+class MDS;
+
+class SnapServer : public MDSTableServer {
+public:
+  
+protected:
+  snapid_t last_snap;
+  map snaps;
+  map > need_to_purge;
+  
+  map pending_create;
+  map > pending_destroy; // (removed_snap, seq)
+  set           pending_noop;
+
+  version_t last_checked_osdmap;
+
+public:
+  SnapServer(MDS *m) : MDSTableServer(m, TABLE_SNAP),
+		       last_checked_osdmap(0) { }
+    
+  void reset_state();
+  void encode_server_state(bufferlist& bl) const {
+    ENCODE_START(3, 3, bl);
+    ::encode(last_snap, bl);
+    ::encode(snaps, bl);
+    ::encode(need_to_purge, bl);
+    ::encode(pending_create, bl);
+    ::encode(pending_destroy, bl);
+    ::encode(pending_noop, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode_server_state(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+    ::decode(last_snap, bl);
+    ::decode(snaps, bl);
+    ::decode(need_to_purge, bl);
+    ::decode(pending_create, bl);
+    if (struct_v >= 2)
+      ::decode(pending_destroy, bl);
+    else {
+      map t;
+      ::decode(t, bl);
+      for (map::iterator p = t.begin(); p != t.end(); ++p)
+	pending_destroy[p->first].first = p->second; 
+    } 
+    ::decode(pending_noop, bl);
+    DECODE_FINISH(bl);
+  }
+
+  // To permit enc/decoding in isolation in dencoder
+  SnapServer() : MDSTableServer(NULL, TABLE_SNAP), last_checked_osdmap(0) {}
+  void encode(bufferlist& bl) const {
+    encode_server_state(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    decode_server_state(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  // server bits
+  void _prepare(bufferlist &bl, uint64_t reqid, int bymds);
+  bool _is_prepared(version_t tid);
+  bool _commit(version_t tid, MMDSTableRequest *req=NULL);
+  void _rollback(version_t tid);
+  void _server_update(bufferlist& bl);
+  void handle_query(MMDSTableRequest *m);
+
+  void check_osd_map(bool force);
+};
+
+#endif
diff --git a/ceph/src/mds/events/ECommitted.h b/ceph/src/mds/events/ECommitted.h
new file mode 100644
index 00000000..2889a3b0
--- /dev/null
+++ b/ceph/src/mds/events/ECommitted.h
@@ -0,0 +1,42 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_ECOMMITTED_H
+#define CEPH_MDS_ECOMMITTED_H
+
+#include "../LogEvent.h"
+#include "EMetaBlob.h"
+
+class ECommitted : public LogEvent {
+public:
+  metareqid_t reqid;
+
+  ECommitted() : LogEvent(EVENT_COMMITTED) { }
+  ECommitted(metareqid_t r) : 
+    LogEvent(EVENT_COMMITTED), reqid(r) { }
+
+  void print(ostream& out) const {
+    out << "ECommitted " << reqid;
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void update_segment() {}
+  void replay(MDS *mds);
+};
+
+#endif
diff --git a/ceph/src/mds/events/EExport.h b/ceph/src/mds/events/EExport.h
new file mode 100644
index 00000000..082e14ba
--- /dev/null
+++ b/ceph/src/mds/events/EExport.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_EEXPORT_H
+#define CEPH_EEXPORT_H
+
+#include "common/config.h"
+#include "include/types.h"
+
+#include "../MDS.h"
+
+#include "EMetaBlob.h"
+#include "../LogEvent.h"
+
+class EExport : public LogEvent {
+public:
+  EMetaBlob metablob; // exported dir
+protected:
+  dirfrag_t      base;
+  set bounds;
+  
+public:
+  EExport() : LogEvent(EVENT_EXPORT) { }
+  EExport(MDLog *mdlog, CDir *dir) : 
+    LogEvent(EVENT_EXPORT), metablob(mdlog),
+    base(dir->dirfrag()) { }
+  
+  set &get_bounds() { return bounds; }
+  
+  void print(ostream& out) const {
+    out << "EExport " << base << " " << metablob;
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  void replay(MDS *mds);
+
+};
+
+#endif
diff --git a/ceph/src/mds/events/EFragment.h b/ceph/src/mds/events/EFragment.h
new file mode 100644
index 00000000..8869137b
--- /dev/null
+++ b/ceph/src/mds/events/EFragment.h
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_EFRAGMENT_H
+#define CEPH_MDS_EFRAGMENT_H
+
+#include "../LogEvent.h"
+#include "EMetaBlob.h"
+
+struct dirfrag_rollback {
+  fnode_t fnode;
+  dirfrag_rollback() { }
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+};
+WRITE_CLASS_ENCODER(dirfrag_rollback)
+
+class EFragment : public LogEvent {
+public:
+  EMetaBlob metablob;
+  __u8 op;
+  inodeno_t ino;
+  frag_t basefrag;
+  __s32 bits;         // positive for split (from basefrag), negative for merge (to basefrag)
+  list orig_frags;
+  bufferlist rollback;
+
+  EFragment() : LogEvent(EVENT_FRAGMENT) { }
+  EFragment(MDLog *mdlog, int o, dirfrag_t df, int b) :
+    LogEvent(EVENT_FRAGMENT), metablob(mdlog), 
+    op(o), ino(df.ino), basefrag(df.frag), bits(b) { }
+
+  void print(ostream& out) const {
+    out << "EFragment " << op_name(op) << " " << ino << " " << basefrag << " by " << bits << " " << metablob;
+  }
+
+  enum {
+    OP_PREPARE = 1,
+    OP_COMMIT = 2,
+    OP_ROLLBACK = 3,
+    OP_FINISH = 4, // finish deleting orphan dirfrags
+    OP_ONESHOT = 5,  // (legacy) PREPARE+COMMIT
+  };
+  static const char *op_name(int o) {
+    switch (o) {
+    case OP_PREPARE: return "prepare";
+    case OP_COMMIT: return "commit";
+    case OP_ROLLBACK: return "rollback";
+    case OP_FINISH: return "finish";
+    default: return "???";
+    }
+  }
+
+  void add_orig_frag(frag_t df, dirfrag_rollback *drb=NULL) {
+    orig_frags.push_back(df);
+    if (drb)
+      ::encode(*drb, rollback);
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  void replay(MDS *mds);
+};
+
+#endif
diff --git a/ceph/src/mds/events/EImportFinish.h b/ceph/src/mds/events/EImportFinish.h
new file mode 100644
index 00000000..7ed25e15
--- /dev/null
+++ b/ceph/src/mds/events/EImportFinish.h
@@ -0,0 +1,52 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_EIMPORTFINISH_H
+#define CEPH_EIMPORTFINISH_H
+
+#include "common/config.h"
+#include "include/types.h"
+
+#include "../MDS.h"
+#include "../LogEvent.h"
+
+class EImportFinish : public LogEvent {
+ protected:
+  dirfrag_t base; // imported dir
+  bool success;
+
+ public:
+  EImportFinish(CDir *dir, bool s) : LogEvent(EVENT_IMPORTFINISH), 
+				     base(dir->dirfrag()),
+				     success(s) { }
+  EImportFinish() : LogEvent(EVENT_IMPORTFINISH), base(), success(false) { }
+  
+  void print(ostream& out) const {
+    out << "EImportFinish " << base;
+    if (success)
+      out << " success";
+    else
+      out << " failed";
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  
+  void replay(MDS *mds);
+
+};
+
+#endif
diff --git a/ceph/src/mds/events/EImportStart.h b/ceph/src/mds/events/EImportStart.h
new file mode 100644
index 00000000..0f551901
--- /dev/null
+++ b/ceph/src/mds/events/EImportStart.h
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_EIMPORTSTART_H
+#define CEPH_EIMPORTSTART_H
+
+#include "common/config.h"
+#include "include/types.h"
+
+#include "../MDS.h"
+
+#include "EMetaBlob.h"
+#include "../LogEvent.h"
+
+class EImportStart : public LogEvent {
+protected:
+  dirfrag_t base;
+  vector bounds;
+
+ public:
+  EMetaBlob metablob;
+  bufferlist client_map;  // encoded map<__u32,entity_inst_t>
+  version_t cmapv;
+
+  EImportStart(MDLog *log,
+	       dirfrag_t di,
+	       vector& b) : LogEvent(EVENT_IMPORTSTART), 
+				       base(di), bounds(b),
+				       metablob(log) { }
+  EImportStart() : LogEvent(EVENT_IMPORTSTART) { }
+  
+  void print(ostream& out) const {
+    out << "EImportStart " << base << " " << metablob;
+  }
+  
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  
+  void update_segment();
+  void replay(MDS *mds);
+
+};
+
+#endif
diff --git a/ceph/src/mds/events/EMetaBlob.h b/ceph/src/mds/events/EMetaBlob.h
new file mode 100644
index 00000000..fb34697c
--- /dev/null
+++ b/ceph/src/mds/events/EMetaBlob.h
@@ -0,0 +1,601 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_EMETABLOB_H
+#define CEPH_MDS_EMETABLOB_H
+
+#include 
+
+#include "../CInode.h"
+#include "../CDir.h"
+#include "../CDentry.h"
+
+#include "include/triple.h"
+#include "include/interval_set.h"
+
+class MDS;
+class MDLog;
+class LogSegment;
+struct MDSlaveUpdate;
+
+/*
+ * a bunch of metadata in the journal
+ */
+
+/* notes:
+ *
+ * - make sure you adjust the inode.version for any modified inode you
+ *   journal.  CDir and CDentry maintain a projected_version, but CInode
+ *   doesn't, since the journaled inode usually has to be modifed 
+ *   manually anyway (to delay the change in the MDS's cache until after
+ *   it is journaled).
+ *
+ */
+
+
+class EMetaBlob {
+
+public:
+  /* fullbit - a regular dentry + inode
+   *
+   * We encode this one a bit weirdly, just because (also, it's marginally faster
+   * on multiple encodes, which I think can happen):
+   * Encode a bufferlist on struct creation with all data members, without a struct_v.
+   * When encode is called, encode struct_v and then append the bufferlist.
+   * Decode straight into the appropriate variables.
+   *
+   * So, if you add members, encode them in the constructor and then change
+   * the struct_v in the encode function!
+   */
+  struct fullbit {
+    static const int STATE_DIRTY =	 (1<<0);
+    static const int STATE_DIRTYPARENT = (1<<1);
+    static const int STATE_DIRTYPOOL   = (1<<2);
+    string  dn;         // dentry
+    snapid_t dnfirst, dnlast;
+    version_t dnv;
+    inode_t inode;      // if it's not
+    fragtree_t dirfragtree;
+    map xattrs;
+    string symlink;
+    bufferlist snapbl;
+    __u8 state;
+    typedef map old_inodes_t;
+    old_inodes_t old_inodes;
+
+    bufferlist _enc;
+
+    fullbit(const fullbit& o);
+    const fullbit& operator=(const fullbit& o);
+
+    fullbit(const string& d, snapid_t df, snapid_t dl, 
+	    version_t v, const inode_t& i, const fragtree_t &dft, 
+	    const map &xa, const string& sym,
+	    const bufferlist &sbl, __u8 st,
+	    const old_inodes_t *oi = NULL) :
+      //dn(d), dnfirst(df), dnlast(dl), dnv(v), 
+      //inode(i), dirfragtree(dft), xattrs(xa), symlink(sym), snapbl(sbl), dirty(dr) 
+      _enc(1024)
+    {
+      ::encode(d, _enc);
+      ::encode(df, _enc);
+      ::encode(dl, _enc);
+      ::encode(v, _enc);
+      ::encode(i, _enc);
+      ::encode(xa, _enc);
+      if (i.is_symlink())
+	::encode(sym, _enc);
+      if (i.is_dir()) {
+	::encode(dft, _enc);
+	::encode(sbl, _enc);
+      }
+      ::encode(st, _enc);
+      ::encode(oi ? true : false, _enc);
+      if (oi)
+	::encode(*oi, _enc);
+    }
+    fullbit(bufferlist::iterator &p) {
+      decode(p);
+    }
+    fullbit() {}
+    ~fullbit() {}
+
+    void encode(bufferlist& bl) const;
+    void decode(bufferlist::iterator &bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+
+    void update_inode(MDS *mds, CInode *in);
+    bool is_dirty() const { return (state & STATE_DIRTY); }
+    bool is_dirty_parent() const { return (state & STATE_DIRTYPARENT); }
+    bool is_dirty_pool() const { return (state & STATE_DIRTYPOOL); }
+
+    void print(ostream& out) const {
+      out << " fullbit dn " << dn << " [" << dnfirst << "," << dnlast << "] dnv " << dnv
+	  << " inode " << inode.ino
+	  << " state=" << state << std::endl;
+    }
+    string state_string() const {
+      string state_string;
+      bool marked_already = false;
+      if (is_dirty()) {
+	state_string.append("dirty");
+	marked_already = true;
+      }
+      if (is_dirty_parent()) {
+	state_string.append(marked_already ? "+dirty_parent" : "dirty_parent");
+	if (is_dirty_pool())
+	  state_string.append("+dirty_pool");
+      }
+      return state_string;
+    }
+  };
+  WRITE_CLASS_ENCODER(fullbit)
+  
+  /* remotebit - a dentry + remote inode link (i.e. just an ino)
+   */
+  struct remotebit {
+    string dn;
+    snapid_t dnfirst, dnlast;
+    version_t dnv;
+    inodeno_t ino;
+    unsigned char d_type;
+    bool dirty;
+
+    bufferlist _enc;
+
+    remotebit(const string& d, snapid_t df, snapid_t dl, version_t v, inodeno_t i, unsigned char dt, bool dr) : 
+      //dn(d), dnfirst(df), dnlast(dl), dnv(v), ino(i), d_type(dt), dirty(dr) { }
+      _enc(256) {
+      ::encode(d, _enc);
+      ::encode(df, _enc);
+      ::encode(dl, _enc);
+      ::encode(v, _enc);
+      ::encode(i, _enc);
+      ::encode(dt, _enc);
+      ::encode(dr, _enc);
+    }
+    remotebit(bufferlist::iterator &p) { decode(p); }
+    remotebit(): dnfirst(0), dnlast(0), dnv(0), ino(0),
+	d_type('\0'), dirty(false) {}
+
+    void encode(bufferlist& bl) const;
+    void decode(bufferlist::iterator &bl);
+    void print(ostream& out) const {
+      out << " remotebit dn " << dn << " [" << dnfirst << "," << dnlast << "] dnv " << dnv
+	  << " ino " << ino
+	  << " dirty=" << dirty << std::endl;
+    }
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+  };
+  WRITE_CLASS_ENCODER(remotebit)
+
+  /*
+   * nullbit - a null dentry
+   */
+  struct nullbit {
+    string dn;
+    snapid_t dnfirst, dnlast;
+    version_t dnv;
+    bool dirty;
+
+    bufferlist _enc;
+
+    nullbit(const string& d, snapid_t df, snapid_t dl, version_t v, bool dr) : 
+      //dn(d), dnfirst(df), dnlast(dl), dnv(v), dirty(dr) { }
+      _enc(128) {
+      ::encode(d, _enc);
+      ::encode(df, _enc);
+      ::encode(dl, _enc);
+      ::encode(v, _enc);
+      ::encode(dr, _enc);
+    }
+    nullbit(bufferlist::iterator &p) { decode(p); }
+    nullbit(): dnfirst(0), dnlast(0), dnv(0), dirty(false) {}
+
+    void encode(bufferlist& bl) const;
+    void decode(bufferlist::iterator &bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+    void print(ostream& out) {
+      out << " nullbit dn " << dn << " [" << dnfirst << "," << dnlast << "] dnv " << dnv
+	  << " dirty=" << dirty << std::endl;
+    }
+  };
+  WRITE_CLASS_ENCODER(nullbit)
+
+
+  /* dirlump - contains metadata for any dir we have contents for.
+   */
+public:
+  struct dirlump {
+    static const int STATE_COMPLETE =    (1<<1);
+    static const int STATE_DIRTY =       (1<<2);  // dirty due to THIS journal item, that is!
+    static const int STATE_NEW =         (1<<3);  // new directory
+    static const int STATE_IMPORTING =	 (1<<4);  // importing directory
+    static const int STATE_DIRTYDFT =	 (1<<5);  // dirty dirfragtree
+
+    //version_t  dirv;
+    fnode_t fnode;
+    __u32 state;
+    __u32 nfull, nremote, nnull;
+
+  private:
+    mutable bufferlist dnbl;
+    bool dn_decoded;
+    list >   dfull;
+    list dremote;
+    list   dnull;
+
+  public:
+    dirlump() : state(0), nfull(0), nremote(0), nnull(0), dn_decoded(true) { }
+    
+    bool is_complete() const { return state & STATE_COMPLETE; }
+    void mark_complete() { state |= STATE_COMPLETE; }
+    bool is_dirty() const { return state & STATE_DIRTY; }
+    void mark_dirty() { state |= STATE_DIRTY; }
+    bool is_new() const { return state & STATE_NEW; }
+    void mark_new() { state |= STATE_NEW; }
+    bool is_importing() { return state & STATE_IMPORTING; }
+    void mark_importing() { state |= STATE_IMPORTING; }
+    bool is_dirty_dft() { return state & STATE_DIRTYDFT; }
+    void mark_dirty_dft() { state |= STATE_DIRTYDFT; }
+
+    list >   &get_dfull()   { return dfull; }
+    list &get_dremote() { return dremote; }
+    list   &get_dnull()   { return dnull; }
+
+    void print(dirfrag_t dirfrag, ostream& out) {
+      out << "dirlump " << dirfrag << " v " << fnode.version
+	  << " state " << state
+	  << " num " << nfull << "/" << nremote << "/" << nnull
+	  << std::endl;
+      _decode_bits();
+      for (list >::iterator p = dfull.begin(); p != dfull.end(); ++p)
+	(*p)->print(out);
+      for (list::iterator p = dremote.begin(); p != dremote.end(); ++p)
+	p->print(out);
+      for (list::iterator p = dnull.begin(); p != dnull.end(); ++p)
+	p->print(out);
+    }
+
+    string state_string() const {
+      string state_string;
+      bool marked_already = false;
+      if (is_complete()) {
+	state_string.append("complete");
+	marked_already = true;
+      }
+      if (is_dirty()) {
+	state_string.append(marked_already ? "+dirty" : "dirty");
+	marked_already = true;
+      }
+      if (is_new()) {
+	state_string.append(marked_already ? "+new" : "new");
+      }
+      return state_string;
+    }
+
+    // if this changes, update the versioning in encode for it!
+    void _encode_bits() const {
+      if (!dn_decoded) return;
+      ::encode(dfull, dnbl);
+      ::encode(dremote, dnbl);
+      ::encode(dnull, dnbl);
+    }
+    void _decode_bits() { 
+      if (dn_decoded) return;
+      bufferlist::iterator p = dnbl.begin();
+      ::decode(dfull, p);
+      ::decode(dremote, p);
+      ::decode(dnull, p);
+      dn_decoded = true;
+    }
+
+    void encode(bufferlist& bl) const;
+    void decode(bufferlist::iterator &bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+  };
+  WRITE_CLASS_ENCODER(dirlump)
+
+private:
+  // my lumps.  preserve the order we added them in a list.
+  list         lump_order;
+  map lump_map;
+  list > roots;
+
+  list > table_tids;  // tableclient transactions
+
+  inodeno_t opened_ino;
+public:
+  inodeno_t renamed_dirino;
+  list renamed_dir_frags;
+private:
+  
+  // ino (pre)allocation.  may involve both inotable AND session state.
+  version_t inotablev, sessionmapv;
+  inodeno_t allocated_ino;            // inotable
+  interval_set preallocated_inos; // inotable + session
+  inodeno_t used_preallocated_ino;    //            session
+  entity_name_t client_name;          //            session
+
+  // inodes i've truncated
+  list truncate_start;        // start truncate 
+  map truncate_finish;  // finished truncate (started in segment blah)
+
+  vector destroyed_inodes;
+
+  // idempotent op(s)
+  list > client_reqs;
+
+ public:
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  // soft stateadd
+  uint64_t last_subtree_map;
+  uint64_t my_offset;
+
+  // for replay, in certain cases
+  //LogSegment *_segment;
+
+  EMetaBlob(MDLog *mdl = 0);  // defined in journal.cc
+  ~EMetaBlob() { }
+
+  void print(ostream& out) {
+    for (list::iterator p = lump_order.begin();
+	 p != lump_order.end();
+	 ++p) {
+      lump_map[*p].print(*p, out);
+    }
+  }
+
+  void add_client_req(metareqid_t r, uint64_t tid=0) {
+    client_reqs.push_back(pair(r, tid));
+  }
+
+  void add_table_transaction(int table, version_t tid) {
+    table_tids.push_back(pair<__u8, version_t>(table, tid));
+  }
+
+  void add_opened_ino(inodeno_t ino) {
+    assert(!opened_ino);
+    opened_ino = ino;
+  }
+
+  void set_ino_alloc(inodeno_t alloc,
+		     inodeno_t used_prealloc,
+		     interval_set& prealloc,
+		     entity_name_t client,
+		     version_t sv, version_t iv) {
+    allocated_ino = alloc;
+    used_preallocated_ino = used_prealloc;
+    preallocated_inos = prealloc;
+    client_name = client;
+    sessionmapv = sv;
+    inotablev = iv;
+  }
+
+  void add_truncate_start(inodeno_t ino) {
+    truncate_start.push_back(ino);
+  }
+  void add_truncate_finish(inodeno_t ino, uint64_t segoff) {
+    truncate_finish[ino] = segoff;
+  }
+
+  void add_destroyed_inode(inodeno_t ino) {
+    destroyed_inodes.push_back(ino);
+  }
+  
+  void add_null_dentry(CDentry *dn, bool dirty) {
+    add_null_dentry(add_dir(dn->get_dir(), false), dn, dirty);
+  }
+  void add_null_dentry(dirlump& lump, CDentry *dn, bool dirty) {
+    // add the dir
+    lump.nnull++;
+    lump.get_dnull().push_back(nullbit(dn->get_name(), 
+				       dn->first, dn->last,
+				       dn->get_projected_version(), 
+				       dirty));
+  }
+
+  void add_remote_dentry(CDentry *dn, bool dirty) {
+    add_remote_dentry(add_dir(dn->get_dir(), false), dn, dirty, 0, 0);
+  }
+  void add_remote_dentry(CDentry *dn, bool dirty, inodeno_t rino, int rdt) {
+    add_remote_dentry(add_dir(dn->get_dir(), false), dn, dirty, rino, rdt);
+  }
+  void add_remote_dentry(dirlump& lump, CDentry *dn, bool dirty, 
+			 inodeno_t rino=0, unsigned char rdt=0) {
+    if (!rino) {
+      rino = dn->get_projected_linkage()->get_remote_ino();
+      rdt = dn->get_projected_linkage()->get_remote_d_type();
+    }
+    lump.nremote++;
+    lump.get_dremote().push_back(remotebit(dn->get_name(), 
+					   dn->first, dn->last,
+					   dn->get_projected_version(), 
+					   rino, rdt,
+					   dirty));
+  }
+
+  // return remote pointer to to-be-journaled inode
+  void add_primary_dentry(CDentry *dn, CInode *in, bool dirty,
+			  bool dirty_parent=false, bool dirty_pool=false) {
+    __u8 state = 0;
+    if (dirty) state |= fullbit::STATE_DIRTY;
+    if (dirty_parent) state |= fullbit::STATE_DIRTYPARENT;
+    if (dirty_pool) state |= fullbit::STATE_DIRTYPOOL;
+    add_primary_dentry(add_dir(dn->get_dir(), false), dn, in, state);
+  }
+  void add_primary_dentry(dirlump& lump, CDentry *dn, CInode *in, __u8 state) {
+    if (!in) 
+      in = dn->get_projected_linkage()->get_inode();
+
+    // make note of where this inode was last journaled
+    in->last_journaled = my_offset;
+    //cout << "journaling " << in->inode.ino << " at " << my_offset << std::endl;
+
+    inode_t *pi = in->get_projected_inode();
+    if ((state & fullbit::STATE_DIRTY) && pi->is_backtrace_updated())
+      state |= fullbit::STATE_DIRTYPARENT;
+
+    bufferlist snapbl;
+    sr_t *sr = in->get_projected_srnode();
+    if (sr)
+      sr->encode(snapbl);
+
+    lump.nfull++;
+    lump.get_dfull().push_back(ceph::shared_ptr(new fullbit(dn->get_name(), 
+									 dn->first, dn->last,
+									 dn->get_projected_version(), 
+									 *pi, in->dirfragtree,
+									 *in->get_projected_xattrs(),
+									 in->symlink, snapbl,
+									 state,
+									 &in->old_inodes)));
+  }
+
+  // convenience: primary or remote?  figure it out.
+  void add_dentry(CDentry *dn, bool dirty) {
+    dirlump& lump = add_dir(dn->get_dir(), false);
+    add_dentry(lump, dn, dirty, false, false);
+  }
+  void add_import_dentry(CDentry *dn) {
+    bool dirty_parent = false;
+    bool dirty_pool = false;
+    if (dn->get_linkage()->is_primary()) {
+      dirty_parent = dn->get_linkage()->get_inode()->is_dirty_parent();
+      dirty_pool = dn->get_linkage()->get_inode()->is_dirty_pool();
+    }
+    dirlump& lump = add_dir(dn->get_dir(), false);
+    add_dentry(lump, dn, dn->is_dirty(), dirty_parent, dirty_pool);
+  }
+  void add_dentry(dirlump& lump, CDentry *dn, bool dirty, bool dirty_parent, bool dirty_pool) {
+    // primary or remote
+    if (dn->get_projected_linkage()->is_remote()) {
+      add_remote_dentry(dn, dirty);
+      return;
+    } else if (dn->get_projected_linkage()->is_null()) {
+      add_null_dentry(dn, dirty);
+      return;
+    }
+    assert(dn->get_projected_linkage()->is_primary());
+    add_primary_dentry(dn, 0, dirty, dirty_parent, dirty_pool);
+  }
+
+  void add_root(bool dirty, CInode *in, inode_t *pi=0, fragtree_t *pdft=0, bufferlist *psnapbl=0,
+		    map *px=0) {
+    in->last_journaled = my_offset;
+    //cout << "journaling " << in->inode.ino << " at " << my_offset << std::endl;
+
+    if (!pi) pi = in->get_projected_inode();
+    if (!pdft) pdft = &in->dirfragtree;
+    if (!px) px = in->get_projected_xattrs();
+
+    bufferlist snapbl;
+    if (psnapbl)
+      snapbl = *psnapbl;
+    else
+      in->encode_snap_blob(snapbl);
+
+    for (list >::iterator p = roots.begin(); p != roots.end(); ++p) {
+      if ((*p)->inode.ino == in->ino()) {
+	roots.erase(p);
+	break;
+      }
+    }
+
+    string empty;
+    roots.push_back(ceph::shared_ptr(new fullbit(empty, in->first, in->last, 0, *pi,
+							      *pdft, *px, in->symlink, snapbl,
+							      dirty ? fullbit::STATE_DIRTY : 0,
+							      &in->old_inodes)));
+  }
+  
+  dirlump& add_dir(CDir *dir, bool dirty, bool complete=false) {
+    return add_dir(dir->dirfrag(), dir->get_projected_fnode(), dir->get_projected_version(),
+		   dirty, complete);
+  }
+  dirlump& add_new_dir(CDir *dir) {
+    return add_dir(dir->dirfrag(), dir->get_projected_fnode(), dir->get_projected_version(),
+		   true, true, true); // dirty AND complete AND new
+  }
+  dirlump& add_import_dir(CDir *dir) {
+    // dirty=false would be okay in some cases
+    return add_dir(dir->dirfrag(), dir->get_projected_fnode(), dir->get_projected_version(),
+		   dir->is_dirty(), dir->is_complete(), false, true, dir->is_dirty_dft());
+  }
+  dirlump& add_fragmented_dir(CDir *dir, bool dirty, bool dirtydft) {
+    return add_dir(dir->dirfrag(), dir->get_projected_fnode(), dir->get_projected_version(),
+		   dirty, false, false, false, dirtydft);
+  }
+  dirlump& add_dir(dirfrag_t df, fnode_t *pf, version_t pv, bool dirty,
+		   bool complete=false, bool isnew=false,
+		   bool importing=false, bool dirty_dft=false) {
+    if (lump_map.count(df) == 0)
+      lump_order.push_back(df);
+
+    dirlump& l = lump_map[df];
+    l.fnode = *pf;
+    l.fnode.version = pv;
+    if (complete) l.mark_complete();
+    if (dirty) l.mark_dirty();
+    if (isnew) l.mark_new();
+    if (importing) l.mark_importing();
+    if (dirty_dft) l.mark_dirty_dft();
+    return l;
+  }
+  
+  static const int TO_AUTH_SUBTREE_ROOT = 0;  // default.
+  static const int TO_ROOT = 1;
+  
+  void add_dir_context(CDir *dir, int mode = TO_AUTH_SUBTREE_ROOT);
+
+  void print(ostream& out) const {
+    out << "[metablob";
+    if (!lump_order.empty()) 
+      out << " " << lump_order.front() << ", " << lump_map.size() << " dirs";
+    if (!table_tids.empty())
+      out << " table_tids=" << table_tids;
+    if (allocated_ino || preallocated_inos.size()) {
+      if (allocated_ino)
+	out << " alloc_ino=" << allocated_ino;
+      if (preallocated_inos.size())
+	out << " prealloc_ino=" << preallocated_inos;
+      if (used_preallocated_ino)
+	out << " used_prealloc_ino=" << used_preallocated_ino;
+      out << " v" << inotablev;
+    }
+    out << "]";
+  }
+
+  void update_segment(LogSegment *ls);
+  void replay(MDS *mds, LogSegment *ls, MDSlaveUpdate *su=NULL);
+};
+WRITE_CLASS_ENCODER(EMetaBlob)
+WRITE_CLASS_ENCODER(EMetaBlob::fullbit)
+WRITE_CLASS_ENCODER(EMetaBlob::remotebit)
+WRITE_CLASS_ENCODER(EMetaBlob::nullbit)
+WRITE_CLASS_ENCODER(EMetaBlob::dirlump)
+
+inline ostream& operator<<(ostream& out, const EMetaBlob& t) {
+  t.print(out);
+  return out;
+}
+
+#endif
diff --git a/ceph/src/mds/events/EOpen.h b/ceph/src/mds/events/EOpen.h
new file mode 100644
index 00000000..1267cf0a
--- /dev/null
+++ b/ceph/src/mds/events/EOpen.h
@@ -0,0 +1,54 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_EOPEN_H
+#define CEPH_MDS_EOPEN_H
+
+#include "../LogEvent.h"
+#include "EMetaBlob.h"
+
+class EOpen : public LogEvent {
+public:
+  EMetaBlob metablob;
+  vector inos;
+
+  EOpen() : LogEvent(EVENT_OPEN) { }
+  EOpen(MDLog *mdlog) : 
+    LogEvent(EVENT_OPEN), metablob(mdlog) { }
+
+  void print(ostream& out) const {
+    out << "EOpen " << metablob << ", " << inos.size() << " open files";
+  }
+
+  void add_clean_inode(CInode *in) {
+    if (!in->is_base()) {
+      metablob.add_dir_context(in->get_projected_parent_dn()->get_dir());
+      metablob.add_primary_dentry(in->get_projected_parent_dn(), 0, false);
+      inos.push_back(in->ino());
+    }
+  }
+  void add_ino(inodeno_t ino) {
+    inos.push_back(ino);
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void update_segment();
+  void replay(MDS *mds);
+};
+
+#endif
diff --git a/ceph/src/mds/events/EResetJournal.h b/ceph/src/mds/events/EResetJournal.h
new file mode 100644
index 00000000..c782f29a
--- /dev/null
+++ b/ceph/src/mds/events/EResetJournal.h
@@ -0,0 +1,38 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDS_ERESETJOURNAL_H
+#define CEPH_MDS_ERESETJOURNAL_H
+
+#include "../LogEvent.h"
+
+// generic log event
+class EResetJournal : public LogEvent {
+ public:
+  EResetJournal() : LogEvent(EVENT_RESETJOURNAL) { }
+  ~EResetJournal() {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  void print(ostream& out) const {
+    out << "EResetJournal";
+  }
+
+  void replay(MDS *mds);
+};
+
+#endif
diff --git a/ceph/src/mds/events/ESession.h b/ceph/src/mds/events/ESession.h
new file mode 100644
index 00000000..95cd690f
--- /dev/null
+++ b/ceph/src/mds/events/ESession.h
@@ -0,0 +1,67 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_ESESSION_H
+#define CEPH_MDS_ESESSION_H
+
+#include "common/config.h"
+#include "include/types.h"
+
+#include "../LogEvent.h"
+
+class ESession : public LogEvent {
+ protected:
+  entity_inst_t client_inst;
+  bool open;    // open or close
+  version_t cmapv;  // client map version
+
+  interval_set inos;
+  version_t inotablev;
+
+ public:
+  ESession() : LogEvent(EVENT_SESSION), open(false) { }
+  ESession(const entity_inst_t& inst, bool o, version_t v) :
+    LogEvent(EVENT_SESSION),
+    client_inst(inst),
+    open(o),
+    cmapv(v),
+    inotablev(0) {
+  }
+  ESession(const entity_inst_t& inst, bool o, version_t v,
+	   const interval_set& i, version_t iv) :
+    LogEvent(EVENT_SESSION),
+    client_inst(inst),
+    open(o),
+    cmapv(v),
+    inos(i), inotablev(iv) { }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void print(ostream& out) const {
+    if (open)
+      out << "ESession " << client_inst << " open cmapv " << cmapv;
+    else
+      out << "ESession " << client_inst << " close cmapv " << cmapv;
+    if (inos.size())
+      out << " (" << inos.size() << " inos, v" << inotablev << ")";
+  }
+  
+  void update_segment();
+  void replay(MDS *mds);  
+};
+
+#endif
diff --git a/ceph/src/mds/events/ESessions.h b/ceph/src/mds/events/ESessions.h
new file mode 100644
index 00000000..fe943a88
--- /dev/null
+++ b/ceph/src/mds/events/ESessions.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_ESESSIONS_H
+#define CEPH_MDS_ESESSIONS_H
+
+#include "common/config.h"
+#include "include/types.h"
+
+#include "../LogEvent.h"
+
+class ESessions : public LogEvent {
+protected:
+  version_t cmapv;  // client map version
+
+public:
+  map client_map;
+  bool old_style_encode;
+
+  ESessions() : LogEvent(EVENT_SESSIONS), old_style_encode(false) { }
+  ESessions(version_t pv, map& cm) :
+    LogEvent(EVENT_SESSIONS),
+    cmapv(pv),
+    old_style_encode(false) {
+    client_map.swap(cm);
+  }
+
+  void mark_old_encoding() { old_style_encode = true; }
+
+  void encode(bufferlist &bl) const;
+  void decode_old(bufferlist::iterator &bl);
+  void decode_new(bufferlist::iterator &bl);
+  void decode(bufferlist::iterator &bl) {
+    if (old_style_encode) decode_old(bl);
+    else decode_new(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void print(ostream& out) const {
+    out << "ESessions " << client_map.size() << " opens cmapv " << cmapv;
+  }
+  
+  void update_segment();
+  void replay(MDS *mds);  
+};
+
+#endif
diff --git a/ceph/src/mds/events/ESlaveUpdate.h b/ceph/src/mds/events/ESlaveUpdate.h
new file mode 100644
index 00000000..b45351ae
--- /dev/null
+++ b/ceph/src/mds/events/ESlaveUpdate.h
@@ -0,0 +1,147 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_ESLAVEUPDATE_H
+#define CEPH_MDS_ESLAVEUPDATE_H
+
+#include "../LogEvent.h"
+#include "EMetaBlob.h"
+
+/*
+ * rollback records, for remote/slave updates, which may need to be manually
+ * rolled back during journal replay.  (or while active if master fails, but in 
+ * that case these records aren't needed.)
+ */ 
+struct link_rollback {
+  metareqid_t reqid;
+  inodeno_t ino;
+  bool was_inc;
+  utime_t old_ctime;
+  utime_t old_dir_mtime;
+  utime_t old_dir_rctime;
+
+  link_rollback() : ino(0), was_inc(false) {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(link_rollback)
+
+/*
+ * this is only used on an empty dir with a dirfrag on a remote node.
+ * we are auth for nothing.  all we need to do is relink the directory
+ * in the hierarchy properly during replay to avoid breaking the
+ * subtree map.
+ */
+struct rmdir_rollback {
+  metareqid_t reqid;
+  dirfrag_t src_dir;
+  string src_dname;
+  dirfrag_t dest_dir;
+  string dest_dname;
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(rmdir_rollback)
+
+struct rename_rollback {
+  struct drec {
+    dirfrag_t dirfrag;
+    utime_t dirfrag_old_mtime;
+    utime_t dirfrag_old_rctime;
+    inodeno_t ino, remote_ino;
+    string dname;
+    char remote_d_type;
+    utime_t old_ctime;
+    
+    drec() : remote_d_type((char)S_IFREG) {}
+
+    void encode(bufferlist& bl) const;
+    void decode(bufferlist::iterator& bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& ls);
+  };
+  WRITE_CLASS_MEMBER_ENCODER(drec)
+
+  metareqid_t reqid;
+  drec orig_src, orig_dest;
+  drec stray; // we know this is null, but we want dname, old mtime/rctime
+  utime_t ctime;
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(rename_rollback)
+
+
+class ESlaveUpdate : public LogEvent {
+public:
+  const static int OP_PREPARE = 1;
+  const static int OP_COMMIT = 2;
+  const static int OP_ROLLBACK = 3;
+  
+  const static int LINK = 1;
+  const static int RENAME = 2;
+  const static int RMDIR = 3;
+
+  /*
+   * we journal a rollback metablob that contains the unmodified metadata
+   * too, because we may be updating previously dirty metadata, which 
+   * will allow old log segments to be trimmed.  if we end of rolling back,
+   * those updates could be lost.. so we re-journal the unmodified metadata,
+   * and replay will apply _either_ commit or rollback.
+   */
+  EMetaBlob commit;
+  bufferlist rollback;
+  string type;
+  metareqid_t reqid;
+  __s32 master;
+  __u8 op;  // prepare, commit, abort
+  __u8 origop; // link | rename
+
+  ESlaveUpdate() : LogEvent(EVENT_SLAVEUPDATE), master(0), op(0), origop(0) { }
+  ESlaveUpdate(MDLog *mdlog, const char *s, metareqid_t ri, int mastermds, int o, int oo) : 
+    LogEvent(EVENT_SLAVEUPDATE), commit(mdlog), 
+    type(s),
+    reqid(ri),
+    master(mastermds),
+    op(o), origop(oo) { }
+  
+  void print(ostream& out) const {
+    if (type.length())
+      out << type << " ";
+    out << " " << (int)op;
+    if (origop == LINK) out << " link";
+    if (origop == RENAME) out << " rename";
+    out << " " << reqid;
+    out << " for mds." << master;
+    out << commit;
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void replay(MDS *mds);
+};
+
+#endif
diff --git a/ceph/src/mds/events/ESubtreeMap.h b/ceph/src/mds/events/ESubtreeMap.h
new file mode 100644
index 00000000..32a4abe5
--- /dev/null
+++ b/ceph/src/mds/events/ESubtreeMap.h
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_ESUBTREEMAP_H
+#define CEPH_MDS_ESUBTREEMAP_H
+
+#include "../LogEvent.h"
+#include "EMetaBlob.h"
+
+class ESubtreeMap : public LogEvent {
+public:
+  EMetaBlob metablob;
+  map > subtrees;
+  set ambiguous_subtrees;
+  uint64_t expire_pos;
+
+  ESubtreeMap() : LogEvent(EVENT_SUBTREEMAP), expire_pos(0) { }
+  
+  void print(ostream& out) const {
+    out << "ESubtreeMap " << subtrees.size() << " subtrees " 
+	<< ", " << ambiguous_subtrees.size() << " ambiguous "
+	<< metablob;
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void replay(MDS *mds);
+};
+
+#endif
diff --git a/ceph/src/mds/events/ETableClient.h b/ceph/src/mds/events/ETableClient.h
new file mode 100644
index 00000000..e415e60b
--- /dev/null
+++ b/ceph/src/mds/events/ETableClient.h
@@ -0,0 +1,48 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_ETABLECLIENT_H
+#define CEPH_MDS_ETABLECLIENT_H
+
+#include "common/config.h"
+#include "include/types.h"
+
+#include "../mds_table_types.h"
+#include "../LogEvent.h"
+
+struct ETableClient : public LogEvent {
+  __u16 table;
+  __s16 op;
+  version_t tid;
+
+  ETableClient() : LogEvent(EVENT_TABLECLIENT), table(0), op(0), tid(0) { }
+  ETableClient(int t, int o, version_t ti) :
+    LogEvent(EVENT_TABLECLIENT),
+    table(t), op(o), tid(ti) { }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void print(ostream& out) const {
+    out << "ETableClient " << get_mdstable_name(table) << " " << get_mdstableserver_opname(op);
+    if (tid) out << " tid " << tid;
+  }  
+
+  //void update_segment();
+  void replay(MDS *mds);  
+};
+
+#endif
diff --git a/ceph/src/mds/events/ETableServer.h b/ceph/src/mds/events/ETableServer.h
new file mode 100644
index 00000000..132d3b6a
--- /dev/null
+++ b/ceph/src/mds/events/ETableServer.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_ETABLESERVER_H
+#define CEPH_MDS_ETABLESERVER_H
+
+#include "common/config.h"
+#include "include/types.h"
+
+#include "../mds_table_types.h"
+#include "../LogEvent.h"
+
+struct ETableServer : public LogEvent {
+  __u16 table;
+  __s16 op;
+  uint64_t reqid;
+  __s32 bymds;
+  bufferlist mutation;
+  version_t tid;
+  version_t version;
+
+  ETableServer() : LogEvent(EVENT_TABLESERVER), table(0), op(0),
+		   reqid(0), bymds(0), tid(0), version(0) { }
+  ETableServer(int t, int o, uint64_t ri, int m, version_t ti, version_t v) :
+    LogEvent(EVENT_TABLESERVER),
+    table(t), op(o), reqid(ri), bymds(m), tid(ti), version(v) { }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void print(ostream& out) const {
+    out << "ETableServer " << get_mdstable_name(table) 
+	<< " " << get_mdstableserver_opname(op);
+    if (reqid) out << " reqid " << reqid;
+    if (bymds >= 0) out << " mds." << bymds;
+    if (tid) out << " tid " << tid;
+    if (version) out << " version " << version;
+    if (mutation.length()) out << " mutation=" << mutation.length() << " bytes";
+  }  
+
+  void update_segment();
+  void replay(MDS *mds);  
+};
+
+#endif
diff --git a/ceph/src/mds/events/EUpdate.h b/ceph/src/mds/events/EUpdate.h
new file mode 100644
index 00000000..645386e2
--- /dev/null
+++ b/ceph/src/mds/events/EUpdate.h
@@ -0,0 +1,50 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_EUPDATE_H
+#define CEPH_MDS_EUPDATE_H
+
+#include "../LogEvent.h"
+#include "EMetaBlob.h"
+
+class EUpdate : public LogEvent {
+public:
+  EMetaBlob metablob;
+  string type;
+  bufferlist client_map;
+  version_t cmapv;
+  metareqid_t reqid;
+  bool had_slaves;
+
+  EUpdate() : LogEvent(EVENT_UPDATE), cmapv(0), had_slaves(false) { }
+  EUpdate(MDLog *mdlog, const char *s) : 
+    LogEvent(EVENT_UPDATE), metablob(mdlog),
+    type(s), cmapv(0), had_slaves(false) { }
+  
+  void print(ostream& out) const {
+    if (type.length())
+      out << "EUpdate " << type << " ";
+    out << metablob;
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  void update_segment();
+  void replay(MDS *mds);
+};
+
+#endif
diff --git a/ceph/src/mds/flock.cc b/ceph/src/mds/flock.cc
new file mode 100644
index 00000000..4e825c9d
--- /dev/null
+++ b/ceph/src/mds/flock.cc
@@ -0,0 +1,490 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include 
+
+#include "common/debug.h"
+#include "mdstypes.h"
+#include "mds/flock.h"
+
+#define dout_subsys ceph_subsys_mds
+
+bool ceph_lock_state_t::is_waiting(ceph_filelock &fl)
+{
+  multimap::iterator p = waiting_locks.find(fl.start);
+  while (p != waiting_locks.end()) {
+    if (p->second.start > fl.start)
+      return false;
+    if (p->second.length == fl.length &&
+	ceph_filelock_owner_equal(p->second, fl))
+      return true;
+    ++p;
+  }
+  return false;
+}
+
+void ceph_lock_state_t::remove_waiting(ceph_filelock& fl)
+{
+  multimap::iterator p = waiting_locks.find(fl.start);
+  while (p != waiting_locks.end()) {
+    if (p->second.start > fl.start)
+      return;
+    if (p->second.length == fl.length &&
+	ceph_filelock_owner_equal(p->second, fl)) {
+      waiting_locks.erase(p);
+      --client_waiting_lock_counts[(client_t)fl.client];
+      if (!client_waiting_lock_counts[(client_t)fl.client]) {
+        client_waiting_lock_counts.erase((client_t)fl.client);
+      }
+      return;
+    }
+    ++p;
+  }
+}
+
+bool ceph_lock_state_t::add_lock(ceph_filelock& new_lock,
+                                 bool wait_on_fail, bool replay)
+{
+  dout(15) << "add_lock " << new_lock << dendl;
+  bool ret = false;
+  list::iterator>
+    overlapping_locks, self_overlapping_locks, neighbor_locks;
+
+  // first, get any overlapping locks and split them into owned-by-us and not
+  if (get_overlapping_locks(new_lock, overlapping_locks, &neighbor_locks)) {
+    dout(15) << "got overlapping lock, splitting by owner" << dendl;
+    split_by_owner(new_lock, overlapping_locks, self_overlapping_locks);
+  }
+  if (!overlapping_locks.empty()) { //overlapping locks owned by others :(
+    if (CEPH_LOCK_EXCL == new_lock.type) {
+      //can't set, we want an exclusive
+      dout(15) << "overlapping lock, and this lock is exclusive, can't set"
+              << dendl;
+      if (wait_on_fail && !replay) {
+        waiting_locks.insert(pair(new_lock.start, new_lock));
+      }
+    } else { //shared lock, check for any exclusive locks blocking us
+      if (contains_exclusive_lock(overlapping_locks)) { //blocked :(
+        dout(15) << " blocked by exclusive lock in overlapping_locks" << dendl;
+        if (wait_on_fail && !replay) {
+          waiting_locks.insert(pair(new_lock.start, new_lock));
+        }
+      } else {
+        //yay, we can insert a shared lock
+        dout(15) << "inserting shared lock" << dendl;
+        remove_waiting(new_lock);
+        adjust_locks(self_overlapping_locks, new_lock, neighbor_locks);
+        held_locks.insert(pair(new_lock.start, new_lock));
+        ret = true;
+      }
+    }
+  } else { //no overlapping locks except our own
+    remove_waiting(new_lock);
+    adjust_locks(self_overlapping_locks, new_lock, neighbor_locks);
+    dout(15) << "no conflicts, inserting " << new_lock << dendl;
+    held_locks.insert(pair
+                      (new_lock.start, new_lock));
+    ret = true;
+  }
+  if (ret) {
+    ++client_held_lock_counts[(client_t)new_lock.client];
+  }
+  else if (wait_on_fail && !replay)
+    ++client_waiting_lock_counts[(client_t)new_lock.client];
+  return ret;
+}
+
+void ceph_lock_state_t::look_for_lock(ceph_filelock& testing_lock)
+{
+  list::iterator> overlapping_locks,
+    self_overlapping_locks;
+  if (get_overlapping_locks(testing_lock, overlapping_locks)) {
+    split_by_owner(testing_lock, overlapping_locks, self_overlapping_locks);
+  }
+  if (!overlapping_locks.empty()) { //somebody else owns overlapping lock
+    if (CEPH_LOCK_EXCL == testing_lock.type) { //any lock blocks it
+      testing_lock = (*overlapping_locks.begin())->second;
+    } else {
+      ceph_filelock *blocking_lock;
+      if ((blocking_lock = contains_exclusive_lock(overlapping_locks))) {
+        testing_lock = *blocking_lock;
+      } else { //nothing blocking!
+        testing_lock.type = CEPH_LOCK_UNLOCK;
+      }
+    }
+    return;
+  }
+  //if we get here, only our own locks block
+  testing_lock.type = CEPH_LOCK_UNLOCK;
+}
+
+void ceph_lock_state_t::remove_lock(ceph_filelock removal_lock,
+                 list& activated_locks)
+{
+  list::iterator> overlapping_locks,
+    self_overlapping_locks;
+  if (get_overlapping_locks(removal_lock, overlapping_locks)) {
+    dout(15) << "splitting by owner" << dendl;
+    split_by_owner(removal_lock, overlapping_locks, self_overlapping_locks);
+  } else dout(15) << "attempt to remove lock at " << removal_lock.start
+                 << " but no locks there!" << dendl;
+  bool remove_to_end = (0 == removal_lock.length);
+  uint64_t removal_start = removal_lock.start;
+  uint64_t removal_end = removal_start + removal_lock.length - 1;
+  uint64_t old_lock_end;
+  __s64 old_lock_client = 0;
+  ceph_filelock *old_lock;
+
+  dout(15) << "examining " << self_overlapping_locks.size()
+          << " self-overlapping locks for removal" << dendl;
+  for (list::iterator>::iterator
+         iter = self_overlapping_locks.begin();
+       iter != self_overlapping_locks.end();
+       ++iter) {
+    dout(15) << "self overlapping lock " << (*iter)->second << dendl;
+    old_lock = &(*iter)->second;
+    bool old_lock_to_end = (0 == old_lock->length);
+    old_lock_end = old_lock->start + old_lock->length - 1;
+    old_lock_client = old_lock->client;
+    if (remove_to_end) {
+      if (old_lock->start < removal_start) {
+        old_lock->length = removal_start - old_lock->start;
+      } else {
+        dout(15) << "erasing " << (*iter)->second << dendl;
+        held_locks.erase(*iter);
+        --client_held_lock_counts[old_lock_client];
+      }
+    } else if (old_lock_to_end) {
+      ceph_filelock append_lock = *old_lock;
+      append_lock.start = removal_end+1;
+      held_locks.insert(pair
+                        (append_lock.start, append_lock));
+      ++client_held_lock_counts[(client_t)old_lock->client];
+      if (old_lock->start >= removal_start) {
+        dout(15) << "erasing " << (*iter)->second << dendl;
+        held_locks.erase(*iter);
+        --client_held_lock_counts[old_lock_client];
+      } else old_lock->length = removal_start - old_lock->start;
+    } else {
+      if (old_lock_end  > removal_end) {
+        ceph_filelock append_lock = *old_lock;
+        append_lock.start = removal_end + 1;
+        append_lock.length = old_lock_end - append_lock.start + 1;
+        held_locks.insert(pair
+                          (append_lock.start, append_lock));
+        ++client_held_lock_counts[(client_t)old_lock->client];
+      }
+      if (old_lock->start < removal_start) {
+        old_lock->length = removal_start - old_lock->start;
+      } else {
+        dout(15) << "erasing " << (*iter)->second << dendl;
+        held_locks.erase(*iter);
+        --client_held_lock_counts[old_lock_client];
+      }
+    }
+    if (!client_held_lock_counts[old_lock_client]) {
+      client_held_lock_counts.erase(old_lock_client);
+    }
+  }
+}
+
+bool ceph_lock_state_t::remove_all_from (client_t client)
+{
+  bool cleared_any = false;
+  if (client_held_lock_counts.count(client)) {
+    remove_all_from(client, held_locks);
+    client_held_lock_counts.erase(client);
+    cleared_any = true;
+  }
+  if (client_waiting_lock_counts.count(client)) {
+    remove_all_from(client, waiting_locks);
+    client_waiting_lock_counts.erase(client);
+  }
+  return cleared_any;
+}
+
+void ceph_lock_state_t::adjust_locks(list::iterator> old_locks,
+                  ceph_filelock& new_lock,
+                  list::iterator>
+                  neighbor_locks)
+{
+  dout(15) << "adjust_locks" << dendl;
+  bool new_lock_to_end = (0 == new_lock.length);
+  uint64_t new_lock_start = new_lock.start;
+  uint64_t new_lock_end = new_lock.start + new_lock.length - 1;
+  uint64_t old_lock_start, old_lock_end;
+  __s64 old_lock_client = 0;
+  ceph_filelock *old_lock;
+  for (list::iterator>::iterator
+         iter = old_locks.begin();
+       iter != old_locks.end();
+       ++iter) {
+    old_lock = &(*iter)->second;
+    dout(15) << "adjusting lock: " << *old_lock << dendl;
+    bool old_lock_to_end = (0 == old_lock->length);
+    old_lock_start = old_lock->start;
+    old_lock_end = old_lock->start + old_lock->length - 1;
+    new_lock_start = new_lock.start;
+    new_lock_end = new_lock.start + new_lock.length - 1;
+    old_lock_client = old_lock->client;
+    if (new_lock_to_end || old_lock_to_end) {
+      //special code path to deal with a length set at 0
+      dout(15) << "one lock extends forever" << dendl;
+      if (old_lock->type == new_lock.type) {
+        //just unify them in new lock, remove old lock
+        dout(15) << "same lock type, unifying" << dendl;
+        new_lock.start = (new_lock_start < old_lock_start) ? new_lock_start :
+          old_lock_start;
+        new_lock.length = 0;
+        held_locks.erase(*iter);
+        --client_held_lock_counts[old_lock_client];
+      } else { //not same type, have to keep any remains of old lock around
+        dout(15) << "shrinking old lock" << dendl;
+        if (new_lock_to_end) {
+          if (old_lock_start < new_lock_start) {
+            old_lock->length = new_lock_start - old_lock_start;
+          } else {
+            held_locks.erase(*iter);
+            --client_held_lock_counts[old_lock_client];
+          }
+        } else { //old lock extends past end of new lock
+          ceph_filelock appended_lock = *old_lock;
+          appended_lock.start = new_lock_end + 1;
+          held_locks.insert(pair
+                            (appended_lock.start, appended_lock));
+          ++client_held_lock_counts[(client_t)old_lock->client];
+          if (old_lock_start < new_lock_start) {
+            old_lock->length = new_lock_start - old_lock_start;
+          } else {
+            held_locks.erase(*iter);
+            --client_held_lock_counts[old_lock_client];
+          }
+        }
+      }
+    } else {
+      if (old_lock->type == new_lock.type) { //just merge them!
+        dout(15) << "merging locks, they're the same type" << dendl;
+        new_lock.start = (old_lock_start < new_lock_start ) ? old_lock_start :
+          new_lock_start;
+        int new_end = (new_lock_end > old_lock_end) ? new_lock_end :
+          old_lock_end;
+        new_lock.length = new_end - new_lock.start + 1;
+        dout(15) << "erasing lock " << (*iter)->second << dendl;
+        held_locks.erase(*iter);
+        --client_held_lock_counts[old_lock_client];
+      } else { //we'll have to update sizes and maybe make new locks
+        dout(15) << "locks aren't same type, changing sizes" << dendl;
+        if (old_lock_end > new_lock_end) { //add extra lock after new_lock
+          ceph_filelock appended_lock = *old_lock;
+          appended_lock.start = new_lock_end + 1;
+          appended_lock.length = old_lock_end - appended_lock.start + 1;
+          held_locks.insert(pair
+                            (appended_lock.start, appended_lock));
+          ++client_held_lock_counts[(client_t)old_lock->client];
+        }
+        if (old_lock_start < new_lock_start) {
+          old_lock->length = new_lock_start - old_lock_start;
+        } else { //old_lock starts inside new_lock, so remove it
+          //if it extended past new_lock_end it's been replaced
+          held_locks.erase(*iter);
+          --client_held_lock_counts[old_lock_client];
+        }
+      }
+    }
+    if (!client_held_lock_counts[old_lock_client]) {
+      client_held_lock_counts.erase(old_lock_client);
+    }
+  }
+
+  //make sure to coalesce neighboring locks
+  for (list::iterator>::iterator
+         iter = neighbor_locks.begin();
+       iter != neighbor_locks.end();
+       ++iter) {
+    old_lock = &(*iter)->second;
+    old_lock_client = old_lock->client;
+    dout(15) << "lock to coalesce: " << *old_lock << dendl;
+    /* because if it's a neighboring lock there can't be any self-overlapping
+       locks that covered it */
+    if (old_lock->type == new_lock.type) { //merge them
+      if (0 == new_lock.length) {
+        if (old_lock->start + old_lock->length == new_lock.start) {
+          new_lock.start = old_lock->start;
+        } else assert(0); /* if there's no end to new_lock, the neighbor
+                             HAS TO be to left side */
+      } else if (0 == old_lock->length) {
+        if (new_lock.start + new_lock.length == old_lock->start) {
+          new_lock.length = 0;
+        } else assert(0); //same as before, but reversed
+      } else {
+        if (old_lock->start + old_lock->length == new_lock.start) {
+          new_lock.start = old_lock->start;
+          new_lock.length = old_lock->length + new_lock.length;
+        } else if (new_lock.start + new_lock.length == old_lock->start) {
+          new_lock.length = old_lock->length + new_lock.length;
+        }
+      }
+      held_locks.erase(*iter);
+      --client_held_lock_counts[old_lock_client];
+    }
+    if (!client_held_lock_counts[old_lock_client]) {
+      client_held_lock_counts.erase(old_lock_client);
+    }
+  }
+}
+
+void ceph_lock_state_t::remove_all_from(client_t client,
+                                        multimap& locks)
+{
+  multimap::iterator iter = locks.begin();
+  while (iter != locks.end()) {
+    if ((client_t)iter->second.client == client) {
+      locks.erase(iter++);
+    } else ++iter;
+  }
+}
+
+multimap::iterator
+ceph_lock_state_t::get_lower_bound(uint64_t start,
+                                   multimap& lock_map)
+{
+   multimap::iterator lower_bound =
+     lock_map.lower_bound(start);
+   if ((lower_bound->first != start)
+       && (start != 0)
+       && (lower_bound != lock_map.begin())) --lower_bound;
+   if (lock_map.end() == lower_bound)
+     dout(15) << "get_lower_dout(15)eturning end()" << dendl;
+   else dout(15) << "get_lower_bound returning iterator pointing to "
+                << lower_bound->second << dendl;
+   return lower_bound;
+ }
+
+multimap::iterator
+ceph_lock_state_t::get_last_before(uint64_t end,
+                                   multimap& lock_map)
+{
+  multimap::iterator last =
+    lock_map.upper_bound(end);
+  if (last != lock_map.begin()) --last;
+  if (lock_map.end() == last)
+    dout(15) << "get_last_before returning end()" << dendl;
+  else dout(15) << "get_last_before returning iterator pointing to "
+               << last->second << dendl;
+  return last;
+}
+
+bool ceph_lock_state_t::share_space(
+    multimap::iterator& iter,
+    uint64_t start, uint64_t end)
+{
+  bool ret = ((iter->first >= start && iter->first <= end) ||
+              ((iter->first < start) &&
+               (((iter->first + iter->second.length - 1) >= start) ||
+                (0 == iter->second.length))));
+  dout(15) << "share_space got start: " << start << ", end: " << end
+          << ", lock: " << iter->second << ", returning " << ret << dendl;
+  return ret;
+}
+
+bool ceph_lock_state_t::get_overlapping_locks(ceph_filelock& lock,
+                           list::iterator> & overlaps,
+                           list::iterator> *self_neighbors)
+{
+  dout(15) << "get_overlapping_locks" << dendl;
+  // create a lock starting one earlier and ending one later
+  // to check for neighbors
+  ceph_filelock neighbor_check_lock = lock;
+  if (neighbor_check_lock.start != 0) {
+    neighbor_check_lock.start = neighbor_check_lock.start - 1;
+    if (neighbor_check_lock.length)
+      neighbor_check_lock.length = neighbor_check_lock.length + 2;
+  } else {
+    if (neighbor_check_lock.length)
+      neighbor_check_lock.length = neighbor_check_lock.length + 1;
+  }
+  //find the last held lock starting at the point after lock
+  uint64_t endpoint = lock.start;
+  if (lock.length) {
+    endpoint += lock.length;
+  } else {
+    endpoint = uint64_t(-1); // max offset
+  }
+  multimap::iterator iter =
+    get_last_before(endpoint, held_locks);
+  bool cont = iter != held_locks.end();
+  while(cont) {
+    if (share_space(iter, lock)) {
+      overlaps.push_front(iter);
+    } else if (self_neighbors &&
+               (neighbor_check_lock.client == iter->second.client) &&
+               (neighbor_check_lock.pid == iter->second.pid) &&
+               share_space(iter, neighbor_check_lock)) {
+      self_neighbors->push_front(iter);
+    }
+    if ((iter->first < lock.start) && (CEPH_LOCK_EXCL == iter->second.type)) {
+      //can't be any more overlapping locks or they'd interfere with this one
+      cont = false;
+    } else if (held_locks.begin() == iter) cont = false;
+    else --iter;
+  }
+  return !overlaps.empty();
+}
+
+bool ceph_lock_state_t::get_waiting_overlaps(ceph_filelock& lock,
+                                             list::iterator>&
+                                               overlaps)
+{
+  dout(15) << "get_waiting_overlaps" << dendl;
+  multimap::iterator iter =
+    get_last_before(lock.start + lock.length - 1, waiting_locks);
+  bool cont = iter != waiting_locks.end();
+  while(cont) {
+    if (share_space(iter, lock)) overlaps.push_front(iter);
+    if (waiting_locks.begin() == iter) cont = false;
+    --iter;
+  }
+  return !overlaps.empty();
+}
+
+void ceph_lock_state_t::split_by_owner(ceph_filelock& owner,
+                                       list::iterator>& locks,
+                                       list::iterator>&
+                                           owned_locks)
+{
+  list::iterator>::iterator
+    iter = locks.begin();
+  dout(15) << "owner lock: " << owner << dendl;
+  while (iter != locks.end()) {
+    dout(15) << "comparing to " << (*iter)->second << dendl;
+    if (ceph_filelock_owner_equal((*iter)->second, owner)) {
+      dout(15) << "success, pushing to owned_locks" << dendl;
+      owned_locks.push_back(*iter);
+      iter = locks.erase(iter);
+    } else {
+      dout(15) << "failure, something not equal in this group "
+              << (*iter)->second.client << ":" << owner.client << ","
+	      << (*iter)->second.owner << ":" << owner.owner << ","
+	      << (*iter)->second.pid << ":" << owner.pid << dendl;
+      ++iter;
+    }
+  }
+}
+
+ceph_filelock *
+ceph_lock_state_t::contains_exclusive_lock(list::iterator>& locks)
+{
+  for (list::iterator>::iterator
+         iter = locks.begin();
+       iter != locks.end();
+       ++iter) {
+    if (CEPH_LOCK_EXCL == (*iter)->second.type) return &(*iter)->second;
+  }
+  return NULL;
+}
diff --git a/ceph/src/mds/flock.h b/ceph/src/mds/flock.h
new file mode 100644
index 00000000..4791b850
--- /dev/null
+++ b/ceph/src/mds/flock.h
@@ -0,0 +1,238 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_MDS_FLOCK_H
+#define CEPH_MDS_FLOCK_H
+
+#include 
+
+#include "common/debug.h"
+#include "mdstypes.h"
+
+
+inline ostream& operator<<(ostream& out, ceph_filelock& l) {
+  out << "start: " << l.start << ", length: " << l.length
+      << ", client: " << l.client << ", owner: " << l.owner
+      << ", pid: " << l.pid << ", type: " << (int)l.type
+      << std::endl;
+  return out;
+}
+
+inline bool ceph_filelock_owner_equal(ceph_filelock& l, ceph_filelock& r)
+{
+  if (l.client != r.client || l.owner != r.owner)
+    return false;
+  // The file lock is from old client if the most significant bit of
+  // 'owner' is not set. Old clients use both 'owner' and 'pid' to
+  // identify the owner of lock.
+  if (l.owner & (1ULL << 63))
+    return true;
+  return l.pid == r.pid;
+}
+
+inline bool operator==(ceph_filelock& l, ceph_filelock& r) {
+  return
+    l.length == r.length &&
+    l.type == r.type &&
+    ceph_filelock_owner_equal(l, r);
+}
+
+class ceph_lock_state_t {
+public:
+  multimap held_locks;    // current locks
+  multimap waiting_locks; // locks waiting for other locks
+  // both of the above are keyed by starting offset
+  map client_held_lock_counts;
+  map client_waiting_lock_counts;
+
+  /**
+   * Check if a lock is on the waiting_locks list.
+   *
+   * @param fl The filelock to check for
+   * @returns True if the lock is waiting, false otherwise
+   */
+  bool is_waiting(ceph_filelock &fl);
+  /**
+   * Remove a lock from the waiting_locks list
+   *
+   * @param fl The filelock to remove
+   */
+  void remove_waiting(ceph_filelock& fl);
+
+  /*
+   * Try to set a new lock. If it's blocked and wait_on_fail is true,
+   * add the lock to waiting_locks.
+   * The lock needs to be of type CEPH_LOCK_EXCL or CEPH_LOCK_SHARED.
+   * This may merge previous locks, or convert the type of already-owned
+   * locks.
+   *
+   * @param new_lock The lock to set
+   * @param wait_on_fail whether to wait until the lock can be set.
+   * Otherwise it fails immediately when blocked.
+   *
+   * @returns true if set, false if not set.
+   */
+  bool add_lock(ceph_filelock& new_lock, bool wait_on_fail, bool replay);
+  /**
+   * See if a lock is blocked by existing locks. If the lock is blocked,
+   * it will be set to the value of the first blocking lock. Otherwise,
+   * it will be returned unchanged, except for setting the type field
+   * to CEPH_LOCK_UNLOCK.
+   *
+   * @param testing_lock The lock to check for conflicts on.
+   */
+  void look_for_lock(ceph_filelock& testing_lock);
+
+  /*
+   * Remove lock(s) described in old_lock. This may involve splitting a
+   * previous lock or making a previous lock smaller.
+   *
+   * @param removal_lock The lock to remove
+   * @param activated_locks A return parameter, holding activated wait locks.
+   */
+  void remove_lock(ceph_filelock removal_lock,
+                   list& activated_locks);
+
+  bool remove_all_from(client_t client);
+private:
+  /**
+   * Adjust old locks owned by a single process so that process can set
+   * a new lock of different type. Handle any changes needed to the old locks
+   * (and the new lock) so that once the new lock is inserted into the 
+   * held_locks list the process has a coherent, non-fragmented set of lock
+   * ranges. Make sure any overlapping locks are combined, trimmed, and removed
+   * as needed.
+   * This function should only be called once you know the lock will be
+   * inserted, as it DOES adjust new_lock. You can call this function
+   * on an empty list, in which case it does nothing.
+   * This function does not remove elements from old_locks, so regard the list
+   * as bad information following function invocation.
+   *
+   * @param new_lock The new lock the process has requested.
+   * @param old_locks list of all locks currently held by same
+   *    client/process that overlap new_lock.
+   * @param neighbor_locks locks owned by same process that neighbor new_lock on
+   *    left or right side.
+   */
+  void adjust_locks(list::iterator> old_locks,
+                    ceph_filelock& new_lock,
+                    list::iterator>
+                      neighbor_locks);
+
+  //this won't reset the counter map value, do that yourself
+  void remove_all_from(client_t client,
+                       multimap& locks);
+
+  //get last lock prior to start position
+  multimap::iterator
+  get_lower_bound(uint64_t start,
+                  multimap& lock_map);
+  //get latest-starting lock that goes over the byte "end"
+  multimap::iterator
+  get_last_before(uint64_t end,
+                  multimap& lock_map);
+
+  /*
+   * See if an iterator's lock covers any of the same bounds as a given range
+   * Rules: locks cover "length" bytes from "start", so the last covered
+   * byte is at start + length - 1.
+   * If the length is 0, the lock covers from "start" to the end of the file.
+   */
+  bool share_space(multimap::iterator& iter,
+		   uint64_t start, uint64_t end);
+  
+  bool share_space(multimap::iterator& iter,
+                   ceph_filelock &lock) {
+    uint64_t end = lock.start;
+    if (lock.length) {
+      end += lock.length - 1;
+    } else { // zero length means end of file
+      end = uint64_t(-1);
+    }
+    return share_space(iter, lock.start, end);
+  }
+  /*
+   *get a list of all locks overlapping with the given lock's range
+   * lock: the lock to compare with.
+   * overlaps: an empty list, to be filled.
+   * Returns: true if at least one lock overlaps.
+   */
+  bool get_overlapping_locks(ceph_filelock& lock,
+                             list::iterator> & overlaps,
+                             list::iterator> *self_neighbors);
+
+  
+  bool get_overlapping_locks(ceph_filelock& lock,
+			     list::iterator>& overlaps) {
+    return get_overlapping_locks(lock, overlaps, NULL);
+  }
+
+  /**
+   * Get a list of all waiting locks that overlap with the given lock's range.
+   * lock: specifies the range to compare with
+   * overlaps: an empty list, to be filled
+   * Returns: true if at least one waiting_lock overlaps
+   */
+  bool get_waiting_overlaps(ceph_filelock& lock,
+                            list::iterator>& overlaps);
+  /*
+   * split a list of locks up by whether they're owned by same
+   * process as given lock
+   * owner: the owning lock
+   * locks: the list of locks (obtained from get_overlapping_locks, probably)
+   *        Will have all locks owned by owner removed
+   * owned_locks: an empty list, to be filled with the locks owned by owner
+   */
+  void split_by_owner(ceph_filelock& owner,
+		      list::iterator> & locks,
+		      list::iterator> & owned_locks);
+
+  ceph_filelock *contains_exclusive_lock(list::iterator>& locks);
+
+public:
+  void encode(bufferlist& bl) const {
+    ::encode(held_locks, bl);
+    ::encode(waiting_locks, bl);
+    ::encode(client_held_lock_counts, bl);
+    ::encode(client_waiting_lock_counts, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(held_locks, bl);
+    ::decode(waiting_locks, bl);
+    ::decode(client_held_lock_counts, bl);
+    ::decode(client_waiting_lock_counts, bl);
+  }
+  void clear() {
+    held_locks.clear();
+    waiting_locks.clear();
+    client_held_lock_counts.clear();
+    client_waiting_lock_counts.clear();
+  }
+};
+WRITE_CLASS_ENCODER(ceph_lock_state_t)
+
+
+inline ostream& operator<<(ostream& out, ceph_lock_state_t& l) {
+  out << "ceph_lock_state_t. held_locks.size()=" << l.held_locks.size()
+      << ", waiting_locks.size()=" << l.waiting_locks.size()
+      << ", client_held_lock_counts -- " << l.client_held_lock_counts
+      << "\n client_waiting_lock_counts -- " << l.client_waiting_lock_counts
+      << "\n held_locks -- ";
+    for (multimap::iterator iter = l.held_locks.begin();
+         iter != l.held_locks.end();
+         ++iter)
+      out << iter->second;
+    out << "\n waiting_locks -- ";
+    for (multimap::iterator iter =l.waiting_locks.begin();
+         iter != l.waiting_locks.end();
+         ++iter)
+      out << iter->second << "\n";
+  return out;
+}
+
+#endif
diff --git a/ceph/src/mds/inode_backtrace.cc b/ceph/src/mds/inode_backtrace.cc
new file mode 100644
index 00000000..e2ab809d
--- /dev/null
+++ b/ceph/src/mds/inode_backtrace.cc
@@ -0,0 +1,120 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#include "inode_backtrace.h"
+
+#include "common/Formatter.h"
+
+/* inode_backpointer_t */
+
+void inode_backpointer_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(dirino, bl);
+  ::encode(dname, bl);
+  ::encode(version, bl);
+  ENCODE_FINISH(bl);
+}
+
+void inode_backpointer_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(dirino, bl);
+  ::decode(dname, bl);
+  ::decode(version, bl);
+  DECODE_FINISH(bl);
+}
+
+void inode_backpointer_t::decode_old(bufferlist::iterator& bl)
+{
+  ::decode(dirino, bl);
+  ::decode(dname, bl);
+  ::decode(version, bl);
+}
+
+void inode_backpointer_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("dirino", dirino);
+  f->dump_string("dname", dname);
+  f->dump_unsigned("version", version);
+}
+
+void inode_backpointer_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new inode_backpointer_t);
+  ls.push_back(new inode_backpointer_t);
+  ls.back()->dirino = 1;
+  ls.back()->dname = "foo";
+  ls.back()->version = 123;
+}
+
+
+/*
+ * inode_backtrace_t
+ */
+
+void inode_backtrace_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(5, 4, bl);
+  ::encode(ino, bl);
+  ::encode(ancestors, bl);
+  ::encode(pool, bl);
+  ::encode(old_pools, bl);
+  ENCODE_FINISH(bl);
+}
+
+void inode_backtrace_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(5, 4, 4, bl);
+  if (struct_v < 3)
+    return;  // sorry, the old data was crap
+  ::decode(ino, bl);
+  if (struct_v >= 4) {
+    ::decode(ancestors, bl);
+  } else {
+    __u32 n;
+    ::decode(n, bl);
+    while (n--) {
+      ancestors.push_back(inode_backpointer_t());
+      ancestors.back().decode_old(bl);
+    }
+  }
+  if (struct_v >= 5) {
+    ::decode(pool, bl);
+    ::decode(old_pools, bl);
+  }
+  DECODE_FINISH(bl);
+}
+
+void inode_backtrace_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("ino", ino);
+  f->open_array_section("ancestors");
+  for (vector::const_iterator p = ancestors.begin(); p != ancestors.end(); ++p) {
+    f->open_object_section("backpointer");
+    p->dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->dump_int("pool", pool);
+  f->open_array_section("old_pools");
+  for (set::iterator p = old_pools.begin(); p != old_pools.end(); ++p) {
+    f->dump_int("old_pool", *p);
+  }
+  f->close_section();
+}
+
+void inode_backtrace_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new inode_backtrace_t);
+  ls.push_back(new inode_backtrace_t);
+  ls.back()->ino = 1;
+  ls.back()->ancestors.push_back(inode_backpointer_t());
+  ls.back()->ancestors.back().dirino = 123;
+  ls.back()->ancestors.back().dname = "bar";
+  ls.back()->ancestors.back().version = 456;
+  ls.back()->pool = 0;
+  ls.back()->old_pools.insert(10);
+  ls.back()->old_pools.insert(7);
+}
+
diff --git a/ceph/src/mds/inode_backtrace.h b/ceph/src/mds/inode_backtrace.h
new file mode 100644
index 00000000..2d80ae3e
--- /dev/null
+++ b/ceph/src/mds/inode_backtrace.h
@@ -0,0 +1,73 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_INODE_BACKTRACE_H
+#define CEPH_INODE_BACKTRACE_H
+
+#include "mdstypes.h"
+
+namespace ceph {
+  class Formatter;
+}
+
+/** metadata backpointers **/
+
+/*
+ * - inode_backpointer_t is just the _pointer_ portion; it doesn't
+ *   tell us who we point _from_.
+ *
+ * - it _does_ include a version of the source object, so we can look
+ *   at two different pointers (from the same inode) and tell which is
+ *   newer.
+ */
+struct inode_backpointer_t {
+  inodeno_t dirino;    // containing directory ino
+  string dname;        // linking dentry name
+  version_t version;   // child's version at time of backpointer creation
+
+  inode_backpointer_t() : version(0) {}
+  inode_backpointer_t(inodeno_t i, const string &d, version_t v) : dirino(i), dname(d), version(v) {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator &bl);
+  void decode_old(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(inode_backpointer_t)
+
+inline bool operator==(const inode_backpointer_t& l, const inode_backpointer_t& r) {
+	return l.dirino == r.dirino && l.version == r.version && l.dname == r.dname;
+}
+
+inline ostream& operator<<(ostream& out, const inode_backpointer_t& ib) {
+  return out << "<" << ib.dirino << "/" << ib.dname << " v" << ib.version << ">";
+}
+
+/*
+ * inode_backtrace_t is a complete ancestor backtraces for a given inode.
+ * we include who _we_ are, so that the backtrace can stand alone (as, say,
+ * an xattr on an object).
+ */
+struct inode_backtrace_t {
+  inodeno_t ino;       // my ino
+  vector ancestors;
+  int64_t pool;
+  // we use a set for old_pools to avoid duplicate entries, e.g. setlayout 0, 1, 0
+  set old_pools;
+
+  inode_backtrace_t() : pool(-1) {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(inode_backtrace_t)
+
+inline ostream& operator<<(ostream& out, const inode_backtrace_t& it) {
+  return out << "(" << it.pool << ")" << it.ino << ":" << it.ancestors << "//" << it.old_pools;
+}
+
+
+#endif
+
diff --git a/ceph/src/mds/journal.cc b/ceph/src/mds/journal.cc
new file mode 100644
index 00000000..2b8bef0b
--- /dev/null
+++ b/ceph/src/mds/journal.cc
@@ -0,0 +1,2787 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "common/config.h"
+#include "osdc/Journaler.h"
+#include "events/ESubtreeMap.h"
+#include "events/ESession.h"
+#include "events/ESessions.h"
+
+#include "events/EMetaBlob.h"
+#include "events/EResetJournal.h"
+
+#include "events/EUpdate.h"
+#include "events/ESlaveUpdate.h"
+#include "events/EOpen.h"
+#include "events/ECommitted.h"
+
+#include "events/EExport.h"
+#include "events/EImportStart.h"
+#include "events/EImportFinish.h"
+#include "events/EFragment.h"
+
+#include "events/ETableClient.h"
+#include "events/ETableServer.h"
+
+#include "include/stringify.h"
+
+#include "LogSegment.h"
+
+#include "MDS.h"
+#include "MDLog.h"
+#include "MDCache.h"
+#include "Server.h"
+#include "Migrator.h"
+#include "Mutation.h"
+
+#include "InoTable.h"
+#include "MDSTableClient.h"
+#include "MDSTableServer.h"
+
+#include "Locker.h"
+
+#define dout_subsys ceph_subsys_mds
+#undef DOUT_COND
+#define DOUT_COND(cct, l) (l<=cct->_conf->debug_mds || l <= cct->_conf->debug_mds_log \
+			      || l <= cct->_conf->debug_mds_log_expire)
+#undef dout_prefix
+#define dout_prefix *_dout << "mds." << mds->get_nodeid() << ".journal "
+
+
+// -----------------------
+// LogSegment
+
+void LogSegment::try_to_expire(MDS *mds, C_GatherBuilder &gather_bld, int op_prio)
+{
+  set commit;
+
+  dout(6) << "LogSegment(" << offset << ").try_to_expire" << dendl;
+
+  assert(g_conf->mds_kill_journal_expire_at != 1);
+
+  // commit dirs
+  for (elist::iterator p = new_dirfrags.begin(); !p.end(); ++p) {
+    dout(20) << " new_dirfrag " << **p << dendl;
+    assert((*p)->is_auth());
+    commit.insert(*p);
+  }
+  for (elist::iterator p = dirty_dirfrags.begin(); !p.end(); ++p) {
+    dout(20) << " dirty_dirfrag " << **p << dendl;
+    assert((*p)->is_auth());
+    commit.insert(*p);
+  }
+  for (elist::iterator p = dirty_dentries.begin(); !p.end(); ++p) {
+    dout(20) << " dirty_dentry " << **p << dendl;
+    assert((*p)->is_auth());
+    commit.insert((*p)->get_dir());
+  }
+  for (elist::iterator p = dirty_inodes.begin(); !p.end(); ++p) {
+    dout(20) << " dirty_inode " << **p << dendl;
+    assert((*p)->is_auth());
+    if ((*p)->is_base()) {
+      (*p)->store(gather_bld.new_sub());
+    } else
+      commit.insert((*p)->get_parent_dn()->get_dir());
+  }
+
+  if (!commit.empty()) {
+    for (set::iterator p = commit.begin();
+	 p != commit.end();
+	 ++p) {
+      CDir *dir = *p;
+      assert(dir->is_auth());
+      if (dir->can_auth_pin()) {
+	dout(15) << "try_to_expire committing " << *dir << dendl;
+	dir->commit(0, gather_bld.new_sub(), false, op_prio);
+      } else {
+	dout(15) << "try_to_expire waiting for unfreeze on " << *dir << dendl;
+	dir->add_waiter(CDir::WAIT_UNFREEZE, gather_bld.new_sub());
+      }
+    }
+  }
+
+  // master ops with possibly uncommitted slaves
+  for (set::iterator p = uncommitted_masters.begin();
+       p != uncommitted_masters.end();
+       ++p) {
+    dout(10) << "try_to_expire waiting for slaves to ack commit on " << *p << dendl;
+    mds->mdcache->wait_for_uncommitted_master(*p, gather_bld.new_sub());
+  }
+
+  // uncommitted fragments
+  for (set::iterator p = uncommitted_fragments.begin();
+       p != uncommitted_fragments.end();
+       ++p) {
+    dout(10) << "try_to_expire waiting for uncommitted fragment " << *p << dendl;
+    mds->mdcache->wait_for_uncommitted_fragment(*p, gather_bld.new_sub());
+  }
+
+  // nudge scatterlocks
+  for (elist::iterator p = dirty_dirfrag_dir.begin(); !p.end(); ++p) {
+    CInode *in = *p;
+    dout(10) << "try_to_expire waiting for dirlock flush on " << *in << dendl;
+    mds->locker->scatter_nudge(&in->filelock, gather_bld.new_sub());
+  }
+  for (elist::iterator p = dirty_dirfrag_dirfragtree.begin(); !p.end(); ++p) {
+    CInode *in = *p;
+    dout(10) << "try_to_expire waiting for dirfragtreelock flush on " << *in << dendl;
+    mds->locker->scatter_nudge(&in->dirfragtreelock, gather_bld.new_sub());
+  }
+  for (elist::iterator p = dirty_dirfrag_nest.begin(); !p.end(); ++p) {
+    CInode *in = *p;
+    dout(10) << "try_to_expire waiting for nest flush on " << *in << dendl;
+    mds->locker->scatter_nudge(&in->nestlock, gather_bld.new_sub());
+  }
+
+  assert(g_conf->mds_kill_journal_expire_at != 2);
+
+  // open files
+  if (!open_files.empty()) {
+    assert(!mds->mdlog->is_capped()); // hmm FIXME
+    EOpen *le = 0;
+    LogSegment *ls = mds->mdlog->get_current_segment();
+    assert(ls != this);
+    elist::iterator p = open_files.begin(member_offset(CInode, item_open_file));
+    while (!p.end()) {
+      CInode *in = *p;
+      assert(in->last == CEPH_NOSNAP);
+      ++p;
+      if (in->is_auth() && !in->is_ambiguous_auth() && in->is_any_caps()) {
+	if (in->is_any_caps_wanted()) {
+	  dout(20) << "try_to_expire requeueing open file " << *in << dendl;
+	  if (!le) {
+	    le = new EOpen(mds->mdlog);
+	    mds->mdlog->start_entry(le);
+	  }
+	  le->add_clean_inode(in);
+	  ls->open_files.push_back(&in->item_open_file);
+	} else {
+	  // drop inodes that aren't wanted
+	  dout(20) << "try_to_expire not requeueing and delisting unwanted file " << *in << dendl;
+	  in->item_open_file.remove_myself();
+	}
+      } else {
+	/*
+	 * we can get a capless inode here if we replay an open file, the client fails to
+	 * reconnect it, but does REPLAY an open request (that adds it to the logseg).  AFAICS
+	 * it's ok for the client to replay an open on a file it doesn't have in it's cache
+	 * anymore.
+	 *
+	 * this makes the mds less sensitive to strict open_file consistency, although it does
+	 * make it easier to miss subtle problems.
+	 */
+	dout(20) << "try_to_expire not requeueing and delisting capless file " << *in << dendl;
+	in->item_open_file.remove_myself();
+      }
+    }
+    if (le) {
+      mds->mdlog->submit_entry(le, gather_bld.new_sub());
+      dout(10) << "try_to_expire waiting for open files to rejournal" << dendl;
+    }
+  }
+
+  assert(g_conf->mds_kill_journal_expire_at != 3);
+
+  // backtraces to be stored/updated
+  for (elist::iterator p = dirty_parent_inodes.begin(); !p.end(); ++p) {
+    CInode *in = *p;
+    assert(in->is_auth());
+    if (in->can_auth_pin()) {
+      dout(15) << "try_to_expire waiting for storing backtrace on " << *in << dendl;
+      in->store_backtrace(gather_bld.new_sub());
+    } else {
+      dout(15) << "try_to_expire waiting for unfreeze on " << *in << dendl;
+      in->add_waiter(CInode::WAIT_UNFREEZE, gather_bld.new_sub());
+    }
+  }
+
+  assert(g_conf->mds_kill_journal_expire_at != 4);
+
+  // slave updates
+  for (elist::iterator p = slave_updates.begin(member_offset(MDSlaveUpdate,
+									     item));
+       !p.end(); ++p) {
+    MDSlaveUpdate *su = *p;
+    dout(10) << "try_to_expire waiting on slave update " << su << dendl;
+    assert(su->waiter == 0);
+    su->waiter = gather_bld.new_sub();
+  }
+
+  // idalloc
+  if (inotablev > mds->inotable->get_committed_version()) {
+    dout(10) << "try_to_expire saving inotable table, need " << inotablev
+	      << ", committed is " << mds->inotable->get_committed_version()
+	      << " (" << mds->inotable->get_committing_version() << ")"
+	      << dendl;
+    mds->inotable->save(gather_bld.new_sub(), inotablev);
+  }
+
+  // sessionmap
+  if (sessionmapv > mds->sessionmap.committed) {
+    dout(10) << "try_to_expire saving sessionmap, need " << sessionmapv 
+	      << ", committed is " << mds->sessionmap.committed
+	      << " (" << mds->sessionmap.committing << ")"
+	      << dendl;
+    mds->sessionmap.save(gather_bld.new_sub(), sessionmapv);
+  }
+
+  // pending commit atids
+  for (map >::iterator p = pending_commit_tids.begin();
+       p != pending_commit_tids.end();
+       ++p) {
+    MDSTableClient *client = mds->get_table_client(p->first);
+    for (ceph::unordered_set::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      dout(10) << "try_to_expire " << get_mdstable_name(p->first) << " transaction " << *q 
+	       << " pending commit (not yet acked), waiting" << dendl;
+      assert(!client->has_committed(*q));
+      client->wait_for_ack(*q, gather_bld.new_sub());
+    }
+  }
+  
+  // table servers
+  for (map::iterator p = tablev.begin();
+       p != tablev.end();
+       ++p) {
+    MDSTableServer *server = mds->get_table_server(p->first);
+    if (p->second > server->get_committed_version()) {
+      dout(10) << "try_to_expire waiting for " << get_mdstable_name(p->first) 
+	       << " to save, need " << p->second << dendl;
+      server->save(gather_bld.new_sub());
+    }
+  }
+
+  // truncating
+  for (set::iterator p = truncating_inodes.begin();
+       p != truncating_inodes.end();
+       ++p) {
+    dout(10) << "try_to_expire waiting for truncate of " << **p << dendl;
+    (*p)->add_waiter(CInode::WAIT_TRUNC, gather_bld.new_sub());
+  }
+  
+  // FIXME client requests...?
+  // audit handling of anchor transactions?
+
+  if (gather_bld.has_subs()) {
+    dout(6) << "LogSegment(" << offset << ").try_to_expire waiting" << dendl;
+    mds->mdlog->flush();
+  } else {
+    assert(g_conf->mds_kill_journal_expire_at != 5);
+    dout(6) << "LogSegment(" << offset << ").try_to_expire success" << dendl;
+  }
+}
+
+#undef DOUT_COND
+#define DOUT_COND(cct, l) (l<=cct->_conf->debug_mds || l <= cct->_conf->debug_mds_log)
+
+
+// -----------------------
+// EMetaBlob
+
+EMetaBlob::EMetaBlob(MDLog *mdlog) : opened_ino(0), renamed_dirino(0),
+				     inotablev(0), sessionmapv(0),
+				     allocated_ino(0),
+				     last_subtree_map(mdlog ? mdlog->get_last_segment_offset() : 0),
+				     my_offset(mdlog ? mdlog->get_write_pos() : 0) //, _segment(0)
+{ }
+
+void EMetaBlob::add_dir_context(CDir *dir, int mode)
+{
+  MDS *mds = dir->cache->mds;
+
+  list parents;
+
+  // it may be okay not to include the maybe items, if
+  //  - we journaled the maybe child inode in this segment
+  //  - that subtree turns out to be unambiguously auth
+  list maybe;
+  bool maybenot = false;
+
+  while (true) {
+    // already have this dir?  (we must always add in order)
+    if (lump_map.count(dir->dirfrag())) {
+      dout(20) << "EMetaBlob::add_dir_context(" << dir << ") have lump " << dir->dirfrag() << dendl;
+      break;
+    }
+      
+    // stop at root/stray
+    CInode *diri = dir->get_inode();
+    CDentry *parent = diri->get_projected_parent_dn();
+
+    if (!parent)
+      break;
+
+    if (mode == TO_AUTH_SUBTREE_ROOT) {
+      // subtree root?
+      if (dir->is_subtree_root() && !dir->state_test(CDir::STATE_EXPORTBOUND)) {
+	if (dir->is_auth() && !dir->is_ambiguous_auth()) {
+	  // it's an auth subtree, we don't need maybe (if any), and we're done.
+	  dout(20) << "EMetaBlob::add_dir_context(" << dir << ") reached unambig auth subtree, don't need " << maybe
+		   << " at " << *dir << dendl;
+	  maybe.clear();
+	  break;
+	} else {
+	  dout(20) << "EMetaBlob::add_dir_context(" << dir << ") reached ambig or !auth subtree, need " << maybe
+		   << " at " << *dir << dendl;
+	  // we need the maybe list after all!
+	  parents.splice(parents.begin(), maybe);
+	  maybenot = false;
+	}
+      }
+      
+      // was the inode journaled in this blob?
+      if (my_offset && diri->last_journaled == my_offset) {
+	dout(20) << "EMetaBlob::add_dir_context(" << dir << ") already have diri this blob " << *diri << dendl;
+	break;
+      }
+
+      // have we journaled this inode since the last subtree map?
+      if (!maybenot && last_subtree_map && diri->last_journaled >= last_subtree_map) {
+	dout(20) << "EMetaBlob::add_dir_context(" << dir << ") already have diri in this segment (" 
+		 << diri->last_journaled << " >= " << last_subtree_map << "), setting maybenot flag "
+		 << *diri << dendl;
+	maybenot = true;
+      }
+    }
+
+    if (maybenot) {
+      dout(25) << "EMetaBlob::add_dir_context(" << dir << ")      maybe " << *parent << dendl;
+      maybe.push_front(parent);
+    } else {
+      dout(25) << "EMetaBlob::add_dir_context(" << dir << ") definitely " << *parent << dendl;
+      parents.push_front(parent);
+    }
+    
+    dir = parent->get_dir();
+  }
+  
+  parents.splice(parents.begin(), maybe);
+
+  dout(20) << "EMetaBlob::add_dir_context final: " << parents << dendl;
+  for (list::iterator p = parents.begin(); p != parents.end(); ++p) {
+    assert((*p)->get_projected_linkage()->is_primary());
+    add_dentry(*p, false);
+  }
+}
+
+void EMetaBlob::update_segment(LogSegment *ls)
+{
+  // atids?
+  //for (list::iterator p = atids.begin(); p != atids.end(); ++p)
+  //  ls->pending_commit_atids[*p] = ls;
+  // -> handled directly by AnchorClient
+
+  // dirty inode mtimes
+  // -> handled directly by Server.cc, replay()
+
+  // alloc table update?
+  if (inotablev)
+    ls->inotablev = inotablev;
+  if (sessionmapv)
+    ls->sessionmapv = sessionmapv;
+
+  // truncated inodes
+  // -> handled directly by Server.cc
+
+  // client requests
+  //  note the newest request per client
+  //if (!client_reqs.empty())
+    //    ls->last_client_tid[client_reqs.rbegin()->client] = client_reqs.rbegin()->tid);
+}
+
+// EMetaBlob::fullbit
+
+void EMetaBlob::fullbit::encode(bufferlist& bl) const {
+  ENCODE_START(6, 5, bl);
+  if (!_enc.length()) {
+    fullbit copy(dn, dnfirst, dnlast, dnv, inode, dirfragtree, xattrs, symlink,
+		 snapbl, state, &old_inodes);
+    bl.append(copy._enc);
+  } else {
+    bl.append(_enc);
+  }
+  ENCODE_FINISH(bl);
+}
+
+void EMetaBlob::fullbit::decode(bufferlist::iterator &bl) {
+  DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, bl);
+  ::decode(dn, bl);
+  ::decode(dnfirst, bl);
+  ::decode(dnlast, bl);
+  ::decode(dnv, bl);
+  ::decode(inode, bl);
+  ::decode(xattrs, bl);
+  if (inode.is_symlink())
+    ::decode(symlink, bl);
+  if (inode.is_dir()) {
+    ::decode(dirfragtree, bl);
+    ::decode(snapbl, bl);
+    if ((struct_v == 2) || (struct_v == 3)) {
+      bool dir_layout_exists;
+      ::decode(dir_layout_exists, bl);
+      if (dir_layout_exists) {
+	__u8 dir_struct_v;
+	::decode(dir_struct_v, bl); // default_file_layout version
+	::decode(inode.layout, bl); // and actual layout, that we care about
+      }
+    }
+  }
+  if (struct_v >= 6) {
+    ::decode(state, bl);
+  } else {
+    bool dirty;
+    ::decode(dirty, bl);
+    state = dirty ? EMetaBlob::fullbit::STATE_DIRTY : 0;
+  }
+
+  if (struct_v >= 3) {
+    bool old_inodes_present;
+    ::decode(old_inodes_present, bl);
+    if (old_inodes_present) {
+      ::decode(old_inodes, bl);
+    }
+  }
+  DECODE_FINISH(bl);
+}
+
+void EMetaBlob::fullbit::dump(Formatter *f) const
+{
+  if (_enc.length() && !dn.length()) {
+    /* if our bufferlist has data but our name is empty, we
+     * haven't initialized ourselves; do so in order to print members!
+     * We use const_cast here because the whole point is we aren't
+     * fully set up and this isn't changing who we "are", just our
+     * representation.
+     */
+    EMetaBlob::fullbit *me = const_cast(this);
+    bufferlist encoded;
+    encode(encoded);
+    bufferlist::iterator p = encoded.begin();
+    me->decode(p);
+  }
+  f->dump_string("dentry", dn);
+  f->dump_stream("snapid.first") << dnfirst;
+  f->dump_stream("snapid.last") << dnlast;
+  f->dump_int("dentry version", dnv);
+  f->open_object_section("inode");
+  inode.dump(f);
+  f->close_section(); // inode
+  f->open_array_section("xattrs");
+  for (map::const_iterator iter = xattrs.begin();
+      iter != xattrs.end(); ++iter) {
+    f->dump_string(iter->first.c_str(), iter->second.c_str());
+  }
+  f->close_section(); // xattrs
+  if (inode.is_symlink()) {
+    f->dump_string("symlink", symlink);
+  }
+  if (inode.is_dir()) {
+    f->dump_stream("frag tree") << dirfragtree;
+    f->dump_string("has_snapbl", snapbl.length() ? "true" : "false");
+    if (inode.has_layout()) {
+      f->open_object_section("file layout policy");
+      // FIXME
+      f->dump_string("layout", "the layout exists");
+      f->close_section(); // file layout policy
+    }
+  }
+  f->dump_string("state", state_string());
+  if (!old_inodes.empty()) {
+    f->open_array_section("old inodes");
+    for (old_inodes_t::const_iterator iter = old_inodes.begin();
+	iter != old_inodes.end(); ++iter) {
+      f->open_object_section("inode");
+      f->dump_int("snapid", iter->first);
+      iter->second.dump(f);
+      f->close_section(); // inode
+    }
+    f->close_section(); // old inodes
+  }
+}
+
+void EMetaBlob::fullbit::generate_test_instances(list& ls)
+{
+  inode_t inode;
+  fragtree_t fragtree;
+  map empty_xattrs;
+  bufferlist empty_snapbl;
+  fullbit *sample = new fullbit("/testdn", 0, 0, 0,
+                                inode, fragtree, empty_xattrs, "", empty_snapbl,
+                                false, NULL);
+  ls.push_back(sample);
+}
+
+void EMetaBlob::fullbit::update_inode(MDS *mds, CInode *in)
+{
+  in->inode = inode;
+  in->xattrs = xattrs;
+  if (in->inode.is_dir()) {
+    if (!(in->dirfragtree == dirfragtree)) {
+      dout(10) << "EMetaBlob::fullbit::update_inode dft " << in->dirfragtree << " -> "
+	       << dirfragtree << " on " << *in << dendl;
+      in->dirfragtree = dirfragtree;
+      in->force_dirfrags();
+      if (in->has_dirfrags() && in->authority() == CDIR_AUTH_UNDEF) {
+	list ls;
+	in->get_nested_dirfrags(ls);
+	for (list::iterator p = ls.begin(); p != ls.end(); ++p) {
+	  CDir *dir = *p;
+	  if (dir->get_num_any() == 0 &&
+	      mds->mdcache->can_trim_non_auth_dirfrag(dir)) {
+	    dout(10) << " closing empty non-auth dirfrag " << *dir << dendl;
+	    in->close_dirfrag(dir->get_frag());
+	  }
+	}
+      }
+    }
+
+    /*
+     * we can do this before linking hte inode bc the split_at would
+     * be a no-op.. we have no children (namely open snaprealms) to
+     * divy up 
+     */
+    in->decode_snap_blob(snapbl);  
+  } else if (in->inode.is_symlink()) {
+    in->symlink = symlink;
+  }
+  in->old_inodes = old_inodes;
+}
+
+// EMetaBlob::remotebit
+
+void EMetaBlob::remotebit::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  if (!_enc.length()) {
+    remotebit copy(dn, dnfirst, dnlast, dnv, ino, d_type, dirty);
+    bl.append(copy._enc);
+  } else {
+    bl.append(_enc);
+  }
+  ENCODE_FINISH(bl);
+}
+
+void EMetaBlob::remotebit::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(dn, bl);
+  ::decode(dnfirst, bl);
+  ::decode(dnlast, bl);
+  ::decode(dnv, bl);
+  ::decode(ino, bl);
+  ::decode(d_type, bl);
+  ::decode(dirty, bl);
+  DECODE_FINISH(bl);
+}
+
+void EMetaBlob::remotebit::dump(Formatter *f) const
+{
+  if (_enc.length() && !dn.length()) {
+    /* if our bufferlist has data but our name is empty, we
+     * haven't initialized ourselves; do so in order to print members!
+     * We use const_cast here because the whole point is we aren't
+     * fully set up and this isn't changing who we "are", just our
+     * representation.
+     */
+    EMetaBlob::remotebit *me = const_cast(this);
+    bufferlist encoded;
+    encode(encoded);
+    bufferlist::iterator p = encoded.begin();
+    me->decode(p);
+  }
+  f->dump_string("dentry", dn);
+  f->dump_int("snapid.first", dnfirst);
+  f->dump_int("snapid.last", dnlast);
+  f->dump_int("dentry version", dnv);
+  f->dump_int("inodeno", ino);
+  uint32_t type = DTTOIF(d_type) & S_IFMT; // convert to type entries
+  string type_string;
+  switch(type) {
+  case S_IFREG:
+    type_string = "file"; break;
+  case S_IFLNK:
+    type_string = "symlink"; break;
+  case S_IFDIR:
+    type_string = "directory"; break;
+  case S_IFIFO:
+    type_string = "fifo"; break;
+  case S_IFCHR:
+    type_string = "chr"; break;
+  case S_IFBLK:
+    type_string = "blk"; break;
+  case S_IFSOCK:
+    type_string = "sock"; break;
+  default:
+    assert (0 == "unknown d_type!");
+  }
+  f->dump_string("d_type", type_string);
+  f->dump_string("dirty", dirty ? "true" : "false");
+}
+
+void EMetaBlob::remotebit::
+generate_test_instances(list& ls)
+{
+  remotebit *remote = new remotebit("/test/dn", 0, 10, 15, 1, IFTODT(S_IFREG), false);
+  ls.push_back(remote);
+}
+
+// EMetaBlob::nullbit
+
+void EMetaBlob::nullbit::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  if (!_enc.length()) {
+    nullbit copy(dn, dnfirst, dnlast, dnv, dirty);
+    bl.append(copy._enc);
+  } else {
+    bl.append(_enc);
+  }
+  ENCODE_FINISH(bl);
+}
+
+void EMetaBlob::nullbit::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(dn, bl);
+  ::decode(dnfirst, bl);
+  ::decode(dnlast, bl);
+  ::decode(dnv, bl);
+  ::decode(dirty, bl);
+  DECODE_FINISH(bl);
+}
+
+void EMetaBlob::nullbit::dump(Formatter *f) const
+{
+  if (_enc.length() && !dn.length()) {
+    /* if our bufferlist has data but our name is empty, we
+     * haven't initialized ourselves; do so in order to print members!
+     * We use const_cast here because the whole point is we aren't
+     * fully set up and this isn't changing who we "are", just our
+     * representation.
+     */
+    EMetaBlob::nullbit *me = const_cast(this);
+    bufferlist encoded;
+    encode(encoded);
+    bufferlist::iterator p = encoded.begin();
+    me->decode(p);
+  }
+  f->dump_string("dentry", dn);
+  f->dump_int("snapid.first", dnfirst);
+  f->dump_int("snapid.last", dnlast);
+  f->dump_int("dentry version", dnv);
+  f->dump_string("dirty", dirty ? "true" : "false");
+}
+
+void EMetaBlob::nullbit::generate_test_instances(list& ls)
+{
+  nullbit *sample = new nullbit("/test/dentry", 0, 10, 15, false);
+  nullbit *sample2 = new nullbit("/test/dirty", 10, 20, 25, true);
+  ls.push_back(sample);
+  ls.push_back(sample2);
+}
+
+// EMetaBlob::dirlump
+
+void EMetaBlob::dirlump::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(fnode, bl);
+  ::encode(state, bl);
+  ::encode(nfull, bl);
+  ::encode(nremote, bl);
+  ::encode(nnull, bl);
+  _encode_bits();
+  ::encode(dnbl, bl);
+  ENCODE_FINISH(bl);
+}
+
+void EMetaBlob::dirlump::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl)
+  ::decode(fnode, bl);
+  ::decode(state, bl);
+  ::decode(nfull, bl);
+  ::decode(nremote, bl);
+  ::decode(nnull, bl);
+  ::decode(dnbl, bl);
+  dn_decoded = false;      // don't decode bits unless we need them.
+  DECODE_FINISH(bl);
+}
+
+void EMetaBlob::dirlump::dump(Formatter *f) const
+{
+  if (!dn_decoded) {
+    dirlump *me = const_cast(this);
+    me->_decode_bits();
+  }
+  f->open_object_section("fnode");
+  fnode.dump(f);
+  f->close_section(); // fnode
+  f->dump_string("state", state_string());
+  f->dump_int("nfull", nfull);
+  f->dump_int("nremote", nremote);
+  f->dump_int("nnull", nnull);
+
+  f->open_array_section("full bits");
+  for (list >::const_iterator
+      iter = dfull.begin(); iter != dfull.end(); ++iter) {
+    f->open_object_section("fullbit");
+    (*iter)->dump(f);
+    f->close_section(); // fullbit
+  }
+  f->close_section(); // full bits
+  f->open_array_section("remote bits");
+  for (list::const_iterator
+      iter = dremote.begin(); iter != dremote.end(); ++iter) {
+    f->open_object_section("remotebit");
+    (*iter).dump(f);
+    f->close_section(); // remotebit
+  }
+  f->close_section(); // remote bits
+  f->open_array_section("null bits");
+  for (list::const_iterator
+      iter = dnull.begin(); iter != dnull.end(); ++iter) {
+    f->open_object_section("null bit");
+    (*iter).dump(f);
+    f->close_section(); // null bit
+  }
+  f->close_section(); // null bits
+}
+
+void EMetaBlob::dirlump::generate_test_instances(list& ls)
+{
+  ls.push_back(new dirlump());
+}
+
+/**
+ * EMetaBlob proper
+ */
+void EMetaBlob::encode(bufferlist& bl) const
+{
+  ENCODE_START(7, 5, bl);
+  ::encode(lump_order, bl);
+  ::encode(lump_map, bl);
+  ::encode(roots, bl);
+  ::encode(table_tids, bl);
+  ::encode(opened_ino, bl);
+  ::encode(allocated_ino, bl);
+  ::encode(used_preallocated_ino, bl);
+  ::encode(preallocated_inos, bl);
+  ::encode(client_name, bl);
+  ::encode(inotablev, bl);
+  ::encode(sessionmapv, bl);
+  ::encode(truncate_start, bl);
+  ::encode(truncate_finish, bl);
+  ::encode(destroyed_inodes, bl);
+  ::encode(client_reqs, bl);
+  ::encode(renamed_dirino, bl);
+  ::encode(renamed_dir_frags, bl);
+  {
+    // make MDS use v6 format happy
+    int64_t i = -1;
+    bool b = false;
+    ::encode(i, bl);
+    ::encode(b, bl);
+  }
+  ENCODE_FINISH(bl);
+}
+void EMetaBlob::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
+  ::decode(lump_order, bl);
+  ::decode(lump_map, bl);
+  if (struct_v >= 4) {
+    ::decode(roots, bl);
+  } else {
+    bufferlist rootbl;
+    ::decode(rootbl, bl);
+    if (rootbl.length()) {
+      bufferlist::iterator p = rootbl.begin();
+      roots.push_back(ceph::shared_ptr(new fullbit(p)));
+    }
+  }
+  ::decode(table_tids, bl);
+  ::decode(opened_ino, bl);
+  ::decode(allocated_ino, bl);
+  ::decode(used_preallocated_ino, bl);
+  ::decode(preallocated_inos, bl);
+  ::decode(client_name, bl);
+  ::decode(inotablev, bl);
+  ::decode(sessionmapv, bl);
+  ::decode(truncate_start, bl);
+  ::decode(truncate_finish, bl);
+  ::decode(destroyed_inodes, bl);
+  if (struct_v >= 2) {
+    ::decode(client_reqs, bl);
+  } else {
+    list r;
+    ::decode(r, bl);
+    while (!r.empty()) {
+	client_reqs.push_back(pair(r.front(), 0));
+	r.pop_front();
+    }
+  }
+  if (struct_v >= 3) {
+    ::decode(renamed_dirino, bl);
+    ::decode(renamed_dir_frags, bl);
+  }
+  if (struct_v >= 6) {
+    // ignore
+    int64_t i;
+    bool b;
+    ::decode(i, bl);
+    ::decode(b, bl);
+  }
+  DECODE_FINISH(bl);
+}
+
+void EMetaBlob::dump(Formatter *f) const
+{
+  f->open_array_section("lumps");
+  for (list::const_iterator i = lump_order.begin();
+       i != lump_order.end(); ++i) {
+    f->open_object_section("lump");
+    f->open_object_section("dirfrag");
+    f->dump_stream("dirfrag") << *i;
+    f->close_section(); // dirfrag
+    f->open_object_section("dirlump");
+    lump_map.at(*i).dump(f);
+    f->close_section(); // dirlump
+    f->close_section(); // lump
+  }
+  f->close_section(); // lumps
+  
+  f->open_array_section("roots");
+  for (list >::const_iterator i = roots.begin();
+       i != roots.end(); ++i) {
+    f->open_object_section("root");
+    (*i)->dump(f);
+    f->close_section(); // root
+  }
+  f->close_section(); // roots
+
+  f->open_array_section("tableclient tranactions");
+  for (list >::const_iterator i = table_tids.begin();
+       i != table_tids.end(); ++i) {
+    f->open_object_section("transaction");
+    f->dump_int("tid", i->first);
+    f->dump_int("version", i->second);
+    f->close_section(); // transaction
+  }
+  f->close_section(); // tableclient transactions
+  
+  f->dump_int("renamed directory inodeno", renamed_dirino);
+  
+  f->open_array_section("renamed directory fragments");
+  for (list::const_iterator i = renamed_dir_frags.begin();
+       i != renamed_dir_frags.end(); ++i) {
+    f->dump_int("frag", *i);
+  }
+  f->close_section(); // renamed directory fragments
+
+  f->dump_int("inotable version", inotablev);
+  f->dump_int("SesionMap version", sessionmapv);
+  f->dump_int("allocated ino", allocated_ino);
+  
+  f->dump_stream("preallocated inos") << preallocated_inos;
+  f->dump_int("used preallocated ino", used_preallocated_ino);
+
+  f->open_object_section("client name");
+  client_name.dump(f);
+  f->close_section(); // client name
+
+  f->open_array_section("inodes starting a truncate");
+  for(list::const_iterator i = truncate_start.begin();
+      i != truncate_start.end(); ++i) {
+    f->dump_int("inodeno", *i);
+  }
+  f->close_section(); // truncate inodes
+  f->open_array_section("inodes finishing a truncated");
+  for(map::const_iterator i = truncate_finish.begin();
+      i != truncate_finish.end(); ++i) {
+    f->open_object_section("inode+segment");
+    f->dump_int("inodeno", i->first);
+    f->dump_int("truncate starting segment", i->second);
+    f->close_section(); // truncated inode
+  }
+  f->close_section(); // truncate finish inodes
+
+  f->open_array_section("destroyed inodes");
+  for(vector::const_iterator i = destroyed_inodes.begin();
+      i != destroyed_inodes.end(); ++i) {
+    f->dump_int("inodeno", *i);
+  }
+  f->close_section(); // destroyed inodes
+
+  f->open_array_section("client requests");
+  for(list >::const_iterator i = client_reqs.begin();
+      i != client_reqs.end(); ++i) {
+    f->open_object_section("Client request");
+    f->dump_stream("request ID") << i->first;
+    f->dump_int("oldest request on client", i->second);
+    f->close_section(); // request
+  }
+  f->close_section(); // client requests
+}
+
+void EMetaBlob::generate_test_instances(list& ls)
+{
+  ls.push_back(new EMetaBlob());
+}
+
+void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
+{
+  dout(10) << "EMetaBlob.replay " << lump_map.size() << " dirlumps by " << client_name << dendl;
+
+  assert(logseg);
+
+  assert(g_conf->mds_kill_journal_replay_at != 1);
+
+  for (list >::iterator p = roots.begin(); p != roots.end(); ++p) {
+    CInode *in = mds->mdcache->get_inode((*p)->inode.ino);
+    bool isnew = in ? false:true;
+    if (!in)
+      in = new CInode(mds->mdcache, true);
+    (*p)->update_inode(mds, in);
+
+    if (isnew)
+      mds->mdcache->add_inode(in);
+    if ((*p)->is_dirty()) in->_mark_dirty(logseg);
+    dout(10) << "EMetaBlob.replay " << (isnew ? " added root ":" updated root ") << *in << dendl;    
+  }
+
+  CInode *renamed_diri = 0;
+  CDir *olddir = 0;
+  if (renamed_dirino) {
+    renamed_diri = mds->mdcache->get_inode(renamed_dirino);
+    if (renamed_diri)
+      dout(10) << "EMetaBlob.replay renamed inode is " << *renamed_diri << dendl;
+    else
+      dout(10) << "EMetaBlob.replay don't have renamed ino " << renamed_dirino << dendl;
+
+    int nnull = 0;
+    for (list::iterator lp = lump_order.begin(); lp != lump_order.end(); ++lp) {
+      dirlump &lump = lump_map[*lp];
+      if (lump.nnull) {
+	dout(10) << "EMetaBlob.replay found null dentry in dir " << *lp << dendl;
+	nnull += lump.nnull;
+      }
+    }
+    assert(nnull <= 1);
+  }
+
+  // keep track of any inodes we unlink and don't relink elsewhere
+  map unlinked;
+  set linked;
+
+  // walk through my dirs (in order!)
+  for (list::iterator lp = lump_order.begin();
+       lp != lump_order.end();
+       ++lp) {
+    dout(10) << "EMetaBlob.replay dir " << *lp << dendl;
+    dirlump &lump = lump_map[*lp];
+
+    // the dir 
+    CDir *dir = mds->mdcache->get_force_dirfrag(*lp);
+    if (!dir) {
+      // hmm.  do i have the inode?
+      CInode *diri = mds->mdcache->get_inode((*lp).ino);
+      if (!diri) {
+	if (MDS_INO_IS_BASE(lp->ino)) {
+	  diri = mds->mdcache->create_system_inode(lp->ino, S_IFDIR|0755);
+	  dout(10) << "EMetaBlob.replay created base " << *diri << dendl;
+	} else {
+	  dout(0) << "EMetaBlob.replay missing dir ino  " << (*lp).ino << dendl;
+	  assert(0);
+	}
+      }
+
+      // create the dirfrag
+      dir = diri->get_or_open_dirfrag(mds->mdcache, (*lp).frag);
+
+      if (MDS_INO_IS_BASE(lp->ino))
+	mds->mdcache->adjust_subtree_auth(dir, CDIR_AUTH_UNKNOWN);
+
+      dout(10) << "EMetaBlob.replay added dir " << *dir << dendl;  
+    }
+    dir->set_version( lump.fnode.version );
+    dir->fnode = lump.fnode;
+
+    if (lump.is_dirty()) {
+      dir->_mark_dirty(logseg);
+
+      if (!(dir->fnode.rstat == dir->fnode.accounted_rstat)) {
+	dout(10) << "EMetaBlob.replay      dirty nestinfo on " << *dir << dendl;
+	mds->locker->mark_updated_scatterlock(&dir->inode->nestlock);
+	logseg->dirty_dirfrag_nest.push_back(&dir->inode->item_dirty_dirfrag_nest);
+      } else {
+	dout(10) << "EMetaBlob.replay      clean nestinfo on " << *dir << dendl;
+      }
+      if (!(dir->fnode.fragstat == dir->fnode.accounted_fragstat)) {
+	dout(10) << "EMetaBlob.replay      dirty fragstat on " << *dir << dendl;
+	mds->locker->mark_updated_scatterlock(&dir->inode->filelock);
+	logseg->dirty_dirfrag_dir.push_back(&dir->inode->item_dirty_dirfrag_dir);
+      } else {
+	dout(10) << "EMetaBlob.replay      clean fragstat on " << *dir << dendl;
+      }
+    }
+    if (lump.is_dirty_dft()) {
+      dout(10) << "EMetaBlob.replay      dirty dirfragtree on " << *dir << dendl;
+      dir->state_set(CDir::STATE_DIRTYDFT);
+      mds->locker->mark_updated_scatterlock(&dir->inode->dirfragtreelock);
+      logseg->dirty_dirfrag_dirfragtree.push_back(&dir->inode->item_dirty_dirfrag_dirfragtree);
+    }
+    if (lump.is_new())
+      dir->mark_new(logseg);
+    if (lump.is_complete())
+      dir->mark_complete();
+    else if (lump.is_importing())
+      dir->state_clear(CDir::STATE_COMPLETE);
+    
+    dout(10) << "EMetaBlob.replay updated dir " << *dir << dendl;  
+
+    // decode bits
+    lump._decode_bits();
+
+    // full dentry+inode pairs
+    for (list >::iterator pp = lump.get_dfull().begin();
+	 pp != lump.get_dfull().end();
+	 ++pp) {
+      ceph::shared_ptr p = *pp;
+      CDentry *dn = dir->lookup_exact_snap(p->dn, p->dnlast);
+      if (!dn) {
+	dn = dir->add_null_dentry(p->dn, p->dnfirst, p->dnlast);
+	dn->set_version(p->dnv);
+	if (p->is_dirty()) dn->_mark_dirty(logseg);
+	dout(10) << "EMetaBlob.replay added " << *dn << dendl;
+      } else {
+	dn->set_version(p->dnv);
+	if (p->is_dirty()) dn->_mark_dirty(logseg);
+	dout(10) << "EMetaBlob.replay for [" << p->dnfirst << "," << p->dnlast << "] had " << *dn << dendl;
+	dn->first = p->dnfirst;
+	assert(dn->last == p->dnlast);
+      }
+
+      CInode *in = mds->mdcache->get_inode(p->inode.ino, p->dnlast);
+      if (!in) {
+	in = new CInode(mds->mdcache, true, p->dnfirst, p->dnlast);
+	p->update_inode(mds, in);
+	mds->mdcache->add_inode(in);
+	if (!dn->get_linkage()->is_null()) {
+	  if (dn->get_linkage()->is_primary()) {
+	    unlinked[dn->get_linkage()->get_inode()] = dir;
+	    stringstream ss;
+	    ss << "EMetaBlob.replay FIXME had dentry linked to wrong inode " << *dn
+	       << " " << *dn->get_linkage()->get_inode() << " should be " << p->inode.ino;
+	    dout(0) << ss.str() << dendl;
+	    mds->clog.warn(ss);
+	  }
+	  dir->unlink_inode(dn);
+	}
+	if (unlinked.count(in))
+	  linked.insert(in);
+	dir->link_primary_inode(dn, in);
+	if (p->is_dirty()) in->_mark_dirty(logseg);
+	dout(10) << "EMetaBlob.replay added " << *in << dendl;
+      } else {
+	if (in->get_parent_dn() && in->inode.anchored != p->inode.anchored)
+	  in->get_parent_dn()->adjust_nested_anchors((int)p->inode.anchored - (int)in->inode.anchored);
+	p->update_inode(mds, in);
+	if (dn->get_linkage()->get_inode() != in && in->get_parent_dn()) {
+	  dout(10) << "EMetaBlob.replay unlinking " << *in << dendl;
+	  unlinked[in] = in->get_parent_dir();
+	  in->get_parent_dir()->unlink_inode(in->get_parent_dn());
+	}
+	if (dn->get_linkage()->get_inode() != in) {
+	  if (!dn->get_linkage()->is_null()) { // note: might be remote.  as with stray reintegration.
+	    if (dn->get_linkage()->is_primary()) {
+	      unlinked[dn->get_linkage()->get_inode()] = dir;
+	      stringstream ss;
+	      ss << "EMetaBlob.replay FIXME had dentry linked to wrong inode " << *dn
+		 << " " << *dn->get_linkage()->get_inode() << " should be " << p->inode.ino;
+	      dout(0) << ss.str() << dendl;
+	      mds->clog.warn(ss);
+	    }
+	    dir->unlink_inode(dn);
+	  }
+	  if (unlinked.count(in))
+	    linked.insert(in);
+	  dir->link_primary_inode(dn, in);
+	  dout(10) << "EMetaBlob.replay linked " << *in << dendl;
+	} else {
+	  dout(10) << "EMetaBlob.replay for [" << p->dnfirst << "," << p->dnlast << "] had " << *in << dendl;
+	}
+	if (p->is_dirty()) in->_mark_dirty(logseg);
+	assert(in->first == p->dnfirst ||
+	       (in->is_multiversion() && in->first > p->dnfirst));
+      }
+      if (p->is_dirty_parent())
+	in->_mark_dirty_parent(logseg, p->is_dirty_pool());
+      assert(g_conf->mds_kill_journal_replay_at != 2);
+    }
+
+    // remote dentries
+    for (list::iterator p = lump.get_dremote().begin();
+	 p != lump.get_dremote().end();
+	 ++p) {
+      CDentry *dn = dir->lookup_exact_snap(p->dn, p->dnlast);
+      if (!dn) {
+	dn = dir->add_remote_dentry(p->dn, p->ino, p->d_type, p->dnfirst, p->dnlast);
+	dn->set_version(p->dnv);
+	if (p->dirty) dn->_mark_dirty(logseg);
+	dout(10) << "EMetaBlob.replay added " << *dn << dendl;
+      } else {
+	if (!dn->get_linkage()->is_null()) {
+	  dout(10) << "EMetaBlob.replay unlinking " << *dn << dendl;
+	  if (dn->get_linkage()->is_primary()) {
+	    unlinked[dn->get_linkage()->get_inode()] = dir;
+	    stringstream ss;
+	    ss << "EMetaBlob.replay FIXME had dentry linked to wrong inode " << *dn
+	       << " " << *dn->get_linkage()->get_inode() << " should be remote " << p->ino;
+	    dout(0) << ss.str() << dendl;
+	  }
+	  dir->unlink_inode(dn);
+	}
+	dir->link_remote_inode(dn, p->ino, p->d_type);
+	dn->set_version(p->dnv);
+	if (p->dirty) dn->_mark_dirty(logseg);
+	dout(10) << "EMetaBlob.replay for [" << p->dnfirst << "," << p->dnlast << "] had " << *dn << dendl;
+	dn->first = p->dnfirst;
+	assert(dn->last == p->dnlast);
+      }
+    }
+
+    // null dentries
+    for (list::iterator p = lump.get_dnull().begin();
+	 p != lump.get_dnull().end();
+	 ++p) {
+      CDentry *dn = dir->lookup_exact_snap(p->dn, p->dnlast);
+      if (!dn) {
+	dn = dir->add_null_dentry(p->dn, p->dnfirst, p->dnlast);
+	dn->set_version(p->dnv);
+	if (p->dirty) dn->_mark_dirty(logseg);
+	dout(10) << "EMetaBlob.replay added " << *dn << dendl;
+      } else {
+	dn->first = p->dnfirst;
+	if (!dn->get_linkage()->is_null()) {
+	  dout(10) << "EMetaBlob.replay unlinking " << *dn << dendl;
+	  CInode *in = dn->get_linkage()->get_inode();
+	  // For renamed inode, We may call CInode::force_dirfrag() later.
+	  // CInode::force_dirfrag() doesn't work well when inode is detached
+	  // from the hierarchy.
+	  if (!renamed_diri || renamed_diri != in) {
+	    if (dn->get_linkage()->is_primary())
+	      unlinked[in] = dir;
+	    dir->unlink_inode(dn);
+	  }
+	}
+	dn->set_version(p->dnv);
+	if (p->dirty) dn->_mark_dirty(logseg);
+	dout(10) << "EMetaBlob.replay had " << *dn << dendl;
+	assert(dn->last == p->dnlast);
+      }
+      olddir = dir;
+    }
+  }
+
+  assert(g_conf->mds_kill_journal_replay_at != 3);
+
+  if (renamed_dirino) {
+    if (renamed_diri) {
+      assert(unlinked.count(renamed_diri));
+      assert(linked.count(renamed_diri));
+      olddir = unlinked[renamed_diri];
+    } else {
+      // we imported a diri we haven't seen before
+      renamed_diri = mds->mdcache->get_inode(renamed_dirino);
+      assert(renamed_diri);  // it was in the metablob
+    }
+
+    if (olddir) {
+      if (olddir->authority() != CDIR_AUTH_UNDEF &&
+	  renamed_diri->authority() == CDIR_AUTH_UNDEF) {
+	assert(slaveup); // auth to non-auth, must be slave prepare
+	list leaves;
+	renamed_diri->dirfragtree.get_leaves(leaves);
+	for (list::iterator p = leaves.begin(); p != leaves.end(); ++p) {
+	  CDir *dir = renamed_diri->get_dirfrag(*p);
+	  assert(dir);
+	  // preserve subtree bound until slave commit
+	  if (dir->get_dir_auth() == CDIR_AUTH_UNDEF)
+	    slaveup->olddirs.insert(dir->inode);
+	}
+      }
+
+      mds->mdcache->adjust_subtree_after_rename(renamed_diri, olddir, false);
+      
+      // see if we can discard the subtree we renamed out of
+      CDir *root = mds->mdcache->get_subtree_root(olddir);
+      if (root->get_dir_auth() == CDIR_AUTH_UNDEF) {
+	if (slaveup) // preserve the old dir until slave commit
+	  slaveup->olddirs.insert(olddir->inode);
+	else
+	  mds->mdcache->try_trim_non_auth_subtree(root);
+      }
+    }
+
+    // if we are the srci importer, we'll also have some dirfrags we have to open up...
+    if (renamed_diri->authority() != CDIR_AUTH_UNDEF) {
+      for (list::iterator p = renamed_dir_frags.begin(); p != renamed_dir_frags.end(); ++p) {
+	CDir *dir = renamed_diri->get_dirfrag(*p);
+	if (dir) {
+	  // we already had the inode before, and we already adjusted this subtree accordingly.
+	  dout(10) << " already had+adjusted rename import bound " << *dir << dendl;
+	  assert(olddir); 
+	  continue;
+	}
+	dir = renamed_diri->get_or_open_dirfrag(mds->mdcache, *p);
+	dout(10) << " creating new rename import bound " << *dir << dendl;
+	mds->mdcache->adjust_subtree_auth(dir, CDIR_AUTH_UNDEF, false);
+      }
+    }
+
+    // rename may overwrite an empty directory and move it into stray dir.
+    unlinked.erase(renamed_diri);
+    for (map::iterator p = unlinked.begin(); p != unlinked.end(); ++p) {
+      if (!linked.count(p->first))
+	continue;
+      assert(p->first->is_dir());
+      mds->mdcache->adjust_subtree_after_rename(p->first, p->second, false);
+    }
+  }
+
+  if (!unlinked.empty()) {
+    for (set::iterator p = linked.begin(); p != linked.end(); ++p)
+      unlinked.erase(*p);
+    dout(10) << " unlinked set contains " << unlinked << dendl;
+    for (map::iterator p = unlinked.begin(); p != unlinked.end(); ++p) {
+      if (slaveup) // preserve unlinked inodes until slave commit
+	slaveup->unlinked.insert(p->first);
+      else
+	mds->mdcache->remove_inode_recursive(p->first);
+    }
+  }
+
+  // table client transactions
+  for (list >::iterator p = table_tids.begin();
+       p != table_tids.end();
+       ++p) {
+    dout(10) << "EMetaBlob.replay noting " << get_mdstable_name(p->first)
+	     << " transaction " << p->second << dendl;
+    MDSTableClient *client = mds->get_table_client(p->first);
+    client->got_journaled_agree(p->second, logseg);
+  }
+
+  // opened ino?
+  if (opened_ino) {
+    CInode *in = mds->mdcache->get_inode(opened_ino);
+    assert(in);
+    dout(10) << "EMetaBlob.replay noting opened inode " << *in << dendl;
+    logseg->open_files.push_back(&in->item_open_file);
+  }
+
+  // allocated_inos
+  if (inotablev) {
+    if (mds->inotable->get_version() >= inotablev) {
+      dout(10) << "EMetaBlob.replay inotable tablev " << inotablev
+	       << " <= table " << mds->inotable->get_version() << dendl;
+    } else {
+      dout(10) << "EMetaBlob.replay inotable v " << inotablev
+	       << " - 1 == table " << mds->inotable->get_version()
+	       << " allocated+used " << allocated_ino
+	       << " prealloc " << preallocated_inos
+	       << dendl;
+      if (allocated_ino)
+	mds->inotable->replay_alloc_id(allocated_ino);
+      if (preallocated_inos.size())
+	mds->inotable->replay_alloc_ids(preallocated_inos);
+
+      // [repair bad inotable updates]
+      if (inotablev > mds->inotable->get_version()) {
+	mds->clog.error() << "journal replay inotablev mismatch "
+	    << mds->inotable->get_version() << " -> " << inotablev << "\n";
+	mds->inotable->force_replay_version(inotablev);
+      }
+
+      assert(inotablev == mds->inotable->get_version());
+    }
+  }
+  if (sessionmapv) {
+    if (mds->sessionmap.version >= sessionmapv) {
+      dout(10) << "EMetaBlob.replay sessionmap v " << sessionmapv
+	       << " <= table " << mds->sessionmap.version << dendl;
+    } else {
+      dout(10) << "EMetaBlob.replay sessionmap v" << sessionmapv
+	       << " -(1|2) == table " << mds->sessionmap.version
+	       << " prealloc " << preallocated_inos
+	       << " used " << used_preallocated_ino
+	       << dendl;
+      Session *session = mds->sessionmap.get_session(client_name);
+      assert(session);
+      dout(20) << " (session prealloc " << session->info.prealloc_inos << ")" << dendl;
+      if (used_preallocated_ino) {
+	if (session->info.prealloc_inos.empty()) {
+	  // HRM: badness in the journal
+	  mds->clog.warn() << " replayed op " << client_reqs << " on session for " << client_name
+			   << " with empty prealloc_inos\n";
+	} else {
+	  inodeno_t next = session->next_ino();
+	  inodeno_t i = session->take_ino(used_preallocated_ino);
+	  if (next != i)
+	    mds->clog.warn() << " replayed op " << client_reqs << " used ino " << i
+			     << " but session next is " << next << "\n";
+	  assert(i == used_preallocated_ino);
+	  session->info.used_inos.clear();
+	}
+	mds->sessionmap.projected = ++mds->sessionmap.version;
+      }
+      if (preallocated_inos.size()) {
+	session->info.prealloc_inos.insert(preallocated_inos);
+	mds->sessionmap.projected = ++mds->sessionmap.version;
+      }
+      assert(sessionmapv == mds->sessionmap.version);
+    }
+  }
+
+  // truncating inodes
+  for (list::iterator p = truncate_start.begin();
+       p != truncate_start.end();
+       ++p) {
+    CInode *in = mds->mdcache->get_inode(*p);
+    assert(in);
+    mds->mdcache->add_recovered_truncate(in, logseg);
+  }
+  for (map::iterator p = truncate_finish.begin();
+       p != truncate_finish.end();
+       ++p) {
+    LogSegment *ls = mds->mdlog->get_segment(p->second);
+    if (ls) {
+      CInode *in = mds->mdcache->get_inode(p->first);
+      assert(in);
+      mds->mdcache->remove_recovered_truncate(in, ls);
+    }
+  }
+
+  // destroyed inodes
+  for (vector::iterator p = destroyed_inodes.begin();
+       p != destroyed_inodes.end();
+       ++p) {
+    CInode *in = mds->mdcache->get_inode(*p);
+    if (in) {
+      dout(10) << "EMetaBlob.replay destroyed " << *p << ", dropping " << *in << dendl;
+      mds->mdcache->remove_inode(in);
+    } else {
+      dout(10) << "EMetaBlob.replay destroyed " << *p << ", not in cache" << dendl;
+    }
+  }
+
+  // client requests
+  for (list >::iterator p = client_reqs.begin();
+       p != client_reqs.end();
+       ++p) {
+    if (p->first.name.is_client()) {
+      dout(10) << "EMetaBlob.replay request " << p->first << " trim_to " << p->second << dendl;
+
+      // if we allocated an inode, there should be exactly one client request id.
+      assert(allocated_ino == inodeno_t() || client_reqs.size() == 1);
+
+      Session *session = mds->sessionmap.get_session(p->first.name);
+      if (session) {
+	session->add_completed_request(p->first.tid, allocated_ino);
+	if (p->second)
+	  session->trim_completed_requests(p->second);
+      }
+    }
+  }
+
+  // update segment
+  update_segment(logseg);
+
+  assert(g_conf->mds_kill_journal_replay_at != 4);
+}
+
+// -----------------------
+// ESession
+
+void ESession::update_segment()
+{
+  _segment->sessionmapv = cmapv;
+  if (inos.size() && inotablev)
+    _segment->inotablev = inotablev;
+}
+
+void ESession::replay(MDS *mds)
+{
+  if (mds->sessionmap.version >= cmapv) {
+    dout(10) << "ESession.replay sessionmap " << mds->sessionmap.version 
+	     << " >= " << cmapv << ", noop" << dendl;
+  } else {
+    dout(10) << "ESession.replay sessionmap " << mds->sessionmap.version
+	     << " < " << cmapv << " " << (open ? "open":"close") << " " << client_inst << dendl;
+    mds->sessionmap.projected = ++mds->sessionmap.version;
+    assert(mds->sessionmap.version == cmapv);
+    Session *session;
+    if (open) {
+      session = mds->sessionmap.get_or_add_session(client_inst);
+      mds->sessionmap.set_state(session, Session::STATE_OPEN);
+      dout(10) << " opened session " << session->info.inst << dendl;
+    } else {
+      session = mds->sessionmap.get_session(client_inst.name);
+      if (session) { // there always should be a session, but there's a bug
+	if (session->connection == NULL) {
+	  dout(10) << " removed session " << session->info.inst << dendl;
+	  mds->sessionmap.remove_session(session);
+	} else {
+	  session->clear();    // the client has reconnected; keep the Session, but reset
+	  dout(10) << " reset session " << session->info.inst << " (they reconnected)" << dendl;
+	}
+      } else {
+	mds->clog.error() << "replayed stray Session close event for " << client_inst
+			  << " from time " << stamp << ", ignoring";
+      }
+    }
+  }
+  
+  if (inos.size() && inotablev) {
+    if (mds->inotable->get_version() >= inotablev) {
+      dout(10) << "ESession.replay inotable " << mds->inotable->get_version()
+	       << " >= " << inotablev << ", noop" << dendl;
+    } else {
+      dout(10) << "ESession.replay inotable " << mds->inotable->get_version()
+	       << " < " << inotablev << " " << (open ? "add":"remove") << dendl;
+      assert(!open);  // for now
+      mds->inotable->replay_release_ids(inos);
+      assert(mds->inotable->get_version() == inotablev);
+    }
+  }
+
+  update_segment();
+}
+
+void ESession::encode(bufferlist &bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(client_inst, bl);
+  ::encode(open, bl);
+  ::encode(cmapv, bl);
+  ::encode(inos, bl);
+  ::encode(inotablev, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ESession::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(client_inst, bl);
+  ::decode(open, bl);
+  ::decode(cmapv, bl);
+  ::decode(inos, bl);
+  ::decode(inotablev, bl);
+  DECODE_FINISH(bl);
+}
+
+void ESession::dump(Formatter *f) const
+{
+  f->dump_stream("client instance") << client_inst;
+  f->dump_string("open", open ? "true" : "false");
+  f->dump_int("client map version", cmapv);
+  f->dump_stream("inos") << inos;
+  f->dump_int("inotable version", inotablev);
+}
+
+void ESession::generate_test_instances(list& ls)
+{
+  ls.push_back(new ESession);
+}
+
+// -----------------------
+// ESessions
+
+void ESessions::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(client_map, bl);
+  ::encode(cmapv, bl);
+  ::encode(stamp, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ESessions::decode_old(bufferlist::iterator &bl)
+{
+  ::decode(client_map, bl);
+  ::decode(cmapv, bl);
+  if (!bl.end())
+    ::decode(stamp, bl);
+}
+
+void ESessions::decode_new(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(client_map, bl);
+  ::decode(cmapv, bl);
+  if (!bl.end())
+    ::decode(stamp, bl);
+  DECODE_FINISH(bl);
+}
+
+void ESessions::dump(Formatter *f) const
+{
+  f->dump_int("client map version", cmapv);
+
+  f->open_array_section("client map");
+  for (map::const_iterator i = client_map.begin();
+       i != client_map.end(); ++i) {
+    f->open_object_section("client");
+    f->dump_int("client id", i->first.v);
+    f->dump_stream("client entity") << i->second;
+    f->close_section(); // client
+  }
+  f->close_section(); // client map
+}
+
+void ESessions::generate_test_instances(list& ls)
+{
+  ls.push_back(new ESessions());
+}
+
+void ESessions::update_segment()
+{
+  _segment->sessionmapv = cmapv;
+}
+
+void ESessions::replay(MDS *mds)
+{
+  if (mds->sessionmap.version >= cmapv) {
+    dout(10) << "ESessions.replay sessionmap " << mds->sessionmap.version
+	     << " >= " << cmapv << ", noop" << dendl;
+  } else {
+    dout(10) << "ESessions.replay sessionmap " << mds->sessionmap.version
+	     << " < " << cmapv << dendl;
+    mds->sessionmap.open_sessions(client_map);
+    assert(mds->sessionmap.version == cmapv);
+    mds->sessionmap.projected = mds->sessionmap.version;
+  }
+  update_segment();
+}
+
+
+// -----------------------
+// ETableServer
+
+void ETableServer::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(table, bl);
+  ::encode(op, bl);
+  ::encode(reqid, bl);
+  ::encode(bymds, bl);
+  ::encode(mutation, bl);
+  ::encode(tid, bl);
+  ::encode(version, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ETableServer::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(table, bl);
+  ::decode(op, bl);
+  ::decode(reqid, bl);
+  ::decode(bymds, bl);
+  ::decode(mutation, bl);
+  ::decode(tid, bl);
+  ::decode(version, bl);
+  DECODE_FINISH(bl);
+}
+
+void ETableServer::dump(Formatter *f) const
+{
+  f->dump_int("table id", table);
+  f->dump_int("op", op);
+  f->dump_int("request id", reqid);
+  f->dump_int("by mds", bymds);
+  f->dump_int("tid", tid);
+  f->dump_int("version", version);
+}
+
+void ETableServer::generate_test_instances(list& ls)
+{
+  ls.push_back(new ETableServer());
+}
+
+
+void ETableServer::update_segment()
+{
+  _segment->tablev[table] = version;
+}
+
+void ETableServer::replay(MDS *mds)
+{
+  MDSTableServer *server = mds->get_table_server(table);
+  if (server->get_version() >= version) {
+    dout(10) << "ETableServer.replay " << get_mdstable_name(table)
+	     << " " << get_mdstableserver_opname(op)
+	     << " event " << version
+	     << " <= table " << server->get_version() << dendl;
+    return;
+  }
+  
+  dout(10) << " ETableServer.replay " << get_mdstable_name(table)
+	   << " " << get_mdstableserver_opname(op)
+	   << " event " << version << " - 1 == table " << server->get_version() << dendl;
+  assert(version-1 == server->get_version());
+
+  switch (op) {
+  case TABLESERVER_OP_PREPARE:
+    server->_prepare(mutation, reqid, bymds);
+    server->_note_prepare(bymds, reqid);
+    break;
+  case TABLESERVER_OP_COMMIT:
+    server->_commit(tid);
+    server->_note_commit(tid);
+    break;
+  case TABLESERVER_OP_ROLLBACK:
+    server->_rollback(tid);
+    server->_note_rollback(tid);
+    break;
+  case TABLESERVER_OP_SERVER_UPDATE:
+    server->_server_update(mutation);
+    break;
+  default:
+    assert(0);
+  }
+  
+  assert(version == server->get_version());
+  update_segment();
+}
+
+
+// ---------------------
+// ETableClient
+
+void ETableClient::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(table, bl);
+  ::encode(op, bl);
+  ::encode(tid, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ETableClient::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(table, bl);
+  ::decode(op, bl);
+  ::decode(tid, bl);
+  DECODE_FINISH(bl);
+}
+
+void ETableClient::dump(Formatter *f) const
+{
+  f->dump_int("table", table);
+  f->dump_int("op", op);
+  f->dump_int("tid", tid);
+}
+
+void ETableClient::generate_test_instances(list& ls)
+{
+  ls.push_back(new ETableClient());
+}
+
+void ETableClient::replay(MDS *mds)
+{
+  dout(10) << " ETableClient.replay " << get_mdstable_name(table)
+	   << " op " << get_mdstableserver_opname(op)
+	   << " tid " << tid << dendl;
+    
+  MDSTableClient *client = mds->get_table_client(table);
+  assert(op == TABLESERVER_OP_ACK);
+  client->got_journaled_ack(tid);
+}
+
+
+// -----------------------
+// ESnap
+/*
+void ESnap::update_segment()
+{
+  _segment->tablev[TABLE_SNAP] = version;
+}
+
+void ESnap::replay(MDS *mds)
+{
+  if (mds->snaptable->get_version() >= version) {
+    dout(10) << "ESnap.replay event " << version
+	     << " <= table " << mds->snaptable->get_version() << dendl;
+    return;
+  } 
+  
+  dout(10) << " ESnap.replay event " << version
+	   << " - 1 == table " << mds->snaptable->get_version() << dendl;
+  assert(version-1 == mds->snaptable->get_version());
+
+  if (create) {
+    version_t v;
+    snapid_t s = mds->snaptable->create(snap.dirino, snap.name, snap.stamp, &v);
+    assert(s == snap.snapid);
+  } else {
+    mds->snaptable->remove(snap.snapid);
+  }
+
+  assert(version == mds->snaptable->get_version());
+}
+*/
+
+
+
+// -----------------------
+// EUpdate
+
+void EUpdate::encode(bufferlist &bl) const
+{
+  ENCODE_START(4, 4, bl);
+  ::encode(stamp, bl);
+  ::encode(type, bl);
+  ::encode(metablob, bl);
+  ::encode(client_map, bl);
+  ::encode(cmapv, bl);
+  ::encode(reqid, bl);
+  ::encode(had_slaves, bl);
+  ENCODE_FINISH(bl);
+}
+ 
+void EUpdate::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(4, 4, 4, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(type, bl);
+  ::decode(metablob, bl);
+  ::decode(client_map, bl);
+  if (struct_v >= 3)
+    ::decode(cmapv, bl);
+  ::decode(reqid, bl);
+  ::decode(had_slaves, bl);
+  DECODE_FINISH(bl);
+}
+
+void EUpdate::dump(Formatter *f) const
+{
+  f->open_object_section("metablob");
+  metablob.dump(f);
+  f->close_section(); // metablob
+
+  f->dump_string("type", type);
+  f->dump_int("client map length", client_map.length());
+  f->dump_int("client map version", cmapv);
+  f->dump_stream("reqid") << reqid;
+  f->dump_string("had slaves", had_slaves ? "true" : "false");
+}
+
+void EUpdate::generate_test_instances(list& ls)
+{
+  ls.push_back(new EUpdate());
+}
+
+
+void EUpdate::update_segment()
+{
+  metablob.update_segment(_segment);
+
+  if (had_slaves)
+    _segment->uncommitted_masters.insert(reqid);
+}
+
+void EUpdate::replay(MDS *mds)
+{
+  metablob.replay(mds, _segment);
+  
+  if (had_slaves) {
+    dout(10) << "EUpdate.replay " << reqid << " had slaves, expecting a matching ECommitted" << dendl;
+    _segment->uncommitted_masters.insert(reqid);
+    set slaves;
+    mds->mdcache->add_uncommitted_master(reqid, _segment, slaves, true);
+  }
+  
+  if (client_map.length()) {
+    if (mds->sessionmap.version >= cmapv) {
+      dout(10) << "EUpdate.replay sessionmap v " << cmapv
+	       << " <= table " << mds->sessionmap.version << dendl;
+    } else {
+      dout(10) << "EUpdate.replay sessionmap " << mds->sessionmap.version
+	       << " < " << cmapv << dendl;
+      // open client sessions?
+      map cm;
+      map seqm;
+      bufferlist::iterator blp = client_map.begin();
+      ::decode(cm, blp);
+      mds->server->prepare_force_open_sessions(cm, seqm);
+      mds->server->finish_force_open_sessions(cm, seqm);
+
+      assert(mds->sessionmap.version == cmapv);
+      mds->sessionmap.projected = mds->sessionmap.version;
+    }
+  }
+}
+
+
+// ------------------------
+// EOpen
+
+void EOpen::encode(bufferlist &bl) const {
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(metablob, bl);
+  ::encode(inos, bl);
+  ENCODE_FINISH(bl);
+} 
+
+void EOpen::decode(bufferlist::iterator &bl) {
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(metablob, bl);
+  ::decode(inos, bl);
+  DECODE_FINISH(bl);
+}
+
+void EOpen::dump(Formatter *f) const
+{
+  f->open_object_section("metablob");
+  metablob.dump(f);
+  f->close_section(); // metablob
+  f->open_array_section("inos involved");
+  for (vector::const_iterator i = inos.begin();
+       i != inos.end(); ++i) {
+    f->dump_int("ino", *i);
+  }
+  f->close_section(); // inos
+}
+
+void EOpen::generate_test_instances(list& ls)
+{
+  ls.push_back(new EOpen());
+  ls.push_back(new EOpen());
+  ls.back()->add_ino(0);
+}
+
+void EOpen::update_segment()
+{
+  // ??
+}
+
+void EOpen::replay(MDS *mds)
+{
+  dout(10) << "EOpen.replay " << dendl;
+  metablob.replay(mds, _segment);
+
+  // note which segments inodes belong to, so we don't have to start rejournaling them
+  for (vector::iterator p = inos.begin();
+       p != inos.end();
+       ++p) {
+    CInode *in = mds->mdcache->get_inode(*p);
+    if (!in) {
+      dout(0) << "EOpen.replay ino " << *p << " not in metablob" << dendl;
+      assert(in);
+    }
+    _segment->open_files.push_back(&in->item_open_file);
+  }
+}
+
+
+// -----------------------
+// ECommitted
+
+void ECommitted::replay(MDS *mds)
+{
+  if (mds->mdcache->uncommitted_masters.count(reqid)) {
+    dout(10) << "ECommitted.replay " << reqid << dendl;
+    mds->mdcache->uncommitted_masters[reqid].ls->uncommitted_masters.erase(reqid);
+    mds->mdcache->uncommitted_masters.erase(reqid);
+  } else {
+    dout(10) << "ECommitted.replay " << reqid << " -- didn't see original op" << dendl;
+  }
+}
+
+void ECommitted::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(reqid, bl);
+  ENCODE_FINISH(bl);
+} 
+
+void ECommitted::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(reqid, bl);
+  DECODE_FINISH(bl);
+}
+
+void ECommitted::dump(Formatter *f) const {
+  f->dump_stream("stamp") << stamp;
+  f->dump_stream("reqid") << reqid;
+}
+
+void ECommitted::generate_test_instances(list& ls)
+{
+  ls.push_back(new ECommitted);
+  ls.push_back(new ECommitted);
+  ls.back()->stamp = utime_t(1, 2);
+  ls.back()->reqid = metareqid_t(entity_name_t::CLIENT(123), 456);
+}
+
+// -----------------------
+// ESlaveUpdate
+
+void link_rollback::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(reqid, bl);
+  ::encode(ino, bl);
+  ::encode(was_inc, bl);
+  ::encode(old_ctime, bl);
+  ::encode(old_dir_mtime, bl);
+  ::encode(old_dir_rctime, bl);
+  ENCODE_FINISH(bl);
+}
+
+void link_rollback::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(reqid, bl);
+  ::decode(ino, bl);
+  ::decode(was_inc, bl);
+  ::decode(old_ctime, bl);
+  ::decode(old_dir_mtime, bl);
+  ::decode(old_dir_rctime, bl);
+  DECODE_FINISH(bl);
+}
+
+void link_rollback::dump(Formatter *f) const
+{
+  f->dump_stream("metareqid") << reqid;
+  f->dump_int("ino", ino);
+  f->dump_string("was incremented", was_inc ? "true" : "false");
+  f->dump_stream("old_ctime") << old_ctime;
+  f->dump_stream("old_dir_mtime") << old_dir_mtime;
+  f->dump_stream("old_dir_rctime") << old_dir_rctime;
+}
+
+void link_rollback::generate_test_instances(list& ls)
+{
+  ls.push_back(new link_rollback());
+}
+
+void rmdir_rollback::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(reqid, bl);
+  ::encode(src_dir, bl);
+  ::encode(src_dname, bl);
+  ::encode(dest_dir, bl);
+  ::encode(dest_dname, bl);
+  ENCODE_FINISH(bl);
+}
+
+void rmdir_rollback::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(reqid, bl);
+  ::decode(src_dir, bl);
+  ::decode(src_dname, bl);
+  ::decode(dest_dir, bl);
+  ::decode(dest_dname, bl);
+  DECODE_FINISH(bl);
+}
+
+void rmdir_rollback::dump(Formatter *f) const
+{
+  f->dump_stream("metareqid") << reqid;
+  f->dump_stream("source directory") << src_dir;
+  f->dump_string("source dname", src_dname);
+  f->dump_stream("destination directory") << dest_dir;
+  f->dump_string("destination dname", dest_dname);
+}
+
+void rmdir_rollback::generate_test_instances(list& ls)
+{
+  ls.push_back(new rmdir_rollback());
+}
+
+void rename_rollback::drec::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(dirfrag, bl);
+  ::encode(dirfrag_old_mtime, bl);
+  ::encode(dirfrag_old_rctime, bl);
+  ::encode(ino, bl);
+  ::encode(remote_ino, bl);
+  ::encode(dname, bl);
+  ::encode(remote_d_type, bl);
+  ::encode(old_ctime, bl);
+  ENCODE_FINISH(bl);
+}
+
+void rename_rollback::drec::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(dirfrag, bl);
+  ::decode(dirfrag_old_mtime, bl);
+  ::decode(dirfrag_old_rctime, bl);
+  ::decode(ino, bl);
+  ::decode(remote_ino, bl);
+  ::decode(dname, bl);
+  ::decode(remote_d_type, bl);
+  ::decode(old_ctime, bl);
+  DECODE_FINISH(bl);
+}
+
+void rename_rollback::drec::dump(Formatter *f) const
+{
+  f->dump_stream("directory fragment") << dirfrag;
+  f->dump_stream("directory old mtime") << dirfrag_old_mtime;
+  f->dump_stream("directory old rctime") << dirfrag_old_rctime;
+  f->dump_int("ino", ino);
+  f->dump_int("remote ino", remote_ino);
+  f->dump_string("dname", dname);
+  uint32_t type = DTTOIF(remote_d_type) & S_IFMT; // convert to type entries
+  string type_string;
+  switch(type) {
+  case S_IFREG:
+    type_string = "file"; break;
+  case S_IFLNK:
+    type_string = "symlink"; break;
+  case S_IFDIR:
+    type_string = "directory"; break;
+  default:
+    type_string = "UNKNOWN-" + stringify((int)type); break;
+  }
+  f->dump_string("remote dtype", type_string);
+  f->dump_stream("old ctime") << old_ctime;
+}
+
+void rename_rollback::drec::generate_test_instances(list& ls)
+{
+  ls.push_back(new drec());
+  ls.back()->remote_d_type = IFTODT(S_IFREG);
+}
+
+void rename_rollback::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(reqid, bl);
+  encode(orig_src, bl);
+  encode(orig_dest, bl);
+  encode(stray, bl);
+  ::encode(ctime, bl);
+  ENCODE_FINISH(bl);
+}
+
+void rename_rollback::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(reqid, bl);
+  decode(orig_src, bl);
+  decode(orig_dest, bl);
+  decode(stray, bl);
+  ::decode(ctime, bl);
+  DECODE_FINISH(bl);
+}
+
+void rename_rollback::dump(Formatter *f) const
+{
+  f->dump_stream("request id") << reqid;
+  f->open_object_section("original src drec");
+  orig_src.dump(f);
+  f->close_section(); // original src drec
+  f->open_object_section("original dest drec");
+  orig_dest.dump(f);
+  f->close_section(); // original dest drec
+  f->open_object_section("stray drec");
+  stray.dump(f);
+  f->close_section(); // stray drec
+  f->dump_stream("ctime") << ctime;
+}
+
+void rename_rollback::generate_test_instances(list& ls)
+{
+  ls.push_back(new rename_rollback());
+  ls.back()->orig_src.remote_d_type = IFTODT(S_IFREG);
+  ls.back()->orig_dest.remote_d_type = IFTODT(S_IFREG);
+  ls.back()->stray.remote_d_type = IFTODT(S_IFREG);
+}
+
+void ESlaveUpdate::encode(bufferlist &bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(type, bl);
+  ::encode(reqid, bl);
+  ::encode(master, bl);
+  ::encode(op, bl);
+  ::encode(origop, bl);
+  ::encode(commit, bl);
+  ::encode(rollback, bl);
+  ENCODE_FINISH(bl);
+} 
+
+void ESlaveUpdate::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(type, bl);
+  ::decode(reqid, bl);
+  ::decode(master, bl);
+  ::decode(op, bl);
+  ::decode(origop, bl);
+  ::decode(commit, bl);
+  ::decode(rollback, bl);
+  DECODE_FINISH(bl);
+}
+
+void ESlaveUpdate::dump(Formatter *f) const
+{
+  f->open_object_section("metablob");
+  commit.dump(f);
+  f->close_section(); // metablob
+
+  f->dump_int("rollback length", rollback.length());
+  f->dump_string("type", type);
+  f->dump_stream("metareqid") << reqid;
+  f->dump_int("master", master);
+  f->dump_int("op", op);
+  f->dump_int("original op", origop);
+}
+
+void ESlaveUpdate::generate_test_instances(list& ls)
+{
+  ls.push_back(new ESlaveUpdate());
+}
+
+
+void ESlaveUpdate::replay(MDS *mds)
+{
+  MDSlaveUpdate *su;
+  switch (op) {
+  case ESlaveUpdate::OP_PREPARE:
+    dout(10) << "ESlaveUpdate.replay prepare " << reqid << " for mds." << master 
+	     << ": applying commit, saving rollback info" << dendl;
+    su = new MDSlaveUpdate(origop, rollback, _segment->slave_updates);
+    commit.replay(mds, _segment, su);
+    mds->mdcache->add_uncommitted_slave_update(reqid, master, su);
+    break;
+
+  case ESlaveUpdate::OP_COMMIT:
+    su = mds->mdcache->get_uncommitted_slave_update(reqid, master);
+    if (su) {
+      dout(10) << "ESlaveUpdate.replay commit " << reqid << " for mds." << master << dendl;
+      mds->mdcache->finish_uncommitted_slave_update(reqid, master);
+    } else {
+      dout(10) << "ESlaveUpdate.replay commit " << reqid << " for mds." << master 
+	       << ": ignoring, no previously saved prepare" << dendl;
+    }
+    break;
+
+  case ESlaveUpdate::OP_ROLLBACK:
+    dout(10) << "ESlaveUpdate.replay abort " << reqid << " for mds." << master
+	     << ": applying rollback commit blob" << dendl;
+    commit.replay(mds, _segment);
+    su = mds->mdcache->get_uncommitted_slave_update(reqid, master);
+    if (su)
+      mds->mdcache->finish_uncommitted_slave_update(reqid, master);
+    break;
+
+  default:
+    assert(0);
+  }
+}
+
+
+// -----------------------
+// ESubtreeMap
+
+void ESubtreeMap::encode(bufferlist& bl) const
+{
+  ENCODE_START(5, 5, bl);
+  ::encode(stamp, bl);
+  ::encode(metablob, bl);
+  ::encode(subtrees, bl);
+  ::encode(ambiguous_subtrees, bl);
+  ::encode(expire_pos, bl);
+  ENCODE_FINISH(bl);
+}
+ 
+void ESubtreeMap::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(5, 5, 5, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(metablob, bl);
+  ::decode(subtrees, bl);
+  if (struct_v >= 4)
+    ::decode(ambiguous_subtrees, bl);
+  if (struct_v >= 3)
+    ::decode(expire_pos, bl);
+  DECODE_FINISH(bl);
+}
+
+void ESubtreeMap::dump(Formatter *f) const
+{
+  f->open_object_section("metablob");
+  metablob.dump(f);
+  f->close_section(); // metablob
+  
+  f->open_array_section("subtrees");
+  for(map >::const_iterator i = subtrees.begin();
+      i != subtrees.end(); ++i) {
+    f->open_object_section("tree");
+    f->dump_stream("root dirfrag") << i->first;
+    for (vector::const_iterator j = i->second.begin();
+	 j != i->second.end(); ++j) {
+      f->dump_stream("bound dirfrag") << *j;
+    }
+    f->close_section(); // tree
+  }
+  f->close_section(); // subtrees
+
+  f->open_array_section("ambiguous subtrees");
+  for(set::const_iterator i = ambiguous_subtrees.begin();
+      i != ambiguous_subtrees.end(); ++i) {
+    f->dump_stream("dirfrag") << *i;
+  }
+  f->close_section(); // ambiguous subtrees
+
+  f->dump_int("expire position", expire_pos);
+}
+
+void ESubtreeMap::generate_test_instances(list& ls)
+{
+  ls.push_back(new ESubtreeMap());
+}
+
+void ESubtreeMap::replay(MDS *mds) 
+{
+  if (expire_pos && expire_pos > mds->mdlog->journaler->get_expire_pos())
+    mds->mdlog->journaler->set_expire_pos(expire_pos);
+
+  // suck up the subtree map?
+  if (mds->mdcache->is_subtrees()) {
+    dout(10) << "ESubtreeMap.replay -- i already have import map; verifying" << dendl;
+    int errors = 0;
+
+    for (map >::iterator p = subtrees.begin();
+	 p != subtrees.end();
+	 ++p) {
+      CDir *dir = mds->mdcache->get_dirfrag(p->first);
+      if (!dir) {
+	mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			  << " subtree root " << p->first << " not in cache";
+	++errors;
+	continue;
+      }
+      
+      if (!mds->mdcache->is_subtree(dir)) {
+	mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			  << " subtree root " << p->first << " not a subtree in cache";
+	++errors;
+	continue;
+      }
+      if (dir->get_dir_auth().first != mds->whoami) {
+	mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			  << " subtree root " << p->first
+			  << " is not mine in cache (it's " << dir->get_dir_auth() << ")";
+	++errors;
+	continue;
+      }
+
+      for (vector::iterator q = p->second.begin(); q != p->second.end(); ++q)
+	mds->mdcache->get_force_dirfrag(*q);
+
+      set bounds;
+      mds->mdcache->get_subtree_bounds(dir, bounds);
+      for (vector::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+	CDir *b = mds->mdcache->get_dirfrag(*q);
+	if (!b) {
+	  mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			    << " subtree " << p->first << " bound " << *q << " not in cache";
+	++errors;
+	  continue;
+	}
+	if (bounds.count(b) == 0) {
+	  mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			    << " subtree " << p->first << " bound " << *q << " not a bound in cache";
+	++errors;
+	  continue;
+	}
+	bounds.erase(b);
+      }
+      for (set::iterator q = bounds.begin(); q != bounds.end(); ++q) {
+	mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			  << " subtree " << p->first << " has extra bound in cache " << (*q)->dirfrag();
+	++errors;
+      }
+      
+      if (ambiguous_subtrees.count(p->first)) {
+	if (!mds->mdcache->have_ambiguous_import(p->first)) {
+	  mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			    << " subtree " << p->first << " is ambiguous but is not in our cache";
+	  ++errors;
+	}
+      } else {
+	if (mds->mdcache->have_ambiguous_import(p->first)) {
+	  mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			    << " subtree " << p->first << " is not ambiguous but is in our cache";
+	  ++errors;
+	}
+      }
+    }
+    
+    list subs;
+    mds->mdcache->list_subtrees(subs);
+    for (list::iterator p = subs.begin(); p != subs.end(); ++p) {
+      CDir *dir = *p;
+      if (dir->get_dir_auth().first != mds->whoami)
+	continue;
+      if (subtrees.count(dir->dirfrag()) == 0) {
+	mds->clog.error() << " replayed ESubtreeMap at " << get_start_off()
+			  << " does not include cache subtree " << dir->dirfrag();
+	++errors;
+      }
+    }
+
+    if (errors) {
+      dout(0) << "journal subtrees: " << subtrees << dendl;
+      dout(0) << "journal ambig_subtrees: " << ambiguous_subtrees << dendl;
+      mds->mdcache->show_subtrees();
+      assert(!g_conf->mds_debug_subtrees || errors == 0);
+    }
+    return;
+  }
+
+  dout(10) << "ESubtreeMap.replay -- reconstructing (auth) subtree spanning tree" << dendl;
+  
+  // first, stick the spanning tree in my cache
+  //metablob.print(*_dout);
+  metablob.replay(mds, _segment);
+  
+  // restore import/export maps
+  for (map >::iterator p = subtrees.begin();
+       p != subtrees.end();
+       ++p) {
+    CDir *dir = mds->mdcache->get_dirfrag(p->first);
+    assert(dir);
+    if (ambiguous_subtrees.count(p->first)) {
+      // ambiguous!
+      mds->mdcache->add_ambiguous_import(p->first, p->second);
+      mds->mdcache->adjust_bounded_subtree_auth(dir, p->second,
+						pair(mds->get_nodeid(), mds->get_nodeid()));
+    } else {
+      // not ambiguous
+      mds->mdcache->adjust_bounded_subtree_auth(dir, p->second, mds->get_nodeid());
+    }
+  }
+  
+  mds->mdcache->show_subtrees();
+}
+
+
+
+// -----------------------
+// EFragment
+
+void EFragment::replay(MDS *mds)
+{
+  dout(10) << "EFragment.replay " << op_name(op) << " " << ino << " " << basefrag << " by " << bits << dendl;
+
+  list resultfrags;
+  list waiters;
+  list old_frags;
+
+  // in may be NULL if it wasn't in our cache yet.  if it's a prepare
+  // it will be once we replay the metablob , but first we need to
+  // refragment anything we already have in the cache.
+  CInode *in = mds->mdcache->get_inode(ino);
+
+  switch (op) {
+  case OP_PREPARE:
+    mds->mdcache->add_uncommitted_fragment(dirfrag_t(ino, basefrag), bits, orig_frags, _segment, &rollback);
+    // fall-thru
+  case OP_ONESHOT:
+    if (in)
+      mds->mdcache->adjust_dir_fragments(in, basefrag, bits, resultfrags, waiters, true);
+    break;
+
+  case OP_ROLLBACK:
+    if (in) {
+      in->dirfragtree.get_leaves_under(basefrag, old_frags);
+      if (orig_frags.empty()) {
+	// old format EFragment
+	mds->mdcache->adjust_dir_fragments(in, basefrag, -bits, resultfrags, waiters, true);
+      } else {
+	for (list::iterator p = orig_frags.begin(); p != orig_frags.end(); ++p)
+	  mds->mdcache->force_dir_fragment(in, *p);
+      }
+    }
+    mds->mdcache->rollback_uncommitted_fragment(dirfrag_t(ino, basefrag), old_frags);
+    break;
+
+  case OP_COMMIT:
+  case OP_FINISH:
+    mds->mdcache->finish_uncommitted_fragment(dirfrag_t(ino, basefrag), op);
+    break;
+
+  default:
+    assert(0);
+  }
+
+  metablob.replay(mds, _segment);
+  if (in && g_conf->mds_debug_frag)
+    in->verify_dirfrags();
+}
+
+void EFragment::encode(bufferlist &bl) const {
+  ENCODE_START(5, 4, bl);
+  ::encode(stamp, bl);
+  ::encode(op, bl);
+  ::encode(ino, bl);
+  ::encode(basefrag, bl);
+  ::encode(bits, bl);
+  ::encode(metablob, bl);
+  ::encode(orig_frags, bl);
+  ::encode(rollback, bl);
+  ENCODE_FINISH(bl);
+}
+
+void EFragment::decode(bufferlist::iterator &bl) {
+  DECODE_START_LEGACY_COMPAT_LEN(5, 4, 4, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  if (struct_v >= 3)
+    ::decode(op, bl);
+  else
+    op = OP_ONESHOT;
+  ::decode(ino, bl);
+  ::decode(basefrag, bl);
+  ::decode(bits, bl);
+  ::decode(metablob, bl);
+  if (struct_v >= 5) {
+    ::decode(orig_frags, bl);
+    ::decode(rollback, bl);
+  }
+  DECODE_FINISH(bl);
+}
+
+void EFragment::dump(Formatter *f) const
+{
+  /*f->open_object_section("Metablob");
+  metablob.dump(f); // sadly we don't have this; dunno if we'll get it
+  f->close_section();*/
+  f->dump_string("op", op_name(op));
+  f->dump_stream("ino") << ino;
+  f->dump_stream("base frag") << basefrag;
+  f->dump_int("bits", bits);
+}
+
+void EFragment::generate_test_instances(list& ls)
+{
+  ls.push_back(new EFragment);
+  ls.push_back(new EFragment);
+  ls.back()->op = OP_PREPARE;
+  ls.back()->ino = 1;
+  ls.back()->bits = 5;
+}
+
+void dirfrag_rollback::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(fnode, bl);
+  ENCODE_FINISH(bl);
+}
+
+void dirfrag_rollback::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(fnode, bl);
+  DECODE_FINISH(bl);
+}
+
+
+
+// =========================================================================
+
+// -----------------------
+// EExport
+
+void EExport::replay(MDS *mds)
+{
+  dout(10) << "EExport.replay " << base << dendl;
+  metablob.replay(mds, _segment);
+  
+  CDir *dir = mds->mdcache->get_dirfrag(base);
+  assert(dir);
+  
+  set realbounds;
+  for (set::iterator p = bounds.begin();
+       p != bounds.end();
+       ++p) {
+    CDir *bd = mds->mdcache->get_dirfrag(*p);
+    assert(bd);
+    realbounds.insert(bd);
+  }
+
+  // adjust auth away
+  mds->mdcache->adjust_bounded_subtree_auth(dir, realbounds, CDIR_AUTH_UNDEF);
+
+  mds->mdcache->try_trim_non_auth_subtree(dir);
+}
+
+void EExport::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(metablob, bl);
+  ::encode(base, bl);
+  ::encode(bounds, bl);
+  ENCODE_FINISH(bl);
+}
+
+void EExport::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(metablob, bl);
+  ::decode(base, bl);
+  ::decode(bounds, bl);
+  DECODE_FINISH(bl);
+}
+
+void EExport::dump(Formatter *f) const
+{
+  f->dump_float("stamp", (double)stamp);
+  /*f->open_object_section("Metablob");
+  metablob.dump(f); // sadly we don't have this; dunno if we'll get it
+  f->close_section();*/
+  f->dump_stream("base dirfrag") << base;
+  f->open_array_section("bounds dirfrags");
+  for (set::const_iterator i = bounds.begin();
+      i != bounds.end(); ++i) {
+    f->dump_stream("dirfrag") << *i;
+  }
+  f->close_section(); // bounds dirfrags
+}
+
+void EExport::generate_test_instances(list& ls)
+{
+  EExport *sample = new EExport();
+  ls.push_back(sample);
+}
+
+
+// -----------------------
+// EImportStart
+
+void EImportStart::update_segment()
+{
+  _segment->sessionmapv = cmapv;
+}
+
+void EImportStart::replay(MDS *mds)
+{
+  dout(10) << "EImportStart.replay " << base << " bounds " << bounds << dendl;
+  //metablob.print(*_dout);
+  metablob.replay(mds, _segment);
+
+  // put in ambiguous import list
+  mds->mdcache->add_ambiguous_import(base, bounds);
+
+  // set auth partially to us so we don't trim it
+  CDir *dir = mds->mdcache->get_dirfrag(base);
+  assert(dir);
+  mds->mdcache->adjust_bounded_subtree_auth(dir, bounds, pair(mds->get_nodeid(), mds->get_nodeid()));
+
+  // open client sessions?
+  if (mds->sessionmap.version >= cmapv) {
+    dout(10) << "EImportStart.replay sessionmap " << mds->sessionmap.version 
+	     << " >= " << cmapv << ", noop" << dendl;
+  } else {
+    dout(10) << "EImportStart.replay sessionmap " << mds->sessionmap.version 
+	     << " < " << cmapv << dendl;
+    map cm;
+    bufferlist::iterator blp = client_map.begin();
+    ::decode(cm, blp);
+    mds->sessionmap.open_sessions(cm);
+    assert(mds->sessionmap.version == cmapv);
+    mds->sessionmap.projected = mds->sessionmap.version;
+  }
+  update_segment();
+}
+
+void EImportStart::encode(bufferlist &bl) const {
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(base, bl);
+  ::encode(metablob, bl);
+  ::encode(bounds, bl);
+  ::encode(cmapv, bl);
+  ::encode(client_map, bl);
+  ENCODE_FINISH(bl);
+}
+
+void EImportStart::decode(bufferlist::iterator &bl) {
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(base, bl);
+  ::decode(metablob, bl);
+  ::decode(bounds, bl);
+  ::decode(cmapv, bl);
+  ::decode(client_map, bl);
+  DECODE_FINISH(bl);
+}
+
+void EImportStart::dump(Formatter *f) const
+{
+  f->dump_stream("base dirfrag") << base;
+  f->open_array_section("boundary dirfrags");
+  for (vector::const_iterator iter = bounds.begin();
+      iter != bounds.end(); ++iter) {
+    f->dump_stream("frag") << *iter;
+  }
+  f->close_section();
+}
+
+void EImportStart::generate_test_instances(list& ls)
+{
+  ls.push_back(new EImportStart);
+}
+
+// -----------------------
+// EImportFinish
+
+void EImportFinish::replay(MDS *mds)
+{
+  if (mds->mdcache->have_ambiguous_import(base)) {
+    dout(10) << "EImportFinish.replay " << base << " success=" << success << dendl;
+    if (success) {
+      mds->mdcache->finish_ambiguous_import(base);
+    } else {
+      CDir *dir = mds->mdcache->get_dirfrag(base);
+      assert(dir);
+      vector bounds;
+      mds->mdcache->get_ambiguous_import_bounds(base, bounds);
+      mds->mdcache->adjust_bounded_subtree_auth(dir, bounds, CDIR_AUTH_UNDEF);
+      mds->mdcache->cancel_ambiguous_import(dir);
+      mds->mdcache->try_trim_non_auth_subtree(dir);
+   }
+  } else {
+    dout(10) << "EImportFinish.replay " << base << " success=" << success
+	     << " on subtree not marked as ambiguous" 
+	     << dendl;
+    assert(0 == "this shouldn't happen unless this is an old journal");
+  }
+}
+
+void EImportFinish::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(stamp, bl);
+  ::encode(base, bl);
+  ::encode(success, bl);
+  ENCODE_FINISH(bl);
+}
+
+void EImportFinish::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+  if (struct_v >= 2)
+    ::decode(stamp, bl);
+  ::decode(base, bl);
+  ::decode(success, bl);
+  DECODE_FINISH(bl);
+}
+
+void EImportFinish::dump(Formatter *f) const
+{
+  f->dump_stream("base dirfrag") << base;
+  f->dump_string("success", success ? "true" : "false");
+}
+void EImportFinish::generate_test_instances(list& ls)
+{
+  ls.push_back(new EImportFinish);
+  ls.push_back(new EImportFinish);
+  ls.back()->success = true;
+}
+
+
+// ------------------------
+// EResetJournal
+
+void EResetJournal::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(stamp, bl);
+  ENCODE_FINISH(bl);
+}
+ 
+void EResetJournal::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(stamp, bl);
+  DECODE_FINISH(bl);
+}
+
+void EResetJournal::dump(Formatter *f) const
+{
+  f->dump_stream("timestamp") << stamp;
+}
+
+void EResetJournal::generate_test_instances(list& ls)
+{
+  ls.push_back(new EResetJournal());
+}
+
+void EResetJournal::replay(MDS *mds)
+{
+  dout(1) << "EResetJournal" << dendl;
+
+  mds->sessionmap.wipe();
+  mds->inotable->replay_reset();
+
+  if (mds->mdsmap->get_root() == mds->whoami) {
+    CDir *rootdir = mds->mdcache->get_root()->get_or_open_dirfrag(mds->mdcache, frag_t());
+    mds->mdcache->adjust_subtree_auth(rootdir, mds->whoami);   
+  }
+
+  CDir *mydir = mds->mdcache->get_myin()->get_or_open_dirfrag(mds->mdcache, frag_t());
+  mds->mdcache->adjust_subtree_auth(mydir, mds->whoami);   
+
+  mds->mdcache->show_subtrees();
+}
+
diff --git a/ceph/src/mds/locks.c b/ceph/src/mds/locks.c
new file mode 100644
index 00000000..f367eda2
--- /dev/null
+++ b/ceph/src/mds/locks.c
@@ -0,0 +1,161 @@
+// there must be a better way?
+typedef char bool;
+#define false 0
+#define true  1
+
+#include "include/int_types.h"
+
+#include 
+#if defined(__linux__)
+#include 
+#elif defined(__FreeBSD__)
+#include 
+#endif
+#include 
+#include 
+
+#include "include/ceph_fs.h"
+#include "locks.h"
+
+static const struct sm_state_t simplelock[LOCK_MAX] = {
+                      // stable     loner  rep state  r     rp   rd   wr   fwr  l    x    caps,other
+    [LOCK_SYNC]      = { 0,         false, LOCK_SYNC, ANY,  0,   ANY, 0,   0,   ANY, 0,   CEPH_CAP_GSHARED,0,0,CEPH_CAP_GSHARED },
+    [LOCK_LOCK_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, ANY,  XCL, XCL, 0,   0,   XCL, 0,   0,0,0,0 },
+    [LOCK_EXCL_SYNC] = { LOCK_SYNC, true,  LOCK_LOCK, 0,    0,   0,   0,   XCL, 0,   0,   0,CEPH_CAP_GSHARED,0,0 },
+    [LOCK_SNAP_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, 0,    0,   0,   0,   AUTH,0,   0,   0,0,0,0 },
+
+    [LOCK_LOCK]      = { 0,         false, LOCK_LOCK, AUTH, 0,   REQ, 0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_SYNC_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, ANY,  0,   0,   0,   0,   0,   0,   0,0,0,0 }, 
+    [LOCK_EXCL_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   XCL, 0,   0,   0,0,0,0 },
+
+    [LOCK_PREXLOCK]  = { LOCK_LOCK, false, LOCK_LOCK, 0,    XCL, 0,   0,   0,   0,   ANY, 0,0,0,0 },
+    [LOCK_XLOCK]     = { LOCK_SYNC, false, LOCK_LOCK, 0,    XCL, 0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_XLOCKDONE] = { LOCK_SYNC, false, LOCK_LOCK, XCL,  XCL, XCL, 0,   0,   XCL, 0,   0,0,CEPH_CAP_GSHARED,0 },
+    [LOCK_LOCK_XLOCK]= { LOCK_PREXLOCK,false,LOCK_LOCK,0,   XCL, 0,   0,   0,   0,   XCL, 0,0,0,0 },
+
+    [LOCK_EXCL]      = { 0,         true,  LOCK_LOCK, 0,    0,   REQ, XCL, 0,   0,   0,   0,CEPH_CAP_GEXCL|CEPH_CAP_GSHARED,0,0 },
+    [LOCK_SYNC_EXCL] = { LOCK_EXCL, true,  LOCK_LOCK, ANY,  0,   0,   0,   0,   0,   0,   0,CEPH_CAP_GSHARED,0,0 },
+    [LOCK_LOCK_EXCL] = { LOCK_EXCL, false, LOCK_LOCK, ANY,  0,   0,   0,   0,   0,   0,   CEPH_CAP_GSHARED,0,0,0 },
+
+    [LOCK_REMOTEXLOCK]={ LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+
+};
+
+const struct sm_t sm_simplelock = {
+	.states = simplelock,
+	.allowed_ever_auth = CEPH_CAP_GSHARED | CEPH_CAP_GEXCL,
+	.allowed_ever_replica = CEPH_CAP_GSHARED,
+	.careful = CEPH_CAP_GSHARED | CEPH_CAP_GEXCL,
+	.can_remote_xlock = 1,
+};
+
+
+// lock state machine states:
+//  Sync  --  Lock  --  sCatter
+//  Tempsync _/
+// (out of date)
+
+static const struct sm_state_t scatterlock[LOCK_MAX] = {
+                      // stable     loner  rep state  r     rp   rd   wr   fwr  l    x    caps,other
+    [LOCK_SYNC]      = { 0,         false, LOCK_SYNC, ANY,  0,   ANY, 0,   0,   ANY, 0,   CEPH_CAP_GSHARED,0,0,CEPH_CAP_GSHARED },
+    [LOCK_LOCK_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, AUTH, 0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_MIX_SYNC]  = { LOCK_SYNC, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_SNAP_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, 0,    0,   0,   0,   AUTH,0,   0,   0,0,0,0 },
+   
+    [LOCK_LOCK]      = { 0,         false, LOCK_LOCK, AUTH, 0,   REQ, AUTH,0,   0,   ANY, 0,0,0,0 },
+    [LOCK_SYNC_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, AUTH, 0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_MIX_LOCK]  = { LOCK_LOCK, false, LOCK_MIX,  0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_MIX_LOCK2] = { LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_TSYN_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    
+    [LOCK_TSYN]      = { 0,         false, LOCK_LOCK, AUTH, 0,   AUTH,0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_LOCK_TSYN] = { LOCK_TSYN, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_MIX_TSYN]  = { LOCK_TSYN, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+
+    [LOCK_MIX]       = { 0,         false, LOCK_MIX,  0,    0,   REQ, ANY, 0,   0,   0,   0,0,0,0 },
+    [LOCK_TSYN_MIX]  = { LOCK_MIX,  false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_SYNC_MIX]  = { LOCK_MIX,  false, LOCK_SYNC_MIX2,0,0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_SYNC_MIX2] = { LOCK_MIX,  false, 0,         0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+};
+
+const struct sm_t sm_scatterlock = {
+	.states = scatterlock,
+	.allowed_ever_auth = CEPH_CAP_GSHARED | CEPH_CAP_GEXCL,
+	.allowed_ever_replica = CEPH_CAP_GSHARED,
+	.careful = CEPH_CAP_GSHARED | CEPH_CAP_GEXCL,
+	.can_remote_xlock = 0,
+};
+
+const struct sm_state_t filelock[LOCK_MAX] = {
+                      // stable     loner  rep state  r     rp   rd   wr   fwr  l    x    caps(any,loner,xlocker,replica)
+    [LOCK_SYNC]      = { 0,         false, LOCK_SYNC, ANY,  0,   ANY, 0,   0,   ANY, 0,   CEPH_CAP_GSHARED|CEPH_CAP_GCACHE|CEPH_CAP_GRD,0,0,CEPH_CAP_GSHARED|CEPH_CAP_GCACHE|CEPH_CAP_GRD },
+    [LOCK_LOCK_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, AUTH, 0,   0,   0,   0,   0,   0,   CEPH_CAP_GCACHE,0,0,0 },
+    [LOCK_EXCL_SYNC] = { LOCK_SYNC, true,  LOCK_LOCK, 0,    0,   0,   0,   XCL, 0,   0,   0,CEPH_CAP_GSHARED|CEPH_CAP_GCACHE|CEPH_CAP_GRD,0,0 },
+    [LOCK_MIX_SYNC]  = { LOCK_SYNC, false, LOCK_MIX_SYNC2,0,0,   0,   0,   0,   0,   0,   CEPH_CAP_GRD|CEPH_CAP_GLAZYIO,0,0,CEPH_CAP_GRD },
+    [LOCK_MIX_SYNC2] = { LOCK_SYNC, false, 0,         0,    0,   0,   0,   0,   0,   0,   CEPH_CAP_GRD|CEPH_CAP_GLAZYIO,0,0,CEPH_CAP_GRD },
+    [LOCK_SNAP_SYNC] = { LOCK_SYNC, false, LOCK_LOCK, 0,    0,   0,   0,   AUTH,0,   0,   0,0,0,0 },
+    [LOCK_XSYN_SYNC] = { LOCK_SYNC, true,  LOCK_LOCK, AUTH, 0,   AUTH,0,   0,   0,   0,   0,CEPH_CAP_GCACHE,0,0 },
+  
+    [LOCK_LOCK]      = { 0,         false, LOCK_LOCK, AUTH, 0,   REQ, AUTH,0,   0,   0,   CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0,0 },
+    [LOCK_SYNC_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, AUTH, 0,   REQ, 0,   0,   0,   0,   CEPH_CAP_GCACHE,0,0,0 },
+    [LOCK_EXCL_LOCK] = { LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   XCL, 0,   0,   CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0,0 },
+    [LOCK_MIX_LOCK]  = { LOCK_LOCK, false, LOCK_MIX,  AUTH, 0,   REQ, 0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_MIX_LOCK2] = { LOCK_LOCK, false, LOCK_LOCK, AUTH, 0,   REQ, 0,   0,   0,   0,   0,0,0,0 },
+
+    [LOCK_PREXLOCK]  = { LOCK_LOCK, false, LOCK_LOCK, 0,    XCL, 0,   0,   0,   0,   ANY, CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0,0 },
+    [LOCK_XLOCK]     = { LOCK_LOCK, false, LOCK_LOCK, 0,    XCL, 0,   0,   0,   0,   0,   CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0,0 },
+    [LOCK_XLOCKDONE] = { LOCK_LOCK, false, LOCK_LOCK, XCL,  XCL, XCL, 0,   0,   XCL, 0,   CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,CEPH_CAP_GSHARED,0 },
+    [LOCK_LOCK_XLOCK]= { LOCK_PREXLOCK,false,LOCK_LOCK,0,   XCL, 0,   0,   0,   0,   XCL, CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0,0 },
+
+    [LOCK_MIX]       = { 0,         false, LOCK_MIX,  0,    0,   REQ, ANY, 0,   0,   0,   CEPH_CAP_GRD|CEPH_CAP_GWR|CEPH_CAP_GLAZYIO,0,0,CEPH_CAP_GRD },
+    [LOCK_SYNC_MIX]  = { LOCK_MIX,  false, LOCK_SYNC_MIX2,ANY,0, 0,   0,   0,   0,   0,   CEPH_CAP_GRD|CEPH_CAP_GLAZYIO,0,0,CEPH_CAP_GRD },
+    [LOCK_SYNC_MIX2] = { LOCK_MIX,  false, 0,         0,    0,   0,   0,   0,   0,   0,   CEPH_CAP_GRD|CEPH_CAP_GLAZYIO,0,0,CEPH_CAP_GRD },
+    [LOCK_EXCL_MIX]  = { LOCK_MIX,  true,  LOCK_LOCK, 0,    0,   0,   XCL, 0,   0,   0,   0,CEPH_CAP_GRD|CEPH_CAP_GWR|CEPH_CAP_GLAZYIO,0,0 },
+    
+    [LOCK_EXCL]      = { 0,         true,  LOCK_LOCK, 0,    0,   XCL, XCL, 0,   0,   0,   0,CEPH_CAP_GSHARED|CEPH_CAP_GEXCL|CEPH_CAP_GCACHE|CEPH_CAP_GRD|CEPH_CAP_GWR|CEPH_CAP_GBUFFER|CEPH_CAP_GLAZYIO,0,0 },
+    [LOCK_SYNC_EXCL] = { LOCK_EXCL, true,  LOCK_LOCK, AUTH, 0,   0,   0,   0,   0,   0,   0,CEPH_CAP_GSHARED|CEPH_CAP_GCACHE|CEPH_CAP_GRD,0,0 },
+    [LOCK_MIX_EXCL]  = { LOCK_EXCL, true,  LOCK_LOCK, 0,    0,   0,   XCL, 0,   0,   0,   0,CEPH_CAP_GRD|CEPH_CAP_GWR|CEPH_CAP_GLAZYIO,0,0 },
+    [LOCK_LOCK_EXCL] = { LOCK_EXCL, true,  LOCK_LOCK, AUTH, 0,   0,   0,   0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
+    [LOCK_XSYN_EXCL] = { LOCK_EXCL, true,  LOCK_LOCK, AUTH, 0,   XCL, 0,   0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
+
+    [LOCK_XSYN]      = { 0,         true,  LOCK_LOCK, AUTH, AUTH,AUTH,XCL, 0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
+    [LOCK_EXCL_XSYN] = { LOCK_XSYN, false, LOCK_LOCK, 0,    0,   XCL, 0,   0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
+
+    [LOCK_PRE_SCAN]  = { LOCK_SCAN, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+    [LOCK_SCAN]      = { LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
+};
+
+const struct sm_t sm_filelock = {
+	.states = filelock,
+	.allowed_ever_auth = (CEPH_CAP_GSHARED |
+			      CEPH_CAP_GEXCL |
+			      CEPH_CAP_GCACHE |
+			      CEPH_CAP_GRD |
+			      CEPH_CAP_GWR |
+			      CEPH_CAP_GWREXTEND |
+			      CEPH_CAP_GBUFFER | 
+			      CEPH_CAP_GLAZYIO),
+	.allowed_ever_replica = (CEPH_CAP_GSHARED |
+				 CEPH_CAP_GCACHE |
+				 CEPH_CAP_GRD | 
+				 CEPH_CAP_GLAZYIO),
+	.careful = (CEPH_CAP_GSHARED | 
+		    CEPH_CAP_GEXCL | 
+		    CEPH_CAP_GCACHE |
+		    CEPH_CAP_GBUFFER),
+	.can_remote_xlock = 0,
+};
+
+
+const struct sm_state_t locallock[LOCK_MAX] = {
+                      // stable     loner  rep state  r     rp   rd   wr   fwr  l    x    caps(any,loner,xlocker,replica)
+    [LOCK_LOCK]      = { 0,         false, LOCK_LOCK, ANY,  0,   ANY, 0,   0,   ANY, AUTH,0,0,0,0 },
+};
+
+const struct sm_t sm_locallock = {
+  .states = locallock,
+  .allowed_ever_auth = 0,
+  .allowed_ever_replica = 0,
+  .careful = 0,
+  .can_remote_xlock = 0,
+};
diff --git a/ceph/src/mds/locks.h b/ceph/src/mds/locks.h
new file mode 100644
index 00000000..d1585cec
--- /dev/null
+++ b/ceph/src/mds/locks.h
@@ -0,0 +1,144 @@
+
+#ifndef CEPH_MDS_LOCKS_H
+#define CEPH_MDS_LOCKS_H
+
+struct sm_state_t {
+  int next;         // 0 if stable
+  char loner;
+  int replica_state;
+  char can_read;
+  char can_read_projected;
+  char can_rdlock;
+  char can_wrlock;
+  char can_force_wrlock;
+  char can_lease;
+  char can_xlock;
+  int caps;
+  int loner_caps;
+  int xlocker_caps;
+  int replica_caps;
+};
+
+struct sm_t {
+  const struct sm_state_t *states;
+  int allowed_ever_auth;
+  int allowed_ever_replica;
+  int careful;
+  int can_remote_xlock;
+};
+
+#define ANY  1 // auth or replica
+#define AUTH 2 // auth only
+#define XCL  3 // auth or exclusive client
+//#define FW   4 // fw to auth, if replica
+#define REQ  5 // req state change from auth, if replica
+
+extern const struct sm_t sm_simplelock;
+extern const struct sm_t sm_filelock;
+extern const struct sm_t sm_scatterlock;
+extern const struct sm_t sm_locallock;
+
+
+
+// -- lock states --
+// sync <-> lock
+enum {
+  LOCK_UNDEF = 0,
+
+  //                                    auth               rep
+  LOCK_SYNC,    // AR   R . RD L . / C .   R RD L . / C . 
+  LOCK_LOCK,    // AR   R . .. . X / . .   . .. . . / . .
+
+  LOCK_PREXLOCK,    // A    . . .. . . / . .   (lock)
+  LOCK_XLOCK,       // A    . . .. . . / . .   (lock)
+  LOCK_XLOCKDONE,   // A    r p rd l x / . .   (lock)  <-- by same client only!!
+  LOCK_LOCK_XLOCK,
+
+  LOCK_SYNC_LOCK,    // AR   R . .. . . / . .   R .. . . / . .
+  LOCK_LOCK_SYNC,    // A    R p rd l . / . .   (lock)  <-- lc by same client only
+
+  LOCK_EXCL,         // A    . . .. . . / c x * (lock)
+  LOCK_EXCL_SYNC,    // A    . . .. . . / c . * (lock)
+  LOCK_EXCL_LOCK,    // A    . . .. . . / . .   (lock)
+  LOCK_SYNC_EXCL,    // Ar   R . .. . . / c . * (sync->lock)
+  LOCK_LOCK_EXCL,    // A    R . .. . . / . .   (lock)
+
+  LOCK_REMOTEXLOCK,  // on NON-auth
+
+  // * = loner mode
+
+  LOCK_MIX,
+  LOCK_SYNC_MIX,
+  LOCK_SYNC_MIX2,
+  LOCK_LOCK_MIX,
+  LOCK_EXCL_MIX,
+  LOCK_MIX_SYNC,
+  LOCK_MIX_SYNC2,
+  LOCK_MIX_LOCK,
+  LOCK_MIX_LOCK2,
+  LOCK_MIX_EXCL,
+
+  LOCK_TSYN,
+  LOCK_TSYN_LOCK,
+  LOCK_TSYN_MIX,
+  LOCK_LOCK_TSYN,
+  LOCK_MIX_TSYN,
+
+  LOCK_PRE_SCAN,
+  LOCK_SCAN,
+
+  LOCK_SNAP_SYNC,
+
+  LOCK_XSYN,
+  LOCK_XSYN_EXCL,
+  LOCK_EXCL_XSYN,
+  LOCK_XSYN_SYNC,
+
+  LOCK_MAX,
+};
+
+// -------------------------
+// lock actions
+
+// for replicas
+#define LOCK_AC_SYNC        -1
+#define LOCK_AC_MIX         -2
+#define LOCK_AC_LOCK        -3
+#define LOCK_AC_LOCKFLUSHED -4
+
+// for auth
+#define LOCK_AC_SYNCACK      1
+#define LOCK_AC_MIXACK     2
+#define LOCK_AC_LOCKACK      3
+
+#define LOCK_AC_REQSCATTER   7
+#define LOCK_AC_REQUNSCATTER 8
+#define LOCK_AC_NUDGE        9
+#define LOCK_AC_REQRDLOCK   10
+
+#define LOCK_AC_FOR_REPLICA(a)  ((a) < 0)
+#define LOCK_AC_FOR_AUTH(a)     ((a) > 0)
+
+
+static inline const char *get_lock_action_name(int a) {
+  switch (a) {
+  case LOCK_AC_SYNC: return "sync";
+  case LOCK_AC_MIX: return "mix";
+  case LOCK_AC_LOCK: return "lock";
+  case LOCK_AC_LOCKFLUSHED: return "lockflushed";
+
+  case LOCK_AC_SYNCACK: return "syncack";
+  case LOCK_AC_MIXACK: return "mixack";
+  case LOCK_AC_LOCKACK: return "lockack";
+
+  case LOCK_AC_REQSCATTER: return "reqscatter";
+  case LOCK_AC_REQUNSCATTER: return "requnscatter";
+  case LOCK_AC_NUDGE: return "nudge";
+  case LOCK_AC_REQRDLOCK: return "reqrdlock";
+  default: return "???";
+  }
+}
+
+
+
+#endif
diff --git a/ceph/src/mds/mds_table_types.h b/ceph/src/mds/mds_table_types.h
new file mode 100644
index 00000000..c08519a8
--- /dev/null
+++ b/ceph/src/mds/mds_table_types.h
@@ -0,0 +1,75 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDSTABLETYPES_H
+#define CEPH_MDSTABLETYPES_H
+
+// MDS TABLES
+
+enum {
+  TABLE_ANCHOR,
+  TABLE_SNAP,
+};
+
+inline const char *get_mdstable_name(int t) {
+  switch (t) {
+  case TABLE_ANCHOR: return "anchortable";
+  case TABLE_SNAP: return "snaptable";
+  default: assert(0);
+  }
+}
+
+enum {
+  TABLESERVER_OP_QUERY        =  1,
+  TABLESERVER_OP_QUERY_REPLY  = -2,
+  TABLESERVER_OP_PREPARE      =  3,
+  TABLESERVER_OP_AGREE        = -4,
+  TABLESERVER_OP_COMMIT       =  5,
+  TABLESERVER_OP_ACK          = -6,
+  TABLESERVER_OP_ROLLBACK     =  7,
+  TABLESERVER_OP_SERVER_UPDATE = 8,
+  TABLESERVER_OP_SERVER_READY = -9,
+};
+
+inline const char *get_mdstableserver_opname(int op) {
+  switch (op) {
+  case TABLESERVER_OP_QUERY: return "query";
+  case TABLESERVER_OP_QUERY_REPLY: return "query_reply";
+  case TABLESERVER_OP_PREPARE: return "prepare";
+  case TABLESERVER_OP_AGREE: return "agree";
+  case TABLESERVER_OP_COMMIT: return "commit";
+  case TABLESERVER_OP_ACK: return "ack";
+  case TABLESERVER_OP_ROLLBACK: return "rollback";
+  case TABLESERVER_OP_SERVER_UPDATE: return "server_update";
+  case TABLESERVER_OP_SERVER_READY: return "server_ready";
+  default: assert(0); return 0;
+  }
+};
+
+enum {
+  TABLE_OP_CREATE,
+  TABLE_OP_UPDATE,
+  TABLE_OP_DESTROY,
+};
+
+inline const char *get_mdstable_opname(int op) {
+  switch (op) {
+  case TABLE_OP_CREATE: return "create";
+  case TABLE_OP_UPDATE: return "update";
+  case TABLE_OP_DESTROY: return "destroy";
+  default: assert(0); return 0;
+  }
+};
+
+#endif
diff --git a/ceph/src/mds/mdstypes.cc b/ceph/src/mds/mdstypes.cc
new file mode 100644
index 00000000..113cce6b
--- /dev/null
+++ b/ceph/src/mds/mdstypes.cc
@@ -0,0 +1,893 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#include "mdstypes.h"
+#include "common/Formatter.h"
+
+void dump(const ceph_file_layout& l, Formatter *f)
+{
+  f->dump_unsigned("stripe_unit", l.fl_stripe_unit);
+  f->dump_unsigned("stripe_count", l.fl_stripe_count);
+  f->dump_unsigned("object_size", l.fl_object_size);
+  if (l.fl_cas_hash)
+    f->dump_unsigned("cas_hash", l.fl_cas_hash);
+  if (l.fl_object_stripe_unit)
+    f->dump_unsigned("object_stripe_unit", l.fl_object_stripe_unit);
+  if (l.fl_pg_pool)
+    f->dump_unsigned("pg_pool", l.fl_pg_pool);
+}
+
+void dump(const ceph_dir_layout& l, Formatter *f)
+{
+  f->dump_unsigned("dir_hash", l.dl_dir_hash);
+}
+
+
+/*
+ * frag_info_t
+ */
+
+void frag_info_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(version, bl);
+  ::encode(mtime, bl);
+  ::encode(nfiles, bl);
+  ::encode(nsubdirs, bl);
+  ENCODE_FINISH(bl);
+}
+
+void frag_info_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(version, bl);
+  ::decode(mtime, bl);
+  ::decode(nfiles, bl);
+  ::decode(nsubdirs, bl);
+  DECODE_FINISH(bl);
+}
+
+void frag_info_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_stream("mtime") << mtime;
+  f->dump_unsigned("num_files", nfiles);
+  f->dump_unsigned("num_subdirs", nsubdirs);
+}
+
+void frag_info_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new frag_info_t);
+  ls.push_back(new frag_info_t);
+  ls.back()->version = 1;
+  ls.back()->mtime = utime_t(2, 3);
+  ls.back()->nfiles = 4;
+  ls.back()->nsubdirs = 5;
+}
+
+ostream& operator<<(ostream &out, const frag_info_t &f)
+{
+  if (f == frag_info_t())
+    return out << "f()";
+  out << "f(v" << f.version;
+  if (f.mtime != utime_t())
+    out << " m" << f.mtime;
+  if (f.nfiles || f.nsubdirs)
+    out << " " << f.size() << "=" << f.nfiles << "+" << f.nsubdirs;
+  out << ")";
+  return out;
+}
+
+
+/*
+ * nest_info_t
+ */
+
+void nest_info_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(version, bl);
+  ::encode(rbytes, bl);
+  ::encode(rfiles, bl);
+  ::encode(rsubdirs, bl);
+  ::encode(ranchors, bl);
+  ::encode(rsnaprealms, bl);
+  ::encode(rctime, bl);
+  ENCODE_FINISH(bl);
+}
+
+void nest_info_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(version, bl);
+  ::decode(rbytes, bl);
+  ::decode(rfiles, bl);
+  ::decode(rsubdirs, bl);
+  ::decode(ranchors, bl);
+  ::decode(rsnaprealms, bl);
+  ::decode(rctime, bl);
+  DECODE_FINISH(bl);
+}
+
+void nest_info_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_unsigned("rbytes", rbytes);
+  f->dump_unsigned("rfiles", rfiles);
+  f->dump_unsigned("rsubdirs", rsubdirs);
+  f->dump_unsigned("ranchors", ranchors);
+  f->dump_unsigned("rsnaprealms", rsnaprealms);
+  f->dump_stream("rctime") << rctime;
+}
+
+void nest_info_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new nest_info_t);
+  ls.push_back(new nest_info_t);
+  ls.back()->version = 1;
+  ls.back()->rbytes = 2;
+  ls.back()->rfiles = 3;
+  ls.back()->rsubdirs = 4;
+  ls.back()->ranchors = 5;
+  ls.back()->rsnaprealms = 6;
+  ls.back()->rctime = utime_t(7, 8);
+}
+
+ostream& operator<<(ostream &out, const nest_info_t &n)
+{
+  if (n == nest_info_t())
+    return out << "n()";
+  out << "n(v" << n.version;
+  if (n.rctime != utime_t())
+    out << " rc" << n.rctime;
+  if (n.rbytes)
+    out << " b" << n.rbytes;
+  if (n.ranchors)
+    out << " a" << n.ranchors;
+  if (n.rsnaprealms)
+    out << " sr" << n.rsnaprealms;
+  if (n.rfiles || n.rsubdirs)
+    out << " " << n.rsize() << "=" << n.rfiles << "+" << n.rsubdirs;
+  out << ")";    
+  return out;
+}
+
+
+/*
+ * client_writeable_range_t
+ */
+
+void client_writeable_range_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(range.first, bl);
+  ::encode(range.last, bl);
+  ::encode(follows, bl);
+  ENCODE_FINISH(bl);
+}
+
+void client_writeable_range_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(range.first, bl);
+  ::decode(range.last, bl);
+  ::decode(follows, bl);
+  DECODE_FINISH(bl);
+}
+
+void client_writeable_range_t::dump(Formatter *f) const
+{
+  f->open_object_section("byte range");
+  f->dump_unsigned("first", range.first);
+  f->dump_unsigned("last", range.last);
+  f->close_section();
+  f->dump_unsigned("follows", follows);
+}
+
+void client_writeable_range_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new client_writeable_range_t);
+  ls.push_back(new client_writeable_range_t);
+  ls.back()->range.first = 123;
+  ls.back()->range.last = 456;
+  ls.back()->follows = 12;
+}
+
+ostream& operator<<(ostream& out, const client_writeable_range_t& r)
+{
+  return out << r.range.first << '-' << r.range.last << "@" << r.follows;
+}
+
+
+/*
+ * inode_t
+ */
+void inode_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(10, 6, bl);
+
+  ::encode(ino, bl);
+  ::encode(rdev, bl);
+  ::encode(ctime, bl);
+
+  ::encode(mode, bl);
+  ::encode(uid, bl);
+  ::encode(gid, bl);
+
+  ::encode(nlink, bl);
+  ::encode(anchored, bl);
+
+  ::encode(dir_layout, bl);
+  ::encode(layout, bl);
+  ::encode(size, bl);
+  ::encode(truncate_seq, bl);
+  ::encode(truncate_size, bl);
+  ::encode(truncate_from, bl);
+  ::encode(truncate_pending, bl);
+  ::encode(mtime, bl);
+  ::encode(atime, bl);
+  ::encode(time_warp_seq, bl);
+  ::encode(client_ranges, bl);
+
+  ::encode(dirstat, bl);
+  ::encode(rstat, bl);
+  ::encode(accounted_rstat, bl);
+
+  ::encode(version, bl);
+  ::encode(file_data_version, bl);
+  ::encode(xattr_version, bl);
+  ::encode(backtrace_version, bl);
+  ::encode(old_pools, bl);
+  ::encode(max_size_ever, bl);
+  ::encode(inline_version, bl);
+  ::encode(inline_data, bl);
+
+  ENCODE_FINISH(bl);
+}
+
+void inode_t::decode(bufferlist::iterator &p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(10, 6, 6, p);
+
+  ::decode(ino, p);
+  ::decode(rdev, p);
+  ::decode(ctime, p);
+
+  ::decode(mode, p);
+  ::decode(uid, p);
+  ::decode(gid, p);
+
+  ::decode(nlink, p);
+  ::decode(anchored, p);
+
+  if (struct_v >= 4)
+    ::decode(dir_layout, p);
+  else
+    memset(&dir_layout, 0, sizeof(dir_layout));
+  ::decode(layout, p);
+  ::decode(size, p);
+  ::decode(truncate_seq, p);
+  ::decode(truncate_size, p);
+  ::decode(truncate_from, p);
+  if (struct_v >= 5)
+    ::decode(truncate_pending, p);
+  else
+    truncate_pending = 0;
+  ::decode(mtime, p);
+  ::decode(atime, p);
+  ::decode(time_warp_seq, p);
+  if (struct_v >= 3) {
+    ::decode(client_ranges, p);
+  } else {
+    map m;
+    ::decode(m, p);
+    for (map::iterator
+	q = m.begin(); q != m.end(); ++q)
+      client_ranges[q->first].range = q->second;
+  }
+    
+  ::decode(dirstat, p);
+  ::decode(rstat, p);
+  ::decode(accounted_rstat, p);
+
+  ::decode(version, p);
+  ::decode(file_data_version, p);
+  ::decode(xattr_version, p);
+  if (struct_v >= 2)
+    ::decode(backtrace_version, p);
+  if (struct_v >= 7)
+    ::decode(old_pools, p);
+  if (struct_v >= 8)
+    ::decode(max_size_ever, p);
+  if (struct_v >= 9) {
+    ::decode(inline_version, p);
+    ::decode(inline_data, p);
+  } else {
+    inline_version = CEPH_INLINE_NONE;
+  }
+  if (struct_v < 10)
+    backtrace_version = 0; // force update backtrace
+
+  DECODE_FINISH(p);
+}
+
+void inode_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("ino", ino);
+  f->dump_unsigned("rdev", rdev);
+  f->dump_stream("ctime") << ctime;
+  f->dump_unsigned("mode", mode);
+  f->dump_unsigned("uid", uid);
+  f->dump_unsigned("gid", gid);
+  f->dump_unsigned("nlink", nlink);
+  f->dump_unsigned("anchored", (int)anchored);
+
+  f->open_object_section("dir_layout");
+  ::dump(dir_layout, f);
+  f->close_section();
+
+  f->open_object_section("layout");
+  ::dump(layout, f);
+  f->close_section();
+
+  f->open_array_section("old_pools");
+  vector::const_iterator i = old_pools.begin();
+  while(i != old_pools.end()) {
+    f->dump_int("pool", *i);
+  }
+  f->close_section();
+
+  f->dump_unsigned("size", size);
+  f->dump_unsigned("truncate_seq", truncate_seq);
+  f->dump_unsigned("truncate_size", truncate_size);
+  f->dump_unsigned("truncate_from", truncate_from);
+  f->dump_unsigned("truncate_pending", truncate_pending);
+  f->dump_stream("mtime") << mtime;
+  f->dump_stream("atime") << atime;
+  f->dump_unsigned("time_warp_seq", time_warp_seq);
+
+  f->open_array_section("client_ranges");
+  for (map::const_iterator p = client_ranges.begin(); p != client_ranges.end(); ++p) {
+    f->open_object_section("client");
+    f->dump_unsigned("client", p->first.v);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_object_section("dirstat");
+  dirstat.dump(f);
+  f->close_section();
+
+  f->open_object_section("rstat");
+  rstat.dump(f);
+  f->close_section();
+
+  f->open_object_section("accounted_rstat");
+  accounted_rstat.dump(f);
+  f->close_section();
+
+  f->dump_unsigned("version", version);
+  f->dump_unsigned("file_data_version", file_data_version);
+  f->dump_unsigned("xattr_version", xattr_version);
+  f->dump_unsigned("backtrace_version", backtrace_version);
+}
+
+void inode_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new inode_t);
+  ls.push_back(new inode_t);
+  ls.back()->ino = 1;
+  // i am lazy.
+}
+
+
+/*
+ * old_inode_t
+ */
+void old_inode_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(first, bl);
+  ::encode(inode, bl);
+  ::encode(xattrs, bl);
+  ENCODE_FINISH(bl);
+}
+
+void old_inode_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(first, bl);
+  ::decode(inode, bl);
+  ::decode(xattrs, bl);
+  DECODE_FINISH(bl);
+}
+
+void old_inode_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("first", first);
+  inode.dump(f);
+  f->open_object_section("xattrs");
+  for (map::const_iterator p = xattrs.begin(); p != xattrs.end(); ++p) {
+    string v(p->second.c_str(), p->second.length());
+    f->dump_string(p->first.c_str(), v);
+  }
+  f->close_section();
+}
+
+void old_inode_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new old_inode_t);
+  ls.push_back(new old_inode_t);
+  ls.back()->first = 2;
+  list ils;
+  inode_t::generate_test_instances(ils);
+  ls.back()->inode = *ils.back();
+  ls.back()->xattrs["user.foo"] = buffer::copy("asdf", 4);
+  ls.back()->xattrs["user.unprintable"] = buffer::copy("\000\001\002", 3);
+}
+
+
+/*
+ * fnode_t
+ */
+void fnode_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(version, bl);
+  ::encode(snap_purged_thru, bl);
+  ::encode(fragstat, bl);
+  ::encode(accounted_fragstat, bl);
+  ::encode(rstat, bl);
+  ::encode(accounted_rstat, bl);
+  ENCODE_FINISH(bl);
+}
+
+void fnode_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(version, bl);
+  ::decode(snap_purged_thru, bl);
+  ::decode(fragstat, bl);
+  ::decode(accounted_fragstat, bl);
+  ::decode(rstat, bl);
+  ::decode(accounted_rstat, bl);
+  DECODE_FINISH(bl);
+}
+
+void fnode_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_unsigned("snap_purged_thru", snap_purged_thru);
+
+  f->open_object_section("fragstat");
+  fragstat.dump(f);
+  f->close_section();
+
+  f->open_object_section("accounted_fragstat");
+  accounted_fragstat.dump(f);
+  f->close_section();
+
+  f->open_object_section("rstat");
+  rstat.dump(f);
+  f->close_section();
+
+  f->open_object_section("accounted_rstat");
+  accounted_rstat.dump(f);
+  f->close_section();
+}
+
+void fnode_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new fnode_t);
+  ls.push_back(new fnode_t);
+  ls.back()->version = 1;
+  ls.back()->snap_purged_thru = 2;
+  list fls;
+  frag_info_t::generate_test_instances(fls);
+  ls.back()->fragstat = *fls.back();
+  ls.back()->accounted_fragstat = *fls.front();
+  list nls;
+  nest_info_t::generate_test_instances(nls);
+  ls.back()->rstat = *nls.front();
+  ls.back()->accounted_rstat = *nls.back();
+}
+
+
+/*
+ * old_rstat_t
+ */
+void old_rstat_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(first, bl);
+  ::encode(rstat, bl);
+  ::encode(accounted_rstat, bl);
+  ENCODE_FINISH(bl);
+}
+
+void old_rstat_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(first, bl);
+  ::decode(rstat, bl);
+  ::decode(accounted_rstat, bl);
+  DECODE_FINISH(bl);
+}
+
+void old_rstat_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("snapid", first);
+  f->open_object_section("rstat");
+  rstat.dump(f);
+  f->close_section();
+  f->open_object_section("accounted_rstat");
+  accounted_rstat.dump(f);
+  f->close_section();
+}
+
+void old_rstat_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new old_rstat_t());
+  ls.push_back(new old_rstat_t());
+  ls.back()->first = 12;
+  list nls;
+  nest_info_t::generate_test_instances(nls);
+  ls.back()->rstat = *nls.back();
+  ls.back()->accounted_rstat = *nls.front();
+}
+
+/*
+ * session_info_t
+ */
+void session_info_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 3, bl);
+  ::encode(inst, bl);
+  ::encode(completed_requests, bl);
+  ::encode(prealloc_inos, bl);   // hacky, see below.
+  ::encode(used_inos, bl);
+  ENCODE_FINISH(bl);
+}
+
+void session_info_t::decode(bufferlist::iterator& p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, p);
+  ::decode(inst, p);
+  if (struct_v <= 2) {
+    set s;
+    ::decode(s, p);
+    while (!s.empty()) {
+      completed_requests[*s.begin()] = inodeno_t();
+      s.erase(s.begin());
+    }
+  } else {
+    ::decode(completed_requests, p);
+  }
+  ::decode(prealloc_inos, p);
+  ::decode(used_inos, p);
+  prealloc_inos.insert(used_inos);
+  used_inos.clear();
+  DECODE_FINISH(p);
+}
+
+void session_info_t::dump(Formatter *f) const
+{
+  f->dump_stream("inst") << inst;
+
+  f->open_array_section("completed_requests");
+  for (map::const_iterator p = completed_requests.begin();
+       p != completed_requests.end();
+       ++p) {
+    f->open_object_section("request");
+    f->dump_unsigned("tid", p->first);
+    f->dump_stream("created_ino") << p->second;
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("prealloc_inos");
+  for (interval_set::const_iterator p = prealloc_inos.begin();
+       p != prealloc_inos.end();
+       ++p) {
+    f->open_object_section("ino_range");
+    f->dump_unsigned("start", p.get_start());
+    f->dump_unsigned("length", p.get_len());
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("used_inos");
+  for (interval_set::const_iterator p = prealloc_inos.begin();
+       p != prealloc_inos.end();
+       ++p) {
+    f->open_object_section("ino_range");
+    f->dump_unsigned("start", p.get_start());
+    f->dump_unsigned("length", p.get_len());
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void session_info_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new session_info_t);
+  ls.push_back(new session_info_t);
+  ls.back()->inst = entity_inst_t(entity_name_t::MDS(12), entity_addr_t());
+  ls.back()->completed_requests.insert(make_pair(234, inodeno_t(111222)));
+  ls.back()->completed_requests.insert(make_pair(237, inodeno_t(222333)));
+  ls.back()->prealloc_inos.insert(333, 12);
+  ls.back()->prealloc_inos.insert(377, 112);
+  // we can't add used inos; they're cleared on decode
+}
+
+
+/*
+ * string_snap_t
+ */
+void string_snap_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(name, bl);
+  ::encode(snapid, bl);
+  ENCODE_FINISH(bl);
+}
+
+void string_snap_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(name, bl);
+  ::decode(snapid, bl);
+  DECODE_FINISH(bl);
+}
+
+void string_snap_t::dump(Formatter *f) const
+{
+  f->dump_string("name", name);
+  f->dump_unsigned("snapid", snapid);
+}
+
+void string_snap_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new string_snap_t);
+  ls.push_back(new string_snap_t);
+  ls.back()->name = "foo";
+  ls.back()->snapid = 123;
+  ls.push_back(new string_snap_t);
+  ls.back()->name = "bar";
+  ls.back()->snapid = 456;
+}
+
+
+/*
+ * MDSCacheObjectInfo
+ */
+void MDSCacheObjectInfo::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(ino, bl);
+  ::encode(dirfrag, bl);
+  ::encode(dname, bl);
+  ::encode(snapid, bl);
+  ENCODE_FINISH(bl);
+}
+
+void MDSCacheObjectInfo::decode(bufferlist::iterator& p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, p);
+  ::decode(ino, p);
+  ::decode(dirfrag, p);
+  ::decode(dname, p);
+  ::decode(snapid, p);
+  DECODE_FINISH(p);
+}
+
+void MDSCacheObjectInfo::dump(Formatter *f) const
+{
+  f->dump_unsigned("ino", ino);
+  f->dump_stream("dirfrag") << dirfrag;
+  f->dump_string("name", dname);
+  f->dump_unsigned("snapid", snapid);
+}
+
+void MDSCacheObjectInfo::generate_test_instances(list& ls)
+{
+  ls.push_back(new MDSCacheObjectInfo);
+  ls.push_back(new MDSCacheObjectInfo);
+  ls.back()->ino = 1;
+  ls.back()->dirfrag = dirfrag_t(2, 3);
+  ls.back()->dname = "fooname";
+  ls.back()->snapid = CEPH_NOSNAP;
+  ls.push_back(new MDSCacheObjectInfo);
+  ls.back()->ino = 121;
+  ls.back()->dirfrag = dirfrag_t(222, 0);
+  ls.back()->dname = "bar foo";
+  ls.back()->snapid = 21322;
+}
+
+
+/*
+ * mds_table_pending_t
+ */
+void mds_table_pending_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(reqid, bl);
+  ::encode(mds, bl);
+  ::encode(tid, bl);
+  ENCODE_FINISH(bl);
+}
+
+void mds_table_pending_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(reqid, bl);
+  ::decode(mds, bl);
+  ::decode(tid, bl);
+  DECODE_FINISH(bl);
+}
+
+void mds_table_pending_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("reqid", reqid);
+  f->dump_unsigned("mds", mds);
+  f->dump_unsigned("tid", tid);
+}
+
+void mds_table_pending_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new mds_table_pending_t);
+  ls.push_back(new mds_table_pending_t);
+  ls.back()->reqid = 234;
+  ls.back()->mds = 2;
+  ls.back()->tid = 35434;
+}
+
+
+/*
+ * inode_load_vec_t
+ */
+void inode_load_vec_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 2, bl);
+  for (int i=0; iopen_array_section("Decay Counters");
+  for (vector::const_iterator i = vec.begin(); i != vec.end(); ++i) {
+    f->open_object_section("Decay Counter");
+    i->dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void inode_load_vec_t::generate_test_instances(list& ls)
+{
+  utime_t sample;
+  ls.push_back(new inode_load_vec_t(sample));
+}
+
+
+/*
+ * dirfrag_load_vec_t
+ */
+void dirfrag_load_vec_t::dump(Formatter *f) const
+{
+  f->open_array_section("Decay Counters");
+  for (vector::const_iterator i = vec.begin(); i != vec.end(); ++i) {
+    f->open_object_section("Decay Counter");
+    i->dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void dirfrag_load_vec_t::generate_test_instances(list& ls)
+{
+  utime_t sample;
+  ls.push_back(new dirfrag_load_vec_t(sample));
+}
+
+/*
+ * mds_load_t
+ */
+void mds_load_t::encode(bufferlist &bl) const {
+  ENCODE_START(2, 2, bl);
+  ::encode(auth, bl);
+  ::encode(all, bl);
+  ::encode(req_rate, bl);
+  ::encode(cache_hit_rate, bl);
+  ::encode(queue_len, bl);
+  ::encode(cpu_load_avg, bl);
+  ENCODE_FINISH(bl);
+}
+
+void mds_load_t::decode(const utime_t &t, bufferlist::iterator &bl) {
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(auth, t, bl);
+  ::decode(all, t, bl);
+  ::decode(req_rate, bl);
+  ::decode(cache_hit_rate, bl);
+  ::decode(queue_len, bl);
+  ::decode(cpu_load_avg, bl);
+  DECODE_FINISH(bl);
+}
+
+void mds_load_t::dump(Formatter *f) const
+{
+  f->dump_float("request rate", req_rate);
+  f->dump_float("cache hit rate", cache_hit_rate);
+  f->dump_float("queue length", queue_len);
+  f->dump_float("cpu load", cpu_load_avg);
+  f->open_object_section("auth dirfrag");
+  auth.dump(f);
+  f->close_section();
+  f->open_object_section("all dirfrags");
+  all.dump(f);
+  f->close_section();
+}
+
+void mds_load_t::generate_test_instances(list& ls)
+{
+  utime_t sample;
+  ls.push_back(new mds_load_t(sample));
+}
+
+/*
+ * cap_reconnect_t
+ */
+void cap_reconnect_t::encode(bufferlist& bl) const {
+  ENCODE_START(1, 1, bl);
+  encode_old(bl); // extract out when something changes
+  ENCODE_FINISH(bl);
+}
+
+void cap_reconnect_t::encode_old(bufferlist& bl) const {
+  ::encode(path, bl);
+  capinfo.flock_len = flockbl.length();
+  ::encode(capinfo, bl);
+  ::encode_nohead(flockbl, bl);
+}
+
+void cap_reconnect_t::decode(bufferlist::iterator& bl) {
+  DECODE_START(1, bl);
+  decode_old(bl); // extract out when something changes
+  DECODE_FINISH(bl);
+}
+
+void cap_reconnect_t::decode_old(bufferlist::iterator& bl) {
+  ::decode(path, bl);
+  ::decode(capinfo, bl);
+  ::decode_nohead(capinfo.flock_len, flockbl, bl);
+}
+
+void cap_reconnect_t::dump(Formatter *f) const
+{
+  f->dump_string("path", path);
+  f->dump_int("cap_id", capinfo.cap_id);
+  f->dump_string("cap wanted", ccap_string(capinfo.wanted));
+  f->dump_string("cap issued", ccap_string(capinfo.issued));
+  f->dump_int("snaprealm", capinfo.snaprealm);
+  f->dump_int("path base ino", capinfo.pathbase);
+  f->dump_string("has file locks", capinfo.flock_len ? "true" : "false");
+}
+
+void cap_reconnect_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new cap_reconnect_t);
+  ls.back()->path = "/test/path";
+  ls.back()->capinfo.cap_id = 1;
+}
diff --git a/ceph/src/mds/mdstypes.h b/ceph/src/mds/mdstypes.h
new file mode 100644
index 00000000..184cf704
--- /dev/null
+++ b/ceph/src/mds/mdstypes.h
@@ -0,0 +1,1454 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_MDSTYPES_H
+#define CEPH_MDSTYPES_H
+
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+using namespace std;
+
+#include "common/config.h"
+#include "common/Clock.h"
+#include "common/DecayCounter.h"
+#include "include/Context.h"
+
+#include "include/frag.h"
+#include "include/xlist.h"
+#include "include/interval_set.h"
+
+#include "inode_backtrace.h"
+
+#include 
+#include "include/assert.h"
+#include "include/hash_namespace.h"
+
+#define CEPH_FS_ONDISK_MAGIC "ceph fs volume v011"
+
+
+#define MDS_REF_SET      // define me for improved debug output, sanity checking
+//#define MDS_AUTHPIN_SET  // define me for debugging auth pin leaks
+//#define MDS_VERIFY_FRAGSTAT    // do do (slow) sanity checking on frags
+
+#define MDS_PORT_CACHE   0x200
+#define MDS_PORT_LOCKER  0x300
+#define MDS_PORT_MIGRATOR 0x400
+
+// FIXME: this should not be hardcoded
+#define MDS_DATA_POOL		0
+#define MDS_METADATA_POOL	1
+
+#define MAX_MDS                   0x100
+#define NUM_STRAY                 10
+
+#define MDS_INO_ROOT              1
+
+// No longer created but recognised in existing filesystems
+// so that we don't try to fragment it.
+#define MDS_INO_CEPH              2
+
+#define MDS_INO_MDSDIR_OFFSET     (1*MAX_MDS)
+#define MDS_INO_LOG_OFFSET        (2*MAX_MDS)
+#define MDS_INO_STRAY_OFFSET      (6*MAX_MDS)
+
+#define MDS_INO_SYSTEM_BASE       ((6*MAX_MDS) + (MAX_MDS * NUM_STRAY))
+
+#define MDS_INO_STRAY(x,i)  (MDS_INO_STRAY_OFFSET+((((unsigned)(x))*NUM_STRAY)+((unsigned)(i))))
+#define MDS_INO_MDSDIR(x) (MDS_INO_MDSDIR_OFFSET+((unsigned)x))
+
+#define MDS_INO_IS_STRAY(i)  ((i) >= MDS_INO_STRAY_OFFSET  && (i) < (MDS_INO_STRAY_OFFSET+(MAX_MDS*NUM_STRAY)))
+#define MDS_INO_IS_MDSDIR(i) ((i) >= MDS_INO_MDSDIR_OFFSET && (i) < (MDS_INO_MDSDIR_OFFSET+MAX_MDS))
+#define MDS_INO_IS_BASE(i)   (MDS_INO_ROOT == (i) || MDS_INO_IS_MDSDIR(i))
+#define MDS_INO_STRAY_OWNER(i) (signed (((unsigned (i)) - MDS_INO_STRAY_OFFSET) / NUM_STRAY))
+#define MDS_INO_STRAY_INDEX(i) (((unsigned (i)) - MDS_INO_STRAY_OFFSET) % NUM_STRAY)
+
+#define MDS_TRAVERSE_FORWARD       1
+#define MDS_TRAVERSE_DISCOVER      2    // skips permissions checks etc.
+#define MDS_TRAVERSE_DISCOVERXLOCK 3    // succeeds on (foreign?) null, xlocked dentries.
+
+
+extern long g_num_ino, g_num_dir, g_num_dn, g_num_cap;
+extern long g_num_inoa, g_num_dira, g_num_dna, g_num_capa;
+extern long g_num_inos, g_num_dirs, g_num_dns, g_num_caps;
+
+
+// CAPS
+
+inline string gcap_string(int cap)
+{
+  string s;
+  if (cap & CEPH_CAP_GSHARED) s += "s";  
+  if (cap & CEPH_CAP_GEXCL) s += "x";
+  if (cap & CEPH_CAP_GCACHE) s += "c";
+  if (cap & CEPH_CAP_GRD) s += "r";
+  if (cap & CEPH_CAP_GWR) s += "w";
+  if (cap & CEPH_CAP_GBUFFER) s += "b";
+  if (cap & CEPH_CAP_GWREXTEND) s += "a";
+  if (cap & CEPH_CAP_GLAZYIO) s += "l";
+  return s;
+}
+inline string ccap_string(int cap)
+{
+  string s;
+  if (cap & CEPH_CAP_PIN) s += "p";
+
+  int a = (cap >> CEPH_CAP_SAUTH) & 3;
+  if (a) s += 'A' + gcap_string(a);
+
+  a = (cap >> CEPH_CAP_SLINK) & 3;
+  if (a) s += 'L' + gcap_string(a);
+
+  a = (cap >> CEPH_CAP_SXATTR) & 3;
+  if (a) s += 'X' + gcap_string(a);
+
+  a = cap >> CEPH_CAP_SFILE;
+  if (a) s += 'F' + gcap_string(a);
+
+  if (s.length() == 0)
+    s = "-";
+  return s;
+}
+
+
+struct scatter_info_t {
+  version_t version;
+
+  scatter_info_t() : version(0) {}
+};
+
+struct frag_info_t : public scatter_info_t {
+  // this frag
+  utime_t mtime;
+  int64_t nfiles;        // files
+  int64_t nsubdirs;      // subdirs
+
+  frag_info_t() : nfiles(0), nsubdirs(0) {}
+
+  int64_t size() const { return nfiles + nsubdirs; }
+
+  void zero() {
+    *this = frag_info_t();
+  }
+
+  // *this += cur - acc;
+  void add_delta(const frag_info_t &cur, frag_info_t &acc, bool& touched_mtime) {
+    if (!(cur.mtime == acc.mtime)) {
+      mtime = cur.mtime;
+      touched_mtime = true;
+    }
+    nfiles += cur.nfiles - acc.nfiles;
+    nsubdirs += cur.nsubdirs - acc.nsubdirs;
+  }
+
+  void add(const frag_info_t& other) {
+    if (other.mtime > mtime)
+      mtime = other.mtime;
+    nfiles += other.nfiles;
+    nsubdirs += other.nsubdirs;
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(frag_info_t)
+
+inline bool operator==(const frag_info_t &l, const frag_info_t &r) {
+  return memcmp(&l, &r, sizeof(l)) == 0;
+}
+
+ostream& operator<<(ostream &out, const frag_info_t &f);
+
+
+struct nest_info_t : public scatter_info_t {
+  // this frag + children
+  utime_t rctime;
+  int64_t rbytes;
+  int64_t rfiles;
+  int64_t rsubdirs;
+  int64_t rsize() const { return rfiles + rsubdirs; }
+
+  int64_t ranchors;  // for dirstat, includes inode's anchored flag.
+  int64_t rsnaprealms;
+
+  nest_info_t() : rbytes(0), rfiles(0), rsubdirs(0),
+		  ranchors(0), rsnaprealms(0) {}
+
+  void zero() {
+    *this = nest_info_t();
+  }
+
+  void sub(const nest_info_t &other) {
+    add(other, -1);
+  }
+  void add(const nest_info_t &other, int fac=1) {
+    if (other.rctime > rctime)
+      rctime = other.rctime;
+    rbytes += fac*other.rbytes;
+    rfiles += fac*other.rfiles;
+    rsubdirs += fac*other.rsubdirs;
+    ranchors += fac*other.ranchors;
+    rsnaprealms += fac*other.rsnaprealms;
+  }
+
+  // *this += cur - acc;
+  void add_delta(const nest_info_t &cur, nest_info_t &acc) {
+    if (cur.rctime > rctime)
+      rctime = cur.rctime;
+    rbytes += cur.rbytes - acc.rbytes;
+    rfiles += cur.rfiles - acc.rfiles;
+    rsubdirs += cur.rsubdirs - acc.rsubdirs;
+    ranchors += cur.ranchors - acc.ranchors;
+    rsnaprealms += cur.rsnaprealms - acc.rsnaprealms;
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(nest_info_t)
+
+inline bool operator==(const nest_info_t &l, const nest_info_t &r) {
+  return memcmp(&l, &r, sizeof(l)) == 0;
+}
+
+ostream& operator<<(ostream &out, const nest_info_t &n);
+
+
+struct vinodeno_t {
+  inodeno_t ino;
+  snapid_t snapid;
+  vinodeno_t() {}
+  vinodeno_t(inodeno_t i, snapid_t s) : ino(i), snapid(s) {}
+
+  void encode(bufferlist& bl) const {
+    ::encode(ino, bl);
+    ::encode(snapid, bl);
+  }
+  void decode(bufferlist::iterator& p) {
+    ::decode(ino, p);
+    ::decode(snapid, p);
+  }
+};
+WRITE_CLASS_ENCODER(vinodeno_t)
+
+inline bool operator==(const vinodeno_t &l, const vinodeno_t &r) {
+  return l.ino == r.ino && l.snapid == r.snapid;
+}
+inline bool operator!=(const vinodeno_t &l, const vinodeno_t &r) {
+  return !(l == r);
+}
+inline bool operator<(const vinodeno_t &l, const vinodeno_t &r) {
+  return 
+    l.ino < r.ino ||
+    (l.ino == r.ino && l.snapid < r.snapid);
+}
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash {
+    size_t operator()(const vinodeno_t &vino) const { 
+      hash H;
+      hash I;
+      return H(vino.ino) ^ I(vino.snapid);
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+
+
+
+inline ostream& operator<<(ostream &out, const vinodeno_t &vino) {
+  out << vino.ino;
+  if (vino.snapid == CEPH_NOSNAP)
+    out << ".head";
+  else if (vino.snapid)
+    out << '.' << vino.snapid;
+  return out;
+}
+
+
+/*
+ * client_writeable_range_t
+ */
+struct client_writeable_range_t {
+  struct byte_range_t {
+    uint64_t first, last;    // interval client can write to
+    byte_range_t() : first(0), last(0) {}
+  };
+
+  byte_range_t range;
+  snapid_t follows;     // aka "data+metadata flushed thru"
+
+  client_writeable_range_t() : follows(0) {}
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+
+inline void decode(client_writeable_range_t::byte_range_t& range, bufferlist::iterator& bl) {
+  ::decode(range.first, bl);
+  ::decode(range.last, bl);
+}
+
+WRITE_CLASS_ENCODER(client_writeable_range_t)
+
+ostream& operator<<(ostream& out, const client_writeable_range_t& r);
+
+inline bool operator==(const client_writeable_range_t& l,
+		       const client_writeable_range_t& r) {
+  return l.range.first == r.range.first && l.range.last == r.range.last &&
+    l.follows == r.follows;
+}
+
+
+/*
+ * inode_t
+ */
+struct inode_t {
+  // base (immutable)
+  inodeno_t ino;
+  uint32_t   rdev;    // if special file
+
+  // affected by any inode change...
+  utime_t    ctime;   // inode change time
+
+  // perm (namespace permissions)
+  uint32_t   mode;
+  uid_t      uid;
+  gid_t      gid;
+
+  // nlink
+  int32_t    nlink;  
+  bool       anchored;          // auth only?
+
+  // file (data access)
+  ceph_dir_layout  dir_layout;    // [dir only]
+  ceph_file_layout layout;
+  vector  old_pools;
+  uint64_t   size;        // on directory, # dentries
+  uint64_t   max_size_ever; // max size the file has ever been
+  uint32_t   truncate_seq;
+  uint64_t   truncate_size, truncate_from;
+  uint32_t   truncate_pending;
+  utime_t    mtime;   // file data modify time.
+  utime_t    atime;   // file data access time.
+  uint32_t   time_warp_seq;  // count of (potential) mtime/atime timewarps (i.e., utimes())
+  bufferlist inline_data;
+  version_t  inline_version;
+
+  map client_ranges;  // client(s) can write to these ranges
+
+  // dirfrag, recursive accountin
+  frag_info_t dirstat;         // protected by my filelock
+  nest_info_t rstat;           // protected by my nestlock
+  nest_info_t accounted_rstat; // protected by parent's nestlock
+ 
+  // special stuff
+  version_t version;           // auth only
+  version_t file_data_version; // auth only
+  version_t xattr_version;
+
+  version_t backtrace_version;
+
+  inode_t() : ino(0), rdev(0),
+	      mode(0), uid(0), gid(0),
+	      nlink(0), anchored(false),
+	      size(0), max_size_ever(0),
+	      truncate_seq(0), truncate_size(0), truncate_from(0),
+	      truncate_pending(0),
+	      time_warp_seq(0),
+	      inline_version(1),
+	      version(0), file_data_version(0), xattr_version(0), backtrace_version(0) {
+    clear_layout();
+    memset(&dir_layout, 0, sizeof(dir_layout));
+  }
+
+  // file type
+  bool is_symlink() const { return (mode & S_IFMT) == S_IFLNK; }
+  bool is_dir()     const { return (mode & S_IFMT) == S_IFDIR; }
+  bool is_file()    const { return (mode & S_IFMT) == S_IFREG; }
+
+  bool is_truncating() const { return (truncate_pending > 0); }
+  void truncate(uint64_t old_size, uint64_t new_size) {
+    assert(new_size < old_size);
+    if (old_size > max_size_ever)
+      max_size_ever = old_size;
+    truncate_from = old_size;
+    size = new_size;
+    rstat.rbytes = new_size;
+    truncate_size = size;
+    truncate_seq++;
+    truncate_pending++;
+  }
+
+  bool has_layout() const {
+    // why on earth is there no converse of memchr() in string.h?
+    const char *p = (const char *)&layout;
+    for (size_t i = 0; i < sizeof(layout); i++)
+      if (p[i] != '\0')
+	return true;
+    return false;
+  }
+
+  void clear_layout() {
+    memset(&layout, 0, sizeof(layout));
+  }
+
+  uint64_t get_layout_size_increment() {
+    return (uint64_t)layout.fl_object_size * (uint64_t)layout.fl_stripe_count;
+  }
+
+  bool is_dirty_rstat() const { return !(rstat == accounted_rstat); }
+
+  uint64_t get_max_size() const {
+    uint64_t max = 0;
+      for (map::const_iterator p = client_ranges.begin();
+	   p != client_ranges.end();
+	   ++p)
+	if (p->second.range.last > max)
+	  max = p->second.range.last;
+      return max;
+  }
+  void set_max_size(uint64_t new_max) {
+    if (new_max == 0) {
+      client_ranges.clear();
+    } else {
+      for (map::iterator p = client_ranges.begin();
+	   p != client_ranges.end();
+	   ++p)
+	p->second.range.last = new_max;
+    }
+  }
+
+  void trim_client_ranges(snapid_t last) {
+    map::iterator p = client_ranges.begin();
+    while (p != client_ranges.end()) {
+      if (p->second.follows >= last)
+	client_ranges.erase(p++);
+      else
+	++p;
+    }
+  }
+
+  bool is_backtrace_updated() {
+    return backtrace_version == version;
+  }
+  void update_backtrace(version_t pv=0) {
+    backtrace_version = pv ? pv : version;
+  }
+
+  void add_old_pool(int64_t l) {
+    backtrace_version = version;
+    old_pools.push_back(l);
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(inode_t)
+
+
+/*
+ * old_inode_t
+ */
+struct old_inode_t {
+  snapid_t first;
+  inode_t inode;
+  map xattrs;
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(old_inode_t)
+
+
+/*
+ * like an inode, but for a dir frag 
+ */
+struct fnode_t {
+  version_t version;
+  snapid_t snap_purged_thru;   // the max_last_destroy snapid we've been purged thru
+  frag_info_t fragstat, accounted_fragstat;
+  nest_info_t rstat, accounted_rstat;
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+  fnode_t() : version(0) {};
+};
+WRITE_CLASS_ENCODER(fnode_t)
+
+
+struct old_rstat_t {
+  snapid_t first;
+  nest_info_t rstat, accounted_rstat;
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& p);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(old_rstat_t)
+
+inline ostream& operator<<(ostream& out, const old_rstat_t& o) {
+  return out << "old_rstat(first " << o.first << " " << o.rstat << " " << o.accounted_rstat << ")";
+}
+
+
+/*
+ * session_info_t
+ */
+
+struct session_info_t {
+  entity_inst_t inst;
+  map completed_requests;
+  interval_set prealloc_inos;   // preallocated, ready to use.
+  interval_set used_inos;       // journaling use
+
+  client_t get_client() const { return client_t(inst.name.num()); }
+
+  void clear_meta() {
+    prealloc_inos.clear();
+    used_inos.clear();
+    completed_requests.clear();
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& p);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(session_info_t)
+
+
+// =======
+// dentries
+
+struct dentry_key_t {
+  snapid_t snapid;
+  const char *name;
+  dentry_key_t() : snapid(0), name(0) {}
+  dentry_key_t(snapid_t s, const char *n) : snapid(s), name(n) {}
+
+  // encode into something that can be decoded as a string.
+  // name_ (head) or name_%x (!head)
+  void encode(bufferlist& bl) const {
+    string key;
+    encode(key);
+    ::encode(key, bl);
+  }
+  void encode(string& key) const {
+    char b[20];
+    if (snapid != CEPH_NOSNAP) {
+      uint64_t val(snapid);
+      snprintf(b, sizeof(b), "%" PRIx64, val);
+    } else {
+      snprintf(b, sizeof(b), "%s", "head");
+    }
+    ostringstream oss;
+    oss << name << "_" << b;
+    key = oss.str();
+  }
+  static void decode_helper(bufferlist::iterator& bl, string& nm, snapid_t& sn) {
+    string key;
+    ::decode(key, bl);
+    decode_helper(key, nm, sn);
+  }
+  static void decode_helper(const string& key, string& nm, snapid_t& sn) {
+    size_t i = key.find_last_of('_');
+    assert(i != string::npos);
+    if (key.compare(i+1, string::npos, "head") == 0) {
+      // name_head
+      sn = CEPH_NOSNAP;
+    } else {
+      // name_%x
+      long long unsigned x = 0;
+      sscanf(key.c_str() + i + 1, "%llx", &x);
+      sn = x;
+    }  
+    nm = string(key.c_str(), i);
+  }
+};
+
+inline ostream& operator<<(ostream& out, const dentry_key_t &k)
+{
+  return out << "(" << k.name << "," << k.snapid << ")";
+}
+
+inline bool operator<(const dentry_key_t& k1, const dentry_key_t& k2)
+{
+  /*
+   * order by name, then snap
+   */
+  int c = strcmp(k1.name, k2.name);
+  return 
+    c < 0 || (c == 0 && k1.snapid < k2.snapid);
+}
+
+
+/*
+ * string_snap_t is a simple (string, snapid_t) pair
+ */
+struct string_snap_t {
+  string name;
+  snapid_t snapid;
+  string_snap_t() {}
+  string_snap_t(const string& n, snapid_t s) : name(n), snapid(s) {}
+  string_snap_t(const char *n, snapid_t s) : name(n), snapid(s) {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& p);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(string_snap_t)
+
+inline bool operator<(const string_snap_t& l, const string_snap_t& r) {
+  int c = strcmp(l.name.c_str(), r.name.c_str());
+  return c < 0 || (c == 0 && l.snapid < r.snapid);
+}
+
+inline ostream& operator<<(ostream& out, const string_snap_t &k)
+{
+  return out << "(" << k.name << "," << k.snapid << ")";
+}
+
+/*
+ * mds_table_pending_t
+ *
+ * mds's requesting any pending ops.  child needs to encode the corresponding
+ * pending mutation state in the table.
+ */
+struct mds_table_pending_t {
+  uint64_t reqid;
+  __s32 mds;
+  version_t tid;
+  mds_table_pending_t() : reqid(0), mds(0), tid(0) {}
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(mds_table_pending_t)
+
+
+// =========
+// requests
+
+struct metareqid_t {
+  entity_name_t name;
+  uint64_t tid;
+  metareqid_t() : tid(0) {}
+  metareqid_t(entity_name_t n, ceph_tid_t t) : name(n), tid(t) {}
+  void encode(bufferlist& bl) const {
+    ::encode(name, bl);
+    ::encode(tid, bl);
+  }
+  void decode(bufferlist::iterator &p) {
+    ::decode(name, p);
+    ::decode(tid, p);
+  }
+};
+WRITE_CLASS_ENCODER(metareqid_t)
+
+inline ostream& operator<<(ostream& out, const metareqid_t& r) {
+  return out << r.name << ":" << r.tid;
+}
+
+inline bool operator==(const metareqid_t& l, const metareqid_t& r) {
+  return (l.name == r.name) && (l.tid == r.tid);
+}
+inline bool operator!=(const metareqid_t& l, const metareqid_t& r) {
+  return (l.name != r.name) || (l.tid != r.tid);
+}
+inline bool operator<(const metareqid_t& l, const metareqid_t& r) {
+  return (l.name < r.name) || 
+    (l.name == r.name && l.tid < r.tid);
+}
+inline bool operator<=(const metareqid_t& l, const metareqid_t& r) {
+  return (l.name < r.name) ||
+    (l.name == r.name && l.tid <= r.tid);
+}
+inline bool operator>(const metareqid_t& l, const metareqid_t& r) { return !(l <= r); }
+inline bool operator>=(const metareqid_t& l, const metareqid_t& r) { return !(l < r); }
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash {
+    size_t operator()(const metareqid_t &r) const { 
+      hash H;
+      return H(r.name.num()) ^ H(r.name.type()) ^ H(r.tid);
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+
+// cap info for client reconnect
+struct cap_reconnect_t {
+  string path;
+  mutable ceph_mds_cap_reconnect capinfo;
+  bufferlist flockbl;
+
+  cap_reconnect_t() {
+    memset(&capinfo, 0, sizeof(capinfo));
+  }
+  cap_reconnect_t(uint64_t cap_id, inodeno_t pino, const string& p, int w, int i, inodeno_t sr) : 
+    path(p) {
+    capinfo.cap_id = cap_id;
+    capinfo.wanted = w;
+    capinfo.issued = i;
+    capinfo.snaprealm = sr;
+    capinfo.pathbase = pino;
+    capinfo.flock_len = 0;
+  }
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void encode_old(bufferlist& bl) const;
+  void decode_old(bufferlist::iterator& bl);
+
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(cap_reconnect_t)
+
+
+// compat for pre-FLOCK feature
+struct old_ceph_mds_cap_reconnect {
+	__le64 cap_id;
+	__le32 wanted;
+	__le32 issued;
+  __le64 old_size;
+  struct ceph_timespec old_mtime, old_atime;
+	__le64 snaprealm;
+	__le64 pathbase;        /* base ino for our path to this ino */
+} __attribute__ ((packed));
+WRITE_RAW_ENCODER(old_ceph_mds_cap_reconnect)
+
+struct old_cap_reconnect_t {
+  string path;
+  old_ceph_mds_cap_reconnect capinfo;
+
+  const old_cap_reconnect_t& operator=(const cap_reconnect_t& n) {
+    path = n.path;
+    capinfo.cap_id = n.capinfo.cap_id;
+    capinfo.wanted = n.capinfo.wanted;
+    capinfo.issued = n.capinfo.issued;
+    capinfo.snaprealm = n.capinfo.snaprealm;
+    capinfo.pathbase = n.capinfo.pathbase;
+    return *this;
+  }
+  operator cap_reconnect_t() {
+    cap_reconnect_t n;
+    n.path = path;
+    n.capinfo.cap_id = capinfo.cap_id;
+    n.capinfo.wanted = capinfo.wanted;
+    n.capinfo.issued = capinfo.issued;
+    n.capinfo.snaprealm = capinfo.snaprealm;
+    n.capinfo.pathbase = capinfo.pathbase;
+    return n;
+  }
+
+  void encode(bufferlist& bl) const {
+    ::encode(path, bl);
+    ::encode(capinfo, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(path, bl);
+    ::decode(capinfo, bl);
+  }
+};
+WRITE_CLASS_ENCODER(old_cap_reconnect_t)
+
+
+// ================================================================
+// dir frag
+
+struct dirfrag_t {
+  inodeno_t ino;
+  frag_t    frag;
+
+  dirfrag_t() : ino(0) { }
+  dirfrag_t(inodeno_t i, frag_t f) : ino(i), frag(f) { }
+
+  void encode(bufferlist& bl) const {
+    ::encode(ino, bl);
+    ::encode(frag, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(ino, bl);
+    ::decode(frag, bl);
+  }
+};
+WRITE_CLASS_ENCODER(dirfrag_t)
+
+
+inline ostream& operator<<(ostream& out, const dirfrag_t df) {
+  out << df.ino;
+  if (!df.frag.is_root()) out << "." << df.frag;
+  return out;
+}
+inline bool operator<(dirfrag_t l, dirfrag_t r) {
+  if (l.ino < r.ino) return true;
+  if (l.ino == r.ino && l.frag < r.frag) return true;
+  return false;
+}
+inline bool operator==(dirfrag_t l, dirfrag_t r) {
+  return l.ino == r.ino && l.frag == r.frag;
+}
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash {
+    size_t operator()(const dirfrag_t &df) const { 
+      static rjhash H;
+      static rjhash I;
+      return H(df.ino) ^ I(df.frag);
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+
+
+// ================================================================
+
+#define META_POP_IRD     0
+#define META_POP_IWR     1
+#define META_POP_READDIR 2
+#define META_POP_FETCH   3
+#define META_POP_STORE   4
+#define META_NPOP        5
+
+class inode_load_vec_t {
+  static const int NUM = 2;
+  std::vector < DecayCounter > vec;
+public:
+  inode_load_vec_t(const utime_t &now)
+     : vec(NUM, DecayCounter(now))
+  {}
+  // for dencoder infrastructure
+  inode_load_vec_t() :
+    vec(NUM, DecayCounter())
+  {}
+  DecayCounter &get(int t) { 
+    assert(t < NUM);
+    return vec[t]; 
+  }
+  void zero(utime_t now) {
+    for (int i=0; i& ls);
+};
+inline void encode(const inode_load_vec_t &c, bufferlist &bl) { c.encode(bl); }
+inline void decode(inode_load_vec_t & c, const utime_t &t, bufferlist::iterator &p) {
+  c.decode(t, p);
+}
+
+class dirfrag_load_vec_t {
+public:
+  static const int NUM = 5;
+  std::vector < DecayCounter > vec;
+  dirfrag_load_vec_t(const utime_t &now)
+     : vec(NUM, DecayCounter(now))
+  { }
+  // for dencoder infrastructure
+  dirfrag_load_vec_t()
+    : vec(NUM, DecayCounter())
+  {}
+  void encode(bufferlist &bl) const {
+    ENCODE_START(2, 2, bl);
+    for (int i=0; i& ls);
+
+  DecayCounter &get(int t) { 
+    assert(t < NUM);
+    return vec[t]; 
+  }
+  void adjust(utime_t now, const DecayRate& rate, double d) {
+    for (int i=0; imds_decay_halflife);
+  return out << "[" << dl.vec[0].get(now, rate) << "," << dl.vec[1].get(now, rate) 
+	     << " " << dl.meta_load(now, rate)
+	     << "]";
+}
+
+
+
+
+
+
+/* mds_load_t
+ * mds load
+ */
+
+struct mds_load_t {
+  dirfrag_load_vec_t auth;
+  dirfrag_load_vec_t all;
+
+  double req_rate;
+  double cache_hit_rate;
+  double queue_len;
+
+  double cpu_load_avg;
+
+  mds_load_t(const utime_t &t) : 
+    auth(t), all(t), req_rate(0), cache_hit_rate(0),
+    queue_len(0), cpu_load_avg(0)
+  {}
+  // mostly for the dencoder infrastructure
+  mds_load_t() :
+    auth(), all(),
+    req_rate(0), cache_hit_rate(0), queue_len(0), cpu_load_avg(0)
+  {}
+  
+  double mds_load();  // defiend in MDBalancer.cc
+  void encode(bufferlist& bl) const;
+  void decode(const utime_t& now, bufferlist::iterator& bl);
+  //this one is for dencoder infrastructure
+  void decode(bufferlist::iterator& bl) { utime_t sample; decode(sample, bl); }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+inline void encode(const mds_load_t &c, bufferlist &bl) { c.encode(bl); }
+inline void decode(mds_load_t &c, const utime_t &t, bufferlist::iterator &p) {
+  c.decode(t, p);
+}
+
+inline ostream& operator<<( ostream& out, mds_load_t& load )
+{
+  return out << "mdsload<" << load.auth << "/" << load.all
+             << ", req " << load.req_rate 
+             << ", hr " << load.cache_hit_rate
+             << ", qlen " << load.queue_len
+	     << ", cpu " << load.cpu_load_avg
+             << ">";
+}
+
+class load_spread_t {
+public:
+  static const int MAX = 4;
+  int last[MAX];
+  int p, n;
+  DecayCounter count;
+
+public:
+  load_spread_t() : p(0), n(0), count(ceph_clock_now(g_ceph_context))
+  {
+    for (int i=0; i= 0 is the auth mds
+#define CDIR_AUTH_PARENT   -1   // default
+#define CDIR_AUTH_UNKNOWN  -2
+#define CDIR_AUTH_DEFAULT   pair(-1, -2)
+#define CDIR_AUTH_UNDEF     pair(-2, -2)
+//#define CDIR_AUTH_ROOTINODE pair( 0, -2)
+
+
+/*
+ * for metadata leases to clients
+ */
+struct ClientLease {
+  client_t client;
+  MDSCacheObject *parent;
+
+  ceph_seq_t seq;
+  utime_t ttl;
+  xlist::item item_session_lease; // per-session list
+  xlist::item item_lease;         // global list
+
+  ClientLease(client_t c, MDSCacheObject *p) : 
+    client(c), parent(p), seq(0),
+    item_session_lease(this),
+    item_lease(this) { }
+};
+
+
+// print hack
+struct mdsco_db_line_prefix {
+  MDSCacheObject *object;
+  mdsco_db_line_prefix(MDSCacheObject *o) : object(o) {}
+};
+ostream& operator<<(ostream& out, mdsco_db_line_prefix o);
+
+// printer
+ostream& operator<<(ostream& out, MDSCacheObject &o);
+
+class MDSCacheObjectInfo {
+public:
+  inodeno_t ino;
+  dirfrag_t dirfrag;
+  string dname;
+  snapid_t snapid;
+
+  MDSCacheObjectInfo() : ino(0) {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+
+inline bool operator==(const MDSCacheObjectInfo& l, const MDSCacheObjectInfo& r) {
+  if (l.ino || r.ino)
+    return l.ino == r.ino && l.snapid == r.snapid;
+  else
+    return l.dirfrag == r.dirfrag && l.dname == r.dname;
+}
+
+WRITE_CLASS_ENCODER(MDSCacheObjectInfo)
+
+
+class MDSCacheObject {
+ public:
+  // -- pins --
+  const static int PIN_REPLICATED =  1000;
+  const static int PIN_DIRTY      =  1001;
+  const static int PIN_LOCK       = -1002;
+  const static int PIN_REQUEST    = -1003;
+  const static int PIN_WAITER     =  1004;
+  const static int PIN_DIRTYSCATTERED = -1005;
+  static const int PIN_AUTHPIN    =  1006;
+  static const int PIN_PTRWAITER  = -1007;
+  const static int PIN_TEMPEXPORTING = 1008;  // temp pin between encode_ and finish_export
+  static const int PIN_CLIENTLEASE = 1009;
+
+  const char *generic_pin_name(int p) {
+    switch (p) {
+    case PIN_REPLICATED: return "replicated";
+    case PIN_DIRTY: return "dirty";
+    case PIN_LOCK: return "lock";
+    case PIN_REQUEST: return "request";
+    case PIN_WAITER: return "waiter";
+    case PIN_DIRTYSCATTERED: return "dirtyscattered";
+    case PIN_AUTHPIN: return "authpin";
+    case PIN_PTRWAITER: return "ptrwaiter";
+    case PIN_TEMPEXPORTING: return "tempexporting";
+    case PIN_CLIENTLEASE: return "clientlease";
+    default: assert(0); return 0;
+    }
+  }
+
+  // -- state --
+  const static int STATE_AUTH      = (1<<30);
+  const static int STATE_DIRTY     = (1<<29);
+  const static int STATE_NOTIFYREF = (1<<28); // notify dropping ref drop through _put()
+  const static int STATE_REJOINING = (1<<27);  // replica has not joined w/ primary copy
+  const static int STATE_REJOINUNDEF = (1<<26);  // contents undefined.
+
+
+  // -- wait --
+  const static uint64_t WAIT_SINGLEAUTH  = (1ull<<60);
+  const static uint64_t WAIT_UNFREEZE    = (1ull<<59); // pka AUTHPINNABLE
+
+
+  // ============================================
+  // cons
+ public:
+  MDSCacheObject() :
+    state(0), 
+    ref(0),
+    replica_nonce(0) {}
+  virtual ~MDSCacheObject() {}
+
+  // printing
+  virtual void print(ostream& out) = 0;
+  virtual ostream& print_db_line_prefix(ostream& out) { 
+    return out << "mdscacheobject(" << this << ") "; 
+  }
+  
+  // --------------------------------------------
+  // state
+ protected:
+  __u32 state;     // state bits
+
+ public:
+  unsigned get_state() const { return state; }
+  unsigned state_test(unsigned mask) const { return (state & mask); }
+  void state_clear(unsigned mask) { state &= ~mask; }
+  void state_set(unsigned mask) { state |= mask; }
+  void state_reset(unsigned s) { state = s; }
+
+  bool is_auth() const { return state_test(STATE_AUTH); }
+  bool is_dirty() const { return state_test(STATE_DIRTY); }
+  bool is_clean() const { return !is_dirty(); }
+  bool is_rejoining() const { return state_test(STATE_REJOINING); }
+
+  // --------------------------------------------
+  // authority
+  virtual pair authority() = 0;
+  bool is_ambiguous_auth() {
+    return authority().second != CDIR_AUTH_UNKNOWN;
+  }
+
+  // --------------------------------------------
+  // pins
+protected:
+  __s32      ref;       // reference count
+#ifdef MDS_REF_SET
+  map ref_map;
+#endif
+
+ public:
+  int get_num_ref(int by = -1) {
+#ifdef MDS_REF_SET
+    if (by >= 0) {
+      if (ref_map.find(by) == ref_map.end())
+	return 0;
+      return ref_map[by];
+    }
+#endif
+    return ref;
+  }
+#ifdef MDS_REF_SET
+  int get_pin_totals() {
+    int total = 0;
+    for(map::iterator i = ref_map.begin(); i != ref_map.end(); ++i) {
+      total += i->second;
+    }
+    return total;
+  }
+#endif
+  virtual const char *pin_name(int by) = 0;
+  //bool is_pinned_by(int by) { return ref_set.count(by); }
+  //multiset& get_ref_set() { return ref_set; }
+
+  virtual void last_put() {}
+  virtual void bad_put(int by) {
+#ifdef MDS_REF_SET
+    assert(ref_map[by] > 0);
+#endif
+    assert(ref > 0);
+  }
+  virtual void _put() {}
+  void put(int by) {
+#ifdef MDS_REF_SET
+    if (ref == 0 || ref_map[by] == 0) {
+#else
+    if (ref == 0) {
+#endif
+      bad_put(by);
+    } else {
+      ref--;
+#ifdef MDS_REF_SET
+      ref_map[by]--;
+      assert(ref == get_pin_totals());
+#endif
+      if (ref == 0)
+	last_put();
+      if (state_test(STATE_NOTIFYREF))
+	_put();
+    }
+  }
+
+  virtual void first_get() {}
+  virtual void bad_get(int by) {
+#ifdef MDS_REF_SET
+    assert(by < 0 || ref_map[by] == 0);
+#endif
+    assert(0);
+  }
+  void get(int by) {
+    if (ref == 0)
+      first_get();
+    ref++;
+#ifdef MDS_REF_SET
+    if (ref_map.find(by) == ref_map.end())
+      ref_map[by] = 0;
+    ref_map[by]++;
+    assert(ref == get_pin_totals());
+#endif
+  }
+
+  void print_pin_set(ostream& out) {
+#ifdef MDS_REF_SET
+    map::iterator it = ref_map.begin();
+    while (it != ref_map.end()) {
+      out << " " << pin_name(it->first) << "=" << it->second;
+      ++it;
+    }
+#else
+    out << " nref=" << ref;
+#endif
+  }
+
+
+  // --------------------------------------------
+  // auth pins
+  virtual bool can_auth_pin() = 0;
+  virtual void auth_pin(void *who) = 0;
+  virtual void auth_unpin(void *who) = 0;
+  virtual bool is_frozen() = 0;
+  virtual bool is_freezing() = 0;
+  virtual bool is_freezing_or_frozen() {
+    return is_frozen() || is_freezing();
+  }
+
+
+  // --------------------------------------------
+  // replication (across mds cluster)
+ protected:
+  unsigned		replica_nonce; // [replica] defined on replica
+  map	replica_map;   // [auth] mds -> nonce
+
+ public:
+  bool is_replicated() { return !replica_map.empty(); }
+  bool is_replica(int mds) { return replica_map.count(mds); }
+  int num_replicas() { return replica_map.size(); }
+  unsigned add_replica(int mds) {
+    if (replica_map.count(mds)) 
+      return ++replica_map[mds];  // inc nonce
+    if (replica_map.empty()) 
+      get(PIN_REPLICATED);
+    return replica_map[mds] = 1;
+  }
+  void add_replica(int mds, unsigned nonce) {
+    if (replica_map.empty()) 
+      get(PIN_REPLICATED);
+    replica_map[mds] = nonce;
+  }
+  unsigned get_replica_nonce(int mds) {
+    assert(replica_map.count(mds));
+    return replica_map[mds];
+  }
+  void remove_replica(int mds) {
+    assert(replica_map.count(mds));
+    replica_map.erase(mds);
+    if (replica_map.empty())
+      put(PIN_REPLICATED);
+  }
+  void clear_replica_map() {
+    if (!replica_map.empty())
+      put(PIN_REPLICATED);
+    replica_map.clear();
+  }
+  map::iterator replicas_begin() { return replica_map.begin(); }
+  map::iterator replicas_end() { return replica_map.end(); }
+  const map& get_replicas() { return replica_map; }
+  void list_replicas(set& ls) {
+    for (map::const_iterator p = replica_map.begin();
+	 p != replica_map.end();
+	 ++p) 
+      ls.insert(p->first);
+  }
+
+  unsigned get_replica_nonce() { return replica_nonce; }
+  void set_replica_nonce(unsigned n) { replica_nonce = n; }
+
+
+  // ---------------------------------------------
+  // waiting
+ protected:
+  multimap  waiting;
+
+ public:
+  bool is_waiter_for(uint64_t mask, uint64_t min=0) {
+    if (!min) {
+      min = mask;
+      while (min & (min-1))  // if more than one bit is set
+	min &= min-1;        //  clear LSB
+    }
+    for (multimap::iterator p = waiting.lower_bound(min);
+	 p != waiting.end();
+	 ++p) {
+      if (p->first & mask) return true;
+      if (p->first > mask) return false;
+    }
+    return false;
+  }
+  virtual void add_waiter(uint64_t mask, Context *c) {
+    if (waiting.empty())
+      get(PIN_WAITER);
+    waiting.insert(pair(mask, c));
+//    pdout(10,g_conf->debug_mds) << (mdsco_db_line_prefix(this)) 
+//			       << "add_waiter " << hex << mask << dec << " " << c
+//			       << " on " << *this
+//			       << dendl;
+    
+  }
+  virtual void take_waiting(uint64_t mask, list& ls) {
+    if (waiting.empty()) return;
+    multimap::iterator it = waiting.begin();
+    while (it != waiting.end()) {
+      if (it->first & mask) {
+	ls.push_back(it->second);
+//	pdout(10,g_conf->debug_mds) << (mdsco_db_line_prefix(this))
+//				   << "take_waiting mask " << hex << mask << dec << " took " << it->second
+//				   << " tag " << hex << it->first << dec
+//				   << " on " << *this
+//				   << dendl;
+	waiting.erase(it++);
+      } else {
+//	pdout(10,g_conf->debug_mds) << "take_waiting mask " << hex << mask << dec << " SKIPPING " << it->second
+//				   << " tag " << hex << it->first << dec
+//				   << " on " << *this 
+//				   << dendl;
+	++it;
+      }
+    }
+    if (waiting.empty())
+      put(PIN_WAITER);
+  }
+  void finish_waiting(uint64_t mask, int result = 0) {
+    list finished;
+    take_waiting(mask, finished);
+    finish_contexts(g_ceph_context, finished, result);
+  }
+
+
+  // ---------------------------------------------
+  // locking
+  // noop unless overloaded.
+  virtual SimpleLock* get_lock(int type) { assert(0); return 0; }
+  virtual void set_object_info(MDSCacheObjectInfo &info) { assert(0); }
+  virtual void encode_lock_state(int type, bufferlist& bl) { assert(0); }
+  virtual void decode_lock_state(int type, bufferlist& bl) { assert(0); }
+  virtual void finish_lock_waiters(int type, uint64_t mask, int r=0) { assert(0); }
+  virtual void add_lock_waiter(int type, uint64_t mask, Context *c) { assert(0); }
+  virtual bool is_lock_waiting(int type, uint64_t mask) { assert(0); return false; }
+
+  virtual void clear_dirty_scattered(int type) { assert(0); }
+
+  // ---------------------------------------------
+  // ordering
+  virtual bool is_lt(const MDSCacheObject *r) const = 0;
+  struct ptr_lt {
+    bool operator()(const MDSCacheObject* l, const MDSCacheObject* r) const {
+      return l->is_lt(r);
+    }
+  };
+
+};
+
+inline ostream& operator<<(ostream& out, MDSCacheObject &o) {
+  o.print(out);
+  return out;
+}
+
+inline ostream& operator<<(ostream& out, const MDSCacheObjectInfo &info) {
+  if (info.ino) return out << info.ino << "." << info.snapid;
+  if (info.dname.length()) return out << info.dirfrag << "/" << info.dname
+				      << " snap " << info.snapid;
+  return out << info.dirfrag;
+}
+
+inline ostream& operator<<(ostream& out, mdsco_db_line_prefix o) {
+  o.object->print_db_line_prefix(out);
+  return out;
+}
+
+
+
+
+
+#endif
diff --git a/ceph/src/mds/snap.cc b/ceph/src/mds/snap.cc
new file mode 100644
index 00000000..06dc9559
--- /dev/null
+++ b/ceph/src/mds/snap.cc
@@ -0,0 +1,195 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004- Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "snap.h"
+
+#include "common/Formatter.h"
+
+/*
+ * SnapInfo
+ */
+
+void SnapInfo::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(snapid, bl);
+  ::encode(ino, bl);
+  ::encode(stamp, bl);
+  ::encode(name, bl);
+  ENCODE_FINISH(bl);
+}
+
+void SnapInfo::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(snapid, bl);
+  ::decode(ino, bl);
+  ::decode(stamp, bl);
+  ::decode(name, bl);
+  DECODE_FINISH(bl);
+}
+
+void SnapInfo::dump(Formatter *f) const
+{
+  f->dump_unsigned("snapid", snapid);
+  f->dump_unsigned("ino", ino);
+  f->dump_stream("stamp") << stamp;
+  f->dump_string("name", name);
+}
+
+void SnapInfo::generate_test_instances(list& ls)
+{
+  ls.push_back(new SnapInfo);
+  ls.push_back(new SnapInfo);
+  ls.back()->snapid = 1;
+  ls.back()->ino = 2;
+  ls.back()->stamp = utime_t(3, 4);
+  ls.back()->name = "foo";
+}
+
+ostream& operator<<(ostream& out, const SnapInfo &sn)
+{
+  return out << "snap(" << sn.snapid
+	     << " " << sn.ino
+	     << " '" << sn.name
+	     << "' " << sn.stamp << ")";
+}
+
+const string& SnapInfo::get_long_name()
+{
+  if (long_name.length() == 0) {
+    char nm[80];
+    snprintf(nm, sizeof(nm), "_%s_%llu", name.c_str(), (unsigned long long)ino);
+    long_name = nm;
+  }
+  return long_name;
+}
+
+/*
+ * snaplink_t
+ */
+
+void snaplink_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(ino, bl);
+  ::encode(first, bl);
+  ENCODE_FINISH(bl);
+}
+
+void snaplink_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(ino, bl);
+  ::decode(first, bl);
+  DECODE_FINISH(bl);
+}
+
+void snaplink_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("ino", ino);
+  f->dump_unsigned("first", first);
+}
+
+void snaplink_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new snaplink_t);
+  ls.push_back(new snaplink_t);
+  ls.back()->ino = 2;
+  ls.back()->first = 123;
+}
+
+ostream& operator<<(ostream& out, const snaplink_t &l)
+{
+  return out << l.ino << "@" << l.first;
+}
+
+/*
+ * sr_t
+ */
+
+void sr_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(4, 4, bl);
+  ::encode(seq, bl);
+  ::encode(created, bl);
+  ::encode(last_created, bl);
+  ::encode(last_destroyed, bl);
+  ::encode(current_parent_since, bl);
+  ::encode(snaps, bl);
+  ::encode(past_parents, bl);
+  ENCODE_FINISH(bl);
+}
+
+void sr_t::decode(bufferlist::iterator& p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(4, 4, 4, p);
+  if (struct_v == 2) {
+    __u8 struct_v;
+    ::decode(struct_v, p);  // yes, really: extra byte for v2 encoding only, see 6ee52e7d.
+  }
+  ::decode(seq, p);
+  ::decode(created, p);
+  ::decode(last_created, p);
+  ::decode(last_destroyed, p);
+  ::decode(current_parent_since, p);
+  ::decode(snaps, p);
+  ::decode(past_parents, p);
+  DECODE_FINISH(p);
+}
+
+void sr_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("seq", seq);
+  f->dump_unsigned("created", created);
+  f->dump_unsigned("last_created", last_created);
+  f->dump_unsigned("last_destroyed", last_destroyed);
+  f->dump_unsigned("current_parent_since", current_parent_since);
+
+  f->open_array_section("snaps");
+  for (map::const_iterator p = snaps.begin(); p != snaps.end(); ++p) {
+    f->open_object_section("snapinfo");
+    f->dump_unsigned("last", p->first);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("past_parents");
+  for (map::const_iterator p = past_parents.begin(); p != past_parents.end(); ++p) {
+    f->open_object_section("past_parent");
+    f->dump_unsigned("last", p->first);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void sr_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new sr_t);
+  ls.push_back(new sr_t);
+  ls.back()->seq = 1;
+  ls.back()->created = 2;
+  ls.back()->last_created = 3;
+  ls.back()->last_destroyed = 4;
+  ls.back()->current_parent_since = 5;
+  ls.back()->snaps[123].snapid = 7;
+  ls.back()->snaps[123].ino = 8;
+  ls.back()->snaps[123].stamp = utime_t(9, 10);
+  ls.back()->snaps[123].name = "name1";
+  ls.back()->past_parents[12].ino = 12;
+  ls.back()->past_parents[12].first = 3;
+}
+
diff --git a/ceph/src/mds/snap.h b/ceph/src/mds/snap.h
new file mode 100644
index 00000000..0216e624
--- /dev/null
+++ b/ceph/src/mds/snap.h
@@ -0,0 +1,94 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDS_SNAP_H
+#define CEPH_MDS_SNAP_H
+
+#include "mdstypes.h"
+#include "common/snap_types.h"
+
+/*
+ * generic snap descriptor.
+ */
+struct SnapInfo {
+  snapid_t snapid;
+  inodeno_t ino;
+  utime_t stamp;
+  string name;
+
+  string long_name; ///< cached _$ino_$name
+  
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+
+  const string& get_long_name();
+};
+WRITE_CLASS_ENCODER(SnapInfo)
+
+ostream& operator<<(ostream& out, const SnapInfo &sn);
+
+
+/*
+ * SnapRealm - a subtree that shares the same set of snapshots.
+ */
+struct SnapRealm;
+struct CapabilityGroup;
+class CInode;
+class MDCache;
+struct MDRequest;
+
+
+
+#include "Capability.h"
+
+struct snaplink_t {
+  inodeno_t ino;
+  snapid_t first;
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(snaplink_t)
+
+ostream& operator<<(ostream& out, const snaplink_t &l);
+
+
+// carry data about a specific version of a SnapRealm
+struct sr_t {
+  snapid_t seq;                     // basically, a version/seq # for changes to _this_ realm.
+  snapid_t created;                 // when this realm was created.
+  snapid_t last_created;            // last snap created in _this_ realm.
+  snapid_t last_destroyed;          // seq for last removal
+  snapid_t current_parent_since;
+  map snaps;
+  map past_parents;  // key is "last" (or NOSNAP)
+
+  sr_t()
+    : seq(0), created(0),
+      last_created(0), last_destroyed(0),
+      current_parent_since(1)
+  {}
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(sr_t);
+
+#endif
diff --git a/ceph/src/messages/MAuth.h b/ceph/src/messages/MAuth.h
new file mode 100644
index 00000000..40a7fa5e
--- /dev/null
+++ b/ceph/src/messages/MAuth.h
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MAUTH_H
+#define CEPH_MAUTH_H
+
+#include "messages/PaxosServiceMessage.h"
+
+struct MAuth : public PaxosServiceMessage {
+  __u32 protocol;
+  bufferlist auth_payload;
+  epoch_t monmap_epoch;
+
+  /* if protocol == 0, then auth_payload is a set<__u32> listing protocols the client supports */
+
+  MAuth() : PaxosServiceMessage(CEPH_MSG_AUTH, 0), protocol(0), monmap_epoch(0) { }
+private:
+  ~MAuth() {}
+
+public:
+  const char *get_type_name() const { return "auth"; }
+  void print(ostream& out) const {
+    out << "auth(proto " << protocol << " " << auth_payload.length() << " bytes"
+	<< " epoch " << monmap_epoch << ")";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(protocol, p);
+    ::decode(auth_payload, p);
+    if (!p.end())
+      ::decode(monmap_epoch, p);
+    else
+      monmap_epoch = 0;
+  }
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(protocol, payload);
+    ::encode(auth_payload, payload);
+    ::encode(monmap_epoch, payload);
+  }
+  bufferlist& get_auth_payload() { return auth_payload; }
+};
+
+#endif
diff --git a/ceph/src/messages/MAuthReply.h b/ceph/src/messages/MAuthReply.h
new file mode 100644
index 00000000..5fea5a55
--- /dev/null
+++ b/ceph/src/messages/MAuthReply.h
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MAUTHREPLY_H
+#define CEPH_MAUTHREPLY_H
+
+#include "msg/Message.h"
+#include "common/errno.h"
+
+struct MAuthReply : public Message {
+  __u32 protocol;
+  __s32 result;
+  uint64_t global_id;      // if zero, meaningless
+  string result_msg;
+  bufferlist result_bl;
+
+  MAuthReply() : Message(CEPH_MSG_AUTH_REPLY), protocol(0), result(0), global_id(0) {}
+  MAuthReply(__u32 p, bufferlist *bl = NULL, int r = 0, uint64_t gid=0, const char *msg = "") :
+    Message(CEPH_MSG_AUTH_REPLY),
+    protocol(p), result(r), global_id(gid),
+    result_msg(msg) {
+    if (bl)
+      result_bl = *bl;
+  }
+private:
+  ~MAuthReply() {}
+
+public:
+  const char *get_type_name() const { return "auth_reply"; }
+  void print(ostream& o) const {
+    o << "auth_reply(proto " << protocol << " " << result << " " << cpp_strerror(result);
+    if (result_msg.length())
+      o << ": " << result_msg;
+    o << ")";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(protocol, p);
+    ::decode(result, p);
+    ::decode(global_id, p);
+    ::decode(result_bl, p);
+    ::decode(result_msg, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(protocol, payload);
+    ::encode(result, payload);
+    ::encode(global_id, payload);
+    ::encode(result_bl, payload);
+    ::encode(result_msg, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MBackfillReserve.h b/ceph/src/messages/MBackfillReserve.h
new file mode 100644
index 00000000..d30e285c
--- /dev/null
+++ b/ceph/src/messages/MBackfillReserve.h
@@ -0,0 +1,91 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MBACKFILL_H
+#define CEPH_MBACKFILL_H
+
+#include "msg/Message.h"
+
+class MBackfillReserve : public Message {
+  static const int HEAD_VERSION = 3;
+  static const int COMPAT_VERSION = 1;
+public:
+  spg_t pgid;
+  epoch_t query_epoch;
+  enum {
+    REQUEST = 0,
+    GRANT = 1,
+    REJECT = 2,
+  };
+  int type;
+  unsigned priority;
+
+  MBackfillReserve()
+    : Message(MSG_OSD_BACKFILL_RESERVE, HEAD_VERSION, COMPAT_VERSION),
+      query_epoch(0), type(-1), priority(-1) {}
+  MBackfillReserve(int type,
+		   spg_t pgid,
+		   epoch_t query_epoch, unsigned prio = -1)
+    : Message(MSG_OSD_BACKFILL_RESERVE, HEAD_VERSION, COMPAT_VERSION),
+      pgid(pgid), query_epoch(query_epoch),
+      type(type), priority(prio) {}
+
+  const char *get_type_name() const {
+    return "MBackfillReserve";
+  }
+
+  void print(ostream& out) const {
+    out << "MBackfillReserve ";
+    switch (type) {
+    case REQUEST:
+      out << "REQUEST ";
+      break;
+    case GRANT:
+      out << "GRANT "; 
+      break;
+    case REJECT:
+      out << "REJECT ";
+      break;
+    }
+    out << " pgid: " << pgid << ", query_epoch: " << query_epoch;
+    if (type == REQUEST) out << ", prio: " << priority;
+    return;
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid.pgid, p);
+    ::decode(query_epoch, p);
+    ::decode(type, p);
+    if (header.version > 1)
+      ::decode(priority, p);
+    else
+      priority = 0;
+    if (header.version >= 3)
+      ::decode(pgid.shard, p);
+    else
+      pgid.shard = ghobject_t::no_shard();
+
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(pgid.pgid, payload);
+    ::encode(query_epoch, payload);
+    ::encode(type, payload);
+    ::encode(priority, payload);
+    ::encode(pgid.shard, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MCacheExpire.h b/ceph/src/messages/MCacheExpire.h
new file mode 100644
index 00000000..b71ada74
--- /dev/null
+++ b/ceph/src/messages/MCacheExpire.h
@@ -0,0 +1,104 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCACHEEXPIRE_H
+#define CEPH_MCACHEEXPIRE_H
+
+#include "mds/mdstypes.h"
+
+class MCacheExpire : public Message {
+  __s32 from;
+
+public:
+  /*
+    group things by realm (auth delgation root), since that's how auth is determined.
+    that makes it less work to process when exports are in progress.
+  */
+  struct realm {
+    map inodes;
+    map dirs;
+    map,uint32_t> > dentries;
+
+    void merge(realm& o) {
+      inodes.insert(o.inodes.begin(), o.inodes.end());
+      dirs.insert(o.dirs.begin(), o.dirs.end());
+      for (map,uint32_t> >::iterator p = o.dentries.begin();
+	   p != o.dentries.end();
+	   ++p) {
+	if (dentries.count(p->first) == 0)
+	  dentries[p->first] = p->second;
+	else
+	  dentries[p->first].insert(p->second.begin(), p->second.end());
+      }
+    }
+
+    void encode(bufferlist &bl) const {
+      ::encode(inodes, bl);
+      ::encode(dirs, bl);
+      ::encode(dentries, bl);
+    }
+    void decode(bufferlist::iterator &bl) {
+      ::decode(inodes, bl);
+      ::decode(dirs, bl);
+      ::decode(dentries, bl);
+    }
+  };
+  WRITE_CLASS_ENCODER(realm)
+
+  map realms;
+
+  int get_from() { return from; }
+
+  MCacheExpire() : Message(MSG_MDS_CACHEEXPIRE), from(-1) {}
+  MCacheExpire(int f) : 
+    Message(MSG_MDS_CACHEEXPIRE),
+    from(f) { }
+private:
+  ~MCacheExpire() {}
+
+public:
+  virtual const char *get_type_name() const { return "cache_expire";}
+  
+  void add_inode(dirfrag_t r, vinodeno_t vino, unsigned nonce) {
+    realms[r].inodes[vino] = nonce;
+  }
+  void add_dir(dirfrag_t r, dirfrag_t df, unsigned nonce) {
+    realms[r].dirs[df] = nonce;
+  }
+  void add_dentry(dirfrag_t r, dirfrag_t df, const string& dn, snapid_t last, unsigned nonce) {
+    realms[r].dentries[df][pair(dn,last)] = nonce;
+  }
+
+  void add_realm(dirfrag_t df, realm& r) {
+    if (realms.count(df) == 0)
+      realms[df] = r;
+    else
+      realms[df].merge(r);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(from, p);
+    ::decode(realms, p);
+  }
+    
+  void encode_payload(uint64_t features) {
+    ::encode(from, payload);
+    ::encode(realms, payload);
+  }
+};
+
+WRITE_CLASS_ENCODER(MCacheExpire::realm)
+
+#endif
diff --git a/ceph/src/messages/MClientCapRelease.h b/ceph/src/messages/MClientCapRelease.h
new file mode 100644
index 00000000..c1281dee
--- /dev/null
+++ b/ceph/src/messages/MClientCapRelease.h
@@ -0,0 +1,51 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCLIENTCAPRELEASE_H
+#define CEPH_MCLIENTCAPRELEASE_H
+
+#include "msg/Message.h"
+
+
+class MClientCapRelease : public Message {
+ public:
+  struct ceph_mds_cap_release head;
+  vector caps;
+
+  MClientCapRelease() : 
+    Message(CEPH_MSG_CLIENT_CAPRELEASE) {
+    memset(&head, 0, sizeof(head));
+  }
+private:
+  ~MClientCapRelease() {}
+
+public:
+  const char *get_type_name() const { return "client_cap_release";}
+  void print(ostream& out) const {
+    out << "client_cap_release(" << caps.size() << ")";
+  }
+  
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(head, p);
+    ::decode_nohead(head.num, caps, p);
+  }
+  void encode_payload(uint64_t features) {
+    head.num = caps.size();
+    ::encode(head, payload);
+    ::encode_nohead(caps, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MClientCaps.h b/ceph/src/messages/MClientCaps.h
new file mode 100644
index 00000000..11c8068d
--- /dev/null
+++ b/ceph/src/messages/MClientCaps.h
@@ -0,0 +1,225 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCLIENTCAPS_H
+#define CEPH_MCLIENTCAPS_H
+
+#include "msg/Message.h"
+#include "include/ceph_features.h"
+
+
+class MClientCaps : public Message {
+
+  static const int HEAD_VERSION = 4;   // added flock metadata, inline data
+  static const int COMPAT_VERSION = 1;
+
+ public:
+  struct ceph_mds_caps head;
+  struct ceph_mds_cap_peer peer;
+  bufferlist snapbl;
+  bufferlist xattrbl;
+  bufferlist flockbl;
+  version_t  inline_version;
+  bufferlist inline_data;
+
+  int      get_caps() { return head.caps; }
+  int      get_wanted() { return head.wanted; }
+  int      get_dirty() { return head.dirty; }
+  ceph_seq_t get_seq() { return head.seq; }
+  ceph_seq_t get_issue_seq() { return head.issue_seq; }
+  ceph_seq_t get_mseq() { return head.migrate_seq; }
+
+  inodeno_t get_ino() { return inodeno_t(head.ino); }
+  inodeno_t get_realm() { return inodeno_t(head.realm); }
+  uint64_t get_cap_id() { return head.cap_id; }
+
+  uint64_t get_size() { return head.size;  }
+  uint64_t get_max_size() { return head.max_size;  }
+  __u32 get_truncate_seq() { return head.truncate_seq; }
+  uint64_t get_truncate_size() { return head.truncate_size; }
+  utime_t get_ctime() { return utime_t(head.ctime); }
+  utime_t get_mtime() { return utime_t(head.mtime); }
+  utime_t get_atime() { return utime_t(head.atime); }
+  __u32 get_time_warp_seq() { return head.time_warp_seq; }
+
+  ceph_file_layout& get_layout() { return head.layout; }
+
+  int       get_migrate_seq() { return head.migrate_seq; }
+  int       get_op() { return head.op; }
+
+  uint64_t get_client_tid() { return get_tid(); }
+  void set_client_tid(uint64_t s) { set_tid(s); }
+
+  snapid_t get_snap_follows() { return snapid_t(head.snap_follows); }
+  void set_snap_follows(snapid_t s) { head.snap_follows = s; }
+
+  void set_caps(int c) { head.caps = c; }
+  void set_wanted(int w) { head.wanted = w; }
+
+  void set_max_size(uint64_t ms) { head.max_size = ms; }
+
+  void set_migrate_seq(unsigned m) { head.migrate_seq = m; }
+  void set_op(int o) { head.op = o; }
+
+  void set_size(loff_t s) { head.size = s; }
+  void set_mtime(const utime_t &t) { t.encode_timeval(&head.mtime); }
+  void set_atime(const utime_t &t) { t.encode_timeval(&head.atime); }
+
+  void set_cap_peer(uint64_t id, ceph_seq_t seq, ceph_seq_t mseq, int mds, int flags) {
+    peer.cap_id = id;
+    peer.seq = seq;
+    peer.mseq = mseq;
+    peer.mds = mds;
+    peer.flags = flags;
+  }
+
+  MClientCaps()
+    : Message(CEPH_MSG_CLIENT_CAPS, HEAD_VERSION, COMPAT_VERSION) {
+    inline_version = 0;
+  }
+  MClientCaps(int op,
+	      inodeno_t ino,
+	      inodeno_t realm,
+	      uint64_t id,
+	      long seq,
+	      int caps,
+	      int wanted,
+	      int dirty,
+	      int mseq)
+    : Message(CEPH_MSG_CLIENT_CAPS, HEAD_VERSION, COMPAT_VERSION) {
+    memset(&head, 0, sizeof(head));
+    head.op = op;
+    head.ino = ino;
+    head.realm = realm;
+    head.cap_id = id;
+    head.seq = seq;
+    head.caps = caps;
+    head.wanted = wanted;
+    head.dirty = dirty;
+    head.migrate_seq = mseq;
+    peer.cap_id = 0;
+    inline_version = 0;
+  }
+  MClientCaps(int op,
+	      inodeno_t ino, inodeno_t realm,
+	      uint64_t id, int mseq)
+    : Message(CEPH_MSG_CLIENT_CAPS, HEAD_VERSION) {
+    memset(&head, 0, sizeof(head));
+    head.op = op;
+    head.ino = ino;
+    head.realm = realm;
+    head.cap_id = id;
+    head.migrate_seq = mseq;
+    peer.cap_id = 0;
+    inline_version = 0;
+  }
+private:
+  ~MClientCaps() {}
+
+public:
+  const char *get_type_name() const { return "Cfcap";}
+  void print(ostream& out) const {
+    out << "client_caps(" << ceph_cap_op_name(head.op)
+	<< " ino " << inodeno_t(head.ino)
+	<< " " << head.cap_id
+	<< " seq " << head.seq;
+    if (get_tid())
+      out << " tid " << get_tid();
+    out << " caps=" << ccap_string(head.caps)
+	<< " dirty=" << ccap_string(head.dirty)
+	<< " wanted=" << ccap_string(head.wanted);
+    out << " follows " << snapid_t(head.snap_follows);
+    if (head.migrate_seq)
+      out << " mseq " << head.migrate_seq;
+
+    out << " size " << head.size << "/" << head.max_size;
+    if (head.truncate_seq)
+      out << " ts " << head.truncate_seq;
+    out << " mtime " << utime_t(head.mtime);
+    if (head.time_warp_seq)
+      out << " tws " << head.time_warp_seq;
+
+    if (head.xattr_version)
+      out << " xattrs(v=" << head.xattr_version << " l=" << xattrbl.length() << ")";
+
+    out << ")";
+  }
+  
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(head, p);
+    ::decode_nohead(head.snap_trace_len, snapbl, p);
+
+    assert(middle.length() == head.xattr_len);
+    if (head.xattr_len)
+      xattrbl = middle;
+
+    // conditionally decode flock metadata
+    if (header.version >= 2)
+      ::decode(flockbl, p);
+
+    if (header.version >= 3) {
+      if (head.op == CEPH_CAP_OP_IMPORT)
+	::decode(peer, p);
+      else if (head.op == CEPH_CAP_OP_EXPORT)
+	memcpy(&peer, &head.peer, sizeof(peer));
+    }
+
+    if (header.version >= 4) {
+      ::decode(inline_version, p);
+      ::decode(inline_data, p);
+    } else {
+      inline_version = CEPH_INLINE_NONE;
+    }
+  }
+  void encode_payload(uint64_t features) {
+    head.snap_trace_len = snapbl.length();
+    head.xattr_len = xattrbl.length();
+
+    // record peer in unused fields of cap export message
+    if ((features & CEPH_FEATURE_EXPORT_PEER) && head.op == CEPH_CAP_OP_EXPORT)
+      memcpy(&head.peer, &peer, sizeof(peer));
+
+    ::encode(head, payload);
+    ::encode_nohead(snapbl, payload);
+
+    middle = xattrbl;
+
+    // conditionally include flock metadata
+    if (features & CEPH_FEATURE_FLOCK) {
+      ::encode(flockbl, payload);
+    } else {
+      header.version = 1;
+      return;
+    }
+
+    if (features & CEPH_FEATURE_EXPORT_PEER) {
+      if (head.op == CEPH_CAP_OP_IMPORT)
+	::encode(peer, payload);
+    } else {
+      header.version = 2;
+      return;
+    }
+
+    if (features & CEPH_FEATURE_MDS_INLINE_DATA) {
+      ::encode(inline_version, payload);
+      ::encode(inline_data, payload);
+    } else {
+      header.version = 3;
+      return;
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MClientLease.h b/ceph/src/messages/MClientLease.h
new file mode 100644
index 00000000..2b80c2ad
--- /dev/null
+++ b/ceph/src/messages/MClientLease.h
@@ -0,0 +1,83 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MCLIENTLEASE_H
+#define CEPH_MCLIENTLEASE_H
+
+#include "msg/Message.h"
+
+struct MClientLease : public Message {
+  struct ceph_mds_lease h;
+  string dname;
+  
+  int get_action() const { return h.action; }
+  ceph_seq_t get_seq() const { return h.seq; }
+  int get_mask() const { return h.mask; }
+  inodeno_t get_ino() const { return inodeno_t(h.ino); }
+  snapid_t get_first() const { return snapid_t(h.first); }
+  snapid_t get_last() const { return snapid_t(h.last); }
+
+  MClientLease() : Message(CEPH_MSG_CLIENT_LEASE) {}
+  MClientLease(int ac, ceph_seq_t seq, int m, uint64_t i, uint64_t sf, uint64_t sl) :
+    Message(CEPH_MSG_CLIENT_LEASE) {
+    h.action = ac;
+    h.seq = seq;
+    h.mask = m;
+    h.ino = i;
+    h.first = sf;
+    h.last = sl;
+    h.duration_ms = 0;
+  }
+  MClientLease(int ac, ceph_seq_t seq, int m, uint64_t i, uint64_t sf, uint64_t sl, const string& d) :
+    Message(CEPH_MSG_CLIENT_LEASE),
+    dname(d) {
+    h.action = ac;
+    h.seq = seq;
+    h.mask = m;
+    h.ino = i;
+    h.first = sf;
+    h.last = sl;
+    h.duration_ms = 0;
+  }
+private:
+  ~MClientLease() {}
+
+public:
+  const char *get_type_name() const { return "client_lease"; }
+  void print(ostream& out) const {
+    out << "client_lease(a=" << ceph_lease_op_name(get_action())
+	<< " seq " << get_seq()
+	<< " mask " << get_mask();
+    out << " " << get_ino();
+    if (h.last != CEPH_NOSNAP)
+      out << " [" << snapid_t(h.first) << "," << snapid_t(h.last) << "]";
+    if (dname.length())
+      out << "/" << dname;
+    out << ")";
+  }
+  
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(h, p);
+    ::decode(dname, p);
+  }
+  virtual void encode_payload(uint64_t features) {
+    ::encode(h, payload);
+    ::encode(dname, payload);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MClientReconnect.h b/ceph/src/messages/MClientReconnect.h
new file mode 100644
index 00000000..4e2839ca
--- /dev/null
+++ b/ceph/src/messages/MClientReconnect.h
@@ -0,0 +1,107 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCLIENTRECONNECT_H
+#define CEPH_MCLIENTRECONNECT_H
+
+#include "msg/Message.h"
+#include "mds/mdstypes.h"
+#include "include/ceph_features.h"
+
+
+class MClientReconnect : public Message {
+
+  const static int HEAD_VERSION = 3;
+
+public:
+  map  caps;   // only head inodes
+  vector realms;
+
+  MClientReconnect() : Message(CEPH_MSG_CLIENT_RECONNECT, HEAD_VERSION) { }
+private:
+  ~MClientReconnect() {}
+
+public:
+  const char *get_type_name() const { return "client_reconnect"; }
+  void print(ostream& out) const {
+    out << "client_reconnect("
+	<< caps.size() << " caps)";
+  }
+
+  void add_cap(inodeno_t ino, uint64_t cap_id, inodeno_t pathbase, const string& path,
+	       int wanted, int issued,
+	       inodeno_t sr) {
+    caps[ino] = cap_reconnect_t(cap_id, pathbase, path, wanted, issued, sr);
+  }
+  void add_snaprealm(inodeno_t ino, snapid_t seq, inodeno_t parent) {
+    ceph_mds_snaprealm_reconnect r;
+    r.ino = ino;
+    r.seq = seq;
+    r.parent = parent;
+    realms.push_back(r);
+  }
+
+  void encode_payload(uint64_t features) {
+    data.clear();
+    if (features & CEPH_FEATURE_MDSENC) {
+      ::encode(caps, data);
+    } else if (features & CEPH_FEATURE_FLOCK) {
+      // encode with old cap_reconnect_t encoding
+      __u32 n = caps.size();
+      ::encode(n, data);
+      for (map::iterator p = caps.begin(); p != caps.end(); ++p) {
+	::encode(p->first, data);
+	p->second.encode_old(data);
+      }
+      header.version = 2;
+    } else {
+      // compat crap
+      header.version = 1;
+      map ocaps;
+      for (map::iterator p = caps.begin(); p != caps.end(); p++)
+	ocaps[p->first] = p->second;
+      ::encode(ocaps, data);
+    }
+    ::encode_nohead(realms, data);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = data.begin();
+    if (header.version >= 3) {
+      // new protocol
+      ::decode(caps, p);
+    } else if (header.version == 2) {
+      __u32 n;
+      ::decode(n, p);
+      inodeno_t ino;
+      while (n--) {
+	::decode(ino, p);
+	caps[ino].decode_old(p);
+      }
+    } else {
+      // compat crap
+      map ocaps;
+      ::decode(ocaps, p);
+      for (map::iterator q = ocaps.begin(); q != ocaps.end(); q++)
+	caps[q->first] = q->second;
+    }
+    while (!p.end()) {
+      realms.push_back(ceph_mds_snaprealm_reconnect());
+      ::decode(realms.back(), p);
+    }
+  }
+
+};
+
+
+#endif
diff --git a/ceph/src/messages/MClientReply.h b/ceph/src/messages/MClientReply.h
new file mode 100644
index 00000000..6507fa65
--- /dev/null
+++ b/ceph/src/messages/MClientReply.h
@@ -0,0 +1,278 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MCLIENTREPLY_H
+#define CEPH_MCLIENTREPLY_H
+
+#include "include/types.h"
+#include "MClientRequest.h"
+
+#include "msg/Message.h"
+#include "include/ceph_features.h"
+#include "common/errno.h"
+
+#include 
+using namespace std;
+
+/***
+ *
+ * MClientReply - container message for MDS reply to a client's MClientRequest
+ *
+ * key fields:
+ *  long tid - transaction id, so the client can match up with pending request
+ *  int result - error code, or fh if it was open
+ *
+ * for most requests:
+ *  trace is a vector of InodeStat's tracing from root to the file/dir/whatever
+ *  the operation referred to, so that the client can update it's info about what
+ *  metadata lives on what MDS.
+ *
+ * for readdir replies:
+ *  dir_contents is a vector of InodeStat*'s.  
+ * 
+ * that's mostly it, i think!
+ *
+ */
+
+
+struct LeaseStat {
+  // this matches ceph_mds_reply_lease
+  __u16 mask;
+  __u32 duration_ms;  
+  __u32 seq;
+
+  LeaseStat() : mask(0), duration_ms(0), seq(0) {}
+
+  void encode(bufferlist &bl) const {
+    ::encode(mask, bl);
+    ::encode(duration_ms, bl);
+    ::encode(seq, bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    ::decode(mask, bl);
+    ::decode(duration_ms, bl);
+    ::decode(seq, bl);
+  }
+};
+WRITE_CLASS_ENCODER(LeaseStat)
+
+inline ostream& operator<<(ostream& out, const LeaseStat& l) {
+  return out << "lease(mask " << l.mask << " dur " << l.duration_ms << ")";
+}
+
+struct DirStat {
+  // mds distribution hints
+  frag_t frag;
+  __s32 auth;
+  set<__s32> dist;
+  
+  DirStat() : auth(CDIR_AUTH_PARENT) {}
+  DirStat(bufferlist::iterator& p) {
+    decode(p);
+  }
+
+  void encode(bufferlist& bl) {
+    ::encode(frag, bl);
+    ::encode(auth, bl);
+    ::encode(dist, bl);
+  }
+  void decode(bufferlist::iterator& p) {
+    ::decode(frag, p);
+    ::decode(auth, p);
+    ::decode(dist, p);
+  }
+
+  // see CDir::encode_dirstat for encoder.
+};
+
+struct InodeStat {
+  vinodeno_t vino;
+  version_t version;
+  ceph_mds_reply_cap cap;
+
+  ceph_file_layout layout;
+  unsigned mode, uid, gid, nlink, rdev;
+  loff_t size, max_size;
+  version_t truncate_seq;
+  uint64_t truncate_size;
+  utime_t ctime, mtime, atime;
+  version_t time_warp_seq;
+  bufferlist inline_data;
+  version_t inline_version;
+
+  frag_info_t dirstat;
+  nest_info_t rstat;
+  
+  string  symlink;   // symlink content (if symlink)
+  fragtree_t dirfragtree;
+
+  version_t xattr_version;
+  bufferlist xattrbl;
+
+  ceph_dir_layout dir_layout;
+
+  //map xattrs;
+
+ public:
+  InodeStat() {}
+  InodeStat(bufferlist::iterator& p, uint64_t features) {
+    decode(p, features);
+  }
+
+  void decode(bufferlist::iterator &p, uint64_t features) {
+    struct ceph_mds_reply_inode e;
+    ::decode(e, p);
+    vino.ino = inodeno_t(e.ino);
+    vino.snapid = snapid_t(e.snapid);
+    version = e.version;
+    layout = e.layout;
+    cap = e.cap;
+    size = e.size;
+    max_size = e.max_size;
+    truncate_seq = e.truncate_seq;
+    truncate_size = e.truncate_size;
+    ctime.decode_timeval(&e.ctime);
+    mtime.decode_timeval(&e.mtime);
+    atime.decode_timeval(&e.atime);
+    time_warp_seq = e.time_warp_seq;
+    mode = e.mode;
+    uid = e.uid;
+    gid = e.gid;
+    nlink = e.nlink;
+    rdev = e.rdev;
+
+    dirstat.nfiles = e.files;
+    dirstat.nsubdirs = e.subdirs;
+
+    rstat.rctime.decode_timeval(&e.rctime);
+    rstat.rbytes = e.rbytes;
+    rstat.rfiles = e.rfiles;
+    rstat.rsubdirs = e.rsubdirs;
+
+    int n = e.fragtree.nsplits;
+    while (n) {
+      ceph_frag_tree_split s;
+      ::decode(s, p);
+      dirfragtree._splits[(__u32)s.frag] = s.by;
+      n--;
+    }
+    ::decode(symlink, p);
+    
+    if (features & CEPH_FEATURE_DIRLAYOUTHASH)
+      ::decode(dir_layout, p);
+    else
+      memset(&dir_layout, 0, sizeof(dir_layout));
+
+    xattr_version = e.xattr_version;
+    ::decode(xattrbl, p);
+
+    if (features & CEPH_FEATURE_MDS_INLINE_DATA) {
+      ::decode(inline_version, p);
+      ::decode(inline_data, p);
+    } else {
+      inline_version = CEPH_INLINE_NONE;
+    }
+  }
+  
+  // see CInode::encode_inodestat for encoder.
+};
+
+
+class MClientReply : public Message {
+  // reply data
+public:
+  struct ceph_mds_reply_head head;
+  bufferlist trace_bl;
+  bufferlist extra_bl;
+  bufferlist snapbl;
+
+ public:
+  int get_op() const { return head.op; }
+
+  void set_mdsmap_epoch(epoch_t e) { head.mdsmap_epoch = e; }
+  epoch_t get_mdsmap_epoch() const { return head.mdsmap_epoch; }
+
+  int get_result() const { return (__s32)(__u32)head.result; }
+
+  void set_result(int r) { head.result = r; }
+
+  void set_unsafe() { head.safe = 0; }
+
+  bool is_safe() const { return head.safe; }
+
+  MClientReply() : Message(CEPH_MSG_CLIENT_REPLY) {}
+  MClientReply(MClientRequest *req, int result = 0) : 
+    Message(CEPH_MSG_CLIENT_REPLY) {
+    memset(&head, 0, sizeof(head));
+    header.tid = req->get_tid();
+    head.op = req->get_op();
+    head.result = result;
+    head.safe = 1;
+  }
+private:
+  ~MClientReply() {}
+
+public:
+  const char *get_type_name() const { return "creply"; }
+  void print(ostream& o) const {
+    o << "client_reply(???:" << get_tid();
+    o << " = " << get_result();
+    if (get_result() <= 0) {
+      o << " " << cpp_strerror(get_result());
+    }
+    if (head.op & CEPH_MDS_OP_WRITE) {
+      if (head.safe)
+	o << " safe";
+      else
+	o << " unsafe";
+    }
+    o << ")";
+  }
+
+  // serialization
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(head, p);
+    ::decode(trace_bl, p);
+    ::decode(extra_bl, p);
+    ::decode(snapbl, p);
+    assert(p.end());
+  }
+  virtual void encode_payload(uint64_t features) {
+    ::encode(head, payload);
+    ::encode(trace_bl, payload);
+    ::encode(extra_bl, payload);
+    ::encode(snapbl, payload);
+  }
+
+
+  // dir contents
+  void set_extra_bl(bufferlist& bl) {
+    extra_bl.claim(bl);
+  }
+  bufferlist &get_extra_bl() {
+    return extra_bl;
+  }
+
+  // trace
+  void set_trace(bufferlist& bl) {
+    trace_bl.claim(bl);
+  }
+  bufferlist& get_trace_bl() {
+    return trace_bl;
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MClientRequest.h b/ceph/src/messages/MClientRequest.h
new file mode 100644
index 00000000..c436368b
--- /dev/null
+++ b/ceph/src/messages/MClientRequest.h
@@ -0,0 +1,204 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MCLIENTREQUEST_H
+#define CEPH_MCLIENTREQUEST_H
+
+/**
+ *
+ * MClientRequest - container for a client METADATA request.  created/sent by clients.  
+ *    can be forwarded around between MDS's.
+ *
+ *   int client - the originating client
+ *   long tid   - transaction id, unique among requests for that client.  probably just a counter!
+ *                -> the MDS passes the Request to the Reply constructor, so this always matches.
+ *  
+ *   int op - the metadata op code.  MDS_OP_RENAME, etc.
+ *   int caller_uid, _gid - guess
+ * 
+ * fixed size arguments are in a union.
+ * there's also a string argument, for e.g. symlink().
+ *  
+ */
+
+#include "msg/Message.h"
+#include "include/filepath.h"
+#include "mds/mdstypes.h"
+
+#include 
+#include 
+#include 
+#include 
+
+
+// metadata ops.
+
+class MClientRequest : public Message {
+public:
+  struct ceph_mds_request_head head;
+
+  struct Release {
+    mutable ceph_mds_request_release item;
+    string dname;
+
+    Release() : item(), dname() {}
+    Release(const ceph_mds_request_release& rel, string name) :
+      item(rel), dname(name) {}
+
+    void encode(bufferlist& bl) const {
+      item.dname_len = dname.length();
+      ::encode(item, bl);
+      ::encode_nohead(dname, bl);
+    }
+    void decode(bufferlist::iterator& bl) {
+      ::decode(item, bl);
+      ::decode_nohead(item.dname_len, dname, bl);
+    }
+  };
+  vector releases;
+
+  // path arguments
+  filepath path, path2;
+
+
+
+ public:
+  // cons
+  MClientRequest() : Message(CEPH_MSG_CLIENT_REQUEST) {}
+  MClientRequest(int op) : Message(CEPH_MSG_CLIENT_REQUEST) {
+    memset(&head, 0, sizeof(head));
+    head.op = op;
+  }
+private:
+  ~MClientRequest() {}
+
+public:
+  void set_mdsmap_epoch(epoch_t e) { head.mdsmap_epoch = e; }
+  epoch_t get_mdsmap_epoch() { return head.mdsmap_epoch; }
+
+  metareqid_t get_reqid() {
+    // FIXME: for now, assume clients always have 1 incarnation
+    return metareqid_t(get_orig_source(), header.tid); 
+  }
+
+  /*bool open_file_mode_is_readonly() {
+    return file_mode_is_readonly(ceph_flags_to_mode(head.args.open.flags));
+    }*/
+  bool may_write() {
+    return
+      (head.op & CEPH_MDS_OP_WRITE) || 
+      (head.op == CEPH_MDS_OP_OPEN && (head.args.open.flags & (O_CREAT|O_TRUNC)));
+  }
+
+  int get_flags() const {
+    return head.flags;
+  }
+  bool is_replay() {
+    return get_flags() & CEPH_MDS_FLAG_REPLAY;
+  }
+
+  // normal fields
+  void set_oldest_client_tid(ceph_tid_t t) { head.oldest_client_tid = t; }
+  void inc_num_fwd() { head.num_fwd = head.num_fwd + 1; }
+  void set_retry_attempt(int a) { head.num_retry = a; }
+  void set_filepath(const filepath& fp) { path = fp; }
+  void set_filepath2(const filepath& fp) { path2 = fp; }
+  void set_string2(const char *s) { path2.set_path(s, 0); }
+  void set_caller_uid(unsigned u) { head.caller_uid = u; }
+  void set_caller_gid(unsigned g) { head.caller_gid = g; }
+  void set_dentry_wanted() {
+    head.flags = head.flags | CEPH_MDS_FLAG_WANT_DENTRY;
+  }
+  void set_replayed_op() {
+    head.flags = head.flags | CEPH_MDS_FLAG_REPLAY;
+  }
+    
+  ceph_tid_t get_oldest_client_tid() const { return head.oldest_client_tid; }
+  int get_num_fwd() const { return head.num_fwd; }
+  int get_retry_attempt() const { return head.num_retry; }
+  int get_op() const { return head.op; }
+  unsigned get_caller_uid() const { return head.caller_uid; }
+  unsigned get_caller_gid() const { return head.caller_gid; }
+
+  const string& get_path() const { return path.get_path(); }
+  const filepath& get_filepath() const { return path; }
+  const string& get_path2() const { return path2.get_path(); }
+  const filepath& get_filepath2() const { return path2; }
+
+  int get_dentry_wanted() { return get_flags() & CEPH_MDS_FLAG_WANT_DENTRY; }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(head, p);
+    ::decode(path, p);
+    ::decode(path2, p);
+    ::decode_nohead(head.num_releases, releases, p);
+  }
+
+  void encode_payload(uint64_t features) {
+    head.num_releases = releases.size();
+    ::encode(head, payload);
+    ::encode(path, payload);
+    ::encode(path2, payload);
+    ::encode_nohead(releases, payload);
+  }
+
+  const char *get_type_name() const { return "creq"; }
+  void print(ostream& out) const {
+    out << "client_request(" << get_orig_source() 
+	<< ":" << get_tid() 
+	<< " " << ceph_mds_op_name(get_op());
+    if (head.op == CEPH_MDS_OP_GETATTR)
+      out << " " << ccap_string(head.args.getattr.mask);
+    if (head.op == CEPH_MDS_OP_SETATTR) {
+      if (head.args.setattr.mask & CEPH_SETATTR_MODE)
+	out << " mode=0" << std::oct << head.args.setattr.mode << std::dec;
+      if (head.args.setattr.mask & CEPH_SETATTR_UID)
+	out << " uid=" << head.args.setattr.uid;
+      if (head.args.setattr.mask & CEPH_SETATTR_GID)
+	out << " gid=" << head.args.setattr.gid;
+      if (head.args.setattr.mask & CEPH_SETATTR_SIZE)
+	out << " size=" << head.args.setattr.size;
+      if (head.args.setattr.mask & CEPH_SETATTR_MTIME)
+	out << " mtime=" << utime_t(head.args.setattr.mtime);
+      if (head.args.setattr.mask & CEPH_SETATTR_ATIME)
+	out << " atime=" << utime_t(head.args.setattr.atime);
+    }
+    if (head.op == CEPH_MDS_OP_SETFILELOCK ||
+	head.op == CEPH_MDS_OP_GETFILELOCK) {
+      out << "rule " << (int)head.args.filelock_change.rule
+	  << ", type " << (int)head.args.filelock_change.type
+	  << ", owner " << head.args.filelock_change.owner
+	  << ", pid " << head.args.filelock_change.pid
+	  << ", start " << head.args.filelock_change.start
+	  << ", length " << head.args.filelock_change.length
+	  << ", wait " << (int)head.args.filelock_change.wait;
+    }
+    //if (!get_filepath().empty()) 
+    out << " " << get_filepath();
+    if (!get_filepath2().empty())
+      out << " " << get_filepath2();
+    if (head.num_retry)
+      out << " RETRY=" << (int)head.num_retry;
+    if (get_flags() & CEPH_MDS_FLAG_REPLAY)
+      out << " REPLAY";
+    out << ")";
+  }
+
+};
+
+WRITE_CLASS_ENCODER(MClientRequest::Release)
+
+#endif
diff --git a/ceph/src/messages/MClientRequestForward.h b/ceph/src/messages/MClientRequestForward.h
new file mode 100644
index 00000000..976d9dcd
--- /dev/null
+++ b/ceph/src/messages/MClientRequestForward.h
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MCLIENTREQUESTFORWARD_H
+#define CEPH_MCLIENTREQUESTFORWARD_H
+
+class MClientRequestForward : public Message {
+  int32_t dest_mds;
+  int32_t num_fwd;
+  bool client_must_resend;
+
+ public:
+  MClientRequestForward()
+    : Message(CEPH_MSG_CLIENT_REQUEST_FORWARD),
+      dest_mds(-1), num_fwd(-1), client_must_resend(false) {}
+  MClientRequestForward(ceph_tid_t t, int dm, int nf, bool cmr) :
+    Message(CEPH_MSG_CLIENT_REQUEST_FORWARD),
+    dest_mds(dm), num_fwd(nf), client_must_resend(cmr) {
+    assert(client_must_resend);
+    header.tid = t;
+  }
+private:
+  ~MClientRequestForward() {}
+
+public:
+  int get_dest_mds() { return dest_mds; }
+  int get_num_fwd() { return num_fwd; }
+  bool must_resend() { return client_must_resend; }
+
+  const char *get_type_name() const { return "client_request_forward"; }
+  void print(ostream& o) const {
+    o << "client_request_forward(" << get_tid()
+      << " to mds." << dest_mds
+      << " num_fwd=" << num_fwd
+      << (client_must_resend ? " client_must_resend":"")
+      << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(dest_mds, payload);
+    ::encode(num_fwd, payload);
+    ::encode(client_must_resend, payload);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dest_mds, p);
+    ::decode(num_fwd, p);
+    ::decode(client_must_resend, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MClientSession.h b/ceph/src/messages/MClientSession.h
new file mode 100644
index 00000000..4b98a983
--- /dev/null
+++ b/ceph/src/messages/MClientSession.h
@@ -0,0 +1,67 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCLIENTSESSION_H
+#define CEPH_MCLIENTSESSION_H
+
+#include "msg/Message.h"
+
+class MClientSession : public Message {
+public:
+  ceph_mds_session_head head;
+
+  int get_op() const { return head.op; }
+  version_t get_seq() const { return head.seq; }
+  utime_t get_stamp() const { return utime_t(head.stamp); }
+  int get_max_caps() const { return head.max_caps; }
+  int get_max_leases() const { return head.max_leases; }
+
+  MClientSession() : Message(CEPH_MSG_CLIENT_SESSION) { }
+  MClientSession(int o, version_t s=0) : 
+    Message(CEPH_MSG_CLIENT_SESSION) {
+    memset(&head, 0, sizeof(head));
+    head.op = o;
+    head.seq = s;
+  }
+  MClientSession(int o, utime_t st) : 
+    Message(CEPH_MSG_CLIENT_SESSION) {
+    memset(&head, 0, sizeof(head));
+    head.op = o;
+    head.seq = 0;
+    st.encode_timeval(&head.stamp);
+  }
+private:
+  ~MClientSession() {}
+
+public:
+  const char *get_type_name() const { return "client_session"; }
+  void print(ostream& out) const {
+    out << "client_session(" << ceph_session_op_name(get_op());
+    if (get_seq())
+      out << " seq " << get_seq();
+    if (get_op() == CEPH_SESSION_RECALL_STATE)
+      out << " max_caps " << head.max_caps << " max_leases " << head.max_leases;
+    out << ")";
+  }
+
+  void decode_payload() { 
+    bufferlist::iterator p = payload.begin();
+    ::decode(head, p);
+  }
+  void encode_payload(uint64_t features) { 
+    ::encode(head, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MClientSnap.h b/ceph/src/messages/MClientSnap.h
new file mode 100644
index 00000000..7f7283e3
--- /dev/null
+++ b/ceph/src/messages/MClientSnap.h
@@ -0,0 +1,66 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCLIENTSNAP_H
+#define CEPH_MCLIENTSNAP_H
+
+#include "msg/Message.h"
+
+struct MClientSnap : public Message {
+  ceph_mds_snap_head head;
+  bufferlist bl;
+  
+  // (for split only)
+  vector split_inos;
+  vector split_realms;
+
+  MClientSnap(int o=0) : 
+    Message(CEPH_MSG_CLIENT_SNAP) {
+    memset(&head, 0, sizeof(head));
+    head.op = o;
+  }
+private:
+  ~MClientSnap() {}
+
+public:  
+  const char *get_type_name() const { return "client_snap"; }
+  void print(ostream& out) const {
+    out << "client_snap(" << ceph_snap_op_name(head.op);
+    if (head.split)
+      out << " split=" << inodeno_t(head.split);
+    out << " tracelen=" << bl.length();
+    out << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    head.num_split_inos = split_inos.size();
+    head.num_split_realms = split_realms.size();
+    head.trace_len = bl.length();
+    ::encode(head, payload);
+    ::encode_nohead(split_inos, payload);
+    ::encode_nohead(split_realms, payload);
+    ::encode_nohead(bl, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(head, p);
+    ::decode_nohead(head.num_split_inos, split_inos, p);
+    ::decode_nohead(head.num_split_realms, split_realms, p);
+    ::decode_nohead(head.trace_len, bl, p);
+    assert(p.end());
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MCommand.h b/ceph/src/messages/MCommand.h
new file mode 100644
index 00000000..e2fe0752
--- /dev/null
+++ b/ceph/src/messages/MCommand.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCOMMAND_H
+#define CEPH_MCOMMAND_H
+
+#include 
+#include 
+
+#include "msg/Message.h"
+
+class MCommand : public Message {
+ public:
+  uuid_d fsid;
+  std::vector cmd;
+
+  MCommand()
+    : Message(MSG_MON_COMMAND) {}
+  MCommand(uuid_d &f)
+    : Message(MSG_COMMAND),
+      fsid(f) { }
+
+private:
+  ~MCommand() {}
+
+public:  
+  const char *get_type_name() const { return "command"; }
+  void print(ostream& o) const {
+    o << "command(tid " << get_tid() << ": ";
+    for (unsigned i=0; i
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MCOMMANDREPLY_H
+#define CEPH_MCOMMANDREPLY_H
+
+#include "msg/Message.h"
+#include "MCommand.h"
+
+class MCommandReply : public Message {
+ public:
+  __s32 r;
+  string rs;
+  
+  MCommandReply()
+    : Message(MSG_COMMAND_REPLY) {}
+  MCommandReply(MCommand *m, int _r)
+    : Message(MSG_COMMAND_REPLY), r(_r) {
+    header.tid = m->get_tid();
+  }
+  MCommandReply(int _r, string s)
+    : Message(MSG_COMMAND_REPLY),
+      r(_r), rs(s) { }
+private:
+  ~MCommandReply() {}
+
+public:
+  const char *get_type_name() const { return "command_reply"; }
+  void print(ostream& o) const {
+    o << "command_reply(tid " << get_tid() << ": " << r << " " << rs << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    ::encode(r, payload);
+    ::encode(rs, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(r, p);
+    ::decode(rs, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MDentryLink.h b/ceph/src/messages/MDentryLink.h
new file mode 100644
index 00000000..b3515327
--- /dev/null
+++ b/ceph/src/messages/MDentryLink.h
@@ -0,0 +1,67 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDENTRYLINK_H
+#define CEPH_MDENTRYLINK_H
+
+class MDentryLink : public Message {
+  dirfrag_t subtree;
+  dirfrag_t dirfrag;
+  string dn;
+  bool is_primary;
+
+ public:
+  dirfrag_t get_subtree() { return subtree; }
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  string& get_dn() { return dn; }
+  bool get_is_primary() { return is_primary; }
+
+  bufferlist bl;
+
+  MDentryLink() :
+    Message(MSG_MDS_DENTRYLINK) { }
+  MDentryLink(dirfrag_t r, dirfrag_t df, string& n, bool p) :
+    Message(MSG_MDS_DENTRYLINK),
+    subtree(r),
+    dirfrag(df),
+    dn(n),
+    is_primary(p) {}
+private:
+  ~MDentryLink() {}
+
+public:
+  const char *get_type_name() const { return "dentry_link";}
+  void print(ostream& o) const {
+    o << "dentry_link(" << dirfrag << " " << dn << ")";
+  }
+  
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(subtree, p);
+    ::decode(dirfrag, p);
+    ::decode(dn, p);
+    ::decode(is_primary, p);
+    ::decode(bl, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(subtree, payload);
+    ::encode(dirfrag, payload);
+    ::encode(dn, payload);
+    ::encode(is_primary, payload);
+    ::encode(bl, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MDentryUnlink.h b/ceph/src/messages/MDentryUnlink.h
new file mode 100644
index 00000000..1cae33fb
--- /dev/null
+++ b/ceph/src/messages/MDentryUnlink.h
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDENTRYUNLINK_H
+#define CEPH_MDENTRYUNLINK_H
+
+class MDentryUnlink : public Message {
+  dirfrag_t dirfrag;
+  string dn;
+
+ public:
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  string& get_dn() { return dn; }
+
+  bufferlist straybl;
+
+  MDentryUnlink() :
+    Message(MSG_MDS_DENTRYUNLINK) { }
+  MDentryUnlink(dirfrag_t df, string& n) :
+    Message(MSG_MDS_DENTRYUNLINK),
+    dirfrag(df),
+    dn(n) {}
+private:
+  ~MDentryUnlink() {}
+
+public:
+  const char *get_type_name() const { return "dentry_unlink";}
+  void print(ostream& o) const {
+    o << "dentry_unlink(" << dirfrag << " " << dn << ")";
+  }
+  
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(dn, p);
+    ::decode(straybl, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+    ::encode(dn, payload);
+    ::encode(straybl, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MDirUpdate.h b/ceph/src/messages/MDirUpdate.h
new file mode 100644
index 00000000..445cd828
--- /dev/null
+++ b/ceph/src/messages/MDirUpdate.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDIRUPDATE_H
+#define CEPH_MDIRUPDATE_H
+
+#include "msg/Message.h"
+
+class MDirUpdate : public Message {
+  int32_t from_mds;
+  dirfrag_t dirfrag;
+  int32_t dir_rep;
+  int32_t discover;
+  set dir_rep_by;
+  filepath path;
+
+ public:
+  int get_source_mds() const { return from_mds; }
+  dirfrag_t get_dirfrag() const { return dirfrag; }
+  int get_dir_rep() const { return dir_rep; }
+  const set& get_dir_rep_by() const { return dir_rep_by; } 
+  bool should_discover() const { return discover > 0; }
+  const filepath& get_path() const { return path; }
+
+  void tried_discover() {
+    if (discover) discover--;
+  }
+
+  MDirUpdate() : Message(MSG_MDS_DIRUPDATE) {}
+  MDirUpdate(int f, 
+	     dirfrag_t dirfrag,
+             int dir_rep,
+             set& dir_rep_by,
+             filepath& path,
+             bool discover = false) :
+    Message(MSG_MDS_DIRUPDATE) {
+    this->from_mds = f;
+    this->dirfrag = dirfrag;
+    this->dir_rep = dir_rep;
+    this->dir_rep_by = dir_rep_by;
+    if (discover) this->discover = 5;
+    this->path = path;
+  }
+private:
+  ~MDirUpdate() {}
+
+public:
+  const char *get_type_name() const { return "dir_update"; }
+  void print(ostream& out) const {
+    out << "dir_update(" << get_dirfrag() << ")";
+  }
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(from_mds, p);
+    ::decode(dirfrag, p);
+    ::decode(dir_rep, p);
+    ::decode(discover, p);
+    ::decode(dir_rep_by, p);
+    ::decode(path, p);
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(from_mds, payload);
+    ::encode(dirfrag, payload);
+    ::encode(dir_rep, payload);
+    ::encode(discover, payload);
+    ::encode(dir_rep_by, payload);
+    ::encode(path, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MDiscover.h b/ceph/src/messages/MDiscover.h
new file mode 100644
index 00000000..296825cc
--- /dev/null
+++ b/ceph/src/messages/MDiscover.h
@@ -0,0 +1,103 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDISCOVER_H
+#define CEPH_MDISCOVER_H
+
+#include "msg/Message.h"
+#include "include/filepath.h"
+
+#include 
+#include 
+using namespace std;
+
+
+class MDiscover : public Message {
+  inodeno_t       base_ino;          // 1 -> root
+  frag_t          base_dir_frag;
+
+  snapid_t        snapid;
+  filepath        want;   // ... [/]need/this/stuff
+  inodeno_t       want_ino;
+
+  bool want_base_dir;
+  bool want_xlocked;
+
+ public:
+  inodeno_t get_base_ino() { return base_ino; }
+  frag_t    get_base_dir_frag() { return base_dir_frag; }
+  snapid_t  get_snapid() { return snapid; }
+
+  filepath& get_want() { return want; }
+  inodeno_t get_want_ino() { return want_ino; }
+  const string& get_dentry(int n) { return want[n]; }
+
+  bool wants_base_dir() { return want_base_dir; }
+  bool wants_xlocked() { return want_xlocked; }
+  
+  void set_base_dir_frag(frag_t f) { base_dir_frag = f; }
+
+  MDiscover() : Message(MSG_MDS_DISCOVER) { }
+  MDiscover(inodeno_t base_ino_,
+	    frag_t base_frag_,
+	    snapid_t s,
+            filepath& want_path_,
+	    inodeno_t want_ino_,
+            bool want_base_dir_ = true,
+	    bool discover_xlocks_ = false) :
+    Message(MSG_MDS_DISCOVER),
+    base_ino(base_ino_),
+    base_dir_frag(base_frag_),
+    snapid(s),
+    want(want_path_),
+    want_ino(want_ino_),
+    want_base_dir(want_base_dir_),
+    want_xlocked(discover_xlocks_) { }
+private:
+  ~MDiscover() {}
+
+public:
+  const char *get_type_name() const { return "Dis"; }
+  void print(ostream &out) const {
+    out << "discover(" << header.tid << " " << base_ino << "." << base_dir_frag
+	<< " " << want;
+    if (want_ino)
+      out << want_ino;
+    out << ")";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(base_ino, p);
+    ::decode(base_dir_frag, p);
+    ::decode(snapid, p);
+    ::decode(want, p);
+    ::decode(want_ino, p);
+    ::decode(want_base_dir, p);
+    ::decode(want_xlocked, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(base_ino, payload);
+    ::encode(base_dir_frag, payload);
+    ::encode(snapid, payload);
+    ::encode(want, payload);
+    ::encode(want_ino, payload);
+    ::encode(want_base_dir, payload);
+    ::encode(want_xlocked, payload);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MDiscoverReply.h b/ceph/src/messages/MDiscoverReply.h
new file mode 100644
index 00000000..361a517b
--- /dev/null
+++ b/ceph/src/messages/MDiscoverReply.h
@@ -0,0 +1,230 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MDISCOVERREPLY_H
+#define CEPH_MDISCOVERREPLY_H
+
+#include "msg/Message.h"
+#include "include/filepath.h"
+
+#include 
+#include 
+using namespace std;
+
+
+
+/**
+ * MDiscoverReply - return new replicas (of inodes, dirs, dentries)
+ *
+ * we group returned items by (dir, dentry, inode).  each
+ * item in each set shares an index (it's "depth").
+ *
+ * we can start and end with any type.
+ *   no_base_dir    = true if the first group has an inode but no dir
+ *   no_base_dentry = true if the first group has an inode but no dentry
+ * they are false if there is no returned data, ie the first group is empty.
+ *
+ * we also return errors:
+ *   error_flag_dn(string) - the specified dentry dne
+ *   error_flag_dir        - the last item wasn't a dir, so we couldn't continue.
+ *
+ * and sometimes,
+ *   dir_auth_hint         - where we think the dir auth is
+ *
+ * depth() gives us the number of depth units/indices for which we have 
+ * information.  this INCLUDES those for which we have errors but no data.
+ *
+ * see MDCache::handle_discover, handle_discover_reply.
+ *
+ *
+ * so basically, we get
+ *
+ *   dir den ino   i
+ *            x    0
+ *    x   x   x    1
+ * or
+ *        x   x    0
+ *    x   x   x    1
+ * or
+ *    x   x   x    0
+ *    x   x   x    1
+ * ...and trail off however we want.    
+ * 
+ * 
+ */
+
+class MDiscoverReply : public Message {
+
+  static const int HEAD_VERSION = 2;
+
+  // info about original request
+  inodeno_t base_ino;
+  frag_t base_dir_frag;  
+  bool wanted_base_dir;
+  bool wanted_xlocked;
+  inodeno_t wanted_ino;
+  snapid_t wanted_snapid;
+
+  // and the response
+  bool flag_error_dn;
+  bool flag_error_ino;
+  bool flag_error_dir;
+  string error_dentry;   // dentry that was not found (to trigger waiters on asker)
+  bool unsolicited;
+
+  __s32 dir_auth_hint;
+
+ public:
+  __u8 starts_with;
+  bufferlist trace;
+
+  enum { DIR, DENTRY, INODE };
+
+  // accessors
+  inodeno_t get_base_ino() { return base_ino; }
+  frag_t get_base_dir_frag() { return base_dir_frag; }
+  bool get_wanted_base_dir() { return wanted_base_dir; }
+  bool get_wanted_xlocked() { return wanted_xlocked; }
+  inodeno_t get_wanted_ino() { return wanted_ino; }
+  snapid_t get_wanted_snapid() { return wanted_snapid; }
+
+  bool is_flag_error_dn() { return flag_error_dn; }
+  bool is_flag_error_ino() { return flag_error_ino; }
+  bool is_flag_error_dir() { return flag_error_dir; }
+  string& get_error_dentry() { return error_dentry; }
+
+  int get_starts_with() { return starts_with; }
+
+  int get_dir_auth_hint() { return dir_auth_hint; }
+
+  bool is_unsolicited() { return unsolicited; }
+  void mark_unsolicited() { unsolicited = true; }
+
+  void set_base_dir_frag(frag_t df) { base_dir_frag = df; }
+
+  // cons
+  MDiscoverReply() : Message(MSG_MDS_DISCOVERREPLY, HEAD_VERSION) { }
+  MDiscoverReply(MDiscover *dis) :
+    Message(MSG_MDS_DISCOVERREPLY, HEAD_VERSION),
+    base_ino(dis->get_base_ino()),
+    base_dir_frag(dis->get_base_dir_frag()),
+    wanted_base_dir(dis->wants_base_dir()),
+    wanted_xlocked(dis->wants_xlocked()),
+    wanted_ino(dis->get_want_ino()),
+    wanted_snapid(dis->get_snapid()),
+    flag_error_dn(false),
+    flag_error_ino(false),
+    flag_error_dir(false),
+    unsolicited(false),
+    dir_auth_hint(CDIR_AUTH_UNKNOWN),
+    starts_with(DIR)
+  {
+    header.tid = dis->get_tid();
+  }
+  MDiscoverReply(dirfrag_t df) :
+    Message(MSG_MDS_DISCOVERREPLY, HEAD_VERSION),
+    base_ino(df.ino),
+    base_dir_frag(df.frag),
+    wanted_base_dir(false),
+    wanted_xlocked(false),
+    wanted_ino(inodeno_t()),
+    wanted_snapid(CEPH_NOSNAP),
+    flag_error_dn(false),
+    flag_error_ino(false),
+    flag_error_dir(false),
+    unsolicited(false),
+    dir_auth_hint(CDIR_AUTH_UNKNOWN),
+    starts_with(DIR)
+  {
+    header.tid = 0;
+  }
+private:
+  ~MDiscoverReply() {}
+
+public:
+  const char *get_type_name() const { return "discover_reply"; }
+  void print(ostream& out) const {
+    out << "discover_reply(" << header.tid << " " << base_ino << ")";
+  }
+  
+  // builders
+  bool is_empty() {
+    return trace.length() == 0 &&
+      !flag_error_dn &&
+      !flag_error_ino &&
+      !flag_error_dir &&
+      dir_auth_hint == CDIR_AUTH_UNKNOWN;
+  }
+
+  //  void set_flag_forward() { flag_forward = true; }
+  void set_flag_error_dn(const string& dn) { 
+    flag_error_dn = true; 
+    error_dentry = dn; 
+  }
+  void set_flag_error_ino() {
+    flag_error_ino = true;
+  }
+  void set_flag_error_dir() { 
+    flag_error_dir = true; 
+  }
+  void set_dir_auth_hint(int a) {
+    dir_auth_hint = a;
+  }
+  void set_error_dentry(const string& dn) {
+    error_dentry = dn;
+  }
+
+
+  // ...
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(base_ino, p);
+    ::decode(base_dir_frag, p);
+    ::decode(wanted_base_dir, p);
+    ::decode(wanted_xlocked, p);
+    ::decode(wanted_snapid, p);
+    ::decode(flag_error_dn, p);
+    ::decode(flag_error_ino, p);
+    ::decode(flag_error_dir, p);
+    ::decode(error_dentry, p);
+    ::decode(dir_auth_hint, p);
+    ::decode(unsolicited, p);
+
+    ::decode(starts_with, p);
+    ::decode(trace, p);
+    if (header.version >= 2)
+      ::decode(wanted_ino, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(base_ino, payload);
+    ::encode(base_dir_frag, payload);
+    ::encode(wanted_base_dir, payload);
+    ::encode(wanted_xlocked, payload);
+    ::encode(wanted_snapid, payload);
+    ::encode(flag_error_dn, payload);
+    ::encode(flag_error_ino, payload);
+    ::encode(flag_error_dir, payload);
+    ::encode(error_dentry, payload);
+    ::encode(dir_auth_hint, payload);
+    ::encode(unsolicited, payload);
+
+    ::encode(starts_with, payload);
+    ::encode(trace, payload);
+    ::encode(wanted_ino, payload);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MExportCaps.h b/ceph/src/messages/MExportCaps.h
new file mode 100644
index 00000000..dcb2298e
--- /dev/null
+++ b/ceph/src/messages/MExportCaps.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MEXPORTCAPS_H
+#define CEPH_MEXPORTCAPS_H
+
+#include "msg/Message.h"
+
+
+class MExportCaps : public Message {
+ public:  
+  inodeno_t ino;
+  bufferlist cap_bl;
+  map client_map;
+
+  MExportCaps() :
+    Message(MSG_MDS_EXPORTCAPS) {}
+private:
+  ~MExportCaps() {}
+
+public:
+  const char *get_type_name() const { return "export_caps"; }
+  void print(ostream& o) const {
+    o << "export_caps(" << ino << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(ino, payload);
+    ::encode(cap_bl, payload);
+    ::encode(client_map, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(ino, p);
+    ::decode(cap_bl, p);
+    ::decode(client_map, p);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MExportCapsAck.h b/ceph/src/messages/MExportCapsAck.h
new file mode 100644
index 00000000..b8fcf1fa
--- /dev/null
+++ b/ceph/src/messages/MExportCapsAck.h
@@ -0,0 +1,49 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MEXPORTCAPSACK_H
+#define CEPH_MEXPORTCAPSACK_H
+
+#include "msg/Message.h"
+
+
+class MExportCapsAck : public Message {
+ public:  
+  inodeno_t ino;
+
+  MExportCapsAck() :
+    Message(MSG_MDS_EXPORTCAPSACK) {}
+  MExportCapsAck(inodeno_t i) :
+    Message(MSG_MDS_EXPORTCAPSACK), ino(i) {}
+private:
+  ~MExportCapsAck() {}
+
+public:
+  const char *get_type_name() const { return "export_caps_ack"; }
+  void print(ostream& o) const {
+    o << "export_caps_ack(" << ino << ")";
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(ino, payload);
+  }
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(ino, p);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDir.h b/ceph/src/messages/MExportDir.h
new file mode 100644
index 00000000..2ff85d4f
--- /dev/null
+++ b/ceph/src/messages/MExportDir.h
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MEXPORTDIR_H
+#define CEPH_MEXPORTDIR_H
+
+#include "msg/Message.h"
+
+
+class MExportDir : public Message {
+ public:  
+  dirfrag_t dirfrag;
+  bufferlist export_data;
+  vector bounds;
+  bufferlist client_map;
+
+  MExportDir() : Message(MSG_MDS_EXPORTDIR) {}
+  MExportDir(dirfrag_t df, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIR), dirfrag(df) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDir() {}
+
+public:
+  const char *get_type_name() const { return "Ex"; }
+  void print(ostream& o) const {
+    o << "export(" << dirfrag << ")";
+  }
+
+  void add_export(dirfrag_t df) { 
+    bounds.push_back(df); 
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+    ::encode(bounds, payload);
+    ::encode(export_data, payload);
+    ::encode(client_map, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(bounds, p);
+    ::decode(export_data, p);
+    ::decode(client_map, p);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirAck.h b/ceph/src/messages/MExportDirAck.h
new file mode 100644
index 00000000..eb23789e
--- /dev/null
+++ b/ceph/src/messages/MExportDirAck.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRACK_H
+#define CEPH_MEXPORTDIRACK_H
+
+#include "MExportDir.h"
+
+class MExportDirAck : public Message {
+public:
+  dirfrag_t dirfrag;
+  bufferlist imported_caps;
+
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  
+  MExportDirAck() : Message(MSG_MDS_EXPORTDIRACK) {}
+  MExportDirAck(dirfrag_t df, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIRACK), dirfrag(df) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirAck() {}
+
+public:
+  const char *get_type_name() const { return "ExAck"; }
+    void print(ostream& o) const {
+    o << "export_ack(" << dirfrag << ")";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(imported_caps, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+    ::encode(imported_caps, payload);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirCancel.h b/ceph/src/messages/MExportDirCancel.h
new file mode 100644
index 00000000..b3b2c582
--- /dev/null
+++ b/ceph/src/messages/MExportDirCancel.h
@@ -0,0 +1,50 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRCANCEL_H
+#define CEPH_MEXPORTDIRCANCEL_H
+
+#include "msg/Message.h"
+#include "include/types.h"
+
+class MExportDirCancel : public Message {
+  dirfrag_t dirfrag;
+
+ public:
+  dirfrag_t get_dirfrag() { return dirfrag; }
+
+  MExportDirCancel() : Message(MSG_MDS_EXPORTDIRCANCEL) {}
+  MExportDirCancel(dirfrag_t df, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIRCANCEL), dirfrag(df) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirCancel() {}
+
+public:
+  const char *get_type_name() const { return "ExCancel"; }
+  void print(ostream& o) const {
+    o << "export_cancel(" << dirfrag << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirDiscover.h b/ceph/src/messages/MExportDirDiscover.h
new file mode 100644
index 00000000..0b29daf1
--- /dev/null
+++ b/ceph/src/messages/MExportDirDiscover.h
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRDISCOVER_H
+#define CEPH_MEXPORTDIRDISCOVER_H
+
+#include "msg/Message.h"
+#include "include/types.h"
+
+class MExportDirDiscover : public Message {
+  int32_t from;
+  dirfrag_t dirfrag;
+  filepath path;
+
+ public:
+  int get_source_mds() { return from; }
+  inodeno_t get_ino() { return dirfrag.ino; }
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  filepath& get_path() { return path; }
+
+  bool started;
+
+  MExportDirDiscover() :     
+    Message(MSG_MDS_EXPORTDIRDISCOVER),
+    started(false) { }
+  MExportDirDiscover(dirfrag_t df, filepath& p, int f, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIRDISCOVER),
+    from(f), dirfrag(df), path(p), started(false) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirDiscover() {}
+
+public:
+  const char *get_type_name() const { return "ExDis"; }
+  void print(ostream& o) const {
+    o << "export_discover(" << dirfrag << " " << path << ")";
+  }
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(from, p);
+    ::decode(dirfrag, p);
+    ::decode(path, p);
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(from, payload);
+    ::encode(dirfrag, payload);
+    ::encode(path, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirDiscoverAck.h b/ceph/src/messages/MExportDirDiscoverAck.h
new file mode 100644
index 00000000..8e20be60
--- /dev/null
+++ b/ceph/src/messages/MExportDirDiscoverAck.h
@@ -0,0 +1,60 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRDISCOVERACK_H
+#define CEPH_MEXPORTDIRDISCOVERACK_H
+
+#include "msg/Message.h"
+#include "include/types.h"
+
+class MExportDirDiscoverAck : public Message {
+  dirfrag_t dirfrag;
+  bool success;
+
+ public:
+  inodeno_t get_ino() { return dirfrag.ino; }
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  bool is_success() { return success; }
+
+  MExportDirDiscoverAck() : Message(MSG_MDS_EXPORTDIRDISCOVERACK) {}
+  MExportDirDiscoverAck(dirfrag_t df, uint64_t tid, bool s=true) :
+    Message(MSG_MDS_EXPORTDIRDISCOVERACK),
+    dirfrag(df), success(s) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirDiscoverAck() {}
+
+public:
+  const char *get_type_name() const { return "ExDisA"; }
+  void print(ostream& o) const {
+    o << "export_discover_ack(" << dirfrag;
+    if (success) 
+      o << " success)";
+    else
+      o << " failure)";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(success, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+    ::encode(success, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirFinish.h b/ceph/src/messages/MExportDirFinish.h
new file mode 100644
index 00000000..dd78dda5
--- /dev/null
+++ b/ceph/src/messages/MExportDirFinish.h
@@ -0,0 +1,54 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRFINISH_H
+#define CEPH_MEXPORTDIRFINISH_H
+
+#include "msg/Message.h"
+
+class MExportDirFinish : public Message {
+  dirfrag_t dirfrag;
+  bool last;
+
+ public:
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  bool is_last() { return last; }
+  
+  MExportDirFinish() {}
+  MExportDirFinish(dirfrag_t df, bool l, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIRFINISH), dirfrag(df), last(l) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirFinish() {}
+
+public:
+  const char *get_type_name() const { return "ExFin"; }
+  void print(ostream& o) const {
+    o << "export_finish(" << dirfrag << (last ? " last" : "") << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+    ::encode(last, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(last, p);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirNotify.h b/ceph/src/messages/MExportDirNotify.h
new file mode 100644
index 00000000..745cf95f
--- /dev/null
+++ b/ceph/src/messages/MExportDirNotify.h
@@ -0,0 +1,81 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRNOTIFY_H
+#define CEPH_MEXPORTDIRNOTIFY_H
+
+#include "msg/Message.h"
+#include 
+using namespace std;
+
+class MExportDirNotify : public Message {
+  dirfrag_t base;
+  bool ack;
+  pair<__s32,__s32> old_auth, new_auth;
+  list bounds;  // bounds; these dirs are _not_ included (tho the dirfragdes are)
+
+ public:
+  dirfrag_t get_dirfrag() { return base; }
+  pair<__s32,__s32> get_old_auth() { return old_auth; }
+  pair<__s32,__s32> get_new_auth() { return new_auth; }
+  bool wants_ack() { return ack; }
+  list& get_bounds() { return bounds; }
+
+  MExportDirNotify() {}
+  MExportDirNotify(dirfrag_t i, uint64_t tid, bool a, pair<__s32,__s32> oa, pair<__s32,__s32> na) :
+    Message(MSG_MDS_EXPORTDIRNOTIFY),
+    base(i), ack(a), old_auth(oa), new_auth(na) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirNotify() {}
+
+public:
+  const char *get_type_name() const { return "ExNot"; }
+  void print(ostream& o) const {
+    o << "export_notify(" << base;
+    o << " " << old_auth << " -> " << new_auth;
+    if (ack) 
+      o << " ack)";
+    else
+      o << " no ack)";
+  }
+  
+  void copy_bounds(list& ex) {
+    this->bounds = ex;
+  }
+  void copy_bounds(set& ex) {
+    for (set::iterator i = ex.begin();
+	 i != ex.end(); ++i)
+      bounds.push_back(*i);
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(base, payload);
+    ::encode(ack, payload);
+    ::encode(old_auth, payload);
+    ::encode(new_auth, payload);
+    ::encode(bounds, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(base, p);
+    ::decode(ack, p);
+    ::decode(old_auth, p);
+    ::decode(new_auth, p);
+    ::decode(bounds, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirNotifyAck.h b/ceph/src/messages/MExportDirNotifyAck.h
new file mode 100644
index 00000000..7b52c4f6
--- /dev/null
+++ b/ceph/src/messages/MExportDirNotifyAck.h
@@ -0,0 +1,52 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRNOTIFYACK_H
+#define CEPH_MEXPORTDIRNOTIFYACK_H
+
+#include "msg/Message.h"
+#include 
+using namespace std;
+
+class MExportDirNotifyAck : public Message {
+  dirfrag_t dirfrag;
+
+ public:
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  
+  MExportDirNotifyAck() {}
+  MExportDirNotifyAck(dirfrag_t df, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIRNOTIFYACK), dirfrag(df) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirNotifyAck() {}
+
+public:
+  const char *get_type_name() const { return "ExNotA"; }
+  void print(ostream& o) const {
+    o << "export_notify_ack(" << dirfrag << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+  }
+  
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirPrep.h b/ceph/src/messages/MExportDirPrep.h
new file mode 100644
index 00000000..791a98ab
--- /dev/null
+++ b/ceph/src/messages/MExportDirPrep.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MEXPORTDIRPREP_H
+#define CEPH_MEXPORTDIRPREP_H
+
+#include "msg/Message.h"
+#include "include/types.h"
+
+class MExportDirPrep : public Message {
+  dirfrag_t dirfrag;
+ public:
+  bufferlist basedir;
+  list bounds;
+  list traces;
+private:
+  set<__s32> bystanders;
+  bool b_did_assim;
+
+public:
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  list& get_bounds() { return bounds; }
+  set<__s32> &get_bystanders() { return bystanders; }
+
+  bool did_assim() { return b_did_assim; }
+  void mark_assim() { b_did_assim = true; }
+
+  MExportDirPrep() {
+    b_did_assim = false;
+  }
+  MExportDirPrep(dirfrag_t df, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIRPREP),
+    dirfrag(df), b_did_assim(false) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirPrep() {}
+
+public:
+  const char *get_type_name() const { return "ExP"; }
+  void print(ostream& o) const {
+    o << "export_prep(" << dirfrag << ")";
+  }
+
+  void add_bound(dirfrag_t df) {
+    bounds.push_back( df );
+  }
+  void add_trace(bufferlist& bl) {
+    traces.push_back(bl);
+  }
+  void add_bystander(int who) {
+    bystanders.insert(who);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(basedir, p);
+    ::decode(bounds, p);
+    ::decode(traces, p);
+    ::decode(bystanders, p);
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+    ::encode(basedir, payload);
+    ::encode(bounds, payload);
+    ::encode(traces, payload);
+    ::encode(bystanders, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MExportDirPrepAck.h b/ceph/src/messages/MExportDirPrepAck.h
new file mode 100644
index 00000000..319194da
--- /dev/null
+++ b/ceph/src/messages/MExportDirPrepAck.h
@@ -0,0 +1,54 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MEXPORTDIRPREPACK_H
+#define CEPH_MEXPORTDIRPREPACK_H
+
+#include "msg/Message.h"
+#include "include/types.h"
+
+class MExportDirPrepAck : public Message {
+  dirfrag_t dirfrag;
+  bool success;
+
+ public:
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  
+  MExportDirPrepAck() {}
+  MExportDirPrepAck(dirfrag_t df, bool s, uint64_t tid) :
+    Message(MSG_MDS_EXPORTDIRPREPACK), dirfrag(df), success(s) {
+    set_tid(tid);
+  }
+private:
+  ~MExportDirPrepAck() {}
+
+public:  
+  bool is_success() { return success; }
+  const char *get_type_name() const { return "ExPAck"; }
+  void print(ostream& o) const {
+    o << "export_prep_ack(" << dirfrag << (success ? " success)" : " fail)");
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(success, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(dirfrag, payload);
+    ::encode(success, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MForward.h b/ceph/src/messages/MForward.h
new file mode 100644
index 00000000..6a0e2bf7
--- /dev/null
+++ b/ceph/src/messages/MForward.h
@@ -0,0 +1,91 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2010 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ * Client requests often need to get forwarded from some monitor
+ * to the leader. This class encapsulates the original message
+ * along with the client's caps so the leader can do proper permissions
+ * checking.
+ */
+
+#ifndef CEPH_MFORWARD_H
+#define CEPH_MFORWARD_H
+
+#include "msg/Message.h"
+#include "mon/MonCap.h"
+#include "include/encoding.h"
+
+struct MForward : public Message {
+  uint64_t tid;
+  PaxosServiceMessage *msg;
+  entity_inst_t client;
+  MonCap client_caps;
+  uint64_t con_features;
+
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+  MForward() : Message(MSG_FORWARD, HEAD_VERSION, COMPAT_VERSION),
+               tid(0), msg(NULL), con_features(0) {}
+  //the message needs to have caps filled in!
+  MForward(uint64_t t, PaxosServiceMessage *m, uint64_t feat) :
+    Message(MSG_FORWARD, HEAD_VERSION, COMPAT_VERSION),
+    tid(t), msg(m) {
+    client = m->get_source_inst();
+    client_caps = m->get_session()->caps;
+    con_features = feat;
+  }
+  MForward(uint64_t t, PaxosServiceMessage *m, uint64_t feat,
+	   const MonCap& caps) :
+    Message(MSG_FORWARD, HEAD_VERSION, COMPAT_VERSION),
+    tid(t), msg(m), client_caps(caps) {
+    client = m->get_source_inst();
+    con_features = feat;
+  }
+private:
+  ~MForward() {
+    if (msg) msg->put();
+  }
+
+public:
+  void encode_payload(uint64_t features) {
+    ::encode(tid, payload);
+    ::encode(client, payload);
+    ::encode(client_caps, payload, features);
+    encode_message(msg, features, payload);
+    ::encode(con_features, payload);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(tid, p);
+    ::decode(client, p);
+    ::decode(client_caps, p);
+    msg = (PaxosServiceMessage *)decode_message(NULL, p);
+    if (header.version >= 2) {
+      ::decode(con_features, p);
+    } else {
+      con_features = 0;
+    }
+
+  }
+
+  const char *get_type_name() const { return "forward"; }
+  void print(ostream& o) const {
+    if (msg)
+      o << "forward(" << *msg << " caps " << client_caps
+	<< " tid " << tid
+        << " con_features " << con_features << ") to leader";
+    else o << "forward(??? ) to leader";
+  }
+};
+  
+#endif
diff --git a/ceph/src/messages/MGenericMessage.h b/ceph/src/messages/MGenericMessage.h
new file mode 100644
index 00000000..6f326aa8
--- /dev/null
+++ b/ceph/src/messages/MGenericMessage.h
@@ -0,0 +1,39 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MGENERICMESSAGE_H
+#define CEPH_MGENERICMESSAGE_H
+
+#include "msg/Message.h"
+
+class MGenericMessage : public Message {
+  char tname[20];
+  //long pcid;
+
+ public:
+  MGenericMessage(int t=0) : Message(t) { 
+    snprintf(tname, sizeof(tname), "generic%d", get_type());
+  }
+
+  //void set_pcid(long pcid) { this->pcid = pcid; }
+  //long get_pcid() { return pcid; }
+
+  const char *get_type_name() const { return tname; }
+
+  void decode_payload() { }
+  void encode_payload(uint64_t features) { }
+};
+
+#endif
diff --git a/ceph/src/messages/MGetPoolStats.h b/ceph/src/messages/MGetPoolStats.h
new file mode 100644
index 00000000..d004e296
--- /dev/null
+++ b/ceph/src/messages/MGetPoolStats.h
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MGETPOOLSTATS_H
+#define CEPH_MGETPOOLSTATS_H
+
+#include 
+
+#include "messages/PaxosServiceMessage.h"
+
+class MGetPoolStats : public PaxosServiceMessage {
+public:
+  uuid_d fsid;
+  list pools;
+
+  MGetPoolStats() : PaxosServiceMessage(MSG_GETPOOLSTATS, 0) {}
+  MGetPoolStats(const uuid_d& f, ceph_tid_t t, list& ls, version_t l) :
+    PaxosServiceMessage(MSG_GETPOOLSTATS, l),
+    fsid(f), pools(ls) {
+    set_tid(t);
+  }
+
+private:
+  ~MGetPoolStats() {}
+
+public:
+  const char *get_type_name() const { return "getpoolstats"; }
+  void print(ostream& out) const {
+    out << "getpoolstats(" << get_tid() << " " << pools << " v" << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(pools, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(pools, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MGetPoolStatsReply.h b/ceph/src/messages/MGetPoolStatsReply.h
new file mode 100644
index 00000000..07b99cff
--- /dev/null
+++ b/ceph/src/messages/MGetPoolStatsReply.h
@@ -0,0 +1,55 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MGETPOOLSTATSREPLY_H
+#define CEPH_MGETPOOLSTATSREPLY_H
+
+#include 
+
+class MGetPoolStatsReply : public PaxosServiceMessage {
+public:
+  uuid_d fsid;
+  map pool_stats;
+
+  MGetPoolStatsReply() : PaxosServiceMessage(MSG_GETPOOLSTATSREPLY, 0) {}
+  MGetPoolStatsReply(uuid_d& f, ceph_tid_t t, version_t v) :
+    PaxosServiceMessage(MSG_GETPOOLSTATSREPLY, v),
+    fsid(f) {
+    set_tid(t);
+  }
+
+private:
+  ~MGetPoolStatsReply() {}
+
+public:
+  const char *get_type_name() const { return "getpoolstats"; }
+  void print(ostream& out) const {
+    out << "getpoolstatsreply(" << get_tid() << " v" << version <<  ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(pool_stats, payload, features);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(pool_stats, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MHeartbeat.h b/ceph/src/messages/MHeartbeat.h
new file mode 100644
index 00000000..b6cfcfaf
--- /dev/null
+++ b/ceph/src/messages/MHeartbeat.h
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MHEARTBEAT_H
+#define CEPH_MHEARTBEAT_H
+
+#include "include/types.h"
+#include "msg/Message.h"
+
+class MHeartbeat : public Message {
+  mds_load_t load;
+  __s32        beat;
+  map<__s32, float> import_map;
+
+ public:
+  mds_load_t& get_load() { return load; }
+  int get_beat() { return beat; }
+
+  map<__s32, float>& get_import_map() {
+    return import_map;
+  }
+
+  MHeartbeat()
+    : Message(MSG_MDS_HEARTBEAT), load(utime_t()) { }
+  MHeartbeat(mds_load_t& load, int beat)
+    : Message(MSG_MDS_HEARTBEAT),
+      load(load) {
+    this->beat = beat;
+  }
+private:
+  ~MHeartbeat() {}
+
+public:
+  const char *get_type_name() const { return "HB"; }
+
+  void encode_payload(uint64_t features) {
+    ::encode(load, payload);
+    ::encode(beat, payload);
+    ::encode(import_map, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    utime_t now(ceph_clock_now(NULL));
+    ::decode(load, now, p);
+    ::decode(beat, p);
+    ::decode(import_map, p);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MInodeFileCaps.h b/ceph/src/messages/MInodeFileCaps.h
new file mode 100644
index 00000000..bc4db2fc
--- /dev/null
+++ b/ceph/src/messages/MInodeFileCaps.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MINODEFILECAPS_H
+#define CEPH_MINODEFILECAPS_H
+
+class MInodeFileCaps : public Message {
+  inodeno_t ino;
+  __u32     caps;
+
+ public:
+  inodeno_t get_ino() { return ino; }
+  int       get_caps() { return caps; }
+
+  MInodeFileCaps() : Message(MSG_MDS_INODEFILECAPS) {}
+  MInodeFileCaps(inodeno_t ino, int caps) :
+    Message(MSG_MDS_INODEFILECAPS) {
+    this->ino = ino;
+    this->caps = caps;
+  }
+private:
+  ~MInodeFileCaps() {}
+
+public:
+  const char *get_type_name() const { return "inode_file_caps";}
+  void print(ostream& out) const {
+    out << "inode_file_caps(" << ino << " " << ccap_string(caps) << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    ::encode(ino, payload);
+    ::encode(caps, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(ino, p);
+    ::decode(caps, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MLock.h b/ceph/src/messages/MLock.h
new file mode 100644
index 00000000..5dd4a858
--- /dev/null
+++ b/ceph/src/messages/MLock.h
@@ -0,0 +1,96 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MLOCK_H
+#define CEPH_MLOCK_H
+
+#include "msg/Message.h"
+#include "mds/locks.h"
+#include "mds/SimpleLock.h"
+
+class MLock : public Message {
+  int32_t     action;  // action type
+  int32_t     asker;  // who is initiating this request
+  metareqid_t reqid;  // for remote lock requests
+  
+  __u16      lock_type;  // lock object type
+  MDSCacheObjectInfo object_info;  
+  
+  bufferlist lockdata;  // and possibly some data
+  
+public:
+  bufferlist& get_data() { return lockdata; }
+  int get_asker() { return asker; }
+  int get_action() { return action; }
+  metareqid_t get_reqid() { return reqid; }
+  
+  int get_lock_type() { return lock_type; }
+  MDSCacheObjectInfo &get_object_info() { return object_info; }
+  
+  MLock() : Message(MSG_MDS_LOCK) {}
+  MLock(int ac, int as) :
+    Message(MSG_MDS_LOCK),
+    action(ac), asker(as),
+    lock_type(0) { }
+  MLock(SimpleLock *lock, int ac, int as) :
+    Message(MSG_MDS_LOCK),
+    action(ac), asker(as),
+    lock_type(lock->get_type()) {
+    lock->get_parent()->set_object_info(object_info);
+  }
+  MLock(SimpleLock *lock, int ac, int as, bufferlist& bl) :
+    Message(MSG_MDS_LOCK),
+    action(ac), asker(as), lock_type(lock->get_type()) {
+    lock->get_parent()->set_object_info(object_info);
+    lockdata.claim(bl);
+  }
+private:
+  ~MLock() {}
+  
+public:
+  const char *get_type_name() const { return "ILock"; }
+  void print(ostream& out) const {
+    out << "lock(a=" << get_lock_action_name(action)
+	<< " " << get_lock_type_name(lock_type)
+	<< " " << object_info
+	<< ")";
+  }
+  
+  void set_reqid(metareqid_t ri) { reqid = ri; }
+  void set_data(const bufferlist& lockdata) {
+    this->lockdata = lockdata;
+  }
+  
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(asker, p);
+    ::decode(action, p);
+    ::decode(reqid, p);
+    ::decode(lock_type, p);
+    ::decode(object_info, p);
+    ::decode(lockdata, p);
+  }
+  virtual void encode_payload(uint64_t features) {
+    ::encode(asker, payload);
+    ::encode(action, payload);
+    ::encode(reqid, payload);
+    ::encode(lock_type, payload);
+    ::encode(object_info, payload);
+    ::encode(lockdata, payload);
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MLog.h b/ceph/src/messages/MLog.h
new file mode 100644
index 00000000..c71f56bd
--- /dev/null
+++ b/ceph/src/messages/MLog.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MLOG_H
+#define CEPH_MLOG_H
+
+#include "common/LogEntry.h"
+#include "messages/PaxosServiceMessage.h"
+
+#include 
+#include 
+
+class MLog : public PaxosServiceMessage {
+public:
+  uuid_d fsid;
+  std::deque entries;
+  
+  MLog() : PaxosServiceMessage(MSG_LOG, 0) {}
+  MLog(const uuid_d& f, const std::deque& e)
+    : PaxosServiceMessage(MSG_LOG, 0), fsid(f), entries(e) { }
+  MLog(const uuid_d& f) : PaxosServiceMessage(MSG_LOG, 0), fsid(f) { }
+private:
+  ~MLog() {}
+
+public:
+  const char *get_type_name() const { return "log"; }
+  void print(ostream& out) const {
+    out << "log(";
+    if (entries.size())
+      out << entries.size() << " entries";
+    out << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(entries, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(entries, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MLogAck.h b/ceph/src/messages/MLogAck.h
new file mode 100644
index 00000000..8022e7ce
--- /dev/null
+++ b/ceph/src/messages/MLogAck.h
@@ -0,0 +1,47 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MLOGACK_H
+#define CEPH_MLOGACK_H
+
+#include 
+
+class MLogAck : public Message {
+public:
+  uuid_d fsid;
+  version_t last;
+  
+  MLogAck() : Message(MSG_LOGACK) {}
+  MLogAck(uuid_d& f, version_t l) : Message(MSG_LOGACK), fsid(f), last(l) {}
+private:
+  ~MLogAck() {}
+
+public:
+  const char *get_type_name() const { return "log_ack"; }
+  void print(ostream& out) const {
+    out << "log(last " << last << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(fsid, payload);
+    ::encode(last, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(fsid, p);
+    ::decode(last, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSBeacon.h b/ceph/src/messages/MMDSBeacon.h
new file mode 100644
index 00000000..64d442bf
--- /dev/null
+++ b/ceph/src/messages/MMDSBeacon.h
@@ -0,0 +1,100 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMDSBEACON_H
+#define CEPH_MMDSBEACON_H
+
+#include "messages/PaxosServiceMessage.h"
+
+#include "include/types.h"
+
+#include "mds/MDSMap.h"
+
+#include 
+
+class MMDSBeacon : public PaxosServiceMessage {
+
+  static const int HEAD_VERSION = 2;
+
+  uuid_d fsid;
+  uint64_t global_id;
+  string name;
+
+  __u32 state;
+  version_t seq;
+  __s32 standby_for_rank;
+  string standby_for_name;
+
+  CompatSet compat;
+
+ public:
+  MMDSBeacon() : PaxosServiceMessage(MSG_MDS_BEACON, 0, HEAD_VERSION) { }
+  MMDSBeacon(const uuid_d &f, uint64_t g, string& n, epoch_t les, int st, version_t se) : 
+    PaxosServiceMessage(MSG_MDS_BEACON, les, HEAD_VERSION), 
+    fsid(f), global_id(g), name(n), state(st), seq(se),
+    standby_for_rank(-1) {
+  }
+private:
+  ~MMDSBeacon() {}
+
+public:
+  uuid_d& get_fsid() { return fsid; }
+  uint64_t get_global_id() { return global_id; }
+  string& get_name() { return name; }
+  epoch_t get_last_epoch_seen() { return version; }
+  int get_state() { return state; }
+  version_t get_seq() { return seq; }
+  const char *get_type_name() const { return "mdsbeacon"; }
+  int get_standby_for_rank() { return standby_for_rank; }
+  const string& get_standby_for_name() { return standby_for_name; }
+
+  CompatSet& get_compat() { return compat; }
+  void set_compat(const CompatSet& c) { compat = c; }
+
+  void set_standby_for_rank(int r) { standby_for_rank = r; }
+  void set_standby_for_name(string& n) { standby_for_name = n; }
+  void set_standby_for_name(const char* c) { standby_for_name.assign(c); }
+
+  void print(ostream& out) const {
+    out << "mdsbeacon(" << global_id << "/" << name << " " << ceph_mds_state_name(state) 
+	<< " seq " << seq << " v" << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(global_id, payload);
+    ::encode(state, payload);
+    ::encode(seq, payload);
+    ::encode(name, payload);
+    ::encode(standby_for_rank, payload);
+    ::encode(standby_for_name, payload);
+    ::encode(compat, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(global_id, p);
+    ::decode(state, p);
+    ::decode(seq, p);
+    ::decode(name, p);
+    ::decode(standby_for_rank, p);
+    ::decode(standby_for_name, p);
+    if (header.version >= 2)
+      ::decode(compat, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSCacheRejoin.h b/ceph/src/messages/MMDSCacheRejoin.h
new file mode 100644
index 00000000..bbafe64a
--- /dev/null
+++ b/ceph/src/messages/MMDSCacheRejoin.h
@@ -0,0 +1,350 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMDSCACHEREJOIN_H
+#define CEPH_MMDSCACHEREJOIN_H
+
+#include "msg/Message.h"
+
+#include "include/types.h"
+
+#include "mds/CInode.h"
+#include "mds/CDir.h"
+
+// sent from replica to auth
+
+class MMDSCacheRejoin : public Message {
+
+  static const int HEAD_VERSION = 1;
+  static const int COMPAT_VERSION = 1;
+
+ public:
+  static const int OP_WEAK    = 1;  // replica -> auth, i exist, + maybe open files.
+  static const int OP_STRONG  = 2;  // replica -> auth, i exist, + open files and lock state.
+  static const int OP_ACK     = 3;  // auth -> replica, here is your lock state.
+  static const int OP_MISSING = 5;  // auth -> replica, i am missing these items
+  static const int OP_FULL    = 6;  // replica -> auth, here is the full object.
+  static const char *get_opname(int op) {
+    switch (op) {
+    case OP_WEAK: return "weak";
+    case OP_STRONG: return "strong";
+    case OP_ACK: return "ack";
+    case OP_MISSING: return "missing";
+    case OP_FULL: return "full";
+    default: assert(0); return 0;
+    }
+  }
+
+  // -- types --
+  struct inode_strong { 
+    uint32_t nonce;
+    int32_t caps_wanted;
+    int32_t filelock, nestlock, dftlock;
+    inode_strong() {}
+    inode_strong(int n, int cw, int dl, int nl, int dftl) :
+      nonce(n), caps_wanted(cw),
+      filelock(dl), nestlock(nl), dftlock(dftl) { }
+    void encode(bufferlist &bl) const {
+      ::encode(nonce, bl);
+      ::encode(caps_wanted, bl);
+      ::encode(filelock, bl);
+      ::encode(nestlock, bl);
+      ::encode(dftlock, bl);
+    }
+    void decode(bufferlist::iterator &bl) {
+      ::decode(nonce, bl);
+      ::decode(caps_wanted, bl);
+      ::decode(filelock, bl);
+      ::decode(nestlock, bl);
+      ::decode(dftlock, bl);
+    }
+  };
+  WRITE_CLASS_ENCODER(inode_strong)
+
+  struct dirfrag_strong {
+    uint32_t nonce;
+    int8_t  dir_rep;
+    dirfrag_strong() {}
+    dirfrag_strong(int n, int dr) : nonce(n), dir_rep(dr) {}
+    void encode(bufferlist &bl) const {
+      ::encode(nonce, bl);
+      ::encode(dir_rep, bl);
+    }
+    void decode(bufferlist::iterator &bl) {
+      ::decode(nonce, bl);
+      ::decode(dir_rep, bl);
+    }
+  };
+  WRITE_CLASS_ENCODER(dirfrag_strong)
+
+  struct dn_strong {
+    snapid_t first;
+    inodeno_t ino;
+    inodeno_t remote_ino;
+    unsigned char remote_d_type;
+    uint32_t nonce;
+    int32_t lock;
+    dn_strong() : 
+      ino(0), remote_ino(0), remote_d_type(0), nonce(0), lock(0) {}
+    dn_strong(snapid_t f, inodeno_t pi, inodeno_t ri, unsigned char rdt, int n, int l) : 
+      first(f), ino(pi), remote_ino(ri), remote_d_type(rdt), nonce(n), lock(l) {}
+    bool is_primary() { return ino > 0; }
+    bool is_remote() { return remote_ino > 0; }
+    bool is_null() { return ino == 0 && remote_ino == 0; }
+    void encode(bufferlist &bl) const {
+      ::encode(first, bl);
+      ::encode(ino, bl);
+      ::encode(remote_ino, bl);
+      ::encode(remote_d_type, bl);
+      ::encode(nonce, bl);
+      ::encode(lock, bl);
+    }
+    void decode(bufferlist::iterator &bl) {
+      ::decode(first, bl);
+      ::decode(ino, bl);
+      ::decode(remote_ino, bl);
+      ::decode(remote_d_type, bl);
+      ::decode(nonce, bl);
+      ::decode(lock, bl);
+    }
+  };
+  WRITE_CLASS_ENCODER(dn_strong)
+
+  struct dn_weak {
+    snapid_t first;
+    inodeno_t ino;
+    dn_weak() : ino(0) {}
+    dn_weak(snapid_t f, inodeno_t pi) : first(f), ino(pi) {}
+    void encode(bufferlist &bl) const {
+      ::encode(first, bl);
+      ::encode(ino, bl);
+    }
+    void decode(bufferlist::iterator &bl) {
+      ::decode(first, bl);
+      ::decode(ino, bl);
+    }
+  };
+  WRITE_CLASS_ENCODER(dn_weak)
+
+  // -- data --
+  int32_t op;
+
+  struct lock_bls {
+    bufferlist file, nest, dft;
+    void encode(bufferlist& bl) const {
+      ::encode(file, bl);
+      ::encode(nest, bl);
+      ::encode(dft, bl);
+    }
+    void decode(bufferlist::iterator& bl) {
+      ::decode(file, bl);
+      ::decode(nest, bl);
+      ::decode(dft, bl);
+    }
+  };
+  WRITE_CLASS_ENCODER(lock_bls)
+
+  // weak
+  map > weak;
+  set weak_dirfrags;
+  set weak_inodes;
+  map inode_scatterlocks;
+
+  // strong
+  map strong_dirfrags;
+  map > strong_dentries;
+  map strong_inodes;
+
+  // open
+  map > cap_exports;
+  map client_map;
+  bufferlist imported_caps;
+
+  // full
+  bufferlist inode_base;
+  bufferlist inode_locks;
+  map dirfrag_bases;
+
+  // authpins, xlocks
+  struct slave_reqid {
+    metareqid_t reqid;
+    __u32 attempt;
+    slave_reqid() : attempt(0) {}
+    slave_reqid(const metareqid_t& r, __u32 a)
+      : reqid(r), attempt(a) {}
+    void encode(bufferlist& bl) const {
+      ::encode(reqid, bl);
+      ::encode(attempt, bl);
+    }
+    void decode(bufferlist::iterator& bl) {
+      ::decode(reqid, bl);
+      ::decode(attempt, bl);
+    }
+  };
+  map > authpinned_inodes;
+  map frozen_authpin_inodes;
+  map > xlocked_inodes;
+  map > > wrlocked_inodes;
+  map > > authpinned_dentries;
+  map > xlocked_dentries;
+  
+  MMDSCacheRejoin() :
+    Message(MSG_MDS_CACHEREJOIN, HEAD_VERSION, COMPAT_VERSION)
+  {}
+  MMDSCacheRejoin(int o) : 
+    Message(MSG_MDS_CACHEREJOIN, HEAD_VERSION, COMPAT_VERSION),
+    op(o) {}
+private:
+  ~MMDSCacheRejoin() {}
+
+public:
+  const char *get_type_name() const { return "cache_rejoin"; }
+  void print(ostream& out) const {
+    out << "cache_rejoin " << get_opname(op);
+  }
+
+  // -- builders --
+  // inodes
+  void add_weak_inode(vinodeno_t i) {
+    weak_inodes.insert(i);
+  }
+  void add_strong_inode(vinodeno_t i, int n, int cw, int dl, int nl, int dftl) {
+    strong_inodes[i] = inode_strong(n, cw, dl, nl, dftl);
+  }
+  void add_inode_locks(CInode *in, __u32 nonce, bufferlist& bl) {
+    ::encode(in->inode.ino, inode_locks);
+    ::encode(in->last, inode_locks);
+    ::encode(nonce, inode_locks);
+    ::encode(bl, inode_locks);
+  }
+  void add_inode_base(CInode *in) {
+    ::encode(in->inode.ino, inode_base);
+    ::encode(in->last, inode_base);
+    bufferlist bl;
+    in->_encode_base(bl);
+    ::encode(bl, inode_base);
+  }
+  void add_inode_authpin(vinodeno_t ino, const metareqid_t& ri, __u32 attempt) {
+    authpinned_inodes[ino].push_back(slave_reqid(ri, attempt));
+  }
+  void add_inode_frozen_authpin(vinodeno_t ino, const metareqid_t& ri, __u32 attempt) {
+    frozen_authpin_inodes[ino] = slave_reqid(ri, attempt);
+  }
+  void add_inode_xlock(vinodeno_t ino, int lt, const metareqid_t& ri, __u32 attempt) {
+    xlocked_inodes[ino][lt] = slave_reqid(ri, attempt);
+  }
+  void add_inode_wrlock(vinodeno_t ino, int lt, const metareqid_t& ri, __u32 attempt) {
+    wrlocked_inodes[ino][lt].push_back(slave_reqid(ri, attempt));
+  }
+
+  void add_scatterlock_state(CInode *in) {
+    if (inode_scatterlocks.count(in->ino()))
+      return;  // already added this one
+    in->encode_lock_state(CEPH_LOCK_IFILE, inode_scatterlocks[in->ino()].file);
+    in->encode_lock_state(CEPH_LOCK_INEST, inode_scatterlocks[in->ino()].nest);
+    in->encode_lock_state(CEPH_LOCK_IDFT, inode_scatterlocks[in->ino()].dft);
+  }
+
+  // dirfrags
+  void add_strong_dirfrag(dirfrag_t df, int n, int dr) {
+    strong_dirfrags[df] = dirfrag_strong(n, dr);
+  }
+  void add_dirfrag_base(CDir *dir) {
+    bufferlist& bl = dirfrag_bases[dir->dirfrag()];
+    dir->_encode_base(bl);
+  }
+
+  // dentries
+  void add_weak_dirfrag(dirfrag_t df) {
+    weak_dirfrags.insert(df);
+  }
+  void add_weak_dentry(inodeno_t dirino, const string& dname, snapid_t last, dn_weak& dnw) {
+    weak[dirino][string_snap_t(dname, last)] = dnw;
+  }
+  void add_weak_primary_dentry(inodeno_t dirino, const string& dname, snapid_t first, snapid_t last, inodeno_t ino) {
+    weak[dirino][string_snap_t(dname, last)] = dn_weak(first, ino);
+  }
+  void add_strong_dentry(dirfrag_t df, const string& dname, snapid_t first, snapid_t last, inodeno_t pi, inodeno_t ri, unsigned char rdt, int n, int ls) {
+    strong_dentries[df][string_snap_t(dname, last)] = dn_strong(first, pi, ri, rdt, n, ls);
+  }
+  void add_dentry_authpin(dirfrag_t df, const string& dname, snapid_t last,
+			  const metareqid_t& ri, __u32 attempt) {
+    authpinned_dentries[df][string_snap_t(dname, last)].push_back(slave_reqid(ri, attempt));
+  }
+  void add_dentry_xlock(dirfrag_t df, const string& dname, snapid_t last,
+			const metareqid_t& ri, __u32 attempt) {
+    xlocked_dentries[df][string_snap_t(dname, last)] = slave_reqid(ri, attempt);
+  }
+
+  // -- encoding --
+  void encode_payload(uint64_t features) {
+    ::encode(op, payload);
+    ::encode(strong_inodes, payload);
+    ::encode(inode_base, payload);
+    ::encode(inode_locks, payload);
+    ::encode(inode_scatterlocks, payload);
+    ::encode(authpinned_inodes, payload);
+    ::encode(frozen_authpin_inodes, payload);
+    ::encode(xlocked_inodes, payload);
+    ::encode(wrlocked_inodes, payload);
+    ::encode(cap_exports, payload);
+    ::encode(client_map, payload);
+    ::encode(imported_caps, payload);
+    ::encode(strong_dirfrags, payload);
+    ::encode(dirfrag_bases, payload);
+    ::encode(weak, payload);
+    ::encode(weak_dirfrags, payload);
+    ::encode(weak_inodes, payload);
+    ::encode(strong_dentries, payload);
+    ::encode(authpinned_dentries, payload);
+    ::encode(xlocked_dentries, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(op, p);
+    ::decode(strong_inodes, p);
+    ::decode(inode_base, p);
+    ::decode(inode_locks, p);
+    ::decode(inode_scatterlocks, p);
+    ::decode(authpinned_inodes, p);
+    ::decode(frozen_authpin_inodes, p);
+    ::decode(xlocked_inodes, p);
+    ::decode(wrlocked_inodes, p);
+    ::decode(cap_exports, p);
+    ::decode(client_map, p);
+    ::decode(imported_caps, p);
+    ::decode(strong_dirfrags, p);
+    ::decode(dirfrag_bases, p);
+    ::decode(weak, p);
+    ::decode(weak_dirfrags, p);
+    ::decode(weak_inodes, p);
+    ::decode(strong_dentries, p);
+    ::decode(authpinned_dentries, p);
+    ::decode(xlocked_dentries, p);
+  }
+
+};
+
+WRITE_CLASS_ENCODER(MMDSCacheRejoin::inode_strong)
+WRITE_CLASS_ENCODER(MMDSCacheRejoin::dirfrag_strong)
+WRITE_CLASS_ENCODER(MMDSCacheRejoin::dn_strong)
+WRITE_CLASS_ENCODER(MMDSCacheRejoin::dn_weak)
+WRITE_CLASS_ENCODER(MMDSCacheRejoin::lock_bls)
+WRITE_CLASS_ENCODER(MMDSCacheRejoin::slave_reqid)
+
+inline ostream& operator<<(ostream& out, const MMDSCacheRejoin::slave_reqid& r) {
+  return out << r.reqid << '.' << r.attempt;
+}
+
+#endif
diff --git a/ceph/src/messages/MMDSFindIno.h b/ceph/src/messages/MMDSFindIno.h
new file mode 100644
index 00000000..859a7c40
--- /dev/null
+++ b/ceph/src/messages/MMDSFindIno.h
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDSFINDINO_H
+#define CEPH_MDSFINDINO_H
+
+#include "msg/Message.h"
+#include "include/filepath.h"
+
+struct MMDSFindIno : public Message {
+  ceph_tid_t tid;
+  inodeno_t ino;
+
+  MMDSFindIno() : Message(MSG_MDS_FINDINO) {}
+  MMDSFindIno(ceph_tid_t t, inodeno_t i) : Message(MSG_MDS_FINDINO), tid(t), ino(i) {}
+
+  const char *get_type_name() const { return "findino"; }
+  void print(ostream &out) const {
+    out << "findino(" << tid << " " << ino << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(tid, payload);
+    ::encode(ino, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(tid, p);
+    ::decode(ino, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSFindInoReply.h b/ceph/src/messages/MMDSFindInoReply.h
new file mode 100644
index 00000000..aaf2a6e7
--- /dev/null
+++ b/ceph/src/messages/MMDSFindInoReply.h
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MDSFINDINOREPLY_H
+#define CEPH_MDSFINDINOREPLY_H
+
+#include "msg/Message.h"
+#include "include/filepath.h"
+
+struct MMDSFindInoReply : public Message {
+  ceph_tid_t tid;
+  filepath path;
+
+  MMDSFindInoReply() : Message(MSG_MDS_FINDINOREPLY) {}
+  MMDSFindInoReply(ceph_tid_t t) : Message(MSG_MDS_FINDINOREPLY), tid(t) {}
+
+  const char *get_type_name() const { return "findinoreply"; }
+  void print(ostream &out) const {
+    out << "findinoreply(" << tid << " " << path << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    ::encode(tid, payload);
+    ::encode(path, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(tid, p);
+    ::decode(path, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSFragmentNotify.h b/ceph/src/messages/MMDSFragmentNotify.h
new file mode 100644
index 00000000..46b88378
--- /dev/null
+++ b/ceph/src/messages/MMDSFragmentNotify.h
@@ -0,0 +1,64 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMDSFRAGMENTNOTIFY_H
+#define CEPH_MMDSFRAGMENTNOTIFY_H
+
+#include "msg/Message.h"
+#include 
+using namespace std;
+
+class MMDSFragmentNotify : public Message {
+  inodeno_t ino;
+  frag_t basefrag;
+  int8_t bits;
+
+ public:
+  inodeno_t get_ino() { return ino; }
+  frag_t get_basefrag() { return basefrag; }
+  int get_bits() { return bits; }
+
+  bufferlist basebl;
+
+  MMDSFragmentNotify() : Message(MSG_MDS_FRAGMENTNOTIFY) {}
+  MMDSFragmentNotify(dirfrag_t df, int b) :
+	Message(MSG_MDS_FRAGMENTNOTIFY),
+    ino(df.ino), basefrag(df.frag), bits(b) { }
+private:
+  ~MMDSFragmentNotify() {}
+
+public:  
+  const char *get_type_name() const { return "fragment_notify"; }
+  void print(ostream& o) const {
+    o << "fragment_notify(" << ino << "." << basefrag
+      << " " << (int)bits << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(ino, payload);
+    ::encode(basefrag, payload);
+    ::encode(bits, payload);
+    ::encode(basebl, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(ino, p);
+    ::decode(basefrag, p);
+    ::decode(bits, p);
+    ::decode(basebl, p);
+  }
+  
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSLoadTargets.h b/ceph/src/messages/MMDSLoadTargets.h
new file mode 100644
index 00000000..4022f32e
--- /dev/null
+++ b/ceph/src/messages/MMDSLoadTargets.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2009 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMDSLoadTargets_H
+#define CEPH_MMDSLoadTargets_H
+
+#include "msg/Message.h"
+#include "messages/PaxosServiceMessage.h"
+#include "include/types.h"
+
+#include 
+using std::map;
+
+class MMDSLoadTargets : public PaxosServiceMessage {
+ public:
+  uint64_t global_id;
+  set targets;
+
+  MMDSLoadTargets() : PaxosServiceMessage(MSG_MDS_OFFLOAD_TARGETS, 0) {}
+
+  MMDSLoadTargets(uint64_t g, set& mds_targets) :
+    PaxosServiceMessage(MSG_MDS_OFFLOAD_TARGETS, 0),
+    global_id(g), targets(mds_targets) {}
+private:
+  ~MMDSLoadTargets() {}
+
+public:
+  const char* get_type_name() const { return "mds_load_targets"; }
+  void print(ostream& o) const {
+    o << "mds_load_targets(" << global_id << " " << targets << ")";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(global_id, p);
+    ::decode(targets, p);
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(global_id, payload);
+    ::encode(targets, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSMap.h b/ceph/src/messages/MMDSMap.h
new file mode 100644
index 00000000..66566d01
--- /dev/null
+++ b/ceph/src/messages/MMDSMap.h
@@ -0,0 +1,97 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MMDSMAP_H
+#define CEPH_MMDSMAP_H
+
+#include "msg/Message.h"
+#include "mds/MDSMap.h"
+#include "include/ceph_features.h"
+
+#include 
+
+class MMDSMap : public Message {
+ public:
+  /*
+  map maps;
+  map incremental_maps;
+
+  epoch_t get_first() {
+    epoch_t e = 0;
+    map::iterator i = maps.begin();
+    if (i != maps.end())  e = i->first;
+    i = incremental_maps.begin();    
+    if (i != incremental_maps.end() &&
+        (e == 0 || i->first < e)) e = i->first;
+    return e;
+  }
+  epoch_t get_last() {
+    epoch_t e = 0;
+    map::reverse_iterator i = maps.rbegin();
+    if (i != maps.rend())  e = i->first;
+    i = incremental_maps.rbegin();    
+    if (i != incremental_maps.rend() &&
+        (e == 0 || i->first > e)) e = i->first;
+    return e;
+  }
+  */
+  
+  uuid_d fsid;
+  epoch_t epoch;
+  bufferlist encoded;
+
+  version_t get_epoch() const { return epoch; }
+  bufferlist& get_encoded() { return encoded; }
+
+  MMDSMap() : 
+    Message(CEPH_MSG_MDS_MAP) {}
+  MMDSMap(const uuid_d &f, MDSMap *mm) :
+    Message(CEPH_MSG_MDS_MAP),
+    fsid(f) {
+    epoch = mm->get_epoch();
+    mm->encode(encoded, -1);  // we will reencode with fewer features as necessary
+  }
+private:
+  ~MMDSMap() {}
+
+public:
+  const char *get_type_name() const { return "mdsmap"; }
+  void print(ostream& out) const {
+    out << "mdsmap(e " << epoch << ")";
+  }
+
+  // marshalling
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(fsid, p);
+    ::decode(epoch, p);
+    ::decode(encoded, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(fsid, payload);
+    ::encode(epoch, payload);
+    if ((features & CEPH_FEATURE_PGID64) == 0 ||
+	(features & CEPH_FEATURE_MDSENC) == 0) {
+      // reencode for old clients.
+      MDSMap m;
+      m.decode(encoded);
+      encoded.clear();
+      m.encode(encoded, features);
+    }
+    ::encode(encoded, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSOpenIno.h b/ceph/src/messages/MMDSOpenIno.h
new file mode 100644
index 00000000..a169fd0e
--- /dev/null
+++ b/ceph/src/messages/MMDSOpenIno.h
@@ -0,0 +1,46 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MDSOPENINO_H
+#define CEPH_MDSOPENINO_H
+
+#include "msg/Message.h"
+
+struct MMDSOpenIno : public Message {
+  inodeno_t ino;
+  vector ancestors;
+
+  MMDSOpenIno() : Message(MSG_MDS_OPENINO) {}
+  MMDSOpenIno(ceph_tid_t t, inodeno_t i, vector& a) :
+    Message(MSG_MDS_OPENINO), ino(i), ancestors(a) {
+    header.tid = t;
+  }
+
+  const char *get_type_name() const { return "openino"; }
+  void print(ostream &out) const {
+    out << "openino(" << header.tid << " " << ino << " " << ancestors << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(ino, payload);
+    ::encode(ancestors, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(ino, p);
+    ::decode(ancestors, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSOpenInoReply.h b/ceph/src/messages/MMDSOpenInoReply.h
new file mode 100644
index 00000000..5204d3c7
--- /dev/null
+++ b/ceph/src/messages/MMDSOpenInoReply.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MDSOPENINOREPLY_H
+#define CEPH_MDSOPENINOREPLY_H
+
+#include "msg/Message.h"
+
+struct MMDSOpenInoReply : public Message {
+  inodeno_t ino;
+  vector ancestors;
+  int32_t hint;
+  int32_t error;
+
+  MMDSOpenInoReply() : Message(MSG_MDS_OPENINOREPLY) {}
+  MMDSOpenInoReply(ceph_tid_t t, inodeno_t i, int h=-1, int e=0) :
+    Message(MSG_MDS_OPENINOREPLY), ino(i), hint(h), error(e) {
+    header.tid = t;
+  }
+
+  const char *get_type_name() const { return "openinoreply"; }
+  void print(ostream &out) const {
+    out << "openinoreply(" << header.tid << " "
+	<< ino << " " << hint << " " << ancestors << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(ino, payload);
+    ::encode(ancestors, payload);
+    ::encode(hint, payload);
+    ::encode(error, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(ino, p);
+    ::decode(ancestors, p);
+    ::decode(hint, p);
+    ::decode(error, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSResolve.h b/ceph/src/messages/MMDSResolve.h
new file mode 100644
index 00000000..263bd547
--- /dev/null
+++ b/ceph/src/messages/MMDSResolve.h
@@ -0,0 +1,73 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMDSRESOLVE_H
+#define CEPH_MMDSRESOLVE_H
+
+#include "msg/Message.h"
+
+#include "include/types.h"
+
+class MMDSResolve : public Message {
+ public:
+  map > subtrees;
+  map > ambiguous_imports;
+  map slave_requests;
+
+  MMDSResolve() : Message(MSG_MDS_RESOLVE) {}
+private:
+  ~MMDSResolve() {}
+
+public:
+  const char *get_type_name() const { return "mds_resolve"; }
+
+  void print(ostream& out) const {
+    out << "mds_resolve(" << subtrees.size()
+	<< "+" << ambiguous_imports.size()
+	<< " subtrees +" << slave_requests.size() << " slave requests)";
+  }
+  
+  void add_subtree(dirfrag_t im) {
+    subtrees[im].clear();
+  }
+  void add_subtree_bound(dirfrag_t im, dirfrag_t ex) {
+    subtrees[im].push_back(ex);
+  }
+
+  void add_ambiguous_import(dirfrag_t im, const vector& m) {
+    ambiguous_imports[im] = m;
+  }
+
+  void add_slave_request(metareqid_t reqid) {
+    slave_requests[reqid].clear();
+  }
+
+  void add_slave_request(metareqid_t reqid, bufferlist& bl) {
+    slave_requests[reqid].claim(bl);
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(subtrees, payload);
+    ::encode(ambiguous_imports, payload);
+    ::encode(slave_requests, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(subtrees, p);
+    ::decode(ambiguous_imports, p);
+    ::decode(slave_requests, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSResolveAck.h b/ceph/src/messages/MMDSResolveAck.h
new file mode 100644
index 00000000..743353fe
--- /dev/null
+++ b/ceph/src/messages/MMDSResolveAck.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMDSRESOLVEACK_H
+#define CEPH_MMDSRESOLVEACK_H
+
+#include "msg/Message.h"
+
+#include "include/types.h"
+
+
+class MMDSResolveAck : public Message {
+ public:
+  map commit;
+  vector abort;
+
+  MMDSResolveAck() : Message(MSG_MDS_RESOLVEACK) {}
+private:
+  ~MMDSResolveAck() {}
+
+public:
+  const char *get_type_name() const { return "resolve_ack"; }
+  /*void print(ostream& out) const {
+    out << "resolve_ack.size()
+	<< "+" << ambiguous_imap.size()
+	<< " imports +" << slave_requests.size() << " slave requests)";
+  }
+  */
+  
+  void add_commit(metareqid_t r) {
+    commit[r].clear();
+  }
+  void add_abort(metareqid_t r) {
+    abort.push_back(r);
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(commit, payload);
+    ::encode(abort, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(commit, p);
+    ::decode(abort, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSSlaveRequest.h b/ceph/src/messages/MMDSSlaveRequest.h
new file mode 100644
index 00000000..15f095bf
--- /dev/null
+++ b/ceph/src/messages/MMDSSlaveRequest.h
@@ -0,0 +1,196 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MMDSSLAVEREQUEST_H
+#define CEPH_MMDSSLAVEREQUEST_H
+
+#include "msg/Message.h"
+#include "mds/mdstypes.h"
+
+class MMDSSlaveRequest : public Message {
+ public:
+  static const int OP_XLOCK =       1;
+  static const int OP_XLOCKACK =   -1;
+  static const int OP_UNXLOCK =     2;
+  static const int OP_AUTHPIN =     3;
+  static const int OP_AUTHPINACK = -3;
+
+  static const int OP_LINKPREP =     4;
+  static const int OP_UNLINKPREP =   5;
+  static const int OP_LINKPREPACK = -4;
+
+  static const int OP_RENAMEPREP =     7;
+  static const int OP_RENAMEPREPACK = -7;
+
+  static const int OP_WRLOCK = 8;
+  static const int OP_WRLOCKACK = -8;
+  static const int OP_UNWRLOCK = 9;
+
+  static const int OP_RMDIRPREP = 10;
+  static const int OP_RMDIRPREPACK = -10;
+
+  static const int OP_DROPLOCKS	= 11;
+
+  static const int OP_RENAMENOTIFY = 12;
+  static const int OP_RENAMENOTIFYACK = -12;
+
+  static const int OP_FINISH = 17;  
+  static const int OP_COMMITTED = -18;  
+
+  static const int OP_ABORT =  20;  // used for recovery only
+  //static const int OP_COMMIT = 21;  // used for recovery only
+
+
+  const static char *get_opname(int o) {
+    switch (o) { 
+    case OP_XLOCK: return "xlock";
+    case OP_XLOCKACK: return "xlock_ack";
+    case OP_UNXLOCK: return "unxlock";
+    case OP_AUTHPIN: return "authpin";
+    case OP_AUTHPINACK: return "authpin_ack";
+
+    case OP_LINKPREP: return "link_prep";
+    case OP_LINKPREPACK: return "link_prep_ack";
+    case OP_UNLINKPREP: return "unlink_prep";
+
+    case OP_RENAMEPREP: return "rename_prep";
+    case OP_RENAMEPREPACK: return "rename_prep_ack";
+
+    case OP_FINISH: return "finish"; // commit
+    case OP_COMMITTED: return "committed";
+
+    case OP_WRLOCK: return "wrlock";
+    case OP_WRLOCKACK: return "wrlock_ack";
+    case OP_UNWRLOCK: return "unwrlock";
+
+    case OP_RMDIRPREP: return "rmdir_prep";
+    case OP_RMDIRPREPACK: return "rmdir_prep_ack";
+
+    case OP_DROPLOCKS: return "drop_locks";
+
+    case OP_RENAMENOTIFY: return "reame_notify";
+    case OP_RENAMENOTIFYACK: return "rename_notify_ack";
+
+    case OP_ABORT: return "abort";
+      //case OP_COMMIT: return "commit";
+
+    default: assert(0); return 0;
+    }
+  }
+
+ private:
+  metareqid_t reqid;
+  __u32 attempt;
+  __s16 op;
+  __u16 flags;
+
+  static const unsigned FLAG_NONBLOCK	= 1;
+  static const unsigned FLAG_WOULDBLOCK	= 2;
+
+  // for locking
+  __u16 lock_type;  // lock object type
+  MDSCacheObjectInfo object_info;
+  
+  // for authpins
+  vector authpins;
+
+ public:
+  // for rename prep
+  filepath srcdnpath;
+  filepath destdnpath;
+  set<__s32> witnesses;
+  bufferlist inode_export;
+  version_t inode_export_v;
+  bufferlist srci_replica;
+  utime_t now;
+
+  bufferlist stray;  // stray dir + dentry
+
+public:
+  metareqid_t get_reqid() { return reqid; }
+  __u32 get_attempt() const { return attempt; }
+  int get_op() { return op; }
+  bool is_reply() { return op < 0; }
+
+  int get_lock_type() { return lock_type; }
+  MDSCacheObjectInfo &get_object_info() { return object_info; }
+  MDSCacheObjectInfo &get_authpin_freeze() { return object_info; }
+
+  vector& get_authpins() { return authpins; }
+  void mark_nonblock() { flags |= FLAG_NONBLOCK; }
+  bool is_nonblock() { return (flags & FLAG_NONBLOCK); }
+  void mark_error_wouldblock() { flags |= FLAG_WOULDBLOCK; }
+  bool is_error_wouldblock() { return (flags & FLAG_WOULDBLOCK); }
+
+  void set_lock_type(int t) { lock_type = t; }
+
+
+  // ----
+  MMDSSlaveRequest() : Message(MSG_MDS_SLAVE_REQUEST) { }
+  MMDSSlaveRequest(metareqid_t ri, __u32 att, int o) : 
+    Message(MSG_MDS_SLAVE_REQUEST),
+    reqid(ri), attempt(att), op(o), flags(0), lock_type(0),
+    inode_export_v(0) { }
+private:
+  ~MMDSSlaveRequest() {}
+
+public:
+  void encode_payload(uint64_t features) {
+    ::encode(reqid, payload);
+    ::encode(attempt, payload);
+    ::encode(op, payload);
+    ::encode(flags, payload);
+    ::encode(lock_type, payload);
+    ::encode(object_info, payload);
+    ::encode(authpins, payload);
+    ::encode(srcdnpath, payload);
+    ::encode(destdnpath, payload);
+    ::encode(witnesses, payload);
+    ::encode(now, payload);
+    ::encode(inode_export, payload);
+    ::encode(inode_export_v, payload);
+    ::encode(srci_replica, payload);
+    ::encode(stray, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(reqid, p);
+    ::decode(attempt, p);
+    ::decode(op, p);
+    ::decode(flags, p);
+    ::decode(lock_type, p);
+    ::decode(object_info, p);
+    ::decode(authpins, p);
+    ::decode(srcdnpath, p);
+    ::decode(destdnpath, p);
+    ::decode(witnesses, p);
+    ::decode(now, p);
+    ::decode(inode_export, p);
+    ::decode(inode_export_v, p);
+    ::decode(srci_replica, p);
+    ::decode(stray, p);
+  }
+
+  const char *get_type_name() const { return "slave_request"; }
+  void print(ostream& out) const {
+    out << "slave_request(" << reqid
+	<< "." << attempt
+	<< " " << get_opname(op) 
+	<< ")";
+  }  
+	
+};
+
+#endif
diff --git a/ceph/src/messages/MMDSTableRequest.h b/ceph/src/messages/MMDSTableRequest.h
new file mode 100644
index 00000000..e67c1f3e
--- /dev/null
+++ b/ceph/src/messages/MMDSTableRequest.h
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MMDSTABLEREQUEST_H
+#define CEPH_MMDSTABLEREQUEST_H
+
+#include "msg/Message.h"
+#include "mds/mds_table_types.h"
+
+class MMDSTableRequest : public Message {
+ public:
+  __u16 table;
+  __s16 op;
+  uint64_t reqid;
+  bufferlist bl;
+
+  MMDSTableRequest() : Message(MSG_MDS_TABLE_REQUEST) {}
+  MMDSTableRequest(int tab, int o, uint64_t r, version_t v=0) : 
+    Message(MSG_MDS_TABLE_REQUEST),
+    table(tab), op(o), reqid(r) {
+    set_tid(v);
+  }
+private:
+  ~MMDSTableRequest() {}
+
+public:  
+  virtual const char *get_type_name() const { return "mds_table_request"; }
+  void print(ostream& o) const {
+    o << "mds_table_request(" << get_mdstable_name(table)
+      << " " << get_mdstableserver_opname(op);
+    if (reqid) o << " " << reqid;
+    if (get_tid()) o << " tid " << get_tid();
+    if (bl.length()) o << " " << bl.length() << " bytes";
+    o << ")";
+  }
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(table, p);
+    ::decode(op, p);
+    ::decode(reqid, p);
+    ::decode(bl, p);
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(table, payload);
+    ::encode(op, payload);
+    ::encode(reqid, payload);
+    ::encode(bl, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonCommand.h b/ceph/src/messages/MMonCommand.h
new file mode 100644
index 00000000..2b79465c
--- /dev/null
+++ b/ceph/src/messages/MMonCommand.h
@@ -0,0 +1,61 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONCOMMAND_H
+#define CEPH_MMONCOMMAND_H
+
+#include "messages/PaxosServiceMessage.h"
+
+#include 
+using std::vector;
+
+class MMonCommand : public PaxosServiceMessage {
+ public:
+  uuid_d fsid;
+  vector cmd;
+
+  MMonCommand() : PaxosServiceMessage(MSG_MON_COMMAND, 0) {}
+  MMonCommand(const uuid_d &f)
+    : PaxosServiceMessage(MSG_MON_COMMAND, 0),
+      fsid(f)
+  { }
+
+private:
+  ~MMonCommand() {}
+
+public:  
+  const char *get_type_name() const { return "mon_command"; }
+  void print(ostream& o) const {
+    o << "mon_command(";
+    for (unsigned i=0; i
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONCOMMANDACK_H
+#define CEPH_MMONCOMMANDACK_H
+
+#include "messages/PaxosServiceMessage.h"
+
+class MMonCommandAck : public PaxosServiceMessage {
+ public:
+  vector cmd;
+  __s32 r;
+  string rs;
+  
+  MMonCommandAck() : PaxosServiceMessage(MSG_MON_COMMAND_ACK, 0) {}
+  MMonCommandAck(vector& c, int _r, string s, version_t v) : 
+    PaxosServiceMessage(MSG_MON_COMMAND_ACK, v),
+    cmd(c), r(_r), rs(s) { }
+private:
+  ~MMonCommandAck() {}
+
+public:
+  const char *get_type_name() const { return "mon_command"; }
+  void print(ostream& o) const {
+    o << "mon_command_ack(" << cmd << "=" << r << " " << rs << " v" << version << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(r, payload);
+    ::encode(rs, payload);
+    ::encode(cmd, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(r, p);
+    ::decode(rs, p);
+    ::decode(cmd, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonElection.h b/ceph/src/messages/MMonElection.h
new file mode 100644
index 00000000..e3ccbf3e
--- /dev/null
+++ b/ceph/src/messages/MMonElection.h
@@ -0,0 +1,120 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MMONELECTION_H
+#define CEPH_MMONELECTION_H
+
+#include "msg/Message.h"
+#include "mon/MonMap.h"
+
+class MMonElection : public Message {
+
+  static const int HEAD_VERSION = 5;
+  static const int COMPAT_VERSION = 2;
+
+public:
+  static const int OP_PROPOSE = 1;
+  static const int OP_ACK     = 2;
+  static const int OP_NAK     = 3;
+  static const int OP_VICTORY = 4;
+  static const char *get_opname(int o) {
+    switch (o) {
+    case OP_PROPOSE: return "propose";
+    case OP_ACK: return "ack";
+    case OP_NAK: return "nak";
+    case OP_VICTORY: return "victory";
+    default: assert(0); return 0;
+    }
+  }
+  
+  uuid_d fsid;
+  int32_t op;
+  epoch_t epoch;
+  bufferlist monmap_bl;
+  set quorum;
+  uint64_t quorum_features;
+  bufferlist sharing_bl;
+  /* the following were both used in the next branch for a while
+   * on user cluster, so we've left them in for compatibility. */
+  version_t defunct_one;
+  version_t defunct_two;
+  
+  MMonElection() : Message(MSG_MON_ELECTION, HEAD_VERSION, COMPAT_VERSION),
+    op(0), epoch(0), quorum_features(0), defunct_one(0),
+    defunct_two(0)
+  { }
+
+  MMonElection(int o, epoch_t e, MonMap *m)
+    : Message(MSG_MON_ELECTION, HEAD_VERSION, COMPAT_VERSION),
+      fsid(m->fsid), op(o), epoch(e), quorum_features(0),
+      defunct_one(0), defunct_two(0)
+  {
+    // encode using full feature set; we will reencode for dest later,
+    // if necessary
+    m->encode(monmap_bl, CEPH_FEATURES_ALL);
+  }
+private:
+  ~MMonElection() {}
+
+public:  
+  const char *get_type_name() const { return "election"; }
+  void print(ostream& out) const {
+    out << "election(" << fsid << " " << get_opname(op) << " " << epoch << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    if (monmap_bl.length() && (features != CEPH_FEATURES_ALL)) {
+      // reencode old-format monmap
+      MonMap t;
+      t.decode(monmap_bl);
+      monmap_bl.clear();
+      t.encode(monmap_bl, features);
+    }
+
+    ::encode(fsid, payload);
+    ::encode(op, payload);
+    ::encode(epoch, payload);
+    ::encode(monmap_bl, payload);
+    ::encode(quorum, payload);
+    ::encode(quorum_features, payload);
+    ::encode(defunct_one, payload);
+    ::encode(defunct_two, payload);
+    ::encode(sharing_bl, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    if (header.version >= 2)
+      ::decode(fsid, p);
+    else
+      memset(&fsid, 0, sizeof(fsid));
+    ::decode(op, p);
+    ::decode(epoch, p);
+    ::decode(monmap_bl, p);
+    ::decode(quorum, p);
+    if (header.version >= 3)
+      ::decode(quorum_features, p);
+    else
+      quorum_features = 0;
+    if (header.version >= 4) {
+      ::decode(defunct_one, p);
+      ::decode(defunct_two, p);
+    }
+    if (header.version >= 5)
+      ::decode(sharing_bl, p);
+  }
+  
+};
+
+#endif
diff --git a/ceph/src/messages/MMonGetMap.h b/ceph/src/messages/MMonGetMap.h
new file mode 100644
index 00000000..d4bb61e7
--- /dev/null
+++ b/ceph/src/messages/MMonGetMap.h
@@ -0,0 +1,35 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONGETMAP_H
+#define CEPH_MMONGETMAP_H
+
+#include "msg/Message.h"
+
+#include "include/types.h"
+
+class MMonGetMap : public Message {
+ public:
+  MMonGetMap() : Message(CEPH_MSG_MON_GET_MAP) { }
+private:
+  ~MMonGetMap() {}
+
+public:
+  const char *get_type_name() const { return "mon_getmap"; }
+  
+  void encode_payload(uint64_t features) { }
+  void decode_payload() { }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonGetVersion.h b/ceph/src/messages/MMonGetVersion.h
new file mode 100644
index 00000000..e74daa65
--- /dev/null
+++ b/ceph/src/messages/MMonGetVersion.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MMONGETVERSION_H
+#define CEPH_MMONGETVERSION_H
+
+#include "msg/Message.h"
+
+#include "include/types.h"
+
+/*
+ * This message is sent to the monitors to verify that the client's
+ * version of the map(s) is the latest available. For example, this
+ * can be used to determine whether a pool actually does not exist, or
+ * if it may have been created but the map was not received yet.
+ */
+class MMonGetVersion : public Message {
+public:
+  MMonGetVersion() : Message(CEPH_MSG_MON_GET_VERSION) {}
+
+  const char *get_type_name() const {
+    return "mon_get_version";
+  }
+
+  void print(ostream& o) const {
+    o << "mon_get_version(what=" << what << " handle=" << handle << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(handle, payload);
+    ::encode(what, payload);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(handle, p);
+    ::decode(what, p);
+  }
+
+  ceph_tid_t handle;
+  string what;
+
+private:
+  ~MMonGetVersion() {}
+};
+
+#endif
diff --git a/ceph/src/messages/MMonGetVersionReply.h b/ceph/src/messages/MMonGetVersionReply.h
new file mode 100644
index 00000000..b13b2f00
--- /dev/null
+++ b/ceph/src/messages/MMonGetVersionReply.h
@@ -0,0 +1,64 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MMONGETVERSIONREPLY_H
+#define CEPH_MMONGETVERSIONREPLY_H
+
+#include "msg/Message.h"
+
+#include "include/types.h"
+
+/*
+ * This message is sent from the monitors to clients in response to a
+ * MMonGetVersion. The latest version of the requested thing is sent
+ * back.
+ */
+class MMonGetVersionReply : public Message {
+
+  static const int HEAD_VERSION = 2;
+
+public:
+  MMonGetVersionReply() : Message(CEPH_MSG_MON_GET_VERSION_REPLY, HEAD_VERSION) { }
+
+  const char *get_type_name() const {
+    return "mon_check_map_ack";
+  }
+
+  void print(ostream& o) const {
+    o << "mon_check_map_ack(handle=" << handle << " version=" << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(handle, payload);
+    ::encode(version, payload);
+    ::encode(oldest_version, payload);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(handle, p);
+    ::decode(version, p);
+    if (header.version >= 2)
+      ::decode(oldest_version, p);
+  }
+
+  ceph_tid_t handle;
+  version_t version;
+  version_t oldest_version;
+
+private:
+  ~MMonGetVersionReply() {}
+};
+
+#endif
diff --git a/ceph/src/messages/MMonGlobalID.h b/ceph/src/messages/MMonGlobalID.h
new file mode 100644
index 00000000..7ec1acc9
--- /dev/null
+++ b/ceph/src/messages/MMonGlobalID.h
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONGLOBALID_H
+#define CEPH_MMONGLOBALID_H
+
+#include "messages/PaxosServiceMessage.h"
+
+struct MMonGlobalID : public PaxosServiceMessage {
+  uint64_t old_max_id;
+  MMonGlobalID() : PaxosServiceMessage(MSG_MON_GLOBAL_ID, 0), old_max_id(0) { }
+private:
+  ~MMonGlobalID() {}
+
+public:
+  const char *get_type_name() const { return "global_id"; }
+  void print(ostream& out) const {
+    out << "global_id  (" << old_max_id << ")";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(old_max_id, p);
+  }
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(old_max_id, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonHealth.h b/ceph/src/messages/MMonHealth.h
new file mode 100644
index 00000000..9608647f
--- /dev/null
+++ b/ceph/src/messages/MMonHealth.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_MMON_HEALTH_H
+#define CEPH_MMON_HEALTH_H
+
+#include "msg/Message.h"
+#include "messages/MMonQuorumService.h"
+#include "mon/mon_types.h"
+
+struct MMonHealth : public MMonQuorumService
+{
+  static const int HEAD_VERSION = 1;
+
+  enum {
+    OP_TELL = 1,
+  };
+
+  int service_type;
+  int service_op;
+
+  // service specific data
+  DataStats data_stats;
+
+  MMonHealth() : MMonQuorumService(MSG_MON_HEALTH, HEAD_VERSION) { }
+  MMonHealth(uint32_t type, int op = 0) :
+    MMonQuorumService(MSG_MON_HEALTH, HEAD_VERSION),
+    service_type(type),
+    service_op(op)
+  { }
+
+private:
+  ~MMonHealth() { }
+
+public:
+  const char *get_type_name() const { return "mon_health"; }
+  const char *get_service_op_name() const {
+    switch (service_op) {
+    case OP_TELL: return "tell";
+    }
+    return "???";
+  }
+  void print(ostream &o) const {
+    o << "mon_health( service " << get_service_type()
+      << " op " << get_service_op_name()
+      << " e " << get_epoch() << " r " << get_round()
+      << " )";
+  }
+
+  int get_service_type() const {
+    return service_type;
+  }
+
+  int get_service_op() {
+    return service_op;
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    service_decode(p);
+    ::decode(service_type, p);
+    ::decode(service_op, p);
+    ::decode(data_stats, p);
+  }
+
+  void encode_payload(uint64_t features) {
+    service_encode();
+    ::encode(service_type, payload);
+    ::encode(service_op, payload);
+    ::encode(data_stats, payload);
+  }
+
+};
+
+#endif /* CEPH_MMON_HEALTH_H */
diff --git a/ceph/src/messages/MMonJoin.h b/ceph/src/messages/MMonJoin.h
new file mode 100644
index 00000000..89e6c804
--- /dev/null
+++ b/ceph/src/messages/MMonJoin.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONJOIN_H
+#define CEPH_MMONJOIN_H
+
+#include "messages/PaxosServiceMessage.h"
+
+#include 
+using std::vector;
+
+class MMonJoin : public PaxosServiceMessage {
+ public:
+  uuid_d fsid;
+  string name;
+  entity_addr_t addr;
+
+  MMonJoin() : PaxosServiceMessage(MSG_MON_JOIN, 0) {}
+  MMonJoin(uuid_d &f, string n, const entity_addr_t& a)
+    : PaxosServiceMessage(MSG_MON_JOIN, 0),
+      fsid(f), name(n), addr(a)
+  { }
+  
+private:
+  ~MMonJoin() {}
+
+public:  
+  const char *get_type_name() const { return "mon_join"; }
+  void print(ostream& o) const {
+    o << "mon_join(" << name << " " << addr << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(name, payload);
+    ::encode(addr, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(name, p);
+    ::decode(addr, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonMap.h b/ceph/src/messages/MMonMap.h
new file mode 100644
index 00000000..3e950fa6
--- /dev/null
+++ b/ceph/src/messages/MMonMap.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONMAP_H
+#define CEPH_MMONMAP_H
+
+#include "include/ceph_features.h"
+#include "msg/Message.h"
+#include "mon/MonMap.h"
+
+class MMonMap : public Message {
+public:
+  bufferlist monmapbl;
+
+  MMonMap() : Message(CEPH_MSG_MON_MAP) { }
+  MMonMap(bufferlist &bl) : Message(CEPH_MSG_MON_MAP) { 
+    monmapbl.claim(bl);
+  }
+private:
+  ~MMonMap() {}
+
+public:
+  const char *get_type_name() const { return "mon_map"; }
+
+  void encode_payload(uint64_t features) { 
+    if (monmapbl.length() && (features & CEPH_FEATURE_MONENC) == 0) {
+      // reencode old-format monmap
+      MonMap t;
+      t.decode(monmapbl);
+      monmapbl.clear();
+      t.encode(monmapbl, features);
+    }
+
+    ::encode(monmapbl, payload);
+  }
+  void decode_payload() { 
+    bufferlist::iterator p = payload.begin();
+    ::decode(monmapbl, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonPaxos.h b/ceph/src/messages/MMonPaxos.h
new file mode 100644
index 00000000..206586cc
--- /dev/null
+++ b/ceph/src/messages/MMonPaxos.h
@@ -0,0 +1,127 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MMONPAXOS_H
+#define CEPH_MMONPAXOS_H
+
+#include "messages/PaxosServiceMessage.h"
+#include "mon/mon_types.h"
+#include "include/ceph_features.h"
+
+class MMonPaxos : public Message {
+
+  static const int HEAD_VERSION = 3;
+  static const int COMPAT_VERSION = 3;
+
+ public:
+  // op types
+  const static int OP_COLLECT =   1; // proposer: propose round
+  const static int OP_LAST =      2; // voter:    accept proposed round
+  const static int OP_BEGIN =     3; // proposer: value proposed for this round
+  const static int OP_ACCEPT =    4; // voter:    accept propsed value
+  const static int OP_COMMIT =    5; // proposer: notify learners of agreed value
+  const static int OP_LEASE =     6; // leader: extend peon lease
+  const static int OP_LEASE_ACK = 7; // peon: lease ack
+  const static char *get_opname(int op) {
+    switch (op) {
+    case OP_COLLECT: return "collect";
+    case OP_LAST: return "last";
+    case OP_BEGIN: return "begin";
+    case OP_ACCEPT: return "accept";
+    case OP_COMMIT: return "commit";
+    case OP_LEASE: return "lease";
+    case OP_LEASE_ACK: return "lease_ack";
+    default: assert(0); return 0;
+    }
+  }
+
+  epoch_t epoch;   // monitor epoch
+  __s32 op;          // paxos op
+
+  version_t first_committed;  // i've committed to
+  version_t last_committed;  // i've committed to
+  version_t pn_from;         // i promise to accept after
+  version_t pn;              // with with proposal
+  version_t uncommitted_pn;     // previous pn, if we are a LAST with an uncommitted value
+  utime_t lease_timestamp;
+  utime_t sent_timestamp;
+
+  version_t latest_version;
+  bufferlist latest_value;
+
+  map values;
+
+  MMonPaxos() : Message(MSG_MON_PAXOS, HEAD_VERSION, COMPAT_VERSION) { }
+  MMonPaxos(epoch_t e, int o, utime_t now) : 
+    Message(MSG_MON_PAXOS, HEAD_VERSION, COMPAT_VERSION),
+    epoch(e),
+    op(o),
+    first_committed(0), last_committed(0), pn_from(0), pn(0), uncommitted_pn(0),
+    sent_timestamp(now),
+    latest_version(0) {
+  }
+
+private:
+  ~MMonPaxos() {}
+
+public:  
+  const char *get_type_name() const { return "paxos"; }
+  
+  void print(ostream& out) const {
+    out << "paxos(" << get_opname(op) 
+	<< " lc " << last_committed
+	<< " fc " << first_committed
+	<< " pn " << pn << " opn " << uncommitted_pn;
+    if (latest_version)
+      out << " latest " << latest_version << " (" << latest_value.length() << " bytes)";
+    out <<  ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    if ((features & CEPH_FEATURE_MONCLOCKCHECK) == 0)
+      header.version = 0;
+    ::encode(epoch, payload);
+    ::encode(op, payload);
+    ::encode(first_committed, payload);
+    ::encode(last_committed, payload);
+    ::encode(pn_from, payload);
+    ::encode(pn, payload);
+    ::encode(uncommitted_pn, payload);
+    ::encode(lease_timestamp, payload);
+    if (features & CEPH_FEATURE_MONCLOCKCHECK)
+      ::encode(sent_timestamp, payload);
+    ::encode(latest_version, payload);
+    ::encode(latest_value, payload);
+    ::encode(values, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+    ::decode(op, p);
+    ::decode(first_committed, p);
+    ::decode(last_committed, p);
+    ::decode(pn_from, p);   
+    ::decode(pn, p);   
+    ::decode(uncommitted_pn, p);
+    ::decode(lease_timestamp, p);
+    if (header.version >= 1)
+      ::decode(sent_timestamp, p);
+    ::decode(latest_version, p);
+    ::decode(latest_value, p);
+    ::decode(values, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonProbe.h b/ceph/src/messages/MMonProbe.h
new file mode 100644
index 00000000..2f8a60b3
--- /dev/null
+++ b/ceph/src/messages/MMonProbe.h
@@ -0,0 +1,128 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MMONPROBE_H
+#define CEPH_MMONPROBE_H
+
+#include "include/ceph_features.h"
+#include "msg/Message.h"
+#include "mon/MonMap.h"
+
+class MMonProbe : public Message {
+public:
+  static const int HEAD_VERSION = 6;
+  static const int COMPAT_VERSION = 5;
+
+  enum {
+    OP_PROBE = 1,
+    OP_REPLY = 2,
+    OP_SLURP = 3,
+    OP_SLURP_LATEST = 4,
+    OP_DATA = 5,
+    OP_MISSING_FEATURES = 6,
+  };
+
+  static const char *get_opname(int o) {
+    switch (o) {
+    case OP_PROBE: return "probe";
+    case OP_REPLY: return "reply";
+    case OP_SLURP: return "slurp";
+    case OP_SLURP_LATEST: return "slurp_latest";
+    case OP_DATA: return "data";
+    case OP_MISSING_FEATURES: return "missing_features";
+    default: assert(0); return 0;
+    }
+  }
+  
+  uuid_d fsid;
+  int32_t op;
+  string name;
+  set quorum;
+  bufferlist monmap_bl;
+  version_t paxos_first_version;
+  version_t paxos_last_version;
+  bool has_ever_joined;
+  uint64_t required_features;
+
+  MMonProbe()
+    : Message(MSG_MON_PROBE, HEAD_VERSION, COMPAT_VERSION) {}
+  MMonProbe(const uuid_d& f, int o, const string& n, bool hej)
+    : Message(MSG_MON_PROBE, HEAD_VERSION, COMPAT_VERSION),
+      fsid(f),
+      op(o),
+      name(n),
+      paxos_first_version(0),
+      paxos_last_version(0),
+      has_ever_joined(hej),
+      required_features(0) {}
+private:
+  ~MMonProbe() {}
+
+public:  
+  const char *get_type_name() const { return "mon_probe"; }
+  void print(ostream& out) const {
+    out << "mon_probe(" << get_opname(op) << " " << fsid << " name " << name;
+    if (quorum.size())
+      out << " quorum " << quorum;
+    if (op == OP_REPLY) {
+      out << " paxos("
+	<< " fc " << paxos_first_version
+	<< " lc " << paxos_last_version
+	<< " )";
+    }
+    if (!has_ever_joined)
+      out << " new";
+    if (required_features)
+      out << " required_features " << required_features;
+    out << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    if (monmap_bl.length() && (features & CEPH_FEATURE_MONENC) == 0) {
+      // reencode old-format monmap
+      MonMap t;
+      t.decode(monmap_bl);
+      monmap_bl.clear();
+      t.encode(monmap_bl, features);
+    }
+
+    ::encode(fsid, payload);
+    ::encode(op, payload);
+    ::encode(name, payload);
+    ::encode(quorum, payload);
+    ::encode(monmap_bl, payload);
+    ::encode(has_ever_joined, payload);
+    ::encode(paxos_first_version, payload);
+    ::encode(paxos_last_version, payload);
+    ::encode(required_features, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(fsid, p);
+    ::decode(op, p);
+    ::decode(name, p);
+    ::decode(quorum, p);
+    ::decode(monmap_bl, p);
+    ::decode(has_ever_joined, p);
+    ::decode(paxos_first_version, p);
+    ::decode(paxos_last_version, p);
+    if (header.version >= 6)
+      ::decode(required_features, p);
+    else
+      required_features = 0;
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonQuorumService.h b/ceph/src/messages/MMonQuorumService.h
new file mode 100644
index 00000000..6a25ae1e
--- /dev/null
+++ b/ceph/src/messages/MMonQuorumService.h
@@ -0,0 +1,72 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_MMON_QUORUM_SERVICE_H
+#define CEPH_MMON_QUORUM_SERVICE_H
+
+#include "msg/Message.h"
+
+struct MMonQuorumService : public Message
+{
+  epoch_t epoch;
+  version_t round;
+
+  MMonQuorumService(int type, int head=1, int compat=1) :
+    Message(type, head, compat),
+    epoch(0),
+    round(0)
+  { }
+
+protected:
+  ~MMonQuorumService() { }
+
+public:
+
+  void set_epoch(epoch_t e) {
+    epoch = e;
+  }
+
+  void set_round(version_t r) {
+    round = r;
+  }
+
+  epoch_t get_epoch() const {
+    return epoch;
+  }
+
+  version_t get_round() const {
+    return round;
+  }
+
+  void service_encode() {
+    ::encode(epoch, payload);
+    ::encode(round, payload);
+  }
+
+  void service_decode(bufferlist::iterator &p) {
+    ::decode(epoch, p);
+    ::decode(round, p);
+  }
+
+  void encode_payload(uint64_t features) {
+    assert(0 == "MMonQuorumService message must always be a base class");
+  }
+
+  void decode_payload() {
+    assert(0 == "MMonQuorumService message must always be a base class");
+  }
+
+  const char *get_type_name() const { return "quorum_service"; }
+};
+
+#endif /* CEPH_MMON_QUORUM_SERVICE_H */
diff --git a/ceph/src/messages/MMonScrub.h b/ceph/src/messages/MMonScrub.h
new file mode 100644
index 00000000..b16728bc
--- /dev/null
+++ b/ceph/src/messages/MMonScrub.h
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2013 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef CEPH_MMONSCRUB_H
+#define CEPH_MMONSCRUB_H
+
+#include "msg/Message.h"
+#include "mon/mon_types.h"
+
+class MMonScrub : public Message
+{
+  static const int HEAD_VERSION = 1;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  typedef enum {
+    OP_SCRUB = 1,         // leader->peon: scrub (a range of) keys
+    OP_RESULT = 2,        // peon->leader: result of a scrub
+  } op_type_t;
+
+  static const char *get_opname(op_type_t op) {
+    switch (op) {
+    case OP_SCRUB: return "scrub";
+    case OP_RESULT: return "result";
+    default: assert(0 == "unknown op type"); return NULL;
+    }
+  }
+
+  op_type_t op;
+  version_t version;
+  ScrubResult result;
+
+  MMonScrub()
+    : Message(MSG_MON_SCRUB, HEAD_VERSION, COMPAT_VERSION)
+  { }
+
+  MMonScrub(op_type_t op, version_t v)
+    : Message(MSG_MON_SCRUB, HEAD_VERSION, COMPAT_VERSION),
+      op(op), version(v)
+  { }
+
+  const char *get_type_name() const { return "mon_scrub"; }
+
+  void print(ostream& out) const {
+    out << "mon_scrub(" << get_opname((op_type_t)op);
+    out << " v " << version;
+    if (op == OP_RESULT)
+      out << " " << result;
+    out << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    uint8_t o = op;
+    ::encode(o, payload);
+    ::encode(version, payload);
+    ::encode(result, payload);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    uint8_t o;
+    ::decode(o, p);
+    op = (op_type_t)o;
+    ::decode(version, p);
+    ::decode(result, p);
+  }
+};
+
+#endif /* CEPH_MMONSCRUB_H */
diff --git a/ceph/src/messages/MMonSubscribe.h b/ceph/src/messages/MMonSubscribe.h
new file mode 100644
index 00000000..0233010f
--- /dev/null
+++ b/ceph/src/messages/MMonSubscribe.h
@@ -0,0 +1,95 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONSUBSCRIBE_H
+#define CEPH_MMONSUBSCRIBE_H
+
+#include "msg/Message.h"
+#include "include/ceph_features.h"
+
+/*
+ * compatibility with old crap
+ */
+struct ceph_mon_subscribe_item_old {
+	__le64 unused;
+	__le64 have;
+	__u8 onetime;
+} __attribute__ ((packed));
+WRITE_RAW_ENCODER(ceph_mon_subscribe_item_old)
+
+
+struct MMonSubscribe : public Message {
+
+  static const int HEAD_VERSION = 2;
+
+  map what;
+  
+  MMonSubscribe() : Message(CEPH_MSG_MON_SUBSCRIBE, HEAD_VERSION) { }
+private:
+  ~MMonSubscribe() {}
+
+public:  
+  void sub_want(const char *w, version_t start, unsigned flags) {
+    what[w].start = start;
+    what[w].flags = flags;
+  }
+
+  const char *get_type_name() const { return "mon_subscribe"; }
+  void print(ostream& o) const {
+    o << "mon_subscribe(" << what << ")";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    if (header.version < 2) {
+      map oldwhat;
+      ::decode(oldwhat, p);
+      what.clear();
+      for (map::iterator q = oldwhat.begin();
+	   q != oldwhat.end();
+	   q++) {
+	if (q->second.have)
+	  what[q->first].start = q->second.have + 1;
+	else
+	  what[q->first].start = 0;
+	what[q->first].flags = 0;
+	if (q->second.onetime)
+	  what[q->first].flags |= CEPH_SUBSCRIBE_ONETIME;
+      }
+    } else {
+      ::decode(what, p);
+    }
+  }
+  void encode_payload(uint64_t features) {
+    if (features & CEPH_FEATURE_SUBSCRIBE2) {
+      ::encode(what, payload);
+    } else {
+      header.version = 0;
+      map oldwhat;
+      for (map::iterator q = what.begin();
+	   q != what.end();
+	   q++) {
+	if (q->second.start)
+	  // warning: start=1 -> have=0, which was ambiguous
+	  oldwhat[q->first].have = q->second.start - 1;
+	else
+	  oldwhat[q->first].have = 0;
+	oldwhat[q->first].onetime = q->second.flags & CEPH_SUBSCRIBE_ONETIME;
+      }
+      ::encode(oldwhat, payload);
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonSubscribeAck.h b/ceph/src/messages/MMonSubscribeAck.h
new file mode 100644
index 00000000..a2a1cded
--- /dev/null
+++ b/ceph/src/messages/MMonSubscribeAck.h
@@ -0,0 +1,50 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MMONSUBSCRIBEACK_H
+#define CEPH_MMONSUBSCRIBEACK_H
+
+#include "msg/Message.h"
+
+struct MMonSubscribeAck : public Message {
+  __u32 interval;
+  uuid_d fsid;
+  
+  MMonSubscribeAck() : Message(CEPH_MSG_MON_SUBSCRIBE_ACK),
+		       interval(0) {
+    memset(&fsid, 0, sizeof(fsid));
+  }
+  MMonSubscribeAck(uuid_d& f, int i) : Message(CEPH_MSG_MON_SUBSCRIBE_ACK),
+				       interval(i), fsid(f) { }
+private:
+  ~MMonSubscribeAck() {}
+
+public:
+  const char *get_type_name() const { return "mon_subscribe_ack"; }
+  void print(ostream& o) const {
+    o << "mon_subscribe_ack(" << interval << "s)";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(interval, p);
+    ::decode(fsid, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(interval, payload);
+    ::encode(fsid, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MMonSync.h b/ceph/src/messages/MMonSync.h
new file mode 100644
index 00000000..48229d15
--- /dev/null
+++ b/ceph/src/messages/MMonSync.h
@@ -0,0 +1,111 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef CEPH_MMONSYNC_H
+#define CEPH_MMONSYNC_H
+
+#include "msg/Message.h"
+
+class MMonSync : public Message
+{
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 2;
+
+public:
+  /**
+  * Operation types
+  */
+  enum {
+    OP_GET_COOKIE_FULL = 1,   // -> start a session (full scan)
+    OP_GET_COOKIE_RECENT = 2, // -> start a session (only recent paxos events)
+    OP_COOKIE = 3,            // <- pass the iterator cookie, or
+    OP_GET_CHUNK = 4,         // -> get some keys
+    OP_CHUNK = 5,             // <- return some keys
+    OP_LAST_CHUNK = 6,        // <- return the last set of keys
+    OP_NO_COOKIE = 8,         // <- sorry, no cookie
+  };
+
+  /**
+  * Obtain a string corresponding to the operation type @p op
+  *
+  * @param op Operation type
+  * @returns A string
+  */
+  static const char *get_opname(int op) {
+    switch (op) {
+    case OP_GET_COOKIE_FULL: return "get_cookie_full";
+    case OP_GET_COOKIE_RECENT: return "get_cookie_recent";
+    case OP_COOKIE: return "cookie";
+    case OP_GET_CHUNK: return "get_chunk";
+    case OP_CHUNK: return "chunk";
+    case OP_LAST_CHUNK: return "last_chunk";
+    case OP_NO_COOKIE: return "no_cookie";
+    default: assert(0 == "unknown op type"); return NULL;
+    }
+  }
+
+  uint32_t op;
+  uint64_t cookie;
+  version_t last_committed;
+  pair last_key;
+  bufferlist chunk_bl;
+  entity_inst_t reply_to;
+
+  MMonSync()
+    : Message(MSG_MON_SYNC, HEAD_VERSION, COMPAT_VERSION)
+  { }
+
+  MMonSync(uint32_t op, uint64_t c = 0)
+    : Message(MSG_MON_SYNC, HEAD_VERSION, COMPAT_VERSION),
+      op(op),
+      cookie(c),
+      last_committed(0)
+  { }
+
+  const char *get_type_name() const { return "mon_sync"; }
+
+  void print(ostream& out) const {
+    out << "mon_sync(" << get_opname(op);
+    if (cookie)
+      out << " cookie " << cookie;
+    if (last_committed > 0)
+      out << " lc " << last_committed;
+    if (chunk_bl.length())
+      out << " bl " << chunk_bl.length() << " bytes";
+    if (!last_key.first.empty() || !last_key.second.empty())
+      out << " last_key " << last_key.first << "," << last_key.second;
+    out << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(op, payload);
+    ::encode(cookie, payload);
+    ::encode(last_committed, payload);
+    ::encode(last_key.first, payload);
+    ::encode(last_key.second, payload);
+    ::encode(chunk_bl, payload);
+    ::encode(reply_to, payload);
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(op, p);
+    ::decode(cookie, p);
+    ::decode(last_committed, p);
+    ::decode(last_key.first, p);
+    ::decode(last_key.second, p);
+    ::decode(chunk_bl, p);
+    ::decode(reply_to, p);
+  }
+};
+
+#endif /* CEPH_MMONSYNC_H */
diff --git a/ceph/src/messages/MOSDAlive.h b/ceph/src/messages/MOSDAlive.h
new file mode 100644
index 00000000..12bbed35
--- /dev/null
+++ b/ceph/src/messages/MOSDAlive.h
@@ -0,0 +1,49 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MOSDALIVE_H
+#define CEPH_MOSDALIVE_H
+
+#include "messages/PaxosServiceMessage.h"
+
+class MOSDAlive : public PaxosServiceMessage {
+ public:
+  epoch_t want;
+
+  MOSDAlive(epoch_t h, epoch_t w) : PaxosServiceMessage(MSG_OSD_ALIVE, h), want(w) { }
+  MOSDAlive() : PaxosServiceMessage(MSG_OSD_ALIVE, 0) {}
+private:
+  ~MOSDAlive() {}
+
+public:
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(want, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(want, p);
+  }
+
+  const char *get_type_name() const { return "osd_alive"; }
+  void print(ostream &out) const {
+    out << "osd_alive(want up_thru " << want << " have " << version << ")";
+  }
+  
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDBoot.h b/ceph/src/messages/MOSDBoot.h
new file mode 100644
index 00000000..bfe77757
--- /dev/null
+++ b/ceph/src/messages/MOSDBoot.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MOSDBOOT_H
+#define CEPH_MOSDBOOT_H
+
+#include "messages/PaxosServiceMessage.h"
+
+#include "include/types.h"
+#include "osd/osd_types.h"
+
+class MOSDBoot : public PaxosServiceMessage {
+
+  static const int HEAD_VERSION = 5;
+  static const int COMPAT_VERSION = 2;
+
+ public:
+  OSDSuperblock sb;
+  entity_addr_t hb_back_addr, hb_front_addr;
+  entity_addr_t cluster_addr;
+  epoch_t boot_epoch;  // last epoch this daemon was added to the map (if any)
+  map metadata; ///< misc metadata about this osd
+
+  MOSDBoot()
+    : PaxosServiceMessage(MSG_OSD_BOOT, 0, HEAD_VERSION, COMPAT_VERSION),
+      boot_epoch(0)
+  { }
+  MOSDBoot(OSDSuperblock& s, epoch_t be,
+	   const entity_addr_t& hb_back_addr_ref,
+	   const entity_addr_t& hb_front_addr_ref,
+           const entity_addr_t& cluster_addr_ref)
+    : PaxosServiceMessage(MSG_OSD_BOOT, s.current_epoch, HEAD_VERSION, COMPAT_VERSION),
+      sb(s),
+      hb_back_addr(hb_back_addr_ref),
+      hb_front_addr(hb_front_addr_ref),
+      cluster_addr(cluster_addr_ref),
+      boot_epoch(be)
+  { }
+  
+private:
+  ~MOSDBoot() { }
+
+public:
+  const char *get_type_name() const { return "osd_boot"; }
+  void print(ostream& out) const {
+    out << "osd_boot(osd." << sb.whoami << " booted " << boot_epoch << " v" << version << ")";
+  }
+  
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(sb, payload);
+    ::encode(hb_back_addr, payload);
+    ::encode(cluster_addr, payload);
+    ::encode(boot_epoch, payload);
+    ::encode(hb_front_addr, payload);
+    ::encode(metadata, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(sb, p);
+    ::decode(hb_back_addr, p);
+    if (header.version >= 2)
+      ::decode(cluster_addr, p);
+    if (header.version >= 3)
+      ::decode(boot_epoch, p);
+    if (header.version >= 4)
+      ::decode(hb_front_addr, p);
+    if (header.version >= 5)
+      ::decode(metadata, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDECSubOpRead.h b/ceph/src/messages/MOSDECSubOpRead.h
new file mode 100644
index 00000000..99e62e6c
--- /dev/null
+++ b/ceph/src/messages/MOSDECSubOpRead.h
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef MOSDECSUBOPREAD_H
+#define MOSDECSUBOPREAD_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
+
+class MOSDECSubOpRead : public Message {
+  static const int HEAD_VERSION = 1;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  spg_t pgid;
+  epoch_t map_epoch;
+  ECSubRead op;
+
+  int get_cost() const {
+    return 0;
+  }
+
+  MOSDECSubOpRead() :
+    Message(MSG_OSD_EC_READ, HEAD_VERSION, COMPAT_VERSION)
+    {}
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid, p);
+    ::decode(map_epoch, p);
+    ::decode(op, p);
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(pgid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(op, payload);
+  }
+
+  const char *get_type_name() const { return "MOSDECSubOpRead"; }
+
+  void print(ostream& out) const {
+    out << "MOSDECSubOpRead(" << pgid
+	<< " " << map_epoch
+	<< " " << op;
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDECSubOpReadReply.h b/ceph/src/messages/MOSDECSubOpReadReply.h
new file mode 100644
index 00000000..28e2cf7e
--- /dev/null
+++ b/ceph/src/messages/MOSDECSubOpReadReply.h
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef MOSDECSUBOPREADREPLY_H
+#define MOSDECSUBOPREADREPLY_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
+
+class MOSDECSubOpReadReply : public Message {
+  static const int HEAD_VERSION = 1;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  spg_t pgid;
+  epoch_t map_epoch;
+  ECSubReadReply op;
+
+  int get_cost() const {
+    return 0;
+  }
+
+  MOSDECSubOpReadReply() :
+    Message(MSG_OSD_EC_READ_REPLY, HEAD_VERSION, COMPAT_VERSION)
+    {}
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid, p);
+    ::decode(map_epoch, p);
+    ::decode(op, p);
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(pgid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(op, payload);
+  }
+
+  const char *get_type_name() const { return "MOSDECSubOpReadReply"; }
+
+  void print(ostream& out) const {
+    out << "MOSDECSubOpReadReply(" << pgid
+	<< " " << map_epoch
+	<< " " << op;
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDECSubOpWrite.h b/ceph/src/messages/MOSDECSubOpWrite.h
new file mode 100644
index 00000000..a47bcef1
--- /dev/null
+++ b/ceph/src/messages/MOSDECSubOpWrite.h
@@ -0,0 +1,70 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef MOSDECSUBOPWRITE_H
+#define MOSDECSUBOPWRITE_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
+
+class MOSDECSubOpWrite : public Message {
+  static const int HEAD_VERSION = 1;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  spg_t pgid;
+  epoch_t map_epoch;
+  ECSubWrite op;
+
+  int get_cost() const {
+    return 0;
+  }
+
+  MOSDECSubOpWrite()
+    : Message(MSG_OSD_EC_WRITE, HEAD_VERSION, COMPAT_VERSION)
+    {}
+  MOSDECSubOpWrite(ECSubWrite &op)
+  : Message(MSG_OSD_EC_WRITE, HEAD_VERSION, COMPAT_VERSION),
+    op(op) {}
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid, p);
+    ::decode(map_epoch, p);
+    ::decode(op, p);
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(pgid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(op, payload);
+  }
+
+  const char *get_type_name() const { return "MOSDECSubOpWrite"; }
+
+  void print(ostream& out) const {
+    out << "MOSDECSubOpWrite(" << pgid
+	<< " " << map_epoch
+	<< " " << op;
+    out << ")";
+  }
+
+  void clear_buffers() {
+    op.t = ObjectStore::Transaction();
+    op.log_entries.clear();
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDECSubOpWriteReply.h b/ceph/src/messages/MOSDECSubOpWriteReply.h
new file mode 100644
index 00000000..c2edfb38
--- /dev/null
+++ b/ceph/src/messages/MOSDECSubOpWriteReply.h
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef MOSDECSUBOPWRITEREPLY_H
+#define MOSDECSUBOPWRITEREPLY_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+#include "osd/ECMsgTypes.h"
+
+class MOSDECSubOpWriteReply : public Message {
+  static const int HEAD_VERSION = 1;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  spg_t pgid;
+  epoch_t map_epoch;
+  ECSubWriteReply op;
+
+  int get_cost() const {
+    return 0;
+  }
+
+  MOSDECSubOpWriteReply() :
+    Message(MSG_OSD_EC_WRITE_REPLY, HEAD_VERSION, COMPAT_VERSION)
+    {}
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid, p);
+    ::decode(map_epoch, p);
+    ::decode(op, p);
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(pgid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(op, payload);
+  }
+
+  const char *get_type_name() const { return "MOSDECSubOpWriteReply"; }
+
+  void print(ostream& out) const {
+    out << "MOSDECSubOpWriteReply(" << pgid
+	<< " " << map_epoch
+	<< " " << op;
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDFailure.h b/ceph/src/messages/MOSDFailure.h
new file mode 100644
index 00000000..a1032e64
--- /dev/null
+++ b/ceph/src/messages/MOSDFailure.h
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDFAILURE_H
+#define CEPH_MOSDFAILURE_H
+
+#include "messages/PaxosServiceMessage.h"
+
+
+class MOSDFailure : public PaxosServiceMessage {
+
+  static const int HEAD_VERSION = 3;
+
+ public:
+  uuid_d fsid;
+  entity_inst_t target_osd;
+  __u8 is_failed;
+  epoch_t       epoch;
+  int32_t failed_for;  // known to be failed since at least this long
+
+  MOSDFailure() : PaxosServiceMessage(MSG_OSD_FAILURE, 0, HEAD_VERSION) { }
+  MOSDFailure(const uuid_d &fs, const entity_inst_t& f, int duration, epoch_t e)
+    : PaxosServiceMessage(MSG_OSD_FAILURE, e, HEAD_VERSION),
+      fsid(fs), target_osd(f), is_failed(true), epoch(e), failed_for(duration) { }
+private:
+  ~MOSDFailure() {}
+
+public: 
+  entity_inst_t get_target() { return target_osd; }
+  bool if_osd_failed() { return is_failed; }
+  epoch_t get_epoch() { return epoch; }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(target_osd, p);
+    ::decode(epoch, p);
+    if (header.version >= 2)
+      ::decode(is_failed, p);
+    else
+      is_failed = true;
+    if (header.version >= 3)
+      ::decode(failed_for, p);
+    else
+      failed_for = 0;
+  }
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(target_osd, payload);
+    ::encode(epoch, payload);
+    ::encode(is_failed, payload);
+    ::encode(failed_for, payload);
+  }
+
+  const char *get_type_name() const { return "osd_failure"; }
+  void print(ostream& out) const {
+    out << "osd_failure("
+	<< (is_failed ? "failed " : "recovered ")
+	<< target_osd << " for " << failed_for << "sec e" << epoch
+	<< " v" << version << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDMap.h b/ceph/src/messages/MOSDMap.h
new file mode 100644
index 00000000..7ed601c6
--- /dev/null
+++ b/ceph/src/messages/MOSDMap.h
@@ -0,0 +1,144 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDMAP_H
+#define CEPH_MOSDMAP_H
+
+#include "msg/Message.h"
+#include "osd/OSDMap.h"
+#include "include/ceph_features.h"
+
+class MOSDMap : public Message {
+
+  static const int HEAD_VERSION = 3;
+
+ public:
+  uuid_d fsid;
+  map maps;
+  map incremental_maps;
+  epoch_t oldest_map, newest_map;
+
+  epoch_t get_first() const {
+    epoch_t e = 0;
+    map::const_iterator i = maps.begin();
+    if (i != maps.end())  e = i->first;
+    i = incremental_maps.begin();    
+    if (i != incremental_maps.end() &&
+        (e == 0 || i->first < e)) e = i->first;
+    return e;
+  }
+  epoch_t get_last() const {
+    epoch_t e = 0;
+    map::const_reverse_iterator i = maps.rbegin();
+    if (i != maps.rend())  e = i->first;
+    i = incremental_maps.rbegin();    
+    if (i != incremental_maps.rend() &&
+        (e == 0 || i->first > e)) e = i->first;
+    return e;
+  }
+  epoch_t get_oldest() {
+    return oldest_map;
+  }
+  epoch_t get_newest() {
+    return newest_map;
+  }
+
+
+  MOSDMap() : Message(CEPH_MSG_OSD_MAP, HEAD_VERSION) { }
+  MOSDMap(const uuid_d &f, OSDMap *oc=0)
+    : Message(CEPH_MSG_OSD_MAP, HEAD_VERSION),
+      fsid(f),
+      oldest_map(0), newest_map(0) {
+    if (oc)
+      oc->encode(maps[oc->get_epoch()]);
+  }
+private:
+  ~MOSDMap() {}
+
+public:
+  // marshalling
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(fsid, p);
+    ::decode(incremental_maps, p);
+    ::decode(maps, p);
+    if (header.version >= 2) {
+      ::decode(oldest_map, p);
+      ::decode(newest_map, p);
+    } else {
+      oldest_map = 0;
+      newest_map = 0;
+    }
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(fsid, payload);
+    if ((features & CEPH_FEATURE_PGID64) == 0 ||
+	(features & CEPH_FEATURE_PGPOOL3) == 0 ||
+	(features & CEPH_FEATURE_OSDENC) == 0 ||
+        (features & CEPH_FEATURE_OSDMAP_ENC) == 0) {
+      if ((features & CEPH_FEATURE_PGID64) == 0 ||
+	  (features & CEPH_FEATURE_PGPOOL3) == 0)
+	header.version = 1;  // old old_client version
+      else if ((features & CEPH_FEATURE_OSDENC) == 0)
+	header.version = 2;  // old pg_pool_t
+
+      // reencode maps using old format
+      //
+      // FIXME: this can probably be done more efficiently higher up
+      // the stack, or maybe replaced with something that only
+      // includes the pools the client cares about.
+      for (map::iterator p = incremental_maps.begin();
+	   p != incremental_maps.end();
+	   ++p) {
+	OSDMap::Incremental inc;
+	bufferlist::iterator q = p->second.begin();
+	inc.decode(q);
+	p->second.clear();
+	if (inc.fullmap.length()) {
+	  // embedded full map?
+	  OSDMap m;
+	  m.decode(inc.fullmap);
+	  inc.fullmap.clear();
+	  m.encode(inc.fullmap, features);
+	}
+	inc.encode(p->second, features);
+      }
+      for (map::iterator p = maps.begin();
+	   p != maps.end();
+	   ++p) {
+	OSDMap m;
+	m.decode(p->second);
+	p->second.clear();
+	m.encode(p->second, features);
+      }
+    }
+    ::encode(incremental_maps, payload);
+    ::encode(maps, payload);
+    if (header.version >= 2) {
+      ::encode(oldest_map, payload);
+      ::encode(newest_map, payload);
+    }
+  }
+
+  const char *get_type_name() const { return "omap"; }
+  void print(ostream& out) const {
+    out << "osd_map(" << get_first() << ".." << get_last();
+    if (oldest_map || newest_map)
+      out << " src has " << oldest_map << ".." << newest_map;
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDMarkMeDown.h b/ceph/src/messages/MOSDMarkMeDown.h
new file mode 100644
index 00000000..1a0475dc
--- /dev/null
+++ b/ceph/src/messages/MOSDMarkMeDown.h
@@ -0,0 +1,69 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MOSDMARKMEDOWN_H
+#define CEPH_MOSDMARKMEDOWN_H
+
+#include "messages/PaxosServiceMessage.h"
+
+class MOSDMarkMeDown : public PaxosServiceMessage {
+
+  static const int HEAD_VERSION = 1;
+
+ public:
+  uuid_d fsid;
+  entity_inst_t target_osd;
+  epoch_t epoch;
+  bool ack;
+
+  MOSDMarkMeDown()
+    : PaxosServiceMessage(MSG_OSD_MARK_ME_DOWN, 0, HEAD_VERSION) { }
+  MOSDMarkMeDown(const uuid_d &fs, const entity_inst_t& f,
+		 epoch_t e, bool ack)
+    : PaxosServiceMessage(MSG_OSD_MARK_ME_DOWN, e, HEAD_VERSION),
+      fsid(fs), target_osd(f), epoch(e), ack(ack) {}
+ private:
+  ~MOSDMarkMeDown() {}
+
+public: 
+  entity_inst_t get_target() { return target_osd; }
+  epoch_t get_epoch() { return epoch; }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(target_osd, p);
+    ::decode(epoch, p);
+    ::decode(ack, p);
+  }
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(target_osd, payload);
+    ::encode(epoch, payload);
+    ::encode(ack, payload);
+  }
+
+  const char *get_type_name() const { return "osd_mark_me_down"; }
+  void print(ostream& out) const {
+    out << "osd_mark_me_down("
+	<< "ack=" << ack
+	<< ", target_osd=" << target_osd
+	<< ", fsid=" << fsid
+	<< ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDOp.h b/ceph/src/messages/MOSDOp.h
new file mode 100644
index 00000000..ed0a669d
--- /dev/null
+++ b/ceph/src/messages/MOSDOp.h
@@ -0,0 +1,380 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDOP_H
+#define CEPH_MOSDOP_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+#include "include/ceph_features.h"
+
+/*
+ * OSD op
+ *
+ * oid - object id
+ * op  - OSD_OP_DELETE, etc.
+ *
+ */
+
+class OSD;
+
+class MOSDOp : public Message {
+
+  static const int HEAD_VERSION = 4;
+  static const int COMPAT_VERSION = 3;
+
+private:
+  uint32_t client_inc;
+  __u32 osdmap_epoch;
+  __u32 flags;
+  utime_t mtime;
+  eversion_t reassert_version;
+  int32_t retry_attempt;   // 0 is first attempt.  -1 if we don't know.
+
+  object_t oid;
+  object_locator_t oloc;
+  pg_t pgid;
+public:
+  vector ops;
+private:
+
+  snapid_t snapid;
+  snapid_t snap_seq;
+  vector snaps;
+
+public:
+  friend class MOSDOpReply;
+
+  // read
+  snapid_t get_snapid() { return snapid; }
+  void set_snapid(snapid_t s) { snapid = s; }
+  // writ
+  snapid_t get_snap_seq() const { return snap_seq; }
+  const vector &get_snaps() const { return snaps; }
+  void set_snaps(const vector& i) {
+    snaps = i;
+  }
+  void set_snap_seq(snapid_t s) { snap_seq = s; }
+
+  osd_reqid_t get_reqid() const {
+    return osd_reqid_t(get_orig_source(),
+		       client_inc,
+		       header.tid);
+  }
+  int get_client_inc() { return client_inc; }
+  ceph_tid_t get_client_tid() { return header.tid; }
+  
+  object_t& get_oid() { return oid; }
+
+  pg_t     get_pg() const { return pgid; }
+
+  object_locator_t get_object_locator() const {
+    return oloc;
+  }
+
+  epoch_t  get_map_epoch() { return osdmap_epoch; }
+
+  eversion_t get_version() { return reassert_version; }
+  
+  utime_t get_mtime() { return mtime; }
+
+  MOSDOp()
+    : Message(CEPH_MSG_OSD_OP, HEAD_VERSION, COMPAT_VERSION) { }
+  MOSDOp(int inc, long tid,
+         object_t& _oid, object_locator_t& _oloc, pg_t _pgid, epoch_t _osdmap_epoch,
+	 int _flags)
+    : Message(CEPH_MSG_OSD_OP, HEAD_VERSION, COMPAT_VERSION),
+      client_inc(inc),
+      osdmap_epoch(_osdmap_epoch), flags(_flags), retry_attempt(-1),
+      oid(_oid), oloc(_oloc), pgid(_pgid) {
+    set_tid(tid);
+  }
+private:
+  ~MOSDOp() {}
+
+public:
+  void set_version(eversion_t v) { reassert_version = v; }
+  void set_mtime(utime_t mt) { mtime = mt; }
+
+  // ops
+  void add_simple_op(int o, uint64_t off, uint64_t len) {
+    OSDOp osd_op;
+    osd_op.op.op = o;
+    osd_op.op.extent.offset = off;
+    osd_op.op.extent.length = len;
+    ops.push_back(osd_op);
+  }
+  void write(uint64_t off, uint64_t len, bufferlist& bl) {
+    add_simple_op(CEPH_OSD_OP_WRITE, off, len);
+    data.claim(bl);
+    header.data_off = off;
+  }
+  void writefull(bufferlist& bl) {
+    add_simple_op(CEPH_OSD_OP_WRITEFULL, 0, bl.length());
+    data.claim(bl);
+    header.data_off = 0;
+  }
+  void zero(uint64_t off, uint64_t len) {
+    add_simple_op(CEPH_OSD_OP_ZERO, off, len);
+  }
+  void truncate(uint64_t off) {
+    add_simple_op(CEPH_OSD_OP_TRUNCATE, off, 0);
+  }
+  void remove() {
+    add_simple_op(CEPH_OSD_OP_DELETE, 0, 0);
+  }
+
+  void read(uint64_t off, uint64_t len) {
+    add_simple_op(CEPH_OSD_OP_READ, off, len);
+  }
+  void stat() {
+    add_simple_op(CEPH_OSD_OP_STAT, 0, 0);
+  }
+
+  // flags
+  int get_flags() const { return flags; }
+
+  bool wants_ack() const { return flags & CEPH_OSD_FLAG_ACK; }
+  bool wants_ondisk() const { return flags & CEPH_OSD_FLAG_ONDISK; }
+  bool wants_onnvram() const { return flags & CEPH_OSD_FLAG_ONNVRAM; }
+
+  void set_want_ack(bool b) { flags |= CEPH_OSD_FLAG_ACK; }
+  void set_want_onnvram(bool b) { flags |= CEPH_OSD_FLAG_ONNVRAM; }
+  void set_want_ondisk(bool b) { flags |= CEPH_OSD_FLAG_ONDISK; }
+
+  bool is_retry_attempt() const { return flags & CEPH_OSD_FLAG_RETRY; }
+  void set_retry_attempt(unsigned a) { 
+    if (a)
+      flags |= CEPH_OSD_FLAG_RETRY;
+    else
+      flags &= ~CEPH_OSD_FLAG_RETRY;
+    retry_attempt = a;
+  }
+
+  /**
+   * get retry attempt
+   *
+   * 0 is the first attempt.
+   *
+   * @return retry attempt, or -1 if we don't know
+   */
+  int get_retry_attempt() const {
+    return retry_attempt;
+  }
+
+  // marshalling
+  virtual void encode_payload(uint64_t features) {
+
+    OSDOp::merge_osd_op_vector_in_data(ops, data);
+
+    if ((features & CEPH_FEATURE_OBJECTLOCATOR) == 0) {
+      // here is the old structure we are encoding to: //
+#if 0
+struct ceph_osd_request_head {
+	__le32 client_inc;                 /* client incarnation */
+	struct ceph_object_layout layout;  /* pgid */
+	__le32 osdmap_epoch;               /* client's osdmap epoch */
+
+	__le32 flags;
+
+	struct ceph_timespec mtime;        /* for mutations only */
+	struct ceph_eversion reassert_version; /* if we are replaying op */
+
+	__le32 object_len;     /* length of object name */
+
+	__le64 snapid;         /* snapid to read */
+	__le64 snap_seq;       /* writer's snap context */
+	__le32 num_snaps;
+
+	__le16 num_ops;
+	struct ceph_osd_op ops[];  /* followed by ops[], obj, ticket, snaps */
+} __attribute__ ((packed));
+#endif
+      header.version = 1;
+
+      ::encode(client_inc, payload);
+
+      __u32 su = 0;
+      ::encode(pgid, payload);
+      ::encode(su, payload);
+
+      ::encode(osdmap_epoch, payload);
+      ::encode(flags, payload);
+      ::encode(mtime, payload);
+      ::encode(reassert_version, payload);
+
+      __u32 oid_len = oid.name.length();
+      ::encode(oid_len, payload);
+      ::encode(snapid, payload);
+      ::encode(snap_seq, payload);
+      __u32 num_snaps = snaps.size();
+      ::encode(num_snaps, payload);
+      
+      //::encode(ops, payload);
+      __u16 num_ops = ops.size();
+      ::encode(num_ops, payload);
+      for (unsigned i = 0; i < ops.size(); i++)
+	::encode(ops[i].op, payload);
+
+      ::encode_nohead(oid.name, payload);
+      ::encode_nohead(snaps, payload);
+    } else {
+      ::encode(client_inc, payload);
+      ::encode(osdmap_epoch, payload);
+      ::encode(flags, payload);
+      ::encode(mtime, payload);
+      ::encode(reassert_version, payload);
+
+      ::encode(oloc, payload);
+      ::encode(pgid, payload);
+      ::encode(oid, payload);
+
+      __u16 num_ops = ops.size();
+      ::encode(num_ops, payload);
+      for (unsigned i = 0; i < ops.size(); i++)
+	::encode(ops[i].op, payload);
+
+      ::encode(snapid, payload);
+      ::encode(snap_seq, payload);
+      ::encode(snaps, payload);
+
+      ::encode(retry_attempt, payload);
+    }
+  }
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+
+    if (header.version < 2) {
+      // old decode
+      ::decode(client_inc, p);
+
+      old_pg_t opgid;
+      ::decode_raw(opgid, p);
+      pgid = opgid;
+
+      __u32 su;
+      ::decode(su, p);
+      oloc.pool = pgid.pool();
+
+      ::decode(osdmap_epoch, p);
+      ::decode(flags, p);
+      ::decode(mtime, p);
+      ::decode(reassert_version, p);
+
+      __u32 oid_len;
+      ::decode(oid_len, p);
+      ::decode(snapid, p);
+      ::decode(snap_seq, p);
+      __u32 num_snaps;
+      ::decode(num_snaps, p);
+      
+      //::decode(ops, p);
+      __u16 num_ops;
+      ::decode(num_ops, p);
+      ops.resize(num_ops);
+      for (unsigned i = 0; i < num_ops; i++)
+	::decode(ops[i].op, p);
+
+      decode_nohead(oid_len, oid.name, p);
+      decode_nohead(num_snaps, snaps, p);
+
+      // recalculate pgid hash value
+      pgid.set_ps(ceph_str_hash(CEPH_STR_HASH_RJENKINS,
+				oid.name.c_str(),
+				oid.name.length()));
+
+      retry_attempt = -1;
+    } else {
+      // new decode 
+      ::decode(client_inc, p);
+      ::decode(osdmap_epoch, p);
+      ::decode(flags, p);
+      ::decode(mtime, p);
+      ::decode(reassert_version, p);
+
+      ::decode(oloc, p);
+
+      if (header.version < 3) {
+	old_pg_t opgid;
+	::decode_raw(opgid, p);
+	pgid = opgid;
+      } else {
+	::decode(pgid, p);
+      }
+
+      ::decode(oid, p);
+
+      //::decode(ops, p);
+      __u16 num_ops;
+      ::decode(num_ops, p);
+      ops.resize(num_ops);
+      for (unsigned i = 0; i < num_ops; i++)
+	::decode(ops[i].op, p);
+
+      ::decode(snapid, p);
+      ::decode(snap_seq, p);
+      ::decode(snaps, p);
+
+      if (header.version >= 4)
+	::decode(retry_attempt, p);
+      else
+	retry_attempt = -1;
+    }
+
+    OSDOp::split_osd_op_vector_in_data(ops, data);
+  }
+
+  void clear_buffers() {
+    ops.clear();
+  }
+
+  const char *get_type_name() const { return "osd_op"; }
+  void print(ostream& out) const {
+    out << "osd_op(" << get_reqid();
+    out << " ";
+    if (!oloc.nspace.empty())
+      out << oloc.nspace << "/";
+    out << oid;
+
+#if 0
+    out << " ";
+    if (may_read())
+      out << "r";
+    if (may_write())
+      out << "w";
+#endif
+    if (snapid != CEPH_NOSNAP)
+      out << "@" << snapid;
+
+    if (oloc.key.size())
+      out << " " << oloc;
+
+    out << " " << ops;
+    out << " " << pgid;
+    if (is_retry_attempt())
+      out << " RETRY=" << get_retry_attempt();
+    if (reassert_version != eversion_t())
+      out << " reassert_version=" << reassert_version;
+    if (get_snap_seq())
+      out << " snapc " << get_snap_seq() << "=" << snaps;
+    out << " " << ceph_osd_flag_string(get_flags());
+    out << " e" << osdmap_epoch;
+    out << ")";
+  }
+};
+
+
+#endif
diff --git a/ceph/src/messages/MOSDOpReply.h b/ceph/src/messages/MOSDOpReply.h
new file mode 100644
index 00000000..91c50e76
--- /dev/null
+++ b/ceph/src/messages/MOSDOpReply.h
@@ -0,0 +1,277 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDOPREPLY_H
+#define CEPH_MOSDOPREPLY_H
+
+#include "msg/Message.h"
+
+#include "MOSDOp.h"
+#include "os/ObjectStore.h"
+#include "common/errno.h"
+
+/*
+ * OSD op reply
+ *
+ * oid - object id
+ * op  - OSD_OP_DELETE, etc.
+ *
+ */
+
+class MOSDOpReply : public Message {
+
+  static const int HEAD_VERSION = 6;
+  static const int COMPAT_VERSION = 2;
+
+  object_t oid;
+  pg_t pgid;
+  vector ops;
+  int64_t flags;
+  int32_t result;
+  eversion_t bad_replay_version;
+  eversion_t replay_version;
+  version_t user_version;
+  epoch_t osdmap_epoch;
+  int32_t retry_attempt;
+  request_redirect_t redirect;
+
+public:
+  object_t get_oid() const { return oid; }
+  pg_t     get_pg() const { return pgid; }
+  int      get_flags() const { return flags; }
+
+  bool     is_ondisk() const { return get_flags() & CEPH_OSD_FLAG_ONDISK; }
+  bool     is_onnvram() const { return get_flags() & CEPH_OSD_FLAG_ONNVRAM; }
+  
+  int get_result() const { return result; }
+  eversion_t get_replay_version() const { return replay_version; }
+  version_t get_user_version() const { return user_version; }
+  
+  void set_result(int r) { result = r; }
+
+  void set_reply_versions(eversion_t v, version_t uv) {
+    replay_version = v;
+    user_version = uv;
+    /* We go through some shenanigans here for backwards compatibility
+     * with old clients, who do not look at our replay_version and
+     * user_version but instead see what we now call the
+     * bad_replay_version. On pools without caching
+     * the user_version infrastructure is a slightly-laggy copy of
+     * the regular pg version/at_version infrastructure; the difference
+     * being it is not updated on watch ops like that is -- but on updates
+     * it is set equal to at_version. This means that for non-watch write ops
+     * on classic pools, all three of replay_version, user_version, and
+     * bad_replay_version are identical. But for watch ops the replay_version
+     * has been updated, while the user_at_version has not, and the semantics
+     * we promised old clients are that the version they see is not an update.
+     * So set the bad_replay_version to be the same as the user_at_version. */
+    bad_replay_version = v;
+    if (uv) {
+      bad_replay_version.version = uv;
+    }
+  }
+
+  /* Don't fill in replay_version for non-write ops */
+  void set_enoent_reply_versions(eversion_t v, version_t uv) {
+    user_version = uv;
+    bad_replay_version = v;
+  }
+
+  void set_redirect(const request_redirect_t& redir) { redirect = redir; }
+  const request_redirect_t& get_redirect() const { return redirect; }
+  bool is_redirect_reply() const { return !redirect.empty(); }
+
+  void add_flags(int f) { flags |= f; }
+
+  void claim_op_out_data(vector& o) {
+    assert(ops.size() == o.size());
+    for (unsigned i = 0; i < o.size(); i++) {
+      ops[i].outdata.claim(o[i].outdata);
+    }
+  }
+  void claim_ops(vector& o) {
+    o.swap(ops);
+  }
+
+  /**
+   * get retry attempt
+   *
+   * If we don't know the attempt (because the server is old), return -1.
+   */
+  int get_retry_attempt() const {
+    return retry_attempt;
+  }
+  
+  // osdmap
+  epoch_t get_map_epoch() const { return osdmap_epoch; }
+
+  /*osd_reqid_t get_reqid() { return osd_reqid_t(get_dest(),
+					       head.client_inc,
+					       head.tid); }
+  */
+
+public:
+  MOSDOpReply()
+    : Message(CEPH_MSG_OSD_OPREPLY, HEAD_VERSION, COMPAT_VERSION) { }
+  MOSDOpReply(MOSDOp *req, int r, epoch_t e, int acktype, bool ignore_out_data)
+    : Message(CEPH_MSG_OSD_OPREPLY, HEAD_VERSION, COMPAT_VERSION) {
+    set_tid(req->get_tid());
+    ops = req->ops;
+    result = r;
+    flags =
+      (req->flags & ~(CEPH_OSD_FLAG_ONDISK|CEPH_OSD_FLAG_ONNVRAM|CEPH_OSD_FLAG_ACK)) | acktype;
+    oid = req->oid;
+    pgid = req->pgid;
+    osdmap_epoch = e;
+    user_version = 0;
+    retry_attempt = req->get_retry_attempt();
+
+    // zero out ops payload_len and possibly out data
+    for (unsigned i = 0; i < ops.size(); i++) {
+      ops[i].op.payload_len = 0;
+      if (ignore_out_data)
+	ops[i].outdata.clear();
+    }
+  }
+private:
+  ~MOSDOpReply() {}
+
+public:
+  virtual void encode_payload(uint64_t features) {
+
+    OSDOp::merge_osd_op_vector_out_data(ops, data);
+
+    if ((features & CEPH_FEATURE_PGID64) == 0) {
+      header.version = 1;
+      ceph_osd_reply_head head;
+      memset(&head, 0, sizeof(head));
+      head.layout.ol_pgid = pgid.get_old_pg().v;
+      head.flags = flags;
+      head.osdmap_epoch = osdmap_epoch;
+      head.reassert_version = bad_replay_version;
+      head.result = result;
+      head.num_ops = ops.size();
+      head.object_len = oid.name.length();
+      ::encode(head, payload);
+      for (unsigned i = 0; i < head.num_ops; i++) {
+	::encode(ops[i].op, payload);
+      }
+      ::encode_nohead(oid.name, payload);
+    } else {
+      ::encode(oid, payload);
+      ::encode(pgid, payload);
+      ::encode(flags, payload);
+      ::encode(result, payload);
+      ::encode(bad_replay_version, payload);
+      ::encode(osdmap_epoch, payload);
+
+      __u32 num_ops = ops.size();
+      ::encode(num_ops, payload);
+      for (unsigned i = 0; i < num_ops; i++)
+	::encode(ops[i].op, payload);
+
+      ::encode(retry_attempt, payload);
+
+      for (unsigned i = 0; i < num_ops; i++)
+	::encode(ops[i].rval, payload);
+
+      ::encode(replay_version, payload);
+      ::encode(user_version, payload);
+      ::encode(redirect, payload);
+    }
+  }
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    if (header.version < 2) {
+      ceph_osd_reply_head head;
+      ::decode(head, p);
+      ops.resize(head.num_ops);
+      for (unsigned i = 0; i < head.num_ops; i++) {
+	::decode(ops[i].op, p);
+      }
+      ::decode_nohead(head.object_len, oid.name, p);
+      pgid = pg_t(head.layout.ol_pgid);
+      result = head.result;
+      flags = head.flags;
+      replay_version = head.reassert_version;
+      user_version = replay_version.version;
+      osdmap_epoch = head.osdmap_epoch;
+      retry_attempt = -1;
+    } else {
+      ::decode(oid, p);
+      ::decode(pgid, p);
+      ::decode(flags, p);
+      ::decode(result, p);
+      ::decode(bad_replay_version, p);
+      ::decode(osdmap_epoch, p);
+
+      __u32 num_ops = ops.size();
+      ::decode(num_ops, p);
+      ops.resize(num_ops);
+      for (unsigned i = 0; i < num_ops; i++)
+	::decode(ops[i].op, p);
+
+      if (header.version >= 3)
+	::decode(retry_attempt, p);
+      else
+	retry_attempt = -1;
+
+      if (header.version >= 4) {
+	for (unsigned i = 0; i < num_ops; ++i)
+	  ::decode(ops[i].rval, p);
+
+	OSDOp::split_osd_op_vector_out_data(ops, data);
+      }
+
+      if (header.version >= 5) {
+	::decode(replay_version, p);
+	::decode(user_version, p);
+      } else {
+	replay_version = bad_replay_version;
+	user_version = replay_version.version;
+      }
+
+      if (header.version >= 6)
+	::decode(redirect, p);
+    }
+  }
+
+  const char *get_type_name() const { return "osd_op_reply"; }
+  
+  void print(ostream& out) const {
+    out << "osd_op_reply(" << get_tid()
+	<< " " << oid << " " << ops
+	<< " v" << get_replay_version()
+	<< " uv" << get_user_version();
+    if (is_ondisk())
+      out << " ondisk";
+    else if (is_onnvram())
+      out << " onnvram";
+    else
+      out << " ack";
+    out << " = " << get_result();
+    if (get_result() < 0) {
+      out << " (" << cpp_strerror(get_result()) << ")";
+    }
+    if (is_redirect_reply()) {
+      out << " redirect: { " << redirect << " }";
+    }
+    out << ")";
+  }
+
+};
+
+
+#endif
diff --git a/ceph/src/messages/MOSDPGBackfill.h b/ceph/src/messages/MOSDPGBackfill.h
new file mode 100644
index 00000000..e9ec661c
--- /dev/null
+++ b/ceph/src/messages/MOSDPGBackfill.h
@@ -0,0 +1,111 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MOSDPGBACKFILL_H
+#define CEPH_MOSDPGBACKFILL_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+class MOSDPGBackfill : public Message {
+  static const int HEAD_VERSION = 3;
+  static const int COMPAT_VERSION = 1;
+public:
+  enum {
+    OP_BACKFILL_PROGRESS = 2,
+    OP_BACKFILL_FINISH = 3,
+    OP_BACKFILL_FINISH_ACK = 4,
+  };
+  const char *get_op_name(int o) const {
+    switch (o) {
+    case OP_BACKFILL_PROGRESS: return "progress";
+    case OP_BACKFILL_FINISH: return "finish";
+    case OP_BACKFILL_FINISH_ACK: return "finish_ack";
+    default: return "???";
+    }
+  }
+
+  __u32 op;
+  epoch_t map_epoch, query_epoch;
+  spg_t pgid;
+  hobject_t last_backfill;
+  bool compat_stat_sum;
+  pg_stat_t stats;
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(op, p);
+    ::decode(map_epoch, p);
+    ::decode(query_epoch, p);
+    ::decode(pgid.pgid, p);
+    ::decode(last_backfill, p);
+
+    // For compatibility with version 1
+    ::decode(stats.stats, p);
+
+    if (header.version >= 2) {
+      ::decode(stats, p);
+    } else {
+      compat_stat_sum = true;
+    }
+
+    // Handle hobject_t format change
+    if (!last_backfill.is_max() &&
+	last_backfill.pool == -1)
+      last_backfill.pool = pgid.pool();
+    if (header.version >= 3)
+      ::decode(pgid.shard, p);
+    else
+      pgid.shard = ghobject_t::no_shard();
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(op, payload);
+    ::encode(map_epoch, payload);
+    ::encode(query_epoch, payload);
+    ::encode(pgid.pgid, payload);
+    ::encode(last_backfill, payload);
+
+    // For compatibility with version 1
+    ::encode(stats.stats, payload);
+
+    ::encode(stats, payload);
+
+    ::encode(pgid.shard, payload);
+  }
+
+  MOSDPGBackfill() :
+    Message(MSG_OSD_PG_BACKFILL, HEAD_VERSION, COMPAT_VERSION),
+    compat_stat_sum(false) {}
+  MOSDPGBackfill(__u32 o, epoch_t e, epoch_t qe, spg_t p)
+    : Message(MSG_OSD_PG_BACKFILL, HEAD_VERSION, COMPAT_VERSION),
+      op(o),
+      map_epoch(e), query_epoch(e),
+      pgid(p),
+      compat_stat_sum(false) {}
+private:
+  ~MOSDPGBackfill() {}
+
+public:
+  const char *get_type_name() const { return "pg_backfill"; }
+  void print(ostream& out) const {
+    out << "pg_backfill(" << get_op_name(op)
+	<< " " << pgid
+	<< " e " << map_epoch << "/" << query_epoch
+	<< " lb " << last_backfill
+	<< ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGCreate.h b/ceph/src/messages/MOSDPGCreate.h
new file mode 100644
index 00000000..fcfb7678
--- /dev/null
+++ b/ceph/src/messages/MOSDPGCreate.h
@@ -0,0 +1,81 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDPGCREATE_H
+#define CEPH_MOSDPGCREATE_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+/*
+ * PGCreate - instruct an OSD to create a pg, if it doesn't already exist
+ */
+
+struct MOSDPGCreate : public Message {
+
+  const static int HEAD_VERSION = 2;
+
+  version_t          epoch;
+  map mkpg;
+
+  MOSDPGCreate()
+    : Message(MSG_OSD_PG_CREATE, HEAD_VERSION) {}
+  MOSDPGCreate(epoch_t e)
+    : Message(MSG_OSD_PG_CREATE, HEAD_VERSION),
+      epoch(e) { }
+private:
+  ~MOSDPGCreate() {}
+
+public:  
+  const char *get_type_name() const { return "pg_create"; }
+
+  void encode_payload(uint64_t features) {
+    ::encode(epoch, payload);
+    ::encode(mkpg, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+    if (header.version >= 2) {
+      ::decode(mkpg, p);
+    } else {
+      __u32 n;
+      ::decode(n, p);
+      while (n--) {
+	pg_t pgid;
+	epoch_t created;   // epoch pg created
+	pg_t parent;       // split from parent (if != pg_t())
+	__s32 split_bits;
+	::decode(pgid, p);
+	::decode(created, p);
+	::decode(parent, p);
+	::decode(split_bits, p);
+	mkpg[pgid] = pg_create_t(created, parent, split_bits);
+      }
+    }
+  }
+
+  void print(ostream& out) const {
+    out << "osd_pg_create(";
+    for (map::const_iterator i = mkpg.begin();
+         i != mkpg.end();
+         ++i) {
+      out << "pg" << i->first << "," << i->second.created << "; ";
+    }
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGInfo.h b/ceph/src/messages/MOSDPGInfo.h
new file mode 100644
index 00000000..83e74fb8
--- /dev/null
+++ b/ceph/src/messages/MOSDPGInfo.h
@@ -0,0 +1,137 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDPGINFO_H
+#define CEPH_MOSDPGINFO_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+class MOSDPGInfo : public Message {
+  static const int HEAD_VERSION = 4;
+  static const int COMPAT_VERSION = 1;
+
+  epoch_t epoch;
+
+public:
+  vector > pg_list;
+
+  epoch_t get_epoch() { return epoch; }
+
+  MOSDPGInfo()
+    : Message(MSG_OSD_PG_INFO, HEAD_VERSION, COMPAT_VERSION) {}
+  MOSDPGInfo(version_t mv)
+    : Message(MSG_OSD_PG_INFO, HEAD_VERSION, COMPAT_VERSION),
+      epoch(mv) { }
+private:
+  ~MOSDPGInfo() {}
+
+public:
+  const char *get_type_name() const { return "pg_info"; }
+  void print(ostream& out) const {
+    out << "pg_info(" << pg_list.size() << " pgs e" << epoch << ":";
+
+    for (vector >::const_iterator i = pg_list.begin();
+         i != pg_list.end();
+         ++i) {
+      if (i != pg_list.begin())
+	out << ",";
+      out << i->first.info.pgid;
+      if (i->second.size())
+	out << "(" << i->second.size() << ")";
+    }
+
+    out << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(epoch, payload);
+
+    // v1 was vector
+    __u32 n = pg_list.size();
+    ::encode(n, payload);
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 p++)
+      ::encode(p->first.info, payload);
+
+    // v2 needs the pg_interval_map_t for each record
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 p++)
+      ::encode(p->second, payload);
+
+    // v3 needs epoch_sent, query_epoch
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 p++)
+      ::encode(pair(
+		 p->first.epoch_sent, p->first.query_epoch), payload);
+
+    // v4 needs from, to
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 ++p) {
+      ::encode(p->first.from, payload);
+      ::encode(p->first.to, payload);
+    }
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+
+    // decode pg_info_t portion of the vector
+    __u32 n;
+    ::decode(n, p);
+    pg_list.resize(n);
+    for (unsigned i=0; i= 2) {
+      // get the pg_interval_map_t portion
+      for (unsigned i=0; i >::iterator i = pg_list.begin();
+	 i != pg_list.end();
+	 i++) {
+      if (header.version >= 3) {
+	pair dec;
+	::decode(dec, p);
+	i->first.epoch_sent = dec.first;
+	i->first.query_epoch = dec.second;
+      } else {
+	i->first.epoch_sent = epoch;
+	i->first.query_epoch = epoch;
+      }
+    }
+
+    // v4 needs from and to
+    if (header.version >= 4) {
+      for (vector >::iterator i = pg_list.begin();
+	   i != pg_list.end();
+	   i++) {
+	::decode(i->first.from, p);
+	::decode(i->first.to, p);
+      }
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGLog.h b/ceph/src/messages/MOSDPGLog.h
new file mode 100644
index 00000000..44cd9896
--- /dev/null
+++ b/ceph/src/messages/MOSDPGLog.h
@@ -0,0 +1,101 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDPGLOG_H
+#define CEPH_MOSDPGLOG_H
+
+#include "msg/Message.h"
+
+class MOSDPGLog : public Message {
+
+  static const int HEAD_VERSION = 4;
+  static const int COMPAT_VERSION = 2;
+
+  epoch_t epoch;
+  /// query_epoch is the epoch of the query being responded to, or
+  /// the current epoch if this is not being sent in response to a
+  /// query. This allows the recipient to disregard responses to old
+  /// queries.
+  epoch_t query_epoch;
+
+public:
+  shard_id_t to;
+  shard_id_t from;
+  pg_info_t info;
+  pg_log_t log;
+  pg_missing_t missing;
+  pg_interval_map_t past_intervals;
+
+  epoch_t get_epoch() { return epoch; }
+  spg_t get_pgid() { return spg_t(info.pgid.pgid, to); }
+  epoch_t get_query_epoch() { return query_epoch; }
+
+  MOSDPGLog() : Message(MSG_OSD_PG_LOG, HEAD_VERSION, COMPAT_VERSION) { }
+  MOSDPGLog(shard_id_t to, shard_id_t from, version_t mv, pg_info_t& i)
+    : Message(MSG_OSD_PG_LOG, HEAD_VERSION, COMPAT_VERSION),
+      epoch(mv), query_epoch(mv),
+      to(to), from(from),
+      info(i)  { }
+  MOSDPGLog(shard_id_t to, shard_id_t from,
+	    version_t mv, pg_info_t& i, epoch_t query_epoch)
+    : Message(MSG_OSD_PG_LOG, HEAD_VERSION, COMPAT_VERSION),
+      epoch(mv), query_epoch(query_epoch),
+      to(to), from(from),
+      info(i)  { }
+
+private:
+  ~MOSDPGLog() {}
+
+public:
+  const char *get_type_name() const { return "PGlog"; }
+  void print(ostream& out) const {
+    out << "pg_log(" << info.pgid << " epoch " << epoch
+	<< " log " << log
+	<< " query_epoch " << query_epoch << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(epoch, payload);
+    ::encode(info, payload);
+    ::encode(log, payload);
+    ::encode(missing, payload);
+    ::encode(query_epoch, payload);
+    ::encode(past_intervals, payload);
+    ::encode(to, payload);
+    ::encode(from, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+    ::decode(info, p);
+    log.decode(p, info.pgid.pool());
+    missing.decode(p, info.pgid.pool());
+    if (header.version >= 2) {
+      ::decode(query_epoch, p);
+    }
+    if (header.version >= 3) {
+      ::decode(past_intervals, p);
+    }
+    if (header.version >= 4) {
+      ::decode(to, p);
+      ::decode(from, p);
+    } else {
+      to = ghobject_t::NO_SHARD;
+      from = ghobject_t::NO_SHARD;
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGMissing.h b/ceph/src/messages/MOSDPGMissing.h
new file mode 100644
index 00000000..b931b363
--- /dev/null
+++ b/ceph/src/messages/MOSDPGMissing.h
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+
+#ifndef CEPH_MOSDPGMISSING_H
+#define CEPH_MOSDPGMISSING_H
+
+#include "msg/Message.h"
+
+class MOSDPGMissing : public Message {
+  epoch_t epoch;
+
+public:
+  pg_info_t info;
+  pg_missing_t missing;
+
+  epoch_t get_epoch() { return epoch; }
+
+  MOSDPGMissing() : Message(MSG_OSD_PG_MISSING) {}
+  MOSDPGMissing(version_t mv, const pg_info_t &info_,
+		const pg_missing_t &missing_)
+    : Message(MSG_OSD_PG_MISSING), epoch(mv), info(info_),
+      missing(missing_) { }
+private:
+  ~MOSDPGMissing() {}
+
+public:
+  const char *get_type_name() const { return "pg_missing"; }
+  void print(ostream& out) const {
+    out << "pg_missing(" << info.pgid << " e" << epoch << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(epoch, payload);
+    ::encode(info, payload);
+    ::encode(missing, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+    ::decode(info, p);
+    missing.decode(p, info.pgid.pool());
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGNotify.h b/ceph/src/messages/MOSDPGNotify.h
new file mode 100644
index 00000000..6b9bdb33
--- /dev/null
+++ b/ceph/src/messages/MOSDPGNotify.h
@@ -0,0 +1,158 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MOSDPGPEERNOTIFY_H
+#define CEPH_MOSDPGPEERNOTIFY_H
+
+#include "msg/Message.h"
+
+#include "osd/osd_types.h"
+
+/*
+ * PGNotify - notify primary of my PGs and versions.
+ */
+
+class MOSDPGNotify : public Message {
+
+  static const int HEAD_VERSION = 5;
+  static const int COMPAT_VERSION = 2;
+
+  epoch_t epoch;
+  /// query_epoch is the epoch of the query being responded to, or
+  /// the current epoch if this is not being sent in response to a
+  /// query. This allows the recipient to disregard responses to old
+  /// queries.
+  vector > pg_list;   // pgid -> version
+
+ public:
+  version_t get_epoch() { return epoch; }
+  vector >& get_pg_list() { return pg_list; }
+
+  MOSDPGNotify()
+    : Message(MSG_OSD_PG_NOTIFY, HEAD_VERSION, COMPAT_VERSION) { }
+  MOSDPGNotify(epoch_t e, vector >& l)
+    : Message(MSG_OSD_PG_NOTIFY, HEAD_VERSION, COMPAT_VERSION),
+      epoch(e) {
+    pg_list.swap(l);
+  }
+private:
+  ~MOSDPGNotify() {}
+
+public:  
+  const char *get_type_name() const { return "PGnot"; }
+
+  void encode_payload(uint64_t features) {
+    // Use query_epoch for first entry for backwards compatibility
+    epoch_t query_epoch = epoch;
+    if (pg_list.size())
+      query_epoch = pg_list.begin()->first.query_epoch;
+    
+    ::encode(epoch, payload);
+
+    // v2 was vector
+    __u32 n = pg_list.size();
+    ::encode(n, payload);
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 p++)
+      ::encode(p->first.info, payload);
+
+    ::encode(query_epoch, payload);
+
+    // v3 needs the pg_interval_map_t for each record
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 p++)
+      ::encode(p->second, payload);
+
+    // v4 needs epoch_sent, query_epoch
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 p++)
+      ::encode(pair(
+	  p->first.epoch_sent, p->first.query_epoch),
+	payload);
+
+    // v5 needs from, to
+    for (vector >::iterator p = pg_list.begin();
+	 p != pg_list.end();
+	 ++p) {
+      ::encode(p->first.from, payload);
+      ::encode(p->first.to, payload);
+    }
+  }
+  void decode_payload() {
+    epoch_t query_epoch;
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+
+    // decode pg_info_t portion of the vector
+    __u32 n;
+    ::decode(n, p);
+    pg_list.resize(n);
+    for (unsigned i=0; i= 3) {
+      // get the pg_interval_map_t portion
+      for (unsigned i=0; i >::iterator i = pg_list.begin();
+	 i != pg_list.end();
+	 i++) {
+      if (header.version >= 4) {
+	pair dec;
+	::decode(dec, p);
+	i->first.epoch_sent = dec.first;
+	i->first.query_epoch = dec.second;
+      } else {
+	i->first.epoch_sent = epoch;
+	i->first.query_epoch = query_epoch;
+      }
+    }
+
+    // v5 needs from and to
+    if (header.version >= 5) {
+      for (vector >::iterator i = pg_list.begin();
+	   i != pg_list.end();
+	   i++) {
+	::decode(i->first.from, p);
+	::decode(i->first.to, p);
+      }
+    }
+  }
+  void print(ostream& out) const {
+    out << "pg_notify(";
+    for (vector >::const_iterator i = pg_list.begin();
+         i != pg_list.end();
+         ++i) {
+      if (i != pg_list.begin())
+	out << ",";
+      out << i->first.info.pgid;
+      if (i->second.size())
+	out << "(" << i->second.size() << ")";
+    }
+    out << " epoch " << epoch
+	<< ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGPull.h b/ceph/src/messages/MOSDPGPull.h
new file mode 100644
index 00000000..91072320
--- /dev/null
+++ b/ceph/src/messages/MOSDPGPull.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef MOSDPGPULL_H
+#define MOSDPGPULL_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+class MOSDPGPull : public Message {
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+
+public:
+  pg_shard_t from;
+  spg_t pgid;
+  epoch_t map_epoch;
+  vector pulls;
+  uint64_t cost;
+
+  MOSDPGPull() :
+    Message(MSG_OSD_PG_PULL, HEAD_VERSION, COMPAT_VERSION),
+    cost(0)
+    {}
+
+  void compute_cost(CephContext *cct) {
+    cost = 0;
+    for (vector::iterator i = pulls.begin();
+	 i != pulls.end();
+	 ++i) {
+      cost += i->cost(cct);
+    }
+  }
+
+  int get_cost() const {
+    return cost;
+  }
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid.pgid, p);
+    ::decode(map_epoch, p);
+    ::decode(pulls, p);
+    ::decode(cost, p);
+    if (header.version >= 2) {
+      ::decode(pgid.shard, p);
+      ::decode(from, p);
+    } else {
+      pgid.shard = ghobject_t::NO_SHARD;
+      from = pg_shard_t(get_source().num(), ghobject_t::NO_SHARD);
+    }
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(pgid.pgid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(pulls, payload);
+    ::encode(cost, payload);
+    ::encode(pgid.shard, payload);
+    ::encode(from, payload);
+  }
+
+  const char *get_type_name() const { return "MOSDPGPull"; }
+
+  void print(ostream& out) const {
+    out << "MOSDPGPull(" << pgid
+	<< " " << map_epoch
+	<< " " << pulls;
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGPush.h b/ceph/src/messages/MOSDPGPush.h
new file mode 100644
index 00000000..46a8f1be
--- /dev/null
+++ b/ceph/src/messages/MOSDPGPush.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef MOSDPGPUSH_H
+#define MOSDPGPUSH_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+class MOSDPGPush : public Message {
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+
+public:
+  pg_shard_t from;
+  spg_t pgid;
+  epoch_t map_epoch;
+  vector pushes;
+  uint64_t cost;
+
+  void compute_cost(CephContext *cct) {
+    cost = 0;
+    for (vector::iterator i = pushes.begin();
+	 i != pushes.end();
+	 ++i) {
+      cost += i->cost(cct);
+    }
+  }
+
+  int get_cost() const {
+    return cost;
+  }
+
+  MOSDPGPush() :
+    Message(MSG_OSD_PG_PUSH, HEAD_VERSION, COMPAT_VERSION),
+    cost(0)
+    {}
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid.pgid, p);
+    ::decode(map_epoch, p);
+    ::decode(pushes, p);
+    ::decode(cost, p);
+    if (header.version >= 2) {
+      ::decode(pgid.shard, p);
+      ::decode(from, p);
+    } else {
+      pgid.shard = ghobject_t::NO_SHARD;
+      from = pg_shard_t(get_source().num(), ghobject_t::NO_SHARD);
+    }
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(pgid.pgid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(pushes, payload);
+    ::encode(cost, payload);
+    ::encode(pgid.shard, payload);
+    ::encode(from, payload);
+  }
+
+  const char *get_type_name() const { return "MOSDPGPush"; }
+
+  void print(ostream& out) const {
+    out << "MOSDPGPush(" << pgid
+	<< " " << map_epoch
+	<< " " << pushes;
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGPushReply.h b/ceph/src/messages/MOSDPGPushReply.h
new file mode 100644
index 00000000..1875235d
--- /dev/null
+++ b/ceph/src/messages/MOSDPGPushReply.h
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef MOSDPGPUSHREPLY_H
+#define MOSDPGPUSHREPLY_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+class MOSDPGPushReply : public Message {
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  pg_shard_t from;
+  spg_t pgid;
+  epoch_t map_epoch;
+  vector replies;
+  uint64_t cost;
+
+  MOSDPGPushReply() :
+    Message(MSG_OSD_PG_PUSH_REPLY, HEAD_VERSION, COMPAT_VERSION),
+    cost(0)
+    {}
+
+  void compute_cost(CephContext *cct) {
+    cost = 0;
+    for (vector::iterator i = replies.begin();
+	 i != replies.end();
+	 ++i) {
+      cost += i->cost(cct);
+    }
+  }
+
+  int get_cost() const {
+    return cost;
+  }
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid.pgid, p);
+    ::decode(map_epoch, p);
+    ::decode(replies, p);
+    ::decode(cost, p);
+
+    if (header.version >= 2) {
+      ::decode(pgid.shard, p);
+      ::decode(from, p);
+    } else {
+      pgid.shard = ghobject_t::NO_SHARD;
+      from = pg_shard_t(get_source().num(), ghobject_t::NO_SHARD);
+    }
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(pgid.pgid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(replies, payload);
+    ::encode(cost, payload);
+    ::encode(pgid.shard, payload);
+    ::encode(from, payload);
+  }
+
+  void print(ostream& out) const {
+    out << "MOSDPGPushReply(" << pgid
+	<< " " << map_epoch
+	<< " " << replies;
+    out << ")";
+  }
+
+  const char *get_type_name() const { return "MOSDPGPushReply"; }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGQuery.h b/ceph/src/messages/MOSDPGQuery.h
new file mode 100644
index 00000000..c2c6f695
--- /dev/null
+++ b/ceph/src/messages/MOSDPGQuery.h
@@ -0,0 +1,103 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDPGQUERY_H
+#define CEPH_MOSDPGQUERY_H
+
+#include "common/hobject.h"
+#include "msg/Message.h"
+
+/*
+ * PGQuery - query another OSD as to the contents of their PGs
+ */
+
+class MOSDPGQuery : public Message {
+  static const int HEAD_VERSION = 3;
+  static const int COMPAT_VERSION = 1;
+  version_t       epoch;
+
+ public:
+  version_t get_epoch() { return epoch; }
+  map  pg_list;
+
+  MOSDPGQuery() : Message(MSG_OSD_PG_QUERY,
+			  HEAD_VERSION,
+			  COMPAT_VERSION) {}
+  MOSDPGQuery(epoch_t e, map& ls) :
+    Message(MSG_OSD_PG_QUERY,
+	    HEAD_VERSION,
+	    COMPAT_VERSION),
+    epoch(e) {
+    pg_list.swap(ls);
+  }
+private:
+  ~MOSDPGQuery() {}
+
+public:  
+  const char *get_type_name() const { return "pg_query"; }
+  void print(ostream& out) const {
+    out << "pg_query(";
+    for (map::const_iterator p = pg_list.begin();
+	 p != pg_list.end(); ++p) {
+      if (p != pg_list.begin())
+	out << ",";
+      out << p->first;
+    }
+    out << " epoch " << epoch << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(epoch, payload);
+    vector > _pg_list;
+    _pg_list.reserve(pg_list.size());
+    vector _shard_list;
+    _shard_list.reserve(pg_list.size());
+    for (map::iterator i = pg_list.begin();
+	 i != pg_list.end();
+	 ++i) {
+      _pg_list.push_back(make_pair(i->first.pgid, i->second));
+      _shard_list.push_back(i->first.shard);
+    }
+    ::encode(_pg_list, payload, features);
+    ::encode(_shard_list, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+    vector > _pg_list;
+    ::decode(_pg_list, p);
+    vector _shard_list(_pg_list.size(), ghobject_t::no_shard());
+    if (header.version >= 3) {
+      _shard_list.clear();
+      ::decode(_shard_list, p);
+    }
+    assert(_pg_list.size() == _shard_list.size());
+    for (unsigned i = 0; i < _pg_list.size(); ++i) {
+      pg_list.insert(
+	make_pair(
+	  spg_t(_pg_list[i].first, _shard_list[i]), _pg_list[i].second));
+    }
+
+    if (header.version < 2) {
+      for (map::iterator i = pg_list.begin();
+	   i != pg_list.end();
+	   ++i) {
+	i->second.epoch_sent = epoch;
+      }
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGRemove.h b/ceph/src/messages/MOSDPGRemove.h
new file mode 100644
index 00000000..b55b5d28
--- /dev/null
+++ b/ceph/src/messages/MOSDPGRemove.h
@@ -0,0 +1,90 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDPGREMOVE_H
+#define CEPH_MOSDPGREMOVE_H
+
+#include "common/hobject.h"
+#include "msg/Message.h"
+
+
+class MOSDPGRemove : public Message {
+
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+  epoch_t epoch;
+
+ public:
+  vector pg_list;
+
+  epoch_t get_epoch() { return epoch; }
+
+  MOSDPGRemove() :
+    Message(MSG_OSD_PG_REMOVE, HEAD_VERSION, COMPAT_VERSION) {}
+  MOSDPGRemove(epoch_t e, vector& l) :
+    Message(MSG_OSD_PG_REMOVE, HEAD_VERSION, COMPAT_VERSION) {
+    this->epoch = e;
+    pg_list.swap(l);
+  }
+private:
+  ~MOSDPGRemove() {}
+
+public:  
+  const char *get_type_name() const { return "PGrm"; }
+
+  void encode_payload(uint64_t features) {
+    ::encode(epoch, payload);
+
+    vector _pg_list;
+    _pg_list.reserve(pg_list.size());
+    vector _shard_list;
+    _shard_list.reserve(pg_list.size());
+    for (vector::iterator i = pg_list.begin(); i != pg_list.end(); ++i) {
+      _pg_list.push_back(i->pgid);
+      _shard_list.push_back(i->shard);
+    }
+    ::encode(_pg_list, payload);
+    ::encode(_shard_list, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+    vector _pg_list;
+    ::decode(_pg_list, p);
+
+    vector _shard_list(_pg_list.size(), ghobject_t::no_shard());
+    if (header.version >= 2) {
+      _shard_list.clear();
+      ::decode(_shard_list, p);
+    }
+    assert(_shard_list.size() == _pg_list.size());
+    pg_list.reserve(_shard_list.size());
+    for (unsigned i = 0; i < _shard_list.size(); ++i) {
+      pg_list.push_back(spg_t(_pg_list[i], _shard_list[i]));
+    }
+  }
+  void print(ostream& out) const {
+    out << "osd pg remove(" << "epoch " << epoch << "; ";
+    for (vector::const_iterator i = pg_list.begin();
+         i != pg_list.end();
+         ++i) {
+      out << "pg" << *i << "; ";
+    }
+    out << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGScan.h b/ceph/src/messages/MOSDPGScan.h
new file mode 100644
index 00000000..2c0c1ad7
--- /dev/null
+++ b/ceph/src/messages/MOSDPGScan.h
@@ -0,0 +1,106 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MOSDPGSCAN_H
+#define CEPH_MOSDPGSCAN_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+class MOSDPGScan : public Message {
+
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  enum {
+    OP_SCAN_GET_DIGEST = 1,      // just objects and versions
+    OP_SCAN_DIGEST = 2,          // result
+  };
+  const char *get_op_name(int o) const {
+    switch (o) {
+    case OP_SCAN_GET_DIGEST: return "get_digest";
+    case OP_SCAN_DIGEST: return "digest";
+    default: return "???";
+    }
+  }
+
+  __u32 op;
+  epoch_t map_epoch, query_epoch;
+  pg_shard_t from;
+  spg_t pgid;
+  hobject_t begin, end;
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(op, p);
+    ::decode(map_epoch, p);
+    ::decode(query_epoch, p);
+    ::decode(pgid.pgid, p);
+    ::decode(begin, p);
+    ::decode(end, p);
+
+    // handle hobject_t format upgrade
+    if (!begin.is_max() && begin.pool == -1)
+      begin.pool = pgid.pool();
+    if (!end.is_max() && end.pool == -1)
+      end.pool = pgid.pool();
+
+    if (header.version >= 2) {
+      ::decode(from, p);
+      ::decode(pgid.shard, p);
+    } else {
+      from = pg_shard_t(
+	get_source().num(),
+	ghobject_t::NO_SHARD);
+      pgid.shard = ghobject_t::NO_SHARD;
+    }
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(op, payload);
+    ::encode(map_epoch, payload);
+    ::encode(query_epoch, payload);
+    ::encode(pgid.pgid, payload);
+    ::encode(begin, payload);
+    ::encode(end, payload);
+    ::encode(from, payload);
+    ::encode(pgid.shard, payload);
+  }
+
+  MOSDPGScan() : Message(MSG_OSD_PG_SCAN, HEAD_VERSION, COMPAT_VERSION) {}
+  MOSDPGScan(__u32 o, pg_shard_t from,
+	     epoch_t e, epoch_t qe, spg_t p, hobject_t be, hobject_t en)
+    : Message(MSG_OSD_PG_SCAN, HEAD_VERSION, COMPAT_VERSION),
+      op(o),
+      map_epoch(e), query_epoch(e),
+      from(from),
+      pgid(p),
+      begin(be), end(en) {
+  }
+private:
+  ~MOSDPGScan() {}
+
+public:
+  const char *get_type_name() const { return "pg_scan"; }
+  void print(ostream& out) const {
+    out << "pg_scan(" << get_op_name(op)
+	<< " " << pgid
+	<< " " << begin << "-" << end
+	<< " e " << map_epoch << "/" << query_epoch
+	<< ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGTemp.h b/ceph/src/messages/MOSDPGTemp.h
new file mode 100644
index 00000000..fe5908ce
--- /dev/null
+++ b/ceph/src/messages/MOSDPGTemp.h
@@ -0,0 +1,52 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MOSDPGTEMP_H
+#define CEPH_MOSDPGTEMP_H
+
+#include "messages/PaxosServiceMessage.h"
+
+class MOSDPGTemp : public PaxosServiceMessage {
+ public:
+  epoch_t map_epoch;
+  map > pg_temp;
+
+  MOSDPGTemp(epoch_t e) : PaxosServiceMessage(MSG_OSD_PGTEMP, e), map_epoch(e) { }
+  MOSDPGTemp() : PaxosServiceMessage(MSG_OSD_PGTEMP, 0) {}
+private:
+  ~MOSDPGTemp() {}
+
+public:
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(map_epoch, payload);
+    ::encode(pg_temp, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(map_epoch, p);
+    ::decode(pg_temp, p);
+  }
+
+  const char *get_type_name() const { return "osd_pgtemp"; }
+  void print(ostream &out) const {
+    out << "osd_pgtemp(e" << map_epoch << " " << pg_temp << " v" << version << ")";
+  }
+  
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPGTrim.h b/ceph/src/messages/MOSDPGTrim.h
new file mode 100644
index 00000000..12a0e7c8
--- /dev/null
+++ b/ceph/src/messages/MOSDPGTrim.h
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MOSDPGTRIM_H
+#define CEPH_MOSDPGTRIM_H
+
+#include "msg/Message.h"
+
+class MOSDPGTrim : public Message {
+
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  epoch_t epoch;
+  spg_t pgid;
+  eversion_t trim_to;
+
+  epoch_t get_epoch() { return epoch; }
+
+  MOSDPGTrim() : Message(MSG_OSD_PG_TRIM, HEAD_VERSION, COMPAT_VERSION) {}
+  MOSDPGTrim(version_t mv, spg_t p, eversion_t tt) :
+    Message(MSG_OSD_PG_TRIM),
+    epoch(mv), pgid(p), trim_to(tt) { }
+private:
+  ~MOSDPGTrim() {}
+
+public:
+  const char *get_type_name() const { return "pg_trim"; }
+  void print(ostream& out) const {
+    out << "pg_trim(" << pgid << " to " << trim_to << " e" << epoch << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(epoch, payload);
+    ::encode(pgid.pgid, payload);
+    ::encode(trim_to, payload);
+    ::encode(pgid.shard, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(epoch, p);
+    ::decode(pgid.pgid, p);
+    ::decode(trim_to, p);
+    if (header.version >= 2)
+      ::decode(pgid.shard, p);
+    else
+      pgid.shard = ghobject_t::no_shard();
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDPing.h b/ceph/src/messages/MOSDPing.h
new file mode 100644
index 00000000..4451a478
--- /dev/null
+++ b/ceph/src/messages/MOSDPing.h
@@ -0,0 +1,96 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MOSDPING_H
+#define CEPH_MOSDPING_H
+
+#include "common/Clock.h"
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+
+class MOSDPing : public Message {
+
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+ public:
+  enum {
+    HEARTBEAT = 0,
+    START_HEARTBEAT = 1,
+    YOU_DIED = 2,
+    STOP_HEARTBEAT = 3,
+    PING = 4,
+    PING_REPLY = 5,
+  };
+  const char *get_op_name(int op) const {
+    switch (op) {
+    case HEARTBEAT: return "heartbeat";
+    case START_HEARTBEAT: return "start_heartbeat";
+    case STOP_HEARTBEAT: return "stop_heartbeat";
+    case YOU_DIED: return "you_died";
+    case PING: return "ping";
+    case PING_REPLY: return "ping_reply";
+    default: return "???";
+    }
+  }
+
+  uuid_d fsid;
+  epoch_t map_epoch, peer_as_of_epoch;
+  __u8 op;
+  osd_peer_stat_t peer_stat;
+  utime_t stamp;
+
+  MOSDPing(const uuid_d& f, epoch_t e, __u8 o, utime_t s)
+    : Message(MSG_OSD_PING, HEAD_VERSION, COMPAT_VERSION),
+      fsid(f), map_epoch(e), peer_as_of_epoch(0), op(o), stamp(s)
+  { }
+  MOSDPing()
+    : Message(MSG_OSD_PING, HEAD_VERSION, COMPAT_VERSION)
+  {}
+private:
+  ~MOSDPing() {}
+
+public:
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(fsid, p);
+    ::decode(map_epoch, p);
+    ::decode(peer_as_of_epoch, p);
+    ::decode(op, p);
+    ::decode(peer_stat, p);
+    if (header.version >= 2)
+      ::decode(stamp, p);
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(fsid, payload);
+    ::encode(map_epoch, payload);
+    ::encode(peer_as_of_epoch, payload);
+    ::encode(op, payload);
+    ::encode(peer_stat, payload);
+    ::encode(stamp, payload);
+  }
+
+  const char *get_type_name() const { return "osd_ping"; }
+  void print(ostream& out) const {
+    out << "osd_ping(" << get_op_name(op)
+	<< " e" << map_epoch
+      //<< " as_of " << peer_as_of_epoch
+	<< " stamp " << stamp
+	<< ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDRepScrub.h b/ceph/src/messages/MOSDRepScrub.h
new file mode 100644
index 00000000..3f67021e
--- /dev/null
+++ b/ceph/src/messages/MOSDRepScrub.h
@@ -0,0 +1,120 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDREPSCRUB_H
+#define CEPH_MOSDREPSCRUB_H
+
+#include "msg/Message.h"
+
+/*
+ * instruct an OSD initiate a replica scrub on a specific PG
+ */
+
+struct MOSDRepScrub : public Message {
+
+  static const int HEAD_VERSION = 5;
+  static const int COMPAT_VERSION = 2;
+
+  spg_t pgid;             // PG to scrub
+  eversion_t scrub_from; // only scrub log entries after scrub_from
+  eversion_t scrub_to;   // last_update_applied when message sent
+  epoch_t map_epoch;
+  bool chunky;           // true for chunky scrubs
+  hobject_t start;       // lower bound of scrub, inclusive
+  hobject_t end;         // upper bound of scrub, exclusive
+  bool deep;             // true if scrub should be deep
+
+  MOSDRepScrub() : Message(MSG_OSD_REP_SCRUB, HEAD_VERSION, COMPAT_VERSION),
+      chunky(false),
+      deep(false) { }
+
+  MOSDRepScrub(spg_t pgid, eversion_t scrub_from, eversion_t scrub_to,
+	       epoch_t map_epoch)
+    : Message(MSG_OSD_REP_SCRUB, HEAD_VERSION, COMPAT_VERSION),
+      pgid(pgid),
+      scrub_from(scrub_from),
+      scrub_to(scrub_to),
+      map_epoch(map_epoch),
+      chunky(false),
+      deep(false) { }
+
+  MOSDRepScrub(spg_t pgid, eversion_t scrub_to, epoch_t map_epoch,
+               hobject_t start, hobject_t end, bool deep)
+    : Message(MSG_OSD_REP_SCRUB, HEAD_VERSION, COMPAT_VERSION),
+      pgid(pgid),
+      scrub_to(scrub_to),
+      map_epoch(map_epoch),
+      chunky(true),
+      start(start),
+      end(end),
+      deep(deep) { }
+
+
+private:
+  ~MOSDRepScrub() {}
+
+public:
+  const char *get_type_name() const { return "replica scrub"; }
+  void print(ostream& out) const {
+    out << "replica scrub(pg: ";
+    out << pgid << ",from:" << scrub_from << ",to:" << scrub_to
+        << ",epoch:" << map_epoch << ",start:" << start << ",end:" << end
+        << ",chunky:" << chunky
+        << ",deep:" << deep
+        << ",version:" << header.version;
+    out << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(pgid.pgid, payload);
+    ::encode(scrub_from, payload);
+    ::encode(scrub_to, payload);
+    ::encode(map_epoch, payload);
+    ::encode(chunky, payload);
+    ::encode(start, payload);
+    ::encode(end, payload);
+    ::encode(deep, payload);
+    ::encode(pgid.shard, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid.pgid, p);
+    ::decode(scrub_from, p);
+    ::decode(scrub_to, p);
+    ::decode(map_epoch, p);
+
+    if (header.version >= 3) {
+      ::decode(chunky, p);
+      ::decode(start, p);
+      ::decode(end, p);
+      if (header.version >= 4) {
+        ::decode(deep, p);
+      } else {
+        deep = false;
+      }
+    } else { // v2 scrub: non-chunky
+      chunky = false;
+      deep = false;
+    }
+
+    if (header.version >= 5) {
+      ::decode(pgid.shard, p);
+    } else {
+      pgid.shard = ghobject_t::no_shard();
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDScrub.h b/ceph/src/messages/MOSDScrub.h
new file mode 100644
index 00000000..72661f89
--- /dev/null
+++ b/ceph/src/messages/MOSDScrub.h
@@ -0,0 +1,79 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDSCRUB_H
+#define CEPH_MOSDSCRUB_H
+
+#include "msg/Message.h"
+
+/*
+ * instruct an OSD to scrub some or all pg(s)
+ */
+
+struct MOSDScrub : public Message {
+
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+
+  uuid_d fsid;
+  vector scrub_pgs;
+  bool repair;
+  bool deep;
+
+  MOSDScrub() : Message(MSG_OSD_SCRUB, HEAD_VERSION, COMPAT_VERSION) {}
+  MOSDScrub(const uuid_d& f, bool r, bool d) :
+    Message(MSG_OSD_SCRUB, HEAD_VERSION, COMPAT_VERSION),
+    fsid(f), repair(r), deep(d) {}
+  MOSDScrub(const uuid_d& f, vector& pgs, bool r, bool d) :
+    Message(MSG_OSD_SCRUB, HEAD_VERSION, COMPAT_VERSION),
+    fsid(f), scrub_pgs(pgs), repair(r), deep(d) {}
+private:
+  ~MOSDScrub() {}
+
+public:
+  const char *get_type_name() const { return "scrub"; }
+  void print(ostream& out) const {
+    out << "scrub(";
+    if (scrub_pgs.empty())
+      out << "osd";
+    else
+      out << scrub_pgs;
+    if (repair)
+      out << " repair";
+    if (deep)
+      out << " deep";
+    out << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(fsid, payload);
+    ::encode(scrub_pgs, payload);
+    ::encode(repair, payload);
+    ::encode(deep, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(fsid, p);
+    ::decode(scrub_pgs, p);
+    ::decode(repair, p);
+    if (header.version >= 2) {
+      ::decode(deep, p);
+    } else {
+      deep = false;
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MOSDSubOp.h b/ceph/src/messages/MOSDSubOp.h
new file mode 100644
index 00000000..7b40c0a6
--- /dev/null
+++ b/ceph/src/messages/MOSDSubOp.h
@@ -0,0 +1,283 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDSUBOP_H
+#define CEPH_MOSDSUBOP_H
+
+#include "msg/Message.h"
+#include "osd/osd_types.h"
+
+/*
+ * OSD sub op - for internal ops on pobjects between primary and replicas(/stripes/whatever)
+ */
+
+class MOSDSubOp : public Message {
+
+  static const int HEAD_VERSION = 11;
+  static const int COMPAT_VERSION = 1;
+
+public:
+  epoch_t map_epoch;
+  
+  // metadata from original request
+  osd_reqid_t reqid;
+  
+  // subop
+  pg_shard_t from;
+  spg_t pgid;
+  hobject_t poid;
+  object_locator_t oloc;
+  
+  __u8 acks_wanted;
+
+  // op to exec
+  vector ops;
+  utime_t mtime;
+  bool noop;
+
+  bool old_exists;
+  uint64_t old_size;
+  eversion_t old_version;
+
+  SnapSet snapset;
+  SnapContext snapc;
+
+  // transaction to exec
+  bufferlist logbl;
+  pg_stat_t pg_stats;
+  
+  // subop metadata
+  eversion_t version;
+
+  // piggybacked osd/og state
+  eversion_t pg_trim_to;   // primary->replica: trim to here
+  eversion_t pg_trim_rollback_to;   // primary->replica: trim rollback
+                                    // info to here
+  osd_peer_stat_t peer_stat;
+
+  map attrset;
+
+  interval_set data_subset;
+  map > clone_subsets;
+
+  bool first, complete;
+
+  interval_set data_included;
+  ObjectRecoveryInfo recovery_info;
+
+  // reflects result of current push
+  ObjectRecoveryProgress recovery_progress;
+
+  // reflects progress before current push
+  ObjectRecoveryProgress current_progress;
+
+  map omap_entries;
+  bufferlist omap_header;
+
+  // indicates that we must fix hobject_t encoding
+  bool hobject_incorrect_pool;
+
+  hobject_t new_temp_oid;      ///< new temp object that we must now start tracking
+  hobject_t discard_temp_oid;  ///< previously used temp object that we can now stop tracking
+
+  /// non-empty if this transaction involves a hit_set history update
+  boost::optional updated_hit_set_history;
+
+  int get_cost() const {
+    if (ops.size() == 1 && ops[0].op.op == CEPH_OSD_OP_PULL)
+      return ops[0].op.extent.length;
+    return data.length();
+  }
+
+  virtual void decode_payload() {
+    hobject_incorrect_pool = false;
+    bufferlist::iterator p = payload.begin();
+    ::decode(map_epoch, p);
+    ::decode(reqid, p);
+    ::decode(pgid.pgid, p);
+    ::decode(poid, p);
+
+    __u32 num_ops;
+    ::decode(num_ops, p);
+    ops.resize(num_ops);
+    unsigned off = 0;
+    for (unsigned i = 0; i < num_ops; i++) {
+      ::decode(ops[i].op, p);
+      ops[i].indata.substr_of(data, off, ops[i].op.payload_len);
+      off += ops[i].op.payload_len;
+    }
+    ::decode(mtime, p);
+    ::decode(noop, p);
+    ::decode(acks_wanted, p);
+    ::decode(version, p);
+    ::decode(old_exists, p);
+    ::decode(old_size, p);
+    ::decode(old_version, p);
+    ::decode(snapset, p);
+    ::decode(snapc, p);
+    ::decode(logbl, p);
+    ::decode(pg_stats, p);
+    ::decode(pg_trim_to, p);
+    ::decode(peer_stat, p);
+    ::decode(attrset, p);
+
+    ::decode(data_subset, p);
+    ::decode(clone_subsets, p);
+    
+    if (header.version >= 2) {
+      ::decode(first, p);
+      ::decode(complete, p);
+    }
+    if (header.version >= 3)
+      ::decode(oloc, p);
+    if (header.version >= 4) {
+      ::decode(data_included, p);
+      recovery_info.decode(p, pgid.pool());
+      ::decode(recovery_progress, p);
+      ::decode(current_progress, p);
+    }
+    if (header.version >= 5)
+      ::decode(omap_entries, p);
+    if (header.version >= 6)
+      ::decode(omap_header, p);
+
+    if (header.version < 7) {
+      // Handle hobject_t format change
+      if (!poid.is_max() && poid.pool == -1)
+	poid.pool = pgid.pool();
+      hobject_incorrect_pool = true;
+    }
+
+    if (header.version >= 8) {
+      ::decode(new_temp_oid, p);
+      ::decode(discard_temp_oid, p);
+    }
+
+    if (header.version >= 9) {
+      ::decode(from, p);
+      ::decode(pgid.shard, p);
+    } else {
+      from = pg_shard_t(
+	get_source().num(),
+	ghobject_t::NO_SHARD);
+      pgid.shard = ghobject_t::NO_SHARD;
+    }
+    if (header.version >= 10) {
+      ::decode(updated_hit_set_history, p);
+    }
+    if (header.version >= 11) {
+      ::decode(pg_trim_rollback_to, p);
+    } else {
+      pg_trim_rollback_to = pg_trim_to;
+    }
+  }
+
+  virtual void encode_payload(uint64_t features) {
+    ::encode(map_epoch, payload);
+    ::encode(reqid, payload);
+    ::encode(pgid.pgid, payload);
+    ::encode(poid, payload);
+
+    __u32 num_ops = ops.size();
+    ::encode(num_ops, payload);
+    for (unsigned i = 0; i < ops.size(); i++) {
+      ops[i].op.payload_len = ops[i].indata.length();
+      ::encode(ops[i].op, payload);
+      data.append(ops[i].indata);
+    }
+    ::encode(mtime, payload);
+    ::encode(noop, payload);
+    ::encode(acks_wanted, payload);
+    ::encode(version, payload);
+    ::encode(old_exists, payload);
+    ::encode(old_size, payload);
+    ::encode(old_version, payload);
+    ::encode(snapset, payload);
+    ::encode(snapc, payload);
+    ::encode(logbl, payload);
+    ::encode(pg_stats, payload);
+    ::encode(pg_trim_to, payload);
+    ::encode(peer_stat, payload);
+    ::encode(attrset, payload);
+    ::encode(data_subset, payload);
+    ::encode(clone_subsets, payload);
+    if (ops.size())
+      header.data_off = ops[0].op.extent.offset;
+    else
+      header.data_off = 0;
+    ::encode(first, payload);
+    ::encode(complete, payload);
+    ::encode(oloc, payload);
+    ::encode(data_included, payload);
+    ::encode(recovery_info, payload);
+    ::encode(recovery_progress, payload);
+    ::encode(current_progress, payload);
+    ::encode(omap_entries, payload);
+    ::encode(omap_header, payload);
+    ::encode(new_temp_oid, payload);
+    ::encode(discard_temp_oid, payload);
+    ::encode(from, payload);
+    ::encode(pgid.shard, payload);
+    ::encode(updated_hit_set_history, payload);
+    ::encode(pg_trim_rollback_to, payload);
+  }
+
+  MOSDSubOp()
+    : Message(MSG_OSD_SUBOP, HEAD_VERSION, COMPAT_VERSION) { }
+  MOSDSubOp(osd_reqid_t r, pg_shard_t from,
+	    spg_t p, const hobject_t& po, bool noop_, int aw,
+	    epoch_t mape, ceph_tid_t rtid, eversion_t v)
+    : Message(MSG_OSD_SUBOP, HEAD_VERSION, COMPAT_VERSION),
+      map_epoch(mape),
+      reqid(r),
+      from(from),
+      pgid(p),
+      poid(po),
+      acks_wanted(aw),
+      noop(noop_),   
+      old_exists(false), old_size(0),
+      version(v),
+      first(false), complete(false),
+      hobject_incorrect_pool(false) {
+    memset(&peer_stat, 0, sizeof(peer_stat));
+    set_tid(rtid);
+  }
+private:
+  ~MOSDSubOp() {}
+
+public:
+  const char *get_type_name() const { return "osd_sub_op"; }
+  void print(ostream& out) const {
+    out << "osd_sub_op(" << reqid
+	<< " " << pgid
+	<< " " << poid
+	<< " " << ops;
+    if (noop)
+      out << " (NOOP)";
+    if (first)
+      out << " first";
+    if (complete)
+      out << " complete";
+    out << " v " << version
+	<< " snapset=" << snapset << " snapc=" << snapc;    
+    if (!data_subset.empty()) out << " subset " << data_subset;
+    if (updated_hit_set_history)
+      out << ", has_updated_hit_set_history";
+    out << ")";
+  }
+};
+
+
+#endif
diff --git a/ceph/src/messages/MOSDSubOpReply.h b/ceph/src/messages/MOSDSubOpReply.h
new file mode 100644
index 00000000..270629f9
--- /dev/null
+++ b/ceph/src/messages/MOSDSubOpReply.h
@@ -0,0 +1,166 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MOSDSUBOPREPLY_H
+#define CEPH_MOSDSUBOPREPLY_H
+
+#include "msg/Message.h"
+
+#include "MOSDSubOp.h"
+#include "os/ObjectStore.h"
+
+/*
+ * OSD op reply
+ *
+ * oid - object id
+ * op  - OSD_OP_DELETE, etc.
+ *
+ */
+
+class MOSDSubOpReply : public Message {
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+public:
+  epoch_t map_epoch;
+  
+  // subop metadata
+  osd_reqid_t reqid;
+  pg_shard_t from;
+  spg_t pgid;
+  hobject_t poid;
+
+  vector ops;
+
+  // result
+  __u8 ack_type;
+  int32_t result;
+  
+  // piggybacked osd state
+  eversion_t last_complete_ondisk;
+  osd_peer_stat_t peer_stat;
+
+  map attrset;
+
+  virtual void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(map_epoch, p);
+    ::decode(reqid, p);
+    ::decode(pgid.pgid, p);
+    ::decode(poid, p);
+
+    unsigned num_ops;
+    ::decode(num_ops, p);
+    ops.resize(num_ops);
+    for (unsigned i = 0; i < num_ops; i++) {
+      ::decode(ops[i].op, p);
+    }
+    ::decode(ack_type, p);
+    ::decode(result, p);
+    ::decode(last_complete_ondisk, p);
+    ::decode(peer_stat, p);
+    ::decode(attrset, p);
+
+    if (!poid.is_max() && poid.pool == -1)
+      poid.pool = pgid.pool();
+
+    if (header.version >= 2) {
+      ::decode(from, p);
+      ::decode(pgid.shard, p);
+    } else {
+      from = pg_shard_t(
+	get_source().num(),
+	ghobject_t::NO_SHARD);
+      pgid.shard = ghobject_t::NO_SHARD;
+    }
+  }
+  virtual void encode_payload(uint64_t features) {
+    ::encode(map_epoch, payload);
+    ::encode(reqid, payload);
+    ::encode(pgid.pgid, payload);
+    ::encode(poid, payload);
+    __u32 num_ops = ops.size();
+    ::encode(num_ops, payload);
+    for (unsigned i = 0; i < ops.size(); i++) {
+      ::encode(ops[i].op, payload);
+    }
+    ::encode(ack_type, payload);
+    ::encode(result, payload);
+    ::encode(last_complete_ondisk, payload);
+    ::encode(peer_stat, payload);
+    ::encode(attrset, payload);
+    ::encode(from, payload);
+    ::encode(pgid.shard, payload);
+  }
+
+  epoch_t get_map_epoch() { return map_epoch; }
+
+  spg_t get_pg() { return pgid; }
+  hobject_t get_poid() { return poid; }
+
+  int get_ack_type() { return ack_type; }
+  bool is_ondisk() { return ack_type & CEPH_OSD_FLAG_ONDISK; }
+  bool is_onnvram() { return ack_type & CEPH_OSD_FLAG_ONNVRAM; }
+
+  int get_result() { return result; }
+
+  void set_last_complete_ondisk(eversion_t v) { last_complete_ondisk = v; }
+  eversion_t get_last_complete_ondisk() { return last_complete_ondisk; }
+
+  void set_peer_stat(const osd_peer_stat_t& stat) { peer_stat = stat; }
+  const osd_peer_stat_t& get_peer_stat() { return peer_stat; }
+
+  void set_attrset(map &as) { attrset = as; }
+  map& get_attrset() { return attrset; } 
+
+public:
+  MOSDSubOpReply(
+    MOSDSubOp *req, pg_shard_t from, int result_, epoch_t e, int at) :
+    Message(MSG_OSD_SUBOPREPLY, HEAD_VERSION, COMPAT_VERSION),
+    map_epoch(e),
+    reqid(req->reqid),
+    from(from),
+    pgid(req->pgid.pgid, req->from.shard),
+    poid(req->poid),
+    ops(req->ops),
+    ack_type(at),
+    result(result_) {
+    memset(&peer_stat, 0, sizeof(peer_stat));
+    set_tid(req->get_tid());
+  }
+  MOSDSubOpReply() : Message(MSG_OSD_SUBOPREPLY) {}
+private:
+  ~MOSDSubOpReply() {}
+
+public:
+  const char *get_type_name() const { return "osd_op_reply"; }
+  
+  void print(ostream& out) const {
+    out << "osd_sub_op_reply(" << reqid
+	<< " " << pgid 
+	<< " " << poid << " " << ops;
+    if (ack_type & CEPH_OSD_FLAG_ONDISK)
+      out << " ondisk";
+    if (ack_type & CEPH_OSD_FLAG_ONNVRAM)
+      out << " onnvram";
+    if (ack_type & CEPH_OSD_FLAG_ACK)
+      out << " ack";
+    out << ", result = " << result;
+    out << ")";
+  }
+
+};
+
+
+#endif
diff --git a/ceph/src/messages/MPGStats.h b/ceph/src/messages/MPGStats.h
new file mode 100644
index 00000000..8c40c4e5
--- /dev/null
+++ b/ceph/src/messages/MPGStats.h
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MPGSTATS_H
+#define CEPH_MPGSTATS_H
+
+#include "osd/osd_types.h"
+#include "messages/PaxosServiceMessage.h"
+
+class MPGStats : public PaxosServiceMessage {
+public:
+  uuid_d fsid;
+  map pg_stat;
+  osd_stat_t osd_stat;
+  epoch_t epoch;
+  utime_t had_map_for;
+  
+  MPGStats() : PaxosServiceMessage(MSG_PGSTATS, 0) {}
+  MPGStats(const uuid_d& f, epoch_t e, utime_t had)
+    : PaxosServiceMessage(MSG_PGSTATS, 0),
+      fsid(f),
+      epoch(e),
+      had_map_for(had)
+  {}
+
+private:
+  ~MPGStats() {};
+
+public:
+  const char *get_type_name() const { return "pg_stats"; }
+  void print(ostream& out) const {
+    out << "pg_stats(" << pg_stat.size() << " pgs tid " << get_tid() << " v " << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(osd_stat, payload);
+    ::encode(pg_stat, payload);
+    ::encode(epoch, payload);
+    ::encode(had_map_for, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(osd_stat, p);
+    ::decode(pg_stat, p);
+    ::decode(epoch, p);
+    ::decode(had_map_for, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MPGStatsAck.h b/ceph/src/messages/MPGStatsAck.h
new file mode 100644
index 00000000..0b7b99e8
--- /dev/null
+++ b/ceph/src/messages/MPGStatsAck.h
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MPGSTATSACK_H
+#define CEPH_MPGSTATSACK_H
+
+#include "osd/osd_types.h"
+
+class MPGStatsAck : public Message {
+public:
+  map > pg_stat;
+  
+  MPGStatsAck() : Message(MSG_PGSTATSACK) {}
+
+private:
+  ~MPGStatsAck() {}
+
+public:
+  const char *get_type_name() const { return "pg_stats_ack"; }
+  void print(ostream& out) const {
+    out << "pg_stats_ack(" << pg_stat.size() << " pgs tid " << get_tid() << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(pg_stat, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pg_stat, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MPing.h b/ceph/src/messages/MPing.h
new file mode 100644
index 00000000..3cd05149
--- /dev/null
+++ b/ceph/src/messages/MPing.h
@@ -0,0 +1,35 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MPING_H
+#define CEPH_MPING_H
+
+#include "msg/Message.h"
+#include "include/encoding.h"
+
+class MPing : public Message {
+ public:
+  MPing() : Message(CEPH_MSG_PING) {}
+private:
+  ~MPing() {}
+
+public:
+  void decode_payload() { }
+  void encode_payload(uint64_t features) { }
+  const char *get_type_name() const { return "ping"; }
+};
+
+#endif
diff --git a/ceph/src/messages/MPoolOp.h b/ceph/src/messages/MPoolOp.h
new file mode 100644
index 00000000..d8647412
--- /dev/null
+++ b/ceph/src/messages/MPoolOp.h
@@ -0,0 +1,101 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MPOOLOP_H
+#define CEPH_MPOOLOP_H
+
+#include "messages/PaxosServiceMessage.h"
+
+
+class MPoolOp : public PaxosServiceMessage {
+
+  static const int HEAD_VERSION = 4;
+  static const int COMPAT_VERSION = 2;
+
+public:
+  uuid_d fsid;
+  __u32 pool;
+  string name;
+  __u32 op;
+  uint64_t auid;
+  snapid_t snapid;
+  __s16 crush_rule;
+
+  MPoolOp()
+    : PaxosServiceMessage(CEPH_MSG_POOLOP, 0, HEAD_VERSION, COMPAT_VERSION) { }
+  MPoolOp(const uuid_d& f, ceph_tid_t t, int p, string& n, int o, version_t v)
+    : PaxosServiceMessage(CEPH_MSG_POOLOP, v, HEAD_VERSION, COMPAT_VERSION),
+      fsid(f), pool(p), name(n), op(o),
+      auid(0), snapid(0), crush_rule(0) {
+    set_tid(t);
+  }
+  MPoolOp(const uuid_d& f, ceph_tid_t t, int p, string& n,
+	  int o, uint64_t uid, version_t v)
+    : PaxosServiceMessage(CEPH_MSG_POOLOP, v, HEAD_VERSION, COMPAT_VERSION),
+      fsid(f), pool(p), name(n), op(o),
+      auid(uid), snapid(0), crush_rule(0) {
+    set_tid(t);
+  }
+
+private:
+  ~MPoolOp() {}
+
+public:
+  const char *get_type_name() const { return "poolop"; }
+  void print(ostream& out) const {
+    out << "pool_op(" << ceph_pool_op_name(op) << " pool " << pool
+	<< " auid " << auid
+	<< " tid " << get_tid()
+	<< " name " << name
+	<< " v" << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(pool, payload);
+    ::encode(op, payload);
+    ::encode(auid, payload);
+    ::encode(snapid, payload);
+    ::encode(name, payload);
+    __u8 pad = 0;
+    ::encode(pad, payload);  /* for v3->v4 encoding change */
+    ::encode(crush_rule, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(pool, p);
+    if (header.version < 2)
+      ::decode(name, p);
+    ::decode(op, p);
+    ::decode(auid, p);
+    ::decode(snapid, p);
+    if (header.version >= 2)
+      ::decode(name, p);
+
+    if (header.version >= 3) {
+      __u8 pad;
+      ::decode(pad, p);
+      if (header.version >= 4)
+	::decode(crush_rule, p);
+      else
+	crush_rule = pad;
+    } else
+      crush_rule = -1;
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MPoolOpReply.h b/ceph/src/messages/MPoolOpReply.h
new file mode 100644
index 00000000..d4fa50a6
--- /dev/null
+++ b/ceph/src/messages/MPoolOpReply.h
@@ -0,0 +1,80 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MPOOLOPREPLY_H
+#define CEPH_MPOOLOPREPLY_H
+
+#include "common/errno.h"
+
+class MPoolOpReply : public PaxosServiceMessage {
+public:
+  uuid_d fsid;
+  __u32 replyCode;
+  epoch_t epoch;
+  bufferlist response_data;
+
+  MPoolOpReply() : PaxosServiceMessage(CEPH_MSG_POOLOP_REPLY, 0)
+  {}
+  MPoolOpReply( uuid_d& f, ceph_tid_t t, int rc, int e, version_t v) :
+    PaxosServiceMessage(CEPH_MSG_POOLOP_REPLY, v),
+    fsid(f),
+    replyCode(rc),
+    epoch(e) {
+    set_tid(t);
+  }
+  MPoolOpReply( uuid_d& f, ceph_tid_t t, int rc, int e, version_t v,
+		bufferlist *blp) :
+    PaxosServiceMessage(CEPH_MSG_POOLOP_REPLY, v),
+    fsid(f),
+    replyCode(rc),
+    epoch(e) {
+    set_tid(t);
+    if (blp)
+      response_data.claim(*blp);
+  }
+
+  const char *get_type_name() const { return "poolopreply"; }
+
+  void print(ostream& out) const {
+    out << "pool_op_reply(tid " << get_tid()
+	<< " " << cpp_strerror(-replyCode)
+	<< " v" << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+    ::encode(replyCode, payload);
+    ::encode(epoch, payload);
+    if (response_data.length()) {
+      ::encode(true, payload);
+      ::encode(response_data, payload);
+    } else
+      ::encode(false, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+    ::decode(replyCode, p);
+    ::decode(epoch, p);
+    bool has_response_data;
+    ::decode(has_response_data, p);
+    if (has_response_data) {
+      ::decode(response_data, p);
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MRecoveryReserve.h b/ceph/src/messages/MRecoveryReserve.h
new file mode 100644
index 00000000..2a88bfde
--- /dev/null
+++ b/ceph/src/messages/MRecoveryReserve.h
@@ -0,0 +1,83 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MRECOVERY_H
+#define CEPH_MRECOVERY_H
+
+#include "msg/Message.h"
+
+class MRecoveryReserve : public Message {
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 1;
+public:
+  spg_t pgid;
+  epoch_t query_epoch;
+  enum {
+    REQUEST = 0,
+    GRANT = 1,
+    RELEASE = 2,
+  };
+  int type;
+
+  MRecoveryReserve()
+    : Message(MSG_OSD_RECOVERY_RESERVE, HEAD_VERSION, COMPAT_VERSION),
+      query_epoch(0), type(-1) {}
+  MRecoveryReserve(int type,
+		   spg_t pgid,
+		   epoch_t query_epoch)
+    : Message(MSG_OSD_RECOVERY_RESERVE, HEAD_VERSION, COMPAT_VERSION),
+      pgid(pgid), query_epoch(query_epoch),
+      type(type) {}
+
+  const char *get_type_name() const {
+    return "MRecoveryReserve";
+  }
+
+  void print(ostream& out) const {
+    out << "MRecoveryReserve ";
+    switch (type) {
+    case REQUEST:
+      out << "REQUEST ";
+      break;
+    case GRANT:
+      out << "GRANT ";
+      break;
+    case RELEASE:
+      out << "RELEASE ";
+      break;
+    }
+    out << " pgid: " << pgid << ", query_epoch: " << query_epoch;
+    return;
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(pgid.pgid, p);
+    ::decode(query_epoch, p);
+    ::decode(type, p);
+    if (header.version >= 2)
+      ::decode(pgid.shard, p);
+    else
+      pgid.shard = ghobject_t::no_shard();
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(pgid.pgid, payload);
+    ::encode(query_epoch, payload);
+    ::encode(type, payload);
+    ::encode(pgid.shard, payload);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MRemoveSnaps.h b/ceph/src/messages/MRemoveSnaps.h
new file mode 100644
index 00000000..6568ffae
--- /dev/null
+++ b/ceph/src/messages/MRemoveSnaps.h
@@ -0,0 +1,51 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MREMOVESNAPS_H
+#define CEPH_MREMOVESNAPS_H
+
+#include "messages/PaxosServiceMessage.h"
+
+struct MRemoveSnaps : public PaxosServiceMessage {
+  map > snaps;
+  
+  MRemoveSnaps() : 
+    PaxosServiceMessage(MSG_REMOVE_SNAPS, 0) { }
+  MRemoveSnaps(map >& s) : 
+    PaxosServiceMessage(MSG_REMOVE_SNAPS, 0) {
+    snaps.swap(s);
+  }
+private:
+  ~MRemoveSnaps() {}
+
+public:
+  const char *get_type_name() const { return "remove_snaps"; }
+  void print(ostream& out) const {
+    out << "remove_snaps(" << snaps << " v" << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(snaps, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(snaps, p);
+    assert(p.end());
+  }
+
+};
+
+#endif
diff --git a/ceph/src/messages/MRoute.h b/ceph/src/messages/MRoute.h
new file mode 100644
index 00000000..49f639ba
--- /dev/null
+++ b/ceph/src/messages/MRoute.h
@@ -0,0 +1,88 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MROUTE_H
+#define CEPH_MROUTE_H
+
+#include "msg/Message.h"
+#include "include/encoding.h"
+
+struct MRoute : public Message {
+
+  static const int HEAD_VERSION = 2;
+  static const int COMPAT_VERSION = 2;
+
+  uint64_t session_mon_tid;
+  Message *msg;
+  entity_inst_t dest;
+  
+  MRoute() : Message(MSG_ROUTE, HEAD_VERSION, COMPAT_VERSION), msg(NULL) {}
+  MRoute(uint64_t t, Message *m)
+    : Message(MSG_ROUTE, HEAD_VERSION, COMPAT_VERSION), session_mon_tid(t), msg(m) {}
+  MRoute(bufferlist bl, const entity_inst_t& i)
+    : Message(MSG_ROUTE, HEAD_VERSION, COMPAT_VERSION), session_mon_tid(0), dest(i) {
+    bufferlist::iterator p = bl.begin();
+    msg = decode_message(NULL, p);
+  }
+private:
+  ~MRoute() {
+    if (msg) msg->put();
+  }
+
+public:
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(session_mon_tid, p);
+    ::decode(dest, p);
+    if (header.version >= 2) {
+      bool m;
+      ::decode(m, p);
+      if (m)
+	msg = decode_message(NULL, p);
+    } else {
+      msg = decode_message(NULL, p);
+    }
+  }
+  void encode_payload(uint64_t features) {
+    ::encode(session_mon_tid, payload);
+    ::encode(dest, payload);
+    if (features & CEPH_FEATURE_MON_NULLROUTE) {
+      bool m = msg ? true : false;
+      ::encode(m, payload);
+      if (msg)
+	encode_message(msg, features, payload);
+    } else {
+      header.version = 1;
+      header.compat_version = 1;
+      assert(msg);
+      encode_message(msg, features, payload);
+    }
+  }
+
+  const char *get_type_name() const { return "route"; }
+  void print(ostream& o) const {
+    if (msg)
+      o << "route(" << *msg;
+    else
+      o << "route(no-reply";
+    if (session_mon_tid)
+      o << " tid " << session_mon_tid << ")";
+    else
+      o << " to " << dest << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MStatfs.h b/ceph/src/messages/MStatfs.h
new file mode 100644
index 00000000..ec19f6ee
--- /dev/null
+++ b/ceph/src/messages/MStatfs.h
@@ -0,0 +1,52 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MSTATFS_H
+#define CEPH_MSTATFS_H
+
+#include     /* or  */
+#include "messages/PaxosServiceMessage.h"
+
+class MStatfs : public PaxosServiceMessage {
+public:
+  uuid_d fsid;
+
+  MStatfs() : PaxosServiceMessage(CEPH_MSG_STATFS, 0) {}
+  MStatfs(const uuid_d& f, ceph_tid_t t, version_t v) :
+    PaxosServiceMessage(CEPH_MSG_STATFS, v), fsid(f) {
+    set_tid(t);
+  }
+
+private:
+  ~MStatfs() {}
+
+public:
+  const char *get_type_name() const { return "statfs"; }
+  void print(ostream& out) const {
+    out << "statfs(" << get_tid() << " v" << version << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    paxos_encode();
+    ::encode(fsid, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+    ::decode(fsid, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MStatfsReply.h b/ceph/src/messages/MStatfsReply.h
new file mode 100644
index 00000000..6bf42d89
--- /dev/null
+++ b/ceph/src/messages/MStatfsReply.h
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MSTATFSREPLY_H
+#define CEPH_MSTATFSREPLY_H
+
+class MStatfsReply : public Message {
+public:
+  struct ceph_mon_statfs_reply h;
+
+  MStatfsReply() : Message(CEPH_MSG_STATFS_REPLY) {}
+  MStatfsReply(uuid_d &f, ceph_tid_t t, epoch_t epoch) : Message(CEPH_MSG_STATFS_REPLY) {
+    memcpy(&h.fsid, f.uuid, sizeof(h.fsid));
+    header.tid = t;
+    h.version = epoch;
+  }
+
+  const char *get_type_name() const { return "statfs_reply"; }
+  void print(ostream& out) const {
+    out << "statfs_reply(" << header.tid << ")";
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(h, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(h, p);
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/MTimeCheck.h b/ceph/src/messages/MTimeCheck.h
new file mode 100644
index 00000000..64db94c9
--- /dev/null
+++ b/ceph/src/messages/MTimeCheck.h
@@ -0,0 +1,87 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#ifndef CEPH_MTIMECHECK_H
+#define CEPH_MTIMECHECK_H
+
+struct MTimeCheck : public Message
+{
+  static const int HEAD_VERSION = 1;
+
+  enum {
+    OP_PING = 1,
+    OP_PONG = 2,
+    OP_REPORT = 3,
+  };
+
+  int op;
+  version_t epoch;
+  version_t round;
+
+  utime_t timestamp;
+  map skews;
+  map latencies;
+
+  MTimeCheck() : Message(MSG_TIMECHECK, HEAD_VERSION) { }
+  MTimeCheck(int op) :
+    Message(MSG_TIMECHECK, HEAD_VERSION),
+    op(op)
+  { }
+
+private:
+  ~MTimeCheck() { }
+
+public:
+  const char *get_type_name() const { return "time_check"; }
+  const char *get_op_name() const {
+    switch (op) {
+    case OP_PING: return "ping";
+    case OP_PONG: return "pong";
+    case OP_REPORT: return "report";
+    }
+    return "???";
+  }
+  void print(ostream &o) const {
+    o << "time_check( " << get_op_name()
+      << " e " << epoch << " r " << round;
+    if (op == OP_PONG) {
+      o << " ts " << timestamp;
+    } else if (op == OP_REPORT) {
+      o << " #skews " << skews.size()
+        << " #latencies " << latencies.size();
+    }
+    o << " )";
+  }
+
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(op, p);
+    ::decode(epoch, p);
+    ::decode(round, p);
+    ::decode(timestamp, p);
+    ::decode(skews, p);
+    ::decode(latencies, p);
+  }
+
+  void encode_payload(uint64_t features) {
+    ::encode(op, payload);
+    ::encode(epoch, payload);
+    ::encode(round, payload);
+    ::encode(timestamp, payload);
+    ::encode(skews, payload);
+    ::encode(latencies, payload);
+  }
+
+};
+
+#endif /* CEPH_MTIMECHECK_H */
diff --git a/ceph/src/messages/MWatchNotify.h b/ceph/src/messages/MWatchNotify.h
new file mode 100644
index 00000000..66632f72
--- /dev/null
+++ b/ceph/src/messages/MWatchNotify.h
@@ -0,0 +1,64 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MWATCHNOTIFY_H
+#define CEPH_MWATCHNOTIFY_H
+
+#include "msg/Message.h"
+
+
+class MWatchNotify : public Message {
+ public:
+  uint64_t cookie;
+  uint64_t ver;
+  uint64_t notify_id;
+  uint8_t opcode;
+  bufferlist bl;
+
+  MWatchNotify() : Message(CEPH_MSG_WATCH_NOTIFY) { }
+  MWatchNotify(uint64_t c, uint64_t v, uint64_t i, uint8_t o, bufferlist b) : Message(CEPH_MSG_WATCH_NOTIFY),
+					cookie(c), ver(v), notify_id(i), opcode(o), bl(b) { }
+private:
+  ~MWatchNotify() {}
+
+public:
+  void decode_payload() {
+    uint8_t msg_ver;
+    bufferlist::iterator p = payload.begin();
+    ::decode(msg_ver, p);
+    ::decode(opcode, p);
+    ::decode(cookie, p);
+    ::decode(ver, p);
+    ::decode(notify_id, p);
+    if (msg_ver >= 1)
+      ::decode(bl, p);
+  }
+  void encode_payload(uint64_t features) {
+    uint8_t msg_ver = 1;
+    ::encode(msg_ver, payload);
+    ::encode(opcode, payload);
+    ::encode(cookie, payload);
+    ::encode(ver, payload);
+    ::encode(notify_id, payload);
+    ::encode(bl, payload);
+  }
+
+  const char *get_type_name() const { return "watch-notify"; }
+  void print(ostream& out) const {
+    out << "watch-notify(c=" << cookie << " v=" << ver << " i=" << notify_id << " opcode=" << (int)opcode << ")";
+  }
+};
+
+#endif
diff --git a/ceph/src/messages/Makefile.am b/ceph/src/messages/Makefile.am
new file mode 100644
index 00000000..cac40482
--- /dev/null
+++ b/ceph/src/messages/Makefile.am
@@ -0,0 +1,117 @@
+
+noinst_HEADERS += \
+	messages/MAuth.h \
+	messages/MAuthReply.h \
+	messages/MCacheExpire.h \
+	messages/MClientCaps.h \
+	messages/MClientCapRelease.h \
+	messages/MClientLease.h \
+	messages/MClientReconnect.h \
+	messages/MClientReply.h \
+	messages/MClientRequest.h \
+	messages/MClientRequestForward.h \
+	messages/MClientSession.h \
+	messages/MClientSnap.h \
+	messages/MCommand.h \
+	messages/MCommandReply.h \
+	messages/MDentryLink.h \
+	messages/MDentryUnlink.h \
+	messages/MDirUpdate.h \
+	messages/MDiscover.h \
+	messages/MDiscoverReply.h \
+	messages/MExportCaps.h \
+	messages/MExportCapsAck.h \
+	messages/MExportDir.h \
+	messages/MExportDirAck.h \
+	messages/MExportDirCancel.h \
+	messages/MExportDirDiscover.h \
+	messages/MExportDirDiscoverAck.h \
+	messages/MExportDirFinish.h \
+	messages/MExportDirNotify.h \
+	messages/MExportDirNotifyAck.h \
+	messages/MExportDirPrep.h \
+	messages/MExportDirPrepAck.h \
+	messages/MGenericMessage.h \
+	messages/MGetPoolStats.h \
+	messages/MGetPoolStatsReply.h \
+	messages/MHeartbeat.h \
+	messages/MInodeFileCaps.h \
+	messages/MLock.h \
+	messages/MLog.h \
+	messages/MLogAck.h \
+	messages/MMDSBeacon.h \
+	messages/MMDSCacheRejoin.h \
+	messages/MMDSLoadTargets.h \
+	messages/MMDSFindIno.h \
+	messages/MMDSFindInoReply.h \
+	messages/MMDSFragmentNotify.h \
+	messages/MMDSMap.h \
+	messages/MMDSOpenIno.h \
+	messages/MMDSOpenInoReply.h \
+	messages/MMDSResolve.h \
+	messages/MMDSResolveAck.h \
+	messages/MMDSSlaveRequest.h \
+	messages/MMDSTableRequest.h \
+	messages/MMonCommand.h \
+	messages/MMonCommandAck.h \
+	messages/MMonElection.h \
+	messages/MMonGetMap.h \
+	messages/MMonGetVersion.h \
+	messages/MMonGetVersionReply.h \
+	messages/MMonGlobalID.h \
+	messages/MMonHealth.h \
+	messages/MMonJoin.h \
+	messages/MMonMap.h \
+	messages/MMonPaxos.h \
+	messages/MMonProbe.h \
+	messages/MMonScrub.h \
+	messages/MMonSubscribe.h \
+	messages/MMonSubscribeAck.h \
+	messages/MMonSync.h \
+	messages/MOSDAlive.h \
+	messages/MOSDBoot.h \
+	messages/MOSDFailure.h \
+	messages/MOSDMarkMeDown.h \
+	messages/MOSDMap.h \
+	messages/MOSDOp.h \
+	messages/MOSDOpReply.h \
+	messages/MOSDPGBackfill.h \
+	messages/MOSDPGCreate.h \
+	messages/MOSDPGPush.h \
+	messages/MOSDPGPull.h \
+	messages/MOSDPGPushReply.h \
+	messages/MOSDPGInfo.h \
+	messages/MOSDPGLog.h \
+	messages/MOSDPGMissing.h \
+	messages/MOSDPGNotify.h \
+	messages/MOSDPGQuery.h \
+	messages/MOSDPGRemove.h \
+	messages/MOSDPGScan.h \
+	messages/MOSDECSubOpWrite.h \
+	messages/MOSDECSubOpWriteReply.h \
+	messages/MOSDECSubOpRead.h \
+	messages/MOSDECSubOpReadReply.h \
+	messages/MBackfillReserve.h \
+	messages/MRecoveryReserve.h \
+	messages/MMonQuorumService.h \
+	messages/MOSDPGTemp.h \
+	messages/MOSDPGTrim.h \
+	messages/MOSDPing.h \
+	messages/MOSDRepScrub.h \
+	messages/MOSDScrub.h \
+	messages/MOSDSubOp.h \
+	messages/MOSDSubOpReply.h \
+	messages/MPGStats.h \
+	messages/MPGStatsAck.h \
+	messages/MPing.h \
+	messages/MPoolOp.h \
+	messages/MPoolOpReply.h \
+	messages/MRemoveSnaps.h \
+	messages/MRoute.h \
+	messages/MForward.h \
+	messages/MStatfs.h \
+	messages/MStatfsReply.h \
+	messages/MTimeCheck.h \
+	messages/MWatchNotify.h \
+	messages/PaxosServiceMessage.h
+
diff --git a/ceph/src/messages/PaxosServiceMessage.h b/ceph/src/messages/PaxosServiceMessage.h
new file mode 100644
index 00000000..285804f2
--- /dev/null
+++ b/ceph/src/messages/PaxosServiceMessage.h
@@ -0,0 +1,70 @@
+#ifndef CEPH_PAXOSSERVICEMESSAGE_H
+#define CEPH_PAXOSSERVICEMESSAGE_H
+
+#include "msg/Message.h"
+#include "mon/Session.h"
+
+class PaxosServiceMessage : public Message {
+ public:
+  version_t version;
+  __s16 deprecated_session_mon;
+  uint64_t deprecated_session_mon_tid;
+
+  // track which epoch the leader received a forwarded request in, so we can
+  // discard forwarded requests appropriately on election boundaries.
+  epoch_t rx_election_epoch;
+  
+  PaxosServiceMessage()
+    : Message(MSG_PAXOS),
+      version(0), deprecated_session_mon(-1), deprecated_session_mon_tid(0),
+      rx_election_epoch(0) { }
+  PaxosServiceMessage(int type, version_t v, int enc_version=1, int compat_enc_version=0)
+    : Message(type, enc_version, compat_enc_version),
+      version(v), deprecated_session_mon(-1), deprecated_session_mon_tid(0),
+      rx_election_epoch(0)  { }
+ protected:
+  ~PaxosServiceMessage() {}
+
+ public:
+  void paxos_encode() {
+    ::encode(version, payload);
+    ::encode(deprecated_session_mon, payload);
+    ::encode(deprecated_session_mon_tid, payload);
+  }
+
+  void paxos_decode( bufferlist::iterator& p ) {
+    ::decode(version, p);
+    ::decode(deprecated_session_mon, p);
+    ::decode(deprecated_session_mon_tid, p);
+  }
+
+  void encode_payload(uint64_t features) {
+    assert(0);
+    paxos_encode();
+  }
+
+  void decode_payload() {
+    assert(0);
+    bufferlist::iterator p = payload.begin();
+    paxos_decode(p);
+  }
+
+  /** 
+   * These messages are only used by the monitors and clients,
+   * and the client doesn't care, so we're creating a monitor-specific
+   * function here. Note that this function explicitly exists to bypass
+   * the normal ref-counting, so don't expect the returned pointer to be
+   * very long-lived -- it will still only last as long as the Session would
+   * normally.
+   */
+  MonSession *get_session() {
+    MonSession *session = (MonSession *)get_connection()->get_priv();
+    if (session)
+      session->put();
+    return session;
+  }
+  
+  const char *get_type_name() const { return "PaxosServiceMessage"; }
+};
+
+#endif
diff --git a/ceph/src/mkcephfs.in b/ceph/src/mkcephfs.in
new file mode 100644
index 00000000..28385686
--- /dev/null
+++ b/ceph/src/mkcephfs.in
@@ -0,0 +1,564 @@
+#!/bin/sh
+#
+# mkcephfs
+#
+# This tool is designed to be flexible.  There are two ways to go:
+#
+# The easy way does everything for you using ssh keys.  This does not
+# scale well for large clusters.
+#
+#  master$ mkcephfs -a -c /etc/ceph/ceph.conf
+#
+# Alternatively, you can use whatever file distribution and/or job
+# launching you want.
+#
+#  master$ mkdir /tmp/foo
+#  master$ mkcephfs -d /tmp/foo -c /etc/ceph/ceph.conf --prepare-monmap
+#
+#     ...copy/share /tmp/foo with all osd and mds nodes at /tmp/bar...
+#
+#  osd$ mkcephfs -d /tmp/bar --init-local-daemons osd
+#  mds$ mkcephfs -d /tmp/bar --init-local-daemons mds
+#
+#     ...gather contents of /tmp/bar's back into /tmp/foo...
+#
+#  master$ mkcephfs -d /tmp/foo --prepare-mon
+#
+#     ...distribute /tmp/foo to each monitor node...
+#
+#  mon$ mkcephfs -d /tmp/foo --init-local-daemons mon
+#  
+#  master$ cp /tmp/foo/keyring.admin /etc/ceph/keyring  # don't forget!
+#
+# In the degenerate case (one node), this is just
+#
+#  mkdir /tmp/foo
+#  mkcephfs -c ceph.conf -d /tmp/foo --prepare-monmap
+#  mkcephfs -d /tmp/foo --init-local-daemons mds
+#  mkcephfs -d /tmp/foo --init-local-daemons osd
+#  mkcephfs -d /tmp/foo --prepare-mon
+#  mkcephfs -d /tmp/foo --init-local-daemons mon
+#  cp /tmp/foo/keyring.admin /etc/ceph/keyring
+#
+# or simply
+#
+#  mkcephfs -a -c ceph.conf
+#
+
+set -e
+
+trap 'echo "\nWARNING: mkcephfs is now deprecated in favour of ceph-deploy. Please see: \n http://github.com/ceph/ceph-deploy"' EXIT
+
+# if we start up as ./mkcephfs, assume everything else is in the
+# current directory too.
+if [ `dirname $0` = "." ] && [ $PWD != "/etc/init.d" ]; then
+    BINDIR=.
+    LIBDIR=.
+    ETCDIR=.
+else
+    BINDIR=@bindir@
+    LIBDIR=@libdir@/ceph
+    ETCDIR=@sysconfdir@/ceph
+fi
+
+usage_exit() {
+    echo "usage: $0 -a -c ceph.conf [-k adminkeyring] [--mkfs]"
+    echo "   to generate a new ceph cluster on all nodes; for advanced usage see man page"
+    echo "   ** be careful, this WILL clobber old data; check your ceph.conf carefully **"
+    exit
+}
+
+. $LIBDIR/ceph_common.sh
+
+
+allhosts=0
+mkfs=0
+preparemonmap=0
+prepareosdfs=""
+initdaemon=""
+initdaemons=""
+preparemon=0
+
+numosd=
+useosdmap=
+usecrushmapsrc=
+usecrushmap=
+verbose=0
+adminkeyring=""
+conf=""
+dir=""
+moreargs=""
+auto_action=0
+manual_action=0
+nocopyconf=0
+
+while [ $# -ge 1 ]; do
+case $1 in
+    -v )
+	    verbose=1;
+	    ;;
+    --dir | -d)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    dir=$1
+	    ;;
+    --allhosts | -a)
+	    allhosts=1
+            auto_action=1
+	    ;;
+    --prepare-monmap)
+	    preparemonmap=1
+            manual_action=1
+	    ;;
+    --prepare-osdfs)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    prepareosdfs=$1
+            manual_action=1
+	    ;;
+    --init-daemon)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    initdaemon=$1
+            manual_action=1
+	    ;;
+    --init-local-daemons)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    initlocaldaemons=$1
+            manual_action=1
+	    ;;
+    --prepare-mon)
+	    preparemon=1
+            manual_action=1
+	    ;;
+    --mkbtrfs | --mkfs)
+	    mkfs=1
+	    ;;
+    --no-copy-conf)
+	    nocopyconf=1
+	    ;;
+    --conf | -c)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    conf=$1
+	    ;;
+    --numosd)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    numosd=$1
+	    moreargs="$moreargs --numosd $1"
+	    ;;
+    --osdmap)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    useosdmap=$1
+	    moreargs="$moreargs --osdmap $1"
+	    ;;
+    --crushmapsrc)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    usecrushmapsrc=$1
+	    moreargs="$moreargs --crushmapsrc $1"
+	    ;;
+    --crushmap)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    usecrushmap=$1
+	    moreargs="$moreargs --crushmap $1"
+	    ;;
+    -k)
+	    [ -z "$2" ] && usage_exit
+	    shift
+	    adminkeyring=$1
+	    ;;
+    *)
+	    echo unrecognized option \'$1\'
+	    usage_exit
+	    ;;
+esac
+shift
+done
+
+
+[ -z "$conf" ] && [ -n "$dir" ] && conf="$dir/conf"
+
+if [ $manual_action -eq 0 ]; then
+    if [ $auto_action -eq 0 ]; then
+        echo "You must specify an action. See man page."
+        usage_exit
+    fi
+elif [ $auto_action -eq 1 ]; then
+    echo "The -a option cannot be combined with other subcommands; see man page."
+    usage_exit
+fi
+
+### prepare-monmap ###
+
+if [ $preparemonmap -eq 1 ]; then
+    echo "preparing monmap in $dir/monmap"
+
+    # first, make a list of monitors
+    mons=`$CCONF -c $conf -l mon | egrep -v '^mon$' | sort`
+    args=""
+
+    type="mon"
+    for name in $mons; do
+	id=`echo $name | cut -c 4- | sed 's/^\\.//'`
+	get_conf addr "" "mon addr"
+	if [ -z "$addr" ]; then
+	    echo "$0: monitor $name has no address defined." 1>&2
+	    exit 1
+	fi
+	args=$args" --add $id $addr"
+    done
+
+    if [ -z "$args" ]; then
+	echo "$0: no monitors found in config, aborting." 1>&2
+	exit 1
+    fi
+
+    # build monmap
+    monmap="$dir/monmap"
+    echo $BINDIR/monmaptool --create --clobber $args --print $monmap || exit 1
+    $BINDIR/monmaptool --create --clobber $args --print $monmap || exit 1
+        
+    # copy conf
+    cp $conf $dir/conf
+
+    exit 0
+fi
+
+
+### init-daemon ###
+
+if [ -n "$initdaemon" ]; then
+    name=$initdaemon
+    type=`echo $name | cut -c 1-3`   # e.g. 'mon', if $name is 'mon1'
+    id=`echo $name | cut -c 4- | sed 's/^\\.//'`
+    name="$type.$id"
+    
+    # create /var/run/ceph (or wherever pid file and/or admin socket live)
+    get_conf pid_file "/var/run/ceph/$name.pid" "pid file"
+    rundir=`dirname $pid_file`
+    if [ "$rundir" != "." ] && [ ! -d "$rundir" ]; then
+	mkdir -p $rundir
+    fi
+    get_conf asok_file "/var/run/ceph/$name.asok" "admin socket"
+    rundir=`dirname $asok_file`
+    if [ "$rundir" != "." ] && [ ! -d "$rundir" ]; then
+	mkdir -p $rundir
+    fi
+
+    if [ $type = "osd" ]; then
+	$BINDIR/ceph-osd -c $conf --monmap $dir/monmap -i $id --mkfs --mkkey
+
+	get_conf osd_data "/var/lib/ceph/osd/ceph-$id" "osd data"
+	get_conf osd_keyring "$osd_data/keyring" "keyring"
+	$BINDIR/ceph-authtool -p -n $name $osd_keyring > $dir/key.$name
+    fi
+    
+    if [ $type = "mds" ]; then
+	get_conf mds_data "/var/lib/ceph/mds/ceph-$id" "mds data"
+	get_conf mds_keyring "$mds_data/keyring" "keyring"
+	test -d $mds_data || mkdir -p $mds_data
+	echo "creating private key for $name keyring $mds_keyring"
+	$BINDIR/ceph-authtool --create-keyring --gen-key -n $name $mds_keyring
+	$BINDIR/ceph-authtool -p -n $name $mds_keyring > $dir/key.$name
+    fi
+
+    if [ $type = "mon" ]; then
+	get_conf mon_data "/var/lib/ceph/mon/ceph-$id" "mon data"
+	mkdir -p "$mon_data"
+	if ! find "$mon_data" -maxdepth 0 -empty | read foo; then
+	    echo "ERROR: $name mon_data directory $mon_data is not empty."
+	    echo "       Please make sure that it is empty before running mkcephfs."
+	    exit 1
+	fi
+	$BINDIR/ceph-mon -c $conf --mkfs -i $id --monmap $dir/monmap --osdmap $dir/osdmap -k $dir/keyring.mon
+    fi
+    
+    exit 0
+fi
+
+
+## init-local-daemons ##
+
+if [ -n "$initlocaldaemons" ]; then
+    get_name_list "$initlocaldaemons"
+    for name in $what; do
+	type=`echo $name | cut -c 1-3`   # e.g. 'mon', if $name is 'mon1'
+	id=`echo $name | cut -c 4- | sed 's/^\\.//'`
+	num=$id
+	name="$type.$id"
+
+	check_host || continue
+
+	$0 -d $dir --init-daemon $name	
+    done
+    exit 0
+fi
+
+
+### prepare-osdfs ###
+
+if [ -n "$prepareosdfs" ]; then
+    name=$prepareosdfs
+    type=`echo $name | cut -c 1-3`   # e.g. 'mon', if $name is 'mon1'
+    id=`echo $name | cut -c 4- | sed 's/^\\.//'`
+    name="$type.$id"
+
+    get_conf osd_data "/var/lib/ceph/osd/ceph-$id" "osd data"
+    get_conf osd_journal "$osd_data/journal" "osd journal"
+    get_conf fs_path "$osd_data" "fs path"  # mount point defaults so osd data
+    get_conf fs_devs "" "devs"
+    get_conf fs_type "" "osd mkfs type"
+
+    if [ -z "$fs_devs" ]; then
+	# try to fallback to old keys
+        get_conf tmp_btrfs_devs "" "btrfs devs"
+        if [ -n "$tmp_btrfs_devs" ]; then
+            fs_devs="$tmp_btrfs_devs"
+        else
+            echo "no devs defined for $name"
+            exit 1
+        fi
+    fi
+    if [ -z "$fs_type" ]; then
+        # try to fallback to to old keys
+        get_conf tmp_devs "" "btrfs devs"
+        if [ -n "$tmp_devs" ]; then
+            fs_type="btrfs"
+        else
+            echo No filesystem type defined!
+            exit 1
+        fi
+    fi
+
+    first_dev=`echo $fs_devs | cut '-d ' -f 1`
+    get_conf fs_opt "" "osd mount options $fs_type"
+    if [ -z "$fs_opt" ]; then
+        if [ "$fs_type" = "btrfs" ]; then
+            #try to fallback to old keys
+            get_conf fs_opt "" "btrfs options"
+        fi
+        if [ -z "$fs_opt" ]; then
+	    if [ "$fs_type" = "xfs" ]; then
+		fs_opt="rw,noatime,inode64"
+	    else
+                #fallback to use at least rw,noatime
+		fs_opt="rw,noatime"
+	    fi
+        fi
+    fi
+    [ -n "$fs_opt" ] && fs_opt="-o $fs_opt"
+    get_conf osd_user "root" "user"
+    
+    if [ -n "$osd_journal" ] && echo "fs_devs" | grep -q -w "$osd_journal" ; then
+        echo "ERROR: osd journal device ($osd_journal) also used by devs ($fs_devs)"
+	exit 1
+    fi
+    
+    test -d $osd_data || mkdir -p $osd_data
+
+    if [ -n "$osd_journal" ]; then
+	test -d $osd_journal || mkdir -p `dirname $osd_journal`
+    fi
+
+    umount $fs_path || true
+    for f in $fs_devs ; do
+	umount $f || true
+    done
+
+    get_conf mkfs_opt "" "osd mkfs options $fs_type"
+    if [ "$fs_type" = "xfs" ] && [ -z "$mkfs_opt" ]; then
+        echo Xfs filesystem found add missing -f mkfs option!
+	mkfs_opt="-f"
+    fi
+    modprobe $fs_type || true
+    mkfs.$fs_type $mkfs_opt $fs_devs
+    mount -t $fs_type $fs_opt $first_dev $fs_path
+    chown $osd_user $fs_path
+    chmod +w $fs_path
+
+    exit 0
+fi
+
+
+
+### prepare-mon ###
+
+if [ $preparemon -eq 1 ]; then
+
+    if [ -n "$useosdmap" ]; then
+	echo "Using osdmap $useosdmap"
+	cp $useosdmap $dir/osdmap
+    else
+	echo "Building generic osdmap from $conf"
+	$BINDIR/osdmaptool --create-from-conf $dir/osdmap -c $conf
+    fi
+
+    # import crush map?
+    get_conf crushmapsrc "" "crush map src" mon global
+    if [ -n "$crushmapsrc" ]; then
+	echo Compiling crush map from $crushmapsrc to $dir/crushmap
+	$BINDIR/crushtool -c $crushmapsrc -o $dir/crushmap
+    fi
+    get_conf crushmap "$usecrushmap" "crush map" mon global
+    if [ -n "$crushmap" ]; then
+	echo Importing crush map from $crushmap
+	$BINDIR/osdmaptool --import-crush $crushmap $dir/osdmap
+    fi
+
+    # admin keyring
+    echo Generating admin key at $dir/keyring.admin
+    $BINDIR/ceph-authtool --create-keyring --gen-key -n client.admin $dir/keyring.admin
+
+    # mon keyring
+    echo Building initial monitor keyring
+    cp $dir/keyring.admin $dir/keyring.mon
+    $BINDIR/ceph-authtool -n client.admin --set-uid=0 \
+	--cap mon 'allow *' \
+	--cap osd 'allow *' \
+	--cap mds 'allow' \
+	$dir/keyring.mon
+
+    $BINDIR/ceph-authtool --gen-key -n mon. $dir/keyring.mon --cap mon 'allow *'
+
+    for k in $dir/key.*
+    do
+	kname=`echo $k | sed 's/.*key\.//'`
+	ktype=`echo $kname | cut -c 1-3`
+	kid=`echo $kname | cut -c 4- | sed 's/^\\.//'`
+	kname="$ktype.$kid"
+	secret=`cat $k`
+	if [ "$ktype" = "osd" ]; then
+	    $BINDIR/ceph-authtool -n $kname --add-key $secret $dir/keyring.mon \
+		--cap mon 'allow rwx' \
+		--cap osd 'allow *'
+	fi
+	if [ "$ktype" = "mds" ]; then
+	    $BINDIR/ceph-authtool -n $kname --add-key $secret $dir/keyring.mon \
+		--cap mon "allow rwx" \
+		--cap osd 'allow *' \
+		--cap mds 'allow'
+	fi
+    done
+
+    exit 0
+fi
+
+
+
+
+
+### do everything via ssh ###
+
+if [ $allhosts -eq 1 ]; then
+
+    verify_conf
+
+    # do it all
+    if [ -z "$dir" ]; then
+	dir=`mktemp -d -t mkcephfs.XXXXXXXXXX` || exit 1
+	echo "temp dir is $dir"
+	trap "rm -rf $dir ; exit" INT TERM EXIT
+    fi
+
+    $0 --prepare-monmap -d $dir -c $conf
+
+    # osd, mds
+    get_name_list "osd mds"
+    for name in $what; do
+	type=`echo $name | cut -c 1-3`   # e.g. 'mon', if $name is 'mon1'
+	id=`echo $name | cut -c 4- | sed 's/^\\.//'`
+	num=$id
+	name="$type.$id"
+
+	check_host || continue
+
+	if [ -n "$ssh" ]; then
+	    rdir=`mktemp -u /tmp/mkfs.ceph.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX` || exit 1
+	    echo pushing conf and monmap to $host:$rdir
+	    do_cmd "mkdir -p $rdir"
+	    scp -q $dir/conf $host:$rdir
+	    scp -q $dir/monmap $host:$rdir
+
+	    if [ $nocopyconf -eq 0 ]; then
+		# also put conf at /etc/ceph/ceph.conf
+		scp -q $dir/conf $host:/etc/ceph/ceph.conf
+	    fi
+	else
+	    rdir=$dir
+
+	    if [ $nocopyconf -eq 0 ]; then
+		# also put conf at /etc/ceph/ceph.conf
+		cp $dir/conf /etc/ceph/ceph.conf
+	    fi
+	fi
+	
+	if [ $mkfs -eq 1 ] && [ "$type" = "osd" ]; then
+	    do_root_cmd "$0 -d $rdir --prepare-osdfs $name"
+	fi
+
+	do_root_cmd "$0 -d $rdir --init-daemon $name"
+
+	# collect the key
+	if [ -n "$ssh" ]; then
+	    echo collecting $name key
+	    scp -q $host:$rdir/key.$name $dir
+	    #cleanup no longer need rdir
+	    do_cmd "rm -r $rdir"
+	fi
+    done
+
+    # prepare monitors
+    $0 -d $dir --prepare-mon $moreargs
+    
+    # mons
+    get_name_list "mon"
+    for name in $what; do
+	type=`echo $name | cut -c 1-3`   # e.g. 'mon', if $name is 'mon1'
+	id=`echo $name | cut -c 4- | sed 's/^\\.//'`
+	num=$id
+	name="$type.$id"
+
+	check_host || continue
+	
+	if [ -n "$ssh" ]; then
+	    rdir=`mktemp -u /tmp/mkfs.ceph.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX` || exit 1
+	    echo pushing everything to $host
+	    ssh $host mkdir -p $rdir
+	    scp -q $dir/* $host:$rdir
+
+	    if [ $nocopyconf -eq 0 ]; then
+		# also put conf at /etc/ceph/ceph.conf
+		scp -q $dir/conf $host:/etc/ceph/ceph.conf
+	    fi
+	else
+	    rdir=$dir
+
+	    if [ $nocopyconf -eq 0 ]; then
+	        # also put conf at /etc/ceph/ceph.conf
+		cp $dir/conf /etc/ceph/ceph.conf
+	    fi
+	fi
+	
+	do_root_cmd "$0 -d $rdir --init-daemon $name"
+
+	if [ -n "$ssh" ]; then
+	    #cleanup no longer need rdir
+	    do_cmd "rm -r $rdir"
+	fi
+    done
+
+    # admin keyring
+    if [ -z "$adminkeyring" ]; then
+	get_conf adminkeyring "/etc/ceph/keyring" "keyring" global
+    fi
+    echo "placing client.admin keyring in $adminkeyring"
+    cp $dir/keyring.admin $adminkeyring
+
+    exit 0
+fi
+
diff --git a/ceph/src/mon/AuthMonitor.cc b/ceph/src/mon/AuthMonitor.cc
new file mode 100644
index 00000000..359c2e10
--- /dev/null
+++ b/ceph/src/mon/AuthMonitor.cc
@@ -0,0 +1,1083 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+
+#include "mon/AuthMonitor.h"
+#include "mon/Monitor.h"
+#include "mon/MonitorDBStore.h"
+
+#include "messages/MMonCommand.h"
+#include "messages/MAuth.h"
+#include "messages/MAuthReply.h"
+#include "messages/MMonGlobalID.h"
+
+#include "include/str_list.h"
+#include "common/Timer.h"
+
+#include "auth/AuthServiceHandler.h"
+#include "auth/KeyRing.h"
+
+#include "osd/osd_types.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+#include "common/cmdparse.h"
+#include "include/str_list.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, get_last_committed())
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, version_t v) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").auth v" << v << " ";
+}
+
+ostream& operator<<(ostream& out, AuthMonitor& pm)
+{
+  return out << "auth";
+}
+
+bool AuthMonitor::check_rotate()
+{
+  KeyServerData::Incremental rot_inc;
+  rot_inc.op = KeyServerData::AUTH_INC_SET_ROTATING;
+  if (!mon->key_server.updated_rotating(rot_inc.rotating_bl, last_rotating_ver))
+    return false;
+  dout(10) << __func__ << " updated rotating" << dendl;
+  push_cephx_inc(rot_inc);
+  return true;
+}
+
+/*
+ Tick function to update the map based on performance every N seconds
+*/
+
+void AuthMonitor::tick()
+{
+  if (!is_active()) return;
+
+  dout(10) << *this << dendl;
+
+  if (!mon->is_leader()) return;
+
+  if (check_rotate())
+    propose_pending();
+}
+
+void AuthMonitor::on_active()
+{
+  dout(10) << "AuthMonitor::on_active()" << dendl;
+
+  if (!mon->is_leader())
+    return;
+  mon->key_server.start_server();
+}
+
+void AuthMonitor::create_initial()
+{
+  dout(10) << "create_initial -- creating initial map" << dendl;
+
+  // initialize rotating keys
+  last_rotating_ver = 0;
+  check_rotate();
+  assert(pending_auth.size() == 1);
+
+  if (mon->is_keyring_required()) {
+    KeyRing keyring;
+    bufferlist bl;
+    int ret = mon->store->get("mkfs", "keyring", bl);
+    assert(ret == 0);
+    bufferlist::iterator p = bl.begin();
+    ::decode(keyring, p);
+
+    import_keyring(keyring);
+  }
+
+  max_global_id = MIN_GLOBAL_ID;
+
+  Incremental inc;
+  inc.inc_type = GLOBAL_ID;
+  inc.max_global_id = max_global_id;
+  pending_auth.push_back(inc);
+
+  format_version = 1;
+}
+
+void AuthMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  dout(10) << __func__ << dendl;
+  version_t version = get_last_committed();
+  version_t keys_ver = mon->key_server.get_ver();
+  if (version == keys_ver)
+    return;
+  assert(version >= keys_ver);
+
+  version_t latest_full = get_version_latest_full();
+
+  dout(10) << __func__ << " version " << version << " keys ver " << keys_ver
+           << " latest " << latest_full << dendl;
+
+  if ((latest_full > 0) && (latest_full > keys_ver)) {
+    bufferlist latest_bl;
+    int err = get_version_full(latest_full, latest_bl);
+    assert(err == 0);
+    assert(latest_bl.length() != 0);
+    dout(7) << __func__ << " loading summary e " << latest_full << dendl;
+    dout(7) << __func__ << " latest length " << latest_bl.length() << dendl;
+    bufferlist::iterator p = latest_bl.begin();
+    __u8 struct_v;
+    ::decode(struct_v, p);
+    ::decode(max_global_id, p);
+    ::decode(mon->key_server, p);
+    mon->key_server.set_ver(latest_full);
+    keys_ver = latest_full;
+  }
+
+  dout(10) << __func__ << " key server version " << mon->key_server.get_ver() << dendl;
+
+  // walk through incrementals
+  while (version > keys_ver) {
+    bufferlist bl;
+    int ret = get_version(keys_ver+1, bl);
+    assert(ret == 0);
+    assert(bl.length());
+
+    // reset if we are moving to initial state.  we will normally have
+    // keys in here temporarily for bootstrapping that we need to
+    // clear out.
+    if (keys_ver == 0)
+      mon->key_server.clear_secrets();
+
+    dout(20) << __func__ << " walking through version " << (keys_ver+1)
+             << " len " << bl.length() << dendl;
+
+    bufferlist::iterator p = bl.begin();
+    __u8 v;
+    ::decode(v, p);
+    while (!p.end()) {
+      Incremental inc;
+      ::decode(inc, p);
+      switch (inc.inc_type) {
+      case GLOBAL_ID:
+	max_global_id = inc.max_global_id;
+	break;
+
+      case AUTH_DATA:
+        {
+          KeyServerData::Incremental auth_inc;
+          bufferlist::iterator iter = inc.auth_data.begin();
+          ::decode(auth_inc, iter);
+          mon->key_server.apply_data_incremental(auth_inc);
+          break;
+        }
+      }
+    }
+
+    keys_ver++;
+    mon->key_server.set_ver(keys_ver);
+
+    if (keys_ver == 1 && mon->is_keyring_required()) {
+      MonitorDBStore::Transaction t;
+      t.erase("mkfs", "keyring");
+      mon->store->apply_transaction(t);
+    }
+  }
+
+  if (last_allocated_id == 0)
+    last_allocated_id = max_global_id;
+
+  dout(10) << "update_from_paxos() last_allocated_id=" << last_allocated_id
+	   << " max_global_id=" << max_global_id
+	   << " format_version " << format_version
+	   << dendl;
+}
+
+void AuthMonitor::increase_max_global_id()
+{
+  assert(mon->is_leader());
+
+  max_global_id += g_conf->mon_globalid_prealloc;
+  dout(10) << "increasing max_global_id to " << max_global_id << dendl;
+  Incremental inc;
+  inc.inc_type = GLOBAL_ID;
+  inc.max_global_id = max_global_id;
+  pending_auth.push_back(inc);
+}
+
+bool AuthMonitor::should_propose(double& delay)
+{
+  return (!pending_auth.empty());
+}
+
+void AuthMonitor::create_pending()
+{
+  pending_auth.clear();
+  dout(10) << "create_pending v " << (get_last_committed() + 1) << dendl;
+}
+
+void AuthMonitor::encode_pending(MonitorDBStore::Transaction *t)
+{
+  dout(10) << __func__ << " v " << (get_last_committed() + 1) << dendl;
+
+  bufferlist bl;
+
+  __u8 v = 1;
+  ::encode(v, bl);
+  vector::iterator p;
+  for (p = pending_auth.begin(); p != pending_auth.end(); ++p)
+    p->encode(bl, mon->get_quorum_features());
+
+  version_t version = get_last_committed() + 1;
+  put_version(t, version, bl);
+  put_last_committed(t, version);
+}
+
+void AuthMonitor::encode_full(MonitorDBStore::Transaction *t)
+{
+  version_t version = mon->key_server.get_ver();
+  // do not stash full version 0 as it will never be removed nor read
+  if (version == 0)
+    return;
+
+  dout(10) << __func__ << " auth v " << version << dendl;
+  assert(get_last_committed() == version);
+
+  bufferlist full_bl;
+  Mutex::Locker l(mon->key_server.get_lock());
+  dout(20) << __func__ << " key server has "
+           << (mon->key_server.has_secrets() ? "" : "no ")
+           << "secrets!" << dendl;
+  __u8 v = 1;
+  ::encode(v, full_bl);
+  ::encode(max_global_id, full_bl);
+  ::encode(mon->key_server, full_bl);
+
+  put_version_full(t, version, full_bl);
+  put_version_latest_full(t, version);
+}
+
+version_t AuthMonitor::get_trim_to()
+{
+  unsigned max = g_conf->paxos_max_join_drift * 2;
+  version_t version = get_last_committed();
+  if (mon->is_leader() && (version > max))
+    return version - max;
+  return 0;
+}
+
+bool AuthMonitor::preprocess_query(PaxosServiceMessage *m)
+{
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    return preprocess_command((MMonCommand*)m);
+
+  case CEPH_MSG_AUTH:
+    return prep_auth((MAuth *)m, false);
+
+  case MSG_MON_GLOBAL_ID:
+    return false;
+
+  default:
+    assert(0);
+    m->put();
+    return true;
+  }
+}
+
+bool AuthMonitor::prepare_update(PaxosServiceMessage *m)
+{
+  dout(10) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    return prepare_command((MMonCommand*)m);
+  case MSG_MON_GLOBAL_ID:
+    return prepare_global_id((MMonGlobalID*)m); 
+  case CEPH_MSG_AUTH:
+    return prep_auth((MAuth *)m, true);
+  default:
+    assert(0);
+    m->put();
+    return false;
+  }
+}
+
+uint64_t AuthMonitor::assign_global_id(MAuth *m, bool should_increase_max)
+{
+  int total_mon = mon->monmap->size();
+  dout(10) << "AuthMonitor::assign_global_id m=" << *m << " mon=" << mon->rank << "/" << total_mon
+	   << " last_allocated=" << last_allocated_id << " max_global_id=" <<  max_global_id << dendl;
+
+  uint64_t next_global_id = last_allocated_id + 1;
+  int remainder = next_global_id % total_mon;
+  if (remainder)
+    remainder = total_mon - remainder;
+  next_global_id += remainder + mon->rank;
+  dout(10) << "next_global_id should be " << next_global_id << dendl;
+
+  // if we can't bump the max, bail out now on an out-of-bounds gid
+  if (next_global_id > max_global_id &&
+      (!mon->is_leader() || !should_increase_max)) {
+    return 0;
+  }
+
+  // can we return a gid?
+  bool return_next = (next_global_id <= max_global_id);
+
+  // bump the max?
+  while (mon->is_leader() &&
+	 next_global_id >= max_global_id - g_conf->mon_globalid_prealloc / 2) {
+    increase_max_global_id();
+  }
+
+  if (return_next) {
+    last_allocated_id = next_global_id;
+    return next_global_id;
+  } else {
+    return 0;
+  }
+}
+
+
+bool AuthMonitor::prep_auth(MAuth *m, bool paxos_writable)
+{
+  dout(10) << "prep_auth() blob_size=" << m->get_auth_payload().length() << dendl;
+
+  MonSession *s = (MonSession *)m->get_connection()->get_priv();
+  if (!s) {
+    dout(10) << "no session, dropping" << dendl;
+    m->put();
+    return true;
+  }
+
+  int ret = 0;
+  AuthCapsInfo caps_info;
+  MAuthReply *reply;
+  bufferlist response_bl;
+  bufferlist::iterator indata = m->auth_payload.begin();
+  __u32 proto = m->protocol;
+  bool start = false;
+  EntityName entity_name;
+
+  // set up handler?
+  if (m->protocol == 0 && !s->auth_handler) {
+    set<__u32> supported;
+
+    try {
+      __u8 struct_v = 1;
+      ::decode(struct_v, indata);
+      ::decode(supported, indata);
+      ::decode(entity_name, indata);
+      ::decode(s->global_id, indata);
+    } catch (const buffer::error &e) {
+      dout(10) << "failed to decode initial auth message" << dendl;
+      ret = -EINVAL;
+      goto reply;
+    }
+
+    // do we require cephx signatures?
+
+    if (!m->get_connection()->has_feature(CEPH_FEATURE_MSG_AUTH)) {
+      if (entity_name.get_type() == CEPH_ENTITY_TYPE_MON ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_MDS) {
+	if (g_conf->cephx_cluster_require_signatures ||
+	    g_conf->cephx_require_signatures) {
+	  dout(1) << m->get_source_inst()
+                  << " supports cephx but not signatures and"
+                  << " 'cephx [cluster] require signatures = true';"
+                  << " disallowing cephx" << dendl;
+	  supported.erase(CEPH_AUTH_CEPHX);
+	}
+      } else {
+	if (g_conf->cephx_service_require_signatures ||
+	    g_conf->cephx_require_signatures) {
+	  dout(1) << m->get_source_inst()
+                  << " supports cephx but not signatures and"
+                  << " 'cephx [service] require signatures = true';"
+                  << " disallowing cephx" << dendl;
+	  supported.erase(CEPH_AUTH_CEPHX);
+	}
+      }
+    }
+
+    int type;
+    if (entity_name.get_type() == CEPH_ENTITY_TYPE_MON ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_MDS)
+      type = mon->auth_cluster_required.pick(supported);
+    else
+      type = mon->auth_service_required.pick(supported);
+
+    s->auth_handler = get_auth_service_handler(type, g_ceph_context, &mon->key_server);
+    if (!s->auth_handler) {
+      dout(1) << "client did not provide supported auth type" << dendl;
+      ret = -ENOTSUP;
+      goto reply;
+    }
+    start = true;
+  } else if (!s->auth_handler) {
+      dout(10) << "protocol specified but no s->auth_handler" << dendl;
+      ret = -EINVAL;
+      goto reply;
+  }
+
+  /* assign a new global_id? we assume this should only happen on the first
+     request. If a client tries to send it later, it'll screw up its auth
+     session */
+  if (!s->global_id) {
+    s->global_id = assign_global_id(m, paxos_writable);
+    if (!s->global_id) {
+
+      delete s->auth_handler;
+      s->auth_handler = NULL;
+
+      if (mon->is_leader() && paxos_writable) {
+        dout(10) << "increasing global id, waitlisting message" << dendl;
+        wait_for_active(new C_RetryMessage(this, m));
+        goto done;
+      }
+
+      s->put();
+
+      if (!mon->is_leader()) {
+	dout(10) << "not the leader, requesting more ids from leader" << dendl;
+	int leader = mon->get_leader();
+	MMonGlobalID *req = new MMonGlobalID();
+	req->old_max_id = max_global_id;
+	mon->messenger->send_message(req, mon->monmap->get_inst(leader));
+	wait_for_finished_proposal(new C_RetryMessage(this, m));
+	return true;
+      }
+
+      assert(!paxos_writable);
+      return false;
+    }
+  }
+
+  try {
+    uint64_t auid = 0;
+    if (start) {
+      // new session
+
+      // always send the latest monmap.
+      if (m->monmap_epoch < mon->monmap->get_epoch())
+	mon->send_latest_monmap(m->get_connection().get());
+
+      proto = s->auth_handler->start_session(entity_name, indata, response_bl, caps_info);
+      ret = 0;
+      if (caps_info.allow_all)
+	s->caps.set_allow_all();
+    } else {
+      // request
+      ret = s->auth_handler->handle_request(indata, response_bl, s->global_id, caps_info, &auid);
+    }
+    if (ret == -EIO) {
+      wait_for_active(new C_RetryMessage(this,m));
+      goto done;
+    }
+    if (caps_info.caps.length()) {
+      bufferlist::iterator p = caps_info.caps.begin();
+      string str;
+      try {
+	::decode(str, p);
+      } catch (const buffer::error &err) {
+	derr << "corrupt cap data for " << entity_name << " in auth db" << dendl;
+	str.clear();
+      }
+      s->caps.parse(str, NULL);
+      s->auid = auid;
+    }
+  } catch (const buffer::error &err) {
+    ret = -EINVAL;
+    dout(0) << "caught error when trying to handle auth request, probably malformed request" << dendl;
+  }
+
+reply:
+  reply = new MAuthReply(proto, &response_bl, ret, s->global_id);
+  mon->send_reply(m, reply);
+  m->put();
+done:
+  s->put();
+  return true;
+}
+
+bool AuthMonitor::preprocess_command(MMonCommand *m)
+{
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss, ds;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+  if (prefix == "auth add" ||
+      prefix == "auth del" ||
+      prefix == "auth get-or-create" ||
+      prefix == "auth get-or-create-key" ||
+      prefix == "auth import" ||
+      prefix == "auth caps") {
+    return false;
+  }
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  // entity might not be supplied, but if it is, it should be valid
+  string entity_name;
+  cmd_getval(g_ceph_context, cmdmap, "entity", entity_name);
+  EntityName entity;
+  if (!entity_name.empty() && !entity.from_str(entity_name)) {
+    ss << "invalid entity_auth " << entity_name;
+    mon->reply_command(m, -EINVAL, ss.str(), get_last_committed());
+    return true;
+  }
+
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr f(new_formatter(format));
+
+  if (prefix == "auth export") {
+    KeyRing keyring;
+    export_keyring(keyring);
+    if (!entity_name.empty()) {
+      EntityAuth eauth;
+      if (keyring.get_auth(entity, eauth)) {
+	KeyRing kr;
+	kr.add(entity, eauth);
+	if (f)
+	  kr.encode_formatted("auth", f.get(), rdata);
+	else
+	  kr.encode_plaintext(rdata);
+	ss << "export " << eauth;
+	r = 0;
+      } else {
+	ss << "no key for " << eauth;
+	r = -ENOENT;
+      }
+    } else {
+      if (f)
+	keyring.encode_formatted("auth", f.get(), rdata);
+      else
+	keyring.encode_plaintext(rdata);
+
+      ss << "exported master keyring";
+      r = 0;
+    }
+  } else if (prefix == "auth get" && !entity_name.empty()) {
+    KeyRing keyring;
+    EntityAuth entity_auth;
+    if(!mon->key_server.get_auth(entity, entity_auth)) {
+      ss << "failed to find " << entity_name << " in keyring";
+      r = -ENOENT;
+    } else {
+      keyring.add(entity, entity_auth);
+      if (f)
+	keyring.encode_formatted("auth", f.get(), rdata);
+      else
+	keyring.encode_plaintext(rdata);
+      ss << "exported keyring for " << entity_name;
+      r = 0;
+    }
+  } else if (prefix == "auth print-key" ||
+	     prefix == "auth print_key" ||
+	     prefix == "auth get-key") {
+    EntityAuth auth;
+    if (!mon->key_server.get_auth(entity, auth)) {
+      ss << "don't have " << entity;
+      r = -ENOENT;
+      goto done;
+    }
+    if (f) {
+      auth.key.encode_formatted("auth", f.get(), rdata);
+    } else {
+      auth.key.encode_plaintext(rdata);
+    }
+    r = 0;
+  } else if (prefix == "auth list") {
+    if (f) {
+      mon->key_server.encode_formatted("auth", f.get(), rdata);
+    } else {
+      mon->key_server.encode_plaintext(rdata);
+      if (rdata.length() > 0)
+        ss << "installed auth entries:" << std::endl;
+      else
+        ss << "no installed auth entries!" << std::endl;
+    }
+    r = 0;
+    goto done;
+  } else {
+    ss << "invalid command";
+    r = -EINVAL;
+  }
+
+ done:
+  rdata.append(ds);
+  string rs;
+  getline(ss, rs, '\0');
+  mon->reply_command(m, r, rs, rdata, get_last_committed());
+  return true;
+}
+
+void AuthMonitor::export_keyring(KeyRing& keyring)
+{
+  mon->key_server.export_keyring(keyring);
+}
+
+void AuthMonitor::import_keyring(KeyRing& keyring)
+{
+  for (map::iterator p = keyring.get_keys().begin();
+       p != keyring.get_keys().end();
+       ++p) {
+    KeyServerData::Incremental auth_inc;
+    auth_inc.name = p->first;
+    auth_inc.auth = p->second;
+    auth_inc.op = KeyServerData::AUTH_INC_ADD;
+    dout(10) << " importing " << auth_inc.name << dendl;
+    dout(30) << "    " << auth_inc.auth << dendl;
+    push_cephx_inc(auth_inc);
+  }
+}
+
+bool AuthMonitor::prepare_command(MMonCommand *m)
+{
+  stringstream ss, ds;
+  bufferlist rdata;
+  string rs;
+  int err = -EINVAL;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  vectorcaps_vec;
+  string entity_name;
+  EntityName entity;
+
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr f(new_formatter(format));
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  cmd_getval(g_ceph_context, cmdmap, "caps", caps_vec);
+  if ((caps_vec.size() % 2) != 0) {
+    ss << "bad capabilities request; odd number of arguments";
+    err = -EINVAL;
+    goto done;
+  }
+
+  cmd_getval(g_ceph_context, cmdmap, "entity", entity_name);
+  if (!entity_name.empty() && !entity.from_str(entity_name)) {
+    ss << "bad entity name";
+    err = -EINVAL;
+    goto done;
+  }
+
+  if (prefix == "auth import") {
+    bufferlist bl = m->get_data();
+    if (bl.length() == 0) {
+      ss << "auth import: no data supplied";
+      getline(ss, rs);
+      mon->reply_command(m, -EINVAL, rs, get_last_committed());
+      return true;
+    }
+    bufferlist::iterator iter = bl.begin();
+    KeyRing keyring;
+    try {
+      ::decode(keyring, iter);
+    } catch (const buffer::error &ex) {
+      ss << "error decoding keyring" << " " << ex.what();
+      rs = err;
+      goto done;
+    }
+    import_keyring(keyring);
+    ss << "imported keyring";
+    getline(ss, rs);
+    err = 0;
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "auth add" && !entity_name.empty()) {
+    /* expected behavior:
+     *  - if command reproduces current state, return 0.
+     *  - if command adds brand new entity, handle it.
+     *  - if command adds new state to existing entity, return error.
+     */
+    KeyServerData::Incremental auth_inc;
+    auth_inc.name = entity;
+    bufferlist bl = m->get_data();
+    bool has_keyring = (bl.length() > 0);
+    map new_caps;
+
+    KeyRing new_keyring;
+    if (has_keyring) {
+      bufferlist::iterator iter = bl.begin();
+      try {
+        ::decode(new_keyring, iter);
+      } catch (const buffer::error &ex) {
+        ss << "error decoding keyring";
+        err = -EINVAL;
+        goto done;
+      }
+    }
+
+    // are we about to have it?
+    for (vector::iterator p = pending_auth.begin();
+        p != pending_auth.end();
+        ++p) {
+      if (p->inc_type == AUTH_DATA) {
+        KeyServerData::Incremental inc;
+        bufferlist::iterator q = p->auth_data.begin();
+        ::decode(inc, q);
+        if (inc.op == KeyServerData::AUTH_INC_ADD &&
+            inc.name == entity) {
+          wait_for_finished_proposal(
+              new Monitor::C_Command(mon, m, 0, rs, get_last_committed() + 1));
+          return true;
+        }
+      }
+    }
+
+    // build new caps from provided arguments (if available)
+    for (vector::iterator it = caps_vec.begin();
+	 it != caps_vec.end() && (it + 1) != caps_vec.end();
+	 it += 2) {
+      string sys = *it;
+      bufferlist cap;
+      ::encode(*(it+1), cap);
+      new_caps[sys] = cap;
+    }
+
+    // pull info out of provided keyring
+    EntityAuth new_inc;
+    if (has_keyring) {
+      if (!new_keyring.get_auth(auth_inc.name, new_inc)) {
+	ss << "key for " << auth_inc.name
+	   << " not found in provided keyring";
+	err = -EINVAL;
+	goto done;
+      }
+      if (!new_caps.empty() && !new_inc.caps.empty()) {
+	ss << "caps cannot be specified both in keyring and in command";
+	err = -EINVAL;
+	goto done;
+      }
+      if (new_caps.empty()) {
+	new_caps = new_inc.caps;
+      }
+    }
+
+    // does entry already exist?
+    if (mon->key_server.get_auth(auth_inc.name, auth_inc.auth)) {
+      // key match?
+      if (has_keyring) {
+        if (auth_inc.auth.key.get_secret().cmp(new_inc.key.get_secret())) {
+	  ss << "entity " << auth_inc.name << " exists but key does not match";
+	  err = -EINVAL;
+	  goto done;
+	}
+      }
+
+      // caps match?
+      if (new_caps.size() != auth_inc.auth.caps.size()) {
+	ss << "entity " << auth_inc.name << " exists but caps do not match";
+	err = -EINVAL;
+	goto done;
+      }
+      for (map::iterator it = new_caps.begin();
+	   it != new_caps.end(); ++it) {
+        if (auth_inc.auth.caps.count(it->first) == 0 ||
+            !auth_inc.auth.caps[it->first].contents_equal(it->second)) {
+          ss << "entity " << auth_inc.name << " exists but cap "
+	     << it->first << " does not match";
+          err = -EINVAL;
+          goto done;
+        }
+      }
+
+      // they match, no-op
+      err = 0;
+      goto done;
+    }
+
+    // okay, add it.
+    auth_inc.op = KeyServerData::AUTH_INC_ADD;
+    auth_inc.auth.caps = new_caps;
+    if (has_keyring) {
+      auth_inc.auth.key = new_inc.key;
+    } else {
+      dout(10) << "AuthMonitor::prepare_command generating random key for "
+               << auth_inc.name << dendl;
+      auth_inc.auth.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    }
+
+    dout(10) << " importing " << auth_inc.name << dendl;
+    dout(30) << "    " << auth_inc.auth << dendl;
+    push_cephx_inc(auth_inc);
+
+    ss << "added key for " << auth_inc.name;
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						   get_last_committed() + 1));
+    return true;
+  } else if ((prefix == "auth get-or-create-key" ||
+	     prefix == "auth get-or-create") &&
+	     !entity_name.empty()) {
+    // auth get-or-create  [mon osdcapa osd osdcapb ...]
+
+    // do we have it?
+    EntityAuth entity_auth;
+    if (mon->key_server.get_auth(entity, entity_auth)) {
+      for (vector::iterator it = caps_vec.begin();
+	   it != caps_vec.end(); it += 2) {
+	string sys = *it;
+	bufferlist cap;
+	::encode(*(it+1), cap);
+	if (entity_auth.caps.count(sys) == 0 ||
+	    !entity_auth.caps[sys].contents_equal(cap)) {
+	  ss << "key for " << entity << " exists but cap " << sys << " does not match";
+	  err = -EINVAL;
+	  goto done;
+	}
+      }
+
+      if (prefix == "auth get-or-create-key") {
+        if (f) {
+          entity_auth.key.encode_formatted("auth", f.get(), rdata);
+        } else {
+          ds << entity_auth.key;
+        }
+      } else {
+	KeyRing kr;
+	kr.add(entity, entity_auth.key);
+        if (f) {
+          kr.encode_formatted("auth", f.get(), rdata);
+        } else {
+          kr.encode_plaintext(rdata);
+        }
+      }
+      err = 0;
+      goto done;
+    }
+
+    // ...or are we about to?
+    for (vector::iterator p = pending_auth.begin();
+	 p != pending_auth.end();
+	 ++p) {
+      if (p->inc_type == AUTH_DATA) {
+	KeyServerData::Incremental auth_inc;
+	bufferlist::iterator q = p->auth_data.begin();
+	::decode(auth_inc, q);
+	if (auth_inc.op == KeyServerData::AUTH_INC_ADD &&
+	    auth_inc.name == entity) {
+	  wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+	  return true;
+	}
+      }
+    }
+
+    // create it
+    KeyServerData::Incremental auth_inc;
+    auth_inc.op = KeyServerData::AUTH_INC_ADD;
+    auth_inc.name = entity;
+    auth_inc.auth.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    for (vector::iterator it = caps_vec.begin();
+	 it != caps_vec.end(); it += 2)
+      ::encode(*(it+1), auth_inc.auth.caps[*it]);
+
+    push_cephx_inc(auth_inc);
+
+    if (prefix == "auth get-or-create-key") {
+      if (f) {
+        auth_inc.auth.key.encode_formatted("auth", f.get(), rdata);
+      } else {
+        ds << auth_inc.auth.key;
+      }
+    } else {
+      KeyRing kr;
+      kr.add(entity, auth_inc.auth.key);
+      if (f) {
+        kr.encode_formatted("auth", f.get(), rdata);
+      } else {
+        kr.encode_plaintext(rdata);
+      }
+    }
+
+    rdata.append(ds);
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, rdata,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "auth caps" && !entity_name.empty()) {
+    KeyServerData::Incremental auth_inc;
+    auth_inc.name = entity;
+    if (!mon->key_server.get_auth(auth_inc.name, auth_inc.auth)) {
+      ss << "couldn't find entry " << auth_inc.name;
+      err = -ENOENT;
+      goto done;
+    }
+
+    map newcaps;
+    for (vector::iterator it = caps_vec.begin();
+	 it != caps_vec.end(); it += 2)
+      ::encode(*(it+1), newcaps[*it]);
+
+    auth_inc.op = KeyServerData::AUTH_INC_ADD;
+    auth_inc.auth.caps = newcaps;
+    push_cephx_inc(auth_inc);
+
+    ss << "updated caps for " << auth_inc.name;
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "auth del" && !entity_name.empty()) {
+    KeyServerData::Incremental auth_inc;
+    auth_inc.name = entity;
+    if (!mon->key_server.contains(auth_inc.name)) {
+      ss << "entity " << entity << " does not exist";
+      err = 0;
+      goto done;
+    }
+    auth_inc.op = KeyServerData::AUTH_INC_DEL;
+    push_cephx_inc(auth_inc);
+
+    ss << "updated";
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  }
+
+done:
+  rdata.append(ds);
+  getline(ss, rs, '\0');
+  mon->reply_command(m, err, rs, rdata, get_last_committed());
+  return false;
+}
+
+bool AuthMonitor::prepare_global_id(MMonGlobalID *m)
+{
+  dout(10) << "AuthMonitor::prepare_global_id" << dendl;
+  increase_max_global_id();
+
+  m->put();
+  return true;
+}
+
+void AuthMonitor::upgrade_format()
+{
+  unsigned int current = 1;
+  if (format_version >= current) {
+    dout(20) << __func__ << " format " << format_version << " is current" << dendl;
+    return;
+  }
+
+  dout(1) << __func__ << " upgrading from format " << format_version << " to " << current << dendl;
+  bool changed = false;
+  map::iterator p;
+  for (p = mon->key_server.secrets_begin();
+       p != mon->key_server.secrets_end();
+       ++p) {
+    // grab mon caps, if any
+    string mon_caps;
+    if (p->second.caps.count("mon") == 0)
+      continue;
+    try {
+      bufferlist::iterator it = p->second.caps["mon"].begin();
+      ::decode(mon_caps, it);
+    }
+    catch (buffer::error) {
+      dout(10) << __func__ << " unable to parse mon cap for "
+               << p->first << dendl;
+      continue;
+    }
+
+    string n = p->first.to_str();
+    string new_caps;
+
+    // set daemon profiles
+    if ((p->first.is_osd() || p->first.is_mds()) &&
+	mon_caps == "allow rwx") {
+      new_caps = string("allow profile ") + string(p->first.get_type_name());
+    }
+
+    // update bootstrap keys
+    if (n == "client.bootstrap-osd") {
+      new_caps = "allow profile bootstrap-osd";
+    }
+    if (n == "client.bootstrap-mds") {
+      new_caps = "allow profile bootstrap-mds";
+    }
+
+    if (new_caps.length() > 0) {
+      dout(5) << __func__ << " updating " << p->first << " mon cap from "
+	      << mon_caps << " to " << new_caps << dendl;
+
+      bufferlist bl;
+      ::encode(new_caps, bl);
+
+      KeyServerData::Incremental auth_inc;
+      auth_inc.name = p->first;
+      auth_inc.auth = p->second;
+      auth_inc.auth.caps["mon"] = bl;
+      auth_inc.op = KeyServerData::AUTH_INC_ADD;
+      push_cephx_inc(auth_inc);
+      changed = true;
+    }
+  }
+
+  if (changed) {
+    // note new format
+    dout(10) << __func__ << " proposing update from format " << format_version
+	     << " -> " << current << dendl;
+    format_version = current;
+    propose_pending();
+  }
+}
+
+void AuthMonitor::dump_info(Formatter *f)
+{
+  /*** WARNING: do not include any privileged information here! ***/
+  f->open_object_section("auth");
+  f->dump_unsigned("first_committed", get_first_committed());
+  f->dump_unsigned("last_committed", get_last_committed());
+  f->dump_unsigned("num_secrets", mon->key_server.get_num_secrets());
+  f->close_section();
+}
diff --git a/ceph/src/mon/AuthMonitor.h b/ceph/src/mon/AuthMonitor.h
new file mode 100644
index 00000000..130616ae
--- /dev/null
+++ b/ceph/src/mon/AuthMonitor.h
@@ -0,0 +1,167 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_AUTHMONITOR_H
+#define CEPH_AUTHMONITOR_H
+
+#include 
+#include 
+using namespace std;
+
+#include "include/ceph_features.h"
+#include "include/types.h"
+#include "msg/Messenger.h"
+#include "mon/PaxosService.h"
+#include "mon/Monitor.h"
+#include "mon/MonitorDBStore.h"
+
+class MMonCommand;
+struct MAuth;
+class MAuthMon;
+struct MMonGlobalID;
+class KeyRing;
+
+#define MIN_GLOBAL_ID 0x1000
+
+class AuthMonitor : public PaxosService {
+  enum IncType {
+    GLOBAL_ID,
+    AUTH_DATA,
+  };
+public:
+  struct Incremental {
+    IncType inc_type;
+    uint64_t max_global_id;
+    uint32_t auth_type;
+    bufferlist auth_data;
+
+    Incremental() : inc_type(GLOBAL_ID), max_global_id(0), auth_type(0) {}
+
+    void encode(bufferlist& bl, uint64_t features=-1) const {
+      if ((features & CEPH_FEATURE_MONENC) == 0) {
+	__u8 v = 1;
+	::encode(v, bl);
+	__u32 _type = (__u32)inc_type;
+	::encode(_type, bl);
+	if (_type == GLOBAL_ID) {
+	  ::encode(max_global_id, bl);
+	} else {
+	  ::encode(auth_type, bl);
+	  ::encode(auth_data, bl);
+	}
+	return;
+      } 
+      ENCODE_START(2, 2, bl);
+      __u32 _type = (__u32)inc_type;
+      ::encode(_type, bl);
+      if (_type == GLOBAL_ID) {
+	::encode(max_global_id, bl);
+      } else {
+	::encode(auth_type, bl);
+	::encode(auth_data, bl);
+      }
+      ENCODE_FINISH(bl);
+    }
+    void decode(bufferlist::iterator& bl) {
+      DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+      __u32 _type;
+      ::decode(_type, bl);
+      inc_type = (IncType)_type;
+      assert(inc_type >= GLOBAL_ID && inc_type <= AUTH_DATA);
+      if (_type == GLOBAL_ID) {
+	::decode(max_global_id, bl);
+      } else {
+	::decode(auth_type, bl);
+	::decode(auth_data, bl);
+      }
+      DECODE_FINISH(bl);
+    }
+    void dump(Formatter *f) const {
+      f->dump_int("type", inc_type);
+      f->dump_int("max_global_id", max_global_id);
+      f->dump_int("auth_type", auth_type);
+      f->dump_int("auth_data_len", auth_data.length());
+    }
+    static void generate_test_instances(list& ls) {
+      ls.push_back(new Incremental);
+      ls.push_back(new Incremental);
+      ls.back()->inc_type = GLOBAL_ID;
+      ls.back()->max_global_id = 1234;
+      ls.push_back(new Incremental);
+      ls.back()->inc_type = AUTH_DATA;
+      ls.back()->auth_type = 12;
+      ls.back()->auth_data.append("foo");
+    }
+  };
+
+private:
+  vector pending_auth;
+  version_t last_rotating_ver;
+  uint64_t max_global_id;
+  uint64_t last_allocated_id;
+
+  void upgrade_format();
+
+  void export_keyring(KeyRing& keyring);
+  void import_keyring(KeyRing& keyring);
+
+  void push_cephx_inc(KeyServerData::Incremental& auth_inc) {
+    Incremental inc;
+    inc.inc_type = AUTH_DATA;
+    ::encode(auth_inc, inc.auth_data);
+    inc.auth_type = CEPH_AUTH_CEPHX;
+    pending_auth.push_back(inc);
+  }
+
+  void on_active();
+  bool should_propose(double& delay);
+  void create_initial();
+  void update_from_paxos(bool *need_bootstrap);
+  void create_pending();  // prepare a new pending
+  bool prepare_global_id(MMonGlobalID *m);
+  void increase_max_global_id();
+  uint64_t assign_global_id(MAuth *m, bool should_increase_max);
+  // propose pending update to peers
+  void encode_pending(MonitorDBStore::Transaction *t);
+  virtual void encode_full(MonitorDBStore::Transaction *t);
+  version_t get_trim_to();
+
+  bool preprocess_query(PaxosServiceMessage *m);  // true if processed.
+  bool prepare_update(PaxosServiceMessage *m);
+
+  bool prep_auth(MAuth *m, bool paxos_writable);
+
+  bool preprocess_command(MMonCommand *m);
+  bool prepare_command(MMonCommand *m);
+
+  bool check_rotate();
+ public:
+  AuthMonitor(Monitor *mn, Paxos *p, const string& service_name)
+    : PaxosService(mn, p, service_name),
+      last_rotating_ver(0),
+      max_global_id(0),
+      last_allocated_id(0)
+  {}
+
+  void pre_auth(MAuth *m);
+  
+  void tick();  // check state, take actions
+
+  void dump_info(Formatter *f);
+};
+
+
+WRITE_CLASS_ENCODER_FEATURES(AuthMonitor::Incremental);
+
+#endif
diff --git a/ceph/src/mon/ConfigKeyService.cc b/ceph/src/mon/ConfigKeyService.cc
new file mode 100644
index 00000000..d2e204d4
--- /dev/null
+++ b/ceph/src/mon/ConfigKeyService.cc
@@ -0,0 +1,203 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+
+#include "mon/Monitor.h"
+#include "mon/QuorumService.h"
+#include "mon/ConfigKeyService.h"
+#include "mon/MonitorDBStore.h"
+
+#include "common/config.h"
+#include "common/cmdparse.h"
+#include "common/errno.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, this)
+static ostream& _prefix(std::ostream *_dout, const Monitor *mon,
+                        const ConfigKeyService *service) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name() << ")." << service->get_name()
+                << "(" << service->get_epoch() << ") ";
+}
+
+const string ConfigKeyService::STORE_PREFIX = "mon_config_key";
+
+int ConfigKeyService::store_get(string key, bufferlist &bl)
+{
+  if (!store_exists(key))
+    return -ENOENT;
+
+  return mon->store->get(STORE_PREFIX, key, bl);
+}
+
+void ConfigKeyService::store_put(string key, bufferlist &bl, Context *cb)
+{
+  bufferlist proposal_bl;
+  MonitorDBStore::Transaction t;
+  t.put(STORE_PREFIX, key, bl);
+  t.encode(proposal_bl);
+
+  paxos->propose_new_value(proposal_bl, cb);
+}
+
+void ConfigKeyService::store_delete(string key, Context *cb)
+{
+  bufferlist proposal_bl;
+  MonitorDBStore::Transaction t;
+  t.erase(STORE_PREFIX, key);
+  t.encode(proposal_bl);
+  paxos->propose_new_value(proposal_bl, cb);
+}
+
+bool ConfigKeyService::store_exists(string key)
+{
+  return mon->store->exists(STORE_PREFIX, key);
+}
+
+void ConfigKeyService::store_list(stringstream &ss)
+{
+  KeyValueDB::Iterator iter =
+    mon->store->get_iterator(STORE_PREFIX);
+
+  JSONFormatter f(true);
+  f.open_array_section("keys");
+
+  while (iter->valid()) {
+    string key(iter->key());
+    f.dump_string("key", key);
+    iter->next();
+  }
+  f.close_section();
+  f.flush(ss);
+}
+
+
+bool ConfigKeyService::service_dispatch(Message *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  if (!in_quorum()) {
+    dout(1) << __func__ << " not in quorum -- ignore message" << dendl;
+    m->put();
+    return false;
+  }
+
+  assert(m != NULL);
+  assert(m->get_type() == MSG_MON_COMMAND);
+
+  MMonCommand *cmd = static_cast(m);
+
+  assert(!cmd->cmd.empty());
+
+  int ret = 0;
+  stringstream ss;
+  bufferlist rdata;
+
+  string prefix;
+  map cmdmap;
+
+  if (!cmdmap_from_json(cmd->cmd, &cmdmap, ss)) {
+    ret = -EINVAL;
+    return false;
+  }
+
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+  string key;
+  cmd_getval(g_ceph_context, cmdmap, "key", key);
+
+  if (prefix == "config-key get") {
+    ret = store_get(key, rdata);
+    if (ret < 0) {
+      assert(!rdata.length());
+      ss << "error obtaining '" << key << "': " << cpp_strerror(ret);
+      goto out;
+    }
+    ss << "obtained '" << key << "'";
+
+  } else if (prefix == "config-key put") {
+    if (!mon->is_leader()) {
+      mon->forward_request_leader(cmd);
+      // we forward the message; so return now.
+      return true;
+    }
+
+    bufferlist data;
+    string val;
+    if (cmd_getval(g_ceph_context, cmdmap, "val", val)) {
+      // they specified a value in the command instead of a file
+      data.append(val);
+    } else if (cmd->get_data_len() > 0) {
+      // they specified '-i '
+      data = cmd->get_data();
+    }
+    if (data.length() > (size_t) g_conf->mon_config_key_max_entry_size) {
+      ret = -EFBIG; // File too large
+      ss << "error: entry size limited to "
+         << g_conf->mon_config_key_max_entry_size << " bytes. "
+         << "Use 'mon config key max entry size' to manually adjust";
+      goto out;
+    }
+    // we'll reply to the message once the proposal has been handled
+    store_put(key, data,
+        new Monitor::C_Command(mon, cmd, 0, "value stored", 0));
+    // return for now; we'll put the message once it's done.
+    return true;
+
+  } else if (prefix == "config-key del") {
+    if (!mon->is_leader()) {
+      mon->forward_request_leader(cmd);
+      return true;
+    }
+
+    if (!store_exists(key)) {
+      ret = 0;
+      ss << "no such key '" << key << "'";
+      goto out;
+    }
+    store_delete(key, new Monitor::C_Command(mon, cmd, 0, "key deleted", 0));
+    // return for now; we'll put the message once it's done
+    return true;
+
+  } else if (prefix == "config-key exists") {
+    bool exists = store_exists(key);
+    ss << "key '" << key << "'";
+    if (exists) {
+      ss << " exists";
+      ret = 0;
+    } else {
+      ss << " doesn't exist";
+      ret = -ENOENT;
+    }
+
+  } else if (prefix == "config-key list") {
+    stringstream tmp_ss;
+    store_list(tmp_ss);
+    rdata.append(tmp_ss);
+    ret = 0;
+  }
+
+out:
+  if (!cmd->get_source().is_mon()) {
+    string rs = ss.str();
+    mon->reply_command(cmd, ret, rs, rdata, 0);
+  } else {
+    cmd->put();
+  }
+
+  return (ret == 0);
+}
+
diff --git a/ceph/src/mon/ConfigKeyService.h b/ceph/src/mon/ConfigKeyService.h
new file mode 100644
index 00000000..e33070b6
--- /dev/null
+++ b/ceph/src/mon/ConfigKeyService.h
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_MON_CONFIG_KEY_SERVICE_H
+#define CEPH_MON_CONFIG_KEY_SERVICE_H
+
+#include "mon/Monitor.h"
+#include "mon/QuorumService.h"
+
+#include "messages/MMonHealth.h"
+
+#include "common/config.h"
+#include "common/Formatter.h"
+
+class Paxos;
+
+class ConfigKeyService : public QuorumService
+{
+  Paxos *paxos;
+
+  int store_get(string key, bufferlist &bl);
+  void store_put(string key, bufferlist &bl, Context *cb = NULL);
+  void store_delete(string key, Context *cb = NULL);
+  void store_list(stringstream &ss);
+  bool store_exists(string key);
+
+  static const string STORE_PREFIX;
+
+protected:
+  virtual void service_shutdown() { }
+
+public:
+  ConfigKeyService(Monitor *m, Paxos *p) :
+    QuorumService(m),
+    paxos(p)
+  { }
+  virtual ~ConfigKeyService() { }
+
+
+  /**
+   * @defgroup ConfigKeyService_Inherited_h Inherited abstract methods
+   * @{
+   */
+  virtual void init() { }
+  virtual void get_health(Formatter *f,
+			  list >& summary,
+                          list > *detail) { }
+  virtual bool service_dispatch(Message *m);
+
+  virtual void start_epoch() { }
+  virtual void finish_epoch() { }
+  virtual void cleanup() { }
+  virtual void service_tick() { }
+
+  virtual int get_type() {
+    return QuorumService::SERVICE_CONFIG_KEY;
+  }
+
+  virtual string get_name() const {
+    return "config_key";
+  }
+
+  /**
+   * @} // ConfigKeyService_Inherited_h
+   */
+};
+
+#endif // CEPH_MON_CONFIG_KEY_SERVICE_H
diff --git a/ceph/src/mon/DataHealthService.cc b/ceph/src/mon/DataHealthService.cc
new file mode 100644
index 00000000..6c6ed290
--- /dev/null
+++ b/ceph/src/mon/DataHealthService.cc
@@ -0,0 +1,268 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include 
+#include "include/memory.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "acconfig.h"
+
+#ifdef HAVE_SYS_VFS_H
+#include 
+#endif
+
+#ifdef HAVE_SYS_MOUNT_H
+#include 
+#endif
+
+#ifdef HAVE_SYS_PARAM_H
+#include 
+#endif
+
+#include "messages/MMonHealth.h"
+#include "include/types.h"
+#include "include/Context.h"
+#include "include/assert.h"
+#include "common/Formatter.h"
+#include "common/errno.h"
+
+#include "mon/Monitor.h"
+#include "mon/QuorumService.h"
+#include "mon/DataHealthService.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, this)
+static ostream& _prefix(std::ostream *_dout, const Monitor *mon,
+                        const DataHealthService *svc) {
+  assert(mon != NULL);
+  assert(svc != NULL);
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name() << ")." << svc->get_name()
+                << "(" << svc->get_epoch() << ") ";
+}
+
+void DataHealthService::start_epoch()
+{
+  dout(10) << __func__ << " epoch " << get_epoch() << dendl;
+  // we are not bound by election epochs, but we should clear the stats
+  // everytime an election is triggerd.  As far as we know, a monitor might
+  // have been running out of disk space and someone fixed it.  We don't want
+  // to hold the cluster back, even confusing the user, due to some possibly
+  // outdated stats.
+  stats.clear();
+  last_warned_percent = 0;
+}
+
+void DataHealthService::get_health(
+    Formatter *f,
+    list >& summary,
+    list > *detail)
+{
+  dout(10) << __func__ << dendl;
+  if (f) {
+    f->open_object_section("data_health");
+    f->open_array_section("mons");
+  }
+
+  for (map::iterator it = stats.begin();
+       it != stats.end(); ++it) {
+    string mon_name = mon->monmap->get_name(it->first.addr);
+    DataStats& stats = it->second;
+
+    health_status_t health_status = HEALTH_OK;
+    string health_detail;
+    if (stats.latest_avail_percent <= g_conf->mon_data_avail_crit) {
+      health_status = HEALTH_ERR;
+      health_detail = "low disk space, shutdown imminent";
+    } else if (stats.latest_avail_percent <= g_conf->mon_data_avail_warn) {
+      health_status = HEALTH_WARN;
+      health_detail = "low disk space";
+    }
+
+    if (stats.store_stats.bytes_total >= g_conf->mon_leveldb_size_warn) {
+      if (health_status > HEALTH_WARN)
+        health_status = HEALTH_WARN;
+      if (!health_detail.empty())
+        health_detail.append("; ");
+      stringstream ss;
+      ss << "store is getting too big! "
+         << prettybyte_t(stats.store_stats.bytes_total)
+         << " >= " << prettybyte_t(g_conf->mon_leveldb_size_warn);
+      health_detail.append(ss.str());
+    }
+
+    if (health_status != HEALTH_OK) {
+      stringstream ss;
+      ss << "mon." << mon_name << " " << health_detail;
+      summary.push_back(make_pair(health_status, ss.str()));
+      ss << " -- " <<  stats.latest_avail_percent << "% avail";
+      if (detail)
+	detail->push_back(make_pair(health_status, ss.str()));
+    }
+
+    if (f) {
+      f->open_object_section("mon");
+      f->dump_string("name", mon_name.c_str());
+      // leave this unenclosed by an object section to avoid breaking backward-compatibility
+      stats.dump(f);
+      f->dump_stream("health") << health_status;
+      if (health_status != HEALTH_OK)
+        f->dump_string("health_detail", health_detail);
+      f->close_section();
+    }
+  }
+
+  if (f) {
+    f->close_section(); // mons
+    f->close_section(); // data_health
+  }
+}
+
+int DataHealthService::update_store_stats(DataStats &ours)
+{
+  map extra;
+  uint64_t store_size = mon->store->get_estimated_size(extra);
+  assert(store_size > 0);
+
+  ours.store_stats.bytes_total = store_size;
+  ours.store_stats.bytes_sst = extra["sst"];
+  ours.store_stats.bytes_log = extra["log"];
+  ours.store_stats.bytes_misc = extra["misc"];
+  ours.last_update = ceph_clock_now(g_ceph_context);
+
+  return 0;
+}
+
+
+int DataHealthService::update_stats()
+{
+  struct statfs stbuf;
+  int err = ::statfs(g_conf->mon_data.c_str(), &stbuf);
+  if (err < 0) {
+    derr << __func__ << " statfs error: " << cpp_strerror(errno) << dendl;
+    return -errno;
+  }
+
+  entity_inst_t our_inst = mon->messenger->get_myinst();
+  DataStats& ours = stats[our_inst];
+
+  ours.kb_total = stbuf.f_blocks * stbuf.f_bsize / 1024;
+  ours.kb_used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize / 1024;
+  ours.kb_avail = stbuf.f_bavail * stbuf.f_bsize / 1024;
+  ours.latest_avail_percent = (((float)ours.kb_avail/ours.kb_total)*100);
+  dout(0) << __func__ << " avail " << ours.latest_avail_percent << "%"
+          << " total " << ours.kb_total << " used " << ours.kb_used << " avail " << ours.kb_avail
+          << dendl;
+  ours.last_update = ceph_clock_now(g_ceph_context);
+
+  return update_store_stats(ours);
+}
+
+void DataHealthService::share_stats()
+{
+  dout(10) << __func__ << dendl;
+  if (!in_quorum())
+    return;
+
+  assert(!stats.empty());
+  entity_inst_t our_inst = mon->messenger->get_myinst();
+  assert(stats.count(our_inst) > 0);
+  DataStats &ours = stats[our_inst];
+  const set& quorum = mon->get_quorum();
+  for (set::const_iterator it = quorum.begin();
+       it != quorum.end(); ++it) {
+    if (mon->monmap->get_name(*it) == mon->name)
+      continue;
+    entity_inst_t inst = mon->monmap->get_inst(*it);
+    MMonHealth *m = new MMonHealth(HealthService::SERVICE_HEALTH_DATA,
+                                   MMonHealth::OP_TELL);
+    m->data_stats = ours;
+    dout(20) << __func__ << " send " << *m << " to " << inst << dendl;
+    mon->messenger->send_message(m, inst);
+  }
+}
+
+void DataHealthService::service_tick()
+{
+  dout(10) << __func__ << dendl;
+
+  int err = update_stats();
+  if (err < 0) {
+    derr << "something went wrong obtaining our disk stats: "
+         << cpp_strerror(err) << dendl;
+    force_shutdown();
+    return;
+  }
+  if (in_quorum())
+    share_stats();
+
+  DataStats &ours = stats[mon->messenger->get_myinst()];
+
+  if (ours.latest_avail_percent <= g_conf->mon_data_avail_crit) {
+    derr << "reached critical levels of available space on local monitor storage"
+         << " -- shutdown!" << dendl;
+    force_shutdown();
+    return;
+  }
+
+  // we must backoff these warnings, and track how much data is being
+  // consumed in-between reports to assess if it's worth to log this info,
+  // otherwise we may very well contribute to the consumption of the
+  // already low available disk space.
+  if (ours.latest_avail_percent <= g_conf->mon_data_avail_warn) {
+    if (ours.latest_avail_percent != last_warned_percent)
+      mon->clog.warn()
+	<< "reached concerning levels of available space on local monitor storage"
+	<< " (" << ours.latest_avail_percent << "% free)\n";
+    last_warned_percent = ours.latest_avail_percent;
+  } else {
+    last_warned_percent = 0;
+  }
+}
+
+void DataHealthService::handle_tell(MMonHealth *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  assert(m->get_service_op() == MMonHealth::OP_TELL);
+
+  stats[m->get_source_inst()] = m->data_stats;
+}
+
+bool DataHealthService::service_dispatch(MMonHealth *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  assert(m->get_service_type() == get_type());
+  if (!in_quorum()) {
+    dout(1) << __func__ << " not in quorum -- drop message" << dendl;
+    m->put();
+    return false;
+  }
+
+  switch (m->service_op) {
+    case MMonHealth::OP_TELL:
+      // someone is telling us their stats
+      handle_tell(m);
+      break;
+    default:
+      dout(0) << __func__ << " unknown op " << m->service_op << dendl;
+      assert(0 == "Unknown service op");
+      break;
+  }
+  m->put();
+  return true;
+}
diff --git a/ceph/src/mon/DataHealthService.h b/ceph/src/mon/DataHealthService.h
new file mode 100644
index 00000000..221e1794
--- /dev/null
+++ b/ceph/src/mon/DataHealthService.h
@@ -0,0 +1,86 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_MON_DATA_HEALTH_SERVICE_H
+#define CEPH_MON_DATA_HEALTH_SERVICE_H
+
+#include 
+
+#include "include/types.h"
+#include "include/Context.h"
+#include "mon/mon_types.h"
+#include "mon/QuorumService.h"
+#include "mon/HealthService.h"
+#include "common/Formatter.h"
+#include "common/config.h"
+#include "global/signal_handler.h"
+
+struct MMonHealth;
+
+class DataHealthService :
+  public HealthService
+{
+  map stats;
+  int last_warned_percent;
+
+  void handle_tell(MMonHealth *m);
+  int update_store_stats(DataStats &ours);
+  int update_stats();
+  void share_stats();
+
+  void force_shutdown() {
+    generic_dout(0) << "** Shutdown via Data Health Service **" << dendl;
+    queue_async_signal(SIGINT);
+  }
+
+protected:
+  virtual void service_tick();
+  virtual bool service_dispatch(Message *m) {
+    assert(0 == "We should never reach this; only the function below");
+    return false;
+  }
+  virtual bool service_dispatch(MMonHealth *m);
+  virtual void service_shutdown() { }
+
+  virtual void start_epoch();
+  virtual void finish_epoch() { }
+  virtual void cleanup() { }
+
+public:
+  DataHealthService(Monitor *m) :
+    HealthService(m),
+    last_warned_percent(0)
+  {
+    set_update_period(g_conf->mon_health_data_update_interval);
+  }
+  virtual ~DataHealthService() { }
+
+  virtual void init() {
+    generic_dout(20) << "data_health " << __func__ << dendl;
+    start_tick();
+  }
+
+  virtual void get_health(Formatter *f,
+                          list >& summary,
+                          list > *detail);
+
+  virtual int get_type() {
+    return HealthService::SERVICE_HEALTH_DATA;
+  }
+
+  virtual string get_name() const {
+    return "data_health";
+  }
+};
+
+#endif /* CEPH_MON_DATA_HEALTH_SERVICE_H */
diff --git a/ceph/src/mon/DumplingMonCommands.h b/ceph/src/mon/DumplingMonCommands.h
new file mode 100644
index 00000000..8e9c2bb3
--- /dev/null
+++ b/ceph/src/mon/DumplingMonCommands.h
@@ -0,0 +1,531 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc. 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/* no guard; may be included multiple times */
+
+/*
+ * Define commands that are reported by the monitor's
+ * "get_command_descriptions" command, and parsed by the Python
+ * frontend 'ceph' (and perhaps by other frontends, such as a RESTful
+ * server). The format is:
+ *
+ * COMMAND(signature, helpstring, modulename, req perms, availability)
+ * where:
+ * signature:  describes the command and its parameters (more below)
+ * helpstring: displays in CLI help, API help (nice if it refers to
+ *             parameter names from signature, 40-a few hundred chars)
+ * modulename: the monitor module or daemon this applies to:
+ *             mds, osd, pg (osd), mon, auth, log, config-key
+ * req perms:  required permission in that modulename space to execute command
+ *             this also controls what type of REST command is accepted
+ * availability: cli, rest, or both
+ *
+ * The commands describe themselves completely enough for the separate
+ * frontend(s) to be able to accept user input and validate it against
+ * the command descriptions, and generate a JSON object that contains
+ * key:value mappings of parameter names to validated parameter values.
+ *
+ * 'signature' is a space-separated list of individual command descriptors;
+ * each descriptor is either a literal string, which can contain no spaces or
+ * '=' signs (for instance, in "pg stat", both "pg" and "stat" are literal
+ * strings representing one descriptor each), or a list of key=val[,key=val...]
+ * which also includes no spaces.  
+ *
+ * The key=val form describes a non-literal parameter.  Each will have at
+ * least a name= and type=, and each type can have its own type-specific
+ * parameters.  The parser is the arbiter of these types and their 
+ * interpretation.  A few more non-type-specific key=val pairs exist:
+ *
+ *    req=false marks an optional parameter (default for req is 'true')
+ *    n= is a repeat count for how many of this argument must be supplied.
+ *          n=1 is the default.
+ *          n=N is a special case that means "1 or more".
+ *
+ * A perhaps-incomplete list of types:
+ *
+ * CephInt: Optional: range=min[|max]
+ * CephFloat: Optional range
+ * CephString: optional badchars
+ * CephSocketpath: validation involves "is it S_ISSOCK"
+ * CephIPAddr: v4 or v6 addr with optional port, syntax validated
+ * CephEntityAddr: CephIPAddr + '/nonce'
+ * CephPoolname: Plainold string
+ * CephObjectname: Another plainold string
+ * CephPgid: n.xxx where n is an int > 0, xxx is a hex number > 0
+ * CephName: daemon name, '*' or '.' (id must be int for type osd)
+ * CephOsdName: osd name, '*' or ' or 'osd.' (id must be int)
+ * CephChoices: strings="foo|bar" means this param can be either
+ * CephFilepath: openable file
+ * CephFragment: cephfs 'fragID': val/bits, val in hex 0xnnn, bits in dec
+ * CephUUID: uuid in text matching Python uuid.UUID()
+ * CephPrefix: special type assigned to literals
+ *
+ * Example:
+ *
+ * COMMAND("auth add " \
+ *   	   "name=entity,type=CephString " \
+ *   	   "name=caps,type=CephString,n=N,req=false", \
+ *   	   "add auth info for  from input file, or random key " \
+ *   	   "if no input given, and/or any caps specified in the command")
+ *
+ * defines a command "auth add" that takes a required argument "entity"
+ * of type "CephString", and from 1 to N arguments named "caps" of type
+ * CephString, at least one of which is required.  The front end will
+ * validate user input against this description.  Let's say the user
+ * enters auth add client.admin 'mon rwx' 'osd *'.  The result will be a
+ * JSON object like {"prefix":"auth add", "entity":"client.admin",
+ * "caps":["mon rwx", "osd *"]}.
+ * Note that 
+ * 	- string literals are accumulated into 'prefix'
+ * 	- n=1 descriptors are given normal string or int object values
+ * 	- n=N descriptors are given array values
+ *
+ * NOTE: be careful with spaces.  Each descriptor must be separated by
+ * one space, no other characters, so if you split lines as above, be
+ * sure to close and reopen the quotes, and be careful to include the '
+ * separating spaces in the quoted string.
+ *
+ * The monitor marshals this JSON into a std::map
+* where cmd_vartype is a boost::variant type-enforcing discriminated
+* type, so the monitor is expected to know the type of each argument.
+* See cmdparse.cc/h for more details.
+*/
+
+/*
+ * pg commands PgMonitor.cc
+ */
+
+COMMAND("pg stat", "show placement group status.", "pg", "r", "cli,rest")
+COMMAND("pg getmap", "get binary pg map to -o/stdout", "pg", "r", "cli,rest")
+COMMAND("pg send_pg_creates", "trigger pg creates to be issued",\
+	"pg", "rw", "cli,rest")
+COMMAND("pg dump " \
+	"name=dumpcontents,type=CephChoices,strings=all|summary|sum|pools|osds|pgs|pgs_brief,n=N,req=false", \
+	"show human-readable versions of pg map", "pg", "r", "cli,rest")
+COMMAND("pg dump_json " \
+	"name=dumpcontents,type=CephChoices,strings=all|summary|sum|pools|osds|pgs,n=N,req=false", \
+	"show human-readable version of pg map in json only",\
+	"pg", "r", "cli,rest")
+COMMAND("pg dump_pools_json", "show pg pools info in json only",\
+	"pg", "r", "cli,rest")
+COMMAND("pg dump_stuck " \
+	"name=stuckops,type=CephChoices,strings=inactive|unclean|stale,n=N,req=false " \
+	"name=threshold,type=CephInt,req=false",
+	"show information about stuck pgs",\
+	"pg", "r", "cli,rest")
+COMMAND("pg map name=pgid,type=CephPgid", "show mapping of pg to osds", \
+	"pg", "r", "cli,rest")
+COMMAND("pg scrub name=pgid,type=CephPgid", "start scrub on ", \
+	"pg", "rw", "cli,rest")
+COMMAND("pg deep-scrub name=pgid,type=CephPgid", "start deep-scrub on ", \
+	"pg", "rw", "cli,rest")
+COMMAND("pg repair name=pgid,type=CephPgid", "start repair on ", \
+	"pg", "rw", "cli,rest")
+COMMAND("pg debug " \
+	"name=debugop,type=CephChoices,strings=unfound_objects_exist|degraded_pgs_exist", \
+	"show debug info about pgs", "pg", "r", "cli,rest")
+COMMAND("pg force_create_pg name=pgid,type=CephPgid", \
+	"force creation of pg ", "pg", "rw", "cli,rest")
+COMMAND("pg set_full_ratio name=ratio,type=CephFloat,range=0.0|1.0", \
+	"set ratio at which pgs are considered full", "pg", "rw", "cli,rest")
+COMMAND("pg set_nearfull_ratio name=ratio,type=CephFloat,range=0.0|1.0", \
+	"set ratio at which pgs are considered nearly full", \
+	"pg", "rw", "cli,rest")
+
+/*
+ * auth commands AuthMonitor.cc
+ */
+
+COMMAND("auth export name=entity,type=CephString,req=false", \
+       	"write keyring for requested entity, or master keyring if none given", \
+	"auth", "r", "cli,rest")
+COMMAND("auth get name=entity,type=CephString", \
+	"write keyring file with requested key", "auth", "r", "cli,rest")
+COMMAND("auth get-key name=entity,type=CephString", "display requested key", \
+	"auth", "r", "cli,rest")
+COMMAND("auth print-key name=entity,type=CephString", "display requested key", \
+	"auth", "r", "cli,rest")
+COMMAND("auth print_key name=entity,type=CephString", "display requested key", \
+	"auth", "r", "cli,rest")
+COMMAND("auth list", "list authentication state", "auth", "r", "cli,rest")
+COMMAND("auth import", "auth import: read keyring file from -i ", \
+	"auth", "rw", "cli,rest")
+COMMAND("auth add " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N,req=false", \
+	"add auth info for  from input file, or random key if no input given, and/or any caps specified in the command",
+	"auth", "rw", "cli,rest")
+COMMAND("auth get-or-create-key " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N,req=false", \
+	"get, or add, key for  from system/caps pairs specified in the command.  If key already exists, any given caps must match the existing caps for that key.", \
+	"auth", "rw", "cli,rest")
+COMMAND("auth get-or-create " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N,req=false", \
+	"add auth info for  from input file, or random key if no input given, and/or any caps specified in the command", \
+	"auth", "rw", "cli,rest")
+COMMAND("auth caps " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N", \
+	"update caps for  from caps specified in the command", \
+	"auth", "rw", "cli,rest")
+COMMAND("auth del " \
+	"name=entity,type=CephString", \
+	"delete all caps for ", \
+	"auth", "rw", "cli,rest")
+
+/*
+ * Monitor commands (Monitor.cc)
+ */
+COMMAND("compact", "cause compaction of monitor's leveldb storage", \
+	"mon", "rw", "cli,rest")
+COMMAND("scrub", "scrub the monitor stores", "mon", "rw", "cli,rest")
+COMMAND("fsid", "show cluster FSID/UUID", "mon", "r", "cli,rest")
+COMMAND("log name=logtext,type=CephString,n=N", \
+	"log supplied text to the monitor log", "mon", "rw", "cli,rest")
+COMMAND("injectargs " \
+	"name=injected_args,type=CephString,n=N", \
+	"inject config arguments into monitor", "mon", "rw", "cli,rest")
+COMMAND("status", "show cluster status", "mon", "r", "cli,rest")
+COMMAND("health name=detail,type=CephChoices,strings=detail,req=false", \
+	"show cluster health", "mon", "r", "cli,rest")
+COMMAND("df name=detail,type=CephChoices,strings=detail,req=false", \
+	"show cluster free space stats", "mon", "r", "cli,rest")
+COMMAND("report name=tags,type=CephString,n=N,req=false", \
+	"report full status of cluster, optional title tag strings", \
+	"mon", "r", "cli,rest")
+COMMAND("quorum_status", "report status of monitor quorum", \
+	"mon", "r", "cli,rest")
+COMMAND("mon_status", "report status of monitors", "mon", "r", "cli,rest")
+COMMAND("sync force " \
+	"name=validate1,type=CephChoices,strings=--yes-i-really-mean-it " \
+	"name=validate2,type=CephChoices,strings=--i-know-what-i-am-doing", \
+	"force sync of and clear monitor store", "mon", "rw", "cli,rest")
+COMMAND("heap " \
+	"name=heapcmd,type=CephChoices,strings=dump|start_profiler|stop_profiler|release|stats", \
+	"show heap usage info (available only if compiled with tcmalloc)", \
+	"mon", "rw", "cli,rest")
+COMMAND("quorum name=quorumcmd,type=CephChoices,strings=enter|exit,n=1", \
+	"enter or exit quorum", "mon", "rw", "cli,rest")
+COMMAND("tell " \
+	"name=target,type=CephName " \
+	"name=args,type=CephString,n=N", \
+	"send a command to a specific daemon", "mon", "rw", "cli,rest")
+
+/*
+ * MDS commands (MDSMonitor.cc)
+ */
+
+COMMAND("mds stat", "show MDS status", "mds", "r", "cli,rest")
+COMMAND("mds dump " 
+	"name=epoch,type=CephInt,req=false,range=0", \
+	"dump info, optionally from epoch", "mds", "r", "cli,rest")
+COMMAND("mds getmap " \
+	"name=epoch,type=CephInt,req=false,range=0", \
+	"get MDS map, optionally from epoch", "mds", "r", "cli,rest")
+COMMAND("mds tell " \
+	"name=who,type=CephString " \
+	"name=args,type=CephString,n=N", \
+	"send command to particular mds", "mds", "rw", "cli,rest")
+COMMAND("mds compat show", "show mds compatibility settings", \
+	"mds", "r", "cli,rest")
+COMMAND("mds stop name=who,type=CephString", "stop mds", \
+	"mds", "rw", "cli,rest")
+COMMAND("mds deactivate name=who,type=CephString", "stop mds", \
+	"mds", "rw", "cli,rest")
+COMMAND("mds set_max_mds " \
+	"name=maxmds,type=CephInt,range=0", \
+	"set max MDS index", "mds", "rw", "cli,rest")
+COMMAND("mds setmap " \
+	"name=epoch,type=CephInt,range=0", \
+	"set mds map; must supply correct epoch number", "mds", "rw", "cli,rest")
+// arbitrary limit 0-20 below; worth standing on head to make it
+// relate to actual state definitions?
+// #include "include/ceph_fs.h"
+COMMAND("mds set_state " \
+	"name=gid,type=CephInt,range=0 " \
+	"name=state,type=CephInt,range=0|20", \
+	"set mds state of  to ", "mds", "rw", "cli,rest")
+COMMAND("mds fail name=who,type=CephString", \
+	"force mds to status failed", "mds", "rw", "cli,rest")
+COMMAND("mds rm " \
+	"name=gid,type=CephInt,range=0 " \
+	"name=who,type=CephName", \
+	"remove nonactive mds", "mds", "rw", "cli,rest")
+COMMAND("mds rmfailed name=who,type=CephInt,range=0", "remove failed mds", \
+	"mds", "rw", "cli,rest")
+COMMAND("mds cluster_down", "take MDS cluster down", "mds", "rw", "cli,rest")
+COMMAND("mds cluster_up", "bring MDS cluster up", "mds", "rw", "cli,rest")
+COMMAND("mds compat rm_compat " \
+	"name=feature,type=CephInt,range=0", \
+	"remove compatible feature", "mds", "rw", "cli,rest")
+COMMAND("mds compat rm_incompat " \
+	"name=feature,type=CephInt,range=0", \
+	"remove incompatible feature", "mds", "rw", "cli,rest")
+COMMAND("mds add_data_pool " \
+	"name=poolid,type=CephInt,range=0", \
+	"add data pool ", "mds", "rw", "cli,rest")
+COMMAND("mds remove_data_pool " \
+	"name=poolid,type=CephInt,range=0", \
+	"remove data pool ", "mds", "rw", "cli,rest")
+COMMAND("mds newfs " \
+	"name=metadata,type=CephInt,range=0 " \
+	"name=data,type=CephInt,range=0 " \
+	"name=sure,type=CephChoices,strings=--yes-i-really-mean-it", \
+	"make new filesystom using pools  and ", \
+	"mds", "rw", "cli,rest")
+/*
+ * Monmap commands
+ */
+COMMAND("mon dump " \
+	"name=epoch,type=CephInt,req=false", \
+	"dump formatted monmap (optionally from epoch)", \
+	"mon", "r", "cli,rest")
+COMMAND("mon stat", "summarize monitor status", "mon", "r", "cli,rest")
+COMMAND("mon getmap " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"get monmap", "mon", "r", "cli,rest")
+COMMAND("mon add " \
+	"name=name,type=CephString " \
+	"name=addr,type=CephIPAddr", \
+	"add new monitor named  at ", "mon", "rw", "cli,rest")
+COMMAND("mon remove " \
+	"name=name,type=CephString", \
+	"remove monitor named ", "mon", "rw", "cli,rest")
+
+
+/*
+ * OSD commands
+ */
+COMMAND("osd stat", "print summary of OSD map", "osd", "r", "cli,rest")
+COMMAND("osd dump " \
+	"name=epoch,type=CephInt,range=0,req=false",
+	"print summary of OSD map", "osd", "r", "cli,rest")
+COMMAND("osd tree " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"print OSD tree", "osd", "r", "cli,rest")
+COMMAND("osd ls " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"show all OSD ids", "osd", "r", "cli,rest")
+COMMAND("osd getmap " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"get OSD map", "osd", "r", "cli,rest")
+COMMAND("osd getcrushmap " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"get CRUSH map", "osd", "r", "cli,rest")
+COMMAND("osd getmaxosd", "show largest OSD id", "osd", "r", "cli,rest")
+COMMAND("osd find " \
+	"name=id,type=CephInt,range=0", \
+	"find osd  in the CRUSH map and show its location", \
+	"osd", "r", "cli,rest")
+COMMAND("osd map " \
+	"name=pool,type=CephPoolname " \
+	"name=object,type=CephObjectname", \
+	"find pg for  in ", "osd", "r", "cli,rest")
+COMMAND("osd scrub " \
+	"name=who,type=CephString", \
+	"initiate scrub on osd ", "osd", "rw", "cli,rest")
+COMMAND("osd deep-scrub " \
+	"name=who,type=CephString", \
+	"initiate deep scrub on osd ", "osd", "rw", "cli,rest")
+COMMAND("osd repair " \
+	"name=who,type=CephString", \
+	"initiate repair on osd ", "osd", "rw", "cli,rest")
+COMMAND("osd lspools " \
+	"name=auid,type=CephInt,req=false", \
+	"list pools", "osd", "r", "cli,rest")
+COMMAND("osd blacklist ls", "show blacklisted clients", "osd", "r", "cli,rest")
+COMMAND("osd crush rule list", "list crush rules", "osd", "r", "cli,rest")
+COMMAND("osd crush rule ls", "list crush rules", "osd", "r", "cli,rest")
+COMMAND("osd crush rule dump", "dump crush rules", "osd", "r", "cli,rest")
+COMMAND("osd crush dump", "dump crush map", "osd", "r", "cli,rest")
+COMMAND("osd setcrushmap", "set crush map from input file", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush set", "set crush map from input file", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush add-bucket " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=type,type=CephString", \
+	"add no-parent (probably root) crush bucket  of type ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush set " \
+	"name=id,type=CephOsdName " \
+	"name=weight,type=CephFloat,range=0.0 " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"set crushmap entry for  to  with location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush add " \
+	"name=id,type=CephOsdName " \
+	"name=weight,type=CephFloat,range=0.0 " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"add crushmap entry for  with  and location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush create-or-move " \
+	"name=id,type=CephOsdName " \
+	"name=weight,type=CephFloat,range=0.0 " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"create entry or move existing entry for   at/to location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush move " \
+	"name=id,type=CephOsdName " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"move existing entry for  to location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush link " \
+	"name=name,type=CephString " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"link existing entry for  under location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush rm " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
+	"remove  from crush map (everywhere, or just at ",\
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush remove " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
+	"remove  from crush map (everywhere, or just at ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush unlink " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
+	"unlink  from crush map (everywhere, or just at ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush reweight " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=weight,type=CephFloat,range=0.0", \
+	"change 's weight to  in crush map", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush tunables " \
+	"name=profile,type=CephChoices,strings=legacy|argonaut|bobtail|optimal|default", \
+	"set crush tunables values to ", "osd", "rw", "cli,rest")
+COMMAND("osd crush rule create-simple " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=root,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=type,type=CephString,goodchars=[A-Za-z0-9-_.]",
+	"create crush rule  in  of type ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush rule rm " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] ",	\
+	"remove crush rule ", "osd", "rw", "cli,rest")
+COMMAND("osd setmaxosd " \
+	"name=newmax,type=CephInt,range=0", \
+	"set new maximum osd value", "osd", "rw", "cli,rest")
+COMMAND("osd pause", "pause osd", "osd", "rw", "cli,rest")
+COMMAND("osd unpause", "unpause osd", "osd", "rw", "cli,rest")
+COMMAND("osd set " \
+	"name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub", \
+	"set ", "osd", "rw", "cli,rest")
+COMMAND("osd unset " \
+	"name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub", \
+	"unset ", "osd", "rw", "cli,rest")
+COMMAND("osd cluster_snap", "take cluster snapshot (disabled)", \
+	"osd", "r", "")
+COMMAND("osd down " \
+	"type=CephString,name=ids,n=N", \
+	"set osd(s)  [...] down", "osd", "rw", "cli,rest")
+COMMAND("osd out " \
+	"name=ids,type=CephString,n=N", \
+	"set osd(s)  [...] out", "osd", "rw", "cli,rest")
+COMMAND("osd in " \
+	"name=ids,type=CephString,n=N", \
+	"set osd(s)  [...] in", "osd", "rw", "cli,rest")
+COMMAND("osd rm " \
+	"name=ids,type=CephString,n=N", \
+	"remove osd(s)  [...] in", "osd", "rw", "cli,rest")
+COMMAND("osd reweight " \
+	"name=id,type=CephInt,range=0 " \
+	"type=CephFloat,name=weight,range=0.0|1.0", \
+	"reweight osd to 0.0 <  < 1.0", "osd", "rw", "cli,rest")
+COMMAND("osd lost " \
+	"name=id,type=CephInt,range=0 " \
+	"name=sure,type=CephChoices,strings=--yes-i-really-mean-it", \
+	"mark osd as permanently lost. THIS DESTROYS DATA IF NO MORE REPLICAS EXIST, BE CAREFUL", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd create " \
+	"name=uuid,type=CephUUID,req=false", \
+	"create new osd (with optional UUID)", "osd", "rw", "cli,rest")
+COMMAND("osd blacklist " \
+	"name=blacklistop,type=CephChoices,strings=add|rm " \
+	"name=addr,type=CephEntityAddr " \
+	"name=expire,type=CephFloat,range=0.0,req=false", \
+	"add (optionally until  seconds from now) or remove  from blacklist", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd pool mksnap " \
+	"name=pool,type=CephPoolname " \
+	"name=snap,type=CephString", \
+	"make snapshot  in ", "osd", "rw", "cli,rest")
+COMMAND("osd pool rmsnap " \
+	"name=pool,type=CephPoolname " \
+	"name=snap,type=CephString", \
+	"remove snapshot  from ", "osd", "rw", "cli,rest")
+COMMAND("osd pool create " \
+	"name=pool,type=CephPoolname " \
+	"name=pg_num,type=CephInt,range=0 " \
+	"name=pgp_num,type=CephInt,range=0,req=false", \
+	"create pool", "osd", "rw", "cli,rest")
+COMMAND("osd pool delete " \
+	"name=pool,type=CephPoolname " \
+	"name=pool2,type=CephPoolname " \
+	"name=sure,type=CephChoices,strings=--yes-i-really-really-mean-it", \
+	"delete pool (say pool twice, add --yes-i-really-really-mean-it)", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd pool rename " \
+	"name=srcpool,type=CephPoolname " \
+	"name=destpool,type=CephPoolname", \
+	"rename  to ", "osd", "rw", "cli,rest")
+COMMAND("osd pool get " \
+	"name=pool,type=CephPoolname " \
+	"name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset", \
+	"get pool parameter ", "osd", "r", "cli,rest")
+COMMAND("osd pool set " \
+	"name=pool,type=CephPoolname " \
+	"name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset " \
+	"name=val,type=CephInt", \
+	"set pool parameter  to ", "osd", "rw", "cli,rest")
+// 'val' is a CephString because it can include a unit.  Perhaps
+// there should be a Python type for validation/conversion of strings
+// with units.
+COMMAND("osd pool set-quota " \
+	"name=pool,type=CephPoolname " \
+	"name=field,type=CephChoices,strings=max_objects|max_bytes " \
+	"name=val,type=CephString",
+	"set object or byte limit on pool", "osd", "rw", "cli,rest")
+COMMAND("osd reweight-by-utilization " \
+	"name=oload,type=CephInt,range=100,req=false", \
+	"reweight OSDs by utilization [overload-percentage-for-consideration, default 120]", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd thrash " \
+	"name=num_epochs,type=CephInt,range=0", \
+	"thrash OSDs for ", "osd", "rw", "cli,rest")
+
+/*
+ * mon/ConfigKeyService.cc
+ */
+
+COMMAND("config-key get " \
+	"name=key,type=CephString", \
+	"get ", "config-key", "r", "cli,rest")
+COMMAND("config-key put " \
+	"name=key,type=CephString " \
+	"name=val,type=CephString,req=false", \
+	"put , value ", "config-key", "rw", "cli,rest")
+COMMAND("config-key del " \
+	"name=key,type=CephString", \
+	"delete ", "config-key", "rw", "cli,rest")
+COMMAND("config-key exists " \
+	"name=key,type=CephString", \
+	"check for 's existence", "config-key", "r", "cli,rest")
+COMMAND("config-key list ", "list keys", "config-key", "r", "cli,rest")
diff --git a/ceph/src/mon/Elector.cc b/ceph/src/mon/Elector.cc
new file mode 100644
index 00000000..2ee10f7b
--- /dev/null
+++ b/ceph/src/mon/Elector.cc
@@ -0,0 +1,489 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "Elector.h"
+#include "Monitor.h"
+
+#include "common/Timer.h"
+#include "MonitorDBStore.h"
+#include "MonmapMonitor.h"
+#include "messages/MMonElection.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, epoch)
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, epoch_t epoch) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").elector(" << epoch << ") ";
+}
+
+
+void Elector::init()
+{
+  epoch = mon->store->get(Monitor::MONITOR_NAME, "election_epoch");
+  if (!epoch)
+    epoch = 1;
+  dout(1) << "init, last seen epoch " << epoch << dendl;
+}
+
+void Elector::shutdown()
+{
+  if (expire_event)
+    mon->timer.cancel_event(expire_event);
+}
+
+void Elector::bump_epoch(epoch_t e) 
+{
+  dout(10) << "bump_epoch " << epoch << " to " << e << dendl;
+  assert(epoch <= e);
+  epoch = e;
+  MonitorDBStore::Transaction t;
+  t.put(Monitor::MONITOR_NAME, "election_epoch", epoch);
+  mon->store->apply_transaction(t);
+
+  mon->join_election();
+
+  // clear up some state
+  electing_me = false;
+  acked_me.clear();
+  classic_mons.clear();
+}
+
+
+void Elector::start()
+{
+  if (!participating) {
+    dout(0) << "not starting new election -- not participating" << dendl;
+    return;
+  }
+  dout(5) << "start -- can i be leader?" << dendl;
+
+  acked_me.clear();
+  classic_mons.clear();
+  init();
+  
+  // start by trying to elect me
+  if (epoch % 2 == 0) 
+    bump_epoch(epoch+1);  // odd == election cycle
+  start_stamp = ceph_clock_now(g_ceph_context);
+  electing_me = true;
+  acked_me[mon->rank] = CEPH_FEATURES_ALL;
+  leader_acked = -1;
+
+  // bcast to everyone else
+  for (unsigned i=0; imonmap->size(); ++i) {
+    if ((int)i == mon->rank) continue;
+    Message *m = new MMonElection(MMonElection::OP_PROPOSE, epoch, mon->monmap);
+    mon->messenger->send_message(m, mon->monmap->get_inst(i));
+  }
+  
+  reset_timer();
+}
+
+void Elector::defer(int who)
+{
+  dout(5) << "defer to " << who << dendl;
+
+  if (electing_me) {
+    // drop out
+    acked_me.clear();
+    classic_mons.clear();
+    electing_me = false;
+  }
+
+  // ack them
+  leader_acked = who;
+  ack_stamp = ceph_clock_now(g_ceph_context);
+  MMonElection *m = new MMonElection(MMonElection::OP_ACK, epoch, mon->monmap);
+  m->sharing_bl = mon->get_supported_commands_bl();
+  mon->messenger->send_message(m, mon->monmap->get_inst(who));
+  
+  // set a timer
+  reset_timer(1.0);  // give the leader some extra time to declare victory
+}
+
+
+void Elector::reset_timer(double plus)
+{
+  // set the timer
+  cancel_timer();
+  expire_event = new C_ElectionExpire(this);
+  mon->timer.add_event_after(g_conf->mon_lease + plus,
+			     expire_event);
+}
+
+
+void Elector::cancel_timer()
+{
+  if (expire_event) {
+    mon->timer.cancel_event(expire_event);
+    expire_event = 0;
+  }
+}
+
+void Elector::expire()
+{
+  dout(5) << "election timer expired" << dendl;
+  
+  // did i win?
+  if (electing_me &&
+      acked_me.size() > (unsigned)(mon->monmap->size() / 2)) {
+    // i win
+    victory();
+  } else {
+    // whoever i deferred to didn't declare victory quickly enough.
+    if (mon->has_ever_joined)
+      start();
+    else
+      mon->bootstrap();
+  }
+}
+
+
+void Elector::victory()
+{
+  leader_acked = -1;
+  electing_me = false;
+
+  uint64_t features = CEPH_FEATURES_ALL;
+  set quorum;
+  for (map::iterator p = acked_me.begin(); p != acked_me.end();
+       ++p) {
+    quorum.insert(p->first);
+    features &= p->second;
+  }
+
+  // decide what command set we're supporting
+  bool use_classic_commands = !classic_mons.empty();
+  // keep a copy to share with the monitor; we clear classic_mons in bump_epoch
+  set copy_classic_mons = classic_mons;
+  
+  cancel_timer();
+  
+  assert(epoch % 2 == 1);  // election
+  bump_epoch(epoch+1);     // is over!
+
+  // decide my supported commands for peons to advertise
+  const bufferlist *cmds_bl = NULL;
+  const MonCommand *cmds;
+  int cmdsize;
+  if (use_classic_commands) {
+    mon->get_classic_monitor_commands(&cmds, &cmdsize);
+    cmds_bl = &mon->get_classic_commands_bl();
+  } else {
+    mon->get_locally_supported_monitor_commands(&cmds, &cmdsize);
+    cmds_bl = &mon->get_supported_commands_bl();
+  }
+  
+  // tell everyone!
+  for (set::iterator p = quorum.begin();
+       p != quorum.end();
+       ++p) {
+    if (*p == mon->rank) continue;
+    MMonElection *m = new MMonElection(MMonElection::OP_VICTORY, epoch, mon->monmap);
+    m->quorum = quorum;
+    m->quorum_features = features;
+    m->sharing_bl = *cmds_bl;
+    mon->messenger->send_message(m, mon->monmap->get_inst(*p));
+  }
+    
+  // tell monitor
+  mon->win_election(epoch, quorum, features, cmds, cmdsize, ©_classic_mons);
+}
+
+
+void Elector::handle_propose(MMonElection *m)
+{
+  dout(5) << "handle_propose from " << m->get_source() << dendl;
+  int from = m->get_source().num();
+
+  assert(m->epoch % 2 == 1); // election
+  uint64_t required_features = mon->get_required_features();
+  if ((required_features ^ m->get_connection()->get_features()) &
+      required_features) {
+    dout(5) << " ignoring propose from mon" << from
+	    << " without required features" << dendl;
+    nak_old_peer(m);
+    return;
+  } else if (m->epoch > epoch) {
+    bump_epoch(m->epoch);
+  } else if (m->epoch < epoch) {
+    // got an "old" propose,
+    if (epoch % 2 == 0 &&    // in a non-election cycle
+	mon->quorum.count(from) == 0) {  // from someone outside the quorum
+      // a mon just started up, call a new election so they can rejoin!
+      dout(5) << " got propose from old epoch, quorum is " << mon->quorum 
+	      << ", " << m->get_source() << " must have just started" << dendl;
+      // we may be active; make sure we reset things in the monitor appropriately.
+      mon->start_election();
+    } else {
+      dout(5) << " ignoring old propose" << dendl;
+      m->put();
+      return;
+    }
+  }
+
+  if (mon->rank < from) {
+    // i would win over them.
+    if (leader_acked >= 0) {        // we already acked someone
+      assert(leader_acked < from);  // and they still win, of course
+      dout(5) << "no, we already acked " << leader_acked << dendl;
+    } else {
+      // wait, i should win!
+      if (!electing_me) {
+	mon->start_election();
+      }
+    }
+  } else {
+    // they would win over me
+    if (leader_acked < 0 ||      // haven't acked anyone yet, or
+	leader_acked > from ||   // they would win over who you did ack, or
+	leader_acked == from) {  // this is the guy we're already deferring to
+      defer(from);
+    } else {
+      // ignore them!
+      dout(5) << "no, we already acked " << leader_acked << dendl;
+    }
+  }
+  
+  m->put();
+}
+ 
+void Elector::handle_ack(MMonElection *m)
+{
+  dout(5) << "handle_ack from " << m->get_source() << dendl;
+  int from = m->get_source().num();
+
+  assert(m->epoch % 2 == 1); // election
+  if (m->epoch > epoch) {
+    dout(5) << "woah, that's a newer epoch, i must have rebooted.  bumping and re-starting!" << dendl;
+    bump_epoch(m->epoch);
+    start();
+    m->put();
+    return;
+  }
+  assert(m->epoch == epoch);
+  uint64_t required_features = mon->get_required_features();
+  if ((required_features ^ m->get_connection()->get_features()) &
+      required_features) {
+    dout(5) << " ignoring ack from mon" << from
+	    << " without required features" << dendl;
+    return;
+  }
+  
+  if (electing_me) {
+    // thanks
+    acked_me[from] = m->get_connection()->get_features();
+    if (!m->sharing_bl.length())
+      classic_mons.insert(from);
+    dout(5) << " so far i have " << acked_me << dendl;
+    
+    // is that _everyone_?
+    if (acked_me.size() == mon->monmap->size()) {
+      // if yes, shortcut to election finish
+      victory();
+    }
+  } else {
+    // ignore, i'm deferring already.
+    assert(leader_acked >= 0);
+  }
+  
+  m->put();
+}
+
+
+void Elector::handle_victory(MMonElection *m)
+{
+  dout(5) << "handle_victory from " << m->get_source() << " quorum_features " << m->quorum_features << dendl;
+  int from = m->get_source().num();
+
+  assert(from < mon->rank);
+  assert(m->epoch % 2 == 0);  
+
+  leader_acked = -1;
+
+  // i should have seen this election if i'm getting the victory.
+  if (m->epoch != epoch + 1) { 
+    dout(5) << "woah, that's a funny epoch, i must have rebooted.  bumping and re-starting!" << dendl;
+    bump_epoch(m->epoch);
+    start();
+    m->put();
+    return;
+  }
+
+  bump_epoch(m->epoch);
+  
+  // they win
+  mon->lose_election(epoch, m->quorum, from, m->quorum_features);
+  
+  // cancel my timer
+  cancel_timer();
+
+  // stash leader's commands
+  if (m->sharing_bl.length()) {
+    MonCommand *new_cmds;
+    int cmdsize;
+    bufferlist::iterator bi = m->sharing_bl.begin();
+    MonCommand::decode_array(&new_cmds, &cmdsize, bi);
+    mon->set_leader_supported_commands(new_cmds, cmdsize);
+  } else { // they are a legacy monitor; use known legacy command set
+    const MonCommand *new_cmds;
+    int cmdsize;
+    mon->get_classic_monitor_commands(&new_cmds, &cmdsize);
+    mon->set_leader_supported_commands(new_cmds, cmdsize);
+  }
+
+  m->put();
+}
+
+void Elector::nak_old_peer(MMonElection *m)
+{
+  uint64_t supported_features = m->get_connection()->get_features();
+
+  if (supported_features & CEPH_FEATURE_OSDMAP_ENC) {
+    uint64_t required_features = mon->get_required_features();
+    dout(10) << "sending nak to peer " << m->get_source()
+	     << " that only supports " << supported_features
+	     << " of the required " << required_features << dendl;
+    
+    MMonElection *reply = new MMonElection(MMonElection::OP_NAK, m->epoch,
+					   mon->monmap);
+    reply->quorum_features = required_features;
+    mon->features.encode(reply->sharing_bl);
+    mon->messenger->send_message(reply, m->get_connection());
+  }
+  m->put();
+}
+
+void Elector::handle_nak(MMonElection *m)
+{
+  dout(1) << "handle_nak from " << m->get_source()
+	  << " quorum_features " << m->quorum_features << dendl;
+
+  CompatSet other;
+  bufferlist::iterator bi = m->sharing_bl.begin();
+  other.decode(bi);
+  CompatSet diff = Monitor::get_supported_features().unsupported(other);
+  
+  derr << "Shutting down because I do not support required monitor features: { "
+       << diff << " }" << dendl;
+  
+  exit(0);
+  // the end!
+}
+
+void Elector::dispatch(Message *m)
+{
+  switch (m->get_type()) {
+    
+  case MSG_MON_ELECTION:
+    {
+      if (!participating) {
+        m->put();
+        return;
+      }
+      if (m->get_source().num() >= mon->monmap->size()) {
+	dout(5) << " ignoring bogus election message with bad mon rank " 
+		<< m->get_source() << dendl;
+	m->put();
+	return;
+      }
+
+      MMonElection *em = static_cast(m);
+
+      // assume an old message encoding would have matched
+      if (em->fsid != mon->monmap->fsid) {
+	dout(0) << " ignoring election msg fsid " 
+		<< em->fsid << " != " << mon->monmap->fsid << dendl;
+	m->put();
+	return;
+      }
+
+      if (!mon->monmap->contains(m->get_source_addr())) {
+	dout(1) << "discarding election message: " << m->get_source_addr()
+		<< " not in my monmap " << *mon->monmap << dendl;
+	m->put();
+	return;
+      }
+
+      MonMap *peermap = new MonMap;
+      peermap->decode(em->monmap_bl);
+      if (peermap->epoch > mon->monmap->epoch) {
+	dout(0) << m->get_source_inst() << " has newer monmap epoch " << peermap->epoch
+		<< " > my epoch " << mon->monmap->epoch 
+		<< ", taking it"
+		<< dendl;
+	mon->monmap->decode(em->monmap_bl);
+        MonitorDBStore::Transaction t;
+        t.put("monmap", mon->monmap->epoch, em->monmap_bl);
+        t.put("monmap", "last_committed", mon->monmap->epoch);
+        mon->store->apply_transaction(t);
+	//mon->monmon()->paxos->stash_latest(mon->monmap->epoch, em->monmap_bl);
+	cancel_timer();
+	mon->bootstrap();
+	m->put();
+	delete peermap;
+	return;
+      }
+      if (peermap->epoch < mon->monmap->epoch) {
+	dout(0) << m->get_source_inst() << " has older monmap epoch " << peermap->epoch
+		<< " < my epoch " << mon->monmap->epoch 
+		<< dendl;
+      } 
+      delete peermap;
+
+      switch (em->op) {
+      case MMonElection::OP_PROPOSE:
+	handle_propose(em);
+	return;
+      }
+
+      if (em->epoch < epoch) {
+	dout(5) << "old epoch, dropping" << dendl;
+	em->put();
+	break;
+      }
+
+      switch (em->op) {
+      case MMonElection::OP_ACK:
+	handle_ack(em);
+	return;
+      case MMonElection::OP_VICTORY:
+	handle_victory(em);
+	return;
+      case MMonElection::OP_NAK:
+	handle_nak(em);
+	return;
+      default:
+	assert(0);
+      }
+    }
+    break;
+    
+  default: 
+    assert(0);
+  }
+}
+
+void Elector::start_participating()
+{
+  if (!participating) {
+    participating = true;
+    call_election();
+  }
+}
diff --git a/ceph/src/mon/Elector.h b/ceph/src/mon/Elector.h
new file mode 100644
index 00000000..007fc116
--- /dev/null
+++ b/ceph/src/mon/Elector.h
@@ -0,0 +1,436 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MON_ELECTOR_H
+#define CEPH_MON_ELECTOR_H
+
+#include 
+using namespace std;
+
+#include "include/types.h"
+#include "msg/Message.h"
+
+#include "include/Context.h"
+
+#include "common/Timer.h"
+
+class Monitor;
+
+/**
+ * This class is responsible for maintaining the local state when electing
+ * a new Leader. We may win or we may lose. If we win, it means we became the
+ * Leader; if we lose, it means we are a Peon.
+ */
+class Elector {
+  /**
+   * @defgroup Elector_h_class Elector
+   * @{
+   */
+ private:
+  /**
+   * The Monitor instance associated with this class.
+   */
+  Monitor *mon;
+
+  /**
+   * Event callback responsible for dealing with an expired election once a
+   * timer runs out and fires up.
+   */
+  Context *expire_event;
+
+  /**
+   * Resets the expire_event timer, by cancelling any existing one and
+   * scheduling a new one.
+   *
+   * @remarks This function assumes as a default firing value the duration of
+   *	      the monitor's lease interval, and adds to it the value specified
+   *	      in @plus
+   *
+   * @post expire_event is set
+   *
+   * @param plus The amount of time to be added to the default firing value.
+   */
+  void reset_timer(double plus=0.0);
+  /**
+   * Cancel the expire_event timer, if it is defined.
+   *
+   * @post expire_event is not set
+   */
+  void cancel_timer();
+
+  /**
+   * Latest epoch we've seen.
+   *
+   * @remarks if its value is odd, we're electing; if it's even, then we're
+   *	      stable.
+   */
+  epoch_t epoch;
+
+  /**
+   * Indicates if we are participating in the quorum.
+   *
+   * @remarks By default, we are created as participating. We may stop
+   *	      participating if the Monitor explicitely calls
+   *	      Elector::stop_participating though. If that happens, it will
+   *	      have to call Elector::start_participating for us to resume
+   *	      participating in the quorum.
+   */
+  bool participating;
+
+  // electing me
+  /**
+   * @defgroup Elector_h_electing_me_vars We are being elected
+   * @{
+   */
+  /**
+   * Indicates if we are the ones being elected.
+   *
+   * We always attempt to be the one being elected if we are the ones starting
+   * the election. If we are not the ones that started it, we will only attempt
+   * to be elected if we think we might have a chance (i.e., the other guy's
+   * rank is lower than ours).
+   */
+  bool     electing_me;
+  /**
+   * Holds the time at which we started the election.
+   */
+  utime_t  start_stamp;
+  /**
+   * Set containing all those that acked our proposal to become the Leader.
+   *
+   * If we are acked by everyone in the MonMap, we will declare
+   * victory.  Also note each peer's feature set.
+   */
+  map acked_me;
+  set classic_mons;
+  /**
+   * @}
+   */
+  /**
+   * @defgroup Elector_h_electing_them_vars We are electing another guy
+   * @{
+   */
+  /**
+   * Indicates who we have acked
+   */
+  int	    leader_acked;
+  /**
+   * Indicates when we have acked him
+   */
+  utime_t   ack_stamp;
+  /**
+   * @}
+   */
+ 
+  /**
+   * Update our epoch.
+   *
+   * If we come across a higher epoch, we simply update ours, also making
+   * sure we are no longer being elected (even though we could have been,
+   * we no longer are since we no longer are on that old epoch).
+   *
+   * @pre Our epoch is lower than @p e
+   * @post Our epoch equals @p e
+   *
+   * @param e Epoch to which we will update our epoch
+   */
+  void bump_epoch(epoch_t e);
+
+  /**
+   * @defgroup Elector_h_callbacks Callbacks
+   * @{
+   */
+  /**
+   * This class is used as the callback when the expire_event timer fires up.
+   *
+   * If the expire_event is fired, then it means that we had an election going,
+   * either started by us or by some other participant, but it took too long,
+   * thus expiring.
+   *
+   * When the election expires, we will check if we were the ones who won, and
+   * if so we will declare victory. If that is not the case, then we assume
+   * that the one we defered to didn't declare victory quickly enough (in fact,
+   * as far as we know, we may even be dead); so, just propose ourselves as the
+   * Leader.
+   */
+  class C_ElectionExpire : public Context {
+    Elector *elector;
+  public:
+    C_ElectionExpire(Elector *e) : elector(e) { }
+    void finish(int r) {
+      elector->expire();
+    }
+  };
+  /**
+   * @}
+   */
+
+  /**
+   * Start new elections by proposing ourselves as the new Leader.
+   *
+   * Basically, send propose messages to all the monitors in the MonMap and
+   * then reset the expire_event timer so we can limit the amount of time we 
+   * will be going at it.
+   *
+   * @pre   participating is true
+   * @post  epoch is an odd value
+   * @post  electing_me is true
+   * @post  we sent propose messages to all the monitors in the MonMap
+   * @post  we reset the expire_event timer
+   */
+  void start();
+  /**
+   * Defer the current election to some other monitor.
+   *
+   * This means that we will ack some other monitor and drop out from the run
+   * to become the Leader. We will only defer an election if the monitor we
+   * are deferring to outranks us.
+   *
+   * @pre   @p who outranks us (i.e., who < our rank)
+   * @pre   @p who outranks any other monitor we have deferred to in the past
+   * @post  electing_me is false
+   * @post  leader_acked equals @p who
+   * @post  we sent an ack message to @p who
+   * @post  we reset the expire_event timer
+   *
+   * @param who Some other monitor's numeric identifier. 
+   */
+  void defer(int who);
+  /**
+   * The election has taken too long and has expired.
+   *
+   * This will happen when no one declared victory or started a new election
+   * during the time span allowed by the expire_event timer.
+   *
+   * When the election expires, we will check if we were the ones who won, and
+   * if so we will declare victory. If that is not the case, then we assume
+   * that the one we defered to didn't declare victory quickly enough (in fact,
+   * as far as we know, we may even be dead); so, just propose ourselves as the
+   * Leader.
+   */
+  void expire();
+  /**
+   * Declare Victory.
+   * 
+   * We won. Or at least we believe we won, but for all intentions and purposes
+   * that does not matter. What matters is that we Won.
+   *
+   * That said, we must now bump our epoch to reflect that the election is over
+   * and then we must let everybody in the quorum know we are their brand new
+   * Leader. And we will also cancel our expire_event timer.
+   *
+   * Actually, the quorum will be now defined as the group of monitors that
+   * acked us during the election process.
+   *
+   * @pre   Election is on-going
+   * @pre   electing_me is true
+   * @post  electing_me is false
+   * @post  epoch is bumped up into an even value
+   * @post  Election is not on-going
+   * @post  We have a quorum, composed of the monitors that acked us
+   * @post  We sent a message of type OP_VICTORY to each quorum member.
+   */
+  void victory();
+
+  /**
+   * Handle a message from some other node proposing himself to become him
+   * the Leader.
+   *
+   * If the message appears to be old (i.e., its epoch is lower than our epoch),
+   * then we may take one of two actions:
+   *
+   *  @li Ignore it because it's nothing more than an old proposal
+   *  @li Start new elections if we verify that it was sent by a monitor from
+   *	  outside the quorum; given its old state, it's fair to assume he just
+   *	  started, so we should start new elections so he may rejoin
+   *
+   * If we did not ignore the received message, then we know that this message
+   * was sent by some other node proposing himself to become the Leader. So, we
+   * will take one of the following actions:
+   *
+   *  @li Ignore him because we already acked another node with higher rank
+   *  @li Ignore him and start a new election because we outrank him
+   *  @li Defer to him because he outranks us and the node we previously
+   *	  acked, if any
+   *
+   *
+   * @invariant The received message is an operation of type OP_PROPOSE
+   *
+   * @param m A message sent by another participant in the quorum.
+   */
+  void handle_propose(class MMonElection *m);
+  /**
+   * Handle a message from some other participant Acking us as the Leader.
+   *
+   * When we receive such a message, one of three thing may be happening:
+   *  @li We received a message with a newer epoch, which means we must have
+   *	  somehow lost track of what was going on (maybe we rebooted), thus we
+   *	  will start a new election
+   *  @li We consider ourselves in the run for the Leader (i.e., @p electing_me 
+   *	  is true), and we are actually being Acked by someone; thus simply add
+   *	  the one acking us to the @p acked_me set. If we do now have acks from
+   *	  all the participants, then we can declare victory
+   *  @li We already deferred the election to somebody else, so we will just
+   *	  ignore this message
+   *
+   * @pre   Election is on-going
+   * @post  Election is on-going if we deferred to somebody else
+   * @post  Election is on-going if we are still waiting for further Acks
+   * @post  Election is not on-going if we are victorious
+   * @post  Election is not on-going if we must start a new one
+   *
+   * @param m A message with an operation type of OP_ACK
+   */
+  void handle_ack(class MMonElection *m);
+  /**
+   * Handle a message from some other participant declaring Victory.
+   *
+   * We just got a message from someone declaring themselves Victorious, thus
+   * the new Leader.
+   *
+   * However, if the message's epoch happens to be different from our epoch+1,
+   * then it means we lost track of something and we must start a new election.
+   *
+   * If that is not the case, then we will simply update our epoch to the one
+   * in the message, cancel our @p expire_event timer and inform our Monitor
+   * that we lost the election and provide it with the new quorum.
+   *
+   * @pre   Election in on-going
+   * @post  Election is not on-going
+   * @post  Updated @p epoch
+   * @post  We have a new quorum if we lost the election
+   *
+   * @param m A message with an operation type of OP_VICTORY
+   */
+  void handle_victory(class MMonElection *m);
+  /**
+   * Send a nak to a peer who's out of date, containing information about why.
+   *
+   * If we get a message from a peer who can't support the required quorum
+   * features, we have to ignore them. This function will at least send
+   * them a message about *why* they're being ignored -- if they're new
+   * enough to support such a message.
+   *
+   * @param m A message from a monitor not supporting required features. We
+   * take ownership of the reference.
+   */
+  void nak_old_peer(class MMonElection *m);
+  /**
+   * Handle a message from some other participant declaring
+   * we cannot join the quorum.
+   *
+   * Apparently the quorum requires some feature that we do not implement. Shut
+   * down gracefully.
+   *
+   * @pre Election is on-going.
+   * @post We've shut down.
+   *
+   * @param m A message with an operation type of OP_NAK
+   */
+  void handle_nak(class MMonElection *m);
+  
+ public:
+  /**
+   * Create an Elector class
+   *
+   * @param m A Monitor instance
+   */
+  Elector(Monitor *m) : mon(m),
+			expire_event(0),
+			epoch(0),
+			participating(true),
+			electing_me(false),
+			leader_acked(-1) { }
+
+  /**
+   * Initiate the Elector class.
+   *
+   * Basically, we will simply read whatever epoch value we have in our stable
+   * storage, or consider it to be 1 if none is read.
+   *
+   * @post @p epoch is set to 1 or higher.
+   */
+  void init();
+  /**
+   * Inform this class it is supposed to shutdown.
+   *
+   * We will simply cancel the @p expire_event if any exists.
+   *
+   * @post @p expire_event is cancelled 
+   */
+  void shutdown();
+
+  /**
+   * Obtain our epoch
+   *
+   * @returns Our current epoch number
+   */
+  epoch_t get_epoch() { return epoch; }
+
+  /**
+   * advance_epoch
+   *
+   * increase election epoch by 1
+   */
+  void advance_epoch() {
+    bump_epoch(epoch + 1);
+  }
+
+  /**
+   * Handle received messages.
+   *
+   * We will ignore all messages that are not of type @p MSG_MON_ELECTION
+   * (i.e., messages whose interface is not of type @p MMonElection). All of
+   * those that are will then be dispatched to their operation-specific
+   * functions.
+   *
+   * @param m A received message
+   */
+  void dispatch(Message *m);
+
+  /**
+   * Call an election.
+   *
+   * This function simply calls Elector::start.
+   */
+  void call_election() {
+    start();
+  }
+
+  /**
+   * Stop participating in subsequent Elections.
+   *
+   * @post @p participating is false
+   */
+  void stop_participating() { participating = false; }
+  /**
+   * Start participating in Elections.
+   *
+   * If we are already participating (i.e., @p participating is true), then
+   * calling this function is moot.
+   *
+   * However, if we are not participating (i.e., @p participating is false),
+   * then we will start participating by setting @p participating to true and
+   * we will call for an Election.
+   *
+   * @post  @p participating is true
+   */
+  void start_participating();
+  /**
+   * @}
+   */
+};
+
+#endif
diff --git a/ceph/src/mon/HealthMonitor.cc b/ceph/src/mon/HealthMonitor.cc
new file mode 100644
index 00000000..7cba39bf
--- /dev/null
+++ b/ceph/src/mon/HealthMonitor.cc
@@ -0,0 +1,103 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+
+#include 
+// Because intusive_ptr clobbers our assert...
+#include "include/assert.h"
+
+#include "mon/Monitor.h"
+#include "mon/QuorumService.h"
+#include "mon/HealthService.h"
+#include "mon/HealthMonitor.h"
+#include "mon/DataHealthService.h"
+
+#include "messages/MMonHealth.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, this)
+static ostream& _prefix(std::ostream *_dout, const Monitor *mon,
+                        const HealthMonitor *hmon) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name() << ")." << hmon->get_name()
+                << "(" << hmon->get_epoch() << ") ";
+}
+
+void HealthMonitor::init()
+{
+  dout(10) << __func__ << dendl;
+  assert(services.empty());
+  services[HealthService::SERVICE_HEALTH_DATA] = new DataHealthService(mon);
+
+  for (map::iterator it = services.begin();
+       it != services.end();
+       ++it) {
+    it->second->init();
+  }
+}
+
+bool HealthMonitor::service_dispatch(Message *m)
+{
+  assert(m->get_type() == MSG_MON_HEALTH);
+  MMonHealth *hm = (MMonHealth*)m;
+  int service_type = hm->get_service_type();
+  if (services.count(service_type) == 0) {
+    dout(1) << __func__ << " service type " << service_type
+            << " not registered -- drop message!" << dendl;
+    m->put();
+    return false;
+  }
+  return services[service_type]->service_dispatch(hm);
+}
+
+void HealthMonitor::service_shutdown()
+{
+  dout(0) << "HealthMonitor::service_shutdown "
+          << services.size() << " services" << dendl;
+  for (map::iterator it = services.begin();
+      it != services.end();
+       ++it) {
+    it->second->shutdown();
+    delete it->second;
+  }
+  services.clear();
+}
+
+void HealthMonitor::get_health(Formatter *f,
+			       list >& summary,
+			       list > *detail)
+{
+  if (f) {
+    f->open_object_section("health");
+    f->open_array_section("health_services");
+  }
+
+  for (map::iterator it = services.begin();
+       it != services.end();
+       ++it) {
+    it->second->get_health(f, summary, detail);
+  }
+
+  if (f) {
+    f->close_section(); // health_services
+    f->close_section(); // health
+  }
+}
+
diff --git a/ceph/src/mon/HealthMonitor.h b/ceph/src/mon/HealthMonitor.h
new file mode 100644
index 00000000..3d842615
--- /dev/null
+++ b/ceph/src/mon/HealthMonitor.h
@@ -0,0 +1,82 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_HEALTH_MONITOR_H
+#define CEPH_HEALTH_MONITOR_H
+
+#include "mon/Monitor.h"
+#include "mon/QuorumService.h"
+#include "mon/HealthService.h"
+
+#include "messages/MMonHealth.h"
+
+#include "common/config.h"
+#include "common/Formatter.h"
+
+class HealthMonitor : public QuorumService
+{
+  map services;
+
+protected:
+  virtual void service_shutdown();
+
+public:
+  HealthMonitor(Monitor *m) : QuorumService(m) { }
+  virtual ~HealthMonitor() {
+    assert(services.empty());
+  }
+
+
+  /**
+   * @defgroup HealthMonitor_Inherited_h Inherited abstract methods
+   * @{
+   */
+  virtual void init();
+  virtual void get_health(Formatter *f,
+		     list >& summary,
+		     list > *detail);
+  virtual bool service_dispatch(Message *m);
+
+  virtual void start_epoch() {
+    for (map::iterator it = services.begin();
+         it != services.end(); ++it) {
+      it->second->start(get_epoch());
+    }
+  }
+
+  virtual void finish_epoch() {
+    generic_dout(20) << "HealthMonitor::finish_epoch()" << dendl;
+    for (map::iterator it = services.begin();
+         it != services.end(); ++it) {
+      assert(it->second != NULL);
+      it->second->finish();
+    }
+  }
+
+  virtual void cleanup() { }
+  virtual void service_tick() { }
+
+  virtual int get_type() {
+    return QuorumService::SERVICE_HEALTH;
+  }
+
+  virtual string get_name() const {
+    return "health";
+  }
+
+  /**
+   * @} // HealthMonitor_Inherited_h
+   */
+};
+
+#endif // CEPH_HEALTH_MONITOR_H
diff --git a/ceph/src/mon/HealthService.h b/ceph/src/mon/HealthService.h
new file mode 100644
index 00000000..2a46f882
--- /dev/null
+++ b/ceph/src/mon/HealthService.h
@@ -0,0 +1,47 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_MON_HEALTH_SERVICE_H
+#define CEPH_MON_HEALTH_SERVICE_H
+
+#include "mon/Monitor.h"
+#include "mon/QuorumService.h"
+
+#include "messages/MMonHealth.h"
+
+#include "common/config.h"
+
+struct HealthService : public QuorumService
+{
+  enum {
+    SERVICE_HEALTH_DATA              = 0x01
+  };
+
+  HealthService(Monitor *m) : QuorumService(m) { }
+  virtual ~HealthService() { }
+
+  virtual bool service_dispatch(Message *m) {
+    return service_dispatch(static_cast(m));
+  }
+
+  virtual bool service_dispatch(MMonHealth *m) = 0;
+
+public:
+  virtual void get_health(Formatter *f,
+			  list >& summary,
+			  list > *detail) = 0;
+  virtual int get_type() = 0;
+  virtual string get_name() const = 0;
+};
+
+#endif // CEPH_MON_HEALTH_SERVICE_H
diff --git a/ceph/src/mon/LogMonitor.cc b/ceph/src/mon/LogMonitor.cc
new file mode 100644
index 00000000..cc985544
--- /dev/null
+++ b/ceph/src/mon/LogMonitor.cc
@@ -0,0 +1,550 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+
+#include "LogMonitor.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+
+#include "messages/MMonCommand.h"
+#include "messages/MLog.h"
+#include "messages/MLogAck.h"
+
+#include "common/Timer.h"
+
+#include "osd/osd_types.h"
+#include "common/errno.h"
+#include "common/config.h"
+#include "include/assert.h"
+#include "include/str_list.h"
+#include "include/compat.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, get_last_committed())
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, version_t v) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").log v" << v << " ";
+}
+
+ostream& operator<<(ostream& out, LogMonitor& pm)
+{
+  /*
+  std::stringstream ss;
+  for (ceph::unordered_map::iterator p = pm.pg_map.num_pg_by_state.begin();
+       p != pm.pg_map.num_pg_by_state.end();
+       ++p) {
+    if (p != pm.pg_map.num_pg_by_state.begin())
+      ss << ", ";
+    ss << p->second << " " << pg_state_string(p->first);
+  }
+  string states = ss.str();
+  return out << "v" << pm.pg_map.version << ": "
+	     << pm.pg_map.pg_stat.size() << " pgs: "
+	     << states << "; "
+	     << kb_t(pm.pg_map.total_pg_kb()) << " data, " 
+	     << kb_t(pm.pg_map.total_used_kb()) << " used, "
+	     << kb_t(pm.pg_map.total_avail_kb()) << " / "
+	     << kb_t(pm.pg_map.total_kb()) << " free";
+  */
+  return out << "log";
+}
+
+/*
+ Tick function to update the map based on performance every N seconds
+*/
+
+void LogMonitor::tick() 
+{
+  if (!is_active()) return;
+
+  dout(10) << *this << dendl;
+
+  if (!mon->is_leader()) return; 
+
+}
+
+void LogMonitor::create_initial()
+{
+  dout(10) << "create_initial -- creating initial map" << dendl;
+  LogEntry e;
+  memset(&e.who, 0, sizeof(e.who));
+  e.stamp = ceph_clock_now(g_ceph_context);
+  e.type = CLOG_INFO;
+  std::stringstream ss;
+  ss << "mkfs " << mon->monmap->get_fsid();
+  e.msg = ss.str();
+  e.seq = 0;
+  pending_log.insert(pair(e.stamp, e));
+}
+
+void LogMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  dout(10) << __func__ << dendl;
+  version_t version = get_last_committed();
+  dout(10) << __func__ << " version " << version
+           << " summary v " << summary.version << dendl;
+  if (version == summary.version)
+    return;
+  assert(version >= summary.version);
+
+  bufferlist blog;
+
+  version_t latest_full = get_version_latest_full();
+  dout(10) << __func__ << " latest full " << latest_full << dendl;
+  if ((latest_full > 0) && (latest_full > summary.version)) {
+    bufferlist latest_bl;
+    get_version_full(latest_full, latest_bl);
+    assert(latest_bl.length() != 0);
+    dout(7) << __func__ << " loading summary e" << latest_full << dendl;
+    bufferlist::iterator p = latest_bl.begin();
+    ::decode(summary, p);
+    dout(7) << __func__ << " loaded summary e" << summary.version << dendl;
+  }
+
+  // walk through incrementals
+  while (version > summary.version) {
+    bufferlist bl;
+    int err = get_version(summary.version+1, bl);
+    assert(err == 0);
+    assert(bl.length());
+
+    bufferlist::iterator p = bl.begin();
+    __u8 v;
+    ::decode(v, p);
+    while (!p.end()) {
+      LogEntry le;
+      le.decode(p);
+      dout(7) << "update_from_paxos applying incremental log " << summary.version+1 <<  " " << le << dendl;
+
+      if (g_conf->mon_cluster_log_to_syslog) {
+	le.log_to_syslog(g_conf->mon_cluster_log_to_syslog_level,
+			 g_conf->mon_cluster_log_to_syslog_facility);
+      }
+      if (g_conf->mon_cluster_log_file.length()) {
+	int min = string_to_syslog_level(g_conf->mon_cluster_log_file_level);
+	int l = clog_type_to_syslog_level(le.type);
+	if (l <= min) {
+	  stringstream ss;
+	  ss << le << "\n";
+	  blog.append(ss.str());
+	}
+      }
+
+      summary.add(le);
+    }
+
+    summary.version++;
+  }
+
+
+  if (blog.length()) {
+    int fd = ::open(g_conf->mon_cluster_log_file.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0600);
+    if (fd < 0) {
+      int err = -errno;
+      dout(1) << "unable to write to " << g_conf->mon_cluster_log_file << ": " << cpp_strerror(err) << dendl;
+    } else {
+      int err = blog.write_fd(fd);
+      if (err < 0) {
+	dout(1) << "error writing to " << g_conf->mon_cluster_log_file
+		<< ": " << cpp_strerror(err) << dendl;
+      }
+      VOID_TEMP_FAILURE_RETRY(::close(fd));
+    }
+  }
+
+  check_subs();
+}
+
+void LogMonitor::store_do_append(MonitorDBStore::Transaction *t,
+    const string& key, bufferlist& bl)
+{
+  bufferlist existing_bl;
+  int err = get_value(key, existing_bl);
+  assert(err == 0);
+
+  existing_bl.append(bl);
+  put_value(t, key, existing_bl);
+}
+
+void LogMonitor::create_pending()
+{
+  pending_log.clear();
+  pending_summary = summary;
+  dout(10) << "create_pending v " << (get_last_committed() + 1) << dendl;
+}
+
+void LogMonitor::encode_pending(MonitorDBStore::Transaction *t)
+{
+  version_t version = get_last_committed() + 1;
+  bufferlist bl;
+  dout(10) << __func__ << " v" << version << dendl;
+  __u8 v = 1;
+  ::encode(v, bl);
+  multimap::iterator p;
+  for (p = pending_log.begin(); p != pending_log.end(); ++p)
+    p->second.encode(bl);
+
+  put_version(t, version, bl);
+  put_last_committed(t, version);
+}
+
+void LogMonitor::encode_full(MonitorDBStore::Transaction *t)
+{
+  dout(10) << __func__ << " log v " << summary.version << dendl;
+  assert(get_last_committed() == summary.version);
+
+  bufferlist summary_bl;
+  ::encode(summary, summary_bl);
+
+  put_version_full(t, summary.version, summary_bl);
+  put_version_latest_full(t, summary.version);
+}
+
+version_t LogMonitor::get_trim_to()
+{
+  unsigned max = g_conf->mon_max_log_epochs;
+  version_t version = get_last_committed();
+  if (mon->is_leader() && version > max)
+    return version - max;
+  return 0;
+}
+
+bool LogMonitor::preprocess_query(PaxosServiceMessage *m)
+{
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    return preprocess_command(static_cast(m));
+
+  case MSG_LOG:
+    return preprocess_log((MLog*)m);
+
+  default:
+    assert(0);
+    m->put();
+    return true;
+  }
+}
+
+bool LogMonitor::prepare_update(PaxosServiceMessage *m)
+{
+  dout(10) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    return prepare_command(static_cast(m));
+  case MSG_LOG:
+    return prepare_log((MLog*)m);
+  default:
+    assert(0);
+    m->put();
+    return false;
+  }
+}
+
+bool LogMonitor::preprocess_log(MLog *m)
+{
+  dout(10) << "preprocess_log " << *m << " from " << m->get_orig_source() << dendl;
+  int num_new = 0;
+
+  MonSession *session = m->get_session();
+  if (!session)
+    goto done;
+  if (!session->is_capable("log", MON_CAP_W)) {
+    dout(0) << "preprocess_log got MLog from entity with insufficient privileges "
+	    << session->caps << dendl;
+    goto done;
+  }
+  
+  for (deque::iterator p = m->entries.begin();
+       p != m->entries.end();
+       ++p) {
+    if (!pending_summary.contains(p->key()))
+      num_new++;
+  }
+  if (!num_new) {
+    dout(10) << "  nothing new" << dendl;
+    goto done;
+  }
+
+  return false;
+
+ done:
+  m->put();
+  return true;
+}
+
+bool LogMonitor::prepare_log(MLog *m) 
+{
+  dout(10) << "prepare_log " << *m << " from " << m->get_orig_source() << dendl;
+
+  if (m->fsid != mon->monmap->fsid) {
+    dout(0) << "handle_log on fsid " << m->fsid << " != " << mon->monmap->fsid 
+	    << dendl;
+    m->put();
+    return false;
+  }
+
+  for (deque::iterator p = m->entries.begin();
+       p != m->entries.end();
+       ++p) {
+    dout(10) << " logging " << *p << dendl;
+    if (!pending_summary.contains(p->key())) {
+      pending_summary.add(*p);
+      pending_log.insert(pair(p->stamp, *p));
+    }
+  }
+  wait_for_finished_proposal(new C_Log(this, m));
+  return true;
+}
+
+void LogMonitor::_updated_log(MLog *m)
+{
+  dout(7) << "_updated_log for " << m->get_orig_source_inst() << dendl;
+  mon->send_reply(m, new MLogAck(m->fsid, m->entries.rbegin()->seq));
+
+  m->put();
+}
+
+bool LogMonitor::should_propose(double& delay)
+{
+  // commit now if we have a lot of pending events
+  if (g_conf->mon_max_log_entries_per_event > 0 &&
+      pending_log.size() >= (unsigned)g_conf->mon_max_log_entries_per_event)
+    return true;
+
+  // otherwise fall back to generic policy
+  return PaxosService::should_propose(delay);
+}
+
+
+bool LogMonitor::preprocess_command(MMonCommand *m)
+{
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss;
+
+  if (r != -1) {
+    string rs;
+    getline(ss, rs);
+    mon->reply_command(m, r, rs, rdata, get_last_committed());
+    return true;
+  } else
+    return false;
+}
+
+
+bool LogMonitor::prepare_command(MMonCommand *m)
+{
+  stringstream ss;
+  string rs;
+  int err = -EINVAL;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  if (prefix == "log") {
+    vector logtext;
+    cmd_getval(g_ceph_context, cmdmap, "logtext", logtext);
+    LogEntry le;
+    le.who = m->get_orig_source_inst();
+    le.stamp = m->get_recv_stamp();
+    le.seq = 0;
+    le.type = CLOG_INFO;
+    le.msg = str_join(logtext, " ");
+    pending_summary.add(le);
+    pending_log.insert(pair(le.stamp, le));
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, string(),
+					      get_last_committed() + 1));
+    return true;
+  }
+
+  getline(ss, rs);
+  mon->reply_command(m, err, rs, get_last_committed());
+  return false;
+}
+
+
+int LogMonitor::sub_name_to_id(const string& n)
+{
+  if (n == "log-debug")
+    return CLOG_DEBUG;
+  if (n == "log-info")
+    return CLOG_INFO;
+  if (n == "log-sec")
+    return CLOG_SEC;
+  if (n == "log-warn")
+    return CLOG_WARN;
+  if (n == "log-error")
+    return CLOG_ERROR;
+  return -1;
+}
+
+void LogMonitor::check_subs()
+{
+  dout(10) << __func__ << dendl;
+  for (map*>::iterator i = mon->session_map.subs.begin();
+       i != mon->session_map.subs.end();
+       ++i) {
+    for (xlist::iterator j = i->second->begin(); !j.end(); ++j) {
+      if (sub_name_to_id((*j)->type) >= 0)
+	check_sub(*j);
+    }
+  }
+}
+
+void LogMonitor::check_sub(Subscription *s)
+{
+  dout(10) << __func__ << " client wants " << s->type << " ver " << s->next << dendl;
+
+  int sub_level = sub_name_to_id(s->type);
+  assert(sub_level >= 0);
+
+  version_t summary_version = summary.version;
+  if (s->next > summary_version) {
+    dout(10) << __func__ << " client " << s->session->inst 
+	    << " requested version (" << s->next << ") is greater than ours (" 
+	    << summary_version << "), which means we already sent him" 
+	    << " everything we have." << dendl;
+    return;
+  } 
+ 
+  MLog *mlog = new MLog(mon->monmap->fsid);
+
+  if (s->next == 0) { 
+    /* First timer, heh? */
+    bool ret = _create_sub_summary(mlog, sub_level);
+    if (!ret) {
+      dout(1) << __func__ << " ret = " << ret << dendl;
+      mlog->put();
+      return;
+    }
+  } else {
+    /* let us send you an incremental log... */
+    _create_sub_incremental(mlog, sub_level, s->next);
+  }
+
+  dout(1) << __func__ << " sending message to " << s->session->inst 
+	  << " with " << mlog->entries.size() << " entries"
+	  << " (version " << mlog->version << ")" << dendl;
+  
+  mon->messenger->send_message(mlog, s->session->inst);
+  if (s->onetime)
+    mon->session_map.remove_sub(s);
+  else
+    s->next = summary_version+1;
+}
+
+/**
+ * Create a log message containing only the last message in the summary.
+ *
+ * @param mlog	Log message we'll send to the client.
+ * @param level Maximum log level the client is interested in.
+ * @return	'true' if we consider we successfully populated @mlog;
+ *		'false' otherwise.
+ */
+bool LogMonitor::_create_sub_summary(MLog *mlog, int level)
+{
+  dout(10) << __func__ << dendl;
+
+  assert(mlog != NULL);
+
+  if (!summary.tail.size())
+    return false;
+
+  list::reverse_iterator it = summary.tail.rbegin();
+  for (; it != summary.tail.rend(); ++it) {
+    LogEntry e = *it;
+    if (e.type < level)
+      continue;
+
+    mlog->entries.push_back(e);
+    mlog->version = summary.version;
+    break;
+  }
+
+  return true;
+}
+
+/**
+ * Create an incremental log message from version @sv to @summary.version
+ *
+ * @param mlog	Log message we'll send to the client with the messages received
+ *		since version @sv, inclusive.
+ * @param level	The max log level of the messages the client is interested in.
+ * @param sv	The version the client is looking for.
+ */
+void LogMonitor::_create_sub_incremental(MLog *mlog, int level, version_t sv)
+{
+  dout(10) << __func__ << " level " << level << " ver " << sv 
+	  << " cur summary ver " << summary.version << dendl; 
+
+  if (sv < get_first_committed()) {
+    dout(10) << __func__ << " skipped from " << sv
+	     << " to first_committed " << get_first_committed() << dendl;
+    LogEntry le;
+    le.stamp = ceph_clock_now(NULL);
+    le.type = CLOG_WARN;
+    ostringstream ss;
+    ss << "skipped log messages from " << sv << " to " << get_first_committed();
+    le.msg = ss.str();
+    mlog->entries.push_back(le);
+    sv = get_first_committed();
+  }
+
+  version_t summary_ver = summary.version;
+  while (sv <= summary_ver) {
+    bufferlist bl;
+    int err = get_version(sv, bl);
+    assert(err == 0);
+    assert(bl.length());
+    bufferlist::iterator p = bl.begin();
+    __u8 v;
+    ::decode(v,p);
+    while (!p.end()) {
+      LogEntry le;
+      le.decode(p);
+
+      if (le.type < level) {
+	dout(20) << __func__ << " requested " << level 
+		 << " entry " << le.type << dendl;
+	continue;
+      }
+
+      mlog->entries.push_back(le);
+    }
+    mlog->version = sv++;
+  }
+
+  dout(10) << __func__ << " incremental message ready (" 
+	   << mlog->entries.size() << " entries)" << dendl;
+}
+
diff --git a/ceph/src/mon/LogMonitor.h b/ceph/src/mon/LogMonitor.h
new file mode 100644
index 00000000..439ea4d9
--- /dev/null
+++ b/ceph/src/mon/LogMonitor.h
@@ -0,0 +1,99 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_LOGMONITOR_H
+#define CEPH_LOGMONITOR_H
+
+#include 
+#include 
+using namespace std;
+
+#include "include/types.h"
+#include "msg/Messenger.h"
+#include "PaxosService.h"
+
+#include "common/LogEntry.h"
+#include "messages/MLog.h"
+
+class MMonCommand;
+
+class LogMonitor : public PaxosService {
+private:
+  multimap pending_log;
+  LogSummary pending_summary, summary;
+
+  void create_initial();
+  void update_from_paxos(bool *need_bootstrap);
+  void create_pending();  // prepare a new pending
+  // propose pending update to peers
+  void encode_pending(MonitorDBStore::Transaction *t);
+  virtual void encode_full(MonitorDBStore::Transaction *t);
+  version_t get_trim_to();
+  bool preprocess_query(PaxosServiceMessage *m);  // true if processed.
+  bool prepare_update(PaxosServiceMessage *m);
+
+  bool preprocess_log(MLog *m);
+  bool prepare_log(MLog *m);
+  void _updated_log(MLog *m);
+
+  bool should_propose(double& delay);
+
+  bool should_stash_full() {
+    // commit a LogSummary on every commit
+    return true;
+  }
+
+  struct C_Log : public Context {
+    LogMonitor *logmon;
+    MLog *ack;
+    C_Log(LogMonitor *p, MLog *a) : logmon(p), ack(a) {}
+    void finish(int r) {
+      if (r == -ECANCELED) {
+	if (ack)
+	  ack->put();
+	return;
+      }
+      logmon->_updated_log(ack);
+    }    
+  };
+
+  bool preprocess_command(MMonCommand *m);
+  bool prepare_command(MMonCommand *m);
+
+  bool _create_sub_summary(MLog *mlog, int level);
+  void _create_sub_incremental(MLog *mlog, int level, version_t sv);
+
+  void store_do_append(MonitorDBStore::Transaction *t,
+		       const string& key, bufferlist& bl);
+
+ public:
+  LogMonitor(Monitor *mn, Paxos *p, const string& service_name) 
+    : PaxosService(mn, p, service_name) { }
+  
+  void tick();  // check state, take actions
+
+  void check_subs();
+  void check_sub(Subscription *s);
+
+  /**
+   * translate log sub name ('log-info') to integer id
+   *
+   * @param n name
+   * @return id, or -1 if unrecognized
+   */
+  int sub_name_to_id(const string& n);
+
+};
+
+#endif
diff --git a/ceph/src/mon/MDSMonitor.cc b/ceph/src/mon/MDSMonitor.cc
new file mode 100644
index 00000000..12755b7c
--- /dev/null
+++ b/ceph/src/mon/MDSMonitor.cc
@@ -0,0 +1,1437 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+
+#include "MDSMonitor.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+#include "OSDMonitor.h"
+
+#include "common/strtol.h"
+#include "common/ceph_argparse.h"
+
+#include "messages/MMDSMap.h"
+#include "messages/MMDSBeacon.h"
+#include "messages/MMDSLoadTargets.h"
+#include "messages/MMonCommand.h"
+
+#include "messages/MGenericMessage.h"
+
+#include "common/perf_counters.h"
+#include "common/Timer.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#include "MonitorDBStore.h"
+#include "common/cmdparse.h"
+#include "include/str_list.h"
+
+#include "mds/mdstypes.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, mdsmap)
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, MDSMap& mdsmap) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").mds e" << mdsmap.get_epoch() << " ";
+}
+
+
+
+// my methods
+
+void MDSMonitor::print_map(MDSMap &m, int dbl)
+{
+  dout(dbl) << "print_map\n";
+  m.print(*_dout);
+  *_dout << dendl;
+}
+
+void MDSMonitor::create_new_fs(MDSMap &m, int metadata_pool, int data_pool)
+{
+  m.max_mds = g_conf->max_mds;
+  m.created = ceph_clock_now(g_ceph_context);
+  m.data_pools.insert(data_pool);
+  m.metadata_pool = metadata_pool;
+  m.cas_pool = -1;
+  m.compat = get_mdsmap_compat_set_default();
+
+  m.session_timeout = g_conf->mds_session_timeout;
+  m.session_autoclose = g_conf->mds_session_autoclose;
+  m.max_file_size = g_conf->mds_max_file_size;
+
+  print_map(m);
+}
+
+
+// service methods
+void MDSMonitor::create_initial()
+{
+  dout(10) << "create_initial" << dendl;
+  create_new_fs(pending_mdsmap, MDS_METADATA_POOL, MDS_DATA_POOL);
+}
+
+
+void MDSMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version_t version = get_last_committed();
+  if (version == mdsmap.epoch)
+    return;
+  assert(version >= mdsmap.epoch);
+
+  dout(10) << __func__ << " version " << version
+	   << ", my e " << mdsmap.epoch << dendl;
+
+  // read and decode
+  mdsmap_bl.clear();
+  int err = get_version(version, mdsmap_bl);
+  assert(err == 0);
+
+  assert(mdsmap_bl.length() > 0);
+  dout(10) << __func__ << " got " << version << dendl;
+  mdsmap.decode(mdsmap_bl);
+
+  // new map
+  dout(4) << "new map" << dendl;
+  print_map(mdsmap, 0);
+
+  check_subs();
+  update_logger();
+}
+
+void MDSMonitor::create_pending()
+{
+  pending_mdsmap = mdsmap;
+  pending_mdsmap.epoch++;
+  dout(10) << "create_pending e" << pending_mdsmap.epoch << dendl;
+}
+
+void MDSMonitor::encode_pending(MonitorDBStore::Transaction *t)
+{
+  dout(10) << "encode_pending e" << pending_mdsmap.epoch << dendl;
+
+  pending_mdsmap.modified = ceph_clock_now(g_ceph_context);
+
+  // print map iff 'debug mon = 30' or higher
+  print_map(pending_mdsmap, 30);
+
+  // apply to paxos
+  assert(get_last_committed() + 1 == pending_mdsmap.epoch);
+  bufferlist mdsmap_bl;
+  pending_mdsmap.encode(mdsmap_bl, mon->get_quorum_features());
+
+  /* put everything in the transaction */
+  put_version(t, pending_mdsmap.epoch, mdsmap_bl);
+  put_last_committed(t, pending_mdsmap.epoch);
+}
+
+version_t MDSMonitor::get_trim_to()
+{
+  version_t floor = 0;
+  if (g_conf->mon_mds_force_trim_to > 0 &&
+      g_conf->mon_mds_force_trim_to < (int)get_last_committed()) {
+    floor = g_conf->mon_mds_force_trim_to;
+    dout(10) << __func__ << " explicit mon_mds_force_trim_to = "
+             << floor << dendl;
+  }
+
+  unsigned max = g_conf->mon_max_mdsmap_epochs;
+  version_t last = get_last_committed();
+
+  if (last - get_first_committed() > max && floor < last - max)
+    return last - max;
+  return floor;
+}
+
+void MDSMonitor::update_logger()
+{
+  dout(10) << "update_logger" << dendl;
+
+  mon->cluster_logger->set(l_cluster_num_mds_up, mdsmap.get_num_up_mds());
+  mon->cluster_logger->set(l_cluster_num_mds_in, mdsmap.get_num_in_mds());
+  mon->cluster_logger->set(l_cluster_num_mds_failed, mdsmap.get_num_failed_mds());
+  mon->cluster_logger->set(l_cluster_mds_epoch, mdsmap.get_epoch());
+}
+
+bool MDSMonitor::preprocess_query(PaxosServiceMessage *m)
+{
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+
+  switch (m->get_type()) {
+    
+  case MSG_MDS_BEACON:
+    return preprocess_beacon(static_cast(m));
+    
+  case MSG_MON_COMMAND:
+    return preprocess_command(static_cast(m));
+
+  case MSG_MDS_OFFLOAD_TARGETS:
+    return preprocess_offload_targets(static_cast(m));
+
+  default:
+    assert(0);
+    m->put();
+    return true;
+  }
+}
+
+void MDSMonitor::_note_beacon(MMDSBeacon *m)
+{
+  uint64_t gid = m->get_global_id();
+  version_t seq = m->get_seq();
+
+  dout(15) << "_note_beacon " << *m << " noting time" << dendl;
+  last_beacon[gid].stamp = ceph_clock_now(g_ceph_context);  
+  last_beacon[gid].seq = seq;
+}
+
+bool MDSMonitor::preprocess_beacon(MMDSBeacon *m)
+{
+  int state = m->get_state();
+  uint64_t gid = m->get_global_id();
+  version_t seq = m->get_seq();
+  MDSMap::mds_info_t info;
+
+  // check privileges, ignore if fails
+  MonSession *session = m->get_session();
+  if (!session)
+    goto out;
+  if (!session->is_capable("mds", MON_CAP_X)) {
+    dout(0) << "preprocess_beacon got MMDSBeacon from entity with insufficient privileges "
+	    << session->caps << dendl;
+    goto out;
+  }
+
+  if (m->get_fsid() != mon->monmap->fsid) {
+    dout(0) << "preprocess_beacon on fsid " << m->get_fsid() << " != " << mon->monmap->fsid << dendl;
+    goto out;
+  }
+
+  dout(12) << "preprocess_beacon " << *m
+	   << " from " << m->get_orig_source_inst()
+	   << " " << m->get_compat()
+	   << dendl;
+
+  // make sure the address has a port
+  if (m->get_orig_source_addr().get_port() == 0) {
+    dout(1) << " ignoring boot message without a port" << dendl;
+    goto out;
+  }
+
+  // check compat
+  if (!m->get_compat().writeable(mdsmap.compat)) {
+    dout(1) << " mds " << m->get_source_inst() << " can't write to mdsmap " << mdsmap.compat << dendl;
+    goto out;
+  }
+
+  // fw to leader?
+  if (!mon->is_leader())
+    return false;
+
+  if (pending_mdsmap.test_flag(CEPH_MDSMAP_DOWN)) {
+    dout(7) << " mdsmap DOWN flag set, ignoring mds " << m->get_source_inst() << " beacon" << dendl;
+    goto out;
+  }
+
+  // booted, but not in map?
+  if (pending_mdsmap.is_dne_gid(gid)) {
+    if (state != MDSMap::STATE_BOOT) {
+      dout(7) << "mds_beacon " << *m << " is not in mdsmap" << dendl;
+      mon->send_reply(m, new MMDSMap(mon->monmap->fsid, &mdsmap));
+      goto out;
+    } else {
+      return false;  // not booted yet.
+    }
+  }
+  info = pending_mdsmap.get_info_gid(gid);
+
+  // old seq?
+  if (info.state_seq > seq) {
+    dout(7) << "mds_beacon " << *m << " has old seq, ignoring" << dendl;
+    goto out;
+  }
+
+  if (mdsmap.get_epoch() != m->get_last_epoch_seen()) {
+    dout(10) << "mds_beacon " << *m
+	     << " ignoring requested state, because mds hasn't seen latest map" << dendl;
+    goto ignore;
+  }
+
+  if (info.laggy()) {
+    _note_beacon(m);
+    return false;  // no longer laggy, need to update map.
+  }
+  if (state == MDSMap::STATE_BOOT) {
+    // ignore, already booted.
+    goto out;
+  }
+  // is there a state change here?
+  if (info.state != state) {    
+    // legal state change?
+    if ((info.state == MDSMap::STATE_STANDBY ||
+	 info.state == MDSMap::STATE_STANDBY_REPLAY ||
+	 info.state == MDSMap::STATE_ONESHOT_REPLAY) && state > 0) {
+      dout(10) << "mds_beacon mds can't activate itself (" << ceph_mds_state_name(info.state)
+	       << " -> " << ceph_mds_state_name(state) << ")" << dendl;
+      goto ignore;
+    }
+    
+    if (info.state == MDSMap::STATE_STANDBY &&
+	(state == MDSMap::STATE_STANDBY_REPLAY ||
+	    state == MDSMap::STATE_ONESHOT_REPLAY) &&
+	(pending_mdsmap.is_degraded() ||
+	 ((m->get_standby_for_rank() >= 0) &&
+	     pending_mdsmap.get_state(m->get_standby_for_rank()) < MDSMap::STATE_ACTIVE))) {
+      dout(10) << "mds_beacon can't standby-replay mds." << m->get_standby_for_rank() << " at this time (cluster degraded, or mds not active)" << dendl;
+      dout(10) << "pending_mdsmap.is_degraded()==" << pending_mdsmap.is_degraded()
+          << " rank state: " << ceph_mds_state_name(pending_mdsmap.get_state(m->get_standby_for_rank())) << dendl;
+      goto ignore;
+    }
+    _note_beacon(m);
+    return false;  // need to update map
+  }
+
+ ignore:
+  // note time and reply
+  _note_beacon(m);
+  mon->send_reply(m,
+		  new MMDSBeacon(mon->monmap->fsid, m->get_global_id(), m->get_name(),
+				 mdsmap.get_epoch(), state, seq));
+  
+  // done
+ out:
+  m->put();
+  return true;
+}
+
+bool MDSMonitor::preprocess_offload_targets(MMDSLoadTargets* m)
+{
+  dout(10) << "preprocess_offload_targets " << *m << " from " << m->get_orig_source() << dendl;
+  uint64_t gid;
+  
+  // check privileges, ignore message if fails
+  MonSession *session = m->get_session();
+  if (!session)
+    goto done;
+  if (!session->is_capable("mds", MON_CAP_X)) {
+    dout(0) << "preprocess_offload_targets got MMDSLoadTargets from entity with insufficient caps "
+	    << session->caps << dendl;
+    goto done;
+  }
+
+  gid = m->global_id;
+  if (mdsmap.mds_info.count(gid) &&
+      m->targets == mdsmap.mds_info[gid].export_targets)
+    goto done;
+
+  return false;
+
+ done:
+  m->put();
+  return true;
+}
+
+
+bool MDSMonitor::prepare_update(PaxosServiceMessage *m)
+{
+  dout(7) << "prepare_update " << *m << dendl;
+
+  switch (m->get_type()) {
+    
+  case MSG_MDS_BEACON:
+    return prepare_beacon(static_cast(m));
+
+  case MSG_MON_COMMAND:
+    return prepare_command(static_cast(m));
+
+  case MSG_MDS_OFFLOAD_TARGETS:
+    return prepare_offload_targets(static_cast(m));
+  
+  default:
+    assert(0);
+    m->put();
+  }
+
+  return true;
+}
+
+
+
+bool MDSMonitor::prepare_beacon(MMDSBeacon *m)
+{
+  // -- this is an update --
+  dout(12) << "prepare_beacon " << *m << " from " << m->get_orig_source_inst() << dendl;
+  entity_addr_t addr = m->get_orig_source_inst().addr;
+  uint64_t gid = m->get_global_id();
+  int state = m->get_state();
+  version_t seq = m->get_seq();
+
+  // boot?
+  if (state == MDSMap::STATE_BOOT) {
+    // zap previous instance of this name?
+    if (g_conf->mds_enforce_unique_name) {
+      bool failed_mds = false;
+      while (uint64_t existing = pending_mdsmap.find_mds_gid_by_name(m->get_name())) {
+        if (!mon->osdmon()->is_writeable()) {
+          mon->osdmon()->wait_for_writeable(new C_RetryMessage(this, m));
+          return false;
+        }
+	fail_mds_gid(existing);
+        failed_mds = true;
+      }
+      if (failed_mds) {
+        assert(mon->osdmon()->is_writeable());
+        request_proposal(mon->osdmon());
+      }
+    }
+
+    // add
+    MDSMap::mds_info_t& info = pending_mdsmap.mds_info[gid];
+    info.global_id = gid;
+    info.name = m->get_name();
+    info.rank = -1;
+    info.addr = addr;
+    info.state = MDSMap::STATE_STANDBY;
+    info.state_seq = seq;
+    info.standby_for_rank = m->get_standby_for_rank();
+    info.standby_for_name = m->get_standby_for_name();
+
+    if (!info.standby_for_name.empty()) {
+      const MDSMap::mds_info_t *leaderinfo = mdsmap.find_by_name(info.standby_for_name);
+      if (leaderinfo && (leaderinfo->rank >= 0)) {
+        info.standby_for_rank =
+            mdsmap.find_by_name(info.standby_for_name)->rank;
+        if (mdsmap.is_followable(info.standby_for_rank)) {
+          info.state = MDSMap::STATE_STANDBY_REPLAY;
+        }
+      }
+    }
+
+    // initialize the beacon timer
+    last_beacon[gid].stamp = ceph_clock_now(g_ceph_context);
+    last_beacon[gid].seq = seq;
+
+    // new incompat?
+    if (!pending_mdsmap.compat.writeable(m->get_compat())) {
+      dout(10) << " mdsmap " << pending_mdsmap.compat << " can't write to new mds' " << m->get_compat()
+	       << ", updating mdsmap and killing old mds's"
+	       << dendl;
+      pending_mdsmap.compat = m->get_compat();
+    }
+
+  } else {
+    // state change
+    MDSMap::mds_info_t& info = pending_mdsmap.get_info_gid(gid);
+
+    if (info.laggy()) {
+      dout(10) << "prepare_beacon clearing laggy flag on " << addr << dendl;
+      info.clear_laggy();
+    }
+  
+    dout(10) << "prepare_beacon mds." << info.rank
+	     << " " << ceph_mds_state_name(info.state)
+	     << " -> " << ceph_mds_state_name(state)
+	     << "  standby_for_rank=" << m->get_standby_for_rank()
+	     << dendl;
+    if (state == MDSMap::STATE_STOPPED) {
+      pending_mdsmap.up.erase(info.rank);
+      pending_mdsmap.in.erase(info.rank);
+      pending_mdsmap.stopped.insert(info.rank);
+      pending_mdsmap.mds_info.erase(gid);  // last! info is a ref into this map
+      last_beacon.erase(gid);
+    } else if (state == MDSMap::STATE_STANDBY_REPLAY) {
+      if (m->get_standby_for_rank() == MDSMap::MDS_STANDBY_NAME) {
+        /* convert name to rank. If we don't have it, do nothing. The
+	 mds will stay in standby and keep requesting the state change */
+        dout(20) << "looking for mds " << m->get_standby_for_name()
+                  << " to STANDBY_REPLAY for" << dendl;
+        const MDSMap::mds_info_t *found_mds = NULL;
+        if ((found_mds = mdsmap.find_by_name(m->get_standby_for_name())) &&
+            (found_mds->rank >= 0) &&
+	    mdsmap.is_followable(found_mds->rank)) {
+          info.standby_for_rank = found_mds->rank;
+          dout(10) <<" found mds " << m->get_standby_for_name()
+                       << "; it has rank " << info.standby_for_rank << dendl;
+          info.state = MDSMap::STATE_STANDBY_REPLAY;
+          info.state_seq = seq;
+        } else {
+          m->put();
+          return false;
+        }
+      } else if (m->get_standby_for_rank() >= 0 &&
+		 mdsmap.is_followable(m->get_standby_for_rank())) {
+        /* switch to standby-replay for this MDS*/
+        info.state = MDSMap::STATE_STANDBY_REPLAY;
+        info.state_seq = seq;
+        info.standby_for_rank = m->get_standby_for_rank();
+      } else { //it's a standby for anybody, and is already in the list
+        assert(pending_mdsmap.get_mds_info().count(info.global_id));
+        m->put();
+        return false;
+      }
+    } else {
+      info.state = state;
+      info.state_seq = seq;
+    }
+  }
+
+  dout(7) << "prepare_beacon pending map now:" << dendl;
+  print_map(pending_mdsmap);
+  
+  wait_for_finished_proposal(new C_Updated(this, m));
+
+  return true;
+}
+
+bool MDSMonitor::prepare_offload_targets(MMDSLoadTargets *m)
+{
+  uint64_t gid = m->global_id;
+  if (pending_mdsmap.mds_info.count(gid)) {
+    dout(10) << "prepare_offload_targets " << gid << " " << m->targets << dendl;
+    pending_mdsmap.mds_info[gid].export_targets = m->targets;
+  } else {
+    dout(10) << "prepare_offload_targets " << gid << " not in map" << dendl;
+  }
+  m->put();
+  return true;
+}
+
+bool MDSMonitor::should_propose(double& delay)
+{
+  // delegate to PaxosService to assess whether we should propose
+  return PaxosService::should_propose(delay);
+}
+
+void MDSMonitor::_updated(MMDSBeacon *m)
+{
+  dout(10) << "_updated " << m->get_orig_source() << " " << *m << dendl;
+  mon->clog.info() << m->get_orig_source_inst() << " "
+	  << ceph_mds_state_name(m->get_state()) << "\n";
+
+  if (m->get_state() == MDSMap::STATE_STOPPED) {
+    // send the map manually (they're out of the map, so they won't get it automatic)
+    mon->send_reply(m, new MMDSMap(mon->monmap->fsid, &mdsmap));
+  }
+
+  m->put();
+}
+
+void MDSMonitor::on_active()
+{
+  tick();
+  update_logger();
+
+  if (mon->is_leader())
+    mon->clog.info() << "mdsmap " << mdsmap << "\n";
+}
+
+void MDSMonitor::get_health(list >& summary,
+			    list > *detail) const
+{
+  mdsmap.get_health(summary, detail);
+}
+
+void MDSMonitor::dump_info(Formatter *f)
+{
+  f->open_object_section("mdsmap");
+  mdsmap.dump(f);
+  f->close_section();
+
+  f->dump_unsigned("mdsmap_first_committed", get_first_committed());
+  f->dump_unsigned("mdsmap_last_committed", get_last_committed());
+}
+
+bool MDSMonitor::preprocess_command(MMonCommand *m)
+{
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss, ds;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr f(new_formatter(format));
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  if (prefix == "mds stat") {
+    if (f) {
+      f->open_object_section("mds_stat");
+      dump_info(f.get());
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << mdsmap;
+    }
+    r = 0;
+  } else if (prefix == "mds dump") {
+    int64_t epocharg;
+    epoch_t epoch;
+
+    MDSMap *p = &mdsmap;
+    if (cmd_getval(g_ceph_context, cmdmap, "epoch", epocharg)) {
+      epoch = epocharg;
+      bufferlist b;
+      int err = get_version(epoch, b);
+      if (err == -ENOENT) {
+	p = 0;
+	r = -ENOENT;
+      } else {
+	assert(err == 0);
+	assert(b.length());
+	p = new MDSMap;
+	p->decode(b);
+      }
+    }
+    if (p) {
+      stringstream ds;
+      if (f != NULL) {
+	f->open_object_section("mdsmap");
+	p->dump(f.get());
+	f->close_section();
+	f->flush(ds);
+	r = 0;
+      } else {
+	p->print(ds);
+	r = 0;
+      } 
+      if (r == 0) {
+	rdata.append(ds);
+	ss << "dumped mdsmap epoch " << p->get_epoch();
+      }
+      if (p != &mdsmap)
+	delete p;
+    }
+  } else if (prefix == "mds getmap") {
+    epoch_t e;
+    int64_t epocharg;
+    bufferlist b;
+    if (cmd_getval(g_ceph_context, cmdmap, "epoch", epocharg)) {
+      e = epocharg;
+      int err = get_version(e, b);
+      if (err == -ENOENT) {
+	r = -ENOENT;
+      } else {
+	assert(r == 0);
+	assert(b.length());
+	MDSMap mm;
+	mm.decode(b);
+	mm.encode(rdata, m->get_connection()->get_features());
+	ss << "got mdsmap epoch " << mm.get_epoch();
+      }
+    } else {
+      mdsmap.encode(rdata, m->get_connection()->get_features());
+      ss << "got mdsmap epoch " << mdsmap.get_epoch();
+    }
+    r = 0;
+  } else if (prefix == "mds tell") {
+    string whostr;
+    cmd_getval(g_ceph_context, cmdmap, "who", whostr);
+    vectorargs_vec;
+    cmd_getval(g_ceph_context, cmdmap, "args", args_vec);
+
+    if (whostr == "*") {
+      r = -ENOENT;
+      const map mds_info = mdsmap.get_mds_info();
+      for (map::const_iterator i = mds_info.begin();
+	   i != mds_info.end();
+	   ++i) {
+	m->cmd = args_vec;
+	mon->send_command(i->second.get_inst(), m->cmd);
+	r = 0;
+      }
+      if (r == -ENOENT) {
+	ss << "no mds active";
+      } else {
+	ss << "ok";
+      }
+    } else {
+      errno = 0;
+      long who = strtol(whostr.c_str(), 0, 10);
+      if (!errno && who >= 0) {
+	if (mdsmap.is_up(who)) {
+	  m->cmd = args_vec;
+	  mon->send_command(mdsmap.get_inst(who), m->cmd);
+	  r = 0;
+	  ss << "ok";
+	} else {
+	  ss << "mds." << who << " not up";
+	  r = -ENOENT;
+	}
+      } else ss << "specify mds number or *";
+    }
+  } else if (prefix == "mds compat show") {
+      if (f) {
+	f->open_object_section("mds_compat");
+	mdsmap.compat.dump(f.get());
+	f->close_section();
+	f->flush(ds);
+      } else {
+	ds << mdsmap.compat;
+      }
+      r = 0;
+  }
+
+  if (r != -1) {
+    rdata.append(ds);
+    string rs;
+    getline(ss, rs);
+    mon->reply_command(m, r, rs, rdata, get_last_committed());
+    return true;
+  } else
+    return false;
+}
+
+void MDSMonitor::fail_mds_gid(uint64_t gid)
+{
+  assert(pending_mdsmap.mds_info.count(gid));
+  MDSMap::mds_info_t& info = pending_mdsmap.mds_info[gid];
+  dout(10) << "fail_mds_gid " << gid << " mds." << info.name << " rank " << info.rank << dendl;
+
+  utime_t until = ceph_clock_now(g_ceph_context);
+  until += g_conf->mds_blacklist_interval;
+
+  pending_mdsmap.last_failure_osd_epoch = mon->osdmon()->blacklist(info.addr, until);
+
+  if (info.rank >= 0) {
+    if (info.state == MDSMap::STATE_CREATING) {
+      // If this gid didn't make it past CREATING, then forget
+      // the rank ever existed so that next time it's handed out
+      // to a gid it'll go back into CREATING.
+      pending_mdsmap.in.erase(info.rank);
+    } else {
+      // Put this rank into the failed list so that the next available STANDBY will
+      // pick it up.
+      pending_mdsmap.failed.insert(info.rank);
+    }
+    pending_mdsmap.up.erase(info.rank);
+  }
+
+  pending_mdsmap.mds_info.erase(gid);
+}
+
+int MDSMonitor::fail_mds(std::ostream &ss, const std::string &arg)
+{
+  std::string err;
+  int w = strict_strtoll(arg.c_str(), 10, &err);
+  if (!err.empty()) {
+    // Try to interpret the arg as an MDS name
+    const MDSMap::mds_info_t *mds_info = mdsmap.find_by_name(arg);
+    if (!mds_info) {
+      ss << "Can't find any MDS named '" << arg << "'";
+      return -ENOENT;
+    }
+    w = mds_info->rank;
+  }
+
+  if (!mon->osdmon()->is_writeable()) {
+    return -EAGAIN;
+ }
+
+  bool failed_mds_gid = false;
+  if (pending_mdsmap.up.count(w)) {
+    uint64_t gid = pending_mdsmap.up[w];
+    if (pending_mdsmap.mds_info.count(gid)) {
+      fail_mds_gid(gid);
+      failed_mds_gid = true;
+    }
+    ss << "failed mds." << w;
+  } else if (pending_mdsmap.mds_info.count(w)) {
+    fail_mds_gid(w);
+    failed_mds_gid = true;
+    ss << "failed mds gid " << w;
+  }
+
+  if (failed_mds_gid) {
+    assert(mon->osdmon()->is_writeable());
+    request_proposal(mon->osdmon());
+  }
+  return 0;
+}
+
+bool MDSMonitor::prepare_command(MMonCommand *m)
+{
+  int r = -EINVAL;
+  stringstream ss;
+  bufferlist rdata;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  string whostr;
+  cmd_getval(g_ceph_context, cmdmap, "who", whostr);
+  if (prefix == "mds stop" ||
+      prefix == "mds deactivate") {
+    int who = parse_pos_long(whostr.c_str(), &ss);
+    if (who < 0)
+      goto out;
+    if (!pending_mdsmap.is_active(who)) {
+      r = -EEXIST;
+      ss << "mds." << who << " not active (" 
+	 << ceph_mds_state_name(pending_mdsmap.get_state(who)) << ")";
+    } else if ((pending_mdsmap.get_root() == who ||
+		pending_mdsmap.get_tableserver() == who) &&
+	       pending_mdsmap.get_num_in_mds() > 1) {
+      r = -EBUSY;
+      ss << "can't tell the root (" << pending_mdsmap.get_root()
+	 << ") or tableserver (" << pending_mdsmap.get_tableserver()
+	 << " to deactivate unless it is the last mds in the cluster";
+    } else if (pending_mdsmap.get_num_in_mds() <= pending_mdsmap.get_max_mds()) {
+      r = -EBUSY;
+      ss << "must decrease max_mds or else MDS will immediately reactivate";
+    } else {
+      r = 0;
+      uint64_t gid = pending_mdsmap.up[who];
+      ss << "telling mds." << who << " " << pending_mdsmap.mds_info[gid].addr << " to deactivate";
+      pending_mdsmap.mds_info[gid].state = MDSMap::STATE_STOPPING;
+    }
+
+  } else if (prefix == "mds set_max_mds") {
+    // NOTE: see also "mds set max_mds", which can modify the same field.
+    int64_t maxmds;
+    if (!cmd_getval(g_ceph_context, cmdmap, "maxmds", maxmds) || maxmds < 0)
+      goto out;
+    pending_mdsmap.max_mds = maxmds;
+    r = 0;
+    ss << "max_mds = " << pending_mdsmap.max_mds;
+  } else if (prefix == "mds set") {
+    string var;
+    if (!cmd_getval(g_ceph_context, cmdmap, "var", var) || var.empty()) {
+      ss << "Invalid variable";
+      goto out;
+    }
+    string val;
+    string interr;
+    int64_t n = 0;
+    if (!cmd_getval(g_ceph_context, cmdmap, "val", val))
+      goto out;
+    // we got a string.  see if it contains an int.
+    n = strict_strtoll(val.c_str(), 10, &interr);
+    if (var == "max_mds") {
+      // NOTE: see also "mds set_max_mds", which can modify the same field.
+      if (interr.length())
+	goto out;
+      pending_mdsmap.max_mds = n;
+    } else if (var == "inline_data") {
+      if (val == "true" || val == "yes" || (!interr.length() && n == 1)) {
+	string confirm;
+	if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) ||
+	    confirm != "--yes-i-really-mean-it") {
+	  ss << "inline data is new and experimental; you must specify --yes-i-really-mean-it";
+	  r = -EPERM;
+	  goto out;
+	}
+	ss << "inline data enabled";
+	pending_mdsmap.set_inline_data_enabled(true);
+	pending_mdsmap.compat.incompat.insert(MDS_FEATURE_INCOMPAT_INLINE);
+      } else if (val == "false" || val == "no" ||
+		 (!interr.length() && n == 0)) {
+	ss << "inline data disabled";
+	pending_mdsmap.set_inline_data_enabled(false);
+      } else {
+	ss << "value must be false|no|0 or true|yes|1";
+	r = -EINVAL;
+	goto out;
+      }
+    } else if (var == "max_file_size") {
+      if (interr.length()) {
+	ss << var << " requires an integer value";
+	goto out;
+      }
+      if (n < CEPH_MIN_STRIPE_UNIT) {
+	r = -ERANGE;
+	ss << var << " must at least " << CEPH_MIN_STRIPE_UNIT;
+	goto out;
+      }
+      pending_mdsmap.max_file_size = n;
+    } else if (var == "allow_new_snaps") {
+      if (val == "false" || val == "no" || (interr.length() == 0 && n == 0)) {
+	pending_mdsmap.clear_snaps_allowed();
+	ss << "disabled new snapshots";
+      } else if (val == "true" || val == "yes" || (interr.length() == 0 && n == 1)) {
+	string confirm;
+	if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) ||
+	    confirm != "--yes-i-really-mean-it") {
+	  ss << "Snapshots are unstable and will probably break your FS! Set to --yes-i-really-mean-it if you are sure you want to enable them";
+	  r = -EPERM;
+	  goto out;
+	}
+	pending_mdsmap.set_snaps_allowed();
+	ss << "enabled new snapshots";
+      } else {
+	ss << "value must be true|yes|1 or false|no|0";
+	r = -EINVAL;
+	goto out;
+      }
+    } else {
+      ss << "unknown variable " << var;
+      goto out;
+    }
+    r = 0;
+  } else if (prefix == "mds setmap") {
+    MDSMap map;
+    map.decode(m->get_data());
+    epoch_t e = 0;
+    int64_t epochnum;
+    if (cmd_getval(g_ceph_context, cmdmap, "epoch", epochnum))
+      e = epochnum;
+
+    if (pending_mdsmap.epoch == e) {
+      map.epoch = pending_mdsmap.epoch;  // make sure epoch is correct
+      pending_mdsmap = map;
+      string rs = "set mds map";
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    } else {
+      ss << "next mdsmap epoch " << pending_mdsmap.epoch << " != " << e;
+    }
+
+  } else if (prefix == "mds set_state") {
+    int64_t gid;
+    if (!cmd_getval(g_ceph_context, cmdmap, "gid", gid)) {
+      ss << "error parsing 'gid' value '"
+         << cmd_vartype_stringify(cmdmap["gid"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    int64_t state;
+    if (!cmd_getval(g_ceph_context, cmdmap, "state", state)) {
+      ss << "error parsing 'state' string value '"
+         << cmd_vartype_stringify(cmdmap["state"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    if (!pending_mdsmap.is_dne_gid(gid)) {
+      MDSMap::mds_info_t& info = pending_mdsmap.get_info_gid(gid);
+      info.state = state;
+      stringstream ss;
+      ss << "set mds gid " << gid << " to state " << state << " " << ceph_mds_state_name(state);
+      string rs;
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+
+  } else if (prefix == "mds fail") {
+    string who;
+    cmd_getval(g_ceph_context, cmdmap, "who", who);
+    r = fail_mds(ss, who);
+    if (r < 0 && r == -EAGAIN) {
+      mon->osdmon()->wait_for_writeable(new C_RetryMessage(this, m));
+      return false; // don't propose yet; wait for message to be retried
+    }
+
+  } else if (prefix == "mds rm") {
+    int64_t gid;
+    if (!cmd_getval(g_ceph_context, cmdmap, "gid", gid)) {
+      ss << "error parsing 'gid' value '"
+         << cmd_vartype_stringify(cmdmap["gid"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    int state = pending_mdsmap.get_state_gid(gid);
+    if (state == 0) {
+      ss << "mds gid " << gid << " dne";
+      r = 0;
+    } else if (state > 0) {
+      ss << "cannot remove active mds." << pending_mdsmap.get_info_gid(gid).name
+	 << " rank " << pending_mdsmap.get_info_gid(gid).rank;
+      r = -EBUSY;
+    } else {
+      pending_mdsmap.mds_info.erase(gid);
+      stringstream ss;
+      ss << "removed mds gid " << gid;
+      string rs;
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+  } else if (prefix == "mds rmfailed") {
+    int64_t w;
+    if (!cmd_getval(g_ceph_context, cmdmap, "who", w)) {
+      ss << "error parsing 'who' value '"
+         << cmd_vartype_stringify(cmdmap["who"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    pending_mdsmap.failed.erase(w);
+    stringstream ss;
+    ss << "removed failed mds." << w;
+    string rs;
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "mds cluster_down") {
+    if (pending_mdsmap.test_flag(CEPH_MDSMAP_DOWN)) {
+      ss << "mdsmap already marked DOWN";
+    } else {
+      pending_mdsmap.set_flag(CEPH_MDSMAP_DOWN);
+      ss << "marked mdsmap DOWN";
+    }
+    r = 0;
+  } else if (prefix == "mds cluster_up") {
+    if (pending_mdsmap.test_flag(CEPH_MDSMAP_DOWN)) {
+      pending_mdsmap.clear_flag(CEPH_MDSMAP_DOWN);
+      ss << "unmarked mdsmap DOWN";
+    } else {
+      ss << "mdsmap not marked DOWN";
+    }
+    r = 0;
+  } else if (prefix == "mds compat rm_compat") {
+    int64_t f;
+    if (!cmd_getval(g_ceph_context, cmdmap, "feature", f)) {
+      ss << "error parsing feature value '"
+         << cmd_vartype_stringify(cmdmap["feature"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    if (pending_mdsmap.compat.compat.contains(f)) {
+      ss << "removing compat feature " << f;
+      pending_mdsmap.compat.compat.remove(f);
+      r = 0;
+    } else {
+      ss << "compat feature " << f << " not present in " << pending_mdsmap.compat;
+      r = 0;
+    }
+  } else if (prefix == "mds compat rm_incompat") {
+    int64_t f;
+    if (!cmd_getval(g_ceph_context, cmdmap, "feature", f)) {
+      ss << "error parsing feature value '"
+         << cmd_vartype_stringify(cmdmap["feature"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    if (pending_mdsmap.compat.incompat.contains(f)) {
+      ss << "removing incompat feature " << f;
+      pending_mdsmap.compat.incompat.remove(f);
+      r = 0;
+    } else {
+      ss << "incompat feature " << f << " not present in " << pending_mdsmap.compat;
+      r = 0;
+    }
+
+  } else if (prefix == "mds add_data_pool") {
+    string poolname;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolname);
+    int64_t poolid = mon->osdmon()->osdmap.lookup_pg_pool_name(poolname);
+    if (poolid < 0) {
+      string err;
+      poolid = strict_strtol(poolname.c_str(), 10, &err);
+      if (err.length()) {
+	r = -ENOENT;
+	poolid = -1;
+	ss << "pool '" << poolname << "' does not exist";
+      }
+    }
+    if (poolid >= 0) {
+      pending_mdsmap.add_data_pool(poolid);
+      ss << "added data pool " << poolid << " to mdsmap";
+      r = 0;
+    }
+  } else if (prefix == "mds remove_data_pool") {
+    string poolname;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolname);
+    int64_t poolid = mon->osdmon()->osdmap.lookup_pg_pool_name(poolname);
+    if (poolid < 0) {
+      string err;
+      poolid = strict_strtol(poolname.c_str(), 10, &err);
+      if (err.length()) {
+	r = -ENOENT;
+	poolid = -1;
+	ss << "pool '" << poolname << "' does not exist";
+      }
+    }
+
+    if (pending_mdsmap.get_first_data_pool() == poolid) {
+      r = -EINVAL;
+      poolid = -1;
+      ss << "cannot remove default data pool";
+    }
+
+    if (poolid >= 0) {
+      cmd_getval(g_ceph_context, cmdmap, "poolid", poolid);
+      r = pending_mdsmap.remove_data_pool(poolid);
+      if (r == -ENOENT)
+	r = 0;
+      if (r == 0)
+	ss << "removed data pool " << poolid << " from mdsmap";
+    }
+  } else if (prefix == "mds newfs") {
+    MDSMap newmap;
+    int64_t metadata, data;
+    if (!cmd_getval(g_ceph_context, cmdmap, "metadata", metadata)) {
+      ss << "error parsing 'metadata' value '"
+         << cmd_vartype_stringify(cmdmap["metadata"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    if (!cmd_getval(g_ceph_context, cmdmap, "data", data)) {
+      ss << "error parsing 'data' value '"
+         << cmd_vartype_stringify(cmdmap["data"]) << "'";
+      r = -EINVAL;
+      goto out;
+    }
+    string sure;
+    cmd_getval(g_ceph_context, cmdmap, "sure", sure);
+    if (sure != "--yes-i-really-mean-it") {
+      ss << "this is DANGEROUS and will wipe out the mdsmap's fs, and may clobber data in the new pools you specify.  add --yes-i-really-mean-it if you do.";
+      r = -EPERM;
+    } else {
+      newmap.inc = pending_mdsmap.inc;
+      pending_mdsmap = newmap;
+      pending_mdsmap.epoch = mdsmap.epoch + 1;
+      create_new_fs(pending_mdsmap, metadata, data);
+      ss << "new fs with metadata pool " << metadata << " and data pool " << data;
+      string rs;
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+  } else {
+    ss << "unrecognized command";
+  }
+ out:
+  string rs;
+  getline(ss, rs);
+
+  if (r >= 0) {
+    // success.. delay reply
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, r, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else {
+    // reply immediately
+    mon->reply_command(m, r, rs, rdata, get_last_committed());
+    return false;
+  }
+}
+
+
+void MDSMonitor::check_subs()
+{
+  string type = "mdsmap";
+  if (mon->session_map.subs.count(type) == 0)
+    return;
+  xlist::iterator p = mon->session_map.subs[type]->begin();
+  while (!p.end()) {
+    Subscription *sub = *p;
+    ++p;
+    check_sub(sub);
+  }
+}
+
+void MDSMonitor::check_sub(Subscription *sub)
+{
+  if (sub->next <= mdsmap.get_epoch()) {
+    mon->messenger->send_message(new MMDSMap(mon->monmap->fsid, &mdsmap),
+				 sub->session->inst);
+    if (sub->onetime)
+      mon->session_map.remove_sub(sub);
+    else
+      sub->next = mdsmap.get_epoch() + 1;
+  }
+}
+
+void MDSMonitor::tick()
+{
+  // make sure mds's are still alive
+  // ...if i am an active leader
+  if (!is_active()) return;
+
+  dout(10) << mdsmap << dendl;
+  
+  bool do_propose = false;
+
+  if (!mon->is_leader()) return;
+
+  // expand mds cluster (add new nodes to @in)?
+  while (pending_mdsmap.get_num_in_mds() < pending_mdsmap.get_max_mds() &&
+	 !pending_mdsmap.is_degraded()) {
+    int mds = 0;
+    string name;
+    while (pending_mdsmap.is_in(mds))
+      mds++;
+    uint64_t newgid = pending_mdsmap.find_replacement_for(mds, name);
+    if (!newgid)
+      break;
+
+    MDSMap::mds_info_t& info = pending_mdsmap.mds_info[newgid];
+    dout(1) << "adding standby " << info.addr << " as mds." << mds << dendl;
+    
+    info.rank = mds;
+    if (pending_mdsmap.stopped.count(mds)) {
+      info.state = MDSMap::STATE_STARTING;
+      pending_mdsmap.stopped.erase(mds);
+    } else
+      info.state = MDSMap::STATE_CREATING;
+    info.inc = ++pending_mdsmap.inc[mds];
+    pending_mdsmap.in.insert(mds);
+    pending_mdsmap.up[mds] = newgid;
+    do_propose = true;
+  }
+
+  // check beacon timestamps
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t cutoff = now;
+  cutoff -= g_conf->mds_beacon_grace;
+
+  // make sure last_beacon is fully populated
+  for (map::iterator p = pending_mdsmap.mds_info.begin();
+       p != pending_mdsmap.mds_info.end();
+       ++p) {
+    if (last_beacon.count(p->first) == 0) {
+      const MDSMap::mds_info_t& info = p->second;
+      dout(10) << " adding " << p->second.addr << " mds." << info.rank << "." << info.inc
+	       << " " << ceph_mds_state_name(info.state)
+	       << " to last_beacon" << dendl;
+      last_beacon[p->first].stamp = ceph_clock_now(g_ceph_context);
+      last_beacon[p->first].seq = 0;
+    }
+  }
+
+  if (mon->osdmon()->is_writeable()) {
+
+    bool propose_osdmap = false;
+
+    map::iterator p = last_beacon.begin();
+    while (p != last_beacon.end()) {
+      uint64_t gid = p->first;
+      utime_t since = p->second.stamp;
+      uint64_t seq = p->second.seq;
+      ++p;
+      
+      if (pending_mdsmap.mds_info.count(gid) == 0) {
+	// clean it out
+	last_beacon.erase(gid);
+	continue;
+      }
+
+      if (since >= cutoff)
+	continue;
+
+      MDSMap::mds_info_t& info = pending_mdsmap.mds_info[gid];
+
+      dout(10) << "no beacon from " << gid << " " << info.addr << " mds." << info.rank << "." << info.inc
+	       << " " << ceph_mds_state_name(info.state)
+	       << " since " << since << dendl;
+      
+      // are we in?
+      // and is there a non-laggy standby that can take over for us?
+      uint64_t sgid;
+      if (info.rank >= 0 &&
+	  info.state != MDSMap::STATE_STANDBY &&
+	  info.state != MDSMap::STATE_STANDBY_REPLAY &&
+	  (sgid = pending_mdsmap.find_replacement_for(info.rank, info.name)) != 0) {
+	MDSMap::mds_info_t& si = pending_mdsmap.mds_info[sgid];
+	dout(10) << " replacing " << gid << " " << info.addr << " mds." << info.rank << "." << info.inc
+		 << " " << ceph_mds_state_name(info.state)
+		 << " with " << sgid << "/" << si.name << " " << si.addr << dendl;
+	switch (info.state) {
+	case MDSMap::STATE_CREATING:
+	case MDSMap::STATE_STARTING:
+	  si.state = info.state;
+	  break;
+	case MDSMap::STATE_REPLAY:
+	case MDSMap::STATE_RESOLVE:
+	case MDSMap::STATE_RECONNECT:
+	case MDSMap::STATE_REJOIN:
+	case MDSMap::STATE_CLIENTREPLAY:
+	case MDSMap::STATE_ACTIVE:
+	case MDSMap::STATE_STOPPING:
+	  si.state = MDSMap::STATE_REPLAY;
+	  break;
+	default:
+	  assert(0);
+	}
+
+	info.state_seq = seq;
+	si.rank = info.rank;
+	si.inc = ++pending_mdsmap.inc[info.rank];
+	pending_mdsmap.up[info.rank] = sgid;
+	if (si.state > 0)
+	  pending_mdsmap.last_failure = pending_mdsmap.epoch;
+	if (si.state > 0 ||
+	    si.state == MDSMap::STATE_CREATING ||
+	    si.state == MDSMap::STATE_STARTING) {
+	  // blacklist laggy mds
+	  utime_t until = now;
+	  until += g_conf->mds_blacklist_interval;
+	  pending_mdsmap.last_failure_osd_epoch = mon->osdmon()->blacklist(info.addr, until);
+	  propose_osdmap = true;
+	}
+	pending_mdsmap.mds_info.erase(gid);
+	last_beacon.erase(gid);
+	do_propose = true;
+      } else if (info.state == MDSMap::STATE_STANDBY_REPLAY) {
+	dout(10) << " failing " << gid << " " << info.addr << " mds." << info.rank << "." << info.inc
+		 << " " << ceph_mds_state_name(info.state)
+		 << dendl;
+	pending_mdsmap.mds_info.erase(gid);
+	last_beacon.erase(gid);
+	do_propose = true;
+      } else {
+	if (info.state == MDSMap::STATE_STANDBY ||
+	    info.state == MDSMap::STATE_STANDBY_REPLAY) {
+	  // remove it
+	  dout(10) << " removing " << gid << " " << info.addr << " mds." << info.rank << "." << info.inc
+		   << " " << ceph_mds_state_name(info.state)
+		   << " (laggy)" << dendl;
+	  pending_mdsmap.mds_info.erase(gid);
+	  do_propose = true;
+	} else if (!info.laggy()) {
+	  dout(10) << " marking " << gid << " " << info.addr << " mds." << info.rank << "." << info.inc
+		   << " " << ceph_mds_state_name(info.state)
+		   << " laggy" << dendl;
+	  info.laggy_since = now;
+	  do_propose = true;
+	}
+	last_beacon.erase(gid);
+      }
+    }
+
+    if (propose_osdmap)
+      request_proposal(mon->osdmon());
+
+  }
+
+
+  // have a standby take over?
+  set failed;
+  pending_mdsmap.get_failed_mds_set(failed);
+  if (!failed.empty()) {
+    set::iterator p = failed.begin();
+    while (p != failed.end()) {
+      int f = *p++;
+      uint64_t sgid;
+      string name;  // FIXME
+      sgid = pending_mdsmap.find_replacement_for(f, name);
+      if (sgid) {
+	MDSMap::mds_info_t& si = pending_mdsmap.mds_info[sgid];
+	dout(0) << " taking over failed mds." << f << " with " << sgid << "/" << si.name << " " << si.addr << dendl;
+	si.state = MDSMap::STATE_REPLAY;
+	si.rank = f;
+	si.inc = ++pending_mdsmap.inc[f];
+	pending_mdsmap.in.insert(f);
+	pending_mdsmap.up[f] = sgid;
+	pending_mdsmap.failed.erase(f);
+	do_propose = true;
+      }
+    }
+  }
+
+  // have a standby follow someone?
+  if (failed.empty()) {
+    for (map::iterator j = pending_mdsmap.mds_info.begin();
+	 j != pending_mdsmap.mds_info.end();
+	 ++j) {
+      MDSMap::mds_info_t& info = j->second;
+      
+      if (info.state != MDSMap::STATE_STANDBY)
+	continue;
+
+      /*
+       * This mds is standby but has no rank assigned.
+       * See if we can find it somebody to shadow
+       */
+      dout(20) << "gid " << j->first << " is standby and following nobody" << dendl;
+      
+      // standby for someone specific?
+      if (info.standby_for_rank >= 0) {
+	if (pending_mdsmap.is_followable(info.standby_for_rank) &&
+	    try_standby_replay(info, pending_mdsmap.mds_info[pending_mdsmap.up[info.standby_for_rank]]))
+	  do_propose = true;
+	continue;
+      }
+
+      // check everyone
+      for (map::iterator i = pending_mdsmap.mds_info.begin();
+	   i != pending_mdsmap.mds_info.end();
+	   ++i) {
+	if (i->second.rank >= 0 && pending_mdsmap.is_followable(i->second.rank)) {
+	  if ((info.standby_for_name.length() && info.standby_for_name != i->second.name) ||
+	      info.standby_for_rank >= 0)
+	    continue;   // we're supposed to follow someone else
+
+	  if (info.standby_for_rank == MDSMap::MDS_STANDBY_ANY &&
+	      try_standby_replay(info, i->second)) {
+	    do_propose = true;
+	    break;
+	  }
+	  continue;
+	}
+      }
+    }
+  }
+
+  if (do_propose)
+    propose_pending();
+}
+
+bool MDSMonitor::try_standby_replay(MDSMap::mds_info_t& finfo, MDSMap::mds_info_t& ainfo)
+{
+  // someone else already following?
+  uint64_t lgid = pending_mdsmap.find_standby_for(ainfo.rank, ainfo.name);
+  if (lgid) {
+    MDSMap::mds_info_t& sinfo = pending_mdsmap.mds_info[lgid];
+    dout(20) << " mds." << ainfo.rank
+	     << " standby gid " << lgid << " with state "
+	     << ceph_mds_state_name(sinfo.state)
+	     << dendl;
+    if (sinfo.state == MDSMap::STATE_STANDBY_REPLAY) {
+      dout(20) << "  skipping this MDS since it has a follower!" << dendl;
+      return false; // this MDS already has a standby
+    }
+  }
+
+  // hey, we found an MDS without a standby. Pair them!
+  finfo.standby_for_rank = ainfo.rank;
+  dout(10) << "  setting to shadow mds rank " << finfo.standby_for_rank << dendl;
+  finfo.state = MDSMap::STATE_STANDBY_REPLAY;
+  return true;
+}
diff --git a/ceph/src/mon/MDSMonitor.h b/ceph/src/mon/MDSMonitor.h
new file mode 100644
index 00000000..901e93e3
--- /dev/null
+++ b/ceph/src/mon/MDSMonitor.h
@@ -0,0 +1,127 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+ 
+/* Metadata Server Monitor
+ */
+
+#ifndef CEPH_MDSMONITOR_H
+#define CEPH_MDSMONITOR_H
+
+#include 
+#include 
+using namespace std;
+
+#include "include/types.h"
+#include "msg/Messenger.h"
+
+#include "mds/MDSMap.h"
+
+#include "PaxosService.h"
+#include "Session.h"
+
+#include "messages/MMDSBeacon.h"
+
+class MMDSGetMap;
+class MMonCommand;
+class MMDSLoadTargets;
+
+class MDSMonitor : public PaxosService {
+ public:
+  // mds maps
+  MDSMap mdsmap;          // current
+  bufferlist mdsmap_bl;   // encoded
+
+  MDSMap pending_mdsmap;  // current + pending updates
+
+  // my helpers
+  void print_map(MDSMap &m, int dbl=7);
+
+  class C_Updated : public Context {
+    MDSMonitor *mm;
+    MMDSBeacon *m;
+  public:
+    C_Updated(MDSMonitor *a, MMDSBeacon *c) :
+      mm(a), m(c) {}
+    void finish(int r) {
+      if (r >= 0)
+	mm->_updated(m);   // success
+      else if (r == -ECANCELED)
+	m->put();
+      else
+	mm->dispatch((PaxosServiceMessage*)m);        // try again
+    }
+  };
+
+  void create_new_fs(MDSMap &m, int metadata_pool, int data_pool);
+
+  version_t get_trim_to();
+
+  // service methods
+  void create_initial();
+  void update_from_paxos(bool *need_bootstrap);
+  void create_pending(); 
+  void encode_pending(MonitorDBStore::Transaction *t);
+  // we don't require full versions; don't encode any.
+  virtual void encode_full(MonitorDBStore::Transaction *t) { }
+
+  void update_logger();
+
+  void _updated(MMDSBeacon *m);
+ 
+  bool preprocess_query(PaxosServiceMessage *m);  // true if processed.
+  bool prepare_update(PaxosServiceMessage *m);
+  bool should_propose(double& delay);
+
+  void on_active();
+
+  void _note_beacon(class MMDSBeacon *m);
+  bool preprocess_beacon(class MMDSBeacon *m);
+  bool prepare_beacon(class MMDSBeacon *m);
+
+  bool preprocess_offload_targets(MMDSLoadTargets *m);
+  bool prepare_offload_targets(MMDSLoadTargets *m);
+
+  void get_health(list >& summary,
+		  list > *detail) const;
+  int fail_mds(std::ostream &ss, const std::string &arg);
+  void fail_mds_gid(uint64_t gid);
+
+  bool preprocess_command(MMonCommand *m);
+  bool prepare_command(MMonCommand *m);
+
+  // beacons
+  struct beacon_info_t {
+    utime_t stamp;
+    uint64_t seq;
+  };
+  map last_beacon;
+
+  bool try_standby_replay(MDSMap::mds_info_t& finfo, MDSMap::mds_info_t& ainfo);
+
+public:
+  MDSMonitor(Monitor *mn, Paxos *p, string service_name)
+    : PaxosService(mn, p, service_name)
+  {
+  }
+
+  void tick();     // check state, take actions
+
+  void dump_info(Formatter *f);
+
+  void check_subs();
+  void check_sub(Subscription *sub);
+
+};
+
+#endif
diff --git a/ceph/src/mon/Makefile.am b/ceph/src/mon/Makefile.am
new file mode 100644
index 00000000..6016355b
--- /dev/null
+++ b/ceph/src/mon/Makefile.am
@@ -0,0 +1,49 @@
+libmon_types_la_SOURCES = \
+	mon/PGMap.cc
+noinst_LTLIBRARIES += libmon_types.la
+
+libmon_la_SOURCES = \
+	mon/Monitor.cc \
+	mon/Paxos.cc \
+	mon/PaxosService.cc \
+	mon/OSDMonitor.cc \
+	mon/MDSMonitor.cc \
+	mon/MonmapMonitor.cc \
+	mon/PGMonitor.cc \
+	mon/LogMonitor.cc \
+	mon/AuthMonitor.cc \
+	mon/Elector.cc \
+	mon/MonitorStore.cc \
+	mon/HealthMonitor.cc \
+	mon/DataHealthService.cc \
+	mon/ConfigKeyService.cc
+libmon_la_LIBADD = $(LIBAUTH) $(LIBCOMMON) $(LIBOS) $(LIBMON_TYPES)
+noinst_LTLIBRARIES += libmon.la
+
+noinst_HEADERS += \
+	mon/AuthMonitor.h \
+	mon/DataHealthService.h \
+	mon/Elector.h \
+	mon/LogMonitor.h \
+	mon/ConfigKeyService.h \
+	mon/HealthMonitor.h \
+	mon/HealthService.h \
+	mon/MDSMonitor.h \
+	mon/MonmapMonitor.h \
+	mon/MonCap.h \
+	mon/MonClient.h \
+	mon/MonCommands.h \
+	mon/DumplingMonCommands.h \
+	mon/MonMap.h \
+	mon/Monitor.h \
+	mon/MonitorStore.h \
+	mon/MonitorDBStore.h \
+	mon/OSDMonitor.h \
+	mon/PGMap.h \
+	mon/PGMonitor.h \
+	mon/Paxos.h \
+	mon/PaxosService.h \
+	mon/QuorumService.h \
+	mon/Session.h \
+	mon/mon_types.h
+
diff --git a/ceph/src/mon/MonCap.cc b/ceph/src/mon/MonCap.cc
new file mode 100644
index 00000000..e8d3f7e8
--- /dev/null
+++ b/ceph/src/mon/MonCap.cc
@@ -0,0 +1,450 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "MonCap.h"
+#include "include/stringify.h"
+#include "common/config.h"
+#include "common/debug.h"
+#include "common/Formatter.h"
+
+#include 
+
+static inline bool is_not_alnum_space(char c)
+{
+  return !(isalpha(c) || isdigit(c) || (c == '-') || (c == '_'));
+}
+
+static string maybe_quote_string(const std::string& str)
+{
+  if (find_if(str.begin(), str.end(), is_not_alnum_space) == str.end())
+    return str;
+  return string("\"") + str + string("\"");
+}
+
+using std::ostream;
+using std::vector;
+
+#define dout_subsys ceph_subsys_mon
+
+ostream& operator<<(ostream& out, mon_rwxa_t p)
+{ 
+  if (p == MON_CAP_ANY)
+    return out << "*";
+
+  if (p & MON_CAP_R)
+    out << "r";
+  if (p & MON_CAP_W)
+    out << "w";
+  if (p & MON_CAP_X)
+    out << "x";
+  return out;
+}
+
+ostream& operator<<(ostream& out, const StringConstraint& c)
+{
+  if (c.prefix.length())
+    return out << "prefix " << c.prefix;
+  else
+    return out << "value " << c.value;
+}
+
+ostream& operator<<(ostream& out, const MonCapGrant& m)
+{
+  out << "allow";
+  if (m.service.length()) {
+    out << " service " << maybe_quote_string(m.service);
+  }
+  if (m.command.length()) {
+    out << " command " << maybe_quote_string(m.command);
+    if (!m.command_args.empty()) {
+      out << " with";
+      for (map::const_iterator p = m.command_args.begin();
+	   p != m.command_args.end();
+	   ++p) {
+	if (p->second.value.length())
+	  out << " " << maybe_quote_string(p->first) << "=" << maybe_quote_string(p->second.value);
+	else
+	  out << " " << maybe_quote_string(p->first) << " prefix " << maybe_quote_string(p->second.prefix);
+      }
+    }
+  }
+  if (m.profile.length()) {
+    out << " profile " << maybe_quote_string(m.profile);
+  }
+  if (m.allow != 0)
+    out << " " << m.allow;
+  return out;
+}
+
+
+// 
+//  fusion lets us easily populate structs via the qi parser.
+
+typedef map kvmap;
+
+BOOST_FUSION_ADAPT_STRUCT(MonCapGrant,
+			  (std::string, service)
+			  (std::string, profile)
+			  (std::string, command)
+			  (kvmap, command_args)
+			  (mon_rwxa_t, allow))
+
+BOOST_FUSION_ADAPT_STRUCT(StringConstraint,
+			  (std::string, value)
+			  (std::string, prefix))
+
+// 
+
+void MonCapGrant::expand_profile(entity_name_t name) const
+{
+  // only generate this list once
+  if (!profile_grants.empty())
+    return;
+
+  if (profile == "mon") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_ALL));
+    profile_grants.push_back(MonCapGrant("log", MON_CAP_ALL));
+  }
+  if (profile == "osd") {
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_ALL));
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("log", MON_CAP_W));
+  }
+  if (profile == "mds") {
+    profile_grants.push_back(MonCapGrant("mds", MON_CAP_ALL));
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("log", MON_CAP_W));
+  }
+  if (profile == "osd" || profile == "mds" || profile == "mon") {
+    string prefix = string("daemon-private/") + stringify(name) + string("/");
+    profile_grants.push_back(MonCapGrant("config-key get", "key", StringConstraint("", prefix)));
+    profile_grants.push_back(MonCapGrant("config-key put", "key", StringConstraint("", prefix)));
+    profile_grants.push_back(MonCapGrant("config-key exists", "key", StringConstraint("", prefix)));
+    profile_grants.push_back(MonCapGrant("config-key delete", "key", StringConstraint("", prefix)));
+  }
+  if (profile == "bootstrap-osd") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));  // read monmap
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));  // read osdmap
+    profile_grants.push_back(MonCapGrant("mon getmap"));
+    profile_grants.push_back(MonCapGrant("osd create"));
+    profile_grants.push_back(MonCapGrant("auth add"));
+    profile_grants.back().command_args["entity"] = StringConstraint("", "osd.");
+    profile_grants.back().command_args["caps_mon"] = StringConstraint("allow profile osd", "");
+    profile_grants.back().command_args["caps_osd"] = StringConstraint("allow *", "");
+  }
+  if (profile == "bootstrap-mds") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));  // read monmap
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));  // read osdmap
+    profile_grants.push_back(MonCapGrant("mon getmap"));
+    profile_grants.push_back(MonCapGrant("auth get-or-create"));  // FIXME: this can expose other mds keys
+    profile_grants.back().command_args["entity"] = StringConstraint("", "mds.");
+    profile_grants.back().command_args["caps_mon"] = StringConstraint("allow profile mds", "");
+    profile_grants.back().command_args["caps_osd"] = StringConstraint("allow rwx", "");
+    profile_grants.back().command_args["caps_mds"] = StringConstraint("allow", "");
+  }
+  if (profile == "fs-client") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("mds", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
+  }
+  if (profile == "simple-rados-client") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
+  }
+}
+
+mon_rwxa_t MonCapGrant::get_allowed(CephContext *cct,
+				    entity_name_t name,
+				    const std::string& s, const std::string& c,
+				    const map& c_args) const
+{
+  if (profile.length()) {
+    expand_profile(name);
+    mon_rwxa_t a;
+    for (list::const_iterator p = profile_grants.begin();
+	 p != profile_grants.end(); ++p)
+      a = a | p->get_allowed(cct, name, s, c, c_args);
+    return a;
+  }
+  if (service.length()) {
+    if (service != s)
+      return 0;
+    return allow;
+  }
+  if (command.length()) {
+    if (command != c)
+      return 0;
+    for (map::const_iterator p = command_args.begin(); p != command_args.end(); ++p) {
+      map::const_iterator q = c_args.find(p->first);
+      // argument must be present if a constraint exists
+      if (q == c_args.end())
+	return 0;
+      if (p->second.value.length()) {
+	// match value
+	if (p->second.value != q->second)
+	  return 0;
+      } else {
+	// match prefix
+	if (q->second.find(p->second.prefix) != 0)
+	  return 0;
+      }
+    }
+    return MON_CAP_ALL;
+  }
+  return allow;
+}
+
+ostream& operator<<(ostream&out, const MonCap& m)
+{
+  for (vector::const_iterator p = m.grants.begin(); p != m.grants.end(); ++p) {
+    if (p != m.grants.begin())
+      out << ", ";
+    out << *p;
+  }
+  return out;
+}
+
+bool MonCap::is_allow_all() const
+{
+  for (vector::const_iterator p = grants.begin(); p != grants.end(); ++p)
+    if (p->is_allow_all())
+      return true;
+  return false;
+}
+
+void MonCap::set_allow_all()
+{
+  grants.clear();
+  grants.push_back(MonCapGrant(MON_CAP_ANY));
+  text = "allow *";
+}
+
+bool MonCap::is_capable(CephContext *cct,
+			entity_name_t name,
+			const string& service,
+			const string& command, const map& command_args,
+			bool op_may_read, bool op_may_write, bool op_may_exec) const
+{
+  if (cct)
+    ldout(cct, 20) << "is_capable service=" << service << " command=" << command
+		   << (op_may_read ? " read":"")
+		   << (op_may_write ? " write":"")
+		   << (op_may_exec ? " exec":"")
+		   << " on cap " << *this
+		   << dendl;
+  mon_rwxa_t allow = 0;
+  for (vector::const_iterator p = grants.begin();
+       p != grants.end(); ++p) {
+    if (cct)
+      ldout(cct, 20) << " allow so far " << allow << ", doing grant " << *p << dendl;
+
+    if (p->is_allow_all()) {
+      if (cct)
+	ldout(cct, 20) << " allow all" << dendl;
+      return true;
+    }
+
+    // check enumerated caps
+    allow = allow | p->get_allowed(cct, name, service, command, command_args);
+    if ((!op_may_read || (allow & MON_CAP_R)) &&
+	(!op_may_write || (allow & MON_CAP_W)) &&
+	(!op_may_exec || (allow & MON_CAP_X))) {
+      if (cct)
+	ldout(cct, 20) << " match" << dendl;
+      return true;
+    }
+  }
+  return false;
+}
+
+void MonCap::encode(bufferlist& bl) const
+{
+  ENCODE_START(4, 4, bl);   // legacy MonCaps was 3, 3
+  ::encode(text, bl);
+  ENCODE_FINISH(bl);
+}
+
+void MonCap::decode(bufferlist::iterator& bl)
+{
+  string s;
+  DECODE_START(4, bl);
+  ::decode(s, bl);
+  DECODE_FINISH(bl);
+  parse(s, NULL);
+}
+
+void MonCap::dump(Formatter *f) const
+{
+  f->dump_string("text", text);
+}
+
+void MonCap::generate_test_instances(list& ls)
+{
+  ls.push_back(new MonCap);
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow *");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow rwx");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow service foo x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow command bar x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow service foo r, allow command bar x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow command bar with k1=v1 x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow command bar with k1=v1 k2=v2 x");
+}
+
+// grammar
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::ascii;
+namespace phoenix = boost::phoenix;
+
+
+template 
+struct MonCapParser : qi::grammar
+{
+  MonCapParser() : MonCapParser::base_type(moncap)
+  {
+    using qi::char_;
+    using qi::int_;
+    using qi::ulong_long;
+    using qi::lexeme;
+    using qi::alnum;
+    using qi::_val;
+    using qi::_1;
+    using qi::_2;
+    using qi::_3;
+    using qi::eps;
+    using qi::lit;
+
+    quoted_string %=
+      lexeme['"' >> +(char_ - '"') >> '"'] | 
+      lexeme['\'' >> +(char_ - '\'') >> '\''];
+    unquoted_word %= +char_("a-zA-Z0-9_.-");
+    str %= quoted_string | unquoted_word;
+
+    spaces = +(lit(' ') | lit('\n') | lit('\t'));
+
+    // command := command[=]cmd [k1=v1 k2=v2 ...]
+    str_match = '=' >> str >> qi::attr(string());
+    str_prefix = spaces >> lit("prefix") >> spaces >> qi::attr(string()) >> str;
+    kv_pair = str >> (str_match | str_prefix);
+    kv_map %= kv_pair >> *(spaces >> kv_pair);
+    command_match = -spaces >> lit("allow") >> spaces >> lit("command") >> (lit('=') | spaces)
+			    >> qi::attr(string()) >> qi::attr(string())
+			    >> str
+			    >> -(spaces >> lit("with") >> spaces >> kv_map)
+			    >> qi::attr(0);
+
+    // service foo rwxa
+    service_match %= -spaces >> lit("allow") >> spaces >> lit("service") >> (lit('=') | spaces)
+			     >> str >> qi::attr(string()) >> qi::attr(string())
+			     >> qi::attr(map())
+                             >> spaces >> rwxa;
+
+    // profile foo
+    profile_match %= -spaces >> lit("allow") >> spaces >> lit("profile") >> (lit('=') | spaces)
+			     >> qi::attr(string())
+			     >> str
+			     >> qi::attr(string())
+			     >> qi::attr(map())
+			     >> qi::attr(0);
+
+    // rwxa
+    rwxa_match %= -spaces >> lit("allow") >> spaces
+			  >> qi::attr(string()) >> qi::attr(string()) >> qi::attr(string())
+			  >> qi::attr(map())
+			  >> rwxa;
+
+    // rwxa := * | [r][w][x]
+    rwxa =
+      (lit("*")[_val = MON_CAP_ANY]) |
+      ( eps[_val = 0] >>
+	( lit('r')[_val |= MON_CAP_R] ||
+	  lit('w')[_val |= MON_CAP_W] ||
+	  lit('x')[_val |= MON_CAP_X]
+	  )
+	);
+
+    // grant := allow ...
+    grant = -spaces >> (rwxa_match | profile_match | service_match | command_match) >> -spaces;
+
+    // moncap := grant [grant ...]
+    grants %= (grant % (*lit(' ') >> (lit(';') | lit(',')) >> *lit(' ')));
+    moncap = grants  [_val = phoenix::construct(_1)]; 
+
+  }
+  qi::rule spaces;
+  qi::rule rwxa;
+  qi::rule quoted_string;
+  qi::rule unquoted_word;
+  qi::rule str;
+
+  qi::rule str_match, str_prefix;
+  qi::rule()> kv_pair;
+  qi::rule()> kv_map;
+
+  qi::rule rwxa_match;
+  qi::rule command_match;
+  qi::rule service_match;
+  qi::rule profile_match;
+  qi::rule grant;
+  qi::rule()> grants;
+  qi::rule moncap;
+};
+
+bool MonCap::parse(const string& str, ostream *err)
+{
+  string s = str;
+  string::iterator iter = s.begin();
+  string::iterator end = s.end();
+
+  MonCapParser g;
+  bool r = qi::parse(iter, end, g, *this);
+  //MonCapGrant foo;
+  //bool r = qi::phrase_parse(iter, end, g, ascii::space, foo);
+  if (r && iter == end) {
+    text = str;
+    return true;
+  }
+
+  // Make sure no grants are kept after parsing failed!
+  grants.clear();
+
+  if (err) {
+    if (iter != end)
+      *err << "moncap parse failed, stopped at '" << std::string(iter, end)
+	   << "' of '" << str << "'\n";
+    else
+      *err << "moncap parse failed, stopped at end of '" << str << "'\n";
+  }
+
+  return false; 
+}
+
diff --git a/ceph/src/mon/MonCap.h b/ceph/src/mon/MonCap.h
new file mode 100644
index 00000000..3e4eda8e
--- /dev/null
+++ b/ceph/src/mon/MonCap.h
@@ -0,0 +1,160 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_MONCAP_H
+#define CEPH_MONCAP_H
+
+#include 
+using std::ostream;
+
+#include "include/types.h"
+#include "msg/msg_types.h"
+
+class CephContext;
+
+static const __u8 MON_CAP_R     = (1 << 1);      // read
+static const __u8 MON_CAP_W     = (1 << 2);      // write
+static const __u8 MON_CAP_X     = (1 << 3);      // execute
+static const __u8 MON_CAP_ALL   = MON_CAP_R | MON_CAP_W | MON_CAP_X;
+static const __u8 MON_CAP_ANY   = 0xff;          // *
+
+struct mon_rwxa_t {
+  __u8 val;
+
+  mon_rwxa_t(__u8 v = 0) : val(v) {}
+  mon_rwxa_t& operator=(__u8 v) {
+    val = v;
+    return *this;
+  }
+  operator __u8() const {
+    return val;
+  }
+};
+
+ostream& operator<<(ostream& out, mon_rwxa_t p);
+
+struct StringConstraint {
+  string value;
+  string prefix;
+
+  StringConstraint() {}
+  StringConstraint(string a, string b) : value(a), prefix(b) {}
+};
+
+ostream& operator<<(ostream& out, const StringConstraint& c);
+
+struct MonCapGrant {
+  /*
+   * A grant can come in one of four forms:
+   *
+   *  - a blanket allow ('allow rw', 'allow *')
+   *    - this will match against any service and the read/write/exec flags
+   *      in the mon code.  semantics of what X means are somewhat ad hoc.
+   *
+   *  - a service allow ('allow service mds rw')
+   *    - this will match against a specific service and the r/w/x flags.
+   *
+   *  - a profile ('allow profile osd')
+   *    - this will match against specific monitor-enforced semantics of what
+   *      this type of user should need to do.  examples include 'osd', 'mds',
+   *      'bootstrap-osd'.
+   *
+   *  - a command ('allow command foo', 'allow command bar with arg1=val1 arg2 prefix val2')
+   *      this includes the command name (the prefix string), and a set
+   *      of key/value pairs that constrain use of that command.  if no pairs
+   *      are specified, any arguments are allowed; if a pair is specified, that
+   *      argument must be present and equal or match a prefix.
+   */
+  std::string service;
+  std::string profile;
+  std::string command;
+  map command_args;
+
+  mon_rwxa_t allow;
+
+  // explicit grants that a profile grant expands to; populated as
+  // needed by expand_profile() (via is_match()) and cached here.
+  mutable list profile_grants;
+
+  void expand_profile(entity_name_t name) const;
+
+  MonCapGrant() : allow(0) {}
+  MonCapGrant(mon_rwxa_t a) : allow(a) {}
+  MonCapGrant(string s, mon_rwxa_t a) : service(s), allow(a) {}
+  MonCapGrant(string c) : command(c) {}
+  MonCapGrant(string c, string a, StringConstraint co) : command(c) {
+    command_args[a] = co;
+  }
+
+  /**
+   * check if given request parameters match our constraints
+   *
+   * @param cct context
+   * @param name entity name
+   * @param service service (if any)
+   * @param command command (if any)
+   * @param command_args command args (if any)
+   * @return bits we allow
+   */
+  mon_rwxa_t get_allowed(CephContext *cct,
+			 entity_name_t name,
+			 const std::string& service,
+			 const std::string& command,
+			 const map& command_args) const;
+
+  bool is_allow_all() const {
+    return
+      allow == MON_CAP_ANY &&
+      service.length() == 0 &&
+      profile.length() == 0 &&
+      command.length() == 0;
+  }
+};
+
+ostream& operator<<(ostream& out, const MonCapGrant& g);
+
+struct MonCap {
+  string text;
+  std::vector grants;
+
+  MonCap() {}
+  MonCap(std::vector g) : grants(g) {}
+
+  string get_str() const {
+    return text;
+  }
+
+  bool is_allow_all() const;
+  void set_allow_all();
+  bool parse(const std::string& str, ostream *err=NULL);
+
+  /**
+   * check if we are capable of something
+   *
+   * This method actually checks a description of a particular operation against
+   * what the capability has specified.
+   *
+   * @param service service name
+   * @param command command id
+   * @param command_args
+   * @param op_may_read whether the operation may need to read
+   * @param op_may_write whether the operation may need to write
+   * @param op_may_exec whether the operation may exec
+   * @return true if the operation is allowed, false otherwise
+   */
+  bool is_capable(CephContext *cct,
+		  entity_name_t name,
+		  const string& service,
+		  const string& command, const map& command_args,
+		  bool op_may_read, bool op_may_write, bool op_may_exec) const;
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& ls);
+};
+WRITE_CLASS_ENCODER(MonCap)
+
+ostream& operator<<(ostream& out, const MonCap& cap);
+
+#endif
diff --git a/ceph/src/mon/MonClient.cc b/ceph/src/mon/MonClient.cc
new file mode 100644
index 00000000..3a6dda46
--- /dev/null
+++ b/ceph/src/mon/MonClient.cc
@@ -0,0 +1,1048 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "msg/SimpleMessenger.h"
+#include "messages/MMonGetMap.h"
+#include "messages/MMonGetVersion.h"
+#include "messages/MMonGetVersionReply.h"
+#include "messages/MMonMap.h"
+#include "messages/MAuth.h"
+#include "messages/MLogAck.h"
+#include "messages/MAuthReply.h"
+#include "messages/MMonCommand.h"
+#include "messages/MMonCommandAck.h"
+#include "messages/MPing.h"
+
+#include "messages/MMonSubscribe.h"
+#include "messages/MMonSubscribeAck.h"
+#include "common/ConfUtils.h"
+#include "common/ceph_argparse.h"
+#include "common/errno.h"
+#include "common/LogClient.h"
+
+#include "MonClient.h"
+#include "MonMap.h"
+
+#include "auth/Auth.h"
+#include "auth/KeyRing.h"
+#include "auth/AuthMethodList.h"
+
+#include "include/str_list.h"
+#include "include/addr_parsing.h"
+
+#include "common/config.h"
+
+
+#define dout_subsys ceph_subsys_monc
+#undef dout_prefix
+#define dout_prefix *_dout << "monclient" << (hunting ? "(hunting)":"") << ": "
+
+MonClient::MonClient(CephContext *cct_) :
+  Dispatcher(cct_),
+  state(MC_STATE_NONE),
+  messenger(NULL),
+  cur_con(NULL),
+  rng(getpid()),
+  monc_lock("MonClient::monc_lock"),
+  timer(cct_, monc_lock), finisher(cct_),
+  authorize_handler_registry(NULL),
+  initialized(false),
+  no_keyring_disabled_cephx(false),
+  log_client(NULL),
+  more_log_pending(false),
+  auth_supported(NULL),
+  hunting(true),
+  want_monmap(true),
+  want_keys(0), global_id(0),
+  authenticate_err(0),
+  session_established_context(NULL),
+  had_a_connection(false),
+  reopen_interval_multiplier(1.0),
+  auth(NULL),
+  keyring(NULL),
+  rotating_secrets(NULL),
+  last_mon_command_tid(0),
+  version_req_id(0)
+{
+}
+
+MonClient::~MonClient()
+{
+  delete auth_supported;
+  delete session_established_context;
+  delete auth;
+  delete keyring;
+  delete rotating_secrets;
+}
+
+int MonClient::build_initial_monmap()
+{
+  ldout(cct, 10) << "build_initial_monmap" << dendl;
+  return monmap.build_initial(cct, cerr);
+}
+
+int MonClient::get_monmap()
+{
+  ldout(cct, 10) << "get_monmap" << dendl;
+  Mutex::Locker l(monc_lock);
+  
+  _sub_want("monmap", 0, 0);
+  if (cur_mon.empty())
+    _reopen_session();
+
+  while (want_monmap)
+    map_cond.Wait(monc_lock);
+
+  ldout(cct, 10) << "get_monmap done" << dendl;
+  return 0;
+}
+
+int MonClient::get_monmap_privately()
+{
+  ldout(cct, 10) << "get_monmap_privately" << dendl;
+  Mutex::Locker l(monc_lock);
+
+  bool temp_msgr = false;
+  SimpleMessenger* smessenger = NULL;
+  if (!messenger) {
+    messenger = smessenger = new SimpleMessenger(cct,
+                                                 entity_name_t::CLIENT(-1),
+                                                 "temp_mon_client", getpid());
+    messenger->add_dispatcher_head(this);
+    smessenger->start();
+    temp_msgr = true;
+  }
+
+  int attempt = 10;
+
+  ldout(cct, 10) << "have " << monmap.epoch << " fsid " << monmap.fsid << dendl;
+
+  while (monmap.fsid.is_zero()) {
+    cur_mon = _pick_random_mon();
+    cur_con = messenger->get_connection(monmap.get_inst(cur_mon));
+    ldout(cct, 10) << "querying mon." << cur_mon << " " << cur_con->get_peer_addr() << dendl;
+    messenger->send_message(new MMonGetMap, cur_con);
+
+    if (--attempt == 0)
+      break;
+
+    utime_t interval;
+    interval.set_from_double(cct->_conf->mon_client_hunt_interval);
+    map_cond.WaitInterval(cct, monc_lock, interval);
+
+    if (monmap.fsid.is_zero()) {
+      messenger->mark_down(cur_con);  // nope, clean that connection up
+    }
+  }
+
+  if (temp_msgr) {
+    messenger->mark_down(cur_con);
+    cur_con.reset(NULL);
+    monc_lock.Unlock();
+    messenger->shutdown();
+    if (smessenger)
+      smessenger->wait();
+    delete messenger;
+    messenger = 0;
+    monc_lock.Lock();
+  }
+
+  hunting = true;  // reset this to true!
+  cur_mon.clear();
+
+  cur_con.reset(NULL);
+
+  if (!monmap.fsid.is_zero())
+    return 0;
+  return -1;
+}
+
+
+/**
+ * Ping the monitor with id @p mon_id and set the resulting reply in
+ * the provided @p result_reply, if this last parameter is not NULL.
+ *
+ * So that we don't rely on the MonClient's default messenger, set up
+ * during connect(), we create our own messenger to comunicate with the
+ * specified monitor.  This is advantageous in the following ways:
+ *
+ * - Isolate the ping procedure from the rest of the MonClient's operations,
+ *   allowing us to not acquire or manage the big monc_lock, thus not
+ *   having to block waiting for some other operation to finish before we
+ *   can proceed.
+ *   * for instance, we can ping mon.FOO even if we are currently hunting
+ *     or blocked waiting for auth to complete with mon.BAR.
+ *
+ * - Ping a monitor prior to establishing a connection (using connect())
+ *   and properly establish the MonClient's messenger.  This frees us
+ *   from dealing with the complex foo that happens in connect().
+ *
+ * We also don't rely on MonClient as a dispatcher for this messenger,
+ * unlike what happens with the MonClient's default messenger.  This allows
+ * us to sandbox the whole ping, having it much as a separate entity in
+ * the MonClient class, considerably simplifying the handling and dispatching
+ * of messages without needing to consider monc_lock.
+ *
+ * Current drawback is that we will establish a messenger for each ping
+ * we want to issue, instead of keeping a single messenger instance that
+ * would be used for all pings.
+ */
+int MonClient::ping_monitor(const string &mon_id, string *result_reply)
+{
+  ldout(cct, 10) << __func__ << dendl;
+
+  if (mon_id.empty()) {
+    ldout(cct, 10) << __func__ << " specified mon id is empty!" << dendl;
+    return -EINVAL;
+  } else if (!monmap.contains(mon_id)) {
+    ldout(cct, 10) << __func__ << " no such monitor 'mon." << mon_id << "'"
+                   << dendl;
+    return -ENOENT;
+  }
+
+  MonClientPinger *pinger = new MonClientPinger(cct, result_reply);
+
+  Messenger *smsgr = new SimpleMessenger(cct,
+                                         entity_name_t::CLIENT(-1),
+                                         "temp_ping_client", getpid());
+  smsgr->add_dispatcher_head(pinger);
+  smsgr->start();
+
+  ConnectionRef con = smsgr->get_connection(monmap.get_inst(mon_id));
+  ldout(cct, 10) << __func__ << " ping mon." << mon_id
+                 << " " << con->get_peer_addr() << dendl;
+  smsgr->send_message(new MPing, con);
+
+  pinger->lock.Lock();
+  int ret = pinger->wait_for_reply(cct->_conf->client_mount_timeout);
+  if (ret == 0) {
+    ldout(cct,10) << __func__ << " got ping reply" << dendl;
+  } else {
+    ret = -ret;
+  }
+  pinger->lock.Unlock();
+
+  smsgr->mark_down(con);
+  smsgr->shutdown();
+  smsgr->wait();
+  delete smsgr;
+  delete pinger;
+  return ret;
+}
+
+bool MonClient::ms_dispatch(Message *m)
+{
+  if (my_addr == entity_addr_t())
+    my_addr = messenger->get_myaddr();
+
+  // we only care about these message types
+  switch (m->get_type()) {
+  case CEPH_MSG_MON_MAP:
+  case CEPH_MSG_AUTH_REPLY:
+  case CEPH_MSG_MON_SUBSCRIBE_ACK:
+  case CEPH_MSG_MON_GET_VERSION_REPLY:
+  case MSG_MON_COMMAND_ACK:
+  case MSG_LOGACK:
+    break;
+  default:
+    return false;
+  }
+
+  Mutex::Locker lock(monc_lock);
+
+  // ignore any messages outside our current session
+  if (m->get_connection() != cur_con) {
+    ldout(cct, 10) << "discarding stray monitor message " << *m << dendl;
+    m->put();
+    return true;
+  }
+
+  switch (m->get_type()) {
+  case CEPH_MSG_MON_MAP:
+    handle_monmap(static_cast(m));
+    break;
+  case CEPH_MSG_AUTH_REPLY:
+    handle_auth(static_cast(m));
+    break;
+  case CEPH_MSG_MON_SUBSCRIBE_ACK:
+    handle_subscribe_ack(static_cast(m));
+    break;
+  case CEPH_MSG_MON_GET_VERSION_REPLY:
+    handle_get_version_reply(static_cast(m));
+    break;
+  case MSG_MON_COMMAND_ACK:
+    handle_mon_command_ack(static_cast(m));
+    break;
+  case MSG_LOGACK:
+    if (log_client) {
+      log_client->handle_log_ack(static_cast(m));
+      if (more_log_pending) {
+	send_log();
+      }
+    } else {
+      m->put();
+    }
+    break;
+  }
+  return true;
+}
+
+void MonClient::send_log()
+{
+  if (log_client) {
+    Message *lm = log_client->get_mon_log_message();
+    if (lm)
+      _send_mon_message(lm);
+    more_log_pending = log_client->are_pending();
+  }
+}
+
+void MonClient::handle_monmap(MMonMap *m)
+{
+  ldout(cct, 10) << "handle_monmap " << *m << dendl;
+  bufferlist::iterator p = m->monmapbl.begin();
+  ::decode(monmap, p);
+
+  assert(!cur_mon.empty());
+  ldout(cct, 10) << " got monmap " << monmap.epoch
+		 << ", mon." << cur_mon << " is now rank " << monmap.get_rank(cur_mon)
+		 << dendl;
+  ldout(cct, 10) << "dump:\n";
+  monmap.print(*_dout);
+  *_dout << dendl;
+
+  _sub_got("monmap", monmap.get_epoch());
+
+  if (!monmap.get_addr_name(cur_con->get_peer_addr(), cur_mon)) {
+    ldout(cct, 10) << "mon." << cur_mon << " went away" << dendl;
+    _reopen_session();  // can't find the mon we were talking to (above)
+  }
+
+  map_cond.Signal();
+  want_monmap = false;
+
+  m->put();
+}
+
+// ----------------------
+
+int MonClient::init()
+{
+  ldout(cct, 10) << "init" << dendl;
+
+  messenger->add_dispatcher_head(this);
+
+  entity_name = cct->_conf->name;
+
+  Mutex::Locker l(monc_lock);
+
+  string method;
+    if (cct->_conf->auth_supported.length() != 0)
+      method = cct->_conf->auth_supported;
+    else if (entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
+             entity_name.get_type() == CEPH_ENTITY_TYPE_MDS ||
+             entity_name.get_type() == CEPH_ENTITY_TYPE_MON)
+      method = cct->_conf->auth_cluster_required;
+    else
+      method = cct->_conf->auth_client_required;
+  auth_supported = new AuthMethodList(cct, method);
+  ldout(cct, 10) << "auth_supported " << auth_supported->get_supported_set() << " method " << method << dendl;
+
+  int r = 0;
+  keyring = new KeyRing; // initializing keyring anyway
+
+  if (auth_supported->is_supported_auth(CEPH_AUTH_CEPHX)) {
+    r = keyring->from_ceph_context(cct);
+    if (r == -ENOENT) {
+      auth_supported->remove_supported_auth(CEPH_AUTH_CEPHX);
+      if (auth_supported->get_supported_set().size() > 0) {
+	r = 0;
+	no_keyring_disabled_cephx = true;
+      } else {
+	lderr(cct) << "ERROR: missing keyring, cannot use cephx for authentication" << dendl;
+      }
+    }
+  }
+
+  if (r < 0) {
+    return r;
+  }
+
+  rotating_secrets = new RotatingKeyRing(cct, cct->get_module_type(), keyring);
+
+  initialized = true;
+
+  timer.init();
+  finisher.start();
+  schedule_tick();
+
+  return 0;
+}
+
+void MonClient::shutdown()
+{
+  ldout(cct, 10) << __func__ << "shutdown" << dendl;
+  monc_lock.Lock();
+  while (!version_requests.empty()) {
+    version_requests.begin()->second->context->complete(-ECANCELED);
+    ldout(cct, 20) << __func__ << " canceling and discarding version request "
+		   << version_requests.begin()->second << dendl;
+    delete version_requests.begin()->second;
+    version_requests.erase(version_requests.begin());
+  }
+
+  while (!waiting_for_session.empty()) {
+    ldout(cct, 20) << __func__ << " discarding pending message " << *waiting_for_session.front() << dendl;
+    waiting_for_session.front()->put();
+    waiting_for_session.pop_front();
+  }
+
+  monc_lock.Unlock();
+
+  if (initialized) {
+    finisher.stop();
+  }
+  monc_lock.Lock();
+  timer.shutdown();
+
+  messenger->mark_down(cur_con);
+  cur_con.reset(NULL);
+
+  monc_lock.Unlock();
+}
+
+int MonClient::authenticate(double timeout)
+{
+  Mutex::Locker lock(monc_lock);
+
+  if (state == MC_STATE_HAVE_SESSION) {
+    ldout(cct, 5) << "already authenticated" << dendl;;
+    return 0;
+  }
+
+  _sub_want("monmap", monmap.get_epoch() ? monmap.get_epoch() + 1 : 0, 0);
+  if (cur_mon.empty())
+    _reopen_session();
+
+  utime_t until = ceph_clock_now(cct);
+  until += timeout;
+  if (timeout > 0.0)
+    ldout(cct, 10) << "authenticate will time out at " << until << dendl;
+  while (state != MC_STATE_HAVE_SESSION && !authenticate_err) {
+    if (timeout > 0.0) {
+      int r = auth_cond.WaitUntil(monc_lock, until);
+      if (r == ETIMEDOUT) {
+	ldout(cct, 0) << "authenticate timed out after " << timeout << dendl;
+	authenticate_err = -r;
+      }
+    } else {
+      auth_cond.Wait(monc_lock);
+    }
+  }
+
+  if (state == MC_STATE_HAVE_SESSION) {
+    ldout(cct, 5) << "authenticate success, global_id " << global_id << dendl;
+  }
+
+  if (authenticate_err < 0 && no_keyring_disabled_cephx) {
+    lderr(cct) << "authenticate NOTE: no keyring found; disabled cephx authentication" << dendl;
+  }
+
+  return authenticate_err;
+}
+
+void MonClient::handle_auth(MAuthReply *m)
+{
+  Context *cb = NULL;
+  bufferlist::iterator p = m->result_bl.begin();
+  if (state == MC_STATE_NEGOTIATING) {
+    if (!auth || (int)m->protocol != auth->get_protocol()) {
+      delete auth;
+      auth = get_auth_client_handler(cct, m->protocol, rotating_secrets);
+      if (!auth) {
+	ldout(cct, 10) << "no handler for protocol " << m->protocol << dendl;
+	if (m->result == -ENOTSUP) {
+	  ldout(cct, 10) << "none of our auth protocols are supported by the server"
+			 << dendl;
+	  authenticate_err = m->result;
+	  auth_cond.SignalAll();
+	}
+	m->put();
+	return;
+      }
+      auth->set_want_keys(want_keys);
+      auth->init(entity_name);
+      auth->set_global_id(global_id);
+    } else {
+      auth->reset();
+    }
+    state = MC_STATE_AUTHENTICATING;
+  }
+  assert(auth);
+  if (m->global_id && m->global_id != global_id) {
+    global_id = m->global_id;
+    auth->set_global_id(global_id);
+    ldout(cct, 10) << "my global_id is " << m->global_id << dendl;
+  }
+
+  int ret = auth->handle_response(m->result, p);
+  m->put();
+
+  if (ret == -EAGAIN) {
+    MAuth *ma = new MAuth;
+    ma->protocol = auth->get_protocol();
+    auth->prepare_build_request();
+    ret = auth->build_request(ma->auth_payload);
+    _send_mon_message(ma, true);
+    return;
+  }
+
+  _finish_hunting();
+
+  authenticate_err = ret;
+  if (ret == 0) {
+    if (state != MC_STATE_HAVE_SESSION) {
+      state = MC_STATE_HAVE_SESSION;
+      while (!waiting_for_session.empty()) {
+	_send_mon_message(waiting_for_session.front());
+	waiting_for_session.pop_front();
+      }
+
+      _resend_mon_commands();
+
+      if (log_client) {
+	log_client->reset_session();
+	send_log();
+      }
+      if (session_established_context) {
+        cb = session_established_context;
+        session_established_context = NULL;
+      }
+    }
+  
+    _check_auth_tickets();
+  }
+  auth_cond.SignalAll();
+  if (cb) {
+    monc_lock.Unlock();
+    cb->complete(0);
+    monc_lock.Lock();
+  }
+}
+
+
+// ---------
+
+void MonClient::_send_mon_message(Message *m, bool force)
+{
+  assert(monc_lock.is_locked());
+  assert(!cur_mon.empty());
+  if (force || state == MC_STATE_HAVE_SESSION) {
+    assert(cur_con);
+    ldout(cct, 10) << "_send_mon_message to mon." << cur_mon
+		   << " at " << cur_con->get_peer_addr() << dendl;
+    messenger->send_message(m, cur_con);
+  } else {
+    waiting_for_session.push_back(m);
+  }
+}
+
+string MonClient::_pick_random_mon()
+{
+  assert(monmap.size() > 0);
+  if (monmap.size() == 1) {
+    return monmap.get_name(0);
+  } else {
+    int max = monmap.size();
+    int o = -1;
+    if (!cur_mon.empty()) {
+      o = monmap.get_rank(cur_mon);
+      if (o >= 0)
+	max--;
+    }
+
+    int32_t n = rng() % max;
+    if (o >= 0 && n >= o)
+      n++;
+    return monmap.get_name(n);
+  }
+}
+
+void MonClient::_reopen_session(int rank, string name)
+{
+  assert(monc_lock.is_locked());
+  ldout(cct, 10) << "_reopen_session rank " << rank << " name " << name << dendl;
+
+  if (rank < 0 && name.length() == 0) {
+    cur_mon = _pick_random_mon();
+  } else if (name.length()) {
+    cur_mon = name;
+  } else {
+    cur_mon = monmap.get_name(rank);
+  }
+
+  if (cur_con) {
+    messenger->mark_down(cur_con);
+  }
+  cur_con = messenger->get_connection(monmap.get_inst(cur_mon));
+	
+  ldout(cct, 10) << "picked mon." << cur_mon << " con " << cur_con
+		 << " addr " << cur_con->get_peer_addr()
+		 << dendl;
+
+  // throw out old queued messages
+  while (!waiting_for_session.empty()) {
+    waiting_for_session.front()->put();
+    waiting_for_session.pop_front();
+  }
+
+  // throw out version check requests
+  while (!version_requests.empty()) {
+    finisher.queue(version_requests.begin()->second->context, -EAGAIN);
+    delete version_requests.begin()->second;
+    version_requests.erase(version_requests.begin());
+  }
+
+  // adjust timeouts if necessary
+  if (had_a_connection) {
+    reopen_interval_multiplier *= cct->_conf->mon_client_hunt_interval_backoff;
+    if (reopen_interval_multiplier >
+          cct->_conf->mon_client_hunt_interval_max_multiple)
+      reopen_interval_multiplier =
+          cct->_conf->mon_client_hunt_interval_max_multiple;
+  }
+
+  // restart authentication handshake
+  state = MC_STATE_NEGOTIATING;
+  hunting = true;
+
+  // send an initial keepalive to ensure our timestamp is valid by the
+  // time we are in an OPENED state (by sequencing this before
+  // authentication).
+  messenger->send_keepalive(cur_con.get());
+
+  MAuth *m = new MAuth;
+  m->protocol = 0;
+  m->monmap_epoch = monmap.get_epoch();
+  __u8 struct_v = 1;
+  ::encode(struct_v, m->auth_payload);
+  ::encode(auth_supported->get_supported_set(), m->auth_payload);
+  ::encode(entity_name, m->auth_payload);
+  ::encode(global_id, m->auth_payload);
+  _send_mon_message(m, true);
+
+  if (!sub_have.empty())
+    _renew_subs();
+}
+
+
+bool MonClient::ms_handle_reset(Connection *con)
+{
+  Mutex::Locker lock(monc_lock);
+
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    if (cur_mon.empty() || con != cur_con) {
+      ldout(cct, 10) << "ms_handle_reset stray mon " << con->get_peer_addr() << dendl;
+      return true;
+    } else {
+      ldout(cct, 10) << "ms_handle_reset current mon " << con->get_peer_addr() << dendl;
+      if (hunting)
+	return true;
+      
+      ldout(cct, 0) << "hunting for new mon" << dendl;
+      _reopen_session();
+    }
+  }
+  return false;
+}
+
+void MonClient::_finish_hunting()
+{
+  assert(monc_lock.is_locked());
+  if (hunting) {
+    ldout(cct, 1) << "found mon." << cur_mon << dendl; 
+    hunting = false;
+    had_a_connection = true;
+    reopen_interval_multiplier /= 2.0;
+    if (reopen_interval_multiplier < 1.0)
+      reopen_interval_multiplier = 1.0;
+  }
+}
+
+void MonClient::tick()
+{
+  ldout(cct, 10) << "tick" << dendl;
+
+  _check_auth_tickets();
+  
+  if (hunting) {
+    ldout(cct, 1) << "continuing hunt" << dendl;
+    _reopen_session();
+  } else if (!cur_mon.empty()) {
+    // just renew as needed
+    utime_t now = ceph_clock_now(cct);
+    ldout(cct, 10) << "renew subs? (now: " << now 
+		   << "; renew after: " << sub_renew_after << ") -- " 
+		   << (now > sub_renew_after ? "yes" : "no") 
+		   << dendl;
+    if (now > sub_renew_after)
+      _renew_subs();
+
+    messenger->send_keepalive(cur_con.get());
+
+    if (state == MC_STATE_HAVE_SESSION) {
+      send_log();
+
+      if (cct->_conf->mon_client_ping_timeout > 0 &&
+	  cur_con->has_feature(CEPH_FEATURE_MSGR_KEEPALIVE2)) {
+	utime_t lk = cur_con->get_last_keepalive_ack();
+	utime_t interval = ceph_clock_now(cct) - lk;
+	if (interval > cct->_conf->mon_client_ping_timeout) {
+	  ldout(cct, 1) << "no keepalive since " << lk << " (" << interval
+			<< " seconds), reconnecting" << dendl;
+	  _reopen_session();
+	}
+      }
+    }
+  }
+
+  schedule_tick();
+}
+
+void MonClient::schedule_tick()
+{
+  if (hunting)
+    timer.add_event_after(cct->_conf->mon_client_hunt_interval
+                          * reopen_interval_multiplier, new C_Tick(this));
+  else
+    timer.add_event_after(cct->_conf->mon_client_ping_interval, new C_Tick(this));
+}
+
+// ---------
+
+void MonClient::_renew_subs()
+{
+  assert(monc_lock.is_locked());
+  if (sub_have.empty()) {
+    ldout(cct, 10) << "renew_subs - empty" << dendl;
+    return;
+  }
+
+  ldout(cct, 10) << "renew_subs" << dendl;
+  if (cur_mon.empty())
+    _reopen_session();
+  else {
+    if (sub_renew_sent == utime_t())
+      sub_renew_sent = ceph_clock_now(cct);
+
+    MMonSubscribe *m = new MMonSubscribe;
+    m->what = sub_have;
+    _send_mon_message(m);
+  }
+}
+
+void MonClient::handle_subscribe_ack(MMonSubscribeAck *m)
+{
+  if (sub_renew_sent != utime_t()) {
+    sub_renew_after = sub_renew_sent;
+    sub_renew_after += m->interval / 2.0;
+    ldout(cct, 10) << "handle_subscribe_ack sent " << sub_renew_sent << " renew after " << sub_renew_after << dendl;
+    sub_renew_sent = utime_t();
+  } else {
+    ldout(cct, 10) << "handle_subscribe_ack sent " << sub_renew_sent << ", ignoring" << dendl;
+  }
+
+  m->put();
+}
+
+int MonClient::_check_auth_tickets()
+{
+  assert(monc_lock.is_locked());
+  if (state == MC_STATE_HAVE_SESSION && auth) {
+    if (auth->need_tickets()) {
+      ldout(cct, 10) << "_check_auth_tickets getting new tickets!" << dendl;
+      MAuth *m = new MAuth;
+      m->protocol = auth->get_protocol();
+      auth->prepare_build_request();
+      auth->build_request(m->auth_payload);
+      _send_mon_message(m);
+    }
+
+    _check_auth_rotating();
+  }
+  return 0;
+}
+
+int MonClient::_check_auth_rotating()
+{
+  assert(monc_lock.is_locked());
+  if (!rotating_secrets ||
+      !auth_principal_needs_rotating_keys(entity_name)) {
+    ldout(cct, 20) << "_check_auth_rotating not needed by " << entity_name << dendl;
+    return 0;
+  }
+
+  if (!auth || state != MC_STATE_HAVE_SESSION) {
+    ldout(cct, 10) << "_check_auth_rotating waiting for auth session" << dendl;
+    return 0;
+  }
+
+  utime_t cutoff = ceph_clock_now(cct);
+  cutoff -= MIN(30.0, cct->_conf->auth_service_ticket_ttl / 4.0);
+  if (!rotating_secrets->need_new_secrets(cutoff)) {
+    ldout(cct, 10) << "_check_auth_rotating have uptodate secrets (they expire after " << cutoff << ")" << dendl;
+    rotating_secrets->dump_rotating();
+    return 0;
+  }
+
+  ldout(cct, 10) << "_check_auth_rotating renewing rotating keys (they expired before " << cutoff << ")" << dendl;
+  MAuth *m = new MAuth;
+  m->protocol = auth->get_protocol();
+  if (auth->build_rotating_request(m->auth_payload)) {
+    _send_mon_message(m);
+  } else {
+    m->put();
+  }
+  return 0;
+}
+
+int MonClient::wait_auth_rotating(double timeout)
+{
+  Mutex::Locker l(monc_lock);
+  utime_t until = ceph_clock_now(cct);
+  until += timeout;
+
+  if (auth->get_protocol() == CEPH_AUTH_NONE)
+    return 0;
+  
+  if (!rotating_secrets)
+    return 0;
+
+  while (auth_principal_needs_rotating_keys(entity_name) &&
+	 rotating_secrets->need_new_secrets()) {
+    utime_t now = ceph_clock_now(cct);
+    if (now >= until) {
+      ldout(cct, 0) << "wait_auth_rotating timed out after " << timeout << dendl;
+      return -ETIMEDOUT;
+    }
+    ldout(cct, 10) << "wait_auth_rotating waiting (until " << until << ")" << dendl;
+    auth_cond.WaitUntil(monc_lock, until);
+  }
+  ldout(cct, 10) << "wait_auth_rotating done" << dendl;
+  return 0;
+}
+
+// ---------
+
+void MonClient::_send_command(MonCommand *r)
+{
+  if (r->target_rank >= 0 &&
+      r->target_rank != monmap.get_rank(cur_mon)) {
+    ldout(cct, 10) << "_send_command " << r->tid << " " << r->cmd
+		   << " wants rank " << r->target_rank
+		   << ", reopening session"
+		   << dendl;
+    if (r->target_rank >= (int)monmap.size()) {
+      ldout(cct, 10) << " target " << r->target_rank << " >= max mon " << monmap.size() << dendl;
+      _finish_command(r, -ENOENT, "mon rank dne");
+      return;
+    }
+    _reopen_session(r->target_rank, string());
+    return;
+  }
+
+  if (r->target_name.length() &&
+      r->target_name != cur_mon) {
+    ldout(cct, 10) << "_send_command " << r->tid << " " << r->cmd
+		   << " wants mon " << r->target_name
+		   << ", reopening session"
+		   << dendl;
+    if (!monmap.contains(r->target_name)) {
+      ldout(cct, 10) << " target " << r->target_name << " not present in monmap" << dendl;
+      _finish_command(r, -ENOENT, "mon dne");
+      return;
+    }
+    _reopen_session(-1, r->target_name);
+    return;
+  }
+
+  ldout(cct, 10) << "_send_command " << r->tid << " " << r->cmd << dendl;
+  MMonCommand *m = new MMonCommand(monmap.fsid);
+  m->set_tid(r->tid);
+  m->cmd = r->cmd;
+  m->set_data(r->inbl);
+  _send_mon_message(m);
+  return;
+}
+
+void MonClient::_resend_mon_commands()
+{
+  // resend any requests
+  for (map::iterator p = mon_commands.begin();
+       p != mon_commands.end();
+       ++p) {
+    _send_command(p->second);
+  }
+}
+
+void MonClient::handle_mon_command_ack(MMonCommandAck *ack)
+{
+  MonCommand *r = NULL;
+  uint64_t tid = ack->get_tid();
+
+  if (tid == 0 && !mon_commands.empty()) {
+    r = mon_commands.begin()->second;
+    ldout(cct, 10) << "handle_mon_command_ack has tid 0, assuming it is " << r->tid << dendl;
+  } else {
+    map::iterator p = mon_commands.find(tid);
+    if (p == mon_commands.end()) {
+      ldout(cct, 10) << "handle_mon_command_ack " << ack->get_tid() << " not found" << dendl;
+      ack->put();
+      return;
+    }
+    r = p->second;
+  }
+
+  ldout(cct, 10) << "handle_mon_command_ack " << r->tid << " " << r->cmd << dendl;
+  if (r->poutbl)
+    r->poutbl->claim(ack->get_data());
+  _finish_command(r, ack->r, ack->rs);
+  ack->put();
+}
+
+int MonClient::_cancel_mon_command(uint64_t tid, int r)
+{
+  assert(monc_lock.is_locked());
+
+  map::iterator it = mon_commands.find(tid);
+  if (it == mon_commands.end()) {
+    ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+    return -ENOENT;
+  }
+
+  ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+  MonCommand *cmd = it->second;
+  _finish_command(cmd, -ETIMEDOUT, "");
+  return 0;
+}
+
+void MonClient::_finish_command(MonCommand *r, int ret, string rs)
+{
+  ldout(cct, 10) << "_finish_command " << r->tid << " = " << ret << " " << rs << dendl;
+  if (r->prval)
+    *(r->prval) = ret;
+  if (r->prs)
+    *(r->prs) = rs;
+  if (r->onfinish)
+    finisher.queue(r->onfinish, ret);
+  mon_commands.erase(r->tid);
+  delete r;
+}
+
+int MonClient::start_mon_command(const vector& cmd,
+				 const bufferlist& inbl,
+				 bufferlist *outbl, string *outs,
+				 Context *onfinish)
+{
+  Mutex::Locker l(monc_lock);
+  MonCommand *r = new MonCommand(++last_mon_command_tid);
+  r->cmd = cmd;
+  r->inbl = inbl;
+  r->poutbl = outbl;
+  r->prs = outs;
+  r->onfinish = onfinish;
+  if (cct->_conf->rados_mon_op_timeout > 0) {
+    r->ontimeout = new C_CancelMonCommand(r->tid, this);
+    timer.add_event_after(cct->_conf->rados_mon_op_timeout, r->ontimeout);
+  }
+  mon_commands[r->tid] = r;
+  _send_command(r);
+  // can't fail
+  return 0;
+}
+
+int MonClient::start_mon_command(const string &mon_name,
+				 const vector& cmd,
+				 const bufferlist& inbl,
+				 bufferlist *outbl, string *outs,
+				 Context *onfinish)
+{
+  Mutex::Locker l(monc_lock);
+  MonCommand *r = new MonCommand(++last_mon_command_tid);
+  r->target_name = mon_name;
+  r->cmd = cmd;
+  r->inbl = inbl;
+  r->poutbl = outbl;
+  r->prs = outs;
+  r->onfinish = onfinish;
+  mon_commands[r->tid] = r;
+  _send_command(r);
+  // can't fail
+  return 0;
+}
+
+int MonClient::start_mon_command(int rank,
+				 const vector& cmd,
+				 const bufferlist& inbl,
+				 bufferlist *outbl, string *outs,
+				 Context *onfinish)
+{
+  Mutex::Locker l(monc_lock);
+  MonCommand *r = new MonCommand(++last_mon_command_tid);
+  r->target_rank = rank;
+  r->cmd = cmd;
+  r->inbl = inbl;
+  r->poutbl = outbl;
+  r->prs = outs;
+  r->onfinish = onfinish;
+  mon_commands[r->tid] = r;
+  _send_command(r);
+  return 0;
+}
+
+// ---------
+
+void MonClient::get_version(string map, version_t *newest, version_t *oldest, Context *onfinish)
+{
+  version_req_d *req = new version_req_d(onfinish, newest, oldest);
+  ldout(cct, 10) << "get_version " << map << " req " << req << dendl;
+  Mutex::Locker l(monc_lock);
+  MMonGetVersion *m = new MMonGetVersion();
+  m->what = map;
+  m->handle = ++version_req_id;
+  version_requests[m->handle] = req;
+  _send_mon_message(m);
+}
+
+void MonClient::handle_get_version_reply(MMonGetVersionReply* m)
+{
+  assert(monc_lock.is_locked());
+  map::iterator iter = version_requests.find(m->handle);
+  if (iter == version_requests.end()) {
+    ldout(cct, 0) << __func__ << " version request with handle " << m->handle
+		  << " not found" << dendl;
+  } else {
+    version_req_d *req = iter->second;
+    ldout(cct, 10) << __func__ << " finishing " << req << " version " << m->version << dendl;
+    version_requests.erase(iter);
+    if (req->newest)
+      *req->newest = m->version;
+    if (req->oldest)
+      *req->oldest = m->oldest_version;
+    finisher.queue(req->context, 0);
+    delete req;
+  }
+  m->put();
+}
diff --git a/ceph/src/mon/MonClient.h b/ceph/src/mon/MonClient.h
new file mode 100644
index 00000000..239d91b4
--- /dev/null
+++ b/ceph/src/mon/MonClient.h
@@ -0,0 +1,446 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MONCLIENT_H
+#define CEPH_MONCLIENT_H
+
+#include "msg/Dispatcher.h"
+#include "msg/Messenger.h"
+
+#include "MonMap.h"
+
+#include "common/Timer.h"
+#include "common/Finisher.h"
+
+#include "auth/AuthClientHandler.h"
+#include "auth/RotatingKeyRing.h"
+
+#include "messages/MMonSubscribe.h"
+
+#include "common/SimpleRNG.h"
+#include "osd/osd_types.h"
+
+#include 
+
+class MonMap;
+class MMonMap;
+class MMonGetVersion;
+class MMonGetVersionReply;
+struct MMonSubscribeAck;
+class MMonCommandAck;
+class MCommandReply;
+struct MAuthReply;
+class MAuthRotating;
+class MPing;
+class LogClient;
+class AuthSupported;
+class AuthAuthorizeHandlerRegistry;
+class AuthMethodList;
+
+enum MonClientState {
+  MC_STATE_NONE,
+  MC_STATE_NEGOTIATING,
+  MC_STATE_AUTHENTICATING,
+  MC_STATE_HAVE_SESSION,
+};
+
+struct MonClientPinger : public Dispatcher {
+
+  Mutex lock;
+  Cond ping_recvd_cond;
+  string *result;
+  bool done;
+
+  MonClientPinger(CephContext *cct_, string *res_) :
+    Dispatcher(cct_),
+    lock("MonClientPinger::lock"),
+    result(res_),
+    done(false)
+  { }
+
+  int wait_for_reply(double timeout = 0.0) {
+    utime_t until = ceph_clock_now(cct);
+    until += (timeout > 0 ? timeout : cct->_conf->client_mount_timeout);
+    done = false;
+
+    int ret = 0;
+    while (!done) {
+      ret = ping_recvd_cond.WaitUntil(lock, until);
+      if (ret == -ETIMEDOUT)
+        break;
+    }
+    return ret;
+  }
+
+  bool ms_dispatch(Message *m) {
+    Mutex::Locker l(lock);
+    if (m->get_type() != CEPH_MSG_PING)
+      return false;
+
+    bufferlist &payload = m->get_payload();
+    if (result && payload.length() > 0) {
+      bufferlist::iterator p = payload.begin();
+      ::decode(*result, p);
+    }
+    done = true;
+    ping_recvd_cond.SignalAll();
+    m->put();
+    return true;
+  }
+  bool ms_handle_reset(Connection *con) {
+    Mutex::Locker l(lock);
+    done = true;
+    ping_recvd_cond.SignalAll();
+    return true;
+  }
+  void ms_handle_remote_reset(Connection *con) {}
+};
+
+class MonClient : public Dispatcher {
+public:
+  MonMap monmap;
+private:
+  MonClientState state;
+
+  Messenger *messenger;
+
+  string cur_mon;
+  ConnectionRef cur_con;
+
+  SimpleRNG rng;
+
+  EntityName entity_name;
+
+  entity_addr_t my_addr;
+
+  Mutex monc_lock;
+  SafeTimer timer;
+  Finisher finisher;
+
+  // Added to support session signatures.  PLR
+
+  AuthAuthorizeHandlerRegistry *authorize_handler_registry;
+
+  bool initialized;
+  bool no_keyring_disabled_cephx;
+
+  LogClient *log_client;
+  bool more_log_pending;
+
+  void send_log();
+
+  AuthMethodList *auth_supported;
+
+  bool ms_dispatch(Message *m);
+  bool ms_handle_reset(Connection *con);
+  void ms_handle_remote_reset(Connection *con) {}
+
+  void handle_monmap(MMonMap *m);
+
+  void handle_auth(MAuthReply *m);
+
+  // monitor session
+  bool hunting;
+
+  struct C_Tick : public Context {
+    MonClient *monc;
+    C_Tick(MonClient *m) : monc(m) {}
+    void finish(int r) {
+      monc->tick();
+    }
+  };
+  void tick();
+  void schedule_tick();
+
+  Cond auth_cond;
+
+  void handle_auth_rotating_response(MAuthRotating *m);
+  // monclient
+  bool want_monmap;
+
+  uint32_t want_keys;
+
+  uint64_t global_id;
+
+  // authenticate
+private:
+  Cond map_cond;
+  int authenticate_err;
+
+  list waiting_for_session;
+  Context *session_established_context;
+  bool had_a_connection;
+  double reopen_interval_multiplier;
+
+  string _pick_random_mon();
+  void _finish_hunting();
+  void _reopen_session(int rank, string name);
+  void _reopen_session() {
+    _reopen_session(-1, string());
+  }
+  void _send_mon_message(Message *m, bool force=false);
+
+public:
+  void set_entity_name(EntityName name) { entity_name = name; }
+
+  int _check_auth_tickets();
+  int _check_auth_rotating();
+  int wait_auth_rotating(double timeout);
+
+  int authenticate(double timeout=0.0);
+
+  // mon subscriptions
+private:
+  map sub_have;  // my subs, and current versions
+  utime_t sub_renew_sent, sub_renew_after;
+
+  void _renew_subs();
+  void handle_subscribe_ack(MMonSubscribeAck* m);
+
+  bool _sub_want(string what, version_t start, unsigned flags) {
+    if (sub_have.count(what) &&
+	sub_have[what].start == start &&
+	sub_have[what].flags == flags)
+      return false;
+    sub_have[what].start = start;
+    sub_have[what].flags = flags;
+    return true;
+  }
+  void _sub_got(string what, version_t got) {
+    if (sub_have.count(what)) {
+      if (sub_have[what].flags & CEPH_SUBSCRIBE_ONETIME)
+	sub_have.erase(what);
+      else
+	sub_have[what].start = got + 1;
+    }
+  }
+  void _sub_unwant(string what) {
+    sub_have.erase(what);
+  }
+
+  // auth tickets
+public:
+  AuthClientHandler *auth;
+public:
+  void renew_subs() {
+    Mutex::Locker l(monc_lock);
+    _renew_subs();
+  }
+  bool sub_want(string what, version_t start, unsigned flags) {
+    Mutex::Locker l(monc_lock);
+    return _sub_want(what, start, flags);
+  }
+  void sub_got(string what, version_t have) {
+    Mutex::Locker l(monc_lock);
+    _sub_got(what, have);
+  }
+  void sub_unwant(string what) {
+    Mutex::Locker l(monc_lock);
+    _sub_unwant(what);
+  }
+  /**
+   * Increase the requested subscription start point. If you do increase
+   * the value, apply the passed-in flags as well; otherwise do nothing.
+   */
+  bool sub_want_increment(string what, version_t start, unsigned flags) {
+    Mutex::Locker l(monc_lock);
+    map::iterator i =
+            sub_have.find(what);
+    if (i == sub_have.end() || i->second.start < start) {
+      ceph_mon_subscribe_item& item = sub_have[what];
+      item.start = start;
+      item.flags = flags;
+      return true;
+    }
+    return false;
+  }
+  
+  KeyRing *keyring;
+  RotatingKeyRing *rotating_secrets;
+
+ public:
+  MonClient(CephContext *cct_);
+  ~MonClient();
+
+  int init();
+  void shutdown();
+
+  void set_log_client(LogClient *clog) {
+    log_client = clog;
+  }
+
+  int build_initial_monmap();
+  int get_monmap();
+  int get_monmap_privately();
+  /**
+   * Ping monitor with ID @p mon_id and record the resulting
+   * reply in @p result_reply.
+   *
+   * @param[in]  mon_id Target monitor's ID
+   * @param[out] Resulting reply from mon.ID, if param != NULL
+   * @returns    0 in case of success; < 0 in case of error,
+   *             -ETIMEDOUT if monitor didn't reply before timeout
+   *             expired (default: conf->client_mount_timeout).
+   */
+  int ping_monitor(const string &mon_id, string *result_reply);
+
+  void send_mon_message(Message *m) {
+    Mutex::Locker l(monc_lock);
+    _send_mon_message(m);
+  }
+  /**
+   * If you specify a callback, you should not call
+   * reopen_session() again until it has been triggered. The MonClient
+   * will behave, but the first callback could be triggered after
+   * the session has been killed and the MonClient has started trying
+   * to reconnect to another monitor.
+   */
+  void reopen_session(Context *cb=NULL) {
+    Mutex::Locker l(monc_lock);
+    if (cb) {
+      delete session_established_context;
+      session_established_context = cb;
+    }
+    _reopen_session();
+  }
+
+  entity_addr_t get_my_addr() const {
+    return my_addr;
+  }
+
+  const uuid_d& get_fsid() {
+    return monmap.fsid;
+  }
+
+  entity_addr_t get_mon_addr(unsigned i) {
+    Mutex::Locker l(monc_lock);
+    if (i < monmap.size())
+      return monmap.get_addr(i);
+    return entity_addr_t();
+  }
+  entity_inst_t get_mon_inst(unsigned i) {
+    Mutex::Locker l(monc_lock);
+    if (i < monmap.size())
+      return monmap.get_inst(i);
+    return entity_inst_t();
+  }
+  int get_num_mon() {
+    Mutex::Locker l(monc_lock);
+    return monmap.size();
+  }
+
+  uint64_t get_global_id() const {
+    return global_id;
+  }
+
+  void set_messenger(Messenger *m) { messenger = m; }
+
+  void send_auth_message(Message *m) {
+    _send_mon_message(m, true);
+  }
+
+  void set_want_keys(uint32_t want) {
+    want_keys = want;
+    if (auth)
+      auth->set_want_keys(want | CEPH_ENTITY_TYPE_MON);
+  }
+
+  void add_want_keys(uint32_t want) {
+    want_keys |= want;
+    if (auth)
+      auth->add_want_keys(want);
+  }
+
+  // admin commands
+private:
+  uint64_t last_mon_command_tid;
+  struct MonCommand {
+    string target_name;
+    int target_rank;
+    uint64_t tid;
+    vector cmd;
+    bufferlist inbl;
+    bufferlist *poutbl;
+    string *prs;
+    int *prval;
+    Context *onfinish, *ontimeout;
+
+    MonCommand(uint64_t t)
+      : target_rank(-1),
+	tid(t),
+	poutbl(NULL), prs(NULL), prval(NULL), onfinish(NULL), ontimeout(NULL)
+    {}
+  };
+  map mon_commands;
+
+  class C_CancelMonCommand : public Context
+  {
+    uint64_t tid;
+    MonClient *monc;
+  public:
+    C_CancelMonCommand(uint64_t tid, MonClient *monc) : tid(tid), monc(monc) {}
+    void finish(int r) {
+      monc->_cancel_mon_command(tid, -ETIMEDOUT);
+    }
+  };
+
+  void _send_command(MonCommand *r);
+  void _resend_mon_commands();
+  int _cancel_mon_command(uint64_t tid, int r);
+  void _finish_command(MonCommand *r, int ret, string rs);
+  void handle_mon_command_ack(MMonCommandAck *ack);
+
+public:
+  int start_mon_command(const vector& cmd, const bufferlist& inbl,
+			bufferlist *outbl, string *outs,
+			Context *onfinish);
+  int start_mon_command(int mon_rank,
+			const vector& cmd, const bufferlist& inbl,
+			bufferlist *outbl, string *outs,
+			Context *onfinish);
+  int start_mon_command(const string &mon_name,  ///< mon name, with mon. prefix
+			const vector& cmd, const bufferlist& inbl,
+			bufferlist *outbl, string *outs,
+			Context *onfinish);
+
+  // version requests
+public:
+  /**
+   * get latest known version(s) of cluster map
+   *
+   * @param map string name of map (e.g., 'osdmap')
+   * @param newest pointer where newest map version will be stored
+   * @param oldest pointer where oldest map version will be stored
+   * @param onfinish context that will be triggered on completion
+   * @return (via context) 0 on success, -EAGAIN if we need to resubmit our request
+   */
+  void get_version(string map, version_t *newest, version_t *oldest, Context *onfinish);
+
+private:
+  struct version_req_d {
+    Context *context;
+    version_t *newest, *oldest;
+    version_req_d(Context *con, version_t *n, version_t *o) : context(con),newest(n), oldest(o) {}
+  };
+
+  map version_requests;
+  ceph_tid_t version_req_id;
+  void handle_get_version_reply(MMonGetVersionReply* m);
+
+
+  MonClient(const MonClient &rhs);
+  MonClient& operator=(const MonClient &rhs);
+};
+
+#endif
diff --git a/ceph/src/mon/MonCommands.h b/ceph/src/mon/MonCommands.h
new file mode 100644
index 00000000..bd9dd2e7
--- /dev/null
+++ b/ceph/src/mon/MonCommands.h
@@ -0,0 +1,635 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc. 
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/* no guard; may be included multiple times */
+
+/*
+ * Define commands that are reported by the monitor's
+ * "get_command_descriptions" command, and parsed by the Python
+ * frontend 'ceph' (and perhaps by other frontends, such as a RESTful
+ * server). The format is:
+ *
+ * COMMAND(signature, helpstring, modulename, req perms, availability)
+ * where:
+ * signature:  describes the command and its parameters (more below)
+ * helpstring: displays in CLI help, API help (nice if it refers to
+ *             parameter names from signature, 40-a few hundred chars)
+ * modulename: the monitor module or daemon this applies to:
+ *             mds, osd, pg (osd), mon, auth, log, config-key
+ * req perms:  required permission in that modulename space to execute command
+ *             this also controls what type of REST command is accepted
+ * availability: cli, rest, or both
+ *
+ * The commands describe themselves completely enough for the separate
+ * frontend(s) to be able to accept user input and validate it against
+ * the command descriptions, and generate a JSON object that contains
+ * key:value mappings of parameter names to validated parameter values.
+ *
+ * 'signature' is a space-separated list of individual command descriptors;
+ * each descriptor is either a literal string, which can contain no spaces or
+ * '=' signs (for instance, in "pg stat", both "pg" and "stat" are literal
+ * strings representing one descriptor each), or a list of key=val[,key=val...]
+ * which also includes no spaces.  
+ *
+ * The key=val form describes a non-literal parameter.  Each will have at
+ * least a name= and type=, and each type can have its own type-specific
+ * parameters.  The parser is the arbiter of these types and their 
+ * interpretation.  A few more non-type-specific key=val pairs exist:
+ *
+ *    req=false marks an optional parameter (default for req is 'true')
+ *    n= is a repeat count for how many of this argument must be supplied.
+ *          n=1 is the default.
+ *          n=N is a special case that means "1 or more".
+ *
+ * A perhaps-incomplete list of types:
+ *
+ * CephInt: Optional: range=min[|max]
+ * CephFloat: Optional range
+ * CephString: optional badchars
+ * CephSocketpath: validation involves "is it S_ISSOCK"
+ * CephIPAddr: v4 or v6 addr with optional port, syntax validated
+ * CephEntityAddr: CephIPAddr + optional '/nonce'
+ * CephPoolname: Plainold string
+ * CephObjectname: Another plainold string
+ * CephPgid: n.xxx where n is an int > 0, xxx is a hex number > 0
+ * CephName: daemon name, '*' or '.' (id must be int for type osd)
+ * CephOsdName: osd name, '*' or ' or 'osd.' (id must be int)
+ * CephChoices: strings="foo|bar" means this param can be either
+ * CephFilepath: openable file
+ * CephFragment: cephfs 'fragID': val/bits, val in hex 0xnnn, bits in dec
+ * CephUUID: uuid in text matching Python uuid.UUID()
+ * CephPrefix: special type assigned to literals
+ *
+ * Example:
+ *
+ * COMMAND("auth add " \
+ *   	   "name=entity,type=CephString " \
+ *   	   "name=caps,type=CephString,n=N,req=false", \
+ *   	   "add auth info for  from input file, or random key " \
+ *   	   "if no input given, and/or any caps specified in the command")
+ *
+ * defines a command "auth add" that takes a required argument "entity"
+ * of type "CephString", and from 1 to N arguments named "caps" of type
+ * CephString, at least one of which is required.  The front end will
+ * validate user input against this description.  Let's say the user
+ * enters auth add client.admin 'mon rwx' 'osd *'.  The result will be a
+ * JSON object like {"prefix":"auth add", "entity":"client.admin",
+ * "caps":["mon rwx", "osd *"]}.
+ * Note that 
+ * 	- string literals are accumulated into 'prefix'
+ * 	- n=1 descriptors are given normal string or int object values
+ * 	- n=N descriptors are given array values
+ *
+ * NOTE: be careful with spaces.  Each descriptor must be separated by
+ * one space, no other characters, so if you split lines as above, be
+ * sure to close and reopen the quotes, and be careful to include the '
+ * separating spaces in the quoted string.
+ *
+ * The monitor marshals this JSON into a std::map
+* where cmd_vartype is a boost::variant type-enforcing discriminated
+* type, so the monitor is expected to know the type of each argument.
+* See cmdparse.cc/h for more details.
+*/
+
+/*
+ * pg commands PgMonitor.cc
+ */
+
+COMMAND("pg stat", "show placement group status.", "pg", "r", "cli,rest")
+COMMAND("pg getmap", "get binary pg map to -o/stdout", "pg", "r", "cli,rest")
+COMMAND("pg send_pg_creates", "trigger pg creates to be issued",\
+	"pg", "rw", "cli,rest")
+COMMAND("pg dump " \
+	"name=dumpcontents,type=CephChoices,strings=all|summary|sum|delta|pools|osds|pgs|pgs_brief,n=N,req=false", \
+	"show human-readable versions of pg map (only 'all' valid with plain)", "pg", "r", "cli,rest")
+COMMAND("pg dump_json " \
+	"name=dumpcontents,type=CephChoices,strings=all|summary|sum|pools|osds|pgs,n=N,req=false", \
+	"show human-readable version of pg map in json only",\
+	"pg", "r", "cli,rest")
+COMMAND("pg dump_pools_json", "show pg pools info in json only",\
+	"pg", "r", "cli,rest")
+COMMAND("pg dump_stuck " \
+	"name=stuckops,type=CephChoices,strings=inactive|unclean|stale,n=N,req=false " \
+	"name=threshold,type=CephInt,req=false",
+	"show information about stuck pgs",\
+	"pg", "r", "cli,rest")
+COMMAND("pg map name=pgid,type=CephPgid", "show mapping of pg to osds", \
+	"pg", "r", "cli,rest")
+COMMAND("pg scrub name=pgid,type=CephPgid", "start scrub on ", \
+	"pg", "rw", "cli,rest")
+COMMAND("pg deep-scrub name=pgid,type=CephPgid", "start deep-scrub on ", \
+	"pg", "rw", "cli,rest")
+COMMAND("pg repair name=pgid,type=CephPgid", "start repair on ", \
+	"pg", "rw", "cli,rest")
+COMMAND("pg debug " \
+	"name=debugop,type=CephChoices,strings=unfound_objects_exist|degraded_pgs_exist", \
+	"show debug info about pgs", "pg", "r", "cli,rest")
+COMMAND("pg force_create_pg name=pgid,type=CephPgid", \
+	"force creation of pg ", "pg", "rw", "cli,rest")
+COMMAND("pg set_full_ratio name=ratio,type=CephFloat,range=0.0|1.0", \
+	"set ratio at which pgs are considered full", "pg", "rw", "cli,rest")
+COMMAND("pg set_nearfull_ratio name=ratio,type=CephFloat,range=0.0|1.0", \
+	"set ratio at which pgs are considered nearly full", \
+	"pg", "rw", "cli,rest")
+
+/*
+ * auth commands AuthMonitor.cc
+ */
+
+COMMAND("auth export name=entity,type=CephString,req=false", \
+       	"write keyring for requested entity, or master keyring if none given", \
+	"auth", "rx", "cli,rest")
+COMMAND("auth get name=entity,type=CephString", \
+	"write keyring file with requested key", "auth", "rx", "cli,rest")
+COMMAND("auth get-key name=entity,type=CephString", "display requested key", \
+	"auth", "rx", "cli,rest")
+COMMAND("auth print-key name=entity,type=CephString", "display requested key", \
+	"auth", "rx", "cli,rest")
+COMMAND("auth print_key name=entity,type=CephString", "display requested key", \
+	"auth", "rx", "cli,rest")
+COMMAND("auth list", "list authentication state", "auth", "rx", "cli,rest")
+COMMAND("auth import", "auth import: read keyring file from -i ", \
+	"auth", "rwx", "cli,rest")
+COMMAND("auth add " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N,req=false", \
+	"add auth info for  from input file, or random key if no input given, and/or any caps specified in the command",
+	"auth", "rwx", "cli,rest")
+COMMAND("auth get-or-create-key " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N,req=false", \
+	"get, or add, key for  from system/caps pairs specified in the command.  If key already exists, any given caps must match the existing caps for that key.", \
+	"auth", "rwx", "cli,rest")
+COMMAND("auth get-or-create " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N,req=false", \
+	"add auth info for  from input file, or random key if no input given, and/or any caps specified in the command", \
+	"auth", "rwx", "cli,rest")
+COMMAND("auth caps " \
+	"name=entity,type=CephString " \
+	"name=caps,type=CephString,n=N", \
+	"update caps for  from caps specified in the command", \
+	"auth", "rwx", "cli,rest")
+COMMAND("auth del " \
+	"name=entity,type=CephString", \
+	"delete all caps for ", \
+	"auth", "rwx", "cli,rest")
+
+/*
+ * Monitor commands (Monitor.cc)
+ */
+COMMAND("compact", "cause compaction of monitor's leveldb storage", \
+	"mon", "rw", "cli,rest")
+COMMAND("scrub", "scrub the monitor stores", "mon", "rw", "cli,rest")
+COMMAND("fsid", "show cluster FSID/UUID", "mon", "r", "cli,rest")
+COMMAND("log name=logtext,type=CephString,n=N", \
+	"log supplied text to the monitor log", "mon", "rw", "cli,rest")
+COMMAND("injectargs " \
+	"name=injected_args,type=CephString,n=N", \
+	"inject config arguments into monitor", "mon", "rw", "cli,rest")
+COMMAND("status", "show cluster status", "mon", "r", "cli,rest")
+COMMAND("health name=detail,type=CephChoices,strings=detail,req=false", \
+	"show cluster health", "mon", "r", "cli,rest")
+COMMAND("df name=detail,type=CephChoices,strings=detail,req=false", \
+	"show cluster free space stats", "mon", "r", "cli,rest")
+COMMAND("report name=tags,type=CephString,n=N,req=false", \
+	"report full status of cluster, optional title tag strings", \
+	"mon", "r", "cli,rest")
+COMMAND("quorum_status", "report status of monitor quorum", \
+	"mon", "r", "cli,rest")
+COMMAND("mon_status", "report status of monitors", "mon", "r", "cli,rest")
+COMMAND("sync force " \
+	"name=validate1,type=CephChoices,strings=--yes-i-really-mean-it,req=false " \
+	"name=validate2,type=CephChoices,strings=--i-know-what-i-am-doing,req=false", \
+	"force sync of and clear monitor store", "mon", "rw", "cli,rest")
+COMMAND("heap " \
+	"name=heapcmd,type=CephChoices,strings=dump|start_profiler|stop_profiler|release|stats", \
+	"show heap usage info (available only if compiled with tcmalloc)", \
+	"mon", "rw", "cli,rest")
+COMMAND("quorum name=quorumcmd,type=CephChoices,strings=enter|exit,n=1", \
+	"enter or exit quorum", "mon", "rw", "cli,rest")
+COMMAND("tell " \
+	"name=target,type=CephName " \
+	"name=args,type=CephString,n=N", \
+	"send a command to a specific daemon", "mon", "rw", "cli,rest")
+
+/*
+ * MDS commands (MDSMonitor.cc)
+ */
+
+COMMAND("mds stat", "show MDS status", "mds", "r", "cli,rest")
+COMMAND("mds dump " 
+	"name=epoch,type=CephInt,req=false,range=0", \
+	"dump info, optionally from epoch", "mds", "r", "cli,rest")
+COMMAND("mds getmap " \
+	"name=epoch,type=CephInt,req=false,range=0", \
+	"get MDS map, optionally from epoch", "mds", "r", "cli,rest")
+COMMAND("mds tell " \
+	"name=who,type=CephString " \
+	"name=args,type=CephString,n=N", \
+	"send command to particular mds", "mds", "rw", "cli,rest")
+COMMAND("mds compat show", "show mds compatibility settings", \
+	"mds", "r", "cli,rest")
+COMMAND("mds stop name=who,type=CephString", "stop mds", \
+	"mds", "rw", "cli,rest")
+COMMAND("mds deactivate name=who,type=CephString", "stop mds", \
+	"mds", "rw", "cli,rest")
+COMMAND("mds set_max_mds " \
+	"name=maxmds,type=CephInt,range=0", \
+	"set max MDS index", "mds", "rw", "cli,rest")
+COMMAND("mds set " \
+	"name=var,type=CephChoices,strings=max_mds|max_file_size|allow_new_snaps|inline_data " \
+	"name=val,type=CephString "					\
+	"name=confirm,type=CephString,req=false",			\
+	"set mds parameter  to ", "mds", "rw", "cli,rest")
+COMMAND("mds setmap " \
+	"name=epoch,type=CephInt,range=0", \
+	"set mds map; must supply correct epoch number", "mds", "rw", "cli,rest")
+// arbitrary limit 0-20 below; worth standing on head to make it
+// relate to actual state definitions?
+// #include "include/ceph_fs.h"
+COMMAND("mds set_state " \
+	"name=gid,type=CephInt,range=0 " \
+	"name=state,type=CephInt,range=0|20", \
+	"set mds state of  to ", "mds", "rw", "cli,rest")
+COMMAND("mds fail name=who,type=CephString", \
+	"force mds to status failed", "mds", "rw", "cli,rest")
+COMMAND("mds rm " \
+	"name=gid,type=CephInt,range=0 " \
+	"name=who,type=CephName", \
+	"remove nonactive mds", "mds", "rw", "cli,rest")
+COMMAND("mds rmfailed name=who,type=CephInt,range=0", "remove failed mds", \
+	"mds", "rw", "cli,rest")
+COMMAND("mds cluster_down", "take MDS cluster down", "mds", "rw", "cli,rest")
+COMMAND("mds cluster_up", "bring MDS cluster up", "mds", "rw", "cli,rest")
+COMMAND("mds compat rm_compat " \
+	"name=feature,type=CephInt,range=0", \
+	"remove compatible feature", "mds", "rw", "cli,rest")
+COMMAND("mds compat rm_incompat " \
+	"name=feature,type=CephInt,range=0", \
+	"remove incompatible feature", "mds", "rw", "cli,rest")
+COMMAND("mds add_data_pool " \
+	"name=pool,type=CephString", \
+	"add data pool ", "mds", "rw", "cli,rest")
+COMMAND("mds remove_data_pool " \
+	"name=pool,type=CephString", \
+	"remove data pool ", "mds", "rw", "cli,rest")
+COMMAND("mds newfs " \
+	"name=metadata,type=CephInt,range=0 " \
+	"name=data,type=CephInt,range=0 " \
+	"name=sure,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
+	"make new filesystom using pools  and ", \
+	"mds", "rw", "cli,rest")
+/*
+ * Monmap commands
+ */
+COMMAND("mon dump " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"dump formatted monmap (optionally from epoch)", \
+	"mon", "r", "cli,rest")
+COMMAND("mon stat", "summarize monitor status", "mon", "r", "cli,rest")
+COMMAND("mon getmap " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"get monmap", "mon", "r", "cli,rest")
+COMMAND("mon add " \
+	"name=name,type=CephString " \
+	"name=addr,type=CephIPAddr", \
+	"add new monitor named  at ", "mon", "rw", "cli,rest")
+COMMAND("mon remove " \
+	"name=name,type=CephString", \
+	"remove monitor named ", "mon", "rw", "cli,rest")
+
+
+/*
+ * OSD commands
+ */
+COMMAND("osd stat", "print summary of OSD map", "osd", "r", "cli,rest")
+COMMAND("osd dump " \
+	"name=epoch,type=CephInt,range=0,req=false",
+	"print summary of OSD map", "osd", "r", "cli,rest")
+COMMAND("osd tree " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"print OSD tree", "osd", "r", "cli,rest")
+COMMAND("osd ls " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"show all OSD ids", "osd", "r", "cli,rest")
+COMMAND("osd getmap " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"get OSD map", "osd", "r", "cli,rest")
+COMMAND("osd getcrushmap " \
+	"name=epoch,type=CephInt,range=0,req=false", \
+	"get CRUSH map", "osd", "r", "cli,rest")
+COMMAND("osd perf", \
+        "print dump of OSD perf summary stats", \
+        "osd", \
+        "r", \
+        "cli,rest")
+COMMAND("osd getmaxosd", "show largest OSD id", "osd", "r", "cli,rest")
+COMMAND("osd find " \
+	"name=id,type=CephInt,range=0", \
+	"find osd  in the CRUSH map and show its location", \
+	"osd", "r", "cli,rest")
+COMMAND("osd metadata " \
+	"name=id,type=CephInt,range=0", \
+	"fetch metadata for osd ", \
+	"osd", "r", "cli,rest")
+COMMAND("osd map " \
+	"name=pool,type=CephPoolname " \
+	"name=object,type=CephObjectname", \
+	"find pg for  in ", "osd", "r", "cli,rest")
+COMMAND("osd scrub " \
+	"name=who,type=CephString", \
+	"initiate scrub on osd ", "osd", "rw", "cli,rest")
+COMMAND("osd deep-scrub " \
+	"name=who,type=CephString", \
+	"initiate deep scrub on osd ", "osd", "rw", "cli,rest")
+COMMAND("osd repair " \
+	"name=who,type=CephString", \
+	"initiate repair on osd ", "osd", "rw", "cli,rest")
+COMMAND("osd lspools " \
+	"name=auid,type=CephInt,req=false", \
+	"list pools", "osd", "r", "cli,rest")
+COMMAND("osd blacklist ls", "show blacklisted clients", "osd", "r", "cli,rest")
+COMMAND("osd crush rule list", "list crush rules", "osd", "r", "cli,rest")
+COMMAND("osd crush rule ls", "list crush rules", "osd", "r", "cli,rest")
+COMMAND("osd crush rule dump " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.],req=false", \
+	"dump crush rule  (default all)", \
+	"osd", "r", "cli,rest")
+COMMAND("osd crush dump", \
+	"dump crush map", \
+	"osd", "r", "cli,rest")
+COMMAND("osd setcrushmap", "set crush map from input file", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush set", "set crush map from input file", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush add-bucket " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=type,type=CephString", \
+	"add no-parent (probably root) crush bucket  of type ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush set " \
+	"name=id,type=CephOsdName " \
+	"name=weight,type=CephFloat,range=0.0 " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"update crushmap position and weight for  to  with location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush add " \
+	"name=id,type=CephOsdName " \
+	"name=weight,type=CephFloat,range=0.0 " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"add or update crushmap position and weight for  with  and location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush create-or-move " \
+	"name=id,type=CephOsdName " \
+	"name=weight,type=CephFloat,range=0.0 " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"create entry or move existing entry for   at/to location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush move " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"move existing entry for  to location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush link " \
+	"name=name,type=CephString " \
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]", \
+	"link existing entry for  under location ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush rm " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
+	"remove  from crush map (everywhere, or just at )",\
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush remove " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
+	"remove  from crush map (everywhere, or just at )", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush unlink " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]", \
+	"unlink  from crush map (everywhere, or just at )", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush reweight " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=weight,type=CephFloat,range=0.0", \
+	"change 's weight to  in crush map", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush tunables " \
+	"name=profile,type=CephChoices,strings=legacy|argonaut|bobtail|firefly|optimal|default", \
+	"set crush tunables values to ", "osd", "rw", "cli,rest")
+COMMAND("osd crush show-tunables", \
+	"show current crush tunables", "osd", "r", "cli,rest")
+COMMAND("osd crush rule create-simple " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=root,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=type,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=mode,type=CephChoices,strings=firstn|indep,req=false",
+	"create crush rule  to start from , replicate across buckets of type , using a choose mode of  (default firstn; indep best for erasure pools)", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush rule create-erasure " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=profile,type=CephString,req=false,goodchars=[A-Za-z0-9-_.=]", \
+	"create crush rule  for erasure coded pool created with  (default default)", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd crush rule rm " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] ",	\
+	"remove crush rule ", "osd", "rw", "cli,rest")
+COMMAND("osd setmaxosd " \
+	"name=newmax,type=CephInt,range=0", \
+	"set new maximum osd value", "osd", "rw", "cli,rest")
+COMMAND("osd pause", "pause osd", "osd", "rw", "cli,rest")
+COMMAND("osd unpause", "unpause osd", "osd", "rw", "cli,rest")
+COMMAND("osd erasure-code-profile set " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
+	"name=profile,type=CephString,n=N,req=false,goodchars=[A-Za-z0-9-_.=]", \
+	"create erasure code profile  with [ ...] pairs. Add a --force at the end to override an existing profile (VERY DANGEROUS)", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd erasure-code-profile get " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.]", \
+	"get erasure code profile ", \
+	"osd", "r", "cli,rest")
+COMMAND("osd erasure-code-profile rm " \
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.]", \
+	"remove erasure code profile ", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd erasure-code-profile ls", \
+	"list all erasure code profiles", \
+	"osd", "r", "cli,rest")
+COMMAND("osd set " \
+	"name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub|notieragent", \
+	"set ", "osd", "rw", "cli,rest")
+COMMAND("osd unset " \
+	"name=key,type=CephChoices,strings=pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub|notieragent", \
+	"unset ", "osd", "rw", "cli,rest")
+COMMAND("osd cluster_snap", "take cluster snapshot (disabled)", \
+	"osd", "r", "")
+COMMAND("osd down " \
+	"type=CephString,name=ids,n=N", \
+	"set osd(s)  [...] down", "osd", "rw", "cli,rest")
+COMMAND("osd out " \
+	"name=ids,type=CephString,n=N", \
+	"set osd(s)  [...] out", "osd", "rw", "cli,rest")
+COMMAND("osd in " \
+	"name=ids,type=CephString,n=N", \
+	"set osd(s)  [...] in", "osd", "rw", "cli,rest")
+COMMAND("osd rm " \
+	"name=ids,type=CephString,n=N", \
+	"remove osd(s)  [...] in", "osd", "rw", "cli,rest")
+COMMAND("osd reweight " \
+	"name=id,type=CephInt,range=0 " \
+	"type=CephFloat,name=weight,range=0.0|1.0", \
+	"reweight osd to 0.0 <  < 1.0", "osd", "rw", "cli,rest")
+COMMAND("osd pg-temp " \
+	"name=pgid,type=CephPgid " \
+	"name=id,type=CephString,n=N,req=false", \
+	"set pg_temp mapping pgid:[ [...]] (developers only)", \
+        "osd", "rw", "cli,rest")
+COMMAND("osd primary-temp " \
+	"name=pgid,type=CephPgid " \
+	"name=id,type=CephString", \
+        "set primary_temp mapping pgid:|-1 (developers only)", \
+        "osd", "rw", "cli,rest")
+COMMAND("osd primary-affinity " \
+	"name=id,type=CephOsdName " \
+	"type=CephFloat,name=weight,range=0.0|1.0", \
+	"adjust osd primary-affinity from 0.0 <=  <= 1.0", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd lost " \
+	"name=id,type=CephInt,range=0 " \
+	"name=sure,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
+	"mark osd as permanently lost. THIS DESTROYS DATA IF NO MORE REPLICAS EXIST, BE CAREFUL", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd create " \
+	"name=uuid,type=CephUUID,req=false", \
+	"create new osd (with optional UUID)", "osd", "rw", "cli,rest")
+COMMAND("osd blacklist " \
+	"name=blacklistop,type=CephChoices,strings=add|rm " \
+	"name=addr,type=CephEntityAddr " \
+	"name=expire,type=CephFloat,range=0.0,req=false", \
+	"add (optionally until  seconds from now) or remove  from blacklist", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd pool mksnap " \
+	"name=pool,type=CephPoolname " \
+	"name=snap,type=CephString", \
+	"make snapshot  in ", "osd", "rw", "cli,rest")
+COMMAND("osd pool rmsnap " \
+	"name=pool,type=CephPoolname " \
+	"name=snap,type=CephString", \
+	"remove snapshot  from ", "osd", "rw", "cli,rest")
+COMMAND("osd pool create " \
+	"name=pool,type=CephPoolname " \
+	"name=pg_num,type=CephInt,range=0 " \
+	"name=pgp_num,type=CephInt,range=0,req=false " \
+        "name=pool_type,type=CephChoices,strings=replicated|erasure,req=false " \
+	"name=erasure_code_profile,type=CephString,req=false,goodchars=[A-Za-z0-9-_.=] " \
+	"name=ruleset,type=CephString,req=false,goodchars=[A-Za-z0-9-_.=]", \
+	"create pool", "osd", "rw", "cli,rest")
+COMMAND("osd pool delete " \
+	"name=pool,type=CephPoolname " \
+	"name=pool2,type=CephPoolname,req=false " \
+	"name=sure,type=CephChoices,strings=--yes-i-really-really-mean-it,req=false", \
+	"delete pool", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd pool rename " \
+	"name=srcpool,type=CephPoolname " \
+	"name=destpool,type=CephPoolname", \
+	"rename  to ", "osd", "rw", "cli,rest")
+COMMAND("osd pool get " \
+	"name=pool,type=CephPoolname " \
+	"name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|auid|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile", \
+	"get pool parameter ", "osd", "r", "cli,rest")
+COMMAND("osd pool set " \
+	"name=pool,type=CephPoolname " \
+	"name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|debug_fake_ec_pool|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|auid " \
+	"name=val,type=CephString " \
+	"name=force,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
+	"set pool parameter  to ", "osd", "rw", "cli,rest")
+// 'val' is a CephString because it can include a unit.  Perhaps
+// there should be a Python type for validation/conversion of strings
+// with units.
+COMMAND("osd pool set-quota " \
+	"name=pool,type=CephPoolname " \
+	"name=field,type=CephChoices,strings=max_objects|max_bytes " \
+	"name=val,type=CephString",
+	"set object or byte limit on pool", "osd", "rw", "cli,rest")
+COMMAND("osd pool get-quota " \
+        "name=pool,type=CephPoolname ",
+        "obtain object or byte limits for pool",
+        "osd", "r", "cli,rest")
+COMMAND("osd pool stats " \
+        "name=name,type=CephString,req=false",
+        "obtain stats from all pools, or from specified pool",
+        "osd", "r", "cli,rest")
+COMMAND("osd reweight-by-utilization " \
+	"name=oload,type=CephInt,range=100,req=false", \
+	"reweight OSDs by utilization [overload-percentage-for-consideration, default 120]", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd thrash " \
+	"name=num_epochs,type=CephInt,range=0", \
+	"thrash OSDs for ", "osd", "rw", "cli,rest")
+
+// tiering
+COMMAND("osd tier add " \
+	"name=pool,type=CephPoolname " \
+	"name=tierpool,type=CephPoolname " \
+	"name=force_nonempty,type=CephChoices,strings=--force-nonempty,req=false",
+	"add the tier  (the second one) to base pool  (the first one)", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd tier remove " \
+	"name=pool,type=CephPoolname " \
+	"name=tierpool,type=CephPoolname",
+	"remove the tier  (the second one) from base pool  (the first one)", \
+	"osd", "rw", "cli,rest")
+COMMAND("osd tier cache-mode " \
+	"name=pool,type=CephPoolname " \
+	"name=mode,type=CephChoices,strings=none|writeback|forward|readonly", \
+	"specify the caching mode for cache tier ", "osd", "rw", "cli,rest")
+COMMAND("osd tier set-overlay " \
+	"name=pool,type=CephPoolname " \
+	"name=overlaypool,type=CephPoolname", \
+	"set the overlay pool for base pool  to be ", "osd", "rw", "cli,rest")
+COMMAND("osd tier remove-overlay " \
+	"name=pool,type=CephPoolname ", \
+	"remove the overlay pool for base pool ", "osd", "rw", "cli,rest")
+
+COMMAND("osd tier add-cache " \
+	"name=pool,type=CephPoolname " \
+	"name=tierpool,type=CephPoolname " \
+	"name=size,type=CephInt,range=0", \
+	"add a cache  (the second one) of size  to existing pool  (the first one)", \
+	"osd", "rw", "cli,rest")
+
+/*
+ * mon/ConfigKeyService.cc
+ */
+
+COMMAND("config-key get " \
+	"name=key,type=CephString", \
+	"get ", "config-key", "r", "cli,rest")
+COMMAND("config-key put " \
+	"name=key,type=CephString " \
+	"name=val,type=CephString,req=false", \
+	"put , value ", "config-key", "rw", "cli,rest")
+COMMAND("config-key del " \
+	"name=key,type=CephString", \
+	"delete ", "config-key", "rw", "cli,rest")
+COMMAND("config-key exists " \
+	"name=key,type=CephString", \
+	"check for 's existence", "config-key", "r", "cli,rest")
+COMMAND("config-key list ", "list keys", "config-key", "r", "cli,rest")
diff --git a/ceph/src/mon/MonMap.cc b/ceph/src/mon/MonMap.cc
new file mode 100644
index 00000000..cf481234
--- /dev/null
+++ b/ceph/src/mon/MonMap.cc
@@ -0,0 +1,337 @@
+
+#include "MonMap.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "include/ceph_features.h"
+#include "include/addr_parsing.h"
+#include "common/ceph_argparse.h"
+#include "common/errno.h"
+
+#include "common/dout.h"
+
+using ceph::Formatter;
+
+void MonMap::encode(bufferlist& blist, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_MONNAMES) == 0) {
+    __u16 v = 1;
+    ::encode(v, blist);
+    ::encode_raw(fsid, blist);
+    ::encode(epoch, blist);
+    vector mon_inst(mon_addr.size());
+    for (unsigned n = 0; n < mon_addr.size(); n++)
+      mon_inst[n] = get_inst(n);
+    ::encode(mon_inst, blist);
+    ::encode(last_changed, blist);
+    ::encode(created, blist);
+    return;
+  }
+
+  if ((features & CEPH_FEATURE_MONENC) == 0) {
+    __u16 v = 2;
+    ::encode(v, blist);
+    ::encode_raw(fsid, blist);
+    ::encode(epoch, blist);
+    ::encode(mon_addr, blist);
+    ::encode(last_changed, blist);
+    ::encode(created, blist);
+  }
+
+  ENCODE_START(3, 3, blist);
+  ::encode_raw(fsid, blist);
+  ::encode(epoch, blist);
+  ::encode(mon_addr, blist);
+  ::encode(last_changed, blist);
+  ::encode(created, blist);
+  ENCODE_FINISH(blist);
+}
+
+void MonMap::decode(bufferlist::iterator &p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN_16(3, 3, 3, p);
+  ::decode_raw(fsid, p);
+  ::decode(epoch, p);
+  if (struct_v == 1) {
+    vector mon_inst;
+    ::decode(mon_inst, p);
+    for (unsigned i = 0; i < mon_inst.size(); i++) {
+      char n[2];
+      n[0] = '0' + i;
+      n[1] = 0;
+      string name = n;
+      mon_addr[name] = mon_inst[i].addr;
+    }
+  } else {
+    ::decode(mon_addr, p);
+  }
+  ::decode(last_changed, p);
+  ::decode(created, p);
+  DECODE_FINISH(p);
+  calc_ranks();
+}
+
+void MonMap::generate_test_instances(list& o)
+{
+  o.push_back(new MonMap);
+  o.push_back(new MonMap);
+  o.back()->epoch = 1;
+  o.back()->last_changed = utime_t(123, 456);
+  o.back()->created = utime_t(789, 101112);
+  o.back()->add("one", entity_addr_t());
+}
+
+// read from/write to a file
+int MonMap::write(const char *fn) 
+{
+  // encode
+  bufferlist bl;
+  encode(bl, CEPH_FEATURES_ALL);
+  
+  return bl.write_file(fn);
+}
+
+int MonMap::read(const char *fn) 
+{
+  // read
+  bufferlist bl;
+  std::string error;
+  int r = bl.read_file(fn, &error);
+  if (r < 0)
+    return r;
+  decode(bl);
+  return 0;
+}
+
+void MonMap::print_summary(ostream& out) const
+{
+  out << "e" << epoch << ": "
+      << mon_addr.size() << " mons at "
+      << mon_addr;
+}
+ 
+void MonMap::print(ostream& out) const
+{
+  out << "epoch " << epoch << "\n";
+  out << "fsid " << fsid << "\n";
+  out << "last_changed " << last_changed << "\n";
+  out << "created " << created << "\n";
+  unsigned i = 0;
+  for (map::const_iterator p = addr_name.begin();
+       p != addr_name.end();
+       ++p)
+    out << i++ << ": " << p->first << " mon." << p->second << "\n";
+}
+
+void MonMap::dump(Formatter *f) const
+{
+  f->dump_int("epoch", epoch);
+  f->dump_stream("fsid") <<  fsid;
+  f->dump_stream("modified") << last_changed;
+  f->dump_stream("created") << created;
+  f->open_array_section("mons");
+  int i = 0;
+  for (map::const_iterator p = addr_name.begin();
+       p != addr_name.end();
+       ++p, ++i) {
+    f->open_object_section("mon");
+    f->dump_int("rank", i);
+    f->dump_string("name", p->second);
+    f->dump_stream("addr") << p->first;
+    f->close_section();
+  }
+  f->close_section();
+}
+
+
+int MonMap::build_from_host_list(std::string hostlist, std::string prefix)
+{
+  vector addrs;
+  if (parse_ip_port_vec(hostlist.c_str(), addrs)) {
+    for (unsigned i=0; i& initial_members,
+				 string my_name, const entity_addr_t& my_addr,
+				 set *removed)
+{
+  // remove non-initial members
+  unsigned i = 0;
+  while (i < size()) {
+    string n = get_name(i);
+    if (std::find(initial_members.begin(), initial_members.end(), n) != initial_members.end()) {
+      lgeneric_dout(cct, 1) << " keeping " << n << " " << get_addr(i) << dendl;
+      i++;
+      continue;
+    }
+
+    lgeneric_dout(cct, 1) << " removing " << get_name(i) << " " << get_addr(i) << dendl;
+    if (removed)
+      removed->insert(get_addr(i));
+    remove(n);
+    assert(!contains(n));
+  }
+
+  // add missing initial members
+  for (list::iterator p = initial_members.begin(); p != initial_members.end(); ++p) {
+    if (!contains(*p)) {
+      if (*p == my_name) {
+	lgeneric_dout(cct, 1) << " adding self " << *p << " " << my_addr << dendl;
+	add(*p, my_addr);
+      } else {
+	entity_addr_t a;
+	a.set_family(AF_INET);
+	for (int n=1; ; n++) {
+	  a.set_nonce(n);
+	  if (!contains(a))
+	    break;
+	}
+	lgeneric_dout(cct, 1) << " adding " << *p << " " << a << dendl;
+	add(*p, a);
+      }
+      assert(contains(*p));
+    }
+  }
+}
+
+
+int MonMap::build_initial(CephContext *cct, ostream& errout)
+{
+  const md_config_t *conf = cct->_conf;
+  // file?
+  if (!conf->monmap.empty()) {
+    int r;
+    try {
+      r = read(conf->monmap.c_str());
+    }
+    catch (const buffer::error &e) {
+      r = -EINVAL;
+    }
+    if (r >= 0)
+      return 0;
+    errout << "unable to read/decode monmap from " << conf->monmap
+	 << ": " << cpp_strerror(-r) << std::endl;
+    return r;
+  }
+
+  // fsid from conf?
+  if (!cct->_conf->fsid.is_zero()) {
+    fsid = cct->_conf->fsid;
+  }
+
+  // -m foo?
+  if (!conf->mon_host.empty()) {
+    int r = build_from_host_list(conf->mon_host, "noname-");
+    if (r < 0) {
+      errout << "unable to parse addrs in '" << conf->mon_host << "'"
+             << std::endl;
+      return r;
+    }
+    return 0;
+  }
+
+  // What monitors are in the config file?
+  std::vector  sections;
+  int ret = conf->get_all_sections(sections);
+  if (ret) {
+    errout << "Unable to find any monitors in the configuration "
+         << "file, because there was an error listing the sections. error "
+	 << ret << std::endl;
+    return -ENOENT;
+  }
+  std::vector  mon_names;
+  for (std::vector ::const_iterator s = sections.begin();
+       s != sections.end(); ++s) {
+    if ((s->substr(0, 4) == "mon.") && (s->size() > 4)) {
+      mon_names.push_back(s->substr(4));
+    }
+  }
+
+  // Find an address for each monitor in the config file.
+  for (std::vector ::const_iterator m = mon_names.begin();
+       m != mon_names.end(); ++m) {
+    std::vector  sections;
+    std::string m_name("mon");
+    m_name += ".";
+    m_name += *m;
+    sections.push_back(m_name);
+    sections.push_back("mon");
+    sections.push_back("global");
+    std::string val;
+    int res = conf->get_val_from_conf_file(sections, "mon addr", val, true);
+    if (res) {
+      errout << "failed to get an address for mon." << *m << ": error "
+	   << res << std::endl;
+      continue;
+    }
+    entity_addr_t addr;
+    if (!addr.parse(val.c_str())) {
+      errout << "unable to parse address for mon." << *m
+	   << ": addr='" << val << "'" << std::endl;
+      continue;
+    }
+    if (addr.get_port() == 0)
+      addr.set_port(CEPH_MON_PORT);
+
+    // the make sure this mon isn't already in the map
+    if (contains(addr))
+      remove(get_name(addr));
+    if (contains(*m))
+      remove(*m);
+
+    add(m->c_str(), addr);
+  }
+
+  if (size() == 0) {
+    errout << "no monitors specified to connect to." << std::endl;
+    return -ENOENT;
+  }
+  return 0;
+}
diff --git a/ceph/src/mon/MonMap.h b/ceph/src/mon/MonMap.h
new file mode 100644
index 00000000..ae81b073
--- /dev/null
+++ b/ceph/src/mon/MonMap.h
@@ -0,0 +1,247 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MONMAP_H
+#define CEPH_MONMAP_H
+
+#include "include/err.h"
+
+#include "msg/Message.h"
+#include "include/types.h"
+//#include "common/config.h"
+
+namespace ceph {
+  class Formatter;
+}
+
+class MonMap {
+ public:
+  epoch_t epoch;       // what epoch/version of the monmap
+  uuid_d fsid;
+  map mon_addr;
+  utime_t last_changed;
+  utime_t created;
+
+  map addr_name;
+  vector rank_name;
+  vector rank_addr;
+
+  void calc_ranks() {
+    rank_name.resize(mon_addr.size());
+    rank_addr.resize(mon_addr.size());
+    addr_name.clear();
+    for (map::iterator p = mon_addr.begin();
+	 p != mon_addr.end();
+	 ++p) {
+      assert(addr_name.count(p->second) == 0);
+      addr_name[p->second] = p->first;
+    }
+    unsigned i = 0;
+    for (map::iterator p = addr_name.begin();
+	 p != addr_name.end();
+	 ++p, i++) {
+      rank_name[i] = p->second;
+      rank_addr[i] = p->first;
+    }
+  }
+
+  MonMap() 
+    : epoch(0) {
+    memset(&fsid, 0, sizeof(fsid));
+  }
+
+  uuid_d& get_fsid() { return fsid; }
+
+  unsigned size() {
+    return mon_addr.size();
+  }
+
+  epoch_t get_epoch() { return epoch; }
+  void set_epoch(epoch_t e) { epoch = e; }
+
+  void list_addrs(list& ls) const {
+    for (map::const_iterator p = mon_addr.begin();
+	 p != mon_addr.end();
+	 ++p)
+      ls.push_back(p->second);
+  }
+
+  void add(const string &name, const entity_addr_t &addr) {
+    assert(mon_addr.count(name) == 0);
+    assert(addr_name.count(addr) == 0);
+    mon_addr[name] = addr;
+    calc_ranks();
+  }
+  
+  void remove(const string &name) {
+    assert(mon_addr.count(name));
+    mon_addr.erase(name);
+    calc_ranks();
+  }
+
+  void rename(string oldname, string newname) {
+    assert(contains(oldname));
+    assert(!contains(newname));
+    mon_addr[newname] = mon_addr[oldname];
+    mon_addr.erase(oldname);
+    calc_ranks();
+  }
+
+  bool contains(const string& name) {
+    return mon_addr.count(name);
+  }
+
+  bool contains(const entity_addr_t &a) {
+    for (map::iterator p = mon_addr.begin();
+	 p != mon_addr.end();
+	 ++p) {
+      if (p->second == a)
+	return true;
+    }
+    return false;
+  }
+
+  string get_name(unsigned n) const {
+    assert(n < rank_name.size());
+    return rank_name[n];
+  }
+  string get_name(const entity_addr_t& a) const {
+    map::const_iterator p = addr_name.find(a);
+    if (p == addr_name.end())
+      return string();
+    else
+      return p->second;
+  }
+
+  int get_rank(const string& n) {
+    for (unsigned i=0; i& initial_members,
+			   string my_name, const entity_addr_t& my_addr,
+			   set *removed);
+
+  void print(ostream& out) const;
+  void print_summary(ostream& out) const;
+  void dump(ceph::Formatter *f) const;
+
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER_FEATURES(MonMap)
+
+inline ostream& operator<<(ostream& out, MonMap& m) {
+  m.print_summary(out);
+  return out;
+}
+
+#endif
diff --git a/ceph/src/mon/Monitor.cc b/ceph/src/mon/Monitor.cc
new file mode 100644
index 00000000..fd3a358d
--- /dev/null
+++ b/ceph/src/mon/Monitor.cc
@@ -0,0 +1,4550 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "Monitor.h"
+#include "common/version.h"
+
+#include "osd/OSDMap.h"
+
+#include "MonitorStore.h"
+#include "MonitorDBStore.h"
+
+#include "msg/Messenger.h"
+
+#include "messages/PaxosServiceMessage.h"
+#include "messages/MMonMap.h"
+#include "messages/MMonGetMap.h"
+#include "messages/MMonGetVersion.h"
+#include "messages/MMonGetVersionReply.h"
+#include "messages/MGenericMessage.h"
+#include "messages/MMonCommand.h"
+#include "messages/MMonCommandAck.h"
+#include "messages/MMonSync.h"
+#include "messages/MMonScrub.h"
+#include "messages/MMonProbe.h"
+#include "messages/MMonJoin.h"
+#include "messages/MMonPaxos.h"
+#include "messages/MRoute.h"
+#include "messages/MForward.h"
+
+#include "messages/MMonSubscribe.h"
+#include "messages/MMonSubscribeAck.h"
+
+#include "messages/MAuthReply.h"
+
+#include "messages/MTimeCheck.h"
+#include "messages/MMonHealth.h"
+#include "messages/MPing.h"
+
+#include "common/strtol.h"
+#include "common/ceph_argparse.h"
+#include "common/Timer.h"
+#include "common/Clock.h"
+#include "common/errno.h"
+#include "common/perf_counters.h"
+#include "common/admin_socket.h"
+
+#include "include/color.h"
+#include "include/ceph_fs.h"
+#include "include/str_list.h"
+
+#include "OSDMonitor.h"
+#include "MDSMonitor.h"
+#include "MonmapMonitor.h"
+#include "PGMonitor.h"
+#include "LogMonitor.h"
+#include "AuthMonitor.h"
+#include "mon/QuorumService.h"
+#include "mon/HealthMonitor.h"
+#include "mon/ConfigKeyService.h"
+
+#include "auth/AuthMethodList.h"
+#include "auth/KeyRing.h"
+
+#include "common/config.h"
+#include "common/cmdparse.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+static ostream& _prefix(std::ostream *_dout, const Monitor *mon) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name() << ") e" << mon->monmap->get_epoch() << " ";
+}
+
+const string Monitor::MONITOR_NAME = "monitor";
+const string Monitor::MONITOR_STORE_PREFIX = "monitor_store";
+
+
+#undef COMMAND
+MonCommand mon_commands[] = {
+#define COMMAND(parsesig, helptext, modulename, req_perms, avail) \
+  {parsesig, helptext, modulename, req_perms, avail},
+#include 
+};
+MonCommand classic_mon_commands[] = {
+#include 
+};
+
+
+long parse_pos_long(const char *s, ostream *pss)
+{
+  if (*s == '-' || *s == '+') {
+    if (pss)
+      *pss << "expected numerical value, got: " << s;
+    return -EINVAL;
+  }
+
+  string err;
+  long r = strict_strtol(s, 10, &err);
+  if ((r == 0) && !err.empty()) {
+    if (pss)
+      *pss << err;
+    return -1;
+  }
+  if (r < 0) {
+    if (pss)
+      *pss << "unable to parse positive integer '" << s << "'";
+    return -1;
+  }
+  return r;
+}
+
+Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s,
+		 Messenger *m, MonMap *map) :
+  Dispatcher(cct_),
+  name(nm),
+  rank(-1), 
+  messenger(m),
+  con_self(m ? m->get_loopback_connection() : NULL),
+  lock("Monitor::lock"),
+  timer(cct_, lock),
+  has_ever_joined(false),
+  logger(NULL), cluster_logger(NULL), cluster_logger_registered(false),
+  monmap(map),
+  clog(cct_, messenger, monmap, LogClient::FLAG_MON),
+  key_server(cct, &keyring),
+  auth_cluster_required(cct,
+			cct->_conf->auth_supported.length() ?
+			cct->_conf->auth_supported : cct->_conf->auth_cluster_required),
+  auth_service_required(cct,
+			cct->_conf->auth_supported.length() ?
+			cct->_conf->auth_supported : cct->_conf->auth_service_required),
+  leader_supported_mon_commands(NULL),
+  leader_supported_mon_commands_size(0),
+  store(s),
+  
+  state(STATE_PROBING),
+  
+  elector(this),
+  required_features(0),
+  leader(0),
+  quorum_features(0),
+  scrub_version(0),
+
+  // sync state
+  sync_provider_count(0),
+  sync_cookie(0),
+  sync_full(false),
+  sync_start_version(0),
+  sync_timeout_event(NULL),
+  sync_last_committed_floor(0),
+
+  timecheck_round(0),
+  timecheck_acks(0),
+  timecheck_event(NULL),
+
+  probe_timeout_event(NULL),
+
+  paxos_service(PAXOS_NUM),
+  admin_hook(NULL),
+  routed_request_tid(0)
+{
+  rank = -1;
+
+  paxos = new Paxos(this, "paxos");
+
+  paxos_service[PAXOS_MDSMAP] = new MDSMonitor(this, paxos, "mdsmap");
+  paxos_service[PAXOS_MONMAP] = new MonmapMonitor(this, paxos, "monmap");
+  paxos_service[PAXOS_OSDMAP] = new OSDMonitor(this, paxos, "osdmap");
+  paxos_service[PAXOS_PGMAP] = new PGMonitor(this, paxos, "pgmap");
+  paxos_service[PAXOS_LOG] = new LogMonitor(this, paxos, "logm");
+  paxos_service[PAXOS_AUTH] = new AuthMonitor(this, paxos, "auth");
+
+  health_monitor = new HealthMonitor(this);
+  config_key_service = new ConfigKeyService(this, paxos);
+
+  mon_caps = new MonCap();
+  bool r = mon_caps->parse("allow *", NULL);
+  assert(r);
+
+  exited_quorum = ceph_clock_now(g_ceph_context);
+
+  // assume our commands until we have an election.  this only means
+  // we won't reply with EINVAL before the election; any command that
+  // actually matters will wait until we have quorum etc and then
+  // retry (and revalidate).
+  const MonCommand *cmds;
+  int cmdsize;
+  get_locally_supported_monitor_commands(&cmds, &cmdsize);
+  set_leader_supported_commands(cmds, cmdsize);
+}
+
+PaxosService *Monitor::get_paxos_service_by_name(const string& name)
+{
+  if (name == "mdsmap")
+    return paxos_service[PAXOS_MDSMAP];
+  if (name == "monmap")
+    return paxos_service[PAXOS_MONMAP];
+  if (name == "osdmap")
+    return paxos_service[PAXOS_OSDMAP];
+  if (name == "pgmap")
+    return paxos_service[PAXOS_PGMAP];
+  if (name == "logm")
+    return paxos_service[PAXOS_LOG];
+  if (name == "auth")
+    return paxos_service[PAXOS_AUTH];
+
+  assert(0 == "given name does not match known paxos service");
+  return NULL;
+}
+
+Monitor::~Monitor()
+{
+  for (vector::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p)
+    delete *p;
+  delete health_monitor;
+  delete config_key_service;
+  delete paxos;
+  assert(session_map.sessions.empty());
+  delete mon_caps;
+  if (leader_supported_mon_commands != mon_commands &&
+      leader_supported_mon_commands != classic_mon_commands)
+    delete[] leader_supported_mon_commands;
+}
+
+
+enum {
+  l_mon_first = 456000,
+  l_mon_last,
+};
+
+
+class AdminHook : public AdminSocketHook {
+  Monitor *mon;
+public:
+  AdminHook(Monitor *m) : mon(m) {}
+  bool call(std::string command, cmdmap_t& cmdmap, std::string format,
+	    bufferlist& out) {
+    stringstream ss;
+    mon->do_admin_command(command, cmdmap, format, ss);
+    out.append(ss);
+    return true;
+  }
+};
+
+void Monitor::do_admin_command(string command, cmdmap_t& cmdmap, string format,
+			       ostream& ss)
+{
+  Mutex::Locker l(lock);
+
+  boost::scoped_ptr f(new_formatter(format));
+
+  if (command == "mon_status") {
+    get_mon_status(f.get(), ss);
+    if (f)
+      f->flush(ss);
+  } else if (command == "quorum_status")
+    _quorum_status(f.get(), ss);
+  else if (command == "sync_force") {
+    string validate;
+    if ((!cmd_getval(g_ceph_context, cmdmap, "validate", validate)) ||
+	(validate != "--yes-i-really-mean-it")) {
+      ss << "are you SURE? this will mean the monitor store will be erased "
+            "the next time the monitor is restarted.  pass "
+            "'--yes-i-really-mean-it' if you really do.";
+      return;
+    }
+    sync_force(f.get(), ss);
+  } else if (command.find("add_bootstrap_peer_hint") == 0) {
+    _add_bootstrap_peer_hint(command, cmdmap, ss);
+  } else if (command.find("osdmonitor_prepare_command") == 0) {
+    _osdmonitor_prepare_command(cmdmap, ss);
+  } else if (command == "quorum enter") {
+    elector.start_participating();
+    start_election();
+    ss << "started responding to quorum, initiated new election";
+  } else if (command == "quorum exit") {
+    start_election();
+    elector.stop_participating();
+    ss << "stopped responding to quorum, initiated new election";
+  } else
+    assert(0 == "bad AdminSocket command binding");
+}
+
+void Monitor::handle_signal(int signum)
+{
+  assert(signum == SIGINT || signum == SIGTERM);
+  derr << "*** Got Signal " << sys_siglist[signum] << " ***" << dendl;
+  shutdown();
+}
+
+CompatSet Monitor::get_supported_features()
+{
+  CompatSet::FeatureSet ceph_mon_feature_compat;
+  CompatSet::FeatureSet ceph_mon_feature_ro_compat;
+  CompatSet::FeatureSet ceph_mon_feature_incompat;
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_BASE);
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS);
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
+  return CompatSet(ceph_mon_feature_compat, ceph_mon_feature_ro_compat,
+		   ceph_mon_feature_incompat);
+}
+
+CompatSet Monitor::get_legacy_features()
+{
+  CompatSet::FeatureSet ceph_mon_feature_compat;
+  CompatSet::FeatureSet ceph_mon_feature_ro_compat;
+  CompatSet::FeatureSet ceph_mon_feature_incompat;
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_BASE);
+  return CompatSet(ceph_mon_feature_compat, ceph_mon_feature_ro_compat,
+		   ceph_mon_feature_incompat);
+}
+
+int Monitor::check_features(MonitorDBStore *store)
+{
+  CompatSet required = get_supported_features();
+  CompatSet ondisk;
+
+  read_features_off_disk(store, &ondisk);
+
+  if (!required.writeable(ondisk)) {
+    CompatSet diff = required.unsupported(ondisk);
+    generic_derr << "ERROR: on disk data includes unsupported features: " << diff << dendl;
+    return -EPERM;
+  }
+
+  return 0;
+}
+
+void Monitor::read_features_off_disk(MonitorDBStore *store, CompatSet *features)
+{
+  bufferlist featuresbl;
+  store->get(MONITOR_NAME, COMPAT_SET_LOC, featuresbl);
+  if (featuresbl.length() == 0) {
+    generic_dout(0) << "WARNING: mon fs missing feature list.\n"
+            << "Assuming it is old-style and introducing one." << dendl;
+    //we only want the baseline ~v.18 features assumed to be on disk.
+    //If new features are introduced this code needs to disappear or
+    //be made smarter.
+    *features = get_legacy_features();
+
+    bufferlist bl;
+    features->encode(bl);
+    MonitorDBStore::Transaction t;
+    t.put(MONITOR_NAME, COMPAT_SET_LOC, bl);
+    store->apply_transaction(t);
+  } else {
+    bufferlist::iterator it = featuresbl.begin();
+    features->decode(it);
+  }
+}
+
+void Monitor::read_features()
+{
+  read_features_off_disk(store, &features);
+  dout(10) << "features " << features << dendl;
+
+  apply_compatset_features_to_quorum_requirements();
+  dout(10) << "required_features " << required_features << dendl;
+}
+
+void Monitor::write_features(MonitorDBStore::Transaction &t)
+{
+  bufferlist bl;
+  features.encode(bl);
+  t.put(MONITOR_NAME, COMPAT_SET_LOC, bl);
+}
+
+int Monitor::preinit()
+{
+  lock.Lock();
+
+  dout(1) << "preinit fsid " << monmap->fsid << dendl;
+  
+  assert(!logger);
+  {
+    PerfCountersBuilder pcb(g_ceph_context, "mon", l_mon_first, l_mon_last);
+    // ...
+    logger = pcb.create_perf_counters();
+    cct->get_perfcounters_collection()->add(logger);
+  }
+
+  assert(!cluster_logger);
+  {
+    PerfCountersBuilder pcb(g_ceph_context, "cluster", l_cluster_first, l_cluster_last);
+    pcb.add_u64(l_cluster_num_mon, "num_mon");
+    pcb.add_u64(l_cluster_num_mon_quorum, "num_mon_quorum");
+    pcb.add_u64(l_cluster_num_osd, "num_osd");
+    pcb.add_u64(l_cluster_num_osd_up, "num_osd_up");
+    pcb.add_u64(l_cluster_num_osd_in, "num_osd_in");
+    pcb.add_u64(l_cluster_osd_epoch, "osd_epoch");
+    pcb.add_u64(l_cluster_osd_kb, "osd_kb");
+    pcb.add_u64(l_cluster_osd_kb_used, "osd_kb_used");
+    pcb.add_u64(l_cluster_osd_kb_avail, "osd_kb_avail");
+    pcb.add_u64(l_cluster_num_pool, "num_pool");
+    pcb.add_u64(l_cluster_num_pg, "num_pg");
+    pcb.add_u64(l_cluster_num_pg_active_clean, "num_pg_active_clean");
+    pcb.add_u64(l_cluster_num_pg_active, "num_pg_active");
+    pcb.add_u64(l_cluster_num_pg_peering, "num_pg_peering");
+    pcb.add_u64(l_cluster_num_object, "num_object");
+    pcb.add_u64(l_cluster_num_object_degraded, "num_object_degraded");
+    pcb.add_u64(l_cluster_num_object_unfound, "num_object_unfound");
+    pcb.add_u64(l_cluster_num_bytes, "num_bytes");
+    pcb.add_u64(l_cluster_num_mds_up, "num_mds_up");
+    pcb.add_u64(l_cluster_num_mds_in, "num_mds_in");
+    pcb.add_u64(l_cluster_num_mds_failed, "num_mds_failed");
+    pcb.add_u64(l_cluster_mds_epoch, "mds_epoch");
+    cluster_logger = pcb.create_perf_counters();
+  }
+
+  // verify cluster_uuid
+  {
+    int r = check_fsid();
+    if (r == -ENOENT)
+      r = write_fsid();
+    if (r < 0) {
+      lock.Unlock();
+      return r;
+    }
+  }
+
+  // open compatset
+  read_features();
+
+  // have we ever joined a quorum?
+  has_ever_joined = (store->get(MONITOR_NAME, "joined") != 0);
+  dout(10) << "has_ever_joined = " << (int)has_ever_joined << dendl;
+
+  if (!has_ever_joined) {
+    // impose initial quorum restrictions?
+    list initial_members;
+    get_str_list(g_conf->mon_initial_members, initial_members);
+
+    if (!initial_members.empty()) {
+      dout(1) << " initial_members " << initial_members << ", filtering seed monmap" << dendl;
+
+      monmap->set_initial_members(g_ceph_context, initial_members, name, messenger->get_myaddr(),
+				  &extra_probe_peers);
+
+      dout(10) << " monmap is " << *monmap << dendl;
+      dout(10) << " extra probe peers " << extra_probe_peers << dendl;
+    }
+  } else if (!monmap->contains(name)) {
+    derr << "not in monmap and have been in a quorum before; "
+         << "must have been removed" << dendl;
+    if (g_conf->mon_force_quorum_join) {
+      dout(0) << "we should have died but "
+              << "'mon_force_quorum_join' is set -- allowing boot" << dendl;
+    } else {
+      derr << "commit suicide!" << dendl;
+      return -ENOENT;
+    }
+  }
+
+  {
+    // We have a potentially inconsistent store state in hands. Get rid of it
+    // and start fresh.
+    bool clear_store = false;
+    if (store->exists("mon_sync", "in_sync")) {
+      dout(1) << __func__ << " clean up potentially inconsistent store state"
+	      << dendl;
+      clear_store = true;
+    }
+
+    if (store->get("mon_sync", "force_sync") > 0) {
+      dout(1) << __func__ << " force sync by clearing store state" << dendl;
+      clear_store = true;
+    }
+
+    if (clear_store) {
+      set sync_prefixes = get_sync_targets_names();
+      store->clear(sync_prefixes);
+    }
+  }
+
+  sync_last_committed_floor = store->get("mon_sync", "last_committed_floor");
+  dout(10) << "sync_last_committed_floor " << sync_last_committed_floor << dendl;
+
+  init_paxos();
+  health_monitor->init();
+
+  int r;
+
+  if (is_keyring_required()) {
+    // we need to bootstrap authentication keys so we can form an
+    // initial quorum.
+    if (authmon()->get_last_committed() == 0) {
+      dout(10) << "loading initial keyring to bootstrap authentication for mkfs" << dendl;
+      bufferlist bl;
+      store->get("mkfs", "keyring", bl);
+      KeyRing keyring;
+      bufferlist::iterator p = bl.begin();
+      ::decode(keyring, p);
+      extract_save_mon_key(keyring);
+    }
+
+    string keyring_loc = g_conf->mon_data + "/keyring";
+
+    r = keyring.load(cct, keyring_loc);
+    if (r < 0) {
+      EntityName mon_name;
+      mon_name.set_type(CEPH_ENTITY_TYPE_MON);
+      EntityAuth mon_key;
+      if (key_server.get_auth(mon_name, mon_key)) {
+	dout(1) << "copying mon. key from old db to external keyring" << dendl;
+	keyring.add(mon_name, mon_key);
+	bufferlist bl;
+	keyring.encode_plaintext(bl);
+	write_default_keyring(bl);
+      } else {
+	derr << "unable to load initial keyring " << g_conf->keyring << dendl;
+	lock.Unlock();
+	return r;
+      }
+    }
+  }
+
+  admin_hook = new AdminHook(this);
+  AdminSocket* admin_socket = cct->get_admin_socket();
+
+  // unlock while registering to avoid mon_lock -> admin socket lock dependency.
+  lock.Unlock();
+  r = admin_socket->register_command("mon_status", "mon_status", admin_hook,
+				     "show current monitor status");
+  assert(r == 0);
+  if (g_conf->mon_advanced_debug_mode) {
+    r = admin_socket->register_command("osdmonitor_prepare_command", "osdmonitor_prepare_command", admin_hook,
+				       "call OSDMonitor::prepare_command");
+    assert(r == 0);
+  }
+  r = admin_socket->register_command("quorum_status", "quorum_status",
+				     admin_hook, "show current quorum status");
+  assert(r == 0);
+  r = admin_socket->register_command("sync_force",
+				     "sync_force name=validate,"
+				     "type=CephChoices,"
+			             "strings=--yes-i-really-mean-it",
+				     admin_hook,
+				     "force sync of and clear monitor store");
+  assert(r == 0);
+  r = admin_socket->register_command("add_bootstrap_peer_hint",
+				     "add_bootstrap_peer_hint name=addr,"
+				     "type=CephIPAddr",
+				     admin_hook,
+				     "add peer address as potential bootstrap"
+				     " peer for cluster bringup");
+  assert(r == 0);
+  r = admin_socket->register_command("quorum enter", "quorum enter",
+                                     admin_hook,
+                                     "force monitor back into quorum");
+  assert(r == 0);
+  r = admin_socket->register_command("quorum exit", "quorum exit",
+                                     admin_hook,
+                                     "force monitor out of the quorum");
+  assert(r == 0);
+  lock.Lock();
+
+  lock.Unlock();
+  return 0;
+}
+
+int Monitor::init()
+{
+  dout(2) << "init" << dendl;
+  lock.Lock();
+
+  // start ticker
+  timer.init();
+  new_tick();
+
+  // i'm ready!
+  messenger->add_dispatcher_tail(this);
+
+  bootstrap();
+
+  // encode command sets
+  const MonCommand *cmds;
+  int cmdsize;
+  get_locally_supported_monitor_commands(&cmds, &cmdsize);
+  MonCommand::encode_array(cmds, cmdsize, supported_commands_bl);
+  get_classic_monitor_commands(&cmds, &cmdsize);
+  MonCommand::encode_array(cmds, cmdsize, classic_commands_bl);
+
+  lock.Unlock();
+  return 0;
+}
+
+void Monitor::init_paxos()
+{
+  dout(10) << __func__ << dendl;
+  paxos->init();
+
+  // init services
+  for (int i = 0; i < PAXOS_NUM; ++i) {
+    paxos_service[i]->init();
+  }
+
+  refresh_from_paxos(NULL);
+}
+
+void Monitor::refresh_from_paxos(bool *need_bootstrap)
+{
+  dout(10) << __func__ << dendl;
+
+  bufferlist bl;
+  int r = store->get(MONITOR_NAME, "cluster_fingerprint", bl);
+  if (r >= 0) {
+    try {
+      bufferlist::iterator p = bl.begin();
+      ::decode(fingerprint, p);
+    }
+    catch (buffer::error& e) {
+      dout(10) << __func__ << " failed to decode cluster_fingerprint" << dendl;
+    }
+  } else {
+    dout(10) << __func__ << " no cluster_fingerprint" << dendl;
+  }
+
+  for (int i = 0; i < PAXOS_NUM; ++i) {
+    paxos_service[i]->refresh(need_bootstrap);
+  }
+  for (int i = 0; i < PAXOS_NUM; ++i) {
+    paxos_service[i]->post_paxos_update();
+  }
+}
+
+void Monitor::register_cluster_logger()
+{
+  if (!cluster_logger_registered) {
+    dout(10) << "register_cluster_logger" << dendl;
+    cluster_logger_registered = true;
+    cct->get_perfcounters_collection()->add(cluster_logger);
+  } else {
+    dout(10) << "register_cluster_logger - already registered" << dendl;
+  }
+}
+
+void Monitor::unregister_cluster_logger()
+{
+  if (cluster_logger_registered) {
+    dout(10) << "unregister_cluster_logger" << dendl;
+    cluster_logger_registered = false;
+    cct->get_perfcounters_collection()->remove(cluster_logger);
+  } else {
+    dout(10) << "unregister_cluster_logger - not registered" << dendl;
+  }
+}
+
+void Monitor::update_logger()
+{
+  cluster_logger->set(l_cluster_num_mon, monmap->size());
+  cluster_logger->set(l_cluster_num_mon_quorum, quorum.size());
+}
+
+void Monitor::shutdown()
+{
+  dout(1) << "shutdown" << dendl;
+  lock.Lock();
+
+  state = STATE_SHUTDOWN;
+
+  if (admin_hook) {
+    AdminSocket* admin_socket = cct->get_admin_socket();
+    admin_socket->unregister_command("mon_status");
+    admin_socket->unregister_command("quorum_status");
+    admin_socket->unregister_command("sync_force");
+    admin_socket->unregister_command("add_bootstrap_peer_hint");
+    delete admin_hook;
+    admin_hook = NULL;
+  }
+
+  elector.shutdown();
+
+  if (logger) {
+    cct->get_perfcounters_collection()->remove(logger);
+    delete logger;
+    logger = NULL;
+  }
+  if (cluster_logger) {
+    if (cluster_logger_registered)
+      cct->get_perfcounters_collection()->remove(cluster_logger);
+    delete cluster_logger;
+    cluster_logger = NULL;
+  }
+  
+  // clean up
+  paxos->shutdown();
+  for (vector::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p)
+    (*p)->shutdown();
+  health_monitor->shutdown();
+
+  finish_contexts(g_ceph_context, waitfor_quorum, -ECANCELED);
+  finish_contexts(g_ceph_context, maybe_wait_for_quorum, -ECANCELED);
+
+  timer.shutdown();
+
+  remove_all_sessions();
+
+  // unlock before msgr shutdown...
+  lock.Unlock();
+
+  messenger->shutdown();  // last thing!  ceph_mon.cc will delete mon.
+}
+
+void Monitor::bootstrap()
+{
+  dout(10) << "bootstrap" << dendl;
+
+  sync_reset_requester();
+  unregister_cluster_logger();
+  cancel_probe_timeout();
+
+  // note my rank
+  int newrank = monmap->get_rank(messenger->get_myaddr());
+  if (newrank < 0 && rank >= 0) {
+    // was i ever part of the quorum?
+    if (has_ever_joined) {
+      dout(0) << " removed from monmap, suicide." << dendl;
+      exit(0);
+    }
+  }
+  if (newrank != rank) {
+    dout(0) << " my rank is now " << newrank << " (was " << rank << ")" << dendl;
+    messenger->set_myname(entity_name_t::MON(newrank));
+    rank = newrank;
+
+    // reset all connections, or else our peers will think we are someone else.
+    messenger->mark_down_all();
+  }
+
+  // reset
+  state = STATE_PROBING;
+
+  _reset();
+
+  // sync store
+  if (g_conf->mon_compact_on_bootstrap) {
+    dout(10) << "bootstrap -- triggering compaction" << dendl;
+    store->compact();
+    dout(10) << "bootstrap -- finished compaction" << dendl;
+  }
+
+  // singleton monitor?
+  if (monmap->size() == 1 && rank == 0) {
+    win_standalone_election();
+    return;
+  }
+
+  reset_probe_timeout();
+
+  // i'm outside the quorum
+  if (monmap->contains(name))
+    outside_quorum.insert(name);
+
+  // probe monitors
+  dout(10) << "probing other monitors" << dendl;
+  for (unsigned i = 0; i < monmap->size(); i++) {
+    if ((int)i != rank)
+      messenger->send_message(new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined),
+			      monmap->get_inst(i));
+  }
+  for (set::iterator p = extra_probe_peers.begin();
+       p != extra_probe_peers.end();
+       ++p) {
+    if (*p != messenger->get_myaddr()) {
+      entity_inst_t i;
+      i.name = entity_name_t::MON(-1);
+      i.addr = *p;
+      messenger->send_message(new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined), i);
+    }
+  }
+}
+
+void Monitor::_osdmonitor_prepare_command(cmdmap_t& cmdmap, ostream& ss)
+{
+  if (!is_leader()) {
+    ss << "mon must be a leader";
+    return;
+  }
+
+  string cmd;
+  cmd_getval(g_ceph_context, cmdmap, "prepare", cmd);
+  cmdmap["prefix"] = cmdmap["prepare"];
+  
+  OSDMonitor *monitor = osdmon();
+  MMonCommand *m = static_cast((new MMonCommand())->get());
+  if (monitor->prepare_command_impl(m, cmdmap))
+    ss << "true";
+  else
+    ss << "false";
+  m->put();
+}
+
+void Monitor::_add_bootstrap_peer_hint(string cmd, cmdmap_t& cmdmap, ostream& ss)
+{
+  string addrstr;
+  if (!cmd_getval(g_ceph_context, cmdmap, "addr", addrstr)) {
+    ss << "unable to parse address string value '"
+         << cmd_vartype_stringify(cmdmap["addr"]) << "'";
+    return;
+  }
+  dout(10) << "_add_bootstrap_peer_hint '" << cmd << "' '"
+           << addrstr << "'" << dendl;
+
+  entity_addr_t addr;
+  const char *end = 0;
+  if (!addr.parse(addrstr.c_str(), &end)) {
+    ss << "failed to parse addr '" << addrstr << "'; syntax is 'add_bootstrap_peer_hint ip[:port]'";
+    return;
+  }
+
+  if (is_leader() || is_peon()) {
+    ss << "mon already active; ignoring bootstrap hint";
+    return;
+  }
+
+  if (addr.get_port() == 0)
+    addr.set_port(CEPH_MON_PORT);
+
+  extra_probe_peers.insert(addr);
+  ss << "adding peer " << addr << " to list: " << extra_probe_peers;
+}
+
+// called by bootstrap(), or on leader|peon -> electing
+void Monitor::_reset()
+{
+  dout(10) << __func__ << dendl;
+
+  assert(state == STATE_ELECTING ||
+	 state == STATE_PROBING);
+
+  cancel_probe_timeout();
+  timecheck_finish();
+
+  leader_since = utime_t();
+  if (!quorum.empty()) {
+    exited_quorum = ceph_clock_now(g_ceph_context);
+  }
+  quorum.clear();
+  outside_quorum.clear();
+
+  scrub_reset();
+
+  paxos->restart();
+
+  for (vector::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p)
+    (*p)->restart();
+  health_monitor->finish();
+}
+
+
+// -----------------------------------------------------------
+// sync
+
+set Monitor::get_sync_targets_names()
+{
+  set targets;
+  targets.insert(paxos->get_name());
+  for (int i = 0; i < PAXOS_NUM; ++i)
+    paxos_service[i]->get_store_prefixes(targets);
+
+  return targets;
+}
+
+
+void Monitor::sync_timeout()
+{
+  dout(10) << __func__ << dendl;
+  assert(state == STATE_SYNCHRONIZING);
+  bootstrap();
+}
+
+void Monitor::sync_obtain_latest_monmap(bufferlist &bl)
+{
+  dout(1) << __func__ << dendl;
+
+  MonMap latest_monmap;
+
+  // Grab latest monmap from MonmapMonitor
+  bufferlist monmon_bl;
+  int err = monmon()->get_monmap(monmon_bl);
+  if (err < 0) {
+    if (err != -ENOENT) {
+      derr << __func__
+           << " something wrong happened while reading the store: "
+           << cpp_strerror(err) << dendl;
+      assert(0 == "error reading the store");
+    }
+  } else {
+    latest_monmap.decode(monmon_bl);
+  }
+
+  // Grab last backed up monmap (if any) and compare epochs
+  if (store->exists("mon_sync", "latest_monmap")) {
+    bufferlist backup_bl;
+    int err = store->get("mon_sync", "latest_monmap", backup_bl);
+    if (err < 0) {
+      assert(err != -ENOENT);
+      derr << __func__
+           << " something wrong happened while reading the store: "
+           << cpp_strerror(err) << dendl;
+      assert(0 == "error reading the store");
+    }
+    assert(backup_bl.length() > 0);
+
+    MonMap backup_monmap;
+    backup_monmap.decode(backup_bl);
+
+    if (backup_monmap.epoch > latest_monmap.epoch)
+      latest_monmap = backup_monmap;
+  }
+
+  // Check if our current monmap's epoch is greater than the one we've
+  // got so far.
+  if (monmap->epoch > latest_monmap.epoch)
+    latest_monmap = *monmap;
+
+  dout(1) << __func__ << " obtained monmap e" << latest_monmap.epoch << dendl;
+
+  latest_monmap.encode(bl, quorum_features);
+}
+
+void Monitor::sync_reset_requester()
+{
+  dout(10) << __func__ << dendl;
+
+  if (sync_timeout_event) {
+    timer.cancel_event(sync_timeout_event);
+    sync_timeout_event = NULL;
+  }
+
+  sync_provider = entity_inst_t();
+  sync_cookie = 0;
+  sync_full = false;
+  sync_start_version = 0;
+}
+
+void Monitor::sync_reset_provider()
+{
+  dout(10) << __func__ << dendl;
+  sync_providers.clear();
+}
+
+void Monitor::sync_start(entity_inst_t &other, bool full)
+{
+  dout(10) << __func__ << " " << other << (full ? " full" : " recent") << dendl;
+
+  assert(state == STATE_PROBING ||
+	 state == STATE_SYNCHRONIZING);
+  state = STATE_SYNCHRONIZING;
+
+  // make sure are not a provider for anyone!
+  sync_reset_provider();
+
+  sync_full = full;
+
+  if (sync_full) {
+    // stash key state, and mark that we are syncing
+    MonitorDBStore::Transaction t;
+    sync_stash_critical_state(&t);
+    t.put("mon_sync", "in_sync", 1);
+
+    sync_last_committed_floor = MAX(sync_last_committed_floor, paxos->get_version());
+    dout(10) << __func__ << " marking sync in progress, storing sync_last_committed_floor "
+	     << sync_last_committed_floor << dendl;
+    t.put("mon_sync", "last_committed_floor", sync_last_committed_floor);
+
+    store->apply_transaction(t);
+
+    assert(g_conf->mon_sync_requester_kill_at != 1);
+
+    // clear the underlying store
+    set targets = get_sync_targets_names();
+    dout(10) << __func__ << " clearing prefixes " << targets << dendl;
+    store->clear(targets);
+
+    // make sure paxos knows it has been reset.  this prevents a
+    // bootstrap and then different probe reply order from possibly
+    // deciding a partial or no sync is needed.
+    paxos->init();
+
+    assert(g_conf->mon_sync_requester_kill_at != 2);
+  }
+
+  // assume 'other' as the leader. We will update the leader once we receive
+  // a reply to the sync start.
+  sync_provider = other;
+
+  sync_reset_timeout();
+
+  MMonSync *m = new MMonSync(sync_full ? MMonSync::OP_GET_COOKIE_FULL : MMonSync::OP_GET_COOKIE_RECENT);
+  if (!sync_full)
+    m->last_committed = paxos->get_version();
+  messenger->send_message(m, sync_provider);
+}
+
+void Monitor::sync_stash_critical_state(MonitorDBStore::Transaction *t)
+{
+  dout(10) << __func__ << dendl;
+  bufferlist backup_monmap;
+  sync_obtain_latest_monmap(backup_monmap);
+  assert(backup_monmap.length() > 0);
+  t->put("mon_sync", "latest_monmap", backup_monmap);
+}
+
+void Monitor::sync_reset_timeout()
+{
+  dout(10) << __func__ << dendl;
+  if (sync_timeout_event)
+    timer.cancel_event(sync_timeout_event);
+  sync_timeout_event = new C_SyncTimeout(this);
+  timer.add_event_after(g_conf->mon_sync_timeout, sync_timeout_event);
+}
+
+void Monitor::sync_finish(version_t last_committed)
+{
+  dout(10) << __func__ << " lc " << last_committed << " from " << sync_provider << dendl;
+
+  assert(g_conf->mon_sync_requester_kill_at != 7);
+
+  if (sync_full) {
+    // finalize the paxos commits
+    MonitorDBStore::Transaction tx;
+    paxos->read_and_prepare_transactions(&tx, sync_start_version, last_committed);
+    tx.put(paxos->get_name(), "last_committed", last_committed);
+
+    dout(30) << __func__ << " final tx dump:\n";
+    JSONFormatter f(true);
+    tx.dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    store->apply_transaction(tx);
+  }
+
+  assert(g_conf->mon_sync_requester_kill_at != 8);
+
+  MonitorDBStore::Transaction t;
+  t.erase("mon_sync", "in_sync");
+  t.erase("mon_sync", "force_sync");
+  t.erase("mon_sync", "last_committed_floor");
+  store->apply_transaction(t);
+
+  assert(g_conf->mon_sync_requester_kill_at != 9);
+
+  init_paxos();
+
+  assert(g_conf->mon_sync_requester_kill_at != 10);
+
+  bootstrap();
+}
+
+void Monitor::handle_sync(MMonSync *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  switch (m->op) {
+
+    // provider ---------
+
+  case MMonSync::OP_GET_COOKIE_FULL:
+  case MMonSync::OP_GET_COOKIE_RECENT:
+    handle_sync_get_cookie(m);
+    break;
+  case MMonSync::OP_GET_CHUNK:
+    handle_sync_get_chunk(m);
+    break;
+
+    // client -----------
+
+  case MMonSync::OP_COOKIE:
+    handle_sync_cookie(m);
+    break;
+
+  case MMonSync::OP_CHUNK:
+  case MMonSync::OP_LAST_CHUNK:
+    handle_sync_chunk(m);
+    break;
+  case MMonSync::OP_NO_COOKIE:
+    handle_sync_no_cookie(m);
+    break;
+
+  default:
+    dout(0) << __func__ << " unknown op " << m->op << dendl;
+    assert(0 == "unknown op");
+  }
+  m->put();
+}
+
+// leader
+
+void Monitor::_sync_reply_no_cookie(MMonSync *m)
+{
+  MMonSync *reply = new MMonSync(MMonSync::OP_NO_COOKIE, m->cookie);
+  messenger->send_message(reply, m->get_connection());
+}
+
+void Monitor::handle_sync_get_cookie(MMonSync *m)
+{
+  if (is_synchronizing()) {
+    _sync_reply_no_cookie(m);
+    return;
+  }
+
+  assert(g_conf->mon_sync_provider_kill_at != 1);
+
+  // make sure they can understand us.
+  if ((required_features ^ m->get_connection()->get_features()) &
+      required_features) {
+    dout(5) << " ignoring peer mon." << m->get_source().num()
+	    << " has features " << std::hex
+	    << m->get_connection()->get_features()
+	    << " but we require " << required_features << std::dec << dendl;
+    return;
+  }
+
+  // make up a unique cookie.  include election epoch (which persists
+  // across restarts for the whole cluster) and a counter for this
+  // process instance.  there is no need to be unique *across*
+  // monitors, though.
+  uint64_t cookie = ((unsigned long long)elector.get_epoch() << 24) + ++sync_provider_count;
+  assert(sync_providers.count(cookie) == 0);
+
+  dout(10) << __func__ << " cookie " << cookie << " for " << m->get_source_inst() << dendl;
+
+  SyncProvider& sp = sync_providers[cookie];
+  sp.cookie = cookie;
+  sp.entity = m->get_source_inst();
+  sp.reset_timeout(g_ceph_context, g_conf->mon_sync_timeout * 2);
+
+  set sync_targets;
+  if (m->op == MMonSync::OP_GET_COOKIE_FULL) {
+    // full scan
+    sync_targets = get_sync_targets_names();
+    sp.last_committed = paxos->get_version();
+    sp.synchronizer = store->get_synchronizer(sp.last_key, sync_targets);
+    sp.full = true;
+    dout(10) << __func__ << " will sync prefixes " << sync_targets << dendl;
+  } else {
+    // just catch up paxos
+    sp.last_committed = m->last_committed;
+  }
+  dout(10) << __func__ << " will sync from version " << sp.last_committed << dendl;
+
+  MMonSync *reply = new MMonSync(MMonSync::OP_COOKIE, sp.cookie);
+  reply->last_committed = sp.last_committed;
+  messenger->send_message(reply, m->get_connection());
+}
+
+void Monitor::handle_sync_get_chunk(MMonSync *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+
+  if (sync_providers.count(m->cookie) == 0) {
+    dout(10) << __func__ << " no cookie " << m->cookie << dendl;
+    _sync_reply_no_cookie(m);
+    return;
+  }
+
+  assert(g_conf->mon_sync_provider_kill_at != 2);
+
+  SyncProvider& sp = sync_providers[m->cookie];
+  sp.reset_timeout(g_ceph_context, g_conf->mon_sync_timeout * 2);
+
+  if (sp.last_committed < paxos->get_first_committed() &&
+      paxos->get_first_committed() > 1) {
+    dout(10) << __func__ << " sync requester fell behind paxos, their lc " << sp.last_committed
+	     << " < our fc " << paxos->get_first_committed() << dendl;
+    sync_providers.erase(m->cookie);
+    _sync_reply_no_cookie(m);
+    return;
+  }
+
+  MMonSync *reply = new MMonSync(MMonSync::OP_CHUNK, sp.cookie);
+  MonitorDBStore::Transaction tx;
+
+  int left = g_conf->mon_sync_max_payload_size;
+  while (sp.last_committed < paxos->get_version() && left > 0) {
+    bufferlist bl;
+    sp.last_committed++;
+    store->get(paxos->get_name(), sp.last_committed, bl);
+    tx.put(paxos->get_name(), sp.last_committed, bl);
+    left -= bl.length();
+    dout(20) << __func__ << " including paxos state " << sp.last_committed << dendl;
+  }
+  reply->last_committed = sp.last_committed;
+
+  if (sp.full && left > 0) {
+    sp.synchronizer->get_chunk_tx(tx, left);
+    sp.last_key = sp.synchronizer->get_last_key();
+    reply->last_key = sp.last_key;
+  }
+
+  if ((sp.full && sp.synchronizer->has_next_chunk()) ||
+      sp.last_committed < paxos->get_version()) {
+    dout(10) << __func__ << " chunk, through version " << sp.last_committed << " key " << sp.last_key << dendl;
+  } else {
+    dout(10) << __func__ << " last chunk, through version " << sp.last_committed << " key " << sp.last_key << dendl;
+    reply->op = MMonSync::OP_LAST_CHUNK;
+
+    assert(g_conf->mon_sync_provider_kill_at != 3);
+
+    // clean up our local state
+    sync_providers.erase(sp.cookie);
+  }
+
+  ::encode(tx, reply->chunk_bl);
+
+  messenger->send_message(reply, m->get_connection());
+}
+
+// requester
+
+void Monitor::handle_sync_cookie(MMonSync *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  if (sync_cookie) {
+    dout(10) << __func__ << " already have a cookie, ignoring" << dendl;
+    return;
+  }
+  if (m->get_source_inst() != sync_provider) {
+    dout(10) << __func__ << " source does not match, discarding" << dendl;
+    return;
+  }
+  sync_cookie = m->cookie;
+  sync_start_version = m->last_committed;
+
+  sync_reset_timeout();
+  sync_get_next_chunk();
+
+  assert(g_conf->mon_sync_requester_kill_at != 3);
+}
+
+void Monitor::sync_get_next_chunk()
+{
+  dout(20) << __func__ << " cookie " << sync_cookie << " provider " << sync_provider << dendl;
+  if (g_conf->mon_inject_sync_get_chunk_delay > 0) {
+    dout(20) << __func__ << " injecting delay of " << g_conf->mon_inject_sync_get_chunk_delay << dendl;
+    usleep((long long)(g_conf->mon_inject_sync_get_chunk_delay * 1000000.0));
+  }
+  MMonSync *r = new MMonSync(MMonSync::OP_GET_CHUNK, sync_cookie);
+  messenger->send_message(r, sync_provider);
+
+  assert(g_conf->mon_sync_requester_kill_at != 4);
+}
+
+void Monitor::handle_sync_chunk(MMonSync *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+
+  if (m->cookie != sync_cookie) {
+    dout(10) << __func__ << " cookie does not match, discarding" << dendl;
+    return;
+  }
+  if (m->get_source_inst() != sync_provider) {
+    dout(10) << __func__ << " source does not match, discarding" << dendl;
+    return;
+  }
+
+  assert(state == STATE_SYNCHRONIZING);
+  assert(g_conf->mon_sync_requester_kill_at != 5);
+
+  MonitorDBStore::Transaction tx;
+  tx.append_from_encoded(m->chunk_bl);
+
+  dout(30) << __func__ << " tx dump:\n";
+  JSONFormatter f(true);
+  tx.dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  store->apply_transaction(tx);
+
+  assert(g_conf->mon_sync_requester_kill_at != 6);
+
+  if (!sync_full) {
+    dout(10) << __func__ << " applying recent paxos transactions as we go" << dendl;
+    MonitorDBStore::Transaction tx;
+    paxos->read_and_prepare_transactions(&tx, paxos->get_version() + 1, m->last_committed);
+    tx.put(paxos->get_name(), "last_committed", m->last_committed);
+
+    dout(30) << __func__ << " tx dump:\n";
+    JSONFormatter f(true);
+    tx.dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    store->apply_transaction(tx);
+    paxos->init();  // to refresh what we just wrote
+  }
+
+  if (m->op == MMonSync::OP_CHUNK) {
+    sync_reset_timeout();
+    sync_get_next_chunk();
+  } else if (m->op == MMonSync::OP_LAST_CHUNK) {
+    sync_finish(m->last_committed);
+  }
+}
+
+void Monitor::handle_sync_no_cookie(MMonSync *m)
+{
+  dout(10) << __func__ << dendl;
+  bootstrap();
+}
+
+void Monitor::sync_trim_providers()
+{
+  dout(20) << __func__ << dendl;
+
+  utime_t now = ceph_clock_now(g_ceph_context);
+  map::iterator p = sync_providers.begin();
+  while (p != sync_providers.end()) {
+    if (now > p->second.timeout) {
+      dout(10) << __func__ << " expiring cookie " << p->second.cookie << " for " << p->second.entity << dendl;
+      sync_providers.erase(p++);
+    } else {
+      ++p;
+    }
+  }
+}
+
+// ---------------------------------------------------
+// probe
+
+void Monitor::cancel_probe_timeout()
+{
+  if (probe_timeout_event) {
+    dout(10) << "cancel_probe_timeout " << probe_timeout_event << dendl;
+    timer.cancel_event(probe_timeout_event);
+    probe_timeout_event = NULL;
+  } else {
+    dout(10) << "cancel_probe_timeout (none scheduled)" << dendl;
+  }
+}
+
+void Monitor::reset_probe_timeout()
+{
+  cancel_probe_timeout();
+  probe_timeout_event = new C_ProbeTimeout(this);
+  double t = g_conf->mon_probe_timeout;
+  timer.add_event_after(t, probe_timeout_event);
+  dout(10) << "reset_probe_timeout " << probe_timeout_event << " after " << t << " seconds" << dendl;
+}
+
+void Monitor::probe_timeout(int r)
+{
+  dout(4) << "probe_timeout " << probe_timeout_event << dendl;
+  assert(is_probing() || is_synchronizing());
+  assert(probe_timeout_event);
+  probe_timeout_event = NULL;
+  bootstrap();
+}
+
+void Monitor::handle_probe(MMonProbe *m)
+{
+  dout(10) << "handle_probe " << *m << dendl;
+
+  if (m->fsid != monmap->fsid) {
+    dout(0) << "handle_probe ignoring fsid " << m->fsid << " != " << monmap->fsid << dendl;
+    m->put();
+    return;
+  }
+
+  switch (m->op) {
+  case MMonProbe::OP_PROBE:
+    handle_probe_probe(m);
+    break;
+
+  case MMonProbe::OP_REPLY:
+    handle_probe_reply(m);
+    break;
+
+  case MMonProbe::OP_MISSING_FEATURES:
+    derr << __func__ << " missing features, have " << CEPH_FEATURES_ALL
+	 << ", required " << required_features
+	 << ", missing " << (required_features & ~CEPH_FEATURES_ALL)
+	 << dendl;
+    break;
+
+  default:
+    m->put();
+  }
+}
+
+/**
+ * @todo fix this. This is going to cause trouble.
+ */
+void Monitor::handle_probe_probe(MMonProbe *m)
+{
+  dout(10) << "handle_probe_probe " << m->get_source_inst() << *m
+	   << " features " << m->get_connection()->get_features() << dendl;
+  uint64_t missing = required_features & ~m->get_connection()->get_features();
+  if (missing) {
+    dout(1) << " peer " << m->get_source_addr() << " missing features "
+	    << missing << dendl;
+    if (m->get_connection()->has_feature(CEPH_FEATURE_OSD_PRIMARY_AFFINITY)) {
+      MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_MISSING_FEATURES,
+				   name, has_ever_joined);
+      m->required_features = required_features;
+      messenger->send_message(r, m->get_connection());
+    }
+    m->put();
+    return;
+  }
+
+  MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_REPLY,
+			       name, has_ever_joined);
+  r->name = name;
+  r->quorum = quorum;
+  monmap->encode(r->monmap_bl, m->get_connection()->get_features());
+  r->paxos_first_version = paxos->get_first_committed();
+  r->paxos_last_version = paxos->get_version();
+  messenger->send_message(r, m->get_connection());
+
+  // did we discover a peer here?
+  if (!monmap->contains(m->get_source_addr())) {
+    dout(1) << " adding peer " << m->get_source_addr()
+	    << " to list of hints" << dendl;
+    extra_probe_peers.insert(m->get_source_addr());
+  }
+
+  m->put();
+}
+
+void Monitor::handle_probe_reply(MMonProbe *m)
+{
+  dout(10) << "handle_probe_reply " << m->get_source_inst() << *m << dendl;
+  dout(10) << " monmap is " << *monmap << dendl;
+
+  // discover name and addrs during probing or electing states.
+  if (!is_probing() && !is_electing()) {
+    m->put();
+    return;
+  }
+
+  // newer map, or they've joined a quorum and we haven't?
+  bufferlist mybl;
+  monmap->encode(mybl, m->get_connection()->get_features());
+  // make sure it's actually different; the checks below err toward
+  // taking the other guy's map, which could cause us to loop.
+  if (!mybl.contents_equal(m->monmap_bl)) {
+    MonMap *newmap = new MonMap;
+    newmap->decode(m->monmap_bl);
+    if (m->has_ever_joined && (newmap->get_epoch() > monmap->get_epoch() ||
+			       !has_ever_joined)) {
+      dout(10) << " got newer/committed monmap epoch " << newmap->get_epoch()
+	       << ", mine was " << monmap->get_epoch() << dendl;
+      delete newmap;
+      monmap->decode(m->monmap_bl);
+      m->put();
+
+      bootstrap();
+      return;
+    }
+    delete newmap;
+  }
+
+  // rename peer?
+  string peer_name = monmap->get_name(m->get_source_addr());
+  if (monmap->get_epoch() == 0 && peer_name.find("noname-") == 0) {
+    dout(10) << " renaming peer " << m->get_source_addr() << " "
+	     << peer_name << " -> " << m->name << " in my monmap"
+	     << dendl;
+    monmap->rename(peer_name, m->name);
+
+    if (is_electing()) {
+      m->put();
+      bootstrap();
+      return;
+    }
+  } else {
+    dout(10) << " peer name is " << peer_name << dendl;
+  }
+
+  // new initial peer?
+  if (monmap->get_epoch() == 0 &&
+      monmap->contains(m->name) &&
+      monmap->get_addr(m->name).is_blank_ip()) {
+    dout(1) << " learned initial mon " << m->name << " addr " << m->get_source_addr() << dendl;
+    monmap->set_addr(m->name, m->get_source_addr());
+    m->put();
+
+    bootstrap();
+    return;
+  }
+
+  // end discover phase
+  if (!is_probing()) {
+    m->put();
+    return;
+  }
+
+  assert(paxos != NULL);
+
+  if (is_synchronizing()) {
+    dout(10) << " currently syncing" << dendl;
+    m->put();
+    return;
+  }
+
+  entity_inst_t other = m->get_source_inst();
+
+  if (m->paxos_last_version < sync_last_committed_floor) {
+    dout(10) << " peer paxos versions [" << m->paxos_first_version
+	     << "," << m->paxos_last_version << "] < my sync_last_committed_floor "
+	     << sync_last_committed_floor << ", ignoring"
+	     << dendl;
+  } else {
+    if (paxos->get_version() < m->paxos_first_version &&
+	m->paxos_first_version > 1) {  // no need to sync if we're 0 and they start at 1.
+      dout(10) << " peer paxos versions [" << m->paxos_first_version
+	       << "," << m->paxos_last_version << "]"
+	       << " vs my version " << paxos->get_version()
+	       << " (too far ahead)"
+	       << dendl;
+      cancel_probe_timeout();
+      sync_start(other, true);
+      m->put();
+      return;
+    }
+    if (paxos->get_version() + g_conf->paxos_max_join_drift < m->paxos_last_version) {
+      dout(10) << " peer paxos version " << m->paxos_last_version
+	       << " vs my version " << paxos->get_version()
+	       << " (too far ahead)"
+	       << dendl;
+      cancel_probe_timeout();
+      sync_start(other, false);
+      m->put();
+      return;
+    }
+  }
+
+  // is there an existing quorum?
+  if (m->quorum.size()) {
+    dout(10) << " existing quorum " << m->quorum << dendl;
+
+    dout(10) << " peer paxos version " << m->paxos_last_version
+             << " vs my version " << paxos->get_version()
+             << " (ok)"
+             << dendl;
+
+    if (monmap->contains(name) &&
+        !monmap->get_addr(name).is_blank_ip()) {
+      // i'm part of the cluster; just initiate a new election
+      start_election();
+    } else {
+      dout(10) << " ready to join, but i'm not in the monmap or my addr is blank, trying to join" << dendl;
+      messenger->send_message(new MMonJoin(monmap->fsid, name, messenger->get_myaddr()),
+                              monmap->get_inst(*m->quorum.begin()));
+    }
+  } else {
+    if (monmap->contains(m->name)) {
+      dout(10) << " mon." << m->name << " is outside the quorum" << dendl;
+      outside_quorum.insert(m->name);
+    } else {
+      dout(10) << " mostly ignoring mon." << m->name << ", not part of monmap" << dendl;
+      m->put();
+      return;
+    }
+
+    unsigned need = monmap->size() / 2 + 1;
+    dout(10) << " outside_quorum now " << outside_quorum << ", need " << need << dendl;
+    if (outside_quorum.size() >= need) {
+      if (outside_quorum.count(name)) {
+        dout(10) << " that's enough to form a new quorum, calling election" << dendl;
+        start_election();
+      } else {
+        dout(10) << " that's enough to form a new quorum, but it does not include me; waiting" << dendl;
+      }
+    } else {
+      dout(10) << " that's not yet enough for a new quorum, waiting" << dendl;
+    }
+  }
+  m->put();
+}
+
+void Monitor::join_election()
+{
+  dout(10) << __func__ << dendl;
+  state = STATE_ELECTING;
+  _reset();
+}
+
+void Monitor::start_election()
+{
+  dout(10) << "start_election" << dendl;
+  state = STATE_ELECTING;
+  _reset();
+
+  cancel_probe_timeout();
+
+  clog.info() << "mon." << name << " calling new monitor election\n";
+  elector.call_election();
+}
+
+void Monitor::win_standalone_election()
+{
+  dout(1) << "win_standalone_election" << dendl;
+
+  // bump election epoch, in case the previous epoch included other
+  // monitors; we need to be able to make the distinction.
+  elector.advance_epoch();
+
+  rank = monmap->get_rank(name);
+  assert(rank == 0);
+  set q;
+  q.insert(rank);
+
+  const MonCommand *my_cmds;
+  int cmdsize;
+  get_locally_supported_monitor_commands(&my_cmds, &cmdsize);
+  win_election(1, q, CEPH_FEATURES_ALL, my_cmds, cmdsize, NULL);
+}
+
+const utime_t& Monitor::get_leader_since() const
+{
+  assert(state == STATE_LEADER);
+  return leader_since;
+}
+
+epoch_t Monitor::get_epoch()
+{
+  return elector.get_epoch();
+}
+
+void Monitor::win_election(epoch_t epoch, set& active, uint64_t features,
+                           const MonCommand *cmdset, int cmdsize,
+                           const set *classic_monitors)
+{
+  dout(10) << __func__ << " epoch " << epoch << " quorum " << active
+	   << " features " << features << dendl;
+  assert(is_electing());
+  state = STATE_LEADER;
+  leader_since = ceph_clock_now(g_ceph_context);
+  leader = rank;
+  quorum = active;
+  quorum_features = features;
+  outside_quorum.clear();
+
+  clog.info() << "mon." << name << "@" << rank
+		<< " won leader election with quorum " << quorum << "\n";
+
+  set_leader_supported_commands(cmdset, cmdsize);
+  if (classic_monitors)
+    classic_mons = *classic_monitors;
+
+  paxos->leader_init();
+  // NOTE: tell monmap monitor first.  This is important for the
+  // bootstrap case to ensure that the very first paxos proposal
+  // codifies the monmap.  Otherwise any manner of chaos can ensue
+  // when monitors are call elections or participating in a paxos
+  // round without agreeing on who the participants are.
+  monmon()->election_finished();
+  for (vector::iterator p = paxos_service.begin();
+       p != paxos_service.end(); ++p) {
+    if (*p != monmon())
+      (*p)->election_finished();
+  }
+  health_monitor->start(epoch);
+
+  finish_election();
+  if (monmap->size() > 1 &&
+      monmap->get_epoch() > 0)
+    timecheck_start();
+}
+
+void Monitor::lose_election(epoch_t epoch, set &q, int l, uint64_t features) 
+{
+  state = STATE_PEON;
+  leader_since = utime_t();
+  leader = l;
+  quorum = q;
+  outside_quorum.clear();
+  quorum_features = features;
+  dout(10) << "lose_election, epoch " << epoch << " leader is mon" << leader
+	   << " quorum is " << quorum << " features are " << quorum_features << dendl;
+
+  paxos->peon_init();
+  for (vector::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p)
+    (*p)->election_finished();
+  health_monitor->start(epoch);
+
+  finish_election();
+}
+
+void Monitor::finish_election()
+{
+  apply_quorum_to_compatset_features();
+  timecheck_finish();
+  exited_quorum = utime_t();
+  finish_contexts(g_ceph_context, waitfor_quorum);
+  finish_contexts(g_ceph_context, maybe_wait_for_quorum);
+  resend_routed_requests();
+  update_logger();
+  register_cluster_logger();
+
+  // am i named properly?
+  string cur_name = monmap->get_name(messenger->get_myaddr());
+  if (cur_name != name) {
+    dout(10) << " renaming myself from " << cur_name << " -> " << name << dendl;
+    messenger->send_message(new MMonJoin(monmap->fsid, name, messenger->get_myaddr()),
+			    monmap->get_inst(*quorum.begin()));
+  }
+}
+
+void Monitor::apply_quorum_to_compatset_features()
+{
+  CompatSet new_features(features);
+  if (quorum_features & CEPH_FEATURE_OSD_ERASURE_CODES) {
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
+  }
+  if (quorum_features & CEPH_FEATURE_OSDMAP_ENC) {
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
+  }
+
+  if (new_features.compare(features) != 0) {
+    CompatSet diff = features.unsupported(new_features);
+    dout(1) << __func__ << " enabling new quorum features: " << diff << dendl;
+    features = new_features;
+
+    MonitorDBStore::Transaction t;
+    write_features(t);
+    store->apply_transaction(t);
+
+    apply_compatset_features_to_quorum_requirements();
+  }
+}
+
+void Monitor::apply_compatset_features_to_quorum_requirements()
+{
+  required_features = 0;
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES)) {
+    required_features |= CEPH_FEATURE_OSD_ERASURE_CODES;
+  }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC)) {
+    required_features |= CEPH_FEATURE_OSDMAP_ENC;
+  }
+  dout(10) << __func__ << " required_features " << required_features << dendl;
+}
+
+void Monitor::sync_force(Formatter *f, ostream& ss)
+{
+  bool free_formatter = false;
+
+  if (!f) {
+    // louzy/lazy hack: default to json if no formatter has been defined
+    f = new JSONFormatter();
+    free_formatter = true;
+  }
+
+  MonitorDBStore::Transaction tx;
+  sync_stash_critical_state(&tx);
+  tx.put("mon_sync", "force_sync", 1);
+  store->apply_transaction(tx);
+
+  f->open_object_section("sync_force");
+  f->dump_int("ret", 0);
+  f->dump_stream("msg") << "forcing store sync the next time the monitor starts";
+  f->close_section(); // sync_force
+  f->flush(ss);
+  if (free_formatter)
+    delete f;
+}
+
+void Monitor::_quorum_status(Formatter *f, ostream& ss)
+{
+  bool free_formatter = false;
+
+  if (!f) {
+    // louzy/lazy hack: default to json if no formatter has been defined
+    f = new JSONFormatter();
+    free_formatter = true;
+  }
+  f->open_object_section("quorum_status");
+  f->dump_int("election_epoch", get_epoch());
+
+  f->open_array_section("quorum");
+  for (set::iterator p = quorum.begin(); p != quorum.end(); ++p)
+    f->dump_int("mon", *p);
+  f->close_section(); // quorum
+
+  list quorum_names = get_quorum_names();
+  f->open_array_section("quorum_names");
+  for (list::iterator p = quorum_names.begin(); p != quorum_names.end(); ++p)
+    f->dump_string("mon", *p);
+  f->close_section(); // quorum_names
+
+  f->dump_string("quorum_leader_name", quorum.empty() ? string() : monmap->get_name(*quorum.begin()));
+
+  f->open_object_section("monmap");
+  monmap->dump(f);
+  f->close_section(); // monmap
+
+  f->close_section(); // quorum_status
+  f->flush(ss);
+  if (free_formatter)
+    delete f;
+}
+
+void Monitor::get_mon_status(Formatter *f, ostream& ss)
+{
+  bool free_formatter = false;
+
+  if (!f) {
+    // louzy/lazy hack: default to json if no formatter has been defined
+    f = new JSONFormatter();
+    free_formatter = true;
+  }
+
+  f->open_object_section("mon_status");
+  f->dump_string("name", name);
+  f->dump_int("rank", rank);
+  f->dump_string("state", get_state_name());
+  f->dump_int("election_epoch", get_epoch());
+
+  f->open_array_section("quorum");
+  for (set::iterator p = quorum.begin(); p != quorum.end(); ++p) {
+    f->dump_int("mon", *p);
+  }
+
+  f->close_section(); // quorum
+
+  f->open_array_section("outside_quorum");
+  for (set::iterator p = outside_quorum.begin(); p != outside_quorum.end(); ++p)
+    f->dump_string("mon", *p);
+  f->close_section(); // outside_quorum
+
+  f->open_array_section("extra_probe_peers");
+  for (set::iterator p = extra_probe_peers.begin();
+       p != extra_probe_peers.end();
+       ++p)
+    f->dump_stream("peer") << *p;
+  f->close_section(); // extra_probe_peers
+
+  f->open_array_section("sync_provider");
+  for (map::const_iterator p = sync_providers.begin();
+       p != sync_providers.end();
+       ++p) {
+    f->dump_unsigned("cookie", p->second.cookie);
+    f->dump_stream("entity") << p->second.entity;
+    f->dump_stream("timeout") << p->second.timeout;
+    f->dump_unsigned("last_committed", p->second.last_committed);
+    f->dump_stream("last_key") << p->second.last_key;
+  }
+  f->close_section();
+
+  if (is_synchronizing()) {
+    f->open_object_section("sync");
+    f->dump_stream("sync_provider") << sync_provider;
+    f->dump_unsigned("sync_cookie", sync_cookie);
+    f->dump_unsigned("sync_start_version", sync_start_version);
+    f->close_section();
+  }
+
+  if (g_conf->mon_sync_provider_kill_at > 0)
+    f->dump_int("provider_kill_at", g_conf->mon_sync_provider_kill_at);
+  if (g_conf->mon_sync_requester_kill_at > 0)
+    f->dump_int("requester_kill_at", g_conf->mon_sync_requester_kill_at);
+
+  f->open_object_section("monmap");
+  monmap->dump(f);
+  f->close_section();
+
+  f->close_section(); // mon_status
+
+  if (free_formatter) {
+    // flush formatter to ss and delete it iff we created the formatter
+    f->flush(ss);
+    delete f;
+  }
+}
+
+void Monitor::get_health(string& status, bufferlist *detailbl, Formatter *f)
+{
+  list > summary;
+  list > detail;
+
+  if (f)
+    f->open_object_section("health");
+
+  for (vector::iterator p = paxos_service.begin();
+       p != paxos_service.end();
+       ++p) {
+    PaxosService *s = *p;
+    s->get_health(summary, detailbl ? &detail : NULL);
+  }
+
+  health_monitor->get_health(f, summary, (detailbl ? &detail : NULL));
+
+  if (f)
+    f->open_array_section("summary");
+  stringstream ss;
+  health_status_t overall = HEALTH_OK;
+  if (!summary.empty()) {
+    ss << ' ';
+    while (!summary.empty()) {
+      if (overall > summary.front().first)
+	overall = summary.front().first;
+      ss << summary.front().second;
+      if (f) {
+        f->open_object_section("item");
+        f->dump_stream("severity") <<  summary.front().first;
+        f->dump_string("summary", summary.front().second);
+        f->close_section();
+      }
+      summary.pop_front();
+      if (!summary.empty())
+	ss << "; ";
+    }
+  }
+  if (f)
+    f->close_section();
+
+  if (f) {
+    f->open_object_section("timechecks");
+    f->dump_int("epoch", get_epoch());
+    f->dump_int("round", timecheck_round);
+    f->dump_stream("round_status")
+      << ((timecheck_round%2) ? "on-going" : "finished");
+  }
+
+  if (!timecheck_skews.empty()) {
+    list warns;
+    if (f)
+      f->open_array_section("mons");
+    for (map::iterator i = timecheck_skews.begin();
+         i != timecheck_skews.end(); ++i) {
+      entity_inst_t inst = i->first;
+      double skew = i->second;
+      double latency = timecheck_latencies[inst];
+      string name = monmap->get_name(inst.addr);
+
+      ostringstream tcss;
+      health_status_t tcstatus = timecheck_status(tcss, skew, latency);
+      if (tcstatus != HEALTH_OK) {
+        if (overall > tcstatus)
+          overall = tcstatus;
+        warns.push_back(name);
+
+        ostringstream tmp_ss;
+        tmp_ss << "mon." << name
+               << " addr " << inst.addr << " " << tcss.str()
+	       << " (latency " << latency << "s)";
+        detail.push_back(make_pair(tcstatus, tmp_ss.str()));
+      }
+
+      if (f) {
+        f->open_object_section("mon");
+        f->dump_string("name", name.c_str());
+        f->dump_float("skew", skew);
+        f->dump_float("latency", latency);
+        f->dump_stream("health") << tcstatus;
+        if (tcstatus != HEALTH_OK)
+          f->dump_stream("details") << tcss.str();
+        f->close_section();
+      }
+    }
+    if (!warns.empty()) {
+      if (!ss.str().empty())
+        ss << ";";
+      ss << " clock skew detected on";
+      while (!warns.empty()) {
+        ss << " mon." << warns.front();
+        warns.pop_front();
+        if (!warns.empty())
+          ss << ",";
+      }
+    }
+    if (f)
+      f->close_section();
+  }
+  if (f)
+    f->close_section();
+
+  stringstream fss;
+  fss << overall;
+  status = fss.str() + ss.str();
+  if (f)
+    f->dump_stream("overall_status") << overall;
+
+  if (f)
+    f->open_array_section("detail");
+  while (!detail.empty()) {
+    if (f)
+      f->dump_string("item", detail.front().second);
+    else if (detailbl != NULL) {
+      detailbl->append(detail.front().second);
+      detailbl->append('\n');
+    }
+    detail.pop_front();
+  }
+  if (f)
+    f->close_section();
+
+  if (f)
+    f->close_section();
+}
+
+void Monitor::get_cluster_status(stringstream &ss, Formatter *f)
+{
+  if (f)
+    f->open_object_section("status");
+
+  // reply with the status for all the components
+  string health;
+  get_health(health, NULL, f);
+
+  if (f) {
+    f->dump_stream("fsid") << monmap->get_fsid();
+    f->dump_unsigned("election_epoch", get_epoch());
+    {
+      f->open_array_section("quorum");
+      for (set::iterator p = quorum.begin(); p != quorum.end(); ++p)
+	f->dump_int("rank", *p);
+      f->close_section();
+      f->open_array_section("quorum_names");
+      for (set::iterator p = quorum.begin(); p != quorum.end(); ++p)
+	f->dump_string("id", monmap->get_name(*p));
+      f->close_section();
+    }
+    f->open_object_section("monmap");
+    monmap->dump(f);
+    f->close_section();
+    f->open_object_section("osdmap");
+    osdmon()->osdmap.print_summary(f, cout);
+    f->close_section();
+    f->open_object_section("pgmap");
+    pgmon()->pg_map.print_summary(f, NULL);
+    f->close_section();
+    f->open_object_section("mdsmap");
+    mdsmon()->mdsmap.print_summary(f, NULL);
+    f->close_section();
+    f->close_section();
+  } else {
+    ss << "    cluster " << monmap->get_fsid() << "\n";
+    ss << "     health " << health << "\n";
+    ss << "     monmap " << *monmap << ", election epoch " << get_epoch()
+       << ", quorum " << get_quorum() << " " << get_quorum_names() << "\n";
+    if (mdsmon()->mdsmap.get_epoch() > 1)
+      ss << "     mdsmap " << mdsmon()->mdsmap << "\n";
+    osdmon()->osdmap.print_summary(NULL, ss);
+    pgmon()->pg_map.print_summary(NULL, &ss);
+  }
+}
+
+void Monitor::_generate_command_map(map& cmdmap,
+                                    map ¶m_str_map)
+{
+  for (map::const_iterator p = cmdmap.begin();
+       p != cmdmap.end(); ++p) {
+    if (p->first == "prefix")
+      continue;
+    if (p->first == "caps") {
+      vector cv;
+      if (cmd_getval(g_ceph_context, cmdmap, "caps", cv) &&
+	  cv.size() % 2 == 0) {
+	for (unsigned i = 0; i < cv.size(); i += 2) {
+	  string k = string("caps_") + cv[i];
+	  param_str_map[k] = cv[i + 1];
+	}
+	continue;
+      }
+    }
+    param_str_map[p->first] = cmd_vartype_stringify(p->second);
+  }
+}
+
+const MonCommand *Monitor::_get_moncommand(const string &cmd_prefix,
+                                           MonCommand *cmds, int cmds_size)
+{
+  MonCommand *this_cmd = NULL;
+  for (MonCommand *cp = cmds;
+       cp < &cmds[cmds_size]; cp++) {
+    if (cp->cmdstring.find(cmd_prefix) != string::npos) {
+      this_cmd = cp;
+      break;
+    }
+  }
+  return this_cmd;
+}
+
+bool Monitor::_allowed_command(MonSession *s, string &module, string &prefix,
+                               const map& cmdmap,
+                               const map& param_str_map,
+                               const MonCommand *this_cmd) {
+
+  bool cmd_r = (this_cmd->req_perms.find('r') != string::npos);
+  bool cmd_w = (this_cmd->req_perms.find('w') != string::npos);
+  bool cmd_x = (this_cmd->req_perms.find('x') != string::npos);
+
+  bool capable = s->caps.is_capable(g_ceph_context, s->inst.name,
+                                    module, prefix, param_str_map,
+                                    cmd_r, cmd_w, cmd_x);
+
+  dout(10) << __func__ << " " << (capable ? "" : "not ") << "capable" << dendl;
+  return capable;
+}
+
+void Monitor::format_command_descriptions(const MonCommand *commands,
+					  unsigned commands_size,
+					  Formatter *f,
+					  bufferlist *rdata)
+{
+  int cmdnum = 0;
+  f->open_object_section("command_descriptions");
+  for (const MonCommand *cp = commands;
+       cp < &commands[commands_size]; cp++) {
+
+    ostringstream secname;
+    secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
+    dump_cmddesc_to_json(f, secname.str(),
+			 cp->cmdstring, cp->helpstring, cp->module,
+			 cp->req_perms, cp->availability);
+    cmdnum++;
+  }
+  f->close_section();	// command_descriptions
+
+  f->flush(*rdata);
+}
+
+void Monitor::get_locally_supported_monitor_commands(const MonCommand **cmds,
+						     int *count)
+{
+  *cmds = mon_commands;
+  *count = ARRAY_SIZE(mon_commands);
+}
+void Monitor::get_leader_supported_commands(const MonCommand **cmds, int *count)
+{
+  *cmds = leader_supported_mon_commands;
+  *count = leader_supported_mon_commands_size;
+}
+void Monitor::get_classic_monitor_commands(const MonCommand **cmds, int *count)
+{
+  *cmds = classic_mon_commands;
+  *count = ARRAY_SIZE(classic_mon_commands);
+}
+void Monitor::set_leader_supported_commands(const MonCommand *cmds, int size)
+{
+  if (leader_supported_mon_commands != mon_commands &&
+      leader_supported_mon_commands != classic_mon_commands)
+    delete[] leader_supported_mon_commands;
+  leader_supported_mon_commands = cmds;
+  leader_supported_mon_commands_size = size;
+}
+
+bool Monitor::is_keyring_required()
+{
+  string auth_cluster_required = g_conf->auth_supported.length() ?
+    g_conf->auth_supported : g_conf->auth_cluster_required;
+  string auth_service_required = g_conf->auth_supported.length() ?
+    g_conf->auth_supported : g_conf->auth_service_required;
+
+  return auth_service_required == "cephx" ||
+    auth_cluster_required == "cephx";
+}
+
+void Monitor::handle_command(MMonCommand *m)
+{
+  if (m->fsid != monmap->fsid) {
+    dout(0) << "handle_command on fsid " << m->fsid << " != " << monmap->fsid << dendl;
+    reply_command(m, -EPERM, "wrong fsid", 0);
+    return;
+  }
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    string rs = "Access denied";
+    reply_command(m, -EACCES, rs, 0);
+    return;
+  }
+
+  if (m->cmd.empty()) {
+    string rs = "No command supplied";
+    reply_command(m, -EINVAL, rs, 0);
+    return;
+  }
+
+  string prefix;
+  vector fullcmd;
+  map cmdmap;
+  stringstream ss, ds;
+  bufferlist rdata;
+  string rs;
+  int r = -EINVAL;
+  rs = "unrecognized command";
+
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    r = -EINVAL;
+    rs = ss.str();
+    if (!m->get_source().is_mon())  // don't reply to mon->mon commands
+      reply_command(m, r, rs, 0);
+    else
+      m->put();
+    return;
+  }
+
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+  if (prefix == "get_command_descriptions") {
+    bufferlist rdata;
+    Formatter *f = new_formatter("json");
+    format_command_descriptions(leader_supported_mon_commands,
+				leader_supported_mon_commands_size, f, &rdata);
+    delete f;
+    reply_command(m, 0, "", rdata, 0);
+    return;
+  }
+
+  string module;
+  string err;
+
+  dout(0) << "handle_command " << *m << dendl;
+
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr f(new_formatter(format));
+
+  get_str_vec(prefix, fullcmd);
+  module = fullcmd[0];
+
+  // validate command is in leader map
+
+  const MonCommand *leader_cmd;
+  leader_cmd = _get_moncommand(prefix,
+                               // the boost underlying this isn't const for some reason
+                               const_cast(leader_supported_mon_commands),
+                               leader_supported_mon_commands_size);
+  if (!leader_cmd) {
+    reply_command(m, -EINVAL, "command not known", 0);
+    return;
+  }
+  // validate command is in our map & matches, or forward
+  const MonCommand *mon_cmd = _get_moncommand(prefix, mon_commands,
+                                              ARRAY_SIZE(mon_commands));
+  if (!is_leader() && (!mon_cmd ||
+      (*leader_cmd != *mon_cmd))) {
+    dout(10) << "We don't match leader, forwarding request " << m << dendl;
+    forward_request_leader(m);
+    return;
+  }
+  // validate user's permissions for requested command
+  map param_str_map;
+  _generate_command_map(cmdmap, param_str_map);
+  if (!_allowed_command(session, module, prefix, cmdmap,
+                        param_str_map, mon_cmd)) {
+    dout(1) << __func__ << " access denied" << dendl;
+    reply_command(m, -EACCES, "access denied", 0);
+    return;
+  }
+
+  if (module == "mds") {
+    mdsmon()->dispatch(m);
+    return;
+  }
+  if (module == "osd") {
+    osdmon()->dispatch(m);
+    return;
+  }
+
+  if (module == "pg") {
+    pgmon()->dispatch(m);
+    return;
+  }
+  if (module == "mon") {
+    monmon()->dispatch(m);
+    return;
+  }
+  if (module == "auth") {
+    authmon()->dispatch(m);
+    return;
+  }
+  if (module == "log") {
+    logmon()->dispatch(m);
+    return;
+  }
+
+  if (module == "config-key") {
+    config_key_service->dispatch(m);
+    return;
+  }
+
+  if (prefix == "fsid") {
+    if (f) {
+      f->open_object_section("fsid");
+      f->dump_stream("fsid") << monmap->fsid;
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ds << monmap->fsid;
+      rdata.append(ds);
+    }
+    reply_command(m, 0, "", rdata, 0);
+    return;
+  }
+
+  if (prefix == "scrub") {
+    if (is_leader()) {
+      int r = scrub();
+      reply_command(m, r, "", rdata, 0);
+    } else if (is_peon()) {
+      forward_request_leader(m);
+    } else {
+      reply_command(m, -EAGAIN, "no quorum", rdata, 0);
+    }
+    return;
+  }
+
+  if (prefix == "compact") {
+    dout(1) << "triggering manual compaction" << dendl;
+    utime_t start = ceph_clock_now(g_ceph_context);
+    store->compact();
+    utime_t end = ceph_clock_now(g_ceph_context);
+    end -= start;
+    dout(1) << "finished manual compaction in " << end << " seconds" << dendl;
+    ostringstream oss;
+    oss << "compacted leveldb in " << end;
+    rs = oss.str();
+    r = 0;
+  }
+  else if (prefix == "injectargs") {
+    vector injected_args;
+    cmd_getval(g_ceph_context, cmdmap, "injected_args", injected_args);
+    if (!injected_args.empty()) {
+      dout(0) << "parsing injected options '" << injected_args << "'" << dendl;
+      ostringstream oss;
+      r = g_conf->injectargs(str_join(injected_args, " "), &oss);
+      ss << "injectargs:"  << oss.str();
+      rs = ss.str();
+      goto out;
+    } else {
+      rs = "must supply options to be parsed in a single string";
+      r = -EINVAL;
+    }
+  } else if (prefix == "status" ||
+	     prefix == "health" ||
+	     prefix == "df") {
+    string detail;
+    cmd_getval(g_ceph_context, cmdmap, "detail", detail);
+
+    if (prefix == "status") {
+      // get_cluster_status handles f == NULL
+      get_cluster_status(ds, f.get());
+
+      if (f) {
+        f->flush(ds);
+        ds << '\n';
+      }
+      rdata.append(ds);
+    } else if (prefix == "health") {
+      string health_str;
+      get_health(health_str, detail == "detail" ? &rdata : NULL, f.get());
+      if (f) {
+        f->flush(ds);
+        ds << '\n';
+      } else {
+        ds << health_str;
+      }
+      bufferlist comb;
+      comb.append(ds);
+      if (detail == "detail")
+	comb.append(rdata);
+      rdata = comb;
+      r = 0;
+    } else if (prefix == "df") {
+      bool verbose = (detail == "detail");
+      if (f)
+        f->open_object_section("stats");
+
+      pgmon()->dump_fs_stats(ds, f.get(), verbose);
+      if (!f)
+        ds << '\n';
+      pgmon()->dump_pool_stats(ds, f.get(), verbose);
+
+      if (f) {
+        f->close_section();
+        f->flush(ds);
+        ds << '\n';
+      }
+    } else {
+      assert(0 == "We should never get here!");
+      return;
+    }
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "report") {
+
+    // this must be formatted, in its current form
+    if (!f)
+      f.reset(new_formatter("json-pretty"));
+    f->open_object_section("report");
+    f->dump_stream("cluster_fingerprint") << fingerprint;
+    f->dump_string("version", ceph_version_to_str());
+    f->dump_string("commit", git_version_to_str());
+    f->dump_stream("timestamp") << ceph_clock_now(NULL);
+
+    vector tagsvec;
+    cmd_getval(g_ceph_context, cmdmap, "tags", tagsvec);
+    string tagstr = str_join(tagsvec, " ");
+    if (!tagstr.empty())
+      tagstr = tagstr.substr(0, tagstr.find_last_of(' '));
+    f->dump_string("tag", tagstr);
+
+    string hs;
+    get_health(hs, NULL, f.get());
+
+    monmon()->dump_info(f.get());
+    osdmon()->dump_info(f.get());
+    mdsmon()->dump_info(f.get());
+    pgmon()->dump_info(f.get());
+    authmon()->dump_info(f.get());
+
+    paxos->dump_info(f.get());
+
+    f->close_section();
+    f->flush(rdata);
+
+    ostringstream ss2;
+    ss2 << "report " << rdata.crc32c(6789);
+    rs = ss2.str();
+    r = 0;
+  } else if (prefix == "quorum_status") {
+    // make sure our map is readable and up to date
+    if (!is_leader() && !is_peon()) {
+      dout(10) << " waiting for quorum" << dendl;
+      waitfor_quorum.push_back(new C_RetryMessage(this, m));
+      return;
+    }
+    _quorum_status(f.get(), ds);
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "mon_status") {
+    get_mon_status(f.get(), ds);
+    if (f)
+      f->flush(ds);
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "sync force") {
+    string validate1, validate2;
+    cmd_getval(g_ceph_context, cmdmap, "validate1", validate1);
+    cmd_getval(g_ceph_context, cmdmap, "validate2", validate2);
+    if (validate1 != "--yes-i-really-mean-it" ||
+	validate2 != "--i-know-what-i-am-doing") {
+      r = -EINVAL;
+      rs = "are you SURE? this will mean the monitor store will be "
+	   "erased.  pass '--yes-i-really-mean-it "
+	   "--i-know-what-i-am-doing' if you really do.";
+      goto out;
+    }
+    sync_force(f.get(), ds);
+    rs = ds.str();
+    r = 0;
+  } else if (prefix == "heap") {
+    if (!ceph_using_tcmalloc())
+      rs = "tcmalloc not enabled, can't use heap profiler commands\n";
+    else {
+      string heapcmd;
+      cmd_getval(g_ceph_context, cmdmap, "heapcmd", heapcmd);
+      // XXX 1-element vector, change at callee or make vector here?
+      vector heapcmd_vec;
+      get_str_vec(heapcmd, heapcmd_vec);
+      ceph_heap_profiler_handle_command(heapcmd_vec, ds);
+      rdata.append(ds);
+      rs = "";
+      r = 0;
+    }
+  } else if (prefix == "quorum") {
+    string quorumcmd;
+    cmd_getval(g_ceph_context, cmdmap, "quorumcmd", quorumcmd);
+    if (quorumcmd == "exit") {
+      start_election();
+      elector.stop_participating();
+      rs = "stopped responding to quorum, initiated new election";
+      r = 0;
+    } else if (quorumcmd == "enter") {
+      elector.start_participating();
+      start_election();
+      rs = "started responding to quorum, initiated new election";
+      r = 0;
+    } else {
+      rs = "needs a valid 'quorum' command";
+      r = -EINVAL;
+    }
+  }
+
+ out:
+  if (!m->get_source().is_mon())  // don't reply to mon->mon commands
+    reply_command(m, r, rs, rdata, 0);
+  else
+    m->put();
+}
+
+void Monitor::reply_command(MMonCommand *m, int rc, const string &rs, version_t version)
+{
+  bufferlist rdata;
+  reply_command(m, rc, rs, rdata, version);
+}
+
+void Monitor::reply_command(MMonCommand *m, int rc, const string &rs, bufferlist& rdata, version_t version)
+{
+  MMonCommandAck *reply = new MMonCommandAck(m->cmd, rc, rs, version);
+  reply->set_tid(m->get_tid());
+  reply->set_data(rdata);
+  send_reply(m, reply);
+  m->put();
+}
+
+
+// ------------------------
+// request/reply routing
+//
+// a client/mds/osd will connect to a random monitor.  we need to forward any
+// messages requiring state updates to the leader, and then route any replies
+// back via the correct monitor and back to them.  (the monitor will not
+// initiate any connections.)
+
+void Monitor::forward_request_leader(PaxosServiceMessage *req)
+{
+  int mon = get_leader();
+  MonSession *session = 0;
+  if (req->get_connection())
+    session = static_cast(req->get_connection()->get_priv());
+  if (req->get_source().is_mon() && req->get_source_addr() != messenger->get_myaddr()) {
+    dout(10) << "forward_request won't forward (non-local) mon request " << *req << dendl;
+    req->put();
+  } else if (session && session->proxy_con) {
+    dout(10) << "forward_request won't double fwd request " << *req << dendl;
+    req->put();
+  } else if (session && !session->closed) {
+    RoutedRequest *rr = new RoutedRequest;
+    rr->tid = ++routed_request_tid;
+    rr->client_inst = req->get_source_inst();
+    rr->con = req->get_connection();
+    rr->con_features = rr->con->get_features();
+    encode_message(req, CEPH_FEATURES_ALL, rr->request_bl);   // for my use only; use all features
+    rr->session = static_cast(session->get());
+    routed_requests[rr->tid] = rr;
+    session->routed_request_tids.insert(rr->tid);
+    
+    dout(10) << "forward_request " << rr->tid << " request " << *req << dendl;
+
+    MForward *forward = new MForward(rr->tid, req,
+				     rr->con_features,
+				     rr->session->caps);
+    forward->set_priority(req->get_priority());
+    messenger->send_message(forward, monmap->get_inst(mon));
+  } else {
+    dout(10) << "forward_request no session for request " << *req << dendl;
+    req->put();
+  }
+  if (session)
+    session->put();
+}
+
+//extract the original message and put it into the regular dispatch function
+void Monitor::handle_forward(MForward *m)
+{
+  dout(10) << "received forwarded message from " << m->client
+	   << " via " << m->get_source_inst() << dendl;
+  MonSession *session = static_cast(m->get_connection()->get_priv());
+  assert(session);
+
+  if (!session->is_capable("mon", MON_CAP_X)) {
+    dout(0) << "forward from entity with insufficient caps! " 
+	    << session->caps << dendl;
+  } else {
+    Connection *c = new Connection(NULL);  // msgr must be null; see PaxosService::dispatch()
+    MonSession *s = new MonSession(m->msg->get_source_inst(), c);
+    c->set_priv(s);
+    c->set_peer_addr(m->client.addr);
+    c->set_peer_type(m->client.name.type());
+    c->set_features(m->con_features);
+
+    s->caps = m->client_caps;
+    dout(10) << " caps are " << s->caps << dendl;
+    s->proxy_con = m->get_connection();
+    s->proxy_tid = m->tid;
+
+    PaxosServiceMessage *req = m->msg;
+    m->msg = NULL;  // so ~MForward doesn't delete it
+    req->set_connection(c);
+
+    // not super accurate, but better than nothing.
+    req->set_recv_stamp(m->get_recv_stamp());
+
+    /*
+     * note which election epoch this is; we will drop the message if
+     * there is a future election since our peers will resend routed
+     * requests in that case.
+     */
+    req->rx_election_epoch = get_epoch();
+
+    /* Because this is a special fake connection, we need to break
+       the ref loop between Connection and MonSession differently
+       than we normally do. Here, the Message refers to the Connection
+       which refers to the Session, and nobody else refers to the Connection
+       or the Session. And due to the special nature of this message,
+       nobody refers to the Connection via the Session. So, clear out that
+       half of the ref loop.*/
+    s->con.reset(NULL);
+
+    dout(10) << " mesg " << req << " from " << m->get_source_addr() << dendl;
+
+    _ms_dispatch(req);
+  }
+  session->put();
+  m->put();
+}
+
+void Monitor::try_send_message(Message *m, const entity_inst_t& to)
+{
+  dout(10) << "try_send_message " << *m << " to " << to << dendl;
+
+  bufferlist bl;
+  encode_message(m, quorum_features, bl);
+
+  messenger->send_message(m, to);
+
+  for (int i=0; i<(int)monmap->size(); i++) {
+    if (i != rank)
+      messenger->send_message(new MRoute(bl, to), monmap->get_inst(i));
+  }
+}
+
+void Monitor::send_reply(PaxosServiceMessage *req, Message *reply)
+{
+  ConnectionRef connection = req->get_connection();
+  if (!connection) {
+    dout(2) << "send_reply no connection, dropping reply " << *reply
+	    << " to " << req << " " << *req << dendl;
+    reply->put();
+    return;
+  }
+  MonSession *session = static_cast(connection->get_priv());
+  if (!session) {
+    dout(2) << "send_reply no session, dropping reply " << *reply
+	    << " to " << req << " " << *req << dendl;
+    reply->put();
+    return;
+  }
+  if (session->proxy_con) {
+    dout(15) << "send_reply routing reply to " << req->get_connection()->get_peer_addr()
+	     << " via " << session->proxy_con->get_peer_addr()
+	     << " for request " << *req << dendl;
+    messenger->send_message(new MRoute(session->proxy_tid, reply),
+			    session->proxy_con);    
+  } else {
+    messenger->send_message(reply, session->con);
+  }
+  session->put();
+}
+
+void Monitor::no_reply(PaxosServiceMessage *req)
+{
+  MonSession *session = static_cast(req->get_connection()->get_priv());
+  if (!session) {
+    dout(2) << "no_reply no session, dropping non-reply to " << req << " " << *req << dendl;
+    return;
+  }
+  if (session->proxy_con) {
+    if (get_quorum_features() & CEPH_FEATURE_MON_NULLROUTE) {
+      dout(10) << "no_reply to " << req->get_source_inst()
+	       << " via " << session->proxy_con->get_peer_addr()
+	       << " for request " << *req << dendl;
+      messenger->send_message(new MRoute(session->proxy_tid, NULL),
+			      session->proxy_con);
+    } else {
+      dout(10) << "no_reply no quorum nullroute feature for " << req->get_source_inst()
+	       << " via " << session->proxy_con->get_peer_addr()
+	       << " for request " << *req << dendl;
+    }
+  } else {
+    dout(10) << "no_reply to " << req->get_source_inst() << " " << *req << dendl;
+  }
+  session->put();
+}
+
+void Monitor::handle_route(MRoute *m)
+{
+  MonSession *session = static_cast(m->get_connection()->get_priv());
+  //check privileges
+  if (session && !session->is_capable("mon", MON_CAP_X)) {
+    dout(0) << "MRoute received from entity without appropriate perms! "
+	    << dendl;
+    session->put();
+    m->put();
+    return;
+  }
+  if (m->msg)
+    dout(10) << "handle_route " << *m->msg << " to " << m->dest << dendl;
+  else
+    dout(10) << "handle_route null to " << m->dest << dendl;
+  
+  // look it up
+  if (m->session_mon_tid) {
+    if (routed_requests.count(m->session_mon_tid)) {
+      RoutedRequest *rr = routed_requests[m->session_mon_tid];
+
+      // reset payload, in case encoding is dependent on target features
+      if (m->msg) {
+	m->msg->clear_payload();
+	messenger->send_message(m->msg, rr->con);
+	m->msg = NULL;
+      }
+      routed_requests.erase(m->session_mon_tid);
+      rr->session->routed_request_tids.insert(rr->tid);
+      delete rr;
+    } else {
+      dout(10) << " don't have routed request tid " << m->session_mon_tid << dendl;
+    }
+  } else {
+    dout(10) << " not a routed request, trying to send anyway" << dendl;
+    if (m->msg) {
+      messenger->lazy_send_message(m->msg, m->dest);
+      m->msg = NULL;
+    }
+  }
+  m->put();
+  if (session)
+    session->put();
+}
+
+void Monitor::resend_routed_requests()
+{
+  dout(10) << "resend_routed_requests" << dendl;
+  int mon = get_leader();
+  list retry;
+  for (map::iterator p = routed_requests.begin();
+       p != routed_requests.end();
+       ++p) {
+    RoutedRequest *rr = p->second;
+
+    bufferlist::iterator q = rr->request_bl.begin();
+    PaxosServiceMessage *req = (PaxosServiceMessage *)decode_message(cct, q);
+
+    if (mon == rank) {
+      dout(10) << " requeue for self tid " << rr->tid << " " << *req << dendl;
+      req->set_connection(rr->con);
+      retry.push_back(new C_RetryMessage(this, req));
+      delete rr;
+    } else {
+      dout(10) << " resend to mon." << mon << " tid " << rr->tid << " " << *req << dendl;
+      MForward *forward = new MForward(rr->tid, req, rr->con_features,
+				       rr->session->caps);
+      forward->client = rr->client_inst;
+      forward->set_priority(req->get_priority());
+      messenger->send_message(forward, monmap->get_inst(mon));
+    }
+  }
+  if (mon == rank) {
+    routed_requests.clear();
+    finish_contexts(g_ceph_context, retry);
+  }
+}
+
+void Monitor::remove_session(MonSession *s)
+{
+  dout(10) << "remove_session " << s << " " << s->inst << dendl;
+  assert(!s->closed);
+  for (set::iterator p = s->routed_request_tids.begin();
+       p != s->routed_request_tids.end();
+       ++p) {
+    if (routed_requests.count(*p)) {
+      RoutedRequest *rr = routed_requests[*p];
+      dout(10) << " dropping routed request " << rr->tid << dendl;
+      delete rr;
+      routed_requests.erase(*p);
+    }
+  }
+  s->con->set_priv(NULL);
+  session_map.remove_session(s);
+}
+
+void Monitor::remove_all_sessions()
+{
+  while (!session_map.sessions.empty()) {
+    MonSession *s = session_map.sessions.front();
+    remove_session(s);
+  }
+}
+
+void Monitor::send_command(const entity_inst_t& inst,
+			   const vector& com)
+{
+  dout(10) << "send_command " << inst << "" << com << dendl;
+  MMonCommand *c = new MMonCommand(monmap->fsid);
+  c->cmd = com;
+  try_send_message(c, inst);
+}
+
+void Monitor::waitlist_or_zap_client(Message *m)
+{
+  /**
+   * Wait list the new session until we're in the quorum, assuming it's
+   * sufficiently new.
+   * tick() will periodically send them back through so we can send
+   * the client elsewhere if we don't think we're getting back in.
+   *
+   * But we whitelist a few sorts of messages:
+   * 1) Monitors can talk to us at any time, of course.
+   * 2) auth messages. It's unlikely to go through much faster, but
+   * it's possible we've just lost our quorum status and we want to take...
+   * 3) command messages. We want to accept these under all possible
+   * circumstances.
+   */
+  ConnectionRef con = m->get_connection();
+  utime_t too_old = ceph_clock_now(g_ceph_context);
+  too_old -= g_ceph_context->_conf->mon_lease;
+  if (m->get_recv_stamp() > too_old &&
+      con->is_connected()) {
+    dout(5) << "waitlisting message " << *m << dendl;
+    maybe_wait_for_quorum.push_back(new C_RetryMessage(this, m));
+  } else {
+    dout(5) << "discarding message " << *m << " and sending client elsewhere" << dendl;
+    messenger->mark_down(con);
+    m->put();
+  }
+}
+
+bool Monitor::_ms_dispatch(Message *m)
+{
+  bool ret = true;
+
+  if (is_shutdown()) {
+    m->put();
+    return true;
+  }
+
+  ConnectionRef connection = m->get_connection();
+  MonSession *s = NULL;
+  MonCap caps;
+  EntityName entity_name;
+  bool src_is_mon;
+
+  // regardless of who we are or who the sender is, the message must
+  // have a connection associated.  If it doesn't then something fishy
+  // is going on.
+  assert(connection);
+
+  src_is_mon = (connection->get_peer_type() & CEPH_ENTITY_TYPE_MON);
+
+  bool reuse_caps = false;
+  dout(20) << "have connection" << dendl;
+  s = static_cast(connection->get_priv());
+  if (s && s->closed) {
+    caps = s->caps;
+    reuse_caps = true;
+    s->put();
+    s = NULL;
+  }
+  if (!s) {
+    // if the sender is not a monitor, make sure their first message for a
+    // session is an MAuth.  If it is not, assume it's a stray message,
+    // and considering that we are creating a new session it is safe to
+    // assume that the sender hasn't authenticated yet, so we have no way
+    // of assessing whether we should handle it or not.
+    if (!src_is_mon && (m->get_type() != CEPH_MSG_AUTH &&
+			m->get_type() != CEPH_MSG_MON_GET_MAP)) {
+      if (m->get_type() == CEPH_MSG_PING) {
+        // let it go through and be dispatched immediately!
+        return dispatch(s, m, false);
+      }
+      dout(1) << __func__ << " dropping stray message " << *m
+	      << " from " << m->get_source_inst() << dendl;
+      m->put();
+      return true;
+    }
+
+    if (!exited_quorum.is_zero() && !src_is_mon) {
+      waitlist_or_zap_client(m);
+      return true;
+    }
+
+    dout(10) << "do not have session, making new one" << dendl;
+    s = session_map.new_session(m->get_source_inst(), m->get_connection().get());
+    m->get_connection()->set_priv(s->get());
+    dout(10) << "ms_dispatch new session " << s << " for " << s->inst << dendl;
+
+    if (!src_is_mon) {
+      dout(10) << "setting timeout on session" << dendl;
+      // set an initial timeout here, so we will trim this session even if they don't
+      // do anything.
+      s->until = ceph_clock_now(g_ceph_context);
+      s->until += g_conf->mon_subscribe_interval;
+    } else {
+      //give it monitor caps; the peer type has been authenticated
+      reuse_caps = false;
+      dout(5) << "setting monitor caps on this connection" << dendl;
+      if (!s->caps.is_allow_all()) //but no need to repeatedly copy
+        s->caps = *mon_caps;
+    }
+    if (reuse_caps)
+      s->caps = caps;
+  } else {
+    dout(20) << "ms_dispatch existing session " << s << " for " << s->inst << dendl;
+  }
+
+  if (s) {
+    if (s->auth_handler) {
+      entity_name = s->auth_handler->get_entity_name();
+    }
+    dout(20) << " caps " << s->caps.get_str() << dendl;
+  }
+
+  if (is_synchronizing() && !src_is_mon) {
+    waitlist_or_zap_client(m);
+    return true;
+  }
+
+  ret = dispatch(s, m, src_is_mon);
+
+  if (s) {
+    s->put();
+  }
+
+  return ret;
+}
+
+bool Monitor::dispatch(MonSession *s, Message *m, const bool src_is_mon)
+{
+  bool ret = true;
+
+  assert(m != NULL);
+
+  switch (m->get_type()) {
+
+    case MSG_ROUTE:
+      handle_route(static_cast(m));
+      break;
+
+    // misc
+    case CEPH_MSG_MON_GET_MAP:
+      handle_mon_get_map(static_cast(m));
+      break;
+
+    case CEPH_MSG_MON_GET_VERSION:
+      handle_get_version(static_cast(m));
+      break;
+
+    case MSG_MON_COMMAND:
+      handle_command(static_cast(m));
+      break;
+
+    case CEPH_MSG_MON_SUBSCRIBE:
+      /* FIXME: check what's being subscribed, filter accordingly */
+      handle_subscribe(static_cast(m));
+      break;
+
+    case MSG_MON_PROBE:
+      handle_probe(static_cast(m));
+      break;
+
+    // Sync (i.e., the new slurp, but on steroids)
+    case MSG_MON_SYNC:
+      handle_sync(static_cast(m));
+      break;
+    case MSG_MON_SCRUB:
+      handle_scrub(static_cast(m));
+      break;
+
+    // OSDs
+    case MSG_OSD_MARK_ME_DOWN:
+    case MSG_OSD_FAILURE:
+    case MSG_OSD_BOOT:
+    case MSG_OSD_ALIVE:
+    case MSG_OSD_PGTEMP:
+      paxos_service[PAXOS_OSDMAP]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    case MSG_REMOVE_SNAPS:
+      paxos_service[PAXOS_OSDMAP]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    // MDSs
+    case MSG_MDS_BEACON:
+    case MSG_MDS_OFFLOAD_TARGETS:
+      paxos_service[PAXOS_MDSMAP]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    // auth
+    case MSG_MON_GLOBAL_ID:
+    case CEPH_MSG_AUTH:
+      /* no need to check caps here */
+      paxos_service[PAXOS_AUTH]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    // pg
+    case CEPH_MSG_STATFS:
+    case MSG_PGSTATS:
+    case MSG_GETPOOLSTATS:
+      paxos_service[PAXOS_PGMAP]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    case CEPH_MSG_POOLOP:
+      paxos_service[PAXOS_OSDMAP]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    // log
+    case MSG_LOG:
+      paxos_service[PAXOS_LOG]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    case MSG_LOGACK:
+      clog.handle_log_ack((MLogAck*)m);
+      break;
+
+    // monmap
+    case MSG_MON_JOIN:
+      paxos_service[PAXOS_MONMAP]->dispatch((PaxosServiceMessage*)m);
+      break;
+
+    // paxos
+    case MSG_MON_PAXOS:
+      {
+        MMonPaxos *pm = static_cast(m);
+        if (!src_is_mon ||
+            !s->is_capable("mon", MON_CAP_X)) {
+          //can't send these!
+          pm->put();
+          break;
+        }
+
+        if (state == STATE_SYNCHRONIZING) {
+          // we are synchronizing. These messages would do us no
+          // good, thus just drop them and ignore them.
+          dout(10) << __func__ << " ignore paxos msg from "
+            << pm->get_source_inst() << dendl;
+          pm->put();
+          break;
+        }
+
+        // sanitize
+        if (pm->epoch > get_epoch()) {
+          bootstrap();
+          pm->put();
+          break;
+        }
+        if (pm->epoch != get_epoch()) {
+          pm->put();
+          break;
+        }
+
+        paxos->dispatch((PaxosServiceMessage*)m);
+      }
+      break;
+
+    // elector messages
+    case MSG_MON_ELECTION:
+      //check privileges here for simplicity
+      if (s &&
+          !s->is_capable("mon", MON_CAP_X)) {
+        dout(0) << "MMonElection received from entity without enough caps!"
+          << s->caps << dendl;
+        m->put();
+        break;
+      }
+      if (!is_probing() && !is_synchronizing()) {
+        elector.dispatch(m);
+      } else {
+        m->put();
+      }
+      break;
+
+    case MSG_FORWARD:
+      handle_forward(static_cast(m));
+      break;
+
+    case MSG_TIMECHECK:
+      handle_timecheck(static_cast(m));
+      break;
+
+    case MSG_MON_HEALTH:
+      health_monitor->dispatch(static_cast(m));
+      break;
+
+    case CEPH_MSG_PING:
+      handle_ping(static_cast(m));
+      break;
+
+    default:
+      ret = false;
+  }
+
+  return ret;
+}
+
+void Monitor::handle_ping(MPing *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  MPing *reply = new MPing;
+  entity_inst_t inst = m->get_source_inst();
+  bufferlist payload;
+  Formatter *f = new JSONFormatter(true);
+  f->open_object_section("pong");
+
+  string health_str;
+  get_health(health_str, NULL, f);
+  {
+    stringstream ss;
+    get_mon_status(f, ss);
+  }
+
+  f->close_section();
+  stringstream ss;
+  f->flush(ss);
+  ::encode(ss.str(), payload);
+  reply->set_payload(payload);
+  dout(10) << __func__ << " reply payload len " << reply->get_payload().length() << dendl;
+  messenger->send_message(reply, inst);
+  m->put();
+}
+
+void Monitor::timecheck_start()
+{
+  dout(10) << __func__ << dendl;
+  timecheck_cleanup();
+  timecheck_start_round();
+}
+
+void Monitor::timecheck_finish()
+{
+  dout(10) << __func__ << dendl;
+  timecheck_cleanup();
+}
+
+void Monitor::timecheck_start_round()
+{
+  dout(10) << __func__ << " curr " << timecheck_round << dendl;
+  assert(is_leader());
+
+  if (monmap->size() == 1) {
+    assert(0 == "We are alone; this shouldn't have been scheduled!");
+    return;
+  }
+
+  if (timecheck_round % 2) {
+    dout(10) << __func__ << " there's a timecheck going on" << dendl;
+    utime_t curr_time = ceph_clock_now(g_ceph_context);
+    double max = g_conf->mon_timecheck_interval*3;
+    if (curr_time - timecheck_round_start > max) {
+      dout(10) << __func__ << " keep current round going" << dendl;
+      goto out;
+    } else {
+      dout(10) << __func__
+               << " finish current timecheck and start new" << dendl;
+      timecheck_cancel_round();
+    }
+  }
+
+  assert(timecheck_round % 2 == 0);
+  timecheck_acks = 0;
+  timecheck_round ++;
+  timecheck_round_start = ceph_clock_now(g_ceph_context);
+  dout(10) << __func__ << " new " << timecheck_round << dendl;
+
+  timecheck();
+out:
+  dout(10) << __func__ << " setting up next event" << dendl;
+  timecheck_event = new C_TimeCheck(this);
+  timer.add_event_after(g_conf->mon_timecheck_interval, timecheck_event);
+}
+
+void Monitor::timecheck_finish_round(bool success)
+{
+  dout(10) << __func__ << " curr " << timecheck_round << dendl;
+  assert(timecheck_round % 2);
+  timecheck_round ++;
+  timecheck_round_start = utime_t();
+
+  if (success) {
+    assert(timecheck_waiting.empty());
+    assert(timecheck_acks == quorum.size());
+    timecheck_report();
+    return;
+  }
+
+  dout(10) << __func__ << " " << timecheck_waiting.size()
+           << " peers still waiting:";
+  for (map::iterator p = timecheck_waiting.begin();
+      p != timecheck_waiting.end(); ++p) {
+    *_dout << " " << p->first.name;
+  }
+  *_dout << dendl;
+  timecheck_waiting.clear();
+
+  dout(10) << __func__ << " finished to " << timecheck_round << dendl;
+}
+
+void Monitor::timecheck_cancel_round()
+{
+  timecheck_finish_round(false);
+}
+
+void Monitor::timecheck_cleanup()
+{
+  timecheck_round = 0;
+  timecheck_acks = 0;
+  timecheck_round_start = utime_t();
+
+  if (timecheck_event) {
+    timer.cancel_event(timecheck_event);
+    timecheck_event = NULL;
+  }
+  timecheck_waiting.clear();
+  timecheck_skews.clear();
+  timecheck_latencies.clear();
+}
+
+void Monitor::timecheck_report()
+{
+  dout(10) << __func__ << dendl;
+  assert(is_leader());
+  assert((timecheck_round % 2) == 0);
+  if (monmap->size() == 1) {
+    assert(0 == "We are alone; we shouldn't have gotten here!");
+    return;
+  }
+
+  assert(timecheck_latencies.size() == timecheck_skews.size());
+  bool do_output = true; // only output report once
+  for (set::iterator q = quorum.begin(); q != quorum.end(); ++q) {
+    if (monmap->get_name(*q) == name)
+      continue;
+
+    MTimeCheck *m = new MTimeCheck(MTimeCheck::OP_REPORT);
+    m->epoch = get_epoch();
+    m->round = timecheck_round;
+
+    for (map::iterator it = timecheck_skews.begin(); it != timecheck_skews.end(); ++it) {
+      double skew = it->second;
+      double latency = timecheck_latencies[it->first];
+
+      m->skews[it->first] = skew;
+      m->latencies[it->first] = latency;
+
+      if (do_output) {
+        dout(25) << __func__ << " " << it->first
+                 << " latency " << latency
+                 << " skew " << skew << dendl;
+      }
+    }
+    do_output = false;
+    entity_inst_t inst = monmap->get_inst(*q);
+    dout(10) << __func__ << " send report to " << inst << dendl;
+    messenger->send_message(m, inst);
+  }
+}
+
+void Monitor::timecheck()
+{
+  dout(10) << __func__ << dendl;
+  assert(is_leader());
+  if (monmap->size() == 1) {
+    assert(0 == "We are alone; we shouldn't have gotten here!");
+    return;
+  }
+  assert(timecheck_round % 2 != 0);
+
+  timecheck_acks = 1; // we ack ourselves
+
+  dout(10) << __func__ << " start timecheck epoch " << get_epoch()
+           << " round " << timecheck_round << dendl;
+
+  // we are at the eye of the storm; the point of reference
+  timecheck_skews[messenger->get_myinst()] = 0.0;
+  timecheck_latencies[messenger->get_myinst()] = 0.0;
+
+  for (set::iterator it = quorum.begin(); it != quorum.end(); ++it) {
+    if (monmap->get_name(*it) == name)
+      continue;
+
+    entity_inst_t inst = monmap->get_inst(*it);
+    utime_t curr_time = ceph_clock_now(g_ceph_context);
+    timecheck_waiting[inst] = curr_time;
+    MTimeCheck *m = new MTimeCheck(MTimeCheck::OP_PING);
+    m->epoch = get_epoch();
+    m->round = timecheck_round;
+    dout(10) << __func__ << " send " << *m << " to " << inst << dendl;
+    messenger->send_message(m, inst);
+  }
+}
+
+health_status_t Monitor::timecheck_status(ostringstream &ss,
+                                          const double skew_bound,
+                                          const double latency)
+{
+  health_status_t status = HEALTH_OK;
+  double abs_skew = (skew_bound > 0 ? skew_bound : -skew_bound);
+  assert(latency >= 0);
+
+  if (abs_skew > g_conf->mon_clock_drift_allowed) {
+    status = HEALTH_WARN;
+    ss << "clock skew " << abs_skew << "s"
+       << " > max " << g_conf->mon_clock_drift_allowed << "s";
+  }
+
+  return status;
+}
+
+void Monitor::handle_timecheck_leader(MTimeCheck *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  /* handles PONG's */
+  assert(m->op == MTimeCheck::OP_PONG);
+
+  entity_inst_t other = m->get_source_inst();
+  if (m->epoch < get_epoch()) {
+    dout(1) << __func__ << " got old timecheck epoch " << m->epoch
+            << " from " << other
+            << " curr " << get_epoch()
+            << " -- severely lagged? discard" << dendl;
+    return;
+  }
+  assert(m->epoch == get_epoch());
+
+  if (m->round < timecheck_round) {
+    dout(1) << __func__ << " got old round " << m->round
+            << " from " << other
+            << " curr " << timecheck_round << " -- discard" << dendl;
+    return;
+  }
+
+  utime_t curr_time = ceph_clock_now(g_ceph_context);
+
+  assert(timecheck_waiting.count(other) > 0);
+  utime_t timecheck_sent = timecheck_waiting[other];
+  timecheck_waiting.erase(other);
+  if (curr_time < timecheck_sent) {
+    // our clock was readjusted -- drop everything until it all makes sense.
+    dout(1) << __func__ << " our clock was readjusted --"
+            << " bump round and drop current check"
+            << dendl;
+    timecheck_cancel_round();
+    return;
+  }
+
+  /* update peer latencies */
+  double latency = (double)(curr_time - timecheck_sent);
+
+  if (timecheck_latencies.count(other) == 0)
+    timecheck_latencies[other] = latency;
+  else {
+    double avg_latency = ((timecheck_latencies[other]*0.8)+(latency*0.2));
+    timecheck_latencies[other] = avg_latency;
+  }
+
+  /*
+   * update skews
+   *
+   * some nasty thing goes on if we were to do 'a - b' between two utime_t,
+   * and 'a' happens to be lower than 'b'; so we use double instead.
+   *
+   * latency is always expected to be >= 0.
+   *
+   * delta, the difference between theirs timestamp and ours, may either be
+   * lower or higher than 0; will hardly ever be 0.
+   *
+   * The absolute skew is the absolute delta minus the latency, which is
+   * taken as a whole instead of an rtt given that there is some queueing
+   * and dispatch times involved and it's hard to assess how long exactly
+   * it took for the message to travel to the other side and be handled. So
+   * we call it a bounded skew, the worst case scenario.
+   *
+   * Now, to math!
+   *
+   * Given that the latency is always positive, we can establish that the
+   * bounded skew will be:
+   *
+   *  1. positive if the absolute delta is higher than the latency and
+   *     delta is positive
+   *  2. negative if the absolute delta is higher than the latency and
+   *     delta is negative.
+   *  3. zero if the absolute delta is lower than the latency.
+   *
+   * On 3. we make a judgement call and treat the skew as non-existent.
+   * This is because that, if the absolute delta is lower than the
+   * latency, then the apparently existing skew is nothing more than a
+   * side-effect of the high latency at work.
+   *
+   * This may not be entirely true though, as a severely skewed clock
+   * may be masked by an even higher latency, but with high latencies
+   * we probably have worse issues to deal with than just skewed clocks.
+   */
+  assert(latency >= 0);
+
+  double delta = ((double) m->timestamp) - ((double) curr_time);
+  double abs_delta = (delta > 0 ? delta : -delta);
+  double skew_bound = abs_delta - latency;
+  if (skew_bound < 0)
+    skew_bound = 0;
+  else if (delta < 0)
+    skew_bound = -skew_bound;
+
+  ostringstream ss;
+  health_status_t status = timecheck_status(ss, skew_bound, latency);
+  if (status == HEALTH_ERR)
+    clog.error() << other << " " << ss.str() << "\n";
+  else if (status == HEALTH_WARN)
+    clog.warn() << other << " " << ss.str() << "\n";
+
+  dout(10) << __func__ << " from " << other << " ts " << m->timestamp
+	   << " delta " << delta << " skew_bound " << skew_bound
+	   << " latency " << latency << dendl;
+
+  if (timecheck_skews.count(other) == 0) {
+    timecheck_skews[other] = skew_bound;
+  } else {
+    timecheck_skews[other] = (timecheck_skews[other]*0.8)+(skew_bound*0.2);
+  }
+
+  timecheck_acks++;
+  if (timecheck_acks == quorum.size()) {
+    dout(10) << __func__ << " got pongs from everybody ("
+             << timecheck_acks << " total)" << dendl;
+    assert(timecheck_skews.size() == timecheck_acks);
+    assert(timecheck_waiting.empty());
+    // everyone has acked, so bump the round to finish it.
+    timecheck_finish_round();
+  }
+}
+
+void Monitor::handle_timecheck_peon(MTimeCheck *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+
+  assert(is_peon());
+  assert(m->op == MTimeCheck::OP_PING || m->op == MTimeCheck::OP_REPORT);
+
+  if (m->epoch != get_epoch()) {
+    dout(1) << __func__ << " got wrong epoch "
+            << "(ours " << get_epoch()
+            << " theirs: " << m->epoch << ") -- discarding" << dendl;
+    return;
+  }
+
+  if (m->round < timecheck_round) {
+    dout(1) << __func__ << " got old round " << m->round
+            << " current " << timecheck_round
+            << " (epoch " << get_epoch() << ") -- discarding" << dendl;
+    return;
+  }
+
+  timecheck_round = m->round;
+
+  if (m->op == MTimeCheck::OP_REPORT) {
+    assert((timecheck_round % 2) == 0);
+    timecheck_latencies.swap(m->latencies);
+    timecheck_skews.swap(m->skews);
+    return;
+  }
+
+  assert((timecheck_round % 2) != 0);
+  MTimeCheck *reply = new MTimeCheck(MTimeCheck::OP_PONG);
+  utime_t curr_time = ceph_clock_now(g_ceph_context);
+  reply->timestamp = curr_time;
+  reply->epoch = m->epoch;
+  reply->round = m->round;
+  dout(10) << __func__ << " send " << *m
+           << " to " << m->get_source_inst() << dendl;
+  messenger->send_message(reply, m->get_connection());
+}
+
+void Monitor::handle_timecheck(MTimeCheck *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+
+  if (is_leader()) {
+    if (m->op != MTimeCheck::OP_PONG) {
+      dout(1) << __func__ << " drop unexpected msg (not pong)" << dendl;
+    } else {
+      handle_timecheck_leader(m);
+    }
+  } else if (is_peon()) {
+    if (m->op != MTimeCheck::OP_PING && m->op != MTimeCheck::OP_REPORT) {
+      dout(1) << __func__ << " drop unexpected msg (not ping or report)" << dendl;
+    } else {
+      handle_timecheck_peon(m);
+    }
+  } else {
+    dout(1) << __func__ << " drop unexpected msg" << dendl;
+  }
+  m->put();
+}
+
+void Monitor::handle_subscribe(MMonSubscribe *m)
+{
+  dout(10) << "handle_subscribe " << *m << dendl;
+  
+  bool reply = false;
+
+  MonSession *s = static_cast(m->get_connection()->get_priv());
+  if (!s) {
+    dout(10) << " no session, dropping" << dendl;
+    m->put();
+    return;
+  }
+
+  s->until = ceph_clock_now(g_ceph_context);
+  s->until += g_conf->mon_subscribe_interval;
+  for (map::iterator p = m->what.begin();
+       p != m->what.end();
+       ++p) {
+    // if there are any non-onetime subscriptions, we need to reply to start the resubscribe timer
+    if ((p->second.flags & CEPH_SUBSCRIBE_ONETIME) == 0)
+      reply = true;
+
+    session_map.add_update_sub(s, p->first, p->second.start, 
+			       p->second.flags & CEPH_SUBSCRIBE_ONETIME,
+			       m->get_connection()->has_feature(CEPH_FEATURE_INCSUBOSDMAP));
+
+    if (p->first == "mdsmap") {
+      if ((int)s->is_capable("mds", MON_CAP_R)) {
+        mdsmon()->check_sub(s->sub_map["mdsmap"]);
+      }
+    } else if (p->first == "osdmap") {
+      if ((int)s->is_capable("osd", MON_CAP_R)) {
+        osdmon()->check_sub(s->sub_map["osdmap"]);
+      }
+    } else if (p->first == "osd_pg_creates") {
+      if ((int)s->is_capable("osd", MON_CAP_W)) {
+	pgmon()->check_sub(s->sub_map["osd_pg_creates"]);
+      }
+    } else if (p->first == "monmap") {
+      check_sub(s->sub_map["monmap"]);
+    } else if (logmon()->sub_name_to_id(p->first) >= 0) {
+      logmon()->check_sub(s->sub_map[p->first]);
+    }
+  }
+
+  // ???
+
+  if (reply)
+    messenger->send_message(new MMonSubscribeAck(monmap->get_fsid(), (int)g_conf->mon_subscribe_interval),
+			    m->get_source_inst());
+
+  s->put();
+  m->put();
+}
+
+void Monitor::handle_get_version(MMonGetVersion *m)
+{
+  dout(10) << "handle_get_version " << *m << dendl;
+  PaxosService *svc = NULL;
+
+  MonSession *s = static_cast(m->get_connection()->get_priv());
+  if (!s) {
+    dout(10) << " no session, dropping" << dendl;
+    m->put();
+    return;
+  }
+
+  if (!is_leader() && !is_peon()) {
+    dout(10) << " waiting for quorum" << dendl;
+    waitfor_quorum.push_back(new C_RetryMessage(this, m));
+    goto out;
+  }
+
+  if (m->what == "mdsmap") {
+    svc = mdsmon();
+  } else if (m->what == "osdmap") {
+    svc = osdmon();
+  } else if (m->what == "monmap") {
+    svc = monmon();
+  } else {
+    derr << "invalid map type " << m->what << dendl;
+  }
+
+  if (svc) {
+    if (!svc->is_readable()) {
+      svc->wait_for_readable(new C_RetryMessage(this, m));
+      goto out;
+    }
+    MMonGetVersionReply *reply = new MMonGetVersionReply();
+    reply->handle = m->handle;
+    reply->version = svc->get_last_committed();
+    reply->oldest_version = svc->get_first_committed();
+    messenger->send_message(reply, m->get_source_inst());
+  }
+
+  m->put();
+
+ out:
+  s->put();
+}
+
+bool Monitor::ms_handle_reset(Connection *con)
+{
+  dout(10) << "ms_handle_reset " << con << " " << con->get_peer_addr() << dendl;
+
+  // ignore lossless monitor sessions
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON)
+    return false;
+
+  MonSession *s = static_cast(con->get_priv());
+  if (!s)
+    return false;
+
+  // break any con <-> session ref cycle
+  s->con->set_priv(NULL);
+
+  if (is_shutdown())
+    return false;
+
+  Mutex::Locker l(lock);
+
+  dout(10) << "reset/close on session " << s->inst << dendl;
+  if (!s->closed)
+    remove_session(s);
+  s->put();
+  return true;
+}
+
+void Monitor::check_subs()
+{
+  string type = "monmap";
+  if (session_map.subs.count(type) == 0)
+    return;
+  xlist::iterator p = session_map.subs[type]->begin();
+  while (!p.end()) {
+    Subscription *sub = *p;
+    ++p;
+    check_sub(sub);
+  }
+}
+
+void Monitor::check_sub(Subscription *sub)
+{
+  dout(10) << "check_sub monmap next " << sub->next << " have " << monmap->get_epoch() << dendl;
+  if (sub->next <= monmap->get_epoch()) {
+    send_latest_monmap(sub->session->con.get());
+    if (sub->onetime)
+      session_map.remove_sub(sub);
+    else
+      sub->next = monmap->get_epoch() + 1;
+  }
+}
+
+
+// -----
+
+void Monitor::send_latest_monmap(Connection *con)
+{
+  bufferlist bl;
+  monmap->encode(bl, con->get_features());
+  messenger->send_message(new MMonMap(bl), con);
+}
+
+void Monitor::handle_mon_get_map(MMonGetMap *m)
+{
+  dout(10) << "handle_mon_get_map" << dendl;
+  send_latest_monmap(m->get_connection().get());
+  m->put();
+}
+
+
+
+// ----------------------------------------------
+// scrub
+
+int Monitor::scrub()
+{
+  dout(10) << __func__ << dendl;
+  assert(is_leader());
+
+  if ((get_quorum_features() & CEPH_FEATURE_MON_SCRUB) == 0) {
+    clog.warn() << "scrub not supported by entire quorum\n";
+    return -EOPNOTSUPP;
+  }
+
+  if (!scrub_result.empty()) {
+    clog.info() << "scrub already in progress\n";
+    return -EBUSY;
+  }
+
+  scrub_result.clear();
+  scrub_version = paxos->get_version();
+
+  for (set::iterator p = quorum.begin();
+       p != quorum.end();
+       ++p) {
+    if (*p == rank)
+      continue;
+    MMonScrub *r = new MMonScrub(MMonScrub::OP_SCRUB, scrub_version);
+    messenger->send_message(r, monmap->get_inst(*p));
+  }
+
+  // scrub my keys
+  _scrub(&scrub_result[rank]);
+
+  if (scrub_result.size() == quorum.size())
+    scrub_finish();
+
+  return 0;
+}
+
+void Monitor::handle_scrub(MMonScrub *m)
+{
+  dout(10) << __func__ << " " << *m << dendl;
+  switch (m->op) {
+  case MMonScrub::OP_SCRUB:
+    {
+      if (!is_peon())
+	break;
+      if (m->version != paxos->get_version())
+	break;
+      MMonScrub *reply = new MMonScrub(MMonScrub::OP_RESULT, m->version);
+      _scrub(&reply->result);
+      messenger->send_message(reply, m->get_connection());
+    }
+    break;
+
+  case MMonScrub::OP_RESULT:
+    {
+      if (!is_leader())
+	break;
+      if (m->version != scrub_version)
+	break;
+      int from = m->get_source().num();
+      assert(scrub_result.count(from) == 0);
+      scrub_result[from] = m->result;
+
+      if (scrub_result.size() == quorum.size())
+	scrub_finish();
+    }
+    break;
+  }
+  m->put();
+}
+
+void Monitor::_scrub(ScrubResult *r)
+{
+  set prefixes = get_sync_targets_names();
+  prefixes.erase("paxos");  // exclude paxos, as this one may have extra states for proposals, etc.
+
+  dout(10) << __func__ << " prefixes " << prefixes << dendl;
+
+  pair start;
+  MonitorDBStore::Synchronizer synchronizer = store->get_synchronizer(start, prefixes);
+
+  while (synchronizer->has_next_chunk()) {
+    pair k = synchronizer->get_next_key();
+    bufferlist bl;
+    store->get(k.first, k.second, bl);
+    dout(30) << __func__ << " " << k << " bl " << bl.length() << " bytes crc " << bl.crc32c(0) << dendl;
+    r->prefix_keys[k.first]++;
+    if (r->prefix_crc.count(k.first) == 0)
+      r->prefix_crc[k.first] = 0;
+    r->prefix_crc[k.first] = bl.crc32c(r->prefix_crc[k.first]);
+  }
+}
+
+void Monitor::scrub_finish()
+{
+  dout(10) << __func__ << dendl;
+
+  // compare
+  int errors = 0;
+  ScrubResult& mine = scrub_result[rank];
+  for (map::iterator p = scrub_result.begin();
+       p != scrub_result.end();
+       ++p) {
+    if (p->first == rank)
+      continue;
+    if (p->second != mine) {
+      ++errors;
+      clog.error() << "scrub mismatch" << "\n";
+      clog.error() << " mon." << rank << " " << mine << "\n";
+      clog.error() << " mon." << p->first << " " << p->second << "\n";
+    }
+  }
+  if (!errors)
+    clog.info() << "scrub ok on " << quorum << ": " << mine << "\n";
+
+  scrub_reset();
+}
+
+void Monitor::scrub_reset()
+{
+  dout(10) << __func__ << dendl;
+  scrub_version = 0;
+  scrub_result.clear();
+}
+
+
+
+/************ TICK ***************/
+
+class C_Mon_Tick : public Context {
+  Monitor *mon;
+public:
+  C_Mon_Tick(Monitor *m) : mon(m) {}
+  void finish(int r) {
+    mon->tick();
+  }
+};
+
+void Monitor::new_tick()
+{
+  C_Mon_Tick *ctx = new C_Mon_Tick(this);
+  timer.add_event_after(g_conf->mon_tick_interval, ctx);
+}
+
+void Monitor::tick()
+{
+  // ok go.
+  dout(11) << "tick" << dendl;
+  
+  for (vector::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p) {
+    (*p)->tick();
+    (*p)->maybe_trim();
+  }
+  
+  // trim sessions
+  utime_t now = ceph_clock_now(g_ceph_context);
+  xlist::iterator p = session_map.sessions.begin();
+  while (!p.end()) {
+    MonSession *s = *p;
+    ++p;
+    
+    // don't trim monitors
+    if (s->inst.name.is_mon())
+      continue; 
+
+    if (!s->until.is_zero() && s->until < now) {
+      dout(10) << " trimming session " << s->con << " " << s->inst
+	       << " (until " << s->until << " < now " << now << ")" << dendl;
+      messenger->mark_down(s->con);
+      remove_session(s);
+    } else if (!exited_quorum.is_zero()) {
+      if (now > (exited_quorum + 2 * g_conf->mon_lease)) {
+        // boot the client Session because we've taken too long getting back in
+        dout(10) << " trimming session " << s->con << " " << s->inst
+		 << " because we've been out of quorum too long" << dendl;
+        messenger->mark_down(s->con);
+        remove_session(s);
+      }
+    }
+  }
+
+  sync_trim_providers();
+
+  if (!maybe_wait_for_quorum.empty()) {
+    finish_contexts(g_ceph_context, maybe_wait_for_quorum);
+  }
+
+  if (is_leader() && paxos->is_active() && fingerprint.is_zero()) {
+    // this is only necessary on upgraded clusters.
+    MonitorDBStore::Transaction t;
+    prepare_new_fingerprint(&t);
+    bufferlist tbl;
+    t.encode(tbl);
+    paxos->propose_new_value(tbl, new C_NoopContext);
+  }
+
+  new_tick();
+}
+
+void Monitor::prepare_new_fingerprint(MonitorDBStore::Transaction *t)
+{
+  uuid_d nf;
+  nf.generate_random();
+  dout(10) << __func__ << " proposing cluster_fingerprint " << nf << dendl;
+
+  bufferlist bl;
+  ::encode(nf, bl);
+  t->put(MONITOR_NAME, "cluster_fingerprint", bl);
+}
+
+int Monitor::check_fsid()
+{
+  if (!store->exists(MONITOR_NAME, "cluster_uuid"))
+    return -ENOENT;
+
+  bufferlist ebl;
+  int r = store->get(MONITOR_NAME, "cluster_uuid", ebl);
+  assert(r == 0);
+
+  string es(ebl.c_str(), ebl.length());
+
+  // only keep the first line
+  size_t pos = es.find_first_of('\n');
+  if (pos != string::npos)
+    es.resize(pos);
+
+  dout(10) << "check_fsid cluster_uuid contains '" << es << "'" << dendl;
+  uuid_d ondisk;
+  if (!ondisk.parse(es.c_str())) {
+    derr << "error: unable to parse uuid" << dendl;
+    return -EINVAL;
+  }
+
+  if (monmap->get_fsid() != ondisk) {
+    derr << "error: cluster_uuid file exists with value " << ondisk
+	 << ", != our uuid " << monmap->get_fsid() << dendl;
+    return -EEXIST;
+  }
+
+  return 0;
+}
+
+int Monitor::write_fsid()
+{
+  MonitorDBStore::Transaction t;
+  int r = write_fsid(t);
+  store->apply_transaction(t);
+  return r;
+}
+
+int Monitor::write_fsid(MonitorDBStore::Transaction &t)
+{
+  ostringstream ss;
+  ss << monmap->get_fsid() << "\n";
+  string us = ss.str();
+
+  bufferlist b;
+  b.append(us);
+
+  t.put(MONITOR_NAME, "cluster_uuid", b);
+  return 0;
+}
+
+/*
+ * this is the closest thing to a traditional 'mkfs' for ceph.
+ * initialize the monitor state machines to their initial values.
+ */
+int Monitor::mkfs(bufferlist& osdmapbl)
+{
+  MonitorDBStore::Transaction t;
+
+  // verify cluster fsid
+  int r = check_fsid();
+  if (r < 0 && r != -ENOENT)
+    return r;
+
+  bufferlist magicbl;
+  magicbl.append(CEPH_MON_ONDISK_MAGIC);
+  magicbl.append("\n");
+  t.put(MONITOR_NAME, "magic", magicbl);
+
+
+  features = get_supported_features();
+  write_features(t);
+
+  // save monmap, osdmap, keyring.
+  bufferlist monmapbl;
+  monmap->encode(monmapbl, CEPH_FEATURES_ALL);
+  monmap->set_epoch(0);     // must be 0 to avoid confusing first MonmapMonitor::update_from_paxos()
+  t.put("mkfs", "monmap", monmapbl);
+
+  if (osdmapbl.length()) {
+    // make sure it's a valid osdmap
+    try {
+      OSDMap om;
+      om.decode(osdmapbl);
+    }
+    catch (buffer::error& e) {
+      derr << "error decoding provided osdmap: " << e.what() << dendl;
+      return -EINVAL;
+    }
+    t.put("mkfs", "osdmap", osdmapbl);
+  }
+
+  if (is_keyring_required()) {
+    KeyRing keyring;
+    string keyring_filename;
+    if (!ceph_resolve_file_search(g_conf->keyring, keyring_filename)) {
+      derr << "unable to find a keyring file on " << g_conf->keyring << dendl;
+      if (g_conf->key != "") {
+	string keyring_plaintext = "[mon.]\n\tkey = " + g_conf->key +
+	  "\n\tcaps mon = \"allow *\"\n";
+	bufferlist bl;
+	bl.append(keyring_plaintext);
+	try {
+	  bufferlist::iterator i = bl.begin();
+	  keyring.decode_plaintext(i);
+	}
+	catch (const buffer::error& e) {
+	  derr << "error decoding keyring " << keyring_plaintext
+	       << ": " << e.what() << dendl;
+	  return -EINVAL;
+	}
+      } else {
+	return -ENOENT;
+      }
+    } else {
+      r = keyring.load(g_ceph_context, keyring_filename);
+      if (r < 0) {
+	derr << "unable to load initial keyring " << g_conf->keyring << dendl;
+	return r;
+      }
+    }
+
+    // put mon. key in external keyring; seed with everything else.
+    extract_save_mon_key(keyring);
+
+    bufferlist keyringbl;
+    keyring.encode_plaintext(keyringbl);
+    t.put("mkfs", "keyring", keyringbl);
+  }
+  write_fsid(t);
+  store->apply_transaction(t);
+
+  return 0;
+}
+
+int Monitor::write_default_keyring(bufferlist& bl)
+{
+  ostringstream os;
+  os << g_conf->mon_data << "/keyring";
+
+  int err = 0;
+  int fd = ::open(os.str().c_str(), O_WRONLY|O_CREAT, 0644);
+  if (fd < 0) {
+    err = -errno;
+    dout(0) << __func__ << " failed to open " << os.str() 
+	    << ": " << cpp_strerror(err) << dendl;
+    return err;
+  }
+
+  err = bl.write_fd(fd);
+  if (!err)
+    ::fsync(fd);
+  ::close(fd);
+
+  return err;
+}
+
+void Monitor::extract_save_mon_key(KeyRing& keyring)
+{
+  EntityName mon_name;
+  mon_name.set_type(CEPH_ENTITY_TYPE_MON);
+  EntityAuth mon_key;
+  if (keyring.get_auth(mon_name, mon_key)) {
+    dout(10) << "extract_save_mon_key moving mon. key to separate keyring" << dendl;
+    KeyRing pkey;
+    pkey.add(mon_name, mon_key);
+    bufferlist bl;
+    pkey.encode_plaintext(bl);
+    write_default_keyring(bl);
+    keyring.remove(mon_name);
+  }
+}
+
+bool Monitor::ms_get_authorizer(int service_id, AuthAuthorizer **authorizer, bool force_new)
+{
+  dout(10) << "ms_get_authorizer for " << ceph_entity_type_name(service_id) << dendl;
+
+  if (is_shutdown())
+    return false;
+
+  // we only connect to other monitors; every else connects to us.
+  if (service_id != CEPH_ENTITY_TYPE_MON)
+    return false;
+
+  if (!auth_cluster_required.is_supported_auth(CEPH_AUTH_CEPHX))
+    return false;
+
+  CephXServiceTicketInfo auth_ticket_info;
+  CephXSessionAuthInfo info;
+  int ret;
+  EntityName name;
+  name.set_type(CEPH_ENTITY_TYPE_MON);
+
+  auth_ticket_info.ticket.name = name;
+  auth_ticket_info.ticket.global_id = 0;
+
+  CryptoKey secret;
+  if (!keyring.get_secret(name, secret) &&
+      !key_server.get_secret(name, secret)) {
+    dout(0) << " couldn't get secret for mon service from keyring or keyserver" << dendl;
+    stringstream ss, ds;
+    int err = key_server.list_secrets(ds);
+    if (err < 0)
+      ss << "no installed auth entries!";
+    else
+      ss << "installed auth entries:";
+    dout(0) << ss.str() << "\n" << ds.str() << dendl;
+    return false;
+  }
+
+  /* mon to mon authentication uses the private monitor shared key and not the
+     rotating key */
+  ret = key_server.build_session_auth_info(service_id, auth_ticket_info, info, secret, (uint64_t)-1);
+  if (ret < 0) {
+    dout(0) << "ms_get_authorizer failed to build session auth_info for use with mon ret " << ret << dendl;
+    return false;
+  }
+
+  CephXTicketBlob blob;
+  if (!cephx_build_service_ticket_blob(cct, info, blob)) {
+    dout(0) << "ms_get_authorizer failed to build service ticket use with mon" << dendl;
+    return false;
+  }
+  bufferlist ticket_data;
+  ::encode(blob, ticket_data);
+
+  bufferlist::iterator iter = ticket_data.begin();
+  CephXTicketHandler handler(g_ceph_context, service_id);
+  ::decode(handler.ticket, iter);
+
+  handler.session_key = info.session_key;
+
+  *authorizer = handler.build_authorizer(0);
+  
+  return true;
+}
+
+bool Monitor::ms_verify_authorizer(Connection *con, int peer_type,
+				   int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply,
+				   bool& isvalid, CryptoKey& session_key)
+{
+  dout(10) << "ms_verify_authorizer " << con->get_peer_addr()
+	   << " " << ceph_entity_type_name(peer_type)
+	   << " protocol " << protocol << dendl;
+
+  if (is_shutdown())
+    return false;
+
+  if (peer_type == CEPH_ENTITY_TYPE_MON &&
+      auth_cluster_required.is_supported_auth(CEPH_AUTH_CEPHX)) {
+    // monitor, and cephx is enabled
+    isvalid = false;
+    if (protocol == CEPH_AUTH_CEPHX) {
+      bufferlist::iterator iter = authorizer_data.begin();
+      CephXServiceTicketInfo auth_ticket_info;
+      
+      if (authorizer_data.length()) {
+	int ret = cephx_verify_authorizer(g_ceph_context, &keyring, iter,
+					  auth_ticket_info, authorizer_reply);
+	if (ret >= 0) {
+	  session_key = auth_ticket_info.session_key;
+	  isvalid = true;
+	} else {
+	  dout(0) << "ms_verify_authorizer bad authorizer from mon " << con->get_peer_addr() << dendl;
+        }
+      }
+    } else {
+      dout(0) << "ms_verify_authorizer cephx enabled, but no authorizer (required for mon)" << dendl;
+    }
+  } else {
+    // who cares.
+    isvalid = true;
+  }
+  return true;
+};
+
+#undef dout_prefix
+#define dout_prefix *_dout
+
+void Monitor::StoreConverter::_convert_finish_features(
+    MonitorDBStore::Transaction &t)
+{
+  dout(20) << __func__ << dendl;
+
+  assert(db->exists(MONITOR_NAME, COMPAT_SET_LOC));
+  bufferlist features_bl;
+  db->get(MONITOR_NAME, COMPAT_SET_LOC, features_bl);
+  assert(features_bl.length());
+
+  CompatSet features;
+  bufferlist::iterator p = features_bl.begin();
+  features.decode(p);
+
+  assert(features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_GV));
+  features.incompat.remove(CEPH_MON_FEATURE_INCOMPAT_GV);
+  assert(!features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_GV));
+
+  features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS);
+  assert(features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS));
+
+  features_bl.clear();
+  features.encode(features_bl);
+
+  dout(20) << __func__ << " new features " << features << dendl;
+  t.put(MONITOR_NAME, COMPAT_SET_LOC, features_bl);
+}
+
+
+bool Monitor::StoreConverter::_check_gv_store()
+{
+  dout(20) << __func__ << dendl;
+  if (!store->exists_bl_ss(COMPAT_SET_LOC, 0))
+    return false;
+
+  bufferlist features_bl;
+  store->get_bl_ss_safe(features_bl, COMPAT_SET_LOC, 0);
+  if (!features_bl.length()) {
+    dout(20) << __func__ << " on-disk features length is zero" << dendl;
+    return false;
+  }
+  CompatSet features;
+  bufferlist::iterator p = features_bl.begin();
+  features.decode(p);
+  return (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_GV));
+}
+
+int Monitor::StoreConverter::needs_conversion()
+{
+  bufferlist magicbl;
+  int ret = 0;
+
+  dout(10) << "check if store needs conversion from legacy format" << dendl;
+  _init();
+
+  int err = store->mount();
+  if (err < 0) {
+    if (err == -ENOENT) {
+      derr << "unable to mount monitor store: "
+	   << cpp_strerror(err) << dendl;
+    } else {
+      derr << "it appears that another monitor is running: "
+	   << cpp_strerror(err) << dendl;
+    }
+    ret = err;
+    goto out;
+  }
+  assert(err == 0);
+
+  if (store->exists_bl_ss("magic", 0)) {
+    if (_check_gv_store()) {
+      dout(1) << "found old GV monitor store format "
+	      << "-- should convert!" << dendl;
+      ret = 1;
+    } else {
+      dout(0) << "Existing monitor store has not been converted "
+	      << "to 0.52 (bobtail) format" << dendl;
+      assert(0 == "Existing store has not been converted to 0.52 format");
+    }
+  }
+  assert(!store->umount());
+
+out:
+  _deinit();
+  return ret;
+}
+
+int Monitor::StoreConverter::convert()
+{
+  _init();
+  assert(!store->mount());
+  if (db->exists("mon_convert", "on_going")) {
+    dout(0) << __func__ << " found a mon store in mid-convertion; abort!"
+      << dendl;
+    return -EEXIST;
+  }
+
+  _mark_convert_start();
+  _convert_monitor();
+  _convert_machines();
+  _convert_paxos();
+  _mark_convert_finish();
+
+  store->umount();
+  _deinit();
+
+  dout(0) << __func__ << " finished conversion" << dendl;
+
+  return 0;
+}
+
+void Monitor::StoreConverter::_convert_monitor()
+{
+  dout(10) << __func__ << dendl;
+
+  assert(store->exists_bl_ss("magic"));
+  assert(store->exists_bl_ss("keyring"));
+  assert(store->exists_bl_ss("feature_set"));
+  assert(store->exists_bl_ss("election_epoch"));
+
+  MonitorDBStore::Transaction tx;
+
+  if (store->exists_bl_ss("joined")) {
+    version_t joined = store->get_int("joined");
+    tx.put(MONITOR_NAME, "joined", joined);
+  }
+
+  vector keys;
+  keys.push_back("magic");
+  keys.push_back("feature_set");
+  keys.push_back("cluster_uuid");
+
+  vector::iterator it;
+  for (it = keys.begin(); it != keys.end(); ++it) {
+    if (!store->exists_bl_ss((*it).c_str()))
+      continue;
+
+    bufferlist bl;
+    int r = store->get_bl_ss(bl, (*it).c_str(), 0);
+    assert(r > 0);
+    tx.put(MONITOR_NAME, *it, bl);
+  }
+  version_t election_epoch = store->get_int("election_epoch");
+  tx.put(MONITOR_NAME, "election_epoch", election_epoch);
+
+  assert(!tx.empty());
+  db->apply_transaction(tx);
+  dout(10) << __func__ << " finished" << dendl;
+}
+
+void Monitor::StoreConverter::_convert_machines(string machine)
+{
+  dout(10) << __func__ << " " << machine << dendl;
+
+  version_t first_committed =
+    store->get_int(machine.c_str(), "first_committed");
+  version_t last_committed =
+    store->get_int(machine.c_str(), "last_committed");
+
+  version_t accepted_pn = store->get_int(machine.c_str(), "accepted_pn");
+  version_t last_pn = store->get_int(machine.c_str(), "last_pn");
+
+  if (accepted_pn > highest_accepted_pn)
+    highest_accepted_pn = accepted_pn;
+  if (last_pn > highest_last_pn)
+    highest_last_pn = last_pn;
+
+  string machine_gv(machine);
+  machine_gv.append("_gv");
+  bool has_gv = true;
+
+  if (!store->exists_bl_ss(machine_gv.c_str())) {
+    dout(1) << __func__ << " " << machine
+      << " no gv dir '" << machine_gv << "'" << dendl;
+    has_gv = false;
+  }
+
+  for (version_t ver = first_committed; ver <= last_committed; ver++) {
+    if (!store->exists_bl_sn(machine.c_str(), ver)) {
+      dout(20) << __func__ << " " << machine
+	       << " ver " << ver << " dne" << dendl;
+      continue;
+    }
+
+    bufferlist bl;
+    int r = store->get_bl_sn(bl, machine.c_str(), ver);
+    assert(r >= 0);
+    dout(20) << __func__ << " " << machine
+	     << " ver " << ver << " bl " << bl.length() << dendl;
+
+    MonitorDBStore::Transaction tx;
+    tx.put(machine, ver, bl);
+    tx.put(machine, "last_committed", ver);
+
+    if (has_gv && store->exists_bl_sn(machine_gv.c_str(), ver)) {
+      stringstream s;
+      s << ver;
+      string ver_str = s.str();
+
+      version_t gv = store->get_int(machine_gv.c_str(), ver_str.c_str());
+      dout(20) << __func__ << " " << machine
+	       << " ver " << ver << " -> " << gv << dendl;
+
+      MonitorDBStore::Transaction paxos_tx;
+
+      if (gvs.count(gv) == 0) {
+        gvs.insert(gv);
+      } else {
+	dout(0) << __func__ << " " << machine
+		<< " gv " << gv << " already exists"
+		<< dendl;
+
+        // Duplicates aren't supposed to happen, but an old bug introduced
+	// them and the mds state machine wasn't ever trimmed, so many users
+	// will see them.  So we'll just merge them all in one
+        // single paxos version.
+        // We know that they are either from another paxos machine or
+        // they are from the same paxos machine but their version is
+        // lower than ours -- given that we are iterating all versions
+        // from the lowest to the highest, duh!
+        // We'll just append our stuff to the existing paxos transaction
+        // as if nothing had happened.
+
+        // Just make sure we are correct. This shouldn't take long and
+        // should never be triggered!
+        set >& s = gv_map[gv];
+        for (set >::iterator it = s.begin();
+             it != s.end(); ++it) {
+          if (it->first == machine)
+            assert(it->second + 1 == ver);
+        }
+
+        bufferlist paxos_bl;
+        int r = db->get("paxos", gv, paxos_bl);
+        assert(r >= 0);
+        paxos_tx.append_from_encoded(paxos_bl);
+      }
+      gv_map[gv].insert(make_pair(machine,ver));
+
+      bufferlist tx_bl;
+      tx.encode(tx_bl);
+      paxos_tx.append_from_encoded(tx_bl);
+      bufferlist paxos_bl;
+      paxos_tx.encode(paxos_bl);
+      tx.put("paxos", gv, paxos_bl);
+    }
+    db->apply_transaction(tx);
+  }
+
+  version_t lc = db->get(machine, "last_committed");
+  dout(20) << __func__ << " lc " << lc << " last_committed " << last_committed << dendl;
+  assert(lc == last_committed);
+
+  MonitorDBStore::Transaction tx;
+  tx.put(machine, "first_committed", first_committed);
+  tx.put(machine, "last_committed", last_committed);
+  tx.put(machine, "conversion_first", first_committed);
+
+  if (store->exists_bl_ss(machine.c_str(), "latest")) {
+    bufferlist latest_bl_raw;
+    int r = store->get_bl_ss(latest_bl_raw, machine.c_str(), "latest");
+    assert(r >= 0);
+    if (!latest_bl_raw.length()) {
+      dout(20) << __func__ << " machine " << machine
+	       << " skip latest with size 0" << dendl;
+      goto out;
+    }
+
+    tx.put(machine, "latest", latest_bl_raw);
+
+    bufferlist::iterator lbl_it = latest_bl_raw.begin();
+    bufferlist latest_bl;
+    version_t latest_ver;
+    ::decode(latest_ver, lbl_it);
+    ::decode(latest_bl, lbl_it);
+
+    dout(20) << __func__ << " machine " << machine
+	     << " latest ver " << latest_ver << dendl;
+
+    tx.put(machine, "full_latest", latest_ver);
+    stringstream os;
+    os << "full_" << latest_ver;
+    tx.put(machine, os.str(), latest_bl);
+  }
+out:
+  db->apply_transaction(tx);
+  dout(10) << __func__ << " machine " << machine << " finished" << dendl;
+}
+
+void Monitor::StoreConverter::_convert_osdmap_full()
+{
+  dout(10) << __func__ << dendl;
+  version_t first_committed =
+    store->get_int("osdmap", "first_committed");
+  version_t last_committed =
+    store->get_int("osdmap", "last_committed");
+
+  int err = 0;
+  for (version_t ver = first_committed; ver <= last_committed; ver++) {
+    if (!store->exists_bl_sn("osdmap_full", ver)) {
+      dout(20) << __func__ << " osdmap_full  ver " << ver << " dne" << dendl;
+      err++;
+      continue;
+    }
+
+    bufferlist bl;
+    int r = store->get_bl_sn(bl, "osdmap_full", ver);
+    assert(r >= 0);
+    dout(20) << __func__ << " osdmap_full ver " << ver
+             << " bl " << bl.length() << " bytes" << dendl;
+
+    string full_key = "full_" + stringify(ver);
+    MonitorDBStore::Transaction tx;
+    tx.put("osdmap", full_key, bl);
+    db->apply_transaction(tx);
+  }
+  dout(10) << __func__ << " found " << err << " conversion errors!" << dendl;
+  assert(err == 0);
+}
+
+void Monitor::StoreConverter::_convert_paxos()
+{
+  dout(10) << __func__ << dendl;
+  assert(!gvs.empty());
+
+  set::reverse_iterator rit = gvs.rbegin();
+  version_t highest_gv = *rit;
+  version_t last_gv = highest_gv;
+
+  int n = 0;
+  int max_versions = (g_conf->paxos_max_join_drift*2);
+  for (; (rit != gvs.rend()) && (n < max_versions); ++rit, ++n) {
+    version_t gv = *rit;
+
+    if (last_gv == gv)
+      continue;
+    if ((last_gv - gv) > 1) {
+      // we are done; we found a gap and we are only interested in keeping
+      // contiguous paxos versions.
+      break;
+    }
+    last_gv = gv;
+  }
+
+  // erase all paxos versions between [first, last_gv[, with first being the
+  // first gv in the map.
+  MonitorDBStore::Transaction tx;
+  set::iterator it = gvs.begin();
+  dout(1) << __func__ << " first gv " << (*it)
+	  << " last gv " << last_gv << dendl;
+  for (; it != gvs.end() && (*it < last_gv); ++it) {
+    tx.erase("paxos", *it);
+  }
+  tx.put("paxos", "first_committed", last_gv);
+  tx.put("paxos", "last_committed", highest_gv);
+  tx.put("paxos", "accepted_pn", highest_accepted_pn);
+  tx.put("paxos", "last_pn", highest_last_pn);
+  tx.put("paxos", "conversion_first", last_gv);
+  db->apply_transaction(tx);
+
+  dout(10) << __func__ << " finished" << dendl;
+}
+
+void Monitor::StoreConverter::_convert_machines()
+{
+  dout(10) << __func__ << dendl;
+  set machine_names = _get_machines_names();
+  set::iterator it = machine_names.begin();
+
+  for (; it != machine_names.end(); ++it) {
+    _convert_machines(*it);
+  }
+  // convert osdmap full versions
+  // this stays here as these aren't really an independent paxos
+  // machine, but rather machine-specific and don't fit on the
+  // _convert_machines(string) function.
+  _convert_osdmap_full();
+
+  dout(10) << __func__ << " finished" << dendl;
+}
diff --git a/ceph/src/mon/Monitor.h b/ceph/src/mon/Monitor.h
new file mode 100644
index 00000000..42e148ee
--- /dev/null
+++ b/ceph/src/mon/Monitor.h
@@ -0,0 +1,965 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/* 
+ * This is the top level monitor. It runs on each machine in the Monitor   
+ * Cluster. The election of a leader for the paxos algorithm only happens 
+ * once per machine via the elector. There is a separate paxos instance (state) 
+ * kept for each of the system components: Object Store Device (OSD) Monitor, 
+ * Placement Group (PG) Monitor, Metadata Server (MDS) Monitor, and Client Monitor.
+ */
+
+#ifndef CEPH_MONITOR_H
+#define CEPH_MONITOR_H
+
+#include "include/types.h"
+#include "msg/Messenger.h"
+
+#include "common/Timer.h"
+
+#include "MonMap.h"
+#include "Elector.h"
+#include "Paxos.h"
+#include "Session.h"
+
+#include "osd/OSDMap.h"
+
+#include "common/LogClient.h"
+#include "common/SimpleRNG.h"
+#include "common/cmdparse.h"
+
+#include "auth/cephx/CephxKeyServer.h"
+#include "auth/AuthMethodList.h"
+#include "auth/KeyRing.h"
+
+#include "perfglue/heap_profiler.h"
+
+#include "messages/MMonCommand.h"
+#include "messages/MPing.h"
+#include "mon/MonitorStore.h"
+#include "mon/MonitorDBStore.h"
+
+#include 
+#include "include/memory.h"
+#include 
+
+
+#define CEPH_MON_PROTOCOL     13 /* cluster internal */
+
+
+enum {
+  l_cluster_first = 555000,
+  l_cluster_num_mon,
+  l_cluster_num_mon_quorum,
+  l_cluster_num_osd,
+  l_cluster_num_osd_up,
+  l_cluster_num_osd_in,
+  l_cluster_osd_epoch,
+  l_cluster_osd_kb,
+  l_cluster_osd_kb_used,
+  l_cluster_osd_kb_avail,
+  l_cluster_num_pool,
+  l_cluster_num_pg,
+  l_cluster_num_pg_active_clean,
+  l_cluster_num_pg_active,
+  l_cluster_num_pg_peering,
+  l_cluster_num_object,
+  l_cluster_num_object_degraded,
+  l_cluster_num_object_unfound,
+  l_cluster_num_bytes,
+  l_cluster_num_mds_up,
+  l_cluster_num_mds_in,
+  l_cluster_num_mds_failed,
+  l_cluster_mds_epoch,
+  l_cluster_last,
+};
+
+class QuorumService;
+class PaxosService;
+
+class PerfCounters;
+class AdminSocketHook;
+
+class MMonGetMap;
+class MMonGetVersion;
+class MMonSync;
+class MMonScrub;
+class MMonProbe;
+struct MMonSubscribe;
+class MAuthRotating;
+struct MRoute;
+struct MForward;
+struct MTimeCheck;
+struct MMonHealth;
+struct MonCommand;
+
+#define COMPAT_SET_LOC "feature_set"
+
+class Monitor : public Dispatcher {
+public:
+  // me
+  string name;
+  int rank;
+  Messenger *messenger;
+  ConnectionRef con_self;
+  Mutex lock;
+  SafeTimer timer;
+  
+  /// true if we have ever joined a quorum.  if false, we are either a
+  /// new cluster, a newly joining monitor, or a just-upgraded
+  /// monitor.
+  bool has_ever_joined;
+
+  PerfCounters *logger, *cluster_logger;
+  bool cluster_logger_registered;
+
+  void register_cluster_logger();
+  void unregister_cluster_logger();
+
+  MonMap *monmap;
+  uuid_d fingerprint;
+
+  set extra_probe_peers;
+
+  LogClient clog;
+  KeyRing keyring;
+  KeyServer key_server;
+
+  AuthMethodList auth_cluster_required;
+  AuthMethodList auth_service_required;
+
+  CompatSet features;
+
+  const MonCommand *leader_supported_mon_commands;
+  int leader_supported_mon_commands_size;
+
+private:
+  void new_tick();
+  friend class C_Mon_Tick;
+
+  // -- local storage --
+public:
+  MonitorDBStore *store;
+  static const string MONITOR_NAME;
+  static const string MONITOR_STORE_PREFIX;
+
+  // -- monitor state --
+private:
+  enum {
+    STATE_PROBING = 1,
+    STATE_SYNCHRONIZING,
+    STATE_ELECTING,
+    STATE_LEADER,
+    STATE_PEON,
+    STATE_SHUTDOWN
+  };
+  int state;
+
+public:
+  static const char *get_state_name(int s) {
+    switch (s) {
+    case STATE_PROBING: return "probing";
+    case STATE_SYNCHRONIZING: return "synchronizing";
+    case STATE_ELECTING: return "electing";
+    case STATE_LEADER: return "leader";
+    case STATE_PEON: return "peon";
+    case STATE_SHUTDOWN: return "shutdown";
+    default: return "???";
+    }
+  }
+  const char *get_state_name() const {
+    return get_state_name(state);
+  }
+
+  bool is_shutdown() const { return state == STATE_SHUTDOWN; }
+  bool is_probing() const { return state == STATE_PROBING; }
+  bool is_synchronizing() const { return state == STATE_SYNCHRONIZING; }
+  bool is_electing() const { return state == STATE_ELECTING; }
+  bool is_leader() const { return state == STATE_LEADER; }
+  bool is_peon() const { return state == STATE_PEON; }
+
+  const utime_t &get_leader_since() const;
+
+  void prepare_new_fingerprint(MonitorDBStore::Transaction *t);
+
+  // -- elector --
+private:
+  Paxos *paxos;
+  Elector elector;
+  friend class Elector;
+
+  /// features we require of peers (based on on-disk compatset)
+  uint64_t required_features;
+  
+  int leader;            // current leader (to best of knowledge)
+  set quorum;       // current active set of monitors (if !starting)
+  utime_t leader_since;  // when this monitor became the leader, if it is the leader
+  utime_t exited_quorum; // time detected as not in quorum; 0 if in
+  uint64_t quorum_features;  ///< intersection of quorum member feature bits
+  bufferlist supported_commands_bl; // encoded MonCommands we support
+  bufferlist classic_commands_bl; // encoded MonCommands supported by Dumpling
+  set classic_mons; // set of "classic" monitors; only valid on leader
+
+  set outside_quorum;
+
+  /**
+   * @defgroup scrub
+   * @{
+   */
+  version_t scrub_version;            ///< paxos version we are scrubbing
+  map scrub_result;  ///< results so far
+
+  /**
+   * trigger a cross-mon scrub
+   *
+   * Verify all mons are storing identical content
+   */
+  int scrub();
+  void handle_scrub(MMonScrub *m);
+  void _scrub(ScrubResult *r);
+  void scrub_finish();
+  void scrub_reset();
+
+  /**
+   * @defgroup Synchronization
+   * @{
+   */
+  /**
+   * @} // provider state
+   */
+  struct SyncProvider {
+    entity_inst_t entity;  ///< who
+    uint64_t cookie;       ///< unique cookie for this sync attempt
+    utime_t timeout;       ///< when we give up and expire this attempt
+    version_t last_committed; ///< last paxos version on peer
+    pair last_key; ///< last key sent to (or on) peer
+    bool full;             ///< full scan?
+    MonitorDBStore::Synchronizer synchronizer;   ///< iterator
+
+    SyncProvider() : cookie(0), last_committed(0), full(false) {}
+
+    void reset_timeout(CephContext *cct, int grace) {
+      timeout = ceph_clock_now(cct);
+      timeout += grace;
+    }
+  };
+
+  map sync_providers;  ///< cookie -> SyncProvider for those syncing from us
+  uint64_t sync_provider_count;   ///< counter for issued cookies to keep them unique
+
+  /**
+   * @} // requester state
+   */
+  entity_inst_t sync_provider;   ///< who we are syncing from
+  uint64_t sync_cookie;          ///< 0 if we are starting, non-zero otherwise
+  bool sync_full;                ///< true if we are a full sync, false for recent catch-up
+  version_t sync_start_version;  ///< last_committed at sync start
+  Context *sync_timeout_event;   ///< timeout event
+
+  /**
+   * floor for sync source
+   *
+   * When we sync we forget about our old last_committed value which
+   * can be dangerous.  For example, if we have a cluster of:
+   *
+   *   mon.a: lc 100
+   *   mon.b: lc 80
+   *   mon.c: lc 100 (us)
+   *
+   * If something forces us to sync (say, corruption, or manual
+   * intervention, or bug), we forget last_committed, and might abort.
+   * If mon.a happens to be down when we come back, we will see:
+   *
+   *   mon.b: lc 80
+   *   mon.c: lc 0 (us)
+   *
+   * and sync from mon.b, at which point a+b will both have lc 80 and
+   * come online with a majority holding out of date commits.
+   *
+   * Avoid this by preserving our old last_committed value prior to
+   * sync and never going backwards.
+   */
+  version_t sync_last_committed_floor;
+
+  struct C_SyncTimeout : public Context {
+    Monitor *mon;
+    C_SyncTimeout(Monitor *m) : mon(m) {}
+    void finish(int r) {
+      mon->sync_timeout();
+    }
+  };
+
+  /**
+   * Obtain the synchronization target prefixes in set form.
+   *
+   * We consider a target prefix all those that are relevant when
+   * synchronizing two stores. That is, all those that hold paxos service's
+   * versions, as well as paxos versions, or any control keys such as the
+   * first or last committed version.
+   *
+   * Given the current design, this function should return the name of all and
+   * any available paxos service, plus the paxos name.
+   *
+   * @returns a set of strings referring to the prefixes being synchronized
+   */
+  set get_sync_targets_names();
+
+  /**
+   * Reset the monitor's sync-related data structures for syncing *from* a peer
+   */
+  void sync_reset_requester();
+
+  /**
+   * Reset sync state related to allowing others to sync from us
+   */
+  void sync_reset_provider();
+
+  /**
+   * Caled when a sync attempt times out (requester-side)
+   */
+  void sync_timeout();
+
+  /**
+   * Get the latest monmap for backup purposes during sync
+   */
+  void sync_obtain_latest_monmap(bufferlist &bl);
+
+  /**
+   * Start sync process
+   *
+   * Start pulling committed state from another monitor.
+   *
+   * @param entity where to pull committed state from
+   * @param full whether to do a full sync or just catch up on recent paxos
+   */
+  void sync_start(entity_inst_t &entity, bool full);
+
+public:
+  /**
+   * force a sync on next mon restart
+   */
+  void sync_force(Formatter *f, ostream& ss);
+
+private:
+  /**
+   * store critical state for safekeeping during sync
+   *
+   * We store a few things on the side that we don't want to get clobbered by sync.  This
+   * includes the latest monmap and a lower bound on last_committed.
+   */
+  void sync_stash_critical_state(MonitorDBStore::Transaction *tx);
+
+  /**
+   * reset the sync timeout
+   *
+   * This is used on the client to restart if things aren't progressing
+   */
+  void sync_reset_timeout();
+
+  /**
+   * trim stale sync provider state
+   *
+   * If someone is syncing from us and hasn't talked to us recently, expire their state.
+   */
+  void sync_trim_providers();
+
+  /**
+   * Complete a sync
+   *
+   * Finish up a sync after we've gotten all of the chunks.
+   *
+   * @param last_committed final last_committed value from provider
+   */
+  void sync_finish(version_t last_committed);
+
+  /**
+   * request the next chunk from the provider
+   */
+  void sync_get_next_chunk();
+
+  /**
+   * handle sync message
+   *
+   * @param m Sync message with operation type MMonSync::OP_START_CHUNKS
+   */
+  void handle_sync(MMonSync *m);
+
+  void _sync_reply_no_cookie(MMonSync *m);
+
+  void handle_sync_get_cookie(MMonSync *m);
+  void handle_sync_get_chunk(MMonSync *m);
+  void handle_sync_finish(MMonSync *m);
+
+  void handle_sync_cookie(MMonSync *m);
+  void handle_sync_forward(MMonSync *m);
+  void handle_sync_chunk(MMonSync *m);
+  void handle_sync_no_cookie(MMonSync *m);
+
+  /**
+   * @} // Synchronization
+   */
+
+  list waitfor_quorum;
+  list maybe_wait_for_quorum;
+
+  /**
+   * @defgroup Monitor_h_TimeCheck Monitor Clock Drift Early Warning System
+   * @{
+   *
+   * We use time checks to keep track of any clock drifting going on in the
+   * cluster. This is accomplished by periodically ping each monitor in the
+   * quorum and register its response time on a map, assessing how much its
+   * clock has drifted. We also take this opportunity to assess the latency
+   * on response.
+   *
+   * This mechanism works as follows:
+   *
+   *  - Leader sends out a 'PING' message to each other monitor in the quorum.
+   *    The message is timestamped with the leader's current time. The leader's
+   *    current time is recorded in a map, associated with each peon's
+   *    instance.
+   *  - The peon replies to the leader with a timestamped 'PONG' message.
+   *  - The leader calculates a delta between the peon's timestamp and its
+   *    current time and stashes it.
+   *  - The leader also calculates the time it took to receive the 'PONG'
+   *    since the 'PING' was sent, and stashes an approximate latency estimate.
+   *  - Once all the quorum members have pong'ed, the leader will share the
+   *    clock skew and latency maps with all the monitors in the quorum.
+   */
+  map timecheck_waiting;
+  map timecheck_skews;
+  map timecheck_latencies;
+  // odd value means we are mid-round; even value means the round has
+  // finished.
+  version_t timecheck_round;
+  unsigned int timecheck_acks;
+  utime_t timecheck_round_start;
+  /**
+   * Time Check event.
+   */
+  Context *timecheck_event;
+
+  struct C_TimeCheck : public Context {
+    Monitor *mon;
+    C_TimeCheck(Monitor *m) : mon(m) { }
+    void finish(int r) {
+      mon->timecheck_start_round();
+    }
+  };
+
+  void timecheck_start();
+  void timecheck_finish();
+  void timecheck_start_round();
+  void timecheck_finish_round(bool success = true);
+  void timecheck_cancel_round();
+  void timecheck_cleanup();
+  void timecheck_report();
+  void timecheck();
+  health_status_t timecheck_status(ostringstream &ss,
+                                   const double skew_bound,
+                                   const double latency);
+  void handle_timecheck_leader(MTimeCheck *m);
+  void handle_timecheck_peon(MTimeCheck *m);
+  void handle_timecheck(MTimeCheck *m);
+  /**
+   * @}
+   */
+  /**
+   * @defgroup Monitor_h_stats Keep track of monitor statistics
+   * @{
+   */
+  struct MonStatsEntry {
+    // data dir
+    uint64_t kb_total;
+    uint64_t kb_used;
+    uint64_t kb_avail;
+    unsigned int latest_avail_ratio;
+    utime_t last_update;
+  };
+
+  struct MonStats {
+    MonStatsEntry ours;
+    map others;
+  };
+
+  MonStats stats;
+
+  void stats_update();
+  /**
+   * @}
+   */
+  /**
+   * Handle ping messages from others.
+   */
+  void handle_ping(MPing *m);
+
+  Context *probe_timeout_event;  // for probing
+
+  struct C_ProbeTimeout : public Context {
+    Monitor *mon;
+    C_ProbeTimeout(Monitor *m) : mon(m) {}
+    void finish(int r) {
+      mon->probe_timeout(r);
+    }
+  };
+
+  void reset_probe_timeout();
+  void cancel_probe_timeout();
+  void probe_timeout(int r);
+
+public:
+  epoch_t get_epoch();
+  int get_leader() { return leader; }
+  const set& get_quorum() { return quorum; }
+  list get_quorum_names() {
+    list q;
+    for (set::iterator p = quorum.begin(); p != quorum.end(); ++p)
+      q.push_back(monmap->get_name(*p));
+    return q;
+  }
+  uint64_t get_quorum_features() const {
+    return quorum_features;
+  }
+  uint64_t get_required_features() const {
+    return quorum_features;
+  }
+  void apply_quorum_to_compatset_features();
+  void apply_compatset_features_to_quorum_requirements();
+
+private:
+  void _reset();   ///< called from bootstrap, start_, or join_election
+public:
+  void bootstrap();
+  void join_election();
+  void start_election();
+  void win_standalone_election();
+  // end election (called by Elector)
+  void win_election(epoch_t epoch, set& q,
+		    uint64_t features,
+		    const MonCommand *cmdset, int cmdsize,
+		    const set *classic_monitors);
+  void lose_election(epoch_t epoch, set& q, int l,
+		     uint64_t features); // end election (called by Elector)
+  void finish_election();
+
+  const bufferlist& get_supported_commands_bl() {
+    return supported_commands_bl;
+  }
+  const bufferlist& get_classic_commands_bl() {
+    return classic_commands_bl;
+  }
+  const set& get_classic_mons() {
+    return classic_mons;
+  }
+
+  void update_logger();
+
+  /**
+   * Vector holding the Services serviced by this Monitor.
+   */
+  vector paxos_service;
+
+  PaxosService *get_paxos_service_by_name(const string& name);
+
+  class PGMonitor *pgmon() {
+    return (class PGMonitor *)paxos_service[PAXOS_PGMAP];
+  }
+
+  class MDSMonitor *mdsmon() {
+    return (class MDSMonitor *)paxos_service[PAXOS_MDSMAP];
+  }
+
+  class MonmapMonitor *monmon() {
+    return (class MonmapMonitor *)paxos_service[PAXOS_MONMAP];
+  }
+
+  class OSDMonitor *osdmon() {
+    return (class OSDMonitor *)paxos_service[PAXOS_OSDMAP];
+  }
+
+  class AuthMonitor *authmon() {
+    return (class AuthMonitor *)paxos_service[PAXOS_AUTH];
+  }
+
+  class LogMonitor *logmon() {
+    return (class LogMonitor*) paxos_service[PAXOS_LOG];
+  }
+
+  friend class Paxos;
+  friend class OSDMonitor;
+  friend class MDSMonitor;
+  friend class MonmapMonitor;
+  friend class PGMonitor;
+  friend class LogMonitor;
+
+  QuorumService *health_monitor;
+  QuorumService *config_key_service;
+
+  // -- sessions --
+  MonSessionMap session_map;
+  AdminSocketHook *admin_hook;
+
+  void check_subs();
+  void check_sub(Subscription *sub);
+
+  void send_latest_monmap(Connection *con);
+
+  // messages
+  void handle_get_version(MMonGetVersion *m);
+  void handle_subscribe(MMonSubscribe *m);
+  void handle_mon_get_map(MMonGetMap *m);
+  static void _generate_command_map(map& cmdmap,
+                                    map ¶m_str_map);
+  static const MonCommand *_get_moncommand(const string &cmd_prefix,
+                                           MonCommand *cmds, int cmds_size);
+  bool _allowed_command(MonSession *s, string &module, string &prefix,
+                        const map& cmdmap,
+                        const map& param_str_map,
+                        const MonCommand *this_cmd);
+  void get_mon_status(Formatter *f, ostream& ss);
+  void _quorum_status(Formatter *f, ostream& ss);
+  void _osdmonitor_prepare_command(cmdmap_t& cmdmap, ostream& ss);
+  void _add_bootstrap_peer_hint(string cmd, cmdmap_t& cmdmap, ostream& ss);
+  void handle_command(class MMonCommand *m);
+  void handle_route(MRoute *m);
+
+  /**
+   * Generate health report
+   *
+   * @param status one-line status summary
+   * @param detailbl optional bufferlist* to fill with a detailed report
+   */
+  void get_health(string& status, bufferlist *detailbl, Formatter *f);
+  void get_cluster_status(stringstream &ss, Formatter *f);
+
+  void reply_command(MMonCommand *m, int rc, const string &rs, version_t version);
+  void reply_command(MMonCommand *m, int rc, const string &rs, bufferlist& rdata, version_t version);
+
+
+  void handle_probe(MMonProbe *m);
+  /**
+   * Handle a Probe Operation, replying with our name, quorum and known versions.
+   *
+   * We use the MMonProbe message class for anything and everything related with
+   * Monitor probing. One of the operations relates directly with the probing
+   * itself, in which we receive a probe request and to which we reply with
+   * our name, our quorum and the known versions for each Paxos service. Thus the
+   * redundant function name. This reply will obviously be sent to the one
+   * probing/requesting these infos.
+   *
+   * @todo Add @pre and @post
+   *
+   * @param m A Probe message, with an operation of type Probe.
+   */
+  void handle_probe_probe(MMonProbe *m);
+  void handle_probe_reply(MMonProbe *m);
+
+  // request routing
+  struct RoutedRequest {
+    uint64_t tid;
+    bufferlist request_bl;
+    MonSession *session;
+    ConnectionRef con;
+    uint64_t con_features;
+    entity_inst_t client_inst;
+
+    RoutedRequest() : tid(0), session(NULL), con_features(0) {}
+    ~RoutedRequest() {
+      if (session)
+	session->put();
+    }
+  };
+  uint64_t routed_request_tid;
+  map routed_requests;
+  
+  void forward_request_leader(PaxosServiceMessage *req);
+  void handle_forward(MForward *m);
+  void try_send_message(Message *m, const entity_inst_t& to);
+  void send_reply(PaxosServiceMessage *req, Message *reply);
+  void no_reply(PaxosServiceMessage *req);
+  void resend_routed_requests();
+  void remove_session(MonSession *s);
+  void remove_all_sessions();
+  void waitlist_or_zap_client(Message *m);
+
+  void send_command(const entity_inst_t& inst,
+		    const vector& com);
+
+public:
+  struct C_Command : public Context {
+    Monitor *mon;
+    MMonCommand *m;
+    int rc;
+    string rs;
+    bufferlist rdata;
+    version_t version;
+    C_Command(Monitor *_mm, MMonCommand *_m, int r, string s, version_t v) :
+      mon(_mm), m(_m), rc(r), rs(s), version(v){}
+    C_Command(Monitor *_mm, MMonCommand *_m, int r, string s, bufferlist rd, version_t v) :
+      mon(_mm), m(_m), rc(r), rs(s), rdata(rd), version(v){}
+    void finish(int r) {
+      if (r >= 0)
+	mon->reply_command(m, rc, rs, rdata, version);
+      else if (r == -ECANCELED)
+	m->put();
+      else if (r == -EAGAIN)
+	mon->_ms_dispatch(m);
+      else
+	assert(0 == "bad C_Command return value");
+    }
+  };
+
+ private:
+  class C_RetryMessage : public Context {
+    Monitor *mon;
+    Message *msg;
+  public:
+    C_RetryMessage(Monitor *m, Message *ms) : mon(m), msg(ms) {}
+    void finish(int r) {
+      if (r == -EAGAIN || r >= 0)
+	mon->_ms_dispatch(msg);
+      else if (r == -ECANCELED)
+	msg->put();
+      else
+	assert(0 == "bad C_RetryMessage return value");
+    }
+  };
+
+  //ms_dispatch handles a lot of logic and we want to reuse it
+  //on forwarded messages, so we create a non-locking version for this class
+  bool _ms_dispatch(Message *m);
+  bool ms_dispatch(Message *m) {
+    lock.Lock();
+    bool ret = _ms_dispatch(m);
+    lock.Unlock();
+    return ret;
+  }
+  // dissociate message handling from session and connection logic
+  bool dispatch(MonSession *s, Message *m, const bool src_is_mon);
+  //mon_caps is used for un-connected messages from monitors
+  MonCap * mon_caps;
+  bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new);
+  bool ms_verify_authorizer(Connection *con, int peer_type,
+			    int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply,
+			    bool& isvalid, CryptoKey& session_key);
+  bool ms_handle_reset(Connection *con);
+  void ms_handle_remote_reset(Connection *con) {}
+
+  int write_default_keyring(bufferlist& bl);
+  void extract_save_mon_key(KeyRing& keyring);
+
+  // features
+  static CompatSet get_supported_features();
+  static CompatSet get_legacy_features();
+  /// read the ondisk features into the CompatSet pointed to by read_features
+  static void read_features_off_disk(MonitorDBStore *store, CompatSet *read_features);
+  void read_features();
+  void write_features(MonitorDBStore::Transaction &t);
+
+ public:
+  Monitor(CephContext *cct_, string nm, MonitorDBStore *s,
+	  Messenger *m, MonMap *map);
+  ~Monitor();
+
+  static int check_features(MonitorDBStore *store);
+
+  int preinit();
+  int init();
+  void init_paxos();
+  void refresh_from_paxos(bool *need_bootstrap);
+  void shutdown();
+  void tick();
+
+  void handle_signal(int sig);
+
+  int mkfs(bufferlist& osdmapbl);
+
+  /**
+   * check cluster_fsid file
+   *
+   * @return EEXIST if file exists and doesn't match, 0 on match, or negative error code
+   */
+  int check_fsid();
+
+  /**
+   * write cluster_fsid file
+   *
+   * @return 0 on success, or negative error code
+   */
+  int write_fsid();
+  int write_fsid(MonitorDBStore::Transaction &t);
+
+  void do_admin_command(std::string command, cmdmap_t& cmdmap,
+			std::string format, ostream& ss);
+
+private:
+  // don't allow copying
+  Monitor(const Monitor& rhs);
+  Monitor& operator=(const Monitor &rhs);
+
+public:
+  class StoreConverter {
+    const string path;
+    MonitorDBStore *db;
+    boost::scoped_ptr store;
+
+    set gvs;
+    map > > gv_map;
+
+    version_t highest_last_pn;
+    version_t highest_accepted_pn;
+
+   public:
+    StoreConverter(string path, MonitorDBStore *d)
+      : path(path), db(d), store(NULL),
+	highest_last_pn(0), highest_accepted_pn(0)
+    { }
+
+    /**
+     * Check if store needs to be converted from old format to a
+     * k/v store.
+     *
+     * @returns 0 if store doesn't need conversion; 1 if it does; <0 if error
+     */
+    int needs_conversion();
+    int convert();
+
+    bool is_converting() {
+      return db->exists("mon_convert", "on_going");
+    }
+
+   private:
+
+    bool _check_gv_store();
+
+    void _init() {
+      assert(!store);
+      MonitorStore *store_ptr = new MonitorStore(path);
+      store.reset(store_ptr);
+    }
+
+    void _deinit() {
+      store.reset(NULL);
+    }
+
+    set _get_machines_names() {
+      set names;
+      names.insert("auth");
+      names.insert("logm");
+      names.insert("mdsmap");
+      names.insert("monmap");
+      names.insert("osdmap");
+      names.insert("pgmap");
+
+      return names;
+    }
+
+    void _mark_convert_start() {
+      MonitorDBStore::Transaction tx;
+      tx.put("mon_convert", "on_going", 1);
+      db->apply_transaction(tx);
+    }
+
+    void _convert_finish_features(MonitorDBStore::Transaction &t);
+    void _mark_convert_finish() {
+      MonitorDBStore::Transaction tx;
+      tx.erase("mon_convert", "on_going");
+      _convert_finish_features(tx);
+      db->apply_transaction(tx);
+    }
+
+    void _convert_monitor();
+    void _convert_machines(string machine);
+    void _convert_osdmap_full();
+    void _convert_machines();
+    void _convert_paxos();
+  };
+
+  static void format_command_descriptions(const MonCommand *commands,
+					  unsigned commands_size,
+					  Formatter *f,
+					  bufferlist *rdata);
+  void get_locally_supported_monitor_commands(const MonCommand **cmds, int *count);
+  void get_classic_monitor_commands(const MonCommand **cmds, int *count);
+  void get_leader_supported_commands(const MonCommand **cmds, int *count);
+  /// the Monitor owns this pointer once you pass it in
+  void set_leader_supported_commands(const MonCommand *cmds, int size);
+  static bool is_keyring_required();
+};
+
+#define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)")
+#define CEPH_MON_FEATURE_INCOMPAT_GV CompatSet::Feature (2, "global version sequencing (v0.52)")
+#define CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS CompatSet::Feature (3, "single paxos with k/v store (v0.\?)")
+#define CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES CompatSet::Feature(4, "support erasure code pools")
+#define CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC CompatSet::Feature(5, "new-style osdmap encoding")
+// make sure you add your feature to Monitor::get_supported_features
+
+long parse_pos_long(const char *s, ostream *pss = NULL);
+
+struct MonCommand {
+  string cmdstring;
+  string helpstring;
+  string module;
+  string req_perms;
+  string availability;
+
+  void encode(bufferlist &bl) const {
+    /*
+     * very naughty: deliberately unversioned because individual commands
+     * shouldn't be encoded standalone, only as a full set (which we do
+     * version, see encode_array() below).
+     */
+    ::encode(cmdstring, bl);
+    ::encode(helpstring, bl);
+    ::encode(module, bl);
+    ::encode(req_perms, bl);
+    ::encode(availability, bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    ::decode(cmdstring, bl);
+    ::decode(helpstring, bl);
+    ::decode(module, bl);
+    ::decode(req_perms, bl);
+    ::decode(availability, bl);
+  }
+  bool operator==(const MonCommand& o) const {
+    return cmdstring == o.cmdstring && helpstring == o.helpstring &&
+	module == o.module && req_perms == o.req_perms &&
+	availability == o.availability;
+  }
+  bool operator!=(const MonCommand& o) const {
+    return !(*this == o);
+  }
+
+  static void encode_array(const MonCommand *cmds, int size, bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    uint16_t s = size;
+    ::encode(s, bl);
+    ::encode_array_nohead(cmds, size, bl);
+    ENCODE_FINISH(bl);
+  }
+  static void decode_array(MonCommand **cmds, int *size,
+                           bufferlist::iterator &bl) {
+    DECODE_START(1, bl);
+    uint16_t s = 0;
+    ::decode(s, bl);
+    *size = s;
+    *cmds = new MonCommand[*size];
+    ::decode_array_nohead(*cmds, *size, bl);
+    DECODE_FINISH(bl);
+  }
+};
+WRITE_CLASS_ENCODER(MonCommand);
+
+#endif
diff --git a/ceph/src/mon/MonitorDBStore.h b/ceph/src/mon/MonitorDBStore.h
new file mode 100644
index 00000000..88c4f936
--- /dev/null
+++ b/ceph/src/mon/MonitorDBStore.h
@@ -0,0 +1,577 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef CEPH_MONITOR_DB_STORE_H
+#define CEPH_MONITOR_DB_STORE_H
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "os/KeyValueDB.h"
+#include "os/LevelDBStore.h"
+
+#include "include/assert.h"
+#include "common/Formatter.h"
+#include "common/errno.h"
+
+class MonitorDBStore
+{
+  boost::scoped_ptr db;
+  bool do_dump;
+  int dump_fd;
+
+ public:
+
+  struct Op {
+    uint8_t type;
+    string prefix;
+    string key, endkey;
+    bufferlist bl;
+
+    Op()
+      : type(0) { }
+    Op(int t, string p, string k)
+      : type(t), prefix(p), key(k) { }
+    Op(int t, const string& p, string k, bufferlist& b)
+      : type(t), prefix(p), key(k), bl(b) { }
+    Op(int t, const string& p, string start, string end)
+      : type(t), prefix(p), key(start), endkey(end) { }
+
+    void encode(bufferlist& encode_bl) const {
+      ENCODE_START(2, 1, encode_bl);
+      ::encode(type, encode_bl);
+      ::encode(prefix, encode_bl);
+      ::encode(key, encode_bl);
+      ::encode(bl, encode_bl);
+      ::encode(endkey, encode_bl);
+      ENCODE_FINISH(encode_bl);
+    }
+
+    void decode(bufferlist::iterator& decode_bl) {
+      DECODE_START(2, decode_bl);
+      ::decode(type, decode_bl);
+      ::decode(prefix, decode_bl);
+      ::decode(key, decode_bl);
+      ::decode(bl, decode_bl);
+      if (struct_v >= 2)
+	::decode(endkey, decode_bl);
+      DECODE_FINISH(decode_bl);
+    }
+
+    void dump(Formatter *f) const {
+      f->dump_int("type", type);
+      f->dump_string("prefix", prefix);
+      f->dump_string("key", key);
+      if (endkey.length())
+	f->dump_string("endkey", endkey);
+    }
+
+    static void generate_test_instances(list& ls) {
+      ls.push_back(new Op);
+      // we get coverage here from the Transaction instances
+    }
+  };
+
+  struct Transaction {
+    list ops;
+
+    enum {
+      OP_PUT	= 1,
+      OP_ERASE	= 2,
+      OP_COMPACT = 3,
+    };
+
+    void put(string prefix, string key, bufferlist& bl) {
+      ops.push_back(Op(OP_PUT, prefix, key, bl));
+    }
+
+    void put(string prefix, version_t ver, bufferlist& bl) {
+      ostringstream os;
+      os << ver;
+      put(prefix, os.str(), bl);
+    }
+
+    void put(string prefix, string key, version_t ver) {
+      bufferlist bl;
+      ::encode(ver, bl);
+      put(prefix, key, bl);
+    }
+
+    void erase(string prefix, string key) {
+      ops.push_back(Op(OP_ERASE, prefix, key));
+    }
+
+    void erase(string prefix, version_t ver) {
+      ostringstream os;
+      os << ver;
+      erase(prefix, os.str());
+    }
+
+    void compact_prefix(string prefix) {
+      ops.push_back(Op(OP_COMPACT, prefix, string()));
+    }
+
+    void compact_range(string prefix, string start, string end) {
+      ops.push_back(Op(OP_COMPACT, prefix, start, end));
+    }
+
+    void encode(bufferlist& bl) const {
+      ENCODE_START(1, 1, bl);
+      ::encode(ops, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(bufferlist::iterator& bl) {
+      DECODE_START(1, bl);
+      ::decode(ops, bl);
+      DECODE_FINISH(bl);
+    }
+
+    static void generate_test_instances(list& ls) {
+      ls.push_back(new Transaction);
+      ls.push_back(new Transaction);
+      bufferlist bl;
+      bl.append("value");
+      ls.back()->put("prefix", "key", bl);
+      ls.back()->erase("prefix2", "key2");
+      ls.back()->compact_prefix("prefix3");
+      ls.back()->compact_range("prefix4", "from", "to");
+    }
+
+    void append(Transaction& other) {
+      ops.splice(ops.end(), other.ops);
+    }
+
+    void append_from_encoded(bufferlist& bl) {
+      Transaction other;
+      bufferlist::iterator it = bl.begin();
+      other.decode(it);
+      append(other);
+    }
+
+    bool empty() {
+      return (size() == 0);
+    }
+
+    bool size() {
+      return ops.size();
+    }
+
+    void dump(ceph::Formatter *f, bool dump_val=false) const {
+      f->open_object_section("transaction");
+      f->open_array_section("ops");
+      list::const_iterator it;
+      int op_num = 0;
+      for (it = ops.begin(); it != ops.end(); ++it) {
+	const Op& op = *it;
+	f->open_object_section("op");
+	f->dump_int("op_num", op_num++);
+	switch (op.type) {
+	case OP_PUT:
+	  {
+	    f->dump_string("type", "PUT");
+	    f->dump_string("prefix", op.prefix);
+	    f->dump_string("key", op.key);
+	    f->dump_unsigned("length", op.bl.length());
+	    if (dump_val) {
+	      ostringstream os;
+	      op.bl.hexdump(os);
+	      f->dump_string("bl", os.str());
+	    }
+	  }
+	  break;
+	case OP_ERASE:
+	  {
+	    f->dump_string("type", "ERASE");
+	    f->dump_string("prefix", op.prefix);
+	    f->dump_string("key", op.key);
+	  }
+	  break;
+	case OP_COMPACT:
+	  {
+	    f->dump_string("type", "COMPACT");
+	    f->dump_string("prefix", op.prefix);
+	    f->dump_string("start", op.key);
+	    f->dump_string("end", op.endkey);
+	  }
+	  break;
+	default:
+	  {
+	    f->dump_string("type", "unknown");
+	    f->dump_unsigned("op_code", op.type);
+	    break;
+	  }
+	}
+	f->close_section();
+      }
+      f->close_section();
+      f->close_section();
+    }
+  };
+
+  int apply_transaction(const MonitorDBStore::Transaction& t) {
+    KeyValueDB::Transaction dbt = db->get_transaction();
+
+    if (do_dump) {
+      bufferlist bl;
+      t.encode(bl);
+      bl.write_fd(dump_fd);
+    }
+
+    list > > compact;
+    for (list::const_iterator it = t.ops.begin(); it != t.ops.end(); ++it) {
+      const Op& op = *it;
+      switch (op.type) {
+      case Transaction::OP_PUT:
+	dbt->set(op.prefix, op.key, op.bl);
+	break;
+      case Transaction::OP_ERASE:
+	dbt->rmkey(op.prefix, op.key);
+	break;
+      case Transaction::OP_COMPACT:
+	compact.push_back(make_pair(op.prefix, make_pair(op.key, op.endkey)));
+	break;
+      default:
+	derr << __func__ << " unknown op type " << op.type << dendl;
+	ceph_assert(0);
+	break;
+      }
+    }
+    int r = db->submit_transaction_sync(dbt);
+    if (r >= 0) {
+      while (!compact.empty()) {
+	if (compact.front().second.first == string() &&
+	    compact.front().second.second == string())
+	  db->compact_prefix_async(compact.front().first);
+	else
+	  db->compact_range_async(compact.front().first, compact.front().second.first, compact.front().second.second);
+	compact.pop_front();
+      }
+    }
+    return r;
+  }
+
+  class StoreIteratorImpl {
+  protected:
+    bool done;
+    pair last_key;
+    bufferlist crc_bl;
+
+    StoreIteratorImpl() : done(false) { }
+    virtual ~StoreIteratorImpl() { }
+
+    bool add_chunk_entry(Transaction &tx,
+			 string &prefix,
+			 string &key,
+			 bufferlist &value,
+			 uint64_t max) {
+      Transaction tmp;
+      bufferlist tmp_bl;
+      tmp.put(prefix, key, value);
+      tmp.encode(tmp_bl);
+
+      bufferlist tx_bl;
+      tx.encode(tx_bl);
+
+      size_t len = tx_bl.length() + tmp_bl.length();
+
+      if (!tx.empty() && (len > max)) {
+	return false;
+      }
+
+      tx.append(tmp);
+      last_key.first = prefix;
+      last_key.second = key;
+
+      if (g_conf->mon_sync_debug) {
+	::encode(prefix, crc_bl);
+	::encode(key, crc_bl);
+	::encode(value, crc_bl);
+      }
+
+      return true;
+    }
+
+    virtual bool _is_valid() = 0;
+
+  public:
+    __u32 crc() {
+      if (g_conf->mon_sync_debug)
+	return crc_bl.crc32c(0);
+      return 0;
+    }
+    pair get_last_key() {
+      return last_key;
+    };
+    virtual bool has_next_chunk() {
+      return !done && _is_valid();
+    }
+    virtual void get_chunk_tx(Transaction &tx, uint64_t max) = 0;
+    virtual pair get_next_key() = 0;
+  };
+  typedef ceph::shared_ptr Synchronizer;
+
+  class WholeStoreIteratorImpl : public StoreIteratorImpl {
+    KeyValueDB::WholeSpaceIterator iter;
+    set sync_prefixes;
+
+  public:
+    WholeStoreIteratorImpl(KeyValueDB::WholeSpaceIterator iter,
+			   set &prefixes)
+      : StoreIteratorImpl(),
+	iter(iter),
+	sync_prefixes(prefixes)
+    { }
+
+    virtual ~WholeStoreIteratorImpl() { }
+
+    /**
+     * Obtain a chunk of the store
+     *
+     * @param bl	    Encoded transaction that will recreate the chunk
+     * @param first_key	    Pair containing the first key to obtain, and that
+     *			    will contain the first key in the chunk (that may
+     *			    differ from the one passed on to the function)
+     * @param last_key[out] Last key in the chunk
+     */
+    virtual void get_chunk_tx(Transaction &tx, uint64_t max) {
+      assert(done == false);
+      assert(iter->valid() == true);
+
+      while (iter->valid()) {
+	string prefix(iter->raw_key().first);
+	string key(iter->raw_key().second);
+	if (sync_prefixes.count(prefix)) {
+	  bufferlist value = iter->value();
+	  if (!add_chunk_entry(tx, prefix, key, value, max))
+	    return;
+	}
+	iter->next();
+      }
+      assert(iter->valid() == false);
+      done = true;
+    }
+
+    virtual pair get_next_key() {
+      assert(iter->valid());
+      pair r = iter->raw_key();
+      do {
+	iter->next();
+      } while (iter->valid() && sync_prefixes.count(iter->raw_key().first) == 0);
+      return r;
+    }
+
+    virtual bool _is_valid() {
+      return iter->valid();
+    }
+  };
+
+  Synchronizer get_synchronizer(pair &key,
+				set &prefixes) {
+    KeyValueDB::WholeSpaceIterator iter;
+    iter = db->get_snapshot_iterator();
+
+    if (!key.first.empty() && !key.second.empty())
+      iter->upper_bound(key.first, key.second);
+    else
+      iter->seek_to_first();
+
+    return ceph::shared_ptr(
+	new WholeStoreIteratorImpl(iter, prefixes)
+    );
+  }
+
+  KeyValueDB::Iterator get_iterator(const string &prefix) {
+    assert(!prefix.empty());
+    KeyValueDB::Iterator iter = db->get_snapshot_iterator(prefix);
+    iter->seek_to_first();
+    return iter;
+  }
+
+  KeyValueDB::WholeSpaceIterator get_iterator() {
+    KeyValueDB::WholeSpaceIterator iter;
+    iter = db->get_snapshot_iterator();
+    iter->seek_to_first();
+    return iter;
+  }
+
+  int get(const string& prefix, const string& key, bufferlist& bl) {
+    set k;
+    k.insert(key);
+    map out;
+
+    db->get(prefix, k, &out);
+    if (out.empty())
+      return -ENOENT;
+    bl.append(out[key]);
+
+    return 0;
+  }
+
+  int get(const string& prefix, const version_t ver, bufferlist& bl) {
+    ostringstream os;
+    os << ver;
+    return get(prefix, os.str(), bl);
+  }
+
+  version_t get(const string& prefix, const string& key) {
+    bufferlist bl;
+    int err = get(prefix, key, bl);
+    if (err < 0) {
+      if (err == -ENOENT) // if key doesn't exist, assume its value is 0
+        return 0;
+      // we're not expecting any other negative return value, and we can't
+      // just return a negative value if we're returning a version_t
+      generic_dout(0) << "MonitorDBStore::get() error obtaining"
+                      << " (" << prefix << ":" << key << "): "
+                      << cpp_strerror(err) << dendl;
+      assert(0 == "error obtaining key");
+    }
+
+    assert(bl.length());
+    version_t ver;
+    bufferlist::iterator p = bl.begin();
+    ::decode(ver, p);
+    return ver;
+  }
+
+  bool exists(const string& prefix, const string& key) {
+    KeyValueDB::Iterator it = db->get_iterator(prefix);
+    int err = it->lower_bound(key);
+    if (err < 0)
+      return false;
+
+    return (it->valid() && it->key() == key);
+  }
+
+  bool exists(const string& prefix, version_t ver) {
+    ostringstream os;
+    os << ver;
+    return exists(prefix, os.str());
+  }
+
+  string combine_strings(const string& prefix, const string& value) {
+    string out = prefix;
+    out.push_back('_');
+    out.append(value);
+    return out;
+  }
+
+  string combine_strings(const string& prefix, const version_t ver) {
+    ostringstream os;
+    os << ver;
+    return combine_strings(prefix, os.str());
+  }
+
+  void clear(set& prefixes) {
+    set::iterator iter;
+    KeyValueDB::Transaction dbt = db->get_transaction();
+
+    for (iter = prefixes.begin(); iter != prefixes.end(); ++iter) {
+      dbt->rmkeys_by_prefix((*iter));
+    }
+    db->submit_transaction_sync(dbt);
+  }
+
+  void init_options() {
+    db->init();
+    if (g_conf->mon_leveldb_write_buffer_size)
+      db->options.write_buffer_size = g_conf->mon_leveldb_write_buffer_size;
+    if (g_conf->mon_leveldb_cache_size)
+      db->options.cache_size = g_conf->mon_leveldb_cache_size;
+    if (g_conf->mon_leveldb_block_size)
+      db->options.block_size = g_conf->mon_leveldb_block_size;
+    if (g_conf->mon_leveldb_bloom_size)
+      db->options.bloom_size = g_conf->mon_leveldb_bloom_size;
+    if (g_conf->mon_leveldb_compression)
+      db->options.compression_enabled = g_conf->mon_leveldb_compression;
+    if (g_conf->mon_leveldb_max_open_files)
+      db->options.max_open_files = g_conf->mon_leveldb_max_open_files;
+    if (g_conf->mon_leveldb_paranoid)
+      db->options.paranoid_checks = g_conf->mon_leveldb_paranoid;
+    if (g_conf->mon_leveldb_log.length())
+      db->options.log_file = g_conf->mon_leveldb_log;
+  }
+
+  int open(ostream &out) {
+    init_options();
+    return db->open(out);
+  }
+
+  int create_and_open(ostream &out) {
+    init_options();
+    return db->create_and_open(out);
+  }
+
+  void compact() {
+    db->compact();
+  }
+
+  void compact_prefix(const string& prefix) {
+    db->compact_prefix(prefix);
+  }
+
+  uint64_t get_estimated_size(map &extras) {
+    return db->get_estimated_size(extras);
+  }
+
+  MonitorDBStore(const string& path) :
+    db(0), do_dump(false), dump_fd(-1) {
+    string::const_reverse_iterator rit;
+    int pos = 0;
+    for (rit = path.rbegin(); rit != path.rend(); ++rit, ++pos) {
+      if (*rit != '/')
+	break;
+    }
+    ostringstream os;
+    os << path.substr(0, path.size() - pos) << "/store.db";
+    string full_path = os.str();
+
+    LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, full_path);
+    if (!db_ptr) {
+      derr << __func__ << " error initializing level db back storage in "
+		<< full_path << dendl;
+      assert(0 != "MonitorDBStore: error initializing level db back storage");
+    }
+    db.reset(db_ptr);
+
+    if (g_conf->mon_debug_dump_transactions) {
+      do_dump = true;
+      dump_fd = ::open(
+	g_conf->mon_debug_dump_location.c_str(),
+	O_CREAT|O_APPEND|O_WRONLY, 0644);
+      if (!dump_fd) {
+	dump_fd = -errno;
+	derr << "Could not open log file, got "
+	     << cpp_strerror(dump_fd) << dendl;
+      }
+    }
+  }
+  MonitorDBStore(LevelDBStore *db_ptr) :
+    db(0), do_dump(false), dump_fd(-1) {
+    db.reset(db_ptr);
+  }
+  ~MonitorDBStore() {
+    if (do_dump)
+      ::close(dump_fd);
+  }
+
+};
+
+WRITE_CLASS_ENCODER(MonitorDBStore::Op);
+WRITE_CLASS_ENCODER(MonitorDBStore::Transaction);
+
+#endif /* CEPH_MONITOR_DB_STORE_H */
diff --git a/ceph/src/mon/MonitorStore.cc b/ceph/src/mon/MonitorStore.cc
new file mode 100644
index 00000000..db21a945
--- /dev/null
+++ b/ceph/src/mon/MonitorStore.cc
@@ -0,0 +1,487 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MonitorStore.h"
+#include "common/Clock.h"
+#include "common/debug.h"
+#include "common/entity_name.h"
+#include "common/errno.h"
+#include "common/run_cmd.h"
+#include "common/safe_io.h"
+#include "common/config.h"
+#include "common/sync_filesystem.h"
+
+#if defined(__FreeBSD__)
+#include 
+#endif
+
+#include "include/compat.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, dir)
+static ostream& _prefix(std::ostream *_dout, const string& dir) {
+  return *_dout << "store(" << dir << ") ";
+}
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+int MonitorStore::mount()
+{
+  char t[1024];
+
+  dout(1) << "mount" << dendl;
+  // verify dir exists
+  DIR *d = ::opendir(dir.c_str());
+  if (!d) {
+    dout(1) << "basedir " << dir << " dne" << dendl;
+    return -ENOENT;
+  }
+  ::closedir(d);
+
+  // open lockfile
+  snprintf(t, sizeof(t), "%s/lock", dir.c_str());
+  lock_fd = ::open(t, O_CREAT|O_RDWR, 0600);
+  if (lock_fd < 0)
+    return -errno;
+  struct flock l;
+  memset(&l, 0, sizeof(l));
+  l.l_type = F_WRLCK;
+  l.l_whence = SEEK_SET;
+  l.l_start = 0;
+  l.l_len = 0;
+  int r = ::fcntl(lock_fd, F_SETLK, &l);
+  if (r < 0) {
+    dout(0) << "failed to lock " << t << ", is another ceph-mon still running?" << dendl;
+    return -errno;
+  }
+
+  if ((!g_conf->chdir.empty()) && (dir[0] != '/')) {
+    // combine it with the cwd, in case fuse screws things up (i.e. fakefuse)
+    string old = dir;
+    char cwd[PATH_MAX];
+    char *p = getcwd(cwd, sizeof(cwd));
+    dir = p;
+    dir += "/";
+    dir += old;
+  }
+  return 0;
+}
+
+int MonitorStore::umount()
+{
+  int close_err = TEMP_FAILURE_RETRY(::close(lock_fd));
+  assert (0 == close_err);
+  return 0;
+}
+
+int MonitorStore::mkfs()
+{
+  int err;
+
+  err = ::mkdir(dir.c_str(), 0700);
+  if (err < 0 && errno != EEXIST) {
+    err = -errno;
+    derr << "MonitorStore::mkfs: unable to create " << dir << ": " << cpp_strerror(err) << dendl;
+    return err;
+  }
+
+  int fd = ::open(dir.c_str(), O_RDONLY);
+  if (fd < 0) {
+    err = -errno;
+    derr << "MonitorStore::mkfs: unable to open " << dir << ": " << cpp_strerror(err) << dendl;
+    return err;
+  }
+  int close_err = TEMP_FAILURE_RETRY(::close(fd));
+  assert (0 == close_err);
+
+  dout(0) << "created monfs at " << dir << " for "
+	  << g_conf->name.get_id() << dendl;
+  return 0;
+}
+
+version_t MonitorStore::get_int(const char *a, const char *b)
+{
+  char fn[1024];
+  if (b)
+    snprintf(fn, sizeof(fn), "%s/%s/%s", dir.c_str(), a, b);
+  else
+    snprintf(fn, sizeof(fn), "%s/%s", dir.c_str(), a);
+  
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    int err = errno;
+    if (err == ENOENT) {
+      // Non-existent files are treated as containing 0.
+      return 0;
+    }
+    derr << "MonitorStore::get_int: failed to open '" << fn << "': "
+	 << cpp_strerror(err) << dendl;
+    assert(0 == "failed to open");
+    return 0;
+  }
+  
+  char buf[20];
+  memset(buf, 0, sizeof(buf));
+  int r = safe_read(fd, buf, sizeof(buf) - 1);
+  if (r < 0) {
+    derr << "MonitorStore::get_int: failed to read '" << fn << "': "
+	 << cpp_strerror(r) << dendl;
+    int close_err = TEMP_FAILURE_RETRY(::close(fd));
+    assert(0 == close_err);
+    assert(0); // the file exists; so this is a different failure
+    return 0;
+  }
+  int close_err = TEMP_FAILURE_RETRY(::close(fd));
+  assert (0 == close_err);
+  
+  version_t val = atoi(buf);
+  
+  if (b) {
+    dout(15) << "get_int " << a << "/" << b << " = " << val << dendl;
+  } else {
+    dout(15) << "get_int " << a << " = " << val << dendl;
+  }
+  return val;
+}
+
+
+void MonitorStore::put_int(version_t val, const char *a, const char *b)
+{
+  char fn[1024];
+  snprintf(fn, sizeof(fn), "%s/%s", dir.c_str(), a);
+  if (b) {
+    int r = ::mkdir(fn, 0755);
+    if ((r < 0) && (errno != EEXIST)) {
+      int err = -errno;
+      derr << __func__ << " failed to create dir " << fn << ": "
+	   << cpp_strerror(err) << dendl;
+      ceph_abort();
+    }
+    dout(15) << "set_int " << a << "/" << b << " = " << val << dendl;
+    snprintf(fn, sizeof(fn), "%s/%s/%s", dir.c_str(), a, b);
+  } else {
+    dout(15) << "set_int " << a << " = " << val << dendl;
+  }
+  
+  char vs[30];
+  snprintf(vs, sizeof(vs), "%lld\n", (unsigned long long)val);
+
+  char tfn[1024];
+  snprintf(tfn, sizeof(tfn), "%s.new", fn);
+
+  int fd = TEMP_FAILURE_RETRY(::open(tfn, O_WRONLY|O_CREAT|O_TRUNC, 0600));
+  if (fd < 0) {
+    int err = errno;
+    derr << "MonitorStore::put_int: failed to open '" << tfn << "': "
+	 << cpp_strerror(err) << dendl;
+    ceph_abort();
+  }
+  int r = safe_write(fd, vs, strlen(vs));
+  if (r) {
+    derr << "MonitorStore::put_int: failed to write to '" << tfn << "': "
+	 << cpp_strerror(r) << dendl;
+    ceph_abort();
+  }
+  r = ::fsync(fd);
+  if (r) {
+    derr << "Monitor::put_int: failed to fsync fd for '" << tfn << "': "
+	 << cpp_strerror(r) << dendl;
+    ceph_abort();
+  }
+  if (TEMP_FAILURE_RETRY(::close(fd))) {
+    derr << "MonitorStore::put_int: failed to close fd for '" << tfn << "': "
+	 << cpp_strerror(r) << dendl;
+    ceph_abort();
+  }
+  if (::rename(tfn, fn)) {
+    int err = errno;
+    derr << "MonitorStore::put_int: failed to rename '" << tfn << "' to "
+	 << "'" << fn << "': " << cpp_strerror(err) << dendl;
+    ceph_abort();
+  }
+}
+
+// kludge to associate a global version number with each per-machine paxos state
+version_t MonitorStore::get_global_version(const char *a, version_t b)
+{
+  char fn[1024], fn2[1024];
+  snprintf(fn, sizeof(fn), "%s_gv", a);
+  snprintf(fn2, sizeof(fn2), "%llu", (long long unsigned)b);
+  return get_int(fn, fn2);
+}
+
+// ----------------------------------------
+// buffers
+
+bool MonitorStore::exists_bl_ss(const char *a, const char *b)
+{
+  char fn[1024];
+  if (b) {
+    dout(15) << "exists_bl " << a << "/" << b << dendl;
+    snprintf(fn, sizeof(fn), "%s/%s/%s", dir.c_str(), a, b);
+  } else {
+    dout(15) << "exists_bl " << a << dendl;
+    snprintf(fn, sizeof(fn), "%s/%s", dir.c_str(), a);
+  }
+  
+  struct stat st;
+  int r = ::stat(fn, &st);
+  //char buf[80];
+  //dout(15) << "exists_bl stat " << fn << " r=" << r << " " << cpp_strerror(errno) << dendl;
+  if (r) {
+    assert (errno == ENOENT);
+  }
+  return r == 0;
+}
+
+void MonitorStore::erase_ss(const char *a, const char *b)
+{
+  char fn[1024];
+  char dr[1024];
+  snprintf(dr, sizeof(dr), "%s/%s", dir.c_str(), a);
+  if (b) {
+    dout(15) << "erase_ss " << a << "/" << b << dendl;
+    snprintf(fn, sizeof(fn), "%s/%s/%s", dir.c_str(), a, b);
+  } else {
+    dout(15) << "erase_ss " << a << dendl;
+    strcpy(fn, dr);
+  }
+  int r = ::unlink(fn);
+  assert(0 == r || ENOENT == errno); // callers don't check for existence first
+
+  ::rmdir(dr);  // sloppy attempt to clean up empty dirs
+}
+
+int MonitorStore::get_bl_ss(bufferlist& bl, const char *a, const char *b)
+{
+  char fn[1024];
+  if (b) {
+    snprintf(fn, sizeof(fn), "%s/%s/%s", dir.c_str(), a, b);
+  } else {
+    snprintf(fn, sizeof(fn), "%s/%s", dir.c_str(), a);
+  }
+  
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    if (b) {
+      dout(15) << "get_bl " << a << "/" << b << " " << cpp_strerror(errno) << dendl;
+    } else {
+      dout(15) << "get_bl " << a << " " << cpp_strerror(errno) << dendl;
+    }
+    return -errno;
+  }
+
+  // get size
+  struct stat st;
+  int rc = ::fstat(fd, &st);
+  assert(rc == 0);
+  __int32_t len = st.st_size;
+ 
+  // read buffer
+  bl.clear();
+  bufferptr bp(len);
+  int off = 0;
+  while (off < len) {
+    dout(20) << "reading at off " << off << " of " << len << dendl;
+    int r = ::read(fd, bp.c_str()+off, len-off);
+    if (r < 0)
+      dout(0) << "errno on read " << cpp_strerror(errno) << dendl;
+    assert(r>0);
+    off += r;
+  }
+  bl.append(bp);
+  int close_err = TEMP_FAILURE_RETRY(::close(fd));
+  assert (0 == close_err);
+
+  if (b) {
+    dout(15) << "get_bl " << a << "/" << b << " = " << bl.length() << " bytes" << dendl;
+  } else {
+    dout(15) << "get_bl " << a << " = " << bl.length() << " bytes" << dendl;
+  }
+
+  return len;
+}
+
+void MonitorStore::write_bl_ss(bufferlist& bl, const char *a, const char *b, bool append)
+{
+  int err = 0;
+  char fn[1024];
+  snprintf(fn, sizeof(fn), "%s/%s", dir.c_str(), a);
+  if (b) {
+    int r = ::mkdir(fn, 0755);
+    if ((r < 0) && (errno != EEXIST)) {
+      err = -errno;
+      derr << __func__ << " failed to create dir " << fn
+	   << ": " << cpp_strerror(err) << dendl;
+      assert(0 == "failed to create dir");
+    }
+    dout(15) << "put_bl " << a << "/" << b << " = " << bl.length() << " bytes" << dendl;
+    snprintf(fn, sizeof(fn), "%s/%s/%s", dir.c_str(), a, b);
+  } else {
+    dout(15) << "put_bl " << a << " = " << bl.length() << " bytes" << dendl;
+  }
+  
+  char tfn[1024];
+  int fd;
+  if (append) {
+    fd = ::open(fn, O_WRONLY|O_CREAT|O_APPEND, 0600);
+    if (fd < 0) {
+      err = -errno;
+      derr << "failed to open " << fn << "for append: "
+	   << cpp_strerror(err) << dendl;
+      assert(0 == "failed to open for append");
+    }
+  } else {
+    snprintf(tfn, sizeof(tfn), "%s.new", fn);
+    fd = ::open(tfn, O_WRONLY|O_CREAT|O_TRUNC, 0600);
+    if (fd < 0) {
+      err = -errno;
+      derr << "failed to open " << tfn << ": " << cpp_strerror(err) << dendl;
+      assert(0 == "failed to open");
+    }
+  }
+  
+  err = bl.write_fd(fd);
+  assert(!err);
+  err = ::fsync(fd);
+  assert(!err);
+  err = TEMP_FAILURE_RETRY(::close(fd));
+  assert (!err); // this really can't fail, right? right?...
+  if (!append) {
+    err = ::rename(tfn, fn);
+    if (err < 0) {
+      err = -errno;
+      derr << __func__ << " failed to rename '" << tfn << "' -> '"
+	   << fn << "': " << cpp_strerror(err) << dendl;
+      assert(0 == "failed to rename");
+    }
+  }
+}
+
+void MonitorStore::put_bl_sn_map(const char *a,
+				map::iterator start,
+				map::iterator end)
+{
+  int err = 0;
+  int close_err = 0;
+  version_t first = start->first;
+  map::iterator lastp = end;
+  --lastp;
+  version_t last = lastp->first;
+  dout(15) <<  "put_bl_sn_map " << a << "/[" << first << ".." << last << "]" << dendl;
+
+  // only do a big sync if there are several values, or if the feature is disabled.
+  if (g_conf->mon_sync_fs_threshold <= 0 ||
+      last - first < (unsigned)g_conf->mon_sync_fs_threshold) {
+    // just do them individually
+    for (map::iterator p = start; p != end; ++p) {
+      put_bl_sn(p->second, a, p->first);
+    }
+    return;
+  }
+
+  // make sure dir exists
+  char dfn[1024];
+  snprintf(dfn, sizeof(dfn), "%s/%s", dir.c_str(), a);
+  int r = ::mkdir(dfn, 0755);
+  if ((r < 0) && (errno != EEXIST)) {
+    err = -errno;
+    derr << __func__ << " failed to create dir " << dfn << ": "
+	 << cpp_strerror(err) << dendl;
+    assert(0 == "failed to create dir");
+  }
+
+  for (map::iterator p = start; p != end; ++p) {
+    char tfn[1024], fn[1024];
+
+    snprintf(fn, sizeof(fn), "%s/%llu", dfn, (long long unsigned)p->first);
+    snprintf(tfn, sizeof(tfn), "%s.new", fn);
+
+    int fd = ::open(tfn, O_WRONLY|O_CREAT|O_TRUNC, 0600);
+    if (fd < 0) {
+      int err = -errno;
+      derr << "failed to open " << tfn << ": " << cpp_strerror(err) << dendl;
+      assert(0 == "failed to open");
+    }
+
+    err = p->second.write_fd(fd);
+    close_err = TEMP_FAILURE_RETRY(::close(fd));
+    assert (0 == close_err);
+    if (err < 0)
+      assert(0 == "failed to write");
+  }
+
+  // sync them all
+  int dirfd = ::open(dir.c_str(), O_RDONLY);
+  if (dirfd < 0) {
+    err = -errno;
+    derr << "failed to open " << dir << ": " << cpp_strerror(err) << dendl;
+    assert(0 == "failed to open temp file");
+  }
+  sync_filesystem(dirfd);
+  close_err = TEMP_FAILURE_RETRY(::close(dirfd));
+  assert (0 == close_err);
+    
+  // rename them all into place
+  for (map::iterator p = start; p != end; ++p) {
+    char tfn[1024], fn[1024];
+    
+    snprintf(fn, sizeof(fn), "%s/%llu", dfn, (long long unsigned)p->first);
+    snprintf(tfn, sizeof(tfn), "%s.new", fn);
+    
+    err = ::rename(tfn, fn);
+    if (err < 0)
+      assert(0 == "failed to rename");
+  }
+    
+  // fsync the dir (to commit the renames)
+  dirfd = ::open(dir.c_str(), O_RDONLY);
+  if (dirfd < 0) {
+    err = -errno;
+    derr << __func__ << " failed to open " << dir
+	 << ": " << cpp_strerror(err) << dendl;
+    assert(0 == "failed to open dir");
+  }
+  err = ::fsync(dirfd);
+  if (err < 0) {
+    err = -errno;
+    derr << __func__ << " failed to fsync " << dir
+	 << ": " << cpp_strerror(err) << dendl;
+    assert(0 == "failed to fsync");
+  }
+  close_err = TEMP_FAILURE_RETRY(::close(dirfd));
+  assert (0 == close_err);
+}
+
+void MonitorStore::sync()
+{
+  int dirfd = ::open(dir.c_str(), O_RDONLY);
+  if (dirfd < 0) {
+    int err = -errno;
+    derr << __func__ << " failed to open " << dir
+	 << ": " << cpp_strerror(err) << dendl;
+    assert(0 == "failed to open dir for syncing");
+  }
+  sync_filesystem(dirfd);
+  int close_err = TEMP_FAILURE_RETRY(::close(dirfd));
+  assert (0 == close_err);
+}
diff --git a/ceph/src/mon/MonitorStore.h b/ceph/src/mon/MonitorStore.h
new file mode 100644
index 00000000..76b83633
--- /dev/null
+++ b/ceph/src/mon/MonitorStore.h
@@ -0,0 +1,109 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MON_MONITORSTORE_H
+#define CEPH_MON_MONITORSTORE_H
+
+#include "include/types.h"
+#include "include/buffer.h"
+
+#include "common/compiler_extensions.h"
+
+#include 
+#include 
+#include 
+
+class MonitorStore {
+  string dir;
+  int lock_fd;
+
+  void write_bl_ss(bufferlist& bl, const char *a, const char *b,
+		  bool append);
+public:
+  MonitorStore(const std::string &d) : dir(d), lock_fd(-1) { }
+  ~MonitorStore() { }
+
+  int mkfs();  // wipe
+  int mount();
+  int umount();
+  void sync();
+
+  // ints (stored as ascii)
+  version_t get_int(const char *a, const char *b=0) WARN_UNUSED_RESULT;
+  void put_int(version_t v, const char *a, const char *b=0);
+
+  version_t get_global_version(const char *a, version_t b) WARN_UNUSED_RESULT;
+
+  // buffers
+  // ss and sn varieties.
+  bool exists_bl_ss(const char *a, const char *b=0);
+  int get_bl_ss(bufferlist& bl, const char *a, const char *b) WARN_UNUSED_RESULT;
+  void get_bl_ss_safe(bufferlist& bl, const char *a, const char *b) {
+    int ret = get_bl_ss(bl, a, b);
+    assert (ret >= 0 || ret == -ENOENT);
+  }
+  void put_bl_ss(bufferlist& bl, const char *a, const char *b) {
+    write_bl_ss(bl, a, b, false);
+  }
+  void append_bl_ss(bufferlist& bl, const char *a, const char *b) {
+    write_bl_ss(bl, a, b, true);
+  }
+  bool exists_bl_sn(const char *a, version_t b) {
+    char bs[20];
+    snprintf(bs, sizeof(bs), "%llu", (unsigned long long)b);
+    return exists_bl_ss(a, bs);
+  }
+  int get_bl_sn(bufferlist& bl, const char *a, version_t b) WARN_UNUSED_RESULT {
+    char bs[20];
+    snprintf(bs, sizeof(bs), "%llu", (unsigned long long)b);
+    return get_bl_ss(bl, a, bs);
+  }
+  void get_bl_sn_safe(bufferlist& bl, const char *a, version_t b) {
+    int ret = get_bl_sn(bl, a, b);
+    assert(ret >= 0 || ret == -ENOENT);
+  }
+  void put_bl_sn(bufferlist& bl, const char *a, version_t b) {
+    char bs[20];
+    snprintf(bs, sizeof(bs), "%llu", (unsigned long long)b);
+    put_bl_ss(bl, a, bs);
+  }
+  /**
+   * Put a whole set of values efficiently and safely.
+   *
+   * @param a - prefix/directory
+   * @param vals - map of int name -> values
+   * @return 0 for success or negative error code
+   */
+  void put_bl_sn_map(const char *a,
+		    map::iterator start,
+		    map::iterator end);
+
+  void erase_ss(const char *a, const char *b);
+  void erase_sn(const char *a, version_t b) {
+    char bs[20];
+    snprintf(bs, sizeof(bs), "%llu", (unsigned long long)b);
+    erase_ss(a, bs);
+  }
+
+  /*
+  version_t get_incarnation() { return get_int("incarnation"); }
+  void set_incarnation(version_t i) { set_int(i, "incarnation"); }
+  
+  version_t get_last_proposal() { return get_int("last_proposal"); }
+  void set_last_proposal(version_t i) { set_int(i, "last_proposal"); }
+  */
+};
+
+
+#endif
diff --git a/ceph/src/mon/MonmapMonitor.cc b/ceph/src/mon/MonmapMonitor.cc
new file mode 100644
index 00000000..3890704e
--- /dev/null
+++ b/ceph/src/mon/MonmapMonitor.cc
@@ -0,0 +1,504 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2009 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MonmapMonitor.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+
+#include "messages/MMonCommand.h"
+#include "messages/MMonJoin.h"
+
+#include "common/Timer.h"
+#include "common/ceph_argparse.h"
+#include "common/errno.h"
+#include "mon/MDSMonitor.h"
+#include "mon/OSDMonitor.h"
+#include "mon/PGMonitor.h"
+
+#include 
+#include "common/config.h"
+#include "common/cmdparse.h"
+#include "include/str_list.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon)
+static ostream& _prefix(std::ostream *_dout, Monitor *mon) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").monmap v" << mon->monmap->epoch << " ";
+}
+
+void MonmapMonitor::create_initial()
+{
+  dout(10) << "create_initial using current monmap" << dendl;
+  pending_map = *mon->monmap;
+  pending_map.epoch = 1;
+}
+
+void MonmapMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version_t version = get_last_committed();
+  if (version <= mon->monmap->get_epoch())
+    return;
+
+  dout(10) << __func__ << " version " << version
+	   << ", my v " << mon->monmap->epoch << dendl;
+  
+  if (need_bootstrap && version != mon->monmap->get_epoch()) {
+    dout(10) << " signaling that we need a bootstrap" << dendl;
+    *need_bootstrap = true;
+  }
+
+  // read and decode
+  monmap_bl.clear();
+  int ret = get_version(version, monmap_bl);
+  assert(ret == 0);
+  assert(monmap_bl.length());
+
+  dout(10) << "update_from_paxos got " << version << dendl;
+  mon->monmap->decode(monmap_bl);
+
+  if (mon->store->exists("mkfs", "monmap")) {
+    MonitorDBStore::Transaction t;
+    t.erase("mkfs", "monmap");
+    mon->store->apply_transaction(t);
+  }
+}
+
+void MonmapMonitor::create_pending()
+{
+  pending_map = *mon->monmap;
+  pending_map.epoch++;
+  pending_map.last_changed = ceph_clock_now(g_ceph_context);
+  dout(10) << "create_pending monmap epoch " << pending_map.epoch << dendl;
+}
+
+void MonmapMonitor::encode_pending(MonitorDBStore::Transaction *t)
+{
+  dout(10) << "encode_pending epoch " << pending_map.epoch << dendl;
+
+  assert(mon->monmap->epoch + 1 == pending_map.epoch ||
+	 pending_map.epoch == 1);  // special case mkfs!
+  bufferlist bl;
+  pending_map.encode(bl, mon->get_quorum_features());
+
+  put_version(t, pending_map.epoch, bl);
+  put_last_committed(t, pending_map.epoch);
+
+  // generate a cluster fingerprint, too?
+  if (pending_map.epoch == 1) {
+    mon->prepare_new_fingerprint(t);
+  }
+}
+
+void MonmapMonitor::on_active()
+{
+  if (get_last_committed() >= 1 && !mon->has_ever_joined) {
+    // make note of the fact that i was, once, part of the quorum.
+    dout(10) << "noting that i was, once, part of an active quorum." << dendl;
+
+    /* This is some form of nasty in-breeding we have between the MonmapMonitor
+       and the Monitor itself. We should find a way to get rid of it given our
+       new architecture. Until then, stick with it since we are a
+       single-threaded process and, truth be told, no one else relies on this
+       thing besides us.
+     */
+    MonitorDBStore::Transaction t;
+    t.put(Monitor::MONITOR_NAME, "joined", 1);
+    mon->store->apply_transaction(t);
+    mon->has_ever_joined = true;
+  }
+
+  if (mon->is_leader())
+    mon->clog.info() << "monmap " << *mon->monmap << "\n";
+}
+
+bool MonmapMonitor::preprocess_query(PaxosServiceMessage *m)
+{
+  switch (m->get_type()) {
+    // READs
+  case MSG_MON_COMMAND:
+    return preprocess_command(static_cast(m));
+  case MSG_MON_JOIN:
+    return preprocess_join(static_cast(m));
+  default:
+    assert(0);
+    m->put();
+    return true;
+  }
+}
+
+void MonmapMonitor::dump_info(Formatter *f)
+{
+  f->dump_unsigned("monmap_first_committed", get_first_committed());
+  f->dump_unsigned("monmap_last_committed", get_last_committed());
+  f->open_object_section("monmap");
+  mon->monmap->dump(f);
+  f->close_section();
+  f->open_array_section("quorum");
+  for (set::iterator q = mon->get_quorum().begin(); q != mon->get_quorum().end(); ++q)
+    f->dump_int("mon", *q);
+  f->close_section();
+}
+
+bool MonmapMonitor::preprocess_command(MMonCommand *m)
+{
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  if (prefix == "mon stat") {
+    mon->monmap->print_summary(ss);
+    ss << ", election epoch " << mon->get_epoch() << ", quorum " << mon->get_quorum()
+       << " " << mon->get_quorum_names();
+    rdata.append(ss);
+    ss.str("");
+    r = 0;
+
+  } else if (prefix == "mon getmap" ||
+             prefix == "mon dump") {
+
+    epoch_t epoch;
+    int64_t epochnum;
+    cmd_getval(g_ceph_context, cmdmap, "epoch", epochnum, (int64_t)0);
+    epoch = epochnum;
+
+    MonMap *p = mon->monmap;
+    if (epoch) {
+      bufferlist bl;
+      r = get_version(epoch, bl);
+      if (r == -ENOENT) {
+        ss << "there is no map for epoch " << epoch;
+        goto reply;
+      }
+      assert(r == 0);
+      assert(bl.length() > 0);
+      p = new MonMap;
+      p->decode(bl);
+    }
+
+    assert(p != NULL);
+
+    if (prefix == "mon getmap") {
+      p->encode(rdata, m->get_connection()->get_features());
+      r = 0;
+      ss << "got monmap epoch " << p->get_epoch();
+    } else if (prefix == "mon dump") {
+      string format;
+      cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+      stringstream ds;
+      boost::scoped_ptr f(new_formatter(format));
+      if (f) {
+        f->open_object_section("monmap");
+        p->dump(f.get());
+        f->open_array_section("quorum");
+        for (set::iterator q = mon->get_quorum().begin();
+            q != mon->get_quorum().end(); ++q) {
+          f->dump_int("mon", *q);
+        }
+        f->close_section();
+        f->close_section();
+        f->flush(ds);
+        r = 0;
+      } else {
+        p->print(ds);
+        r = 0;
+      }
+      rdata.append(ds);
+      ss << "dumped monmap epoch " << p->get_epoch();
+    }
+    if (p != mon->monmap)
+       delete p;
+  }
+  else if (prefix == "mon add")
+    return false;
+  else if (prefix == "mon remove")
+    return false;
+
+reply:
+  if (r != -1) {
+    string rs;
+    getline(ss, rs);
+
+    mon->reply_command(m, r, rs, rdata, get_last_committed());
+    return true;
+  } else
+    return false;
+}
+
+
+bool MonmapMonitor::prepare_update(PaxosServiceMessage *m)
+{
+  dout(7) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+  
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    return prepare_command(static_cast(m));
+  case MSG_MON_JOIN:
+    return prepare_join(static_cast(m));
+  default:
+    assert(0);
+    m->put();
+  }
+
+  return false;
+}
+
+bool MonmapMonitor::prepare_command(MMonCommand *m)
+{
+  stringstream ss;
+  string rs;
+  int err = -EINVAL;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  if (prefix == "mon add") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    string addrstr;
+    cmd_getval(g_ceph_context, cmdmap, "addr", addrstr);
+    entity_addr_t addr;
+    bufferlist rdata;
+
+    if (!addr.parse(addrstr.c_str())) {
+      err = -EINVAL;
+      ss << "addr " << addrstr << "does not parse";
+      goto out;
+    }
+
+    if (addr.get_port() == 0) {
+      ss << "port defaulted to " << CEPH_MON_PORT;
+      addr.set_port(CEPH_MON_PORT);
+    }
+
+    /**
+     * If we have a monitor with the same name and different addr, then EEXIST
+     * If we have a monitor with the same addr and different name, then EEXIST
+     * If we have a monitor with the same addr and same name, then return as if
+     * we had just added the monitor.
+     * If we don't have the monitor, add it.
+     */
+
+    err = 0;
+    if (!ss.str().empty())
+      ss << "; ";
+
+    do {
+      if (pending_map.contains(addr)) {
+        string n = pending_map.get_name(addr);
+        if (n == name)
+          break;
+      } else if (pending_map.contains(name)) {
+        entity_addr_t tmp_addr = pending_map.get_addr(name);
+        if (tmp_addr == addr)
+          break;
+      } else {
+        break;
+      }
+      err = -EEXIST;
+      ss << "mon." << name << " at " << addr << " already exists";
+      goto out;
+    } while (false);
+
+    ss << "added mon." << name << " at " << addr;
+    if (pending_map.contains(name)) {
+      goto out;
+    }
+
+    pending_map.add(name, addr);
+    pending_map.last_changed = ceph_clock_now(g_ceph_context);
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+                                                     get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "mon remove") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    if (!pending_map.contains(name)) {
+      err = 0;
+      ss << "mon " << name << " does not exist or has already been removed";
+      goto out;
+    }
+
+    if (pending_map.size() == 1) {
+      err = -EINVAL;
+      ss << "error: refusing removal of last monitor " << name;
+      goto out;
+    }
+    entity_addr_t addr = pending_map.get_addr(name);
+    pending_map.remove(name);
+    pending_map.last_changed = ceph_clock_now(g_ceph_context);
+    ss << "removed mon." << name << " at " << addr << ", there are now " << pending_map.size() << " monitors" ;
+    getline(ss, rs);
+    // send reply immediately in case we get removed
+    mon->reply_command(m, 0, rs, get_last_committed());
+    return true;
+  }
+  else
+    ss << "unknown command " << prefix;
+
+out:
+  getline(ss, rs);
+  mon->reply_command(m, err, rs, get_last_committed());
+  return false;
+}
+
+bool MonmapMonitor::preprocess_join(MMonJoin *join)
+{
+  dout(10) << "preprocess_join " << join->name << " at " << join->addr << dendl;
+
+  MonSession *session = join->get_session();
+  if (!session ||
+      !session->is_capable("mon", MON_CAP_W | MON_CAP_X)) {
+    dout(10) << " insufficient caps" << dendl;
+    join->put();
+    return true;
+  }
+
+  if (pending_map.contains(join->name) && !pending_map.get_addr(join->name).is_blank_ip()) {
+    dout(10) << " already have " << join->name << dendl;
+    join->put();
+    return true;
+  }
+  if (pending_map.contains(join->addr) && pending_map.get_name(join->addr) == join->name) {
+    dout(10) << " already have " << join->addr << dendl;
+    join->put();
+    return true;
+  }
+  return false;
+}
+bool MonmapMonitor::prepare_join(MMonJoin *join)
+{
+  dout(0) << "adding/updating " << join->name << " at " << join->addr << " to monitor cluster" << dendl;
+  if (pending_map.contains(join->name))
+    pending_map.remove(join->name);
+  if (pending_map.contains(join->addr))
+    pending_map.remove(pending_map.get_name(join->addr));
+  pending_map.add(join->name, join->addr);
+  pending_map.last_changed = ceph_clock_now(g_ceph_context);
+  join->put();
+  return true;
+}
+
+bool MonmapMonitor::should_propose(double& delay)
+{
+  delay = 0.0;
+  return true;
+}
+
+void MonmapMonitor::tick()
+{
+}
+
+void MonmapMonitor::get_health(list >& summary,
+			       list > *detail) const
+{
+  int max = mon->monmap->size();
+  int actual = mon->get_quorum().size();
+  if (actual < max) {
+    ostringstream ss;
+    ss << (max-actual) << " mons down, quorum " << mon->get_quorum() << " " << mon->get_quorum_names();
+    summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+    if (detail) {
+      set q = mon->get_quorum();
+      for (int i=0; imonmap->get_name(i) << " (rank " << i
+	     << ") addr " << mon->monmap->get_addr(i)
+	     << " is down (out of quorum)";
+	  detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+	}
+      }
+    }
+  }
+  if (g_conf->mon_warn_on_old_mons && !mon->get_classic_mons().empty()) {
+    ostringstream ss;
+    ss << "some monitors are running older code";
+    summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+    if (detail) {
+      for (set::const_iterator i = mon->get_classic_mons().begin();
+	  i != mon->get_classic_mons().end();
+	  ++i) {
+	ostringstream ss;
+	ss << "mon." << mon->monmap->get_name(*i)
+	     << " only supports the \"classic\" command set";
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+  }
+}
+
+int MonmapMonitor::get_monmap(bufferlist &bl)
+{
+  version_t latest_ver = get_last_committed();
+  dout(10) << __func__ << " ver " << latest_ver << dendl;
+
+  if (!mon->store->exists(get_service_name(), stringify(latest_ver)))
+    return -ENOENT;
+
+  int err = get_version(latest_ver, bl);
+  if (err < 0) {
+    dout(1) << __func__ << " error obtaining monmap: "
+            << cpp_strerror(err) << dendl;
+    return err;
+  }
+  return 0;
+}
+
+int MonmapMonitor::get_monmap(MonMap &m)
+{
+  dout(10) << __func__ << dendl;
+  bufferlist monmap_bl;
+
+  int err = get_monmap(monmap_bl);
+  if (err < 0) {
+    return err;
+  }
+  m.decode(monmap_bl);
+  return 0;
+}
diff --git a/ceph/src/mon/MonmapMonitor.h b/ceph/src/mon/MonmapMonitor.h
new file mode 100644
index 00000000..198489d7
--- /dev/null
+++ b/ceph/src/mon/MonmapMonitor.h
@@ -0,0 +1,89 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2009 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/*
+ * The Monmap Monitor is used to track the monitors in the cluster.
+ */
+
+#ifndef CEPH_MONMAPMONITOR_H
+#define CEPH_MONMAPMONITOR_H
+
+#include 
+#include 
+
+using namespace std;
+
+#include "include/types.h"
+#include "msg/Messenger.h"
+
+#include "PaxosService.h"
+#include "MonMap.h"
+#include "MonitorDBStore.h"
+
+class MMonGetMap;
+class MMonMap;
+class MMonCommand;
+class MMonJoin;
+
+class MonmapMonitor : public PaxosService {
+ public:
+  MonmapMonitor(Monitor *mn, Paxos *p, const string& service_name)
+    : PaxosService(mn, p, service_name)
+  {
+  }
+  MonMap pending_map; //the pending map awaiting passage
+
+  void create_initial();
+
+  void update_from_paxos(bool *need_bootstrap);
+
+  void create_pending();
+
+  void encode_pending(MonitorDBStore::Transaction *t);
+  // we always encode the full map; we have no use for full versions
+  virtual void encode_full(MonitorDBStore::Transaction *t) { }
+
+  void on_active();
+
+  void dump_info(Formatter *f);
+
+  bool preprocess_query(PaxosServiceMessage *m);
+  bool prepare_update(PaxosServiceMessage *m);
+
+  bool preprocess_join(MMonJoin *m);
+  bool prepare_join(MMonJoin *m);
+
+  bool preprocess_command(MMonCommand *m);
+  bool prepare_command(MMonCommand *m);
+
+  void get_health(list >& summary,
+		  list > *detail) const;
+
+  int get_monmap(bufferlist &bl);
+  int get_monmap(MonMap &m);
+
+  /*
+   * Since monitors are pretty
+   * important, this implementation will just write 0.0.
+   */
+  bool should_propose(double& delay);
+
+  void tick();
+
+ private:
+  bufferlist monmap_bl;
+};
+
+
+#endif
diff --git a/ceph/src/mon/OSDMonitor.cc b/ceph/src/mon/OSDMonitor.cc
new file mode 100644
index 00000000..7e469b2d
--- /dev/null
+++ b/ceph/src/mon/OSDMonitor.cc
@@ -0,0 +1,5967 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+
+#include "OSDMonitor.h"
+#include "Monitor.h"
+#include "MDSMonitor.h"
+#include "PGMonitor.h"
+
+#include "MonitorDBStore.h"
+
+#include "crush/CrushWrapper.h"
+#include "crush/CrushTester.h"
+
+#include "messages/MOSDFailure.h"
+#include "messages/MOSDMarkMeDown.h"
+#include "messages/MOSDMap.h"
+#include "messages/MOSDBoot.h"
+#include "messages/MOSDAlive.h"
+#include "messages/MPoolOp.h"
+#include "messages/MPoolOpReply.h"
+#include "messages/MOSDPGTemp.h"
+#include "messages/MMonCommand.h"
+#include "messages/MRemoveSnaps.h"
+#include "messages/MOSDScrub.h"
+
+#include "common/Timer.h"
+#include "common/ceph_argparse.h"
+#include "common/perf_counters.h"
+#include "common/strtol.h"
+
+#include "common/config.h"
+#include "common/errno.h"
+
+#include "erasure-code/ErasureCodePlugin.h"
+
+#include "include/compat.h"
+#include "include/assert.h"
+#include "include/stringify.h"
+#include "include/util.h"
+#include "common/cmdparse.h"
+#include "include/str_list.h"
+#include "include/str_map.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, osdmap)
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, OSDMap& osdmap) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").osd e" << osdmap.get_epoch() << " ";
+}
+
+bool OSDMonitor::_have_pending_crush()
+{
+  return pending_inc.crush.length();
+}
+
+CrushWrapper &OSDMonitor::_get_stable_crush()
+{
+  return *osdmap.crush;
+}
+
+void OSDMonitor::_get_pending_crush(CrushWrapper& newcrush)
+{
+  bufferlist bl;
+  if (pending_inc.crush.length())
+    bl = pending_inc.crush;
+  else
+    osdmap.crush->encode(bl);
+
+  bufferlist::iterator p = bl.begin();
+  newcrush.decode(p);
+}
+
+void OSDMonitor::create_initial()
+{
+  dout(10) << "create_initial for " << mon->monmap->fsid << dendl;
+
+  OSDMap newmap;
+
+  bufferlist bl;
+  mon->store->get("mkfs", "osdmap", bl);
+
+  if (bl.length()) {
+    newmap.decode(bl);
+    newmap.set_fsid(mon->monmap->fsid);
+  } else {
+    newmap.build_simple(g_ceph_context, 0, mon->monmap->fsid, 0,
+			g_conf->osd_pg_bits, g_conf->osd_pgp_bits);
+  }
+  newmap.set_epoch(1);
+  newmap.created = newmap.modified = ceph_clock_now(g_ceph_context);
+
+  // encode into pending incremental
+  newmap.encode(pending_inc.fullmap, mon->quorum_features);
+}
+
+void OSDMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version_t version = get_last_committed();
+  if (version == osdmap.epoch)
+    return;
+  assert(version >= osdmap.epoch);
+
+  dout(15) << "update_from_paxos paxos e " << version
+	   << ", my e " << osdmap.epoch << dendl;
+
+
+  /*
+   * We will possibly have a stashed latest that *we* wrote, and we will
+   * always be sure to have the oldest full map in the first..last range
+   * due to encode_trim_extra(), which includes the oldest full map in the trim
+   * transaction.
+   *
+   * encode_trim_extra() does not however write the full map's
+   * version to 'full_latest'.  This is only done when we are building the
+   * full maps from the incremental versions.  But don't panic!  We make sure
+   * that the following conditions find whichever full map version is newer.
+   */
+  version_t latest_full = get_version_latest_full();
+  if (latest_full == 0 && get_first_committed() > 1)
+    latest_full = get_first_committed();
+
+  if (latest_full > 0) {
+    // make sure we can really believe get_version_latest_full(); see
+    // 76cd7ac1c2094b34ad36bea89b2246fa90eb2f6d
+    bufferlist test;
+    get_version_full(latest_full, test);
+    if (test.length() == 0) {
+      dout(10) << __func__ << " ignoring recorded latest_full as it is missing; fallback to search" << dendl;
+      latest_full = 0;
+    }
+  }
+  if (get_first_committed() > 1 &&
+      latest_full < get_first_committed()) {
+    /* a bug introduced in 7fb3804fb860dcd0340dd3f7c39eec4315f8e4b6 would lead
+     * us to not update the on-disk latest_full key.  Upon trim, the actual
+     * version would cease to exist but we would still point to it.  This
+     * makes sure we get it pointing to a proper version.
+     */
+    version_t lc = get_last_committed();
+    version_t fc = get_first_committed();
+
+    dout(10) << __func__ << " looking for valid full map in interval"
+             << " [" << fc << ", " << lc << "]" << dendl;
+
+    latest_full = 0;
+    for (version_t v = lc; v >= fc; v--) {
+      string full_key = "full_" + stringify(v);
+      if (mon->store->exists(get_service_name(), full_key)) {
+        dout(10) << __func__ << " found latest full map v " << v << dendl;
+        latest_full = v;
+        break;
+      }
+    }
+
+    // if we trigger this, then there's something else going with the store
+    // state, and we shouldn't want to work around it without knowing what
+    // exactly happened.
+    assert(latest_full > 0);
+    MonitorDBStore::Transaction t;
+    put_version_latest_full(&t, latest_full);
+    mon->store->apply_transaction(t);
+    dout(10) << __func__ << " updated the on-disk full map version to "
+             << latest_full << dendl;
+  }
+
+  if ((latest_full > 0) && (latest_full > osdmap.epoch)) {
+    bufferlist latest_bl;
+    get_version_full(latest_full, latest_bl);
+    assert(latest_bl.length() != 0);
+    dout(7) << __func__ << " loading latest full map e" << latest_full << dendl;
+    osdmap.decode(latest_bl);
+  }
+
+  // walk through incrementals
+  MonitorDBStore::Transaction *t = NULL;
+  size_t tx_size = 0;
+  while (version > osdmap.epoch) {
+    bufferlist inc_bl;
+    int err = get_version(osdmap.epoch+1, inc_bl);
+    assert(err == 0);
+    assert(inc_bl.length());
+
+    dout(7) << "update_from_paxos  applying incremental " << osdmap.epoch+1 << dendl;
+    OSDMap::Incremental inc(inc_bl);
+    err = osdmap.apply_incremental(inc);
+    assert(err == 0);
+
+    if (t == NULL)
+      t = new MonitorDBStore::Transaction;
+
+    // Write out the full map for all past epochs.  Encode the full
+    // map with the same features as the incremental.  If we don't
+    // know, use the quorum features.  If we don't know those either,
+    // encode with all features.
+    uint64_t f = inc.encode_features;
+    if (!f)
+      f = mon->quorum_features;
+    if (!f)
+      f = -1;
+    bufferlist full_bl;
+    osdmap.encode(full_bl, f);
+    tx_size += full_bl.length();
+
+    put_version_full(t, osdmap.epoch, full_bl);
+    put_version_latest_full(t, osdmap.epoch);
+
+    // share
+    dout(1) << osdmap << dendl;
+
+    if (osdmap.epoch == 1) {
+      t->erase("mkfs", "osdmap");
+    }
+
+    if (tx_size > g_conf->mon_sync_max_payload_size*2) {
+      mon->store->apply_transaction(*t);
+      delete t;
+      t = NULL;
+      tx_size = 0;
+    }
+  }
+
+  if (t != NULL) {
+    mon->store->apply_transaction(*t);
+    delete t;
+  }
+
+  for (int o = 0; o < osdmap.get_max_osd(); o++) {
+    if (osdmap.is_down(o)) {
+      // invalidate osd_epoch cache
+      osd_epoch.erase(o);
+
+      // populate down -> out map
+      if (osdmap.is_in(o) &&
+	  down_pending_out.count(o) == 0) {
+	dout(10) << " adding osd." << o << " to down_pending_out map" << dendl;
+	down_pending_out[o] = ceph_clock_now(g_ceph_context);
+      }
+    }
+  }
+  // blow away any osd_epoch items beyond max_osd
+  map::iterator p = osd_epoch.upper_bound(osdmap.get_max_osd());
+  while (p != osd_epoch.end()) {
+    osd_epoch.erase(p++);
+  }
+
+  /** we don't have any of the feature bit infrastructure in place for
+   * supporting primary_temp mappings without breaking old clients/OSDs.*/
+  assert(g_conf->mon_osd_allow_primary_temp || osdmap.primary_temp->empty());
+
+  if (mon->is_leader()) {
+    // kick pgmon, make sure it's seen the latest map
+    mon->pgmon()->check_osd_map(osdmap.epoch);
+  }
+
+  check_subs();
+
+  share_map_with_random_osd();
+  update_logger();
+
+  process_failures();
+
+  // make sure our feature bits reflect the latest map
+  update_msgr_features();
+}
+
+void OSDMonitor::update_msgr_features()
+{
+  set types;
+  types.insert((int)entity_name_t::TYPE_OSD);
+  types.insert((int)entity_name_t::TYPE_CLIENT);
+  types.insert((int)entity_name_t::TYPE_MDS);
+  types.insert((int)entity_name_t::TYPE_MON);
+  for (set::iterator q = types.begin(); q != types.end(); ++q) {
+    uint64_t mask;
+    uint64_t features = osdmap.get_features(*q, &mask);
+    if ((mon->messenger->get_policy(*q).features_required & mask) != features) {
+      dout(0) << "crush map has features " << features << ", adjusting msgr requires" << dendl;
+      Messenger::Policy p = mon->messenger->get_policy(*q);
+      p.features_required = (p.features_required & ~mask) | features;
+      mon->messenger->set_policy(*q, p);
+    }
+  }
+}
+
+bool OSDMonitor::thrash()
+{
+  if (!thrash_map)
+    return false;
+
+  thrash_map--;
+  int o;
+
+  // mark a random osd up_thru.. 
+  if (rand() % 4 == 0 || thrash_last_up_osd < 0)
+    o = rand() % osdmap.get_num_osds();
+  else
+    o = thrash_last_up_osd;
+  if (osdmap.is_up(o)) {
+    dout(5) << "thrash_map osd." << o << " up_thru" << dendl;
+    pending_inc.new_up_thru[o] = osdmap.get_epoch();
+  }
+
+  // mark a random osd up/down
+  o = rand() % osdmap.get_num_osds();
+  if (osdmap.is_up(o)) {
+    dout(5) << "thrash_map osd." << o << " down" << dendl;
+    pending_inc.new_state[o] = CEPH_OSD_UP;
+  } else if (osdmap.exists(o)) {
+    dout(5) << "thrash_map osd." << o << " up" << dendl;
+    pending_inc.new_state[o] = CEPH_OSD_UP;
+    pending_inc.new_up_client[o] = entity_addr_t();
+    pending_inc.new_up_cluster[o] = entity_addr_t();
+    pending_inc.new_hb_back_up[o] = entity_addr_t();
+    pending_inc.new_weight[o] = CEPH_OSD_IN;
+    thrash_last_up_osd = o;
+  }
+
+  // mark a random osd in
+  o = rand() % osdmap.get_num_osds();
+  if (osdmap.exists(o)) {
+    dout(5) << "thrash_map osd." << o << " in" << dendl;
+    pending_inc.new_weight[o] = CEPH_OSD_IN;
+  }
+
+  // mark a random osd out
+  o = rand() % osdmap.get_num_osds();
+  if (osdmap.exists(o)) {
+    dout(5) << "thrash_map osd." << o << " out" << dendl;
+    pending_inc.new_weight[o] = CEPH_OSD_OUT;
+  }
+
+  // generate some pg_temp entries.
+  // let's assume the ceph::unordered_map iterates in a random-ish order.
+  int n = rand() % mon->pgmon()->pg_map.pg_stat.size();
+  ceph::unordered_map::iterator p = mon->pgmon()->pg_map.pg_stat.begin();
+  ceph::unordered_map::iterator e = mon->pgmon()->pg_map.pg_stat.end();
+  while (n--)
+    ++p;
+  for (int i=0; i<50; i++) {
+    unsigned size = osdmap.get_pg_size(p->first);
+    vector v;
+    bool have_real_osd = false;
+    for (int j=0; j < (int)size; j++) {
+      o = rand() % osdmap.get_num_osds();
+      if (osdmap.exists(o) && std::find(v.begin(), v.end(), o) == v.end()) {
+	have_real_osd = true;
+	v.push_back(o);
+      }
+    }
+    for (vector::iterator q = p->second.acting.begin();
+	 q != p->second.acting.end() && v.size() < size;
+	 ++q) {
+      if (std::find(v.begin(), v.end(), *q) == v.end()) {
+	if (*q != CRUSH_ITEM_NONE)
+	  have_real_osd = true;
+	v.push_back(*q);
+      }
+    }
+    if (osdmap.pg_is_ec(p->first)) {
+      while (v.size() < size)
+	v.push_back(CRUSH_ITEM_NONE);
+    }
+    if (!v.empty() && have_real_osd)
+      pending_inc.new_pg_temp[p->first] = v;
+    dout(5) << "thrash_map pg " << p->first << " pg_temp remapped to " << v << dendl;
+
+    ++p;
+    if (p == e)
+      p = mon->pgmon()->pg_map.pg_stat.begin();
+  }
+  return true;
+}
+
+void OSDMonitor::on_active()
+{
+  update_logger();
+
+  if (thrash_map) {
+    if (mon->is_leader()) {
+      if (thrash())
+	propose_pending();
+    } else {
+      thrash_map = 0;
+    }
+  }
+
+  if (mon->is_leader())
+    mon->clog.info() << "osdmap " << osdmap << "\n"; 
+
+  if (!mon->is_leader()) {
+    list ls;
+    take_all_failures(ls);
+    while (!ls.empty()) {
+      dispatch(ls.front());
+      ls.pop_front();
+    }
+  }
+}
+
+void OSDMonitor::on_shutdown()
+{
+  dout(10) << __func__ << dendl;
+
+  // discard failure info, waiters
+  list ls;
+  take_all_failures(ls);
+  while (!ls.empty()) {
+    ls.front()->put();
+    ls.pop_front();
+  }
+}
+
+void OSDMonitor::update_logger()
+{
+  dout(10) << "update_logger" << dendl;
+  
+  mon->cluster_logger->set(l_cluster_num_osd, osdmap.get_num_osds());
+  mon->cluster_logger->set(l_cluster_num_osd_up, osdmap.get_num_up_osds());
+  mon->cluster_logger->set(l_cluster_num_osd_in, osdmap.get_num_in_osds());
+  mon->cluster_logger->set(l_cluster_osd_epoch, osdmap.get_epoch());
+}
+
+/* Assign a lower weight to overloaded OSDs.
+ *
+ * The osds that will get a lower weight are those with with a utilization
+ * percentage 'oload' percent greater than the average utilization.
+ */
+int OSDMonitor::reweight_by_utilization(int oload, std::string& out_str)
+{
+  if (oload <= 100) {
+    ostringstream oss;
+    oss << "You must give a percentage higher than 100. "
+      "The reweighting threshold will be calculated as  "
+      "times . For example, an argument of 200 would "
+      "reweight OSDs which are twice as utilized as the average OSD.\n";
+    out_str = oss.str();
+    dout(0) << "reweight_by_utilization: " << out_str << dendl;
+    return -EINVAL;
+  }
+
+  // Avoid putting a small number (or 0) in the denominator when calculating
+  // average_util
+  const PGMap &pgm = mon->pgmon()->pg_map;
+  if (pgm.osd_sum.kb < 1024) {
+    ostringstream oss;
+    oss << "Refusing to reweight: we only have " << pgm.osd_sum << " kb "
+      "across all osds!\n";
+    out_str = oss.str();
+    dout(0) << "reweight_by_utilization: " << out_str << dendl;
+    return -EDOM;
+  }
+
+  if (pgm.osd_sum.kb_used < 5 * 1024) {
+    ostringstream oss;
+    oss << "Refusing to reweight: we only have " << pgm.osd_sum << " kb "
+      "used across all osds!\n";
+    out_str = oss.str();
+    dout(0) << "reweight_by_utilization: " << out_str << dendl;
+    return -EDOM;
+  }
+
+  float average_util = pgm.osd_sum.kb_used;
+  average_util /= pgm.osd_sum.kb;
+  float overload_util = average_util * oload / 100.0;
+
+  ostringstream oss;
+  char buf[128];
+  snprintf(buf, sizeof(buf), "average_util: %04f, overload_util: %04f. ",
+	   average_util, overload_util);
+  oss << buf;
+  std::string sep;
+  oss << "overloaded osds: ";
+  bool changed = false;
+  for (ceph::unordered_map::const_iterator p = pgm.osd_stat.begin();
+       p != pgm.osd_stat.end();
+       ++p) {
+    float util = p->second.kb_used;
+    util /= p->second.kb;
+    if (util >= overload_util) {
+      sep = ", ";
+      // Assign a lower weight to overloaded OSDs. The current weight
+      // is a factor to take into account the original weights,
+      // to represent e.g. differing storage capacities
+      unsigned weight = osdmap.get_weight(p->first);
+      unsigned new_weight = (unsigned)((average_util / util) * (float)weight);
+      pending_inc.new_weight[p->first] = new_weight;
+      char buf[128];
+      snprintf(buf, sizeof(buf), "%d [%04f -> %04f]", p->first,
+	       (float)weight / (float)0x10000,
+	       (float)new_weight / (float)0x10000);
+      oss << buf << sep;
+      changed = true;
+    }
+  }
+  if (sep.empty()) {
+    oss << "(none)";
+  }
+  out_str = oss.str();
+  dout(0) << "reweight_by_utilization: finished with " << out_str << dendl;
+  return changed;
+}
+
+void OSDMonitor::create_pending()
+{
+  pending_inc = OSDMap::Incremental(osdmap.epoch+1);
+  pending_inc.fsid = mon->monmap->fsid;
+  
+  dout(10) << "create_pending e " << pending_inc.epoch << dendl;
+
+  // drop any redundant pg_temp entries
+  OSDMap::remove_redundant_temporaries(g_ceph_context, osdmap, &pending_inc);
+
+  // drop any pg or primary_temp entries with no up entries
+  OSDMap::remove_down_temps(g_ceph_context, osdmap, &pending_inc);
+}
+
+/**
+ * @note receiving a transaction in this function gives a fair amount of
+ * freedom to the service implementation if it does need it. It shouldn't.
+ */
+void OSDMonitor::encode_pending(MonitorDBStore::Transaction *t)
+{
+  dout(10) << "encode_pending e " << pending_inc.epoch
+	   << dendl;
+  
+  // finalize up pending_inc
+  pending_inc.modified = ceph_clock_now(g_ceph_context);
+
+  int r = pending_inc.propagate_snaps_to_tiers(g_ceph_context, osdmap);
+  assert(r == 0);
+
+  bufferlist bl;
+
+  // tell me about it
+  for (map::iterator i = pending_inc.new_state.begin();
+       i != pending_inc.new_state.end();
+       ++i) {
+    int s = i->second ? i->second : CEPH_OSD_UP;
+    if (s & CEPH_OSD_UP)
+      dout(2) << " osd." << i->first << " DOWN" << dendl;
+    if (s & CEPH_OSD_EXISTS)
+      dout(2) << " osd." << i->first << " DNE" << dendl;
+  }
+  for (map::iterator i = pending_inc.new_up_client.begin();
+       i != pending_inc.new_up_client.end();
+       ++i) { 
+    //FIXME: insert cluster addresses too
+    dout(2) << " osd." << i->first << " UP " << i->second << dendl;
+  }
+  for (map::iterator i = pending_inc.new_weight.begin();
+       i != pending_inc.new_weight.end();
+       ++i) {
+    if (i->second == CEPH_OSD_OUT) {
+      dout(2) << " osd." << i->first << " OUT" << dendl;
+    } else if (i->second == CEPH_OSD_IN) {
+      dout(2) << " osd." << i->first << " IN" << dendl;
+    } else {
+      dout(2) << " osd." << i->first << " WEIGHT " << hex << i->second << dec << dendl;
+    }
+  }
+
+  // encode
+  assert(get_last_committed() + 1 == pending_inc.epoch);
+  ::encode(pending_inc, bl, mon->quorum_features);
+
+  /* put everything in the transaction */
+  put_version(t, pending_inc.epoch, bl);
+  put_last_committed(t, pending_inc.epoch);
+
+  // metadata, too!
+  for (map::iterator p = pending_metadata.begin();
+       p != pending_metadata.end();
+       ++p)
+    t->put(OSD_METADATA_PREFIX, stringify(p->first), p->second);
+  for (set::iterator p = pending_metadata_rm.begin();
+       p != pending_metadata_rm.end();
+       ++p)
+    t->erase(OSD_METADATA_PREFIX, stringify(*p));
+  pending_metadata.clear();
+  pending_metadata_rm.clear();
+}
+
+int OSDMonitor::dump_osd_metadata(int osd, Formatter *f, ostream *err)
+{
+  bufferlist bl;
+  int r = mon->store->get(OSD_METADATA_PREFIX, stringify(osd), bl);
+  if (r < 0)
+    return r;
+  map m;
+  try {
+    bufferlist::iterator p = bl.begin();
+    ::decode(m, p);
+  }
+  catch (buffer::error& e) {
+    if (err)
+      *err << "osd." << osd << " metadata is corrupt";
+    return -EIO;
+  }
+  for (map::iterator p = m.begin(); p != m.end(); ++p)
+    f->dump_string(p->first.c_str(), p->second);
+  return 0;
+}
+
+void OSDMonitor::share_map_with_random_osd()
+{
+  if (osdmap.get_num_up_osds() == 0) {
+    dout(10) << __func__ << " no up osds, don't share with anyone" << dendl;
+    return;
+  }
+
+  MonSession *s = mon->session_map.get_random_osd_session(&osdmap);
+  if (!s) {
+    dout(10) << __func__ << " no up osd on our session map" << dendl;
+    return;
+  }
+
+  dout(10) << "committed, telling random " << s->inst << " all about it" << dendl;
+  // whatev, they'll request more if they need it
+  MOSDMap *m = build_incremental(osdmap.get_epoch() - 1, osdmap.get_epoch());
+  mon->messenger->send_message(m, s->inst);
+}
+
+version_t OSDMonitor::get_trim_to()
+{
+  if (mon->pgmon()->is_readable() &&
+      mon->pgmon()->pg_map.creating_pgs.empty()) {
+    epoch_t floor = mon->pgmon()->pg_map.get_min_last_epoch_clean();
+    dout(10) << " min_last_epoch_clean " << floor << dendl;
+    if (g_conf->mon_osd_force_trim_to > 0 &&
+	g_conf->mon_osd_force_trim_to < (int)get_last_committed()) {
+      floor = g_conf->mon_osd_force_trim_to;
+      dout(10) << " explicit mon_osd_force_trim_to = " << floor << dendl;
+    }
+    unsigned min = g_conf->mon_min_osdmap_epochs;
+    if (floor + min > get_last_committed()) {
+      if (min < get_last_committed())
+	floor = get_last_committed() - min;
+      else
+	floor = 0;
+    }
+    if (floor > get_first_committed())
+      return floor;
+  }
+  return 0;
+}
+
+void OSDMonitor::encode_trim_extra(MonitorDBStore::Transaction *tx, version_t first)
+{
+  dout(10) << __func__ << " including full map for e " << first << dendl;
+  bufferlist bl;
+  get_version_full(first, bl);
+  put_version_full(tx, first, bl);
+}
+
+// -------------
+
+bool OSDMonitor::preprocess_query(PaxosServiceMessage *m)
+{
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+
+  switch (m->get_type()) {
+    // READs
+  case MSG_MON_COMMAND:
+    return preprocess_command(static_cast(m));
+
+    // damp updates
+  case MSG_OSD_MARK_ME_DOWN:
+    return preprocess_mark_me_down(static_cast(m));
+  case MSG_OSD_FAILURE:
+    return preprocess_failure(static_cast(m));
+  case MSG_OSD_BOOT:
+    return preprocess_boot(static_cast(m));
+  case MSG_OSD_ALIVE:
+    return preprocess_alive(static_cast(m));
+  case MSG_OSD_PGTEMP:
+    return preprocess_pgtemp(static_cast(m));
+
+  case CEPH_MSG_POOLOP:
+    return preprocess_pool_op(static_cast(m));
+
+  case MSG_REMOVE_SNAPS:
+    return preprocess_remove_snaps(static_cast(m));
+    
+  default:
+    assert(0);
+    m->put();
+    return true;
+  }
+}
+
+bool OSDMonitor::prepare_update(PaxosServiceMessage *m)
+{
+  dout(7) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+  
+  switch (m->get_type()) {
+    // damp updates
+  case MSG_OSD_MARK_ME_DOWN:
+    return prepare_mark_me_down(static_cast(m));
+  case MSG_OSD_FAILURE:
+    return prepare_failure(static_cast(m));
+  case MSG_OSD_BOOT:
+    return prepare_boot(static_cast(m));
+  case MSG_OSD_ALIVE:
+    return prepare_alive(static_cast(m));
+  case MSG_OSD_PGTEMP:
+    return prepare_pgtemp(static_cast(m));
+
+  case MSG_MON_COMMAND:
+    return prepare_command(static_cast(m));
+    
+  case CEPH_MSG_POOLOP:
+    return prepare_pool_op(static_cast(m));
+
+  case MSG_REMOVE_SNAPS:
+    return prepare_remove_snaps(static_cast(m));
+
+  default:
+    assert(0);
+    m->put();
+  }
+
+  return false;
+}
+
+bool OSDMonitor::should_propose(double& delay)
+{
+  dout(10) << "should_propose" << dendl;
+
+  // if full map, propose immediately!  any subsequent changes will be clobbered.
+  if (pending_inc.fullmap.length())
+    return true;
+
+  // adjust osd weights?
+  if (!osd_weight.empty() &&
+      osd_weight.size() == (unsigned)osdmap.get_max_osd()) {
+    dout(0) << " adjusting osd weights based on " << osd_weight << dendl;
+    osdmap.adjust_osd_weights(osd_weight, pending_inc);
+    delay = 0.0;
+    osd_weight.clear();
+    return true;
+  }
+
+  return PaxosService::should_propose(delay);
+}
+
+
+
+// ---------------------------
+// READs
+
+
+// ---------------------------
+// UPDATEs
+
+// failure --
+
+bool OSDMonitor::check_source(PaxosServiceMessage *m, uuid_d fsid) {
+  // check permissions
+  MonSession *session = m->get_session();
+  if (!session)
+    return true;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "got MOSDFailure from entity with insufficient caps "
+	    << session->caps << dendl;
+    return true;
+  }
+  if (fsid != mon->monmap->fsid) {
+    dout(0) << "check_source: on fsid " << fsid
+	    << " != " << mon->monmap->fsid << dendl;
+    return true;
+  }
+  return false;
+}
+
+
+bool OSDMonitor::preprocess_failure(MOSDFailure *m)
+{
+  // who is target_osd
+  int badboy = m->get_target().name.num();
+
+  // check permissions
+  if (check_source(m, m->fsid))
+    goto didit;
+
+  // first, verify the reporting host is valid
+  if (m->get_orig_source().is_osd()) {
+    int from = m->get_orig_source().num();
+    if (!osdmap.exists(from) ||
+	osdmap.get_addr(from) != m->get_orig_source_inst().addr ||
+	osdmap.is_down(from)) {
+      dout(5) << "preprocess_failure from dead osd." << from << ", ignoring" << dendl;
+      send_incremental(m, m->get_epoch()+1);
+      goto didit;
+    }
+  }
+  
+
+  // weird?
+  if (!osdmap.have_inst(badboy)) {
+    dout(5) << "preprocess_failure dne(/dup?): " << m->get_target() << ", from " << m->get_orig_source_inst() << dendl;
+    if (m->get_epoch() < osdmap.get_epoch())
+      send_incremental(m, m->get_epoch()+1);
+    goto didit;
+  }
+  if (osdmap.get_inst(badboy) != m->get_target()) {
+    dout(5) << "preprocess_failure wrong osd: report " << m->get_target() << " != map's " << osdmap.get_inst(badboy)
+	    << ", from " << m->get_orig_source_inst() << dendl;
+    if (m->get_epoch() < osdmap.get_epoch())
+      send_incremental(m, m->get_epoch()+1);
+    goto didit;
+  }
+
+  // already reported?
+  if (osdmap.is_down(badboy)) {
+    dout(5) << "preprocess_failure dup: " << m->get_target() << ", from " << m->get_orig_source_inst() << dendl;
+    if (m->get_epoch() < osdmap.get_epoch())
+      send_incremental(m, m->get_epoch()+1);
+    goto didit;
+  }
+
+  if (!can_mark_down(badboy)) {
+    dout(5) << "preprocess_failure ignoring report of " << m->get_target() << " from " << m->get_orig_source_inst() << dendl;
+    goto didit;
+  }
+
+  dout(10) << "preprocess_failure new: " << m->get_target() << ", from " << m->get_orig_source_inst() << dendl;
+  return false;
+
+ didit:
+  m->put();
+  return true;
+}
+
+class C_AckMarkedDown : public Context {
+  OSDMonitor *osdmon;
+  MOSDMarkMeDown *m;
+public:
+  C_AckMarkedDown(
+    OSDMonitor *osdmon,
+    MOSDMarkMeDown *m)
+    : osdmon(osdmon), m(m) {}
+
+  void finish(int) {
+    osdmon->mon->send_reply(
+      m,
+      new MOSDMarkMeDown(
+	m->fsid,
+	m->get_target(),
+	m->get_epoch(),
+	m->ack));
+  }
+  ~C_AckMarkedDown() {
+    m->put();
+  }
+};
+
+bool OSDMonitor::preprocess_mark_me_down(MOSDMarkMeDown *m)
+{
+  int requesting_down = m->get_target().name.num();
+  int from = m->get_orig_source().num();
+
+  // check permissions
+  if (check_source(m, m->fsid))
+    goto reply;
+
+  // first, verify the reporting host is valid
+  if (!m->get_orig_source().is_osd())
+    goto reply;
+
+  if (!osdmap.exists(from) ||
+      osdmap.is_down(from) ||
+      osdmap.get_addr(from) != m->get_target().addr) {
+    dout(5) << "preprocess_mark_me_down from dead osd."
+	    << from << ", ignoring" << dendl;
+    send_incremental(m, m->get_epoch()+1);
+    goto reply;
+  }
+
+  // no down might be set
+  if (!can_mark_down(requesting_down))
+    goto reply;
+
+  dout(10) << "MOSDMarkMeDown for: " << m->get_target() << dendl;
+  return false;
+
+ reply:
+  Context *c(new C_AckMarkedDown(this, m));
+  c->complete(0);
+  return true;
+}
+
+bool OSDMonitor::prepare_mark_me_down(MOSDMarkMeDown *m)
+{
+  int target_osd = m->get_target().name.num();
+
+  assert(osdmap.is_up(target_osd));
+  assert(osdmap.get_addr(target_osd) == m->get_target().addr);
+
+  mon->clog.info() << "osd." << target_osd << " marked itself down\n";
+  pending_inc.new_state[target_osd] = CEPH_OSD_UP;
+  wait_for_finished_proposal(new C_AckMarkedDown(this, m));
+  return true;
+}
+
+bool OSDMonitor::can_mark_down(int i)
+{
+  if (osdmap.test_flag(CEPH_OSDMAP_NODOWN)) {
+    dout(5) << "can_mark_down NODOWN flag set, will not mark osd." << i << " down" << dendl;
+    return false;
+  }
+  int up = osdmap.get_num_up_osds() - pending_inc.get_net_marked_down(&osdmap);
+  float up_ratio = (float)up / (float)osdmap.get_num_osds();
+  if (up_ratio < g_conf->mon_osd_min_up_ratio) {
+    dout(5) << "can_mark_down current up_ratio " << up_ratio << " < min "
+	    << g_conf->mon_osd_min_up_ratio
+	    << ", will not mark osd." << i << " down" << dendl;
+    return false;
+  }
+  return true;
+}
+
+bool OSDMonitor::can_mark_up(int i)
+{
+  if (osdmap.test_flag(CEPH_OSDMAP_NOUP)) {
+    dout(5) << "can_mark_up NOUP flag set, will not mark osd." << i << " up" << dendl;
+    return false;
+  }
+  return true;
+}
+
+/**
+ * @note the parameter @p i apparently only exists here so we can output the
+ *	 osd's id on messages.
+ */
+bool OSDMonitor::can_mark_out(int i)
+{
+  if (osdmap.test_flag(CEPH_OSDMAP_NOOUT)) {
+    dout(5) << "can_mark_out NOOUT flag set, will not mark osds out" << dendl;
+    return false;
+  }
+  int in = osdmap.get_num_in_osds() - pending_inc.get_net_marked_out(&osdmap);
+  float in_ratio = (float)in / (float)osdmap.get_num_osds();
+  if (in_ratio < g_conf->mon_osd_min_in_ratio) {
+    if (i >= 0)
+      dout(5) << "can_mark_down current in_ratio " << in_ratio << " < min "
+	      << g_conf->mon_osd_min_in_ratio
+	      << ", will not mark osd." << i << " out" << dendl;
+    else
+      dout(5) << "can_mark_down current in_ratio " << in_ratio << " < min "
+	      << g_conf->mon_osd_min_in_ratio
+	      << ", will not mark osds out" << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+bool OSDMonitor::can_mark_in(int i)
+{
+  if (osdmap.test_flag(CEPH_OSDMAP_NOIN)) {
+    dout(5) << "can_mark_in NOIN flag set, will not mark osd." << i << " in" << dendl;
+    return false;
+  }
+  return true;
+}
+
+void OSDMonitor::check_failures(utime_t now)
+{
+  for (map::iterator p = failure_info.begin();
+       p != failure_info.end();
+       ++p) {
+    check_failure(now, p->first, p->second);
+  }
+}
+
+bool OSDMonitor::check_failure(utime_t now, int target_osd, failure_info_t& fi)
+{
+  utime_t orig_grace(g_conf->osd_heartbeat_grace, 0);
+  utime_t max_failed_since = fi.get_failed_since();
+  utime_t failed_for = now - max_failed_since;
+
+  utime_t grace = orig_grace;
+  double my_grace = 0, peer_grace = 0;
+  if (g_conf->mon_osd_adjust_heartbeat_grace) {
+    double halflife = (double)g_conf->mon_osd_laggy_halflife;
+    double decay_k = ::log(.5) / halflife;
+
+    // scale grace period based on historical probability of 'lagginess'
+    // (false positive failures due to slowness).
+    const osd_xinfo_t& xi = osdmap.get_xinfo(target_osd);
+    double decay = exp((double)failed_for * decay_k);
+    dout(20) << " halflife " << halflife << " decay_k " << decay_k
+	     << " failed_for " << failed_for << " decay " << decay << dendl;
+    my_grace = decay * (double)xi.laggy_interval * xi.laggy_probability;
+    grace += my_grace;
+
+    // consider the peers reporting a failure a proxy for a potential
+    // 'subcluster' over the overall cluster that is similarly
+    // laggy.  this is clearly not true in all cases, but will sometimes
+    // help us localize the grace correction to a subset of the system
+    // (say, a rack with a bad switch) that is unhappy.
+    assert(fi.reporters.size());
+    for (map::iterator p = fi.reporters.begin();
+	 p != fi.reporters.end();
+	 ++p) {
+      const osd_xinfo_t& xi = osdmap.get_xinfo(p->first);
+      utime_t elapsed = now - xi.down_stamp;
+      double decay = exp((double)elapsed * decay_k);
+      peer_grace += decay * (double)xi.laggy_interval * xi.laggy_probability;
+    }
+    peer_grace /= (double)fi.reporters.size();
+    grace += peer_grace;
+  }
+
+  dout(10) << " osd." << target_osd << " has "
+	   << fi.reporters.size() << " reporters and "
+	   << fi.num_reports << " reports, "
+	   << grace << " grace (" << orig_grace << " + " << my_grace << " + " << peer_grace << "), max_failed_since " << max_failed_since
+	   << dendl;
+
+  // already pending failure?
+  if (pending_inc.new_state.count(target_osd) &&
+      pending_inc.new_state[target_osd] & CEPH_OSD_UP) {
+    dout(10) << " already pending failure" << dendl;
+    return true;
+  }
+
+  if (failed_for >= grace &&
+      ((int)fi.reporters.size() >= g_conf->mon_osd_min_down_reporters) &&
+      (fi.num_reports >= g_conf->mon_osd_min_down_reports)) {
+    dout(1) << " we have enough reports/reporters to mark osd." << target_osd << " down" << dendl;
+    pending_inc.new_state[target_osd] = CEPH_OSD_UP;
+
+    mon->clog.info() << osdmap.get_inst(target_osd) << " failed ("
+		     << fi.num_reports << " reports from " << (int)fi.reporters.size() << " peers after "
+		     << failed_for << " >= grace " << grace << ")\n";
+    return true;
+  }
+  return false;
+}
+
+bool OSDMonitor::prepare_failure(MOSDFailure *m)
+{
+  dout(1) << "prepare_failure " << m->get_target() << " from " << m->get_orig_source_inst()
+          << " is reporting failure:" << m->if_osd_failed() << dendl;
+
+  int target_osd = m->get_target().name.num();
+  int reporter = m->get_orig_source().num();
+  assert(osdmap.is_up(target_osd));
+  assert(osdmap.get_addr(target_osd) == m->get_target().addr);
+
+  // calculate failure time
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t failed_since = m->get_recv_stamp() - utime_t(m->failed_for ? m->failed_for : g_conf->osd_heartbeat_grace, 0);
+  
+  if (m->if_osd_failed()) {
+    // add a report
+    mon->clog.debug() << m->get_target() << " reported failed by "
+		      << m->get_orig_source_inst() << "\n";
+    failure_info_t& fi = failure_info[target_osd];
+    MOSDFailure *old = fi.add_report(reporter, failed_since, m);
+    if (old) {
+      mon->no_reply(old);
+      old->put();
+    }
+
+    return check_failure(now, target_osd, fi);
+  } else {
+    // remove the report
+    mon->clog.debug() << m->get_target() << " failure report canceled by "
+		      << m->get_orig_source_inst() << "\n";
+    if (failure_info.count(target_osd)) {
+      failure_info_t& fi = failure_info[target_osd];
+      list ls;
+      fi.take_report_messages(ls);
+      fi.cancel_report(reporter);
+      while (!ls.empty()) {
+	mon->no_reply(ls.front());
+	ls.front()->put();
+	ls.pop_front();
+      }
+      if (fi.reporters.empty()) {
+	dout(10) << " removing last failure_info for osd." << target_osd << dendl;
+	failure_info.erase(target_osd);
+      } else {
+	dout(10) << " failure_info for osd." << target_osd << " now "
+		 << fi.reporters.size() << " reporters and "
+		 << fi.num_reports << " reports" << dendl;
+      }
+    } else {
+      dout(10) << " no failure_info for osd." << target_osd << dendl;
+    }
+    mon->no_reply(m);
+    m->put();
+  }
+  
+  return false;
+}
+
+void OSDMonitor::process_failures()
+{
+  map::iterator p = failure_info.begin();
+  while (p != failure_info.end()) {
+    if (osdmap.is_up(p->first)) {
+      ++p;
+    } else {
+      dout(10) << "process_failures osd." << p->first << dendl;
+      list ls;
+      p->second.take_report_messages(ls);
+      failure_info.erase(p++);
+
+      while (!ls.empty()) {
+	send_latest(ls.front(), ls.front()->get_epoch());
+	ls.pop_front();
+      }
+    }
+  }
+}
+
+void OSDMonitor::take_all_failures(list& ls)
+{
+  dout(10) << __func__ << " on " << failure_info.size() << " osds" << dendl;
+
+  for (map::iterator p = failure_info.begin();
+       p != failure_info.end();
+       ++p) {
+    p->second.take_report_messages(ls);
+  }
+  failure_info.clear();
+}
+
+
+// boot --
+
+bool OSDMonitor::preprocess_boot(MOSDBoot *m)
+{
+  int from = m->get_orig_source_inst().name.num();
+
+  // check permissions, ignore if failed (no response expected)
+  MonSession *session = m->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "got preprocess_boot message from entity with insufficient caps"
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (m->sb.cluster_fsid != mon->monmap->fsid) {
+    dout(0) << "preprocess_boot on fsid " << m->sb.cluster_fsid
+	    << " != " << mon->monmap->fsid << dendl;
+    goto ignore;
+  }
+
+  if (m->get_orig_source_inst().addr.is_blank_ip()) {
+    dout(0) << "preprocess_boot got blank addr for " << m->get_orig_source_inst() << dendl;
+    goto ignore;
+  }
+
+  assert(m->get_orig_source_inst().name.is_osd());
+
+  // check if osd has required features to boot
+  if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) &
+       CEPH_FEATURE_OSD_ERASURE_CODES) &&
+      !(m->get_connection()->get_features() & CEPH_FEATURE_OSD_ERASURE_CODES)) {
+    dout(0) << __func__ << " osdmap requires Erasure Codes but osd at "
+            << m->get_orig_source_inst()
+            << " doesn't announce support -- ignore" << dendl;
+    goto ignore;
+  }
+  
+  // already booted?
+  if (osdmap.is_up(from) &&
+      osdmap.get_inst(from) == m->get_orig_source_inst()) {
+    // yup.
+    dout(7) << "preprocess_boot dup from " << m->get_orig_source_inst()
+	    << " == " << osdmap.get_inst(from) << dendl;
+    _booted(m, false);
+    return true;
+  }
+
+  if (osdmap.exists(from) &&
+      !osdmap.get_uuid(from).is_zero() &&
+      osdmap.get_uuid(from) != m->sb.osd_fsid) {
+    dout(7) << __func__ << " from " << m->get_orig_source_inst()
+            << " clashes with existing osd: different fsid"
+            << " (ours: " << osdmap.get_uuid(from)
+            << " ; theirs: " << m->sb.osd_fsid << ")" << dendl;
+    goto ignore;
+  }
+
+  if (osdmap.exists(from) &&
+      osdmap.get_info(from).up_from > m->version) {
+    dout(7) << "prepare_boot msg from before last up_from, ignoring" << dendl;
+    send_latest(m, m->sb.current_epoch+1);
+    goto ignore;
+  }
+
+  // noup?
+  if (!can_mark_up(from)) {
+    dout(7) << "preprocess_boot ignoring boot from " << m->get_orig_source_inst() << dendl;
+    send_latest(m, m->sb.current_epoch+1);
+    return true;
+  }
+
+  dout(10) << "preprocess_boot from " << m->get_orig_source_inst() << dendl;
+  return false;
+
+ ignore:
+  m->put();
+  return true;
+}
+
+bool OSDMonitor::prepare_boot(MOSDBoot *m)
+{
+  dout(7) << "prepare_boot from " << m->get_orig_source_inst() << " sb " << m->sb
+	  << " cluster_addr " << m->cluster_addr
+	  << " hb_back_addr " << m->hb_back_addr
+	  << " hb_front_addr " << m->hb_front_addr
+	  << dendl;
+
+  assert(m->get_orig_source().is_osd());
+  int from = m->get_orig_source().num();
+
+  // does this osd exist?
+  if (from >= osdmap.get_max_osd()) {
+    dout(1) << "boot from osd." << from << " >= max_osd " << osdmap.get_max_osd() << dendl;
+    m->put();
+    return false;
+  }
+
+  int oldstate = osdmap.exists(from) ? osdmap.get_state(from) : CEPH_OSD_NEW;
+  if (pending_inc.new_state.count(from))
+    oldstate ^= pending_inc.new_state[from];
+
+  // already up?  mark down first?
+  if (osdmap.is_up(from)) {
+    dout(7) << "prepare_boot was up, first marking down " << osdmap.get_inst(from) << dendl;
+    // preprocess should have caught these;  if not, assert.
+    assert(osdmap.get_inst(from) != m->get_orig_source_inst());
+    assert(osdmap.get_uuid(from) == m->sb.osd_fsid);
+    
+    if (pending_inc.new_state.count(from) == 0 ||
+	(pending_inc.new_state[from] & CEPH_OSD_UP) == 0) {
+      // mark previous guy down
+      pending_inc.new_state[from] = CEPH_OSD_UP;
+    }
+    wait_for_finished_proposal(new C_RetryMessage(this, m));
+  } else if (pending_inc.new_up_client.count(from)) { //FIXME: should this be using new_up_client?
+    // already prepared, just wait
+    dout(7) << "prepare_boot already prepared, waiting on " << m->get_orig_source_addr() << dendl;
+    wait_for_finished_proposal(new C_RetryMessage(this, m));
+  } else {
+    // mark new guy up.
+    pending_inc.new_up_client[from] = m->get_orig_source_addr();
+    if (!m->cluster_addr.is_blank_ip())
+      pending_inc.new_up_cluster[from] = m->cluster_addr;
+    pending_inc.new_hb_back_up[from] = m->hb_back_addr;
+    if (!m->hb_front_addr.is_blank_ip())
+      pending_inc.new_hb_front_up[from] = m->hb_front_addr;
+
+    // mark in?
+    if ((g_conf->mon_osd_auto_mark_auto_out_in && (oldstate & CEPH_OSD_AUTOOUT)) ||
+	(g_conf->mon_osd_auto_mark_new_in && (oldstate & CEPH_OSD_NEW)) ||
+	(g_conf->mon_osd_auto_mark_in)) {
+      if (can_mark_in(from)) {
+	pending_inc.new_weight[from] = CEPH_OSD_IN;
+      } else {
+	dout(7) << "prepare_boot NOIN set, will not mark in " << m->get_orig_source_addr() << dendl;
+      }
+    }
+
+    down_pending_out.erase(from);  // if any
+
+    if (m->sb.weight)
+      osd_weight[from] = m->sb.weight;
+
+    // set uuid?
+    dout(10) << " setting osd." << from << " uuid to " << m->sb.osd_fsid << dendl;
+    if (!osdmap.exists(from) || osdmap.get_uuid(from) != m->sb.osd_fsid) {
+      // preprocess should have caught this;  if not, assert.
+      assert(!osdmap.exists(from) || osdmap.get_uuid(from).is_zero());
+      pending_inc.new_uuid[from] = m->sb.osd_fsid;
+    }
+
+    // fresh osd?
+    if (m->sb.newest_map == 0 && osdmap.exists(from)) {
+      const osd_info_t& i = osdmap.get_info(from);
+      if (i.up_from > i.lost_at) {
+	dout(10) << " fresh osd; marking lost_at too" << dendl;
+	pending_inc.new_lost[from] = osdmap.get_epoch();
+      }
+    }
+
+    // metadata
+    bufferlist osd_metadata;
+    ::encode(m->metadata, osd_metadata);
+    pending_metadata[from] = osd_metadata;
+
+    // adjust last clean unmount epoch?
+    const osd_info_t& info = osdmap.get_info(from);
+    dout(10) << " old osd_info: " << info << dendl;
+    if (m->sb.mounted > info.last_clean_begin ||
+	(m->sb.mounted == info.last_clean_begin &&
+	 m->sb.clean_thru > info.last_clean_end)) {
+      epoch_t begin = m->sb.mounted;
+      epoch_t end = m->sb.clean_thru;
+
+      dout(10) << "prepare_boot osd." << from << " last_clean_interval "
+	       << "[" << info.last_clean_begin << "," << info.last_clean_end << ")"
+	       << " -> [" << begin << "-" << end << ")"
+	       << dendl;
+      pending_inc.new_last_clean_interval[from] = pair(begin, end);
+    }
+
+    osd_xinfo_t xi = osdmap.get_xinfo(from);
+    if (m->boot_epoch == 0) {
+      xi.laggy_probability *= (1.0 - g_conf->mon_osd_laggy_weight);
+      xi.laggy_interval *= (1.0 - g_conf->mon_osd_laggy_weight);
+      dout(10) << " not laggy, new xi " << xi << dendl;
+    } else {
+      if (xi.down_stamp.sec()) {
+	int interval = ceph_clock_now(g_ceph_context).sec() - xi.down_stamp.sec();
+	xi.laggy_interval =
+	  interval * g_conf->mon_osd_laggy_weight +
+	  xi.laggy_interval * (1.0 - g_conf->mon_osd_laggy_weight);
+      }
+      xi.laggy_probability =
+	g_conf->mon_osd_laggy_weight +
+	xi.laggy_probability * (1.0 - g_conf->mon_osd_laggy_weight);
+      dout(10) << " laggy, now xi " << xi << dendl;
+    }
+    // set features shared by the osd
+    xi.features = m->get_connection()->get_features();
+    pending_inc.new_xinfo[from] = xi;
+
+    // wait
+    wait_for_finished_proposal(new C_Booted(this, m));
+  }
+  return true;
+}
+
+void OSDMonitor::_booted(MOSDBoot *m, bool logit)
+{
+  dout(7) << "_booted " << m->get_orig_source_inst() 
+	  << " w " << m->sb.weight << " from " << m->sb.current_epoch << dendl;
+
+  if (logit) {
+    mon->clog.info() << m->get_orig_source_inst() << " boot\n";
+  }
+
+  send_latest(m, m->sb.current_epoch+1);
+}
+
+
+// -------------
+// alive
+
+bool OSDMonitor::preprocess_alive(MOSDAlive *m)
+{
+  int from = m->get_orig_source().num();
+
+  // check permissions, ignore if failed
+  MonSession *session = m->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "attempt to send MOSDAlive from entity with insufficient privileges:"
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (!osdmap.is_up(from) ||
+      osdmap.get_inst(from) != m->get_orig_source_inst()) {
+    dout(7) << "preprocess_alive ignoring alive message from down " << m->get_orig_source_inst() << dendl;
+    goto ignore;
+  }
+
+  if (osdmap.get_up_thru(from) >= m->want) {
+    // yup.
+    dout(7) << "preprocess_alive want up_thru " << m->want << " dup from " << m->get_orig_source_inst() << dendl;
+    _reply_map(m, m->version);
+    return true;
+  }
+
+  dout(10) << "preprocess_alive want up_thru " << m->want
+	   << " from " << m->get_orig_source_inst() << dendl;
+  return false;
+
+ ignore:
+  m->put();
+  return true;
+}
+
+bool OSDMonitor::prepare_alive(MOSDAlive *m)
+{
+  int from = m->get_orig_source().num();
+
+  if (0) {  // we probably don't care much about these
+    mon->clog.debug() << m->get_orig_source_inst() << " alive\n";
+  }
+
+  dout(7) << "prepare_alive want up_thru " << m->want << " have " << m->version
+	  << " from " << m->get_orig_source_inst() << dendl;
+  pending_inc.new_up_thru[from] = m->version;  // set to the latest map the OSD has
+  wait_for_finished_proposal(new C_ReplyMap(this, m, m->version));
+  return true;
+}
+
+void OSDMonitor::_reply_map(PaxosServiceMessage *m, epoch_t e)
+{
+  dout(7) << "_reply_map " << e
+	  << " from " << m->get_orig_source_inst()
+	  << dendl;
+  send_latest(m, e);
+}
+
+// -------------
+// pg_temp changes
+
+bool OSDMonitor::preprocess_pgtemp(MOSDPGTemp *m)
+{
+  dout(10) << "preprocess_pgtemp " << *m << dendl;
+  vector empty;
+  int from = m->get_orig_source().num();
+  size_t ignore_cnt = 0;
+
+  // check caps
+  MonSession *session = m->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "attempt to send MOSDPGTemp from entity with insufficient caps "
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (!osdmap.is_up(from) ||
+      osdmap.get_inst(from) != m->get_orig_source_inst()) {
+    dout(7) << "ignoring pgtemp message from down " << m->get_orig_source_inst() << dendl;
+    goto ignore;
+  }
+
+  for (map >::iterator p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p) {
+    dout(20) << " " << p->first
+	     << (osdmap.pg_temp->count(p->first) ? (*osdmap.pg_temp)[p->first] : empty)
+             << " -> " << p->second << dendl;
+
+    // does the pool exist?
+    if (!osdmap.have_pg_pool(p->first.pool())) {
+      /*
+       * 1. If the osdmap does not have the pool, it means the pool has been
+       *    removed in-between the osd sending this message and us handling it.
+       * 2. If osdmap doesn't have the pool, it is safe to assume the pool does
+       *    not exist in the pending either, as the osds would not send a
+       *    message about a pool they know nothing about (yet).
+       * 3. However, if the pool does exist in the pending, then it must be a
+       *    new pool, and not relevant to this message (see 1).
+       */
+      dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+               << ": pool has been removed" << dendl;
+      ignore_cnt++;
+      continue;
+    }
+
+    // removal?
+    if (p->second.empty() && (osdmap.pg_temp->count(p->first) ||
+			      osdmap.primary_temp->count(p->first)))
+      return false;
+    // change?
+    //  NOTE: we assume that this will clear pg_primary, so consider
+    //        an existing pg_primary field to imply a change
+    if (p->second.size() && (osdmap.pg_temp->count(p->first) == 0 ||
+			     (*osdmap.pg_temp)[p->first] != p->second ||
+			     osdmap.primary_temp->count(p->first)))
+      return false;
+  }
+
+  // should we ignore all the pgs?
+  if (ignore_cnt == m->pg_temp.size())
+    goto ignore;
+
+  dout(7) << "preprocess_pgtemp e" << m->map_epoch << " no changes from " << m->get_orig_source_inst() << dendl;
+  _reply_map(m, m->map_epoch);
+  return true;
+
+ ignore:
+  m->put();
+  return true;
+}
+
+bool OSDMonitor::prepare_pgtemp(MOSDPGTemp *m)
+{
+  int from = m->get_orig_source().num();
+  dout(7) << "prepare_pgtemp e" << m->map_epoch << " from " << m->get_orig_source_inst() << dendl;
+  for (map >::iterator p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p) {
+    uint64_t pool = p->first.pool();
+    if (pending_inc.old_pools.count(pool)) {
+      dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+               << ": pool pending removal" << dendl;
+      continue;
+    }
+    if (!osdmap.have_pg_pool(pool)) {
+      dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+               << ": pool has been removed" << dendl;
+      continue;
+    }
+    pending_inc.new_pg_temp[p->first] = p->second;
+
+    // unconditionally clear pg_primary (until this message can encode
+    // a change for that, too.. at which point we need to also fix
+    // preprocess_pg_temp)
+    if (osdmap.primary_temp->count(p->first) ||
+	pending_inc.new_primary_temp.count(p->first))
+      pending_inc.new_primary_temp[p->first] = -1;
+  }
+  pending_inc.new_up_thru[from] = m->map_epoch;   // set up_thru too, so the osd doesn't have to ask again
+  wait_for_finished_proposal(new C_ReplyMap(this, m, m->map_epoch));
+  return true;
+}
+
+
+// ---
+
+bool OSDMonitor::preprocess_remove_snaps(MRemoveSnaps *m)
+{
+  dout(7) << "preprocess_remove_snaps " << *m << dendl;
+
+  // check privilege, ignore if failed
+  MonSession *session = m->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_R | MON_CAP_W)) {
+    dout(0) << "got preprocess_remove_snaps from entity with insufficient caps "
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  for (map >::iterator q = m->snaps.begin();
+       q != m->snaps.end();
+       ++q) {
+    if (!osdmap.have_pg_pool(q->first)) {
+      dout(10) << " ignoring removed_snaps " << q->second << " on non-existent pool " << q->first << dendl;
+      continue;
+    }
+    const pg_pool_t *pi = osdmap.get_pg_pool(q->first);
+    for (vector::iterator p = q->second.begin(); 
+	 p != q->second.end();
+	 ++p) {
+      if (*p > pi->get_snap_seq() ||
+	  !pi->removed_snaps.contains(*p))
+	return false;
+    }
+  }
+
+ ignore:
+  m->put();
+  return true;
+}
+
+bool OSDMonitor::prepare_remove_snaps(MRemoveSnaps *m)
+{
+  dout(7) << "prepare_remove_snaps " << *m << dendl;
+
+  for (map >::iterator p = m->snaps.begin(); 
+       p != m->snaps.end();
+       ++p) {
+    pg_pool_t& pi = osdmap.pools[p->first];
+    for (vector::iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      if (!pi.removed_snaps.contains(*q) &&
+	  (!pending_inc.new_pools.count(p->first) ||
+	   !pending_inc.new_pools[p->first].removed_snaps.contains(*q))) {
+	pg_pool_t *newpi = pending_inc.get_new_pool(p->first, &pi);
+	newpi->removed_snaps.insert(*q);
+	dout(10) << " pool " << p->first << " removed_snaps added " << *q
+		 << " (now " << newpi->removed_snaps << ")" << dendl;
+	if (*q > newpi->get_snap_seq()) {
+	  dout(10) << " pool " << p->first << " snap_seq " << newpi->get_snap_seq() << " -> " << *q << dendl;
+	  newpi->set_snap_seq(*q);
+	}
+	newpi->set_snap_epoch(pending_inc.epoch);
+      }
+    }
+  }
+
+  m->put();
+  return true;
+}
+
+
+// ---------------
+// map helpers
+
+void OSDMonitor::send_latest(PaxosServiceMessage *m, epoch_t start)
+{
+  dout(5) << "send_latest to " << m->get_orig_source_inst()
+	  << " start " << start << dendl;
+  if (start == 0)
+    send_full(m);
+  else
+    send_incremental(m, start);
+  m->put();
+}
+
+
+MOSDMap *OSDMonitor::build_latest_full()
+{
+  MOSDMap *r = new MOSDMap(mon->monmap->fsid, &osdmap);
+  r->oldest_map = get_first_committed();
+  r->newest_map = osdmap.get_epoch();
+  return r;
+}
+
+MOSDMap *OSDMonitor::build_incremental(epoch_t from, epoch_t to)
+{
+  dout(10) << "build_incremental [" << from << ".." << to << "]" << dendl;
+  MOSDMap *m = new MOSDMap(mon->monmap->fsid);
+  m->oldest_map = get_first_committed();
+  m->newest_map = osdmap.get_epoch();
+
+  for (epoch_t e = to; e >= from && e > 0; e--) {
+    bufferlist bl;
+    int err = get_version(e, bl);
+    if (err == 0) {
+      assert(bl.length());
+      // if (get_version(e, bl) > 0) {
+      dout(20) << "build_incremental    inc " << e << " "
+	       << bl.length() << " bytes" << dendl;
+      m->incremental_maps[e] = bl;
+    } else {
+      assert(err == -ENOENT);
+      assert(!bl.length());
+      get_version_full(e, bl);
+      if (bl.length() > 0) {
+      //else if (get_version("full", e, bl) > 0) {
+      dout(20) << "build_incremental   full " << e << " "
+	       << bl.length() << " bytes" << dendl;
+      m->maps[e] = bl;
+      } else {
+	assert(0);  // we should have all maps.
+      }
+    }
+  }
+  return m;
+}
+
+void OSDMonitor::send_full(PaxosServiceMessage *m)
+{
+  dout(5) << "send_full to " << m->get_orig_source_inst() << dendl;
+  mon->send_reply(m, build_latest_full());
+}
+
+/* TBH, I'm fairly certain these two functions could somehow be using a single
+ * helper function to do the heavy lifting. As this is not our main focus right
+ * now, I'm leaving it to the next near-future iteration over the services'
+ * code. We should not forget it though.
+ *
+ * TODO: create a helper function and get rid of the duplicated code.
+ */
+void OSDMonitor::send_incremental(PaxosServiceMessage *req, epoch_t first)
+{
+  dout(5) << "send_incremental [" << first << ".." << osdmap.get_epoch() << "]"
+	  << " to " << req->get_orig_source_inst()
+	  << dendl;
+
+  int osd = -1;
+  if (req->get_source().is_osd()) {
+    osd = req->get_source().num();
+    map::iterator p = osd_epoch.find(osd);
+    if (p != osd_epoch.end()) {
+      dout(10) << " osd." << osd << " should have epoch " << p->second << dendl;
+      first = p->second + 1;
+      if (first > osdmap.get_epoch())
+	return;
+    }
+  }
+
+  if (first < get_first_committed()) {
+    first = get_first_committed();
+    bufferlist bl;
+    int err = get_version_full(first, bl);
+    assert(err == 0);
+    assert(bl.length());
+
+    dout(20) << "send_incremental starting with base full "
+	     << first << " " << bl.length() << " bytes" << dendl;
+
+    MOSDMap *m = new MOSDMap(osdmap.get_fsid());
+    m->oldest_map = first;
+    m->newest_map = osdmap.get_epoch();
+    m->maps[first] = bl;
+    mon->send_reply(req, m);
+
+    if (osd >= 0)
+      osd_epoch[osd] = osdmap.get_epoch();
+    return;
+  }
+
+  // send some maps.  it may not be all of them, but it will get them
+  // started.
+  epoch_t last = MIN(first + g_conf->osd_map_message_max, osdmap.get_epoch());
+  MOSDMap *m = build_incremental(first, last);
+  m->oldest_map = get_first_committed();
+  m->newest_map = osdmap.get_epoch();
+  mon->send_reply(req, m);
+
+  if (osd >= 0)
+    osd_epoch[osd] = last;
+}
+
+void OSDMonitor::send_incremental(epoch_t first, entity_inst_t& dest, bool onetime)
+{
+  dout(5) << "send_incremental [" << first << ".." << osdmap.get_epoch() << "]"
+	  << " to " << dest << dendl;
+
+  if (first < get_first_committed()) {
+    first = get_first_committed();
+    bufferlist bl;
+    int err = get_version_full(first, bl);
+    assert(err == 0);
+    assert(bl.length());
+
+    dout(20) << "send_incremental starting with base full "
+	     << first << " " << bl.length() << " bytes" << dendl;
+
+    MOSDMap *m = new MOSDMap(osdmap.get_fsid());
+    m->oldest_map = first;
+    m->newest_map = osdmap.get_epoch();
+    m->maps[first] = bl;
+    mon->messenger->send_message(m, dest);
+    first++;
+  }
+
+  while (first <= osdmap.get_epoch()) {
+    epoch_t last = MIN(first + g_conf->osd_map_message_max, osdmap.get_epoch());
+    MOSDMap *m = build_incremental(first, last);
+    mon->messenger->send_message(m, dest);
+    first = last + 1;
+    if (onetime)
+      break;
+  }
+}
+
+
+
+
+epoch_t OSDMonitor::blacklist(const entity_addr_t& a, utime_t until)
+{
+  dout(10) << "blacklist " << a << " until " << until << dendl;
+  pending_inc.new_blacklist[a] = until;
+  return pending_inc.epoch;
+}
+
+
+void OSDMonitor::check_subs()
+{
+  dout(10) << __func__ << dendl;
+  string type = "osdmap";
+  if (mon->session_map.subs.count(type) == 0)
+    return;
+  xlist::iterator p = mon->session_map.subs[type]->begin();
+  while (!p.end()) {
+    Subscription *sub = *p;
+    ++p;
+    check_sub(sub);
+  }
+}
+
+void OSDMonitor::check_sub(Subscription *sub)
+{
+  dout(10) << __func__ << " " << sub << " next " << sub->next
+	   << (sub->onetime ? " (onetime)":" (ongoing)") << dendl;
+  if (sub->next <= osdmap.get_epoch()) {
+    if (sub->next >= 1)
+      send_incremental(sub->next, sub->session->inst, sub->incremental_onetime);
+    else
+      mon->messenger->send_message(build_latest_full(),
+				   sub->session->inst);
+    if (sub->onetime)
+      mon->session_map.remove_sub(sub);
+    else
+      sub->next = osdmap.get_epoch() + 1;
+  }
+}
+
+// TICK
+
+
+void OSDMonitor::tick()
+{
+  if (!is_active()) return;
+
+  dout(10) << osdmap << dendl;
+
+  if (!mon->is_leader()) return;
+
+  bool do_propose = false;
+  utime_t now = ceph_clock_now(g_ceph_context);
+
+  // mark osds down?
+  check_failures(now);
+
+  // mark down osds out?
+
+  /* can_mark_out() checks if we can mark osds as being out. The -1 has no
+   * influence at all. The decision is made based on the ratio of "in" osds,
+   * and the function returns false if this ratio is lower that the minimum
+   * ratio set by g_conf->mon_osd_min_in_ratio. So it's not really up to us.
+   */
+  if (can_mark_out(-1)) {
+    set down_cache;  // quick cache of down subtrees
+
+    map::iterator i = down_pending_out.begin();
+    while (i != down_pending_out.end()) {
+      int o = i->first;
+      utime_t down = now;
+      down -= i->second;
+      ++i;
+
+      if (osdmap.is_down(o) &&
+	  osdmap.is_in(o) &&
+	  can_mark_out(o)) {
+	utime_t orig_grace(g_conf->mon_osd_down_out_interval, 0);
+	utime_t grace = orig_grace;
+	double my_grace = 0.0;
+
+	if (g_conf->mon_osd_adjust_down_out_interval) {
+	  // scale grace period the same way we do the heartbeat grace.
+	  const osd_xinfo_t& xi = osdmap.get_xinfo(o);
+	  double halflife = (double)g_conf->mon_osd_laggy_halflife;
+	  double decay_k = ::log(.5) / halflife;
+	  double decay = exp((double)down * decay_k);
+	  dout(20) << "osd." << o << " laggy halflife " << halflife << " decay_k " << decay_k
+		   << " down for " << down << " decay " << decay << dendl;
+	  my_grace = decay * (double)xi.laggy_interval * xi.laggy_probability;
+	  grace += my_grace;
+	}
+
+	// is this an entire large subtree down?
+	if (g_conf->mon_osd_down_out_subtree_limit.length()) {
+	  int type = osdmap.crush->get_type_id(g_conf->mon_osd_down_out_subtree_limit);
+	  if (type > 0) {
+	    if (osdmap.containing_subtree_is_down(g_ceph_context, o, type, &down_cache)) {
+	      dout(10) << "tick entire containing " << g_conf->mon_osd_down_out_subtree_limit
+		       << " subtree for osd." << o << " is down; resetting timer" << dendl;
+	      // reset timer, too.
+	      down_pending_out[o] = now;
+	      continue;
+	    }
+	  }
+	}
+
+	if (g_conf->mon_osd_down_out_interval > 0 &&
+	    down.sec() >= grace) {
+	  dout(10) << "tick marking osd." << o << " OUT after " << down
+		   << " sec (target " << grace << " = " << orig_grace << " + " << my_grace << ")" << dendl;
+	  pending_inc.new_weight[o] = CEPH_OSD_OUT;
+
+	  // set the AUTOOUT bit.
+	  if (pending_inc.new_state.count(o) == 0)
+	    pending_inc.new_state[o] = 0;
+	  pending_inc.new_state[o] |= CEPH_OSD_AUTOOUT;
+
+	  do_propose = true;
+	
+	  mon->clog.info() << "osd." << o << " out (down for " << down << ")\n";
+	} else
+	  continue;
+      }
+
+      down_pending_out.erase(o);
+    }
+  } else {
+    dout(10) << "tick NOOUT flag set, not checking down osds" << dendl;
+  }
+
+  // expire blacklisted items?
+  for (ceph::unordered_map::iterator p = osdmap.blacklist.begin();
+       p != osdmap.blacklist.end();
+       ++p) {
+    if (p->second < now) {
+      dout(10) << "expiring blacklist item " << p->first << " expired " << p->second << " < now " << now << dendl;
+      pending_inc.old_blacklist.push_back(p->first);
+      do_propose = true;
+    }
+  }
+
+  //if map full setting has changed, get that info out there!
+  if (mon->pgmon()->is_readable()) {
+    if (!mon->pgmon()->pg_map.full_osds.empty()) {
+      dout(5) << "There are full osds, setting full flag" << dendl;
+      add_flag(CEPH_OSDMAP_FULL);
+    } else if (osdmap.test_flag(CEPH_OSDMAP_FULL)){
+      dout(10) << "No full osds, removing full flag" << dendl;
+      remove_flag(CEPH_OSDMAP_FULL);
+    }
+    if (pending_inc.new_flags != -1 &&
+	(pending_inc.new_flags ^ osdmap.flags) & CEPH_OSDMAP_FULL) {
+      dout(1) << "New setting for CEPH_OSDMAP_FULL -- doing propose" << dendl;
+      do_propose = true;
+    }
+  }
+  // ---------------
+#define SWAP_PRIMARIES_AT_START 0
+#define SWAP_TIME 1
+#if 0
+  if (SWAP_PRIMARIES_AT_START) {
+    // For all PGs that have OSD 0 as the primary,
+    // switch them to use the first replca
+    ps_t numps = osdmap.get_pg_num();
+    for (int64_t pool=0; pool<1; pool++)
+      for (ps_t ps = 0; ps < numps; ++ps) {
+	pg_t pgid = pg_t(pg_t::TYPE_REPLICATED, ps, pool, -1);
+	vector osds;
+	osdmap.pg_to_osds(pgid, osds); 
+	if (osds[0] == 0) {
+	  pending_inc.new_pg_swap_primary[pgid] = osds[1];
+	  dout(3) << "Changing primary for PG " << pgid << " from " << osds[0] << " to "
+		  << osds[1] << dendl;
+	  do_propose = true;
+	}
+      }
+  }
+#endif
+  // ---------------
+
+  if (update_pools_status())
+    do_propose = true;
+
+  if (do_propose ||
+      !pending_inc.new_pg_temp.empty())  // also propose if we adjusted pg_temp
+    propose_pending();
+}
+
+void OSDMonitor::handle_osd_timeouts(const utime_t &now,
+				     std::map &last_osd_report)
+{
+  utime_t timeo(g_conf->mon_osd_report_timeout, 0);
+  int max_osd = osdmap.get_max_osd();
+  bool new_down = false;
+
+  for (int i=0; i < max_osd; ++i) {
+    dout(30) << "handle_osd_timeouts: checking up on osd " << i << dendl;
+    if (!osdmap.exists(i))
+      continue;
+    if (!osdmap.is_up(i))
+      continue;
+    const std::map::const_iterator t = last_osd_report.find(i);
+    if (t == last_osd_report.end()) {
+      // it wasn't in the map; start the timer.
+      last_osd_report[i] = now;
+    } else if (can_mark_down(i)) {
+      utime_t diff = now - t->second;
+      if (diff > timeo) {
+	mon->clog.info() << "osd." << i << " marked down after no pg stats for " << diff << "seconds\n";
+	derr << "no osd or pg stats from osd." << i << " since " << t->second << ", " << diff
+	     << " seconds ago.  marking down" << dendl;
+	pending_inc.new_state[i] = CEPH_OSD_UP;
+	new_down = true;
+      }
+    }
+  }
+  if (new_down) {
+    propose_pending();
+  }
+}
+
+void OSDMonitor::mark_all_down()
+{
+  assert(mon->is_leader());
+
+  dout(7) << "mark_all_down" << dendl;
+
+  set ls;
+  osdmap.get_all_osds(ls);
+  for (set::iterator it = ls.begin();
+       it != ls.end();
+       ++it) {
+    if (osdmap.is_down(*it)) continue;
+    pending_inc.new_state[*it] = CEPH_OSD_UP;
+  }
+
+  propose_pending();
+}
+
+void OSDMonitor::get_health(list >& summary,
+			    list > *detail) const
+{
+  int num_osds = osdmap.get_num_osds();
+
+  if (num_osds == 0) {
+    summary.push_back(make_pair(HEALTH_ERR, "no osds"));
+  } else {
+    int num_in_osds = 0;
+    int num_down_in_osds = 0;
+    for (int i = 0; i < osdmap.get_max_osd(); i++) {
+      if (!osdmap.exists(i) || osdmap.is_out(i))
+	continue;
+      ++num_in_osds;
+      if (!osdmap.is_up(i)) {
+	++num_down_in_osds;
+	if (detail) {
+	  const osd_info_t& info = osdmap.get_info(i);
+	  ostringstream ss;
+	  ss << "osd." << i << " is down since epoch " << info.down_at
+	     << ", last address " << osdmap.get_addr(i);
+	  detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+	}
+      }
+    }
+    assert(num_down_in_osds <= num_in_osds);
+    if (num_down_in_osds > 0) {
+      ostringstream ss;
+      ss << num_down_in_osds << "/" << num_in_osds << " in osds are down";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+
+    // warn about flags
+    if (osdmap.test_flag(CEPH_OSDMAP_PAUSERD |
+			 CEPH_OSDMAP_PAUSEWR |
+			 CEPH_OSDMAP_NOUP |
+			 CEPH_OSDMAP_NODOWN |
+			 CEPH_OSDMAP_NOIN |
+			 CEPH_OSDMAP_NOOUT |
+			 CEPH_OSDMAP_NOBACKFILL |
+			 CEPH_OSDMAP_NORECOVER |
+			 CEPH_OSDMAP_NOSCRUB |
+			 CEPH_OSDMAP_NODEEP_SCRUB |
+			 CEPH_OSDMAP_NOTIERAGENT)) {
+      ostringstream ss;
+      ss << osdmap.get_flag_string() << " flag(s) set";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+      if (detail)
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+
+    // old crush tunables?
+    if (g_conf->mon_warn_on_legacy_crush_tunables) {
+      if (osdmap.crush->has_legacy_tunables()) {
+	ostringstream ss;
+	ss << "crush map has legacy tunables";
+	summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+	if (detail) {
+	  ss << "; see http://ceph.com/docs/master/rados/operations/crush-map/#tunables";
+	  detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+	}
+      }
+    }
+
+    // hit_set-less cache_mode?
+    if (g_conf->mon_warn_on_cache_pools_without_hit_sets) {
+      int problem_cache_pools = 0;
+      for (map::const_iterator p = osdmap.pools.begin();
+	   p != osdmap.pools.end();
+	   ++p) {
+	const pg_pool_t& info = p->second;
+	if (info.cache_mode_requires_hit_set() &&
+	    info.hit_set_params.get_type() == HitSet::TYPE_NONE) {
+	  ++problem_cache_pools;
+	  if (detail) {
+	    ostringstream ss;
+	    ss << "pool '" << osdmap.get_pool_name(p->first)
+	       << "' with cache_mode " << info.get_cache_mode_name()
+	       << " needs hit_set_type to be set but it is not";
+	    detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+	  }
+	}
+      }
+      if (problem_cache_pools) {
+	ostringstream ss;
+	ss << problem_cache_pools << " cache pools are missing hit_sets";
+	summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+
+    // Warn if 'mon_osd_down_out_interval' is set to zero.
+    // Having this option set to zero on the leader acts much like the
+    // 'noout' flag.  It's hard to figure out what's going wrong with clusters
+    // without the 'noout' flag set but acting like that just the same, so
+    // we report a HEALTH_WARN in case this option is set to zero.
+    // This is an ugly hack to get the warning out, but until we find a way
+    // to spread global options throughout the mon cluster and have all mons
+    // using a base set of the same options, we need to work around this sort
+    // of things.
+    // There's also the obvious drawback that if this is set on a single
+    // monitor on a 3-monitor cluster, this warning will only be shown every
+    // third monitor connection.
+    if (g_conf->mon_warn_on_osd_down_out_interval_zero &&
+        g_conf->mon_osd_down_out_interval == 0) {
+      ostringstream ss;
+      ss << "mon." << mon->name << " has mon_osd_down_out_interval set to 0";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+      if (detail) {
+        ss << "; this has the same effect as the 'noout' flag";
+        detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+
+    get_pools_health(summary, detail);
+  }
+}
+
+void OSDMonitor::dump_info(Formatter *f)
+{
+  f->open_object_section("osdmap");
+  osdmap.dump(f);
+  f->close_section();
+
+  f->open_array_section("osd_metadata");
+  for (int i=0; iopen_object_section("osd");
+      f->dump_unsigned("id", i);
+      dump_osd_metadata(i, f, NULL);
+      f->close_section();
+    }
+  }
+  f->close_section();
+
+  f->dump_unsigned("osdmap_first_committed", get_first_committed());
+  f->dump_unsigned("osdmap_last_committed", get_last_committed());
+
+  f->open_object_section("crushmap");
+  osdmap.crush->dump(f);
+  f->close_section();
+}
+
+bool OSDMonitor::preprocess_command(MMonCommand *m)
+{
+  int r = 0;
+  bufferlist rdata;
+  stringstream ss, ds;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr f(new_formatter(format));
+
+  if (prefix == "osd stat") {
+    osdmap.print_summary(f.get(), ds);
+    if (f)
+      f->flush(rdata);
+    else
+      rdata.append(ds);
+  }
+  else if (prefix == "osd dump" ||
+	   prefix == "osd tree" ||
+	   prefix == "osd ls" ||
+	   prefix == "osd getmap" ||
+	   prefix == "osd getcrushmap" ||
+	   prefix == "osd perf") {
+    string val;
+
+    epoch_t epoch = 0;
+    int64_t epochnum;
+    cmd_getval(g_ceph_context, cmdmap, "epoch", epochnum, (int64_t)0);
+    epoch = epochnum;
+
+    OSDMap *p = &osdmap;
+    if (epoch) {
+      bufferlist b;
+      int err = get_version_full(epoch, b);
+      if (err == -ENOENT) {
+	r = -ENOENT;
+        ss << "there is no map for epoch " << epoch;
+	goto reply;
+      }
+      assert(err == 0);
+      assert(b.length());
+      p = new OSDMap;
+      p->decode(b);
+    }
+    if (prefix == "osd dump") {
+      stringstream ds;
+      if (f) {
+	f->open_object_section("osdmap");
+	p->dump(f.get());
+	f->close_section();
+	f->flush(ds);
+      } else {
+	p->print(ds);
+      } 
+      rdata.append(ds);
+      if (!f)
+	ds << " ";
+    } else if (prefix == "osd ls") {
+      if (f) {
+	f->open_array_section("osds");
+	for (int i = 0; i < osdmap.get_max_osd(); i++) {
+	  if (osdmap.exists(i)) {
+	    f->dump_int("osd", i);
+	  }
+	}
+	f->close_section();
+	f->flush(ds);
+      } else {
+	bool first = true;
+	for (int i = 0; i < osdmap.get_max_osd(); i++) {
+	  if (osdmap.exists(i)) {
+	    if (!first)
+	      ds << "\n";
+	    first = false;
+	    ds << i;
+	  }
+	}
+      } 
+      rdata.append(ds);
+    } else if (prefix == "osd tree") {
+      if (f) {
+	f->open_object_section("tree");
+	p->print_tree(NULL, f.get());
+	f->close_section();
+	f->flush(ds);
+      } else {
+	p->print_tree(&ds, NULL);
+      } 
+      rdata.append(ds);
+    } else if (prefix == "osd getmap") {
+      p->encode(rdata, m->get_connection()->get_features());
+      ss << "got osdmap epoch " << p->get_epoch();
+    } else if (prefix == "osd getcrushmap") {
+      p->crush->encode(rdata);
+      ss << "got crush map from osdmap epoch " << p->get_epoch();
+    } else if (prefix == "osd perf") {
+      const PGMap &pgm = mon->pgmon()->pg_map;
+      if (f) {
+	f->open_object_section("osdstats");
+	pgm.dump_osd_perf_stats(f.get());
+	f->close_section();
+	f->flush(ds);
+      } else {
+	pgm.print_osd_perf_stats(&ds);
+      }
+      rdata.append(ds);
+    }
+    if (p != &osdmap)
+      delete p;
+  } else if (prefix == "osd getmaxosd") {
+    if (f) {
+      f->open_object_section("getmaxosd");
+      f->dump_int("epoch", osdmap.get_epoch());
+      f->dump_int("max_osd", osdmap.get_max_osd());
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ds << "max_osd = " << osdmap.get_max_osd() << " in epoch " << osdmap.get_epoch();
+      rdata.append(ds);
+    }
+  } else if (prefix  == "osd find") {
+    int64_t osd;
+    if (!cmd_getval(g_ceph_context, cmdmap, "id", osd)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.exists(osd)) {
+      ss << "osd." << osd << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    string format;
+    cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+    boost::scoped_ptr f(new_formatter(format));
+
+    f->open_object_section("osd_location");
+    f->dump_int("osd", osd);
+    f->dump_stream("ip") << osdmap.get_addr(osd);
+    f->open_object_section("crush_location");
+    map loc = osdmap.crush->get_full_location(osd);
+    for (map::iterator p = loc.begin(); p != loc.end(); ++p)
+      f->dump_string(p->first.c_str(), p->second);
+    f->close_section();
+    f->close_section();
+    f->flush(rdata);
+  } else if (prefix == "osd metadata") {
+    int64_t osd;
+    if (!cmd_getval(g_ceph_context, cmdmap, "id", osd)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.exists(osd)) {
+      ss << "osd." << osd << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    string format;
+    cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+    boost::scoped_ptr f(new_formatter(format));
+    f->open_object_section("osd_metadata");
+    r = dump_osd_metadata(osd, f.get(), &ss);
+    if (r < 0)
+      goto reply;
+    f->close_section();
+    f->flush(rdata);
+  } else if (prefix == "osd map") {
+    string poolstr, objstr, namespacestr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    cmd_getval(g_ceph_context, cmdmap, "object", objstr);
+    cmd_getval(g_ceph_context, cmdmap, "nspace", namespacestr);
+
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "pool " << poolstr << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    object_locator_t oloc(pool, namespacestr);
+    object_t oid(objstr);
+    pg_t pgid = osdmap.object_locator_to_pg(oid, oloc);
+    pg_t mpgid = osdmap.raw_pg_to_pg(pgid);
+    vector up, acting;
+    int up_p, acting_p;
+    osdmap.pg_to_up_acting_osds(mpgid, &up, &up_p, &acting, &acting_p);
+
+    string fullobjname;
+    if (!namespacestr.empty())
+      fullobjname = namespacestr + string("/") + oid.name;
+    else
+      fullobjname = oid.name;
+    if (f) {
+      f->open_object_section("osd_map");
+      f->dump_int("epoch", osdmap.get_epoch());
+      f->dump_string("pool", poolstr);
+      f->dump_int("pool_id", pool);
+      f->dump_stream("objname") << fullobjname;
+      f->dump_stream("raw_pgid") << pgid;
+      f->dump_stream("pgid") << mpgid;
+      f->dump_stream("up") << up;
+      f->dump_int("up_primary", up_p);
+      f->dump_stream("acting") << acting;
+      f->dump_int("acting_primary", acting_p);
+      f->close_section(); // osd_map
+      f->flush(rdata);
+    } else {
+      ds << "osdmap e" << osdmap.get_epoch()
+        << " pool '" << poolstr << "' (" << pool << ")"
+        << " object '" << fullobjname << "' ->"
+        << " pg " << pgid << " (" << mpgid << ")"
+        << " -> up (" << up << ", p" << up_p << ") acting ("
+        << acting << ", p" << acting_p << ")";
+      rdata.append(ds);
+    }
+  } else if ((prefix == "osd scrub" ||
+	      prefix == "osd deep-scrub" ||
+	      prefix == "osd repair")) {
+    string whostr;
+    cmd_getval(g_ceph_context, cmdmap, "who", whostr);
+    vector pvec;
+    get_str_vec(prefix, pvec);
+
+    if (whostr == "*") {
+      ss << "osds ";
+      int c = 0;
+      for (int i = 0; i < osdmap.get_max_osd(); i++)
+	if (osdmap.is_up(i)) {
+	  ss << (c++ ? "," : "") << i;
+	  mon->try_send_message(new MOSDScrub(osdmap.get_fsid(),
+					      pvec.back() == "repair",
+					      pvec.back() == "deep-scrub"),
+				osdmap.get_inst(i));
+	}
+      r = 0;
+      ss << " instructed to " << pvec.back();
+    } else {
+      long osd = parse_osd_id(whostr.c_str(), &ss);
+      if (osd < 0) {
+	r = -EINVAL;
+      } else if (osdmap.is_up(osd)) {
+	mon->try_send_message(new MOSDScrub(osdmap.get_fsid(),
+					    pvec.back() == "repair",
+					    pvec.back() == "deep-scrub"),
+			      osdmap.get_inst(osd));
+	ss << "osd." << osd << " instructed to " << pvec.back();
+      } else {
+	ss << "osd." << osd << " is not up";
+	r = -EAGAIN;
+      }
+    }
+  } else if (prefix == "osd lspools") {
+    int64_t auid;
+    cmd_getval(g_ceph_context, cmdmap, "auid", auid, int64_t(0));
+    if (f)
+      f->open_array_section("pools");
+    for (map::iterator p = osdmap.pools.begin();
+	 p != osdmap.pools.end();
+	 ++p) {
+      if (!auid || p->second.auid == (uint64_t)auid) {
+	if (f) {
+	  f->open_object_section("pool");
+	  f->dump_int("poolnum", p->first);
+	  f->dump_string("poolname", osdmap.pool_name[p->first]);
+	  f->close_section();
+	} else {
+	  ds << p->first << ' ' << osdmap.pool_name[p->first] << ',';
+	}
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(ds);
+    }
+    rdata.append(ds);
+  } else if (prefix == "osd blacklist ls") {
+    if (f)
+      f->open_array_section("blacklist");
+
+    for (ceph::unordered_map::iterator p = osdmap.blacklist.begin();
+	 p != osdmap.blacklist.end();
+	 ++p) {
+      if (f) {
+	f->open_object_section("entry");
+	f->dump_stream("addr") << p->first;
+	f->dump_stream("until") << p->second;
+	f->close_section();
+      } else {
+	stringstream ss;
+	string s;
+	ss << p->first << " " << p->second;
+	getline(ss, s);
+	s += "\n";
+	rdata.append(s);
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(rdata);
+    }
+    ss << "listed " << osdmap.blacklist.size() << " entries";
+
+  } else if (prefix == "osd pool get") {
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      r = -ENOENT;
+      goto reply;
+    }
+
+    const pg_pool_t *p = osdmap.get_pg_pool(pool);
+    string var;
+    cmd_getval(g_ceph_context, cmdmap, "var", var);
+
+    if (!p->is_tier() &&
+        (var == "hit_set_type" || var == "hit_set_period" ||
+         var == "hit_set_count" || var == "hit_set_fpp" ||
+         var == "target_max_objects" || var == "target_max_bytes" ||
+         var == "cache_target_full_ratio" ||
+         var == "cache_target_dirty_ratio" ||
+         var == "cache_min_flush_age" || var == "cache_min_evict_age")) {
+      ss << "pool '" << poolstr
+         << "' is not a tier pool: variable not applicable";
+      r = -EACCES;
+      goto reply;
+    }
+
+    if (!p->is_erasure() && var == "erasure_code_profile") {
+      ss << "pool '" << poolstr
+         << "' is not a erasure pool: variable not applicable";
+      r = -EACCES;
+      goto reply;
+    }
+
+    if (f) {
+      f->open_object_section("pool");
+      f->dump_string("pool", poolstr);
+      f->dump_int("pool_id", pool);
+
+      if (var == "pg_num") {
+        f->dump_int("pg_num", p->get_pg_num());
+      } else if (var == "pgp_num") {
+        f->dump_int("pgp_num", p->get_pgp_num());
+      } else if (var == "auid") {
+        f->dump_int("auid", p->get_auid());
+      } else if (var == "size") {
+        f->dump_int("size", p->get_size());
+      } else if (var == "min_size") {
+        f->dump_int("min_size", p->get_min_size());
+      } else if (var == "crash_replay_interval") {
+        f->dump_int("crash_replay_interval", p->get_crash_replay_interval());
+      } else if (var == "crush_ruleset") {
+        f->dump_int("crush_ruleset", p->get_crush_ruleset());
+      } else if (var == "hit_set_period") {
+	f->dump_int("hit_set_period", p->hit_set_period);
+      } else if (var == "hit_set_count") {
+	f->dump_int("hit_set_count", p->hit_set_count);
+      } else if (var == "hit_set_type") {
+	f->dump_string("hit_set_type", HitSet::get_type_name(p->hit_set_params.get_type()));
+      } else if (var == "hit_set_fpp") {
+	if (p->hit_set_params.get_type() != HitSet::TYPE_BLOOM) {
+	  f->close_section();
+	  ss << "hit set is no of type Bloom; invalid to get a false positive rate!";
+	  r = -EINVAL;
+	  goto reply;
+	} else {
+	  BloomHitSet::Params *bloomp = static_cast(p->hit_set_params.impl.get());
+	  f->dump_float("hit_set_fpp", bloomp->get_fpp());
+	}
+      } else if (var == "target_max_objects") {
+        f->dump_unsigned("target_max_objects", p->target_max_objects);
+      } else if (var == "target_max_bytes") {
+        f->dump_unsigned("target_max_bytes", p->target_max_bytes);
+      } else if (var == "cache_target_dirty_ratio") {
+        f->dump_unsigned("cache_target_dirty_ratio_micro",
+                         p->cache_target_dirty_ratio_micro);
+        f->dump_float("cache_target_dirty_ratio",
+                      ((float)p->cache_target_dirty_ratio_micro/1000000));
+      } else if (var == "cache_target_full_ratio") {
+        f->dump_unsigned("cache_target_full_ratio_micro",
+                         p->cache_target_full_ratio_micro);
+        f->dump_float("cache_target_full_ratio",
+                      ((float)p->cache_target_full_ratio_micro/1000000));
+      } else if (var == "cache_min_flush_age") {
+        f->dump_unsigned("cache_min_flush_age", p->cache_min_flush_age);
+      } else if (var == "cache_min_evict_age") {
+        f->dump_unsigned("cache_min_evict_age", p->cache_min_evict_age);
+      } else if (var == "erasure_code_profile") {
+       f->dump_string("erasure_code_profile", p->erasure_code_profile);
+      }
+
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      if (var == "pg_num") {
+        ss << "pg_num: " << p->get_pg_num();
+      } else if (var == "pgp_num") {
+        ss << "pgp_num: " << p->get_pgp_num();
+      } else if (var == "auid") {
+        ss << "auid: " << p->get_auid();
+      } else if (var == "size") {
+        ss << "size: " << p->get_size();
+      } else if (var == "min_size") {
+        ss << "min_size: " << p->get_min_size();
+      } else if (var == "crash_replay_interval") {
+        ss << "crash_replay_interval: " << p->get_crash_replay_interval();
+      } else if (var == "crush_ruleset") {
+        ss << "crush_ruleset: " << p->get_crush_ruleset();
+      } else if (var == "hit_set_period") {
+	ss << "hit_set_period: " << p->hit_set_period;
+      } else if (var == "hit_set_count") {
+	ss << "hit_set_count: " << p->hit_set_count;
+      } else if (var == "hit_set_type") {
+	ss << "hit_set_type: " <<  HitSet::get_type_name(p->hit_set_params.get_type());
+      } else if (var == "hit_set_fpp") {
+	if (p->hit_set_params.get_type() != HitSet::TYPE_BLOOM) {
+	  ss << "hit set is no of type Bloom; invalid to get a false positive rate!";
+	  r = -EINVAL;
+	  goto reply;
+	}
+	BloomHitSet::Params *bloomp = static_cast(p->hit_set_params.impl.get());
+	ss << "hit_set_fpp: " << bloomp->get_fpp();
+      } else if (var == "target_max_objects") {
+        ss << "target_max_objects: " << p->target_max_objects;
+      } else if (var == "target_max_bytes") {
+        ss << "target_max_bytes: " << p->target_max_bytes;
+      } else if (var == "cache_target_dirty_ratio") {
+        ss << "cache_target_dirty_ratio: "
+          << ((float)p->cache_target_dirty_ratio_micro/1000000);
+      } else if (var == "cache_target_full_ratio") {
+        ss << "cache_target_full_ratio: "
+          << ((float)p->cache_target_full_ratio_micro/1000000);
+      } else if (var == "cache_min_flush_age") {
+        ss << "cache_min_flush_age: " << p->cache_min_flush_age;
+      } else if (var == "cache_min_evict_age") {
+        ss << "cache_min_evict_age: " << p->cache_min_evict_age;
+      } else if (var == "erasure_code_profile") {
+       ss << "erasure_code_profile: " << p->erasure_code_profile;
+      }
+
+      rdata.append(ss);
+      ss.str("");
+    }
+    r = 0;
+
+  } else if (prefix == "osd pool stats") {
+    string pool_name;
+    cmd_getval(g_ceph_context, cmdmap, "name", pool_name);
+
+    PGMap& pg_map = mon->pgmon()->pg_map;
+
+    int64_t poolid = -ENOENT;
+    bool one_pool = false;
+    if (!pool_name.empty()) {
+      poolid = osdmap.lookup_pg_pool_name(pool_name);
+      if (poolid < 0) {
+        assert(poolid == -ENOENT);
+        ss << "unrecognized pool '" << pool_name << "'";
+        r = -ENOENT;
+        goto reply;
+      }
+      one_pool = true;
+    }
+
+    stringstream rs;
+
+    if (f)
+      f->open_array_section("pool_stats");
+    if (osdmap.get_pools().size() == 0) {
+      if (!f)
+        ss << "there are no pools!";
+      goto stats_out;
+    }
+
+    for (map::const_iterator it = osdmap.get_pools().begin();
+         it != osdmap.get_pools().end();
+         ++it) {
+
+      if (!one_pool)
+        poolid = it->first;
+
+      pool_name = osdmap.get_pool_name(poolid);
+
+      if (f) {
+        f->open_object_section("pool");
+        f->dump_string("pool_name", pool_name.c_str());
+        f->dump_int("pool_id", poolid);
+        f->open_object_section("recovery");
+      }
+
+      stringstream rss, tss;
+      pg_map.pool_recovery_summary(f.get(), &rss, poolid);
+      if (!f && !rss.str().empty())
+        tss << "  " << rss.str() << "\n";
+
+      if (f) {
+        f->close_section();
+        f->open_object_section("recovery_rate");
+      }
+
+      rss.clear();
+      rss.str("");
+
+      pg_map.pool_recovery_rate_summary(f.get(), &rss, poolid);
+      if (!f && !rss.str().empty())
+        tss << "  recovery io " << rss.str() << "\n";
+
+      if (f) {
+        f->close_section();
+        f->open_object_section("client_io_rate");
+      }
+
+      rss.clear();
+      rss.str("");
+
+      pg_map.pool_client_io_rate_summary(f.get(), &rss, poolid);
+      if (!f && !rss.str().empty())
+        tss << "  client io " << rss.str() << "\n";
+
+      if (f) {
+        f->close_section();
+        f->close_section();
+      } else {
+        rs << "pool " << pool_name << " id " << poolid << "\n";
+        if (!tss.str().empty())
+          rs << tss.str() << "\n";
+        else
+          rs << "  nothing is going on\n\n";
+      }
+
+      if (one_pool)
+        break;
+    }
+
+stats_out:
+    if (f) {
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      rdata.append(rs.str());
+    }
+    rdata.append("\n");
+    r = 0;
+
+  } else if (prefix == "osd pool get-quota") {
+    string pool_name;
+    cmd_getval(g_ceph_context, cmdmap, "pool", pool_name);
+
+    int64_t poolid = osdmap.lookup_pg_pool_name(pool_name);
+    if (poolid < 0) {
+      assert(poolid == -ENOENT);
+      ss << "unrecognized pool '" << pool_name << "'";
+      r = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(poolid);
+
+    if (f) {
+      f->open_object_section("pool_quotas");
+      f->dump_string("pool_name", pool_name);
+      f->dump_unsigned("pool_id", poolid);
+      f->dump_unsigned("quota_max_objects", p->quota_max_objects);
+      f->dump_unsigned("quota_max_bytes", p->quota_max_bytes);
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      stringstream rs;
+      rs << "quotas for pool '" << pool_name << "':\n"
+         << "  max objects: ";
+      if (p->quota_max_objects == 0)
+        rs << "N/A";
+      else
+        rs << si_t(p->quota_max_objects) << " objects";
+      rs << "\n"
+         << "  max bytes  : ";
+      if (p->quota_max_bytes == 0)
+        rs << "N/A";
+      else
+        rs << si_t(p->quota_max_bytes) << "B";
+      rdata.append(rs.str());
+    }
+    rdata.append("\n");
+    r = 0;
+  } else if (prefix == "osd crush rule list" ||
+	     prefix == "osd crush rule ls") {
+    string format;
+    cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+    Formatter *fp = new_formatter(format);
+    if (!fp)
+      fp = new_formatter("json-pretty");
+    boost::scoped_ptr f(fp);
+    f->open_array_section("rules");
+    osdmap.crush->list_rules(f.get());
+    f->close_section();
+    ostringstream rs;
+    f->flush(rs);
+    rs << "\n";
+    rdata.append(rs.str());
+  } else if (prefix == "osd crush rule dump") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    string format;
+    cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+    Formatter *fp = new_formatter(format);
+    if (!fp)
+      fp = new_formatter("json-pretty");
+    boost::scoped_ptr f(fp);
+    if (name == "") {
+      f->open_array_section("rules");
+      osdmap.crush->dump_rules(f.get());
+      f->close_section();
+    } else {
+      int ruleset = osdmap.crush->get_rule_id(name);
+      if (ruleset < 0) {
+	ss << "unknown crush ruleset '" << name << "'";
+	r = ruleset;
+	goto reply;
+      }
+      osdmap.crush->dump_rule(ruleset, f.get());
+    }
+    ostringstream rs;
+    f->flush(rs);
+    rs << "\n";
+    rdata.append(rs.str());
+  } else if (prefix == "osd crush dump") {
+    string format;
+    cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+    Formatter *fp = new_formatter(format);
+    if (!fp)
+      fp = new_formatter("json-pretty");
+    boost::scoped_ptr f(fp);
+    f->open_object_section("crush_map");
+    osdmap.crush->dump(f.get());
+    f->close_section();
+    ostringstream rs;
+    f->flush(rs);
+    rs << "\n";
+    rdata.append(rs.str());
+  } else if (prefix == "osd crush show-tunables") {
+    string format;
+    cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
+    Formatter *fp = new_formatter(format);
+    if (!fp)
+      fp = new_formatter("json-pretty");
+    boost::scoped_ptr f(fp);
+    f->open_object_section("crush_map_tunables");
+    osdmap.crush->dump_tunables(f.get());
+    f->close_section();
+    ostringstream rs;
+    f->flush(rs);
+    rs << "\n";
+    rdata.append(rs.str());
+  } else if (prefix == "osd erasure-code-profile ls") {
+    const map > &profiles =
+      osdmap.get_erasure_code_profiles();
+    if (f)
+      f->open_array_section("erasure-code-profiles");
+    for(map >::const_iterator i = profiles.begin();
+	i != profiles.end();
+	i++) {
+      if (f)
+        f->dump_string("profile", i->first.c_str());
+      else
+	rdata.append(i->first + "\n");
+    }
+    if (f) {
+      f->close_section();
+      ostringstream rs;
+      f->flush(rs);
+      rs << "\n";
+      rdata.append(rs.str());
+    }
+  } else if (prefix == "osd erasure-code-profile get") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    if (!osdmap.has_erasure_code_profile(name)) {
+      ss << "unknown erasure code profile '" << name << "'";
+      r = -ENOENT;
+      goto reply;
+    }
+    const map &profile = osdmap.get_erasure_code_profile(name);
+    if (f)
+      f->open_object_section("profile");
+    for (map::const_iterator i = profile.begin();
+	 i != profile.end();
+	 i++) {
+      if (f)
+        f->dump_string(i->first.c_str(), i->second.c_str());
+      else
+	rdata.append(i->first + "=" + i->second + "\n");
+    }
+    if (f) {
+      f->close_section();
+      ostringstream rs;
+      f->flush(rs);
+      rs << "\n";
+      rdata.append(rs.str());
+    }
+  } else {
+    // try prepare update
+    return false;
+  }
+
+ reply:
+  string rs;
+  getline(ss, rs);
+  mon->reply_command(m, r, rs, rdata, get_last_committed());
+  return true;
+}
+
+void OSDMonitor::update_pool_flags(int64_t pool_id, uint64_t flags)
+{
+  const pg_pool_t *pool = osdmap.get_pg_pool(pool_id);
+  pending_inc.get_new_pool(pool_id, pool)->flags = flags;
+}
+
+bool OSDMonitor::update_pools_status()
+{
+  if (!mon->pgmon()->is_readable())
+    return false;
+
+  bool ret = false;
+
+  const map& pools = osdmap.get_pools();
+  for (map::const_iterator it = pools.begin();
+       it != pools.end();
+       ++it) {
+    if (!mon->pgmon()->pg_map.pg_pool_sum.count(it->first))
+      continue;
+    pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first];
+    object_stat_sum_t& sum = stats.stats.sum;
+    const pg_pool_t &pool = it->second;
+    const char *pool_name = osdmap.get_pool_name(it->first);
+
+    bool pool_is_full =
+      (pool.quota_max_bytes > 0 && (uint64_t)sum.num_bytes >= pool.quota_max_bytes) ||
+      (pool.quota_max_objects > 0 && (uint64_t)sum.num_objects >= pool.quota_max_objects);
+
+    if (pool.get_flags() & pg_pool_t::FLAG_FULL) {
+      if (pool_is_full)
+        continue;
+
+      mon->clog.info() << "pool '" << pool_name
+                       << "' no longer full; removing FULL flag";
+
+      update_pool_flags(it->first, pool.get_flags() & ~pg_pool_t::FLAG_FULL);
+      ret = true;
+    } else {
+      if (!pool_is_full)
+	continue;
+
+      if (pool.quota_max_bytes > 0 &&
+          (uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
+        mon->clog.warn() << "pool '" << pool_name << "' is full"
+                         << " (reached quota's max_bytes: "
+                         << si_t(pool.quota_max_bytes) << ")";
+      } else if (pool.quota_max_objects > 0 &&
+		 (uint64_t)sum.num_objects >= pool.quota_max_objects) {
+        mon->clog.warn() << "pool '" << pool_name << "' is full"
+                         << " (reached quota's max_objects: "
+                         << pool.quota_max_objects << ")";
+      } else {
+        assert(0 == "we shouldn't reach this");
+      }
+      update_pool_flags(it->first, pool.get_flags() | pg_pool_t::FLAG_FULL);
+      ret = true;
+    }
+  }
+  return ret;
+}
+
+void OSDMonitor::get_pools_health(
+    list >& summary,
+    list > *detail) const
+{
+  const map& pools = osdmap.get_pools();
+  for (map::const_iterator it = pools.begin();
+       it != pools.end(); ++it) {
+    if (!mon->pgmon()->pg_map.pg_pool_sum.count(it->first))
+      continue;
+    pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first];
+    object_stat_sum_t& sum = stats.stats.sum;
+    const pg_pool_t &pool = it->second;
+    const char *pool_name = osdmap.get_pool_name(it->first);
+
+    if (pool.get_flags() & pg_pool_t::FLAG_FULL) {
+      // uncomment these asserts if/when we update the FULL flag on pg_stat update
+      //assert((pool.quota_max_objects > 0) || (pool.quota_max_bytes > 0));
+
+      stringstream ss;
+      ss << "pool '" << pool_name << "' is full";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+      if (detail)
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+
+    float warn_threshold = g_conf->mon_pool_quota_warn_threshold/100;
+    float crit_threshold = g_conf->mon_pool_quota_crit_threshold/100;
+
+    if (pool.quota_max_objects > 0) {
+      stringstream ss;
+      health_status_t status = HEALTH_OK;
+      if ((uint64_t)sum.num_objects >= pool.quota_max_objects) {
+	// uncomment these asserts if/when we update the FULL flag on pg_stat update
+        //assert(pool.get_flags() & pg_pool_t::FLAG_FULL);
+      } else if (crit_threshold > 0 &&
+		 sum.num_objects >= pool.quota_max_objects*crit_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << sum.num_objects << " objects"
+           << " (max " << pool.quota_max_objects << ")";
+        status = HEALTH_ERR;
+      } else if (warn_threshold > 0 &&
+		 sum.num_objects >= pool.quota_max_objects*warn_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << sum.num_objects << " objects"
+           << " (max " << pool.quota_max_objects << ")";
+        status = HEALTH_WARN;
+      }
+      if (status != HEALTH_OK) {
+        pair s(status, ss.str());
+        summary.push_back(s);
+        if (detail)
+          detail->push_back(s);
+      }
+    }
+
+    if (pool.quota_max_bytes > 0) {
+      health_status_t status = HEALTH_OK;
+      stringstream ss;
+      if ((uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
+	// uncomment these asserts if/when we update the FULL flag on pg_stat update
+	//assert(pool.get_flags() & pg_pool_t::FLAG_FULL);
+      } else if (crit_threshold > 0 &&
+		 sum.num_bytes >= pool.quota_max_bytes*crit_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << si_t(sum.num_bytes) << " bytes"
+           << " (max " << si_t(pool.quota_max_bytes) << ")";
+        status = HEALTH_ERR;
+      } else if (warn_threshold > 0 &&
+		 sum.num_bytes >= pool.quota_max_bytes*warn_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << si_t(sum.num_bytes) << " objects"
+           << " (max " << si_t(pool.quota_max_bytes) << ")";
+        status = HEALTH_WARN;
+      }
+      if (status != HEALTH_OK) {
+        pair s(status, ss.str());
+        summary.push_back(s);
+        if (detail)
+          detail->push_back(s);
+      }
+    }
+  }
+}
+
+
+int OSDMonitor::prepare_new_pool(MPoolOp *m)
+{
+  dout(10) << "prepare_new_pool from " << m->get_connection() << dendl;
+  MonSession *session = m->get_session();
+  if (!session)
+    return -EPERM;
+  string erasure_code_profile;
+  stringstream ss;
+  string ruleset_name;
+  if (m->auid)
+    return prepare_new_pool(m->name, m->auid, m->crush_rule, ruleset_name,
+			    0, 0,
+                            erasure_code_profile,
+			    pg_pool_t::TYPE_REPLICATED, ss);
+  else
+    return prepare_new_pool(m->name, session->auid, m->crush_rule, ruleset_name,
+			    0, 0,
+                            erasure_code_profile,
+			    pg_pool_t::TYPE_REPLICATED, ss);
+}
+
+int OSDMonitor::crush_ruleset_create_erasure(const string &name,
+					     const string &profile,
+					     int *ruleset,
+					     stringstream &ss)
+{
+  int ruleid = osdmap.crush->get_rule_id(name);
+  if (ruleid != -ENOENT) {
+    *ruleset = osdmap.crush->get_rule_mask_ruleset(ruleid);
+    return -EEXIST;
+  }
+
+  CrushWrapper newcrush;
+  _get_pending_crush(newcrush);
+
+  ruleid = newcrush.get_rule_id(name);
+  if (ruleid != -ENOENT) {
+    *ruleset = newcrush.get_rule_mask_ruleset(ruleid);
+    return -EALREADY;
+  } else {
+    ErasureCodeInterfaceRef erasure_code;
+    int err = get_erasure_code(profile, &erasure_code, ss);
+    if (err) {
+      ss << "failed to load plugin using profile " << profile;
+      return err;
+    }
+
+    err = erasure_code->create_ruleset(name, newcrush, &ss);
+    erasure_code.reset();
+    if (err < 0)
+      return err;
+    *ruleset = err;
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush);
+    return 0;
+  }
+}
+
+int OSDMonitor::get_erasure_code(const string &erasure_code_profile,
+				 ErasureCodeInterfaceRef *erasure_code,
+				 stringstream &ss) const
+{
+  if (pending_inc.has_erasure_code_profile(erasure_code_profile))
+    return -EAGAIN;
+  const map &profile =
+    osdmap.get_erasure_code_profile(erasure_code_profile);
+  map::const_iterator plugin =
+    profile.find("plugin");
+  if (plugin == profile.end()) {
+    ss << "cannot determine the erasure code plugin"
+       << " because there is no 'plugin' entry in the erasure_code_profile "
+       << profile;
+    return -EINVAL;
+  }
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  return instance.factory(plugin->second, profile, erasure_code, ss);
+}
+
+int OSDMonitor::check_cluster_features(uint64_t features,
+				       stringstream &ss)
+{
+  stringstream unsupported_ss;
+  int unsupported_count = 0;
+  if ((mon->get_quorum_features() & features) != features) {
+    unsupported_ss << "the monitor cluster";
+    ++unsupported_count;
+  }
+
+  set up_osds;
+  osdmap.get_up_osds(up_osds);
+  for (set::iterator it = up_osds.begin();
+       it != up_osds.end(); ++it) {
+    const osd_xinfo_t &xi = osdmap.get_xinfo(*it);
+    if ((xi.features & features) != features) {
+      if (unsupported_count > 0)
+	unsupported_ss << ", ";
+      unsupported_ss << "osd." << *it;
+      unsupported_count ++;
+    }
+  }
+
+  if (unsupported_count > 0) {
+    ss << "features " << features << " unsupported by: "
+       << unsupported_ss.str();
+    return -ENOTSUP;
+  }
+
+  // check pending osd state, too!
+  for (map::const_iterator p =
+	 pending_inc.new_xinfo.begin();
+       p != pending_inc.new_xinfo.end(); ++p) {
+    const osd_xinfo_t &xi = p->second;
+    if ((xi.features & features) != features) {
+      dout(10) << __func__ << " pending osd." << p->first
+	       << " features are insufficient; retry" << dendl;
+      return -EAGAIN;
+    }
+  }
+
+  return 0;
+}
+
+bool OSDMonitor::validate_crush_against_features(const CrushWrapper *newcrush,
+                                                 stringstream& ss)
+{
+  OSDMap::Incremental new_pending = pending_inc;
+  ::encode(*newcrush, new_pending.crush);
+  OSDMap newmap;
+  newmap.deepish_copy_from(osdmap);
+  newmap.apply_incremental(new_pending);
+  uint64_t features = newmap.get_features(CEPH_ENTITY_TYPE_MON, NULL);
+
+  stringstream features_ss;
+
+  int r = check_cluster_features(features, features_ss);
+
+  if (!r)
+    return true;
+
+  ss << "Could not change CRUSH: " << features_ss.str();
+  return false;
+}
+
+bool OSDMonitor::erasure_code_profile_in_use(const map &pools,
+					     const string &profile,
+					     ostream &ss)
+{
+  bool found = false;
+  for (map::const_iterator p = pools.begin();
+       p != pools.end();
+       ++p) {
+    if (p->second.erasure_code_profile == profile) {
+      ss << osdmap.pool_name[p->first] << " ";
+      found = true;
+    }
+  }
+  if (found) {
+    ss << "pool(s) are using the erasure code profile '" << profile << "'";
+  }
+  return found;
+}
+
+int OSDMonitor::parse_erasure_code_profile(const vector &erasure_code_profile,
+					   map *erasure_code_profile_map,
+					   stringstream &ss)
+{
+  int r = get_str_map(g_conf->osd_pool_default_erasure_code_profile,
+		      ss,
+		      erasure_code_profile_map);
+  if (r)
+    return r;
+  (*erasure_code_profile_map)["directory"] =
+    g_conf->osd_pool_default_erasure_code_directory;
+
+  for (vector::const_iterator i = erasure_code_profile.begin();
+       i != erasure_code_profile.end();
+       ++i) {
+    size_t equal = i->find('=');
+    if (equal == string::npos)
+      (*erasure_code_profile_map)[*i] = string();
+    else {
+      const string key = i->substr(0, equal);
+      equal++;
+      const string value = i->substr(equal);
+      (*erasure_code_profile_map)[key] = value;
+    }
+  }
+
+  return 0;
+}
+
+int OSDMonitor::prepare_pool_size(const unsigned pool_type,
+				  const string &erasure_code_profile,
+				  unsigned *size, unsigned *min_size,
+				  stringstream &ss)
+{
+  int err = 0;
+  switch (pool_type) {
+  case pg_pool_t::TYPE_REPLICATED:
+    *size = g_conf->osd_pool_default_size;
+    *min_size = g_conf->get_osd_pool_default_min_size();
+    break;
+  case pg_pool_t::TYPE_ERASURE:
+    {
+      ErasureCodeInterfaceRef erasure_code;
+      err = get_erasure_code(erasure_code_profile, &erasure_code, ss);
+      if (err == 0) {
+	*size = erasure_code->get_chunk_count();
+	*min_size = erasure_code->get_data_chunk_count();
+      }
+    }
+    break;
+  default:
+    ss << "prepare_pool_size: " << pool_type << " is not a known pool type";
+    err = -EINVAL;
+    break;
+  }
+  return err;
+}
+
+int OSDMonitor::prepare_pool_stripe_width(const unsigned pool_type,
+					  const string &erasure_code_profile,
+					  uint32_t *stripe_width,
+					  stringstream &ss)
+{
+  int err = 0;
+  switch (pool_type) {
+  case pg_pool_t::TYPE_REPLICATED:
+    // ignored
+    break;
+  case pg_pool_t::TYPE_ERASURE:
+    {
+      ErasureCodeInterfaceRef erasure_code;
+      err = get_erasure_code(erasure_code_profile, &erasure_code, ss);
+      uint32_t desired_stripe_width = g_conf->osd_pool_erasure_code_stripe_width;
+      if (err == 0)
+	*stripe_width = erasure_code->get_data_chunk_count() *
+	  erasure_code->get_chunk_size(desired_stripe_width);
+    }
+    break;
+  default:
+    ss << "prepare_pool_stripe_width: "
+       << pool_type << " is not a known pool type";
+    err = -EINVAL;
+    break;
+  }
+  return err;
+}
+
+int OSDMonitor::prepare_pool_crush_ruleset(const unsigned pool_type,
+					   const string &erasure_code_profile,
+					   const string &ruleset_name,
+					   int *crush_ruleset,
+					   stringstream &ss)
+{
+  if (*crush_ruleset < 0) {
+    switch (pool_type) {
+    case pg_pool_t::TYPE_REPLICATED:
+      *crush_ruleset = osdmap.crush->get_osd_pool_default_crush_replicated_ruleset(g_ceph_context);
+      if (*crush_ruleset < 0) {
+        // Errors may happen e.g. if no valid ruleset is available
+        ss << "No suitable CRUSH ruleset exists";
+        return *crush_ruleset;
+      }
+      break;
+    case pg_pool_t::TYPE_ERASURE:
+      {
+	int err = crush_ruleset_create_erasure(ruleset_name,
+					       erasure_code_profile,
+					       crush_ruleset, ss);
+	switch (err) {
+	case -EALREADY:
+	  dout(20) << "prepare_pool_crush_ruleset: ruleset "
+		   << ruleset_name << " try again" << dendl;
+	case 0:
+	  // need to wait for the crush rule to be proposed before proceeding
+	  err = -EAGAIN;
+	  break;
+	case -EEXIST:
+	  err = 0;
+	  break;
+ 	}
+	return err;
+      }
+      break;
+    default:
+      ss << "prepare_pool_crush_ruleset: " << pool_type
+	 << " is not a known pool type";
+      return -EINVAL;
+      break;
+    }
+  } else {
+    if (!osdmap.crush->ruleset_exists(*crush_ruleset)) {
+      ss << "CRUSH ruleset " << *crush_ruleset << " not found";
+      return -ENOENT;
+    }
+  }
+
+  return 0;
+}
+/**
+ * @param name The name of the new pool
+ * @param auid The auid of the pool owner. Can be -1
+ * @param crush_ruleset The crush rule to use. If <0, will use the system default
+ * @param crush_ruleset_name The crush rule to use, if crush_rulset <0
+ * @param pg_num The pg_num to use. If set to 0, will use the system default
+ * @param pgp_num The pgp_num to use. If set to 0, will use the system default
+ * @param erasure_code_profile The profile name in OSDMap to be used for erasure code
+ * @param pool_type TYPE_ERASURE, TYPE_REP or TYPE_RAID4
+ * @param ss human readable error message, if any.
+ *
+ * @return 0 on success, negative errno on failure.
+ */
+int OSDMonitor::prepare_new_pool(string& name, uint64_t auid,
+				 int crush_ruleset,
+				 const string &crush_ruleset_name,
+                                 unsigned pg_num, unsigned pgp_num,
+				 const string &erasure_code_profile,
+                                 const unsigned pool_type,
+				 stringstream &ss)
+{
+  int r;
+  r = prepare_pool_crush_ruleset(pool_type, erasure_code_profile,
+				 crush_ruleset_name, &crush_ruleset, ss);
+  if (r)
+    return r;
+  unsigned size, min_size;
+  r = prepare_pool_size(pool_type, erasure_code_profile, &size, &min_size, ss);
+  if (r)
+    return r;
+  uint32_t stripe_width = 0;
+  r = prepare_pool_stripe_width(pool_type, erasure_code_profile, &stripe_width, ss);
+  if (r)
+    return r;
+
+  for (map::iterator p = pending_inc.new_pool_names.begin();
+       p != pending_inc.new_pool_names.end();
+       ++p) {
+    if (p->second == name)
+      return 0;
+  }
+
+  if (-1 == pending_inc.new_pool_max)
+    pending_inc.new_pool_max = osdmap.pool_max;
+  int64_t pool = ++pending_inc.new_pool_max;
+  pg_pool_t empty;
+  pg_pool_t *pi = pending_inc.get_new_pool(pool, &empty);
+  pi->type = pool_type;
+  pi->flags = g_conf->osd_pool_default_flags;
+  if (g_conf->osd_pool_default_flag_hashpspool)
+    pi->flags |= pg_pool_t::FLAG_HASHPSPOOL;
+
+  pi->size = size;
+  pi->min_size = min_size;
+  pi->crush_ruleset = crush_ruleset;
+  pi->object_hash = CEPH_STR_HASH_RJENKINS;
+  pi->set_pg_num(pg_num ? pg_num : g_conf->osd_pool_default_pg_num);
+  pi->set_pgp_num(pgp_num ? pgp_num : g_conf->osd_pool_default_pgp_num);
+  pi->last_change = pending_inc.epoch;
+  pi->auid = auid;
+  pi->erasure_code_profile = erasure_code_profile;
+  pi->stripe_width = stripe_width;
+  pi->cache_target_dirty_ratio_micro =
+    g_conf->osd_pool_default_cache_target_dirty_ratio * 1000000;
+  pi->cache_target_full_ratio_micro =
+    g_conf->osd_pool_default_cache_target_full_ratio * 1000000;
+  pi->cache_min_flush_age = g_conf->osd_pool_default_cache_min_flush_age;
+  pi->cache_min_evict_age = g_conf->osd_pool_default_cache_min_evict_age;
+  pending_inc.new_pool_names[pool] = name;
+  return 0;
+}
+
+bool OSDMonitor::prepare_set_flag(MMonCommand *m, int flag)
+{
+  ostringstream ss;
+  if (pending_inc.new_flags < 0)
+    pending_inc.new_flags = osdmap.get_flags();
+  pending_inc.new_flags |= flag;
+  ss << "set " << OSDMap::get_flag_string(flag);
+  wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+						    get_last_committed() + 1));
+  return true;
+}
+
+bool OSDMonitor::prepare_unset_flag(MMonCommand *m, int flag)
+{
+  ostringstream ss;
+  if (pending_inc.new_flags < 0)
+    pending_inc.new_flags = osdmap.get_flags();
+  pending_inc.new_flags &= ~flag;
+  ss << "unset " << OSDMap::get_flag_string(flag);
+  wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+						    get_last_committed() + 1));
+  return true;
+}
+
+int OSDMonitor::parse_osd_id(const char *s, stringstream *pss)
+{
+  // osd.NNN?
+  if (strncmp(s, "osd.", 4) == 0) {
+    s += 4;
+  }
+
+  // NNN?
+  ostringstream ss;
+  long id = parse_pos_long(s, &ss);
+  if (id < 0) {
+    *pss << ss.str();
+    return id;
+  }
+  if (id > 0xffff) {
+    *pss << "osd id " << id << " is too large";
+    return -ERANGE;
+  }
+  return id;
+}
+
+int OSDMonitor::prepare_command_pool_set(map &cmdmap,
+                                         stringstream& ss)
+{
+  string poolstr;
+  cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+  int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+  if (pool < 0) {
+    ss << "unrecognized pool '" << poolstr << "'";
+    return -ENOENT;
+  }
+  string var;
+  cmd_getval(g_ceph_context, cmdmap, "var", var);
+
+  pg_pool_t p = *osdmap.get_pg_pool(pool);
+  if (pending_inc.new_pools.count(pool))
+    p = pending_inc.new_pools[pool];
+
+  // accept val as a json string in the normal case (current
+  // generation monitor).  parse out int or float values from the
+  // string as needed.  however, if it is not a string, try to pull
+  // out an int, in case an older monitor with an older json schema is
+  // forwarding a request.
+  string val;
+  string interr, floaterr;
+  int64_t n = 0;
+  double f = 0;
+  int64_t uf = 0;  // micro-f
+  if (!cmd_getval(g_ceph_context, cmdmap, "val", val)) {
+    // wasn't a string; maybe an older mon forwarded json with an int?
+    if (!cmd_getval(g_ceph_context, cmdmap, "val", n))
+      return -EINVAL;  // no value!
+  } else {
+    // we got a string.  see if it contains an int.
+    n = strict_strtoll(val.c_str(), 10, &interr);
+    // or a float
+    f = strict_strtod(val.c_str(), &floaterr);
+    uf = llrintl(f * (double)1000000.0);
+  }
+
+  if (!p.is_tier() &&
+      (var == "hit_set_type" || var == "hit_set_period" ||
+       var == "hit_set_count" || var == "hit_set_fpp" ||
+       var == "target_max_objects" || var == "target_max_bytes" ||
+       var == "cache_target_full_ratio" || var == "cache_target_dirty_ratio" ||
+       var == "cache_min_flush_age" || var == "cache_min_evict_age")) {
+    ss << "pool '" << poolstr << "' is not a tier pool: variable not applicable";
+    return -EACCES;
+  }
+
+  if (var == "size") {
+    if (p.type == pg_pool_t::TYPE_ERASURE) {
+      ss << "can not change the size of an erasure-coded pool";
+      return -ENOTSUP;
+    }
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n == 0 || n > 10) {
+      ss << "pool size must be between 1 and 10";
+      return -EINVAL;
+    }
+    p.size = n;
+    if (n < p.min_size)
+      p.min_size = n;
+  } else if (var == "min_size") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.min_size = n;
+  } else if (var == "auid") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.auid = n;
+  } else if (var == "crash_replay_interval") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.crash_replay_interval = n;
+  } else if (var == "pg_num") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n <= (int)p.get_pg_num()) {
+      ss << "specified pg_num " << n << " <= current " << p.get_pg_num();
+      if (n < (int)p.get_pg_num())
+	return -EEXIST;
+      return 0;
+    }
+    string force;
+    cmd_getval(g_ceph_context,cmdmap, "force", force);
+    if (p.cache_mode != pg_pool_t::CACHEMODE_NONE &&
+	force != "--yes-i-really-mean-it") {
+      ss << "splits in cache pools must be followed by scrubs and leave sufficient free space to avoid overfilling.  use --yes-i-really-mean-it to force.";
+      return -EPERM;
+    }
+    int expected_osds = MAX(1, MIN(p.get_pg_num(), osdmap.get_num_osds()));
+    int64_t new_pgs = n - p.get_pg_num();
+    int64_t pgs_per_osd = new_pgs / expected_osds;
+    if (pgs_per_osd > g_conf->mon_osd_max_split_count) {
+      ss << "specified pg_num " << n << " is too large (creating "
+	 << new_pgs << " new PGs on ~" << expected_osds
+	 << " OSDs exceeds per-OSD max of " << g_conf->mon_osd_max_split_count
+	 << ')';
+      return -E2BIG;
+    }
+    for(set::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
+	i != mon->pgmon()->pg_map.creating_pgs.end();
+	++i) {
+      if (i->m_pool == static_cast(pool)) {
+	ss << "currently creating pgs, wait";
+	return -EBUSY;
+      }
+    }
+    p.set_pg_num(n);
+  } else if (var == "pgp_num") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n <= 0) {
+      ss << "specified pgp_num must > 0, but you set to " << n;
+      return -EINVAL;
+    }
+    if (n > (int)p.get_pg_num()) {
+      ss << "specified pgp_num " << n << " > pg_num " << p.get_pg_num();
+      return -EINVAL;
+    }
+    for(set::iterator i = mon->pgmon()->pg_map.creating_pgs.begin();
+	i != mon->pgmon()->pg_map.creating_pgs.end();
+	++i) {
+      if (i->m_pool == static_cast(pool)) {
+	ss << "currently creating pgs, wait";
+	return -EBUSY;
+      }
+    }
+    p.set_pgp_num(n);
+  } else if (var == "crush_ruleset") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (!osdmap.crush->ruleset_exists(n)) {
+      ss << "crush ruleset " << n << " does not exist";
+      return -ENOENT;
+    }
+    p.crush_ruleset = n;
+  } else if (var == "hashpspool") {
+    // make sure we only compare against 'n' if we didn't receive a string
+    if (val == "true" || (interr.empty() && n == 1)) {
+      p.flags |= pg_pool_t::FLAG_HASHPSPOOL;
+    } else if (val == "false" || (interr.empty() && n == 0)) {
+      p.flags &= ~pg_pool_t::FLAG_HASHPSPOOL;
+    } else {
+      ss << "expecting value 'true', 'false', '0', or '1'";
+      return -EINVAL;
+    }
+  } else if (var == "hit_set_type") {
+    if (val == "none")
+      p.hit_set_params = HitSet::Params();
+    else {
+      int err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+      if (err)
+	return err;
+      if (val == "bloom") {
+	BloomHitSet::Params *bsp = new BloomHitSet::Params;
+	bsp->set_fpp(g_conf->osd_pool_default_hit_set_bloom_fpp);
+	p.hit_set_params = HitSet::Params(bsp);
+      } else if (val == "explicit_hash")
+	p.hit_set_params = HitSet::Params(new ExplicitHashHitSet::Params);
+      else if (val == "explicit_object")
+	p.hit_set_params = HitSet::Params(new ExplicitObjectHitSet::Params);
+      else {
+	ss << "unrecognized hit_set type '" << val << "'";
+	return -EINVAL;
+      }
+    }
+  } else if (var == "hit_set_period") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.hit_set_period = n;
+  } else if (var == "hit_set_count") {
+
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.hit_set_count = n;
+  } else if (var == "hit_set_fpp") {
+    if (floaterr.length()) {
+      ss << "error parsing floating point value '" << val << "': " << floaterr;
+      return -EINVAL;
+    }
+    if (p.hit_set_params.get_type() != HitSet::TYPE_BLOOM) {
+      ss << "hit set is not of type Bloom; invalid to set a false positive rate!";
+      return -EINVAL;
+    }
+    BloomHitSet::Params *bloomp = static_cast(p.hit_set_params.impl.get());
+    bloomp->set_fpp(f);
+  } else if (var == "debug_fake_ec_pool") {
+    if (val == "true" || (interr.empty() && n == 1)) {
+      p.flags |= pg_pool_t::FLAG_DEBUG_FAKE_EC_POOL;
+    }
+  } else if (var == "target_max_objects") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.target_max_objects = n;
+  } else if (var == "target_max_bytes") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.target_max_bytes = n;
+  } else if (var == "cache_target_dirty_ratio") {
+    if (floaterr.length()) {
+      ss << "error parsing float '" << val << "': " << floaterr;
+      return -EINVAL;
+    }
+    if (f < 0 || f > 1.0) {
+      ss << "value must be in the range 0..1";
+      return -ERANGE;
+    }
+    p.cache_target_dirty_ratio_micro = uf;
+  } else if (var == "cache_target_full_ratio") {
+    if (floaterr.length()) {
+      ss << "error parsing float '" << val << "': " << floaterr;
+      return -EINVAL;
+    }
+    if (f < 0 || f > 1.0) {
+      ss << "value must be in the range 0..1";
+      return -ERANGE;
+    }
+    p.cache_target_full_ratio_micro = uf;
+  } else if (var == "cache_min_flush_age") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.cache_min_flush_age = n;
+  } else if (var == "cache_min_evict_age") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.cache_min_evict_age = n;
+  } else {
+    ss << "unrecognized variable '" << var << "'";
+    return -EINVAL;
+  }
+  ss << "set pool " << pool << " " << var << " to " << val;
+  p.last_change = pending_inc.epoch;
+  pending_inc.new_pools[pool] = p;
+  return 0;
+}
+
+bool OSDMonitor::prepare_command(MMonCommand *m)
+{
+  stringstream ss;
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  return prepare_command_impl(m, cmdmap);
+}
+
+bool OSDMonitor::prepare_command_impl(MMonCommand *m,
+				      map &cmdmap)
+{
+  bool ret = false;
+  stringstream ss;
+  string rs;
+  bufferlist rdata;
+  int err = 0;
+
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr f(new_formatter(format));
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  int64_t id;
+  string name;
+  bool osdid_present = cmd_getval(g_ceph_context, cmdmap, "id", id);
+  if (osdid_present) {
+    ostringstream oss;
+    oss << "osd." << id;
+    name = oss.str();
+  }
+
+  // Even if there's a pending state with changes that could affect
+  // a command, considering that said state isn't yet committed, we
+  // just don't care about those changes if the command currently being
+  // handled acts as a no-op against the current committed state.
+  // In a nutshell, we assume this command  happens *before*.
+  //
+  // Let me make this clearer:
+  //
+  //   - If we have only one client, and that client issues some
+  //     operation that would conflict with this operation  but is
+  //     still on the pending state, then we would be sure that said
+  //     operation wouldn't have returned yet, so the client wouldn't
+  //     issue this operation (unless the client didn't wait for the
+  //     operation to finish, and that would be the client's own fault).
+  //
+  //   - If we have more than one client, each client will observe
+  //     whatever is the state at the moment of the commit.  So, if we
+  //     have two clients, one issuing an unlink and another issuing a
+  //     link, and if the link happens while the unlink is still on the
+  //     pending state, from the link's point-of-view this is a no-op.
+  //     If different clients are issuing conflicting operations and
+  //     they care about that, then the clients should make sure they
+  //     enforce some kind of concurrency mechanism -- from our
+  //     perspective that's what Douglas Adams would call an SEP.
+  //
+  // This should be used as a general guideline for most commands handled
+  // in this function.  Adapt as you see fit, but please bear in mind that
+  // this is the expected behavior.
+
+
+  if (prefix == "osd setcrushmap" ||
+      (prefix == "osd crush set" && !osdid_present)) {
+    dout(10) << "prepare_command setting new crush map" << dendl;
+    bufferlist data(m->get_data());
+    CrushWrapper crush;
+    try {
+      bufferlist::iterator bl(data.begin());
+      crush.decode(bl);
+    }
+    catch (const std::exception &e) {
+      err = -EINVAL;
+      ss << "Failed to parse crushmap: " << e.what();
+      goto reply;
+    }
+
+    if (!validate_crush_against_features(&crush, ss)) {
+      err = -EINVAL;
+      goto reply;
+    }
+
+    // sanity check: test some inputs to make sure this map isn't totally broken
+    dout(10) << " testing map" << dendl;
+    stringstream ess;
+    CrushTester tester(crush, ess);
+    tester.test();
+    dout(10) << " result " << ess.str() << dendl;
+
+    pending_inc.crush = data;
+    ss << "set crush map";
+    goto update;
+  } else if (prefix == "osd crush add-bucket") {
+    // os crush add-bucket  
+    string name, typestr;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    cmd_getval(g_ceph_context, cmdmap, "type", typestr);
+
+    if (!_have_pending_crush() &&
+	_get_stable_crush().name_exists(name)) {
+      ss << "bucket '" << name << "' already exists";
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (newcrush.name_exists(name)) {
+      ss << "bucket '" << name << "' already exists";
+      goto update;
+    }
+    int type = newcrush.get_type_id(typestr);
+    if (type < 0) {
+      ss << "type '" << typestr << "' does not exist";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (type == 0) {
+      ss << "type '" << typestr << "' is for devices, not buckets";
+      err = -EINVAL;
+      goto reply;
+    }
+    int bucketno;
+    err = newcrush.add_bucket(0, CRUSH_BUCKET_STRAW,
+				       CRUSH_HASH_DEFAULT, type, 0, NULL,
+				       NULL, &bucketno);
+    if (err < 0) {
+      ss << "add_bucket error: '" << cpp_strerror(err) << "'";
+      goto reply;
+    }
+    err = newcrush.set_item_name(bucketno, name);
+    if (err < 0) {
+      ss << "error setting bucket name to '" << name << "'";
+      goto reply;
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush);
+    ss << "added bucket " << name << " type " << typestr
+       << " to crush map";
+    goto update;
+  } else if (osdid_present &&
+	     (prefix == "osd crush set" || prefix == "osd crush add")) {
+    //  is 'osd.' or '', passed as int64_t id
+    // osd crush set    [ ...]
+    // osd crush add    [ ...]
+
+    if (!osdmap.exists(id)) {
+      err = -ENOENT;
+      ss << name << " does not exist.  create it before updating the crush map";
+      goto reply;
+    }
+
+    double weight;
+    if (!cmd_getval(g_ceph_context, cmdmap, "weight", weight)) {
+      ss << "unable to parse weight value '"
+         << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    string args;
+    vector argvec;
+    cmd_getval(g_ceph_context, cmdmap, "args", argvec);
+    map loc;
+    CrushWrapper::parse_loc_map(argvec, &loc);
+
+    if (prefix == "osd crush set"
+        && !_get_stable_crush().item_exists(id)) {
+      err = -ENOENT;
+      ss << "unable to set item id " << id << " name '" << name
+         << "' weight " << weight << " at location " << loc
+         << ": does not exist";
+      goto reply;
+    }
+
+    dout(5) << "adding/updating crush item id " << id << " name '"
+      << name << "' weight " << weight << " at location "
+      << loc << dendl;
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    string action;
+    if (prefix == "osd crush set" ||
+        newcrush.check_item_loc(g_ceph_context, id, loc, (int *)NULL)) {
+      action = "set";
+      err = newcrush.update_item(g_ceph_context, id, weight, name, loc);
+    } else {
+      action = "add";
+      err = newcrush.insert_item(g_ceph_context, id, weight, name, loc);
+      if (err == 0)
+        err = 1;
+    }
+
+    if (err < 0)
+      goto reply;
+
+    if (err == 0 && !_have_pending_crush()) {
+      ss << action << " item id " << id << " name '" << name << "' weight "
+        << weight << " at location " << loc << ": no change";
+      goto reply;
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush);
+    ss << action << " item id " << id << " name '" << name << "' weight "
+      << weight << " at location " << loc << " to crush map";
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush create-or-move") {
+    do {
+      // osd crush create-or-move    [ ...]
+      if (!osdmap.exists(id)) {
+	err = -ENOENT;
+	ss << name << " does not exist.  create it before updating the crush map";
+	goto reply;
+      }
+
+      double weight;
+      if (!cmd_getval(g_ceph_context, cmdmap, "weight", weight)) {
+        ss << "unable to parse weight value '"
+           << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+        err = -EINVAL;
+        goto reply;
+      }
+
+      string args;
+      vector argvec;
+      cmd_getval(g_ceph_context, cmdmap, "args", argvec);
+      map loc;
+      CrushWrapper::parse_loc_map(argvec, &loc);
+
+      dout(0) << "create-or-move crush item name '" << name << "' initial_weight " << weight
+	      << " at location " << loc << dendl;
+
+      CrushWrapper newcrush;
+      _get_pending_crush(newcrush);
+
+      err = newcrush.create_or_move_item(g_ceph_context, id, weight, name, loc);
+      if (err == 0) {
+	ss << "create-or-move updated item name '" << name << "' weight " << weight
+	   << " at location " << loc << " to crush map";
+	break;
+      }
+      if (err > 0) {
+	pending_inc.crush.clear();
+	newcrush.encode(pending_inc.crush);
+	ss << "create-or-move updating item name '" << name << "' weight " << weight
+	   << " at location " << loc << " to crush map";
+	getline(ss, rs);
+	wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      }
+    } while (false);
+
+  } else if (prefix == "osd crush move") {
+    do {
+      // osd crush move   [ ...]
+
+      string args;
+      vector argvec;
+      cmd_getval(g_ceph_context, cmdmap, "name", name);
+      cmd_getval(g_ceph_context, cmdmap, "args", argvec);
+      map loc;
+      CrushWrapper::parse_loc_map(argvec, &loc);
+
+      dout(0) << "moving crush item name '" << name << "' to location " << loc << dendl;
+      CrushWrapper newcrush;
+      _get_pending_crush(newcrush);
+
+      if (!newcrush.name_exists(name)) {
+	err = -ENOENT;
+	ss << "item " << name << " does not exist";
+	break;
+      }
+      int id = newcrush.get_item_id(name);
+
+      if (!newcrush.check_item_loc(g_ceph_context, id, loc, (int *)NULL)) {
+	err = newcrush.move_bucket(g_ceph_context, id, loc);
+	if (err >= 0) {
+	  ss << "moved item id " << id << " name '" << name << "' to location " << loc << " in crush map";
+	  pending_inc.crush.clear();
+	  newcrush.encode(pending_inc.crush);
+	  getline(ss, rs);
+	  wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						   get_last_committed() + 1));
+	  return true;
+	}
+      } else {
+	ss << "no need to move item id " << id << " name '" << name << "' to location " << loc << " in crush map";
+	err = 0;
+      }
+    } while (false);
+
+  } else if (prefix == "osd crush link") {
+    // osd crush link   [ ...]
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    vector argvec;
+    cmd_getval(g_ceph_context, cmdmap, "args", argvec);
+    map loc;
+    CrushWrapper::parse_loc_map(argvec, &loc);
+
+    if (!osdmap.crush->name_exists(name)) {
+      err = -ENOENT;
+      ss << "item " << name << " does not exist";
+      goto reply;
+    }
+    if (osdmap.crush->check_item_loc(g_ceph_context, id, loc, (int*) NULL)) {
+      ss << "no need to move item id " << id << " name '" << name
+	 << "' to location " << loc << " in crush map";
+      err = 0;
+      goto reply;
+    }
+
+    dout(5) << "linking crush item name '" << name << "' at location " << loc << dendl;
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (!newcrush.name_exists(name)) {
+      err = -ENOENT;
+      ss << "item " << name << " does not exist";
+    } else {
+      int id = newcrush.get_item_id(name);
+      if (!newcrush.check_item_loc(g_ceph_context, id, loc, (int *)NULL)) {
+	err = newcrush.link_bucket(g_ceph_context, id, loc);
+	if (err >= 0) {
+	  ss << "linked item id " << id << " name '" << name
+             << "' to location " << loc << " in crush map";
+	  pending_inc.crush.clear();
+	  newcrush.encode(pending_inc.crush);
+	}
+      } else {
+	ss << "no need to move item id " << id << " name '" << name
+           << "' to location " << loc << " in crush map";
+	err = 0;
+      }
+    }
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, err, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush rm" ||
+	     prefix == "osd crush remove" ||
+	     prefix == "osd crush unlink") {
+    do {
+      // osd crush rm  [ancestor]
+      CrushWrapper newcrush;
+      _get_pending_crush(newcrush);
+
+      string name;
+      cmd_getval(g_ceph_context, cmdmap, "name", name);
+
+      if (!osdmap.crush->name_exists(name)) {
+	err = 0;
+	ss << "device '" << name << "' does not appear in the crush map";
+	break;
+      }
+      if (!newcrush.name_exists(name)) {
+	err = 0;
+	ss << "device '" << name << "' does not appear in the crush map";
+	getline(ss, rs);
+	wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      }
+      int id = newcrush.get_item_id(name);
+      bool unlink_only = prefix == "osd crush unlink";
+      string ancestor_str;
+      if (cmd_getval(g_ceph_context, cmdmap, "ancestor", ancestor_str)) {
+	if (!newcrush.name_exists(ancestor_str)) {
+	  err = -ENOENT;
+	  ss << "ancestor item '" << ancestor_str
+	     << "' does not appear in the crush map";
+	  break;
+	}
+	int ancestor = newcrush.get_item_id(ancestor_str);
+	err = newcrush.remove_item_under(g_ceph_context, id, ancestor,
+					 unlink_only);
+      } else {
+	err = newcrush.remove_item(g_ceph_context, id, unlink_only);
+      }
+      if (err == -ENOENT) {
+	ss << "item " << id << " does not appear in that position";
+	err = 0;
+	break;
+      }
+      if (err == 0) {
+	pending_inc.crush.clear();
+	newcrush.encode(pending_inc.crush);
+	ss << "removed item id " << id << " name '" << name << "' from crush map";
+	getline(ss, rs);
+	wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      }
+    } while (false);
+
+  } else if (prefix == "osd crush reweight") {
+    do {
+      // osd crush reweight  
+      CrushWrapper newcrush;
+      _get_pending_crush(newcrush);
+
+      string name;
+      cmd_getval(g_ceph_context, cmdmap, "name", name);
+      if (!newcrush.name_exists(name)) {
+	err = -ENOENT;
+	ss << "device '" << name << "' does not appear in the crush map";
+	break;
+      }
+
+      int id = newcrush.get_item_id(name);
+      if (id < 0) {
+	ss << "device '" << name << "' is not a leaf in the crush map";
+	break;
+      }
+      double w;
+      if (!cmd_getval(g_ceph_context, cmdmap, "weight", w)) {
+        ss << "unable to parse weight value '"
+           << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+        err = -EINVAL;
+        break;
+      }
+
+      err = newcrush.adjust_item_weightf(g_ceph_context, id, w);
+      if (err >= 0) {
+	pending_inc.crush.clear();
+	newcrush.encode(pending_inc.crush);
+	ss << "reweighted item id " << id << " name '" << name << "' to " << w
+	   << " in crush map";
+	getline(ss, rs);
+	wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      }
+    } while (false);
+
+  } else if (prefix == "osd crush tunables") {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    err = 0;
+    string profile;
+    cmd_getval(g_ceph_context, cmdmap, "profile", profile);
+    if (profile == "legacy" || profile == "argonaut") {
+      newcrush.set_tunables_legacy();
+    } else if (profile == "bobtail") {
+      newcrush.set_tunables_bobtail();
+    } else if (profile == "firefly") {
+      newcrush.set_tunables_firefly();
+    } else if (profile == "optimal") {
+      newcrush.set_tunables_optimal();
+    } else if (profile == "default") {
+      newcrush.set_tunables_default();
+    } else {
+      ss << "unrecognized profile '" << profile << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    if (!validate_crush_against_features(&newcrush, ss)) {
+      err = -EINVAL;
+      goto reply;
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush);
+    ss << "adjusted tunables profile to " << profile;
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule create-simple") {
+    string name, root, type, mode;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    cmd_getval(g_ceph_context, cmdmap, "root", root);
+    cmd_getval(g_ceph_context, cmdmap, "type", type);
+    cmd_getval(g_ceph_context, cmdmap, "mode", mode);
+    if (mode == "")
+      mode = "firstn";
+
+    if (osdmap.crush->rule_exists(name)) {
+      ss << "rule " << name << " already exists";
+      err = 0;
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (newcrush.rule_exists(name)) {
+      ss << "rule " << name << " already exists";
+      err = 0;
+    } else {
+      int rule = newcrush.add_simple_ruleset(name, root, type, mode,
+					     pg_pool_t::TYPE_REPLICATED, &ss);
+      if (rule < 0) {
+	err = rule;
+	goto reply;
+      }
+
+      pending_inc.crush.clear();
+      newcrush.encode(pending_inc.crush);
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd erasure-code-profile rm") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+
+    if (erasure_code_profile_in_use(pending_inc.new_pools, name, ss))
+      goto wait;
+
+    if (erasure_code_profile_in_use(osdmap.pools, name, ss)) {
+      err = -EBUSY;
+      goto reply;
+    }
+
+    if (osdmap.has_erasure_code_profile(name) ||
+	pending_inc.new_erasure_code_profiles.count(name)) {
+      if (osdmap.has_erasure_code_profile(name)) {
+	pending_inc.old_erasure_code_profiles.push_back(name);
+      } else {
+	dout(20) << "erasure code profile rm " << name << ": creation canceled" << dendl;
+	pending_inc.new_erasure_code_profiles.erase(name);
+      }
+
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+							get_last_committed() + 1));
+      return true;
+    } else {
+      ss << "erasure-code-profile " << name << " does not exist";
+      err = 0;
+      goto reply;
+    }
+
+  } else if (prefix == "osd erasure-code-profile set") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    vector profile;
+    cmd_getval(g_ceph_context, cmdmap, "profile", profile);
+    bool force;
+    if (profile.size() > 0 && profile.back() == "--force") {
+      profile.pop_back();
+      force = true;
+    } else {
+      force = false;
+    }
+    map profile_map;
+    err = parse_erasure_code_profile(profile, &profile_map, ss);
+    if (err)
+      goto reply;
+
+    if (osdmap.has_erasure_code_profile(name)) {
+      if (osdmap.get_erasure_code_profile(name) == profile_map) {
+	err = 0;
+	goto reply;
+      }
+      if (!force) {
+	err = -EPERM;
+	ss << "will not override erasure code profile " << name;
+	goto reply;
+      }
+    }
+
+    if (pending_inc.has_erasure_code_profile(name)) {
+      dout(20) << "erasure code profile " << name << " try again" << dendl;
+      goto wait;
+    } else {
+      dout(20) << "erasure code profile " << name << " set" << dendl;
+      pending_inc.set_erasure_code_profile(name, profile_map);
+    }
+
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+                                                      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule create-erasure") {
+    err = check_cluster_features(CEPH_FEATURE_CRUSH_V2, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string name, poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+    string profile;
+    cmd_getval(g_ceph_context, cmdmap, "profile", profile);
+    if (profile == "")
+      profile = "default";
+    if (profile == "default") {
+      if (!osdmap.has_erasure_code_profile(profile)) {
+	if (pending_inc.has_erasure_code_profile(profile)) {
+	  dout(20) << "erasure code profile " << profile << " already pending" << dendl;
+	  goto wait;
+	}
+
+	map profile_map;
+	err = osdmap.get_erasure_code_profile_default(g_ceph_context,
+						      profile_map,
+						      &ss);
+	if (err)
+	  goto reply;
+	dout(20) << "erasure code profile " << profile << " set" << dendl;
+	pending_inc.set_erasure_code_profile(profile, profile_map);
+	goto wait;
+      }
+    }
+
+    int ruleset;
+    err = crush_ruleset_create_erasure(name, profile, &ruleset, ss);
+    if (err < 0) {
+      switch(err) {
+      case -EEXIST: // return immediately
+	ss << "rule " << name << " already exists";
+	err = 0;
+	goto reply;
+	break;
+      case -EALREADY: // wait for pending to be proposed
+	ss << "rule " << name << " already exists";
+	err = 0;
+	break;
+      default: // non recoverable error
+ 	goto reply;
+	break;
+      }
+    } else {
+      ss << "created ruleset " << name << " at " << ruleset;
+    }
+
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+                                                      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule rm") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "name", name);
+
+    if (!osdmap.crush->rule_exists(name)) {
+      ss << "rule " << name << " does not exist";
+      err = 0;
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (!newcrush.rule_exists(name)) {
+      ss << "rule " << name << " does not exist";
+      err = 0;
+    } else {
+      int ruleno = newcrush.get_rule_id(name);
+      assert(ruleno >= 0);
+
+      // make sure it is not in use.
+      // FIXME: this is ok in some situations, but let's not bother with that
+      // complexity now.
+      int ruleset = newcrush.get_rule_mask_ruleset(ruleno);
+      if (osdmap.crush_ruleset_in_use(ruleset)) {
+	ss << "crush rule " << name << " ruleset " << ruleset << " is in use";
+	err = -EBUSY;
+	goto reply;
+      }
+
+      err = newcrush.remove_rule(ruleno);
+      if (err < 0) {
+	goto reply;
+      }
+
+      pending_inc.crush.clear();
+      newcrush.encode(pending_inc.crush);
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd setmaxosd") {
+    int64_t newmax;
+    if (!cmd_getval(g_ceph_context, cmdmap, "newmax", newmax)) {
+      ss << "unable to parse 'newmax' value '"
+         << cmd_vartype_stringify(cmdmap["newmax"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    if (newmax > g_conf->mon_max_osd) {
+      err = -ERANGE;
+      ss << "cannot set max_osd to " << newmax << " which is > conf.mon_max_osd ("
+	 << g_conf->mon_max_osd << ")";
+      goto reply;
+    }
+
+    pending_inc.new_max_osd = newmax;
+    ss << "set new max_osd = " << pending_inc.new_max_osd;
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd pause") {
+    return prepare_set_flag(m, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+
+  } else if (prefix == "osd unpause") {
+    return prepare_unset_flag(m, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+
+  } else if (prefix == "osd set") {
+    string key;
+    cmd_getval(g_ceph_context, cmdmap, "key", key);
+    if (key == "pause")
+      return prepare_set_flag(m, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+    else if (key == "noup")
+      return prepare_set_flag(m, CEPH_OSDMAP_NOUP);
+    else if (key == "nodown")
+      return prepare_set_flag(m, CEPH_OSDMAP_NODOWN);
+    else if (key == "noout")
+      return prepare_set_flag(m, CEPH_OSDMAP_NOOUT);
+    else if (key == "noin")
+      return prepare_set_flag(m, CEPH_OSDMAP_NOIN);
+    else if (key == "nobackfill")
+      return prepare_set_flag(m, CEPH_OSDMAP_NOBACKFILL);
+    else if (key == "norecover")
+      return prepare_set_flag(m, CEPH_OSDMAP_NORECOVER);
+    else if (key == "noscrub")
+      return prepare_set_flag(m, CEPH_OSDMAP_NOSCRUB);
+    else if (key == "nodeep-scrub")
+      return prepare_set_flag(m, CEPH_OSDMAP_NODEEP_SCRUB);
+    else if (key == "notieragent")
+      return prepare_set_flag(m, CEPH_OSDMAP_NOTIERAGENT);
+    else {
+      ss << "unrecognized flag '" << key << "'";
+      err = -EINVAL;
+    }
+
+  } else if (prefix == "osd unset") {
+    string key;
+    cmd_getval(g_ceph_context, cmdmap, "key", key);
+    if (key == "pause")
+      return prepare_unset_flag(m, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+    else if (key == "noup")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NOUP);
+    else if (key == "nodown")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NODOWN);
+    else if (key == "noout")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NOOUT);
+    else if (key == "noin")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NOIN);
+    else if (key == "nobackfill")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NOBACKFILL);
+    else if (key == "norecover")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NORECOVER);
+    else if (key == "noscrub")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NOSCRUB);
+    else if (key == "nodeep-scrub")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NODEEP_SCRUB);
+    else if (key == "notieragent")
+      return prepare_unset_flag(m, CEPH_OSDMAP_NOTIERAGENT);
+    else {
+      ss << "unrecognized flag '" << key << "'";
+      err = -EINVAL;
+    }
+
+  } else if (prefix == "osd cluster_snap") {
+    // ** DISABLE THIS FOR NOW **
+    ss << "cluster snapshot currently disabled (broken implementation)";
+    // ** DISABLE THIS FOR NOW **
+
+  } else if (prefix == "osd down" ||
+	     prefix == "osd out" ||
+	     prefix == "osd in" ||
+	     prefix == "osd rm") {
+
+    bool any = false;
+
+    vector idvec;
+    cmd_getval(g_ceph_context, cmdmap, "ids", idvec);
+    for (unsigned j = 0; j < idvec.size(); j++) {
+      long osd = parse_osd_id(idvec[j].c_str(), &ss);
+      if (osd < 0) {
+	ss << "invalid osd id" << osd;
+	err = -EINVAL;
+	continue;
+      } else if (!osdmap.exists(osd)) {
+	ss << "osd." << osd << " does not exist. ";
+	continue;
+      }
+      if (prefix == "osd down") {
+	if (osdmap.is_down(osd)) {
+	  ss << "osd." << osd << " is already down. ";
+	} else {
+	  pending_inc.new_state[osd] = CEPH_OSD_UP;
+	  ss << "marked down osd." << osd << ". ";
+	  any = true;
+	}
+      } else if (prefix == "osd out") {
+	if (osdmap.is_out(osd)) {
+	  ss << "osd." << osd << " is already out. ";
+	} else {
+	  pending_inc.new_weight[osd] = CEPH_OSD_OUT;
+	  ss << "marked out osd." << osd << ". ";
+	  any = true;
+	}
+      } else if (prefix == "osd in") {
+	if (osdmap.is_in(osd)) {
+	  ss << "osd." << osd << " is already in. ";
+	} else {
+	  pending_inc.new_weight[osd] = CEPH_OSD_IN;
+	  ss << "marked in osd." << osd << ". ";
+	  any = true;
+	}
+      } else if (prefix == "osd rm") {
+	if (osdmap.is_up(osd)) {
+	  if (any)
+	    ss << ", ";
+          ss << "osd." << osd << " is still up; must be down before removal. ";
+	  err = -EBUSY;
+	} else {
+	  pending_inc.new_state[osd] = osdmap.get_state(osd);
+          pending_inc.new_uuid[osd] = uuid_d();
+	  pending_metadata_rm.insert(osd);
+	  if (any) {
+	    ss << ", osd." << osd;
+          } else {
+	    ss << "removed osd." << osd;
+          }
+	  any = true;
+	}
+      }
+    }
+    if (any) {
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, err, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+  } else if (prefix == "osd pg-temp") {
+    string pgidstr;
+    if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) {
+      ss << "unable to parse 'pgid' value '"
+         << cmd_vartype_stringify(cmdmap["pgid"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    pg_t pgid;
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    PGMap& pg_map = mon->pgmon()->pg_map;
+    if (!pg_map.pg_stat.count(pgid)) {
+      ss << "pg " << pgid << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    vector id_vec;
+    vector new_pg_temp;
+    if (!cmd_getval(g_ceph_context, cmdmap, "id", id_vec)) {
+      ss << "unable to parse 'id' value(s) '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    for (unsigned i = 0; i < id_vec.size(); i++) {
+      int32_t osd = parse_osd_id(id_vec[i].c_str(), &ss);
+      if (osd < 0) {
+        err = -EINVAL;
+        goto reply;
+      }
+      if (!osdmap.exists(osd)) {
+        ss << "osd." << osd << " does not exist";
+        err = -ENOENT;
+        goto reply;
+      }
+
+      new_pg_temp.push_back(osd);
+    }
+
+    pending_inc.new_pg_temp[pgid] = new_pg_temp;
+    ss << "set " << pgid << " pg_temp mapping to " << new_pg_temp;
+    goto update;
+  } else if (prefix == "osd primary-temp") {
+    string pgidstr;
+    if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) {
+      ss << "unable to parse 'pgid' value '"
+         << cmd_vartype_stringify(cmdmap["pgid"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    pg_t pgid;
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    PGMap& pg_map = mon->pgmon()->pg_map;
+    if (!pg_map.pg_stat.count(pgid)) {
+      ss << "pg " << pgid << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    string id;
+    int32_t osd;
+    if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+      ss << "unable to parse 'id' value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (strcmp(id.c_str(), "-1")) {
+      osd = parse_osd_id(id.c_str(), &ss);
+      if (osd < 0) {
+        err = -EINVAL;
+        goto reply;
+      }
+      if (!osdmap.exists(osd)) {
+        ss << "osd." << osd << " does not exist";
+        err = -ENOENT;
+        goto reply;
+      }
+    } else {
+      osd = -1;
+    }
+
+    if (!g_conf->mon_osd_allow_primary_temp) {
+      ss << "you must enable 'mon osd allow primary temp = true' on the mons before you can set primary_temp mappings.  note that this is for developers only: older clients/OSDs will break and there is no feature bit infrastructure in place.";
+      err = -EPERM;
+      goto reply;
+    }
+
+    pending_inc.new_primary_temp[pgid] = osd;
+    ss << "set " << pgid << " primary_temp mapping to " << osd;
+    goto update;
+  } else if (prefix == "osd primary-affinity") {
+    int64_t id;
+    if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+      ss << "invalid osd id value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    double w;
+    if (!cmd_getval(g_ceph_context, cmdmap, "weight", w)) {
+      ss << "unable to parse 'weight' value '"
+           << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    long ww = (int)((double)CEPH_OSD_MAX_PRIMARY_AFFINITY*w);
+    if (ww < 0L) {
+      ss << "weight must be >= 0";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!g_conf->mon_osd_allow_primary_affinity) {
+      ss << "you must enable 'mon osd allow primary affinity = true' on the mons before you can adjust primary-affinity.  note that older clients will no longer be able to communicate with the cluster.";
+      err = -EPERM;
+      goto reply;
+    }
+    err = check_cluster_features(CEPH_FEATURE_OSD_PRIMARY_AFFINITY, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err < 0)
+      goto reply;
+    if (osdmap.exists(id)) {
+      pending_inc.new_primary_affinity[id] = ww;
+      ss << "set osd." << id << " primary-affinity to " << w << " (" << ios::hex << ww << ios::dec << ")";
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+                                                get_last_committed() + 1));
+      return true;
+    }
+  } else if (prefix == "osd reweight") {
+    int64_t id;
+    if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    double w;
+    if (!cmd_getval(g_ceph_context, cmdmap, "weight", w)) {
+      ss << "unable to parse weight value '"
+         << cmd_vartype_stringify(cmdmap["weight"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    long ww = (int)((double)CEPH_OSD_IN*w);
+    if (ww < 0L) {
+      ss << "weight must be >= 0";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (osdmap.exists(id)) {
+      pending_inc.new_weight[id] = ww;
+      ss << "reweighted osd." << id << " to " << w << " (" << ios::hex << ww << ios::dec << ")";
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+
+  } else if (prefix == "osd lost") {
+    int64_t id;
+    if (!cmd_getval(g_ceph_context, cmdmap, "id", id)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    string sure;
+    if (!cmd_getval(g_ceph_context, cmdmap, "sure", sure) || sure != "--yes-i-really-mean-it") {
+      ss << "are you SURE?  this might mean real, permanent data loss.  pass "
+	    "--yes-i-really-mean-it if you really do.";
+      err = -EPERM;
+      goto reply;
+    } else if (!osdmap.exists(id) || !osdmap.is_down(id)) {
+      ss << "osd." << id << " is not down or doesn't exist";
+    } else {
+      epoch_t e = osdmap.get_info(id).down_at;
+      pending_inc.new_lost[id] = e;
+      ss << "marked osd lost in epoch " << e;
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+
+  } else if (prefix == "osd create") {
+    int i = -1;
+
+    // optional uuid provided?
+    uuid_d uuid;
+    string uuidstr;
+    if (cmd_getval(g_ceph_context, cmdmap, "uuid", uuidstr)) {
+      if (!uuid.parse(uuidstr.c_str())) {
+	err = -EINVAL;
+	goto reply;
+      }
+      dout(10) << " osd create got uuid " << uuid << dendl;
+      i = osdmap.identify_osd(uuid);
+      if (i >= 0) {
+	// osd already exists
+	err = 0;
+	if (f) {
+	  f->open_object_section("created_osd");
+	  f->dump_int("osdid", i);
+	  f->close_section();
+	  f->flush(rdata);
+	} else {
+	  ss << i;
+	  rdata.append(ss);
+	}
+	goto reply;
+      }
+      i = pending_inc.identify_osd(uuid);
+      if (i >= 0) {
+	// osd is about to exist
+	wait_for_finished_proposal(new C_RetryMessage(this, m));
+	return true;
+      }
+    }
+
+    // allocate a new id
+    for (i=0; i < osdmap.get_max_osd(); i++) {
+      if (!osdmap.exists(i) &&
+	  pending_inc.new_up_client.count(i) == 0 &&
+	  (pending_inc.new_state.count(i) == 0 ||
+	   (pending_inc.new_state[i] & CEPH_OSD_EXISTS) == 0))
+	goto done;
+    }
+
+    // raise max_osd
+    if (pending_inc.new_max_osd < 0)
+      pending_inc.new_max_osd = osdmap.get_max_osd() + 1;
+    else
+      pending_inc.new_max_osd++;
+    i = pending_inc.new_max_osd - 1;
+
+done:
+    dout(10) << " creating osd." << i << dendl;
+    pending_inc.new_state[i] |= CEPH_OSD_EXISTS | CEPH_OSD_NEW;
+    if (!uuid.is_zero())
+      pending_inc.new_uuid[i] = uuid;
+    if (f) {
+      f->open_object_section("created_osd");
+      f->dump_int("osdid", i);
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ss << i;
+      rdata.append(ss);
+    }
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, rdata,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd blacklist") {
+    string addrstr;
+    cmd_getval(g_ceph_context, cmdmap, "addr", addrstr);
+    entity_addr_t addr;
+    if (!addr.parse(addrstr.c_str(), 0))
+      ss << "unable to parse address " << addrstr;
+    else {
+      string blacklistop;
+      cmd_getval(g_ceph_context, cmdmap, "blacklistop", blacklistop);
+      if (blacklistop == "add") {
+	utime_t expires = ceph_clock_now(g_ceph_context);
+	double d;
+	// default one hour
+	cmd_getval(g_ceph_context, cmdmap, "expire", d, double(60*60));
+	expires += d;
+
+	pending_inc.new_blacklist[addr] = expires;
+	ss << "blacklisting " << addr << " until " << expires << " (" << d << " sec)";
+	getline(ss, rs);
+	wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      } else if (blacklistop == "rm") {
+	if (osdmap.is_blacklisted(addr) ||
+	    pending_inc.new_blacklist.count(addr)) {
+	  if (osdmap.is_blacklisted(addr))
+	    pending_inc.old_blacklist.push_back(addr);
+	  else
+	    pending_inc.new_blacklist.erase(addr);
+	  ss << "un-blacklisting " << addr;
+	  getline(ss, rs);
+	  wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						    get_last_committed() + 1));
+	  return true;
+	}
+	ss << addr << " isn't blacklisted";
+	err = 0;
+	goto reply;
+      }
+    }
+  } else if (prefix == "osd pool mksnap") {
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string snapname;
+    cmd_getval(g_ceph_context, cmdmap, "snap", snapname);
+    const pg_pool_t *p = osdmap.get_pg_pool(pool);
+    if (p->is_unmanaged_snaps_mode()) {
+      ss << "pool " << poolstr << " is in unmanaged snaps mode";
+      err = -EINVAL;
+      goto reply;
+    } else if (p->snap_exists(snapname.c_str())) {
+      ss << "pool " << poolstr << " snap " << snapname << " already exists";
+      err = 0;
+      goto reply;
+    }
+    pg_pool_t *pp = 0;
+    if (pending_inc.new_pools.count(pool))
+      pp = &pending_inc.new_pools[pool];
+    if (!pp) {
+      pp = &pending_inc.new_pools[pool];
+      *pp = *p;
+    }
+    if (pp->snap_exists(snapname.c_str())) {
+      ss << "pool " << poolstr << " snap " << snapname << " already exists";
+    } else {
+      pp->add_snap(snapname.c_str(), ceph_clock_now(g_ceph_context));
+      pp->set_snap_epoch(pending_inc.epoch);
+      ss << "created pool " << poolstr << " snap " << snapname;
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pool rmsnap") {
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string snapname;
+    cmd_getval(g_ceph_context, cmdmap, "snap", snapname);
+    const pg_pool_t *p = osdmap.get_pg_pool(pool);
+    if (p->is_unmanaged_snaps_mode()) {
+      ss << "pool " << poolstr << " is in unmanaged snaps mode";
+      err = -EINVAL;
+      goto reply;
+    } else if (!p->snap_exists(snapname.c_str())) {
+      ss << "pool " << poolstr << " snap " << snapname << " does not exist";
+      err = 0;
+      goto reply;
+    }
+    pg_pool_t *pp = 0;
+    if (pending_inc.new_pools.count(pool))
+      pp = &pending_inc.new_pools[pool];
+    if (!pp) {
+      pp = &pending_inc.new_pools[pool];
+      *pp = *p;
+    }
+    snapid_t sn = pp->snap_exists(snapname.c_str());
+    if (sn) {
+      pp->remove_snap(sn);
+      pp->set_snap_epoch(pending_inc.epoch);
+      ss << "removed pool " << poolstr << " snap " << snapname;
+    } else {
+      ss << "already removed pool " << poolstr << " snap " << snapname;
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pool create") {
+    int64_t  pg_num;
+    int64_t pgp_num;
+    cmd_getval(g_ceph_context, cmdmap, "pg_num", pg_num, int64_t(0));
+    if ((pg_num == 0) || (pg_num > g_conf->mon_max_pool_pg_num)) {
+      ss << "'pg_num' must be greater than 0 and less than or equal to "
+	 << g_conf->mon_max_pool_pg_num
+	 << " (you may adjust 'mon max pool pg num' for higher values)";
+      err = -ERANGE;
+      goto reply;
+    }
+
+    cmd_getval(g_ceph_context, cmdmap, "pgp_num", pgp_num, pg_num);
+    if ((pgp_num == 0) || (pgp_num > pg_num)) {
+      ss << "'pgp_num' must be greater than 0 and lower or equal than 'pg_num'"
+	 << ", which in this case is " << pg_num;
+      err = -ERANGE;
+      goto reply;
+    }
+
+    string pool_type_str;
+    cmd_getval(g_ceph_context, cmdmap, "pool_type", pool_type_str);
+    if (pool_type_str.empty())
+      pool_type_str = pg_pool_t::get_default_type();
+
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id >= 0) {
+      const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+      if (pool_type_str != p->get_type_name()) {
+	ss << "pool '" << poolstr << "' cannot change to type " << pool_type_str;
+ 	err = -EINVAL;
+      } else {
+	ss << "pool '" << poolstr << "' already exists";
+	err = 0;
+      }
+      goto reply;
+    }
+
+    int pool_type;
+    if (pool_type_str == "replicated") {
+      pool_type = pg_pool_t::TYPE_REPLICATED;
+    } else if (pool_type_str == "erasure") {
+      err = check_cluster_features(CEPH_FEATURE_CRUSH_V2 |
+				   CEPH_FEATURE_OSD_ERASURE_CODES,
+				   ss);
+      if (err == -EAGAIN)
+	goto wait;
+      if (err)
+	goto reply;
+      pool_type = pg_pool_t::TYPE_ERASURE;
+    } else {
+      ss << "unknown pool type '" << pool_type_str << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    string ruleset_name;
+    cmd_getval(g_ceph_context, cmdmap, "ruleset", ruleset_name);
+    string erasure_code_profile;
+    cmd_getval(g_ceph_context, cmdmap, "erasure_code_profile", erasure_code_profile);
+    if (erasure_code_profile == "")
+      erasure_code_profile = "default";
+    if (erasure_code_profile == "default") {
+      if (!osdmap.has_erasure_code_profile(erasure_code_profile)) {
+	if (pending_inc.has_erasure_code_profile(erasure_code_profile)) {
+	  dout(20) << "erasure code profile " << erasure_code_profile << " already pending" << dendl;
+	  goto wait;
+	}
+
+	map profile_map;
+	err = osdmap.get_erasure_code_profile_default(g_ceph_context,
+						      profile_map,
+						      &ss);
+	if (err)
+	  goto reply;
+	dout(20) << "erasure code profile " << erasure_code_profile << " set" << dendl;
+	pending_inc.set_erasure_code_profile(erasure_code_profile, profile_map);
+	goto wait;
+      }
+    }
+
+    if (ruleset_name == "") {
+      if (erasure_code_profile == "default") {
+	ruleset_name = "erasure-code";
+      } else {
+	dout(1) << "implicitly use ruleset named after the pool: "
+		<< poolstr << dendl;
+	ruleset_name = poolstr;
+      }
+    }
+
+    err = prepare_new_pool(poolstr, 0, // auid=0 for admin created pool
+			   -1, // default crush rule
+			   ruleset_name,
+			   pg_num, pgp_num,
+			   erasure_code_profile, pool_type,
+			   ss);
+    if (err < 0) {
+      switch(err) {
+      case -EEXIST:
+	ss << "pool '" << poolstr << "' already exists";
+	break;
+      case -EAGAIN:
+	wait_for_finished_proposal(new C_RetryMessage(this, m));
+	return true;
+      default:
+	goto reply;
+	break;
+      }
+    } else {
+      ss << "pool '" << poolstr << "' created";
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd pool delete") {
+    // osd pool delete   --yes-i-really-really-mean-it
+    string poolstr, poolstr2, sure;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    cmd_getval(g_ceph_context, cmdmap, "pool2", poolstr2);
+    cmd_getval(g_ceph_context, cmdmap, "sure", sure);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "pool '" << poolstr << "' does not exist";
+      err = 0;
+      goto reply;
+    }
+
+    if (poolstr2 != poolstr || sure != "--yes-i-really-really-mean-it") {
+      ss << "WARNING: this will *PERMANENTLY DESTROY* all data stored in pool " << poolstr
+	 << ".  If you are *ABSOLUTELY CERTAIN* that is what you want, pass the pool name *twice*, "
+	 << "followed by --yes-i-really-really-mean-it.";
+      err = -EPERM;
+      goto reply;
+    }
+    err = _prepare_remove_pool(pool, &ss);
+    if (err == -EAGAIN) {
+      wait_for_finished_proposal(new C_RetryMessage(this, m));
+      return true;
+    }
+    if (err < 0)
+      goto reply;
+    goto update;
+  } else if (prefix == "osd pool rename") {
+    string srcpoolstr, destpoolstr;
+    cmd_getval(g_ceph_context, cmdmap, "srcpool", srcpoolstr);
+    cmd_getval(g_ceph_context, cmdmap, "destpool", destpoolstr);
+    int64_t pool_src = osdmap.lookup_pg_pool_name(srcpoolstr.c_str());
+    int64_t pool_dst = osdmap.lookup_pg_pool_name(destpoolstr.c_str());
+
+    if (pool_src < 0) {
+      if (pool_dst >= 0) {
+        // src pool doesn't exist, dst pool does exist: to ensure idempotency
+        // of operations, assume this rename succeeded, as it is not changing
+        // the current state.  Make sure we output something understandable
+        // for whoever is issuing the command, if they are paying attention,
+        // in case it was not intentional; or to avoid a "wtf?" and a bug
+        // report in case it was intentional, while expecting a failure.
+        ss << "pool '" << srcpoolstr << "' does not exist; pool '"
+          << destpoolstr << "' does -- assuming successful rename";
+        err = 0;
+      } else {
+        ss << "unrecognized pool '" << srcpoolstr << "'";
+        err = -ENOENT;
+      }
+      goto reply;
+    } else if (pool_dst >= 0) {
+      // source pool exists and so does the destination pool
+      ss << "pool '" << destpoolstr << "' already exists";
+      err = -EEXIST;
+      goto reply;
+    }
+
+    int ret = _prepare_rename_pool(pool_src, destpoolstr);
+    if (ret == 0) {
+      ss << "pool '" << srcpoolstr << "' renamed to '" << destpoolstr << "'";
+    } else {
+      ss << "failed to rename pool '" << srcpoolstr << "' to '" << destpoolstr << "': "
+        << cpp_strerror(ret);
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, ret, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd pool set") {
+    err = prepare_command_pool_set(cmdmap, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err < 0)
+      goto reply;
+
+    getline(ss, rs);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						   get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier add") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string tierpoolstr;
+    cmd_getval(g_ceph_context, cmdmap, "tierpool", tierpoolstr);
+    int64_t tierpool_id = osdmap.lookup_pg_pool_name(tierpoolstr);
+    if (tierpool_id < 0) {
+      ss << "unrecognized pool '" << tierpoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    assert(p);
+    const pg_pool_t *tp = osdmap.get_pg_pool(tierpool_id);
+    assert(tp);
+    if (p->tiers.count(tierpool_id)) {
+      assert(tp->tier_of == pool_id);
+      err = 0;
+      ss << "pool '" << tierpoolstr << "' is now (or already was) a tier of '" << poolstr << "'";
+      goto reply;
+    }
+    if (tp->is_tier()) {
+      ss << "tier pool '" << tierpoolstr << "' is already a tier of '"
+	 << osdmap.get_pool_name(tp->tier_of) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    // make sure new tier is empty
+    string force_nonempty;
+    cmd_getval(g_ceph_context, cmdmap, "force_nonempty", force_nonempty);
+    const pool_stat_t& tier_stats =
+      mon->pgmon()->pg_map.get_pg_pool_sum_stat(tierpool_id);
+    if (tier_stats.stats.sum.num_objects != 0 &&
+	force_nonempty != "--force-nonempty") {
+      ss << "tier pool '" << tierpoolstr << "' is not empty; --force-nonempty to force";
+      err = -ENOTEMPTY;
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+    if (np->tiers.count(tierpool_id) || ntp->is_tier()) {
+      wait_for_finished_proposal(new C_RetryMessage(this, m));
+      return true;
+    }
+    np->tiers.insert(tierpool_id);
+    np->set_snap_epoch(pending_inc.epoch); // tier will update to our snap info
+    ntp->tier_of = pool_id;
+    ss << "pool '" << tierpoolstr << "' is now (or already was) a tier of '" << poolstr << "'";
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier remove") {
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string tierpoolstr;
+    cmd_getval(g_ceph_context, cmdmap, "tierpool", tierpoolstr);
+    int64_t tierpool_id = osdmap.lookup_pg_pool_name(tierpoolstr);
+    if (tierpool_id < 0) {
+      ss << "unrecognized pool '" << tierpoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    assert(p);
+    const pg_pool_t *tp = osdmap.get_pg_pool(tierpool_id);
+    assert(tp);
+    if (p->tiers.count(tierpool_id) == 0) {
+      ss << "pool '" << tierpoolstr << "' is now (or already was) not a tier of '" << poolstr << "'";
+      err = 0;
+      goto reply;
+    }
+    if (tp->tier_of != pool_id) {
+      ss << "tier pool '" << tierpoolstr << "' is a tier of '"
+         << osdmap.get_pool_name(tp->tier_of) << "': "
+         // be scary about it; this is an inconsistency and bells must go off
+         << "THIS SHOULD NOT HAVE HAPPENED AT ALL";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (p->read_tier == tierpool_id) {
+      ss << "tier pool '" << tierpoolstr << "' is the overlay for '" << poolstr << "'; please remove-overlay first";
+      err = -EBUSY;
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+    if (np->tiers.count(tierpool_id) == 0 ||
+	ntp->tier_of != pool_id ||
+	np->read_tier == tierpool_id) {
+      wait_for_finished_proposal(new C_RetryMessage(this, m));
+      return true;
+    }
+    np->tiers.erase(tierpool_id);
+    ntp->clear_tier();
+    ss << "pool '" << tierpoolstr << "' is now (or already was) not a tier of '" << poolstr << "'";
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier set-overlay") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string overlaypoolstr;
+    cmd_getval(g_ceph_context, cmdmap, "overlaypool", overlaypoolstr);
+    int64_t overlaypool_id = osdmap.lookup_pg_pool_name(overlaypoolstr);
+    if (overlaypool_id < 0) {
+      ss << "unrecognized pool '" << overlaypoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    assert(p);
+    if (p->tiers.count(overlaypool_id) == 0) {
+      ss << "tier pool '" << overlaypoolstr << "' is not a tier of '" << poolstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (p->read_tier == overlaypool_id) {
+      err = 0;
+      ss << "overlay for '" << poolstr << "' is now (or already was) '" << overlaypoolstr << "'";
+      goto reply;
+    }
+    if (p->has_read_tier()) {
+      ss << "pool '" << poolstr << "' has overlay '"
+	 << osdmap.get_pool_name(p->read_tier)
+	 << "'; please remove-overlay first";
+      err = -EINVAL;
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    np->read_tier = overlaypool_id;
+    np->write_tier = overlaypool_id;
+    np->last_force_op_resend = pending_inc.epoch;
+    ss << "overlay for '" << poolstr << "' is now (or already was) '" << overlaypoolstr << "'";
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier remove-overlay") {
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    assert(p);
+    if (!p->has_read_tier()) {
+      err = 0;
+      ss << "there is now (or already was) no overlay for '" << poolstr << "'";
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    np->clear_read_tier();
+    np->clear_write_tier();
+    np->last_force_op_resend = pending_inc.epoch;
+    ss << "there is now (or already was) no overlay for '" << poolstr << "'";
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier cache-mode") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    assert(p);
+    if (!p->is_tier()) {
+      ss << "pool '" << poolstr << "' is not a tier";
+      err = -EINVAL;
+      goto reply;
+    }
+    string modestr;
+    cmd_getval(g_ceph_context, cmdmap, "mode", modestr);
+    pg_pool_t::cache_mode_t mode = pg_pool_t::get_cache_mode_from_str(modestr);
+    if (mode < 0) {
+      ss << "'" << modestr << "' is not a valid cache mode";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    // pool already has this cache-mode set and there are no pending changes
+    if (p->cache_mode == mode &&
+	(pending_inc.new_pools.count(pool_id) == 0 ||
+	 pending_inc.new_pools[pool_id].cache_mode == p->cache_mode)) {
+      ss << "set cache-mode for pool '" << poolstr << "'"
+         << " to " << pg_pool_t::get_cache_mode_name(mode);
+      err = 0;
+      goto reply;
+    }
+
+    /* Mode description:
+     *
+     *  none:       No cache-mode defined
+     *  forward:    Forward all reads and writes to base pool
+     *  writeback:  Cache writes, promote reads from base pool
+     *  readonly:   Forward writes to base pool
+     *
+     * Hence, these are the allowed transitions:
+     *
+     *  none -> any
+     *  forward -> writeback || any IF num_objects_dirty == 0
+     *  writeback -> forward
+     *  readonly -> any
+     */
+
+    // We check if the transition is valid against the current pool mode, as
+    // it is the only committed state thus far.  We will blantly squash
+    // whatever mode is on the pending state.
+
+    if (p->cache_mode == pg_pool_t::CACHEMODE_WRITEBACK &&
+        mode != pg_pool_t::CACHEMODE_FORWARD) {
+      ss << "unable to set cache-mode '" << pg_pool_t::get_cache_mode_name(mode)
+         << "' on a '" << pg_pool_t::get_cache_mode_name(p->cache_mode)
+         << "' pool; only '"
+         << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_FORWARD)
+        << "' allowed.";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (p->cache_mode == pg_pool_t::CACHEMODE_FORWARD &&
+               mode != pg_pool_t::CACHEMODE_WRITEBACK) {
+
+      const pool_stat_t& tier_stats =
+        mon->pgmon()->pg_map.get_pg_pool_sum_stat(pool_id);
+
+      if (tier_stats.stats.sum.num_objects_dirty > 0) {
+        ss << "unable to set cache-mode '"
+           << pg_pool_t::get_cache_mode_name(mode) << "' on pool '" << poolstr
+           << "': dirty objects found";
+        err = -EBUSY;
+        goto reply;
+      }
+    }
+
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    np->cache_mode = mode;
+    // set this both when moving to and from cache_mode NONE.  this is to
+    // capture legacy pools that were set up before this flag existed.
+    np->flags |= pg_pool_t::FLAG_INCOMPLETE_CLONES;
+    ss << "set cache-mode for pool '" << poolstr
+	<< "' to " << pg_pool_t::get_cache_mode_name(mode);
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier add-cache") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string tierpoolstr;
+    cmd_getval(g_ceph_context, cmdmap, "tierpool", tierpoolstr);
+    int64_t tierpool_id = osdmap.lookup_pg_pool_name(tierpoolstr);
+    if (tierpool_id < 0) {
+      ss << "unrecognized pool '" << tierpoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    assert(p);
+    const pg_pool_t *tp = osdmap.get_pg_pool(tierpool_id);
+    assert(tp);
+    if (p->tiers.count(tierpool_id)) {
+      assert(tp->tier_of == pool_id);
+      err = 0;
+      ss << "pool '" << tierpoolstr << "' is now (or already was) a tier of '" << poolstr << "'";
+      goto reply;
+    }
+    if (tp->is_tier()) {
+      ss << "tier pool '" << tierpoolstr << "' is already a tier of '"
+	 << osdmap.get_pool_name(tp->tier_of) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    int64_t size = 0;
+    if (!cmd_getval(g_ceph_context, cmdmap, "size", size)) {
+      ss << "unable to parse 'size' value '"
+         << cmd_vartype_stringify(cmdmap["size"]) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    // make sure new tier is empty
+    const pool_stat_t& tier_stats =
+      mon->pgmon()->pg_map.get_pg_pool_sum_stat(tierpool_id);
+    if (tier_stats.stats.sum.num_objects != 0) {
+      ss << "tier pool '" << tierpoolstr << "' is not empty";
+      err = -ENOTEMPTY;
+      goto reply;
+    }
+    string modestr = g_conf->osd_tier_default_cache_mode;
+    pg_pool_t::cache_mode_t mode = pg_pool_t::get_cache_mode_from_str(modestr);
+    if (mode < 0) {
+      ss << "osd tier cache default mode '" << modestr << "' is not a valid cache mode";
+      err = -EINVAL;
+      goto reply;
+    }
+    HitSet::Params hsp;
+    if (g_conf->osd_tier_default_cache_hit_set_type == "bloom") {
+      BloomHitSet::Params *bsp = new BloomHitSet::Params;
+      bsp->set_fpp(g_conf->osd_pool_default_hit_set_bloom_fpp);
+      hsp = HitSet::Params(bsp);
+    } else if (g_conf->osd_tier_default_cache_hit_set_type == "explicit_hash") {
+      hsp = HitSet::Params(new ExplicitHashHitSet::Params);
+    }
+    else if (g_conf->osd_tier_default_cache_hit_set_type == "explicit_object") {
+      hsp = HitSet::Params(new ExplicitObjectHitSet::Params);
+    } else {
+      ss << "osd tier cache default hit set type '" <<
+	g_conf->osd_tier_default_cache_hit_set_type << "' is not a known type";
+      err = -EINVAL;
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+    if (np->tiers.count(tierpool_id) || ntp->is_tier()) {
+      wait_for_finished_proposal(new C_RetryMessage(this, m));
+      return true;
+    }
+    np->tiers.insert(tierpool_id);
+    np->set_snap_epoch(pending_inc.epoch); // tier will update to our snap info
+    ntp->tier_of = pool_id;
+    ntp->cache_mode = mode;
+    ntp->hit_set_count = g_conf->osd_tier_default_cache_hit_set_count;
+    ntp->hit_set_period = g_conf->osd_tier_default_cache_hit_set_period;
+    ntp->hit_set_params = hsp;
+    ntp->target_max_bytes = size;
+    ss << "pool '" << tierpoolstr << "' is now (or already was) a cache tier of '" << poolstr << "'";
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pool set-quota") {
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    string field;
+    cmd_getval(g_ceph_context, cmdmap, "field", field);
+    if (field != "max_objects" && field != "max_bytes") {
+      ss << "unrecognized field '" << field << "'; max_bytes of max_objects";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    // val could contain unit designations, so we treat as a string
+    string val;
+    cmd_getval(g_ceph_context, cmdmap, "val", val);
+    stringstream tss;
+    int64_t value = unit_to_bytesize(val, &tss);
+    if (value < 0) {
+      ss << "error parsing value '" << value << "': " << tss.str();
+      err = value;
+      goto reply;
+    }
+
+    pg_pool_t *pi = pending_inc.get_new_pool(pool_id, osdmap.get_pg_pool(pool_id));
+    if (field == "max_objects") {
+      pi->quota_max_objects = value;
+    } else if (field == "max_bytes") {
+      pi->quota_max_bytes = value;
+    } else {
+      assert(0 == "unrecognized option");
+    }
+    ss << "set-quota " << field << " = " << value << " for pool " << poolstr;
+    rs = ss.str();
+    wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd reweight-by-utilization") {
+    int64_t oload;
+    cmd_getval(g_ceph_context, cmdmap, "oload", oload, int64_t(120));
+    string out_str;
+    err = reweight_by_utilization(oload, out_str);
+    if (err < 0) {
+      ss << "FAILED reweight-by-utilization: " << out_str;
+    }
+    else if (err == 0) {
+      ss << "no change: " << out_str;
+    } else {
+      ss << "SUCCESSFUL reweight-by-utilization: " << out_str;
+      getline(ss, rs);
+      wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+  } else if (prefix == "osd thrash") {
+    int64_t num_epochs;
+    cmd_getval(g_ceph_context, cmdmap, "num_epochs", num_epochs, int64_t(0));
+    // thrash_map is a member var
+    thrash_map = num_epochs;
+    ss << "will thrash map for " << thrash_map << " epochs";
+    ret = thrash();
+    err = 0;
+  } else {
+    err = -EINVAL;
+  }
+
+ reply:
+  getline(ss, rs);
+  if (err < 0 && rs.length() == 0)
+    rs = cpp_strerror(err);
+  mon->reply_command(m, err, rs, rdata, get_last_committed());
+  return ret;
+
+ update:
+  getline(ss, rs);
+  wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs,
+					    get_last_committed() + 1));
+  return true;
+
+ wait:
+  wait_for_finished_proposal(new C_RetryMessage(this, m));
+  return true;
+}
+
+bool OSDMonitor::preprocess_pool_op(MPoolOp *m) 
+{
+  if (m->op == POOL_OP_CREATE)
+    return preprocess_pool_op_create(m);
+
+  if (!osdmap.get_pg_pool(m->pool)) {
+    dout(10) << "attempt to delete non-existent pool id " << m->pool << dendl;
+    _pool_op_reply(m, 0, osdmap.get_epoch());
+    return true;
+  }
+
+  // check if the snap and snapname exists
+  bool snap_exists = false;
+  const pg_pool_t *p = osdmap.get_pg_pool(m->pool);
+  if (p->snap_exists(m->name.c_str()))
+    snap_exists = true;
+  
+  switch (m->op) {
+  case POOL_OP_CREATE_SNAP:
+    if (p->is_unmanaged_snaps_mode()) {
+      _pool_op_reply(m, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    if (snap_exists) {
+      _pool_op_reply(m, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_CREATE_UNMANAGED_SNAP:
+    if (p->is_pool_snaps_mode()) {
+      _pool_op_reply(m, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_DELETE_SNAP:
+    if (p->is_unmanaged_snaps_mode()) {
+      _pool_op_reply(m, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    if (!snap_exists) {
+      _pool_op_reply(m, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_DELETE_UNMANAGED_SNAP:
+    if (p->is_pool_snaps_mode()) {
+      _pool_op_reply(m, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    if (p->is_removed_snap(m->snapid)) {
+      _pool_op_reply(m, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_DELETE:
+    if (osdmap.lookup_pg_pool_name(m->name.c_str()) >= 0) {
+      _pool_op_reply(m, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_AUID_CHANGE:
+    return false;
+  default:
+    assert(0);
+    break;
+  }
+
+  return false;
+}
+
+bool OSDMonitor::preprocess_pool_op_create(MPoolOp *m)
+{
+  MonSession *session = m->get_session();
+  if (!session) {
+    _pool_op_reply(m, -EPERM, osdmap.get_epoch());
+    return true;
+  }
+  if (!session->is_capable("osd", MON_CAP_W)) {
+    dout(5) << "attempt to create new pool without sufficient auid privileges!"
+	    << "message: " << *m  << std::endl
+	    << "caps: " << session->caps << dendl;
+    _pool_op_reply(m, -EPERM, osdmap.get_epoch());
+    return true;
+  }
+
+  int64_t pool = osdmap.lookup_pg_pool_name(m->name.c_str());
+  if (pool >= 0) {
+    _pool_op_reply(m, 0, osdmap.get_epoch());
+    return true;
+  }
+
+  return false;
+}
+
+bool OSDMonitor::prepare_pool_op(MPoolOp *m)
+{
+  dout(10) << "prepare_pool_op " << *m << dendl;
+  if (m->op == POOL_OP_CREATE) {
+    return prepare_pool_op_create(m);
+  } else if (m->op == POOL_OP_DELETE) {
+    return prepare_pool_op_delete(m);
+  }
+
+  int ret = 0;
+  bool changed = false;
+
+  if (!osdmap.have_pg_pool(m->pool)) {
+    _pool_op_reply(m, -ENOENT, osdmap.get_epoch());
+    return false;
+  }
+
+  const pg_pool_t *pool = osdmap.get_pg_pool(m->pool);
+
+  switch (m->op) {
+    case POOL_OP_CREATE_SNAP:
+    case POOL_OP_DELETE_SNAP:
+      if (!pool->is_unmanaged_snaps_mode()) {
+        bool snap_exists = pool->snap_exists(m->name.c_str());
+        if ((m->op == POOL_OP_CREATE_SNAP && snap_exists)
+          || (m->op == POOL_OP_DELETE_SNAP && !snap_exists)) {
+          ret = 0;
+        } else {
+          break;
+        }
+      } else {
+        ret = -EINVAL;
+      }
+      _pool_op_reply(m, ret, osdmap.get_epoch());
+      return false;
+
+    case POOL_OP_DELETE_UNMANAGED_SNAP:
+      // we won't allow removal of an unmanaged snapshot from a pool
+      // not in unmanaged snaps mode.
+      if (!pool->is_unmanaged_snaps_mode()) {
+        _pool_op_reply(m, -ENOTSUP, osdmap.get_epoch());
+        return false;
+      }
+      /* fall-thru */
+    case POOL_OP_CREATE_UNMANAGED_SNAP:
+      // but we will allow creating an unmanaged snapshot on any pool
+      // as long as it is not in 'pool' snaps mode.
+      if (pool->is_pool_snaps_mode()) {
+        _pool_op_reply(m, -EINVAL, osdmap.get_epoch());
+        return false;
+      }
+  }
+
+  // projected pool info
+  pg_pool_t pp;
+  if (pending_inc.new_pools.count(m->pool))
+    pp = pending_inc.new_pools[m->pool];
+  else
+    pp = *osdmap.get_pg_pool(m->pool);
+
+  bufferlist reply_data;
+
+  // pool snaps vs unmanaged snaps are mutually exclusive
+  switch (m->op) {
+  case POOL_OP_CREATE_SNAP:
+  case POOL_OP_DELETE_SNAP:
+    if (pp.is_unmanaged_snaps_mode()) {
+      ret = -EINVAL;
+      goto out;
+    }
+    break;
+
+  case POOL_OP_CREATE_UNMANAGED_SNAP:
+  case POOL_OP_DELETE_UNMANAGED_SNAP:
+    if (pp.is_pool_snaps_mode()) {
+      ret = -EINVAL;
+      goto out;
+    }
+  }
+
+  switch (m->op) {
+  case POOL_OP_CREATE_SNAP:
+    if (!pp.snap_exists(m->name.c_str())) {
+      pp.add_snap(m->name.c_str(), ceph_clock_now(g_ceph_context));
+      dout(10) << "create snap in pool " << m->pool << " " << m->name << " seq " << pp.get_snap_epoch() << dendl;
+      changed = true;
+    }
+    break;
+
+  case POOL_OP_DELETE_SNAP:
+    {
+      snapid_t s = pp.snap_exists(m->name.c_str());
+      if (s) {
+	pp.remove_snap(s);
+	changed = true;
+      }
+    }
+    break;
+
+  case POOL_OP_CREATE_UNMANAGED_SNAP: 
+    {
+      uint64_t snapid;
+      pp.add_unmanaged_snap(snapid);
+      ::encode(snapid, reply_data);
+      changed = true;
+    }
+    break;
+
+  case POOL_OP_DELETE_UNMANAGED_SNAP:
+    if (!pp.is_removed_snap(m->snapid)) {
+      pp.remove_unmanaged_snap(m->snapid);
+      changed = true;
+    }
+    break;
+
+  case POOL_OP_AUID_CHANGE:
+    if (pp.auid != m->auid) {
+      pp.auid = m->auid;
+      changed = true;
+    }
+    break;
+
+  default:
+    assert(0);
+    break;
+  }
+
+  if (changed) {
+    pp.set_snap_epoch(pending_inc.epoch);
+    pending_inc.new_pools[m->pool] = pp;
+  }
+
+ out:
+  wait_for_finished_proposal(new OSDMonitor::C_PoolOp(this, m, ret, pending_inc.epoch, &reply_data));
+  return true;
+}
+
+bool OSDMonitor::prepare_pool_op_create(MPoolOp *m)
+{
+  int err = prepare_new_pool(m);
+  wait_for_finished_proposal(new OSDMonitor::C_PoolOp(this, m, err, pending_inc.epoch));
+  return true;
+}
+
+int OSDMonitor::_check_remove_pool(int64_t pool, const pg_pool_t *p,
+				   ostream *ss)
+{
+  string poolstr = osdmap.get_pool_name(pool);
+
+  if (p->tier_of >= 0) {
+    *ss << "pool '" << poolstr << "' is a tier of '"
+	<< osdmap.get_pool_name(p->tier_of) << "'";
+    return -EBUSY;
+  }
+  if (!p->tiers.empty()) {
+    *ss << "pool '" << poolstr << "' has tiers";
+    for(std::set::iterator i = p->tiers.begin(); i != p->tiers.end(); ++i) {
+      const char *name = osdmap.get_pool_name(*i);
+      assert(name != NULL);
+      *ss << " " << name;
+    }
+    return -EBUSY;
+  }
+  *ss << "pool '" << poolstr << "' removed";
+  return 0;
+}
+
+int OSDMonitor::_prepare_remove_pool(int64_t pool, ostream *ss)
+{
+  dout(10) << "_prepare_remove_pool " << pool << dendl;
+  const pg_pool_t *p = osdmap.get_pg_pool(pool);
+  int r = _check_remove_pool(pool, p, ss);
+  if (r < 0)
+    return r;
+
+  if (pending_inc.new_pools.count(pool)) {
+    // if there is a problem with the pending info, wait and retry
+    // this op.
+    pg_pool_t *p = &pending_inc.new_pools[pool];
+    int r = _check_remove_pool(pool, p, ss);
+    if (r < 0)
+      return -EAGAIN;
+  }
+
+  if (pending_inc.old_pools.count(pool)) {
+    dout(10) << "_prepare_remove_pool " << pool << " already pending removal"
+	     << dendl;
+    return 0;
+  }
+
+  // remove
+  pending_inc.old_pools.insert(pool);
+
+  // remove any pg_temp mappings for this pool too
+  for (map >::iterator p = osdmap.pg_temp->begin();
+       p != osdmap.pg_temp->end();
+       ++p) {
+    if (p->first.pool() == (uint64_t)pool) {
+      dout(10) << "_prepare_remove_pool " << pool << " removing obsolete pg_temp "
+	       << p->first << dendl;
+      pending_inc.new_pg_temp[p->first].clear();
+    }
+  }
+  for (map::iterator p = osdmap.primary_temp->begin();
+      p != osdmap.primary_temp->end();
+      ++p) {
+    if (p->first.pool() == (uint64_t)pool) {
+      dout(10) << "_prepare_remove_pool " << pool
+               << " removing obsolete primary_temp" << p->first << dendl;
+      pending_inc.new_primary_temp[p->first] = -1;
+    }
+  }
+  return 0;
+}
+
+int OSDMonitor::_prepare_rename_pool(int64_t pool, string newname)
+{
+  dout(10) << "_prepare_rename_pool " << pool << dendl;
+  if (pending_inc.old_pools.count(pool)) {
+    dout(10) << "_prepare_rename_pool " << pool << " pending removal" << dendl;    
+    return -ENOENT;
+  }
+  for (map::iterator p = pending_inc.new_pool_names.begin();
+       p != pending_inc.new_pool_names.end();
+       ++p) {
+    if (p->second == newname && p->first != pool) {
+      return -EEXIST;
+    }
+  }
+
+  pending_inc.new_pool_names[pool] = newname;
+  return 0;
+}
+
+bool OSDMonitor::prepare_pool_op_delete(MPoolOp *m)
+{
+  ostringstream ss;
+  int ret = _prepare_remove_pool(m->pool, &ss);
+  if (ret == -EAGAIN) {
+    wait_for_finished_proposal(new C_RetryMessage(this, m));
+    return true;
+  }
+  if (ret < 0)
+    dout(10) << __func__ << " got " << ret << " " << ss.str() << dendl;
+  wait_for_finished_proposal(new OSDMonitor::C_PoolOp(this, m, ret,
+						      pending_inc.epoch));
+  return true;
+}
+
+void OSDMonitor::_pool_op_reply(MPoolOp *m, int ret, epoch_t epoch, bufferlist *blp)
+{
+  dout(20) << "_pool_op_reply " << ret << dendl;
+  MPoolOpReply *reply = new MPoolOpReply(m->fsid, m->get_tid(),
+					 ret, epoch, get_last_committed(), blp);
+  mon->send_reply(m, reply);
+  m->put();
+}
diff --git a/ceph/src/mon/OSDMonitor.h b/ceph/src/mon/OSDMonitor.h
new file mode 100644
index 00000000..fbce5fec
--- /dev/null
+++ b/ceph/src/mon/OSDMonitor.h
@@ -0,0 +1,412 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/* Object Store Device (OSD) Monitor
+ */
+
+#ifndef CEPH_OSDMONITOR_H
+#define CEPH_OSDMONITOR_H
+
+#include 
+#include 
+using namespace std;
+
+#include "include/types.h"
+#include "msg/Messenger.h"
+
+#include "osd/OSDMap.h"
+
+#include "PaxosService.h"
+#include "Session.h"
+
+class Monitor;
+#include "messages/MOSDBoot.h"
+#include "messages/MMonCommand.h"
+#include "messages/MOSDMap.h"
+#include "messages/MOSDFailure.h"
+#include "messages/MPoolOp.h"
+
+#include "erasure-code/ErasureCodeInterface.h"
+
+#define OSD_METADATA_PREFIX "osd_metadata"
+
+/// information about a particular peer's failure reports for one osd
+struct failure_reporter_t {
+  int num_reports;          ///< reports from this reporter
+  utime_t failed_since;     ///< when they think it failed
+  MOSDFailure *msg;         ///< most recent failure message
+
+  failure_reporter_t() : num_reports(0), msg(NULL) {}
+  failure_reporter_t(utime_t s) : num_reports(1), failed_since(s), msg(NULL) {}
+  ~failure_reporter_t() {
+    // caller should have taken this message before removing the entry.
+    assert(!msg);
+  }
+};
+
+/// information about all failure reports for one osd
+struct failure_info_t {
+  map reporters;  ///< reporter -> # reports
+  utime_t max_failed_since;                ///< most recent failed_since
+  int num_reports;
+
+  failure_info_t() : num_reports(0) {}
+
+  utime_t get_failed_since() {
+    if (max_failed_since == utime_t() && !reporters.empty()) {
+      // the old max must have canceled; recalculate.
+      for (map::iterator p = reporters.begin();
+	   p != reporters.end();
+	   ++p)
+	if (p->second.failed_since > max_failed_since)
+	  max_failed_since = p->second.failed_since;
+    }
+    return max_failed_since;
+  }
+
+  // set the message for the latest report.  return any old message we had,
+  // if any, so we can discard it.
+  MOSDFailure *add_report(int who, utime_t failed_since, MOSDFailure *msg) {
+    map::iterator p = reporters.find(who);
+    if (p == reporters.end()) {
+      if (max_failed_since == utime_t())
+	max_failed_since = failed_since;
+      else if (max_failed_since < failed_since)
+	max_failed_since = failed_since;
+      p = reporters.insert(map::value_type(who, failure_reporter_t(failed_since))).first;
+    } else {
+      p->second.num_reports++;
+    }
+    num_reports++;
+
+    MOSDFailure *ret = p->second.msg;
+    p->second.msg = msg;
+    return ret;
+  }
+
+  void take_report_messages(list& ls) {
+    for (map::iterator p = reporters.begin();
+	 p != reporters.end();
+	 ++p) {
+      if (p->second.msg) {
+	ls.push_back(p->second.msg);
+	p->second.msg = NULL;
+      }
+    }
+  }
+
+  void cancel_report(int who) {
+    map::iterator p = reporters.find(who);
+    if (p == reporters.end())
+      return;
+    num_reports -= p->second.num_reports;
+    reporters.erase(p);
+    if (reporters.empty())
+      max_failed_since = utime_t();
+  }
+};
+
+class OSDMonitor : public PaxosService {
+public:
+  OSDMap osdmap;
+
+private:
+  // [leader]
+  OSDMap::Incremental pending_inc;
+  map pending_metadata;
+  set             pending_metadata_rm;
+  map failure_info;
+  map    down_pending_out;  // osd down -> out
+
+  map osd_weight;
+
+  /*
+   * cache what epochs we think osds have.  this is purely
+   * optimization to try to avoid sending the same inc maps twice.
+   */
+  map osd_epoch;
+
+  void check_failures(utime_t now);
+  bool check_failure(utime_t now, int target_osd, failure_info_t& fi);
+
+  // map thrashing
+  int thrash_map;
+  int thrash_last_up_osd;
+  bool thrash();
+
+  bool _have_pending_crush();
+  CrushWrapper &_get_stable_crush();
+  void _get_pending_crush(CrushWrapper& newcrush);
+
+  // svc
+public:  
+  void create_initial();
+private:
+  void update_from_paxos(bool *need_bootstrap);
+  void create_pending();  // prepare a new pending
+  void encode_pending(MonitorDBStore::Transaction *t);
+  void on_active();
+  void on_shutdown();
+
+  /**
+   * we haven't delegated full version stashing to paxosservice for some time
+   * now, making this function useless in current context.
+   */
+  virtual void encode_full(MonitorDBStore::Transaction *t) { }
+  /**
+   * do not let paxosservice periodically stash full osdmaps, or we will break our
+   * locally-managed full maps.  (update_from_paxos loads the latest and writes them
+   * out going forward from there, but if we just synced that may mean we skip some.)
+   */
+  virtual bool should_stash_full() {
+    return false;
+  }
+
+  /**
+   * hook into trim to include the oldest full map in the trim transaction
+   *
+   * This ensures that anyone post-sync will have enough to rebuild their
+   * full osdmaps.
+   */
+  void encode_trim_extra(MonitorDBStore::Transaction *tx, version_t first);
+
+  void update_msgr_features();
+  int check_cluster_features(uint64_t features, stringstream &ss);
+  /**
+   * check if the cluster supports the features required by the
+   * given crush map. Outputs the daemons which don't support it
+   * to the stringstream.
+   *
+   * @returns true if the map is passable, false otherwise
+   */
+  bool validate_crush_against_features(const CrushWrapper *newcrush,
+                                      stringstream &ss);
+
+  void share_map_with_random_osd();
+
+  void update_logger();
+
+  void handle_query(PaxosServiceMessage *m);
+  bool preprocess_query(PaxosServiceMessage *m);  // true if processed.
+  bool prepare_update(PaxosServiceMessage *m);
+  bool should_propose(double &delay);
+
+  version_t get_trim_to();
+
+  bool can_mark_down(int o);
+  bool can_mark_up(int o);
+  bool can_mark_out(int o);
+  bool can_mark_in(int o);
+
+  // ...
+  MOSDMap *build_latest_full();
+  MOSDMap *build_incremental(epoch_t first, epoch_t last);
+  void send_full(PaxosServiceMessage *m);
+  void send_incremental(PaxosServiceMessage *m, epoch_t first);
+  void send_incremental(epoch_t first, entity_inst_t& dest, bool onetime);
+
+  int reweight_by_utilization(int oload, std::string& out_str);
+
+  bool check_source(PaxosServiceMessage *m, uuid_d fsid);
+ 
+  bool preprocess_mark_me_down(class MOSDMarkMeDown *m);
+
+  friend class C_AckMarkedDown;
+  bool preprocess_failure(class MOSDFailure *m);
+  bool prepare_failure(class MOSDFailure *m);
+  bool prepare_mark_me_down(class MOSDMarkMeDown *m);
+  void process_failures();
+  void take_all_failures(list& ls);
+
+  bool preprocess_boot(class MOSDBoot *m);
+  bool prepare_boot(class MOSDBoot *m);
+  void _booted(MOSDBoot *m, bool logit);
+
+  bool preprocess_alive(class MOSDAlive *m);
+  bool prepare_alive(class MOSDAlive *m);
+  void _reply_map(PaxosServiceMessage *m, epoch_t e);
+
+  bool preprocess_pgtemp(class MOSDPGTemp *m);
+  bool prepare_pgtemp(class MOSDPGTemp *m);
+
+  int _check_remove_pool(int64_t pool, const pg_pool_t *pi, ostream *ss);
+  int _prepare_remove_pool(int64_t pool, ostream *ss);
+  int _prepare_rename_pool(int64_t pool, string newname);
+
+  bool preprocess_pool_op ( class MPoolOp *m);
+  bool preprocess_pool_op_create ( class MPoolOp *m);
+  bool prepare_pool_op (MPoolOp *m);
+  bool prepare_pool_op_create (MPoolOp *m);
+  bool prepare_pool_op_delete(MPoolOp *m);
+  int crush_ruleset_create_erasure(const string &name,
+				   const string &profile,
+				   int *ruleset,
+				   stringstream &ss);
+  int get_erasure_code(const string &erasure_code_profile,
+		       ErasureCodeInterfaceRef *erasure_code,
+		       stringstream &ss) const;
+  int prepare_pool_crush_ruleset(const unsigned pool_type,
+				 const string &erasure_code_profile,
+				 const string &ruleset_name,
+				 int *crush_ruleset,
+				 stringstream &ss);
+  bool erasure_code_profile_in_use(const map &pools,
+				   const string &profile,
+				   ostream &ss);
+  int parse_erasure_code_profile(const vector &erasure_code_profile,
+				 map *erasure_code_profile_map,
+				 stringstream &ss);
+  int prepare_pool_size(const unsigned pool_type,
+			const string &erasure_code_profile,
+			unsigned *size, unsigned *min_size,
+			stringstream &ss);
+  int prepare_pool_stripe_width(const unsigned pool_type,
+				const string &erasure_code_profile,
+				unsigned *stripe_width,
+				stringstream &ss);
+  int prepare_new_pool(string& name, uint64_t auid,
+		       int crush_ruleset,
+		       const string &crush_ruleset_name,
+                       unsigned pg_num, unsigned pgp_num,
+		       const string &erasure_code_profile,
+                       const unsigned pool_type,
+		       stringstream &ss);
+  int prepare_new_pool(MPoolOp *m);
+
+  void update_pool_flags(int64_t pool_id, uint64_t flags);
+  bool update_pools_status();
+  void get_pools_health(list >& summary,
+                        list > *detail) const;
+
+  bool prepare_set_flag(MMonCommand *m, int flag);
+  bool prepare_unset_flag(MMonCommand *m, int flag);
+
+  void _pool_op_reply(MPoolOp *m, int ret, epoch_t epoch, bufferlist *blp=NULL);
+
+  struct C_Booted : public Context {
+    OSDMonitor *cmon;
+    MOSDBoot *m;
+    bool logit;
+    C_Booted(OSDMonitor *cm, MOSDBoot *m_, bool l=true) : 
+      cmon(cm), m(m_), logit(l) {}
+    void finish(int r) {
+      if (r >= 0)
+	cmon->_booted(m, logit);
+      else if (r == -ECANCELED)
+	m->put();
+      else if (r == -EAGAIN)
+	cmon->dispatch((PaxosServiceMessage*)m);
+      else
+	assert(0 == "bad C_Booted return value");
+    }
+  };
+
+  struct C_ReplyMap : public Context {
+    OSDMonitor *osdmon;
+    PaxosServiceMessage *m;
+    epoch_t e;
+    C_ReplyMap(OSDMonitor *o, PaxosServiceMessage *mm, epoch_t ee) : osdmon(o), m(mm), e(ee) {}
+    void finish(int r) {
+      if (r >= 0)
+	osdmon->_reply_map(m, e);
+      else if (r == -ECANCELED)
+	m->put();
+      else if (r == -EAGAIN)
+	osdmon->dispatch(m);
+      else
+	assert(0 == "bad C_ReplyMap return value");
+    }    
+  };
+  struct C_PoolOp : public Context {
+    OSDMonitor *osdmon;
+    MPoolOp *m;
+    int replyCode;
+    int epoch;
+    bufferlist reply_data;
+    C_PoolOp(OSDMonitor * osd, MPoolOp *m_, int rc, int e, bufferlist *rd=NULL) :
+      osdmon(osd), m(m_), replyCode(rc), epoch(e) {
+      if (rd)
+	reply_data = *rd;
+    }
+    void finish(int r) {
+      if (r >= 0)
+	osdmon->_pool_op_reply(m, replyCode, epoch, &reply_data);
+      else if (r == -ECANCELED)
+	m->put();
+      else if (r == -EAGAIN)
+	osdmon->dispatch(m);
+      else
+	assert(0 == "bad C_PoolOp return value");
+    }
+  };
+
+  bool preprocess_remove_snaps(struct MRemoveSnaps *m);
+  bool prepare_remove_snaps(struct MRemoveSnaps *m);
+
+ public:
+  OSDMonitor(Monitor *mn, Paxos *p, string service_name)
+  : PaxosService(mn, p, service_name),
+    thrash_map(0), thrash_last_up_osd(-1) { }
+
+  void tick();  // check state, take actions
+
+  int parse_osd_id(const char *s, stringstream *pss);
+
+  void get_health(list >& summary,
+		  list > *detail) const;
+  bool preprocess_command(MMonCommand *m);
+  bool prepare_command(MMonCommand *m);
+  bool prepare_command_impl(MMonCommand *m, map &cmdmap);
+
+  int prepare_command_pool_set(map &cmdmap,
+                               stringstream& ss);
+
+  void handle_osd_timeouts(const utime_t &now,
+			   std::map &last_osd_report);
+  void mark_all_down();
+
+  void send_latest(PaxosServiceMessage *m, epoch_t start=0);
+  void send_latest_now_nodelete(PaxosServiceMessage *m, epoch_t start=0) {
+    send_incremental(m, start);
+  }
+
+  epoch_t blacklist(const entity_addr_t& a, utime_t until);
+
+  void dump_info(Formatter *f);
+  int dump_osd_metadata(int osd, Formatter *f, ostream *err);
+
+  void check_subs();
+  void check_sub(Subscription *sub);
+
+  void add_flag(int flag) {
+    if (!(osdmap.flags & flag)) {
+      if (pending_inc.new_flags < 0)
+	pending_inc.new_flags = osdmap.flags;
+      pending_inc.new_flags |= flag;
+    }
+  }
+
+  void remove_flag(int flag) {
+    if(osdmap.flags & flag) {
+      if (pending_inc.new_flags < 0)
+	pending_inc.new_flags = osdmap.flags;
+      pending_inc.new_flags &= ~flag;
+    }
+  }
+};
+
+#endif
diff --git a/ceph/src/mon/PGMap.cc b/ceph/src/mon/PGMap.cc
new file mode 100644
index 00000000..5ec8ee2a
--- /dev/null
+++ b/ceph/src/mon/PGMap.cc
@@ -0,0 +1,1224 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "PGMap.h"
+
+#define dout_subsys ceph_subsys_mon
+#include "common/debug.h"
+#include "common/TextTable.h"
+#include "include/stringify.h"
+#include "common/Formatter.h"
+#include "include/ceph_features.h"
+#include "mon/MonitorDBStore.h"
+
+// --
+
+void PGMap::Incremental::encode(bufferlist &bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_MONENC) == 0) {
+    __u8 v = 4;
+    ::encode(v, bl);
+    ::encode(version, bl);
+    ::encode(pg_stat_updates, bl);
+    ::encode(osd_stat_updates, bl);
+    ::encode(osd_stat_rm, bl);
+    ::encode(osdmap_epoch, bl);
+    ::encode(pg_scan, bl);
+    ::encode(full_ratio, bl);
+    ::encode(nearfull_ratio, bl);
+    ::encode(pg_remove, bl);
+    return;
+  }
+
+  ENCODE_START(7, 5, bl);
+  ::encode(version, bl);
+  ::encode(pg_stat_updates, bl);
+  ::encode(osd_stat_updates, bl);
+  ::encode(osd_stat_rm, bl);
+  ::encode(osdmap_epoch, bl);
+  ::encode(pg_scan, bl);
+  ::encode(full_ratio, bl);
+  ::encode(nearfull_ratio, bl);
+  ::encode(pg_remove, bl);
+  ::encode(stamp, bl);
+  ::encode(osd_epochs, bl);
+  ENCODE_FINISH(bl);
+}
+
+void PGMap::Incremental::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
+  ::decode(version, bl);
+  if (struct_v < 3) {
+    pg_stat_updates.clear();
+    __u32 n;
+    ::decode(n, bl);
+    while (n--) {
+      old_pg_t opgid;
+      ::decode(opgid, bl);
+      pg_t pgid = opgid;
+      ::decode(pg_stat_updates[pgid], bl);
+    }
+  } else {
+    ::decode(pg_stat_updates, bl);
+  }
+  ::decode(osd_stat_updates, bl);
+  ::decode(osd_stat_rm, bl);
+  ::decode(osdmap_epoch, bl);
+  ::decode(pg_scan, bl);
+  if (struct_v >= 2) {
+    ::decode(full_ratio, bl);
+    ::decode(nearfull_ratio, bl);
+  }
+  if (struct_v < 3) {
+    pg_remove.clear();
+    __u32 n;
+    ::decode(n, bl);
+    while (n--) {
+      old_pg_t opgid;
+      ::decode(opgid, bl);
+      pg_remove.insert(pg_t(opgid));
+    }
+  } else {
+    ::decode(pg_remove, bl);
+  }
+  if (struct_v < 4 && full_ratio == 0) {
+    full_ratio = -1;
+  }
+  if (struct_v < 4 && nearfull_ratio == 0) {
+    nearfull_ratio = -1;
+  }
+  if (struct_v >= 6)
+    ::decode(stamp, bl);
+  if (struct_v >= 7) {
+    ::decode(osd_epochs, bl);
+  } else {
+    for (map::iterator i = osd_stat_updates.begin();
+	 i != osd_stat_updates.end();
+	 ++i) {
+      // This isn't accurate, but will cause trimming to behave like
+      // previously.
+      osd_epochs.insert(make_pair(i->first, osdmap_epoch));
+    }
+  }
+  DECODE_FINISH(bl);
+}
+
+void PGMap::Incremental::dump(Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_stream("stamp") << stamp;
+  f->dump_unsigned("osdmap_epoch", osdmap_epoch);
+  f->dump_unsigned("pg_scan_epoch", pg_scan);
+  f->dump_float("full_ratio", full_ratio);
+  f->dump_float("nearfull_ratio", nearfull_ratio);
+
+  f->open_array_section("pg_stat_updates");
+  for (map::const_iterator p = pg_stat_updates.begin(); p != pg_stat_updates.end(); ++p) {
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << p->first;
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("osd_stat_updates");
+  for (map::const_iterator p = osd_stat_updates.begin(); p != osd_stat_updates.end(); ++p) {
+    f->open_object_section("osd_stat");
+    f->dump_int("osd", p->first);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("osd_stat_removals");
+  for (set::const_iterator p = osd_stat_rm.begin(); p != osd_stat_rm.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+
+  f->open_array_section("pg_removals");
+  for (set::const_iterator p = pg_remove.begin(); p != pg_remove.end(); ++p)
+    f->dump_stream("pgid") << *p;
+  f->close_section();
+}
+
+void PGMap::Incremental::generate_test_instances(list& o)
+{
+  o.push_back(new Incremental);
+  o.push_back(new Incremental);
+  o.back()->version = 1;
+  o.back()->stamp = utime_t(123,345);
+  o.push_back(new Incremental);
+  o.back()->version = 2;
+  o.back()->pg_stat_updates[pg_t(1,2,3)] = pg_stat_t();
+  o.back()->osd_stat_updates[5] = osd_stat_t();
+  o.back()->osd_epochs[5] = 12;
+  o.push_back(new Incremental);
+  o.back()->version = 3;
+  o.back()->osdmap_epoch = 1;
+  o.back()->pg_scan = 2;
+  o.back()->full_ratio = .2;
+  o.back()->nearfull_ratio = .3;
+  o.back()->pg_stat_updates[pg_t(4,5,6)] = pg_stat_t();
+  o.back()->osd_stat_updates[6] = osd_stat_t();
+  o.back()->osd_epochs[6] = 12;
+  o.back()->pg_remove.insert(pg_t(1,2,3));
+  o.back()->osd_stat_rm.insert(5);
+}
+
+
+// --
+
+void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
+{
+  assert(inc.version == version+1);
+  version++;
+
+  utime_t delta_t;
+  delta_t = inc.stamp;
+  delta_t -= stamp;
+  stamp = inc.stamp;
+
+  pool_stat_t pg_sum_old = pg_sum;
+  ceph::unordered_map pg_pool_sum_old;
+
+  bool ratios_changed = false;
+  if (inc.full_ratio != full_ratio && inc.full_ratio != -1) {
+    full_ratio = inc.full_ratio;
+    ratios_changed = true;
+  }
+  if (inc.nearfull_ratio != nearfull_ratio && inc.nearfull_ratio != -1) {
+    nearfull_ratio = inc.nearfull_ratio;
+    ratios_changed = true;
+  }
+  if (ratios_changed)
+    redo_full_sets();
+
+  for (map::const_iterator p = inc.pg_stat_updates.begin();
+       p != inc.pg_stat_updates.end();
+       ++p) {
+    const pg_t &update_pg(p->first);
+    const pg_stat_t &update_stat(p->second);
+
+    if (pg_pool_sum_old.count(update_pg.pool()) == 0)
+      pg_pool_sum_old[update_pg.pool()] = pg_pool_sum[update_pg.pool()];
+
+    ceph::unordered_map::iterator t = pg_stat.find(update_pg);
+    if (t == pg_stat.end()) {
+      ceph::unordered_map::value_type v(update_pg, update_stat);
+      pg_stat.insert(v);
+      // did we affect the min?
+      if (min_last_epoch_clean &&
+	  update_stat.get_effective_last_epoch_clean() < min_last_epoch_clean)
+	min_last_epoch_clean = 0;
+    } else {
+      // did we (or might we) affect the min?
+      epoch_t lec = update_stat.get_effective_last_epoch_clean();
+      if (min_last_epoch_clean &&
+	  (lec < min_last_epoch_clean ||  // we did
+	   (lec > min_last_epoch_clean && // we might
+	    t->second.get_effective_last_epoch_clean() == min_last_epoch_clean)
+	   ))
+	min_last_epoch_clean = 0;
+
+      stat_pg_sub(update_pg, t->second);
+      t->second = update_stat;
+    }
+    stat_pg_add(update_pg, update_stat);
+  }
+  assert(osd_stat.size() == osd_epochs.size());
+  for (map::const_iterator p =
+	 inc.get_osd_stat_updates().begin();
+       p != inc.get_osd_stat_updates().end();
+       ++p) {
+    int osd = p->first;
+    const osd_stat_t &new_stats(p->second);
+
+    ceph::unordered_map::iterator t = osd_stat.find(osd);
+    if (t == osd_stat.end()) {
+      ceph::unordered_map::value_type v(osd, new_stats);
+      osd_stat.insert(v);
+    } else {
+      stat_osd_sub(t->second);
+      t->second = new_stats;
+    }
+    ceph::unordered_map::iterator i = osd_epochs.find(osd);
+    map::const_iterator j = inc.get_osd_epochs().find(osd);
+    assert(j != inc.get_osd_epochs().end());
+
+    // will we potentially affect the min?
+    if (min_last_epoch_clean &&
+	(i == osd_epochs.end() ||
+	 j->second < min_last_epoch_clean ||
+	 (j->second > min_last_epoch_clean &&
+	  i->second == min_last_epoch_clean)))
+      min_last_epoch_clean = 0;
+
+    if (i == osd_epochs.end())
+      osd_epochs.insert(*j);
+    else
+      i->second = j->second;
+
+    stat_osd_add(new_stats);
+
+    // adjust [near]full status
+    register_nearfull_status(osd, new_stats);
+  }
+  set deleted_pools;
+  for (set::const_iterator p = inc.pg_remove.begin();
+       p != inc.pg_remove.end();
+       ++p) {
+    const pg_t &removed_pg(*p);
+    ceph::unordered_map::iterator s = pg_stat.find(removed_pg);
+    if (s != pg_stat.end()) {
+      stat_pg_sub(removed_pg, s->second);
+      pg_stat.erase(s);
+    }
+    if (removed_pg.ps() == 0)
+      deleted_pools.insert(removed_pg.pool());
+  }
+  for (set::iterator p = deleted_pools.begin();
+       p != deleted_pools.end();
+       ++p) {
+    dout(20) << " deleted pool " << *p << dendl;
+    deleted_pool(*p);
+  }
+
+  for (set::iterator p = inc.get_osd_stat_rm().begin();
+       p != inc.get_osd_stat_rm().end();
+       ++p) {
+    ceph::unordered_map::iterator t = osd_stat.find(*p);
+    if (t != osd_stat.end()) {
+      stat_osd_sub(t->second);
+      osd_stat.erase(t);
+    }
+
+    // remove these old osds from full/nearfull set(s), too
+    nearfull_osds.erase(*p);
+    full_osds.erase(*p);
+  }
+
+  // calculate a delta, and average over the last 2 deltas.
+  pool_stat_t d = pg_sum;
+  d.stats.sub(pg_sum_old.stats);
+  pg_sum_deltas.push_back(make_pair(d, delta_t));
+  stamp_delta += delta_t;
+
+  pg_sum_delta.stats.add(d.stats);
+  if (pg_sum_deltas.size() > (std::list< pair >::size_type)MAX(1, cct ? cct->_conf->mon_stat_smooth_intervals : 1)) {
+    pg_sum_delta.stats.sub(pg_sum_deltas.front().first.stats);
+    stamp_delta -= pg_sum_deltas.front().second;
+    pg_sum_deltas.pop_front();
+  }
+
+  update_pool_deltas(cct, inc.stamp, pg_pool_sum_old);
+
+  if (inc.osdmap_epoch)
+    last_osdmap_epoch = inc.osdmap_epoch;
+  if (inc.pg_scan)
+    last_pg_scan = inc.pg_scan;
+
+  min_last_epoch_clean = 0;  // invalidate
+}
+
+void PGMap::redo_full_sets()
+{
+  full_osds.clear();
+  nearfull_osds.clear();
+  for (ceph::unordered_map::iterator i = osd_stat.begin();
+       i != osd_stat.end();
+       ++i) {
+    register_nearfull_status(i->first, i->second);
+  }
+}
+
+void PGMap::register_nearfull_status(int osd, const osd_stat_t& s)
+{
+  float ratio = ((float)s.kb_used) / ((float)s.kb);
+
+  if (full_ratio > 0 && ratio > full_ratio) {
+    // full
+    full_osds.insert(osd);
+    nearfull_osds.erase(osd);
+  } else if (nearfull_ratio > 0 && ratio > nearfull_ratio) {
+    // nearfull
+    full_osds.erase(osd);
+    nearfull_osds.insert(osd);
+  } else {
+    // ok
+    full_osds.erase(osd);
+    nearfull_osds.erase(osd);
+  }
+}
+
+void PGMap::calc_stats()
+{
+  num_pg_by_state.clear();
+  num_pg = 0;
+  num_osd = 0;
+  pg_pool_sum.clear();
+  pg_sum = pool_stat_t();
+  osd_sum = osd_stat_t();
+
+  for (ceph::unordered_map::iterator p = pg_stat.begin();
+       p != pg_stat.end();
+       ++p) {
+    stat_pg_add(p->first, p->second);
+  }
+  for (ceph::unordered_map::iterator p = osd_stat.begin();
+       p != osd_stat.end();
+       ++p)
+    stat_osd_add(p->second);
+
+  redo_full_sets();
+
+  calc_min_last_epoch_clean();
+}
+
+void PGMap::update_pg(pg_t pgid, bufferlist& bl)
+{
+  bufferlist::iterator p = bl.begin();
+  ceph::unordered_map::iterator s = pg_stat.find(pgid);
+  if (s != pg_stat.end())
+    stat_pg_sub(pgid, s->second);
+  pg_stat_t& r = pg_stat[pgid];
+  ::decode(r, p);
+  stat_pg_add(pgid, r);
+}
+
+void PGMap::remove_pg(pg_t pgid)
+{
+  ceph::unordered_map::iterator s = pg_stat.find(pgid);
+  if (s != pg_stat.end()) {
+    stat_pg_sub(pgid, s->second);
+    pg_stat.erase(s);
+  }
+}
+
+void PGMap::update_osd(int osd, bufferlist& bl)
+{
+  bufferlist::iterator p = bl.begin();
+  ceph::unordered_map::iterator o = osd_stat.find(osd);
+  if (o != osd_stat.end())
+    stat_osd_sub(o->second);
+  osd_stat_t& r = osd_stat[osd];
+  ::decode(r, p);
+  stat_osd_add(r);
+
+  // adjust [near]full status
+  register_nearfull_status(osd, r);
+}
+
+void PGMap::remove_osd(int osd)
+{
+  ceph::unordered_map::iterator o = osd_stat.find(osd);
+  if (o != osd_stat.end()) {
+    stat_osd_sub(o->second);
+    osd_stat.erase(o);
+
+    // remove these old osds from full/nearfull set(s), too
+    nearfull_osds.erase(osd);
+    full_osds.erase(osd);
+  }
+}
+
+void PGMap::stat_pg_add(const pg_t &pgid, const pg_stat_t &s)
+{
+  num_pg++;
+  num_pg_by_state[s.state]++;
+  pg_pool_sum[pgid.pool()].add(s);
+  pg_sum.add(s);
+  if (s.state & PG_STATE_CREATING) {
+    creating_pgs.insert(pgid);
+    if (s.acting_primary >= 0)
+      creating_pgs_by_osd[s.acting_primary].insert(pgid);
+  }
+}
+
+void PGMap::stat_pg_sub(const pg_t &pgid, const pg_stat_t &s)
+{
+  num_pg--;
+  if (--num_pg_by_state[s.state] == 0)
+    num_pg_by_state.erase(s.state);
+
+  pool_stat_t& ps = pg_pool_sum[pgid.pool()];
+  ps.sub(s);
+  if (ps.is_zero())
+    pg_pool_sum.erase(pgid.pool());
+
+  pg_sum.sub(s);
+  if (s.state & PG_STATE_CREATING) {
+    creating_pgs.erase(pgid);
+    if (s.acting_primary >= 0) {
+      creating_pgs_by_osd[s.acting_primary].erase(pgid);
+      if (creating_pgs_by_osd[s.acting_primary].size() == 0)
+        creating_pgs_by_osd.erase(s.acting_primary);
+    }
+  }
+}
+
+void PGMap::stat_osd_add(const osd_stat_t &s)
+{
+  num_osd++;
+  osd_sum.add(s);
+}
+
+void PGMap::stat_osd_sub(const osd_stat_t &s)
+{
+  num_osd--;
+  osd_sum.sub(s);
+}
+
+epoch_t PGMap::calc_min_last_epoch_clean() const
+{
+  if (pg_stat.empty())
+    return 0;
+  ceph::unordered_map::const_iterator p = pg_stat.begin();
+  epoch_t min = p->second.get_effective_last_epoch_clean();
+  for (++p; p != pg_stat.end(); ++p) {
+    epoch_t lec = p->second.get_effective_last_epoch_clean();
+    if (lec < min)
+      min = lec;
+  }
+  // also scan osd epochs
+  // don't trim past the oldest reported osd epoch
+  for (ceph::unordered_map::const_iterator i = osd_epochs.begin();
+       i != osd_epochs.end();
+       ++i) {
+    if (i->second < min)
+      min = i->second;
+  }
+  return min;
+}
+
+void PGMap::encode(bufferlist &bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_MONENC) == 0) {
+    __u8 v = 3;
+    ::encode(v, bl);
+    ::encode(version, bl);
+    ::encode(pg_stat, bl);
+    ::encode(osd_stat, bl);
+    ::encode(last_osdmap_epoch, bl);
+    ::encode(last_pg_scan, bl);
+    ::encode(full_ratio, bl);
+    ::encode(nearfull_ratio, bl);
+    return;
+  }
+
+  ENCODE_START(6, 4, bl);
+  ::encode(version, bl);
+  ::encode(pg_stat, bl);
+  ::encode(osd_stat, bl);
+  ::encode(last_osdmap_epoch, bl);
+  ::encode(last_pg_scan, bl);
+  ::encode(full_ratio, bl);
+  ::encode(nearfull_ratio, bl);
+  ::encode(stamp, bl);
+  ::encode(osd_epochs, bl);
+  ENCODE_FINISH(bl);
+}
+
+void PGMap::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(6, 4, 4, bl);
+  ::decode(version, bl);
+  if (struct_v < 3) {
+    pg_stat.clear();
+    __u32 n;
+    ::decode(n, bl);
+    while (n--) {
+      old_pg_t opgid;
+      ::decode(opgid, bl);
+      pg_t pgid = opgid;
+      ::decode(pg_stat[pgid], bl);
+    }
+  } else {
+    ::decode(pg_stat, bl);
+  }
+  ::decode(osd_stat, bl);
+  ::decode(last_osdmap_epoch, bl);
+  ::decode(last_pg_scan, bl);
+  if (struct_v >= 2) {
+    ::decode(full_ratio, bl);
+    ::decode(nearfull_ratio, bl);
+  }
+  if (struct_v >= 5)
+    ::decode(stamp, bl);
+  if (struct_v >= 6) {
+    ::decode(osd_epochs, bl);
+  } else {
+    for (ceph::unordered_map::iterator i = osd_stat.begin();
+	 i != osd_stat.end();
+	 ++i) {
+      // This isn't accurate, but will cause trimming to behave like
+      // previously.
+      osd_epochs.insert(make_pair(i->first, last_osdmap_epoch));
+    }
+  }
+  DECODE_FINISH(bl);
+
+  calc_stats();
+}
+
+void PGMap::dirty_all(Incremental& inc)
+{
+  inc.osdmap_epoch = last_osdmap_epoch;
+  inc.pg_scan = last_pg_scan;
+  inc.full_ratio = full_ratio;
+  inc.nearfull_ratio = nearfull_ratio;
+
+  for (ceph::unordered_map::const_iterator p = pg_stat.begin(); p != pg_stat.end(); ++p) {
+    inc.pg_stat_updates[p->first] = p->second;
+  }
+  for (ceph::unordered_map::const_iterator p = osd_stat.begin(); p != osd_stat.end(); ++p) {
+    assert(osd_epochs.count(p->first));
+    inc.update_stat(p->first,
+		   inc.get_osd_epochs().find(p->first)->second,
+		   p->second);
+  }
+}
+
+void PGMap::dump(Formatter *f) const
+{
+  dump_basic(f);
+  dump_pg_stats(f, false);
+  dump_pool_stats(f);
+  dump_osd_stats(f);
+}
+
+void PGMap::dump_basic(Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_stream("stamp") << stamp;
+  f->dump_unsigned("last_osdmap_epoch", last_osdmap_epoch);
+  f->dump_unsigned("last_pg_scan", last_pg_scan);
+  f->dump_float("full_ratio", full_ratio);
+  f->dump_float("near_full_ratio", nearfull_ratio);
+  
+  f->open_object_section("pg_stats_sum");
+  pg_sum.dump(f);
+  f->close_section();
+
+  f->open_object_section("osd_stats_sum");
+  osd_sum.dump(f);
+  f->close_section();
+
+  dump_delta(f);
+}
+
+void PGMap::dump_delta(Formatter *f) const
+{
+  f->open_object_section("pg_stats_delta");
+  pg_sum_delta.dump(f);
+  f->close_section();
+}
+
+void PGMap::dump_pg_stats(Formatter *f, bool brief) const
+{
+  f->open_array_section("pg_stats");
+  for (ceph::unordered_map::const_iterator i = pg_stat.begin();
+       i != pg_stat.end();
+       ++i) {
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << i->first;
+    if (brief)
+      i->second.dump_brief(f);
+    else
+      i->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_pool_stats(Formatter *f) const
+{
+  f->open_array_section("pool_stats");
+  for (ceph::unordered_map::const_iterator p = pg_pool_sum.begin();
+       p != pg_pool_sum.end();
+       ++p) {
+    f->open_object_section("pool_stat");
+    f->dump_int("poolid", p->first);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_osd_stats(Formatter *f) const
+{
+  f->open_array_section("osd_stats");
+  for (ceph::unordered_map::const_iterator q = osd_stat.begin();
+       q != osd_stat.end();
+       ++q) {
+    f->open_object_section("osd_stat");
+    f->dump_int("osd", q->first);
+    q->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_pg_stats_plain(ostream& ss,
+				const ceph::unordered_map& pg_stats) const
+{
+  ss << "pg_stat\tobjects\tmip\tdegr\tunf\tbytes\tlog\tdisklog\tstate\tstate_stamp\tv\treported\tup\tup_primary\tacting\tacting_primary\tlast_scrub\tscrub_stamp\tlast_deep_scrub\tdeep_scrub_stamp" << std::endl;
+  for (ceph::unordered_map::const_iterator i = pg_stats.begin();
+       i != pg_stats.end(); ++i) {
+    const pg_stat_t &st(i->second);
+    ss << i->first
+       << "\t" << st.stats.sum.num_objects
+      //<< "\t" << st.num_object_copies
+       << "\t" << st.stats.sum.num_objects_missing_on_primary
+       << "\t" << st.stats.sum.num_objects_degraded
+       << "\t" << st.stats.sum.num_objects_unfound
+       << "\t" << st.stats.sum.num_bytes
+       << "\t" << st.log_size
+       << "\t" << st.ondisk_log_size
+       << "\t" << pg_state_string(st.state)
+       << "\t" << st.last_change
+       << "\t" << st.version
+       << "\t" << st.reported_epoch << ":" << st.reported_seq
+       << "\t" << st.up
+       << "\t" << st.up_primary
+       << "\t" << st.acting
+       << "\t" << st.acting_primary
+       << "\t" << st.last_scrub << "\t" << st.last_scrub_stamp
+       << "\t" << st.last_deep_scrub << "\t" << st.last_deep_scrub_stamp
+       << std::endl;
+  }
+}
+
+void PGMap::dump(ostream& ss) const
+{
+  ss << "version " << version << std::endl;
+  ss << "stamp " << stamp << std::endl;
+  ss << "last_osdmap_epoch " << last_osdmap_epoch << std::endl;
+  ss << "last_pg_scan " << last_pg_scan << std::endl;
+  ss << "full_ratio " << full_ratio << std::endl;
+  ss << "nearfull_ratio " << nearfull_ratio << std::endl;
+  dump_pg_stats_plain(ss, pg_stat);
+  for (ceph::unordered_map::const_iterator p = pg_pool_sum.begin();
+       p != pg_pool_sum.end();
+       ++p)
+    ss << "pool " << p->first
+       << "\t" << p->second.stats.sum.num_objects
+      //<< "\t" << p->second.num_object_copies
+       << "\t" << p->second.stats.sum.num_objects_missing_on_primary
+       << "\t" << p->second.stats.sum.num_objects_degraded
+       << "\t" << p->second.stats.sum.num_objects_unfound
+       << "\t" << p->second.stats.sum.num_bytes
+       << "\t" << p->second.log_size
+       << "\t" << p->second.ondisk_log_size
+       << std::endl;
+  ss << " sum\t" << pg_sum.stats.sum.num_objects
+    //<< "\t" << pg_sum.num_object_copies
+     << "\t" << pg_sum.stats.sum.num_objects_missing_on_primary
+     << "\t" << pg_sum.stats.sum.num_objects_degraded
+     << "\t" << pg_sum.stats.sum.num_objects_unfound
+     << "\t" << pg_sum.stats.sum.num_bytes
+     << "\t" << pg_sum.log_size
+     << "\t" << pg_sum.ondisk_log_size
+     << std::endl;
+  ss << "osdstat\tkbused\tkbavail\tkb\thb in\thb out" << std::endl;
+  for (ceph::unordered_map::const_iterator p = osd_stat.begin();
+       p != osd_stat.end();
+       ++p)
+    ss << p->first
+       << "\t" << p->second.kb_used
+       << "\t" << p->second.kb_avail 
+       << "\t" << p->second.kb
+       << "\t" << p->second.hb_in
+       << "\t" << p->second.hb_out
+       << std::endl;
+  ss << " sum\t" << osd_sum.kb_used
+     << "\t" << osd_sum.kb_avail 
+     << "\t" << osd_sum.kb
+     << std::endl;
+}
+
+void PGMap::get_stuck_stats(PGMap::StuckPG type, utime_t cutoff,
+			    ceph::unordered_map& stuck_pgs) const
+{
+  for (ceph::unordered_map::const_iterator i = pg_stat.begin();
+       i != pg_stat.end();
+       ++i) {
+    utime_t val;
+    switch (type) {
+    case STUCK_INACTIVE:
+      if (i->second.state & PG_STATE_ACTIVE)
+	continue;
+      val = i->second.last_active;
+      break;
+    case STUCK_UNCLEAN:
+      if (i->second.state & PG_STATE_CLEAN)
+	continue;
+      val = i->second.last_clean;
+      break;
+    case STUCK_STALE:
+      if ((i->second.state & PG_STATE_STALE) == 0)
+	continue;
+      val = i->second.last_unstale;
+      break;
+    default:
+      assert(0 == "invalid type");
+    }
+
+    if (val < cutoff) {
+      stuck_pgs[i->first] = i->second;
+    }
+  }
+}
+
+void PGMap::dump_stuck(Formatter *f, PGMap::StuckPG type, utime_t cutoff) const
+{
+  ceph::unordered_map stuck_pg_stats;
+  get_stuck_stats(type, cutoff, stuck_pg_stats);
+  f->open_array_section("stuck_pg_stats");
+  for (ceph::unordered_map::const_iterator i = stuck_pg_stats.begin();
+       i != stuck_pg_stats.end();
+       ++i) {
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << i->first;
+    i->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_stuck_plain(ostream& ss, PGMap::StuckPG type, utime_t cutoff) const
+{
+  ceph::unordered_map stuck_pg_stats;
+  get_stuck_stats(type, cutoff, stuck_pg_stats);
+  if (!stuck_pg_stats.empty())
+    dump_pg_stats_plain(ss, stuck_pg_stats);
+}
+
+void PGMap::dump_osd_perf_stats(Formatter *f) const
+{
+  f->open_array_section("osd_perf_infos");
+  for (ceph::unordered_map::const_iterator i = osd_stat.begin();
+       i != osd_stat.end();
+       ++i) {
+    f->open_object_section("osd");
+    f->dump_int("id", i->first);
+    {
+      f->open_object_section("perf_stats");
+      i->second.fs_perf_stat.dump(f);
+      f->close_section();
+    }
+    f->close_section();
+  }
+  f->close_section();
+}
+void PGMap::print_osd_perf_stats(std::ostream *ss) const
+{
+  TextTable tab;
+  tab.define_column("osdid", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("fs_commit_latency(ms)", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("fs_apply_latency(ms)", TextTable::LEFT, TextTable::RIGHT);
+  for (ceph::unordered_map::const_iterator i = osd_stat.begin();
+       i != osd_stat.end();
+       ++i) {
+    tab << i->first;
+    tab << i->second.fs_perf_stat.filestore_commit_latency;
+    tab << i->second.fs_perf_stat.filestore_apply_latency;
+    tab << TextTable::endrow;
+  }
+  (*ss) << tab;
+}
+
+void PGMap::recovery_summary(Formatter *f, ostream *out,
+                             const pool_stat_t& delta_sum) const
+{
+  bool first = true;
+  if (delta_sum.stats.sum.num_objects_degraded) {
+    double pc = (double)delta_sum.stats.sum.num_objects_degraded /
+      (double)delta_sum.stats.sum.num_object_copies * (double)100.0;
+    char b[20];
+    snprintf(b, sizeof(b), "%.3lf", pc);
+    if (f) {
+      f->dump_unsigned("degraded_objects", delta_sum.stats.sum.num_objects_degraded);
+      f->dump_unsigned("degraded_total", delta_sum.stats.sum.num_object_copies);
+      f->dump_string("degraded_ratio", b);
+    } else {
+      *out << delta_sum.stats.sum.num_objects_degraded
+	   << "/" << delta_sum.stats.sum.num_object_copies << " objects degraded (" << b << "%)";
+    }
+    first = false;
+  }
+  if (delta_sum.stats.sum.num_objects_unfound) {
+    double pc = (double)delta_sum.stats.sum.num_objects_unfound /
+      (double)delta_sum.stats.sum.num_objects * (double)100.0;
+    char b[20];
+    snprintf(b, sizeof(b), "%.3lf", pc);
+    if (f) {
+      f->dump_unsigned("unfound_objects", delta_sum.stats.sum.num_objects_unfound);
+      f->dump_unsigned("unfound_total", delta_sum.stats.sum.num_objects);
+      f->dump_string("unfound_ratio", b);
+    } else {
+      if (!first)
+	*out << "; ";
+      *out << delta_sum.stats.sum.num_objects_unfound
+	   << "/" << delta_sum.stats.sum.num_objects << " unfound (" << b << "%)";
+    }
+    first = false;
+  }
+}
+
+void PGMap::recovery_rate_summary(Formatter *f, ostream *out,
+                                  const pool_stat_t& delta_sum,
+                                  utime_t delta_stamp) const
+{
+  // make non-negative; we can get negative values if osds send
+  // uncommitted stats and then "go backward" or if they are just
+  // buggy/wrong.
+  pool_stat_t pos_delta = delta_sum;
+  pos_delta.floor(0);
+  if (pos_delta.stats.sum.num_objects_recovered ||
+      pos_delta.stats.sum.num_bytes_recovered ||
+      pos_delta.stats.sum.num_keys_recovered) {
+    int64_t objps = pos_delta.stats.sum.num_objects_recovered / (double)delta_stamp;
+    int64_t bps = pos_delta.stats.sum.num_bytes_recovered / (double)delta_stamp;
+    int64_t kps = pos_delta.stats.sum.num_keys_recovered / (double)delta_stamp;
+    if (f) {
+      f->dump_int("recovering_objects_per_sec", objps);
+      f->dump_int("recovering_bytes_per_sec", bps);
+      f->dump_int("recovering_keys_per_sec", kps);
+    } else {
+      *out << pretty_si_t(bps) << "B/s";
+      if (pos_delta.stats.sum.num_keys_recovered)
+	*out << ", " << pretty_si_t(kps) << "keys/s";
+      *out << ", " << pretty_si_t(objps) << "objects/s";
+    }
+  }
+}
+
+void PGMap::overall_recovery_rate_summary(Formatter *f, ostream *out) const
+{
+  recovery_rate_summary(f, out, pg_sum_delta, stamp_delta);
+}
+
+void PGMap::overall_recovery_summary(Formatter *f, ostream *out) const
+{
+  recovery_summary(f, out, pg_sum);
+}
+
+void PGMap::pool_recovery_rate_summary(Formatter *f, ostream *out,
+                                       uint64_t poolid) const
+{
+  ceph::unordered_map >::const_iterator p =
+    per_pool_sum_delta.find(poolid);
+  if (p == per_pool_sum_delta.end())
+    return;
+  ceph::unordered_map::const_iterator ts =
+    per_pool_sum_deltas_stamps.find(p->first);
+  assert(ts != per_pool_sum_deltas_stamps.end());
+  recovery_rate_summary(f, out, p->second.first, ts->second);
+}
+
+void PGMap::pool_recovery_summary(Formatter *f, ostream *out,
+                                  uint64_t poolid) const
+{
+  ceph::unordered_map >::const_iterator p =
+    per_pool_sum_delta.find(poolid);
+  if (p == per_pool_sum_delta.end())
+    return;
+  recovery_summary(f, out, p->second.first);
+}
+
+void PGMap::client_io_rate_summary(Formatter *f, ostream *out,
+                                   const pool_stat_t& delta_sum,
+                                   utime_t delta_stamp) const
+{
+  pool_stat_t pos_delta = delta_sum;
+  pos_delta.floor(0);
+  if (pos_delta.stats.sum.num_rd ||
+      pos_delta.stats.sum.num_wr) {
+    if (pos_delta.stats.sum.num_rd) {
+      int64_t rd = (pos_delta.stats.sum.num_rd_kb << 10) / (double)delta_stamp;
+      if (f) {
+	f->dump_int("read_bytes_sec", rd);
+      } else {
+	*out << pretty_si_t(rd) << "B/s rd, ";
+      }
+    }
+    if (pos_delta.stats.sum.num_wr) {
+      int64_t wr = (pos_delta.stats.sum.num_wr_kb << 10) / (double)delta_stamp;
+      if (f) {
+	f->dump_int("write_bytes_sec", wr);
+      } else {
+	*out << pretty_si_t(wr) << "B/s wr, ";
+      }
+    }
+    int64_t iops = (pos_delta.stats.sum.num_rd + pos_delta.stats.sum.num_wr) / (double)delta_stamp;
+    if (f) {
+      f->dump_int("op_per_sec", iops);
+    } else {
+      *out << pretty_si_t(iops) << "op/s";
+    }
+  }
+}
+
+void PGMap::overall_client_io_rate_summary(Formatter *f, ostream *out) const
+{
+  client_io_rate_summary(f, out, pg_sum_delta, stamp_delta);
+}
+
+void PGMap::pool_client_io_rate_summary(Formatter *f, ostream *out,
+                                        uint64_t poolid) const
+{
+  ceph::unordered_map >::const_iterator p =
+    per_pool_sum_delta.find(poolid);
+  if (p == per_pool_sum_delta.end())
+    return;
+  ceph::unordered_map::const_iterator ts =
+    per_pool_sum_deltas_stamps.find(p->first);
+  assert(ts != per_pool_sum_deltas_stamps.end());
+  client_io_rate_summary(f, out, p->second.first, ts->second);
+}
+
+/**
+ * update aggregated delta
+ *
+ * @param cct               ceph context
+ * @param ts                Timestamp for the stats being delta'ed
+ * @param old_pool_sum      Previous stats sum
+ * @param last_ts           Last timestamp for pool
+ * @param result_pool_sum   Resulting stats
+ * @param result_ts_delta   Resulting timestamp delta
+ * @param delta_avg_list    List of last N computed deltas, used to average
+ */
+void PGMap::update_delta(CephContext *cct,
+                         const utime_t ts,
+                         const pool_stat_t& old_pool_sum,
+                         utime_t *last_ts,
+                         const pool_stat_t& current_pool_sum,
+                         pool_stat_t *result_pool_delta,
+                         utime_t *result_ts_delta,
+                         list > *delta_avg_list)
+{
+  /* @p ts is the timestamp we want to associate with the data
+   * in @p old_pool_sum, and on which we will base ourselves to
+   * calculate the delta, stored in 'delta_t'.
+   */
+  utime_t delta_t;
+  delta_t = ts;         // start with the provided timestamp
+  delta_t -= *last_ts;  // take the last timestamp we saw
+  *last_ts = ts;        // @p ts becomes the last timestamp we saw
+
+  // calculate a delta, and average over the last 2 deltas.
+  /* start by taking a copy of our current @p result_pool_sum, and by
+   * taking out the stats from @p old_pool_sum.  This generates a stats
+   * delta.  Stash this stats delta in @p delta_avg_list, along with the
+   * timestamp delta for these results.
+   */
+  pool_stat_t d = current_pool_sum;
+  d.stats.sub(old_pool_sum.stats);
+  delta_avg_list->push_back(make_pair(d,delta_t));
+  *result_ts_delta += delta_t;
+
+  /* Aggregate current delta, and take out the last seen delta (if any) to
+   * average it out.
+   */
+  result_pool_delta->stats.add(d.stats);
+  size_t s = MAX(1, cct ? cct->_conf->mon_stat_smooth_intervals : 1);
+  if (delta_avg_list->size() > s) {
+    result_pool_delta->stats.sub(delta_avg_list->front().first.stats);
+    *result_ts_delta -= delta_avg_list->front().second;
+    delta_avg_list->pop_front();
+  }
+}
+
+/**
+ * update aggregated delta
+ *
+ * @param cct            ceph context
+ * @param ts             Timestamp
+ * @param pg_sum_old     Old pg_sum
+ */
+void PGMap::update_global_delta(CephContext *cct,
+                         const utime_t ts, const pool_stat_t& pg_sum_old)
+{
+  update_delta(cct, ts, pg_sum_old, &stamp, pg_sum, &pg_sum_delta,
+               &stamp_delta, &pg_sum_deltas);
+}
+
+/**
+ * Update a given pool's deltas
+ *
+ * @param cct           Ceph Context
+ * @param ts            Timestamp for the stats being delta'ed
+ * @param pool          Pool's id
+ * @param old_pool_sum  Previous stats sum
+ */
+void PGMap::update_one_pool_delta(CephContext *cct,
+                                  const utime_t ts,
+                                  const uint64_t pool,
+                                  const pool_stat_t& old_pool_sum)
+{
+  if (per_pool_sum_deltas.count(pool) == 0) {
+    assert(per_pool_sum_deltas_stamps.count(pool) == 0);
+    assert(per_pool_sum_delta.count(pool) == 0);
+  }
+
+  pair& sum_delta = per_pool_sum_delta[pool];
+
+  update_delta(cct, ts, old_pool_sum, &sum_delta.second, pg_pool_sum[pool],
+               &sum_delta.first, &per_pool_sum_deltas_stamps[pool],
+               &per_pool_sum_deltas[pool]);
+}
+
+/**
+ * Update pools' deltas
+ *
+ * @param cct               CephContext
+ * @param ts                Timestamp for the stats being delta'ed
+ * @param pg_pool_sum_old   Map of pool stats for delta calcs.
+ */
+void PGMap::update_pool_deltas(CephContext *cct, const utime_t ts,
+                               const ceph::unordered_map& pg_pool_sum_old)
+{
+  for (ceph::unordered_map::const_iterator it = pg_pool_sum_old.begin();
+       it != pg_pool_sum_old.end(); ++it) {
+    update_one_pool_delta(cct, ts, it->first, it->second);
+  }
+}
+
+void PGMap::clear_delta()
+{
+  pg_sum_delta = pool_stat_t();
+  pg_sum_deltas.clear();
+  stamp_delta = utime_t();
+}
+
+void PGMap::print_summary(Formatter *f, ostream *out) const
+{
+  std::stringstream ss;
+  if (f)
+    f->open_array_section("pgs_by_state");
+
+  for (ceph::unordered_map::const_iterator p = num_pg_by_state.begin();
+       p != num_pg_by_state.end();
+       ++p) {
+    if (f) {
+      f->open_object_section("pgs_by_state_element");
+      f->dump_string("state_name", pg_state_string(p->first));
+      f->dump_unsigned("count", p->second);
+      f->close_section();
+    } else {
+      ss.setf(std::ios::right);
+      ss << "             " << std::setw(7) << p->second << " " << pg_state_string(p->first) << "\n";
+      ss.unsetf(std::ios::right);
+    }
+  }
+  if (f)
+    f->close_section();
+
+  if (f) {
+    f->dump_unsigned("version", version);
+    f->dump_unsigned("num_pgs", pg_stat.size());
+    f->dump_unsigned("data_bytes", pg_sum.stats.sum.num_bytes);
+    f->dump_unsigned("bytes_used", osd_sum.kb_used * 1024ull);
+    f->dump_unsigned("bytes_avail", osd_sum.kb_avail * 1024ull);
+    f->dump_unsigned("bytes_total", osd_sum.kb * 1024ull);
+  } else {
+    *out << "      pgmap v" << version << ": "
+	 << pg_stat.size() << " pgs, " << pg_pool_sum.size() << " pools, "
+	 << prettybyte_t(pg_sum.stats.sum.num_bytes) << " data, "
+	 << pretty_si_t(pg_sum.stats.sum.num_objects) << "objects\n";
+    *out << "            "
+	 << kb_t(osd_sum.kb_used) << " used, "
+	 << kb_t(osd_sum.kb_avail) << " / "
+	 << kb_t(osd_sum.kb) << " avail\n";
+  }
+
+  std::stringstream ssr;
+  overall_recovery_summary(f, &ssr);
+  if (!f && ssr.str().length())
+    *out << "            " << ssr.str() << "\n";
+  ssr.clear();
+  ssr.str("");
+
+  if (!f)
+    *out << ss.str();   // pgs by state
+
+  overall_recovery_rate_summary(f, &ssr);
+  if (!f && ssr.str().length())
+    *out << "recovery io " << ssr.str() << "\n";
+
+  ssr.clear();
+  ssr.str("");
+
+  overall_client_io_rate_summary(f, &ssr);
+  if (!f && ssr.str().length())
+    *out << "  client io " << ssr.str() << "\n";
+
+
+}
+
+void PGMap::print_oneline_summary(ostream *out) const
+{
+  std::stringstream ss;
+
+  for (ceph::unordered_map::const_iterator p = num_pg_by_state.begin();
+       p != num_pg_by_state.end();
+       ++p) {
+    if (p != num_pg_by_state.begin())
+      ss << ", ";
+    ss << p->second << " " << pg_state_string(p->first);
+  }
+
+  string states = ss.str();
+  *out << "v" << version << ": "
+       << pg_stat.size() << " pgs: "
+       << states << "; "
+       << prettybyte_t(pg_sum.stats.sum.num_bytes) << " data, "
+       << kb_t(osd_sum.kb_used) << " used, "
+       << kb_t(osd_sum.kb_avail) << " / "
+       << kb_t(osd_sum.kb) << " avail";
+
+  // make non-negative; we can get negative values if osds send
+  // uncommitted stats and then "go backward" or if they are just
+  // buggy/wrong.
+  pool_stat_t pos_delta = pg_sum_delta;
+  pos_delta.floor(0);
+  if (pos_delta.stats.sum.num_rd ||
+      pos_delta.stats.sum.num_wr) {
+    *out << "; ";
+    if (pos_delta.stats.sum.num_rd) {
+      int64_t rd = (pos_delta.stats.sum.num_rd_kb << 10) / (double)stamp_delta;
+      *out << pretty_si_t(rd) << "B/s rd, ";
+    }
+    if (pos_delta.stats.sum.num_wr) {
+      int64_t wr = (pos_delta.stats.sum.num_wr_kb << 10) / (double)stamp_delta;
+      *out << pretty_si_t(wr) << "B/s wr, ";
+    }
+    int64_t iops = (pos_delta.stats.sum.num_rd + pos_delta.stats.sum.num_wr) / (double)stamp_delta;
+    *out << pretty_si_t(iops) << "op/s";
+  }
+
+  std::stringstream ssr;
+  overall_recovery_summary(NULL, &ssr);
+  if (ssr.str().length())
+    *out << "; " << ssr.str();
+  ssr.clear();
+  ssr.str("");
+  overall_recovery_rate_summary(NULL, &ssr);
+  if (ssr.str().length())
+    *out << "; " << ssr.str() << " recovering";
+}
+
+void PGMap::generate_test_instances(list& o)
+{
+  o.push_back(new PGMap);
+  o.push_back(new PGMap);
+  list inc;
+  Incremental::generate_test_instances(inc);
+  inc.pop_front();
+  while (!inc.empty()) {
+    o.back()->apply_incremental(NULL, *inc.front());
+    delete inc.front();
+    inc.pop_front();
+  }
+}
diff --git a/ceph/src/mon/PGMap.h b/ceph/src/mon/PGMap.h
new file mode 100644
index 00000000..99ce521b
--- /dev/null
+++ b/ceph/src/mon/PGMap.h
@@ -0,0 +1,316 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+ 
+/*
+ * Placement Group Map. Placement Groups are logical sets of objects
+ * that are replicated by the same set of devices. pgid=(r,hash(o)&m)
+ * where & is a bit-wise AND and m=2^k-1
+ */
+
+#ifndef CEPH_PGMAP_H
+#define CEPH_PGMAP_H
+
+#include "common/debug.h"
+#include "osd/osd_types.h"
+#include "common/config.h"
+#include 
+
+#include "MonitorDBStore.h"
+
+namespace ceph { class Formatter; }
+
+class PGMap {
+public:
+  // the map
+  version_t version;
+  epoch_t last_osdmap_epoch;   // last osdmap epoch i applied to the pgmap
+  epoch_t last_pg_scan;  // osdmap epoch
+  ceph::unordered_map pg_stat;
+  ceph::unordered_map osd_stat;
+  set full_osds;
+  set nearfull_osds;
+  float full_ratio;
+  float nearfull_ratio;
+
+  // mapping of osd to most recently reported osdmap epoch
+  ceph::unordered_map osd_epochs;
+
+  class Incremental {
+  public:
+    version_t version;
+    map pg_stat_updates;
+    epoch_t osdmap_epoch;
+    epoch_t pg_scan;  // osdmap epoch
+    set pg_remove;
+    float full_ratio;
+    float nearfull_ratio;
+    utime_t stamp;
+
+  private:
+    map osd_stat_updates;
+    set osd_stat_rm;
+
+    // mapping of osd to most recently reported osdmap epoch
+    map osd_epochs;
+  public:
+
+    const map &get_osd_stat_updates() const {
+      return osd_stat_updates;
+    }
+    const set &get_osd_stat_rm() const {
+      return osd_stat_rm;
+    }
+    const map &get_osd_epochs() const {
+      return osd_epochs;
+    }
+
+    void update_stat(int32_t osd, epoch_t epoch, const osd_stat_t &stat) {
+      osd_stat_updates[osd] = stat;
+      osd_epochs[osd] = epoch;
+      assert(osd_epochs.size() == osd_stat_updates.size());
+    }
+    void stat_osd_out(int32_t osd) {
+      // 0 the stats for the osd
+      osd_stat_updates[osd] = osd_stat_t();
+    }
+    void rm_stat(int32_t osd) {
+      osd_stat_rm.insert(osd);
+      osd_epochs.erase(osd);
+      osd_stat_updates.erase(osd);
+    }
+    void encode(bufferlist &bl, uint64_t features=-1) const;
+    void decode(bufferlist::iterator &bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& o);
+
+    Incremental() : version(0), osdmap_epoch(0), pg_scan(0),
+        full_ratio(0), nearfull_ratio(0) {}
+  };
+
+
+  // aggregate stats (soft state), generated by calc_stats()
+  ceph::unordered_map num_pg_by_state;
+  int64_t num_pg, num_osd;
+  ceph::unordered_map pg_pool_sum;
+  pool_stat_t pg_sum;
+  osd_stat_t osd_sum;
+  mutable epoch_t min_last_epoch_clean;
+
+  utime_t stamp;
+
+  // recent deltas, and summation
+  /**
+   * keep track of last deltas for each pool, calculated using
+   * @p pg_pool_sum as baseline.
+   */
+  ceph::unordered_map > > per_pool_sum_deltas;
+  /**
+   * keep track of per-pool timestamp deltas, according to last update on
+   * each pool.
+   */
+  ceph::unordered_map per_pool_sum_deltas_stamps;
+  /**
+   * keep track of sum deltas, per-pool, taking into account any previous
+   * deltas existing in @p per_pool_sum_deltas.  The utime_t as second member
+   * of the pair is the timestamp refering to the last update (i.e., the first
+   * member of the pair) for a given pool.
+   */
+  ceph::unordered_map > per_pool_sum_delta;
+
+  list< pair > pg_sum_deltas;
+  pool_stat_t pg_sum_delta;
+  utime_t stamp_delta;
+
+  void update_global_delta(CephContext *cct,
+                           const utime_t ts, const pool_stat_t& pg_sum_old);
+  void update_pool_deltas(CephContext *cct,
+                          const utime_t ts,
+                          const ceph::unordered_map& pg_pool_sum_old);
+  void clear_delta();
+
+  void deleted_pool(int64_t pool) {
+    pg_pool_sum.erase(pool);
+    per_pool_sum_deltas.erase(pool);
+    per_pool_sum_deltas_stamps.erase(pool);
+    per_pool_sum_delta.erase(pool);
+  }
+
+ private:
+  void update_delta(CephContext *cct,
+                    const utime_t ts,
+                    const pool_stat_t& old_pool_sum,
+                    utime_t *last_ts,
+                    const pool_stat_t& current_pool_sum,
+                    pool_stat_t *result_pool_delta,
+                    utime_t *result_ts_delta,
+                    list > *delta_avg_list);
+
+  void update_one_pool_delta(CephContext *cct,
+                             const utime_t ts,
+                             const uint64_t pool,
+                             const pool_stat_t& old_pool_sum);
+
+  epoch_t calc_min_last_epoch_clean() const;
+
+ public:
+
+  set creating_pgs;   // lru: front = new additions, back = recently pinged
+  map > creating_pgs_by_osd;
+
+  enum StuckPG {
+    STUCK_INACTIVE,
+    STUCK_UNCLEAN,
+    STUCK_STALE,
+    STUCK_NONE
+  };
+  
+  PGMap()
+    : version(0),
+      last_osdmap_epoch(0), last_pg_scan(0),
+      full_ratio(0), nearfull_ratio(0),
+      num_pg(0),
+      num_osd(0),
+      min_last_epoch_clean(0)
+  {}
+
+  void set_full_ratios(float full, float nearfull) {
+    if (full_ratio == full && nearfull_ratio == nearfull)
+      return;
+    full_ratio = full;
+    nearfull_ratio = nearfull;
+    redo_full_sets();
+  }
+
+  version_t get_version() const {
+    return version;
+  }
+  void set_version(version_t v) {
+    version = v;
+  }
+  epoch_t get_last_osdmap_epoch() const {
+    return last_osdmap_epoch;
+  }
+  void set_last_osdmap_epoch(epoch_t e) {
+    last_osdmap_epoch = e;
+  }
+  epoch_t get_last_pg_scan() const {
+    return last_pg_scan;
+  }
+  void set_last_pg_scan(epoch_t e) {
+    last_pg_scan = e;
+  }
+  utime_t get_stamp() const {
+    return stamp;
+  }
+  void set_stamp(utime_t s) {
+    stamp = s;
+  }
+
+  pool_stat_t get_pg_pool_sum_stat(int64_t pool) const {
+    ceph::unordered_map::const_iterator p =
+      pg_pool_sum.find(pool);
+    if (p != pg_pool_sum.end())
+      return p->second;
+    return pool_stat_t();
+  }
+
+  void update_pg(pg_t pgid, bufferlist& bl);
+  void remove_pg(pg_t pgid);
+  void update_osd(int osd, bufferlist& bl);
+  void remove_osd(int osd);
+
+  void apply_incremental(CephContext *cct, const Incremental& inc);
+  void redo_full_sets();
+  void register_nearfull_status(int osd, const osd_stat_t& s);
+  void calc_stats();
+  void stat_pg_add(const pg_t &pgid, const pg_stat_t &s);
+  void stat_pg_sub(const pg_t &pgid, const pg_stat_t &s);
+  void stat_osd_add(const osd_stat_t &s);
+  void stat_osd_sub(const osd_stat_t &s);
+  
+  void encode(bufferlist &bl, uint64_t features=-1) const;
+  void decode(bufferlist::iterator &bl);
+
+  void dirty_all(Incremental& inc);
+
+  void dump(Formatter *f) const; 
+  void dump_basic(Formatter *f) const;
+  void dump_pg_stats(Formatter *f, bool brief) const;
+  void dump_pool_stats(Formatter *f) const;
+  void dump_osd_stats(Formatter *f) const;
+  void dump_delta(Formatter *f) const;
+
+  void dump_pg_stats_plain(ostream& ss,
+			   const ceph::unordered_map& pg_stats) const;
+  void get_stuck_stats(StuckPG type, utime_t cutoff,
+		       ceph::unordered_map& stuck_pgs) const;
+  void dump_stuck(Formatter *f, StuckPG type, utime_t cutoff) const;
+  void dump_stuck_plain(ostream& ss, StuckPG type, utime_t cutoff) const;
+
+  void dump(ostream& ss) const;
+
+  void dump_osd_perf_stats(Formatter *f) const;
+  void print_osd_perf_stats(std::ostream *ss) const;
+
+  void recovery_summary(Formatter *f, ostream *out,
+                        const pool_stat_t& delta_sum) const;
+  void overall_recovery_summary(Formatter *f, ostream *out) const;
+  void pool_recovery_summary(Formatter *f, ostream *out,
+                             uint64_t poolid) const;
+  void recovery_rate_summary(Formatter *f, ostream *out,
+                             const pool_stat_t& delta_sum,
+                             utime_t delta_stamp) const;
+  void overall_recovery_rate_summary(Formatter *f, ostream *out) const;
+  void pool_recovery_rate_summary(Formatter *f, ostream *out,
+                                  uint64_t poolid) const;
+  /**
+   * Obtain a formatted/plain output for client I/O, source from stats for a
+   * given @p delta_sum pool over a given @p delta_stamp period of time.
+   */
+  void client_io_rate_summary(Formatter *f, ostream *out,
+                              const pool_stat_t& delta_sum,
+                              utime_t delta_stamp) const;
+  /**
+   * Obtain a formatted/plain output for the overall client I/O, which is
+   * calculated resorting to @p pg_sum_delta and @p stamp_delta.
+   */
+  void overall_client_io_rate_summary(Formatter *f, ostream *out) const;
+  /**
+   * Obtain a formatted/plain output for client I/O over a given pool
+   * with id @p pool_id.  We will then obtain pool-specific data
+   * from @p per_pool_sum_delta.
+   */
+  void pool_client_io_rate_summary(Formatter *f, ostream *out,
+                                   uint64_t poolid) const;
+
+  void print_summary(Formatter *f, ostream *out) const;
+  void print_oneline_summary(ostream *out) const;
+
+  epoch_t get_min_last_epoch_clean() const {
+    if (!min_last_epoch_clean)
+      min_last_epoch_clean = calc_min_last_epoch_clean();
+    return min_last_epoch_clean;
+  }
+
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER_FEATURES(PGMap::Incremental)
+WRITE_CLASS_ENCODER_FEATURES(PGMap)
+
+inline ostream& operator<<(ostream& out, const PGMap& m) {
+  m.print_oneline_summary(&out);
+  return out;
+}
+
+#endif
diff --git a/ceph/src/mon/PGMonitor.cc b/ceph/src/mon/PGMonitor.cc
new file mode 100644
index 00000000..15f67466
--- /dev/null
+++ b/ceph/src/mon/PGMonitor.cc
@@ -0,0 +1,2129 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include "json_spirit/json_spirit.h"
+#include "common/debug.h"		// undo damage
+#include "PGMonitor.h"
+#include "Monitor.h"
+#include "MDSMonitor.h"
+#include "OSDMonitor.h"
+#include "MonitorDBStore.h"
+
+#include "messages/MPGStats.h"
+#include "messages/MPGStatsAck.h"
+#include "messages/MGetPoolStats.h"
+#include "messages/MGetPoolStatsReply.h"
+
+#include "messages/MStatfs.h"
+#include "messages/MStatfsReply.h"
+#include "messages/MOSDPGCreate.h"
+#include "messages/MMonCommand.h"
+#include "messages/MOSDScrub.h"
+
+#include "common/Timer.h"
+#include "common/Formatter.h"
+#include "common/ceph_argparse.h"
+#include "common/perf_counters.h"
+#include "common/TextTable.h"
+
+#include "include/stringify.h"
+
+#include "osd/osd_types.h"
+
+#include "common/config.h"
+#include "common/errno.h"
+#include "common/strtol.h"
+#include "include/str_list.h"
+#include 
+#include 
+#include "common/cmdparse.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, pg_map)
+static ostream& _prefix(std::ostream *_dout, const Monitor *mon, const PGMap& pg_map) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").pg v" << pg_map.version << " ";
+}
+
+/*
+ Tick function to update the map based on performance every N seconds
+*/
+
+void PGMonitor::on_restart()
+{
+  // clear leader state
+  last_sent_pg_create.clear();
+  last_osd_report.clear();
+}
+
+void PGMonitor::on_active()
+{
+  if (mon->is_leader()) {
+    check_osd_map(mon->osdmon()->osdmap.epoch);
+    need_check_down_pgs = true;
+  }
+
+  update_logger();
+
+  if (mon->is_leader())
+    mon->clog.info() << "pgmap " << pg_map << "\n";
+}
+
+void PGMonitor::update_logger()
+{
+  dout(10) << "update_logger" << dendl;
+
+  mon->cluster_logger->set(l_cluster_osd_kb, pg_map.osd_sum.kb);
+  mon->cluster_logger->set(l_cluster_osd_kb_used, pg_map.osd_sum.kb_used);
+  mon->cluster_logger->set(l_cluster_osd_kb_avail, pg_map.osd_sum.kb_avail);
+
+  mon->cluster_logger->set(l_cluster_num_pool, pg_map.pg_pool_sum.size());
+  mon->cluster_logger->set(l_cluster_num_pg, pg_map.pg_stat.size());
+
+  unsigned active = 0, active_clean = 0, peering = 0;
+  for (ceph::unordered_map::iterator p = pg_map.num_pg_by_state.begin();
+       p != pg_map.num_pg_by_state.end();
+       ++p) {
+    if (p->first & PG_STATE_ACTIVE) {
+      active += p->second;
+      if (p->first & PG_STATE_CLEAN)
+	active_clean += p->second;
+    }
+    if (p->first & PG_STATE_PEERING)
+      peering += p->second;
+  }
+  mon->cluster_logger->set(l_cluster_num_pg_active_clean, active_clean);
+  mon->cluster_logger->set(l_cluster_num_pg_active, active);
+  mon->cluster_logger->set(l_cluster_num_pg_peering, peering);
+
+  mon->cluster_logger->set(l_cluster_num_object, pg_map.pg_sum.stats.sum.num_objects);
+  mon->cluster_logger->set(l_cluster_num_object_degraded, pg_map.pg_sum.stats.sum.num_objects_degraded);
+  mon->cluster_logger->set(l_cluster_num_object_unfound, pg_map.pg_sum.stats.sum.num_objects_unfound);
+  mon->cluster_logger->set(l_cluster_num_bytes, pg_map.pg_sum.stats.sum.num_bytes);
+}
+
+void PGMonitor::tick() 
+{
+  if (!is_active()) return;
+
+  handle_osd_timeouts();
+
+  if (mon->is_leader()) {
+    bool propose = false;
+    
+    if (need_check_down_pgs && check_down_pgs())
+      propose = true;
+    
+    if (propose) {
+      propose_pending();
+    }
+  }
+
+  if (!pg_map.pg_sum_deltas.empty()) {
+    utime_t age = ceph_clock_now(g_ceph_context) - pg_map.stamp;
+    if (age > 2 * g_conf->mon_delta_reset_interval) {
+      dout(10) << " clearing pg_map delta (" << age << " > " << g_conf->mon_delta_reset_interval << " seconds old)" << dendl;
+      pg_map.clear_delta();
+    }
+  }
+
+  /* If we have deltas for pools, run through pgmap's 'per_pool_sum_delta' and
+   * clear any deltas that are old enough.
+   *
+   * Note that 'per_pool_sum_delta' keeps a pool id as key, and a pair containing
+   * the calc'ed stats delta and an absolute timestamp from when those stats were
+   * obtained -- the timestamp IS NOT a delta itself.
+   */
+  if (!pg_map.per_pool_sum_deltas.empty()) {
+    ceph::unordered_map >::iterator it;
+    for (it = pg_map.per_pool_sum_delta.begin();
+         it != pg_map.per_pool_sum_delta.end(); ) {
+      utime_t age = ceph_clock_now(g_ceph_context) - it->second.second;
+      if (age > 2*g_conf->mon_delta_reset_interval) {
+        dout(10) << " clearing pg_map delta for pool " << it->first
+                 << " (" << age << " > " << g_conf->mon_delta_reset_interval
+                 << " seconds old)" << dendl;
+        pg_map.per_pool_sum_deltas.erase(it->first);
+        pg_map.per_pool_sum_deltas_stamps.erase(it->first);
+        pg_map.per_pool_sum_delta.erase((it++)->first);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  dout(10) << pg_map << dendl;
+}
+
+void PGMonitor::create_initial()
+{
+  dout(10) << "create_initial -- creating initial map" << dendl;
+  format_version = 1;
+}
+
+void PGMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version_t version = get_last_committed();
+  if (version == pg_map.version)
+    return;
+  assert(version >= pg_map.version);
+
+  if (format_version == 0) {
+    // old format
+
+    /* Obtain latest full pgmap version, if available and whose version is
+     * greater than the current pgmap's version.
+     */
+    version_t latest_full = get_version_latest_full();
+    if ((latest_full > 0) && (latest_full > pg_map.version)) {
+      bufferlist latest_bl;
+      int err = get_version_full(latest_full, latest_bl);
+      assert(err == 0);
+      dout(7) << __func__ << " loading latest full pgmap v"
+	      << latest_full << dendl;
+      try {
+	PGMap tmp_pg_map;
+	bufferlist::iterator p = latest_bl.begin();
+	tmp_pg_map.decode(p);
+	pg_map = tmp_pg_map;
+      } catch (const std::exception& e) {
+	dout(0) << __func__ << ": error parsing update: "
+		<< e.what() << dendl;
+	assert(0 == "update_from_paxos: error parsing update");
+	return;
+      }
+    }
+
+    // walk through incrementals
+    while (version > pg_map.version) {
+      bufferlist bl;
+      int err = get_version(pg_map.version+1, bl);
+      assert(err == 0);
+      assert(bl.length());
+
+      dout(7) << "update_from_paxos  applying incremental " << pg_map.version+1 << dendl;
+      PGMap::Incremental inc;
+      try {
+	bufferlist::iterator p = bl.begin();
+	inc.decode(p);
+      }
+      catch (const std::exception &e) {
+	dout(0) << "update_from_paxos: error parsing "
+		<< "incremental update: " << e.what() << dendl;
+	assert(0 == "update_from_paxos: error parsing incremental update");
+	return;
+      }
+
+      pg_map.apply_incremental(g_ceph_context, inc);
+
+      dout(10) << pg_map << dendl;
+
+      if (inc.pg_scan)
+	last_sent_pg_create.clear();  // reset pg_create throttle timer
+    }
+
+  } else if (format_version == 1) {
+    // pg/osd keys in leveldb
+
+    // read meta
+    epoch_t last_pg_scan = pg_map.last_pg_scan;
+
+    while (version > pg_map.version) {
+      // load full state?
+      if (pg_map.version == 0) {
+	dout(10) << __func__ << " v0, read_full" << dendl;
+	read_pgmap_full();
+	goto out;
+      }
+
+      // incremental state?
+      dout(10) << __func__ << " read_incremental" << dendl;
+      bufferlist bl;
+      int r = get_version(pg_map.version + 1, bl);
+      if (r == -ENOENT) {
+	dout(10) << __func__ << " failed to read_incremental, read_full" << dendl;
+	read_pgmap_full();
+	goto out;
+      }
+      assert(r == 0);
+      apply_pgmap_delta(bl);
+    }
+
+    read_pgmap_meta();
+
+  out:
+    if (last_pg_scan != pg_map.last_pg_scan)
+      last_sent_pg_create.clear();  // reset pg_create throttle timer
+  }
+
+  assert(version == pg_map.version);
+
+  if (mon->osdmon()->osdmap.get_epoch()) {
+    map_pg_creates();
+    send_pg_creates();
+  }
+
+  update_logger();
+}
+
+void PGMonitor::on_upgrade()
+{
+  dout(1) << __func__ << " discarding in-core PGMap" << dendl;
+  pg_map = PGMap();
+}
+
+void PGMonitor::upgrade_format()
+{
+  unsigned current = 1;
+  assert(format_version <= current);
+  if (format_version == current)
+    return;
+
+  dout(1) << __func__ << " to " << current << dendl;
+
+  // upgrade by dirtying it all
+  pg_map.dirty_all(pending_inc);
+
+  format_version = current;
+  propose_pending();
+}
+
+void PGMonitor::post_paxos_update()
+{
+  if (mon->osdmon()->osdmap.get_epoch()) {
+    map_pg_creates();
+    send_pg_creates();
+  }
+}
+
+void PGMonitor::handle_osd_timeouts()
+{
+  if (!mon->is_leader())
+    return;
+  utime_t now(ceph_clock_now(g_ceph_context));
+  utime_t timeo(g_conf->mon_osd_report_timeout, 0);
+  if (now - mon->get_leader_since() < timeo) {
+    // We haven't been the leader for long enough to consider OSD timeouts
+    return;
+  }
+
+  if (mon->osdmon()->is_writeable())
+    mon->osdmon()->handle_osd_timeouts(now, last_osd_report);
+}
+
+void PGMonitor::create_pending()
+{
+  pending_inc = PGMap::Incremental();
+  pending_inc.version = pg_map.version + 1;
+  if (pg_map.version == 0) {
+    // pull initial values from first leader mon's config
+    pending_inc.full_ratio = g_conf->mon_osd_full_ratio;
+    if (pending_inc.full_ratio > 1.0)
+      pending_inc.full_ratio /= 100.0;
+    pending_inc.nearfull_ratio = g_conf->mon_osd_nearfull_ratio;
+    if (pending_inc.nearfull_ratio > 1.0)
+      pending_inc.nearfull_ratio /= 100.0;
+  } else {
+    pending_inc.full_ratio = pg_map.full_ratio;
+    pending_inc.nearfull_ratio = pg_map.nearfull_ratio;
+  }
+  dout(10) << "create_pending v " << pending_inc.version << dendl;
+}
+
+void PGMonitor::read_pgmap_meta()
+{
+  dout(10) << __func__ << dendl;
+
+  string prefix = pgmap_meta_prefix;
+
+  version_t version = mon->store->get(prefix, "version");
+  epoch_t last_osdmap_epoch = mon->store->get(prefix, "last_osdmap_epoch");
+  epoch_t last_pg_scan = mon->store->get(prefix, "last_pg_scan");
+  pg_map.set_version(version);
+  pg_map.set_last_osdmap_epoch(last_osdmap_epoch);
+
+  if (last_pg_scan != pg_map.get_last_pg_scan()) {
+    pg_map.set_last_pg_scan(last_pg_scan);
+    // clear our osdmap epoch so that map_pg_creates() will re-run
+    last_map_pg_create_osd_epoch = 0;
+  }
+
+  float full_ratio, nearfull_ratio;
+  {
+    bufferlist bl;
+    mon->store->get(prefix, "full_ratio", bl);
+    bufferlist::iterator p = bl.begin();
+    ::decode(full_ratio, p);
+  }
+  {
+    bufferlist bl;
+    mon->store->get(prefix, "nearfull_ratio", bl);
+    bufferlist::iterator p = bl.begin();
+    ::decode(nearfull_ratio, p);
+  }
+  pg_map.set_full_ratios(full_ratio, nearfull_ratio);
+  {
+    bufferlist bl;
+    mon->store->get(prefix, "stamp", bl);
+    bufferlist::iterator p = bl.begin();
+    utime_t stamp;
+    ::decode(stamp, p);
+    pg_map.set_stamp(stamp);
+  }
+}
+
+void PGMonitor::read_pgmap_full()
+{
+  read_pgmap_meta();
+
+  string prefix = pgmap_pg_prefix;
+  for (KeyValueDB::Iterator i = mon->store->get_iterator(prefix); i->valid(); i->next()) {
+    string key = i->key();
+    pg_t pgid;
+    if (!pgid.parse(key.c_str())) {
+      dout(0) << "unable to parse key " << key << dendl;
+      continue;
+    }
+    bufferlist bl = i->value();
+    pg_map.update_pg(pgid, bl);
+    dout(20) << " got " << pgid << dendl;
+  }
+
+  prefix = pgmap_osd_prefix;
+  for (KeyValueDB::Iterator i = mon->store->get_iterator(prefix); i->valid(); i->next()) {
+    string key = i->key();
+    int osd = atoi(key.c_str());
+    bufferlist bl = i->value();
+    pg_map.update_osd(osd, bl);
+    dout(20) << " got osd." << osd << dendl;
+  }
+}
+
+void PGMonitor::apply_pgmap_delta(bufferlist& bl)
+{
+  version_t v = pg_map.version + 1;
+
+  utime_t inc_stamp;
+  bufferlist dirty_pgs, dirty_osds;
+  {
+    bufferlist::iterator p = bl.begin();
+    ::decode(inc_stamp, p);
+    ::decode(dirty_pgs, p);
+    ::decode(dirty_osds, p);
+  }
+
+  pool_stat_t pg_sum_old = pg_map.pg_sum;
+  ceph::unordered_map pg_pool_sum_old;
+
+  // pgs
+  set deleted_pools;
+  bufferlist::iterator p = dirty_pgs.begin();
+  while (!p.end()) {
+    pg_t pgid;
+    ::decode(pgid, p);
+    bufferlist bl;
+    int r = mon->store->get(pgmap_pg_prefix, stringify(pgid), bl);
+    dout(20) << " refreshing pg " << pgid << " got " << r << " len "
+	     << bl.length() << dendl;
+
+    if (pg_pool_sum_old.count(pgid.pool()) == 0)
+      pg_pool_sum_old[pgid.pool()] = pg_map.pg_pool_sum[pgid.pool()];
+
+    if (r >= 0) {
+      pg_map.update_pg(pgid, bl);
+    } else {
+      pg_map.remove_pg(pgid);
+      if (pgid.ps() == 0)
+	deleted_pools.insert(pgid.pool());
+    }
+  }
+
+  // osds
+  p = dirty_osds.begin();
+  while (!p.end()) {
+    int32_t osd;
+    ::decode(osd, p);
+    dout(20) << " refreshing osd." << osd << dendl;
+    bufferlist bl;
+    int r = mon->store->get(pgmap_osd_prefix, stringify(osd), bl);
+    if (r >= 0) {
+      pg_map.update_osd(osd, bl);
+    } else {
+      pg_map.remove_osd(osd);
+    }
+  }
+
+  pg_map.update_global_delta(g_ceph_context, inc_stamp, pg_sum_old);
+  pg_map.update_pool_deltas(g_ceph_context, inc_stamp, pg_pool_sum_old);
+
+  // clean up deleted pools after updating the deltas
+  for (set::iterator p = deleted_pools.begin();
+       p != deleted_pools.end();
+       ++p) {
+    dout(20) << " deleted pool " << *p << dendl;
+    pg_map.deleted_pool(*p);
+  }
+
+  // ok, we're now on the new version
+  pg_map.version = v;
+}
+
+
+void PGMonitor::encode_pending(MonitorDBStore::Transaction *t)
+{
+  version_t version = pending_inc.version;
+  dout(10) << __func__ << " v " << version << dendl;
+  assert(get_last_committed() + 1 == version);
+  pending_inc.stamp = ceph_clock_now(g_ceph_context);
+
+  uint64_t features = mon->get_quorum_features();
+
+  string prefix = pgmap_meta_prefix;
+
+  t->put(prefix, "version", pending_inc.version);
+  {
+    bufferlist bl;
+    ::encode(pending_inc.stamp, bl);
+    t->put(prefix, "stamp", bl);
+  }
+
+  if (pending_inc.osdmap_epoch)
+    t->put(prefix, "last_osdmap_epoch", pending_inc.osdmap_epoch);
+  if (pending_inc.pg_scan)
+    t->put(prefix, "last_pg_scan", pending_inc.pg_scan);
+  if (pending_inc.full_ratio > 0) {
+    bufferlist bl;
+    ::encode(pending_inc.full_ratio, bl);
+    t->put(prefix, "full_ratio", bl);
+  }
+  if (pending_inc.nearfull_ratio > 0) {
+    bufferlist bl;
+    ::encode(pending_inc.nearfull_ratio, bl);
+    t->put(prefix, "nearfull_ratio", bl);
+  }
+
+  bufferlist incbl;
+  ::encode(pending_inc.stamp, incbl);
+  {
+    bufferlist dirty;
+    string prefix = pgmap_pg_prefix;
+    for (map::const_iterator p = pending_inc.pg_stat_updates.begin();
+	 p != pending_inc.pg_stat_updates.end();
+	 ++p) {
+      ::encode(p->first, dirty);
+      bufferlist bl;
+      ::encode(p->second, bl, features);
+      t->put(prefix, stringify(p->first), bl);
+    }
+    for (set::const_iterator p = pending_inc.pg_remove.begin(); p != pending_inc.pg_remove.end(); ++p) {
+      ::encode(*p, dirty);
+      t->erase(prefix, stringify(*p));
+    }
+    ::encode(dirty, incbl);
+  }
+  {
+    bufferlist dirty;
+    string prefix = pgmap_osd_prefix;
+    for (map::const_iterator p =
+	   pending_inc.get_osd_stat_updates().begin();
+	 p != pending_inc.get_osd_stat_updates().end();
+	 ++p) {
+      ::encode(p->first, dirty);
+      bufferlist bl;
+      ::encode(p->second, bl, features);
+      t->put(prefix, stringify(p->first), bl);
+    }
+    for (set::const_iterator p =
+	   pending_inc.get_osd_stat_rm().begin();
+	 p != pending_inc.get_osd_stat_rm().end();
+	 ++p) {
+      ::encode(*p, dirty);
+      t->erase(prefix, stringify(*p));
+    }
+    ::encode(dirty, incbl);
+  }
+
+  put_version(t, version, incbl);
+
+  put_last_committed(t, version);
+}
+
+version_t PGMonitor::get_trim_to()
+{
+  unsigned max = g_conf->mon_max_pgmap_epochs;
+  version_t version = get_last_committed();
+  if (mon->is_leader() && (version > max))
+    return version - max;
+  return 0;
+}
+
+bool PGMonitor::preprocess_query(PaxosServiceMessage *m)
+{
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case CEPH_MSG_STATFS:
+    handle_statfs(static_cast(m));
+    return true;
+  case MSG_GETPOOLSTATS:
+    return preprocess_getpoolstats(static_cast(m));
+    
+  case MSG_PGSTATS:
+    return preprocess_pg_stats(static_cast(m));
+
+  case MSG_MON_COMMAND:
+    return preprocess_command(static_cast(m));
+
+
+  default:
+    assert(0);
+    m->put();
+    return true;
+  }
+}
+
+bool PGMonitor::prepare_update(PaxosServiceMessage *m)
+{
+  dout(10) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_PGSTATS:
+    return prepare_pg_stats((MPGStats*)m);
+
+  case MSG_MON_COMMAND:
+    return prepare_command(static_cast(m));
+
+  default:
+    assert(0);
+    m->put();
+    return false;
+  }
+}
+
+void PGMonitor::handle_statfs(MStatfs *statfs)
+{
+  // check caps
+  MonSession *session = statfs->get_session();
+  if (!session)
+    goto out;
+  if (!session->is_capable("pg", MON_CAP_R)) {
+    dout(0) << "MStatfs received from entity with insufficient privileges "
+	    << session->caps << dendl;
+    goto out;
+  }
+  MStatfsReply *reply;
+
+  dout(10) << "handle_statfs " << *statfs << " from " << statfs->get_orig_source() << dendl;
+
+  if (statfs->fsid != mon->monmap->fsid) {
+    dout(0) << "handle_statfs on fsid " << statfs->fsid << " != " << mon->monmap->fsid << dendl;
+    goto out;
+  }
+
+  // fill out stfs
+  reply = new MStatfsReply(mon->monmap->fsid, statfs->get_tid(), get_last_committed());
+
+  // these are in KB.
+  reply->h.st.kb = pg_map.osd_sum.kb;
+  reply->h.st.kb_used = pg_map.osd_sum.kb_used;
+  reply->h.st.kb_avail = pg_map.osd_sum.kb_avail;
+  reply->h.st.num_objects = pg_map.pg_sum.stats.sum.num_objects;
+
+  // reply
+  mon->send_reply(statfs, reply);
+ out:
+  statfs->put();
+}
+
+bool PGMonitor::preprocess_getpoolstats(MGetPoolStats *m)
+{
+  MGetPoolStatsReply *reply;
+
+  MonSession *session = m->get_session();
+  if (!session)
+    goto out;
+  if (!session->is_capable("pg", MON_CAP_R)) {
+    dout(0) << "MGetPoolStats received from entity with insufficient caps "
+	    << session->caps << dendl;
+    goto out;
+  }
+
+  if (m->fsid != mon->monmap->fsid) {
+    dout(0) << "preprocess_getpoolstats on fsid " << m->fsid << " != " << mon->monmap->fsid << dendl;
+    goto out;
+  }
+  
+  reply = new MGetPoolStatsReply(m->fsid, m->get_tid(), get_last_committed());
+
+  for (list::iterator p = m->pools.begin();
+       p != m->pools.end();
+       ++p) {
+    int64_t poolid = mon->osdmon()->osdmap.lookup_pg_pool_name(p->c_str());
+    if (poolid < 0)
+      continue;
+    if (pg_map.pg_pool_sum.count(poolid) == 0)
+      continue;
+    reply->pool_stats[*p] = pg_map.pg_pool_sum[poolid];
+  }
+
+  mon->send_reply(m, reply);
+
+ out:
+  m->put();
+  return true;
+}
+
+
+bool PGMonitor::preprocess_pg_stats(MPGStats *stats)
+{
+  // check caps
+  MonSession *session = stats->get_session();
+  if (!session) {
+    dout(10) << "PGMonitor::preprocess_pg_stats: no monitor session!" << dendl;
+    stats->put();
+    return true;
+  }
+  if (!session->is_capable("pg", MON_CAP_R)) {
+    derr << "PGMonitor::preprocess_pg_stats: MPGStats received from entity "
+         << "with insufficient privileges " << session->caps << dendl;
+    stats->put();
+    return true;
+  }
+
+  // First, just see if they need a new osdmap. But
+  // only if they've had the map for a while.
+  if (stats->had_map_for > 30.0 && 
+      mon->osdmon()->is_readable() &&
+      stats->epoch < mon->osdmon()->osdmap.get_epoch())
+    mon->osdmon()->send_latest_now_nodelete(stats, stats->epoch+1);
+
+  // Always forward the PGStats to the leader, even if they are the same as
+  // the old PGStats. The leader will mark as down osds that haven't sent
+  // PGStats for a few minutes.
+  return false;
+}
+
+bool PGMonitor::pg_stats_have_changed(int from, const MPGStats *stats) const
+{
+  // any new osd info?
+  ceph::unordered_map::const_iterator s = pg_map.osd_stat.find(from);
+  if (s == pg_map.osd_stat.end())
+    return true;
+  if (s->second != stats->osd_stat)
+    return true;
+
+  // any new pg info?
+  for (map::const_iterator p = stats->pg_stat.begin();
+       p != stats->pg_stat.end(); ++p) {
+    ceph::unordered_map::const_iterator t = pg_map.pg_stat.find(p->first);
+    if (t == pg_map.pg_stat.end())
+      return true;
+    if (t->second.reported_epoch != p->second.reported_epoch ||
+	t->second.reported_seq != p->second.reported_seq)
+      return true;
+  }
+
+  return false;
+}
+
+bool PGMonitor::prepare_pg_stats(MPGStats *stats) 
+{
+  dout(10) << "prepare_pg_stats " << *stats << " from " << stats->get_orig_source() << dendl;
+  int from = stats->get_orig_source().num();
+
+  if (stats->fsid != mon->monmap->fsid) {
+    dout(0) << "prepare_pg_stats on fsid " << stats->fsid << " != " << mon->monmap->fsid << dendl;
+    stats->put();
+    return false;
+  }
+
+  last_osd_report[from] = ceph_clock_now(g_ceph_context);
+
+  if (!stats->get_orig_source().is_osd() ||
+      !mon->osdmon()->osdmap.is_up(from) ||
+      stats->get_orig_source_inst() != mon->osdmon()->osdmap.get_inst(from)) {
+    dout(1) << " ignoring stats from non-active osd." << dendl;
+    stats->put();
+    return false;
+  }
+      
+  if (!pg_stats_have_changed(from, stats)) {
+    dout(10) << " message contains no new osd|pg stats" << dendl;
+    MPGStatsAck *ack = new MPGStatsAck;
+    ack->set_tid(stats->get_tid());
+    for (map::const_iterator p = stats->pg_stat.begin();
+	 p != stats->pg_stat.end();
+	 ++p) {
+      ack->pg_stat[p->first] = make_pair(p->second.reported_seq, p->second.reported_epoch);
+    }
+    mon->send_reply(stats, ack);
+    stats->put();
+    return false;
+  }
+
+  // osd stat
+  if (mon->osdmon()->osdmap.is_in(from)) {
+    pending_inc.update_stat(from, stats->epoch, stats->osd_stat);
+  } else {
+    pending_inc.update_stat(from, stats->epoch, osd_stat_t());
+  }
+  
+  if (pg_map.osd_stat.count(from))
+    dout(10) << " got osd." << from << " " << stats->osd_stat << " (was " << pg_map.osd_stat[from] << ")" << dendl;
+  else
+    dout(10) << " got osd." << from << " " << stats->osd_stat << " (first report)" << dendl;
+
+  // pg stats
+  MPGStatsAck *ack = new MPGStatsAck;
+  ack->set_tid(stats->get_tid());
+  for (map::iterator p = stats->pg_stat.begin();
+       p != stats->pg_stat.end();
+       ++p) {
+    pg_t pgid = p->first;
+    ack->pg_stat[pgid] = make_pair(p->second.reported_seq, p->second.reported_epoch);
+
+    if (pg_map.pg_stat.count(pgid) &&
+        pg_map.pg_stat[pgid].get_version_pair() > p->second.get_version_pair()) {
+      dout(15) << " had " << pgid << " from " << pg_map.pg_stat[pgid].reported_epoch << ":"
+	       << pg_map.pg_stat[pgid].reported_seq << dendl;
+      continue;
+    }
+    if (pending_inc.pg_stat_updates.count(pgid) && 
+        pending_inc.pg_stat_updates[pgid].get_version_pair() > p->second.get_version_pair()) {
+      dout(15) << " had " << pgid << " from " << pending_inc.pg_stat_updates[pgid].reported_epoch << ":"
+	       << pending_inc.pg_stat_updates[pgid].reported_seq << " (pending)" << dendl;
+      continue;
+    }
+
+    if (pg_map.pg_stat.count(pgid) == 0) {
+      dout(15) << " got " << pgid << " reported at " << p->second.reported_epoch << ":"
+	       << p->second.reported_seq
+	       << " state " << pg_state_string(p->second.state)
+	       << " but DNE in pg_map; pool was probably deleted."
+	       << dendl;
+      continue;
+    }
+      
+    dout(15) << " got " << pgid
+	     << " reported at " << p->second.reported_epoch << ":" << p->second.reported_seq
+	     << " state " << pg_state_string(pg_map.pg_stat[pgid].state)
+	     << " -> " << pg_state_string(p->second.state)
+	     << dendl;
+    pending_inc.pg_stat_updates[pgid] = p->second;
+
+    /*
+    // we don't care much about consistency, here; apply to live map.
+    pg_map.stat_pg_sub(pgid, pg_map.pg_stat[pgid]);
+    pg_map.pg_stat[pgid] = p->second;
+    pg_map.stat_pg_add(pgid, pg_map.pg_stat[pgid]);
+    */
+  }
+  
+  wait_for_finished_proposal(new C_Stats(this, stats, ack));
+  return true;
+}
+
+void PGMonitor::_updated_stats(MPGStats *req, MPGStatsAck *ack)
+{
+  dout(7) << "_updated_stats for " << req->get_orig_source_inst() << dendl;
+  mon->send_reply(req, ack);
+  req->put();
+}
+
+
+
+// ------------------------
+
+struct RetryCheckOSDMap : public Context {
+  PGMonitor *pgmon;
+  epoch_t epoch;
+  RetryCheckOSDMap(PGMonitor *p, epoch_t e) : pgmon(p), epoch(e) {}
+  void finish(int r) {
+    if (r == -ECANCELED)
+      return;
+    pgmon->check_osd_map(epoch);
+  }
+};
+
+void PGMonitor::check_osd_map(epoch_t epoch)
+{
+  if (mon->is_peon()) 
+    return; // whatever.
+
+  if (pg_map.last_osdmap_epoch >= epoch) {
+    dout(10) << "check_osd_map already seen " << pg_map.last_osdmap_epoch << " >= " << epoch << dendl;
+    return;
+  }
+
+  if (!mon->osdmon()->is_readable()) {
+    dout(10) << "check_osd_map -- osdmap not readable, waiting" << dendl;
+    mon->osdmon()->wait_for_readable(new RetryCheckOSDMap(this, epoch));
+    return;
+  }
+
+  if (!is_writeable()) {
+    dout(10) << "check_osd_map -- pgmap not writeable, waiting" << dendl;
+    wait_for_writeable(new RetryCheckOSDMap(this, epoch));
+    return;
+  }
+
+  // apply latest map(s)
+  for (epoch_t e = pg_map.last_osdmap_epoch+1;
+       e <= epoch;
+       e++) {
+    dout(10) << "check_osd_map applying osdmap e" << e << " to pg_map" << dendl;
+    bufferlist bl;
+    int err = mon->osdmon()->get_version(e, bl);
+    assert(err == 0);
+
+    assert(bl.length());
+    OSDMap::Incremental inc(bl);
+    for (map::iterator p = inc.new_weight.begin();
+	 p != inc.new_weight.end();
+	 ++p)
+      if (p->second == CEPH_OSD_OUT) {
+	dout(10) << "check_osd_map  osd." << p->first << " went OUT" << dendl;
+	pending_inc.stat_osd_out(p->first);
+      }
+
+    // this is conservative: we want to know if any osds (maybe) got marked down.
+    for (map::iterator p = inc.new_state.begin();
+	 p != inc.new_state.end();
+	 ++p) {
+      if (p->second & CEPH_OSD_UP) {   // true if marked up OR down, but we're too lazy to check which
+	need_check_down_pgs = true;
+
+	// clear out the last_osd_report for this OSD
+        map::iterator report = last_osd_report.find(p->first);
+        if (report != last_osd_report.end()) {
+          last_osd_report.erase(report);
+        }
+      }
+
+      if (p->second & CEPH_OSD_EXISTS) {
+	// whether it was created *or* destroyed, we can safely drop
+	// it's osd_stat_t record.
+	dout(10) << "check_osd_map  osd." << p->first << " created or destroyed" << dendl;
+	pending_inc.rm_stat(p->first);
+
+	// and adjust full, nearfull set
+	pg_map.nearfull_osds.erase(p->first);
+	pg_map.full_osds.erase(p->first);
+      }
+    }
+  }
+
+  bool propose = false;
+  if (pg_map.last_osdmap_epoch < epoch) {
+    pending_inc.osdmap_epoch = epoch;
+    propose = true;
+  }
+
+  // scan pg space?
+  if (register_new_pgs())
+    propose = true;
+
+  if (need_check_down_pgs && check_down_pgs())
+    propose = true;
+  
+  if (propose)
+    propose_pending();
+
+  if (mon->osdmon()->osdmap.get_epoch()) {
+    map_pg_creates();
+    send_pg_creates();
+  }
+}
+
+void PGMonitor::register_pg(pg_pool_t& pool, pg_t pgid, epoch_t epoch, bool new_pool)
+{
+  pg_t parent;
+  int split_bits = 0;
+  if (!new_pool) {
+    parent = pgid;
+    while (1) {
+      // remove most significant bit
+      int msb = pool.calc_bits_of(parent.ps());
+      if (!msb) break;
+      parent.set_ps(parent.ps() & ~(1<<(msb-1)));
+      split_bits++;
+      dout(10) << " is " << pgid << " parent " << parent << " ?" << dendl;
+      //if (parent.u.pg.ps < mon->osdmon->osdmap.get_pgp_num()) {
+      if (pg_map.pg_stat.count(parent) &&
+	  pg_map.pg_stat[parent].state != PG_STATE_CREATING) {
+	dout(10) << "  parent is " << parent << dendl;
+	break;
+      }
+    }
+  }
+  
+  pending_inc.pg_stat_updates[pgid].state = PG_STATE_CREATING;
+  pending_inc.pg_stat_updates[pgid].created = epoch;
+  pending_inc.pg_stat_updates[pgid].parent = parent;
+  pending_inc.pg_stat_updates[pgid].parent_split_bits = split_bits;
+
+  if (split_bits == 0) {
+    dout(10) << "register_new_pgs  will create " << pgid << dendl;
+  } else {
+    dout(10) << "register_new_pgs  will create " << pgid
+	     << " parent " << parent
+	     << " by " << split_bits << " bits"
+	     << dendl;
+  }
+}
+
+bool PGMonitor::register_new_pgs()
+{
+  // iterate over crush mapspace
+  epoch_t epoch = mon->osdmon()->osdmap.get_epoch();
+  dout(10) << "register_new_pgs checking pg pools for osdmap epoch " << epoch
+	   << ", last_pg_scan " << pg_map.last_pg_scan << dendl;
+
+  OSDMap *osdmap = &mon->osdmon()->osdmap;
+
+  int created = 0;
+  for (map::iterator p = osdmap->pools.begin();
+       p != osdmap->pools.end();
+       ++p) {
+    int64_t poolid = p->first;
+    pg_pool_t &pool = p->second;
+    int ruleno = pool.get_crush_ruleset();
+    if (!osdmap->crush->rule_exists(ruleno)) 
+      continue;
+
+    if (pool.get_last_change() <= pg_map.last_pg_scan ||
+	pool.get_last_change() <= pending_inc.pg_scan) {
+      dout(10) << " no change in pool " << p->first << " " << pool << dendl;
+      continue;
+    }
+
+    dout(10) << "register_new_pgs scanning pool " << p->first << " " << pool << dendl;
+
+    bool new_pool = pg_map.pg_pool_sum.count(poolid) == 0;  // first pgs in this pool
+
+    for (ps_t ps = 0; ps < pool.get_pg_num(); ps++) {
+      pg_t pgid(ps, poolid, -1);
+      if (pg_map.pg_stat.count(pgid)) {
+	dout(20) << "register_new_pgs  have " << pgid << dendl;
+	continue;
+      }
+      created++;
+      register_pg(pool, pgid, pool.get_last_change(), new_pool);
+    }
+  }
+
+  int removed = 0;
+  for (set::iterator p = pg_map.creating_pgs.begin();
+       p != pg_map.creating_pgs.end();
+       ++p) {
+    if (p->preferred() >= 0) {
+      dout(20) << " removing creating_pg " << *p << " because it is localized and obsolete" << dendl;
+      pending_inc.pg_remove.insert(*p);
+      removed++;
+    }
+    if (!osdmap->have_pg_pool(p->pool())) {
+      dout(20) << " removing creating_pg " << *p << " because containing pool deleted" << dendl;
+      pending_inc.pg_remove.insert(*p);
+      ++removed;
+    }
+  }
+
+  // deleted pools?
+  for (ceph::unordered_map::const_iterator p = pg_map.pg_stat.begin();
+       p != pg_map.pg_stat.end(); ++p) {
+    if (!osdmap->have_pg_pool(p->first.pool())) {
+      dout(20) << " removing pg_stat " << p->first << " because "
+	       << "containing pool deleted" << dendl;
+      pending_inc.pg_remove.insert(p->first);
+      ++removed;
+    }
+    if (p->first.preferred() >= 0) {
+      dout(20) << " removing localized pg " << p->first << dendl;
+      pending_inc.pg_remove.insert(p->first);
+      ++removed;
+    }
+  }
+
+  dout(10) << "register_new_pgs registered " << created << " new pgs, removed "
+	   << removed << " uncreated pgs" << dendl;
+  if (created || removed) {
+    pending_inc.pg_scan = epoch;
+    return true;
+  }
+  return false;
+}
+
+void PGMonitor::map_pg_creates()
+{
+  OSDMap *osdmap = &mon->osdmon()->osdmap;
+  if (osdmap->get_epoch() == last_map_pg_create_osd_epoch) {
+    dout(10) << "map_pg_creates to " << pg_map.creating_pgs.size() << " pgs -- no change" << dendl;
+    return;
+  }
+
+  dout(10) << "map_pg_creates to " << pg_map.creating_pgs.size() << " pgs osdmap epoch " << osdmap->get_epoch() << dendl;
+  last_map_pg_create_osd_epoch = osdmap->get_epoch();
+
+  for (set::iterator p = pg_map.creating_pgs.begin();
+       p != pg_map.creating_pgs.end();
+       ++p) {
+    pg_t pgid = *p;
+    pg_t on = pgid;
+    pg_stat_t& s = pg_map.pg_stat[pgid];
+    if (s.parent_split_bits)
+      on = s.parent;
+
+    vector up, acting;
+    int up_primary, acting_primary;
+    osdmap->pg_to_up_acting_osds(
+      on,
+      &up,
+      &up_primary,
+      &acting,
+      &acting_primary);
+
+    if (s.acting_primary != -1) {
+      pg_map.creating_pgs_by_osd[s.acting_primary].erase(pgid);
+      if (pg_map.creating_pgs_by_osd[s.acting_primary].size() == 0)
+        pg_map.creating_pgs_by_osd.erase(s.acting_primary);
+    }
+    s.up = up;
+    s.up_primary = up_primary;
+    s.acting = acting;
+    s.acting_primary = acting_primary;
+
+    // don't send creates for localized pgs
+    if (pgid.preferred() >= 0)
+      continue;
+
+    // don't send creates for splits
+    if (s.parent_split_bits)
+      continue;
+
+    if (acting_primary != -1) {
+      pg_map.creating_pgs_by_osd[acting_primary].insert(pgid);
+    } else {
+      dout(20) << "map_pg_creates  " << pgid << " -> no osds in epoch "
+	       << mon->osdmon()->osdmap.get_epoch() << ", skipping" << dendl;
+      continue;  // blarney!
+    }
+  }
+  for (map >::iterator p = pg_map.creating_pgs_by_osd.begin();
+       p != pg_map.creating_pgs_by_osd.end();
+       ++p) {
+    dout(10) << "map_pg_creates osd." << p->first << " has " << p->second.size() << " pgs" << dendl;
+  }
+}
+
+void PGMonitor::send_pg_creates()
+{
+  dout(10) << "send_pg_creates to " << pg_map.creating_pgs.size() << " pgs" << dendl;
+
+  utime_t now = ceph_clock_now(g_ceph_context);
+  for (map >::iterator p = pg_map.creating_pgs_by_osd.begin();
+       p != pg_map.creating_pgs_by_osd.end();
+       ++p) {
+    int osd = p->first;
+
+    // throttle?
+    if (last_sent_pg_create.count(osd) &&
+	now - g_conf->mon_pg_create_interval < last_sent_pg_create[osd]) 
+      continue;
+      
+    if (mon->osdmon()->osdmap.is_up(osd))
+      send_pg_creates(osd, NULL);
+  }
+}
+
+void PGMonitor::send_pg_creates(int osd, Connection *con)
+{
+  map >::iterator p = pg_map.creating_pgs_by_osd.find(osd);
+  if (p == pg_map.creating_pgs_by_osd.end())
+    return;
+  assert(p->second.size() > 0);
+
+  dout(20) << "send_pg_creates osd." << osd << " pgs " << p->second << dendl;
+  MOSDPGCreate *m = new MOSDPGCreate(mon->osdmon()->osdmap.get_epoch());
+  for (set::iterator q = p->second.begin(); q != p->second.end(); ++q) {
+    m->mkpg[*q] = pg_create_t(pg_map.pg_stat[*q].created,
+			      pg_map.pg_stat[*q].parent,
+			      pg_map.pg_stat[*q].parent_split_bits);
+  }
+
+  if (con) {
+    mon->messenger->send_message(m, con);
+  } else {
+    assert(mon->osdmon()->osdmap.is_up(osd));
+    mon->messenger->send_message(m, mon->osdmon()->osdmap.get_inst(osd));
+  }
+  last_sent_pg_create[osd] = ceph_clock_now(g_ceph_context);
+}
+
+bool PGMonitor::check_down_pgs()
+{
+  dout(10) << "check_down_pgs" << dendl;
+
+  OSDMap *osdmap = &mon->osdmon()->osdmap;
+  bool ret = false;
+
+  for (ceph::unordered_map::iterator p = pg_map.pg_stat.begin();
+       p != pg_map.pg_stat.end();
+       ++p) {
+    if ((p->second.state & PG_STATE_STALE) == 0 &&
+	p->second.acting_primary != -1 &&
+	osdmap->is_down(p->second.acting_primary)) {
+      dout(10) << " marking pg " << p->first << " stale with acting " << p->second.acting << dendl;
+
+      map::iterator q = pending_inc.pg_stat_updates.find(p->first);
+      pg_stat_t *stat;
+      if (q == pending_inc.pg_stat_updates.end()) {
+	stat = &pending_inc.pg_stat_updates[p->first];
+	*stat = p->second;
+      } else {
+	stat = &q->second;
+      }
+      stat->state |= PG_STATE_STALE;
+      stat->last_unstale = ceph_clock_now(g_ceph_context);
+      ret = true;
+    }
+  }
+  need_check_down_pgs = false;
+
+  return ret;
+}
+
+inline string percentify(const float& a) {
+  stringstream ss;
+  if (a < 0.01)
+    ss << "0";
+  else
+    ss << std::fixed << std::setprecision(2) << a;
+  return ss.str();
+}
+
+//void PGMonitor::dump_object_stat_sum(stringstream& ss, Formatter *f,
+void PGMonitor::dump_object_stat_sum(TextTable &tbl, Formatter *f,
+				     object_stat_sum_t &sum, uint64_t avail,
+				     bool verbose)
+{
+  if (f) {
+    f->dump_int("kb_used", SHIFT_ROUND_UP(sum.num_bytes, 10));
+    f->dump_int("bytes_used", sum.num_bytes);
+    f->dump_unsigned("max_avail", avail);
+    f->dump_int("objects", sum.num_objects);
+    if (verbose) {
+      f->dump_int("dirty", sum.num_objects_dirty);
+      f->dump_int("rd", sum.num_rd);
+      f->dump_int("rd_kb", sum.num_rd_kb);
+      f->dump_int("wr", sum.num_wr);
+      f->dump_int("wr_kb", sum.num_wr_kb);
+    }
+  } else {
+    tbl << stringify(si_t(sum.num_bytes));
+    int64_t kb_used = SHIFT_ROUND_UP(sum.num_bytes, 10);
+    tbl << percentify(((float)kb_used / pg_map.osd_sum.kb)*100);
+    tbl << si_t(avail);
+    tbl << sum.num_objects;
+    if (verbose) {
+      tbl << stringify(si_t(sum.num_objects_dirty))
+	  << stringify(si_t(sum.num_rd))
+          << stringify(si_t(sum.num_wr));
+    }
+  }
+}
+
+int64_t PGMonitor::get_rule_avail(OSDMap& osdmap, int ruleno)
+{
+  map wm;
+  int r = osdmap.crush->get_rule_weight_osd_map(ruleno, &wm);
+  if (r < 0)
+    return r;
+  if(wm.size() == 0)
+    return 0;
+  int64_t min = -1;
+  for (map::iterator p = wm.begin(); p != wm.end(); ++p) {
+    int64_t proj = (float)(pg_map.osd_stat[p->first].kb_avail * 1024ull) /
+      (double)p->second;
+    if (min < 0 || proj < min)
+      min = proj;
+  }
+  return min;
+}
+
+void PGMonitor::dump_pool_stats(stringstream &ss, Formatter *f, bool verbose)
+{
+  TextTable tbl;
+
+  if (f) {
+    f->open_array_section("pools");
+  } else {
+    tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("ID", TextTable::LEFT, TextTable::LEFT);
+    if (verbose)
+      tbl.define_column("CATEGORY", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("USED", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("%USED", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("MAX AVAIL", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("OBJECTS", TextTable::LEFT, TextTable::RIGHT);
+    if (verbose) {
+      tbl.define_column("DIRTY", TextTable::LEFT, TextTable::RIGHT);
+      tbl.define_column("READ", TextTable::LEFT, TextTable::RIGHT);
+      tbl.define_column("WRITE", TextTable::LEFT, TextTable::RIGHT);
+    }
+  }
+
+  map avail_by_rule;
+  OSDMap &osdmap = mon->osdmon()->osdmap;
+  for (map::const_iterator p = osdmap.get_pools().begin();
+       p != osdmap.get_pools().end(); ++p) {
+    int64_t pool_id = p->first;
+    if ((pool_id < 0) || (pg_map.pg_pool_sum.count(pool_id) == 0))
+      continue;
+    string pool_name = osdmap.get_pool_name(pool_id);
+    pool_stat_t &stat = pg_map.pg_pool_sum[pool_id];
+
+    const pg_pool_t *pool = osdmap.get_pg_pool(pool_id);
+    int ruleno = osdmap.crush->find_rule(pool->get_crush_ruleset(),
+					 pool->get_type(),
+					 pool->get_size());
+    uint64_t avail;
+    if (avail_by_rule.count(ruleno) == 0) {
+      avail = get_rule_avail(osdmap, ruleno);
+      avail_by_rule[ruleno] = avail;
+    } else {
+      avail = avail_by_rule[ruleno];
+    }
+    switch (pool->get_type()) {
+    case pg_pool_t::TYPE_REPLICATED:
+      avail /= pool->get_size();
+      break;
+    case pg_pool_t::TYPE_ERASURE:
+      {
+	const map& ecp =
+	  osdmap.get_erasure_code_profile(pool->erasure_code_profile);
+	map::const_iterator pm = ecp.find("m");
+	map::const_iterator pk = ecp.find("k");
+	if (pm != ecp.end() && pk != ecp.end()) {
+	  int k = atoi(pk->second.c_str());
+	  int m = atoi(pm->second.c_str());
+	  avail = avail * k / (m + k);
+	}
+      }
+      break;
+    default:
+      assert(0 == "unrecognized pool type");
+    }
+
+    if (f) {
+      f->open_object_section("pool");
+      f->dump_string("name", pool_name);
+      f->dump_int("id", pool_id);
+      f->open_object_section("stats");
+    } else {
+      tbl << pool_name
+          << pool_id;
+      if (verbose)
+        tbl << "-";
+    }
+    dump_object_stat_sum(tbl, f, stat.stats.sum, avail, verbose);
+    if (f)
+      f->close_section(); // stats
+    else
+      tbl << TextTable::endrow;
+
+    if (verbose) {
+      if (f)
+        f->open_array_section("categories");
+
+      for (map::iterator it = stat.stats.cat_sum.begin();
+          it != stat.stats.cat_sum.end(); ++it) {
+        if (f) {
+          f->open_object_section(it->first.c_str());
+        } else {
+          tbl << ""
+              << ""
+              << it->first;
+        }
+        dump_object_stat_sum(tbl, f, it->second, avail, verbose);
+        if (f)
+          f->close_section(); // category name
+        else
+          tbl << TextTable::endrow;
+      }
+      if (f)
+        f->close_section(); // categories
+    }
+    if (f)
+      f->close_section(); // pool
+  }
+  if (f)
+    f->close_section();
+  else {
+    ss << "POOLS:\n";
+    tbl.set_indent(4);
+    ss << tbl;
+  }
+}
+
+void PGMonitor::dump_fs_stats(stringstream &ss, Formatter *f, bool verbose)
+{
+  if (f) {
+    f->open_object_section("stats");
+    f->dump_int("total_space", pg_map.osd_sum.kb);
+    f->dump_int("total_used", pg_map.osd_sum.kb_used);
+    f->dump_int("total_avail", pg_map.osd_sum.kb_avail);
+    if (verbose) {
+      f->dump_int("total_objects", pg_map.pg_sum.stats.sum.num_objects);
+    }
+    f->close_section();
+  } else {
+    TextTable tbl;
+    tbl.define_column("SIZE", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("RAW USED", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("%RAW USED", TextTable::LEFT, TextTable::RIGHT);
+    if (verbose) {
+      tbl.define_column("OBJECTS", TextTable::LEFT, TextTable::RIGHT);
+    }
+    tbl << stringify(si_t(pg_map.osd_sum.kb*1024))
+        << stringify(si_t(pg_map.osd_sum.kb_avail*1024))
+        << stringify(si_t(pg_map.osd_sum.kb_used*1024));
+    tbl << percentify(((float)pg_map.osd_sum.kb_used / pg_map.osd_sum.kb)*100);
+    if (verbose) {
+      tbl << stringify(si_t(pg_map.pg_sum.stats.sum.num_objects));
+    }
+    tbl << TextTable::endrow;
+
+    ss << "GLOBAL:\n";
+    tbl.set_indent(4);
+    ss << tbl;
+  }
+}
+
+
+void PGMonitor::dump_info(Formatter *f)
+{
+  f->open_object_section("pgmap");
+  pg_map.dump(f);
+  f->close_section();
+
+  f->dump_unsigned("pgmap_first_committed", get_first_committed());
+  f->dump_unsigned("pgmap_last_committed", get_last_committed());
+}
+
+bool PGMonitor::preprocess_command(MMonCommand *m)
+{
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss, ds;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  // perhaps these would be better in the parsing, but it's weird
+  if (prefix == "pg dump_json") {
+    vector v;
+    v.push_back(string("all"));
+    cmd_putval(g_ceph_context, cmdmap, "format", string("json"));
+    cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v);
+    prefix = "pg dump";
+  } else if (prefix == "pg dump_pools_json") {
+    vector v;
+    v.push_back(string("pools"));
+    cmd_putval(g_ceph_context, cmdmap, "format", string("json"));
+    cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v);
+    prefix = "pg dump";
+  }
+
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr f(new_formatter(format));
+
+  if (prefix == "pg stat") {
+    if (f) {
+      f->open_object_section("pg_map");
+      pg_map.dump(f.get());
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << pg_map;
+    }
+    r = 0;
+  } else if (prefix == "pg getmap") {
+    pg_map.encode(rdata);
+    ss << "got pgmap version " << pg_map.version;
+    r = 0;
+  } else if (prefix == "pg map_pg_creates") {
+    map_pg_creates();
+    ss << "mapped pg creates ";
+    r = 0;
+  } else if (prefix == "pg send_pg_creates") {
+    send_pg_creates();
+    ss << "sent pg creates ";
+    r = 0;
+  } else if (prefix == "pg dump") {
+    string val;
+    vector dumpcontents;
+    set what;
+    if (cmd_getval(g_ceph_context, cmdmap, "dumpcontents", dumpcontents)) {
+      copy(dumpcontents.begin(), dumpcontents.end(),
+	   inserter(what, what.end()));
+    }
+    if (what.empty())
+      what.insert("all");
+    if (f) {
+      vector dumpcontents;
+      if (cmd_getval(g_ceph_context, cmdmap, "dumpcontents", dumpcontents)) {
+	copy(dumpcontents.begin(), dumpcontents.end(),
+	     inserter(what, what.end()));
+      }
+      if (what.count("all")) {
+	f->open_object_section("pg_map");
+	pg_map.dump(f.get());
+	f->close_section();
+      } else if (what.count("summary") || what.count("sum")) {
+	f->open_object_section("pg_map");
+	pg_map.dump_basic(f.get());
+	f->close_section();
+      } else {
+	if (what.count("pools")) {
+	  pg_map.dump_pool_stats(f.get());
+	}
+	if (what.count("osds")) {
+	  pg_map.dump_osd_stats(f.get());
+	}
+	if (what.count("pgs")) {
+	  pg_map.dump_pg_stats(f.get(), false);
+	}
+	if (what.count("pgs_brief")) {
+	  pg_map.dump_pg_stats(f.get(), true);
+	}
+	if (what.count("delta")) {
+	  f->open_object_section("delta");
+	  pg_map.dump_delta(f.get());
+	  f->close_section();
+	}
+      }
+      f->flush(ds);
+    } else {
+      // plain format ignores dumpcontents
+      pg_map.dump(ds);
+    }
+    ss << "dumped " << what << " in format " << format;
+    r = 0;
+  } else if (prefix == "pg dump_stuck") {
+    vector stuckop_vec;
+    cmd_getval(g_ceph_context, cmdmap, "stuckops", stuckop_vec);
+    if (stuckop_vec.empty())
+      stuckop_vec.push_back("unclean");
+    int64_t threshold;
+    cmd_getval(g_ceph_context, cmdmap, "threshold", threshold,
+	       int64_t(g_conf->mon_pg_stuck_threshold));
+
+    r = dump_stuck_pg_stats(ds, f.get(), (int)threshold, stuckop_vec);
+    ss << "ok";
+    r = 0;
+  } else if (prefix == "pg map") {
+    pg_t pgid;
+    string pgidstr;
+    cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr);
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    vector up, acting;
+    if (!mon->osdmon()->osdmap.have_pg_pool(pgid.pool())) {
+      r = -ENOENT;
+      ss << "pg '" << pgidstr << "' does not exist";
+      goto reply;
+    }
+    pg_t mpgid = mon->osdmon()->osdmap.raw_pg_to_pg(pgid);
+    mon->osdmon()->osdmap.pg_to_up_acting_osds(pgid, up, acting);
+    if (f) {
+      f->open_object_section("pg_map");
+      f->dump_stream("epoch") << mon->osdmon()->osdmap.get_epoch();
+      f->dump_stream("raw_pgid") << pgid;
+      f->dump_stream("pgid") << mpgid;
+
+      f->open_array_section("up");
+      for (vector::iterator it = up.begin(); it != up.end(); ++it)
+	f->dump_int("up_osd", *it);
+      f->close_section();
+
+      f->open_array_section("acting");
+      for (vector::iterator it = acting.begin(); it != acting.end(); ++it)
+	f->dump_int("acting_osd", *it);
+      f->close_section();
+
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << "osdmap e" << mon->osdmon()->osdmap.get_epoch()
+	 << " pg " << pgid << " (" << mpgid << ")"
+	 << " -> up " << up << " acting " << acting;
+    }
+    r = 0;
+  } else if (prefix == "pg scrub" || 
+	     prefix == "pg repair" || 
+	     prefix == "pg deep-scrub") {
+    string scrubop = prefix.substr(3, string::npos);
+    pg_t pgid;
+    string pgidstr;
+    cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr);
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    if (!pg_map.pg_stat.count(pgid)) {
+      ss << "pg " << pgid << " dne";
+      r = -ENOENT;
+      goto reply;
+    }
+    if (pg_map.pg_stat[pgid].acting_primary == -1) {
+      ss << "pg " << pgid << " has no primary osd";
+      r = -EAGAIN;
+      goto reply;
+    }
+    int osd = pg_map.pg_stat[pgid].acting_primary;
+    if (!mon->osdmon()->osdmap.is_up(osd)) {
+      ss << "pg " << pgid << " primary osd." << osd << " not up";
+      r = -EAGAIN;
+      goto reply;
+    }
+    vector pgs(1);
+    pgs[0] = pgid;
+    mon->try_send_message(new MOSDScrub(mon->monmap->fsid, pgs,
+					scrubop == "repair",
+					scrubop == "deep-scrub"),
+			  mon->osdmon()->osdmap.get_inst(osd));
+    ss << "instructing pg " << pgid << " on osd." << osd << " to " << scrubop;
+    r = 0;
+  } else if (prefix == "pg debug") {
+    string debugop;
+    cmd_getval(g_ceph_context, cmdmap, "debugop", debugop, string("unfound_objects_exist"));
+    if (debugop == "unfound_objects_exist") {
+      bool unfound_objects_exist = false;
+      ceph::unordered_map::const_iterator end = pg_map.pg_stat.end();
+      for (ceph::unordered_map::const_iterator s = pg_map.pg_stat.begin();
+	   s != end; ++s) {
+	if (s->second.stats.sum.num_objects_unfound > 0) {
+	  unfound_objects_exist = true;
+	  break;
+	}
+      }
+      if (unfound_objects_exist)
+	ds << "TRUE";
+      else
+	ds << "FALSE";
+      r = 0;
+    } else if (debugop == "degraded_pgs_exist") {
+      bool degraded_pgs_exist = false;
+      ceph::unordered_map::const_iterator end = pg_map.pg_stat.end();
+      for (ceph::unordered_map::const_iterator s = pg_map.pg_stat.begin();
+	   s != end; ++s) {
+	if (s->second.stats.sum.num_objects_degraded > 0) {
+	  degraded_pgs_exist = true;
+	  break;
+	}
+      }
+      if (degraded_pgs_exist)
+	ds << "TRUE";
+      else
+	ds << "FALSE";
+      r = 0;
+    }
+  }
+
+  if (r == -1)
+    return false;
+
+ reply:
+  string rs;
+  getline(ss, rs);
+  rdata.append(ds);
+  mon->reply_command(m, r, rs, rdata, get_last_committed());
+  return true;
+}
+
+bool PGMonitor::prepare_command(MMonCommand *m)
+{
+  stringstream ss;
+  pg_t pgid;
+  epoch_t epoch = mon->osdmon()->osdmap.get_epoch();
+  int r = 0;
+  string rs;
+
+  map cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon->reply_command(m, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
+
+  MonSession *session = m->get_session();
+  if (!session) {
+    mon->reply_command(m, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  if (prefix == "pg force_create_pg") {
+    string pgidstr;
+    cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr);
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "pg " << pgidstr << " invalid";
+      r = -EINVAL;
+      goto reply;
+    }
+    if (!pg_map.pg_stat.count(pgid)) {
+      ss << "pg " << pgid << " dne";
+      r = -ENOENT;
+      goto reply;
+    }
+    if (pg_map.creating_pgs.count(pgid)) {
+      ss << "pg " << pgid << " already creating";
+      r = 0;
+      goto reply;
+    }
+    {
+      pg_stat_t& s = pending_inc.pg_stat_updates[pgid];
+      s.state = PG_STATE_CREATING;
+      s.created = epoch;
+      s.last_change = ceph_clock_now(g_ceph_context);
+    }
+    ss << "pg " << pgidstr << " now creating, ok";
+    goto update;
+  } else if (prefix == "pg set_full_ratio" ||
+	     prefix == "pg set_nearfull_ratio") {
+    double n;
+    if (!cmd_getval(g_ceph_context, cmdmap, "ratio", n)) {
+      ss << "unable to parse 'ratio' value '"
+         << cmd_vartype_stringify(cmdmap["who"]) << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    string op = prefix.substr(3, string::npos);
+    if (op == "set_full_ratio")
+      pending_inc.full_ratio = n;
+    else if (op == "set_nearfull_ratio")
+      pending_inc.nearfull_ratio = n;
+    goto update;
+  } else {
+    r = -EINVAL;
+    goto reply;
+  }
+
+ reply:
+  getline(ss, rs);
+  if (r < 0 && rs.length() == 0)
+    rs = cpp_strerror(r);
+  mon->reply_command(m, r, rs, get_last_committed());
+  return false;
+
+ update:
+  getline(ss, rs);
+  wait_for_finished_proposal(new Monitor::C_Command(mon, m, r, rs,
+						    get_last_committed() + 1));
+  return true;
+}
+
+static void note_stuck_detail(enum PGMap::StuckPG what,
+			      ceph::unordered_map& stuck_pgs,
+			      list > *detail)
+{
+  for (ceph::unordered_map::iterator p = stuck_pgs.begin();
+       p != stuck_pgs.end();
+       ++p) {
+    ostringstream ss;
+    utime_t since;
+    const char *whatname = 0;
+    switch (what) {
+    case PGMap::STUCK_INACTIVE:
+      since = p->second.last_active;
+      whatname = "inactive";
+      break;
+    case PGMap::STUCK_UNCLEAN:
+      since = p->second.last_clean;
+      whatname = "unclean";
+      break;
+    case PGMap::STUCK_STALE:
+      since = p->second.last_unstale;
+      whatname = "stale";
+      break;
+    default:
+      assert(0);
+    }
+    ss << "pg " << p->first << " is stuck " << whatname;
+    if (since == utime_t()) {
+      ss << " since forever";
+    }else {
+      utime_t dur = ceph_clock_now(g_ceph_context) - since;
+      ss << " for " << dur;
+    }
+    ss << ", current state " << pg_state_string(p->second.state)
+       << ", last acting " << p->second.acting;
+    detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+  }
+}
+
+int PGMonitor::_warn_slow_request_histogram(const pow2_hist_t& h, string suffix,
+					    list >& summary,
+					    list > *detail) const
+{
+  unsigned sum = 0;
+  for (unsigned i = h.h.size() - 1; i > 0; --i) {
+    float ub = (float)(1 << i) / 1000.0;
+    if (ub < g_conf->mon_osd_max_op_age)
+      break;
+    ostringstream ss;
+    if (h.h[i]) {
+      ss << h.h[i] << " ops are blocked > " << ub << " sec" << suffix;
+      if (detail)
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      sum += h.h[i];
+    }
+  }
+  return sum;
+}
+
+void PGMonitor::get_health(list >& summary,
+			   list > *detail) const
+{
+  map note;
+  ceph::unordered_map::const_iterator p = pg_map.num_pg_by_state.begin();
+  ceph::unordered_map::const_iterator p_end = pg_map.num_pg_by_state.end();
+  for (; p != p_end; ++p) {
+    if (p->first & PG_STATE_STALE)
+      note["stale"] += p->second;
+    if (p->first & PG_STATE_DOWN)
+      note["down"] += p->second;
+    if (p->first & PG_STATE_DEGRADED)
+      note["degraded"] += p->second;
+    if (p->first & PG_STATE_INCONSISTENT)
+      note["inconsistent"] += p->second;
+    if (p->first & PG_STATE_PEERING)
+      note["peering"] += p->second;
+    if (p->first & PG_STATE_REPAIR)
+      note["repair"] += p->second;
+    if (p->first & PG_STATE_SPLITTING)
+      note["splitting"] += p->second;
+    if (p->first & PG_STATE_RECOVERING)
+      note["recovering"] += p->second;
+    if (p->first & PG_STATE_RECOVERY_WAIT)
+      note["recovery_wait"] += p->second;
+    if (p->first & PG_STATE_INCOMPLETE)
+      note["incomplete"] += p->second;
+    if (p->first & PG_STATE_BACKFILL_WAIT)
+      note["backfill"] += p->second;
+    if (p->first & PG_STATE_BACKFILL)
+      note["backfilling"] += p->second;
+    if (p->first & PG_STATE_BACKFILL_TOOFULL)
+      note["backfill_toofull"] += p->second;
+  }
+
+  ceph::unordered_map stuck_pgs;
+  utime_t now(ceph_clock_now(g_ceph_context));
+  utime_t cutoff = now - utime_t(g_conf->mon_pg_stuck_threshold, 0);
+
+  pg_map.get_stuck_stats(PGMap::STUCK_INACTIVE, cutoff, stuck_pgs);
+  if (!stuck_pgs.empty()) {
+    note["stuck inactive"] = stuck_pgs.size();
+    if (detail)
+      note_stuck_detail(PGMap::STUCK_INACTIVE, stuck_pgs, detail);
+  }
+  stuck_pgs.clear();
+
+  pg_map.get_stuck_stats(PGMap::STUCK_UNCLEAN, cutoff, stuck_pgs);
+  if (!stuck_pgs.empty()) {
+    note["stuck unclean"] = stuck_pgs.size();
+    if (detail)
+      note_stuck_detail(PGMap::STUCK_UNCLEAN, stuck_pgs, detail);
+  }
+  stuck_pgs.clear();
+
+  pg_map.get_stuck_stats(PGMap::STUCK_STALE, cutoff, stuck_pgs);
+  if (!stuck_pgs.empty()) {
+    note["stuck stale"] = stuck_pgs.size();
+    if (detail)
+      note_stuck_detail(PGMap::STUCK_STALE, stuck_pgs, detail);
+  }
+
+  if (!note.empty()) {
+    for (map::iterator p = note.begin(); p != note.end(); ++p) {
+      ostringstream ss;
+      ss << p->second << " pgs " << p->first;
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+    if (detail) {
+      for (ceph::unordered_map::const_iterator p = pg_map.pg_stat.begin();
+	   p != pg_map.pg_stat.end();
+	   ++p) {
+	if ((p->second.state & (PG_STATE_STALE |
+			       PG_STATE_DOWN |
+			       PG_STATE_DEGRADED |
+			       PG_STATE_INCONSISTENT |
+			       PG_STATE_PEERING |
+			       PG_STATE_REPAIR |
+			       PG_STATE_SPLITTING |
+			       PG_STATE_RECOVERING |
+			       PG_STATE_RECOVERY_WAIT |
+			       PG_STATE_INCOMPLETE |
+			       PG_STATE_BACKFILL_WAIT |
+			       PG_STATE_BACKFILL |
+			       PG_STATE_BACKFILL_TOOFULL)) &&
+	    stuck_pgs.count(p->first) == 0) {
+	  ostringstream ss;
+	  ss << "pg " << p->first << " is " << pg_state_string(p->second.state);
+	  ss << ", acting " << p->second.acting;
+	  if (p->second.stats.sum.num_objects_unfound)
+	    ss << ", " << p->second.stats.sum.num_objects_unfound << " unfound";
+	  if (p->second.state & PG_STATE_INCOMPLETE) {
+	    const pg_pool_t *pi = mon->osdmon()->osdmap.get_pg_pool(p->first.pool());
+	    if (pi && pi->min_size > 1) {
+	      ss << " (reducing pool " << mon->osdmon()->osdmap.get_pool_name(p->first.pool())
+		 << " min_size from " << (int)pi->min_size << " may help; search ceph.com/docs for 'incomplete')";
+	    }
+	  }
+	  detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+	}
+      }
+    }
+  }
+
+  // slow requests
+  if (g_conf->mon_osd_max_op_age > 0 &&
+      pg_map.osd_sum.op_queue_age_hist.upper_bound() > g_conf->mon_osd_max_op_age) {
+    unsigned sum = _warn_slow_request_histogram(pg_map.osd_sum.op_queue_age_hist, "", summary, detail);
+    if (sum > 0) {
+      ostringstream ss;
+      ss << sum << " requests are blocked > " << g_conf->mon_osd_max_op_age << " sec";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+
+      if (detail) {
+        unsigned num_slow_osds = 0;
+	// do per-osd warnings
+	for (ceph::unordered_map::const_iterator p = pg_map.osd_stat.begin();
+	     p != pg_map.osd_stat.end();
+	     ++p) {
+	  if (_warn_slow_request_histogram(p->second.op_queue_age_hist,
+					   string(" on osd.") + stringify(p->first),
+					   summary, detail))
+	    ++num_slow_osds;
+	}
+	ostringstream ss2;
+	ss2 << num_slow_osds << " osds have slow requests";
+	summary.push_back(make_pair(HEALTH_WARN, ss2.str()));
+	detail->push_back(make_pair(HEALTH_WARN, ss2.str()));
+      }
+    }
+  }
+
+  // recovery
+  stringstream rss;
+  pg_map.overall_recovery_summary(NULL, &rss);
+  if (!rss.str().empty()) {
+    summary.push_back(make_pair(HEALTH_WARN, "recovery " + rss.str()));
+    if (detail)
+      detail->push_back(make_pair(HEALTH_WARN, "recovery " + rss.str()));
+  }
+  
+  // full/nearfull
+  check_full_osd_health(summary, detail, pg_map.full_osds, "full", HEALTH_ERR);
+  check_full_osd_health(summary, detail, pg_map.nearfull_osds, "near full", HEALTH_WARN);
+
+  // near-target max pools
+  const map& pools = mon->osdmon()->osdmap.get_pools();
+  for (map::const_iterator p = pools.begin();
+       p != pools.end(); ++p) {
+    if ((!p->second.target_max_objects && !p->second.target_max_bytes) ||
+	!pg_map.pg_pool_sum.count(p->first))
+      continue;
+    bool nearfull = false;
+    const char *name = mon->osdmon()->osdmap.get_pool_name(p->first);
+    const pool_stat_t& st = pg_map.get_pg_pool_sum_stat(p->first);
+    uint64_t ratio = p->second.cache_target_full_ratio_micro +
+      ((1000000 - p->second.cache_target_full_ratio_micro) *
+       g_conf->mon_cache_target_full_warn_ratio);
+    if (p->second.target_max_objects && (uint64_t)st.stats.sum.num_objects >
+	p->second.target_max_objects * ratio / 1000000) {
+      nearfull = true;
+      if (detail) {
+	ostringstream ss;
+	ss << "cache pool '" << name << "' with "
+	   << si_t(st.stats.sum.num_objects)
+	   << " objects at/near target max "
+	   << si_t(p->second.target_max_objects) << " objects";
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+    if (p->second.target_max_bytes && (uint64_t)st.stats.sum.num_bytes >
+	p->second.target_max_bytes * ratio / 1000000) {
+      nearfull = true;
+      if (detail) {
+	ostringstream ss;
+	ss << "cache pool '" << mon->osdmon()->osdmap.get_pool_name(p->first)
+	   << "' with " << si_t(st.stats.sum.num_bytes)
+	   << "B at/near target max "
+	   << si_t(p->second.target_max_bytes) << "B";
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+    if (nearfull) {
+      ostringstream ss;
+      ss << "'" << name << "' at/near target max";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+  }
+
+  // scrub
+  if (pg_map.pg_sum.stats.sum.num_scrub_errors) {
+    ostringstream ss;
+    ss << pg_map.pg_sum.stats.sum.num_scrub_errors << " scrub errors";
+    summary.push_back(make_pair(HEALTH_ERR, ss.str()));
+    if (detail) {
+      detail->push_back(make_pair(HEALTH_ERR, ss.str()));
+    }
+  }
+
+  // pg skew
+  int num_in = mon->osdmon()->osdmap.get_num_in_osds();
+  if (num_in && g_conf->mon_pg_warn_min_per_osd > 0) {
+    int per = pg_map.pg_stat.size() / num_in;
+    if (per < g_conf->mon_pg_warn_min_per_osd) {
+      ostringstream ss;
+      ss << "too few pgs per osd (" << per << " < min " << g_conf->mon_pg_warn_min_per_osd << ")";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+      if (detail)
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+  }
+  if (!pg_map.pg_stat.empty()) {
+    for (ceph::unordered_map::const_iterator p = pg_map.pg_pool_sum.begin();
+	 p != pg_map.pg_pool_sum.end();
+	 ++p) {
+      const pg_pool_t *pi = mon->osdmon()->osdmap.get_pg_pool(p->first);
+      if (!pi)
+	continue;   // in case osdmap changes haven't propagated to PGMap yet
+      if (pi->get_pg_num() > pi->get_pgp_num()) {
+	ostringstream ss;
+	ss << "pool " << mon->osdmon()->osdmap.get_pool_name(p->first) << " pg_num "
+	   << pi->get_pg_num() << " > pgp_num " << pi->get_pgp_num();
+	summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+	if (detail)
+	  detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+      int average_objects_per_pg = pg_map.pg_sum.stats.sum.num_objects / pg_map.pg_stat.size();
+      if (average_objects_per_pg > 0 &&
+	  pg_map.pg_sum.stats.sum.num_objects >= g_conf->mon_pg_warn_min_objects &&
+	  p->second.stats.sum.num_objects >= g_conf->mon_pg_warn_min_pool_objects) {
+	int objects_per_pg = p->second.stats.sum.num_objects / pi->get_pg_num();
+	float ratio = (float)objects_per_pg / (float)average_objects_per_pg;
+	if (g_conf->mon_pg_warn_max_object_skew > 0 &&
+	    ratio > g_conf->mon_pg_warn_max_object_skew) {
+	  ostringstream ss;
+	  ss << "pool " << mon->osdmon()->osdmap.get_pool_name(p->first) << " has too few pgs";
+	  summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+	  if (detail) {
+	    ostringstream ss;
+	    ss << "pool " << mon->osdmon()->osdmap.get_pool_name(p->first) << " objects per pg ("
+	       << objects_per_pg << ") is more than " << ratio << " times cluster average ("
+	       << average_objects_per_pg << ")";
+	    detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+	  }
+	}
+      }
+    }
+  }
+}
+
+void PGMonitor::check_full_osd_health(list >& summary,
+				      list > *detail,
+				      const set& s, const char *desc,
+				      health_status_t sev) const
+{
+  if (!s.empty()) {
+    ostringstream ss;
+    ss << s.size() << " " << desc << " osd(s)";
+    summary.push_back(make_pair(sev, ss.str()));
+    if (detail) {
+      for (set::const_iterator p = s.begin(); p != s.end(); ++p) {
+	ostringstream ss;
+	const osd_stat_t& os = pg_map.osd_stat.find(*p)->second;
+	int ratio = (int)(((float)os.kb_used) / (float) os.kb * 100.0);
+	ss << "osd." << *p << " is " << desc << " at " << ratio << "%";
+	detail->push_back(make_pair(sev, ss.str()));
+      }
+    }
+  }
+}
+
+int PGMonitor::dump_stuck_pg_stats(stringstream &ds,
+				   Formatter *f,
+				   int threshold,
+				   vector& args) const
+{
+  PGMap::StuckPG stuck_type;
+  string type = args[0];
+
+  if (type == "inactive")
+    stuck_type = PGMap::STUCK_INACTIVE;
+  else if (type == "unclean")
+    stuck_type = PGMap::STUCK_UNCLEAN;
+  else if (type == "stale")
+    stuck_type = PGMap::STUCK_STALE;
+  else {
+    ds << "Unknown type: " << type << std::endl;
+    return 0;
+  }
+
+  utime_t now(ceph_clock_now(g_ceph_context));
+  utime_t cutoff = now - utime_t(threshold, 0);
+
+  if (!f) {
+    pg_map.dump_stuck_plain(ds, stuck_type, cutoff);
+  } else {
+    pg_map.dump_stuck(f, stuck_type, cutoff);
+    f->flush(ds);
+  }
+
+  return 0;
+}
+
+void PGMonitor::check_sub(Subscription *sub)
+{
+  if (sub->type == "osd_pg_creates") {
+    send_pg_creates(sub->session->inst.name.num(),
+		    sub->session->con.get());
+  }
+}
diff --git a/ceph/src/mon/PGMonitor.h b/ceph/src/mon/PGMonitor.h
new file mode 100644
index 00000000..f0073789
--- /dev/null
+++ b/ceph/src/mon/PGMonitor.h
@@ -0,0 +1,215 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/*
+ * Placement Group Monitor. Placement Groups are logical sets of objects
+ * that are replicated by the same set of devices.
+ */
+
+#ifndef CEPH_PGMONITOR_H
+#define CEPH_PGMONITOR_H
+
+#include 
+#include 
+using namespace std;
+
+#include "PGMap.h"
+#include "PaxosService.h"
+#include "include/types.h"
+#include "include/utime.h"
+#include "common/histogram.h"
+#include "msg/Messenger.h"
+#include "common/config.h"
+#include "mon/MonitorDBStore.h"
+
+#include "messages/MPGStats.h"
+#include "messages/MPGStatsAck.h"
+class MStatfs;
+class MMonCommand;
+class MGetPoolStats;
+
+class RatioMonitor;
+class TextTable;
+
+class PGMonitor : public PaxosService {
+public:
+  PGMap pg_map;
+
+  bool need_check_down_pgs;
+
+  epoch_t last_map_pg_create_osd_epoch;
+
+
+private:
+  PGMap::Incremental pending_inc;
+
+  const char *pgmap_meta_prefix;
+  const char *pgmap_pg_prefix;
+  const char *pgmap_osd_prefix;
+
+  void create_initial();
+  void update_from_paxos(bool *need_bootstrap);
+  void upgrade_format();
+  void on_upgrade();
+  void post_paxos_update();
+  void handle_osd_timeouts();
+  void create_pending();  // prepare a new pending
+  // propose pending update to peers
+  version_t get_trim_to();
+  void update_logger();
+
+  void encode_pending(MonitorDBStore::Transaction *t);
+  void read_pgmap_meta();
+  void read_pgmap_full();
+  void apply_pgmap_delta(bufferlist& bl);
+
+  bool preprocess_query(PaxosServiceMessage *m);  // true if processed.
+  bool prepare_update(PaxosServiceMessage *m);
+
+  bool preprocess_pg_stats(MPGStats *stats);
+  bool pg_stats_have_changed(int from, const MPGStats *stats) const;
+  bool prepare_pg_stats(MPGStats *stats);
+  void _updated_stats(MPGStats *req, MPGStatsAck *ack);
+
+  struct C_Stats : public Context {
+    PGMonitor *pgmon;
+    MPGStats *req;
+    MPGStatsAck *ack;
+    entity_inst_t who;
+    C_Stats(PGMonitor *p, MPGStats *r, MPGStatsAck *a) : pgmon(p), req(r), ack(a) {}
+    void finish(int r) {
+      if (r >= 0) {
+	pgmon->_updated_stats(req, ack);
+      } else if (r == -ECANCELED) {
+	req->put();
+	ack->put();
+      } else if (r == -EAGAIN) {
+	pgmon->dispatch(req);
+	ack->put();
+      } else {
+	assert(0 == "bad C_Stats return value");
+      }
+    }    
+  };
+
+  void handle_statfs(MStatfs *statfs);
+  bool preprocess_getpoolstats(MGetPoolStats *m);
+
+  bool preprocess_command(MMonCommand *m);
+  bool prepare_command(MMonCommand *m);
+
+  map last_sent_pg_create;  // per osd throttle
+
+  // when we last received PG stats from each osd
+  map last_osd_report;
+
+  void register_pg(pg_pool_t& pool, pg_t pgid, epoch_t epoch, bool new_pool);
+
+  /**
+   * check latest osdmap for new pgs to register
+   *
+   * @return true if we updated pending_inc (and should propose)
+   */
+  bool register_new_pgs();
+
+  void map_pg_creates();
+  void send_pg_creates();
+  void send_pg_creates(int osd, Connection *con);
+
+  /**
+   * check pgs for down primary osds
+   *
+   * clears need_check_down_pgs
+   *
+   * @return true if we updated pending_inc (and should propose)
+   */
+  bool check_down_pgs();
+
+  /**
+   * Dump stats from pgs stuck in specified states.
+   *
+   * @return 0 on success, negative error code on failure
+   */
+  int dump_stuck_pg_stats(stringstream &ds, Formatter *f,
+			  int threshold,
+			  vector& args) const;
+
+  void dump_object_stat_sum(TextTable &tbl, Formatter *f,
+                            object_stat_sum_t &sum,
+			    uint64_t avail,
+			    bool verbose);
+
+  int64_t get_rule_avail(OSDMap& osdmap, int ruleno);
+
+public:
+  PGMonitor(Monitor *mn, Paxos *p, const string& service_name)
+    : PaxosService(mn, p, service_name),
+      need_check_down_pgs(false),
+      last_map_pg_create_osd_epoch(0),
+      pgmap_meta_prefix("pgmap_meta"),
+      pgmap_pg_prefix("pgmap_pg"),
+      pgmap_osd_prefix("pgmap_osd")
+  { }
+  ~PGMonitor() { }
+
+  virtual void get_store_prefixes(set& s) {
+    s.insert(get_service_name());
+    s.insert(pgmap_meta_prefix);
+    s.insert(pgmap_pg_prefix);
+    s.insert(pgmap_osd_prefix);
+  }
+
+  virtual void on_restart();
+
+  /* Courtesy function provided by PaxosService, called when an election
+   * finishes and the cluster goes active. We use it here to make sure we
+   * haven't lost any PGs from new pools. */
+  virtual void on_active();
+
+  bool should_stash_full() {
+    return false;  // never
+  }
+  virtual void encode_full(MonitorDBStore::Transaction *t) {
+    assert(0 == "unimplemented encode_full");
+  }
+
+
+  void tick();  // check state, take actions
+
+  void check_osd_map(epoch_t epoch);
+
+  void dump_pool_stats(stringstream &ss, Formatter *f, bool verbose);
+  void dump_fs_stats(stringstream &ss, Formatter *f, bool verbose);
+
+  void dump_info(Formatter *f);
+
+  int _warn_slow_request_histogram(const pow2_hist_t& h, string suffix,
+				   list >& summary,
+				   list > *detail) const;
+
+  void get_health(list >& summary,
+		  list > *detail) const;
+  void check_full_osd_health(list >& summary,
+			     list > *detail,
+			     const set& s, const char *desc, health_status_t sev) const;
+
+  void check_sub(Subscription *sub);
+
+private:
+  // no copying allowed
+  PGMonitor(const PGMonitor &rhs);
+  PGMonitor &operator=(const PGMonitor &rhs);
+};
+
+#endif
diff --git a/ceph/src/mon/Paxos.cc b/ceph/src/mon/Paxos.cc
new file mode 100644
index 00000000..b38b1116
--- /dev/null
+++ b/ceph/src/mon/Paxos.cc
@@ -0,0 +1,1371 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include "Paxos.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+
+#include "messages/MMonPaxos.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+#include "include/stringify.h"
+#include "common/Formatter.h"
+
+#define dout_subsys ceph_subsys_paxos
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, mon->name, mon->rank, paxos_name, state, first_committed, last_committed)
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, const string& name,
+		        int rank, const string& paxos_name, int state,
+			version_t first_committed, version_t last_committed)
+{
+  return *_dout << "mon." << name << "@" << rank
+		<< "(" << mon->get_state_name() << ")"
+		<< ".paxos(" << paxos_name << " " << Paxos::get_statename(state)
+		<< " c " << first_committed << ".." << last_committed
+		<< ") ";
+}
+
+MonitorDBStore *Paxos::get_store()
+{
+  return mon->store;
+}
+
+void Paxos::read_and_prepare_transactions(MonitorDBStore::Transaction *tx, version_t first, version_t last)
+{
+  dout(10) << __func__ << " first " << first << " last " << last << dendl;
+  for (version_t v = first; v <= last; ++v) {
+    dout(30) << __func__ << " apply version " << v << dendl;
+    bufferlist bl;
+    int err = get_store()->get(get_name(), v, bl);
+    assert(err == 0);
+    assert(bl.length());
+    decode_append_transaction(*tx, bl);
+  }
+  dout(15) << __func__ << " total versions " << (last-first) << dendl;
+}
+
+void Paxos::init()
+{
+  // load paxos variables from stable storage
+  last_pn = get_store()->get(get_name(), "last_pn");
+  accepted_pn = get_store()->get(get_name(), "accepted_pn");
+  last_committed = get_store()->get(get_name(), "last_committed");
+  first_committed = get_store()->get(get_name(), "first_committed");
+
+  dout(10) << __func__ << " last_pn: " << last_pn << " accepted_pn: "
+	   << accepted_pn << " last_committed: " << last_committed
+	   << " first_committed: " << first_committed << dendl;
+
+  dout(10) << "init" << dendl;
+  assert(is_consistent());
+}
+
+void Paxos::dump_info(Formatter *f)
+{
+  f->open_object_section("paxos");
+  f->dump_unsigned("first_committed", first_committed);
+  f->dump_unsigned("last_committed", last_committed);
+  f->dump_unsigned("last_pn", last_pn);
+  f->dump_unsigned("accepted_pn", accepted_pn);
+  f->close_section();
+}
+
+// ---------------------------------
+
+// PHASE 1
+
+// leader
+void Paxos::collect(version_t oldpn)
+{
+  // we're recoverying, it seems!
+  state = STATE_RECOVERING;
+  assert(mon->is_leader());
+
+  // reset the number of lasts received
+  uncommitted_v = 0;
+  uncommitted_pn = 0;
+  uncommitted_value.clear();
+  peer_first_committed.clear();
+  peer_last_committed.clear();
+
+  // look for uncommitted value
+  if (get_store()->exists(get_name(), last_committed+1)) {
+    version_t v = get_store()->get(get_name(), "pending_v");
+    version_t pn = get_store()->get(get_name(), "pending_pn");
+    if (v && pn && v == last_committed + 1) {
+      uncommitted_pn = pn;
+    } else {
+      dout(10) << "WARNING: no pending_pn on disk, using previous accepted_pn " << accepted_pn
+	       << " and crossing our fingers" << dendl;
+      uncommitted_pn = accepted_pn;
+    }
+    uncommitted_v = last_committed+1;
+
+    get_store()->get(get_name(), last_committed+1, uncommitted_value);
+    assert(uncommitted_value.length());
+    dout(10) << "learned uncommitted " << (last_committed+1)
+	     << " pn " << uncommitted_pn
+	     << " (" << uncommitted_value.length() << " bytes) from myself" 
+	     << dendl;
+  }
+
+  // pick new pn
+  accepted_pn = get_new_proposal_number(MAX(accepted_pn, oldpn));
+  accepted_pn_from = last_committed;
+  num_last = 1;
+  dout(10) << "collect with pn " << accepted_pn << dendl;
+
+  // send collect
+  for (set::const_iterator p = mon->get_quorum().begin();
+       p != mon->get_quorum().end();
+       ++p) {
+    if (*p == mon->rank) continue;
+    
+    MMonPaxos *collect = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_COLLECT,
+				       ceph_clock_now(g_ceph_context));
+    collect->last_committed = last_committed;
+    collect->first_committed = first_committed;
+    collect->pn = accepted_pn;
+    mon->messenger->send_message(collect, mon->monmap->get_inst(*p));
+  }
+
+  // set timeout event
+  collect_timeout_event = new C_CollectTimeout(this);
+  mon->timer.add_event_after(g_conf->mon_accept_timeout, collect_timeout_event);
+}
+
+
+// peon
+void Paxos::handle_collect(MMonPaxos *collect)
+{
+  dout(10) << "handle_collect " << *collect << dendl;
+
+  assert(mon->is_peon()); // mon epoch filter should catch strays
+
+  // we're recoverying, it seems!
+  state = STATE_RECOVERING;
+
+  if (collect->first_committed > last_committed+1) {
+    dout(5) << __func__
+            << " leader's lowest version is too high for our last committed"
+            << " (theirs: " << collect->first_committed
+            << "; ours: " << last_committed << ") -- bootstrap!" << dendl;
+    collect->put();
+    mon->bootstrap();
+    return;
+  }
+
+  // reply
+  MMonPaxos *last = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_LAST,
+				  ceph_clock_now(g_ceph_context));
+  last->last_committed = last_committed;
+  last->first_committed = first_committed;
+  
+  version_t previous_pn = accepted_pn;
+
+  // can we accept this pn?
+  if (collect->pn > accepted_pn) {
+    // ok, accept it
+    accepted_pn = collect->pn;
+    accepted_pn_from = collect->pn_from;
+    dout(10) << "accepting pn " << accepted_pn << " from " 
+	     << accepted_pn_from << dendl;
+  
+    MonitorDBStore::Transaction t;
+    t.put(get_name(), "accepted_pn", accepted_pn);
+
+    dout(30) << __func__ << " transaction dump:\n";
+    JSONFormatter f(true);
+    t.dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    get_store()->apply_transaction(t);
+  } else {
+    // don't accept!
+    dout(10) << "NOT accepting pn " << collect->pn << " from " << collect->pn_from
+	     << ", we already accepted " << accepted_pn
+	     << " from " << accepted_pn_from << dendl;
+  }
+  last->pn = accepted_pn;
+  last->pn_from = accepted_pn_from;
+
+  // share whatever committed values we have
+  if (collect->last_committed < last_committed)
+    share_state(last, collect->first_committed, collect->last_committed);
+
+  // do we have an accepted but uncommitted value?
+  //  (it'll be at last_committed+1)
+  bufferlist bl;
+  if (collect->last_committed <= last_committed &&
+      get_store()->exists(get_name(), last_committed+1)) {
+    get_store()->get(get_name(), last_committed+1, bl);
+    assert(bl.length() > 0);
+    dout(10) << " sharing our accepted but uncommitted value for " 
+	     << last_committed+1 << " (" << bl.length() << " bytes)" << dendl;
+    last->values[last_committed+1] = bl;
+
+    version_t v = get_store()->get(get_name(), "pending_v");
+    version_t pn = get_store()->get(get_name(), "pending_pn");
+    if (v && pn && v == last_committed + 1) {
+      last->uncommitted_pn = pn;
+    } else {
+      // previously we didn't record which pn a value was accepted
+      // under!  use the pn value we just had...  :(
+      dout(10) << "WARNING: no pending_pn on disk, using previous accepted_pn " << previous_pn
+	       << " and crossing our fingers" << dendl;
+      last->uncommitted_pn = previous_pn;
+    }
+  }
+
+  // send reply
+  mon->messenger->send_message(last, collect->get_source_inst());
+  collect->put();
+}
+
+/**
+ * @note This is Okay. We share our versions between peer_last_committed and
+ *	 our last_committed (inclusive), and add their bufferlists to the
+ *	 message. It will be the peer's job to apply them to his store, as
+ *	 these bufferlists will contain raw transactions.
+ *	 This function is called by both the Peon and the Leader. The Peon will
+ *	 share the state with the Leader during handle_collect(), sharing any
+ *	 values the leader may be missing (i.e., the leader's last_committed is
+ *	 lower than the peon's last_committed). The Leader will share the state
+ *	 with the Peon during handle_last(), if the peon's last_committed is
+ *	 lower than the leader's last_committed.
+ */
+void Paxos::share_state(MMonPaxos *m, version_t peer_first_committed,
+			version_t peer_last_committed)
+{
+  assert(peer_last_committed < last_committed);
+
+  dout(10) << "share_state peer has fc " << peer_first_committed 
+	   << " lc " << peer_last_committed << dendl;
+  version_t v = peer_last_committed + 1;
+
+  // include incrementals
+  for ( ; v <= last_committed; v++) {
+    if (get_store()->exists(get_name(), v)) {
+      get_store()->get(get_name(), v, m->values[v]);
+      assert(m->values[v].length());
+      dout(10) << " sharing " << v << " ("
+	       << m->values[v].length() << " bytes)" << dendl;
+    }
+  }
+
+  m->last_committed = last_committed;
+}
+
+/**
+ * Store on disk a state that was shared with us
+ *
+ * Basically, we received a set of version. Or just one. It doesn't matter.
+ * What matters is that we have to stash it in the store. So, we will simply
+ * write every single bufferlist into their own versions on our side (i.e.,
+ * onto paxos-related keys), and then we will decode those same bufferlists
+ * we just wrote and apply the transactions they hold. We will also update
+ * our first and last committed values to point to the new values, if need
+ * be. All all this is done tightly wrapped in a transaction to ensure we
+ * enjoy the atomicity guarantees given by our awesome k/v store.
+ */
+bool Paxos::store_state(MMonPaxos *m)
+{
+  MonitorDBStore::Transaction t;
+  map::iterator start = m->values.begin();
+  bool changed = false;
+
+  // build map of values to store
+  // we want to write the range [last_committed, m->last_committed] only.
+  if (start != m->values.end() &&
+      start->first > last_committed + 1) {
+    // ignore everything if values start in the future.
+    dout(10) << "store_state ignoring all values, they start at " << start->first
+	     << " > last_committed+1" << dendl;
+    start = m->values.end();
+  }
+
+  // push forward the start position on the message's values iterator, up until
+  // we run out of positions or we find a position matching 'last_committed'.
+  while (start != m->values.end() && start->first <= last_committed) {
+    ++start;
+  }
+
+  // make sure we get the right interval of values to apply by pushing forward
+  // the 'end' iterator until it matches the message's 'last_committed'.
+  map::iterator end = start;
+  while (end != m->values.end() && end->first <= m->last_committed) {
+    last_committed = end->first;
+    ++end;
+  }
+
+  if (start == end) {
+    dout(10) << "store_state nothing to commit" << dendl;
+  } else {
+    dout(10) << "store_state [" << start->first << ".." 
+	     << last_committed << "]" << dendl;
+    t.put(get_name(), "last_committed", last_committed);
+    // we should apply the state here -- decode every single bufferlist in the
+    // map and append the transactions to 't'.
+    map::iterator it;
+    for (it = start; it != end; ++it) {
+      // write the bufferlist as the version's value
+      t.put(get_name(), it->first, it->second);
+      // decode the bufferlist and append it to the transaction we will shortly
+      // apply.
+      decode_append_transaction(t, it->second);
+    }
+
+    // discard obsolete uncommitted value?
+    if (uncommitted_v && uncommitted_v <= last_committed) {
+      dout(10) << " forgetting obsolete uncommitted value " << uncommitted_v
+	       << " pn " << uncommitted_pn << dendl;
+      uncommitted_v = 0;
+      uncommitted_pn = 0;
+      uncommitted_value.clear();
+    }
+  }
+  if (!t.empty()) {
+    dout(30) << __func__ << " transaction dump:\n";
+    JSONFormatter f(true);
+    t.dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    get_store()->apply_transaction(t);
+
+    // refresh first_committed; this txn may have trimmed.
+    first_committed = get_store()->get(get_name(), "first_committed");
+
+    _sanity_check_store();
+    changed = true;
+  }
+
+  remove_legacy_versions();
+
+  return changed;
+}
+
+void Paxos::remove_legacy_versions()
+{
+  if (get_store()->exists(get_name(), "conversion_first")) {
+    MonitorDBStore::Transaction t;
+    version_t v = get_store()->get(get_name(), "conversion_first");
+    dout(10) << __func__ << " removing pre-conversion paxos states from " << v
+	     << " until " << first_committed << dendl;
+    for (; v < first_committed; ++v) {
+      t.erase(get_name(), v);
+    }
+    t.erase(get_name(), "conversion_first");
+    get_store()->apply_transaction(t);
+  }
+}
+
+void Paxos::_sanity_check_store()
+{
+  version_t lc = get_store()->get(get_name(), "last_committed");
+  assert(lc == last_committed);
+}
+
+
+// leader
+void Paxos::handle_last(MMonPaxos *last)
+{
+  bool need_refresh = false;
+
+  dout(10) << "handle_last " << *last << dendl;
+
+  if (!mon->is_leader()) {
+    dout(10) << "not leader, dropping" << dendl;
+    last->put();
+    return;
+  }
+
+  // note peer's first_ and last_committed, in case we learn a new
+  // commit and need to push it to them.
+  peer_first_committed[last->get_source().num()] = last->first_committed;
+  peer_last_committed[last->get_source().num()] = last->last_committed;
+
+  if (last->first_committed > last_committed+1) {
+    dout(5) << __func__
+            << " peon's lowest version is too high for our last committed"
+            << " (theirs: " << last->first_committed
+            << "; ours: " << last_committed << ") -- bootstrap!" << dendl;
+    last->put();
+    mon->bootstrap();
+    return;
+  }
+
+  assert(g_conf->paxos_kill_at != 1);
+
+  // store any committed values if any are specified in the message
+  need_refresh = store_state(last);
+
+  assert(g_conf->paxos_kill_at != 2);
+
+  // do they accept your pn?
+  if (last->pn > accepted_pn) {
+    // no, try again.
+    dout(10) << " they had a higher pn than us, picking a new one." << dendl;
+
+    // cancel timeout event
+    mon->timer.cancel_event(collect_timeout_event);
+    collect_timeout_event = 0;
+
+    collect(last->pn);
+  } else if (last->pn == accepted_pn) {
+    // yes, they accepted our pn.  great.
+    num_last++;
+    dout(10) << " they accepted our pn, we now have " 
+	     << num_last << " peons" << dendl;
+
+    // did this person send back an accepted but uncommitted value?
+    if (last->uncommitted_pn) {
+      if (last->uncommitted_pn >= uncommitted_pn &&
+	  last->last_committed >= last_committed &&
+	  last->last_committed + 1 >= uncommitted_v) {
+	uncommitted_v = last->last_committed+1;
+	uncommitted_pn = last->uncommitted_pn;
+	uncommitted_value = last->values[uncommitted_v];
+	dout(10) << "we learned an uncommitted value for " << uncommitted_v
+		 << " pn " << uncommitted_pn
+		 << " " << uncommitted_value.length() << " bytes"
+		 << dendl;
+      } else {
+	dout(10) << "ignoring uncommitted value for " << (last->last_committed+1)
+		 << " pn " << last->uncommitted_pn
+		 << " " << last->values[last->last_committed+1].length() << " bytes"
+		 << dendl;
+      }
+    }
+    
+    // is that everyone?
+    if (num_last == mon->get_quorum().size()) {
+      // cancel timeout event
+      mon->timer.cancel_event(collect_timeout_event);
+      collect_timeout_event = 0;
+
+      // share committed values?
+      for (map::iterator p = peer_last_committed.begin();
+	   p != peer_last_committed.end();
+	   ++p) {
+	if (p->second < last_committed) {
+	  // share committed values
+	  dout(10) << " sending commit to mon." << p->first << dendl;
+	  MMonPaxos *commit = new MMonPaxos(mon->get_epoch(),
+					    MMonPaxos::OP_COMMIT,
+					    ceph_clock_now(g_ceph_context));
+	  share_state(commit, peer_first_committed[p->first], p->second);
+	  mon->messenger->send_message(commit, mon->monmap->get_inst(p->first));
+	}
+      }
+      peer_first_committed.clear();
+      peer_last_committed.clear();
+
+      // almost...
+
+      // did we learn an old value?
+      if (uncommitted_v == last_committed+1 &&
+	  uncommitted_value.length()) {
+	dout(10) << "that's everyone.  begin on old learned value" << dendl;
+	state = STATE_UPDATING_PREVIOUS;
+	begin(uncommitted_value);
+      } else {
+	// active!
+	dout(10) << "that's everyone.  active!" << dendl;
+	extend_lease();
+
+	need_refresh = false;
+	if (do_refresh()) {
+	  finish_round();
+
+	  finish_contexts(g_ceph_context, waiting_for_active);
+	  finish_contexts(g_ceph_context, waiting_for_readable);
+	  finish_contexts(g_ceph_context, waiting_for_writeable);
+	}
+      }
+    }
+  } else {
+    // no, this is an old message, discard
+    dout(10) << "old pn, ignoring" << dendl;
+  }
+
+  if (need_refresh)
+    (void)do_refresh();
+
+  last->put();
+}
+
+void Paxos::collect_timeout()
+{
+  dout(1) << "collect timeout, calling fresh election" << dendl;
+  collect_timeout_event = 0;
+  assert(mon->is_leader());
+  mon->bootstrap();
+}
+
+
+// leader
+void Paxos::begin(bufferlist& v)
+{
+  dout(10) << "begin for " << last_committed+1 << " " 
+	   << v.length() << " bytes"
+	   << dendl;
+
+  assert(mon->is_leader());
+  assert(is_updating() || is_updating_previous());
+
+  // we must already have a majority for this to work.
+  assert(mon->get_quorum().size() == 1 ||
+	 num_last > (unsigned)mon->monmap->size()/2);
+  
+  // and no value, yet.
+  assert(new_value.length() == 0);
+  
+  // accept it ourselves
+  accepted.clear();
+  accepted.insert(mon->rank);
+  new_value = v;
+
+  if (last_committed == 0) {
+    MonitorDBStore::Transaction t;
+    // initial base case; set first_committed too
+    t.put(get_name(), "first_committed", 1);
+    decode_append_transaction(t, new_value);
+
+    bufferlist tx_bl;
+    t.encode(tx_bl);
+
+    new_value = tx_bl;
+  }
+
+  // store the proposed value in the store. IF it is accepted, we will then
+  // have to decode it into a transaction and apply it.
+  MonitorDBStore::Transaction t;
+  t.put(get_name(), last_committed+1, new_value);
+
+  // note which pn this pending value is for.
+  t.put(get_name(), "pending_v", last_committed + 1);
+  t.put(get_name(), "pending_pn", accepted_pn);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t.dump(&f);
+  f.flush(*_dout);
+  MonitorDBStore::Transaction debug_tx;
+  bufferlist::iterator new_value_it = new_value.begin();
+  debug_tx.decode(new_value_it);
+  debug_tx.dump(&f);
+  *_dout << "\nbl dump:\n";
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  get_store()->apply_transaction(t);
+
+  assert(g_conf->paxos_kill_at != 3);
+
+  if (mon->get_quorum().size() == 1) {
+    // we're alone, take it easy
+    commit();
+    if (do_refresh()) {
+      assert(is_updating());  // we can't be updating-previous with quorum of 1
+      commit_proposal();
+      finish_round();
+      finish_contexts(g_ceph_context, waiting_for_active);
+      finish_contexts(g_ceph_context, waiting_for_commit);
+      finish_contexts(g_ceph_context, waiting_for_readable);
+      finish_contexts(g_ceph_context, waiting_for_writeable);
+    }
+    return;
+  }
+
+  // ask others to accept it too!
+  for (set::const_iterator p = mon->get_quorum().begin();
+       p != mon->get_quorum().end();
+       ++p) {
+    if (*p == mon->rank) continue;
+    
+    dout(10) << " sending begin to mon." << *p << dendl;
+    MMonPaxos *begin = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_BEGIN,
+				     ceph_clock_now(g_ceph_context));
+    begin->values[last_committed+1] = new_value;
+    begin->last_committed = last_committed;
+    begin->pn = accepted_pn;
+    
+    mon->messenger->send_message(begin, mon->monmap->get_inst(*p));
+  }
+
+  // set timeout event
+  accept_timeout_event = new C_AcceptTimeout(this);
+  mon->timer.add_event_after(g_conf->mon_accept_timeout, accept_timeout_event);
+}
+
+// peon
+void Paxos::handle_begin(MMonPaxos *begin)
+{
+  dout(10) << "handle_begin " << *begin << dendl;
+
+  // can we accept this?
+  if (begin->pn < accepted_pn) {
+    dout(10) << " we accepted a higher pn " << accepted_pn << ", ignoring" << dendl;
+    begin->put();
+    return;
+  }
+  assert(begin->pn == accepted_pn);
+  assert(begin->last_committed == last_committed);
+  
+  assert(g_conf->paxos_kill_at != 4);
+
+  // set state.
+  state = STATE_UPDATING;
+  lease_expire = utime_t();  // cancel lease
+
+  // yes.
+  version_t v = last_committed+1;
+  dout(10) << "accepting value for " << v << " pn " << accepted_pn << dendl;
+  // store the accepted value onto our store. We will have to decode it and
+  // apply its transaction once we receive permission to commit.
+  MonitorDBStore::Transaction t;
+  t.put(get_name(), v, begin->values[v]);
+
+  // note which pn this pending value is for.
+  t.put(get_name(), "pending_v", v);
+  t.put(get_name(), "pending_pn", accepted_pn);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t.dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  get_store()->apply_transaction(t);
+
+  assert(g_conf->paxos_kill_at != 5);
+
+  // reply
+  MMonPaxos *accept = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_ACCEPT,
+				    ceph_clock_now(g_ceph_context));
+  accept->pn = accepted_pn;
+  accept->last_committed = last_committed;
+  mon->messenger->send_message(accept, begin->get_source_inst());
+  
+  begin->put();
+}
+
+// leader
+void Paxos::handle_accept(MMonPaxos *accept)
+{
+  dout(10) << "handle_accept " << *accept << dendl;
+  int from = accept->get_source().num();
+
+  if (accept->pn != accepted_pn) {
+    // we accepted a higher pn, from some other leader
+    dout(10) << " we accepted a higher pn " << accepted_pn << ", ignoring" << dendl;
+    goto out;
+  }
+  if (last_committed > 0 &&
+      accept->last_committed < last_committed-1) {
+    dout(10) << " this is from an old round, ignoring" << dendl;
+    goto out;
+  }
+  assert(accept->last_committed == last_committed ||   // not committed
+	 accept->last_committed == last_committed-1);  // committed
+
+  assert(is_updating() || is_updating_previous());
+  assert(accepted.count(from) == 0);
+  accepted.insert(from);
+  dout(10) << " now " << accepted << " have accepted" << dendl;
+
+  assert(g_conf->paxos_kill_at != 6);
+
+  // only commit (and expose committed state) when we get *all* quorum
+  // members to accept.  otherwise, they may still be sharing the now
+  // stale state.
+  // FIXME: we can improve this with an additional lease revocation message
+  // that doesn't block for the persist.
+  if (accepted == mon->get_quorum()) {
+    // yay, commit!
+    dout(10) << " got majority, committing, done with update" << dendl;
+    commit();
+    if (!do_refresh())
+      goto out;
+    if (is_updating())
+      commit_proposal();
+    finish_contexts(g_ceph_context, waiting_for_commit);
+
+    // cancel timeout event
+    mon->timer.cancel_event(accept_timeout_event);
+    accept_timeout_event = 0;
+
+    // yay!
+    extend_lease();
+
+    assert(g_conf->paxos_kill_at != 10);
+
+    finish_round();
+
+    // wake people up
+    finish_contexts(g_ceph_context, waiting_for_active);
+    finish_contexts(g_ceph_context, waiting_for_readable);
+    finish_contexts(g_ceph_context, waiting_for_writeable);
+  }
+
+ out:
+  accept->put();
+}
+
+void Paxos::accept_timeout()
+{
+  dout(1) << "accept timeout, calling fresh election" << dendl;
+  accept_timeout_event = 0;
+  assert(mon->is_leader());
+  assert(is_updating() || is_updating_previous());
+  mon->bootstrap();
+}
+
+void Paxos::commit()
+{
+  dout(10) << "commit " << last_committed+1 << dendl;
+
+  // cancel lease - it was for the old value.
+  //  (this would only happen if message layer lost the 'begin', but
+  //   leader still got a majority and committed with out us.)
+  lease_expire = utime_t();  // cancel lease
+
+  assert(g_conf->paxos_kill_at != 7);
+
+  MonitorDBStore::Transaction t;
+
+  // commit locally
+  last_committed++;
+  last_commit_time = ceph_clock_now(g_ceph_context);
+  t.put(get_name(), "last_committed", last_committed);
+
+  // decode the value and apply its transaction to the store.
+  // this value can now be read from last_committed.
+  decode_append_transaction(t, new_value);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t.dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  get_store()->apply_transaction(t);
+
+  assert(g_conf->paxos_kill_at != 8);
+
+  // refresh first_committed; this txn may have trimmed.
+  first_committed = get_store()->get(get_name(), "first_committed");
+
+  _sanity_check_store();
+
+  // tell everyone
+  for (set::const_iterator p = mon->get_quorum().begin();
+       p != mon->get_quorum().end();
+       ++p) {
+    if (*p == mon->rank) continue;
+
+    dout(10) << " sending commit to mon." << *p << dendl;
+    MMonPaxos *commit = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_COMMIT,
+				      ceph_clock_now(g_ceph_context));
+    commit->values[last_committed] = new_value;
+    commit->pn = accepted_pn;
+    commit->last_committed = last_committed;
+
+    mon->messenger->send_message(commit, mon->monmap->get_inst(*p));
+  }
+
+  assert(g_conf->paxos_kill_at != 9);
+
+  // get ready for a new round.
+  new_value.clear();
+
+  remove_legacy_versions();
+}
+
+
+void Paxos::handle_commit(MMonPaxos *commit)
+{
+  dout(10) << "handle_commit on " << commit->last_committed << dendl;
+
+  if (!mon->is_peon()) {
+    dout(10) << "not a peon, dropping" << dendl;
+    assert(0);
+    commit->put();
+    return;
+  }
+
+  store_state(commit);
+
+  if (do_refresh()) {
+    finish_contexts(g_ceph_context, waiting_for_commit);
+  }
+
+  commit->put();
+}
+
+void Paxos::extend_lease()
+{
+  assert(mon->is_leader());
+  //assert(is_active());
+
+  lease_expire = ceph_clock_now(g_ceph_context);
+  lease_expire += g_conf->mon_lease;
+  acked_lease.clear();
+  acked_lease.insert(mon->rank);
+
+  dout(7) << "extend_lease now+" << g_conf->mon_lease 
+	  << " (" << lease_expire << ")" << dendl;
+
+  // bcast
+  for (set::const_iterator p = mon->get_quorum().begin();
+      p != mon->get_quorum().end(); ++p) {
+
+    if (*p == mon->rank) continue;
+    MMonPaxos *lease = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_LEASE,
+				     ceph_clock_now(g_ceph_context));
+    lease->last_committed = last_committed;
+    lease->lease_timestamp = lease_expire;
+    lease->first_committed = first_committed;
+    mon->messenger->send_message(lease, mon->monmap->get_inst(*p));
+  }
+
+  // set timeout event.
+  //  if old timeout is still in place, leave it.
+  if (!lease_ack_timeout_event) {
+    lease_ack_timeout_event = new C_LeaseAckTimeout(this);
+    mon->timer.add_event_after(g_conf->mon_lease_ack_timeout, 
+			       lease_ack_timeout_event);
+  }
+
+  // set renew event
+  lease_renew_event = new C_LeaseRenew(this);
+  utime_t at = lease_expire;
+  at -= g_conf->mon_lease;
+  at += g_conf->mon_lease_renew_interval;
+  mon->timer.add_event_at(at, lease_renew_event);
+}
+
+void Paxos::warn_on_future_time(utime_t t, entity_name_t from)
+{
+  utime_t now = ceph_clock_now(g_ceph_context);
+  if (t > now) {
+    utime_t diff = t - now;
+    if (diff > g_conf->mon_clock_drift_allowed) {
+      utime_t warn_diff = now - last_clock_drift_warn;
+      if (warn_diff >
+	  pow(g_conf->mon_clock_drift_warn_backoff, clock_drift_warned)) {
+	mon->clog.warn() << "message from " << from << " was stamped " << diff
+			 << "s in the future, clocks not synchronized";
+	last_clock_drift_warn = ceph_clock_now(g_ceph_context);
+	++clock_drift_warned;
+      }
+    }
+  }
+
+}
+
+bool Paxos::do_refresh()
+{
+  bool need_bootstrap = false;
+
+  // make sure we have the latest state loaded up
+  mon->refresh_from_paxos(&need_bootstrap);
+
+  if (need_bootstrap) {
+    dout(10) << " doing requested bootstrap" << dendl;
+    mon->bootstrap();
+    return false;
+  }
+
+  return true;
+}
+
+void Paxos::commit_proposal()
+{
+  dout(10) << __func__ << dendl;
+  assert(mon->is_leader());
+  assert(!proposals.empty());
+  assert(is_updating());
+
+  C_Proposal *proposal = static_cast(proposals.front());
+  assert(proposal->proposed);
+  dout(10) << __func__ << " proposal " << proposal << " took "
+	   << (ceph_clock_now(NULL) - proposal->proposal_time)
+	   << " to finish" << dendl;
+  proposals.pop_front();
+  proposal->complete(0);
+}
+
+void Paxos::finish_round()
+{
+  assert(mon->is_leader());
+
+  // ok, now go active!
+  state = STATE_ACTIVE;
+
+  dout(10) << __func__ << " state " << state
+	   << " proposals left " << proposals.size() << dendl;
+
+  if (should_trim()) {
+    trim();
+  }
+
+  if (is_active() && !proposals.empty()) {
+    propose_queued();
+  }
+}
+
+// peon
+void Paxos::handle_lease(MMonPaxos *lease)
+{
+  // sanity
+  if (!mon->is_peon() ||
+      last_committed != lease->last_committed) {
+    dout(10) << "handle_lease i'm not a peon, or they're not the leader,"
+	     << " or the last_committed doesn't match, dropping" << dendl;
+    lease->put();
+    return;
+  }
+
+  warn_on_future_time(lease->sent_timestamp, lease->get_source());
+
+  // extend lease
+  if (lease_expire < lease->lease_timestamp) {
+    lease_expire = lease->lease_timestamp;
+
+    utime_t now = ceph_clock_now(g_ceph_context);
+    if (lease_expire < now) {
+      utime_t diff = now - lease_expire;
+      derr << "lease_expire from " << lease->get_source_inst() << " is " << diff << " seconds in the past; mons are probably laggy (or possibly clocks are too skewed)" << dendl;
+    }
+  }
+
+  state = STATE_ACTIVE;
+
+  dout(10) << "handle_lease on " << lease->last_committed
+	   << " now " << lease_expire << dendl;
+
+  // ack
+  MMonPaxos *ack = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_LEASE_ACK,
+				 ceph_clock_now(g_ceph_context));
+  ack->last_committed = last_committed;
+  ack->first_committed = first_committed;
+  ack->lease_timestamp = ceph_clock_now(g_ceph_context);
+  mon->messenger->send_message(ack, lease->get_source_inst());
+
+  // (re)set timeout event.
+  reset_lease_timeout();
+
+  // kick waiters
+  finish_contexts(g_ceph_context, waiting_for_active);
+  if (is_readable())
+    finish_contexts(g_ceph_context, waiting_for_readable);
+
+  lease->put();
+}
+
+void Paxos::handle_lease_ack(MMonPaxos *ack)
+{
+  int from = ack->get_source().num();
+
+  if (!lease_ack_timeout_event) {
+    dout(10) << "handle_lease_ack from " << ack->get_source() 
+	     << " -- stray (probably since revoked)" << dendl;
+  }
+  else if (acked_lease.count(from) == 0) {
+    acked_lease.insert(from);
+    
+    if (acked_lease == mon->get_quorum()) {
+      // yay!
+      dout(10) << "handle_lease_ack from " << ack->get_source() 
+	       << " -- got everyone" << dendl;
+      mon->timer.cancel_event(lease_ack_timeout_event);
+      lease_ack_timeout_event = 0;
+
+
+    } else {
+      dout(10) << "handle_lease_ack from " << ack->get_source() 
+	       << " -- still need "
+	       << mon->get_quorum().size() - acked_lease.size()
+	       << " more" << dendl;
+    }
+  } else {
+    dout(10) << "handle_lease_ack from " << ack->get_source() 
+	     << " dup (lagging!), ignoring" << dendl;
+  }
+
+  warn_on_future_time(ack->sent_timestamp, ack->get_source());
+
+  ack->put();
+}
+
+void Paxos::lease_ack_timeout()
+{
+  dout(1) << "lease_ack_timeout -- calling new election" << dendl;
+  assert(mon->is_leader());
+  assert(is_active());
+
+  lease_ack_timeout_event = 0;
+  mon->bootstrap();
+}
+
+void Paxos::reset_lease_timeout()
+{
+  dout(20) << "reset_lease_timeout - setting timeout event" << dendl;
+  if (lease_timeout_event)
+    mon->timer.cancel_event(lease_timeout_event);
+  lease_timeout_event = new C_LeaseTimeout(this);
+  mon->timer.add_event_after(g_conf->mon_lease_ack_timeout, lease_timeout_event);
+}
+
+void Paxos::lease_timeout()
+{
+  dout(1) << "lease_timeout -- calling new election" << dendl;
+  assert(mon->is_peon());
+
+  lease_timeout_event = 0;
+  mon->bootstrap();
+}
+
+void Paxos::lease_renew_timeout()
+{
+  lease_renew_event = 0;
+  extend_lease();
+}
+
+
+/*
+ * trim old states
+ */
+void Paxos::trim()
+{
+  assert(should_trim());
+  version_t end = MIN(get_version() - g_conf->paxos_min,
+		      get_first_committed() + g_conf->paxos_trim_max);
+
+  if (first_committed >= end)
+    return;
+
+  dout(10) << "trim to " << end << " (was " << first_committed << ")" << dendl;
+
+  MonitorDBStore::Transaction t;
+
+  for (version_t v = first_committed; v < end; ++v) {
+    dout(10) << "trim " << v << dendl;
+    t.erase(get_name(), v);
+  }
+  t.put(get_name(), "first_committed", end);
+  if (g_conf->mon_compact_on_trim) {
+    dout(10) << " compacting trimmed range" << dendl;
+    t.compact_range(get_name(), stringify(first_committed - 1), stringify(end));
+  }
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t.dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  bufferlist bl;
+  t.encode(bl);
+
+  trimming = true;
+  queue_proposal(bl, new C_Trimmed(this));
+}
+
+/*
+ * return a globally unique, monotonically increasing proposal number
+ */
+version_t Paxos::get_new_proposal_number(version_t gt)
+{
+  if (last_pn < gt) 
+    last_pn = gt;
+  
+  // update. make it unique among all monitors.
+  last_pn /= 100;
+  last_pn++;
+  last_pn *= 100;
+  last_pn += (version_t)mon->rank;
+
+  // write
+  MonitorDBStore::Transaction t;
+  t.put(get_name(), "last_pn", last_pn);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t.dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  get_store()->apply_transaction(t);
+
+  dout(10) << "get_new_proposal_number = " << last_pn << dendl;
+  return last_pn;
+}
+
+
+void Paxos::cancel_events()
+{
+  if (collect_timeout_event) {
+    mon->timer.cancel_event(collect_timeout_event);
+    collect_timeout_event = 0;
+  }
+  if (accept_timeout_event) {
+    mon->timer.cancel_event(accept_timeout_event);
+    accept_timeout_event = 0;
+  }
+  if (lease_renew_event) {
+    mon->timer.cancel_event(lease_renew_event);
+    lease_renew_event = 0;
+  }
+  if (lease_ack_timeout_event) {
+    mon->timer.cancel_event(lease_ack_timeout_event);
+    lease_ack_timeout_event = 0;
+  }  
+  if (lease_timeout_event) {
+    mon->timer.cancel_event(lease_timeout_event);
+    lease_timeout_event = 0;
+  }
+}
+
+void Paxos::shutdown() {
+  dout(10) << __func__ << " cancel all contexts" << dendl;
+  finish_contexts(g_ceph_context, waiting_for_writeable, -ECANCELED);
+  finish_contexts(g_ceph_context, waiting_for_commit, -ECANCELED);
+  finish_contexts(g_ceph_context, waiting_for_readable, -ECANCELED);
+  finish_contexts(g_ceph_context, waiting_for_active, -ECANCELED);
+  finish_contexts(g_ceph_context, proposals, -ECANCELED);
+}
+
+void Paxos::leader_init()
+{
+  cancel_events();
+  new_value.clear();
+
+  finish_contexts(g_ceph_context, proposals, -EAGAIN);
+
+  if (mon->get_quorum().size() == 1) {
+    state = STATE_ACTIVE;
+    return;
+  }
+
+  state = STATE_RECOVERING;
+  lease_expire = utime_t();
+  dout(10) << "leader_init -- starting paxos recovery" << dendl;
+  collect(0);
+}
+
+void Paxos::peon_init()
+{
+  cancel_events();
+  new_value.clear();
+
+  state = STATE_RECOVERING;
+  lease_expire = utime_t();
+  dout(10) << "peon_init -- i am a peon" << dendl;
+
+  // start a timer, in case the leader never manages to issue a lease
+  reset_lease_timeout();
+
+  // no chance to write now!
+  finish_contexts(g_ceph_context, waiting_for_writeable, -EAGAIN);
+  finish_contexts(g_ceph_context, waiting_for_commit, -EAGAIN);
+  finish_contexts(g_ceph_context, proposals, -EAGAIN);
+}
+
+void Paxos::restart()
+{
+  dout(10) << "restart -- canceling timeouts" << dendl;
+  cancel_events();
+  new_value.clear();
+
+  state = STATE_RECOVERING;
+
+  finish_contexts(g_ceph_context, proposals, -EAGAIN);
+  finish_contexts(g_ceph_context, waiting_for_commit, -EAGAIN);
+  finish_contexts(g_ceph_context, waiting_for_active, -EAGAIN);
+}
+
+
+void Paxos::dispatch(PaxosServiceMessage *m)
+{
+  // election in progress?
+  if (!mon->is_leader() && !mon->is_peon()) {
+    dout(5) << "election in progress, dropping " << *m << dendl;
+    m->put();
+    return;    
+  }
+
+  // check sanity
+  assert(mon->is_leader() || 
+	 (mon->is_peon() && m->get_source().num() == mon->get_leader()));
+  
+  switch (m->get_type()) {
+
+  case MSG_MON_PAXOS:
+    {
+      MMonPaxos *pm = (MMonPaxos*)m;
+
+      // NOTE: these ops are defined in messages/MMonPaxos.h
+      switch (pm->op) {
+	// learner
+      case MMonPaxos::OP_COLLECT:
+	handle_collect(pm);
+	break;
+      case MMonPaxos::OP_LAST:
+	handle_last(pm);
+	break;
+      case MMonPaxos::OP_BEGIN:
+	handle_begin(pm);
+	break;
+      case MMonPaxos::OP_ACCEPT:
+	handle_accept(pm);
+	break;		
+      case MMonPaxos::OP_COMMIT:
+	handle_commit(pm);
+	break;
+      case MMonPaxos::OP_LEASE:
+	handle_lease(pm);
+	break;
+      case MMonPaxos::OP_LEASE_ACK:
+	handle_lease_ack(pm);
+	break;
+      default:
+	assert(0);
+      }
+    }
+    break;
+    
+  default:
+    assert(0);
+  }
+}
+
+
+// -----------------
+// service interface
+
+// -- READ --
+
+bool Paxos::is_readable(version_t v)
+{
+  dout(5) << "is_readable now=" << ceph_clock_now(g_ceph_context)
+	  << " lease_expire=" << lease_expire
+	  << " has v" << v << " lc " << last_committed << dendl;
+  if (v > last_committed)
+    return false;
+  return 
+    (mon->is_peon() || mon->is_leader()) &&
+    (is_active() || is_updating()) &&
+    last_committed > 0 &&           // must have a value
+    (mon->get_quorum().size() == 1 ||  // alone, or
+     is_lease_valid()); // have lease
+}
+
+bool Paxos::read(version_t v, bufferlist &bl)
+{
+  if (!get_store()->get(get_name(), v, bl))
+    return false;
+  return true;
+}
+
+version_t Paxos::read_current(bufferlist &bl)
+{
+  if (read(last_committed, bl))
+    return last_committed;
+  return 0;
+}
+
+
+bool Paxos::is_lease_valid()
+{
+  return ((mon->get_quorum().size() == 1)
+      || (ceph_clock_now(g_ceph_context) < lease_expire));
+}
+
+// -- WRITE --
+
+bool Paxos::is_writeable()
+{
+  return
+    mon->is_leader() &&
+    is_active() &&
+    is_lease_valid();
+}
+
+void Paxos::list_proposals(ostream& out)
+{
+  out << __func__ << " " << proposals.size() << " in queue:\n";
+  list::iterator p_it = proposals.begin();
+  for (int i = 0; p_it != proposals.end(); ++p_it, ++i) {
+    C_Proposal *p = (C_Proposal*) *p_it;
+    out << "-- entry #" << i << "\n";
+    out << *p << "\n";
+  }
+}
+
+void Paxos::propose_queued()
+{
+  assert(is_active());
+  assert(!proposals.empty());
+
+  C_Proposal *proposal = static_cast(proposals.front());
+  assert(!proposal->proposed);
+
+  cancel_events();
+  dout(10) << __func__ << " " << (last_committed + 1)
+	  << " " << proposal->bl.length() << " bytes" << dendl;
+  proposal->proposed = true;
+
+  dout(30) << __func__ << " ";
+  list_proposals(*_dout);
+  *_dout << dendl;
+
+  state = STATE_UPDATING;
+  begin(proposal->bl);
+}
+
+void Paxos::queue_proposal(bufferlist& bl, Context *onfinished)
+{
+  dout(5) << __func__ << " bl " << bl.length() << " bytes;"
+	  << " ctx = " << onfinished << dendl;
+
+  proposals.push_back(new C_Proposal(onfinished, bl));
+}
+
+bool Paxos::propose_new_value(bufferlist& bl, Context *onfinished)
+{
+  assert(mon->is_leader());
+
+  queue_proposal(bl, onfinished);
+
+  if (!is_active()) {
+    dout(5) << __func__ << " not active; proposal queued" << dendl; 
+    return true;
+  }
+
+  propose_queued();
+  
+  return true;
+}
+
+bool Paxos::is_consistent()
+{
+  return (first_committed <= last_committed);
+}
+
diff --git a/ceph/src/mon/Paxos.h b/ceph/src/mon/Paxos.h
new file mode 100644
index 00000000..b9e43a14
--- /dev/null
+++ b/ceph/src/mon/Paxos.h
@@ -0,0 +1,1322 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/*
+time---->
+
+cccccccccccccccccca????????????????????????????????????????
+cccccccccccccccccca????????????????????????????????????????
+cccccccccccccccccca???????????????????????????????????????? leader
+cccccccccccccccccc????????????????????????????????????????? 
+ccccc?????????????????????????????????????????????????????? 
+
+last_committed
+
+pn_from
+pn
+
+a 12v 
+b 12v
+c 14v
+d
+e 12v
+*/
+
+/**
+ * Paxos storage layout and behavior
+ *
+ * Currently, we use a key/value store to hold all the Paxos-related data, but
+ * it can logically be depicted as this:
+ *
+ *  paxos:
+ *    first_committed -> 1
+ *     last_committed -> 4
+ *		    1 -> value_1
+ *		    2 -> value_2
+ *		    3 -> value_3
+ *		    4 -> value_4
+ *
+ * Since we are relying on a k/v store supporting atomic transactions, we can
+ * guarantee that if 'last_committed' has a value of '4', then we have up to
+ * version 4 on the store, and no more than that; the same applies to
+ * 'first_committed', which holding '1' will strictly meaning that our lowest
+ * version is 1.
+ *
+ * Each version's value (value_1, value_2, ..., value_n) is a blob of data,
+ * incomprehensible to the Paxos. These values are proposed to the Paxos on
+ * propose_new_value() and each one is a transaction encoded in a bufferlist.
+ *
+ * The Paxos will write the value to disk, associating it with its version,
+ * but will take a step further: the value shall be decoded, and the operations
+ * on that transaction shall be applied during the same transaction that will
+ * write the value's encoded bufferlist to disk. This behavior ensures that
+ * whatever is being proposed will only be available on the store when it is
+ * applied by Paxos, which will then be aware of such new values, guaranteeing
+ * the store state is always consistent without requiring shady workarounds.
+ *
+ * So, let's say that FooMonitor proposes the following transaction, neatly
+ * encoded on a bufferlist of course:
+ *
+ *  Tx_Foo
+ *    put(foo, last_committed, 3)
+ *    put(foo, 3, foo_value_3)
+ *    erase(foo, 2)
+ *    erase(foo, 1)
+ *    put(foo, first_committed, 3)
+ *
+ * And knowing that the Paxos is proposed Tx_Foo as a bufferlist, once it is
+ * ready to commit, and assuming we are now committing version 5 of the Paxos,
+ * we will do something along the lines of:
+ *
+ *  Tx proposed_tx;
+ *  proposed_tx.decode(Tx_foo_bufferlist);
+ *
+ *  Tx our_tx;
+ *  our_tx.put(paxos, last_committed, 5);
+ *  our_tx.put(paxos, 5, Tx_foo_bufferlist);
+ *  our_tx.append(proposed_tx);
+ *
+ *  store_apply(our_tx);
+ *
+ * And the store should look like this after we apply 'our_tx':
+ *
+ *  paxos:
+ *    first_committed -> 1
+ *     last_committed -> 5
+ *		    1 -> value_1
+ *		    2 -> value_2
+ *		    3 -> value_3
+ *		    4 -> value_4
+ *		    5 -> Tx_foo_bufferlist
+ *  foo:
+ *    first_committed -> 3
+ *     last_committed -> 3
+ *		    3 -> foo_value_3
+ *
+ */
+
+#ifndef CEPH_MON_PAXOS_H
+#define CEPH_MON_PAXOS_H
+
+#include "include/types.h"
+#include "mon_types.h"
+#include "include/buffer.h"
+#include "messages/PaxosServiceMessage.h"
+#include "msg/msg_types.h"
+
+#include "include/Context.h"
+
+#include "common/Timer.h"
+#include 
+
+#include "MonitorDBStore.h"
+
+class Monitor;
+class MMonPaxos;
+class Paxos;
+
+// i am one state machine.
+/**
+ * This libary is based on the Paxos algorithm, but varies in a few key ways:
+ *  1- Only a single new value is generated at a time, simplifying the recovery logic.
+ *  2- Nodes track "committed" values, and share them generously (and trustingly)
+ *  3- A 'leasing' mechanism is built-in, allowing nodes to determine when it is 
+ *     safe to "read" their copy of the last committed value.
+ *
+ * This provides a simple replication substrate that services can be built on top of.
+ * See PaxosService.h
+ */
+class Paxos {
+  /**
+   * @defgroup Paxos_h_class Paxos
+   * @{
+   */
+  /**
+   * The Monitor to which this Paxos class is associated with.
+   */
+  Monitor *mon;
+
+  // my state machine info
+  const string paxos_name;
+
+  friend class Monitor;
+  friend class PaxosService;
+
+  list extra_state_dirs;
+
+  // LEADER+PEON
+
+  // -- generic state --
+public:
+  /**
+   * @defgroup Paxos_h_states States on which the leader/peon may be.
+   * @{
+   */
+  enum {
+    /**
+     * Leader/Peon is in Paxos' Recovery state
+     */
+    STATE_RECOVERING,
+    /**
+     * Leader/Peon is idle, and the Peon may or may not have a valid lease.
+     */
+    STATE_ACTIVE,
+    /**
+     * Leader/Peon is updating to a new value.
+     */
+    STATE_UPDATING,
+    /*
+     * Leader proposing an old value
+     */
+    STATE_UPDATING_PREVIOUS,
+  };
+
+  /**
+   * Obtain state name from constant value.
+   *
+   * @note This function will raise a fatal error if @p s is not
+   *	   a valid state value.
+   *
+   * @param s State value.
+   * @return The state's name.
+   */
+  static const string get_statename(int s) {
+    switch (s) {
+    case STATE_RECOVERING:
+      return "recovering";
+    case STATE_ACTIVE:
+      return "active";
+    case STATE_UPDATING:
+      return "updating";
+    case STATE_UPDATING_PREVIOUS:
+      return "updating-previous";
+    default:
+      return "UNKNOWN";
+    }
+  }
+
+private:
+  /**
+   * The state we are in.
+   */
+  int state;
+  /**
+   * @}
+   */
+
+public:
+  /**
+   * Check if we are recovering.
+   *
+   * @return 'true' if we are on the Recovering state; 'false' otherwise.
+   */
+  bool is_recovering() const { return (state & STATE_RECOVERING); }
+  /**
+   * Check if we are active.
+   *
+   * @return 'true' if we are on the Active state; 'false' otherwise.
+   */
+  bool is_active() const { return state == STATE_ACTIVE; }
+  /**
+   * Check if we are updating.
+   *
+   * @return 'true' if we are on the Updating state; 'false' otherwise.
+   */
+  bool is_updating() const { return state == STATE_UPDATING; }
+
+  /**
+   * Check if we are updating/proposing a previous value from a
+   * previous quorum
+   */
+  bool is_updating_previous() const { return state == STATE_UPDATING_PREVIOUS; }
+
+private:
+  /**
+   * @defgroup Paxos_h_recovery_vars Common recovery-related member variables
+   * @note These variables are common to both the Leader and the Peons.
+   * @{
+   */
+  /**
+   *
+   */
+  version_t first_committed;
+  /**
+   * Last Proposal Number
+   *
+   * @todo Expand description
+   */
+  version_t last_pn;
+  /**
+   * Last committed value's version.
+   *
+   * On both the Leader and the Peons, this is the last value's version that 
+   * was accepted by a given quorum and thus committed, that this instance 
+   * knows about.
+   *
+   * @note It may not be the last committed value's version throughout the
+   *	   system. If we are a Peon, we may have not been part of the quorum
+   *	   that accepted the value, and for this very same reason we may still
+   *	   be a (couple of) version(s) behind, until we learn about the most
+   *	   recent version. This should only happen if we are not active (i.e.,
+   *	   part of the quorum), which should not happen if we are up, running
+   *	   and able to communicate with others -- thus able to be part of the
+   *	   monmap and trigger new elections.
+   */
+  version_t last_committed;
+  /**
+   * Last committed value's time.
+   *
+   * When the commit happened.
+   */
+  utime_t last_commit_time;
+  /**
+   * The last Proposal Number we have accepted.
+   *
+   * On the Leader, it will be the Proposal Number picked by the Leader 
+   * itself. On the Peon, however, it will be the proposal sent by the Leader
+   * and it will only be updated iif its value is higher than the one
+   * already known by the Peon.
+   */
+  version_t accepted_pn;
+  /**
+   * The last_committed epoch of the leader at the time we accepted the last pn.
+   *
+   * This has NO SEMANTIC MEANING, and is there only for the debug output.
+   */
+  version_t accepted_pn_from;
+  /**
+   * Map holding the first committed version by each quorum member.
+   *
+   * The versions kept in this map are updated during the collect phase.
+   * When the Leader starts the collect phase, each Peon will reply with its
+   * first committed version, which will then be kept in this map.
+   */
+  map peer_first_committed;
+  /**
+   * Map holding the last committed version by each quorum member.
+   *
+   * The versions kept in this map are updated during the collect phase.
+   * When the Leader starts the collect phase, each Peon will reply with its
+   * last committed version, which will then be kept in this map.
+   */
+  map peer_last_committed;
+  /**
+   * @}
+   */
+
+  // active (phase 2)
+  /**
+   * @defgroup Paxos_h_active_vars Common active-related member variables
+   * @{
+   */
+  /**
+   * When does our read lease expires.
+   *
+   * Instead of performing a full commit each time a read is requested, we
+   * keep leases. Each lease will have an expiration date, which may or may
+   * not be extended. 
+   */
+  utime_t lease_expire;
+  /**
+   * List of callbacks waiting for our state to change into STATE_ACTIVE.
+   */
+  list waiting_for_active;
+  /**
+   * List of callbacks waiting for the chance to read a version from us.
+   *
+   * Each entry on the list may result from an attempt to read a version that
+   * wasn't available at the time, or an attempt made during a period during
+   * which we could not satisfy the read request. The first case happens if
+   * the requested version is greater than our last committed version. The
+   * second scenario may happen if we are recovering, or if we don't have a
+   * valid lease.
+   *
+   * The list will be woken up once we change to STATE_ACTIVE with an extended
+   * lease -- which can be achieved if we have everyone on the quorum on board
+   * with the latest proposal, or if we don't really care about the remaining
+   * uncommitted values --, or if we're on a quorum of one.
+   */
+  list waiting_for_readable;
+  /**
+   * @}
+   */
+
+  // -- leader --
+  // recovery (paxos phase 1)
+  /**
+   * @defgroup Paxos_h_leader_recovery Leader-specific Recovery-related vars
+   * @{
+   */
+  /**
+   * Number of replies to the collect phase we've received so far.
+   *
+   * This variable is reset to 1 each time we start a collect phase; it is
+   * incremented each time we receive a reply to the collect message, and
+   * is used to determine whether or not we have received replies from the
+   * whole quorum.
+   */
+  unsigned   num_last;
+  /**
+   * Uncommitted value's version.
+   *
+   * If we have, or end up knowing about, an uncommitted value, then its
+   * version will be kept in this variable.
+   *
+   * @note If this version equals @p last_committed+1 when we reach the final
+   *	   steps of recovery, then the algorithm will assume this is a value
+   *	   the Leader does not know about, and trustingly the Leader will 
+   *	   propose this version's value.
+   */
+  version_t  uncommitted_v;
+  /**
+   * Uncommitted value's Proposal Number.
+   *
+   * We use this variable to assess if the Leader should take into consideration
+   * an uncommitted value sent by a Peon. Given that the Peon will send back to
+   * the Leader the last Proposal Number he accepted, the Leader will be able
+   * to infer if this value is more recent than the one the Leader has, thus
+   * more relevant.
+   */
+  version_t  uncommitted_pn;
+  /**
+   * Uncommitted Value.
+   *
+   * If the system fails in-between the accept replies from the Peons and the
+   * instruction to commit from the Leader, then we may end up with accepted
+   * but yet-uncommitted values. During the Leader's recovery, he will attempt
+   * to bring the whole system to the latest state, and that means committing
+   * past accepted but uncommitted values.
+   *
+   * This variable will hold an uncommitted value, which may originate either
+   * on the Leader, or learnt by the Leader from a Peon during the collect
+   * phase.
+   */
+  bufferlist uncommitted_value;
+  /**
+   * Used to specify when an on-going collect phase times out.
+   */
+  Context    *collect_timeout_event;
+  /**
+   * @}
+   */
+
+  // active
+  /**
+   * @defgroup Paxos_h_leader_active Leader-specific Active-related vars
+   * @{
+   */
+  /**
+   * Set of participants (Leader & Peons) that have acked a lease extension.
+   *
+   * Each Peon that acknowledges a lease extension will have its place in this
+   * set, which will be used to account for all the acks from all the quorum
+   * members, guaranteeing that we trigger new elections if some don't ack in
+   * the expected timeframe.
+   */
+  set   acked_lease;
+  /**
+   * Callback responsible for extending the lease periodically.
+   */
+  Context    *lease_renew_event;
+  /**
+   * Callback to trigger new elections once the time for acks is out.
+   */
+  Context    *lease_ack_timeout_event;
+  /**
+   * @}
+   */
+  /**
+   * @defgroup Paxos_h_peon_active Peon-specific Active-related vars
+   * @{
+   */
+  /**
+   * Callback to trigger new elections when the Peon's lease times out.
+   *
+   * If the Peon's lease is extended, this callback will be reset (i.e.,
+   * we cancel the event and reschedule a new one with starting from the
+   * beginning).
+   */
+  Context    *lease_timeout_event;
+  /**
+   * @}
+   */
+
+  // updating (paxos phase 2)
+  /**
+   * @defgroup Paxos_h_leader_updating Leader-specific Updating-related vars
+   * @{
+   */
+  /**
+   * New Value being proposed to the Peons.
+   *
+   * This bufferlist holds the value the Leader is proposing to the Peons, and
+   * that will be committed if the Peons do accept the proposal.
+   */
+  bufferlist new_value;
+  /**
+   * Set of participants (Leader & Peons) that accepted the new proposed value.
+   *
+   * This set is used to keep track of those who have accepted the proposed
+   * value, so the leader may know when to issue a commit (when a majority of
+   * participants has accepted the proposal), and when to extend the lease
+   * (when all the quorum members have accepted the proposal).
+   */
+  set   accepted;
+  /**
+   * Callback to trigger a new election if the proposal is not accepted by the
+   * full quorum within a given timeframe.
+   *
+   * If the full quorum does not accept the proposal, then it means that the
+   * Leader may no longer be recognized as the leader, or that the quorum has
+   * changed, and the value may have not reached all the participants. Thus,
+   * the leader must call new elections, and go through a recovery phase in
+   * order to propagate the new value throughout the system.
+   *
+   * This does not mean that we won't commit. We will commit as soon as we
+   * have a majority of acceptances. But if we do not have full acceptance
+   * from the quorum, then we cannot extend the lease, as some participants
+   * may not have the latest committed value.
+   */
+  Context    *accept_timeout_event;
+
+  /**
+   * List of callbacks waiting for it to be possible to write again.
+   *
+   * @remarks It is not possible to write if we are not the Leader, or we are
+   *	      not on the active state, or if the lease has expired.
+   */
+  list waiting_for_writeable;
+  /**
+   * List of callbacks waiting for a commit to finish.
+   *
+   * @remarks This may be used to a) wait for an on-going commit to finish
+   *	      before we proceed with, say, a new proposal; or b) wait for the
+   *	      next commit to be finished so we are sure that our value was
+   *	      fully committed.
+   */
+  list waiting_for_commit;
+  /**
+   *
+   */
+  list proposals;
+  /**
+   * @}
+   */
+
+  /**
+   * @defgroup Paxos_h_sync_warns Synchronization warnings
+   * @todo Describe these variables
+   * @{
+   */
+  utime_t last_clock_drift_warn;
+  int clock_drift_warned;
+  /**
+   * @}
+   */
+
+  /**
+   * Should be true if we have proposed to trim, or are in the middle of
+   * trimming; false otherwise.
+   */
+  bool trimming;
+
+  /**
+   * @defgroup Paxos_h_callbacks Callback classes.
+   * @{
+   */
+  /**
+   * Callback class responsible for handling a Collect Timeout.
+   */
+  class C_CollectTimeout : public Context {
+    Paxos *paxos;
+  public:
+    C_CollectTimeout(Paxos *p) : paxos(p) {}
+    void finish(int r) {
+      if (r == -ECANCELED)
+	return;
+      paxos->collect_timeout();
+    }
+  };
+
+  /**
+   * Callback class responsible for handling an Accept Timeout.
+   */
+  class C_AcceptTimeout : public Context {
+    Paxos *paxos;
+  public:
+    C_AcceptTimeout(Paxos *p) : paxos(p) {}
+    void finish(int r) {
+      if (r == -ECANCELED)
+	return;
+      paxos->accept_timeout();
+    }
+  };
+
+  /**
+   * Callback class responsible for handling a Lease Ack Timeout.
+   */
+  class C_LeaseAckTimeout : public Context {
+    Paxos *paxos;
+  public:
+    C_LeaseAckTimeout(Paxos *p) : paxos(p) {}
+    void finish(int r) {
+      if (r == -ECANCELED)
+	return;
+      paxos->lease_ack_timeout();
+    }
+  };
+
+  /**
+   * Callback class responsible for handling a Lease Timeout.
+   */
+  class C_LeaseTimeout : public Context {
+    Paxos *paxos;
+  public:
+    C_LeaseTimeout(Paxos *p) : paxos(p) {}
+    void finish(int r) {
+      if (r == -ECANCELED)
+	return;
+      paxos->lease_timeout();
+    }
+  };
+
+  /**
+   * Callback class responsible for handling a Lease Renew Timeout.
+   */
+  class C_LeaseRenew : public Context {
+    Paxos *paxos;
+  public:
+    C_LeaseRenew(Paxos *p) : paxos(p) {}
+    void finish(int r) {
+      if (r == -ECANCELED)
+	return;
+      paxos->lease_renew_timeout();
+    }
+  };
+
+  class C_Trimmed : public Context {
+    Paxos *paxos;
+  public:
+    C_Trimmed(Paxos *p) : paxos(p) { }
+    void finish(int r) {
+      paxos->trimming = false;
+    }
+  };
+  /**
+   *
+   */
+public:
+  class C_Proposal : public Context {
+    Context *proposer_context;
+  public:
+    bufferlist bl;
+    // for debug purposes. Will go away. Soon.
+    bool proposed;
+    utime_t proposal_time;
+
+    C_Proposal(Context *c, bufferlist& proposal_bl) :
+	proposer_context(c),
+	bl(proposal_bl),
+        proposed(false),
+	proposal_time(ceph_clock_now(NULL))
+      { }
+
+    void finish(int r) {
+      if (proposer_context) {
+	proposer_context->complete(r);
+	proposer_context = NULL;
+      }
+    }
+  };
+  /**
+   * @}
+   */
+private:
+  /**
+   * @defgroup Paxos_h_election_triggered Steps triggered by an election.
+   *
+   * @note All these functions play a significant role in the Recovery Phase,
+   *	   which is triggered right after an election once someone becomes
+   *	   the Leader.
+   * @{
+   */
+  /**
+   * Create a new Proposal Number and propose it to the Peons.
+   *
+   * This function starts the Recovery Phase, which can be directly mapped
+   * onto the original Paxos' Prepare phase. Basically, we'll generate a
+   * Proposal Number, taking @p oldpn into consideration, and we will send
+   * it to a quorum, along with our first and last committed versions. By
+   * sending these information in a message to the quorum, we expect to
+   * obtain acceptances from a majority, allowing us to commit, or be
+   * informed of a higher Proposal Number known by one or more of the Peons
+   * in the quorum.
+   *
+   * @pre We are the Leader.
+   * @post Recovery Phase initiated by sending messages to the quorum.
+   *
+   * @param oldpn A proposal number taken as the highest known so far, that
+   *		  should be taken into consideration when generating a new 
+   *		  Proposal Number for the Recovery Phase.
+   */
+  void collect(version_t oldpn);
+  /**
+   * Handle the reception of a collect message from the Leader and reply
+   * accordingly.
+   *
+   * Once a Peon receives a collect message from the Leader it will reply
+   * with its first and last committed versions, as well as information so
+   * the Leader may know if his Proposal Number was, or was not, accepted by
+   * the Peon. The Peon will accept the Leader's Proposal Number iif it is
+   * higher than the Peon's currently accepted Proposal Number. The Peon may
+   * also inform the Leader of accepted but uncommitted values.
+   *
+   * @invariant The message is an operation of type OP_COLLECT.
+   * @pre We are a Peon.
+   * @post Replied to the Leader, accepting or not accepting his PN.
+   *
+   * @param collect The collect message sent by the Leader to the Peon.
+   */
+  void handle_collect(MMonPaxos *collect);
+  /**
+   * Handle a response from a Peon to the Leader's collect phase.
+   *
+   * The received message will state the Peon's last committed version, as 
+   * well as its last proposal number. This will lead to one of the following
+   * scenarios: if the replied Proposal Number is equal to the one we proposed,
+   * then the Peon has accepted our proposal, and if all the Peons do accept
+   * our Proposal Number, then we are allowed to proceed with the commit;
+   * however, if a Peon replies with a higher Proposal Number, we assume he
+   * knows something we don't and the Leader will have to abort the current
+   * proposal in order to retry with the Proposal Number specified by the Peon.
+   * It may also occur that the Peon replied with a lower Proposal Number, in
+   * which case we assume it is a reply to an an older value and we'll simply
+   * drop it.
+   * This function will also check if the Peon replied with an accepted but
+   * yet uncommitted value. In this case, if its version is higher than our
+   * last committed value by one, we assume that the Peon knows a value from a
+   * previous proposal that has never been committed, and we should try to
+   * commit that value by proposing it next. On the other hand, if that is
+   * not the case, we'll assume it is an old, uncommitted value, we do not
+   * care about and we'll consider the system active by extending the leases.
+   *
+   * @invariant The message is an operation of type OP_LAST.
+   * @pre We are the Leader.
+   * @post We initiate a commit, or we retry with a higher Proposal Number, 
+   *	   or we drop the message.
+   * @post We move from STATE_RECOVERING to STATE_ACTIVE.
+   *
+   * @param last The message sent by the Peon to the Leader.
+   */
+  void handle_last(MMonPaxos *last);
+  /**
+   * The Recovery Phase timed out, meaning that a significant part of the
+   * quorum does not believe we are the Leader, and we thus should trigger new
+   * elections.
+   *
+   * @pre We believe to be the Leader.
+   * @post Trigger new elections.
+   */
+  void collect_timeout();
+  /**
+   * @}
+   */
+
+  /**
+   * @defgroup Paxos_h_updating_funcs Functions used during the Updating State
+   * 
+   * These functions may easily be mapped to the original Paxos Algorithm's 
+   * phases. 
+   *
+   * Taking into account the algorithm can be divided in 4 phases (Prepare,
+   * Promise, Accept Request and Accepted), we can easily map Paxos::begin to
+   * both the Prepare and Accept Request phases; the Paxos::handle_begin to
+   * the Promise phase; and the Paxos::handle_accept to the Accepted phase.
+   * @{
+   */
+  /**
+   * Start a new proposal with the intent of committing @p value.
+   *
+   * If we are alone on the system (i.e., a quorum of one), then we will
+   * simply commit the value, but if we are not alone, then we need to propose
+   * the value to the quorum.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post We commit, iif we are alone, or we send a message to each quorum 
+   *	   member
+   * @post We are on STATE_ACTIVE, iif we are alone, or on 
+   *	   STATE_UPDATING otherwise
+   *
+   * @param value The value being proposed to the quorum
+   */
+  void begin(bufferlist& value);
+  /**
+   * Accept or decline (by ignoring) a proposal from the Leader.
+   *
+   * We will decline the proposal (by ignoring it) if we have promised to
+   * accept a higher numbered proposal. If that is not the case, we will
+   * accept it and accordingly reply to the Leader.
+   *
+   * @pre We are a Peon
+   * @pre We are on STATE_ACTIVE
+   * @post We are on STATE_UPDATING iif we accept the Leader's proposal
+   * @post We send a reply message to the Leader iif we accept his proposal
+   *
+   * @invariant The received message is an operation of type OP_BEGIN
+   *
+   * @param begin The message sent by the Leader to the Peon during the
+   *		  Paxos::begin function
+   *
+   */
+  void handle_begin(MMonPaxos *begin);
+  /**
+   * Handle an Accept message sent by a Peon.
+   *
+   * In order to commit, the Leader has to receive accepts from a majority of
+   * the quorum. If that does happen, then the Leader may proceed with the
+   * commit. However, the Leader needs the accepts from all the quorum members
+   * in order to extend the lease and move on to STATE_ACTIVE.
+   *
+   * This function handles these two situations, accounting for the amount of
+   * received accepts.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_UPDATING
+   * @post We are on STATE_ACTIVE iif we received accepts from the full quorum
+   * @post We extended the lease iif we moved on to STATE_ACTIVE
+   * @post We are on STATE_UPDATING iif we didn't received accepts from the
+   *	   full quorum
+   * @post We have committed iif we received accepts from a majority
+   *
+   * @invariant The received message is an operation of type OP_ACCEPT
+   *
+   * @param accept The message sent by the Peons to the Leader during the
+   *		   Paxos::handle_begin function
+   */
+  void handle_accept(MMonPaxos *accept);
+  /**
+   * Trigger a fresh election.
+   *
+   * During Paxos::begin we set a Callback of type Paxos::C_AcceptTimeout in
+   * order to limit the amount of time we spend waiting for Accept replies.
+   * This callback will call Paxos::accept_timeout when it is fired.
+   *
+   * This is essential to the algorithm because there may be the chance that
+   * we are no longer the Leader (i.e., others don't believe in us) and we
+   * are getting ignored, or we dropped out of the quorum and haven't realised
+   * it. So, our only option is to trigger fresh elections.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_UPDATING
+   * @post Triggered fresh elections
+   */
+  void accept_timeout();
+  /**
+   * @}
+   */
+
+  /**
+   * Commit a value throughout the system.
+   *
+   * The Leader will cancel the current lease (as it was for the old value),
+   * and will store the committed value locally. It will then instruct every
+   * quorum member to do so as well.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_UPDATING
+   * @pre A majority of quorum members accepted our proposal
+   * @post Value locally stored
+   * @post Quorum members instructed to commit the new value.
+   */
+  void commit();
+  /**
+   * Commit the new value to stable storage as being the latest available
+   * version.
+   *
+   * @pre We are a Peon
+   * @post The new value is locally stored
+   * @post Fire up the callbacks waiting on waiting_for_commit
+   *
+   * @invariant The received message is an operation of type OP_COMMIT
+   *
+   * @param commit The message sent by the Leader to the Peon during
+   *		   Paxos::commit
+   */
+  void handle_commit(MMonPaxos *commit);
+  /**
+   * Extend the system's lease.
+   *
+   * This means that the Leader considers that it should now safe to read from
+   * any node on the system, since every quorum member is now in possession of
+   * the latest version. Therefore, the Leader will send a message stating just
+   * this to each quorum member, and will impose a limited timeframe during
+   * which acks will be accepted. If there aren't as many acks as expected
+   * (i.e, if at least one quorum member does not ack the lease) during this
+   * timeframe, then we will force fresh elections.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post A message extending the lease is sent to each quorum member
+   * @post A timeout callback is set to limit the amount of time we will wait
+   *	   for lease acks.
+   * @post A timer is set in order to renew the lease after a certain amount
+   *	   of time.
+   */
+  void extend_lease();
+  /**
+   * Update the lease on the Peon's side of things.
+   *
+   * Once a Peon receives a Lease message, it will update its lease_expire
+   * variable, reply to the Leader acknowledging the lease update and set a
+   * timeout callback to be fired upon the lease's expiration. Finally, the
+   * Peon will fire up all the callbacks waiting for it to become active,
+   * which it just did, and all those waiting for it to become readable,
+   * which should be true if the Peon's lease didn't expire in the mean time.
+   *
+   * @pre We are a Peon
+   * @post We update the lease accordingly
+   * @post A lease timeout callback is set
+   * @post Move to STATE_ACTIVE
+   * @post Fire up all the callbacks waiting for STATE_ACTIVE
+   * @post Fire up all the callbacks waiting for readable iif we are readable
+   * @post Ack the lease to the Leader
+   *
+   * @invariant The received message is an operation of type OP_LEASE
+   *
+   * @param The message sent by the Leader to the Peon during the
+   *	    Paxos::extend_lease function
+   */
+  void handle_lease(MMonPaxos *lease);
+  /**
+   * Account for all the Lease Acks the Leader receives from the Peons.
+   *
+   * Once the Leader receives all the Lease Acks from the Peons, it will be
+   * able to cancel the Lease Ack timeout callback, thus avoiding calling
+   * fresh elections.
+   *
+   * @pre We are the Leader
+   * @post Cancel the Lease Ack timeout callback iif we receive acks from all
+   *	   the quorum members
+   *
+   * @invariant The received message is an operation of type OP_LEASE_ACK
+   *
+   * @param ack The message sent by a Peon to the Leader during the
+   *		Paxos::handle_lease function
+   */
+  void handle_lease_ack(MMonPaxos *ack);
+  /**
+   * Call fresh elections because at least one Peon didn't acked our lease.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post Trigger fresh elections
+   */
+  void lease_ack_timeout();
+  /**
+   * Extend lease since we haven't had new committed values meanwhile.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post Go through with Paxos::extend_lease
+   */
+  void lease_renew_timeout();
+  /**
+   * Call fresh elections because the Peon's lease expired without being
+   * renewed or receiving a fresh lease.
+   *
+   * This means that the Peon is no longer assumed as being in the quorum
+   * (or there is no Leader to speak of), so just trigger fresh elections
+   * to circumvent this issue.
+   *
+   * @pre We are a Peon
+   * @post Trigger fresh elections
+   */
+  void lease_timeout();        // on peon, if lease isn't extended
+
+  /// restart the lease timeout timer
+  void reset_lease_timeout();
+
+  /**
+   * Cancel all of Paxos' timeout/renew events. 
+   */
+  void cancel_events();
+  /**
+   * Shutdown this Paxos machine
+   */
+  void shutdown();
+
+  /**
+   * Generate a new Proposal Number based on @p gt
+   *
+   * @todo Check what @p gt actually means and what its usage entails
+   * @param gt A hint for the geration of the Proposal Number
+   * @return A globally unique, monotonically increasing Proposal Number
+   */
+  version_t get_new_proposal_number(version_t gt=0);
+ 
+  /**
+   * @todo document sync function
+   */
+  void warn_on_future_time(utime_t t, entity_name_t from);
+
+  /**
+   * Queue a new proposal by pushing it at the back of the queue; do not
+   * propose it.
+   *
+   * @param bl The bufferlist to be proposed
+   * @param onfinished The callback to be called once the proposal finishes
+   */
+  void queue_proposal(bufferlist& bl, Context *onfinished);
+  /**
+   * Begin proposing the Proposal at the front of the proposals queue.
+   */
+  void propose_queued();
+
+  /**
+   * refresh state from store
+   *
+   * Called when we have new state for the mon to consume.  If we return false,
+   * abort (we triggered a bootstrap).
+   *
+   * @returns true on success, false if we are now bootstrapping
+   */
+  bool do_refresh();
+
+  void commit_proposal();
+  void finish_round();
+
+public:
+  /**
+   * @param m A monitor
+   * @param mid A machine id
+   */
+  Paxos(Monitor *m, const string &name) 
+		 : mon(m),
+		   paxos_name(name),
+		   state(STATE_RECOVERING),
+		   first_committed(0),
+		   last_pn(0),
+		   last_committed(0),
+		   accepted_pn(0),
+		   accepted_pn_from(0),
+		   num_last(0),
+		   uncommitted_v(0), uncommitted_pn(0),
+		   collect_timeout_event(0),
+		   lease_renew_event(0),
+		   lease_ack_timeout_event(0),
+		   lease_timeout_event(0),
+		   accept_timeout_event(0),
+		   clock_drift_warned(0),
+		   trimming(false) { }
+
+  const string get_name() const {
+    return paxos_name;
+  }
+
+  void dispatch(PaxosServiceMessage *m);
+
+  void read_and_prepare_transactions(MonitorDBStore::Transaction *tx, version_t from, version_t last);
+
+  void init();
+
+  /**
+   * dump state info to a formatter
+   */
+  void dump_info(Formatter *f);
+
+  /**
+   * This function runs basic consistency checks. Importantly, if
+   * it is inconsistent and shouldn't be, it asserts out.
+   *
+   * @return True if consistent, false if not.
+   */
+  bool is_consistent();
+
+  void restart();
+  /**
+   * Initiate the Leader after it wins an election.
+   *
+   * Once an election is won, the Leader will be initiated and there are two
+   * possible outcomes of this method: the Leader directly jumps to the active
+   * state (STATE_ACTIVE) if it believes to be the only one in the quorum, or
+   * will start recovering (STATE_RECOVERING) by initiating the collect phase. 
+   *
+   * @pre Our monitor is the Leader.
+   * @post We are either on STATE_ACTIVE if we're the only one in the quorum,
+   *	   or on STATE_RECOVERING otherwise.
+   */
+  void leader_init();
+  /**
+   * Initiate a Peon after it loses an election.
+   *
+   * If we are a Peon, then there must be a Leader and we are not alone in the
+   * quorum, thus automatically assume we are on STATE_RECOVERING, which means
+   * we will soon be enrolled into the Leader's collect phase.
+   *
+   * @pre There is a Leader, and he's about to start the collect phase.
+   * @post We are on STATE_RECOVERING and will soon receive collect phase's 
+   *	   messages.
+   */
+  void peon_init();
+
+  /**
+   * Include an incremental state of values, ranging from peer_first_committed
+   * to the last committed value, on the message m
+   *
+   * @param m A message
+   * @param peer_first_committed Lowest version to take into account
+   * @param peer_last_committed Highest version to take into account
+   */
+  void share_state(MMonPaxos *m, version_t peer_first_committed,
+		   version_t peer_last_committed);
+  /**
+   * Store on disk a state that was shared with us
+   *
+   * Basically, we received a set of version. Or just one. It doesn't matter.
+   * What matters is that we have to stash it in the store. So, we will simply
+   * write every single bufferlist into their own versions on our side (i.e.,
+   * onto paxos-related keys), and then we will decode those same bufferlists
+   * we just wrote and apply the transactions they hold. We will also update
+   * our first and last committed values to point to the new values, if need
+   * be. All this is done tightly wrapped in a transaction to ensure we
+   * enjoy the atomicity guarantees given by our awesome k/v store.
+   *
+   * @param m A message
+   * @returns true if we stored something new; false otherwise
+   */
+  bool store_state(MMonPaxos *m);
+  void _sanity_check_store();
+
+  /**
+   * remove legacy paxos versions from before conversion
+   */
+  void remove_legacy_versions();
+
+  /**
+   * Helper function to decode a bufferlist into a transaction and append it
+   * to another transaction.
+   *
+   * This function is used during the Leader's commit and during the
+   * Paxos::store_state in order to apply the bufferlist's transaction onto
+   * the store.
+   *
+   * @param t The transaction to which we will append the operations
+   * @param bl A bufferlist containing an encoded transaction
+   */
+  static void decode_append_transaction(MonitorDBStore::Transaction& t,
+				 bufferlist& bl) {
+    MonitorDBStore::Transaction vt;
+    bufferlist::iterator it = bl.begin();
+    vt.decode(it);
+    t.append(vt);
+  }
+
+  /**
+   * @todo This appears to be used only by the OSDMonitor, and I would say
+   *	   its objective is to allow a third-party to have a "private"
+   *	   state dir. -JL
+   */
+  void add_extra_state_dir(string s) {
+    extra_state_dirs.push_back(s);
+  }
+
+  // -- service interface --
+  /**
+   * Add c to the list of callbacks waiting for us to become active.
+   *
+   * @param c A callback
+   */
+  void wait_for_active(Context *c) {
+    waiting_for_active.push_back(c);
+  }
+
+  /**
+   * Trim the Paxos state as much as we can.
+   */
+  void trim();
+
+  /**
+   * Check if we should trim.
+   *
+   * If trimming is disabled, we must take that into consideration and only
+   * return true if we are positively sure that we should trim soon.
+   *
+   * @returns true if we should trim; false otherwise.
+   */
+  bool should_trim() {
+    int available_versions = get_version() - get_first_committed();
+    int maximum_versions = g_conf->paxos_min + g_conf->paxos_trim_min;
+
+    if (trimming || (available_versions <= maximum_versions))
+      return false;
+
+    return true;
+  }
+ 
+  // read
+  /**
+   * @defgroup Paxos_h_read_funcs Read-related functions
+   * @{
+   */
+  /**
+   * Get latest committed version
+   *
+   * @return latest committed version
+   */
+  version_t get_version() { return last_committed; }
+  /**
+   * Get first committed version
+   *
+   * @return the first committed version
+   */
+  version_t get_first_committed() { return first_committed; }
+  /**
+   * Check if a given version is readable.
+   *
+   * A version may not be readable for a myriad of reasons:
+   *  @li the version @v is higher that the last committed version
+   *  @li we are not the Leader nor a Peon (election may be on-going)
+   *  @li we do not have a committed value yet
+   *  @li we do not have a valid lease
+   *
+   * @param seen The version we want to check if it is readable.
+   * @return 'true' if the version is readable; 'false' otherwise.
+   */
+  bool is_readable(version_t seen=0);
+  /**
+   * Read version @v and store its value in @bl
+   *
+   * @param[in] v The version we want to read
+   * @param[out] bl The version's value
+   * @return 'true' if we successfully read the value; 'false' otherwise
+   */
+  bool read(version_t v, bufferlist &bl);
+  /**
+   * Read the latest committed version
+   *
+   * @param[out] bl The version's value
+   * @return the latest committed version if we successfully read the value;
+   *	     or 0 (zero) otherwise.
+   */
+  version_t read_current(bufferlist &bl);
+  /**
+   * Add onreadable to the list of callbacks waiting for us to become readable.
+   *
+   * @param onreadable A callback
+   */
+  void wait_for_readable(Context *onreadable) {
+    assert(!is_readable());
+    waiting_for_readable.push_back(onreadable);
+  }
+  /**
+   * @}
+   */
+
+  /**
+   * Check if we have a valid lease.
+   *
+   * @returns true if the lease is still valid; false otherwise.
+   */
+  bool is_lease_valid();
+  // write
+  /**
+   * @defgroup Paxos_h_write_funcs Write-related functions
+   * @{
+   */
+  /**
+   * Check if we are writeable.
+   *
+   * We are writeable if we are alone (i.e., a quorum of one), or if we match
+   * all the following conditions:
+   *  @li We are the Leader
+   *  @li We are on STATE_ACTIVE
+   *  @li We have a valid lease
+   *
+   * @return 'true' if we are writeable; 'false' otherwise.
+   */
+  bool is_writeable();
+  /**
+   * Add c to the list of callbacks waiting for us to become writeable.
+   *
+   * @param c A callback
+   */
+  void wait_for_writeable(Context *c) {
+    assert(!is_writeable());
+    waiting_for_writeable.push_back(c);
+  }
+
+  /**
+   * List all queued proposals
+   *
+   * @param out[out] Output Stream onto which we will output the list
+   *		     of queued proposals.
+   */
+  void list_proposals(ostream& out);
+  /**
+   * Propose a new value to the Leader.
+   *
+   * This function enables the submission of a new value to the Leader, which
+   * will trigger a new proposal.
+   *
+   * @param bl A bufferlist holding the value to be proposed
+   * @param onfinish A callback to be fired up once we finish the proposal
+   */
+  bool propose_new_value(bufferlist& bl, Context *onfinished=0);
+  /**
+   * Add oncommit to the back of the list of callbacks waiting for us to
+   * finish committing.
+   *
+   * @param oncommit A callback
+   */
+  void wait_for_commit(Context *oncommit) {
+    waiting_for_commit.push_back(oncommit);
+  }
+  /**
+   * Add oncommit to the front of the list of callbacks waiting for us to
+   * finish committing.
+   *
+   * @param oncommit A callback
+   */
+  void wait_for_commit_front(Context *oncommit) {
+    waiting_for_commit.push_front(oncommit);
+  }
+  /**
+   * @}
+   */
+
+  /**
+   * @}
+   */
+ protected:
+  MonitorDBStore *get_store();
+};
+
+inline ostream& operator<<(ostream& out, Paxos::C_Proposal& p)
+{
+  string proposed = (p.proposed ? "proposed" : "unproposed");
+  out << " " << proposed
+      << " queued " << (ceph_clock_now(NULL) - p.proposal_time)
+      << " tx dump:\n";
+  MonitorDBStore::Transaction t;
+  bufferlist::iterator p_it = p.bl.begin();
+  t.decode(p_it);
+  JSONFormatter f(true);
+  t.dump(&f);
+  f.flush(out);
+  return out;
+}
+
+#endif
+
diff --git a/ceph/src/mon/PaxosService.cc b/ceph/src/mon/PaxosService.cc
new file mode 100644
index 00000000..1b216898
--- /dev/null
+++ b/ceph/src/mon/PaxosService.cc
@@ -0,0 +1,386 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "PaxosService.h"
+#include "common/Clock.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+
+
+#include "common/config.h"
+#include "include/assert.h"
+#include "common/Formatter.h"
+
+#define dout_subsys ceph_subsys_paxos
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, paxos, service_name, get_first_committed(), get_last_committed())
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, Paxos *paxos, string service_name,
+			version_t fc, version_t lc) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").paxosservice(" << service_name << " " << fc << ".." << lc << ") ";
+}
+
+bool PaxosService::dispatch(PaxosServiceMessage *m)
+{
+  dout(10) << "dispatch " << *m << " from " << m->get_orig_source_inst() << dendl;
+
+  if (mon->is_shutdown()) {
+    m->put();
+    return true;
+  }
+
+  // make sure this message isn't forwarded from a previous election epoch
+  if (m->rx_election_epoch &&
+      m->rx_election_epoch < mon->get_epoch()) {
+    dout(10) << " discarding forwarded message from previous election epoch "
+	     << m->rx_election_epoch << " < " << mon->get_epoch() << dendl;
+    m->put();
+    return true;
+  }
+
+  // make sure the client is still connected.  note that a proxied
+  // connection will be disconnected with a null message; don't drop
+  // those.  also ignore loopback (e.g., log) messages.
+  if (!m->get_connection()->is_connected() &&
+      m->get_connection() != mon->con_self &&
+      m->get_connection()->get_messenger() != NULL) {
+    dout(10) << " discarding message from disconnected client "
+	     << m->get_source_inst() << " " << *m << dendl;
+    m->put();
+    return true;
+  }
+
+  // make sure our map is readable and up to date
+  if (!is_readable(m->version)) {
+    dout(10) << " waiting for paxos -> readable (v" << m->version << ")" << dendl;
+    wait_for_readable(new C_RetryMessage(this, m), m->version);
+    return true;
+  }
+
+  // preprocess
+  if (preprocess_query(m)) 
+    return true;  // easy!
+
+  // leader?
+  if (!mon->is_leader()) {
+    mon->forward_request_leader(m);
+    return true;
+  }
+  
+  // writeable?
+  if (!is_writeable()) {
+    dout(10) << " waiting for paxos -> writeable" << dendl;
+    wait_for_writeable(new C_RetryMessage(this, m));
+    return true;
+  }
+
+  // update
+  if (prepare_update(m)) {
+    double delay = 0.0;
+    if (should_propose(delay)) {
+      if (delay == 0.0) {
+	propose_pending();
+      } else {
+	// delay a bit
+	if (!proposal_timer) {
+	  proposal_timer = new C_Propose(this);
+	  dout(10) << " setting proposal_timer " << proposal_timer << " with delay of " << delay << dendl;
+	  mon->timer.add_event_after(delay, proposal_timer);
+	} else { 
+	  dout(10) << " proposal_timer already set" << dendl;
+	}
+      }
+    } else {
+      dout(10) << " not proposing" << dendl;
+    }
+  }     
+  return true;
+}
+
+void PaxosService::refresh(bool *need_bootstrap)
+{
+  // update cached versions
+  cached_first_committed = mon->store->get(get_service_name(), first_committed_name);
+  cached_last_committed = mon->store->get(get_service_name(), last_committed_name);
+
+  version_t new_format = get_value("format_version");
+  if (new_format != format_version) {
+    dout(1) << __func__ << " upgraded, format " << format_version << " -> " << new_format << dendl;
+    on_upgrade();
+  }
+  format_version = new_format;
+
+  dout(10) << __func__ << dendl;
+
+  update_from_paxos(need_bootstrap);
+}
+
+
+void PaxosService::remove_legacy_versions()
+{
+  dout(10) << __func__ << dendl;
+  if (!mon->store->exists(get_service_name(), "conversion_first"))
+    return;
+
+  version_t cf = mon->store->get(get_service_name(), "conversion_first");
+  version_t fc = get_first_committed();
+
+  dout(10) << __func__ << " conversion_first " << cf
+	   << " first committed " << fc << dendl;
+
+  MonitorDBStore::Transaction t;
+  if (cf < fc) {
+    trim(&t, cf, fc);
+  }
+  t.erase(get_service_name(), "conversion_first");
+  mon->store->apply_transaction(t);
+}
+
+bool PaxosService::should_propose(double& delay)
+{
+  // simple default policy: quick startup, then some damping.
+  if (get_last_committed() <= 1)
+    delay = 0.0;
+  else {
+    utime_t now = ceph_clock_now(g_ceph_context);
+    if ((now - paxos->last_commit_time) > g_conf->paxos_propose_interval)
+      delay = (double)g_conf->paxos_min_wait;
+    else
+      delay = (double)(g_conf->paxos_propose_interval + paxos->last_commit_time
+		       - now);
+  }
+  return true;
+}
+
+
+void PaxosService::propose_pending()
+{
+  dout(10) << "propose_pending" << dendl;
+  assert(have_pending);
+  assert(!proposing);
+  assert(mon->is_leader());
+  assert(is_active());
+
+  if (proposal_timer) {
+    dout(10) << " canceling proposal_timer " << proposal_timer << dendl;
+    mon->timer.cancel_event(proposal_timer);
+    proposal_timer = NULL;
+  }
+
+  /**
+   * @note The value we propose is encoded in a bufferlist, passed to 
+   *	   Paxos::propose_new_value and it is obtained by calling a 
+   *	   function that must be implemented by the class implementing us.
+   *	   I.e., the function encode_pending will be the one responsible
+   *	   to encode whatever is pending on the implementation class into a
+   *	   bufferlist, so we can then propose that as a value through Paxos.
+   */
+  MonitorDBStore::Transaction t;
+  bufferlist bl;
+
+  if (should_stash_full())
+    encode_full(&t);
+
+  encode_pending(&t);
+  have_pending = false;
+
+  if (format_version > 0) {
+    t.put(get_service_name(), "format_version", format_version);
+  }
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t.dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  t.encode(bl);
+
+  // apply to paxos
+  proposing = true;
+  paxos->propose_new_value(bl, new C_Committed(this));
+}
+
+bool PaxosService::should_stash_full()
+{
+  version_t latest_full = get_version_latest_full();
+  /* @note The first member of the condition is moot and it is here just for
+   *	   clarity's sake. The second member would end up returing true
+   *	   nonetheless because, in that event,
+   *	      latest_full == get_trim_to() == 0.
+   */
+  return (!latest_full ||
+	  (latest_full <= get_trim_to()) ||
+	  (get_last_committed() - latest_full > (unsigned)g_conf->paxos_stash_full_interval));
+}
+
+void PaxosService::restart()
+{
+  dout(10) << "restart" << dendl;
+  if (proposal_timer) {
+    dout(10) << " canceling proposal_timer " << proposal_timer << dendl;
+    mon->timer.cancel_event(proposal_timer);
+    proposal_timer = 0;
+  }
+
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, -EAGAIN);
+
+  if (have_pending) {
+    discard_pending();
+    have_pending = false;
+  }
+  proposing = false;
+
+  on_restart();
+}
+
+void PaxosService::election_finished()
+{
+  dout(10) << "election_finished" << dendl;
+
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, -EAGAIN);
+
+  // make sure we update our state
+  _active();
+}
+
+void PaxosService::_active()
+{
+  if (is_proposing()) {
+    dout(10) << "_acting - proposing" << dendl;
+    return;
+  }
+  if (!is_active()) {
+    dout(10) << "_active - not active" << dendl;
+    wait_for_active(new C_Active(this));
+    return;
+  }
+  dout(10) << "_active" << dendl;
+
+  remove_legacy_versions();
+
+  // create pending state?
+  if (mon->is_leader() && is_active()) {
+    dout(7) << "_active creating new pending" << dendl;
+    if (!have_pending) {
+      create_pending();
+      have_pending = true;
+    }
+
+    if (get_last_committed() == 0) {
+      // create initial state
+      create_initial();
+      propose_pending();
+      return;
+    }
+  } else {
+    if (!mon->is_leader()) {
+      dout(7) << __func__ << " we are not the leader, hence we propose nothing!" << dendl;
+    } else if (!is_active()) {
+      dout(7) << __func__ << " we are not active, hence we propose nothing!" << dendl;
+    }
+  }
+
+  // wake up anyone who came in while we were proposing.  note that
+  // anyone waiting for the previous proposal to commit is no longer
+  // on this list; it is on Paxos's.
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, 0);
+
+  if (is_active() && mon->is_leader())
+    upgrade_format();
+
+  // NOTE: it's possible that this will get called twice if we commit
+  // an old paxos value.  Implementations should be mindful of that.
+  if (is_active())
+    on_active();
+}
+
+
+void PaxosService::shutdown()
+{
+  cancel_events();
+
+  if (proposal_timer) {
+    dout(10) << " canceling proposal_timer " << proposal_timer << dendl;
+    mon->timer.cancel_event(proposal_timer);
+    proposal_timer = 0;
+  }
+
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, -EAGAIN);
+
+  on_shutdown();
+}
+
+void PaxosService::maybe_trim()
+{
+  if (!is_writeable())
+    return;
+
+  version_t trim_to = get_trim_to();
+  if (trim_to < get_first_committed())
+    return;
+
+  version_t to_remove = trim_to - get_first_committed();
+  if (g_conf->paxos_service_trim_min > 0 &&
+      to_remove < (version_t)g_conf->paxos_service_trim_min) {
+    dout(10) << __func__ << " trim_to " << trim_to << " would only trim " << to_remove
+	     << " < paxos_service_trim_min " << g_conf->paxos_service_trim_min << dendl;
+    return;
+  }
+
+  if (g_conf->paxos_service_trim_max > 0 &&
+      to_remove > (version_t)g_conf->paxos_service_trim_max) {
+    dout(10) << __func__ << " trim_to " << trim_to << " would only trim " << to_remove
+	     << " > paxos_service_trim_max, limiting to " << g_conf->paxos_service_trim_max
+	     << dendl;
+    trim_to = get_first_committed() + g_conf->paxos_service_trim_max;
+    to_remove = trim_to - get_first_committed();
+  }
+
+  dout(10) << __func__ << " trimming to " << trim_to << ", " << to_remove << " states" << dendl;
+  MonitorDBStore::Transaction t;
+  trim(&t, get_first_committed(), trim_to);
+  put_first_committed(&t, trim_to);
+
+  // let the service add any extra stuff
+  encode_trim_extra(&t, trim_to);
+
+  bufferlist bl;
+  t.encode(bl);
+  paxos->propose_new_value(bl, NULL);
+}
+
+void PaxosService::trim(MonitorDBStore::Transaction *t,
+			version_t from, version_t to)
+{
+  dout(10) << __func__ << " from " << from << " to " << to << dendl;
+  assert(from != to);
+
+  for (version_t v = from; v < to; ++v) {
+    dout(20) << __func__ << " " << v << dendl;
+    t->erase(get_service_name(), v);
+
+    string full_key = mon->store->combine_strings("full", v);
+    if (mon->store->exists(get_service_name(), full_key)) {
+      dout(20) << __func__ << " " << full_key << dendl;
+      t->erase(get_service_name(), full_key);
+    }
+  }
+  if (g_conf->mon_compact_on_trim) {
+    dout(20) << " compacting prefix " << get_service_name() << dendl;
+    t->compact_range(get_service_name(), stringify(from - 1), stringify(to));
+  }
+}
+
diff --git a/ceph/src/mon/PaxosService.h b/ceph/src/mon/PaxosService.h
new file mode 100644
index 00000000..5321bebc
--- /dev/null
+++ b/ceph/src/mon/PaxosService.h
@@ -0,0 +1,911 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_PAXOSSERVICE_H
+#define CEPH_PAXOSSERVICE_H
+
+#include "messages/PaxosServiceMessage.h"
+#include "include/Context.h"
+#include "include/stringify.h"
+#include 
+#include "Paxos.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+
+class Monitor;
+class Paxos;
+
+/**
+ * A Paxos Service is an abstraction that easily allows one to obtain an
+ * association between a Monitor and a Paxos class, in order to implement any
+ * service.
+ */
+class PaxosService {
+  /**
+   * @defgroup PaxosService_h_class Paxos Service
+   * @{
+   */
+ public:
+  /**
+   * The Monitor to which this class is associated with
+   */
+  Monitor *mon;
+  /**
+   * The Paxos instance to which this class is associated with
+   */
+  Paxos *paxos;
+  /**
+   * Our name. This will be associated with the class implementing us, and will
+   * be used mainly for store-related operations.
+   */
+  string service_name;
+  /**
+   * If we are or have queued anything for proposal, this variable will be true
+   * until our proposal has been finished.
+   */
+  bool proposing;
+
+ protected:
+  /**
+   * Services implementing us used to depend on the Paxos version, back when
+   * each service would have a Paxos instance for itself. However, now we only
+   * have a single Paxos instance, shared by all the services. Each service now
+   * must keep its own version, if so they wish. This variable should be used
+   * for that purpose.
+   */
+  version_t service_version;
+
+ private:
+  /**
+   * Event callback responsible for proposing our pending value once a timer 
+   * runs out and fires.
+   */
+  Context *proposal_timer;
+  /**
+   * If the implementation class has anything pending to be proposed to Paxos,
+   * then have_pending should be true; otherwise, false.
+   */
+  bool have_pending; 
+
+protected:
+
+  /**
+   * format of our state in leveldb, 0 for default
+   */
+  version_t format_version;
+
+
+
+  /**
+   * @defgroup PaxosService_h_callbacks Callback classes
+   * @{
+   */
+  /**
+   * Retry dispatching a given service message
+   *
+   * This callback class is used when we had to wait for some condition to
+   * become true while we were dispatching it.
+   *
+   * For instance, if the message's version isn't readable, according to Paxos,
+   * then we must wait for it to become readable. So, we just queue an
+   * instance of this class onto the Paxos::wait_for_readable function, and
+   * we will retry the whole dispatch again once the callback is fired.
+   */
+  class C_RetryMessage : public Context {
+    PaxosService *svc;
+    PaxosServiceMessage *m;
+  public:
+    C_RetryMessage(PaxosService *s, PaxosServiceMessage *m_) : svc(s), m(m_) {}
+    void finish(int r) {
+      if (r == -EAGAIN || r >= 0)
+	svc->dispatch(m);
+      else if (r == -ECANCELED)
+	m->put();
+      else
+	assert(0 == "bad C_RetryMessage return value");
+    }
+  };
+
+  /**
+   * Callback used to make sure we call the PaxosService::_active function
+   * whenever a condition is fulfilled.
+   *
+   * This is used in multiple situations, from waiting for the Paxos to commit
+   * our proposed value, to waiting for the Paxos to become active once an
+   * election is finished.
+   */
+  class C_Active : public Context {
+    PaxosService *svc;
+  public:
+    C_Active(PaxosService *s) : svc(s) {}
+    void finish(int r) {
+      if (r >= 0)
+	svc->_active();
+    }
+  };
+
+  /**
+   * Callback class used to propose the pending value once the proposal_timer
+   * fires up.
+   */
+  class C_Propose : public Context {
+    PaxosService *ps;
+  public:
+    C_Propose(PaxosService *p) : ps(p) { }
+    void finish(int r) {
+      ps->proposal_timer = 0;
+      if (r >= 0)
+	ps->propose_pending();
+      else if (r == -ECANCELED || r == -EAGAIN)
+	return;
+      else
+	assert(0 == "bad return value for C_Propose");
+    }
+  };
+
+  /**
+   * Callback class used to mark us as active once a proposal finishes going
+   * through Paxos.
+   *
+   * We should wake people up *only* *after* we inform the service we
+   * just went active. And we should wake people up only once we finish
+   * going active. This is why we first go active, avoiding to wake up the
+   * wrong people at the wrong time, such as waking up a C_RetryMessage
+   * before waking up a C_Active, thus ending up without a pending value.
+   */
+  class C_Committed : public Context {
+    PaxosService *ps;
+  public:
+    C_Committed(PaxosService *p) : ps(p) { }
+    void finish(int r) {
+      ps->proposing = false;
+      if (r >= 0)
+	ps->_active();
+      else if (r == -ECANCELED || r == -EAGAIN)
+	return;
+      else
+	assert(0 == "bad return value for C_Committed");
+    }
+  };
+  /**
+   * @}
+   */
+  friend class C_Propose;
+  
+
+public:
+  /**
+   * @param mn A Monitor instance
+   * @param p A Paxos instance
+   * @parem name Our service's name.
+   */
+  PaxosService(Monitor *mn, Paxos *p, string name) 
+    : mon(mn), paxos(p), service_name(name),
+      proposing(false),
+      service_version(0), proposal_timer(0), have_pending(false),
+      format_version(0),
+      last_committed_name("last_committed"),
+      first_committed_name("first_committed"),
+      full_prefix_name("full"), full_latest_name("latest"),
+      cached_first_committed(0), cached_last_committed(0)
+  {
+  }
+
+  virtual ~PaxosService() {}
+
+  /**
+   * Get the service's name.
+   *
+   * @returns The service's name.
+   */
+  string get_service_name() { return service_name; }
+
+  /**
+   * Get the store prefixes we utilize
+   */
+  virtual void get_store_prefixes(set& s) {
+    s.insert(service_name);
+  }
+  
+  // i implement and you ignore
+  /**
+   * Informs this instance that it should consider itself restarted.
+   *
+   * This means that we will cancel our proposal_timer event, if any exists.
+   */
+  void restart();
+  /**
+   * Informs this instance that an election has finished.
+   *
+   * This means that we will invoke a PaxosService::discard_pending while
+   * setting have_pending to false (basically, ignore our pending state) and
+   * we will then make sure we obtain a new state.
+   *
+   * Our state shall be updated by PaxosService::_active if the Paxos is
+   * active; otherwise, we will wait for it to become active by adding a 
+   * PaxosService::C_Active callback to it.
+   */
+  void election_finished();
+  /**
+   * Informs this instance that it is supposed to shutdown.
+   *
+   * Basically, it will instruct Paxos to cancel all events/callbacks and then
+   * will cancel the proposal_timer event if any exists.
+   */
+  void shutdown();
+
+private:
+  /**
+   * Update our state by updating it from Paxos, and then creating a new
+   * pending state if need be.
+   *
+   * @remarks We only create a pending state we our Monitor is the Leader.
+   *
+   * @pre Paxos is active
+   * @post have_pending is true iif our Monitor is the Leader and Paxos is
+   *	   active
+   */
+  void _active();
+  /**
+   * Scrub our versions after we convert the store from the old layout to
+   * the new k/v store.
+   */
+  void remove_legacy_versions();
+
+public:
+  /**
+   * Propose a new value through Paxos.
+   *
+   * This function should be called by the classes implementing 
+   * PaxosService, in order to propose a new value through Paxos.
+   *
+   * @pre The implementation class implements the encode_pending function.
+   * @pre have_pending is true
+   * @pre Our monitor is the Leader
+   * @pre Paxos is active
+   * @post Cancel the proposal timer, if any
+   * @post have_pending is false
+   * @post propose pending value through Paxos
+   *
+   * @note This function depends on the implementation of encode_pending on
+   *	   the class that is implementing PaxosService
+   */
+  void propose_pending();
+
+  /**
+   * Let others request us to propose.
+   *
+   * At the moment, this is just a wrapper to propose_pending() with an
+   * extra check for is_writeable(), but it's a good practice to dissociate
+   * requests for proposals from direct usage of propose_pending() for
+   * future use -- we might want to perform additional checks or put a
+   * request on hold, for instance.
+   */
+  void request_proposal() {
+    assert(is_writeable());
+
+    propose_pending();
+  }
+  /**
+   * Request service @p other to perform a proposal.
+   *
+   * We could simply use the function above, requesting @p other directly,
+   * but we might eventually want to do something to the request -- say,
+   * set a flag stating we're waiting on a cross-proposal to be finished.
+   */
+  void request_proposal(PaxosService *other) {
+    assert(other != NULL);
+    assert(other->is_writeable());
+
+    other->request_proposal();
+  }
+
+  /**
+   * Dispatch a message by passing it to several different functions that are
+   * either implemented directly by this service, or that should be implemented
+   * by the class implementing this service.
+   *
+   * @param m A message
+   * @returns 'true' on successful dispatch; 'false' otherwise.
+   */
+  bool dispatch(PaxosServiceMessage *m);
+
+  void refresh(bool *need_bootstrap);
+
+  /**
+   * @defgroup PaxosService_h_override_funcs Functions that should be
+   *					     overridden.
+   *
+   * These functions should be overridden at will by the class implementing
+   * this service.
+   * @{
+   */
+  /**
+   * Create the initial state for your system.
+   *
+   * In some of ours the state is actually set up elsewhere so this does
+   * nothing.
+   */
+  virtual void create_initial() = 0;
+
+  /**
+   * Query the Paxos system for the latest state and apply it if it's newer
+   * than the current Monitor state.
+   *
+   * @returns 'true' on success; 'false' otherwise.
+   */
+  virtual void update_from_paxos(bool *need_bootstrap) = 0;
+
+  /**
+   * Hook called after all services have refreshed their state from paxos
+   *
+   * This is useful for doing any update work that depends on other
+   * service's having up-to-date state.
+   */
+  virtual void post_paxos_update() {}
+
+  /**
+   * Init on startup
+   *
+   * This is called on mon startup, after all of the PaxosService instances'
+   * update_from_paxos() methods have been called
+   */
+  virtual void init() {}
+
+  /**
+   * Create the pending state.
+   *
+   * @invariant This function is only called on a Leader.
+   * @remarks This created state is then modified by incoming messages.
+   * @remarks Called at startup and after every Paxos ratification round.
+   */
+  virtual void create_pending() = 0;
+
+  /**
+   * Encode the pending state into a bufferlist for ratification and
+   * transmission as the next state.
+   *
+   * @invariant This function is only called on a Leader.
+   *
+   * @param t The transaction to hold all changes.
+   */
+  virtual void encode_pending(MonitorDBStore::Transaction *t) = 0;
+
+  /**
+   * Discard the pending state
+   *
+   * @invariant This function is only called on a Leader.
+   *
+   * @remarks This function is NOT overridden in any of our code, but it is
+   *	      called in PaxosService::election_finished if have_pending is
+   *	      true.
+   */
+  virtual void discard_pending() { }
+
+  /**
+   * Look at the query; if the query can be handled without changing state,
+   * do so.
+   *
+   * @param m A query message
+   * @returns 'true' if the query was handled (e.g., was a read that got
+   *	      answered, was a state change that has no effect); 'false' 
+   *	      otherwise.
+   */
+  virtual bool preprocess_query(PaxosServiceMessage *m) = 0;
+
+  /**
+   * Apply the message to the pending state.
+   *
+   * @invariant This function is only called on a Leader.
+   *
+   * @param m An update message
+   * @returns 'true' if the update message was handled (e.g., a command that
+   *	      went through); 'false' otherwise.
+   */
+  virtual bool prepare_update(PaxosServiceMessage *m) = 0;
+  /**
+   * @}
+   */
+
+  /**
+   * Determine if the Paxos system should vote on pending, and if so how long
+   * it should wait to vote.
+   *
+   * @param[out] delay The wait time, used so we can limit the update traffic
+   *		       spamming.
+   * @returns 'true' if the Paxos system should propose; 'false' otherwise.
+   */
+  virtual bool should_propose(double &delay);
+
+  /**
+   * @defgroup PaxosService_h_courtesy Courtesy functions
+   *
+   * Courtesy functions, in case the class implementing this service has
+   * anything it wants/needs to do at these times.
+   * @{
+   */
+  /**
+   * This is called when the Paxos state goes to active.
+   *
+   * On the peon, this is after each election.
+   * On the leader, this is after each election, *and* after each completed
+   * proposal.
+   *
+   * @note This function may get called twice in certain recovery cases.
+   */
+  virtual void on_active() { }
+
+  /**
+   * This is called when we are shutting down
+   */
+  virtual void on_shutdown() {}
+
+  /**
+   * this is called when activating on the leader
+   *
+   * it should conditionally upgrade the on-disk format by proposing a transaction
+   */
+  virtual void upgrade_format() { }
+
+  /**
+   * this is called when we detect the store has just upgraded underneath us
+   */
+  virtual void on_upgrade() {}
+
+  /**
+   * Called when the Paxos system enters a Leader election.
+   *
+   * @remarks It's a courtesy method, in case the class implementing this
+   *	      service has anything it wants/needs to do at that time.
+   */
+  virtual void on_restart() { }
+  /**
+   * @}
+   */
+
+  /**
+   * Tick.
+   */
+  virtual void tick() {}
+
+  /**
+   * Get health information
+   *
+   * @param summary list of summary strings and associated severity
+   * @param detail optional list of detailed problem reports; may be NULL
+   */
+  virtual void get_health(list >& summary,
+			  list > *detail) const { }
+
+ private:
+  /**
+   * @defgroup PaxosService_h_store_keys Set of keys that are usually used on
+   *					 all the services implementing this
+   *					 class, and, being almost the only keys
+   *					 used, should be standardized to avoid
+   *					 mistakes.
+   * @{
+   */
+  const string last_committed_name;
+  const string first_committed_name;
+  const string full_prefix_name;
+  const string full_latest_name;
+  /**
+   * @}
+   */
+
+  /**
+   * @defgroup PaxosService_h_version_cache Variables holding cached values
+   *                                        for the most used versions (first
+   *                                        and last committed); we only have
+   *                                        to read them when the store is
+   *                                        updated, so in-between updates we
+   *                                        may very well use cached versions
+   *                                        and avoid the overhead.
+   * @{
+   */
+  version_t cached_first_committed;
+  version_t cached_last_committed;
+  /**
+   * @}
+   */
+
+  /**
+   * Callback list to be used whenever we are running a proposal through
+   * Paxos. These callbacks will be awaken whenever the said proposal
+   * finishes.
+   */
+  list waiting_for_finished_proposal;
+
+ public:
+
+  /**
+   * Check if we are proposing a value through Paxos
+   *
+   * @returns true if we are proposing; false otherwise.
+   */
+  bool is_proposing() {
+    return proposing;
+  }
+
+  /**
+   * Check if we are in the Paxos ACTIVE state.
+   *
+   * @note This function is a wrapper for Paxos::is_active
+   *
+   * @returns true if in state ACTIVE; false otherwise.
+   */
+  bool is_active() {
+    return
+      !is_proposing() &&
+      (paxos->is_active() || paxos->is_updating());
+  }
+
+  /**
+   * Check if we are readable.
+   *
+   * This mirrors on the paxos check, except that we also verify that
+   *
+   *  - the client hasn't seen the future relative to this PaxosService
+   *  - this service isn't proposing.
+   *  - we have committed our initial state (last_committed > 0)
+   *
+   * @param ver The version we want to check if is readable
+   * @returns true if it is readable; false otherwise
+   */
+  bool is_readable(version_t ver = 0) {
+    if (ver > get_last_committed() ||
+	is_proposing() ||
+	!paxos->is_readable(0) ||
+	get_last_committed() == 0)
+      return false;
+    return true;
+  }
+
+  /**
+   * Check if we are writeable.
+   *
+   * We consider to be writeable iff:
+   *
+   *  - we are not proposing a new version;
+   *  - we are ready to be written to -- i.e., we have a pending value.
+   *  - paxos is (active or updating)
+   *
+   * @returns true if writeable; false otherwise
+   */
+  bool is_writeable() {
+    return
+      !is_proposing() &&
+      is_write_ready() &&
+      (paxos->is_active() || paxos->is_updating());
+  }
+
+  /**
+   * Check if we are ready to be written to.  This means we must have a
+   * pending value and be active.
+   *
+   * @returns true if we are ready to be written to; false otherwise.
+   */
+  bool is_write_ready() {
+    return is_active() && have_pending;
+  }
+
+  /**
+   * Wait for a proposal to finish.
+   *
+   * Add a callback to be awaken whenever our current proposal finishes being
+   * proposed through Paxos.
+   *
+   * @param c The callback to be awaken once the proposal is finished.
+   */
+  void wait_for_finished_proposal(Context *c) {
+    waiting_for_finished_proposal.push_back(c);
+  }
+
+  /**
+   * Wait for us to become active
+   *
+   * @param c The callback to be awaken once we become active.
+   */
+  void wait_for_active(Context *c) {
+    if (!is_proposing()) {
+      paxos->wait_for_active(c);
+      return;
+    }
+    wait_for_finished_proposal(c);
+  }
+
+  /**
+   * Wait for us to become readable
+   *
+   * @param c The callback to be awaken once we become active.
+   * @param ver The version we want to wait on.
+   */
+  void wait_for_readable(Context *c, version_t ver = 0) {
+    /* This is somewhat of a hack. We only do check if a version is readable on
+     * PaxosService::dispatch(), but, nonetheless, we must make sure that if that
+     * is why we are not readable, then we must wait on PaxosService and not on
+     * Paxos; otherwise, we may assert on Paxos::wait_for_readable() if it
+     * happens to be readable at that specific point in time.
+     */
+    if (is_proposing() ||
+	ver > get_last_committed() ||
+	get_last_committed() == 0)
+      wait_for_finished_proposal(c);
+    else
+      paxos->wait_for_readable(c);
+  }
+
+  /**
+   * Wait for us to become writeable
+   *
+   * @param c The callback to be awaken once we become writeable.
+   */
+  void wait_for_writeable(Context *c) {
+    if (is_proposing())
+      wait_for_finished_proposal(c);
+    else if (!is_write_ready())
+      wait_for_active(c);
+    else
+      paxos->wait_for_writeable(c);
+  }
+
+  /**
+   * @defgroup PaxosService_h_Trim
+   * @{
+   */
+  /**
+   * trim service states if appropriate
+   *
+   * Called at same interval as tick()
+   */
+  void maybe_trim();
+
+  /**
+   * Auxiliary function to trim our state from version @from to version @to,
+   * not including; i.e., the interval [from, to[
+   *
+   * @param t The transaction to which we will add the trim operations.
+   * @param from the lower limit of the interval to be trimmed
+   * @param to the upper limit of the interval to be trimmed (not including)
+   */
+  void trim(MonitorDBStore::Transaction *t, version_t from, version_t to);
+
+  /**
+   * encode service-specific extra bits into trim transaction
+   *
+   * @param tx transaction
+   * @param first new first_committed value
+   */
+  virtual void encode_trim_extra(MonitorDBStore::Transaction *tx, version_t first) {}
+
+  /**
+   * Get the version we should trim to.
+   *
+   * Should be overloaded by service if it wants to trim states.
+   *
+   * @returns the version we should trim to; if we return zero, it should be
+   *	      assumed that there's no version to trim to.
+   */
+  virtual version_t get_trim_to() {
+    return 0;
+  }
+
+  /**
+   * @}
+   */
+  /**
+   * @defgroup PaxosService_h_Stash_Full
+   * @{
+   */
+  virtual bool should_stash_full();
+  /**
+   * Encode a full version on @p t
+   *
+   * @note We force every service to implement this function, since we strongly
+   *	   desire the encoding of full versions.
+   * @note Services that do not trim their state, will be bound to only create
+   *	   one full version. Full version stashing is determined/controled by
+   *	   trimming: we stash a version each time a trim is bound to erase the
+   *	   latest full version.
+   *
+   * @param t Transaction on which the full version shall be encoded.
+   */
+  virtual void encode_full(MonitorDBStore::Transaction *t) = 0;
+
+  /**
+   * @}
+   */
+
+  /**
+   * Cancel events.
+   *
+   * @note This function is a wrapper for Paxos::cancel_events
+   */
+  void cancel_events() {
+    paxos->cancel_events();
+  }
+
+  /**
+   * @defgroup PaxosService_h_store_funcs Back storage interface functions
+   * @{
+   */
+  /**
+   * @defgroup PaxosService_h_store_modify Wrapper function interface to access
+   *					   the back store for modification
+   *					   purposes
+   * @{
+   */
+  void put_first_committed(MonitorDBStore::Transaction *t, version_t ver) {
+    t->put(get_service_name(), first_committed_name, ver);
+  }
+  /**
+   * Set the last committed version to @p ver
+   *
+   * @param t A transaction to which we add this put operation
+   * @param ver The last committed version number being put
+   */
+  void put_last_committed(MonitorDBStore::Transaction *t, version_t ver) {
+    t->put(get_service_name(), last_committed_name, ver);
+
+    /* We only need to do this once, and that is when we are about to make our
+     * first proposal. There are some services that rely on first_committed
+     * being set -- and it should! -- so we need to guarantee that it is,
+     * specially because the services itself do not do it themselves. They do
+     * rely on it, but they expect us to deal with it, and so we shall.
+     */
+    if (!get_first_committed())
+      put_first_committed(t, ver);
+  }
+  /**
+   * Put the contents of @p bl into version @p ver
+   *
+   * @param t A transaction to which we will add this put operation
+   * @param ver The version to which we will add the value
+   * @param bl A bufferlist containing the version's value
+   */
+  void put_version(MonitorDBStore::Transaction *t, version_t ver,
+		   bufferlist& bl) {
+    t->put(get_service_name(), ver, bl);
+  }
+  /**
+   * Put the contents of @p bl into a full version key for this service, that
+   * will be created with @p ver in mind.
+   *
+   * @param t The transaction to which we will add this put operation
+   * @param ver A version number
+   * @param bl A bufferlist containing the version's value
+   */
+  void put_version_full(MonitorDBStore::Transaction *t,
+			version_t ver, bufferlist& bl) {
+    string key = mon->store->combine_strings(full_prefix_name, ver);
+    t->put(get_service_name(), key, bl);
+  }
+  /**
+   * Put the version number in @p ver into the key pointing to the latest full
+   * version of this service.
+   *
+   * @param t The transaction to which we will add this put operation
+   * @param ver A version number
+   */
+  void put_version_latest_full(MonitorDBStore::Transaction *t, version_t ver) {
+    string key = mon->store->combine_strings(full_prefix_name, full_latest_name);
+    t->put(get_service_name(), key, ver);
+  }
+  /**
+   * Put the contents of @p bl into the key @p key.
+   *
+   * @param t A transaction to which we will add this put operation
+   * @param key The key to which we will add the value
+   * @param bl A bufferlist containing the value
+   */
+  void put_value(MonitorDBStore::Transaction *t, const string& key, bufferlist& bl) {
+    t->put(get_service_name(), key, bl);
+  }
+
+  /**
+   * @}
+   */
+
+  /**
+   * @defgroup PaxosService_h_store_get Wrapper function interface to access
+   *					the back store for reading purposes
+   * @{
+   */
+
+  /**
+   * @defgroup PaxosService_h_version_cache Obtain cached versions for this
+   *                                        service.
+   * @{
+   */
+  /**
+   * Get the first committed version
+   *
+   * @returns Our first committed version (that is available)
+   */
+  version_t get_first_committed() {
+    return cached_first_committed;
+  }
+  /**
+   * Get the last committed version
+   *
+   * @returns Our last committed version
+   */
+  version_t get_last_committed() {
+    return cached_last_committed;
+  }
+
+  /**
+   * @}
+   */
+
+  /**
+   * Get the contents of a given version @p ver
+   *
+   * @param ver The version being obtained
+   * @param bl The bufferlist to be populated
+   * @return 0 on success; <0 otherwise
+   */
+  int get_version(version_t ver, bufferlist& bl) {
+    return mon->store->get(get_service_name(), ver, bl);
+  }
+  /**
+   * Get the contents of a given full version of this service.
+   *
+   * @param ver A version number
+   * @param bl The bufferlist to be populated
+   * @returns 0 on success; <0 otherwise
+   */
+  int get_version_full(version_t ver, bufferlist& bl) {
+    string key = mon->store->combine_strings(full_prefix_name, ver);
+    return mon->store->get(get_service_name(), key, bl);
+  }
+  /**
+   * Get the latest full version number
+   *
+   * @returns A version number
+   */
+  version_t get_version_latest_full() {
+    string key = mon->store->combine_strings(full_prefix_name, full_latest_name);
+    return mon->store->get(get_service_name(), key);
+  }
+
+  /**
+   * Get a value from a given key.
+   *
+   * @param[in] key The key
+   * @param[out] bl The bufferlist to be populated with the value
+   */
+  int get_value(const string& key, bufferlist& bl) {
+    return mon->store->get(get_service_name(), key, bl);
+  }
+  /**
+   * Get an integer value from a given key.
+   *
+   * @param[in] key The key
+   */
+  version_t get_value(const string& key) {
+    return mon->store->get(get_service_name(), key);
+  }
+
+  /**
+   * @}
+   */
+  /**
+   * @}
+   */
+};
+
+#endif
+
diff --git a/ceph/src/mon/QuorumService.h b/ceph/src/mon/QuorumService.h
new file mode 100644
index 00000000..ef9dcdcf
--- /dev/null
+++ b/ceph/src/mon/QuorumService.h
@@ -0,0 +1,135 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_MON_QUORUM_SERVICE_H
+#define CEPH_MON_QUORUM_SERVICE_H
+
+#include 
+
+#include "include/types.h"
+#include "include/Context.h"
+#include "common/RefCountedObj.h"
+#include "common/config.h"
+
+#include "mon/Monitor.h"
+
+class QuorumService
+{
+  Context *tick_event;
+  double tick_period;
+
+  struct C_Tick : public Context {
+    QuorumService *s;
+    C_Tick(QuorumService *qs) : s(qs) { }
+    void finish(int r) {
+      if (r < 0)
+        return;
+      s->tick();
+    }
+  };
+
+public:
+  enum {
+    SERVICE_HEALTH                   = 0x01,
+    SERVICE_TIMECHECK                = 0x02,
+    SERVICE_CONFIG_KEY               = 0x03,
+  };
+
+protected:
+  Monitor *mon;
+  epoch_t epoch;
+
+  QuorumService(Monitor *m) :
+    tick_event(NULL),
+    tick_period(g_conf->mon_tick_interval),
+    mon(m),
+    epoch(0)
+  {
+  }
+
+  void cancel_tick() {
+    if (tick_event)
+      mon->timer.cancel_event(tick_event);
+    tick_event = NULL;
+  }
+
+  void start_tick() {
+    generic_dout(10) << __func__ << dendl;
+
+    cancel_tick();
+    if (tick_period <= 0)
+      return;
+
+    tick_event = new C_Tick(this);
+    mon->timer.add_event_after(tick_period, tick_event);
+  }
+
+  void set_update_period(double t) {
+    tick_period = t;
+  }
+
+  bool in_quorum() {
+    return (mon->is_leader() || mon->is_peon());
+  }
+
+  virtual bool service_dispatch(Message *m) = 0;
+  virtual void service_tick() = 0;
+  virtual void service_shutdown() = 0;
+
+  virtual void start_epoch() = 0;
+  virtual void finish_epoch() = 0;
+  virtual void cleanup() = 0;
+
+public:
+  virtual ~QuorumService() { }
+
+  void start(epoch_t new_epoch) {
+    epoch = new_epoch;
+    start_epoch();
+  }
+
+  void finish() {
+    generic_dout(20) << "QuorumService::finish" << dendl;
+    finish_epoch();
+  }
+
+  epoch_t get_epoch() const {
+    return epoch;
+  }
+
+  bool dispatch(Message *m) {
+    return service_dispatch(m);
+  }
+
+  void tick() {
+    service_tick();
+    start_tick();
+  }
+
+  void shutdown() {
+    generic_dout(0) << "quorum service shutdown" << dendl;
+    cancel_tick();
+    service_shutdown();
+  }
+
+  virtual void init() { }
+
+  virtual void get_health(Formatter *f,
+			  list >& summary,
+                          list > *detail) = 0;
+  virtual int get_type() = 0;
+  virtual string get_name() const = 0;
+
+};
+
+#endif /* CEPH_MON_QUORUM_SERVICE_H */
diff --git a/ceph/src/mon/Session.h b/ceph/src/mon/Session.h
new file mode 100644
index 00000000..a5f67d9e
--- /dev/null
+++ b/ceph/src/mon/Session.h
@@ -0,0 +1,206 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MON_SESSION_H
+#define CEPH_MON_SESSION_H
+
+#include "include/xlist.h"
+#include "msg/msg_types.h"
+
+#include "auth/AuthServiceHandler.h"
+#include "osd/OSDMap.h"
+
+#include "MonCap.h"
+
+struct MonSession;
+
+struct Subscription {
+  MonSession *session;
+  string type;
+  xlist::item type_item;
+  version_t next;
+  bool onetime;
+  bool incremental_onetime;  // has CEPH_FEATURE_INCSUBOSDMAP
+  
+  Subscription(MonSession *s, const string& t) : session(s), type(t), type_item(this),
+						 next(0), onetime(false), incremental_onetime(false) {};
+};
+
+struct MonSession : public RefCountedObject {
+  ConnectionRef con;
+  entity_inst_t inst;
+  utime_t until;
+  utime_t time_established;
+  bool closed;
+  xlist::item item;
+  set routed_request_tids;
+  MonCap caps;
+  uint64_t auid;
+  uint64_t global_id;
+  uint64_t notified_global_id;
+
+  map sub_map;
+
+  AuthServiceHandler *auth_handler;
+
+  ConnectionRef proxy_con;
+  uint64_t proxy_tid;
+
+  MonSession(const entity_inst_t& i, Connection *c) :
+    con(c), inst(i), closed(false), item(this),
+    auid(0),
+    global_id(0), notified_global_id(0), auth_handler(NULL),
+    proxy_con(NULL), proxy_tid(0) {
+    time_established = ceph_clock_now(g_ceph_context);
+  }
+  ~MonSession() {
+    //generic_dout(0) << "~MonSession " << this << dendl;
+    // we should have been removed before we get destructed; see MonSessionMap::remove_session()
+    assert(!item.is_on_list());
+    assert(sub_map.empty());
+    delete auth_handler;
+  }
+
+  bool is_capable(string service, int mask) {
+    map args;
+    return caps.is_capable(g_ceph_context,
+			   inst.name,
+			   service, "", args,
+			   mask & MON_CAP_R, mask & MON_CAP_W, mask & MON_CAP_X);
+  }
+};
+
+
+struct MonSessionMap {
+  xlist sessions;
+  map* > subs;
+  multimap by_osd;
+
+  MonSessionMap() {}
+  ~MonSessionMap() {
+    while (!subs.empty()) {
+      assert(subs.begin()->second->empty());
+      delete subs.begin()->second;
+      subs.erase(subs.begin());
+    }
+  }
+
+  void remove_session(MonSession *s) {
+    assert(!s->closed);
+    for (map::iterator p = s->sub_map.begin(); p != s->sub_map.end(); ++p) {
+      p->second->type_item.remove_myself();
+      delete p->second;
+    }
+    s->sub_map.clear();
+    s->item.remove_myself();
+    if (s->inst.name.is_osd()) {
+      for (multimap::iterator p = by_osd.find(s->inst.name.num());
+	   p->first == s->inst.name.num();
+	   ++p)
+	if (p->second == s) {
+	  by_osd.erase(p);
+	  break;
+	}
+    }
+    s->closed = true;
+    s->put();
+  }
+
+  MonSession *new_session(const entity_inst_t& i, Connection *c) {
+    MonSession *s = new MonSession(i, c);
+    sessions.push_back(&s->item);
+    if (i.name.is_osd())
+      by_osd.insert(pair(i.name.num(), s));
+    s->get();  // caller gets a ref
+    return s;
+  }
+
+  MonSession *get_random_osd_session(OSDMap *osdmap) {
+    // ok, this isn't actually random, but close enough.
+    if (by_osd.empty())
+      return 0;
+    int n = by_osd.rbegin()->first + 1;
+    int r = rand() % n;
+
+    multimap::iterator p = by_osd.lower_bound(r);
+    if (p == by_osd.end())
+      --p;
+
+    if (!osdmap) {
+      return p->second;
+    }
+
+    MonSession *s = NULL;
+
+    multimap::iterator b = p, f = p;
+    bool backward = true, forward = true;
+    while (backward || forward) {
+      if (backward) {
+        if (osdmap->is_up(b->first) &&
+	    osdmap->get_addr(b->first) == b->second->con->get_peer_addr()) {
+          s = b->second;
+          break;
+        }
+        if (b != by_osd.begin())
+          --b;
+        else
+          backward = false;
+      }
+
+      forward = (f != by_osd.end());
+      if (forward) {
+        if (osdmap->is_up(f->first)) {
+          s = f->second;
+          break;
+        }
+        ++f;
+      }
+    }
+
+    return s;
+  }
+
+  void add_update_sub(MonSession *s, const string& what, version_t start, bool onetime, bool incremental_onetime) {
+    Subscription *sub = 0;
+    if (s->sub_map.count(what)) {
+      sub = s->sub_map[what];
+    } else {
+      sub = new Subscription(s, what);
+      s->sub_map[what] = sub;
+      
+      if (!subs.count(what))
+	subs[what] = new xlist;
+      subs[what]->push_back(&sub->type_item);
+    }
+    sub->next = start;
+    sub->onetime = onetime;
+    sub->incremental_onetime = onetime && incremental_onetime;
+  }
+
+  void remove_sub(Subscription *sub) {
+    sub->session->sub_map.erase(sub->type);
+    sub->type_item.remove_myself();
+    delete sub;
+  }
+};
+
+inline ostream& operator<<(ostream& out, const MonSession *s)
+{
+  out << "MonSession: " << s->inst << " is "
+      << (s->closed ? "closed" : "open");
+  out << s->caps;
+  return out;
+}
+
+#endif
diff --git a/ceph/src/mon/mon_types.h b/ceph/src/mon/mon_types.h
new file mode 100644
index 00000000..0ae1aaf8
--- /dev/null
+++ b/ceph/src/mon/mon_types.h
@@ -0,0 +1,182 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MON_TYPES_H
+#define CEPH_MON_TYPES_H
+
+#include "include/utime.h"
+#include "common/Formatter.h"
+
+#define PAXOS_PGMAP      0  // before osd, for pg kick to behave
+#define PAXOS_MDSMAP     1
+#define PAXOS_OSDMAP     2
+#define PAXOS_LOG        3
+#define PAXOS_MONMAP     4
+#define PAXOS_AUTH       5
+#define PAXOS_NUM        6
+
+inline const char *get_paxos_name(int p) {
+  switch (p) {
+  case PAXOS_MDSMAP: return "mdsmap";
+  case PAXOS_MONMAP: return "monmap";
+  case PAXOS_OSDMAP: return "osdmap";
+  case PAXOS_PGMAP: return "pgmap";
+  case PAXOS_LOG: return "logm";
+  case PAXOS_AUTH: return "auth";
+  default: assert(0); return 0;
+  }
+}
+
+#define CEPH_MON_ONDISK_MAGIC "ceph mon volume v012"
+
+/**
+ * leveldb store stats
+ *
+ * If we ever decide to support multiple backends for the monitor store,
+ * we should then create an abstract class 'MonitorStoreStats' of sorts
+ * and inherit it on LevelDBStoreStats.  I'm sure you'll figure something
+ * out.
+ */
+struct LevelDBStoreStats {
+  uint64_t bytes_total;
+  uint64_t bytes_sst;
+  uint64_t bytes_log;
+  uint64_t bytes_misc;
+  utime_t last_update;
+
+  void dump(Formatter *f) const {
+    assert(f != NULL);
+    f->dump_int("bytes_total", bytes_total);
+    f->dump_int("bytes_sst", bytes_sst);
+    f->dump_int("bytes_log", bytes_log);
+    f->dump_int("bytes_misc", bytes_misc);
+    f->dump_stream("last_updated") << last_update;
+  }
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(bytes_total, bl);
+    ::encode(bytes_sst, bl);
+    ::encode(bytes_log, bl);
+    ::encode(bytes_misc, bl);
+    ::encode(last_update, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator &p) {
+    DECODE_START(1, p);
+    ::decode(bytes_total, p);
+    ::decode(bytes_sst, p);
+    ::decode(bytes_log, p);
+    ::decode(bytes_misc, p);
+    ::decode(last_update, p);
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(LevelDBStoreStats);
+
+// data stats
+
+struct DataStats {
+  // data dir
+  uint64_t kb_total;
+  uint64_t kb_used;
+  uint64_t kb_avail;
+  int latest_avail_percent;
+  utime_t last_update;
+
+  LevelDBStoreStats store_stats;
+
+  void dump(Formatter *f) const {
+    assert(f != NULL);
+    f->dump_int("kb_total", kb_total);
+    f->dump_int("kb_used", kb_used);
+    f->dump_int("kb_avail", kb_avail);
+    f->dump_int("avail_percent", latest_avail_percent);
+    f->dump_stream("last_updated") << last_update;
+
+    f->open_object_section("store_stats");
+    store_stats.dump(f);
+    f->close_section();
+  }
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(2, 1, bl);
+    ::encode(kb_total, bl);
+    ::encode(kb_used, bl);
+    ::encode(kb_avail, bl);
+    ::encode(latest_avail_percent, bl);
+    ::encode(last_update, bl);
+    ::encode(store_stats, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &p) {
+    DECODE_START(1, p);
+    ::decode(kb_total, p);
+    ::decode(kb_used, p);
+    ::decode(kb_avail, p);
+    ::decode(latest_avail_percent, p);
+    ::decode(last_update, p);
+    if (struct_v > 1)
+      ::decode(store_stats, p);
+
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(DataStats);
+
+struct ScrubResult {
+  map prefix_crc;  ///< prefix -> crc
+  map prefix_keys; ///< prefix -> key count
+
+  bool operator!=(const ScrubResult& other) {
+    return prefix_crc != other.prefix_crc || prefix_keys != other.prefix_keys;
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(prefix_crc, bl);
+    ::encode(prefix_keys, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& p) {
+    DECODE_START(1, p);
+    ::decode(prefix_crc, p);
+    ::decode(prefix_keys, p);
+    DECODE_FINISH(p);
+  }
+  void dump(Formatter *f) const {
+    f->open_object_section("crc");
+    for (map::const_iterator p = prefix_crc.begin(); p != prefix_crc.end(); ++p)
+      f->dump_unsigned(p->first.c_str(), p->second);
+    f->close_section();
+    f->open_object_section("keys");
+    for (map::const_iterator p = prefix_keys.begin(); p != prefix_keys.end(); ++p)
+      f->dump_unsigned(p->first.c_str(), p->second);
+    f->close_section();
+  }
+  static void generate_test_instances(list& ls) {
+    ls.push_back(new ScrubResult);
+    ls.push_back(new ScrubResult);
+    ls.back()->prefix_crc["foo"] = 123;
+    ls.back()->prefix_keys["bar"] = 456;
+  }
+};
+WRITE_CLASS_ENCODER(ScrubResult);
+
+static inline ostream& operator<<(ostream& out, const ScrubResult& r) {
+  return out << "ScrubResult(keys " << r.prefix_keys << " crc " << r.prefix_crc << ")";
+}
+
+#endif
diff --git a/ceph/src/mount.fuse.ceph b/ceph/src/mount.fuse.ceph
new file mode 100755
index 00000000..785df6c1
--- /dev/null
+++ b/ceph/src/mount.fuse.ceph
@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# Helper to mount ceph-fuse from /etc/fstab.  To use, add an entry
+# like:
+#
+# # DEVICE                           PATH         TYPE        OPTIONS
+# id=admin                           /mnt/ceph    fuse.ceph   defaults   0 0
+# id=myuser,conf=/etc/ceph/foo.conf  /mnt/ceph2   fuse.ceph   defaults   0 0
+#
+# where the device field is a comma-separated list of options to pass on
+# the command line.  The examples above, for example, specify that
+# ceph-fuse will authenticated as client.admin and client.myuser
+# (respectively), and the second example also sets the 'conf' option to
+# '/etc/ceph/foo.conf' via the ceph-fuse command line.  Any valid
+# ceph-fuse can be passed in this way.
+
+set -e
+
+# convert device string to options
+cephargs='--'`echo $1 | sed 's/,/ --/g'`
+
+# strip out 'noauto' option; libfuse doesn't like it
+opts=`echo $4 | sed 's/,noauto//' | sed 's/noauto,//'`
+
+# strip out '_netdev' option; libfuse doesn't like it
+opts=`echo $opts | sed 's/,_netdev//' | sed 's/_netdev,//'`
+
+# go
+exec ceph-fuse $cephargs $2 $3 $opts
diff --git a/ceph/src/mount/canonicalize.c b/ceph/src/mount/canonicalize.c
new file mode 100644
index 00000000..7561e41a
--- /dev/null
+++ b/ceph/src/mount/canonicalize.c
@@ -0,0 +1,203 @@
+/*
+ * canonicalize.c -- canonicalize pathname by removing symlinks
+ * Copyright (C) 1993 Rick Sladkey 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+/*
+ * This routine is part of libc.  We include it nevertheless,
+ * since the libc version has some security flaws.
+ *
+ * TODO: use canonicalize_file_name() when exist in glibc
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#ifndef MAXSYMLINKS
+# define MAXSYMLINKS 256
+#endif
+
+static char *
+myrealpath(const char *path, char *resolved_path, int maxreslth) {
+	int readlinks = 0;
+	char *npath;
+	char *link_path;
+	int n;
+	char *buf = NULL;
+
+	npath = resolved_path;
+
+	/* If it's a relative pathname use getcwd for starters. */
+	if (*path != '/') {
+		if (!getcwd(npath, maxreslth-2))
+			return NULL;
+		npath += strlen(npath);
+		if (npath[-1] != '/')
+			*npath++ = '/';
+	} else {
+		*npath++ = '/';
+		path++;
+	}
+
+	/* Expand each slash-separated pathname component. */
+	link_path = malloc(PATH_MAX+1);
+	if (!link_path)
+		return NULL;
+	while (*path != '\0') {
+		/* Ignore stray "/" */
+		if (*path == '/') {
+			path++;
+			continue;
+		}
+		if (*path == '.' && (path[1] == '\0' || path[1] == '/')) {
+			/* Ignore "." */
+			path++;
+			continue;
+		}
+		if (*path == '.' && path[1] == '.' &&
+		    (path[2] == '\0' || path[2] == '/')) {
+			/* Backup for ".." */
+			path += 2;
+			while (npath > resolved_path+1 &&
+			       (--npath)[-1] != '/')
+				;
+			continue;
+		}
+		/* Safely copy the next pathname component. */
+		while (*path != '\0' && *path != '/') {
+			if (npath-resolved_path > maxreslth-2) {
+				errno = ENAMETOOLONG;
+				goto err;
+			}
+			*npath++ = *path++;
+		}
+
+		/* Protect against infinite loops. */
+		if (readlinks++ > MAXSYMLINKS) {
+			errno = ELOOP;
+			goto err;
+		}
+
+		/* See if last pathname component is a symlink. */
+		*npath = '\0';
+
+		n = readlink(resolved_path, link_path, PATH_MAX);
+		if (n < 0) {
+			/* EINVAL means the file exists but isn't a symlink. */
+			if (errno != EINVAL)
+				goto err;
+		} else {
+			int m;
+			char *newbuf;
+
+			/* Note: readlink doesn't add the null byte. */
+			link_path[n] = '\0';
+			if (*link_path == '/')
+				/* Start over for an absolute symlink. */
+				npath = resolved_path;
+			else
+				/* Otherwise back up over this component. */
+				while (*(--npath) != '/')
+					;
+
+			/* Insert symlink contents into path. */
+			m = strlen(path);
+			newbuf = malloc(m + n + 1);
+			if (!newbuf)
+				goto err;
+			memcpy(newbuf, link_path, n);
+			memcpy(newbuf + n, path, m + 1);
+			free(buf);
+			path = buf = newbuf;
+		}
+		*npath++ = '/';
+	}
+	/* Delete trailing slash but don't whomp a lone slash. */
+	if (npath != resolved_path+1 && npath[-1] == '/')
+		npath--;
+	/* Make sure it's null terminated. */
+	*npath = '\0';
+
+	free(link_path);
+	free(buf);
+	return resolved_path;
+
+ err:
+	free(link_path);
+	free(buf);
+	return NULL;
+}
+
+/*
+ * Converts private "dm-N" names to "/dev/mapper/"
+ *
+ * Since 2.6.29 (patch 784aae735d9b0bba3f8b9faef4c8b30df3bf0128) kernel sysfs
+ * provides the real DM device names in /sys/block//dm/name
+ */
+char *
+canonicalize_dm_name(const char *ptname)
+{
+	FILE	*f;
+	size_t	sz;
+	char	path[256], name[256], *res = NULL;
+
+	snprintf(path, sizeof(path), "/sys/block/%s/dm/name", ptname);
+	if (!(f = fopen(path, "r")))
+		return NULL;
+
+	/* read "\n" from sysfs */
+	if (fgets(name, sizeof(name), f) && (sz = strlen(name)) > 1) {
+		name[sz - 1] = '\0';
+		snprintf(path, sizeof(path), "/dev/mapper/%s", name);
+		res = strdup(path);
+	}
+	fclose(f);
+	return res;
+}
+
+char *
+canonicalize_path(const char *path)
+{
+	char *canonical;
+	char *p;
+
+	if (path == NULL)
+		return NULL;
+
+	canonical = malloc(PATH_MAX+2);
+	if (!canonical)
+		return NULL;
+	if (!myrealpath(path, canonical, PATH_MAX+1)) {
+		free(canonical);
+		return strdup(path);
+	}
+
+
+	p = strrchr(canonical, '/');
+	if (p && strncmp(p, "/dm-", 4) == 0 && isdigit(*(p + 4))) {
+		p = canonicalize_dm_name(p+1);
+		if (p) {
+			free(canonical);
+			return p;
+		}
+	}
+
+	return canonical;
+}
+
+
diff --git a/ceph/src/mount/mount.ceph.c b/ceph/src/mount/mount.ceph.c
new file mode 100644
index 00000000..563cd07e
--- /dev/null
+++ b/ceph/src/mount/mount.ceph.c
@@ -0,0 +1,380 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/secret.h"
+#include "include/addr_parsing.h"
+
+#ifndef MS_RELATIME
+# define MS_RELATIME (1<<21)
+#endif
+
+#define MAX_SECRET_LEN 1000
+#define MAX_SECRET_OPTION_LEN (MAX_SECRET_LEN + 7)
+
+int verboseflag = 0;
+int skip_mtab_flag = 0;
+static const char * const EMPTY_STRING = "";
+
+/* TODO duplicates logic from kernel */
+#define CEPH_AUTH_NAME_DEFAULT "guest"
+
+#include "mtab.c"
+
+static void block_signals (int how)
+{
+     sigset_t sigs;
+
+     sigfillset (&sigs);
+     sigdelset(&sigs, SIGTRAP);
+     sigdelset(&sigs, SIGSEGV);
+     sigprocmask (how, &sigs, (sigset_t *) 0);
+}
+
+static char *mount_resolve_src(const char *orig_str)
+{
+	int len, pos;
+	char *mount_path;
+	char *src;
+	char *buf = strdup(orig_str);
+
+	mount_path = strstr(buf, ":/");
+	if (!mount_path) {
+		printf("source mount path was not specified\n");
+		free(buf);
+		return NULL;
+	}
+	if (mount_path == buf) {
+		printf("server address expected\n");
+		free(buf);
+		return NULL;
+	}
+
+	*mount_path = '\0';
+	mount_path++;
+
+	if (!*mount_path) {
+		printf("incorrect source mount path\n");
+		free(buf);
+		return NULL;
+	}
+
+	src = resolve_addrs(buf);
+	if (!src)
+		return NULL;
+
+	len = strlen(src);
+	pos = safe_cat(&src, &len, len, ":");
+	safe_cat(&src, &len, pos, mount_path);
+
+	free(buf);
+	return src;
+}
+
+/*
+ * this one is partialy based on parse_options() from cifs.mount.c
+ */
+static char *parse_options(const char *data, int *filesys_flags)
+{
+	char * next_keyword = NULL;
+	char * out = NULL;
+	int out_len = 0;
+	int pos = 0;
+	char *name = NULL;
+	int name_len = 0;
+	int name_pos = 0;
+	char secret[MAX_SECRET_LEN];
+	char *saw_name = NULL;
+	char *saw_secret = NULL;
+
+	if(verboseflag)
+		printf("parsing options: %s\n", data);
+
+	do {
+		char * value = NULL;
+		/*  check if ends with trailing comma */
+		if(*data == 0)
+			break;
+		next_keyword = strchr(data,',');
+	
+		/* temporarily null terminate end of keyword=value pair */
+		if(next_keyword)
+			*next_keyword++ = 0;
+
+		/* temporarily null terminate keyword to make keyword and value distinct */
+		if ((value = strchr(data, '=')) != NULL) {
+			*value = '\0';
+			value++;
+		}
+
+		int skip = 1;
+
+		if (strncmp(data, "ro", 2) == 0) {
+			*filesys_flags |= MS_RDONLY;
+		} else if (strncmp(data, "rw", 2) == 0) {
+			*filesys_flags &= ~MS_RDONLY;
+		} else if (strncmp(data, "nosuid", 6) == 0) {
+			*filesys_flags |= MS_NOSUID;
+		} else if (strncmp(data, "suid", 4) == 0) {
+			*filesys_flags &= ~MS_NOSUID;
+		} else if (strncmp(data, "dev", 3) == 0) {
+			*filesys_flags &= ~MS_NODEV;
+		} else if (strncmp(data, "nodev", 5) == 0) {
+			*filesys_flags |= MS_NODEV;
+		} else if (strncmp(data, "noexec", 6) == 0) {
+			*filesys_flags |= MS_NOEXEC;
+		} else if (strncmp(data, "exec", 4) == 0) {
+			*filesys_flags &= ~MS_NOEXEC;
+                } else if (strncmp(data, "sync", 4) == 0) {
+                        *filesys_flags |= MS_SYNCHRONOUS;
+                } else if (strncmp(data, "remount", 7) == 0) {
+                        *filesys_flags |= MS_REMOUNT;
+                } else if (strncmp(data, "mandlock", 8) == 0) {
+                        *filesys_flags |= MS_MANDLOCK;
+		} else if ((strncmp(data, "nobrl", 5) == 0) || 
+			   (strncmp(data, "nolock", 6) == 0)) {
+			*filesys_flags &= ~MS_MANDLOCK;
+		} else if (strncmp(data, "noatime", 7) == 0) {
+			*filesys_flags |= MS_NOATIME;
+		} else if (strncmp(data, "nodiratime", 10) == 0) {
+			*filesys_flags |= MS_NODIRATIME;
+		} else if (strncmp(data, "relatime", 8) == 0) {
+			*filesys_flags |= MS_RELATIME;
+
+		} else if (strncmp(data, "noauto", 6) == 0) {
+			skip = 1;  /* ignore */
+		} else if (strncmp(data, "_netdev", 7) == 0) {
+			skip = 1;  /* ignore */
+
+		} else if (strncmp(data, "secretfile", 10) == 0) {
+			if (!value || !*value) {
+				printf("keyword secretfile found, but no secret file specified\n");
+				return NULL;
+			}
+
+			if (read_secret_from_file(value, secret, sizeof(secret)) < 0) {
+				printf("error reading secret file\n");
+				return NULL;
+			}
+
+			/* see comment for "secret" */
+			saw_secret = secret;
+			skip = 1;
+		} else if (strncmp(data, "secret", 6) == 0) {
+			if (!value || !*value) {
+				printf("mount option secret requires a value.\n");
+				return NULL;
+			}
+
+			/* secret is only added to kernel options as
+			   backwards compatilbity, if add_key doesn't
+			   recognize our keytype; hence, it is skipped
+			   here and appended to options on add_key
+			   failure */
+			strncpy(secret, value, sizeof(secret));
+			saw_secret = secret;
+			skip = 1;
+		} else if (strncmp(data, "name", 4) == 0) {
+			if (!value || !*value) {
+				printf("mount option name requires a value.\n");
+				return NULL;
+			}
+
+			/* take a copy of the name, to be used for
+			   naming the keys that we add to kernel;
+			   ignore memleak as mount.ceph is
+			   short-lived */
+			saw_name = strdup(value);
+			if (!saw_name) {
+				printf("out of memory.\n");
+				return NULL;
+			}
+			skip = 0;
+		} else {
+			skip = 0;
+			if (verboseflag)
+				printf("ceph: Unknown mount option %s\n",data);
+		}
+
+		/* Copy (possibly modified) option to out */
+		if (!skip) {
+			if (pos)
+				pos = safe_cat(&out, &out_len, pos, ",");
+
+			if (value) {
+				pos = safe_cat(&out, &out_len, pos, data);
+				pos = safe_cat(&out, &out_len, pos, "=");
+				pos = safe_cat(&out, &out_len, pos, value);
+			} else {
+				pos = safe_cat(&out, &out_len, pos, data);
+			}
+			
+		}
+		data = next_keyword;
+	} while (data);
+
+	name_pos = safe_cat(&name, &name_len, name_pos, "client.");
+	if (!saw_name) {
+		name_pos = safe_cat(&name, &name_len, name_pos, CEPH_AUTH_NAME_DEFAULT);
+	} else {
+		name_pos = safe_cat(&name, &name_len, name_pos, saw_name);
+	}
+	if (saw_secret || is_kernel_secret(name)) {
+		int ret;
+		char secret_option[MAX_SECRET_OPTION_LEN];
+		ret = get_secret_option(saw_secret, name, secret_option, sizeof(secret_option));
+		if (ret < 0) {
+			return NULL;
+		} else {
+			if (pos) {
+				pos = safe_cat(&out, &out_len, pos, ",");
+			}
+			pos = safe_cat(&out, &out_len, pos, secret_option);
+		}
+	}
+
+	if (!out)
+		return strdup(EMPTY_STRING);
+	return out;
+}
+
+
+static int parse_arguments(int argc, char *const *const argv,
+		const char **src, const char **node, const char **opts)
+{
+	int i;
+
+	if (argc < 2) {
+		// There were no arguments. Just show the usage.
+		return 1;
+	}
+	if ((!strcmp(argv[1], "-h")) || (!strcmp(argv[1], "--help"))) {
+		// The user asked for help.
+		return 1;
+	}
+
+	// The first two arguments are positional
+	if (argc < 3)
+		return -EINVAL;
+	*src = argv[1];
+	*node = argv[2];
+
+	// Parse the remaining options
+	*opts = EMPTY_STRING;
+	for (i = 3; i < argc; ++i) {
+		if (!strcmp("-h", argv[i]))
+			return 1;
+		else if (!strcmp("-n", argv[i]))
+			skip_mtab_flag = 1;
+		else if (!strcmp("-v", argv[i]))
+			verboseflag = 1;
+		else if (!strcmp("-o", argv[i])) {
+			++i;
+			if (i >= argc) {
+				printf("Option -o requires an argument.\n\n");
+				return -EINVAL;
+			}
+			*opts = argv[i];
+		}
+		else {
+			printf("Can't understand option: '%s'\n\n", argv[i]);
+			return -EINVAL;
+		}
+	}
+	return 0;
+}
+
+/* modprobe failing doesn't necessarily prevent from working, so this
+   returns void */
+static void modprobe(void) {
+	int status;
+	status = system("/sbin/modprobe ceph");
+	if (status < 0) {
+		char error_buf[80];
+		fprintf(stderr, "mount.ceph: cannot run modprobe: %s\n",
+				strerror_r(errno, error_buf, sizeof(error_buf)));
+	} else if (WIFEXITED(status)) {
+		status = WEXITSTATUS(status);
+		if (status != 0) {
+			fprintf(stderr,
+				"mount.ceph: modprobe failed, exit status %d\n",
+				status);
+		}
+	} else if (WIFSIGNALED(status)) {
+		fprintf(stderr,
+			"mount.ceph: modprobe failed with signal %d\n",
+			WTERMSIG(status));
+	} else {
+		fprintf(stderr, "mount.ceph: weird status from modprobe: %d\n",
+			status);
+	}
+}
+
+static void usage(const char *prog_name)
+{
+	printf("usage: %s [src] [mount-point] [-n] [-v] [-o ceph-options]\n",
+		prog_name);
+	printf("options:\n");
+	printf("\t-h: Print this help\n");
+	printf("\t-n: Do not update /etc/mtab\n");
+	printf("\t-v: Verbose\n");
+	printf("\tceph-options: refer to mount.ceph(8)\n");
+	printf("\n");
+}
+
+int main(int argc, char *argv[])
+{
+	const char *src, *node, *opts;
+	char *rsrc = NULL;
+	char *popts = NULL;
+	int flags = 0;
+	int retval = 0;
+
+	retval = parse_arguments(argc, argv, &src, &node, &opts);
+	if (retval) {
+		usage(argv[0]);
+		exit((retval > 0) ? EXIT_SUCCESS : EXIT_FAILURE);
+	}
+
+	rsrc = mount_resolve_src(src);
+	if (!rsrc) {
+		printf("failed to resolve source\n");
+		exit(1);
+	}
+
+	modprobe();
+
+	popts = parse_options(opts, &flags);
+	if (!popts) {
+		printf("failed to parse ceph_options\n");
+		exit(1);
+	}
+
+	block_signals(SIG_BLOCK);
+
+	if (mount(rsrc, node, "ceph", flags, popts)) {
+		retval = errno;
+		switch (errno) {
+		case ENODEV:
+			printf("mount error: ceph filesystem not supported by the system\n");
+			break;
+		default:
+			printf("mount error %d = %s\n",errno,strerror(errno));
+		}
+	} else {
+		if (!skip_mtab_flag) {
+			update_mtab_entry(rsrc, node, "ceph", popts, flags, 0, 0);
+		}
+	}
+
+	block_signals(SIG_UNBLOCK);
+
+	free(popts);
+	free(rsrc);
+	exit(retval);
+}
+
diff --git a/ceph/src/mount/mtab.c b/ceph/src/mount/mtab.c
new file mode 100644
index 00000000..9b2c9237
--- /dev/null
+++ b/ceph/src/mount/mtab.c
@@ -0,0 +1,280 @@
+
+/*
+ * this code lifted from util-linux-ng, licensed GPLv2+,
+ *
+ *  git://git.kernel.org/pub/scm/utils/util-linux-ng/util-linux-ng.git
+ *
+ * whoever decided that each special mount program is responsible
+ * for updating /etc/mtab should be spanked.
+ *
+ * 
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "mount/canonicalize.c"
+
+
+/* Updating mtab ----------------------------------------------*/
+
+/* Flag for already existing lock file. */
+static int we_created_lockfile = 0;
+static int lockfile_fd = -1;
+
+/* Flag to indicate that signals have been set up. */
+static int signals_have_been_setup = 0;
+
+/* Ensure that the lock is released if we are interrupted.  */
+extern char *strsignal(int sig);	/* not always in  */
+
+static void
+setlkw_timeout (int sig) {
+     /* nothing, fcntl will fail anyway */
+}
+
+#define _PATH_MOUNTED "/etc/mtab"
+#define _PATH_MOUNTED_LOCK "/etc/mtab~"
+
+/* exit status - bits below are ORed */
+#define EX_USAGE        1       /* incorrect invocation or permission */
+#define EX_SYSERR       2       /* out of memory, cannot fork, ... */
+#define EX_SOFTWARE     4       /* internal mount bug or wrong version */
+#define EX_USER         8       /* user interrupt */
+#define EX_FILEIO      16       /* problems writing, locking, ... mtab/fstab */
+#define EX_FAIL        32       /* mount failure */
+#define EX_SOMEOK      64       /* some mount succeeded */
+
+int die(int err, const char *fmt, ...) {
+        va_list args;
+
+        va_start(args, fmt);
+        vfprintf(stderr, fmt, args);
+        fprintf(stderr, "\n");
+        va_end(args);
+
+        exit(err);
+}
+
+static void
+handler (int sig) {
+	die(EX_USER, "%s", strsignal(sig));
+}
+
+/* Remove lock file.  */
+void
+unlock_mtab (void) {
+	if (we_created_lockfile) {
+		close(lockfile_fd);
+		lockfile_fd = -1;
+		unlink (_PATH_MOUNTED_LOCK);
+		we_created_lockfile = 0;
+	}
+}
+
+/* Create the lock file.
+   The lock file will be removed if we catch a signal or when we exit. */
+/* The old code here used flock on a lock file /etc/mtab~ and deleted
+   this lock file afterwards. However, as rgooch remarks, that has a
+   race: a second mount may be waiting on the lock and proceed as
+   soon as the lock file is deleted by the first mount, and immediately
+   afterwards a third mount comes, creates a new /etc/mtab~, applies
+   flock to that, and also proceeds, so that the second and third mount
+   now both are scribbling in /etc/mtab.
+   The new code uses a link() instead of a creat(), where we proceed
+   only if it was us that created the lock, and hence we always have
+   to delete the lock afterwards. Now the use of flock() is in principle
+   superfluous, but avoids an arbitrary sleep(). */
+
+/* Where does the link point to? Obvious choices are mtab and mtab~~.
+   HJLu points out that the latter leads to races. Right now we use
+   mtab~. instead. Use 20 as upper bound for the length of %d. */
+#define MOUNTLOCK_LINKTARGET		_PATH_MOUNTED_LOCK "%d"
+#define MOUNTLOCK_LINKTARGET_LTH	(sizeof(_PATH_MOUNTED_LOCK)+20)
+
+/*
+ * The original mount locking code has used sleep(1) between attempts and
+ * maximal number of attemps has been 5.
+ *
+ * There was very small number of attempts and extremely long waiting (1s)
+ * that is useless on machines with large number of concurret mount processes.
+ *
+ * Now we wait few thousand microseconds between attempts and we have global
+ * time limit (30s) rather than limit for number of attempts. The advantage
+ * is that this method also counts time which we spend in fcntl(F_SETLKW) and
+ * number of attempts is not so much restricted.
+ *
+ * -- kzak@redhat.com [2007-Mar-2007]
+ */
+
+/* maximum seconds between first and last attempt */
+#define MOUNTLOCK_MAXTIME		30
+
+/* sleep time (in microseconds, max=999999) between attempts */
+#define MOUNTLOCK_WAITTIME		5000
+
+void
+lock_mtab (void) {
+	int i;
+	struct timespec waittime;
+	struct timeval maxtime;
+	char linktargetfile[MOUNTLOCK_LINKTARGET_LTH];
+
+	if (!signals_have_been_setup) {
+		int sig = 0;
+		struct sigaction sa;
+
+		sa.sa_handler = handler;
+		sa.sa_flags = 0;
+		sigfillset (&sa.sa_mask);
+
+		while (sigismember (&sa.sa_mask, ++sig) != -1
+		       && sig != SIGCHLD) {
+			if (sig == SIGALRM)
+				sa.sa_handler = setlkw_timeout;
+			else
+				sa.sa_handler = handler;
+			sigaction (sig, &sa, (struct sigaction *) 0);
+		}
+		signals_have_been_setup = 1;
+	}
+
+	snprintf(linktargetfile, sizeof(linktargetfile), MOUNTLOCK_LINKTARGET,
+		 getpid ());
+
+	i = open (linktargetfile, O_WRONLY|O_CREAT, S_IRUSR|S_IWUSR);
+	if (i < 0) {
+		int errsv = errno;
+		/* linktargetfile does not exist (as a file)
+		   and we cannot create it. Read-only filesystem?
+		   Too many files open in the system?
+		   Filesystem full? */
+		die (EX_FILEIO, "can't create lock file %s: %s "
+		     "(use -n flag to override)",
+		     linktargetfile, strerror (errsv));
+	}
+	close(i);
+
+	gettimeofday(&maxtime, NULL);
+	maxtime.tv_sec += MOUNTLOCK_MAXTIME;
+
+	waittime.tv_sec = 0;
+	waittime.tv_nsec = (1000 * MOUNTLOCK_WAITTIME);
+
+	/* Repeat until it was us who made the link */
+	while (!we_created_lockfile) {
+		struct timeval now;
+		struct flock flock;
+		int errsv, j;
+
+		j = link(linktargetfile, _PATH_MOUNTED_LOCK);
+		errsv = errno;
+
+		if (j == 0)
+			we_created_lockfile = 1;
+
+		if (j < 0 && errsv != EEXIST) {
+			(void) unlink(linktargetfile);
+			die (EX_FILEIO, "can't link lock file %s: %s "
+			     "(use -n flag to override)",
+			     _PATH_MOUNTED_LOCK, strerror (errsv));
+		}
+
+		lockfile_fd = open (_PATH_MOUNTED_LOCK, O_WRONLY);
+
+		if (lockfile_fd < 0) {
+			/* Strange... Maybe the file was just deleted? */
+			int errsv = errno;
+			gettimeofday(&now, NULL);
+			if (errno == ENOENT && now.tv_sec < maxtime.tv_sec) {
+				we_created_lockfile = 0;
+				continue;
+			}
+			(void) unlink(linktargetfile);
+			die (EX_FILEIO, "can't open lock file %s: %s "
+			     "(use -n flag to override)",
+			     _PATH_MOUNTED_LOCK, strerror (errsv));
+		}
+
+		flock.l_type = F_WRLCK;
+		flock.l_whence = SEEK_SET;
+		flock.l_start = 0;
+		flock.l_len = 0;
+
+		if (j == 0) {
+			/* We made the link. Now claim the lock. */
+			if (fcntl (lockfile_fd, F_SETLK, &flock) == -1) {
+				/* proceed, since it was us who created the lockfile anyway */
+			}
+			(void) unlink(linktargetfile);
+		} else {
+			/* Someone else made the link. Wait. */
+			gettimeofday(&now, NULL);
+			if (now.tv_sec < maxtime.tv_sec) {
+				alarm(maxtime.tv_sec - now.tv_sec);
+				if (fcntl (lockfile_fd, F_SETLKW, &flock) == -1) {
+					int errsv = errno;
+					(void) unlink(linktargetfile);
+					die (EX_FILEIO, "can't lock lock file %s: %s",
+					     _PATH_MOUNTED_LOCK, (errno == EINTR) ?
+					     "timed out" : strerror (errsv));
+				}
+				alarm(0);
+
+				nanosleep(&waittime, NULL);
+			} else {
+				(void) unlink(linktargetfile);
+				die (EX_FILEIO, "Cannot create link %s\n"
+				     "Perhaps there is a stale lock file?\n",
+					 _PATH_MOUNTED_LOCK);
+			}
+			close(lockfile_fd);
+		}
+	}
+}
+
+static void
+update_mtab_entry(const char *spec, const char *node, const char *type,
+		  const char *opts, int flags, int freq, int pass) {
+	struct mntent mnt;
+
+	if (!opts)
+		opts = "rw";
+
+	mnt.mnt_fsname = strdup(spec);
+	mnt.mnt_dir = canonicalize_path(node);
+	mnt.mnt_type = strdup(type);
+	mnt.mnt_opts = strdup(opts);
+	mnt.mnt_freq = freq;
+	mnt.mnt_passno = pass;
+
+	FILE *fp;
+	
+	lock_mtab();
+	fp = setmntent(_PATH_MOUNTED, "a+");
+	if (fp == NULL) {
+		int errsv = errno;
+		printf("mount: can't open %s: %s", _PATH_MOUNTED,
+		       strerror (errsv));
+	} else {
+		if ((addmntent (fp, &mnt)) == 1) {
+			int errsv = errno;
+			printf("mount: error writing %s: %s",
+			      _PATH_MOUNTED, strerror (errsv));
+		}
+	}
+	endmntent(fp);
+	unlock_mtab();
+
+	free(mnt.mnt_fsname);
+	free(mnt.mnt_dir);
+}
diff --git a/ceph/src/msg/Accepter.cc b/ceph/src/msg/Accepter.cc
new file mode 100644
index 00000000..718d4785
--- /dev/null
+++ b/ceph/src/msg/Accepter.cc
@@ -0,0 +1,258 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "Accepter.h"
+#include "SimpleMessenger.h"
+
+#include "Message.h"
+#include "Pipe.h"
+
+#include "common/debug.h"
+#include "common/errno.h"
+
+#define dout_subsys ceph_subsys_ms
+
+#undef dout_prefix
+#define dout_prefix *_dout << "accepter."
+
+
+/********************************************
+ * Accepter
+ */
+
+int Accepter::bind(const entity_addr_t &bind_addr, const set& avoid_ports)
+{
+  const md_config_t *conf = msgr->cct->_conf;
+  // bind to a socket
+  ldout(msgr->cct,10) << "accepter.bind" << dendl;
+  
+  int family;
+  switch (bind_addr.get_family()) {
+  case AF_INET:
+  case AF_INET6:
+    family = bind_addr.get_family();
+    break;
+
+  default:
+    // bind_addr is empty
+    family = conf->ms_bind_ipv6 ? AF_INET6 : AF_INET;
+  }
+
+  /* socket creation */
+  listen_sd = ::socket(family, SOCK_STREAM, 0);
+  if (listen_sd < 0) {
+    lderr(msgr->cct) << "accepter.bind unable to create socket: "
+		     << cpp_strerror(errno) << dendl;
+    return -errno;
+  }
+
+  // use whatever user specified (if anything)
+  entity_addr_t listen_addr = bind_addr;
+  listen_addr.set_family(family);
+
+  /* bind to port */
+  int rc = -1;
+  if (listen_addr.get_port()) {
+    // specific port
+
+    // reuse addr+port when possible
+    int on = 1;
+    rc = ::setsockopt(listen_sd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+    if (rc < 0) {
+      lderr(msgr->cct) << "accepter.bind unable to setsockopt: "
+			 << cpp_strerror(errno) << dendl;
+      return -errno;
+    }
+
+    rc = ::bind(listen_sd, (struct sockaddr *) &listen_addr.ss_addr(), listen_addr.addr_size());
+    if (rc < 0) {
+      lderr(msgr->cct) << "accepter.bind unable to bind to " << listen_addr.ss_addr()
+		       << ": " << cpp_strerror(errno) << dendl;
+      return -errno;
+    }
+  } else {
+    // try a range of ports
+    for (int port = msgr->cct->_conf->ms_bind_port_min; port <= msgr->cct->_conf->ms_bind_port_max; port++) {
+      if (avoid_ports.count(port))
+	continue;
+      listen_addr.set_port(port);
+      rc = ::bind(listen_sd, (struct sockaddr *) &listen_addr.ss_addr(), listen_addr.addr_size());
+      if (rc == 0)
+	break;
+    }
+    if (rc < 0) {
+      lderr(msgr->cct) << "accepter.bind unable to bind to " << listen_addr.ss_addr()
+		       << " on any port in range " << msgr->cct->_conf->ms_bind_port_min
+		       << "-" << msgr->cct->_conf->ms_bind_port_max
+		       << ": " << cpp_strerror(errno)
+		       << dendl;
+      return -errno;
+    }
+    ldout(msgr->cct,10) << "accepter.bind bound on random port " << listen_addr << dendl;
+  }
+
+  // what port did we get?
+  socklen_t llen = sizeof(listen_addr.ss_addr());
+  rc = getsockname(listen_sd, (sockaddr*)&listen_addr.ss_addr(), &llen);
+  if (rc < 0) {
+    rc = -errno;
+    lderr(msgr->cct) << "accepter.bind failed getsockname: " << cpp_strerror(rc) << dendl;
+    return rc;
+  }
+  
+  ldout(msgr->cct,10) << "accepter.bind bound to " << listen_addr << dendl;
+
+  // listen!
+  rc = ::listen(listen_sd, 128);
+  if (rc < 0) {
+    rc = -errno;
+    lderr(msgr->cct) << "accepter.bind unable to listen on " << listen_addr
+		     << ": " << cpp_strerror(rc) << dendl;
+    return rc;
+  }
+  
+  msgr->set_myaddr(bind_addr);
+  if (bind_addr != entity_addr_t())
+    msgr->learned_addr(bind_addr);
+  else
+    assert(msgr->get_need_addr());  // should still be true.
+
+  if (msgr->get_myaddr().get_port() == 0) {
+    msgr->set_myaddr(listen_addr);
+  }
+  entity_addr_t addr = msgr->get_myaddr();
+  addr.nonce = nonce;
+  msgr->set_myaddr(addr);
+
+  msgr->init_local_connection();
+
+  ldout(msgr->cct,1) << "accepter.bind my_inst.addr is " << msgr->get_myaddr()
+		     << " need_addr=" << msgr->get_need_addr() << dendl;
+  return 0;
+}
+
+int Accepter::rebind(const set& avoid_ports)
+{
+  ldout(msgr->cct,1) << "accepter.rebind avoid " << avoid_ports << dendl;
+  
+  // invalidate our previously learned address.
+  msgr->unlearn_addr();
+
+  entity_addr_t addr = msgr->get_myaddr();
+  set new_avoid = avoid_ports;
+  new_avoid.insert(addr.get_port());
+  addr.set_port(0);
+
+  // adjust the nonce; we want our entity_addr_t to be truly unique.
+  nonce += 1000000;
+  msgr->my_inst.addr.nonce = nonce;
+  ldout(msgr->cct,10) << " new nonce " << nonce << " and inst " << msgr->my_inst << dendl;
+
+  ldout(msgr->cct,10) << " will try " << addr << " and avoid ports " << new_avoid << dendl;
+  int r = bind(addr, new_avoid);
+  if (r == 0)
+    start();
+  return r;
+}
+
+int Accepter::start()
+{
+  ldout(msgr->cct,1) << "accepter.start" << dendl;
+
+  // start thread
+  create();
+
+  return 0;
+}
+
+void *Accepter::entry()
+{
+  ldout(msgr->cct,10) << "accepter starting" << dendl;
+  
+  int errors = 0;
+
+  struct pollfd pfd;
+  pfd.fd = listen_sd;
+  pfd.events = POLLIN | POLLERR | POLLNVAL | POLLHUP;
+  while (!done) {
+    ldout(msgr->cct,20) << "accepter calling poll" << dendl;
+    int r = poll(&pfd, 1, -1);
+    if (r < 0)
+      break;
+    ldout(msgr->cct,20) << "accepter poll got " << r << dendl;
+
+    if (pfd.revents & (POLLERR | POLLNVAL | POLLHUP))
+      break;
+
+    ldout(msgr->cct,10) << "pfd.revents=" << pfd.revents << dendl;
+    if (done) break;
+
+    // accept
+    entity_addr_t addr;
+    socklen_t slen = sizeof(addr.ss_addr());
+    int sd = ::accept(listen_sd, (sockaddr*)&addr.ss_addr(), &slen);
+    if (sd >= 0) {
+      errors = 0;
+      ldout(msgr->cct,10) << "accepted incoming on sd " << sd << dendl;
+      
+      msgr->add_accept_pipe(sd);
+    } else {
+      ldout(msgr->cct,0) << "accepter no incoming connection?  sd = " << sd
+	      << " errno " << errno << " " << cpp_strerror(errno) << dendl;
+      if (++errors > 4)
+	break;
+    }
+  }
+
+  ldout(msgr->cct,20) << "accepter closing" << dendl;
+  // don't close socket, in case we start up again?  blech.
+  if (listen_sd >= 0) {
+    ::close(listen_sd);
+    listen_sd = -1;
+  }
+  ldout(msgr->cct,10) << "accepter stopping" << dendl;
+  return 0;
+}
+
+void Accepter::stop()
+{
+  done = true;
+  ldout(msgr->cct,10) << "stop accepter" << dendl;
+
+  if (listen_sd >= 0) {
+    ::shutdown(listen_sd, SHUT_RDWR);
+  }
+
+  // wait for thread to stop before closing the socket, to avoid
+  // racing against fd re-use.
+  if (is_started()) {
+    join();
+  }
+
+  if (listen_sd >= 0) {
+    ::close(listen_sd);
+    listen_sd = -1;
+  }
+  done = false;
+}
+
+
+
+
diff --git a/ceph/src/msg/Accepter.h b/ceph/src/msg/Accepter.h
new file mode 100644
index 00000000..4b1421f9
--- /dev/null
+++ b/ceph/src/msg/Accepter.h
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MSG_ACCEPTER_H
+#define CEPH_MSG_ACCEPTER_H
+
+#include "msg/msg_types.h"
+#include "common/Thread.h"
+
+class SimpleMessenger;
+
+/**
+ * If the SimpleMessenger binds to a specific address, the Accepter runs
+ * and listens for incoming connections.
+ */
+class Accepter : public Thread {
+  SimpleMessenger *msgr;
+  bool done;
+  int listen_sd;
+  uint64_t nonce;
+
+public:
+  Accepter(SimpleMessenger *r, uint64_t n) : msgr(r), done(false), listen_sd(-1), nonce(n) {}
+    
+  void *entry();
+  void stop();
+  int bind(const entity_addr_t &bind_addr, const set& avoid_ports);
+  int rebind(const set& avoid_port);
+  int start();
+};
+
+
+#endif
diff --git a/ceph/src/msg/DispatchQueue.cc b/ceph/src/msg/DispatchQueue.cc
new file mode 100644
index 00000000..8a79ce0d
--- /dev/null
+++ b/ceph/src/msg/DispatchQueue.cc
@@ -0,0 +1,177 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "msg/Message.h"
+#include "DispatchQueue.h"
+#include "SimpleMessenger.h"
+#include "common/ceph_context.h"
+
+#define dout_subsys ceph_subsys_ms
+#include "common/debug.h"
+
+
+/*******************
+ * DispatchQueue
+ */
+
+#undef dout_prefix
+#define dout_prefix *_dout << "-- " << msgr->get_myaddr() << " "
+
+double DispatchQueue::get_max_age(utime_t now) {
+  Mutex::Locker l(lock);
+  if (marrival.empty())
+    return 0;
+  else
+    return (now - marrival.begin()->first);
+}
+
+void DispatchQueue::enqueue(Message *m, int priority, uint64_t id)
+{
+  Mutex::Locker l(lock);
+  ldout(cct,20) << "queue " << m << " prio " << priority << dendl;
+  add_arrival(m);
+  if (priority >= CEPH_MSG_PRIO_LOW) {
+    mqueue.enqueue_strict(
+      id, priority, QueueItem(m));
+  } else {
+    mqueue.enqueue(
+      id, priority, m->get_cost(), QueueItem(m));
+  }
+  cond.Signal();
+}
+
+void DispatchQueue::local_delivery(Message *m, int priority)
+{
+  Mutex::Locker l(lock);
+  m->set_connection(msgr->local_connection.get());
+  m->set_recv_stamp(ceph_clock_now(msgr->cct));
+  add_arrival(m);
+  if (priority >= CEPH_MSG_PRIO_LOW) {
+    mqueue.enqueue_strict(
+      0, priority, QueueItem(m));
+  } else {
+    mqueue.enqueue(
+      0, priority, m->get_cost(), QueueItem(m));
+  }
+  cond.Signal();
+}
+
+/*
+ * This function delivers incoming messages to the Messenger.
+ * Pipes with messages are kept in queues; when beginning a message
+ * delivery the highest-priority queue is selected, the pipe from the
+ * front of the queue is removed, and its message read. If the pipe
+ * has remaining messages at that priority level, it is re-placed on to the
+ * end of the queue. If the queue is empty; it's removed.
+ * The message is then delivered and the process starts again.
+ */
+void DispatchQueue::entry()
+{
+  lock.Lock();
+  while (true) {
+    while (!mqueue.empty()) {
+      QueueItem qitem = mqueue.dequeue();
+      if (!qitem.is_code())
+	remove_arrival(qitem.get_message());
+      lock.Unlock();
+
+      if (qitem.is_code()) {
+	switch (qitem.get_code()) {
+	case D_BAD_REMOTE_RESET:
+	  msgr->ms_deliver_handle_remote_reset(qitem.get_connection());
+	  break;
+	case D_CONNECT:
+	  msgr->ms_deliver_handle_connect(qitem.get_connection());
+	  break;
+	case D_ACCEPT:
+	  msgr->ms_deliver_handle_accept(qitem.get_connection());
+	  break;
+	case D_BAD_RESET:
+	  msgr->ms_deliver_handle_reset(qitem.get_connection());
+	  break;
+	default:
+	  assert(0);
+	}
+      } else {
+	Message *m = qitem.get_message();
+	if (stop) {
+	  ldout(cct,10) << " stop flag set, discarding " << m << " " << *m << dendl;
+	  m->put();
+	} else {
+	  uint64_t msize = m->get_dispatch_throttle_size();
+	  m->set_dispatch_throttle_size(0);  // clear it out, in case we requeue this message.
+
+	  ldout(cct,1) << "<== " << m->get_source_inst()
+		       << " " << m->get_seq()
+		       << " ==== " << *m
+		       << " ==== " << m->get_payload().length() << "+" << m->get_middle().length()
+		       << "+" << m->get_data().length()
+		       << " (" << m->get_footer().front_crc << " " << m->get_footer().middle_crc
+		       << " " << m->get_footer().data_crc << ")"
+		       << " " << m << " con " << m->get_connection()
+		       << dendl;
+	  msgr->ms_deliver_dispatch(m);
+
+	  msgr->dispatch_throttle_release(msize);
+
+	  ldout(cct,20) << "done calling dispatch on " << m << dendl;
+	}
+      }
+
+      lock.Lock();
+    }
+    if (stop)
+      break;
+
+    // wait for something to be put on queue
+    cond.Wait(lock);
+  }
+  lock.Unlock();
+}
+
+void DispatchQueue::discard_queue(uint64_t id) {
+  Mutex::Locker l(lock);
+  list removed;
+  mqueue.remove_by_class(id, &removed);
+  for (list::iterator i = removed.begin();
+       i != removed.end();
+       ++i) {
+    assert(!(i->is_code())); // We don't discard id 0, ever!
+    Message *m = i->get_message();
+    remove_arrival(m);
+    msgr->dispatch_throttle_release(m->get_dispatch_throttle_size());
+    m->put();
+  }
+}
+
+void DispatchQueue::start()
+{
+  assert(!stop);
+  assert(!dispatch_thread.is_started());
+  dispatch_thread.create();
+}
+
+void DispatchQueue::wait()
+{
+  dispatch_thread.join();
+}
+
+void DispatchQueue::shutdown()
+{
+  // stop my dispatch thread
+  lock.Lock();
+  stop = true;
+  cond.Signal();
+  lock.Unlock();
+}
diff --git a/ceph/src/msg/DispatchQueue.h b/ceph/src/msg/DispatchQueue.h
new file mode 100644
index 00000000..dffd65cc
--- /dev/null
+++ b/ceph/src/msg/DispatchQueue.h
@@ -0,0 +1,184 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_DISPATCHQUEUE_H
+#define CEPH_DISPATCHQUEUE_H
+
+#include 
+#include 
+#include "include/assert.h"
+#include "include/xlist.h"
+#include "include/atomic.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/Thread.h"
+#include "common/RefCountedObj.h"
+#include "common/PrioritizedQueue.h"
+
+class CephContext;
+class DispatchQueue;
+class Pipe;
+class SimpleMessenger;
+class Message;
+struct Connection;
+
+/**
+ * The DispatchQueue contains all the Pipes which have Messages
+ * they want to be dispatched, carefully organized by Message priority
+ * and permitted to deliver in a round-robin fashion.
+ * See SimpleMessenger::dispatch_entry for details.
+ */
+class DispatchQueue {
+  class QueueItem {
+    int type;
+    ConnectionRef con;
+    MessageRef m;
+  public:
+    QueueItem(Message *m) : type(-1), con(0), m(m) {}
+    QueueItem(int type, Connection *con) : type(type), con(con), m(0) {}
+    bool is_code() const {
+      return type != -1;
+    }
+    int get_code () {
+      assert(is_code());
+      return type;
+    }
+    Message *get_message() {
+      assert(!is_code());
+      return m.get();
+    }
+    Connection *get_connection() {
+      assert(is_code());
+      return con.get();
+    }
+  };
+    
+  CephContext *cct;
+  SimpleMessenger *msgr;
+  Mutex lock;
+  Cond cond;
+
+  PrioritizedQueue mqueue;
+
+  set > marrival;
+  map >::iterator> marrival_map;
+  void add_arrival(Message *m) {
+    marrival_map.insert(
+      make_pair(
+	m,
+	marrival.insert(make_pair(m->get_recv_stamp(), m)).first
+	)
+      );
+  }
+  void remove_arrival(Message *m) {
+    map >::iterator>::iterator i =
+      marrival_map.find(m);
+    assert(i != marrival_map.end());
+    marrival.erase(i->second);
+    marrival_map.erase(i);
+  }
+
+  uint64_t next_pipe_id;
+    
+  enum { D_CONNECT = 1, D_ACCEPT, D_BAD_REMOTE_RESET, D_BAD_RESET, D_NUM_CODES };
+
+  /**
+   * The DispatchThread runs dispatch_entry to empty out the dispatch_queue.
+   */
+  class DispatchThread : public Thread {
+    DispatchQueue *dq;
+  public:
+    DispatchThread(DispatchQueue *dq) : dq(dq) {}
+    void *entry() {
+      dq->entry();
+      return 0;
+    }
+  } dispatch_thread;
+
+  public:
+  bool stop;
+  void local_delivery(Message *m, int priority);
+
+  double get_max_age(utime_t now);
+
+  int get_queue_len() {
+    Mutex::Locker l(lock);
+    return mqueue.length();
+  }
+    
+  void queue_connect(Connection *con) {
+    Mutex::Locker l(lock);
+    if (stop)
+      return;
+    mqueue.enqueue_strict(
+      0,
+      CEPH_MSG_PRIO_HIGHEST,
+      QueueItem(D_CONNECT, con));
+    cond.Signal();
+  }
+  void queue_accept(Connection *con) {
+    Mutex::Locker l(lock);
+    if (stop)
+      return;
+    mqueue.enqueue_strict(
+      0,
+      CEPH_MSG_PRIO_HIGHEST,
+      QueueItem(D_ACCEPT, con));
+    cond.Signal();
+  }
+  void queue_remote_reset(Connection *con) {
+    Mutex::Locker l(lock);
+    if (stop)
+      return;
+    mqueue.enqueue_strict(
+      0,
+      CEPH_MSG_PRIO_HIGHEST,
+      QueueItem(D_BAD_REMOTE_RESET, con));
+    cond.Signal();
+  }
+  void queue_reset(Connection *con) {
+    Mutex::Locker l(lock);
+    if (stop)
+      return;
+    mqueue.enqueue_strict(
+      0,
+      CEPH_MSG_PRIO_HIGHEST,
+      QueueItem(D_BAD_RESET, con));
+    cond.Signal();
+  }
+
+  void enqueue(Message *m, int priority, uint64_t id);
+  void discard_queue(uint64_t id);
+  uint64_t get_id() {
+    Mutex::Locker l(lock);
+    return next_pipe_id++;
+  }
+  void start();
+  void entry();
+  void wait();
+  void shutdown();
+  bool is_started() {return dispatch_thread.is_started();}
+
+  DispatchQueue(CephContext *cct, SimpleMessenger *msgr)
+    : cct(cct), msgr(msgr),
+      lock("SimpleMessenger::DispatchQeueu::lock"), 
+      mqueue(cct->_conf->ms_pq_max_tokens_per_priority,
+	     cct->_conf->ms_pq_min_cost),
+      next_pipe_id(1),
+      dispatch_thread(this),
+      stop(false)
+    {}
+};
+
+#endif
diff --git a/ceph/src/msg/Dispatcher.h b/ceph/src/msg/Dispatcher.h
new file mode 100644
index 00000000..9f837cd6
--- /dev/null
+++ b/ceph/src/msg/Dispatcher.h
@@ -0,0 +1,120 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_DISPATCHER_H
+#define CEPH_DISPATCHER_H
+
+#include "Message.h"
+#include "common/config.h"
+#include "auth/Auth.h"
+
+class Messenger;
+
+class Dispatcher {
+public:
+  Dispatcher(CephContext *cct_)
+    : cct(cct_)
+  {
+  }
+  virtual ~Dispatcher() { }
+
+  // how i receive messages
+  virtual bool ms_dispatch(Message *m) = 0;
+
+  /**
+   * This function will be called whenever a new Connection is made to the
+   * Messenger.
+   *
+   * @param con The new Connection which has been established. You are not
+   * granted a reference to it -- take one if you need one!
+   */
+  virtual void ms_handle_connect(Connection *con) { };
+
+  /**
+   * Callback indicating we have accepted an incoming connection.
+   *
+   * @param con The (new or existing) Connection associated with the session
+   */
+  virtual void ms_handle_accept(Connection *con) { };
+
+  /*
+   * this indicates that the ordered+reliable delivery semantics have 
+   * been violated.  Messages may have been lost due to a fault
+   * in the network connection.
+   * Only called on lossy Connections or those you've
+   * designated mark_down_on_empty().
+   *
+   * @param con The Connection which broke. You are not granted
+   * a reference to it.
+   */
+  virtual bool ms_handle_reset(Connection *con) = 0;
+
+  /**
+   * This indicates that the ordered+reliable delivery semantics
+   * have been violated because the remote somehow reset.
+   * It implies that incoming messages were dropped, and
+   * probably some of our previous outgoing messages were too.
+   *
+   * @param con The Connection which broke. You are not granted
+   * a reference to it.
+   */
+  virtual void ms_handle_remote_reset(Connection *con) = 0;
+  
+  /**
+   * @defgroup Authentication
+   * @{
+   */
+  /**
+   * Retrieve the AuthAuthorizer for the given peer type. It might not
+   * provide one if it knows there is no AuthAuthorizer for that type.
+   *
+   * @param dest_type The peer type we want the authorizer for.
+   * @param a Double pointer to an AuthAuthorizer. The Dispatcher will fill
+   * in *a with the correct AuthAuthorizer, if it can. Make sure that you have
+   * set *a to NULL before calling in.
+   * @param force_new Force the Dispatcher to wait for a new set of keys before
+   * returning the authorizer.
+   *
+   * @return True if this function call properly filled in *a, false otherwise.
+   */
+  virtual bool ms_get_authorizer(int dest_type, AuthAuthorizer **a, bool force_new) { return false; };
+  /**
+   * Verify the authorizer for a new incoming Connection.
+   *
+   * @param con The new incoming Connection
+   * @param peer_type The type of the endpoint which initiated this Connection
+   * @param protocol The ID of the protocol in use (at time of writing, cephx or none)
+   * @param authorizer The authorization string supplied by the remote
+   * @param authorizer_reply Output param: The string we should send back to
+   * the remote to authorize ourselves. Only filled in if isvalid
+   * @param isvalid Output param: True if authorizer is valid, false otherwise
+   *
+   * @return True if we were able to prove or disprove correctness of
+   * authorizer, false otherwise.
+   */
+  virtual bool ms_verify_authorizer(Connection *con, int peer_type,
+				    int protocol, bufferlist& authorizer, bufferlist& authorizer_reply,
+				    bool& isvalid, CryptoKey& session_key) { return false; };
+  /**
+   * @} //Authentication
+   */
+protected:
+  CephContext *cct;
+private:
+  Dispatcher(const Dispatcher &rhs);
+  Dispatcher& operator=(const Dispatcher &rhs);
+};
+
+#endif
diff --git a/ceph/src/msg/Makefile.am b/ceph/src/msg/Makefile.am
new file mode 100644
index 00000000..a849a1ca
--- /dev/null
+++ b/ceph/src/msg/Makefile.am
@@ -0,0 +1,20 @@
+libmsg_la_SOURCES = \
+	msg/Accepter.cc \
+	msg/DispatchQueue.cc \
+	msg/Message.cc \
+	msg/Messenger.cc \
+	msg/Pipe.cc \
+	msg/SimpleMessenger.cc \
+	msg/msg_types.cc
+
+noinst_HEADERS += \
+	msg/Accepter.h \
+	msg/DispatchQueue.h \
+	msg/Dispatcher.h \
+	msg/Message.h \
+	msg/Messenger.h \
+	msg/Pipe.h \
+	msg/SimpleMessenger.h \
+	msg/msg_types.h
+
+noinst_LTLIBRARIES += libmsg.la
diff --git a/ceph/src/msg/Message.cc b/ceph/src/msg/Message.cc
new file mode 100644
index 00000000..6ecce716
--- /dev/null
+++ b/ceph/src/msg/Message.cc
@@ -0,0 +1,787 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#ifdef ENCODE_DUMP
+# include 
+# include 
+#endif
+
+#include 
+using namespace std;
+
+#include "include/types.h"
+
+#include "global/global_context.h"
+
+#include "Message.h"
+#include "Pipe.h"
+#include "messages/MPGStats.h"
+
+#include "messages/MGenericMessage.h"
+
+#include "messages/MPGStatsAck.h"
+
+#include "messages/MStatfs.h"
+#include "messages/MStatfsReply.h"
+
+#include "messages/MGetPoolStats.h"
+#include "messages/MGetPoolStatsReply.h"
+
+
+#include "messages/MPoolOp.h"
+#include "messages/MPoolOpReply.h"
+
+#include "messages/PaxosServiceMessage.h"
+#include "messages/MMonCommand.h"
+#include "messages/MMonCommandAck.h"
+#include "messages/MMonPaxos.h"
+
+#include "messages/MMonProbe.h"
+#include "messages/MMonJoin.h"
+#include "messages/MMonElection.h"
+#include "messages/MMonSync.h"
+#include "messages/MMonScrub.h"
+
+#include "messages/MLog.h"
+#include "messages/MLogAck.h"
+
+#include "messages/MPing.h"
+
+#include "messages/MCommand.h"
+#include "messages/MCommandReply.h"
+#include "messages/MBackfillReserve.h"
+#include "messages/MRecoveryReserve.h"
+
+#include "messages/MRoute.h"
+#include "messages/MForward.h"
+
+#include "messages/MOSDBoot.h"
+#include "messages/MOSDAlive.h"
+#include "messages/MOSDPGTemp.h"
+#include "messages/MOSDFailure.h"
+#include "messages/MOSDMarkMeDown.h"
+#include "messages/MOSDPing.h"
+#include "messages/MOSDOp.h"
+#include "messages/MOSDOpReply.h"
+#include "messages/MOSDSubOp.h"
+#include "messages/MOSDSubOpReply.h"
+#include "messages/MOSDMap.h"
+
+#include "messages/MOSDPGNotify.h"
+#include "messages/MOSDPGQuery.h"
+#include "messages/MOSDPGLog.h"
+#include "messages/MOSDPGRemove.h"
+#include "messages/MOSDPGInfo.h"
+#include "messages/MOSDPGCreate.h"
+#include "messages/MOSDPGTrim.h"
+#include "messages/MOSDPGMissing.h"
+#include "messages/MOSDScrub.h"
+#include "messages/MOSDRepScrub.h"
+#include "messages/MOSDPGScan.h"
+#include "messages/MOSDPGBackfill.h"
+
+#include "messages/MRemoveSnaps.h"
+
+#include "messages/MMonMap.h"
+#include "messages/MMonGetMap.h"
+#include "messages/MMonGetVersion.h"
+#include "messages/MMonGetVersionReply.h"
+#include "messages/MMonHealth.h"
+
+#include "messages/MAuth.h"
+#include "messages/MAuthReply.h"
+#include "messages/MMonSubscribe.h"
+#include "messages/MMonSubscribeAck.h"
+#include "messages/MMonGlobalID.h"
+#include "messages/MClientSession.h"
+#include "messages/MClientReconnect.h"
+#include "messages/MClientRequest.h"
+#include "messages/MClientRequestForward.h"
+#include "messages/MClientReply.h"
+#include "messages/MClientCaps.h"
+#include "messages/MClientCapRelease.h"
+#include "messages/MClientLease.h"
+#include "messages/MClientSnap.h"
+
+#include "messages/MMDSSlaveRequest.h"
+
+#include "messages/MMDSMap.h"
+#include "messages/MMDSBeacon.h"
+#include "messages/MMDSLoadTargets.h"
+#include "messages/MMDSResolve.h"
+#include "messages/MMDSResolveAck.h"
+#include "messages/MMDSCacheRejoin.h"
+#include "messages/MMDSFindIno.h"
+#include "messages/MMDSFindInoReply.h"
+#include "messages/MMDSOpenIno.h"
+#include "messages/MMDSOpenInoReply.h"
+
+#include "messages/MDirUpdate.h"
+#include "messages/MDiscover.h"
+#include "messages/MDiscoverReply.h"
+
+#include "messages/MMDSFragmentNotify.h"
+
+#include "messages/MExportDirDiscover.h"
+#include "messages/MExportDirDiscoverAck.h"
+#include "messages/MExportDirCancel.h"
+#include "messages/MExportDirPrep.h"
+#include "messages/MExportDirPrepAck.h"
+#include "messages/MExportDir.h"
+#include "messages/MExportDirAck.h"
+#include "messages/MExportDirNotify.h"
+#include "messages/MExportDirNotifyAck.h"
+#include "messages/MExportDirFinish.h"
+
+#include "messages/MExportCaps.h"
+#include "messages/MExportCapsAck.h"
+
+
+#include "messages/MDentryUnlink.h"
+#include "messages/MDentryLink.h"
+
+#include "messages/MHeartbeat.h"
+
+#include "messages/MMDSTableRequest.h"
+
+//#include "messages/MInodeUpdate.h"
+#include "messages/MCacheExpire.h"
+#include "messages/MInodeFileCaps.h"
+
+#include "messages/MLock.h"
+
+#include "messages/MWatchNotify.h"
+#include "messages/MTimeCheck.h"
+
+#include "common/config.h"
+
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPushReply.h"
+#include "messages/MOSDPGPull.h"
+
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
+
+#define DEBUGLVL  10    // debug level of output
+
+#define dout_subsys ceph_subsys_ms
+
+void Message::encode(uint64_t features, bool datacrc)
+{
+  // encode and copy out of *m
+  if (empty_payload()) {
+    encode_payload(features);
+
+    // if the encoder didn't specify past compatibility, we assume it
+    // is incompatible.
+    if (header.compat_version == 0)
+      header.compat_version = header.version;
+  }
+  calc_front_crc();
+
+  // update envelope
+  header.front_len = get_payload().length();
+  header.middle_len = get_middle().length();
+  header.data_len = get_data().length();
+  calc_header_crc();
+
+  footer.flags = CEPH_MSG_FOOTER_COMPLETE;
+
+  if (datacrc) {
+    calc_data_crc();
+
+#ifdef ENCODE_DUMP
+    bufferlist bl;
+    ::encode(get_header(), bl);
+
+    // dump the old footer format
+    ceph_msg_footer_old old_footer;
+    old_footer.front_crc = footer.front_crc;
+    old_footer.middle_crc = footer.middle_crc;
+    old_footer.data_crc = footer.data_crc;
+    old_footer.flags = footer.flags;
+    ::encode(old_footer, bl);
+
+    ::encode(get_payload(), bl);
+    ::encode(get_middle(), bl);
+    ::encode(get_data(), bl);
+
+    // this is almost an exponential backoff, except because we count
+    // bits we tend to sample things we encode later, which should be
+    // more representative.
+    static int i = 0;
+    i++;
+    int bits = 0;
+    for (unsigned t = i; t; bits++)
+      t &= t - 1;
+    if (bits <= 2) {
+      char fn[200];
+      int status;
+      snprintf(fn, sizeof(fn), ENCODE_STRINGIFY(ENCODE_DUMP) "/%s__%d.%x",
+	       abi::__cxa_demangle(typeid(*this).name(), 0, 0, &status),
+	       getpid(), i++);
+      int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT, 0644);
+      if (fd >= 0) {
+	bl.write_fd(fd);
+	::close(fd);
+      }
+    }
+#endif
+  } else {
+    footer.flags = (unsigned)footer.flags | CEPH_MSG_FOOTER_NOCRC;
+  }
+}
+
+void Message::dump(Formatter *f) const
+{
+  stringstream ss;
+  print(ss);
+  f->dump_string("summary", ss.str());
+}
+
+Message *decode_message(CephContext *cct, ceph_msg_header& header, ceph_msg_footer& footer,
+			bufferlist& front, bufferlist& middle, bufferlist& data)
+{
+  // verify crc
+  if (!cct || !cct->_conf->ms_nocrc) {
+    __u32 front_crc = front.crc32c(0);
+    __u32 middle_crc = middle.crc32c(0);
+
+    if (front_crc != footer.front_crc) {
+      if (cct) {
+	ldout(cct, 0) << "bad crc in front " << front_crc << " != exp " << footer.front_crc << dendl;
+	ldout(cct, 20) << " ";
+	front.hexdump(*_dout);
+	*_dout << dendl;
+      }
+      return 0;
+    }
+    if (middle_crc != footer.middle_crc) {
+      if (cct) {
+	ldout(cct, 0) << "bad crc in middle " << middle_crc << " != exp " << footer.middle_crc << dendl;
+	ldout(cct, 20) << " ";
+	middle.hexdump(*_dout);
+	*_dout << dendl;
+      }
+      return 0;
+    }
+
+    if ((footer.flags & CEPH_MSG_FOOTER_NOCRC) == 0) {
+      __u32 data_crc = data.crc32c(0);
+      if (data_crc != footer.data_crc) {
+	if (cct) {
+	  ldout(cct, 0) << "bad crc in data " << data_crc << " != exp " << footer.data_crc << dendl;
+	  ldout(cct, 20) << " ";
+	  data.hexdump(*_dout);
+	  *_dout << dendl;
+	}
+	return 0;
+      }
+    }
+  } 
+
+  // make message
+  Message *m = 0;
+  int type = header.type;
+  switch (type) {
+
+    // -- with payload --
+
+  case MSG_PGSTATS:
+    m = new MPGStats;
+    break;
+  case MSG_PGSTATSACK:
+    m = new MPGStatsAck;
+    break;
+
+  case CEPH_MSG_STATFS:
+    m = new MStatfs;
+    break;
+  case CEPH_MSG_STATFS_REPLY:
+    m = new MStatfsReply;
+    break;
+  case MSG_GETPOOLSTATS:
+    m = new MGetPoolStats;
+    break;
+  case MSG_GETPOOLSTATSREPLY:
+    m = new MGetPoolStatsReply;
+    break;
+  case CEPH_MSG_POOLOP:
+    m = new MPoolOp;
+    break;
+  case CEPH_MSG_POOLOP_REPLY:
+    m = new MPoolOpReply;
+    break;
+  case MSG_MON_COMMAND:
+    m = new MMonCommand;
+    break;
+  case MSG_MON_COMMAND_ACK:
+    m = new MMonCommandAck;
+    break;
+  case MSG_MON_PAXOS:
+    m = new MMonPaxos;
+    break;
+
+  case MSG_MON_PROBE:
+    m = new MMonProbe;
+    break;
+  case MSG_MON_JOIN:
+    m = new MMonJoin;
+    break;
+  case MSG_MON_ELECTION:
+    m = new MMonElection;
+    break;
+  case MSG_MON_SYNC:
+    m = new MMonSync;
+    break;
+  case MSG_MON_SCRUB:
+    m = new MMonScrub;
+    break;
+
+  case MSG_LOG:
+    m = new MLog;
+    break;
+  case MSG_LOGACK:
+    m = new MLogAck;
+    break;
+
+  case CEPH_MSG_PING:
+    m = new MPing();
+    break;
+  case MSG_COMMAND:
+    m = new MCommand;
+    break;
+  case MSG_COMMAND_REPLY:
+    m = new MCommandReply;
+    break;
+  case MSG_OSD_BACKFILL_RESERVE:
+    m = new MBackfillReserve;
+    break;
+  case MSG_OSD_RECOVERY_RESERVE:
+    m = new MRecoveryReserve;
+    break;
+
+  case MSG_ROUTE:
+    m = new MRoute;
+    break;
+  case MSG_FORWARD:
+    m = new MForward;
+    break;
+    
+  case CEPH_MSG_MON_MAP:
+    m = new MMonMap;
+    break;
+  case CEPH_MSG_MON_GET_MAP:
+    m = new MMonGetMap;
+    break;
+  case CEPH_MSG_MON_GET_VERSION:
+    m = new MMonGetVersion();
+    break;
+  case CEPH_MSG_MON_GET_VERSION_REPLY:
+    m = new MMonGetVersionReply();
+    break;
+
+  case MSG_OSD_BOOT:
+    m = new MOSDBoot();
+    break;
+  case MSG_OSD_ALIVE:
+    m = new MOSDAlive();
+    break;
+  case MSG_OSD_PGTEMP:
+    m = new MOSDPGTemp;
+    break;
+  case MSG_OSD_FAILURE:
+    m = new MOSDFailure();
+    break;
+  case MSG_OSD_MARK_ME_DOWN:
+    m = new MOSDMarkMeDown();
+    break;
+  case MSG_OSD_PING:
+    m = new MOSDPing();
+    break;
+  case CEPH_MSG_OSD_OP:
+    m = new MOSDOp();
+    break;
+  case CEPH_MSG_OSD_OPREPLY:
+    m = new MOSDOpReply();
+    break;
+  case MSG_OSD_SUBOP:
+    m = new MOSDSubOp();
+    break;
+  case MSG_OSD_SUBOPREPLY:
+    m = new MOSDSubOpReply();
+    break;
+
+  case CEPH_MSG_OSD_MAP:
+    m = new MOSDMap;
+    break;
+
+  case CEPH_MSG_WATCH_NOTIFY:
+    m = new MWatchNotify;
+    break;
+
+  case MSG_OSD_PG_NOTIFY:
+    m = new MOSDPGNotify;
+    break;
+  case MSG_OSD_PG_QUERY:
+    m = new MOSDPGQuery;
+    break;
+  case MSG_OSD_PG_LOG:
+    m = new MOSDPGLog;
+    break;
+  case MSG_OSD_PG_REMOVE:
+    m = new MOSDPGRemove;
+    break;
+  case MSG_OSD_PG_INFO:
+    m = new MOSDPGInfo;
+    break;
+  case MSG_OSD_PG_CREATE:
+    m = new MOSDPGCreate;
+    break;
+  case MSG_OSD_PG_TRIM:
+    m = new MOSDPGTrim;
+    break;
+
+  case MSG_OSD_SCRUB:
+    m = new MOSDScrub;
+    break;
+  case MSG_REMOVE_SNAPS:
+    m = new MRemoveSnaps;
+    break;
+  case MSG_OSD_PG_MISSING:
+    m = new MOSDPGMissing;
+    break;
+  case MSG_OSD_REP_SCRUB:
+    m = new MOSDRepScrub;
+    break;
+  case MSG_OSD_PG_SCAN:
+    m = new MOSDPGScan;
+    break;
+  case MSG_OSD_PG_BACKFILL:
+    m = new MOSDPGBackfill;
+    break;
+  case MSG_OSD_PG_PUSH:
+    m = new MOSDPGPush;
+    break;
+  case MSG_OSD_PG_PULL:
+    m = new MOSDPGPull;
+    break;
+  case MSG_OSD_PG_PUSH_REPLY:
+    m = new MOSDPGPushReply;
+    break;
+  case MSG_OSD_EC_WRITE:
+    m = new MOSDECSubOpWrite;
+    break;
+  case MSG_OSD_EC_WRITE_REPLY:
+    m = new MOSDECSubOpWriteReply;
+    break;
+  case MSG_OSD_EC_READ:
+    m = new MOSDECSubOpRead;
+    break;
+  case MSG_OSD_EC_READ_REPLY:
+    m = new MOSDECSubOpReadReply;
+    break;
+   // auth
+  case CEPH_MSG_AUTH:
+    m = new MAuth;
+    break;
+  case CEPH_MSG_AUTH_REPLY:
+    m = new MAuthReply;
+    break;
+
+  case MSG_MON_GLOBAL_ID:
+    m = new MMonGlobalID;
+    break; 
+
+    // clients
+  case CEPH_MSG_MON_SUBSCRIBE:
+    m = new MMonSubscribe;
+    break;
+  case CEPH_MSG_MON_SUBSCRIBE_ACK:
+    m = new MMonSubscribeAck;
+    break;
+  case CEPH_MSG_CLIENT_SESSION:
+    m = new MClientSession;
+    break;
+  case CEPH_MSG_CLIENT_RECONNECT:
+    m = new MClientReconnect;
+    break;
+  case CEPH_MSG_CLIENT_REQUEST:
+    m = new MClientRequest;
+    break;
+  case CEPH_MSG_CLIENT_REQUEST_FORWARD:
+    m = new MClientRequestForward;
+    break;
+  case CEPH_MSG_CLIENT_REPLY:
+    m = new MClientReply;
+    break;
+  case CEPH_MSG_CLIENT_CAPS:
+    m = new MClientCaps;
+    break;
+  case CEPH_MSG_CLIENT_CAPRELEASE:
+    m = new MClientCapRelease;
+    break;
+  case CEPH_MSG_CLIENT_LEASE:
+    m = new MClientLease;
+    break;
+  case CEPH_MSG_CLIENT_SNAP:
+    m = new MClientSnap;
+    break;
+
+    // mds
+  case MSG_MDS_SLAVE_REQUEST:
+    m = new MMDSSlaveRequest;
+    break;
+
+  case CEPH_MSG_MDS_MAP:
+    m = new MMDSMap;
+    break;
+  case MSG_MDS_BEACON:
+    m = new MMDSBeacon;
+    break;
+  case MSG_MDS_OFFLOAD_TARGETS:
+    m = new MMDSLoadTargets;
+    break;
+  case MSG_MDS_RESOLVE:
+    m = new MMDSResolve;
+    break;
+  case MSG_MDS_RESOLVEACK:
+    m = new MMDSResolveAck;
+    break;
+  case MSG_MDS_CACHEREJOIN:
+    m = new MMDSCacheRejoin;
+	break;
+	/*
+  case MSG_MDS_CACHEREJOINACK:
+	m = new MMDSCacheRejoinAck;
+	break;
+	*/
+
+  case MSG_MDS_DIRUPDATE:
+    m = new MDirUpdate();
+    break;
+
+  case MSG_MDS_DISCOVER:
+    m = new MDiscover();
+    break;
+  case MSG_MDS_DISCOVERREPLY:
+    m = new MDiscoverReply();
+    break;
+
+  case MSG_MDS_FINDINO:
+    m = new MMDSFindIno;
+    break;
+  case MSG_MDS_FINDINOREPLY:
+    m = new MMDSFindInoReply;
+    break;
+
+  case MSG_MDS_OPENINO:
+    m = new MMDSOpenIno;
+    break;
+  case MSG_MDS_OPENINOREPLY:
+    m = new MMDSOpenInoReply;
+    break;
+
+  case MSG_MDS_FRAGMENTNOTIFY:
+    m = new MMDSFragmentNotify;
+    break;
+
+  case MSG_MDS_EXPORTDIRDISCOVER:
+    m = new MExportDirDiscover();
+    break;
+  case MSG_MDS_EXPORTDIRDISCOVERACK:
+    m = new MExportDirDiscoverAck();
+    break;
+  case MSG_MDS_EXPORTDIRCANCEL:
+    m = new MExportDirCancel();
+    break;
+
+  case MSG_MDS_EXPORTDIR:
+    m = new MExportDir;
+    break;
+  case MSG_MDS_EXPORTDIRACK:
+    m = new MExportDirAck;
+    break;
+  case MSG_MDS_EXPORTDIRFINISH:
+    m = new MExportDirFinish;
+    break;
+
+  case MSG_MDS_EXPORTDIRNOTIFY:
+    m = new MExportDirNotify();
+    break;
+
+  case MSG_MDS_EXPORTDIRNOTIFYACK:
+    m = new MExportDirNotifyAck();
+    break;
+
+  case MSG_MDS_EXPORTDIRPREP:
+    m = new MExportDirPrep();
+    break;
+
+  case MSG_MDS_EXPORTDIRPREPACK:
+    m = new MExportDirPrepAck();
+    break;
+
+  case MSG_MDS_EXPORTCAPS:
+    m = new MExportCaps;
+    break;
+  case MSG_MDS_EXPORTCAPSACK:
+    m = new MExportCapsAck;
+    break;
+
+
+  case MSG_MDS_DENTRYUNLINK:
+    m = new MDentryUnlink;
+    break;
+  case MSG_MDS_DENTRYLINK:
+    m = new MDentryLink;
+    break;
+
+  case MSG_MDS_HEARTBEAT:
+    m = new MHeartbeat();
+    break;
+
+  case MSG_MDS_CACHEEXPIRE:
+    m = new MCacheExpire();
+    break;
+
+  case MSG_MDS_TABLE_REQUEST:
+    m = new MMDSTableRequest;
+    break;
+
+	/*  case MSG_MDS_INODEUPDATE:
+    m = new MInodeUpdate();
+    break;
+	*/
+
+  case MSG_MDS_INODEFILECAPS:
+    m = new MInodeFileCaps();
+    break;
+
+  case MSG_MDS_LOCK:
+    m = new MLock();
+    break;
+
+  case MSG_TIMECHECK:
+    m = new MTimeCheck();
+    break;
+
+  case MSG_MON_HEALTH:
+    m = new MMonHealth();
+    break;
+
+    // -- simple messages without payload --
+
+  case CEPH_MSG_SHUTDOWN:
+    m = new MGenericMessage(type);
+    break;
+
+  default:
+    if (cct) {
+      ldout(cct, 0) << "can't decode unknown message type " << type << " MSG_AUTH=" << CEPH_MSG_AUTH << dendl;
+      if (cct->_conf->ms_die_on_bad_msg)
+	assert(0);
+    }
+    return 0;
+  }
+
+  // m->header.version, if non-zero, should be populated with the
+  // newest version of the encoding the code supports.  If set, check
+  // it against compat_version.
+  if (m->get_header().version &&
+      m->get_header().version < header.compat_version) {
+    if (cct) {
+      ldout(cct, 0) << "will not decode message of type " << type
+		    << " version " << header.version
+		    << " because compat_version " << header.compat_version
+		    << " > supported version " << m->get_header().version << dendl;
+      if (cct->_conf->ms_die_on_bad_msg)
+	assert(0);
+    }
+    m->put();
+    return 0;
+  }
+  
+  m->set_header(header);
+  m->set_footer(footer);
+  m->set_payload(front);
+  m->set_middle(middle);
+  m->set_data(data);
+
+  try {
+    m->decode_payload();
+  }
+  catch (const buffer::error &e) {
+    if (cct) {
+      lderr(cct) << "failed to decode message of type " << type
+		 << " v" << header.version
+		 << ": " << e.what() << dendl;
+      ldout(cct, 30) << "dump: \n";
+      m->get_payload().hexdump(*_dout);
+      *_dout << dendl;
+      if (cct->_conf->ms_die_on_bad_msg)
+	assert(0);
+    }
+    m->put();
+    return 0;
+  }
+
+  // done!
+  return m;
+}
+
+
+// This routine is not used for ordinary messages, but only when encapsulating a message
+// for forwarding and routing.  It's also used in a backward compatibility test, which only
+// effectively tests backward compability for those functions.  To avoid backward compatibility
+// problems, we currently always encode and decode using the old footer format that doesn't
+// allow for message authentication.  Eventually we should fix that.  PLR
+
+void encode_message(Message *msg, uint64_t features, bufferlist& payload)
+{
+  bufferlist front, middle, data;
+  ceph_msg_footer_old old_footer;
+  ceph_msg_footer footer;
+  msg->encode(features, true);
+  ::encode(msg->get_header(), payload);
+
+  // Here's where we switch to the old footer format.  PLR
+
+  footer = msg->get_footer();
+  old_footer.front_crc = footer.front_crc;   
+  old_footer.middle_crc = footer.middle_crc;   
+  old_footer.data_crc = footer.data_crc;   
+  old_footer.flags = footer.flags;   
+  ::encode(old_footer, payload);
+
+  ::encode(msg->get_payload(), payload);
+  ::encode(msg->get_middle(), payload);
+  ::encode(msg->get_data(), payload);
+}
+
+// See above for somewhat bogus use of the old message footer.  We switch to the current footer
+// after decoding the old one so the other form of decode_message() doesn't have to change.
+// We've slipped in a 0 signature at this point, so any signature checking after this will
+// fail.  PLR
+
+Message *decode_message(CephContext *cct, bufferlist::iterator& p)
+{
+  ceph_msg_header h;
+  ceph_msg_footer_old fo;
+  ceph_msg_footer f;
+  bufferlist fr, mi, da;
+  ::decode(h, p);
+  ::decode(fo, p);
+  f.front_crc = fo.front_crc;
+  f.middle_crc = fo.middle_crc;
+  f.data_crc = fo.data_crc;
+  f.flags = fo.flags;
+  f.sig = 0;
+  ::decode(fr, p);
+  ::decode(mi, p);
+  ::decode(da, p);
+  return decode_message(cct, h, f, fr, mi, da);
+}
+
diff --git a/ceph/src/msg/Message.h b/ceph/src/msg/Message.h
new file mode 100644
index 00000000..a69944f8
--- /dev/null
+++ b/ceph/src/msg/Message.h
@@ -0,0 +1,544 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MESSAGE_H
+#define CEPH_MESSAGE_H
+ 
+#include 
+#include 
+
+#include 
+// Because intusive_ptr clobbers our assert...
+#include "include/assert.h"
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "common/Throttle.h"
+#include "msg_types.h"
+
+#include "common/RefCountedObj.h"
+
+#include "common/debug.h"
+#include "common/config.h"
+
+// monitor internal
+#define MSG_MON_SCRUB              64
+#define MSG_MON_ELECTION           65
+#define MSG_MON_PAXOS              66
+#define MSG_MON_PROBE              67
+#define MSG_MON_JOIN               68
+#define MSG_MON_SYNC		   69
+
+/* monitor <-> mon admin tool */
+#define MSG_MON_COMMAND            50
+#define MSG_MON_COMMAND_ACK        51
+#define MSG_LOG                    52
+#define MSG_LOGACK                 53
+//#define MSG_MON_OBSERVE            54
+//#define MSG_MON_OBSERVE_NOTIFY     55
+#define MSG_CLASS                  56
+#define MSG_CLASS_ACK              57
+
+#define MSG_GETPOOLSTATS           58
+#define MSG_GETPOOLSTATSREPLY      59
+
+#define MSG_MON_GLOBAL_ID          60
+
+// #define MSG_POOLOP                 49
+// #define MSG_POOLOPREPLY            48
+
+#define MSG_ROUTE                  47
+#define MSG_FORWARD                46
+
+#define MSG_PAXOS                  40
+
+
+// osd internal
+#define MSG_OSD_PING         70
+#define MSG_OSD_BOOT         71
+#define MSG_OSD_FAILURE      72
+#define MSG_OSD_ALIVE        73
+#define MSG_OSD_MARK_ME_DOWN 74
+
+#define MSG_OSD_SUBOP        76
+#define MSG_OSD_SUBOPREPLY   77
+
+#define MSG_OSD_PGTEMP       78
+
+#define MSG_OSD_PG_NOTIFY      80
+#define MSG_OSD_PG_QUERY       81
+#define MSG_OSD_PG_SUMMARY     82
+#define MSG_OSD_PG_LOG         83
+#define MSG_OSD_PG_REMOVE      84
+#define MSG_OSD_PG_INFO        85
+#define MSG_OSD_PG_TRIM        86
+
+#define MSG_PGSTATS            87
+#define MSG_PGSTATSACK         88
+
+#define MSG_OSD_PG_CREATE      89
+#define MSG_REMOVE_SNAPS       90
+
+#define MSG_OSD_SCRUB          91
+#define MSG_OSD_PG_MISSING     92
+#define MSG_OSD_REP_SCRUB      93
+
+#define MSG_OSD_PG_SCAN        94
+#define MSG_OSD_PG_BACKFILL    95
+
+#define MSG_COMMAND            97
+#define MSG_COMMAND_REPLY      98
+
+#define MSG_OSD_BACKFILL_RESERVE 99
+#define MSG_OSD_RECOVERY_RESERVE 150
+
+#define MSG_OSD_PG_PUSH        105
+#define MSG_OSD_PG_PULL        106
+#define MSG_OSD_PG_PUSH_REPLY  107
+
+#define MSG_OSD_EC_WRITE       108
+#define MSG_OSD_EC_WRITE_REPLY 109
+#define MSG_OSD_EC_READ        110
+#define MSG_OSD_EC_READ_REPLY  111
+
+// *** MDS ***
+
+#define MSG_MDS_BEACON             100  // to monitor
+#define MSG_MDS_SLAVE_REQUEST      101
+#define MSG_MDS_TABLE_REQUEST      102
+
+                                // 150 already in use (MSG_OSD_RECOVERY_RESERVE)
+
+#define MSG_MDS_RESOLVE            0x200
+#define MSG_MDS_RESOLVEACK         0x201
+#define MSG_MDS_CACHEREJOIN        0x202
+#define MSG_MDS_DISCOVER           0x203
+#define MSG_MDS_DISCOVERREPLY      0x204
+#define MSG_MDS_INODEUPDATE        0x205
+#define MSG_MDS_DIRUPDATE          0x206
+#define MSG_MDS_CACHEEXPIRE        0x207
+#define MSG_MDS_DENTRYUNLINK       0x208
+#define MSG_MDS_FRAGMENTNOTIFY     0x209
+#define MSG_MDS_OFFLOAD_TARGETS    0x20a
+#define MSG_MDS_DENTRYLINK         0x20c
+#define MSG_MDS_FINDINO            0x20d
+#define MSG_MDS_FINDINOREPLY       0x20e
+#define MSG_MDS_OPENINO            0x20f
+#define MSG_MDS_OPENINOREPLY       0x210
+
+#define MSG_MDS_LOCK               0x300
+#define MSG_MDS_INODEFILECAPS      0x301
+
+#define MSG_MDS_EXPORTDIRDISCOVER     0x449
+#define MSG_MDS_EXPORTDIRDISCOVERACK  0x450
+#define MSG_MDS_EXPORTDIRCANCEL       0x451
+#define MSG_MDS_EXPORTDIRPREP         0x452
+#define MSG_MDS_EXPORTDIRPREPACK      0x453
+#define MSG_MDS_EXPORTDIRWARNING      0x454
+#define MSG_MDS_EXPORTDIRWARNINGACK   0x455
+#define MSG_MDS_EXPORTDIR             0x456
+#define MSG_MDS_EXPORTDIRACK          0x457
+#define MSG_MDS_EXPORTDIRNOTIFY       0x458
+#define MSG_MDS_EXPORTDIRNOTIFYACK    0x459
+#define MSG_MDS_EXPORTDIRFINISH       0x460
+
+#define MSG_MDS_EXPORTCAPS            0x470
+#define MSG_MDS_EXPORTCAPSACK         0x471
+
+#define MSG_MDS_HEARTBEAT          0x500  // for mds load balancer
+
+// *** generic ***
+#define MSG_TIMECHECK             0x600
+#define MSG_MON_HEALTH            0x601
+
+
+
+
+// ======================================================
+
+// abstract Connection, for keeping per-connection state
+
+class Messenger;
+
+struct Connection : private RefCountedObject {
+  Mutex lock;
+  Messenger *msgr;
+  RefCountedObject *priv;
+  int peer_type;
+  entity_addr_t peer_addr;
+  utime_t last_keepalive_ack;
+private:
+  uint64_t features;
+public:
+  RefCountedObject *pipe;
+  bool failed;              /// true if we are a lossy connection that has failed.
+
+  int rx_buffers_version;
+  map > rx_buffers;
+
+  friend class boost::intrusive_ptr;
+
+public:
+  Connection(Messenger *m)
+    : lock("Connection::lock"),
+      msgr(m),
+      priv(NULL),
+      peer_type(-1),
+      features(0),
+      pipe(NULL),
+      failed(false),
+      rx_buffers_version(0) {
+    // we are managed exlusively by ConnectionRef; make it so you can
+    //   ConnectionRef foo = new Connection;
+    nref.set(0);
+  }
+  ~Connection() {
+    //generic_dout(0) << "~Connection " << this << dendl;
+    if (priv) {
+      //generic_dout(0) << "~Connection " << this << " dropping priv " << priv << dendl;
+      priv->put();
+    }
+    if (pipe)
+      pipe->put();
+  }
+
+  void set_priv(RefCountedObject *o) {
+    Mutex::Locker l(lock);
+    if (priv)
+      priv->put();
+    priv = o;
+  }
+  RefCountedObject *get_priv() {
+    Mutex::Locker l(lock);
+    if (priv)
+      return priv->get();
+    return NULL;
+  }
+
+  RefCountedObject *get_pipe() {
+    Mutex::Locker l(lock);
+    if (pipe)
+      return pipe->get();
+    return NULL;
+  }
+  bool try_get_pipe(RefCountedObject **p) {
+    Mutex::Locker l(lock);
+    if (failed) {
+      *p = NULL;
+    } else {
+      if (pipe)
+	*p = pipe->get();
+      else
+	*p = NULL;
+    }
+    return !failed;
+  }
+  bool clear_pipe(RefCountedObject *old_p) {
+    if (old_p == pipe) {
+      Mutex::Locker l(lock);
+      pipe->put();
+      pipe = NULL;
+      failed = true;
+      return true;
+    }
+    return false;
+  }
+  void reset_pipe(RefCountedObject *p) {
+    Mutex::Locker l(lock);
+    if (pipe)
+      pipe->put();
+    pipe = p->get();
+  }
+  bool is_connected() {
+    Mutex::Locker l(lock);
+    return pipe != NULL;
+  }
+
+  Messenger *get_messenger() {
+    return msgr;
+  }
+
+  int get_peer_type() { return peer_type; }
+  void set_peer_type(int t) { peer_type = t; }
+  
+  bool peer_is_mon() { return peer_type == CEPH_ENTITY_TYPE_MON; }
+  bool peer_is_mds() { return peer_type == CEPH_ENTITY_TYPE_MDS; }
+  bool peer_is_osd() { return peer_type == CEPH_ENTITY_TYPE_OSD; }
+  bool peer_is_client() { return peer_type == CEPH_ENTITY_TYPE_CLIENT; }
+
+  const entity_addr_t& get_peer_addr() { return peer_addr; }
+  void set_peer_addr(const entity_addr_t& a) { peer_addr = a; }
+
+  uint64_t get_features() const { return features; }
+  bool has_feature(uint64_t f) const { return features & f; }
+  void set_features(uint64_t f) { features = f; }
+  void set_feature(uint64_t f) { features |= f; }
+
+  void post_rx_buffer(ceph_tid_t tid, bufferlist& bl) {
+    Mutex::Locker l(lock);
+    ++rx_buffers_version;
+    rx_buffers[tid] = pair(bl, rx_buffers_version);
+  }
+  void revoke_rx_buffer(ceph_tid_t tid) {
+    Mutex::Locker l(lock);
+    rx_buffers.erase(tid);
+  }
+
+  utime_t get_last_keepalive_ack() const {
+    return last_keepalive_ack;
+  }
+};
+typedef boost::intrusive_ptr ConnectionRef;
+
+
+
+// abstract Message class
+
+class Message : public RefCountedObject {
+protected:
+  ceph_msg_header  header;      // headerelope
+  ceph_msg_footer  footer;
+  bufferlist       payload;  // "front" unaligned blob
+  bufferlist       middle;   // "middle" unaligned blob
+  bufferlist       data;     // data payload (page-alignment will be preserved where possible)
+  
+  /* recv_stamp is set when the Messenger starts reading the
+   * Message off the wire */
+  utime_t recv_stamp;
+  /* dispatch_stamp is set when the Messenger starts calling dispatch() on
+   * its endpoints */
+  utime_t dispatch_stamp;
+  /* throttle_stamp is the point at which we got throttle */
+  utime_t throttle_stamp;
+  /* time at which message was fully read */
+  utime_t recv_complete_stamp;
+
+  ConnectionRef connection;
+
+  // release our size in bytes back to this throttler when our payload
+  // is adjusted or when we are destroyed.
+  Throttle *byte_throttler;
+
+  // release a count back to this throttler when we are destroyed
+  Throttle *msg_throttler;
+
+  // keep track of how big this message was when we reserved space in
+  // the msgr dispatch_throttler, so that we can properly release it
+  // later.  this is necessary because messages can enter the dispatch
+  // queue locally (not via read_message()), and those are not
+  // currently throttled.
+  uint64_t dispatch_throttle_size;
+
+  friend class Messenger;
+
+public:
+  Message()
+    : connection(NULL),
+      byte_throttler(NULL),
+      msg_throttler(NULL),
+      dispatch_throttle_size(0) {
+    memset(&header, 0, sizeof(header));
+    memset(&footer, 0, sizeof(footer));
+  };
+  Message(int t, int version=1, int compat_version=0)
+    : connection(NULL),
+      byte_throttler(NULL),
+      msg_throttler(NULL),
+      dispatch_throttle_size(0) {
+    memset(&header, 0, sizeof(header));
+    header.type = t;
+    header.version = version;
+    header.compat_version = compat_version;
+    header.priority = 0;  // undef
+    header.data_off = 0;
+    memset(&footer, 0, sizeof(footer));
+  }
+
+  Message *get() {
+    return static_cast(RefCountedObject::get());
+  }
+
+protected:
+  virtual ~Message() { 
+    assert(nref.read() == 0);
+    if (byte_throttler)
+      byte_throttler->put(payload.length() + middle.length() + data.length());
+    if (msg_throttler)
+      msg_throttler->put();
+  }
+public:
+  const ConnectionRef& get_connection() { return connection; }
+  void set_connection(const ConnectionRef& c) {
+    connection = c;
+  }
+  void set_byte_throttler(Throttle *t) { byte_throttler = t; }
+  Throttle *get_byte_throttler() { return byte_throttler; }
+  void set_message_throttler(Throttle *t) { msg_throttler = t; }
+  Throttle *get_message_throttler() { return msg_throttler; }
+ 
+  void set_dispatch_throttle_size(uint64_t s) { dispatch_throttle_size = s; }
+  uint64_t get_dispatch_throttle_size() { return dispatch_throttle_size; }
+
+  ceph_msg_header &get_header() { return header; }
+  void set_header(const ceph_msg_header &e) { header = e; }
+  void set_footer(const ceph_msg_footer &e) { footer = e; }
+  ceph_msg_footer &get_footer() { return footer; }
+
+  /*
+   * If you use get_[data, middle, payload] you shouldn't
+   * use it to change those bufferlists unless you KNOW
+   * there is no throttle being used. The other
+   * functions are throttling-aware as appropriate.
+   */
+
+  void clear_payload() {
+    if (byte_throttler)
+      byte_throttler->put(payload.length() + middle.length());
+    payload.clear();
+    middle.clear();
+  }
+
+  virtual void clear_buffers() {}
+  void clear_data() {
+    if (byte_throttler)
+      byte_throttler->put(data.length());
+    data.clear();
+    clear_buffers(); // let subclass drop buffers as well
+  }
+
+  bool empty_payload() { return payload.length() == 0; }
+  bufferlist& get_payload() { return payload; }
+  void set_payload(bufferlist& bl) {
+    if (byte_throttler)
+      byte_throttler->put(payload.length());
+    payload.claim(bl);
+    if (byte_throttler)
+      byte_throttler->take(payload.length());
+  }
+
+  void set_middle(bufferlist& bl) {
+    if (byte_throttler)
+      byte_throttler->put(payload.length());
+    middle.claim(bl);
+    if (byte_throttler)
+      byte_throttler->take(payload.length());
+  }
+  bufferlist& get_middle() { return middle; }
+
+  void set_data(const bufferlist &d) {
+    if (byte_throttler)
+      byte_throttler->put(data.length());
+    data = d;
+    if (byte_throttler)
+      byte_throttler->take(data.length());
+  }
+
+  bufferlist& get_data() { return data; }
+  void claim_data(bufferlist& bl) {
+    if (byte_throttler)
+      byte_throttler->put(data.length());
+    bl.claim(data);
+  }
+  off_t get_data_len() { return data.length(); }
+
+  void set_recv_stamp(utime_t t) { recv_stamp = t; }
+  const utime_t& get_recv_stamp() const { return recv_stamp; }
+  void set_dispatch_stamp(utime_t t) { dispatch_stamp = t; }
+  const utime_t& get_dispatch_stamp() const { return dispatch_stamp; }
+  void set_throttle_stamp(utime_t t) { throttle_stamp = t; }
+  const utime_t& get_throttle_stamp() const { return throttle_stamp; }
+  void set_recv_complete_stamp(utime_t t) { recv_complete_stamp = t; }
+  const utime_t& get_recv_complete_stamp() const { return recv_complete_stamp; }
+
+  void calc_header_crc() {
+    header.crc = ceph_crc32c(0, (unsigned char*)&header,
+			     sizeof(header) - sizeof(header.crc));
+  }
+  void calc_front_crc() {
+    footer.front_crc = payload.crc32c(0);
+    footer.middle_crc = middle.crc32c(0);
+  }
+  void calc_data_crc() {
+    footer.data_crc = data.crc32c(0);
+  }
+
+  virtual int get_cost() const {
+    return data.length();
+  }
+
+  // type
+  int get_type() const { return header.type; }
+  void set_type(int t) { header.type = t; }
+
+  uint64_t get_tid() const { return header.tid; }
+  void set_tid(uint64_t t) { header.tid = t; }
+
+  unsigned get_seq() const { return header.seq; }
+  void set_seq(unsigned s) { header.seq = s; }
+
+  unsigned get_priority() const { return header.priority; }
+  void set_priority(__s16 p) { header.priority = p; }
+
+  // source/dest
+  entity_inst_t get_source_inst() const {
+    return entity_inst_t(get_source(), get_source_addr());
+  }
+  entity_name_t get_source() const {
+    return entity_name_t(header.src);
+  }
+  entity_addr_t get_source_addr() const {
+    if (connection)
+      return connection->get_peer_addr();
+    return entity_addr_t();
+  }
+
+  // forwarded?
+  entity_inst_t get_orig_source_inst() const {
+    return get_source_inst();
+  }
+  entity_name_t get_orig_source() const {
+    return get_orig_source_inst().name;
+  }
+  entity_addr_t get_orig_source_addr() const {
+    return get_orig_source_inst().addr;
+  }
+
+  // virtual bits
+  virtual void decode_payload() = 0;
+  virtual void encode_payload(uint64_t features) = 0;
+  virtual const char *get_type_name() const = 0;
+  virtual void print(ostream& out) const {
+    out << get_type_name();
+  }
+
+  virtual void dump(Formatter *f) const;
+
+  void encode(uint64_t features, bool datacrc);
+};
+typedef boost::intrusive_ptr MessageRef;
+
+extern Message *decode_message(CephContext *cct, ceph_msg_header &header,
+			       ceph_msg_footer& footer, bufferlist& front,
+			       bufferlist& middle, bufferlist& data);
+inline ostream& operator<<(ostream& out, Message& m) {
+  m.print(out);
+  if (m.get_header().version)
+    out << " v" << m.get_header().version;
+  return out;
+}
+
+extern void encode_message(Message *m, uint64_t features, bufferlist& bl);
+extern Message *decode_message(CephContext *cct, bufferlist::iterator& bl);
+
+#endif
diff --git a/ceph/src/msg/Messenger.cc b/ceph/src/msg/Messenger.cc
new file mode 100644
index 00000000..b80782de
--- /dev/null
+++ b/ceph/src/msg/Messenger.cc
@@ -0,0 +1,13 @@
+
+#include "include/types.h"
+#include "Messenger.h"
+
+#include "SimpleMessenger.h"
+
+Messenger *Messenger::create(CephContext *cct,
+			     entity_name_t name,
+			     string lname,
+			     uint64_t nonce)
+{
+  return new SimpleMessenger(cct, name, lname, nonce);
+}
diff --git a/ceph/src/msg/Messenger.h b/ceph/src/msg/Messenger.h
new file mode 100644
index 00000000..42feaf22
--- /dev/null
+++ b/ceph/src/msg/Messenger.h
@@ -0,0 +1,699 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+
+#ifndef CEPH_MESSENGER_H
+#define CEPH_MESSENGER_H
+
+#include 
+using namespace std;
+
+#include "Message.h"
+#include "Dispatcher.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "include/Context.h"
+#include "include/types.h"
+#include "include/ceph_features.h"
+#include "auth/Crypto.h"
+
+#include 
+#include 
+
+class MDS;
+class Timer;
+
+
+class Messenger {
+private:
+  list dispatchers;
+
+protected:
+  /// the "name" of the local daemon. eg client.99
+  entity_inst_t my_inst;
+  int default_send_priority;
+  /// set to true once the Messenger has started, and set to false on shutdown
+  bool started;
+
+public:
+  /**
+   *  The CephContext this Messenger uses. Many other components initialize themselves
+   *  from this value.
+   */
+  CephContext *cct;
+
+  /**
+   * A Policy describes the rules of a Connection. Is there a limit on how
+   * much data this Connection can have locally? When the underlying connection
+   * experiences an error, does the Connection disappear? Can this Messenger
+   * re-establish the underlying connection?
+   */
+  struct Policy {
+    /// If true, the Connection is tossed out on errors.
+    bool lossy;
+    /// If true, the underlying connection can't be re-established from this end.
+    bool server;
+    /// If true, we will standby when idle
+    bool standby;
+    /// If true, we will try to detect session resets
+    bool resetcheck;
+    /**
+     *  The throttler is used to limit how much data is held by Messages from
+     *  the associated Connection(s). When reading in a new Message, the Messenger
+     *  will call throttler->throttle() for the size of the new Message.
+     */
+    Throttle *throttler_bytes;
+    Throttle *throttler_messages;
+
+    /// Specify features supported locally by the endpoint.
+    uint64_t features_supported;
+    /// Specify features any remotes must have to talk to this endpoint.
+    uint64_t features_required;
+
+    Policy()
+      : lossy(false), server(false), standby(false), resetcheck(true),
+	throttler_bytes(NULL),
+	throttler_messages(NULL),
+	features_supported(CEPH_FEATURES_SUPPORTED_DEFAULT),
+	features_required(0) {}
+  private:
+    Policy(bool l, bool s, bool st, bool r, uint64_t sup, uint64_t req)
+      : lossy(l), server(s), standby(st), resetcheck(r),
+	throttler_bytes(NULL),
+	throttler_messages(NULL),
+	features_supported(sup | CEPH_FEATURES_SUPPORTED_DEFAULT),
+	features_required(req) {}
+
+  public:
+    static Policy stateful_server(uint64_t sup, uint64_t req) {
+      return Policy(false, true, true, true, sup, req);
+    }
+    static Policy stateless_server(uint64_t sup, uint64_t req) {
+      return Policy(true, true, false, false, sup, req);
+    }
+    static Policy lossless_peer(uint64_t sup, uint64_t req) {
+      return Policy(false, false, true, false, sup, req);
+    }
+    static Policy lossless_peer_reuse(uint64_t sup, uint64_t req) {
+      return Policy(false, false, true, true, sup, req);
+    }
+    static Policy lossy_client(uint64_t sup, uint64_t req) {
+      return Policy(true, false, false, false, sup, req);
+    }
+    static Policy lossless_client(uint64_t sup, uint64_t req) {
+      return Policy(false, false, false, true, sup, req);
+    }
+  };
+
+  /**
+   * Messenger constructor. Call this from your implementation.
+   * Messenger users should construct full implementations directly,
+   * or use the create() function.
+   */
+  Messenger(CephContext *cct_, entity_name_t w)
+    : my_inst(),
+      default_send_priority(CEPH_MSG_PRIO_DEFAULT), started(false),
+      cct(cct_)
+  {
+    my_inst.name = w;
+  }
+  virtual ~Messenger() {}
+
+  /**
+   * create a new messenger
+   *
+   * Create a new messenger instance, with whatever implementation is
+   * available or specified via the configuration in cct.
+   *
+   * @param cct context
+   * @param name entity name to register
+   * @param lname logical name of the messenger in this process (e.g., "client")
+   * @param nonce nonce value to uniquely identify this instance on the current host
+   */
+  static Messenger *create(CephContext *cct,
+                           entity_name_t name,
+			   string lname,
+                           uint64_t nonce);
+
+  /**
+   * @defgroup Accessors
+   * @{
+   */
+  /**
+   * Retrieve the Messenger's instance.
+   *
+   * @return A const reference to the instance this Messenger
+   * currently believes to be its own.
+   */
+  const entity_inst_t& get_myinst() { return my_inst; }
+  /**
+   * set messenger's instance
+   */
+  void set_myinst(entity_inst_t i) { my_inst = i; }
+  /**
+   * Retrieve the Messenger's address.
+   *
+   * @return A const reference to the address this Messenger
+   * currently believes to be its own.
+   */
+  const entity_addr_t& get_myaddr() { return my_inst.addr; }
+protected:
+  /**
+   * set messenger's address
+   */
+  void set_myaddr(const entity_addr_t& a) { my_inst.addr = a; }
+public:
+  /**
+   * Retrieve the Messenger's name.
+   *
+   * @return A const reference to the name this Messenger
+   * currently believes to be its own.
+   */
+  const entity_name_t& get_myname() { return my_inst.name; }
+  /**
+   * Set the name of the local entity. The name is reported to others and
+   * can be changed while the system is running, but doing so at incorrect
+   * times may have bad results.
+   *
+   * @param m The name to set.
+   */
+  void set_myname(const entity_name_t& m) { my_inst.name = m; }
+  /**
+   * Set the unknown address components for this Messenger.
+   * This is useful if the Messenger doesn't know its full address just by
+   * binding, but another Messenger on the same interface has already learned
+   * its full address. This function does not fill in known address elements,
+   * cause a rebind, or do anything of that sort.
+   *
+   * @param addr The address to use as a template.
+   */
+  virtual void set_addr_unknowns(entity_addr_t &addr) = 0;
+  /// Get the default send priority.
+  int get_default_send_priority() { return default_send_priority; }
+  /**
+   * Get the number of Messages which the Messenger has received
+   * but not yet dispatched.
+   */
+  virtual int get_dispatch_queue_len() = 0;
+
+  /**
+   * Get age of oldest undelivered message
+   * (0 if the queue is empty)
+   */
+  virtual double get_dispatch_queue_max_age(utime_t now) = 0;
+
+  /**
+   * @} // Accessors
+   */
+
+  /**
+   * @defgroup Configuration
+   * @{
+   */
+  /**
+   * Set the cluster protocol in use by this daemon.
+   * This is an init-time function and cannot be called after calling
+   * start() or bind().
+   *
+   * @param p The cluster protocol to use. Defined externally.
+   */
+  virtual void set_cluster_protocol(int p) = 0;
+  /**
+   * Set a policy which is applied to all peers who do not have a type-specific
+   * Policy.
+   * This is an init-time function and cannot be called after calling
+   * start() or bind().
+   *
+   * @param p The Policy to apply.
+   */
+  virtual void set_default_policy(Policy p) = 0;
+  /**
+   * Set a policy which is applied to all peers of the given type.
+   * This is an init-time function and cannot be called after calling
+   * start() or bind().
+   *
+   * @param type The peer type this policy applies to.
+   * @param p The policy to apply.
+   */
+  virtual void set_policy(int type, Policy p) = 0;
+  /**
+   * Set the Policy associated with a type of peer.
+   *
+   * This can be called either on initial setup, or after connections
+   * are already established.  However, the policies for existing
+   * connections will not be affected; the new policy will only apply
+   * to future connections.
+   *
+   * @param t The peer type to get the default policy for.
+   * @return A const Policy reference.
+   */
+  virtual Policy get_policy(int t) = 0;
+  /**
+   * Get the default Policy
+   *
+   * @return A const Policy reference.
+   */
+  virtual Policy get_default_policy() = 0;
+  /**
+   * Set a Throttler which is applied to all Messages from the given
+   * type of peer.
+   *
+   * This is an init-time function and cannot be called after calling
+   * start() or bind().
+   *
+   * @param type The peer type this Throttler will apply to.
+   * @param t The Throttler to apply. The Messenger does not take
+   * ownership of this pointer, but you must not destroy it before
+   * you destroy the Messenger.
+   */
+  virtual void set_policy_throttlers(int type, Throttle *bytes, Throttle *msgs=NULL) = 0;
+  /**
+   * Set the default send priority
+   *
+   * This is an init-time function and must be called *before* calling
+   * start().
+   *
+   * @param p The cluster protocol to use. Defined externally.
+   */
+  void set_default_send_priority(int p) {
+    assert(!started);
+    default_send_priority = p;
+  }
+  /**
+   * Add a new Dispatcher to the front of the list. If you add
+   * a Dispatcher which is already included, it will get a duplicate
+   * entry. This will reduce efficiency but not break anything.
+   *
+   * @param d The Dispatcher to insert into the list.
+   */
+  void add_dispatcher_head(Dispatcher *d) { 
+    bool first = dispatchers.empty();
+    dispatchers.push_front(d);
+    if (first)
+      ready();
+  }
+  /**
+   * Add a new Dispatcher to the end of the list. If you add
+   * a Dispatcher which is already included, it will get a duplicate
+   * entry. This will reduce efficiency but not break anything.
+   *
+   * @param d The Dispatcher to insert into the list.
+   */
+  void add_dispatcher_tail(Dispatcher *d) { 
+    bool first = dispatchers.empty();
+    dispatchers.push_back(d);
+    if (first)
+      ready();
+  }
+  /**
+   * Bind the Messenger to a specific address. If bind_addr
+   * is not completely filled in the system will use the
+   * valid portions and cycle through the unset ones (eg, the port)
+   * in an unspecified order.
+   *
+   * @param bind_addr The address to bind to.
+   * @return 0 on success, or -1 on error, or -errno if
+   * we can be more specific about the failure.
+   */
+  virtual int bind(const entity_addr_t& bind_addr) = 0;
+  /**
+   * This function performs a full restart of the Messenger component,
+   * whatever that means.  Other entities who connect to this
+   * Messenger post-rebind() should perceive it as a new entity which
+   * they have not previously contacted, and it MUST bind to a
+   * different address than it did previously.
+   *
+   * @param avoid_ports Additional port to avoid binding to.
+   */
+  virtual int rebind(const set& avoid_ports) { return -EOPNOTSUPP; }
+  /**
+   * @} // Configuration
+   */
+
+  /**
+   * @defgroup Startup/Shutdown
+   * @{
+   */
+  /**
+   * Perform any resource allocation, thread startup, etc
+   * that is required before attempting to connect to other
+   * Messengers or transmit messages.
+   * Once this function completes, started shall be set to true.
+   *
+   * @return 0 on success; -errno on failure.
+   */
+  virtual int start() { started = true; return 0; }
+
+  // shutdown
+  /**
+   * Block until the Messenger has finished shutting down (according
+   * to the shutdown() function).
+   * It is valid to call this after calling shutdown(), but it must
+   * be called before deleting the Messenger.
+   */
+  virtual void wait() = 0;
+  /**
+   * Initiate a shutdown of the Messenger.
+   *
+   * @return 0 on success, -errno otherwise.
+   */
+  virtual int shutdown() { started = false; return 0; }
+  /**
+   * @} // Startup/Shutdown
+   */
+
+  /**
+   * @defgroup Messaging
+   * @{
+   */
+  /**
+   * Queue the given Message for the given entity.
+   * Success in this function does not guarantee Message delivery, only
+   * success in queueing the Message. Other guarantees may be provided based
+   * on the Connection policy associated with the dest.
+   *
+   * @param m The Message to send. The Messenger consumes a single reference
+   * when you pass it in.
+   * @param dest The entity to send the Message to.
+   *
+   * DEPRECATED: please do not use this interface for any new code;
+   * use the Connection* variant.
+   *
+   * @return 0 on success, or -errno on failure.
+   */
+  virtual int send_message(Message *m, const entity_inst_t& dest) = 0;
+  /**
+   * Queue the given Message to send out on the given Connection.
+   * Success in this function does not guarantee Message delivery, only
+   * success in queueing the Message. Other guarantees may be provided based
+   * on the Connection policy.
+   *
+   * @param m The Message to send. The Messenger consumes a single reference
+   * when you pass it in.
+   * @param con The Connection to send the Message out on.
+   *
+   * @return 0 on success, or -errno on failure.
+   */
+  virtual int send_message(Message *m, Connection *con) = 0;
+  int send_message(Message *m, const ConnectionRef& con) {
+    return send_message(m, con.get());
+  }
+  /**
+   * Lazily queue the given Message for the given entity. Unlike with
+   * send_message(), lazy_send_message() will not establish a
+   * Connection if none exists, re-establish the connection if it
+   * has broken, or queue the Message if the connection is broken.
+   *
+   * @param m The Message to send. The Messenger consumes a single reference
+   * when you pass it in.
+   * @param dest The entity to send the Message to.
+   *
+   * DEPRECATED: please do not use this interface for any new code;
+   * use the Connection* variant.
+   *
+   * @return 0.
+   */
+  virtual int lazy_send_message(Message *m, const entity_inst_t& dest) = 0;
+  /**
+   * Lazily queue the given Message for the given Connection. Unlike with
+   * send_message(), lazy_send_message() does not necessarily re-establish
+   * the connection if it has broken, or even queue the Message if the
+   * connection is broken.
+   *
+   * @param m The Message to send. The Messenger consumes a single reference
+   * when you pass it in.
+   * @param dest The entity to send the Message to.
+   *
+   * @return 0.
+   */
+  virtual int lazy_send_message(Message *m, Connection *con) = 0;
+
+  /**
+   * @} // Messaging
+   */
+  /**
+   * @defgroup Connection Management
+   * @{
+   */
+  /**
+   * Get the Connection object associated with a given entity. If a
+   * Connection does not exist, create one and establish a logical connection.
+   * The caller owns a reference when this returns. Call ->put() when you're
+   * done!
+   *
+   * @param dest The entity to get a connection for.
+   */
+  virtual ConnectionRef get_connection(const entity_inst_t& dest) = 0;
+  /**
+   * Get the Connection object associated with ourselves.
+   */
+  virtual ConnectionRef get_loopback_connection() = 0;
+  /**
+   * Send a "keepalive" ping to the given dest, if it has a working Connection.
+   * If the Messenger doesn't already have a Connection, or if the underlying
+   * connection has broken, this function does nothing.
+   *
+   * @param dest The entity to send the keepalive to.
+   * @return 0, or implementation-defined error numbers.
+   */
+  virtual int send_keepalive(const entity_inst_t& dest) = 0;
+  /**
+   * Send a "keepalive" ping along the given Connection, if it's working.
+   * If the underlying connection has broken, this function does nothing.
+   *
+   * @param dest The entity to send the keepalive to.
+   * @return 0, or implementation-defined error numbers.
+   */
+  virtual int send_keepalive(Connection *con) = 0;
+  /**
+   * Mark down a Connection to a remote.
+   *
+   * This will cause us to discard our outgoing queue for them, and if
+   * reset detection is enabled in the policy and the endpoint tries
+   * to reconnect they will discard their queue when we inform them of
+   * the session reset.
+   *
+   * If there is no Connection to the given dest, it is a no-op.
+   *
+   * This generates a RESET notification to the Dispatcher.
+   *
+   * DEPRECATED: please do not use this interface for any new code;
+   * use the Connection* variant.
+   *
+   * @param a The address to mark down.
+   */
+  virtual void mark_down(const entity_addr_t& a) = 0;
+  /**
+   * Mark down the given Connection.
+   *
+   * This will cause us to discard its outgoing queue, and if reset
+   * detection is enabled in the policy and the endpoint tries to
+   * reconnect they will discard their queue when we inform them of
+   * the session reset.
+   *
+   * If the Connection* is NULL, this is a no-op.
+   *
+   * It does not generate any notifications to the Dispatcher.
+   *
+   * @param con The Connection to mark down.
+   */
+  virtual void mark_down(Connection *con) = 0;
+  void mark_down(const ConnectionRef& con) {
+    mark_down(con.get());
+  }
+  /**
+   * Mark all the existing Connections down. This is equivalent
+   * to iterating over all Connections and calling mark_down()
+   * on each.
+   *
+   * This will generate a RESET event for each closed connections.
+   */
+  virtual void mark_down_all() = 0;
+  /**
+   * Unlike mark_down, this function will try and deliver
+   * all messages before ending the connection, and it will use
+   * the Pipe's existing semantics to do so. Once the Messages
+   * all been sent out the Connection will be closed and
+   * generate an ms_handle_reset notification to the
+   * Dispatcher.
+   * This function means that you will get standard delivery to endpoints,
+   * and then the Connection will be cleaned up.
+   *
+   * @param con The Connection to mark down.
+   */
+  virtual void mark_down_on_empty(Connection *con) = 0;
+  /**
+   * Mark a Connection as "disposable", setting it to lossy
+   * (regardless of initial Policy). Unlike mark_down_on_empty()
+   * this does not immediately close the Connection once
+   * Messages have been delivered, so as long as there are no errors you can
+   * continue to receive responses; but it will not attempt
+   * to reconnect for message delivery or preserve your old
+   * delivery semantics, either.
+   *
+   * TODO: There's some odd stuff going on in our SimpleMessenger
+   * implementation during connect that looks unused; is there
+   * more of a contract that that's enforcing?
+   *
+   * @param con The Connection to mark as disposable.
+   */
+  virtual void mark_disposable(Connection *con) = 0;
+  /**
+   * @} // Connection Management
+   */
+protected:
+  /**
+   * @defgroup Subclass Interfacing
+   * @{
+   */
+  /**
+   * A courtesy function for Messenger implementations which
+   * will be called when we receive our first Dispatcher.
+   */
+  virtual void ready() { }
+  /**
+   * @} // Subclass Interfacing
+   */
+  /**
+   * @defgroup Dispatcher Interfacing
+   * @{
+   */
+public:
+  /**
+   *  Deliver a single Message. Send it to each Dispatcher
+   *  in sequence until one of them handles it.
+   *  If none of our Dispatchers can handle it, assert(0).
+   *
+   *  @param m The Message to deliver. We take ownership of
+   *  one reference to it.
+   */
+  void ms_deliver_dispatch(Message *m) {
+    m->set_dispatch_stamp(ceph_clock_now(cct));
+    for (list::iterator p = dispatchers.begin();
+	 p != dispatchers.end();
+	 ++p) {
+      if ((*p)->ms_dispatch(m))
+	return;
+    }
+    lsubdout(cct, ms, 0) << "ms_deliver_dispatch: unhandled message " << m << " " << *m << " from "
+			 << m->get_source_inst() << dendl;
+    assert(!cct->_conf->ms_die_on_unhandled_msg);
+    m->put();
+  }
+  /**
+   * Notify each Dispatcher of a new Connection. Call
+   * this function whenever a new Connection is initiated.
+   *
+   * @param con Pointer to the new Connection.
+   */
+  void ms_deliver_handle_connect(Connection *con) {
+    for (list::iterator p = dispatchers.begin();
+	 p != dispatchers.end();
+	 ++p)
+      (*p)->ms_handle_connect(con);
+  }
+
+  /**
+   * Notify each Dispatcher of a new incomming Connection. Call
+   * this function whenever a new Connection is accepted.
+   *
+   * @param con Pointer to the new Connection.
+   */
+  void ms_deliver_handle_accept(Connection *con) {
+    for (list::iterator p = dispatchers.begin();
+	 p != dispatchers.end();
+	 ++p)
+      (*p)->ms_handle_accept(con);
+  }
+
+  /**
+   * Notify each Dispatcher of a Connection which may have lost
+   * Messages. Call this function whenever you detect that a lossy Connection
+   * has been disconnected.
+   *
+   * @param con Pointer to the broken Connection.
+   */
+  void ms_deliver_handle_reset(Connection *con) {
+    for (list::iterator p = dispatchers.begin();
+	 p != dispatchers.end();
+	 ++p) {
+      if ((*p)->ms_handle_reset(con))
+	return;
+    }
+  }
+  /**
+   * Notify each Dispatcher of a Connection which has been "forgotten" about
+   * by the remote end, implying that messages have probably been lost.
+   * Call this function whenever you detect a reset.
+   *
+   * @param con Pointer to the broken Connection.
+   */
+  void ms_deliver_handle_remote_reset(Connection *con) {
+    for (list::iterator p = dispatchers.begin();
+	 p != dispatchers.end();
+	 ++p)
+      (*p)->ms_handle_remote_reset(con);
+  }
+  /**
+   * Get the AuthAuthorizer for a new outgoing Connection.
+   *
+   * @param peer_type The peer type for the new Connection
+   * @param force_new True if we want to wait for new keys, false otherwise.
+   * @return A pointer to the AuthAuthorizer, if we have one; NULL otherwise
+   */
+  AuthAuthorizer *ms_deliver_get_authorizer(int peer_type, bool force_new) {
+    AuthAuthorizer *a = 0;
+    for (list::iterator p = dispatchers.begin();
+	 p != dispatchers.end();
+	 ++p) {
+      if ((*p)->ms_get_authorizer(peer_type, &a, force_new))
+	return a;
+    }
+    return NULL;
+  }
+  /**
+   * Verify that the authorizer on a new incoming Connection is correct.
+   *
+   * @param con The new incoming Connection
+   * @param peer_type The type of the endpoint on the new Connection
+   * @param protocol The ID of the protocol in use (at time of writing, cephx or none)
+   * @param authorizer The authorization string supplied by the remote
+   * @param authorizer_reply Output param: The string we should send back to
+   * the remote to authorize ourselves. Only filled in if isvalid
+   * @param isvalid Output param: True if authorizer is valid, false otherwise
+   *
+   * @return True if we were able to prove or disprove correctness of
+   * authorizer, false otherwise.
+   */
+  bool ms_deliver_verify_authorizer(Connection *con, int peer_type,
+				    int protocol, bufferlist& authorizer, bufferlist& authorizer_reply,
+				    bool& isvalid, CryptoKey& session_key) {
+    for (list::iterator p = dispatchers.begin();
+	 p != dispatchers.end();
+	 ++p) {
+      if ((*p)->ms_verify_authorizer(con, peer_type, protocol, authorizer, authorizer_reply, isvalid, session_key))
+	return true;
+    }
+    return false;
+  }
+
+  /**
+   * @} // Dispatcher Interfacing
+   */
+};
+
+
+
+#endif
diff --git a/ceph/src/msg/Pipe.cc b/ceph/src/msg/Pipe.cc
new file mode 100644
index 00000000..54ce6799
--- /dev/null
+++ b/ceph/src/msg/Pipe.cc
@@ -0,0 +1,2358 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "Message.h"
+#include "Pipe.h"
+#include "SimpleMessenger.h"
+
+#include "common/debug.h"
+#include "common/errno.h"
+
+// Below included to get encode_encrypt(); That probably should be in Crypto.h, instead
+
+#include "auth/Crypto.h"
+#include "auth/cephx/CephxProtocol.h"
+#include "auth/AuthSessionHandler.h"
+
+// Constant to limit starting sequence number to 2^31.  Nothing special about it, just a big number.  PLR
+#define SEQ_MASK  0x7fffffff 
+#define dout_subsys ceph_subsys_ms
+
+#undef dout_prefix
+#define dout_prefix _pipe_prefix(_dout)
+ostream& Pipe::_pipe_prefix(std::ostream *_dout) {
+  return *_dout << "-- " << msgr->get_myinst().addr << " >> " << peer_addr << " pipe(" << this
+		<< " sd=" << sd << " :" << port
+		<< " s=" << state
+		<< " pgs=" << peer_global_seq
+		<< " cs=" << connect_seq
+		<< " l=" << policy.lossy
+		<< " c=" << connection_state
+		<< ").";
+}
+
+/*
+ * This optimization may not be available on all platforms (e.g. OSX).
+ * Apparently a similar approach based on TCP_CORK can be used.
+ */
+#ifndef MSG_MORE
+# define MSG_MORE 0
+#endif
+
+/*
+ * On BSD SO_NOSIGPIPE can be set via setsockopt to block SIGPIPE.
+ */
+#ifndef MSG_NOSIGNAL
+# define MSG_NOSIGNAL 0
+# ifdef SO_NOSIGPIPE
+#  define CEPH_USE_SO_NOSIGPIPE
+# else
+#  error "Cannot block SIGPIPE!"
+# endif
+#endif
+
+/**************************************
+ * Pipe
+ */
+
+Pipe::Pipe(SimpleMessenger *r, int st, Connection *con)
+  : reader_thread(this), writer_thread(this),
+    delay_thread(NULL),
+    msgr(r),
+    conn_id(r->dispatch_queue.get_id()),
+    sd(-1), port(0),
+    peer_type(-1),
+    pipe_lock("SimpleMessenger::Pipe::pipe_lock"),
+    state(st),
+    connection_state(NULL),
+    reader_running(false), reader_needs_join(false),
+    writer_running(false),
+    in_q(&(r->dispatch_queue)),
+    send_keepalive(false),
+    send_keepalive_ack(false),
+    close_on_empty(false),
+    connect_seq(0), peer_global_seq(0),
+    out_seq(0), in_seq(0), in_seq_acked(0) {
+  if (con) {
+    connection_state = con;
+    connection_state->reset_pipe(this);
+  } else {
+    connection_state = new Connection(msgr);
+    connection_state->pipe = get();
+  }
+
+  if (randomize_out_seq()) {
+    lsubdout(msgr->cct,ms,15) << "Pipe(): Could not get random bytes to set seq number for session reset; set seq number to " << out_seq << dendl;
+  }
+    
+
+  msgr->timeout = msgr->cct->_conf->ms_tcp_read_timeout * 1000; //convert to ms
+  if (msgr->timeout == 0)
+    msgr->timeout = -1;
+}
+
+Pipe::~Pipe()
+{
+  assert(out_q.empty());
+  assert(sent.empty());
+  delete delay_thread;
+}
+
+void Pipe::handle_ack(uint64_t seq)
+{
+  lsubdout(msgr->cct, ms, 15) << "reader got ack seq " << seq << dendl;
+  // trim sent list
+  while (!sent.empty() &&
+	 sent.front()->get_seq() <= seq) {
+    Message *m = sent.front();
+    sent.pop_front();
+    lsubdout(msgr->cct, ms, 10) << "reader got ack seq "
+				<< seq << " >= " << m->get_seq() << " on " << m << " " << *m << dendl;
+    m->put();
+  }
+
+  if (sent.empty() && close_on_empty) {
+    lsubdout(msgr->cct, ms, 10) << "reader got last ack, queue empty, closing" << dendl;
+    stop();
+  }
+}
+
+void Pipe::start_reader()
+{
+  assert(pipe_lock.is_locked());
+  assert(!reader_running);
+  if (reader_needs_join) {
+    reader_thread.join();
+    reader_needs_join = false;
+  }
+  reader_running = true;
+  reader_thread.create(msgr->cct->_conf->ms_rwthread_stack_bytes);
+}
+
+void Pipe::maybe_start_delay_thread()
+{
+  if (!delay_thread &&
+      msgr->cct->_conf->ms_inject_delay_type.find(ceph_entity_type_name(connection_state->peer_type)) != string::npos) {
+    lsubdout(msgr->cct, ms, 1) << "setting up a delay queue on Pipe " << this << dendl;
+    delay_thread = new DelayedDelivery(this);
+    delay_thread->create();
+  }
+}
+
+void Pipe::start_writer()
+{
+  assert(pipe_lock.is_locked());
+  assert(!writer_running);
+  writer_running = true;
+  writer_thread.create(msgr->cct->_conf->ms_rwthread_stack_bytes);
+}
+
+void Pipe::join_reader()
+{
+  if (!reader_running)
+    return;
+  cond.Signal();
+  pipe_lock.Unlock();
+  reader_thread.join();
+  pipe_lock.Lock();
+  reader_needs_join = false;
+}
+
+void Pipe::DelayedDelivery::discard()
+{
+  lgeneric_subdout(pipe->msgr->cct, ms, 20) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::discard" << dendl;
+  Mutex::Locker l(delay_lock);
+  while (!delay_queue.empty()) {
+    Message *m = delay_queue.front().second;
+    pipe->msgr->dispatch_throttle_release(m->get_dispatch_throttle_size());
+    m->put();
+    delay_queue.pop_front();
+  }
+}
+
+void Pipe::DelayedDelivery::flush()
+{
+  lgeneric_subdout(pipe->msgr->cct, ms, 20) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::flush" << dendl;
+  Mutex::Locker l(delay_lock);
+  while (!delay_queue.empty()) {
+    Message *m = delay_queue.front().second;
+    delay_queue.pop_front();
+    pipe->in_q->enqueue(m, m->get_priority(), pipe->conn_id);
+  }
+}
+
+void *Pipe::DelayedDelivery::entry()
+{
+  Mutex::Locker locker(delay_lock);
+  lgeneric_subdout(pipe->msgr->cct, ms, 20) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::entry start" << dendl;
+
+  while (!stop_delayed_delivery) {
+    if (delay_queue.empty()) {
+      lgeneric_subdout(pipe->msgr->cct, ms, 30) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::entry sleeping on delay_cond because delay queue is empty" << dendl;
+      delay_cond.Wait(delay_lock);
+      continue;
+    }
+    utime_t release = delay_queue.front().first;
+    Message *m = delay_queue.front().second;
+    string delay_msg_type = pipe->msgr->cct->_conf->ms_inject_delay_msg_type;
+    if (release > ceph_clock_now(pipe->msgr->cct) &&
+	(delay_msg_type.empty() || m->get_type_name() == delay_msg_type)) {
+      lgeneric_subdout(pipe->msgr->cct, ms, 10) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::entry sleeping on delay_cond until " << release << dendl;
+      delay_cond.WaitUntil(delay_lock, release);
+      continue;
+    }
+    lgeneric_subdout(pipe->msgr->cct, ms, 10) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::entry dequeuing message " << m << " for delivery, past " << release << dendl;
+    delay_queue.pop_front();
+    pipe->in_q->enqueue(m, m->get_priority(), pipe->conn_id);
+  }
+  lgeneric_subdout(pipe->msgr->cct, ms, 20) << pipe->_pipe_prefix(_dout) << "DelayedDelivery::entry stop" << dendl;
+  return NULL;
+}
+
+int Pipe::accept()
+{
+  ldout(msgr->cct,10) << "accept" << dendl;
+  assert(pipe_lock.is_locked());
+  assert(state == STATE_ACCEPTING);
+
+  pipe_lock.Unlock();
+
+  // vars
+  bufferlist addrs;
+  entity_addr_t socket_addr;
+  socklen_t len;
+  int r;
+  char banner[strlen(CEPH_BANNER)+1];
+  bufferlist addrbl;
+  ceph_msg_connect connect;
+  ceph_msg_connect_reply reply;
+  Pipe *existing = 0;
+  bufferptr bp;
+  bufferlist authorizer, authorizer_reply;
+  bool authorizer_valid;
+  uint64_t feat_missing;
+  bool replaced = false;
+  // this variable denotes if the connection attempt from peer is a hard 
+  // reset or not, it is true if there is an existing connection and the
+  // connection sequence from peer is equal to zero
+  bool is_reset_from_peer = false;
+  CryptoKey session_key;
+  int removed; // single-use down below
+
+  // this should roughly mirror pseudocode at
+  //  http://ceph.newdream.net/wiki/Messaging_protocol
+  int reply_tag = 0;
+  uint64_t existing_seq = -1;
+
+  // used for reading in the remote acked seq on connect
+  uint64_t newly_acked_seq = 0;
+
+  set_socket_options();
+
+  // announce myself.
+  r = tcp_write(CEPH_BANNER, strlen(CEPH_BANNER));
+  if (r < 0) {
+    ldout(msgr->cct,10) << "accept couldn't write banner" << dendl;
+    goto fail_unlocked;
+  }
+
+  // and my addr
+  ::encode(msgr->my_inst.addr, addrs);
+
+  port = msgr->my_inst.addr.get_port();
+
+  // and peer's socket addr (they might not know their ip)
+  len = sizeof(socket_addr.ss_addr());
+  r = ::getpeername(sd, (sockaddr*)&socket_addr.ss_addr(), &len);
+  if (r < 0) {
+    ldout(msgr->cct,0) << "accept failed to getpeername " << cpp_strerror(errno) << dendl;
+    goto fail_unlocked;
+  }
+  ::encode(socket_addr, addrs);
+
+  r = tcp_write(addrs.c_str(), addrs.length());
+  if (r < 0) {
+    ldout(msgr->cct,10) << "accept couldn't write my+peer addr" << dendl;
+    goto fail_unlocked;
+  }
+
+  ldout(msgr->cct,1) << "accept sd=" << sd << " " << socket_addr << dendl;
+  
+  // identify peer
+  if (tcp_read(banner, strlen(CEPH_BANNER)) < 0) {
+    ldout(msgr->cct,10) << "accept couldn't read banner" << dendl;
+    goto fail_unlocked;
+  }
+  if (memcmp(banner, CEPH_BANNER, strlen(CEPH_BANNER))) {
+    banner[strlen(CEPH_BANNER)] = 0;
+    ldout(msgr->cct,1) << "accept peer sent bad banner '" << banner << "' (should be '" << CEPH_BANNER << "')" << dendl;
+    goto fail_unlocked;
+  }
+  {
+    bufferptr tp(sizeof(peer_addr));
+    addrbl.push_back(tp);
+  }
+  if (tcp_read(addrbl.c_str(), addrbl.length()) < 0) {
+    ldout(msgr->cct,10) << "accept couldn't read peer_addr" << dendl;
+    goto fail_unlocked;
+  }
+  {
+    bufferlist::iterator ti = addrbl.begin();
+    ::decode(peer_addr, ti);
+  }
+
+  ldout(msgr->cct,10) << "accept peer addr is " << peer_addr << dendl;
+  if (peer_addr.is_blank_ip()) {
+    // peer apparently doesn't know what ip they have; figure it out for them.
+    int port = peer_addr.get_port();
+    peer_addr.addr = socket_addr.addr;
+    peer_addr.set_port(port);
+    ldout(msgr->cct,0) << "accept peer addr is really " << peer_addr
+	    << " (socket is " << socket_addr << ")" << dendl;
+  }
+  set_peer_addr(peer_addr);  // so that connection_state gets set up
+  
+  while (1) {
+    if (tcp_read((char*)&connect, sizeof(connect)) < 0) {
+      ldout(msgr->cct,10) << "accept couldn't read connect" << dendl;
+      goto fail_unlocked;
+    }
+
+    // sanitize features
+    connect.features = ceph_sanitize_features(connect.features);
+
+    authorizer.clear();
+    if (connect.authorizer_len) {
+      bp = buffer::create(connect.authorizer_len);
+      if (tcp_read(bp.c_str(), connect.authorizer_len) < 0) {
+        ldout(msgr->cct,10) << "accept couldn't read connect authorizer" << dendl;
+        goto fail_unlocked;
+      }
+      authorizer.push_back(bp);
+      authorizer_reply.clear();
+    }
+
+    ldout(msgr->cct,20) << "accept got peer connect_seq " << connect.connect_seq
+	     << " global_seq " << connect.global_seq
+	     << dendl;
+    
+    msgr->lock.Lock();   // FIXME
+    pipe_lock.Lock();
+    if (msgr->dispatch_queue.stop)
+      goto shutting_down;
+    if (state != STATE_ACCEPTING) {
+      goto shutting_down;
+    }
+
+    // note peer's type, flags
+    set_peer_type(connect.host_type);
+    policy = msgr->get_policy(connect.host_type);
+    ldout(msgr->cct,10) << "accept of host_type " << connect.host_type
+			<< ", policy.lossy=" << policy.lossy
+			<< " policy.server=" << policy.server
+			<< " policy.standby=" << policy.standby
+			<< " policy.resetcheck=" << policy.resetcheck
+			<< dendl;
+
+    memset(&reply, 0, sizeof(reply));
+    reply.protocol_version = msgr->get_proto_version(peer_type, false);
+    msgr->lock.Unlock();
+
+    // mismatch?
+    ldout(msgr->cct,10) << "accept my proto " << reply.protocol_version
+	     << ", their proto " << connect.protocol_version << dendl;
+    if (connect.protocol_version != reply.protocol_version) {
+      reply.tag = CEPH_MSGR_TAG_BADPROTOVER;
+      goto reply;
+    }
+
+    // require signatures for cephx?
+    if (connect.authorizer_protocol == CEPH_AUTH_CEPHX) {
+      if (peer_type == CEPH_ENTITY_TYPE_OSD ||
+	  peer_type == CEPH_ENTITY_TYPE_MDS) {
+	if (msgr->cct->_conf->cephx_require_signatures ||
+	    msgr->cct->_conf->cephx_cluster_require_signatures) {
+	  ldout(msgr->cct,10) << "using cephx, requiring MSG_AUTH feature bit for cluster" << dendl;
+	  policy.features_required |= CEPH_FEATURE_MSG_AUTH;
+	}
+      } else {
+	if (msgr->cct->_conf->cephx_require_signatures ||
+	    msgr->cct->_conf->cephx_service_require_signatures) {
+	  ldout(msgr->cct,10) << "using cephx, requiring MSG_AUTH feature bit for service" << dendl;
+	  policy.features_required |= CEPH_FEATURE_MSG_AUTH;
+	}
+      }
+    }
+
+    feat_missing = policy.features_required & ~(uint64_t)connect.features;
+    if (feat_missing) {
+      ldout(msgr->cct,1) << "peer missing required features " << std::hex << feat_missing << std::dec << dendl;
+      reply.tag = CEPH_MSGR_TAG_FEATURES;
+      goto reply;
+    }
+    
+    // Check the authorizer.  If not good, bail out.
+
+    pipe_lock.Unlock();
+
+    if (!msgr->verify_authorizer(connection_state.get(), peer_type, connect.authorizer_protocol, authorizer,
+				 authorizer_reply, authorizer_valid, session_key) ||
+	!authorizer_valid) {
+      ldout(msgr->cct,0) << "accept: got bad authorizer" << dendl;
+      pipe_lock.Lock();
+      if (state != STATE_ACCEPTING)
+	goto shutting_down_msgr_unlocked;
+      reply.tag = CEPH_MSGR_TAG_BADAUTHORIZER;
+      session_security.reset();
+      goto reply;
+    } 
+
+    // We've verified the authorizer for this pipe, so set up the session security structure.  PLR
+
+    ldout(msgr->cct,10) << "accept:  setting up session_security." << dendl;
+
+    msgr->lock.Lock();
+    pipe_lock.Lock();
+    if (msgr->dispatch_queue.stop)
+      goto shutting_down;
+    if (state != STATE_ACCEPTING)
+      goto shutting_down;
+    
+    // existing?
+    existing = msgr->_lookup_pipe(peer_addr);
+    if (existing) {
+      existing->pipe_lock.Lock(true);  // skip lockdep check (we are locking a second Pipe here)
+
+      if (connect.global_seq < existing->peer_global_seq) {
+	ldout(msgr->cct,10) << "accept existing " << existing << ".gseq " << existing->peer_global_seq
+		 << " > " << connect.global_seq << ", RETRY_GLOBAL" << dendl;
+	reply.tag = CEPH_MSGR_TAG_RETRY_GLOBAL;
+	reply.global_seq = existing->peer_global_seq;  // so we can send it below..
+	existing->pipe_lock.Unlock();
+	msgr->lock.Unlock();
+	goto reply;
+      } else {
+	ldout(msgr->cct,10) << "accept existing " << existing << ".gseq " << existing->peer_global_seq
+		 << " <= " << connect.global_seq << ", looks ok" << dendl;
+      }
+      
+      if (existing->policy.lossy) {
+	ldout(msgr->cct,0) << "accept replacing existing (lossy) channel (new one lossy="
+	        << policy.lossy << ")" << dendl;
+	existing->was_session_reset();
+	goto replace;
+      }
+
+      ldout(msgr->cct,0) << "accept connect_seq " << connect.connect_seq
+			 << " vs existing " << existing->connect_seq
+			 << " state " << existing->get_state_name() << dendl;
+
+      if (connect.connect_seq == 0 && existing->connect_seq > 0) {
+	ldout(msgr->cct,0) << "accept peer reset, then tried to connect to us, replacing" << dendl;
+        // this is a hard reset from peer
+        is_reset_from_peer = true;
+	if (policy.resetcheck)
+	  existing->was_session_reset(); // this resets out_queue, msg_ and connect_seq #'s
+	goto replace;
+      }
+
+      if (connect.connect_seq < existing->connect_seq) {
+	// old attempt, or we sent READY but they didn't get it.
+	ldout(msgr->cct,10) << "accept existing " << existing << ".cseq " << existing->connect_seq
+			    << " > " << connect.connect_seq << ", RETRY_SESSION" << dendl;
+	goto retry_session;
+      }
+
+      if (connect.connect_seq == existing->connect_seq) {
+	// if the existing connection successfully opened, and/or
+	// subsequently went to standby, then the peer should bump
+	// their connect_seq and retry: this is not a connection race
+	// we need to resolve here.
+	if (existing->state == STATE_OPEN ||
+	    existing->state == STATE_STANDBY) {
+	  ldout(msgr->cct,10) << "accept connection race, existing " << existing
+			      << ".cseq " << existing->connect_seq
+			      << " == " << connect.connect_seq
+			      << ", OPEN|STANDBY, RETRY_SESSION" << dendl;
+	  goto retry_session;
+	}
+
+	// connection race?
+	if (peer_addr < msgr->my_inst.addr ||
+	    existing->policy.server) {
+	  // incoming wins
+	  ldout(msgr->cct,10) << "accept connection race, existing " << existing << ".cseq " << existing->connect_seq
+		   << " == " << connect.connect_seq << ", or we are server, replacing my attempt" << dendl;
+	  if (!(existing->state == STATE_CONNECTING ||
+		existing->state == STATE_WAIT))
+	    lderr(msgr->cct) << "accept race bad state, would replace, existing="
+			     << existing->get_state_name()
+			     << " " << existing << ".cseq=" << existing->connect_seq
+			     << " == " << connect.connect_seq
+			     << dendl;
+	  assert(existing->state == STATE_CONNECTING ||
+		 existing->state == STATE_WAIT);
+	  goto replace;
+	} else {
+	  // our existing outgoing wins
+	  ldout(msgr->cct,10) << "accept connection race, existing " << existing << ".cseq " << existing->connect_seq
+		   << " == " << connect.connect_seq << ", sending WAIT" << dendl;
+	  assert(peer_addr > msgr->my_inst.addr);
+	  if (!(existing->state == STATE_CONNECTING))
+	    lderr(msgr->cct) << "accept race bad state, would send wait, existing="
+			     << existing->get_state_name()
+			     << " " << existing << ".cseq=" << existing->connect_seq
+			     << " == " << connect.connect_seq
+			     << dendl;
+	  assert(existing->state == STATE_CONNECTING);
+	  // make sure our outgoing connection will follow through
+	  existing->_send_keepalive();
+	  reply.tag = CEPH_MSGR_TAG_WAIT;
+	  existing->pipe_lock.Unlock();
+	  msgr->lock.Unlock();
+	  goto reply;
+	}
+      }
+
+      assert(connect.connect_seq > existing->connect_seq);
+      assert(connect.global_seq >= existing->peer_global_seq);
+      if (policy.resetcheck &&   // RESETSESSION only used by servers; peers do not reset each other
+	  existing->connect_seq == 0) {
+	ldout(msgr->cct,0) << "accept we reset (peer sent cseq " << connect.connect_seq 
+		 << ", " << existing << ".cseq = " << existing->connect_seq
+		 << "), sending RESETSESSION" << dendl;
+	reply.tag = CEPH_MSGR_TAG_RESETSESSION;
+	msgr->lock.Unlock();
+	existing->pipe_lock.Unlock();
+	goto reply;
+      }
+
+      // reconnect
+      ldout(msgr->cct,10) << "accept peer sent cseq " << connect.connect_seq
+	       << " > " << existing->connect_seq << dendl;
+      goto replace;
+    } // existing
+    else if (policy.resetcheck && connect.connect_seq > 0) {
+      // we reset, and they are opening a new session
+      ldout(msgr->cct,0) << "accept we reset (peer sent cseq " << connect.connect_seq << "), sending RESETSESSION" << dendl;
+      msgr->lock.Unlock();
+      reply.tag = CEPH_MSGR_TAG_RESETSESSION;
+      goto reply;
+    } else {
+      // new session
+      ldout(msgr->cct,10) << "accept new session" << dendl;
+      existing = NULL;
+      goto open;
+    }
+    assert(0);
+
+  retry_session:
+    assert(existing->pipe_lock.is_locked());
+    assert(pipe_lock.is_locked());
+    reply.tag = CEPH_MSGR_TAG_RETRY_SESSION;
+    reply.connect_seq = existing->connect_seq + 1;
+    existing->pipe_lock.Unlock();
+    msgr->lock.Unlock();
+    goto reply;    
+
+  reply:
+    assert(pipe_lock.is_locked());
+    reply.features = ((uint64_t)connect.features & policy.features_supported) | policy.features_required;
+    reply.authorizer_len = authorizer_reply.length();
+    pipe_lock.Unlock();
+    r = tcp_write((char*)&reply, sizeof(reply));
+    if (r < 0)
+      goto fail_unlocked;
+    if (reply.authorizer_len) {
+      r = tcp_write(authorizer_reply.c_str(), authorizer_reply.length());
+      if (r < 0)
+	goto fail_unlocked;
+    }
+  }
+  
+ replace:
+  assert(existing->pipe_lock.is_locked());
+  assert(pipe_lock.is_locked());
+  // if it is a hard reset from peer, we don't need a round-trip to negotiate in/out sequence
+  if ((connect.features & CEPH_FEATURE_RECONNECT_SEQ) && !is_reset_from_peer) {
+    reply_tag = CEPH_MSGR_TAG_SEQ;
+    existing_seq = existing->in_seq;
+  }
+  ldout(msgr->cct,10) << "accept replacing " << existing << dendl;
+  existing->stop();
+  existing->unregister_pipe();
+  replaced = true;
+
+  if (existing->policy.lossy) {
+    // disconnect from the Connection
+    assert(existing->connection_state);
+    if (existing->connection_state->clear_pipe(existing))
+      msgr->dispatch_queue.queue_reset(existing->connection_state.get());
+  } else {
+    // queue a reset on the old connection
+    msgr->dispatch_queue.queue_reset(connection_state.get());
+
+    // drop my Connection, and take a ref to the existing one. do not
+    // clear existing->connection_state, since read_message and
+    // write_message both dereference it without pipe_lock.
+    connection_state = existing->connection_state;
+
+    // make existing Connection reference us
+    connection_state->reset_pipe(this);
+
+    // flush/queue any existing delayed messages
+    if (existing->delay_thread)
+      existing->delay_thread->flush();
+
+    // steal incoming queue
+    uint64_t replaced_conn_id = conn_id;
+    conn_id = existing->conn_id;
+    existing->conn_id = replaced_conn_id;
+
+    // reset the in_seq if this is a hard reset from peer,
+    // otherwise we respect our original connection's value
+    in_seq = is_reset_from_peer ? 0 : existing->in_seq;
+    in_seq_acked = in_seq;
+
+    // steal outgoing queue and out_seq
+    existing->requeue_sent();
+    out_seq = existing->out_seq;
+    ldout(msgr->cct,10) << "accept re-queuing on out_seq " << out_seq << " in_seq " << in_seq << dendl;
+    for (map >::iterator p = existing->out_q.begin();
+         p != existing->out_q.end();
+         ++p)
+      out_q[p->first].splice(out_q[p->first].begin(), p->second);
+  }
+  existing->pipe_lock.Unlock();
+
+ open:
+  // open
+  assert(pipe_lock.is_locked());
+  connect_seq = connect.connect_seq + 1;
+  peer_global_seq = connect.global_seq;
+  assert(state == STATE_ACCEPTING);
+  state = STATE_OPEN;
+  ldout(msgr->cct,10) << "accept success, connect_seq = " << connect_seq << ", sending READY" << dendl;
+
+  // send READY reply
+  reply.tag = (reply_tag ? reply_tag : CEPH_MSGR_TAG_READY);
+  reply.features = policy.features_supported;
+  reply.global_seq = msgr->get_global_seq();
+  reply.connect_seq = connect_seq;
+  reply.flags = 0;
+  reply.authorizer_len = authorizer_reply.length();
+  if (policy.lossy)
+    reply.flags = reply.flags | CEPH_MSG_CONNECT_LOSSY;
+
+  connection_state->set_features((uint64_t)reply.features & (uint64_t)connect.features);
+  ldout(msgr->cct,10) << "accept features " << connection_state->get_features() << dendl;
+
+  session_security.reset(
+      get_auth_session_handler(msgr->cct,
+			       connect.authorizer_protocol,
+			       session_key,
+			       connection_state->get_features()));
+
+  // notify
+  msgr->dispatch_queue.queue_accept(connection_state.get());
+
+  // ok!
+  if (msgr->dispatch_queue.stop)
+    goto shutting_down;
+  removed = msgr->accepting_pipes.erase(this);
+  assert(removed == 1);
+  register_pipe();
+  msgr->lock.Unlock();
+  pipe_lock.Unlock();
+
+  r = tcp_write((char*)&reply, sizeof(reply));
+  if (r < 0) {
+    goto fail_registered;
+  }
+
+  if (reply.authorizer_len) {
+    r = tcp_write(authorizer_reply.c_str(), authorizer_reply.length());
+    if (r < 0) {
+      goto fail_registered;
+    }
+  }
+
+  if (reply_tag == CEPH_MSGR_TAG_SEQ) {
+    if (tcp_write((char*)&existing_seq, sizeof(existing_seq)) < 0) {
+      ldout(msgr->cct,2) << "accept write error on in_seq" << dendl;
+      goto fail_registered;
+    }
+    if (tcp_read((char*)&newly_acked_seq, sizeof(newly_acked_seq)) < 0) {
+      ldout(msgr->cct,2) << "accept read error on newly_acked_seq" << dendl;
+      goto fail_registered;
+    }
+  }
+
+  pipe_lock.Lock();
+  discard_requeued_up_to(newly_acked_seq);
+  if (state != STATE_CLOSED) {
+    ldout(msgr->cct,10) << "accept starting writer, state " << get_state_name() << dendl;
+    start_writer();
+  }
+  ldout(msgr->cct,20) << "accept done" << dendl;
+
+  maybe_start_delay_thread();
+
+  return 0;   // success.
+
+ fail_registered:
+  ldout(msgr->cct, 10) << "accept fault after register" << dendl;
+
+  if (msgr->cct->_conf->ms_inject_internal_delays) {
+    ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+    utime_t t;
+    t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+    t.sleep();
+  }
+
+ fail_unlocked:
+  pipe_lock.Lock();
+  if (state != STATE_CLOSED) {
+    bool queued = is_queued();
+    ldout(msgr->cct, 10) << "  queued = " << (int)queued << dendl;
+    if (queued) {
+      state = policy.server ? STATE_STANDBY : STATE_CONNECTING;
+    } else if (replaced) {
+      state = STATE_STANDBY;
+    } else {
+      state = STATE_CLOSED;
+      state_closed.set(1);
+    }
+    fault();
+    if (queued || replaced)
+      start_writer();
+  }
+  return -1;
+
+ shutting_down:
+  msgr->lock.Unlock();
+ shutting_down_msgr_unlocked:
+  assert(pipe_lock.is_locked());
+
+  if (msgr->cct->_conf->ms_inject_internal_delays) {
+    ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+    utime_t t;
+    t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+    t.sleep();
+  }
+
+  state = STATE_CLOSED;
+  state_closed.set(1);
+  fault();
+  return -1;
+}
+
+void Pipe::set_socket_options()
+{
+  // disable Nagle algorithm?
+  if (msgr->cct->_conf->ms_tcp_nodelay) {
+    int flag = 1;
+    int r = ::setsockopt(sd, IPPROTO_TCP, TCP_NODELAY, (char*)&flag, sizeof(flag));
+    if (r < 0) {
+      r = -errno;
+      ldout(msgr->cct,0) << "couldn't set TCP_NODELAY: " << cpp_strerror(r) << dendl;
+    }
+  }
+  if (msgr->cct->_conf->ms_tcp_rcvbuf) {
+    int size = msgr->cct->_conf->ms_tcp_rcvbuf;
+    int r = ::setsockopt(sd, SOL_SOCKET, SO_RCVBUF, (void*)&size, sizeof(size));
+    if (r < 0)  {
+      r = -errno;
+      ldout(msgr->cct,0) << "couldn't set SO_RCVBUF to " << size << ": " << cpp_strerror(r) << dendl;
+    }
+  }
+
+  // block ESIGPIPE
+#ifdef CEPH_USE_SO_NOSIGPIPE
+  int val = 1;
+  int r = ::setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, (void*)&val, sizeof(val));
+  if (r) {
+    r = -errno;
+    ldout(msgr->cct,0) << "couldn't set SO_NOSIGPIPE: " << cpp_strerror(r) << dendl;
+  }
+#endif
+}
+
+int Pipe::connect()
+{
+  bool got_bad_auth = false;
+
+  ldout(msgr->cct,10) << "connect " << connect_seq << dendl;
+  assert(pipe_lock.is_locked());
+
+  __u32 cseq = connect_seq;
+  __u32 gseq = msgr->get_global_seq();
+
+  // stop reader thrad
+  join_reader();
+
+  pipe_lock.Unlock();
+  
+  char tag = -1;
+  int rc;
+  struct msghdr msg;
+  struct iovec msgvec[2];
+  int msglen;
+  char banner[strlen(CEPH_BANNER) + 1];  // extra byte makes coverity happy
+  entity_addr_t paddr;
+  entity_addr_t peer_addr_for_me, socket_addr;
+  AuthAuthorizer *authorizer = NULL;
+  bufferlist addrbl, myaddrbl;
+  const md_config_t *conf = msgr->cct->_conf;
+
+  // close old socket.  this is safe because we stopped the reader thread above.
+  if (sd >= 0)
+    ::close(sd);
+
+  // create socket?
+  sd = ::socket(peer_addr.get_family(), SOCK_STREAM, 0);
+  if (sd < 0) {
+    lderr(msgr->cct) << "connect couldn't created socket " << cpp_strerror(errno) << dendl;
+    goto fail;
+  }
+
+  // connect!
+  ldout(msgr->cct,10) << "connecting to " << peer_addr << dendl;
+  rc = ::connect(sd, (sockaddr*)&peer_addr.addr, peer_addr.addr_size());
+  if (rc < 0) {
+    ldout(msgr->cct,2) << "connect error " << peer_addr
+	     << ", " << cpp_strerror(errno) << dendl;
+    goto fail;
+  }
+
+  set_socket_options();
+
+  // verify banner
+  // FIXME: this should be non-blocking, or in some other way verify the banner as we get it.
+  if (tcp_read((char*)&banner, strlen(CEPH_BANNER)) < 0) {
+    ldout(msgr->cct,2) << "connect couldn't read banner, " << cpp_strerror(errno) << dendl;
+    goto fail;
+  }
+  if (memcmp(banner, CEPH_BANNER, strlen(CEPH_BANNER))) {
+    ldout(msgr->cct,0) << "connect protocol error (bad banner) on peer " << paddr << dendl;
+    goto fail;
+  }
+
+  memset(&msg, 0, sizeof(msg));
+  msgvec[0].iov_base = banner;
+  msgvec[0].iov_len = strlen(CEPH_BANNER);
+  msg.msg_iov = msgvec;
+  msg.msg_iovlen = 1;
+  msglen = msgvec[0].iov_len;
+  if (do_sendmsg(&msg, msglen)) {
+    ldout(msgr->cct,2) << "connect couldn't write my banner, " << cpp_strerror(errno) << dendl;
+    goto fail;
+  }
+
+  // identify peer
+  {
+    bufferptr p(sizeof(paddr) * 2);
+    addrbl.push_back(p);
+  }
+  if (tcp_read(addrbl.c_str(), addrbl.length()) < 0) {
+    ldout(msgr->cct,2) << "connect couldn't read peer addrs, " << cpp_strerror(errno) << dendl;
+    goto fail;
+  }
+  {
+    bufferlist::iterator p = addrbl.begin();
+    ::decode(paddr, p);
+    ::decode(peer_addr_for_me, p);
+    port = peer_addr_for_me.get_port();
+  }
+
+  ldout(msgr->cct,20) << "connect read peer addr " << paddr << " on socket " << sd << dendl;
+  if (peer_addr != paddr) {
+    if (paddr.is_blank_ip() &&
+	peer_addr.get_port() == paddr.get_port() &&
+	peer_addr.get_nonce() == paddr.get_nonce()) {
+      ldout(msgr->cct,0) << "connect claims to be " 
+	      << paddr << " not " << peer_addr << " - presumably this is the same node!" << dendl;
+    } else {
+      ldout(msgr->cct,0) << "connect claims to be " 
+	      << paddr << " not " << peer_addr << " - wrong node!" << dendl;
+      goto fail;
+    }
+  }
+
+  ldout(msgr->cct,20) << "connect peer addr for me is " << peer_addr_for_me << dendl;
+
+  msgr->learned_addr(peer_addr_for_me);
+
+  ::encode(msgr->my_inst.addr, myaddrbl);
+
+  memset(&msg, 0, sizeof(msg));
+  msgvec[0].iov_base = myaddrbl.c_str();
+  msgvec[0].iov_len = myaddrbl.length();
+  msg.msg_iov = msgvec;
+  msg.msg_iovlen = 1;
+  msglen = msgvec[0].iov_len;
+  if (do_sendmsg(&msg, msglen)) {
+    ldout(msgr->cct,2) << "connect couldn't write my addr, " << cpp_strerror(errno) << dendl;
+    goto fail;
+  }
+  ldout(msgr->cct,10) << "connect sent my addr " << msgr->my_inst.addr << dendl;
+
+
+  while (1) {
+    delete authorizer;
+    authorizer = msgr->get_authorizer(peer_type, false);
+    bufferlist authorizer_reply;
+
+    ceph_msg_connect connect;
+    connect.features = policy.features_supported;
+    connect.host_type = msgr->my_type;
+    connect.global_seq = gseq;
+    connect.connect_seq = cseq;
+    connect.protocol_version = msgr->get_proto_version(peer_type, true);
+    connect.authorizer_protocol = authorizer ? authorizer->protocol : 0;
+    connect.authorizer_len = authorizer ? authorizer->bl.length() : 0;
+    if (authorizer) 
+      ldout(msgr->cct,10) << "connect.authorizer_len=" << connect.authorizer_len
+	       << " protocol=" << connect.authorizer_protocol << dendl;
+    connect.flags = 0;
+    if (policy.lossy)
+      connect.flags |= CEPH_MSG_CONNECT_LOSSY;  // this is fyi, actually, server decides!
+    memset(&msg, 0, sizeof(msg));
+    msgvec[0].iov_base = (char*)&connect;
+    msgvec[0].iov_len = sizeof(connect);
+    msg.msg_iov = msgvec;
+    msg.msg_iovlen = 1;
+    msglen = msgvec[0].iov_len;
+    if (authorizer) {
+      msgvec[1].iov_base = authorizer->bl.c_str();
+      msgvec[1].iov_len = authorizer->bl.length();
+      msg.msg_iovlen++;
+      msglen += msgvec[1].iov_len;
+    }
+
+    ldout(msgr->cct,10) << "connect sending gseq=" << gseq << " cseq=" << cseq
+	     << " proto=" << connect.protocol_version << dendl;
+    if (do_sendmsg(&msg, msglen)) {
+      ldout(msgr->cct,2) << "connect couldn't write gseq, cseq, " << cpp_strerror(errno) << dendl;
+      goto fail;
+    }
+
+    ldout(msgr->cct,20) << "connect wrote (self +) cseq, waiting for reply" << dendl;
+    ceph_msg_connect_reply reply;
+    if (tcp_read((char*)&reply, sizeof(reply)) < 0) {
+      ldout(msgr->cct,2) << "connect read reply " << cpp_strerror(errno) << dendl;
+      goto fail;
+    }
+
+    // sanitize features
+    reply.features = ceph_sanitize_features(reply.features);
+
+    ldout(msgr->cct,20) << "connect got reply tag " << (int)reply.tag
+			<< " connect_seq " << reply.connect_seq
+			<< " global_seq " << reply.global_seq
+			<< " proto " << reply.protocol_version
+			<< " flags " << (int)reply.flags
+			<< " features " << reply.features
+			<< dendl;
+
+    authorizer_reply.clear();
+
+    if (reply.authorizer_len) {
+      ldout(msgr->cct,10) << "reply.authorizer_len=" << reply.authorizer_len << dendl;
+      bufferptr bp = buffer::create(reply.authorizer_len);
+      if (tcp_read(bp.c_str(), reply.authorizer_len) < 0) {
+        ldout(msgr->cct,10) << "connect couldn't read connect authorizer_reply" << dendl;
+	goto fail;
+      }
+      authorizer_reply.push_back(bp);
+    }
+
+    if (authorizer) {
+      bufferlist::iterator iter = authorizer_reply.begin();
+      if (!authorizer->verify_reply(iter)) {
+        ldout(msgr->cct,0) << "failed verifying authorize reply" << dendl;
+	goto fail;
+      }
+    }
+
+    if (conf->ms_inject_internal_delays) {
+      ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+      utime_t t;
+      t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+      t.sleep();
+    }
+
+    pipe_lock.Lock();
+    if (state != STATE_CONNECTING) {
+      ldout(msgr->cct,0) << "connect got RESETSESSION but no longer connecting" << dendl;
+      goto stop_locked;
+    }
+
+    if (reply.tag == CEPH_MSGR_TAG_FEATURES) {
+      ldout(msgr->cct,0) << "connect protocol feature mismatch, my " << std::hex
+	      << connect.features << " < peer " << reply.features
+	      << " missing " << (reply.features & ~policy.features_supported)
+	      << std::dec << dendl;
+      goto fail_locked;
+    }
+
+    if (reply.tag == CEPH_MSGR_TAG_BADPROTOVER) {
+      ldout(msgr->cct,0) << "connect protocol version mismatch, my " << connect.protocol_version
+	      << " != " << reply.protocol_version << dendl;
+      goto fail_locked;
+    }
+
+    if (reply.tag == CEPH_MSGR_TAG_BADAUTHORIZER) {
+      ldout(msgr->cct,0) << "connect got BADAUTHORIZER" << dendl;
+      if (got_bad_auth)
+        goto stop_locked;
+      got_bad_auth = true;
+      pipe_lock.Unlock();
+      delete authorizer;
+      authorizer = msgr->get_authorizer(peer_type, true);  // try harder
+      continue;
+    }
+    if (reply.tag == CEPH_MSGR_TAG_RESETSESSION) {
+      ldout(msgr->cct,0) << "connect got RESETSESSION" << dendl;
+      was_session_reset();
+      cseq = 0;
+      pipe_lock.Unlock();
+      continue;
+    }
+    if (reply.tag == CEPH_MSGR_TAG_RETRY_GLOBAL) {
+      gseq = msgr->get_global_seq(reply.global_seq);
+      ldout(msgr->cct,10) << "connect got RETRY_GLOBAL " << reply.global_seq
+	       << " chose new " << gseq << dendl;
+      pipe_lock.Unlock();
+      continue;
+    }
+    if (reply.tag == CEPH_MSGR_TAG_RETRY_SESSION) {
+      assert(reply.connect_seq > connect_seq);
+      ldout(msgr->cct,10) << "connect got RETRY_SESSION " << connect_seq
+	       << " -> " << reply.connect_seq << dendl;
+      cseq = connect_seq = reply.connect_seq;
+      pipe_lock.Unlock();
+      continue;
+    }
+
+    if (reply.tag == CEPH_MSGR_TAG_WAIT) {
+      ldout(msgr->cct,3) << "connect got WAIT (connection race)" << dendl;
+      state = STATE_WAIT;
+      goto stop_locked;
+    }
+
+    if (reply.tag == CEPH_MSGR_TAG_READY ||
+        reply.tag == CEPH_MSGR_TAG_SEQ) {
+      uint64_t feat_missing = policy.features_required & ~(uint64_t)reply.features;
+      if (feat_missing) {
+	ldout(msgr->cct,1) << "missing required features " << std::hex << feat_missing << std::dec << dendl;
+	goto fail_locked;
+      }
+
+      if (reply.tag == CEPH_MSGR_TAG_SEQ) {
+        ldout(msgr->cct,10) << "got CEPH_MSGR_TAG_SEQ, reading acked_seq and writing in_seq" << dendl;
+        uint64_t newly_acked_seq = 0;
+        if (tcp_read((char*)&newly_acked_seq, sizeof(newly_acked_seq)) < 0) {
+          ldout(msgr->cct,2) << "connect read error on newly_acked_seq" << dendl;
+          goto fail_locked;
+        }
+	ldout(msgr->cct,2) << " got newly_acked_seq " << newly_acked_seq
+			   << " vs out_seq " << out_seq << dendl;
+	while (newly_acked_seq > out_seq) {
+	  Message *m = _get_next_outgoing();
+	  assert(m);
+	  ldout(msgr->cct,2) << " discarding previously sent " << m->get_seq()
+			     << " " << *m << dendl;
+	  assert(m->get_seq() <= newly_acked_seq);
+	  m->put();
+	  ++out_seq;
+	}
+        if (tcp_write((char*)&in_seq, sizeof(in_seq)) < 0) {
+          ldout(msgr->cct,2) << "connect write error on in_seq" << dendl;
+          goto fail_locked;
+        }
+      }
+
+      // hooray!
+      peer_global_seq = reply.global_seq;
+      policy.lossy = reply.flags & CEPH_MSG_CONNECT_LOSSY;
+      state = STATE_OPEN;
+      connect_seq = cseq + 1;
+      assert(connect_seq == reply.connect_seq);
+      backoff = utime_t();
+      connection_state->set_features((uint64_t)reply.features & (uint64_t)connect.features);
+      ldout(msgr->cct,10) << "connect success " << connect_seq << ", lossy = " << policy.lossy
+	       << ", features " << connection_state->get_features() << dendl;
+      
+
+      // If we have an authorizer, get a new AuthSessionHandler to deal with ongoing security of the
+      // connection.  PLR
+
+      if (authorizer != NULL) {
+	session_security.reset(
+            get_auth_session_handler(msgr->cct,
+				     authorizer->protocol,
+				     authorizer->session_key,
+				     connection_state->get_features()));
+      }  else {
+        // We have no authorizer, so we shouldn't be applying security to messages in this pipe.  PLR
+	session_security.reset();
+      }
+
+      msgr->dispatch_queue.queue_connect(connection_state.get());
+      
+      if (!reader_running) {
+	ldout(msgr->cct,20) << "connect starting reader" << dendl;
+	start_reader();
+      }
+      maybe_start_delay_thread();
+      delete authorizer;
+      return 0;
+    }
+    
+    // protocol error
+    ldout(msgr->cct,0) << "connect got bad tag " << (int)tag << dendl;
+    goto fail_locked;
+  }
+
+ fail:
+  if (conf->ms_inject_internal_delays) {
+    ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+    utime_t t;
+    t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+    t.sleep();
+  }
+
+  pipe_lock.Lock();
+ fail_locked:
+  if (state == STATE_CONNECTING)
+    fault();
+  else
+    ldout(msgr->cct,3) << "connect fault, but state = " << get_state_name()
+		       << " != connecting, stopping" << dendl;
+
+ stop_locked:
+  delete authorizer;
+  return -1;
+}
+
+void Pipe::register_pipe()
+{
+  ldout(msgr->cct,10) << "register_pipe" << dendl;
+  assert(msgr->lock.is_locked());
+  Pipe *existing = msgr->_lookup_pipe(peer_addr);
+  assert(existing == NULL);
+  msgr->rank_pipe[peer_addr] = this;
+}
+
+void Pipe::unregister_pipe()
+{
+  assert(msgr->lock.is_locked());
+  ceph::unordered_map::iterator p = msgr->rank_pipe.find(peer_addr);
+  if (p != msgr->rank_pipe.end() && p->second == this) {
+    ldout(msgr->cct,10) << "unregister_pipe" << dendl;
+    msgr->rank_pipe.erase(p);
+  } else {
+    ldout(msgr->cct,10) << "unregister_pipe - not registered" << dendl;
+    msgr->accepting_pipes.erase(this);  // somewhat overkill, but safe.
+  }
+}
+
+void Pipe::join()
+{
+  ldout(msgr->cct, 20) << "join" << dendl;
+  if (writer_thread.is_started())
+    writer_thread.join();
+  if (reader_thread.is_started())
+    reader_thread.join();
+  if (delay_thread) {
+    ldout(msgr->cct, 20) << "joining delay_thread" << dendl;
+    delay_thread->stop();
+    delay_thread->join();
+  }
+}
+
+void Pipe::requeue_sent()
+{
+  if (sent.empty())
+    return;
+
+  list& rq = out_q[CEPH_MSG_PRIO_HIGHEST];
+  while (!sent.empty()) {
+    Message *m = sent.back();
+    sent.pop_back();
+    ldout(msgr->cct,10) << "requeue_sent " << *m << " for resend seq " << out_seq
+			<< " (" << m->get_seq() << ")" << dendl;
+    rq.push_front(m);
+    out_seq--;
+  }
+}
+
+void Pipe::discard_requeued_up_to(uint64_t seq)
+{
+  ldout(msgr->cct, 10) << "discard_requeued_up_to " << seq << dendl;
+  if (out_q.count(CEPH_MSG_PRIO_HIGHEST) == 0)
+    return;
+  list& rq = out_q[CEPH_MSG_PRIO_HIGHEST];
+  while (!rq.empty()) {
+    Message *m = rq.front();
+    if (m->get_seq() == 0 || m->get_seq() > seq)
+      break;
+    ldout(msgr->cct,10) << "discard_requeued_up_to " << *m << " for resend seq " << out_seq
+			<< " <= " << seq << ", discarding" << dendl;
+    m->put();
+    rq.pop_front();
+    out_seq++;
+  }
+  if (rq.empty())
+    out_q.erase(CEPH_MSG_PRIO_HIGHEST);
+}
+
+/*
+ * Tears down the Pipe's message queues, and removes them from the DispatchQueue
+ * Must hold pipe_lock prior to calling.
+ */
+void Pipe::discard_out_queue()
+{
+  ldout(msgr->cct,10) << "discard_queue" << dendl;
+
+  for (list::iterator p = sent.begin(); p != sent.end(); ++p) {
+    ldout(msgr->cct,20) << "  discard " << *p << dendl;
+    (*p)->put();
+  }
+  sent.clear();
+  for (map >::iterator p = out_q.begin(); p != out_q.end(); ++p)
+    for (list::iterator r = p->second.begin(); r != p->second.end(); ++r) {
+      ldout(msgr->cct,20) << "  discard " << *r << dendl;
+      (*r)->put();
+    }
+  out_q.clear();
+}
+
+void Pipe::fault(bool onread)
+{
+  const md_config_t *conf = msgr->cct->_conf;
+  assert(pipe_lock.is_locked());
+  cond.Signal();
+
+  if (onread && state == STATE_CONNECTING) {
+    ldout(msgr->cct,10) << "fault already connecting, reader shutting down" << dendl;
+    return;
+  }
+  
+  ldout(msgr->cct,2) << "fault " << cpp_strerror(errno) << dendl;
+
+  if (state == STATE_CLOSED ||
+      state == STATE_CLOSING) {
+    ldout(msgr->cct,10) << "fault already closed|closing" << dendl;
+    if (connection_state->clear_pipe(this))
+      msgr->dispatch_queue.queue_reset(connection_state.get());
+    return;
+  }
+
+  shutdown_socket();
+
+  // lossy channel?
+  if (policy.lossy && state != STATE_CONNECTING) {
+    ldout(msgr->cct,10) << "fault on lossy channel, failing" << dendl;
+
+    stop();
+
+    // crib locks, blech.  note that Pipe is now STATE_CLOSED and the
+    // rank_pipe entry is ignored by others.
+    pipe_lock.Unlock();
+
+    if (conf->ms_inject_internal_delays) {
+      ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+      utime_t t;
+      t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+      t.sleep();
+    }
+
+    msgr->lock.Lock();
+    pipe_lock.Lock();
+    unregister_pipe();
+    msgr->lock.Unlock();
+
+    in_q->discard_queue(conn_id);
+    if (delay_thread)
+      delay_thread->discard();
+    discard_out_queue();
+
+    // disconnect from Connection, and mark it failed.  future messages
+    // will be dropped.
+    assert(connection_state);
+    if (connection_state->clear_pipe(this))
+      msgr->dispatch_queue.queue_reset(connection_state.get());
+    return;
+  }
+
+  // queue delayed items immediately
+  if (delay_thread)
+    delay_thread->flush();
+
+  // requeue sent items
+  requeue_sent();
+
+  if (policy.standby && !is_queued()) {
+    ldout(msgr->cct,0) << "fault with nothing to send, going to standby" << dendl;
+    state = STATE_STANDBY;
+    return;
+  }
+
+  if (state != STATE_CONNECTING) {
+    if (policy.server) {
+      ldout(msgr->cct,0) << "fault, server, going to standby" << dendl;
+      state = STATE_STANDBY;
+    } else {
+      ldout(msgr->cct,0) << "fault, initiating reconnect" << dendl;
+      connect_seq++;
+      state = STATE_CONNECTING;
+    }
+    backoff = utime_t();
+  } else if (backoff == utime_t()) {
+    ldout(msgr->cct,0) << "fault" << dendl;
+    backoff.set_from_double(conf->ms_initial_backoff);
+  } else {
+    ldout(msgr->cct,10) << "fault waiting " << backoff << dendl;
+    cond.WaitInterval(msgr->cct, pipe_lock, backoff);
+    backoff += backoff;
+    if (backoff > conf->ms_max_backoff)
+      backoff.set_from_double(conf->ms_max_backoff);
+    ldout(msgr->cct,10) << "fault done waiting or woke up" << dendl;
+  }
+}
+
+int Pipe::randomize_out_seq()
+{
+  if (connection_state->get_features() & CEPH_FEATURE_MSG_AUTH) {
+    // Set out_seq to a random value, so CRC won't be predictable.   Don't bother checking seq_error
+    // here.  We'll check it on the call.  PLR
+    int seq_error = get_random_bytes((char *)&out_seq, sizeof(out_seq));
+    out_seq &= SEQ_MASK;
+    lsubdout(msgr->cct, ms, 10) << "randomize_out_seq " << out_seq << dendl;
+    return seq_error;
+  } else {
+    // previously, seq #'s always started at 0.
+    out_seq = 0;
+    return 0;
+  }
+}
+
+void Pipe::was_session_reset()
+{
+  assert(pipe_lock.is_locked());
+
+  ldout(msgr->cct,10) << "was_session_reset" << dendl;
+  in_q->discard_queue(conn_id);
+  if (delay_thread)
+    delay_thread->discard();
+  discard_out_queue();
+
+  msgr->dispatch_queue.queue_remote_reset(connection_state.get());
+
+  if (randomize_out_seq()) {
+    lsubdout(msgr->cct,ms,15) << "was_session_reset(): Could not get random bytes to set seq number for session reset; set seq number to " << out_seq << dendl;
+  }
+
+  in_seq = 0;
+  connect_seq = 0;
+}
+
+void Pipe::stop()
+{
+  ldout(msgr->cct,10) << "stop" << dendl;
+  assert(pipe_lock.is_locked());
+  state = STATE_CLOSED;
+  state_closed.set(1);
+  cond.Signal();
+  shutdown_socket();
+}
+
+
+/* read msgs from socket.
+ * also, server.
+ */
+void Pipe::reader()
+{
+  pipe_lock.Lock();
+
+  if (state == STATE_ACCEPTING) {
+    accept();
+    assert(pipe_lock.is_locked());
+  }
+
+  // loop.
+  while (state != STATE_CLOSED &&
+	 state != STATE_CONNECTING) {
+    assert(pipe_lock.is_locked());
+
+    // sleep if (re)connecting
+    if (state == STATE_STANDBY) {
+      ldout(msgr->cct,20) << "reader sleeping during reconnect|standby" << dendl;
+      cond.Wait(pipe_lock);
+      continue;
+    }
+
+    // get a reference to the AuthSessionHandler while we have the pipe_lock
+    ceph::shared_ptr auth_handler = session_security;
+
+    pipe_lock.Unlock();
+
+    char tag = -1;
+    ldout(msgr->cct,20) << "reader reading tag..." << dendl;
+    if (tcp_read((char*)&tag, 1) < 0) {
+      pipe_lock.Lock();
+      ldout(msgr->cct,2) << "reader couldn't read tag, " << cpp_strerror(errno) << dendl;
+      fault(true);
+      continue;
+    }
+
+    if (tag == CEPH_MSGR_TAG_KEEPALIVE) {
+      ldout(msgr->cct,20) << "reader got KEEPALIVE" << dendl;
+      pipe_lock.Lock();
+      continue;
+    }
+    if (tag == CEPH_MSGR_TAG_KEEPALIVE2) {
+      ldout(msgr->cct,30) << "reader got KEEPALIVE2 tag ..." << dendl;
+      ceph_timespec t;
+      int rc = tcp_read((char*)&t, sizeof(t));
+      pipe_lock.Lock();
+      if (rc < 0) {
+	ldout(msgr->cct,2) << "reader couldn't read KEEPALIVE2 stamp "
+			   << cpp_strerror(errno) << dendl;
+	fault(true);
+      } else {
+	send_keepalive_ack = true;
+	keepalive_ack_stamp = utime_t(t);
+	ldout(msgr->cct,20) << "reader got KEEPALIVE2 " << keepalive_ack_stamp
+			    << dendl;
+	cond.Signal();
+      }
+      continue;
+    }
+    if (tag == CEPH_MSGR_TAG_KEEPALIVE2_ACK) {
+      ldout(msgr->cct,20) << "reader got KEEPALIVE_ACK" << dendl;
+      struct ceph_timespec t;
+      int rc = tcp_read((char*)&t, sizeof(t));
+      pipe_lock.Lock();
+      if (rc < 0) {
+	ldout(msgr->cct,2) << "reader couldn't read KEEPALIVE2 stamp " << cpp_strerror(errno) << dendl;
+	fault(true);
+      } else {
+	connection_state->last_keepalive_ack = utime_t(t);
+      }
+      continue;
+    }
+
+    // open ...
+    if (tag == CEPH_MSGR_TAG_ACK) {
+      ldout(msgr->cct,20) << "reader got ACK" << dendl;
+      ceph_le64 seq;
+      int rc = tcp_read((char*)&seq, sizeof(seq));
+      pipe_lock.Lock();
+      if (rc < 0) {
+	ldout(msgr->cct,2) << "reader couldn't read ack seq, " << cpp_strerror(errno) << dendl;
+	fault(true);
+      } else if (state != STATE_CLOSED) {
+        handle_ack(seq);
+      }
+      continue;
+    }
+
+    else if (tag == CEPH_MSGR_TAG_MSG) {
+      ldout(msgr->cct,20) << "reader got MSG" << dendl;
+      Message *m = 0;
+      int r = read_message(&m, auth_handler.get());
+
+      pipe_lock.Lock();
+      
+      if (!m) {
+	if (r < 0)
+	  fault(true);
+	continue;
+      }
+
+      if (state == STATE_CLOSED ||
+	  state == STATE_CONNECTING) {
+	msgr->dispatch_throttle_release(m->get_dispatch_throttle_size());
+	m->put();
+	continue;
+      }
+
+      // check received seq#.  if it is old, drop the message.  
+      // note that incoming messages may skip ahead.  this is convenient for the client
+      // side queueing because messages can't be renumbered, but the (kernel) client will
+      // occasionally pull a message out of the sent queue to send elsewhere.  in that case
+      // it doesn't matter if we "got" it or not.
+      if (m->get_seq() <= in_seq) {
+	ldout(msgr->cct,0) << "reader got old message "
+		<< m->get_seq() << " <= " << in_seq << " " << m << " " << *m
+		<< ", discarding" << dendl;
+	msgr->dispatch_throttle_release(m->get_dispatch_throttle_size());
+	m->put();
+	if (connection_state->has_feature(CEPH_FEATURE_RECONNECT_SEQ) &&
+	    msgr->cct->_conf->ms_die_on_old_message)
+	  assert(0 == "old msgs despite reconnect_seq feature");
+	continue;
+      }
+
+      m->set_connection(connection_state.get());
+
+      // note last received message.
+      in_seq = m->get_seq();
+
+      cond.Signal();  // wake up writer, to ack this
+      
+      ldout(msgr->cct,10) << "reader got message "
+	       << m->get_seq() << " " << m << " " << *m
+	       << dendl;
+
+      if (delay_thread) {
+	utime_t release;
+	if (rand() % 10000 < msgr->cct->_conf->ms_inject_delay_probability * 10000.0) {
+	  release = m->get_recv_stamp();
+	  release += msgr->cct->_conf->ms_inject_delay_max * (double)(rand() % 10000) / 10000.0;
+	  lsubdout(msgr->cct, ms, 1) << "queue_received will delay until " << release << " on " << m << " " << *m << dendl;
+	}
+	delay_thread->queue(release, m);
+      } else {
+	in_q->enqueue(m, m->get_priority(), conn_id);
+      }
+    } 
+    
+    else if (tag == CEPH_MSGR_TAG_CLOSE) {
+      ldout(msgr->cct,20) << "reader got CLOSE" << dendl;
+      pipe_lock.Lock();
+      if (state == STATE_CLOSING) {
+	state = STATE_CLOSED;
+	state_closed.set(1);
+      } else {
+	state = STATE_CLOSING;
+      }
+      cond.Signal();
+      break;
+    }
+    else {
+      ldout(msgr->cct,0) << "reader bad tag " << (int)tag << dendl;
+      pipe_lock.Lock();
+      fault(true);
+    }
+  }
+
+ 
+  // reap?
+  reader_running = false;
+  reader_needs_join = true;
+  unlock_maybe_reap();
+  ldout(msgr->cct,10) << "reader done" << dendl;
+}
+
+/* write msgs to socket.
+ * also, client.
+ */
+void Pipe::writer()
+{
+  pipe_lock.Lock();
+  while (state != STATE_CLOSED) {// && state != STATE_WAIT) {
+    ldout(msgr->cct,10) << "writer: state = " << get_state_name()
+			<< " policy.server=" << policy.server << dendl;
+
+    // standby?
+    if (is_queued() && state == STATE_STANDBY && !policy.server) {
+      connect_seq++;
+      state = STATE_CONNECTING;
+    }
+
+    // connect?
+    if (state == STATE_CONNECTING) {
+      assert(!policy.server);
+      connect();
+      continue;
+    }
+    
+    if (state == STATE_CLOSING) {
+      // write close tag
+      ldout(msgr->cct,20) << "writer writing CLOSE tag" << dendl;
+      char tag = CEPH_MSGR_TAG_CLOSE;
+      state = STATE_CLOSED;
+      state_closed.set(1);
+      pipe_lock.Unlock();
+      if (sd) {
+	int r = ::write(sd, &tag, 1);
+	// we can ignore r, actually; we don't care if this succeeds.
+	r++; r = 0; // placate gcc
+      }
+      pipe_lock.Lock();
+      continue;
+    }
+
+    if (state != STATE_CONNECTING && state != STATE_WAIT && state != STATE_STANDBY &&
+	(is_queued() || in_seq > in_seq_acked)) {
+
+      // keepalive?
+      if (send_keepalive) {
+	int rc;
+	if (connection_state->has_feature(CEPH_FEATURE_MSGR_KEEPALIVE2)) {
+	  pipe_lock.Unlock();
+	  rc = write_keepalive2(CEPH_MSGR_TAG_KEEPALIVE2,
+				ceph_clock_now(msgr->cct));
+	} else {
+	  pipe_lock.Unlock();
+	  rc = write_keepalive();
+	}
+	pipe_lock.Lock();
+	if (rc < 0) {
+	  ldout(msgr->cct,2) << "writer couldn't write keepalive[2], "
+			     << cpp_strerror(errno) << dendl;
+	  fault();
+ 	  continue;
+	}
+	send_keepalive = false;
+      }
+      if (send_keepalive_ack) {
+	utime_t t = keepalive_ack_stamp;
+	pipe_lock.Unlock();
+	int rc = write_keepalive2(CEPH_MSGR_TAG_KEEPALIVE2_ACK, t);
+	pipe_lock.Lock();
+	if (rc < 0) {
+	  ldout(msgr->cct,2) << "writer couldn't write keepalive_ack, " << cpp_strerror(errno) << dendl;
+	  fault();
+	  continue;
+	}
+	send_keepalive_ack = false;
+      }
+
+      // send ack?
+      if (in_seq > in_seq_acked) {
+	uint64_t send_seq = in_seq;
+	pipe_lock.Unlock();
+	int rc = write_ack(send_seq);
+	pipe_lock.Lock();
+	if (rc < 0) {
+	  ldout(msgr->cct,2) << "writer couldn't write ack, " << cpp_strerror(errno) << dendl;
+	  fault();
+ 	  continue;
+	}
+	in_seq_acked = send_seq;
+      }
+
+      // grab outgoing message
+      Message *m = _get_next_outgoing();
+      if (m) {
+	m->set_seq(++out_seq);
+	if (!policy.lossy || close_on_empty) {
+	  // put on sent list
+	  sent.push_back(m); 
+	  m->get();
+	}
+
+	// associate message with Connection (for benefit of encode_payload)
+	m->set_connection(connection_state.get());
+
+	uint64_t features = connection_state->get_features();
+	if (m->empty_payload())
+	  ldout(msgr->cct,20) << "writer encoding " << m->get_seq() << " features " << features
+			      << " " << m << " " << *m << dendl;
+	else
+	  ldout(msgr->cct,20) << "writer half-reencoding " << m->get_seq() << " features " << features
+			      << " " << m << " " << *m << dendl;
+
+	// encode and copy out of *m
+	m->encode(features, !msgr->cct->_conf->ms_nocrc);
+
+	// prepare everything
+	ceph_msg_header& header = m->get_header();
+	ceph_msg_footer& footer = m->get_footer();
+
+	// Now that we have all the crcs calculated, handle the
+	// digital signature for the message, if the pipe has session
+	// security set up.  Some session security options do not
+	// actually calculate and check the signature, but they should
+	// handle the calls to sign_message and check_signature.  PLR
+	if (session_security.get() == NULL) {
+	  ldout(msgr->cct, 20) << "writer no session security" << dendl;
+	} else {
+	  if (session_security->sign_message(m)) {
+	    ldout(msgr->cct, 20) << "writer failed to sign seq # " << header.seq
+				 << "): sig = " << footer.sig << dendl;
+	  } else {
+	    ldout(msgr->cct, 20) << "writer signed seq # " << header.seq
+				 << "): sig = " << footer.sig << dendl;
+	  }
+	}
+
+	bufferlist blist = m->get_payload();
+	blist.append(m->get_middle());
+	blist.append(m->get_data());
+
+	pipe_lock.Unlock();
+
+        ldout(msgr->cct,20) << "writer sending " << m->get_seq() << " " << m << dendl;
+	int rc = write_message(header, footer, blist);
+
+	pipe_lock.Lock();
+	if (rc < 0) {
+          ldout(msgr->cct,1) << "writer error sending " << m << ", "
+		  << cpp_strerror(errno) << dendl;
+	  fault();
+        }
+	m->put();
+      }
+      continue;
+    }
+    
+    if (sent.empty() && close_on_empty) {
+      ldout(msgr->cct,10) << "writer out and sent queues empty, closing" << dendl;
+      stop();
+      continue;
+    }
+
+    // wait
+    ldout(msgr->cct,20) << "writer sleeping" << dendl;
+    cond.Wait(pipe_lock);
+  }
+  
+  ldout(msgr->cct,20) << "writer finishing" << dendl;
+
+  // reap?
+  writer_running = false;
+  unlock_maybe_reap();
+  ldout(msgr->cct,10) << "writer done" << dendl;
+}
+
+void Pipe::unlock_maybe_reap()
+{
+  if (!reader_running && !writer_running) {
+    shutdown_socket();
+    pipe_lock.Unlock();
+    msgr->queue_reap(this);
+  } else {
+    pipe_lock.Unlock();
+  }
+}
+
+static void alloc_aligned_buffer(bufferlist& data, unsigned len, unsigned off)
+{
+  // create a buffer to read into that matches the data alignment
+  unsigned left = len;
+  if (off & ~CEPH_PAGE_MASK) {
+    // head
+    unsigned head = 0;
+    head = MIN(CEPH_PAGE_SIZE - (off & ~CEPH_PAGE_MASK), left);
+    bufferptr bp = buffer::create(head);
+    data.push_back(bp);
+    left -= head;
+  }
+  unsigned middle = left & CEPH_PAGE_MASK;
+  if (middle > 0) {
+    bufferptr bp = buffer::create_page_aligned(middle);
+    data.push_back(bp);
+    left -= middle;
+  }
+  if (left) {
+    bufferptr bp = buffer::create(left);
+    data.push_back(bp);
+  }
+}
+
+int Pipe::read_message(Message **pm, AuthSessionHandler* auth_handler)
+{
+  int ret = -1;
+  // envelope
+  //ldout(msgr->cct,10) << "receiver.read_message from sd " << sd  << dendl;
+  
+  ceph_msg_header header; 
+  ceph_msg_footer footer;
+  __u32 header_crc;
+  
+  if (connection_state->has_feature(CEPH_FEATURE_NOSRCADDR)) {
+    if (tcp_read((char*)&header, sizeof(header)) < 0)
+      return -1;
+    header_crc = ceph_crc32c(0, (unsigned char *)&header, sizeof(header) - sizeof(header.crc));
+  } else {
+    ceph_msg_header_old oldheader;
+    if (tcp_read((char*)&oldheader, sizeof(oldheader)) < 0)
+      return -1;
+    // this is fugly
+    memcpy(&header, &oldheader, sizeof(header));
+    header.src = oldheader.src.name;
+    header.reserved = oldheader.reserved;
+    header.crc = oldheader.crc;
+    header_crc = ceph_crc32c(0, (unsigned char *)&oldheader, sizeof(oldheader) - sizeof(oldheader.crc));
+  }
+
+  ldout(msgr->cct,20) << "reader got envelope type=" << header.type
+           << " src " << entity_name_t(header.src)
+           << " front=" << header.front_len
+	   << " data=" << header.data_len
+	   << " off " << header.data_off
+           << dendl;
+
+  // verify header crc
+  if (header_crc != header.crc) {
+    ldout(msgr->cct,0) << "reader got bad header crc " << header_crc << " != " << header.crc << dendl;
+    return -1;
+  }
+
+  bufferlist front, middle, data;
+  int front_len, middle_len;
+  unsigned data_len, data_off;
+  int aborted;
+  Message *message;
+  utime_t recv_stamp = ceph_clock_now(msgr->cct);
+
+  if (policy.throttler_messages) {
+    ldout(msgr->cct,10) << "reader wants " << 1 << " message from policy throttler "
+			<< policy.throttler_messages->get_current() << "/"
+			<< policy.throttler_messages->get_max() << dendl;
+    policy.throttler_messages->get();
+  }
+
+  uint64_t message_size = header.front_len + header.middle_len + header.data_len;
+  if (message_size) {
+    if (policy.throttler_bytes) {
+      ldout(msgr->cct,10) << "reader wants " << message_size << " bytes from policy throttler "
+	       << policy.throttler_bytes->get_current() << "/"
+	       << policy.throttler_bytes->get_max() << dendl;
+      policy.throttler_bytes->get(message_size);
+    }
+
+    // throttle total bytes waiting for dispatch.  do this _after_ the
+    // policy throttle, as this one does not deadlock (unless dispatch
+    // blocks indefinitely, which it shouldn't).  in contrast, the
+    // policy throttle carries for the lifetime of the message.
+    ldout(msgr->cct,10) << "reader wants " << message_size << " from dispatch throttler "
+	     << msgr->dispatch_throttler.get_current() << "/"
+	     << msgr->dispatch_throttler.get_max() << dendl;
+    msgr->dispatch_throttler.get(message_size);
+  }
+
+  utime_t throttle_stamp = ceph_clock_now(msgr->cct);
+
+  // read front
+  front_len = header.front_len;
+  if (front_len) {
+    bufferptr bp = buffer::create(front_len);
+    if (tcp_read(bp.c_str(), front_len) < 0)
+      goto out_dethrottle;
+    front.push_back(bp);
+    ldout(msgr->cct,20) << "reader got front " << front.length() << dendl;
+  }
+
+  // read middle
+  middle_len = header.middle_len;
+  if (middle_len) {
+    bufferptr bp = buffer::create(middle_len);
+    if (tcp_read(bp.c_str(), middle_len) < 0)
+      goto out_dethrottle;
+    middle.push_back(bp);
+    ldout(msgr->cct,20) << "reader got middle " << middle.length() << dendl;
+  }
+
+
+  // read data
+  data_len = le32_to_cpu(header.data_len);
+  data_off = le32_to_cpu(header.data_off);
+  if (data_len) {
+    unsigned offset = 0;
+    unsigned left = data_len;
+
+    bufferlist newbuf, rxbuf;
+    bufferlist::iterator blp;
+    int rxbuf_version = 0;
+	
+    while (left > 0) {
+      // wait for data
+      if (tcp_read_wait() < 0)
+	goto out_dethrottle;
+
+      // get a buffer
+      connection_state->lock.Lock();
+      map >::iterator p = connection_state->rx_buffers.find(header.tid);
+      if (p != connection_state->rx_buffers.end()) {
+	if (rxbuf.length() == 0 || p->second.second != rxbuf_version) {
+	  ldout(msgr->cct,10) << "reader seleting rx buffer v " << p->second.second
+		   << " at offset " << offset
+		   << " len " << p->second.first.length() << dendl;
+	  rxbuf = p->second.first;
+	  rxbuf_version = p->second.second;
+	  // make sure it's big enough
+	  if (rxbuf.length() < data_len)
+	    rxbuf.push_back(buffer::create(data_len - rxbuf.length()));
+	  blp = p->second.first.begin();
+	  blp.advance(offset);
+	}
+      } else {
+	if (!newbuf.length()) {
+	  ldout(msgr->cct,20) << "reader allocating new rx buffer at offset " << offset << dendl;
+	  alloc_aligned_buffer(newbuf, data_len, data_off);
+	  blp = newbuf.begin();
+	  blp.advance(offset);
+	}
+      }
+      bufferptr bp = blp.get_current_ptr();
+      int read = MIN(bp.length(), left);
+      ldout(msgr->cct,20) << "reader reading nonblocking into " << (void*)bp.c_str() << " len " << bp.length() << dendl;
+      int got = tcp_read_nonblocking(bp.c_str(), read);
+      ldout(msgr->cct,30) << "reader read " << got << " of " << read << dendl;
+      connection_state->lock.Unlock();
+      if (got < 0)
+	goto out_dethrottle;
+      if (got > 0) {
+	blp.advance(got);
+	data.append(bp, 0, got);
+	offset += got;
+	left -= got;
+      } // else we got a signal or something; just loop.
+    }
+  }
+
+  // footer
+  if (connection_state->has_feature(CEPH_FEATURE_MSG_AUTH)) {
+    if (tcp_read((char*)&footer, sizeof(footer)) < 0)
+      goto out_dethrottle;
+  } else {
+    ceph_msg_footer_old old_footer;
+    if (tcp_read((char*)&old_footer, sizeof(old_footer)) < 0)
+      goto out_dethrottle;
+    footer.front_crc = old_footer.front_crc;
+    footer.middle_crc = old_footer.middle_crc;
+    footer.data_crc = old_footer.data_crc;
+    footer.sig = 0;
+    footer.flags = old_footer.flags;
+  }
+  
+  aborted = (footer.flags & CEPH_MSG_FOOTER_COMPLETE) == 0;
+  ldout(msgr->cct,10) << "aborted = " << aborted << dendl;
+  if (aborted) {
+    ldout(msgr->cct,0) << "reader got " << front.length() << " + " << middle.length() << " + " << data.length()
+	    << " byte message.. ABORTED" << dendl;
+    ret = 0;
+    goto out_dethrottle;
+  }
+
+  ldout(msgr->cct,20) << "reader got " << front.length() << " + " << middle.length() << " + " << data.length()
+	   << " byte message" << dendl;
+  message = decode_message(msgr->cct, header, footer, front, middle, data);
+  if (!message) {
+    ret = -EINVAL;
+    goto out_dethrottle;
+  }
+
+  //
+  //  Check the signature if one should be present.  A zero return indicates success. PLR
+  //
+
+  if (auth_handler == NULL) {
+    ldout(msgr->cct, 10) << "No session security set" << dendl;
+  } else {
+    if (auth_handler->check_message_signature(message)) {
+      ldout(msgr->cct, 0) << "Signature check failed" << dendl;
+      ret = -EINVAL;
+      goto out_dethrottle;
+    } 
+  }
+
+  message->set_byte_throttler(policy.throttler_bytes);
+  message->set_message_throttler(policy.throttler_messages);
+
+  // store reservation size in message, so we don't get confused
+  // by messages entering the dispatch queue through other paths.
+  message->set_dispatch_throttle_size(message_size);
+
+  message->set_recv_stamp(recv_stamp);
+  message->set_throttle_stamp(throttle_stamp);
+  message->set_recv_complete_stamp(ceph_clock_now(msgr->cct));
+
+  *pm = message;
+  return 0;
+
+ out_dethrottle:
+  // release bytes reserved from the throttlers on failure
+  if (policy.throttler_messages) {
+    ldout(msgr->cct,10) << "reader releasing " << 1 << " message to policy throttler "
+			<< policy.throttler_messages->get_current() << "/"
+			<< policy.throttler_messages->get_max() << dendl;
+    policy.throttler_messages->put();
+  }
+  if (message_size) {
+    if (policy.throttler_bytes) {
+      ldout(msgr->cct,10) << "reader releasing " << message_size << " bytes to policy throttler "
+			  << policy.throttler_bytes->get_current() << "/"
+			  << policy.throttler_bytes->get_max() << dendl;
+      policy.throttler_bytes->put(message_size);
+    }
+
+    msgr->dispatch_throttle_release(message_size);
+  }
+  return ret;
+}
+
+int Pipe::do_sendmsg(struct msghdr *msg, int len, bool more)
+{
+  while (len > 0) {
+    if (0) { // sanity
+      int l = 0;
+      for (unsigned i=0; imsg_iovlen; i++)
+	l += msg->msg_iov[i].iov_len;
+      assert(l == len);
+    }
+
+    int r = ::sendmsg(sd, msg, MSG_NOSIGNAL | (more ? MSG_MORE : 0));
+    if (r == 0) 
+      ldout(msgr->cct,10) << "do_sendmsg hmm do_sendmsg got r==0!" << dendl;
+    if (r < 0) { 
+      ldout(msgr->cct,1) << "do_sendmsg error " << cpp_strerror(errno) << dendl;
+      return -1;
+    }
+    if (state == STATE_CLOSED) {
+      ldout(msgr->cct,10) << "do_sendmsg oh look, state == CLOSED, giving up" << dendl;
+      errno = EINTR;
+      return -1; // close enough
+    }
+
+    len -= r;
+    if (len == 0) break;
+    
+    // hrmph.  trim r bytes off the front of our message.
+    ldout(msgr->cct,20) << "do_sendmsg short write did " << r << ", still have " << len << dendl;
+    while (r > 0) {
+      if (msg->msg_iov[0].iov_len <= (size_t)r) {
+	// lose this whole item
+	//ldout(msgr->cct,30) << "skipping " << msg->msg_iov[0].iov_len << ", " << (msg->msg_iovlen-1) << " v, " << r << " left" << dendl;
+	r -= msg->msg_iov[0].iov_len;
+	msg->msg_iov++;
+	msg->msg_iovlen--;
+      } else {
+	// partial!
+	//ldout(msgr->cct,30) << "adjusting " << msg->msg_iov[0].iov_len << ", " << msg->msg_iovlen << " v, " << r << " left" << dendl;
+	msg->msg_iov[0].iov_base = (char *)msg->msg_iov[0].iov_base + r;
+	msg->msg_iov[0].iov_len -= r;
+	break;
+      }
+    }
+  }
+  return 0;
+}
+
+
+int Pipe::write_ack(uint64_t seq)
+{
+  ldout(msgr->cct,10) << "write_ack " << seq << dendl;
+
+  char c = CEPH_MSGR_TAG_ACK;
+  ceph_le64 s;
+  s = seq;
+
+  struct msghdr msg;
+  memset(&msg, 0, sizeof(msg));
+  struct iovec msgvec[2];
+  msgvec[0].iov_base = &c;
+  msgvec[0].iov_len = 1;
+  msgvec[1].iov_base = &s;
+  msgvec[1].iov_len = sizeof(s);
+  msg.msg_iov = msgvec;
+  msg.msg_iovlen = 2;
+  
+  if (do_sendmsg(&msg, 1 + sizeof(s), true) < 0)
+    return -1;	
+  return 0;
+}
+
+int Pipe::write_keepalive()
+{
+  ldout(msgr->cct,10) << "write_keepalive" << dendl;
+
+  char c = CEPH_MSGR_TAG_KEEPALIVE;
+
+  struct msghdr msg;
+  memset(&msg, 0, sizeof(msg));
+  struct iovec msgvec[2];
+  msgvec[0].iov_base = &c;
+  msgvec[0].iov_len = 1;
+  msg.msg_iov = msgvec;
+  msg.msg_iovlen = 1;
+  
+  if (do_sendmsg(&msg, 1) < 0)
+    return -1;	
+  return 0;
+}
+
+int Pipe::write_keepalive2(char tag, const utime_t& t)
+{
+  ldout(msgr->cct,10) << "write_keepalive2 " << (int)tag << " " << t << dendl;
+  struct ceph_timespec ts;
+  t.encode_timeval(&ts);
+  struct msghdr msg;
+  memset(&msg, 0, sizeof(msg));
+  struct iovec msgvec[2];
+  msgvec[0].iov_base = &tag;
+  msgvec[0].iov_len = 1;
+  msgvec[1].iov_base = &ts;
+  msgvec[1].iov_len = sizeof(ts);
+  msg.msg_iov = msgvec;
+  msg.msg_iovlen = 2;
+
+  if (do_sendmsg(&msg, 1 + sizeof(ts)) < 0)
+    return -1;
+  return 0;
+}
+
+
+int Pipe::write_message(ceph_msg_header& header, ceph_msg_footer& footer, bufferlist& blist)
+{
+  int ret;
+
+  // set up msghdr and iovecs
+  struct msghdr msg;
+  memset(&msg, 0, sizeof(msg));
+  struct iovec *msgvec = new iovec[3 + blist.buffers().size()];  // conservative upper bound
+  msg.msg_iov = msgvec;
+  int msglen = 0;
+  
+  // send tag
+  char tag = CEPH_MSGR_TAG_MSG;
+  msgvec[msg.msg_iovlen].iov_base = &tag;
+  msgvec[msg.msg_iovlen].iov_len = 1;
+  msglen++;
+  msg.msg_iovlen++;
+
+  // send envelope
+  ceph_msg_header_old oldheader;
+  if (connection_state->has_feature(CEPH_FEATURE_NOSRCADDR)) {
+    msgvec[msg.msg_iovlen].iov_base = (char*)&header;
+    msgvec[msg.msg_iovlen].iov_len = sizeof(header);
+    msglen += sizeof(header);
+    msg.msg_iovlen++;
+  } else {
+    memcpy(&oldheader, &header, sizeof(header));
+    oldheader.src.name = header.src;
+    oldheader.src.addr = connection_state->get_peer_addr();
+    oldheader.orig_src = oldheader.src;
+    oldheader.reserved = header.reserved;
+    oldheader.crc = ceph_crc32c(0, (unsigned char*)&oldheader,
+				sizeof(oldheader) - sizeof(oldheader.crc));
+    msgvec[msg.msg_iovlen].iov_base = (char*)&oldheader;
+    msgvec[msg.msg_iovlen].iov_len = sizeof(oldheader);
+    msglen += sizeof(oldheader);
+    msg.msg_iovlen++;
+  }
+
+  // payload (front+data)
+  list::const_iterator pb = blist.buffers().begin();
+  int b_off = 0;  // carry-over buffer offset, if any
+  int bl_pos = 0; // blist pos
+  int left = blist.length();
+
+  while (left > 0) {
+    int donow = MIN(left, (int)pb->length()-b_off);
+    if (donow == 0) {
+      ldout(msgr->cct,0) << "donow = " << donow << " left " << left << " pb->length " << pb->length()
+	      << " b_off " << b_off << dendl;
+    }
+    assert(donow > 0);
+    ldout(msgr->cct,30) << " bl_pos " << bl_pos << " b_off " << b_off
+	     << " leftinchunk " << left
+	     << " buffer len " << pb->length()
+	     << " writing " << donow 
+	     << dendl;
+    
+    if (msg.msg_iovlen >= IOV_MAX-2) {
+      if (do_sendmsg(&msg, msglen, true))
+	goto fail;
+      
+      // and restart the iov
+      msg.msg_iov = msgvec;
+      msg.msg_iovlen = 0;
+      msglen = 0;
+    }
+    
+    msgvec[msg.msg_iovlen].iov_base = (void*)(pb->c_str()+b_off);
+    msgvec[msg.msg_iovlen].iov_len = donow;
+    msglen += donow;
+    msg.msg_iovlen++;
+    
+    left -= donow;
+    assert(left >= 0);
+    b_off += donow;
+    bl_pos += donow;
+    if (left == 0)
+      break;
+    while (b_off == (int)pb->length()) {
+      ++pb;
+      b_off = 0;
+    }
+  }
+  assert(left == 0);
+
+  // send footer; if receiver doesn't support signatures, use the old footer format
+
+  ceph_msg_footer_old old_footer;
+  if (connection_state->has_feature(CEPH_FEATURE_MSG_AUTH)) {
+    msgvec[msg.msg_iovlen].iov_base = (void*)&footer;
+    msgvec[msg.msg_iovlen].iov_len = sizeof(footer);
+    msglen += sizeof(footer);
+    msg.msg_iovlen++;
+  } else {
+    old_footer.front_crc = footer.front_crc;   
+    old_footer.middle_crc = footer.middle_crc;   
+    old_footer.data_crc = footer.data_crc;   
+    old_footer.flags = footer.flags;   
+    msgvec[msg.msg_iovlen].iov_base = (char*)&old_footer;
+    msgvec[msg.msg_iovlen].iov_len = sizeof(old_footer);
+    msglen += sizeof(old_footer);
+    msg.msg_iovlen++;
+  }
+
+  // send
+  if (do_sendmsg(&msg, msglen))
+    goto fail;
+
+  ret = 0;
+
+ out:
+  delete[] msgvec;
+  return ret;
+
+ fail:
+  ret = -1;
+  goto out;
+}
+
+
+int Pipe::tcp_read(char *buf, int len)
+{
+  if (sd < 0)
+    return -1;
+
+  while (len > 0) {
+
+    if (msgr->cct->_conf->ms_inject_socket_failures && sd >= 0) {
+      if (rand() % msgr->cct->_conf->ms_inject_socket_failures == 0) {
+	ldout(msgr->cct, 0) << "injecting socket failure" << dendl;
+	::shutdown(sd, SHUT_RDWR);
+      }
+    }
+
+    if (tcp_read_wait() < 0)
+      return -1;
+
+    int got = tcp_read_nonblocking(buf, len);
+
+    if (got < 0)
+      return -1;
+
+    len -= got;
+    buf += got;
+    //lgeneric_dout(cct, DBL) << "tcp_read got " << got << ", " << len << " left" << dendl;
+  }
+  return len;
+}
+
+int Pipe::tcp_read_wait()
+{
+  if (sd < 0)
+    return -1;
+  struct pollfd pfd;
+  short evmask;
+  pfd.fd = sd;
+  pfd.events = POLLIN;
+#if defined(__linux__)
+  pfd.events |= POLLRDHUP;
+#endif
+
+  if (poll(&pfd, 1, msgr->timeout) <= 0)
+    return -1;
+
+  evmask = POLLERR | POLLHUP | POLLNVAL;
+#if defined(__linux__)
+  evmask |= POLLRDHUP;
+#endif
+  if (pfd.revents & evmask)
+    return -1;
+
+  if (!(pfd.revents & POLLIN))
+    return -1;
+
+  return 0;
+}
+
+int Pipe::tcp_read_nonblocking(char *buf, int len)
+{
+again:
+  int got = ::recv( sd, buf, len, MSG_DONTWAIT );
+  if (got < 0) {
+    if (errno == EAGAIN || errno == EINTR) {
+      goto again;
+    } else {
+      ldout(msgr->cct, 10) << "tcp_read_nonblocking socket " << sd << " returned "
+		     << got << " " << cpp_strerror(errno) << dendl;
+      return -1;
+    }
+  } else if (got == 0) {
+    /* poll() said there was data, but we didn't read any - peer
+     * sent a FIN.  Maybe POLLRDHUP signals this, but this is
+     * standard socket behavior as documented by Stevens.
+     */
+    return -1;
+  }
+  return got;
+}
+
+int Pipe::tcp_write(const char *buf, int len)
+{
+  if (sd < 0)
+    return -1;
+  struct pollfd pfd;
+  pfd.fd = sd;
+  pfd.events = POLLOUT | POLLHUP | POLLNVAL | POLLERR;
+#if defined(__linux__)
+  pfd.events |= POLLRDHUP;
+#endif
+
+  if (msgr->cct->_conf->ms_inject_socket_failures && sd >= 0) {
+    if (rand() % msgr->cct->_conf->ms_inject_socket_failures == 0) {
+      ldout(msgr->cct, 0) << "injecting socket failure" << dendl;
+      ::shutdown(sd, SHUT_RDWR);
+    }
+  }
+
+  if (poll(&pfd, 1, -1) < 0)
+    return -1;
+
+  if (!(pfd.revents & POLLOUT))
+    return -1;
+
+  //lgeneric_dout(cct, DBL) << "tcp_write writing " << len << dendl;
+  assert(len > 0);
+  while (len > 0) {
+    int did = ::send( sd, buf, len, MSG_NOSIGNAL );
+    if (did < 0) {
+      //lgeneric_dout(cct, 1) << "tcp_write error did = " << did << " " << cpp_strerror(errno) << dendl;
+      //lgeneric_derr(cct, 1) << "tcp_write error did = " << did << " " << cpp_strerror(errno) << dendl;
+      return did;
+    }
+    len -= did;
+    buf += did;
+    //lgeneric_dout(cct, DBL) << "tcp_write did " << did << ", " << len << " left" << dendl;
+  }
+  return 0;
+}
diff --git a/ceph/src/msg/Pipe.h b/ceph/src/msg/Pipe.h
new file mode 100644
index 00000000..468a6a52
--- /dev/null
+++ b/ceph/src/msg/Pipe.h
@@ -0,0 +1,322 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MSGR_PIPE_H
+#define CEPH_MSGR_PIPE_H
+
+#include "include/memory.h"
+
+#include "msg_types.h"
+#include "Messenger.h"
+#include "auth/AuthSessionHandler.h"
+
+
+class SimpleMessenger;
+class IncomingQueue;
+class DispatchQueue;
+
+  /**
+   * The Pipe is the most complex SimpleMessenger component. It gets
+   * two threads, one each for reading and writing on a socket it's handed
+   * at creation time, and is responsible for everything that happens on
+   * that socket. Besides message transmission, it's responsible for
+   * propagating socket errors to the SimpleMessenger and then sticking
+   * around in a state where it can provide enough data for the SimpleMessenger
+   * to provide reliable Message delivery when it manages to reconnect.
+   */
+  class Pipe : public RefCountedObject {
+    /**
+     * The Reader thread handles all reads off the socket -- not just
+     * Messages, but also acks and other protocol bits (excepting startup,
+     * when the Writer does a couple of reads).
+     * All the work is implemented in Pipe itself, of course.
+     */
+    class Reader : public Thread {
+      Pipe *pipe;
+    public:
+      Reader(Pipe *p) : pipe(p) {}
+      void *entry() { pipe->reader(); return 0; }
+    } reader_thread;
+    friend class Reader;
+
+    /**
+     * The Writer thread handles all writes to the socket (after startup).
+     * All the work is implemented in Pipe itself, of course.
+     */
+    class Writer : public Thread {
+      Pipe *pipe;
+    public:
+      Writer(Pipe *p) : pipe(p) {}
+      void *entry() { pipe->writer(); return 0; }
+    } writer_thread;
+    friend class Writer;
+
+    /**
+     * The DelayedDelivery is for injecting delays into Message delivery off
+     * the socket. It is only enabled if delays are requested, and if they
+     * are then it pulls Messages off the DelayQueue and puts them into the
+     * in_q (SimpleMessenger::dispatch_queue).
+     * Please note that this probably has issues with Pipe shutdown and
+     * replacement semantics. I've tried, but no guarantees.
+     */
+    class DelayedDelivery: public Thread {
+      Pipe *pipe;
+      std::deque< pair > delay_queue;
+      Mutex delay_lock;
+      Cond delay_cond;
+      bool stop_delayed_delivery;
+
+    public:
+      DelayedDelivery(Pipe *p)
+	: pipe(p),
+	  delay_lock("Pipe::DelayedDelivery::delay_lock"),
+	  stop_delayed_delivery(false) { }
+      ~DelayedDelivery() {
+	discard();
+      }
+      void *entry();
+      void queue(utime_t release, Message *m) {
+	Mutex::Locker l(delay_lock);
+	delay_queue.push_back(make_pair(release, m));
+	delay_cond.Signal();
+      }
+      void discard();
+      void flush();
+      void stop() {
+	delay_lock.Lock();
+	stop_delayed_delivery = true;
+	delay_cond.Signal();
+	delay_lock.Unlock();
+      }
+    } *delay_thread;
+    friend class DelayedDelivery;
+
+  public:
+    Pipe(SimpleMessenger *r, int st, Connection *con);
+    ~Pipe();
+
+    SimpleMessenger *msgr;
+    uint64_t conn_id;
+    ostream& _pipe_prefix(std::ostream *_dout);
+
+    enum {
+      STATE_ACCEPTING,
+      STATE_CONNECTING,
+      STATE_OPEN,
+      STATE_STANDBY,
+      STATE_CLOSED,
+      STATE_CLOSING,
+      STATE_WAIT       // just wait for racing connection
+    };
+
+    static const char *get_state_name(int s) {
+      switch (s) {
+      case STATE_ACCEPTING: return "accepting";
+      case STATE_CONNECTING: return "connecting";
+      case STATE_OPEN: return "open";
+      case STATE_STANDBY: return "standby";
+      case STATE_CLOSED: return "closed";
+      case STATE_CLOSING: return "closing";
+      case STATE_WAIT: return "wait";
+      default: return "UNKNOWN";
+      }
+    }
+    const char *get_state_name() {
+      return get_state_name(state);
+    }
+
+    int sd;
+    int port;
+    int peer_type;
+    entity_addr_t peer_addr;
+    Messenger::Policy policy;
+    
+    Mutex pipe_lock;
+    int state;
+    atomic_t state_closed; // non-zero iff state = STATE_CLOSED
+
+    // session_security handles any signatures or encryptions required for this pipe's msgs. PLR
+
+    ceph::shared_ptr session_security;
+
+  protected:
+    friend class SimpleMessenger;
+    ConnectionRef connection_state;
+
+    utime_t backoff;         // backoff time
+
+    bool reader_running, reader_needs_join;
+    bool writer_running;
+
+    map > out_q;  // priority queue for outbound msgs
+    DispatchQueue *in_q;
+    list sent;
+    Cond cond;
+    bool send_keepalive;
+    bool send_keepalive_ack;
+    utime_t keepalive_ack_stamp;
+    bool halt_delivery; //if a pipe's queue is destroyed, stop adding to it
+    bool close_on_empty;
+    
+    __u32 connect_seq, peer_global_seq;
+    uint64_t out_seq;
+    uint64_t in_seq, in_seq_acked;
+    
+    void set_socket_options();
+
+    int accept();   // server handshake
+    int connect();  // client handshake
+    void reader();
+    void writer();
+    void unlock_maybe_reap();
+
+    int randomize_out_seq();
+
+    int read_message(Message **pm,
+		     AuthSessionHandler *session_security_copy);
+    int write_message(ceph_msg_header& h, ceph_msg_footer& f, bufferlist& body);
+    /**
+     * Write the given data (of length len) to the Pipe's socket. This function
+     * will loop until all passed data has been written out.
+     * If more is set, the function will optimize socket writes
+     * for additional data (by passing the MSG_MORE flag, aka TCP_CORK).
+     *
+     * @param msg The msghdr to write out
+     * @param len The length of the data in msg
+     * @param more Should be set true if this is one part of a larger message
+     * @return 0, or -1 on failure (unrecoverable -- close the socket).
+     */
+    int do_sendmsg(struct msghdr *msg, int len, bool more=false);
+    int write_ack(uint64_t s);
+    int write_keepalive();
+    int write_keepalive2(char tag, const utime_t &t);
+
+    void fault(bool reader=false);
+
+    void was_session_reset();
+
+    /* Clean up sent list */
+    void handle_ack(uint64_t seq);
+
+    public:
+    Pipe(const Pipe& other);
+    const Pipe& operator=(const Pipe& other);
+
+    void start_reader();
+    void start_writer();
+    void maybe_start_delay_thread();
+    void join_reader();
+
+    // public constructors
+    static const Pipe& Server(int s);
+    static const Pipe& Client(const entity_addr_t& pi);
+
+    __u32 get_out_seq() { return out_seq; }
+
+    bool is_queued() { return !out_q.empty() || send_keepalive || send_keepalive_ack; }
+
+    entity_addr_t& get_peer_addr() { return peer_addr; }
+
+    void set_peer_addr(const entity_addr_t& a) {
+      if (&peer_addr != &a)  // shut up valgrind
+        peer_addr = a;
+      connection_state->set_peer_addr(a);
+    }
+    void set_peer_type(int t) {
+      peer_type = t;
+      connection_state->set_peer_type(t);
+    }
+
+    void register_pipe();
+    void unregister_pipe();
+    void join();
+    void stop();
+
+    void _send(Message *m) {
+      assert(pipe_lock.is_locked());
+      out_q[m->get_priority()].push_back(m);
+      cond.Signal();
+    }
+    void _send_keepalive() {
+      assert(pipe_lock.is_locked());
+      send_keepalive = true;
+      cond.Signal();
+    }
+    Message *_get_next_outgoing() {
+      assert(pipe_lock.is_locked());
+      Message *m = 0;
+      while (!m && !out_q.empty()) {
+        map >::reverse_iterator p = out_q.rbegin();
+        if (!p->second.empty()) {
+          m = p->second.front();
+          p->second.pop_front();
+        }
+        if (p->second.empty())
+          out_q.erase(p->first);
+      }
+      return m;
+    }
+
+    /// move all messages in the sent list back into the queue at the highest priority.
+    void requeue_sent();
+    /// discard messages requeued by requeued_sent() up to a given seq
+    void discard_requeued_up_to(uint64_t seq);
+    void discard_out_queue();
+
+    void shutdown_socket() {
+      if (sd >= 0)
+        ::shutdown(sd, SHUT_RDWR);
+    }
+
+    /**
+     * do a blocking read of len bytes from socket
+     *
+     * @param buf buffer to read into
+     * @param len exact number of bytes to read
+     * @return 0 for success, or -1 on error
+     */
+    int tcp_read(char *buf, int len);
+
+    /**
+     * wait for bytes to become available on the socket
+     *
+     * @return 0 for success, or -1 on error
+     */
+    int tcp_read_wait();
+
+    /**
+     * non-blocking read of available bytes on socket
+     *
+     * This is expected to be used after tcp_read_wait(), and will return
+     * an error if there is no data on the socket to consume.
+     *
+     * @param buf buffer to read into
+     * @param len maximum number of bytes to read
+     * @return bytes read, or -1 on error or when there is no data
+     */
+    int tcp_read_nonblocking(char *buf, int len);
+
+    /**
+     * blocking write of bytes to socket
+     *
+     * @param buf buffer
+     * @param len number of bytes to write
+     * @return 0 for success, or -1 on error
+     */
+    int tcp_write(const char *buf, int len);
+
+  };
+
+
+#endif
diff --git a/ceph/src/msg/SimpleMessenger.cc b/ceph/src/msg/SimpleMessenger.cc
new file mode 100644
index 00000000..ce7f1fde
--- /dev/null
+++ b/ceph/src/msg/SimpleMessenger.cc
@@ -0,0 +1,737 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+
+
+#include "SimpleMessenger.h"
+
+#include "common/config.h"
+#include "common/Timer.h"
+#include "common/errno.h"
+#include "auth/Crypto.h"
+#include "include/Spinlock.h"
+
+#define dout_subsys ceph_subsys_ms
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+static ostream& _prefix(std::ostream *_dout, SimpleMessenger *msgr) {
+  return *_dout << "-- " << msgr->get_myaddr() << " ";
+}
+
+
+/*******************
+ * SimpleMessenger
+ */
+
+SimpleMessenger::SimpleMessenger(CephContext *cct, entity_name_t name,
+				 string mname, uint64_t _nonce)
+  : Messenger(cct, name),
+    accepter(this, _nonce),
+    dispatch_queue(cct, this),
+    reaper_thread(this),
+    my_type(name.type()),
+    nonce(_nonce),
+    lock("SimpleMessenger::lock"), need_addr(true), did_bind(false),
+    global_seq(0),
+    cluster_protocol(0),
+    policy_lock("SimpleMessenger::policy_lock"),
+    dispatch_throttler(cct, string("msgr_dispatch_throttler-") + mname, cct->_conf->ms_dispatch_throttle_bytes),
+    reaper_started(false), reaper_stop(false),
+    timeout(0),
+    local_connection(new Connection(this))
+{
+  ceph_spin_init(&global_seq_lock);
+  init_local_connection();
+}
+
+/**
+ * Destroy the SimpleMessenger. Pretty simple since all the work is done
+ * elsewhere.
+ */
+SimpleMessenger::~SimpleMessenger()
+{
+  assert(!did_bind); // either we didn't bind or we shut down the Accepter
+  assert(rank_pipe.empty()); // we don't have any running Pipes.
+  assert(reaper_stop && !reaper_started); // the reaper thread is stopped
+}
+
+void SimpleMessenger::ready()
+{
+  ldout(cct,10) << "ready " << get_myaddr() << dendl;
+  dispatch_queue.start();
+
+  lock.Lock();
+  if (did_bind)
+    accepter.start();
+  lock.Unlock();
+}
+
+
+int SimpleMessenger::shutdown()
+{
+  ldout(cct,10) << "shutdown " << get_myaddr() << dendl;
+  mark_down_all();
+  dispatch_queue.shutdown();
+
+  // break ref cycles on the loopback connection
+  local_connection->set_priv(NULL);
+  return 0;
+}
+
+int SimpleMessenger::_send_message(Message *m, const entity_inst_t& dest,
+                                   bool lazy)
+{
+  // set envelope
+  m->get_header().src = get_myname();
+
+  if (!m->get_priority()) m->set_priority(get_default_send_priority());
+ 
+  ldout(cct,1) << (lazy ? "lazy " : "") <<"--> " << dest.name << " "
+          << dest.addr << " -- " << *m
+    	  << " -- ?+" << m->get_data().length()
+	  << " " << m 
+	  << dendl;
+
+  if (dest.addr == entity_addr_t()) {
+    ldout(cct,0) << (lazy ? "lazy_" : "") << "send_message message " << *m
+                 << " with empty dest " << dest.addr << dendl;
+    m->put();
+    return -EINVAL;
+  }
+
+  lock.Lock();
+  Pipe *pipe = _lookup_pipe(dest.addr);
+  submit_message(m, (pipe ? pipe->connection_state.get() : NULL),
+                 dest.addr, dest.name.type(), lazy);
+  lock.Unlock();
+  return 0;
+}
+
+int SimpleMessenger::_send_message(Message *m, Connection *con, bool lazy)
+{
+  //set envelope
+  m->get_header().src = get_myname();
+
+  if (!m->get_priority()) m->set_priority(get_default_send_priority());
+
+  ldout(cct,1) << (lazy ? "lazy " : "") << "--> " << con->get_peer_addr()
+      << " -- " << *m
+      << " -- ?+" << m->get_data().length()
+      << " " << m << " con " << con
+      << dendl;
+
+  lock.Lock();
+  submit_message(m, con, con->get_peer_addr(), con->get_peer_type(), lazy);
+  lock.Unlock();
+  return 0;
+}
+
+/**
+ * If my_inst.addr doesn't have an IP set, this function
+ * will fill it in from the passed addr. Otherwise it does nothing and returns.
+ */
+void SimpleMessenger::set_addr_unknowns(entity_addr_t &addr)
+{
+  if (my_inst.addr.is_blank_ip()) {
+    int port = my_inst.addr.get_port();
+    my_inst.addr.addr = addr.addr;
+    my_inst.addr.set_port(port);
+    init_local_connection();
+  }
+}
+
+int SimpleMessenger::get_proto_version(int peer_type, bool connect)
+{
+  // set reply protocol version
+  if (peer_type == my_type) {
+    // internal
+    return cluster_protocol;
+  } else {
+    // public
+    if (connect) {
+      switch (peer_type) {
+      case CEPH_ENTITY_TYPE_OSD: return CEPH_OSDC_PROTOCOL;
+      case CEPH_ENTITY_TYPE_MDS: return CEPH_MDSC_PROTOCOL;
+      case CEPH_ENTITY_TYPE_MON: return CEPH_MONC_PROTOCOL;
+      }
+    } else {
+      switch (my_type) {
+      case CEPH_ENTITY_TYPE_OSD: return CEPH_OSDC_PROTOCOL;
+      case CEPH_ENTITY_TYPE_MDS: return CEPH_MDSC_PROTOCOL;
+      case CEPH_ENTITY_TYPE_MON: return CEPH_MONC_PROTOCOL;
+      }
+    }
+  }
+  return 0;
+}
+
+
+
+
+
+
+
+/********************************************
+ * SimpleMessenger
+ */
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+
+void SimpleMessenger::dispatch_throttle_release(uint64_t msize)
+{
+  if (msize) {
+    ldout(cct,10) << "dispatch_throttle_release " << msize << " to dispatch throttler "
+	    << dispatch_throttler.get_current() << "/"
+	    << dispatch_throttler.get_max() << dendl;
+    dispatch_throttler.put(msize);
+  }
+}
+
+void SimpleMessenger::reaper_entry()
+{
+  ldout(cct,10) << "reaper_entry start" << dendl;
+  lock.Lock();
+  while (!reaper_stop) {
+    reaper();
+    reaper_cond.Wait(lock);
+  }
+  lock.Unlock();
+  ldout(cct,10) << "reaper_entry done" << dendl;
+}
+
+/*
+ * note: assumes lock is held
+ */
+void SimpleMessenger::reaper()
+{
+  ldout(cct,10) << "reaper" << dendl;
+  assert(lock.is_locked());
+
+  while (!pipe_reap_queue.empty()) {
+    Pipe *p = pipe_reap_queue.front();
+    pipe_reap_queue.pop_front();
+    ldout(cct,10) << "reaper reaping pipe " << p << " " << p->get_peer_addr() << dendl;
+    p->pipe_lock.Lock();
+    p->discard_out_queue();
+    if (p->connection_state) {
+      // mark_down, mark_down_all, or fault() should have done this,
+      // or accept() may have switch the Connection to a different
+      // Pipe... but make sure!
+      bool cleared = p->connection_state->clear_pipe(p);
+      assert(!cleared);
+    }
+    p->pipe_lock.Unlock();
+    p->unregister_pipe();
+    assert(pipes.count(p));
+    pipes.erase(p);
+    p->join();
+    if (p->sd >= 0)
+      ::close(p->sd);
+    ldout(cct,10) << "reaper reaped pipe " << p << " " << p->get_peer_addr() << dendl;
+    p->put();
+    ldout(cct,10) << "reaper deleted pipe " << p << dendl;
+  }
+  ldout(cct,10) << "reaper done" << dendl;
+}
+
+void SimpleMessenger::queue_reap(Pipe *pipe)
+{
+  ldout(cct,10) << "queue_reap " << pipe << dendl;
+  lock.Lock();
+  pipe_reap_queue.push_back(pipe);
+  reaper_cond.Signal();
+  lock.Unlock();
+}
+
+
+
+int SimpleMessenger::bind(const entity_addr_t &bind_addr)
+{
+  lock.Lock();
+  if (started) {
+    ldout(cct,10) << "rank.bind already started" << dendl;
+    lock.Unlock();
+    return -1;
+  }
+  ldout(cct,10) << "rank.bind " << bind_addr << dendl;
+  lock.Unlock();
+
+  // bind to a socket
+  set avoid_ports;
+  int r = accepter.bind(bind_addr, avoid_ports);
+  if (r >= 0)
+    did_bind = true;
+  return r;
+}
+
+int SimpleMessenger::rebind(const set& avoid_ports)
+{
+  ldout(cct,1) << "rebind avoid " << avoid_ports << dendl;
+  assert(did_bind);
+  accepter.stop();
+  mark_down_all();
+  return accepter.rebind(avoid_ports);
+}
+
+int SimpleMessenger::start()
+{
+  lock.Lock();
+  ldout(cct,1) << "messenger.start" << dendl;
+
+  // register at least one entity, first!
+  assert(my_type >= 0);
+
+  assert(!started);
+  started = true;
+
+  if (!did_bind) {
+    my_inst.addr.nonce = nonce;
+    init_local_connection();
+  }
+
+  lock.Unlock();
+
+  reaper_started = true;
+  reaper_thread.create();
+  return 0;
+}
+
+Pipe *SimpleMessenger::add_accept_pipe(int sd)
+{
+  lock.Lock();
+  Pipe *p = new Pipe(this, Pipe::STATE_ACCEPTING, NULL);
+  p->sd = sd;
+  p->pipe_lock.Lock();
+  p->start_reader();
+  p->pipe_lock.Unlock();
+  pipes.insert(p);
+  accepting_pipes.insert(p);
+  lock.Unlock();
+  return p;
+}
+
+/* connect_rank
+ * NOTE: assumes messenger.lock held.
+ */
+Pipe *SimpleMessenger::connect_rank(const entity_addr_t& addr,
+				    int type,
+				    Connection *con,
+				    Message *first)
+{
+  assert(lock.is_locked());
+  assert(addr != my_inst.addr);
+  
+  ldout(cct,10) << "connect_rank to " << addr << ", creating pipe and registering" << dendl;
+  
+  // create pipe
+  Pipe *pipe = new Pipe(this, Pipe::STATE_CONNECTING, con);
+  pipe->pipe_lock.Lock();
+  pipe->set_peer_type(type);
+  pipe->set_peer_addr(addr);
+  pipe->policy = get_policy(type);
+  pipe->start_writer();
+  if (first)
+    pipe->_send(first);
+  pipe->pipe_lock.Unlock();
+  pipe->register_pipe();
+  pipes.insert(pipe);
+
+  return pipe;
+}
+
+
+
+
+
+
+AuthAuthorizer *SimpleMessenger::get_authorizer(int peer_type, bool force_new)
+{
+  return ms_deliver_get_authorizer(peer_type, force_new);
+}
+
+bool SimpleMessenger::verify_authorizer(Connection *con, int peer_type,
+					int protocol, bufferlist& authorizer, bufferlist& authorizer_reply,
+					bool& isvalid,CryptoKey& session_key)
+{
+  return ms_deliver_verify_authorizer(con, peer_type, protocol, authorizer, authorizer_reply, isvalid,session_key);
+}
+
+ConnectionRef SimpleMessenger::get_connection(const entity_inst_t& dest)
+{
+  Mutex::Locker l(lock);
+  if (my_inst.addr == dest.addr) {
+    // local
+    return local_connection;
+  }
+
+  // remote
+  while (true) {
+    Pipe *pipe = _lookup_pipe(dest.addr);
+    if (pipe) {
+      ldout(cct, 10) << "get_connection " << dest << " existing " << pipe << dendl;
+    } else {
+      pipe = connect_rank(dest.addr, dest.name.type(), NULL, NULL);
+      ldout(cct, 10) << "get_connection " << dest << " new " << pipe << dendl;
+    }
+    Mutex::Locker l(pipe->pipe_lock);
+    if (pipe->connection_state)
+      return pipe->connection_state;
+    // we failed too quickly!  retry.  FIXME.
+  }
+}
+
+ConnectionRef SimpleMessenger::get_loopback_connection()
+{
+  return local_connection;
+}
+
+void SimpleMessenger::submit_message(Message *m, Connection *con,
+				     const entity_addr_t& dest_addr, int dest_type, bool lazy)
+{
+
+  if (cct->_conf->ms_dump_on_send) {
+    m->encode(-1, true);
+    ldout(cct, 0) << "submit_message " << *m << "\n";
+    m->get_payload().hexdump(*_dout);
+    if (m->get_data().length() > 0) {
+      *_dout << " data:\n";
+      m->get_data().hexdump(*_dout);
+    }
+    *_dout << dendl;
+    m->clear_payload();
+  }
+
+  // existing connection?
+  if (con) {
+    Pipe *pipe = NULL;
+    bool ok = con->try_get_pipe((RefCountedObject**)&pipe);
+    if (!ok) {
+      ldout(cct,0) << "submit_message " << *m << " remote, " << dest_addr
+		   << ", failed lossy con, dropping message " << m << dendl;
+      m->put();
+      return;
+    }
+    if (pipe) {
+      pipe->pipe_lock.Lock();
+      if (pipe->state != Pipe::STATE_CLOSED) {
+	ldout(cct,20) << "submit_message " << *m << " remote, " << dest_addr << ", have pipe." << dendl;
+	pipe->_send(m);
+	pipe->pipe_lock.Unlock();
+	pipe->put();
+	return;
+      }
+      pipe->pipe_lock.Unlock();
+      pipe->put();
+      ldout(cct,20) << "submit_message " << *m << " remote, " << dest_addr
+		    << ", had pipe " << pipe << ", but it closed." << dendl;
+      m->put();
+      return;
+    }
+  }
+
+  // local?
+  if (my_inst.addr == dest_addr) {
+    // local
+    ldout(cct,20) << "submit_message " << *m << " local" << dendl;
+    dispatch_queue.local_delivery(m, m->get_priority());
+    return;
+  }
+
+  // remote, no existing pipe.
+  const Policy& policy = get_policy(dest_type);
+  if (policy.server) {
+    ldout(cct,20) << "submit_message " << *m << " remote, " << dest_addr << ", lossy server for target type "
+		  << ceph_entity_type_name(dest_type) << ", no session, dropping." << dendl;
+    m->put();
+  } else if (lazy) {
+    ldout(cct,20) << "submit_message " << *m << " remote, " << dest_addr << ", lazy, dropping." << dendl;
+    m->put();
+  } else {
+    ldout(cct,20) << "submit_message " << *m << " remote, " << dest_addr << ", new pipe." << dendl;
+    connect_rank(dest_addr, dest_type, con, m);
+  }
+}
+
+int SimpleMessenger::send_keepalive(const entity_inst_t& dest)
+{
+  const entity_addr_t dest_addr = dest.addr;
+  entity_addr_t dest_proc_addr = dest_addr;
+  int ret = 0;
+
+  lock.Lock();
+  {
+    // local?
+    if (my_inst.addr != dest_addr) {
+      // remote.
+      Pipe *pipe = _lookup_pipe(dest_proc_addr);
+      if (pipe) {
+        // connected?
+	pipe->pipe_lock.Lock();
+	ldout(cct,20) << "send_keepalive remote, " << dest_addr << ", have pipe." << dendl;
+	pipe->_send_keepalive();
+	pipe->pipe_lock.Unlock();
+      } else {
+        ret = -EINVAL;
+      }
+      if (!pipe) {
+	ldout(cct,20) << "send_keepalive no pipe for " << dest_addr << ", doing nothing." << dendl;
+      }
+    }
+  }
+  lock.Unlock();
+  return ret;
+}
+
+int SimpleMessenger::send_keepalive(Connection *con)
+{
+  int ret = 0;
+  Pipe *pipe = static_cast(con->get_pipe());
+  if (pipe) {
+    ldout(cct,20) << "send_keepalive con " << con << ", have pipe." << dendl;
+    assert(pipe->msgr == this);
+    pipe->pipe_lock.Lock();
+    pipe->_send_keepalive();
+    pipe->pipe_lock.Unlock();
+    pipe->put();
+  } else {
+    ldout(cct,0) << "send_keepalive con " << con << ", no pipe." << dendl;
+    ret = -EPIPE;
+  }
+  return ret;
+}
+
+
+
+void SimpleMessenger::wait()
+{
+  lock.Lock();
+  if (!started) {
+    lock.Unlock();
+    return;
+  }
+  lock.Unlock();
+
+  if(dispatch_queue.is_started()) {
+    ldout(cct,10) << "wait: waiting for dispatch queue" << dendl;
+    dispatch_queue.wait();
+    ldout(cct,10) << "wait: dispatch queue is stopped" << dendl;
+  }
+
+  // done!  clean up.
+  if (did_bind) {
+    ldout(cct,20) << "wait: stopping accepter thread" << dendl;
+    accepter.stop();
+    did_bind = false;
+    ldout(cct,20) << "wait: stopped accepter thread" << dendl;
+  }
+
+  if (reaper_started) {
+    ldout(cct,20) << "wait: stopping reaper thread" << dendl;
+    lock.Lock();
+    reaper_cond.Signal();
+    reaper_stop = true;
+    lock.Unlock();
+    reaper_thread.join();
+    reaper_started = false;
+    ldout(cct,20) << "wait: stopped reaper thread" << dendl;
+  }
+
+  // close+reap all pipes
+  lock.Lock();
+  {
+    ldout(cct,10) << "wait: closing pipes" << dendl;
+
+    while (!rank_pipe.empty()) {
+      Pipe *p = rank_pipe.begin()->second;
+      p->unregister_pipe();
+      p->pipe_lock.Lock();
+      p->stop();
+      p->pipe_lock.Unlock();
+    }
+
+    reaper();
+    ldout(cct,10) << "wait: waiting for pipes " << pipes << " to close" << dendl;
+    while (!pipes.empty()) {
+      reaper_cond.Wait(lock);
+      reaper();
+    }
+  }
+  lock.Unlock();
+
+  ldout(cct,10) << "wait: done." << dendl;
+  ldout(cct,1) << "shutdown complete." << dendl;
+  started = false;
+  my_type = -1;
+}
+
+
+void SimpleMessenger::mark_down_all()
+{
+  ldout(cct,1) << "mark_down_all" << dendl;
+  lock.Lock();
+  for (set::iterator q = accepting_pipes.begin(); q != accepting_pipes.end(); ++q) {
+    Pipe *p = *q;
+    ldout(cct,5) << "mark_down_all accepting_pipe " << p << dendl;
+    p->pipe_lock.Lock();
+    p->stop();
+    ConnectionRef con = p->connection_state;
+    if (con && con->clear_pipe(p))
+      dispatch_queue.queue_reset(con.get());
+    p->pipe_lock.Unlock();
+  }
+  accepting_pipes.clear();
+
+  while (!rank_pipe.empty()) {
+    ceph::unordered_map::iterator it = rank_pipe.begin();
+    Pipe *p = it->second;
+    ldout(cct,5) << "mark_down_all " << it->first << " " << p << dendl;
+    rank_pipe.erase(it);
+    p->unregister_pipe();
+    p->pipe_lock.Lock();
+    p->stop();
+    ConnectionRef con = p->connection_state;
+    if (con && con->clear_pipe(p))
+      dispatch_queue.queue_reset(con.get());
+    p->pipe_lock.Unlock();
+  }
+  lock.Unlock();
+}
+
+void SimpleMessenger::mark_down(const entity_addr_t& addr)
+{
+  lock.Lock();
+  Pipe *p = _lookup_pipe(addr);
+  if (p) {
+    ldout(cct,1) << "mark_down " << addr << " -- " << p << dendl;
+    p->unregister_pipe();
+    p->pipe_lock.Lock();
+    p->stop();
+    if (p->connection_state) {
+      // generate a reset event for the caller in this case, even
+      // though they asked for it, since this is the addr-based (and
+      // not Connection* based) interface
+      ConnectionRef con = p->connection_state;
+      if (con && con->clear_pipe(p))
+	dispatch_queue.queue_reset(con.get());
+    }
+    p->pipe_lock.Unlock();
+  } else {
+    ldout(cct,1) << "mark_down " << addr << " -- pipe dne" << dendl;
+  }
+  lock.Unlock();
+}
+
+void SimpleMessenger::mark_down(Connection *con)
+{
+  if (con == NULL)
+    return;
+  lock.Lock();
+  Pipe *p = static_cast(con->get_pipe());
+  if (p) {
+    ldout(cct,1) << "mark_down " << con << " -- " << p << dendl;
+    assert(p->msgr == this);
+    p->unregister_pipe();
+    p->pipe_lock.Lock();
+    p->stop();
+    if (p->connection_state) {
+      // do not generate a reset event for the caller in this case,
+      // since they asked for it.
+      p->connection_state->clear_pipe(p);
+    }
+    p->pipe_lock.Unlock();
+    p->put();
+  } else {
+    ldout(cct,1) << "mark_down " << con << " -- pipe dne" << dendl;
+  }
+  lock.Unlock();
+}
+
+void SimpleMessenger::mark_down_on_empty(Connection *con)
+{
+  lock.Lock();
+  Pipe *p = static_cast(con->get_pipe());
+  if (p) {
+    assert(p->msgr == this);
+    p->pipe_lock.Lock();
+    p->unregister_pipe();
+    if (p->out_q.empty()) {
+      ldout(cct,1) << "mark_down_on_empty " << con << " -- " << p << " closing (queue is empty)" << dendl;
+      p->stop();
+    } else {
+      ldout(cct,1) << "mark_down_on_empty " << con << " -- " << p << " marking (queue is not empty)" << dendl;
+      p->close_on_empty = true;
+    }
+    p->pipe_lock.Unlock();
+    p->put();
+  } else {
+    ldout(cct,1) << "mark_down_on_empty " << con << " -- pipe dne" << dendl;
+  }
+  lock.Unlock();
+}
+
+void SimpleMessenger::mark_disposable(Connection *con)
+{
+  lock.Lock();
+  Pipe *p = static_cast(con->get_pipe());
+  if (p) {
+    ldout(cct,1) << "mark_disposable " << con << " -- " << p << dendl;
+    assert(p->msgr == this);
+    p->pipe_lock.Lock();
+    p->policy.lossy = true;
+    p->pipe_lock.Unlock();
+    p->put();
+  } else {
+    ldout(cct,1) << "mark_disposable " << con << " -- pipe dne" << dendl;
+  }
+  lock.Unlock();
+}
+
+void SimpleMessenger::learned_addr(const entity_addr_t &peer_addr_for_me)
+{
+  // be careful here: multiple threads may block here, and readers of
+  // my_inst.addr do NOT hold any lock.
+
+  // this always goes from true -> false under the protection of the
+  // mutex.  if it is already false, we need not retake the mutex at
+  // all.
+  if (!need_addr)
+    return;
+
+  lock.Lock();
+  if (need_addr) {
+    entity_addr_t t = peer_addr_for_me;
+    t.set_port(my_inst.addr.get_port());
+    my_inst.addr.addr = t.addr;
+    ldout(cct,1) << "learned my addr " << my_inst.addr << dendl;
+    need_addr = false;
+    init_local_connection();
+  }
+  lock.Unlock();
+}
+
+void SimpleMessenger::unlearn_addr()
+{
+  lock.Lock();
+  need_addr = true;
+  lock.Unlock();
+}
+
+void SimpleMessenger::init_local_connection()
+{
+  local_connection->peer_addr = my_inst.addr;
+  local_connection->peer_type = my_type;
+}
diff --git a/ceph/src/msg/SimpleMessenger.h b/ceph/src/msg/SimpleMessenger.h
new file mode 100644
index 00000000..e6e1fb1d
--- /dev/null
+++ b/ceph/src/msg/SimpleMessenger.h
@@ -0,0 +1,472 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_SIMPLEMESSENGER_H
+#define CEPH_SIMPLEMESSENGER_H
+
+#include "include/types.h"
+#include "include/xlist.h"
+
+#include 
+#include 
+using namespace std;
+#include "include/unordered_map.h"
+#include "include/unordered_set.h"
+
+#include "common/Mutex.h"
+#include "include/atomic.h"
+#include "common/Cond.h"
+#include "common/Thread.h"
+#include "common/Throttle.h"
+
+#include "Messenger.h"
+#include "Message.h"
+#include "include/assert.h"
+#include "DispatchQueue.h"
+
+#include "Pipe.h"
+#include "Accepter.h"
+#include "include/Spinlock.h"
+
+/*
+ * This class handles transmission and reception of messages. Generally
+ * speaking, there are several major components:
+ *
+ * - Connection
+ *    Each logical session is associated with a Connection.
+ * - Pipe
+ *    Each network connection is handled through a pipe, which handles
+ *    the input and output of each message.  There is normally a 1:1
+ *    relationship between Pipe and Connection, but logical sessions may
+ *    get handed off between Pipes when sockets reconnect or during
+ *    connection races.
+ * - IncomingQueue
+ *    Incoming messages are associated with an IncomingQueue, and there
+ *    is one such queue associated with each Pipe.
+ * - DispatchQueue
+ *    IncomingQueues get queued in the DIspatchQueue, which is responsible
+ *    for doing a round-robin sweep and processing them via a worker thread.
+ * - SimpleMessenger
+ *    It's the exterior class passed to the external message handler and
+ *    most of the API details.
+ *
+ * Lock ordering:
+ *
+ *   SimpleMessenger::lock
+ *       Pipe::pipe_lock
+ *           DispatchQueue::lock
+ *               IncomingQueue::lock
+ */
+
+class SimpleMessenger : public Messenger {
+  // First we have the public Messenger interface implementation...
+public:
+  /**
+   * Initialize the SimpleMessenger!
+   *
+   * @param cct The CephContext to use
+   * @param name The name to assign ourselves
+   * _nonce A unique ID to use for this SimpleMessenger. It should not
+   * be a value that will be repeated if the daemon restarts.
+   */
+  SimpleMessenger(CephContext *cct, entity_name_t name,
+		  string mname, uint64_t _nonce);
+
+  /**
+   * Destroy the SimpleMessenger. Pretty simple since all the work is done
+   * elsewhere.
+   */
+  virtual ~SimpleMessenger();
+
+  /** @defgroup Accessors
+   * @{
+   */
+  void set_addr_unknowns(entity_addr_t& addr);
+
+  int get_dispatch_queue_len() {
+    return dispatch_queue.get_queue_len();
+  }
+
+  double get_dispatch_queue_max_age(utime_t now) {
+    return dispatch_queue.get_max_age(now);
+  }
+  /** @} Accessors */
+
+  /**
+   * @defgroup Configuration functions
+   * @{
+   */
+  void set_cluster_protocol(int p) {
+    assert(!started && !did_bind);
+    cluster_protocol = p;
+  }
+
+  void set_default_policy(Policy p) {
+    Mutex::Locker l(policy_lock);
+    default_policy = p;
+  }
+
+  void set_policy(int type, Policy p) {
+    Mutex::Locker l(policy_lock);
+    policy_map[type] = p;
+  }
+
+  void set_policy_throttlers(int type, Throttle *byte_throttle, Throttle *msg_throttle) {
+    Mutex::Locker l(policy_lock);
+    if (policy_map.count(type)) {
+      policy_map[type].throttler_bytes = byte_throttle;
+      policy_map[type].throttler_messages = msg_throttle;
+    } else {
+      default_policy.throttler_bytes = byte_throttle;
+      default_policy.throttler_messages = msg_throttle;
+    }
+  }
+
+  int bind(const entity_addr_t& bind_addr);
+  int rebind(const set& avoid_ports);
+
+  /** @} Configuration functions */
+
+  /**
+   * @defgroup Startup/Shutdown
+   * @{
+   */
+  virtual int start();
+  virtual void wait();
+  virtual int shutdown();
+
+  /** @} // Startup/Shutdown */
+
+  /**
+   * @defgroup Messaging
+   * @{
+   */
+  virtual int send_message(Message *m, const entity_inst_t& dest) {
+    return _send_message(m, dest, false);
+  }
+
+  virtual int send_message(Message *m, Connection *con) {
+    return _send_message(m, con, false);
+  }
+
+  virtual int lazy_send_message(Message *m, const entity_inst_t& dest) {
+    return _send_message(m, dest, true);
+  }
+
+  virtual int lazy_send_message(Message *m, Connection *con) {
+    return _send_message(m, con, true);
+  }
+  /** @} // Messaging */
+
+  /**
+   * @defgroup Connection Management
+   * @{
+   */
+  virtual ConnectionRef get_connection(const entity_inst_t& dest);
+  virtual ConnectionRef get_loopback_connection();
+  virtual int send_keepalive(const entity_inst_t& addr);
+  virtual int send_keepalive(Connection *con);
+  virtual void mark_down(const entity_addr_t& addr);
+  virtual void mark_down(Connection *con);
+  virtual void mark_down_on_empty(Connection *con);
+  virtual void mark_disposable(Connection *con);
+  virtual void mark_down_all();
+  /** @} // Connection Management */
+protected:
+  /**
+   * @defgroup Messenger Interfaces
+   * @{
+   */
+  /**
+   * Start up the DispatchQueue thread once we have somebody to dispatch to.
+   */
+  virtual void ready();
+  /** @} // Messenger Interfaces */
+private:
+  /**
+   * @defgroup Inner classes
+   * @{
+   */
+
+public:
+  Accepter accepter;
+  DispatchQueue dispatch_queue;
+
+  friend class Accepter;
+
+  /**
+   * Register a new pipe for accept
+   *
+   * @param sd socket
+   */
+  Pipe *add_accept_pipe(int sd);
+
+private:
+
+  /**
+   * A thread used to tear down Pipes when they're complete.
+   */
+  class ReaperThread : public Thread {
+    SimpleMessenger *msgr;
+  public:
+    ReaperThread(SimpleMessenger *m) : msgr(m) {}
+    void *entry() {
+      msgr->reaper_entry();
+      return 0;
+    }
+  } reaper_thread;
+
+  /**
+   * @} // Inner classes
+   */
+
+  /**
+   * @defgroup Utility functions
+   * @{
+   */
+
+  /**
+   * Create a Pipe associated with the given entity (of the given type).
+   * Initiate the connection. (This function returning does not guarantee
+   * connection success.)
+   *
+   * @param addr The address of the entity to connect to.
+   * @param type The peer type of the entity at the address.
+   * @param con An existing Connection to associate with the new Pipe. If
+   * NULL, it creates a new Connection.
+   * @param msg an initial message to queue on the new pipe
+   *
+   * @return a pointer to the newly-created Pipe. Caller does not own a
+   * reference; take one if you need it.
+   */
+  Pipe *connect_rank(const entity_addr_t& addr, int type, Connection *con, Message *first);
+  /**
+   * Send a message, lazily or not.
+   * This just glues [lazy_]send_message together and passes
+   * the input on to submit_message.
+   */
+  int _send_message(Message *m, const entity_inst_t& dest, bool lazy);
+  /**
+   * Same as above, but for the Connection-based variants.
+   */
+  int _send_message(Message *m, Connection *con, bool lazy);
+  /**
+   * Queue up a Message for delivery to the entity specified
+   * by addr and dest_type.
+   * submit_message() is responsible for creating
+   * new Pipes (and closing old ones) as necessary.
+   *
+   * @param m The Message to queue up. This function eats a reference.
+   * @param con The existing Connection to use, or NULL if you don't know of one.
+   * @param addr The address to send the Message to.
+   * @param dest_type The peer type of the address we're sending to
+   * @param lazy If true, do not establish or fix a Connection to send the Message;
+   * just drop silently under failure.
+   */
+  void submit_message(Message *m, Connection *con,
+                      const entity_addr_t& addr, int dest_type, bool lazy);
+  /**
+   * Look through the pipes in the pipe_reap_queue and tear them down.
+   */
+  void reaper();
+  /**
+   * @} // Utility functions
+   */
+
+  // SimpleMessenger stuff
+  /// the peer type of our endpoint
+  int my_type;
+  /// approximately unique ID set by the Constructor for use in entity_addr_t
+  uint64_t nonce;
+  /// overall lock used for SimpleMessenger data structures
+  Mutex lock;
+  /// true, specifying we haven't learned our addr; set false when we find it.
+  // maybe this should be protected by the lock?
+  bool need_addr;
+
+public:
+  bool get_need_addr() const { return need_addr; }
+
+private:
+  /**
+   *  false; set to true if the SimpleMessenger bound to a specific address;
+   *  and set false again by Accepter::stop(). This isn't lock-protected
+   *  since you shouldn't be able to race the only writers.
+   */
+  bool did_bind;
+  /// counter for the global seq our connection protocol uses
+  __u32 global_seq;
+  /// lock to protect the global_seq
+  ceph_spinlock_t global_seq_lock;
+
+  /**
+   * hash map of addresses to Pipes
+   *
+   * NOTE: a Pipe* with state CLOSED may still be in the map but is considered
+   * invalid and can be replaced by anyone holding the msgr lock
+   */
+  ceph::unordered_map rank_pipe;
+  /**
+   * list of pipes are in teh process of accepting
+   *
+   * These are not yet in the rank_pipe map.
+   */
+  set accepting_pipes;
+  /// a set of all the Pipes we have which are somehow active
+  set      pipes;
+  /// a list of Pipes we want to tear down
+  list     pipe_reap_queue;
+
+  /// internal cluster protocol version, if any, for talking to entities of the same type.
+  int cluster_protocol;
+
+  /// lock protecting policy
+  Mutex policy_lock;
+  /// the default Policy we use for Pipes
+  Policy default_policy;
+  /// map specifying different Policies for specific peer types
+  map policy_map; // entity_name_t::type -> Policy
+
+  /// Throttle preventing us from building up a big backlog waiting for dispatch
+  Throttle dispatch_throttler;
+
+  bool reaper_started, reaper_stop;
+  Cond reaper_cond;
+
+  /// This Cond is slept on by wait() and signaled by dispatch_entry()
+  Cond  wait_cond;
+
+  friend class Pipe;
+
+  Pipe *_lookup_pipe(const entity_addr_t& k) {
+    ceph::unordered_map::iterator p = rank_pipe.find(k);
+    if (p == rank_pipe.end())
+      return NULL;
+    // see lock cribbing in Pipe::fault()
+    if (p->second->state_closed.read())
+      return NULL;
+    return p->second;
+  }
+
+public:
+
+  int timeout;
+
+  /// con used for sending messages to ourselves
+  ConnectionRef local_connection;
+
+  /**
+   * @defgroup SimpleMessenger internals
+   * @{
+   */
+
+  /**
+   * This wraps ms_deliver_get_authorizer. We use it for Pipe.
+   */
+  AuthAuthorizer *get_authorizer(int peer_type, bool force_new);
+  /**
+   * This wraps ms_deliver_verify_authorizer; we use it for Pipe.
+   */
+  bool verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& auth, bufferlist& auth_reply,
+                         bool& isvalid,CryptoKey& session_key);
+  /**
+   * Increment the global sequence for this SimpleMessenger and return it.
+   * This is for the connect protocol, although it doesn't hurt if somebody
+   * else calls it.
+   *
+   * @return a global sequence ID that nobody else has seen.
+   */
+  __u32 get_global_seq(__u32 old=0) {
+    ceph_spin_lock(&global_seq_lock);
+    if (old > global_seq)
+      global_seq = old;
+    __u32 ret = ++global_seq;
+    ceph_spin_unlock(&global_seq_lock);
+    return ret;
+  }
+  /**
+   * Get the protocol version we support for the given peer type: either
+   * a peer protocol (if it matches our own), the protocol version for the
+   * peer (if we're connecting), or our protocol version (if we're accepting).
+   */
+  int get_proto_version(int peer_type, bool connect);
+
+  /**
+   * Fill in the address and peer type for the local connection, which
+   * is used for delivering messages back to ourself.
+   */
+  void init_local_connection();
+  /**
+   * Tell the SimpleMessenger its full IP address.
+   *
+   * This is used by Pipes when connecting to other endpoints, and
+   * probably shouldn't be called by anybody else.
+   */
+  void learned_addr(const entity_addr_t& peer_addr_for_me);
+
+  /**
+   * Tell the SimpleMessenger its address is no longer known
+   *
+   * This happens when we rebind to a new port.
+   */
+  void unlearn_addr();
+
+  /**
+   * Get the Policy associated with a type of peer.
+   * @param t The peer type to get the default policy for.
+   *
+   * @return A const Policy reference.
+   */
+  Policy get_policy(int t) {
+    Mutex::Locker l(policy_lock);
+    if (policy_map.count(t))
+      return policy_map[t];
+    else
+      return default_policy;
+  }
+  Policy get_default_policy() {
+    Mutex::Locker l(policy_lock);
+    return default_policy;
+  }
+
+  /**
+   * Release memory accounting back to the dispatch throttler.
+   *
+   * @param msize The amount of memory to release.
+   */
+  void dispatch_throttle_release(uint64_t msize);
+
+  /**
+   * This function is used by the reaper thread. As long as nobody
+   * has set reaper_stop, it calls the reaper function, then
+   * waits to be signaled when it needs to reap again (or when it needs
+   * to stop).
+   */
+  void reaper_entry();
+  /**
+   * Add a pipe to the pipe_reap_queue, to be torn down on
+   * the next call to reaper().
+   * It should really only be the Pipe calling this, in our current
+   * implementation.
+   *
+   * @param pipe A Pipe which has stopped its threads and is
+   * ready to be torn down.
+   */
+  void queue_reap(Pipe *pipe);
+  /**
+   * @} // SimpleMessenger Internals
+   */
+} ;
+
+#endif
diff --git a/ceph/src/msg/msg_types.cc b/ceph/src/msg/msg_types.cc
new file mode 100644
index 00000000..b02db768
--- /dev/null
+++ b/ceph/src/msg/msg_types.cc
@@ -0,0 +1,154 @@
+
+#include "msg_types.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+void entity_name_t::dump(Formatter *f) const
+{
+  f->dump_string("type", type_str());
+  f->dump_unsigned("num", num());
+}
+
+
+void entity_addr_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("nonce", nonce);
+  f->dump_stream("addr") << addr;
+}
+
+void entity_name_t::generate_test_instances(list& o)
+{
+  o.push_back(new entity_name_t(entity_name_t::MON()));
+  o.push_back(new entity_name_t(entity_name_t::MON(1)));
+  o.push_back(new entity_name_t(entity_name_t::OSD(1)));
+  o.push_back(new entity_name_t(entity_name_t::CLIENT(1)));
+}
+
+void entity_addr_t::generate_test_instances(list& o)
+{
+  o.push_back(new entity_addr_t());
+  entity_addr_t *a = new entity_addr_t();
+  a->set_nonce(1);
+  o.push_back(a);
+  entity_addr_t *b = new entity_addr_t();
+  b->set_nonce(5);
+  b->set_family(AF_INET);
+  b->set_in4_quad(0, 127);
+  b->set_in4_quad(1, 0);
+  b->set_in4_quad(2, 1);
+  b->set_in4_quad(3, 2);
+  b->set_port(2);
+  o.push_back(b);
+}
+
+bool entity_addr_t::parse(const char *s, const char **end)
+{
+  memset(this, 0, sizeof(*this));
+
+  const char *start = s;
+  bool brackets = false;
+  if (*start == '[') {
+    start++;
+    brackets = true;
+  }
+  
+  // inet_pton() requires a null terminated input, so let's fill two
+  // buffers, one with ipv4 allowed characters, and one with ipv6, and
+  // then see which parses.
+  char buf4[39];
+  char *o = buf4;
+  const char *p = start;
+  while (o < buf4 + sizeof(buf4) &&
+	 *p && ((*p == '.') ||
+		(*p >= '0' && *p <= '9'))) {
+    *o++ = *p++;
+  }
+  *o = 0;
+
+  char buf6[64];  // actually 39 + null is sufficient.
+  o = buf6;
+  p = start;
+  while (o < buf6 + sizeof(buf6) &&
+	 *p && ((*p == ':') ||
+		(*p >= '0' && *p <= '9') ||
+		(*p >= 'a' && *p <= 'f') ||
+		(*p >= 'A' && *p <= 'F'))) {
+    *o++ = *p++;
+  }
+  *o = 0;
+  //cout << "buf4 is '" << buf4 << "', buf6 is '" << buf6 << "'" << std::endl;
+
+  // ipv4?
+  struct in_addr a4;
+  struct in6_addr a6;
+  if (inet_pton(AF_INET, buf4, &a4)) {
+    addr4.sin_addr.s_addr = a4.s_addr;
+    addr.ss_family = AF_INET;
+    p = start + strlen(buf4);
+  } else if (inet_pton(AF_INET6, buf6, &a6)) {
+    addr.ss_family = AF_INET6;
+    memcpy(&addr6.sin6_addr, &a6, sizeof(a6));
+    p = start + strlen(buf6);
+  } else {
+    return false;
+  }
+
+  if (brackets) {
+    if (*p != ']')
+      return false;
+    p++;
+  }
+  
+  //cout << "p is " << *p << std::endl;
+  if (*p == ':') {
+    // parse a port, too!
+    p++;
+    int port = atoi(p);
+    set_port(port);
+    while (*p && *p >= '0' && *p <= '9')
+      p++;
+  }
+
+  if (*p == '/') {
+    // parse nonce, too
+    p++;
+    int non = atoi(p);
+    set_nonce(non);
+    while (*p && *p >= '0' && *p <= '9')
+      p++;
+  }
+
+  if (end)
+    *end = p;
+
+  //cout << *this << std::endl;
+  return true;
+}
+
+
+
+ostream& operator<<(ostream& out, const sockaddr_storage &ss)
+{
+  char buf[NI_MAXHOST] = { 0 };
+  char serv[NI_MAXSERV] = { 0 };
+  size_t hostlen;
+
+  if (ss.ss_family == AF_INET)
+    hostlen = sizeof(struct sockaddr_in);
+  else if (ss.ss_family == AF_INET6)
+    hostlen = sizeof(struct sockaddr_in6);
+  else
+    hostlen = sizeof(struct sockaddr_storage);
+  getnameinfo((struct sockaddr *)&ss, hostlen, buf, sizeof(buf),
+	      serv, sizeof(serv),
+	      NI_NUMERICHOST | NI_NUMERICSERV);
+  if (ss.ss_family == AF_INET6)
+    return out << '[' << buf << "]:" << serv;
+  return out //<< ss.ss_family << ":"
+	     << buf << ':' << serv;
+}
diff --git a/ceph/src/msg/msg_types.h b/ceph/src/msg/msg_types.h
new file mode 100644
index 00000000..5d86b3a2
--- /dev/null
+++ b/ceph/src/msg/msg_types.h
@@ -0,0 +1,444 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MSG_TYPES_H
+#define CEPH_MSG_TYPES_H
+
+#include 
+
+#include "include/types.h"
+#include "include/blobhash.h"
+#include "include/encoding.h"
+#include "include/hash_namespace.h"
+
+namespace ceph {
+  class Formatter;
+}
+
+inline bool operator==(const sockaddr_in& a, const sockaddr_in& b) {
+  return strncmp((const char*)&a, (const char*)&b, sizeof(a)) == 0;
+}
+inline bool operator!=(const sockaddr_in& a, const sockaddr_in& b) {
+  return strncmp((const char*)&a, (const char*)&b, sizeof(a)) != 0;
+}
+
+extern ostream& operator<<(ostream& out, const sockaddr_storage &ss);
+
+class entity_name_t {
+public:
+  __u8 _type;
+  int64_t _num;
+
+public:
+  static const int TYPE_MON = CEPH_ENTITY_TYPE_MON;
+  static const int TYPE_MDS = CEPH_ENTITY_TYPE_MDS;
+  static const int TYPE_OSD = CEPH_ENTITY_TYPE_OSD;
+  static const int TYPE_CLIENT = CEPH_ENTITY_TYPE_CLIENT;
+
+  static const int NEW = -1;
+
+  // cons
+  entity_name_t() : _type(0), _num(0) { }
+  entity_name_t(int t, int64_t n) : _type(t), _num(n) { }
+  entity_name_t(const ceph_entity_name &n) : 
+    _type(n.type), _num(n.num) { }
+
+  // static cons
+  static entity_name_t MON(int i=NEW) { return entity_name_t(TYPE_MON, i); }
+  static entity_name_t MDS(int i=NEW) { return entity_name_t(TYPE_MDS, i); }
+  static entity_name_t OSD(int i=NEW) { return entity_name_t(TYPE_OSD, i); }
+  static entity_name_t CLIENT(int i=NEW) { return entity_name_t(TYPE_CLIENT, i); }
+  
+  int64_t num() const { return _num; }
+  int type() const { return _type; }
+  const char *type_str() const {
+    return ceph_entity_type_name(type());
+  }
+
+  bool is_new() const { return num() < 0; }
+
+  bool is_client() const { return type() == TYPE_CLIENT; }
+  bool is_mds() const { return type() == TYPE_MDS; }
+  bool is_osd() const { return type() == TYPE_OSD; }
+  bool is_mon() const { return type() == TYPE_MON; }
+
+  operator ceph_entity_name() const {
+    ceph_entity_name n = { _type, init_le64(_num) };
+    return n;
+  }
+
+  bool parse(const string& s) {
+    const char *start = s.c_str();
+    char *end;
+    bool got = parse(start, &end);
+    return got && end == start + s.length();
+  }
+  bool parse(const char *start, char **end) {
+    if (strstr(start, "mon.") == start) {
+      _type = TYPE_MON;
+      start += 4;
+    } else if (strstr(start, "osd.") == start) {
+      _type = TYPE_OSD;
+      start += 4;
+    } else if (strstr(start, "mds.") == start) {
+      _type = TYPE_MDS;
+      start += 4;
+    } else if (strstr(start, "client.") == start) {
+      _type = TYPE_CLIENT;
+      start += 7;
+    } else {
+      return false;
+    }
+    if (isspace(*start))
+      return false;
+    _num = strtoll(start, end, 10);
+    if (*end == NULL || *end == start)
+      return false;
+    return true;
+  }
+
+  void encode(bufferlist& bl) const {
+    ::encode(_type, bl);
+    ::encode(_num, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(_type, bl);
+    ::decode(_num, bl);
+  }
+  void dump(Formatter *f) const;
+
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(entity_name_t)
+
+inline bool operator== (const entity_name_t& l, const entity_name_t& r) { 
+  return (l.type() == r.type()) && (l.num() == r.num()); }
+inline bool operator!= (const entity_name_t& l, const entity_name_t& r) { 
+  return (l.type() != r.type()) || (l.num() != r.num()); }
+inline bool operator< (const entity_name_t& l, const entity_name_t& r) { 
+  return (l.type() < r.type()) || (l.type() == r.type() && l.num() < r.num()); }
+
+inline std::ostream& operator<<(std::ostream& out, const entity_name_t& addr) {
+  //if (addr.is_namer()) return out << "namer";
+  if (addr.is_new() || addr.num() < 0)
+    return out << addr.type_str() << ".?";
+  else
+    return out << addr.type_str() << '.' << addr.num();
+}
+inline std::ostream& operator<<(std::ostream& out, const ceph_entity_name& addr) {
+  return out << *(const entity_name_t*)&addr;
+}
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash< entity_name_t >
+  {
+    size_t operator()( const entity_name_t &m ) const
+    {
+      return rjhash32(m.type() ^ m.num());
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+
+
+/*
+ * an entity's network address.
+ * includes a random value that prevents it from being reused.
+ * thus identifies a particular process instance.
+ * ipv4 for now.
+ */
+
+/*
+ * encode sockaddr.ss_family in big endian
+ */
+static inline void encode(const sockaddr_storage& a, bufferlist& bl) {
+  struct sockaddr_storage ss = a;
+#if !defined(__FreeBSD__)
+  ss.ss_family = htons(ss.ss_family);
+#endif
+  ::encode_raw(ss, bl);
+}
+static inline void decode(sockaddr_storage& a, bufferlist::iterator& bl) {
+  ::decode_raw(a, bl);
+#if !defined(__FreeBSD__)
+  a.ss_family = ntohs(a.ss_family);
+#endif
+}
+
+struct entity_addr_t {
+  __u32 type;
+  __u32 nonce;
+  union {
+    sockaddr_storage addr;
+    sockaddr_in addr4;
+    sockaddr_in6 addr6;
+  };
+
+  unsigned int addr_size() const {
+    switch (addr.ss_family) {
+    case AF_INET:
+      return sizeof(addr4);
+      break;
+    case AF_INET6:
+      return sizeof(addr6);
+      break;
+    }
+    return sizeof(addr);
+  }
+
+  entity_addr_t() : type(0), nonce(0) { 
+    memset(&addr, 0, sizeof(addr));
+  }
+  entity_addr_t(const ceph_entity_addr &o) {
+    type = o.type;
+    nonce = o.nonce;
+    addr = o.in_addr;
+#if !defined(__FreeBSD__)
+    addr.ss_family = ntohs(addr.ss_family);
+#endif
+  }
+
+  __u32 get_nonce() const { return nonce; }
+  void set_nonce(__u32 n) { nonce = n; }
+
+  int get_family() const {
+    return addr.ss_family;
+  }
+  void set_family(int f) {
+    addr.ss_family = f;
+  }
+  
+  sockaddr_storage &ss_addr() {
+    return addr;
+  }
+  sockaddr_in &in4_addr() {
+    return addr4;
+  }
+  sockaddr_in6 &in6_addr() {
+    return addr6;
+  }
+
+  bool set_sockaddr(struct sockaddr *sa)
+  {
+    switch (sa->sa_family) {
+    case AF_INET:
+      memcpy(&addr4, sa, sizeof(sockaddr_in));
+      break;
+    case AF_INET6:
+      memcpy(&addr6, sa, sizeof(sockaddr_in6));
+      break;
+    default:
+      return false;
+    }
+    return true;
+  }
+
+  void set_in4_quad(int pos, int val) {
+    addr4.sin_family = AF_INET;
+    unsigned char *ipq = (unsigned char*)&addr4.sin_addr.s_addr;
+    ipq[pos] = val;
+  }
+  void set_port(int port) {
+    switch (addr.ss_family) {
+    case AF_INET:
+      addr4.sin_port = htons(port);
+      break;
+    case AF_INET6:
+      addr6.sin6_port = htons(port);
+      break;
+    default:
+      assert(0);
+    }
+  }
+  int get_port() const {
+    switch (addr.ss_family) {
+    case AF_INET:
+      return ntohs(addr4.sin_port);
+      break;
+    case AF_INET6:
+      return ntohs(addr6.sin6_port);
+      break;
+    }
+    return 0;
+  }
+
+  operator ceph_entity_addr() const {
+    ceph_entity_addr a;
+    a.type = 0;
+    a.nonce = nonce;
+    a.in_addr = addr;
+#if !defined(__FreeBSD__)
+    a.in_addr.ss_family = htons(addr.ss_family);
+#endif
+    return a;
+  }
+
+  bool probably_equals(const entity_addr_t &o) const {
+    if (get_port() != o.get_port())
+      return false;
+    if (get_nonce() != o.get_nonce())
+      return false;
+    if (is_blank_ip() || o.is_blank_ip())
+      return true;
+    if (memcmp(&addr, &o.addr, sizeof(addr)) == 0)
+      return true;
+    return false;
+  }
+  
+  bool is_same_host(const entity_addr_t &o) const {
+    if (addr.ss_family != o.addr.ss_family)
+      return false;
+    if (addr.ss_family == AF_INET)
+      return addr4.sin_addr.s_addr == o.addr4.sin_addr.s_addr;
+    if (addr.ss_family == AF_INET6)
+      return memcmp(addr6.sin6_addr.s6_addr,
+		    o.addr6.sin6_addr.s6_addr,
+		    sizeof(addr6.sin6_addr.s6_addr)) == 0;
+    return false;
+  }
+
+  bool is_blank_ip() const {
+    switch (addr.ss_family) {
+    case AF_INET:
+      return addr4.sin_addr.s_addr == INADDR_ANY;
+    case AF_INET6:
+      return memcmp(&addr6.sin6_addr, &in6addr_any, sizeof(in6addr_any)) == 0;
+    default:
+      return true;
+    }
+  }
+
+  bool is_ip() const {
+    switch (addr.ss_family) {
+    case AF_INET:
+    case AF_INET6:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  bool parse(const char *s, const char **end = 0);
+
+  void encode(bufferlist& bl) const {
+    ::encode(type, bl);
+    ::encode(nonce, bl);
+    ::encode(addr, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(type, bl);
+    ::decode(nonce, bl);
+    ::decode(addr, bl);
+  }
+
+  void dump(Formatter *f) const;
+
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(entity_addr_t)
+
+inline ostream& operator<<(ostream& out, const entity_addr_t &addr)
+{
+  return out << addr.addr << '/' << addr.nonce;
+}
+
+inline bool operator==(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) == 0; }
+inline bool operator!=(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) != 0; }
+inline bool operator<(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) < 0; }
+inline bool operator<=(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) <= 0; }
+inline bool operator>(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) > 0; }
+inline bool operator>=(const entity_addr_t& a, const entity_addr_t& b) { return memcmp(&a, &b, sizeof(a)) >= 0; }
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash< entity_addr_t >
+  {
+    size_t operator()( const entity_addr_t& x ) const
+    {
+      static blobhash H;
+      return H((const char*)&x, sizeof(x));
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+
+/*
+ * a particular entity instance
+ */
+struct entity_inst_t {
+  entity_name_t name;
+  entity_addr_t addr;
+  entity_inst_t() {}
+  entity_inst_t(entity_name_t n, const entity_addr_t& a) : name(n), addr(a) {}
+  entity_inst_t(const ceph_entity_inst& i) : name(i.name), addr(i.addr) { }
+  entity_inst_t(const ceph_entity_name& n, const ceph_entity_addr &a) : name(n), addr(a) {}
+  operator ceph_entity_inst() {
+    ceph_entity_inst i = {name, addr};
+    return i;
+  }
+
+  void encode(bufferlist& bl) const {
+    ::encode(name, bl);
+    ::encode(addr, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(name, bl);
+    ::decode(addr, bl);
+  }
+};
+WRITE_CLASS_ENCODER(entity_inst_t)
+
+
+inline bool operator==(const entity_inst_t& a, const entity_inst_t& b) { 
+  return a.name == b.name && a.addr == b.addr;
+}
+inline bool operator!=(const entity_inst_t& a, const entity_inst_t& b) { 
+  return a.name != b.name || a.addr != b.addr;
+}
+inline bool operator<(const entity_inst_t& a, const entity_inst_t& b) { 
+  return a.name < b.name || (a.name == b.name && a.addr < b.addr);
+}
+inline bool operator<=(const entity_inst_t& a, const entity_inst_t& b) {
+  return a.name < b.name || (a.name == b.name && a.addr <= b.addr);
+}
+inline bool operator>(const entity_inst_t& a, const entity_inst_t& b) { return b < a; }
+inline bool operator>=(const entity_inst_t& a, const entity_inst_t& b) { return b <= a; }
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash< entity_inst_t >
+  {
+    size_t operator()( const entity_inst_t& x ) const
+    {
+      static hash< entity_name_t > H;
+      static hash< entity_addr_t > I;
+      return H(x.name) ^ I(x.addr);
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+
+inline ostream& operator<<(ostream& out, const entity_inst_t &i)
+{
+  return out << i.name << " " << i.addr;
+}
+inline ostream& operator<<(ostream& out, const ceph_entity_inst &i)
+{
+  entity_inst_t n = i;
+  return out << n;
+}
+
+
+
+
+
+#endif
diff --git a/ceph/src/objclass/class_api.cc b/ceph/src/objclass/class_api.cc
new file mode 100644
index 00000000..46a8ff54
--- /dev/null
+++ b/ceph/src/objclass/class_api.cc
@@ -0,0 +1,625 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/config.h"
+#include "common/debug.h"
+
+#include "objclass/objclass.h"
+#include "osd/ReplicatedPG.h"
+
+#include "osd/ClassHandler.h"
+
+#include "auth/Crypto.h"
+#include "common/armor.h"
+
+static ClassHandler *ch;
+
+#define dout_subsys ceph_subsys_objclass
+
+void cls_initialize(ClassHandler *h)
+{
+  ch = h;
+}
+
+void cls_finalize()
+{
+  ch = NULL;
+}
+
+
+void *cls_alloc(size_t size)
+{
+  return malloc(size);
+}
+
+void cls_free(void *p)
+{
+  free(p);
+}
+
+int cls_register(const char *name, cls_handle_t *handle)
+{
+  ClassHandler::ClassData *cls = ch->register_class(name);
+  *handle = (cls_handle_t)cls;
+  return (cls != NULL);
+}
+
+int cls_unregister(cls_handle_t handle)
+{
+  ClassHandler::ClassData *cls = (ClassHandler::ClassData *)handle;
+  ch->unregister_class(cls);
+  return 1;
+}
+
+int cls_register_method(cls_handle_t hclass, const char *method,
+                        int flags,
+                        cls_method_call_t class_call, cls_method_handle_t *handle)
+{
+  if (!(flags & (CLS_METHOD_RD | CLS_METHOD_WR)))
+    return -EINVAL;
+  ClassHandler::ClassData *cls = (ClassHandler::ClassData *)hclass;
+  cls_method_handle_t hmethod =(cls_method_handle_t)cls->register_method(method, flags, class_call);
+  if (handle)
+    *handle = hmethod;
+  return (hmethod != NULL);
+}
+
+int cls_register_cxx_method(cls_handle_t hclass, const char *method,
+                            int flags,
+			    cls_method_cxx_call_t class_call, cls_method_handle_t *handle)
+{
+  ClassHandler::ClassData *cls = (ClassHandler::ClassData *)hclass;
+  cls_method_handle_t hmethod = (cls_method_handle_t)cls->register_cxx_method(method, flags, class_call);
+  if (handle)
+    *handle = hmethod;
+  return (hmethod != NULL);
+}
+
+int cls_unregister_method(cls_method_handle_t handle)
+{
+  ClassHandler::ClassMethod *method = (ClassHandler::ClassMethod *)handle;
+  method->unregister();
+  return 1;
+}
+
+int cls_call(cls_method_context_t hctx, const char *cls, const char *method,
+                                 char *indata, int datalen,
+                                 char **outdata, int *outdatalen)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  bufferlist idata;
+  vector nops(1);
+  OSDOp& op = nops[0];
+  int r;
+
+  op.op.op = CEPH_OSD_OP_CALL;
+  op.op.cls.class_len = strlen(cls);
+  op.op.cls.method_len = strlen(method);
+  op.op.cls.indata_len = datalen;
+  op.indata.append(cls, op.op.cls.class_len);
+  op.indata.append(method, op.op.cls.method_len);
+  op.indata.append(indata, datalen);
+  r = (*pctx)->pg->do_osd_ops(*pctx, nops);
+
+  *outdata = (char *)malloc(op.outdata.length());
+  if (!*outdata)
+    return -ENOMEM;
+  memcpy(*outdata, op.outdata.c_str(), op.outdata.length());
+  *outdatalen = op.outdata.length();
+
+  return r;
+}
+
+int cls_getxattr(cls_method_context_t hctx, const char *name,
+                                 char **outdata, int *outdatalen)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  bufferlist name_data;
+  vector nops(1);
+  OSDOp& op = nops[0];
+  int r;
+
+  op.op.op = CEPH_OSD_OP_GETXATTR;
+  op.indata.append(name);
+  op.op.xattr.name_len = strlen(name);
+  r = (*pctx)->pg->do_osd_ops(*pctx, nops);
+
+  *outdata = (char *)malloc(op.outdata.length());
+  if (!*outdata)
+    return -ENOMEM;
+  memcpy(*outdata, op.outdata.c_str(), op.outdata.length());
+  *outdatalen = op.outdata.length();
+
+  return r;
+}
+
+int cls_setxattr(cls_method_context_t hctx, const char *name,
+                                 const char *value, int val_len)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  bufferlist name_data;
+  vector nops(1);
+  OSDOp& op = nops[0];
+  int r;
+
+  op.op.op = CEPH_OSD_OP_SETXATTR;
+  op.indata.append(name);
+  op.indata.append(value);
+  op.op.xattr.name_len = strlen(name);
+  op.op.xattr.value_len = val_len;
+  r = (*pctx)->pg->do_osd_ops(*pctx, nops);
+
+  return r;
+}
+
+int cls_read(cls_method_context_t hctx, int ofs, int len,
+                                 char **outdata, int *outdatalen)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  ops[0].op.op = CEPH_OSD_OP_SYNC_READ;
+  ops[0].op.extent.offset = ofs;
+  ops[0].op.extent.length = len;
+  int r = (*pctx)->pg->do_osd_ops(*pctx, ops);
+
+  *outdata = (char *)malloc(ops[0].outdata.length());
+  if (!*outdata)
+    return -ENOMEM;
+  memcpy(*outdata, ops[0].outdata.c_str(), ops[0].outdata.length());
+  *outdatalen = ops[0].outdata.length();
+
+  if (r < 0)
+    return r;
+
+  return *outdatalen;
+}
+
+int cls_get_request_origin(cls_method_context_t hctx, entity_inst_t *origin)
+{
+  ReplicatedPG::OpContext **pctx = static_cast(hctx);
+  *origin = (*pctx)->op->get_req()->get_orig_source_inst();
+  return 0;
+}
+
+int cls_cxx_create(cls_method_context_t hctx, bool exclusive)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  ops[0].op.op = CEPH_OSD_OP_CREATE;
+  ops[0].op.flags = (exclusive ? CEPH_OSD_OP_FLAG_EXCL : 0);
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_remove(cls_method_context_t hctx)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  ops[0].op.op = CEPH_OSD_OP_DELETE;
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_stat(cls_method_context_t hctx, uint64_t *size, time_t *mtime)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  int ret;
+  ops[0].op.op = CEPH_OSD_OP_STAT;
+  ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
+  if (ret < 0)
+    return ret;
+  bufferlist::iterator iter = ops[0].outdata.begin();
+  utime_t ut;
+  uint64_t s;
+  try {
+    ::decode(s, iter);
+    ::decode(ut, iter);
+  } catch (buffer::error& err) {
+    return -EIO;
+  }
+  if (size)
+    *size = s;
+  if (mtime)
+    *mtime = ut.sec();
+  return 0;
+}
+
+int cls_cxx_read(cls_method_context_t hctx, int ofs, int len, bufferlist *outbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  int ret;
+  ops[0].op.op = CEPH_OSD_OP_SYNC_READ;
+  ops[0].op.extent.offset = ofs;
+  ops[0].op.extent.length = len;
+  ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
+  if (ret < 0)
+    return ret;
+  outbl->claim(ops[0].outdata);
+  return outbl->length();
+}
+
+int cls_cxx_write(cls_method_context_t hctx, int ofs, int len, bufferlist *inbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  ops[0].op.op = CEPH_OSD_OP_WRITE;
+  ops[0].op.extent.offset = ofs;
+  ops[0].op.extent.length = len;
+  ops[0].indata = *inbl;
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_write_full(cls_method_context_t hctx, bufferlist *inbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  ops[0].op.op = CEPH_OSD_OP_WRITEFULL;
+  ops[0].op.extent.offset = 0;
+  ops[0].op.extent.length = inbl->length();
+  ops[0].indata = *inbl;
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_replace(cls_method_context_t hctx, int ofs, int len, bufferlist *inbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(2);
+  ops[0].op.op = CEPH_OSD_OP_TRUNCATE;
+  ops[0].op.extent.offset = 0;
+  ops[0].op.extent.length = 0;
+  ops[1].op.op = CEPH_OSD_OP_WRITE;
+  ops[1].op.extent.offset = ofs;
+  ops[1].op.extent.length = len;
+  ops[1].indata = *inbl;
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_getxattr(cls_method_context_t hctx, const char *name,
+                     bufferlist *outbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  bufferlist name_data;
+  vector nops(1);
+  OSDOp& op = nops[0];
+  int r;
+
+  op.op.op = CEPH_OSD_OP_GETXATTR;
+  op.indata.append(name);
+  op.op.xattr.name_len = strlen(name);
+  r = (*pctx)->pg->do_osd_ops(*pctx, nops);
+  if (r < 0)
+    return r;
+
+  outbl->claim(op.outdata);
+  return outbl->length();
+}
+
+int cls_cxx_getxattrs(cls_method_context_t hctx, map *attrset)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector nops(1);
+  OSDOp& op = nops[0];
+  int r;
+
+  op.op.op = CEPH_OSD_OP_GETXATTRS;
+  r = (*pctx)->pg->do_osd_ops(*pctx, nops);
+  if (r < 0)
+    return r;
+
+  bufferlist::iterator iter = op.outdata.begin();
+  try {
+    ::decode(*attrset, iter);
+  } catch (buffer::error& err) {
+    return -EIO;
+  }
+  return 0;
+}
+
+int cls_cxx_setxattr(cls_method_context_t hctx, const char *name,
+                     bufferlist *inbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  bufferlist name_data;
+  vector nops(1);
+  OSDOp& op = nops[0];
+  int r;
+
+  op.op.op = CEPH_OSD_OP_SETXATTR;
+  op.indata.append(name);
+  op.indata.append(*inbl);
+  op.op.xattr.name_len = strlen(name);
+  op.op.xattr.value_len = inbl->length();
+  r = (*pctx)->pg->do_osd_ops(*pctx, nops);
+
+  return r;
+}
+
+int cls_cxx_snap_revert(cls_method_context_t hctx, snapid_t snapid)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  ops[0].op.op = CEPH_OSD_OP_ROLLBACK;
+  ops[0].op.snap.snapid = snapid;
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_map_get_all_vals(cls_method_context_t hctx, map* vals)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  int ret;
+
+  string start_after;
+  string filter_prefix;
+  uint64_t max = (uint64_t)-1;
+  bufferlist inbl;
+
+  ::encode(start_after, op.indata);
+  ::encode(max, op.indata);
+  ::encode(filter_prefix, op.indata);
+
+  op.op.op = CEPH_OSD_OP_OMAPGETVALS;
+  
+  ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
+  if (ret < 0)
+    return ret;
+
+  bufferlist::iterator iter = op.outdata.begin();
+  try {
+    ::decode(*vals, iter);
+  } catch (buffer::error& err) {
+    return -EIO;
+  }
+  return vals->size();
+}
+
+int cls_cxx_map_get_keys(cls_method_context_t hctx, const string &start_obj,
+			 uint64_t max_to_get, set *keys)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  int ret;
+
+  ::encode(start_obj, op.indata);
+  ::encode(max_to_get, op.indata);
+
+  op.op.op = CEPH_OSD_OP_OMAPGETKEYS;
+
+  ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
+  if (ret < 0)
+    return ret;
+
+  bufferlist::iterator iter = op.outdata.begin();
+  try {
+    ::decode(*keys, iter);
+  } catch (buffer::error& err) {
+    return -EIO;
+  }
+  return keys->size();
+}
+
+int cls_cxx_map_get_vals(cls_method_context_t hctx, const string &start_obj,
+			 const string &filter_prefix, uint64_t max_to_get,
+			 map *vals)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  int ret;
+
+  bufferlist inbl;
+
+  ::encode(start_obj, op.indata);
+  ::encode(max_to_get, op.indata);
+  ::encode(filter_prefix, op.indata);
+
+  op.op.op = CEPH_OSD_OP_OMAPGETVALS;
+  
+  ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
+  if (ret < 0)
+    return ret;
+
+  bufferlist::iterator iter = op.outdata.begin();
+  try {
+    ::decode(*vals, iter);
+  } catch (buffer::error& err) {
+    return -EIO;
+  }
+  return vals->size();
+}
+
+int cls_cxx_map_read_header(cls_method_context_t hctx, bufferlist *outbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  int ret;
+  op.op.op = CEPH_OSD_OP_OMAPGETHEADER;
+  ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
+  if (ret < 0)
+    return ret;
+
+  outbl->claim(op.outdata);
+
+  return 0;
+}
+
+int cls_cxx_map_get_val(cls_method_context_t hctx, const string &key,
+			bufferlist *outbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  int ret;
+
+  set k;
+  k.insert(key);
+  ::encode(k, op.indata);
+
+  op.op.op = CEPH_OSD_OP_OMAPGETVALSBYKEYS;
+  ret = (*pctx)->pg->do_osd_ops(*pctx, ops);
+  if (ret < 0)
+    return ret;
+
+  bufferlist::iterator iter = op.outdata.begin();
+  try {
+    map m;
+
+    ::decode(m, iter);
+    map::iterator iter = m.begin();
+    if (iter == m.end())
+      return -ENOENT;
+
+    *outbl = iter->second;
+  } catch (buffer::error& e) {
+    return -EIO;
+  }
+  return 0;
+}
+
+int cls_cxx_map_set_val(cls_method_context_t hctx, const string &key,
+			bufferlist *inbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  bufferlist& update_bl = op.indata;
+  map m;
+  m[key] = *inbl;
+  ::encode(m, update_bl);
+
+  op.op.op = CEPH_OSD_OP_OMAPSETVALS;
+
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_map_set_vals(cls_method_context_t hctx,
+			 const std::map *map)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  bufferlist& update_bl = op.indata;
+  ::encode(*map, update_bl);
+
+  op.op.op = CEPH_OSD_OP_OMAPSETVALS;
+
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_map_clear(cls_method_context_t hctx)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+
+  op.op.op = CEPH_OSD_OP_OMAPCLEAR;
+
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_map_write_header(cls_method_context_t hctx, bufferlist *inbl)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  op.indata.claim(*inbl);
+
+  op.op.op = CEPH_OSD_OP_OMAPSETHEADER;
+
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_cxx_map_remove_key(cls_method_context_t hctx, const string &key)
+{
+  ReplicatedPG::OpContext **pctx = (ReplicatedPG::OpContext **)hctx;
+  vector ops(1);
+  OSDOp& op = ops[0];
+  bufferlist& update_bl = op.indata;
+  set to_rm;
+  to_rm.insert(key);
+
+  ::encode(to_rm, update_bl);
+
+  op.op.op = CEPH_OSD_OP_OMAPRMKEYS;
+
+  return (*pctx)->pg->do_osd_ops(*pctx, ops);
+}
+
+int cls_gen_random_bytes(char *buf, int size)
+{
+  return get_random_bytes(buf, size);
+}
+
+int cls_gen_rand_base64(char *dest, int size) /* size should be the required string size + 1 */
+{
+  char buf[size];
+  char tmp_dest[size + 4]; /* so that there's space for the extra '=' characters, and some */
+  int ret;
+
+  ret = cls_gen_random_bytes(buf, sizeof(buf));
+  if (ret < 0) {
+    generic_derr << "cannot get random bytes: " << ret << dendl;
+    return -1;
+  }
+
+  ret = ceph_armor(tmp_dest, &tmp_dest[sizeof(tmp_dest)],
+		   (const char *)buf, ((const char *)buf) + ((size - 1) * 3 + 4 - 1) / 4);
+  if (ret < 0) {
+    generic_derr << "ceph_armor failed" << dendl;
+    return -1;
+  }
+  tmp_dest[ret] = '\0';
+  memcpy(dest, tmp_dest, size);
+  dest[size] = '\0';
+
+  return 0;
+}
+
+uint64_t cls_current_version(cls_method_context_t hctx)
+{
+  ReplicatedPG::OpContext *ctx = *(ReplicatedPG::OpContext **)hctx;
+
+  return ctx->pg->info.last_user_version;
+}
+
+
+int cls_current_subop_num(cls_method_context_t hctx)
+{
+  ReplicatedPG::OpContext *ctx = *(ReplicatedPG::OpContext **)hctx;
+
+  return ctx->current_osd_subop_num;
+}
+
+void cls_cxx_subop_version(cls_method_context_t hctx, string *s)
+{
+  if (!s)
+    return;
+
+  char buf[32];
+  uint64_t ver = cls_current_version(hctx);
+  int subop_num = cls_current_subop_num(hctx);
+  snprintf(buf, sizeof(buf), "%lld.%d", (long long)ver, subop_num);
+
+  *s = buf;
+}
+
+int cls_log(int level, const char *format, ...)
+{
+   int size = 256;
+   va_list ap;
+   while (1) {
+     char buf[size];
+     va_start(ap, format);
+     int n = vsnprintf(buf, size, format, ap);
+     va_end(ap);
+#define MAX_SIZE 8196
+     if ((n > -1 && n < size) || size > MAX_SIZE) {
+       dout(level) << buf << dendl;
+       return n;
+     }
+     size *= 2;
+   }
+}
diff --git a/ceph/src/objclass/objclass.h b/ceph/src/objclass/objclass.h
new file mode 100644
index 00000000..6f0de282
--- /dev/null
+++ b/ceph/src/objclass/objclass.h
@@ -0,0 +1,156 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_OBJCLASS_H
+#define CEPH_OBJCLASS_H
+
+#ifdef __cplusplus
+
+#include "../include/types.h"
+#include "msg/msg_types.h"
+
+extern "C" {
+#endif
+
+#define CLS_VER(maj,min) \
+int __cls_ver__## maj ## _ ##min = 0; \
+int __cls_ver_maj = maj; \
+int __cls_ver_min = min;
+
+#define CLS_NAME(name) \
+int __cls_name__## name = 0; \
+const char *__cls_name = #name;
+
+#define CLS_METHOD_RD		0x1
+#define CLS_METHOD_WR		0x2
+#define CLS_METHOD_PUBLIC	0x4
+
+
+#define CLS_LOG(level, fmt, ...)					\
+  cls_log(level, " %s:%d: " fmt, __FILE__, __LINE__, ##__VA_ARGS__)
+#define CLS_ERR(fmt, ...) CLS_LOG(0, fmt, ##__VA_ARGS__)
+
+void __cls_init();
+
+typedef void *cls_handle_t;
+typedef void *cls_method_handle_t;
+typedef void *cls_method_context_t;
+typedef int (*cls_method_call_t)(cls_method_context_t ctx,
+				 char *indata, int datalen,
+				 char **outdata, int *outdatalen);
+typedef struct {
+	const char *name;
+	const char *ver;
+} cls_deps_t;
+
+/* class utils */
+extern int cls_log(int level, const char *format, ...)
+  __attribute__((__format__(printf, 2, 3)));
+extern void *cls_alloc(size_t size);
+extern void cls_free(void *p);
+
+extern int cls_read(cls_method_context_t hctx, int ofs, int len,
+                                 char **outdata, int *outdatalen);
+extern int cls_call(cls_method_context_t hctx, const char *cls, const char *method,
+                                 char *indata, int datalen,
+                                 char **outdata, int *outdatalen);
+extern int cls_getxattr(cls_method_context_t hctx, const char *name,
+                                 char **outdata, int *outdatalen);
+extern int cls_setxattr(cls_method_context_t hctx, const char *name,
+                                 const char *value, int val_len);
+/** This will fill in the passed origin pointer with the origin of the
+ * request which activated your class call. */
+extern int cls_get_request_origin(cls_method_context_t hctx,
+                                  entity_inst_t *origin);
+
+/* class registration api */
+extern int cls_register(const char *name, cls_handle_t *handle);
+extern int cls_unregister(cls_handle_t);
+
+extern int cls_register_method(cls_handle_t hclass, const char *method, int flags,
+                        cls_method_call_t class_call, cls_method_handle_t *handle);
+extern int cls_unregister_method(cls_method_handle_t handle);
+
+
+
+/* triggers */
+#define OBJ_READ    0x1
+#define OBJ_WRITE   0x2
+
+typedef int cls_trigger_t;
+
+extern int cls_link(cls_method_handle_t handle, int priority, cls_trigger_t trigger);
+extern int cls_unlink(cls_method_handle_t handle);
+
+
+/* should be defined by the class implementation
+   defined here inorder to get it compiled without C++ mangling */
+extern void class_init(void);
+extern void class_fini(void);
+
+#ifdef __cplusplus
+}
+
+typedef int (*cls_method_cxx_call_t)(cls_method_context_t ctx,
+				     class buffer::list *inbl, class buffer::list *outbl);
+
+extern int cls_register_cxx_method(cls_handle_t hclass, const char *method, int flags,
+				   cls_method_cxx_call_t class_call, cls_method_handle_t *handle);
+
+extern int cls_cxx_create(cls_method_context_t hctx, bool exclusive);
+extern int cls_cxx_remove(cls_method_context_t hctx);
+extern int cls_cxx_stat(cls_method_context_t hctx, uint64_t *size, time_t *mtime);
+extern int cls_cxx_read(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
+extern int cls_cxx_write(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
+extern int cls_cxx_write_full(cls_method_context_t hctx, bufferlist *bl);
+extern int cls_cxx_getxattr(cls_method_context_t hctx, const char *name,
+                            bufferlist *outbl);
+extern int cls_cxx_getxattrs(cls_method_context_t hctx, map *attrset);
+extern int cls_cxx_setxattr(cls_method_context_t hctx, const char *name,
+                            bufferlist *inbl);
+extern int cls_cxx_replace(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
+extern int cls_cxx_snap_revert(cls_method_context_t hctx, snapid_t snapid);
+extern int cls_cxx_map_clear(cls_method_context_t hctx);
+extern int cls_cxx_map_get_all_vals(cls_method_context_t hctx,
+                                    std::map *vals);
+extern int cls_cxx_map_get_keys(cls_method_context_t hctx,
+                                const string &start_after,
+                                uint64_t max_to_get,
+                                std::set *keys);
+extern int cls_cxx_map_get_vals(cls_method_context_t hctx,
+                                const string &start_after,
+                                const string &filter_prefix,
+                                uint64_t max_to_get,
+                                std::map *vals);
+extern int cls_cxx_map_read_header(cls_method_context_t hctx, bufferlist *outbl);
+extern int cls_cxx_map_get_val(cls_method_context_t hctx,
+                               const string &key, bufferlist *outbl);
+extern int cls_cxx_map_set_val(cls_method_context_t hctx,
+                               const string &key, bufferlist *inbl);
+extern int cls_cxx_map_set_vals(cls_method_context_t hctx,
+                                const std::map *map);
+extern int cls_cxx_map_write_header(cls_method_context_t hctx, bufferlist *inbl);
+extern int cls_cxx_map_remove_key(cls_method_context_t hctx, const string &key);
+extern int cls_cxx_map_update(cls_method_context_t hctx, bufferlist *inbl);
+
+/* utility functions */
+extern int cls_gen_random_bytes(char *buf, int size);
+extern int cls_gen_rand_base64(char *dest, int size); /* size should be the required string size + 1 */
+
+/* environment */
+extern uint64_t cls_current_version(cls_method_context_t hctx);
+extern int cls_current_subop_num(cls_method_context_t hctx);
+
+/* helpers */
+extern void cls_cxx_subop_version(cls_method_context_t hctx, string *s);
+
+/* These are also defined in rados.h and librados.h. Keep them in sync! */
+#define CEPH_OSD_TMAP_HDR 'h'
+#define CEPH_OSD_TMAP_SET 's'
+#define CEPH_OSD_TMAP_CREATE 'c'
+#define CEPH_OSD_TMAP_RM 'r'
+
+
+#endif
+
+#endif
diff --git a/ceph/src/ocf/Makefile.am b/ceph/src/ocf/Makefile.am
new file mode 100644
index 00000000..569f3abe
--- /dev/null
+++ b/ceph/src/ocf/Makefile.am
@@ -0,0 +1,23 @@
+EXTRA_DIST = ceph.in Makefile.in
+
+if WITH_OCF
+# The root of the OCF resource agent hierarchy
+# Per the OCF standard, it's always "lib",
+# not "lib64" (even on 64-bit platforms).
+ocfdir = $(prefix)/lib/ocf
+
+# The ceph provider directory
+radir = $(ocfdir)/resource.d/$(PACKAGE_NAME)
+
+ra_SCRIPTS = ceph rbd
+
+install-data-hook:
+	$(LN_S) ceph $(DESTDIR)$(radir)/osd
+	$(LN_S) ceph $(DESTDIR)$(radir)/mds
+	$(LN_S) ceph $(DESTDIR)$(radir)/mon
+
+uninstall-hook:
+	rm -f $(DESTDIR)$(radir)/osd
+	rm -f $(DESTDIR)$(radir)/mds
+	rm -f $(DESTDIR)$(radir)/mon
+endif
diff --git a/ceph/src/ocf/Makefile.in b/ceph/src/ocf/Makefile.in
new file mode 100644
index 00000000..4f243be0
--- /dev/null
+++ b/ceph/src/ocf/Makefile.in
@@ -0,0 +1,534 @@
+# Makefile.in generated by automake 1.11.3 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+# Foundation, Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+VPATH = @srcdir@
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+target_triplet = @target@
+subdir = src/ocf
+DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in \
+	$(srcdir)/ceph.in $(srcdir)/rbd.in
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_check_classpath.m4 \
+	$(top_srcdir)/m4/ac_prog_jar.m4 \
+	$(top_srcdir)/m4/ac_prog_javac.m4 \
+	$(top_srcdir)/m4/ac_prog_javac_works.m4 \
+	$(top_srcdir)/m4/ac_prog_javah.m4 \
+	$(top_srcdir)/m4/acx_pthread.m4 \
+	$(top_srcdir)/m4/ax_c_pretty_func.m4 \
+	$(top_srcdir)/m4/ax_c_var_func.m4 \
+	$(top_srcdir)/m4/ax_check_compile_flag.m4 \
+	$(top_srcdir)/m4/ax_cxx_static_cast.m4 \
+	$(top_srcdir)/m4/ax_intel.m4 $(top_srcdir)/m4/libtool.m4 \
+	$(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \
+	$(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \
+	$(top_srcdir)/m4/pkg.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/src/acconfig.h
+CONFIG_CLEAN_FILES = ceph rbd
+CONFIG_CLEAN_VPATH_FILES =
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(radir)"
+SCRIPTS = $(ra_SCRIPTS)
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN   " $@;
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+SOURCES =
+DIST_SOURCES =
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AM_CXXFLAGS = @AM_CXXFLAGS@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+AR = @AR@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BOOST_PROGRAM_OPTIONS_LIBS = @BOOST_PROGRAM_OPTIONS_LIBS@
+CC = @CC@
+CCAS = @CCAS@
+CCASDEPMODE = @CCASDEPMODE@
+CCASFLAGS = @CCASFLAGS@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPTOPP_CFLAGS = @CRYPTOPP_CFLAGS@
+CRYPTOPP_LIBS = @CRYPTOPP_LIBS@
+CRYPTO_CFLAGS = @CRYPTO_CFLAGS@
+CRYPTO_LIBS = @CRYPTO_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+EXTRA_CLASSPATH_JAR = @EXTRA_CLASSPATH_JAR@
+FGREP = @FGREP@
+GCOV_PREFIX_STRIP = @GCOV_PREFIX_STRIP@
+GIT_CHECK = @GIT_CHECK@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+INTEL_FLAGS = @INTEL_FLAGS@
+INTEL_PCLMUL_FLAGS = @INTEL_PCLMUL_FLAGS@
+INTEL_SSE2_FLAGS = @INTEL_SSE2_FLAGS@
+INTEL_SSE3_FLAGS = @INTEL_SSE3_FLAGS@
+INTEL_SSE4_1_FLAGS = @INTEL_SSE4_1_FLAGS@
+INTEL_SSE4_2_FLAGS = @INTEL_SSE4_2_FLAGS@
+INTEL_SSE_FLAGS = @INTEL_SSE_FLAGS@
+INTEL_SSSE3_FLAGS = @INTEL_SSSE3_FLAGS@
+JAR = @JAR@
+JAVAC = @JAVAC@
+JAVAH = @JAVAH@
+JDK_CPPFLAGS = @JDK_CPPFLAGS@
+KEYUTILS_LIB = @KEYUTILS_LIB@
+LD = @LD@
+LDFLAGS = @LDFLAGS@
+LIBEDIT_CFLAGS = @LIBEDIT_CFLAGS@
+LIBEDIT_LIBS = @LIBEDIT_LIBS@
+LIBFUSE = @LIBFUSE@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBTCMALLOC = @LIBTCMALLOC@
+LIBTOOL = @LIBTOOL@
+LIBZFS_CFLAGS = @LIBZFS_CFLAGS@
+LIBZFS_LIBS = @LIBZFS_LIBS@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBOBJS = @LTLIBOBJS@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NSS_CFLAGS = @NSS_CFLAGS@
+NSS_LIBS = @NSS_LIBS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
+PYTHON = @PYTHON@
+PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@
+PYTHON_PLATFORM = @PYTHON_PLATFORM@
+PYTHON_PREFIX = @PYTHON_PREFIX@
+PYTHON_VERSION = @PYTHON_VERSION@
+RANLIB = @RANLIB@
+RESOLV_LIBS = @RESOLV_LIBS@
+RPM_RELEASE = @RPM_RELEASE@
+SED = @SED@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+STRIP = @STRIP@
+VERSION = @VERSION@
+WARN_IGNORED_QUALIFIERS = @WARN_IGNORED_QUALIFIERS@
+WARN_TYPE_LIMITS = @WARN_TYPE_LIMITS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+acx_pthread_config = @acx_pthread_config@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+pkgpyexecdir = @pkgpyexecdir@
+pkgpythondir = @pkgpythondir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+pyexecdir = @pyexecdir@
+pythondir = @pythondir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+subdirs = @subdirs@
+sysconfdir = @sysconfdir@
+target = @target@
+target_alias = @target_alias@
+target_cpu = @target_cpu@
+target_os = @target_os@
+target_vendor = @target_vendor@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+EXTRA_DIST = ceph.in Makefile.in
+
+# The root of the OCF resource agent hierarchy
+# Per the OCF standard, it's always "lib",
+# not "lib64" (even on 64-bit platforms).
+@WITH_OCF_TRUE@ocfdir = $(prefix)/lib/ocf
+
+# The ceph provider directory
+@WITH_OCF_TRUE@radir = $(ocfdir)/resource.d/$(PACKAGE_NAME)
+@WITH_OCF_TRUE@ra_SCRIPTS = ceph rbd
+all: all-am
+
+.SUFFIXES:
+$(srcdir)/Makefile.in:  $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/ocf/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/ocf/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure:  $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4):  $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+ceph: $(top_builddir)/config.status $(srcdir)/ceph.in
+	cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@
+rbd: $(top_builddir)/config.status $(srcdir)/rbd.in
+	cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@
+install-raSCRIPTS: $(ra_SCRIPTS)
+	@$(NORMAL_INSTALL)
+	test -z "$(radir)" || $(MKDIR_P) "$(DESTDIR)$(radir)"
+	@list='$(ra_SCRIPTS)'; test -n "$(radir)" || list=; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \
+	done | \
+	sed -e 'p;s,.*/,,;n' \
+	    -e 'h;s|.*|.|' \
+	    -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \
+	$(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \
+	  { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+	    if ($$2 == $$4) { files[d] = files[d] " " $$1; \
+	      if (++n[d] == $(am__install_max)) { \
+		print "f", d, files[d]; n[d] = 0; files[d] = "" } } \
+	    else { print "f", d "/" $$4, $$1 } } \
+	  END { for (d in files) print "f", d, files[d] }' | \
+	while read type dir files; do \
+	     if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+	     test -z "$$files" || { \
+	       echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(radir)$$dir'"; \
+	       $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(radir)$$dir" || exit $$?; \
+	     } \
+	; done
+
+uninstall-raSCRIPTS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(ra_SCRIPTS)'; test -n "$(radir)" || exit 0; \
+	files=`for p in $$list; do echo "$$p"; done | \
+	       sed -e 's,.*/,,;$(transform)'`; \
+	dir='$(DESTDIR)$(radir)'; $(am__uninstall_files_from_dir)
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+tags: TAGS
+TAGS:
+
+ctags: CTAGS
+CTAGS:
+
+
+distdir: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(SCRIPTS)
+installdirs:
+	for dir in "$(DESTDIR)$(radir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+@WITH_OCF_FALSE@install-data-hook:
+@WITH_OCF_FALSE@uninstall-hook:
+clean: clean-am
+
+clean-am: clean-generic clean-libtool mostlyclean-am
+
+distclean: distclean-am
+	-rm -f Makefile
+distclean-am: clean-am distclean-generic
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-raSCRIPTS
+	@$(NORMAL_INSTALL)
+	$(MAKE) $(AM_MAKEFLAGS) install-data-hook
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-generic mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-raSCRIPTS
+	@$(NORMAL_INSTALL)
+	$(MAKE) $(AM_MAKEFLAGS) uninstall-hook
+.MAKE: install-am install-data-am install-strip uninstall-am
+
+.PHONY: all all-am check check-am clean clean-generic clean-libtool \
+	distclean distclean-generic distclean-libtool distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-data-hook install-dvi \
+	install-dvi-am install-exec install-exec-am install-html \
+	install-html-am install-info install-info-am install-man \
+	install-pdf install-pdf-am install-ps install-ps-am \
+	install-raSCRIPTS install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-generic mostlyclean-libtool pdf pdf-am \
+	ps ps-am uninstall uninstall-am uninstall-hook \
+	uninstall-raSCRIPTS
+
+
+@WITH_OCF_TRUE@install-data-hook:
+@WITH_OCF_TRUE@	$(LN_S) ceph $(DESTDIR)$(radir)/osd
+@WITH_OCF_TRUE@	$(LN_S) ceph $(DESTDIR)$(radir)/mds
+@WITH_OCF_TRUE@	$(LN_S) ceph $(DESTDIR)$(radir)/mon
+
+@WITH_OCF_TRUE@uninstall-hook:
+@WITH_OCF_TRUE@	rm -f $(DESTDIR)$(radir)/osd
+@WITH_OCF_TRUE@	rm -f $(DESTDIR)$(radir)/mds
+@WITH_OCF_TRUE@	rm -f $(DESTDIR)$(radir)/mon
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/ceph/src/ocf/ceph.in b/ceph/src/ocf/ceph.in
new file mode 100644
index 00000000..9448a298
--- /dev/null
+++ b/ceph/src/ocf/ceph.in
@@ -0,0 +1,177 @@
+#!/bin/sh
+
+# Initialization:
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+# Convenience variables
+# When sysconfdir isn't passed in as a configure flag,
+# it's defined in terms of prefix
+prefix=@prefix@
+CEPH_INIT=@sysconfdir@/init.d/ceph
+
+ceph_meta_data() {
+    local longdesc
+    local shortdesc
+    case $__SCRIPT_NAME in
+	"osd")
+	    longdesc="Wraps the ceph init script to provide an OCF resource agent that manages and monitors the Ceph OSD service."
+	    shortdesc="Manages a Ceph OSD instance."
+	    ;;
+	"mds")
+	    longdesc="Wraps the ceph init script to provide an OCF resource agent that manages and monitors the Ceph MDS service."
+	    shortdesc="Manages a Ceph MDS instance."
+	    ;;
+	"mon")
+	    longdesc="Wraps the ceph init script to provide an OCF resource agent that manages and monitors the Ceph MON service."
+	    shortdesc="Manages a Ceph MON instance."
+	    ;;
+    esac
+    
+cat <
+
+
+  0.1
+  ${longdesc}
+  ${shortdesc}
+  
+  
+    
+    
+    
+    
+    
+  
+
+EOF
+}
+
+ceph_action() {
+    local init_action
+    init_action="$1"
+
+    case ${__SCRIPT_NAME} in
+	osd|mds|mon)
+	    ocf_run $CEPH_INIT $init_action ${__SCRIPT_NAME}
+	    ;;
+	*)
+	    ocf_run $CEPH_INIT $init_action
+	    ;;
+    esac
+}
+
+ceph_validate_all() {
+    # Do we have the ceph init script?
+    check_binary @sysconfdir@/init.d/ceph
+
+    # Do we have a configuration file?
+    [ -e @sysconfdir@/ceph/ceph.conf ] || exit $OCF_ERR_INSTALLED
+}
+
+ceph_monitor() {
+    local rc
+
+    ceph_action status
+
+    # 0: running, and fully caught up with master
+    # 3: gracefully stopped
+    # any other: error
+    case "$?" in
+        0)
+            rc=$OCF_SUCCESS
+            ocf_log debug "Resource is running"
+            ;;
+        3)
+            rc=$OCF_NOT_RUNNING
+            ocf_log debug "Resource is not running"
+            ;;
+        *)
+            ocf_log err "Resource has failed"
+            rc=$OCF_ERR_GENERIC
+    esac
+
+    return $rc
+}
+
+ceph_start() {
+    # if resource is already running, bail out early
+    if ceph_monitor; then
+        ocf_log info "Resource is already running"
+        return $OCF_SUCCESS
+    fi
+
+    ceph_action start
+
+    while ! ceph_monitor; do
+        ocf_log debug "Resource has not started yet, waiting"
+        sleep 1
+    done
+
+    return $OCF_SUCCESS
+}
+
+ceph_stop() {
+    local rc
+
+    # exit immediately if configuration is not valid
+    ceph_validate_all || exit $?
+
+    ceph_monitor
+    rc=$?
+    case "$rc" in
+        "$OCF_SUCCESS")
+            # Currently running. Normal, expected behavior.
+            ocf_log debug "Resource is currently running"
+            ;;
+        "$OCF_NOT_RUNNING")
+            # Currently not running. Nothing to do.
+            ocf_log info "Resource is already stopped"
+            return $OCF_SUCCESS
+            ;;
+    esac
+
+    ceph_action stop
+
+    while ceph_monitor; do
+        ocf_log debug "Resource has not stopped yet, waiting"
+        sleep 1
+    done
+
+    # only return $OCF_SUCCESS if _everything_ succeeded as expected
+    return $OCF_SUCCESS
+
+}
+
+
+
+# Make sure meta-data and usage always succeed
+case $__OCF_ACTION in
+meta-data)      ceph_meta_data
+                exit $OCF_SUCCESS
+                ;;
+usage|help)     ceph_usage
+                exit $OCF_SUCCESS
+                ;;
+esac
+
+# Anything other than meta-data and usage must pass validation
+ceph_validate_all || exit $?
+
+# Translate each action into the appropriate function call
+case $__OCF_ACTION in
+start)          ceph_start;;
+stop)           ceph_stop;;
+status|monitor) ceph_monitor;;
+reload)         ocf_log info "Reloading..."
+                ceph_start
+                ;;
+validate-all)   ;;
+*)              ceph_usage
+                exit $OCF_ERR_UNIMPLEMENTED
+                ;;
+esac
+rc=$?
+
+exit $rc
diff --git a/ceph/src/ocf/rbd.in b/ceph/src/ocf/rbd.in
new file mode 100644
index 00000000..150ad6e6
--- /dev/null
+++ b/ceph/src/ocf/rbd.in
@@ -0,0 +1,296 @@
+#!/bin/sh
+#
+#   OCF resource agent for mapping and unmapping
+#   RADOS Block Devices (RBDs)
+#
+#   License:      GNU Lesser General Public License (LGPL) 2.1
+#   (c) 2012 Florian Haas, hastexo
+#
+
+# Initialization:
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+# Convenience variables
+# When sysconfdir isn't passed in as a configure flag,
+# it's defined in terms of prefix
+prefix=@prefix@
+
+# Defaults
+OCF_RESKEY_pool_default="rbd"
+OCF_RESKEY_cephconf_default="@sysconfdir@/@PACKAGE_TARNAME@/@PACKAGE_TARNAME@.conf"
+: ${OCF_RESKEY_pool=${OCF_RESKEY_pool_default}}
+: ${OCF_RESKEY_cephconf=${OCF_RESKEY_cephconf_default}}
+
+rbd_meta_data() {
+    cat <
+
+
+  0.1
+  
+Manages RADOS Block Devices (RBDs) as a highly available
+resource. Maps and unmaps RBDs as needed.
+  
+  Maps and unmaps RADOS Block Devices
+  
+    
+      
+      Name of the RBD device.
+      
+      RBD device name
+      
+    
+    
+      
+      Name of the RADOS pool where the RBD has been created
+      
+      RADOS pool name
+      
+    
+    
+      
+      Name of the device snapshot to map.
+      
+      Snapshot name
+      
+    
+    
+      
+      Location of the Ceph configuration file
+      
+      Ceph configuration file
+      
+    
+    
+      
+      Address (or comma-separated list of addresses) of
+      monitor servers to connect to. Overrides values from
+      configuration file.
+      
+      Monitor address(es)
+      
+    
+    
+      
+      Username to use when mapping the device. Required
+      if Ceph authentication is enabled on the monitor.
+      
+      Authentication username
+      
+    
+    
+      
+      File containing an authentication secret. Required
+      if Ceph authentication is enabled on the monitor.
+      
+      Authentication secret file
+      
+    
+  
+  
+    
+    
+    
+    
+    
+  
+
+EOF
+}
+
+rbd_usage() {
+        cat <
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/int_types.h"
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "include/compat.h"
+#include "include/linux_fiemap.h"
+#include "include/color.h"
+#include "include/buffer.h"
+#include "include/assert.h"
+
+#ifndef __CYGWIN__
+#include "btrfs_ioctl.h"
+#endif
+
+#include 
+#include 
+#include 
+
+#include "BtrfsFileStoreBackend.h"
+
+#include "common/errno.h"
+#include "common/config.h"
+
+#if defined(__linux__)
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "btrfsfilestorebackend(" << get_basedir_path() << ") "
+
+#define ALIGN_DOWN(x, by) ((x) - ((x) % (by)))
+#define ALIGNED(x, by) (!((x) % (by)))
+#define ALIGN_UP(x, by) (ALIGNED((x), (by)) ? (x) : (ALIGN_DOWN((x), (by)) + (by)))
+
+BtrfsFileStoreBackend::BtrfsFileStoreBackend(FileStore *fs):
+    GenericFileStoreBackend(fs), has_clone_range(false),
+    has_snap_create(false), has_snap_destroy(false),
+    has_snap_create_v2(false), has_wait_sync(false), stable_commits(false),
+    m_filestore_btrfs_clone_range(g_conf->filestore_btrfs_clone_range),
+    m_filestore_btrfs_snap (g_conf->filestore_btrfs_snap) { }
+
+int BtrfsFileStoreBackend::detect_features()
+{
+  int r;
+
+  r = GenericFileStoreBackend::detect_features();
+  if (r < 0)
+    return r;
+
+  // clone_range?
+  if (m_filestore_btrfs_clone_range) {
+    int fd = ::openat(get_basedir_fd(), "clone_range_test", O_CREAT|O_WRONLY, 0600);
+    if (fd >= 0) {
+      ::unlinkat(get_basedir_fd(), "clone_range_test", 0);
+      if (fd < 0) {
+	r = -errno;
+	dout(0) << "detect_feature: failed to unlink test file for CLONE_RANGE ioctl: "
+		<< cpp_strerror(r) << dendl;
+      }
+      btrfs_ioctl_clone_range_args clone_args;
+      memset(&clone_args, 0, sizeof(clone_args));
+      clone_args.src_fd = -1;
+      r = ::ioctl(fd, BTRFS_IOC_CLONE_RANGE, &clone_args);
+      if (r < 0 && errno == EBADF) {
+	dout(0) << "detect_feature: CLONE_RANGE ioctl is supported" << dendl;
+	has_clone_range = true;
+      } else {
+	r = -errno;
+	dout(0) << "detect_feature: CLONE_RANGE ioctl is NOT supported: " << cpp_strerror(r) << dendl;
+      }
+      TEMP_FAILURE_RETRY(::close(fd));
+    } else {
+      r = -errno;
+      dout(0) << "detect_feature: failed to create test file for CLONE_RANGE ioctl: "
+	      << cpp_strerror(r) << dendl;
+    }
+  } else {
+    dout(0) << "detect_feature: CLONE_RANGE ioctl is DISABLED via 'filestore btrfs clone range' option" << dendl;
+  }
+
+  struct btrfs_ioctl_vol_args vol_args;
+  memset(&vol_args, 0, sizeof(vol_args));
+
+  // create test source volume
+  vol_args.fd = 0;
+  strcpy(vol_args.name, "test_subvol");
+  r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SUBVOL_CREATE, &vol_args);
+  if (r != 0) {
+    r = -errno;
+    dout(0) << "detect_feature: failed to create simple subvolume " << vol_args.name << ": " << cpp_strerror(r) << dendl;
+  }
+  int srcfd = ::openat(get_basedir_fd(), vol_args.name, O_RDONLY);
+  if (srcfd < 0) {
+    r = -errno;
+    dout(0) << "detect_feature: failed to open " << vol_args.name << ": " << cpp_strerror(r) << dendl;
+  }
+
+  // snap_create and snap_destroy?
+  vol_args.fd = srcfd;
+  strcpy(vol_args.name, "sync_snap_test");
+  r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_CREATE, &vol_args);
+  int err = errno;
+  if (r == 0 || errno == EEXIST) {
+    dout(0) << "detect_feature: SNAP_CREATE is supported" << dendl;
+    has_snap_create = true;
+
+    r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_DESTROY, &vol_args);
+    if (r == 0) {
+      dout(0) << "detect_feature: SNAP_DESTROY is supported" << dendl;
+      has_snap_destroy = true;
+    } else {
+      err = -errno;
+      dout(0) << "detect_feature: SNAP_DESTROY failed: " << cpp_strerror(err) << dendl;
+
+      if (err == -EPERM && getuid() != 0) {
+	dout(0) << "detect_feature: failed with EPERM as non-root; remount with -o user_subvol_rm_allowed" << dendl;
+	cerr << TEXT_YELLOW
+	     << "btrfs SNAP_DESTROY failed as non-root; remount with -o user_subvol_rm_allowed"
+	     << TEXT_NORMAL << std::endl;
+      } else if (err == -EOPNOTSUPP) {
+	derr << "btrfs SNAP_DESTROY ioctl not supported; you need a kernel newer than 2.6.32" << dendl;
+      }
+    }
+  } else {
+    dout(0) << "detect_feature: SNAP_CREATE failed: " << cpp_strerror(err) << dendl;
+  }
+
+  if (m_filestore_btrfs_snap) {
+    if (has_snap_destroy)
+      stable_commits = true;
+    else
+      dout(0) << "detect_feature: snaps enabled, but no SNAP_DESTROY ioctl; DISABLING" << dendl;
+  }
+
+  // start_sync?
+  __u64 transid = 0;
+  r = ::ioctl(get_basedir_fd(), BTRFS_IOC_START_SYNC, &transid);
+  if (r < 0) {
+    int err = errno;
+    dout(0) << "detect_feature: START_SYNC got " << cpp_strerror(err) << dendl;
+  }
+  if (r == 0 && transid > 0) {
+    dout(0) << "detect_feature: START_SYNC is supported (transid " << transid << ")" << dendl;
+
+    // do we have wait_sync too?
+    r = ::ioctl(get_basedir_fd(), BTRFS_IOC_WAIT_SYNC, &transid);
+    if (r == 0 || errno == ERANGE) {
+      dout(0) << "detect_feature: WAIT_SYNC is supported" << dendl;
+      has_wait_sync = true;
+    } else {
+      int err = errno;
+      dout(0) << "detect_feature: WAIT_SYNC is NOT supported: " << cpp_strerror(err) << dendl;
+    }
+  } else {
+    int err = errno;
+    dout(0) << "detect_feature: START_SYNC is NOT supported: " << cpp_strerror(err) << dendl;
+  }
+
+  if (has_wait_sync) {
+    // async snap creation?
+    struct btrfs_ioctl_vol_args_v2 async_args;
+    memset(&async_args, 0, sizeof(async_args));
+    async_args.fd = srcfd;
+    async_args.flags = BTRFS_SUBVOL_CREATE_ASYNC;
+    strcpy(async_args.name, "async_snap_test");
+
+    // remove old one, first
+    struct stat st;
+    strcpy(vol_args.name, async_args.name);
+    if (::fstatat(get_basedir_fd(), vol_args.name, &st, 0) == 0) {
+      dout(0) << "detect_feature: removing old async_snap_test" << dendl;
+      r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_DESTROY, &vol_args);
+      if (r != 0) {
+	int err = errno;
+	dout(0) << "detect_feature: failed to remove old async_snap_test: " << cpp_strerror(err) << dendl;
+      }
+    }
+
+    r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_CREATE_V2, &async_args);
+    if (r == 0 || errno == EEXIST) {
+      dout(0) << "detect_feature: SNAP_CREATE_V2 is supported" << dendl;
+      has_snap_create_v2 = true;
+
+      // clean up
+      strcpy(vol_args.name, "async_snap_test");
+      r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_DESTROY, &vol_args);
+      if (r != 0) {
+	int err = errno;
+	dout(0) << "detect_feature: SNAP_DESTROY failed: " << cpp_strerror(err) << dendl;
+      }
+    } else {
+      int err = errno;
+      dout(0) << "detect_feature: SNAP_CREATE_V2 is NOT supported: " << cpp_strerror(err) << dendl;
+    }
+  }
+
+  // clean up test subvol
+  if (srcfd >= 0)
+    TEMP_FAILURE_RETRY(::close(srcfd));
+
+  strcpy(vol_args.name, "test_subvol");
+  r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_DESTROY, &vol_args);
+  if (r < 0) {
+    r = -errno;
+    dout(0) << "detect_feature: failed to remove " << vol_args.name << ": " << cpp_strerror(r) << dendl;
+  }
+
+  if (m_filestore_btrfs_snap && !has_snap_create_v2) {
+    dout(0) << "mount WARNING: btrfs snaps enabled, but no SNAP_CREATE_V2 ioctl (from kernel 2.6.37+)" << dendl;
+    cerr << TEXT_YELLOW
+      << " ** WARNING: 'filestore btrfs snap' is enabled (for safe transactions,\n"
+      << "             rollback), but btrfs does not support the SNAP_CREATE_V2 ioctl\n"
+      << "             (added in Linux 2.6.37).  Expect slow btrfs sync/commit\n"
+      << "             performance.\n"
+      << TEXT_NORMAL;
+  }
+
+  return 0;
+}
+
+bool BtrfsFileStoreBackend::can_checkpoint()
+{
+  return stable_commits;
+}
+
+int BtrfsFileStoreBackend::create_current()
+{
+  struct stat st;
+  int ret = ::stat(get_current_path().c_str(), &st);
+  if (ret == 0) {
+    // current/ exists
+    if (!S_ISDIR(st.st_mode)) {
+      dout(0) << "create_current: current/ exists but is not a directory" << dendl;
+      return -EINVAL;
+    }
+
+    struct stat basest;
+    struct statfs currentfs;
+    ret = ::fstat(get_basedir_fd(), &basest);
+    if (ret < 0) {
+      ret = -errno;
+      dout(0) << "create_current: cannot fstat basedir " << cpp_strerror(ret) << dendl;
+      return ret;
+    }
+    ret = ::statfs(get_current_path().c_str(), ¤tfs);
+    if (ret < 0) {
+      ret = -errno;
+      dout(0) << "create_current: cannot statsf basedir " << cpp_strerror(ret) << dendl;
+      return ret;
+    }
+    if (currentfs.f_type == BTRFS_SUPER_MAGIC && basest.st_dev != st.st_dev) {
+      dout(2) << "create_current: current appears to be a btrfs subvolume" << dendl;
+      stable_commits = true;
+    }
+    return 0;
+  }
+
+  struct btrfs_ioctl_vol_args volargs;
+  memset(&volargs, 0, sizeof(volargs));
+
+  volargs.fd = 0;
+  strcpy(volargs.name, "current");
+  if (::ioctl(get_basedir_fd(), BTRFS_IOC_SUBVOL_CREATE, (unsigned long int)&volargs) < 0) {
+    ret = -errno;
+    dout(0) << "create_current: BTRFS_IOC_SUBVOL_CREATE failed with error "
+	    << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  dout(2) << "create_current: created btrfs subvol " << get_current_path() << dendl;
+  if (::chmod(get_current_path().c_str(), 0755) < 0) {
+    ret = -errno;
+    dout(0) << "create_current: failed to chmod " << get_current_path() << " to 0755: "
+	    << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  stable_commits = true;
+  return 0;
+}
+
+int BtrfsFileStoreBackend::list_checkpoints(list& ls)
+{
+  int ret, err = 0;
+
+  struct stat basest;
+  ret = ::fstat(get_basedir_fd(), &basest);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "list_checkpoints: cannot fstat basedir " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  // get snap list
+  DIR *dir = ::opendir(get_basedir_path().c_str());
+  if (!dir) {
+    ret = -errno;
+    dout(0) << "list_checkpoints: opendir '" << get_basedir_path() << "' failed: "
+	    << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  list snaps;
+  char path[PATH_MAX];
+  char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
+  struct dirent *de;
+  while (::readdir_r(dir, (struct dirent *)&buf, &de) == 0) {
+    if (!de)
+      break;
+
+    snprintf(path, sizeof(path), "%s/%s", get_basedir_path().c_str(), de->d_name);
+
+    struct stat st;
+    ret = ::stat(path, &st);
+    if (ret < 0) {
+      err = -errno;
+      dout(0) << "list_checkpoints: stat '" << path << "' failed: "
+	      << cpp_strerror(err) << dendl;
+      break;
+    }
+
+    if (!S_ISDIR(st.st_mode))
+      continue;
+
+    struct statfs fs;
+    ret = ::statfs(path, &fs);
+    if (ret < 0) {
+      err = -errno;
+      dout(0) << "list_checkpoints: statfs '" << path << "' failed: "
+	      << cpp_strerror(err) << dendl;
+      break;
+    }
+
+    if (fs.f_type == BTRFS_SUPER_MAGIC && basest.st_dev != st.st_dev)
+      snaps.push_back(string(de->d_name));
+  }
+
+  if (::closedir(dir) < 0) {
+      ret = -errno;
+      dout(0) << "list_checkpoints: closedir failed: " << cpp_strerror(ret) << dendl;
+      if (!err)
+	err = ret;
+  }
+
+  if (err)
+    return err;
+
+  ls.swap(snaps);
+  return 0;
+}
+
+int BtrfsFileStoreBackend::create_checkpoint(const string& name, uint64_t *transid)
+{
+  dout(10) << "create_checkpoint: '" << name << "'" << dendl;
+  if (has_snap_create_v2 && transid) {
+    struct btrfs_ioctl_vol_args_v2 async_args;
+    memset(&async_args, 0, sizeof(async_args));
+    async_args.fd = get_current_fd();
+    async_args.flags = BTRFS_SUBVOL_CREATE_ASYNC;
+    strncpy(async_args.name, name.c_str(), sizeof(async_args.name));
+
+    int r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_CREATE_V2, &async_args);
+    if (r < 0) {
+      r = -errno;
+      dout(0) << "create_checkpoint: async snap create '" << name << "' got " << cpp_strerror(r) << dendl;
+      return r;
+    }
+    dout(20) << "create_checkpoint: async snap create '" << name << "' transid " << async_args.transid << dendl;
+    *transid = async_args.transid;
+  } else {
+    struct btrfs_ioctl_vol_args vol_args;
+    memset(&vol_args, 0, sizeof(vol_args));
+    vol_args.fd = get_current_fd();
+    strcpy(vol_args.name, name.c_str());
+
+    int r = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_CREATE, &vol_args);
+    if (r < 0) {
+      r = -errno;
+      dout(0) << "create_checkpoint: snap create '" << name << "' got " << cpp_strerror(r) << dendl;
+      return r;
+    }
+    if (transid)
+      *transid = 0;
+  }
+  return 0;
+}
+
+int BtrfsFileStoreBackend::sync_checkpoint(uint64_t transid)
+{
+  // wait for commit
+  dout(10) << "sync_checkpoint: transid " << transid << " to complete" << dendl;
+  int ret = ::ioctl(get_op_fd(), BTRFS_IOC_WAIT_SYNC, &transid);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "sync_checkpoint: ioctl WAIT_SYNC got " << cpp_strerror(ret) << dendl;
+    return -errno;
+  }
+  dout(20) << "sync_checkpoint: done waiting for transid " << transid << dendl;
+  return 0;
+}
+
+int BtrfsFileStoreBackend::rollback_to(const string& name)
+{
+  dout(10) << "rollback_to: to '" << name << "'" << dendl;
+  char s[PATH_MAX];
+  btrfs_ioctl_vol_args vol_args;
+
+  memset(&vol_args, 0, sizeof(vol_args));
+  vol_args.fd = 0;
+  strcpy(vol_args.name, "current");
+
+  int ret = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_DESTROY, &vol_args);
+  if (ret && errno != ENOENT) {
+    dout(0) << "rollback_to: error removing old current subvol: " << cpp_strerror(ret) << dendl;
+    snprintf(s, sizeof(s), "%s/current.remove.me.%d", get_basedir_path().c_str(), rand());
+    if (::rename(get_current_path().c_str(), s)) {
+      ret = -errno;
+      dout(0) << "rollback_to: error renaming old current subvol: "
+	      << cpp_strerror(ret) << dendl;
+      return ret;
+    }
+  }
+
+  snprintf(s, sizeof(s), "%s/%s", get_basedir_path().c_str(), name.c_str());
+
+  // roll back
+  vol_args.fd = ::open(s, O_RDONLY);
+  if (vol_args.fd < 0) {
+    ret = -errno;
+    dout(0) << "rollback_to: error opening '" << s << "': " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+  ret = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_CREATE, &vol_args);
+  if (ret < 0 ) {
+    ret = -errno;
+    dout(0) << "rollback_to: ioctl SNAP_CREATE got " << cpp_strerror(ret) << dendl;
+  }
+  TEMP_FAILURE_RETRY(::close(vol_args.fd));
+  return ret;
+}
+
+int BtrfsFileStoreBackend::destroy_checkpoint(const string& name)
+{
+  dout(10) << "destroy_checkpoint: '" << name << "'" << dendl;
+  btrfs_ioctl_vol_args vol_args;
+  memset(&vol_args, 0, sizeof(vol_args));
+  vol_args.fd = 0;
+  strncpy(vol_args.name, name.c_str(), sizeof(vol_args.name));
+
+  int ret = ::ioctl(get_basedir_fd(), BTRFS_IOC_SNAP_DESTROY, &vol_args);
+  if (ret) {
+    ret = -errno;
+    dout(0) << "destroy_checkpoint: ioctl SNAP_DESTROY got " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+  return 0;
+}
+
+int BtrfsFileStoreBackend::syncfs()
+{
+  dout(15) << "syncfs" << dendl;
+  // do a full btrfs commit
+  int ret = ::ioctl(get_op_fd(), BTRFS_IOC_SYNC);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "syncfs: btrfs IOC_SYNC got " << cpp_strerror(ret) << dendl;
+  }
+  return ret;
+}
+
+int BtrfsFileStoreBackend::clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff)
+{
+  dout(20) << "clone_range: " << srcoff << "~" << len << " to " << dstoff << dendl;
+  size_t blk_size = get_blksize();
+  if (!has_clone_range ||
+      srcoff % blk_size != dstoff % blk_size) {
+    dout(20) << "clone_range: using copy" << dendl;
+    return _copy_range(from, to, srcoff, len, dstoff);
+  }
+
+  int err = 0;
+  int r = 0;
+
+  uint64_t srcoffclone = ALIGN_UP(srcoff, blk_size);
+  uint64_t dstoffclone = ALIGN_UP(dstoff, blk_size);
+  if (srcoffclone >= srcoff + len) {
+    dout(20) << "clone_range: using copy, extent too short to align srcoff" << dendl;
+    return _copy_range(from, to, srcoff, len, dstoff);
+  }
+
+  uint64_t lenclone = len - (srcoffclone - srcoff);
+  if (!ALIGNED(lenclone, blk_size)) {
+    struct stat from_stat, to_stat;
+    err = ::fstat(from, &from_stat);
+    if (err) return -errno;
+    err = ::fstat(to , &to_stat);
+    if (err) return -errno;
+
+    if (srcoff + len != (uint64_t)from_stat.st_size ||
+	dstoff + len < (uint64_t)to_stat.st_size) {
+      // Not to the end of the file, need to align length as well
+      lenclone = ALIGN_DOWN(lenclone, blk_size);
+    }
+  }
+  if (lenclone == 0) {
+    // too short
+    return _copy_range(from, to, srcoff, len, dstoff);
+  }
+
+  dout(20) << "clone_range: cloning " << srcoffclone << "~" << lenclone
+	   << " to " << dstoffclone << " = " << r << dendl;
+  btrfs_ioctl_clone_range_args a;
+  a.src_fd = from;
+  a.src_offset = srcoffclone;
+  a.src_length = lenclone;
+  a.dest_offset = dstoffclone;
+  err = ::ioctl(to, BTRFS_IOC_CLONE_RANGE, &a);
+  if (err >= 0) {
+    r += err;
+  } else if (errno == EINVAL) {
+    // Still failed, might be compressed
+    dout(20) << "clone_range: failed CLONE_RANGE call with -EINVAL, using copy" << dendl;
+    return _copy_range(from, to, srcoff, len, dstoff);
+  } else {
+    return -errno;
+  }
+
+  // Take care any trimmed from front
+  if (srcoffclone != srcoff) {
+    err = _copy_range(from, to, srcoff, srcoffclone - srcoff, dstoff);
+    if (err >= 0) {
+      r += err;
+    } else {
+      return err;
+    }
+  }
+
+  // Copy end
+  if (srcoffclone + lenclone != srcoff + len) {
+    err = _copy_range(from, to,
+			 srcoffclone + lenclone,
+			 (srcoff + len) - (srcoffclone + lenclone),
+			 dstoffclone + lenclone);
+    if (err >= 0) {
+      r += err;
+    } else {
+      return err;
+    }
+  }
+  dout(20) << "clone_range: finished " << srcoff << "~" << len
+	   << " to " << dstoff << " = " << r << dendl;
+  return r;
+}
+#endif
diff --git a/ceph/src/os/BtrfsFileStoreBackend.h b/ceph/src/os/BtrfsFileStoreBackend.h
new file mode 100644
index 00000000..f63a2a8b
--- /dev/null
+++ b/ceph/src/os/BtrfsFileStoreBackend.h
@@ -0,0 +1,46 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_BTRFSFILESTOREBACKEDN_H
+#define CEPH_BTRFSFILESTOREBACKEDN_H
+
+#if defined(__linux__)
+#include "GenericFileStoreBackend.h"
+
+class BtrfsFileStoreBackend : public GenericFileStoreBackend {
+private:
+  bool has_clone_range;       ///< clone range ioctl is supported
+  bool has_snap_create;       ///< snap create ioctl is supported
+  bool has_snap_destroy;      ///< snap destroy ioctl is supported
+  bool has_snap_create_v2;    ///< snap create v2 ioctl (async!) is supported
+  bool has_wait_sync;         ///< wait sync ioctl is supported
+  bool stable_commits;
+  bool m_filestore_btrfs_clone_range;
+  bool m_filestore_btrfs_snap;
+public:
+  BtrfsFileStoreBackend(FileStore *fs);
+  ~BtrfsFileStoreBackend() {};
+  int detect_features();
+  bool can_checkpoint();
+  int create_current();
+  int list_checkpoints(list& ls);
+  int create_checkpoint(const string& name, uint64_t *cid);
+  int sync_checkpoint(uint64_t cid);
+  int rollback_to(const string& name);
+  int destroy_checkpoint(const string& name);
+  int syncfs();
+  int clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff);
+};
+#endif
+#endif
diff --git a/ceph/src/os/CollectionIndex.h b/ceph/src/os/CollectionIndex.h
new file mode 100644
index 00000000..529db015
--- /dev/null
+++ b/ceph/src/os/CollectionIndex.h
@@ -0,0 +1,190 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef OS_COLLECTIONINDEX_H
+#define OS_COLLECTIONINDEX_H
+
+#include 
+#include 
+#include "include/memory.h"
+
+#include "osd/osd_types.h"
+#include "include/object.h"
+
+/**
+ * CollectionIndex provides an interface for manipulating indexed collections
+ */
+class CollectionIndex {
+protected:
+  /** 
+   * Object encapsulating a returned path.
+   *
+   * A path to an object (existent or non-existent) becomes invalid
+   * when a different object is created in the index.  Path stores
+   * a shared_ptr to the CollectionIndex to keep the index alive
+   * during its lifetime.
+   * @see IndexManager
+   * @see self_ref
+   * @see set_ref
+   */
+  class Path {
+  public:
+    /// Returned path
+    string full_path;
+    /// Ref to parent Index
+    ceph::shared_ptr parent_ref;
+    /// coll_t for parent Index
+    coll_t parent_coll;
+
+    /// Normal Constructor
+    Path(
+      string path,                              ///< [in] Path to return.
+      ceph::weak_ptr ref)  ///< [in] weak_ptr to parent.
+      : full_path(path), parent_ref(ref), parent_coll(parent_ref->coll()) {}
+
+    /// Debugging Constructor
+    Path(
+      string path,                              ///< [in] Path to return.
+      coll_t coll)                              ///< [in] collection
+      : full_path(path), parent_coll(coll) {}
+      
+    /// Getter for the stored path.
+    const char *path() const { return full_path.c_str(); }
+
+    /// Getter for collection
+    coll_t coll() const { return parent_coll; }
+
+    /// Getter for parent
+    ceph::shared_ptr get_index() const {
+      return parent_ref;
+    }
+  };
+ public:
+  /// Type of returned paths
+  typedef ceph::shared_ptr IndexedPath;
+
+  static IndexedPath get_testing_path(string path, coll_t collection) {
+    return IndexedPath(new Path(path, collection));
+  }
+
+  static const uint32_t FLAT_INDEX_TAG = 0;
+  static const uint32_t HASH_INDEX_TAG = 1;
+  static const uint32_t HASH_INDEX_TAG_2 = 2;
+  static const uint32_t HOBJECT_WITH_POOL = 3;
+  /**
+   * For tracking Filestore collection versions.
+   *
+   * @return Collection version represented by the Index implementation
+   */
+  virtual uint32_t collection_version() = 0;
+
+  /**
+   * Returns the collection managed by this CollectionIndex
+   */
+  virtual coll_t coll() const = 0;
+
+  /** 
+   * For setting the internal weak_ptr to a shared_ptr to this.
+   *
+   * @see IndexManager
+   */
+  virtual void set_ref(ceph::shared_ptr ref) = 0;
+
+  /** 
+   * Initializes the index.
+   *
+   * @return Error Code, 0 for success
+   */
+  virtual int init() = 0;
+
+  /**
+   * Cleanup before replaying journal
+   *
+   * Index implemenations may need to perform compound operations
+   * which may leave the collection unstable if interupted.  cleanup
+   * is called on mount to allow the CollectionIndex implementation
+   * to stabilize.
+   *
+   * @see HashIndex
+   * @return Error Code, 0 for success
+   */
+  virtual int cleanup() = 0;
+
+  /**
+   * Call when a file is created using a path returned from lookup.
+   *
+   * @return Error Code, 0 for success
+   */
+  virtual int created(
+    const ghobject_t &oid, ///< [in] Created object.
+    const char *path       ///< [in] Path to created object.
+    ) = 0;
+
+  /**
+   * Removes oid from the collection
+   *
+   * @return Error Code, 0 for success
+   */
+  virtual int unlink(
+    const ghobject_t &oid ///< [in] Object to remove
+    ) = 0;
+
+  /**
+   * Gets the IndexedPath for oid.
+   *
+   * @return Error Code, 0 for success
+   */
+  virtual int lookup(
+    const ghobject_t &oid, ///< [in] Object to lookup
+    IndexedPath *path,	   ///< [out] Path to object
+    int *exist	           ///< [out] True if the object exists, else false
+    ) = 0;
+
+  /**
+   * Moves objects matching  in the lsb 
+   *
+   * dest and this must be the same subclass
+   *
+   * @return Error Code, 0 for success
+   */
+  virtual int split(
+    uint32_t match,                             //< [in] value to match
+    uint32_t bits,                              //< [in] bits to check
+    ceph::shared_ptr dest  //< [in] destination index
+    ) { assert(0); return 0; }
+
+
+  /// List contents of collection by hash
+  virtual int collection_list_partial(
+    const ghobject_t &start, ///< [in] object at which to start
+    int min_count,          ///< [in] get at least min_count objects
+    int max_count,          ///< [in] return at most max_count objects
+    snapid_t seq,           ///< [in] list only objects with snap >= seq
+    vector *ls,  ///< [out] Listed objects
+    ghobject_t *next         ///< [out] Next object to list
+    ) = 0;
+
+  /// List contents of collection.
+  virtual int collection_list(
+    vector *ls ///< [out] Listed Objects
+    ) = 0;
+
+  /// Call prior to removing directory
+  virtual int prep_delete() { return 0; }
+
+  /// Virtual destructor
+  virtual ~CollectionIndex() {}
+};
+
+#endif
diff --git a/ceph/src/os/DBObjectMap.cc b/ceph/src/os/DBObjectMap.cc
new file mode 100644
index 00000000..5d2a2e61
--- /dev/null
+++ b/ceph/src/os/DBObjectMap.cc
@@ -0,0 +1,1268 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "include/int_types.h"
+#include "include/buffer.h"
+
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+#include 
+
+#include "ObjectMap.h"
+#include "KeyValueDB.h"
+#include "DBObjectMap.h"
+#include 
+
+#include "common/debug.h"
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "filestore "
+
+const string DBObjectMap::USER_PREFIX = "_USER_";
+const string DBObjectMap::XATTR_PREFIX = "_AXATTR_";
+const string DBObjectMap::SYS_PREFIX = "_SYS_";
+const string DBObjectMap::COMPLETE_PREFIX = "_COMPLETE_";
+const string DBObjectMap::HEADER_KEY = "HEADER";
+const string DBObjectMap::USER_HEADER_KEY = "USER_HEADER";
+const string DBObjectMap::GLOBAL_STATE_KEY = "HEADER";
+const string DBObjectMap::HOBJECT_TO_SEQ = "_HOBJTOSEQ_";
+
+// Legacy
+const string DBObjectMap::LEAF_PREFIX = "_LEAF_";
+const string DBObjectMap::REVERSE_LEAF_PREFIX = "_REVLEAF_";
+
+static void append_escaped(const string &in, string *out)
+{
+  for (string::const_iterator i = in.begin(); i != in.end(); ++i) {
+    if (*i == '%') {
+      out->push_back('%');
+      out->push_back('p');
+    } else if (*i == '.') {
+      out->push_back('%');
+      out->push_back('e');
+    } else if (*i == '_') {
+      out->push_back('%');
+      out->push_back('u');
+    } else {
+      out->push_back(*i);
+    }
+  }
+}
+
+static bool append_unescaped(string::const_iterator begin,
+			     string::const_iterator end, 
+			     string *out) {
+  for (string::const_iterator i = begin; i != end; ++i) {
+    if (*i == '%') {
+      ++i;
+      if (*i == 'p')
+	out->push_back('%');
+      else if (*i == 'e')
+	out->push_back('.');
+      else if (*i == 'u')
+	out->push_back('_');
+      else
+	return false;
+    } else {
+      out->push_back(*i);
+    }
+  }
+  return true;
+}
+
+bool DBObjectMap::check(std::ostream &out)
+{
+  bool retval = true;
+  map parent_to_num_children;
+  map parent_to_actual_num_children;
+  KeyValueDB::Iterator iter = db->get_iterator(HOBJECT_TO_SEQ);
+  for (iter->seek_to_first(); iter->valid(); iter->next()) {
+    _Header header;
+    assert(header.num_children == 1);
+    header.num_children = 0; // Hack for leaf node
+    bufferlist bl = iter->value();
+    while (true) {
+      bufferlist::iterator bliter = bl.begin();
+      header.decode(bliter);
+      if (header.seq != 0)
+	parent_to_actual_num_children[header.seq] = header.num_children;
+      if (header.parent == 0)
+	break;
+
+      if (!parent_to_num_children.count(header.parent))
+	parent_to_num_children[header.parent] = 0;
+      parent_to_num_children[header.parent]++;
+      if (parent_to_actual_num_children.count(header.parent))
+	break;
+
+      set to_get;
+      map got;
+      to_get.insert(HEADER_KEY);
+      db->get(sys_parent_prefix(header), to_get, &got);
+      if (got.empty()) {
+	out << "Missing: seq " << header.parent << std::endl;
+	retval = false;
+	break;
+      } else {
+	bl = got.begin()->second;
+      }
+    }
+  }
+
+  for (map::iterator i = parent_to_num_children.begin();
+       i != parent_to_num_children.end();
+       parent_to_num_children.erase(i++)) {
+    if (!parent_to_actual_num_children.count(i->first))
+      continue;
+    if (parent_to_actual_num_children[i->first] != i->second) {
+      out << "Invalid: seq " << i->first << " recorded children: "
+	  << parent_to_actual_num_children[i->first] << " found: "
+	  << i->second << std::endl;
+      retval = false;
+    }
+    parent_to_actual_num_children.erase(i->first);
+  }
+  return retval;
+}
+
+string DBObjectMap::ghobject_key(const ghobject_t &oid)
+{
+  string out;
+  append_escaped(oid.hobj.oid.name, &out);
+  out.push_back('.');
+  append_escaped(oid.hobj.get_key(), &out);
+  out.push_back('.');
+  append_escaped(oid.hobj.nspace, &out);
+  out.push_back('.');
+
+  char snap_with_hash[1000];
+  char *t = snap_with_hash;
+  char *end = t + sizeof(snap_with_hash);
+  if (oid.hobj.snap == CEPH_NOSNAP)
+    t += snprintf(t, end - t, "head");
+  else if (oid.hobj.snap == CEPH_SNAPDIR)
+    t += snprintf(t, end - t, "snapdir");
+  else
+    t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.snap);
+
+  if (oid.hobj.pool == -1)
+    t += snprintf(t, end - t, ".none");
+  else
+    t += snprintf(t, end - t, ".%llx", (long long unsigned)oid.hobj.pool);
+  snprintf(t, end - t, ".%.*X", (int)(sizeof(oid.hobj.hash)*2), oid.hobj.hash);
+
+  if (oid.generation != ghobject_t::NO_GEN ||
+      oid.shard_id != ghobject_t::NO_SHARD) {
+    t += snprintf(t, end - t, ".%llx", (long long unsigned)oid.generation);
+    t += snprintf(t, end - t, ".%x", (int)oid.shard_id);
+  }
+  out += string(snap_with_hash);
+  return out;
+}
+
+string DBObjectMap::ghobject_key_v0(coll_t c, const ghobject_t &oid)
+{
+  string out;
+  append_escaped(c.to_str(), &out);
+  out.push_back('.');
+  append_escaped(oid.hobj.oid.name, &out);
+  out.push_back('.');
+  append_escaped(oid.hobj.get_key(), &out);
+  out.push_back('.');
+
+  char snap_with_hash[1000];
+  char *t = snap_with_hash;
+  char *end = t + sizeof(snap_with_hash);
+  if (oid.hobj.snap == CEPH_NOSNAP)
+    t += snprintf(t, end - t, ".head");
+  else if (oid.hobj.snap == CEPH_SNAPDIR)
+    t += snprintf(t, end - t, ".snapdir");
+  else
+    t += snprintf(t, end - t, ".%llx", (long long unsigned)oid.hobj.snap);
+  snprintf(t, end - t, ".%.*X", (int)(sizeof(oid.hobj.hash)*2), oid.hobj.hash);
+  out += string(snap_with_hash);
+  return out;
+}
+
+bool DBObjectMap::parse_ghobject_key_v0(const string &in, coll_t *c,
+				       ghobject_t *oid)
+{
+  string coll;
+  string name;
+  string key;
+  snapid_t snap;
+  uint32_t hash;
+
+  string::const_iterator current = in.begin();
+  string::const_iterator end;
+  for (end = current; end != in.end() && *end != '.'; ++end) ;
+  if (end == in.end())
+    return false;
+  if (!append_unescaped(current, end, &coll))
+    return false;
+
+  current = ++end;
+  for (; end != in.end() && *end != '.'; ++end) ;
+  if (end == in.end())
+    return false;
+  if (!append_unescaped(current, end, &name))
+    return false;
+
+  current = ++end;
+  for (; end != in.end() && *end != '.'; ++end) ;
+  if (end == in.end())
+    return false;
+  if (!append_unescaped(current, end, &key))
+    return false;
+
+  current = ++end;
+  // Bug in old encoding, double .
+  if (*current != '.')
+    return false;
+
+  current = ++end;
+  for (; end != in.end() && *end != '.'; ++end) ;
+  if (end == in.end())
+    return false;
+  string snap_str(current, end);
+  
+  current = ++end;
+  for (; end != in.end() && *end != '.'; ++end) ;
+  if (end != in.end())
+    return false;
+  string hash_str(current, end);
+
+  if (snap_str == "head")
+    snap = CEPH_NOSNAP;
+  else if (snap_str == "snapdir")
+    snap = CEPH_SNAPDIR;
+  else
+    snap = strtoull(snap_str.c_str(), NULL, 16);
+  sscanf(hash_str.c_str(), "%X", &hash);
+
+  *c = coll_t(coll);
+  int64_t pool = -1;
+  spg_t pg;
+  if (c->is_pg_prefix(pg))
+    pool = (int64_t)pg.pgid.pool();
+  (*oid) = ghobject_t(hobject_t(name, key, snap, hash, pool, ""));
+  return true;
+}
+
+string DBObjectMap::map_header_key(const ghobject_t &oid)
+{
+  return ghobject_key(oid);
+}
+
+string DBObjectMap::header_key(uint64_t seq)
+{
+  char buf[100];
+  snprintf(buf, sizeof(buf), "%.*" PRId64, (int)(2*sizeof(seq)), seq);
+  return string(buf);
+}
+
+string DBObjectMap::complete_prefix(Header header)
+{
+  return USER_PREFIX + header_key(header->seq) + COMPLETE_PREFIX;
+}
+
+string DBObjectMap::user_prefix(Header header)
+{
+  return USER_PREFIX + header_key(header->seq) + USER_PREFIX;
+}
+
+string DBObjectMap::sys_prefix(Header header)
+{
+  return USER_PREFIX + header_key(header->seq) + SYS_PREFIX;
+}
+
+string DBObjectMap::xattr_prefix(Header header)
+{
+  return USER_PREFIX + header_key(header->seq) + XATTR_PREFIX;
+}
+
+string DBObjectMap::sys_parent_prefix(_Header header)
+{
+  return USER_PREFIX + header_key(header.parent) + SYS_PREFIX;
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::init()
+{
+  invalid = false;
+  if (ready) {
+    return 0;
+  }
+  assert(!parent_iter);
+  if (header->parent) {
+    Header parent = map->lookup_parent(header);
+    if (!parent) {
+      assert(0);
+      return -EINVAL;
+    }
+    parent_iter.reset(new DBObjectMapIteratorImpl(map, parent));
+  }
+  key_iter = map->db->get_iterator(map->user_prefix(header));
+  assert(key_iter);
+  complete_iter = map->db->get_iterator(map->complete_prefix(header));
+  assert(complete_iter);
+  cur_iter = key_iter;
+  assert(cur_iter);
+  ready = true;
+  return 0;
+}
+
+ObjectMap::ObjectMapIterator DBObjectMap::get_iterator(
+  const ghobject_t &oid)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return ObjectMapIterator(new EmptyIteratorImpl());
+  return _get_iterator(header);
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::seek_to_first()
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->seek_to_first();
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->seek_to_first();
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::seek_to_last()
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->seek_to_last();
+    if (r < 0)
+      return r;
+    if (parent_iter->valid())
+      r = parent_iter->next();
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->seek_to_last();
+  if (r < 0)
+    return r;
+  if (key_iter->valid())
+    r = key_iter->next();
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::lower_bound(const string &to)
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->lower_bound(to);
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->lower_bound(to);
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::upper_bound(const string &after)
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->upper_bound(after);
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->upper_bound(after);
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+bool DBObjectMap::DBObjectMapIteratorImpl::valid()
+{
+  bool valid = !invalid && ready;
+  assert(!valid || cur_iter->valid());
+  return valid;
+}
+
+bool DBObjectMap::DBObjectMapIteratorImpl::valid_parent()
+{
+  if (parent_iter && parent_iter->valid() &&
+      (!key_iter->valid() || key_iter->key() > parent_iter->key()))
+    return true;
+  return false;
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::next()
+{
+  assert(cur_iter->valid());
+  assert(valid());
+  cur_iter->next();
+  return adjust();
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::next_parent()
+{
+  if (!parent_iter || !parent_iter->valid()) {
+    invalid = true;
+    return 0;
+  }
+  r = next();
+  if (r < 0)
+    return r;
+  if (!valid() || on_parent() || !parent_iter->valid())
+    return 0;
+
+  return lower_bound(parent_iter->key());
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::in_complete_region(const string &to_test,
+							     string *begin,
+							     string *end)
+{
+  complete_iter->upper_bound(to_test);
+  if (complete_iter->valid())
+    complete_iter->prev();
+  else
+    complete_iter->seek_to_last();
+
+  if (!complete_iter->valid())
+    return false;
+
+  string _end;
+  if (begin)
+    *begin = complete_iter->key();
+  _end = string(complete_iter->value().c_str());
+  if (end)
+    *end = _end;
+  return (to_test >= complete_iter->key()) && (!_end.size() || _end > to_test);
+}
+
+/**
+ * Moves parent_iter to the next position both out of the complete_region and 
+ * not equal to key_iter.  Then, we set cur_iter to parent_iter if valid and
+ * less than key_iter and key_iter otherwise.
+ */
+int DBObjectMap::DBObjectMapIteratorImpl::adjust()
+{
+  string begin, end;
+  while (parent_iter && parent_iter->valid()) {
+    if (in_complete_region(parent_iter->key(), &begin, &end)) {
+      if (end.size() == 0) {
+	parent_iter->seek_to_last();
+	if (parent_iter->valid())
+	  parent_iter->next();
+      } else
+	parent_iter->lower_bound(end);
+    } else if (key_iter->valid() && key_iter->key() == parent_iter->key()) {
+      parent_iter->next();
+    } else {
+      break;
+    }
+  }
+  if (valid_parent()) {
+    cur_iter = parent_iter;
+  } else if (key_iter->valid()) {
+    cur_iter = key_iter;
+  } else {
+    invalid = true;
+  }
+  assert(invalid || cur_iter->valid());
+  return 0;
+}
+
+
+string DBObjectMap::DBObjectMapIteratorImpl::key()
+{
+  return cur_iter->key();
+}
+
+bufferlist DBObjectMap::DBObjectMapIteratorImpl::value()
+{
+  return cur_iter->value();
+}
+
+int DBObjectMap::DBObjectMapIteratorImpl::status()
+{
+  return r;
+}
+
+int DBObjectMap::set_keys(const ghobject_t &oid,
+			  const map &set,
+			  const SequencerPosition *spos)
+{
+  KeyValueDB::Transaction t = db->get_transaction();
+  Header header = lookup_create_map_header(oid, t);
+  if (!header)
+    return -EINVAL;
+  if (check_spos(oid, header, spos))
+    return 0;
+
+  t->set(user_prefix(header), set);
+
+  return db->submit_transaction(t);
+}
+
+int DBObjectMap::set_header(const ghobject_t &oid,
+			    const bufferlist &bl,
+			    const SequencerPosition *spos)
+{
+  KeyValueDB::Transaction t = db->get_transaction();
+  Header header = lookup_create_map_header(oid, t);
+  if (!header)
+    return -EINVAL;
+  if (check_spos(oid, header, spos))
+    return 0;
+  _set_header(header, bl, t);
+  return db->submit_transaction(t);
+}
+
+void DBObjectMap::_set_header(Header header, const bufferlist &bl,
+			      KeyValueDB::Transaction t)
+{
+  map to_set;
+  to_set[USER_HEADER_KEY] = bl;
+  t->set(sys_prefix(header), to_set);
+}
+
+int DBObjectMap::get_header(const ghobject_t &oid,
+			    bufferlist *bl)
+{
+  Header header = lookup_map_header(oid);
+  if (!header) {
+    return 0;
+  }
+  return _get_header(header, bl);
+}
+
+int DBObjectMap::_get_header(Header header,
+			     bufferlist *bl)
+{
+  map out;
+  while (true) {
+    out.clear();
+    set to_get;
+    to_get.insert(USER_HEADER_KEY);
+    int r = db->get(sys_prefix(header), to_get, &out);
+    if (r == 0 && !out.empty())
+      break;
+    if (r < 0)
+      return r;
+    Header current(header);
+    if (!current->parent)
+      break;
+    header = lookup_parent(current);
+  }
+
+  if (!out.empty())
+    bl->swap(out.begin()->second);
+  return 0;
+}
+
+int DBObjectMap::clear(const ghobject_t &oid,
+		       const SequencerPosition *spos)
+{
+  KeyValueDB::Transaction t = db->get_transaction();
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  if (check_spos(oid, header, spos))
+    return 0;
+  remove_map_header(oid, header, t);
+  assert(header->num_children > 0);
+  header->num_children--;
+  int r = _clear(header, t);
+  if (r < 0)
+    return r;
+  return db->submit_transaction(t);
+}
+
+int DBObjectMap::_clear(Header header,
+			KeyValueDB::Transaction t)
+{
+  while (1) {
+    if (header->num_children) {
+      set_header(header, t);
+      break;
+    }
+    clear_header(header, t);
+    if (!header->parent)
+      break;
+    Header parent = lookup_parent(header);
+    if (!parent) {
+      return -EINVAL;
+    }
+    assert(parent->num_children > 0);
+    parent->num_children--;
+    header.swap(parent);
+  }
+  return 0;
+}
+
+int DBObjectMap::merge_new_complete(Header header,
+				    const map &new_complete,
+				    DBObjectMapIterator iter,
+				    KeyValueDB::Transaction t)
+{
+  KeyValueDB::Iterator complete_iter = db->get_iterator(
+    complete_prefix(header)
+    );
+  map::const_iterator i = new_complete.begin();
+  set to_remove;
+  map to_add;
+
+  string begin, end;
+  while (i != new_complete.end()) {
+    string new_begin = i->first;
+    string new_end = i->second;
+    int r = iter->in_complete_region(new_begin, &begin, &end);
+    if (r < 0)
+      return r;
+    if (r) {
+      to_remove.insert(begin);
+      new_begin = begin;
+    }
+    ++i;
+    while (i != new_complete.end()) {
+      if (!new_end.size() || i->first <= new_end) {
+	if (!new_end.size() && i->second > new_end) {
+	  new_end = i->second;
+	}
+	++i;
+	continue;
+      }
+
+      r = iter->in_complete_region(new_end, &begin, &end);
+      if (r < 0)
+	return r;
+      if (r) {
+	to_remove.insert(begin);
+	new_end = end;
+	continue;
+      }
+      break;
+    }
+    bufferlist bl;
+    bl.append(bufferptr(new_end.c_str(), new_end.size() + 1));
+    to_add.insert(make_pair(new_begin, bl));
+  }
+  t->rmkeys(complete_prefix(header), to_remove);
+  t->set(complete_prefix(header), to_add);
+  return 0;
+}
+
+int DBObjectMap::copy_up_header(Header header,
+				KeyValueDB::Transaction t)
+{
+  bufferlist bl;
+  int r = _get_header(header, &bl);
+  if (r < 0)
+    return r;
+
+  _set_header(header, bl, t);
+  return 0;
+}
+
+int DBObjectMap::need_parent(DBObjectMapIterator iter)
+{
+  int r = iter->seek_to_first();
+  if (r < 0)
+    return r;
+
+  if (!iter->valid())
+    return 0;
+
+  string begin, end;
+  if (iter->in_complete_region(iter->key(), &begin, &end) && end == "") {
+    return 0;
+  }
+  return 1;
+}
+
+int DBObjectMap::rm_keys(const ghobject_t &oid,
+			 const set &to_clear,
+			 const SequencerPosition *spos)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  KeyValueDB::Transaction t = db->get_transaction();
+  if (check_spos(oid, header, spos))
+    return 0;
+  t->rmkeys(user_prefix(header), to_clear);
+  if (!header->parent) {
+    return db->submit_transaction(t);
+  }
+
+  // Copy up keys from parent around to_clear
+  int keep_parent;
+  {
+    DBObjectMapIterator iter = _get_iterator(header);
+    iter->seek_to_first();
+    map new_complete;
+    map to_write;
+    for(set::const_iterator i = to_clear.begin();
+	i != to_clear.end();
+      ) {
+      unsigned copied = 0;
+      iter->lower_bound(*i);
+      ++i;
+      if (!iter->valid())
+	break;
+      string begin = iter->key();
+      if (!iter->on_parent())
+	iter->next_parent();
+      if (new_complete.size() && new_complete.rbegin()->second == begin) {
+	begin = new_complete.rbegin()->first;
+      }
+      while (iter->valid() && copied < 20) {
+	if (!to_clear.count(iter->key()))
+	  to_write[iter->key()].append(iter->value());
+	if (i != to_clear.end() && *i <= iter->key()) {
+	  ++i;
+	  copied = 0;
+	}
+
+	iter->next_parent();
+	copied++;
+      }
+      if (iter->valid()) {
+	new_complete[begin] = iter->key();
+      } else {
+	new_complete[begin] = "";
+	break;
+      }
+    }
+    t->set(user_prefix(header), to_write);
+    merge_new_complete(header, new_complete, iter, t);
+    keep_parent = need_parent(iter);
+    if (keep_parent < 0)
+      return keep_parent;
+  }
+  if (!keep_parent) {
+    copy_up_header(header, t);
+    Header parent = lookup_parent(header);
+    if (!parent)
+      return -EINVAL;
+    parent->num_children--;
+    _clear(parent, t);
+    header->parent = 0;
+    set_map_header(oid, *header, t);
+    t->rmkeys_by_prefix(complete_prefix(header));
+  }
+  return db->submit_transaction(t);
+}
+
+int DBObjectMap::clear_keys_header(const ghobject_t &oid,
+				   const SequencerPosition *spos)
+{
+  KeyValueDB::Transaction t = db->get_transaction();
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  if (check_spos(oid, header, spos))
+    return 0;
+
+  // save old attrs
+  KeyValueDB::Iterator iter = db->get_iterator(xattr_prefix(header));
+  if (!iter)
+    return -EINVAL;
+  map attrs;
+  for (iter->seek_to_first(); !iter->status() && iter->valid(); iter->next())
+    attrs.insert(make_pair(iter->key(), iter->value()));
+  if (iter->status())
+    return iter->status();
+
+  // remove current header
+  remove_map_header(oid, header, t);
+  assert(header->num_children > 0);
+  header->num_children--;
+  int r = _clear(header, t);
+  if (r < 0)
+    return r;
+
+  // create new header
+  Header newheader = generate_new_header(oid, Header());
+  set_map_header(oid, *newheader, t);
+  if (!attrs.empty())
+    t->set(xattr_prefix(newheader), attrs);
+  return db->submit_transaction(t);
+}
+
+int DBObjectMap::get(const ghobject_t &oid,
+		     bufferlist *_header,
+		     map *out)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  _get_header(header, _header);
+  ObjectMapIterator iter = _get_iterator(header);
+  for (iter->seek_to_first(); iter->valid(); iter->next()) {
+    if (iter->status())
+      return iter->status();
+    out->insert(make_pair(iter->key(), iter->value()));
+  }
+  return 0;
+}
+
+int DBObjectMap::get_keys(const ghobject_t &oid,
+			  set *keys)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  ObjectMapIterator iter = get_iterator(oid);
+  for (; iter->valid(); iter->next()) {
+    if (iter->status())
+      return iter->status();
+    keys->insert(iter->key());
+  }
+  return 0;
+}
+
+int DBObjectMap::scan(Header header,
+		      const set &in_keys,
+		      set *out_keys,
+		      map *out_values)
+{
+  ObjectMapIterator db_iter = _get_iterator(header);
+  for (set::const_iterator key_iter = in_keys.begin();
+       key_iter != in_keys.end();
+       ++key_iter) {
+    db_iter->lower_bound(*key_iter);
+    if (db_iter->status())
+      return db_iter->status();
+    if (db_iter->valid() && db_iter->key() == *key_iter) {
+      if (out_keys)
+	out_keys->insert(*key_iter);
+      if (out_values)
+	out_values->insert(make_pair(db_iter->key(), db_iter->value()));
+    }
+  }
+  return 0;
+}
+
+int DBObjectMap::get_values(const ghobject_t &oid,
+			    const set &keys,
+			    map *out)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  return scan(header, keys, 0, out);
+}
+
+int DBObjectMap::check_keys(const ghobject_t &oid,
+			    const set &keys,
+			    set *out)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  return scan(header, keys, out, 0);
+}
+
+int DBObjectMap::get_xattrs(const ghobject_t &oid,
+			    const set &to_get,
+			    map *out)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  return db->get(xattr_prefix(header), to_get, out);
+}
+
+int DBObjectMap::get_all_xattrs(const ghobject_t &oid,
+				set *out)
+{
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  KeyValueDB::Iterator iter = db->get_iterator(xattr_prefix(header));
+  if (!iter)
+    return -EINVAL;
+  for (iter->seek_to_first(); !iter->status() && iter->valid(); iter->next())
+    out->insert(iter->key());
+  return iter->status();
+}
+
+int DBObjectMap::set_xattrs(const ghobject_t &oid,
+			    const map &to_set,
+			    const SequencerPosition *spos)
+{
+  KeyValueDB::Transaction t = db->get_transaction();
+  Header header = lookup_create_map_header(oid, t);
+  if (!header)
+    return -EINVAL;
+  if (check_spos(oid, header, spos))
+    return 0;
+  t->set(xattr_prefix(header), to_set);
+  return db->submit_transaction(t);
+}
+
+int DBObjectMap::remove_xattrs(const ghobject_t &oid,
+			       const set &to_remove,
+			       const SequencerPosition *spos)
+{
+  KeyValueDB::Transaction t = db->get_transaction();
+  Header header = lookup_map_header(oid);
+  if (!header)
+    return -ENOENT;
+  if (check_spos(oid, header, spos))
+    return 0;
+  t->rmkeys(xattr_prefix(header), to_remove);
+  return db->submit_transaction(t);
+}
+
+int DBObjectMap::clone(const ghobject_t &oid,
+		       const ghobject_t &target,
+		       const SequencerPosition *spos)
+{
+  if (oid == target)
+    return 0;
+
+  KeyValueDB::Transaction t = db->get_transaction();
+  {
+    Header destination = lookup_map_header(target);
+    if (destination) {
+      remove_map_header(target, destination, t);
+      if (check_spos(target, destination, spos))
+	return 0;
+      destination->num_children--;
+      _clear(destination, t);
+    }
+  }
+
+  Header parent = lookup_map_header(oid);
+  if (!parent)
+    return db->submit_transaction(t);
+
+  Header source = generate_new_header(oid, parent);
+  Header destination = generate_new_header(target, parent);
+  if (spos)
+    destination->spos = *spos;
+
+  parent->num_children = 2;
+  set_header(parent, t);
+  set_map_header(oid, *source, t);
+  set_map_header(target, *destination, t);
+
+  map to_set;
+  KeyValueDB::Iterator xattr_iter = db->get_iterator(xattr_prefix(parent));
+  for (xattr_iter->seek_to_first();
+       xattr_iter->valid();
+       xattr_iter->next())
+    to_set.insert(make_pair(xattr_iter->key(), xattr_iter->value()));
+  t->set(xattr_prefix(source), to_set);
+  t->set(xattr_prefix(destination), to_set);
+  t->rmkeys_by_prefix(xattr_prefix(parent));
+  return db->submit_transaction(t);
+}
+
+int DBObjectMap::upgrade()
+{
+  while (1) {
+    unsigned count = 0;
+    KeyValueDB::Iterator iter = db->get_iterator(LEAF_PREFIX);
+    iter->seek_to_first();
+    if (!iter->valid())
+      break;
+    KeyValueDB::Transaction t = db->get_transaction();
+    set legacy_to_remove;
+    set moved_seqs;
+    map new_map_headers;
+    for (;
+	 iter->valid() && count < 300;
+	 iter->next(), ++count) {
+      bufferlist bl = iter->value();
+      _Header hdr;
+      bufferlist::iterator bliter = bl.begin();
+      hdr.decode(bliter);
+
+      legacy_to_remove.insert(iter->key());
+      if (moved_seqs.count(hdr.parent))
+	continue; // Moved already in this transaction
+      moved_seqs.insert(hdr.parent);
+
+      set to_get;
+      to_get.insert(HEADER_KEY);
+      map got;
+      int r = db->get(USER_PREFIX + header_key(hdr.parent) + SYS_PREFIX,
+		      to_get,
+		      &got);
+      if (r < 0)
+	return r;
+      if (got.empty())
+	continue; // Moved in a previous transaction
+
+      t->rmkeys(USER_PREFIX + header_key(hdr.parent) + SYS_PREFIX,
+		 to_get);
+
+      coll_t coll;
+      ghobject_t oid;
+      assert(parse_ghobject_key_v0(iter->key(), &coll, &oid));
+      new_map_headers[ghobject_key(oid)] = got.begin()->second;
+    }
+
+    t->rmkeys(LEAF_PREFIX, legacy_to_remove);
+    t->set(HOBJECT_TO_SEQ, new_map_headers);
+    int r = db->submit_transaction(t);
+    if (r < 0)
+      return r;
+  }
+
+  
+  while (1) {
+    KeyValueDB::Transaction t = db->get_transaction();
+    KeyValueDB::Iterator iter = db->get_iterator(REVERSE_LEAF_PREFIX);
+    iter->seek_to_first();
+    if (!iter->valid())
+      break;
+    set to_remove;
+    unsigned count = 0;
+    for (; iter->valid() && count < 1000; iter->next(), ++count)
+      to_remove.insert(iter->key());
+    t->rmkeys(REVERSE_LEAF_PREFIX, to_remove);
+    db->submit_transaction(t);
+  }
+  state.v = 1;
+  KeyValueDB::Transaction t = db->get_transaction();
+  write_state(t);
+  db->submit_transaction_sync(t);
+  return 0;
+}
+
+int DBObjectMap::init(bool do_upgrade)
+{
+  map result;
+  set to_get;
+  to_get.insert(GLOBAL_STATE_KEY);
+  int r = db->get(SYS_PREFIX, to_get, &result);
+  if (r < 0)
+    return r;
+  if (!result.empty()) {
+    bufferlist::iterator bliter = result.begin()->second.begin();
+    state.decode(bliter);
+    if (state.v < 1) { // Needs upgrade
+      if (!do_upgrade) {
+	dout(1) << "DOBjbectMap requires an upgrade,"
+		<< " set filestore_update_to"
+		<< dendl;
+	return -ENOTSUP;
+      } else {
+	r = upgrade();
+	if (r < 0)
+	  return r;
+      }
+    }
+  } else {
+    // New store
+    state.v = 1;
+    state.seq = 1;
+  }
+  dout(20) << "(init)dbobjectmap: seq is " << state.seq << dendl;
+  return 0;
+}
+
+int DBObjectMap::sync(const ghobject_t *oid,
+		      const SequencerPosition *spos) {
+  KeyValueDB::Transaction t = db->get_transaction();
+  write_state(t);
+  if (oid) {
+    assert(spos);
+    Header header = lookup_map_header(*oid);
+    if (header) {
+      dout(10) << "oid: " << *oid << " setting spos to "
+	       << *spos << dendl;
+      header->spos = *spos;
+      set_map_header(*oid, *header, t);
+    }
+  }
+  return db->submit_transaction_sync(t);
+}
+
+int DBObjectMap::write_state(KeyValueDB::Transaction _t) {
+  dout(20) << "dbobjectmap: seq is " << state.seq << dendl;
+  KeyValueDB::Transaction t = _t ? _t : db->get_transaction();
+  bufferlist bl;
+  state.encode(bl);
+  map to_write;
+  to_write[GLOBAL_STATE_KEY] = bl;
+  t->set(SYS_PREFIX, to_write);
+  return _t ? 0 : db->submit_transaction(t);
+}
+
+
+DBObjectMap::Header DBObjectMap::_lookup_map_header(const ghobject_t &oid)
+{
+  while (map_header_in_use.count(oid))
+    header_cond.Wait(header_lock);
+
+  map out;
+  set to_get;
+  to_get.insert(map_header_key(oid));
+  int r = db->get(HOBJECT_TO_SEQ, to_get, &out);
+  if (r < 0)
+    return Header();
+  if (out.empty())
+    return Header();
+  
+  Header ret(new _Header(), RemoveMapHeaderOnDelete(this, oid));
+  bufferlist::iterator iter = out.begin()->second.begin();
+  ret->decode(iter);
+  return ret;
+}
+
+DBObjectMap::Header DBObjectMap::_generate_new_header(const ghobject_t &oid,
+						      Header parent)
+{
+  Header header = Header(new _Header(), RemoveOnDelete(this));
+  header->seq = state.seq++;
+  if (parent) {
+    header->parent = parent->seq;
+    header->spos = parent->spos;
+  }
+  header->num_children = 1;
+  header->oid = oid;
+  assert(!in_use.count(header->seq));
+  in_use.insert(header->seq);
+
+  write_state();
+  return header;
+}
+
+DBObjectMap::Header DBObjectMap::lookup_parent(Header input)
+{
+  Mutex::Locker l(header_lock);
+  while (in_use.count(input->parent))
+    header_cond.Wait(header_lock);
+  map out;
+  set keys;
+  keys.insert(HEADER_KEY);
+
+  dout(20) << "lookup_parent: parent " << input->parent
+       << " for seq " << input->seq << dendl;
+  int r = db->get(sys_parent_prefix(input), keys, &out);
+  if (r < 0) {
+    assert(0);
+    return Header();
+  }
+  if (out.empty()) {
+    assert(0);
+    return Header();
+  }
+
+  Header header = Header(new _Header(), RemoveOnDelete(this));
+  header->seq = input->parent;
+  bufferlist::iterator iter = out.begin()->second.begin();
+  header->decode(iter);
+  dout(20) << "lookup_parent: parent seq is " << header->seq << " with parent "
+       << header->parent << dendl;
+  in_use.insert(header->seq);
+  return header;
+}
+
+DBObjectMap::Header DBObjectMap::lookup_create_map_header(
+  const ghobject_t &oid,
+  KeyValueDB::Transaction t)
+{
+  Mutex::Locker l(header_lock);
+  Header header = _lookup_map_header(oid);
+  if (!header) {
+    header = _generate_new_header(oid, Header());
+    set_map_header(oid, *header, t);
+  }
+  return header;
+}
+
+void DBObjectMap::clear_header(Header header, KeyValueDB::Transaction t)
+{
+  dout(20) << "clear_header: clearing seq " << header->seq << dendl;
+  t->rmkeys_by_prefix(user_prefix(header));
+  t->rmkeys_by_prefix(sys_prefix(header));
+  t->rmkeys_by_prefix(complete_prefix(header));
+  t->rmkeys_by_prefix(xattr_prefix(header));
+  set keys;
+  keys.insert(header_key(header->seq));
+  t->rmkeys(USER_PREFIX, keys);
+}
+
+void DBObjectMap::set_header(Header header, KeyValueDB::Transaction t)
+{
+  dout(20) << "set_header: setting seq " << header->seq << dendl;
+  map to_write;
+  header->encode(to_write[HEADER_KEY]);
+  t->set(sys_prefix(header), to_write);
+}
+
+void DBObjectMap::remove_map_header(const ghobject_t &oid,
+				    Header header,
+				    KeyValueDB::Transaction t)
+{
+  dout(20) << "remove_map_header: removing " << header->seq
+	   << " oid " << oid << dendl;
+  set to_remove;
+  to_remove.insert(map_header_key(oid));
+  t->rmkeys(HOBJECT_TO_SEQ, to_remove);
+}
+
+void DBObjectMap::set_map_header(const ghobject_t &oid, _Header header,
+				 KeyValueDB::Transaction t)
+{
+  dout(20) << "set_map_header: setting " << header.seq
+	   << " oid " << oid << " parent seq "
+	   << header.parent << dendl;
+  map to_set;
+  header.encode(to_set[map_header_key(oid)]);
+  t->set(HOBJECT_TO_SEQ, to_set);
+}
+
+bool DBObjectMap::check_spos(const ghobject_t &oid,
+			     Header header,
+			     const SequencerPosition *spos)
+{
+  if (!spos || *spos > header->spos) {
+    stringstream out;
+    if (spos)
+      dout(10) << "oid: " << oid << " not skipping op, *spos "
+	       << *spos << dendl;
+    else
+      dout(10) << "oid: " << oid << " not skipping op, *spos "
+	       << "empty" << dendl;
+    dout(10) << " > header.spos " << header->spos << dendl;
+    return false;
+  } else {
+    dout(10) << "oid: " << oid << " skipping op, *spos " << *spos
+	     << " <= header.spos " << header->spos << dendl;
+    return true;
+  }
+}
+
+int DBObjectMap::list_objects(vector *out)
+{
+  KeyValueDB::Iterator iter = db->get_iterator(HOBJECT_TO_SEQ);
+  for (iter->seek_to_first(); iter->valid(); iter->next()) {
+    bufferlist bl = iter->value();
+    bufferlist::iterator bliter = bl.begin();
+    _Header header;
+    header.decode(bliter);
+    out->push_back(header.oid);
+  }
+  return 0;
+}
diff --git a/ceph/src/os/DBObjectMap.h b/ceph/src/os/DBObjectMap.h
new file mode 100644
index 00000000..a71c369f
--- /dev/null
+++ b/ceph/src/os/DBObjectMap.h
@@ -0,0 +1,489 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#ifndef DBOBJECTMAP_DB_H
+#define DBOBJECTMAP_DB_H
+
+#include "include/buffer.h"
+#include 
+#include 
+#include 
+
+#include 
+#include "include/memory.h"
+#include 
+
+#include "ObjectMap.h"
+#include "KeyValueDB.h"
+#include "osd/osd_types.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+
+/**
+ * DBObjectMap: Implements ObjectMap in terms of KeyValueDB
+ *
+ * Prefix space structure:
+ *
+ * @see complete_prefix
+ * @see user_prefix
+ * @see sys_prefix
+ *
+ * - GHOBJECT_TO_SEQ: Contains leaf mapping from ghobject_t->hobj.seq and
+ *                   corresponding omap header
+ * - SYS_PREFIX: GLOBAL_STATE_KEY - contains next seq number
+ *                                  @see State
+ *                                  @see write_state
+ *                                  @see init
+ *                                  @see generate_new_header
+ * - USER_PREFIX + header_key(header->seq) + USER_PREFIX
+ *              : key->value for header->seq
+ * - USER_PREFIX + header_key(header->seq) + COMPLETE_PREFIX: see below
+ * - USER_PREFIX + header_key(header->seq) + XATTR_PREFIX: xattrs
+ * - USER_PREFIX + header_key(header->seq) + SYS_PREFIX
+ *              : USER_HEADER_KEY - omap header for header->seq
+ *              : HEADER_KEY - encoding of header for header->seq
+ *
+ * For each node (represented by a header), we
+ * store three mappings: the key mapping, the complete mapping, and the parent.
+ * The complete mapping (COMPLETE_PREFIX space) is key->key.  Each x->y entry in
+ * this mapping indicates that the key mapping contains all entries on [x,y).
+ * Note, max string is represented by "", so ""->"" indicates that the parent
+ * is unnecessary (@see rm_keys).  When looking up a key not contained in the
+ * the complete set, we have to check the parent if we don't find it in the
+ * key set.  During rm_keys, we copy keys from the parent and update the
+ * complete set to reflect the change @see rm_keys.
+ */
+class DBObjectMap : public ObjectMap {
+public:
+  boost::scoped_ptr db;
+
+  /**
+   * Serializes access to next_seq as well as the in_use set
+   */
+  Mutex header_lock;
+  Cond header_cond;
+  Cond map_header_cond;
+
+  /**
+   * Set of headers currently in use
+   */
+  set in_use;
+  set map_header_in_use;
+
+  DBObjectMap(KeyValueDB *db) : db(db),
+				header_lock("DBOBjectMap")
+    {}
+
+  int set_keys(
+    const ghobject_t &oid,
+    const map &set,
+    const SequencerPosition *spos=0
+    );
+
+  int set_header(
+    const ghobject_t &oid,
+    const bufferlist &bl,
+    const SequencerPosition *spos=0
+    );
+
+  int get_header(
+    const ghobject_t &oid,
+    bufferlist *bl
+    );
+
+  int clear(
+    const ghobject_t &oid,
+    const SequencerPosition *spos=0
+    );
+
+  int clear_keys_header(
+    const ghobject_t &oid,
+    const SequencerPosition *spos=0
+    );
+
+  int rm_keys(
+    const ghobject_t &oid,
+    const set &to_clear,
+    const SequencerPosition *spos=0
+    );
+
+  int get(
+    const ghobject_t &oid,
+    bufferlist *header,
+    map *out
+    );
+
+  int get_keys(
+    const ghobject_t &oid,
+    set *keys
+    );
+
+  int get_values(
+    const ghobject_t &oid,
+    const set &keys,
+    map *out
+    );
+
+  int check_keys(
+    const ghobject_t &oid,
+    const set &keys,
+    set *out
+    );
+
+  int get_xattrs(
+    const ghobject_t &oid,
+    const set &to_get,
+    map *out
+    );
+
+  int get_all_xattrs(
+    const ghobject_t &oid,
+    set *out
+    );
+
+  int set_xattrs(
+    const ghobject_t &oid,
+    const map &to_set,
+    const SequencerPosition *spos=0
+    );
+
+  int remove_xattrs(
+    const ghobject_t &oid,
+    const set &to_remove,
+    const SequencerPosition *spos=0
+    );
+
+  int clone(
+    const ghobject_t &oid,
+    const ghobject_t &target,
+    const SequencerPosition *spos=0
+    );
+
+  /// Read initial state from backing store
+  int init(bool upgrade = false);
+
+  /// Upgrade store to current version
+  int upgrade();
+
+  /// Consistency check, debug, there must be no parallel writes
+  bool check(std::ostream &out);
+
+  /// Ensure that all previous operations are durable
+  int sync(const ghobject_t *oid=0, const SequencerPosition *spos=0);
+
+  /// Util, list all objects, there must be no other concurrent access
+  int list_objects(vector *objs ///< [out] objects
+    );
+
+  ObjectMapIterator get_iterator(const ghobject_t &oid);
+
+  static const string USER_PREFIX;
+  static const string XATTR_PREFIX;
+  static const string SYS_PREFIX;
+  static const string COMPLETE_PREFIX;
+  static const string HEADER_KEY;
+  static const string USER_HEADER_KEY;
+  static const string GLOBAL_STATE_KEY;
+  static const string HOBJECT_TO_SEQ;
+
+  /// Legacy
+  static const string LEAF_PREFIX;
+  static const string REVERSE_LEAF_PREFIX;
+
+  /// persistent state for store @see generate_header
+  struct State {
+    __u8 v;
+    uint64_t seq;
+    State() : v(0), seq(1) {}
+    State(uint64_t seq) : v(0), seq(seq) {}
+
+    void encode(bufferlist &bl) const {
+      ENCODE_START(2, 1, bl);
+      ::encode(v, bl);
+      ::encode(seq, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(bufferlist::iterator &bl) {
+      DECODE_START(2, bl);
+      if (struct_v >= 2)
+	::decode(v, bl);
+      else
+	v = 0;
+      ::decode(seq, bl);
+      DECODE_FINISH(bl);
+    }
+
+    void dump(Formatter *f) const {
+      f->dump_unsigned("seq", seq);
+    }
+
+    static void generate_test_instances(list &o) {
+      o.push_back(new State(0));
+      o.push_back(new State(20));
+    }
+  } state;
+
+  struct _Header {
+    uint64_t seq;
+    uint64_t parent;
+    uint64_t num_children;
+
+    coll_t c;
+    ghobject_t oid;
+
+    SequencerPosition spos;
+
+    void encode(bufferlist &bl) const {
+      ENCODE_START(2, 1, bl);
+      ::encode(seq, bl);
+      ::encode(parent, bl);
+      ::encode(num_children, bl);
+      ::encode(c, bl);
+      ::encode(oid, bl);
+      ::encode(spos, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(bufferlist::iterator &bl) {
+      DECODE_START(2, bl);
+      ::decode(seq, bl);
+      ::decode(parent, bl);
+      ::decode(num_children, bl);
+      ::decode(c, bl);
+      ::decode(oid, bl);
+      if (struct_v >= 2)
+	::decode(spos, bl);
+      DECODE_FINISH(bl);
+    }
+
+    void dump(Formatter *f) const {
+      f->dump_unsigned("seq", seq);
+      f->dump_unsigned("parent", parent);
+      f->dump_unsigned("num_children", num_children);
+      f->dump_stream("coll") << c;
+      f->dump_stream("oid") << oid;
+    }
+
+    static void generate_test_instances(list<_Header*> &o) {
+      o.push_back(new _Header);
+      o.push_back(new _Header);
+      o.back()->parent = 20;
+      o.back()->seq = 30;
+    }
+
+    _Header() : seq(0), parent(0), num_children(1) {}
+  };
+
+  /// String munging (public for testing)
+  static string ghobject_key(const ghobject_t &oid);
+  static string ghobject_key_v0(coll_t c, const ghobject_t &oid);
+  static bool parse_ghobject_key_v0(const string &in,
+				   coll_t *c, ghobject_t *oid);
+private:
+  /// Implicit lock on Header->seq
+  typedef ceph::shared_ptr<_Header> Header;
+
+  string map_header_key(const ghobject_t &oid);
+  string header_key(uint64_t seq);
+  string complete_prefix(Header header);
+  string user_prefix(Header header);
+  string sys_prefix(Header header);
+  string xattr_prefix(Header header);
+  string sys_parent_prefix(_Header header);
+  string sys_parent_prefix(Header header) {
+    return sys_parent_prefix(*header);
+  }
+
+  class EmptyIteratorImpl : public ObjectMapIteratorImpl {
+  public:
+    int seek_to_first() { return 0; }
+    int seek_to_last() { return 0; }
+    int upper_bound(const string &after) { return 0; }
+    int lower_bound(const string &to) { return 0; }
+    bool valid() { return false; }
+    int next() { assert(0); return 0; }
+    string key() { assert(0); return ""; }
+    bufferlist value() { assert(0); return bufferlist(); }
+    int status() { return 0; }
+  };
+
+
+  /// Iterator
+  class DBObjectMapIteratorImpl : public ObjectMapIteratorImpl {
+  public:
+    DBObjectMap *map;
+
+    /// NOTE: implicit lock on header->seq AND for all ancestors
+    Header header;
+
+    /// parent_iter == NULL iff no parent
+    ceph::shared_ptr parent_iter;
+    KeyValueDB::Iterator key_iter;
+    KeyValueDB::Iterator complete_iter;
+
+    /// cur_iter points to currently valid iterator
+    ceph::shared_ptr cur_iter;
+    int r;
+
+    /// init() called, key_iter, complete_iter, parent_iter filled in
+    bool ready;
+    /// past end
+    bool invalid;
+
+    DBObjectMapIteratorImpl(DBObjectMap *map, Header header) :
+      map(map), header(header), r(0), ready(false), invalid(true) {}
+    int seek_to_first();
+    int seek_to_last();
+    int upper_bound(const string &after);
+    int lower_bound(const string &to);
+    bool valid();
+    int next();
+    string key();
+    bufferlist value();
+    int status();
+
+    bool on_parent() {
+      return cur_iter == parent_iter;
+    }
+
+    /// skips to next valid parent entry
+    int next_parent();
+
+    /// Tests whether to_test is in complete region
+    int in_complete_region(const string &to_test, ///< [in] key to test
+			   string *begin,         ///< [out] beginning of region
+			   string *end            ///< [out] end of region
+      ); ///< @returns true if to_test is in the complete region, else false
+
+  private:
+    int init();
+    bool valid_parent();
+    int adjust();
+  };
+
+  typedef ceph::shared_ptr DBObjectMapIterator;
+  DBObjectMapIterator _get_iterator(Header header) {
+    return DBObjectMapIterator(new DBObjectMapIteratorImpl(this, header));
+  }
+
+  /// sys
+
+  /// Removes node corresponding to header
+  void clear_header(Header header, KeyValueDB::Transaction t);
+
+  /// Set node containing input to new contents
+  void set_header(Header input, KeyValueDB::Transaction t);
+
+  /// Remove leaf node corresponding to oid in c
+  void remove_map_header(const ghobject_t &oid,
+			 Header header,
+			 KeyValueDB::Transaction t);
+
+  /// Set leaf node for c and oid to the value of header
+  void set_map_header(const ghobject_t &oid, _Header header,
+		      KeyValueDB::Transaction t);
+
+  /// Set leaf node for c and oid to the value of header
+  bool check_spos(const ghobject_t &oid,
+		  Header header,
+		  const SequencerPosition *spos);
+
+  /// Lookup or create header for c oid
+  Header lookup_create_map_header(const ghobject_t &oid,
+				  KeyValueDB::Transaction t);
+
+  /**
+   * Generate new header for c oid with new seq number
+   *
+   * Has the side effect of syncronously saving the new DBObjectMap state
+   */
+  Header _generate_new_header(const ghobject_t &oid, Header parent);
+  Header generate_new_header(const ghobject_t &oid, Header parent) {
+    Mutex::Locker l(header_lock);
+    return _generate_new_header(oid, parent);
+  }
+
+  /// Lookup leaf header for c oid
+  Header _lookup_map_header(const ghobject_t &oid);
+  Header lookup_map_header(const ghobject_t &oid) {
+    Mutex::Locker l(header_lock);
+    return _lookup_map_header(oid);
+  }
+
+  /// Lookup header node for input
+  Header lookup_parent(Header input);
+
+
+  /// Helpers
+  int _get_header(Header header, bufferlist *bl);
+
+  /// Scan keys in header into out_keys and out_values (if nonnull)
+  int scan(Header header,
+	   const set &in_keys,
+	   set *out_keys,
+	   map *out_values);
+
+  /// Remove header and all related prefixes
+  int _clear(Header header,
+	     KeyValueDB::Transaction t);
+  /// Adds to t operations necessary to add new_complete to the complete set
+  int merge_new_complete(Header header,
+			 const map &new_complete,
+			 DBObjectMapIterator iter,
+			 KeyValueDB::Transaction t);
+
+  /// Writes out State (mainly next_seq)
+  int write_state(KeyValueDB::Transaction _t =
+		  KeyValueDB::Transaction());
+
+  /// 0 if the complete set now contains all of key space, < 0 on error, 1 else
+  int need_parent(DBObjectMapIterator iter);
+
+  /// Copies header entry from parent @see rm_keys
+  int copy_up_header(Header header,
+		     KeyValueDB::Transaction t);
+
+  /// Sets header @see set_header
+  void _set_header(Header header, const bufferlist &bl,
+		   KeyValueDB::Transaction t);
+
+  /** 
+   * Removes map header lock once Header is out of scope
+   * @see lookup_map_header
+   */
+  class RemoveMapHeaderOnDelete {
+  public:
+    DBObjectMap *db;
+    ghobject_t oid;
+    RemoveMapHeaderOnDelete(DBObjectMap *db, const ghobject_t &oid) :
+      db(db), oid(oid) {}
+    void operator() (_Header *header) {
+      Mutex::Locker l(db->header_lock);
+      db->map_header_in_use.erase(oid);
+      db->map_header_cond.Signal();
+      delete header;
+    }
+  };
+
+  /** 
+   * Removes header seq lock once Header is out of scope
+   * @see lookup_parent
+   * @see generate_new_header
+   */
+  class RemoveOnDelete {
+  public:
+    DBObjectMap *db;
+    RemoveOnDelete(DBObjectMap *db) :
+      db(db) {}
+    void operator() (_Header *header) {
+      Mutex::Locker l(db->header_lock);
+      db->in_use.erase(header->seq);
+      db->header_cond.Signal();
+      delete header;
+    }
+  };
+  friend class RemoveOnDelete;
+};
+WRITE_CLASS_ENCODER(DBObjectMap::_Header)
+WRITE_CLASS_ENCODER(DBObjectMap::State)
+
+#endif
diff --git a/ceph/src/os/FDCache.h b/ceph/src/os/FDCache.h
new file mode 100644
index 00000000..ba11e120
--- /dev/null
+++ b/ceph/src/os/FDCache.h
@@ -0,0 +1,98 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_FDCACHE_H
+#define CEPH_FDCACHE_H
+
+#include 
+#include 
+#include 
+#include "common/hobject.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/shared_cache.hpp"
+#include "include/compat.h"
+
+/**
+ * FD Cache
+ */
+class FDCache : public md_config_obs_t {
+public:
+  /**
+   * FD
+   *
+   * Wrapper for an fd.  Destructor closes the fd.
+   */
+  class FD {
+  public:
+    const int fd;
+    FD(int _fd) : fd(_fd) {
+      assert(_fd >= 0);
+    }
+    int operator*() const {
+      return fd;
+    }
+    ~FD() {
+      VOID_TEMP_FAILURE_RETRY(::close(fd));
+    }
+  };
+
+private:
+  SharedLRU registry;
+  CephContext *cct;
+
+public:
+  FDCache(CephContext *cct) : cct(cct) {
+    assert(cct);
+    cct->_conf->add_observer(this);
+    registry.set_size(cct->_conf->filestore_fd_cache_size);
+  }
+  ~FDCache() {
+    cct->_conf->remove_observer(this);
+  }
+  typedef ceph::shared_ptr FDRef;
+
+  FDRef lookup(const ghobject_t &hoid) {
+    return registry.lookup(hoid);
+  }
+
+  FDRef add(const ghobject_t &hoid, int fd) {
+    return registry.add(hoid, new FD(fd));
+  }
+
+  /// clear cached fd for hoid, subsequent lookups will get an empty FD
+  void clear(const ghobject_t &hoid) {
+    registry.clear(hoid);
+    assert(!registry.lookup(hoid));
+  }
+
+  /// md_config_obs_t
+  const char** get_tracked_conf_keys() const {
+    static const char* KEYS[] = {
+      "filestore_fd_cache_size",
+      NULL
+    };
+    return KEYS;
+  }
+  void handle_conf_change(const md_config_t *conf,
+			  const std::set &changed) {
+    if (changed.count("filestore_fd_cache_size")) {
+      registry.set_size(conf->filestore_fd_cache_size);
+    }
+  }
+
+};
+typedef FDCache::FDRef FDRef;
+
+#endif
diff --git a/ceph/src/os/FileJournal.cc b/ceph/src/os/FileJournal.cc
new file mode 100644
index 00000000..7eb7927e
--- /dev/null
+++ b/ceph/src/os/FileJournal.cc
@@ -0,0 +1,1877 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include "acconfig.h"
+
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "FileJournal.h"
+#include "include/color.h"
+#include "common/perf_counters.h"
+#include "os/FileStore.h"
+
+#include "include/compat.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/blkdev.h"
+#include "common/linux_version.h"
+
+#define dout_subsys ceph_subsys_journal
+#undef dout_prefix
+#define dout_prefix *_dout << "journal "
+
+const static int64_t ONE_MEG(1 << 20);
+
+int FileJournal::_open(bool forwrite, bool create)
+{
+  int flags, ret;
+
+  if (aio && !directio) {
+    derr << "FileJournal::_open: aio not supported without directio; disabling aio" << dendl;
+    aio = false;
+  }
+#ifndef HAVE_LIBAIO
+  if (aio) {
+    derr << "FileJournal::_open: libaio not compiled in; disabling aio" << dendl;
+    aio = false;
+  }
+#endif
+
+  if (forwrite) {
+    flags = O_RDWR;
+    if (directio)
+      flags |= O_DIRECT | O_DSYNC;
+  } else {
+    flags = O_RDONLY;
+  }
+  if (create)
+    flags |= O_CREAT;
+  
+  if (fd >= 0) {
+    if (TEMP_FAILURE_RETRY(::close(fd))) {
+      int err = errno;
+      derr << "FileJournal::_open: error closing old fd: "
+	   << cpp_strerror(err) << dendl;
+    }
+  }
+  fd = TEMP_FAILURE_RETRY(::open(fn.c_str(), flags, 0644));
+  if (fd < 0) {
+    int err = errno;
+    dout(2) << "FileJournal::_open unable to open journal "
+	    << fn << ": " << cpp_strerror(err) << dendl;
+    return -err;
+  }
+
+  struct stat st;
+  ret = ::fstat(fd, &st);
+  if (ret) {
+    ret = errno;
+    derr << "FileJournal::_open: unable to fstat journal: " << cpp_strerror(ret) << dendl;
+    goto out_fd;
+  }
+
+  if (S_ISBLK(st.st_mode)) {
+    ret = _open_block_device();
+  } else {
+    if (aio && !force_aio) {
+      derr << "FileJournal::_open: disabling aio for non-block journal.  Use "
+	   << "journal_force_aio to force use of aio anyway" << dendl;
+      aio = false;
+    }
+    ret = _open_file(st.st_size, st.st_blksize, create);
+  }
+
+  if (ret)
+    goto out_fd;
+
+#ifdef HAVE_LIBAIO
+  aio_ctx = 0;
+  ret = io_setup(128, &aio_ctx);
+  if (ret < 0) {
+    ret = errno;
+    derr << "FileJournal::_open: unable to setup io_context " << cpp_strerror(ret) << dendl;
+    goto out_fd;
+  }
+#endif
+
+  /* We really want max_size to be a multiple of block_size. */
+  max_size -= max_size % block_size;
+
+  dout(1) << "_open " << fn << " fd " << fd
+	  << ": " << max_size 
+	  << " bytes, block size " << block_size
+	  << " bytes, directio = " << directio
+	  << ", aio = " << aio
+	  << dendl;
+  return 0;
+
+ out_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+  return ret;
+}
+
+int FileJournal::_open_block_device()
+{
+  int64_t bdev_sz = 0;
+  int ret = get_block_device_size(fd, &bdev_sz);
+  if (ret) {
+    dout(0) << __func__ << ": failed to read block device size." << dendl;
+    return -EIO;
+  }
+
+  /* Check for bdev_sz too small */
+  if (bdev_sz < ONE_MEG) {
+    dout(0) << __func__ << ": your block device must be at least "
+      << ONE_MEG << " bytes to be used for a Ceph journal." << dendl;
+    return -EINVAL;
+  }
+
+  int64_t conf_journal_sz(g_conf->osd_journal_size);
+  conf_journal_sz <<= 20;
+
+  dout(10) << __func__ << ": ignoring osd journal size. "
+	   << "We'll use the entire block device (size: " << bdev_sz << ")"
+	   << dendl;
+  max_size = bdev_sz;
+
+  /* block devices have to write in blocks of CEPH_PAGE_SIZE */
+  block_size = CEPH_PAGE_SIZE;
+
+  _check_disk_write_cache();
+  return 0;
+}
+
+void FileJournal::_check_disk_write_cache() const
+{
+  ostringstream hdparm_cmd;
+  FILE *fp = NULL;
+
+  if (geteuid() != 0) {
+    dout(10) << "_check_disk_write_cache: not root, NOT checking disk write "
+      << "cache on raw block device " << fn << dendl;
+    goto done;
+  }
+
+  hdparm_cmd << "/sbin/hdparm -W " << fn;
+  fp = popen(hdparm_cmd.str().c_str(), "r");
+  if (!fp) {
+    dout(10) << "_check_disk_write_cache: failed to run /sbin/hdparm: NOT "
+      << "checking disk write cache on raw block device " << fn << dendl;
+    goto done;
+  }
+
+  while (true) {
+    char buf[256];
+    memset(buf, 0, sizeof(buf));
+    char *line = fgets(buf, sizeof(buf) - 1, fp);
+    if (!line) {
+      if (ferror(fp)) {
+	int ret = -errno;
+	derr << "_check_disk_write_cache: fgets error: " << cpp_strerror(ret)
+	     << dendl;
+	goto close_f;
+      }
+      else {
+	// EOF.
+	break;
+      }
+    }
+
+    int on;
+    if (sscanf(line, " write-caching =  %d", &on) != 1)
+      continue;
+    if (!on) {
+      dout(10) << "_check_disk_write_cache: disk write cache is off (good) on "
+	       << fn << dendl;
+      break;
+    }
+
+    // is our kernel new enough?
+    int ver = get_linux_version();
+    if (ver == 0) {
+      dout(10) << "_check_disk_write_cache: get_linux_version failed" << dendl;
+    } else if (ver >= KERNEL_VERSION(2, 6, 33)) {
+      dout(20) << "_check_disk_write_cache: disk write cache is on, but your "
+	       << "kernel is new enough to handle it correctly. (fn:"
+	       << fn << ")" << dendl;
+      break;
+    }
+    derr << TEXT_RED
+	 << " ** WARNING: disk write cache is ON on " << fn << ".\n"
+	 << "    Journaling will not be reliable on kernels prior to 2.6.33\n"
+	 << "    (recent kernels are safe).  You can disable the write cache with\n"
+	 << "    'hdparm -W 0 " << fn << "'"
+	 << TEXT_NORMAL
+	 << dendl;
+    break;
+  }
+
+close_f:
+  if (pclose(fp)) {
+    int ret = -errno;
+    derr << "_check_disk_write_cache: pclose failed: " << cpp_strerror(ret)
+	 << dendl;
+  }
+done:
+  ;
+}
+
+int FileJournal::_open_file(int64_t oldsize, blksize_t blksize,
+			    bool create)
+{
+  int ret;
+  int64_t conf_journal_sz(g_conf->osd_journal_size);
+  conf_journal_sz <<= 20;
+
+  if ((g_conf->osd_journal_size == 0) && (oldsize < ONE_MEG)) {
+    derr << "I'm sorry, I don't know how large of a journal to create."
+	 << "Please specify a block device to use as the journal OR "
+	 << "set osd_journal_size in your ceph.conf" << dendl;
+    return -EINVAL;
+  }
+
+  if (create && (oldsize < conf_journal_sz)) {
+    uint64_t newsize(g_conf->osd_journal_size);
+    newsize <<= 20;
+    dout(10) << "_open extending to " << newsize << " bytes" << dendl;
+    ret = ::ftruncate(fd, newsize);
+    if (ret < 0) {
+      int err = errno;
+      derr << "FileJournal::_open_file : unable to extend journal to "
+	   << newsize << " bytes: " << cpp_strerror(err) << dendl;
+      return -err;
+    }
+#ifdef HAVE_POSIX_FALLOCATE
+    ret = ::posix_fallocate(fd, 0, newsize);
+    if (ret) {
+      derr << "FileJournal::_open_file : unable to preallocation journal to "
+	   << newsize << " bytes: " << cpp_strerror(ret) << dendl;
+      return -ret;
+    }
+    max_size = newsize;
+#elif defined(__APPLE__)
+    fstore_t store;
+    store.fst_flags = F_ALLOCATECONTIG;
+    store.fst_posmode = F_PEOFPOSMODE;
+    store.fst_offset = 0;
+    store.fst_length = newsize;
+
+    ret = ::fcntl(fd, F_PREALLOCATE, &store);
+    if (ret == -1) {
+      ret = -errno;
+      derr << "FileJournal::_open_file : unable to preallocation journal to "
+	   << newsize << " bytes: " << cpp_strerror(ret) << dendl;
+      return ret;
+    }
+    max_size = newsize;
+#else
+# error "Journal pre-allocation not supported on platform."
+#endif
+  }
+  else {
+    max_size = oldsize;
+  }
+  block_size = MAX(blksize, (blksize_t)CEPH_PAGE_SIZE);
+
+  if (create && g_conf->journal_zero_on_create) {
+    derr << "FileJournal::_open_file : zeroing journal" << dendl;
+    uint64_t write_size = 1 << 20;
+    char *buf = new char[write_size];
+    memset(static_cast(buf), 0, write_size);
+    uint64_t i = 0;
+    for (; (i + write_size) <= (unsigned)max_size; i += write_size) {
+      ret = ::pwrite(fd, static_cast(buf), write_size, i);
+      if (ret < 0) {
+	delete [] buf;
+	return -errno;
+      }
+    }
+    if (i < (unsigned)max_size) {
+      ret = ::pwrite(fd, static_cast(buf), max_size - i, i);
+      if (ret < 0) {
+	delete [] buf;
+	return -errno;
+      }
+    }
+    delete [] buf;
+  }
+      
+
+  dout(10) << "_open journal is not a block device, NOT checking disk "
+           << "write cache on '" << fn << "'" << dendl;
+
+  return 0;
+}
+
+int FileJournal::check()
+{
+  int ret;
+
+  ret = _open(false, false);
+  if (ret < 0)
+    goto done;
+
+  ret = read_header();
+  if (ret < 0)
+    goto done;
+
+  if (header.fsid != fsid) {
+    derr << "check: ondisk fsid " << header.fsid << " doesn't match expected " << fsid
+	 << ", invalid (someone else's?) journal" << dendl;
+    ret = -EINVAL;
+    goto done;
+  }
+
+  dout(1) << "check: header looks ok" << dendl;
+  ret = 0;
+
+ done:
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+  fd = -1;
+  return ret;
+}
+
+
+int FileJournal::create()
+{
+  void *buf = 0;
+  int64_t needed_space;
+  int ret;
+  buffer::ptr bp;
+  dout(2) << "create " << fn << " fsid " << fsid << dendl;
+
+  ret = _open(true, true);
+  if (ret < 0)
+    goto done;
+
+  // write empty header
+  header = header_t();
+  header.flags = header_t::FLAG_CRC;  // enable crcs on any new journal.
+  header.fsid = fsid;
+  header.max_size = max_size;
+  header.block_size = block_size;
+  if (g_conf->journal_block_align || directio)
+    header.alignment = block_size;
+  else
+    header.alignment = 16;  // at least stay word aligned on 64bit machines...
+
+  header.start = get_top();
+  header.start_seq = 0;
+
+  print_header();
+
+  // static zeroed buffer for alignment padding
+  delete [] zero_buf;
+  zero_buf = new char[header.alignment];
+  memset(zero_buf, 0, header.alignment);
+
+  bp = prepare_header();
+  if (TEMP_FAILURE_RETRY(::pwrite(fd, bp.c_str(), bp.length(), 0)) < 0) {
+    ret = errno;
+    derr << "FileJournal::create : create write header error "
+         << cpp_strerror(ret) << dendl;
+    goto close_fd;
+  }
+
+  // zero first little bit, too.
+  ret = posix_memalign(&buf, block_size, block_size);
+  if (ret) {
+    derr << "FileJournal::create: failed to allocate " << block_size
+	 << " bytes of memory: " << cpp_strerror(ret) << dendl;
+    goto close_fd;
+  }
+  memset(buf, 0, block_size);
+  if (TEMP_FAILURE_RETRY(::pwrite(fd, buf, block_size, get_top())) < 0) {
+    ret = errno;
+    derr << "FileJournal::create: error zeroing first " << block_size
+	 << " bytes " << cpp_strerror(ret) << dendl;
+    goto free_buf;
+  }
+
+  needed_space = ((int64_t)g_conf->osd_max_write_size) << 20;
+  needed_space += (2 * sizeof(entry_header_t)) + get_top();
+  if (header.max_size - header.start < needed_space) {
+    derr << "FileJournal::create: OSD journal is not large enough to hold "
+	 << "osd_max_write_size bytes!" << dendl;
+    ret = -ENOSPC;
+    goto free_buf;
+  }
+
+  dout(2) << "create done" << dendl;
+  ret = 0;
+
+free_buf:
+  free(buf);
+  buf = 0;
+close_fd:
+  if (TEMP_FAILURE_RETRY(::close(fd)) < 0) {
+    ret = errno;
+    derr << "FileJournal::create: error closing fd: " << cpp_strerror(ret)
+	 << dendl;
+    goto done;
+  }
+done:
+  fd = -1;
+  return ret;
+}
+
+int FileJournal::peek_fsid(uuid_d& fsid)
+{
+  int r = _open(false, false);
+  if (r < 0)
+    return r;
+  r = read_header();
+  if (r < 0)
+    return r;
+  fsid = header.fsid;
+  return 0;
+}
+
+int FileJournal::open(uint64_t fs_op_seq)
+{
+  dout(2) << "open " << fn << " fsid " << fsid << " fs_op_seq " << fs_op_seq << dendl;
+
+  last_committed_seq = fs_op_seq;
+  uint64_t next_seq = fs_op_seq + 1;
+
+  int err = _open(false);
+  if (err < 0) 
+    return err;
+
+  // assume writeable, unless...
+  read_pos = 0;
+  write_pos = get_top();
+
+  // read header?
+  err = read_header();
+  if (err < 0)
+    return err;
+
+  // static zeroed buffer for alignment padding
+  delete [] zero_buf;
+  zero_buf = new char[header.alignment];
+  memset(zero_buf, 0, header.alignment);
+
+  dout(10) << "open header.fsid = " << header.fsid 
+    //<< " vs expected fsid = " << fsid 
+	   << dendl;
+  if (header.fsid != fsid) {
+    derr << "FileJournal::open: ondisk fsid " << header.fsid << " doesn't match expected " << fsid
+         << ", invalid (someone else's?) journal" << dendl;
+    return -EINVAL;
+  }
+  if (header.max_size > max_size) {
+    dout(2) << "open journal size " << header.max_size << " > current " << max_size << dendl;
+    return -EINVAL;
+  }
+  if (header.block_size != block_size) {
+    dout(2) << "open journal block size " << header.block_size << " != current " << block_size << dendl;
+    return -EINVAL;
+  }
+  if (header.max_size % header.block_size) {
+    dout(2) << "open journal max size " << header.max_size
+	    << " not a multiple of block size " << header.block_size << dendl;
+    return -EINVAL;
+  }
+  if (header.alignment != block_size && directio) {
+    dout(0) << "open journal alignment " << header.alignment << " does not match block size " 
+	    << block_size << " (required for direct_io journal mode)" << dendl;
+    return -EINVAL;
+  }
+  if ((header.alignment % CEPH_PAGE_SIZE) && directio) {
+    dout(0) << "open journal alignment " << header.alignment << " is not multiple of page size " << CEPH_PAGE_SIZE
+	    << " (required for direct_io journal mode)" << dendl;
+    return -EINVAL;
+  }
+
+  // looks like a valid header.
+  write_pos = 0;  // not writeable yet
+
+  // find next entry
+  read_pos = header.start;
+  uint64_t seq = header.start_seq;
+  while (1) {
+    bufferlist bl;
+    off64_t old_pos = read_pos;
+    if (!read_entry(bl, seq)) {
+      dout(10) << "open reached end of journal." << dendl;
+      break;
+    }
+    if (seq > next_seq) {
+      dout(10) << "open entry " << seq << " len " << bl.length() << " > next_seq " << next_seq
+	       << ", ignoring journal contents"
+	       << dendl;
+      read_pos = -1;
+      last_committed_seq = 0;
+      seq = 0;
+      return 0;
+    }
+    if (seq == next_seq) {
+      dout(10) << "open reached seq " << seq << dendl;
+      read_pos = old_pos;
+      break;
+    }
+    seq++;  // next event should follow.
+  }
+
+  return 0;
+}
+
+void FileJournal::close()
+{
+  dout(1) << "close " << fn << dendl;
+
+  // stop writer thread
+  stop_writer();
+
+  // close
+  assert(writeq_empty());
+  assert(fd >= 0);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+  fd = -1;
+}
+
+
+int FileJournal::dump(ostream& out)
+{
+  dout(10) << "dump" << dendl;
+  _open(false, false);
+
+  int err = read_header();
+  if (err < 0)
+    return err;
+
+  read_pos = header.start;
+
+  JSONFormatter f(true);
+
+  f.open_array_section("journal");
+  while (1) {
+    bufferlist bl;
+    uint64_t seq = 0;
+    uint64_t pos = read_pos;
+    if (!read_entry(bl, seq)) {
+      dout(3) << "journal_replay: end of journal, done." << dendl;
+      break;
+    }
+
+    f.open_object_section("entry");
+    f.dump_unsigned("offset", pos);
+    f.dump_unsigned("seq", seq);
+    f.open_array_section("transactions");
+    bufferlist::iterator p = bl.begin();
+    int trans_num = 0;
+    while (!p.end()) {
+      ObjectStore::Transaction *t = new ObjectStore::Transaction(p);
+      f.open_object_section("transaction");
+      f.dump_unsigned("trans_num", trans_num);
+      t->dump(&f);
+      f.close_section();
+      delete t;
+      trans_num++;
+    }
+    f.close_section();
+    f.close_section();
+    f.flush(cout);
+  }
+
+  f.close_section();
+  dout(10) << "dump finish" << dendl;
+  return 0;
+}
+
+
+void FileJournal::start_writer()
+{
+  write_stop = false;
+  write_thread.create();
+#ifdef HAVE_LIBAIO
+  write_finish_thread.create();
+#endif
+}
+
+void FileJournal::stop_writer()
+{
+  {
+    Mutex::Locker l(write_lock);
+#ifdef HAVE_LIBAIO
+    Mutex::Locker q(aio_lock);
+#endif
+    Mutex::Locker p(writeq_lock);
+    write_stop = true;
+    writeq_cond.Signal();
+#ifdef HAVE_LIBAIO
+    aio_cond.Signal();
+    write_finish_cond.Signal();
+#endif
+  } 
+  write_thread.join();
+#ifdef HAVE_LIBAIO
+  write_finish_thread.join();
+#endif
+}
+
+
+
+void FileJournal::print_header()
+{
+  dout(10) << "header: block_size " << header.block_size
+	   << " alignment " << header.alignment
+	   << " max_size " << header.max_size
+	   << dendl;
+  dout(10) << "header: start " << header.start << dendl;
+  dout(10) << " write_pos " << write_pos << dendl;
+}
+
+int FileJournal::read_header()
+{
+  dout(10) << "read_header" << dendl;
+  bufferlist bl;
+
+  buffer::ptr bp = buffer::create_page_aligned(block_size);
+  bp.zero();
+  int r = ::pread(fd, bp.c_str(), bp.length(), 0);
+  bl.push_back(bp);
+
+  try {
+    bufferlist::iterator p = bl.begin();
+    ::decode(header, p);
+  }
+  catch (buffer::error& e) {
+    derr << "read_header error decoding journal header" << dendl;
+    return -EINVAL;
+  }
+
+  if (r < 0) {
+    int err = errno;
+    dout(0) << "read_header got " << cpp_strerror(err) << dendl;
+    return -err;
+  }
+  
+  /*
+   * Unfortunately we weren't initializing the flags field for new
+   * journals!  Aie.  This is safe(ish) now that we have only one
+   * flag.  Probably around when we add the next flag we need to
+   * remove this or else this (eventually old) code will clobber newer
+   * code's flags.
+   */
+  if (header.flags > 3) {
+    derr << "read_header appears to have gibberish flags; assuming 0" << dendl;
+    header.flags = 0;
+  }
+
+  print_header();
+
+  return 0;
+}
+
+bufferptr FileJournal::prepare_header()
+{
+  bufferlist bl;
+  {
+    Mutex::Locker l(finisher_lock);
+    header.committed_up_to = journaled_seq;
+  }
+  ::encode(header, bl);
+  bufferptr bp = buffer::create_page_aligned(get_top());
+  bp.zero();
+  memcpy(bp.c_str(), bl.c_str(), bl.length());
+  return bp;
+}
+
+
+
+int FileJournal::check_for_full(uint64_t seq, off64_t pos, off64_t size)
+{
+  // already full?
+  if (full_state != FULL_NOTFULL)
+    return -ENOSPC;
+
+  // take 1 byte off so that we only get pos == header.start on EMPTY, never on FULL.
+  off64_t room;
+  if (pos >= header.start)
+    room = (header.max_size - pos) + (header.start - get_top()) - 1;
+  else
+    room = header.start - pos - 1;
+  dout(10) << "room " << room << " max_size " << max_size << " pos " << pos << " header.start " << header.start
+	   << " top " << get_top() << dendl;
+
+  if (do_sync_cond) {
+    if (room < (header.max_size >> 1) &&
+	room + size > (header.max_size >> 1)) {
+      dout(10) << " passing half full mark, triggering commit" << dendl;
+      do_sync_cond->SloppySignal();  // initiate a real commit so we can trim
+    }
+  }
+
+  if (room >= size) {
+    dout(10) << "check_for_full at " << pos << " : " << size << " < " << room << dendl;
+    if (pos + size > header.max_size)
+      must_write_header = true;
+    return 0;
+  }
+
+  // full
+  dout(1) << "check_for_full at " << pos << " : JOURNAL FULL "
+	  << pos << " >= " << room
+	  << " (max_size " << header.max_size << " start " << header.start << ")"
+	  << dendl;
+
+  off64_t max = header.max_size - get_top();
+  if (size > max)
+    dout(0) << "JOURNAL TOO SMALL: continuing, but slow: item " << size << " > journal " << max << " (usable)" << dendl;
+
+  return -ENOSPC;
+}
+
+int FileJournal::prepare_multi_write(bufferlist& bl, uint64_t& orig_ops, uint64_t& orig_bytes)
+{
+  // gather queued writes
+  off64_t queue_pos = write_pos;
+
+  int eleft = g_conf->journal_max_write_entries;
+  unsigned bmax = g_conf->journal_max_write_bytes;
+
+  if (full_state != FULL_NOTFULL)
+    return -ENOSPC;
+  
+  while (!writeq_empty()) {
+    int r = prepare_single_write(bl, queue_pos, orig_ops, orig_bytes);
+    if (r == -ENOSPC) {
+      if (orig_ops)
+	break;         // commit what we have
+
+      if (logger)
+	logger->inc(l_os_j_full);
+
+      if (wait_on_full) {
+	dout(20) << "prepare_multi_write full on first entry, need to wait" << dendl;
+      } else {
+	dout(20) << "prepare_multi_write full on first entry, restarting journal" << dendl;
+
+	// throw out what we have so far
+	full_state = FULL_FULL;
+	while (!writeq_empty()) {
+	  put_throttle(1, peek_write().bl.length());
+	  pop_write();
+	}  
+	print_header();
+      }
+
+      return -ENOSPC;  // hrm, full on first op
+    }
+
+    if (eleft) {
+      if (--eleft == 0) {
+	dout(20) << "prepare_multi_write hit max events per write " << g_conf->journal_max_write_entries << dendl;
+	break;
+      }
+    }
+    if (bmax) {
+      if (bl.length() >= bmax) {
+	dout(20) << "prepare_multi_write hit max write size " << g_conf->journal_max_write_bytes << dendl;
+	break;
+      }
+    }
+  }
+
+  dout(20) << "prepare_multi_write queue_pos now " << queue_pos << dendl;
+  //assert(write_pos + bl.length() == queue_pos);
+  return 0;
+}
+
+/*
+void FileJournal::queue_write_fin(uint64_t seq, Context *fin)
+{
+  writing_seq.push_back(seq);
+  if (!waiting_for_notfull.empty()) {
+    // make sure previously unjournaled stuff waiting for UNFULL triggers
+    // _before_ newly journaled stuff does
+    dout(10) << "queue_write_fin will defer seq " << seq << " callback " << fin
+	     << " until after UNFULL" << dendl;
+    C_Gather *g = new C_Gather(writeq.front().fin);
+    writing_fin.push_back(g->new_sub());
+    waiting_for_notfull.push_back(g->new_sub());
+  } else {
+    writing_fin.push_back(writeq.front().fin);
+    dout(20) << "queue_write_fin seq " << seq << " callback " << fin << dendl;
+  }
+}
+*/
+
+void FileJournal::queue_completions_thru(uint64_t seq)
+{
+  assert(finisher_lock.is_locked());
+  utime_t now = ceph_clock_now(g_ceph_context);
+  while (!completions_empty()) {
+    completion_item next = completion_peek_front();
+    if (next.seq > seq)
+      break;
+    completion_pop_front();
+    utime_t lat = now;
+    lat -= next.start;
+    dout(10) << "queue_completions_thru seq " << seq
+	     << " queueing seq " << next.seq
+	     << " " << next.finish
+	     << " lat " << lat << dendl;
+    if (logger) {
+      logger->tinc(l_os_j_lat, lat);
+    }
+    if (next.finish)
+      finisher->queue(next.finish);
+    if (next.tracked_op)
+      next.tracked_op->mark_event("journaled_completion_queued");
+  }
+  finisher_cond.Signal();
+}
+
+int FileJournal::prepare_single_write(bufferlist& bl, off64_t& queue_pos, uint64_t& orig_ops, uint64_t& orig_bytes)
+{
+  // grab next item
+  write_item &next_write = peek_write();
+  uint64_t seq = next_write.seq;
+  bufferlist &ebl = next_write.bl;
+  unsigned head_size = sizeof(entry_header_t);
+  off64_t base_size = 2*head_size + ebl.length();
+
+  int alignment = next_write.alignment; // we want to start ebl with this alignment
+  unsigned pre_pad = 0;
+  if (alignment >= 0)
+    pre_pad = ((unsigned int)alignment - (unsigned int)head_size) & ~CEPH_PAGE_MASK;
+  off64_t size = ROUND_UP_TO(base_size + pre_pad, header.alignment);
+  unsigned post_pad = size - base_size - pre_pad;
+
+  int r = check_for_full(seq, queue_pos, size);
+  if (r < 0)
+    return r;   // ENOSPC or EAGAIN
+
+  orig_bytes += ebl.length();
+  orig_ops++;
+
+  // add to write buffer
+  dout(15) << "prepare_single_write " << orig_ops << " will write " << queue_pos << " : seq " << seq
+	   << " len " << ebl.length() << " -> " << size
+	   << " (head " << head_size << " pre_pad " << pre_pad
+	   << " ebl " << ebl.length() << " post_pad " << post_pad << " tail " << head_size << ")"
+	   << " (ebl alignment " << alignment << ")"
+	   << dendl;
+    
+  // add it this entry
+  entry_header_t h;
+  memset(&h, 0, sizeof(h));
+  h.seq = seq;
+  h.pre_pad = pre_pad;
+  h.len = ebl.length();
+  h.post_pad = post_pad;
+  h.make_magic(queue_pos, header.get_fsid64());
+  h.crc32c = ebl.crc32c(0);
+
+  bl.append((const char*)&h, sizeof(h));
+  if (pre_pad) {
+    bufferptr bp = buffer::create_static(pre_pad, zero_buf);
+    bl.push_back(bp);
+  }
+  bl.claim_append(ebl);
+
+  if (h.post_pad) {
+    bufferptr bp = buffer::create_static(post_pad, zero_buf);
+    bl.push_back(bp);
+  }
+  bl.append((const char*)&h, sizeof(h));
+
+  if (next_write.tracked_op)
+    next_write.tracked_op->mark_event("write_thread_in_journal_buffer");
+
+  // pop from writeq
+  pop_write();
+  journalq.push_back(pair(seq, queue_pos));
+  writing_seq = seq;
+
+  queue_pos += size;
+  if (queue_pos >= header.max_size)
+    queue_pos = queue_pos + get_top() - header.max_size;
+
+  return 0;
+}
+
+void FileJournal::align_bl(off64_t pos, bufferlist& bl)
+{
+  // make sure list segments are page aligned
+  if (directio && (!bl.is_page_aligned() ||
+		   !bl.is_n_page_sized())) {
+    bl.rebuild_page_aligned();
+    if ((bl.length() & ~CEPH_PAGE_MASK) != 0 ||
+	(pos & ~CEPH_PAGE_MASK) != 0)
+      dout(0) << "rebuild_page_aligned failed, " << bl << dendl;
+    assert((bl.length() & ~CEPH_PAGE_MASK) == 0);
+    assert((pos & ~CEPH_PAGE_MASK) == 0);
+  }
+}
+
+int FileJournal::write_bl(off64_t& pos, bufferlist& bl)
+{
+  int ret;
+
+  off64_t spos = ::lseek64(fd, pos, SEEK_SET);
+  if (spos < 0) {
+    ret = -errno;
+    derr << "FileJournal::write_bl : lseek64 failed " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+  ret = bl.write_fd(fd);
+  if (ret) {
+    derr << "FileJournal::write_bl : write_fd failed: " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+  pos += bl.length();
+  if (pos == header.max_size)
+    pos = get_top();
+  return 0;
+}
+
+void FileJournal::do_write(bufferlist& bl)
+{
+  // nothing to do?
+  if (bl.length() == 0 && !must_write_header) 
+    return;
+
+  buffer::ptr hbp;
+  if (g_conf->journal_write_header_frequency &&
+      (((++journaled_since_start) %
+	g_conf->journal_write_header_frequency) == 0)) {
+    must_write_header = true;
+  }
+
+  if (must_write_header) {
+    must_write_header = false;
+    hbp = prepare_header();
+  }
+
+  dout(15) << "do_write writing " << write_pos << "~" << bl.length() 
+	   << (hbp.length() ? " + header":"")
+	   << dendl;
+  
+  utime_t from = ceph_clock_now(g_ceph_context);
+
+  // entry
+  off64_t pos = write_pos;
+
+  // Adjust write_pos
+  align_bl(pos, bl);
+  write_pos += bl.length();
+  if (write_pos >= header.max_size)
+    write_pos = write_pos - header.max_size + get_top();
+
+  write_lock.Unlock();
+
+  // split?
+  off64_t split = 0;
+  if (pos + bl.length() > header.max_size) {
+    bufferlist first, second;
+    split = header.max_size - pos;
+    first.substr_of(bl, 0, split);
+    second.substr_of(bl, split, bl.length() - split);
+    assert(first.length() + second.length() == bl.length());
+    dout(10) << "do_write wrapping, first bit at " << pos << " len " << first.length()
+	     << " second bit len " << second.length() << " (orig len " << bl.length() << ")" << dendl;
+
+    if (write_bl(pos, first)) {
+      derr << "FileJournal::do_write: write_bl(pos=" << pos
+	   << ") failed" << dendl;
+      ceph_abort();
+    }
+    assert(pos == get_top());
+    if (hbp.length()) {
+      // be sneaky: include the header in the second fragment
+      second.push_front(hbp);
+      pos = 0;          // we included the header
+    }
+    if (write_bl(pos, second)) {
+      derr << "FileJournal::do_write: write_bl(pos=" << pos
+	   << ") failed" << dendl;
+      ceph_abort();
+    }
+  } else {
+    // header too?
+    if (hbp.length()) {
+      if (TEMP_FAILURE_RETRY(::pwrite(fd, hbp.c_str(), hbp.length(), 0)) < 0) {
+	int err = errno;
+	derr << "FileJournal::do_write: pwrite(fd=" << fd
+	     << ", hbp.length=" << hbp.length() << ") failed :"
+	     << cpp_strerror(err) << dendl;
+	ceph_abort();
+      }
+    }
+
+    if (write_bl(pos, bl)) {
+      derr << "FileJournal::do_write: write_bl(pos=" << pos
+	   << ") failed" << dendl;
+      ceph_abort();
+    }
+  }
+
+  if (!directio) {
+    dout(20) << "do_write fsync" << dendl;
+
+    /*
+     * We'd really love to have a fsync_range or fdatasync_range and do a:
+     *
+     *  if (split) {
+     *    ::fsync_range(fd, header.max_size - split, split)l
+     *    ::fsync_range(fd, get_top(), bl.length() - split);
+     *  else
+     *    ::fsync_range(fd, write_pos, bl.length())
+     *
+     * NetBSD and AIX apparently have it, and adding it to Linux wouldn't be
+     * too hard given all the underlying infrastructure already exist.
+     *
+     * NOTE: using sync_file_range here would not be safe as it does not
+     * flush disk caches or commits any sort of metadata.
+     */
+#if defined(DARWIN) || defined(__FreeBSD__)
+    ::fsync(fd);
+#else
+    ::fdatasync(fd);
+#endif
+  }
+
+  utime_t lat = ceph_clock_now(g_ceph_context) - from;    
+  dout(20) << "do_write latency " << lat << dendl;
+
+  write_lock.Lock();    
+
+  assert(write_pos == pos);
+  assert(write_pos % header.alignment == 0);
+
+  {
+    Mutex::Locker locker(finisher_lock);
+    journaled_seq = writing_seq;
+
+    // kick finisher?
+    //  only if we haven't filled up recently!
+    if (full_state != FULL_NOTFULL) {
+      dout(10) << "do_write NOT queueing finisher seq " << journaled_seq
+	       << ", full_commit_seq|full_restart_seq" << dendl;
+    } else {
+      if (plug_journal_completions) {
+	dout(20) << "do_write NOT queueing finishers through seq " << journaled_seq
+		 << " due to completion plug" << dendl;
+      } else {
+	dout(20) << "do_write queueing finishers through seq " << journaled_seq << dendl;
+	queue_completions_thru(journaled_seq);
+      }
+    }
+  }
+}
+
+void FileJournal::flush()
+{
+  dout(10) << "waiting for completions to empty" << dendl;
+  {
+    Mutex::Locker l(finisher_lock);
+    while (!completions_empty())
+      finisher_cond.Wait(finisher_lock);
+  }
+  dout(10) << "flush waiting for finisher" << dendl;
+  finisher->wait_for_empty();
+  dout(10) << "flush done" << dendl;
+}
+
+
+void FileJournal::write_thread_entry()
+{
+  dout(10) << "write_thread_entry start" << dendl;
+  while (1) {
+    {
+      Mutex::Locker locker(writeq_lock);
+      if (writeq.empty()) {
+	if (write_stop)
+	  break;
+	dout(20) << "write_thread_entry going to sleep" << dendl;
+	writeq_cond.Wait(writeq_lock);
+	dout(20) << "write_thread_entry woke up" << dendl;
+	continue;
+      }
+    }
+    
+#ifdef HAVE_LIBAIO
+    if (aio) {
+      Mutex::Locker locker(aio_lock);
+      // should we back off to limit aios in flight?  try to do this
+      // adaptively so that we submit larger aios once we have lots of
+      // them in flight.
+      //
+      // NOTE: our condition here is based on aio_num (protected by
+      // aio_lock) and throttle_bytes (part of the write queue).  when
+      // we sleep, we *only* wait for aio_num to change, and do not
+      // wake when more data is queued.  this is not strictly correct,
+      // but should be fine given that we will have plenty of aios in
+      // flight if we hit this limit to ensure we keep the device
+      // saturated.
+      while (aio_num > 0) {
+	int exp = MIN(aio_num * 2, 24);
+	long unsigned min_new = 1ull << exp;
+	long unsigned cur = throttle_bytes.get_current();
+	dout(20) << "write_thread_entry aio throttle: aio num " << aio_num << " bytes " << aio_bytes
+		 << " ... exp " << exp << " min_new " << min_new
+		 << " ... pending " << cur << dendl;
+	if (cur >= min_new)
+	  break;
+	dout(20) << "write_thread_entry deferring until more aios complete: "
+		 << aio_num << " aios with " << aio_bytes << " bytes needs " << min_new
+		 << " bytes to start a new aio (currently " << cur << " pending)" << dendl;
+	aio_cond.Wait(aio_lock);
+	dout(20) << "write_thread_entry woke up" << dendl;
+      }
+    }
+#endif
+
+    Mutex::Locker locker(write_lock);
+    uint64_t orig_ops = 0;
+    uint64_t orig_bytes = 0;
+
+    bufferlist bl;
+    int r = prepare_multi_write(bl, orig_ops, orig_bytes);
+    if (r == -ENOSPC) {
+      dout(20) << "write_thread_entry full, going to sleep (waiting for commit)" << dendl;
+      commit_cond.Wait(write_lock);
+      dout(20) << "write_thread_entry woke up" << dendl;
+      continue;
+    }
+    assert(r == 0);
+
+    if (logger) {
+      logger->inc(l_os_j_wr);
+      logger->inc(l_os_j_wr_bytes, bl.length());
+    }
+
+#ifdef HAVE_LIBAIO
+    if (aio)
+      do_aio_write(bl);
+    else
+      do_write(bl);
+#else
+    do_write(bl);
+#endif
+    put_throttle(orig_ops, orig_bytes);
+  }
+
+  dout(10) << "write_thread_entry finish" << dendl;
+}
+
+#ifdef HAVE_LIBAIO
+void FileJournal::do_aio_write(bufferlist& bl)
+{
+
+  if (g_conf->journal_write_header_frequency &&
+      (((++journaled_since_start) %
+	g_conf->journal_write_header_frequency) == 0)) {
+    must_write_header = true;
+  }
+
+  // nothing to do?
+  if (bl.length() == 0 && !must_write_header) 
+    return;
+
+  buffer::ptr hbp;
+  if (must_write_header) {
+    must_write_header = false;
+    hbp = prepare_header();
+  }
+
+  // entry
+  off64_t pos = write_pos;
+
+  dout(15) << "do_aio_write writing " << pos << "~" << bl.length() 
+	   << (hbp.length() ? " + header":"")
+	   << dendl;
+  
+  // split?
+  off64_t split = 0;
+  if (pos + bl.length() > header.max_size) {
+    bufferlist first, second;
+    split = header.max_size - pos;
+    first.substr_of(bl, 0, split);
+    second.substr_of(bl, split, bl.length() - split);
+    assert(first.length() + second.length() == bl.length());
+    dout(10) << "do_aio_write wrapping, first bit at " << pos << "~" << first.length() << dendl;
+
+    if (write_aio_bl(pos, first, 0)) {
+      derr << "FileJournal::do_aio_write: write_aio_bl(pos=" << pos
+	   << ") failed" << dendl;
+      ceph_abort();
+    }
+    assert(pos == header.max_size);
+    if (hbp.length()) {
+      // be sneaky: include the header in the second fragment
+      second.push_front(hbp);
+      pos = 0;          // we included the header
+    } else
+      pos = get_top();  // no header, start after that
+    if (write_aio_bl(pos, second, writing_seq)) {
+      derr << "FileJournal::do_aio_write: write_aio_bl(pos=" << pos
+	   << ") failed" << dendl;
+      ceph_abort();
+    }
+  } else {
+    // header too?
+    if (hbp.length()) {
+      bufferlist hbl;
+      hbl.push_back(hbp);
+      loff_t pos = 0;
+      if (write_aio_bl(pos, hbl, 0)) {
+	derr << "FileJournal::do_aio_write: write_aio_bl(header) failed" << dendl;
+	ceph_abort();
+      }
+    }
+
+    if (write_aio_bl(pos, bl, writing_seq)) {
+      derr << "FileJournal::do_aio_write: write_aio_bl(pos=" << pos
+	   << ") failed" << dendl;
+      ceph_abort();
+    }
+  }
+
+  write_pos = pos;
+  if (write_pos == header.max_size)
+    write_pos = get_top();
+  assert(write_pos % header.alignment == 0);
+}
+
+/**
+ * write a buffer using aio
+ *
+ * @param seq seq to trigger when this aio completes.  if 0, do not update any state
+ * on completion.
+ */
+int FileJournal::write_aio_bl(off64_t& pos, bufferlist& bl, uint64_t seq)
+{
+  Mutex::Locker locker(aio_lock);
+  align_bl(pos, bl);
+
+  dout(20) << "write_aio_bl " << pos << "~" << bl.length() << " seq " << seq << dendl;
+  
+  while (bl.length() > 0) {
+    int max = MIN(bl.buffers().size(), IOV_MAX-1);
+    iovec *iov = new iovec[max];
+    int n = 0;
+    unsigned len = 0;
+    for (std::list::const_iterator p = bl.buffers().begin();
+	 n < max;
+	 ++p, ++n) {
+      assert(p != bl.buffers().end());
+      iov[n].iov_base = (void *)p->c_str();
+      iov[n].iov_len = p->length();
+      len += p->length();
+    }
+
+    bufferlist tbl;
+    bl.splice(0, len, &tbl);  // move bytes from bl -> tbl
+
+    aio_queue.push_back(aio_info(tbl, pos, bl.length() > 0 ? 0 : seq));
+    aio_info& aio = aio_queue.back();
+    aio.iov = iov;
+
+    io_prep_pwritev(&aio.iocb, fd, aio.iov, n, pos);
+
+    dout(20) << "write_aio_bl .. " << aio.off << "~" << aio.len
+	     << " in " << n << dendl;
+
+    aio_num++;
+    aio_bytes += aio.len;
+
+    iocb *piocb = &aio.iocb;
+    int attempts = 10;
+    do {
+      int r = io_submit(aio_ctx, 1, &piocb);
+      if (r < 0) {
+	derr << "io_submit to " << aio.off << "~" << aio.len
+	     << " got " << cpp_strerror(r) << dendl;
+	if (r == -EAGAIN && attempts-- > 0) {
+	  usleep(500);
+	  continue;
+	}
+	assert(0 == "io_submit got unexpected error");
+      }
+    } while (false);
+    pos += aio.len;
+  }
+  write_finish_cond.Signal();
+  return 0;
+}
+#endif
+
+void FileJournal::write_finish_thread_entry()
+{
+#ifdef HAVE_LIBAIO
+  dout(10) << "write_finish_thread_entry enter" << dendl;
+  while (true) {
+    {
+      Mutex::Locker locker(aio_lock);
+      if (aio_queue.empty()) {
+	if (write_stop)
+	  break;
+	dout(20) << "write_finish_thread_entry sleeping" << dendl;
+	write_finish_cond.Wait(aio_lock);
+	continue;
+      }
+    }
+    
+    dout(20) << "write_finish_thread_entry waiting for aio(s)" << dendl;
+    io_event event[16];
+    int r = io_getevents(aio_ctx, 1, 16, event, NULL);
+    if (r < 0) {
+      if (r == -EINTR) {
+	dout(0) << "io_getevents got " << cpp_strerror(r) << dendl;
+	continue;
+      }
+      derr << "io_getevents got " << cpp_strerror(r) << dendl;
+      assert(0 == "got unexpected error from io_getevents");
+    }
+    
+    {
+      Mutex::Locker locker(aio_lock);
+      for (int i=0; ilen) {
+	  derr << "aio to " << ai->off << "~" << ai->len
+	       << " got " << cpp_strerror(event[i].res) << dendl;
+	  assert(0 == "unexpected aio error");
+	}
+	dout(10) << "write_finish_thread_entry aio " << ai->off
+		 << "~" << ai->len << " done" << dendl;
+	ai->done = true;
+      }
+      check_aio_completion();
+    }
+  }
+  dout(10) << "write_finish_thread_entry exit" << dendl;
+#endif
+}
+
+#ifdef HAVE_LIBAIO
+/**
+ * check aio_wait for completed aio, and update state appropriately.
+ */
+void FileJournal::check_aio_completion()
+{
+  assert(aio_lock.is_locked());
+  dout(20) << "check_aio_completion" << dendl;
+
+  bool completed_something = false;
+  uint64_t new_journaled_seq = 0;
+
+  list::iterator p = aio_queue.begin();
+  while (p != aio_queue.end() && p->done) {
+    dout(20) << "check_aio_completion completed seq " << p->seq << " "
+	     << p->off << "~" << p->len << dendl;
+    if (p->seq) {
+      new_journaled_seq = p->seq;
+      completed_something = true;
+    }
+    aio_num--;
+    aio_bytes -= p->len;
+    aio_queue.erase(p++);
+  }
+
+  if (completed_something) {
+    // kick finisher?  
+    //  only if we haven't filled up recently!
+    Mutex::Locker locker(finisher_lock);
+    journaled_seq = new_journaled_seq;
+    if (full_state != FULL_NOTFULL) {
+      dout(10) << "check_aio_completion NOT queueing finisher seq " << journaled_seq
+	       << ", full_commit_seq|full_restart_seq" << dendl;
+    } else {
+      if (plug_journal_completions) {
+	dout(20) << "check_aio_completion NOT queueing finishers through seq " << journaled_seq
+		 << " due to completion plug" << dendl;
+      } else {
+	dout(20) << "check_aio_completion queueing finishers through seq " << journaled_seq << dendl;
+	queue_completions_thru(journaled_seq);
+      }
+    }
+
+    // maybe write queue was waiting for aio count to drop?
+    aio_cond.Signal();
+  }
+}
+#endif
+
+void FileJournal::submit_entry(uint64_t seq, bufferlist& e, int alignment,
+			       Context *oncommit, TrackedOpRef osd_op)
+{
+  // dump on queue
+  dout(5) << "submit_entry seq " << seq
+	  << " len " << e.length()
+	  << " (" << oncommit << ")" << dendl;
+  assert(e.length() > 0);
+
+  dout(30) << "XXX throttle take " << e.length() << dendl;
+  throttle_ops.take(1);
+  throttle_bytes.take(e.length());
+  if (osd_op)
+    osd_op->mark_event("commit_queued_for_journal_write");
+  if (logger) {
+    logger->set(l_os_jq_max_ops, throttle_ops.get_max());
+    logger->set(l_os_jq_max_bytes, throttle_bytes.get_max());
+    logger->set(l_os_jq_ops, throttle_ops.get_current());
+    logger->set(l_os_jq_bytes, throttle_bytes.get_current());
+  }
+
+  {
+    Mutex::Locker l1(writeq_lock);  // ** lock **
+    Mutex::Locker l2(completions_lock);  // ** lock **
+    completions.push_back(
+      completion_item(
+	seq, oncommit, ceph_clock_now(g_ceph_context), osd_op));
+    writeq.push_back(write_item(seq, e, alignment, osd_op));
+    writeq_cond.Signal();
+  }
+}
+
+bool FileJournal::writeq_empty()
+{
+  Mutex::Locker locker(writeq_lock);
+  return writeq.empty();
+}
+
+FileJournal::write_item &FileJournal::peek_write()
+{
+  assert(write_lock.is_locked());
+  Mutex::Locker locker(writeq_lock);
+  return writeq.front();
+}
+
+void FileJournal::pop_write()
+{
+  assert(write_lock.is_locked());
+  Mutex::Locker locker(writeq_lock);
+  writeq.pop_front();
+}
+
+void FileJournal::commit_start(uint64_t seq)
+{
+  dout(10) << "commit_start" << dendl;
+
+  // was full?
+  switch (full_state) {
+  case FULL_NOTFULL:
+    break; // all good
+
+  case FULL_FULL:
+    if (seq >= journaled_seq) {
+      dout(1) << " FULL_FULL -> FULL_WAIT.  commit_start on seq "
+	      << seq << " > journaled_seq " << journaled_seq
+	      << ", moving to FULL_WAIT."
+	      << dendl;
+      full_state = FULL_WAIT;
+    } else {
+      dout(1) << "FULL_FULL commit_start on seq "
+	      << seq << " < journaled_seq " << journaled_seq
+	      << ", remaining in FULL_FULL"
+	      << dendl;
+    }
+    break;
+
+  case FULL_WAIT:
+    dout(1) << " FULL_WAIT -> FULL_NOTFULL.  journal now active, setting completion plug." << dendl;
+    full_state = FULL_NOTFULL;
+    plug_journal_completions = true;
+    break;
+  }
+}
+
+void FileJournal::committed_thru(uint64_t seq)
+{
+  Mutex::Locker locker(write_lock);
+
+  if (seq < last_committed_seq) {
+    dout(5) << "committed_thru " << seq << " < last_committed_seq " << last_committed_seq << dendl;
+    assert(seq >= last_committed_seq);
+    return;
+  }
+  if (seq == last_committed_seq) {
+    dout(5) << "committed_thru " << seq << " == last_committed_seq " << last_committed_seq << dendl;
+    return;
+  }
+
+  dout(5) << "committed_thru " << seq << " (last_committed_seq " << last_committed_seq << ")" << dendl;
+  last_committed_seq = seq;
+
+  // completions!
+  {
+    Mutex::Locker locker(finisher_lock);
+    queue_completions_thru(seq);
+    if (plug_journal_completions && seq >= header.start_seq) {
+      dout(10) << " removing completion plug, queuing completions thru journaled_seq " << journaled_seq << dendl;
+      plug_journal_completions = false;
+      queue_completions_thru(journaled_seq);
+    }
+  }
+
+  // adjust start pointer
+  while (!journalq.empty() && journalq.front().first <= seq) {
+    journalq.pop_front();
+  }
+  if (!journalq.empty()) {
+    header.start = journalq.front().second;
+    header.start_seq = journalq.front().first;
+  } else {
+    header.start = write_pos;
+    header.start_seq = seq + 1;
+  }
+  must_write_header = true;
+  print_header();
+
+  // committed but unjournaled items
+  while (!writeq_empty() && peek_write().seq <= seq) {
+    dout(15) << " dropping committed but unwritten seq " << peek_write().seq 
+	     << " len " << peek_write().bl.length()
+	     << dendl;
+    put_throttle(1, peek_write().bl.length());
+    pop_write();
+  }
+  
+  commit_cond.Signal();
+
+  dout(10) << "committed_thru done" << dendl;
+}
+
+
+void FileJournal::put_throttle(uint64_t ops, uint64_t bytes)
+{
+  uint64_t new_ops = throttle_ops.put(ops);
+  uint64_t new_bytes = throttle_bytes.put(bytes);
+  dout(5) << "put_throttle finished " << ops << " ops and " 
+	   << bytes << " bytes, now "
+	   << new_ops << " ops and " << new_bytes << " bytes"
+	   << dendl;
+
+  if (logger) {
+    logger->inc(l_os_j_ops, ops);
+    logger->inc(l_os_j_bytes, bytes);
+    logger->set(l_os_jq_ops, new_ops);
+    logger->set(l_os_jq_bytes, new_bytes);
+    logger->set(l_os_jq_max_ops, throttle_ops.get_max());
+    logger->set(l_os_jq_max_bytes, throttle_bytes.get_max());
+  }
+}
+
+int FileJournal::make_writeable()
+{
+  dout(10) << __func__ << dendl;
+  int r = _open(true);
+  if (r < 0)
+    return r;
+
+  if (read_pos > 0)
+    write_pos = read_pos;
+  else
+    write_pos = get_top();
+  read_pos = 0;
+
+  must_write_header = true;
+  start_writer();
+  return 0;
+}
+
+void FileJournal::wrap_read_bl(
+  off64_t pos,
+  int64_t olen,
+  bufferlist* bl,
+  off64_t *out_pos
+  )
+{
+  while (olen > 0) {
+    while (pos >= header.max_size)
+      pos = pos + get_top() - header.max_size;
+
+    int64_t len;
+    if (pos + olen > header.max_size)
+      len = header.max_size - pos;        // partial
+    else
+      len = olen;                         // rest
+    
+    int64_t actual = ::lseek64(fd, pos, SEEK_SET);
+    assert(actual == pos);
+    
+    bufferptr bp = buffer::create(len);
+    int r = safe_read_exact(fd, bp.c_str(), len);
+    if (r) {
+      derr << "FileJournal::wrap_read_bl: safe_read_exact " << pos << "~" << len << " returned "
+	   << r << dendl;
+      ceph_abort();
+    }
+    bl->push_back(bp);
+    pos += len;
+    olen -= len;
+  }
+  if (pos >= header.max_size)
+    pos = pos + get_top() - header.max_size;
+  if (out_pos)
+    *out_pos = pos;
+}
+
+bool FileJournal::read_entry(
+  bufferlist &bl,
+  uint64_t &next_seq,
+  bool *corrupt)
+{
+  if (corrupt)
+    *corrupt = false;
+  uint64_t seq = next_seq;
+
+  if (!read_pos) {
+    dout(2) << "read_entry -- not readable" << dendl;
+    return false;
+  }
+
+  off64_t pos = read_pos;
+  off64_t next_pos = pos;
+  stringstream ss;
+  read_entry_result result = do_read_entry(
+    pos,
+    &next_pos,
+    &bl,
+    &seq,
+    &ss);
+  if (result == SUCCESS) {
+    if (next_seq > seq) {
+      return false;
+    } else {
+      read_pos = next_pos;
+      next_seq = seq;
+      return true;
+    }
+  }
+
+  stringstream errss;
+  if (seq < header.committed_up_to) {
+    derr << "Unable to read past sequence " << seq
+	 << " but header indicates the journal has committed up through "
+	 << header.committed_up_to << ", journal is corrupt" << dendl;
+    if (g_conf->journal_ignore_corruption) {
+      if (corrupt)
+	*corrupt = true;
+      return false;
+    } else {
+      assert(0);
+    }
+  }
+
+  dout(25) << errss.str() << dendl;
+  dout(2) << "No further valid entries found, journal is most likely valid"
+	  << dendl;
+  return false;
+}
+
+FileJournal::read_entry_result FileJournal::do_read_entry(
+  off64_t pos,
+  off64_t *next_pos,
+  bufferlist *bl,
+  uint64_t *seq,
+  ostream *ss,
+  entry_header_t *_h)
+{
+  bufferlist _bl;
+  if (!bl)
+    bl = &_bl;
+
+  // header
+  entry_header_t *h;
+  bufferlist hbl;
+  off64_t _next_pos;
+  wrap_read_bl(pos, sizeof(*h), &hbl, &_next_pos);
+  h = (entry_header_t *)hbl.c_str();
+
+  if (!h->check_magic(pos, header.get_fsid64())) {
+    dout(25) << "read_entry " << pos
+	     << " : bad header magic, end of journal" << dendl;
+    if (ss)
+      *ss << "bad header magic";
+    if (next_pos)
+      *next_pos = pos + (4<<10); // check 4k ahead
+    return MAYBE_CORRUPT;
+  }
+  pos = _next_pos;
+
+  // pad + body + pad
+  if (h->pre_pad)
+    pos += h->pre_pad;
+
+  bl->clear();
+  wrap_read_bl(pos, h->len, bl, &pos);
+
+  if (h->post_pad)
+    pos += h->post_pad;
+
+  // footer
+  entry_header_t *f;
+  bufferlist fbl;
+  wrap_read_bl(pos, sizeof(*f), &fbl, &pos);
+  f = (entry_header_t *)fbl.c_str();
+  if (memcmp(f, h, sizeof(*f))) {
+    if (ss)
+      *ss << "bad footer magic, partial entry";
+    if (next_pos)
+      *next_pos = pos;
+    return MAYBE_CORRUPT;
+  }
+
+  if ((header.flags & header_t::FLAG_CRC) ||   // if explicitly enabled (new journal)
+      h->crc32c != 0) {                        // newer entry in old journal
+    uint32_t actual_crc = bl->crc32c(0);
+    if (actual_crc != h->crc32c) {
+      if (ss)
+	*ss << "header crc (" << h->crc32c
+	    << ") doesn't match body crc (" << actual_crc << ")";
+      if (next_pos)
+	*next_pos = pos;
+      return MAYBE_CORRUPT;
+    }
+  }
+
+  // yay!
+  dout(2) << "read_entry " << pos << " : seq " << h->seq
+	  << " " << h->len << " bytes"
+	  << dendl;
+
+  // ok!
+  if (seq)
+    *seq = h->seq;
+
+  // works around an apparent GCC 4.8(?) compiler bug about unaligned
+  // bind by reference to (packed) h->seq
+  journalq.push_back(
+    pair(static_cast(h->seq),
+			   static_cast(pos)));
+
+  if (next_pos)
+    *next_pos = pos;
+
+  if (_h)
+    *_h = *h;
+
+  assert(pos % header.alignment == 0);
+  return SUCCESS;
+}
+
+void FileJournal::throttle()
+{
+  if (throttle_ops.wait(g_conf->journal_queue_max_ops))
+    dout(2) << "throttle: waited for ops" << dendl;
+  if (throttle_bytes.wait(g_conf->journal_queue_max_bytes))
+    dout(2) << "throttle: waited for bytes" << dendl;
+}
+
+void FileJournal::get_header(
+  uint64_t wanted_seq,
+  off64_t *_pos,
+  entry_header_t *h)
+{
+  off64_t pos = header.start;
+  off64_t next_pos = pos;
+  bufferlist bl;
+  uint64_t seq = 0;
+  while (1) {
+    bl.clear();
+    pos = next_pos;
+    read_entry_result result = do_read_entry(
+      pos,
+      &next_pos,
+      &bl,
+      &seq,
+      0,
+      h);
+    if (result == FAILURE || result == MAYBE_CORRUPT)
+      assert(0);
+    if (seq == wanted_seq) {
+      if (_pos)
+	*_pos = pos;
+      return;
+    }
+  }
+  assert(0); // not reachable
+}
+
+void FileJournal::corrupt(
+  int wfd,
+  off64_t corrupt_at)
+{
+  if (corrupt_at >= header.max_size)
+    corrupt_at = corrupt_at + get_top() - header.max_size;
+
+    int64_t actual = ::lseek64(fd, corrupt_at, SEEK_SET);
+    assert(actual == corrupt_at);
+
+    char buf[10];
+    int r = safe_read_exact(fd, buf, 1);
+    assert(r == 0);
+
+    actual = ::lseek64(wfd, corrupt_at, SEEK_SET);
+    assert(actual == corrupt_at);
+
+    buf[0]++;
+    r = safe_write(wfd, buf, 1);
+    assert(r == 0);
+}
+
+void FileJournal::corrupt_payload(
+  int wfd,
+  uint64_t seq)
+{
+  off64_t pos = 0;
+  entry_header_t h;
+  get_header(seq, &pos, &h);
+  off64_t corrupt_at =
+    pos + sizeof(entry_header_t) + h.pre_pad;
+  corrupt(wfd, corrupt_at);
+}
+
+
+void FileJournal::corrupt_footer_magic(
+  int wfd,
+  uint64_t seq)
+{
+  off64_t pos = 0;
+  entry_header_t h;
+  get_header(seq, &pos, &h);
+  off64_t corrupt_at =
+    pos + sizeof(entry_header_t) + h.pre_pad +
+    h.len + h.post_pad +
+    (reinterpret_cast(&h.magic2) - reinterpret_cast(&h));
+  corrupt(wfd, corrupt_at);
+}
+
+
+void FileJournal::corrupt_header_magic(
+  int wfd,
+  uint64_t seq)
+{
+  off64_t pos = 0;
+  entry_header_t h;
+  get_header(seq, &pos, &h);
+  off64_t corrupt_at =
+    pos +
+    (reinterpret_cast(&h.magic2) - reinterpret_cast(&h));
+  corrupt(wfd, corrupt_at);
+}
diff --git a/ceph/src/os/FileJournal.h b/ceph/src/os/FileJournal.h
new file mode 100644
index 00000000..dbb1181b
--- /dev/null
+++ b/ceph/src/os/FileJournal.h
@@ -0,0 +1,479 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_FILEJOURNAL_H
+#define CEPH_FILEJOURNAL_H
+
+#include 
+using std::deque;
+
+#include "Journal.h"
+#include "common/Cond.h"
+#include "common/Mutex.h"
+#include "common/Thread.h"
+#include "common/Throttle.h"
+
+#ifdef HAVE_LIBAIO
+# include 
+#endif
+
+/**
+ * Implements journaling on top of block device or file.
+ *
+ * Lock ordering is write_lock > aio_lock > finisher_lock
+ */
+class FileJournal : public Journal {
+public:
+  /// Protected by finisher_lock
+  struct completion_item {
+    uint64_t seq;
+    Context *finish;
+    utime_t start;
+    TrackedOpRef tracked_op;
+    completion_item(uint64_t o, Context *c, utime_t s,
+		    TrackedOpRef opref)
+      : seq(o), finish(c), start(s), tracked_op(opref) {}
+    completion_item() : seq(0), finish(0), start(0) {}
+  };
+  struct write_item {
+    uint64_t seq;
+    bufferlist bl;
+    int alignment;
+    TrackedOpRef tracked_op;
+    write_item(uint64_t s, bufferlist& b, int al, TrackedOpRef opref) :
+      seq(s), alignment(al), tracked_op(opref) {
+      bl.claim(b);
+    }
+    write_item() : seq(0), alignment(0) {}
+  };
+
+  Mutex finisher_lock;
+  Cond finisher_cond;
+  uint64_t journaled_seq;
+  bool plug_journal_completions;
+
+  Mutex writeq_lock;
+  Cond writeq_cond;
+  deque writeq;
+  bool writeq_empty();
+  write_item &peek_write();
+  void pop_write();
+
+  Mutex completions_lock;
+  deque completions;
+  bool completions_empty() {
+    Mutex::Locker l(completions_lock);
+    return completions.empty();
+  }
+  completion_item completion_peek_front() {
+    Mutex::Locker l(completions_lock);
+    assert(!completions.empty());
+    return completions.front();
+  }
+  void completion_pop_front() {
+    Mutex::Locker l(completions_lock);
+    assert(!completions.empty());
+    completions.pop_front();
+  }
+
+  void submit_entry(uint64_t seq, bufferlist& bl, int alignment,
+		    Context *oncommit,
+		    TrackedOpRef osd_op = TrackedOpRef());
+  /// End protected by finisher_lock
+
+  /*
+   * journal header
+   */
+  struct header_t {
+    enum {
+      FLAG_CRC = (1<<0),
+      // NOTE: remove kludgey weirdness in read_header() next time a flag is added.
+    };
+
+    uint64_t flags;
+    uuid_d fsid;
+    __u32 block_size;
+    __u32 alignment;
+    int64_t max_size;   // max size of journal ring buffer
+    int64_t start;      // offset of first entry
+    uint64_t committed_up_to; // committed up to
+
+    /**
+     * start_seq
+     *
+     * entry at header.start has sequence >= start_seq
+     *
+     * Generally, the entry at header.start will have sequence
+     * start_seq if it exists.  The only exception is immediately
+     * after journal creation since the first sequence number is
+     * not known.
+     *
+     * If the first read on open fails, we can assume corruption
+     * if start_seq > committed_up_thru because the entry would have
+     * a sequence >= start_seq and therefore > committed_up_thru.
+     */
+    uint64_t start_seq;
+
+    header_t() :
+      flags(0), block_size(0), alignment(0), max_size(0), start(0),
+      committed_up_to(0), start_seq(0) {}
+
+    void clear() {
+      start = block_size;
+    }
+
+    uint64_t get_fsid64() {
+      return *(uint64_t*)&fsid.uuid[0];
+    }
+
+    void encode(bufferlist& bl) const {
+      __u32 v = 4;
+      ::encode(v, bl);
+      bufferlist em;
+      {
+	::encode(flags, em);
+	::encode(fsid, em);
+	::encode(block_size, em);
+	::encode(alignment, em);
+	::encode(max_size, em);
+	::encode(start, em);
+	::encode(committed_up_to, em);
+	::encode(start_seq, em);
+      }
+      ::encode(em, bl);
+    }
+    void decode(bufferlist::iterator& bl) {
+      __u32 v;
+      ::decode(v, bl);
+      if (v < 2) {  // normally 0, but concievably 1
+	// decode old header_t struct (pre v0.40).
+	bl.advance(4); // skip __u32 flags (it was unused by any old code)
+	flags = 0;
+	uint64_t tfsid;
+	::decode(tfsid, bl);
+	*(uint64_t*)&fsid.uuid[0] = tfsid;
+	*(uint64_t*)&fsid.uuid[8] = tfsid;
+	::decode(block_size, bl);
+	::decode(alignment, bl);
+	::decode(max_size, bl);
+	::decode(start, bl);
+	committed_up_to = 0;
+	start_seq = 0;
+	return;
+      }
+      bufferlist em;
+      ::decode(em, bl);
+      bufferlist::iterator t = em.begin();
+      ::decode(flags, t);
+      ::decode(fsid, t);
+      ::decode(block_size, t);
+      ::decode(alignment, t);
+      ::decode(max_size, t);
+      ::decode(start, t);
+
+      if (v > 2)
+	::decode(committed_up_to, t);
+      else
+	committed_up_to = 0;
+
+      if (v > 3)
+	::decode(start_seq, t);
+      else
+	start_seq = 0;
+    }
+  } header;
+
+  struct entry_header_t {
+    uint64_t seq;     // fs op seq #
+    uint32_t crc32c;  // payload only.  not header, pre_pad, post_pad, or footer.
+    uint32_t len;
+    uint32_t pre_pad, post_pad;
+    uint64_t magic1;
+    uint64_t magic2;
+    
+    void make_magic(off64_t pos, uint64_t fsid) {
+      magic1 = pos;
+      magic2 = fsid ^ seq ^ len;
+    }
+    bool check_magic(off64_t pos, uint64_t fsid) {
+      return
+	magic1 == (uint64_t)pos &&
+	magic2 == (fsid ^ seq ^ len);
+    }
+  } __attribute__((__packed__, aligned(4)));
+
+private:
+  string fn;
+
+  char *zero_buf;
+
+  off64_t max_size;
+  size_t block_size;
+  bool is_bdev;
+  bool directio, aio, force_aio;
+  bool must_write_header;
+  off64_t write_pos;      // byte where the next entry to be written will go
+  off64_t read_pos;       // 
+
+#ifdef HAVE_LIBAIO
+  /// state associated with an in-flight aio request
+  /// Protected by aio_lock
+  struct aio_info {
+    struct iocb iocb;
+    bufferlist bl;
+    struct iovec *iov;
+    bool done;
+    uint64_t off, len;    ///< these are for debug only
+    uint64_t seq;         ///< seq number to complete on aio completion, if non-zero
+
+    aio_info(bufferlist& b, uint64_t o, uint64_t s)
+      : iov(NULL), done(false), off(o), len(b.length()), seq(s) {
+      bl.claim(b);
+      memset((void*)&iocb, 0, sizeof(iocb));
+    }
+    ~aio_info() {
+      delete[] iov;
+    }
+  };
+  Mutex aio_lock;
+  Cond aio_cond;
+  Cond write_finish_cond;
+  io_context_t aio_ctx;
+  list aio_queue;
+  int aio_num, aio_bytes;
+  /// End protected by aio_lock
+#endif
+
+  uint64_t last_committed_seq;
+  uint64_t journaled_since_start;
+
+  /*
+   * full states cycle at the beginnging of each commit epoch, when commit_start()
+   * is called.
+   *   FULL - we just filled up during this epoch.
+   *   WAIT - we filled up last epoch; now we have to wait until everything during
+   *          that epoch commits to the fs before we can start writing over it.
+   *   NOTFULL - all good, journal away.
+   */
+  enum {
+    FULL_NOTFULL = 0,
+    FULL_FULL = 1,
+    FULL_WAIT = 2,
+  } full_state;
+
+  int fd;
+
+  // in journal
+  deque > journalq;  // track seq offsets, so we can trim later.
+  uint64_t writing_seq;
+
+  
+  // throttle
+  Throttle throttle_ops, throttle_bytes;
+
+  void put_throttle(uint64_t ops, uint64_t bytes);
+
+  // write thread
+  Mutex write_lock;
+  bool write_stop;
+
+  Cond commit_cond;
+
+  int _open(bool wr, bool create=false);
+  int _open_block_device();
+  void _check_disk_write_cache() const;
+  int _open_file(int64_t oldsize, blksize_t blksize, bool create);
+  void print_header();
+  int read_header();
+  bufferptr prepare_header();
+  void start_writer();
+  void stop_writer();
+  void write_thread_entry();
+
+  void queue_completions_thru(uint64_t seq);
+
+  int check_for_full(uint64_t seq, off64_t pos, off64_t size);
+  int prepare_multi_write(bufferlist& bl, uint64_t& orig_ops, uint64_t& orig_bytee);
+  int prepare_single_write(bufferlist& bl, off64_t& queue_pos, uint64_t& orig_ops, uint64_t& orig_bytes);
+  void do_write(bufferlist& bl);
+
+  void write_finish_thread_entry();
+  void check_aio_completion();
+  void do_aio_write(bufferlist& bl);
+  int write_aio_bl(off64_t& pos, bufferlist& bl, uint64_t seq);
+
+
+  void align_bl(off64_t pos, bufferlist& bl);
+  int write_bl(off64_t& pos, bufferlist& bl);
+
+  /// read len from journal starting at in_pos and wrapping up to len
+  void wrap_read_bl(
+    off64_t in_pos,   ///< [in] start position
+    int64_t len,      ///< [in] length to read
+    bufferlist* bl,   ///< [out] result
+    off64_t *out_pos  ///< [out] next position to read, will be wrapped
+    );
+
+  class Writer : public Thread {
+    FileJournal *journal;
+  public:
+    Writer(FileJournal *fj) : journal(fj) {}
+    void *entry() {
+      journal->write_thread_entry();
+      return 0;
+    }
+  } write_thread;
+
+  class WriteFinisher : public Thread {
+    FileJournal *journal;
+  public:
+    WriteFinisher(FileJournal *fj) : journal(fj) {}
+    void *entry() {
+      journal->write_finish_thread_entry();
+      return 0;
+    }
+  } write_finish_thread;
+
+  off64_t get_top() {
+    return ROUND_UP_TO(sizeof(header), block_size);
+  }
+
+ public:
+  FileJournal(uuid_d fsid, Finisher *fin, Cond *sync_cond, const char *f, bool dio=false, bool ai=true, bool faio=false) :
+    Journal(fsid, fin, sync_cond),
+    finisher_lock("FileJournal::finisher_lock", false, true, false, g_ceph_context),
+    journaled_seq(0),
+    plug_journal_completions(false),
+    writeq_lock("FileJournal::writeq_lock", false, true, false, g_ceph_context),
+    completions_lock(
+      "FileJournal::completions_lock", false, true, false, g_ceph_context),
+    fn(f),
+    zero_buf(NULL),
+    max_size(0), block_size(0),
+    is_bdev(false), directio(dio), aio(ai), force_aio(faio),
+    must_write_header(false),
+    write_pos(0), read_pos(0),
+#ifdef HAVE_LIBAIO
+    aio_lock("FileJournal::aio_lock"),
+    aio_ctx(0),
+    aio_num(0), aio_bytes(0),
+#endif
+    last_committed_seq(0), 
+    journaled_since_start(0),
+    full_state(FULL_NOTFULL),
+    fd(-1),
+    writing_seq(0),
+    throttle_ops(g_ceph_context, "filestore_ops"),
+    throttle_bytes(g_ceph_context, "filestore_bytes"),
+    write_lock("FileJournal::write_lock", false, true, false, g_ceph_context),
+    write_stop(false),
+    write_thread(this),
+    write_finish_thread(this) { }
+  ~FileJournal() {
+    delete[] zero_buf;
+  }
+
+  int check();
+  int create();
+  int open(uint64_t fs_op_seq);
+  void close();
+  int peek_fsid(uuid_d& fsid);
+
+  int dump(ostream& out);
+
+  void flush();
+
+  void throttle();
+
+  bool is_writeable() {
+    return read_pos == 0;
+  }
+  int make_writeable();
+
+  // writes
+  void commit_start(uint64_t seq);
+  void committed_thru(uint64_t seq);
+  bool should_commit_now() {
+    return full_state != FULL_NOTFULL;
+  }
+
+  void set_wait_on_full(bool b) { wait_on_full = b; }
+
+  // reads
+
+  /// Result code for read_entry
+  enum read_entry_result {
+    SUCCESS,
+    FAILURE,
+    MAYBE_CORRUPT
+  };
+
+  /**
+   * read_entry
+   *
+   * Reads next entry starting at pos.  If the entry appears
+   * clean, *bl will contain the payload, *seq will contain
+   * the sequence number, and *out_pos will reflect the next
+   * read position.  If the entry is invalid *ss will contain
+   * debug text, while *seq, *out_pos, and *bl will be unchanged.
+   *
+   * If the entry suggests a corrupt log, *ss will contain debug
+   * text, *out_pos will contain the next index to check.  If
+   * we find an entry in this way that returns SUCCESS, the journal
+   * is most likely corrupt.
+   */
+  read_entry_result do_read_entry(
+    off64_t pos,          ///< [in] position to read
+    off64_t *next_pos,    ///< [out] next position to read
+    bufferlist* bl,       ///< [out] payload for successful read
+    uint64_t *seq,        ///< [out] seq of successful read
+    ostream *ss,          ///< [out] error output
+    entry_header_t *h = 0 ///< [out] header
+    ); ///< @return result code
+
+  bool read_entry(
+    bufferlist &bl,
+    uint64_t &last_seq,
+    bool *corrupt
+    );
+
+  bool read_entry(
+    bufferlist &bl,
+    uint64_t &last_seq) {
+    return read_entry(bl, last_seq, 0);
+  }
+
+  // Debug/Testing
+  void get_header(
+    uint64_t wanted_seq,
+    off64_t *_pos,
+    entry_header_t *h);
+  void corrupt(
+    int wfd,
+    off64_t corrupt_at);
+  void corrupt_payload(
+    int wfd,
+    uint64_t seq);
+  void corrupt_footer_magic(
+    int wfd,
+    uint64_t seq);
+  void corrupt_header_magic(
+    int wfd,
+    uint64_t seq);
+};
+
+WRITE_CLASS_ENCODER(FileJournal::header_t)
+
+#endif
diff --git a/ceph/src/os/FileStore.cc b/ceph/src/os/FileStore.cc
new file mode 100644
index 00000000..1b3dd5ec
--- /dev/null
+++ b/ceph/src/os/FileStore.cc
@@ -0,0 +1,4964 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#if defined(__linux__)
+#include 
+#endif
+
+#include 
+#include 
+
+#include "include/compat.h"
+#include "include/linux_fiemap.h"
+
+#include "common/xattr.h"
+#include "chain_xattr.h"
+
+#if defined(DARWIN) || defined(__FreeBSD__)
+#include 
+#include 
+#endif // DARWIN
+
+
+#include 
+#include 
+
+#include "FileStore.h"
+#include "GenericFileStoreBackend.h"
+#include "BtrfsFileStoreBackend.h"
+#include "XfsFileStoreBackend.h"
+#include "ZFSFileStoreBackend.h"
+#include "common/BackTrace.h"
+#include "include/types.h"
+#include "FileJournal.h"
+
+#include "osd/osd_types.h"
+#include "include/color.h"
+#include "include/buffer.h"
+
+#include "common/Timer.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/run_cmd.h"
+#include "common/safe_io.h"
+#include "common/perf_counters.h"
+#include "common/sync_filesystem.h"
+#include "common/fd.h"
+#include "HashIndex.h"
+#include "DBObjectMap.h"
+#include "LevelDBStore.h"
+
+#include "common/ceph_crypto.h"
+using ceph::crypto::SHA1;
+
+#include "include/assert.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "filestore(" << basedir << ") "
+
+#define COMMIT_SNAP_ITEM "snap_%lld"
+#define CLUSTER_SNAP_ITEM "clustersnap_%s"
+
+#define REPLAY_GUARD_XATTR "user.cephos.seq"
+#define GLOBAL_REPLAY_GUARD_XATTR "user.cephos.gseq"
+
+// XATTR_SPILL_OUT_NAME as a xattr is used to maintain that indicates whether
+// xattrs spill over into DBObjectMap, if XATTR_SPILL_OUT_NAME exists in file
+// xattrs and the value is "no", it indicates no xattrs in DBObjectMap
+#define XATTR_SPILL_OUT_NAME "user.cephos.spill_out"
+#define XATTR_NO_SPILL_OUT "0"
+#define XATTR_SPILL_OUT "1"
+
+//Initial features in new superblock.
+static CompatSet get_fs_initial_compat_set() {
+  CompatSet::FeatureSet ceph_osd_feature_compat;
+  CompatSet::FeatureSet ceph_osd_feature_ro_compat;
+  CompatSet::FeatureSet ceph_osd_feature_incompat;
+  return CompatSet(ceph_osd_feature_compat, ceph_osd_feature_ro_compat,
+		   ceph_osd_feature_incompat);
+}
+
+//Features are added here that this FileStore supports.
+static CompatSet get_fs_supported_compat_set() {
+  CompatSet compat =  get_fs_initial_compat_set();
+  //Any features here can be set in code, but not in initial superblock
+  compat.incompat.insert(CEPH_FS_FEATURE_INCOMPAT_SHARDS);
+  return compat;
+}
+
+
+int FileStore::peek_journal_fsid(uuid_d *fsid)
+{
+  // make sure we don't try to use aio or direct_io (and get annoying
+  // error messages from failing to do so); performance implications
+  // should be irrelevant for this use
+  FileJournal j(*fsid, 0, 0, journalpath.c_str(), false, false);
+  return j.peek_fsid(*fsid);
+}
+
+void FileStore::FSPerfTracker::update_from_perfcounters(
+  PerfCounters &logger)
+{
+  os_commit_latency.consume_next(
+    logger.get_tavg_ms(
+      l_os_j_lat));
+  os_apply_latency.consume_next(
+    logger.get_tavg_ms(
+      l_os_apply_lat));
+}
+
+
+ostream& operator<<(ostream& out, const FileStore::OpSequencer& s)
+{
+  assert(&out);
+  return out << *s.parent;
+}
+
+int FileStore::get_cdir(coll_t cid, char *s, int len) 
+{
+  const string &cid_str(cid.to_str());
+  return snprintf(s, len, "%s/current/%s", basedir.c_str(), cid_str.c_str());
+}
+
+int FileStore::get_index(coll_t cid, Index *index)
+{
+  char path[PATH_MAX];
+  get_cdir(cid, path, sizeof(path));
+  int r = index_manager.get_index(cid, path, index);
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::init_index(coll_t cid)
+{
+  char path[PATH_MAX];
+  get_cdir(cid, path, sizeof(path));
+  int r = index_manager.init_index(cid, path, target_version);
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::lfn_find(coll_t cid, const ghobject_t& oid, IndexedPath *path)
+{
+  Index index; 
+  int r, exist;
+  r = get_index(cid, &index);
+  if (r < 0)
+    return r;
+
+  r = index->lookup(oid, path, &exist);
+  if (r < 0) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  if (!exist)
+    return -ENOENT;
+  return 0;
+}
+
+int FileStore::lfn_truncate(coll_t cid, const ghobject_t& oid, off_t length)
+{
+  IndexedPath path;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd, &path);
+  if (r < 0)
+    return r;
+  r = ::ftruncate(**fd, length);
+  if (r < 0)
+    r = -errno;
+  if (r >= 0 && m_filestore_sloppy_crc) {
+    int rc = backend->_crc_update_truncate(**fd, length);
+    assert(rc >= 0);
+  }
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::lfn_stat(coll_t cid, const ghobject_t& oid, struct stat *buf)
+{
+  IndexedPath path;
+  int r = lfn_find(cid, oid, &path);
+  if (r < 0)
+    return r;
+  r = ::stat(path->path(), buf);
+  if (r < 0)
+    r = -errno;
+  return r;
+}
+
+int FileStore::lfn_open(coll_t cid,
+			const ghobject_t& oid,
+			bool create,
+			FDRef *outfd,
+			IndexedPath *path,
+			Index *index) 
+{
+  assert(get_allow_sharded_objects() ||
+	 ( oid.shard_id == ghobject_t::NO_SHARD &&
+	   oid.generation == ghobject_t::NO_GEN ));
+  assert(outfd);
+  int flags = O_RDWR;
+  if (create)
+    flags |= O_CREAT;
+  Index index2;
+  if (!index) {
+    index = &index2;
+  }
+  int r = 0;
+  if (!(*index)) {
+    r = get_index(cid, index);
+  }
+
+  int fd, exist;
+  if (!replaying) {
+    Mutex::Locker l(fdcache_lock);
+    *outfd = fdcache.lookup(oid);
+    if (*outfd)
+      return 0;
+  }
+
+  {
+    IndexedPath path2;
+    if (!path)
+      path = &path2;
+    if (r < 0) {
+      derr << "error getting collection index for " << cid
+	   << ": " << cpp_strerror(-r) << dendl;
+      goto fail;
+    }
+    r = (*index)->lookup(oid, path, &exist);
+    if (r < 0) {
+      derr << "could not find " << oid << " in index: "
+	   << cpp_strerror(-r) << dendl;
+      goto fail;
+    }
+
+    r = ::open((*path)->path(), flags, 0644);
+    if (r < 0) {
+      r = -errno;
+      dout(10) << "error opening file " << (*path)->path() << " with flags="
+	       << flags << ": " << cpp_strerror(-r) << dendl;
+      goto fail;
+    }
+    fd = r;
+
+    if (create && (!exist)) {
+      r = (*index)->created(oid, (*path)->path());
+      if (r < 0) {
+	VOID_TEMP_FAILURE_RETRY(::close(fd));
+	derr << "error creating " << oid << " (" << (*path)->path()
+	     << ") in index: " << cpp_strerror(-r) << dendl;
+	goto fail;
+      }
+      r = chain_fsetxattr(fd, XATTR_SPILL_OUT_NAME,
+                          XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT));
+      if (r < 0) {
+        VOID_TEMP_FAILURE_RETRY(::close(fd));
+        derr << "error setting spillout xattr for oid " << oid << " (" << (*path)->path()
+                       << "):" << cpp_strerror(-r) << dendl;
+        goto fail;
+      }
+    }
+  }
+
+  if (!replaying) {
+    Mutex::Locker l(fdcache_lock);
+    *outfd = fdcache.lookup(oid);
+    if (*outfd) {
+      VOID_TEMP_FAILURE_RETRY(::close(fd));
+      return 0;
+    } else {
+      *outfd = fdcache.add(oid, fd);
+    }
+  } else {
+    *outfd = FDRef(new FDCache::FD(fd));
+  }
+  return 0;
+
+ fail:
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+void FileStore::lfn_close(FDRef fd)
+{
+}
+
+int FileStore::lfn_link(coll_t c, coll_t newcid, const ghobject_t& o, const ghobject_t& newoid)
+{
+  Index index_new, index_old;
+  IndexedPath path_new, path_old;
+  int exist;
+  int r;
+  if (c < newcid) {
+    r = get_index(newcid, &index_new);
+    if (r < 0)
+      return r;
+    r = get_index(c, &index_old);
+    if (r < 0)
+      return r;
+  } else if (c == newcid) {
+    r = get_index(c, &index_old);
+    if (r < 0)
+      return r;
+    index_new = index_old;
+  } else {
+    r = get_index(c, &index_old);
+    if (r < 0)
+      return r;
+    r = get_index(newcid, &index_new);
+    if (r < 0)
+      return r;
+  }
+
+  r = index_old->lookup(o, &path_old, &exist);
+  if (r < 0) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  if (!exist)
+    return -ENOENT;
+
+  r = index_new->lookup(newoid, &path_new, &exist);
+  if (r < 0) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  if (exist)
+    return -EEXIST;
+
+  dout(25) << "lfn_link path_old: " << path_old << dendl;
+  dout(25) << "lfn_link path_new: " << path_new << dendl;
+  r = ::link(path_old->path(), path_new->path());
+  if (r < 0)
+    return -errno;
+
+  r = index_new->created(newoid, path_new->path());
+  if (r < 0) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  return 0;
+}
+
+int FileStore::lfn_unlink(coll_t cid, const ghobject_t& o,
+			  const SequencerPosition &spos,
+			  bool force_clear_omap)
+{
+  Index index;
+  int r = get_index(cid, &index);
+  if (r < 0)
+    return r;
+  Mutex::Locker l(fdcache_lock);
+  {
+    IndexedPath path;
+    int exist;
+    r = index->lookup(o, &path, &exist);
+    if (r < 0) {
+      assert(!m_filestore_fail_eio || r != -EIO);
+      return r;
+    }
+
+    if (!force_clear_omap) {
+      struct stat st;
+      r = ::stat(path->path(), &st);
+      if (r < 0) {
+	r = -errno;
+	assert(!m_filestore_fail_eio || r != -EIO);
+	return r;
+      }
+      if (st.st_nlink == 1)
+	force_clear_omap = true;
+    }
+    if (force_clear_omap) {
+      dout(20) << __func__ << ": clearing omap on " << o
+	       << " in cid " << cid << dendl;
+      r = object_map->clear(o, &spos);
+      if (r < 0 && r != -ENOENT) {
+	assert(!m_filestore_fail_eio || r != -EIO);
+	return r;
+      }
+      if (g_conf->filestore_debug_inject_read_err) {
+	debug_obj_on_delete(o);
+      }
+      wbthrottle.clear_object(o); // should be only non-cache ref
+      fdcache.clear(o);
+    } else {
+      /* Ensure that replay of this op doesn't result in the object_map
+       * going away.
+       */
+      if (!backend->can_checkpoint())
+	object_map->sync(&o, &spos);
+    }
+  }
+  return index->unlink(o);
+}
+
+FileStore::FileStore(const std::string &base, const std::string &jdev, const char *name, bool do_update) :
+  JournalingObjectStore(base),
+  internal_name(name),
+  basedir(base), journalpath(jdev),
+  blk_size(0),
+  fsid_fd(-1), op_fd(-1),
+  basedir_fd(-1), current_fd(-1),
+  generic_backend(NULL), backend(NULL),
+  index_manager(do_update),
+  ondisk_finisher(g_ceph_context),
+  lock("FileStore::lock"),
+  force_sync(false), sync_epoch(0),
+  sync_entry_timeo_lock("sync_entry_timeo_lock"),
+  timer(g_ceph_context, sync_entry_timeo_lock),
+  stop(false), sync_thread(this),
+  fdcache_lock("fdcache_lock"),
+  fdcache(g_ceph_context),
+  wbthrottle(g_ceph_context),
+  default_osr("default"),
+  op_queue_len(0), op_queue_bytes(0),
+  op_throttle_lock("FileStore::op_throttle_lock"),
+  op_finisher(g_ceph_context),
+  op_tp(g_ceph_context, "FileStore::op_tp", g_conf->filestore_op_threads, "filestore_op_threads"),
+  op_wq(this, g_conf->filestore_op_thread_timeout,
+	g_conf->filestore_op_thread_suicide_timeout, &op_tp),
+  logger(NULL),
+  read_error_lock("FileStore::read_error_lock"),
+  m_filestore_commit_timeout(g_conf->filestore_commit_timeout),
+  m_filestore_journal_parallel(g_conf->filestore_journal_parallel ),
+  m_filestore_journal_trailing(g_conf->filestore_journal_trailing),
+  m_filestore_journal_writeahead(g_conf->filestore_journal_writeahead),
+  m_filestore_fiemap_threshold(g_conf->filestore_fiemap_threshold),
+  m_filestore_max_sync_interval(g_conf->filestore_max_sync_interval),
+  m_filestore_min_sync_interval(g_conf->filestore_min_sync_interval),
+  m_filestore_fail_eio(g_conf->filestore_fail_eio),
+  m_filestore_replica_fadvise(g_conf->filestore_replica_fadvise),
+  do_update(do_update),
+  m_journal_dio(g_conf->journal_dio),
+  m_journal_aio(g_conf->journal_aio),
+  m_journal_force_aio(g_conf->journal_force_aio),
+  m_osd_rollback_to_cluster_snap(g_conf->osd_rollback_to_cluster_snap),
+  m_osd_use_stale_snap(g_conf->osd_use_stale_snap),
+  m_filestore_queue_max_ops(g_conf->filestore_queue_max_ops),
+  m_filestore_queue_max_bytes(g_conf->filestore_queue_max_bytes),
+  m_filestore_queue_committing_max_ops(g_conf->filestore_queue_committing_max_ops),
+  m_filestore_queue_committing_max_bytes(g_conf->filestore_queue_committing_max_bytes),
+  m_filestore_do_dump(false),
+  m_filestore_dump_fmt(true),
+  m_filestore_sloppy_crc(g_conf->filestore_sloppy_crc),
+  m_filestore_sloppy_crc_block_size(g_conf->filestore_sloppy_crc_block_size),
+  m_filestore_max_alloc_hint_size(g_conf->filestore_max_alloc_hint_size),
+  m_fs_type(FS_TYPE_NONE),
+  m_filestore_max_inline_xattr_size(0),
+  m_filestore_max_inline_xattrs(0)
+{
+  m_filestore_kill_at.set(g_conf->filestore_kill_at);
+
+  ostringstream oss;
+  oss << basedir << "/current";
+  current_fn = oss.str();
+
+  ostringstream sss;
+  sss << basedir << "/current/commit_op_seq";
+  current_op_seq_fn = sss.str();
+
+  ostringstream omss;
+  omss << basedir << "/current/omap";
+  omap_dir = omss.str();
+
+  // initialize logger
+  PerfCountersBuilder plb(g_ceph_context, internal_name, l_os_first, l_os_last);
+
+  plb.add_u64(l_os_jq_max_ops, "journal_queue_max_ops");
+  plb.add_u64(l_os_jq_ops, "journal_queue_ops");
+  plb.add_u64_counter(l_os_j_ops, "journal_ops");
+  plb.add_u64(l_os_jq_max_bytes, "journal_queue_max_bytes");
+  plb.add_u64(l_os_jq_bytes, "journal_queue_bytes");
+  plb.add_u64_counter(l_os_j_bytes, "journal_bytes");
+  plb.add_time_avg(l_os_j_lat, "journal_latency");
+  plb.add_u64_counter(l_os_j_wr, "journal_wr");
+  plb.add_u64_avg(l_os_j_wr_bytes, "journal_wr_bytes");
+  plb.add_u64(l_os_oq_max_ops, "op_queue_max_ops");
+  plb.add_u64(l_os_oq_ops, "op_queue_ops");
+  plb.add_u64_counter(l_os_ops, "ops");
+  plb.add_u64(l_os_oq_max_bytes, "op_queue_max_bytes");
+  plb.add_u64(l_os_oq_bytes, "op_queue_bytes");
+  plb.add_u64_counter(l_os_bytes, "bytes");
+  plb.add_time_avg(l_os_apply_lat, "apply_latency");
+  plb.add_u64(l_os_committing, "committing");
+
+  plb.add_u64_counter(l_os_commit, "commitcycle");
+  plb.add_time_avg(l_os_commit_len, "commitcycle_interval");
+  plb.add_time_avg(l_os_commit_lat, "commitcycle_latency");
+  plb.add_u64_counter(l_os_j_full, "journal_full");
+  plb.add_time_avg(l_os_queue_lat, "queue_transaction_latency_avg");
+
+  logger = plb.create_perf_counters();
+
+  g_ceph_context->get_perfcounters_collection()->add(logger);
+  g_ceph_context->_conf->add_observer(this);
+
+  generic_backend = new GenericFileStoreBackend(this);
+  backend = generic_backend;
+
+  superblock.compat_features = get_fs_initial_compat_set();
+}
+
+FileStore::~FileStore()
+{
+  g_ceph_context->_conf->remove_observer(this);
+  g_ceph_context->get_perfcounters_collection()->remove(logger);
+
+  delete generic_backend;
+
+  if (journal)
+    journal->logger = NULL;
+  delete logger;
+
+  if (m_filestore_do_dump) {
+    dump_stop();
+  }
+}
+
+static void get_attrname(const char *name, char *buf, int len)
+{
+  snprintf(buf, len, "user.ceph.%s", name);
+}
+
+bool parse_attrname(char **name)
+{
+  if (strncmp(*name, "user.ceph.", 10) == 0) {
+    *name += 10;
+    return true;
+  }
+  return false;
+}
+
+int FileStore::statfs(struct statfs *buf)
+{
+  if (::statfs(basedir.c_str(), buf) < 0) {
+    int r = -errno;
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  return 0;
+}
+
+
+int FileStore::open_journal()
+{
+  if (journalpath.length()) {
+    dout(10) << "open_journal at " << journalpath << dendl;
+    journal = new FileJournal(fsid, &finisher, &sync_cond, journalpath.c_str(),
+			      m_journal_dio, m_journal_aio, m_journal_force_aio);
+    if (journal)
+      journal->logger = logger;
+  }
+  return 0;
+}
+
+int FileStore::dump_journal(ostream& out)
+{
+  int r;
+
+  if (!journalpath.length())
+    return -EINVAL;
+
+  FileJournal *journal = new FileJournal(fsid, &finisher, &sync_cond, journalpath.c_str(), m_journal_dio);
+  r = journal->dump(out);
+  delete journal;
+  return r;
+}
+
+int FileStore::mkfs()
+{
+  int ret = 0;
+  char fsid_fn[PATH_MAX];
+  uuid_d old_fsid;
+
+  dout(1) << "mkfs in " << basedir << dendl;
+  basedir_fd = ::open(basedir.c_str(), O_RDONLY);
+  if (basedir_fd < 0) {
+    ret = -errno;
+    derr << "mkfs failed to open base dir " << basedir << ": " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  // open+lock fsid
+  snprintf(fsid_fn, sizeof(fsid_fn), "%s/fsid", basedir.c_str());
+  fsid_fd = ::open(fsid_fn, O_RDWR|O_CREAT, 0644);
+  if (fsid_fd < 0) {
+    ret = -errno;
+    derr << "mkfs: failed to open " << fsid_fn << ": " << cpp_strerror(ret) << dendl;
+    goto close_basedir_fd;
+  }
+
+  if (lock_fsid() < 0) {
+    ret = -EBUSY;
+    goto close_fsid_fd;
+  }
+
+  if (read_fsid(fsid_fd, &old_fsid) < 0 || old_fsid.is_zero()) {
+    if (fsid.is_zero()) {
+      fsid.generate_random();
+      dout(1) << "mkfs generated fsid " << fsid << dendl;
+    } else {
+      dout(1) << "mkfs using provided fsid " << fsid << dendl;
+    }
+
+    char fsid_str[40];
+    fsid.print(fsid_str);
+    strcat(fsid_str, "\n");
+    ret = ::ftruncate(fsid_fd, 0);
+    if (ret < 0) {
+      ret = -errno;
+      derr << "mkfs: failed to truncate fsid: "
+	   << cpp_strerror(ret) << dendl;
+      goto close_fsid_fd;
+    }
+    ret = safe_write(fsid_fd, fsid_str, strlen(fsid_str));
+    if (ret < 0) {
+      derr << "mkfs: failed to write fsid: "
+	   << cpp_strerror(ret) << dendl;
+      goto close_fsid_fd;
+    }
+    if (::fsync(fsid_fd) < 0) {
+      ret = errno;
+      derr << "mkfs: close failed: can't write fsid: "
+	   << cpp_strerror(ret) << dendl;
+      goto close_fsid_fd;
+    }
+    dout(10) << "mkfs fsid is " << fsid << dendl;
+  } else {
+    if (!fsid.is_zero() && fsid != old_fsid) {
+      derr << "mkfs on-disk fsid " << old_fsid << " != provided " << fsid << dendl;
+      ret = -EINVAL;
+      goto close_fsid_fd;
+    }
+    fsid = old_fsid;
+    dout(1) << "mkfs fsid is already set to " << fsid << dendl;
+  }
+
+  // version stamp
+  ret = write_version_stamp();
+  if (ret < 0) {
+    derr << "mkfs: write_version_stamp() failed: "
+	 << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  ret = write_superblock();
+  if (ret < 0) {
+    derr << "mkfs: write_superblock() failed: "
+	 << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  struct statfs basefs;
+  ret = ::fstatfs(basedir_fd, &basefs);
+  if (ret < 0) {
+    ret = -errno;
+    derr << "mkfs cannot fstatfs basedir "
+	 << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  if (basefs.f_type == BTRFS_SUPER_MAGIC) {
+#if defined(__linux__)
+    backend = new BtrfsFileStoreBackend(this);
+#endif
+  } else if (basefs.f_type == XFS_SUPER_MAGIC) {
+#ifdef HAVE_LIBXFS
+    backend = new XfsFileStoreBackend(this);
+#endif
+  } else if (basefs.f_type == ZFS_SUPER_MAGIC) {
+#ifdef HAVE_LIBZFS
+    backend = new ZFSFileStoreBackend(this);
+#endif
+  }
+
+  ret = backend->create_current();
+  if (ret < 0) {
+    derr << "mkfs: failed to create current/ " << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  // write initial op_seq
+  {
+    uint64_t initial_seq = 0;
+    int fd = read_op_seq(&initial_seq);
+    if (fd < 0) {
+      derr << "mkfs: failed to create " << current_op_seq_fn << ": "
+	   << cpp_strerror(fd) << dendl;
+      goto close_fsid_fd;
+    }
+    if (initial_seq == 0) {
+      int err = write_op_seq(fd, 1);
+      if (err < 0) {
+	VOID_TEMP_FAILURE_RETRY(::close(fd));
+	derr << "mkfs: failed to write to " << current_op_seq_fn << ": "
+	     << cpp_strerror(err) << dendl;
+	goto close_fsid_fd;
+      }
+
+      if (backend->can_checkpoint()) {
+	// create snap_1 too
+	current_fd = ::open(current_fn.c_str(), O_RDONLY);
+	assert(current_fd >= 0);
+	char s[NAME_MAX];
+	snprintf(s, sizeof(s), COMMIT_SNAP_ITEM, 1ull);
+	ret = backend->create_checkpoint(s, NULL);
+	VOID_TEMP_FAILURE_RETRY(::close(current_fd));
+	if (ret < 0 && ret != -EEXIST) {
+	  VOID_TEMP_FAILURE_RETRY(::close(fd));  
+	  derr << "mkfs: failed to create snap_1: " << cpp_strerror(ret) << dendl;
+	  goto close_fsid_fd;
+	}
+      }
+    }
+    VOID_TEMP_FAILURE_RETRY(::close(fd));  
+  }
+
+  {
+    leveldb::Options options;
+    options.create_if_missing = true;
+    leveldb::DB *db;
+    leveldb::Status status = leveldb::DB::Open(options, omap_dir, &db);
+    if (status.ok()) {
+      delete db;
+      dout(1) << "leveldb db exists/created" << dendl;
+    } else {
+      derr << "mkfs failed to create leveldb: " << status.ToString() << dendl;
+      ret = -1;
+      goto close_fsid_fd;
+    }
+  }
+
+  // journal?
+  ret = mkjournal();
+  if (ret)
+    goto close_fsid_fd;
+
+  dout(1) << "mkfs done in " << basedir << dendl;
+  ret = 0;
+
+ close_fsid_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+  fsid_fd = -1;
+ close_basedir_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(basedir_fd));
+  if (backend != generic_backend) {
+    delete backend;
+    backend = generic_backend;
+  }
+  return ret;
+}
+
+int FileStore::mkjournal()
+{
+  // read fsid
+  int ret;
+  char fn[PATH_MAX];
+  snprintf(fn, sizeof(fn), "%s/fsid", basedir.c_str());
+  int fd = ::open(fn, O_RDONLY, 0644);
+  if (fd < 0) {
+    int err = errno;
+    derr << "FileStore::mkjournal: open error: " << cpp_strerror(err) << dendl;
+    return -err;
+  }
+  ret = read_fsid(fd, &fsid);
+  if (ret < 0) {
+    derr << "FileStore::mkjournal: read error: " << cpp_strerror(ret) << dendl;
+    VOID_TEMP_FAILURE_RETRY(::close(fd));
+    return ret;
+  }
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+
+  ret = 0;
+
+  open_journal();
+  if (journal) {
+    ret = journal->check();
+    if (ret < 0) {
+      ret = journal->create();
+      if (ret)
+	derr << "mkjournal error creating journal on " << journalpath
+		<< ": " << cpp_strerror(ret) << dendl;
+      else
+	dout(0) << "mkjournal created journal on " << journalpath << dendl;
+    }
+    delete journal;
+    journal = 0;
+  }
+  return ret;
+}
+
+int FileStore::read_fsid(int fd, uuid_d *uuid)
+{
+  char fsid_str[40];
+  int ret = safe_read(fd, fsid_str, sizeof(fsid_str));
+  if (ret < 0)
+    return ret;
+  if (ret == 8) {
+    // old 64-bit fsid... mirror it.
+    *(uint64_t*)&uuid->uuid[0] = *(uint64_t*)fsid_str;
+    *(uint64_t*)&uuid->uuid[8] = *(uint64_t*)fsid_str;
+    return 0;
+  }
+
+  if (ret > 36)
+    fsid_str[36] = 0;
+  if (!uuid->parse(fsid_str))
+    return -EINVAL;
+  return 0;
+}
+
+int FileStore::lock_fsid()
+{
+  struct flock l;
+  memset(&l, 0, sizeof(l));
+  l.l_type = F_WRLCK;
+  l.l_whence = SEEK_SET;
+  l.l_start = 0;
+  l.l_len = 0;
+  int r = ::fcntl(fsid_fd, F_SETLK, &l);
+  if (r < 0) {
+    int err = errno;
+    dout(0) << "lock_fsid failed to lock " << basedir << "/fsid, is another ceph-osd still running? "
+	    << cpp_strerror(err) << dendl;
+    return -err;
+  }
+  return 0;
+}
+
+bool FileStore::test_mount_in_use()
+{
+  dout(5) << "test_mount basedir " << basedir << " journal " << journalpath << dendl;
+  char fn[PATH_MAX];
+  snprintf(fn, sizeof(fn), "%s/fsid", basedir.c_str());
+
+  // verify fs isn't in use
+
+  fsid_fd = ::open(fn, O_RDWR, 0644);
+  if (fsid_fd < 0)
+    return 0;   // no fsid, ok.
+  bool inuse = lock_fsid() < 0;
+  VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+  fsid_fd = -1;
+  return inuse;
+}
+
+int FileStore::_detect_fs()
+{
+  struct statfs st;
+  int r = ::fstatfs(basedir_fd, &st);
+  if (r < 0)
+    return -errno;
+
+  blk_size = st.f_bsize;
+
+  m_fs_type = FS_TYPE_OTHER;
+  if (st.f_type == BTRFS_SUPER_MAGIC) {
+#if defined(__linux__)
+    dout(0) << "mount detected btrfs" << dendl;
+    backend = new BtrfsFileStoreBackend(this);
+    m_fs_type = FS_TYPE_BTRFS;
+
+    wbthrottle.set_fs(WBThrottle::BTRFS);
+#endif
+  } else if (st.f_type == XFS_SUPER_MAGIC) {
+#ifdef HAVE_LIBXFS
+    dout(0) << "mount detected xfs (libxfs)" << dendl;
+    backend = new XfsFileStoreBackend(this);
+#else
+    dout(0) << "mount detected xfs" << dendl;
+#endif
+    m_fs_type = FS_TYPE_XFS;
+
+    // wbthrottle is constructed with fs(WBThrottle::XFS)
+    if (m_filestore_replica_fadvise) {
+      dout(1) << " disabling 'filestore replica fadvise' due to known issues with fadvise(DONTNEED) on xfs" << dendl;
+      g_conf->set_val("filestore_replica_fadvise", "false");
+      g_conf->apply_changes(NULL);
+      assert(m_filestore_replica_fadvise == false);
+    }
+  } else if (st.f_type == ZFS_SUPER_MAGIC) {
+#ifdef HAVE_LIBZFS
+    dout(0) << "mount detected zfs (libzfs)" << dendl;
+    backend = new ZFSFileStoreBackend(this);
+    m_fs_type = FS_TYPE_ZFS;
+#endif
+  }
+
+  set_xattr_limits_via_conf();
+
+  r = backend->detect_features();
+  if (r < 0) {
+    derr << "_detect_fs: detect_features error: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  // test xattrs
+  char fn[PATH_MAX];
+  int x = rand();
+  int y = x+1;
+  snprintf(fn, sizeof(fn), "%s/xattr_test", basedir.c_str());
+  int tmpfd = ::open(fn, O_CREAT|O_WRONLY|O_TRUNC, 0700);
+  if (tmpfd < 0) {
+    int ret = -errno;
+    derr << "_detect_fs unable to create " << fn << ": " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  int ret = chain_fsetxattr(tmpfd, "user.test", &x, sizeof(x));
+  if (ret >= 0)
+    ret = chain_fgetxattr(tmpfd, "user.test", &y, sizeof(y));
+  if ((ret < 0) || (x != y)) {
+    derr << "Extended attributes don't appear to work. ";
+    if (ret)
+      *_dout << "Got error " + cpp_strerror(ret) + ". ";
+    *_dout << "If you are using ext3 or ext4, be sure to mount the underlying "
+	   << "file system with the 'user_xattr' option." << dendl;
+    ::unlink(fn);
+    VOID_TEMP_FAILURE_RETRY(::close(tmpfd));
+    return -ENOTSUP;
+  }
+
+  char buf[1000];
+  memset(buf, 0, sizeof(buf)); // shut up valgrind
+  chain_fsetxattr(tmpfd, "user.test", &buf, sizeof(buf));
+  chain_fsetxattr(tmpfd, "user.test2", &buf, sizeof(buf));
+  chain_fsetxattr(tmpfd, "user.test3", &buf, sizeof(buf));
+  chain_fsetxattr(tmpfd, "user.test4", &buf, sizeof(buf));
+  ret = chain_fsetxattr(tmpfd, "user.test5", &buf, sizeof(buf));
+  if (ret == -ENOSPC) {
+    dout(0) << "limited size xattrs" << dendl;
+  }
+  chain_fremovexattr(tmpfd, "user.test");
+  chain_fremovexattr(tmpfd, "user.test2");
+  chain_fremovexattr(tmpfd, "user.test3");
+  chain_fremovexattr(tmpfd, "user.test4");
+  chain_fremovexattr(tmpfd, "user.test5");
+
+  ::unlink(fn);
+  VOID_TEMP_FAILURE_RETRY(::close(tmpfd));
+
+  return 0;
+}
+
+int FileStore::_sanity_check_fs()
+{
+  // sanity check(s)
+
+  if (((int)m_filestore_journal_writeahead +
+      (int)m_filestore_journal_parallel +
+      (int)m_filestore_journal_trailing) > 1) {
+    dout(0) << "mount ERROR: more than one of filestore journal {writeahead,parallel,trailing} enabled" << dendl;
+    cerr << TEXT_RED 
+	 << " ** WARNING: more than one of 'filestore journal {writeahead,parallel,trailing}'\n"
+	 << "             is enabled in ceph.conf.  You must choose a single journal mode."
+	 << TEXT_NORMAL << std::endl;
+    return -EINVAL;
+  }
+
+  if (!backend->can_checkpoint()) {
+    if (!journal || !m_filestore_journal_writeahead) {
+      dout(0) << "mount WARNING: no btrfs, and no journal in writeahead mode; data may be lost" << dendl;
+      cerr << TEXT_RED 
+	   << " ** WARNING: no btrfs AND (no journal OR journal not in writeahead mode)\n"
+	   << "             For non-btrfs volumes, a writeahead journal is required to\n"
+	   << "             maintain on-disk consistency in the event of a crash.  Your conf\n"
+	   << "             should include something like:\n"
+	   << "        osd journal = /path/to/journal_device_or_file\n"
+	   << "        filestore journal writeahead = true\n"
+	   << TEXT_NORMAL;
+    }
+  }
+
+  if (!journal) {
+    dout(0) << "mount WARNING: no journal" << dendl;
+    cerr << TEXT_YELLOW
+	 << " ** WARNING: No osd journal is configured: write latency may be high.\n"
+	 << "             If you will not be using an osd journal, write latency may be\n"
+	 << "             relatively high.  It can be reduced somewhat by lowering\n"
+	 << "             filestore_max_sync_interval, but lower values mean lower write\n"
+	 << "             throughput, especially with spinning disks.\n"
+	 << TEXT_NORMAL;
+  }
+
+  return 0;
+}
+
+int FileStore::write_superblock()
+{
+  bufferlist bl;
+  ::encode(superblock, bl);
+  return safe_write_file(basedir.c_str(), "superblock",
+      bl.c_str(), bl.length());
+}
+
+int FileStore::read_superblock()
+{
+  bufferptr bp(PATH_MAX);
+  int ret = safe_read_file(basedir.c_str(), "superblock",
+      bp.c_str(), bp.length());
+  if (ret < 0) {
+    if (ret == -ENOENT) {
+      // If the file doesn't exist write initial CompatSet
+      return write_superblock();
+    }
+    return ret;
+  }
+
+  bufferlist bl;
+  bl.push_back(bp);
+  bufferlist::iterator i = bl.begin();
+  ::decode(superblock, i);
+  return 0;
+}
+
+void FileStore::set_allow_sharded_objects()
+{
+  if (!get_allow_sharded_objects()) {
+    superblock.compat_features.incompat.insert(CEPH_FS_FEATURE_INCOMPAT_SHARDS);
+    int ret = write_superblock();
+    assert(ret == 0);	//Should we return error and make caller handle it?
+  }
+  return;
+}
+
+bool FileStore::get_allow_sharded_objects()
+{
+  return g_conf->filestore_debug_disable_sharded_check ||
+    superblock.compat_features.incompat.contains(CEPH_FS_FEATURE_INCOMPAT_SHARDS);
+}
+
+int FileStore::update_version_stamp()
+{
+  return write_version_stamp();
+}
+
+int FileStore::version_stamp_is_valid(uint32_t *version)
+{
+  bufferptr bp(PATH_MAX);
+  int ret = safe_read_file(basedir.c_str(), "store_version",
+      bp.c_str(), bp.length());
+  if (ret < 0) {
+    if (ret == -ENOENT)
+      return 0;
+    return ret;
+  }
+  bufferlist bl;
+  bl.push_back(bp);
+  bufferlist::iterator i = bl.begin();
+  ::decode(*version, i);
+  if (*version == target_version)
+    return 1;
+  else
+    return 0;
+}
+
+int FileStore::write_version_stamp()
+{
+  bufferlist bl;
+  ::encode(target_version, bl);
+
+  return safe_write_file(basedir.c_str(), "store_version",
+      bl.c_str(), bl.length());
+}
+
+int FileStore::read_op_seq(uint64_t *seq)
+{
+  int op_fd = ::open(current_op_seq_fn.c_str(), O_CREAT|O_RDWR, 0644);
+  if (op_fd < 0) {
+    int r = -errno;
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  char s[40];
+  memset(s, 0, sizeof(s));
+  int ret = safe_read(op_fd, s, sizeof(s) - 1);
+  if (ret < 0) {
+    derr << "error reading " << current_op_seq_fn << ": " << cpp_strerror(ret) << dendl;
+    VOID_TEMP_FAILURE_RETRY(::close(op_fd));
+    assert(!m_filestore_fail_eio || ret != -EIO);
+    return ret;
+  }
+  *seq = atoll(s);
+  return op_fd;
+}
+
+int FileStore::write_op_seq(int fd, uint64_t seq)
+{
+  char s[30];
+  snprintf(s, sizeof(s), "%" PRId64 "\n", seq);
+  int ret = TEMP_FAILURE_RETRY(::pwrite(fd, s, strlen(s), 0));
+  if (ret < 0) {
+    ret = -errno;
+    assert(!m_filestore_fail_eio || ret != -EIO);
+  }
+  return ret;
+}
+
+int FileStore::mount() 
+{
+  int ret;
+  char buf[PATH_MAX];
+  uint64_t initial_op_seq;
+  set cluster_snaps;
+  CompatSet supported_compat_set = get_fs_supported_compat_set();
+
+  dout(5) << "basedir " << basedir << " journal " << journalpath << dendl;
+  
+  // make sure global base dir exists
+  if (::access(basedir.c_str(), R_OK | W_OK)) {
+    ret = -errno;
+    derr << "FileStore::mount: unable to access basedir '" << basedir << "': "
+	 << cpp_strerror(ret) << dendl;
+    goto done;
+  }
+
+  // get fsid
+  snprintf(buf, sizeof(buf), "%s/fsid", basedir.c_str());
+  fsid_fd = ::open(buf, O_RDWR, 0644);
+  if (fsid_fd < 0) {
+    ret = -errno;
+    derr << "FileStore::mount: error opening '" << buf << "': "
+	 << cpp_strerror(ret) << dendl;
+    goto done;
+  }
+
+  ret = read_fsid(fsid_fd, &fsid);
+  if (ret < 0) {
+    derr << "FileStore::mount: error reading fsid_fd: " << cpp_strerror(ret)
+	 << dendl;
+    goto close_fsid_fd;
+  }
+
+  if (lock_fsid() < 0) {
+    derr << "FileStore::mount: lock_fsid failed" << dendl;
+    ret = -EBUSY;
+    goto close_fsid_fd;
+  }
+
+  dout(10) << "mount fsid is " << fsid << dendl;
+
+
+  uint32_t version_stamp;
+  ret = version_stamp_is_valid(&version_stamp);
+  if (ret < 0) {
+    derr << "FileStore::mount : error in version_stamp_is_valid: "
+	 << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  } else if (ret == 0) {
+    if (do_update) {
+      derr << "FileStore::mount : stale version stamp detected: "
+	   << version_stamp 
+	   << ". Proceeding, do_update "
+	   << "is set, performing disk format upgrade."
+	   << dendl;
+    } else {
+      ret = -EINVAL;
+      derr << "FileStore::mount : stale version stamp " << version_stamp
+	   << ". Please run the FileStore update script before starting the "
+	   << "OSD, or set filestore_update_to to " << target_version
+	   << dendl;
+      goto close_fsid_fd;
+    }
+  }
+
+  ret = read_superblock();
+  if (ret < 0) {
+    ret = -EINVAL;
+    goto close_fsid_fd;
+  }
+
+  // Check if this FileStore supports all the necessary features to mount
+  if (supported_compat_set.compare(superblock.compat_features) == -1) {
+    derr << "FileStore::mount : Incompatible features set "
+	   << superblock.compat_features << dendl;
+    ret = -EINVAL;
+    goto close_fsid_fd;
+  }
+
+  // open some dir handles
+  basedir_fd = ::open(basedir.c_str(), O_RDONLY);
+  if (basedir_fd < 0) {
+    ret = -errno;
+    derr << "FileStore::mount: failed to open " << basedir << ": "
+	 << cpp_strerror(ret) << dendl;
+    basedir_fd = -1;
+    goto close_fsid_fd;
+  }
+
+  // test for btrfs, xattrs, etc.
+  ret = _detect_fs();
+  if (ret < 0) {
+    derr << "FileStore::mount : error in _detect_fs: "
+	 << cpp_strerror(ret) << dendl;
+    goto close_basedir_fd;
+  }
+
+  {
+    list ls;
+    ret = backend->list_checkpoints(ls);
+    if (ret < 0) {
+      derr << "FileStore::mount : error in _list_snaps: "<< cpp_strerror(ret) << dendl;
+      goto close_basedir_fd;
+    }
+
+    long long unsigned c, prev = 0;
+    char clustersnap[NAME_MAX];
+    for (list::iterator it = ls.begin(); it != ls.end(); ++it) {
+      if (sscanf(it->c_str(), COMMIT_SNAP_ITEM, &c) == 1) {
+	assert(c > prev);
+	prev = c;
+	snaps.push_back(c);
+      } else if (sscanf(it->c_str(), CLUSTER_SNAP_ITEM, clustersnap) == 1)
+	cluster_snaps.insert(*it);
+    }
+  }
+
+  if (m_osd_rollback_to_cluster_snap.length() &&
+      cluster_snaps.count(m_osd_rollback_to_cluster_snap) == 0) {
+    derr << "rollback to cluster snapshot '" << m_osd_rollback_to_cluster_snap << "': not found" << dendl;
+    ret = -ENOENT;
+    goto close_basedir_fd;
+  }
+
+  char nosnapfn[200];
+  snprintf(nosnapfn, sizeof(nosnapfn), "%s/nosnap", current_fn.c_str());
+
+  if (backend->can_checkpoint()) {
+    if (snaps.empty()) {
+      dout(0) << "mount WARNING: no consistent snaps found, store may be in inconsistent state" << dendl;
+    } else {
+      char s[NAME_MAX];
+      uint64_t curr_seq = 0;
+
+      if (m_osd_rollback_to_cluster_snap.length()) {
+	derr << TEXT_RED
+	     << " ** NOTE: rolling back to cluster snapshot " << m_osd_rollback_to_cluster_snap << " **"
+	     << TEXT_NORMAL
+	     << dendl;
+	assert(cluster_snaps.count(m_osd_rollback_to_cluster_snap));
+	snprintf(s, sizeof(s), CLUSTER_SNAP_ITEM, m_osd_rollback_to_cluster_snap.c_str());
+      } else {
+	{
+	  int fd = read_op_seq(&curr_seq);
+	  if (fd >= 0) {
+	    VOID_TEMP_FAILURE_RETRY(::close(fd));
+	  }
+	}
+	if (curr_seq)
+	  dout(10) << " current/ seq was " << curr_seq << dendl;
+	else
+	  dout(10) << " current/ missing entirely (unusual, but okay)" << dendl;
+
+	uint64_t cp = snaps.back();
+	dout(10) << " most recent snap from " << snaps << " is " << cp << dendl;
+
+	// if current/ is marked as non-snapshotted, refuse to roll
+	// back (without clear direction) to avoid throwing out new
+	// data.
+	struct stat st;
+	if (::stat(nosnapfn, &st) == 0) {
+	  if (!m_osd_use_stale_snap) {
+	    derr << "ERROR: " << nosnapfn << " exists, not rolling back to avoid losing new data" << dendl;
+	    derr << "Force rollback to old snapshotted version with 'osd use stale snap = true'" << dendl;
+	    derr << "config option for --osd-use-stale-snap startup argument." << dendl;
+	    ret = -ENOTSUP;
+	    goto close_basedir_fd;
+	  }
+	  derr << "WARNING: user forced start with data sequence mismatch: current was " << curr_seq
+	       << ", newest snap is " << cp << dendl;
+	  cerr << TEXT_YELLOW
+	       << " ** WARNING: forcing the use of stale snapshot data **"
+	       << TEXT_NORMAL << std::endl;
+	}
+
+        dout(10) << "mount rolling back to consistent snap " << cp << dendl;
+	snprintf(s, sizeof(s), COMMIT_SNAP_ITEM, (long long unsigned)cp);
+      }
+
+      // drop current?
+      ret = backend->rollback_to(s);
+      if (ret) {
+	derr << "FileStore::mount: error rolling back to " << s << ": "
+	     << cpp_strerror(ret) << dendl;
+	goto close_basedir_fd;
+      }
+    }
+  }
+  initial_op_seq = 0;
+
+  current_fd = ::open(current_fn.c_str(), O_RDONLY);
+  if (current_fd < 0) {
+    ret = -errno;
+    derr << "FileStore::mount: error opening: " << current_fn << ": " << cpp_strerror(ret) << dendl;
+    goto close_basedir_fd;
+  }
+
+  assert(current_fd >= 0);
+
+  op_fd = read_op_seq(&initial_op_seq);
+  if (op_fd < 0) {
+    derr << "FileStore::mount: read_op_seq failed" << dendl;
+    goto close_current_fd;
+  }
+
+  dout(5) << "mount op_seq is " << initial_op_seq << dendl;
+  if (initial_op_seq == 0) {
+    derr << "mount initial op seq is 0; something is wrong" << dendl;
+    ret = -EINVAL;
+    goto close_current_fd;
+  }
+
+  if (!backend->can_checkpoint()) {
+    // mark current/ as non-snapshotted so that we don't rollback away
+    // from it.
+    int r = ::creat(nosnapfn, 0644);
+    if (r < 0) {
+      derr << "FileStore::mount: failed to create current/nosnap" << dendl;
+      goto close_current_fd;
+    }
+    VOID_TEMP_FAILURE_RETRY(::close(r));
+  } else {
+    // clear nosnap marker, if present.
+    ::unlink(nosnapfn);
+  }
+
+  {
+    LevelDBStore *omap_store = new LevelDBStore(g_ceph_context, omap_dir);
+
+    omap_store->init();
+    if (g_conf->osd_leveldb_write_buffer_size)
+      omap_store->options.write_buffer_size = g_conf->osd_leveldb_write_buffer_size;
+    if (g_conf->osd_leveldb_cache_size)
+      omap_store->options.cache_size = g_conf->osd_leveldb_cache_size;
+    if (g_conf->osd_leveldb_block_size)
+      omap_store->options.block_size = g_conf->osd_leveldb_block_size;
+    if (g_conf->osd_leveldb_bloom_size)
+      omap_store->options.bloom_size = g_conf->osd_leveldb_bloom_size;
+    if (g_conf->osd_leveldb_compression)
+      omap_store->options.compression_enabled = g_conf->osd_leveldb_compression;
+    if (g_conf->osd_leveldb_paranoid)
+      omap_store->options.paranoid_checks = g_conf->osd_leveldb_paranoid;
+    if (g_conf->osd_leveldb_max_open_files)
+      omap_store->options.max_open_files = g_conf->osd_leveldb_max_open_files;
+    if (g_conf->osd_leveldb_log.length())
+      omap_store->options.log_file = g_conf->osd_leveldb_log;
+
+    stringstream err;
+    if (omap_store->create_and_open(err)) {
+      delete omap_store;
+      derr << "Error initializing leveldb: " << err.str() << dendl;
+      ret = -1;
+      goto close_current_fd;
+    }
+
+    if (g_conf->osd_compact_leveldb_on_mount) {
+      derr << "Compacting store..." << dendl;
+      omap_store->compact();
+      derr << "...finished compacting store" << dendl;
+    }
+
+    DBObjectMap *dbomap = new DBObjectMap(omap_store);
+    ret = dbomap->init(do_update);
+    if (ret < 0) {
+      delete dbomap;
+      derr << "Error initializing DBObjectMap: " << ret << dendl;
+      goto close_current_fd;
+    }
+    stringstream err2;
+
+    if (g_conf->filestore_debug_omap_check && !dbomap->check(err2)) {
+      derr << err2.str() << dendl;;
+      delete dbomap;
+      ret = -EINVAL;
+      goto close_current_fd;
+    }
+    object_map.reset(dbomap);
+  }
+
+  // journal
+  open_journal();
+
+  // select journal mode?
+  if (journal) {
+    if (!m_filestore_journal_writeahead &&
+	!m_filestore_journal_parallel &&
+	!m_filestore_journal_trailing) {
+      if (!backend->can_checkpoint()) {
+	m_filestore_journal_writeahead = true;
+	dout(0) << "mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled" << dendl;
+      } else {
+	m_filestore_journal_parallel = true;
+	dout(0) << "mount: enabling PARALLEL journal mode: fs, checkpoint is enabled" << dendl;
+      }
+    } else {
+      if (m_filestore_journal_writeahead)
+	dout(0) << "mount: WRITEAHEAD journal mode explicitly enabled in conf" << dendl;
+      if (m_filestore_journal_parallel)
+	dout(0) << "mount: PARALLEL journal mode explicitly enabled in conf" << dendl;
+      if (m_filestore_journal_trailing)
+	dout(0) << "mount: TRAILING journal mode explicitly enabled in conf" << dendl;
+    }
+    if (m_filestore_journal_writeahead)
+      journal->set_wait_on_full(true);
+  }
+
+  ret = _sanity_check_fs();
+  if (ret) {
+    derr << "FileStore::mount: _sanity_check_fs failed with error "
+	 << ret << dendl;
+    goto close_current_fd;
+  }
+
+  // Cleanup possibly invalid collections
+  {
+    vector collections;
+    ret = list_collections(collections);
+    if (ret < 0) {
+      derr << "Error " << ret << " while listing collections" << dendl;
+      goto close_current_fd;
+    }
+    for (vector::iterator i = collections.begin();
+	 i != collections.end();
+	 ++i) {
+      Index index;
+      ret = get_index(*i, &index);
+      if (ret < 0) {
+	derr << "Unable to mount index " << *i 
+	     << " with error: " << ret << dendl;
+	goto close_current_fd;
+      }
+      index->cleanup();
+    }
+  }
+
+  wbthrottle.start();
+  sync_thread.create();
+
+  ret = journal_replay(initial_op_seq);
+  if (ret < 0) {
+    derr << "mount failed to open journal " << journalpath << ": " << cpp_strerror(ret) << dendl;
+    if (ret == -ENOTTY) {
+      derr << "maybe journal is not pointing to a block device and its size "
+	   << "wasn't configured?" << dendl;
+    }
+
+    // stop sync thread
+    lock.Lock();
+    stop = true;
+    sync_cond.Signal();
+    lock.Unlock();
+    sync_thread.join();
+
+    wbthrottle.stop();
+
+    goto close_current_fd;
+  }
+
+  {
+    stringstream err2;
+    if (g_conf->filestore_debug_omap_check && !object_map->check(err2)) {
+      derr << err2.str() << dendl;;
+      ret = -EINVAL;
+      goto close_current_fd;
+    }
+  }
+
+  journal_start();
+
+  op_tp.start();
+  op_finisher.start();
+  ondisk_finisher.start();
+
+  timer.init();
+
+  // all okay.
+  return 0;
+
+close_current_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(current_fd));
+  current_fd = -1;
+close_basedir_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(basedir_fd));
+  basedir_fd = -1;
+close_fsid_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+  fsid_fd = -1;
+done:
+  assert(!m_filestore_fail_eio || ret != -EIO);
+  return ret;
+}
+
+int FileStore::umount() 
+{
+  dout(5) << "umount " << basedir << dendl;
+  
+
+  start_sync();
+
+  lock.Lock();
+  stop = true;
+  sync_cond.Signal();
+  lock.Unlock();
+  sync_thread.join();
+  wbthrottle.stop();
+  op_tp.stop();
+
+  journal_stop();
+
+  op_finisher.stop();
+  ondisk_finisher.stop();
+
+  if (fsid_fd >= 0) {
+    VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+    fsid_fd = -1;
+  }
+  if (op_fd >= 0) {
+    VOID_TEMP_FAILURE_RETRY(::close(op_fd));
+    op_fd = -1;
+  }
+  if (current_fd >= 0) {
+    VOID_TEMP_FAILURE_RETRY(::close(current_fd));
+    current_fd = -1;
+  }
+  if (basedir_fd >= 0) {
+    VOID_TEMP_FAILURE_RETRY(::close(basedir_fd));
+    basedir_fd = -1;
+  }
+
+  if (backend != generic_backend) {
+    delete backend;
+    backend = generic_backend;
+  }
+
+  force_sync = false;
+
+  object_map.reset();
+
+  {
+    Mutex::Locker l(sync_entry_timeo_lock);
+    timer.shutdown();
+  }
+
+  // nothing
+  return 0;
+}
+
+
+int FileStore::get_max_object_name_length()
+{
+  lock.Lock();
+  int ret = pathconf(basedir.c_str(), _PC_NAME_MAX);
+  if (ret < 0) {
+    int err = errno;
+    lock.Unlock();
+    if (err == 0)
+      return -EDOM;
+    return -err;
+  }
+  lock.Unlock();
+  return ret;
+}
+
+
+
+/// -----------------------------
+
+FileStore::Op *FileStore::build_op(list& tls,
+				   Context *onreadable,
+				   Context *onreadable_sync,
+				   TrackedOpRef osd_op)
+{
+  uint64_t bytes = 0, ops = 0;
+  for (list::iterator p = tls.begin();
+       p != tls.end();
+       ++p) {
+    bytes += (*p)->get_num_bytes();
+    ops += (*p)->get_num_ops();
+  }
+
+  Op *o = new Op;
+  o->start = ceph_clock_now(g_ceph_context);
+  o->tls.swap(tls);
+  o->onreadable = onreadable;
+  o->onreadable_sync = onreadable_sync;
+  o->ops = ops;
+  o->bytes = bytes;
+  o->osd_op = osd_op;
+  return o;
+}
+
+
+
+void FileStore::queue_op(OpSequencer *osr, Op *o)
+{
+  // queue op on sequencer, then queue sequencer for the threadpool,
+  // so that regardless of which order the threads pick up the
+  // sequencer, the op order will be preserved.
+
+  osr->queue(o);
+
+  logger->inc(l_os_ops);
+  logger->inc(l_os_bytes, o->bytes);
+
+  dout(5) << "queue_op " << o << " seq " << o->op
+	  << " " << *osr
+	  << " " << o->bytes << " bytes"
+	  << "   (queue has " << op_queue_len << " ops and " << op_queue_bytes << " bytes)"
+	  << dendl;
+  op_wq.queue(osr);
+}
+
+void FileStore::op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle)
+{
+  // Do not call while holding the journal lock!
+  uint64_t max_ops = m_filestore_queue_max_ops;
+  uint64_t max_bytes = m_filestore_queue_max_bytes;
+
+  if (backend->can_checkpoint() && is_committing()) {
+    max_ops += m_filestore_queue_committing_max_ops;
+    max_bytes += m_filestore_queue_committing_max_bytes;
+  }
+
+  logger->set(l_os_oq_max_ops, max_ops);
+  logger->set(l_os_oq_max_bytes, max_bytes);
+
+  utime_t start = ceph_clock_now(g_ceph_context);
+  {
+    Mutex::Locker l(op_throttle_lock);
+    while ((max_ops && (op_queue_len + 1) > max_ops) ||
+           (max_bytes && op_queue_bytes      // let single large ops through!
+	      && (op_queue_bytes + o->bytes) > max_bytes)) {
+      dout(2) << "waiting " << op_queue_len + 1 << " > " << max_ops << " ops || "
+	      << op_queue_bytes + o->bytes << " > " << max_bytes << dendl;
+      if (handle)
+	handle->suspend_tp_timeout();
+      op_throttle_cond.Wait(op_throttle_lock);
+      if (handle)
+	handle->reset_tp_timeout();
+    }
+
+    op_queue_len++;
+    op_queue_bytes += o->bytes;
+  }
+  utime_t end = ceph_clock_now(g_ceph_context);
+  logger->tinc(l_os_queue_lat, end - start);
+
+  logger->set(l_os_oq_ops, op_queue_len);
+  logger->set(l_os_oq_bytes, op_queue_bytes);
+}
+
+void FileStore::op_queue_release_throttle(Op *o)
+{
+  {
+    Mutex::Locker l(op_throttle_lock);
+    op_queue_len--;
+    op_queue_bytes -= o->bytes;
+    op_throttle_cond.Signal();
+  }
+
+  logger->set(l_os_oq_ops, op_queue_len);
+  logger->set(l_os_oq_bytes, op_queue_bytes);
+}
+
+void FileStore::_do_op(OpSequencer *osr, ThreadPool::TPHandle &handle)
+{
+  wbthrottle.throttle();
+  // inject a stall?
+  if (g_conf->filestore_inject_stall) {
+    int orig = g_conf->filestore_inject_stall;
+    dout(5) << "_do_op filestore_inject_stall " << orig << ", sleeping" << dendl;
+    for (int n = 0; n < g_conf->filestore_inject_stall; n++)
+      sleep(1);
+    g_conf->set_val("filestore_inject_stall", "0");
+    dout(5) << "_do_op done stalling" << dendl;
+  }
+
+  osr->apply_lock.Lock();
+  Op *o = osr->peek_queue();
+  apply_manager.op_apply_start(o->op);
+  dout(5) << "_do_op " << o << " seq " << o->op << " " << *osr << "/" << osr->parent << " start" << dendl;
+  int r = _do_transactions(o->tls, o->op, &handle);
+  apply_manager.op_apply_finish(o->op);
+  dout(10) << "_do_op " << o << " seq " << o->op << " r = " << r
+	   << ", finisher " << o->onreadable << " " << o->onreadable_sync << dendl;
+}
+
+void FileStore::_finish_op(OpSequencer *osr)
+{
+  list to_queue;
+  Op *o = osr->dequeue(&to_queue);
+  
+  dout(10) << "_finish_op " << o << " seq " << o->op << " " << *osr << "/" << osr->parent << dendl;
+  osr->apply_lock.Unlock();  // locked in _do_op
+
+  // called with tp lock held
+  op_queue_release_throttle(o);
+
+  utime_t lat = ceph_clock_now(g_ceph_context);
+  lat -= o->start;
+  logger->tinc(l_os_apply_lat, lat);
+
+  if (o->onreadable_sync) {
+    o->onreadable_sync->complete(0);
+  }
+  if (o->onreadable) {
+    op_finisher.queue(o->onreadable);
+  }
+  op_finisher.queue(to_queue);
+  delete o;
+}
+
+
+struct C_JournaledAhead : public Context {
+  FileStore *fs;
+  FileStore::OpSequencer *osr;
+  FileStore::Op *o;
+  Context *ondisk;
+
+  C_JournaledAhead(FileStore *f, FileStore::OpSequencer *os, FileStore::Op *o, Context *ondisk):
+    fs(f), osr(os), o(o), ondisk(ondisk) { }
+  void finish(int r) {
+    fs->_journaled_ahead(osr, o, ondisk);
+  }
+};
+
+int FileStore::queue_transactions(Sequencer *posr, list &tls,
+				  TrackedOpRef osd_op,
+				  ThreadPool::TPHandle *handle)
+{
+  Context *onreadable;
+  Context *ondisk;
+  Context *onreadable_sync;
+  ObjectStore::Transaction::collect_contexts(
+    tls, &onreadable, &ondisk, &onreadable_sync);
+  if (g_conf->filestore_blackhole) {
+    dout(0) << "queue_transactions filestore_blackhole = TRUE, dropping transaction" << dendl;
+    delete ondisk;
+    delete onreadable;
+    delete onreadable_sync;
+    return 0;
+  }
+
+  // set up the sequencer
+  OpSequencer *osr;
+  if (!posr)
+    posr = &default_osr;
+  if (posr->p) {
+    osr = static_cast(posr->p);
+    dout(5) << "queue_transactions existing " << *osr << "/" << osr->parent << dendl; //<< " w/ q " << osr->q << dendl;
+  } else {
+    osr = new OpSequencer;
+    osr->parent = posr;
+    posr->p = osr;
+    dout(5) << "queue_transactions new " << *osr << "/" << osr->parent << dendl;
+  }
+
+  if (journal && journal->is_writeable() && !m_filestore_journal_trailing) {
+    Op *o = build_op(tls, onreadable, onreadable_sync, osd_op);
+    op_queue_reserve_throttle(o, handle);
+    journal->throttle();
+    uint64_t op_num = submit_manager.op_submit_start();
+    o->op = op_num;
+
+    if (m_filestore_do_dump)
+      dump_transactions(o->tls, o->op, osr);
+
+    if (m_filestore_journal_parallel) {
+      dout(5) << "queue_transactions (parallel) " << o->op << " " << o->tls << dendl;
+      
+      _op_journal_transactions(o->tls, o->op, ondisk, osd_op);
+      
+      // queue inside submit_manager op submission lock
+      queue_op(osr, o);
+    } else if (m_filestore_journal_writeahead) {
+      dout(5) << "queue_transactions (writeahead) " << o->op << " " << o->tls << dendl;
+      
+      osr->queue_journal(o->op);
+
+      _op_journal_transactions(o->tls, o->op,
+			       new C_JournaledAhead(this, osr, o, ondisk),
+			       osd_op);
+    } else {
+      assert(0);
+    }
+    submit_manager.op_submit_finish(op_num);
+    return 0;
+  }
+
+  uint64_t op = submit_manager.op_submit_start();
+  dout(5) << "queue_transactions (trailing journal) " << op << " " << tls << dendl;
+
+  if (m_filestore_do_dump)
+    dump_transactions(tls, op, osr);
+
+  apply_manager.op_apply_start(op);
+  int r = do_transactions(tls, op);
+    
+  if (r >= 0) {
+    _op_journal_transactions(tls, op, ondisk, osd_op);
+  } else {
+    delete ondisk;
+  }
+
+  // start on_readable finisher after we queue journal item, as on_readable callback
+  // is allowed to delete the Transaction
+  if (onreadable_sync) {
+    onreadable_sync->complete(r);
+  }
+  op_finisher.queue(onreadable, r);
+
+  submit_manager.op_submit_finish(op);
+  apply_manager.op_apply_finish(op);
+
+  return r;
+}
+
+void FileStore::_journaled_ahead(OpSequencer *osr, Op *o, Context *ondisk)
+{
+  dout(5) << "_journaled_ahead " << o << " seq " << o->op << " " << *osr << " " << o->tls << dendl;
+
+  // this should queue in order because the journal does it's completions in order.
+  queue_op(osr, o);
+
+  list to_queue;
+  osr->dequeue_journal(&to_queue);
+
+  // do ondisk completions async, to prevent any onreadable_sync completions
+  // getting blocked behind an ondisk completion.
+  if (ondisk) {
+    dout(10) << " queueing ondisk " << ondisk << dendl;
+    ondisk_finisher.queue(ondisk);
+  }
+  ondisk_finisher.queue(to_queue);
+}
+
+int FileStore::_do_transactions(
+  list &tls,
+  uint64_t op_seq,
+  ThreadPool::TPHandle *handle)
+{
+  int r = 0;
+
+  uint64_t bytes = 0, ops = 0;
+  for (list::iterator p = tls.begin();
+       p != tls.end();
+       ++p) {
+    bytes += (*p)->get_num_bytes();
+    ops += (*p)->get_num_ops();
+  }
+
+  int trans_num = 0;
+  for (list::iterator p = tls.begin();
+       p != tls.end();
+       ++p, trans_num++) {
+    r = _do_transaction(**p, op_seq, trans_num, handle);
+    if (r < 0)
+      break;
+    if (handle)
+      handle->reset_tp_timeout();
+  }
+  
+  return r;
+}
+
+void FileStore::_set_global_replay_guard(coll_t cid,
+					 const SequencerPosition &spos)
+{
+  if (backend->can_checkpoint())
+    return;
+
+  // sync all previous operations on this sequencer
+  sync_filesystem(basedir_fd);
+
+  char fn[PATH_MAX];
+  get_cdir(cid, fn, sizeof(fn));
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    int err = errno;
+    derr << __func__ << ": " << cid << " error " << cpp_strerror(err) << dendl;
+    assert(0 == "_set_global_replay_guard failed");
+  }
+
+  _inject_failure();
+
+  // then record that we did it
+  bufferlist v;
+  ::encode(spos, v);
+  int r = chain_fsetxattr(fd, GLOBAL_REPLAY_GUARD_XATTR, v.c_str(), v.length());
+  if (r < 0) {
+    derr << __func__ << ": fsetxattr " << GLOBAL_REPLAY_GUARD_XATTR
+	 << " got " << cpp_strerror(r) << dendl;
+    assert(0 == "fsetxattr failed");
+  }
+
+  // and make sure our xattr is durable.
+  ::fsync(fd);
+
+  _inject_failure();
+
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+  dout(10) << __func__ << ": " << spos << " done" << dendl;
+}
+
+int FileStore::_check_global_replay_guard(coll_t cid,
+					  const SequencerPosition& spos)
+{
+  if (!replaying || backend->can_checkpoint())
+    return 1;
+
+  char fn[PATH_MAX];
+  get_cdir(cid, fn, sizeof(fn));
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    dout(10) << __func__ << ": " << cid << " dne" << dendl;
+    return 1;  // if collection does not exist, there is no guard, and we can replay.
+  }
+
+  char buf[100];
+  int r = chain_fgetxattr(fd, GLOBAL_REPLAY_GUARD_XATTR, buf, sizeof(buf));
+  if (r < 0) {
+    dout(20) << __func__ << " no xattr" << dendl;
+    assert(!m_filestore_fail_eio || r != -EIO);
+    VOID_TEMP_FAILURE_RETRY(::close(fd));
+    return 1;  // no xattr
+  }
+  bufferlist bl;
+  bl.append(buf, r);
+
+  SequencerPosition opos;
+  bufferlist::iterator p = bl.begin();
+  ::decode(opos, p);
+
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+  return spos >= opos ? 1 : -1;
+}
+
+
+void FileStore::_set_replay_guard(coll_t cid,
+                                  const SequencerPosition &spos,
+                                  bool in_progress=false)
+{
+  char fn[PATH_MAX];
+  get_cdir(cid, fn, sizeof(fn));
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    int err = errno;
+    derr << "_set_replay_guard " << cid << " error " << cpp_strerror(err) << dendl;
+    assert(0 == "_set_replay_guard failed");
+  }
+  _set_replay_guard(fd, spos, 0, in_progress);
+  ::close(fd);
+} 
+
+
+void FileStore::_set_replay_guard(int fd,
+				  const SequencerPosition& spos,
+				  const ghobject_t *hoid,
+				  bool in_progress)
+{
+  if (backend->can_checkpoint())
+    return;
+
+  dout(10) << "_set_replay_guard " << spos << (in_progress ? " START" : "") << dendl;
+
+  _inject_failure();
+
+  // first make sure the previous operation commits
+  ::fsync(fd);
+
+  // sync object_map too.  even if this object has a header or keys,
+  // it have had them in the past and then removed them, so always
+  // sync.
+  object_map->sync(hoid, &spos);
+
+  _inject_failure();
+
+  // then record that we did it
+  bufferlist v(40);
+  ::encode(spos, v);
+  ::encode(in_progress, v);
+  int r = chain_fsetxattr(fd, REPLAY_GUARD_XATTR, v.c_str(), v.length());
+  if (r < 0) {
+    derr << "fsetxattr " << REPLAY_GUARD_XATTR << " got " << cpp_strerror(r) << dendl;
+    assert(0 == "fsetxattr failed");
+  }
+
+  // and make sure our xattr is durable.
+  ::fsync(fd);
+
+  _inject_failure();
+
+  dout(10) << "_set_replay_guard " << spos << " done" << dendl;
+}
+
+void FileStore::_close_replay_guard(coll_t cid,
+                                    const SequencerPosition &spos)
+{
+  char fn[PATH_MAX];
+  get_cdir(cid, fn, sizeof(fn));
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    int err = errno;
+    derr << "_close_replay_guard " << cid << " error " << cpp_strerror(err) << dendl;
+    assert(0 == "_close_replay_guard failed");
+  }
+  _close_replay_guard(fd, spos);
+  ::close(fd);
+} 
+
+void FileStore::_close_replay_guard(int fd, const SequencerPosition& spos)
+{
+  if (backend->can_checkpoint())
+    return;
+
+  dout(10) << "_close_replay_guard " << spos << dendl;
+
+  _inject_failure();
+
+  // then record that we are done with this operation
+  bufferlist v(40);
+  ::encode(spos, v);
+  bool in_progress = false;
+  ::encode(in_progress, v);
+  int r = chain_fsetxattr(fd, REPLAY_GUARD_XATTR, v.c_str(), v.length());
+  if (r < 0) {
+    derr << "fsetxattr " << REPLAY_GUARD_XATTR << " got " << cpp_strerror(r) << dendl;
+    assert(0 == "fsetxattr failed");
+  }
+
+  // and make sure our xattr is durable.
+  ::fsync(fd);
+
+  _inject_failure();
+
+  dout(10) << "_close_replay_guard " << spos << " done" << dendl;
+}
+
+int FileStore::_check_replay_guard(coll_t cid, ghobject_t oid, const SequencerPosition& spos)
+{
+  if (!replaying || backend->can_checkpoint())
+    return 1;
+
+  int r = _check_global_replay_guard(cid, spos);
+  if (r < 0)
+    return r;
+
+  FDRef fd;
+  r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    dout(10) << "_check_replay_guard " << cid << " " << oid << " dne" << dendl;
+    return 1;  // if file does not exist, there is no guard, and we can replay.
+  }
+  int ret = _check_replay_guard(**fd, spos);
+  lfn_close(fd);
+  return ret;
+}
+
+int FileStore::_check_replay_guard(coll_t cid, const SequencerPosition& spos)
+{
+  if (!replaying || backend->can_checkpoint())
+    return 1;
+
+  char fn[PATH_MAX];
+  get_cdir(cid, fn, sizeof(fn));
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    dout(10) << "_check_replay_guard " << cid << " dne" << dendl;
+    return 1;  // if collection does not exist, there is no guard, and we can replay.
+  }
+  int ret = _check_replay_guard(fd, spos);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+  return ret;
+}
+
+int FileStore::_check_replay_guard(int fd, const SequencerPosition& spos)
+{
+  if (!replaying || backend->can_checkpoint())
+    return 1;
+
+  char buf[100];
+  int r = chain_fgetxattr(fd, REPLAY_GUARD_XATTR, buf, sizeof(buf));
+  if (r < 0) {
+    dout(20) << "_check_replay_guard no xattr" << dendl;
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return 1;  // no xattr
+  }
+  bufferlist bl;
+  bl.append(buf, r);
+
+  SequencerPosition opos;
+  bufferlist::iterator p = bl.begin();
+  ::decode(opos, p);
+  bool in_progress = false;
+  if (!p.end())   // older journals don't have this
+    ::decode(in_progress, p);
+  if (opos > spos) {
+    dout(10) << "_check_replay_guard object has " << opos << " > current pos " << spos
+	     << ", now or in future, SKIPPING REPLAY" << dendl;
+    return -1;
+  } else if (opos == spos) {
+    if (in_progress) {
+      dout(10) << "_check_replay_guard object has " << opos << " == current pos " << spos
+	       << ", in_progress=true, CONDITIONAL REPLAY" << dendl;
+      return 0;
+    } else {
+      dout(10) << "_check_replay_guard object has " << opos << " == current pos " << spos
+	       << ", in_progress=false, SKIPPING REPLAY" << dendl;
+      return -1;
+    }
+  } else {
+    dout(10) << "_check_replay_guard object has " << opos << " < current pos " << spos
+	     << ", in past, will replay" << dendl;
+    return 1;
+  }
+}
+
+unsigned FileStore::_do_transaction(
+  Transaction& t, uint64_t op_seq, int trans_num,
+  ThreadPool::TPHandle *handle)
+{
+  dout(10) << "_do_transaction on " << &t << dendl;
+
+  Transaction::iterator i = t.begin();
+  
+  SequencerPosition spos(op_seq, trans_num, 0);
+  while (i.have_op()) {
+    if (handle)
+      handle->reset_tp_timeout();
+
+    int op = i.get_op();
+    int r = 0;
+
+    _inject_failure();
+
+    switch (op) {
+    case Transaction::OP_NOP:
+      break;
+    case Transaction::OP_TOUCH:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _touch(cid, oid);
+      }
+      break;
+      
+    case Transaction::OP_WRITE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	bool replica = i.get_replica();
+	bufferlist bl;
+	i.get_bl(bl);
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _write(cid, oid, off, len, bl, replica);
+      }
+      break;
+      
+    case Transaction::OP_ZERO:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _zero(cid, oid, off, len);
+      }
+      break;
+      
+    case Transaction::OP_TRIMCACHE:
+      {
+	i.get_cid();
+	i.get_oid();
+	i.get_length();
+	i.get_length();
+	// deprecated, no-op
+      }
+      break;
+      
+    case Transaction::OP_TRUNCATE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _truncate(cid, oid, off);
+      }
+      break;
+      
+    case Transaction::OP_REMOVE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _remove(cid, oid, spos);
+      }
+      break;
+      
+    case Transaction::OP_SETATTR:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	string name = i.get_attrname();
+	bufferlist bl;
+	i.get_bl(bl);
+	if (_check_replay_guard(cid, oid, spos) > 0) {
+	  map to_set;
+	  to_set[name] = bufferptr(bl.c_str(), bl.length());
+	  r = _setattrs(cid, oid, to_set, spos);
+	  if (r == -ENOSPC)
+	    dout(0) << " ENOSPC on setxattr on " << cid << "/" << oid
+		    << " name " << name << " size " << bl.length() << dendl;
+	}
+      }
+      break;
+      
+    case Transaction::OP_SETATTRS:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	map aset;
+	i.get_attrset(aset);
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _setattrs(cid, oid, aset, spos);
+  	if (r == -ENOSPC)
+	  dout(0) << " ENOSPC on setxattrs on " << cid << "/" << oid << dendl;
+      }
+      break;
+
+    case Transaction::OP_RMATTR:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	string name = i.get_attrname();
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _rmattr(cid, oid, name.c_str(), spos);
+      }
+      break;
+
+    case Transaction::OP_RMATTRS:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _rmattrs(cid, oid, spos);
+      }
+      break;
+      
+    case Transaction::OP_CLONE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+	r = _clone(cid, oid, noid, spos);
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+ 	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	r = _clone_range(cid, oid, noid, off, len, off, spos);
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE2:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+ 	uint64_t srcoff = i.get_length();
+	uint64_t len = i.get_length();
+ 	uint64_t dstoff = i.get_length();
+	r = _clone_range(cid, oid, noid, srcoff, len, dstoff, spos);
+      }
+      break;
+
+    case Transaction::OP_MKCOLL:
+      {
+	coll_t cid = i.get_cid();
+	if (_check_replay_guard(cid, spos) > 0)
+	  r = _create_collection(cid, spos);
+      }
+      break;
+
+    case Transaction::OP_RMCOLL:
+      {
+	coll_t cid = i.get_cid();
+	if (_check_replay_guard(cid, spos) > 0)
+	  r = _destroy_collection(cid);
+      }
+      break;
+
+    case Transaction::OP_COLL_ADD:
+      {
+	coll_t ncid = i.get_cid();
+	coll_t ocid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	r = _collection_add(ncid, ocid, oid, spos);
+      }
+      break;
+
+    case Transaction::OP_COLL_REMOVE:
+       {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	if (_check_replay_guard(cid, oid, spos) > 0)
+	  r = _remove(cid, oid, spos);
+       }
+      break;
+
+    case Transaction::OP_COLL_MOVE:
+      {
+	// WARNING: this is deprecated and buggy; only here to replay old journals.
+	coll_t ocid = i.get_cid();
+	coll_t ncid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	r = _collection_add(ocid, ncid, oid, spos);
+	if (r == 0 &&
+	    (_check_replay_guard(ocid, oid, spos) > 0))
+	  r = _remove(ocid, oid, spos);
+      }
+      break;
+
+    case Transaction::OP_COLL_MOVE_RENAME:
+      {
+	coll_t oldcid = i.get_cid();
+	ghobject_t oldoid = i.get_oid();
+	coll_t newcid = i.get_cid();
+	ghobject_t newoid = i.get_oid();
+	r = _collection_move_rename(oldcid, oldoid, newcid, newoid, spos);
+      }
+      break;
+
+    case Transaction::OP_COLL_SETATTR:
+      {
+	coll_t cid = i.get_cid();
+	string name = i.get_attrname();
+	bufferlist bl;
+	i.get_bl(bl);
+	if (_check_replay_guard(cid, spos) > 0)
+	  r = _collection_setattr(cid, name.c_str(), bl.c_str(), bl.length());
+      }
+      break;
+
+    case Transaction::OP_COLL_RMATTR:
+      {
+	coll_t cid = i.get_cid();
+	string name = i.get_attrname();
+	if (_check_replay_guard(cid, spos) > 0)
+	  r = _collection_rmattr(cid, name.c_str());
+      }
+      break;
+
+    case Transaction::OP_STARTSYNC:
+      _start_sync();
+      break;
+
+    case Transaction::OP_COLL_RENAME:
+      {
+	coll_t cid(i.get_cid());
+	coll_t ncid(i.get_cid());
+	r = _collection_rename(cid, ncid, spos);
+      }
+      break;
+
+    case Transaction::OP_OMAP_CLEAR:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	r = _omap_clear(cid, oid, spos);
+      }
+      break;
+    case Transaction::OP_OMAP_SETKEYS:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	map aset;
+	i.get_attrset(aset);
+	r = _omap_setkeys(cid, oid, aset, spos);
+      }
+      break;
+    case Transaction::OP_OMAP_RMKEYS:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	set keys;
+	i.get_keyset(keys);
+	r = _omap_rmkeys(cid, oid, keys, spos);
+      }
+      break;
+    case Transaction::OP_OMAP_RMKEYRANGE:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	string first, last;
+	first = i.get_key();
+	last = i.get_key();
+	r = _omap_rmkeyrange(cid, oid, first, last, spos);
+      }
+      break;
+    case Transaction::OP_OMAP_SETHEADER:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	bufferlist bl;
+	i.get_bl(bl);
+	r = _omap_setheader(cid, oid, bl, spos);
+      }
+      break;
+    case Transaction::OP_SPLIT_COLLECTION:
+      {
+	coll_t cid(i.get_cid());
+	uint32_t bits(i.get_u32());
+	uint32_t rem(i.get_u32());
+	coll_t dest(i.get_cid());
+	r = _split_collection_create(cid, bits, rem, dest, spos);
+      }
+      break;
+    case Transaction::OP_SPLIT_COLLECTION2:
+      {
+	coll_t cid(i.get_cid());
+	uint32_t bits(i.get_u32());
+	uint32_t rem(i.get_u32());
+	coll_t dest(i.get_cid());
+	r = _split_collection(cid, bits, rem, dest, spos);
+      }
+      break;
+
+    case Transaction::OP_SETALLOCHINT:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        uint64_t expected_object_size = i.get_length();
+        uint64_t expected_write_size = i.get_length();
+        if (_check_replay_guard(cid, oid, spos) > 0)
+          r = _set_alloc_hint(cid, oid, expected_object_size,
+                              expected_write_size);
+      }
+      break;
+
+    default:
+      derr << "bad op " << op << dendl;
+      assert(0);
+    }
+
+    if (r < 0) {
+      bool ok = false;
+
+      if (r == -ENOENT && !(op == Transaction::OP_CLONERANGE ||
+			    op == Transaction::OP_CLONE ||
+			    op == Transaction::OP_CLONERANGE2 ||
+			    op == Transaction::OP_COLL_ADD))
+	// -ENOENT is normally okay
+	// ...including on a replayed OP_RMCOLL with checkpoint mode
+	ok = true;
+      if (r == -ENOENT && (
+	  op == Transaction::OP_COLL_ADD &&
+	  i.tolerate_collection_add_enoent()))
+	ok = true; // Hack for upgrade from snapcolls to snapmapper
+      if (r == -ENODATA)
+	ok = true;
+
+      if (op == Transaction::OP_SETALLOCHINT)
+        // Either EOPNOTSUPP or EINVAL most probably.  EINVAL in most
+        // cases means invalid hint size (e.g. too big, not a multiple
+        // of block size, etc) or, at least on xfs, an attempt to set
+        // or change it when the file is not empty.  However,
+        // OP_SETALLOCHINT is advisory, so ignore all errors.
+        ok = true;
+
+      if (replaying && !backend->can_checkpoint()) {
+	if (r == -EEXIST && op == Transaction::OP_MKCOLL) {
+	  dout(10) << "tolerating EEXIST during journal replay since checkpoint is not enabled" << dendl;
+	  ok = true;
+	}
+	if (r == -EEXIST && op == Transaction::OP_COLL_ADD) {
+	  dout(10) << "tolerating EEXIST during journal replay since checkpoint is not enabled" << dendl;
+	  ok = true;
+	}
+	if (r == -EEXIST && op == Transaction::OP_COLL_MOVE) {
+	  dout(10) << "tolerating EEXIST during journal replay since checkpoint is not enabled" << dendl;
+	  ok = true;
+	}
+	if (r == -ERANGE) {
+	  dout(10) << "tolerating ERANGE on replay" << dendl;
+	  ok = true;
+	}
+	if (r == -ENOENT) {
+	  dout(10) << "tolerating ENOENT on replay" << dendl;
+	  ok = true;
+	}
+      }
+
+      if (!ok) {
+	const char *msg = "unexpected error code";
+
+	if (r == -ENOENT && (op == Transaction::OP_CLONERANGE ||
+			     op == Transaction::OP_CLONE ||
+			     op == Transaction::OP_CLONERANGE2))
+	  msg = "ENOENT on clone suggests osd bug";
+
+	if (r == -ENOSPC)
+	  // For now, if we hit _any_ ENOSPC, crash, before we do any damage
+	  // by partially applying transactions.
+	  msg = "ENOSPC handling not implemented";
+
+	if (r == -ENOTEMPTY) {
+	  msg = "ENOTEMPTY suggests garbage data in osd data dir";
+	}
+
+	dout(0) << " error " << cpp_strerror(r) << " not handled on operation " << op
+		<< " (" << spos << ", or op " << spos.op << ", counting from 0)" << dendl;
+	dout(0) << msg << dendl;
+	dout(0) << " transaction dump:\n";
+	JSONFormatter f(true);
+	f.open_object_section("transaction");
+	t.dump(&f);
+	f.close_section();
+	f.flush(*_dout);
+	*_dout << dendl;
+
+	if (r == -EMFILE) {
+	  dump_open_fds(g_ceph_context);
+	}
+
+	assert(0 == "unexpected error");
+      }
+    }
+
+    spos.op++;
+  }
+
+  _inject_failure();
+
+  return 0;  // FIXME count errors
+}
+
+  /*********************************************/
+
+
+
+// --------------------
+// objects
+
+bool FileStore::exists(coll_t cid, const ghobject_t& oid)
+{
+  struct stat st;
+  if (stat(cid, oid, &st) == 0)
+    return true;
+  else 
+    return false;
+}
+  
+int FileStore::stat(
+  coll_t cid, const ghobject_t& oid, struct stat *st, bool allow_eio)
+{
+  int r = lfn_stat(cid, oid, st);
+  assert(allow_eio || !m_filestore_fail_eio || r != -EIO);
+  if (r < 0) {
+    dout(10) << "stat " << cid << "/" << oid
+	     << " = " << r << dendl;
+  } else {
+    dout(10) << "stat " << cid << "/" << oid
+	     << " = " << r
+	     << " (size " << st->st_size << ")" << dendl;
+  }
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_mdata_eio(oid)) {
+    return -EIO;
+  } else {
+    return r;
+  }
+}
+
+int FileStore::read(
+  coll_t cid,
+  const ghobject_t& oid,
+  uint64_t offset,
+  size_t len,
+  bufferlist& bl,
+  bool allow_eio)
+{
+  int got;
+
+  dout(15) << "read " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    dout(10) << "FileStore::read(" << cid << "/" << oid << ") open error: "
+	     << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  if (len == 0) {
+    struct stat st;
+    memset(&st, 0, sizeof(struct stat));
+    int r = ::fstat(**fd, &st);
+    assert(r == 0);
+    len = st.st_size;
+  }
+
+  bufferptr bptr(len);  // prealloc space for entire read
+  got = safe_pread(**fd, bptr.c_str(), len, offset);
+  if (got < 0) {
+    dout(10) << "FileStore::read(" << cid << "/" << oid << ") pread error: " << cpp_strerror(got) << dendl;
+    lfn_close(fd);
+    assert(allow_eio || !m_filestore_fail_eio || got != -EIO);
+    return got;
+  }
+  bptr.set_length(got);   // properly size the buffer
+  bl.push_back(bptr);   // put it in the target bufferlist
+
+  if (m_filestore_sloppy_crc && (!replaying || backend->can_checkpoint())) {
+    ostringstream ss;
+    int errors = backend->_crc_verify_read(**fd, offset, got, bl, &ss);
+    if (errors > 0) {
+      dout(0) << "FileStore::read " << cid << "/" << oid << " " << offset << "~"
+	      << got << " ... BAD CRC:\n" << ss.str() << dendl;
+      assert(0 == "bad crc on read");
+    }
+  }
+
+  lfn_close(fd);
+
+  dout(10) << "FileStore::read " << cid << "/" << oid << " " << offset << "~"
+	   << got << "/" << len << dendl;
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_data_eio(oid)) {
+    return -EIO;
+  } else {
+    return got;
+  }
+}
+
+int FileStore::fiemap(coll_t cid, const ghobject_t& oid,
+                    uint64_t offset, size_t len,
+                    bufferlist& bl)
+{
+  if (!backend->has_fiemap() || len <= (size_t)m_filestore_fiemap_threshold) {
+    map m;
+    m[offset] = len;
+    ::encode(m, bl);
+    return 0;
+  }
+
+
+  struct fiemap *fiemap = NULL;
+  map exomap;
+
+  dout(15) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    dout(10) << "read couldn't open " << cid << "/" << oid << ": " << cpp_strerror(r) << dendl;
+  } else {
+    uint64_t i;
+
+    r = backend->do_fiemap(**fd, offset, len, &fiemap);
+    if (r < 0)
+      goto done;
+
+    if (fiemap->fm_mapped_extents == 0) {
+      free(fiemap);
+      goto done;
+    }
+
+    struct fiemap_extent *extent = &fiemap->fm_extents[0];
+
+    /* start where we were asked to start */
+    if (extent->fe_logical < offset) {
+      extent->fe_length -= offset - extent->fe_logical;
+      extent->fe_logical = offset;
+    }
+
+    i = 0;
+
+    while (i < fiemap->fm_mapped_extents) {
+      struct fiemap_extent *next = extent + 1;
+
+      dout(10) << "FileStore::fiemap() fm_mapped_extents=" << fiemap->fm_mapped_extents
+	       << " fe_logical=" << extent->fe_logical << " fe_length=" << extent->fe_length << dendl;
+
+      /* try to merge extents */
+      while ((i < fiemap->fm_mapped_extents - 1) &&
+             (extent->fe_logical + extent->fe_length == next->fe_logical)) {
+          next->fe_length += extent->fe_length;
+          next->fe_logical = extent->fe_logical;
+          extent = next;
+          next = extent + 1;
+          i++;
+      }
+
+      if (extent->fe_logical + extent->fe_length > offset + len)
+        extent->fe_length = offset + len - extent->fe_logical;
+      exomap[extent->fe_logical] = extent->fe_length;
+      i++;
+      extent++;
+    }
+    free(fiemap);
+  }
+
+done:
+  if (r >= 0) {
+    lfn_close(fd);
+    ::encode(exomap, bl);
+  }
+
+  dout(10) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len << " = " << r << " num_extents=" << exomap.size() << " " << exomap << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+
+int FileStore::_remove(coll_t cid, const ghobject_t& oid,
+		       const SequencerPosition &spos) 
+{
+  dout(15) << "remove " << cid << "/" << oid << dendl;
+  int r = lfn_unlink(cid, oid, spos);
+  dout(10) << "remove " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+int FileStore::_truncate(coll_t cid, const ghobject_t& oid, uint64_t size)
+{
+  dout(15) << "truncate " << cid << "/" << oid << " size " << size << dendl;
+  int r = lfn_truncate(cid, oid, size);
+  dout(10) << "truncate " << cid << "/" << oid << " size " << size << " = " << r << dendl;
+  return r;
+}
+
+
+int FileStore::_touch(coll_t cid, const ghobject_t& oid)
+{
+  dout(15) << "touch " << cid << "/" << oid << dendl;
+
+  FDRef fd;
+  int r = lfn_open(cid, oid, true, &fd);
+  if (r < 0) {
+    return r;
+  } else {
+    lfn_close(fd);
+  }
+  dout(10) << "touch " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+int FileStore::_write(coll_t cid, const ghobject_t& oid,
+                     uint64_t offset, size_t len,
+                     const bufferlist& bl, bool replica)
+{
+  dout(15) << "write " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+  int r;
+
+  int64_t actual;
+
+  FDRef fd;
+  r = lfn_open(cid, oid, true, &fd);
+  if (r < 0) {
+    dout(0) << "write couldn't open " << cid << "/"
+	    << oid << ": "
+	    << cpp_strerror(r) << dendl;
+    goto out;
+  }
+    
+  // seek
+  actual = ::lseek64(**fd, offset, SEEK_SET);
+  if (actual < 0) {
+    r = -errno;
+    dout(0) << "write lseek64 to " << offset << " failed: " << cpp_strerror(r) << dendl;
+    lfn_close(fd);
+    goto out;
+  }
+  if (actual != (int64_t)offset) {
+    dout(0) << "write lseek64 to " << offset << " gave bad offset " << actual << dendl;
+    r = -EIO;
+    lfn_close(fd);
+    goto out;
+  }
+
+  // write
+  r = bl.write_fd(**fd);
+  if (r == 0)
+    r = bl.length();
+
+  if (r >= 0 && m_filestore_sloppy_crc) {
+    int rc = backend->_crc_update_write(**fd, offset, len, bl);
+    assert(rc >= 0);
+  }
+
+  // flush?
+  if (!replaying &&
+      g_conf->filestore_wbthrottle_enable)
+    wbthrottle.queue_wb(fd, oid, offset, len, replica);
+  lfn_close(fd);
+
+ out:
+  dout(10) << "write " << cid << "/" << oid << " " << offset << "~" << len << " = " << r << dendl;
+  return r;
+}
+
+int FileStore::_zero(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len)
+{
+  dout(15) << "zero " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+  int ret = 0;
+
+#ifdef CEPH_HAVE_FALLOCATE
+# if !defined(DARWIN) && !defined(__FreeBSD__)
+  // first try to punch a hole.
+  FDRef fd;
+  ret = lfn_open(cid, oid, false, &fd);
+  if (ret < 0) {
+    goto out;
+  }
+
+  // first try fallocate
+  ret = fallocate(**fd, FALLOC_FL_PUNCH_HOLE, offset, len);
+  if (ret < 0)
+    ret = -errno;
+  lfn_close(fd);
+
+  if (ret >= 0 && m_filestore_sloppy_crc) {
+    int rc = backend->_crc_update_zero(**fd, offset, len);
+    assert(rc >= 0);
+  }
+
+  if (ret == 0)
+    goto out;  // yay!
+  if (ret != -EOPNOTSUPP)
+    goto out;  // some other error
+# endif
+#endif
+
+  // lame, kernel is old and doesn't support it.
+  // write zeros.. yuck!
+  dout(20) << "zero FALLOC_FL_PUNCH_HOLE not supported, falling back to writing zeros" << dendl;
+  {
+    bufferptr bp(len);
+    bp.zero();
+    bufferlist bl;
+    bl.push_back(bp);
+    ret = _write(cid, oid, offset, len, bl);
+  }
+
+ out:
+  dout(20) << "zero " << cid << "/" << oid << " " << offset << "~" << len << " = " << ret << dendl;
+  return ret;
+}
+
+int FileStore::_clone(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid,
+		      const SequencerPosition& spos)
+{
+  dout(15) << "clone " << cid << "/" << oldoid << " -> " << cid << "/" << newoid << dendl;
+
+  if (_check_replay_guard(cid, newoid, spos) < 0)
+    return 0;
+
+  int r;
+  FDRef o, n;
+  {
+    Index index;
+    IndexedPath from, to;
+    r = lfn_open(cid, oldoid, false, &o, &from, &index);
+    if (r < 0) {
+      goto out2;
+    }
+    r = lfn_open(cid, newoid, true, &n, &to, &index);
+    if (r < 0) {
+      goto out;
+    }
+    r = ::ftruncate(**n, 0);
+    if (r < 0) {
+      goto out;
+    }
+    struct stat st;
+    ::fstat(**o, &st);
+    r = _do_clone_range(**o, **n, 0, st.st_size, 0);
+    if (r < 0) {
+      r = -errno;
+      goto out3;
+    }
+    dout(20) << "objectmap clone" << dendl;
+    r = object_map->clone(oldoid, newoid, &spos);
+    if (r < 0 && r != -ENOENT)
+      goto out3;
+  }
+
+  {
+    char buf[2];
+    map aset;
+    r = _fgetattrs(**o, aset, false);
+    if (r < 0)
+      goto out3;
+
+    r = chain_fgetxattr(**o, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+    if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT))) {
+      r = chain_fsetxattr(**n, XATTR_SPILL_OUT_NAME, XATTR_NO_SPILL_OUT,
+                          sizeof(XATTR_NO_SPILL_OUT));
+    } else {
+      r = chain_fsetxattr(**n, XATTR_SPILL_OUT_NAME, XATTR_SPILL_OUT,
+                          sizeof(XATTR_SPILL_OUT));
+    }
+    if (r < 0)
+      goto out3;
+
+    r = _fsetattrs(**n, aset);
+    if (r < 0)
+      goto out3;
+  }
+
+  // clone is non-idempotent; record our work.
+  _set_replay_guard(**n, spos, &newoid);
+
+ out3:
+  lfn_close(n);
+ out:
+  lfn_close(o);
+ out2:
+  dout(10) << "clone " << cid << "/" << oldoid << " -> " << cid << "/" << newoid << " = " << r << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::_do_clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff)
+{
+  dout(20) << "_do_clone_range copy " << srcoff << "~" << len << " to " << dstoff << dendl;
+  return backend->clone_range(from, to, srcoff, len, dstoff);
+}
+
+int FileStore::_do_copy_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff)
+{
+  dout(20) << "_do_copy_range " << srcoff << "~" << len << " to " << dstoff << dendl;
+  int r = 0;
+  int64_t actual;
+
+  actual = ::lseek64(from, srcoff, SEEK_SET);
+  if (actual != (int64_t)srcoff) {
+    r = errno;
+    derr << "lseek64 to " << srcoff << " got " << cpp_strerror(r) << dendl;
+    return r;
+  }
+  actual = ::lseek64(to, dstoff, SEEK_SET);
+  if (actual != (int64_t)dstoff) {
+    r = errno;
+    derr << "lseek64 to " << dstoff << " got " << cpp_strerror(r) << dendl;
+    return r;
+  }
+  
+  loff_t pos = srcoff;
+  loff_t end = srcoff + len;
+  int buflen = 4096*32;
+  char buf[buflen];
+  while (pos < end) {
+    int l = MIN(end-pos, buflen);
+    r = ::read(from, buf, l);
+    dout(25) << "  read from " << pos << "~" << l << " got " << r << dendl;
+    if (r < 0) {
+      if (errno == EINTR) {
+	continue;
+      } else {
+	r = -errno;
+	derr << "FileStore::_do_copy_range: read error at " << pos << "~" << len
+	     << ", " << cpp_strerror(r) << dendl;
+	break;
+      }
+    }
+    if (r == 0) {
+      // hrm, bad source range, wtf.
+      r = -ERANGE;
+      derr << "FileStore::_do_copy_range got short read result at " << pos
+	      << " of fd " << from << " len " << len << dendl;
+      break;
+    }
+    int op = 0;
+    while (op < r) {
+      int r2 = safe_write(to, buf+op, r-op);
+      dout(25) << " write to " << to << " len " << (r-op)
+	       << " got " << r2 << dendl;
+      if (r2 < 0) {
+	r = r2;
+	derr << "FileStore::_do_copy_range: write error at " << pos << "~"
+	     << r-op << ", " << cpp_strerror(r) << dendl;
+
+	break;
+      }
+      op += (r-op);
+    }
+    if (r < 0)
+      break;
+    pos += r;
+  }
+  if (r >= 0 && m_filestore_sloppy_crc) {
+    int rc = backend->_crc_update_clone_range(from, to, srcoff, len, dstoff);
+    assert(rc >= 0);
+  }
+  dout(20) << "_do_copy_range " << srcoff << "~" << len << " to " << dstoff << " = " << r << dendl;
+  return r;
+}
+
+int FileStore::_clone_range(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid,
+			    uint64_t srcoff, uint64_t len, uint64_t dstoff,
+			    const SequencerPosition& spos)
+{
+  dout(15) << "clone_range " << cid << "/" << oldoid << " -> " << cid << "/" << newoid << " " << srcoff << "~" << len << " to " << dstoff << dendl;
+
+  if (_check_replay_guard(cid, newoid, spos) < 0)
+    return 0;
+
+  int r;
+  FDRef o, n;
+  r = lfn_open(cid, oldoid, false, &o);
+  if (r < 0) {
+    goto out2;
+  }
+  r = lfn_open(cid, newoid, true, &n);
+  if (r < 0) {
+    goto out;
+  }
+  r = _do_clone_range(**o, **n, srcoff, len, dstoff);
+
+  // clone is non-idempotent; record our work.
+  _set_replay_guard(**n, spos, &newoid);
+
+  lfn_close(n);
+ out:
+  lfn_close(o);
+ out2:
+  dout(10) << "clone_range " << cid << "/" << oldoid << " -> " << cid << "/" << newoid << " "
+	   << srcoff << "~" << len << " to " << dstoff << " = " << r << dendl;
+  return r;
+}
+
+class SyncEntryTimeout : public Context {
+public:
+  SyncEntryTimeout(int commit_timeo) 
+    : m_commit_timeo(commit_timeo)
+  {
+  }
+
+  void finish(int r) {
+    BackTrace *bt = new BackTrace(1);
+    generic_dout(-1) << "FileStore: sync_entry timed out after "
+	   << m_commit_timeo << " seconds.\n";
+    bt->print(*_dout);
+    *_dout << dendl;
+    delete bt;
+    ceph_abort();
+  }
+private:
+  int m_commit_timeo;
+};
+
+void FileStore::sync_entry()
+{
+  lock.Lock();
+  while (!stop) {
+    utime_t max_interval;
+    max_interval.set_from_double(m_filestore_max_sync_interval);
+    utime_t min_interval;
+    min_interval.set_from_double(m_filestore_min_sync_interval);
+
+    utime_t startwait = ceph_clock_now(g_ceph_context);
+    if (!force_sync) {
+      dout(20) << "sync_entry waiting for max_interval " << max_interval << dendl;
+      sync_cond.WaitInterval(g_ceph_context, lock, max_interval);
+    } else {
+      dout(20) << "sync_entry not waiting, force_sync set" << dendl;
+    }
+
+    if (force_sync) {
+      dout(20) << "sync_entry force_sync set" << dendl;
+      force_sync = false;
+    } else {
+      // wait for at least the min interval
+      utime_t woke = ceph_clock_now(g_ceph_context);
+      woke -= startwait;
+      dout(20) << "sync_entry woke after " << woke << dendl;
+      if (woke < min_interval) {
+	utime_t t = min_interval;
+	t -= woke;
+	dout(20) << "sync_entry waiting for another " << t 
+		 << " to reach min interval " << min_interval << dendl;
+	sync_cond.WaitInterval(g_ceph_context, lock, t);
+      }
+    }
+
+    list fin;
+  again:
+    fin.swap(sync_waiters);
+    lock.Unlock();
+    
+    op_tp.pause();
+    if (apply_manager.commit_start()) {
+      utime_t start = ceph_clock_now(g_ceph_context);
+      uint64_t cp = apply_manager.get_committing_seq();
+
+      sync_entry_timeo_lock.Lock();
+      SyncEntryTimeout *sync_entry_timeo =
+	new SyncEntryTimeout(m_filestore_commit_timeout);
+      timer.add_event_after(m_filestore_commit_timeout, sync_entry_timeo);
+      sync_entry_timeo_lock.Unlock();
+
+      logger->set(l_os_committing, 1);
+
+      // make flusher stop flushing previously queued stuff
+      sync_epoch++;
+
+      dout(15) << "sync_entry committing " << cp << " sync_epoch " << sync_epoch << dendl;
+      stringstream errstream;
+      if (g_conf->filestore_debug_omap_check && !object_map->check(errstream)) {
+	derr << errstream.str() << dendl;
+	assert(0);
+      }
+
+      if (backend->can_checkpoint()) {
+	int err = write_op_seq(op_fd, cp);
+	if (err < 0) {
+	  derr << "Error during write_op_seq: " << cpp_strerror(err) << dendl;
+	  assert(0 == "error during write_op_seq");
+	}
+
+	char s[NAME_MAX];
+	snprintf(s, sizeof(s), COMMIT_SNAP_ITEM, (long long unsigned)cp);
+	uint64_t cid = 0;
+	err = backend->create_checkpoint(s, &cid);
+	if (err < 0) {
+	    int err = errno;
+	    derr << "snap create '" << s << "' got error " << err << dendl;
+	    assert(err == 0);
+	}
+
+	snaps.push_back(cp);
+	apply_manager.commit_started();
+	op_tp.unpause();
+
+	if (cid > 0) {
+	  dout(20) << " waiting for checkpoint " << cid << " to complete" << dendl;
+	  err = backend->sync_checkpoint(cid);
+	  if (err < 0) {
+	    derr << "ioctl WAIT_SYNC got " << cpp_strerror(err) << dendl;
+	    assert(0 == "wait_sync got error");
+	  }
+	  dout(20) << " done waiting for checkpoint" << cid << " to complete" << dendl;
+	}
+      } else
+      {
+	apply_manager.commit_started();
+	op_tp.unpause();
+
+	int err = backend->syncfs();
+	if (err < 0) {
+	  derr << "syncfs got " << cpp_strerror(err) << dendl;
+	  assert(0 == "syncfs returned error");
+	}
+
+	err = write_op_seq(op_fd, cp);
+	if (err < 0) {
+	  derr << "Error during write_op_seq: " << cpp_strerror(err) << dendl;
+	  assert(0 == "error during write_op_seq");
+	}
+	err = ::fsync(op_fd);
+	if (err < 0) {
+	  derr << "Error during fsync of op_seq: " << cpp_strerror(err) << dendl;
+	  assert(0 == "error during fsync of op_seq");
+	}
+      }
+      
+      utime_t done = ceph_clock_now(g_ceph_context);
+      utime_t lat = done - start;
+      utime_t dur = done - startwait;
+      dout(10) << "sync_entry commit took " << lat << ", interval was " << dur << dendl;
+
+      logger->inc(l_os_commit);
+      logger->tinc(l_os_commit_lat, lat);
+      logger->tinc(l_os_commit_len, dur);
+
+      apply_manager.commit_finish();
+      wbthrottle.clear();
+
+      logger->set(l_os_committing, 0);
+
+      // remove old snaps?
+      if (backend->can_checkpoint()) {
+	char s[NAME_MAX];
+	while (snaps.size() > 2) {
+	  snprintf(s, sizeof(s), COMMIT_SNAP_ITEM, (long long unsigned)snaps.front());
+	  snaps.pop_front();
+	  dout(10) << "removing snap '" << s << "'" << dendl;
+	  int r = backend->destroy_checkpoint(s);
+	  if (r) {
+	    int err = errno;
+	    derr << "unable to destroy snap '" << s << "' got " << cpp_strerror(err) << dendl;
+	  }
+	}
+      }
+
+      dout(15) << "sync_entry committed to op_seq " << cp << dendl;
+
+      sync_entry_timeo_lock.Lock();
+      timer.cancel_event(sync_entry_timeo);
+      sync_entry_timeo_lock.Unlock();
+    } else {
+      op_tp.unpause();
+    }
+    
+    lock.Lock();
+    finish_contexts(g_ceph_context, fin, 0);
+    fin.clear();
+    if (!sync_waiters.empty()) {
+      dout(10) << "sync_entry more waiters, committing again" << dendl;
+      goto again;
+    }
+    if (journal && journal->should_commit_now()) {
+      dout(10) << "sync_entry journal says we should commit again (probably is/was full)" << dendl;
+      goto again;
+    }
+  }
+  stop = false;
+  lock.Unlock();
+}
+
+void FileStore::_start_sync()
+{
+  if (!journal) {  // don't do a big sync if the journal is on
+    dout(10) << "start_sync" << dendl;
+    sync_cond.Signal();
+  } else {
+    dout(10) << "start_sync - NOOP (journal is on)" << dendl;
+  }
+}
+
+void FileStore::start_sync()
+{
+  Mutex::Locker l(lock);
+  force_sync = true;
+  sync_cond.Signal();
+}
+
+void FileStore::start_sync(Context *onsafe)
+{
+  Mutex::Locker l(lock);
+  sync_waiters.push_back(onsafe);
+  sync_cond.Signal();
+  dout(10) << "start_sync" << dendl;
+}
+
+void FileStore::sync()
+{
+  Mutex l("FileStore::sync");
+  Cond c;
+  bool done;
+  C_SafeCond *fin = new C_SafeCond(&l, &c, &done);
+
+  start_sync(fin);
+
+  l.Lock();
+  while (!done) {
+    dout(10) << "sync waiting" << dendl;
+    c.Wait(l);
+  }
+  l.Unlock();
+  dout(10) << "sync done" << dendl;
+}
+
+void FileStore::_flush_op_queue()
+{
+  dout(10) << "_flush_op_queue draining op tp" << dendl;
+  op_wq.drain();
+  dout(10) << "_flush_op_queue waiting for apply finisher" << dendl;
+  op_finisher.wait_for_empty();
+}
+
+/*
+ * flush - make every queued write readable
+ */
+void FileStore::flush()
+{
+  dout(10) << "flush" << dendl;
+
+  if (g_conf->filestore_blackhole) {
+    // wait forever
+    Mutex lock("FileStore::flush::lock");
+    Cond cond;
+    lock.Lock();
+    while (true)
+      cond.Wait(lock);
+    assert(0);
+  }
+ 
+  if (m_filestore_journal_writeahead) {
+    if (journal)
+      journal->flush();
+    dout(10) << "flush draining ondisk finisher" << dendl;
+    ondisk_finisher.wait_for_empty();
+  }
+
+  _flush_op_queue();
+  dout(10) << "flush complete" << dendl;
+}
+
+/*
+ * sync_and_flush - make every queued write readable AND committed to disk
+ */
+void FileStore::sync_and_flush()
+{
+  dout(10) << "sync_and_flush" << dendl;
+
+  if (m_filestore_journal_writeahead) {
+    if (journal)
+      journal->flush();
+    _flush_op_queue();
+  } else {
+    // includes m_filestore_journal_parallel
+    _flush_op_queue();
+    sync();
+  }
+  dout(10) << "sync_and_flush done" << dendl;
+}
+
+int FileStore::snapshot(const string& name)
+{
+  dout(10) << "snapshot " << name << dendl;
+  sync_and_flush();
+
+  if (!backend->can_checkpoint()) {
+    dout(0) << "snapshot " << name << " failed, not supported" << dendl;
+    return -EOPNOTSUPP;
+  }
+
+  char s[NAME_MAX];
+  snprintf(s, sizeof(s), CLUSTER_SNAP_ITEM, name.c_str());
+
+  int r = backend->create_checkpoint(s, NULL);
+  if (r) {
+    r = -errno;
+    derr << "snapshot " << name << " failed: " << cpp_strerror(r) << dendl;
+  }
+
+  return r;
+}
+
+// -------------------------------
+// attributes
+
+int FileStore::_fgetattr(int fd, const char *name, bufferptr& bp)
+{
+  char val[100];
+  int l = chain_fgetxattr(fd, name, val, sizeof(val));
+  if (l >= 0) {
+    bp = buffer::create(l);
+    memcpy(bp.c_str(), val, l);
+  } else if (l == -ERANGE) {
+    l = chain_fgetxattr(fd, name, 0, 0);
+    if (l > 0) {
+      bp = buffer::create(l);
+      l = chain_fgetxattr(fd, name, bp.c_str(), l);
+    }
+  }
+  assert(!m_filestore_fail_eio || l != -EIO);
+  return l;
+}
+
+int FileStore::_fgetattrs(int fd, map& aset, bool user_only)
+{
+  // get attr list
+  char names1[100];
+  int len = chain_flistxattr(fd, names1, sizeof(names1)-1);
+  char *names2 = 0;
+  char *name = 0;
+  if (len == -ERANGE) {
+    len = chain_flistxattr(fd, 0, 0);
+    if (len < 0) {
+      assert(!m_filestore_fail_eio || len != -EIO);
+      return len;
+    }
+    dout(10) << " -ERANGE, len is " << len << dendl;
+    names2 = new char[len+1];
+    len = chain_flistxattr(fd, names2, len);
+    dout(10) << " -ERANGE, got " << len << dendl;
+    if (len < 0) {
+      assert(!m_filestore_fail_eio || len != -EIO);
+      return len;
+    }
+    name = names2;
+  } else if (len < 0) {
+    assert(!m_filestore_fail_eio || len != -EIO);
+    return len;
+  } else {
+    name = names1;
+  }
+  name[len] = 0;
+
+  char *end = name + len;
+  while (name < end) {
+    char *attrname = name;
+    if (parse_attrname(&name)) {
+      char *set_name = name;
+      bool can_get = true;
+      if (user_only) {
+	if (*set_name =='_')
+	  set_name++;
+	else
+	  can_get = false;
+      }
+      if (*set_name && can_get) {
+        dout(20) << "fgetattrs " << fd << " getting '" << name << "'" << dendl;
+        int r = _fgetattr(fd, attrname, aset[set_name]);
+        if (r < 0)
+	  return r;
+      }
+    }
+    name += strlen(name) + 1;
+  }
+
+  delete[] names2;
+  return 0;
+}
+
+int FileStore::_fsetattrs(int fd, map &aset)
+{
+  for (map::iterator p = aset.begin();
+       p != aset.end();
+       ++p) {
+    char n[CHAIN_XATTR_MAX_NAME_LEN];
+    get_attrname(p->first.c_str(), n, CHAIN_XATTR_MAX_NAME_LEN);
+    const char *val;
+    if (p->second.length())
+      val = p->second.c_str();
+    else
+      val = "";
+    // ??? Why do we skip setting all the other attrs if one fails?
+    int r = chain_fsetxattr(fd, n, val, p->second.length());
+    if (r < 0) {
+      derr << "FileStore::_setattrs: chain_setxattr returned " << r << dendl;
+      return r;
+    }
+  }
+  return 0;
+}
+
+// debug EIO injection
+void FileStore::inject_data_error(const ghobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  dout(10) << __func__ << ": init error on " << oid << dendl;
+  data_error_set.insert(oid);
+}
+void FileStore::inject_mdata_error(const ghobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  dout(10) << __func__ << ": init error on " << oid << dendl;
+  mdata_error_set.insert(oid);
+}
+void FileStore::debug_obj_on_delete(const ghobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  dout(10) << __func__ << ": clear error on " << oid << dendl;
+  data_error_set.erase(oid);
+  mdata_error_set.erase(oid);
+}
+bool FileStore::debug_data_eio(const ghobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  if (data_error_set.count(oid)) {
+    dout(10) << __func__ << ": inject error on " << oid << dendl;
+    return true;
+  } else {
+    return false;
+  }
+}
+bool FileStore::debug_mdata_eio(const ghobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  if (mdata_error_set.count(oid)) {
+    dout(10) << __func__ << ": inject error on " << oid << dendl;
+    return true;
+  } else {
+    return false;
+  }
+}
+
+
+// objects
+
+int FileStore::getattr(coll_t cid, const ghobject_t& oid, const char *name, bufferptr &bp)
+{
+  dout(15) << "getattr " << cid << "/" << oid << " '" << name << "'" << dendl;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    goto out;
+  }
+  char n[CHAIN_XATTR_MAX_NAME_LEN];
+  get_attrname(name, n, CHAIN_XATTR_MAX_NAME_LEN);
+  r = _fgetattr(**fd, n, bp);
+  lfn_close(fd);
+  if (r == -ENODATA) {
+    map got;
+    set to_get;
+    to_get.insert(string(name));
+    Index index;
+    r = get_index(cid, &index);
+    if (r < 0) {
+      dout(10) << __func__ << " could not get index r = " << r << dendl;
+      goto out;
+    }
+    r = object_map->get_xattrs(oid, to_get, &got);
+    if (r < 0 && r != -ENOENT) {
+      dout(10) << __func__ << " get_xattrs err r =" << r << dendl;
+      goto out;
+    }
+    if (got.empty()) {
+      dout(10) << __func__ << " got.size() is 0" << dendl;
+      return -ENODATA;
+    }
+    bp = bufferptr(got.begin()->second.c_str(),
+		   got.begin()->second.length());
+    r = bp.length();
+  }
+ out:
+  dout(10) << "getattr " << cid << "/" << oid << " '" << name << "' = " << r << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_mdata_eio(oid)) {
+    return -EIO;
+  } else {
+    return r;
+  }
+}
+
+int FileStore::getattrs(coll_t cid, const ghobject_t& oid, map& aset, bool user_only)
+{
+  set omap_attrs;
+  map omap_aset;
+  Index index;
+  dout(15) << "getattrs " << cid << "/" << oid << dendl;
+  FDRef fd;
+  bool spill_out = true;
+  char buf[2];
+
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    goto out;
+  }
+
+  r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+  if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT)))
+    spill_out = false;
+
+  r = _fgetattrs(**fd, aset, user_only);
+  if (r < 0) {
+    goto out;
+  }
+  lfn_close(fd);
+
+  if (!spill_out) {
+    dout(10) << __func__ << " no xattr exists in object_map r = " << r << dendl;
+    goto out;
+  }
+
+  r = get_index(cid, &index);
+  if (r < 0) {
+    dout(10) << __func__ << " could not get index r = " << r << dendl;
+    goto out;
+  }
+  r = object_map->get_all_xattrs(oid, &omap_attrs);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " could not get omap_attrs r = " << r << dendl;
+    goto out;
+  }
+
+  r = object_map->get_xattrs(oid, omap_attrs, &omap_aset);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " could not get omap_attrs r = " << r << dendl;
+    goto out;
+  }
+  if (r == -ENOENT)
+    r = 0;
+  assert(omap_attrs.size() == omap_aset.size());
+  for (map::iterator i = omap_aset.begin();
+	 i != omap_aset.end();
+	 ++i) {
+    string key;
+    if (user_only) {
+	if (i->first[0] != '_')
+	  continue;
+	if (i->first == "_")
+	  continue;
+	key = i->first.substr(1, i->first.size());
+    } else {
+	key = i->first;
+    }
+    aset.insert(make_pair(key,
+			    bufferptr(i->second.c_str(), i->second.length())));
+  }
+ out:
+  dout(10) << "getattrs " << cid << "/" << oid << " = " << r << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_mdata_eio(oid)) {
+    return -EIO;
+  } else {
+    return r;
+  }
+}
+
+int FileStore::_setattrs(coll_t cid, const ghobject_t& oid, map& aset,
+			 const SequencerPosition &spos)
+{
+  map omap_set;
+  set omap_remove;
+  map inline_set;
+  map inline_to_set;
+  FDRef fd;
+  int spill_out = -1;
+
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    goto out;
+  }
+
+  char buf[2];
+  r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+  if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT)))
+    spill_out = 0;
+  else
+    spill_out = 1;
+
+  r = _fgetattrs(**fd, inline_set, false);
+  assert(!m_filestore_fail_eio || r != -EIO);
+  dout(15) << "setattrs " << cid << "/" << oid << dendl;
+  r = 0;
+
+  for (map::iterator p = aset.begin();
+       p != aset.end();
+       ++p) {
+    char n[CHAIN_XATTR_MAX_NAME_LEN];
+    get_attrname(p->first.c_str(), n, CHAIN_XATTR_MAX_NAME_LEN);
+
+    if (p->second.length() > m_filestore_max_inline_xattr_size) {
+	if (inline_set.count(p->first)) {
+	  inline_set.erase(p->first);
+	  r = chain_fremovexattr(**fd, n);
+	  if (r < 0)
+	    goto out_close;
+	}
+	omap_set[p->first].push_back(p->second);
+	continue;
+    }
+
+    if (!inline_set.count(p->first) &&
+	  inline_set.size() >= m_filestore_max_inline_xattrs) {
+	if (inline_set.count(p->first)) {
+	  inline_set.erase(p->first);
+	  r = chain_fremovexattr(**fd, n);
+	  if (r < 0)
+	    goto out_close;
+	}
+	omap_set[p->first].push_back(p->second);
+	continue;
+    }
+    omap_remove.insert(p->first);
+    inline_set.insert(*p);
+
+    inline_to_set.insert(*p);
+  }
+
+  if (spill_out != 1 && !omap_set.empty()) {
+    chain_fsetxattr(**fd, XATTR_SPILL_OUT_NAME, XATTR_SPILL_OUT,
+		    sizeof(XATTR_SPILL_OUT));
+  }
+
+  r = _fsetattrs(**fd, inline_to_set);
+  if (r < 0)
+    goto out_close;
+
+  if (spill_out && !omap_remove.empty()) {
+    r = object_map->remove_xattrs(oid, omap_remove, &spos);
+    if (r < 0 && r != -ENOENT) {
+      dout(10) << __func__ << " could not remove_xattrs r = " << r << dendl;
+      assert(!m_filestore_fail_eio || r != -EIO);
+      goto out_close;
+    } else {
+      r = 0; // don't confuse the debug output
+    }
+  }
+
+  if (!omap_set.empty()) {
+    r = object_map->set_xattrs(oid, omap_set, &spos);
+    if (r < 0) {
+      dout(10) << __func__ << " could not set_xattrs r = " << r << dendl;
+      assert(!m_filestore_fail_eio || r != -EIO);
+      goto out_close;
+    }
+  }
+ out_close:
+  lfn_close(fd);
+ out:
+  dout(10) << "setattrs " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+
+int FileStore::_rmattr(coll_t cid, const ghobject_t& oid, const char *name,
+		       const SequencerPosition &spos)
+{
+  dout(15) << "rmattr " << cid << "/" << oid << " '" << name << "'" << dendl;
+  FDRef fd;
+  bool spill_out = true;
+  bufferptr bp;
+
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    goto out;
+  }
+
+  char buf[2];
+  r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+  if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT))) {
+    spill_out = false;
+  }
+
+  char n[CHAIN_XATTR_MAX_NAME_LEN];
+  get_attrname(name, n, CHAIN_XATTR_MAX_NAME_LEN);
+  r = chain_fremovexattr(**fd, n);
+  if (r == -ENODATA && spill_out) {
+    Index index;
+    r = get_index(cid, &index);
+    if (r < 0) {
+      dout(10) << __func__ << " could not get index r = " << r << dendl;
+      goto out_close;
+    }
+    set to_remove;
+    to_remove.insert(string(name));
+    r = object_map->remove_xattrs(oid, to_remove, &spos);
+    if (r < 0 && r != -ENOENT) {
+      dout(10) << __func__ << " could not remove_xattrs index r = " << r << dendl;
+      assert(!m_filestore_fail_eio || r != -EIO);
+      goto out_close;
+    }
+  }
+ out_close:
+  lfn_close(fd);
+ out:
+  dout(10) << "rmattr " << cid << "/" << oid << " '" << name << "' = " << r << dendl;
+  return r;
+}
+
+int FileStore::_rmattrs(coll_t cid, const ghobject_t& oid,
+			const SequencerPosition &spos)
+{
+  dout(15) << "rmattrs " << cid << "/" << oid << dendl;
+
+  map aset;
+  FDRef fd;
+  set omap_attrs;
+  Index index;
+  bool spill_out = true;
+
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    goto out;
+  }
+
+  char buf[2];
+  r = chain_fgetxattr(**fd, XATTR_SPILL_OUT_NAME, buf, sizeof(buf));
+  if (r >= 0 && !strncmp(buf, XATTR_NO_SPILL_OUT, sizeof(XATTR_NO_SPILL_OUT))) {
+    spill_out = false;
+  }
+
+  r = _fgetattrs(**fd, aset, false);
+  if (r >= 0) {
+    for (map::iterator p = aset.begin(); p != aset.end(); ++p) {
+      char n[CHAIN_XATTR_MAX_NAME_LEN];
+      get_attrname(p->first.c_str(), n, CHAIN_XATTR_MAX_NAME_LEN);
+      r = chain_fremovexattr(**fd, n);
+      if (r < 0)
+	break;
+    }
+  }
+
+  if (!spill_out) {
+    dout(10) << __func__ << " no xattr exists in object_map r = " << r << dendl;
+    goto out_close;
+  }
+
+  r = get_index(cid, &index);
+  if (r < 0) {
+    dout(10) << __func__ << " could not get index r = " << r << dendl;
+    goto out_close;
+  }
+  r = object_map->get_all_xattrs(oid, &omap_attrs);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " could not get omap_attrs r = " << r << dendl;
+    assert(!m_filestore_fail_eio || r != -EIO);
+    goto out_close;
+  }
+  r = object_map->remove_xattrs(oid, omap_attrs, &spos);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " could not remove omap_attrs r = " << r << dendl;
+    goto out_close;
+  }
+  if (r == -ENOENT)
+    r = 0;
+
+  chain_fsetxattr(**fd, XATTR_SPILL_OUT_NAME, XATTR_NO_SPILL_OUT,
+		  sizeof(XATTR_NO_SPILL_OUT));
+
+ out_close:
+  lfn_close(fd);
+ out:
+  dout(10) << "rmattrs " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+
+
+// collections
+
+int FileStore::collection_getattr(coll_t c, const char *name,
+				  void *value, size_t size) 
+{
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(15) << "collection_getattr " << fn << " '" << name << "' len " << size << dendl;
+  int r;
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    r = -errno;
+    goto out;
+  }
+  char n[PATH_MAX];
+  get_attrname(name, n, PATH_MAX);
+  r = chain_fgetxattr(fd, n, value, size);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+ out:
+  dout(10) << "collection_getattr " << fn << " '" << name << "' len " << size << " = " << r << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::collection_getattr(coll_t c, const char *name, bufferlist& bl)
+{
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(15) << "collection_getattr " << fn << " '" << name << "'" << dendl;
+  char n[PATH_MAX];
+  get_attrname(name, n, PATH_MAX);
+  buffer::ptr bp;
+  int r;
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    r = -errno;
+    goto out;
+  }
+  r = _fgetattr(fd, n, bp);
+  bl.push_back(bp);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+ out:
+  dout(10) << "collection_getattr " << fn << " '" << name << "' = " << r << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::collection_getattrs(coll_t cid, map& aset) 
+{
+  char fn[PATH_MAX];
+  get_cdir(cid, fn, sizeof(fn));
+  dout(10) << "collection_getattrs " << fn << dendl;
+  int r = 0;
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    r = -errno;
+    goto out;
+  }
+  r = _fgetattrs(fd, aset, true);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+ out:
+  dout(10) << "collection_getattrs " << fn << " = " << r << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+
+int FileStore::_collection_setattr(coll_t c, const char *name,
+				  const void *value, size_t size) 
+{
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(10) << "collection_setattr " << fn << " '" << name << "' len " << size << dendl;
+  char n[PATH_MAX];
+  int r;
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    r = -errno;
+    goto out;
+  }
+  get_attrname(name, n, PATH_MAX);
+  r = chain_fsetxattr(fd, n, value, size);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+ out:
+  dout(10) << "collection_setattr " << fn << " '" << name << "' len " << size << " = " << r << dendl;
+  return r;
+}
+
+int FileStore::_collection_rmattr(coll_t c, const char *name) 
+{
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(15) << "collection_rmattr " << fn << dendl;
+  char n[PATH_MAX];
+  get_attrname(name, n, PATH_MAX);
+  int r;
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    r = -errno;
+    goto out;
+  }
+  r = chain_fremovexattr(fd, n);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+ out:
+  dout(10) << "collection_rmattr " << fn << " = " << r << dendl;
+  return r;
+}
+
+
+int FileStore::_collection_setattrs(coll_t cid, map& aset) 
+{
+  char fn[PATH_MAX];
+  get_cdir(cid, fn, sizeof(fn));
+  dout(15) << "collection_setattrs " << fn << dendl;
+  int r = 0;
+  int fd = ::open(fn, O_RDONLY);
+  if (fd < 0) {
+    r = -errno;
+    goto out;
+  }
+  for (map::iterator p = aset.begin();
+       p != aset.end();
+       ++p) {
+    char n[PATH_MAX];
+    get_attrname(p->first.c_str(), n, PATH_MAX);
+    r = chain_fsetxattr(fd, n, p->second.c_str(), p->second.length());
+    if (r < 0)
+      break;
+  }
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+ out:
+  dout(10) << "collection_setattrs " << fn << " = " << r << dendl;
+  return r;
+}
+
+int FileStore::_collection_remove_recursive(const coll_t &cid,
+					    const SequencerPosition &spos)
+{
+  struct stat st;
+  int r = collection_stat(cid, &st);
+  if (r < 0) {
+    if (r == -ENOENT)
+      return 0;
+    return r;
+  }
+
+  vector objects;
+  ghobject_t max;
+  r = 0;
+  while (!max.is_max()) {
+    r = collection_list_partial(cid, max, 200, 300, 0, &objects, &max);
+    if (r < 0)
+      return r;
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i) {
+      assert(_check_replay_guard(cid, *i, spos));
+      r = _remove(cid, *i, spos);
+      if (r < 0)
+	return r;
+    }
+  }
+  return _destroy_collection(cid);
+}
+
+int FileStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
+				  const SequencerPosition& spos)
+{
+  char new_coll[PATH_MAX], old_coll[PATH_MAX];
+  get_cdir(cid, old_coll, sizeof(old_coll));
+  get_cdir(ncid, new_coll, sizeof(new_coll));
+
+  if (_check_replay_guard(cid, spos) < 0) {
+    return 0;
+  }
+
+  if (_check_replay_guard(ncid, spos) < 0) {
+    return _collection_remove_recursive(cid, spos);
+  }
+
+  if (!collection_exists(cid)) {
+    if (replaying) {
+      // already happened
+      return 0;
+    } else {
+      return -ENOENT;
+    }
+  }
+  _set_global_replay_guard(cid, spos);
+
+  int ret = 0;
+  if (::rename(old_coll, new_coll)) {
+    if (replaying && !backend->can_checkpoint() &&
+	(errno == EEXIST || errno == ENOTEMPTY))
+      ret = _collection_remove_recursive(cid, spos);
+    else
+      ret = -errno;
+
+    dout(10) << "collection_rename '" << cid << "' to '" << ncid << "'"
+	     << ": ret = " << ret << dendl;
+    return ret;
+  }
+
+  if (ret >= 0) {
+    int fd = ::open(new_coll, O_RDONLY);
+    assert(fd >= 0);
+    _set_replay_guard(fd, spos);
+    VOID_TEMP_FAILURE_RETRY(::close(fd));
+  }
+
+  dout(10) << "collection_rename '" << cid << "' to '" << ncid << "'"
+	   << ": ret = " << ret << dendl;
+  return ret;
+}
+
+// --------------------------
+// collections
+
+int FileStore::collection_version_current(coll_t c, uint32_t *version)
+{
+  Index index;
+  int r = get_index(c, &index);
+  if (r < 0)
+    return r;
+  *version = index->collection_version();
+  if (*version == target_version)
+    return 1;
+  else 
+    return 0;
+}
+
+int FileStore::list_collections(vector& ls) 
+{
+  dout(10) << "list_collections" << dendl;
+
+  char fn[PATH_MAX];
+  snprintf(fn, sizeof(fn), "%s/current", basedir.c_str());
+
+  int r = 0;
+  DIR *dir = ::opendir(fn);
+  if (!dir) {
+    r = -errno;
+    derr << "tried opening directory " << fn << ": " << cpp_strerror(-r) << dendl;
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+
+  char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
+  struct dirent *de;
+  while ((r = ::readdir_r(dir, (struct dirent *)&buf, &de)) == 0) {
+    if (!de)
+      break;
+    if (de->d_type == DT_UNKNOWN) {
+      // d_type not supported (non-ext[234], btrfs), must stat
+      struct stat sb;
+      char filename[PATH_MAX];
+      snprintf(filename, sizeof(filename), "%s/%s", fn, de->d_name);
+
+      r = ::stat(filename, &sb);
+      if (r < 0) {
+	r = -errno;
+	derr << "stat on " << filename << ": " << cpp_strerror(-r) << dendl;
+	assert(!m_filestore_fail_eio || r != -EIO);
+	break;
+      }
+      if (!S_ISDIR(sb.st_mode)) {
+	continue;
+      }
+    } else if (de->d_type != DT_DIR) {
+      continue;
+    }
+    if (strcmp(de->d_name, "omap") == 0) {
+      continue;
+    }
+    if (de->d_name[0] == '.' &&
+	(de->d_name[1] == '\0' ||
+	 (de->d_name[1] == '.' &&
+	  de->d_name[2] == '\0')))
+      continue;
+    ls.push_back(coll_t(de->d_name));
+  }
+
+  if (r > 0) {
+    derr << "trying readdir_r " << fn << ": " << cpp_strerror(r) << dendl;
+    r = -r;
+  }
+
+  ::closedir(dir);
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::collection_stat(coll_t c, struct stat *st) 
+{
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(15) << "collection_stat " << fn << dendl;
+  int r = ::stat(fn, st);
+  if (r < 0)
+    r = -errno;
+  dout(10) << "collection_stat " << fn << " = " << r << dendl;
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+bool FileStore::collection_exists(coll_t c) 
+{
+  struct stat st;
+  return collection_stat(c, &st) == 0;
+}
+
+bool FileStore::collection_empty(coll_t c) 
+{  
+  dout(15) << "collection_empty " << c << dendl;
+  Index index;
+  int r = get_index(c, &index);
+  if (r < 0)
+    return false;
+  vector ls;
+  collection_list_handle_t handle;
+  r = index->collection_list_partial(ghobject_t(), 1, 1, 0, &ls, NULL);
+  if (r < 0) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return false;
+  }
+  return ls.empty();
+}
+
+int FileStore::collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
+                                     snapid_t seq, vector *ls)
+{
+  bool done = false;
+  ghobject_t next = start;
+
+  while (!done) {
+    vector next_objects;
+    int r = collection_list_partial(c, next,
+                                get_ideal_list_min(), get_ideal_list_max(),
+                                seq, &next_objects, &next);
+    if (r < 0)
+      return r;
+
+    ls->insert(ls->end(), next_objects.begin(), next_objects.end());
+
+    // special case for empty collection
+    if (ls->empty()) {
+      break;
+    }
+
+    while (!ls->empty() && ls->back() >= end) {
+      ls->pop_back();
+      done = true;
+    }
+
+    if (next >= end) {
+      done = true;
+    }
+  }
+
+  return 0;
+}
+
+int FileStore::collection_list_partial(coll_t c, ghobject_t start,
+				       int min, int max, snapid_t seq,
+				       vector *ls, ghobject_t *next)
+{
+  dout(10) << "collection_list_partial: " << c << dendl;
+  Index index;
+  int r = get_index(c, &index);
+  if (r < 0)
+    return r;
+  r = index->collection_list_partial(start,
+				     min, max, seq,
+				     ls, next);
+  if (r < 0) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  if (ls)
+    dout(20) << "objects: " << *ls << dendl;
+  return 0;
+}
+
+int FileStore::collection_list(coll_t c, vector& ls)
+{  
+  Index index;
+  int r = get_index(c, &index);
+  if (r < 0)
+    return r;
+  r = index->collection_list(&ls);
+  assert(!m_filestore_fail_eio || r != -EIO);
+  return r;
+}
+
+int FileStore::omap_get(coll_t c, const ghobject_t &hoid,
+			bufferlist *header,
+			map *out)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(c, hoid, &path);
+  if (r < 0)
+    return r;
+  r = object_map->get(hoid, header, out);
+  if (r < 0 && r != -ENOENT) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  return 0;
+}
+
+int FileStore::omap_get_header(
+  coll_t c,
+  const ghobject_t &hoid,
+  bufferlist *bl,
+  bool allow_eio)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(c, hoid, &path);
+  if (r < 0)
+    return r;
+  r = object_map->get_header(hoid, bl);
+  if (r < 0 && r != -ENOENT) {
+    assert(allow_eio || !m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  return 0;
+}
+
+int FileStore::omap_get_keys(coll_t c, const ghobject_t &hoid, set *keys)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(c, hoid, &path);
+  if (r < 0)
+    return r;
+  r = object_map->get_keys(hoid, keys);
+  if (r < 0 && r != -ENOENT) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  return 0;
+}
+
+int FileStore::omap_get_values(coll_t c, const ghobject_t &hoid,
+			       const set &keys,
+			       map *out)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(c, hoid, &path);
+  if (r < 0)
+    return r;
+  r = object_map->get_values(hoid, keys, out);
+  if (r < 0 && r != -ENOENT) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  return 0;
+}
+
+int FileStore::omap_check_keys(coll_t c, const ghobject_t &hoid,
+			       const set &keys,
+			       set *out)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(c, hoid, &path);
+  if (r < 0)
+    return r;
+  r = object_map->check_keys(hoid, keys, out);
+  if (r < 0 && r != -ENOENT) {
+    assert(!m_filestore_fail_eio || r != -EIO);
+    return r;
+  }
+  return 0;
+}
+
+ObjectMap::ObjectMapIterator FileStore::get_omap_iterator(coll_t c,
+							  const ghobject_t &hoid)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(c, hoid, &path);
+  if (r < 0)
+    return ObjectMap::ObjectMapIterator();
+  return object_map->get_iterator(hoid);
+}
+
+int FileStore::_create_collection(
+  coll_t c,
+  const SequencerPosition &spos)
+{
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(15) << "create_collection " << fn << dendl;
+  int r = ::mkdir(fn, 0755);
+  if (r < 0)
+    r = -errno;
+  if (r == -EEXIST && replaying)
+    r = 0;
+  dout(10) << "create_collection " << fn << " = " << r << dendl;
+
+  if (r < 0)
+    return r;
+  r = init_index(c);
+  if (r < 0)
+    return r;
+  _set_replay_guard(c, spos);
+  return 0;
+}
+
+// DEPRECATED -- remove with _split_collection_create
+int FileStore::_create_collection(coll_t c) 
+{
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(15) << "create_collection " << fn << dendl;
+  int r = ::mkdir(fn, 0755);
+  if (r < 0)
+    r = -errno;
+  dout(10) << "create_collection " << fn << " = " << r << dendl;
+
+  if (r < 0)
+    return r;
+  return init_index(c);
+}
+
+int FileStore::_destroy_collection(coll_t c) 
+{
+  {
+    Index from;
+    int r = get_index(c, &from);
+    if (r < 0)
+      return r;
+    r = from->prep_delete();
+    if (r < 0)
+      return r;
+  }
+  char fn[PATH_MAX];
+  get_cdir(c, fn, sizeof(fn));
+  dout(15) << "_destroy_collection " << fn << dendl;
+  int r = ::rmdir(fn);
+  if (r < 0)
+    r = -errno;
+  dout(10) << "_destroy_collection " << fn << " = " << r << dendl;
+  return r;
+}
+
+
+int FileStore::_collection_add(coll_t c, coll_t oldcid, const ghobject_t& o,
+			       const SequencerPosition& spos)
+{
+  dout(15) << "collection_add " << c << "/" << o << " from " << oldcid << "/" << o << dendl;
+  
+  int dstcmp = _check_replay_guard(c, o, spos);
+  if (dstcmp < 0)
+    return 0;
+
+  // check the src name too; it might have a newer guard, and we don't
+  // want to clobber it
+  int srccmp = _check_replay_guard(oldcid, o, spos);
+  if (srccmp < 0)
+    return 0;
+
+  // open guard on object so we don't any previous operations on the
+  // new name that will modify the source inode.
+  FDRef fd;
+  int r = lfn_open(oldcid, o, 0, &fd);
+  if (r < 0) {
+    // the source collection/object does not exist. If we are replaying, we
+    // should be safe, so just return 0 and move on.
+    assert(replaying);
+    dout(10) << "collection_add " << c << "/" << o << " from "
+	     << oldcid << "/" << o << " (dne, continue replay) " << dendl;
+    return 0;
+  }
+  if (dstcmp > 0) {      // if dstcmp == 0 the guard already says "in-progress"
+    _set_replay_guard(**fd, spos, &o, true);
+  }
+
+  r = lfn_link(oldcid, c, o, o);
+  if (replaying && !backend->can_checkpoint() &&
+      r == -EEXIST)    // crashed between link() and set_replay_guard()
+    r = 0;
+
+  _inject_failure();
+
+  // close guard on object so we don't do this again
+  if (r == 0) {
+    _close_replay_guard(**fd, spos);
+  }
+  lfn_close(fd);
+
+  dout(10) << "collection_add " << c << "/" << o << " from " << oldcid << "/" << o << " = " << r << dendl;
+  return r;
+}
+
+int FileStore::_collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+				       coll_t c, const ghobject_t& o,
+				       const SequencerPosition& spos)
+{
+  dout(15) << __func__ << " " << c << "/" << o << " from " << oldcid << "/" << oldoid << dendl;
+  int r = 0;
+  int dstcmp, srccmp;
+
+  if (replaying) {
+    /* If the destination collection doesn't exist during replay,
+     * we need to delete the src object and continue on
+     */
+    if (!collection_exists(c))
+      goto out_rm_src;
+  }
+
+  dstcmp = _check_replay_guard(c, o, spos);
+  if (dstcmp < 0)
+    goto out_rm_src;
+
+  // check the src name too; it might have a newer guard, and we don't
+  // want to clobber it
+  srccmp = _check_replay_guard(oldcid, oldoid, spos);
+  if (srccmp < 0)
+    return 0;
+
+  {
+    // open guard on object so we don't any previous operations on the
+    // new name that will modify the source inode.
+    FDRef fd;
+    r = lfn_open(oldcid, oldoid, 0, &fd);
+    if (r < 0) {
+      // the source collection/object does not exist. If we are replaying, we
+      // should be safe, so just return 0 and move on.
+      assert(replaying);
+      dout(10) << __func__ << " " << c << "/" << o << " from "
+	       << oldcid << "/" << oldoid << " (dne, continue replay) " << dendl;
+      return 0;
+    }
+    if (dstcmp > 0) {      // if dstcmp == 0 the guard already says "in-progress"
+      _set_replay_guard(**fd, spos, &o, true);
+    }
+
+    r = lfn_link(oldcid, c, oldoid, o);
+    if (replaying && !backend->can_checkpoint() &&
+	r == -EEXIST)    // crashed between link() and set_replay_guard()
+      r = 0;
+
+    _inject_failure();
+
+    if (r == 0) {
+      // the name changed; link the omap content
+      r = object_map->clone(oldoid, o, &spos);
+      if (r == -ENOENT)
+	r = 0;
+    }
+
+    _inject_failure();
+
+    lfn_close(fd);
+    fd = FDRef();
+
+    if (r == 0)
+      r = lfn_unlink(oldcid, oldoid, spos, true);
+
+    if (r == 0)
+      r = lfn_open(c, o, 0, &fd);
+
+    // close guard on object so we don't do this again
+    if (r == 0)
+      _close_replay_guard(**fd, spos);
+
+    lfn_close(fd);
+  }
+
+  dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/" << oldoid
+	   << " = " << r << dendl;
+  return r;
+
+ out_rm_src:
+  // remove source
+  if (_check_replay_guard(oldcid, oldoid, spos) > 0) {
+    r = lfn_unlink(oldcid, oldoid, spos, true);
+  }
+
+  dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/" << oldoid
+	   << " = " << r << dendl;
+  return r;
+}
+
+void FileStore::_inject_failure()
+{
+  if (m_filestore_kill_at.read()) {
+    int final = m_filestore_kill_at.dec();
+    dout(5) << "_inject_failure " << (final+1) << " -> " << final << dendl;
+    if (final == 0) {
+      derr << "_inject_failure KILLING" << dendl;
+      g_ceph_context->_log->flush();
+      _exit(1);
+    }
+  }
+}
+
+int FileStore::_omap_clear(coll_t cid, const ghobject_t &hoid,
+			   const SequencerPosition &spos) {
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(cid, hoid, &path);
+  if (r < 0)
+    return r;
+  r = object_map->clear_keys_header(hoid, &spos);
+  if (r < 0 && r != -ENOENT)
+    return r;
+  return 0;
+}
+
+int FileStore::_omap_setkeys(coll_t cid, const ghobject_t &hoid,
+			     const map &aset,
+			     const SequencerPosition &spos) {
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(cid, hoid, &path);
+  if (r < 0)
+    return r;
+  return object_map->set_keys(hoid, aset, &spos);
+}
+
+int FileStore::_omap_rmkeys(coll_t cid, const ghobject_t &hoid,
+			    const set &keys,
+			    const SequencerPosition &spos) {
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(cid, hoid, &path);
+  if (r < 0)
+    return r;
+  r = object_map->rm_keys(hoid, keys, &spos);
+  if (r < 0 && r != -ENOENT)
+    return r;
+  return 0;
+}
+
+int FileStore::_omap_rmkeyrange(coll_t cid, const ghobject_t &hoid,
+				const string& first, const string& last,
+				const SequencerPosition &spos) {
+  dout(15) << __func__ << " " << cid << "/" << hoid << " [" << first << "," << last << "]" << dendl;
+  set keys;
+  {
+    ObjectMap::ObjectMapIterator iter = get_omap_iterator(cid, hoid);
+    if (!iter)
+      return -ENOENT;
+    for (iter->lower_bound(first); iter->valid() && iter->key() < last;
+	 iter->next()) {
+      keys.insert(iter->key());
+    }
+  }
+  return _omap_rmkeys(cid, hoid, keys, spos);
+}
+
+int FileStore::_omap_setheader(coll_t cid, const ghobject_t &hoid,
+			       const bufferlist &bl,
+			       const SequencerPosition &spos)
+{
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+  IndexedPath path;
+  int r = lfn_find(cid, hoid, &path);
+  if (r < 0)
+    return r;
+  return object_map->set_header(hoid, bl, &spos);
+}
+
+int FileStore::_split_collection(coll_t cid,
+				 uint32_t bits,
+				 uint32_t rem,
+				 coll_t dest,
+				 const SequencerPosition &spos)
+{
+  int r;
+  {
+    dout(15) << __func__ << " " << cid << " bits: " << bits << dendl;
+    if (!collection_exists(cid)) {
+      dout(2) << __func__ << ": " << cid << " DNE" << dendl;
+      assert(replaying);
+      return 0;
+    }
+    if (!collection_exists(dest)) {
+      dout(2) << __func__ << ": " << dest << " DNE" << dendl;
+      assert(replaying);
+      return 0;
+    }
+
+    int dstcmp = _check_replay_guard(dest, spos);
+    if (dstcmp < 0)
+      return 0;
+    if (dstcmp > 0 && !collection_empty(dest))
+      return -ENOTEMPTY;
+
+    int srccmp = _check_replay_guard(cid, spos);
+    if (srccmp < 0)
+      return 0;
+
+    _set_global_replay_guard(cid, spos);
+    _set_replay_guard(cid, spos, true);
+    _set_replay_guard(dest, spos, true);
+
+    Index from;
+    r = get_index(cid, &from);
+
+    Index to;
+    if (!r)
+      r = get_index(dest, &to);
+
+    if (!r)
+      r = from->split(rem, bits, to);
+
+    _close_replay_guard(cid, spos);
+    _close_replay_guard(dest, spos);
+  }
+  if (g_conf->filestore_debug_verify_split) {
+    vector objects;
+    ghobject_t next;
+    while (1) {
+      collection_list_partial(
+	cid,
+	next,
+	get_ideal_list_min(), get_ideal_list_max(), 0,
+	&objects,
+	&next);
+      if (objects.empty())
+	break;
+      for (vector::iterator i = objects.begin();
+	   i != objects.end();
+	   ++i) {
+	dout(20) << __func__ << ": " << *i << " still in source "
+		 << cid << dendl;
+	assert(!i->match(bits, rem));
+      }
+      objects.clear();
+    }
+    next = ghobject_t();
+    while (1) {
+      collection_list_partial(
+	dest,
+	next,
+	get_ideal_list_min(), get_ideal_list_max(), 0,
+	&objects,
+	&next);
+      if (objects.empty())
+	break;
+      for (vector::iterator i = objects.begin();
+	   i != objects.end();
+	   ++i) {
+	dout(20) << __func__ << ": " << *i << " now in dest "
+		 << *i << dendl;
+	assert(i->match(bits, rem));
+      }
+      objects.clear();
+    }
+  }
+  return r;
+}
+
+// DEPRECATED: remove once we are sure there won't be any such transactions
+// replayed
+int FileStore::_split_collection_create(coll_t cid,
+					uint32_t bits,
+					uint32_t rem,
+					coll_t dest,
+					const SequencerPosition &spos)
+{
+  dout(15) << __func__ << " " << cid << " bits: " << bits << dendl;
+  int r = _create_collection(dest);
+  if (r < 0 && !(r == -EEXIST && replaying))
+    return r;
+
+  int dstcmp = _check_replay_guard(cid, spos);
+  if (dstcmp < 0)
+    return 0;
+
+  int srccmp = _check_replay_guard(dest, spos);
+  if (srccmp < 0)
+    return 0;
+
+  _set_replay_guard(cid, spos, true);
+  _set_replay_guard(dest, spos, true);
+
+  Index from;
+  r = get_index(cid, &from);
+
+  Index to;
+  if (!r) 
+    r = get_index(dest, &to);
+
+  if (!r) 
+    r = from->split(rem, bits, to);
+
+  _close_replay_guard(cid, spos);
+  _close_replay_guard(dest, spos);
+  return r;
+}
+
+int FileStore::_set_alloc_hint(coll_t cid, const ghobject_t& oid,
+                               uint64_t expected_object_size,
+                               uint64_t expected_write_size)
+{
+  dout(15) << "set_alloc_hint " << cid << "/" << oid << " object_size " << expected_object_size << " write_size " << expected_write_size << dendl;
+
+  FDRef fd;
+  int ret;
+
+  ret = lfn_open(cid, oid, false, &fd);
+  if (ret < 0)
+    goto out;
+
+  {
+    // TODO: a more elaborate hint calculation
+    uint64_t hint = MIN(expected_write_size, m_filestore_max_alloc_hint_size);
+
+    ret = backend->set_alloc_hint(**fd, hint);
+    dout(20) << "set_alloc_hint hint " << hint << " ret " << ret << dendl;
+  }
+
+  lfn_close(fd);
+out:
+  dout(10) << "set_alloc_hint " << cid << "/" << oid << " object_size " << expected_object_size << " write_size " << expected_write_size << " = " << ret << dendl;
+  assert(!m_filestore_fail_eio || ret != -EIO);
+  return ret;
+}
+
+const char** FileStore::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "filestore_min_sync_interval",
+    "filestore_max_sync_interval",
+    "filestore_queue_max_ops",
+    "filestore_queue_max_bytes",
+    "filestore_queue_committing_max_ops",
+    "filestore_queue_committing_max_bytes",
+    "filestore_commit_timeout",
+    "filestore_dump_file",
+    "filestore_kill_at",
+    "filestore_fail_eio",
+    "filestore_replica_fadvise",
+    "filestore_sloppy_crc",
+    "filestore_sloppy_crc_block_size",
+    "filestore_max_alloc_hint_size",
+    NULL
+  };
+  return KEYS;
+}
+
+void FileStore::handle_conf_change(const struct md_config_t *conf,
+			  const std::set  &changed)
+{
+  if (changed.count("filestore_max_inline_xattr_size") ||
+      changed.count("filestore_max_inline_xattr_size_xfs") ||
+      changed.count("filestore_max_inline_xattr_size_btrfs") ||
+      changed.count("filestore_max_inline_xattr_size_other") ||
+      changed.count("filestore_max_inline_xattrs") ||
+      changed.count("filestore_max_inline_xattrs_xfs") ||
+      changed.count("filestore_max_inline_xattrs_btrfs") ||
+      changed.count("filestore_max_inline_xattrs_other")) {
+    Mutex::Locker l(lock);
+    set_xattr_limits_via_conf();
+  }
+  if (changed.count("filestore_min_sync_interval") ||
+      changed.count("filestore_max_sync_interval") ||
+      changed.count("filestore_queue_max_ops") ||
+      changed.count("filestore_queue_max_bytes") ||
+      changed.count("filestore_queue_committing_max_ops") ||
+      changed.count("filestore_queue_committing_max_bytes") ||
+      changed.count("filestore_kill_at") ||
+      changed.count("filestore_fail_eio") ||
+      changed.count("filestore_sloppy_crc") ||
+      changed.count("filestore_sloppy_crc_block_size") ||
+      changed.count("filestore_max_alloc_hint_size") ||
+      changed.count("filestore_replica_fadvise")) {
+    Mutex::Locker l(lock);
+    m_filestore_min_sync_interval = conf->filestore_min_sync_interval;
+    m_filestore_max_sync_interval = conf->filestore_max_sync_interval;
+    m_filestore_queue_max_ops = conf->filestore_queue_max_ops;
+    m_filestore_queue_max_bytes = conf->filestore_queue_max_bytes;
+    m_filestore_queue_committing_max_ops = conf->filestore_queue_committing_max_ops;
+    m_filestore_queue_committing_max_bytes = conf->filestore_queue_committing_max_bytes;
+    m_filestore_kill_at.set(conf->filestore_kill_at);
+    m_filestore_fail_eio = conf->filestore_fail_eio;
+    m_filestore_replica_fadvise = conf->filestore_replica_fadvise;
+    m_filestore_sloppy_crc = conf->filestore_sloppy_crc;
+    m_filestore_sloppy_crc_block_size = conf->filestore_sloppy_crc_block_size;
+    m_filestore_max_alloc_hint_size = conf->filestore_max_alloc_hint_size;
+  }
+  if (changed.count("filestore_commit_timeout")) {
+    Mutex::Locker l(sync_entry_timeo_lock);
+    m_filestore_commit_timeout = conf->filestore_commit_timeout;
+  }
+  if (changed.count("filestore_dump_file")) {
+    if (conf->filestore_dump_file.length() &&
+	conf->filestore_dump_file != "-") {
+      dump_start(conf->filestore_dump_file);
+    } else {
+      dump_stop();
+    }
+  }
+}
+
+void FileStore::dump_start(const std::string& file)
+{
+  dout(10) << "dump_start " << file << dendl;
+  if (m_filestore_do_dump) {
+    dump_stop();
+  }
+  m_filestore_dump_fmt.reset();
+  m_filestore_dump_fmt.open_array_section("dump");
+  m_filestore_dump.open(file.c_str());
+  m_filestore_do_dump = true;
+}
+
+void FileStore::dump_stop()
+{
+  dout(10) << "dump_stop" << dendl;
+  m_filestore_do_dump = false;
+  if (m_filestore_dump.is_open()) {
+    m_filestore_dump_fmt.close_section();
+    m_filestore_dump_fmt.flush(m_filestore_dump);
+    m_filestore_dump.flush();
+    m_filestore_dump.close();
+  }
+}
+
+void FileStore::dump_transactions(list& ls, uint64_t seq, OpSequencer *osr)
+{
+  m_filestore_dump_fmt.open_array_section("transactions");
+  unsigned trans_num = 0;
+  for (list::iterator i = ls.begin(); i != ls.end(); ++i, ++trans_num) {
+    m_filestore_dump_fmt.open_object_section("transaction");
+    m_filestore_dump_fmt.dump_string("osr", osr->get_name());
+    m_filestore_dump_fmt.dump_unsigned("seq", seq);
+    m_filestore_dump_fmt.dump_unsigned("trans_num", trans_num);
+    (*i)->dump(&m_filestore_dump_fmt);
+    m_filestore_dump_fmt.close_section();
+  }
+  m_filestore_dump_fmt.close_section();
+  m_filestore_dump_fmt.flush(m_filestore_dump);
+  m_filestore_dump.flush();
+}
+
+void FileStore::set_xattr_limits_via_conf()
+{
+  uint32_t fs_xattr_size;
+  uint32_t fs_xattrs;
+
+  assert(m_fs_type != FS_TYPE_NONE);
+
+  switch(m_fs_type) {
+    case FS_TYPE_XFS:
+      fs_xattr_size = g_conf->filestore_max_inline_xattr_size_xfs;
+      fs_xattrs = g_conf->filestore_max_inline_xattrs_xfs;
+      break;
+    case FS_TYPE_BTRFS:
+      fs_xattr_size = g_conf->filestore_max_inline_xattr_size_btrfs;
+      fs_xattrs = g_conf->filestore_max_inline_xattrs_btrfs;
+      break;
+    case FS_TYPE_ZFS:
+    case FS_TYPE_OTHER:
+      fs_xattr_size = g_conf->filestore_max_inline_xattr_size_other;
+      fs_xattrs = g_conf->filestore_max_inline_xattrs_other;
+      break;
+    default:
+      assert(!"Unknown fs type");
+  }
+
+  //Use override value if set
+  if (g_conf->filestore_max_inline_xattr_size)
+    m_filestore_max_inline_xattr_size = g_conf->filestore_max_inline_xattr_size;
+  else
+    m_filestore_max_inline_xattr_size = fs_xattr_size;
+
+  //Use override value if set
+  if (g_conf->filestore_max_inline_xattrs)
+    m_filestore_max_inline_xattrs = g_conf->filestore_max_inline_xattrs;
+  else
+    m_filestore_max_inline_xattrs = fs_xattrs;
+}
+
+// -- FSSuperblock --
+
+void FSSuperblock::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  compat_features.encode(bl);
+  ENCODE_FINISH(bl);
+}
+
+void FSSuperblock::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  compat_features.decode(bl);
+  DECODE_FINISH(bl);
+}
+
+void FSSuperblock::dump(Formatter *f) const
+{
+  f->open_object_section("compat");
+  compat_features.dump(f);
+  f->close_section();
+}
+
+void FSSuperblock::generate_test_instances(list& o)
+{
+  FSSuperblock z;
+  o.push_back(new FSSuperblock(z));
+  CompatSet::FeatureSet feature_compat;
+  CompatSet::FeatureSet feature_ro_compat;
+  CompatSet::FeatureSet feature_incompat;
+  feature_incompat.insert(CEPH_FS_FEATURE_INCOMPAT_SHARDS);
+  z.compat_features = CompatSet(feature_compat, feature_ro_compat,
+                                feature_incompat);
+  o.push_back(new FSSuperblock(z));
+}
diff --git a/ceph/src/os/FileStore.h b/ceph/src/os/FileStore.h
new file mode 100644
index 00000000..3fcd89a2
--- /dev/null
+++ b/ceph/src/os/FileStore.h
@@ -0,0 +1,773 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_FILESTORE_H
+#define CEPH_FILESTORE_H
+
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+#include 
+using namespace std;
+
+#include "include/unordered_map.h"
+
+#include "include/assert.h"
+
+#include "ObjectStore.h"
+#include "JournalingObjectStore.h"
+
+#include "common/Timer.h"
+#include "common/WorkQueue.h"
+
+#include "common/Mutex.h"
+#include "HashIndex.h"
+#include "IndexManager.h"
+#include "ObjectMap.h"
+#include "SequencerPosition.h"
+#include "FDCache.h"
+#include "WBThrottle.h"
+
+#include "include/uuid.h"
+
+
+// from include/linux/falloc.h:
+#ifndef FALLOC_FL_PUNCH_HOLE
+# define FALLOC_FL_PUNCH_HOLE 0x2
+#endif
+
+#if defined(__linux__)
+# ifndef BTRFS_SUPER_MAGIC
+static const __SWORD_TYPE BTRFS_SUPER_MAGIC(0x9123683E);
+# endif
+# ifndef XFS_SUPER_MAGIC
+static const __SWORD_TYPE XFS_SUPER_MAGIC(0x58465342);
+# endif
+#endif
+
+#ifndef ZFS_SUPER_MAGIC
+static const __SWORD_TYPE ZFS_SUPER_MAGIC(0x2fc12fc1);
+#endif
+
+
+enum fs_types {
+  FS_TYPE_NONE = 0,
+  FS_TYPE_XFS,
+  FS_TYPE_BTRFS,
+  FS_TYPE_ZFS,
+  FS_TYPE_OTHER
+};
+
+class FileStoreBackend;
+
+#define CEPH_FS_FEATURE_INCOMPAT_SHARDS CompatSet::Feature(1, "sharded objects")
+
+class FSSuperblock {
+public:
+  CompatSet compat_features;
+
+  FSSuperblock() { }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(FSSuperblock)
+
+inline ostream& operator<<(ostream& out, const FSSuperblock& sb)
+{
+  return out << "sb(" << sb.compat_features << ")";
+}
+
+class FileStore : public JournalingObjectStore,
+                  public md_config_obs_t
+{
+  static const uint32_t target_version = 3;
+public:
+  uint32_t get_target_version() {
+    return target_version;
+  }
+
+  int peek_journal_fsid(uuid_d *fsid);
+
+  struct FSPerfTracker {
+    PerfCounters::avg_tracker os_commit_latency;
+    PerfCounters::avg_tracker os_apply_latency;
+
+    objectstore_perf_stat_t get_cur_stats() const {
+      objectstore_perf_stat_t ret;
+      ret.filestore_commit_latency = os_commit_latency.avg();
+      ret.filestore_apply_latency = os_apply_latency.avg();
+      return ret;
+    }
+
+    void update_from_perfcounters(PerfCounters &logger);
+  } perf_tracker;
+  objectstore_perf_stat_t get_cur_stats() {
+    perf_tracker.update_from_perfcounters(*logger);
+    return perf_tracker.get_cur_stats();
+  }
+
+private:
+  string internal_name;         ///< internal name, used to name the perfcounter instance
+  string basedir, journalpath;
+  std::string current_fn;
+  std::string current_op_seq_fn;
+  std::string omap_dir;
+  uuid_d fsid;
+  
+  size_t blk_size;            ///< fs block size
+
+  int fsid_fd, op_fd, basedir_fd, current_fd;
+
+  FileStoreBackend *generic_backend;
+  FileStoreBackend *backend;
+
+  deque snaps;
+
+  // Indexed Collections
+  IndexManager index_manager;
+  int get_index(coll_t c, Index *index);
+  int init_index(coll_t c);
+
+  // ObjectMap
+  boost::scoped_ptr object_map;
+  
+  Finisher ondisk_finisher;
+
+  // helper fns
+  int get_cdir(coll_t cid, char *s, int len);
+  
+  /// read a uuid from fd
+  int read_fsid(int fd, uuid_d *uuid);
+
+  /// lock fsid_fd
+  int lock_fsid();
+
+  // sync thread
+  Mutex lock;
+  bool force_sync;
+  Cond sync_cond;
+  uint64_t sync_epoch;
+
+  Mutex sync_entry_timeo_lock;
+  SafeTimer timer;
+
+  list sync_waiters;
+  bool stop;
+  void sync_entry();
+  struct SyncThread : public Thread {
+    FileStore *fs;
+    SyncThread(FileStore *f) : fs(f) {}
+    void *entry() {
+      fs->sync_entry();
+      return 0;
+    }
+  } sync_thread;
+
+  // -- op workqueue --
+  struct Op {
+    utime_t start;
+    uint64_t op;
+    list tls;
+    Context *onreadable, *onreadable_sync;
+    uint64_t ops, bytes;
+    TrackedOpRef osd_op;
+  };
+  class OpSequencer : public Sequencer_impl {
+    Mutex qlock; // to protect q, for benefit of flush (peek/dequeue also protected by lock)
+    list q;
+    list jq;
+    list > flush_commit_waiters;
+    Cond cond;
+  public:
+    Sequencer *parent;
+    Mutex apply_lock;  // for apply mutual exclusion
+    
+    /// get_max_uncompleted
+    bool _get_max_uncompleted(
+      uint64_t *seq ///< [out] max uncompleted seq
+      ) {
+      assert(qlock.is_locked());
+      assert(seq);
+      *seq = 0;
+      if (q.empty() && jq.empty())
+	return true;
+
+      if (!q.empty())
+	*seq = q.back()->op;
+      if (!jq.empty() && jq.back() > *seq)
+	*seq = jq.back();
+
+      return false;
+    } /// @returns true if both queues are empty
+
+    /// get_min_uncompleted
+    bool _get_min_uncompleted(
+      uint64_t *seq ///< [out] min uncompleted seq
+      ) {
+      assert(qlock.is_locked());
+      assert(seq);
+      *seq = 0;
+      if (q.empty() && jq.empty())
+	return true;
+
+      if (!q.empty())
+	*seq = q.front()->op;
+      if (!jq.empty() && jq.front() < *seq)
+	*seq = jq.front();
+
+      return false;
+    } /// @returns true if both queues are empty
+
+    void _wake_flush_waiters(list *to_queue) {
+      uint64_t seq;
+      if (_get_min_uncompleted(&seq))
+	seq = -1;
+
+      for (list >::iterator i =
+	     flush_commit_waiters.begin();
+	   i != flush_commit_waiters.end() && i->first < seq;
+	   flush_commit_waiters.erase(i++)) {
+	to_queue->push_back(i->second);
+      }
+    }
+
+    void queue_journal(uint64_t s) {
+      Mutex::Locker l(qlock);
+      jq.push_back(s);
+    }
+    void dequeue_journal(list *to_queue) {
+      Mutex::Locker l(qlock);
+      jq.pop_front();
+      cond.Signal();
+      _wake_flush_waiters(to_queue);
+    }
+    void queue(Op *o) {
+      Mutex::Locker l(qlock);
+      q.push_back(o);
+    }
+    Op *peek_queue() {
+      assert(apply_lock.is_locked());
+      return q.front();
+    }
+
+    Op *dequeue(list *to_queue) {
+      assert(to_queue);
+      assert(apply_lock.is_locked());
+      Mutex::Locker l(qlock);
+      Op *o = q.front();
+      q.pop_front();
+      cond.Signal();
+
+      _wake_flush_waiters(to_queue);
+      return o;
+    }
+
+    void flush() {
+      Mutex::Locker l(qlock);
+
+      while (g_conf->filestore_blackhole)
+	cond.Wait(qlock);  // wait forever
+
+
+      // get max for journal _or_ op queues
+      uint64_t seq = 0;
+      if (!q.empty())
+	seq = q.back()->op;
+      if (!jq.empty() && jq.back() > seq)
+	seq = jq.back();
+
+      if (seq) {
+	// everything prior to our watermark to drain through either/both queues
+	while ((!q.empty() && q.front()->op <= seq) ||
+	       (!jq.empty() && jq.front() <= seq))
+	  cond.Wait(qlock);
+      }
+    }
+    bool flush_commit(Context *c) {
+      Mutex::Locker l(qlock);
+      uint64_t seq = 0;
+      if (_get_max_uncompleted(&seq)) {
+	delete c;
+	return true;
+      } else {
+	flush_commit_waiters.push_back(make_pair(seq, c));
+	return false;
+      }
+    }
+
+    OpSequencer()
+      : qlock("FileStore::OpSequencer::qlock", false, false),
+	parent(0),
+	apply_lock("FileStore::OpSequencer::apply_lock", false, false) {}
+    ~OpSequencer() {
+      assert(q.empty());
+    }
+
+    const string& get_name() const {
+      return parent->get_name();
+    }
+  };
+
+  friend ostream& operator<<(ostream& out, const OpSequencer& s);
+
+  Mutex fdcache_lock;
+  FDCache fdcache;
+  WBThrottle wbthrottle;
+
+  Sequencer default_osr;
+  deque op_queue;
+  uint64_t op_queue_len, op_queue_bytes;
+  Cond op_throttle_cond;
+  Mutex op_throttle_lock;
+  Finisher op_finisher;
+
+  ThreadPool op_tp;
+  struct OpWQ : public ThreadPool::WorkQueue {
+    FileStore *store;
+    OpWQ(FileStore *fs, time_t timeout, time_t suicide_timeout, ThreadPool *tp)
+      : ThreadPool::WorkQueue("FileStore::OpWQ", timeout, suicide_timeout, tp), store(fs) {}
+
+    bool _enqueue(OpSequencer *osr) {
+      store->op_queue.push_back(osr);
+      return true;
+    }
+    void _dequeue(OpSequencer *o) {
+      assert(0);
+    }
+    bool _empty() {
+      return store->op_queue.empty();
+    }
+    OpSequencer *_dequeue() {
+      if (store->op_queue.empty())
+	return NULL;
+      OpSequencer *osr = store->op_queue.front();
+      store->op_queue.pop_front();
+      return osr;
+    }
+    void _process(OpSequencer *osr, ThreadPool::TPHandle &handle) {
+      store->_do_op(osr, handle);
+    }
+    void _process_finish(OpSequencer *osr) {
+      store->_finish_op(osr);
+    }
+    void _clear() {
+      assert(store->op_queue.empty());
+    }
+  } op_wq;
+
+  void _do_op(OpSequencer *o, ThreadPool::TPHandle &handle);
+  void _finish_op(OpSequencer *o);
+  Op *build_op(list& tls,
+	       Context *onreadable, Context *onreadable_sync,
+	       TrackedOpRef osd_op);
+  void queue_op(OpSequencer *osr, Op *o);
+  void op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle = NULL);
+  void op_queue_release_throttle(Op *o);
+  void _journaled_ahead(OpSequencer *osr, Op *o, Context *ondisk);
+  friend struct C_JournaledAhead;
+  int write_version_stamp();
+
+  int open_journal();
+
+  PerfCounters *logger;
+
+public:
+  int lfn_find(coll_t cid, const ghobject_t& oid, IndexedPath *path);
+  int lfn_truncate(coll_t cid, const ghobject_t& oid, off_t length);
+  int lfn_stat(coll_t cid, const ghobject_t& oid, struct stat *buf);
+  int lfn_open(
+    coll_t cid,
+    const ghobject_t& oid,
+    bool create,
+    FDRef *outfd,
+    IndexedPath *path = 0,
+    Index *index = 0);
+  void lfn_close(FDRef fd);
+  int lfn_link(coll_t c, coll_t newcid, const ghobject_t& o, const ghobject_t& newoid) ;
+  int lfn_unlink(coll_t cid, const ghobject_t& o, const SequencerPosition &spos,
+		 bool force_clear_omap=false);
+
+public:
+  FileStore(const std::string &base, const std::string &jdev, const char *internal_name = "filestore", bool update_to=false);
+  ~FileStore();
+
+  int _detect_fs();
+  int _sanity_check_fs();
+  
+  bool test_mount_in_use();
+  int version_stamp_is_valid(uint32_t *version);
+  int update_version_stamp();
+  int read_op_seq(uint64_t *seq);
+  int write_op_seq(int, uint64_t seq);
+  int mount();
+  int umount();
+  int get_max_object_name_length();
+  int mkfs();
+  int mkjournal();
+
+  /**
+   * set_allow_sharded_objects()
+   *
+   * Before sharded ghobject_t can be specified this function must be called
+   *
+   * Once this function is called the FileStore is not mountable by prior releases
+   */
+  void set_allow_sharded_objects();
+
+  /**
+   * get_allow_sharded_objects()
+   *
+   * return value: true if set_allow_sharded_objects() called, otherwise false
+   */
+  bool get_allow_sharded_objects();
+
+  int statfs(struct statfs *buf);
+
+  int _do_transactions(
+    list &tls, uint64_t op_seq,
+    ThreadPool::TPHandle *handle);
+  int do_transactions(list &tls, uint64_t op_seq) {
+    return _do_transactions(tls, op_seq, 0);
+  }
+  unsigned _do_transaction(
+    Transaction& t, uint64_t op_seq, int trans_num,
+    ThreadPool::TPHandle *handle);
+
+  int queue_transactions(Sequencer *osr, list& tls,
+			 TrackedOpRef op = TrackedOpRef(),
+			 ThreadPool::TPHandle *handle = NULL);
+
+  /**
+   * set replay guard xattr on given file
+   *
+   * This will ensure that we will not replay this (or any previous) operation
+   * against this particular inode/object.
+   *
+   * @param fd open file descriptor for the file/object
+   * @param spos sequencer position of the last operation we should not replay
+   */
+  void _set_replay_guard(int fd,
+			 const SequencerPosition& spos,
+			 const ghobject_t *oid=0,
+			 bool in_progress=false);
+  void _set_replay_guard(coll_t cid,
+                         const SequencerPosition& spos,
+                         bool in_progress);
+  void _set_global_replay_guard(coll_t cid,
+				const SequencerPosition &spos);
+
+  /// close a replay guard opened with in_progress=true
+  void _close_replay_guard(int fd, const SequencerPosition& spos);
+  void _close_replay_guard(coll_t cid, const SequencerPosition& spos);
+
+  /**
+   * check replay guard xattr on given file
+   *
+   * Check the current position against any marker on the file that
+   * indicates which operations have already been applied.  If the
+   * current or a newer operation has been marked as applied, we
+   * should not replay the current operation again.
+   *
+   * If we are not replaying the journal, we already return true.  It
+   * is only on replay that we might return false, indicated that the
+   * operation should not be performed (again).
+   *
+   * @param fd open fd on the file/object in question
+   * @param spos sequencerposition for an operation we could apply/replay
+   * @return 1 if we can apply (maybe replay) this operation, -1 if spos has already been applied, 0 if it was in progress
+   */
+  int _check_replay_guard(int fd, const SequencerPosition& spos);
+  int _check_replay_guard(coll_t cid, const SequencerPosition& spos);
+  int _check_replay_guard(coll_t cid, ghobject_t oid, const SequencerPosition& pos);
+  int _check_global_replay_guard(coll_t cid, const SequencerPosition& spos);
+
+  // ------------------
+  // objects
+  int pick_object_revision_lt(ghobject_t& oid) {
+    return 0;
+  }
+  bool exists(coll_t cid, const ghobject_t& oid);
+  int stat(
+    coll_t cid,
+    const ghobject_t& oid,
+    struct stat *st,
+    bool allow_eio = false);
+  int read(
+    coll_t cid,
+    const ghobject_t& oid,
+    uint64_t offset,
+    size_t len,
+    bufferlist& bl,
+    bool allow_eio = false);
+  int fiemap(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, bufferlist& bl);
+
+  int _touch(coll_t cid, const ghobject_t& oid);
+  int _write(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, const bufferlist& bl,
+      bool replica = false);
+  int _zero(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len);
+  int _truncate(coll_t cid, const ghobject_t& oid, uint64_t size);
+  int _clone(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid,
+	     const SequencerPosition& spos);
+  int _clone_range(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid,
+		   uint64_t srcoff, uint64_t len, uint64_t dstoff,
+		   const SequencerPosition& spos);
+  int _do_clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff);
+  int _do_copy_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff);
+  int _remove(coll_t cid, const ghobject_t& oid, const SequencerPosition &spos);
+
+  int _fgetattr(int fd, const char *name, bufferptr& bp);
+  int _fgetattrs(int fd, map& aset, bool user_only);
+  int _fsetattrs(int fd, map &aset);
+
+  void _start_sync();
+
+  void start_sync();
+  void start_sync(Context *onsafe);
+  void sync();
+  void _flush_op_queue();
+  void flush();
+  void sync_and_flush();
+
+  int dump_journal(ostream& out);
+
+  void set_fsid(uuid_d u) {
+    fsid = u;
+  }
+  uuid_d get_fsid() { return fsid; }
+
+  // DEBUG read error injection, an object is removed from both on delete()
+  Mutex read_error_lock;
+  set data_error_set; // read() will return -EIO
+  set mdata_error_set; // getattr(),stat() will return -EIO
+  void inject_data_error(const ghobject_t &oid);
+  void inject_mdata_error(const ghobject_t &oid);
+  void debug_obj_on_delete(const ghobject_t &oid);
+  bool debug_data_eio(const ghobject_t &oid);
+  bool debug_mdata_eio(const ghobject_t &oid);
+
+  int snapshot(const string& name);
+
+  // attrs
+  int getattr(coll_t cid, const ghobject_t& oid, const char *name, bufferptr &bp);
+  int getattrs(coll_t cid, const ghobject_t& oid, map& aset, bool user_only = false);
+
+  int _setattrs(coll_t cid, const ghobject_t& oid, map& aset,
+		const SequencerPosition &spos);
+  int _rmattr(coll_t cid, const ghobject_t& oid, const char *name,
+	      const SequencerPosition &spos);
+  int _rmattrs(coll_t cid, const ghobject_t& oid,
+	       const SequencerPosition &spos);
+
+  int collection_getattr(coll_t c, const char *name, void *value, size_t size);
+  int collection_getattr(coll_t c, const char *name, bufferlist& bl);
+  int collection_getattrs(coll_t cid, map &aset);
+
+  int _collection_setattr(coll_t c, const char *name, const void *value, size_t size);
+  int _collection_rmattr(coll_t c, const char *name);
+  int _collection_setattrs(coll_t cid, map &aset);
+  int _collection_remove_recursive(const coll_t &cid,
+				   const SequencerPosition &spos);
+  int _collection_rename(const coll_t &cid, const coll_t &ncid,
+			 const SequencerPosition& spos);
+
+  // collections
+  int list_collections(vector& ls);
+  int collection_version_current(coll_t c, uint32_t *version);
+  int collection_stat(coll_t c, struct stat *st);
+  bool collection_exists(coll_t c);
+  bool collection_empty(coll_t c);
+  int collection_list(coll_t c, vector& oid);
+  int collection_list_partial(coll_t c, ghobject_t start,
+			      int min, int max, snapid_t snap,
+			      vector *ls, ghobject_t *next);
+  int collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
+                            snapid_t seq, vector *ls);
+
+  // omap (see ObjectStore.h for documentation)
+  int omap_get(coll_t c, const ghobject_t &oid, bufferlist *header,
+	       map *out);
+  int omap_get_header(
+    coll_t c,
+    const ghobject_t &oid,
+    bufferlist *out,
+    bool allow_eio = false);
+  int omap_get_keys(coll_t c, const ghobject_t &oid, set *keys);
+  int omap_get_values(coll_t c, const ghobject_t &oid, const set &keys,
+		      map *out);
+  int omap_check_keys(coll_t c, const ghobject_t &oid, const set &keys,
+		      set *out);
+  ObjectMap::ObjectMapIterator get_omap_iterator(coll_t c, const ghobject_t &oid);
+
+  int _create_collection(coll_t c);
+  int _create_collection(coll_t c, const SequencerPosition &spos);
+  int _destroy_collection(coll_t c);
+  int _collection_add(coll_t c, coll_t ocid, const ghobject_t& oid,
+		      const SequencerPosition& spos);
+  int _collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+			      coll_t c, const ghobject_t& o,
+			      const SequencerPosition& spos);
+
+  int _set_alloc_hint(coll_t cid, const ghobject_t& oid,
+                      uint64_t expected_object_size,
+                      uint64_t expected_write_size);
+
+  void dump_start(const std::string& file);
+  void dump_stop();
+  void dump_transactions(list& ls, uint64_t seq, OpSequencer *osr);
+
+private:
+  void _inject_failure();
+
+  // omap
+  int _omap_clear(coll_t cid, const ghobject_t &oid,
+		  const SequencerPosition &spos);
+  int _omap_setkeys(coll_t cid, const ghobject_t &oid,
+		    const map &aset,
+		    const SequencerPosition &spos);
+  int _omap_rmkeys(coll_t cid, const ghobject_t &oid, const set &keys,
+		   const SequencerPosition &spos);
+  int _omap_rmkeyrange(coll_t cid, const ghobject_t &oid,
+		       const string& first, const string& last,
+		       const SequencerPosition &spos);
+  int _omap_setheader(coll_t cid, const ghobject_t &oid, const bufferlist &bl,
+		      const SequencerPosition &spos);
+  int _split_collection(coll_t cid, uint32_t bits, uint32_t rem, coll_t dest,
+                        const SequencerPosition &spos);
+  int _split_collection_create(coll_t cid, uint32_t bits, uint32_t rem,
+			       coll_t dest,
+			       const SequencerPosition &spos);
+
+  virtual const char** get_tracked_conf_keys() const;
+  virtual void handle_conf_change(const struct md_config_t *conf,
+			  const std::set  &changed);
+  float m_filestore_commit_timeout;
+  bool m_filestore_journal_parallel;
+  bool m_filestore_journal_trailing;
+  bool m_filestore_journal_writeahead;
+  int m_filestore_fiemap_threshold;
+  double m_filestore_max_sync_interval;
+  double m_filestore_min_sync_interval;
+  bool m_filestore_fail_eio;
+  bool m_filestore_replica_fadvise;
+  int do_update;
+  bool m_journal_dio, m_journal_aio, m_journal_force_aio;
+  std::string m_osd_rollback_to_cluster_snap;
+  bool m_osd_use_stale_snap;
+  int m_filestore_queue_max_ops;
+  int m_filestore_queue_max_bytes;
+  int m_filestore_queue_committing_max_ops;
+  int m_filestore_queue_committing_max_bytes;
+  bool m_filestore_do_dump;
+  std::ofstream m_filestore_dump;
+  JSONFormatter m_filestore_dump_fmt;
+  atomic_t m_filestore_kill_at;
+  bool m_filestore_sloppy_crc;
+  int m_filestore_sloppy_crc_block_size;
+  uint64_t m_filestore_max_alloc_hint_size;
+  enum fs_types m_fs_type;
+
+  //Determined xattr handling based on fs type
+  void set_xattr_limits_via_conf();
+  uint32_t m_filestore_max_inline_xattr_size;
+  uint32_t m_filestore_max_inline_xattrs;
+
+  FSSuperblock superblock;
+
+  /**
+   * write_superblock()
+   *
+   * Write superblock to persisent storage
+   *
+   * return value: 0 on success, otherwise negative errno
+   */
+  int write_superblock();
+
+  /**
+   * read_superblock()
+   *
+   * Fill in FileStore::superblock by reading persistent storage
+   *
+   * return value: 0 on success, otherwise negative errno
+   */
+  int read_superblock();
+
+  friend class FileStoreBackend;
+};
+
+ostream& operator<<(ostream& out, const FileStore::OpSequencer& s);
+
+struct fiemap;
+
+class FileStoreBackend {
+private:
+  FileStore *filestore;
+protected:
+  int get_basedir_fd() {
+    return filestore->basedir_fd;
+  }
+  int get_current_fd() {
+    return filestore->current_fd;
+  }
+  int get_op_fd() {
+    return filestore->op_fd;
+  }
+  size_t get_blksize() {
+    return filestore->blk_size;
+  }
+  const string& get_basedir_path() {
+    return filestore->basedir;
+  }
+  const string& get_current_path() {
+    return filestore->current_fn;
+  }
+  int _copy_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff) {
+    return filestore->_do_copy_range(from, to, srcoff, len, dstoff);
+  }
+  int get_crc_block_size() {
+    return filestore->m_filestore_sloppy_crc_block_size;
+  }
+public:
+  FileStoreBackend(FileStore *fs) : filestore(fs) {}
+  virtual ~FileStoreBackend() {};
+  virtual int detect_features() = 0;
+  virtual int create_current() = 0;
+  virtual bool can_checkpoint() = 0;
+  virtual int list_checkpoints(list& ls) = 0;
+  virtual int create_checkpoint(const string& name, uint64_t *cid) = 0;
+  virtual int sync_checkpoint(uint64_t id) = 0;
+  virtual int rollback_to(const string& name) = 0;
+  virtual int destroy_checkpoint(const string& name) = 0;
+  virtual int syncfs() = 0;
+  virtual bool has_fiemap() = 0;
+  virtual int do_fiemap(int fd, off_t start, size_t len, struct fiemap **pfiemap) = 0;
+  virtual int clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff) = 0;
+  virtual int set_alloc_hint(int fd, uint64_t hint) = 0;
+
+  // hooks for (sloppy) crc tracking
+  virtual int _crc_update_write(int fd, loff_t off, size_t len, const bufferlist& bl) = 0;
+  virtual int _crc_update_truncate(int fd, loff_t off) = 0;
+  virtual int _crc_update_zero(int fd, loff_t off, size_t len) = 0;
+  virtual int _crc_update_clone_range(int srcfd, int destfd,
+				      loff_t srcoff, size_t len, loff_t dstoff) = 0;
+  virtual int _crc_verify_read(int fd, loff_t off, size_t len, const bufferlist& bl,
+			       ostream *out) = 0;
+};
+
+#endif
diff --git a/ceph/src/os/FlatIndex.cc b/ceph/src/os/FlatIndex.cc
new file mode 100644
index 00000000..2f1a1d16
--- /dev/null
+++ b/ceph/src/os/FlatIndex.cc
@@ -0,0 +1,430 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#if defined(__FreeBSD__)
+#include 
+#include 
+#endif
+
+#include "FlatIndex.h"
+#include "CollectionIndex.h"
+#include "common/ceph_crypto.h"
+#include "osd/osd_types.h"
+#include 
+
+#include "chain_xattr.h"
+
+using ceph::crypto::SHA1;
+
+/*
+ * long file names will have the following format:
+ *
+ * prefix_hash_index_cookie
+ *
+ * The prefix will just be the first X bytes of the original file name.
+ * The cookie is a constant string that shows whether this file name
+ * is hashed
+ */
+
+#define FILENAME_LFN_DIGEST_SIZE CEPH_CRYPTO_SHA1_DIGESTSIZE
+
+#define FILENAME_MAX_LEN        4096    // the long file name size
+#define FILENAME_SHORT_LEN      255     // the short file name size
+#define FILENAME_COOKIE         "long"  // ceph long file name
+#define FILENAME_HASH_LEN       FILENAME_LFN_DIGEST_SIZE
+#define FILENAME_EXTRA	        4       // underscores and digit
+
+#define LFN_ATTR "user.cephos.lfn"
+
+#define FILENAME_PREFIX_LEN (FILENAME_SHORT_LEN - FILENAME_HASH_LEN - (sizeof(FILENAME_COOKIE) - 1) - FILENAME_EXTRA)
+
+void FlatIndex::set_ref(ceph::shared_ptr ref) {
+  self_ref = ref;
+}
+
+int FlatIndex::cleanup() {
+  return 0;
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+static int hash_filename(const char *filename, char *hash, int buf_len)
+{
+  if (buf_len < FILENAME_HASH_LEN + 1)
+    return -EINVAL;
+
+  char buf[FILENAME_LFN_DIGEST_SIZE];
+  char hex[FILENAME_LFN_DIGEST_SIZE * 2];
+
+  SHA1 h;
+  h.Update((const byte *)filename, strlen(filename));
+  h.Final((byte *)buf);
+
+  buf_to_hex((byte *)buf, (FILENAME_HASH_LEN + 1) / 2, hex);
+  strncpy(hash, hex, FILENAME_HASH_LEN);
+  hash[FILENAME_HASH_LEN] = '\0';
+  return 0;
+}
+
+static void build_filename(char *filename, int len, const char *old_filename, int i)
+{
+  char hash[FILENAME_HASH_LEN + 1];
+
+  assert(len >= FILENAME_SHORT_LEN + 4);
+
+  strncpy(filename, old_filename, FILENAME_PREFIX_LEN);
+  filename[FILENAME_PREFIX_LEN] = '\0';
+  if (strlen(filename) < FILENAME_PREFIX_LEN)
+    return;
+  if (old_filename[FILENAME_PREFIX_LEN] == '\0')
+    return;
+
+  hash_filename(old_filename, hash, sizeof(hash));
+  int ofs = FILENAME_PREFIX_LEN;
+  while (1) {
+    int suffix_len = sprintf(filename + ofs, "_%s_%d_" FILENAME_COOKIE, hash, i);
+    if (ofs + suffix_len <= FILENAME_SHORT_LEN || !ofs)
+      break;
+    ofs--;
+  }
+}
+
+/* is this a candidate? */
+static int lfn_is_hashed_filename(const char *filename)
+{
+  int len = strlen(filename);
+  if (len < FILENAME_SHORT_LEN)
+    return 0;
+  return (strcmp(filename + len - (sizeof(FILENAME_COOKIE) - 1), FILENAME_COOKIE) == 0);
+}
+
+static void lfn_translate(const char *path, const char *name, char *new_name, int len)
+{
+  if (!lfn_is_hashed_filename(name)) {
+    strncpy(new_name, name, len);
+    return;
+  }
+
+  char buf[PATH_MAX];
+
+  snprintf(buf, sizeof(buf), "%s/%s", path, name);
+  int r = chain_getxattr(buf, LFN_ATTR, new_name, len - 1);
+  if (r < 0)
+    strncpy(new_name, name, len);
+  else
+    new_name[r] = '\0';
+  return;
+}
+
+static int append_oname(const ghobject_t &oid, char *s, int len)
+{
+  //assert(sizeof(oid) == 28);
+  char *end = s + len;
+  char *t = s + strlen(s);
+
+  const char *i = oid.hobj.oid.name.c_str();
+  while (*i && t < end) {
+    if (*i == '\\') {
+      *t++ = '\\';
+      *t++ = '\\';      
+    } else if (*i == '.' && i == oid.hobj.oid.name.c_str()) {  // only escape leading .
+      *t++ = '\\';
+      *t++ = '.';
+    } else if (*i == '/') {
+      *t++ = '\\';
+      *t++ = 's';
+    } else
+      *t++ = *i;
+    i++;
+  }
+
+  int size = t - s;
+
+  if (oid.hobj.snap == CEPH_NOSNAP)
+    size += snprintf(t, end - t, "_head");
+  else if (oid.hobj.snap == CEPH_SNAPDIR)
+    size += snprintf(t, end - t, "_snapdir");
+  else
+    size += snprintf(t, end - t, "_%llx", (long long unsigned)oid.hobj.snap);
+
+  return size;
+}
+
+static bool parse_object(char *s, ghobject_t& oid)
+{
+  sobject_t o;
+  char *bar = s + strlen(s) - 1;
+  while (*bar != '_' &&
+	 bar > s)
+    bar--;
+  if (*bar == '_') {
+    char buf[bar-s + 1];
+    char *t = buf;
+    char *i = s;
+    while (i < bar) {
+      if (*i == '\\') {
+	i++;
+	switch (*i) {
+	case '\\': *t++ = '\\'; break;
+	case '.': *t++ = '.'; break;
+	case 's': *t++ = '/'; break;
+	default: assert(0);
+	}
+      } else {
+	*t++ = *i;
+      }
+      i++;
+    }
+    *t = 0;
+    o.oid.name = string(buf, t-buf);
+    if (strcmp(bar+1, "head") == 0)
+      o.snap = CEPH_NOSNAP;
+    else if (strcmp(bar+1, "snapdir") == 0)
+      o.snap = CEPH_SNAPDIR;
+    else
+      o.snap = strtoull(bar+1, &s, 16);
+    oid = ghobject_t(hobject_t(o));
+    return true;
+  }
+  return false;
+}
+
+static int lfn_get(const char *coll_path, const ghobject_t& oid, char *pathname, int len, char *lfn, int lfn_len, int *exist, int *is_lfn)
+{
+  int i = 0;
+  strncpy(pathname, coll_path, len);
+  size_t path_len = strlen(coll_path);
+  pathname[path_len] = '/';
+  path_len++;
+  pathname[path_len] = '\0';
+  char *filename = pathname + path_len;
+
+  *lfn = '\0';
+  int actual_len = append_oname(oid, lfn, lfn_len);
+
+  if (actual_len < (int)FILENAME_PREFIX_LEN) {
+    /* not a long file name, just build it as it is */
+    strncpy(filename, lfn, len - path_len);
+    *is_lfn = 0;
+    struct stat buf;
+    int r = ::stat(pathname, &buf);
+    if (r < 0) {
+      if (errno == ENOENT) {
+	*exist = 0;
+      } else {
+	return -errno;
+      }
+    } else {
+      *exist = 1;
+    }
+
+    return 0;
+  }
+
+  *is_lfn = 1;
+  *exist = 0;
+
+  while (1) {
+    char buf[PATH_MAX];
+    int r;
+
+    build_filename(filename, len - path_len, lfn, i);
+    r = chain_getxattr(pathname, LFN_ATTR, buf, sizeof(buf));
+    if (r < 0)
+      r = -errno;
+    if (r > 0) {
+      buf[MIN((int)sizeof(buf)-1, r)] = '\0';
+      if (strcmp(buf, lfn) == 0) { // a match?
+        *exist = 1;
+        return i;
+      }
+    }
+    switch (r) {
+    case -ENOENT:
+      return i;
+    case -ERANGE:
+      assert(0); // shouldn't happen
+    default:
+      break;
+    }
+    if (r < 0)
+      break;
+    i++;
+  }
+
+  return 0; // unreachable anyway
+}
+
+int FlatIndex::init() {
+  return 0;
+}
+
+int FlatIndex::created(const ghobject_t &hoid, const char *path) {
+  char long_name[PATH_MAX];
+  long_name[0] = '\0';
+  int actual_len = append_oname(hoid, long_name, sizeof(long_name));
+  if (actual_len < (int)FILENAME_PREFIX_LEN) {
+    return 0;
+  }
+  assert(long_name[actual_len] == '\0');
+  assert(long_name[actual_len - 1] != '\0');
+  int r = chain_setxattr(path, LFN_ATTR, long_name, actual_len);
+  if (r < 0)
+    return r;
+  return 0;
+}
+
+int FlatIndex::unlink(const ghobject_t &o) {
+  char long_fn[PATH_MAX];
+  char short_fn[PATH_MAX];
+  char short_fn2[PATH_MAX];
+  int r, i, exist, err;
+  int path_len;
+  int is_lfn;
+
+  r = lfn_get(base_path.c_str(), o, short_fn, sizeof(short_fn), 
+	      long_fn, sizeof(long_fn), &exist, &is_lfn);
+  if (r < 0)
+    return r;
+  if (!is_lfn) {
+    r = ::unlink(short_fn);
+    if (r < 0) {
+      return -errno;
+    }
+    return 0;
+  }
+  if (!exist)
+    return -ENOENT;
+
+  const char *next = strncpy(short_fn2, base_path.c_str(), sizeof(short_fn2));
+  path_len = next - short_fn2;
+  short_fn2[path_len] = '/';
+  path_len++;
+  short_fn2[path_len] = '\0';
+
+  for (i = r + 1; ; i++) {
+    struct stat buf;
+    int ret;
+
+    build_filename(&short_fn2[path_len], sizeof(short_fn2) - path_len, long_fn, i);
+    ret = ::stat(short_fn2, &buf);
+    if (ret < 0) {
+      if (i == r + 1) {
+        err = ::unlink(short_fn);
+        if (err < 0)
+          return err;
+        return 0;
+      }
+      break;
+    }
+  }
+
+  build_filename(&short_fn2[path_len], sizeof(short_fn2) - path_len, long_fn, i - 1);
+
+  if (rename(short_fn2, short_fn) < 0) {
+    assert(0);
+  }
+
+  return 0;
+}
+
+int FlatIndex::lookup(const ghobject_t &hoid, IndexedPath *path, int *exist) {
+  char long_fn[PATH_MAX];
+  char short_fn[PATH_MAX];
+  int r;
+  int is_lfn;
+  r = lfn_get(base_path.c_str(), hoid, 
+	      short_fn, sizeof(short_fn), long_fn, 
+	      sizeof(long_fn), exist, &is_lfn);
+  if (r < 0)
+    return r;
+  *path = IndexedPath(new Path(string(short_fn), self_ref));
+  return 0;
+}
+
+static int get_hobject_from_oinfo(const char *dir, const char *file, 
+				  ghobject_t *o) {
+  char path[PATH_MAX];
+  bufferptr bp(PATH_MAX);
+  snprintf(path, sizeof(path), "%s/%s", dir, file);
+  // Hack, user.ceph._ is the attribute used to store the object info
+  int r = chain_getxattr(path, "user.ceph._", bp.c_str(), bp.length());
+  if (r < 0)
+    return r;
+  bufferlist bl;
+  bl.push_back(bp);
+  object_info_t oi(bl);
+  *o = oi.soid;
+  return 0;
+}
+
+int FlatIndex::collection_list_partial(const ghobject_t &start,
+				       int min_count,
+				       int max_count,
+				       snapid_t seq,
+				       vector *ls,
+				       ghobject_t *next) {
+  assert(0); // Should not be called
+  return 0;
+}
+
+int FlatIndex::collection_list(vector *ls) {
+  char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
+  char dir_name[PATH_MAX], new_name[PATH_MAX];
+  strncpy(dir_name, base_path.c_str(), sizeof(dir_name));
+  dir_name[sizeof(dir_name)-1]='\0';
+
+  DIR *dir = ::opendir(dir_name);
+  if (!dir)
+    return -errno;
+  
+  // first, build (ino, object) list
+  vector< pair > inolist;
+
+  struct dirent *de;
+  while (::readdir_r(dir, (struct dirent *)buf, &de) == 0) {
+    if (!de)
+      break;
+    // parse
+    if (de->d_name[0] == '.')
+      continue;
+    //cout << "  got object " << de->d_name << std::endl;
+    ghobject_t o;
+    lfn_translate(dir_name, de->d_name, new_name, sizeof(new_name));
+    if (parse_object(new_name, o)) {
+      get_hobject_from_oinfo(dir_name, de->d_name, &o);
+      inolist.push_back(pair(de->d_ino, o));
+      ls->push_back(o);
+    }
+  }
+
+  // sort
+  sort(inolist.begin(), inolist.end());
+
+  // build final list
+  ls->resize(inolist.size());
+  int i = 0;
+  for (vector< pair >::iterator p = inolist.begin(); p != inolist.end(); ++p)
+    (*ls)[i++].swap(p->second);
+  
+  ::closedir(dir);
+  return 0;
+}
diff --git a/ceph/src/os/FlatIndex.h b/ceph/src/os/FlatIndex.h
new file mode 100644
index 00000000..e55bd142
--- /dev/null
+++ b/ceph/src/os/FlatIndex.h
@@ -0,0 +1,87 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_FLATINDEX_H
+#define CEPH_FLATINDEX_H
+
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+
+#include "CollectionIndex.h"
+
+/**
+ * FlatIndex implements the collection layout prior to CollectionIndex
+ *
+ * This class should only be used for converting old filestores.
+ */
+class FlatIndex : public CollectionIndex {
+  ceph::weak_ptr self_ref;
+  string base_path;
+  coll_t collection;
+public:
+  FlatIndex(coll_t collection, string base_path) : base_path(base_path),
+						   collection(collection) {}
+
+  /// @see CollectionIndex
+  uint32_t collection_version() { return FLAT_INDEX_TAG; }
+
+  coll_t coll() const { return collection; }
+
+  /// @see CollectionIndex
+  void set_ref(ceph::shared_ptr ref);
+
+  /// @see CollectionIndex
+  int cleanup();
+
+  /// @see CollectionIndex
+  int init();
+
+  /// @see CollectionIndex
+  int created(
+    const ghobject_t &oid,
+    const char *path
+    );
+
+  /// @see CollectionIndex
+  int unlink(
+    const ghobject_t &oid
+    );
+
+  /// @see CollectionIndex
+  int lookup(
+    const ghobject_t &oid,
+    IndexedPath *path,
+    int *exist
+    );
+
+  /// @see CollectionIndex
+  int collection_list(
+    vector *ls
+    );
+
+  /// @see CollectionIndex
+  int collection_list_partial(
+    const ghobject_t &start,
+    int min_count,
+    int max_count,
+    snapid_t seq,
+    vector *ls,
+    ghobject_t *next
+    );
+};
+
+#endif
diff --git a/ceph/src/os/GenericFileStoreBackend.cc b/ceph/src/os/GenericFileStoreBackend.cc
new file mode 100644
index 00000000..56deeb21
--- /dev/null
+++ b/ceph/src/os/GenericFileStoreBackend.cc
@@ -0,0 +1,366 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/int_types.h"
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#if defined(__linux__)
+#include 
+#endif
+
+#include "include/compat.h"
+#include "include/linux_fiemap.h"
+
+#include 
+#include 
+#include 
+
+#include "GenericFileStoreBackend.h"
+
+#include "common/errno.h"
+#include "common/config.h"
+#include "common/sync_filesystem.h"
+
+#include "common/SloppyCRCMap.h"
+#include "os/chain_xattr.h"
+
+#define SLOPPY_CRC_XATTR "user.cephos.scrc"
+
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "genericfilestorebackend(" << get_basedir_path() << ") "
+
+#define ALIGN_DOWN(x, by) ((x) - ((x) % (by)))
+#define ALIGNED(x, by) (!((x) % (by)))
+#define ALIGN_UP(x, by) (ALIGNED((x), (by)) ? (x) : (ALIGN_DOWN((x), (by)) + (by)))
+
+GenericFileStoreBackend::GenericFileStoreBackend(FileStore *fs):
+  FileStoreBackend(fs),
+  ioctl_fiemap(false),
+  m_filestore_fiemap(g_conf->filestore_fiemap),
+  m_filestore_fsync_flushes_journal_data(g_conf->filestore_fsync_flushes_journal_data) {}
+
+int GenericFileStoreBackend::detect_features()
+{
+  char fn[PATH_MAX];
+  snprintf(fn, sizeof(fn), "%s/fiemap_test", get_basedir_path().c_str());
+
+  int fd = ::open(fn, O_CREAT|O_RDWR|O_TRUNC, 0644);
+  if (fd < 0) {
+    fd = -errno;
+    derr << "detect_features: unable to create " << fn << ": " << cpp_strerror(fd) << dendl;
+    return fd;
+  }
+
+  // ext4 has a bug in older kernels where fiemap will return an empty
+  // result in some cases.  this is a file layout that triggers the bug
+  // on 2.6.34-rc5.
+  int v[] = {
+    0x0000000000016000, 0x0000000000007000,
+    0x000000000004a000, 0x0000000000007000,
+    0x0000000000060000, 0x0000000000001000,
+    0x0000000000061000, 0x0000000000008000,
+    0x0000000000069000, 0x0000000000007000,
+    0x00000000000a3000, 0x000000000000c000,
+    0x000000000024e000, 0x000000000000c000,
+    0x000000000028b000, 0x0000000000009000,
+    0x00000000002b1000, 0x0000000000003000,
+    0, 0
+  };
+  for (int i=0; v[i]; i++) {
+    int off = v[i++];
+    int len = v[i];
+
+    // write a large extent
+    char buf[len];
+    memset(buf, 1, sizeof(buf));
+    int r = ::lseek(fd, off, SEEK_SET);
+    if (r < 0) {
+      r = -errno;
+      derr << "detect_features: failed to lseek " << fn << ": " << cpp_strerror(r) << dendl;
+      VOID_TEMP_FAILURE_RETRY(::close(fd));
+      return r;
+    }
+    r = write(fd, buf, sizeof(buf));
+    if (r < 0) {
+      derr << "detect_features: failed to write to " << fn << ": " << cpp_strerror(r) << dendl;
+      VOID_TEMP_FAILURE_RETRY(::close(fd));
+      return r;
+    }
+  }
+  ::fsync(fd);
+
+  // fiemap an extent inside that
+  struct fiemap *fiemap;
+  int r = do_fiemap(fd, 2430421, 59284, &fiemap);
+  if (r < 0) {
+    dout(0) << "detect_features: FIEMAP ioctl is NOT supported" << dendl;
+    ioctl_fiemap = false;
+  } else {
+    if (fiemap->fm_mapped_extents == 0) {
+      dout(0) << "detect_features: FIEMAP ioctl is supported, but buggy -- upgrade your kernel" << dendl;
+      ioctl_fiemap = false;
+    } else {
+      dout(0) << "detect_features: FIEMAP ioctl is supported and appears to work" << dendl;
+      ioctl_fiemap = true;
+    }
+    free(fiemap);
+  }
+  if (!m_filestore_fiemap) {
+    dout(0) << "detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option" << dendl;
+    ioctl_fiemap = false;
+  }
+
+  ::unlink(fn);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+
+
+  bool have_syncfs = false;
+#ifdef HAVE_SYS_SYNCFS
+  if (::syncfs(get_basedir_fd()) == 0) {
+    dout(0) << "detect_features: syncfs(2) syscall fully supported (by glibc and kernel)" << dendl;
+    have_syncfs = true;
+  } else {
+    dout(0) << "detect_features: syncfs(2) syscall supported by glibc BUT NOT the kernel" << dendl;
+  }
+#elif defined(SYS_syncfs)
+  if (syscall(SYS_syncfs, get_basedir_fd()) == 0) {
+    dout(0) << "detect_features: syscall(SYS_syncfs, fd) fully supported" << dendl;
+    have_syncfs = true;
+  } else {
+    dout(0) << "detect_features: syscall(SYS_syncfs, fd) supported by libc BUT NOT the kernel" << dendl;
+  }
+#elif defined(__NR_syncfs)
+  if (syscall(__NR_syncfs, get_basedir_fd()) == 0) {
+    dout(0) << "detect_features: syscall(__NR_syncfs, fd) fully supported" << dendl;
+    have_syncfs = true;
+  } else {
+    dout(0) << "detect_features: syscall(__NR_syncfs, fd) supported by libc BUT NOT the kernel" << dendl;
+  }
+#endif
+  if (!have_syncfs) {
+    dout(0) << "detect_features: syncfs(2) syscall not supported" << dendl;
+    if (m_filestore_fsync_flushes_journal_data) {
+      dout(0) << "detect_features: no syncfs(2), but 'filestore fsync flushes journal data = true', so fsync will suffice." << dendl;
+    } else {
+      dout(0) << "detect_features: no syncfs(2), must use sync(2)." << dendl;
+      dout(0) << "detect_features: WARNING: multiple ceph-osd daemons on the same host will be slow" << dendl;
+    }
+  }
+
+  return 0;
+}
+
+int GenericFileStoreBackend::create_current()
+{
+  struct stat st;
+  int ret = ::stat(get_current_path().c_str(), &st);
+  if (ret == 0) {
+    // current/ exists
+    if (!S_ISDIR(st.st_mode)) {
+      dout(0) << "_create_current: current/ exists but is not a directory" << dendl;
+      ret = -EINVAL;
+    }
+  } else {
+    ret = ::mkdir(get_current_path().c_str(), 0755);
+    if (ret < 0) {
+      ret = -errno;
+      dout(0) << "_create_current: mkdir " << get_current_path() << " failed: "<< cpp_strerror(ret) << dendl;
+    }
+  }
+  return ret;
+}
+
+int GenericFileStoreBackend::syncfs()
+{
+  int ret;
+  if (m_filestore_fsync_flushes_journal_data) {
+    dout(15) << "syncfs: doing fsync on " << get_op_fd() << dendl;
+    // make the file system's journal commit.
+    //  this works with ext3, but NOT ext4
+    ret = ::fsync(get_op_fd());
+    if (ret < 0)
+      ret = -errno;
+  } else {
+    dout(15) << "syncfs: doing a full sync (syncfs(2) if possible)" << dendl;
+    ret = sync_filesystem(get_current_fd());
+  }
+  return ret;
+}
+
+int GenericFileStoreBackend::do_fiemap(int fd, off_t start, size_t len, struct fiemap **pfiemap)
+{
+  struct fiemap *fiemap = NULL;
+  struct fiemap *_realloc_fiemap = NULL;
+  int size;
+  int ret;
+
+  fiemap = (struct fiemap*)calloc(sizeof(struct fiemap), 1);
+  if (!fiemap)
+    return -ENOMEM;
+
+  fiemap->fm_start = start;
+  fiemap->fm_length = len;
+
+  fsync(fd); /* flush extents to disk if needed */
+
+  if (ioctl(fd, FS_IOC_FIEMAP, fiemap) < 0) {
+    ret = -errno;
+    goto done_err;
+  }
+
+  size = sizeof(struct fiemap_extent) * (fiemap->fm_mapped_extents);
+
+  _realloc_fiemap = (struct fiemap *)realloc(fiemap, sizeof(struct fiemap) + size);
+  if (!_realloc_fiemap) {
+    ret = -ENOMEM;
+    goto done_err;
+  } else {
+    fiemap = _realloc_fiemap;
+  }
+
+  memset(fiemap->fm_extents, 0, size);
+
+  fiemap->fm_extent_count = fiemap->fm_mapped_extents;
+  fiemap->fm_mapped_extents = 0;
+
+  if (ioctl(fd, FS_IOC_FIEMAP, fiemap) < 0) {
+    ret = -errno;
+    goto done_err;
+  }
+  *pfiemap = fiemap;
+
+  return 0;
+
+done_err:
+  *pfiemap = NULL;
+  free(fiemap);
+  return ret;
+}
+
+
+int GenericFileStoreBackend::_crc_load_or_init(int fd, SloppyCRCMap *cm)
+{
+  char buf[100];
+  bufferptr bp;
+  int r = 0;
+  int l = chain_fgetxattr(fd, SLOPPY_CRC_XATTR, buf, sizeof(buf));
+  if (l == -ENODATA) {
+    return 0;
+  }
+  if (l >= 0) {
+    bp = buffer::create(l);
+    memcpy(bp.c_str(), buf, l);
+  } else if (l == -ERANGE) {
+    l = chain_fgetxattr(fd, SLOPPY_CRC_XATTR, 0, 0);
+    if (l > 0) {
+      bp = buffer::create(l);
+      l = chain_fgetxattr(fd, SLOPPY_CRC_XATTR, bp.c_str(), l);
+    }
+  }
+  bufferlist bl;
+  bl.append(bp);
+  bufferlist::iterator p = bl.begin();
+  try {
+    ::decode(*cm, p);
+  }
+  catch (buffer::error &e) {
+    r = -EIO;
+  }
+  if (r < 0)
+    derr << __func__ << " got " << cpp_strerror(r) << dendl;
+  return r;
+}
+
+int GenericFileStoreBackend::_crc_save(int fd, SloppyCRCMap *cm)
+{
+  bufferlist bl;
+  ::encode(*cm, bl);
+  int r = chain_fsetxattr(fd, SLOPPY_CRC_XATTR, bl.c_str(), bl.length());
+  if (r < 0)
+    derr << __func__ << " got " << cpp_strerror(r) << dendl;
+  return r;
+}
+
+int GenericFileStoreBackend::_crc_update_write(int fd, loff_t off, size_t len, const bufferlist& bl)
+{
+  SloppyCRCMap scm(get_crc_block_size());
+  int r = _crc_load_or_init(fd, &scm);
+  if (r < 0)
+    return r;
+  ostringstream ss;
+  scm.write(off, len, bl, &ss);
+  dout(30) << __func__ << "\n" << ss.str() << dendl;
+  r = _crc_save(fd, &scm);
+  return r;
+}
+
+int GenericFileStoreBackend::_crc_update_truncate(int fd, loff_t off)
+{
+  SloppyCRCMap scm(get_crc_block_size());
+  int r = _crc_load_or_init(fd, &scm);
+  if (r < 0)
+    return r;
+  scm.truncate(off);
+  r = _crc_save(fd, &scm);
+  return r;
+}
+
+int GenericFileStoreBackend::_crc_update_zero(int fd, loff_t off, size_t len)
+{
+  SloppyCRCMap scm(get_crc_block_size());
+  int r = _crc_load_or_init(fd, &scm);
+  if (r < 0)
+    return r;
+  scm.zero(off, len);
+  r = _crc_save(fd, &scm);
+  return r;
+}
+
+int GenericFileStoreBackend::_crc_update_clone_range(int srcfd, int destfd,
+						     loff_t srcoff, size_t len, loff_t dstoff)
+{
+  SloppyCRCMap scm_src(get_crc_block_size());
+  SloppyCRCMap scm_dst(get_crc_block_size());
+  int r = _crc_load_or_init(srcfd, &scm_src);
+  if (r < 0)
+    return r;
+  r = _crc_load_or_init(destfd, &scm_dst);
+  if (r < 0)
+    return r;
+  ostringstream ss;
+  scm_dst.clone_range(srcoff, len, dstoff, scm_src, &ss);
+  dout(30) << __func__ << "\n" << ss.str() << dendl;
+  r = _crc_save(destfd, &scm_dst);
+  return r;
+}
+
+int GenericFileStoreBackend::_crc_verify_read(int fd, loff_t off, size_t len, const bufferlist& bl,
+					      ostream *out)
+{
+  SloppyCRCMap scm(get_crc_block_size());
+  int r = _crc_load_or_init(fd, &scm);
+  if (r < 0)
+    return r;
+  return scm.read(off, len, bl, out);
+}
diff --git a/ceph/src/os/GenericFileStoreBackend.h b/ceph/src/os/GenericFileStoreBackend.h
new file mode 100644
index 00000000..374e193e
--- /dev/null
+++ b/ceph/src/os/GenericFileStoreBackend.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_GENERICFILESTOREBACKEDN_H
+#define CEPH_GENERICFILESTOREBACKEDN_H
+
+#include "FileStore.h"
+
+class SloppyCRCMap;
+
+class GenericFileStoreBackend : public FileStoreBackend {
+private:
+  bool ioctl_fiemap;
+  bool m_filestore_fiemap;
+  bool m_filestore_fsync_flushes_journal_data;
+public:
+  GenericFileStoreBackend(FileStore *fs);
+  virtual ~GenericFileStoreBackend() {};
+
+  virtual int detect_features();
+  virtual int create_current();
+  virtual bool can_checkpoint() { return false; };
+  virtual int list_checkpoints(list& ls) { return 0; }
+  virtual int create_checkpoint(const string& name, uint64_t *cid) { return -EOPNOTSUPP; }
+  virtual int sync_checkpoint(uint64_t id) { return -EOPNOTSUPP; }
+  virtual int rollback_to(const string& name) { return -EOPNOTSUPP; }
+  virtual int destroy_checkpoint(const string& name) { return -EOPNOTSUPP; }
+  virtual int syncfs();
+  virtual bool has_fiemap() { return ioctl_fiemap; }
+  virtual int do_fiemap(int fd, off_t start, size_t len, struct fiemap **pfiemap);
+  virtual int clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff) {
+    return _copy_range(from, to, srcoff, len, dstoff);
+  }
+  virtual int set_alloc_hint(int fd, uint64_t hint) { return -EOPNOTSUPP; }
+
+private:
+  int _crc_load_or_init(int fd, SloppyCRCMap *cm);
+  int _crc_save(int fd, SloppyCRCMap *cm);
+public:
+  virtual int _crc_update_write(int fd, loff_t off, size_t len, const bufferlist& bl);
+  virtual int _crc_update_truncate(int fd, loff_t off);
+  virtual int _crc_update_zero(int fd, loff_t off, size_t len);
+  virtual int _crc_update_clone_range(int srcfd, int destfd,
+				      loff_t srcoff, size_t len, loff_t dstoff);
+  virtual int _crc_verify_read(int fd, loff_t off, size_t len, const bufferlist& bl,
+			       ostream *out);
+};
+#endif
diff --git a/ceph/src/os/GenericObjectMap.cc b/ceph/src/os/GenericObjectMap.cc
new file mode 100644
index 00000000..011c83b5
--- /dev/null
+++ b/ceph/src/os/GenericObjectMap.cc
@@ -0,0 +1,1111 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack 
+ *
+ * Author: Haomai Wang 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/int_types.h"
+#include "include/buffer.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "GenericObjectMap.h"
+#include "common/debug.h"
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_keyvaluestore
+const string GenericObjectMap::GLOBAL_STATE_KEY = "HEADER";
+
+const string GenericObjectMap::USER_PREFIX = "_SEQ_";
+const string GenericObjectMap::INTERN_PREFIX = "_INTERN_";
+const string GenericObjectMap::COMPLETE_PREFIX = "_COMPLETE_";
+const string GenericObjectMap::GHOBJECT_TO_SEQ_PREFIX = "_GHOBJTOSEQ_";
+const string GenericObjectMap::PARENT_KEY = "_PARENT_HEADER_";
+
+// In order to make right ordering for leveldb matching with hobject_t,
+// so use "!" to separated
+const string GenericObjectMap::GHOBJECT_KEY_SEP_S = "!";
+const char GenericObjectMap::GHOBJECT_KEY_SEP_C = '!';
+
+// ============== GenericObjectMap Key Function =================
+
+static void append_escaped(const string &in, string *out)
+{
+  for (string::const_iterator i = in.begin(); i != in.end(); ++i) {
+    if (*i == '%') {
+      out->push_back('%');
+      out->push_back('p');
+    } else if (*i == '.') {
+      out->push_back('%');
+      out->push_back('e');
+    } else if (*i == GenericObjectMap::GHOBJECT_KEY_SEP_C) {
+      out->push_back('%');
+      out->push_back('u');
+    } else if (*i == '!') {
+      out->push_back('%');
+      out->push_back('s');
+    } else {
+      out->push_back(*i);
+    }
+  }
+}
+
+static bool append_unescaped(string::const_iterator begin,
+                             string::const_iterator end,
+                             string *out) 
+{
+  for (string::const_iterator i = begin; i != end; ++i) {
+    if (*i == '%') {
+      ++i;
+      if (*i == 'p')
+        out->push_back('%');
+      else if (*i == 'e')
+        out->push_back('.');
+      else if (*i == 'u')
+        out->push_back(GenericObjectMap::GHOBJECT_KEY_SEP_C);
+      else if (*i == 's')
+        out->push_back('!');
+      else
+        return false;
+    } else {
+      out->push_back(*i);
+    }
+  }
+  return true;
+}
+
+string GenericObjectMap::header_key(const coll_t &cid)
+{
+  string full_name;
+
+  append_escaped(cid.to_str(), &full_name);
+  full_name.append(GHOBJECT_KEY_SEP_S);
+  return full_name;
+}
+
+string GenericObjectMap::header_key(const coll_t &cid, const ghobject_t &oid)
+{
+  string full_name;
+
+  append_escaped(cid.to_str(), &full_name);
+  full_name.append(GHOBJECT_KEY_SEP_S);
+
+  char buf[PATH_MAX];
+  char *t = buf;
+  char *end = t + sizeof(buf);
+
+  // make field ordering match with hobject_t compare operations
+  snprintf(t, end - t, "%.*X", (int)(sizeof(oid.hobj.hash)*2),
+           (uint32_t)oid.get_filestore_key_u32());
+  full_name += string(buf);
+  full_name.append(GHOBJECT_KEY_SEP_S);
+
+  append_escaped(oid.hobj.nspace, &full_name);
+  full_name.append(GHOBJECT_KEY_SEP_S);
+
+  t = buf;
+  if (oid.hobj.pool == -1)
+    t += snprintf(t, end - t, "none");
+  else
+    t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.pool);
+  full_name += string(buf);
+  full_name.append(GHOBJECT_KEY_SEP_S);
+
+  append_escaped(oid.hobj.get_key(), &full_name);
+  full_name.append(GHOBJECT_KEY_SEP_S);
+
+  append_escaped(oid.hobj.oid.name, &full_name);
+  full_name.append(GHOBJECT_KEY_SEP_S);
+
+  t = buf;
+  if (oid.hobj.snap == CEPH_NOSNAP)
+    t += snprintf(t, end - t, "head");
+  else if (oid.hobj.snap == CEPH_SNAPDIR)
+    t += snprintf(t, end - t, "snapdir");
+  else
+    // Keep length align
+    t += snprintf(t, end - t, "%016llx", (long long unsigned)oid.hobj.snap);
+  full_name += string(buf);
+
+  if (oid.generation != ghobject_t::NO_GEN) {
+    assert(oid.shard_id != ghobject_t::NO_SHARD);
+    full_name.append(GHOBJECT_KEY_SEP_S);
+
+    t = buf;
+    end = t + sizeof(buf);
+    t += snprintf(t, end - t, "%llx", (long long unsigned)oid.generation);
+    full_name += string(buf);
+
+    full_name.append(GHOBJECT_KEY_SEP_S);
+
+    t = buf;
+    end = t + sizeof(buf);
+    t += snprintf(t, end - t, "%x", (int)oid.shard_id);
+    full_name += string(buf);
+  }
+
+  return full_name;
+}
+
+bool GenericObjectMap::parse_header_key(const string &long_name,
+                                          coll_t *out_coll, ghobject_t *out)
+{
+  string coll;
+  string name;
+  string key;
+  string ns;
+  uint32_t hash;
+  snapid_t snap;
+  uint64_t pool;
+  gen_t generation = ghobject_t::NO_GEN;
+  shard_t shard_id = ghobject_t::NO_SHARD;
+
+  string::const_iterator current = long_name.begin();
+  string::const_iterator end;
+
+  for (end = current; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+  if (!append_unescaped(current, end, &coll))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+  if (end == long_name.end())
+    return false;
+  string hash_str(current, end);
+  sscanf(hash_str.c_str(), "%X", &hash);
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &ns))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+  if (end == long_name.end())
+    return false;
+  string pstring(current, end);
+  if (pstring == "none")
+    pool = (uint64_t)-1;
+  else
+    pool = strtoull(pstring.c_str(), NULL, 16);
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &key))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &name))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+  string snap_str(current, end);
+  if (snap_str == "head")
+    snap = CEPH_NOSNAP;
+  else if (snap_str == "snapdir")
+    snap = CEPH_SNAPDIR;
+  else
+    snap = strtoull(snap_str.c_str(), NULL, 16);
+
+  // Optional generation/shard_id
+  string genstring, shardstring;
+  if (end != long_name.end()) {
+    current = ++end;
+    for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+    if (end == long_name.end())
+      return false;
+    genstring = string(current, end);
+
+    generation = (gen_t)strtoull(genstring.c_str(), NULL, 16);
+
+    current = ++end;
+    for ( ; end != long_name.end() && *end != GHOBJECT_KEY_SEP_C; ++end) ;
+    if (end != long_name.end())
+      return false;
+    shardstring = string(current, end);
+
+    shard_id = (shard_t)strtoul(shardstring.c_str(), NULL, 16);
+  }
+
+  if (out) {
+    (*out) = ghobject_t(hobject_t(name, key, snap, hash, (int64_t)pool, ns),
+                        generation, shard_id);
+    // restore reversed hash. see calculate_key
+    out->hobj.hash = out->get_filestore_key();
+  }
+
+  if (out_coll)
+    *out_coll = coll_t(coll);
+
+  return true;
+}
+
+
+// ============== GenericObjectMap Prefix =================
+
+string GenericObjectMap::user_prefix(Header header, const string &prefix)
+{
+  return USER_PREFIX + seq_key(header->seq) + prefix;
+}
+
+string GenericObjectMap::complete_prefix(Header header)
+{
+  return INTERN_PREFIX + seq_key(header->seq) + COMPLETE_PREFIX;
+}
+
+string GenericObjectMap::parent_seq_prefix(uint64_t seq)
+{
+  return INTERN_PREFIX + seq_key(seq) + PARENT_KEY;
+}
+
+
+// ============== GenericObjectMapIteratorImpl =================
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::init()
+{
+  invalid = false;
+  if (ready) {
+    return 0;
+  }
+
+  assert(!parent_iter);
+  if (header->parent) {
+    Header parent = map->lookup_parent(header);
+    if (!parent) {
+      assert(0);
+      return -EINVAL;
+    }
+    parent_iter.reset(new GenericObjectMapIteratorImpl(map, parent, prefix));
+  }
+
+  key_iter = map->db->get_iterator(map->user_prefix(header, prefix));
+  assert(key_iter);
+  complete_iter = map->db->get_iterator(map->complete_prefix(header));
+  assert(complete_iter);
+  cur_iter = key_iter;
+  assert(cur_iter);
+  ready = true;
+  return 0;
+}
+
+ObjectMap::ObjectMapIterator GenericObjectMap::get_iterator(
+    const coll_t &cid, const ghobject_t &oid, const string &prefix)
+{
+  Header header = lookup_header(cid, oid);
+  if (!header)
+    return ObjectMap::ObjectMapIterator(new EmptyIteratorImpl());
+  return _get_iterator(header, prefix);
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::seek_to_first()
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->seek_to_first();
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->seek_to_first();
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::seek_to_last()
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->seek_to_last();
+    if (r < 0)
+      return r;
+    if (parent_iter->valid())
+      r = parent_iter->next();
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->seek_to_last();
+  if (r < 0)
+    return r;
+  if (key_iter->valid())
+    r = key_iter->next();
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::lower_bound(const string &to)
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->lower_bound(to);
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->lower_bound(to);
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::upper_bound(const string &after)
+{
+  init();
+  r = 0;
+  if (parent_iter) {
+    r = parent_iter->upper_bound(after);
+    if (r < 0)
+      return r;
+  }
+  r = key_iter->upper_bound(after);
+  if (r < 0)
+    return r;
+  return adjust();
+}
+
+bool GenericObjectMap::GenericObjectMapIteratorImpl::valid()
+{
+  bool valid = !invalid && ready;
+  assert(!valid || cur_iter->valid());
+  return valid;
+}
+
+bool GenericObjectMap::GenericObjectMapIteratorImpl::valid_parent()
+{
+  if (parent_iter && parent_iter->valid() &&
+      (!key_iter->valid() || key_iter->key() > parent_iter->key()))
+    return true;
+  return false;
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::next()
+{
+  assert(cur_iter->valid());
+  assert(valid());
+  cur_iter->next();
+  return adjust();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::next_parent()
+{
+  if (!parent_iter || !parent_iter->valid()) {
+    invalid = true;
+    return 0;
+  }
+  r = next();
+  if (r < 0)
+    return r;
+  if (!valid() || on_parent() || !parent_iter->valid())
+    return 0;
+
+  return lower_bound(parent_iter->key());
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::in_complete_region(
+    const string &to_test, string *begin, string *end)
+{
+  complete_iter->upper_bound(to_test);
+  if (complete_iter->valid())
+    complete_iter->prev();
+  else
+    complete_iter->seek_to_last();
+
+  if (!complete_iter->valid())
+    return false;
+
+  string _end;
+  if (begin)
+    *begin = complete_iter->key();
+  _end = string(complete_iter->value().c_str());
+  if (end)
+    *end = _end;
+  return (to_test >= complete_iter->key()) && (!_end.size() || _end > to_test);
+}
+
+/**
+ * Moves parent_iter to the next position both out of the complete_region and 
+ * not equal to key_iter.  Then, we set cur_iter to parent_iter if valid and
+ * less than key_iter and key_iter otherwise.
+ */
+int GenericObjectMap::GenericObjectMapIteratorImpl::adjust()
+{
+  string begin, end;
+  while (parent_iter && parent_iter->valid()) {
+    if (in_complete_region(parent_iter->key(), &begin, &end)) {
+      if (end.size() == 0) {
+        parent_iter->seek_to_last();
+        if (parent_iter->valid())
+          parent_iter->next();
+      } else {
+        parent_iter->lower_bound(end);
+      }
+    } else if (key_iter->valid() && key_iter->key() == parent_iter->key()) {
+      parent_iter->next();
+    } else {
+      break;
+    }
+  }
+  if (valid_parent()) {
+    cur_iter = parent_iter;
+  } else if (key_iter->valid()) {
+    cur_iter = key_iter;
+  } else {
+    invalid = true;
+  }
+  assert(invalid || cur_iter->valid());
+  return 0;
+}
+
+string GenericObjectMap::GenericObjectMapIteratorImpl::key()
+{
+  return cur_iter->key();
+}
+
+bufferlist GenericObjectMap::GenericObjectMapIteratorImpl::value()
+{
+  return cur_iter->value();
+}
+
+int GenericObjectMap::GenericObjectMapIteratorImpl::status()
+{
+  return r;
+}
+
+
+// ============== GenericObjectMap Public API =================
+
+void GenericObjectMap::set_keys(const Header header,
+                                const string &prefix,
+                                const map &set,
+                                KeyValueDB::Transaction t)
+{
+  t->set(user_prefix(header, prefix), set);
+}
+
+int GenericObjectMap::clear(const Header header,
+                            KeyValueDB::Transaction t)
+{
+  remove_header(header->cid, header->oid, header, t);
+  assert(header->num_children > 0);
+  header->num_children--;
+  int r = _clear(header, t);
+  if (r < 0)
+    return r;
+  return 0;
+}
+
+int GenericObjectMap::rm_keys(const Header header,
+                              const string &prefix,
+                              const set &to_clear,
+                              KeyValueDB::Transaction t)
+{
+  t->rmkeys(user_prefix(header, prefix), to_clear);
+  if (!header->parent) {
+    return 0;
+  }
+
+  // Copy up keys from parent around to_clear
+  int keep_parent;
+  {
+    GenericObjectMapIterator iter = _get_iterator(header, prefix);
+    iter->seek_to_first();
+    map new_complete;
+    map to_write;
+    for(set::const_iterator i = to_clear.begin();
+        i != to_clear.end(); ) {
+      unsigned copied = 0;
+      iter->lower_bound(*i);
+      ++i;
+      if (!iter->valid())
+        break;
+      string begin = iter->key();
+      if (!iter->on_parent())
+        iter->next_parent();
+      if (new_complete.size() && new_complete.rbegin()->second == begin) {
+        begin = new_complete.rbegin()->first;
+      }
+      while (iter->valid() && copied < 20) {
+        if (!to_clear.count(iter->key()))
+          to_write[iter->key()].append(iter->value());
+        if (i != to_clear.end() && *i <= iter->key()) {
+          ++i;
+          copied = 0;
+        }
+
+        iter->next_parent();
+        copied++;
+      }
+      if (iter->valid()) {
+        new_complete[begin] = iter->key();
+      } else {
+        new_complete[begin] = "";
+        break;
+      }
+    }
+    t->set(user_prefix(header, prefix), to_write);
+    merge_new_complete(header, new_complete, iter, t);
+    keep_parent = need_parent(iter);
+    if (keep_parent < 0)
+      return keep_parent;
+  }
+
+  if (!keep_parent) {
+    Header parent = lookup_parent(header);
+    if (!parent)
+      return -EINVAL;
+    parent->num_children--;
+    _clear(parent, t);
+    header->parent = 0;
+    set_header(header->cid, header->oid, *header, t);
+    t->rmkeys_by_prefix(complete_prefix(header));
+  }
+
+  return 0;
+}
+
+int GenericObjectMap::get(const coll_t &cid, const ghobject_t &oid,
+                          const string &prefix,
+                          map *out)
+{
+  Header header = lookup_header(cid, oid);
+  if (!header)
+    return -ENOENT;
+
+  ObjectMap::ObjectMapIterator iter = _get_iterator(header, prefix);
+  for (iter->seek_to_first(); iter->valid(); iter->next()) {
+    if (iter->status())
+      return iter->status();
+    out->insert(make_pair(iter->key(), iter->value()));
+  }
+
+  return 0;
+}
+
+int GenericObjectMap::get_keys(const coll_t &cid, const ghobject_t &oid,
+                               const string &prefix,
+                               set *keys)
+{
+  Header header = lookup_header(cid, oid);
+  if (!header)
+    return -ENOENT;
+
+  ObjectMap::ObjectMapIterator iter = _get_iterator(header, prefix);
+  for (; iter->valid(); iter->next()) {
+    if (iter->status())
+      return iter->status();
+    keys->insert(iter->key());
+  }
+  return 0;
+}
+
+int GenericObjectMap::get_values(const coll_t &cid, const ghobject_t &oid,
+                                 const string &prefix,
+                                 const set &keys,
+                                 map *out)
+{
+  Header header = lookup_header(cid, oid);
+  if (!header)
+    return -ENOENT;
+  return scan(header, prefix, keys, 0, out);
+}
+
+int GenericObjectMap::check_keys(const coll_t &cid, const ghobject_t &oid,
+                                 const string &prefix,
+                                 const set &keys,
+                                 set *out)
+{
+  Header header = lookup_header(cid, oid);
+  if (!header)
+    return -ENOENT;
+  return scan(header, prefix, keys, out, 0);
+}
+
+void GenericObjectMap::clone(const Header parent, const coll_t &cid,
+                             const ghobject_t &target,
+                             KeyValueDB::Transaction t,
+                             Header *old_header, Header *new_header)
+{
+  {
+    Header destination = lookup_header(cid, target);
+    if (destination) {
+      remove_header(cid, target, destination, t);
+      destination->num_children--;
+      _clear(destination, t);
+    }
+  }
+
+  Header source = generate_new_header(parent->cid, parent->oid, parent, t);
+  Header destination = generate_new_header(cid, target, parent, t);
+
+  destination->data = parent->data;
+  source->data = parent->data;
+
+  parent->num_children = 2;
+  set_parent_header(parent, t);
+  set_header(parent->cid, parent->oid, *source, t);
+  set_header(cid, target, *destination, t);
+
+  if (new_header)
+    *old_header = source;
+  if (new_header)
+    *new_header = destination;
+}
+
+void GenericObjectMap::rename(const Header old_header, const coll_t &cid,
+                              const ghobject_t &target,
+                              KeyValueDB::Transaction t)
+{
+  if (old_header->oid == target && old_header->cid == cid)
+    return ;
+
+  remove_header(old_header->cid, old_header->oid, old_header, t);
+  old_header->cid = cid;
+  old_header->oid = target;
+  set_header(cid, target, *old_header, t);
+}
+
+int GenericObjectMap::init(bool do_upgrade)
+{
+  map result;
+  set to_get;
+  to_get.insert(GLOBAL_STATE_KEY);
+  int r = db->get(INTERN_PREFIX, to_get, &result);
+  if (r < 0)
+    return r;
+  if (!result.empty()) {
+    bufferlist::iterator bliter = result.begin()->second.begin();
+    state.decode(bliter);
+    if (state.v < 1) { // Needs upgrade
+      if (!do_upgrade) {
+        dout(1) << "GenericObjbectMap requires an upgrade,"
+                << " set filestore_update_to"
+                << dendl;
+        return -ENOTSUP;
+      } else {
+        r = upgrade();
+        if (r < 0)
+          return r;
+      }
+    }
+  } else {
+    // New store
+    state.v = 1;
+    state.seq = 1;
+  }
+  dout(20) << "(init)genericobjectmap: seq is " << state.seq << dendl;
+  return 0;
+}
+
+int GenericObjectMap::sync(const Header header, KeyValueDB::Transaction t)
+{
+  write_state(t);
+  if (header) {
+    set_header(header->cid, header->oid, *header, t);
+  }
+  return 0;
+}
+
+bool GenericObjectMap::check(std::ostream &out)
+{
+  bool retval = true;
+  map parent_to_num_children;
+  map parent_to_actual_num_children;
+  KeyValueDB::Iterator iter = db->get_iterator(GHOBJECT_TO_SEQ_PREFIX);
+
+  for (iter->seek_to_first(); iter->valid(); iter->next()) {
+    _Header header;
+    assert(header.num_children == 1);
+    header.num_children = 0; // Hack for leaf node
+    bufferlist bl = iter->value();
+    while (true) {
+      bufferlist::iterator bliter = bl.begin();
+      header.decode(bliter);
+      if (header.seq != 0)
+        parent_to_actual_num_children[header.seq] = header.num_children;
+      if (header.parent == 0)
+        break;
+
+      if (!parent_to_num_children.count(header.parent))
+        parent_to_num_children[header.parent] = 0;
+      parent_to_num_children[header.parent]++;
+      if (parent_to_actual_num_children.count(header.parent))
+        break;
+
+      set to_get;
+      map got;
+      to_get.insert(PARENT_KEY);
+      db->get(parent_seq_prefix(header.parent), to_get, &got);
+      if (got.empty()) {
+        out << "Missing: seq " << header.parent << std::endl;
+        retval = false;
+        break;
+      } else {
+        bl = got.begin()->second;
+      }
+    }
+  }
+
+  for (map::iterator i = parent_to_num_children.begin();
+       i != parent_to_num_children.end();
+       parent_to_num_children.erase(i++)) {
+    if (!parent_to_actual_num_children.count(i->first))
+      continue;
+    if (parent_to_actual_num_children[i->first] != i->second) {
+      out << "Invalid: seq " << i->first << " recorded children: "
+          << parent_to_actual_num_children[i->first] << " found: "
+          << i->second << std::endl;
+      retval = false;
+    }
+    parent_to_actual_num_children.erase(i->first);
+  }
+  return retval;
+}
+
+
+// ============== GenericObjectMap Intern Implementation =================
+
+int GenericObjectMap::scan(Header header,
+                           const string &prefix,
+                           const set &in_keys,
+                           set *out_keys,
+                           map *out_values)
+{
+  ObjectMap::ObjectMapIterator db_iter = _get_iterator(header, prefix);
+  for (set::const_iterator key_iter = in_keys.begin();
+       key_iter != in_keys.end();
+       ++key_iter) {
+    db_iter->lower_bound(*key_iter);
+    if (db_iter->status())
+      return db_iter->status();
+
+    if (db_iter->valid() && db_iter->key() == *key_iter) {
+      if (out_keys)
+        out_keys->insert(*key_iter);
+      if (out_values)
+        out_values->insert(make_pair(db_iter->key(), db_iter->value()));
+    }
+  }
+  return 0;
+}
+
+int GenericObjectMap::_clear(Header header, KeyValueDB::Transaction t)
+{
+  while (1) {
+    if (header->num_children) {
+      set_parent_header(header, t);
+      break;
+    }
+
+    clear_header(header, t);
+    if (!header->parent)
+      break;
+
+    Header parent = lookup_parent(header);
+    if (!parent) {
+      return -EINVAL;
+    }
+    assert(parent->num_children > 0);
+    parent->num_children--;
+    header.swap(parent);
+  }
+  return 0;
+}
+
+int GenericObjectMap::merge_new_complete(
+    Header header, const map &new_complete,
+    GenericObjectMapIterator iter, KeyValueDB::Transaction t)
+{
+  KeyValueDB::Iterator complete_iter = db->get_iterator(
+    complete_prefix(header));
+  map::const_iterator i = new_complete.begin();
+  set to_remove;
+  map to_add;
+
+  string begin, end;
+  while (i != new_complete.end()) {
+    string new_begin = i->first;
+    string new_end = i->second;
+    int r = iter->in_complete_region(new_begin, &begin, &end);
+    if (r < 0)
+      return r;
+    if (r) {
+      to_remove.insert(begin);
+      new_begin = begin;
+    }
+    ++i;
+    while (i != new_complete.end()) {
+      if (!new_end.size() || i->first <= new_end) {
+        if (!new_end.size() && i->second > new_end) {
+          new_end = i->second;
+        }
+        ++i;
+        continue;
+      }
+
+      r = iter->in_complete_region(new_end, &begin, &end);
+      if (r < 0)
+        return r;
+      if (r) {
+        to_remove.insert(begin);
+        new_end = end;
+        continue;
+      }
+      break;
+    }
+    bufferlist bl;
+    bl.append(bufferptr(new_end.c_str(), new_end.size() + 1));
+    to_add.insert(make_pair(new_begin, bl));
+  }
+  t->rmkeys(complete_prefix(header), to_remove);
+  t->set(complete_prefix(header), to_add);
+  return 0;
+}
+
+int GenericObjectMap::need_parent(GenericObjectMapIterator iter)
+{
+  int r = iter->seek_to_first();
+  if (r < 0)
+    return r;
+
+  if (!iter->valid())
+    return 0;
+
+  string begin, end;
+  if (iter->in_complete_region(iter->key(), &begin, &end) && end == "") {
+    return 0;
+  }
+  return 1;
+}
+
+int GenericObjectMap::write_state(KeyValueDB::Transaction t)
+{
+  dout(20) << __func__ << " seq is " << state.seq << dendl;
+  bufferlist bl;
+  state.encode(bl);
+  map to_write;
+  to_write[GLOBAL_STATE_KEY] = bl;
+  t->set(INTERN_PREFIX, to_write);
+  return 0;
+}
+
+// NOTE(haomai): It may occur dead lock if thread A hold header A try to header
+// B and thread hold header B try to get header A
+GenericObjectMap::Header GenericObjectMap::_lookup_header(
+    const coll_t &cid, const ghobject_t &oid)
+{
+  set to_get;
+  to_get.insert(header_key(cid, oid));
+  _Header header;
+
+  map out;
+
+  int r = db->get(GHOBJECT_TO_SEQ_PREFIX, to_get, &out);
+  if (r < 0)
+    return Header();
+  if (out.empty())
+    return Header();
+
+  bufferlist::iterator iter = out.begin()->second.begin();
+  header.decode(iter);
+
+  Header ret = Header(new _Header(header));
+  return ret;
+}
+
+GenericObjectMap::Header GenericObjectMap::_generate_new_header(
+    const coll_t &cid, const ghobject_t &oid, Header parent,
+    KeyValueDB::Transaction t)
+{
+  Header header = Header(new _Header());
+  header->seq = state.seq++;
+  if (parent) {
+    header->parent = parent->seq;
+  }
+  header->num_children = 1;
+  header->oid = oid;
+  header->cid = cid;
+
+  write_state(t);
+  return header;
+}
+
+GenericObjectMap::Header GenericObjectMap::lookup_parent(Header input)
+{
+  Mutex::Locker l(header_lock);
+  map out;
+  set keys;
+  keys.insert(PARENT_KEY);
+
+  dout(20) << "lookup_parent: parent " << input->parent
+       << " for seq " << input->seq << dendl;
+
+  int r = db->get(parent_seq_prefix(input->parent), keys, &out);
+  if (r < 0) {
+    assert(0);
+    return Header();
+  }
+  if (out.empty()) {
+    assert(0);
+    return Header();
+  }
+
+  Header header = Header(new _Header());
+  header->seq = input->parent;
+  bufferlist::iterator iter = out.begin()->second.begin();
+  header->decode(iter);
+  dout(20) << "lookup_parent: parent seq is " << header->seq << " with parent "
+           << header->parent << dendl;
+  return header;
+}
+
+GenericObjectMap::Header GenericObjectMap::lookup_create_header(
+    const coll_t &cid, const ghobject_t &oid, KeyValueDB::Transaction t)
+{
+  Mutex::Locker l(header_lock);
+  Header header = _lookup_header(cid, oid);
+  if (!header) {
+    header = _generate_new_header(cid, oid, Header(), t);
+    set_header(cid, oid, *header, t);
+  }
+  return header;
+}
+
+void GenericObjectMap::set_parent_header(Header header, KeyValueDB::Transaction t)
+{
+  dout(20) << __func__ << " setting seq " << header->seq << dendl;
+  map to_write;
+  header->encode(to_write[PARENT_KEY]);
+  t->set(parent_seq_prefix(header->seq), to_write);
+}
+
+void GenericObjectMap::clear_header(Header header, KeyValueDB::Transaction t)
+{
+  dout(20) << __func__ << " clearing seq " << header->seq << dendl;
+  t->rmkeys_by_prefix(user_prefix(header, string()));
+  t->rmkeys_by_prefix(complete_prefix(header));
+  set keys;
+  keys.insert(PARENT_KEY);
+  t->rmkeys(parent_seq_prefix(header->seq), keys);
+}
+
+// only remove GHOBJECT_TO_SEQ
+void GenericObjectMap::remove_header(const coll_t &cid,
+                                     const ghobject_t &oid, Header header,
+                                     KeyValueDB::Transaction t)
+{
+  dout(20) << __func__ << " removing " << header->seq
+           << " cid " << cid << " oid " << oid << dendl;
+  set to_remove;
+  to_remove.insert(header_key(cid, oid));
+  t->rmkeys(GHOBJECT_TO_SEQ_PREFIX, to_remove);
+}
+
+void GenericObjectMap::set_header(const coll_t &cid, const ghobject_t &oid,
+                                  _Header header, KeyValueDB::Transaction t)
+{
+  dout(20) << __func__ << " setting " << header.seq
+           << " cid " << cid << " oid " << oid << " parent seq "
+           << header.parent << dendl;
+  map to_set;
+  header.encode(to_set[header_key(cid, oid)]);
+  t->set(GHOBJECT_TO_SEQ_PREFIX, to_set);
+}
+
+int GenericObjectMap::list_objects(const coll_t &cid, ghobject_t start, int max,
+                                   vector *out, ghobject_t *next)
+{
+  // FIXME
+  Mutex::Locker l(header_lock);
+
+  if (start.is_max())
+      return 0;
+
+  if (start.hobj.is_min()) {
+    vector oids;
+
+    KeyValueDB::Iterator iter = db->get_iterator(GHOBJECT_TO_SEQ_PREFIX);
+    for (iter->lower_bound(header_key(cid)); iter->valid(); iter->next()) {
+      bufferlist bl = iter->value();
+      bufferlist::iterator bliter = bl.begin();
+      _Header header;
+      header.decode(bliter);
+
+      if (header.cid == cid)
+        oids.push_back(header.oid);
+
+      break;
+    }
+
+    if (oids.empty()) {
+      if (next)
+        *next = ghobject_t::get_max();
+      return 0;
+    }
+    start = oids[0];
+  }
+
+  int size = 0;
+  KeyValueDB::Iterator iter = db->get_iterator(GHOBJECT_TO_SEQ_PREFIX);
+  for (iter->lower_bound(header_key(cid, start)); iter->valid(); iter->next()) {
+    bufferlist bl = iter->value();
+    bufferlist::iterator bliter = bl.begin();
+    _Header header;
+    header.decode(bliter);
+
+    if (header.cid != cid) {
+      if (next)
+        *next = ghobject_t::get_max();
+      break;
+    }
+
+    if (max && size >= max) {
+      if (next)
+        *next = header.oid;
+      break;
+    }
+
+    assert(start <= header.oid);
+
+
+    size++;
+    if (out)
+      out->push_back(header.oid);
+    start = header.oid;
+  }
+
+  if (out->size())
+    dout(20) << "objects: " << *out << dendl;
+
+  if (!iter->valid())
+    if (next)
+      *next = ghobject_t::get_max();
+
+  return 0;
+}
diff --git a/ceph/src/os/GenericObjectMap.h b/ceph/src/os/GenericObjectMap.h
new file mode 100644
index 00000000..3c5e3cb9
--- /dev/null
+++ b/ceph/src/os/GenericObjectMap.h
@@ -0,0 +1,427 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack 
+ *
+ * Author: Haomai Wang 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_GENERICOBJECTMAP_H
+#define CEPH_GENERICOBJECTMAP_H
+
+#include "include/buffer.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/memory.h"
+#include "ObjectMap.h"
+#include "KeyValueDB.h"
+#include "osd/osd_types.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/simple_cache.hpp"
+
+
+/**
+ * Genericobjectmap: Provide with key/value associated to ghobject_t APIs to caller
+ * and avoid concerning too much. Wrap and combine KeyValueDB/ObjectMap APIs
+ * with ghobject_t and adding clone capacity.
+ *
+ * Prefix space structure:
+ *
+ * - GHOBJECT_TO_SEQ: Contains leaf mapping from ghobject_t->Header(including
+ *                    hobj.seq and related metadata)
+ * - INTERN_PREFIX: GLOBAL_STATE_KEY - contains the global state
+ *                                  @see State
+ *                                  @see write_state
+ *                                  @see init
+ *                                  @see generate_new_header
+ * - INTERN_PREFIX + header_key(header->seq) + COMPLETE_PREFIX: see below
+ * - INTERN_PREFIX + header_key(header->seq) + PARENT_KEY
+ *              : used to store parent header(same as headers in GHOBJECT_TO_SEQ)
+ * - USER_PREFIX + header_key(header->seq) + [CUSTOM_PREFIX]
+ *              : key->value which set by callers
+ *
+ * For each node (represented by a header), we
+ * store three mappings: the key mapping, the complete mapping, and the parent.
+ * The complete mapping (COMPLETE_PREFIX space) is key->key.  Each x->y entry in
+ * this mapping indicates that the key mapping contains all entries on [x,y).
+ * Note, max string is represented by "", so ""->"" indicates that the parent
+ * is unnecessary (@see rm_keys).  When looking up a key not contained in the
+ * the complete set, we have to check the parent if we don't find it in the
+ * key set.  During rm_keys, we copy keys from the parent and update the
+ * complete set to reflect the change @see rm_keys.
+ */
+
+// This class only provide basic read capacity, suggest inherit it to
+// implement write transaction to use it. @see StripObjectMap
+class GenericObjectMap {
+ public:
+  boost::scoped_ptr db;
+
+  /**
+   * Serializes access to next_seq as well as the in_use set
+   */
+  Mutex header_lock;
+
+  GenericObjectMap(KeyValueDB *db) : db(db), header_lock("GenericObjectMap") {}
+
+  int get(
+    const coll_t &cid,
+    const ghobject_t &oid,
+    const string &prefix,
+    map *out
+    );
+
+  int get_keys(
+    const coll_t &cid,
+    const ghobject_t &oid,
+    const string &prefix,
+    set *keys
+    );
+
+  int get_values(
+    const coll_t &cid,
+    const ghobject_t &oid,
+    const string &prefix,
+    const set &keys,
+    map *out
+    );
+
+  int check_keys(
+    const coll_t &cid,
+    const ghobject_t &oid,
+    const string &prefix,
+    const set &keys,
+    set *out
+    );
+
+  /// Read initial state from backing store
+  int init(bool upgrade = false);
+
+  /// Upgrade store to current version
+  int upgrade() {return 0;}
+
+  /// Consistency check, debug, there must be no parallel writes
+  bool check(std::ostream &out);
+
+  /// Util, list all objects, there must be no other concurrent access
+  int list_objects(const coll_t &cid, ghobject_t start, int max,
+                   vector *objs, ///< [out] objects
+                   ghobject_t *next);
+
+  ObjectMap::ObjectMapIterator get_iterator(const coll_t &cid,
+                                            const ghobject_t &oid,
+                                            const string &prefix);
+
+  KeyValueDB::Transaction get_transaction() { return db->get_transaction(); }
+  int submit_transaction(KeyValueDB::Transaction t) {
+    return db->submit_transaction(t);
+  }
+
+  /// persistent state for store @see generate_header
+  struct State {
+    __u8 v;
+    uint64_t seq;
+    State() : v(0), seq(1) {}
+    State(uint64_t seq) : v(0), seq(seq) {}
+
+    void encode(bufferlist &bl) const {
+      ENCODE_START(1, 1, bl);
+      ::encode(v, bl);
+      ::encode(seq, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(bufferlist::iterator &bl) {
+      DECODE_START(1, bl);
+      ::decode(v, bl);
+      ::decode(seq, bl);
+      DECODE_FINISH(bl);
+    }
+
+    void dump(Formatter *f) const {
+      f->dump_unsigned("seq", seq);
+    }
+
+    static void generate_test_instances(list &o) {
+      o.push_back(new State(0));
+      o.push_back(new State(20));
+    }
+  } state;
+
+  struct _Header {
+    uint64_t seq;
+    uint64_t parent;
+    uint64_t num_children;
+
+    coll_t cid;
+    ghobject_t oid;
+
+    // Used by successor
+    bufferlist data;
+
+    void encode(bufferlist &bl) const {
+      ENCODE_START(1, 1, bl);
+      ::encode(seq, bl);
+      ::encode(parent, bl);
+      ::encode(num_children, bl);
+      ::encode(cid, bl);
+      ::encode(oid, bl);
+      ::encode(data, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(bufferlist::iterator &bl) {
+      DECODE_START(1, bl);
+      ::decode(seq, bl);
+      ::decode(parent, bl);
+      ::decode(num_children, bl);
+      ::decode(cid, bl);
+      ::decode(oid, bl);
+      ::decode(data, bl);
+      DECODE_FINISH(bl);
+    }
+
+    void dump(Formatter *f) const {
+      f->dump_unsigned("seq", seq);
+      f->dump_unsigned("parent", parent);
+      f->dump_unsigned("num_children", num_children);
+      f->dump_stream("coll") << cid;
+      f->dump_stream("oid") << oid;
+    }
+
+    _Header() : seq(0), parent(0), num_children(1) {}
+  };
+
+  typedef ceph::shared_ptr<_Header> Header;
+
+  Header lookup_header(const coll_t &cid, const ghobject_t &oid) {
+    Mutex::Locker l(header_lock);
+    return _lookup_header(cid, oid);
+  }
+
+  /// Lookup or create header for c oid
+  Header lookup_create_header(const coll_t &cid, const ghobject_t &oid,
+    KeyValueDB::Transaction t);
+
+  /// Set leaf node for c and oid to the value of header
+  void set_header(const coll_t &cid, const ghobject_t &oid, _Header header,
+    KeyValueDB::Transaction t);
+
+  // Move all modify member function to "protect", in order to indicate these
+  // should be made use of by sub-class
+  void set_keys(
+    const Header header,
+    const string &prefix,
+    const map &set,
+    KeyValueDB::Transaction t
+    );
+
+  int clear(
+    const Header header,
+    KeyValueDB::Transaction t
+    );
+
+  int rm_keys(
+    const Header header,
+    const string &prefix,
+    const set &to_clear,
+    KeyValueDB::Transaction t
+    );
+
+  void clone(
+    const Header origin_header,
+    const coll_t &cid,
+    const ghobject_t &target,
+    KeyValueDB::Transaction t,
+    Header *old_header,
+    Header *new_header
+    );
+
+  void rename(
+    const Header header,
+    const coll_t &cid,
+    const ghobject_t &target,
+    KeyValueDB::Transaction t
+    );
+
+  /// Ensure that all previous operations are durable
+  int sync(const Header header, KeyValueDB::Transaction t);
+
+  static const string GLOBAL_STATE_KEY;
+  static const string PARENT_KEY;
+
+  static const string USER_PREFIX;
+  static const string INTERN_PREFIX;
+  static const string PARENT_PREFIX;
+  static const string COMPLETE_PREFIX;
+  static const string GHOBJECT_TO_SEQ_PREFIX;
+
+  static const string GHOBJECT_KEY_SEP_S;
+  static const char GHOBJECT_KEY_SEP_C;
+
+private:
+  /// Implicit lock on Header->seq
+
+  static string header_key(const coll_t &cid);
+  static string header_key(const coll_t &cid, const ghobject_t &oid);
+  static bool parse_header_key(const string &in, coll_t *c, ghobject_t *oid);
+
+  string seq_key(uint64_t seq) {
+    char buf[100];
+    snprintf(buf, sizeof(buf), "%.*" PRId64, (int)(2*sizeof(seq)), seq);
+    return string(buf);
+  }
+
+  string user_prefix(Header header, const string &prefix);
+  string complete_prefix(Header header);
+  string parent_seq_prefix(uint64_t seq);
+
+  class EmptyIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
+  public:
+    int seek_to_first() { return 0; }
+    int seek_to_last() { return 0; }
+    int upper_bound(const string &after) { return 0; }
+    int lower_bound(const string &to) { return 0; }
+    bool valid() { return false; }
+    int next() { assert(0); return 0; }
+    string key() { assert(0); return ""; }
+    bufferlist value() { assert(0); return bufferlist(); }
+    int status() { return 0; }
+  };
+
+
+  /// Iterator
+  class GenericObjectMapIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
+  public:
+    GenericObjectMap *map;
+
+    /// NOTE: implicit lock on header->seq AND for all ancestors
+    Header header;
+
+    /// parent_iter == NULL iff no parent
+    ceph::shared_ptr parent_iter;
+    KeyValueDB::Iterator key_iter;
+    KeyValueDB::Iterator complete_iter;
+
+    /// cur_iter points to currently valid iterator
+    ceph::shared_ptr cur_iter;
+    int r;
+
+    /// init() called, key_iter, complete_iter, parent_iter filled in
+    bool ready;
+    /// past end
+    bool invalid;
+
+    string prefix;
+
+    GenericObjectMapIteratorImpl(GenericObjectMap *map, Header header,
+        const string &_prefix) : map(map), header(header), r(0), ready(false),
+                                 invalid(true), prefix(_prefix) { }
+    int seek_to_first();
+    int seek_to_last();
+    int upper_bound(const string &after);
+    int lower_bound(const string &to);
+    bool valid();
+    int next();
+    string key();
+    bufferlist value();
+    int status();
+
+    bool on_parent() {
+      return cur_iter == parent_iter;
+    }
+
+    /// skips to next valid parent entry
+    int next_parent();
+
+    /// Tests whether to_test is in complete region
+    int in_complete_region(const string &to_test, ///[in] key to test
+                           string *begin,         ///[out] beginning of region
+                           string *end            ///[out] end of region
+      ); ///< @returns true if to_test is in the complete region, else false
+
+  private:
+    int init();
+    bool valid_parent();
+    int adjust();
+  };
+
+protected:
+  typedef ceph::shared_ptr GenericObjectMapIterator;
+  GenericObjectMapIterator _get_iterator(Header header, string prefix) {
+    return GenericObjectMapIterator(new GenericObjectMapIteratorImpl(this, header, prefix));
+  }
+
+  Header generate_new_header(const coll_t &cid, const ghobject_t &oid,
+                             Header parent, KeyValueDB::Transaction t) {
+    Mutex::Locker l(header_lock);
+    return _generate_new_header(cid, oid, parent, t);
+  }
+
+  // Scan keys in header into out_keys and out_values (if nonnull)
+  int scan(Header header, const string &prefix, const set &in_keys,
+           set *out_keys, map *out_values);
+
+ private:
+
+  /// Removes node corresponding to header
+  void clear_header(Header header, KeyValueDB::Transaction t);
+
+  /// Set node containing input to new contents
+  void set_parent_header(Header input, KeyValueDB::Transaction t);
+
+    /// Remove leaf node corresponding to oid in c
+  void remove_header(const coll_t &cid, const ghobject_t &oid, Header header,
+      KeyValueDB::Transaction t);
+
+  /**
+   * Generate new header for c oid with new seq number
+   *
+   * Has the side effect of syncronously saving the new GenericObjectMap state
+   */
+  Header _generate_new_header(const coll_t &cid, const ghobject_t &oid,
+                              Header parent, KeyValueDB::Transaction t);
+
+  // Lookup leaf header for c oid
+  Header _lookup_header(const coll_t &cid, const ghobject_t &oid);
+
+  // Lookup header node for input
+  Header lookup_parent(Header input);
+
+  // Remove header and all related prefixes
+  int _clear(Header header, KeyValueDB::Transaction t);
+
+  // Adds to t operations necessary to add new_complete to the complete set
+  int merge_new_complete(Header header, const map &new_complete,
+      GenericObjectMapIterator iter, KeyValueDB::Transaction t);
+
+  // Writes out State (mainly next_seq)
+  int write_state(KeyValueDB::Transaction _t);
+
+  // 0 if the complete set now contains all of key space, < 0 on error, 1 else
+  int need_parent(GenericObjectMapIterator iter);
+
+  // Copies header entry from parent @see rm_keys
+  int copy_up_header(Header header, KeyValueDB::Transaction t);
+
+  // Sets header @see set_header
+  void _set_header(Header header, const bufferlist &bl,
+                   KeyValueDB::Transaction t);
+};
+WRITE_CLASS_ENCODER(GenericObjectMap::_Header)
+WRITE_CLASS_ENCODER(GenericObjectMap::State)
+
+#endif
diff --git a/ceph/src/os/HashIndex.cc b/ceph/src/os/HashIndex.cc
new file mode 100644
index 00000000..35cb49bb
--- /dev/null
+++ b/ceph/src/os/HashIndex.cc
@@ -0,0 +1,737 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "osd/osd_types.h"
+#include 
+
+#include "HashIndex.h"
+
+#include "common/debug.h"
+#define dout_subsys ceph_subsys_filestore
+
+const string HashIndex::SUBDIR_ATTR = "contents";
+const string HashIndex::IN_PROGRESS_OP_TAG = "in_progress_op";
+
+int HashIndex::cleanup() {
+  bufferlist bl;
+  int r = get_attr_path(vector(), IN_PROGRESS_OP_TAG, bl);
+  if (r < 0) {
+    // No in progress operations!
+    return 0;
+  }
+  bufferlist::iterator i = bl.begin();
+  InProgressOp in_progress(i);
+  subdir_info_s info;
+  r = get_info(in_progress.path, &info);
+  if (r == -ENOENT) {
+    return end_split_or_merge(in_progress.path);
+  } else if (r < 0) {
+    return r;
+  }
+
+  if (in_progress.is_split())
+    return complete_split(in_progress.path, info);
+  else if (in_progress.is_merge())
+    return complete_merge(in_progress.path, info);
+  else if (in_progress.is_col_split()) {
+    for (vector::iterator i = in_progress.path.begin();
+	 i != in_progress.path.end();
+	 ++i) {
+      vector path(in_progress.path.begin(), i);
+      int r = reset_attr(path);
+      if (r < 0)
+	return r;
+    }
+    return 0;
+  }
+  else
+    return -EINVAL;
+}
+
+int HashIndex::reset_attr(
+  const vector &path)
+{
+  int exists = 0;
+  int r = path_exists(path, &exists);
+  if (r < 0)
+    return r;
+  if (!exists)
+    return 0;
+  map objects;
+  set subdirs;
+  r = list_objects(path, 0, 0, &objects);
+  if (r < 0)
+    return r;
+  r = list_subdirs(path, &subdirs);
+  if (r < 0)
+    return r;
+
+  subdir_info_s info;
+  info.hash_level = path.size();
+  info.objs = objects.size();
+  info.subdirs = subdirs.size();
+  return set_info(path, info);
+}
+
+int HashIndex::col_split_level(
+  HashIndex &from,
+  HashIndex &to,
+  const vector &path,
+  uint32_t inbits,
+  uint32_t match,
+  unsigned *mkdirred)
+{
+  /* For each subdir, move, recurse, or ignore based on comparing the low order
+   * bits of the hash represented by the subdir path with inbits, match passed
+   * in.
+   */
+  set subdirs;
+  int r = from.list_subdirs(path, &subdirs);
+  if (r < 0)
+    return r;
+  map objects;
+  r = from.list_objects(path, 0, 0, &objects);
+  if (r < 0)
+    return r;
+
+  set to_move;
+  for (set::iterator i = subdirs.begin();
+       i != subdirs.end();
+       ++i) {
+    uint32_t bits = 0;
+    uint32_t hash = 0;
+    vector sub_path(path.begin(), path.end());
+    sub_path.push_back(*i);
+    path_to_hobject_hash_prefix(sub_path, &bits, &hash);
+    if (bits < inbits) {
+      if (hobject_t::match_hash(hash, bits, match)) {
+	r = col_split_level(
+	  from,
+	  to,
+	  sub_path,
+	  inbits,
+	  match,
+	  mkdirred);
+	if (r < 0)
+	  return r;
+	if (*mkdirred > path.size())
+	  *mkdirred = path.size();
+      } // else, skip, doesn't need to be moved or recursed into
+    } else {
+      if (hobject_t::match_hash(hash, inbits, match)) {
+	to_move.insert(*i);
+      }
+    } // else, skip, doesn't need to be moved or recursed into
+  }
+
+  /* Then, do the same for each object */
+  map objs_to_move;
+  for (map::iterator i = objects.begin();
+       i != objects.end();
+       ++i) {
+    if (i->second.match(inbits, match)) {
+      objs_to_move.insert(*i);
+    }
+  }
+
+  if (objs_to_move.empty() && to_move.empty())
+    return 0;
+
+  // Make parent directories as needed
+  while (*mkdirred < path.size()) {
+    ++*mkdirred;
+    int exists = 0;
+    vector creating_path(path.begin(), path.begin()+*mkdirred);
+    r = to.path_exists(creating_path, &exists);
+    if (r < 0)
+      return r;
+    if (exists)
+      continue;
+    subdir_info_s info;
+    info.objs = 0;
+    info.subdirs = 0;
+    info.hash_level = creating_path.size();
+    if (*mkdirred < path.size() - 1)
+      info.subdirs = 1;
+    r = to.start_col_split(creating_path);
+    if (r < 0)
+      return r;
+    r = to.create_path(creating_path);
+    if (r < 0)
+      return r;
+    r = to.set_info(creating_path, info);
+    if (r < 0)
+      return r;
+    r = to.end_split_or_merge(creating_path);
+    if (r < 0)
+      return r;
+  }
+
+  subdir_info_s from_info;
+  subdir_info_s to_info;
+  r = from.get_info(path, &from_info);
+  if (r < 0)
+    return r;
+  r = to.get_info(path, &to_info);
+  if (r < 0)
+    return r;
+
+  from.start_col_split(path);
+  to.start_col_split(path);
+
+  // Do subdir moves
+  for (set::iterator i = to_move.begin();
+       i != to_move.end();
+       ++i) {
+    from_info.subdirs--;
+    to_info.subdirs++;
+    r = move_subdir(from, to, path, *i);
+    if (r < 0)
+      return r;
+  }
+
+  for (map::iterator i = objs_to_move.begin();
+       i != objs_to_move.end();
+       ++i) {
+    from_info.objs--;
+    to_info.objs++;
+    r = move_object(from, to, path, *i);
+    if (r < 0)
+      return r;
+  }
+       
+
+  r = to.set_info(path, to_info);
+  if (r < 0)
+    return r;
+  r = from.set_info(path, from_info);
+  if (r < 0)
+    return r;
+  from.end_split_or_merge(path);
+  to.end_split_or_merge(path);
+  return 0;
+}
+
+int HashIndex::_split(
+  uint32_t match,
+  uint32_t bits,
+  ceph::shared_ptr dest) {
+  assert(collection_version() == dest->collection_version());
+  unsigned mkdirred = 0;
+  return col_split_level(
+    *this,
+    *static_cast(dest.get()),
+    vector(),
+    bits,
+    match,
+    &mkdirred);
+}
+
+int HashIndex::_init() {
+  subdir_info_s info;
+  vector path;
+  return set_info(path, info);
+}
+
+/* LFNIndex virtual method implementations */
+int HashIndex::_created(const vector &path,
+			const ghobject_t &oid,
+			const string &mangled_name) {
+  subdir_info_s info;
+  int r;
+  r = get_info(path, &info);
+  if (r < 0)
+    return r;
+  info.objs++;
+  r = set_info(path, info);
+  if (r < 0)
+    return r;
+
+  if (must_split(info)) {
+    int r = initiate_split(path, info);
+    if (r < 0)
+      return r;
+    return complete_split(path, info);
+  } else {
+    return 0;
+  }
+}
+
+int HashIndex::_remove(const vector &path,
+		       const ghobject_t &oid,
+		       const string &mangled_name) {
+  int r;
+  r = remove_object(path, oid);
+  if (r < 0)
+    return r;
+  subdir_info_s info;
+  r = get_info(path, &info);
+  if (r < 0)
+    return r;
+  info.objs--;
+  r = set_info(path, info);
+  if (r < 0)
+    return r;
+  if (must_merge(info)) {
+    r = initiate_merge(path, info);
+    if (r < 0)
+      return r;
+    return complete_merge(path, info);
+  } else {
+    return 0;
+  }
+}
+
+int HashIndex::_lookup(const ghobject_t &oid,
+		       vector *path,
+		       string *mangled_name,
+		       int *exists_out) {
+  vector path_comp;
+  get_path_components(oid, &path_comp);
+  vector::iterator next = path_comp.begin();
+  int exists;
+  while (1) {
+    int r = path_exists(*path, &exists);
+    if (r < 0)
+      return r;
+    if (!exists) {
+      if (path->empty())
+	return -ENOENT;
+      path->pop_back();
+      break;
+    }
+    if (next == path_comp.end())
+      break;
+    path->push_back(*(next++));
+  }
+  return get_mangled_name(*path, oid, mangled_name, exists_out);
+}
+
+int HashIndex::_collection_list(vector *ls) {
+  vector path;
+  return list_by_hash(path, 0, 0, 0, 0, ls);
+}
+
+int HashIndex::_collection_list_partial(const ghobject_t &start,
+					int min_count,
+					int max_count,
+					snapid_t seq,
+					vector *ls,
+					ghobject_t *next) {
+  vector path;
+  ghobject_t _next;
+  if (!next)
+    next = &_next;
+  *next = start;
+  dout(20) << "_collection_list_partial " << start << " " << min_count << "-" << max_count << " ls.size " << ls->size() << dendl;
+  return list_by_hash(path, min_count, max_count, seq, next, ls);
+}
+
+int HashIndex::prep_delete() {
+  return recursive_remove(vector());
+}
+
+int HashIndex::recursive_remove(const vector &path) {
+  set subdirs;
+  int r = list_subdirs(path, &subdirs);
+  if (r < 0)
+    return r;
+  map objects;
+  r = list_objects(path, 0, 0, &objects);
+  if (r < 0)
+    return r;
+  if (!objects.empty())
+    return -ENOTEMPTY;
+  vector subdir(path);
+  for (set::iterator i = subdirs.begin();
+       i != subdirs.end();
+       ++i) {
+    subdir.push_back(*i);
+    r = recursive_remove(subdir);
+    if (r < 0)
+      return r;
+    subdir.pop_back();
+  }
+  return remove_path(path);
+}
+
+int HashIndex::start_col_split(const vector &path) {
+  bufferlist bl;
+  InProgressOp op_tag(InProgressOp::COL_SPLIT, path);
+  op_tag.encode(bl);
+  int r = add_attr_path(vector(), IN_PROGRESS_OP_TAG, bl);
+  if (r < 0)
+    return r;
+  return fsync_dir(vector());
+}
+
+int HashIndex::start_split(const vector &path) {
+  bufferlist bl;
+  InProgressOp op_tag(InProgressOp::SPLIT, path);
+  op_tag.encode(bl);
+  int r = add_attr_path(vector(), IN_PROGRESS_OP_TAG, bl);
+  if (r < 0)
+    return r;
+  return fsync_dir(vector());
+}
+
+int HashIndex::start_merge(const vector &path) {
+  bufferlist bl;
+  InProgressOp op_tag(InProgressOp::MERGE, path);
+  op_tag.encode(bl);
+  int r = add_attr_path(vector(), IN_PROGRESS_OP_TAG, bl);
+  if (r < 0)
+    return r;
+  return fsync_dir(vector());
+}
+
+int HashIndex::end_split_or_merge(const vector &path) {
+  return remove_attr_path(vector(), IN_PROGRESS_OP_TAG);
+}
+
+int HashIndex::get_info(const vector &path, subdir_info_s *info) {
+  bufferlist buf;
+  int r = get_attr_path(path, SUBDIR_ATTR, buf);
+  if (r < 0)
+    return r;
+  bufferlist::iterator bufiter = buf.begin();
+  info->decode(bufiter);
+  assert(path.size() == (unsigned)info->hash_level);
+  return 0;
+}
+
+int HashIndex::set_info(const vector &path, const subdir_info_s &info) {
+  bufferlist buf;
+  assert(path.size() == (unsigned)info.hash_level);
+  info.encode(buf);
+  return add_attr_path(path, SUBDIR_ATTR, buf);
+}
+
+bool HashIndex::must_merge(const subdir_info_s &info) {
+  return (info.hash_level > 0 &&
+          merge_threshold > 0 &&
+	  info.objs < (unsigned)merge_threshold &&
+	  info.subdirs == 0);
+}
+
+bool HashIndex::must_split(const subdir_info_s &info) {
+  return (info.hash_level < (unsigned)MAX_HASH_LEVEL &&
+	  info.objs > ((unsigned)(abs(merge_threshold)) * 16 * split_multiplier));
+			    
+}
+
+int HashIndex::initiate_merge(const vector &path, subdir_info_s info) {
+  return start_merge(path);
+}
+
+int HashIndex::complete_merge(const vector &path, subdir_info_s info) {
+  vector dst = path;
+  dst.pop_back();
+  subdir_info_s dstinfo;
+  int r, exists;
+  r = path_exists(path, &exists);
+  if (r < 0)
+    return r;
+  r = get_info(dst, &dstinfo);
+  if (r < 0)
+    return r;
+  if (exists) {
+    r = move_objects(path, dst);
+    if (r < 0)
+      return r;
+    r = reset_attr(dst);
+    if (r < 0)
+      return r;
+    r = remove_path(path);
+    if (r < 0)
+      return r;
+  }
+  if (must_merge(dstinfo)) {
+    r = initiate_merge(dst, dstinfo);
+    if (r < 0)
+      return r;
+    r = fsync_dir(dst);
+    if (r < 0)
+      return r;
+    return complete_merge(dst, dstinfo);
+  }
+  r = fsync_dir(dst);
+  if (r < 0)
+    return r;
+  return end_split_or_merge(path);
+}
+
+int HashIndex::initiate_split(const vector &path, subdir_info_s info) {
+  return start_split(path);
+}
+
+int HashIndex::complete_split(const vector &path, subdir_info_s info) {
+  int level = info.hash_level;
+  map objects;
+  vector dst = path;
+  int r;
+  dst.push_back("");
+  r = list_objects(path, 0, 0, &objects);
+  if (r < 0)
+    return r;
+  set subdirs;
+  r = list_subdirs(path, &subdirs);
+  if (r < 0)
+    return r;
+  map > mapped;
+  map moved;
+  int num_moved = 0;
+  for (map::iterator i = objects.begin();
+       i != objects.end();
+       ++i) {
+    vector new_path;
+    get_path_components(i->second, &new_path);
+    mapped[new_path[level]][i->first] = i->second;
+  }
+  for (map >::iterator i = mapped.begin();
+       i != mapped.end();
+       ) {
+    dst[level] = i->first;
+    /* If the info already exists, it must be correct,
+     * we may be picking up a partially finished split */
+    subdir_info_s temp;
+    // subdir has already been fully copied
+    if (subdirs.count(i->first) && !get_info(dst, &temp)) {
+      for (map::iterator j = i->second.begin();
+	   j != i->second.end();
+	   ++j) {
+	moved[j->first] = j->second;
+	num_moved++;
+	objects.erase(j->first);
+      }
+      ++i;
+      continue;
+    }
+
+    subdir_info_s info_new;
+    info_new.objs = i->second.size();
+    info_new.subdirs = 0;
+    info_new.hash_level = level + 1;
+    if (must_merge(info_new) && !subdirs.count(i->first)) {
+      mapped.erase(i++);
+      continue;
+    }
+
+    // Subdir doesn't yet exist
+    if (!subdirs.count(i->first)) {
+      info.subdirs += 1;
+      r = create_path(dst);
+      if (r < 0)
+	return r;
+    } // else subdir has been created but only partially copied
+
+    for (map::iterator j = i->second.begin();
+	 j != i->second.end();
+	 ++j) {
+      moved[j->first] = j->second;
+      num_moved++;
+      objects.erase(j->first);
+      r = link_object(path, dst, j->second, j->first);
+      // May be a partially finished split
+      if (r < 0 && r != -EEXIST) {
+	return r;
+      }
+    }
+
+    r = fsync_dir(dst);
+    if (r < 0)
+      return r;
+
+    // Presence of info must imply that all objects have been copied
+    r = set_info(dst, info_new);
+    if (r < 0)
+      return r;
+
+    r = fsync_dir(dst);
+    if (r < 0)
+      return r;
+
+    ++i;
+  }
+  r = remove_objects(path, moved, &objects);
+  if (r < 0)
+    return r;
+  info.objs = objects.size();
+  r = reset_attr(path);
+  if (r < 0)
+    return r;
+  r = fsync_dir(path);
+  if (r < 0)
+    return r;
+  return end_split_or_merge(path);
+}
+
+void HashIndex::get_path_components(const ghobject_t &oid,
+				    vector *path) {
+  char buf[MAX_HASH_LEVEL + 1];
+  snprintf(buf, sizeof(buf), "%.*X", MAX_HASH_LEVEL, (uint32_t)oid.hobj.get_filestore_key());
+
+  // Path components are the hex characters of oid.hobj.hash, least
+  // significant first
+  for (int i = 0; i < MAX_HASH_LEVEL; ++i) {
+    path->push_back(string(&buf[i], 1));
+  }
+}
+
+string HashIndex::get_hash_str(uint32_t hash) {
+  char buf[MAX_HASH_LEVEL + 1];
+  snprintf(buf, sizeof(buf), "%.*X", MAX_HASH_LEVEL, hash);
+  string retval;
+  for (int i = 0; i < MAX_HASH_LEVEL; ++i) {
+    retval.push_back(buf[MAX_HASH_LEVEL - 1 - i]);
+  }
+  return retval;
+}
+
+string HashIndex::get_path_str(const ghobject_t &oid) {
+  assert(!oid.is_max());
+  return get_hash_str(oid.hobj.hash);
+}
+
+uint32_t HashIndex::hash_prefix_to_hash(string prefix) {
+  while (prefix.size() < sizeof(uint32_t) * 2) {
+    prefix.push_back('0');
+  }
+  uint32_t hash;
+  sscanf(prefix.c_str(), "%x", &hash);
+  // nibble reverse
+  hash = ((hash & 0x0f0f0f0f) << 4) | ((hash & 0xf0f0f0f0) >> 4);
+  hash = ((hash & 0x00ff00ff) << 8) | ((hash & 0xff00ff00) >> 8);
+  hash = ((hash & 0x0000ffff) << 16) | ((hash & 0xffff0000) >> 16);
+  return hash;
+}
+
+int HashIndex::get_path_contents_by_hash(const vector &path,
+					 const string *lower_bound,
+					 const ghobject_t *next_object,
+					 const snapid_t *seq,
+					 set *hash_prefixes,
+					 set > *objects) {
+  set subdirs;
+  map rev_objects;
+  int r;
+  string cur_prefix;
+  for (vector::const_iterator i = path.begin();
+       i != path.end();
+       ++i) {
+    cur_prefix.append(*i);
+  }
+  r = list_objects(path, 0, 0, &rev_objects);
+  if (r < 0)
+    return r;
+  for (map::iterator i = rev_objects.begin();
+       i != rev_objects.end();
+       ++i) {
+    string hash_prefix = get_path_str(i->second);
+    if (lower_bound && hash_prefix < *lower_bound)
+      continue;
+    if (next_object && i->second < *next_object)
+      continue;
+    if (seq && i->second.hobj.snap < *seq)
+      continue;
+    hash_prefixes->insert(hash_prefix);
+    objects->insert(pair(hash_prefix, i->second));
+  }
+  r = list_subdirs(path, &subdirs);
+  if (r < 0)
+    return r;
+  for (set::iterator i = subdirs.begin();
+       i != subdirs.end();
+       ++i) {
+    string candidate = cur_prefix + *i;
+    if (lower_bound && candidate < lower_bound->substr(0, candidate.size()))
+      continue;
+    if (next_object &&
+        (next_object->is_max() ||
+	 candidate < get_path_str(*next_object).substr(0, candidate.size())))
+      continue;
+    hash_prefixes->insert(cur_prefix + *i);
+  }
+  return 0;
+}
+
+int HashIndex::list_by_hash(const vector &path,
+			    int min_count,
+			    int max_count,
+			    snapid_t seq,
+			    ghobject_t *next,
+			    vector *out) {
+  assert(out);
+  vector next_path = path;
+  next_path.push_back("");
+  set hash_prefixes;
+  set > objects;
+  int r = get_path_contents_by_hash(path,
+				    NULL,
+				    next,
+				    &seq,
+				    &hash_prefixes,
+				    &objects);
+  if (r < 0)
+    return r;
+  dout(20) << " prefixes " << hash_prefixes << dendl;
+  for (set::iterator i = hash_prefixes.begin();
+       i != hash_prefixes.end();
+       ++i) {
+    set >::iterator j = objects.lower_bound(
+      make_pair(*i, ghobject_t()));
+    if (j == objects.end() || j->first != *i) {
+      if (min_count > 0 && out->size() > (unsigned)min_count) {
+	if (next)
+	  *next = ghobject_t(hobject_t("", "", CEPH_NOSNAP, hash_prefix_to_hash(*i), -1, ""));
+	return 0;
+      }
+      *(next_path.rbegin()) = *(i->rbegin());
+      ghobject_t next_recurse;
+      if (next)
+	next_recurse = *next;
+      r = list_by_hash(next_path,
+		       min_count,
+		       max_count,
+		       seq,
+		       &next_recurse,
+		       out);
+
+      if (r < 0)
+	return r;
+      if (!next_recurse.is_max()) {
+	if (next)
+	  *next = next_recurse;
+	return 0;
+      }
+    } else {
+      while (j != objects.end() && j->first == *i) {
+	if (max_count > 0 && out->size() == (unsigned)max_count) {
+	  if (next)
+	    *next = j->second;
+	  return 0;
+	}
+	if (!next || j->second >= *next) {
+	  out->push_back(j->second);
+	}
+	++j;
+      }
+    }
+  }
+  if (next)
+    *next = ghobject_t(hobject_t::get_max());
+  return 0;
+}
diff --git a/ceph/src/os/HashIndex.h b/ceph/src/os/HashIndex.h
new file mode 100644
index 00000000..4bf5c317
--- /dev/null
+++ b/ceph/src/os/HashIndex.h
@@ -0,0 +1,340 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_HASHINDEX_H
+#define CEPH_HASHINDEX_H
+
+#include "include/buffer.h"
+#include "include/encoding.h"
+#include "LFNIndex.h"
+
+
+/**
+ * Implements collection prehashing.
+ *
+ * @verbatim
+ *     (root) - 0 - 0
+ *                - 1
+ *                - E
+ *            - 1
+ *            - 2 - D - 0
+ *            .
+ *            .
+ *            .
+ *            - F - 0
+ * @endverbatim
+ *
+ * A file is located at the longest existing directory from the root 
+ * given by the hex characters in the hash beginning with the least
+ * significant.
+ * 
+ * ex: ghobject_t("object", CEPH_NO_SNAP, 0xA4CEE0D2)
+ * would be located in (root)/2/D/0/
+ * 
+ * Subdirectories are created when the number of objects in a directory
+ * exceed (abs(merge_threshhold)) * 16 * split_multiplier.  The number of objects in a directory 
+ * is encoded as subdir_info_s in an xattr on the directory.
+ */
+class HashIndex : public LFNIndex {
+private:
+  /// Attribute name for storing subdir info @see subdir_info_s
+  static const string SUBDIR_ATTR;
+  /// Attribute name for storing in progress op tag
+  static const string IN_PROGRESS_OP_TAG;
+  /// Size (bits) in object hash
+  static const int PATH_HASH_LEN = 32;
+  /// Max length of hashed path
+  static const int MAX_HASH_LEVEL = (PATH_HASH_LEN/4);
+
+  /**
+   * Merges occur when the number of object drops below
+   * merge_threshold and splits occur when the number of objects
+   * exceeds 16 * abs(merge_threshold) * split_multiplier.
+   * Please note if merge_threshold is less than zero, it will never do merging
+   */
+  int merge_threshold;
+  int split_multiplier;
+
+  /// Encodes current subdir state for determining when to split/merge.
+  struct subdir_info_s {
+    uint64_t objs;       ///< Objects in subdir.
+    uint32_t subdirs;    ///< Subdirs in subdir.
+    uint32_t hash_level; ///< Hashlevel of subdir.
+
+    subdir_info_s() : objs(0), subdirs(0), hash_level(0) {}
+    
+    void encode(bufferlist &bl) const
+    {
+      __u8 v = 1;
+      ::encode(v, bl);
+      ::encode(objs, bl);
+      ::encode(subdirs, bl);
+      ::encode(hash_level, bl);
+    }
+    
+    void decode(bufferlist::iterator &bl)
+    {
+      __u8 v;
+      ::decode(v, bl);
+      assert(v == 1);
+      ::decode(objs, bl);
+      ::decode(subdirs, bl);
+      ::decode(hash_level, bl);
+    }
+  };
+
+  /// Encodes in progress split or merge
+  struct InProgressOp {
+    static const int SPLIT = 0;
+    static const int MERGE = 1;
+    static const int COL_SPLIT = 2;
+    int op;
+    vector path;
+
+    InProgressOp(int op, const vector &path) 
+      : op(op), path(path) {}
+
+    InProgressOp(bufferlist::iterator &bl) {
+      decode(bl);
+    }
+
+    bool is_split() const { return op == SPLIT; }
+    bool is_col_split() const { return op == COL_SPLIT; }
+    bool is_merge() const { return op == MERGE; }
+
+    void encode(bufferlist &bl) const {
+      __u8 v = 1;
+      ::encode(v, bl);
+      ::encode(op, bl);
+      ::encode(path, bl);
+    }
+
+    void decode(bufferlist::iterator &bl) {
+      __u8 v;
+      ::decode(v, bl);
+      assert(v == 1);
+      ::decode(op, bl);
+      ::decode(path, bl);
+    }
+  };
+    
+    
+public:
+  /// Constructor.
+  HashIndex(
+    coll_t collection,     ///< [in] Collection
+    const char *base_path, ///< [in] Path to the index root.
+    int merge_at,          ///< [in] Merge threshhold.
+    int split_multiple,	   ///< [in] Split threshhold.
+    uint32_t index_version,///< [in] Index version
+    double retry_probability=0) ///< [in] retry probability
+    : LFNIndex(collection, base_path, index_version, retry_probability),
+      merge_threshold(merge_at),
+      split_multiplier(split_multiple) {}
+
+  /// @see CollectionIndex
+  uint32_t collection_version() { return index_version; }
+
+  /// @see CollectionIndex
+  int cleanup();
+
+  /// @see CollectionIndex
+  int prep_delete();
+
+  /// @see CollectionIndex
+  int _split(
+    uint32_t match,
+    uint32_t bits,
+    ceph::shared_ptr dest
+    );
+	
+protected:
+  int _init();
+
+  int _created(
+    const vector &path,
+    const ghobject_t &oid,
+    const string &mangled_name
+    );
+  int _remove(
+    const vector &path,
+    const ghobject_t &oid,
+    const string &mangled_name
+    );
+  int _lookup(
+    const ghobject_t &oid,
+    vector *path,
+    string *mangled_name,
+    int *exists
+    );
+  int _collection_list(
+    vector *ls
+    );
+  int _collection_list_partial(
+    const ghobject_t &start,
+    int min_count,
+    int max_count,
+    snapid_t seq,
+    vector *ls,
+    ghobject_t *next
+    );
+private:
+  /// Recursively remove path and its subdirs
+  int recursive_remove(
+    const vector &path ///< [in] path to remove
+    ); /// @return Error Code, 0 on success
+  /// Tag root directory at beginning of col_split
+  int start_col_split(
+    const vector &path ///< [in] path to split
+    ); ///< @return Error Code, 0 on success
+  /// Tag root directory at beginning of split
+  int start_split(
+    const vector &path ///< [in] path to split
+    ); ///< @return Error Code, 0 on success
+  /// Tag root directory at beginning of split
+  int start_merge(
+    const vector &path ///< [in] path to merge
+    ); ///< @return Error Code, 0 on success
+  /// Remove tag at end of split or merge
+  int end_split_or_merge(
+    const vector &path ///< [in] path to split or merged
+    ); ///< @return Error Code, 0 on success
+  /// Gets info from the xattr on the subdir represented by path
+  int get_info(
+    const vector &path, ///< [in] Path from which to read attribute.
+    subdir_info_s *info		///< [out] Attribute value
+    ); /// @return Error Code, 0 on success
+
+  /// Sets info to the xattr on the subdir represented by path
+  int set_info(
+    const vector &path, ///< [in] Path on which to set attribute.
+    const subdir_info_s &info  	///< [in] Value to set
+    ); /// @return Error Code, 0 on success
+
+  /// Encapsulates logic for when to split.
+  bool must_merge(
+    const subdir_info_s &info ///< [in] Info to check
+    ); /// @return True if info must be merged, False otherwise
+
+  /// Encapsulates logic for when to merge.
+  bool must_split(
+    const subdir_info_s &info ///< [in] Info to check
+    ); /// @return True if info must be split, False otherwise
+
+  /// Initiates merge
+  int initiate_merge(
+    const vector &path, ///< [in] Subdir to merge
+    subdir_info_s info		///< [in] Info attached to path
+    ); /// @return Error Code, 0 on success
+
+  /// Completes merge
+  int complete_merge(
+    const vector &path, ///< [in] Subdir to merge
+    subdir_info_s info		///< [in] Info attached to path
+    ); /// @return Error Code, 0 on success
+
+  /// Resets attr to match actual subdir contents
+  int reset_attr(
+    const vector &path ///< [in] path to cleanup
+    );
+
+  /// Initiate Split
+  int initiate_split(
+    const vector &path, ///< [in] Subdir to split
+    subdir_info_s info		///< [in] Info attached to path
+    ); /// @return Error Code, 0 on success
+
+  /// Completes Split
+  int complete_split(
+    const vector &path, ///< [in] Subdir to split
+    subdir_info_s info	       ///< [in] Info attached to path
+    ); /// @return Error Code, 0 on success
+
+  /// Determine path components from hoid hash
+  void get_path_components(
+    const ghobject_t &oid, ///< [in] Object for which to get path components
+    vector *path   ///< [out] Path components for hoid.
+    );
+
+  /// do collection split for path
+  static int col_split_level(
+    HashIndex &from,            ///< [in] from index
+    HashIndex &dest,            ///< [in] to index
+    const vector &path, ///< [in] path to split
+    uint32_t bits,              ///< [in] num bits to match
+    uint32_t match,             ///< [in] bits to match
+    unsigned *mkdirred          ///< [in,out] path[:mkdirred] has been mkdirred
+    );
+    
+
+  /** 
+   * Get string representation of ghobject_t/hash
+   *
+   * e.g: 0x01234567 -> "76543210"
+   */
+  static string get_path_str(
+    const ghobject_t &oid ///< [in] Object to get hash string for
+    ); ///< @return Hash string for hoid.
+
+  /// Get string from hash, @see get_path_str
+  static string get_hash_str(
+    uint32_t hash ///< [in] Hash to convert to a string.
+    ); ///< @return String representation of hash
+
+  /// Get hash from hash prefix string e.g. "FFFFAB" -> 0xFFFFAB00
+  static uint32_t hash_prefix_to_hash(
+    string prefix ///< [in] string to convert
+    ); ///< @return Hash
+
+  /// Get hash mod from path
+  static void path_to_hobject_hash_prefix(
+    const vector &path,///< [in] path to convert
+    uint32_t *bits,            ///< [out] bits
+    uint32_t *hash             ///< [out] hash
+    ) {
+    string hash_str;
+    for (vector::const_iterator i = path.begin();
+	 i != path.end();
+	 ++i) {
+      hash_str.push_back(*i->begin());
+    }
+    uint32_t rev_hash = hash_prefix_to_hash(hash_str);
+    if (hash)
+      *hash = rev_hash;
+    if (bits)
+      *bits = path.size() * 4;
+  }
+
+  /// Get path contents by hash
+  int get_path_contents_by_hash(
+    const vector &path,            /// [in] Path to list
+    const string *lower_bound,             /// [in] list > *lower_bound
+    const ghobject_t *next_object,          /// [in] list > *next_object
+    const snapid_t *seq,                   /// [in] list >= *seq
+    set *hash_prefixes,            /// [out] prefixes in dir
+    set > *objects /// [out] objects
+    );
+
+  /// List objects in collection in ghobject_t order
+  int list_by_hash(
+    const vector &path, /// [in] Path to list
+    int min_count,              /// [in] List at least min_count
+    int max_count,              /// [in] List at most max_count
+    snapid_t seq,               /// [in] list only objects where snap >= seq
+    ghobject_t *next,            /// [in,out] List objects >= *next
+    vector *out      /// [out] Listed objects
+    ); ///< @return Error Code, 0 on success
+};
+
+#endif
diff --git a/ceph/src/os/IndexManager.cc b/ceph/src/os/IndexManager.cc
new file mode 100644
index 00000000..e8a37858
--- /dev/null
+++ b/ceph/src/os/IndexManager.cc
@@ -0,0 +1,136 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "include/memory.h"
+#include 
+
+#if defined(__FreeBSD__)
+#include 
+#endif
+
+#include 
+
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/config.h"
+#include "common/debug.h"
+#include "include/buffer.h"
+
+#include "IndexManager.h"
+#include "FlatIndex.h"
+#include "HashIndex.h"
+#include "CollectionIndex.h"
+
+#include "chain_xattr.h"
+
+static int set_version(const char *path, uint32_t version) {
+  bufferlist bl;
+  ::encode(version, bl);
+  return chain_setxattr(path, "user.cephos.collection_version", bl.c_str(), 
+		     bl.length());
+}
+
+static int get_version(const char *path, uint32_t *version) {
+  bufferptr bp(PATH_MAX);
+  int r = chain_getxattr(path, "user.cephos.collection_version", 
+		      bp.c_str(), bp.length());
+  if (r < 0) {
+    if (r != -ENOENT) {
+      *version = 0;
+      return 0;
+    } else {
+      return r;
+    }
+  }
+  bp.set_length(r);
+  bufferlist bl;
+  bl.push_back(bp);
+  bufferlist::iterator i = bl.begin();
+  ::decode(*version, i);
+  return 0;
+}
+
+void IndexManager::put_index(coll_t c) {
+  Mutex::Locker l(lock);
+  assert(col_indices.count(c));
+  col_indices.erase(c);
+  cond.Signal();
+}
+
+int IndexManager::init_index(coll_t c, const char *path, uint32_t version) {
+  Mutex::Locker l(lock);
+  int r = set_version(path, version);
+  if (r < 0)
+    return r;
+  HashIndex index(c, path, g_conf->filestore_merge_threshold,
+		  g_conf->filestore_split_multiple,
+		  version,
+		  g_conf->filestore_index_retry_probability);
+  return index.init();
+}
+
+int IndexManager::build_index(coll_t c, const char *path, Index *index) {
+  if (upgrade) {
+    // Need to check the collection generation
+    int r;
+    uint32_t version = 0;
+    r = get_version(path, &version);
+    if (r < 0)
+      return r;
+
+    switch (version) {
+    case CollectionIndex::FLAT_INDEX_TAG: {
+      *index = Index(new FlatIndex(c, path),
+		     RemoveOnDelete(c, this));
+      return 0;
+    }
+    case CollectionIndex::HASH_INDEX_TAG: // fall through
+    case CollectionIndex::HASH_INDEX_TAG_2: // fall through
+    case CollectionIndex::HOBJECT_WITH_POOL: {
+      // Must be a HashIndex
+      *index = Index(new HashIndex(c, path, g_conf->filestore_merge_threshold,
+				   g_conf->filestore_split_multiple, version), 
+		     RemoveOnDelete(c, this));
+      return 0;
+    }
+    default: assert(0);
+    }
+
+  } else {
+    // No need to check
+    *index = Index(new HashIndex(c, path, g_conf->filestore_merge_threshold,
+				 g_conf->filestore_split_multiple,
+				 CollectionIndex::HOBJECT_WITH_POOL,
+				 g_conf->filestore_index_retry_probability),
+		   RemoveOnDelete(c, this));
+    return 0;
+  }
+}
+
+int IndexManager::get_index(coll_t c, const char *path, Index *index) {
+  Mutex::Locker l(lock);
+  while (1) {
+    if (!col_indices.count(c)) {
+      int r = build_index(c, path, index);
+      if (r < 0)
+	return r;
+      (*index)->set_ref(*index);
+      col_indices[c] = (*index);
+      break;
+    } else {
+      cond.Wait(lock);
+    }
+  }
+  return 0;
+}
diff --git a/ceph/src/os/IndexManager.h b/ceph/src/os/IndexManager.h
new file mode 100644
index 00000000..2aa7c66a
--- /dev/null
+++ b/ceph/src/os/IndexManager.h
@@ -0,0 +1,112 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#ifndef OS_INDEXMANAGER_H
+#define OS_INDEXMANAGER_H
+
+#include "include/memory.h"
+#include 
+
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/config.h"
+#include "common/debug.h"
+
+#include "CollectionIndex.h"
+#include "HashIndex.h"
+#include "FlatIndex.h"
+
+
+/// Public type for Index
+typedef ceph::shared_ptr Index;
+/**
+ * Encapsulates mutual exclusion for CollectionIndexes.
+ *
+ * Allowing a modification (removal or addition of an object) to occur
+ * while a read is occuring (lookup of an object's path and use of
+ * that path) may result in the path becoming invalid.  Thus, during
+ * the lifetime of a CollectionIndex object and any paths returned
+ * by it, no other concurrent accesses may be allowed.
+ *
+ * This is enforced using shared_ptr.  A shared_ptr
+ * is returned from get_index.  Any paths generated using that object
+ * carry a reference to the parrent index.  Once all
+ * shared_ptr references have expired, the destructor
+ * removes the weak_ptr from col_indices and wakes waiters.
+ */
+class IndexManager {
+  Mutex lock; ///< Lock for Index Manager
+  Cond cond;  ///< Cond for waiters on col_indices
+  bool upgrade;
+
+  /// Currently in use CollectionIndices
+  map > col_indices;
+
+  /// Cleans up state for c @see RemoveOnDelete
+  void put_index(
+    coll_t c ///< Put the index for c
+    );
+
+  /// Callback for shared_ptr release @see get_index
+  class RemoveOnDelete {
+  public:
+    coll_t c;
+    IndexManager *manager;
+    RemoveOnDelete(coll_t c, IndexManager *manager) : 
+      c(c), manager(manager) {}
+
+    void operator()(CollectionIndex *index) {
+      manager->put_index(c);
+      delete index;
+    }
+  };
+
+  /**
+   * Index factory
+   *
+   * Encapsulates logic for handling legacy FileStore
+   * layouts
+   *
+   * @param [in] c Collection for which to get index
+   * @param [in] path Path to collection
+   * @param [out] index Index for c
+   * @return error code
+   */
+  int build_index(coll_t c, const char *path, Index *index);
+public:
+  /// Constructor
+  IndexManager(bool upgrade) : lock("IndexManager lock"),
+			       upgrade(upgrade) {}
+
+  /**
+   * Reserve and return index for c
+   *
+   * @param [in] c Collection for which to get index
+   * @param [in] path Path to collection
+   * @param [out] index Index for c
+   * @return error code
+   */
+  int get_index(coll_t c, const char *path, Index *index);
+
+  /**
+   * Initialize index for collection c at path
+   *
+   * @param [in] c Collection for which to init Index
+   * @param [in] path Path to collection
+   * @param [in] filestore_version version of containing FileStore
+   * @return error code
+   */
+  int init_index(coll_t c, const char *path, uint32_t filestore_version);
+};
+
+#endif
diff --git a/ceph/src/os/Journal.h b/ceph/src/os/Journal.h
new file mode 100644
index 00000000..4f8658fb
--- /dev/null
+++ b/ceph/src/os/Journal.h
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_JOURNAL_H
+#define CEPH_JOURNAL_H
+
+#include 
+
+#include "include/buffer.h"
+#include "include/Context.h"
+#include "common/Finisher.h"
+#include "common/TrackedOp.h"
+
+class PerfCounters;
+
+class Journal {
+protected:
+  uuid_d fsid;
+  Finisher *finisher;
+public:
+  PerfCounters *logger;
+protected:
+  Cond *do_sync_cond;
+  bool wait_on_full;
+
+public:
+  Journal(uuid_d f, Finisher *fin, Cond *c=0) :
+    fsid(f), finisher(fin), logger(NULL),
+    do_sync_cond(c),
+    wait_on_full(false) { }
+  virtual ~Journal() { }
+
+  virtual int check() = 0;   ///< check if journal appears valid
+  virtual int create() = 0;  ///< create a fresh journal
+  virtual int open(uint64_t fs_op_seq) = 0;  ///< open an existing journal
+  virtual void close() = 0;  ///< close an open journal
+
+  virtual void flush() = 0;
+  virtual void throttle() = 0;
+
+  virtual int dump(ostream& out) { return -EOPNOTSUPP; }
+
+  void set_wait_on_full(bool b) { wait_on_full = b; }
+
+  // writes
+  virtual bool is_writeable() = 0;
+  virtual int make_writeable() = 0;
+  virtual void submit_entry(uint64_t seq, bufferlist& e, int alignment,
+			    Context *oncommit,
+			    TrackedOpRef osd_op = TrackedOpRef()) = 0;
+  virtual void commit_start(uint64_t seq) = 0;
+  virtual void committed_thru(uint64_t seq) = 0;
+
+  /// Read next journal entry - asserts on invalid journal
+  virtual bool read_entry(
+    bufferlist &bl, ///< [out] payload on successful read
+    uint64_t &seq   ///< [in,out] sequence number on last successful read
+    ) = 0; ///< @return true on successful read, false on journal end
+
+  virtual bool should_commit_now() = 0;
+
+  // reads/recovery
+  
+};
+
+#endif
diff --git a/ceph/src/os/JournalingObjectStore.cc b/ceph/src/os/JournalingObjectStore.cc
new file mode 100644
index 00000000..7616fe23
--- /dev/null
+++ b/ceph/src/os/JournalingObjectStore.cc
@@ -0,0 +1,270 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "JournalingObjectStore.h"
+
+#include "common/errno.h"
+#include "common/debug.h"
+
+#define dout_subsys ceph_subsys_journal
+#undef dout_prefix
+#define dout_prefix *_dout << "journal "
+
+
+
+void JournalingObjectStore::journal_start()
+{
+  dout(10) << "journal_start" << dendl;
+  finisher.start();
+}
+ 
+void JournalingObjectStore::journal_stop() 
+{
+  dout(10) << "journal_stop" << dendl;
+  finisher.stop();
+  if (journal) {
+    journal->close();
+    delete journal;
+    journal = 0;
+  }
+  apply_manager.reset();
+}
+
+int JournalingObjectStore::journal_replay(uint64_t fs_op_seq)
+{
+  dout(10) << "journal_replay fs op_seq " << fs_op_seq << dendl;
+
+  if (g_conf->journal_replay_from) {
+    dout(0) << "journal_replay forcing replay from " << g_conf->journal_replay_from
+	    << " instead of " << fs_op_seq << dendl;
+    // the previous op is the last one committed
+    fs_op_seq = g_conf->journal_replay_from - 1;
+  }
+
+  uint64_t op_seq = fs_op_seq;
+  apply_manager.init_seq(fs_op_seq);
+
+  if (!journal)
+    return 0;
+
+  int err = journal->open(op_seq);
+  if (err < 0) {
+    dout(3) << "journal_replay open failed with " 
+	    << cpp_strerror(err) << dendl;
+    delete journal;
+    journal = 0;
+    return err;
+  }
+
+  replaying = true;
+
+  int count = 0;
+  while (1) {
+    bufferlist bl;
+    uint64_t seq = op_seq + 1;
+    if (!journal->read_entry(bl, seq)) {
+      dout(3) << "journal_replay: end of journal, done." << dendl;
+      break;
+    }
+
+    if (seq <= op_seq) {
+      dout(3) << "journal_replay: skipping old op seq " << seq << " <= " << op_seq << dendl;
+      continue;
+    }
+    assert(op_seq == seq-1);
+    
+    dout(3) << "journal_replay: applying op seq " << seq << dendl;
+    bufferlist::iterator p = bl.begin();
+    list tls;
+    while (!p.end()) {
+      Transaction *t = new Transaction(p);
+      tls.push_back(t);
+    }
+
+    apply_manager.op_apply_start(seq);
+    int r = do_transactions(tls, seq);
+    apply_manager.op_apply_finish(seq);
+
+    op_seq = seq;
+
+    while (!tls.empty()) {
+      delete tls.front(); 
+      tls.pop_front();
+    }
+
+    dout(3) << "journal_replay: r = " << r << ", op_seq now " << op_seq << dendl;
+  }
+
+  replaying = false;
+
+  submit_manager.set_op_seq(op_seq);
+
+  // done reading, make writeable.
+  err = journal->make_writeable();
+  if (err < 0)
+    return err;
+
+  return count;
+}
+
+
+// ------------------------------------
+
+uint64_t JournalingObjectStore::ApplyManager::op_apply_start(uint64_t op)
+{
+  Mutex::Locker l(apply_lock);
+  while (blocked) {
+    // note: this only happens during journal replay
+    dout(10) << "op_apply_start blocked, waiting" << dendl;
+    blocked_cond.Wait(apply_lock);
+  }
+  dout(10) << "op_apply_start " << op << " open_ops " << open_ops << " -> " << (open_ops+1) << dendl;
+  assert(!blocked);
+  assert(op > committed_seq);
+  open_ops++;
+  return op;
+}
+
+void JournalingObjectStore::ApplyManager::op_apply_finish(uint64_t op)
+{
+  Mutex::Locker l(apply_lock);
+  dout(10) << "op_apply_finish " << op << " open_ops " << open_ops
+	   << " -> " << (open_ops-1)
+	   << ", max_applied_seq " << max_applied_seq << " -> " << MAX(op, max_applied_seq)
+	   << dendl;
+  --open_ops;
+  assert(open_ops >= 0);
+
+  // signal a blocked commit_start (only needed during journal replay)
+  if (blocked) {
+    blocked_cond.Signal();
+  }
+
+  // there can be multiple applies in flight; track the max value we
+  // note.  note that we can't _read_ this value and learn anything
+  // meaningful unless/until we've quiesced all in-flight applies.
+  if (op > max_applied_seq)
+    max_applied_seq = op;
+}
+
+uint64_t JournalingObjectStore::SubmitManager::op_submit_start()
+{
+  lock.Lock();
+  uint64_t op = ++op_seq;
+  dout(10) << "op_submit_start " << op << dendl;
+  return op;
+}
+
+void JournalingObjectStore::SubmitManager::op_submit_finish(uint64_t op)
+{
+  dout(10) << "op_submit_finish " << op << dendl;
+  if (op != op_submitted + 1) {
+    dout(0) << "op_submit_finish " << op << " expected " << (op_submitted + 1)
+	    << ", OUT OF ORDER" << dendl;
+    assert(0 == "out of order op_submit_finish");
+  }
+  op_submitted = op;
+  lock.Unlock();
+}
+
+
+// ------------------------------------------
+
+void JournalingObjectStore::ApplyManager::add_waiter(uint64_t op, Context *c)
+{
+  Mutex::Locker l(com_lock);
+  assert(c);
+  commit_waiters[op].push_back(c);
+}
+
+bool JournalingObjectStore::ApplyManager::commit_start()
+{
+  bool ret = false;
+
+  uint64_t _committing_seq = 0;
+  {
+    Mutex::Locker l(apply_lock);
+    dout(10) << "commit_start max_applied_seq " << max_applied_seq
+	     << ", open_ops " << open_ops
+	     << dendl;
+    blocked = true;
+    while (open_ops > 0) {
+      dout(10) << "commit_start waiting for " << open_ops << " open ops to drain" << dendl;
+      blocked_cond.Wait(apply_lock);
+    }
+    assert(open_ops == 0);
+    dout(10) << "commit_start blocked, all open_ops have completed" << dendl;
+    {
+      Mutex::Locker l(com_lock);
+      if (max_applied_seq == committed_seq) {
+	dout(10) << "commit_start nothing to do" << dendl;
+	blocked = false;
+	assert(commit_waiters.empty());
+	goto out;
+      }
+
+      _committing_seq = committing_seq = max_applied_seq;
+
+      dout(10) << "commit_start committing " << committing_seq
+	       << ", still blocked" << dendl;
+    }
+  }
+  ret = true;
+
+ out:
+  if (journal)
+    journal->commit_start(_committing_seq);  // tell the journal too
+  return ret;
+}
+
+void JournalingObjectStore::ApplyManager::commit_started()
+{
+  Mutex::Locker l(apply_lock);
+  // allow new ops. (underlying fs should now be committing all prior ops)
+  dout(10) << "commit_started committing " << committing_seq << ", unblocking" << dendl;
+  blocked = false;
+  blocked_cond.Signal();
+}
+
+void JournalingObjectStore::ApplyManager::commit_finish()
+{
+  Mutex::Locker l(com_lock);
+  dout(10) << "commit_finish thru " << committing_seq << dendl;
+  
+  if (journal)
+    journal->committed_thru(committing_seq);
+
+  committed_seq = committing_seq;
+  
+  map >::iterator p = commit_waiters.begin();
+  while (p != commit_waiters.end() &&
+    p->first <= committing_seq) {
+    finisher.queue(p->second);
+    commit_waiters.erase(p++);
+  }
+}
+
+void JournalingObjectStore::_op_journal_transactions(
+  list& tls, uint64_t op,
+  Context *onjournal, TrackedOpRef osd_op)
+{
+  dout(10) << "op_journal_transactions " << op << " " << tls << dendl;
+
+  if (journal && journal->is_writeable()) {
+    bufferlist tbl;
+    unsigned data_len = 0;
+    int data_align = -1; // -1 indicates that we don't care about the alignment
+    for (list::iterator p = tls.begin();
+	 p != tls.end(); ++p) {
+      ObjectStore::Transaction *t = *p;
+      if (t->get_data_length() > data_len &&
+	(int)t->get_data_length() >= g_conf->journal_align_min_size) {
+	data_len = t->get_data_length();
+	data_align = (t->get_data_alignment() - tbl.length()) & ~CEPH_PAGE_MASK;
+      }
+      ::encode(*t, tbl);
+    }
+    journal->submit_entry(op, tbl, data_align, onjournal, osd_op);
+  } else if (onjournal) {
+    apply_manager.add_waiter(op, onjournal);
+  }
+}
diff --git a/ceph/src/os/JournalingObjectStore.h b/ceph/src/os/JournalingObjectStore.h
new file mode 100644
index 00000000..fb7f0eca
--- /dev/null
+++ b/ceph/src/os/JournalingObjectStore.h
@@ -0,0 +1,139 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_JOURNALINGOBJECTSTORE_H
+#define CEPH_JOURNALINGOBJECTSTORE_H
+
+#include "ObjectStore.h"
+#include "Journal.h"
+#include "common/RWLock.h"
+
+class JournalingObjectStore : public ObjectStore {
+protected:
+  Journal *journal;
+  Finisher finisher;
+
+
+  class SubmitManager {
+    Mutex lock;
+    uint64_t op_seq;
+    uint64_t op_submitted;
+  public:
+    SubmitManager() :
+      lock("JOS::SubmitManager::lock", false, true, false, g_ceph_context),
+      op_seq(0), op_submitted(0)
+    {}
+    uint64_t op_submit_start();
+    void op_submit_finish(uint64_t op);
+    void set_op_seq(uint64_t seq) {
+      Mutex::Locker l(lock);
+      op_submitted = op_seq = seq;
+    }
+    uint64_t get_op_seq() {
+      return op_seq;
+    }
+  } submit_manager;
+
+  class ApplyManager {
+    Journal *&journal;
+    Finisher &finisher;
+
+    Mutex apply_lock;
+    bool blocked;
+    Cond blocked_cond;
+    int open_ops;
+    uint64_t max_applied_seq;
+
+    Mutex com_lock;
+    map > commit_waiters;
+    uint64_t committing_seq, committed_seq;
+
+  public:
+    ApplyManager(Journal *&j, Finisher &f) :
+      journal(j), finisher(f),
+      apply_lock("JOS::ApplyManager::apply_lock", false, true, false, g_ceph_context),
+      blocked(false),
+      open_ops(0),
+      max_applied_seq(0),
+      com_lock("JOS::ApplyManager::com_lock", false, true, false, g_ceph_context),
+      committing_seq(0), committed_seq(0) {}
+    void reset() {
+      assert(open_ops == 0);
+      assert(blocked == false);
+      max_applied_seq = 0;
+      committing_seq = 0;
+      committed_seq = 0;
+    }
+    void add_waiter(uint64_t, Context*);
+    uint64_t op_apply_start(uint64_t op);
+    void op_apply_finish(uint64_t op);
+    bool commit_start();
+    void commit_started();
+    void commit_finish();
+    bool is_committing() {
+      Mutex::Locker l(com_lock);
+      return committing_seq != committed_seq;
+    }
+    uint64_t get_committed_seq() {
+      Mutex::Locker l(com_lock);
+      return committed_seq;
+    }
+    uint64_t get_committing_seq() {
+      Mutex::Locker l(com_lock);
+      return committing_seq;
+    }
+    void init_seq(uint64_t fs_op_seq) {
+      {
+	Mutex::Locker l(com_lock);
+	committed_seq = fs_op_seq;
+	committing_seq = fs_op_seq;
+      }
+      {
+	Mutex::Locker l(apply_lock);
+	max_applied_seq = fs_op_seq;
+      }
+    }
+  } apply_manager;
+
+  bool replaying;
+
+protected:
+  void journal_start();
+  void journal_stop();
+  int journal_replay(uint64_t fs_op_seq);
+
+  void _op_journal_transactions(list& tls, uint64_t op,
+				Context *onjournal, TrackedOpRef osd_op);
+
+  virtual int do_transactions(list& tls, uint64_t op_seq) = 0;
+
+public:
+  bool is_committing() {
+    return apply_manager.is_committing();
+  }
+  uint64_t get_committed_seq() {
+    return apply_manager.get_committed_seq();
+  }
+
+public:
+  JournalingObjectStore(const std::string& path)
+    : ObjectStore(path),
+      journal(NULL),
+      finisher(g_ceph_context),
+      apply_manager(journal, finisher),
+      replaying(false) {}
+  
+};
+
+#endif
diff --git a/ceph/src/os/KeyValueDB.h b/ceph/src/os/KeyValueDB.h
new file mode 100644
index 00000000..c581aa52
--- /dev/null
+++ b/ceph/src/os/KeyValueDB.h
@@ -0,0 +1,181 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef KEY_VALUE_DB_H
+#define KEY_VALUE_DB_H
+
+#include "include/buffer.h"
+#include 
+#include 
+#include 
+#include "include/memory.h"
+#include 
+#include "ObjectMap.h"
+
+using std::string;
+/**
+ * Defines virtual interface to be implemented by key value store
+ *
+ * Kyoto Cabinet or LevelDB should implement this
+ */
+class KeyValueDB {
+public:
+  class TransactionImpl {
+  public:
+    /// Set Keys
+    void set(
+      const string &prefix,                 ///< [in] Prefix for keys
+      const std::map &to_set ///< [in] keys/values to set
+    ) {
+      std::map::const_iterator it;
+      for (it = to_set.begin(); it != to_set.end(); ++it)
+	set(prefix, it->first, it->second);
+    }
+
+    /// Set Key
+    virtual void set(
+      const string &prefix,   ///< [in] Prefix for the key
+      const string &k,	      ///< [in] Key to set
+      const bufferlist &bl    ///< [in] Value to set
+      ) = 0;
+
+
+    /// Removes Keys
+    void rmkeys(
+      const string &prefix,   ///< [in] Prefix to search for
+      const std::set &keys ///< [in] Keys to remove
+    ) {
+      std::set::const_iterator it;
+      for (it = keys.begin(); it != keys.end(); ++it)
+	rmkey(prefix, *it);
+    }
+
+    /// Remove Key
+    virtual void rmkey(
+      const string &prefix,   ///< [in] Prefix to search for
+      const string &k	      ///< [in] Key to remove
+      ) = 0;
+
+    /// Removes keys beginning with prefix
+    virtual void rmkeys_by_prefix(
+      const string &prefix ///< [in] Prefix by which to remove keys
+      ) = 0;
+
+    virtual ~TransactionImpl() {};
+  };
+  typedef ceph::shared_ptr< TransactionImpl > Transaction;
+
+  virtual int init() = 0;
+  virtual int open(ostream &out) = 0;
+  virtual int create_and_open(ostream &out) = 0;
+
+  virtual Transaction get_transaction() = 0;
+  virtual int submit_transaction(Transaction) = 0;
+  virtual int submit_transaction_sync(Transaction t) {
+    return submit_transaction(t);
+  }
+
+  /// Retrieve Keys
+  virtual int get(
+    const string &prefix,        ///< [in] Prefix for key
+    const std::set &key,      ///< [in] Key to retrieve
+    std::map *out ///< [out] Key value retrieved
+    ) = 0;
+
+  class WholeSpaceIteratorImpl {
+  public:
+    virtual int seek_to_first() = 0;
+    virtual int seek_to_first(const string &prefix) = 0;
+    virtual int seek_to_last() = 0;
+    virtual int seek_to_last(const string &prefix) = 0;
+    virtual int upper_bound(const string &prefix, const string &after) = 0;
+    virtual int lower_bound(const string &prefix, const string &to) = 0;
+    virtual bool valid() = 0;
+    virtual int next() = 0;
+    virtual int prev() = 0;
+    virtual string key() = 0;
+    virtual pair raw_key() = 0;
+    virtual bufferlist value() = 0;
+    virtual int status() = 0;
+    virtual ~WholeSpaceIteratorImpl() { }
+  };
+  typedef ceph::shared_ptr< WholeSpaceIteratorImpl > WholeSpaceIterator;
+
+  class IteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
+    const string prefix;
+    WholeSpaceIterator generic_iter;
+  public:
+    IteratorImpl(const string &prefix, WholeSpaceIterator iter) :
+      prefix(prefix), generic_iter(iter) { }
+    virtual ~IteratorImpl() { }
+
+    int seek_to_first() {
+      return generic_iter->seek_to_first(prefix);
+    }
+    int seek_to_last() {
+      return generic_iter->seek_to_last(prefix);
+    }
+    int upper_bound(const string &after) {
+      return generic_iter->upper_bound(prefix, after);
+    }
+    int lower_bound(const string &to) {
+      return generic_iter->lower_bound(prefix, to);
+    }
+    bool valid() {
+      if (!generic_iter->valid())
+	return false;
+      pair raw_key = generic_iter->raw_key();
+      return (raw_key.first == prefix);
+    }
+    int next() {
+      if (valid())
+	return generic_iter->next();
+      return status();
+    }
+    int prev() {
+      if (valid())
+	return generic_iter->prev();
+      return status();
+    }
+    string key() {
+      return generic_iter->key();
+    }
+    bufferlist value() {
+      return generic_iter->value();
+    }
+    int status() {
+      return generic_iter->status();
+    }
+  };
+
+  typedef ceph::shared_ptr< IteratorImpl > Iterator;
+
+  WholeSpaceIterator get_iterator() {
+    return _get_iterator();
+  }
+
+  Iterator get_iterator(const string &prefix) {
+    return ceph::shared_ptr(
+      new IteratorImpl(prefix, get_iterator())
+    );
+  }
+
+  WholeSpaceIterator get_snapshot_iterator() {
+    return _get_snapshot_iterator();
+  }
+
+  Iterator get_snapshot_iterator(const string &prefix) {
+    return ceph::shared_ptr(
+      new IteratorImpl(prefix, get_snapshot_iterator())
+    );
+  }
+
+  virtual uint64_t get_estimated_size(map &extra) = 0;
+
+  virtual ~KeyValueDB() {}
+
+protected:
+  virtual WholeSpaceIterator _get_iterator() = 0;
+  virtual WholeSpaceIterator _get_snapshot_iterator() = 0;
+};
+
+#endif
diff --git a/ceph/src/os/KeyValueStore.cc b/ceph/src/os/KeyValueStore.cc
new file mode 100644
index 00000000..17c0c3b4
--- /dev/null
+++ b/ceph/src/os/KeyValueStore.cc
@@ -0,0 +1,3013 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack 
+ *
+ * Author: Haomai Wang 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+#include "include/compat.h"
+
+#include 
+#include 
+
+#include "KeyValueStore.h"
+#include "common/BackTrace.h"
+#include "include/types.h"
+
+#include "osd/osd_types.h"
+#include "include/color.h"
+#include "include/buffer.h"
+
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/run_cmd.h"
+#include "common/safe_io.h"
+#include "common/perf_counters.h"
+#include "common/sync_filesystem.h"
+#include "LevelDBStore.h"
+
+#include "common/ceph_crypto.h"
+using ceph::crypto::SHA1;
+
+#include "include/assert.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_keyvaluestore
+
+const string KeyValueStore::OBJECT_STRIP_PREFIX = "_STRIP_";
+const string KeyValueStore::OBJECT_XATTR = "__OBJATTR__";
+const string KeyValueStore::OBJECT_OMAP = "__OBJOMAP__";
+const string KeyValueStore::OBJECT_OMAP_HEADER = "__OBJOMAP_HEADER__";
+const string KeyValueStore::OBJECT_OMAP_HEADER_KEY = "__OBJOMAP_HEADER__KEY_";
+const string KeyValueStore::COLLECTION = "__COLLECTION__";
+const string KeyValueStore::COLLECTION_ATTR = "__COLL_ATTR__";
+
+// ============== StripObjectMap Implementation =================
+
+int StripObjectMap::save_strip_header(StripObjectHeaderRef strip_header,
+                                      KeyValueDB::Transaction t)
+{
+  strip_header->header->data.clear();
+  ::encode(*strip_header, strip_header->header->data);
+
+  set_header(strip_header->cid, strip_header->oid, *(strip_header->header), t);
+  return 0;
+}
+
+int StripObjectMap::create_strip_header(const coll_t &cid,
+                                        const ghobject_t &oid,
+                                        StripObjectHeaderRef *strip_header,
+                                        KeyValueDB::Transaction t)
+{
+  Header header = generate_new_header(cid, oid, Header(), t);
+  if (!header)
+    return -EINVAL;
+
+  StripObjectHeaderRef tmp = StripObjectHeaderRef(new StripObjectHeader());
+  tmp->oid = oid;
+  tmp->cid = cid;
+  tmp->header = header;
+  if (strip_header)
+    *strip_header = tmp;
+
+  return 0;
+}
+
+int StripObjectMap::lookup_strip_header(const coll_t &cid,
+                                        const ghobject_t &oid,
+                                        StripObjectHeaderRef *strip_header)
+{
+  if (cid != coll_t()) {
+    Mutex::Locker l(lock);
+    pair p;
+    if (caches.lookup(oid, &p)) {
+      if (p.first == cid) {
+        *strip_header = p.second;
+        return 0;
+      }
+    }
+  }
+  Header header = lookup_header(cid, oid);
+
+  if (!header) {
+    dout(20) << "lookup_strip_header failed to get strip_header "
+             << " cid " << cid <<" oid " << oid << dendl;
+    return -ENOENT;
+  }
+
+
+  StripObjectHeaderRef tmp = StripObjectHeaderRef(new StripObjectHeader());
+  if (header->data.length()) {
+    bufferlist::iterator bliter = header->data.begin();
+    ::decode(*tmp, bliter);
+  }
+
+  if (tmp->strip_size == 0)
+    tmp->strip_size = default_strip_size;
+
+  tmp->oid = oid;
+  tmp->cid = cid;
+  tmp->header = header;
+
+  {
+    Mutex::Locker l(lock);
+    caches.add(oid, make_pair(cid, tmp));
+  }
+  *strip_header = tmp;
+  dout(10) << "lookup_strip_header done " << " cid " << cid << " oid "
+           << oid << dendl;
+  return 0;
+}
+
+int StripObjectMap::file_to_extents(uint64_t offset, size_t len,
+                                    uint64_t strip_size,
+                                    vector &extents)
+{
+  if (len == 0)
+    return 0;
+
+  uint64_t start, end, strip_offset, extent_offset, extent_len;
+  start = offset / strip_size;
+  end = (offset + len) / strip_size;
+  strip_offset = start * strip_size;
+
+  // "offset" may in the middle of first strip object
+  if (offset > strip_offset) {
+    extent_offset = offset - strip_offset;
+    if (extent_offset + len <= strip_size)
+      extent_len = len;
+    else
+      extent_len = strip_size - extent_offset;
+    extents.push_back(StripExtent(start, extent_offset, extent_len));
+    start++;
+    strip_offset += strip_size;
+  }
+
+  for (; start < end; ++start) {
+    extents.push_back(StripExtent(start, 0, strip_size));
+    strip_offset += strip_size;
+  }
+
+  // The end of strip object may be partial
+  if (offset + len > strip_offset)
+    extents.push_back(StripExtent(start, 0, offset+len-strip_offset));
+
+  assert(extents.size());
+  dout(10) << "file_to_extents done " << dendl;
+  return 0;
+}
+
+void StripObjectMap::clone_wrap(StripObjectHeaderRef old_header,
+                                const coll_t &cid, const ghobject_t &oid,
+                                KeyValueDB::Transaction t,
+                                StripObjectHeaderRef *target_header)
+{
+  Header new_origin_header;
+  StripObjectHeaderRef tmp = StripObjectHeaderRef(new StripObjectHeader());
+
+  clone(old_header->header, cid, oid, t, &new_origin_header,
+        &tmp->header);
+
+  tmp->oid = oid;
+  tmp->cid = cid;
+  tmp->strip_size = old_header->strip_size;
+  tmp->max_size = old_header->max_size;
+  tmp->bits = old_header->bits;
+  old_header->header = new_origin_header;
+
+  if (target_header)
+    *target_header = tmp;
+}
+
+void StripObjectMap::rename_wrap(StripObjectHeaderRef old_header, const coll_t &cid, const ghobject_t &oid,
+                                 KeyValueDB::Transaction t,
+                                 StripObjectHeaderRef *new_header)
+{
+  rename(old_header->header, cid, oid, t);
+
+  StripObjectHeaderRef tmp = StripObjectHeaderRef(new StripObjectHeader());
+  tmp->strip_size = old_header->strip_size;
+  tmp->max_size = old_header->max_size;
+  tmp->bits = old_header->bits;
+  tmp->header = old_header->header;
+  tmp->oid = oid;
+  tmp->cid = cid;
+
+  if (new_header)
+    *new_header = tmp;
+
+  old_header->header = Header();
+  old_header->deleted = true;
+}
+
+int StripObjectMap::get_values_with_header(const StripObjectHeaderRef header,
+                                           const string &prefix,
+                                           const set &keys,
+                                           map *out)
+{
+  return scan(header->header, prefix, keys, 0, out);
+}
+
+int StripObjectMap::get_keys_with_header(const StripObjectHeaderRef header,
+                                         const string &prefix,
+                                         set *keys)
+{
+  ObjectMap::ObjectMapIterator iter = _get_iterator(header->header, prefix);
+  for (; iter->valid(); iter->next()) {
+    if (iter->status())
+      return iter->status();
+    keys->insert(iter->key());
+  }
+  return 0;
+}
+
+int StripObjectMap::get_with_header(const StripObjectHeaderRef header,
+                        const string &prefix, map *out)
+{
+  ObjectMap::ObjectMapIterator iter = _get_iterator(header->header, prefix);
+  for (iter->seek_to_first(); iter->valid(); iter->next()) {
+    if (iter->status())
+      return iter->status();
+    out->insert(make_pair(iter->key(), iter->value()));
+  }
+
+  return 0;
+}
+
+// ========= KeyValueStore::BufferTransaction Implementation ============
+
+int KeyValueStore::BufferTransaction::lookup_cached_header(
+    const coll_t &cid, const ghobject_t &oid,
+    StripObjectMap::StripObjectHeaderRef *strip_header,
+    bool create_if_missing)
+{
+  StripObjectMap::StripObjectHeaderRef header;
+  int r = 0;
+
+  StripHeaderMap::iterator it = strip_headers.find(make_pair(cid, oid));
+  if (it != strip_headers.end()) {
+
+    if (!it->second->deleted) {
+      if (strip_header)
+        *strip_header = it->second;
+      return 0;
+    } else if (!create_if_missing) {
+      return -ENOENT;
+    }
+
+    // If (it->second.deleted && create_if_missing) go down
+    r = -ENOENT;
+  } else {
+    r = store->backend->lookup_strip_header(cid, oid, &header);
+  }
+
+  if (r == -ENOENT && create_if_missing) {
+    r = store->backend->create_strip_header(cid, oid, &header, t);
+  }
+
+  if (r < 0) {
+    dout(10) << __func__  << " " << cid << "/" << oid << " "
+             << " r = " << r << dendl;
+    return r;
+  }
+
+  strip_headers[make_pair(cid, oid)] = header;
+  if (strip_header)
+    *strip_header = strip_headers[make_pair(cid, oid)];
+  return r;
+}
+
+int KeyValueStore::BufferTransaction::get_buffer_keys(
+    StripObjectMap::StripObjectHeaderRef strip_header, const string &prefix,
+    const set &keys, map *out)
+{
+  set need_lookup;
+
+  for (set::iterator it = keys.begin(); it != keys.end(); ++it) {
+    map, bufferlist>::iterator i =
+        strip_header->buffers.find(make_pair(prefix, *it));
+
+    if (i != strip_header->buffers.end()) {
+      (*out)[*it].swap(i->second);
+    } else {
+      need_lookup.insert(*it);
+    }
+  }
+
+  if (!need_lookup.empty()) {
+    int r = store->backend->get_values_with_header(strip_header, prefix,
+                                                   need_lookup, out);
+    if (r < 0) {
+      dout(10) << __func__  << " " << strip_header->cid << "/"
+               << strip_header->oid << " " << " r = " << r << dendl;
+      return r;
+    }
+  }
+
+  return 0;
+}
+
+void KeyValueStore::BufferTransaction::set_buffer_keys(
+     StripObjectMap::StripObjectHeaderRef strip_header,
+     const string &prefix, map &values)
+{
+  store->backend->set_keys(strip_header->header, prefix, values, t);
+
+  for (map::iterator iter = values.begin();
+       iter != values.end(); ++iter) {
+    strip_header->buffers[make_pair(prefix, iter->first)].swap(iter->second);
+  }
+}
+
+int KeyValueStore::BufferTransaction::remove_buffer_keys(
+     StripObjectMap::StripObjectHeaderRef strip_header, const string &prefix,
+     const set &keys)
+{
+  for (set::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
+    strip_header->buffers[make_pair(prefix, *iter)] = bufferlist();
+  }
+
+  return store->backend->rm_keys(strip_header->header, prefix, keys, t);
+}
+
+void KeyValueStore::BufferTransaction::clear_buffer_keys(
+     StripObjectMap::StripObjectHeaderRef strip_header, const string &prefix)
+{
+  for (map, bufferlist>::iterator iter = strip_header->buffers.begin();
+       iter != strip_header->buffers.end(); ++iter) {
+    if (iter->first.first == prefix)
+      iter->second = bufferlist();
+  }
+}
+
+int KeyValueStore::BufferTransaction::clear_buffer(
+     StripObjectMap::StripObjectHeaderRef strip_header)
+{
+  strip_header->deleted = true;
+
+  InvalidateCacheContext *c = new InvalidateCacheContext(store, strip_header->cid, strip_header->oid);
+  finishes.push_back(c);
+  return store->backend->clear(strip_header->header, t);
+}
+
+void KeyValueStore::BufferTransaction::clone_buffer(
+    StripObjectMap::StripObjectHeaderRef old_header,
+    const coll_t &cid, const ghobject_t &oid)
+{
+  // Remove target ahead to avoid dead lock
+  strip_headers.erase(make_pair(cid, oid));
+
+  StripObjectMap::StripObjectHeaderRef new_target_header;
+
+  store->backend->clone_wrap(old_header, cid, oid, t, &new_target_header);
+
+  // FIXME: Lacking of lock for origin header(now become parent), it will
+  // cause other operation can get the origin header while submitting
+  // transactions
+  strip_headers[make_pair(cid, oid)] = new_target_header;
+}
+
+void KeyValueStore::BufferTransaction::rename_buffer(
+    StripObjectMap::StripObjectHeaderRef old_header,
+    const coll_t &cid, const ghobject_t &oid)
+{
+  // FIXME: Lacking of lock for origin header, it will cause other operation
+  // can get the origin header while submitting transactions
+  StripObjectMap::StripObjectHeaderRef new_header;
+  store->backend->rename_wrap(old_header, cid, oid, t, &new_header);
+
+  InvalidateCacheContext *c = new InvalidateCacheContext(store, old_header->cid, old_header->oid);
+  finishes.push_back(c);
+  strip_headers[make_pair(cid, oid)] = new_header;
+}
+
+int KeyValueStore::BufferTransaction::submit_transaction()
+{
+  int r = 0;
+
+  for (StripHeaderMap::iterator header_iter = strip_headers.begin();
+       header_iter != strip_headers.end(); ++header_iter) {
+    StripObjectMap::StripObjectHeaderRef header = header_iter->second;
+
+    if (header->deleted)
+      continue;
+
+    r = store->backend->save_strip_header(header, t);
+
+    if (r < 0) {
+      dout(10) << __func__ << " save strip header failed " << dendl;
+      goto out;
+    }
+  }
+
+  r = store->backend->submit_transaction(t);
+  for (list::iterator it = finishes.begin(); it != finishes.end(); ++it) {
+    (*it)->complete(r);
+  }
+
+out:
+  dout(5) << __func__ << " r = " << r << dendl;
+  return r;
+}
+
+// =========== KeyValueStore Intern Helper Implementation ==============
+
+ostream& operator<<(ostream& out, const KeyValueStore::OpSequencer& s)
+{
+  assert(&out);
+  return out << *s.parent;
+}
+
+int KeyValueStore::_create_current()
+{
+  struct stat st;
+  int ret = ::stat(current_fn.c_str(), &st);
+  if (ret == 0) {
+    // current/ exists
+    if (!S_ISDIR(st.st_mode)) {
+      dout(0) << "_create_current: current/ exists but is not a directory" << dendl;
+      ret = -EINVAL;
+    }
+  } else {
+    ret = ::mkdir(current_fn.c_str(), 0755);
+    if (ret < 0) {
+      ret = -errno;
+      dout(0) << "_create_current: mkdir " << current_fn << " failed: "<< cpp_strerror(ret) << dendl;
+    }
+  }
+
+  return ret;
+}
+
+
+
+// =========== KeyValueStore API Implementation ==============
+
+KeyValueStore::KeyValueStore(const std::string &base,
+                             const char *name, bool do_update) :
+  ObjectStore(base),
+  internal_name(name),
+  basedir(base),
+  fsid_fd(-1), current_fd(-1),
+  kv_type(KV_TYPE_NONE),
+  backend(NULL),
+  ondisk_finisher(g_ceph_context),
+  lock("KeyValueStore::lock"),
+  default_osr("default"),
+  op_queue_len(0), op_queue_bytes(0),
+  op_throttle_lock("KeyValueStore::op_throttle_lock"),
+  op_finisher(g_ceph_context),
+  op_tp(g_ceph_context, "KeyValueStore::op_tp",
+        g_conf->keyvaluestore_op_threads, "keyvaluestore_op_threads"),
+  op_wq(this, g_conf->keyvaluestore_op_thread_timeout,
+        g_conf->keyvaluestore_op_thread_suicide_timeout, &op_tp),
+  logger(NULL),
+  m_keyvaluestore_queue_max_ops(g_conf->keyvaluestore_queue_max_ops),
+  m_keyvaluestore_queue_max_bytes(g_conf->keyvaluestore_queue_max_bytes),
+  do_update(do_update)
+{
+  ostringstream oss;
+  oss << basedir << "/current";
+  current_fn = oss.str();
+
+  ostringstream sss;
+  sss << basedir << "/current/commit_op_seq";
+  current_op_seq_fn = sss.str();
+
+  // initialize logger
+  PerfCountersBuilder plb(g_ceph_context, internal_name, l_os_commit_lat, l_os_last);
+
+  plb.add_u64(l_os_oq_max_ops, "op_queue_max_ops");
+  plb.add_u64(l_os_oq_ops, "op_queue_ops");
+  plb.add_u64_counter(l_os_ops, "ops");
+  plb.add_u64(l_os_oq_max_bytes, "op_queue_max_bytes");
+  plb.add_u64(l_os_oq_bytes, "op_queue_bytes");
+  plb.add_u64_counter(l_os_bytes, "bytes");
+  plb.add_time_avg(l_os_apply_lat, "apply_latency");
+  plb.add_time_avg(l_os_queue_lat, "queue_transaction_latency_avg");
+
+  logger = plb.create_perf_counters();
+
+  g_ceph_context->get_perfcounters_collection()->add(logger);
+  g_ceph_context->_conf->add_observer(this);
+}
+
+KeyValueStore::~KeyValueStore()
+{
+  g_ceph_context->_conf->remove_observer(this);
+  g_ceph_context->get_perfcounters_collection()->remove(logger);
+
+  delete logger;
+}
+
+int KeyValueStore::statfs(struct statfs *buf)
+{
+  if (::statfs(basedir.c_str(), buf) < 0) {
+    int r = -errno;
+    return r;
+  }
+  return 0;
+}
+
+int KeyValueStore::mkfs()
+{
+  int ret = 0;
+  char fsid_fn[PATH_MAX];
+  uuid_d old_fsid;
+
+  dout(1) << "mkfs in " << basedir << dendl;
+
+  // open+lock fsid
+  snprintf(fsid_fn, sizeof(fsid_fn), "%s/fsid", basedir.c_str());
+  fsid_fd = ::open(fsid_fn, O_RDWR|O_CREAT, 0644);
+  if (fsid_fd < 0) {
+    ret = -errno;
+    derr << "mkfs: failed to open " << fsid_fn << ": " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  if (lock_fsid() < 0) {
+    ret = -EBUSY;
+    goto close_fsid_fd;
+  }
+
+  if (read_fsid(fsid_fd, &old_fsid) < 0 || old_fsid.is_zero()) {
+    if (fsid.is_zero()) {
+      fsid.generate_random();
+      dout(1) << "mkfs generated fsid " << fsid << dendl;
+    } else {
+      dout(1) << "mkfs using provided fsid " << fsid << dendl;
+    }
+
+    char fsid_str[40];
+    fsid.print(fsid_str);
+    strcat(fsid_str, "\n");
+    ret = ::ftruncate(fsid_fd, 0);
+    if (ret < 0) {
+      ret = -errno;
+      derr << "mkfs: failed to truncate fsid: " << cpp_strerror(ret) << dendl;
+      goto close_fsid_fd;
+    }
+    ret = safe_write(fsid_fd, fsid_str, strlen(fsid_str));
+    if (ret < 0) {
+      derr << "mkfs: failed to write fsid: " << cpp_strerror(ret) << dendl;
+      goto close_fsid_fd;
+    }
+    if (::fsync(fsid_fd) < 0) {
+      ret = errno;
+      derr << "mkfs: close failed: can't write fsid: "
+           << cpp_strerror(ret) << dendl;
+      goto close_fsid_fd;
+    }
+    dout(10) << "mkfs fsid is " << fsid << dendl;
+  } else {
+    if (!fsid.is_zero() && fsid != old_fsid) {
+      derr << "mkfs on-disk fsid " << old_fsid << " != provided " << fsid << dendl;
+      ret = -EINVAL;
+      goto close_fsid_fd;
+    }
+    fsid = old_fsid;
+    dout(1) << "mkfs fsid is already set to " << fsid << dendl;
+  }
+
+  // version stamp
+  ret = write_version_stamp();
+  if (ret < 0) {
+    derr << "mkfs: write_version_stamp() failed: "
+         << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  ret = _create_current();
+  if (ret < 0) {
+    derr << "mkfs: failed to create current/ " << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  if (_detect_backend()) {
+    derr << "KeyValueStore::mkfs error in _detect_backend" << dendl;
+    ret = -1;
+    goto close_fsid_fd;
+  }
+
+  {
+    KeyValueDB *store;
+    if (kv_type == KV_TYPE_LEVELDB) {
+      store = new LevelDBStore(g_ceph_context, current_fn);
+    } else {
+      derr << "KeyValueStore::mkfs error: unknown backend type" << kv_type << dendl;
+      ret = -1;
+      goto close_fsid_fd;
+    }
+
+    store->init();
+    stringstream err;
+    if (store->create_and_open(err)) {
+      derr << "KeyValueStore::mkfs failed to create keyvaluestore backend: "
+           << err.str() << dendl;
+      ret = -1;
+      delete store;
+      goto close_fsid_fd;
+    } else {
+      delete store;
+      dout(1) << "keyvaluestore backend exists/created" << dendl;
+    }
+  }
+
+  dout(1) << "mkfs done in " << basedir << dendl;
+  ret = 0;
+
+ close_fsid_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+  fsid_fd = -1;
+  return ret;
+}
+
+int KeyValueStore::read_fsid(int fd, uuid_d *uuid)
+{
+  char fsid_str[40];
+  int ret = safe_read(fd, fsid_str, sizeof(fsid_str));
+  if (ret < 0)
+    return ret;
+  if (ret == 8) {
+    // old 64-bit fsid... mirror it.
+    *(uint64_t*)&uuid->uuid[0] = *(uint64_t*)fsid_str;
+    *(uint64_t*)&uuid->uuid[8] = *(uint64_t*)fsid_str;
+    return 0;
+  }
+
+  if (ret > 36)
+    fsid_str[36] = 0;
+  if (!uuid->parse(fsid_str))
+    return -EINVAL;
+  return 0;
+}
+
+int KeyValueStore::lock_fsid()
+{
+  struct flock l;
+  memset(&l, 0, sizeof(l));
+  l.l_type = F_WRLCK;
+  l.l_whence = SEEK_SET;
+  l.l_start = 0;
+  l.l_len = 0;
+  int r = ::fcntl(fsid_fd, F_SETLK, &l);
+  if (r < 0) {
+    int err = errno;
+    dout(0) << "lock_fsid failed to lock " << basedir
+            << "/fsid, is another ceph-osd still running? "
+            << cpp_strerror(err) << dendl;
+    return -err;
+  }
+  return 0;
+}
+
+bool KeyValueStore::test_mount_in_use()
+{
+  dout(5) << "test_mount basedir " << basedir << dendl;
+  char fn[PATH_MAX];
+  snprintf(fn, sizeof(fn), "%s/fsid", basedir.c_str());
+
+  // verify fs isn't in use
+
+  fsid_fd = ::open(fn, O_RDWR, 0644);
+  if (fsid_fd < 0)
+    return 0;   // no fsid, ok.
+  bool inuse = lock_fsid() < 0;
+  VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+  fsid_fd = -1;
+  return inuse;
+}
+
+int KeyValueStore::update_version_stamp()
+{
+  return write_version_stamp();
+}
+
+int KeyValueStore::version_stamp_is_valid(uint32_t *version)
+{
+  bufferptr bp(PATH_MAX);
+  int ret = safe_read_file(basedir.c_str(), "store_version",
+      bp.c_str(), bp.length());
+  if (ret < 0) {
+    if (ret == -ENOENT)
+      return 0;
+    return ret;
+  }
+  bufferlist bl;
+  bl.push_back(bp);
+  bufferlist::iterator i = bl.begin();
+  ::decode(*version, i);
+  if (*version == target_version)
+    return 1;
+  else
+    return 0;
+}
+
+int KeyValueStore::write_version_stamp()
+{
+  bufferlist bl;
+  ::encode(target_version, bl);
+
+  return safe_write_file(basedir.c_str(), "store_version",
+      bl.c_str(), bl.length());
+}
+
+int KeyValueStore::mount()
+{
+  int ret;
+  char buf[PATH_MAX];
+
+  dout(5) << "basedir " << basedir << dendl;
+
+  // make sure global base dir exists
+  if (::access(basedir.c_str(), R_OK | W_OK)) {
+    ret = -errno;
+    derr << "KeyValueStore::mount: unable to access basedir '" << basedir
+         << "': " << cpp_strerror(ret) << dendl;
+    goto done;
+  }
+
+  // get fsid
+  snprintf(buf, sizeof(buf), "%s/fsid", basedir.c_str());
+  fsid_fd = ::open(buf, O_RDWR, 0644);
+  if (fsid_fd < 0) {
+    ret = -errno;
+    derr << "KeyValueStore::mount: error opening '" << buf << "': "
+         << cpp_strerror(ret) << dendl;
+    goto done;
+  }
+
+  ret = read_fsid(fsid_fd, &fsid);
+  if (ret < 0) {
+    derr << "KeyValueStore::mount: error reading fsid_fd: "
+         << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  if (lock_fsid() < 0) {
+    derr << "KeyValueStore::mount: lock_fsid failed" << dendl;
+    ret = -EBUSY;
+    goto close_fsid_fd;
+  }
+
+  dout(10) << "mount fsid is " << fsid << dendl;
+
+  uint32_t version_stamp;
+  ret = version_stamp_is_valid(&version_stamp);
+  if (ret < 0) {
+    derr << "KeyValueStore::mount : error in version_stamp_is_valid: "
+         << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  } else if (ret == 0) {
+    if (do_update) {
+      derr << "KeyValueStore::mount : stale version stamp detected: "
+           << version_stamp << ". Proceeding, do_update "
+           << "is set, performing disk format upgrade." << dendl;
+    } else {
+      ret = -EINVAL;
+      derr << "KeyValueStore::mount : stale version stamp " << version_stamp
+           << ". Please run the KeyValueStore update script before starting "
+           << "the OSD, or set keyvaluestore_update_to to " << target_version
+           << dendl;
+      goto close_fsid_fd;
+    }
+  }
+
+  current_fd = ::open(current_fn.c_str(), O_RDONLY);
+  if (current_fd < 0) {
+    ret = -errno;
+    derr << "KeyValueStore::mount: error opening: " << current_fn << ": "
+         << cpp_strerror(ret) << dendl;
+    goto close_fsid_fd;
+  }
+
+  assert(current_fd >= 0);
+
+  if (_detect_backend()) {
+    derr << "KeyValueStore::mount error in _detect_backend" << dendl;
+    ret = -1;
+    goto close_current_fd;
+  }
+
+  {
+    KeyValueDB *store;
+    if (kv_type == KV_TYPE_LEVELDB) {
+      store = new LevelDBStore(g_ceph_context, current_fn);
+    } else {
+      derr << "KeyValueStore::mount error: unknown backend type" << kv_type
+           << dendl;
+      ret = -1;
+      goto close_current_fd;
+    }
+
+    store->init();
+    stringstream err;
+    if (store->open(err)) {
+      derr << "KeyValueStore::mount Error initializing keyvaluestore backend: "
+           << err.str() << dendl;
+      ret = -1;
+      delete store;
+      goto close_current_fd;
+    }
+
+    StripObjectMap *dbomap = new StripObjectMap(store);
+    ret = dbomap->init(do_update);
+    if (ret < 0) {
+      delete dbomap;
+      derr << "Error initializing StripObjectMap: " << ret << dendl;
+      goto close_current_fd;
+    }
+    stringstream err2;
+
+    if (g_conf->keyvaluestore_debug_check_backend && !dbomap->check(err2)) {
+      derr << err2.str() << dendl;;
+      delete dbomap;
+      ret = -EINVAL;
+      goto close_current_fd;
+    }
+
+    backend.reset(dbomap);
+  }
+
+  op_tp.start();
+  op_finisher.start();
+  ondisk_finisher.start();
+
+  // all okay.
+  return 0;
+
+close_current_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(current_fd));
+  current_fd = -1;
+close_fsid_fd:
+  VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+  fsid_fd = -1;
+done:
+  return ret;
+}
+
+int KeyValueStore::umount()
+{
+  dout(5) << "umount " << basedir << dendl;
+
+  op_tp.stop();
+  op_finisher.stop();
+  ondisk_finisher.stop();
+
+  if (fsid_fd >= 0) {
+    VOID_TEMP_FAILURE_RETRY(::close(fsid_fd));
+    fsid_fd = -1;
+  }
+  if (current_fd >= 0) {
+    VOID_TEMP_FAILURE_RETRY(::close(current_fd));
+    current_fd = -1;
+  }
+
+  backend.reset();
+
+  // nothing
+  return 0;
+}
+
+int KeyValueStore::get_max_object_name_length()
+{
+  lock.Lock();
+  int ret = pathconf(basedir.c_str(), _PC_NAME_MAX);
+  if (ret < 0) {
+    int err = errno;
+    lock.Unlock();
+    if (err == 0)
+      return -EDOM;
+    return -err;
+  }
+  lock.Unlock();
+  return ret;
+}
+
+int KeyValueStore::queue_transactions(Sequencer *posr, list &tls,
+                                      TrackedOpRef osd_op,
+                                      ThreadPool::TPHandle *handle)
+{
+  Context *onreadable;
+  Context *ondisk;
+  Context *onreadable_sync;
+  ObjectStore::Transaction::collect_contexts(
+    tls, &onreadable, &ondisk, &onreadable_sync);
+
+  // set up the sequencer
+  OpSequencer *osr;
+  if (!posr)
+    posr = &default_osr;
+  if (posr->p) {
+    osr = static_cast(posr->p);
+    dout(5) << "queue_transactions existing " << *osr << "/" << osr->parent
+            << dendl; //<< " w/ q " << osr->q << dendl;
+  } else {
+    osr = new OpSequencer;
+    osr->parent = posr;
+    posr->p = osr;
+    dout(5) << "queue_transactions new " << *osr << "/" << osr->parent << dendl;
+  }
+
+  Op *o = build_op(tls, ondisk, onreadable, onreadable_sync, osd_op);
+  op_queue_reserve_throttle(o, handle);
+  dout(5) << "queue_transactions (trailing journal) " << " " << tls <& tls,
+        Context *ondisk, Context *onreadable, Context *onreadable_sync,
+        TrackedOpRef osd_op)
+{
+  uint64_t bytes = 0, ops = 0;
+  for (list::iterator p = tls.begin();
+       p != tls.end();
+       ++p) {
+    bytes += (*p)->get_num_bytes();
+    ops += (*p)->get_num_ops();
+  }
+
+  Op *o = new Op;
+  o->start = ceph_clock_now(g_ceph_context);
+  o->tls.swap(tls);
+  o->ondisk = ondisk;
+  o->onreadable = onreadable;
+  o->onreadable_sync = onreadable_sync;
+  o->ops = ops;
+  o->bytes = bytes;
+  o->osd_op = osd_op;
+  return o;
+}
+
+void KeyValueStore::queue_op(OpSequencer *osr, Op *o)
+{
+  // queue op on sequencer, then queue sequencer for the threadpool,
+  // so that regardless of which order the threads pick up the
+  // sequencer, the op order will be preserved.
+
+  osr->queue(o);
+
+  logger->inc(l_os_ops);
+  logger->inc(l_os_bytes, o->bytes);
+
+  dout(5) << "queue_op " << o << " seq " << o->op << " " << *osr << " "
+          << o->bytes << " bytes" << "   (queue has " << op_queue_len
+          << " ops and " << op_queue_bytes << " bytes)" << dendl;
+  op_wq.queue(osr);
+}
+
+void KeyValueStore::op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle)
+{
+  uint64_t max_ops = m_keyvaluestore_queue_max_ops;
+  uint64_t max_bytes = m_keyvaluestore_queue_max_bytes;
+
+  logger->set(l_os_oq_max_ops, max_ops);
+  logger->set(l_os_oq_max_bytes, max_bytes);
+
+  utime_t start = ceph_clock_now(g_ceph_context);
+  {
+    Mutex::Locker l(op_throttle_lock);
+    while ((max_ops && (op_queue_len + 1) > max_ops) ||
+           (max_bytes && op_queue_bytes      // let single large ops through!
+           && (op_queue_bytes + o->bytes) > max_bytes)) {
+      dout(2) << "waiting " << op_queue_len + 1 << " > " << max_ops
+              << " ops || " << op_queue_bytes + o->bytes << " > " << max_bytes
+              << dendl;
+      if (handle)
+        handle->suspend_tp_timeout();
+      op_throttle_cond.Wait(op_throttle_lock);
+      if (handle)
+        handle->reset_tp_timeout();
+    }
+
+    op_queue_len++;
+    op_queue_bytes += o->bytes;
+  }
+  utime_t end = ceph_clock_now(g_ceph_context);
+  logger->tinc(l_os_queue_lat, end - start);
+
+  logger->set(l_os_oq_ops, op_queue_len);
+  logger->set(l_os_oq_bytes, op_queue_bytes);
+}
+
+void KeyValueStore::op_queue_release_throttle(Op *o)
+{
+  {
+    Mutex::Locker l(op_throttle_lock);
+    op_queue_len--;
+    op_queue_bytes -= o->bytes;
+    op_throttle_cond.Signal();
+  }
+
+  logger->set(l_os_oq_ops, op_queue_len);
+  logger->set(l_os_oq_bytes, op_queue_bytes);
+}
+
+void KeyValueStore::_do_op(OpSequencer *osr, ThreadPool::TPHandle &handle)
+{
+  // FIXME: Suppose the collection of transaction only affect objects in the
+  // one PG, so this lock will ensure no other concurrent write operation
+  osr->apply_lock.Lock();
+  Op *o = osr->peek_queue();
+  dout(5) << "_do_op " << o << " seq " << o->op << " " << *osr << "/" << osr->parent << " start" << dendl;
+  int r = _do_transactions(o->tls, o->op, &handle);
+  dout(10) << "_do_op " << o << " seq " << o->op << " r = " << r
+           << ", finisher " << o->onreadable << " " << o->onreadable_sync << dendl;
+
+  if (o->ondisk) {
+    if (r < 0) {
+      delete o->ondisk;
+      o->ondisk = 0;
+    } else {
+      ondisk_finisher.queue(o->ondisk, r);
+    }
+  }
+}
+
+void KeyValueStore::_finish_op(OpSequencer *osr)
+{
+  list to_queue;
+  Op *o = osr->dequeue(&to_queue);
+
+  dout(10) << "_finish_op " << o << " seq " << o->op << " " << *osr << "/" << osr->parent << dendl;
+  osr->apply_lock.Unlock();  // locked in _do_op
+  op_queue_release_throttle(o);
+
+  utime_t lat = ceph_clock_now(g_ceph_context);
+  lat -= o->start;
+  logger->tinc(l_os_apply_lat, lat);
+
+  if (o->onreadable_sync) {
+    o->onreadable_sync->complete(0);
+  }
+  op_finisher.queue(o->onreadable);
+  op_finisher.queue(to_queue);
+  delete o;
+}
+
+// Combine all the ops in the same transaction using "BufferTransaction" and
+// cache the middle results in order to make visible to the following ops.
+//
+// Lock: KeyValueStore use "in_use" in GenericObjectMap to avoid concurrent
+// operation on the same object. Not sure ReadWrite lock should be applied to
+// improve concurrent performance. In the future, I'd like to remove apply_lock
+// on "osr" and introduce PG RWLock.
+int KeyValueStore::_do_transactions(list &tls, uint64_t op_seq,
+  ThreadPool::TPHandle *handle)
+{
+  int r = 0;
+
+  uint64_t bytes = 0, ops = 0;
+  for (list::iterator p = tls.begin();
+       p != tls.end();
+       ++p) {
+    bytes += (*p)->get_num_bytes();
+    ops += (*p)->get_num_ops();
+  }
+
+  int trans_num = 0;
+  BufferTransaction bt(this);
+
+  for (list::iterator p = tls.begin();
+       p != tls.end();
+       ++p, trans_num++) {
+    r = _do_transaction(**p, bt, handle);
+    if (r < 0)
+      break;
+    if (handle)
+      handle->reset_tp_timeout();
+  }
+
+  r = bt.submit_transaction();
+  if (r < 0) {
+    assert(0 == "unexpected error");  // FIXME
+  }
+
+  return r;
+}
+
+unsigned KeyValueStore::_do_transaction(Transaction& transaction,
+                                        BufferTransaction &t,
+                                        ThreadPool::TPHandle *handle)
+{
+  dout(10) << "_do_transaction on " << &transaction << dendl;
+
+  Transaction::iterator i = transaction.begin();
+  uint64_t op_num = 0;
+
+  while (i.have_op()) {
+    if (handle)
+      handle->reset_tp_timeout();
+
+    int op = i.get_op();
+    int r = 0;
+
+    switch (op) {
+    case Transaction::OP_NOP:
+      break;
+
+    case Transaction::OP_TOUCH:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        r = _touch(cid, oid, t);
+      }
+      break;
+
+    case Transaction::OP_WRITE:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        uint64_t off = i.get_length();
+        uint64_t len = i.get_length();
+        bool replica = i.get_replica();
+        bufferlist bl;
+        i.get_bl(bl);
+        r = _write(cid, oid, off, len, bl, t, replica);
+      }
+      break;
+
+    case Transaction::OP_ZERO:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        uint64_t off = i.get_length();
+        uint64_t len = i.get_length();
+        r = _zero(cid, oid, off, len, t);
+      }
+      break;
+
+    case Transaction::OP_TRIMCACHE:
+      {
+        i.get_cid();
+        i.get_oid();
+        i.get_length();
+        i.get_length();
+        // deprecated, no-op
+      }
+      break;
+
+    case Transaction::OP_TRUNCATE:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        uint64_t off = i.get_length();
+        r = _truncate(cid, oid, off, t);
+      }
+      break;
+
+    case Transaction::OP_REMOVE:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        r = _remove(cid, oid, t);
+      }
+      break;
+
+    case Transaction::OP_SETATTR:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        string name = i.get_attrname();
+        bufferlist bl;
+        i.get_bl(bl);
+        map to_set;
+        to_set[name] = bufferptr(bl.c_str(), bl.length());
+        r = _setattrs(cid, oid, to_set, t);
+        if (r == -ENOSPC)
+          dout(0) << " ENOSPC on setxattr on " << cid << "/" << oid
+                  << " name " << name << " size " << bl.length() << dendl;
+      }
+      break;
+
+    case Transaction::OP_SETATTRS:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        map aset;
+        i.get_attrset(aset);
+        r = _setattrs(cid, oid, aset, t);
+        if (r == -ENOSPC)
+          dout(0) << " ENOSPC on setxattrs on " << cid << "/" << oid << dendl;
+      }
+      break;
+
+    case Transaction::OP_RMATTR:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        string name = i.get_attrname();
+        r = _rmattr(cid, oid, name.c_str(), t);
+      }
+      break;
+
+    case Transaction::OP_RMATTRS:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        r = _rmattrs(cid, oid, t);
+      }
+      break;
+
+    case Transaction::OP_CLONE:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        ghobject_t noid = i.get_oid();
+        r = _clone(cid, oid, noid, t);
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        ghobject_t noid = i.get_oid();
+        uint64_t off = i.get_length();
+        uint64_t len = i.get_length();
+        r = _clone_range(cid, oid, noid, off, len, off, t);
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE2:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        ghobject_t noid = i.get_oid();
+        uint64_t srcoff = i.get_length();
+        uint64_t len = i.get_length();
+        uint64_t dstoff = i.get_length();
+        r = _clone_range(cid, oid, noid, srcoff, len, dstoff, t);
+      }
+      break;
+
+    case Transaction::OP_MKCOLL:
+      {
+        coll_t cid = i.get_cid();
+        r = _create_collection(cid, t);
+      }
+      break;
+
+    case Transaction::OP_RMCOLL:
+      {
+        coll_t cid = i.get_cid();
+        r = _destroy_collection(cid, t);
+      }
+      break;
+
+    case Transaction::OP_COLL_ADD:
+      {
+        coll_t ncid = i.get_cid();
+        coll_t ocid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        r = _collection_add(ncid, ocid, oid, t);
+      }
+      break;
+
+    case Transaction::OP_COLL_REMOVE:
+       {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        r = _remove(cid, oid, t);
+       }
+      break;
+
+    case Transaction::OP_COLL_MOVE:
+      {
+        // WARNING: this is deprecated and buggy; only here to replay old journals.
+        coll_t ocid = i.get_cid();
+        coll_t ncid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        r = _collection_move_rename(ocid, oid, ncid, oid, t);
+      }
+      break;
+
+    case Transaction::OP_COLL_MOVE_RENAME:
+      {
+        coll_t oldcid = i.get_cid();
+        ghobject_t oldoid = i.get_oid();
+        coll_t newcid = i.get_cid();
+        ghobject_t newoid = i.get_oid();
+        r = _collection_move_rename(oldcid, oldoid, newcid, newoid, t);
+      }
+      break;
+
+    case Transaction::OP_COLL_SETATTR:
+      {
+        coll_t cid = i.get_cid();
+        string name = i.get_attrname();
+        bufferlist bl;
+        i.get_bl(bl);
+        r = _collection_setattr(cid, name.c_str(), bl.c_str(), bl.length(), t);
+      }
+      break;
+
+    case Transaction::OP_COLL_RMATTR:
+      {
+        coll_t cid = i.get_cid();
+        string name = i.get_attrname();
+        r = _collection_rmattr(cid, name.c_str(), t);
+      }
+      break;
+
+    case Transaction::OP_STARTSYNC:
+      {
+        start_sync();
+        break;
+      }
+
+    case Transaction::OP_COLL_RENAME:
+      {
+        coll_t cid(i.get_cid());
+        coll_t ncid(i.get_cid());
+        r = _collection_rename(cid, ncid, t);
+      }
+      break;
+
+    case Transaction::OP_OMAP_CLEAR:
+      {
+        coll_t cid(i.get_cid());
+        ghobject_t oid = i.get_oid();
+        r = _omap_clear(cid, oid, t);
+      }
+      break;
+    case Transaction::OP_OMAP_SETKEYS:
+      {
+        coll_t cid(i.get_cid());
+        ghobject_t oid = i.get_oid();
+        map aset;
+        i.get_attrset(aset);
+        r = _omap_setkeys(cid, oid, aset, t);
+      }
+      break;
+    case Transaction::OP_OMAP_RMKEYS:
+      {
+        coll_t cid(i.get_cid());
+        ghobject_t oid = i.get_oid();
+        set keys;
+        i.get_keyset(keys);
+        r = _omap_rmkeys(cid, oid, keys, t);
+      }
+      break;
+    case Transaction::OP_OMAP_RMKEYRANGE:
+      {
+        coll_t cid(i.get_cid());
+        ghobject_t oid = i.get_oid();
+        string first, last;
+        first = i.get_key();
+        last = i.get_key();
+        r = _omap_rmkeyrange(cid, oid, first, last, t);
+      }
+      break;
+    case Transaction::OP_OMAP_SETHEADER:
+      {
+        coll_t cid(i.get_cid());
+        ghobject_t oid = i.get_oid();
+        bufferlist bl;
+        i.get_bl(bl);
+        r = _omap_setheader(cid, oid, bl, t);
+      }
+      break;
+    case Transaction::OP_SPLIT_COLLECTION:
+      {
+        coll_t cid(i.get_cid());
+        uint32_t bits(i.get_u32());
+        uint32_t rem(i.get_u32());
+        coll_t dest(i.get_cid());
+        r = _split_collection_create(cid, bits, rem, dest, t);
+      }
+      break;
+    case Transaction::OP_SPLIT_COLLECTION2:
+      {
+        coll_t cid(i.get_cid());
+        uint32_t bits(i.get_u32());
+        uint32_t rem(i.get_u32());
+        coll_t dest(i.get_cid());
+        r = _split_collection(cid, bits, rem, dest, t);
+      }
+      break;
+
+    case Transaction::OP_SETALLOCHINT:
+      {
+        // TODO: can kvstore make use of the hint?
+        coll_t cid(i.get_cid());
+        ghobject_t oid = i.get_oid();
+        (void)i.get_length();  // discard result
+        (void)i.get_length();  // discard result
+      }
+      break;
+
+    default:
+      derr << "bad op " << op << dendl;
+      assert(0);
+    }
+
+    if (r < 0) {
+      bool ok = false;
+
+      if (r == -ENOENT && !(op == Transaction::OP_CLONERANGE ||
+                            op == Transaction::OP_CLONE ||
+                            op == Transaction::OP_CLONERANGE2))
+        // -ENOENT is normally okay
+        // ...including on a replayed OP_RMCOLL with checkpoint mode
+        ok = true;
+      if (r == -ENODATA)
+        ok = true;
+
+      if (!ok) {
+        const char *msg = "unexpected error code";
+
+        if (r == -ENOENT && (op == Transaction::OP_CLONERANGE ||
+                            op == Transaction::OP_CLONE ||
+                            op == Transaction::OP_CLONERANGE2))
+          msg = "ENOENT on clone suggests osd bug";
+
+        if (r == -ENOSPC)
+          // For now, if we hit _any_ ENOSPC, crash, before we do any damage
+          // by partially applying transactions.
+          msg = "ENOSPC handling not implemented";
+
+        if (r == -ENOTEMPTY) {
+          msg = "ENOTEMPTY suggests garbage data in osd data dir";
+        }
+
+        dout(0) << " error " << cpp_strerror(r) << " not handled on operation "
+                << op << " op " << op_num << ", counting from 0)" << dendl;
+        dout(0) << msg << dendl;
+        dout(0) << " transaction dump:\n";
+        JSONFormatter f(true);
+        f.open_object_section("transaction");
+        transaction.dump(&f);
+        f.close_section();
+        f.flush(*_dout);
+        *_dout << dendl;
+        assert(0 == "unexpected error");
+
+        if (r == -EMFILE) {
+          dump_open_fds(g_ceph_context);
+        }
+      }
+    }
+
+    op_num++;
+  }
+
+  return 0;  // FIXME count errors
+}
+
+
+// =========== KeyValueStore Op Implementation ==============
+// objects
+
+bool KeyValueStore::exists(coll_t cid, const ghobject_t& oid)
+{
+  dout(10) << __func__ << "collection: " << cid << " object: " << oid
+           << dendl;
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = backend->lookup_strip_header(cid, oid, &header);
+  if (r < 0) {
+    return false;
+  }
+
+  return true;
+}
+
+int KeyValueStore::stat(coll_t cid, const ghobject_t& oid,
+                        struct stat *st, bool allow_eio)
+{
+  dout(10) << "stat " << cid << "/" << oid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = backend->lookup_strip_header(cid, oid, &header);
+  if (r < 0) {
+    dout(10) << "stat " << cid << "/" << oid << "=" << r << dendl;
+    return -ENOENT;
+  }
+
+  st->st_blocks = header->max_size / header->strip_size;
+  if (header->max_size % header->strip_size)
+    st->st_blocks++;
+  st->st_nlink = 1;
+  st->st_size = header->max_size;
+  st->st_blksize = header->strip_size;
+
+  return r;
+}
+
+int KeyValueStore::_generic_read(StripObjectMap::StripObjectHeaderRef header,
+                                 uint64_t offset, size_t len, bufferlist& bl,
+                                 bool allow_eio, BufferTransaction *bt)
+{
+  if (header->max_size < offset) {
+    dout(10) << __func__ << " " << header->cid << "/" << header->oid << ")"
+             << " offset exceed the length of bl"<< dendl;
+    return 0;
+  }
+
+  if (len == 0)
+    len = header->max_size - offset;
+
+  if (offset + len > header->max_size)
+    len = header->max_size - offset;
+
+  vector extents;
+  StripObjectMap::file_to_extents(offset, len, header->strip_size,
+                                  extents);
+  map out;
+  set keys;
+
+  for (vector::iterator iter = extents.begin();
+       iter != extents.end(); ++iter) {
+    bufferlist old;
+    string key = strip_object_key(iter->no);
+
+    if (bt && header->buffers.count(make_pair(OBJECT_STRIP_PREFIX, key))) {
+      // use strip_header buffer
+      assert(header->bits[iter->no]);
+      out[key] = header->buffers[make_pair(OBJECT_STRIP_PREFIX, key)];
+    } else if (header->bits[iter->no]) {
+      keys.insert(key);
+    }
+  }
+
+  int r = backend->get_values_with_header(header, OBJECT_STRIP_PREFIX, keys, &out);
+  if (r < 0) {
+    dout(10) << __func__ << " " << header->cid << "/" << header->oid << " "
+             << offset << "~" << len << " = " << r << dendl;
+    return r;
+  } else if (out.size() != keys.size()) {
+    dout(0) << __func__ << " broken header or missing data in backend "
+            << header->cid << "/" << header->oid << " " << offset << "~"
+            << len << " = " << r << dendl;
+    return -EBADF;
+  }
+
+  for (vector::iterator iter = extents.begin();
+       iter != extents.end(); ++iter) {
+    string key = strip_object_key(iter->no);
+
+    if (header->bits[iter->no]) {
+      if (iter->len == header->strip_size) {
+        bl.claim_append(out[key]);
+      } else {
+        out[key].copy(iter->offset, iter->len, bl);
+      }
+    } else {
+      bl.append_zero(iter->len);
+    }
+  }
+
+  dout(10) << __func__ << " " << header->cid << "/" << header->oid << " "
+           << offset << "~" << bl.length() << "/" << len << " r = " << r
+           << dendl;
+
+  return bl.length();
+}
+
+
+int KeyValueStore::read(coll_t cid, const ghobject_t& oid, uint64_t offset,
+                        size_t len, bufferlist& bl, bool allow_eio)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+           << len << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = backend->lookup_strip_header(cid, oid, &header);
+
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+              << len << " header isn't exist: r = " << r << dendl;
+    return r;
+  }
+
+  return _generic_read(header, offset, len, bl, allow_eio);
+}
+
+int KeyValueStore::fiemap(coll_t cid, const ghobject_t& oid,
+                          uint64_t offset, size_t len, bufferlist& bl)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " " << offset << "~"
+           << len << dendl;
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = backend->lookup_strip_header(cid, oid, &header);
+  if (r < 0) {
+    dout(10) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len
+             << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  vector extents;
+  StripObjectMap::file_to_extents(offset, len, header->strip_size,
+                                  extents);
+
+  map m;
+  for (vector::iterator iter = extents.begin();
+       iter != extents.end(); ++iter) {
+    uint64_t off = iter->no * header->strip_size + iter->offset;
+    m[off] = iter->len;
+  }
+  ::encode(m, bl);
+  return 0;
+}
+
+int KeyValueStore::_remove(coll_t cid, const ghobject_t& oid,
+                           BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(cid, oid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oid << " "
+             << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  header->max_size = 0;
+  header->bits.clear();
+  r = t.clear_buffer(header);
+
+  dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_truncate(coll_t cid, const ghobject_t& oid, uint64_t size,
+                             BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << " size " << size
+           << dendl;
+
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(cid, oid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oid << " " << size
+             << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  if (header->max_size == size)
+    return 0;
+
+  if (header->max_size > size) {
+    vector extents;
+    StripObjectMap::file_to_extents(size, header->max_size-size,
+                                    header->strip_size, extents);
+    assert(extents.size());
+
+    vector::iterator iter = extents.begin();
+    if (header->bits[iter->no] && iter->offset != 0) {
+      bufferlist value;
+      map values;
+      set lookup_keys;
+      string key = strip_object_key(iter->no);
+
+      lookup_keys.insert(key);
+      r = t.get_buffer_keys(header, OBJECT_STRIP_PREFIX,
+                            lookup_keys, &values);
+      if (r < 0) {
+        dout(10) << __func__ << " " << cid << "/" << oid << " "
+                 << size << " = " << r << dendl;
+        return r;
+      } else if (values.size() != lookup_keys.size()) {
+        dout(0) << __func__ << " broken header or missing data in backend "
+                << header->cid << "/" << header->oid << " size " << size
+                <<  " r = " << r << dendl;
+        return -EBADF;
+      }
+
+      values[key].copy(0, iter->offset, value);
+      value.append_zero(header->strip_size-iter->offset);
+      assert(value.length() == header->strip_size);
+      value.swap(values[key]);
+
+      t.set_buffer_keys(header, OBJECT_STRIP_PREFIX, values);
+      ++iter;
+    }
+
+    set keys;
+    for (; iter != extents.end(); ++iter) {
+      if (header->bits[iter->no]) {
+        keys.insert(strip_object_key(iter->no));
+        header->bits[iter->no] = 0;
+      }
+    }
+    r = t.remove_buffer_keys(header, OBJECT_STRIP_PREFIX, keys);
+    if (r < 0) {
+      dout(10) << __func__ << " " << cid << "/" << oid << " "
+               << size << " = " << r << dendl;
+      return r;
+    }
+  }
+
+  header->bits.resize(size/header->strip_size+1);
+  header->max_size = size;
+
+  dout(10) << __func__ << " " << cid << "/" << oid << " size " << size << " = "
+           << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_touch(coll_t cid, const ghobject_t& oid,
+                          BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(cid, oid, &header, true);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oid << " "
+             << " failed to get header: r = " << r << dendl;
+    r = -EINVAL;
+    return r;
+  }
+
+  dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_generic_write(StripObjectMap::StripObjectHeaderRef header,
+                                  uint64_t offset, size_t len,
+                                  const bufferlist& bl, BufferTransaction &t,
+                                  bool replica)
+{
+  if (len > bl.length())
+    len = bl.length();
+
+  if (len + offset > header->max_size) {
+    header->max_size = len + offset;
+    header->bits.resize(header->max_size/header->strip_size+1);
+  }
+
+  vector extents;
+  StripObjectMap::file_to_extents(offset, len, header->strip_size,
+                                  extents);
+
+  map out;
+  set keys;
+  for (vector::iterator iter = extents.begin();
+       iter != extents.end(); ++iter) {
+    if (header->bits[iter->no] && !(iter->offset == 0 &&
+                                   iter->len == header->strip_size))
+      keys.insert(strip_object_key(iter->no));
+  }
+
+  int r = t.get_buffer_keys(header, OBJECT_STRIP_PREFIX, keys, &out);
+  if (r < 0) {
+    dout(10) << __func__ << " failed to get value " << header->cid << "/"
+              << header->oid << " " << offset << "~" << len << " = " << r
+              << dendl;
+    return r;
+  } else if (keys.size() != out.size()) {
+    // Error on header.bits or the corresponding key/value pair is missing
+    dout(0) << __func__ << " broken header or missing data in backend "
+            << header->cid << "/" << header->oid << " " << offset << "~"
+            << len << " = " << r << dendl;
+    return -EBADF;
+  }
+
+  uint64_t bl_offset = 0;
+  map values;
+  for (vector::iterator iter = extents.begin();
+       iter != extents.end(); ++iter) {
+    bufferlist value;
+    string key = strip_object_key(iter->no);
+    if (header->bits[iter->no]) {
+      if (iter->offset == 0 && iter->len == header->strip_size) {
+        bl.copy(bl_offset, iter->len, value);
+        bl_offset += iter->len;
+      } else {
+        assert(out[key].length() == header->strip_size);
+
+        out[key].copy(0, iter->offset, value);
+        bl.copy(bl_offset, iter->len, value);
+        bl_offset += iter->len;
+
+        if (value.length() != header->strip_size)
+          out[key].copy(value.length(), header->strip_size-value.length(),
+                        value);
+      }
+    } else {
+      if (iter->offset)
+        value.append_zero(iter->offset);
+      bl.copy(bl_offset, iter->len, value);
+      bl_offset += iter->len;
+
+      if (value.length() < header->strip_size)
+        value.append_zero(header->strip_size-value.length());
+
+      header->bits[iter->no] = 1;
+    }
+    assert(value.length() == header->strip_size);
+    values[key].swap(value);
+  }
+  assert(bl_offset == len);
+
+  t.set_buffer_keys(header, OBJECT_STRIP_PREFIX, values);
+  dout(10) << __func__ << " " << header->cid << "/" << header->oid << " "
+           << offset << "~" << len << " = " << r << dendl;
+
+  return r;
+}
+
+int KeyValueStore::_write(coll_t cid, const ghobject_t& oid,
+                          uint64_t offset, size_t len, const bufferlist& bl,
+                          BufferTransaction &t, bool replica)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+           << len << dendl;
+
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(cid, oid, &header, true);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oid << " " << offset
+             << "~" << len << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  return _generic_write(header, offset, len, bl, t, replica);
+}
+
+int KeyValueStore::_zero(coll_t cid, const ghobject_t& oid, uint64_t offset,
+                         size_t len, BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << " " << offset << "~" << len << dendl;
+
+  bufferptr bp(len);
+  bp.zero();
+  bufferlist bl;
+  bl.push_back(bp);
+  int r = _write(cid, oid, offset, len, bl, t);
+
+  dout(10) << __func__ << " " << cid << "/" << oid << " " << offset << "~"
+           << len << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_clone(coll_t cid, const ghobject_t& oldoid,
+                          const ghobject_t& newoid, BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+           << newoid << dendl;
+
+  if (oldoid == newoid)
+    return 0;
+
+  int r;
+  StripObjectMap::StripObjectHeaderRef old_header;
+
+  r = t.lookup_cached_header(cid, oldoid, &old_header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+             << newoid << " = " << r << dendl;
+    return r;
+  }
+
+  t.clone_buffer(old_header, cid, newoid);
+
+  dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+           << newoid << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_clone_range(coll_t cid, const ghobject_t& oldoid,
+                                const ghobject_t& newoid, uint64_t srcoff,
+                                uint64_t len, uint64_t dstoff,
+                                BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+           << newoid << " " << srcoff << "~" << len << " to " << dstoff
+           << dendl;
+
+  int r;
+  bufferlist bl;
+
+  StripObjectMap::StripObjectHeaderRef old_header, new_header;
+
+  r = t.lookup_cached_header(cid, oldoid, &old_header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+           << newoid << " " << srcoff << "~" << len << " to " << dstoff
+           << " header isn't exist: r = " << r << dendl;
+    return r;
+  }
+
+  r = t.lookup_cached_header(cid, newoid, &new_header, true);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+           << newoid << " " << srcoff << "~" << len << " to " << dstoff
+           << " can't create header: r = " << r << dendl;
+    return r;
+  }
+
+  r = _generic_read(old_header, srcoff, len, bl, &t);
+  if (r < 0)
+    goto out;
+
+  r = _generic_write(new_header, dstoff, len, bl, t);
+
+ out:
+  dout(10) << __func__ << " " << cid << "/" << oldoid << " -> " << cid << "/"
+           << newoid << " " << srcoff << "~" << len << " to " << dstoff
+           << " = " << r << dendl;
+  return r;
+}
+
+// attrs
+
+int KeyValueStore::getattr(coll_t cid, const ghobject_t& oid, const char *name,
+                           bufferptr &bp)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << " '" << name << "'"
+           << dendl;
+
+  int r;
+  map got;
+  set to_get;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  to_get.insert(string(name));
+
+  r = backend->lookup_strip_header(cid, oid, &header);
+  if (r < 0) {
+    dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+    return r;
+  }
+
+  r = backend->get_values_with_header(header, OBJECT_XATTR, to_get, &got);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " get_xattrs err r =" << r << dendl;
+    goto out;
+  }
+  if (got.empty()) {
+    dout(10) << __func__ << " got.size() is 0" << dendl;
+    return -ENODATA;
+  }
+  bp = bufferptr(got.begin()->second.c_str(),
+                 got.begin()->second.length());
+  r = 0;
+
+ out:
+  dout(10) << __func__ << " " << cid << "/" << oid << " '" << name << "' = "
+           << r << dendl;
+  return r;
+}
+
+int KeyValueStore::getattrs(coll_t cid, const ghobject_t& oid,
+                           map& aset, bool user_only)
+{
+  int r;
+  map attr_aset;
+
+  r = backend->get(cid, oid, OBJECT_XATTR, &attr_aset);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " could not get attrs r = " << r << dendl;
+    goto out;
+  }
+
+  if (r == -ENOENT)
+    r = 0;
+
+  for (map::iterator i = attr_aset.begin();
+       i != attr_aset.end(); ++i) {
+    string key;
+    if (user_only) {
+      if (i->first[0] != '_')
+        continue;
+      if (i->first == "_")
+        continue;
+      key = i->first.substr(1, i->first.size());
+    } else {
+      key = i->first;
+    }
+    aset.insert(make_pair(key,
+                bufferptr(i->second.c_str(), i->second.length())));
+  }
+
+ out:
+  dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+
+  return r;
+}
+
+int KeyValueStore::_setattrs(coll_t cid, const ghobject_t& oid,
+                             map& aset,
+                             BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+  int r;
+
+  StripObjectMap::StripObjectHeaderRef header;
+  map attrs;
+
+  r = t.lookup_cached_header(cid, oid, &header, false);
+  if (r < 0)
+    goto out;
+
+  for (map::iterator it = aset.begin();
+       it != aset.end(); ++it) {
+    attrs[it->first].push_back(it->second);
+  }
+
+  t.set_buffer_keys(header, OBJECT_XATTR, attrs);
+
+out:
+  dout(10) << __func__ << " " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+
+int KeyValueStore::_rmattr(coll_t cid, const ghobject_t& oid, const char *name,
+                           BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << " '" << name << "'"
+           << dendl;
+
+  int r;
+  set to_remove;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(cid, oid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " could not find header r = " << r
+             << dendl;
+    return r;
+  }
+
+  to_remove.insert(string(name));
+  r = t.remove_buffer_keys(header, OBJECT_XATTR, to_remove);
+
+  dout(10) << __func__ << " " << cid << "/" << oid << " '" << name << "' = "
+           << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_rmattrs(coll_t cid, const ghobject_t& oid,
+                            BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << oid << dendl;
+
+  int r;
+  set attrs;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(cid, oid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " could not find header r = " << r
+             << dendl;
+    return r;
+  }
+
+  r = backend->get_keys_with_header(header, OBJECT_XATTR, &attrs);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " could not get attrs r = " << r << dendl;
+    return r;
+  }
+
+  r = t.remove_buffer_keys(header, OBJECT_XATTR, attrs);
+  t.clear_buffer_keys(header, OBJECT_XATTR);
+
+  dout(10) << __func__ <<  " " << cid << "/" << oid << " = " << r << dendl;
+  return r;
+}
+
+// collection attrs
+
+int KeyValueStore::collection_getattr(coll_t c, const char *name,
+                                      void *value, size_t size)
+{
+  dout(15) << __func__ << " " << c.to_str() << " '" << name << "' len "
+           << size << dendl;
+
+  bufferlist bl;
+  int r;
+
+  r = collection_getattr(c, name, bl);
+  if (r < 0)
+      goto out;
+
+  if (bl.length() < size) {
+    r = bl.length();
+    bl.copy(0, bl.length(), static_cast(value));
+  } else {
+    r = size;
+    bl.copy(0, size, static_cast(value));
+  }
+
+out:
+  dout(10) << __func__ << " " << c.to_str() << " '" << name << "' len "
+           << size << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::collection_getattr(coll_t c, const char *name,
+                                      bufferlist& bl)
+{
+  dout(15) << __func__ << " " << c.to_str() << " '" << name
+           << "'" << dendl;
+
+  set keys;
+  map out;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  keys.insert(string(name));
+
+  int r = backend->lookup_strip_header(get_coll_for_coll(),
+                                       make_ghobject_for_coll(c), &header);
+  if (r < 0) {
+    dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+    return r;
+  }
+
+  r = backend->get_values_with_header(header, COLLECTION_ATTR, keys, &out);
+  if (r < 0) {
+    dout(10) << __func__ << " could not get key" << string(name) << dendl;
+    r = -EINVAL;
+  }
+
+  assert(out.size());
+  bl.swap(out.begin()->second);
+
+  dout(10) << __func__ << " " << c.to_str() << " '" << name << "' len "
+           << bl.length() << " = " << r << dendl;
+  return bl.length();
+}
+
+int KeyValueStore::collection_getattrs(coll_t cid,
+                                       map &aset)
+{
+  dout(10) << __func__ << " " << cid.to_str() << dendl;
+
+  map out;
+  set keys;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  for (map::iterator it = aset.begin();
+       it != aset.end(); ++it) {
+      keys.insert(it->first);
+  }
+
+  int r = backend->lookup_strip_header(get_coll_for_coll(),
+                                       make_ghobject_for_coll(cid), &header);
+  if (r < 0) {
+    dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+    return r;
+  }
+
+  r = backend->get_values_with_header(header, COLLECTION_ATTR, keys, &out);
+  if (r < 0) {
+    dout(10) << __func__ << " could not get keys" << dendl;
+    r = -EINVAL;
+    goto out;
+  }
+
+  for (map::iterator it = out.begin(); it != out.end();
+       ++it) {
+    bufferptr ptr(it->second.c_str(), it->second.length());
+    aset.insert(make_pair(it->first, ptr));
+  }
+
+ out:
+  dout(10) << __func__ << " " << cid.to_str() << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_collection_setattr(coll_t c, const char *name,
+                                       const void *value, size_t size,
+                                       BufferTransaction &t)
+{
+  dout(10) << __func__ << " " << c << " '" << name << "' len "
+           << size << dendl;
+
+  int r;
+  bufferlist bl;
+  map out;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(get_coll_for_coll(),
+                             make_ghobject_for_coll(c),
+                             &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " could not find header r = " << r << dendl;
+    return r;
+  }
+
+  bl.append(reinterpret_cast(value), size);
+  out.insert(make_pair(string(name), bl));
+
+  t.set_buffer_keys(header, COLLECTION_ATTR, out);
+
+  dout(10) << __func__ << " " << c << " '"
+           << name << "' len " << size << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_collection_rmattr(coll_t c, const char *name,
+                                      BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << c << dendl;
+
+  bufferlist bl;
+  set out;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = t.lookup_cached_header(get_coll_for_coll(),
+                                 make_ghobject_for_coll(c), &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " could not find header r = " << r << dendl;
+    return r;
+  }
+
+  out.insert(string(name));
+  r = t.remove_buffer_keys(header, COLLECTION_ATTR, out);
+
+  dout(10) << __func__ << " " << c << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_collection_setattrs(coll_t cid,
+                                        map& aset,
+                                        BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << dendl;
+
+  map attrs;
+  StripObjectMap::StripObjectHeaderRef header;
+  int r = t.lookup_cached_header(get_coll_for_coll(),
+                                 make_ghobject_for_coll(cid),
+                                 &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " could not find header r = " << r << dendl;
+    return r;
+  }
+
+  for (map::iterator it = aset.begin(); it != aset.end();
+       ++it) {
+    attrs[it->first].push_back(it->second);
+  }
+
+  t.set_buffer_keys(header, COLLECTION_ATTR, attrs);
+
+  dout(10) << __func__ << " " << cid << " = " << r << dendl;
+  return r;
+}
+
+
+// collections
+
+int KeyValueStore::_create_collection(coll_t c, BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << c << dendl;
+
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+  bufferlist bl;
+
+  r = t.lookup_cached_header(get_coll_for_coll(),
+                             make_ghobject_for_coll(c), &header,
+                             false);
+  if (r == 0) {
+    r = -EEXIST;
+    return r;
+  }
+
+  r = t.lookup_cached_header(get_coll_for_coll(),
+                             make_ghobject_for_coll(c), &header,
+                             true);
+
+  dout(10) << __func__ << " cid " << c << " r = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_destroy_collection(coll_t c, BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << c << dendl;
+
+  int r;
+  uint64_t modified_object = 0;
+  StripObjectMap::StripObjectHeaderRef header;
+  vector oids;
+
+  r = t.lookup_cached_header(get_coll_for_coll(), make_ghobject_for_coll(c),
+                             &header, false);
+  if (r < 0) {
+    goto out;
+  }
+
+  // All modified objects are marked deleted
+  for (BufferTransaction::StripHeaderMap::iterator iter = t.strip_headers.begin();
+       iter != t.strip_headers.end(); ++iter) {
+    // sum the total modified object in this PG
+    if (iter->first.first != c)
+      continue;
+
+    modified_object++;
+    if (!iter->second->deleted) {
+      r = -ENOTEMPTY;
+      goto out;
+    }
+  }
+
+  r = backend->list_objects(c, ghobject_t(), modified_object+1, &oids,
+                            0);
+  // No other object
+  if (oids.size() != modified_object && oids.size() != 0) {
+    r = -ENOTEMPTY;
+    goto out;
+  }
+
+  for(vector::iterator iter = oids.begin();
+      iter != oids.end(); ++iter) {
+    if (!t.strip_headers.count(make_pair(c, *iter))) {
+      r = -ENOTEMPTY;
+      goto out;
+    }
+  }
+
+  r = t.clear_buffer(header);
+
+out:
+  dout(10) << __func__ << " " << c << " = " << r << dendl;
+  return r;
+}
+
+
+int KeyValueStore::_collection_add(coll_t c, coll_t oldcid,
+                                   const ghobject_t& o,
+                                   BufferTransaction &t)
+{
+  dout(15) << __func__ <<  " " << c << "/" << o << " from " << oldcid << "/"
+           << o << dendl;
+
+  bufferlist bl;
+  StripObjectMap::StripObjectHeaderRef header, old_header;
+
+  int r = t.lookup_cached_header(oldcid, o, &old_header, false);
+  if (r < 0) {
+    goto out;
+  }
+
+  r = t.lookup_cached_header(c, o, &header, false);
+  if (r == 0) {
+    r = -EEXIST;
+    dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+             << o << " already exist " << dendl;
+    goto out;
+  }
+
+  r = _generic_read(old_header, 0, old_header->max_size, bl, &t);
+  if (r < 0) {
+    r = -EINVAL;
+    goto out;
+  }
+
+  r = _generic_write(header, 0, bl.length(), bl, t);
+  if (r < 0) {
+    r = -EINVAL;
+  }
+
+out:
+  dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+           << o << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_collection_move_rename(coll_t oldcid,
+                                           const ghobject_t& oldoid,
+                                           coll_t c, const ghobject_t& o,
+                                           BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+           << oldoid << dendl;
+  int r;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  r = t.lookup_cached_header(c, o, &header, false);
+  if (r == 0) {
+    dout(10) << __func__ << " " << oldcid << "/" << oldoid << " -> " << c
+             << "/" << o << " = " << r << dendl;
+    return -EEXIST;
+  }
+
+  r = t.lookup_cached_header(oldcid, oldoid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << oldcid << "/" << oldoid << " -> " << c
+             << "/" << o << " = " << r << dendl;
+    return r;
+  }
+
+  t.rename_buffer(header, c, o);
+
+  dout(10) << __func__ << " " << c << "/" << o << " from " << oldcid << "/"
+           << oldoid << " = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_collection_remove_recursive(const coll_t &cid,
+                                                BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = t.lookup_cached_header(get_coll_for_coll(),
+                                 make_ghobject_for_coll(cid),
+                                 &header, false);
+  if (r < 0) {
+    return 0;
+  }
+
+  vector objects;
+  ghobject_t max;
+  while (!max.is_max()) {
+    r = collection_list_partial(cid, max, 200, 300, 0, &objects, &max);
+    if (r < 0)
+      return r;
+
+    for (vector::iterator i = objects.begin();
+         i != objects.end(); ++i) {
+      r = _remove(cid, *i, t);
+
+      if (r < 0)
+        return r;
+    }
+  }
+
+  r = t.clear_buffer(header);
+
+  dout(10) << __func__ << " " << cid  << " r = " << r << dendl;
+  return 0;
+}
+
+int KeyValueStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
+                                      BufferTransaction &t)
+{
+  dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
+           << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = t.lookup_cached_header(get_coll_for_coll(),
+                                 make_ghobject_for_coll(ncid),
+                                 &header, false);
+  if (r == 0) {
+    dout(2) << __func__ << ": " << ncid << " DNE" << dendl;
+    return -EEXIST;
+  }
+
+  r = t.lookup_cached_header(get_coll_for_coll(), make_ghobject_for_coll(cid),
+                             &header, false);
+  if (r < 0) {
+    dout(2) << __func__ << ": " << cid << " DNE" << dendl;
+    return 0;
+  }
+
+  vector objects;
+  ghobject_t next, current;
+  int move_size = 0;
+  while (1) {
+    collection_list_partial(cid, current, get_ideal_list_min(),
+                            get_ideal_list_max(), 0, &objects, &next);
+
+    dout(20) << __func__ << cid << "objects size: " << objects.size()
+             << dendl;
+
+    if (objects.empty())
+      break;
+
+    for (vector::iterator i = objects.begin();
+        i != objects.end(); ++i) {
+      if (_collection_move_rename(cid, *i, ncid, *i, t) < 0) {
+        return -1;
+      }
+      move_size++;
+    }
+
+    objects.clear();
+    current = next;
+  }
+
+  t.rename_buffer(header, get_coll_for_coll(), make_ghobject_for_coll(ncid));
+
+  dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
+           << dendl;
+  return 0;
+}
+
+int KeyValueStore::list_collections(vector& ls)
+{
+  dout(10) << __func__ << " " << dendl;
+
+  vector oids;
+  ghobject_t next;
+  backend->list_objects(get_coll_for_coll(), ghobject_t(), 0, &oids, &next);
+  assert(next == ghobject_t::get_max());
+
+  for (vector::const_iterator iter = oids.begin();
+       iter != oids.end(); ++iter) {
+    ls.push_back(coll_t(iter->hobj.oid.name));
+  }
+
+  return 0;
+}
+
+bool KeyValueStore::collection_exists(coll_t c)
+{
+  dout(10) << __func__ << " " << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+  int r = backend->lookup_strip_header(get_coll_for_coll(),
+                                       make_ghobject_for_coll(c), &header);
+  if (r < 0) {
+    return false;
+  }
+  return true;
+}
+
+bool KeyValueStore::collection_empty(coll_t c)
+{
+  dout(10) << __func__ << " " << dendl;
+
+  vector oids;
+  backend->list_objects(c, ghobject_t(), 1, &oids, 0);
+
+  return oids.empty();
+}
+
+int KeyValueStore::collection_list_range(coll_t c, ghobject_t start,
+                                         ghobject_t end, snapid_t seq,
+                                         vector *ls)
+{
+  bool done = false;
+  ghobject_t next = start;
+
+  while (!done) {
+    vector next_objects;
+    int r = collection_list_partial(c, next, get_ideal_list_min(),
+                                    get_ideal_list_max(), seq,
+                                    &next_objects, &next);
+    if (r < 0)
+      return r;
+
+    ls->insert(ls->end(), next_objects.begin(), next_objects.end());
+
+    // special case for empty collection
+    if (ls->empty()) {
+      break;
+    }
+
+    while (!ls->empty() && ls->back() >= end) {
+      ls->pop_back();
+      done = true;
+    }
+
+    if (next >= end) {
+      done = true;
+    }
+  }
+
+  return 0;
+}
+
+int KeyValueStore::collection_list_partial(coll_t c, ghobject_t start,
+                                           int min, int max, snapid_t seq,
+                                           vector *ls,
+                                           ghobject_t *next)
+{
+  dout(10) << __func__ << " " << c << " start:" << start << " is_max:"
+           << start.is_max() << dendl;
+
+  if (min < 0 || max < 0)
+      return -EINVAL;
+
+  if (start.is_max())
+      return 0;
+
+  return backend->list_objects(c, start, max, ls, next);
+}
+
+int KeyValueStore::collection_list(coll_t c, vector& ls)
+{
+  return collection_list_partial(c, ghobject_t(), 0, 0, 0, &ls, 0);
+}
+
+int KeyValueStore::collection_version_current(coll_t c, uint32_t *version)
+{
+  *version = COLLECTION_VERSION;
+  if (*version == target_version)
+    return 1;
+  else
+    return 0;
+}
+
+// omap
+
+int KeyValueStore::omap_get(coll_t c, const ghobject_t &hoid,
+                            bufferlist *bl, map *out)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = backend->lookup_strip_header(c, hoid, &header);
+  if (r < 0) {
+    dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+    return r;
+  }
+
+  r = backend->get_with_header(header, OBJECT_OMAP, out);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " err r =" << r << dendl;
+    return r;
+  }
+
+  set keys;
+  map got;
+
+  keys.insert(OBJECT_OMAP_HEADER_KEY);
+  r = backend->get_values_with_header(header, OBJECT_OMAP_HEADER, keys, &got);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " err r =" << r << dendl;
+    return r;
+  }
+
+  if (!got.empty()) {
+    assert(got.size() == 1);
+    bl->swap(got.begin()->second);
+  }
+
+  return 0;
+}
+
+int KeyValueStore::omap_get_header(coll_t c, const ghobject_t &hoid,
+                                   bufferlist *bl, bool allow_eio)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+  set keys;
+  map got;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = backend->lookup_strip_header(c, hoid, &header);
+  if (r < 0) {
+    dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+    return r;
+  }
+
+  keys.insert(OBJECT_OMAP_HEADER_KEY);
+  r = backend->get_values_with_header(header, OBJECT_OMAP_HEADER, keys, &got);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " err r =" << r << dendl;
+    return r;
+  }
+
+  if (!got.empty()) {
+    assert(got.size() == 1);
+    bl->swap(got.begin()->second);
+  }
+
+  return 0;
+}
+
+int KeyValueStore::omap_get_keys(coll_t c, const ghobject_t &hoid, set *keys)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+  int r = backend->lookup_strip_header(c, hoid, &header);
+  if (r < 0) {
+    dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+    return r;
+  }
+
+  r = backend->get_keys_with_header(header, OBJECT_OMAP, keys);
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  }
+  return 0;
+}
+
+int KeyValueStore::omap_get_values(coll_t c, const ghobject_t &hoid,
+                                   const set &keys,
+                                   map *out)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+  int r = backend->lookup_strip_header(c, hoid, &header);
+  if (r < 0) {
+    dout(10) << __func__ << " lookup_strip_header failed: r =" << r << dendl;
+    return r;
+  }
+
+  r = backend->get_values_with_header(header, OBJECT_OMAP, keys, out);
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  }
+  return 0;
+}
+
+int KeyValueStore::omap_check_keys(coll_t c, const ghobject_t &hoid,
+                                   const set &keys, set *out)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+
+  int r = backend->check_keys(c, hoid, OBJECT_OMAP, keys, out);
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  }
+  return 0;
+}
+
+ObjectMap::ObjectMapIterator KeyValueStore::get_omap_iterator(
+    coll_t c, const ghobject_t &hoid)
+{
+  dout(15) << __func__ << " " << c << "/" << hoid << dendl;
+  return backend->get_iterator(c, hoid, OBJECT_OMAP);
+}
+
+int KeyValueStore::_omap_clear(coll_t cid, const ghobject_t &hoid,
+                               BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = t.lookup_cached_header(cid, hoid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << hoid << " "
+             << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  set keys;
+  r = backend->get_keys_with_header(header, OBJECT_OMAP, &keys);
+  if (r < 0 && r != -ENOENT) {
+    dout(10) << __func__ << " could not get omap_keys r = " << r << dendl;
+    return r;
+  }
+
+  r = t.remove_buffer_keys(header, OBJECT_OMAP, keys);
+  if (r < 0) {
+    dout(10) << __func__ << " could not remove keys r = " << r << dendl;
+    return r;
+  }
+
+  keys.clear();
+  keys.insert(OBJECT_OMAP_HEADER_KEY);
+  r = t.remove_buffer_keys(header, OBJECT_OMAP_HEADER, keys);
+  if (r < 0) {
+    dout(10) << __func__ << " could not remove keys r = " << r << dendl;
+    return r;
+  }
+
+  t.clear_buffer_keys(header, OBJECT_OMAP_HEADER);
+
+  dout(10) << __func__ << " " << cid << "/" << hoid << " r = " << r << dendl;
+  return 0;
+}
+
+int KeyValueStore::_omap_setkeys(coll_t cid, const ghobject_t &hoid,
+                                 map &aset,
+                                 BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = t.lookup_cached_header(cid, hoid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << hoid << " "
+             << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  t.set_buffer_keys(header, OBJECT_OMAP, aset);
+
+  return 0;
+}
+
+int KeyValueStore::_omap_rmkeys(coll_t cid, const ghobject_t &hoid,
+                                const set &keys,
+                                BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = t.lookup_cached_header(cid, hoid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << hoid << " "
+             << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  r = t.remove_buffer_keys(header, OBJECT_OMAP, keys);
+
+  dout(10) << __func__ << " " << cid << "/" << hoid << " r = " << r << dendl;
+  return r;
+}
+
+int KeyValueStore::_omap_rmkeyrange(coll_t cid, const ghobject_t &hoid,
+                                    const string& first, const string& last,
+                                    BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << hoid << " [" << first << ","
+           << last << "]" << dendl;
+
+  set keys;
+  {
+    ObjectMap::ObjectMapIterator iter = get_omap_iterator(cid, hoid);
+    if (!iter)
+      return -ENOENT;
+
+    for (iter->lower_bound(first); iter->valid() && iter->key() < last;
+         iter->next()) {
+      keys.insert(iter->key());
+    }
+  }
+  return _omap_rmkeys(cid, hoid, keys, t);
+}
+
+int KeyValueStore::_omap_setheader(coll_t cid, const ghobject_t &hoid,
+                                   const bufferlist &bl,
+                                   BufferTransaction &t)
+{
+  dout(15) << __func__ << " " << cid << "/" << hoid << dendl;
+
+  map sets;
+  StripObjectMap::StripObjectHeaderRef header;
+
+  int r = t.lookup_cached_header(cid, hoid, &header, false);
+  if (r < 0) {
+    dout(10) << __func__ << " " << cid << "/" << hoid << " "
+             << " failed to get header: r = " << r << dendl;
+    return r;
+  }
+
+  sets[OBJECT_OMAP_HEADER_KEY] = bl;
+  t.set_buffer_keys(header, OBJECT_OMAP_HEADER, sets);
+  return 0;
+}
+
+int KeyValueStore::_split_collection(coll_t cid, uint32_t bits, uint32_t rem,
+                                     coll_t dest, BufferTransaction &t)
+{
+  {
+    dout(15) << __func__ << " " << cid << " bits: " << bits << dendl;
+
+    StripObjectMap::StripObjectHeaderRef header;
+
+    int r = t.lookup_cached_header(get_coll_for_coll(),
+                                   make_ghobject_for_coll(cid),
+                                   &header, false);
+    if (r < 0) {
+      dout(2) << __func__ << ": " << cid << " DNE" << dendl;
+      return 0;
+    }
+
+    r = t.lookup_cached_header(get_coll_for_coll(),
+                               make_ghobject_for_coll(dest),
+                               &header, false);
+    if (r < 0) {
+      dout(2) << __func__ << ": " << dest << " DNE" << dendl;
+      return 0;
+    }
+
+    vector objects;
+    ghobject_t next, current;
+    int move_size = 0;
+    while (1) {
+      collection_list_partial(cid, current, get_ideal_list_min(),
+                              get_ideal_list_max(), 0, &objects, &next);
+
+      dout(20) << __func__ << cid << "objects size: " << objects.size()
+              << dendl;
+
+      if (objects.empty())
+        break;
+
+      for (vector::iterator i = objects.begin();
+          i != objects.end(); ++i) {
+        if (i->match(bits, rem)) {
+          if (_collection_move_rename(cid, *i, dest, *i, t) < 0) {
+            return -1;
+          }
+          move_size++;
+        }
+      }
+
+      objects.clear();
+      current = next;
+    }
+
+    dout(20) << __func__ << "move" << move_size << " object from " << cid
+             << "to " << dest << dendl;
+  }
+
+  if (g_conf->filestore_debug_verify_split) {
+    vector objects;
+    ghobject_t next;
+    while (1) {
+      collection_list_partial(cid, next, get_ideal_list_min(),
+                              get_ideal_list_max(), 0, &objects, &next);
+      if (objects.empty())
+        break;
+
+      for (vector::iterator i = objects.begin();
+           i != objects.end(); ++i) {
+        dout(20) << __func__ << ": " << *i << " still in source "
+                 << cid << dendl;
+        assert(!i->match(bits, rem));
+      }
+      objects.clear();
+    }
+
+    next = ghobject_t();
+    while (1) {
+      collection_list_partial(dest, next, get_ideal_list_min(),
+                              get_ideal_list_max(), 0, &objects, &next);
+      if (objects.empty())
+        break;
+
+      for (vector::iterator i = objects.begin();
+           i != objects.end(); ++i) {
+        dout(20) << __func__ << ": " << *i << " now in dest "
+                 << *i << dendl;
+        assert(i->match(bits, rem));
+      }
+      objects.clear();
+    }
+  }
+  return 0;
+}
+
+const char** KeyValueStore::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "keyvaluestore_queue_max_ops",
+    "keyvaluestore_queue_max_bytes",
+    NULL
+  };
+  return KEYS;
+}
+
+void KeyValueStore::handle_conf_change(const struct md_config_t *conf,
+                                       const std::set  &changed)
+{
+  if (changed.count("keyvaluestore_queue_max_ops") ||
+      changed.count("keyvaluestore_queue_max_bytes")) {
+    m_keyvaluestore_queue_max_ops = conf->keyvaluestore_queue_max_ops;
+    m_keyvaluestore_queue_max_bytes = conf->keyvaluestore_queue_max_bytes;
+  }
+}
+
+void KeyValueStore::dump_transactions(list& ls, uint64_t seq, OpSequencer *osr)
+{
+}
diff --git a/ceph/src/os/KeyValueStore.h b/ceph/src/os/KeyValueStore.h
new file mode 100644
index 00000000..bc36103a
--- /dev/null
+++ b/ceph/src/os/KeyValueStore.h
@@ -0,0 +1,647 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 UnitedStack 
+ *
+ * Author: Haomai Wang 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+
+#ifndef CEPH_KEYVALUESTORE_H
+#define CEPH_KEYVALUESTORE_H
+
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+#include 
+using namespace std;
+
+#include "include/assert.h"
+
+#include "ObjectStore.h"
+
+#include "common/WorkQueue.h"
+#include "common/Finisher.h"
+#include "common/fd.h"
+
+#include "common/Mutex.h"
+#include "GenericObjectMap.h"
+#include "KeyValueDB.h"
+#include "common/random_cache.hpp"
+
+#include "include/uuid.h"
+
+enum kvstore_types {
+    KV_TYPE_NONE = 0,
+    KV_TYPE_LEVELDB,
+    KV_TYPE_OTHER
+};
+
+
+static uint64_t default_strip_size = 1024;
+
+class StripObjectMap: public GenericObjectMap {
+ public:
+
+  struct StripExtent {
+    uint64_t no;
+    uint64_t offset;    // in key
+    uint64_t len;    // in key
+    StripExtent(uint64_t n, uint64_t off, size_t len):
+      no(n), offset(off), len(len) {}
+  };
+
+  // -- strip object --
+  struct StripObjectHeader {
+    // Persistent state
+    uint64_t strip_size;
+    uint64_t max_size;
+    vector bits;
+
+    // soft state
+    Header header; // FIXME: Hold lock to avoid concurrent operations, it will
+                   // also block read operation which not should be permitted.
+    coll_t cid;
+    ghobject_t oid;
+    bool deleted;
+    map, bufferlist> buffers;  // pair(prefix, key)
+
+    StripObjectHeader(): strip_size(default_strip_size), max_size(0), deleted(false) {}
+
+    void encode(bufferlist &bl) const {
+      ENCODE_START(1, 1, bl);
+      ::encode(strip_size, bl);
+      ::encode(max_size, bl);
+      ::encode(bits, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(bufferlist::iterator &bl) {
+      DECODE_START(1, bl);
+      ::decode(strip_size, bl);
+      ::decode(max_size, bl);
+      ::decode(bits, bl);
+      DECODE_FINISH(bl);
+    }
+  };
+  typedef ceph::shared_ptr StripObjectHeaderRef;
+
+  static int file_to_extents(uint64_t offset, size_t len, uint64_t strip_size,
+                             vector &extents);
+  int lookup_strip_header(const coll_t & cid, const ghobject_t &oid,
+                          StripObjectHeaderRef *header);
+  int save_strip_header(StripObjectHeaderRef header, KeyValueDB::Transaction t);
+  int create_strip_header(const coll_t &cid, const ghobject_t &oid,
+                          StripObjectHeaderRef *strip_header,
+                          KeyValueDB::Transaction t);
+  void clone_wrap(StripObjectHeaderRef old_header,
+                  const coll_t &cid, const ghobject_t &oid,
+                  KeyValueDB::Transaction t,
+                  StripObjectHeaderRef *target_header);
+  void rename_wrap(StripObjectHeaderRef old_header, const coll_t &cid, const ghobject_t &oid,
+                   KeyValueDB::Transaction t,
+                   StripObjectHeaderRef *new_header);
+  // Already hold header to avoid lock header seq again
+  int get_with_header(
+    const StripObjectHeaderRef header,
+    const string &prefix,
+    map *out
+    );
+
+  int get_values_with_header(
+    const StripObjectHeaderRef header,
+    const string &prefix,
+    const set &keys,
+    map *out
+    );
+  int get_keys_with_header(
+    const StripObjectHeaderRef header,
+    const string &prefix,
+    set *keys
+    );
+
+  Mutex lock;
+  void invalidate_cache(const coll_t &c, const ghobject_t &oid) {
+    Mutex::Locker l(lock);
+    caches.clear(oid);
+  }
+
+  RandomCache > caches;
+  StripObjectMap(KeyValueDB *db): GenericObjectMap(db),
+                                  lock("StripObjectMap::lock"),
+                                  caches(g_conf->keyvaluestore_header_cache_size)
+  {}
+};
+
+
+class KeyValueStore : public ObjectStore,
+                      public md_config_obs_t {
+ public:
+  objectstore_perf_stat_t get_cur_stats() {
+    objectstore_perf_stat_t ret;
+    return ret;
+  }
+
+  static const uint32_t target_version = 1;
+
+ private:
+  string internal_name; // internal name, used to name the perfcounter instance
+  string basedir;
+  std::string current_fn;
+  std::string current_op_seq_fn;
+  uuid_d fsid;
+
+  int fsid_fd, current_fd;
+
+  enum kvstore_types kv_type;
+
+  deque snaps;
+
+  // ObjectMap
+  boost::scoped_ptr backend;
+
+  Finisher ondisk_finisher;
+
+  Mutex lock;
+
+  int _create_current();
+
+  /// read a uuid from fd
+  int read_fsid(int fd, uuid_d *uuid);
+
+  /// lock fsid_fd
+  int lock_fsid();
+
+  string strip_object_key(uint64_t no) {
+    char n[100];
+    snprintf(n, 100, "%lld", (long long)no);
+    return string(n);
+  }
+
+  // A special coll used by store collection info, each obj in this coll
+  // represent a coll_t
+  static bool is_coll_obj(coll_t c) {
+    return c == coll_t("COLLECTIONS");
+  }
+  static coll_t get_coll_for_coll() {
+    return coll_t("COLLECTIONS");
+  }
+  static ghobject_t make_ghobject_for_coll(const coll_t &col) {
+    return ghobject_t(hobject_t(sobject_t(col.to_str(), CEPH_NOSNAP)));
+  }
+
+  // Each transaction has side effect which may influent the following
+  // operations, we need to make it visible for the following within
+  // transaction by caching middle result.
+  // Side effects contains:
+  // 1. Creating/Deleting collection
+  // 2. Creating/Deleting object
+  // 3. Object modify(including omap, xattr)
+  // 4. Clone or rename
+  struct BufferTransaction {
+    typedef pair uniq_id;
+    typedef map StripHeaderMap;
+
+    //Dirty records
+    StripHeaderMap strip_headers;
+    list finishes;
+
+    KeyValueStore *store;
+
+    KeyValueDB::Transaction t;
+
+    int lookup_cached_header(const coll_t &cid, const ghobject_t &oid,
+                             StripObjectMap::StripObjectHeaderRef *strip_header,
+                             bool create_if_missing);
+    int get_buffer_keys(StripObjectMap::StripObjectHeaderRef strip_header,
+                        const string &prefix, const set &keys,
+                        map *out);
+    void set_buffer_keys(StripObjectMap::StripObjectHeaderRef strip_header,
+                         const string &prefix, map &bl);
+    int remove_buffer_keys(StripObjectMap::StripObjectHeaderRef strip_header,
+                           const string &prefix, const set &keys);
+    void clear_buffer_keys(StripObjectMap::StripObjectHeaderRef strip_header,
+                           const string &prefix);
+    int clear_buffer(StripObjectMap::StripObjectHeaderRef strip_header);
+    void clone_buffer(StripObjectMap::StripObjectHeaderRef old_header,
+                      const coll_t &cid, const ghobject_t &oid);
+    void rename_buffer(StripObjectMap::StripObjectHeaderRef old_header,
+                       const coll_t &cid, const ghobject_t &oid);
+    int submit_transaction();
+
+    BufferTransaction(KeyValueStore *store): store(store) {
+      t = store->backend->get_transaction();
+    }
+
+    struct InvalidateCacheContext : public Context {
+      KeyValueStore *store;
+      const coll_t cid;
+      const ghobject_t oid;
+      InvalidateCacheContext(KeyValueStore *s, const coll_t &c, const ghobject_t &oid): store(s), cid(c), oid(oid) {}
+      void finish(int r) {
+      if (r == 0)
+        store->backend->invalidate_cache(cid, oid);
+      }
+    };
+  };
+
+  // -- op workqueue --
+  struct Op {
+    utime_t start;
+    uint64_t op;
+    list tls;
+    Context *ondisk, *onreadable, *onreadable_sync;
+    uint64_t ops, bytes;
+    TrackedOpRef osd_op;
+  };
+  class OpSequencer : public Sequencer_impl {
+    Mutex qlock; // to protect q, for benefit of flush (peek/dequeue also protected by lock)
+    list q;
+    Cond cond;
+    list > flush_commit_waiters;
+    uint64_t op; // used by flush() to know the sequence of op
+   public:
+    Sequencer *parent;
+    Mutex apply_lock;  // for apply mutual exclusion
+    
+    /// get_max_uncompleted
+    bool _get_max_uncompleted(
+      uint64_t *seq ///< [out] max uncompleted seq
+      ) {
+      assert(qlock.is_locked());
+      assert(seq);
+      *seq = 0;
+      if (q.empty()) {
+	return true;
+      } else {
+	*seq = q.back()->op;
+	return false;
+      }
+    } /// @returns true if the queue is empty
+
+    /// get_min_uncompleted
+    bool _get_min_uncompleted(
+      uint64_t *seq ///< [out] min uncompleted seq
+      ) {
+      assert(qlock.is_locked());
+      assert(seq);
+      *seq = 0;
+      if (q.empty()) {
+	return true;
+      } else {
+	*seq = q.front()->op;
+	return false;
+      }
+    } /// @returns true if both queues are empty
+
+    void _wake_flush_waiters(list *to_queue) {
+      uint64_t seq;
+      if (_get_min_uncompleted(&seq))
+	seq = -1;
+
+      for (list >::iterator i =
+	     flush_commit_waiters.begin();
+	   i != flush_commit_waiters.end() && i->first < seq;
+	   flush_commit_waiters.erase(i++)) {
+	to_queue->push_back(i->second);
+      }
+    }
+
+    void queue(Op *o) {
+      Mutex::Locker l(qlock);
+      q.push_back(o);
+      op++;
+      o->op = op;
+    }
+    Op *peek_queue() {
+      assert(apply_lock.is_locked());
+      return q.front();
+    }
+
+    Op *dequeue(list *to_queue) {
+      assert(to_queue);
+      assert(apply_lock.is_locked());
+      Mutex::Locker l(qlock);
+      Op *o = q.front();
+      q.pop_front();
+      cond.Signal();
+
+      _wake_flush_waiters(to_queue);
+      return o;
+    }
+
+    void flush() {
+      Mutex::Locker l(qlock);
+
+      // get max for journal _or_ op queues
+      uint64_t seq = 0;
+      if (!q.empty())
+        seq = q.back()->op;
+
+      if (seq) {
+        // everything prior to our watermark to drain through either/both
+        // queues
+        while (!q.empty() && q.front()->op <= seq)
+          cond.Wait(qlock);
+      }
+    }
+    bool flush_commit(Context *c) {
+      Mutex::Locker l(qlock);
+      uint64_t seq = 0;
+      if (_get_max_uncompleted(&seq)) {
+	delete c;
+	return true;
+      } else {
+	flush_commit_waiters.push_back(make_pair(seq, c));
+	return false;
+      }
+    }
+
+    OpSequencer()
+      : qlock("KeyValueStore::OpSequencer::qlock", false, false),
+        op(0), parent(0),
+	apply_lock("KeyValueStore::OpSequencer::apply_lock", false, false) {}
+    ~OpSequencer() {
+      assert(q.empty());
+    }
+
+    const string& get_name() const {
+      return parent->get_name();
+    }
+  };
+
+  friend ostream& operator<<(ostream& out, const OpSequencer& s);
+
+  Sequencer default_osr;
+  deque op_queue;
+  uint64_t op_queue_len, op_queue_bytes;
+  Cond op_throttle_cond;
+  Mutex op_throttle_lock;
+  Finisher op_finisher;
+
+  ThreadPool op_tp;
+  struct OpWQ : public ThreadPool::WorkQueue {
+    KeyValueStore *store;
+    OpWQ(KeyValueStore *fs, time_t timeout, time_t suicide_timeout,
+         ThreadPool *tp) :
+      ThreadPool::WorkQueue("KeyValueStore::OpWQ",
+                                         timeout, suicide_timeout, tp),
+      store(fs) {}
+
+    bool _enqueue(OpSequencer *osr) {
+      store->op_queue.push_back(osr);
+      return true;
+    }
+    void _dequeue(OpSequencer *o) {
+      assert(0);
+    }
+    bool _empty() {
+      return store->op_queue.empty();
+    }
+    OpSequencer *_dequeue() {
+      if (store->op_queue.empty())
+	return NULL;
+      OpSequencer *osr = store->op_queue.front();
+      store->op_queue.pop_front();
+      return osr;
+    }
+    void _process(OpSequencer *osr, ThreadPool::TPHandle &handle) {
+      store->_do_op(osr, handle);
+    }
+    void _process_finish(OpSequencer *osr) {
+      store->_finish_op(osr);
+    }
+    void _clear() {
+      assert(store->op_queue.empty());
+    }
+  } op_wq;
+
+  Op *build_op(list& tls, Context *ondisk, Context *onreadable,
+               Context *onreadable_sync, TrackedOpRef osd_op);
+  void queue_op(OpSequencer *osr, Op *o);
+  void op_queue_reserve_throttle(Op *o, ThreadPool::TPHandle *handle = NULL);
+  void _do_op(OpSequencer *osr, ThreadPool::TPHandle &handle);
+  void op_queue_release_throttle(Op *o);
+  void _finish_op(OpSequencer *osr);
+
+  PerfCounters *logger;
+
+ public:
+
+  KeyValueStore(const std::string &base,
+                const char *internal_name = "keyvaluestore-dev",
+                bool update_to=false);
+  ~KeyValueStore();
+
+  int _detect_backend() { kv_type = KV_TYPE_LEVELDB; return 0; }
+  bool test_mount_in_use();
+  int version_stamp_is_valid(uint32_t *version);
+  int update_version_stamp();
+  uint32_t get_target_version() {
+    return target_version;
+  }
+  int peek_journal_fsid(uuid_d *id) {
+    *id = fsid;
+    return 0;
+  }
+
+  int write_version_stamp();
+  int mount();
+  int umount();
+  int get_max_object_name_length();
+  int mkfs();
+  int mkjournal() {return 0;}
+
+  /**
+   ** set_allow_sharded_objects()
+   **
+   ** Before sharded ghobject_t can be specified this function must be called
+   **/
+  void set_allow_sharded_objects() {}
+
+  /**
+   ** get_allow_sharded_objects()
+   **
+   ** return value: true if set_allow_sharded_objects() called, otherwise false
+   **/
+  bool get_allow_sharded_objects() {return false;}
+
+  int statfs(struct statfs *buf);
+
+  int _do_transactions(
+    list &tls, uint64_t op_seq,
+    ThreadPool::TPHandle *handle);
+  int do_transactions(list &tls, uint64_t op_seq) {
+    return _do_transactions(tls, op_seq, 0);
+  }
+  unsigned _do_transaction(Transaction& transaction,
+                           BufferTransaction &bt,
+                           ThreadPool::TPHandle *handle);
+
+  int queue_transactions(Sequencer *osr, list& tls,
+                         TrackedOpRef op = TrackedOpRef(),
+                         ThreadPool::TPHandle *handle = NULL);
+
+
+  // ------------------
+  // objects
+
+  int _generic_read(StripObjectMap::StripObjectHeaderRef header,
+                    uint64_t offset, size_t len, bufferlist& bl,
+                    bool allow_eio = false, BufferTransaction *bt = 0);
+  int _generic_write(StripObjectMap::StripObjectHeaderRef header,
+                     uint64_t offset, size_t len, const bufferlist& bl,
+                     BufferTransaction &t, bool replica = false);
+
+  bool exists(coll_t cid, const ghobject_t& oid);
+  int stat(coll_t cid, const ghobject_t& oid, struct stat *st,
+           bool allow_eio = false);
+  int read(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+           bufferlist& bl, bool allow_eio = false);
+  int fiemap(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+             bufferlist& bl);
+
+  int _touch(coll_t cid, const ghobject_t& oid, BufferTransaction &t);
+  int _write(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+             const bufferlist& bl, BufferTransaction &t, bool replica = false);
+  int _zero(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len,
+            BufferTransaction &t);
+  int _truncate(coll_t cid, const ghobject_t& oid, uint64_t size,
+                BufferTransaction &t);
+  int _clone(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid,
+             BufferTransaction &t);
+  int _clone_range(coll_t cid, const ghobject_t& oldoid,
+                   const ghobject_t& newoid, uint64_t srcoff,
+                   uint64_t len, uint64_t dstoff, BufferTransaction &t);
+  int _remove(coll_t cid, const ghobject_t& oid, BufferTransaction &t);
+
+
+  void start_sync() {}
+  void sync() {}
+  void flush() {}
+  void sync_and_flush() {}
+
+  void set_fsid(uuid_d u) { fsid = u; }
+  uuid_d get_fsid() { return fsid; }
+
+  // attrs
+  int getattr(coll_t cid, const ghobject_t& oid, const char *name,
+              bufferptr &bp);
+  int getattrs(coll_t cid, const ghobject_t& oid, map& aset,
+               bool user_only = false);
+
+  int _setattrs(coll_t cid, const ghobject_t& oid,
+                map& aset, BufferTransaction &t);
+  int _rmattr(coll_t cid, const ghobject_t& oid, const char *name,
+              BufferTransaction &t);
+  int _rmattrs(coll_t cid, const ghobject_t& oid, BufferTransaction &t);
+
+  int collection_getattr(coll_t c, const char *name, void *value, size_t size);
+  int collection_getattr(coll_t c, const char *name, bufferlist& bl);
+  int collection_getattrs(coll_t cid, map &aset);
+
+  int _collection_setattr(coll_t c, const char *name, const void *value,
+                          size_t size, BufferTransaction &t);
+  int _collection_rmattr(coll_t c, const char *name, BufferTransaction &t);
+  int _collection_setattrs(coll_t cid, map &aset,
+                           BufferTransaction &t);
+
+  // collections
+  int _create_collection(coll_t c, BufferTransaction &t);
+  int _destroy_collection(coll_t c, BufferTransaction &t);
+  int _collection_add(coll_t c, coll_t ocid, const ghobject_t& oid,
+                      BufferTransaction &t);
+  int _collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+                              coll_t c, const ghobject_t& o,
+                              BufferTransaction &t);
+  int _collection_remove_recursive(const coll_t &cid,
+                                   BufferTransaction &t);
+  int _collection_rename(const coll_t &cid, const coll_t &ncid,
+                         BufferTransaction &t);
+  int list_collections(vector& ls);
+  bool collection_exists(coll_t c);
+  bool collection_empty(coll_t c);
+  int collection_list(coll_t c, vector& oid);
+  int collection_list_partial(coll_t c, ghobject_t start,
+                              int min, int max, snapid_t snap,
+                              vector *ls, ghobject_t *next);
+  int collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
+                            snapid_t seq, vector *ls);
+  int collection_version_current(coll_t c, uint32_t *version);
+
+  // omap (see ObjectStore.h for documentation)
+  int omap_get(coll_t c, const ghobject_t &oid, bufferlist *header,
+               map *out);
+  int omap_get_header(
+    coll_t c,
+    const ghobject_t &oid,
+    bufferlist *out,
+    bool allow_eio = false);
+  int omap_get_keys(coll_t c, const ghobject_t &oid, set *keys);
+  int omap_get_values(coll_t c, const ghobject_t &oid, const set &keys,
+                      map *out);
+  int omap_check_keys(coll_t c, const ghobject_t &oid, const set &keys,
+                      set *out);
+  ObjectMap::ObjectMapIterator get_omap_iterator(coll_t c,
+                                                 const ghobject_t &oid);
+
+  void dump_transactions(list& ls, uint64_t seq,
+                         OpSequencer *osr);
+
+ private:
+  void _inject_failure() {}
+
+  // omap
+  int _omap_clear(coll_t cid, const ghobject_t &oid,
+                  BufferTransaction &t);
+  int _omap_setkeys(coll_t cid, const ghobject_t &oid,
+                    map &aset,
+                    BufferTransaction &t);
+  int _omap_rmkeys(coll_t cid, const ghobject_t &oid, const set &keys,
+                   BufferTransaction &t);
+  int _omap_rmkeyrange(coll_t cid, const ghobject_t &oid,
+                       const string& first, const string& last,
+                       BufferTransaction &t);
+  int _omap_setheader(coll_t cid, const ghobject_t &oid, const bufferlist &bl,
+                      BufferTransaction &t);
+  int _split_collection(coll_t cid, uint32_t bits, uint32_t rem, coll_t dest,
+                        BufferTransaction &t);
+  int _split_collection_create(coll_t cid, uint32_t bits, uint32_t rem,
+                               coll_t dest, BufferTransaction &t){
+    return 0;
+  }
+
+  virtual const char** get_tracked_conf_keys() const;
+  virtual void handle_conf_change(const struct md_config_t *conf,
+                                  const std::set  &changed);
+
+  std::string m_osd_rollback_to_cluster_snap;
+  int m_keyvaluestore_queue_max_ops;
+  int m_keyvaluestore_queue_max_bytes;
+
+  int do_update;
+
+
+  static const string OBJECT_STRIP_PREFIX;
+  static const string OBJECT_XATTR;
+  static const string OBJECT_OMAP;
+  static const string OBJECT_OMAP_HEADER;
+  static const string OBJECT_OMAP_HEADER_KEY;
+  static const string COLLECTION;
+  static const string COLLECTION_ATTR;
+  static const uint32_t COLLECTION_VERSION = 1;
+};
+
+WRITE_CLASS_ENCODER(StripObjectMap::StripObjectHeader)
+
+#endif
diff --git a/ceph/src/os/LFNIndex.cc b/ceph/src/os/LFNIndex.cc
new file mode 100644
index 00000000..e017f83a
--- /dev/null
+++ b/ceph/src/os/LFNIndex.cc
@@ -0,0 +1,1367 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#if defined(__FreeBSD__)
+#include 
+#endif
+
+#include "osd/osd_types.h"
+#include "include/object.h"
+#include "common/config.h"
+#include "common/debug.h"
+#include "include/buffer.h"
+#include "common/ceph_crypto.h"
+#include "include/compat.h"
+#include "chain_xattr.h"
+
+#include "LFNIndex.h"
+using ceph::crypto::SHA1;
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "LFNIndex(" << get_base_path() << ") "
+
+
+const string LFNIndex::LFN_ATTR = "user.cephos.lfn";
+const string LFNIndex::PHASH_ATTR_PREFIX = "user.cephos.phash.";
+const string LFNIndex::SUBDIR_PREFIX = "DIR_";
+const string LFNIndex::FILENAME_COOKIE = "long";
+const int LFNIndex::FILENAME_PREFIX_LEN =  FILENAME_SHORT_LEN - FILENAME_HASH_LEN - 
+								FILENAME_COOKIE.size() - 
+								FILENAME_EXTRA;
+void LFNIndex::maybe_inject_failure()
+{
+  if (error_injection_enabled) {
+    if (current_failure > last_failure &&
+	(((double)(rand() % 10000))/((double)(10000))
+	 < error_injection_probability)) {
+      last_failure = current_failure;
+      current_failure = 0;
+      throw RetryException();
+    }
+    ++current_failure;
+  }
+}
+
+// Helper to close fd's when we leave scope.  This is useful when used
+// in combination with RetryException, thrown by the above.
+struct FDCloser {
+  int fd;
+  FDCloser(int f) : fd(f) {}
+  ~FDCloser() {
+    VOID_TEMP_FAILURE_RETRY(::close(fd));
+  }
+};
+
+
+/* Public methods */
+
+void LFNIndex::set_ref(ceph::shared_ptr ref)
+{
+  self_ref = ref;
+}
+
+int LFNIndex::init()
+{
+  return _init();
+}
+
+int LFNIndex::created(const ghobject_t &oid, const char *path)
+{
+  WRAP_RETRY(
+  vector path_comp;
+  string short_name;
+  r = decompose_full_path(path, &path_comp, 0, &short_name);
+  if (r < 0)
+    goto out;
+  r = lfn_created(path_comp, oid, short_name);
+  if (r < 0)
+    goto out;
+  r = _created(path_comp, oid, short_name);
+  if (r < 0)
+    goto out;
+    );
+}
+
+int LFNIndex::unlink(const ghobject_t &oid)
+{
+  WRAP_RETRY(
+  vector path;
+  string short_name;
+  r = _lookup(oid, &path, &short_name, NULL);
+  if (r < 0) {
+    goto out;
+  }
+  r = _remove(path, oid, short_name);
+  if (r < 0) {
+    goto out;
+  }
+  );
+}
+
+int LFNIndex::lookup(const ghobject_t &oid,
+		     IndexedPath *out_path,
+		     int *exist)
+{
+  WRAP_RETRY(
+  vector path;
+  string short_name;
+  r = _lookup(oid, &path, &short_name, exist);
+  if (r < 0)
+    goto out;
+  string full_path = get_full_path(path, short_name);
+  struct stat buf;
+  maybe_inject_failure();
+  r = ::stat(full_path.c_str(), &buf);
+  maybe_inject_failure();
+  if (r < 0) {
+    if (errno == ENOENT) {
+      *exist = 0;
+    } else {
+      r = -errno;
+      goto out;
+    }
+  } else {
+    *exist = 1;
+  }
+  *out_path = IndexedPath(new Path(full_path, self_ref));
+  r = 0;
+  );
+}
+
+int LFNIndex::collection_list(vector *ls)
+{
+  return _collection_list(ls);
+}
+
+
+int LFNIndex::collection_list_partial(const ghobject_t &start,
+				      int min_count,
+				      int max_count,
+				      snapid_t seq,
+				      vector *ls,
+				      ghobject_t *next)
+{
+  return _collection_list_partial(start, min_count, max_count, seq, ls, next);
+}
+
+/* Derived class utility methods */
+
+int LFNIndex::fsync_dir(const vector &path)
+{
+  maybe_inject_failure();
+  int fd = ::open(get_full_path_subdir(path).c_str(), O_RDONLY);
+  if (fd < 0)
+    return -errno;
+  FDCloser f(fd);
+  maybe_inject_failure();
+  int r = ::fsync(fd);
+  maybe_inject_failure();
+  if (r < 0)
+    return -errno;
+  else
+    return 0;
+}
+
+int LFNIndex::link_object(const vector &from,
+			  const vector &to,
+			  const ghobject_t &oid,
+			  const string &from_short_name)
+{
+  int r;
+  string from_path = get_full_path(from, from_short_name);
+  string to_path;
+  maybe_inject_failure();
+  r = lfn_get_name(to, oid, 0, &to_path, 0);
+  if (r < 0)
+    return r;
+  maybe_inject_failure();
+  r = ::link(from_path.c_str(), to_path.c_str());
+  maybe_inject_failure();
+  if (r < 0)
+    return -errno;
+  else
+    return 0;
+}
+
+int LFNIndex::remove_objects(const vector &dir,
+			     const map &to_remove,
+			     map *remaining)
+{
+  set clean_chains;
+  for (map::const_iterator to_clean = to_remove.begin();
+       to_clean != to_remove.end();
+       ++to_clean) {
+    if (!lfn_is_hashed_filename(to_clean->first)) {
+      maybe_inject_failure();
+      int r = ::unlink(get_full_path(dir, to_clean->first).c_str());
+      maybe_inject_failure();
+      if (r < 0)
+	return -errno;
+      continue;
+    }
+    if (clean_chains.count(lfn_get_short_name(to_clean->second, 0)))
+      continue;
+    set holes;
+    map > chain;
+    for (int i = 0; ; ++i) {
+      string short_name = lfn_get_short_name(to_clean->second, i);
+      if (remaining->count(short_name)) {
+	chain[i] = *(remaining->find(short_name));
+      } else if (to_remove.count(short_name)) {
+	holes.insert(i);
+      } else {
+	break;
+      }
+    }
+
+    map >::reverse_iterator candidate = chain.rbegin();
+    for (set::iterator i = holes.begin();
+	 i != holes.end();
+	 ++i) {
+      if (candidate == chain.rend() || *i > candidate->first) {
+	string remove_path_name = 
+	  get_full_path(dir, lfn_get_short_name(to_clean->second, *i)); 
+	maybe_inject_failure();
+	int r = ::unlink(remove_path_name.c_str());
+	maybe_inject_failure();
+	if (r < 0)
+	  return -errno;
+	continue;
+      }
+      string from = get_full_path(dir, candidate->second.first);
+      string to = get_full_path(dir, lfn_get_short_name(candidate->second.second, *i));
+      maybe_inject_failure();
+      int r = ::rename(from.c_str(), to.c_str());
+      maybe_inject_failure();
+      if (r < 0)
+	return -errno;
+      remaining->erase(candidate->second.first);
+      remaining->insert(pair(
+			  lfn_get_short_name(candidate->second.second, *i),
+					     candidate->second.second));
+      ++candidate;
+    }
+    if (!holes.empty())
+      clean_chains.insert(lfn_get_short_name(to_clean->second, 0));
+  }
+  return 0;
+}
+
+int LFNIndex::move_objects(const vector &from,
+			   const vector &to)
+{
+  map to_move;
+  int r;
+  r = list_objects(from, 0, NULL, &to_move);
+  if (r < 0)
+    return r;
+  for (map::iterator i = to_move.begin();
+       i != to_move.end();
+       ++i) {
+    string from_path = get_full_path(from, i->first);
+    string to_path, to_name;
+    r = lfn_get_name(to, i->second, &to_name, &to_path, 0);
+    if (r < 0)
+      return r;
+    maybe_inject_failure();
+    r = ::link(from_path.c_str(), to_path.c_str());
+    if (r < 0 && errno != EEXIST)
+      return -errno;
+    maybe_inject_failure();
+    r = lfn_created(to, i->second, to_name);
+    maybe_inject_failure();
+    if (r < 0)
+      return r;
+  }
+  r = fsync_dir(to);
+  if (r < 0)
+    return r;
+  for (map::iterator i = to_move.begin();
+       i != to_move.end();
+       ++i) {
+    maybe_inject_failure();
+    r = ::unlink(get_full_path(from, i->first).c_str());
+    maybe_inject_failure();
+    if (r < 0)
+      return -errno;
+  }
+  return fsync_dir(from);
+}
+
+int LFNIndex::remove_object(const vector &from,
+			    const ghobject_t &oid)
+{
+  string short_name;
+  int r, exist;
+  maybe_inject_failure();
+  r = get_mangled_name(from, oid, &short_name, &exist);
+  maybe_inject_failure();
+  if (r < 0)
+    return r;
+  return lfn_unlink(from, oid, short_name);
+}
+
+int LFNIndex::get_mangled_name(const vector &from,
+			       const ghobject_t &oid,
+			       string *mangled_name, int *exists)
+{
+  return lfn_get_name(from, oid, mangled_name, 0, exists);
+}
+
+int LFNIndex::move_subdir(
+  LFNIndex &from,
+  LFNIndex &dest,
+  const vector &path,
+  string dir
+  )
+{
+  vector sub_path(path.begin(), path.end());
+  sub_path.push_back(dir);
+  string from_path(from.get_full_path_subdir(sub_path));
+  string to_path(dest.get_full_path_subdir(sub_path));
+  int r = ::rename(from_path.c_str(), to_path.c_str());
+  if (r < 0)
+    return -errno;
+  return 0;
+}
+
+int LFNIndex::move_object(
+  LFNIndex &from,
+  LFNIndex &dest,
+  const vector &path,
+  const pair &obj
+  )
+{
+  string from_path(from.get_full_path(path, obj.first));
+  string to_path;
+  string to_name;
+  int exists;
+  int r = dest.lfn_get_name(path, obj.second, &to_name, &to_path, &exists);
+  if (r < 0)
+    return r;
+  if (!exists) {
+    r = ::link(from_path.c_str(), to_path.c_str());
+    if (r < 0)
+      return r;
+  }
+  r = dest.lfn_created(path, obj.second, to_name);
+  if (r < 0)
+    return r;
+  r = dest.fsync_dir(path);
+  if (r < 0)
+    return r;
+  r = from.remove_object(path, obj.second);
+  if (r < 0)
+    return r;
+  return from.fsync_dir(path);
+}
+
+
+static int get_hobject_from_oinfo(const char *dir, const char *file, 
+				  ghobject_t *o)
+{
+  char path[PATH_MAX];
+  bufferptr bp(PATH_MAX);
+  snprintf(path, sizeof(path), "%s/%s", dir, file);
+  // Hack, user.ceph._ is the attribute used to store the object info
+  int r = chain_getxattr(path, "user.ceph._", bp.c_str(), bp.length());
+  if (r < 0)
+    return r;
+  bufferlist bl;
+  bl.push_back(bp);
+  object_info_t oi(bl);
+  *o = oi.soid;
+  return 0;
+}
+
+
+int LFNIndex::list_objects(const vector &to_list, int max_objs,
+			   long *handle, map *out)
+{
+  string to_list_path = get_full_path_subdir(to_list);
+  DIR *dir = ::opendir(to_list_path.c_str());
+  char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
+  int r;
+  if (!dir) {
+    return -errno;
+  }
+
+  if (handle && *handle) {
+    seekdir(dir, *handle);
+  }
+
+  struct dirent *de;
+  int listed = 0;
+  bool end = false;
+  while (!::readdir_r(dir, reinterpret_cast(buf), &de)) {
+    if (!de) {
+      end = true;
+      break;
+    }
+    if (max_objs > 0 && listed >= max_objs) {
+      break;
+    }
+    if (de->d_name[0] == '.')
+      continue;
+    string short_name(de->d_name);
+    ghobject_t obj;
+    if (lfn_is_object(short_name)) {
+      r = lfn_translate(to_list, short_name, &obj);
+      if (r < 0) {
+	r = -errno;
+	goto cleanup;
+      } else if (r > 0) {
+	string long_name = lfn_generate_object_name(obj);
+	if (!lfn_must_hash(long_name)) {
+	  assert(long_name == short_name);
+	}
+	if (index_version == HASH_INDEX_TAG)
+	  get_hobject_from_oinfo(to_list_path.c_str(), short_name.c_str(), &obj);
+	  
+	out->insert(pair(short_name, obj));
+	++listed;
+      } else {
+	continue;
+      }
+    }
+  }
+
+  if (handle && !end) {
+    *handle = telldir(dir);
+  }
+
+  r = 0;
+ cleanup:
+  ::closedir(dir);
+  return r;
+}
+
+int LFNIndex::list_subdirs(const vector &to_list,
+				  set *out)
+{
+  string to_list_path = get_full_path_subdir(to_list);
+  DIR *dir = ::opendir(to_list_path.c_str());
+  char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
+  if (!dir)
+    return -errno;
+
+  struct dirent *de;
+  while (!::readdir_r(dir, reinterpret_cast(buf), &de)) {
+    if (!de) {
+      break;
+    }
+    string short_name(de->d_name);
+    string demangled_name;
+    ghobject_t obj;
+    if (lfn_is_subdir(short_name, &demangled_name)) {
+      out->insert(demangled_name);
+    }
+  }
+
+  ::closedir(dir);
+  return 0;
+}
+
+int LFNIndex::create_path(const vector &to_create)
+{
+  maybe_inject_failure();
+  int r = ::mkdir(get_full_path_subdir(to_create).c_str(), 0777);
+  maybe_inject_failure();
+  if (r < 0)
+    return -errno;
+  else
+    return 0;
+}
+
+int LFNIndex::remove_path(const vector &to_remove)
+{
+  maybe_inject_failure();
+  int r = ::rmdir(get_full_path_subdir(to_remove).c_str());
+  maybe_inject_failure();
+  if (r < 0)
+    return -errno;
+  else
+    return 0;
+}
+
+int LFNIndex::path_exists(const vector &to_check, int *exists)
+{
+  string full_path = get_full_path_subdir(to_check);
+  struct stat buf;
+  if (::stat(full_path.c_str(), &buf)) {
+    int r = -errno;
+    if (r == -ENOENT) {
+      *exists = 0;
+      return 0;
+    } else {
+      return r;
+    }
+  } else {
+    *exists = 1;
+    return 0;
+  }
+}
+
+int LFNIndex::add_attr_path(const vector &path,
+			    const string &attr_name, 
+			    bufferlist &attr_value)
+{
+  string full_path = get_full_path_subdir(path);
+  maybe_inject_failure();
+  return chain_setxattr(full_path.c_str(), mangle_attr_name(attr_name).c_str(),
+		     reinterpret_cast(attr_value.c_str()),
+		     attr_value.length());
+}
+
+int LFNIndex::get_attr_path(const vector &path,
+			    const string &attr_name, 
+			    bufferlist &attr_value)
+{
+  string full_path = get_full_path_subdir(path);
+  size_t size = 1024; // Initial
+  while (1) {
+    bufferptr buf(size);
+    int r = chain_getxattr(full_path.c_str(), mangle_attr_name(attr_name).c_str(),
+			 reinterpret_cast(buf.c_str()),
+			 size);
+    if (r > 0) {
+      buf.set_length(r);
+      attr_value.push_back(buf);
+      break;
+    } else {
+      r = -errno;
+      if (r == -ERANGE) {
+	size *= 2;
+      } else {
+	return r;
+      }
+    }
+  }
+  return 0;
+}
+
+int LFNIndex::remove_attr_path(const vector &path,
+			       const string &attr_name)
+{
+  string full_path = get_full_path_subdir(path);
+  string mangled_attr_name = mangle_attr_name(attr_name);
+  maybe_inject_failure();
+  return chain_removexattr(full_path.c_str(), mangled_attr_name.c_str());
+}
+  
+string LFNIndex::lfn_generate_object_name_keyless(const ghobject_t &oid)
+{
+  char s[FILENAME_MAX_LEN];
+  char *end = s + sizeof(s);
+  char *t = s;
+
+  assert(oid.generation == ghobject_t::NO_GEN);
+  const char *i = oid.hobj.oid.name.c_str();
+  // Escape subdir prefix
+  if (oid.hobj.oid.name.substr(0, 4) == "DIR_") {
+    *t++ = '\\';
+    *t++ = 'd';
+    i += 4;
+  }
+  while (*i && t < end) {
+    if (*i == '\\') {
+      *t++ = '\\';
+      *t++ = '\\';      
+    } else if (*i == '.' && i == oid.hobj.oid.name.c_str()) {  // only escape leading .
+      *t++ = '\\';
+      *t++ = '.';
+    } else if (*i == '/') {
+      *t++ = '\\';
+      *t++ = 's';
+    } else
+      *t++ = *i;
+    i++;
+  }
+
+  if (oid.hobj.snap == CEPH_NOSNAP)
+    t += snprintf(t, end - t, "_head");
+  else if (oid.hobj.snap == CEPH_SNAPDIR)
+    t += snprintf(t, end - t, "_snapdir");
+  else
+    t += snprintf(t, end - t, "_%llx", (long long unsigned)oid.hobj.snap);
+  snprintf(t, end - t, "_%.*X", (int)(sizeof(oid.hobj.hash)*2), oid.hobj.hash);
+
+  return string(s);
+}
+
+static void append_escaped(string::const_iterator begin,
+			   string::const_iterator end, 
+			   string *out)
+{
+  for (string::const_iterator i = begin; i != end; ++i) {
+    if (*i == '\\') {
+      out->append("\\\\");
+    } else if (*i == '/') {
+      out->append("\\s");
+    } else if (*i == '_') {
+      out->append("\\u");
+    } else if (*i == '\0') {
+      out->append("\\n");
+    } else {
+      out->append(i, i+1);
+    }
+  }
+}
+
+string LFNIndex::lfn_generate_object_name(const ghobject_t &oid)
+{
+  if (index_version == HASH_INDEX_TAG)
+    return lfn_generate_object_name_keyless(oid);
+  if (index_version == HASH_INDEX_TAG_2)
+    return lfn_generate_object_name_poolless(oid);
+
+  string full_name;
+  string::const_iterator i = oid.hobj.oid.name.begin();
+  if (oid.hobj.oid.name.substr(0, 4) == "DIR_") {
+    full_name.append("\\d");
+    i += 4;
+  } else if (oid.hobj.oid.name[0] == '.') {
+    full_name.append("\\.");
+    ++i;
+  }
+  append_escaped(i, oid.hobj.oid.name.end(), &full_name);
+  full_name.append("_");
+  append_escaped(oid.hobj.get_key().begin(), oid.hobj.get_key().end(), &full_name);
+  full_name.append("_");
+
+  char buf[PATH_MAX];
+  char *t = buf;
+  char *end = t + sizeof(buf);
+  if (oid.hobj.snap == CEPH_NOSNAP)
+    t += snprintf(t, end - t, "head");
+  else if (oid.hobj.snap == CEPH_SNAPDIR)
+    t += snprintf(t, end - t, "snapdir");
+  else
+    t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.snap);
+  snprintf(t, end - t, "_%.*X", (int)(sizeof(oid.hobj.hash)*2), oid.hobj.hash);
+  full_name += string(buf);
+  full_name.append("_");
+
+  append_escaped(oid.hobj.nspace.begin(), oid.hobj.nspace.end(), &full_name);
+  full_name.append("_");
+
+  t = buf;
+  end = t + sizeof(buf);
+  if (oid.hobj.pool == -1)
+    t += snprintf(t, end - t, "none");
+  else
+    t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.pool);
+  full_name += string(buf);
+
+  if (oid.generation != ghobject_t::NO_GEN ||
+      oid.shard_id != ghobject_t::NO_SHARD) {
+    full_name.append("_");
+
+    t = buf;
+    end = t + sizeof(buf);
+    t += snprintf(t, end - t, "%llx", (long long unsigned)oid.generation);
+    full_name += string(buf);
+
+    full_name.append("_");
+
+    t = buf;
+    end = t + sizeof(buf);
+    t += snprintf(t, end - t, "%x", (int)oid.shard_id);
+    full_name += string(buf);
+  }
+
+  return full_name;
+}
+
+string LFNIndex::lfn_generate_object_name_poolless(const ghobject_t &oid)
+{
+  if (index_version == HASH_INDEX_TAG)
+    return lfn_generate_object_name_keyless(oid);
+
+  assert(oid.generation == ghobject_t::NO_GEN);
+  string full_name;
+  string::const_iterator i = oid.hobj.oid.name.begin();
+  if (oid.hobj.oid.name.substr(0, 4) == "DIR_") {
+    full_name.append("\\d");
+    i += 4;
+  } else if (oid.hobj.oid.name[0] == '.') {
+    full_name.append("\\.");
+    ++i;
+  }
+  append_escaped(i, oid.hobj.oid.name.end(), &full_name);
+  full_name.append("_");
+  append_escaped(oid.hobj.get_key().begin(), oid.hobj.get_key().end(), &full_name);
+  full_name.append("_");
+
+  char snap_with_hash[PATH_MAX];
+  char *t = snap_with_hash;
+  char *end = t + sizeof(snap_with_hash);
+  if (oid.hobj.snap == CEPH_NOSNAP)
+    t += snprintf(t, end - t, "head");
+  else if (oid.hobj.snap == CEPH_SNAPDIR)
+    t += snprintf(t, end - t, "snapdir");
+  else
+    t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.snap);
+  snprintf(t, end - t, "_%.*X", (int)(sizeof(oid.hobj.hash)*2), oid.hobj.hash);
+  full_name += string(snap_with_hash);
+  return full_name;
+}
+
+int LFNIndex::lfn_get_name(const vector &path, 
+			   const ghobject_t &oid,
+			   string *mangled_name, string *out_path,
+			   int *exists)
+{
+  string subdir_path = get_full_path_subdir(path);
+  string full_name = lfn_generate_object_name(oid);
+  int r;
+
+  if (!lfn_must_hash(full_name)) {
+    if (mangled_name)
+      *mangled_name = full_name;
+    if (out_path)
+      *out_path = get_full_path(path, full_name);
+    if (exists) {
+      struct stat buf;
+      string full_path = get_full_path(path, full_name);
+      maybe_inject_failure();
+      r = ::stat(full_path.c_str(), &buf);
+      if (r < 0) {
+	if (errno == ENOENT)
+	  *exists = 0;
+	else
+	  return -errno;
+      } else {
+	*exists = 1;
+      }
+    }
+    return 0;
+  }
+
+  int i = 0;
+  string candidate;
+  string candidate_path;
+  char buf[FILENAME_MAX_LEN + 1];
+  for ( ; ; ++i) {
+    candidate = lfn_get_short_name(oid, i);
+    candidate_path = get_full_path(path, candidate);
+    r = chain_getxattr(candidate_path.c_str(), get_lfn_attr().c_str(),
+		       buf, sizeof(buf));
+    if (r < 0) {
+      if (errno != ENODATA && errno != ENOENT)
+	return -errno;
+      if (errno == ENODATA) {
+	// Left over from incomplete transaction, it'll be replayed
+	maybe_inject_failure();
+	r = ::unlink(candidate_path.c_str());
+	maybe_inject_failure();
+	if (r < 0)
+	  return -errno;
+      }
+      if (mangled_name)
+	*mangled_name = candidate;
+      if (out_path)
+	*out_path = candidate_path;
+      if (exists)
+	*exists = 0;
+      return 0;
+    }
+    assert(r > 0);
+    buf[MIN((int)sizeof(buf) - 1, r)] = '\0';
+    if (!strcmp(buf, full_name.c_str())) {
+      if (mangled_name)
+	*mangled_name = candidate;
+      if (out_path)
+	*out_path = candidate_path;
+      if (exists)
+	*exists = 1;
+      return 0;
+    }
+    r = chain_getxattr(candidate_path.c_str(), get_alt_lfn_attr().c_str(),
+		       buf, sizeof(buf));
+    if (r > 0) {
+      // only consider alt name if nlink > 1
+      struct stat st;
+      int rc = ::stat(candidate_path.c_str(), &st);
+      if (rc < 0)
+	return -errno;
+      if (st.st_nlink <= 1) {
+	// left over from incomplete unlink, remove
+	maybe_inject_failure();
+	dout(20) << __func__ << " found extra alt attr for " << candidate_path
+		 << ", long name " << string(buf, r) << dendl;
+	rc = chain_removexattr(candidate_path.c_str(),
+			       get_alt_lfn_attr().c_str());
+	maybe_inject_failure();
+	if (rc < 0)
+	  return rc;
+	continue;
+      }
+      buf[MIN((int)sizeof(buf) - 1, r)] = '\0';
+      if (!strcmp(buf, full_name.c_str())) {
+	dout(20) << __func__ << " used alt attr for " << full_name << dendl;
+	if (mangled_name)
+	  *mangled_name = candidate;
+	if (out_path)
+	  *out_path = candidate_path;
+	if (exists)
+	  *exists = 1;
+	return 0;
+      }
+    }
+  }
+  assert(0); // Unreachable
+  return 0;
+}
+
+int LFNIndex::lfn_created(const vector &path,
+			  const ghobject_t &oid,
+			  const string &mangled_name)
+{
+  if (!lfn_is_hashed_filename(mangled_name))
+    return 0;
+  string full_path = get_full_path(path, mangled_name);
+  string full_name = lfn_generate_object_name(oid);
+  maybe_inject_failure();
+
+  // if the main attr exists and is different, move it to the alt attr.
+  char buf[FILENAME_MAX_LEN + 1];
+  int r = chain_getxattr(full_path.c_str(), get_lfn_attr().c_str(),
+			 buf, sizeof(buf));
+  if (r >= 0 && (r != (int)full_name.length() ||
+		 memcmp(buf, full_name.c_str(), full_name.length()))) {
+    dout(20) << __func__ << " " << mangled_name
+	     << " moving old name to alt attr "
+	     << string(buf, r)
+	     << ", new name is " << full_name << dendl;
+    r = chain_setxattr(full_path.c_str(), get_alt_lfn_attr().c_str(),
+		       buf, r);
+    if (r < 0)
+      return r;
+  }
+
+  return chain_setxattr(full_path.c_str(), get_lfn_attr().c_str(),
+		     full_name.c_str(), full_name.size());
+}
+
+int LFNIndex::lfn_unlink(const vector &path,
+			 const ghobject_t &oid,
+			 const string &mangled_name)
+{
+  if (!lfn_is_hashed_filename(mangled_name)) {
+    string full_path = get_full_path(path, mangled_name);
+    maybe_inject_failure();
+    int r = ::unlink(full_path.c_str());
+    maybe_inject_failure();
+    if (r < 0)
+      return -errno;
+    return 0;
+  }
+  string subdir_path = get_full_path_subdir(path);
+  
+  
+  int i = 0;
+  for ( ; ; ++i) {
+    string candidate = lfn_get_short_name(oid, i);
+    if (candidate == mangled_name)
+      break;
+  }
+  int removed_index = i;
+  ++i;
+  for ( ; ; ++i) {
+    struct stat buf;
+    string to_check = lfn_get_short_name(oid, i);
+    string to_check_path = get_full_path(path, to_check);
+    int r = ::stat(to_check_path.c_str(), &buf);
+    if (r < 0) {
+      if (errno == ENOENT) {
+	break;
+      } else {
+	return -errno;
+      }
+    }
+  }
+  string full_path = get_full_path(path, mangled_name);
+  int fd = ::open(full_path.c_str(), O_RDONLY);
+  if (fd < 0)
+    return -errno;
+  FDCloser f(fd);
+  if (i == removed_index + 1) {
+    maybe_inject_failure();
+    int r = ::unlink(full_path.c_str());
+    maybe_inject_failure();
+    if (r < 0)
+      return -errno;
+  } else {
+    string& rename_to = full_path;
+    string rename_from = get_full_path(path, lfn_get_short_name(oid, i - 1));
+    maybe_inject_failure();
+    int r = ::rename(rename_from.c_str(), rename_to.c_str());
+    maybe_inject_failure();
+    if (r < 0)
+      return -errno;
+  }
+  struct stat st;
+  int r = ::fstat(fd, &st);
+  if (r == 0 && st.st_nlink > 0) {
+    // remove alt attr
+    dout(20) << __func__ << " removing alt attr from " << full_path << dendl;
+    fsync_dir(path);
+    chain_fremovexattr(fd, get_alt_lfn_attr().c_str());
+  }
+  return r;
+}
+
+int LFNIndex::lfn_translate(const vector &path,
+			    const string &short_name,
+			    ghobject_t *out)
+{
+  if (!lfn_is_hashed_filename(short_name)) {
+    return lfn_parse_object_name(short_name, out);
+  }
+  // Get lfn_attr
+  string full_path = get_full_path(path, short_name);
+  char attr[PATH_MAX];
+  int r = chain_getxattr(full_path.c_str(), get_lfn_attr().c_str(), attr, sizeof(attr) - 1);
+  if (r < 0)
+    return -errno;
+  if (r < (int)sizeof(attr))
+    attr[r] = '\0';
+
+  string long_name(attr);
+  return lfn_parse_object_name(long_name, out);
+}
+
+bool LFNIndex::lfn_is_object(const string &short_name)
+{
+  return lfn_is_hashed_filename(short_name) || !lfn_is_subdir(short_name, 0);
+}
+
+bool LFNIndex::lfn_is_subdir(const string &name, string *demangled)
+{
+  if (name.substr(0, SUBDIR_PREFIX.size()) == SUBDIR_PREFIX) {
+    if (demangled)
+      *demangled = demangle_path_component(name);
+    return 1;
+  }
+  return 0;
+}
+
+static int parse_object(const char *s, ghobject_t& o)
+{
+  const char *hash = s + strlen(s) - 1;
+  while (*hash != '_' &&
+	 hash > s)
+    hash--;
+  const char *bar = hash - 1;
+  while (*bar != '_' &&
+	 bar > s)
+    bar--;
+  if (*bar == '_') {
+    char buf[bar-s + 1];
+    char *t = buf;
+    const char *i = s;
+    while (i < bar) {
+      if (*i == '\\') {
+	i++;
+	switch (*i) {
+	case '\\': *t++ = '\\'; break;
+	case '.': *t++ = '.'; break;
+	case 's': *t++ = '/'; break;
+	case 'd': {
+	  *t++ = 'D';
+	  *t++ = 'I';
+	  *t++ = 'R';
+	  *t++ = '_';
+	  break;
+	}
+	default: assert(0);
+	}
+      } else {
+	*t++ = *i;
+      }
+      i++;
+    }
+    *t = 0;
+    o.hobj.oid.name = string(buf, t-buf);
+    if (strncmp(bar+1, "head", 4) == 0)
+      o.hobj.snap = CEPH_NOSNAP;
+    else if (strncmp(bar+1, "snapdir", 7) == 0)
+      o.hobj.snap = CEPH_SNAPDIR;
+    else 
+      o.hobj.snap = strtoull(bar+1, NULL, 16);
+    sscanf(hash, "_%X", &o.hobj.hash);
+
+    return 1;
+  }
+  return 0;
+}
+
+bool LFNIndex::lfn_parse_object_name_keyless(const string &long_name, ghobject_t *out)
+{
+  bool r = parse_object(long_name.c_str(), *out);
+  int64_t pool = -1;
+  spg_t pg;
+  if (coll().is_pg_prefix(pg))
+    pool = (int64_t)pg.pgid.pool();
+  out->hobj.pool = pool;
+  if (!r) return r;
+  string temp = lfn_generate_object_name(*out);
+  return r;
+}
+
+static bool append_unescaped(string::const_iterator begin,
+			     string::const_iterator end, 
+			     string *out)
+{
+  for (string::const_iterator i = begin; i != end; ++i) {
+    if (*i == '\\') {
+      ++i;
+      if (*i == '\\')
+	out->append("\\");
+      else if (*i == 's')
+	out->append("/");
+      else if (*i == 'n')
+	(*out) += '\0';
+      else if (*i == 'u')
+	out->append("_");
+      else
+	return false;
+    } else {
+      out->append(i, i+1);
+    }
+  }
+  return true;
+}
+
+bool LFNIndex::lfn_parse_object_name_poolless(const string &long_name,
+					      ghobject_t *out)
+{
+  string name;
+  string key;
+  uint32_t hash;
+  snapid_t snap;
+
+  string::const_iterator current = long_name.begin();
+  if (*current == '\\') {
+    ++current;
+    if (current == long_name.end()) {
+      return false;
+    } else if (*current == 'd') {
+      name.append("DIR_");
+      ++current;
+    } else if (*current == '.') {
+      name.append(".");
+      ++current;
+    } else {
+      --current;
+    }
+  }
+
+  string::const_iterator end = current;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &name))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &key))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  string snap_str(current, end);
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end != long_name.end())
+    return false;
+  string hash_str(current, end);
+
+  if (snap_str == "head")
+    snap = CEPH_NOSNAP;
+  else if (snap_str == "snapdir")
+    snap = CEPH_SNAPDIR;
+  else
+    snap = strtoull(snap_str.c_str(), NULL, 16);
+  sscanf(hash_str.c_str(), "%X", &hash);
+
+
+  int64_t pool = -1;
+  spg_t pg;
+  if (coll().is_pg_prefix(pg))
+    pool = (int64_t)pg.pgid.pool();
+  (*out) = ghobject_t(hobject_t(name, key, snap, hash, pool, ""));
+  return true;
+}
+
+
+bool LFNIndex::lfn_parse_object_name(const string &long_name, ghobject_t *out)
+{
+  string name;
+  string key;
+  string ns;
+  uint32_t hash;
+  snapid_t snap;
+  uint64_t pool;
+  gen_t generation = ghobject_t::NO_GEN;
+  shard_t shard_id = ghobject_t::NO_SHARD;
+
+  if (index_version == HASH_INDEX_TAG)
+    return lfn_parse_object_name_keyless(long_name, out);
+  if (index_version == HASH_INDEX_TAG_2)
+    return lfn_parse_object_name_poolless(long_name, out);
+
+  string::const_iterator current = long_name.begin();
+  if (*current == '\\') {
+    ++current;
+    if (current == long_name.end()) {
+      return false;
+    } else if (*current == 'd') {
+      name.append("DIR_");
+      ++current;
+    } else if (*current == '.') {
+      name.append(".");
+      ++current;
+    } else {
+      --current;
+    }
+  }
+
+  string::const_iterator end = current;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &name))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &key))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  string snap_str(current, end);
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  string hash_str(current, end);
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  if (end == long_name.end())
+    return false;
+  if (!append_unescaped(current, end, &ns))
+    return false;
+
+  current = ++end;
+  for ( ; end != long_name.end() && *end != '_'; ++end) ;
+  string pstring(current, end);
+
+  // Optional generation/shard_id
+  string genstring, shardstring;
+  if (end != long_name.end()) {
+    current = ++end;
+    for ( ; end != long_name.end() && *end != '_'; ++end) ;
+    if (end == long_name.end())
+      return false;
+    genstring = string(current, end);
+
+    generation = (gen_t)strtoull(genstring.c_str(), NULL, 16);
+
+    current = ++end;
+    for ( ; end != long_name.end() && *end != '_'; ++end) ;
+    if (end != long_name.end())
+      return false;
+    shardstring = string(current, end);
+
+    shard_id = (shard_t)strtoul(shardstring.c_str(), NULL, 16);
+  }
+
+  if (snap_str == "head")
+    snap = CEPH_NOSNAP;
+  else if (snap_str == "snapdir")
+    snap = CEPH_SNAPDIR;
+  else
+    snap = strtoull(snap_str.c_str(), NULL, 16);
+  sscanf(hash_str.c_str(), "%X", &hash);
+
+  if (pstring == "none")
+    pool = (uint64_t)-1;
+  else
+    pool = strtoull(pstring.c_str(), NULL, 16);
+
+  (*out) = ghobject_t(hobject_t(name, key, snap, hash, (int64_t)pool, ns), generation, shard_id);
+  return true;
+}
+
+bool LFNIndex::lfn_is_hashed_filename(const string &name)
+{
+  if (name.size() < (unsigned)FILENAME_SHORT_LEN) {
+    return 0;
+  }
+  if (name.substr(name.size() - FILENAME_COOKIE.size(), FILENAME_COOKIE.size())
+      == FILENAME_COOKIE) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+bool LFNIndex::lfn_must_hash(const string &long_name)
+{
+  return (int)long_name.size() >= FILENAME_SHORT_LEN;
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+int LFNIndex::hash_filename(const char *filename, char *hash, int buf_len)
+{
+  if (buf_len < FILENAME_HASH_LEN + 1)
+    return -EINVAL;
+
+  char buf[FILENAME_LFN_DIGEST_SIZE];
+  char hex[FILENAME_LFN_DIGEST_SIZE * 2];
+
+  SHA1 h;
+  h.Update((const byte *)filename, strlen(filename));
+  h.Final((byte *)buf);
+
+  buf_to_hex((byte *)buf, (FILENAME_HASH_LEN + 1) / 2, hex);
+  strncpy(hash, hex, FILENAME_HASH_LEN);
+  hash[FILENAME_HASH_LEN] = '\0';
+  return 0;
+}
+
+void LFNIndex::build_filename(const char *old_filename, int i, char *filename, int len)
+{
+  char hash[FILENAME_HASH_LEN + 1];
+
+  assert(len >= FILENAME_SHORT_LEN + 4);
+
+  strncpy(filename, old_filename, FILENAME_PREFIX_LEN);
+  filename[FILENAME_PREFIX_LEN] = '\0';
+  if ((int)strlen(filename) < FILENAME_PREFIX_LEN)
+    return;
+  if (old_filename[FILENAME_PREFIX_LEN] == '\0')
+    return;
+
+  hash_filename(old_filename, hash, sizeof(hash));
+  int ofs = FILENAME_PREFIX_LEN;
+  while (1) {
+    int suffix_len = sprintf(filename + ofs, "_%s_%d_%s", hash, i, FILENAME_COOKIE.c_str());
+    if (ofs + suffix_len <= FILENAME_SHORT_LEN || !ofs)
+      break;
+    ofs--;
+  }
+}
+
+string LFNIndex::lfn_get_short_name(const ghobject_t &oid, int i)
+{
+  string long_name = lfn_generate_object_name(oid);
+  assert(lfn_must_hash(long_name));
+  char buf[FILENAME_SHORT_LEN + 4];
+  build_filename(long_name.c_str(), i, buf, sizeof(buf));
+  return string(buf);
+}
+
+const string &LFNIndex::get_base_path()
+{
+  return base_path;
+}
+
+string LFNIndex::get_full_path_subdir(const vector &rel)
+{
+  string retval = get_base_path();
+  for (vector::const_iterator i = rel.begin();
+       i != rel.end();
+       ++i) {
+    retval += "/";
+    retval += mangle_path_component(*i);
+  }
+  return retval;
+}
+
+string LFNIndex::get_full_path(const vector &rel, const string &name)
+{
+  return get_full_path_subdir(rel) + "/" + name;
+}
+
+string LFNIndex::mangle_path_component(const string &component)
+{
+  return SUBDIR_PREFIX + component;
+}
+
+string LFNIndex::demangle_path_component(const string &component)
+{
+  return component.substr(SUBDIR_PREFIX.size(), component.size() - SUBDIR_PREFIX.size());
+}
+
+int LFNIndex::decompose_full_path(const char *in, vector *out,
+				  ghobject_t *oid, string *shortname)
+{
+  const char *beginning = in + get_base_path().size();
+  const char *end = beginning;
+  while (1) {
+    end++;
+    beginning = end++;
+    for ( ; *end != '\0' && *end != '/'; ++end) ;
+    if (*end != '\0') {
+      out->push_back(demangle_path_component(string(beginning, end - beginning)));
+      continue;
+    } else {
+      break;
+    }
+  }
+  *shortname = string(beginning, end - beginning);
+  if (oid) {
+    int r = lfn_translate(*out, *shortname, oid);
+    if (r < 0)
+      return r;
+  }
+  return 0;
+}
+
+string LFNIndex::mangle_attr_name(const string &attr)
+{
+  return PHASH_ATTR_PREFIX + attr;
+}
diff --git a/ceph/src/os/LFNIndex.h b/ceph/src/os/LFNIndex.h
new file mode 100644
index 00000000..646e7267
--- /dev/null
+++ b/ceph/src/os/LFNIndex.h
@@ -0,0 +1,589 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef OS_LFNINDEX_H
+#define OS_LFNINDEX_H
+
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+#include 
+
+#include "osd/osd_types.h"
+#include "include/object.h"
+#include "common/ceph_crypto.h"
+
+#include "CollectionIndex.h"
+
+/** 
+ * LFNIndex also encapsulates logic for manipulating
+ * subdirectories of of a collection as well as the long filename
+ * logic.
+ *
+ * The protected methods provide machinery for derived classes to
+ * manipulate subdirectories and objects.
+ *  
+ * The virtual methods are to be overridden to provide the actual
+ * hashed layout.
+ *  
+ * User must call created when an object is created.
+ *  
+ * Syncronization: Calling code must ensure that there are no object
+ * creations or deletions during the lifetime of a Path object (except
+ * of an object at that path).
+ *
+ * Unless otherwise noted, methods which return an int return 0 on sucess
+ * and a negative error code on failure.
+ */
+#define WRAP_RETRY(x) {				\
+  bool failed = false;				\
+  int r = 0;					\
+  init_inject_failure();			\
+  while (1) {					\
+    try {					\
+      if (failed) {				\
+	r = cleanup();				\
+	assert(r == 0);				\
+      }						\
+      { x }					\
+      out:					\
+      complete_inject_failure();		\
+      return r;					\
+    } catch (RetryException) {			\
+      failed = true;				\
+    } catch (...) {				\
+      assert(0);				\
+    }						\
+  }						\
+  return -1;					\
+  }						\
+
+  
+
+class LFNIndex : public CollectionIndex {
+  /// Hash digest output size.
+  static const int FILENAME_LFN_DIGEST_SIZE = CEPH_CRYPTO_SHA1_DIGESTSIZE;
+  /// Length of filename hash.
+  static const int FILENAME_HASH_LEN = FILENAME_LFN_DIGEST_SIZE;
+  /// Max filename size.
+  static const int FILENAME_MAX_LEN = 4096;
+  /// Length of hashed filename.
+  static const int FILENAME_SHORT_LEN = 255;
+  /// Length of hashed filename prefix.
+  static const int FILENAME_PREFIX_LEN;
+  /// Length of hashed filename cookie.
+  static const int FILENAME_EXTRA = 4;
+  /// Lfn cookie value.
+  static const string FILENAME_COOKIE;
+  /// Name of LFN attribute for storing full name.
+  static const string LFN_ATTR;
+  /// Prefix for subdir index attributes.
+  static const string PHASH_ATTR_PREFIX;
+  /// Prefix for index subdirectories.
+  static const string SUBDIR_PREFIX;
+
+  /// Path to Index base.
+  const string base_path;
+  /// For reference counting the collection @see Path
+  ceph::weak_ptr self_ref;
+
+protected:
+  const uint32_t index_version;
+
+  /// true if retry injection is enabled
+  struct RetryException : public exception {};
+  bool error_injection_enabled;
+  bool error_injection_on;
+  double error_injection_probability;
+  uint64_t last_failure;
+  uint64_t current_failure;
+  void init_inject_failure() {
+    if (error_injection_on) {
+      error_injection_enabled = true;
+      last_failure = current_failure = 0;
+    }
+  }
+  void maybe_inject_failure();
+  void complete_inject_failure() {
+    error_injection_enabled = false;
+  }
+
+private:
+  string lfn_attribute, lfn_alt_attribute;
+  coll_t collection;
+
+public:
+  /// Constructor
+  LFNIndex(
+    coll_t collection,
+    const char *base_path, ///< [in] path to Index root
+    uint32_t index_version,
+    double _error_injection_probability=0)
+    : base_path(base_path),
+      index_version(index_version),
+      error_injection_enabled(false),
+      error_injection_on(_error_injection_probability != 0),
+      error_injection_probability(_error_injection_probability),
+      last_failure(0), current_failure(0),
+      collection(collection) {
+    if (index_version == HASH_INDEX_TAG) {
+      lfn_attribute = LFN_ATTR;
+    } else {
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%d", index_version);
+      lfn_attribute = LFN_ATTR + string(buf);
+      lfn_alt_attribute = LFN_ATTR + string(buf) + "-alt";
+   }
+  }
+
+  coll_t coll() const { return collection; }
+
+  /// Virtual destructor
+  virtual ~LFNIndex() {}
+
+  /// @see CollectionIndex
+  void set_ref(ceph::shared_ptr ref);
+
+  /// @see CollectionIndex
+  int init();
+
+  /// @see CollectionIndex
+  int cleanup() = 0;
+
+  /// @see CollectionIndex
+  int created(
+    const ghobject_t &oid,
+    const char *path
+    );
+
+  /// @see CollectionIndex
+  int unlink(
+    const ghobject_t &oid
+    );
+
+  /// @see CollectionIndex
+  int lookup(
+    const ghobject_t &oid,
+    IndexedPath *path,
+    int *exist
+    );
+
+  /// @see CollectionIndex
+  int collection_list(
+    vector *ls
+    );
+
+  /// @see CollectionIndex
+  int collection_list_partial(
+    const ghobject_t &start,
+    int min_count,
+    int max_count,
+    snapid_t seq,
+    vector *ls,
+    ghobject_t *next
+    );
+
+  virtual int _split(
+    uint32_t match,                             //< [in] value to match
+    uint32_t bits,                              //< [in] bits to check
+    ceph::shared_ptr dest  //< [in] destination index
+    ) = 0;
+  
+  /// @see CollectionIndex
+  int split(
+    uint32_t match,
+    uint32_t bits,
+    ceph::shared_ptr dest
+    ) {
+    WRAP_RETRY(
+      r = _split(match, bits, dest);
+      goto out;
+      );
+  }
+
+
+protected:
+  virtual int _init() = 0;
+
+  /// Will be called upon object creation
+  virtual int _created(
+    const vector &path, ///< [in] Path to subdir.
+    const ghobject_t &oid,      ///< [in] Object created.
+    const string &mangled_name  ///< [in] Mangled filename.
+    ) = 0;
+
+  /// Will be called to remove an object
+  virtual int _remove(
+    const vector &path,     ///< [in] Path to subdir.
+    const ghobject_t &oid,          ///< [in] Object to remove.
+    const string &mangled_name	    ///< [in] Mangled filename.
+    ) = 0;
+
+  /// Return the path and mangled_name for oid.
+  virtual int _lookup(
+    const ghobject_t &oid,///< [in] Object for lookup.
+    vector *path, ///< [out] Path to the object.
+    string *mangled_name, ///< [out] Mangled filename.
+    int *exists		  ///< [out] True if the object exists.
+    ) = 0;
+
+  /**
+   * List contents of the collection, must be implemented by derived class.
+   *
+   * @param [out] seq Snapid to list.
+   * @param [in] max_count Max number to list (0 for no limit).
+   * @param [out] ls Container for listed objects.
+   * @param [in,out] last List handle.  0 for beginning.  Passing the same
+   * cookie location will cause the next max_count to be listed.
+   * @return Error code.  0 on success.
+   */
+  /// List contents of collection.
+  virtual int _collection_list(
+    vector *ls ///< [out] Listed objects.
+    ) = 0;
+
+  /// @see CollectionIndex
+  virtual int _collection_list_partial(
+    const ghobject_t &start,
+    int min_count,
+    int max_count,
+    snapid_t seq,
+    vector *ls,
+    ghobject_t *next
+    ) = 0;
+
+protected:
+
+  /* Non-virtual utility methods */
+
+  /// Sync a subdirectory
+  int fsync_dir(
+    const vector &path ///< [in] Path to sync
+    ); ///< @return Error Code, 0 on success
+
+  /// Link an object from from into to
+  int link_object(
+    const vector &from,   ///< [in] Source subdirectory.
+    const vector &to,     ///< [in] Dest subdirectory.
+    const ghobject_t &oid,        ///< [in] Object to move.
+    const string &from_short_name ///< [in] Mangled filename of oid.
+    ); ///< @return Error Code, 0 on success
+
+  /**
+   * Efficiently remove objects from a subdirectory
+   *
+   * remove_object invalidates mangled names in the directory requiring
+   * the mangled name of each additional object to be looked up a second
+   * time.  remove_objects removes the need for additional lookups
+   *
+   * @param [in] dir Directory from which to remove.
+   * @param [in] map of objects to remove to mangle names
+   * @param [in,out] map of filenames to objects
+   * @return Error Code, 0 on success.
+   */
+  int remove_objects(
+    const vector &dir,             
+    const map &to_remove,
+    map *remaining
+    );
+	
+
+  /** 
+   * Moves contents of from into to.
+   *
+   * Invalidates mangled names in to.  If interupted, all objects will be
+   * present in to before objects are removed from from.  Ignores EEXIST 
+   * while linking into to.
+   * @return Error Code, 0 on success
+   */
+  int move_objects(
+    const vector &from, ///< [in] Source subdirectory.
+    const vector &to    ///< [in] Dest subdirectory.
+    );
+
+  /** 
+   * Remove an object from from.
+   *
+   * Invalidates mangled names in from.
+   * @return Error Code, 0 on success
+   */
+  int remove_object(
+    const vector &from,  ///< [in] Directory from which to remove.
+    const ghobject_t &to_remove   ///< [in] Object to remove.
+    );
+
+  /**
+   * Gets the filename corresponding to oid in from.
+   * 
+   * The filename may differ between subdirectories.  Furthermore,
+   * file creations ore removals in from may invalidate the name.
+   * @return Error code on failure, 0 on success
+   */
+  int get_mangled_name(
+    const vector &from, ///< [in] Subdirectory
+    const ghobject_t &oid,	///< [in] Object
+    string *mangled_name,	///< [out] Filename
+    int *exists			///< [out] 1 if the file exists, else 0
+    );
+
+  /// do move subdir from from to dest
+  static int move_subdir(
+    LFNIndex &from,             ///< [in] from index
+    LFNIndex &dest,             ///< [in] to index
+    const vector &path, ///< [in] path containing dir
+    string dir                  ///< [in] dir to move
+    );
+
+  /// do move object from from to dest
+  static int move_object(
+    LFNIndex &from,             ///< [in] from index
+    LFNIndex &dest,             ///< [in] to index
+    const vector &path, ///< [in] path to split
+    const pair &obj ///< [in] obj to move
+    );
+
+  /**
+   * Lists objects in to_list.
+   *
+   * @param [in] to_list Directory to list.
+   * @param [in] max_objects Max number to list.
+   * @param [in,out] handle Cookie for continuing the listing.
+   * Initialize to zero to start at the beginning of the directory.
+   * @param [out] out Mapping of listed object filenames to objects.
+   * @return Error code on failure, 0 on success
+   */
+  int list_objects(
+    const vector &to_list,
+    int max_objects,
+    long *handle,
+    map *out
+    );
+
+  /// Lists subdirectories.
+  int list_subdirs(
+    const vector &to_list, ///< [in] Directory to list.
+    set *out		   ///< [out] Subdirectories listed. 
+    );
+
+  /// Create subdirectory.
+  int create_path(
+    const vector &to_create ///< [in] Subdirectory to create.
+    );
+
+  /// Remove subdirectory.
+  int remove_path(
+    const vector &to_remove ///< [in] Subdirectory to remove.
+    );
+
+  /// Check whether to_check exists.
+  int path_exists(
+    const vector &to_check, ///< [in] Subdirectory to check.
+    int *exists			    ///< [out] 1 if it exists, 0 else
+    );
+
+  /// Save attr_value to attr_name attribute on path.
+  int add_attr_path(
+    const vector &path, ///< [in] Path to modify.
+    const string &attr_name, 	///< [in] Name of attribute.
+    bufferlist &attr_value	///< [in] Value to save.
+    );
+
+  /// Read into attr_value atribute attr_name on path.
+  int get_attr_path(
+    const vector &path, ///< [in] Path to read.
+    const string &attr_name, 	///< [in] Attribute to read. 
+    bufferlist &attr_value	///< [out] Attribute value read.
+    );
+
+  /// Remove attr from path
+  int remove_attr_path(
+    const vector &path, ///< [in] path from which to remove attr
+    const string &attr_name	///< [in] attr to remove
+    ); ///< @return Error code, 0 on success
+
+private:
+  /* lfn translation functions */
+
+  /**
+   * Gets the version specific lfn attribute tag
+   */
+  const string &get_lfn_attr() const {
+    return lfn_attribute;
+  }
+  const string &get_alt_lfn_attr() const {
+    return lfn_alt_attribute;
+  }
+
+  /**
+   * Gets the filename corresponsing to oid in path.
+   *
+   * @param [in] path Path in which to get filename for oid.
+   * @param [in] oid Object for which to get filename.
+   * @param [out] mangled_name Filename for oid, pass NULL if not needed.
+   * @param [out] full_path Fullpath for oid, pass NULL if not needed.
+   * @param [out] exists 1 if the file exists, 0 otherwise, pass NULL if 
+   * not needed
+   * @return Error Code, 0 on success.
+   */
+  int lfn_get_name(
+    const vector &path,
+    const ghobject_t &oid,
+    string *mangled_name,
+    string *full_path,
+    int *exists
+    );
+
+  /// Adjusts path contents when oid is created at name mangled_name.
+  int lfn_created(
+    const vector &path, ///< [in] Path to adjust.
+    const ghobject_t &oid,	///< [in] Object created.
+    const string &mangled_name  ///< [in] Filename of created object.
+    );
+
+  /// Removes oid from path while adjusting path contents
+  int lfn_unlink(
+    const vector &path, ///< [in] Path containing oid.
+    const ghobject_t &oid,	///< [in] Object to remove.
+    const string &mangled_name	///< [in] Filename of object to remove.
+    );
+
+  ///Transate a file into and ghobject_t.
+  int lfn_translate(
+    const vector &path, ///< [in] Path containing the file.
+    const string &short_name,	///< [in] Filename to translate. 
+    ghobject_t *out		///< [out] Object found.
+    ); ///< @return Negative error code on error, 0 if not an object, 1 else
+
+  /* manglers/demanglers */
+  /// Filters object filenames
+  bool lfn_is_object(
+    const string &short_name ///< [in] Filename to check
+    ); ///< True if short_name is an object, false otherwise
+
+  /// Filters subdir filenames
+  bool lfn_is_subdir(
+    const string &short_name, ///< [in] Filename to check.
+    string *demangled_name    ///< [out] Demangled subdir name.
+    ); ///< @return True if short_name is a subdir, false otherwise
+
+  /// Generate object name
+  string lfn_generate_object_name_keyless(
+    const ghobject_t &oid ///< [in] Object for which to generate.
+    ); ///< @return Generated object name.
+
+  /// Generate object name
+  string lfn_generate_object_name_poolless(
+    const ghobject_t &oid ///< [in] Object for which to generate.
+    ); ///< @return Generated object name.
+
+  /// Generate object name
+  string lfn_generate_object_name(
+    const ghobject_t &oid ///< [in] Object for which to generate.
+    ); ///< @return Generated object name.
+
+  /// Parse object name
+  bool lfn_parse_object_name_keyless(
+    const string &long_name, ///< [in] Name to parse
+    ghobject_t *out	     ///< [out] Resulting Object
+    ); ///< @return True if successfull, False otherwise.
+
+  /// Parse object name
+  bool lfn_parse_object_name_poolless(
+    const string &long_name, ///< [in] Name to parse
+    ghobject_t *out	     ///< [out] Resulting Object
+    ); ///< @return True if successfull, False otherwise.
+
+  /// Parse object name
+  bool lfn_parse_object_name(
+    const string &long_name, ///< [in] Name to parse
+    ghobject_t *out	     ///< [out] Resulting Object
+    ); ///< @return True if successfull, False otherwise.
+
+  /// Checks whether short_name is a hashed filename.
+  bool lfn_is_hashed_filename(
+    const string &short_name ///< [in] Name to check.
+    ); ///< @return True if short_name is hashed, False otherwise.
+
+  /// Checks whether long_name must be hashed.
+  bool lfn_must_hash(
+    const string &long_name ///< [in] Name to check.
+    ); ///< @return True if long_name must be hashed, False otherwise.
+
+  /// Generate hashed name.
+  string lfn_get_short_name(
+    const ghobject_t &oid, ///< [in] Object for which to generate.
+    int i		   ///< [in] Index of hashed name to generate.
+    ); ///< @return Hashed filename.
+
+  /* other common methods */
+  /// Gets the base path
+  const string &get_base_path(); ///< @return Index base_path
+
+  /// Get full path the subdir
+  string get_full_path_subdir(
+    const vector &rel ///< [in] The subdir.
+    ); ///< @return Full path to rel.
+
+  /// Get full path to object
+  string get_full_path(
+    const vector &rel, ///< [in] Path to object.
+    const string &name	       ///< [in] Filename of object.
+    ); ///< @return Fullpath to object at name in rel.
+
+  /// Get mangled path component
+  string mangle_path_component(
+    const string &component ///< [in] Component to mangle
+    ); /// @return Mangled component
+
+  /// Demangle component
+  string demangle_path_component(
+    const string &component ///< [in] Subdir name to demangle
+    ); ///< @return Demangled path component.
+
+  /// Decompose full path into object name and filename.
+  int decompose_full_path(
+    const char *in,      ///< [in] Full path to object.
+    vector *out, ///< [out] Path to object at in.
+    ghobject_t *oid,	 ///< [out] Object at in.
+    string *shortname	 ///< [out] Filename of object at in.
+    ); ///< @return Error Code, 0 on success.
+
+  /// Mangle attribute name
+  string mangle_attr_name(
+    const string &attr ///< [in] Attribute to mangle.
+    ); ///< @return Mangled attribute name.
+
+  /// Builds hashed filename
+  void build_filename(
+    const char *old_filename, ///< [in] Filename to convert.
+    int i,		      ///< [in] Index of hash.
+    char *filename,	      ///< [out] Resulting filename.
+    int len		      ///< [in] Size of buffer for filename
+    ); ///< @return Error Code, 0 on success
+
+  /// Get hash of filename
+  int hash_filename(
+    const char *filename, ///< [in] Filename to hash.
+    char *hash,		  ///< [out] Hash of filename.
+    int len		  ///< [in] Size of hash buffer.
+    ); ///< @return Error Code, 0 on success.
+
+  friend class TestWrapLFNIndex;
+};
+typedef LFNIndex::IndexedPath IndexedPath;
+
+#endif
diff --git a/ceph/src/os/LevelDBStore.cc b/ceph/src/os/LevelDBStore.cc
new file mode 100644
index 00000000..326862f8
--- /dev/null
+++ b/ceph/src/os/LevelDBStore.cc
@@ -0,0 +1,288 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "LevelDBStore.h"
+
+#include 
+#include 
+#include 
+#include "include/memory.h"
+#include 
+using std::string;
+#include "common/perf_counters.h"
+
+int LevelDBStore::init()
+{
+  // init defaults.  caller can override these if they want
+  // prior to calling open.
+  options.write_buffer_size = g_conf->leveldb_write_buffer_size;
+  options.cache_size = g_conf->leveldb_cache_size;
+  options.block_size = g_conf->leveldb_block_size;
+  options.bloom_size = g_conf->leveldb_bloom_size;
+  options.compression_enabled = g_conf->leveldb_compression;
+  options.paranoid_checks = g_conf->leveldb_paranoid;
+  options.max_open_files = g_conf->leveldb_max_open_files;
+  options.log_file = g_conf->leveldb_log;
+  return 0;
+}
+
+int LevelDBStore::do_open(ostream &out, bool create_if_missing)
+{
+  leveldb::Options ldoptions;
+
+  if (options.write_buffer_size)
+    ldoptions.write_buffer_size = options.write_buffer_size;
+  if (options.max_open_files)
+    ldoptions.max_open_files = options.max_open_files;
+  if (options.cache_size) {
+    leveldb::Cache *_db_cache = leveldb::NewLRUCache(options.cache_size);
+    db_cache.reset(_db_cache);
+    ldoptions.block_cache = db_cache.get();
+  }
+  if (options.block_size)
+    ldoptions.block_size = options.block_size;
+  if (options.bloom_size) {
+#ifdef HAVE_LEVELDB_FILTER_POLICY
+    const leveldb::FilterPolicy *_filterpolicy =
+	leveldb::NewBloomFilterPolicy(options.bloom_size);
+    filterpolicy.reset(_filterpolicy);
+    ldoptions.filter_policy = filterpolicy.get();
+#else
+    assert(0 == "bloom size set but installed leveldb doesn't support bloom filters");
+#endif
+  }
+  if (options.compression_enabled)
+    ldoptions.compression = leveldb::kSnappyCompression;
+  else
+    ldoptions.compression = leveldb::kNoCompression;
+  if (options.block_restart_interval)
+    ldoptions.block_restart_interval = options.block_restart_interval;
+
+  ldoptions.error_if_exists = options.error_if_exists;
+  ldoptions.paranoid_checks = options.paranoid_checks;
+  ldoptions.create_if_missing = create_if_missing;
+
+  if (options.log_file.length()) {
+    leveldb::Env *env = leveldb::Env::Default();
+    env->NewLogger(options.log_file, &ldoptions.info_log);
+  }
+
+  leveldb::DB *_db;
+  leveldb::Status status = leveldb::DB::Open(ldoptions, path, &_db);
+  db.reset(_db);
+  if (!status.ok()) {
+    out << status.ToString() << std::endl;
+    return -EINVAL;
+  }
+
+  if (g_conf->leveldb_compact_on_mount) {
+    derr << "Compacting leveldb store..." << dendl;
+    compact();
+    derr << "Finished compacting leveldb store" << dendl;
+  }
+
+  PerfCountersBuilder plb(g_ceph_context, "leveldb", l_leveldb_first, l_leveldb_last);
+  plb.add_u64_counter(l_leveldb_gets, "leveldb_get");
+  plb.add_u64_counter(l_leveldb_txns, "leveldb_transaction");
+  plb.add_u64_counter(l_leveldb_compact, "leveldb_compact");
+  plb.add_u64_counter(l_leveldb_compact_range, "leveldb_compact_range");
+  plb.add_u64_counter(l_leveldb_compact_queue_merge, "leveldb_compact_queue_merge");
+  plb.add_u64(l_leveldb_compact_queue_len, "leveldb_compact_queue_len");
+  logger = plb.create_perf_counters();
+  cct->get_perfcounters_collection()->add(logger);
+  return 0;
+}
+
+LevelDBStore::~LevelDBStore()
+{
+  close();
+  delete logger;
+
+  // Ensure db is destroyed before dependent db_cache and filterpolicy
+  db.reset();
+}
+
+void LevelDBStore::close()
+{
+  // stop compaction thread
+  compact_queue_lock.Lock();
+  if (compact_thread.is_started()) {
+    compact_queue_stop = true;
+    compact_queue_cond.Signal();
+    compact_queue_lock.Unlock();
+    compact_thread.join();
+  } else {
+    compact_queue_lock.Unlock();
+  }
+
+  if (logger)
+    cct->get_perfcounters_collection()->remove(logger);
+}
+
+int LevelDBStore::submit_transaction(KeyValueDB::Transaction t)
+{
+  LevelDBTransactionImpl * _t =
+    static_cast(t.get());
+  leveldb::Status s = db->Write(leveldb::WriteOptions(), &(_t->bat));
+  logger->inc(l_leveldb_txns);
+  return s.ok() ? 0 : -1;
+}
+
+int LevelDBStore::submit_transaction_sync(KeyValueDB::Transaction t)
+{
+  LevelDBTransactionImpl * _t =
+    static_cast(t.get());
+  leveldb::WriteOptions options;
+  options.sync = true;
+  leveldb::Status s = db->Write(options, &(_t->bat));
+  logger->inc(l_leveldb_txns);
+  return s.ok() ? 0 : -1;
+}
+
+void LevelDBStore::LevelDBTransactionImpl::set(
+  const string &prefix,
+  const string &k,
+  const bufferlist &to_set_bl)
+{
+  buffers.push_back(to_set_bl);
+  buffers.rbegin()->rebuild();
+  bufferlist &bl = *(buffers.rbegin());
+  string key = combine_strings(prefix, k);
+  keys.push_back(key);
+  bat.Delete(leveldb::Slice(*(keys.rbegin())));
+  bat.Put(leveldb::Slice(*(keys.rbegin())),
+	  leveldb::Slice(bl.c_str(), bl.length()));
+}
+
+void LevelDBStore::LevelDBTransactionImpl::rmkey(const string &prefix,
+					         const string &k)
+{
+  string key = combine_strings(prefix, k);
+  keys.push_back(key);
+  bat.Delete(leveldb::Slice(*(keys.rbegin())));
+}
+
+void LevelDBStore::LevelDBTransactionImpl::rmkeys_by_prefix(const string &prefix)
+{
+  KeyValueDB::Iterator it = db->get_iterator(prefix);
+  for (it->seek_to_first();
+       it->valid();
+       it->next()) {
+    string key = combine_strings(prefix, it->key());
+    keys.push_back(key);
+    bat.Delete(*(keys.rbegin()));
+  }
+}
+
+int LevelDBStore::get(
+    const string &prefix,
+    const std::set &keys,
+    std::map *out)
+{
+  KeyValueDB::Iterator it = get_iterator(prefix);
+  for (std::set::const_iterator i = keys.begin();
+       i != keys.end();
+       ++i) {
+    it->lower_bound(*i);
+    if (it->valid() && it->key() == *i) {
+      out->insert(make_pair(*i, it->value()));
+    } else if (!it->valid())
+      break;
+  }
+  logger->inc(l_leveldb_gets);
+  return 0;
+}
+
+string LevelDBStore::combine_strings(const string &prefix, const string &value)
+{
+  string out = prefix;
+  out.push_back(0);
+  out.append(value);
+  return out;
+}
+
+bufferlist LevelDBStore::to_bufferlist(leveldb::Slice in)
+{
+  bufferlist bl;
+  bl.append(bufferptr(in.data(), in.size()));
+  return bl;
+}
+
+int LevelDBStore::split_key(leveldb::Slice in, string *prefix, string *key)
+{
+  string in_prefix = in.ToString();
+  size_t prefix_len = in_prefix.find('\0');
+  if (prefix_len >= in_prefix.size())
+    return -EINVAL;
+
+  if (prefix)
+    *prefix = string(in_prefix, 0, prefix_len);
+  if (key)
+    *key= string(in_prefix, prefix_len + 1);
+  return 0;
+}
+
+void LevelDBStore::compact()
+{
+  logger->inc(l_leveldb_compact);
+  db->CompactRange(NULL, NULL);
+}
+
+
+void LevelDBStore::compact_thread_entry()
+{
+  compact_queue_lock.Lock();
+  while (!compact_queue_stop) {
+    while (!compact_queue.empty()) {
+      pair range = compact_queue.front();
+      compact_queue.pop_front();
+      logger->set(l_leveldb_compact_queue_len, compact_queue.size());
+      compact_queue_lock.Unlock();
+      logger->inc(l_leveldb_compact_range);
+      compact_range(range.first, range.second);
+      compact_queue_lock.Lock();
+      continue;
+    }
+    compact_queue_cond.Wait(compact_queue_lock);
+  }
+  compact_queue_lock.Unlock();
+}
+
+void LevelDBStore::compact_range_async(const string& start, const string& end)
+{
+  Mutex::Locker l(compact_queue_lock);
+
+  // try to merge adjacent ranges.  this is O(n), but the queue should
+  // be short.  note that we do not cover all overlap cases and merge
+  // opportunities here, but we capture the ones we currently need.
+  list< pair >::iterator p = compact_queue.begin();
+  while (p != compact_queue.end()) {
+    if (p->first == start && p->second == end) {
+      // dup; no-op
+      return;
+    }
+    if (p->first <= end && p->first > start) {
+      // merge with existing range to the right
+      compact_queue.push_back(make_pair(start, p->second));
+      compact_queue.erase(p);
+      logger->inc(l_leveldb_compact_queue_merge);
+      break;
+    }
+    if (p->second >= start && p->second < end) {
+      // merge with existing range to the left
+      compact_queue.push_back(make_pair(p->first, end));
+      compact_queue.erase(p);
+      logger->inc(l_leveldb_compact_queue_merge);
+      break;
+    }
+    ++p;
+  }
+  if (p == compact_queue.end()) {
+    // no merge, new entry.
+    compact_queue.push_back(make_pair(start, end));
+    logger->set(l_leveldb_compact_queue_len, compact_queue.size());
+  }
+  compact_queue_cond.Signal();
+  if (!compact_thread.is_started()) {
+    compact_thread.create();
+  }
+}
diff --git a/ceph/src/os/LevelDBStore.h b/ceph/src/os/LevelDBStore.h
new file mode 100644
index 00000000..26e7bbe7
--- /dev/null
+++ b/ceph/src/os/LevelDBStore.h
@@ -0,0 +1,399 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef LEVEL_DB_STORE_H
+#define LEVEL_DB_STORE_H
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "KeyValueDB.h"
+#include 
+#include 
+#include 
+#include "include/memory.h"
+#include 
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "leveldb/write_batch.h"
+#include "leveldb/slice.h"
+#include "leveldb/cache.h"
+#ifdef HAVE_LEVELDB_FILTER_POLICY
+#include "leveldb/filter_policy.h"
+#endif
+
+#include 
+#include "common/errno.h"
+#include "common/dout.h"
+#include "include/assert.h"
+#include "common/Formatter.h"
+
+#include "common/ceph_context.h"
+
+class PerfCounters;
+
+enum {
+  l_leveldb_first = 34300,
+  l_leveldb_gets,
+  l_leveldb_txns,
+  l_leveldb_compact,
+  l_leveldb_compact_range,
+  l_leveldb_compact_queue_merge,
+  l_leveldb_compact_queue_len,
+  l_leveldb_last,
+};
+
+/**
+ * Uses LevelDB to implement the KeyValueDB interface
+ */
+class LevelDBStore : public KeyValueDB {
+  CephContext *cct;
+  PerfCounters *logger;
+  string path;
+  boost::scoped_ptr db_cache;
+#ifdef HAVE_LEVELDB_FILTER_POLICY
+  boost::scoped_ptr filterpolicy;
+#endif
+  boost::scoped_ptr db;
+
+  int do_open(ostream &out, bool create_if_missing);
+
+  // manage async compactions
+  Mutex compact_queue_lock;
+  Cond compact_queue_cond;
+  list< pair > compact_queue;
+  bool compact_queue_stop;
+  class CompactThread : public Thread {
+    LevelDBStore *db;
+  public:
+    CompactThread(LevelDBStore *d) : db(d) {}
+    void *entry() {
+      db->compact_thread_entry();
+      return NULL;
+    }
+    friend class LevelDBStore;
+  } compact_thread;
+
+  void compact_thread_entry();
+
+  void compact_range(const string& start, const string& end) {
+    leveldb::Slice cstart(start);
+    leveldb::Slice cend(end);
+    db->CompactRange(&cstart, &cend);
+  }
+  void compact_range_async(const string& start, const string& end);
+
+public:
+  /// compact the underlying leveldb store
+  void compact();
+
+  /// compact leveldb for all keys with a given prefix
+  void compact_prefix(const string& prefix) {
+    compact_range(prefix, past_prefix(prefix));
+  }
+  void compact_prefix_async(const string& prefix) {
+    compact_range_async(prefix, past_prefix(prefix));
+  }
+
+  void compact_range(const string& prefix, const string& start, const string& end) {
+    compact_range(combine_strings(prefix, start), combine_strings(prefix, end));
+  }
+  void compact_range_async(const string& prefix, const string& start, const string& end) {
+    compact_range_async(combine_strings(prefix, start), combine_strings(prefix, end));
+  }
+
+  /**
+   * options_t: Holds options which are minimally interpreted
+   * on initialization and then passed through to LevelDB.
+   * We transform a couple of these into actual LevelDB
+   * structures, but the rest are simply passed through unchanged. See
+   * leveldb/options.h for more precise details on each.
+   *
+   * Set them after constructing the LevelDBStore, but before calling
+   * open() or create_and_open().
+   */
+  struct options_t {
+    uint64_t write_buffer_size; /// in-memory write buffer size
+    int max_open_files; /// maximum number of files LevelDB can open at once
+    uint64_t cache_size; /// size of extra decompressed cache to use
+    uint64_t block_size; /// user data per block
+    int bloom_size; /// number of bits per entry to put in a bloom filter
+    bool compression_enabled; /// whether to use libsnappy compression or not
+
+    // don't change these ones. No, seriously
+    int block_restart_interval;
+    bool error_if_exists;
+    bool paranoid_checks;
+
+    string log_file;
+
+    options_t() :
+      write_buffer_size(0), //< 0 means default
+      max_open_files(0), //< 0 means default
+      cache_size(0), //< 0 means no cache (default)
+      block_size(0), //< 0 means default
+      bloom_size(0), //< 0 means no bloom filter (default)
+      compression_enabled(true), //< set to false for no compression
+      block_restart_interval(0), //< 0 means default
+      error_if_exists(false), //< set to true if you want to check nonexistence
+      paranoid_checks(false) //< set to true if you want paranoid checks
+    {}
+  } options;
+
+  LevelDBStore(CephContext *c, const string &path) :
+    cct(c),
+    logger(NULL),
+    path(path),
+    db_cache(NULL),
+#ifdef HAVE_LEVELDB_FILTER_POLICY
+    filterpolicy(NULL),
+#endif
+    compact_queue_lock("LevelDBStore::compact_thread_lock"),
+    compact_queue_stop(false),
+    compact_thread(this),
+    options()
+  {}
+
+  ~LevelDBStore();
+
+  int init();
+
+  /// Opens underlying db
+  int open(ostream &out) {
+    return do_open(out, false);
+  }
+  /// Creates underlying db if missing and opens it
+  int create_and_open(ostream &out) {
+    return do_open(out, true);
+  }
+
+  void close();
+
+  class LevelDBTransactionImpl : public KeyValueDB::TransactionImpl {
+  public:
+    leveldb::WriteBatch bat;
+    list buffers;
+    list keys;
+    LevelDBStore *db;
+
+    LevelDBTransactionImpl(LevelDBStore *db) : db(db) {}
+    void set(
+      const string &prefix,
+      const string &k,
+      const bufferlist &bl);
+    void rmkey(
+      const string &prefix,
+      const string &k);
+    void rmkeys_by_prefix(
+      const string &prefix
+      );
+  };
+
+  KeyValueDB::Transaction get_transaction() {
+    return ceph::shared_ptr< LevelDBTransactionImpl >(
+      new LevelDBTransactionImpl(this));
+  }
+
+  int submit_transaction(KeyValueDB::Transaction t);
+  int submit_transaction_sync(KeyValueDB::Transaction t);
+  int get(
+    const string &prefix,
+    const std::set &key,
+    std::map *out
+    );
+
+  class LevelDBWholeSpaceIteratorImpl :
+    public KeyValueDB::WholeSpaceIteratorImpl {
+  protected:
+    boost::scoped_ptr dbiter;
+  public:
+    LevelDBWholeSpaceIteratorImpl(leveldb::Iterator *iter) :
+      dbiter(iter) { }
+    virtual ~LevelDBWholeSpaceIteratorImpl() { }
+
+    int seek_to_first() {
+      dbiter->SeekToFirst();
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    int seek_to_first(const string &prefix) {
+      leveldb::Slice slice_prefix(prefix);
+      dbiter->Seek(slice_prefix);
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    int seek_to_last() {
+      dbiter->SeekToLast();
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    int seek_to_last(const string &prefix) {
+      string limit = past_prefix(prefix);
+      leveldb::Slice slice_limit(limit);
+      dbiter->Seek(slice_limit);
+
+      if (!dbiter->Valid()) {
+        dbiter->SeekToLast();
+      } else {
+        dbiter->Prev();
+      }
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    int upper_bound(const string &prefix, const string &after) {
+      lower_bound(prefix, after);
+      if (valid()) {
+	pair key = raw_key();
+	if (key.first == prefix && key.second == after)
+	  next();
+      }
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    int lower_bound(const string &prefix, const string &to) {
+      string bound = combine_strings(prefix, to);
+      leveldb::Slice slice_bound(bound);
+      dbiter->Seek(slice_bound);
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    bool valid() {
+      return dbiter->Valid();
+    }
+    int next() {
+      if (valid())
+	dbiter->Next();
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    int prev() {
+      if (valid())
+	dbiter->Prev();
+      return dbiter->status().ok() ? 0 : -1;
+    }
+    string key() {
+      string out_key;
+      split_key(dbiter->key(), 0, &out_key);
+      return out_key;
+    }
+    pair raw_key() {
+      string prefix, key;
+      split_key(dbiter->key(), &prefix, &key);
+      return make_pair(prefix, key);
+    }
+    bufferlist value() {
+      return to_bufferlist(dbiter->value());
+    }
+    int status() {
+      return dbiter->status().ok() ? 0 : -1;
+    }
+  };
+
+  class LevelDBSnapshotIteratorImpl : public LevelDBWholeSpaceIteratorImpl {
+    leveldb::DB *db;
+    const leveldb::Snapshot *snapshot;
+  public:
+    LevelDBSnapshotIteratorImpl(leveldb::DB *db, const leveldb::Snapshot *s,
+				leveldb::Iterator *iter) :
+      LevelDBWholeSpaceIteratorImpl(iter), db(db), snapshot(s) { }
+
+    ~LevelDBSnapshotIteratorImpl() {
+      assert(snapshot != NULL);
+      db->ReleaseSnapshot(snapshot);
+    }
+  };
+
+  /// Utility
+  static string combine_strings(const string &prefix, const string &value);
+  static int split_key(leveldb::Slice in, string *prefix, string *key);
+  static bufferlist to_bufferlist(leveldb::Slice in);
+  static bool in_prefix(const string &prefix, leveldb::Slice key) {
+    return (key.compare(leveldb::Slice(past_prefix(prefix))) < 0) &&
+      (key.compare(leveldb::Slice(prefix)) > 0);
+  }
+  static string past_prefix(const string &prefix) {
+    string limit = prefix;
+    limit.push_back(1);
+    return limit;
+  }
+
+  virtual uint64_t get_estimated_size(map &extra) {
+    DIR *store_dir = opendir(path.c_str());
+    if (!store_dir) {
+      lderr(cct) << __func__ << " something happened opening the store: "
+                 << cpp_strerror(errno) << dendl;
+      return 0;
+    }
+
+    uint64_t total_size = 0;
+    uint64_t sst_size = 0;
+    uint64_t log_size = 0;
+    uint64_t misc_size = 0;
+
+    struct dirent *entry = NULL;
+    while ((entry = readdir(store_dir)) != NULL) {
+      string n(entry->d_name);
+
+      if (n == "." || n == "..")
+        continue;
+
+      string fpath = path + '/' + n;
+      struct stat s;
+      int err = stat(fpath.c_str(), &s);
+      if (err < 0)
+	err = -errno;
+      // we may race against leveldb while reading files; this should only
+      // happen when those files are being updated, data is being shuffled
+      // and files get removed, in which case there's not much of a problem
+      // as we'll get to them next time around.
+      if ((err < 0) && (err != -ENOENT)) {
+        lderr(cct) << __func__ << " error obtaining stats for " << fpath
+                   << ": " << cpp_strerror(err) << dendl;
+        goto err;
+      }
+
+      size_t pos = n.find_last_of('.');
+      if (pos == string::npos) {
+        misc_size += s.st_size;
+        continue;
+      }
+
+      string ext = n.substr(pos+1);
+      if (ext == "sst") {
+        sst_size += s.st_size;
+      } else if (ext == "log") {
+        log_size += s.st_size;
+      } else {
+        misc_size += s.st_size;
+      }
+    }
+
+    total_size = sst_size + log_size + misc_size;
+
+    extra["sst"] = sst_size;
+    extra["log"] = log_size;
+    extra["misc"] = misc_size;
+    extra["total"] = total_size;
+
+err:
+    closedir(store_dir);
+    return total_size;
+  }
+
+
+protected:
+  WholeSpaceIterator _get_iterator() {
+    return ceph::shared_ptr(
+      new LevelDBWholeSpaceIteratorImpl(
+	db->NewIterator(leveldb::ReadOptions())
+      )
+    );
+  }
+
+  WholeSpaceIterator _get_snapshot_iterator() {
+    const leveldb::Snapshot *snapshot;
+    leveldb::ReadOptions options;
+
+    snapshot = db->GetSnapshot();
+    options.snapshot = snapshot;
+
+    return ceph::shared_ptr(
+      new LevelDBSnapshotIteratorImpl(db.get(), snapshot,
+	db->NewIterator(options))
+    );
+  }
+
+};
+
+#endif
diff --git a/ceph/src/os/Makefile.am b/ceph/src/os/Makefile.am
new file mode 100644
index 00000000..63a18461
--- /dev/null
+++ b/ceph/src/os/Makefile.am
@@ -0,0 +1,75 @@
+libos_types_la_SOURCES = \
+	os/Transaction.cc
+libos_types_la_CXXFLAGS = ${AM_CXXFLAGS}
+noinst_LTLIBRARIES += libos_types.la
+
+libos_la_SOURCES = \
+	os/chain_xattr.cc \
+	os/DBObjectMap.cc \
+	os/GenericObjectMap.cc \
+	os/FileJournal.cc \
+	os/FileStore.cc \
+	os/FlatIndex.cc \
+	os/GenericFileStoreBackend.cc \
+	os/HashIndex.cc \
+	os/IndexManager.cc \
+	os/JournalingObjectStore.cc \
+	os/LevelDBStore.cc \
+	os/LFNIndex.cc \
+	os/MemStore.cc \
+	os/KeyValueStore.cc \
+	os/ObjectStore.cc \
+	os/WBThrottle.cc \
+	common/TrackedOp.cc
+
+if LINUX
+libos_la_SOURCES += os/BtrfsFileStoreBackend.cc
+endif
+
+if WITH_LIBXFS
+libos_la_SOURCES += os/XfsFileStoreBackend.cc
+endif
+
+if WITH_LIBZFS
+libos_la_SOURCES += os/ZFSFileStoreBackend.cc
+endif
+
+libos_la_CXXFLAGS = ${AM_CXXFLAGS}
+libos_la_LIBADD = $(LIBOS_TYPES)
+noinst_LTLIBRARIES += libos.la
+
+noinst_HEADERS += \
+	os/btrfs_ioctl.h \
+	os/chain_xattr.h \
+	os/BtrfsFileStoreBackend.h \
+	os/CollectionIndex.h \
+	os/DBObjectMap.h \
+	os/GenericObjectMap.h \
+	os/FileJournal.h \
+	os/FileStore.h \
+	os/FlatIndex.h \
+	os/FDCache.h \
+	os/GenericFileStoreBackend.h \
+	os/HashIndex.h \
+	os/IndexManager.h \
+	os/Journal.h \
+	os/JournalingObjectStore.h \
+	os/KeyValueDB.h \
+	os/LevelDBStore.h \
+	os/LFNIndex.h \
+	os/MemStore.h \
+	os/KeyValueStore.h \
+	os/ObjectMap.h \
+	os/ObjectStore.h \
+	os/SequencerPosition.h \
+	os/WBThrottle.h \
+	os/XfsFileStoreBackend.h \
+	os/ZFSFileStoreBackend.h
+
+if WITH_LIBZFS
+libos_zfs_a_SOURCES = os/ZFS.cc
+libos_zfs_a_CXXFLAGS = ${AM_CXXFLAGS} ${LIBZFS_CFLAGS}
+noinst_LIBRARIES += libos_zfs.a
+noinst_HEADERS += os/ZFS.h
+endif
+
diff --git a/ceph/src/os/MemStore.cc b/ceph/src/os/MemStore.cc
new file mode 100644
index 00000000..952866a0
--- /dev/null
+++ b/ceph/src/os/MemStore.cc
@@ -0,0 +1,1487 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "acconfig.h"
+
+#ifdef HAVE_SYS_MOUNT_H
+#include 
+#endif
+
+#ifdef HAVE_SYS_PARAM_H
+#include 
+#endif
+
+#include "include/types.h"
+#include "include/stringify.h"
+#include "include/unordered_map.h"
+#include "include/memory.h"
+#include "common/errno.h"
+#include "MemStore.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "memstore(" << path << ") "
+
+// for comparing collections for lock ordering
+bool operator>(const MemStore::CollectionRef& l,
+	       const MemStore::CollectionRef& r)
+{
+  return (unsigned long)l.get() > (unsigned long)r.get();
+}
+
+
+int MemStore::peek_journal_fsid(uuid_d *fsid)
+{
+  *fsid = uuid_d();
+  return 0;
+}
+
+int MemStore::mount()
+{
+  int r = _load();
+  if (r < 0)
+    return r;
+  finisher.start();
+  return 0;
+}
+
+int MemStore::umount()
+{
+  finisher.stop();
+  return _save();
+}
+
+int MemStore::_save()
+{
+  dout(10) << __func__ << dendl;
+  Mutex::Locker l(apply_lock); // block any writer
+  dump_all();
+  set collections;
+  for (ceph::unordered_map::iterator p = coll_map.begin();
+       p != coll_map.end();
+       ++p) {
+    dout(20) << __func__ << " coll " << p->first << " " << p->second << dendl;
+    collections.insert(p->first);
+    bufferlist bl;
+    assert(p->second);
+    p->second->encode(bl);
+    string fn = path + "/" + stringify(p->first);
+    int r = bl.write_file(fn.c_str());
+    if (r < 0)
+      return r;
+  }
+
+  string fn = path + "/collections";
+  bufferlist bl;
+  ::encode(collections, bl);
+  int r = bl.write_file(fn.c_str());
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+void MemStore::dump_all()
+{
+  Formatter *f = new_formatter("json-pretty");
+  f->open_object_section("store");
+  dump(f);
+  f->close_section();
+  dout(0) << "dump:";
+  f->flush(*_dout);
+  *_dout << dendl;
+  delete f;
+}
+
+void MemStore::dump(Formatter *f)
+{
+  f->open_array_section("collections");
+  for (ceph::unordered_map::iterator p = coll_map.begin();
+       p != coll_map.end();
+       ++p) {
+    f->open_object_section("collection");
+    f->dump_string("name", stringify(p->first));
+
+    f->open_array_section("xattrs");
+    for (map::iterator q = p->second->xattr.begin();
+	 q != p->second->xattr.end();
+	 ++q) {
+      f->open_object_section("xattr");
+      f->dump_string("name", q->first);
+      f->dump_int("length", q->second.length());
+      f->close_section();
+    }
+    f->close_section();
+
+    f->open_array_section("objects");
+    for (map::iterator q = p->second->object_map.begin();
+	 q != p->second->object_map.end();
+	 ++q) {
+      f->open_object_section("object");
+      f->dump_string("name", stringify(q->first));
+      if (q->second)
+	q->second->dump(f);
+      f->close_section();
+    }
+    f->close_section();
+
+    f->close_section();
+  }
+  f->close_section();
+}
+
+int MemStore::_load()
+{
+  dout(10) << __func__ << dendl;
+  bufferlist bl;
+  string fn = path + "/collections";
+  string err;
+  int r = bl.read_file(fn.c_str(), &err);
+  if (r < 0)
+    return r;
+
+  set collections;
+  bufferlist::iterator p = bl.begin();
+  ::decode(collections, p);
+
+  for (set::iterator q = collections.begin();
+       q != collections.end();
+       ++q) {
+    string fn = path + "/" + stringify(*q);
+    bufferlist cbl;
+    int r = cbl.read_file(fn.c_str(), &err);
+    if (r < 0)
+      return r;
+    CollectionRef c(new Collection);
+    bufferlist::iterator p = cbl.begin();
+    c->decode(p);
+    coll_map[*q] = c;
+  }
+
+  dump_all();
+
+  return 0;  
+}
+
+void MemStore::set_fsid(uuid_d u)
+{
+  int r = write_meta("fs_fsid", stringify(u));
+  assert(r >= 0);
+}
+
+uuid_d MemStore::get_fsid()
+{
+  string fsid_str;
+  int r = read_meta("fs_fsid", &fsid_str);
+  assert(r >= 0);
+  uuid_d uuid;
+  bool b = uuid.parse(fsid_str.c_str());
+  assert(b);
+  return uuid;
+}
+
+int MemStore::mkfs()
+{
+  string fsid_str;
+  int r = read_meta("fs_fsid", &fsid_str);
+  if (r == -ENOENT) {
+    uuid_d fsid;
+    fsid.generate_random();
+    fsid_str = stringify(fsid);
+    r = write_meta("fs_fsid", fsid_str);
+    if (r < 0)
+      return r;
+    dout(1) << __func__ << " new fsid " << fsid_str << dendl;
+  } else {
+    dout(1) << __func__ << " had fsid " << fsid_str << dendl;
+  }
+
+  string fn = path + "/collections";
+  derr << path << dendl;
+  bufferlist bl;
+  set collections;
+  ::encode(collections, bl);
+  r = bl.write_file(fn.c_str());
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int MemStore::statfs(struct statfs *st)
+{
+  dout(10) << __func__ << dendl;
+  // make some shit up.  these are the only fields that matter.
+  st->f_bsize = 1024;
+  st->f_blocks = 1000000;
+  st->f_bfree =  1000000;
+  st->f_bavail = 1000000;
+  return 0;
+}
+
+objectstore_perf_stat_t MemStore::get_cur_stats()
+{
+  // fixme
+  return objectstore_perf_stat_t();
+}
+
+MemStore::CollectionRef MemStore::get_collection(coll_t cid)
+{
+  RWLock::RLocker l(coll_lock);
+  ceph::unordered_map::iterator cp = coll_map.find(cid);
+  if (cp == coll_map.end())
+    return CollectionRef();
+  return cp->second;
+}
+
+
+// ---------------
+// read operations
+
+bool MemStore::exists(coll_t cid, const ghobject_t& oid)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return false;
+  RWLock::RLocker l(c->lock);
+
+  // Perform equivalent of c->get_object_(oid) != NULL. In C++11 the
+  // shared_ptr needs to be compared to nullptr.
+  return (bool)c->get_object(oid);
+}
+
+int MemStore::stat(
+    coll_t cid,
+    const ghobject_t& oid,
+    struct stat *st,
+    bool allow_eio)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  st->st_size = o->data.length();
+  st->st_blksize = 4096;
+  st->st_blocks = (st->st_size + st->st_blksize - 1) / st->st_blksize;
+  st->st_nlink = 1;
+  return 0;
+}
+
+int MemStore::read(
+    coll_t cid,
+    const ghobject_t& oid,
+    uint64_t offset,
+    size_t len,
+    bufferlist& bl,
+    bool allow_eio)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " "
+	   << offset << "~" << len << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker lc(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  if (offset >= o->data.length())
+    return 0;
+  size_t l = len;
+  if (l == 0)  // note: len == 0 means read the entire object
+    l = o->data.length();
+  else if (offset + l > o->data.length())
+    l = o->data.length() - offset;
+  bl.clear();
+  bl.substr_of(o->data, offset, l);
+  return bl.length();
+}
+
+int MemStore::fiemap(coll_t cid, const ghobject_t& oid,
+		     uint64_t offset, size_t len, bufferlist& bl)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " " << offset << "~"
+	   << len << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker lc(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  if (offset >= o->data.length())
+    return 0;
+  size_t l = len;
+  if (offset + l > o->data.length())
+    l = o->data.length() - offset;
+  map m;
+  m[offset] = l;
+  ::encode(m, bl);
+  return 0;  
+}
+
+int MemStore::getattr(coll_t cid, const ghobject_t& oid,
+		      const char *name, bufferptr& value)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " " << name << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  string k(name);
+  if (!o->xattr.count(k)) {
+    return -ENODATA;
+  }
+  value = o->xattr[k];
+  return 0;
+}
+
+int MemStore::getattrs(coll_t cid, const ghobject_t& oid,
+		       map& aset, bool user_only)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  if (user_only) {
+    for (map::iterator p = o->xattr.begin();
+	 p != o->xattr.end();
+	 ++p) {
+      if (p->first.length() > 1 && p->first[0] == '_') {
+	aset[p->first.substr(1)] = p->second;
+      }
+    }
+  } else {
+    aset = o->xattr;
+  }
+  return 0;
+}
+
+int MemStore::list_collections(vector& ls)
+{
+  dout(10) << __func__ << dendl;
+  RWLock::RLocker l(coll_lock);
+  for (ceph::unordered_map::iterator p = coll_map.begin();
+       p != coll_map.end();
+       ++p) {
+    ls.push_back(p->first);
+  }
+  return 0;
+}
+
+bool MemStore::collection_exists(coll_t cid)
+{
+  dout(10) << __func__ << " " << cid << dendl;
+  RWLock::RLocker l(coll_lock);
+  return coll_map.count(cid);
+}
+
+int MemStore::collection_getattr(coll_t cid, const char *name,
+				 void *value, size_t size)
+{
+  dout(10) << __func__ << " " << cid << " " << name << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker lc(c->lock);
+
+  if (!c->xattr.count(name))
+    return -ENOENT;
+  bufferlist bl;
+  bl.append(c->xattr[name]);
+  size_t l = MIN(size, bl.length());
+  bl.copy(0, size, (char *)value);
+  return l;
+}
+
+int MemStore::collection_getattr(coll_t cid, const char *name, bufferlist& bl)
+{
+  dout(10) << __func__ << " " << cid << " " << name << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  if (!c->xattr.count(name))
+    return -ENOENT;
+  bl.clear();
+  bl.append(c->xattr[name]);
+  return bl.length();
+}
+
+int MemStore::collection_getattrs(coll_t cid, map &aset)
+{
+  dout(10) << __func__ << " " << cid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  aset = c->xattr;
+  return 0;
+}
+
+bool MemStore::collection_empty(coll_t cid)
+{
+  dout(10) << __func__ << " " << cid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  return c->object_map.empty();
+}
+
+int MemStore::collection_list(coll_t cid, vector& o)
+{
+  dout(10) << __func__ << " " << cid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  for (map::iterator p = c->object_map.begin();
+       p != c->object_map.end();
+       ++p)
+    o.push_back(p->first);
+  return 0;
+}
+
+int MemStore::collection_list_partial(coll_t cid, ghobject_t start,
+				      int min, int max, snapid_t snap, 
+				      vector *ls, ghobject_t *next)
+{
+  dout(10) << __func__ << " " << cid << " " << start << " " << min << "-"
+	   << max << " " << snap << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  map::iterator p = c->object_map.lower_bound(start);
+  while (p != c->object_map.end() &&
+	 ls->size() < (unsigned)max) {
+    ls->push_back(p->first);
+    ++p;
+  }
+  if (p == c->object_map.end())
+    *next = ghobject_t::get_max();
+  else
+    *next = p->first;
+  return 0;
+}
+
+int MemStore::collection_list_range(coll_t cid,
+				    ghobject_t start, ghobject_t end,
+				    snapid_t seq, vector *ls)
+{
+  dout(10) << __func__ << " " << cid << " " << start << " " << end
+	   << " " << seq << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  map::iterator p = c->object_map.lower_bound(start);
+  while (p != c->object_map.end() &&
+	 p->first < end) {
+    ls->push_back(p->first);
+    ++p;
+  }
+  return 0;
+}
+
+int MemStore::omap_get(
+    coll_t cid,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    bufferlist *header,      ///< [out] omap header
+    map *out /// < [out] Key to value map
+    )
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  *header = o->omap_header;
+  *out = o->omap;
+  return 0;
+}
+
+int MemStore::omap_get_header(
+    coll_t cid,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    bufferlist *header,      ///< [out] omap header
+    bool allow_eio ///< [in] don't assert on eio
+    )
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  *header = o->omap_header;
+  return 0;
+}
+
+int MemStore::omap_get_keys(
+    coll_t cid,              ///< [in] Collection containing oid
+    const ghobject_t &oid, ///< [in] Object containing omap
+    set *keys      ///< [out] Keys defined on oid
+    )
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  for (map::iterator p = o->omap.begin();
+       p != o->omap.end();
+       ++p)
+    keys->insert(p->first);
+  return 0;
+}
+
+int MemStore::omap_get_values(
+    coll_t cid,                    ///< [in] Collection containing oid
+    const ghobject_t &oid,       ///< [in] Object containing omap
+    const set &keys,     ///< [in] Keys to get
+    map *out ///< [out] Returned keys and values
+    )
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  for (set::const_iterator p = keys.begin();
+       p != keys.end();
+       ++p) {
+    map::iterator q = o->omap.find(*p);
+    if (q != o->omap.end())
+      out->insert(*q);
+  }
+  return 0;
+}
+
+int MemStore::omap_check_keys(
+    coll_t cid,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    const set &keys, ///< [in] Keys to check
+    set *out         ///< [out] Subset of keys defined on oid
+    )
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  for (set::const_iterator p = keys.begin();
+       p != keys.end();
+       ++p) {
+    map::iterator q = o->omap.find(*p);
+    if (q != o->omap.end())
+      out->insert(*p);
+  }
+  return 0;
+}
+
+ObjectMap::ObjectMapIterator MemStore::get_omap_iterator(coll_t cid,
+							 const ghobject_t& oid)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return ObjectMap::ObjectMapIterator();
+  RWLock::RLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return ObjectMap::ObjectMapIterator();
+  return ObjectMap::ObjectMapIterator(new OmapIteratorImpl(c, o));
+}
+
+
+// ---------------
+// write operations
+
+int MemStore::queue_transactions(Sequencer *osr,
+				 list& tls,
+				 TrackedOpRef op,
+				 ThreadPool::TPHandle *handle)
+{
+  // fixme: ignore the Sequencer and serialize everything.
+  Mutex::Locker l(apply_lock);
+
+  for (list::iterator p = tls.begin(); p != tls.end(); ++p) {
+    // poke the TPHandle heartbeat just to exercise that code path
+    if (handle)
+      handle->reset_tp_timeout();
+
+    _do_transaction(**p);
+  }
+
+  Context *on_apply = NULL, *on_apply_sync = NULL, *on_commit = NULL;
+  ObjectStore::Transaction::collect_contexts(tls, &on_apply, &on_commit,
+					     &on_apply_sync);
+  if (on_apply_sync)
+    on_apply_sync->complete(0);
+  if (on_apply)
+    finisher.queue(on_apply);
+  if (on_commit)
+    finisher.queue(on_commit);
+  return 0;
+}
+
+void MemStore::_do_transaction(Transaction& t)
+{
+  Transaction::iterator i = t.begin();
+  int pos = 0;
+
+  while (i.have_op()) {
+    int op = i.get_op();
+    int r = 0;
+
+    switch (op) {
+    case Transaction::OP_NOP:
+      break;
+    case Transaction::OP_TOUCH:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	r = _touch(cid, oid);
+      }
+      break;
+      
+    case Transaction::OP_WRITE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	bool replica = i.get_replica();
+	bufferlist bl;
+	i.get_bl(bl);
+	r = _write(cid, oid, off, len, bl, replica);
+      }
+      break;
+      
+    case Transaction::OP_ZERO:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	r = _zero(cid, oid, off, len);
+      }
+      break;
+      
+    case Transaction::OP_TRIMCACHE:
+      {
+	i.get_cid();
+	i.get_oid();
+	i.get_length();
+	i.get_length();
+	// deprecated, no-op
+      }
+      break;
+      
+    case Transaction::OP_TRUNCATE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	r = _truncate(cid, oid, off);
+      }
+      break;
+      
+    case Transaction::OP_REMOVE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	r = _remove(cid, oid);
+      }
+      break;
+      
+    case Transaction::OP_SETATTR:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	string name = i.get_attrname();
+	bufferlist bl;
+	i.get_bl(bl);
+	map to_set;
+	to_set[name] = bufferptr(bl.c_str(), bl.length());
+	r = _setattrs(cid, oid, to_set);
+      }
+      break;
+      
+    case Transaction::OP_SETATTRS:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	map aset;
+	i.get_attrset(aset);
+	r = _setattrs(cid, oid, aset);
+      }
+      break;
+
+    case Transaction::OP_RMATTR:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	string name = i.get_attrname();
+	r = _rmattr(cid, oid, name.c_str());
+      }
+      break;
+
+    case Transaction::OP_RMATTRS:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	r = _rmattrs(cid, oid);
+      }
+      break;
+      
+    case Transaction::OP_CLONE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+	r = _clone(cid, oid, noid);
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+ 	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	r = _clone_range(cid, oid, noid, off, len, off);
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE2:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+ 	uint64_t srcoff = i.get_length();
+	uint64_t len = i.get_length();
+ 	uint64_t dstoff = i.get_length();
+	r = _clone_range(cid, oid, noid, srcoff, len, dstoff);
+      }
+      break;
+
+    case Transaction::OP_MKCOLL:
+      {
+	coll_t cid = i.get_cid();
+	r = _create_collection(cid);
+      }
+      break;
+
+    case Transaction::OP_RMCOLL:
+      {
+	coll_t cid = i.get_cid();
+	r = _destroy_collection(cid);
+      }
+      break;
+
+    case Transaction::OP_COLL_ADD:
+      {
+	coll_t ncid = i.get_cid();
+	coll_t ocid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	r = _collection_add(ncid, ocid, oid);
+      }
+      break;
+
+    case Transaction::OP_COLL_REMOVE:
+       {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	r = _remove(cid, oid);
+       }
+      break;
+
+    case Transaction::OP_COLL_MOVE:
+      assert(0 == "deprecated");
+      break;
+
+    case Transaction::OP_COLL_MOVE_RENAME:
+      {
+	coll_t oldcid = i.get_cid();
+	ghobject_t oldoid = i.get_oid();
+	coll_t newcid = i.get_cid();
+	ghobject_t newoid = i.get_oid();
+	r = _collection_move_rename(oldcid, oldoid, newcid, newoid);
+      }
+      break;
+
+    case Transaction::OP_COLL_SETATTR:
+      {
+	coll_t cid = i.get_cid();
+	string name = i.get_attrname();
+	bufferlist bl;
+	i.get_bl(bl);
+	r = _collection_setattr(cid, name.c_str(), bl.c_str(), bl.length());
+      }
+      break;
+
+    case Transaction::OP_COLL_RMATTR:
+      {
+	coll_t cid = i.get_cid();
+	string name = i.get_attrname();
+	r = _collection_rmattr(cid, name.c_str());
+      }
+      break;
+
+    case Transaction::OP_COLL_RENAME:
+      {
+	coll_t cid(i.get_cid());
+	coll_t ncid(i.get_cid());
+	r = _collection_rename(cid, ncid);
+      }
+      break;
+
+    case Transaction::OP_OMAP_CLEAR:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	r = _omap_clear(cid, oid);
+      }
+      break;
+    case Transaction::OP_OMAP_SETKEYS:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	map aset;
+	i.get_attrset(aset);
+	r = _omap_setkeys(cid, oid, aset);
+      }
+      break;
+    case Transaction::OP_OMAP_RMKEYS:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	set keys;
+	i.get_keyset(keys);
+	r = _omap_rmkeys(cid, oid, keys);
+      }
+      break;
+    case Transaction::OP_OMAP_RMKEYRANGE:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	string first, last;
+	first = i.get_key();
+	last = i.get_key();
+	r = _omap_rmkeyrange(cid, oid, first, last);
+      }
+      break;
+    case Transaction::OP_OMAP_SETHEADER:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	bufferlist bl;
+	i.get_bl(bl);
+	r = _omap_setheader(cid, oid, bl);
+      }
+      break;
+    case Transaction::OP_SPLIT_COLLECTION:
+      assert(0 == "deprecated");
+      break;
+    case Transaction::OP_SPLIT_COLLECTION2:
+      {
+	coll_t cid(i.get_cid());
+	uint32_t bits(i.get_u32());
+	uint32_t rem(i.get_u32());
+	coll_t dest(i.get_cid());
+	r = _split_collection(cid, bits, rem, dest);
+      }
+      break;
+
+    case Transaction::OP_SETALLOCHINT:
+      {
+        coll_t cid(i.get_cid());
+        ghobject_t oid = i.get_oid();
+        (void)i.get_length();  // discard result
+        (void)i.get_length();  // discard result
+      }
+      break;
+
+    default:
+      derr << "bad op " << op << dendl;
+      assert(0);
+    }
+
+    if (r < 0) {
+      bool ok = false;
+
+      if (r == -ENOENT && !(op == Transaction::OP_CLONERANGE ||
+			    op == Transaction::OP_CLONE ||
+			    op == Transaction::OP_CLONERANGE2 ||
+			    op == Transaction::OP_COLL_ADD))
+	// -ENOENT is usually okay
+	ok = true;
+      if (r == -ENODATA)
+	ok = true;
+
+      if (!ok) {
+	const char *msg = "unexpected error code";
+
+	if (r == -ENOENT && (op == Transaction::OP_CLONERANGE ||
+			     op == Transaction::OP_CLONE ||
+			     op == Transaction::OP_CLONERANGE2))
+	  msg = "ENOENT on clone suggests osd bug";
+
+	if (r == -ENOSPC)
+	  // For now, if we hit _any_ ENOSPC, crash, before we do any damage
+	  // by partially applying transactions.
+	  msg = "ENOSPC handling not implemented";
+
+	if (r == -ENOTEMPTY) {
+	  msg = "ENOTEMPTY suggests garbage data in osd data dir";
+	  dump_all();
+	}
+
+	dout(0) << " error " << cpp_strerror(r) << " not handled on operation " << op
+		<< " (op " << pos << ", counting from 0)" << dendl;
+	dout(0) << msg << dendl;
+	dout(0) << " transaction dump:\n";
+	JSONFormatter f(true);
+	f.open_object_section("transaction");
+	t.dump(&f);
+	f.close_section();
+	f.flush(*_dout);
+	*_dout << dendl;
+	assert(0 == "unexpected error");
+      }
+    }
+
+    ++pos;
+  }
+}
+
+int MemStore::_touch(coll_t cid, const ghobject_t& oid)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o) {
+    o.reset(new Object);
+    c->object_map[oid] = o;
+    c->object_hash[oid] = o;
+  }
+  return 0;
+}
+
+int MemStore::_write(coll_t cid, const ghobject_t& oid,
+		     uint64_t offset, size_t len, const bufferlist& bl,
+		     bool replica)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " "
+	   << offset << "~" << len << dendl;
+  assert(len == bl.length());
+
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o) {
+    // write implicitly creates a missing object
+    o.reset(new Object);
+    c->object_map[oid] = o;
+    c->object_hash[oid] = o;
+  }
+
+  _write_into_bl(bl, offset, &o->data);
+  return 0;
+}
+
+void MemStore::_write_into_bl(const bufferlist& src, unsigned offset,
+			      bufferlist *dst)
+{
+  unsigned len = src.length();
+
+  // before
+  bufferlist newdata;
+  if (dst->length() >= offset) {
+    newdata.substr_of(*dst, 0, offset);
+  } else {
+    newdata.substr_of(*dst, 0, dst->length());
+    bufferptr bp(offset - dst->length());
+    bp.zero();
+    newdata.append(bp);
+  }
+
+  newdata.append(src);
+
+  // after
+  if (dst->length() > offset + len) {
+    bufferlist tail;
+    tail.substr_of(*dst, offset + len, dst->length() - (offset + len));
+    newdata.append(tail);
+  }
+
+  dst->claim(newdata);
+}
+
+int MemStore::_zero(coll_t cid, const ghobject_t& oid,
+		    uint64_t offset, size_t len)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " " << offset << "~"
+	   << len << dendl;
+  bufferptr bp(len);
+  bp.zero();
+  bufferlist bl;
+  bl.push_back(bp);
+  return _write(cid, oid, offset, len, bl);
+}
+
+int MemStore::_truncate(coll_t cid, const ghobject_t& oid, uint64_t size)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " " << size << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  if (o->data.length() > size) {
+    bufferlist bl;
+    bl.substr_of(o->data, 0, size);
+    o->data.claim(bl);
+  } else if (o->data.length() == size) {
+    // do nothing
+  } else {
+    bufferptr bp(size - o->data.length());
+    bp.zero();
+    o->data.append(bp);
+  }
+  return 0;
+}
+
+int MemStore::_remove(coll_t cid, const ghobject_t& oid)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  c->object_map.erase(oid);
+  c->object_hash.erase(oid);
+  return 0;
+}
+
+int MemStore::_setattrs(coll_t cid, const ghobject_t& oid,
+			map& aset)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  for (map::const_iterator p = aset.begin(); p != aset.end(); ++p)
+    o->xattr[p->first] = p->second;
+  return 0;
+}
+
+int MemStore::_rmattr(coll_t cid, const ghobject_t& oid, const char *name)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " " << name << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  if (!o->xattr.count(name))
+    return -ENODATA;
+  o->xattr.erase(name);
+  return 0;
+}
+
+int MemStore::_rmattrs(coll_t cid, const ghobject_t& oid)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  o->xattr.clear();
+  return 0;
+}
+
+int MemStore::_clone(coll_t cid, const ghobject_t& oldoid,
+		     const ghobject_t& newoid)
+{
+  dout(10) << __func__ << " " << cid << " " << oldoid
+	   << " -> " << newoid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef oo = c->get_object(oldoid);
+  if (!oo)
+    return -ENOENT;
+  ObjectRef no = c->get_object(newoid);
+  if (!no) {
+    no.reset(new Object);
+    c->object_map[newoid] = no;
+    c->object_hash[newoid] = no;
+  }
+  no->data = oo->data;
+  no->omap_header = oo->omap_header;
+  no->omap = oo->omap;
+  return 0;
+}
+
+int MemStore::_clone_range(coll_t cid, const ghobject_t& oldoid,
+			   const ghobject_t& newoid,
+			   uint64_t srcoff, uint64_t len, uint64_t dstoff)
+{
+  dout(10) << __func__ << " " << cid << " "
+	   << oldoid << " " << srcoff << "~" << len << " -> "
+	   << newoid << " " << dstoff << "~" << len
+	   << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef oo = c->get_object(oldoid);
+  if (!oo)
+    return -ENOENT;
+  ObjectRef no = c->get_object(newoid);
+  if (!no) {
+    no.reset(new Object);
+    c->object_map[newoid] = no;
+    c->object_hash[newoid] = no;
+  }
+  if (srcoff >= oo->data.length())
+    return 0;
+  if (srcoff + len >= oo->data.length())
+    len = oo->data.length() - srcoff;
+  bufferlist bl;
+  bl.substr_of(oo->data, srcoff, len);
+  _write_into_bl(bl, dstoff, &no->data);
+  return len;
+}
+
+int MemStore::_omap_clear(coll_t cid, const ghobject_t &oid)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  o->omap.clear();
+  return 0;
+}
+
+int MemStore::_omap_setkeys(coll_t cid, const ghobject_t &oid,
+			    const map &aset)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  for (map::const_iterator p = aset.begin(); p != aset.end(); ++p)
+    o->omap[p->first] = p->second;
+  return 0;
+}
+
+int MemStore::_omap_rmkeys(coll_t cid, const ghobject_t &oid,
+			   const set &keys)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  for (set::const_iterator p = keys.begin(); p != keys.end(); ++p)
+    o->omap.erase(*p);
+  return 0;
+}
+
+int MemStore::_omap_rmkeyrange(coll_t cid, const ghobject_t &oid,
+			       const string& first, const string& last)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << " " << first
+	   << " " << last << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  map::iterator p = o->omap.upper_bound(first);
+  map::iterator e = o->omap.lower_bound(last);
+  while (p != e)
+    o->omap.erase(p++);
+  return 0;
+}
+
+int MemStore::_omap_setheader(coll_t cid, const ghobject_t &oid,
+			      const bufferlist &bl)
+{
+  dout(10) << __func__ << " " << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  RWLock::WLocker l(c->lock);
+
+  ObjectRef o = c->get_object(oid);
+  if (!o)
+    return -ENOENT;
+  o->omap_header = bl;
+  return 0;
+}
+
+int MemStore::_create_collection(coll_t cid)
+{
+  dout(10) << __func__ << " " << cid << dendl;
+  RWLock::WLocker l(coll_lock);
+  ceph::unordered_map::iterator cp = coll_map.find(cid);
+  if (cp != coll_map.end())
+    return -EEXIST;
+  coll_map[cid].reset(new Collection);
+  return 0;
+}
+
+int MemStore::_destroy_collection(coll_t cid)
+{
+  dout(10) << __func__ << " " << cid << dendl;
+  RWLock::WLocker l(coll_lock);
+  ceph::unordered_map::iterator cp = coll_map.find(cid);
+  if (cp == coll_map.end())
+    return -ENOENT;
+  {
+    RWLock::RLocker l2(cp->second->lock);
+    if (!cp->second->object_map.empty())
+      return -ENOTEMPTY;
+  }
+  coll_map.erase(cp);
+  return 0;
+}
+
+int MemStore::_collection_add(coll_t cid, coll_t ocid, const ghobject_t& oid)
+{
+  dout(10) << __func__ << " " << cid << " " << ocid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  CollectionRef oc = get_collection(ocid);
+  if (!oc)
+    return -ENOENT;
+  RWLock::WLocker l1(MIN(c, oc)->lock);
+  RWLock::WLocker l2(MAX(c, oc)->lock);
+
+  if (c->object_hash.count(oid))
+    return -EEXIST;
+  if (oc->object_hash.count(oid) == 0)
+    return -ENOENT;
+  ObjectRef o = oc->object_hash[oid];
+  c->object_map[oid] = o;
+  c->object_hash[oid] = o;
+  return 0;
+}
+
+int MemStore::_collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+				      coll_t cid, const ghobject_t& oid)
+{
+  dout(10) << __func__ << " " << oldcid << " " << oldoid << " -> "
+	   << cid << " " << oid << dendl;
+  CollectionRef c = get_collection(cid);
+  if (!c)
+    return -ENOENT;
+  CollectionRef oc = get_collection(oldcid);
+  if (!oc)
+    return -ENOENT;
+  RWLock::WLocker l1(MIN(c, oc)->lock);
+  RWLock::WLocker l2(MAX(c, oc)->lock);
+
+  if (c->object_hash.count(oid))
+    return -EEXIST;
+  if (oc->object_hash.count(oldoid) == 0)
+    return -ENOENT;
+  ObjectRef o = oc->object_hash[oldoid];
+  c->object_map[oid] = o;
+  c->object_hash[oid] = o;
+  oc->object_map.erase(oldoid);
+  oc->object_hash.erase(oldoid);
+  return 0; 
+}
+
+int MemStore::_collection_setattr(coll_t cid, const char *name,
+				  const void *value, size_t size)
+{
+  dout(10) << __func__ << " " << cid << " " << name << dendl;
+  ceph::unordered_map::iterator cp = coll_map.find(cid);
+  if (cp == coll_map.end())
+    return -ENOENT;
+  RWLock::WLocker l(cp->second->lock);
+
+  cp->second->xattr[name] = bufferptr((const char *)value, size);
+  return 0;
+}
+
+int MemStore::_collection_setattrs(coll_t cid, map &aset)
+{
+  dout(10) << __func__ << " " << cid << dendl;
+  ceph::unordered_map::iterator cp = coll_map.find(cid);
+  if (cp == coll_map.end())
+    return -ENOENT;
+  RWLock::WLocker l(cp->second->lock);
+
+  for (map::const_iterator p = aset.begin();
+       p != aset.end();
+       ++p) {
+    cp->second->xattr[p->first] = p->second;
+  }
+  return 0;
+}
+
+int MemStore::_collection_rmattr(coll_t cid, const char *name)
+{
+  dout(10) << __func__ << " " << cid << " " << name << dendl;
+  ceph::unordered_map::iterator cp = coll_map.find(cid);
+  if (cp == coll_map.end())
+    return -ENOENT;
+  RWLock::WLocker l(cp->second->lock);
+
+  if (cp->second->xattr.count(name) == 0)
+    return -ENODATA;
+  cp->second->xattr.erase(name);
+  return 0;
+}
+
+int MemStore::_collection_rename(const coll_t &cid, const coll_t &ncid)
+{
+  dout(10) << __func__ << " " << cid << " -> " << ncid << dendl;
+  RWLock::WLocker l(coll_lock);
+  if (coll_map.count(cid) == 0)
+    return -ENOENT;
+  if (coll_map.count(ncid))
+    return -EEXIST;
+  coll_map[ncid] = coll_map[cid];
+  coll_map.erase(cid);
+  return 0;
+}
+
+int MemStore::_split_collection(coll_t cid, uint32_t bits, uint32_t match,
+				coll_t dest)
+{
+  dout(10) << __func__ << " " << cid << " " << bits << " " << match << " "
+	   << dest << dendl;
+  CollectionRef sc = get_collection(cid);
+  if (!sc)
+    return -ENOENT;
+  CollectionRef dc = get_collection(dest);
+  if (!dc)
+    return -ENOENT;
+  RWLock::WLocker l1(MIN(sc, dc)->lock);
+  RWLock::WLocker l2(MAX(sc, dc)->lock);
+
+  map::iterator p = sc->object_map.begin();
+  while (p != sc->object_map.end()) {
+    if (p->first.match(bits, match)) {
+      dout(20) << " moving " << p->first << dendl;
+      dc->object_map.insert(make_pair(p->first, p->second));
+      dc->object_hash.insert(make_pair(p->first, p->second));
+      sc->object_hash.erase(p->first);
+      sc->object_map.erase(p++);
+    } else {
+      ++p;
+    }
+  }
+
+  return 0;
+}
diff --git a/ceph/src/os/MemStore.h b/ceph/src/os/MemStore.h
new file mode 100644
index 00000000..c33dd04c
--- /dev/null
+++ b/ceph/src/os/MemStore.h
@@ -0,0 +1,357 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013- Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MEMSTORE_H
+#define CEPH_MEMSTORE_H
+
+#include "include/assert.h"
+#include "include/unordered_map.h"
+#include "include/memory.h"
+#include "common/Finisher.h"
+#include "common/RWLock.h"
+#include "ObjectStore.h"
+
+class MemStore : public ObjectStore {
+public:
+  struct Object {
+    bufferlist data;
+    map xattr;
+    bufferlist omap_header;
+    map omap;
+
+    void encode(bufferlist& bl) const {
+      ENCODE_START(1, 1, bl);
+      ::encode(data, bl);
+      ::encode(xattr, bl);
+      ::encode(omap_header, bl);
+      ::encode(omap, bl);
+      ENCODE_FINISH(bl);
+    }
+    void decode(bufferlist::iterator& p) {
+      DECODE_START(1, p);
+      ::decode(data, p);
+      ::decode(xattr, p);
+      ::decode(omap_header, p);
+      ::decode(omap, p);
+      DECODE_FINISH(p);
+    }
+    void dump(Formatter *f) const {
+      f->dump_int("data_len", data.length());
+      f->dump_int("omap_header_len", omap_header.length());
+
+      f->open_array_section("xattrs");
+      for (map::const_iterator p = xattr.begin();
+	   p != xattr.end();
+	   ++p) {
+	f->open_object_section("xattr");
+	f->dump_string("name", p->first);
+	f->dump_int("length", p->second.length());
+	f->close_section();	
+      }
+      f->close_section();
+
+      f->open_array_section("omap");
+      for (map::const_iterator p = omap.begin();
+	   p != omap.end();
+	   ++p) {
+	f->open_object_section("pair");
+	f->dump_string("key", p->first);
+	f->dump_int("length", p->second.length());
+	f->close_section();	
+      }
+      f->close_section();
+    }
+  };
+  typedef ceph::shared_ptr ObjectRef;
+
+  struct Collection {
+    ceph::unordered_map object_hash;  ///< for lookup
+    map object_map;        ///< for iteration
+    map xattr;
+    RWLock lock;   ///< for object_{map,hash}
+
+    // NOTE: The lock only needs to protect the object_map/hash, not the
+    // contents of individual objects.  The osd is already sequencing
+    // reads and writes, so we will never see them concurrently at this
+    // level.
+
+    ObjectRef get_object(ghobject_t oid) {
+      ceph::unordered_map::iterator o = object_hash.find(oid);
+      if (o == object_hash.end())
+	return ObjectRef();
+      return o->second;
+    }
+
+    void encode(bufferlist& bl) const {
+      ENCODE_START(1, 1, bl);
+      ::encode(xattr, bl);
+      uint32_t s = object_map.size();
+      ::encode(s, bl);
+      for (map::const_iterator p = object_map.begin();
+	   p != object_map.end();
+	   ++p) {
+	::encode(p->first, bl);
+	p->second->encode(bl);
+      }
+      ENCODE_FINISH(bl);
+    }
+    void decode(bufferlist::iterator& p) {
+      DECODE_START(1, p);
+      ::decode(xattr, p);
+      uint32_t s;
+      ::decode(s, p);
+      while (s--) {
+	ghobject_t k;
+	::decode(k, p);
+	ObjectRef o(new Object);
+	o->decode(p);
+	object_map.insert(make_pair(k, o));
+	object_hash.insert(make_pair(k, o));
+      }
+      DECODE_FINISH(p);
+    }
+
+    Collection() : lock("MemStore::Collection::lock") {}
+  };
+  typedef ceph::shared_ptr CollectionRef;
+
+private:
+  class OmapIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
+    CollectionRef c;
+    ObjectRef o;
+    map::iterator it;
+  public:
+    OmapIteratorImpl(CollectionRef c, ObjectRef o)
+      : c(c), o(o), it(o->omap.begin()) {}
+
+    int seek_to_first() {
+      RWLock::RLocker l(c->lock);
+      it = o->omap.begin();
+      return 0;
+    }
+    int upper_bound(const string &after) {
+      RWLock::RLocker l(c->lock);
+      it = o->omap.upper_bound(after);
+      return 0;
+    }
+    int lower_bound(const string &to) {
+      RWLock::RLocker l(c->lock);
+      it = o->omap.lower_bound(to);
+      return 0;
+    }
+    bool valid() {
+      RWLock::RLocker l(c->lock);
+      return it != o->omap.end();      
+    }
+    int next() {
+      RWLock::RLocker l(c->lock);
+      ++it;
+      return 0;
+    }
+    string key() {
+      RWLock::RLocker l(c->lock);
+      return it->first;
+    }
+    bufferlist value() {
+      RWLock::RLocker l(c->lock);
+      return it->second;
+    }
+    int status() {
+      return 0;
+    }
+  };
+
+
+  ceph::unordered_map coll_map;
+  RWLock coll_lock;    ///< rwlock to protect coll_map
+  Mutex apply_lock;    ///< serialize all updates
+
+  CollectionRef get_collection(coll_t cid);
+
+  Finisher finisher;
+
+  void _do_transaction(Transaction& t);
+
+  void _write_into_bl(const bufferlist& src, unsigned offset, bufferlist *dst);
+
+  int _touch(coll_t cid, const ghobject_t& oid);
+  int _write(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, const bufferlist& bl,
+      bool replica = false);
+  int _zero(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len);
+  int _truncate(coll_t cid, const ghobject_t& oid, uint64_t size);
+  int _remove(coll_t cid, const ghobject_t& oid);
+  int _setattrs(coll_t cid, const ghobject_t& oid, map& aset);
+  int _rmattr(coll_t cid, const ghobject_t& oid, const char *name);
+  int _rmattrs(coll_t cid, const ghobject_t& oid);
+  int _clone(coll_t cid, const ghobject_t& oldoid, const ghobject_t& newoid);
+  int _clone_range(coll_t cid, const ghobject_t& oldoid,
+		   const ghobject_t& newoid,
+		   uint64_t srcoff, uint64_t len, uint64_t dstoff);
+  int _omap_clear(coll_t cid, const ghobject_t &oid);
+  int _omap_setkeys(coll_t cid, const ghobject_t &oid,
+		    const map &aset);
+  int _omap_rmkeys(coll_t cid, const ghobject_t &oid, const set &keys);
+  int _omap_rmkeyrange(coll_t cid, const ghobject_t &oid,
+		       const string& first, const string& last);
+  int _omap_setheader(coll_t cid, const ghobject_t &oid, const bufferlist &bl);
+
+  int _create_collection(coll_t c);
+  int _destroy_collection(coll_t c);
+  int _collection_add(coll_t cid, coll_t ocid, const ghobject_t& oid);
+  int _collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+			      coll_t cid, const ghobject_t& o);
+  int _collection_setattr(coll_t cid, const char *name, const void *value,
+			  size_t size);
+  int _collection_setattrs(coll_t cid, map &aset);
+  int _collection_rmattr(coll_t cid, const char *name);
+  int _collection_rename(const coll_t &cid, const coll_t &ncid);
+  int _split_collection(coll_t cid, uint32_t bits, uint32_t rem, coll_t dest);
+
+  int _save();
+  int _load();
+
+  void dump(Formatter *f);
+  void dump_all();
+
+public:
+  MemStore(CephContext *cct, const string& path)
+    : ObjectStore(path),
+      coll_lock("MemStore::coll_lock"),
+      apply_lock("MemStore::apply_lock"),
+      finisher(cct) { }
+  ~MemStore() { }
+
+  int update_version_stamp() {
+    return 0;
+  }
+  uint32_t get_target_version() {
+    return 1;
+  }
+
+  int peek_journal_fsid(uuid_d *fsid);
+
+  bool test_mount_in_use() {
+    return false;
+  }
+
+  int mount();
+  int umount();
+
+  int get_max_object_name_length() {
+    return 4096;
+  }
+
+  int mkfs();
+  int mkjournal() {
+    return 0;
+  }
+
+  void set_allow_sharded_objects() {
+  }
+  bool get_allow_sharded_objects() {
+    return true;
+  }
+
+  int statfs(struct statfs *buf);
+
+  bool exists(coll_t cid, const ghobject_t& oid);
+  int stat(
+    coll_t cid,
+    const ghobject_t& oid,
+    struct stat *st,
+    bool allow_eio = false); // struct stat?
+  int read(
+    coll_t cid,
+    const ghobject_t& oid,
+    uint64_t offset,
+    size_t len,
+    bufferlist& bl,
+    bool allow_eio = false);
+  int fiemap(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, bufferlist& bl);
+  int getattr(coll_t cid, const ghobject_t& oid, const char *name, bufferptr& value);
+  int getattrs(coll_t cid, const ghobject_t& oid, map& aset, bool user_only = false);
+
+  int list_collections(vector& ls);
+  bool collection_exists(coll_t c);
+  int collection_getattr(coll_t cid, const char *name,
+			 void *value, size_t size);
+  int collection_getattr(coll_t cid, const char *name, bufferlist& bl);
+  int collection_getattrs(coll_t cid, map &aset);
+  bool collection_empty(coll_t c);
+  int collection_list(coll_t cid, vector& o);
+  int collection_list_partial(coll_t cid, ghobject_t start,
+			      int min, int max, snapid_t snap, 
+			      vector *ls, ghobject_t *next);
+  int collection_list_range(coll_t cid, ghobject_t start, ghobject_t end,
+			    snapid_t seq, vector *ls);
+
+  int omap_get(
+    coll_t cid,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    bufferlist *header,      ///< [out] omap header
+    map *out /// < [out] Key to value map
+    );
+
+  /// Get omap header
+  int omap_get_header(
+    coll_t cid,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    bufferlist *header,      ///< [out] omap header
+    bool allow_eio = false ///< [in] don't assert on eio
+    );
+
+  /// Get keys defined on oid
+  int omap_get_keys(
+    coll_t cid,              ///< [in] Collection containing oid
+    const ghobject_t &oid, ///< [in] Object containing omap
+    set *keys      ///< [out] Keys defined on oid
+    );
+
+  /// Get key values
+  int omap_get_values(
+    coll_t cid,                    ///< [in] Collection containing oid
+    const ghobject_t &oid,       ///< [in] Object containing omap
+    const set &keys,     ///< [in] Keys to get
+    map *out ///< [out] Returned keys and values
+    );
+
+  /// Filters keys into out which are defined on oid
+  int omap_check_keys(
+    coll_t cid,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    const set &keys, ///< [in] Keys to check
+    set *out         ///< [out] Subset of keys defined on oid
+    );
+
+  ObjectMap::ObjectMapIterator get_omap_iterator(
+    coll_t cid,              ///< [in] collection
+    const ghobject_t &oid  ///< [in] object
+    );
+
+  void set_fsid(uuid_d u);
+  uuid_d get_fsid();
+
+  objectstore_perf_stat_t get_cur_stats();
+
+  int queue_transactions(
+    Sequencer *osr, list& tls,
+    TrackedOpRef op = TrackedOpRef(),
+    ThreadPool::TPHandle *handle = NULL);
+};
+
+
+
+
+#endif
diff --git a/ceph/src/os/ObjectMap.h b/ceph/src/os/ObjectMap.h
new file mode 100644
index 00000000..86f9e3e5
--- /dev/null
+++ b/ceph/src/os/ObjectMap.h
@@ -0,0 +1,161 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef OS_KEYVALUESTORE_H
+#define OS_KEYVALUESTORE_H
+
+#include "IndexManager.h"
+#include "SequencerPosition.h"
+#include 
+#include 
+#include "include/memory.h"
+
+/**
+ * Encapsulates the FileStore key value store
+ *
+ * Implementations of this interface will be used to implement TMAP
+ */
+class ObjectMap {
+public:
+  /// Set keys and values from specified map
+  virtual int set_keys(
+    const ghobject_t &oid,              ///< [in] object containing map
+    const map &set,  ///< [in] key to value map to set
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) = 0;
+
+  /// Set header
+  virtual int set_header(
+    const ghobject_t &oid,              ///< [in] object containing map
+    const bufferlist &bl,               ///< [in] header to set
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) = 0;
+
+  /// Retrieve header
+  virtual int get_header(
+    const ghobject_t &oid,              ///< [in] object containing map
+    bufferlist *bl                      ///< [out] header to set
+    ) = 0;
+
+  /// Clear all map keys and values from oid
+  virtual int clear(
+    const ghobject_t &oid,             ///< [in] object containing map
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) = 0;
+
+  /// Clear all map keys and values in to_clear from oid
+  virtual int rm_keys(
+    const ghobject_t &oid,              ///< [in] object containing map
+    const set &to_clear,        ///< [in] Keys to clear
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) = 0;
+
+  /// Clear all omap keys and the header
+  virtual int clear_keys_header(
+    const ghobject_t &oid,              ///< [in] oid to clear
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) = 0;
+
+  /// Get all keys and values
+  virtual int get(
+    const ghobject_t &oid,             ///< [in] object containing map
+    bufferlist *header,                ///< [out] Returned Header
+    map *out       ///< [out] Returned keys and values
+    ) = 0;
+
+  /// Get values for supplied keys
+  virtual int get_keys(
+    const ghobject_t &oid,             ///< [in] object containing map
+    set *keys                  ///< [out] Keys defined on oid
+    ) = 0;
+
+  /// Get values for supplied keys
+  virtual int get_values(
+    const ghobject_t &oid,             ///< [in] object containing map
+    const set &keys,           ///< [in] Keys to get
+    map *out       ///< [out] Returned keys and values
+    ) = 0;
+
+  /// Check key existence
+  virtual int check_keys(
+    const ghobject_t &oid,             ///< [in] object containing map
+    const set &keys,           ///< [in] Keys to check
+    set *out                   ///< [out] Subset of keys defined on oid
+    ) = 0;
+
+  /// Get xattrs
+  virtual int get_xattrs(
+    const ghobject_t &oid,             ///< [in] object
+    const set &to_get,         ///< [in] keys to get
+    map *out       ///< [out] subset of attrs/vals defined
+    ) = 0;
+
+  /// Get all xattrs
+  virtual int get_all_xattrs(
+    const ghobject_t &oid,             ///< [in] object
+    set *out                   ///< [out] attrs and values
+    ) = 0;
+
+  /// set xattrs in to_set
+  virtual int set_xattrs(
+    const ghobject_t &oid,                ///< [in] object
+    const map &to_set,///< [in] attrs/values to set
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) = 0;
+
+  /// remove xattrs in to_remove
+  virtual int remove_xattrs(
+    const ghobject_t &oid,               ///< [in] object
+    const set &to_remove,        ///< [in] attrs to remove
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) = 0;
+
+
+  /// Clone keys efficiently from oid map to target map
+  virtual int clone(
+    const ghobject_t &oid,             ///< [in] object containing map
+    const ghobject_t &target,           ///< [in] target of clone
+    const SequencerPosition *spos=0     ///< [in] sequencer position
+    ) { return 0; }
+
+  /// Ensure all previous writes are durable
+  virtual int sync(
+    const ghobject_t *oid=0,          ///< [in] object
+    const SequencerPosition *spos=0   ///< [in] Sequencer
+    ) { return 0; }
+
+  virtual bool check(std::ostream &out) { return true; }
+
+  class ObjectMapIteratorImpl {
+  public:
+    virtual int seek_to_first() = 0;
+    virtual int upper_bound(const string &after) = 0;
+    virtual int lower_bound(const string &to) = 0;
+    virtual bool valid() = 0;
+    virtual int next() = 0;
+    virtual string key() = 0;
+    virtual bufferlist value() = 0;
+    virtual int status() = 0;
+    virtual ~ObjectMapIteratorImpl() {}
+  };
+  typedef ceph::shared_ptr ObjectMapIterator;
+  virtual ObjectMapIterator get_iterator(const ghobject_t &oid) {
+    return ObjectMapIterator();
+  }
+
+
+  virtual ~ObjectMap() {}
+};
+
+#endif
diff --git a/ceph/src/os/ObjectStore.cc b/ceph/src/os/ObjectStore.cc
new file mode 100644
index 00000000..afa90b16
--- /dev/null
+++ b/ceph/src/os/ObjectStore.cc
@@ -0,0 +1,159 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include 
+#include 
+#include "include/memory.h"
+#include "ObjectStore.h"
+#include "common/Formatter.h"
+#include "FileStore.h"
+#include "MemStore.h"
+#include "KeyValueStore.h"
+#include "common/safe_io.h"
+
+ObjectStore *ObjectStore::create(CephContext *cct,
+				 const string& type,
+				 const string& data,
+				 const string& journal)
+{
+  if (type == "filestore") {
+    return new FileStore(data, journal);
+  }
+  if (type == "memstore") {
+    return new MemStore(cct, data);
+  }
+  if (type == "keyvaluestore-dev") {
+    return new KeyValueStore(data);
+  }
+  return NULL;
+}
+
+int ObjectStore::write_meta(const std::string& key,
+			    const std::string& value)
+{
+  string v = value;
+  v += "\n";
+  int r = safe_write_file(path.c_str(), key.c_str(),
+			  v.c_str(), v.length());
+  if (r < 0)
+    return r;
+  return 0;
+}
+
+int ObjectStore::read_meta(const std::string& key,
+			   std::string *value)
+{
+  char buf[4096];
+  int r = safe_read_file(path.c_str(), key.c_str(),
+			 buf, sizeof(buf));
+  if (r <= 0)
+    return r;
+  // drop trailing newlines
+  while (r && isspace(buf[r-1])) {
+    --r;
+  }
+  *value = string(buf, r);
+  return 0;
+}
+
+
+
+
+ostream& operator<<(ostream& out, const ObjectStore::Sequencer& s)
+{
+  return out << "osr(" << s.get_name() << " " << &s << ")";
+}
+
+unsigned ObjectStore::apply_transactions(Sequencer *osr,
+					 list &tls,
+					 Context *ondisk)
+{
+  // use op pool
+  Cond my_cond;
+  Mutex my_lock("ObjectStore::apply_transaction::my_lock");
+  int r = 0;
+  bool done;
+  C_SafeCond *onreadable = new C_SafeCond(&my_lock, &my_cond, &done, &r);
+
+  queue_transactions(osr, tls, onreadable, ondisk);
+
+  my_lock.Lock();
+  while (!done)
+    my_cond.Wait(my_lock);
+  my_lock.Unlock();
+  return r;
+}
+
+int ObjectStore::queue_transactions(
+  Sequencer *osr,
+  list& tls,
+  Context *onreadable,
+  Context *oncommit,
+  Context *onreadable_sync,
+  Context *oncomplete,
+  TrackedOpRef op = TrackedOpRef())
+{
+  RunOnDeleteRef _complete(new RunOnDelete(oncomplete));
+  Context *_onreadable = new Wrapper(
+    onreadable, _complete);
+  Context *_oncommit = new Wrapper(
+    oncommit, _complete);
+  return queue_transactions(osr, tls, _onreadable, _oncommit,
+			    onreadable_sync, op);
+}
+
+int ObjectStore::collection_list(coll_t c, vector& o)
+{
+  vector go;
+  int ret = collection_list(c, go);
+  if (ret == 0) {
+    o.reserve(go.size());
+    for (vector::iterator i = go.begin(); i != go.end() ; ++i)
+      o.push_back(i->hobj);
+  }
+  return ret;
+}
+
+int ObjectStore::collection_list_partial(coll_t c, hobject_t start,
+			      int min, int max, snapid_t snap,
+				      vector *ls, hobject_t *next)
+{
+  vector go;
+  ghobject_t gnext, gstart(start);
+  int ret = collection_list_partial(c, gstart, min, max, snap, &go, &gnext);
+  if (ret == 0) {
+    *next = gnext.hobj;
+    ls->reserve(go.size());
+    for (vector::iterator i = go.begin(); i != go.end() ; ++i)
+      ls->push_back(i->hobj);
+  }
+  return ret;
+}
+
+int ObjectStore::collection_list_range(coll_t c, hobject_t start, hobject_t end,
+			    snapid_t seq, vector *ls)
+{
+  vector go;
+  // Starts with the smallest shard id and generation to
+  // make sure the result list has the marker object
+  ghobject_t gstart(start, 0, shard_id_t(0));
+  // Exclusive end, choose the smallest end ghobject
+  ghobject_t gend(end, 0, shard_id_t(0));
+  int ret = collection_list_range(c, gstart, gend, seq, &go);
+  if (ret == 0) {
+    ls->reserve(go.size());
+    for (vector::iterator i = go.begin(); i != go.end() ; ++i)
+      ls->push_back(i->hobj);
+  }
+  return ret;
+}
diff --git a/ceph/src/os/ObjectStore.h b/ceph/src/os/ObjectStore.h
new file mode 100644
index 00000000..a5f5fcb0
--- /dev/null
+++ b/ceph/src/os/ObjectStore.h
@@ -0,0 +1,1577 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_OBJECTSTORE_H
+#define CEPH_OBJECTSTORE_H
+
+#include "include/Context.h"
+#include "include/buffer.h"
+#include "include/types.h"
+#include "osd/osd_types.h"
+#include "common/TrackedOp.h"
+#include "common/WorkQueue.h"
+#include "ObjectMap.h"
+
+#include 
+#include 
+#include 
+
+#if defined(DARWIN) || defined(__FreeBSD__)
+#include 
+#else
+#include     /* or  */
+#endif /* DARWIN */
+
+class CephContext;
+
+using std::vector;
+using std::string;
+
+namespace ceph {
+  class Formatter;
+}
+
+enum {
+  l_os_first = 84000,
+  l_os_jq_max_ops,
+  l_os_jq_ops,
+  l_os_j_ops,
+  l_os_jq_max_bytes,
+  l_os_jq_bytes,
+  l_os_j_bytes,
+  l_os_j_lat,
+  l_os_j_wr,
+  l_os_j_wr_bytes,
+  l_os_j_full,
+  l_os_committing,
+  l_os_commit,
+  l_os_commit_len,
+  l_os_commit_lat,
+  l_os_oq_max_ops,
+  l_os_oq_ops,
+  l_os_ops,
+  l_os_oq_max_bytes,
+  l_os_oq_bytes,
+  l_os_bytes,
+  l_os_apply_lat,
+  l_os_queue_lat,
+  l_os_last,
+};
+
+
+/*
+ * low-level interface to the local OSD file system
+ */
+
+class Logger;
+
+
+static inline void encode(const map *attrset, bufferlist &bl) {
+  ::encode(*attrset, bl);
+}
+
+class ObjectStore {
+protected:
+  string path;
+
+public:
+  /**
+   * create - create an ObjectStore instance.
+   *
+   * This is invoked once at initialization time.
+   *
+   * @param type type of store. This is a string from the configuration file.
+   * @param data path (or other descriptor) for data
+   * @param journal path (or other descriptor) for journal (optional)
+   */
+  static ObjectStore *create(CephContext *cct,
+			     const string& type,
+			     const string& data,
+			     const string& journal);
+
+  Logger *logger;
+
+  /**
+   * Fetch Object Store statistics.
+   *
+   * Currently only latency of write and apply times are measured.
+   *
+   * This appears to be called with nothing locked.
+   */
+  virtual objectstore_perf_stat_t get_cur_stats() = 0;
+
+  /**
+   * a sequencer orders transactions
+   *
+   * Any transactions queued under a given sequencer will be applied in
+   * sequence.  Transactions queued under different sequencers may run
+   * in parallel.
+   *
+   * Clients of ObjectStore create and maintain their own Sequencer objects.
+   * When a list of transactions is queued the caller specifies a Sequencer to be used.
+   *
+   */
+
+  /**
+   * ABC for Sequencer implementation, private to the ObjectStore derived class.
+   * created in ...::queue_transaction(s)
+   */
+  struct Sequencer_impl {
+    virtual void flush() = 0;
+
+    /**
+     * Async flush_commit
+     *
+     * There are two cases:
+     * 1) sequencer is currently idle: the method returns true and
+     *    c is deleted
+     * 2) sequencer is not idle: the method returns false and c is
+     *    called asyncronously with a value of 0 once all transactions
+     *    queued on this sequencer prior to the call have been applied
+     *    and committed.
+     */
+    virtual bool flush_commit(
+      Context *c ///< [in] context to call upon flush/commit
+      ) = 0; ///< @return true if idle, false otherwise
+
+    virtual ~Sequencer_impl() {}
+  };
+
+  /**
+   * External (opaque) sequencer implementation
+   */
+  struct Sequencer {
+    string name;
+    Sequencer_impl *p;
+
+    Sequencer(string n)
+      : name(n), p(NULL) {}
+    ~Sequencer() {
+      delete p;
+    }
+
+    /// return a unique string identifier for this sequencer
+    const string& get_name() const {
+      return name;
+    }
+    /// wait for any queued transactions on this sequencer to apply
+    void flush() {
+      if (p)
+	p->flush();
+    }
+
+    /// @see Sequencer_impl::flush_commit()
+    bool flush_commit(Context *c) {
+      if (!p) {
+	delete c;
+	return true;
+      } else {
+	return p->flush_commit(c);
+      }
+    }
+  };
+
+  /*********************************
+   *
+   * Object Contents and semantics
+   *
+   * All ObjectStore objects are identified as a named object
+   * (ghobject_t and hobject_t) in a named collection (coll_t).
+   * ObjectStore operations support the creation, mutation, deletion
+   * and enumeration of objects within a collection.  Enumeration is
+   * in sorted key order (where keys are sorted by hash). Object names
+   * are globally unique.
+   *
+   * Each object has four distinct parts: byte data, xattrs, omap_header
+   * and omap entries.
+   *
+   * The data portion of an object is conceptually equivalent to a
+   * file in a file system. Random and Partial access for both read
+   * and operations is required. The ability to have a sparse
+   * implementation of the data portion of an object is beneficial for
+   * some workloads, but not required. There is a system-wide limit on
+   * the maximum size of an object, which is typically around 100 MB.
+   *
+   * Xattrs are equivalent to the extended attributes of file
+   * systems. Xattrs are a set of key/value pairs.  Sub-value access
+   * is not required. It is possible to enumerate the set of xattrs in
+   * key order.  At the implementation level, xattrs are used
+   * exclusively internal to Ceph and the implementer can expect the
+   * total size of all of the xattrs on an object to be relatively
+   * small, i.e., less than 64KB. Much of Ceph assumes that accessing
+   * xattrs on temporally adjacent object accesses (recent past or
+   * near future) is inexpensive.
+   *
+   * omap_header is a single blob of data. It can be read or written
+   * in total.
+   *
+   * Omap entries are conceptually the same as xattrs
+   * but in a different address space. In other words, you can have
+   * the same key as an xattr and an omap entry and they have distinct
+   * values. Enumeration of xattrs doesn't include omap entries and
+   * vice versa. The size and access characteristics of omap entries
+   * are very different from xattrs. In particular, the value portion
+   * of an omap entry can be quite large (MBs).  More importantly, the
+   * interface must support efficient range queries on omap entries even
+   * when there are a large numbers of entries.
+   *
+   *********************************/
+
+  /*******************************
+   *
+   * Collections
+   *
+   * A collection is simply a grouping of objects. Collections have
+   * names (coll_t) and can be enumerated in order.  Like an
+   * individual object, a collection also has a set of xattrs.
+   *
+   *
+   */
+
+
+  /*********************************
+   * transaction
+   *
+   * A Transaction represents a sequence of primitive mutation
+   * operations.
+   *
+   * Three events in the life of a Transaction result in
+   * callbacks. Any Transaction can contain any number of callback
+   * objects (Context) for any combination of the three classes of
+   * callbacks:
+   *
+   *    on_applied_sync, on_applied, and on_commit.
+   *
+   * The "on_applied" and "on_applied_sync" callbacks are invoked when
+   * the modifications requested by the Transaction are visible to
+   * subsequent ObjectStore operations, i.e., the results are
+   * readable. The only conceptual difference between on_applied and
+   * on_applied_sync is the specific thread and locking environment in
+   * which the callbacks operate.  "on_applied_sync" is called
+   * directly by an ObjectStore execution thread. It is expected to
+   * execute quickly and must not acquire any locks of the calling
+   * environment. Conversely, "on_applied" is called from the separate
+   * Finisher thread, meaning that it can contend for calling
+   * environment locks. NB, on_applied and on_applied sync are
+   * sometimes called on_readable and on_readable_sync.
+   *
+   * The "on_commit" callback is also called from the Finisher thread
+   * and indicates that all of the mutations have been durably
+   * committed to stable storage (i.e., are now software/hardware
+   * crashproof).
+   *
+   * At the implementation level, each mutation primitive (and its
+   * associated data) can be serialized to a single buffer.  That
+   * serialization, however, does not copy any data, but (using the
+   * bufferlist library) will reference the original buffers.  This
+   * implies that the buffer that contains the data being submitted
+   * must remain stable until the on_commit callback completes.  In
+   * practice, bufferlist handles all of this for you and this
+   * subtlety is only relevant if you are referencing an existing
+   * buffer via buffer::raw_static.
+   *
+   * Some implementations of ObjectStore choose to implement their own
+   * form of journaling that uses the serialized form of a
+   * Transaction. This requires that the encode/decode logic properly
+   * version itself and handle version upgrades that might change the
+   * format of the encoded Transaction. This has already happened a
+   * couple of times and the Transaction object contains some helper
+   * variables that aid in this legacy decoding:
+   *
+   *   sobject_encoding detects an older/simpler version of oid
+   *   present in pre-bobtail versions of ceph.  use_pool_override
+   *   also detects a situation where the pool of an oid can be
+   *   override for legacy operations/buffers.  For non-legacy
+   *   implementation of ObjectStore, neither of these fields is
+   *   relevant.
+   *
+   *
+   * TRANSACTION ISOLATION
+   *
+   * Except as noted below, isolation is the responsibility of the
+   * caller. In other words, if any storage element (storage element
+   * == any of the four portions of an object as described above) is
+   * altered by a transaction (including deletion), the caller
+   * promises not to attempt to read that element while the
+   * transaction is pending (here pending means from the time of
+   * issuance until the "on_applied_sync" callback has been
+   * received). Violations of isolation need not be detected by
+   * ObjectStore and there is no corresponding error mechanism for
+   * reporting an isolation violation (crashing would be the
+   * appropriate way to report an isolation violation if detected).
+   *
+   * Enumeration operations may violate transaction isolation as
+   * described above when a storage element is being created or
+   * deleted as part of a transaction. In this case, ObjectStore is
+   * allowed to consider the enumeration operation to either preceed
+   * or follow the violating transaction element. In other words, the
+   * presence/absence of the mutated element in the enumeration is
+   * entirely at the discretion of ObjectStore. The arbitrary ordering
+   * applies independently to each transaction element. For example,
+   * if a transaction contains two mutating elements "create A" and
+   * "delete B". And an enumeration operation is performed while this
+   * transaction is pending. It is permissable for ObjectStore to
+   * report any of the four possible combinations of the existance of
+   * A and B.
+   *
+   */
+  class Transaction {
+  public:
+    enum {
+      OP_NOP =          0,
+      OP_TOUCH =        9,   // cid, oid
+      OP_WRITE =        10,  // cid, oid, offset, len, bl
+      OP_ZERO =         11,  // cid, oid, offset, len
+      OP_TRUNCATE =     12,  // cid, oid, len
+      OP_REMOVE =       13,  // cid, oid
+      OP_SETATTR =      14,  // cid, oid, attrname, bl
+      OP_SETATTRS =     15,  // cid, oid, attrset
+      OP_RMATTR =       16,  // cid, oid, attrname
+      OP_CLONE =        17,  // cid, oid, newoid
+      OP_CLONERANGE =   18,  // cid, oid, newoid, offset, len
+      OP_CLONERANGE2 =  30,  // cid, oid, newoid, srcoff, len, dstoff
+
+      OP_TRIMCACHE =    19,  // cid, oid, offset, len  **DEPRECATED**
+
+      OP_MKCOLL =       20,  // cid
+      OP_RMCOLL =       21,  // cid
+      OP_COLL_ADD =     22,  // cid, oldcid, oid
+      OP_COLL_REMOVE =  23,  // cid, oid
+      OP_COLL_SETATTR = 24,  // cid, attrname, bl
+      OP_COLL_RMATTR =  25,  // cid, attrname
+      OP_COLL_SETATTRS = 26,  // cid, attrset
+      OP_COLL_MOVE =    8,   // newcid, oldcid, oid
+
+      OP_STARTSYNC =    27,  // start a sync
+
+      OP_RMATTRS =      28,  // cid, oid
+      OP_COLL_RENAME =       29,  // cid, newcid
+
+      OP_OMAP_CLEAR = 31,   // cid
+      OP_OMAP_SETKEYS = 32, // cid, attrset
+      OP_OMAP_RMKEYS = 33,  // cid, keyset
+      OP_OMAP_SETHEADER = 34, // cid, header
+      OP_SPLIT_COLLECTION = 35, // cid, bits, destination
+      OP_SPLIT_COLLECTION2 = 36, /* cid, bits, destination
+				    doesn't create the destination */
+      OP_OMAP_RMKEYRANGE = 37,  // cid, oid, firstkey, lastkey
+      OP_COLL_MOVE_RENAME = 38,   // oldcid, oldoid, newcid, newoid
+
+      OP_SETALLOCHINT = 39,  // cid, oid, object_size, write_size
+    };
+
+  private:
+    uint64_t ops;
+    uint64_t pad_unused_bytes;
+    uint32_t largest_data_len, largest_data_off, largest_data_off_in_tbl;
+    bufferlist tbl;
+    bool sobject_encoding;
+    int64_t pool_override;
+    bool use_pool_override;
+    bool replica;
+    bool tolerate_collection_add_enoent;
+
+    list on_applied;
+    list on_commit;
+    list on_applied_sync;
+
+  public:
+    void set_tolerate_collection_add_enoent() {
+      tolerate_collection_add_enoent = true;
+    }
+
+    /* Operations on callback contexts */
+    void register_on_applied(Context *c) {
+      if (!c) return;
+      on_applied.push_back(c);
+    }
+    void register_on_commit(Context *c) {
+      if (!c) return;
+      on_commit.push_back(c);
+    }
+    void register_on_applied_sync(Context *c) {
+      if (!c) return;
+      on_applied_sync.push_back(c);
+    }
+    void register_on_complete(Context *c) {
+      if (!c) return;
+      RunOnDeleteRef _complete(new RunOnDelete(c));
+      register_on_applied(new ContainerContext(_complete));
+      register_on_commit(new ContainerContext(_complete));
+    }
+
+    static void collect_contexts(
+      list &t,
+      Context **out_on_applied,
+      Context **out_on_commit,
+      Context **out_on_applied_sync) {
+      assert(out_on_applied);
+      assert(out_on_commit);
+      assert(out_on_applied_sync);
+      list on_applied, on_commit, on_applied_sync;
+      for (list::iterator i = t.begin();
+	   i != t.end();
+	   ++i) {
+	on_applied.splice(on_applied.end(), (*i)->on_applied);
+	on_commit.splice(on_commit.end(), (*i)->on_commit);
+	on_applied_sync.splice(on_applied_sync.end(), (*i)->on_applied_sync);
+      }
+      *out_on_applied = C_Contexts::list_to_context(on_applied);
+      *out_on_commit = C_Contexts::list_to_context(on_commit);
+      *out_on_applied_sync = C_Contexts::list_to_context(on_applied_sync);
+    }
+
+    Context *get_on_applied() {
+      return C_Contexts::list_to_context(on_applied);
+    }
+    Context *get_on_commit() {
+      return C_Contexts::list_to_context(on_commit);
+    }
+    Context *get_on_applied_sync() {
+      return C_Contexts::list_to_context(on_applied_sync);
+    }
+
+    /// For legacy transactions, provide the pool to override the encoded pool with
+    void set_pool_override(int64_t pool) {
+      pool_override = pool;
+    }
+    void set_replica() {
+      replica = true;
+    }
+    bool get_replica() { return replica; }
+
+    void swap(Transaction& other) {
+      std::swap(ops, other.ops);
+      std::swap(largest_data_len, other.largest_data_len);
+      std::swap(largest_data_off, other.largest_data_off);
+      std::swap(largest_data_off_in_tbl, other.largest_data_off_in_tbl);
+      std::swap(on_applied, other.on_applied);
+      std::swap(on_commit, other.on_commit);
+      std::swap(on_applied_sync, other.on_applied_sync);
+      tbl.swap(other.tbl);
+    }
+
+    /// Append the operations of the parameter to this Transaction. Those operations are removed from the parameter Transaction
+    void append(Transaction& other) {
+      ops += other.ops;
+      assert(pad_unused_bytes == 0);
+      assert(other.pad_unused_bytes == 0);
+      if (other.largest_data_len > largest_data_len) {
+	largest_data_len = other.largest_data_len;
+	largest_data_off = other.largest_data_off;
+	largest_data_off_in_tbl = tbl.length() + other.largest_data_off_in_tbl;
+      }
+      tbl.append(other.tbl);
+      on_applied.splice(on_applied.end(), other.on_applied);
+      on_commit.splice(on_commit.end(), other.on_commit);
+      on_applied_sync.splice(on_applied_sync.end(), other.on_applied_sync);
+    }
+
+    /** Inquires about the Transaction as a whole. */
+
+    /// How big is the encoded Transaction buffer?
+    uint64_t get_encoded_bytes() {
+      return 1 + 8 + 8 + 4 + 4 + 4 + 4 + tbl.length();
+    }
+
+    uint64_t get_num_bytes() {
+      return get_encoded_bytes();
+    }
+    /// Size of largest data buffer to the "write" operation encountered so far
+    uint32_t get_data_length() {
+      return largest_data_len;
+    }
+    /// offset within the encoded buffer to the start of the first data buffer that's encoded
+    uint32_t get_data_offset() {
+      if (largest_data_off_in_tbl) {
+	return largest_data_off_in_tbl +
+	  sizeof(__u8) +  // encode struct_v
+	  sizeof(__u8) +  // encode compat_v
+	  sizeof(__u32) + // encode len
+	  sizeof(ops) +
+	  sizeof(pad_unused_bytes) +
+	  sizeof(largest_data_len) +
+	  sizeof(largest_data_off) +
+	  sizeof(largest_data_off_in_tbl) +
+	  sizeof(__u32);  // tbl length
+      }
+      return 0;  // none
+    }
+    /// offset of buffer as aligned to destination within object.
+    int get_data_alignment() {
+      if (!largest_data_len)
+	return -1;
+      return (largest_data_off - get_data_offset()) & ~CEPH_PAGE_MASK;
+    }
+    /// Is the Transaction empty (no operations)
+    bool empty() {
+      return !ops;
+    }
+    /// Number of operations in the transation
+    int get_num_ops() {
+      return ops;
+    }
+
+    /**
+     * iterator
+     *
+     * Helper object to parse Transactions.
+     *
+     * ObjectStore instances use this object to step down the encoded
+     * buffer decoding operation codes and parameters as we go.
+     *
+     */
+    class iterator {
+      bufferlist::iterator p;
+      bool sobject_encoding;
+      int64_t pool_override;
+      bool use_pool_override;
+      bool replica;
+      bool _tolerate_collection_add_enoent;
+
+      iterator(Transaction *t)
+	: p(t->tbl.begin()),
+	  sobject_encoding(t->sobject_encoding),
+	  pool_override(t->pool_override),
+	  use_pool_override(t->use_pool_override),
+	  replica(t->replica),
+	  _tolerate_collection_add_enoent(
+	    t->tolerate_collection_add_enoent) {}
+
+      friend class Transaction;
+
+    public:
+      bool tolerate_collection_add_enoent() const {
+	return _tolerate_collection_add_enoent;
+      }
+      /// true if there are more operations left to be enumerated
+      bool have_op() {
+	return !p.end();
+      }
+
+      /* Decode the specified type of object from the input
+       * stream. There is no checking that the encoded data is of the
+       * correct type.
+       */
+      int get_op() {
+	__u32 op;
+	::decode(op, p);
+	return op;
+      }
+      void get_bl(bufferlist& bl) {
+	::decode(bl, p);
+      }
+      /// Get an oid, recognize various legacy forms and update them.
+      ghobject_t get_oid() {
+	ghobject_t oid;
+	if (sobject_encoding) {
+	  sobject_t soid;
+	  ::decode(soid, p);
+	  oid.hobj.snap = soid.snap;
+	  oid.hobj.oid = soid.oid;
+	  oid.generation = ghobject_t::NO_GEN;
+	  oid.shard_id = ghobject_t::NO_SHARD;
+	} else {
+	  ::decode(oid, p);
+	  if (use_pool_override && pool_override != -1 &&
+	      !oid.hobj.is_max() && oid.hobj.pool == -1) {
+	    oid.hobj.pool = pool_override;
+	  }
+	}
+	return oid;
+      }
+      coll_t get_cid() {
+	coll_t c;
+	::decode(c, p);
+	return c;
+      }
+      uint64_t get_length() {
+	uint64_t len;
+	::decode(len, p);
+	return len;
+      }
+      string get_attrname() {
+	string s;
+	::decode(s, p);
+	return s;
+      }
+      string get_key() {
+	string s;
+	::decode(s, p);
+	return s;
+      }
+      void get_attrset(map& aset) {
+	::decode(aset, p);
+      }
+      void get_attrset(map& aset) {
+	::decode(aset, p);
+      }
+      void get_keyset(set &keys) {
+	::decode(keys, p);
+      }
+      uint32_t get_u32() {
+	uint32_t bits;
+	::decode(bits, p);
+	return bits;
+      }
+      bool get_replica() { return replica; }
+    };
+
+    iterator begin() {
+      return iterator(this);
+    }
+
+    /**
+     * Helper functions to encode the various mutation elements of a
+     * transaction.  These are 1:1 with the operation codes (see
+     * enumeration above).  These routines ensure that the
+     * encoder/creator of a transaction gets the right data in the
+     * right place. Sadly, there's no corresponding version nor any
+     * form of seat belts for the decoder.
+     */
+
+    /// Commence a global file system sync operation.
+    void start_sync() {
+      __u32 op = OP_STARTSYNC;
+      ::encode(op, tbl);
+      ops++;
+    }
+    /// noop. 'nuf said
+    void nop() {
+      __u32 op = OP_NOP;
+      ::encode(op, tbl);
+      ops++;
+    }
+    /**
+     * touch
+     *
+     * Ensure the existance of an object in a collection. Create an
+     * empty object if necessary
+     */
+    void touch(coll_t cid, const ghobject_t& oid) {
+      __u32 op = OP_TOUCH;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ops++;
+    }
+    /**
+     * Write data to an offset within an object. If the object is too
+     * small, it is expanded as needed.  It is possible to specify an
+     * offset beyond the current end of an object and it will be
+     * expanded as needed. Simple implementations of ObjectStore will
+     * just zero the data between the old end of the object and the
+     * newly provided data. More sophisticated implementations of
+     * ObjectStore will omit the untouched data and store it as a
+     * "hole" in the file.
+     */
+    void write(coll_t cid, const ghobject_t& oid, uint64_t off, uint64_t len,
+	       const bufferlist& data) {
+      __u32 op = OP_WRITE;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(off, tbl);
+      ::encode(len, tbl);
+      assert(len == data.length());
+      if (data.length() > largest_data_len) {
+	largest_data_len = data.length();
+	largest_data_off = off;
+	largest_data_off_in_tbl = tbl.length() + sizeof(__u32);  // we are about to
+      }
+      ::encode(data, tbl);
+      ops++;
+    }
+    /**
+     * zero out the indicated byte range within an object. Some
+     * ObjectStore instances may optimize this to release the
+     * underlying storage space.
+     */
+    void zero(coll_t cid, const ghobject_t& oid, uint64_t off, uint64_t len) {
+      __u32 op = OP_ZERO;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(off, tbl);
+      ::encode(len, tbl);
+      ops++;
+    }
+    /// Discard all data in the object beyond the specified size.
+    void truncate(coll_t cid, const ghobject_t& oid, uint64_t off) {
+      __u32 op = OP_TRUNCATE;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(off, tbl);
+      ops++;
+    }
+    /// Remove an object. All four parts of the object are removed.
+    void remove(coll_t cid, const ghobject_t& oid) {
+      __u32 op = OP_REMOVE;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ops++;
+    }
+    /// Set an xattr of an object
+    void setattr(coll_t cid, const ghobject_t& oid, const char* name, bufferlist& val) {
+      string n(name);
+      setattr(cid, oid, n, val);
+    }
+    /// Set an xattr of an object
+    void setattr(coll_t cid, const ghobject_t& oid, const string& s, bufferlist& val) {
+      __u32 op = OP_SETATTR;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(s, tbl);
+      ::encode(val, tbl);
+      ops++;
+    }
+    /// Set multiple xattrs of an object
+    void setattrs(coll_t cid, const ghobject_t& oid, map& attrset) {
+      __u32 op = OP_SETATTRS;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(attrset, tbl);
+      ops++;
+    }
+    /// Set multiple xattrs of an object
+    void setattrs(coll_t cid, const ghobject_t& oid, map& attrset) {
+      __u32 op = OP_SETATTRS;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(attrset, tbl);
+      ops++;
+    }
+    /// remove an xattr from an object
+    void rmattr(coll_t cid, const ghobject_t& oid, const char *name) {
+      string n(name);
+      rmattr(cid, oid, n);
+    }
+    /// remove an xattr from an object
+    void rmattr(coll_t cid, const ghobject_t& oid, const string& s) {
+      __u32 op = OP_RMATTR;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(s, tbl);
+      ops++;
+    }
+    /// remove all xattrs from an object
+    void rmattrs(coll_t cid, const ghobject_t& oid) {
+      __u32 op = OP_RMATTRS;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ops++;
+    }
+    /**
+     * Clone an object into another object.
+     *
+     * Low-cost (e.g., O(1)) cloning (if supported) is best, but
+     * fallback to an O(n) copy is allowed.  All four parts of the
+     * object are cloned (data, xattrs, omap header, omap
+     * entries).
+     *
+     * The destination named object may already exist in
+     * which case its previous contents are discarded.
+     */
+    void clone(coll_t cid, const ghobject_t& oid, ghobject_t noid) {
+      __u32 op = OP_CLONE;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(noid, tbl);
+      ops++;
+    }
+    /**
+     * Clone a byte range from one object to another.
+     *
+     * The data portion of the destination object receives a copy of a
+     * portion of the data from the source object. None of the other
+     * three parts of an object is copied from the source.
+     */
+    void clone_range(coll_t cid, const ghobject_t& oid, ghobject_t noid,
+		     uint64_t srcoff, uint64_t srclen, uint64_t dstoff) {
+      __u32 op = OP_CLONERANGE2;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(noid, tbl);
+      ::encode(srcoff, tbl);
+      ::encode(srclen, tbl);
+      ::encode(dstoff, tbl);
+      ops++;
+    }
+    /// Create the collection
+    void create_collection(coll_t cid) {
+      __u32 op = OP_MKCOLL;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ops++;
+    }
+    /// remove the collection, the collection must be empty
+    void remove_collection(coll_t cid) {
+      __u32 op = OP_RMCOLL;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ops++;
+    }
+    /**
+     * Add object to another collection (DEPRECATED)
+     *
+     * The Object is added to the new collection. This is a virtual
+     * add, we now have two names for the same object.  This is only
+     * used for conversion of old stores to new stores and is not
+     * needed for new implementations unless they expect to make use
+     * of the conversion infrastructure.
+     */
+    void collection_add(coll_t cid, coll_t ocid, const ghobject_t& oid) {
+      __u32 op = OP_COLL_ADD;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(ocid, tbl);
+      ::encode(oid, tbl);
+      ops++;
+    }
+    void collection_remove(coll_t cid, const ghobject_t& oid) {
+      __u32 op = OP_COLL_REMOVE;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ops++;
+    }
+    void collection_move(coll_t cid, coll_t oldcid, const ghobject_t& oid) {
+      collection_add(cid, oldcid, oid);
+      collection_remove(oldcid, oid);
+      return;
+    }
+    void collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
+				coll_t cid, const ghobject_t& oid) {
+      __u32 op = OP_COLL_MOVE_RENAME;
+      ::encode(op, tbl);
+      ::encode(oldcid, tbl);
+      ::encode(oldoid, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ops++;
+    }
+
+    /// Set an xattr on a collection
+    void collection_setattr(coll_t cid, const char* name, bufferlist& val) {
+      string n(name);
+      collection_setattr(cid, n, val);
+    }
+    /// Set an xattr on a collection
+    void collection_setattr(coll_t cid, const string& name, bufferlist& val) {
+      __u32 op = OP_COLL_SETATTR;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(name, tbl);
+      ::encode(val, tbl);
+      ops++;
+    }
+
+    /// Remove an xattr from a collection
+    void collection_rmattr(coll_t cid, const char* name) {
+      string n(name);
+      collection_rmattr(cid, n);
+    }
+    /// Remove an xattr from a collection
+    void collection_rmattr(coll_t cid, const string& name) {
+      __u32 op = OP_COLL_RMATTR;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(name, tbl);
+      ops++;
+    }
+    /// Set multiple xattrs on a collection
+    void collection_setattrs(coll_t cid, map& aset) {
+      __u32 op = OP_COLL_SETATTRS;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(aset, tbl);
+      ops++;
+    }
+    /// Set multiple xattrs on a collection
+    void collection_setattrs(coll_t cid, map& aset) {
+      __u32 op = OP_COLL_SETATTRS;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(aset, tbl);
+      ops++;
+    }
+    /// Change the name of a collection
+    void collection_rename(coll_t cid, coll_t ncid) {
+      __u32 op = OP_COLL_RENAME;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(ncid, tbl);
+      ops++;
+    }
+
+    /// Remove omap from oid
+    void omap_clear(
+      coll_t cid,           ///< [in] Collection containing oid
+      const ghobject_t &oid  ///< [in] Object from which to remove omap
+      ) {
+      __u32 op = OP_OMAP_CLEAR;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ops++;
+    }
+    /// Set keys on oid omap.  Replaces duplicate keys.
+    void omap_setkeys(
+      coll_t cid,                           ///< [in] Collection containing oid
+      const ghobject_t &oid,                ///< [in] Object to update
+      const map &attrset ///< [in] Replacement keys and values
+      ) {
+      __u32 op = OP_OMAP_SETKEYS;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(attrset, tbl);
+      ops++;
+    }
+    /// Remove keys from oid omap
+    void omap_rmkeys(
+      coll_t cid,             ///< [in] Collection containing oid
+      const ghobject_t &oid,  ///< [in] Object from which to remove the omap
+      const set &keys ///< [in] Keys to clear
+      ) {
+      __u32 op = OP_OMAP_RMKEYS;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(keys, tbl);
+      ops++;
+    }
+
+    /// Remove key range from oid omap
+    void omap_rmkeyrange(
+      coll_t cid,             ///< [in] Collection containing oid
+      const ghobject_t &oid,  ///< [in] Object from which to remove the omap keys
+      const string& first,    ///< [in] first key in range
+      const string& last      ///< [in] first key past range, range is [first,last)
+      ) {
+      __u32 op = OP_OMAP_RMKEYRANGE;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(first, tbl);
+      ::encode(last, tbl);
+      ops++;
+    }
+
+    /// Set omap header
+    void omap_setheader(
+      coll_t cid,             ///< [in] Collection containing oid
+      const ghobject_t &oid,  ///< [in] Object
+      const bufferlist &bl    ///< [in] Header value
+      ) {
+      __u32 op = OP_OMAP_SETHEADER;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(bl, tbl);
+      ops++;
+    }
+
+    /// Split collection based on given prefixes, objects matching the specified bits/rem are
+    /// moved to the new collection
+    void split_collection(
+      coll_t cid,
+      uint32_t bits,
+      uint32_t rem,
+      coll_t destination) {
+      __u32 op = OP_SPLIT_COLLECTION2;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(bits, tbl);
+      ::encode(rem, tbl);
+      ::encode(destination, tbl);
+      ++ops;
+    }
+
+    void set_alloc_hint(
+      coll_t cid,
+      const ghobject_t &oid,
+      uint64_t expected_object_size,
+      uint64_t expected_write_size
+    ) {
+      __u32 op = OP_SETALLOCHINT;
+      ::encode(op, tbl);
+      ::encode(cid, tbl);
+      ::encode(oid, tbl);
+      ::encode(expected_object_size, tbl);
+      ::encode(expected_write_size, tbl);
+      ++ops;
+    }
+
+    // etc.
+    Transaction() :
+      ops(0), pad_unused_bytes(0), largest_data_len(0), largest_data_off(0), largest_data_off_in_tbl(0),
+      sobject_encoding(false), pool_override(-1), use_pool_override(false),
+      replica(false),
+      tolerate_collection_add_enoent(false) {}
+
+    Transaction(bufferlist::iterator &dp) :
+      ops(0), pad_unused_bytes(0), largest_data_len(0), largest_data_off(0), largest_data_off_in_tbl(0),
+      sobject_encoding(false), pool_override(-1), use_pool_override(false),
+      replica(false),
+      tolerate_collection_add_enoent(false) {
+      decode(dp);
+    }
+
+    Transaction(bufferlist &nbl) :
+      ops(0), pad_unused_bytes(0), largest_data_len(0), largest_data_off(0), largest_data_off_in_tbl(0),
+      sobject_encoding(false), pool_override(-1), use_pool_override(false),
+      replica(false),
+      tolerate_collection_add_enoent(false) {
+      bufferlist::iterator dp = nbl.begin();
+      decode(dp);
+    }
+
+    void encode(bufferlist& bl) const {
+      ENCODE_START(7, 5, bl);
+      ::encode(ops, bl);
+      ::encode(pad_unused_bytes, bl);
+      ::encode(largest_data_len, bl);
+      ::encode(largest_data_off, bl);
+      ::encode(largest_data_off_in_tbl, bl);
+      ::encode(tbl, bl);
+      ::encode(tolerate_collection_add_enoent, bl);
+      ENCODE_FINISH(bl);
+    }
+    void decode(bufferlist::iterator &bl) {
+      DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
+      DECODE_OLDEST(2);
+      if (struct_v < 4)
+	sobject_encoding = true;
+      else
+	sobject_encoding = false;
+      ::decode(ops, bl);
+      ::decode(pad_unused_bytes, bl);
+      if (struct_v >= 3) {
+	::decode(largest_data_len, bl);
+	::decode(largest_data_off, bl);
+	::decode(largest_data_off_in_tbl, bl);
+      }
+      ::decode(tbl, bl);
+      if (struct_v < 6) {
+	use_pool_override = true;
+      }
+      if (struct_v >= 7) {
+	::decode(tolerate_collection_add_enoent, bl);
+      }
+      DECODE_FINISH(bl);
+    }
+
+    void dump(ceph::Formatter *f);
+    static void generate_test_instances(list& o);
+  };
+
+  struct C_DeleteTransaction : public Context {
+    ObjectStore::Transaction *t;
+    C_DeleteTransaction(ObjectStore::Transaction *tt) : t(tt) {}
+    void finish(int r) {
+      delete t;
+    }
+  };
+  template
+  struct C_DeleteTransactionHolder : public Context {
+    ObjectStore::Transaction *t;
+    T obj;
+    C_DeleteTransactionHolder(ObjectStore::Transaction *tt, T &obj) :
+      t(tt), obj(obj) {}
+    void finish(int r) {
+      delete t;
+    }
+  };
+
+  // synchronous wrappers
+  unsigned apply_transaction(Transaction& t, Context *ondisk=0) {
+    list tls;
+    tls.push_back(&t);
+    return apply_transactions(NULL, tls, ondisk);
+  }
+  unsigned apply_transaction(Sequencer *osr, Transaction& t, Context *ondisk=0) {
+    list tls;
+    tls.push_back(&t);
+    return apply_transactions(osr, tls, ondisk);
+  }
+  unsigned apply_transactions(list& tls, Context *ondisk=0) {
+    return apply_transactions(NULL, tls, ondisk);
+  }
+  unsigned apply_transactions(Sequencer *osr, list& tls, Context *ondisk=0);
+
+  int queue_transaction_and_cleanup(Sequencer *osr, Transaction* t,
+				    ThreadPool::TPHandle *handle = NULL) {
+    list tls;
+    tls.push_back(t);
+    return queue_transactions(osr, tls, new C_DeleteTransaction(t),
+	                      NULL, NULL, TrackedOpRef(), handle);
+  }
+
+  int queue_transaction(Sequencer *osr, Transaction *t, Context *onreadable, Context *ondisk=0,
+				Context *onreadable_sync=0,
+				TrackedOpRef op = TrackedOpRef(),
+				ThreadPool::TPHandle *handle = NULL) {
+    list tls;
+    tls.push_back(t);
+    return queue_transactions(osr, tls, onreadable, ondisk, onreadable_sync,
+	                      op, handle);
+  }
+
+  int queue_transactions(Sequencer *osr, list& tls,
+			 Context *onreadable, Context *ondisk=0,
+			 Context *onreadable_sync=0,
+			 TrackedOpRef op = TrackedOpRef(),
+			 ThreadPool::TPHandle *handle = NULL) {
+    assert(!tls.empty());
+    tls.back()->register_on_applied(onreadable);
+    tls.back()->register_on_commit(ondisk);
+    tls.back()->register_on_applied_sync(onreadable_sync);
+    return queue_transactions(osr, tls, op, handle);
+  }
+
+  virtual int queue_transactions(
+    Sequencer *osr, list& tls,
+    TrackedOpRef op = TrackedOpRef(),
+    ThreadPool::TPHandle *handle = NULL) = 0;
+
+
+  int queue_transactions(
+    Sequencer *osr,
+    list& tls,
+    Context *onreadable,
+    Context *oncommit,
+    Context *onreadable_sync,
+    Context *oncomplete,
+    TrackedOpRef op);
+
+  int queue_transaction(
+    Sequencer *osr,
+    Transaction* t,
+    Context *onreadable,
+    Context *oncommit,
+    Context *onreadable_sync,
+    Context *oncomplete,
+    TrackedOpRef op) {
+    list tls;
+    tls.push_back(t);
+    return queue_transactions(
+      osr, tls, onreadable, oncommit, onreadable_sync, oncomplete, op);
+  }
+
+ public:
+  ObjectStore(const std::string& path_) : path(path_), logger(NULL) {}
+  virtual ~ObjectStore() {}
+
+  // no copying
+  ObjectStore(const ObjectStore& o);
+  const ObjectStore& operator=(const ObjectStore& o);
+
+  // mgmt
+  virtual int version_stamp_is_valid(uint32_t *version) { return 1; }
+  virtual int update_version_stamp() = 0;
+  virtual bool test_mount_in_use() = 0;
+  virtual int mount() = 0;
+  virtual int umount() = 0;
+  virtual int get_max_object_name_length() = 0;
+  virtual int mkfs() = 0;  // wipe
+  virtual int mkjournal() = 0; // journal only
+  virtual void set_allow_sharded_objects() = 0;
+  virtual bool get_allow_sharded_objects() = 0;
+
+  virtual int statfs(struct statfs *buf) = 0;
+
+  /**
+   * get the most recent "on-disk format version" supported
+   */
+  virtual uint32_t get_target_version() = 0;
+
+  /**
+   * check the journal uuid/fsid, without opening
+   */
+  virtual int peek_journal_fsid(uuid_d *fsid) = 0;
+
+  /**
+   * write_meta - write a simple configuration key out-of-band
+   *
+   * Write a simple key/value pair for basic store configuration
+   * (e.g., a uuid or magic number) to an unopened/unmounted store.
+   * The default implementation writes this to a plaintext file in the
+   * path.
+   *
+   * A newline is appended.
+   *
+   * @param key key name (e.g., "fsid")
+   * @param value value (e.g., a uuid rendered as a string)
+   * @returns 0 for success, or an error code
+   */
+  virtual int write_meta(const std::string& key,
+			 const std::string& value);
+
+  /**
+   * read_meta - read a simple configuration key out-of-band
+   *
+   * Read a simple key value to an unopened/mounted store.
+   *
+   * Trailing whitespace is stripped off.
+   *
+   * @param key key name
+   * @param value pointer to value string
+   * @returns 0 for success, or an error code
+   */
+  virtual int read_meta(const std::string& key,
+			std::string *value);
+
+  /**
+   * get ideal min value for collection_list_partial()
+   *
+   * default to some arbitrary values; the implementation will override.
+   */
+  virtual int get_ideal_list_min() { return 32; }
+
+  /**
+   * get ideal max value for collection_list_partial()
+   *
+   * default to some arbitrary values; the implementation will override.
+   */
+  virtual int get_ideal_list_max() { return 64; }
+
+  /**
+   * Synchronous read operations
+   */
+
+
+  /**
+   * exists -- Test for existance of object
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @returns true if object exists, false otherwise
+   */
+  virtual bool exists(coll_t cid, const ghobject_t& oid) = 0;                   // useful?
+
+  /**
+   * stat -- get information for an object
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @param st output information for the object
+   * @param allow_eio if false, assert on -EIO operation failure
+   * @returns 0 on success, negative error code on failure.
+   */
+  virtual int stat(
+    coll_t cid,
+    const ghobject_t& oid,
+    struct stat *st,
+    bool allow_eio = false) = 0; // struct stat?
+
+  /**
+   * read -- read a byte range of data from an object
+   *
+   * Note: if reading from an offset past the end of the object, we
+   * return 0 (not, say, -EINVAL).
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @param offset location offset of first byte to be read
+   * @param len number of bytes to be read
+   * @param bl output bufferlist
+   * @param allow_eio if false, assert on -EIO operation failure
+   * @returns number of bytes read on success, or negative error code on failure.
+   */
+   virtual int read(
+    coll_t cid,
+    const ghobject_t& oid,
+    uint64_t offset,
+    size_t len,
+    bufferlist& bl,
+    bool allow_eio = false) = 0;
+
+  /**
+   * fiemap -- get extent map of data of an object
+   *
+   * Returns an encoded map of the extents of an object's data portion
+   * (map).
+   *
+   * A non-enlightend implementation is free to return the extent (offset, len)
+   * as the sole extent.
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @param offset location offset of first byte to be read
+   * @param len number of bytes to be read
+   * @param bl output bufferlist for extent map information.
+   * @returns 0 on success, negative error code on failure.
+   */
+  virtual int fiemap(coll_t cid, const ghobject_t& oid, uint64_t offset, size_t len, bufferlist& bl) = 0;
+
+  /**
+   * getattr -- get an xattr of an object
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @param name name of attr to read
+   * @param value place to put output result.
+   * @returns 0 on success, negative error code on failure.
+   */
+  virtual int getattr(coll_t cid, const ghobject_t& oid, const char *name, bufferptr& value) = 0;
+
+  /**
+   * getattr -- get an xattr of an object
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @param name name of attr to read
+   * @param value place to put output result.
+   * @returns 0 on success, negative error code on failure.
+   */
+  int getattr(coll_t cid, const ghobject_t& oid, const char *name, bufferlist& value) {
+    bufferptr bp;
+    int r = getattr(cid, oid, name, bp);
+    if (bp.length())
+      value.push_back(bp);
+    return r;
+  }
+  int getattr(
+    coll_t cid, const ghobject_t& oid,
+    const string name, bufferlist& value) {
+    bufferptr bp;
+    int r = getattr(cid, oid, name.c_str(), bp);
+    value.push_back(bp);
+    return r;
+  }
+
+  /**
+   * getattrs -- get all of the xattrs of an object
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @param aset place to put output result.
+   * @param user_only true -> only user attributes are return else all attributes are returned
+   * @returns 0 on success, negative error code on failure.
+   */
+  virtual int getattrs(coll_t cid, const ghobject_t& oid, map& aset, bool user_only = false) = 0;
+
+  /**
+   * getattrs -- get all of the xattrs of an object
+   *
+   * @param cid collection for object
+   * @param oid oid of object
+   * @param aset place to put output result.
+   * @param user_only true -> only user attributes are return else all attributes are returned
+   * @returns 0 on success, negative error code on failure.
+   */
+  int getattrs(coll_t cid, const ghobject_t& oid, map& aset, bool user_only = false) {
+    map bmap;
+    int r = getattrs(cid, oid, bmap, user_only);
+    for (map::iterator i = bmap.begin();
+	i != bmap.end();
+	++i) {
+      aset[i->first].append(i->second);
+    }
+    return r;
+  }
+
+
+  // collections
+
+  /**
+   * list_collections -- get all of the collections known to this ObjectStore
+   *
+   * @param ls list of the collections in sorted order.
+   * @returns 0 on success, negative error code on failure.
+   */
+  virtual int list_collections(vector& ls) = 0;
+
+  virtual int collection_version_current(coll_t c, uint32_t *version) {
+    *version = 0;
+    return 1;
+  }
+  /**
+   * does a collection exist?
+   *
+   * @param c collection
+   * @returns true if it exists, false otherwise
+   */
+  virtual bool collection_exists(coll_t c) = 0;
+  /**
+   * collection_getattr - get an xattr of a collection
+   *
+   * @param cid collection name
+   * @param name xattr name
+   * @param value pointer of buffer to receive value
+   * @param size size of buffer to receive value
+   * @returns 0 on success, negative error code on failure
+   */
+  virtual int collection_getattr(coll_t cid, const char *name,
+	                         void *value, size_t size) = 0;
+  /**
+   * collection_getattr - get an xattr of a collection
+   *
+   * @param cid collection name
+   * @param name xattr name
+   * @param bl buffer to receive value
+   * @returns 0 on success, negative error code on failure
+   */
+  virtual int collection_getattr(coll_t cid, const char *name, bufferlist& bl) = 0;
+  /**
+   * collection_getattrs - get all xattrs of a collection
+   *
+   * @param cid collection name
+   * @param asert map of keys and buffers that contain the values
+   * @returns 0 on success, negative error code on failure
+   */
+  virtual int collection_getattrs(coll_t cid, map &aset) = 0;
+  /**
+   * is a collection empty?
+   *
+   * @param c collection
+   * @returns true if empty, false otherwise
+   */
+  virtual bool collection_empty(coll_t c) = 0;
+
+  /**
+   * collection_list - get all objects of a collection in sorted order
+   *
+   * @param c collection name
+   * @param o [out] list of objects
+   * @returns 0 on success, negative error code on failure
+   */
+  virtual int collection_list(coll_t c, vector& o) = 0;
+
+  /**
+   * list partial contents of collection relative to a hash offset/position
+   *
+   * @param c collection
+   * @param start list objects that sort >= this value
+   * @param min return at least this many results, unless we reach the end
+   * @param max return no more than this many results
+   * @param snapid return no objects with snap < snapid
+   * @param ls [out] result
+   * @param next [out] next item sorts >= this value
+   * @return zero on success, or negative error
+   */
+  virtual int collection_list_partial(coll_t c, ghobject_t start,
+				      int min, int max, snapid_t snap,
+				      vector *ls, ghobject_t *next) = 0;
+
+  /**
+   * list contents of a collection that fall in the range [start, end)
+   *
+   * @param c collection
+   * @param start list object that sort >= this value
+   * @param end list objects that sort < this value
+   * @param snapid return no objects with snap < snapid
+   * @param ls [out] result
+   * @return zero on success, or negative error
+   */
+  virtual int collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
+	                            snapid_t seq, vector *ls) = 0;
+
+  //TODO: Remove
+  int collection_list(coll_t c, vector& o);
+
+  int collection_list_partial(coll_t c, hobject_t start,
+				      int min, int max, snapid_t snap,
+				      vector *ls, hobject_t *next);
+
+  int collection_list_range(coll_t c, hobject_t start, hobject_t end,
+	                            snapid_t seq, vector *ls);
+
+  /// OMAP
+  /// Get omap contents
+  virtual int omap_get(
+    coll_t c,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    bufferlist *header,      ///< [out] omap header
+    map *out /// < [out] Key to value map
+    ) = 0;
+
+  /// Get omap header
+  virtual int omap_get_header(
+    coll_t c,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    bufferlist *header,      ///< [out] omap header
+    bool allow_eio = false ///< [in] don't assert on eio
+    ) = 0;
+
+  /// Get keys defined on oid
+  virtual int omap_get_keys(
+    coll_t c,              ///< [in] Collection containing oid
+    const ghobject_t &oid, ///< [in] Object containing omap
+    set *keys      ///< [out] Keys defined on oid
+    ) = 0;
+
+  /// Get key values
+  virtual int omap_get_values(
+    coll_t c,                    ///< [in] Collection containing oid
+    const ghobject_t &oid,       ///< [in] Object containing omap
+    const set &keys,     ///< [in] Keys to get
+    map *out ///< [out] Returned keys and values
+    ) = 0;
+
+  /// Filters keys into out which are defined on oid
+  virtual int omap_check_keys(
+    coll_t c,                ///< [in] Collection containing oid
+    const ghobject_t &oid,   ///< [in] Object containing omap
+    const set &keys, ///< [in] Keys to check
+    set *out         ///< [out] Subset of keys defined on oid
+    ) = 0;
+
+  /**
+   * Returns an object map iterator
+   *
+   * Warning!  The returned iterator is an implicit lock on filestore
+   * operations in c.  Do not use filestore methods on c while the returned
+   * iterator is live.  (Filling in a transaction is no problem).
+   *
+   * @return iterator, null on error
+   */
+  virtual ObjectMap::ObjectMapIterator get_omap_iterator(
+    coll_t c,              ///< [in] collection
+    const ghobject_t &oid  ///< [in] object
+    ) = 0;
+
+  virtual void sync(Context *onsync) {}
+  virtual void sync() {}
+  virtual void flush() {}
+  virtual void sync_and_flush() {}
+
+  virtual int dump_journal(ostream& out) { return -EOPNOTSUPP; }
+
+  virtual int snapshot(const string& name) { return -EOPNOTSUPP; }
+
+  /**
+   * Set and get internal fsid for this instance. No external data is modified
+   */
+  virtual void set_fsid(uuid_d u) = 0;
+  virtual uuid_d get_fsid() = 0;
+
+  // DEBUG
+  virtual void inject_data_error(const ghobject_t &oid) {}
+  virtual void inject_mdata_error(const ghobject_t &oid) {}
+};
+WRITE_CLASS_ENCODER(ObjectStore::Transaction)
+
+ostream& operator<<(ostream& out, const ObjectStore::Sequencer& s);
+
+#endif
diff --git a/ceph/src/os/SequencerPosition.h b/ceph/src/os/SequencerPosition.h
new file mode 100644
index 00000000..38f11f08
--- /dev/null
+++ b/ceph/src/os/SequencerPosition.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#ifndef __CEPH_OS_SEQUENCERPOSITION_H
+#define __CEPH_OS_SEQUENCERPOSITION_H
+
+#include "include/types.h"
+#include "include/cmp.h"
+#include "include/encoding.h"
+#include "common/Formatter.h"
+
+#include 
+
+/**
+ * transaction and op offset
+ */
+struct SequencerPosition {
+  uint64_t seq;  ///< seq
+  uint32_t trans; ///< transaction in that seq (0-based)
+  uint32_t op;    ///< op in that transaction (0-based)
+
+  SequencerPosition(uint64_t s=0, int32_t t=0, int32_t o=0) : seq(s), trans(t), op(o) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(seq, bl);
+    ::encode(trans, bl);
+    ::encode(op, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& p) {
+    DECODE_START(1, p);
+    ::decode(seq, p);
+    ::decode(trans, p);
+    ::decode(op, p);
+    DECODE_FINISH(p);
+  }
+  void dump(Formatter *f) const {
+    f->dump_unsigned("seq", seq);
+    f->dump_unsigned("trans", trans);
+    f->dump_unsigned("op", op);
+  }
+  static void generate_test_instances(list& o) {
+    o.push_back(new SequencerPosition);
+    o.push_back(new SequencerPosition(1, 2, 3));
+    o.push_back(new SequencerPosition(4, 5, 6));
+  }
+};
+WRITE_CLASS_ENCODER(SequencerPosition)
+
+inline ostream& operator<<(ostream& out, const SequencerPosition& t) {
+  return out << t.seq << "." << t.trans << "." << t.op;
+}
+
+WRITE_EQ_OPERATORS_3(SequencerPosition, seq, trans, op)
+WRITE_CMP_OPERATORS_3(SequencerPosition, seq, trans, op)
+
+
+#endif
diff --git a/ceph/src/os/Transaction.cc b/ceph/src/os/Transaction.cc
new file mode 100644
index 00000000..cdfdf0e1
--- /dev/null
+++ b/ceph/src/os/Transaction.cc
@@ -0,0 +1,471 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#include "ObjectStore.h"
+#include "common/Formatter.h"
+
+void ObjectStore::Transaction::dump(ceph::Formatter *f)
+{
+  f->open_array_section("ops");
+  iterator i = begin();
+  int op_num = 0;
+  bool stop_looping = false;
+  while (i.have_op() && !stop_looping) {
+    int op = i.get_op();
+    f->open_object_section("op");
+    f->dump_int("op_num", op_num);
+
+    switch (op) {
+    case Transaction::OP_NOP:
+      f->dump_string("op_name", "nop");
+      break;
+    case Transaction::OP_TOUCH:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	f->dump_string("op_name", "touch");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+      }
+      break;
+      
+    case Transaction::OP_WRITE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	bufferlist bl;
+	i.get_bl(bl);
+	f->dump_string("op_name", "write");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+        f->dump_unsigned("length", len);
+        f->dump_unsigned("offset", off);
+        f->dump_unsigned("bufferlist length", bl.length());
+      }
+      break;
+      
+    case Transaction::OP_ZERO:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	f->dump_string("op_name", "zero");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+        f->dump_unsigned("offset", off);
+	f->dump_unsigned("length", len);
+      }
+      break;
+      
+    case Transaction::OP_TRIMCACHE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	f->dump_string("op_name", "trim_cache");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->dump_unsigned("offset", off);
+	f->dump_unsigned("length", len);
+      }
+      break;
+      
+    case Transaction::OP_TRUNCATE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	uint64_t off = i.get_length();
+	f->dump_string("op_name", "truncate");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->dump_unsigned("offset", off);
+      }
+      break;
+      
+    case Transaction::OP_REMOVE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	f->dump_string("op_name", "remove");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+      }
+      break;
+      
+    case Transaction::OP_SETATTR:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	string name = i.get_attrname();
+	bufferlist bl;
+	i.get_bl(bl);
+	f->dump_string("op_name", "setattr");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->dump_string("name", name);
+	f->dump_unsigned("length", bl.length());
+      }
+      break;
+      
+    case Transaction::OP_SETATTRS:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	map aset;
+	i.get_attrset(aset);
+	f->dump_string("op_name", "setattrs");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->open_object_section("attr_lens");
+	for (map::iterator p = aset.begin();
+	    p != aset.end(); ++p) {
+	  f->dump_unsigned(p->first.c_str(), p->second.length());
+	}
+	f->close_section();
+      }
+      break;
+
+    case Transaction::OP_RMATTR:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	string name = i.get_attrname();
+	f->dump_string("op_name", "rmattr");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->dump_string("name", name);
+      }
+      break;
+
+    case Transaction::OP_RMATTRS:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	f->dump_string("op_name", "rmattrs");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+      }
+      break;
+      
+    case Transaction::OP_CLONE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+	f->dump_string("op_name", "clone");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("src_oid") << oid;
+	f->dump_stream("dst_oid") << noid;
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+ 	uint64_t off = i.get_length();
+	uint64_t len = i.get_length();
+	f->dump_string("op_name", "clonerange");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("src_oid") << oid;
+	f->dump_stream("dst_oid") << noid;
+	f->dump_unsigned("offset", off);
+	f->dump_unsigned("len", len);
+      }
+      break;
+
+    case Transaction::OP_CLONERANGE2:
+      {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	ghobject_t noid = i.get_oid();
+ 	uint64_t srcoff = i.get_length();
+	uint64_t len = i.get_length();
+ 	uint64_t dstoff = i.get_length();
+	f->dump_string("op_name", "clonerange2");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("src_oid") << oid;
+	f->dump_stream("dst_oid") << noid;
+	f->dump_unsigned("src_offset", srcoff);
+	f->dump_unsigned("len", len);
+	f->dump_unsigned("dst_offset", dstoff);
+      }
+      break;
+
+    case Transaction::OP_MKCOLL:
+      {
+	coll_t cid = i.get_cid();
+	f->dump_string("op_name", "mkcoll");
+	f->dump_stream("collection") << cid;
+      }
+      break;
+
+    case Transaction::OP_RMCOLL:
+      {
+	coll_t cid = i.get_cid();
+	f->dump_string("op_name", "rmcoll");
+	f->dump_stream("collection") << cid;
+      }
+      break;
+
+    case Transaction::OP_COLL_ADD:
+      {
+	coll_t ncid = i.get_cid();
+	coll_t ocid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	f->dump_string("op_name", "collection_add");
+	f->dump_stream("src_collection") << ocid;
+	f->dump_stream("dst_collection") << ncid;
+	f->dump_stream("oid") << oid;
+      }
+      break;
+
+    case Transaction::OP_COLL_REMOVE:
+       {
+	coll_t cid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	f->dump_string("op_name", "collection_remove");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+       }
+      break;
+
+    case Transaction::OP_COLL_MOVE:
+       {
+	coll_t ocid = i.get_cid();
+	coll_t ncid = i.get_cid();
+	ghobject_t oid = i.get_oid();
+	f->open_object_section("collection_move");
+	f->dump_stream("src_collection") << ocid;
+	f->dump_stream("dst_collection") << ncid;
+	f->dump_stream("oid") << oid;
+	f->close_section();
+       }
+      break;
+
+
+    case Transaction::OP_COLL_SETATTR:
+      {
+	coll_t cid = i.get_cid();
+	string name = i.get_attrname();
+	bufferlist bl;
+	i.get_bl(bl);
+	f->dump_string("op_name", "collection_setattr");
+	f->dump_stream("collection") << cid;
+	f->dump_string("name", name);
+	f->dump_unsigned("length", bl.length());
+      }
+      break;
+
+    case Transaction::OP_COLL_RMATTR:
+      {
+	coll_t cid = i.get_cid();
+	string name = i.get_attrname();
+	f->dump_string("op_name", "collection_rmattr");
+	f->dump_stream("collection") << cid;
+	f->dump_string("name", name);
+      }
+      break;
+
+    case Transaction::OP_STARTSYNC:
+      f->dump_string("op_name", "startsync");
+      break;
+
+    case Transaction::OP_COLL_RENAME:
+      {
+	coll_t cid(i.get_cid());
+	coll_t ncid(i.get_cid());
+	f->dump_string("op_name", "collection_rename");
+	f->dump_stream("src_collection") << cid;
+	f->dump_stream("dst_collection") << ncid;
+      }
+      break;
+
+    case Transaction::OP_OMAP_CLEAR:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	f->dump_string("op_name", "omap_clear");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+      }
+      break;
+
+    case Transaction::OP_OMAP_SETKEYS:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	map aset;
+	i.get_attrset(aset);
+	f->dump_string("op_name", "omap_setkeys");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->open_object_section("attr_lens");
+	for (map::iterator p = aset.begin();
+	    p != aset.end(); ++p) {
+	  f->dump_unsigned(p->first.c_str(), p->second.length());
+	}
+	f->close_section();
+      }
+      break;
+
+    case Transaction::OP_OMAP_RMKEYS:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	set keys;
+	i.get_keyset(keys);
+	f->dump_string("op_name", "omap_rmkeys");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+      }
+      break;
+
+    case Transaction::OP_OMAP_SETHEADER:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	bufferlist bl;
+	i.get_bl(bl);
+	f->dump_string("op_name", "omap_setheader");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->dump_stream("header_length") << bl.length();
+      }
+      break;
+
+    case Transaction::OP_SPLIT_COLLECTION:
+      {
+	coll_t cid(i.get_cid());
+	uint32_t bits(i.get_u32());
+	uint32_t rem(i.get_u32());
+	coll_t dest(i.get_cid());
+	f->dump_string("op_name", "op_split_collection_create");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("bits") << bits;
+	f->dump_stream("rem") << rem;
+	f->dump_stream("dest") << dest;
+      }
+      break;
+
+    case Transaction::OP_SPLIT_COLLECTION2:
+      {
+	coll_t cid(i.get_cid());
+	uint32_t bits(i.get_u32());
+	uint32_t rem(i.get_u32());
+	coll_t dest(i.get_cid());
+	f->dump_string("op_name", "op_split_collection");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("bits") << bits;
+	f->dump_stream("rem") << rem;
+	f->dump_stream("dest") << dest;
+      }
+      break;
+
+    case Transaction::OP_OMAP_RMKEYRANGE:
+      {
+	coll_t cid(i.get_cid());
+	ghobject_t oid = i.get_oid();
+	string first, last;
+	first = i.get_key();
+	last = i.get_key();
+	f->dump_string("op_name", "op_omap_rmkeyrange");
+	f->dump_stream("collection") << cid;
+	f->dump_stream("oid") << oid;
+	f->dump_string("first", first);
+	f->dump_string("last", last);
+      }
+      break;
+
+    case Transaction::OP_COLL_MOVE_RENAME:
+      {
+	coll_t old_cid(i.get_cid());
+	ghobject_t old_oid = i.get_oid();
+	coll_t new_cid(i.get_cid());
+	ghobject_t new_oid = i.get_oid();
+	f->dump_string("op_name", "op_coll_move_rename");
+	f->dump_stream("old_collection") << old_cid;
+	f->dump_stream("old_oid") << old_oid;
+	f->dump_stream("new_collection") << new_cid;
+	f->dump_stream("new_oid") << new_oid;
+      }
+      break;
+
+    case Transaction::OP_SETALLOCHINT:
+      {
+        coll_t cid = i.get_cid();
+        ghobject_t oid = i.get_oid();
+        uint64_t expected_object_size = i.get_length();
+        uint64_t expected_write_size = i.get_length();
+        f->dump_string("op_name", "op_setallochint");
+        f->dump_stream("collection") << cid;
+        f->dump_stream("oid") << oid;
+        f->dump_stream("expected_object_size") << expected_object_size;
+        f->dump_stream("expected_write_size") << expected_write_size;
+      }
+      break;
+
+    default:
+      f->dump_string("op_name", "unknown");
+      f->dump_unsigned("op_code", op);
+      stop_looping = true;
+      break;
+    }
+    f->close_section();
+    op_num++;
+  }
+  f->close_section();
+}
+
+void ObjectStore::Transaction::generate_test_instances(list& o)
+{
+  o.push_back(new Transaction);
+
+  Transaction *t = new Transaction;
+  t->nop();
+  o.push_back(t);
+  
+  t = new Transaction;
+  coll_t c("foocoll");
+  coll_t c2("foocoll2");
+  ghobject_t o1(hobject_t("obj", "", 123, 456, -1, ""));
+  ghobject_t o2(hobject_t("obj2", "", 123, 456, -1, ""));
+  ghobject_t o3(hobject_t("obj3", "", 123, 456, -1, ""));
+  t->touch(c, o1);
+  bufferlist bl;
+  bl.append("some data");
+  t->write(c, o1, 1, bl.length(), bl);
+  t->zero(c, o1, 22, 33);
+  t->truncate(c, o1, 99);
+  t->remove(c, o1);
+  o.push_back(t);
+
+  t = new Transaction;
+  t->setattr(c, o1, "key", bl);
+  map m;
+  m["a"] = buffer::copy("this", 4);
+  m["b"] = buffer::copy("that", 4);
+  t->setattrs(c, o1, m);
+  t->rmattr(c, o1, "b");
+  t->rmattrs(c, o1);
+
+  t->clone(c, o1, o2);
+  t->clone(c, o1, o3);
+  t->clone_range(c, o1, o2, 1, 12, 99);
+
+  t->create_collection(c);
+  t->collection_add(c, c2, o1);
+  t->collection_add(c, c2, o2);
+  t->collection_move(c, c2, o3);
+  t->remove_collection(c);
+  t->collection_setattr(c, "this", bl);
+  t->collection_rmattr(c, "foo");
+  t->collection_setattrs(c, m);
+  t->collection_rename(c, c2);
+  o.push_back(t);  
+}
diff --git a/ceph/src/os/WBThrottle.cc b/ceph/src/os/WBThrottle.cc
new file mode 100644
index 00000000..0354ceb8
--- /dev/null
+++ b/ceph/src/os/WBThrottle.cc
@@ -0,0 +1,260 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "acconfig.h"
+
+#include "os/WBThrottle.h"
+#include "common/perf_counters.h"
+
+WBThrottle::WBThrottle(CephContext *cct) :
+  cur_ios(0), cur_size(0),
+  cct(cct),
+  logger(NULL),
+  stopping(true),
+  lock("WBThrottle::lock", false, true, false, cct),
+  fs(XFS)
+{
+  {
+    Mutex::Locker l(lock);
+    set_from_conf();
+  }
+  assert(cct);
+  PerfCountersBuilder b(
+    cct, string("WBThrottle"),
+    l_wbthrottle_first, l_wbthrottle_last);
+  b.add_u64(l_wbthrottle_bytes_dirtied, "bytes_dirtied");
+  b.add_u64(l_wbthrottle_bytes_wb, "bytes_wb");
+  b.add_u64(l_wbthrottle_ios_dirtied, "ios_dirtied");
+  b.add_u64(l_wbthrottle_ios_wb, "ios_wb");
+  b.add_u64(l_wbthrottle_inodes_dirtied, "inodes_dirtied");
+  b.add_u64(l_wbthrottle_inodes_wb, "inodes_wb");
+  logger = b.create_perf_counters();
+  cct->get_perfcounters_collection()->add(logger);
+  for (unsigned i = l_wbthrottle_first + 1; i != l_wbthrottle_last; ++i)
+    logger->set(i, 0);
+
+  cct->_conf->add_observer(this);
+}
+
+WBThrottle::~WBThrottle() {
+  assert(cct);
+  cct->get_perfcounters_collection()->remove(logger);
+  delete logger;
+  cct->_conf->remove_observer(this);
+}
+
+void WBThrottle::start()
+{
+  {
+    Mutex::Locker l(lock);
+    stopping = false;
+  }
+  create();
+}
+
+void WBThrottle::stop()
+{
+  {
+    Mutex::Locker l(lock);
+    stopping = true;
+    cond.Signal();
+  }
+
+  join();
+}
+
+const char** WBThrottle::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "filestore_wbthrottle_btrfs_bytes_start_flusher",
+    "filestore_wbthrottle_btrfs_bytes_hard_limit",
+    "filestore_wbthrottle_btrfs_ios_start_flusher",
+    "filestore_wbthrottle_btrfs_ios_hard_limit",
+    "filestore_wbthrottle_btrfs_inodes_start_flusher",
+    "filestore_wbthrottle_btrfs_inodes_hard_limit",
+    "filestore_wbthrottle_xfs_bytes_start_flusher",
+    "filestore_wbthrottle_xfs_bytes_hard_limit",
+    "filestore_wbthrottle_xfs_ios_start_flusher",
+    "filestore_wbthrottle_xfs_ios_hard_limit",
+    "filestore_wbthrottle_xfs_inodes_start_flusher",
+    "filestore_wbthrottle_xfs_inodes_hard_limit",
+    NULL
+  };
+  return KEYS;
+}
+
+void WBThrottle::set_from_conf()
+{
+  assert(lock.is_locked());
+  if (fs == BTRFS) {
+    size_limits.first =
+      cct->_conf->filestore_wbthrottle_btrfs_bytes_start_flusher;
+    size_limits.second =
+      cct->_conf->filestore_wbthrottle_btrfs_bytes_hard_limit;
+    io_limits.first =
+      cct->_conf->filestore_wbthrottle_btrfs_ios_start_flusher;
+    io_limits.second =
+      cct->_conf->filestore_wbthrottle_btrfs_ios_hard_limit;
+    fd_limits.first =
+      cct->_conf->filestore_wbthrottle_btrfs_inodes_start_flusher;
+    fd_limits.second =
+      cct->_conf->filestore_wbthrottle_btrfs_inodes_hard_limit;
+  } else if (fs == XFS) {
+    size_limits.first =
+      cct->_conf->filestore_wbthrottle_xfs_bytes_start_flusher;
+    size_limits.second =
+      cct->_conf->filestore_wbthrottle_xfs_bytes_hard_limit;
+    io_limits.first =
+      cct->_conf->filestore_wbthrottle_xfs_ios_start_flusher;
+    io_limits.second =
+      cct->_conf->filestore_wbthrottle_xfs_ios_hard_limit;
+    fd_limits.first =
+      cct->_conf->filestore_wbthrottle_xfs_inodes_start_flusher;
+    fd_limits.second =
+      cct->_conf->filestore_wbthrottle_xfs_inodes_hard_limit;
+  } else {
+    assert(0 == "invalid value for fs");
+  }
+  cond.Signal();
+}
+
+void WBThrottle::handle_conf_change(const md_config_t *conf,
+				    const std::set &changed)
+{
+  Mutex::Locker l(lock);
+  for (const char** i = get_tracked_conf_keys(); *i; ++i) {
+    if (changed.count(*i)) {
+      set_from_conf();
+      return;
+    }
+  }
+}
+
+bool WBThrottle::get_next_should_flush(
+  boost::tuple *next)
+{
+  assert(lock.is_locked());
+  assert(next);
+  while (!stopping &&
+         cur_ios < io_limits.first &&
+         pending_wbs.size() < fd_limits.first &&
+         cur_size < size_limits.first)
+         cond.Wait(lock);
+  if (stopping)
+    return false;
+  assert(!pending_wbs.empty());
+  ghobject_t obj(pop_object());
+  
+  map >::iterator i =
+    pending_wbs.find(obj);
+  *next = boost::make_tuple(obj, i->second.second, i->second.first);
+  pending_wbs.erase(i);
+  return true;
+}
+
+
+void *WBThrottle::entry()
+{
+  Mutex::Locker l(lock);
+  boost::tuple wb;
+  while (get_next_should_flush(&wb)) {
+    clearing = wb.get<0>();
+    lock.Unlock();
+#ifdef HAVE_FDATASYNC
+    ::fdatasync(**wb.get<1>());
+#else
+    ::fsync(**wb.get<1>());
+#endif
+#ifdef HAVE_POSIX_FADVISE
+    if (wb.get<2>().nocache) {
+      int fa_r = posix_fadvise(**wb.get<1>(), 0, 0, POSIX_FADV_DONTNEED);
+      assert(fa_r == 0);
+    }
+#endif
+    lock.Lock();
+    clearing = ghobject_t();
+    cur_ios -= wb.get<2>().ios;
+    logger->dec(l_wbthrottle_ios_dirtied, wb.get<2>().ios);
+    cur_size -= wb.get<2>().size;
+    logger->dec(l_wbthrottle_bytes_dirtied, wb.get<2>().size);
+    logger->dec(l_wbthrottle_inodes_dirtied);
+    cond.Signal();
+    wb = boost::tuple();
+  }
+  return 0;
+}
+
+void WBThrottle::queue_wb(
+  FDRef fd, const ghobject_t &hoid, uint64_t offset, uint64_t len,
+  bool nocache)
+{
+  Mutex::Locker l(lock);
+  map >::iterator wbiter =
+    pending_wbs.find(hoid);
+  if (wbiter == pending_wbs.end()) {
+    wbiter = pending_wbs.insert(
+      make_pair(hoid,
+	make_pair(
+	  PendingWB(),
+	  fd))).first;
+    logger->inc(l_wbthrottle_inodes_dirtied);
+  } else {
+    remove_object(hoid);
+  }
+
+  cur_ios++;
+  logger->inc(l_wbthrottle_ios_dirtied);
+  cur_size += len;
+  logger->inc(l_wbthrottle_bytes_dirtied, len);
+
+  wbiter->second.first.add(nocache, len, 1);
+  insert_object(hoid);
+  cond.Signal();
+}
+
+void WBThrottle::clear()
+{
+  Mutex::Locker l(lock);
+  for (map >::iterator i =
+	 pending_wbs.begin();
+       i != pending_wbs.end();
+       ++i) {
+    cur_ios -= i->second.first.ios;
+    logger->dec(l_wbthrottle_ios_dirtied, i->second.first.ios);
+    cur_size -= i->second.first.size;
+    logger->dec(l_wbthrottle_bytes_dirtied, i->second.first.size);
+    logger->dec(l_wbthrottle_inodes_dirtied);
+  }
+  pending_wbs.clear();
+  lru.clear();
+  rev_lru.clear();
+  cond.Signal();
+}
+
+void WBThrottle::clear_object(const ghobject_t &hoid)
+{
+  Mutex::Locker l(lock);
+  while (clearing == hoid)
+    cond.Wait(lock);
+  map >::iterator i =
+    pending_wbs.find(hoid);
+  if (i == pending_wbs.end())
+    return;
+
+  cur_ios -= i->second.first.ios;
+  cur_size -= i->second.first.size;
+
+  pending_wbs.erase(i);
+  remove_object(hoid);
+}
+
+void WBThrottle::throttle()
+{
+  Mutex::Locker l(lock);
+  while (!stopping && !(
+	   cur_ios < io_limits.second &&
+	   pending_wbs.size() < fd_limits.second &&
+	   cur_size < size_limits.second)) {
+    cond.Wait(lock);
+  }
+}
diff --git a/ceph/src/os/WBThrottle.h b/ceph/src/os/WBThrottle.h
new file mode 100644
index 00000000..f643e39f
--- /dev/null
+++ b/ceph/src/os/WBThrottle.h
@@ -0,0 +1,173 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef WBTHROTTLE_H
+#define WBTHROTTLE_H
+
+#include 
+#include 
+#include "include/memory.h"
+#include "include/buffer.h"
+#include "common/Formatter.h"
+#include "common/hobject.h"
+#include "include/interval_set.h"
+#include "FDCache.h"
+#include "common/Thread.h"
+#include "common/ceph_context.h"
+
+class PerfCounters;
+enum {
+  l_wbthrottle_first = 999090,
+  l_wbthrottle_bytes_dirtied,
+  l_wbthrottle_bytes_wb,
+  l_wbthrottle_ios_dirtied,
+  l_wbthrottle_ios_wb,
+  l_wbthrottle_inodes_dirtied,
+  l_wbthrottle_inodes_wb,
+  l_wbthrottle_last
+};
+
+/**
+ * WBThrottle
+ *
+ * Tracks, throttles, and flushes outstanding IO
+ */
+class WBThrottle : Thread, public md_config_obs_t {
+  ghobject_t clearing;
+
+  /* *_limits.first is the start_flusher limit and
+   * *_limits.second is the hard limit
+   */
+
+  /// Limits on unflushed bytes
+  pair size_limits;
+
+  /// Limits on unflushed ios
+  pair io_limits;
+
+  /// Limits on unflushed objects
+  pair fd_limits;
+
+  uint64_t cur_ios;  /// Currently unflushed IOs
+  uint64_t cur_size; /// Currently unflushed bytes
+
+  /**
+   * PendingWB tracks the ios pending on an object.
+   */
+  class PendingWB {
+  public:
+    bool nocache;
+    uint64_t size;
+    uint64_t ios;
+    PendingWB() : nocache(true), size(0), ios(0) {}
+    void add(bool _nocache, uint64_t _size, uint64_t _ios) {
+      if (!_nocache)
+	nocache = false; // only nocache if all writes are nocache
+      size += _size;
+      ios += _ios;
+    }
+  };
+
+  CephContext *cct;
+  PerfCounters *logger;
+  bool stopping;
+  Mutex lock;
+  Cond cond;
+
+
+  /**
+   * Flush objects in lru order
+   */
+  list lru;
+  map::iterator> rev_lru;
+  void remove_object(const ghobject_t &oid) {
+    assert(lock.is_locked());
+    map::iterator>::iterator iter =
+      rev_lru.find(oid);
+    if (iter == rev_lru.end())
+      return;
+
+    lru.erase(iter->second);
+    rev_lru.erase(iter);
+  }
+  ghobject_t pop_object() {
+    assert(!lru.empty());
+    ghobject_t oid(lru.front());
+    lru.pop_front();
+    rev_lru.erase(oid);
+    return oid;
+  }
+  void insert_object(const ghobject_t &oid) {
+    assert(rev_lru.find(oid) == rev_lru.end());
+    lru.push_back(oid);
+    rev_lru.insert(make_pair(oid, --lru.end()));
+  }
+
+  map > pending_wbs;
+
+  /// get next flush to perform
+  bool get_next_should_flush(
+    boost::tuple *next ///< [out] next to flush
+    ); ///< @return false if we are shutting down
+public:
+  enum FS {
+    BTRFS,
+    XFS
+  };
+
+private:
+  FS fs;
+
+  void set_from_conf();
+public:
+  WBThrottle(CephContext *cct);
+  ~WBThrottle();
+
+  void start();
+  void stop();
+  /// Set fs as XFS or BTRFS
+  void set_fs(FS new_fs) {
+    Mutex::Locker l(lock);
+    fs = new_fs;
+    set_from_conf();
+  }
+
+  /// Queue wb on oid, fd taking throttle (does not block)
+  void queue_wb(
+    FDRef fd,              ///< [in] FDRef to oid
+    const ghobject_t &oid, ///< [in] object
+    uint64_t offset,       ///< [in] offset written
+    uint64_t len,          ///< [in] length written
+    bool nocache           ///< [in] try to clear out of cache after write
+    );
+
+  /// Clear all wb (probably due to sync)
+  void clear();
+
+  /// Clear object
+  void clear_object(const ghobject_t &oid);
+
+  /// Block until there is throttle available
+  void throttle();
+
+  /// md_config_obs_t
+  const char** get_tracked_conf_keys() const;
+  void handle_conf_change(const md_config_t *conf,
+			  const std::set &changed);
+
+  /// Thread
+  void *entry();
+};
+
+#endif
diff --git a/ceph/src/os/XfsFileStoreBackend.cc b/ceph/src/os/XfsFileStoreBackend.cc
new file mode 100644
index 00000000..e4d4aadf
--- /dev/null
+++ b/ceph/src/os/XfsFileStoreBackend.cc
@@ -0,0 +1,137 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "XfsFileStoreBackend.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "common/errno.h"
+#include "include/assert.h"
+#include "include/compat.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "xfsfilestorebackend(" << get_basedir_path() << ") "
+
+XfsFileStoreBackend::XfsFileStoreBackend(FileStore *fs):
+  GenericFileStoreBackend(fs), m_has_extsize(false) { }
+
+/*
+ * Set extsize attr on a file to val.  Should be a free-standing
+ * function, but dout_prefix expanding to a call to get_basedir_path()
+ * protected member function won't let it.
+ */
+int XfsFileStoreBackend::set_extsize(int fd, unsigned int val)
+{
+  struct fsxattr fsx;
+  struct stat sb;
+  int ret;
+
+  if (fstat(fd, &sb) < 0) {
+    ret = -errno;
+    dout(0) << "set_extsize: fstat: " << cpp_strerror(ret) << dendl;
+    goto out;
+  }
+  if (!S_ISREG(sb.st_mode)) {
+    ret = -EINVAL;
+    dout(0) << "set_extsize: invalid target file type" << dendl;
+    goto out;
+  }
+
+  if (ioctl(fd, XFS_IOC_FSGETXATTR, &fsx) < 0) {
+    ret = -errno;
+    dout(0) << "set_extsize: FSGETXATTR: " << cpp_strerror(ret) << dendl;
+    goto out;
+  }
+
+  // already set?
+  if ((fsx.fsx_xflags & XFS_XFLAG_EXTSIZE) && fsx.fsx_extsize == val)
+    return 0;
+
+  // xfs won't change extent size if any extents are allocated
+  if (fsx.fsx_nextents != 0)
+    return 0;
+
+  fsx.fsx_xflags |= XFS_XFLAG_EXTSIZE;
+  fsx.fsx_extsize = val;
+
+  if (ioctl(fd, XFS_IOC_FSSETXATTR, &fsx) < 0) {
+    ret = -errno;
+    dout(0) << "set_extsize: FSSETXATTR: " << cpp_strerror(ret) << dendl;
+    goto out;
+  }
+  ret = 0;
+
+out:
+  return ret;
+}
+
+int XfsFileStoreBackend::detect_features()
+{
+  int ret;
+
+  ret = GenericFileStoreBackend::detect_features();
+  if (ret < 0)
+    return ret;
+
+  // extsize?
+  int fd = ::openat(get_basedir_fd(), "extsize_test", O_CREAT|O_WRONLY, 0600);
+  if (fd < 0) {
+    ret = -errno;
+    dout(0) << "detect_feature: failed to create test file for extsize attr: "
+            << cpp_strerror(ret) << dendl;
+    goto out;
+  }
+  if (::unlinkat(get_basedir_fd(), "extsize_test", 0) < 0) {
+    ret = -errno;
+    dout(0) << "detect_feature: failed to unlink test file for extsize attr: "
+            << cpp_strerror(ret) << dendl;
+    goto out_close;
+  }
+
+  if (g_conf->filestore_xfs_extsize) {
+    ret = set_extsize(fd, 1U << 15); // a few pages
+    if (ret) {
+      ret = 0;
+      dout(0) << "detect_feature: failed to set test file extsize, assuming extsize is NOT supported" << dendl;
+      goto out_close;
+    } else {
+      dout(0) << "detect_feature: extsize is supported" << dendl;
+      m_has_extsize = true;
+    }
+  } else {
+    dout(0) << "detect_feature: extsize is disabled by conf" << dendl;
+  }
+
+
+out_close:
+  TEMP_FAILURE_RETRY(::close(fd));
+out:
+  return ret;
+}
+
+int XfsFileStoreBackend::set_alloc_hint(int fd, uint64_t hint)
+{
+  if (!m_has_extsize)
+    return -EOPNOTSUPP;
+
+  assert(hint < UINT_MAX);
+  return set_extsize(fd, hint);
+}
diff --git a/ceph/src/os/XfsFileStoreBackend.h b/ceph/src/os/XfsFileStoreBackend.h
new file mode 100644
index 00000000..cb19bf7b
--- /dev/null
+++ b/ceph/src/os/XfsFileStoreBackend.h
@@ -0,0 +1,33 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_XFSFILESTOREBACKEND_H
+#define CEPH_XFSFILESTOREBACKEND_H
+
+#include "GenericFileStoreBackend.h"
+
+#include "include/int_types.h"
+
+class XfsFileStoreBackend : public GenericFileStoreBackend {
+private:
+  bool m_has_extsize;
+  int set_extsize(int fd, unsigned int val);
+public:
+  XfsFileStoreBackend(FileStore *fs);
+  ~XfsFileStoreBackend() {};
+  int detect_features();
+  int set_alloc_hint(int fd, uint64_t hint);
+};
+
+#endif /* CEPH_XFSFILESTOREBACKEND_H */
diff --git a/ceph/src/os/ZFS.cc b/ceph/src/os/ZFS.cc
new file mode 100644
index 00000000..02520796
--- /dev/null
+++ b/ceph/src/os/ZFS.cc
@@ -0,0 +1,83 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#define HAVE_IOCTL_IN_SYS_IOCTL_H
+#include 
+#include "ZFS.h"
+
+const int ZFS::TYPE_FILESYSTEM 	= ZFS_TYPE_FILESYSTEM;
+const int ZFS::TYPE_SNAPSHOT	= ZFS_TYPE_SNAPSHOT;
+const int ZFS::TYPE_VOLUME	= ZFS_TYPE_VOLUME;
+const int ZFS::TYPE_DATASET	= ZFS_TYPE_DATASET;
+
+ZFS::~ZFS()
+{
+  if (g_zfs)
+    ::libzfs_fini((libzfs_handle_t*)g_zfs);
+}
+
+int ZFS::init()
+{
+  g_zfs = ::libzfs_init();
+  return g_zfs ? 0 : -EINVAL;
+}
+
+ZFS::Handle *ZFS::open(const char *n, int t)
+{
+  return (ZFS::Handle*)::zfs_open((libzfs_handle_t*)g_zfs, n, (zfs_type_t)t);
+}
+
+void ZFS::close(ZFS::Handle *h)
+{
+  ::zfs_close((zfs_handle_t*)h);
+}
+
+const char *ZFS::get_name(ZFS::Handle *h)
+{
+  return ::zfs_get_name((zfs_handle_t*)h);
+}
+
+ZFS::Handle *ZFS::path_to_zhandle(const char *p, int t)
+{
+  return ::zfs_path_to_zhandle((libzfs_handle_t*)g_zfs, (char *)p, (zfs_type_t)t);
+}
+
+int ZFS::create(const char *n, int t)
+{
+  return ::zfs_create((libzfs_handle_t*)g_zfs, n, (zfs_type_t)t, NULL);
+}
+
+int ZFS::snapshot(const char *n, bool r)
+{
+  return ::zfs_snapshot((libzfs_handle_t*)g_zfs, n, (boolean_t)r, NULL);
+}
+
+int ZFS::rollback(ZFS::Handle *h, ZFS::Handle *snap, bool f)
+{
+  return ::zfs_rollback((zfs_handle_t*)h, (zfs_handle_t*)snap, (boolean_t)f);
+}
+
+int ZFS::destroy_snaps(ZFS::Handle *h, const char *n, bool d)
+{
+  return ::zfs_destroy_snaps((zfs_handle_t*)h, (char *)n, (boolean_t)d);
+}
+
+bool ZFS::is_mounted(ZFS::Handle *h, char **p)
+{
+  return (bool)::zfs_is_mounted((zfs_handle_t*)h, p);
+}
+
+int ZFS::mount(ZFS::Handle *h, const char *o, int f)
+{
+  return ::zfs_mount((zfs_handle_t*)h, o, f);
+}
+
+int ZFS::umount(ZFS::Handle *h, const char *o, int f)
+{
+  return ::zfs_unmount((zfs_handle_t*)h, o, f);
+}
+
+int ZFS::iter_snapshots_sorted(ZFS::Handle *h, ZFS::iter_func f, void *d)
+{
+  return ::zfs_iter_snapshots_sorted((zfs_handle_t*)h, (zfs_iter_f)f, d);
+}
diff --git a/ceph/src/os/ZFS.h b/ceph/src/os/ZFS.h
new file mode 100644
index 00000000..3ebe1110
--- /dev/null
+++ b/ceph/src/os/ZFS.h
@@ -0,0 +1,39 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_ZFS_H
+#define CEPH_ZFS_H
+
+// Simple wrapper to hide libzfs.h. (it conflicts with standard linux headers)
+class ZFS {
+  void *g_zfs;
+public:
+
+  static const int TYPE_FILESYSTEM;
+  static const int TYPE_SNAPSHOT;
+  static const int TYPE_VOLUME;
+  static const int TYPE_POOL;
+  static const int TYPE_DATASET;
+
+  typedef void Handle;
+  typedef int (*iter_func)(Handle *, void *);
+
+  static const char *get_name(Handle *);
+
+  ZFS() : g_zfs(NULL) {}
+  ~ZFS();
+  int init();
+  Handle *open(const char *, int);
+  void close(Handle *);
+  Handle *path_to_zhandle(const char *, int);
+  int create(const char *, int);
+  int snapshot(const char *, bool);
+  int rollback(Handle *, Handle *, bool);
+  int destroy_snaps(Handle *, const char *, bool);
+  int iter_snapshots_sorted(Handle *, iter_func, void *);
+  int mount(Handle *, const char *, int);
+  int umount(Handle *, const char *, int);
+  bool is_mounted(Handle *, char **);
+};
+
+#endif
diff --git a/ceph/src/os/ZFSFileStoreBackend.cc b/ceph/src/os/ZFSFileStoreBackend.cc
new file mode 100644
index 00000000..aa52b8d2
--- /dev/null
+++ b/ceph/src/os/ZFSFileStoreBackend.cc
@@ -0,0 +1,260 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/int_types.h"
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/compat.h"
+#include "include/linux_fiemap.h"
+#include "include/color.h"
+#include "include/buffer.h"
+#include "include/assert.h"
+
+#include 
+#include 
+#include 
+
+#include "common/errno.h"
+#include "common/config.h"
+#include "common/sync_filesystem.h"
+
+#ifdef HAVE_LIBZFS
+
+#include "ZFSFileStoreBackend.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "zfsfilestorebackend(" << get_basedir_path() << ") "
+
+ZFSFileStoreBackend::ZFSFileStoreBackend(FileStore *fs) :
+  GenericFileStoreBackend(fs), base_zh(NULL), current_zh(NULL),
+  m_filestore_zfs_snap(g_conf->filestore_zfs_snap)
+{
+  int ret = zfs.init();
+  if (ret < 0) {
+    dout(0) << "ZFSFileStoreBackend: failed to init libzfs" << dendl;
+    return;
+  }
+
+  base_zh = zfs.path_to_zhandle(get_basedir_path().c_str(), ZFS::TYPE_FILESYSTEM);
+  if (!base_zh) {
+    dout(0) << "ZFSFileStoreBackend: failed to get zfs handler for basedir" << dendl;
+    return;
+  }
+
+  update_current_zh();
+}
+
+ZFSFileStoreBackend::~ZFSFileStoreBackend()
+{
+  if (base_zh)
+    zfs.close(base_zh);
+  if (current_zh)
+    zfs.close(current_zh);
+}
+
+int ZFSFileStoreBackend::update_current_zh()
+{
+  char path[PATH_MAX];
+  snprintf(path, sizeof(path), "%s/current", zfs.get_name(base_zh));
+  ZFS::Handle *zh = zfs.open(path, ZFS::TYPE_FILESYSTEM);
+  if (zh) {
+    char *mnt;
+    if (zfs.is_mounted(zh, &mnt)) {
+      int ret = get_current_path() == mnt;
+      free(mnt);
+      if (ret) {
+	current_zh = zh;
+	return 0;
+      }
+    } else {
+      int ret = zfs.mount(zh, NULL, 0);
+      if (ret < 0) {
+	ret = -errno;
+	dout(0) << "update_current_zh: zfs_mount '" << zfs.get_name(zh)
+		<< "' got " << cpp_strerror(ret) << dendl;
+	return ret;
+      }
+    }
+    zfs.close(zh);
+  } else {
+    dout(0) << "update_current_zh: zfs_open '" << path << "' got NULL" << dendl;
+    return -ENOENT;
+  }
+
+  zh = zfs.path_to_zhandle(get_current_path().c_str(), ZFS::TYPE_FILESYSTEM);
+  if (zh) {
+    if (strcmp(zfs.get_name(base_zh), zfs.get_name(zh))) {
+      current_zh = zh;
+      return 0;
+    }
+    zfs.close(zh);
+    dout(0) << "update_current_zh: basedir and current/ on the same filesystem" << dendl;
+  } else {
+    dout(0) << "update_current_zh: current/ not exist" << dendl;
+  }
+  return -ENOENT;
+}
+
+int ZFSFileStoreBackend::detect_features()
+{
+  if (!current_zh)
+    dout(0) << "detect_features: null zfs handle for current/" << dendl;
+  return 0;
+}
+
+bool ZFSFileStoreBackend::can_checkpoint()
+{
+  return m_filestore_zfs_snap && current_zh != NULL;
+}
+
+int ZFSFileStoreBackend::create_current()
+{
+  struct stat st;
+  int ret = ::stat(get_current_path().c_str(), &st);
+  if (ret == 0) {
+    // current/ exists
+    if (!S_ISDIR(st.st_mode)) {
+      dout(0) << "create_current: current/ exists but is not a directory" << dendl;
+      return -ENOTDIR;
+    }
+    return 0;
+  } else if (errno != ENOENT) {
+    ret = -errno;
+    dout(0) << "create_current: cannot stat current/ " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  char path[PATH_MAX];
+  snprintf(path, sizeof(path), "%s/current", zfs.get_name(base_zh));
+  ret = zfs.create(path, ZFS::TYPE_FILESYSTEM);
+  if (ret < 0 && errno != EEXIST) {
+    ret = -errno;
+    dout(0) << "create_current: zfs_create '" << path << "' got " << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  ret = update_current_zh();
+  return ret;
+}
+
+static int list_checkpoints_callback(ZFS::Handle *zh, void *data)
+{
+  list *ls = static_cast *>(data);
+  string str = ZFS::get_name(zh);
+  size_t pos = str.find('@');
+  assert(pos != string::npos && pos + 1 != str.length());
+  ls->push_back(str.substr(pos + 1));
+  return 0;
+}
+
+int ZFSFileStoreBackend::list_checkpoints(list& ls)
+{
+  dout(10) << "list_checkpoints:" << dendl;
+  if (!current_zh)
+    return -EINVAL;
+
+  list snaps;
+  int ret = zfs.iter_snapshots_sorted(current_zh, list_checkpoints_callback, &snaps);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "list_checkpoints: zfs_iter_snapshots_sorted got" << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+  ls.swap(snaps);
+  return 0;
+}
+
+int ZFSFileStoreBackend::create_checkpoint(const string& name, uint64_t *cid)
+{
+  dout(10) << "create_checkpoint: '" << name << "'" << dendl;
+  if (!current_zh)
+    return -EINVAL;
+
+  // looks like zfsonlinux doesn't flush dirty data when taking snapshot
+  int ret = sync_filesystem(get_current_fd());
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "create_checkpoint: sync_filesystem got" << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+
+  char path[PATH_MAX];
+  snprintf(path, sizeof(path), "%s@%s", zfs.get_name(current_zh), name.c_str());
+  ret = zfs.snapshot(path, false);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "create_checkpoint: zfs_snapshot '" << path << "' got" << cpp_strerror(ret) << dendl;
+    return ret;
+  }
+  if (cid)
+    *cid = 0;
+  return 0;
+}
+
+int ZFSFileStoreBackend::rollback_to(const string& name)
+{
+  dout(10) << "rollback_to: '" << name << "'" << dendl;
+  if (!current_zh)
+    return -EINVAL;
+
+  // umount current to avoid triggering online rollback deadlock
+  int ret;
+  if (zfs.is_mounted(current_zh, NULL)) {
+    ret = zfs.umount(current_zh, NULL, 0);
+    if (ret < 0) {
+      ret = -errno;
+      dout(0) << "rollback_to: zfs_umount '" << zfs.get_name(current_zh) << "' got" << cpp_strerror(ret) << dendl;
+    }
+  }
+
+  char path[PATH_MAX];
+  snprintf(path, sizeof(path), "%s@%s", zfs.get_name(current_zh), name.c_str());
+
+  ZFS::Handle *snap_zh = zfs.open(path, ZFS::TYPE_SNAPSHOT);
+  if (!snap_zh) {
+    dout(0) << "rollback_to: zfs_open '" << path << "' got NULL" << dendl;
+    return -ENOENT;
+  }
+
+  ret = zfs.rollback(current_zh, snap_zh, false);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "rollback_to: zfs_rollback '" << zfs.get_name(snap_zh) << "' got" << cpp_strerror(ret) << dendl;
+  }
+
+  if (!zfs.is_mounted(current_zh, NULL)) {
+    int ret = zfs.mount(current_zh, NULL, 0);
+    if (ret < 0) {
+      ret = -errno;
+      dout(0) << "update_current_zh: zfs_mount '" << zfs.get_name(current_zh) << "' got " << cpp_strerror(ret) << dendl;
+      return ret;
+    }
+  }
+
+  zfs.close(snap_zh);
+  return ret;
+}
+
+int ZFSFileStoreBackend::destroy_checkpoint(const string& name)
+{
+  dout(10) << "destroy_checkpoint: '" << name << "'" << dendl;
+  if (!current_zh)
+    return -EINVAL;
+
+  int ret = zfs.destroy_snaps(current_zh, name.c_str(), true);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "destroy_checkpoint: zfs_destroy_snaps '" << name << "' got" << cpp_strerror(ret) << dendl;
+  }
+  return ret;
+}
+#endif
diff --git a/ceph/src/os/ZFSFileStoreBackend.h b/ceph/src/os/ZFSFileStoreBackend.h
new file mode 100644
index 00000000..8186d9ca
--- /dev/null
+++ b/ceph/src/os/ZFSFileStoreBackend.h
@@ -0,0 +1,30 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_ZFSFILESTOREBACKEND_H
+#define CEPH_ZFSFILESTOREBACKEND_H
+
+#ifdef HAVE_LIBZFS
+#include "GenericFileStoreBackend.h"
+#include "ZFS.h"
+
+class ZFSFileStoreBackend : public GenericFileStoreBackend {
+private:
+  ZFS zfs;
+  ZFS::Handle *base_zh;
+  ZFS::Handle *current_zh;
+  bool m_filestore_zfs_snap;
+  int update_current_zh();
+public:
+  ZFSFileStoreBackend(FileStore *fs);
+  ~ZFSFileStoreBackend();
+  int detect_features();
+  bool can_checkpoint();
+  int create_current();
+  int list_checkpoints(list& ls);
+  int create_checkpoint(const string& name, uint64_t *cid);
+  int rollback_to(const string& name);
+  int destroy_checkpoint(const string& name);
+};
+#endif
+#endif
diff --git a/ceph/src/os/btrfs_ioctl.h b/ceph/src/os/btrfs_ioctl.h
new file mode 100644
index 00000000..277498ca
--- /dev/null
+++ b/ceph/src/os/btrfs_ioctl.h
@@ -0,0 +1,201 @@
+/*
+ * Copyright (C) 2007 Oracle.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#ifndef __IOCTL_
+#define __IOCTL_
+
+#if defined(__linux__)
+#include 
+#elif defined(__FreeBSD__)
+#include 
+#endif
+
+#define BTRFS_IOCTL_MAGIC 0x94
+#define BTRFS_VOL_NAME_MAX 255
+
+/* this should be 4k */
+#define BTRFS_PATH_NAME_MAX 4087
+struct btrfs_ioctl_vol_args {
+	__s64 fd;
+	char name[BTRFS_PATH_NAME_MAX + 1];
+};
+
+#define BTRFS_SUBVOL_CREATE_ASYNC	(1ULL << 0)
+
+#define BTRFS_SUBVOL_NAME_MAX 4039
+struct btrfs_ioctl_vol_args_v2 {
+	__s64 fd;
+	__u64 transid;
+	__u64 flags;
+	__u64 unused[4];
+	char name[BTRFS_SUBVOL_NAME_MAX + 1];
+};
+
+#define BTRFS_INO_LOOKUP_PATH_MAX 4080
+struct btrfs_ioctl_ino_lookup_args {
+	__u64 treeid;
+	__u64 objectid;
+	char name[BTRFS_INO_LOOKUP_PATH_MAX];
+};
+
+struct btrfs_ioctl_search_key {
+	/* which root are we searching.  0 is the tree of tree roots */
+	__u64 tree_id;
+
+	/* keys returned will be >= min and <= max */
+	__u64 min_objectid;
+	__u64 max_objectid;
+
+	/* keys returned will be >= min and <= max */
+	__u64 min_offset;
+	__u64 max_offset;
+
+	/* max and min transids to search for */
+	__u64 min_transid;
+	__u64 max_transid;
+
+	/* keys returned will be >= min and <= max */
+	__u32 min_type;
+	__u32 max_type;
+
+	/*
+	 * how many items did userland ask for, and how many are we
+	 * returning
+	 */
+	__u32 nr_items;
+
+	/* align to 64 bits */
+	__u32 unused;
+
+	/* some extra for later */
+	__u64 unused1;
+	__u64 unused2;
+	__u64 unused3;
+	__u64 unused4;
+};
+
+struct btrfs_ioctl_search_header {
+	__u64 transid;
+	__u64 objectid;
+	__u64 offset;
+	__u32 type;
+	__u32 len;
+};
+
+#define BTRFS_SEARCH_ARGS_BUFSIZE (4096 - sizeof(struct btrfs_ioctl_search_key))
+/*
+ * the buf is an array of search headers where
+ * each header is followed by the actual item
+ * the type field is expanded to 32 bits for alignment
+ */
+struct btrfs_ioctl_search_args {
+	struct btrfs_ioctl_search_key key;
+	char buf[BTRFS_SEARCH_ARGS_BUFSIZE];
+};
+
+struct btrfs_ioctl_clone_range_args {
+  __s64 src_fd;
+  __u64 src_offset, src_length;
+  __u64 dest_offset;
+};
+
+/* flags for the defrag range ioctl */
+#define BTRFS_DEFRAG_RANGE_COMPRESS 1
+#define BTRFS_DEFRAG_RANGE_START_IO 2
+
+struct btrfs_ioctl_defrag_range_args {
+	/* start of the defrag operation */
+	__u64 start;
+
+	/* number of bytes to defrag, use (u64)-1 to say all */
+	__u64 len;
+
+	/*
+	 * flags for the operation, which can include turning
+	 * on compression for this one defrag
+	 */
+	__u64 flags;
+
+	/*
+	 * any extent bigger than this will be considered
+	 * already defragged.  Use 0 to take the kernel default
+	 * Use 1 to say every single extent must be rewritten
+	 */
+	__u32 extent_thresh;
+
+	/* spare for later */
+	__u32 unused[5];
+};
+
+struct btrfs_ioctl_space_info {
+	__u64 flags;
+	__u64 total_bytes;
+	__u64 used_bytes;
+};
+
+struct btrfs_ioctl_space_args {
+	__u64 space_slots;
+	__u64 total_spaces;
+	struct btrfs_ioctl_space_info spaces[0];
+};
+
+#define BTRFS_IOC_SNAP_CREATE _IOW(BTRFS_IOCTL_MAGIC, 1, \
+				   struct btrfs_ioctl_vol_args)
+#define BTRFS_IOC_DEFRAG _IOW(BTRFS_IOCTL_MAGIC, 2, \
+				   struct btrfs_ioctl_vol_args)
+#define BTRFS_IOC_RESIZE _IOW(BTRFS_IOCTL_MAGIC, 3, \
+				   struct btrfs_ioctl_vol_args)
+#define BTRFS_IOC_SCAN_DEV _IOW(BTRFS_IOCTL_MAGIC, 4, \
+				   struct btrfs_ioctl_vol_args)
+/* trans start and trans end are dangerous, and only for
+ * use by applications that know how to avoid the
+ * resulting deadlocks
+ */
+#define BTRFS_IOC_TRANS_START  _IO(BTRFS_IOCTL_MAGIC, 6)
+#define BTRFS_IOC_TRANS_END    _IO(BTRFS_IOCTL_MAGIC, 7)
+#define BTRFS_IOC_SYNC         _IO(BTRFS_IOCTL_MAGIC, 8)
+
+#define BTRFS_IOC_CLONE        _IOW(BTRFS_IOCTL_MAGIC, 9, int)
+#define BTRFS_IOC_ADD_DEV _IOW(BTRFS_IOCTL_MAGIC, 10, \
+				   struct btrfs_ioctl_vol_args)
+#define BTRFS_IOC_RM_DEV _IOW(BTRFS_IOCTL_MAGIC, 11, \
+				   struct btrfs_ioctl_vol_args)
+#define BTRFS_IOC_BALANCE _IOW(BTRFS_IOCTL_MAGIC, 12, \
+				   struct btrfs_ioctl_vol_args)
+
+#define BTRFS_IOC_CLONE_RANGE _IOW(BTRFS_IOCTL_MAGIC, 13, \
+				  struct btrfs_ioctl_clone_range_args)
+
+#define BTRFS_IOC_SUBVOL_CREATE _IOW(BTRFS_IOCTL_MAGIC, 14, \
+				   struct btrfs_ioctl_vol_args)
+#define BTRFS_IOC_SNAP_DESTROY _IOW(BTRFS_IOCTL_MAGIC, 15, \
+				struct btrfs_ioctl_vol_args)
+#define BTRFS_IOC_DEFRAG_RANGE _IOW(BTRFS_IOCTL_MAGIC, 16, \
+				struct btrfs_ioctl_defrag_range_args)
+#define BTRFS_IOC_TREE_SEARCH _IOWR(BTRFS_IOCTL_MAGIC, 17, \
+				   struct btrfs_ioctl_search_args)
+#define BTRFS_IOC_INO_LOOKUP _IOWR(BTRFS_IOCTL_MAGIC, 18, \
+				   struct btrfs_ioctl_ino_lookup_args)
+#define BTRFS_IOC_DEFAULT_SUBVOL _IOW(BTRFS_IOCTL_MAGIC, 19, u64)
+#define BTRFS_IOC_SPACE_INFO _IOWR(BTRFS_IOCTL_MAGIC, 20, \
+				    struct btrfs_ioctl_space_args)
+#define BTRFS_IOC_START_SYNC _IOR(BTRFS_IOCTL_MAGIC, 24, __u64)
+#define BTRFS_IOC_WAIT_SYNC  _IOW(BTRFS_IOCTL_MAGIC, 22, __u64)
+#define BTRFS_IOC_SNAP_CREATE_V2 _IOW(BTRFS_IOCTL_MAGIC, 23, \
+				   struct btrfs_ioctl_vol_args_v2)
+#endif
diff --git a/ceph/src/os/chain_xattr.cc b/ceph/src/os/chain_xattr.cc
new file mode 100644
index 00000000..c020c9db
--- /dev/null
+++ b/ceph/src/os/chain_xattr.cc
@@ -0,0 +1,437 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "chain_xattr.h"
+
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "include/assert.h"
+
+#if defined(__linux__)
+#include 
+#endif
+
+#include "common/xattr.h"
+
+/*
+ * chaining xattrs
+ *
+ * In order to support xattrs that are larger than the xattr size limit that some file systems
+ * impose, we use multiple xattrs to store the value of a single xattr. The xattrs keys
+ * are set as follows:
+ * The first xattr in the chain, has a key that holds the original xattr name, with any '@' char
+ * being esacped ("@@").
+ * The chained keys will have the first xattr's key (with the escaping), and a suffix: "@"
+ * where  marks the num of xattr in the chain.
+ */
+
+static void get_raw_xattr_name(const char *name, int i, char *raw_name, int raw_len)
+{
+  int pos = 0;
+
+  while (*name) {
+    switch (*name) {
+    case '@': /* escape it */
+      pos += 2;
+      assert (pos < raw_len - 1);
+      *raw_name = '@';
+      raw_name++;
+      *raw_name = '@';
+      break;
+    default:
+      pos++;
+      assert(pos < raw_len - 1);
+      *raw_name = *name;
+      break;
+    }
+    name++;
+    raw_name++;
+  }
+
+  if (!i) {
+    *raw_name = '\0';
+  } else {
+    int r = snprintf(raw_name, raw_len, "@%d", i);
+    assert(r < raw_len - pos);
+  }
+}
+
+static int translate_raw_name(const char *raw_name, char *name, int name_len, bool *is_first)
+{
+  int pos = 0;
+
+  *is_first = true;
+  while (*raw_name) {
+    switch (*raw_name) {
+    case '@': /* escape it */
+      raw_name++;
+      if (!*raw_name)
+        break;
+      if (*raw_name != '@') {
+        *is_first = false;
+        goto done;
+      }
+
+    /* fall through */
+    default:
+      *name = *raw_name;
+      break;
+    }
+    pos++;
+    assert(pos < name_len);
+    name++;
+    raw_name++;
+  }
+done:
+  *name = '\0';
+  return pos;
+}
+
+
+// setxattr
+
+static int getxattr_len(const char *fn, const char *name)
+{
+  int i = 0, total = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int r;
+
+  do {
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    r = sys_getxattr(fn, raw_name, 0, 0);
+    if (!i && r < 0)
+      return r;
+    if (r < 0)
+      break;
+    total += r;
+    i++;
+  } while (r == CHAIN_XATTR_MAX_BLOCK_LEN);
+
+  return total;
+}
+
+int chain_getxattr(const char *fn, const char *name, void *val, size_t size)
+{
+  int i = 0, pos = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int ret = 0;
+  int r;
+  size_t chunk_size;
+
+  if (!size)
+    return getxattr_len(fn, name);
+
+  do {
+    chunk_size = (size < CHAIN_XATTR_MAX_BLOCK_LEN ? size : CHAIN_XATTR_MAX_BLOCK_LEN);
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    size -= chunk_size;
+
+    r = sys_getxattr(fn, raw_name, (char *)val + pos, chunk_size);
+    if (r < 0) {
+      ret = r;
+      break;
+    }
+
+    if (r > 0)
+      pos += r;
+
+    i++;
+  } while (size && r == CHAIN_XATTR_MAX_BLOCK_LEN);
+
+  if (r >= 0) {
+    ret = pos;
+    /* is there another chunk? that can happen if the last read size span over
+       exactly one block */
+    if (chunk_size == CHAIN_XATTR_MAX_BLOCK_LEN) {
+      get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+      r = sys_getxattr(fn, raw_name, 0, 0);
+      if (r > 0) { // there's another chunk.. the original buffer was too small
+        ret = -ERANGE;
+      }
+    }
+  }
+  return ret;
+}
+
+static int chain_fgetxattr_len(int fd, const char *name)
+{
+  int i = 0, total = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int r;
+
+  do {
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    r = sys_fgetxattr(fd, raw_name, 0, 0);
+    if (!i && r < 0)
+      return r;
+    if (r < 0)
+      break;
+    total += r;
+    i++;
+  } while (r == CHAIN_XATTR_MAX_BLOCK_LEN);
+
+  return total;
+}
+
+int chain_fgetxattr(int fd, const char *name, void *val, size_t size)
+{
+  int i = 0, pos = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int ret = 0;
+  int r;
+  size_t chunk_size;
+
+  if (!size)
+    return chain_fgetxattr_len(fd, name);
+
+  do {
+    chunk_size = (size < CHAIN_XATTR_MAX_BLOCK_LEN ? size : CHAIN_XATTR_MAX_BLOCK_LEN);
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    size -= chunk_size;
+
+    r = sys_fgetxattr(fd, raw_name, (char *)val + pos, chunk_size);
+    if (r < 0) {
+      ret = r;
+      break;
+    }
+
+    if (r > 0)
+      pos += r;
+
+    i++;
+  } while (size && r == CHAIN_XATTR_MAX_BLOCK_LEN);
+
+  if (r >= 0) {
+    ret = pos;
+    /* is there another chunk? that can happen if the last read size span over
+       exactly one block */
+    if (chunk_size == CHAIN_XATTR_MAX_BLOCK_LEN) {
+      get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+      r = sys_fgetxattr(fd, raw_name, 0, 0);
+      if (r > 0) { // there's another chunk.. the original buffer was too small
+        ret = -ERANGE;
+      }
+    }
+  }
+  return ret;
+}
+
+
+// setxattr
+
+int chain_setxattr(const char *fn, const char *name, const void *val, size_t size)
+{
+  int i = 0, pos = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int ret = 0;
+  size_t chunk_size;
+
+  do {
+    chunk_size = (size < CHAIN_XATTR_MAX_BLOCK_LEN ? size : CHAIN_XATTR_MAX_BLOCK_LEN);
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    size -= chunk_size;
+
+    int r = sys_setxattr(fn, raw_name, (char *)val + pos, chunk_size);
+    if (r < 0) {
+      ret = r;
+      break;
+    }
+    pos  += chunk_size;
+    ret = pos;
+    i++;
+  } while (size);
+
+  /* if we're exactly at a chunk size, remove the next one (if wasn't removed
+     before) */
+  if (ret >= 0 && chunk_size == CHAIN_XATTR_MAX_BLOCK_LEN) {
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    int r = sys_removexattr(fn, raw_name);
+    if (r < 0 && r != -ENODATA)
+      ret = r;
+  }
+  
+  return ret;
+}
+
+int chain_fsetxattr(int fd, const char *name, const void *val, size_t size)
+{
+  int i = 0, pos = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int ret = 0;
+  size_t chunk_size;
+
+  do {
+    chunk_size = (size < CHAIN_XATTR_MAX_BLOCK_LEN ? size : CHAIN_XATTR_MAX_BLOCK_LEN);
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    size -= chunk_size;
+
+    int r = sys_fsetxattr(fd, raw_name, (char *)val + pos, chunk_size);
+    if (r < 0) {
+      ret = r;
+      break;
+    }
+    pos  += chunk_size;
+    ret = pos;
+    i++;
+  } while (size);
+
+  /* if we're exactly at a chunk size, remove the next one (if wasn't removed
+     before) */
+  if (ret >= 0 && chunk_size == CHAIN_XATTR_MAX_BLOCK_LEN) {
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    int r = sys_fremovexattr(fd, raw_name);
+    if (r < 0 && r != -ENODATA)
+      ret = r;
+  }
+  
+  return ret;
+}
+
+
+// removexattr
+
+int chain_removexattr(const char *fn, const char *name)
+{
+  int i = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int r;
+
+  do {
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    r = sys_removexattr(fn, raw_name);
+    if (!i && r < 0) {
+      return r;
+    }
+    i++;
+  } while (r >= 0);
+  return 0;
+}
+
+int chain_fremovexattr(int fd, const char *name)
+{
+  int i = 0;
+  char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+  int r;
+
+  do {
+    get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
+    r = sys_fremovexattr(fd, raw_name);
+    if (!i && r < 0) {
+      return r;
+    }
+    i++;
+  } while (r >= 0);
+  return 0;
+}
+
+
+// listxattr
+
+int chain_listxattr(const char *fn, char *names, size_t len) {
+  int r;
+
+  if (!len)
+    return sys_listxattr(fn, names, len) * 2;
+
+  r = sys_listxattr(fn, 0, 0);
+  if (r < 0)
+    return r;
+
+  size_t total_len = r * 2; // should be enough
+  char *full_buf = (char *)malloc(total_len);
+  if (!full_buf)
+    return -ENOMEM;
+
+  r = sys_listxattr(fn, full_buf, total_len);
+  if (r < 0) {
+    free(full_buf);
+    return r;
+  }
+
+  char *p = full_buf;
+  const char *end = full_buf + r;
+  char *dest = names;
+  char *dest_end = names + len;
+
+  while (p < end) {
+    char name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+    int attr_len = strlen(p);
+    bool is_first;
+    int name_len = translate_raw_name(p, name, sizeof(name), &is_first);
+    if (is_first)  {
+      if (dest + name_len > dest_end) {
+        r = -ERANGE;
+        goto done;
+      }
+      strcpy(dest, name);
+      dest += name_len + 1;
+    }
+    p += attr_len + 1;
+  }
+  r = dest - names;
+
+done:
+  free(full_buf);
+  return r;
+}
+
+int chain_flistxattr(int fd, char *names, size_t len) {
+  int r;
+  char *p;
+  const char * end;
+  char *dest;
+  char *dest_end;
+
+  if (!len)
+    return sys_flistxattr(fd, names, len) * 2;
+
+  r = sys_flistxattr(fd, 0, 0);
+  if (r < 0)
+    return r;
+
+  size_t total_len = r * 2; // should be enough
+  char *full_buf = (char *)malloc(total_len);
+  if (!full_buf)
+    return -ENOMEM;
+
+  r = sys_flistxattr(fd, full_buf, total_len);
+  if (r < 0)
+    goto done;
+
+  p = full_buf;
+  end = full_buf + r;
+  dest = names;
+  dest_end = names + len;
+
+  while (p < end) {
+    char name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
+    int attr_len = strlen(p);
+    bool is_first;
+    int name_len = translate_raw_name(p, name, sizeof(name), &is_first);
+    if (is_first)  {
+      if (dest + name_len > dest_end) {
+        r = -ERANGE;
+        goto done;
+      }
+      strcpy(dest, name);
+      dest += name_len + 1;
+    }
+    p += attr_len + 1;
+  }
+  r = dest - names;
+
+done:
+  free(full_buf);
+  return r;
+}
diff --git a/ceph/src/os/chain_xattr.h b/ceph/src/os/chain_xattr.h
new file mode 100644
index 00000000..7e8312ff
--- /dev/null
+++ b/ceph/src/os/chain_xattr.h
@@ -0,0 +1,73 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef __CEPH_OSD_CHAIN_XATTR_H
+#define __CEPH_OSD_CHAIN_XATTR_H
+
+#include "common/xattr.h"
+
+#include 
+
+#define CHAIN_XATTR_MAX_NAME_LEN  128
+#define CHAIN_XATTR_MAX_BLOCK_LEN 2048
+
+
+// wrappers to hide annoying errno handling.
+
+static inline int sys_fgetxattr(int fd, const char *name, void *val, size_t size)
+{
+  int r = ::ceph_os_fgetxattr(fd, name, val, size);
+  return (r < 0 ? -errno : r);
+}
+static inline int sys_getxattr(const char *fn, const char *name, void *val, size_t size)
+{
+  int r = ::ceph_os_getxattr(fn, name, val, size);
+  return (r < 0 ? -errno : r);
+}
+
+static inline int sys_setxattr(const char *fn, const char *name, const void *val, size_t size)
+{
+  int r = ::ceph_os_setxattr(fn, name, val, size);
+  return (r < 0 ? -errno : r);
+}
+static inline int sys_fsetxattr(int fd, const char *name, const void *val, size_t size)
+{
+  int r = ::ceph_os_fsetxattr(fd, name, val, size);
+  return (r < 0 ? -errno : r);
+}
+
+static inline int sys_listxattr(const char *fn, char *names, size_t len)
+{
+  int r = ::ceph_os_listxattr(fn, names, len);
+  return (r < 0 ? -errno : r);
+}
+static inline int sys_flistxattr(int fd, char *names, size_t len)
+{
+  int r = ::ceph_os_flistxattr(fd, names, len);
+  return (r < 0 ? -errno : r);
+}
+
+static inline int sys_removexattr(const char *fn, const char *name)
+{
+  int r = ::ceph_os_removexattr(fn, name);
+  return (r < 0 ? -errno : r);
+}
+static inline int sys_fremovexattr(int fd, const char *name)
+{
+  int r = ::ceph_os_fremovexattr(fd, name);
+  return (r < 0 ? -errno : r);
+}
+
+
+// wrappers to chain large values across multiple xattrs
+
+int chain_getxattr(const char *fn, const char *name, void *val, size_t size);
+int chain_fgetxattr(int fd, const char *name, void *val, size_t size);
+int chain_setxattr(const char *fn, const char *name, const void *val, size_t size);
+int chain_fsetxattr(int fd, const char *name, const void *val, size_t size);
+int chain_listxattr(const char *fn, char *names, size_t len);
+int chain_flistxattr(int fd, char *names, size_t len);
+int chain_removexattr(const char *fn, const char *name);
+int chain_fremovexattr(int fd, const char *name);
+
+#endif
diff --git a/ceph/src/osd/Ager.cc b/ceph/src/osd/Ager.cc
new file mode 100644
index 00000000..f94da1f4
--- /dev/null
+++ b/ceph/src/osd/Ager.cc
@@ -0,0 +1,271 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+
+#include "Ager.h"
+#include "os/ObjectStore.h"
+
+#include "common/Clock.h"
+#include "common/debug.h"
+
+// ick
+#include 
+#include 
+#include 
+
+#if defined(DARWIN) || defined(__FreeBSD__)
+#include 
+#include 
+#endif // DARWIN || __FreeBSD__
+
+
+int myrand() 
+{
+  if (0) 
+    return rand();
+  else {
+    static int n = 0;
+    srand(n++);
+    return rand();
+  }
+}
+
+
+file_object_t Ager::age_get_oid() {
+  if (!age_free_oids.empty()) {
+    file_object_t o = age_free_oids.front();
+    age_free_oids.pop_front();
+    return o;
+  }
+  file_object_t last = age_cur_oid;
+  ++age_cur_oid.bno;
+  return last;
+}
+
+ssize_t Ager::age_pick_size() {
+  ssize_t max = file_size_distn.sample() * 1024;
+  return max/2 + (myrand() % 100) * max/200 + 1;
+}
+
+bool start_debug = false;
+
+uint64_t Ager::age_fill(float pc, utime_t until) {
+  int max = 1024*1024;
+  bufferptr bp(max);
+  bp.zero();
+  bufferlist bl;
+  bl.push_back(bp);
+  uint64_t wrote = 0;
+  while (1) {
+    if (ceph_clock_now(cct) > until) break;
+    
+    struct statfs st;
+    store->statfs(&st);
+    float free = 1.0 - ((float)(st.f_bfree) / (float)st.f_blocks);
+    float avail = 1.0 - ((float)(st.f_bavail) / (float)st.f_blocks);  // to write to
+    //float a = (float)(st.f_bfree) / (float)st.f_blocks;
+    //dout(10) << "age_fill at " << a << " / " << pc << " .. " << st.f_blocks << " " << st.f_bavail << dendl;
+    if (free >= pc) {
+      generic_dout(2) << "age_fill at " << free << " / " << avail << " / " << " / " << pc << " stopping" << dendl;
+      break;
+    }
+
+    // make sure we can write to it..
+    if (avail > .98 ||
+        avail - free > .02) 
+      store->sync();
+
+    file_object_t poid = age_get_oid();
+    
+    int b = myrand() % 10;
+    age_objects[b].push_back(poid);
+    
+    ssize_t s = age_pick_size();
+    wrote += (s + 4095) / 4096;
+
+
+
+
+    generic_dout(2) << "age_fill at " << free << " / " << avail << " / " << pc << " creating " << hex << poid << dec << " sz " << s << dendl;
+    
+
+    if (false && start_debug && wrote > 1000000ULL) { 
+      /*
+
+
+      1005700
+?
+1005000
+1005700
+      1005710
+      1005725ULL
+      1005750ULL
+      1005800
+      1006000
+
+//  99  1000500 ? 1000750 1006000
+*/
+    }
+
+    off_t off = 0;
+    while (s) {
+      ssize_t t = MIN(s, max);
+      bufferlist sbl;
+      sbl.substr_of(bl, 0, t);
+      ObjectStore::Transaction tr;
+      hobject_t oid(sobject_t(poid, 0));
+      tr.write(coll_t(), oid, off, t, sbl);
+      store->apply_transaction(tr);
+      off += t;
+      s -= t;
+    }
+    poid.bno++;
+  }
+
+  return wrote*4; // KB
+}
+
+void Ager::age_empty(float pc) {
+  int nper = 20;
+  int n = nper;
+
+  while (1) {
+    struct statfs st;
+    store->statfs(&st);
+    float free = 1.0 - ((float)(st.f_bfree) / (float)st.f_blocks);
+    float avail = 1.0 - ((float)(st.f_bavail) / (float)st.f_blocks);  // to write to
+    generic_dout(2) << "age_empty at " << free << " / " << avail << " / " << pc << dendl;//" stopping" << dendl;
+    if (free <= pc) {
+      generic_dout(2) << "age_empty at " << free << " / " << avail << " / " << pc << " stopping" << dendl;
+      break;
+    }
+    
+    int b = myrand() % 10;
+    n--;
+    if (n == 0 || age_objects[b].empty()) {
+      generic_dout(2) << "age_empty sync" << dendl;
+      //sync();
+      //sync();
+      n = nper;
+      continue;
+    }
+    file_object_t poid = age_objects[b].front();
+    age_objects[b].pop_front();
+    
+    generic_dout(2) << "age_empty at " << free << " / " << avail << " / " << pc << " removing " << hex << poid << dec << dendl;
+    
+    ObjectStore::Transaction t;
+    hobject_t oid(sobject_t(poid, 0));
+    t.remove(coll_t(), oid);
+    store->apply_transaction(t);
+    age_free_oids.push_back(poid);
+  }
+}
+
+
+
+
+void Ager::age(int time,
+               float high_water,    // fill to this %
+               float low_water,     // then empty to this %
+               int count,         // this many times
+               float final_water,   // and end here ( <= low_water)
+               int fake_size_mb) { 
+
+  srand(0);
+
+  utime_t start = ceph_clock_now(cct);
+  utime_t until = start;
+  until.sec_ref() += time;
+  
+  //int elapsed = 0;
+  int freelist_inc = 60;
+  utime_t nextfl = start;
+  nextfl.sec_ref() += freelist_inc;
+
+  while (age_objects.size() < 10) age_objects.push_back( list() );
+  
+  if (fake_size_mb) {
+    int fake_bl = fake_size_mb * 256;
+    struct statfs st;
+    store->statfs(&st);
+    float f = (float)fake_bl / (float)st.f_blocks;
+    high_water = (float)high_water * f;
+    low_water = (float)low_water * f;
+    final_water = (float)final_water * f;
+    generic_dout(2) << "fake " << fake_bl << " / " << st.f_blocks << " is " << f << ", high " << high_water << " low " << low_water << " final " << final_water << dendl;
+  }
+  
+  // init size distn (once)
+  if (!did_distn) {
+    did_distn = true;
+    age_cur_oid = file_object_t(888, 0);
+    file_size_distn.add(1, 19.0758125+0.65434375);
+    file_size_distn.add(512, 35.6566);
+    file_size_distn.add(1024, 27.7271875);
+    file_size_distn.add(2*1024, 16.63503125);
+    //file_size_distn.add(4*1024, 106.82384375);
+    //file_size_distn.add(8*1024, 81.493375);
+    //file_size_distn.add(16*1024, 14.13553125);
+    //file_size_distn.add(32*1024, 2.176);
+    //file_size_distn.add(256*1024, 0.655938);
+    //file_size_distn.add(512*1024, 0.1480625);
+    //file_size_distn.add(1*1024*1024, 0.020125); // actually 2, but 32bit
+    file_size_distn.normalize();
+  }
+  
+  // clear
+  for (int i=0; i<10; i++)
+    age_objects[i].clear();
+  
+  uint64_t wrote = 0;
+
+  for (int c=1; c<=count; c++) {
+    if (ceph_clock_now(cct) > until) break;
+    
+    //if (c == 7) start_debug = true;
+    
+    generic_dout(1) << "#age " << c << "/" << count << " filling to " << high_water << dendl;
+    uint64_t w = age_fill(high_water, until);
+    //dout(1) << "age wrote " << w << dendl;
+    wrote += w;
+    //store->sync();
+    //store->_get_frag_stat(st);
+    //pfrag(st);
+
+
+    if (c == count) {
+      generic_dout(1) << "#age final empty to " << final_water << dendl;
+      age_empty(final_water);    
+    } else {
+      generic_dout(1) << "#age " << c << "/" << count << " emptying to " << low_water << dendl;
+      age_empty(low_water);
+    }
+    //store->sync();
+    //store->sync();
+
+    // show frag state
+    /*store->_get_frag_stat(st);
+    pfrag(wrote / (1024ULL*1024ULL) ,  // GB
+    st);*/
+
+    // dump freelist?
+    /*
+    if (ceph_clock_now(cct) > nextfl) {
+      elapsed += freelist_inc;
+      save_freelist(elapsed);
+      nextfl.sec_ref() += freelist_inc;
+    }
+    */
+  }
+
+  // dump the freelist
+  //save_freelist(0);
+  exit(0);   // hack
+
+  // ok!
+  store->sync();
+  store->sync();
+  generic_dout(1) << "age finished" << dendl;
+}  
diff --git a/ceph/src/osd/Ager.h b/ceph/src/osd/Ager.h
new file mode 100644
index 00000000..face0a62
--- /dev/null
+++ b/ceph/src/osd/Ager.h
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_AGER_H
+#define CEPH_AGER_H
+
+#include "include/types.h"
+#include "include/Distribution.h"
+#include "os/ObjectStore.h"
+#include "common/Clock.h"
+#include "common/ceph_context.h"
+
+#include 
+#include 
+using namespace std;
+
+class Ager {
+  CephContext *cct;
+  ObjectStore *store;
+
+ private:
+  list           age_free_oids;
+  file_object_t                 age_cur_oid;
+  vector< list > age_objects;
+  Distribution file_size_distn; //kb
+  bool         did_distn;
+
+  void age_empty(float pc);
+  uint64_t age_fill(float pc, utime_t until);
+  ssize_t age_pick_size();
+  file_object_t age_get_oid();
+
+ public:
+  Ager(CephContext *cct_, ObjectStore *s) : cct(cct_), store(s), did_distn(false) {}
+
+  void age(int time,
+           float high_water,    // fill to this %
+          float low_water,     // then empty to this %
+          int count,         // this many times
+          float final_water,   // and end here ( <= low_water)
+          int fake_size_mb=0);
+};
+
+#endif
diff --git a/ceph/src/osd/ClassHandler.cc b/ceph/src/osd/ClassHandler.cc
new file mode 100644
index 00000000..b1afe1e0
--- /dev/null
+++ b/ceph/src/osd/ClassHandler.cc
@@ -0,0 +1,267 @@
+
+#include "include/types.h"
+#include "msg/Message.h"
+#include "osd/OSD.h"
+#include "ClassHandler.h"
+
+#include 
+
+#include 
+
+#if defined(__FreeBSD__)
+#include 
+#endif
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix *_dout
+
+
+#define CLS_PREFIX "libcls_"
+#define CLS_SUFFIX ".so"
+
+
+int ClassHandler::open_class(const string& cname, ClassData **pcls)
+{
+  Mutex::Locker lock(mutex);
+  ClassData *cls = _get_class(cname);
+  if (cls->status != ClassData::CLASS_OPEN) {
+    int r = _load_class(cls);
+    if (r)
+      return r;
+  }
+  *pcls = cls;
+  return 0;
+}
+
+int ClassHandler::open_all_classes()
+{
+  dout(10) << __func__ << dendl;
+  DIR *dir = ::opendir(cct->_conf->osd_class_dir.c_str());
+  if (!dir)
+    return -errno;
+
+  char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
+  struct dirent *pde;
+  int r = 0;
+  while ((r = ::readdir_r(dir, (dirent *)&buf, &pde)) == 0 && pde) {
+    if (pde->d_name[0] == '.')
+      continue;
+    if (strlen(pde->d_name) > sizeof(CLS_PREFIX) - 1 + sizeof(CLS_SUFFIX) - 1 &&
+	strncmp(pde->d_name, CLS_PREFIX, sizeof(CLS_PREFIX) - 1) == 0 &&
+	strcmp(pde->d_name + strlen(pde->d_name) - (sizeof(CLS_SUFFIX) - 1), CLS_SUFFIX) == 0) {
+      char cname[PATH_MAX + 1];
+      strcpy(cname, pde->d_name + sizeof(CLS_PREFIX) - 1);
+      cname[strlen(cname) - (sizeof(CLS_SUFFIX) - 1)] = '\0';
+      dout(10) << __func__ << " found " << cname << dendl;
+      ClassData *cls;
+      r = open_class(cname, &cls);
+      if (r < 0)
+	goto out;
+    }
+  }
+ out:
+  closedir(dir);
+  return r;
+}
+
+void ClassHandler::shutdown()
+{
+  for (map::iterator p = classes.begin(); p != classes.end(); ++p) {
+    dlclose(p->second.handle);
+  }
+  classes.clear();
+}
+
+ClassHandler::ClassData *ClassHandler::_get_class(const string& cname)
+{
+  ClassData *cls;
+  map::iterator iter = classes.find(cname);
+
+  if (iter != classes.end()) {
+    cls = &iter->second;
+  } else {
+    cls = &classes[cname];
+    dout(10) << "_get_class adding new class name " << cname << " " << cls << dendl;
+    cls->name = cname;
+    cls->handler = this;
+  }
+  return cls;
+}
+
+int ClassHandler::_load_class(ClassData *cls)
+{
+  // already open
+  if (cls->status == ClassData::CLASS_OPEN)
+    return 0;
+
+  if (cls->status == ClassData::CLASS_UNKNOWN ||
+      cls->status == ClassData::CLASS_MISSING) {
+    char fname[PATH_MAX];
+    snprintf(fname, sizeof(fname), "%s/" CLS_PREFIX "%s" CLS_SUFFIX,
+	     cct->_conf->osd_class_dir.c_str(),
+	     cls->name.c_str());
+    dout(10) << "_load_class " << cls->name << " from " << fname << dendl;
+
+    struct stat st;
+    int r = ::stat(fname, &st);
+    if (r < 0)
+      return -errno;
+
+    cls->handle = dlopen(fname, RTLD_NOW);
+    if (!cls->handle) {
+      dout(0) << "_load_class could not open class " << fname
+	      << " (dlopen failed): " << dlerror() << dendl;
+      cls->status = ClassData::CLASS_MISSING;
+      return -EIO;
+    }
+
+    cls_deps_t *(*cls_deps)();
+    cls_deps = (cls_deps_t *(*)())dlsym(cls->handle, "class_deps");
+    if (cls_deps) {
+      cls_deps_t *deps = cls_deps();
+      while (deps) {
+	if (!deps->name)
+	  break;
+	ClassData *cls_dep = _get_class(deps->name);
+	cls->dependencies.insert(cls_dep);
+	if (cls_dep->status != ClassData::CLASS_OPEN)
+	  cls->missing_dependencies.insert(cls_dep);
+	deps++;
+      }
+    }
+  }
+
+  // resolve dependencies
+  set::iterator p = cls->missing_dependencies.begin();
+  while (p != cls->missing_dependencies.end()) {
+    ClassData *dc = *p;
+    int r = _load_class(dc);
+    if (r < 0) {
+      cls->status = ClassData::CLASS_MISSING_DEPS;
+      return r;
+    }
+    
+    dout(10) << "_load_class " << cls->name << " satisfied dependency " << dc->name << dendl;
+    cls->missing_dependencies.erase(p++);
+  }
+  
+  // initialize
+  void (*cls_init)() = (void (*)())dlsym(cls->handle, "__cls_init");
+  if (cls_init) {
+    cls->status = ClassData::CLASS_INITIALIZING;
+    cls_init();
+  }
+  
+  dout(10) << "_load_class " << cls->name << " success" << dendl;
+  cls->status = ClassData::CLASS_OPEN;
+  return 0;
+}
+
+
+
+ClassHandler::ClassData *ClassHandler::register_class(const char *cname)
+{
+  assert(mutex.is_locked());
+
+  ClassData *cls = _get_class(cname);
+  dout(10) << "register_class " << cname << " status " << cls->status << dendl;
+
+  if (cls->status != ClassData::CLASS_INITIALIZING) {
+    dout(0) << "class " << cname << " isn't loaded; is the class registering under the wrong name?" << dendl;
+    return NULL;
+  }
+  return cls;
+}
+
+void ClassHandler::unregister_class(ClassHandler::ClassData *cls)
+{
+  /* FIXME: do we really need this one? */
+}
+
+ClassHandler::ClassMethod *ClassHandler::ClassData::register_method(const char *mname,
+                                                                    int flags,
+								    cls_method_call_t func)
+{
+  /* no need for locking, called under the class_init mutex */
+  if (!flags) {
+    derr << "register_method " << name << "." << mname << " flags " << flags << " " << (void*)func
+	 << " FAILED -- flags must be non-zero" << dendl;
+    return NULL;
+  }
+  dout(10) << "register_method " << name << "." << mname << " flags " << flags << " " << (void*)func << dendl;
+  ClassMethod& method = methods_map[mname];
+  method.func = func;
+  method.name = mname;
+  method.flags = flags;
+  method.cls = this;
+  return &method;
+}
+
+ClassHandler::ClassMethod *ClassHandler::ClassData::register_cxx_method(const char *mname,
+                                                                        int flags,
+									cls_method_cxx_call_t func)
+{
+  /* no need for locking, called under the class_init mutex */
+  dout(10) << "register_cxx_method " << name << "." << mname << " flags " << flags << " " << (void*)func << dendl;
+  ClassMethod& method = methods_map[mname];
+  method.cxx_func = func;
+  method.name = mname;
+  method.flags = flags;
+  method.cls = this;
+  return &method;
+}
+
+ClassHandler::ClassMethod *ClassHandler::ClassData::_get_method(const char *mname)
+{
+  map::iterator iter = methods_map.find(mname);
+  if (iter == methods_map.end())
+    return NULL;
+  return &(iter->second);
+}
+
+int ClassHandler::ClassData::get_method_flags(const char *mname)
+{
+  Mutex::Locker l(handler->mutex);
+  ClassMethod *method = _get_method(mname);
+  if (!method)
+    return -ENOENT;
+  return method->flags;
+}
+
+void ClassHandler::ClassData::unregister_method(ClassHandler::ClassMethod *method)
+{
+  /* no need for locking, called under the class_init mutex */
+   map::iterator iter = methods_map.find(method->name);
+   if (iter == methods_map.end())
+     return;
+   methods_map.erase(iter);
+}
+
+void ClassHandler::ClassMethod::unregister()
+{
+  cls->unregister_method(this);
+}
+
+int ClassHandler::ClassMethod::exec(cls_method_context_t ctx, bufferlist& indata, bufferlist& outdata)
+{
+  int ret;
+  if (cxx_func) {
+    // C++ call version
+    ret = cxx_func(ctx, &indata, &outdata);
+  } else {
+    // C version
+    char *out = NULL;
+    int olen = 0;
+    ret = func(ctx, indata.c_str(), indata.length(), &out, &olen);
+    if (out) {
+      // assume *out was allocated via cls_alloc (which calls malloc!)
+      buffer::ptr bp = buffer::claim_malloc(olen, out);
+      outdata.push_back(bp);
+    }
+  }
+  return ret;
+}
+
diff --git a/ceph/src/osd/ClassHandler.h b/ceph/src/osd/ClassHandler.h
new file mode 100644
index 00000000..93cf3c07
--- /dev/null
+++ b/ceph/src/osd/ClassHandler.h
@@ -0,0 +1,95 @@
+#ifndef CEPH_CLASSHANDLER_H
+#define CEPH_CLASSHANDLER_H
+
+#include "include/types.h"
+
+#include "objclass/objclass.h"
+
+#include "common/Cond.h"
+#include "common/Mutex.h"
+#include "common/ceph_context.h"
+
+
+class ClassHandler
+{
+public:
+  CephContext *cct;
+
+  struct ClassData;
+
+  struct ClassMethod {
+    struct ClassHandler::ClassData *cls;
+    string name;
+    int flags;
+    cls_method_call_t func;
+    cls_method_cxx_call_t cxx_func;
+
+    int exec(cls_method_context_t ctx, bufferlist& indata, bufferlist& outdata);
+    void unregister();
+
+    int get_flags() {
+      Mutex::Locker l(cls->handler->mutex);
+      return flags;
+    }
+
+    ClassMethod() : cls(0), flags(0), func(0), cxx_func(0) {}
+  };
+
+  struct ClassData {
+    enum Status { 
+      CLASS_UNKNOWN,
+      CLASS_MISSING,         // missing
+      CLASS_MISSING_DEPS,    // missing dependencies
+      CLASS_INITIALIZING,    // calling init() right now
+      CLASS_OPEN,            // initialized, usable
+    } status;
+
+    string name;
+    ClassHandler *handler;
+    void *handle;
+
+    map methods_map;
+
+    set dependencies;         /* our dependencies */
+    set missing_dependencies; /* only missing dependencies */
+
+    ClassMethod *_get_method(const char *mname);
+
+    ClassData() : status(CLASS_UNKNOWN), 
+		  handler(NULL),
+		  handle(NULL) {}
+    ~ClassData() { }
+
+    ClassMethod *register_method(const char *mname, int flags, cls_method_call_t func);
+    ClassMethod *register_cxx_method(const char *mname, int flags, cls_method_cxx_call_t func);
+    void unregister_method(ClassMethod *method);
+
+    ClassMethod *get_method(const char *mname) {
+      Mutex::Locker l(handler->mutex);
+      return _get_method(mname);
+    }
+    int get_method_flags(const char *mname);
+  };
+
+private:
+  Mutex mutex;
+  map classes;
+
+  ClassData *_get_class(const string& cname);
+  int _load_class(ClassData *cls);
+
+public:
+  ClassHandler(CephContext *cct_) : cct(cct_), mutex("ClassHandler") {}
+  
+  int open_all_classes();
+
+  int open_class(const string& cname, ClassData **pcls);
+  
+  ClassData *register_class(const char *cname);
+  void unregister_class(ClassData *cls);
+
+  void shutdown();
+};
+
+
+#endif
diff --git a/ceph/src/osd/ECBackend.cc b/ceph/src/osd/ECBackend.cc
new file mode 100644
index 00000000..aefbb5e3
--- /dev/null
+++ b/ceph/src/osd/ECBackend.cc
@@ -0,0 +1,1785 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "ECUtil.h"
+#include "ECBackend.h"
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPushReply.h"
+
+#define dout_subsys ceph_subsys_osd
+#define DOUT_PREFIX_ARGS this
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+static ostream& _prefix(std::ostream *_dout, ECBackend *pgb) {
+  return *_dout << pgb->get_parent()->gen_dbg_prefix();
+}
+
+struct ECRecoveryHandle : public PGBackend::RecoveryHandle {
+  list ops;
+};
+
+static ostream &operator<<(ostream &lhs, const map &rhs)
+{
+  lhs << "[";
+  for (map::const_iterator i = rhs.begin();
+       i != rhs.end();
+       ++i) {
+    if (i != rhs.begin())
+      lhs << ", ";
+    lhs << make_pair(i->first, i->second.length());
+  }
+  return lhs << "]";
+}
+
+static ostream &operator<<(ostream &lhs, const map &rhs)
+{
+  lhs << "[";
+  for (map::const_iterator i = rhs.begin();
+       i != rhs.end();
+       ++i) {
+    if (i != rhs.begin())
+      lhs << ", ";
+    lhs << make_pair(i->first, i->second.length());
+  }
+  return lhs << "]";
+}
+
+static ostream &operator<<(
+  ostream &lhs,
+  const boost::tuple > &rhs)
+{
+  return lhs << "(" << rhs.get<0>() << ", "
+	     << rhs.get<1>() << ", " << rhs.get<2>() << ")";
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::read_request_t &rhs)
+{
+  return lhs << "read_request_t(to_read=[" << rhs.to_read << "]"
+	     << ", need=" << rhs.need
+	     << ", want_attrs=" << rhs.want_attrs
+	     << ")";
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::read_result_t &rhs)
+{
+  lhs << "read_result_t(r=" << rhs.r
+      << ", errors=" << rhs.errors;
+  if (rhs.attrs) {
+    lhs << ", attrs=" << rhs.attrs;
+  } else {
+    lhs << ", noattrs";
+  }
+  return lhs << ", returned=" << rhs.returned;
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::ReadOp &rhs)
+{
+  lhs << "ReadOp(tid=" << rhs.tid;
+  if (rhs.op && rhs.op->get_req()) {
+    lhs << ", op=";
+    rhs.op->get_req()->print(lhs);
+  }
+  return lhs << ", to_read=" << rhs.to_read
+	     << ", complete=" << rhs.complete
+	     << ", priority=" << rhs.priority
+	     << ", obj_to_source=" << rhs.obj_to_source
+	     << ", source_to_obj=" << rhs.source_to_obj
+	     << ", in_progress=" << rhs.in_progress << ")";
+}
+
+void ECBackend::ReadOp::dump(Formatter *f) const
+{
+  f->dump_unsigned("tid", tid);
+  if (op && op->get_req()) {
+    f->dump_stream("op") << *(op->get_req());
+  }
+  f->dump_stream("to_read") << to_read;
+  f->dump_stream("complete") << complete;
+  f->dump_int("priority", priority);
+  f->dump_stream("obj_to_source") << obj_to_source;
+  f->dump_stream("source_to_obj") << source_to_obj;
+  f->dump_stream("in_progress") << in_progress;
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::Op &rhs)
+{
+  lhs << "Op(" << rhs.hoid
+      << " v=" << rhs.version
+      << " tt=" << rhs.trim_to
+      << " tid=" << rhs.tid
+      << " reqid=" << rhs.reqid;
+  if (rhs.client_op && rhs.client_op->get_req()) {
+    lhs << " client_op=";
+    rhs.client_op->get_req()->print(lhs);
+  }
+  lhs << " pending_commit=" << rhs.pending_commit
+      << " pending_apply=" << rhs.pending_apply
+      << ")";
+  return lhs;
+}
+
+ostream &operator<<(ostream &lhs, const ECBackend::RecoveryOp &rhs)
+{
+  return lhs << "RecoveryOp("
+	     << "hoid=" << rhs.hoid
+	     << " v=" << rhs.v
+	     << " missing_on=" << rhs.missing_on
+	     << " missing_on_shards=" << rhs.missing_on_shards
+	     << " recovery_info=" << rhs.recovery_info
+	     << " recovery_progress=" << rhs.recovery_progress
+	     << " pending_read=" << rhs.pending_read
+	     << " obc refcount=" << rhs.obc.use_count()
+	     << " state=" << ECBackend::RecoveryOp::tostr(rhs.state)
+	     << " waiting_on_pushes=" << rhs.waiting_on_pushes
+	     << " extent_requested=" << rhs.extent_requested;
+}
+
+void ECBackend::RecoveryOp::dump(Formatter *f) const
+{
+  f->dump_stream("hoid") << hoid;
+  f->dump_stream("v") << v;
+  f->dump_stream("missing_on") << missing_on;
+  f->dump_stream("missing_on_shards") << missing_on_shards;
+  f->dump_stream("recovery_info") << recovery_info;
+  f->dump_stream("recovery_progress") << recovery_progress;
+  f->dump_bool("pending_read", pending_read);
+  f->dump_stream("state") << tostr(state);
+  f->dump_stream("waiting_on_pushes") << waiting_on_pushes;
+  f->dump_stream("extent_requested") << extent_requested;
+}
+
+ECBackend::ECBackend(
+  PGBackend::Listener *pg,
+  coll_t coll,
+  coll_t temp_coll,
+  ObjectStore *store,
+  CephContext *cct,
+  ErasureCodeInterfaceRef ec_impl,
+  uint64_t stripe_width)
+  : PGBackend(pg, store, coll, temp_coll),
+    cct(cct),
+    ec_impl(ec_impl),
+    sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
+  assert((ec_impl->get_data_chunk_count() *
+	  ec_impl->get_chunk_size(stripe_width)) == stripe_width);
+}
+
+PGBackend::RecoveryHandle *ECBackend::open_recovery_op()
+{
+  return new ECRecoveryHandle;
+}
+
+struct OnRecoveryReadComplete :
+  public GenContext &> {
+  ECBackend *pg;
+  hobject_t hoid;
+  set want;
+  OnRecoveryReadComplete(ECBackend *pg, const hobject_t &hoid)
+    : pg(pg), hoid(hoid) {}
+  void finish(pair &in) {
+    ECBackend::read_result_t &res = in.second;
+    assert(res.r == 0);
+    assert(res.errors.empty());
+    assert(res.returned.size() == 1);
+    pg->handle_recovery_read_complete(
+      hoid,
+      res.returned.back(),
+      res.attrs,
+      in.first);
+  }
+};
+
+struct RecoveryMessages {
+  map reads;
+  void read(
+    ECBackend *ec,
+    const hobject_t &hoid, uint64_t off, uint64_t len,
+    const set &need,
+    bool attrs) {
+    list > to_read;
+    to_read.push_back(make_pair(off, len));
+    assert(!reads.count(hoid));
+    reads.insert(
+      make_pair(
+	hoid,
+	ECBackend::read_request_t(
+	  hoid,
+	  to_read,
+	  need,
+	  attrs,
+	  new OnRecoveryReadComplete(
+	    ec,
+	    hoid))));
+  }
+
+  map > pushes;
+  map > push_replies;
+  ObjectStore::Transaction *t;
+  RecoveryMessages() : t(new ObjectStore::Transaction) {}
+  ~RecoveryMessages() { assert(!t); }
+};
+
+void ECBackend::handle_recovery_push(
+  PushOp &op,
+  RecoveryMessages *m)
+{
+  bool oneshot = op.before_progress.first && op.after_progress.data_complete;
+  coll_t tcoll = oneshot ? coll : get_temp_coll(m->t);
+  if (op.before_progress.first) {
+    get_parent()->on_local_recover_start(
+      op.soid,
+      m->t);
+    m->t->remove(
+      get_temp_coll(m->t),
+      ghobject_t(
+	op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+    m->t->touch(
+      tcoll,
+      ghobject_t(
+	op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+  }
+
+  if (!op.data_included.empty()) {
+    uint64_t start = op.data_included.range_start();
+    uint64_t end = op.data_included.range_end();
+    assert(op.data.length() == (end - start));
+
+    m->t->write(
+      tcoll,
+      ghobject_t(
+	op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+      start,
+      op.data.length(),
+      op.data);
+  } else {
+    assert(op.data.length() == 0);
+  }
+
+  if (op.before_progress.first) {
+    if (!oneshot)
+      add_temp_obj(op.soid);
+    assert(op.attrset.count(string("_")));
+    m->t->setattrs(
+      tcoll,
+      ghobject_t(
+	op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+      op.attrset);
+  }
+
+  if (op.after_progress.data_complete && !oneshot) {
+    clear_temp_obj(op.soid);
+    m->t->collection_move(
+      coll,
+      tcoll,
+      ghobject_t(
+	op.soid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+  }
+  if (op.after_progress.data_complete) {
+    if ((get_parent()->pgb_is_primary())) {
+      assert(recovery_ops.count(op.soid));
+      assert(recovery_ops[op.soid].obc);
+      object_stat_sum_t stats;
+      stats.num_objects_recovered = 1;
+      stats.num_bytes_recovered = recovery_ops[op.soid].obc->obs.oi.size;
+      get_parent()->on_local_recover(
+	op.soid,
+	stats,
+	op.recovery_info,
+	recovery_ops[op.soid].obc,
+	m->t);
+    } else {
+      get_parent()->on_local_recover(
+	op.soid,
+	object_stat_sum_t(),
+	op.recovery_info,
+	ObjectContextRef(),
+	m->t);
+    }
+  }
+  m->push_replies[get_parent()->primary_shard()].push_back(PushReplyOp());
+  m->push_replies[get_parent()->primary_shard()].back().soid = op.soid;
+}
+
+void ECBackend::handle_recovery_push_reply(
+  PushReplyOp &op,
+  pg_shard_t from,
+  RecoveryMessages *m)
+{
+  if (!recovery_ops.count(op.soid))
+    return;
+  RecoveryOp &rop = recovery_ops[op.soid];
+  assert(rop.waiting_on_pushes.count(from));
+  rop.waiting_on_pushes.erase(from);
+  continue_recovery_op(rop, m);
+}
+
+void ECBackend::handle_recovery_read_complete(
+  const hobject_t &hoid,
+  boost::tuple > &to_read,
+  boost::optional > attrs,
+  RecoveryMessages *m)
+{
+  dout(10) << __func__ << ": returned " << hoid << " "
+	   << "(" << to_read.get<0>()
+	   << ", " << to_read.get<1>()
+	   << ", " << to_read.get<2>()
+	   << ")"
+	   << dendl;
+  assert(recovery_ops.count(hoid));
+  RecoveryOp &op = recovery_ops[hoid];
+  assert(op.returned_data.empty());
+  map target;
+  for (set::iterator i = op.missing_on_shards.begin();
+       i != op.missing_on_shards.end();
+       ++i) {
+    target[*i] = &(op.returned_data[*i]);
+  }
+  map from;
+  for(map::iterator i = to_read.get<2>().begin();
+      i != to_read.get<2>().end();
+      ++i) {
+    from[i->first.shard].claim(i->second);
+  }
+  dout(10) << __func__ << ": " << from << dendl;
+  ECUtil::decode(sinfo, ec_impl, from, target);
+  if (attrs) {
+    op.xattrs.swap(*attrs);
+
+    if (!op.obc) {
+      op.obc = get_parent()->get_obc(hoid, op.xattrs);
+      op.recovery_info.size = op.obc->obs.oi.size;
+      op.recovery_info.oi = op.obc->obs.oi;
+    }
+
+    ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
+    if (op.obc->obs.oi.size > 0) {
+      assert(op.xattrs.count(ECUtil::get_hinfo_key()));
+      bufferlist::iterator bp = op.xattrs[ECUtil::get_hinfo_key()].begin();
+      ::decode(hinfo, bp);
+    }
+    op.hinfo = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
+  }
+  assert(op.xattrs.size());
+  assert(op.obc);
+  continue_recovery_op(op, m);
+}
+
+struct SendPushReplies : public Context {
+  PGBackend::Listener *l;
+  epoch_t epoch;
+  map replies;
+  SendPushReplies(
+    PGBackend::Listener *l,
+    epoch_t epoch,
+    map &in) : l(l), epoch(epoch) {
+    replies.swap(in);
+  }
+  void finish(int) {
+    for (map::iterator i = replies.begin();
+	 i != replies.end();
+	 ++i) {
+      l->send_message_osd_cluster(i->first, i->second, epoch);
+    }
+    replies.clear();
+  }
+  ~SendPushReplies() {
+    for (map::iterator i = replies.begin();
+	 i != replies.end();
+	 ++i) {
+      i->second->put();
+    }
+    replies.clear();
+  }
+};
+
+void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
+{
+  for (map >::iterator i = m.pushes.begin();
+       i != m.pushes.end();
+       m.pushes.erase(i++)) {
+    MOSDPGPush *msg = new MOSDPGPush();
+    msg->set_priority(priority);
+    msg->map_epoch = get_parent()->get_epoch();
+    msg->from = get_parent()->whoami_shard();
+    msg->pgid = spg_t(get_parent()->get_info().pgid.pgid, i->first.shard);
+    msg->pushes.swap(i->second);
+    msg->compute_cost(cct);
+    get_parent()->send_message(
+      i->first.osd,
+      msg);
+  }
+  map replies;
+  for (map >::iterator i =
+	 m.push_replies.begin();
+       i != m.push_replies.end();
+       m.push_replies.erase(i++)) {
+    MOSDPGPushReply *msg = new MOSDPGPushReply();
+    msg->set_priority(priority);
+    msg->map_epoch = get_parent()->get_epoch();
+    msg->from = get_parent()->whoami_shard();
+    msg->pgid = spg_t(get_parent()->get_info().pgid.pgid, i->first.shard);
+    msg->replies.swap(i->second);
+    msg->compute_cost(cct);
+    replies.insert(make_pair(i->first.osd, msg));
+  }
+  m.t->register_on_complete(
+    get_parent()->bless_context(
+      new SendPushReplies(
+	get_parent(),
+	get_parent()->get_epoch(),
+	replies)));
+  m.t->register_on_applied(
+    new ObjectStore::C_DeleteTransaction(m.t));
+  get_parent()->queue_transaction(m.t);
+  m.t = NULL;
+  if (m.reads.empty())
+    return;
+  start_read_op(
+    priority,
+    m.reads,
+    OpRequestRef());
+}
+
+void ECBackend::continue_recovery_op(
+  RecoveryOp &op,
+  RecoveryMessages *m)
+{
+  dout(10) << __func__ << ": continuing " << op << dendl;
+  while (1) {
+    switch (op.state) {
+    case RecoveryOp::IDLE: {
+      // start read
+      op.state = RecoveryOp::READING;
+      assert(!op.recovery_progress.data_complete);
+      set want(op.missing_on_shards.begin(), op.missing_on_shards.end());
+      set to_read;
+      int r = get_min_avail_to_read_shards(
+	op.hoid, want, true, &to_read);
+      if (r != 0) {
+	// we must have lost a recovery source
+	assert(!op.recovery_progress.first);
+	dout(10) << __func__ << ": canceling recovery op for obj " << op.hoid
+		 << dendl;
+	get_parent()->cancel_pull(op.hoid);
+	recovery_ops.erase(op.hoid);
+	return;
+      }
+      assert(r == 0);
+      m->read(
+	this,
+	op.hoid,
+	op.recovery_progress.data_recovered_to,
+	get_recovery_chunk_size(),
+	to_read,
+	op.recovery_progress.first);
+      op.extent_requested = make_pair(op.recovery_progress.data_recovered_to,
+				      get_recovery_chunk_size());
+      dout(10) << __func__ << ": IDLE return " << op << dendl;
+      return;
+    }
+    case RecoveryOp::READING: {
+      // read completed, start write
+      assert(op.xattrs.size());
+      assert(op.returned_data.size());
+      op.state = RecoveryOp::WRITING;
+      ObjectRecoveryProgress after_progress = op.recovery_progress;
+      after_progress.data_recovered_to += get_recovery_chunk_size();
+      after_progress.first = false;
+      if (after_progress.data_recovered_to >= op.obc->obs.oi.size) {
+	after_progress.data_recovered_to =
+	  sinfo.logical_to_next_stripe_offset(
+	    op.obc->obs.oi.size);
+	after_progress.data_complete = true;
+      }
+      for (set::iterator mi = op.missing_on.begin();
+	   mi != op.missing_on.end();
+	   ++mi) {
+	assert(op.returned_data.count(mi->shard));
+	m->pushes[*mi].push_back(PushOp());
+	PushOp &pop = m->pushes[*mi].back();
+	pop.soid = op.hoid;
+	pop.version = op.v;
+	pop.data = op.returned_data[mi->shard];
+	dout(10) << __func__ << ": before_progress=" << op.recovery_progress
+		 << ", after_progress=" << after_progress
+		 << ", pop.data.length()=" << pop.data.length()
+		 << ", size=" << op.obc->obs.oi.size << dendl;
+	assert(
+	  pop.data.length() ==
+	  sinfo.aligned_logical_offset_to_chunk_offset(
+	    after_progress.data_recovered_to -
+	    op.recovery_progress.data_recovered_to)
+	  );
+	if (pop.data.length())
+	  pop.data_included.insert(
+	    sinfo.aligned_logical_offset_to_chunk_offset(
+	      op.recovery_progress.data_recovered_to),
+	    pop.data.length()
+	    );
+	if (op.recovery_progress.first) {
+	  pop.attrset = op.xattrs;
+	}
+	pop.recovery_info = op.recovery_info;
+	pop.before_progress = op.recovery_progress;
+	pop.after_progress = after_progress;
+	if (*mi != get_parent()->primary_shard())
+	  get_parent()->begin_peer_recover(
+	    *mi,
+	    op.hoid);
+      }
+      op.returned_data.clear();
+      op.waiting_on_pushes = op.missing_on;
+      op.recovery_progress = after_progress;
+      dout(10) << __func__ << ": READING return " << op << dendl;
+      return;
+    }
+    case RecoveryOp::WRITING: {
+      if (op.waiting_on_pushes.empty()) {
+	if (op.recovery_progress.data_complete) {
+	  op.state = RecoveryOp::COMPLETE;
+	  for (set::iterator i = op.missing_on.begin();
+	       i != op.missing_on.end();
+	       ++i) {
+	    if (*i != get_parent()->primary_shard()) {
+	      dout(10) << __func__ << ": on_peer_recover on " << *i
+		       << ", obj " << op.hoid << dendl;
+	      get_parent()->on_peer_recover(
+		*i,
+		op.hoid,
+		op.recovery_info,
+		object_stat_sum_t());
+	    }
+	  }
+	  get_parent()->on_global_recover(op.hoid);
+	  dout(10) << __func__ << ": WRITING return " << op << dendl;
+	  recovery_ops.erase(op.hoid);
+	  return;
+	} else {
+	  op.state = RecoveryOp::IDLE;
+	  dout(10) << __func__ << ": WRITING continue " << op << dendl;
+	  continue;
+	}
+      }
+      return;
+    }
+    case RecoveryOp::COMPLETE: {
+      assert(0); // should never be called once complete
+    };
+    default:
+      assert(0);
+    }
+  }
+}
+
+void ECBackend::run_recovery_op(
+  RecoveryHandle *_h,
+  int priority)
+{
+  ECRecoveryHandle *h = static_cast(_h);
+  RecoveryMessages m;
+  for (list::iterator i = h->ops.begin();
+       i != h->ops.end();
+       ++i) {
+    dout(10) << __func__ << ": starting " << *i << dendl;
+    assert(!recovery_ops.count(i->hoid));
+    RecoveryOp &op = recovery_ops.insert(make_pair(i->hoid, *i)).first->second;
+    continue_recovery_op(op, &m);
+  }
+  dispatch_recovery_messages(m, priority);
+  delete _h;
+}
+
+void ECBackend::recover_object(
+  const hobject_t &hoid,
+  eversion_t v,
+  ObjectContextRef head,
+  ObjectContextRef obc,
+  RecoveryHandle *_h)
+{
+  ECRecoveryHandle *h = static_cast(_h);
+  h->ops.push_back(RecoveryOp());
+  h->ops.back().v = v;
+  h->ops.back().hoid = hoid;
+  h->ops.back().obc = obc;
+  h->ops.back().recovery_info.soid = hoid;
+  h->ops.back().recovery_info.version = v;
+  if (obc) {
+    h->ops.back().recovery_info.size = obc->obs.oi.size;
+    h->ops.back().recovery_info.oi = obc->obs.oi;
+  }
+  h->ops.back().recovery_progress.omap_complete = true;
+  for (set::const_iterator i =
+	 get_parent()->get_actingbackfill_shards().begin();
+       i != get_parent()->get_actingbackfill_shards().end();
+       ++i) {
+    dout(10) << "checking " << *i << dendl;
+    if (get_parent()->get_shard_missing(*i).is_missing(hoid)) {
+      h->ops.back().missing_on.insert(*i);
+      h->ops.back().missing_on_shards.insert(i->shard);
+    }
+  }
+  dout(10) << __func__ << ": built op " << h->ops.back() << dendl;
+}
+
+bool ECBackend::can_handle_while_inactive(
+  OpRequestRef _op)
+{
+  return false;
+}
+
+bool ECBackend::handle_message(
+  OpRequestRef _op)
+{
+  dout(10) << __func__ << ": " << *_op->get_req() << dendl;
+  int priority = _op->get_req()->get_priority();
+  switch (_op->get_req()->get_type()) {
+  case MSG_OSD_EC_WRITE: {
+    MOSDECSubOpWrite *op = static_cast(_op->get_req());
+    handle_sub_write(op->op.from, _op, op->op);
+    return true;
+  }
+  case MSG_OSD_EC_WRITE_REPLY: {
+    MOSDECSubOpWriteReply *op = static_cast(
+      _op->get_req());
+    op->set_priority(priority);
+    handle_sub_write_reply(op->op.from, op->op);
+    return true;
+  }
+  case MSG_OSD_EC_READ: {
+    MOSDECSubOpRead *op = static_cast(_op->get_req());
+    MOSDECSubOpReadReply *reply = new MOSDECSubOpReadReply;
+    reply->pgid = get_parent()->primary_spg_t();
+    reply->map_epoch = get_parent()->get_epoch();
+    handle_sub_read(op->op.from, op->op, &(reply->op));
+    op->set_priority(priority);
+    get_parent()->send_message_osd_cluster(
+      op->op.from.osd, reply, get_parent()->get_epoch());
+    return true;
+  }
+  case MSG_OSD_EC_READ_REPLY: {
+    MOSDECSubOpReadReply *op = static_cast(
+      _op->get_req());
+    RecoveryMessages rm;
+    handle_sub_read_reply(op->op.from, op->op, &rm);
+    dispatch_recovery_messages(rm, priority);
+    return true;
+  }
+  case MSG_OSD_PG_PUSH: {
+    MOSDPGPush *op = static_cast(_op->get_req());
+    RecoveryMessages rm;
+    for (vector::iterator i = op->pushes.begin();
+	 i != op->pushes.end();
+	 ++i) {
+      handle_recovery_push(*i, &rm);
+    }
+    dispatch_recovery_messages(rm, priority);
+    return true;
+  }
+  case MSG_OSD_PG_PUSH_REPLY: {
+    MOSDPGPushReply *op = static_cast(_op->get_req());
+    RecoveryMessages rm;
+    for (vector::iterator i = op->replies.begin();
+	 i != op->replies.end();
+	 ++i) {
+      handle_recovery_push_reply(*i, op->from, &rm);
+    }
+    dispatch_recovery_messages(rm, priority);
+    return true;
+  }
+  default:
+    return false;
+  }
+  return false;
+}
+
+struct SubWriteCommitted : public Context {
+  ECBackend *pg;
+  OpRequestRef msg;
+  ceph_tid_t tid;
+  eversion_t version;
+  eversion_t last_complete;
+  SubWriteCommitted(
+    ECBackend *pg,
+    OpRequestRef msg,
+    ceph_tid_t tid,
+    eversion_t version,
+    eversion_t last_complete)
+    : pg(pg), msg(msg), tid(tid),
+      version(version), last_complete(last_complete) {}
+  void finish(int) {
+    if (msg)
+      msg->mark_event("sub_op_committed");
+    pg->sub_write_committed(tid, version, last_complete);
+  }
+};
+void ECBackend::sub_write_committed(
+  ceph_tid_t tid, eversion_t version, eversion_t last_complete) {
+  if (get_parent()->pgb_is_primary()) {
+    ECSubWriteReply reply;
+    reply.tid = tid;
+    reply.last_complete = last_complete;
+    reply.committed = true;
+    reply.from = get_parent()->whoami_shard();
+    handle_sub_write_reply(
+      get_parent()->whoami_shard(),
+      reply);
+  } else {
+    get_parent()->update_last_complete_ondisk(last_complete);
+    MOSDECSubOpWriteReply *r = new MOSDECSubOpWriteReply;
+    r->pgid = get_parent()->primary_spg_t();
+    r->map_epoch = get_parent()->get_epoch();
+    r->op.tid = tid;
+    r->op.last_complete = last_complete;
+    r->op.committed = true;
+    r->op.from = get_parent()->whoami_shard();
+    get_parent()->send_message_osd_cluster(
+      get_parent()->primary_shard().osd, r, get_parent()->get_epoch());
+  }
+}
+
+struct SubWriteApplied : public Context {
+  ECBackend *pg;
+  OpRequestRef msg;
+  ceph_tid_t tid;
+  eversion_t version;
+  SubWriteApplied(
+    ECBackend *pg,
+    OpRequestRef msg,
+    ceph_tid_t tid,
+    eversion_t version)
+    : pg(pg), msg(msg), tid(tid), version(version) {}
+  void finish(int) {
+    if (msg)
+      msg->mark_event("sub_op_applied");
+    pg->sub_write_applied(tid, version);
+  }
+};
+void ECBackend::sub_write_applied(
+  ceph_tid_t tid, eversion_t version) {
+  parent->op_applied(version);
+  if (get_parent()->pgb_is_primary()) {
+    ECSubWriteReply reply;
+    reply.from = get_parent()->whoami_shard();
+    reply.tid = tid;
+    reply.applied = true;
+    handle_sub_write_reply(
+      get_parent()->whoami_shard(),
+      reply);
+  } else {
+    MOSDECSubOpWriteReply *r = new MOSDECSubOpWriteReply;
+    r->pgid = get_parent()->primary_spg_t();
+    r->map_epoch = get_parent()->get_epoch();
+    r->op.from = get_parent()->whoami_shard();
+    r->op.tid = tid;
+    r->op.applied = true;
+    get_parent()->send_message_osd_cluster(
+      get_parent()->primary_shard().osd, r, get_parent()->get_epoch());
+  }
+}
+
+void ECBackend::handle_sub_write(
+  pg_shard_t from,
+  OpRequestRef msg,
+  ECSubWrite &op,
+  Context *on_local_applied_sync)
+{
+  if (msg)
+    msg->mark_started();
+  assert(!get_parent()->get_log().get_missing().is_missing(op.soid));
+  if (!get_parent()->pgb_is_primary())
+    get_parent()->update_stats(op.stats);
+  ObjectStore::Transaction *localt = new ObjectStore::Transaction;
+  if (!op.temp_added.empty()) {
+    get_temp_coll(localt);
+    add_temp_objs(op.temp_added);
+  }
+  if (op.t.empty()) {
+    for (set::iterator i = op.temp_removed.begin();
+	 i != op.temp_removed.end();
+	 ++i) {
+      dout(10) << __func__ << ": removing object " << *i
+	       << " since we won't get the transaction" << dendl;
+      localt->remove(
+	temp_coll,
+	ghobject_t(
+	  *i,
+	  ghobject_t::NO_GEN,
+	  get_parent()->whoami_shard().shard));
+    }
+  }
+  clear_temp_objs(op.temp_removed);
+  get_parent()->log_operation(
+    op.log_entries,
+    op.updated_hit_set_history,
+    op.trim_to,
+    op.trim_rollback_to,
+    !(op.t.empty()),
+    localt);
+  localt->append(op.t);
+  if (on_local_applied_sync) {
+    dout(10) << "Queueing onreadable_sync: " << on_local_applied_sync << dendl;
+    localt->register_on_applied_sync(on_local_applied_sync);
+  }
+  localt->register_on_commit(
+    get_parent()->bless_context(
+      new SubWriteCommitted(
+	this, msg, op.tid,
+	op.at_version,
+	get_parent()->get_info().last_complete)));
+  localt->register_on_applied(
+    get_parent()->bless_context(
+      new SubWriteApplied(this, msg, op.tid, op.at_version)));
+  localt->register_on_applied(
+    new ObjectStore::C_DeleteTransaction(localt));
+  get_parent()->queue_transaction(localt, msg);
+}
+
+void ECBackend::handle_sub_read(
+  pg_shard_t from,
+  ECSubRead &op,
+  ECSubReadReply *reply)
+{
+  for(map > >::iterator i =
+        op.to_read.begin();
+      i != op.to_read.end();
+      ++i) {
+    for (list >::iterator j = i->second.begin();
+	 j != i->second.end();
+	 ++j) {
+      bufferlist bl;
+      int r = store->read(
+	i->first.is_temp() ? temp_coll : coll,
+	ghobject_t(
+	  i->first, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+	j->first,
+	j->second,
+	bl,
+	false);
+      if (r < 0) {
+	assert(0);
+	reply->buffers_read.erase(i->first);
+	reply->errors[i->first] = r;
+	break;
+      } else {
+	reply->buffers_read[i->first].push_back(
+	  make_pair(
+	    j->first,
+	    bl)
+	  );
+      }
+    }
+  }
+  for (set::iterator i = op.attrs_to_read.begin();
+       i != op.attrs_to_read.end();
+       ++i) {
+    dout(10) << __func__ << ": fulfilling attr request on "
+	     << *i << dendl;
+    if (reply->errors.count(*i))
+      continue;
+    int r = store->getattrs(
+      i->is_temp() ? temp_coll : coll,
+      ghobject_t(
+	*i, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+      reply->attrs_read[*i]);
+    if (r < 0) {
+      assert(0);
+      reply->buffers_read.erase(*i);
+      reply->errors[*i] = r;
+    }
+  }
+  reply->from = get_parent()->whoami_shard();
+  reply->tid = op.tid;
+}
+
+void ECBackend::handle_sub_write_reply(
+  pg_shard_t from,
+  ECSubWriteReply &op)
+{
+  map::iterator i = tid_to_op_map.find(op.tid);
+  assert(i != tid_to_op_map.end());
+  if (op.committed) {
+    assert(i->second.pending_commit.count(from));
+    i->second.pending_commit.erase(from);
+    if (from != get_parent()->whoami_shard()) {
+      get_parent()->update_peer_last_complete_ondisk(from, op.last_complete);
+    }
+  }
+  if (op.applied) {
+    assert(i->second.pending_apply.count(from));
+    i->second.pending_apply.erase(from);
+  }
+  check_op(&(i->second));
+}
+
+void ECBackend::handle_sub_read_reply(
+  pg_shard_t from,
+  ECSubReadReply &op,
+  RecoveryMessages *m)
+{
+  dout(10) << __func__ << ": reply " << op << dendl;
+  map::iterator iter = tid_to_read_map.find(op.tid);
+  if (iter == tid_to_read_map.end()) {
+    //canceled
+    return;
+  }
+  ReadOp &rop = iter->second;
+  for (map > >::iterator i =
+	 op.buffers_read.begin();
+       i != op.buffers_read.end();
+       ++i) {
+    assert(!op.errors.count(i->first));
+    if (!rop.to_read.count(i->first)) {
+      // We canceled this read! @see filter_read_op
+      continue;
+    }
+    list >::const_iterator req_iter =
+      rop.to_read.find(i->first)->second.to_read.begin();
+    list<
+      boost::tuple<
+	uint64_t, uint64_t, map > >::iterator riter =
+      rop.complete[i->first].returned.begin();
+    for (list >::iterator j = i->second.begin();
+	 j != i->second.end();
+	 ++j, ++req_iter, ++riter) {
+      assert(req_iter != rop.to_read.find(i->first)->second.to_read.end());
+      assert(riter != rop.complete[i->first].returned.end());
+      pair adjusted =
+	sinfo.aligned_offset_len_to_chunk(
+	  *req_iter);
+      assert(adjusted.first == j->first);
+      riter->get<2>()[from].claim(j->second);
+    }
+  }
+  for (map >::iterator i = op.attrs_read.begin();
+       i != op.attrs_read.end();
+       ++i) {
+    assert(!op.errors.count(i->first));
+    if (!rop.to_read.count(i->first)) {
+      // We canceled this read! @see filter_read_op
+      continue;
+    }
+    rop.complete[i->first].attrs = map();
+    (*(rop.complete[i->first].attrs)).swap(i->second);
+  }
+  for (map::iterator i = op.errors.begin();
+       i != op.errors.end();
+       ++i) {
+    rop.complete[i->first].errors.insert(
+      make_pair(
+	from,
+	i->second));
+    if (rop.complete[i->first].r == 0)
+      rop.complete[i->first].r = i->second;
+  }
+
+  map >::iterator siter =
+shard_to_read_map.find(from);
+  assert(siter != shard_to_read_map.end());
+  assert(siter->second.count(op.tid));
+  siter->second.erase(op.tid);
+
+  assert(rop.in_progress.count(from));
+  rop.in_progress.erase(from);
+  if (!rop.in_progress.empty()) {
+    dout(10) << __func__ << " readop not complete: " << rop << dendl;
+  } else {
+    dout(10) << __func__ << " readop complete: " << rop << dendl;
+    complete_read_op(rop, m);
+  }
+}
+
+void ECBackend::complete_read_op(ReadOp &rop, RecoveryMessages *m)
+{
+  map::iterator reqiter =
+    rop.to_read.begin();
+  map::iterator resiter =
+    rop.complete.begin();
+  assert(rop.to_read.size() == rop.complete.size());
+  for (; reqiter != rop.to_read.end(); ++reqiter, ++resiter) {
+    if (reqiter->second.cb) {
+      pair arg(
+	m, resiter->second);
+      reqiter->second.cb->complete(arg);
+      reqiter->second.cb = NULL;
+    }
+  }
+  tid_to_read_map.erase(rop.tid);
+}
+
+struct FinishReadOp : public GenContext  {
+  ECBackend *ec;
+  ceph_tid_t tid;
+  FinishReadOp(ECBackend *ec, ceph_tid_t tid) : ec(ec), tid(tid) {}
+  void finish(ThreadPool::TPHandle &handle) {
+    assert(ec->tid_to_read_map.count(tid));
+    int priority = ec->tid_to_read_map[tid].priority;
+    RecoveryMessages rm;
+    ec->complete_read_op(ec->tid_to_read_map[tid], &rm);
+    ec->dispatch_recovery_messages(rm, priority);
+  }
+};
+
+void ECBackend::filter_read_op(
+  const OSDMapRef osdmap,
+  ReadOp &op)
+{
+  set to_cancel;
+  for (map >::iterator i = op.source_to_obj.begin();
+       i != op.source_to_obj.end();
+       ++i) {
+    if (osdmap->is_down(i->first.osd)) {
+      to_cancel.insert(i->second.begin(), i->second.end());
+      op.in_progress.erase(i->first);
+      continue;
+    }
+  }
+
+  if (to_cancel.empty())
+    return;
+
+  for (map >::iterator i = op.source_to_obj.begin();
+       i != op.source_to_obj.end();
+       ) {
+    for (set::iterator j = i->second.begin();
+	 j != i->second.end();
+	 ) {
+      if (to_cancel.count(*j))
+	i->second.erase(j++);
+      else
+	++j;
+    }
+    if (i->second.empty()) {
+      op.source_to_obj.erase(i++);
+    } else {
+      assert(!osdmap->is_down(i->first.osd));
+      ++i;
+    }
+  }
+
+  for (set::iterator i = to_cancel.begin();
+       i != to_cancel.end();
+       ++i) {
+    get_parent()->cancel_pull(*i);
+
+    assert(op.to_read.count(*i));
+    read_request_t &req = op.to_read.find(*i)->second;
+    dout(10) << __func__ << ": canceling " << req
+	     << "  for obj " << *i << dendl;
+    assert(req.cb);
+    delete req.cb;
+    req.cb = NULL;
+
+    op.to_read.erase(*i);
+    op.complete.erase(*i);
+    recovery_ops.erase(*i);
+  }
+
+  if (op.in_progress.empty()) {
+    get_parent()->schedule_work(
+      get_parent()->bless_gencontext(
+	new FinishReadOp(this, op.tid)));
+  }
+};
+
+void ECBackend::check_recovery_sources(const OSDMapRef osdmap)
+{
+  set tids_to_filter;
+  for (map >::iterator 
+       i = shard_to_read_map.begin();
+       i != shard_to_read_map.end();
+       ) {
+    if (osdmap->is_down(i->first.osd)) {
+      tids_to_filter.insert(i->second.begin(), i->second.end());
+      shard_to_read_map.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+  for (set::iterator i = tids_to_filter.begin();
+       i != tids_to_filter.end();
+       ++i) {
+    map::iterator j = tid_to_read_map.find(*i);
+    assert(j != tid_to_read_map.end());
+    filter_read_op(osdmap, j->second);
+  }
+}
+
+void ECBackend::on_change()
+{
+  dout(10) << __func__ << dendl;
+  writing.clear();
+  tid_to_op_map.clear();
+  for (map::iterator i = tid_to_read_map.begin();
+       i != tid_to_read_map.end();
+       ++i) {
+    dout(10) << __func__ << ": cancelling " << i->second << dendl;
+    for (map::iterator j =
+	   i->second.to_read.begin();
+	 j != i->second.to_read.end();
+	 ++j) {
+      delete j->second.cb;
+      j->second.cb = 0;
+    }
+  }
+  tid_to_read_map.clear();
+  for (list::iterator i = in_progress_client_reads.begin();
+       i != in_progress_client_reads.end();
+       ++i) {
+    delete i->on_complete;
+    i->on_complete = NULL;
+  }
+  in_progress_client_reads.clear();
+  shard_to_read_map.clear();
+  clear_state();
+}
+
+void ECBackend::clear_state()
+{
+  recovery_ops.clear();
+}
+
+void ECBackend::on_flushed()
+{
+}
+
+void ECBackend::dump_recovery_info(Formatter *f) const
+{
+  f->open_array_section("recovery_ops");
+  for (map::const_iterator i = recovery_ops.begin();
+       i != recovery_ops.end();
+       ++i) {
+    f->open_object_section("op");
+    i->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("read_ops");
+  for (map::const_iterator i = tid_to_read_map.begin();
+       i != tid_to_read_map.end();
+       ++i) {
+    f->open_object_section("read_op");
+    i->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+PGBackend::PGTransaction *ECBackend::get_transaction()
+{
+  return new ECTransaction;
+}
+
+struct MustPrependHashInfo : public ObjectModDesc::Visitor {
+  enum { EMPTY, FOUND_APPEND, FOUND_CREATE_STASH } state;
+  MustPrependHashInfo() : state(EMPTY) {}
+  void append(uint64_t) {
+    if (state == EMPTY) {
+      state = FOUND_APPEND;
+    }
+  }
+  void rmobject(version_t) {
+    if (state == EMPTY) {
+      state = FOUND_CREATE_STASH;
+    }
+  }
+  void create() {
+    if (state == EMPTY) {
+      state = FOUND_CREATE_STASH;
+    }
+  }
+  bool must_prepend_hash_info() const { return state == FOUND_APPEND; }
+};
+
+void ECBackend::submit_transaction(
+  const hobject_t &hoid,
+  const eversion_t &at_version,
+  PGTransaction *_t,
+  const eversion_t &trim_to,
+  const eversion_t &trim_rollback_to,
+  vector &log_entries,
+  boost::optional &hset_history,
+  Context *on_local_applied_sync,
+  Context *on_all_applied,
+  Context *on_all_commit,
+  ceph_tid_t tid,
+  osd_reqid_t reqid,
+  OpRequestRef client_op
+  )
+{
+  assert(!tid_to_op_map.count(tid));
+  Op *op = &(tid_to_op_map[tid]);
+  op->hoid = hoid;
+  op->version = at_version;
+  op->trim_to = trim_to;
+  op->trim_rollback_to = trim_rollback_to;
+  op->log_entries.swap(log_entries);
+  std::swap(op->updated_hit_set_history, hset_history);
+  op->on_local_applied_sync = on_local_applied_sync;
+  op->on_all_applied = on_all_applied;
+  op->on_all_commit = on_all_commit;
+  op->tid = tid;
+  op->reqid = reqid;
+  op->client_op = client_op;
+  
+  op->t = static_cast(_t);
+
+  set need_hinfos;
+  op->t->get_append_objects(&need_hinfos);
+  for (set::iterator i = need_hinfos.begin();
+       i != need_hinfos.end();
+       ++i) {
+    op->unstable_hash_infos.insert(
+      make_pair(
+	*i,
+	get_hash_info(*i)));
+  }
+
+  for (vector::iterator i = op->log_entries.begin();
+       i != op->log_entries.end();
+       ++i) {
+    MustPrependHashInfo vis;
+    i->mod_desc.visit(&vis);
+    if (vis.must_prepend_hash_info()) {
+      dout(10) << __func__ << ": stashing HashInfo for "
+	       << i->soid << " for entry " << *i << dendl;
+      assert(op->unstable_hash_infos.count(i->soid));
+      ObjectModDesc desc;
+      map > old_attrs;
+      bufferlist old_hinfo;
+      ::encode(*(op->unstable_hash_infos[i->soid]), old_hinfo);
+      old_attrs[ECUtil::get_hinfo_key()] = old_hinfo;
+      desc.setattrs(old_attrs);
+      i->mod_desc.swap(desc);
+      i->mod_desc.claim_append(desc);
+      assert(i->mod_desc.can_rollback());
+    }
+  }
+
+  dout(10) << __func__ << ": op " << *op << " starting" << dendl;
+  start_write(op);
+  writing.push_back(op);
+  dout(10) << "onreadable_sync: " << op->on_local_applied_sync << dendl;
+}
+
+int ECBackend::get_min_avail_to_read_shards(
+  const hobject_t &hoid,
+  const set &want,
+  bool for_recovery,
+  set *to_read)
+{
+  map >::const_iterator miter =
+    get_parent()->get_missing_loc_shards().find(hoid);
+
+  set have;
+  map shards;
+
+  for (set::const_iterator i =
+	 get_parent()->get_acting_shards().begin();
+       i != get_parent()->get_acting_shards().end();
+       ++i) {
+    dout(10) << __func__ << ": checking acting " << *i << dendl;
+    const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
+    if (!missing.is_missing(hoid)) {
+      assert(!have.count(i->shard));
+      have.insert(i->shard);
+      assert(!shards.count(i->shard));
+      shards.insert(make_pair(i->shard, *i));
+    }
+  }
+
+  if (for_recovery) {
+    for (set::const_iterator i =
+	   get_parent()->get_backfill_shards().begin();
+	 i != get_parent()->get_backfill_shards().end();
+	 ++i) {
+      if (have.count(i->shard)) {
+	assert(shards.count(i->shard));
+	continue;
+      }
+      dout(10) << __func__ << ": checking backfill " << *i << dendl;
+      assert(!shards.count(i->shard));
+      const pg_info_t &info = get_parent()->get_shard_info(*i);
+      const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
+      if (hoid < info.last_backfill && !missing.is_missing(hoid)) {
+	have.insert(i->shard);
+	shards.insert(make_pair(i->shard, *i));
+      }
+    }
+
+    if (miter != get_parent()->get_missing_loc_shards().end()) {
+      for (set::iterator i = miter->second.begin();
+	   i != miter->second.end();
+	   ++i) {
+	dout(10) << __func__ << ": checking missing_loc " << *i << dendl;
+	boost::optional m =
+	  get_parent()->maybe_get_shard_missing(*i);
+	if (m) {
+	  assert(!(*m).is_missing(hoid));
+	}
+	have.insert(i->shard);
+	shards.insert(make_pair(i->shard, *i));
+      }
+    }
+  }
+
+  set need;
+  int r = ec_impl->minimum_to_decode(want, have, &need);
+  if (r < 0)
+    return r;
+
+  if (!to_read)
+    return 0;
+
+  for (set::iterator i = need.begin();
+       i != need.end();
+       ++i) {
+    assert(shards.count(*i));
+    to_read->insert(shards[*i]);
+  }
+  return 0;
+}
+
+void ECBackend::start_read_op(
+  int priority,
+  map &to_read,
+  OpRequestRef _op)
+{
+  ceph_tid_t tid = get_parent()->get_tid();
+  assert(!tid_to_read_map.count(tid));
+  ReadOp &op(tid_to_read_map[tid]);
+  op.priority = priority;
+  op.tid = tid;
+  op.to_read.swap(to_read);
+  op.op = _op;
+  dout(10) << __func__ << ": starting " << op << dendl;
+
+  map messages;
+  for (map::iterator i = op.to_read.begin();
+       i != op.to_read.end();
+       ++i) {
+    list > > &reslist =
+      op.complete[i->first].returned;
+    bool need_attrs = i->second.want_attrs;
+    for (set::const_iterator j = i->second.need.begin();
+	 j != i->second.need.end();
+	 ++j) {
+      if (need_attrs) {
+	messages[*j].attrs_to_read.insert(i->first);
+	need_attrs = false;
+      }
+      op.obj_to_source[i->first].insert(*j);
+      op.source_to_obj[*j].insert(i->first);
+    }
+    for (list >::const_iterator j =
+	   i->second.to_read.begin();
+	 j != i->second.to_read.end();
+	 ++j) {
+      reslist.push_back(
+	boost::make_tuple(
+	  j->first,
+	  j->second,
+	  map()));
+      pair chunk_off_len =
+	sinfo.aligned_offset_len_to_chunk(
+	  *j);
+      for (set::const_iterator k = i->second.need.begin();
+	   k != i->second.need.end();
+	   ++k) {
+	messages[*k].to_read[i->first].push_back(chunk_off_len);
+      }
+      assert(!need_attrs);
+    }
+  }
+
+  for (map::iterator i = messages.begin();
+       i != messages.end();
+       ++i) {
+    op.in_progress.insert(i->first);
+    shard_to_read_map[i->first].insert(op.tid);
+    i->second.tid = tid;
+    MOSDECSubOpRead *msg = new MOSDECSubOpRead;
+    msg->set_priority(priority);
+    msg->pgid = spg_t(
+      get_parent()->whoami_spg_t().pgid,
+      i->first.shard);
+    msg->map_epoch = get_parent()->get_epoch();
+    msg->op = i->second;
+    msg->op.from = get_parent()->whoami_shard();
+    msg->op.tid = tid;
+    get_parent()->send_message_osd_cluster(
+      i->first.osd,
+      msg,
+      get_parent()->get_epoch());
+  }
+  dout(10) << __func__ << ": started " << op << dendl;
+}
+
+ECUtil::HashInfoRef ECBackend::get_hash_info(
+  const hobject_t &hoid)
+{
+  dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
+  ECUtil::HashInfoRef ref = unstable_hashinfo_registry.lookup(hoid);
+  if (!ref) {
+    dout(10) << __func__ << ": not in cache " << hoid << dendl;
+    struct stat st;
+    int r = store->stat(
+      hoid.is_temp() ? temp_coll : coll,
+      ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+      &st);
+    ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
+    if (r >= 0 && st.st_size > 0) {
+      dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl;
+      bufferlist bl;
+      r = store->getattr(
+	hoid.is_temp() ? temp_coll : coll,
+	ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+	ECUtil::get_hinfo_key(),
+	bl);
+      if (r >= 0) {
+	bufferlist::iterator bp = bl.begin();
+	::decode(hinfo, bp);
+	assert(hinfo.get_total_chunk_size() == (unsigned)st.st_size);
+      } else {
+	assert(0 == "missing hash attr");
+      }
+    }
+    ref = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
+  }
+  return ref;
+}
+
+void ECBackend::check_op(Op *op)
+{
+  if (op->pending_apply.empty() && op->on_all_applied) {
+    dout(10) << __func__ << " Calling on_all_applied on " << *op << dendl;
+    op->on_all_applied->complete(0);
+    op->on_all_applied = 0;
+  }
+  if (op->pending_commit.empty() && op->on_all_commit) {
+    dout(10) << __func__ << " Calling on_all_commit on " << *op << dendl;
+    op->on_all_commit->complete(0);
+    op->on_all_commit = 0;
+  }
+  if (op->pending_apply.empty() && op->pending_commit.empty()) {
+    // done!
+    assert(writing.front() == op);
+    dout(10) << __func__ << " Completing " << *op << dendl;
+    writing.pop_front();
+    tid_to_op_map.erase(op->tid);
+  }
+  for (map::iterator i = tid_to_op_map.begin();
+       i != tid_to_op_map.end();
+       ++i) {
+    dout(20) << __func__ << " tid " << i->first <<": " << i->second << dendl;
+  }
+}
+
+void ECBackend::start_write(Op *op) {
+  map trans;
+  for (set::const_iterator i =
+	 get_parent()->get_actingbackfill_shards().begin();
+       i != get_parent()->get_actingbackfill_shards().end();
+       ++i) {
+    trans[i->shard];
+  }
+  op->t->generate_transactions(
+    op->unstable_hash_infos,
+    ec_impl,
+    get_parent()->get_info().pgid.pgid,
+    sinfo,
+    &trans,
+    &(op->temp_added),
+    &(op->temp_cleared));
+
+  dout(10) << "onreadable_sync: " << op->on_local_applied_sync << dendl;
+
+  for (set::const_iterator i =
+	 get_parent()->get_actingbackfill_shards().begin();
+       i != get_parent()->get_actingbackfill_shards().end();
+       ++i) {
+    op->pending_apply.insert(*i);
+    op->pending_commit.insert(*i);
+    map::iterator iter =
+      trans.find(i->shard);
+    assert(iter != trans.end());
+    bool should_send = get_parent()->should_send_op(*i, op->hoid);
+    pg_stat_t stats =
+      should_send ?
+      get_info().stats :
+      parent->get_shard_info().find(*i)->second.stats;
+
+    ECSubWrite sop(
+      get_parent()->whoami_shard(),
+      op->tid,
+      op->reqid,
+      op->hoid,
+      stats,
+      should_send ? iter->second : ObjectStore::Transaction(),
+      op->version,
+      op->trim_to,
+      op->trim_rollback_to,
+      op->log_entries,
+      op->updated_hit_set_history,
+      op->temp_added,
+      op->temp_cleared);
+    if (*i == get_parent()->whoami_shard()) {
+      handle_sub_write(
+	get_parent()->whoami_shard(),
+	op->client_op,
+	sop,
+	op->on_local_applied_sync);
+      op->on_local_applied_sync = 0;
+    } else {
+      MOSDECSubOpWrite *r = new MOSDECSubOpWrite(sop);
+      r->set_priority(cct->_conf->osd_client_op_priority);
+      r->pgid = spg_t(get_parent()->primary_spg_t().pgid, i->shard);
+      r->map_epoch = get_parent()->get_epoch();
+      get_parent()->send_message_osd_cluster(
+	i->osd, r, get_parent()->get_epoch());
+    }
+  }
+}
+
+int ECBackend::objects_read_sync(
+  const hobject_t &hoid,
+  uint64_t off,
+  uint64_t len,
+  bufferlist *bl)
+{
+  return -EOPNOTSUPP;
+}
+
+struct CallClientContexts :
+  public GenContext &> {
+  ECBackend *ec;
+  ECBackend::ClientAsyncReadStatus *status;
+  list,
+	    pair > > to_read;
+  CallClientContexts(
+    ECBackend *ec,
+    ECBackend::ClientAsyncReadStatus *status,
+    const list,
+		    pair > > &to_read)
+    : ec(ec), status(status), to_read(to_read) {}
+  void finish(pair &in) {
+    ECBackend::read_result_t &res = in.second;
+    assert(res.returned.size() == to_read.size());
+    assert(res.r == 0);
+    assert(res.errors.empty());
+    for (list,
+		   pair > >::iterator i = to_read.begin();
+	 i != to_read.end();
+	 to_read.erase(i++)) {
+      pair adjusted =
+	ec->sinfo.offset_len_to_stripe_bounds(i->first);
+      assert(res.returned.front().get<0>() == adjusted.first &&
+	     res.returned.front().get<1>() == adjusted.second);
+      map to_decode;
+      bufferlist bl;
+      for (map::iterator j =
+	     res.returned.front().get<2>().begin();
+	   j != res.returned.front().get<2>().end();
+	   ++j) {
+	to_decode[j->first.shard].claim(j->second);
+      }
+      ECUtil::decode(
+	ec->sinfo,
+	ec->ec_impl,
+	to_decode,
+	&bl);
+      assert(i->second.second);
+      assert(i->second.first);
+      i->second.first->substr_of(
+	bl,
+	i->first.first - adjusted.first,
+	MIN(i->first.second, bl.length() - (i->first.first - adjusted.first)));
+      if (i->second.second) {
+	i->second.second->complete(i->second.first->length());
+      }
+      res.returned.pop_front();
+    }
+    status->complete = true;
+    list &ip =
+      ec->in_progress_client_reads;
+    while (ip.size() && ip.front().complete) {
+      if (ip.front().on_complete) {
+	ip.front().on_complete->complete(0);
+	ip.front().on_complete = NULL;
+      }
+      ip.pop_front();
+    }
+  }
+  ~CallClientContexts() {
+    for (list,
+		   pair > >::iterator i = to_read.begin();
+	 i != to_read.end();
+	 to_read.erase(i++)) {
+      delete i->second.second;
+    }
+  }
+};
+
+void ECBackend::objects_read_async(
+  const hobject_t &hoid,
+  const list,
+		  pair > > &to_read,
+  Context *on_complete)
+{
+  in_progress_client_reads.push_back(ClientAsyncReadStatus(on_complete));
+  CallClientContexts *c = new CallClientContexts(
+    this, &(in_progress_client_reads.back()), to_read);
+  list > offsets;
+  for (list,
+		 pair > >::const_iterator i =
+	 to_read.begin();
+       i != to_read.end();
+       ++i) {
+    offsets.push_back(
+      sinfo.offset_len_to_stripe_bounds(i->first));
+  }
+
+  set want_to_read;
+  for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) {
+    want_to_read.insert(i);
+  }
+  set shards;
+  int r = get_min_avail_to_read_shards(
+    hoid,
+    want_to_read,
+    false,
+    &shards);
+  assert(r == 0);
+
+  map for_read_op;
+  for_read_op.insert(
+    make_pair(
+      hoid,
+      read_request_t(
+	hoid,
+	offsets,
+	shards,
+	false,
+	c)));
+
+  start_read_op(
+    cct->_conf->osd_client_op_priority,
+    for_read_op,
+    OpRequestRef());
+  return;
+}
+
+
+int ECBackend::objects_get_attrs(
+  const hobject_t &hoid,
+  map *out)
+{
+  int r = store->getattrs(
+    coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+    *out);
+  if (r < 0)
+    return r;
+
+  for (map::iterator i = out->begin();
+       i != out->end();
+       ) {
+    if (ECUtil::is_hinfo_key_string(i->first))
+      out->erase(i++);
+    else
+      ++i;
+  }
+  return r;
+}
+
+void ECBackend::rollback_append(
+  const hobject_t &hoid,
+  uint64_t old_size,
+  ObjectStore::Transaction *t)
+{
+  assert(old_size % sinfo.get_stripe_width() == 0);
+  t->truncate(
+    coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+    sinfo.aligned_logical_offset_to_chunk_offset(
+      old_size));
+}
+
+void ECBackend::be_deep_scrub(
+  const hobject_t &poid,
+  ScrubMap::object &o,
+  ThreadPool::TPHandle &handle) {
+  bufferhash h(-1);
+  int r;
+  uint64_t stride = cct->_conf->osd_deep_scrub_stride;
+  if (stride % sinfo.get_chunk_size())
+    stride += sinfo.get_chunk_size() - (stride % sinfo.get_chunk_size());
+  uint64_t pos = 0;
+  while (true) {
+    bufferlist bl;
+    handle.reset_tp_timeout();
+    r = store->read(
+      coll,
+      ghobject_t(
+	poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+      pos,
+      stride, bl,
+      true);
+    if (r < 0)
+      break;
+    if (bl.length() % sinfo.get_chunk_size()) {
+      r = -EIO;
+      break;
+    }
+    pos += r;
+    h << bl;
+    if ((unsigned)r < stride)
+      break;
+  }
+
+  ECUtil::HashInfoRef hinfo = get_hash_info(poid);
+  if (r == -EIO) {
+    dout(0) << "_scan_list  " << poid << " got "
+	    << r << " on read, read_error" << dendl;
+    o.read_error = true;
+  }
+
+  if (hinfo->get_chunk_hash(get_parent()->whoami_shard().shard) != h.digest()) {
+    dout(0) << "_scan_list  " << poid << " got incorrect hash on read" << dendl;
+    o.read_error = true;
+  }
+
+  if (hinfo->get_total_chunk_size() != pos) {
+    dout(0) << "_scan_list  " << poid << " got incorrect size on read" << dendl;
+    o.read_error = true;
+  }
+
+  /* We checked above that we match our own stored hash.  We cannot
+   * send a hash of the actual object, so instead we simply send
+   * our locally stored hash of shard 0 on the assumption that if
+   * we match our chunk hash and our recollection of the hash for
+   * chunk 0 matches that of our peers, there is likely no corruption.
+   */
+  o.digest = hinfo->get_chunk_hash(0);
+  o.digest_present = true;
+
+  o.omap_digest = 0;
+  o.omap_digest_present = true;
+}
diff --git a/ceph/src/osd/ECBackend.h b/ceph/src/osd/ECBackend.h
new file mode 100644
index 00000000..28bcf8a5
--- /dev/null
+++ b/ceph/src/osd/ECBackend.h
@@ -0,0 +1,480 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef ECBACKEND_H
+#define ECBACKEND_H
+
+#include "OSD.h"
+#include "PGBackend.h"
+#include "osd_types.h"
+#include 
+#include "erasure-code/ErasureCodeInterface.h"
+#include "ECTransaction.h"
+#include "ECMsgTypes.h"
+#include "ECUtil.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
+
+struct RecoveryMessages;
+class ECBackend : public PGBackend {
+public:
+  RecoveryHandle *open_recovery_op();
+
+  void run_recovery_op(
+    RecoveryHandle *h,
+    int priority
+    );
+
+  void recover_object(
+    const hobject_t &hoid,
+    eversion_t v,
+    ObjectContextRef head,
+    ObjectContextRef obc,
+    RecoveryHandle *h
+    );
+
+  bool handle_message(
+    OpRequestRef op
+    );
+  bool can_handle_while_inactive(
+    OpRequestRef op
+    );
+  friend struct SubWriteApplied;
+  friend struct SubWriteCommitted;
+  void sub_write_applied(
+    ceph_tid_t tid, eversion_t version);
+  void sub_write_committed(
+    ceph_tid_t tid, eversion_t version, eversion_t last_complete);
+  void handle_sub_write(
+    pg_shard_t from,
+    OpRequestRef msg,
+    ECSubWrite &op,
+    Context *on_local_applied_sync = 0
+    );
+  void handle_sub_read(
+    pg_shard_t from,
+    ECSubRead &op,
+    ECSubReadReply *reply
+    );
+  void handle_sub_write_reply(
+    pg_shard_t from,
+    ECSubWriteReply &op
+    );
+  void handle_sub_read_reply(
+    pg_shard_t from,
+    ECSubReadReply &op,
+    RecoveryMessages *m
+    );
+
+  /// @see ReadOp below
+  void check_recovery_sources(const OSDMapRef osdmap);
+
+  void on_change();
+  void clear_state();
+
+  void on_flushed();
+
+  void dump_recovery_info(Formatter *f) const;
+
+  /// @see osd/ECTransaction.cc/h
+  PGTransaction *get_transaction();
+
+  void submit_transaction(
+    const hobject_t &hoid,
+    const eversion_t &at_version,
+    PGTransaction *t,
+    const eversion_t &trim_to,
+    const eversion_t &trim_rollback_to,
+    vector &log_entries,
+    boost::optional &hset_history,
+    Context *on_local_applied_sync,
+    Context *on_all_applied,
+    Context *on_all_commit,
+    ceph_tid_t tid,
+    osd_reqid_t reqid,
+    OpRequestRef op
+    );
+
+  int objects_read_sync(
+    const hobject_t &hoid,
+    uint64_t off,
+    uint64_t len,
+    bufferlist *bl);
+
+  /**
+   * Async read mechanism
+   *
+   * Async reads use the same async read mechanism as does recovery.
+   * CallClientContexts is responsible for reconstructing the response
+   * buffer as well as for calling the callbacks.
+   *
+   * One tricky bit is that two reads may possibly not read from the same
+   * set of replicas.  This could result in two reads completing in the
+   * wrong (from the interface user's point of view) order.  Thus, we
+   * maintain a queue of in progress reads (@see in_progress_client_reads)
+   * to ensure that we always call the completion callback in order.
+   *
+   * Another subtely is that while we may read a degraded object, we will
+   * still only perform a client read from shards in the acting set.  This
+   * ensures that we won't ever have to restart a client initiated read in
+   * check_recovery_sources.
+   */
+  friend struct CallClientContexts;
+  struct ClientAsyncReadStatus {
+    bool complete;
+    Context *on_complete;
+    ClientAsyncReadStatus(Context *on_complete)
+    : complete(false), on_complete(on_complete) {}
+  };
+  list in_progress_client_reads;
+  void objects_read_async(
+    const hobject_t &hoid,
+    const list,
+		    pair > > &to_read,
+    Context *on_complete);
+
+private:
+  friend struct ECRecoveryHandle;
+  uint64_t get_recovery_chunk_size() const {
+    uint64_t max = cct->_conf->osd_recovery_max_chunk;
+    max -= max % sinfo.get_stripe_width();
+    max += sinfo.get_stripe_width();
+    return max;
+  }
+
+  /**
+   * Recovery
+   *
+   * Recovery uses the same underlying read mechanism as client reads
+   * with the slight difference that recovery reads may come from non
+   * acting shards.  Thus, check_recovery_sources may wind up calling
+   * cancel_pull for a read originating with RecoveryOp.
+   *
+   * The recovery process is expressed as a state machine:
+   * - IDLE: Nothing is currently in progress, reads will be started and
+   *         we will transition to READING
+   * - READING: We are awaiting a pending read op.  Once complete, we will
+   *            decode the buffers and proceed to WRITING
+   * - WRITING: We are awaiting a completed push.  Once complete, we will
+   *            either transition to COMPLETE or to IDLE to continue.
+   * - COMPLETE: complete
+   *
+   * We use the existing Push and PushReply messages and structures to
+   * handle actually shuffling the data over to the replicas.  recovery_info
+   * and recovery_progress are expressed in terms of the logical offset
+   * space except for data_included which is in terms of the chunked object
+   * space (to match the passed buffer).
+   *
+   * xattrs are requested on the first read and used to initialize the
+   * object_context if missing on completion of the first read.
+   *
+   * In order to batch up reads and writes, we batch Push, PushReply,
+   * Transaction, and reads in a RecoveryMessages object which is passed
+   * among the recovery methods.
+   */
+  struct RecoveryOp {
+    hobject_t hoid;
+    eversion_t v;
+    set missing_on;
+    set missing_on_shards;
+
+    ObjectRecoveryInfo recovery_info;
+    ObjectRecoveryProgress recovery_progress;
+
+    bool pending_read;
+    enum state_t { IDLE, READING, WRITING, COMPLETE } state;
+
+    static const char* tostr(state_t state) {
+      switch (state) {
+      case ECBackend::RecoveryOp::IDLE:
+	return "IDLE";
+	break;
+      case ECBackend::RecoveryOp::READING:
+	return "READING";
+	break;
+      case ECBackend::RecoveryOp::WRITING:
+	return "WRITING";
+	break;
+      case ECBackend::RecoveryOp::COMPLETE:
+	return "COMPLETE";
+	break;
+      default:
+	assert(0);
+	return "";
+      }
+    }
+
+    // must be filled if state == WRITING
+    map returned_data;
+    map xattrs;
+    ECUtil::HashInfoRef hinfo;
+    ObjectContextRef obc;
+    set waiting_on_pushes;
+
+    // valid in state READING
+    pair extent_requested;
+
+    void dump(Formatter *f) const;
+
+    RecoveryOp() : pending_read(false), state(IDLE) {}
+  };
+  friend ostream &operator<<(ostream &lhs, const RecoveryOp &rhs);
+  map recovery_ops;
+
+public:
+  /**
+   * Low level async read mechanism
+   *
+   * To avoid duplicating the logic for requesting and waiting for
+   * multiple object shards, there is a common async read mechanism
+   * taking a map of hobject_t->read_request_t which defines callbacks
+   * taking read_result_ts as arguments.
+   *
+   * tid_to_read_map gives open read ops.  check_recovery_sources uses
+   * shard_to_read_map and ReadOp::source_to_obj to restart reads
+   * involving down osds.
+   *
+   * The user is responsible for specifying replicas on which to read
+   * and for reassembling the buffer on the other side since client
+   * reads require the original object buffer while recovery only needs
+   * the missing pieces.
+   *
+   * Rather than handling reads on the primary directly, we simply send
+   * ourselves a message.  This avoids a dedicated primary path for that
+   * part.
+   */
+  struct read_result_t {
+    int r;
+    map errors;
+    boost::optional > attrs;
+    list<
+      boost::tuple<
+	uint64_t, uint64_t, map > > returned;
+    read_result_t() : r(0) {}
+  };
+  struct read_request_t {
+    const list > to_read;
+    const set need;
+    const bool want_attrs;
+    GenContext &> *cb;
+    read_request_t(
+      const hobject_t &hoid,
+      const list > &to_read,
+      const set &need,
+      bool want_attrs,
+      GenContext &> *cb)
+      : to_read(to_read), need(need), want_attrs(want_attrs),
+	cb(cb) {}
+  };
+  friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
+
+  struct ReadOp {
+    int priority;
+    ceph_tid_t tid;
+    OpRequestRef op; // may be null if not on behalf of a client
+
+    map to_read;
+    map complete;
+
+    map > obj_to_source;
+    map > source_to_obj;
+
+    void dump(Formatter *f) const;
+
+    set in_progress;
+  };
+  friend struct FinishReadOp;
+  void filter_read_op(
+    const OSDMapRef osdmap,
+    ReadOp &op);
+  void complete_read_op(ReadOp &rop, RecoveryMessages *m);
+  friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
+  map tid_to_read_map;
+  map > shard_to_read_map;
+  void start_read_op(
+    int priority,
+    map &to_read,
+    OpRequestRef op);
+
+
+  /**
+   * Client writes
+   *
+   * ECTransaction is responsible for generating a transaction for
+   * each shard to which we need to send the write.  As required
+   * by the PGBackend interface, the ECBackend write mechanism
+   * passes trim information with the write and last_complete back
+   * with the reply.
+   *
+   * As with client reads, there is a possibility of out-of-order
+   * completions. Thus, callbacks and completion are called in order
+   * on the writing list.
+   */
+  struct Op {
+    hobject_t hoid;
+    eversion_t version;
+    eversion_t trim_to;
+    eversion_t trim_rollback_to;
+    vector log_entries;
+    boost::optional updated_hit_set_history;
+    Context *on_local_applied_sync;
+    Context *on_all_applied;
+    Context *on_all_commit;
+    ceph_tid_t tid;
+    osd_reqid_t reqid;
+    OpRequestRef client_op;
+
+    ECTransaction *t;
+
+    set temp_added;
+    set temp_cleared;
+
+    set pending_commit;
+    set pending_apply;
+
+    map unstable_hash_infos;
+    ~Op() {
+      delete t;
+      delete on_local_applied_sync;
+      delete on_all_applied;
+      delete on_all_commit;
+    }
+  };
+  friend ostream &operator<<(ostream &lhs, const Op &rhs);
+
+  void continue_recovery_op(
+    RecoveryOp &op,
+    RecoveryMessages *m);
+
+  void dispatch_recovery_messages(RecoveryMessages &m, int priority);
+  friend struct OnRecoveryReadComplete;
+  void handle_recovery_read_complete(
+    const hobject_t &hoid,
+    boost::tuple > &to_read,
+    boost::optional > attrs,
+    RecoveryMessages *m);
+  void handle_recovery_push(
+    PushOp &op,
+    RecoveryMessages *m);
+  void handle_recovery_push_reply(
+    PushReplyOp &op,
+    pg_shard_t from,
+    RecoveryMessages *m);
+
+  map tid_to_op_map; /// lists below point into here
+  list writing;
+
+  CephContext *cct;
+  ErasureCodeInterfaceRef ec_impl;
+
+
+  /**
+   * ECRecPred
+   *
+   * Determines the whether _have is suffient to recover an object
+   */
+  class ECRecPred : public IsRecoverablePredicate {
+    set want;
+    ErasureCodeInterfaceRef ec_impl;
+  public:
+    ECRecPred(ErasureCodeInterfaceRef ec_impl) : ec_impl(ec_impl) {
+      for (unsigned i = 0; i < ec_impl->get_data_chunk_count(); ++i) {
+	want.insert(i);
+      }
+    }
+    bool operator()(const set &_have) const {
+      set have;
+      for (set::const_iterator i = _have.begin();
+	   i != _have.end();
+	   ++i) {
+	have.insert(i->shard);
+      }
+      set min;
+      return ec_impl->minimum_to_decode(want, have, &min) == 0;
+    }
+  };
+  IsRecoverablePredicate *get_is_recoverable_predicate() {
+    return new ECRecPred(ec_impl);
+  }
+
+  /**
+   * ECReadPred
+   *
+   * Determines the whether _have is suffient to read an object
+   */
+  class ECReadPred : public IsReadablePredicate {
+    pg_shard_t whoami;
+    ECRecPred rec_pred;
+  public:
+    ECReadPred(
+      pg_shard_t whoami,
+      ErasureCodeInterfaceRef ec_impl) : whoami(whoami), rec_pred(ec_impl) {}
+    bool operator()(const set &_have) const {
+      return _have.count(whoami) && rec_pred(_have);
+    }
+  };
+  IsReadablePredicate *get_is_readable_predicate() {
+    return new ECReadPred(get_parent()->whoami_shard(), ec_impl);
+  }
+
+
+  const ECUtil::stripe_info_t sinfo;
+  /// If modified, ensure that the ref is held until the update is applied
+  SharedPtrRegistry unstable_hashinfo_registry;
+  ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid);
+
+  friend struct ReadCB;
+  void check_op(Op *op);
+  void start_write(Op *op);
+public:
+  ECBackend(
+    PGBackend::Listener *pg,
+    coll_t coll,
+    coll_t temp_coll,
+    ObjectStore *store,
+    CephContext *cct,
+    ErasureCodeInterfaceRef ec_impl,
+    uint64_t stripe_width);
+
+  /// Returns to_read replicas sufficient to reconstruct want
+  int get_min_avail_to_read_shards(
+    const hobject_t &hoid,     ///< [in] object
+    const set &want,      ///< [in] desired shards
+    bool for_recovery,         ///< [in] true if we may use non-acting replicas
+    set *to_read   ///< [out] shards to read
+    ); ///< @return error code, 0 on success
+
+  int objects_get_attrs(
+    const hobject_t &hoid,
+    map *out);
+
+  void rollback_append(
+    const hobject_t &hoid,
+    uint64_t old_size,
+    ObjectStore::Transaction *t);
+
+  bool scrub_supported() { return true; }
+
+  void be_deep_scrub(
+    const hobject_t &obj,
+    ScrubMap::object &o,
+    ThreadPool::TPHandle &handle);
+  uint64_t be_get_ondisk_size(uint64_t logical_size) {
+    return sinfo.logical_to_next_chunk_offset(logical_size);
+  }
+};
+
+#endif
diff --git a/ceph/src/osd/ECMsgTypes.cc b/ceph/src/osd/ECMsgTypes.cc
new file mode 100644
index 00000000..ba02d835
--- /dev/null
+++ b/ceph/src/osd/ECMsgTypes.cc
@@ -0,0 +1,355 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "ECMsgTypes.h"
+
+void ECSubWrite::encode(bufferlist &bl) const
+{
+  ENCODE_START(3, 1, bl);
+  ::encode(from, bl);
+  ::encode(tid, bl);
+  ::encode(reqid, bl);
+  ::encode(soid, bl);
+  ::encode(stats, bl);
+  ::encode(t, bl);
+  ::encode(at_version, bl);
+  ::encode(trim_to, bl);
+  ::encode(log_entries, bl);
+  ::encode(temp_added, bl);
+  ::encode(temp_removed, bl);
+  ::encode(updated_hit_set_history, bl);
+  ::encode(trim_rollback_to, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ECSubWrite::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(3, bl);
+  ::decode(from, bl);
+  ::decode(tid, bl);
+  ::decode(reqid, bl);
+  ::decode(soid, bl);
+  ::decode(stats, bl);
+  ::decode(t, bl);
+  ::decode(at_version, bl);
+  ::decode(trim_to, bl);
+  ::decode(log_entries, bl);
+  ::decode(temp_added, bl);
+  ::decode(temp_removed, bl);
+  if (struct_v >= 2) {
+    ::decode(updated_hit_set_history, bl);
+  }
+  if (struct_v >= 3) {
+    ::decode(trim_rollback_to, bl);
+  } else {
+    trim_rollback_to = trim_to;
+  }
+  DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubWrite &rhs)
+{
+  lhs << "ECSubWrite(tid=" << rhs.tid
+      << ", reqid=" << rhs.reqid
+      << ", at_version=" << rhs.at_version
+      << ", trim_to=" << rhs.trim_to
+      << ", trim_rollback_to=" << rhs.trim_rollback_to;
+  if (rhs.updated_hit_set_history)
+    lhs << ", has_updated_hit_set_history";
+  return lhs <<  ")";
+}
+
+void ECSubWrite::dump(Formatter *f) const
+{
+  f->dump_unsigned("tid", tid);
+  f->dump_stream("reqid") << reqid;
+  f->dump_stream("at_version") << at_version;
+  f->dump_stream("trim_to") << trim_to;
+  f->dump_stream("trim_rollback_to") << trim_rollback_to;
+  f->dump_stream("has_updated_hit_set_history")
+    << static_cast(updated_hit_set_history);
+}
+
+void ECSubWrite::generate_test_instances(list &o)
+{
+  o.push_back(new ECSubWrite());
+  o.back()->tid = 1;
+  o.back()->at_version = eversion_t(2, 100);
+  o.back()->trim_to = eversion_t(1, 40);
+  o.push_back(new ECSubWrite());
+  o.back()->tid = 4;
+  o.back()->reqid = osd_reqid_t(entity_name_t::CLIENT(123), 1, 45678);
+  o.back()->at_version = eversion_t(10, 300);
+  o.back()->trim_to = eversion_t(5, 42);
+  o.push_back(new ECSubWrite());
+  o.back()->tid = 9;
+  o.back()->reqid = osd_reqid_t(entity_name_t::CLIENT(123), 1, 45678);
+  o.back()->at_version = eversion_t(10, 300);
+  o.back()->trim_to = eversion_t(5, 42);
+  o.back()->trim_rollback_to = eversion_t(8, 250);
+}
+
+void ECSubWriteReply::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(from, bl);
+  ::encode(tid, bl);
+  ::encode(last_complete, bl);
+  ::encode(committed, bl);
+  ::encode(applied, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ECSubWriteReply::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(from, bl);
+  ::decode(tid, bl);
+  ::decode(last_complete, bl);
+  ::decode(committed, bl);
+  ::decode(applied, bl);
+  DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubWriteReply &rhs)
+{
+  return lhs
+    << "ECSubWriteReply(tid=" << rhs.tid
+    << ", last_complete=" << rhs.last_complete
+    << ", committed=" << rhs.committed
+    << ", applied=" << rhs.applied << ")";
+}
+
+void ECSubWriteReply::dump(Formatter *f) const
+{
+  f->dump_unsigned("tid", tid);
+  f->dump_stream("last_complete") << last_complete;
+  f->dump_stream("committed") << committed;
+  f->dump_stream("applied") << applied;
+}
+
+void ECSubWriteReply::generate_test_instances(list& o)
+{
+  o.push_back(new ECSubWriteReply());
+  o.back()->tid = 20;
+  o.back()->last_complete = eversion_t(100, 2000);
+  o.back()->committed = true;
+  o.push_back(new ECSubWriteReply());
+  o.back()->tid = 80;
+  o.back()->last_complete = eversion_t(50, 200);
+  o.back()->applied = true;
+}
+
+void ECSubRead::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(from, bl);
+  ::encode(tid, bl);
+  ::encode(to_read, bl);
+  ::encode(attrs_to_read, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ECSubRead::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(from, bl);
+  ::decode(tid, bl);
+  ::decode(to_read, bl);
+  ::decode(attrs_to_read, bl);
+  DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubRead &rhs)
+{
+  return lhs
+    << "ECSubRead(tid=" << rhs.tid
+    << ", to_read=" << rhs.to_read
+    << ", attrs_to_read=" << rhs.attrs_to_read << ")";
+}
+
+void ECSubRead::dump(Formatter *f) const
+{
+  f->dump_stream("from") << from;
+  f->dump_unsigned("tid", tid);
+  f->open_array_section("objects");
+  for (map > >::const_iterator i =
+	 to_read.begin();
+       i != to_read.end();
+       ++i) {
+    f->open_object_section("object");
+    f->dump_stream("oid") << i->first;
+    f->open_array_section("extents");
+    for (list >::const_iterator j =
+	   i->second.begin();
+	 j != i->second.end();
+	 ++j) {
+      f->open_object_section("extent");
+      f->dump_unsigned("off", j->first);
+      f->dump_unsigned("len", j->second);
+      f->close_section();
+    }
+    f->close_section();
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("object_attrs_requested");
+  for (set::const_iterator i = attrs_to_read.begin();
+       i != attrs_to_read.end();
+       ++i) {
+    f->open_object_section("object");
+    f->dump_stream("oid") << *i;
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ECSubRead::generate_test_instances(list& o)
+{
+  hobject_t hoid1(sobject_t("asdf", 1));
+  hobject_t hoid2(sobject_t("asdf2", CEPH_NOSNAP));
+  o.push_back(new ECSubRead());
+  o.back()->from = pg_shard_t(2, 255);
+  o.back()->tid = 1;
+  o.back()->to_read[hoid1].push_back(make_pair(100, 200));
+  o.back()->to_read[hoid1].push_back(make_pair(400, 600));
+  o.back()->to_read[hoid2].push_back(make_pair(400, 600));
+  o.back()->attrs_to_read.insert(hoid1);
+  o.push_back(new ECSubRead());
+  o.back()->from = pg_shard_t(2, 255);
+  o.back()->tid = 300;
+  o.back()->to_read[hoid1].push_back(make_pair(300, 200));
+  o.back()->to_read[hoid2].push_back(make_pair(400, 600));
+  o.back()->to_read[hoid2].push_back(make_pair(2000, 600));
+  o.back()->attrs_to_read.insert(hoid2);
+}
+
+void ECSubReadReply::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(from, bl);
+  ::encode(tid, bl);
+  ::encode(buffers_read, bl);
+  ::encode(attrs_read, bl);
+  ::encode(errors, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ECSubReadReply::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(from, bl);
+  ::decode(tid, bl);
+  ::decode(buffers_read, bl);
+  ::decode(attrs_read, bl);
+  ::decode(errors, bl);
+  DECODE_FINISH(bl);
+}
+
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubReadReply &rhs)
+{
+  return lhs
+    << "ECSubReadReply(tid=" << rhs.tid
+    << ", attrs_read=" << rhs.attrs_read.size()
+    << ")";
+}
+
+void ECSubReadReply::dump(Formatter *f) const
+{
+  f->dump_stream("from") << from;
+  f->dump_unsigned("tid", tid);
+  f->open_array_section("buffers_read");
+  for (map > >::const_iterator i =
+	 buffers_read.begin();
+       i != buffers_read.end();
+       ++i) {
+    f->open_object_section("object");
+    f->dump_stream("oid") << i->first;
+    f->open_array_section("data");
+    for (list >::const_iterator j =
+	   i->second.begin();
+	 j != i->second.end();
+	 ++j) {
+      f->open_object_section("extent");
+      f->dump_unsigned("off", j->first);
+      f->dump_unsigned("buf_len", j->second.length());
+      f->close_section();
+    }
+    f->close_section();
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("attrs_returned");
+  for (map >::const_iterator i =
+	 attrs_read.begin();
+       i != attrs_read.end();
+       ++i) {
+    f->open_object_section("object_attrs");
+    f->dump_stream("oid") << i->first;
+    f->open_array_section("attrs");
+    for (map::const_iterator j = i->second.begin();
+	 j != i->second.end();
+	 ++j) {
+      f->open_object_section("attr");
+      f->dump_string("attr", j->first);
+      f->dump_unsigned("val_len", j->second.length());
+      f->close_section();
+    }
+    f->close_section();
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("errors");
+  for (map::const_iterator i = errors.begin();
+       i != errors.end();
+       ++i) {
+    f->open_object_section("error_pair");
+    f->dump_stream("oid") << i->first;
+    f->dump_int("error", i->second);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ECSubReadReply::generate_test_instances(list& o)
+{
+  hobject_t hoid1(sobject_t("asdf", 1));
+  hobject_t hoid2(sobject_t("asdf2", CEPH_NOSNAP));
+  bufferlist bl;
+  bl.append_zero(100);
+  bufferlist bl2;
+  bl2.append_zero(200);
+  o.push_back(new ECSubReadReply());
+  o.back()->from = pg_shard_t(2, 255);
+  o.back()->tid = 1;
+  o.back()->buffers_read[hoid1].push_back(make_pair(20, bl));
+  o.back()->buffers_read[hoid1].push_back(make_pair(2000, bl2));
+  o.back()->buffers_read[hoid2].push_back(make_pair(0, bl));
+  o.back()->attrs_read[hoid1]["foo"] = bl;
+  o.back()->attrs_read[hoid1]["_"] = bl2;
+  o.push_back(new ECSubReadReply());
+  o.back()->from = pg_shard_t(2, 255);
+  o.back()->tid = 300;
+  o.back()->buffers_read[hoid2].push_back(make_pair(0, bl2));
+  o.back()->attrs_read[hoid2]["foo"] = bl;
+  o.back()->attrs_read[hoid2]["_"] = bl2;
+  o.back()->errors[hoid1] = -2;
+}
diff --git a/ceph/src/osd/ECMsgTypes.h b/ceph/src/osd/ECMsgTypes.h
new file mode 100644
index 00000000..1cdfa57e
--- /dev/null
+++ b/ceph/src/osd/ECMsgTypes.h
@@ -0,0 +1,114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef ECBMSGTYPES_H
+#define ECBMSGTYPES_H
+
+#include "osd_types.h"
+#include "include/buffer.h"
+#include "os/ObjectStore.h"
+
+struct ECSubWrite {
+  pg_shard_t from;
+  ceph_tid_t tid;
+  osd_reqid_t reqid;
+  hobject_t soid;
+  pg_stat_t stats;
+  ObjectStore::Transaction t;
+  eversion_t at_version;
+  eversion_t trim_to;
+  eversion_t trim_rollback_to;
+  vector log_entries;
+  set temp_added;
+  set temp_removed;
+  boost::optional updated_hit_set_history;
+  ECSubWrite() {}
+  ECSubWrite(
+    pg_shard_t from,
+    ceph_tid_t tid,
+    osd_reqid_t reqid,
+    hobject_t soid,
+    const pg_stat_t &stats,
+    const ObjectStore::Transaction &t,
+    eversion_t at_version,
+    eversion_t trim_to,
+    eversion_t trim_rollback_to,
+    vector log_entries,
+    boost::optional updated_hit_set_history,
+    const set &temp_added,
+    const set &temp_removed)
+    : from(from), tid(tid), reqid(reqid),
+      soid(soid), stats(stats), t(t),
+      at_version(at_version),
+      trim_to(trim_to), trim_rollback_to(trim_rollback_to),
+      log_entries(log_entries),
+      temp_added(temp_added),
+      temp_removed(temp_removed),
+      updated_hit_set_history(updated_hit_set_history) {}
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ECSubWrite)
+
+struct ECSubWriteReply {
+  pg_shard_t from;
+  ceph_tid_t tid;
+  eversion_t last_complete;
+  bool committed;
+  bool applied;
+  ECSubWriteReply() : committed(false), applied(false) {}
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ECSubWriteReply)
+
+struct ECSubRead {
+  pg_shard_t from;
+  ceph_tid_t tid;
+  map > > to_read;
+  set attrs_to_read;
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ECSubRead)
+
+struct ECSubReadReply {
+  pg_shard_t from;
+  ceph_tid_t tid;
+  map > > buffers_read;
+  map > attrs_read;
+  map errors;
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ECSubReadReply)
+
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubWrite &rhs);
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubWriteReply &rhs);
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubRead &rhs);
+std::ostream &operator<<(
+  std::ostream &lhs, const ECSubReadReply &rhs);
+
+#endif
diff --git a/ceph/src/osd/ECTransaction.cc b/ceph/src/osd/ECTransaction.cc
new file mode 100644
index 00000000..c25023c4
--- /dev/null
+++ b/ceph/src/osd/ECTransaction.cc
@@ -0,0 +1,283 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "ECBackend.h"
+#include "ECUtil.h"
+#include "os/ObjectStore.h"
+
+struct AppendObjectsGenerator: public boost::static_visitor {
+  typedef void result_type;
+  set *out;
+  AppendObjectsGenerator(set *out) : out(out) {}
+  void operator()(const ECTransaction::AppendOp &op) {
+    out->insert(op.oid);
+  }
+  void operator()(const ECTransaction::TouchOp &op) {}
+  void operator()(const ECTransaction::CloneOp &op) {
+    out->insert(op.source);
+    out->insert(op.target);
+  }
+  void operator()(const ECTransaction::RenameOp &op) {
+    out->insert(op.source);
+    out->insert(op.destination);
+  }
+  void operator()(const ECTransaction::StashOp &op) {
+    out->insert(op.oid);
+  }
+  void operator()(const ECTransaction::RemoveOp &op) {
+    out->insert(op.oid);
+  }
+  void operator()(const ECTransaction::SetAttrsOp &op) {}
+  void operator()(const ECTransaction::RmAttrOp &op) {}
+  void operator()(const ECTransaction::AllocHintOp &op) {}
+  void operator()(const ECTransaction::NoOp &op) {}
+};
+void ECTransaction::get_append_objects(
+  set *out) const
+{
+  AppendObjectsGenerator gen(out);
+  reverse_visit(gen);
+}
+
+struct TransGenerator : public boost::static_visitor {
+  typedef void result_type;
+  map &hash_infos;
+
+  ErasureCodeInterfaceRef &ecimpl;
+  const pg_t pgid;
+  const ECUtil::stripe_info_t sinfo;
+  map *trans;
+  set want;
+  set *temp_added;
+  set *temp_removed;
+  stringstream *out;
+  TransGenerator(
+    map &hash_infos,
+    ErasureCodeInterfaceRef &ecimpl,
+    pg_t pgid,
+    const ECUtil::stripe_info_t &sinfo,
+    map *trans,
+    set *temp_added,
+    set *temp_removed,
+    stringstream *out)
+    : hash_infos(hash_infos),
+      ecimpl(ecimpl), pgid(pgid),
+      sinfo(sinfo),
+      trans(trans),
+      temp_added(temp_added), temp_removed(temp_removed),
+      out(out) {
+    for (unsigned i = 0; i < ecimpl->get_chunk_count(); ++i) {
+      want.insert(i);
+    }
+  }
+
+  coll_t get_coll_ct(shard_id_t shard, const hobject_t &hoid) {
+    if (hoid.is_temp()) {
+      temp_removed->erase(hoid);
+      temp_added->insert(hoid);
+    }
+    return get_coll(shard, hoid);
+  }
+  coll_t get_coll_rm(shard_id_t shard, const hobject_t &hoid) {
+    if (hoid.is_temp()) {
+      temp_added->erase(hoid);
+      temp_removed->insert(hoid);
+    }
+    return get_coll(shard, hoid);
+  }
+  coll_t get_coll(shard_id_t shard, const hobject_t &hoid) {
+    if (hoid.is_temp())
+      return coll_t::make_temp_coll(spg_t(pgid, shard));
+    else
+      return coll_t(spg_t(pgid, shard));
+  }
+
+  void operator()(const ECTransaction::TouchOp &op) {
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      i->second.touch(
+	get_coll_ct(i->first, op.oid),
+	ghobject_t(op.oid, ghobject_t::NO_GEN, i->first));
+    }
+  }
+  void operator()(const ECTransaction::AppendOp &op) {
+    uint64_t offset = op.off;
+    bufferlist bl(op.bl);
+    assert(bl.length());
+    assert(offset % sinfo.get_stripe_width() == 0);
+    map buffers;
+
+    assert(hash_infos.count(op.oid));
+    ECUtil::HashInfoRef hinfo = hash_infos[op.oid];
+
+    // align
+    if (bl.length() % sinfo.get_stripe_width())
+      bl.append_zero(
+	sinfo.get_stripe_width() -
+	((offset + bl.length()) % sinfo.get_stripe_width()));
+    assert(bl.length() - op.bl.length() < sinfo.get_stripe_width());
+    int r = ECUtil::encode(
+      sinfo, ecimpl, bl, want, &buffers);
+
+    hinfo->append(
+      sinfo.aligned_logical_offset_to_chunk_offset(op.off),
+      buffers);
+    bufferlist hbuf;
+    ::encode(
+      *hinfo,
+      hbuf);
+
+    assert(r == 0);
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      assert(buffers.count(i->first));
+      bufferlist &enc_bl = buffers[i->first];
+      i->second.write(
+	get_coll_ct(i->first, op.oid),
+	ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+	sinfo.logical_to_prev_chunk_offset(
+	  offset),
+	enc_bl.length(),
+	enc_bl);
+      i->second.setattr(
+	get_coll_ct(i->first, op.oid),
+	ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+	ECUtil::get_hinfo_key(),
+	hbuf);
+    }
+  }
+  void operator()(const ECTransaction::CloneOp &op) {
+    assert(hash_infos.count(op.source));
+    assert(hash_infos.count(op.target));
+    *(hash_infos[op.target]) = *(hash_infos[op.source]);
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      i->second.clone(
+	get_coll_ct(i->first, op.source),
+	ghobject_t(op.source, ghobject_t::NO_GEN, i->first),
+	ghobject_t(op.target, ghobject_t::NO_GEN, i->first));
+    }
+  }
+  void operator()(const ECTransaction::RenameOp &op) {
+    assert(hash_infos.count(op.source));
+    assert(hash_infos.count(op.destination));
+    *(hash_infos[op.destination]) = *(hash_infos[op.source]);
+    hash_infos[op.source]->clear();
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      i->second.collection_move_rename(
+	get_coll_rm(i->first, op.source),
+	ghobject_t(op.source, ghobject_t::NO_GEN, i->first),
+	get_coll_ct(i->first, op.destination),
+	ghobject_t(op.destination, ghobject_t::NO_GEN, i->first));
+    }
+  }
+  void operator()(const ECTransaction::StashOp &op) {
+    assert(hash_infos.count(op.oid));
+    hash_infos[op.oid]->clear();
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      coll_t cid(get_coll_rm(i->first, op.oid));
+      i->second.collection_move_rename(
+	cid,
+	ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+	cid,
+	ghobject_t(op.oid, op.version, i->first));
+    }
+  }
+  void operator()(const ECTransaction::RemoveOp &op) {
+    assert(hash_infos.count(op.oid));
+    hash_infos[op.oid]->clear();
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      i->second.remove(
+	get_coll_rm(i->first, op.oid),
+	ghobject_t(op.oid, ghobject_t::NO_GEN, i->first));
+    }
+  }
+  void operator()(const ECTransaction::SetAttrsOp &op) {
+    map attrs(op.attrs);
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      i->second.setattrs(
+	get_coll_ct(i->first, op.oid),
+	ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+	attrs);
+    }
+  }
+  void operator()(const ECTransaction::RmAttrOp &op) {
+    for (map::iterator i = trans->begin();
+	 i != trans->end();
+	 ++i) {
+      i->second.rmattr(
+	get_coll_ct(i->first, op.oid),
+	ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+	op.key);
+    }
+  }
+  void operator()(const ECTransaction::AllocHintOp &op) {
+    // logical_to_next_chunk_offset() scales down both aligned and
+    // unaligned offsets
+    uint64_t object_size = sinfo.logical_to_next_chunk_offset(
+                                                    op.expected_object_size);
+    uint64_t write_size = sinfo.logical_to_next_chunk_offset(
+                                                   op.expected_write_size);
+
+    for (map::iterator i = trans->begin();
+         i != trans->end();
+         ++i) {
+      i->second.set_alloc_hint(
+        get_coll_ct(i->first, op.oid),
+        ghobject_t(op.oid, ghobject_t::NO_GEN, i->first),
+        object_size, write_size);
+    }
+  }
+  void operator()(const ECTransaction::NoOp &op) {}
+};
+
+
+void ECTransaction::generate_transactions(
+  map &hash_infos,
+  ErasureCodeInterfaceRef &ecimpl,
+  pg_t pgid,
+  const ECUtil::stripe_info_t &sinfo,
+  map *transactions,
+  set *temp_added,
+  set *temp_removed,
+  stringstream *out) const
+{
+  TransGenerator gen(
+    hash_infos,
+    ecimpl,
+    pgid,
+    sinfo,
+    transactions,
+    temp_added,
+    temp_removed,
+    out);
+  visit(gen);
+}
diff --git a/ceph/src/osd/ECTransaction.h b/ceph/src/osd/ECTransaction.h
new file mode 100644
index 00000000..7b104c7e
--- /dev/null
+++ b/ceph/src/osd/ECTransaction.h
@@ -0,0 +1,207 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef ECTRANSACTION_H
+#define ECTRANSACTION_H
+
+#include "OSD.h"
+#include "PGBackend.h"
+#include "osd_types.h"
+#include "ECUtil.h"
+#include 
+#include "erasure-code/ErasureCodeInterface.h"
+
+class ECTransaction : public PGBackend::PGTransaction {
+public:
+  struct AppendOp {
+    hobject_t oid;
+    uint64_t off;
+    bufferlist bl;
+    AppendOp(const hobject_t &oid, uint64_t off, bufferlist &bl)
+      : oid(oid), off(off), bl(bl) {}
+  };
+  struct CloneOp {
+    hobject_t source;
+    hobject_t target;
+    CloneOp(const hobject_t &source, const hobject_t &target)
+      : source(source), target(target) {}
+  };
+  struct RenameOp {
+    hobject_t source;
+    hobject_t destination;
+    RenameOp(const hobject_t &source, const hobject_t &destination)
+      : source(source), destination(destination) {}
+  };
+  struct StashOp {
+    hobject_t oid;
+    version_t version;
+    StashOp(const hobject_t &oid, version_t version)
+      : oid(oid), version(version) {}
+  };
+  struct TouchOp {
+    hobject_t oid;
+    TouchOp(const hobject_t &oid) : oid(oid) {}
+  };
+  struct RemoveOp {
+    hobject_t oid;
+    RemoveOp(const hobject_t &oid) : oid(oid) {}
+  };
+  struct SetAttrsOp {
+    hobject_t oid;
+    map attrs;
+    SetAttrsOp(const hobject_t &oid, map &_attrs)
+      : oid(oid) {
+      attrs.swap(_attrs);
+    }
+    SetAttrsOp(const hobject_t &oid, const string &key, bufferlist &val)
+      : oid(oid) {
+      attrs.insert(make_pair(key, val));
+    }
+  };
+  struct RmAttrOp {
+    hobject_t oid;
+    string key;
+    RmAttrOp(const hobject_t &oid, const string &key) : oid(oid), key(key) {}
+  };
+  struct AllocHintOp {
+    hobject_t oid;
+    uint64_t expected_object_size;
+    uint64_t expected_write_size;
+    AllocHintOp(const hobject_t &oid,
+                uint64_t expected_object_size,
+                uint64_t expected_write_size)
+      : oid(oid), expected_object_size(expected_object_size),
+        expected_write_size(expected_write_size) {}
+  };
+  struct NoOp {};
+  typedef boost::variant<
+    AppendOp,
+    CloneOp,
+    RenameOp,
+    StashOp,
+    TouchOp,
+    RemoveOp,
+    SetAttrsOp,
+    RmAttrOp,
+    AllocHintOp,
+    NoOp> Op;
+  list ops;
+  uint64_t written;
+
+  ECTransaction() : written(0) {}
+  /// Write
+  void touch(
+    const hobject_t &hoid) {
+    bufferlist bl;
+    ops.push_back(TouchOp(hoid));
+  }
+  void append(
+    const hobject_t &hoid,
+    uint64_t off,
+    uint64_t len,
+    bufferlist &bl) {
+    if (len == 0) {
+      touch(hoid);
+      return;
+    }
+    written += len;
+    assert(len == bl.length());
+    ops.push_back(AppendOp(hoid, off, bl));
+  }
+  void stash(
+    const hobject_t &hoid,
+    version_t former_version) {
+    ops.push_back(StashOp(hoid, former_version));
+  }
+  void remove(
+    const hobject_t &hoid) {
+    ops.push_back(RemoveOp(hoid));
+  }
+  void setattrs(
+    const hobject_t &hoid,
+    map &attrs) {
+    ops.push_back(SetAttrsOp(hoid, attrs));
+  }
+  void setattr(
+    const hobject_t &hoid,
+    const string &attrname,
+    bufferlist &bl) {
+    ops.push_back(SetAttrsOp(hoid, attrname, bl));
+  }
+  void rmattr(
+    const hobject_t &hoid,
+    const string &attrname) {
+    ops.push_back(RmAttrOp(hoid, attrname));
+  }
+  void clone(
+    const hobject_t &from,
+    const hobject_t &to) {
+    ops.push_back(CloneOp(from, to));
+  }
+  void rename(
+    const hobject_t &from,
+    const hobject_t &to) {
+    ops.push_back(RenameOp(from, to));
+  }
+  void set_alloc_hint(
+    const hobject_t &hoid,
+    uint64_t expected_object_size,
+    uint64_t expected_write_size) {
+    ops.push_back(AllocHintOp(hoid, expected_object_size, expected_write_size));
+  }
+
+  void append(PGTransaction *_to_append) {
+    ECTransaction *to_append = static_cast(_to_append);
+    written += to_append->written;
+    to_append->written = 0;
+    ops.splice(ops.end(), to_append->ops,
+	       to_append->ops.begin(), to_append->ops.end());
+  }
+  void nop() {
+    ops.push_back(NoOp());
+  }
+  bool empty() const {
+    return ops.empty();
+  }
+  uint64_t get_bytes_written() const {
+    return written;
+  }
+  template 
+  void visit(T &vis) const {
+    for (list::const_iterator i = ops.begin(); i != ops.end(); ++i) {
+      boost::apply_visitor(vis, *i);
+    }
+  }
+  template 
+  void reverse_visit(T &vis) const {
+    for (list::const_reverse_iterator i = ops.rbegin();
+	 i != ops.rend();
+	 ++i) {
+      boost::apply_visitor(vis, *i);
+    }
+  }
+  void get_append_objects(
+    set *out) const;
+  void generate_transactions(
+    map &hash_infos,
+    ErasureCodeInterfaceRef &ecimpl,
+    pg_t pgid,
+    const ECUtil::stripe_info_t &sinfo,
+    map *transactions,
+    set *temp_added,
+    set *temp_removed,
+    stringstream *out = 0) const;
+};
+
+#endif
diff --git a/ceph/src/osd/ECUtil.cc b/ceph/src/osd/ECUtil.cc
new file mode 100644
index 00000000..1f3b4585
--- /dev/null
+++ b/ceph/src/osd/ECUtil.cc
@@ -0,0 +1,196 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include 
+#include "include/encoding.h"
+#include "ECUtil.h"
+
+int ECUtil::decode(
+  const stripe_info_t &sinfo,
+  ErasureCodeInterfaceRef &ec_impl,
+  map &to_decode,
+  bufferlist *out) {
+
+  uint64_t total_chunk_size = to_decode.begin()->second.length();
+
+  assert(to_decode.size());
+  assert(total_chunk_size % sinfo.get_chunk_size() == 0);
+  assert(out);
+  assert(out->length() == 0);
+
+  for (map::iterator i = to_decode.begin();
+       i != to_decode.end();
+       ++i) {
+    assert(i->second.length() == total_chunk_size);
+  }
+
+  if (total_chunk_size == 0)
+    return 0;
+
+  for (uint64_t i = 0; i < total_chunk_size; i += sinfo.get_chunk_size()) {
+    map chunks;
+    for (map::iterator j = to_decode.begin();
+	 j != to_decode.end();
+	 ++j) {
+      chunks[j->first].substr_of(j->second, i, sinfo.get_chunk_size());
+    }
+    bufferlist bl;
+    int r = ec_impl->decode_concat(chunks, &bl);
+    assert(bl.length() == sinfo.get_stripe_width());
+    assert(r == 0);
+    out->claim_append(bl);
+  }
+  return 0;
+}
+
+int ECUtil::decode(
+  const stripe_info_t &sinfo,
+  ErasureCodeInterfaceRef &ec_impl,
+  map &to_decode,
+  map &out) {
+
+  uint64_t total_chunk_size = to_decode.begin()->second.length();
+
+  assert(to_decode.size());
+  assert(total_chunk_size % sinfo.get_chunk_size() == 0);
+
+  for (map::iterator i = to_decode.begin();
+       i != to_decode.end();
+       ++i) {
+    assert(i->second.length() == total_chunk_size);
+  }
+
+  if (total_chunk_size == 0)
+    return 0;
+
+  set need;
+  for (map::iterator i = out.begin();
+       i != out.end();
+       ++i) {
+    assert(i->second);
+    assert(i->second->length() == 0);
+    need.insert(i->first);
+  }
+
+  for (uint64_t i = 0; i < total_chunk_size; i += sinfo.get_chunk_size()) {
+    map chunks;
+    for (map::iterator j = to_decode.begin();
+	 j != to_decode.end();
+	 ++j) {
+      chunks[j->first].substr_of(j->second, i, sinfo.get_chunk_size());
+    }
+    map out_bls;
+    int r = ec_impl->decode(need, chunks, &out_bls);
+    assert(r == 0);
+    for (map::iterator j = out.begin();
+	 j != out.end();
+	 ++j) {
+      assert(out_bls.count(j->first));
+      assert(out_bls[j->first].length() == sinfo.get_chunk_size());
+      j->second->claim_append(out_bls[j->first]);
+    }
+  }
+  for (map::iterator i = out.begin();
+       i != out.end();
+       ++i) {
+    assert(i->second->length() == total_chunk_size);
+  }
+  return 0;
+}
+
+int ECUtil::encode(
+  const stripe_info_t &sinfo,
+  ErasureCodeInterfaceRef &ec_impl,
+  bufferlist &in,
+  const set &want,
+  map *out) {
+
+  uint64_t logical_size = in.length();
+
+  assert(logical_size % sinfo.get_stripe_width() == 0);
+  assert(out);
+  assert(out->empty());
+
+  if (logical_size == 0)
+    return 0;
+
+  for (uint64_t i = 0; i < logical_size; i += sinfo.get_stripe_width()) {
+    map encoded;
+    bufferlist buf;
+    buf.substr_of(in, i, sinfo.get_stripe_width());
+    int r = ec_impl->encode(want, buf, &encoded);
+    assert(r == 0);
+    for (map::iterator i = encoded.begin();
+	 i != encoded.end();
+	 ++i) {
+      assert(i->second.length() == sinfo.get_chunk_size());
+      (*out)[i->first].claim_append(i->second);
+    }
+  }
+
+  for (map::iterator i = out->begin();
+       i != out->end();
+       ++i) {
+    assert(i->second.length() % sinfo.get_chunk_size() == 0);
+    assert(
+      sinfo.aligned_chunk_offset_to_logical_offset(i->second.length()) ==
+      logical_size);
+  }
+  return 0;
+}
+
+void ECUtil::HashInfo::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(total_chunk_size, bl);
+  ::encode(cumulative_shard_hashes, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ECUtil::HashInfo::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(total_chunk_size, bl);
+  ::decode(cumulative_shard_hashes, bl);
+  DECODE_FINISH(bl);
+}
+
+void ECUtil::HashInfo::dump(Formatter *f) const
+{
+  f->dump_unsigned("total_chunk_size", total_chunk_size);
+  f->open_object_section("cumulative_shard_hashes");
+  for (unsigned i = 0; i != cumulative_shard_hashes.size(); ++i) {
+    f->open_object_section("hash");
+    f->dump_unsigned("shard", i);
+    f->dump_unsigned("hash", cumulative_shard_hashes[i]);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ECUtil::HashInfo::generate_test_instances(list& o)
+{
+  o.push_back(new HashInfo(3));
+  {
+    bufferlist bl;
+    bl.append_zero(20);
+    map buffers;
+    buffers[0] = bl;
+    buffers[1] = bl;
+    buffers[2] = bl;
+    o.back()->append(0, buffers);
+    o.back()->append(20, buffers);
+  }
+  o.push_back(new HashInfo(4));
+}
+
+const string HINFO_KEY = "hinfo_key";
+
+bool ECUtil::is_hinfo_key_string(const string &key)
+{
+  return key == HINFO_KEY;
+}
+
+const string &ECUtil::get_hinfo_key()
+{
+  return HINFO_KEY;
+}
diff --git a/ceph/src/osd/ECUtil.h b/ceph/src/osd/ECUtil.h
new file mode 100644
index 00000000..52d79aab
--- /dev/null
+++ b/ceph/src/osd/ECUtil.h
@@ -0,0 +1,154 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef ECUTIL_H
+#define ECUTIL_H
+
+#include 
+#include 
+
+#include "include/memory.h"
+#include "erasure-code/ErasureCodeInterface.h"
+#include "include/buffer.h"
+#include "include/assert.h"
+#include "include/encoding.h"
+#include "common/Formatter.h"
+
+namespace ECUtil {
+
+const uint64_t CHUNK_ALIGNMENT = 64;
+const uint64_t CHUNK_INFO = 8;
+const uint64_t CHUNK_PADDING = 8;
+const uint64_t CHUNK_OVERHEAD = 16; // INFO + PADDING
+
+class stripe_info_t {
+  const uint64_t stripe_size;
+  const uint64_t stripe_width;
+  const uint64_t chunk_size;
+public:
+  stripe_info_t(uint64_t stripe_size, uint64_t stripe_width)
+    : stripe_size(stripe_size), stripe_width(stripe_width),
+      chunk_size(stripe_width / stripe_size) {
+    assert(stripe_width % stripe_size == 0);
+  }
+  uint64_t get_stripe_width() const {
+    return stripe_width;
+  }
+  uint64_t get_chunk_size() const {
+    return chunk_size;
+  }
+  uint64_t logical_to_prev_chunk_offset(uint64_t offset) const {
+    return (offset / stripe_width) * chunk_size;
+  }
+  uint64_t logical_to_next_chunk_offset(uint64_t offset) const {
+    return ((offset + stripe_width - 1)/ stripe_width) * chunk_size;
+  }
+  uint64_t logical_to_prev_stripe_offset(uint64_t offset) const {
+    return offset - (offset % stripe_width);
+  }
+  uint64_t logical_to_next_stripe_offset(uint64_t offset) const {
+    return offset % stripe_width ?
+      offset - (offset % stripe_width) + stripe_width :
+      offset;
+  }
+  uint64_t aligned_logical_offset_to_chunk_offset(uint64_t offset) const {
+    assert(offset % stripe_width == 0);
+    return (offset / stripe_width) * chunk_size;
+  }
+  uint64_t aligned_chunk_offset_to_logical_offset(uint64_t offset) const {
+    assert(offset % chunk_size == 0);
+    return (offset / chunk_size) * stripe_width;
+  }
+  pair aligned_offset_len_to_chunk(
+    pair in) const {
+    return make_pair(
+      aligned_logical_offset_to_chunk_offset(in.first),
+      aligned_logical_offset_to_chunk_offset(in.second));
+  }
+  pair offset_len_to_stripe_bounds(
+    pair in) const {
+    uint64_t off = logical_to_prev_stripe_offset(in.first);
+    uint64_t len = logical_to_next_stripe_offset(
+      (in.first - off) + in.second);
+    return make_pair(off, len);
+  }
+};
+
+int decode(
+  const stripe_info_t &sinfo,
+  ErasureCodeInterfaceRef &ec_impl,
+  map &to_decode,
+  bufferlist *out);
+
+int decode(
+  const stripe_info_t &sinfo,
+  ErasureCodeInterfaceRef &ec_impl,
+  map &to_decode,
+  map &out);
+
+int encode(
+  const stripe_info_t &sinfo,
+  ErasureCodeInterfaceRef &ec_impl,
+  bufferlist &in,
+  const set &want,
+  map *out);
+
+class HashInfo {
+  uint64_t total_chunk_size;
+  vector cumulative_shard_hashes;
+public:
+  HashInfo() : total_chunk_size(0) {}
+  HashInfo(unsigned num_chunks)
+  : total_chunk_size(0),
+    cumulative_shard_hashes(num_chunks, -1) {}
+  void append(uint64_t old_size, map &to_append) {
+    assert(to_append.size() == cumulative_shard_hashes.size());
+    assert(old_size == total_chunk_size);
+    uint64_t size_to_append = to_append.begin()->second.length();
+    for (map::iterator i = to_append.begin();
+	 i != to_append.end();
+	 ++i) {
+      assert(size_to_append == i->second.length());
+      assert((unsigned)i->first < cumulative_shard_hashes.size());
+      uint32_t new_hash = i->second.crc32c(cumulative_shard_hashes[i->first]);
+      cumulative_shard_hashes[i->first] = new_hash;
+    }
+    total_chunk_size += size_to_append;
+  }
+  void clear() {
+    total_chunk_size = 0;
+    cumulative_shard_hashes = vector(
+      cumulative_shard_hashes.size(),
+      -1);
+  }
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+  uint32_t get_chunk_hash(int shard) const {
+    assert((unsigned)shard < cumulative_shard_hashes.size());
+    return cumulative_shard_hashes[shard];
+  }
+  uint64_t get_total_chunk_size() const {
+    return total_chunk_size;
+  }
+};
+typedef ceph::shared_ptr HashInfoRef;
+
+bool is_hinfo_key_string(const string &key);
+const string &get_hinfo_key();
+
+};
+WRITE_CLASS_ENCODER(ECUtil::HashInfo)
+#endif
diff --git a/ceph/src/osd/HitSet.cc b/ceph/src/osd/HitSet.cc
new file mode 100644
index 00000000..700da5d4
--- /dev/null
+++ b/ceph/src/osd/HitSet.cc
@@ -0,0 +1,218 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "HitSet.h"
+
+// -- HitSet --
+
+HitSet::HitSet(const HitSet::Params& params)
+  : sealed(false)
+{
+  switch (params.get_type()) {
+  case TYPE_BLOOM:
+    {
+      BloomHitSet::Params *p =
+	static_cast(params.impl.get());
+      impl.reset(new BloomHitSet(p));
+    }
+    break;
+
+  case TYPE_EXPLICIT_HASH:
+    impl.reset(new ExplicitHashHitSet(static_cast(params.impl.get())));
+    break;
+
+  case TYPE_EXPLICIT_OBJECT:
+    impl.reset(new ExplicitObjectHitSet(static_cast(params.impl.get())));
+    break;
+
+  case TYPE_NONE:
+    break;
+
+  default:
+    assert (0 == "unknown HitSet type");
+  }
+}
+
+void HitSet::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(sealed, bl);
+  if (impl) {
+    ::encode((__u8)impl->get_type(), bl);
+    impl->encode(bl);
+  } else {
+    ::encode((__u8)TYPE_NONE, bl);
+  }
+  ENCODE_FINISH(bl);
+}
+
+void HitSet::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(sealed, bl);
+  __u8 type;
+  ::decode(type, bl);
+  switch ((impl_type_t)type) {
+  case TYPE_EXPLICIT_HASH:
+    impl.reset(new ExplicitHashHitSet);
+    break;
+  case TYPE_EXPLICIT_OBJECT:
+    impl.reset(new ExplicitObjectHitSet);
+    break;
+  case TYPE_BLOOM:
+    impl.reset(new BloomHitSet);
+    break;
+  case TYPE_NONE:
+    impl.reset(NULL);
+    break;
+  default:
+    throw buffer::malformed_input("unrecognized HitMap type");
+  }
+  if (impl)
+    impl->decode(bl);
+  DECODE_FINISH(bl);
+}
+
+void HitSet::dump(Formatter *f) const
+{
+  f->dump_string("type", get_type_name());
+  f->dump_string("sealed", sealed ? "yes" : "no");
+  if (impl)
+    impl->dump(f);
+}
+
+void HitSet::generate_test_instances(list& o)
+{
+  o.push_back(new HitSet);
+  o.push_back(new HitSet(new BloomHitSet(10, .1, 1)));
+  o.back()->insert(hobject_t());
+  o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+  o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+  o.push_back(new HitSet(new ExplicitHashHitSet));
+  o.back()->insert(hobject_t());
+  o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+  o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+  o.push_back(new HitSet(new ExplicitObjectHitSet));
+  o.back()->insert(hobject_t());
+  o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+  o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+}
+
+HitSet::Params::Params(const Params& o)
+{
+  if (o.get_type() != TYPE_NONE) {
+    create_impl(o.get_type());
+    // it's annoying to write virtual operator= methods; use encode/decode
+    // instead.
+    bufferlist bl;
+    o.impl->encode(bl);
+    bufferlist::iterator p = bl.begin();
+    impl->decode(p);
+  } // else we don't need to do anything
+}
+
+const HitSet::Params& HitSet::Params::operator=(const Params& o)
+{
+  create_impl(o.get_type());
+  if (o.impl) {
+    // it's annoying to write virtual operator= methods; use encode/decode
+    // instead.
+    bufferlist bl;
+    o.impl->encode(bl);
+    bufferlist::iterator p = bl.begin();
+    impl->decode(p);
+  }
+  return *this;
+}
+
+void HitSet::Params::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  if (impl) {
+    ::encode((__u8)impl->get_type(), bl);
+    impl->encode(bl);
+  } else {
+    ::encode((__u8)TYPE_NONE, bl);
+  }
+  ENCODE_FINISH(bl);
+}
+
+bool HitSet::Params::create_impl(impl_type_t type)
+{
+  switch ((impl_type_t)type) {
+  case TYPE_EXPLICIT_HASH:
+    impl.reset(new ExplicitHashHitSet::Params);
+    break;
+  case TYPE_EXPLICIT_OBJECT:
+    impl.reset(new ExplicitObjectHitSet::Params);
+    break;
+  case TYPE_BLOOM:
+    impl.reset(new BloomHitSet::Params);
+    break;
+  case TYPE_NONE:
+    impl.reset(NULL);
+    break;
+  default:
+    return false;
+  }
+  return true;
+}
+
+void HitSet::Params::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  __u8 type;
+  ::decode(type, bl);
+  if (!create_impl((impl_type_t)type))
+    throw buffer::malformed_input("unrecognized HitMap type");
+  if (impl)
+    impl->decode(bl);
+  DECODE_FINISH(bl);
+}
+
+void HitSet::Params::dump(Formatter *f) const
+{
+  f->dump_string("type", HitSet::get_type_name(get_type()));
+  if (impl)
+    impl->dump(f);
+}
+
+void HitSet::Params::generate_test_instances(list& o)
+{
+#define loop_hitset_params(kind) \
+{ \
+  list params; \
+  kind::Params::generate_test_instances(params); \
+  for (list::iterator i = params.begin(); \
+  i != params.end(); ++i) \
+    o.push_back(new Params(*i)); \
+}
+  o.push_back(new Params);
+  o.push_back(new Params(new BloomHitSet::Params));
+  loop_hitset_params(BloomHitSet);
+  o.push_back(new Params(new ExplicitHashHitSet::Params));
+  loop_hitset_params(ExplicitHashHitSet);
+  o.push_back(new Params(new ExplicitObjectHitSet::Params));
+  loop_hitset_params(ExplicitObjectHitSet);
+}
+
+ostream& operator<<(ostream& out, const HitSet::Params& p) {
+  out << HitSet::get_type_name(p.get_type());
+  if (p.impl) {
+    out << "{";
+    p.impl->dump_stream(out);
+  }
+  out << "}";
+  return out;
+}
diff --git a/ceph/src/osd/HitSet.h b/ceph/src/osd/HitSet.h
new file mode 100644
index 00000000..476678ef
--- /dev/null
+++ b/ceph/src/osd/HitSet.h
@@ -0,0 +1,477 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_OSD_HITSET_H
+#define CEPH_OSD_HITSET_H
+
+#include 
+
+#include "include/encoding.h"
+#include "include/unordered_set.h"
+#include "common/bloom_filter.hpp"
+#include "common/hobject.h"
+#include "common/Formatter.h"
+
+/**
+ * generic container for a HitSet
+ *
+ * Encapsulate a HitSetImpl of any type.  Expose a generic interface
+ * to users and wrap the encoded object with a type so that it can be
+ * safely decoded later.
+ */
+
+class HitSet {
+public:
+  typedef enum {
+    TYPE_NONE = 0,
+    TYPE_EXPLICIT_HASH = 1,
+    TYPE_EXPLICIT_OBJECT = 2,
+    TYPE_BLOOM = 3
+  } impl_type_t;
+
+  static const char *get_type_name(impl_type_t t) {
+    switch (t) {
+    case TYPE_NONE: return "none";
+    case TYPE_EXPLICIT_HASH: return "explicit_hash";
+    case TYPE_EXPLICIT_OBJECT: return "explicit_object";
+    case TYPE_BLOOM: return "bloom";
+    default: return "???";
+    }
+  }
+  const char *get_type_name() const {
+    if (impl)
+      return get_type_name(impl->get_type());
+    return get_type_name(TYPE_NONE);
+  }
+
+  /// abstract interface for a HitSet implementation
+  class Impl {
+  public:
+    virtual impl_type_t get_type() const = 0;
+    virtual bool is_full() const = 0;
+    virtual void insert(const hobject_t& o) = 0;
+    virtual bool contains(const hobject_t& o) const = 0;
+    virtual unsigned insert_count() const = 0;
+    virtual unsigned approx_unique_insert_count() const = 0;
+    virtual void encode(bufferlist &bl) const = 0;
+    virtual void decode(bufferlist::iterator& p) = 0;
+    virtual void dump(Formatter *f) const = 0;
+    virtual Impl* clone() const = 0;
+    virtual void seal() {}
+    virtual ~Impl() {}
+  };
+
+  boost::scoped_ptr impl;
+  bool sealed;
+
+  class Params {
+    /// create an Impl* of the given type
+    bool create_impl(impl_type_t t);
+
+  public:
+    class Impl {
+    public:
+      virtual impl_type_t get_type() const = 0;
+      virtual HitSet::Impl *get_new_impl() const = 0;
+      virtual void encode(bufferlist &bl) const {}
+      virtual void decode(bufferlist::iterator& p) {}
+      virtual void dump(Formatter *f) const {}
+      virtual void dump_stream(ostream& o) const {}
+      virtual ~Impl() {}
+    };
+
+    Params()  {}
+    Params(Impl *i) : impl(i) {}
+    virtual ~Params() {}
+
+    boost::scoped_ptr impl;
+
+    impl_type_t get_type() const {
+      if (impl)
+	return impl->get_type();
+      return TYPE_NONE;
+    }
+
+    Params(const Params& o);
+    const Params& operator=(const Params& o);
+
+    void encode(bufferlist &bl) const;
+    void decode(bufferlist::iterator &bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& o);
+
+    friend ostream& operator<<(ostream& out, const HitSet::Params& p);
+  };
+
+  HitSet() : impl(NULL), sealed(false) {}
+  HitSet(Impl *i) : impl(i), sealed(false) {}
+  HitSet(const HitSet::Params& params);
+
+  HitSet(const HitSet& o) {
+    sealed = o.sealed;
+    if (o.impl)
+      impl.reset(o.impl->clone());
+    else
+      impl.reset(NULL);
+  }
+  const HitSet& operator=(const HitSet& o) {
+    sealed = o.sealed;
+    if (o.impl)
+      impl.reset(o.impl->clone());
+    else
+      impl.reset(NULL);
+    return *this;
+  }
+
+
+  bool is_full() const {
+    return impl->is_full();
+  }
+  /// insert a hash into the set
+  void insert(const hobject_t& o) {
+    impl->insert(o);
+  }
+  /// query whether a hash is in the set
+  bool contains(const hobject_t& o) const {
+    return impl->contains(o);
+  }
+
+  unsigned insert_count() const {
+    return impl->insert_count();
+  }
+  unsigned approx_unique_insert_count() const {
+    return impl->approx_unique_insert_count();
+  }
+  void seal() {
+    assert(!sealed);
+    sealed = true;
+    impl->seal();
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+private:
+  void reset_to_type(impl_type_t type);
+};
+WRITE_CLASS_ENCODER(HitSet);
+WRITE_CLASS_ENCODER(HitSet::Params);
+
+typedef boost::shared_ptr HitSetRef;
+
+ostream& operator<<(ostream& out, const HitSet::Params& p);
+
+/**
+ * explicitly enumerate hash hits in the set
+ */
+class ExplicitHashHitSet : public HitSet::Impl {
+  uint64_t count;
+  ceph::unordered_set hits;
+public:
+  class Params : public HitSet::Params::Impl {
+  public:
+    virtual HitSet::impl_type_t get_type() const {
+      return HitSet::TYPE_EXPLICIT_HASH;
+    }
+    virtual HitSet::Impl *get_new_impl() const {
+      return new ExplicitHashHitSet;
+    }
+    static void generate_test_instances(list& o) {
+      o.push_back(new Params);
+    }
+  };
+
+  ExplicitHashHitSet() : count(0) {}
+  ExplicitHashHitSet(const ExplicitHashHitSet::Params *p) : count(0) {}
+  ExplicitHashHitSet(const ExplicitHashHitSet &o) : count(o.count),
+      hits(o.hits) {}
+
+  HitSet::Impl *clone() const {
+    return new ExplicitHashHitSet(*this);
+  }
+
+  HitSet::impl_type_t get_type() const {
+    return HitSet::TYPE_EXPLICIT_HASH;
+  }
+  bool is_full() const {
+    return false;
+  }
+  void insert(const hobject_t& o) {
+    hits.insert(o.hash);
+    ++count;
+  }
+  bool contains(const hobject_t& o) const {
+    return hits.count(o.hash);
+  }
+  unsigned insert_count() const {
+    return count;
+  }
+  unsigned approx_unique_insert_count() const {
+    return hits.size();
+  }
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(count, bl);
+    ::encode(hits, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    DECODE_START(1, bl);
+    ::decode(count, bl);
+    ::decode(hits, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const {
+    f->dump_unsigned("insert_count", count);
+    f->open_array_section("hash_set");
+    for (ceph::unordered_set::const_iterator p = hits.begin(); p != hits.end(); ++p)
+      f->dump_unsigned("hash", *p);
+    f->close_section();
+  }
+  static void generate_test_instances(list& o) {
+    o.push_back(new ExplicitHashHitSet);
+    o.push_back(new ExplicitHashHitSet);
+    o.back()->insert(hobject_t());
+    o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+    o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+  }
+};
+WRITE_CLASS_ENCODER(ExplicitHashHitSet)
+
+/**
+ * explicitly enumerate objects in the set
+ */
+class ExplicitObjectHitSet : public HitSet::Impl {
+  uint64_t count;
+  ceph::unordered_set hits;
+public:
+  class Params : public HitSet::Params::Impl {
+  public:
+    virtual HitSet::impl_type_t get_type() const {
+      return HitSet::TYPE_EXPLICIT_OBJECT;
+    }
+    virtual HitSet::Impl *get_new_impl() const {
+      return new ExplicitObjectHitSet;
+    }
+    static void generate_test_instances(list& o) {
+      o.push_back(new Params);
+    }
+  };
+
+  ExplicitObjectHitSet() : count(0) {}
+  ExplicitObjectHitSet(const ExplicitObjectHitSet::Params *p) : count(0) {}
+  ExplicitObjectHitSet(const ExplicitObjectHitSet &o) : count(o.count),
+      hits(o.hits) {}
+
+  HitSet::Impl *clone() const {
+    return new ExplicitObjectHitSet(*this);
+  }
+
+  HitSet::impl_type_t get_type() const {
+    return HitSet::TYPE_EXPLICIT_OBJECT;
+  }
+  bool is_full() const {
+    return false;
+  }
+  void insert(const hobject_t& o) {
+    hits.insert(o);
+    ++count;
+  }
+  bool contains(const hobject_t& o) const {
+    return hits.count(o);
+  }
+  unsigned insert_count() const {
+    return count;
+  }
+  unsigned approx_unique_insert_count() const {
+    return hits.size();
+  }
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(count, bl);
+    ::encode(hits, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    DECODE_START(1, bl);
+    ::decode(count, bl);
+    ::decode(hits, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const {
+    f->dump_unsigned("insert_count", count);
+    f->open_array_section("set");
+    for (ceph::unordered_set::const_iterator p = hits.begin(); p != hits.end(); ++p) {
+      f->open_object_section("object");
+      p->dump(f);
+      f->close_section();
+    }
+    f->close_section();
+  }
+  static void generate_test_instances(list& o) {
+    o.push_back(new ExplicitObjectHitSet);
+    o.push_back(new ExplicitObjectHitSet);
+    o.back()->insert(hobject_t());
+    o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+    o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+  }
+};
+WRITE_CLASS_ENCODER(ExplicitObjectHitSet)
+
+/**
+ * use a bloom_filter to track hits to the set
+ */
+class BloomHitSet : public HitSet::Impl {
+  compressible_bloom_filter bloom;
+
+public:
+  HitSet::impl_type_t get_type() const {
+    return HitSet::TYPE_BLOOM;
+  }
+
+  class Params : public HitSet::Params::Impl {
+  public:
+    virtual HitSet::impl_type_t get_type() const {
+      return HitSet::TYPE_BLOOM;
+    }
+    virtual HitSet::Impl *get_new_impl() const {
+      return new BloomHitSet;
+    }
+
+    uint32_t fpp_micro;    ///< false positive probability / 1M
+    uint64_t target_size;  ///< number of unique insertions we expect to this HitSet
+    uint64_t seed;         ///< seed to use when initializing the bloom filter
+
+    Params()
+      : fpp_micro(0), target_size(0), seed(0) {}
+    Params(double fpp, uint64_t t, uint64_t s)
+      : fpp_micro(fpp * 1000000.0), target_size(t), seed(s) {}
+    Params(const Params &o)
+      : fpp_micro(o.fpp_micro),
+	target_size(o.target_size),
+	seed(o.seed) {}
+    ~Params() {}
+
+    double get_fpp() const {
+      return (double)fpp_micro / 1000000.0;
+    }
+    void set_fpp(double f) {
+      fpp_micro = (unsigned)(llrintl(f * (double)1000000.0));
+    }
+
+    void encode(bufferlist& bl) const {
+      ENCODE_START(1, 1, bl);
+      ::encode(fpp_micro, bl);
+      ::encode(target_size, bl);
+      ::encode(seed, bl);
+      ENCODE_FINISH(bl);
+    }
+    void decode(bufferlist::iterator& bl) {
+      DECODE_START(1, bl);
+      ::decode(fpp_micro, bl);
+      ::decode(target_size, bl);
+      ::decode(seed, bl);
+      DECODE_FINISH(bl);
+    }
+    void dump(Formatter *f) const {
+      f->dump_float("false_positive_probability", get_fpp());
+      f->dump_int("target_size", target_size);
+      f->dump_int("seed", seed);
+    }
+    void dump_stream(ostream& o) const {
+      o << "false_positive_probability: "
+	<< get_fpp() << ", target_size: " << target_size
+	<< ", seed: " << seed;
+    }
+    static void generate_test_instances(list& o) {
+      o.push_back(new Params);
+      o.push_back(new Params);
+      (*o.rbegin())->fpp_micro = 123456;
+      (*o.rbegin())->target_size = 300;
+      (*o.rbegin())->seed = 99;
+    }
+  };
+
+  BloomHitSet() {}
+  BloomHitSet(unsigned inserts, double fpp, int seed)
+    : bloom(inserts, fpp, seed)
+  {}
+  BloomHitSet(const BloomHitSet::Params *p) : bloom(p->target_size,
+                                                    p->get_fpp(),
+                                                    p->seed)
+  {}
+
+  BloomHitSet(const BloomHitSet &o) {
+    // oh god
+    bufferlist bl;
+    o.encode(bl);
+    bufferlist::iterator bli = bl.begin();
+    this->decode(bli);
+  }
+
+  HitSet::Impl *clone() const {
+    return new BloomHitSet(*this);
+  }
+
+  bool is_full() const {
+    return bloom.is_full();
+  }
+
+  void insert(const hobject_t& o) {
+    bloom.insert(o.hash);
+  }
+  bool contains(const hobject_t& o) const {
+    return bloom.contains(o.hash);
+  }
+  unsigned insert_count() const {
+    return bloom.element_count();
+  }
+  unsigned approx_unique_insert_count() const {
+    return bloom.approx_unique_element_count();
+  }
+  void seal() {
+    // aim for a density of .5 (50% of bit set)
+    double pc = (double)bloom.density() * 2.0;
+    if (pc < 1.0)
+      bloom.compress(pc);
+  }
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(bloom, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    DECODE_START(1, bl);
+    ::decode(bloom, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const {
+    f->open_object_section("bloom_filter");
+    bloom.dump(f);
+    f->close_section();
+  }
+  static void generate_test_instances(list& o) {
+    o.push_back(new BloomHitSet);
+    o.push_back(new BloomHitSet(10, .1, 1));
+    o.back()->insert(hobject_t());
+    o.back()->insert(hobject_t("asdf", "", CEPH_NOSNAP, 123, 1, ""));
+    o.back()->insert(hobject_t("qwer", "", CEPH_NOSNAP, 456, 1, ""));
+  }
+};
+WRITE_CLASS_ENCODER(BloomHitSet)
+
+#endif
diff --git a/ceph/src/osd/Makefile.am b/ceph/src/osd/Makefile.am
new file mode 100644
index 00000000..75e48771
--- /dev/null
+++ b/ceph/src/osd/Makefile.am
@@ -0,0 +1,52 @@
+libosd_types_la_SOURCES = \
+	osd/PGLog.cc \
+	osd/osd_types.cc \
+	osd/ECUtil.cc
+libosd_types_la_CXXFLAGS = ${AM_CXXFLAGS}
+noinst_LTLIBRARIES += libosd_types.la
+
+libosd_la_SOURCES = \
+	osd/PG.cc \
+	osd/ReplicatedPG.cc \
+	osd/ReplicatedBackend.cc \
+	osd/ECBackend.cc \
+	osd/ECMsgTypes.cc \
+	osd/ECTransaction.cc \
+	osd/PGBackend.cc \
+	osd/Ager.cc \
+	osd/HitSet.cc \
+	osd/OSD.cc \
+	osd/OSDCap.cc \
+	osd/Watch.cc \
+	osd/ClassHandler.cc \
+	osd/OpRequest.cc \
+	common/TrackedOp.cc \
+	osd/SnapMapper.cc \
+	objclass/class_api.cc
+libosd_la_CXXFLAGS = ${AM_CXXFLAGS}
+libosd_la_LIBADD = $(LIBOSDC) $(LIBOS) $(LIBOSD_TYPES) $(LIBOS_TYPES)
+noinst_LTLIBRARIES += libosd.la
+
+noinst_HEADERS += \
+	osd/Ager.h \
+	osd/ClassHandler.h \
+	osd/HitSet.h \
+	osd/OSD.h \
+	osd/OSDCap.h \
+	osd/OSDMap.h \
+	osd/ObjectVersioner.h \
+	osd/OpRequest.h \
+	osd/SnapMapper.h \
+	osd/PG.h \
+	osd/PGLog.h \
+	osd/ReplicatedPG.h \
+	osd/PGBackend.h \
+	osd/ReplicatedBackend.h \
+	osd/TierAgentState.h \
+	osd/ECBackend.h \
+	osd/ECUtil.h \
+	osd/ECMsgTypes.h \
+	osd/ECTransaction.h \
+	osd/Watch.h \
+	osd/osd_types.h
+
diff --git a/ceph/src/osd/OSD.cc b/ceph/src/osd/OSD.cc
new file mode 100644
index 00000000..dc67fdd7
--- /dev/null
+++ b/ceph/src/osd/OSD.cc
@@ -0,0 +1,8043 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include "acconfig.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#ifdef HAVE_SYS_PARAM_H
+#include 
+#endif
+
+#ifdef HAVE_SYS_MOUNT_H
+#include 
+#endif
+
+#include "osd/PG.h"
+
+#include "include/types.h"
+#include "include/compat.h"
+
+#include "OSD.h"
+#include "OSDMap.h"
+#include "Watch.h"
+#include "osdc/Objecter.h"
+
+#include "common/ceph_argparse.h"
+#include "common/version.h"
+#include "common/io_priority.h"
+
+#include "os/ObjectStore.h"
+
+#include "ReplicatedPG.h"
+
+#include "Ager.h"
+
+
+#include "msg/Messenger.h"
+#include "msg/Message.h"
+
+#include "mon/MonClient.h"
+
+#include "messages/MLog.h"
+
+#include "messages/MGenericMessage.h"
+#include "messages/MPing.h"
+#include "messages/MOSDPing.h"
+#include "messages/MOSDFailure.h"
+#include "messages/MOSDMarkMeDown.h"
+#include "messages/MOSDOp.h"
+#include "messages/MOSDOpReply.h"
+#include "messages/MOSDSubOp.h"
+#include "messages/MOSDSubOpReply.h"
+#include "messages/MOSDBoot.h"
+#include "messages/MOSDPGTemp.h"
+
+#include "messages/MOSDMap.h"
+#include "messages/MOSDPGNotify.h"
+#include "messages/MOSDPGQuery.h"
+#include "messages/MOSDPGLog.h"
+#include "messages/MOSDPGRemove.h"
+#include "messages/MOSDPGInfo.h"
+#include "messages/MOSDPGCreate.h"
+#include "messages/MOSDPGTrim.h"
+#include "messages/MOSDPGScan.h"
+#include "messages/MOSDPGBackfill.h"
+#include "messages/MOSDPGMissing.h"
+#include "messages/MBackfillReserve.h"
+#include "messages/MRecoveryReserve.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
+
+#include "messages/MOSDAlive.h"
+
+#include "messages/MOSDScrub.h"
+#include "messages/MOSDRepScrub.h"
+
+#include "messages/MMonCommand.h"
+#include "messages/MCommand.h"
+#include "messages/MCommandReply.h"
+
+#include "messages/MPGStats.h"
+#include "messages/MPGStatsAck.h"
+
+#include "messages/MWatchNotify.h"
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPushReply.h"
+#include "messages/MOSDPGPull.h"
+
+#include "common/perf_counters.h"
+#include "common/Timer.h"
+#include "common/LogClient.h"
+#include "common/HeartbeatMap.h"
+#include "common/admin_socket.h"
+
+#include "global/signal_handler.h"
+#include "global/pidfile.h"
+
+#include "include/color.h"
+#include "perfglue/cpu_profiler.h"
+#include "perfglue/heap_profiler.h"
+
+#include "osd/ClassHandler.h"
+#include "osd/OpRequest.h"
+
+#include "auth/AuthAuthorizeHandler.h"
+
+#include "common/errno.h"
+
+#include "objclass/objclass.h"
+
+#include "common/cmdparse.h"
+#include "include/str_list.h"
+
+#include "include/assert.h"
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, whoami, get_osdmap())
+
+static ostream& _prefix(std::ostream* _dout, int whoami, OSDMapRef osdmap) {
+  return *_dout << "osd." << whoami << " "
+		<< (osdmap ? osdmap->get_epoch():0)
+		<< " ";
+}
+
+//Initial features in new superblock.
+//Features here are also automatically upgraded
+CompatSet OSD::get_osd_initial_compat_set() {
+  CompatSet::FeatureSet ceph_osd_feature_compat;
+  CompatSet::FeatureSet ceph_osd_feature_ro_compat;
+  CompatSet::FeatureSet ceph_osd_feature_incompat;
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_PGINFO);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_OLOC);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEC);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_CATEGORIES);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_HOBJECTPOOL);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BIGINFO);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBINFO);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBLOG);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SNAPMAPPER);
+  return CompatSet(ceph_osd_feature_compat, ceph_osd_feature_ro_compat,
+		   ceph_osd_feature_incompat);
+}
+
+//Features are added here that this OSD supports.
+CompatSet OSD::get_osd_compat_set() {
+  CompatSet compat =  get_osd_initial_compat_set();
+  //Any features here can be set in code, but not in initial superblock
+  compat.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
+  return compat;
+}
+
+OSDService::OSDService(OSD *osd) :
+  osd(osd),
+  cct(osd->cct),
+  whoami(osd->whoami), store(osd->store), clog(osd->clog),
+  pg_recovery_stats(osd->pg_recovery_stats),
+  infos_oid(OSD::make_infos_oid()),
+  cluster_messenger(osd->cluster_messenger),
+  client_messenger(osd->client_messenger),
+  logger(osd->logger),
+  recoverystate_perf(osd->recoverystate_perf),
+  monc(osd->monc),
+  op_wq(osd->op_wq),
+  peering_wq(osd->peering_wq),
+  recovery_wq(osd->recovery_wq),
+  snap_trim_wq(osd->snap_trim_wq),
+  scrub_wq(osd->scrub_wq),
+  scrub_finalize_wq(osd->scrub_finalize_wq),
+  rep_scrub_wq(osd->rep_scrub_wq),
+  push_wq("push_wq", cct->_conf->osd_recovery_thread_timeout, &osd->recovery_tp),
+  gen_wq("gen_wq", cct->_conf->osd_recovery_thread_timeout, &osd->recovery_tp),
+  class_handler(osd->class_handler),
+  pg_epoch_lock("OSDService::pg_epoch_lock"),
+  publish_lock("OSDService::publish_lock"),
+  pre_publish_lock("OSDService::pre_publish_lock"),
+  sched_scrub_lock("OSDService::sched_scrub_lock"), scrubs_pending(0),
+  scrubs_active(0),
+  agent_lock("OSD::agent_lock"),
+  agent_valid_iterator(false),
+  agent_ops(0),
+  agent_active(true),
+  agent_thread(this),
+  agent_stop_flag(false),
+  agent_timer_lock("OSD::agent_timer_lock"),
+  agent_timer(osd->client_messenger->cct, agent_timer_lock),
+  objecter_lock("OSD::objecter_lock"),
+  objecter_timer(osd->client_messenger->cct, objecter_lock),
+  objecter(new Objecter(osd->client_messenger->cct, osd->objecter_messenger, osd->monc, &objecter_osdmap,
+			objecter_lock, objecter_timer, 0, 0)),
+  objecter_finisher(osd->client_messenger->cct),
+  objecter_dispatcher(this),
+  watch_lock("OSD::watch_lock"),
+  watch_timer(osd->client_messenger->cct, watch_lock),
+  next_notif_id(0),
+  backfill_request_lock("OSD::backfill_request_lock"),
+  backfill_request_timer(cct, backfill_request_lock, false),
+  last_tid(0),
+  tid_lock("OSDService::tid_lock"),
+  reserver_finisher(cct),
+  local_reserver(&reserver_finisher, cct->_conf->osd_max_backfills),
+  remote_reserver(&reserver_finisher, cct->_conf->osd_max_backfills),
+  pg_temp_lock("OSDService::pg_temp_lock"),
+  map_cache_lock("OSDService::map_lock"),
+  map_cache(cct->_conf->osd_map_cache_size),
+  map_bl_cache(cct->_conf->osd_map_cache_size),
+  map_bl_inc_cache(cct->_conf->osd_map_cache_size),
+  in_progress_split_lock("OSDService::in_progress_split_lock"),
+  full_status_lock("OSDService::full_status_lock"),
+  cur_state(NONE),
+  last_msg(0),
+  cur_ratio(0),
+  is_stopping_lock("OSDService::is_stopping_lock"),
+  state(NOT_STOPPING)
+#ifdef PG_DEBUG_REFS
+  , pgid_lock("OSDService::pgid_lock")
+#endif
+{}
+
+OSDService::~OSDService()
+{
+  delete objecter;
+}
+
+void OSDService::_start_split(spg_t parent, const set &children)
+{
+  for (set::const_iterator i = children.begin();
+       i != children.end();
+       ++i) {
+    dout(10) << __func__ << ": Starting split on pg " << *i
+	     << ", parent=" << parent << dendl;
+    assert(!pending_splits.count(*i));
+    assert(!in_progress_splits.count(*i));
+    pending_splits.insert(make_pair(*i, parent));
+
+    assert(!rev_pending_splits[parent].count(*i));
+    rev_pending_splits[parent].insert(*i);
+  }
+}
+
+void OSDService::mark_split_in_progress(spg_t parent, const set &children)
+{
+  Mutex::Locker l(in_progress_split_lock);
+  map >::iterator piter = rev_pending_splits.find(parent);
+  assert(piter != rev_pending_splits.end());
+  for (set::const_iterator i = children.begin();
+       i != children.end();
+       ++i) {
+    assert(piter->second.count(*i));
+    assert(pending_splits.count(*i));
+    assert(!in_progress_splits.count(*i));
+    assert(pending_splits[*i] == parent);
+
+    pending_splits.erase(*i);
+    piter->second.erase(*i);
+    in_progress_splits.insert(*i);
+  }
+  if (piter->second.empty())
+    rev_pending_splits.erase(piter);
+}
+
+void OSDService::cancel_pending_splits_for_parent(spg_t parent)
+{
+  Mutex::Locker l(in_progress_split_lock);
+  return _cancel_pending_splits_for_parent(parent);
+}
+
+void OSDService::_cancel_pending_splits_for_parent(spg_t parent)
+{
+  map >::iterator piter = rev_pending_splits.find(parent);
+  if (piter == rev_pending_splits.end())
+    return;
+
+  for (set::iterator i = piter->second.begin();
+       i != piter->second.end();
+       ++i) {
+    assert(pending_splits.count(*i));
+    assert(!in_progress_splits.count(*i));
+    pending_splits.erase(*i);
+    dout(10) << __func__ << ": Completing split on pg " << *i
+	     << " for parent: " << parent << dendl;
+    _cancel_pending_splits_for_parent(*i);
+  }
+  rev_pending_splits.erase(piter);
+}
+
+void OSDService::_maybe_split_pgid(OSDMapRef old_map,
+				  OSDMapRef new_map,
+				  spg_t pgid)
+{
+  assert(old_map->have_pg_pool(pgid.pool()));
+  if (pgid.ps() < static_cast(old_map->get_pg_num(pgid.pool()))) {
+    set children;
+    pgid.is_split(old_map->get_pg_num(pgid.pool()),
+		  new_map->get_pg_num(pgid.pool()), &children);
+    _start_split(pgid, children);
+  } else {
+    assert(pgid.ps() < static_cast(new_map->get_pg_num(pgid.pool())));
+  }
+}
+
+void OSDService::init_splits_between(spg_t pgid,
+				     OSDMapRef frommap,
+				     OSDMapRef tomap)
+{
+  // First, check whether we can avoid this potentially expensive check
+  if (tomap->have_pg_pool(pgid.pool()) &&
+      pgid.is_split(
+	frommap->get_pg_num(pgid.pool()),
+	tomap->get_pg_num(pgid.pool()),
+	NULL)) {
+    // Ok, a split happened, so we need to walk the osdmaps
+    set new_pgs; // pgs to scan on each map
+    new_pgs.insert(pgid);
+    OSDMapRef curmap(get_map(frommap->get_epoch()));
+    for (epoch_t e = frommap->get_epoch() + 1;
+	 e <= tomap->get_epoch();
+	 ++e) {
+      OSDMapRef nextmap(try_get_map(e));
+      if (!nextmap)
+	continue;
+      set even_newer_pgs; // pgs added in this loop
+      for (set::iterator i = new_pgs.begin(); i != new_pgs.end(); ++i) {
+	set split_pgs;
+	if (i->is_split(curmap->get_pg_num(i->pool()),
+			nextmap->get_pg_num(i->pool()),
+			&split_pgs)) {
+	  start_split(*i, split_pgs);
+	  even_newer_pgs.insert(split_pgs.begin(), split_pgs.end());
+	}
+      }
+      new_pgs.insert(even_newer_pgs.begin(), even_newer_pgs.end());
+      curmap = nextmap;
+    }
+    assert(curmap == tomap); // we must have had both frommap and tomap
+  }
+}
+
+void OSDService::expand_pg_num(OSDMapRef old_map,
+			       OSDMapRef new_map)
+{
+  Mutex::Locker l(in_progress_split_lock);
+  for (set::iterator i = in_progress_splits.begin();
+       i != in_progress_splits.end();
+    ) {
+    if (!new_map->have_pg_pool(i->pool())) {
+      in_progress_splits.erase(i++);
+    } else {
+      _maybe_split_pgid(old_map, new_map, *i);
+      ++i;
+    }
+  }
+  for (map::iterator i = pending_splits.begin();
+       i != pending_splits.end();
+    ) {
+    if (!new_map->have_pg_pool(i->first.pool())) {
+      rev_pending_splits.erase(i->second);
+      pending_splits.erase(i++);
+    } else {
+      _maybe_split_pgid(old_map, new_map, i->first);
+      ++i;
+    }
+  }
+}
+
+bool OSDService::splitting(spg_t pgid)
+{
+  Mutex::Locker l(in_progress_split_lock);
+  return in_progress_splits.count(pgid) ||
+    pending_splits.count(pgid);
+}
+
+void OSDService::complete_split(const set &pgs)
+{
+  Mutex::Locker l(in_progress_split_lock);
+  for (set::const_iterator i = pgs.begin();
+       i != pgs.end();
+       ++i) {
+    dout(10) << __func__ << ": Completing split on pg " << *i << dendl;
+    assert(!pending_splits.count(*i));
+    assert(in_progress_splits.count(*i));
+    in_progress_splits.erase(*i);
+  }
+}
+
+void OSDService::need_heartbeat_peer_update()
+{
+  osd->need_heartbeat_peer_update();
+}
+
+void OSDService::pg_stat_queue_enqueue(PG *pg)
+{
+  osd->pg_stat_queue_enqueue(pg);
+}
+
+void OSDService::pg_stat_queue_dequeue(PG *pg)
+{
+  osd->pg_stat_queue_dequeue(pg);
+}
+
+void OSDService::start_shutdown()
+{
+  {
+    Mutex::Locker l(agent_timer_lock);
+    agent_timer.cancel_all_events();
+    agent_timer.shutdown();
+  }
+}
+
+void OSDService::shutdown()
+{
+  reserver_finisher.stop();
+  {
+    Mutex::Locker l(watch_lock);
+    watch_timer.shutdown();
+  }
+
+  {
+    Mutex::Locker l(objecter_lock);
+    objecter_timer.shutdown();
+    objecter->shutdown_locked();
+  }
+  objecter->shutdown_unlocked();
+  objecter_finisher.stop();
+
+  {
+    Mutex::Locker l(backfill_request_lock);
+    backfill_request_timer.shutdown();
+  }
+  osdmap = OSDMapRef();
+  next_osdmap = OSDMapRef();
+}
+
+void OSDService::init()
+{
+  reserver_finisher.start();
+  {
+    objecter_finisher.start();
+    objecter->init_unlocked();
+    Mutex::Locker l(objecter_lock);
+    objecter_timer.init();
+    objecter->set_client_incarnation(0);
+    objecter->init_locked();
+  }
+  watch_timer.init();
+  agent_timer.init();
+
+  agent_thread.create();
+}
+
+void OSDService::activate_map()
+{
+  // wake/unwake the tiering agent
+  agent_lock.Lock();
+  agent_active =
+    !osdmap->test_flag(CEPH_OSDMAP_NOTIERAGENT) &&
+    osd->is_active();
+  agent_cond.Signal();
+  agent_lock.Unlock();
+}
+
+class AgentTimeoutCB : public Context {
+  PGRef pg;
+public:
+  AgentTimeoutCB(PGRef _pg) : pg(_pg) {}
+  void finish(int) {
+    pg->agent_choose_mode_restart();
+  }
+};
+
+void OSDService::agent_entry()
+{
+  dout(10) << __func__ << " start" << dendl;
+  agent_lock.Lock();
+
+  while (!agent_stop_flag) {
+    if (agent_queue.empty()) {
+      dout(20) << __func__ << " empty queue" << dendl;
+      agent_cond.Wait(agent_lock);
+      continue;
+    }
+    uint64_t level = agent_queue.rbegin()->first;
+    set& top = agent_queue.rbegin()->second;
+    dout(10) << __func__
+	     << " tiers " << agent_queue.size()
+	     << ", top is " << level
+	     << " with pgs " << top.size()
+	     << ", ops " << agent_ops << "/"
+	     << g_conf->osd_agent_max_ops
+	     << (agent_active ? " active" : " NOT ACTIVE")
+	     << dendl;
+    dout(20) << __func__ << " oids " << agent_oids << dendl;
+    if (agent_ops >= g_conf->osd_agent_max_ops || top.empty() ||
+	!agent_active) {
+      agent_cond.Wait(agent_lock);
+      continue;
+    }
+
+    if (!agent_valid_iterator || agent_queue_pos == top.end()) {
+      agent_queue_pos = top.begin();
+      agent_valid_iterator = true;
+    }
+    PGRef pg = *agent_queue_pos;
+    int max = g_conf->osd_agent_max_ops - agent_ops;
+    agent_lock.Unlock();
+    if (!pg->agent_work(max)) {
+      dout(10) << __func__ << " " << *pg
+	<< " no agent_work, delay for " << g_conf->osd_agent_delay_time
+	<< " seconds" << dendl;
+
+      osd->logger->inc(l_osd_tier_delay);
+      // Queue a timer to call agent_choose_mode for this pg in 5 seconds
+      agent_timer_lock.Lock();
+      Context *cb = new AgentTimeoutCB(pg);
+      agent_timer.add_event_after(g_conf->osd_agent_delay_time, cb);
+      agent_timer_lock.Unlock();
+    }
+    agent_lock.Lock();
+  }
+  agent_lock.Unlock();
+  dout(10) << __func__ << " finish" << dendl;
+}
+
+void OSDService::agent_stop()
+{
+  {
+    Mutex::Locker l(agent_lock);
+
+    // By this time all ops should be cancelled
+    assert(agent_ops == 0);
+    // By this time all PGs are shutdown and dequeued
+    if (!agent_queue.empty()) {
+      set& top = agent_queue.rbegin()->second;
+      derr << "agent queue not empty, for example " << (*top.begin())->info.pgid << dendl;
+      assert(0 == "agent queue not empty");
+    }
+
+    agent_stop_flag = true;
+    agent_cond.Signal();
+  }
+  agent_thread.join();
+}
+
+
+#undef dout_prefix
+#define dout_prefix *_dout
+
+int OSD::convert_collection(ObjectStore *store, coll_t cid)
+{
+  coll_t tmp0("convertfs_temp");
+  coll_t tmp1("convertfs_temp1");
+  vector objects;
+
+  map aset;
+  int r = store->collection_getattrs(cid, aset);
+  if (r < 0)
+    return r;
+
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(tmp0);
+    for (map::iterator i = aset.begin();
+	 i != aset.end();
+	 ++i) {
+      bufferlist val;
+      val.push_back(i->second);
+      t.collection_setattr(tmp0, i->first, val);
+    }
+    store->apply_transaction(t);
+  }
+
+  ghobject_t next;
+  while (!next.is_max()) {
+    objects.clear();
+    ghobject_t start = next;
+    r = store->collection_list_partial(cid, start,
+				       200, 300, 0,
+				       &objects, &next);
+    if (r < 0)
+      return r;
+
+    ObjectStore::Transaction t;
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i) {
+      t.collection_add(tmp0, cid, *i);
+    }
+    store->apply_transaction(t);
+  }
+
+  {
+    ObjectStore::Transaction t;
+    t.collection_rename(cid, tmp1);
+    t.collection_rename(tmp0, cid);
+    store->apply_transaction(t);
+  }
+
+  recursive_remove_collection(store, tmp1);
+  store->sync_and_flush();
+  store->sync();
+  return 0;
+}
+
+int OSD::do_convertfs(ObjectStore *store)
+{
+  int r = store->mount();
+  if (r < 0)
+    return r;
+
+  uint32_t version;
+  r = store->version_stamp_is_valid(&version);
+  if (r < 0)
+    return r;
+  if (r == 1)
+    return store->umount();
+
+  derr << "ObjectStore is old at version " << version << ".  Updating..."  << dendl;
+
+  derr << "Removing tmp pgs" << dendl;
+  vector collections;
+  r = store->list_collections(collections);
+  if (r < 0)
+    return r;
+  for (vector::iterator i = collections.begin();
+       i != collections.end();
+       ++i) {
+    spg_t pgid;
+    if (i->is_temp(pgid))
+      recursive_remove_collection(store, *i);
+    else if (i->to_str() == "convertfs_temp" ||
+	     i->to_str() == "convertfs_temp1")
+      recursive_remove_collection(store, *i);
+  }
+  store->flush();
+
+
+  derr << "Getting collections" << dendl;
+
+  derr << collections.size() << " to process." << dendl;
+  collections.clear();
+  r = store->list_collections(collections);
+  if (r < 0)
+    return r;
+  int processed = 0;
+  for (vector::iterator i = collections.begin();
+       i != collections.end();
+       ++i, ++processed) {
+    derr << processed << "/" << collections.size() << " processed" << dendl;
+    uint32_t collection_version;
+    r = store->collection_version_current(*i, &collection_version);
+    if (r < 0) {
+      return r;
+    } else if (r == 1) {
+      derr << "Collection " << *i << " is up to date" << dendl;
+    } else {
+      derr << "Updating collection " << *i << " current version is " 
+	   << collection_version << dendl;
+      r = convert_collection(store, *i);
+      if (r < 0)
+	return r;
+      derr << "collection " << *i << " updated" << dendl;
+    }
+  }
+  derr << "All collections up to date, updating version stamp..." << dendl;
+  r = store->update_version_stamp();
+  if (r < 0)
+    return r;
+  store->sync_and_flush();
+  store->sync();
+  derr << "Version stamp updated, done with upgrade!" << dendl;
+  return store->umount();
+}
+
+int OSD::mkfs(CephContext *cct, ObjectStore *store, const string &dev,
+	      uuid_d fsid, int whoami)
+{
+  int ret;
+
+  try {
+    // if we are fed a uuid for this osd, use it.
+    store->set_fsid(cct->_conf->osd_uuid);
+
+    ret = store->mkfs();
+    if (ret) {
+      derr << "OSD::mkfs: ObjectStore::mkfs failed with error " << ret << dendl;
+      goto free_store;
+    }
+
+    ret = store->mount();
+    if (ret) {
+      derr << "OSD::mkfs: couldn't mount ObjectStore: error " << ret << dendl;
+      goto free_store;
+    }
+
+    // age?
+    if (cct->_conf->osd_age_time != 0) {
+      if (cct->_conf->osd_age_time >= 0) {
+        dout(0) << "aging..." << dendl;
+        Ager ager(cct, store);
+        ager.age(cct->_conf->osd_age_time,
+          cct->_conf->osd_age,
+          cct->_conf->osd_age - .05,
+          50000,
+          cct->_conf->osd_age - .05);
+      }
+    }
+
+    OSDSuperblock sb;
+    bufferlist sbbl;
+    ret = store->read(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, 0, sbbl);
+    if (ret >= 0) {
+      dout(0) << " have superblock" << dendl;
+      if (whoami != sb.whoami) {
+	derr << "provided osd id " << whoami << " != superblock's " << sb.whoami << dendl;
+	ret = -EINVAL;
+	goto umount_store;
+      }
+      if (fsid != sb.cluster_fsid) {
+	derr << "provided cluster fsid " << fsid << " != superblock's " << sb.cluster_fsid << dendl;
+	ret = -EINVAL;
+	goto umount_store;
+      }
+    } else {
+      // create superblock
+      if (fsid.is_zero()) {
+	derr << "must specify cluster fsid" << dendl;
+	ret = -EINVAL;
+	goto umount_store;
+      }
+
+      sb.cluster_fsid = fsid;
+      sb.osd_fsid = store->get_fsid();
+      sb.whoami = whoami;
+      sb.compat_features = get_osd_initial_compat_set();
+
+      // benchmark?
+      if (cct->_conf->osd_auto_weight) {
+	bufferlist bl;
+	bufferptr bp(1048576);
+	bp.zero();
+	bl.push_back(bp);
+	dout(0) << "testing disk bandwidth..." << dendl;
+	utime_t start = ceph_clock_now(cct);
+	object_t oid("disk_bw_test");
+	for (int i=0; i<1000; i++) {
+	  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+	  t->write(coll_t::META_COLL, hobject_t(sobject_t(oid, 0)), i*bl.length(), bl.length(), bl);
+	  store->queue_transaction_and_cleanup(NULL, t);
+	}
+	store->sync();
+	utime_t end = ceph_clock_now(cct);
+	end -= start;
+	dout(0) << "measured " << (1000.0 / (double)end) << " mb/sec" << dendl;
+	ObjectStore::Transaction tr;
+	tr.remove(coll_t::META_COLL, hobject_t(sobject_t(oid, 0)));
+	ret = store->apply_transaction(tr);
+	if (ret) {
+	  derr << "OSD::mkfs: error while benchmarking: apply_transaction returned "
+	       << ret << dendl;
+	  goto umount_store;
+	}
+	
+	// set osd weight
+	sb.weight = (1000.0 / (double)end);
+      }
+
+      bufferlist bl;
+      ::encode(sb, bl);
+
+      ObjectStore::Transaction t;
+      t.create_collection(coll_t::META_COLL);
+      t.write(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, bl.length(), bl);
+      ret = store->apply_transaction(t);
+      if (ret) {
+	derr << "OSD::mkfs: error while writing OSD_SUPERBLOCK_POBJECT: "
+	     << "apply_transaction returned " << ret << dendl;
+	goto umount_store;
+      }
+    }
+
+    store->sync_and_flush();
+
+    ret = write_meta(store, sb.cluster_fsid, sb.osd_fsid, whoami);
+    if (ret) {
+      derr << "OSD::mkfs: failed to write fsid file: error " << ret << dendl;
+      goto umount_store;
+    }
+
+  }
+  catch (const std::exception &se) {
+    derr << "OSD::mkfs: caught exception " << se.what() << dendl;
+    ret = 1000;
+  }
+  catch (...) {
+    derr << "OSD::mkfs: caught unknown exception." << dendl;
+    ret = 1000;
+  }
+
+umount_store:
+  store->umount();
+free_store:
+  delete store;
+  return ret;
+}
+
+int OSD::write_meta(ObjectStore *store, uuid_d& cluster_fsid, uuid_d& osd_fsid, int whoami)
+{
+  char val[80];
+  int r;
+  
+  snprintf(val, sizeof(val), "%s", CEPH_OSD_ONDISK_MAGIC);
+  r = store->write_meta("magic", val);
+  if (r < 0)
+    return r;
+
+  snprintf(val, sizeof(val), "%d", whoami);
+  r = store->write_meta("whoami", val);
+  if (r < 0)
+    return r;
+
+  cluster_fsid.print(val);
+  r = store->write_meta("ceph_fsid", val);
+  if (r < 0)
+    return r;
+
+  r = store->write_meta("ready", "ready");
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int OSD::peek_meta(ObjectStore *store, std::string& magic,
+		   uuid_d& cluster_fsid, uuid_d& osd_fsid, int& whoami)
+{
+  string val;
+
+  int r = store->read_meta("magic", &val);
+  if (r < 0)
+    return r;
+  magic = val;
+
+  r = store->read_meta("whoami", &val);
+  if (r < 0)
+    return r;
+  whoami = atoi(val.c_str());
+
+  r = store->read_meta("ceph_fsid", &val);
+  if (r < 0)
+    return r;
+  r = cluster_fsid.parse(val.c_str());
+  if (r < 0)
+    return r;
+
+  r = store->read_meta("fsid", &val);
+  if (r < 0) {
+    osd_fsid = uuid_d();
+  } else {
+    r = osd_fsid.parse(val.c_str());
+    if (r < 0)
+      return r;
+  }
+
+  return 0;
+}
+
+
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, whoami, osdmap)
+
+// cons/des
+
+OSD::OSD(CephContext *cct_, ObjectStore *store_,
+	 int id, Messenger *internal_messenger, Messenger *external_messenger,
+	 Messenger *hb_clientm,
+	 Messenger *hb_front_serverm,
+	 Messenger *hb_back_serverm,
+	 Messenger *osdc_messenger,
+	 MonClient *mc,
+	 const std::string &dev, const std::string &jdev) :
+  Dispatcher(cct_),
+  osd_lock("OSD::osd_lock"),
+  tick_timer(cct, osd_lock),
+  authorize_handler_cluster_registry(new AuthAuthorizeHandlerRegistry(cct,
+								      cct->_conf->auth_supported.length() ?
+								      cct->_conf->auth_supported :
+								      cct->_conf->auth_cluster_required)),
+  authorize_handler_service_registry(new AuthAuthorizeHandlerRegistry(cct,
+								      cct->_conf->auth_supported.length() ?
+								      cct->_conf->auth_supported :
+								      cct->_conf->auth_service_required)),
+  cluster_messenger(internal_messenger),
+  client_messenger(external_messenger),
+  objecter_messenger(osdc_messenger),
+  monc(mc),
+  logger(NULL),
+  recoverystate_perf(NULL),
+  store(store_),
+  clog(cct, client_messenger, &mc->monmap, LogClient::NO_FLAGS),
+  whoami(id),
+  dev_path(dev), journal_path(jdev),
+  dispatch_running(false),
+  asok_hook(NULL),
+  osd_compat(get_osd_compat_set()),
+  state(STATE_INITIALIZING), boot_epoch(0), up_epoch(0), bind_epoch(0),
+  op_tp(cct, "OSD::op_tp", cct->_conf->osd_op_threads, "osd_op_threads"),
+  recovery_tp(cct, "OSD::recovery_tp", cct->_conf->osd_recovery_threads, "osd_recovery_threads"),
+  disk_tp(cct, "OSD::disk_tp", cct->_conf->osd_disk_threads, "osd_disk_threads"),
+  command_tp(cct, "OSD::command_tp", 1),
+  paused_recovery(false),
+  heartbeat_lock("OSD::heartbeat_lock"),
+  heartbeat_stop(false), heartbeat_need_update(true), heartbeat_epoch(0),
+  hbclient_messenger(hb_clientm),
+  hb_front_server_messenger(hb_front_serverm),
+  hb_back_server_messenger(hb_back_serverm),
+  heartbeat_thread(this),
+  heartbeat_dispatcher(this),
+  stat_lock("OSD::stat_lock"),
+  finished_lock("OSD::finished_lock"),
+  op_tracker(cct, cct->_conf->osd_enable_op_tracker),
+  test_ops_hook(NULL),
+  op_wq(this, cct->_conf->osd_op_thread_timeout, &op_tp),
+  peering_wq(this, cct->_conf->osd_op_thread_timeout, &op_tp),
+  map_lock("OSD::map_lock"),
+  peer_map_epoch_lock("OSD::peer_map_epoch_lock"),
+  debug_drop_pg_create_probability(cct->_conf->osd_debug_drop_pg_create_probability),
+  debug_drop_pg_create_duration(cct->_conf->osd_debug_drop_pg_create_duration),
+  debug_drop_pg_create_left(-1),
+  outstanding_pg_stats(false),
+  timeout_mon_on_pg_stats(true),
+  up_thru_wanted(0), up_thru_pending(0),
+  pg_stat_queue_lock("OSD::pg_stat_queue_lock"),
+  osd_stat_updated(false),
+  pg_stat_tid(0), pg_stat_tid_flushed(0),
+  command_wq(this, cct->_conf->osd_command_thread_timeout, &command_tp),
+  recovery_ops_active(0),
+  recovery_wq(this, cct->_conf->osd_recovery_thread_timeout, &recovery_tp),
+  replay_queue_lock("OSD::replay_queue_lock"),
+  snap_trim_wq(this, cct->_conf->osd_snap_trim_thread_timeout, &disk_tp),
+  scrub_wq(this, cct->_conf->osd_scrub_thread_timeout, &disk_tp),
+  scrub_finalize_wq(cct->_conf->osd_scrub_finalize_thread_timeout, &op_tp),
+  rep_scrub_wq(this, cct->_conf->osd_scrub_thread_timeout, &disk_tp),
+  remove_wq(store, cct->_conf->osd_remove_thread_timeout, &disk_tp),
+  next_removal_seq(0),
+  service(this)
+{
+  monc->set_messenger(client_messenger);
+  op_tracker.set_complaint_and_threshold(cct->_conf->osd_op_complaint_time,
+                                         cct->_conf->osd_op_log_threshold);
+  op_tracker.set_history_size_and_duration(cct->_conf->osd_op_history_size,
+                                           cct->_conf->osd_op_history_duration);
+}
+
+OSD::~OSD()
+{
+  delete authorize_handler_cluster_registry;
+  delete authorize_handler_service_registry;
+  delete class_handler;
+  cct->get_perfcounters_collection()->remove(recoverystate_perf);
+  cct->get_perfcounters_collection()->remove(logger);
+  delete recoverystate_perf;
+  delete logger;
+  delete store;
+}
+
+void cls_initialize(ClassHandler *ch);
+
+void OSD::handle_signal(int signum)
+{
+  assert(signum == SIGINT || signum == SIGTERM);
+  derr << "*** Got signal " << sys_siglist[signum] << " ***" << dendl;
+  //suicide(128 + signum);
+  shutdown();
+}
+
+int OSD::pre_init()
+{
+  Mutex::Locker lock(osd_lock);
+  if (is_stopping())
+    return 0;
+  
+  if (store->test_mount_in_use()) {
+    derr << "OSD::pre_init: object store '" << dev_path << "' is "
+         << "currently in use. (Is ceph-osd already running?)" << dendl;
+    return -EBUSY;
+  }
+
+  cct->_conf->add_observer(this);
+  return 0;
+}
+
+// asok
+
+class OSDSocketHook : public AdminSocketHook {
+  OSD *osd;
+public:
+  OSDSocketHook(OSD *o) : osd(o) {}
+  bool call(std::string command, cmdmap_t& cmdmap, std::string format,
+	    bufferlist& out) {
+    stringstream ss;
+    bool r = osd->asok_command(command, cmdmap, format, ss);
+    out.append(ss);
+    return r;
+  }
+};
+
+bool OSD::asok_command(string command, cmdmap_t& cmdmap, string format,
+		       ostream& ss)
+{
+  Formatter *f = new_formatter(format);
+  if (!f)
+    f = new_formatter("json-pretty");
+  if (command == "status") {
+    f->open_object_section("status");
+    f->dump_stream("cluster_fsid") << superblock.cluster_fsid;
+    f->dump_stream("osd_fsid") << superblock.osd_fsid;
+    f->dump_unsigned("whoami", superblock.whoami);
+    f->dump_string("state", get_state_name(state));
+    f->dump_unsigned("oldest_map", superblock.oldest_map);
+    f->dump_unsigned("newest_map", superblock.newest_map);
+    osd_lock.Lock();
+    f->dump_unsigned("num_pgs", pg_map.size());
+    osd_lock.Unlock();
+    f->close_section();
+  } else if (command == "flush_journal") {
+    store->sync_and_flush();
+  } else if (command == "dump_ops_in_flight") {
+    op_tracker.dump_ops_in_flight(f);
+  } else if (command == "dump_historic_ops") {
+    op_tracker.dump_historic_ops(f);
+  } else if (command == "dump_op_pq_state") {
+    f->open_object_section("pq");
+    op_wq.dump(f);
+    f->close_section();
+  } else if (command == "dump_blacklist") {
+    list > bl;
+    OSDMapRef curmap = service.get_osdmap();
+
+    f->open_array_section("blacklist");
+    curmap->get_blacklist(&bl);
+    for (list >::iterator it = bl.begin();
+	it != bl.end(); ++it) {
+      f->open_array_section("entry");
+      f->open_object_section("entity_addr_t");
+      it->first.dump(f);
+      f->close_section(); //entity_addr_t
+      it->second.localtime(f->dump_stream("expire_time"));
+      f->close_section(); //entry
+    }
+    f->close_section(); //blacklist
+  } else if (command == "dump_watchers") {
+    list watchers;
+    osd_lock.Lock();
+    // scan pg's
+    for (ceph::unordered_map::iterator it = pg_map.begin();
+	 it != pg_map.end();
+	 ++it) {
+
+      list pg_watchers;
+      PG *pg = it->second;
+      pg->lock();
+      pg->get_watchers(pg_watchers);
+      pg->unlock();
+      watchers.splice(watchers.end(), pg_watchers);
+    }
+    osd_lock.Unlock();
+
+    f->open_array_section("watchers");
+    for (list::iterator it = watchers.begin();
+	it != watchers.end(); ++it) {
+
+      f->open_array_section("watch");
+
+      f->dump_string("namespace", it->obj.nspace);
+      f->dump_string("object", it->obj.oid.name);
+
+      f->open_object_section("entity_name");
+      it->wi.name.dump(f);
+      f->close_section(); //entity_name_t
+
+      f->dump_int("cookie", it->wi.cookie);
+      f->dump_int("timeout", it->wi.timeout_seconds);
+
+      f->open_object_section("entity_addr_t");
+      it->wi.addr.dump(f);
+      f->close_section(); //entity_addr_t
+
+      f->close_section(); //watch
+    }
+
+    f->close_section(); //watches
+  } else {
+    assert(0 == "broken asok registration");
+  }
+  f->flush(ss);
+  delete f;
+  return true;
+}
+
+class TestOpsSocketHook : public AdminSocketHook {
+  OSDService *service;
+  ObjectStore *store;
+public:
+  TestOpsSocketHook(OSDService *s, ObjectStore *st) : service(s), store(st) {}
+  bool call(std::string command, cmdmap_t& cmdmap, std::string format,
+	    bufferlist& out) {
+    stringstream ss;
+    test_ops(service, store, command, cmdmap, ss);
+    out.append(ss);
+    return true;
+  }
+  void test_ops(OSDService *service, ObjectStore *store, std::string command,
+     cmdmap_t& cmdmap, ostream &ss);
+
+};
+
+int OSD::init()
+{
+  CompatSet initial, diff;
+  Mutex::Locker lock(osd_lock);
+  if (is_stopping())
+    return 0;
+
+  tick_timer.init();
+  service.backfill_request_timer.init();
+
+  // mount.
+  dout(2) << "mounting " << dev_path << " "
+	  << (journal_path.empty() ? "(no journal)" : journal_path) << dendl;
+  assert(store);  // call pre_init() first!
+
+  int r = store->mount();
+  if (r < 0) {
+    derr << "OSD:init: unable to mount object store" << dendl;
+    return r;
+  }
+
+  dout(2) << "boot" << dendl;
+
+  // read superblock
+  r = read_superblock();
+  if (r < 0) {
+    derr << "OSD::init() : unable to read osd superblock" << dendl;
+    r = -EINVAL;
+    goto out;
+  }
+
+  if (osd_compat.compare(superblock.compat_features) < 0) {
+    derr << "The disk uses features unsupported by the executable." << dendl;
+    derr << " ondisk features " << superblock.compat_features << dendl;
+    derr << " daemon features " << osd_compat << dendl;
+
+    if (osd_compat.writeable(superblock.compat_features)) {
+      CompatSet diff = osd_compat.unsupported(superblock.compat_features);
+      derr << "it is still writeable, though. Missing features: " << diff << dendl;
+      r = -EOPNOTSUPP;
+      goto out;
+    }
+    else {
+      CompatSet diff = osd_compat.unsupported(superblock.compat_features);
+      derr << "Cannot write to disk! Missing features: " << diff << dendl;
+      r = -EOPNOTSUPP;
+      goto out;
+    }
+  }
+
+  assert_warn(whoami == superblock.whoami);
+  if (whoami != superblock.whoami) {
+    derr << "OSD::init: superblock says osd"
+	 << superblock.whoami << " but i am osd." << whoami << dendl;
+    r = -EINVAL;
+    goto out;
+  }
+
+  initial = get_osd_initial_compat_set();
+  diff = superblock.compat_features.unsupported(initial);
+  if (superblock.compat_features.merge(initial)) {
+    // We need to persist the new compat_set before we
+    // do anything else
+    dout(5) << "Upgrading superblock adding: " << diff << dendl;
+    ObjectStore::Transaction t;
+    write_superblock(t);
+    r = store->apply_transaction(t);
+    if (r < 0)
+      goto out;
+  }
+
+  // make sure info object exists
+  if (!store->exists(coll_t::META_COLL, service.infos_oid)) {
+    dout(10) << "init creating/touching snapmapper object" << dendl;
+    ObjectStore::Transaction t;
+    t.touch(coll_t::META_COLL, service.infos_oid);
+    r = store->apply_transaction(t);
+    if (r < 0)
+      goto out;
+  }
+
+  // make sure snap mapper object exists
+  if (!store->exists(coll_t::META_COLL, OSD::make_snapmapper_oid())) {
+    dout(10) << "init creating/touching infos object" << dendl;
+    ObjectStore::Transaction t;
+    t.touch(coll_t::META_COLL, OSD::make_snapmapper_oid());
+    r = store->apply_transaction(t);
+    if (r < 0)
+      goto out;
+  }
+
+  class_handler = new ClassHandler(cct);
+  cls_initialize(class_handler);
+
+  if (cct->_conf->osd_open_classes_on_start) {
+    int r = class_handler->open_all_classes();
+    if (r)
+      dout(1) << "warning: got an error loading one or more classes: " << cpp_strerror(r) << dendl;
+  }
+
+  // load up "current" osdmap
+  assert_warn(!osdmap);
+  if (osdmap) {
+    derr << "OSD::init: unable to read current osdmap" << dendl;
+    r = -EINVAL;
+    goto out;
+  }
+  osdmap = get_map(superblock.current_epoch);
+  check_osdmap_features(store);
+
+  create_recoverystate_perf();
+
+  bind_epoch = osdmap->get_epoch();
+
+  // load up pgs (as they previously existed)
+  load_pgs();
+
+  dout(2) << "superblock: i am osd." << superblock.whoami << dendl;
+
+  create_logger();
+    
+  // i'm ready!
+  client_messenger->add_dispatcher_head(this);
+  cluster_messenger->add_dispatcher_head(this);
+
+  hbclient_messenger->add_dispatcher_head(&heartbeat_dispatcher);
+  hb_front_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);
+  hb_back_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);
+
+  objecter_messenger->add_dispatcher_head(&service.objecter_dispatcher);
+
+  monc->set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD);
+  r = monc->init();
+  if (r < 0)
+    goto out;
+
+  // tell monc about log_client so it will know about mon session resets
+  monc->set_log_client(&clog);
+
+  op_tp.start();
+  recovery_tp.start();
+  disk_tp.start();
+  command_tp.start();
+
+  set_disk_tp_priority();
+
+  // start the heartbeat
+  heartbeat_thread.create();
+
+  // tick
+  tick_timer.add_event_after(cct->_conf->osd_heartbeat_interval, new C_Tick(this));
+
+  service.init();
+  service.publish_map(osdmap);
+  service.publish_superblock(superblock);
+
+  osd_lock.Unlock();
+
+  r = monc->authenticate();
+  if (r < 0) {
+    osd_lock.Lock(); // locker is going to unlock this on function exit
+    if (is_stopping())
+      r =  0;
+    goto monout;
+  }
+
+  while (monc->wait_auth_rotating(30.0) < 0) {
+    derr << "unable to obtain rotating service keys; retrying" << dendl;
+  }
+
+  osd_lock.Lock();
+  if (is_stopping())
+    return 0;
+
+  check_config();
+
+  dout(10) << "ensuring pgs have consumed prior maps" << dendl;
+  consume_map();
+  peering_wq.drain();
+
+  dout(0) << "done with init, starting boot process" << dendl;
+  state = STATE_BOOTING;
+  start_boot();
+
+  return 0;
+monout:
+  monc->shutdown();
+
+out:
+  store->umount();
+  delete store;
+  return r;
+}
+
+void OSD::final_init()
+{
+  int r;
+  AdminSocket *admin_socket = cct->get_admin_socket();
+  asok_hook = new OSDSocketHook(this);
+  r = admin_socket->register_command("status", "status", asok_hook,
+				     "high-level status of OSD");
+  assert(r == 0);
+  r = admin_socket->register_command("flush_journal", "flush_journal",
+                                     asok_hook,
+                                     "flush the journal to permanent store");
+  assert(r == 0);
+  r = admin_socket->register_command("dump_ops_in_flight",
+				     "dump_ops_in_flight", asok_hook,
+				     "show the ops currently in flight");
+  assert(r == 0);
+  r = admin_socket->register_command("dump_historic_ops", "dump_historic_ops",
+				     asok_hook,
+				     "show slowest recent ops");
+  assert(r == 0);
+  r = admin_socket->register_command("dump_op_pq_state", "dump_op_pq_state",
+				     asok_hook,
+				     "dump op priority queue state");
+  assert(r == 0);
+  r = admin_socket->register_command("dump_blacklist", "dump_blacklist",
+				     asok_hook,
+				     "dump blacklisted clients and times");
+  assert(r == 0);
+  r = admin_socket->register_command("dump_watchers", "dump_watchers",
+				     asok_hook,
+				     "show clients which have active watches,"
+				     " and on which objects");
+  assert(r == 0);
+
+  test_ops_hook = new TestOpsSocketHook(&(this->service), this->store);
+  // Note: pools are CephString instead of CephPoolname because
+  // these commands traditionally support both pool names and numbers
+  r = admin_socket->register_command(
+   "setomapval",
+   "setomapval " \
+   "name=pool,type=CephString " \
+   "name=objname,type=CephObjectname " \
+   "name=key,type=CephString "\
+   "name=val,type=CephString",
+   test_ops_hook,
+   "set omap key");
+  assert(r == 0);
+  r = admin_socket->register_command(
+    "rmomapkey",
+    "rmomapkey " \
+    "name=pool,type=CephString " \
+    "name=objname,type=CephObjectname " \
+    "name=key,type=CephString",
+    test_ops_hook,
+    "remove omap key");
+  assert(r == 0);
+  r = admin_socket->register_command(
+    "setomapheader",
+    "setomapheader " \
+    "name=pool,type=CephString " \
+    "name=objname,type=CephObjectname " \
+    "name=header,type=CephString",
+    test_ops_hook,
+    "set omap header");
+  assert(r == 0);
+
+  r = admin_socket->register_command(
+    "getomap",
+    "getomap " \
+    "name=pool,type=CephString " \
+    "name=objname,type=CephObjectname",
+    test_ops_hook,
+    "output entire object map");
+  assert(r == 0);
+
+  r = admin_socket->register_command(
+    "truncobj",
+    "truncobj " \
+    "name=pool,type=CephString " \
+    "name=objname,type=CephObjectname " \
+    "name=len,type=CephInt",
+    test_ops_hook,
+    "truncate object to length");
+  assert(r == 0);
+
+  r = admin_socket->register_command(
+    "injectdataerr",
+    "injectdataerr " \
+    "name=pool,type=CephString " \
+    "name=objname,type=CephObjectname",
+    test_ops_hook,
+    "inject data error into omap");
+  assert(r == 0);
+
+  r = admin_socket->register_command(
+    "injectmdataerr",
+    "injectmdataerr " \
+    "name=pool,type=CephString " \
+    "name=objname,type=CephObjectname",
+    test_ops_hook,
+    "inject metadata error");
+  assert(r == 0);
+}
+
+void OSD::create_logger()
+{
+  dout(10) << "create_logger" << dendl;
+
+  PerfCountersBuilder osd_plb(cct, "osd", l_osd_first, l_osd_last);
+
+  osd_plb.add_u64(l_osd_opq, "opq");       // op queue length (waiting to be processed yet)
+  osd_plb.add_u64(l_osd_op_wip, "op_wip");   // rep ops currently being processed (primary)
+
+  osd_plb.add_u64_counter(l_osd_op,       "op");           // client ops
+  osd_plb.add_u64_counter(l_osd_op_inb,   "op_in_bytes");       // client op in bytes (writes)
+  osd_plb.add_u64_counter(l_osd_op_outb,  "op_out_bytes");      // client op out bytes (reads)
+  osd_plb.add_time_avg(l_osd_op_lat,   "op_latency");       // client op latency
+  osd_plb.add_time_avg(l_osd_op_process_lat, "op_process_latency");   // client op process latency
+
+  osd_plb.add_u64_counter(l_osd_op_r,      "op_r");        // client reads
+  osd_plb.add_u64_counter(l_osd_op_r_outb, "op_r_out_bytes");   // client read out bytes
+  osd_plb.add_time_avg(l_osd_op_r_lat,  "op_r_latency");    // client read latency
+  osd_plb.add_time_avg(l_osd_op_r_process_lat, "op_r_process_latency");   // client read process latency
+  osd_plb.add_u64_counter(l_osd_op_w,      "op_w");        // client writes
+  osd_plb.add_u64_counter(l_osd_op_w_inb,  "op_w_in_bytes");    // client write in bytes
+  osd_plb.add_time_avg(l_osd_op_w_rlat, "op_w_rlat");   // client write readable/applied latency
+  osd_plb.add_time_avg(l_osd_op_w_lat,  "op_w_latency");    // client write latency
+  osd_plb.add_time_avg(l_osd_op_w_process_lat, "op_w_process_latency");   // client write process latency
+  osd_plb.add_u64_counter(l_osd_op_rw,     "op_rw");       // client rmw
+  osd_plb.add_u64_counter(l_osd_op_rw_inb, "op_rw_in_bytes");   // client rmw in bytes
+  osd_plb.add_u64_counter(l_osd_op_rw_outb,"op_rw_out_bytes");  // client rmw out bytes
+  osd_plb.add_time_avg(l_osd_op_rw_rlat,"op_rw_rlat");  // client rmw readable/applied latency
+  osd_plb.add_time_avg(l_osd_op_rw_lat, "op_rw_latency");   // client rmw latency
+  osd_plb.add_time_avg(l_osd_op_rw_process_lat, "op_rw_process_latency");   // client rmw process latency
+
+  osd_plb.add_u64_counter(l_osd_sop,       "subop");         // subops
+  osd_plb.add_u64_counter(l_osd_sop_inb,   "subop_in_bytes");     // subop in bytes
+  osd_plb.add_time_avg(l_osd_sop_lat,   "subop_latency");     // subop latency
+
+  osd_plb.add_u64_counter(l_osd_sop_w,     "subop_w");          // replicated (client) writes
+  osd_plb.add_u64_counter(l_osd_sop_w_inb, "subop_w_in_bytes");      // replicated write in bytes
+  osd_plb.add_time_avg(l_osd_sop_w_lat, "subop_w_latency");      // replicated write latency
+  osd_plb.add_u64_counter(l_osd_sop_pull,     "subop_pull");       // pull request
+  osd_plb.add_time_avg(l_osd_sop_pull_lat, "subop_pull_latency");
+  osd_plb.add_u64_counter(l_osd_sop_push,     "subop_push");       // push (write)
+  osd_plb.add_u64_counter(l_osd_sop_push_inb, "subop_push_in_bytes");
+  osd_plb.add_time_avg(l_osd_sop_push_lat, "subop_push_latency");
+
+  osd_plb.add_u64_counter(l_osd_pull,      "pull");       // pull requests sent
+  osd_plb.add_u64_counter(l_osd_push,      "push");       // push messages
+  osd_plb.add_u64_counter(l_osd_push_outb, "push_out_bytes");  // pushed bytes
+
+  osd_plb.add_u64_counter(l_osd_push_in,    "push_in");        // inbound push messages
+  osd_plb.add_u64_counter(l_osd_push_inb,   "push_in_bytes");  // inbound pushed bytes
+
+  osd_plb.add_u64_counter(l_osd_rop, "recovery_ops");       // recovery ops (started)
+
+  osd_plb.add_u64(l_osd_loadavg, "loadavg");
+  osd_plb.add_u64(l_osd_buf, "buffer_bytes");       // total ceph::buffer bytes
+
+  osd_plb.add_u64(l_osd_pg, "numpg");   // num pgs
+  osd_plb.add_u64(l_osd_pg_primary, "numpg_primary"); // num primary pgs
+  osd_plb.add_u64(l_osd_pg_replica, "numpg_replica"); // num replica pgs
+  osd_plb.add_u64(l_osd_pg_stray, "numpg_stray");   // num stray pgs
+  osd_plb.add_u64(l_osd_hb_to, "heartbeat_to_peers");     // heartbeat peers we send to
+  osd_plb.add_u64(l_osd_hb_from, "heartbeat_from_peers"); // heartbeat peers we recv from
+  osd_plb.add_u64_counter(l_osd_map, "map_messages");           // osdmap messages
+  osd_plb.add_u64_counter(l_osd_mape, "map_message_epochs");         // osdmap epochs
+  osd_plb.add_u64_counter(l_osd_mape_dup, "map_message_epoch_dups"); // dup osdmap epochs
+  osd_plb.add_u64_counter(l_osd_waiting_for_map,
+			  "messages_delayed_for_map"); // dup osdmap epochs
+
+  osd_plb.add_u64(l_osd_stat_bytes, "stat_bytes");
+  osd_plb.add_u64(l_osd_stat_bytes_used, "stat_bytes_used");
+  osd_plb.add_u64(l_osd_stat_bytes_avail, "stat_bytes_avail");
+
+  osd_plb.add_u64_counter(l_osd_copyfrom, "copyfrom");
+
+  osd_plb.add_u64_counter(l_osd_tier_promote, "tier_promote");
+  osd_plb.add_u64_counter(l_osd_tier_flush, "tier_flush");
+  osd_plb.add_u64_counter(l_osd_tier_flush_fail, "tier_flush_fail");
+  osd_plb.add_u64_counter(l_osd_tier_try_flush, "tier_try_flush");
+  osd_plb.add_u64_counter(l_osd_tier_try_flush_fail, "tier_try_flush_fail");
+  osd_plb.add_u64_counter(l_osd_tier_evict, "tier_evict");
+  osd_plb.add_u64_counter(l_osd_tier_whiteout, "tier_whiteout");
+  osd_plb.add_u64_counter(l_osd_tier_dirty, "tier_dirty");
+  osd_plb.add_u64_counter(l_osd_tier_clean, "tier_clean");
+  osd_plb.add_u64_counter(l_osd_tier_delay, "tier_delay");
+
+  osd_plb.add_u64_counter(l_osd_agent_wake, "agent_wake");
+  osd_plb.add_u64_counter(l_osd_agent_skip, "agent_skip");
+  osd_plb.add_u64_counter(l_osd_agent_flush, "agent_flush");
+  osd_plb.add_u64_counter(l_osd_agent_evict, "agent_evict");
+
+  logger = osd_plb.create_perf_counters();
+  cct->get_perfcounters_collection()->add(logger);
+}
+
+void OSD::create_recoverystate_perf()
+{
+  dout(10) << "create_recoverystate_perf" << dendl;
+
+  PerfCountersBuilder rs_perf(cct, "recoverystate_perf", rs_first, rs_last);
+
+  rs_perf.add_time_avg(rs_initial_latency, "initial_latency");
+  rs_perf.add_time_avg(rs_started_latency, "started_latency");
+  rs_perf.add_time_avg(rs_reset_latency, "reset_latency");
+  rs_perf.add_time_avg(rs_start_latency, "start_latency");
+  rs_perf.add_time_avg(rs_primary_latency, "primary_latency");
+  rs_perf.add_time_avg(rs_peering_latency, "peering_latency");
+  rs_perf.add_time_avg(rs_backfilling_latency, "backfilling_latency");
+  rs_perf.add_time_avg(rs_waitremotebackfillreserved_latency, "waitremotebackfillreserved_latency");
+  rs_perf.add_time_avg(rs_waitlocalbackfillreserved_latency, "waitlocalbackfillreserved_latency");
+  rs_perf.add_time_avg(rs_notbackfilling_latency, "notbackfilling_latency");
+  rs_perf.add_time_avg(rs_repnotrecovering_latency, "repnotrecovering_latency");
+  rs_perf.add_time_avg(rs_repwaitrecoveryreserved_latency, "repwaitrecoveryreserved_latency");
+  rs_perf.add_time_avg(rs_repwaitbackfillreserved_latency, "repwaitbackfillreserved_latency");
+  rs_perf.add_time_avg(rs_RepRecovering_latency, "RepRecovering_latency");
+  rs_perf.add_time_avg(rs_activating_latency, "activating_latency");
+  rs_perf.add_time_avg(rs_waitlocalrecoveryreserved_latency, "waitlocalrecoveryreserved_latency");
+  rs_perf.add_time_avg(rs_waitremoterecoveryreserved_latency, "waitremoterecoveryreserved_latency");
+  rs_perf.add_time_avg(rs_recovering_latency, "recovering_latency");
+  rs_perf.add_time_avg(rs_recovered_latency, "recovered_latency");
+  rs_perf.add_time_avg(rs_clean_latency, "clean_latency");
+  rs_perf.add_time_avg(rs_active_latency, "active_latency");
+  rs_perf.add_time_avg(rs_replicaactive_latency, "replicaactive_latency");
+  rs_perf.add_time_avg(rs_stray_latency, "stray_latency");
+  rs_perf.add_time_avg(rs_getinfo_latency, "getinfo_latency");
+  rs_perf.add_time_avg(rs_getlog_latency, "getlog_latency");
+  rs_perf.add_time_avg(rs_waitactingchange_latency, "waitactingchange_latency");
+  rs_perf.add_time_avg(rs_incomplete_latency, "incomplete_latency");
+  rs_perf.add_time_avg(rs_getmissing_latency, "getmissing_latency");
+  rs_perf.add_time_avg(rs_waitupthru_latency, "waitupthru_latency");
+
+  recoverystate_perf = rs_perf.create_perf_counters();
+  cct->get_perfcounters_collection()->add(recoverystate_perf);
+}
+
+void OSD::suicide(int exitcode)
+{
+  if (cct->_conf->filestore_blackhole) {
+    derr << " filestore_blackhole=true, doing abbreviated shutdown" << dendl;
+    _exit(exitcode);
+  }
+
+  // turn off lockdep; the surviving threads tend to fight with exit() below
+  g_lockdep = 0;
+
+  derr << " pausing thread pools" << dendl;
+  op_tp.pause();
+  disk_tp.pause();
+  recovery_tp.pause();
+  command_tp.pause();
+
+  derr << " flushing io" << dendl;
+  store->sync_and_flush();
+
+  derr << " removing pid file" << dendl;
+  pidfile_remove();
+
+  derr << " exit" << dendl;
+  exit(exitcode);
+}
+
+int OSD::shutdown()
+{
+  if (!service.prepare_to_stop())
+    return 0; // already shutting down
+  osd_lock.Lock();
+  if (is_stopping()) {
+    osd_lock.Unlock();
+    return 0;
+  }
+  derr << "shutdown" << dendl;
+
+  heartbeat_lock.Lock();
+  state = STATE_STOPPING;
+  heartbeat_lock.Unlock();
+
+  // Debugging
+  cct->_conf->set_val("debug_osd", "100");
+  cct->_conf->set_val("debug_journal", "100");
+  cct->_conf->set_val("debug_filestore", "100");
+  cct->_conf->set_val("debug_ms", "100");
+  cct->_conf->apply_changes(NULL);
+
+  service.start_shutdown();
+
+  // Shutdown PGs
+  for (ceph::unordered_map::iterator p = pg_map.begin();
+       p != pg_map.end();
+       ++p) {
+    dout(20) << " kicking pg " << p->first << dendl;
+    p->second->lock();
+    p->second->on_shutdown();
+    p->second->unlock();
+    p->second->osr->flush();
+  }
+  
+  // finish ops
+  op_wq.drain(); // should already be empty except for lagard PGs
+  {
+    Mutex::Locker l(finished_lock);
+    finished.clear(); // zap waiters (bleh, this is messy)
+  }
+
+  // unregister commands
+  cct->get_admin_socket()->unregister_command("status");
+  cct->get_admin_socket()->unregister_command("flush_journal");
+  cct->get_admin_socket()->unregister_command("dump_ops_in_flight");
+  cct->get_admin_socket()->unregister_command("dump_historic_ops");
+  cct->get_admin_socket()->unregister_command("dump_op_pq_state");
+  cct->get_admin_socket()->unregister_command("dump_blacklist");
+  cct->get_admin_socket()->unregister_command("dump_watchers");
+  delete asok_hook;
+  asok_hook = NULL;
+
+  cct->get_admin_socket()->unregister_command("setomapval");
+  cct->get_admin_socket()->unregister_command("rmomapkey");
+  cct->get_admin_socket()->unregister_command("setomapheader");
+  cct->get_admin_socket()->unregister_command("getomap");
+  cct->get_admin_socket()->unregister_command("truncobj");
+  cct->get_admin_socket()->unregister_command("injectdataerr");
+  cct->get_admin_socket()->unregister_command("injectmdataerr");
+  delete test_ops_hook;
+  test_ops_hook = NULL;
+
+  osd_lock.Unlock();
+
+  heartbeat_lock.Lock();
+  heartbeat_stop = true;
+  heartbeat_cond.Signal();
+  heartbeat_lock.Unlock();
+  heartbeat_thread.join();
+
+  recovery_tp.drain();
+  recovery_tp.stop();
+  dout(10) << "recovery tp stopped" << dendl;
+
+  op_tp.drain();
+  peering_wq.clear();
+  scrub_finalize_wq.clear();
+  op_tp.stop();
+  dout(10) << "osd tp stopped" << dendl;
+
+  command_tp.drain();
+  command_tp.stop();
+  dout(10) << "command tp stopped" << dendl;
+
+  disk_tp.drain();
+  disk_tp.stop();
+  dout(10) << "disk tp paused (new)" << dendl;
+
+  dout(10) << "stopping agent" << dendl;
+  service.agent_stop();
+
+  osd_lock.Lock();
+
+  reset_heartbeat_peers();
+
+  tick_timer.shutdown();
+
+  // note unmount epoch
+  dout(10) << "noting clean unmount in epoch " << osdmap->get_epoch() << dendl;
+  superblock.mounted = boot_epoch;
+  superblock.clean_thru = osdmap->get_epoch();
+  ObjectStore::Transaction t;
+  write_superblock(t);
+  int r = store->apply_transaction(t);
+  if (r) {
+    derr << "OSD::shutdown: error writing superblock: "
+	 << cpp_strerror(r) << dendl;
+  }
+
+  dout(10) << "syncing store" << dendl;
+  store->flush();
+  store->sync();
+  store->umount();
+  delete store;
+  store = 0;
+  dout(10) << "Store synced" << dendl;
+
+  {
+    Mutex::Locker l(pg_stat_queue_lock);
+    assert(pg_stat_queue.empty());
+  }
+
+  // Remove PGs
+#ifdef PG_DEBUG_REFS
+  service.dump_live_pgids();
+#endif
+  for (ceph::unordered_map::iterator p = pg_map.begin();
+       p != pg_map.end();
+       ++p) {
+    dout(20) << " kicking pg " << p->first << dendl;
+    p->second->lock();
+    if (p->second->ref.read() != 1) {
+      derr << "pgid " << p->first << " has ref count of "
+	   << p->second->ref.read() << dendl;
+      assert(0);
+    }
+    p->second->unlock();
+    p->second->put("PGMap");
+  }
+  pg_map.clear();
+#ifdef PG_DEBUG_REFS
+  service.dump_live_pgids();
+#endif
+  cct->_conf->remove_observer(this);
+
+  monc->shutdown();
+  osd_lock.Unlock();
+
+  osdmap = OSDMapRef();
+  service.shutdown();
+  op_tracker.on_shutdown();
+
+  class_handler->shutdown();
+  client_messenger->shutdown();
+  cluster_messenger->shutdown();
+  hbclient_messenger->shutdown();
+  objecter_messenger->shutdown();
+  hb_front_server_messenger->shutdown();
+  hb_back_server_messenger->shutdown();
+  peering_wq.clear();
+  return r;
+}
+
+void OSD::write_superblock(ObjectStore::Transaction& t)
+{
+  dout(10) << "write_superblock " << superblock << dendl;
+
+  //hack: at minimum it's using the baseline feature set
+  if (!superblock.compat_features.incompat.mask |
+      CEPH_OSD_FEATURE_INCOMPAT_BASE.id)
+    superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE);
+
+  bufferlist bl;
+  ::encode(superblock, bl);
+  t.write(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, bl.length(), bl);
+}
+
+int OSD::read_superblock()
+{
+  bufferlist bl;
+  int r = store->read(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, 0, bl);
+  if (r < 0)
+    return r;
+
+  bufferlist::iterator p = bl.begin();
+  ::decode(superblock, p);
+
+  dout(10) << "read_superblock " << superblock << dendl;
+  
+  return 0;
+}
+
+
+
+void OSD::recursive_remove_collection(ObjectStore *store, coll_t tmp)
+{
+  OSDriver driver(
+    store,
+    coll_t(),
+    make_snapmapper_oid());
+
+  spg_t pg;
+  tmp.is_pg_prefix(pg);
+
+  ObjectStore::Transaction t;
+  SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
+
+  vector objects;
+  store->collection_list(tmp, objects);
+
+  // delete them.
+  unsigned removed = 0;
+  for (vector::iterator p = objects.begin();
+       p != objects.end();
+       ++p, removed++) {
+    OSDriver::OSTransaction _t(driver.get_transaction(&t));
+    int r = mapper.remove_oid(p->hobj, &_t);
+    if (r != 0 && r != -ENOENT)
+      assert(0);
+    t.collection_remove(tmp, *p);
+    if (removed > 300) {
+      int r = store->apply_transaction(t);
+      assert(r == 0);
+      t = ObjectStore::Transaction();
+      removed = 0;
+    }
+  }
+  t.remove_collection(tmp);
+  int r = store->apply_transaction(t);
+  assert(r == 0);
+  store->sync_and_flush();
+}
+
+
+// ======================================================
+// PG's
+
+PGPool OSD::_get_pool(int id, OSDMapRef createmap)
+{
+  if (!createmap->have_pg_pool(id)) {
+    dout(5) << __func__ << ": the OSDmap does not contain a PG pool with id = "
+	    << id << dendl;
+    assert(0);
+  }
+
+  PGPool p = PGPool(id, createmap->get_pool_name(id),
+		    createmap->get_pg_pool(id)->auid);
+    
+  const pg_pool_t *pi = createmap->get_pg_pool(id);
+  p.info = *pi;
+  p.snapc = pi->get_snap_context();
+
+  pi->build_removed_snaps(p.cached_removed_snaps);
+  dout(10) << "_get_pool " << p.id << dendl;
+  return p;
+}
+
+PG *OSD::_open_lock_pg(
+  OSDMapRef createmap,
+  spg_t pgid, bool no_lockdep_check, bool hold_map_lock)
+{
+  assert(osd_lock.is_locked());
+
+  PG* pg = _make_pg(createmap, pgid);
+
+  pg_map[pgid] = pg;
+
+  service.pg_add_epoch(pg->info.pgid, createmap->get_epoch());
+
+  pg->lock(no_lockdep_check);
+  pg->get("PGMap");  // because it's in pg_map
+  return pg;
+}
+
+PG* OSD::_make_pg(
+  OSDMapRef createmap,
+  spg_t pgid)
+{
+  dout(10) << "_open_lock_pg " << pgid << dendl;
+  PGPool pool = _get_pool(pgid.pool(), createmap);
+
+  // create
+  PG *pg;
+  hobject_t logoid = make_pg_log_oid(pgid);
+  hobject_t infooid = make_pg_biginfo_oid(pgid);
+  if (createmap->get_pg_type(pgid.pgid) == pg_pool_t::TYPE_REPLICATED ||
+      createmap->get_pg_type(pgid.pgid) == pg_pool_t::TYPE_ERASURE)
+    pg = new ReplicatedPG(&service, createmap, pool, pgid, logoid, infooid);
+  else 
+    assert(0);
+
+  return pg;
+}
+
+
+void OSD::add_newly_split_pg(PG *pg, PG::RecoveryCtx *rctx)
+{
+  epoch_t e(service.get_osdmap()->get_epoch());
+  pg->get("PGMap");  // For pg_map
+  pg_map[pg->info.pgid] = pg;
+  service.pg_add_epoch(pg->info.pgid, pg->get_osdmap()->get_epoch());
+  dout(10) << "Adding newly split pg " << *pg << dendl;
+  vector up, acting;
+  pg->get_osdmap()->pg_to_up_acting_osds(pg->info.pgid.pgid, up, acting);
+  int role = OSDMap::calc_pg_role(service.whoami, acting);
+  pg->set_role(role);
+  pg->reg_next_scrub();
+  pg->handle_loaded(rctx);
+  pg->write_if_dirty(*(rctx->transaction));
+  pg->queue_null(e, e);
+  map >::iterator to_wake =
+    peering_wait_for_split.find(pg->info.pgid);
+  if (to_wake != peering_wait_for_split.end()) {
+    for (list::iterator i =
+	   to_wake->second.begin();
+	 i != to_wake->second.end();
+	 ++i) {
+      pg->queue_peering_event(*i);
+    }
+    peering_wait_for_split.erase(to_wake);
+  }
+  wake_pg_waiters(pg->info.pgid);
+  if (!service.get_osdmap()->have_pg_pool(pg->info.pgid.pool()))
+    _remove_pg(pg);
+}
+
+OSD::res_result OSD::_try_resurrect_pg(
+  OSDMapRef curmap, spg_t pgid, spg_t *resurrected, PGRef *old_pg_state)
+{
+  assert(resurrected);
+  assert(old_pg_state);
+  // find nearest ancestor
+  DeletingStateRef df;
+  spg_t cur(pgid);
+  while (true) {
+    df = service.deleting_pgs.lookup(cur);
+    if (df)
+      break;
+    if (!cur.ps())
+      break;
+    cur = cur.get_parent();
+  }
+  if (!df)
+    return RES_NONE; // good to go
+
+  df->old_pg_state->lock();
+  OSDMapRef create_map = df->old_pg_state->get_osdmap();
+  df->old_pg_state->unlock();
+
+  set children;
+  if (cur == pgid) {
+    if (df->try_stop_deletion()) {
+      dout(10) << __func__ << ": halted deletion on pg " << pgid << dendl;
+      *resurrected = cur;
+      *old_pg_state = df->old_pg_state;
+      service.deleting_pgs.remove(pgid); // PG is no longer being removed!
+      return RES_SELF;
+    } else {
+      // raced, ensure we don't see DeletingStateRef when we try to
+      // delete this pg
+      service.deleting_pgs.remove(pgid);
+      return RES_NONE;
+    }
+  } else if (cur.is_split(create_map->get_pg_num(cur.pool()),
+			  curmap->get_pg_num(cur.pool()),
+			  &children) &&
+	     children.count(pgid)) {
+    if (df->try_stop_deletion()) {
+      dout(10) << __func__ << ": halted deletion on ancestor pg " << pgid
+	       << dendl;
+      *resurrected = cur;
+      *old_pg_state = df->old_pg_state;
+      service.deleting_pgs.remove(cur); // PG is no longer being removed!
+      return RES_PARENT;
+    } else {
+      /* this is not a problem, failing to cancel proves that all objects
+       * have been removed, so no hobject_t overlap is possible
+       */
+      return RES_NONE;
+    }
+  }
+  return RES_NONE;
+}
+
+PG *OSD::_create_lock_pg(
+  OSDMapRef createmap,
+  spg_t pgid,
+  bool newly_created,
+  bool hold_map_lock,
+  bool backfill,
+  int role,
+  vector& up, int up_primary,
+  vector& acting, int acting_primary,
+  pg_history_t history,
+  pg_interval_map_t& pi,
+  ObjectStore::Transaction& t)
+{
+  assert(osd_lock.is_locked());
+  dout(20) << "_create_lock_pg pgid " << pgid << dendl;
+
+  PG *pg = _open_lock_pg(createmap, pgid, true, hold_map_lock);
+
+  service.init_splits_between(pgid, pg->get_osdmap(), service.get_osdmap());
+
+  pg->init(
+    role,
+    up,
+    up_primary,
+    acting,
+    acting_primary,
+    history,
+    pi,
+    backfill,
+    &t);
+
+  dout(7) << "_create_lock_pg " << *pg << dendl;
+  return pg;
+}
+
+
+bool OSD::_have_pg(spg_t pgid)
+{
+  assert(osd_lock.is_locked());
+  return pg_map.count(pgid);
+}
+
+PG *OSD::_lookup_lock_pg(spg_t pgid)
+{
+  assert(osd_lock.is_locked());
+  if (!pg_map.count(pgid))
+    return NULL;
+  PG *pg = pg_map[pgid];
+  pg->lock();
+  return pg;
+}
+
+
+PG *OSD::_lookup_pg(spg_t pgid)
+{
+  assert(osd_lock.is_locked());
+  if (!pg_map.count(pgid))
+    return NULL;
+  PG *pg = pg_map[pgid];
+  return pg;
+}
+
+PG *OSD::_lookup_lock_pg_with_map_lock_held(spg_t pgid)
+{
+  assert(osd_lock.is_locked());
+  assert(pg_map.count(pgid));
+  PG *pg = pg_map[pgid];
+  pg->lock();
+  return pg;
+}
+
+void OSD::load_pgs()
+{
+  assert(osd_lock.is_locked());
+  dout(0) << "load_pgs" << dendl;
+  assert(pg_map.empty());
+
+  vector ls;
+  int r = store->list_collections(ls);
+  if (r < 0) {
+    derr << "failed to list pgs: " << cpp_strerror(-r) << dendl;
+  }
+
+  set head_pgs;
+  map > pgs;
+  for (vector::iterator it = ls.begin();
+       it != ls.end();
+       ++it) {
+    spg_t pgid;
+    snapid_t snap;
+    uint64_t seq;
+
+    if (it->is_temp(pgid) ||
+	it->is_removal(&seq, &pgid)) {
+      dout(10) << "load_pgs " << *it << " clearing temp" << dendl;
+      recursive_remove_collection(store, *it);
+      continue;
+    }
+
+    if (it->is_pg(pgid, snap)) {
+      if (snap != CEPH_NOSNAP) {
+	dout(10) << "load_pgs skipping snapped dir " << *it
+		 << " (pg " << pgid << " snap " << snap << ")" << dendl;
+	pgs[pgid].insert(snap);
+      } else {
+	pgs[pgid];
+	head_pgs.insert(pgid);
+      }
+      continue;
+    }
+
+    dout(10) << "load_pgs ignoring unrecognized " << *it << dendl;
+  }
+
+  bool has_upgraded = false;
+  for (map >::iterator i = pgs.begin();
+       i != pgs.end();
+       ++i) {
+    spg_t pgid(i->first);
+
+    if (!head_pgs.count(pgid)) {
+      dout(10) << __func__ << ": " << pgid << " has orphan snap collections " << i->second
+	       << " with no head" << dendl;
+      continue;
+    }
+
+    if (!osdmap->have_pg_pool(pgid.pool())) {
+      dout(10) << __func__ << ": skipping PG " << pgid << " because we don't have pool "
+	       << pgid.pool() << dendl;
+      continue;
+    }
+
+    if (pgid.preferred() >= 0) {
+      dout(10) << __func__ << ": skipping localized PG " << pgid << dendl;
+      // FIXME: delete it too, eventually
+      continue;
+    }
+
+    dout(10) << "pgid " << pgid << " coll " << coll_t(pgid) << dendl;
+    bufferlist bl;
+    epoch_t map_epoch = PG::peek_map_epoch(store, coll_t(pgid), service.infos_oid, &bl);
+
+    PG *pg = _open_lock_pg(map_epoch == 0 ? osdmap : service.get_map(map_epoch), pgid);
+
+    // read pg state, log
+    pg->read_state(store, bl);
+
+    if (pg->must_upgrade()) {
+      if (!has_upgraded) {
+	derr << "PGs are upgrading" << dendl;
+	has_upgraded = true;
+      }
+      dout(10) << "PG " << pg->info.pgid
+	       << " must upgrade..." << dendl;
+      pg->upgrade(store, i->second);
+    } else if (!i->second.empty()) {
+      // handle upgrade bug
+      for (interval_set::iterator j = i->second.begin();
+	   j != i->second.end();
+	   ++j) {
+	for (snapid_t k = j.get_start();
+	     k != j.get_start() + j.get_len();
+	     ++k) {
+	  assert(store->collection_empty(coll_t(pgid, k)));
+	  ObjectStore::Transaction t;
+	  t.remove_collection(coll_t(pgid, k));
+	  store->apply_transaction(t);
+	}
+      }
+    }
+
+    if (!pg->snap_collections.empty()) {
+      pg->snap_collections.clear();
+      pg->dirty_big_info = true;
+      pg->dirty_info = true;
+      ObjectStore::Transaction t;
+      pg->write_if_dirty(t);
+      store->apply_transaction(t);
+    }
+
+    service.init_splits_between(pg->info.pgid, pg->get_osdmap(), osdmap);
+
+    // generate state for PG's current mapping
+    int primary, up_primary;
+    vector acting, up;
+    pg->get_osdmap()->pg_to_up_acting_osds(
+      pgid.pgid, &up, &up_primary, &acting, &primary);
+    pg->init_primary_up_acting(
+      up,
+      acting,
+      up_primary,
+      primary);
+    int role = OSDMap::calc_pg_role(whoami, pg->acting);
+    pg->set_role(role);
+
+    pg->reg_next_scrub();
+
+    PG::RecoveryCtx rctx(0, 0, 0, 0, 0, 0);
+    pg->handle_loaded(&rctx);
+
+    dout(10) << "load_pgs loaded " << *pg << " " << pg->pg_log.get_log() << dendl;
+    pg->unlock();
+  }
+  dout(0) << "load_pgs opened " << pg_map.size() << " pgs" << dendl;
+  
+  build_past_intervals_parallel();
+}
+
+
+/*
+ * build past_intervals efficiently on old, degraded, and buried
+ * clusters.  this is important for efficiently catching up osds that
+ * are way behind on maps to the current cluster state.
+ *
+ * this is a parallel version of PG::generate_past_intervals().
+ * follow the same logic, but do all pgs at the same time so that we
+ * can make a single pass across the osdmap history.
+ */
+struct pistate {
+  epoch_t start, end;
+  vector old_acting, old_up;
+  epoch_t same_interval_since;
+  int primary;
+  int up_primary;
+};
+
+void OSD::build_past_intervals_parallel()
+{
+  map pis;
+
+  // calculate untion of map range
+  epoch_t end_epoch = superblock.oldest_map;
+  epoch_t cur_epoch = superblock.newest_map;
+  for (ceph::unordered_map::iterator i = pg_map.begin();
+       i != pg_map.end();
+       ++i) {
+    PG *pg = i->second;
+
+    epoch_t start, end;
+    if (!pg->_calc_past_interval_range(&start, &end))
+      continue;
+
+    dout(10) << pg->info.pgid << " needs " << start << "-" << end << dendl;
+    pistate& p = pis[pg];
+    p.start = start;
+    p.end = end;
+    p.same_interval_since = 0;
+
+    if (start < cur_epoch)
+      cur_epoch = start;
+    if (end > end_epoch)
+      end_epoch = end;
+  }
+  if (pis.empty()) {
+    dout(10) << __func__ << " nothing to build" << dendl;
+    return;
+  }
+
+  dout(1) << __func__ << " over " << cur_epoch << "-" << end_epoch << dendl;
+  assert(cur_epoch <= end_epoch);
+
+  OSDMapRef cur_map, last_map;
+  for ( ; cur_epoch <= end_epoch; cur_epoch++) {
+    dout(10) << __func__ << " epoch " << cur_epoch << dendl;
+    last_map = cur_map;
+    cur_map = get_map(cur_epoch);
+
+    for (map::iterator i = pis.begin(); i != pis.end(); ++i) {
+      PG *pg = i->first;
+      pistate& p = i->second;
+
+      if (cur_epoch < p.start || cur_epoch > p.end)
+	continue;
+
+      vector acting, up;
+      int up_primary;
+      int primary;
+      cur_map->pg_to_up_acting_osds(
+	pg->info.pgid.pgid, &up, &up_primary, &acting, &primary);
+
+      if (p.same_interval_since == 0) {
+	dout(10) << __func__ << " epoch " << cur_epoch << " pg " << pg->info.pgid
+		 << " first map, acting " << acting
+		 << " up " << up << ", same_interval_since = " << cur_epoch << dendl;
+	p.same_interval_since = cur_epoch;
+	p.old_up = up;
+	p.old_acting = acting;
+	p.primary = primary;
+	p.up_primary = up_primary;
+	continue;
+      }
+      assert(last_map);
+
+      std::stringstream debug;
+      bool new_interval = pg_interval_t::check_new_interval(
+	p.primary,
+	primary,
+	p.old_acting, acting,
+	p.up_primary,
+	up_primary,
+	p.old_up, up,
+	p.same_interval_since,
+	pg->info.history.last_epoch_clean,
+	cur_map, last_map,
+	pg->info.pgid.pool(),
+	pg->info.pgid.pgid,
+	&pg->past_intervals,
+	&debug);
+      if (new_interval) {
+	dout(10) << __func__ << " epoch " << cur_epoch << " pg " << pg->info.pgid
+		 << " " << debug.str() << dendl;
+	p.old_up = up;
+	p.old_acting = acting;
+	p.same_interval_since = cur_epoch;
+      }
+    }
+  }
+
+  // write info only at the end.  this is necessary because we check
+  // whether the past_intervals go far enough back or forward in time,
+  // but we don't check for holes.  we could avoid it by discarding
+  // the previous past_intervals and rebuilding from scratch, or we
+  // can just do this and commit all our work at the end.
+  ObjectStore::Transaction t;
+  int num = 0;
+  for (map::iterator i = pis.begin(); i != pis.end(); ++i) {
+    PG *pg = i->first;
+    pg->lock();
+    pg->dirty_big_info = true;
+    pg->dirty_info = true;
+    pg->write_if_dirty(t);
+    pg->unlock();
+
+    // don't let the transaction get too big
+    if (++num >= cct->_conf->osd_target_transaction_size) {
+      store->apply_transaction(t);
+      t = ObjectStore::Transaction();
+      num = 0;
+    }
+  }
+  if (!t.empty())
+    store->apply_transaction(t);
+}
+
+/*
+ * look up a pg.  if we have it, great.  if not, consider creating it IF the pg mapping
+ * hasn't changed since the given epoch and we are the primary.
+ */
+void OSD::handle_pg_peering_evt(
+  spg_t pgid,
+  const pg_info_t& info,
+  pg_interval_map_t& pi,
+  epoch_t epoch,
+  pg_shard_t from,
+  bool primary,
+  PG::CephPeeringEvtRef evt)
+{
+  if (service.splitting(pgid)) {
+    peering_wait_for_split[pgid].push_back(evt);
+    return;
+  }
+
+  if (!_have_pg(pgid)) {
+    // same primary?
+    if (!osdmap->have_pg_pool(pgid.pool()))
+      return;
+    int up_primary, acting_primary;
+    vector up, acting;
+    osdmap->pg_to_up_acting_osds(
+      pgid.pgid, &up, &up_primary, &acting, &acting_primary);
+    int role = osdmap->calc_pg_role(whoami, acting, acting.size());
+
+    pg_history_t history = info.history;
+    bool valid_history = project_pg_history(
+      pgid, history, epoch, up, up_primary, acting, acting_primary);
+
+    if (!valid_history || epoch < history.same_interval_since) {
+      dout(10) << "get_or_create_pg " << pgid << " acting changed in "
+	       << history.same_interval_since << " (msg from " << epoch << ")" << dendl;
+      return;
+    }
+
+    if (service.splitting(pgid)) {
+      assert(0);
+    }
+
+    bool create = false;
+    if (primary) {
+      // DNE on source?
+      if (info.dne()) {
+	// is there a creation pending on this pg?
+	if (creating_pgs.count(pgid)) {
+	  creating_pgs[pgid].prior.erase(from);
+	  if (!can_create_pg(pgid))
+	    return;
+	  history = creating_pgs[pgid].history;
+	  create = true;
+	} else {
+	  dout(10) << "get_or_create_pg " << pgid
+		   << " DNE on source, but creation probe, ignoring" << dendl;
+	  return;
+	}
+      }
+      creating_pgs.erase(pgid);
+    } else {
+      assert(!info.dne());  // pg exists if we are hearing about it
+    }
+
+    // do we need to resurrect a deleting pg?
+    spg_t resurrected;
+    PGRef old_pg_state;
+    res_result result = _try_resurrect_pg(
+      service.get_osdmap(),
+      pgid,
+      &resurrected,
+      &old_pg_state);
+
+    PG::RecoveryCtx rctx = create_context();
+    switch (result) {
+    case RES_NONE: {
+      // ok, create the pg locally using provided Info and History
+      rctx.transaction->create_collection(coll_t(pgid));
+      PG *pg = _create_lock_pg(
+	get_map(epoch),
+	pgid, create, false, result == RES_SELF,
+	role,
+	up, up_primary,
+	acting, acting_primary,
+	history, pi,
+	*rctx.transaction);
+      pg->handle_create(&rctx);
+      pg->write_if_dirty(*rctx.transaction);
+      dispatch_context(rctx, pg, osdmap);
+
+      dout(10) << *pg << " is new" << dendl;
+
+      // kick any waiters
+      wake_pg_waiters(pg->info.pgid);
+      
+      pg->queue_peering_event(evt);
+      pg->unlock();
+      return;
+    }
+    case RES_SELF: {
+      old_pg_state->lock();
+      PG *pg = _create_lock_pg(
+	old_pg_state->get_osdmap(),
+	resurrected,
+	false,
+	false,
+	true,
+	old_pg_state->role,
+	old_pg_state->up,
+	old_pg_state->up_primary.osd,
+	old_pg_state->acting,
+	old_pg_state->primary.osd,
+	old_pg_state->info.history,
+	old_pg_state->past_intervals,
+	*rctx.transaction);
+      old_pg_state->unlock();
+      pg->handle_create(&rctx);
+      pg->write_if_dirty(*rctx.transaction);
+      dispatch_context(rctx, pg, osdmap);
+
+      dout(10) << *pg << " is new (resurrected)" << dendl;
+
+      // kick any waiters
+      wake_pg_waiters(pg->info.pgid);
+
+      pg->queue_peering_event(evt);
+      pg->unlock();
+      return;
+    }
+    case RES_PARENT: {
+      assert(old_pg_state);
+      old_pg_state->lock();
+      PG *parent = _create_lock_pg(
+	old_pg_state->get_osdmap(),
+	resurrected,
+	false,
+	false,
+	true,
+	old_pg_state->role,
+	old_pg_state->up,
+	old_pg_state->up_primary.osd,
+	old_pg_state->acting,
+	old_pg_state->primary.osd,
+	old_pg_state->info.history,
+	old_pg_state->past_intervals,
+	*rctx.transaction
+	);
+      old_pg_state->unlock();
+      parent->handle_create(&rctx);
+      parent->write_if_dirty(*rctx.transaction);
+      dispatch_context(rctx, parent, osdmap);
+
+      dout(10) << *parent << " is new" << dendl;
+
+      // kick any waiters
+      wake_pg_waiters(parent->info.pgid);
+
+      assert(service.splitting(pgid));
+      peering_wait_for_split[pgid].push_back(evt);
+
+      //parent->queue_peering_event(evt);
+      parent->queue_null(osdmap->get_epoch(), osdmap->get_epoch());
+      parent->unlock();
+      return;
+    }
+    }
+  } else {
+    // already had it.  did the mapping change?
+    PG *pg = _lookup_lock_pg(pgid);
+    if (epoch < pg->info.history.same_interval_since) {
+      dout(10) << *pg << " get_or_create_pg acting changed in "
+	       << pg->info.history.same_interval_since
+	       << " (msg from " << epoch << ")" << dendl;
+      pg->unlock();
+      return;
+    }
+    pg->queue_peering_event(evt);
+    pg->unlock();
+    return;
+  }
+}
+
+
+/*
+ * calculate prior pg members during an epoch interval [start,end)
+ *  - from each epoch, include all osds up then AND now
+ *  - if no osds from then are up now, include them all, even tho they're not reachable now
+ */
+void OSD::calc_priors_during(
+  spg_t pgid, epoch_t start, epoch_t end, set& pset)
+{
+  dout(15) << "calc_priors_during " << pgid << " [" << start
+	   << "," << end << ")" << dendl;
+  
+  for (epoch_t e = start; e < end; e++) {
+    OSDMapRef oldmap = get_map(e);
+    vector acting;
+    oldmap->pg_to_acting_osds(pgid.pgid, acting);
+    dout(20) << "  " << pgid << " in epoch " << e << " was " << acting << dendl;
+    int up = 0;
+    int actual_osds = 0;
+    for (unsigned i=0; iis_up(acting[i])) {
+	  if (acting[i] != whoami) {
+	    pset.insert(
+	      pg_shard_t(
+		acting[i],
+		osdmap->pg_is_ec(pgid.pgid) ? shard_id_t(i) : ghobject_t::NO_SHARD));
+	  }
+	  up++;
+	}
+	actual_osds++;
+      }
+    }
+    if (!up && actual_osds) {
+      // sucky.  add down osds, even tho we can't reach them right now.
+      for (unsigned i=0; ipg_is_ec(pgid.pgid) ? i : ghobject_t::NO_SHARD));
+	}
+      }
+    }
+  }
+  dout(10) << "calc_priors_during " << pgid
+	   << " [" << start << "," << end 
+	   << ") = " << pset << dendl;
+}
+
+
+/**
+ * Fill in the passed history so you know same_interval_since, same_up_since,
+ * and same_primary_since.
+ */
+bool OSD::project_pg_history(spg_t pgid, pg_history_t& h, epoch_t from,
+			     const vector& currentup,
+			     int currentupprimary,
+			     const vector& currentacting,
+			     int currentactingprimary)
+{
+  dout(15) << "project_pg_history " << pgid
+           << " from " << from << " to " << osdmap->get_epoch()
+           << ", start " << h
+           << dendl;
+
+  epoch_t e;
+  for (e = osdmap->get_epoch();
+       e > from;
+       e--) {
+    // verify during intermediate epoch (e-1)
+    OSDMapRef oldmap = service.try_get_map(e-1);
+    if (!oldmap) {
+      dout(15) << __func__ << ": found map gap, returning false" << dendl;
+      return false;
+    }
+    assert(oldmap->have_pg_pool(pgid.pool()));
+
+    int upprimary, actingprimary;
+    vector up, acting;
+    oldmap->pg_to_up_acting_osds(
+      pgid.pgid,
+      &up,
+      &upprimary,
+      &acting,
+      &actingprimary);
+
+    // acting set change?
+    if ((actingprimary != currentactingprimary ||
+	 upprimary != currentupprimary ||
+	 acting != currentacting ||
+	 up != currentup) && e > h.same_interval_since) {
+      dout(15) << "project_pg_history " << pgid << " acting|up changed in " << e 
+	       << " from " << acting << "/" << up
+	       << " " << actingprimary << "/" << upprimary
+	       << " -> " << currentacting << "/" << currentup
+	       << " " << currentactingprimary << "/" << currentupprimary 
+	       << dendl;
+      h.same_interval_since = e;
+    }
+    // split?
+    if (pgid.is_split(oldmap->get_pg_num(pgid.pool()),
+		      osdmap->get_pg_num(pgid.pool()),
+		      0)) {
+      h.same_interval_since = e;
+    }
+    // up set change?
+    if ((up != currentup || upprimary != currentupprimary)
+	&& e > h.same_up_since) {
+      dout(15) << "project_pg_history " << pgid << " up changed in " << e 
+	       << " from " << up << " " << upprimary
+	       << " -> " << currentup << " " << currentupprimary << dendl;
+      h.same_up_since = e;
+    }
+
+    // primary change?
+    if (OSDMap::primary_changed(
+	  actingprimary,
+	  acting,
+	  currentactingprimary,
+	  currentacting) &&
+        e > h.same_primary_since) {
+      dout(15) << "project_pg_history " << pgid << " primary changed in " << e << dendl;
+      h.same_primary_since = e;
+    }
+
+    if (h.same_interval_since >= e && h.same_up_since >= e && h.same_primary_since >= e)
+      break;
+  }
+
+  // base case: these floors should be the creation epoch if we didn't
+  // find any changes.
+  if (e == h.epoch_created) {
+    if (!h.same_interval_since)
+      h.same_interval_since = e;
+    if (!h.same_up_since)
+      h.same_up_since = e;
+    if (!h.same_primary_since)
+      h.same_primary_since = e;
+  }
+
+  dout(15) << "project_pg_history end " << h << dendl;
+  return true;
+}
+
+// -------------------------------------
+
+float OSDService::get_full_ratio()
+{
+  float full_ratio = cct->_conf->osd_failsafe_full_ratio;
+  if (full_ratio > 1.0) full_ratio /= 100.0;
+  return full_ratio;
+}
+
+float OSDService::get_nearfull_ratio()
+{
+  float nearfull_ratio = cct->_conf->osd_failsafe_nearfull_ratio;
+  if (nearfull_ratio > 1.0) nearfull_ratio /= 100.0;
+  return nearfull_ratio;
+}
+
+void OSDService::check_nearfull_warning(const osd_stat_t &osd_stat)
+{
+  Mutex::Locker l(full_status_lock);
+  enum s_names new_state;
+
+  time_t now = ceph_clock_gettime(NULL);
+
+  // We base ratio on kb_avail rather than kb_used because they can
+  // differ significantly e.g. on btrfs volumes with a large number of
+  // chunks reserved for metadata, and for our purposes (avoiding
+  // completely filling the disk) it's far more important to know how
+  // much space is available to use than how much we've already used.
+  float ratio = ((float)(osd_stat.kb - osd_stat.kb_avail)) / ((float)osd_stat.kb);
+  float nearfull_ratio = get_nearfull_ratio();
+  float full_ratio = get_full_ratio();
+  cur_ratio = ratio;
+
+  if (full_ratio > 0 && ratio > full_ratio) {
+    new_state = FULL;
+  } else if (nearfull_ratio > 0 && ratio > nearfull_ratio) {
+    new_state = NEAR;
+  } else {
+    cur_state = NONE;
+    return;
+  }
+
+  if (cur_state != new_state) {
+    cur_state = new_state;
+  } else if (now - last_msg < cct->_conf->osd_op_complaint_time) {
+    return;
+  }
+  last_msg = now;
+  if (cur_state == FULL)
+    clog.error() << "OSD full dropping all updates " << (int)(ratio * 100) << "% full";
+  else
+    clog.warn() << "OSD near full (" << (int)(ratio * 100) << "%)";
+}
+
+bool OSDService::check_failsafe_full()
+{
+  Mutex::Locker l(full_status_lock);
+  if (cur_state == FULL)
+    return true;
+  return false;
+}
+
+bool OSDService::too_full_for_backfill(double *_ratio, double *_max_ratio)
+{
+  Mutex::Locker l(full_status_lock);
+  double max_ratio;
+  max_ratio = cct->_conf->osd_backfill_full_ratio;
+  if (_ratio)
+    *_ratio = cur_ratio;
+  if (_max_ratio)
+    *_max_ratio = max_ratio;
+  return cur_ratio >= max_ratio;
+}
+
+
+void OSD::update_osd_stat()
+{
+  // fill in osd stats too
+  struct statfs stbuf;
+  store->statfs(&stbuf);
+
+  uint64_t bytes = stbuf.f_blocks * stbuf.f_bsize;
+  uint64_t used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize;
+  uint64_t avail = stbuf.f_bavail * stbuf.f_bsize;
+
+  osd_stat.kb = bytes >> 10;
+  osd_stat.kb_used = used >> 10;
+  osd_stat.kb_avail = avail >> 10;
+
+  logger->set(l_osd_stat_bytes, bytes);
+  logger->set(l_osd_stat_bytes_used, used);
+  logger->set(l_osd_stat_bytes_avail, avail);
+
+  osd_stat.hb_in.clear();
+  for (map::iterator p = heartbeat_peers.begin(); p != heartbeat_peers.end(); ++p)
+    osd_stat.hb_in.push_back(p->first);
+  osd_stat.hb_out.clear();
+
+  service.check_nearfull_warning(osd_stat);
+
+  op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
+
+  dout(20) << "update_osd_stat " << osd_stat << dendl;
+}
+
+void OSD::_add_heartbeat_peer(int p)
+{
+  if (p == whoami)
+    return;
+  HeartbeatInfo *hi;
+
+  map::iterator i = heartbeat_peers.find(p);
+  if (i == heartbeat_peers.end()) {
+    pair cons = service.get_con_osd_hb(p, osdmap->get_epoch());
+    if (!cons.first)
+      return;
+    hi = &heartbeat_peers[p];
+    hi->peer = p;
+    HeartbeatSession *s = new HeartbeatSession(p);
+    hi->con_back = cons.first.get();
+    hi->con_back->set_priv(s);
+    if (cons.second) {
+      hi->con_front = cons.second.get();
+      hi->con_front->set_priv(s->get());
+      dout(10) << "_add_heartbeat_peer: new peer osd." << p
+	       << " " << hi->con_back->get_peer_addr()
+	       << " " << hi->con_front->get_peer_addr()
+	       << dendl;
+    } else {
+      hi->con_front.reset(NULL);
+      dout(10) << "_add_heartbeat_peer: new peer osd." << p
+	       << " " << hi->con_back->get_peer_addr()
+	       << dendl;
+    }
+  } else {
+    hi = &i->second;
+  }
+  hi->epoch = osdmap->get_epoch();
+}
+
+void OSD::_remove_heartbeat_peer(int n)
+{
+  map::iterator q = heartbeat_peers.find(n);
+  assert(q != heartbeat_peers.end());
+  dout(20) << " removing heartbeat peer osd." << n
+	   << " " << q->second.con_back->get_peer_addr()
+	   << " " << (q->second.con_front ? q->second.con_front->get_peer_addr() : entity_addr_t())
+	   << dendl;
+  hbclient_messenger->mark_down(q->second.con_back);
+  if (q->second.con_front) {
+    hbclient_messenger->mark_down(q->second.con_front);
+  }
+  heartbeat_peers.erase(q);
+}
+
+void OSD::need_heartbeat_peer_update()
+{
+  Mutex::Locker l(heartbeat_lock);
+  if (is_stopping())
+    return;
+  dout(20) << "need_heartbeat_peer_update" << dendl;
+  heartbeat_need_update = true;
+}
+
+void OSD::maybe_update_heartbeat_peers()
+{
+  assert(osd_lock.is_locked());
+
+  if (is_waiting_for_healthy()) {
+    utime_t now = ceph_clock_now(cct);
+    if (last_heartbeat_resample == utime_t()) {
+      last_heartbeat_resample = now;
+      heartbeat_need_update = true;
+    } else if (!heartbeat_need_update) {
+      utime_t dur = now - last_heartbeat_resample;
+      if (dur > cct->_conf->osd_heartbeat_grace) {
+	dout(10) << "maybe_update_heartbeat_peers forcing update after " << dur << " seconds" << dendl;
+	heartbeat_need_update = true;
+	last_heartbeat_resample = now;
+	reset_heartbeat_peers();   // we want *new* peers!
+      }
+    }
+  }
+
+  Mutex::Locker l(heartbeat_lock);
+  if (!heartbeat_need_update)
+    return;
+  heartbeat_need_update = false;
+
+  dout(10) << "maybe_update_heartbeat_peers updating" << dendl;
+
+  heartbeat_epoch = osdmap->get_epoch();
+
+  // build heartbeat from set
+  if (is_active()) {
+    for (ceph::unordered_map::iterator i = pg_map.begin();
+	 i != pg_map.end();
+	 ++i) {
+      PG *pg = i->second;
+      pg->heartbeat_peer_lock.Lock();
+      dout(20) << i->first << " heartbeat_peers " << pg->heartbeat_peers << dendl;
+      for (set::iterator p = pg->heartbeat_peers.begin();
+	   p != pg->heartbeat_peers.end();
+	   ++p)
+	if (osdmap->is_up(*p))
+	  _add_heartbeat_peer(*p);
+      for (set::iterator p = pg->probe_targets.begin();
+	   p != pg->probe_targets.end();
+	   ++p)
+	if (osdmap->is_up(*p))
+	  _add_heartbeat_peer(*p);
+      pg->heartbeat_peer_lock.Unlock();
+    }
+  }
+
+  // include next and previous up osds to ensure we have a fully-connected set
+  set want, extras;
+  int next = osdmap->get_next_up_osd_after(whoami);
+  if (next >= 0)
+    want.insert(next);
+  int prev = osdmap->get_previous_up_osd_before(whoami);
+  if (prev >= 0)
+    want.insert(prev);
+
+  for (set::iterator p = want.begin(); p != want.end(); ++p) {
+    dout(10) << " adding neighbor peer osd." << *p << dendl;
+    extras.insert(*p);
+    _add_heartbeat_peer(*p);
+  }
+
+  // remove down peers; enumerate extras
+  map::iterator p = heartbeat_peers.begin();
+  while (p != heartbeat_peers.end()) {
+    if (!osdmap->is_up(p->first)) {
+      int o = p->first;
+      ++p;
+      _remove_heartbeat_peer(o);
+      continue;
+    }
+    if (p->second.epoch < osdmap->get_epoch()) {
+      extras.insert(p->first);
+    }
+    ++p;
+  }
+
+  // too few?
+  int start = osdmap->get_next_up_osd_after(whoami);
+  for (int n = start; n >= 0; ) {
+    if ((int)heartbeat_peers.size() >= cct->_conf->osd_heartbeat_min_peers)
+      break;
+    if (!extras.count(n) && !want.count(n) && n != whoami) {
+      dout(10) << " adding random peer osd." << n << dendl;
+      extras.insert(n);
+      _add_heartbeat_peer(n);
+    }
+    n = osdmap->get_next_up_osd_after(n);
+    if (n == start)
+      break;  // came full circle; stop
+  }
+
+  // too many?
+  for (set::iterator p = extras.begin();
+       (int)heartbeat_peers.size() > cct->_conf->osd_heartbeat_min_peers && p != extras.end();
+       ++p) {
+    if (want.count(*p))
+      continue;
+    _remove_heartbeat_peer(*p);
+  }
+
+  dout(10) << "maybe_update_heartbeat_peers " << heartbeat_peers.size() << " peers, extras " << extras << dendl;
+}
+
+void OSD::reset_heartbeat_peers()
+{
+  assert(osd_lock.is_locked());
+  dout(10) << "reset_heartbeat_peers" << dendl;
+  Mutex::Locker l(heartbeat_lock);
+  while (!heartbeat_peers.empty()) {
+    HeartbeatInfo& hi = heartbeat_peers.begin()->second;
+    hbclient_messenger->mark_down(hi.con_back);
+    if (hi.con_front) {
+      hbclient_messenger->mark_down(hi.con_front);
+    }
+    heartbeat_peers.erase(heartbeat_peers.begin());
+  }
+  failure_queue.clear();
+}
+
+void OSD::handle_osd_ping(MOSDPing *m)
+{
+  if (superblock.cluster_fsid != m->fsid) {
+    dout(20) << "handle_osd_ping from " << m->get_source_inst()
+	     << " bad fsid " << m->fsid << " != " << superblock.cluster_fsid << dendl;
+    m->put();
+    return;
+  }
+
+  int from = m->get_source().num();
+
+  heartbeat_lock.Lock();
+  if (is_stopping()) {
+    heartbeat_lock.Unlock();
+    m->put();
+    return;
+  }
+
+  OSDMapRef curmap = service.get_osdmap();
+  
+  switch (m->op) {
+
+  case MOSDPing::PING:
+    {
+      if (cct->_conf->osd_debug_drop_ping_probability > 0) {
+	if (debug_heartbeat_drops_remaining.count(from)) {
+	  if (debug_heartbeat_drops_remaining[from] == 0) {
+	    debug_heartbeat_drops_remaining.erase(from);
+	  } else {
+	    debug_heartbeat_drops_remaining[from]--;
+	    dout(5) << "Dropping heartbeat from " << from
+		    << ", " << debug_heartbeat_drops_remaining[from]
+		    << " remaining to drop" << dendl;
+	    break;
+	  }
+	} else if (cct->_conf->osd_debug_drop_ping_probability >
+	           ((((double)(rand()%100))/100.0))) {
+	  debug_heartbeat_drops_remaining[from] =
+	    cct->_conf->osd_debug_drop_ping_duration;
+	  dout(5) << "Dropping heartbeat from " << from
+		  << ", " << debug_heartbeat_drops_remaining[from]
+		  << " remaining to drop" << dendl;
+	  break;
+	}
+      }
+
+      if (!cct->get_heartbeat_map()->is_healthy()) {
+	dout(10) << "internal heartbeat not healthy, dropping ping request" << dendl;
+	break;
+      }
+
+      Message *r = new MOSDPing(monc->get_fsid(),
+				curmap->get_epoch(),
+				MOSDPing::PING_REPLY,
+				m->stamp);
+      m->get_connection()->get_messenger()->send_message(r, m->get_connection());
+
+      if (curmap->is_up(from)) {
+	note_peer_epoch(from, m->map_epoch);
+	if (is_active()) {
+	  ConnectionRef con = service.get_con_osd_cluster(from, curmap->get_epoch());
+	  if (con) {
+	    _share_map_outgoing(from, con.get());
+	  }
+	}
+      } else if (!curmap->exists(from) ||
+		 curmap->get_down_at(from) > m->map_epoch) {
+	// tell them they have died
+	Message *r = new MOSDPing(monc->get_fsid(),
+				  curmap->get_epoch(),
+				  MOSDPing::YOU_DIED,
+				  m->stamp);
+	m->get_connection()->get_messenger()->send_message(r, m->get_connection());
+      }
+    }
+    break;
+
+  case MOSDPing::PING_REPLY:
+    {
+      map::iterator i = heartbeat_peers.find(from);
+      if (i != heartbeat_peers.end()) {
+	if (m->get_connection() == i->second.con_back) {
+	  dout(25) << "handle_osd_ping got reply from osd." << from
+		   << " first_rx " << i->second.first_tx
+		   << " last_tx " << i->second.last_tx
+		   << " last_rx_back " << i->second.last_rx_back << " -> " << m->stamp
+		   << " last_rx_front " << i->second.last_rx_front
+		   << dendl;
+	  i->second.last_rx_back = m->stamp;
+	  // if there is no front con, set both stamps.
+	  if (i->second.con_front == NULL)
+	    i->second.last_rx_front = m->stamp;
+	} else if (m->get_connection() == i->second.con_front) {
+	  dout(25) << "handle_osd_ping got reply from osd." << from
+		   << " first_rx " << i->second.first_tx
+		   << " last_tx " << i->second.last_tx
+		   << " last_rx_back " << i->second.last_rx_back
+		   << " last_rx_front " << i->second.last_rx_front << " -> " << m->stamp
+		   << dendl;
+	  i->second.last_rx_front = m->stamp;
+	}
+      }
+
+      if (m->map_epoch &&
+	  curmap->is_up(from)) {
+	note_peer_epoch(from, m->map_epoch);
+	if (is_active()) {
+	  ConnectionRef con = service.get_con_osd_cluster(from, curmap->get_epoch());
+	  if (con) {
+	    _share_map_outgoing(from, con.get());
+	  }
+	}
+      }
+
+      utime_t cutoff = ceph_clock_now(cct);
+      cutoff -= cct->_conf->osd_heartbeat_grace;
+      if (i->second.is_healthy(cutoff)) {
+	// Cancel false reports
+	if (failure_queue.count(from)) {
+	  dout(10) << "handle_osd_ping canceling queued failure report for osd." << from<< dendl;
+	  failure_queue.erase(from);
+	}
+	if (failure_pending.count(from)) {
+	  dout(10) << "handle_osd_ping canceling in-flight failure report for osd." << from<< dendl;
+	  send_still_alive(curmap->get_epoch(), failure_pending[from]);
+	  failure_pending.erase(from);
+	}
+      }
+    }
+    break;
+
+  case MOSDPing::YOU_DIED:
+    dout(10) << "handle_osd_ping " << m->get_source_inst()
+	     << " says i am down in " << m->map_epoch << dendl;
+    osdmap_subscribe(curmap->get_epoch()+1, false);
+    break;
+  }
+
+  heartbeat_lock.Unlock();
+  m->put();
+}
+
+void OSD::heartbeat_entry()
+{
+  Mutex::Locker l(heartbeat_lock);
+  if (is_stopping())
+    return;
+  while (!heartbeat_stop) {
+    heartbeat();
+
+    double wait = .5 + ((float)(rand() % 10)/10.0) * (float)cct->_conf->osd_heartbeat_interval;
+    utime_t w;
+    w.set_from_double(wait);
+    dout(30) << "heartbeat_entry sleeping for " << wait << dendl;
+    heartbeat_cond.WaitInterval(cct, heartbeat_lock, w);
+    if (is_stopping())
+      return;
+    dout(30) << "heartbeat_entry woke up" << dendl;
+  }
+}
+
+void OSD::heartbeat_check()
+{
+  assert(heartbeat_lock.is_locked());
+  utime_t now = ceph_clock_now(cct);
+  double age = hbclient_messenger->get_dispatch_queue_max_age(now);
+  if (age > (cct->_conf->osd_heartbeat_grace / 2)) {
+    derr << "skipping heartbeat_check, hbqueue max age: " << age << dendl;
+    return; // hb dispatch is too backed up for our hb status to be meaningful
+  }
+
+  // check for incoming heartbeats (move me elsewhere?)
+  utime_t cutoff = now;
+  cutoff -= cct->_conf->osd_heartbeat_grace;
+  for (map::iterator p = heartbeat_peers.begin();
+       p != heartbeat_peers.end();
+       ++p) {
+    dout(25) << "heartbeat_check osd." << p->first
+	     << " first_tx " << p->second.first_tx
+	     << " last_tx " << p->second.last_tx
+	     << " last_rx_back " << p->second.last_rx_back
+	     << " last_rx_front " << p->second.last_rx_front
+	     << dendl;
+    if (p->second.is_unhealthy(cutoff)) {
+      if (p->second.last_rx_back == utime_t() ||
+	  p->second.last_rx_front == utime_t()) {
+	derr << "heartbeat_check: no reply from osd." << p->first
+	     << " ever on either front or back, first ping sent " << p->second.first_tx
+	     << " (cutoff " << cutoff << ")" << dendl;
+	// fail
+	failure_queue[p->first] = p->second.last_tx;
+      } else {
+	derr << "heartbeat_check: no reply from osd." << p->first
+	     << " since back " << p->second.last_rx_back
+	     << " front " << p->second.last_rx_front
+	     << " (cutoff " << cutoff << ")" << dendl;
+	// fail
+	failure_queue[p->first] = MIN(p->second.last_rx_back, p->second.last_rx_front);
+      }
+    }
+  }
+}
+
+void OSD::heartbeat()
+{
+  dout(30) << "heartbeat" << dendl;
+
+  // get CPU load avg
+  double loadavgs[1];
+  if (getloadavg(loadavgs, 1) == 1)
+    logger->set(l_osd_loadavg, 100 * loadavgs[0]);
+
+  dout(30) << "heartbeat checking stats" << dendl;
+
+  // refresh stats?
+  {
+    Mutex::Locker lock(stat_lock);
+    update_osd_stat();
+  }
+
+  dout(5) << "heartbeat: " << osd_stat << dendl;
+
+  utime_t now = ceph_clock_now(cct);
+
+  // send heartbeats
+  for (map::iterator i = heartbeat_peers.begin();
+       i != heartbeat_peers.end();
+       ++i) {
+    int peer = i->first;
+    i->second.last_tx = now;
+    if (i->second.first_tx == utime_t())
+      i->second.first_tx = now;
+    dout(30) << "heartbeat sending ping to osd." << peer << dendl;
+    hbclient_messenger->send_message(new MOSDPing(monc->get_fsid(),
+						  service.get_osdmap()->get_epoch(),
+						  MOSDPing::PING,
+						  now),
+				     i->second.con_back);
+    if (i->second.con_front)
+      hbclient_messenger->send_message(new MOSDPing(monc->get_fsid(),
+						    service.get_osdmap()->get_epoch(),
+						    MOSDPing::PING,
+						    now),
+				       i->second.con_front);
+  }
+
+  dout(30) << "heartbeat check" << dendl;
+  heartbeat_check();
+
+  logger->set(l_osd_hb_to, heartbeat_peers.size());
+  logger->set(l_osd_hb_from, 0);
+  
+  // hmm.. am i all alone?
+  dout(30) << "heartbeat lonely?" << dendl;
+  if (heartbeat_peers.empty()) {
+    if (now - last_mon_heartbeat > cct->_conf->osd_mon_heartbeat_interval && is_active()) {
+      last_mon_heartbeat = now;
+      dout(10) << "i have no heartbeat peers; checking mon for new map" << dendl;
+      osdmap_subscribe(osdmap->get_epoch() + 1, true);
+    }
+  }
+
+  dout(30) << "heartbeat done" << dendl;
+}
+
+bool OSD::heartbeat_reset(Connection *con)
+{
+  HeartbeatSession *s = static_cast(con->get_priv());
+  if (s) {
+    heartbeat_lock.Lock();
+    if (is_stopping()) {
+      heartbeat_lock.Unlock();
+      s->put();
+      return true;
+    }
+    map::iterator p = heartbeat_peers.find(s->peer);
+    if (p != heartbeat_peers.end() &&
+	(p->second.con_back == con ||
+	 p->second.con_front == con)) {
+      dout(10) << "heartbeat_reset failed hb con " << con << " for osd." << p->second.peer
+	       << ", reopening" << dendl;
+      if (con != p->second.con_back) {
+	hbclient_messenger->mark_down(p->second.con_back);
+      }
+      p->second.con_back.reset(NULL);
+      if (p->second.con_front && con != p->second.con_front) {
+	hbclient_messenger->mark_down(p->second.con_front);
+      }
+      p->second.con_front.reset(NULL);
+      pair newcon = service.get_con_osd_hb(p->second.peer, p->second.epoch);
+      if (newcon.first) {
+	p->second.con_back = newcon.first.get();
+	p->second.con_back->set_priv(s->get());
+	if (newcon.second) {
+	  p->second.con_front = newcon.second.get();
+	  p->second.con_front->set_priv(s->get());
+	}
+      } else {
+	dout(10) << "heartbeat_reset failed hb con " << con << " for osd." << p->second.peer
+		 << ", raced with osdmap update, closing out peer" << dendl;
+	heartbeat_peers.erase(p);
+      }
+    } else {
+      dout(10) << "heartbeat_reset closing (old) failed hb con " << con << dendl;
+    }
+    heartbeat_lock.Unlock();
+    s->put();
+  }
+  return true;
+}
+
+
+
+// =========================================
+
+void OSD::tick()
+{
+  assert(osd_lock.is_locked());
+  dout(5) << "tick" << dendl;
+
+  logger->set(l_osd_buf, buffer::get_total_alloc());
+
+  if (is_active() || is_waiting_for_healthy()) {
+    map_lock.get_read();
+
+    maybe_update_heartbeat_peers();
+
+    heartbeat_lock.Lock();
+    heartbeat_check();
+    heartbeat_lock.Unlock();
+
+    // mon report?
+    utime_t now = ceph_clock_now(cct);
+    if (outstanding_pg_stats && timeout_mon_on_pg_stats &&
+	(now - cct->_conf->osd_mon_ack_timeout) > last_pg_stats_ack) {
+      dout(1) << "mon hasn't acked PGStats in " << now - last_pg_stats_ack
+	      << " seconds, reconnecting elsewhere" << dendl;
+      monc->reopen_session(new C_MonStatsAckTimer(this));
+      timeout_mon_on_pg_stats = false;
+      last_pg_stats_ack = ceph_clock_now(cct);  // reset clock
+      last_pg_stats_sent = utime_t();
+    }
+    if (now - last_pg_stats_sent > cct->_conf->osd_mon_report_interval_max) {
+      osd_stat_updated = true;
+      do_mon_report();
+    } else if (now - last_mon_report > cct->_conf->osd_mon_report_interval_min) {
+      do_mon_report();
+    }
+
+    map_lock.put_read();
+  }
+
+  if (is_waiting_for_healthy()) {
+    if (_is_healthy()) {
+      dout(1) << "healthy again, booting" << dendl;
+      state = STATE_BOOTING;
+      start_boot();
+    }
+  }
+
+  if (is_active()) {
+    // periodically kick recovery work queue
+    recovery_tp.wake();
+
+    if (!scrub_random_backoff()) {
+      sched_scrub();
+    }
+
+    check_replay_queue();
+  }
+
+  // only do waiters if dispatch() isn't currently running.  (if it is,
+  // it'll do the waiters, and doing them here may screw up ordering
+  // of op_queue vs handle_osd_map.)
+  if (!dispatch_running) {
+    dispatch_running = true;
+    do_waiters();
+    dispatch_running = false;
+    dispatch_cond.Signal();
+  }
+
+  check_ops_in_flight();
+
+  tick_timer.add_event_after(1.0, new C_Tick(this));
+}
+
+void OSD::check_ops_in_flight()
+{
+  vector warnings;
+  if (op_tracker.check_ops_in_flight(warnings)) {
+    for (vector::iterator i = warnings.begin();
+        i != warnings.end();
+        ++i) {
+      clog.warn() << *i;
+    }
+  }
+  return;
+}
+
+// Usage:
+//   setomapval  [namespace/]  
+//   rmomapkey  [namespace/] 
+//   setomapheader  [namespace/] 

+//   getomap  [namespace/]
+//   truncobj  [namespace/] 
+//   injectmdataerr [namespace/]
+//   injectdataerr [namespace/]
+void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store,
+     std::string command, cmdmap_t& cmdmap, ostream &ss)
+{
+  //Test support
+  //Support changing the omap on a single osd by using the Admin Socket to
+  //directly request the osd make a change.
+  if (command == "setomapval" || command == "rmomapkey" ||
+      command == "setomapheader" || command == "getomap" ||
+      command == "truncobj" || command == "injectmdataerr" ||
+      command == "injectdataerr"
+    ) {
+    pg_t rawpg;
+    int64_t pool;
+    OSDMapRef curmap = service->get_osdmap();
+    int r;
+
+    string poolstr;
+
+    cmd_getval(service->cct, cmdmap, "pool", poolstr);
+    pool = curmap->const_lookup_pg_pool_name(poolstr.c_str());
+    //If we can't find it by name then maybe id specified
+    if (pool < 0 && isdigit(poolstr[0]))
+      pool = atoll(poolstr.c_str());
+    if (pool < 0) {
+      ss << "Invalid pool" << poolstr;
+      return;
+    }
+    r = -1;
+    string objname, nspace;
+    cmd_getval(service->cct, cmdmap, "objname", objname);
+    std::size_t found = objname.find_first_of('/');
+    if (found != string::npos) {
+      nspace = objname.substr(0, found);
+      objname = objname.substr(found+1);
+    }
+    object_locator_t oloc(pool, nspace);
+    r = curmap->object_locator_to_pg(object_t(objname), oloc,  rawpg);
+
+    if (r < 0) {
+      ss << "Invalid namespace/objname";
+      return;
+    }
+    if (curmap->pg_is_ec(rawpg)) {
+      ss << "Must not call on ec pool";
+      return;
+    }
+    spg_t pgid = spg_t(curmap->raw_pg_to_pg(rawpg), ghobject_t::no_shard());
+
+    hobject_t obj(object_t(objname), string(""), CEPH_NOSNAP, rawpg.ps(), pool, nspace);
+    ObjectStore::Transaction t;
+
+    if (command == "setomapval") {
+      map newattrs;
+      bufferlist val;
+      string key, valstr;
+      cmd_getval(service->cct, cmdmap, "key", key);
+      cmd_getval(service->cct, cmdmap, "val", valstr);
+
+      val.append(valstr);
+      newattrs[key] = val;
+      t.omap_setkeys(coll_t(pgid), obj, newattrs);
+      r = store->apply_transaction(t);
+      if (r < 0)
+        ss << "error=" << r;
+      else
+        ss << "ok";
+    } else if (command == "rmomapkey") {
+      string key;
+      set keys;
+      cmd_getval(service->cct, cmdmap, "key", key);
+
+      keys.insert(key);
+      t.omap_rmkeys(coll_t(pgid), obj, keys);
+      r = store->apply_transaction(t);
+      if (r < 0)
+        ss << "error=" << r;
+      else
+        ss << "ok";
+    } else if (command == "setomapheader") {
+      bufferlist newheader;
+      string headerstr;
+
+      cmd_getval(service->cct, cmdmap, "header", headerstr);
+      newheader.append(headerstr);
+      t.omap_setheader(coll_t(pgid), obj, newheader);
+      r = store->apply_transaction(t);
+      if (r < 0)
+        ss << "error=" << r;
+      else
+        ss << "ok";
+    } else if (command == "getomap") {
+      //Debug: Output entire omap
+      bufferlist hdrbl;
+      map keyvals;
+      r = store->omap_get(coll_t(pgid), obj, &hdrbl, &keyvals);
+      if (r >= 0) {
+          ss << "header=" << string(hdrbl.c_str(), hdrbl.length());
+          for (map::iterator it = keyvals.begin();
+              it != keyvals.end(); ++it)
+            ss << " key=" << (*it).first << " val="
+               << string((*it).second.c_str(), (*it).second.length());
+      } else {
+          ss << "error=" << r;
+      }
+    } else if (command == "truncobj") {
+      int64_t trunclen;
+      cmd_getval(service->cct, cmdmap, "len", trunclen);
+      t.truncate(coll_t(pgid), obj, trunclen);
+      r = store->apply_transaction(t);
+      if (r < 0)
+	ss << "error=" << r;
+      else
+	ss << "ok";
+    } else if (command == "injectdataerr") {
+      store->inject_data_error(obj);
+      ss << "ok";
+    } else if (command == "injectmdataerr") {
+      store->inject_mdata_error(obj);
+      ss << "ok";
+    }
+    return;
+  }
+  ss << "Internal error - command=" << command;
+  return;
+}
+
+// =========================================
+bool remove_dir(
+  CephContext *cct,
+  ObjectStore *store, SnapMapper *mapper,
+  OSDriver *osdriver,
+  ObjectStore::Sequencer *osr,
+  coll_t coll, DeletingStateRef dstate,
+  ThreadPool::TPHandle &handle)
+{
+  vector olist;
+  int64_t num = 0;
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  ghobject_t next;
+  while (!next.is_max()) {
+    handle.reset_tp_timeout();
+    store->collection_list_partial(
+      coll,
+      next,
+      store->get_ideal_list_min(),
+      store->get_ideal_list_max(),
+      0,
+      &olist,
+      &next);
+    for (vector::iterator i = olist.begin();
+	 i != olist.end();
+	 ++i, ++num) {
+      OSDriver::OSTransaction _t(osdriver->get_transaction(t));
+      int r = mapper->remove_oid(i->hobj, &_t);
+      if (r != 0 && r != -ENOENT) {
+	assert(0);
+      }
+      t->remove(coll, *i);
+      if (num >= cct->_conf->osd_target_transaction_size) {
+	C_SaferCond waiter;
+	store->queue_transaction(osr, t, &waiter);
+	bool cont = dstate->pause_clearing();
+	handle.suspend_tp_timeout();
+	waiter.wait();
+	handle.reset_tp_timeout();
+	if (cont)
+	  cont = dstate->resume_clearing();
+	delete t;
+	if (!cont)
+	  return false;
+	t = new ObjectStore::Transaction;
+	num = 0;
+      }
+    }
+    olist.clear();
+  }
+
+  C_SaferCond waiter;
+  store->queue_transaction(osr, t, &waiter);
+  bool cont = dstate->pause_clearing();
+  handle.suspend_tp_timeout();
+  waiter.wait();
+  handle.reset_tp_timeout();
+  if (cont)
+    cont = dstate->resume_clearing();
+  delete t;
+  return cont;
+}
+
+void OSD::RemoveWQ::_process(
+  pair item,
+  ThreadPool::TPHandle &handle)
+{
+  PGRef pg(item.first);
+  SnapMapper &mapper = pg->snap_mapper;
+  OSDriver &driver = pg->osdriver;
+  coll_t coll = coll_t(pg->info.pgid);
+  pg->osr->flush();
+
+  if (!item.second->start_clearing())
+    return;
+
+  list colls_to_remove;
+  pg->get_colls(&colls_to_remove);
+  for (list::iterator i = colls_to_remove.begin();
+       i != colls_to_remove.end();
+       ++i) {
+    bool cont = remove_dir(
+      pg->cct, store, &mapper, &driver, pg->osr.get(), *i, item.second,
+      handle);
+    if (!cont)
+      return;
+  }
+
+  if (!item.second->start_deleting())
+    return;
+
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  PGLog::clear_info_log(
+    pg->info.pgid,
+    OSD::make_infos_oid(),
+    pg->log_oid,
+    t);
+
+  for (list::iterator i = colls_to_remove.begin();
+       i != colls_to_remove.end();
+       ++i) {
+    t->remove_collection(*i);
+  }
+
+  // We need the sequencer to stick around until the op is complete
+  store->queue_transaction(
+    pg->osr.get(),
+    t,
+    0, // onapplied
+    0, // oncommit
+    0, // onreadable sync
+    new ObjectStore::C_DeleteTransactionHolder(
+      t, pg), // oncomplete
+    TrackedOpRef());
+
+  item.second->finish_deleting();
+}
+// =========================================
+
+void OSD::do_mon_report()
+{
+  dout(7) << "do_mon_report" << dendl;
+
+  utime_t now(ceph_clock_now(cct));
+  last_mon_report = now;
+
+  // do any pending reports
+  send_alive();
+  service.send_pg_temp();
+  send_failures();
+  send_pg_stats(now);
+}
+
+void OSD::ms_handle_connect(Connection *con)
+{
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    Mutex::Locker l(osd_lock);
+    if (is_stopping())
+      return;
+    dout(10) << "ms_handle_connect on mon" << dendl;
+    if (is_booting()) {
+      start_boot();
+    } else {
+      send_alive();
+      service.send_pg_temp();
+      send_failures();
+      send_pg_stats(ceph_clock_now(cct));
+
+      monc->sub_want("osd_pg_creates", 0, CEPH_SUBSCRIBE_ONETIME);
+      monc->renew_subs();
+    }
+  }
+}
+
+bool OSD::ms_handle_reset(Connection *con)
+{
+  OSD::Session *session = (OSD::Session *)con->get_priv();
+  dout(1) << "ms_handle_reset con " << con << " session " << session << dendl;
+  if (!session)
+    return false;
+  session->wstate.reset();
+  session->con.reset(NULL);  // break con <-> session ref cycle
+  session->put();
+  return true;
+}
+
+struct C_OSD_GetVersion : public Context {
+  OSD *osd;
+  uint64_t oldest, newest;
+  C_OSD_GetVersion(OSD *o) : osd(o), oldest(0), newest(0) {}
+  void finish(int r) {
+    if (r >= 0)
+      osd->_maybe_boot(oldest, newest);
+  }
+};
+
+void OSD::start_boot()
+{
+  dout(10) << "start_boot - have maps " << superblock.oldest_map
+	   << ".." << superblock.newest_map << dendl;
+  C_OSD_GetVersion *c = new C_OSD_GetVersion(this);
+  monc->get_version("osdmap", &c->newest, &c->oldest, c);
+}
+
+void OSD::_maybe_boot(epoch_t oldest, epoch_t newest)
+{
+  Mutex::Locker l(osd_lock);
+  if (is_stopping())
+    return;
+  dout(10) << "_maybe_boot mon has osdmaps " << oldest << ".." << newest << dendl;
+
+  if (is_initializing()) {
+    dout(10) << "still initializing" << dendl;
+    return;
+  }
+
+  // if our map within recent history, try to add ourselves to the osdmap.
+  if (osdmap->test_flag(CEPH_OSDMAP_NOUP)) {
+    dout(5) << "osdmap NOUP flag is set, waiting for it to clear" << dendl;
+  } else if (is_waiting_for_healthy() || !_is_healthy()) {
+    // if we are not healthy, do not mark ourselves up (yet)
+    dout(1) << "not healthy; waiting to boot" << dendl;
+    if (!is_waiting_for_healthy())
+      start_waiting_for_healthy();
+    // send pings sooner rather than later
+    heartbeat_kick();
+  } else if (osdmap->get_epoch() >= oldest - 1 &&
+	     osdmap->get_epoch() + cct->_conf->osd_map_message_max > newest) {
+    _send_boot();
+    return;
+  }
+  
+  // get all the latest maps
+  if (osdmap->get_epoch() + 1 >= oldest)
+    osdmap_subscribe(osdmap->get_epoch() + 1, true);
+  else
+    osdmap_subscribe(oldest - 1, true);
+}
+
+void OSD::start_waiting_for_healthy()
+{
+  dout(1) << "start_waiting_for_healthy" << dendl;
+  state = STATE_WAITING_FOR_HEALTHY;
+  last_heartbeat_resample = utime_t();
+}
+
+bool OSD::_is_healthy()
+{
+  if (!cct->get_heartbeat_map()->is_healthy()) {
+    dout(1) << "is_healthy false -- internal heartbeat failed" << dendl;
+    return false;
+  }
+
+  if (is_waiting_for_healthy()) {
+    Mutex::Locker l(heartbeat_lock);
+    utime_t cutoff = ceph_clock_now(cct);
+    cutoff -= cct->_conf->osd_heartbeat_grace;
+    int num = 0, up = 0;
+    for (map::iterator p = heartbeat_peers.begin();
+	 p != heartbeat_peers.end();
+	 ++p) {
+      if (p->second.is_healthy(cutoff))
+	++up;
+      ++num;
+    }
+    if ((float)up < (float)num * cct->_conf->osd_heartbeat_min_healthy_ratio) {
+      dout(1) << "is_healthy false -- only " << up << "/" << num << " up peers (less than 1/3)" << dendl;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+void OSD::_send_boot()
+{
+  dout(10) << "_send_boot" << dendl;
+  entity_addr_t cluster_addr = cluster_messenger->get_myaddr();
+  if (cluster_addr.is_blank_ip()) {
+    int port = cluster_addr.get_port();
+    cluster_addr = client_messenger->get_myaddr();
+    cluster_addr.set_port(port);
+    cluster_messenger->set_addr_unknowns(cluster_addr);
+    dout(10) << " assuming cluster_addr ip matches client_addr" << dendl;
+  }
+  entity_addr_t hb_back_addr = hb_back_server_messenger->get_myaddr();
+  if (hb_back_addr.is_blank_ip()) {
+    int port = hb_back_addr.get_port();
+    hb_back_addr = cluster_addr;
+    hb_back_addr.set_port(port);
+    hb_back_server_messenger->set_addr_unknowns(hb_back_addr);
+    dout(10) << " assuming hb_back_addr ip matches cluster_addr" << dendl;
+  }
+  entity_addr_t hb_front_addr = hb_front_server_messenger->get_myaddr();
+  if (hb_front_addr.is_blank_ip()) {
+    int port = hb_front_addr.get_port();
+    hb_front_addr = client_messenger->get_myaddr();
+    hb_front_addr.set_port(port);
+    hb_front_server_messenger->set_addr_unknowns(hb_front_addr);
+    dout(10) << " assuming hb_front_addr ip matches client_addr" << dendl;
+  }
+
+  MOSDBoot *mboot = new MOSDBoot(superblock, boot_epoch, hb_back_addr, hb_front_addr, cluster_addr);
+  dout(10) << " client_addr " << client_messenger->get_myaddr()
+	   << ", cluster_addr " << cluster_addr
+	   << ", hb_back_addr " << hb_back_addr
+	   << ", hb_front_addr " << hb_front_addr
+	   << dendl;
+  _collect_metadata(&mboot->metadata);
+  monc->send_mon_message(mboot);
+}
+
+void OSD::_collect_metadata(map *pm)
+{
+  (*pm)["ceph_version"] = pretty_version_to_str();
+
+  // config info
+  (*pm)["osd_data"] = dev_path;
+  (*pm)["osd_journal"] = journal_path;
+  (*pm)["front_addr"] = stringify(client_messenger->get_myaddr());
+  (*pm)["back_addr"] = stringify(cluster_messenger->get_myaddr());
+  (*pm)["hb_front_addr"] = stringify(hb_front_server_messenger->get_myaddr());
+  (*pm)["hb_back_addr"] = stringify(hb_back_server_messenger->get_myaddr());
+
+  // kernel info
+  struct utsname u;
+  int r = uname(&u);
+  if (r >= 0) {
+    (*pm)["os"] = u.sysname;
+    (*pm)["kernel_version"] = u.release;
+    (*pm)["kernel_description"] = u.version;
+    (*pm)["hostname"] = u.nodename;
+    (*pm)["arch"] = u.machine;
+  }
+
+  // memory
+  FILE *f = fopen("/proc/meminfo", "r");
+  if (f) {
+    char buf[100];
+    while (!feof(f)) {
+      char *line = fgets(buf, sizeof(buf), f);
+      if (!line)
+	break;
+      char key[40];
+      long long value;
+      int r = sscanf(line, "%s %lld", key, &value);
+      if (r == 2) {
+	if (strcmp(key, "MemTotal:") == 0)
+	  (*pm)["mem_total_kb"] = stringify(value);
+	else if (strcmp(key, "SwapTotal:") == 0)
+	  (*pm)["mem_swap_kb"] = stringify(value);
+      }
+    }
+    fclose(f);
+  }
+
+  // processor
+  f = fopen("/proc/cpuinfo", "r");
+  if (f) {
+    char buf[100];
+    while (!feof(f)) {
+      char *line = fgets(buf, sizeof(buf), f);
+      if (!line)
+	break;
+      if (strncmp(line, "model name", 10) == 0) {
+	char *c = strchr(buf, ':');
+	c++;
+	while (*c == ' ')
+	  ++c;
+	char *nl = c;
+	while (*nl != '\n')
+	  ++nl;
+	*nl = '\0';
+	(*pm)["cpu"] = c;
+	break;
+      }
+    }
+    fclose(f);
+  }
+
+  // distro info
+  f = fopen("/etc/lsb-release", "r");
+  if (f) {
+    char buf[100];
+    while (!feof(f)) {
+      char *line = fgets(buf, sizeof(buf), f);
+      if (!line)
+	break;
+      char *eq = strchr(buf, '=');
+      if (!eq)
+	break;
+      *eq = '\0';
+      ++eq;
+      while (*eq == '\"')
+	++eq;
+      while (*eq && (eq[strlen(eq)-1] == '\n' ||
+		     eq[strlen(eq)-1] == '\"'))
+	eq[strlen(eq)-1] = '\0';
+      if (strcmp(buf, "DISTRIB_ID") == 0)
+	(*pm)["distro"] = eq;
+      else if (strcmp(buf, "DISTRIB_RELEASE") == 0)
+	(*pm)["distro_version"] = eq;
+      else if (strcmp(buf, "DISTRIB_CODENAME") == 0)
+	(*pm)["distro_codename"] = eq;
+      else if (strcmp(buf, "DISTRIB_DESCRIPTION") == 0)
+	(*pm)["distro_description"] = eq;
+    }
+    fclose(f);
+  }
+
+  dout(10) << __func__ << " " << *pm << dendl;
+}
+
+void OSD::queue_want_up_thru(epoch_t want)
+{
+  map_lock.get_read();
+  epoch_t cur = osdmap->get_up_thru(whoami);
+  if (want > up_thru_wanted) {
+    dout(10) << "queue_want_up_thru now " << want << " (was " << up_thru_wanted << ")" 
+	     << ", currently " << cur
+	     << dendl;
+    up_thru_wanted = want;
+
+    // expedite, a bit.  WARNING this will somewhat delay other mon queries.
+    last_mon_report = ceph_clock_now(cct);
+    send_alive();
+  } else {
+    dout(10) << "queue_want_up_thru want " << want << " <= queued " << up_thru_wanted 
+	     << ", currently " << cur
+	     << dendl;
+  }
+  map_lock.put_read();
+}
+
+void OSD::send_alive()
+{
+  if (!osdmap->exists(whoami))
+    return;
+  epoch_t up_thru = osdmap->get_up_thru(whoami);
+  dout(10) << "send_alive up_thru currently " << up_thru << " want " << up_thru_wanted << dendl;
+  if (up_thru_wanted > up_thru) {
+    up_thru_pending = up_thru_wanted;
+    dout(10) << "send_alive want " << up_thru_wanted << dendl;
+    monc->send_mon_message(new MOSDAlive(osdmap->get_epoch(), up_thru_wanted));
+  }
+}
+
+void OSDService::send_message_osd_cluster(int peer, Message *m, epoch_t from_epoch)
+{
+  Mutex::Locker l(pre_publish_lock);
+
+  // service map is always newer/newest
+  assert(from_epoch <= next_osdmap->get_epoch());
+
+  if (next_osdmap->is_down(peer) ||
+      next_osdmap->get_info(peer).up_from > from_epoch) {
+    m->put();
+    return;
+  }
+  const entity_inst_t& peer_inst = next_osdmap->get_cluster_inst(peer);
+  Connection *peer_con = osd->cluster_messenger->get_connection(peer_inst).get();
+  osd->_share_map_outgoing(peer, peer_con, next_osdmap);
+  osd->cluster_messenger->send_message(m, peer_inst);
+}
+
+ConnectionRef OSDService::get_con_osd_cluster(int peer, epoch_t from_epoch)
+{
+  Mutex::Locker l(pre_publish_lock);
+
+  // service map is always newer/newest
+  assert(from_epoch <= next_osdmap->get_epoch());
+
+  if (next_osdmap->is_down(peer) ||
+      next_osdmap->get_info(peer).up_from > from_epoch) {
+    return NULL;
+  }
+  return osd->cluster_messenger->get_connection(next_osdmap->get_cluster_inst(peer));
+}
+
+pair OSDService::get_con_osd_hb(int peer, epoch_t from_epoch)
+{
+  Mutex::Locker l(pre_publish_lock);
+
+  // service map is always newer/newest
+  assert(from_epoch <= next_osdmap->get_epoch());
+
+  pair ret;
+  if (next_osdmap->is_down(peer) ||
+      next_osdmap->get_info(peer).up_from > from_epoch) {
+    return ret;
+  }
+  ret.first = osd->hbclient_messenger->get_connection(next_osdmap->get_hb_back_inst(peer));
+  if (next_osdmap->get_hb_front_addr(peer) != entity_addr_t())
+    ret.second = osd->hbclient_messenger->get_connection(next_osdmap->get_hb_front_inst(peer));
+  return ret;
+}
+
+void OSDService::queue_want_pg_temp(pg_t pgid, vector& want)
+{
+  Mutex::Locker l(pg_temp_lock);
+  pg_temp_wanted[pgid] = want;
+}
+
+void OSDService::send_pg_temp()
+{
+  Mutex::Locker l(pg_temp_lock);
+  if (pg_temp_wanted.empty())
+    return;
+  dout(10) << "send_pg_temp " << pg_temp_wanted << dendl;
+  MOSDPGTemp *m = new MOSDPGTemp(osdmap->get_epoch());
+  m->pg_temp = pg_temp_wanted;
+  monc->send_mon_message(m);
+}
+
+void OSD::send_failures()
+{
+  assert(osd_lock.is_locked());
+  bool locked = false;
+  if (!failure_queue.empty()) {
+    heartbeat_lock.Lock();
+    locked = true;
+  }
+  utime_t now = ceph_clock_now(cct);
+  while (!failure_queue.empty()) {
+    int osd = failure_queue.begin()->first;
+    int failed_for = (int)(double)(now - failure_queue.begin()->second);
+    entity_inst_t i = osdmap->get_inst(osd);
+    monc->send_mon_message(new MOSDFailure(monc->get_fsid(), i, failed_for, osdmap->get_epoch()));
+    failure_pending[osd] = i;
+    failure_queue.erase(osd);
+  }
+  if (locked) heartbeat_lock.Unlock();
+}
+
+void OSD::send_still_alive(epoch_t epoch, const entity_inst_t &i)
+{
+  MOSDFailure *m = new MOSDFailure(monc->get_fsid(), i, 0, epoch);
+  m->is_failed = false;
+  monc->send_mon_message(m);
+}
+
+void OSD::send_pg_stats(const utime_t &now)
+{
+  assert(osd_lock.is_locked());
+
+  dout(20) << "send_pg_stats" << dendl;
+
+  stat_lock.Lock();
+  osd_stat_t cur_stat = osd_stat;
+  stat_lock.Unlock();
+
+  cur_stat.fs_perf_stat = store->get_cur_stats();
+   
+  pg_stat_queue_lock.Lock();
+
+  if (osd_stat_updated || !pg_stat_queue.empty()) {
+    last_pg_stats_sent = now;
+    osd_stat_updated = false;
+
+    dout(10) << "send_pg_stats - " << pg_stat_queue.size() << " pgs updated" << dendl;
+
+    utime_t had_for(now);
+    had_for -= had_map_since;
+
+    MPGStats *m = new MPGStats(monc->get_fsid(), osdmap->get_epoch(), had_for);
+    m->set_tid(++pg_stat_tid);
+    m->osd_stat = cur_stat;
+
+    xlist::iterator p = pg_stat_queue.begin();
+    while (!p.end()) {
+      PG *pg = *p;
+      ++p;
+      if (!pg->is_primary()) {  // we hold map_lock; role is stable.
+	pg->stat_queue_item.remove_myself();
+	pg->put("pg_stat_queue");
+	continue;
+      }
+      pg->pg_stats_publish_lock.Lock();
+      if (pg->pg_stats_publish_valid) {
+	m->pg_stat[pg->info.pgid.pgid] = pg->pg_stats_publish;
+	dout(25) << " sending " << pg->info.pgid << " " << pg->pg_stats_publish.reported_epoch << ":"
+		 << pg->pg_stats_publish.reported_seq << dendl;
+      } else {
+	dout(25) << " NOT sending " << pg->info.pgid << " " << pg->pg_stats_publish.reported_epoch << ":"
+		 << pg->pg_stats_publish.reported_seq << ", not valid" << dendl;
+      }
+      pg->pg_stats_publish_lock.Unlock();
+    }
+
+    if (!outstanding_pg_stats) {
+      outstanding_pg_stats = true;
+      last_pg_stats_ack = ceph_clock_now(cct);
+    }
+    monc->send_mon_message(m);
+  }
+
+  pg_stat_queue_lock.Unlock();
+}
+
+void OSD::handle_pg_stats_ack(MPGStatsAck *ack)
+{
+  dout(10) << "handle_pg_stats_ack " << dendl;
+
+  if (!require_mon_peer(ack)) {
+    ack->put();
+    return;
+  }
+
+  last_pg_stats_ack = ceph_clock_now(cct);
+
+  pg_stat_queue_lock.Lock();
+
+  if (ack->get_tid() > pg_stat_tid_flushed) {
+    pg_stat_tid_flushed = ack->get_tid();
+    pg_stat_queue_cond.Signal();
+  }
+
+  xlist::iterator p = pg_stat_queue.begin();
+  while (!p.end()) {
+    PG *pg = *p;
+    PGRef _pg(pg);
+    ++p;
+
+    if (ack->pg_stat.count(pg->info.pgid.pgid)) {
+      pair acked = ack->pg_stat[pg->info.pgid.pgid];
+      pg->pg_stats_publish_lock.Lock();
+      if (acked.first == pg->pg_stats_publish.reported_seq &&
+	  acked.second == pg->pg_stats_publish.reported_epoch) {
+	dout(25) << " ack on " << pg->info.pgid << " " << pg->pg_stats_publish.reported_epoch
+		 << ":" << pg->pg_stats_publish.reported_seq << dendl;
+	pg->stat_queue_item.remove_myself();
+	pg->put("pg_stat_queue");
+      } else {
+	dout(25) << " still pending " << pg->info.pgid << " " << pg->pg_stats_publish.reported_epoch
+		 << ":" << pg->pg_stats_publish.reported_seq << " > acked " << acked << dendl;
+      }
+      pg->pg_stats_publish_lock.Unlock();
+    } else {
+      dout(30) << " still pending " << pg->info.pgid << " " << pg->pg_stats_publish.reported_epoch
+	       << ":" << pg->pg_stats_publish.reported_seq << dendl;
+    }
+  }
+  
+  if (!pg_stat_queue.size()) {
+    outstanding_pg_stats = false;
+  }
+
+  pg_stat_queue_lock.Unlock();
+
+  ack->put();
+}
+
+void OSD::flush_pg_stats()
+{
+  dout(10) << "flush_pg_stats" << dendl;
+  utime_t now = ceph_clock_now(cct);
+  send_pg_stats(now);
+
+  osd_lock.Unlock();
+
+  pg_stat_queue_lock.Lock();
+  uint64_t tid = pg_stat_tid;
+  dout(10) << "flush_pg_stats waiting for stats tid " << tid << " to flush" << dendl;
+  while (tid > pg_stat_tid_flushed)
+    pg_stat_queue_cond.Wait(pg_stat_queue_lock);
+  dout(10) << "flush_pg_stats finished waiting for stats tid " << tid << " to flush" << dendl;
+  pg_stat_queue_lock.Unlock();
+
+  osd_lock.Lock();
+}
+
+
+void OSD::handle_command(MMonCommand *m)
+{
+  if (!require_mon_peer(m))
+    return;
+
+  Command *c = new Command(m->cmd, m->get_tid(), m->get_data(), NULL);
+  command_wq.queue(c);
+  m->put();
+}
+
+void OSD::handle_command(MCommand *m)
+{
+  ConnectionRef con = m->get_connection();
+  Session *session = static_cast(con->get_priv());
+  if (!session) {
+    client_messenger->send_message(new MCommandReply(m, -EPERM), con);
+    m->put();
+    return;
+  }
+
+  OSDCap& caps = session->caps;
+  session->put();
+
+  if (!caps.allow_all() || m->get_source().is_mon()) {
+    client_messenger->send_message(new MCommandReply(m, -EPERM), con);
+    m->put();
+    return;
+  }
+
+  Command *c = new Command(m->cmd, m->get_tid(), m->get_data(), con.get());
+  command_wq.queue(c);
+
+  m->put();
+}
+
+struct OSDCommand {
+  string cmdstring;
+  string helpstring;
+  string module;
+  string perm;
+  string availability;
+} osd_commands[] = {
+
+#define COMMAND(parsesig, helptext, module, perm, availability) \
+  {parsesig, helptext, module, perm, availability},
+
+// yes, these are really pg commands, but there's a limit to how
+// much work it's worth.  The OSD returns all of them.  Make this
+// form (pg  ) valid only for the cli. 
+// Rest uses "tell  "
+
+COMMAND("pg " \
+	"name=pgid,type=CephPgid " \
+	"name=cmd,type=CephChoices,strings=query", \
+	"show details of a specific pg", "osd", "r", "cli")
+COMMAND("pg " \
+	"name=pgid,type=CephPgid " \
+	"name=cmd,type=CephChoices,strings=mark_unfound_lost " \
+	"name=mulcmd,type=CephChoices,strings=revert|delete", \
+	"mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available",
+	"osd", "rw", "cli")
+COMMAND("pg " \
+	"name=pgid,type=CephPgid " \
+	"name=cmd,type=CephChoices,strings=list_missing " \
+	"name=offset,type=CephString,req=false",
+	"list missing objects on this pg, perhaps starting at an offset given in JSON",
+	"osd", "r", "cli")
+
+// new form: tell   for both cli and rest 
+
+COMMAND("query",
+	"show details of a specific pg", "osd", "r", "cli,rest")
+COMMAND("mark_unfound_lost " \
+	"name=mulcmd,type=CephChoices,strings=revert|delete", \
+	"mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available",
+	"osd", "rw", "cli,rest")
+COMMAND("list_missing " \
+	"name=offset,type=CephString,req=false",
+	"list missing objects on this pg, perhaps starting at an offset given in JSON",
+	"osd", "r", "cli,rest")
+
+// tell  commands.  Validation of osd.n must be special-cased in client
+COMMAND("version", "report version of OSD", "osd", "r", "cli,rest")
+COMMAND("injectargs " \
+	"name=injected_args,type=CephString,n=N",
+	"inject configuration arguments into running OSD",
+	"osd", "rw", "cli,rest")
+COMMAND("bench " \
+	"name=count,type=CephInt,req=false " \
+	"name=size,type=CephInt,req=false ", \
+	"OSD benchmark: write  -byte objects, " \
+	"(default 1G size 4MB). Results in log.",
+	"osd", "rw", "cli,rest")
+COMMAND("flush_pg_stats", "flush pg stats", "osd", "rw", "cli,rest")
+COMMAND("heap " \
+	"name=heapcmd,type=CephChoices,strings=dump|start_profiler|stop_profiler|release|stats", \
+	"show heap usage info (available only if compiled with tcmalloc)", \
+	"osd", "rw", "cli,rest")
+COMMAND("debug_dump_missing " \
+	"name=filename,type=CephFilepath",
+	"dump missing objects to a named file", "osd", "r", "cli,rest")
+COMMAND("debug kick_recovery_wq " \
+	"name=delay,type=CephInt,range=0",
+	"set osd_recovery_delay_start to ", "osd", "rw", "cli,rest")
+COMMAND("cpu_profiler " \
+	"name=arg,type=CephChoices,strings=status|flush",
+	"run cpu profiling on daemon", "osd", "rw", "cli,rest")
+COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics",
+	"osd", "r", "cli,rest")
+COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics",
+	"osd", "rw", "cli,rest")
+};
+
+void OSD::do_command(Connection *con, ceph_tid_t tid, vector& cmd, bufferlist& data)
+{
+  int r = 0;
+  stringstream ss, ds;
+  string rs;
+  bufferlist odata;
+
+  dout(20) << "do_command tid " << tid << " " << cmd << dendl;
+
+  map cmdmap;
+  string prefix;
+  string format;
+  string pgidstr;
+  boost::scoped_ptr f;
+
+  if (cmd.empty()) {
+    ss << "no command given";
+    goto out;
+  }
+
+  if (!cmdmap_from_json(cmd, &cmdmap, ss)) {
+    r = -EINVAL;
+    goto out;
+  }
+
+  cmd_getval(cct, cmdmap, "prefix", prefix);
+
+  if (prefix == "get_command_descriptions") {
+    int cmdnum = 0;
+    JSONFormatter *f = new JSONFormatter();
+    f->open_object_section("command_descriptions");
+    for (OSDCommand *cp = osd_commands;
+	 cp < &osd_commands[ARRAY_SIZE(osd_commands)]; cp++) {
+
+      ostringstream secname;
+      secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
+      dump_cmddesc_to_json(f, secname.str(), cp->cmdstring, cp->helpstring,
+			   cp->module, cp->perm, cp->availability);
+      cmdnum++;
+    }
+    f->close_section();	// command_descriptions
+
+    f->flush(ds);
+    delete f;
+    goto out;
+  }
+
+  cmd_getval(cct, cmdmap, "format", format);
+  f.reset(new_formatter(format));
+
+  if (prefix == "version") {
+    if (f) {
+      f->open_object_section("version");
+      f->dump_string("version", pretty_version_to_str());
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << pretty_version_to_str();
+    }
+    goto out;
+  }
+  else if (prefix == "injectargs") {
+    vector argsvec;
+    cmd_getval(cct, cmdmap, "injected_args", argsvec);
+
+    if (argsvec.empty()) {
+      r = -EINVAL;
+      ss << "ignoring empty injectargs";
+      goto out;
+    }
+    string args = argsvec.front();
+    for (vector::iterator a = ++argsvec.begin(); a != argsvec.end(); ++a)
+      args += " " + *a;
+    osd_lock.Unlock();
+    cct->_conf->injectargs(args, &ss);
+    osd_lock.Lock();
+  }
+
+  // either 'pg  ' or
+  // 'tell ' (which comes in without any of that prefix)?
+
+  else if (prefix == "pg" ||
+	   (cmd_getval(cct, cmdmap, "pgid", pgidstr) &&
+	     (prefix == "query" ||
+	      prefix == "mark_unfound_lost" ||
+	      prefix == "list_missing")
+	   )) {
+    pg_t pgid;
+
+    if (!cmd_getval(cct, cmdmap, "pgid", pgidstr)) {
+      ss << "no pgid specified";
+      r = -EINVAL;
+    } else if (!pgid.parse(pgidstr.c_str())) {
+      ss << "couldn't parse pgid '" << pgidstr << "'";
+      r = -EINVAL;
+    } else {
+      spg_t pcand;
+      if (osdmap->get_primary_shard(pgid, &pcand) &&
+	  _have_pg(pcand)) {
+	PG *pg = _lookup_lock_pg(pcand);
+	assert(pg);
+	if (pg->is_primary()) {
+	  // simulate pg  cmd= for pg->do-command
+	  if (prefix != "pg")
+	    cmd_putval(cct, cmdmap, "cmd", prefix);
+	  r = pg->do_command(cmdmap, ss, data, odata);
+	} else {
+	  ss << "not primary for pgid " << pgid;
+
+	  // send them the latest diff to ensure they realize the mapping
+	  // has changed.
+	  send_incremental_map(osdmap->get_epoch() - 1, con);
+
+	  // do not reply; they will get newer maps and realize they
+	  // need to resend.
+	  pg->unlock();
+	  return;
+	}
+	pg->unlock();
+      } else {
+	ss << "i don't have pgid " << pgid;
+	r = -ENOENT;
+      }
+    }
+  }
+
+  else if (prefix == "bench") {
+    int64_t count;
+    int64_t bsize;
+    // default count 1G, size 4MB
+    cmd_getval(cct, cmdmap, "count", count, (int64_t)1 << 30);
+    cmd_getval(cct, cmdmap, "size", bsize, (int64_t)4 << 20);
+
+    uint32_t duration = g_conf->osd_bench_duration;
+
+    if (bsize > (int64_t) g_conf->osd_bench_max_block_size) {
+      // let us limit the block size because the next checks rely on it
+      // having a sane value.  If we allow any block size to be set things
+      // can still go sideways.
+      ss << "block 'size' values are capped at "
+         << prettybyte_t(g_conf->osd_bench_max_block_size) << ". If you wish to use"
+         << " a higher value, please adjust 'osd_bench_max_block_size'";
+      r = -EINVAL;
+      goto out;
+    } else if (bsize < (int64_t) (1 << 20)) {
+      // entering the realm of small block sizes.
+      // limit the count to a sane value, assuming a configurable amount of
+      // IOPS and duration, so that the OSD doesn't get hung up on this,
+      // preventing timeouts from going off
+      int64_t max_count =
+        bsize * duration * g_conf->osd_bench_small_size_max_iops;
+      if (count > max_count) {
+        ss << "'count' values greater than " << max_count
+           << " for a block size of " << prettybyte_t(bsize) << ", assuming "
+           << g_conf->osd_bench_small_size_max_iops << " IOPS,"
+           << " for " << duration << " seconds,"
+           << " can cause ill effects on osd. "
+           << " Please adjust 'osd_bench_small_size_max_iops' with a higher"
+           << " value if you wish to use a higher 'count'.";
+        r = -EINVAL;
+        goto out;
+      }
+    } else {
+      // 1MB block sizes are big enough so that we get more stuff done.
+      // However, to avoid the osd from getting hung on this and having
+      // timers being triggered, we are going to limit the count assuming
+      // a configurable throughput and duration.
+      int64_t max_count =
+        g_conf->osd_bench_large_size_max_throughput * duration;
+      if (count > max_count) {
+        ss << "'count' values greater than " << max_count
+           << " for a block size of " << prettybyte_t(bsize) << ", assuming "
+           << prettybyte_t(g_conf->osd_bench_large_size_max_throughput) << "/s,"
+           << " for " << duration << " seconds,"
+           << " can cause ill effects on osd. "
+           << " Please adjust 'osd_bench_large_size_max_throughput'"
+           << " with a higher value if you wish to use a higher 'count'.";
+        r = -EINVAL;
+        goto out;
+      }
+    }
+
+    dout(1) << " bench count " << count
+            << " bsize " << prettybyte_t(bsize) << dendl;
+
+    bufferlist bl;
+    bufferptr bp(bsize);
+    bp.zero();
+    bl.push_back(bp);
+
+    ObjectStore::Transaction *cleanupt = new ObjectStore::Transaction;
+
+    store->sync_and_flush();
+    utime_t start = ceph_clock_now(cct);
+    for (int64_t pos = 0; pos < count; pos += bsize) {
+      char nm[30];
+      snprintf(nm, sizeof(nm), "disk_bw_test_%lld", (long long)pos);
+      object_t oid(nm);
+      hobject_t soid(sobject_t(oid, 0));
+      ObjectStore::Transaction *t = new ObjectStore::Transaction;
+      t->write(coll_t::META_COLL, soid, 0, bsize, bl);
+      store->queue_transaction_and_cleanup(NULL, t);
+      cleanupt->remove(coll_t::META_COLL, soid);
+    }
+    store->sync_and_flush();
+    utime_t end = ceph_clock_now(cct);
+
+    // clean up
+    store->queue_transaction_and_cleanup(NULL, cleanupt);
+
+    uint64_t rate = (double)count / (end - start);
+    if (f) {
+      f->open_object_section("osd_bench_results");
+      f->dump_int("bytes_written", count);
+      f->dump_int("blocksize", bsize);
+      f->dump_float("bytes_per_sec", rate);
+      f->close_section();
+      f->flush(ss);
+    } else {
+      ss << "bench: wrote " << prettybyte_t(count)
+	 << " in blocks of " << prettybyte_t(bsize) << " in "
+	 << (end-start) << " sec at " << prettybyte_t(rate) << "/sec";
+    }
+  }
+
+  else if (prefix == "flush_pg_stats") {
+    flush_pg_stats();
+  }
+  
+  else if (prefix == "heap") {
+    if (!ceph_using_tcmalloc()) {
+      r = -EOPNOTSUPP;
+      ss << "could not issue heap profiler command -- not using tcmalloc!";
+    } else {
+      string heapcmd;
+      cmd_getval(cct, cmdmap, "heapcmd", heapcmd);
+      // XXX 1-element vector, change at callee or make vector here?
+      vector heapcmd_vec;
+      get_str_vec(heapcmd, heapcmd_vec);
+      ceph_heap_profiler_handle_command(heapcmd_vec, ds);
+    }
+  }
+
+  else if (prefix == "debug dump_missing") {
+    string file_name;
+    cmd_getval(cct, cmdmap, "filename", file_name);
+    std::ofstream fout(file_name.c_str());
+    if (!fout.is_open()) {
+	ss << "failed to open file '" << file_name << "'";
+	r = -EINVAL;
+	goto out;
+    }
+
+    std::set  keys;
+    for (ceph::unordered_map::const_iterator pg_map_e = pg_map.begin();
+	 pg_map_e != pg_map.end(); ++pg_map_e) {
+      keys.insert(pg_map_e->first);
+    }
+
+    fout << "*** osd " << whoami << ": dump_missing ***" << std::endl;
+    for (std::set ::iterator p = keys.begin();
+	 p != keys.end(); ++p) {
+      ceph::unordered_map::iterator q = pg_map.find(*p);
+      assert(q != pg_map.end());
+      PG *pg = q->second;
+      pg->lock();
+
+      fout << *pg << std::endl;
+      std::map::const_iterator mend =
+	pg->pg_log.get_missing().missing.end();
+      std::map::const_iterator mi =
+	pg->pg_log.get_missing().missing.begin();
+      for (; mi != mend; ++mi) {
+	fout << mi->first << " -> " << mi->second << std::endl;
+	if (!pg->missing_loc.needs_recovery(mi->first))
+	  continue;
+	if (pg->missing_loc.is_unfound(mi->first))
+	  fout << " unfound ";
+	const set &mls(pg->missing_loc.get_locations(mi->first));
+	if (mls.empty())
+	  continue;
+	fout << "missing_loc: " << mls << std::endl;
+      }
+      pg->unlock();
+      fout << std::endl;
+    }
+
+    fout.close();
+  }
+  else if (prefix == "debug kick_recovery_wq") {
+    int64_t delay;
+    cmd_getval(cct, cmdmap, "delay", delay);
+    ostringstream oss;
+    oss << delay;
+    r = cct->_conf->set_val("osd_recovery_delay_start", oss.str().c_str());
+    if (r != 0) {
+      ss << "kick_recovery_wq: error setting "
+	 << "osd_recovery_delay_start to '" << delay << "': error "
+	 << r;
+      goto out;
+    }
+    cct->_conf->apply_changes(NULL);
+    ss << "kicking recovery queue. set osd_recovery_delay_start "
+       << "to " << cct->_conf->osd_recovery_delay_start;
+    defer_recovery_until = ceph_clock_now(cct);
+    defer_recovery_until += cct->_conf->osd_recovery_delay_start;
+    recovery_wq.wake();
+  }
+
+  else if (prefix == "cpu_profiler") {
+    string arg;
+    cmd_getval(cct, cmdmap, "arg", arg);
+    vector argvec;
+    get_str_vec(arg, argvec);
+    cpu_profiler_handle_command(argvec, ds);
+  }
+
+  else if (prefix == "dump_pg_recovery_stats") {
+    stringstream s;
+    if (f) {
+      pg_recovery_stats.dump_formatted(f.get());
+      f->flush(ds);
+    } else {
+      pg_recovery_stats.dump(s);
+      ds << "dump pg recovery stats: " << s.str();
+    }
+  }
+
+  else if (prefix == "reset_pg_recovery_stats") {
+    ss << "reset pg recovery stats";
+    pg_recovery_stats.reset();
+  }
+
+  else {
+    ss << "unrecognized command! " << cmd;
+    r = -EINVAL;
+  }
+
+ out:
+  rs = ss.str();
+  odata.append(ds);
+  dout(0) << "do_command r=" << r << " " << rs << dendl;
+  clog.info() << rs << "\n";
+  if (con) {
+    MCommandReply *reply = new MCommandReply(r, rs);
+    reply->set_tid(tid);
+    reply->set_data(odata);
+    client_messenger->send_message(reply, con);
+  }
+  return;
+}
+
+
+
+// --------------------------------------
+// dispatch
+
+epoch_t OSD::get_peer_epoch(int peer)
+{
+  Mutex::Locker l(peer_map_epoch_lock);
+  map::iterator p = peer_map_epoch.find(peer);
+  if (p == peer_map_epoch.end())
+    return 0;
+  return p->second;
+}
+
+epoch_t OSD::note_peer_epoch(int peer, epoch_t e)
+{
+  Mutex::Locker l(peer_map_epoch_lock);
+  map::iterator p = peer_map_epoch.find(peer);
+  if (p != peer_map_epoch.end()) {
+    if (p->second < e) {
+      dout(10) << "note_peer_epoch osd." << peer << " has " << e << dendl;
+      p->second = e;
+    } else {
+      dout(30) << "note_peer_epoch osd." << peer << " has " << p->second << " >= " << e << dendl;
+    }
+    return p->second;
+  } else {
+    dout(10) << "note_peer_epoch osd." << peer << " now has " << e << dendl;
+    peer_map_epoch[peer] = e;
+    return e;
+  }
+}
+ 
+void OSD::forget_peer_epoch(int peer, epoch_t as_of) 
+{
+  Mutex::Locker l(peer_map_epoch_lock);
+  map::iterator p = peer_map_epoch.find(peer);
+  if (p != peer_map_epoch.end()) {
+    if (p->second <= as_of) {
+      dout(10) << "forget_peer_epoch osd." << peer << " as_of " << as_of
+	       << " had " << p->second << dendl;
+      peer_map_epoch.erase(p);
+    } else {
+      dout(10) << "forget_peer_epoch osd." << peer << " as_of " << as_of
+	       << " has " << p->second << " - not forgetting" << dendl;
+    }
+  }
+}
+
+
+bool OSD::_share_map_incoming(entity_name_t name, Connection *con, epoch_t epoch, Session* session)
+{
+  bool shared = false;
+  dout(20) << "_share_map_incoming " << name << " " << con->get_peer_addr() << " " << epoch << dendl;
+  //assert(osd_lock.is_locked());
+
+  assert(is_active());
+
+  // does client have old map?
+  if (name.is_client()) {
+    bool sendmap = epoch < osdmap->get_epoch();
+    if (sendmap && session) {
+      if (session->last_sent_epoch < osdmap->get_epoch()) {
+	session->last_sent_epoch = osdmap->get_epoch();
+      } else {
+	sendmap = false; //we don't need to send it out again
+	dout(15) << name << " already sent incremental to update from epoch "<< epoch << dendl;
+      }
+    }
+    if (sendmap) {
+      dout(10) << name << " has old map " << epoch << " < " << osdmap->get_epoch() << dendl;
+      send_incremental_map(epoch, con);
+      shared = true;
+    }
+  }
+
+  // does peer have old map?
+  if (con->get_messenger() == cluster_messenger &&
+      osdmap->is_up(name.num()) &&
+      (osdmap->get_cluster_addr(name.num()) == con->get_peer_addr() ||
+       osdmap->get_hb_back_addr(name.num()) == con->get_peer_addr())) {
+    // remember
+    epoch_t has = note_peer_epoch(name.num(), epoch);
+
+    // share?
+    if (has < osdmap->get_epoch()) {
+      dout(10) << name << " " << con->get_peer_addr() << " has old map " << epoch << " < " << osdmap->get_epoch() << dendl;
+      note_peer_epoch(name.num(), osdmap->get_epoch());
+      send_incremental_map(epoch, con);
+      shared = true;
+    }
+  }
+
+  if (session)
+    session->put();
+  return shared;
+}
+
+
+void OSD::_share_map_outgoing(int peer, Connection *con, OSDMapRef map)
+{
+  if (!map)
+    map = service.get_osdmap();
+
+  // send map?
+  epoch_t pe = get_peer_epoch(peer);
+  if (pe) {
+    if (pe < map->get_epoch()) {
+      send_incremental_map(pe, con);
+      note_peer_epoch(peer, map->get_epoch());
+    } else
+      dout(20) << "_share_map_outgoing " << con << " already has epoch " << pe << dendl;
+  } else {
+    dout(20) << "_share_map_outgoing " << con << " don't know epoch, doing nothing" << dendl;
+    // no idea about peer's epoch.
+    // ??? send recent ???
+    // do nothing.
+  }
+}
+
+
+bool OSD::heartbeat_dispatch(Message *m)
+{
+  dout(30) << "heartbeat_dispatch " << m << dendl;
+  switch (m->get_type()) {
+    
+  case CEPH_MSG_PING:
+    dout(10) << "ping from " << m->get_source_inst() << dendl;
+    m->put();
+    break;
+
+  case MSG_OSD_PING:
+    handle_osd_ping(static_cast(m));
+    break;
+
+  case CEPH_MSG_OSD_MAP:
+    {
+      ConnectionRef self = cluster_messenger->get_loopback_connection();
+      cluster_messenger->send_message(m, self);
+    }
+    break;
+
+  default:
+    dout(0) << "dropping unexpected message " << *m << " from " << m->get_source_inst() << dendl;
+    m->put();
+  }
+
+  return true;
+}
+
+bool OSDService::ObjecterDispatcher::ms_dispatch(Message *m)
+{
+  Mutex::Locker l(osd->objecter_lock);
+  osd->objecter->dispatch(m);
+  return true;
+}
+
+bool OSDService::ObjecterDispatcher::ms_handle_reset(Connection *con)
+{
+  Mutex::Locker l(osd->objecter_lock);
+  osd->objecter->ms_handle_reset(con);
+  return true;
+}
+
+void OSDService::ObjecterDispatcher::ms_handle_connect(Connection *con)
+{
+  Mutex::Locker l(osd->objecter_lock);
+  return osd->objecter->ms_handle_connect(con);
+}
+
+bool OSDService::ObjecterDispatcher::ms_get_authorizer(int dest_type,
+						       AuthAuthorizer **authorizer,
+						       bool force_new)
+{
+  if (dest_type == CEPH_ENTITY_TYPE_MON)
+    return true;
+  *authorizer = osd->monc->auth->build_authorizer(dest_type);
+  return *authorizer != NULL;
+}
+
+
+bool OSD::ms_dispatch(Message *m)
+{
+  if (m->get_type() == MSG_OSD_MARK_ME_DOWN) {
+    service.got_stop_ack();
+    m->put();
+    return true;
+  }
+
+  // lock!
+
+  osd_lock.Lock();
+  if (is_stopping()) {
+    osd_lock.Unlock();
+    m->put();
+    return true;
+  }
+
+  while (dispatch_running) {
+    dout(10) << "ms_dispatch waiting for other dispatch thread to complete" << dendl;
+    dispatch_cond.Wait(osd_lock);
+  }
+  dispatch_running = true;
+
+  do_waiters();
+  _dispatch(m);
+  do_waiters();
+
+  dispatch_running = false;
+  dispatch_cond.Signal();
+
+  osd_lock.Unlock();
+
+  return true;
+}
+
+bool OSD::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new)
+{
+  dout(10) << "OSD::ms_get_authorizer type=" << ceph_entity_type_name(dest_type) << dendl;
+
+  if (dest_type == CEPH_ENTITY_TYPE_MON)
+    return true;
+
+  if (force_new) {
+    /* the MonClient checks keys every tick(), so we should just wait for that cycle
+       to get through */
+    if (monc->wait_auth_rotating(10) < 0)
+      return false;
+  }
+
+  *authorizer = monc->auth->build_authorizer(dest_type);
+  return *authorizer != NULL;
+}
+
+
+bool OSD::ms_verify_authorizer(Connection *con, int peer_type,
+			       int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply,
+			       bool& isvalid, CryptoKey& session_key)
+{
+  AuthAuthorizeHandler *authorize_handler = 0;
+  switch (peer_type) {
+  case CEPH_ENTITY_TYPE_MDS:
+    /*
+     * note: mds is technically a client from our perspective, but
+     * this makes the 'cluster' consistent w/ monitor's usage.
+     */
+  case CEPH_ENTITY_TYPE_OSD:
+    authorize_handler = authorize_handler_cluster_registry->get_handler(protocol);
+    break;
+  default:
+    authorize_handler = authorize_handler_service_registry->get_handler(protocol);
+  }
+  if (!authorize_handler) {
+    dout(0) << "No AuthAuthorizeHandler found for protocol " << protocol << dendl;
+    isvalid = false;
+    return true;
+  }
+
+  AuthCapsInfo caps_info;
+  EntityName name;
+  uint64_t global_id;
+  uint64_t auid = CEPH_AUTH_UID_DEFAULT;
+
+  isvalid = authorize_handler->verify_authorizer(cct, monc->rotating_secrets,
+						 authorizer_data, authorizer_reply, name, global_id, caps_info, session_key, &auid);
+
+  if (isvalid) {
+    Session *s = static_cast(con->get_priv());
+    if (!s) {
+      s = new Session;
+      con->set_priv(s->get());
+      s->con = con;
+      dout(10) << " new session " << s << " con=" << s->con << " addr=" << s->con->get_peer_addr() << dendl;
+    }
+
+    s->entity_name = name;
+    if (caps_info.allow_all)
+      s->caps.set_allow_all();
+    s->auid = auid;
+ 
+    if (caps_info.caps.length() > 0) {
+      bufferlist::iterator p = caps_info.caps.begin();
+      string str;
+      try {
+	::decode(str, p);
+      }
+      catch (buffer::error& e) {
+      }
+      bool success = s->caps.parse(str);
+      if (success)
+	dout(10) << " session " << s << " " << s->entity_name << " has caps " << s->caps << " '" << str << "'" << dendl;
+      else
+	dout(10) << " session " << s << " " << s->entity_name << " failed to parse caps '" << str << "'" << dendl;
+    }
+    
+    s->put();
+  }
+  return true;
+};
+
+
+void OSD::do_waiters()
+{
+  assert(osd_lock.is_locked());
+  
+  dout(10) << "do_waiters -- start" << dendl;
+  finished_lock.Lock();
+  while (!finished.empty()) {
+    OpRequestRef next = finished.front();
+    finished.pop_front();
+    finished_lock.Unlock();
+    dispatch_op(next);
+    finished_lock.Lock();
+  }
+  finished_lock.Unlock();
+  dout(10) << "do_waiters -- finish" << dendl;
+}
+
+void OSD::dispatch_op(OpRequestRef op)
+{
+  switch (op->get_req()->get_type()) {
+
+  case MSG_OSD_PG_CREATE:
+    handle_pg_create(op);
+    break;
+
+  case MSG_OSD_PG_NOTIFY:
+    handle_pg_notify(op);
+    break;
+  case MSG_OSD_PG_QUERY:
+    handle_pg_query(op);
+    break;
+  case MSG_OSD_PG_LOG:
+    handle_pg_log(op);
+    break;
+  case MSG_OSD_PG_REMOVE:
+    handle_pg_remove(op);
+    break;
+  case MSG_OSD_PG_INFO:
+    handle_pg_info(op);
+    break;
+  case MSG_OSD_PG_TRIM:
+    handle_pg_trim(op);
+    break;
+  case MSG_OSD_PG_MISSING:
+    assert(0 ==
+	   "received MOSDPGMissing; this message is supposed to be unused!?!");
+    break;
+  case MSG_OSD_PG_SCAN:
+    handle_pg_scan(op);
+    break;
+  case MSG_OSD_PG_BACKFILL:
+    handle_pg_backfill(op);
+    break;
+
+  case MSG_OSD_BACKFILL_RESERVE:
+    handle_pg_backfill_reserve(op);
+    break;
+  case MSG_OSD_RECOVERY_RESERVE:
+    handle_pg_recovery_reserve(op);
+    break;
+
+    // client ops
+  case CEPH_MSG_OSD_OP:
+    handle_op(op);
+    break;
+
+    // for replication etc.
+  case MSG_OSD_SUBOP:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_SUBOPREPLY:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_PG_PUSH:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_PG_PULL:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_PG_PUSH_REPLY:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_EC_WRITE:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_EC_WRITE_REPLY:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_EC_READ:
+    handle_replica_op(op);
+    break;
+  case MSG_OSD_EC_READ_REPLY:
+    handle_replica_op(op);
+    break;
+  }
+}
+
+void OSD::_dispatch(Message *m)
+{
+  assert(osd_lock.is_locked());
+  dout(20) << "_dispatch " << m << " " << *m << dendl;
+  Session *session = NULL;
+
+  logger->set(l_osd_buf, buffer::get_total_alloc());
+
+  switch (m->get_type()) {
+
+    // -- don't need lock -- 
+  case CEPH_MSG_PING:
+    dout(10) << "ping from " << m->get_source() << dendl;
+    m->put();
+    break;
+
+    // -- don't need OSDMap --
+
+    // map and replication
+  case CEPH_MSG_OSD_MAP:
+    handle_osd_map(static_cast(m));
+    break;
+
+    // osd
+  case CEPH_MSG_SHUTDOWN:
+    session = static_cast(m->get_connection()->get_priv());
+    if (!session ||
+	session->entity_name.is_mon() ||
+	session->entity_name.is_osd())
+      shutdown();
+    else dout(0) << "shutdown message from connection with insufficient privs!"
+		 << m->get_connection() << dendl;
+    m->put();
+    if (session)
+      session->put();
+    break;
+
+  case MSG_PGSTATSACK:
+    handle_pg_stats_ack(static_cast(m));
+    break;
+
+  case MSG_MON_COMMAND:
+    handle_command(static_cast(m));
+    break;
+  case MSG_COMMAND:
+    handle_command(static_cast(m));
+    break;
+
+  case MSG_OSD_SCRUB:
+    handle_scrub(static_cast(m));
+    break;    
+
+  case MSG_OSD_REP_SCRUB:
+    handle_rep_scrub(static_cast(m));
+    break;    
+
+    // -- need OSDMap --
+
+  default:
+    {
+      OpRequestRef op = op_tracker.create_request(m);
+      op->mark_event("waiting_for_osdmap");
+      // no map?  starting up?
+      if (!osdmap) {
+        dout(7) << "no OSDMap, not booted" << dendl;
+        waiting_for_osdmap.push_back(op);
+        break;
+      }
+      
+      // need OSDMap
+      dispatch_op(op);
+    }
+  }
+
+  logger->set(l_osd_buf, buffer::get_total_alloc());
+
+}
+
+void OSD::handle_rep_scrub(MOSDRepScrub *m)
+{
+  dout(10) << "queueing MOSDRepScrub " << *m << dendl;
+  rep_scrub_wq.queue(m);
+}
+
+void OSD::handle_scrub(MOSDScrub *m)
+{
+  dout(10) << "handle_scrub " << *m << dendl;
+  if (!require_mon_peer(m))
+    return;
+  if (m->fsid != monc->get_fsid()) {
+    dout(0) << "handle_scrub fsid " << m->fsid << " != " << monc->get_fsid() << dendl;
+    m->put();
+    return;
+  }
+
+  if (m->scrub_pgs.empty()) {
+    for (ceph::unordered_map::iterator p = pg_map.begin();
+	 p != pg_map.end();
+	 ++p) {
+      PG *pg = p->second;
+      pg->lock();
+      if (pg->is_primary()) {
+	pg->unreg_next_scrub();
+	pg->scrubber.must_scrub = true;
+	pg->scrubber.must_deep_scrub = m->deep || m->repair;
+	pg->scrubber.must_repair = m->repair;
+	pg->reg_next_scrub();
+	dout(10) << "marking " << *pg << " for scrub" << dendl;
+      }
+      pg->unlock();
+    }
+  } else {
+    for (vector::iterator p = m->scrub_pgs.begin();
+	 p != m->scrub_pgs.end();
+	 ++p) {
+      spg_t pcand;
+      if (osdmap->get_primary_shard(*p, &pcand) &&
+	  pg_map.count(pcand)) {
+	PG *pg = pg_map[pcand];
+	pg->lock();
+	if (pg->is_primary()) {
+	  pg->unreg_next_scrub();
+	  pg->scrubber.must_scrub = true;
+	  pg->scrubber.must_deep_scrub = m->deep || m->repair;
+	  pg->scrubber.must_repair = m->repair;
+	  pg->reg_next_scrub();
+	  dout(10) << "marking " << *pg << " for scrub" << dendl;
+	}
+	pg->unlock();
+      }
+    }
+  }
+  
+  m->put();
+}
+
+bool OSD::scrub_random_backoff()
+{
+  bool coin_flip = (rand() % 3) == whoami % 3;
+  if (!coin_flip) {
+    dout(20) << "scrub_random_backoff lost coin flip, randomly backing off" << dendl;
+    return true;
+  }
+  return false;
+}
+
+bool OSD::scrub_should_schedule()
+{
+  double loadavgs[1];
+  if (getloadavg(loadavgs, 1) != 1) {
+    dout(10) << "scrub_should_schedule couldn't read loadavgs\n" << dendl;
+    return false;
+  }
+
+  if (loadavgs[0] >= cct->_conf->osd_scrub_load_threshold) {
+    dout(20) << "scrub_should_schedule loadavg " << loadavgs[0]
+	     << " >= max " << cct->_conf->osd_scrub_load_threshold
+	     << " = no, load too high" << dendl;
+    return false;
+  }
+
+  dout(20) << "scrub_should_schedule loadavg " << loadavgs[0]
+	   << " < max " << cct->_conf->osd_scrub_load_threshold
+	   << " = yes" << dendl;
+  return loadavgs[0] < cct->_conf->osd_scrub_load_threshold;
+}
+
+void OSD::sched_scrub()
+{
+  assert(osd_lock.is_locked());
+
+  bool load_is_low = scrub_should_schedule();
+
+  dout(20) << "sched_scrub load_is_low=" << (int)load_is_low << dendl;
+
+  utime_t now = ceph_clock_now(cct);
+  
+  //dout(20) << " " << last_scrub_pg << dendl;
+
+  pair pos;
+  if (service.first_scrub_stamp(&pos)) {
+    do {
+      utime_t t = pos.first;
+      spg_t pgid = pos.second;
+      dout(30) << "sched_scrub examine " << pgid << " at " << t << dendl;
+
+      utime_t diff = now - t;
+      if ((double)diff < cct->_conf->osd_scrub_min_interval) {
+	dout(10) << "sched_scrub " << pgid << " at " << t
+		 << ": " << (double)diff << " < min (" << cct->_conf->osd_scrub_min_interval << " seconds)" << dendl;
+	break;
+      }
+      if ((double)diff < cct->_conf->osd_scrub_max_interval && !load_is_low) {
+	// save ourselves some effort
+	dout(10) << "sched_scrub " << pgid << " high load at " << t
+		 << ": " << (double)diff << " < max (" << cct->_conf->osd_scrub_max_interval << " seconds)" << dendl;
+	break;
+      }
+
+      PG *pg = _lookup_lock_pg(pgid);
+      if (pg) {
+	if (pg->get_pgbackend()->scrub_supported() && pg->is_active() &&
+	    (load_is_low ||
+	     (double)diff >= cct->_conf->osd_scrub_max_interval ||
+	     pg->scrubber.must_scrub)) {
+	  dout(10) << "sched_scrub scrubbing " << pgid << " at " << t
+		   << (pg->scrubber.must_scrub ? ", explicitly requested" :
+		   ( (double)diff >= cct->_conf->osd_scrub_max_interval ? ", diff >= max" : ""))
+		   << dendl;
+	  if (pg->sched_scrub()) {
+	    pg->unlock();
+	    break;
+	  }
+	}
+	pg->unlock();
+      }
+    } while  (service.next_scrub_stamp(pos, &pos));
+  }    
+  dout(20) << "sched_scrub done" << dendl;
+}
+
+bool OSDService::inc_scrubs_pending()
+{
+  bool result = false;
+
+  sched_scrub_lock.Lock();
+  if (scrubs_pending + scrubs_active < cct->_conf->osd_max_scrubs) {
+    dout(20) << "inc_scrubs_pending " << scrubs_pending << " -> " << (scrubs_pending+1)
+	     << " (max " << cct->_conf->osd_max_scrubs << ", active " << scrubs_active << ")" << dendl;
+    result = true;
+    ++scrubs_pending;
+  } else {
+    dout(20) << "inc_scrubs_pending " << scrubs_pending << " + " << scrubs_active << " active >= max " << cct->_conf->osd_max_scrubs << dendl;
+  }
+  sched_scrub_lock.Unlock();
+
+  return result;
+}
+
+void OSDService::dec_scrubs_pending()
+{
+  sched_scrub_lock.Lock();
+  dout(20) << "dec_scrubs_pending " << scrubs_pending << " -> " << (scrubs_pending-1)
+	   << " (max " << cct->_conf->osd_max_scrubs << ", active " << scrubs_active << ")" << dendl;
+  --scrubs_pending;
+  assert(scrubs_pending >= 0);
+  sched_scrub_lock.Unlock();
+}
+
+void OSDService::inc_scrubs_active(bool reserved)
+{
+  sched_scrub_lock.Lock();
+  ++(scrubs_active);
+  if (reserved) {
+    --(scrubs_pending);
+    dout(20) << "inc_scrubs_active " << (scrubs_active-1) << " -> " << scrubs_active
+	     << " (max " << cct->_conf->osd_max_scrubs
+	     << ", pending " << (scrubs_pending+1) << " -> " << scrubs_pending << ")" << dendl;
+    assert(scrubs_pending >= 0);
+  } else {
+    dout(20) << "inc_scrubs_active " << (scrubs_active-1) << " -> " << scrubs_active
+	     << " (max " << cct->_conf->osd_max_scrubs
+	     << ", pending " << scrubs_pending << ")" << dendl;
+  }
+  sched_scrub_lock.Unlock();
+}
+
+void OSDService::dec_scrubs_active()
+{
+  sched_scrub_lock.Lock();
+  dout(20) << "dec_scrubs_active " << scrubs_active << " -> " << (scrubs_active-1)
+	   << " (max " << cct->_conf->osd_max_scrubs << ", pending " << scrubs_pending << ")" << dendl;
+  --scrubs_active;
+  sched_scrub_lock.Unlock();
+}
+
+bool OSDService::prepare_to_stop()
+{
+  Mutex::Locker l(is_stopping_lock);
+  if (state != NOT_STOPPING)
+    return false;
+
+  OSDMapRef osdmap = get_osdmap();
+  if (osdmap && osdmap->is_up(whoami)) {
+    dout(0) << __func__ << " telling mon we are shutting down" << dendl;
+    state = PREPARING_TO_STOP;
+    monc->send_mon_message(new MOSDMarkMeDown(monc->get_fsid(),
+					      osdmap->get_inst(whoami),
+					      osdmap->get_epoch(),
+					      false
+					      ));
+    utime_t now = ceph_clock_now(cct);
+    utime_t timeout;
+    timeout.set_from_double(now + cct->_conf->osd_mon_shutdown_timeout);
+    while ((ceph_clock_now(cct) < timeout) &&
+	   (state != STOPPING)) {
+      is_stopping_cond.WaitUntil(is_stopping_lock, timeout);
+    }
+  }
+  dout(0) << __func__ << " starting shutdown" << dendl;
+  state = STOPPING;
+  return true;
+}
+
+void OSDService::got_stop_ack()
+{
+  Mutex::Locker l(is_stopping_lock);
+  dout(0) << __func__ << " starting shutdown" << dendl;
+  state = STOPPING;
+  is_stopping_cond.Signal();
+}
+
+
+// =====================================================
+// MAP
+
+void OSD::wait_for_new_map(OpRequestRef op)
+{
+  // ask?
+  if (waiting_for_osdmap.empty()) {
+    osdmap_subscribe(osdmap->get_epoch() + 1, true);
+  }
+  
+  logger->inc(l_osd_waiting_for_map);
+  waiting_for_osdmap.push_back(op);
+  op->mark_delayed("wait for new map");
+}
+
+
+/** update_map
+ * assimilate new OSDMap(s).  scan pgs, etc.
+ */
+
+void OSD::note_down_osd(int peer)
+{
+  assert(osd_lock.is_locked());
+  cluster_messenger->mark_down(osdmap->get_cluster_addr(peer));
+
+  heartbeat_lock.Lock();
+  failure_queue.erase(peer);
+  failure_pending.erase(peer);
+  map::iterator p = heartbeat_peers.find(peer);
+  if (p != heartbeat_peers.end()) {
+    hbclient_messenger->mark_down(p->second.con_back);
+    if (p->second.con_front) {
+      hbclient_messenger->mark_down(p->second.con_front);
+    }
+    heartbeat_peers.erase(p);
+  }
+  heartbeat_lock.Unlock();
+}
+
+void OSD::note_up_osd(int peer)
+{
+  forget_peer_epoch(peer, osdmap->get_epoch() - 1);
+}
+
+struct C_OnMapApply : public Context {
+  OSDService *service;
+  boost::scoped_ptr t;
+  list pinned_maps;
+  epoch_t e;
+  C_OnMapApply(OSDService *service,
+	       ObjectStore::Transaction *t,
+	       const list &pinned_maps,
+	       epoch_t e)
+    : service(service), t(t), pinned_maps(pinned_maps), e(e) {}
+  void finish(int r) {
+    service->clear_map_bl_cache_pins(e);
+  }
+};
+
+void OSD::osdmap_subscribe(version_t epoch, bool force_request)
+{
+  OSDMapRef osdmap = service.get_osdmap();
+  if (osdmap->get_epoch() >= epoch)
+    return;
+
+  if (monc->sub_want_increment("osdmap", epoch, CEPH_SUBSCRIBE_ONETIME) ||
+      force_request) {
+    monc->renew_subs();
+  }
+}
+
+void OSD::handle_osd_map(MOSDMap *m)
+{
+  assert(osd_lock.is_locked());
+  list pinned_maps;
+  if (m->fsid != monc->get_fsid()) {
+    dout(0) << "handle_osd_map fsid " << m->fsid << " != " << monc->get_fsid() << dendl;
+    m->put();
+    return;
+  }
+  if (is_initializing()) {
+    dout(0) << "ignoring osdmap until we have initialized" << dendl;
+    m->put();
+    return;
+  }
+
+  Session *session = static_cast(m->get_connection()->get_priv());
+  if (session && !(session->entity_name.is_mon() || session->entity_name.is_osd())) {
+    //not enough perms!
+    m->put();
+    session->put();
+    return;
+  }
+  if (session)
+    session->put();
+
+  // share with the objecter
+  {
+    Mutex::Locker l(service.objecter_lock);
+    m->get();
+    service.objecter->handle_osd_map(m);
+  }
+
+  epoch_t first = m->get_first();
+  epoch_t last = m->get_last();
+  dout(3) << "handle_osd_map epochs [" << first << "," << last << "], i have "
+	  << osdmap->get_epoch()
+	  << ", src has [" << m->oldest_map << "," << m->newest_map << "]"
+	  << dendl;
+
+  logger->inc(l_osd_map);
+  logger->inc(l_osd_mape, last - first + 1);
+  if (first <= osdmap->get_epoch())
+    logger->inc(l_osd_mape_dup, osdmap->get_epoch() - first + 1);
+
+  // make sure there is something new, here, before we bother flushing the queues and such
+  if (last <= osdmap->get_epoch()) {
+    dout(10) << " no new maps here, dropping" << dendl;
+    m->put();
+    return;
+  }
+
+  // even if this map isn't from a mon, we may have satisfied our subscription
+  monc->sub_got("osdmap", last);
+
+  // missing some?
+  bool skip_maps = false;
+  if (first > osdmap->get_epoch() + 1) {
+    dout(10) << "handle_osd_map message skips epochs " << osdmap->get_epoch() + 1
+	     << ".." << (first-1) << dendl;
+    if (m->oldest_map <= osdmap->get_epoch() + 1) {
+      osdmap_subscribe(osdmap->get_epoch()+1, true);
+      m->put();
+      return;
+    }
+    // always try to get the full range of maps--as many as we can.  this
+    //  1- is good to have
+    //  2- is at present the only way to ensure that we get a *full* map as
+    //     the first map!
+    if (m->oldest_map < first) {
+      osdmap_subscribe(m->oldest_map - 1, true);
+      m->put();
+      return;
+    }
+    skip_maps = true;
+  }
+
+  ObjectStore::Transaction *_t = new ObjectStore::Transaction;
+  ObjectStore::Transaction &t = *_t;
+
+  // store new maps: queue for disk and put in the osdmap cache
+  epoch_t last_marked_full = 0;
+  epoch_t start = MAX(osdmap->get_epoch() + 1, first);
+  for (epoch_t e = start; e <= last; e++) {
+    map::iterator p;
+    p = m->maps.find(e);
+    if (p != m->maps.end()) {
+      dout(10) << "handle_osd_map  got full map for epoch " << e << dendl;
+      OSDMap *o = new OSDMap;
+      bufferlist& bl = p->second;
+      
+      o->decode(bl);
+      if (o->test_flag(CEPH_OSDMAP_FULL))
+	last_marked_full = e;
+      pinned_maps.push_back(add_map(o));
+
+      hobject_t fulloid = get_osdmap_pobject_name(e);
+      t.write(coll_t::META_COLL, fulloid, 0, bl.length(), bl);
+      pin_map_bl(e, bl);
+      continue;
+    }
+
+    p = m->incremental_maps.find(e);
+    if (p != m->incremental_maps.end()) {
+      dout(10) << "handle_osd_map  got inc map for epoch " << e << dendl;
+      bufferlist& bl = p->second;
+      hobject_t oid = get_inc_osdmap_pobject_name(e);
+      t.write(coll_t::META_COLL, oid, 0, bl.length(), bl);
+      pin_map_inc_bl(e, bl);
+
+      OSDMap *o = new OSDMap;
+      if (e > 1) {
+	bufferlist obl;
+	OSDMapRef prev = get_map(e - 1);
+	prev->encode(obl);
+	o->decode(obl);
+      }
+
+      OSDMap::Incremental inc;
+      bufferlist::iterator p = bl.begin();
+      inc.decode(p);
+      if (o->apply_incremental(inc) < 0) {
+	derr << "ERROR: bad fsid?  i have " << osdmap->get_fsid() << " and inc has " << inc.fsid << dendl;
+	assert(0 == "bad fsid");
+      }
+
+      if (o->test_flag(CEPH_OSDMAP_FULL))
+	last_marked_full = e;
+      pinned_maps.push_back(add_map(o));
+
+      bufferlist fbl;
+      o->encode(fbl);
+
+      hobject_t fulloid = get_osdmap_pobject_name(e);
+      t.write(coll_t::META_COLL, fulloid, 0, fbl.length(), fbl);
+      pin_map_bl(e, fbl);
+      continue;
+    }
+
+    assert(0 == "MOSDMap lied about what maps it had?");
+  }
+
+  if (superblock.oldest_map) {
+    int num = 0;
+    epoch_t min(
+      MIN(m->oldest_map,
+	  service.map_cache.cached_key_lower_bound()));
+    for (epoch_t e = superblock.oldest_map; e < min; ++e) {
+      dout(20) << " removing old osdmap epoch " << e << dendl;
+      t.remove(coll_t::META_COLL, get_osdmap_pobject_name(e));
+      t.remove(coll_t::META_COLL, get_inc_osdmap_pobject_name(e));
+      superblock.oldest_map = e+1;
+      num++;
+      if (num >= cct->_conf->osd_target_transaction_size &&
+	  (uint64_t)num > (last - first))  // make sure we at least keep pace with incoming maps
+	break;
+    }
+  }
+
+  if (!superblock.oldest_map || skip_maps)
+    superblock.oldest_map = first;
+  superblock.newest_map = last;
+
+  if (last_marked_full > superblock.last_map_marked_full)
+    superblock.last_map_marked_full = last_marked_full;
+ 
+  map_lock.get_write();
+
+  C_Contexts *fin = new C_Contexts(cct);
+
+  // advance through the new maps
+  for (epoch_t cur = start; cur <= superblock.newest_map; cur++) {
+    dout(10) << " advance to epoch " << cur << " (<= newest " << superblock.newest_map << ")" << dendl;
+
+    OSDMapRef newmap = get_map(cur);
+    assert(newmap);  // we just cached it above!
+
+    // start blacklisting messages sent to peers that go down.
+    service.pre_publish_map(newmap);
+
+    // kill connections to newly down osds
+    set old;
+    osdmap->get_all_osds(old);
+    for (set::iterator p = old.begin(); p != old.end(); ++p) {
+      if (*p != whoami &&
+	  osdmap->have_inst(*p) &&                        // in old map
+	  (!newmap->exists(*p) || !newmap->is_up(*p))) {  // but not the new one
+	note_down_osd(*p);
+      }
+    }
+    
+    osdmap = newmap;
+
+    superblock.current_epoch = cur;
+    advance_map(t, fin);
+    had_map_since = ceph_clock_now(cct);
+  }
+
+  if (osdmap->is_up(whoami) &&
+      osdmap->get_addr(whoami) == client_messenger->get_myaddr() &&
+      bind_epoch < osdmap->get_up_from(whoami)) {
+
+    if (is_booting()) {
+      dout(1) << "state: booting -> active" << dendl;
+      state = STATE_ACTIVE;
+
+      // set incarnation so that osd_reqid_t's we generate for our
+      // objecter requests are unique across restarts.
+      service.objecter->set_client_incarnation(osdmap->get_epoch());
+    }
+  }
+
+  bool do_shutdown = false;
+  bool do_restart = false;
+  if (osdmap->get_epoch() > 0 &&
+      state == STATE_ACTIVE) {
+    if (!osdmap->exists(whoami)) {
+      dout(0) << "map says i do not exist.  shutting down." << dendl;
+      do_shutdown = true;   // don't call shutdown() while we have everything paused
+    } else if (!osdmap->is_up(whoami) ||
+	       !osdmap->get_addr(whoami).probably_equals(client_messenger->get_myaddr()) ||
+	       !osdmap->get_cluster_addr(whoami).probably_equals(cluster_messenger->get_myaddr()) ||
+	       !osdmap->get_hb_back_addr(whoami).probably_equals(hb_back_server_messenger->get_myaddr()) ||
+	       (osdmap->get_hb_front_addr(whoami) != entity_addr_t() &&
+                !osdmap->get_hb_front_addr(whoami).probably_equals(hb_front_server_messenger->get_myaddr()))) {
+      if (!osdmap->is_up(whoami)) {
+	if (service.is_preparing_to_stop() || service.is_stopping()) {
+	  service.got_stop_ack();
+	} else {
+	  clog.warn() << "map e" << osdmap->get_epoch()
+		      << " wrongly marked me down";
+	}
+      }
+      else if (!osdmap->get_addr(whoami).probably_equals(client_messenger->get_myaddr()))
+	clog.error() << "map e" << osdmap->get_epoch()
+		    << " had wrong client addr (" << osdmap->get_addr(whoami)
+		     << " != my " << client_messenger->get_myaddr() << ")";
+      else if (!osdmap->get_cluster_addr(whoami).probably_equals(cluster_messenger->get_myaddr()))
+	clog.error() << "map e" << osdmap->get_epoch()
+		    << " had wrong cluster addr (" << osdmap->get_cluster_addr(whoami)
+		     << " != my " << cluster_messenger->get_myaddr() << ")";
+      else if (!osdmap->get_hb_back_addr(whoami).probably_equals(hb_back_server_messenger->get_myaddr()))
+	clog.error() << "map e" << osdmap->get_epoch()
+		    << " had wrong hb back addr (" << osdmap->get_hb_back_addr(whoami)
+		     << " != my " << hb_back_server_messenger->get_myaddr() << ")";
+      else if (osdmap->get_hb_front_addr(whoami) != entity_addr_t() &&
+               !osdmap->get_hb_front_addr(whoami).probably_equals(hb_front_server_messenger->get_myaddr()))
+	clog.error() << "map e" << osdmap->get_epoch()
+		    << " had wrong hb front addr (" << osdmap->get_hb_front_addr(whoami)
+		     << " != my " << hb_front_server_messenger->get_myaddr() << ")";
+      
+      if (!service.is_stopping()) {
+	up_epoch = 0;
+	do_restart = true;
+	bind_epoch = osdmap->get_epoch();
+
+	start_waiting_for_healthy();
+
+	set avoid_ports;
+	avoid_ports.insert(cluster_messenger->get_myaddr().get_port());
+	avoid_ports.insert(hb_back_server_messenger->get_myaddr().get_port());
+	avoid_ports.insert(hb_front_server_messenger->get_myaddr().get_port());
+
+	int r = cluster_messenger->rebind(avoid_ports);
+	if (r != 0)
+	  do_shutdown = true;  // FIXME: do_restart?
+
+	r = hb_back_server_messenger->rebind(avoid_ports);
+	if (r != 0)
+	  do_shutdown = true;  // FIXME: do_restart?
+
+	r = hb_front_server_messenger->rebind(avoid_ports);
+	if (r != 0)
+	  do_shutdown = true;  // FIXME: do_restart?
+
+	hbclient_messenger->mark_down_all();
+
+	reset_heartbeat_peers();
+      }
+    }
+  }
+
+
+  // note in the superblock that we were clean thru the prior epoch
+  if (boot_epoch && boot_epoch >= superblock.mounted) {
+    superblock.mounted = boot_epoch;
+    superblock.clean_thru = osdmap->get_epoch();
+  }
+
+  // superblock and commit
+  write_superblock(t);
+  store->queue_transaction(
+    0,
+    _t,
+    new C_OnMapApply(&service, _t, pinned_maps, osdmap->get_epoch()),
+    0, fin);
+  service.publish_superblock(superblock);
+
+  map_lock.put_write();
+
+  check_osdmap_features(store);
+
+  // yay!
+  consume_map();
+
+  if (is_active() || is_waiting_for_healthy())
+    maybe_update_heartbeat_peers();
+
+  if (!is_active()) {
+    dout(10) << " not yet active; waiting for peering wq to drain" << dendl;
+    peering_wq.drain();
+  } else {
+    activate_map();
+  }
+
+  if (m->newest_map && m->newest_map > last) {
+    dout(10) << " msg say newest map is " << m->newest_map << ", requesting more" << dendl;
+    osdmap_subscribe(osdmap->get_epoch()+1, true);
+  }
+  else if (is_booting()) {
+    start_boot();  // retry
+  }
+  else if (do_restart)
+    start_boot();
+
+  if (do_shutdown)
+    shutdown();
+
+  m->put();
+}
+
+void OSD::check_osdmap_features(ObjectStore *fs)
+{
+  // adjust required feature bits?
+
+  // we have to be a bit careful here, because we are accessing the
+  // Policy structures without taking any lock.  in particular, only
+  // modify integer values that can safely be read by a racing CPU.
+  // since we are only accessing existing Policy structures a their
+  // current memory location, and setting or clearing bits in integer
+  // fields, and we are the only writer, this is not a problem.
+
+  {
+    Messenger::Policy p = client_messenger->get_default_policy();
+    uint64_t mask;
+    uint64_t features = osdmap->get_features(entity_name_t::TYPE_CLIENT, &mask);
+    if ((p.features_required & mask) != features) {
+      dout(0) << "crush map has features " << features
+	      << ", adjusting msgr requires for clients" << dendl;
+      p.features_required = (p.features_required & ~mask) | features;
+      client_messenger->set_default_policy(p);
+    }
+  }
+  {
+    Messenger::Policy p = client_messenger->get_policy(entity_name_t::TYPE_MON);
+    uint64_t mask;
+    uint64_t features = osdmap->get_features(entity_name_t::TYPE_MON, &mask);
+    if ((p.features_required & mask) != features) {
+      dout(0) << "crush map has features " << features
+	      << " was " << p.features_required
+	      << ", adjusting msgr requires for mons" << dendl;
+      p.features_required = (p.features_required & ~mask) | features;
+      client_messenger->set_policy(entity_name_t::TYPE_MON, p);
+    }
+  }
+  {
+    Messenger::Policy p = cluster_messenger->get_policy(entity_name_t::TYPE_OSD);
+    uint64_t mask;
+    uint64_t features = osdmap->get_features(entity_name_t::TYPE_OSD, &mask);
+
+    if ((p.features_required & mask) != features) {
+      dout(0) << "crush map has features " << features
+	      << ", adjusting msgr requires for osds" << dendl;
+      p.features_required = (p.features_required & ~mask) | features;
+      cluster_messenger->set_policy(entity_name_t::TYPE_OSD, p);
+    }
+
+    if ((features & CEPH_FEATURE_OSD_ERASURE_CODES) &&
+	!fs->get_allow_sharded_objects()) {
+      dout(0) << __func__ << " enabling on-disk ERASURE CODES compat feature" << dendl;
+      superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
+      ObjectStore::Transaction *t = new ObjectStore::Transaction;
+      write_superblock(*t);
+      int err = store->queue_transaction_and_cleanup(NULL, t);
+      assert(err == 0);
+      fs->set_allow_sharded_objects();
+    }
+  }
+}
+
+bool OSD::advance_pg(
+  epoch_t osd_epoch, PG *pg,
+  ThreadPool::TPHandle &handle,
+  PG::RecoveryCtx *rctx,
+  set > *new_pgs)
+{
+  assert(pg->is_locked());
+  epoch_t next_epoch = pg->get_osdmap()->get_epoch() + 1;
+  OSDMapRef lastmap = pg->get_osdmap();
+
+  if (lastmap->get_epoch() == osd_epoch)
+    return true;
+  assert(lastmap->get_epoch() < osd_epoch);
+
+  epoch_t min_epoch = service.get_min_pg_epoch();
+  epoch_t max;
+  if (min_epoch) {
+    max = min_epoch + g_conf->osd_map_max_advance;
+  } else {
+    max = next_epoch + g_conf->osd_map_max_advance;
+  }
+
+  for (;
+       next_epoch <= osd_epoch && next_epoch <= max;
+       ++next_epoch) {
+    OSDMapRef nextmap = service.try_get_map(next_epoch);
+    if (!nextmap)
+      continue;
+
+    vector newup, newacting;
+    int up_primary, acting_primary;
+    nextmap->pg_to_up_acting_osds(
+      pg->info.pgid.pgid,
+      &newup, &up_primary,
+      &newacting, &acting_primary);
+    pg->handle_advance_map(
+      nextmap, lastmap, newup, up_primary,
+      newacting, acting_primary, rctx);
+
+    // Check for split!
+    set children;
+    spg_t parent(pg->info.pgid);
+    if (parent.is_split(
+	lastmap->get_pg_num(pg->pool.id),
+	nextmap->get_pg_num(pg->pool.id),
+	&children)) {
+      service.mark_split_in_progress(pg->info.pgid, children);
+      split_pgs(
+	pg, children, new_pgs, lastmap, nextmap,
+	rctx);
+    }
+
+    lastmap = nextmap;
+    handle.reset_tp_timeout();
+  }
+  service.pg_update_epoch(pg->info.pgid, lastmap->get_epoch());
+  pg->handle_activate_map(rctx);
+  if (next_epoch <= osd_epoch) {
+    dout(10) << __func__ << " advanced by max " << g_conf->osd_map_max_advance
+	     << " past min epoch " << min_epoch
+	     << " ... will requeue " << *pg << dendl;
+    return false;
+  }
+  return true;
+}
+
+/** 
+ * scan placement groups, initiate any replication
+ * activities.
+ */
+void OSD::advance_map(ObjectStore::Transaction& t, C_Contexts *tfin)
+{
+  assert(osd_lock.is_locked());
+
+  dout(7) << "advance_map epoch " << osdmap->get_epoch()
+          << "  " << pg_map.size() << " pgs"
+          << dendl;
+
+  if (!up_epoch &&
+      osdmap->is_up(whoami) &&
+      osdmap->get_inst(whoami) == client_messenger->get_myinst()) {
+    up_epoch = osdmap->get_epoch();
+    dout(10) << "up_epoch is " << up_epoch << dendl;
+    if (!boot_epoch) {
+      boot_epoch = osdmap->get_epoch();
+      dout(10) << "boot_epoch is " << boot_epoch << dendl;
+    }
+  }
+
+  // scan pg creations
+  ceph::unordered_map::iterator n = creating_pgs.begin();
+  while (n != creating_pgs.end()) {
+    ceph::unordered_map::iterator p = n++;
+    spg_t pgid = p->first;
+
+    // am i still primary?
+    vector acting;
+    int primary;
+    osdmap->pg_to_acting_osds(pgid.pgid, &acting, &primary);
+    if (primary != whoami) {
+      dout(10) << " no longer primary for " << pgid << ", stopping creation" << dendl;
+      creating_pgs.erase(p);
+    } else {
+      /*
+       * adding new ppl to our pg has no effect, since we're still primary,
+       * and obviously haven't given the new nodes any data.
+       */
+      p->second.acting.swap(acting);  // keep the latest
+    }
+  }
+
+  // scan pgs with waiters
+  map >::iterator p = waiting_for_pg.begin();
+  while (p != waiting_for_pg.end()) {
+    spg_t pgid = p->first;
+
+    vector acting;
+    int nrep = osdmap->pg_to_acting_osds(pgid.pgid, acting);
+    int role = osdmap->calc_pg_role(whoami, acting, nrep);
+    if (role >= 0) {
+      ++p;  // still me
+    } else {
+      dout(10) << " discarding waiting ops for " << pgid << dendl;
+      while (!p->second.empty()) {
+	p->second.pop_front();
+      }
+      waiting_for_pg.erase(p++);
+    }
+  }
+}
+
+void OSD::consume_map()
+{
+  assert(osd_lock.is_locked());
+  dout(7) << "consume_map version " << osdmap->get_epoch() << dendl;
+
+  int num_pg_primary = 0, num_pg_replica = 0, num_pg_stray = 0;
+  list to_remove;
+
+  // scan pg's
+  for (ceph::unordered_map::iterator it = pg_map.begin();
+       it != pg_map.end();
+       ++it) {
+    PG *pg = it->second;
+    pg->lock();
+    if (pg->is_primary())
+      num_pg_primary++;
+    else if (pg->is_replica())
+      num_pg_replica++;
+    else
+      num_pg_stray++;
+
+    if (!osdmap->have_pg_pool(pg->info.pgid.pool())) {
+      //pool is deleted!
+      to_remove.push_back(PGRef(pg));
+    } else {
+      service.init_splits_between(it->first, service.get_osdmap(), osdmap);
+    }
+
+    pg->unlock();
+  }
+
+  for (list::iterator i = to_remove.begin();
+       i != to_remove.end();
+       to_remove.erase(i++)) {
+    (*i)->lock();
+    _remove_pg(&**i);
+    (*i)->unlock();
+  }
+  to_remove.clear();
+
+  service.expand_pg_num(service.get_osdmap(), osdmap);
+
+  service.pre_publish_map(osdmap);
+  service.publish_map(osdmap);
+
+  // scan pg's
+  for (ceph::unordered_map::iterator it = pg_map.begin();
+       it != pg_map.end();
+       ++it) {
+    PG *pg = it->second;
+    pg->lock();
+    pg->queue_null(osdmap->get_epoch(), osdmap->get_epoch());
+    pg->unlock();
+  }
+  
+  logger->set(l_osd_pg, pg_map.size());
+  logger->set(l_osd_pg_primary, num_pg_primary);
+  logger->set(l_osd_pg_replica, num_pg_replica);
+  logger->set(l_osd_pg_stray, num_pg_stray);
+}
+
+void OSD::activate_map()
+{
+  assert(osd_lock.is_locked());
+
+  dout(7) << "activate_map version " << osdmap->get_epoch() << dendl;
+
+  wake_all_pg_waiters();   // the pg mapping may have shifted
+
+  if (osdmap->test_flag(CEPH_OSDMAP_FULL)) {
+    dout(10) << " osdmap flagged full, doing onetime osdmap subscribe" << dendl;
+    osdmap_subscribe(osdmap->get_epoch() + 1, true);
+  }
+
+  // norecover?
+  if (osdmap->test_flag(CEPH_OSDMAP_NORECOVER)) {
+    if (!paused_recovery) {
+      dout(1) << "pausing recovery (NORECOVER flag set)" << dendl;
+      paused_recovery = true;
+      recovery_tp.pause_new();
+    }
+  } else {
+    if (paused_recovery) {
+      dout(1) << "resuming recovery (NORECOVER flag cleared)" << dendl;
+      paused_recovery = false;
+      recovery_tp.unpause();
+    }
+  }
+
+  service.activate_map();
+
+  // process waiters
+  take_waiters(waiting_for_osdmap);
+}
+
+
+MOSDMap *OSD::build_incremental_map_msg(epoch_t since, epoch_t to)
+{
+  MOSDMap *m = new MOSDMap(monc->get_fsid());
+  m->oldest_map = superblock.oldest_map;
+  m->newest_map = superblock.newest_map;
+  
+  for (epoch_t e = to; e > since; e--) {
+    bufferlist bl;
+    if (e > m->oldest_map && get_inc_map_bl(e, bl)) {
+      m->incremental_maps[e].claim(bl);
+    } else if (get_map_bl(e, bl)) {
+      m->maps[e].claim(bl);
+      break;
+    } else {
+      derr << "since " << since << " to " << to
+	   << " oldest " << m->oldest_map << " newest " << m->newest_map
+	   << dendl;
+      assert(0 == "missing an osdmap on disk");  // we should have all maps.
+    }
+  }
+  return m;
+}
+
+void OSD::send_map(MOSDMap *m, Connection *con)
+{
+  Messenger *msgr = client_messenger;
+  if (entity_name_t::TYPE_OSD == con->get_peer_type())
+    msgr = cluster_messenger;
+  msgr->send_message(m, con);
+}
+
+void OSD::send_incremental_map(epoch_t since, Connection *con)
+{
+  epoch_t to = osdmap->get_epoch();
+  dout(10) << "send_incremental_map " << since << " -> " << to
+           << " to " << con << " " << con->get_peer_addr() << dendl;
+
+  if (since < superblock.oldest_map) {
+    // just send latest full map
+    MOSDMap *m = new MOSDMap(monc->get_fsid());
+    m->oldest_map = superblock.oldest_map;
+    m->newest_map = superblock.newest_map;
+    get_map_bl(to, m->maps[to]);
+    send_map(m, con);
+    return;
+  }
+
+  if (to > since && (int64_t)(to - since) > cct->_conf->osd_map_share_max_epochs) {
+    dout(10) << "  " << (to - since) << " > max " << cct->_conf->osd_map_share_max_epochs
+	     << ", only sending most recent" << dendl;
+    since = to - cct->_conf->osd_map_share_max_epochs;
+  }
+
+  while (since < to) {
+    if (to - since > (epoch_t)cct->_conf->osd_map_message_max)
+      to = since + cct->_conf->osd_map_message_max;
+    MOSDMap *m = build_incremental_map_msg(since, to);
+    send_map(m, con);
+    since = to;
+  }
+}
+
+bool OSDService::_get_map_bl(epoch_t e, bufferlist& bl)
+{
+  bool found = map_bl_cache.lookup(e, &bl);
+  if (found)
+    return true;
+  found = store->read(
+    coll_t::META_COLL, OSD::get_osdmap_pobject_name(e), 0, 0, bl) >= 0;
+  if (found)
+    _add_map_bl(e, bl);
+  return found;
+}
+
+bool OSDService::get_inc_map_bl(epoch_t e, bufferlist& bl)
+{
+  Mutex::Locker l(map_cache_lock);
+  bool found = map_bl_inc_cache.lookup(e, &bl);
+  if (found)
+    return true;
+  found = store->read(
+    coll_t::META_COLL, OSD::get_inc_osdmap_pobject_name(e), 0, 0, bl) >= 0;
+  if (found)
+    _add_map_inc_bl(e, bl);
+  return found;
+}
+
+void OSDService::_add_map_bl(epoch_t e, bufferlist& bl)
+{
+  dout(10) << "add_map_bl " << e << " " << bl.length() << " bytes" << dendl;
+  map_bl_cache.add(e, bl);
+}
+
+void OSDService::_add_map_inc_bl(epoch_t e, bufferlist& bl)
+{
+  dout(10) << "add_map_inc_bl " << e << " " << bl.length() << " bytes" << dendl;
+  map_bl_inc_cache.add(e, bl);
+}
+
+void OSDService::pin_map_inc_bl(epoch_t e, bufferlist &bl)
+{
+  Mutex::Locker l(map_cache_lock);
+  map_bl_inc_cache.pin(e, bl);
+}
+
+void OSDService::pin_map_bl(epoch_t e, bufferlist &bl)
+{
+  Mutex::Locker l(map_cache_lock);
+  map_bl_cache.pin(e, bl);
+}
+
+void OSDService::clear_map_bl_cache_pins(epoch_t e)
+{
+  Mutex::Locker l(map_cache_lock);
+  map_bl_inc_cache.clear_pinned(e);
+  map_bl_cache.clear_pinned(e);
+}
+
+OSDMapRef OSDService::_add_map(OSDMap *o)
+{
+  epoch_t e = o->get_epoch();
+
+  if (cct->_conf->osd_map_dedup) {
+    // Dedup against an existing map at a nearby epoch
+    OSDMapRef for_dedup = map_cache.lower_bound(e);
+    if (for_dedup) {
+      OSDMap::dedup(for_dedup.get(), o);
+    }
+  }
+  OSDMapRef l = map_cache.add(e, o);
+  return l;
+}
+
+OSDMapRef OSDService::try_get_map(epoch_t epoch)
+{
+  Mutex::Locker l(map_cache_lock);
+  OSDMapRef retval = map_cache.lookup(epoch);
+  if (retval) {
+    dout(30) << "get_map " << epoch << " -cached" << dendl;
+    return retval;
+  }
+
+  OSDMap *map = new OSDMap;
+  if (epoch > 0) {
+    dout(20) << "get_map " << epoch << " - loading and decoding " << map << dendl;
+    bufferlist bl;
+    if (!_get_map_bl(epoch, bl)) {
+      delete map;
+      return OSDMapRef();
+    }
+    map->decode(bl);
+  } else {
+    dout(20) << "get_map " << epoch << " - return initial " << map << dendl;
+  }
+  return _add_map(map);
+}
+
+bool OSD::require_mon_peer(Message *m)
+{
+  if (!m->get_connection()->peer_is_mon()) {
+    dout(0) << "require_mon_peer received from non-mon " << m->get_connection()->get_peer_addr()
+	    << " " << *m << dendl;
+    m->put();
+    return false;
+  }
+  return true;
+}
+
+bool OSD::require_osd_peer(OpRequestRef& op)
+{
+  if (!op->get_req()->get_connection()->peer_is_osd()) {
+    dout(0) << "require_osd_peer received from non-osd " << op->get_req()->get_connection()->get_peer_addr()
+	    << " " << *op->get_req() << dendl;
+    return false;
+  }
+  return true;
+}
+
+bool OSD::require_self_aliveness(OpRequestRef& op, epoch_t epoch)
+{
+  if (epoch < up_epoch) {
+    dout(7) << "from pre-up epoch " << epoch << " < " << up_epoch << dendl;
+    return false;
+  }
+
+  if (!is_active()) {
+    dout(7) << "still in boot state, dropping message " << *op->get_req() << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+bool OSD::require_same_peer_instance(OpRequestRef& op, OSDMapRef& map)
+{
+  Message *m = op->get_req();
+  int from = m->get_source().num();
+
+  if (!map->have_inst(from) ||
+      (map->get_cluster_addr(from) != m->get_source_inst().addr)) {
+    dout(5) << "from dead osd." << from << ", marking down, "
+	    << " msg was " << m->get_source_inst().addr
+	    << " expected " << (map->have_inst(from) ?
+				map->get_cluster_addr(from) : entity_addr_t())
+	    << dendl;
+    ConnectionRef con = m->get_connection();
+    cluster_messenger->mark_down(con.get());
+    Session *s = static_cast(con->get_priv());
+    if (s) {
+      con->set_priv(NULL);   // break ref <-> session cycle, if any
+      s->put();
+    }
+    return false;
+  }
+  return true;
+}
+
+bool OSD::require_up_osd_peer(OpRequestRef& op, OSDMapRef& map,
+                              epoch_t their_epoch)
+{
+  if (!require_self_aliveness(op, their_epoch)) {
+    return false;
+  } else if (!require_osd_peer(op)) {
+    return false;
+  } else if (map->get_epoch() >= their_epoch &&
+	     !require_same_peer_instance(op, map)) {
+    return false;
+  }
+  return true;
+}
+
+/*
+ * require that we have same (or newer) map, and that
+ * the source is the pg primary.
+ */
+bool OSD::require_same_or_newer_map(OpRequestRef& op, epoch_t epoch)
+{
+  Message *m = op->get_req();
+  dout(15) << "require_same_or_newer_map " << epoch << " (i am " << osdmap->get_epoch() << ") " << m << dendl;
+
+  assert(osd_lock.is_locked());
+
+  // do they have a newer map?
+  if (epoch > osdmap->get_epoch()) {
+    dout(7) << "waiting for newer map epoch " << epoch << " > my " << osdmap->get_epoch() << " with " << m << dendl;
+    wait_for_new_map(op);
+    return false;
+  }
+
+  if (!require_self_aliveness(op, epoch)) {
+    return false;
+  }
+
+  // ok, our map is same or newer.. do they still exist?
+  if (m->get_connection()->get_messenger() == cluster_messenger &&
+      !require_same_peer_instance(op, osdmap)) {
+    return false;
+  }
+
+  return true;
+}
+
+
+
+
+
+// ----------------------------------------
+// pg creation
+
+
+bool OSD::can_create_pg(spg_t pgid)
+{
+  assert(creating_pgs.count(pgid));
+
+  // priors empty?
+  if (!creating_pgs[pgid].prior.empty()) {
+    dout(10) << "can_create_pg " << pgid
+	     << " - waiting for priors " << creating_pgs[pgid].prior << dendl;
+    return false;
+  }
+
+  dout(10) << "can_create_pg " << pgid << " - can create now" << dendl;
+  return true;
+}
+
+void OSD::split_pgs(
+  PG *parent,
+  const set &childpgids, set > *out_pgs,
+  OSDMapRef curmap,
+  OSDMapRef nextmap,
+  PG::RecoveryCtx *rctx)
+{
+  unsigned pg_num = nextmap->get_pg_num(
+    parent->pool.id);
+  parent->update_snap_mapper_bits(
+    parent->info.pgid.get_split_bits(pg_num)
+    );
+
+  vector updated_stats(childpgids.size() + 1);
+  parent->info.stats.stats.sum.split(updated_stats);
+
+  vector::iterator stat_iter = updated_stats.begin();
+  for (set::const_iterator i = childpgids.begin();
+       i != childpgids.end();
+       ++i, ++stat_iter) {
+    assert(stat_iter != updated_stats.end());
+    dout(10) << "Splitting " << *parent << " into " << *i << dendl;
+    assert(service.splitting(*i));
+    PG* child = _make_pg(nextmap, *i);
+    child->lock(true);
+    out_pgs->insert(child);
+
+    unsigned split_bits = i->get_split_bits(pg_num);
+    dout(10) << "pg_num is " << pg_num << dendl;
+    dout(10) << "m_seed " << i->ps() << dendl;
+    dout(10) << "split_bits is " << split_bits << dendl;
+
+    parent->split_colls(
+      *i,
+      split_bits,
+      i->ps(),
+      rctx->transaction);
+    parent->split_into(
+      i->pgid,
+      child,
+      split_bits);
+    child->info.stats.stats.sum = *stat_iter;
+
+    child->write_if_dirty(*(rctx->transaction));
+    child->unlock();
+  }
+  assert(stat_iter != updated_stats.end());
+  parent->info.stats.stats.sum = *stat_iter;
+  parent->write_if_dirty(*(rctx->transaction));
+}
+  
+/*
+ * holding osd_lock
+ */
+void OSD::handle_pg_create(OpRequestRef op)
+{
+  MOSDPGCreate *m = (MOSDPGCreate*)op->get_req();
+  assert(m->get_header().type == MSG_OSD_PG_CREATE);
+
+  dout(10) << "handle_pg_create " << *m << dendl;
+
+  // drop the next N pg_creates in a row?
+  if (debug_drop_pg_create_left < 0 &&
+      cct->_conf->osd_debug_drop_pg_create_probability >
+      ((((double)(rand()%100))/100.0))) {
+    debug_drop_pg_create_left = debug_drop_pg_create_duration;
+  }
+  if (debug_drop_pg_create_left >= 0) {
+    --debug_drop_pg_create_left;
+    if (debug_drop_pg_create_left >= 0) {
+      dout(0) << "DEBUG dropping/ignoring pg_create, will drop the next "
+	      << debug_drop_pg_create_left << " too" << dendl;
+      return;
+    }
+  }
+
+  /* we have to hack around require_mon_peer's interface limits, so
+   * grab an extra reference before going in. If the peer isn't
+   * a Monitor, the reference is put for us (and then cleared
+   * up automatically by our OpTracker infrastructure). Otherwise,
+   * we put the extra ref ourself.
+   */
+  if (!require_mon_peer(op->get_req()->get())) {
+    return;
+  }
+  op->get_req()->put();
+
+  if (!require_same_or_newer_map(op, m->epoch)) return;
+
+  op->mark_started();
+
+  int num_created = 0;
+
+  for (map::iterator p = m->mkpg.begin();
+       p != m->mkpg.end();
+       ++p) {
+    epoch_t created = p->second.created;
+    pg_t parent = p->second.parent;
+    if (p->second.split_bits) // Skip split pgs
+      continue;
+    pg_t on = p->first;
+
+    if (on.preferred() >= 0) {
+      dout(20) << "ignoring localized pg " << on << dendl;
+      continue;
+    }
+
+    if (!osdmap->have_pg_pool(on.pool())) {
+      dout(20) << "ignoring pg on deleted pool " << on << dendl;
+      continue;
+    }
+
+    dout(20) << "mkpg " << on << " e" << created << dendl;
+   
+    // is it still ours?
+    vector up, acting;
+    int up_primary = -1;
+    int acting_primary = -1;
+    osdmap->pg_to_up_acting_osds(on, &up, &up_primary, &acting, &acting_primary);
+    int role = osdmap->calc_pg_role(whoami, acting, acting.size());
+
+    if (up_primary != whoami) {
+      dout(10) << "mkpg " << on << "  not primary (role="
+	       << role << "), skipping" << dendl;
+      continue;
+    }
+    if (up != acting) {
+      dout(10) << "mkpg " << on << "  up " << up
+	       << " != acting " << acting << ", ignoring" << dendl;
+      // we'll get a query soon anyway, since we know the pg
+      // must exist. we can ignore this.
+      continue;
+    }
+
+    spg_t pgid;
+    bool mapped = osdmap->get_primary_shard(on, &pgid);
+    assert(mapped);
+
+    // does it already exist?
+    if (_have_pg(pgid)) {
+      dout(10) << "mkpg " << pgid << "  already exists, skipping" << dendl;
+      continue;
+    }
+
+    // figure history
+    pg_history_t history;
+    history.epoch_created = created;
+    history.last_epoch_clean = created;
+    // Newly created PGs don't need to scrub immediately, so mark them
+    // as scrubbed at creation time.
+    utime_t now = ceph_clock_now(NULL);
+    history.last_scrub_stamp = now;
+    history.last_deep_scrub_stamp = now;
+    bool valid_history = project_pg_history(
+      pgid, history, created, up, up_primary, acting, acting_primary);
+    /* the pg creation message must have come from a mon and therefore
+     * cannot be on the other side of a map gap
+     */
+    assert(valid_history);
+    
+    // register.
+    creating_pgs[pgid].history = history;
+    creating_pgs[pgid].parent = parent;
+    creating_pgs[pgid].acting.swap(acting);
+    calc_priors_during(pgid, created, history.same_interval_since, 
+		       creating_pgs[pgid].prior);
+
+    PG::RecoveryCtx rctx = create_context();
+    // poll priors
+    set& pset = creating_pgs[pgid].prior;
+    dout(10) << "mkpg " << pgid << " e" << created
+	     << " h " << history
+	     << " : querying priors " << pset << dendl;
+    for (set::iterator p = pset.begin(); p != pset.end(); ++p)
+      if (osdmap->is_up(p->osd))
+	(*rctx.query_map)[p->osd][spg_t(pgid.pgid, p->shard)] =
+	  pg_query_t(
+	    pg_query_t::INFO,
+	    p->shard, pgid.shard,
+	    history,
+	    osdmap->get_epoch());
+
+    PG *pg = NULL;
+    if (can_create_pg(pgid)) {
+      pg_interval_map_t pi;
+      rctx.transaction->create_collection(coll_t(pgid));
+      pg = _create_lock_pg(
+	osdmap, pgid, true, false, false,
+	0, creating_pgs[pgid].acting, whoami,
+	creating_pgs[pgid].acting, whoami,
+	history, pi,
+	*rctx.transaction);
+      pg->info.last_epoch_started = pg->info.history.last_epoch_started;
+      creating_pgs.erase(pgid);
+      wake_pg_waiters(pg->info.pgid);
+      pg->handle_create(&rctx);
+      pg->write_if_dirty(*rctx.transaction);
+      pg->publish_stats_to_osd();
+      pg->unlock();
+      num_created++;
+    }
+    dispatch_context(rctx, pg, osdmap);
+  }
+
+  maybe_update_heartbeat_peers();
+}
+
+
+// ----------------------------------------
+// peering and recovery
+
+PG::RecoveryCtx OSD::create_context()
+{
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  C_Contexts *on_applied = new C_Contexts(cct);
+  C_Contexts *on_safe = new C_Contexts(cct);
+  map > *query_map =
+    new map >;
+  map > > *notify_list =
+    new map > >;
+  map > > *info_map =
+    new map > >;
+  PG::RecoveryCtx rctx(query_map, info_map, notify_list,
+		       on_applied, on_safe, t);
+  return rctx;
+}
+
+void OSD::dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg,
+                                       ThreadPool::TPHandle *handle)
+{
+  if (!ctx.transaction->empty()) {
+    ctx.on_applied->add(new ObjectStore::C_DeleteTransaction(ctx.transaction));
+    int tr = store->queue_transaction(
+      pg->osr.get(),
+      ctx.transaction, ctx.on_applied, ctx.on_safe, NULL,
+      TrackedOpRef(), handle);
+    assert(tr == 0);
+    ctx.transaction = new ObjectStore::Transaction;
+    ctx.on_applied = new C_Contexts(cct);
+    ctx.on_safe = new C_Contexts(cct);
+  }
+}
+
+bool OSD::compat_must_dispatch_immediately(PG *pg)
+{
+  assert(pg->is_locked());
+  set tmpacting;
+  if (!pg->actingbackfill.empty()) {
+    tmpacting = pg->actingbackfill;
+  } else {
+    for (unsigned i = 0; i < pg->acting.size(); ++i) {
+      tmpacting.insert(
+	pg_shard_t(
+	  pg->acting[i],
+	  pg->pool.info.ec_pool() ? i : ghobject_t::NO_SHARD));
+    }
+  }
+
+  for (set::iterator i = tmpacting.begin();
+       i != tmpacting.end();
+       ++i) {
+    if (i->osd == whoami)
+      continue;
+    ConnectionRef conn =
+      service.get_con_osd_cluster(i->osd, pg->get_osdmap()->get_epoch());
+    if (conn && !conn->has_feature(CEPH_FEATURE_INDEP_PG_MAP)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+void OSD::dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap,
+                           ThreadPool::TPHandle *handle)
+{
+  if (service.get_osdmap()->is_up(whoami) &&
+      is_active()) {
+    do_notifies(*ctx.notify_list, curmap);
+    do_queries(*ctx.query_map, curmap);
+    do_infos(*ctx.info_map, curmap);
+  }
+  delete ctx.notify_list;
+  delete ctx.query_map;
+  delete ctx.info_map;
+  if ((ctx.on_applied->empty() &&
+       ctx.on_safe->empty() &&
+       ctx.transaction->empty()) || !pg) {
+    delete ctx.transaction;
+    delete ctx.on_applied;
+    delete ctx.on_safe;
+  } else {
+    ctx.on_applied->add(new ObjectStore::C_DeleteTransaction(ctx.transaction));
+    int tr = store->queue_transaction(
+      pg->osr.get(),
+      ctx.transaction, ctx.on_applied, ctx.on_safe, NULL, TrackedOpRef(),
+      handle);
+    assert(tr == 0);
+  }
+}
+
+/** do_notifies
+ * Send an MOSDPGNotify to a primary, with a list of PGs that I have
+ * content for, and they are primary for.
+ */
+
+void OSD::do_notifies(
+  map > >& notify_list,
+  OSDMapRef curmap)
+{
+  for (map > >::iterator it =
+	 notify_list.begin();
+       it != notify_list.end();
+       ++it) {
+    if (!curmap->is_up(it->first))
+      continue;
+    ConnectionRef con = service.get_con_osd_cluster(
+      it->first, curmap->get_epoch());
+    if (!con)
+      continue;
+    _share_map_outgoing(it->first, con.get(), curmap);
+    if (con->has_feature(CEPH_FEATURE_INDEP_PG_MAP)) {
+      dout(7) << "do_notify osd " << it->first
+	      << " on " << it->second.size() << " PGs" << dendl;
+      MOSDPGNotify *m = new MOSDPGNotify(curmap->get_epoch(),
+					 it->second);
+      cluster_messenger->send_message(m, con.get());
+    } else {
+      dout(7) << "do_notify osd " << it->first
+	      << " sending separate messages" << dendl;
+      for (vector >::iterator i =
+	     it->second.begin();
+	   i != it->second.end();
+	   ++i) {
+	vector > list(1);
+	list[0] = *i;
+	MOSDPGNotify *m = new MOSDPGNotify(i->first.epoch_sent,
+					   list);
+	cluster_messenger->send_message(m, con.get());
+      }
+    }
+  }
+}
+
+
+/** do_queries
+ * send out pending queries for info | summaries
+ */
+void OSD::do_queries(map >& query_map,
+		     OSDMapRef curmap)
+{
+  for (map >::iterator pit = query_map.begin();
+       pit != query_map.end();
+       ++pit) {
+    if (!curmap->is_up(pit->first))
+      continue;
+    int who = pit->first;
+    ConnectionRef con = service.get_con_osd_cluster(who, curmap->get_epoch());
+    if (!con)
+      continue;
+    _share_map_outgoing(who, con.get(), curmap);
+    if (con->has_feature(CEPH_FEATURE_INDEP_PG_MAP)) {
+      dout(7) << "do_queries querying osd." << who
+	      << " on " << pit->second.size() << " PGs" << dendl;
+      MOSDPGQuery *m = new MOSDPGQuery(curmap->get_epoch(), pit->second);
+      cluster_messenger->send_message(m, con.get());
+    } else {
+      dout(7) << "do_queries querying osd." << who
+	      << " sending saperate messages "
+	      << " on " << pit->second.size() << " PGs" << dendl;
+      for (map::iterator i = pit->second.begin();
+	   i != pit->second.end();
+	   ++i) {
+	map to_send;
+	to_send.insert(*i);
+	MOSDPGQuery *m = new MOSDPGQuery(i->second.epoch_sent, to_send);
+	cluster_messenger->send_message(m, con.get());
+      }
+    }
+  }
+}
+
+
+void OSD::do_infos(map > >& info_map,
+		   OSDMapRef curmap)
+{
+  for (map > >::iterator p =
+	 info_map.begin();
+       p != info_map.end();
+       ++p) { 
+    if (!curmap->is_up(p->first))
+      continue;
+    for (vector >::iterator i = p->second.begin();
+	 i != p->second.end();
+	 ++i) {
+      dout(20) << "Sending info " << i->first.info
+	       << " to shard " << p->first << dendl;
+    }
+    ConnectionRef con = service.get_con_osd_cluster(
+      p->first, curmap->get_epoch());
+    if (!con)
+      continue;
+    _share_map_outgoing(p->first, con.get(), curmap);
+    if (con->has_feature(CEPH_FEATURE_INDEP_PG_MAP)) {
+      MOSDPGInfo *m = new MOSDPGInfo(curmap->get_epoch());
+      m->pg_list = p->second;
+      cluster_messenger->send_message(m, con.get());
+    } else {
+      for (vector >::iterator i =
+	     p->second.begin();
+	   i != p->second.end();
+	   ++i) {
+	vector > to_send(1);
+	to_send[0] = *i;
+	MOSDPGInfo *m = new MOSDPGInfo(i->first.epoch_sent);
+	m->pg_list = to_send;
+	cluster_messenger->send_message(m, con.get());
+      }
+    }
+  }
+  info_map.clear();
+}
+
+
+/** PGNotify
+ * from non-primary to primary
+ * includes pg_info_t.
+ * NOTE: called with opqueue active.
+ */
+void OSD::handle_pg_notify(OpRequestRef op)
+{
+  MOSDPGNotify *m = (MOSDPGNotify*)op->get_req();
+  assert(m->get_header().type == MSG_OSD_PG_NOTIFY);
+
+  dout(7) << "handle_pg_notify from " << m->get_source() << dendl;
+  int from = m->get_source().num();
+
+  if (!require_osd_peer(op))
+    return;
+
+  if (!require_same_or_newer_map(op, m->get_epoch())) return;
+
+  op->mark_started();
+
+  for (vector >::iterator it = m->get_pg_list().begin();
+       it != m->get_pg_list().end();
+       ++it) {
+
+    if (it->first.info.pgid.preferred() >= 0) {
+      dout(20) << "ignoring localized pg " << it->first.info.pgid << dendl;
+      continue;
+    }
+
+    handle_pg_peering_evt(
+      spg_t(it->first.info.pgid.pgid, it->first.to),
+      it->first.info, it->second,
+      it->first.query_epoch, pg_shard_t(from, it->first.from), true,
+      PG::CephPeeringEvtRef(
+	new PG::CephPeeringEvt(
+	  it->first.epoch_sent, it->first.query_epoch,
+	  PG::MNotifyRec(pg_shard_t(from, it->first.from), it->first)))
+      );
+  }
+}
+
+void OSD::handle_pg_log(OpRequestRef op)
+{
+  MOSDPGLog *m = (MOSDPGLog*) op->get_req();
+  assert(m->get_header().type == MSG_OSD_PG_LOG);
+  dout(7) << "handle_pg_log " << *m << " from " << m->get_source() << dendl;
+
+  if (!require_osd_peer(op))
+    return;
+
+  int from = m->get_source().num();
+  if (!require_same_or_newer_map(op, m->get_epoch())) return;
+
+  if (m->info.pgid.preferred() >= 0) {
+    dout(10) << "ignoring localized pg " << m->info.pgid << dendl;
+    return;
+  }
+
+  op->mark_started();
+  handle_pg_peering_evt(
+    spg_t(m->info.pgid.pgid, m->to),
+    m->info, m->past_intervals, m->get_epoch(),
+    pg_shard_t(from, m->from), false,
+    PG::CephPeeringEvtRef(
+      new PG::CephPeeringEvt(
+	m->get_epoch(), m->get_query_epoch(),
+	PG::MLogRec(pg_shard_t(from, m->from), m)))
+    );
+}
+
+void OSD::handle_pg_info(OpRequestRef op)
+{
+  MOSDPGInfo *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_INFO);
+  dout(7) << "handle_pg_info " << *m << " from " << m->get_source() << dendl;
+
+  if (!require_osd_peer(op))
+    return;
+
+  int from = m->get_source().num();
+  if (!require_same_or_newer_map(op, m->get_epoch())) return;
+
+  op->mark_started();
+
+  for (vector >::iterator p = m->pg_list.begin();
+       p != m->pg_list.end();
+       ++p) {
+    if (p->first.info.pgid.preferred() >= 0) {
+      dout(10) << "ignoring localized pg " << p->first.info.pgid << dendl;
+      continue;
+    }
+
+    handle_pg_peering_evt(
+      spg_t(p->first.info.pgid.pgid, p->first.to),
+      p->first.info, p->second, p->first.epoch_sent,
+      pg_shard_t(from, p->first.from), false,
+      PG::CephPeeringEvtRef(
+	new PG::CephPeeringEvt(
+	  p->first.epoch_sent, p->first.query_epoch,
+	  PG::MInfoRec(
+	    pg_shard_t(
+	      from, p->first.from), p->first.info, p->first.epoch_sent)))
+      );
+  }
+}
+
+void OSD::handle_pg_trim(OpRequestRef op)
+{
+  MOSDPGTrim *m = (MOSDPGTrim *)op->get_req();
+  assert(m->get_header().type == MSG_OSD_PG_TRIM);
+
+  dout(7) << "handle_pg_trim " << *m << " from " << m->get_source() << dendl;
+
+  if (!require_osd_peer(op))
+    return;
+
+  int from = m->get_source().num();
+  if (!require_same_or_newer_map(op, m->epoch)) return;
+
+  if (m->pgid.preferred() >= 0) {
+    dout(10) << "ignoring localized pg " << m->pgid << dendl;
+    return;
+  }
+
+  op->mark_started();
+
+  if (!_have_pg(m->pgid)) {
+    dout(10) << " don't have pg " << m->pgid << dendl;
+  } else {
+    PG *pg = _lookup_lock_pg(m->pgid);
+    if (m->epoch < pg->info.history.same_interval_since) {
+      dout(10) << *pg << " got old trim to " << m->trim_to << ", ignoring" << dendl;
+      pg->unlock();
+      return;
+    }
+    assert(pg);
+
+    if (pg->is_primary()) {
+      // peer is informing us of their last_complete_ondisk
+      dout(10) << *pg << " replica osd." << from << " lcod " << m->trim_to << dendl;
+      pg->peer_last_complete_ondisk[pg_shard_t(from, m->pgid.shard)] =
+	m->trim_to;
+      if (pg->calc_min_last_complete_ondisk()) {
+	dout(10) << *pg << " min lcod now " << pg->min_last_complete_ondisk << dendl;
+	pg->trim_peers();
+      }
+    } else {
+      // primary is instructing us to trim
+      ObjectStore::Transaction *t = new ObjectStore::Transaction;
+      PG::PGLogEntryHandler handler;
+      pg->pg_log.trim(&handler, m->trim_to, pg->info);
+      handler.apply(pg, t);
+      pg->dirty_info = true;
+      pg->write_if_dirty(*t);
+      int tr = store->queue_transaction(
+	pg->osr.get(), t,
+	new ObjectStore::C_DeleteTransaction(t));
+      assert(tr == 0);
+    }
+    pg->unlock();
+  }
+}
+
+void OSD::handle_pg_scan(OpRequestRef op)
+{
+  MOSDPGScan *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_SCAN);
+  dout(10) << "handle_pg_scan " << *m << " from " << m->get_source() << dendl;
+  
+  if (!require_osd_peer(op))
+    return;
+  if (!require_same_or_newer_map(op, m->query_epoch))
+    return;
+
+  if (m->pgid.preferred() >= 0) {
+    dout(10) << "ignoring localized pg " << m->pgid << dendl;
+    return;
+  }
+
+  PG *pg;
+  
+  if (!_have_pg(m->pgid)) {
+    return;
+  }
+
+  pg = _lookup_pg(m->pgid);
+  assert(pg);
+
+  enqueue_op(pg, op);
+}
+
+void OSD::handle_pg_backfill(OpRequestRef op)
+{
+  MOSDPGBackfill *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_BACKFILL);
+  dout(10) << "handle_pg_backfill " << *m << " from " << m->get_source() << dendl;
+  
+  if (!require_osd_peer(op))
+    return;
+  if (!require_same_or_newer_map(op, m->query_epoch))
+    return;
+
+  if (m->pgid.preferred() >= 0) {
+    dout(10) << "ignoring localized pg " << m->pgid << dendl;
+    return;
+  }
+
+  PG *pg;
+  
+  if (!_have_pg(m->pgid)) {
+    return;
+  }
+
+  pg = _lookup_pg(m->pgid);
+  assert(pg);
+
+  enqueue_op(pg, op);
+}
+
+void OSD::handle_pg_backfill_reserve(OpRequestRef op)
+{
+  MBackfillReserve *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_BACKFILL_RESERVE);
+
+  if (!require_osd_peer(op))
+    return;
+  if (!require_same_or_newer_map(op, m->query_epoch))
+    return;
+
+  PG::CephPeeringEvtRef evt;
+  if (m->type == MBackfillReserve::REQUEST) {
+    evt = PG::CephPeeringEvtRef(
+      new PG::CephPeeringEvt(
+	m->query_epoch,
+	m->query_epoch,
+	PG::RequestBackfillPrio(m->priority)));
+  } else if (m->type == MBackfillReserve::GRANT) {
+    evt = PG::CephPeeringEvtRef(
+      new PG::CephPeeringEvt(
+	m->query_epoch,
+	m->query_epoch,
+	PG::RemoteBackfillReserved()));
+  } else if (m->type == MBackfillReserve::REJECT) {
+    evt = PG::CephPeeringEvtRef(
+      new PG::CephPeeringEvt(
+	m->query_epoch,
+	m->query_epoch,
+	PG::RemoteReservationRejected()));
+  } else {
+    assert(0);
+  }
+
+  if (service.splitting(m->pgid)) {
+    peering_wait_for_split[m->pgid].push_back(evt);
+    return;
+  }
+
+  PG *pg = 0;
+  if (!_have_pg(m->pgid))
+    return;
+
+  pg = _lookup_lock_pg(m->pgid);
+  assert(pg);
+
+  pg->queue_peering_event(evt);
+  pg->unlock();
+}
+
+void OSD::handle_pg_recovery_reserve(OpRequestRef op)
+{
+  MRecoveryReserve *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_RECOVERY_RESERVE);
+
+  if (!require_osd_peer(op))
+    return;
+  if (!require_same_or_newer_map(op, m->query_epoch))
+    return;
+
+  PG::CephPeeringEvtRef evt;
+  if (m->type == MRecoveryReserve::REQUEST) {
+    evt = PG::CephPeeringEvtRef(
+      new PG::CephPeeringEvt(
+	m->query_epoch,
+	m->query_epoch,
+	PG::RequestRecovery()));
+  } else if (m->type == MRecoveryReserve::GRANT) {
+    evt = PG::CephPeeringEvtRef(
+      new PG::CephPeeringEvt(
+	m->query_epoch,
+	m->query_epoch,
+	PG::RemoteRecoveryReserved()));
+  } else if (m->type == MRecoveryReserve::RELEASE) {
+    evt = PG::CephPeeringEvtRef(
+      new PG::CephPeeringEvt(
+	m->query_epoch,
+	m->query_epoch,
+	PG::RecoveryDone()));
+  } else {
+    assert(0);
+  }
+
+  if (service.splitting(m->pgid)) {
+    peering_wait_for_split[m->pgid].push_back(evt);
+    return;
+  }
+
+  PG *pg = 0;
+  if (!_have_pg(m->pgid))
+    return;
+
+  pg = _lookup_lock_pg(m->pgid);
+  assert(pg);
+
+  pg->queue_peering_event(evt);
+  pg->unlock();
+}
+
+
+/** PGQuery
+ * from primary to replica | stray
+ * NOTE: called with opqueue active.
+ */
+void OSD::handle_pg_query(OpRequestRef op)
+{
+  assert(osd_lock.is_locked());
+
+  MOSDPGQuery *m = (MOSDPGQuery*)op->get_req();
+  assert(m->get_header().type == MSG_OSD_PG_QUERY);
+
+  if (!require_osd_peer(op))
+    return;
+
+  dout(7) << "handle_pg_query from " << m->get_source() << " epoch " << m->get_epoch() << dendl;
+  int from = m->get_source().num();
+  
+  if (!require_same_or_newer_map(op, m->get_epoch())) return;
+
+  op->mark_started();
+
+  map< int, vector > > notify_list;
+  
+  for (map::iterator it = m->pg_list.begin();
+       it != m->pg_list.end();
+       ++it) {
+    spg_t pgid = it->first;
+
+    if (pgid.preferred() >= 0) {
+      dout(10) << "ignoring localized pg " << pgid << dendl;
+      continue;
+    }
+
+    if (service.splitting(pgid)) {
+      peering_wait_for_split[pgid].push_back(
+	PG::CephPeeringEvtRef(
+	  new PG::CephPeeringEvt(
+	    it->second.epoch_sent, it->second.epoch_sent,
+	    PG::MQuery(pg_shard_t(from, it->second.from),
+		       it->second, it->second.epoch_sent))));
+      continue;
+    }
+
+    if (pg_map.count(pgid)) {
+      PG *pg = 0;
+      pg = _lookup_lock_pg(pgid);
+      pg->queue_query(
+	it->second.epoch_sent, it->second.epoch_sent,
+	pg_shard_t(from, it->second.from), it->second);
+      pg->unlock();
+      continue;
+    }
+
+    if (!osdmap->have_pg_pool(pgid.pool()))
+      continue;
+
+    // get active crush mapping
+    int up_primary, acting_primary;
+    vector up, acting;
+    osdmap->pg_to_up_acting_osds(
+      pgid.pgid, &up, &up_primary, &acting, &acting_primary);
+
+    // same primary?
+    pg_history_t history = it->second.history;
+    bool valid_history = project_pg_history(
+      pgid, history, it->second.epoch_sent,
+      up, up_primary, acting, acting_primary);
+
+    if (!valid_history ||
+        it->second.epoch_sent < history.same_interval_since) {
+      dout(10) << " pg " << pgid << " dne, and pg has changed in "
+	       << history.same_interval_since
+	       << " (msg from " << it->second.epoch_sent << ")" << dendl;
+      continue;
+    }
+
+    dout(10) << " pg " << pgid << " dne" << dendl;
+    pg_info_t empty(spg_t(pgid.pgid, it->second.to));
+    /* This is racy, but that should be ok: if we complete the deletion
+     * before the pg is recreated, we'll just start it off backfilling
+     * instead of just empty */
+    if (service.deleting_pgs.lookup(pgid))
+      empty.last_backfill = hobject_t();
+    if (it->second.type == pg_query_t::LOG ||
+	it->second.type == pg_query_t::FULLLOG) {
+      ConnectionRef con = service.get_con_osd_cluster(from, osdmap->get_epoch());
+      if (con) {
+	MOSDPGLog *mlog = new MOSDPGLog(
+	  it->second.from, it->second.to,
+	  osdmap->get_epoch(), empty,
+	  it->second.epoch_sent);
+	_share_map_outgoing(from, con.get(), osdmap);
+	cluster_messenger->send_message(mlog, con.get());
+      }
+    } else {
+      notify_list[from].push_back(
+	make_pair(
+	  pg_notify_t(
+	    it->second.from, it->second.to,
+	    it->second.epoch_sent,
+	    osdmap->get_epoch(),
+	    empty),
+	  pg_interval_map_t()));
+    }
+  }
+  do_notifies(notify_list, osdmap);
+}
+
+
+void OSD::handle_pg_remove(OpRequestRef op)
+{
+  MOSDPGRemove *m = (MOSDPGRemove *)op->get_req();
+  assert(m->get_header().type == MSG_OSD_PG_REMOVE);
+  assert(osd_lock.is_locked());
+
+  if (!require_osd_peer(op))
+    return;
+
+  dout(7) << "handle_pg_remove from " << m->get_source() << " on "
+	  << m->pg_list.size() << " pgs" << dendl;
+  
+  if (!require_same_or_newer_map(op, m->get_epoch())) return;
+  
+  op->mark_started();
+
+  for (vector::iterator it = m->pg_list.begin();
+       it != m->pg_list.end();
+       ++it) {
+    spg_t pgid = *it;
+    if (pgid.preferred() >= 0) {
+      dout(10) << "ignoring localized pg " << pgid << dendl;
+      continue;
+    }
+    
+    if (pg_map.count(pgid) == 0) {
+      dout(10) << " don't have pg " << pgid << dendl;
+      continue;
+    }
+    dout(5) << "queue_pg_for_deletion: " << pgid << dendl;
+    PG *pg = _lookup_lock_pg(pgid);
+    pg_history_t history = pg->info.history;
+    int up_primary, acting_primary;
+    vector up, acting;
+    osdmap->pg_to_up_acting_osds(
+      pgid.pgid, &up, &up_primary, &acting, &acting_primary);
+    bool valid_history = project_pg_history(
+      pg->info.pgid, history, pg->get_osdmap()->get_epoch(),
+      up, up_primary, acting, acting_primary);
+    if (valid_history &&
+        history.same_interval_since <= m->get_epoch()) {
+      assert(pg->get_primary().osd == m->get_source().num());
+      PGRef _pg(pg);
+      _remove_pg(pg);
+      pg->unlock();
+    } else {
+      dout(10) << *pg << " ignoring remove request, pg changed in epoch "
+	       << history.same_interval_since
+	       << " > " << m->get_epoch() << dendl;
+      pg->unlock();
+    }
+  }
+}
+
+void OSD::_remove_pg(PG *pg)
+{
+  ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
+
+  // on_removal, which calls remove_watchers_and_notifies, and the erasure from
+  // the pg_map must be done together without unlocking the pg lock,
+  // to avoid racing with watcher cleanup in ms_handle_reset
+  // and handle_notify_timeout
+  pg->on_removal(rmt);
+
+  service.cancel_pending_splits_for_parent(pg->info.pgid);
+
+  store->queue_transaction(
+    pg->osr.get(), rmt,
+    new ObjectStore::C_DeleteTransactionHolder<
+      SequencerRef>(rmt, pg->osr),
+    new ContainerContext<
+      SequencerRef>(pg->osr));
+
+  DeletingStateRef deleting = service.deleting_pgs.lookup_or_create(
+    pg->info.pgid,
+    make_pair(
+      pg->info.pgid,
+      PGRef(pg))
+    );
+  remove_wq.queue(make_pair(PGRef(pg), deleting));
+
+  service.pg_remove_epoch(pg->info.pgid);
+
+  // remove from map
+  pg_map.erase(pg->info.pgid);
+  pg->put("PGMap"); // since we've taken it out of map
+}
+
+
+// =========================================================
+// RECOVERY
+
+/*
+ * caller holds osd_lock
+ */
+void OSD::check_replay_queue()
+{
+  assert(osd_lock.is_locked());
+
+  utime_t now = ceph_clock_now(cct);
+  list< pair > pgids;
+  replay_queue_lock.Lock();
+  while (!replay_queue.empty() &&
+	 replay_queue.front().second <= now) {
+    pgids.push_back(replay_queue.front());
+    replay_queue.pop_front();
+  }
+  replay_queue_lock.Unlock();
+
+  for (list< pair >::iterator p = pgids.begin(); p != pgids.end(); ++p) {
+    spg_t pgid = p->first;
+    if (pg_map.count(pgid)) {
+      PG *pg = _lookup_lock_pg_with_map_lock_held(pgid);
+      dout(10) << "check_replay_queue " << *pg << dendl;
+      if (pg->is_active() &&
+	  pg->is_replay() &&
+	  pg->is_primary() &&
+	  pg->replay_until == p->second) {
+	pg->replay_queued_ops();
+      }
+      pg->unlock();
+    } else {
+      dout(10) << "check_replay_queue pgid " << pgid << " (not found)" << dendl;
+    }
+  }
+  
+  // wake up _all_ pg waiters; raw pg -> actual pg mapping may have shifted
+  wake_all_pg_waiters();
+}
+
+
+bool OSDService::queue_for_recovery(PG *pg)
+{
+  bool b = recovery_wq.queue(pg);
+  if (b)
+    dout(10) << "queue_for_recovery queued " << *pg << dendl;
+  else
+    dout(10) << "queue_for_recovery already queued " << *pg << dendl;
+  return b;
+}
+
+bool OSD::_recover_now()
+{
+  if (recovery_ops_active >= cct->_conf->osd_recovery_max_active) {
+    dout(15) << "_recover_now active " << recovery_ops_active
+	     << " >= max " << cct->_conf->osd_recovery_max_active << dendl;
+    return false;
+  }
+  if (ceph_clock_now(cct) < defer_recovery_until) {
+    dout(15) << "_recover_now defer until " << defer_recovery_until << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+void OSD::do_recovery(PG *pg, ThreadPool::TPHandle &handle)
+{
+  // see how many we should try to start.  note that this is a bit racy.
+  recovery_wq.lock();
+  int max = MIN(cct->_conf->osd_recovery_max_active - recovery_ops_active,
+      cct->_conf->osd_recovery_max_single_start);
+  if (max > 0) {
+    dout(10) << "do_recovery can start " << max << " (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active
+	     << " rops)" << dendl;
+    recovery_ops_active += max;  // take them now, return them if we don't use them.
+  } else {
+    dout(10) << "do_recovery can start 0 (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active
+	     << " rops)" << dendl;
+  }
+  recovery_wq.unlock();
+
+  if (max <= 0) {
+    dout(10) << "do_recovery raced and failed to start anything; requeuing " << *pg << dendl;
+    recovery_wq.queue(pg);
+    return;
+  } else {
+    pg->lock_suspend_timeout(handle);
+    if (pg->deleting || !(pg->is_active() && pg->is_primary())) {
+      pg->unlock();
+      goto out;
+    }
+    
+    dout(10) << "do_recovery starting " << max << " " << *pg << dendl;
+#ifdef DEBUG_RECOVERY_OIDS
+    dout(20) << "  active was " << recovery_oids[pg->info.pgid] << dendl;
+#endif
+    
+    PG::RecoveryCtx rctx = create_context();
+
+    int started;
+    bool more = pg->start_recovery_ops(max, &rctx, handle, &started);
+    dout(10) << "do_recovery started " << started << "/" << max << " on " << *pg << dendl;
+
+    /*
+     * if we couldn't start any recovery ops and things are still
+     * unfound, see if we can discover more missing object locations.
+     * It may be that our initial locations were bad and we errored
+     * out while trying to pull.
+     */
+    if (!more && pg->have_unfound()) {
+      pg->discover_all_missing(*rctx.query_map);
+      if (rctx.query_map->empty()) {
+	dout(10) << "do_recovery  no luck, giving up on this pg for now" << dendl;
+	recovery_wq.lock();
+	recovery_wq._dequeue(pg);
+	recovery_wq.unlock();
+      }
+    }
+
+    pg->write_if_dirty(*rctx.transaction);
+    OSDMapRef curmap = pg->get_osdmap();
+    pg->unlock();
+    dispatch_context(rctx, pg, curmap);
+  }
+
+ out:
+  recovery_wq.lock();
+  if (max > 0) {
+    assert(recovery_ops_active >= max);
+    recovery_ops_active -= max;
+  }
+  recovery_wq._wake();
+  recovery_wq.unlock();
+}
+
+void OSD::start_recovery_op(PG *pg, const hobject_t& soid)
+{
+  recovery_wq.lock();
+  dout(10) << "start_recovery_op " << *pg << " " << soid
+	   << " (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active << " rops)"
+	   << dendl;
+  assert(recovery_ops_active >= 0);
+  recovery_ops_active++;
+
+#ifdef DEBUG_RECOVERY_OIDS
+  dout(20) << "  active was " << recovery_oids[pg->info.pgid] << dendl;
+  assert(recovery_oids[pg->info.pgid].count(soid) == 0);
+  recovery_oids[pg->info.pgid].insert(soid);
+#endif
+
+  recovery_wq.unlock();
+}
+
+void OSD::finish_recovery_op(PG *pg, const hobject_t& soid, bool dequeue)
+{
+  recovery_wq.lock();
+  dout(10) << "finish_recovery_op " << *pg << " " << soid
+	   << " dequeue=" << dequeue
+	   << " (" << recovery_ops_active << "/" << cct->_conf->osd_recovery_max_active << " rops)"
+	   << dendl;
+
+  // adjust count
+  recovery_ops_active--;
+  assert(recovery_ops_active >= 0);
+
+#ifdef DEBUG_RECOVERY_OIDS
+  dout(20) << "  active oids was " << recovery_oids[pg->info.pgid] << dendl;
+  assert(recovery_oids[pg->info.pgid].count(soid));
+  recovery_oids[pg->info.pgid].erase(soid);
+#endif
+
+  if (dequeue)
+    recovery_wq._dequeue(pg);
+  else {
+    recovery_wq._queue_front(pg);
+  }
+
+  recovery_wq._wake();
+  recovery_wq.unlock();
+}
+
+// =========================================================
+// OPS
+
+void OSDService::reply_op_error(OpRequestRef op, int err)
+{
+  reply_op_error(op, err, eversion_t(), 0);
+}
+
+void OSDService::reply_op_error(OpRequestRef op, int err, eversion_t v,
+                                version_t uv)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == CEPH_MSG_OSD_OP);
+  int flags;
+  flags = m->get_flags() & (CEPH_OSD_FLAG_ACK|CEPH_OSD_FLAG_ONDISK);
+
+  MOSDOpReply *reply = new MOSDOpReply(m, err, osdmap->get_epoch(), flags,
+				       true);
+  reply->set_reply_versions(v, uv);
+  m->get_connection()->get_messenger()->send_message(reply, m->get_connection());
+}
+
+void OSDService::handle_misdirected_op(PG *pg, OpRequestRef op)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == CEPH_MSG_OSD_OP);
+
+  assert(m->get_map_epoch() >= pg->info.history.same_primary_since);
+
+  if (pg->is_ec_pg()) {
+    /**
+       * OSD recomputes op target based on current OSDMap. With an EC pg, we
+       * can get this result:
+       * 1) client at map 512 sends an op to osd 3, pg_t 3.9 based on mapping
+       *    [CRUSH_ITEM_NONE, 2, 3]/3
+       * 2) OSD 3 at map 513 remaps op to osd 3, spg_t 3.9s0 based on mapping
+       *    [3, 2, 3]/3
+       * 3) PG 3.9s0 dequeues the op at epoch 512 and notices that it isn't primary
+       *    -- misdirected op
+       * 4) client resends and this time PG 3.9s0 having caught up to 513 gets
+       *    it and fulfils it
+       *
+       * We can't compute the op target based on the sending map epoch due to
+       * splitting.  The simplest thing is to detect such cases here and drop
+       * them without an error (the client will resend anyway).
+       */
+    OSDMapRef opmap = try_get_map(m->get_map_epoch());
+    if (!opmap) {
+      dout(7) << __func__ << ": " << *pg << " no longer have map for "
+	      << m->get_map_epoch() << ", dropping" << dendl;
+      return;
+    }
+    pg_t _pgid = m->get_pg();
+    spg_t pgid;
+    if ((m->get_flags() & CEPH_OSD_FLAG_PGOP) == 0)
+      _pgid = opmap->raw_pg_to_pg(_pgid);
+    if (opmap->get_primary_shard(_pgid, &pgid) &&
+	pgid.shard != pg->info.pgid.shard) {
+      dout(7) << __func__ << ": " << *pg << " primary changed since "
+	      << m->get_map_epoch() << ", dropping" << dendl;
+      return;
+    }
+  }
+
+  dout(7) << *pg << " misdirected op in " << m->get_map_epoch() << dendl;
+  clog.warn() << m->get_source_inst() << " misdirected " << m->get_reqid()
+	      << " pg " << m->get_pg()
+	      << " to osd." << whoami
+	      << " not " << pg->acting
+	      << " in e" << m->get_map_epoch() << "/" << osdmap->get_epoch() << "\n";
+  reply_op_error(op, -ENXIO);
+}
+
+void OSD::handle_op(OpRequestRef op)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == CEPH_MSG_OSD_OP);
+  if (op_is_discardable(m)) {
+    dout(10) << " discardable " << *m << dendl;
+    return;
+  }
+
+  // we don't need encoded payload anymore
+  m->clear_payload();
+
+  // require same or newer map
+  if (!require_same_or_newer_map(op, m->get_map_epoch()))
+    return;
+
+  // object name too long?
+  if (m->get_oid().name.size() > MAX_CEPH_OBJECT_NAME_LEN) {
+    dout(4) << "handle_op '" << m->get_oid().name << "' is longer than "
+	    << MAX_CEPH_OBJECT_NAME_LEN << " bytes!" << dendl;
+    service.reply_op_error(op, -ENAMETOOLONG);
+    return;
+  }
+
+  // blacklisted?
+  if (osdmap->is_blacklisted(m->get_source_addr())) {
+    dout(4) << "handle_op " << m->get_source_addr() << " is blacklisted" << dendl;
+    service.reply_op_error(op, -EBLACKLISTED);
+    return;
+  }
+  // share our map with sender, if they're old
+  _share_map_incoming(m->get_source(), m->get_connection().get(), m->get_map_epoch(),
+		      static_cast(m->get_connection()->get_priv()));
+
+  if (op->rmw_flags == 0) {
+    int r = init_op_flags(op);
+    if (r) {
+      service.reply_op_error(op, r);
+      return;
+    }
+  }
+
+  if (cct->_conf->osd_debug_drop_op_probability > 0 &&
+      !m->get_source().is_mds()) {
+    if ((double)rand() / (double)RAND_MAX < cct->_conf->osd_debug_drop_op_probability) {
+      dout(0) << "handle_op DEBUG artificially dropping op " << *m << dendl;
+      return;
+    }
+  }
+
+  if (op->may_write()) {
+    // full?
+    if ((service.check_failsafe_full() ||
+	 osdmap->test_flag(CEPH_OSDMAP_FULL) ||
+	 m->get_map_epoch() < superblock.last_map_marked_full) &&
+	!m->get_source().is_mds()) {  // FIXME: we'll exclude mds writes for now.
+      // Drop the request, since the client will retry when the full
+      // flag is unset.
+      return;
+    }
+
+    // invalid?
+    if (m->get_snapid() != CEPH_NOSNAP) {
+      service.reply_op_error(op, -EINVAL);
+      return;
+    }
+
+    // too big?
+    if (cct->_conf->osd_max_write_size &&
+	m->get_data_len() > cct->_conf->osd_max_write_size << 20) {
+      // journal can't hold commit!
+      derr << "handle_op msg data len " << m->get_data_len()
+	   << " > osd_max_write_size " << (cct->_conf->osd_max_write_size << 20)
+	   << " on " << *m << dendl;
+      service.reply_op_error(op, -OSD_WRITETOOBIG);
+      return;
+    }
+  }
+  // calc actual pgid
+  pg_t _pgid = m->get_pg();
+  int64_t pool = _pgid.pool();
+  if ((m->get_flags() & CEPH_OSD_FLAG_PGOP) == 0 &&
+      osdmap->have_pg_pool(pool))
+    _pgid = osdmap->raw_pg_to_pg(_pgid);
+
+  spg_t pgid;
+  if (!osdmap->get_primary_shard(_pgid, &pgid)) {
+    // missing pool or acting set empty -- drop
+    return;
+  }
+
+  // get and lock *pg.
+  PG *pg = _have_pg(pgid) ? _lookup_pg(pgid) : NULL;
+  if (!pg) {
+    dout(7) << "hit non-existent pg " << pgid << dendl;
+
+    if (osdmap->get_pg_acting_role(pgid.pgid, whoami) >= 0) {
+      dout(7) << "we are valid target for op, waiting" << dendl;
+      waiting_for_pg[pgid].push_back(op);
+      op->mark_delayed("waiting for pg to exist locally");
+      return;
+    }
+
+    // okay, we aren't valid now; check send epoch
+    if (m->get_map_epoch() < superblock.oldest_map) {
+      dout(7) << "don't have sender's osdmap; assuming it was valid and that client will resend" << dendl;
+      return;
+    }
+    OSDMapRef send_map = get_map(m->get_map_epoch());
+
+    if (send_map->get_pg_acting_role(pgid.pgid, whoami) >= 0) {
+      dout(7) << "dropping request; client will resend when they get new map" << dendl;
+    } else if (!send_map->have_pg_pool(pgid.pool())) {
+      dout(7) << "dropping request; pool did not exist" << dendl;
+      clog.warn() << m->get_source_inst() << " invalid " << m->get_reqid()
+		  << " pg " << m->get_pg()
+		  << " to osd." << whoami
+		  << " in e" << osdmap->get_epoch()
+		  << ", client e" << m->get_map_epoch()
+		  << " when pool " << m->get_pg().pool() << " did not exist"
+		  << "\n";
+    } else {
+      dout(7) << "we are invalid target" << dendl;
+      clog.warn() << m->get_source_inst() << " misdirected " << m->get_reqid()
+		  << " pg " << m->get_pg()
+		  << " to osd." << whoami
+		  << " in e" << osdmap->get_epoch()
+		  << ", client e" << m->get_map_epoch()
+		  << " pg " << pgid
+		  << " features " << m->get_connection()->get_features()
+		  << "\n";
+      service.reply_op_error(op, -ENXIO);
+    }
+    return;
+  }
+
+  enqueue_op(pg, op);
+}
+
+template
+void OSD::handle_replica_op(OpRequestRef op)
+{
+  T *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSGTYPE);
+
+  dout(10) << __func__ << " " << *m << " epoch " << m->map_epoch << dendl;
+  if (m->map_epoch < up_epoch) {
+    dout(3) << "replica op from before up" << dendl;
+    return;
+  }
+
+  if (!require_up_osd_peer(op, osdmap, m->map_epoch))
+    return;
+
+  // must be a rep op.
+  assert(m->get_source().is_osd());
+  
+  // require same or newer map
+  if (!require_same_or_newer_map(op, m->map_epoch))
+    return;
+
+  // share our map with sender, if they're old
+  _share_map_incoming(m->get_source(), m->get_connection().get(), m->map_epoch,
+		      static_cast(m->get_connection()->get_priv()));
+
+  // make sure we have the pg
+  const spg_t pgid = m->pgid;
+  if (service.splitting(pgid)) {
+    waiting_for_pg[pgid].push_back(op);
+    return;
+  }
+
+  PG *pg = _have_pg(pgid) ? _lookup_pg(pgid) : NULL;
+  if (!pg) {
+    return;
+  }
+  enqueue_op(pg, op);
+}
+
+bool OSD::op_is_discardable(MOSDOp *op)
+{
+  // drop client request if they are not connected and can't get the
+  // reply anyway.  unless this is a replayed op, in which case we
+  // want to do what we can to apply it.
+  if (!op->get_connection()->is_connected() &&
+      op->get_version().version == 0) {
+    return true;
+  }
+  return false;
+}
+
+/*
+ * enqueue called with osd_lock held
+ */
+void OSD::enqueue_op(PG *pg, OpRequestRef op)
+{
+  utime_t latency = ceph_clock_now(cct) - op->get_req()->get_recv_stamp();
+  dout(15) << "enqueue_op " << op << " prio " << op->get_req()->get_priority()
+	   << " cost " << op->get_req()->get_cost()
+	   << " latency " << latency
+	   << " " << *(op->get_req()) << dendl;
+  pg->queue_op(op);
+}
+
+void OSD::OpWQ::_enqueue(pair item)
+{
+  unsigned priority = item.second->get_req()->get_priority();
+  unsigned cost = item.second->get_req()->get_cost();
+  if (priority >= CEPH_MSG_PRIO_LOW)
+    pqueue.enqueue_strict(
+      item.second->get_req()->get_source_inst(),
+      priority, item);
+  else
+    pqueue.enqueue(item.second->get_req()->get_source_inst(),
+      priority, cost, item);
+  osd->logger->set(l_osd_opq, pqueue.length());
+}
+
+void OSD::OpWQ::_enqueue_front(pair item)
+{
+  Mutex::Locker l(qlock);
+  if (pg_for_processing.count(&*(item.first))) {
+    pg_for_processing[&*(item.first)].push_front(item.second);
+    item.second = pg_for_processing[&*(item.first)].back();
+    pg_for_processing[&*(item.first)].pop_back();
+  }
+  unsigned priority = item.second->get_req()->get_priority();
+  unsigned cost = item.second->get_req()->get_cost();
+  if (priority >= CEPH_MSG_PRIO_LOW)
+    pqueue.enqueue_strict_front(
+      item.second->get_req()->get_source_inst(),
+      priority, item);
+  else
+    pqueue.enqueue_front(item.second->get_req()->get_source_inst(),
+      priority, cost, item);
+  osd->logger->set(l_osd_opq, pqueue.length());
+}
+
+PGRef OSD::OpWQ::_dequeue()
+{
+  assert(!pqueue.empty());
+  PGRef pg;
+  {
+    Mutex::Locker l(qlock);
+    pair ret = pqueue.dequeue();
+    pg = ret.first;
+    pg_for_processing[&*pg].push_back(ret.second);
+  }
+  osd->logger->set(l_osd_opq, pqueue.length());
+  return pg;
+}
+
+void OSD::OpWQ::_process(PGRef pg, ThreadPool::TPHandle &handle)
+{
+  pg->lock_suspend_timeout(handle);
+  OpRequestRef op;
+  {
+    Mutex::Locker l(qlock);
+    if (!pg_for_processing.count(&*pg)) {
+      pg->unlock();
+      return;
+    }
+    assert(pg_for_processing[&*pg].size());
+    op = pg_for_processing[&*pg].front();
+    pg_for_processing[&*pg].pop_front();
+    if (!(pg_for_processing[&*pg].size()))
+      pg_for_processing.erase(&*pg);
+  }
+
+  lgeneric_subdout(osd->cct, osd, 30) << "dequeue status: ";
+  Formatter *f = new_formatter("json");
+  f->open_object_section("q");
+  dump(f);
+  f->close_section();
+  f->flush(*_dout);
+  delete f;
+  *_dout << dendl;
+
+  osd->dequeue_op(pg, op, handle);
+  pg->unlock();
+}
+
+
+void OSDService::dequeue_pg(PG *pg, list *dequeued)
+{
+  osd->op_wq.dequeue(pg, dequeued);
+}
+
+/*
+ * NOTE: dequeue called in worker thread, with pg lock
+ */
+void OSD::dequeue_op(
+  PGRef pg, OpRequestRef op,
+  ThreadPool::TPHandle &handle)
+{
+  utime_t now = ceph_clock_now(cct);
+  op->set_dequeued_time(now);
+  utime_t latency = now - op->get_req()->get_recv_stamp();
+  dout(10) << "dequeue_op " << op << " prio " << op->get_req()->get_priority()
+	   << " cost " << op->get_req()->get_cost()
+	   << " latency " << latency
+	   << " " << *(op->get_req())
+	   << " pg " << *pg << dendl;
+  if (pg->deleting)
+    return;
+
+  op->mark_reached_pg();
+
+  pg->do_request(op, handle);
+
+  // finish
+  dout(10) << "dequeue_op " << op << " finish" << dendl;
+}
+
+
+void OSDService::queue_for_peering(PG *pg)
+{
+  peering_wq.queue(pg);
+}
+
+struct C_CompleteSplits : public Context {
+  OSD *osd;
+  set > pgs;
+  C_CompleteSplits(OSD *osd, const set > &in)
+    : osd(osd), pgs(in) {}
+  void finish(int r) {
+    Mutex::Locker l(osd->osd_lock);
+    if (osd->is_stopping())
+      return;
+    PG::RecoveryCtx rctx = osd->create_context();
+    set to_complete;
+    for (set >::iterator i = pgs.begin();
+	 i != pgs.end();
+	 ++i) {
+      (*i)->lock();
+      osd->add_newly_split_pg(&**i, &rctx);
+      osd->dispatch_context_transaction(rctx, &**i);
+      if (!((*i)->deleting))
+	to_complete.insert((*i)->info.pgid);
+      (*i)->unlock();
+    }
+    osd->service.complete_split(to_complete);
+    osd->dispatch_context(rctx, 0, osd->service.get_osdmap());
+  }
+};
+
+void OSD::process_peering_events(
+  const list &pgs,
+  ThreadPool::TPHandle &handle
+  )
+{
+  bool need_up_thru = false;
+  epoch_t same_interval_since = 0;
+  OSDMapRef curmap = service.get_osdmap();
+  PG::RecoveryCtx rctx = create_context();
+  for (list::const_iterator i = pgs.begin();
+       i != pgs.end();
+       ++i) {
+    set > split_pgs;
+    PG *pg = *i;
+    pg->lock_suspend_timeout(handle);
+    curmap = service.get_osdmap();
+    if (pg->deleting) {
+      pg->unlock();
+      continue;
+    }
+    if (!advance_pg(curmap->get_epoch(), pg, handle, &rctx, &split_pgs)) {
+      pg->queue_null(curmap->get_epoch(), curmap->get_epoch());
+    } else if (!pg->peering_queue.empty()) {
+      PG::CephPeeringEvtRef evt = pg->peering_queue.front();
+      pg->peering_queue.pop_front();
+      pg->handle_peering_event(evt, &rctx);
+    }
+    need_up_thru = pg->need_up_thru || need_up_thru;
+    same_interval_since = MAX(pg->info.history.same_interval_since,
+			      same_interval_since);
+    pg->write_if_dirty(*rctx.transaction);
+    if (!split_pgs.empty()) {
+      rctx.on_applied->add(new C_CompleteSplits(this, split_pgs));
+      split_pgs.clear();
+    }
+    if (compat_must_dispatch_immediately(pg)) {
+      dispatch_context(rctx, pg, curmap, &handle);
+      rctx = create_context();
+    } else {
+      dispatch_context_transaction(rctx, pg, &handle);
+    }
+    pg->unlock();
+    handle.reset_tp_timeout();
+  }
+  if (need_up_thru)
+    queue_want_up_thru(same_interval_since);
+  dispatch_context(rctx, 0, curmap, &handle);
+
+  service.send_pg_temp();
+}
+
+// --------------------------------
+
+const char** OSD::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "osd_max_backfills",
+    "osd_op_complaint_time", "osd_op_log_threshold",
+    "osd_op_history_size", "osd_op_history_duration",
+    "osd_map_cache_size",
+    "osd_map_max_advance",
+    "osd_pg_epoch_persisted_max_stale",
+    "osd_disk_thread_ioprio_class",
+    "osd_disk_thread_ioprio_priority",
+    NULL
+  };
+  return KEYS;
+}
+
+void OSD::handle_conf_change(const struct md_config_t *conf,
+			     const std::set  &changed)
+{
+  if (changed.count("osd_max_backfills")) {
+    service.local_reserver.set_max(cct->_conf->osd_max_backfills);
+    service.remote_reserver.set_max(cct->_conf->osd_max_backfills);
+  }
+  if (changed.count("osd_op_complaint_time") ||
+      changed.count("osd_op_log_threshold")) {
+    op_tracker.set_complaint_and_threshold(cct->_conf->osd_op_complaint_time,
+                                           cct->_conf->osd_op_log_threshold);
+  }
+  if (changed.count("osd_op_history_size") ||
+      changed.count("osd_op_history_duration")) {
+    op_tracker.set_history_size_and_duration(cct->_conf->osd_op_history_size,
+                                             cct->_conf->osd_op_history_duration);
+  }
+  if (changed.count("osd_disk_thread_ioprio_class") ||
+      changed.count("osd_disk_thread_ioprio_priority")) {
+    set_disk_tp_priority();
+  }
+
+  check_config();
+}
+
+void OSD::check_config()
+{
+  // some sanity checks
+  if (g_conf->osd_map_cache_size <= g_conf->osd_map_max_advance + 2) {
+    clog.warn() << "osd_map_cache_size (" << g_conf->osd_map_cache_size << ")"
+		<< " is not > osd_map_max_advance ("
+		<< g_conf->osd_map_max_advance << ")";
+  }
+  if (g_conf->osd_map_cache_size <= (int)g_conf->osd_pg_epoch_persisted_max_stale + 2) {
+    clog.warn() << "osd_map_cache_size (" << g_conf->osd_map_cache_size << ")"
+		<< " is not > osd_pg_epoch_persisted_max_stale ("
+		<< g_conf->osd_pg_epoch_persisted_max_stale << ")";
+  }
+}
+
+void OSD::set_disk_tp_priority()
+{
+  dout(10) << __func__
+	   << " class " << cct->_conf->osd_disk_thread_ioprio_class
+	   << " priority " << cct->_conf->osd_disk_thread_ioprio_priority
+	   << dendl;
+  int cls =
+    ceph_ioprio_string_to_class(cct->_conf->osd_disk_thread_ioprio_class);
+  disk_tp.set_ioprio(cls, cct->_conf->osd_disk_thread_ioprio_priority);
+}
+
+// --------------------------------
+
+int OSD::init_op_flags(OpRequestRef op)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  vector::iterator iter;
+
+  // client flags have no bearing on whether an op is a read, write, etc.
+  op->rmw_flags = 0;
+
+  // set bits based on op codes, called methods.
+  for (iter = m->ops.begin(); iter != m->ops.end(); ++iter) {
+    if (ceph_osd_op_mode_modify(iter->op.op))
+      op->set_write();
+    if (ceph_osd_op_mode_read(iter->op.op))
+      op->set_read();
+
+    // set READ flag if there are src_oids
+    if (iter->soid.oid.name.length())
+      op->set_read();
+
+    // set PGOP flag if there are PG ops
+    if (ceph_osd_op_type_pg(iter->op.op))
+      op->set_pg_op();
+
+    if (ceph_osd_op_mode_cache(iter->op.op))
+      op->set_cache();
+
+    switch (iter->op.op) {
+    case CEPH_OSD_OP_CALL:
+      {
+	bufferlist::iterator bp = iter->indata.begin();
+	int is_write, is_read;
+	string cname, mname;
+	bp.copy(iter->op.cls.class_len, cname);
+	bp.copy(iter->op.cls.method_len, mname);
+
+	ClassHandler::ClassData *cls;
+	int r = class_handler->open_class(cname, &cls);
+	if (r) {
+	  derr << "class " << cname << " open got " << cpp_strerror(r) << dendl;
+	  if (r == -ENOENT)
+	    r = -EOPNOTSUPP;
+	  else
+	    r = -EIO;
+	  return r;
+	}
+	int flags = cls->get_method_flags(mname.c_str());
+	if (flags < 0) {
+	  if (flags == -ENOENT)
+	    r = -EOPNOTSUPP;
+	  else
+	    r = flags;
+	  return r;
+	}
+	is_read = flags & CLS_METHOD_RD;
+	is_write = flags & CLS_METHOD_WR;
+
+	dout(10) << "class " << cname << " method " << mname
+		<< " flags=" << (is_read ? "r" : "") << (is_write ? "w" : "") << dendl;
+	if (is_read)
+	  op->set_class_read();
+	if (is_write)
+	  op->set_class_write();
+	break;
+      }
+    default:
+      break;
+    }
+  }
+
+  if (op->rmw_flags == 0)
+    return -EINVAL;
+
+  return 0;
+}
+
+bool OSD::RecoveryWQ::_enqueue(PG *pg) {
+  if (!pg->recovery_item.is_on_list()) {
+    pg->get("RecoveryWQ");
+    osd->recovery_queue.push_back(&pg->recovery_item);
+
+    if (osd->cct->_conf->osd_recovery_delay_start > 0) {
+      osd->defer_recovery_until = ceph_clock_now(osd->cct);
+      osd->defer_recovery_until += osd->cct->_conf->osd_recovery_delay_start;
+    }
+    return true;
+  }
+  return false;
+}
+
+void OSD::PeeringWQ::_dequeue(list *out) {
+  set got;
+  for (list::iterator i = peering_queue.begin();
+      i != peering_queue.end() &&
+      out->size() < osd->cct->_conf->osd_peering_wq_batch_size;
+      ) {
+        if (in_use.count(*i)) {
+          ++i;
+        } else {
+          out->push_back(*i);
+          got.insert(*i);
+          peering_queue.erase(i++);
+        }
+  }
+  in_use.insert(got.begin(), got.end());
+}
diff --git a/ceph/src/osd/OSD.h b/ceph/src/osd/OSD.h
new file mode 100644
index 00000000..e2a3c8e3
--- /dev/null
+++ b/ceph/src/osd/OSD.h
@@ -0,0 +1,2019 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_OSD_H
+#define CEPH_OSD_H
+
+#include "boost/tuple/tuple.hpp"
+
+#include "PG.h"
+
+#include "msg/Dispatcher.h"
+
+#include "common/Mutex.h"
+#include "common/RWLock.h"
+#include "common/Timer.h"
+#include "common/WorkQueue.h"
+#include "common/LogClient.h"
+#include "common/AsyncReserver.h"
+#include "common/ceph_context.h"
+
+#include "os/ObjectStore.h"
+#include "OSDCap.h"
+
+#include "osd/ClassHandler.h"
+
+#include "include/CompatSet.h"
+
+#include "auth/KeyRing.h"
+#include "messages/MOSDRepScrub.h"
+#include "OpRequest.h"
+
+#include 
+#include 
+#include "include/memory.h"
+using namespace std;
+
+#include "include/unordered_map.h"
+#include "include/unordered_set.h"
+
+#include "Watch.h"
+#include "common/shared_cache.hpp"
+#include "common/simple_cache.hpp"
+#include "common/sharedptr_registry.hpp"
+#include "common/PrioritizedQueue.h"
+
+#define CEPH_OSD_PROTOCOL    10 /* cluster internal */
+
+
+enum {
+  l_osd_first = 10000,
+  l_osd_opq,
+  l_osd_op_wip,
+  l_osd_op,
+  l_osd_op_inb,
+  l_osd_op_outb,
+  l_osd_op_lat,
+  l_osd_op_process_lat,
+  l_osd_op_r,
+  l_osd_op_r_outb,
+  l_osd_op_r_lat,
+  l_osd_op_r_process_lat,
+  l_osd_op_w,
+  l_osd_op_w_inb,
+  l_osd_op_w_rlat,
+  l_osd_op_w_lat,
+  l_osd_op_w_process_lat,
+  l_osd_op_rw,
+  l_osd_op_rw_inb,
+  l_osd_op_rw_outb,
+  l_osd_op_rw_rlat,
+  l_osd_op_rw_lat,
+  l_osd_op_rw_process_lat,
+
+  l_osd_sop,
+  l_osd_sop_inb,
+  l_osd_sop_lat,
+  l_osd_sop_w,
+  l_osd_sop_w_inb,
+  l_osd_sop_w_lat,
+  l_osd_sop_pull,
+  l_osd_sop_pull_lat,
+  l_osd_sop_push,
+  l_osd_sop_push_inb,
+  l_osd_sop_push_lat,
+
+  l_osd_pull,
+  l_osd_push,
+  l_osd_push_outb,
+
+  l_osd_push_in,
+  l_osd_push_inb,
+
+  l_osd_rop,
+
+  l_osd_loadavg,
+  l_osd_buf,
+
+  l_osd_pg,
+  l_osd_pg_primary,
+  l_osd_pg_replica,
+  l_osd_pg_stray,
+  l_osd_hb_to,
+  l_osd_hb_from,
+  l_osd_map,
+  l_osd_mape,
+  l_osd_mape_dup,
+
+  l_osd_waiting_for_map,
+
+  l_osd_stat_bytes,
+  l_osd_stat_bytes_used,
+  l_osd_stat_bytes_avail,
+
+  l_osd_copyfrom,
+
+  l_osd_tier_promote,
+  l_osd_tier_flush,
+  l_osd_tier_flush_fail,
+  l_osd_tier_try_flush,
+  l_osd_tier_try_flush_fail,
+  l_osd_tier_evict,
+  l_osd_tier_whiteout,
+  l_osd_tier_dirty,
+  l_osd_tier_clean,
+  l_osd_tier_delay,
+
+  l_osd_agent_wake,
+  l_osd_agent_skip,
+  l_osd_agent_flush,
+  l_osd_agent_evict,
+
+  l_osd_last,
+};
+
+// RecoveryState perf counters
+enum {
+  rs_first = 20000,
+  rs_initial_latency,
+  rs_started_latency,
+  rs_reset_latency,
+  rs_start_latency,
+  rs_primary_latency,
+  rs_peering_latency,
+  rs_backfilling_latency,
+  rs_waitremotebackfillreserved_latency,
+  rs_waitlocalbackfillreserved_latency,
+  rs_notbackfilling_latency,
+  rs_repnotrecovering_latency,
+  rs_repwaitrecoveryreserved_latency,
+  rs_repwaitbackfillreserved_latency,
+  rs_RepRecovering_latency,
+  rs_activating_latency,
+  rs_waitlocalrecoveryreserved_latency,
+  rs_waitremoterecoveryreserved_latency,
+  rs_recovering_latency,
+  rs_recovered_latency,
+  rs_clean_latency,
+  rs_active_latency,
+  rs_replicaactive_latency,
+  rs_stray_latency,
+  rs_getinfo_latency,
+  rs_getlog_latency,
+  rs_waitactingchange_latency,
+  rs_incomplete_latency,
+  rs_getmissing_latency,
+  rs_waitupthru_latency,
+  rs_last,
+};
+
+class Messenger;
+class Message;
+class MonClient;
+class PerfCounters;
+class ObjectStore;
+class OSDMap;
+class MLog;
+class MClass;
+class MOSDPGMissing;
+class Objecter;
+
+class Watch;
+class Notification;
+class ReplicatedPG;
+
+class AuthAuthorizeHandlerRegistry;
+
+class OpsFlightSocketHook;
+class HistoricOpsSocketHook;
+class TestOpsSocketHook;
+struct C_CompleteSplits;
+
+typedef ceph::shared_ptr SequencerRef;
+
+class DeletingState {
+  Mutex lock;
+  Cond cond;
+  enum {
+    QUEUED,
+    CLEARING_DIR,
+    CLEARING_WAITING,
+    DELETING_DIR,
+    DELETED_DIR,
+    CANCELED,
+  } status;
+  bool stop_deleting;
+public:
+  const spg_t pgid;
+  const PGRef old_pg_state;
+  DeletingState(const pair &in) :
+    lock("DeletingState::lock"), status(QUEUED), stop_deleting(false),
+    pgid(in.first), old_pg_state(in.second) {}
+
+  /// transition status to clearing
+  bool start_clearing() {
+    Mutex::Locker l(lock);
+    assert(
+      status == QUEUED ||
+      status == DELETED_DIR);
+    if (stop_deleting) {
+      status = CANCELED;
+      cond.Signal();
+      return false;
+    }
+    status = CLEARING_DIR;
+    return true;
+  } ///< @return false if we should cancel deletion
+
+  /// transition status to CLEARING_WAITING
+  bool pause_clearing() {
+    Mutex::Locker l(lock);
+    assert(status == CLEARING_DIR);
+    if (stop_deleting) {
+      status = CANCELED;
+      cond.Signal();
+      return false;
+    }
+    status = CLEARING_WAITING;
+    return true;
+  } ///< @return false if we should cancel deletion
+
+  /// transition status to CLEARING_DIR
+  bool resume_clearing() {
+    Mutex::Locker l(lock);
+    assert(status == CLEARING_WAITING);
+    if (stop_deleting) {
+      status = CANCELED;
+      cond.Signal();
+      return false;
+    }
+    status = CLEARING_DIR;
+    return true;
+  } ///< @return false if we should cancel deletion
+
+  /// transition status to deleting
+  bool start_deleting() {
+    Mutex::Locker l(lock);
+    assert(status == CLEARING_DIR);
+    if (stop_deleting) {
+      status = CANCELED;
+      cond.Signal();
+      return false;
+    }
+    status = DELETING_DIR;
+    return true;
+  } ///< @return false if we should cancel deletion
+
+  /// signal collection removal queued
+  void finish_deleting() {
+    Mutex::Locker l(lock);
+    assert(status == DELETING_DIR);
+    status = DELETED_DIR;
+    cond.Signal();
+  }
+
+  /// try to halt the deletion
+  bool try_stop_deletion() {
+    Mutex::Locker l(lock);
+    stop_deleting = true;
+    /**
+     * If we are in DELETING_DIR or CLEARING_DIR, there are in progress
+     * operations we have to wait for before continuing on.  States
+     * CLEARING_WAITING and QUEUED indicate that the remover will check
+     * stop_deleting before queueing any further operations.  CANCELED
+     * indicates that the remover has already halted.  DELETED_DIR
+     * indicates that the deletion has been fully queueud.
+     */
+    while (status == DELETING_DIR || status == CLEARING_DIR)
+      cond.Wait(lock);
+    return status != DELETED_DIR;
+  } ///< @return true if we don't need to recreate the collection
+};
+typedef ceph::shared_ptr DeletingStateRef;
+
+class OSD;
+class OSDService {
+public:
+  OSD *osd;
+  CephContext *cct;
+  SharedPtrRegistry osr_registry;
+  SharedPtrRegistry deleting_pgs;
+  const int whoami;
+  ObjectStore *&store;
+  LogClient &clog;
+  PGRecoveryStats &pg_recovery_stats;
+  hobject_t infos_oid;
+private:
+  Messenger *&cluster_messenger;
+  Messenger *&client_messenger;
+public:
+  PerfCounters *&logger;
+  PerfCounters *&recoverystate_perf;
+  MonClient   *&monc;
+  ThreadPool::WorkQueueVal, PGRef> &op_wq;
+  ThreadPool::BatchWorkQueue &peering_wq;
+  ThreadPool::WorkQueue &recovery_wq;
+  ThreadPool::WorkQueue &snap_trim_wq;
+  ThreadPool::WorkQueue &scrub_wq;
+  ThreadPool::WorkQueue &scrub_finalize_wq;
+  ThreadPool::WorkQueue &rep_scrub_wq;
+  GenContextWQ push_wq;
+  GenContextWQ gen_wq;
+  ClassHandler  *&class_handler;
+
+  void dequeue_pg(PG *pg, list *dequeued);
+
+  // -- map epoch lower bound --
+  Mutex pg_epoch_lock;
+  multiset pg_epochs;
+  map pg_epoch;
+
+  void pg_add_epoch(spg_t pgid, epoch_t epoch) {
+    Mutex::Locker l(pg_epoch_lock);
+    map::iterator t = pg_epoch.find(pgid);
+    assert(t == pg_epoch.end());
+    pg_epoch[pgid] = epoch;
+    pg_epochs.insert(epoch);
+  }
+  void pg_update_epoch(spg_t pgid, epoch_t epoch) {
+    Mutex::Locker l(pg_epoch_lock);
+    map::iterator t = pg_epoch.find(pgid);
+    assert(t != pg_epoch.end());
+    pg_epochs.erase(pg_epochs.find(t->second));
+    t->second = epoch;
+    pg_epochs.insert(epoch);
+  }
+  void pg_remove_epoch(spg_t pgid) {
+    Mutex::Locker l(pg_epoch_lock);
+    map::iterator t = pg_epoch.find(pgid);
+    if (t != pg_epoch.end()) {
+      pg_epochs.erase(pg_epochs.find(t->second));
+      pg_epoch.erase(t);
+    }
+  }
+  epoch_t get_min_pg_epoch() {
+    Mutex::Locker l(pg_epoch_lock);
+    if (pg_epochs.empty())
+      return 0;
+    else
+      return *pg_epochs.begin();
+  }
+
+  // -- superblock --
+  Mutex publish_lock, pre_publish_lock; // pre-publish orders before publish
+  OSDSuperblock superblock;
+  OSDSuperblock get_superblock() {
+    Mutex::Locker l(publish_lock);
+    return superblock;
+  }
+  void publish_superblock(const OSDSuperblock &block) {
+    Mutex::Locker l(publish_lock);
+    superblock = block;
+  }
+
+  int get_nodeid() const { return whoami; }
+
+  OSDMapRef osdmap;
+  OSDMapRef get_osdmap() {
+    Mutex::Locker l(publish_lock);
+    return osdmap;
+  }
+  void publish_map(OSDMapRef map) {
+    Mutex::Locker l(publish_lock);
+    osdmap = map;
+  }
+
+  /*
+   * osdmap - current published amp
+   * next_osdmap - pre_published map that is about to be published.
+   *
+   * We use the next_osdmap to send messages and initiate connections,
+   * but only if the target is the same instance as the one in the map
+   * epoch the current user is working from (i.e., the result is
+   * equivalent to what is in next_osdmap).
+   *
+   * This allows the helpers to start ignoring osds that are about to
+   * go down, and let OSD::handle_osd_map()/note_down_osd() mark them
+   * down, without worrying about reopening connections from threads
+   * working from old maps.
+   */
+  OSDMapRef next_osdmap;
+  void pre_publish_map(OSDMapRef map) {
+    Mutex::Locker l(pre_publish_lock);
+    next_osdmap = map;
+  }
+
+  void activate_map();
+
+  ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch);
+  pair get_con_osd_hb(int peer, epoch_t from_epoch);  // (back, front)
+  void send_message_osd_cluster(int peer, Message *m, epoch_t from_epoch);
+  void send_message_osd_cluster(Message *m, Connection *con) {
+    cluster_messenger->send_message(m, con);
+  }
+  void send_message_osd_cluster(Message *m, const ConnectionRef& con) {
+    cluster_messenger->send_message(m, con.get());
+  }
+  void send_message_osd_client(Message *m, Connection *con) {
+    client_messenger->send_message(m, con);
+  }
+  void send_message_osd_client(Message *m, const ConnectionRef& con) {
+    client_messenger->send_message(m, con.get());
+  }
+  entity_name_t get_cluster_msgr_name() {
+    return cluster_messenger->get_myname();
+  }
+
+  // -- scrub scheduling --
+  Mutex sched_scrub_lock;
+  int scrubs_pending;
+  int scrubs_active;
+  set< pair > last_scrub_pg;
+
+  void reg_last_pg_scrub(spg_t pgid, utime_t t) {
+    Mutex::Locker l(sched_scrub_lock);
+    last_scrub_pg.insert(pair(t, pgid));
+  }
+  void unreg_last_pg_scrub(spg_t pgid, utime_t t) {
+    Mutex::Locker l(sched_scrub_lock);
+    pair p(t, pgid);
+    set >::iterator it = last_scrub_pg.find(p);
+    assert(it != last_scrub_pg.end());
+    last_scrub_pg.erase(it);
+  }
+  bool first_scrub_stamp(pair *out) {
+    Mutex::Locker l(sched_scrub_lock);
+    if (last_scrub_pg.empty())
+      return false;
+    set< pair >::iterator iter = last_scrub_pg.begin();
+    *out = *iter;
+    return true;
+  }
+  bool next_scrub_stamp(pair next,
+			pair *out) {
+    Mutex::Locker l(sched_scrub_lock);
+    if (last_scrub_pg.empty())
+      return false;
+    set< pair >::iterator iter = last_scrub_pg.lower_bound(next);
+    if (iter == last_scrub_pg.end())
+      return false;
+    ++iter;
+    if (iter == last_scrub_pg.end())
+      return false;
+    *out = *iter;
+    return true;
+  }
+
+  bool inc_scrubs_pending();
+  void inc_scrubs_active(bool reserved);
+  void dec_scrubs_pending();
+  void dec_scrubs_active();
+
+  void reply_op_error(OpRequestRef op, int err);
+  void reply_op_error(OpRequestRef op, int err, eversion_t v, version_t uv);
+  void handle_misdirected_op(PG *pg, OpRequestRef op);
+
+
+  // -- agent shared state --
+  Mutex agent_lock;
+  Cond agent_cond;
+  map > agent_queue;
+  set::iterator agent_queue_pos;
+  bool agent_valid_iterator;
+  int agent_ops;
+  set agent_oids;
+  bool agent_active;
+  struct AgentThread : public Thread {
+    OSDService *osd;
+    AgentThread(OSDService *o) : osd(o) {}
+    void *entry() {
+      osd->agent_entry();
+      return NULL;
+    }
+  } agent_thread;
+  bool agent_stop_flag;
+  Mutex agent_timer_lock;
+  SafeTimer agent_timer;
+
+  void agent_entry();
+  void agent_stop();
+
+  void _enqueue(PG *pg, uint64_t priority) {
+    if (!agent_queue.empty() &&
+	agent_queue.rbegin()->first < priority)
+      agent_valid_iterator = false;  // inserting higher-priority queue
+    set& nq = agent_queue[priority];
+    if (nq.empty())
+      agent_cond.Signal();
+    nq.insert(pg);
+  }
+
+  void _dequeue(PG *pg, uint64_t old_priority) {
+    set& oq = agent_queue[old_priority];
+    set::iterator p = oq.find(pg);
+    assert(p != oq.end());
+    if (p == agent_queue_pos)
+      ++agent_queue_pos;
+    oq.erase(p);
+    if (oq.empty()) {
+      if (agent_queue.rbegin()->first == old_priority)
+	agent_valid_iterator = false;
+      agent_queue.erase(old_priority);
+    }
+  }
+
+  /// enable agent for a pg
+  void agent_enable_pg(PG *pg, uint64_t priority) {
+    Mutex::Locker l(agent_lock);
+    _enqueue(pg, priority);
+  }
+
+  /// adjust priority for an enagled pg
+  void agent_adjust_pg(PG *pg, uint64_t old_priority, uint64_t new_priority) {
+    Mutex::Locker l(agent_lock);
+    assert(new_priority != old_priority);
+    _enqueue(pg, new_priority);
+    _dequeue(pg, old_priority);
+  }
+
+  /// disable agent for a pg
+  void agent_disable_pg(PG *pg, uint64_t old_priority) {
+    Mutex::Locker l(agent_lock);
+    _dequeue(pg, old_priority);
+  }
+
+  /// note start of an async (flush) op
+  void agent_start_op(const hobject_t& oid) {
+    Mutex::Locker l(agent_lock);
+    ++agent_ops;
+    assert(agent_oids.count(oid) == 0);
+    agent_oids.insert(oid);
+  }
+
+  /// note finish or cancellation of an async (flush) op
+  void agent_finish_op(const hobject_t& oid) {
+    Mutex::Locker l(agent_lock);
+    assert(agent_ops > 0);
+    --agent_ops;
+    assert(agent_oids.count(oid) == 1);
+    agent_oids.erase(oid);
+    agent_cond.Signal();
+  }
+
+  /// check if we are operating on an object
+  bool agent_is_active_oid(const hobject_t& oid) {
+    Mutex::Locker l(agent_lock);
+    return agent_oids.count(oid);
+  }
+
+  /// get count of active agent ops
+  int agent_get_num_ops() {
+    Mutex::Locker l(agent_lock);
+    return agent_ops;
+  }
+
+
+  // -- Objecter, for teiring reads/writes from/to other OSDs --
+  Mutex objecter_lock;
+  SafeTimer objecter_timer;
+  OSDMap objecter_osdmap;
+  Objecter *objecter;
+  Finisher objecter_finisher;
+  struct ObjecterDispatcher : public Dispatcher {
+    OSDService *osd;
+    bool ms_dispatch(Message *m);
+    bool ms_handle_reset(Connection *con);
+    void ms_handle_remote_reset(Connection *con) {}
+    void ms_handle_connect(Connection *con);
+    bool ms_get_authorizer(int dest_type,
+			   AuthAuthorizer **authorizer,
+			   bool force_new);
+    ObjecterDispatcher(OSDService *o) : Dispatcher(cct), osd(o) {}
+  } objecter_dispatcher;
+  friend struct ObjecterDispatcher;
+
+
+  // -- Watch --
+  Mutex watch_lock;
+  SafeTimer watch_timer;
+  uint64_t next_notif_id;
+  uint64_t get_next_id(epoch_t cur_epoch) {
+    Mutex::Locker l(watch_lock);
+    return (((uint64_t)cur_epoch) << 32) | ((uint64_t)(next_notif_id++));
+  }
+
+  // -- Backfill Request Scheduling --
+  Mutex backfill_request_lock;
+  SafeTimer backfill_request_timer;
+
+  // -- tids --
+  // for ops i issue
+  ceph_tid_t last_tid;
+  Mutex tid_lock;
+  ceph_tid_t get_tid() {
+    ceph_tid_t t;
+    tid_lock.Lock();
+    t = ++last_tid;
+    tid_lock.Unlock();
+    return t;
+  }
+
+  // -- backfill_reservation --
+  enum {
+    BACKFILL_LOW = 0,   // backfill non-degraded PGs
+    BACKFILL_HIGH = 1,	// backfill degraded PGs
+    RECOVERY = AsyncReserver::MAX_PRIORITY  // log based recovery
+  };
+  Finisher reserver_finisher;
+  AsyncReserver local_reserver;
+  AsyncReserver remote_reserver;
+
+  // -- pg_temp --
+  Mutex pg_temp_lock;
+  map > pg_temp_wanted;
+  void queue_want_pg_temp(pg_t pgid, vector& want);
+  void remove_want_pg_temp(pg_t pgid) {
+    Mutex::Locker l(pg_temp_lock);
+    pg_temp_wanted.erase(pgid);
+  }
+  void send_pg_temp();
+
+  void queue_for_peering(PG *pg);
+  bool queue_for_recovery(PG *pg);
+  bool queue_for_snap_trim(PG *pg) {
+    return snap_trim_wq.queue(pg);
+  }
+  bool queue_for_scrub(PG *pg) {
+    return scrub_wq.queue(pg);
+  }
+
+  // osd map cache (past osd maps)
+  Mutex map_cache_lock;
+  SharedLRU map_cache;
+  SimpleLRU map_bl_cache;
+  SimpleLRU map_bl_inc_cache;
+
+  OSDMapRef try_get_map(epoch_t e);
+  OSDMapRef get_map(epoch_t e) {
+    OSDMapRef ret(try_get_map(e));
+    assert(ret);
+    return ret;
+  }
+  OSDMapRef add_map(OSDMap *o) {
+    Mutex::Locker l(map_cache_lock);
+    return _add_map(o);
+  }
+  OSDMapRef _add_map(OSDMap *o);
+
+  void add_map_bl(epoch_t e, bufferlist& bl) {
+    Mutex::Locker l(map_cache_lock);
+    return _add_map_bl(e, bl);
+  }
+  void pin_map_bl(epoch_t e, bufferlist &bl);
+  void _add_map_bl(epoch_t e, bufferlist& bl);
+  bool get_map_bl(epoch_t e, bufferlist& bl) {
+    Mutex::Locker l(map_cache_lock);
+    return _get_map_bl(e, bl);
+  }
+  bool _get_map_bl(epoch_t e, bufferlist& bl);
+
+  void add_map_inc_bl(epoch_t e, bufferlist& bl) {
+    Mutex::Locker l(map_cache_lock);
+    return _add_map_inc_bl(e, bl);
+  }
+  void pin_map_inc_bl(epoch_t e, bufferlist &bl);
+  void _add_map_inc_bl(epoch_t e, bufferlist& bl);
+  bool get_inc_map_bl(epoch_t e, bufferlist& bl);
+
+  void clear_map_bl_cache_pins(epoch_t e);
+
+  void need_heartbeat_peer_update();
+
+  void pg_stat_queue_enqueue(PG *pg);
+  void pg_stat_queue_dequeue(PG *pg);
+
+  void init();
+  void start_shutdown();
+  void shutdown();
+
+  // split
+  Mutex in_progress_split_lock;
+  map pending_splits; // child -> parent
+  map > rev_pending_splits; // parent -> [children]
+  set in_progress_splits;       // child
+
+  void _start_split(spg_t parent, const set &children);
+  void start_split(spg_t parent, const set &children) {
+    Mutex::Locker l(in_progress_split_lock);
+    return _start_split(parent, children);
+  }
+  void mark_split_in_progress(spg_t parent, const set &pgs);
+  void complete_split(const set &pgs);
+  void cancel_pending_splits_for_parent(spg_t parent);
+  void _cancel_pending_splits_for_parent(spg_t parent);
+  bool splitting(spg_t pgid);
+  void expand_pg_num(OSDMapRef old_map,
+		     OSDMapRef new_map);
+  void _maybe_split_pgid(OSDMapRef old_map,
+			 OSDMapRef new_map,
+			 spg_t pgid);
+  void init_splits_between(spg_t pgid, OSDMapRef frommap, OSDMapRef tomap);
+
+  // -- OSD Full Status --
+  Mutex full_status_lock;
+  enum s_names { NONE, NEAR, FULL } cur_state;
+  time_t last_msg;
+  double cur_ratio;
+  float get_full_ratio();
+  float get_nearfull_ratio();
+  void check_nearfull_warning(const osd_stat_t &stat);
+  bool check_failsafe_full();
+  bool too_full_for_backfill(double *ratio, double *max_ratio);
+
+
+  // -- stopping --
+  Mutex is_stopping_lock;
+  Cond is_stopping_cond;
+  enum {
+    NOT_STOPPING,
+    PREPARING_TO_STOP,
+    STOPPING } state;
+  bool is_stopping() {
+    Mutex::Locker l(is_stopping_lock);
+    return state == STOPPING;
+  }
+  bool is_preparing_to_stop() {
+    Mutex::Locker l(is_stopping_lock);
+    return state == PREPARING_TO_STOP;
+  }
+  bool prepare_to_stop();
+  void got_stop_ack();
+
+
+#ifdef PG_DEBUG_REFS
+  Mutex pgid_lock;
+  map pgid_tracker;
+  map live_pgs;
+  void add_pgid(spg_t pgid, PG *pg) {
+    Mutex::Locker l(pgid_lock);
+    if (!pgid_tracker.count(pgid)) {
+      pgid_tracker[pgid] = 0;
+      live_pgs[pgid] = pg;
+    }
+    pgid_tracker[pgid]++;
+  }
+  void remove_pgid(spg_t pgid, PG *pg) {
+    Mutex::Locker l(pgid_lock);
+    assert(pgid_tracker.count(pgid));
+    assert(pgid_tracker[pgid] > 0);
+    pgid_tracker[pgid]--;
+    if (pgid_tracker[pgid] == 0) {
+      pgid_tracker.erase(pgid);
+      live_pgs.erase(pgid);
+    }
+  }
+  void dump_live_pgids() {
+    Mutex::Locker l(pgid_lock);
+    derr << "live pgids:" << dendl;
+    for (map::iterator i = pgid_tracker.begin();
+	 i != pgid_tracker.end();
+	 ++i) {
+      derr << "\t" << *i << dendl;
+      live_pgs[i->first]->dump_live_ids();
+    }
+  }
+#endif
+
+  OSDService(OSD *osd);
+  ~OSDService();
+};
+
+struct C_OSD_SendMessageOnConn: public Context {
+  OSDService *osd;
+  Message *reply;
+  ConnectionRef conn;
+  C_OSD_SendMessageOnConn(
+    OSDService *osd,
+    Message *reply,
+    ConnectionRef conn) : osd(osd), reply(reply), conn(conn) {}
+  void finish(int) {
+    osd->send_message_osd_cluster(reply, conn.get());
+  }
+};
+
+class OSD : public Dispatcher,
+	    public md_config_obs_t {
+  /** OSD **/
+public:
+  // config observer bits
+  virtual const char** get_tracked_conf_keys() const;
+  virtual void handle_conf_change(const struct md_config_t *conf,
+				  const std::set  &changed);
+  void check_config();
+
+protected:
+  Mutex osd_lock;			// global lock
+  SafeTimer tick_timer;    // safe timer (osd_lock)
+
+  AuthAuthorizeHandlerRegistry *authorize_handler_cluster_registry;
+  AuthAuthorizeHandlerRegistry *authorize_handler_service_registry;
+
+  Messenger   *cluster_messenger;
+  Messenger   *client_messenger;
+  Messenger   *objecter_messenger;
+  MonClient   *monc; // check the "monc helpers" list before accessing directly
+  PerfCounters      *logger;
+  PerfCounters      *recoverystate_perf;
+  ObjectStore *store;
+
+  LogClient clog;
+
+  int whoami;
+  std::string dev_path, journal_path;
+
+  class C_Tick : public Context {
+    OSD *osd;
+  public:
+    C_Tick(OSD *o) : osd(o) {}
+    void finish(int r) {
+      osd->tick();
+    }
+  };
+
+  Cond dispatch_cond;
+  int dispatch_running;
+
+  void create_logger();
+  void create_recoverystate_perf();
+  void tick();
+  void _dispatch(Message *m);
+  void dispatch_op(OpRequestRef op);
+
+  void check_osdmap_features(ObjectStore *store);
+
+  // asok
+  friend class OSDSocketHook;
+  class OSDSocketHook *asok_hook;
+  bool asok_command(string command, cmdmap_t& cmdmap, string format, ostream& ss);
+
+public:
+  ClassHandler  *class_handler;
+  int get_nodeid() { return whoami; }
+  
+  static hobject_t get_osdmap_pobject_name(epoch_t epoch) { 
+    char foo[20];
+    snprintf(foo, sizeof(foo), "osdmap.%d", epoch);
+    return hobject_t(sobject_t(object_t(foo), 0)); 
+  }
+  static hobject_t get_inc_osdmap_pobject_name(epoch_t epoch) { 
+    char foo[20];
+    snprintf(foo, sizeof(foo), "inc_osdmap.%d", epoch);
+    return hobject_t(sobject_t(object_t(foo), 0)); 
+  }
+
+  static hobject_t make_snapmapper_oid() {
+    return hobject_t(
+      sobject_t(
+	object_t("snapmapper"),
+	0));
+  }
+
+  static hobject_t make_pg_log_oid(spg_t pg) {
+    stringstream ss;
+    ss << "pglog_" << pg;
+    string s;
+    getline(ss, s);
+    return hobject_t(sobject_t(object_t(s.c_str()), 0));
+  }
+  
+  static hobject_t make_pg_biginfo_oid(spg_t pg) {
+    stringstream ss;
+    ss << "pginfo_" << pg;
+    string s;
+    getline(ss, s);
+    return hobject_t(sobject_t(object_t(s.c_str()), 0));
+  }
+  static hobject_t make_infos_oid() {
+    hobject_t oid(sobject_t("infos", CEPH_NOSNAP));
+    return oid;
+  }
+  static void recursive_remove_collection(ObjectStore *store, coll_t tmp);
+
+  /**
+   * get_osd_initial_compat_set()
+   *
+   * Get the initial feature set for this OSD.  Features
+   * here are automatically upgraded.
+   *
+   * Return value: Initial osd CompatSet
+   */
+  static CompatSet get_osd_initial_compat_set();
+
+  /**
+   * get_osd_compat_set()
+   *
+   * Get all features supported by this OSD
+   *
+   * Return value: CompatSet of all supported features
+   */
+  static CompatSet get_osd_compat_set();
+  
+
+private:
+  // -- superblock --
+  OSDSuperblock superblock;
+
+  void write_superblock();
+  void write_superblock(ObjectStore::Transaction& t);
+  int read_superblock();
+
+  CompatSet osd_compat;
+
+  // -- state --
+public:
+  static const int STATE_INITIALIZING = 1;
+  static const int STATE_BOOTING = 2;
+  static const int STATE_ACTIVE = 3;
+  static const int STATE_STOPPING = 4;
+  static const int STATE_WAITING_FOR_HEALTHY = 5;
+
+  static const char *get_state_name(int s) {
+    switch (s) {
+    case STATE_INITIALIZING: return "initializing";
+    case STATE_BOOTING: return "booting";
+    case STATE_ACTIVE: return "active";
+    case STATE_STOPPING: return "stopping";
+    case STATE_WAITING_FOR_HEALTHY: return "waiting_for_healthy";
+    default: return "???";
+    }
+  }
+
+private:
+  int state;
+  epoch_t boot_epoch;  // _first_ epoch we were marked up (after this process started)
+  epoch_t up_epoch;    // _most_recent_ epoch we were marked up
+  epoch_t bind_epoch;  // epoch we last did a bind to new ip:ports
+
+public:
+  bool is_initializing() { return state == STATE_INITIALIZING; }
+  bool is_booting() { return state == STATE_BOOTING; }
+  bool is_active() { return state == STATE_ACTIVE; }
+  bool is_stopping() { return state == STATE_STOPPING; }
+  bool is_waiting_for_healthy() { return state == STATE_WAITING_FOR_HEALTHY; }
+
+private:
+
+  ThreadPool op_tp;
+  ThreadPool recovery_tp;
+  ThreadPool disk_tp;
+  ThreadPool command_tp;
+
+  bool paused_recovery;
+
+  void set_disk_tp_priority();
+
+  // -- sessions --
+public:
+  struct Session : public RefCountedObject {
+    EntityName entity_name;
+    OSDCap caps;
+    int64_t auid;
+    epoch_t last_sent_epoch;
+    ConnectionRef con;
+    WatchConState wstate;
+
+    Session() : auid(-1), last_sent_epoch(0), con(0) {}
+  };
+
+private:
+  /**
+   *  @defgroup monc helpers
+   *
+   *  Right now we only have the one
+   */
+
+  /**
+   * Ask the Monitors for a sequence of OSDMaps.
+   *
+   * @param epoch The epoch to start with when replying
+   * @param force_request True if this request forces a new subscription to
+   * the monitors; false if an outstanding request that encompasses it is
+   * sufficient.
+   */
+  void osdmap_subscribe(version_t epoch, bool force_request);
+  /** @} monc helpers */
+
+  // -- heartbeat --
+  /// information about a heartbeat peer
+  struct HeartbeatInfo {
+    int peer;           ///< peer
+    ConnectionRef con_front;   ///< peer connection (front)
+    ConnectionRef con_back;    ///< peer connection (back)
+    utime_t first_tx;   ///< time we sent our first ping request
+    utime_t last_tx;    ///< last time we sent a ping request
+    utime_t last_rx_front;  ///< last time we got a ping reply on the front side
+    utime_t last_rx_back;   ///< last time we got a ping reply on the back side
+    epoch_t epoch;      ///< most recent epoch we wanted this peer
+
+    bool is_unhealthy(utime_t cutoff) {
+      return
+	! ((last_rx_front > cutoff ||
+	    (last_rx_front == utime_t() && (last_tx == utime_t() ||
+					    first_tx > cutoff))) &&
+	   (last_rx_back > cutoff ||
+	    (last_rx_back == utime_t() && (last_tx == utime_t() ||
+					   first_tx > cutoff))));
+    }
+    bool is_healthy(utime_t cutoff) {
+      return last_rx_front > cutoff && last_rx_back > cutoff;
+    }
+
+  };
+  /// state attached to outgoing heartbeat connections
+  struct HeartbeatSession : public RefCountedObject {
+    int peer;
+    HeartbeatSession(int p) : peer(p) {}
+  };
+  Mutex heartbeat_lock;
+  map debug_heartbeat_drops_remaining;
+  Cond heartbeat_cond;
+  bool heartbeat_stop;
+  bool heartbeat_need_update;   ///< true if we need to refresh our heartbeat peers
+  epoch_t heartbeat_epoch;      ///< last epoch we updated our heartbeat peers
+  map heartbeat_peers;  ///< map of osd id to HeartbeatInfo
+  utime_t last_mon_heartbeat;
+  Messenger *hbclient_messenger;
+  Messenger *hb_front_server_messenger;
+  Messenger *hb_back_server_messenger;
+  utime_t last_heartbeat_resample;   ///< last time we chose random peers in waiting-for-healthy state
+  
+  void _add_heartbeat_peer(int p);
+  void _remove_heartbeat_peer(int p);
+  bool heartbeat_reset(Connection *con);
+  void maybe_update_heartbeat_peers();
+  void reset_heartbeat_peers();
+  void heartbeat();
+  void heartbeat_check();
+  void heartbeat_entry();
+  void need_heartbeat_peer_update();
+
+  void heartbeat_kick() {
+    Mutex::Locker l(heartbeat_lock);
+    heartbeat_cond.Signal();
+  }
+
+  struct T_Heartbeat : public Thread {
+    OSD *osd;
+    T_Heartbeat(OSD *o) : osd(o) {}
+    void *entry() {
+      osd->heartbeat_entry();
+      return 0;
+    }
+  } heartbeat_thread;
+
+public:
+  bool heartbeat_dispatch(Message *m);
+
+  struct HeartbeatDispatcher : public Dispatcher {
+    OSD *osd;
+    HeartbeatDispatcher(OSD *o) : Dispatcher(cct), osd(o) {}
+    bool ms_dispatch(Message *m) {
+      return osd->heartbeat_dispatch(m);
+    };
+    bool ms_handle_reset(Connection *con) {
+      return osd->heartbeat_reset(con);
+    }
+    void ms_handle_remote_reset(Connection *con) {}
+    bool ms_verify_authorizer(Connection *con, int peer_type,
+			      int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply,
+			      bool& isvalid, CryptoKey& session_key) {
+      isvalid = true;
+      return true;
+    }
+  } heartbeat_dispatcher;
+
+private:
+  // -- stats --
+  Mutex stat_lock;
+  osd_stat_t osd_stat;
+
+  void update_osd_stat();
+  
+  // -- waiters --
+  list finished;
+  Mutex finished_lock;
+  
+  void take_waiters(list& ls) {
+    finished_lock.Lock();
+    finished.splice(finished.end(), ls);
+    finished_lock.Unlock();
+  }
+  void take_waiters_front(list& ls) {
+    finished_lock.Lock();
+    finished.splice(finished.begin(), ls);
+    finished_lock.Unlock();
+  }
+  void take_waiter(OpRequestRef op) {
+    finished_lock.Lock();
+    finished.push_back(op);
+    finished_lock.Unlock();
+  }
+  void do_waiters();
+  
+  // -- op tracking --
+  OpTracker op_tracker;
+  void check_ops_in_flight();
+  void test_ops(std::string command, std::string args, ostream& ss);
+  friend class TestOpsSocketHook;
+  TestOpsSocketHook *test_ops_hook;
+  friend struct C_CompleteSplits;
+
+  // -- op queue --
+
+  struct OpWQ: public ThreadPool::WorkQueueVal,
+					       PGRef > {
+    Mutex qlock;
+    map > pg_for_processing;
+    OSD *osd;
+    PrioritizedQueue, entity_inst_t > pqueue;
+    OpWQ(OSD *o, time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueueVal, PGRef >(
+	"OSD::OpWQ", ti, ti*10, tp),
+	qlock("OpWQ::qlock"),
+	osd(o),
+	pqueue(o->cct->_conf->osd_op_pq_max_tokens_per_priority,
+	       o->cct->_conf->osd_op_pq_min_cost)
+    {}
+
+    void dump(Formatter *f) {
+      lock();
+      pqueue.dump(f);
+      unlock();
+    }
+
+    void _enqueue_front(pair item);
+    void _enqueue(pair item);
+    PGRef _dequeue();
+
+    struct Pred {
+      PG *pg;
+      Pred(PG *pg) : pg(pg) {}
+      bool operator()(const pair &op) {
+	return op.first == pg;
+      }
+    };
+    void dequeue(PG *pg, list *dequeued = 0) {
+      lock();
+      if (!dequeued) {
+	pqueue.remove_by_filter(Pred(pg));
+	pg_for_processing.erase(pg);
+      } else {
+	list > _dequeued;
+	pqueue.remove_by_filter(Pred(pg), &_dequeued);
+	for (list >::iterator i = _dequeued.begin();
+	     i != _dequeued.end();
+	     ++i) {
+	  dequeued->push_back(i->second);
+	}
+	if (pg_for_processing.count(pg)) {
+	  dequeued->splice(
+	    dequeued->begin(),
+	    pg_for_processing[pg]);
+	  pg_for_processing.erase(pg);
+	}
+      }
+      unlock();
+    }
+    bool _empty() {
+      return pqueue.empty();
+    }
+    void _process(PGRef pg, ThreadPool::TPHandle &handle);
+  } op_wq;
+
+  void enqueue_op(PG *pg, OpRequestRef op);
+  void dequeue_op(
+    PGRef pg, OpRequestRef op,
+    ThreadPool::TPHandle &handle);
+
+  // -- peering queue --
+  struct PeeringWQ : public ThreadPool::BatchWorkQueue {
+    list peering_queue;
+    OSD *osd;
+    set in_use;
+    PeeringWQ(OSD *o, time_t ti, ThreadPool *tp)
+      : ThreadPool::BatchWorkQueue(
+	"OSD::PeeringWQ", ti, ti*10, tp), osd(o) {}
+
+    void _dequeue(PG *pg) {
+      for (list::iterator i = peering_queue.begin();
+	   i != peering_queue.end();
+	   ) {
+	if (*i == pg) {
+	  peering_queue.erase(i++);
+	  pg->put("PeeringWQ");
+	} else {
+	  ++i;
+	}
+      }
+    }
+    bool _enqueue(PG *pg) {
+      pg->get("PeeringWQ");
+      peering_queue.push_back(pg);
+      return true;
+    }
+    bool _empty() {
+      return peering_queue.empty();
+    }
+    void _dequeue(list *out);
+    void _process(
+      const list &pgs,
+      ThreadPool::TPHandle &handle) {
+      osd->process_peering_events(pgs, handle);
+      for (list::const_iterator i = pgs.begin();
+	   i != pgs.end();
+	   ++i) {
+	(*i)->put("PeeringWQ");
+      }
+    }
+    void _process_finish(const list &pgs) {
+      for (list::const_iterator i = pgs.begin();
+	   i != pgs.end();
+	   ++i) {
+	in_use.erase(*i);
+      }
+    }
+    void _clear() {
+      assert(peering_queue.empty());
+    }
+  } peering_wq;
+
+  void process_peering_events(
+    const list &pg,
+    ThreadPool::TPHandle &handle);
+
+  friend class PG;
+  friend class ReplicatedPG;
+
+
+ protected:
+
+  // -- osd map --
+  OSDMapRef       osdmap;
+  OSDMapRef get_osdmap() {
+    return osdmap;
+  }
+  utime_t         had_map_since;
+  RWLock          map_lock;
+  list  waiting_for_osdmap;
+
+  Mutex peer_map_epoch_lock;
+  map peer_map_epoch;
+  
+  epoch_t get_peer_epoch(int p);
+  epoch_t note_peer_epoch(int p, epoch_t e);
+  void forget_peer_epoch(int p, epoch_t e);
+
+  bool _share_map_incoming(entity_name_t name, Connection *con, epoch_t epoch,
+			   Session *session = 0);
+  void _share_map_outgoing(int peer, Connection *con,
+			   OSDMapRef map = OSDMapRef());
+
+  void wait_for_new_map(OpRequestRef op);
+  void handle_osd_map(class MOSDMap *m);
+  void note_down_osd(int osd);
+  void note_up_osd(int osd);
+  
+  bool advance_pg(
+    epoch_t advance_to, PG *pg,
+    ThreadPool::TPHandle &handle,
+    PG::RecoveryCtx *rctx,
+    set > *split_pgs
+  );
+  void advance_map(ObjectStore::Transaction& t, C_Contexts *tfin);
+  void consume_map();
+  void activate_map();
+
+  // osd map cache (past osd maps)
+  OSDMapRef get_map(epoch_t e) {
+    return service.get_map(e);
+  }
+  OSDMapRef add_map(OSDMap *o) {
+    return service.add_map(o);
+  }
+  void add_map_bl(epoch_t e, bufferlist& bl) {
+    return service.add_map_bl(e, bl);
+  }
+  void pin_map_bl(epoch_t e, bufferlist &bl) {
+    return service.pin_map_bl(e, bl);
+  }
+  bool get_map_bl(epoch_t e, bufferlist& bl) {
+    return service.get_map_bl(e, bl);
+  }
+  void add_map_inc_bl(epoch_t e, bufferlist& bl) {
+    return service.add_map_inc_bl(e, bl);
+  }
+  void pin_map_inc_bl(epoch_t e, bufferlist &bl) {
+    return service.pin_map_inc_bl(e, bl);
+  }
+  bool get_inc_map_bl(epoch_t e, bufferlist& bl) {
+    return service.get_inc_map_bl(e, bl);
+  }
+
+  MOSDMap *build_incremental_map_msg(epoch_t from, epoch_t to);
+  void send_incremental_map(epoch_t since, Connection *con);
+  void send_map(MOSDMap *m, Connection *con);
+
+protected:
+  // -- placement groups --
+  ceph::unordered_map pg_map;
+  map > waiting_for_pg;
+  map > peering_wait_for_split;
+  PGRecoveryStats pg_recovery_stats;
+
+  PGPool _get_pool(int id, OSDMapRef createmap);
+
+  bool  _have_pg(spg_t pgid);
+  PG   *_lookup_lock_pg_with_map_lock_held(spg_t pgid);
+  PG   *_lookup_lock_pg(spg_t pgid);
+  PG   *_lookup_pg(spg_t pgid);
+  PG   *_open_lock_pg(OSDMapRef createmap,
+		      spg_t pg, bool no_lockdep_check=false,
+		      bool hold_map_lock=false);
+  enum res_result {
+    RES_PARENT,    // resurrected a parent
+    RES_SELF,      // resurrected self
+    RES_NONE       // nothing relevant deleting
+  };
+  res_result _try_resurrect_pg(
+    OSDMapRef curmap, spg_t pgid, spg_t *resurrected, PGRef *old_pg_state);
+  PG   *_create_lock_pg(
+    OSDMapRef createmap,
+    spg_t pgid,
+    bool newly_created,
+    bool hold_map_lock,
+    bool backfill,
+    int role,
+    vector& up, int up_primary,
+    vector& acting, int acting_primary,
+    pg_history_t history,
+    pg_interval_map_t& pi,
+    ObjectStore::Transaction& t);
+  PG   *_lookup_qlock_pg(spg_t pgid);
+
+  PG* _make_pg(OSDMapRef createmap, spg_t pgid);
+  void add_newly_split_pg(PG *pg,
+			  PG::RecoveryCtx *rctx);
+
+  void handle_pg_peering_evt(
+    spg_t pgid,
+    const pg_info_t& info,
+    pg_interval_map_t& pi,
+    epoch_t epoch,
+    pg_shard_t from,
+    bool primary,
+    PG::CephPeeringEvtRef evt);
+  
+  void load_pgs();
+  void build_past_intervals_parallel();
+
+  void calc_priors_during(
+    spg_t pgid, epoch_t start, epoch_t end, set& pset);
+
+  /// project pg history from from to now
+  bool project_pg_history(
+    spg_t pgid, pg_history_t& h, epoch_t from,
+    const vector& lastup,
+    int lastupprimary,
+    const vector& lastacting,
+    int lastactingprimary
+    ); ///< @return false if there was a map gap between from and now
+
+  void wake_pg_waiters(spg_t pgid) {
+    if (waiting_for_pg.count(pgid)) {
+      take_waiters_front(waiting_for_pg[pgid]);
+      waiting_for_pg.erase(pgid);
+    }
+  }
+  void wake_all_pg_waiters() {
+    for (map >::iterator p = waiting_for_pg.begin();
+	 p != waiting_for_pg.end();
+	 ++p)
+      take_waiters_front(p->second);
+    waiting_for_pg.clear();
+  }
+
+
+  // -- pg creation --
+  struct create_pg_info {
+    pg_history_t history;
+    vector acting;
+    set prior;
+    pg_t parent;
+  };
+  ceph::unordered_map creating_pgs;
+  double debug_drop_pg_create_probability;
+  int debug_drop_pg_create_duration;
+  int debug_drop_pg_create_left;  // 0 if we just dropped the last one, -1 if we can drop more
+
+  bool can_create_pg(spg_t pgid);
+  void handle_pg_create(OpRequestRef op);
+
+  void split_pgs(
+    PG *parent,
+    const set &childpgids, set > *out_pgs,
+    OSDMapRef curmap,
+    OSDMapRef nextmap,
+    PG::RecoveryCtx *rctx);
+
+  // == monitor interaction ==
+  utime_t last_mon_report;
+  utime_t last_pg_stats_sent;
+
+  /* if our monitor dies, we want to notice it and reconnect.
+   *  So we keep track of when it last acked our stat updates,
+   *  and if too much time passes (and we've been sending
+   *  more updates) then we can call it dead and reconnect
+   *  elsewhere.
+   */
+  utime_t last_pg_stats_ack;
+  bool outstanding_pg_stats; // some stat updates haven't been acked yet
+  bool timeout_mon_on_pg_stats;
+  void restart_stats_timer() {
+    Mutex::Locker l(osd_lock);
+    last_pg_stats_ack = ceph_clock_now(cct);
+    timeout_mon_on_pg_stats = true;
+  }
+
+  class C_MonStatsAckTimer : public Context {
+    OSD *osd;
+  public:
+    C_MonStatsAckTimer(OSD *o) : osd(o) {}
+    void finish(int r) {
+      osd->restart_stats_timer();
+    }
+  };
+  friend class C_MonStatsAckTimer;
+
+  void do_mon_report();
+
+  // -- boot --
+  void start_boot();
+  void _maybe_boot(epoch_t oldest, epoch_t newest);
+  void _send_boot();
+  void _collect_metadata(map *pmeta);
+
+  void start_waiting_for_healthy();
+  bool _is_healthy();
+  
+  friend struct C_OSD_GetVersion;
+
+  // -- alive --
+  epoch_t up_thru_wanted;
+  epoch_t up_thru_pending;
+
+  void queue_want_up_thru(epoch_t want);
+  void send_alive();
+
+  // -- failures --
+  map failure_queue;
+  map failure_pending;
+
+
+  void send_failures();
+  void send_still_alive(epoch_t epoch, const entity_inst_t &i);
+
+  // -- pg stats --
+  Mutex pg_stat_queue_lock;
+  Cond pg_stat_queue_cond;
+  xlist pg_stat_queue;
+  bool osd_stat_updated;
+  uint64_t pg_stat_tid, pg_stat_tid_flushed;
+
+  void send_pg_stats(const utime_t &now);
+  void handle_pg_stats_ack(class MPGStatsAck *ack);
+  void flush_pg_stats();
+
+  void pg_stat_queue_enqueue(PG *pg) {
+    pg_stat_queue_lock.Lock();
+    if (pg->is_primary() && !pg->stat_queue_item.is_on_list()) {
+      pg->get("pg_stat_queue");
+      pg_stat_queue.push_back(&pg->stat_queue_item);
+    }
+    osd_stat_updated = true;
+    pg_stat_queue_lock.Unlock();
+  }
+  void pg_stat_queue_dequeue(PG *pg) {
+    pg_stat_queue_lock.Lock();
+    if (pg->stat_queue_item.remove_myself())
+      pg->put("pg_stat_queue");
+    pg_stat_queue_lock.Unlock();
+  }
+  void clear_pg_stat_queue() {
+    pg_stat_queue_lock.Lock();
+    while (!pg_stat_queue.empty()) {
+      PG *pg = pg_stat_queue.front();
+      pg_stat_queue.pop_front();
+      pg->put("pg_stat_queue");
+    }
+    pg_stat_queue_lock.Unlock();
+  }
+
+  ceph_tid_t get_tid() {
+    return service.get_tid();
+  }
+
+  // -- generic pg peering --
+  PG::RecoveryCtx create_context();
+  bool compat_must_dispatch_immediately(PG *pg);
+  void dispatch_context(PG::RecoveryCtx &ctx, PG *pg, OSDMapRef curmap,
+                        ThreadPool::TPHandle *handle = NULL);
+  void dispatch_context_transaction(PG::RecoveryCtx &ctx, PG *pg,
+                                    ThreadPool::TPHandle *handle = NULL);
+  void do_notifies(map > >&
+		       notify_list,
+		   OSDMapRef map);
+  void do_queries(map >& query_map,
+		  OSDMapRef map);
+  void do_infos(map > >& info_map,
+		OSDMapRef map);
+  void repeer(PG *pg, map< int, map >& query_map);
+
+  bool require_mon_peer(Message *m);
+  bool require_osd_peer(OpRequestRef& op);
+  /***
+   * Verifies that we were alive in the given epoch, and that
+   * still are.
+   */
+  bool require_self_aliveness(OpRequestRef& op, epoch_t alive_since);
+  /**
+   * Verifies that the OSD who sent the given op has the same
+   * address as in the given map.
+   * @pre op was sent by an OSD using the cluster messenger
+   */
+  bool require_same_peer_instance(OpRequestRef& op, OSDMapRef& map);
+  bool require_up_osd_peer(OpRequestRef& Op, OSDMapRef& map,
+                           epoch_t their_epoch);
+
+  bool require_same_or_newer_map(OpRequestRef& op, epoch_t e);
+
+  void handle_pg_query(OpRequestRef op);
+  void handle_pg_notify(OpRequestRef op);
+  void handle_pg_log(OpRequestRef op);
+  void handle_pg_info(OpRequestRef op);
+  void handle_pg_trim(OpRequestRef op);
+
+  void handle_pg_scan(OpRequestRef op);
+
+  void handle_pg_backfill(OpRequestRef op);
+  void handle_pg_backfill_reserve(OpRequestRef op);
+  void handle_pg_recovery_reserve(OpRequestRef op);
+
+  void handle_pg_remove(OpRequestRef op);
+  void _remove_pg(PG *pg);
+
+  // -- commands --
+  struct Command {
+    vector cmd;
+    ceph_tid_t tid;
+    bufferlist indata;
+    ConnectionRef con;
+
+    Command(vector& c, ceph_tid_t t, bufferlist& bl, Connection *co)
+      : cmd(c), tid(t), indata(bl), con(co) {}
+  };
+  list command_queue;
+  struct CommandWQ : public ThreadPool::WorkQueue {
+    OSD *osd;
+    CommandWQ(OSD *o, time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueue("OSD::CommandWQ", ti, 0, tp), osd(o) {}
+
+    bool _empty() {
+      return osd->command_queue.empty();
+    }
+    bool _enqueue(Command *c) {
+      osd->command_queue.push_back(c);
+      return true;
+    }
+    void _dequeue(Command *pg) {
+      assert(0);
+    }
+    Command *_dequeue() {
+      if (osd->command_queue.empty())
+	return NULL;
+      Command *c = osd->command_queue.front();
+      osd->command_queue.pop_front();
+      return c;
+    }
+    void _process(Command *c) {
+      osd->osd_lock.Lock();
+      if (osd->is_stopping()) {
+	osd->osd_lock.Unlock();
+	delete c;
+	return;
+      }
+      osd->do_command(c->con.get(), c->tid, c->cmd, c->indata);
+      osd->osd_lock.Unlock();
+      delete c;
+    }
+    void _clear() {
+      while (!osd->command_queue.empty()) {
+	Command *c = osd->command_queue.front();
+	osd->command_queue.pop_front();
+	delete c;
+      }
+    }
+  } command_wq;
+
+  void handle_command(class MMonCommand *m);
+  void handle_command(class MCommand *m);
+  void do_command(Connection *con, ceph_tid_t tid, vector& cmd, bufferlist& data);
+
+  // -- pg recovery --
+  xlist recovery_queue;
+  utime_t defer_recovery_until;
+  int recovery_ops_active;
+#ifdef DEBUG_RECOVERY_OIDS
+  map > recovery_oids;
+#endif
+
+  struct RecoveryWQ : public ThreadPool::WorkQueue {
+    OSD *osd;
+    RecoveryWQ(OSD *o, time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueue("OSD::RecoveryWQ", ti, ti*10, tp), osd(o) {}
+
+    bool _empty() {
+      return osd->recovery_queue.empty();
+    }
+    bool _enqueue(PG *pg);
+    void _dequeue(PG *pg) {
+      if (pg->recovery_item.remove_myself())
+	pg->put("RecoveryWQ");
+    }
+    PG *_dequeue() {
+      if (osd->recovery_queue.empty())
+	return NULL;
+      
+      if (!osd->_recover_now())
+	return NULL;
+
+      PG *pg = osd->recovery_queue.front();
+      osd->recovery_queue.pop_front();
+      return pg;
+    }
+    void _queue_front(PG *pg) {
+      if (!pg->recovery_item.is_on_list()) {
+	pg->get("RecoveryWQ");
+	osd->recovery_queue.push_front(&pg->recovery_item);
+      }
+    }
+    void _process(PG *pg, ThreadPool::TPHandle &handle) {
+      osd->do_recovery(pg, handle);
+      pg->put("RecoveryWQ");
+    }
+    void _clear() {
+      while (!osd->recovery_queue.empty()) {
+	PG *pg = osd->recovery_queue.front();
+	osd->recovery_queue.pop_front();
+	pg->put("RecoveryWQ");
+      }
+    }
+  } recovery_wq;
+
+  void start_recovery_op(PG *pg, const hobject_t& soid);
+  void finish_recovery_op(PG *pg, const hobject_t& soid, bool dequeue);
+  void do_recovery(PG *pg, ThreadPool::TPHandle &handle);
+  bool _recover_now();
+
+  // replay / delayed pg activation
+  Mutex replay_queue_lock;
+  list< pair > replay_queue;
+  
+  void check_replay_queue();
+
+
+  // -- snap trimming --
+  xlist snap_trim_queue;
+  
+  struct SnapTrimWQ : public ThreadPool::WorkQueue {
+    OSD *osd;
+    SnapTrimWQ(OSD *o, time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueue("OSD::SnapTrimWQ", ti, 0, tp), osd(o) {}
+
+    bool _empty() {
+      return osd->snap_trim_queue.empty();
+    }
+    bool _enqueue(PG *pg) {
+      if (pg->snap_trim_item.is_on_list())
+	return false;
+      pg->get("SnapTrimWQ");
+      osd->snap_trim_queue.push_back(&pg->snap_trim_item);
+      return true;
+    }
+    void _dequeue(PG *pg) {
+      if (pg->snap_trim_item.remove_myself())
+	pg->put("SnapTrimWQ");
+    }
+    PG *_dequeue() {
+      if (osd->snap_trim_queue.empty())
+	return NULL;
+      PG *pg = osd->snap_trim_queue.front();
+      osd->snap_trim_queue.pop_front();
+      return pg;
+    }
+    void _process(PG *pg) {
+      pg->snap_trimmer();
+      pg->put("SnapTrimWQ");
+    }
+    void _clear() {
+      osd->snap_trim_queue.clear();
+    }
+  } snap_trim_wq;
+
+
+  // -- scrubbing --
+  void sched_scrub();
+  bool scrub_random_backoff();
+  bool scrub_should_schedule();
+
+  xlist scrub_queue;
+
+  struct ScrubWQ : public ThreadPool::WorkQueue {
+    OSD *osd;
+    ScrubWQ(OSD *o, time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueue("OSD::ScrubWQ", ti, 0, tp), osd(o) {}
+
+    bool _empty() {
+      return osd->scrub_queue.empty();
+    }
+    bool _enqueue(PG *pg) {
+      if (pg->scrub_item.is_on_list()) {
+	return false;
+      }
+      pg->get("ScrubWQ");
+      osd->scrub_queue.push_back(&pg->scrub_item);
+      return true;
+    }
+    void _dequeue(PG *pg) {
+      if (pg->scrub_item.remove_myself()) {
+	pg->put("ScrubWQ");
+      }
+    }
+    PG *_dequeue() {
+      if (osd->scrub_queue.empty())
+	return NULL;
+      PG *pg = osd->scrub_queue.front();
+      osd->scrub_queue.pop_front();
+      return pg;
+    }
+    void _process(
+      PG *pg,
+      ThreadPool::TPHandle &handle) {
+      pg->scrub(handle);
+      pg->put("ScrubWQ");
+    }
+    void _clear() {
+      while (!osd->scrub_queue.empty()) {
+	PG *pg = osd->scrub_queue.front();
+	osd->scrub_queue.pop_front();
+	pg->put("ScrubWQ");
+      }
+    }
+  } scrub_wq;
+
+  struct ScrubFinalizeWQ : public ThreadPool::WorkQueue {
+  private:
+    xlist scrub_finalize_queue;
+
+  public:
+    ScrubFinalizeWQ(time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueue("OSD::ScrubFinalizeWQ", ti, ti*10, tp) {}
+
+    bool _empty() {
+      return scrub_finalize_queue.empty();
+    }
+    bool _enqueue(PG *pg) {
+      if (pg->scrub_finalize_item.is_on_list()) {
+	return false;
+      }
+      pg->get("ScrubFinalizeWQ");
+      scrub_finalize_queue.push_back(&pg->scrub_finalize_item);
+      return true;
+    }
+    void _dequeue(PG *pg) {
+      if (pg->scrub_finalize_item.remove_myself()) {
+	pg->put("ScrubFinalizeWQ");
+      }
+    }
+    PG *_dequeue() {
+      if (scrub_finalize_queue.empty())
+	return NULL;
+      PG *pg = scrub_finalize_queue.front();
+      scrub_finalize_queue.pop_front();
+      return pg;
+    }
+    void _process(PG *pg) {
+      pg->scrub_finalize();
+      pg->put("ScrubFinalizeWQ");
+    }
+    void _clear() {
+      while (!scrub_finalize_queue.empty()) {
+	PG *pg = scrub_finalize_queue.front();
+	scrub_finalize_queue.pop_front();
+	pg->put("ScrubFinalizeWQ");
+      }
+    }
+  } scrub_finalize_wq;
+
+  struct RepScrubWQ : public ThreadPool::WorkQueue {
+  private: 
+    OSD *osd;
+    list rep_scrub_queue;
+
+  public:
+    RepScrubWQ(OSD *o, time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueue("OSD::RepScrubWQ", ti, 0, tp), osd(o) {}
+
+    bool _empty() {
+      return rep_scrub_queue.empty();
+    }
+    bool _enqueue(MOSDRepScrub *msg) {
+      rep_scrub_queue.push_back(msg);
+      return true;
+    }
+    void _dequeue(MOSDRepScrub *msg) {
+      assert(0); // Not applicable for this wq
+      return;
+    }
+    MOSDRepScrub *_dequeue() {
+      if (rep_scrub_queue.empty())
+	return NULL;
+      MOSDRepScrub *msg = rep_scrub_queue.front();
+      rep_scrub_queue.pop_front();
+      return msg;
+    }
+    void _process(
+      MOSDRepScrub *msg,
+      ThreadPool::TPHandle &handle) {
+      osd->osd_lock.Lock();
+      if (osd->is_stopping()) {
+	osd->osd_lock.Unlock();
+	return;
+      }
+      if (osd->_have_pg(msg->pgid)) {
+	PG *pg = osd->_lookup_lock_pg(msg->pgid);
+	osd->osd_lock.Unlock();
+	pg->replica_scrub(msg, handle);
+	msg->put();
+	pg->unlock();
+      } else {
+	msg->put();
+	osd->osd_lock.Unlock();
+      }
+    }
+    void _clear() {
+      while (!rep_scrub_queue.empty()) {
+	MOSDRepScrub *msg = rep_scrub_queue.front();
+	rep_scrub_queue.pop_front();
+	msg->put();
+      }
+    }
+  } rep_scrub_wq;
+
+  // -- removing --
+  struct RemoveWQ :
+    public ThreadPool::WorkQueueVal > {
+    ObjectStore *&store;
+    list > remove_queue;
+    RemoveWQ(ObjectStore *&o, time_t ti, ThreadPool *tp)
+      : ThreadPool::WorkQueueVal >(
+	"OSD::RemoveWQ", ti, 0, tp),
+	store(o) {}
+
+    bool _empty() {
+      return remove_queue.empty();
+    }
+    void _enqueue(pair item) {
+      remove_queue.push_back(item);
+    }
+    void _enqueue_front(pair item) {
+      remove_queue.push_front(item);
+    }
+    bool _dequeue(pair item) {
+      assert(0);
+    }
+    pair _dequeue() {
+      assert(!remove_queue.empty());
+      pair item = remove_queue.front();
+      remove_queue.pop_front();
+      return item;
+    }
+    void _process(pair, ThreadPool::TPHandle &);
+    void _clear() {
+      remove_queue.clear();
+    }
+  } remove_wq;
+  uint64_t next_removal_seq;
+  coll_t get_next_removal_coll(spg_t pgid) {
+    return coll_t::make_removal_coll(next_removal_seq++, pgid);
+  }
+
+ private:
+  bool ms_dispatch(Message *m);
+  bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new);
+  bool ms_verify_authorizer(Connection *con, int peer_type,
+			    int protocol, bufferlist& authorizer, bufferlist& authorizer_reply,
+			    bool& isvalid, CryptoKey& session_key);
+  void ms_handle_connect(Connection *con);
+  bool ms_handle_reset(Connection *con);
+  void ms_handle_remote_reset(Connection *con) {}
+
+ public:
+  /* internal and external can point to the same messenger, they will still
+   * be cleaned up properly*/
+  OSD(CephContext *cct_,
+      ObjectStore *store_,
+      int id,
+      Messenger *internal,
+      Messenger *external,
+      Messenger *hb_client,
+      Messenger *hb_front_server,
+      Messenger *hb_back_server,
+      Messenger *osdc_messenger,
+      MonClient *mc, const std::string &dev, const std::string &jdev);
+  ~OSD();
+
+  // static bits
+  static int find_osd_dev(char *result, int whoami);
+  static int do_convertfs(ObjectStore *store);
+  static int convert_collection(ObjectStore *store, coll_t cid);
+  static int mkfs(CephContext *cct, ObjectStore *store,
+		  const string& dev,
+		  uuid_d fsid, int whoami);
+  /* remove any non-user xattrs from a map of them */
+  void filter_xattrs(map& attrs) {
+    for (map::iterator iter = attrs.begin();
+	 iter != attrs.end();
+	 ) {
+      if (('_' != iter->first.at(0)) || (iter->first.size() == 1))
+	attrs.erase(iter++);
+      else ++iter;
+    }
+  }
+
+private:
+  static int write_meta(ObjectStore *store,
+			uuid_d& cluster_fsid, uuid_d& osd_fsid, int whoami);
+public:
+  static int peek_meta(ObjectStore *store, string& magic,
+		       uuid_d& cluster_fsid, uuid_d& osd_fsid, int& whoami);
+  
+
+  // startup/shutdown
+  int pre_init();
+  int init();
+  void final_init();
+
+  void suicide(int exitcode);
+  int shutdown();
+
+  void handle_signal(int signum);
+
+  void handle_rep_scrub(MOSDRepScrub *m);
+  void handle_scrub(struct MOSDScrub *m);
+  void handle_osd_ping(class MOSDPing *m);
+  void handle_op(OpRequestRef op);
+
+  template 
+  void handle_replica_op(OpRequestRef op);
+
+  /// check if we can throw out op from a disconnected client
+  static bool op_is_discardable(class MOSDOp *m);
+  /// check if op should be (re)queued for processing
+public:
+  void force_remount();
+
+  int init_op_flags(OpRequestRef op);
+
+  OSDService service;
+  friend class OSDService;
+};
+
+//compatibility of the executable
+extern const CompatSet::Feature ceph_osd_feature_compat[];
+extern const CompatSet::Feature ceph_osd_feature_ro_compat[];
+extern const CompatSet::Feature ceph_osd_feature_incompat[];
+
+#endif
diff --git a/ceph/src/osd/OSDCap.cc b/ceph/src/osd/OSDCap.cc
new file mode 100644
index 00000000..383674fd
--- /dev/null
+++ b/ceph/src/osd/OSDCap.cc
@@ -0,0 +1,260 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2009-2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "OSDCap.h"
+#include "common/config.h"
+#include "common/debug.h"
+
+using std::ostream;
+using std::vector;
+
+ostream& operator<<(ostream& out, osd_rwxa_t p)
+{ 
+  if (p == OSD_CAP_ANY)
+    return out << "*";
+
+  if (p & OSD_CAP_R)
+    out << "r";
+  if (p & OSD_CAP_W)
+    out << "w";
+  if ((p & OSD_CAP_X) == OSD_CAP_X) {
+    out << "x";
+  } else {
+    if (p & OSD_CAP_CLS_R)
+      out << " class-read";
+    if (p & OSD_CAP_CLS_W)
+      out << " class-write";
+  }
+  return out;
+}
+
+ostream& operator<<(ostream& out, const OSDCapSpec& s)
+{
+  if (s.allow)
+    return out << s.allow;
+  if (s.class_name.length())
+    return out << "class '" << s.class_name << "' '" << s.class_allow << "'";
+  return out;
+}
+
+ostream& operator<<(ostream& out, const OSDCapMatch& m)
+{
+  if (m.auid != -1LL) {
+    out << "auid " << m.auid << " ";
+  }
+  if (m.object_prefix.length()) {
+    out << "object_prefix " << m.object_prefix << " ";
+  }
+  if (m.pool_name.length()) {
+    out << "pool " << m.pool_name << " ";
+  }
+  if (m.is_nspace) {
+    out << "namespace ";
+    if (m.nspace.length() == 0)
+      out << "\"\"";
+    else
+      out << m.nspace;
+    out << " ";
+  }
+  return out;
+}
+
+bool OSDCapMatch::is_match(const string& pn, const string& ns, int64_t pool_auid, const string& object) const
+{
+  if (auid >= 0) {
+    if (auid != pool_auid)
+      return false;
+  }
+  if (pool_name.length()) {
+    if (pool_name != pn)
+      return false;
+  }
+  if (is_nspace) {
+    if (nspace != ns)
+      return false;
+  }
+  if (object_prefix.length()) {
+    if (object.find(object_prefix) != 0)
+      return false;
+  }
+  return true;
+}
+
+bool OSDCapMatch::is_match_all() const
+{
+  if (auid >= 0)
+    return false;
+  if (pool_name.length())
+    return false;
+  if (is_nspace)
+    return false;
+  if (object_prefix.length())
+    return false;
+  return true;
+}
+
+ostream& operator<<(ostream& out, const OSDCapGrant& g)
+{
+  return out << "grant(" << g.match << g.spec << ")";
+}
+
+
+bool OSDCap::allow_all() const
+{
+  for (vector::const_iterator p = grants.begin(); p != grants.end(); ++p)
+    if (p->match.is_match_all() && p->spec.allow_all())
+      return true;
+  return false;
+}
+
+void OSDCap::set_allow_all()
+{
+  grants.clear();
+  grants.push_back(OSDCapGrant(OSDCapMatch(), OSDCapSpec(OSD_CAP_ANY)));
+}
+
+bool OSDCap::is_capable(const string& pool_name, const string& ns, int64_t pool_auid,
+			const string& object, bool op_may_read,
+			bool op_may_write, bool op_may_class_read,
+			bool op_may_class_write) const
+{
+  osd_rwxa_t allow = 0;
+  for (vector::const_iterator p = grants.begin();
+       p != grants.end(); ++p) {
+    if (p->match.is_match(pool_name, ns, pool_auid, object)) {
+      allow = allow | p->spec.allow;
+      if ((op_may_read && !(allow & OSD_CAP_R)) ||
+	  (op_may_write && !(allow & OSD_CAP_W)) ||
+	  (op_may_class_read && !(allow & OSD_CAP_CLS_R)) ||
+	  (op_may_class_write && !(allow & OSD_CAP_CLS_W)))
+	continue;
+      return true;
+    }
+  }
+  return false;
+}
+
+
+// grammar
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::ascii;
+namespace phoenix = boost::phoenix;
+
+template 
+struct OSDCapParser : qi::grammar
+{
+  OSDCapParser() : OSDCapParser::base_type(osdcap)
+  {
+    using qi::char_;
+    using qi::int_;
+    using qi::lexeme;
+    using qi::alnum;
+    using qi::_val;
+    using qi::_1;
+    using qi::_2;
+    using qi::_3;
+    using qi::eps;
+    using qi::lit;
+
+    quoted_string %=
+      lexeme['"' >> +(char_ - '"') >> '"'] | 
+      lexeme['\'' >> +(char_ - '\'') >> '\''];
+    equoted_string %=
+      lexeme['"' >> *(char_ - '"') >> '"'] |
+      lexeme['\'' >> *(char_ - '\'') >> '\''];
+    unquoted_word %= +char_("a-zA-Z0-9_.-");
+    str %= quoted_string | unquoted_word;
+    estr %= equoted_string | unquoted_word;
+
+    spaces = +(lit(' ') | lit('\n') | lit('\t'));
+
+
+    // match := [pool[=] [namespace[=]] | auid <123>] [object_prefix ]
+    pool_name %= -(spaces >> lit("pool") >> (lit('=') | spaces) >> str);
+    nspace %= (spaces >> lit("namespace") >> (lit('=') | spaces) >> estr);
+    auid %= (spaces >> lit("auid") >> spaces >> int_);
+    object_prefix %= -(spaces >> lit("object_prefix") >> spaces >> str);
+
+    match = ( (auid >> object_prefix)                 [_val = phoenix::construct(_1, _2)] |
+             (pool_name >> nspace >> object_prefix)   [_val = phoenix::construct(_1, _2, _3)] |
+             (pool_name >> object_prefix)             [_val = phoenix::construct(_1, _2)]);
+
+    // rwxa := * | [r][w][x] [class-read] [class-write]
+    rwxa =
+      (spaces >> lit("*")[_val = OSD_CAP_ANY]) |
+      ( eps[_val = 0] >>
+	(
+	 spaces >>
+	 ( lit('r')[_val |= OSD_CAP_R] ||
+	   lit('w')[_val |= OSD_CAP_W] ||
+	   lit('x')[_val |= OSD_CAP_X] )) ||
+	( (spaces >> lit("class-read")[_val |= OSD_CAP_CLS_R]) ||
+	  (spaces >> lit("class-write")[_val |= OSD_CAP_CLS_W]) ));
+
+    // capspec := * | rwx | class  [classcap]
+    capspec =
+      rwxa                                            [_val = phoenix::construct(_1)] |
+      ( spaces >> lit("class") >> spaces >> ((str >> spaces >> str)   [_val = phoenix::construct(_1, _2)] |
+			 str                          [_val = phoenix::construct(_1, string())] ));
+
+    // grant := allow match capspec
+    grant = (*lit(' ') >> lit("allow") >>
+	     ((capspec >> match)       [_val = phoenix::construct(_2, _1)] |
+	      (match >> capspec)       [_val = phoenix::construct(_1, _2)]) >>
+	     *lit(' '));
+    // osdcap := grant [grant ...]
+    grants %= (grant % (*lit(' ') >> (lit(';') | lit(',')) >> *lit(' ')));
+    osdcap = grants  [_val = phoenix::construct(_1)]; 
+  }
+  qi::rule spaces;
+  qi::rule rwxa;
+  qi::rule quoted_string, equoted_string;
+  qi::rule unquoted_word;
+  qi::rule str, estr;
+  qi::rule auid;
+  qi::rule capspec;
+  qi::rule pool_name;
+  qi::rule nspace;
+  qi::rule object_prefix;
+  qi::rule match;
+  qi::rule grant;
+  qi::rule()> grants;
+  qi::rule osdcap;
+};
+
+bool OSDCap::parse(const string& str, ostream *err)
+{
+  OSDCapParser g;
+  string::const_iterator iter = str.begin();
+  string::const_iterator end = str.end();
+
+  bool r = qi::phrase_parse(iter, end, g, ascii::space, *this);
+  if (r && iter == end)
+    return true;
+
+  // Make sure no grants are kept after parsing failed!
+  grants.clear();
+
+  if (err)
+    *err << "osdcap parse failed, stopped at '" << std::string(iter, end)
+	 << "' of '" << str << "'\n";
+
+  return false; 
+}
+
diff --git a/ceph/src/osd/OSDCap.h b/ceph/src/osd/OSDCap.h
new file mode 100644
index 00000000..3fc7fb67
--- /dev/null
+++ b/ceph/src/osd/OSDCap.h
@@ -0,0 +1,158 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ * OSDCaps: Hold the capabilities associated with a single authenticated 
+ * user key. These are specified by text strings of the form
+ * "allow r" (which allows reading anything on the OSD)
+ * "allow rwx auid foo" (which allows full access to listed auids)
+ *  "allow rwx pool foo" (which allows full access to listed pools)
+ * "allow *" (which allows full access to EVERYTHING)
+ *
+ * The full grammar is documented in the parser in OSDCap.cc.
+ *
+ * The OSD assumes that anyone with * caps is an admin and has full
+ * message permissions. This means that only the monitor and the OSDs
+ * should get *
+ */
+
+#ifndef CEPH_OSDCAP_H
+#define CEPH_OSDCAP_H
+
+#include 
+using std::ostream;
+
+#include "include/types.h"
+
+static const __u8 OSD_CAP_R     = (1 << 1);      // read
+static const __u8 OSD_CAP_W     = (1 << 2);      // write
+static const __u8 OSD_CAP_CLS_R = (1 << 3);      // class read
+static const __u8 OSD_CAP_CLS_W = (1 << 4);      // class write
+static const __u8 OSD_CAP_X     = (OSD_CAP_CLS_R | OSD_CAP_CLS_W); // execute
+static const __u8 OSD_CAP_ANY   = 0xff;          // *
+
+struct osd_rwxa_t {
+  __u8 val;
+
+  osd_rwxa_t(__u8 v = 0) : val(v) {}
+  osd_rwxa_t& operator=(__u8 v) {
+    val = v;
+    return *this;
+  }
+  operator __u8() const {
+    return val;
+  }
+};
+
+ostream& operator<<(ostream& out, osd_rwxa_t p);
+
+struct OSDCapSpec {
+  osd_rwxa_t allow;
+  std::string class_name;
+  std::string class_allow;
+
+  OSDCapSpec() : allow(0) {}
+  OSDCapSpec(osd_rwxa_t v) : allow(v) {}
+  OSDCapSpec(std::string n) : allow(0), class_name(n) {}
+  OSDCapSpec(std::string n, std::string a) : allow(0), class_name(n), class_allow(a) {}
+
+  bool allow_all() const {
+    return allow == OSD_CAP_ANY;
+  }
+};
+
+ostream& operator<<(ostream& out, const OSDCapSpec& s);
+
+
+struct OSDCapMatch {
+  // auid and pool_name/nspace are mutually exclusive
+  int64_t auid;
+  std::string pool_name;
+  bool is_nspace;      // true if nspace is defined; false if not constrained.
+  std::string nspace;
+
+  std::string object_prefix;
+
+  OSDCapMatch() : auid(CEPH_AUTH_UID_DEFAULT), is_nspace(false) {}
+  OSDCapMatch(std::string pl, std::string pre) :
+	auid(CEPH_AUTH_UID_DEFAULT), pool_name(pl), is_nspace(false), object_prefix(pre) {}
+  OSDCapMatch(std::string pl, std::string ns, std::string pre) :
+	auid(CEPH_AUTH_UID_DEFAULT), pool_name(pl), is_nspace(true), nspace(ns), object_prefix(pre) {}
+  OSDCapMatch(uint64_t auid, std::string pre) : auid(auid), is_nspace(false), object_prefix(pre) {}
+
+  /**
+   * check if given request parameters match our constraints
+   *
+   * @param auid requesting user's auid
+   * @param pool_name pool name
+   * @param nspace_name namespace name
+   * @param pool_auid pool's auid
+   * @param object object name
+   * @return true if we match, false otherwise
+   */
+  bool is_match(const std::string& pool_name, const std::string& nspace_name, int64_t pool_auid, const std::string& object) const;
+  bool is_match_all() const;
+};
+
+ostream& operator<<(ostream& out, const OSDCapMatch& m);
+
+
+struct OSDCapGrant {
+  OSDCapMatch match;
+  OSDCapSpec spec;
+
+  OSDCapGrant() {}
+  OSDCapGrant(OSDCapMatch m, OSDCapSpec s) : match(m), spec(s) {}
+};
+
+ostream& operator<<(ostream& out, const OSDCapGrant& g);
+
+
+struct OSDCap {
+  std::vector grants;
+
+  OSDCap() {}
+  OSDCap(std::vector g) : grants(g) {}
+
+  bool allow_all() const;
+  void set_allow_all();
+  bool parse(const std::string& str, ostream *err=NULL);
+
+  /**
+   * check if we are capable of something
+   *
+   * This method actually checks a description of a particular operation against
+   * what the capability has specified.  Currently that is just rwx with matches
+   * against pool, pool auid, and object name prefix.
+   *
+   * @param pool_name name of the pool we are accessing
+   * @param ns name of the namespace we are accessing
+   * @param pool_auid owner of the pool we are accessing
+   * @param object name of the object we are accessing
+   * @param op_may_read whether the operation may need to read
+   * @param op_may_write whether the operation may need to write
+   * @param op_may_class_read whether the operation needs to call a
+   *                          read class method
+   * @param op_may_class_write whether the operation needs to call a
+   *                          write class method
+   * @return true if the operation is allowed, false otherwise
+   */
+  bool is_capable(const string& pool_name, const string& ns, int64_t pool_auid,
+		  const string& object, bool op_may_read, bool op_may_write,
+		  bool op_may_class_read, bool op_may_class_write) const;
+};
+
+static inline ostream& operator<<(ostream& out, const OSDCap& cap) 
+{
+  return out << "osdcap" << cap.grants;
+}
+
+#endif
diff --git a/ceph/src/osd/OSDMap.cc b/ceph/src/osd/OSDMap.cc
new file mode 100644
index 00000000..645a6f70
--- /dev/null
+++ b/ceph/src/osd/OSDMap.cc
@@ -0,0 +1,2727 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "OSDMap.h"
+
+#include "common/config.h"
+#include "common/Formatter.h"
+#include "include/ceph_features.h"
+#include "include/str_map.h"
+
+#include "common/code_environment.h"
+
+#define dout_subsys ceph_subsys_osd
+
+// ----------------------------------
+// osd_info_t
+
+void osd_info_t::dump(Formatter *f) const
+{
+  f->dump_int("last_clean_begin", last_clean_begin);
+  f->dump_int("last_clean_end", last_clean_end);
+  f->dump_int("up_from", up_from);
+  f->dump_int("up_thru", up_thru);
+  f->dump_int("down_at", down_at);
+  f->dump_int("lost_at", lost_at);
+}
+
+void osd_info_t::encode(bufferlist& bl) const
+{
+  __u8 struct_v = 1;
+  ::encode(struct_v, bl);
+  ::encode(last_clean_begin, bl);
+  ::encode(last_clean_end, bl);
+  ::encode(up_from, bl);
+  ::encode(up_thru, bl);
+  ::encode(down_at, bl);
+  ::encode(lost_at, bl);
+}
+
+void osd_info_t::decode(bufferlist::iterator& bl)
+{
+  __u8 struct_v;
+  ::decode(struct_v, bl);
+  ::decode(last_clean_begin, bl);
+  ::decode(last_clean_end, bl);
+  ::decode(up_from, bl);
+  ::decode(up_thru, bl);
+  ::decode(down_at, bl);
+  ::decode(lost_at, bl);
+}
+
+void osd_info_t::generate_test_instances(list& o)
+{
+  o.push_back(new osd_info_t);
+  o.push_back(new osd_info_t);
+  o.back()->last_clean_begin = 1;
+  o.back()->last_clean_end = 2;
+  o.back()->up_from = 30;
+  o.back()->up_thru = 40;
+  o.back()->down_at = 5;
+  o.back()->lost_at = 6;
+}
+
+ostream& operator<<(ostream& out, const osd_info_t& info)
+{
+  out << "up_from " << info.up_from
+      << " up_thru " << info.up_thru
+      << " down_at " << info.down_at
+      << " last_clean_interval [" << info.last_clean_begin << "," << info.last_clean_end << ")";
+  if (info.lost_at)
+    out << " lost_at " << info.lost_at;
+  return out;
+}
+
+// ----------------------------------
+// osd_xinfo_t
+
+void osd_xinfo_t::dump(Formatter *f) const
+{
+  f->dump_stream("down_stamp") << down_stamp;
+  f->dump_float("laggy_probability", laggy_probability);
+  f->dump_int("laggy_interval", laggy_interval);
+  f->dump_int("features", features);
+}
+
+void osd_xinfo_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 1, bl);
+  ::encode(down_stamp, bl);
+  __u32 lp = laggy_probability * 0xfffffffful;
+  ::encode(lp, bl);
+  ::encode(laggy_interval, bl);
+  ::encode(features, bl);
+  ENCODE_FINISH(bl);
+}
+
+void osd_xinfo_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START(1, bl);
+  ::decode(down_stamp, bl);
+  __u32 lp;
+  ::decode(lp, bl);
+  laggy_probability = (float)lp / (float)0xffffffff;
+  ::decode(laggy_interval, bl);
+  if (struct_v >= 2)
+    ::decode(features, bl);
+  else
+    features = 0;
+  DECODE_FINISH(bl);
+}
+
+void osd_xinfo_t::generate_test_instances(list& o)
+{
+  o.push_back(new osd_xinfo_t);
+  o.push_back(new osd_xinfo_t);
+  o.back()->down_stamp = utime_t(2, 3);
+  o.back()->laggy_probability = .123;
+  o.back()->laggy_interval = 123456;
+}
+
+ostream& operator<<(ostream& out, const osd_xinfo_t& xi)
+{
+  return out << "down_stamp " << xi.down_stamp
+	     << " laggy_probability " << xi.laggy_probability
+	     << " laggy_interval " << xi.laggy_interval;
+}
+
+// ----------------------------------
+// OSDMap::Incremental
+
+int OSDMap::Incremental::get_net_marked_out(const OSDMap *previous) const
+{
+  int n = 0;
+  for (map::const_iterator p = new_weight.begin();
+       p != new_weight.end();
+       ++p) {
+    if (p->second == CEPH_OSD_OUT && !previous->is_out(p->first))
+      n++;  // marked out
+    if (p->second != CEPH_OSD_OUT && previous->is_out(p->first))
+      n--;  // marked in
+  }
+  return n;
+}
+
+int OSDMap::Incremental::get_net_marked_down(const OSDMap *previous) const
+{
+  int n = 0;
+  for (map::const_iterator p = new_state.begin();
+       p != new_state.end();
+       ++p) {
+    if (p->second & CEPH_OSD_UP) {
+      if (previous->is_up(p->first))
+	n++;  // marked down
+      else
+	n--;  // marked up
+    }
+  }
+  return n;
+}
+
+int OSDMap::Incremental::identify_osd(uuid_d u) const
+{
+  for (map::const_iterator p = new_uuid.begin();
+       p != new_uuid.end();
+       ++p)
+    if (p->second == u)
+      return p->first;
+  return -1;
+}
+
+int OSDMap::Incremental::propagate_snaps_to_tiers(CephContext *cct,
+						  const OSDMap& osdmap)
+{
+  assert(epoch == osdmap.get_epoch() + 1);
+  for (map::iterator p = new_pools.begin();
+       p != new_pools.end(); ++p) {
+    if (!p->second.tiers.empty()) {
+      pg_pool_t& base = p->second;
+      for (set::const_iterator q = base.tiers.begin();
+	   q != base.tiers.end();
+	   ++q) {
+	map::iterator r = new_pools.find(*q);
+	pg_pool_t *tier = 0;
+	if (r == new_pools.end()) {
+	  const pg_pool_t *orig = osdmap.get_pg_pool(*q);
+	  if (!orig) {
+	    lderr(cct) << __func__ << " no pool " << *q << dendl;
+	    return -EIO;
+	  }
+	  tier = get_new_pool(*q, orig);
+	} else {
+	  tier = &r->second;
+	}
+	if (tier->tier_of != p->first) {
+	  lderr(cct) << __func__ << " " << r->first << " tier_of != " << p->first << dendl;
+	  return -EIO;
+	}
+
+	ldout(cct, 10) << __func__ << " from " << p->first << " to "
+		       << r->first << dendl;
+	tier->snap_seq = base.snap_seq;
+	tier->snap_epoch = base.snap_epoch;
+	tier->snaps = base.snaps;
+	tier->removed_snaps = base.removed_snaps;
+      }
+    }
+  }
+  return 0;
+}
+
+
+bool OSDMap::subtree_is_down(int id, set *down_cache) const
+{
+  if (id >= 0)
+    return is_down(id);
+
+  if (down_cache &&
+      down_cache->count(id)) {
+    return true;
+  }
+
+  list children;
+  crush->get_children(id, &children);
+  for (list::iterator p = children.begin(); p != children.end(); ++p) {
+    if (!subtree_is_down(*p, down_cache)) {
+      return false;
+    }
+  }
+  if (down_cache) {
+    down_cache->insert(id);
+  }
+  return true;
+}
+
+bool OSDMap::containing_subtree_is_down(CephContext *cct, int id, int subtree_type, set *down_cache) const
+{
+  // use a stack-local down_cache if we didn't get one from the
+  // caller.  then at least this particular call will avoid duplicated
+  // work.
+  set local_down_cache;
+  if (!down_cache) {
+    down_cache = &local_down_cache;
+  }
+
+  int current = id;
+  while (true) {
+    int type;
+    if (current >= 0) {
+      type = 0;
+    } else {
+      type = crush->get_bucket_type(current);
+    }
+    assert(type >= 0);
+
+    if (!subtree_is_down(current, down_cache)) {
+      ldout(cct, 30) << "containing_subtree_is_down(" << id << ") = false" << dendl;
+      return false;
+    }
+
+    // is this a big enough subtree to be done?
+    if (type >= subtree_type) {
+      ldout(cct, 30) << "containing_subtree_is_down(" << id << ") = true ... " << type << " >= " << subtree_type << dendl;
+      return true;
+    }
+
+    int r = crush->get_immediate_parent_id(current, ¤t);
+    if (r < 0) {
+      return false;
+    }
+  }
+}
+
+void OSDMap::Incremental::encode_client_old(bufferlist& bl) const
+{
+  __u16 v = 5;
+  ::encode(v, bl);
+  ::encode(fsid, bl);
+  ::encode(epoch, bl);
+  ::encode(modified, bl);
+  int32_t new_t = new_pool_max;
+  ::encode(new_t, bl);
+  ::encode(new_flags, bl);
+  ::encode(fullmap, bl);
+  ::encode(crush, bl);
+
+  ::encode(new_max_osd, bl);
+  // for ::encode(new_pools, bl);
+  __u32 n = new_pools.size();
+  ::encode(n, bl);
+  for (map::const_iterator p = new_pools.begin();
+       p != new_pools.end();
+       ++p) {
+    n = p->first;
+    ::encode(n, bl);
+    ::encode(p->second, bl, 0);
+  }
+  // for ::encode(new_pool_names, bl);
+  n = new_pool_names.size();
+  ::encode(n, bl);
+  for (map::const_iterator p = new_pool_names.begin(); p != new_pool_names.end(); ++p) {
+    n = p->first;
+    ::encode(n, bl);
+    ::encode(p->second, bl);
+  }
+  // for ::encode(old_pools, bl);
+  n = old_pools.size();
+  ::encode(n, bl);
+  for (set::iterator p = old_pools.begin(); p != old_pools.end(); ++p) {
+    n = *p;
+    ::encode(n, bl);
+  }
+  ::encode(new_up_client, bl);
+  ::encode(new_state, bl);
+  ::encode(new_weight, bl);
+  // for ::encode(new_pg_temp, bl);
+  n = new_pg_temp.size();
+  ::encode(n, bl);
+  for (map >::const_iterator p = new_pg_temp.begin();
+       p != new_pg_temp.end();
+       ++p) {
+    old_pg_t opg = p->first.get_old_pg();
+    ::encode(opg, bl);
+    ::encode(p->second, bl);
+  }
+}
+
+void OSDMap::Incremental::encode_classic(bufferlist& bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_PGID64) == 0) {
+    encode_client_old(bl);
+    return;
+  }
+
+  // base
+  __u16 v = 6;
+  ::encode(v, bl);
+  ::encode(fsid, bl);
+  ::encode(epoch, bl);
+  ::encode(modified, bl);
+  ::encode(new_pool_max, bl);
+  ::encode(new_flags, bl);
+  ::encode(fullmap, bl);
+  ::encode(crush, bl);
+
+  ::encode(new_max_osd, bl);
+  ::encode(new_pools, bl, features);
+  ::encode(new_pool_names, bl);
+  ::encode(old_pools, bl);
+  ::encode(new_up_client, bl);
+  ::encode(new_state, bl);
+  ::encode(new_weight, bl);
+  ::encode(new_pg_temp, bl);
+
+  // extended
+  __u16 ev = 10;
+  ::encode(ev, bl);
+  ::encode(new_hb_back_up, bl);
+  ::encode(new_up_thru, bl);
+  ::encode(new_last_clean_interval, bl);
+  ::encode(new_lost, bl);
+  ::encode(new_blacklist, bl);
+  ::encode(old_blacklist, bl);
+  ::encode(new_up_cluster, bl);
+  ::encode(cluster_snapshot, bl);
+  ::encode(new_uuid, bl);
+  ::encode(new_xinfo, bl);
+  ::encode(new_hb_front_up, bl);
+}
+
+void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_OSDMAP_ENC) == 0) {
+    encode_classic(bl, features);
+    return;
+  }
+
+  // meta-encoding: how we include client-used and osd-specific data
+  ENCODE_START(7, 7, bl);
+
+  {
+    ENCODE_START(3, 1, bl); // client-usable data
+    ::encode(fsid, bl);
+    ::encode(epoch, bl);
+    ::encode(modified, bl);
+    ::encode(new_pool_max, bl);
+    ::encode(new_flags, bl);
+    ::encode(fullmap, bl);
+    ::encode(crush, bl);
+
+    ::encode(new_max_osd, bl);
+    ::encode(new_pools, bl, features);
+    ::encode(new_pool_names, bl);
+    ::encode(old_pools, bl);
+    ::encode(new_up_client, bl);
+    ::encode(new_state, bl);
+    ::encode(new_weight, bl);
+    ::encode(new_pg_temp, bl);
+    ::encode(new_primary_temp, bl);
+    ::encode(new_primary_affinity, bl);
+    ::encode(new_erasure_code_profiles, bl);
+    ::encode(old_erasure_code_profiles, bl);
+    ENCODE_FINISH(bl); // client-usable data
+  }
+
+  {
+    ENCODE_START(2, 1, bl); // extended, osd-only data
+    ::encode(new_hb_back_up, bl);
+    ::encode(new_up_thru, bl);
+    ::encode(new_last_clean_interval, bl);
+    ::encode(new_lost, bl);
+    ::encode(new_blacklist, bl);
+    ::encode(old_blacklist, bl);
+    ::encode(new_up_cluster, bl);
+    ::encode(cluster_snapshot, bl);
+    ::encode(new_uuid, bl);
+    ::encode(new_xinfo, bl);
+    ::encode(new_hb_front_up, bl);
+    ::encode(features, bl);         // NOTE: features arg, not the member
+    ENCODE_FINISH(bl); // osd-only data
+  }
+
+  ENCODE_FINISH(bl); // meta-encoding wrapper
+
+}
+
+void OSDMap::Incremental::decode_classic(bufferlist::iterator &p)
+{
+  __u32 n, t;
+  // base
+  __u16 v;
+  ::decode(v, p);
+  ::decode(fsid, p);
+  ::decode(epoch, p);
+  ::decode(modified, p);
+  if (v == 4 || v == 5) {
+    ::decode(n, p);
+    new_pool_max = n;
+  } else if (v >= 6)
+    ::decode(new_pool_max, p);
+  ::decode(new_flags, p);
+  ::decode(fullmap, p);
+  ::decode(crush, p);
+
+  ::decode(new_max_osd, p);
+  if (v < 6) {
+    new_pools.clear();
+    ::decode(n, p);
+    while (n--) {
+      ::decode(t, p);
+      ::decode(new_pools[t], p);
+    }
+  } else {
+    ::decode(new_pools, p);
+  }
+  if (v == 5) {
+    new_pool_names.clear();
+    ::decode(n, p);
+    while (n--) {
+      ::decode(t, p);
+      ::decode(new_pool_names[t], p);
+    }
+  } else if (v >= 6) {
+    ::decode(new_pool_names, p);
+  }
+  if (v < 6) {
+    old_pools.clear();
+    ::decode(n, p);
+    while (n--) {
+      ::decode(t, p);
+      old_pools.insert(t);
+    }
+  } else {
+    ::decode(old_pools, p);
+  }
+  ::decode(new_up_client, p);
+  ::decode(new_state, p);
+  ::decode(new_weight, p);
+
+  if (v < 6) {
+    new_pg_temp.clear();
+    ::decode(n, p);
+    while (n--) {
+      old_pg_t opg;
+      ::decode_raw(opg, p);
+      ::decode(new_pg_temp[pg_t(opg)], p);
+    }
+  } else {
+    ::decode(new_pg_temp, p);
+  }
+
+  // decode short map, too.
+  if (v == 5 && p.end())
+    return;
+
+  // extended
+  __u16 ev = 0;
+  if (v >= 5)
+    ::decode(ev, p);
+  ::decode(new_hb_back_up, p);
+  if (v < 5)
+    ::decode(new_pool_names, p);
+  ::decode(new_up_thru, p);
+  ::decode(new_last_clean_interval, p);
+  ::decode(new_lost, p);
+  ::decode(new_blacklist, p);
+  ::decode(old_blacklist, p);
+  if (ev >= 6)
+    ::decode(new_up_cluster, p);
+  if (ev >= 7)
+    ::decode(cluster_snapshot, p);
+  if (ev >= 8)
+    ::decode(new_uuid, p);
+  if (ev >= 9)
+    ::decode(new_xinfo, p);
+  if (ev >= 10)
+    ::decode(new_hb_front_up, p);
+}
+
+void OSDMap::Incremental::decode(bufferlist::iterator& bl)
+{
+  /**
+   * Older encodings of the Incremental had a single struct_v which
+   * covered the whole encoding, and was prior to our modern
+   * stuff which includes a compatv and a size. So if we see
+   * a struct_v < 7, we must rewind to the beginning and use our
+   * classic decoder.
+   */
+  DECODE_START_LEGACY_COMPAT_LEN(7, 7, 7, bl); // wrapper
+  if (struct_v < 7) {
+    int struct_v_size = sizeof(struct_v);
+    bl.advance(-struct_v_size);
+    decode_classic(bl);
+    encode_features = 0;
+    return;
+  }
+  {
+    DECODE_START(3, bl); // client-usable data
+    ::decode(fsid, bl);
+    ::decode(epoch, bl);
+    ::decode(modified, bl);
+    ::decode(new_pool_max, bl);
+    ::decode(new_flags, bl);
+    ::decode(fullmap, bl);
+    ::decode(crush, bl);
+
+    ::decode(new_max_osd, bl);
+    ::decode(new_pools, bl);
+    ::decode(new_pool_names, bl);
+    ::decode(old_pools, bl);
+    ::decode(new_up_client, bl);
+    ::decode(new_state, bl);
+    ::decode(new_weight, bl);
+    ::decode(new_pg_temp, bl);
+    ::decode(new_primary_temp, bl);
+    if (struct_v >= 2)
+      ::decode(new_primary_affinity, bl);
+    else
+      new_primary_affinity.clear();
+    if (struct_v >= 3) {
+      ::decode(new_erasure_code_profiles, bl);
+      ::decode(old_erasure_code_profiles, bl);
+    } else {
+      new_erasure_code_profiles.clear();
+      old_erasure_code_profiles.clear();
+    }
+    DECODE_FINISH(bl); // client-usable data
+  }
+
+  {
+    DECODE_START(2, bl); // extended, osd-only data
+    ::decode(new_hb_back_up, bl);
+    ::decode(new_up_thru, bl);
+    ::decode(new_last_clean_interval, bl);
+    ::decode(new_lost, bl);
+    ::decode(new_blacklist, bl);
+    ::decode(old_blacklist, bl);
+    ::decode(new_up_cluster, bl);
+    ::decode(cluster_snapshot, bl);
+    ::decode(new_uuid, bl);
+    ::decode(new_xinfo, bl);
+    ::decode(new_hb_front_up, bl);
+    if (struct_v >= 2)
+      ::decode(encode_features, bl);
+    else
+      encode_features = 0;
+    DECODE_FINISH(bl); // osd-only data
+  }
+
+  DECODE_FINISH(bl); // wrapper
+}
+
+void OSDMap::Incremental::dump(Formatter *f) const
+{
+  f->dump_int("epoch", epoch);
+  f->dump_stream("fsid") << fsid;
+  f->dump_stream("modified") << modified;
+  f->dump_int("new_pool_max", new_pool_max);
+  f->dump_int("new_flags", new_flags);
+
+  if (fullmap.length()) {
+    f->open_object_section("full_map");
+    OSDMap full;
+    bufferlist fbl = fullmap;  // kludge around constness.
+    bufferlist::iterator p = fbl.begin();
+    full.decode(p);
+    full.dump(f);
+    f->close_section();
+  }
+  if (crush.length()) {
+    f->open_object_section("crush");
+    CrushWrapper c;
+    bufferlist tbl = crush;  // kludge around constness.
+    bufferlist::iterator p = tbl.begin();
+    c.decode(p);
+    c.dump(f);
+    f->close_section();
+  }
+
+  f->dump_int("new_max_osd", new_max_osd);
+
+  f->open_array_section("new_pools");
+  for (map::const_iterator p = new_pools.begin(); p != new_pools.end(); ++p) {
+    f->open_object_section("pool");
+    f->dump_int("pool", p->first);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("new_pool_names");
+  for (map::const_iterator p = new_pool_names.begin(); p != new_pool_names.end(); ++p) {
+    f->open_object_section("pool_name");
+    f->dump_int("pool", p->first);
+    f->dump_string("name", p->second);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("old_pools");
+  for (set::const_iterator p = old_pools.begin(); p != old_pools.end(); ++p)
+    f->dump_int("pool", *p);
+  f->close_section();
+
+  f->open_array_section("new_up_osds");
+  for (map::const_iterator p = new_up_client.begin(); p != new_up_client.end(); ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    f->dump_stream("public_addr") << p->second;
+    f->dump_stream("cluster_addr") << new_up_cluster.find(p->first)->second;
+    f->dump_stream("heartbeat_back_addr") << new_hb_back_up.find(p->first)->second;
+    map::const_iterator q;
+    if ((q = new_hb_front_up.find(p->first)) != new_hb_front_up.end())
+      f->dump_stream("heartbeat_front_addr") << q->second;
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("new_weight");
+  for (map::const_iterator p = new_weight.begin(); p != new_weight.end(); ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    f->dump_int("weight", p->second);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("osd_state_xor");
+  for (map::const_iterator p = new_state.begin(); p != new_state.end(); ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    set st;
+    calc_state_set(new_state.find(p->first)->second, st);
+    f->open_array_section("state_xor");
+    for (set::iterator p = st.begin(); p != st.end(); ++p)
+      f->dump_string("state", *p);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("new_pg_temp");
+  for (map >::const_iterator p = new_pg_temp.begin();
+       p != new_pg_temp.end();
+       ++p) {
+    f->open_object_section("pg");
+    f->dump_stream("pgid") << p->first;
+    f->open_array_section("osds");
+    for (vector::const_iterator q = p->second.begin(); q != p->second.end(); ++q)
+      f->dump_int("osd", *q);
+    f->close_section();
+    f->close_section();    
+  }
+  f->close_section();
+
+  f->open_array_section("primary_temp");
+  for (map::const_iterator p = new_primary_temp.begin();
+      p != new_primary_temp.end();
+      ++p) {
+    f->dump_stream("pgid") << p->first;
+    f->dump_int("osd", p->second);
+  }
+  f->close_section(); // primary_temp
+
+  f->open_array_section("new_up_thru");
+  for (map::const_iterator p = new_up_thru.begin(); p != new_up_thru.end(); ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    f->dump_int("up_thru", p->second);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("new_lost");
+  for (map::const_iterator p = new_lost.begin(); p != new_lost.end(); ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    f->dump_int("epoch_lost", p->second);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("new_last_clean_interval");
+  for (map >::const_iterator p = new_last_clean_interval.begin();
+       p != new_last_clean_interval.end();
+       ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    f->dump_int("first", p->second.first);
+    f->dump_int("last", p->second.second);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("new_blacklist");
+  for (map::const_iterator p = new_blacklist.begin();
+       p != new_blacklist.end();
+       ++p) {
+    stringstream ss;
+    ss << p->first;
+    f->dump_stream(ss.str().c_str()) << p->second;
+  }
+  f->close_section();
+  f->open_array_section("old_blacklist");
+  for (vector::const_iterator p = old_blacklist.begin(); p != old_blacklist.end(); ++p)
+    f->dump_stream("addr") << *p;
+  f->close_section();
+
+  f->open_array_section("new_xinfo");
+  for (map::const_iterator p = new_xinfo.begin(); p != new_xinfo.end(); ++p) {
+    f->open_object_section("xinfo");
+    f->dump_int("osd", p->first);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  if (cluster_snapshot.size())
+    f->dump_string("cluster_snapshot", cluster_snapshot);
+
+  f->open_array_section("new_uuid");
+  for (map::const_iterator p = new_uuid.begin(); p != new_uuid.end(); ++p) {
+    f->open_object_section("osd");
+    f->dump_int("osd", p->first);
+    f->dump_stream("uuid") << p->second;
+    f->close_section();
+  }
+  f->close_section();
+
+  OSDMap::dump_erasure_code_profiles(new_erasure_code_profiles, f);
+  f->open_array_section("old_erasure_code_profiles");
+  for (vector::const_iterator p = old_erasure_code_profiles.begin();
+       p != old_erasure_code_profiles.end();
+       p++) {
+    f->dump_string("old", p->c_str());
+  }
+  f->close_section();
+}
+
+void OSDMap::Incremental::generate_test_instances(list& o)
+{
+  o.push_back(new Incremental);
+}
+
+// ----------------------------------
+// OSDMap
+
+void OSDMap::set_epoch(epoch_t e)
+{
+  epoch = e;
+  for (map::iterator p = pools.begin();
+       p != pools.end();
+       ++p)
+    p->second.last_change = e;
+}
+
+bool OSDMap::is_blacklisted(const entity_addr_t& a) const
+{
+  if (blacklist.empty())
+    return false;
+
+  // this specific instance?
+  if (blacklist.count(a))
+    return true;
+
+  // is entire ip blacklisted?
+  if (a.is_ip()) {
+    entity_addr_t b = a;
+    b.set_port(0);
+    b.set_nonce(0);
+    if (blacklist.count(b)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void OSDMap::get_blacklist(list > *bl) const
+{
+  for (ceph::unordered_map::const_iterator it = blacklist.begin() ;
+			 it != blacklist.end(); ++it) {
+    bl->push_back(*it);
+  }
+}
+
+void OSDMap::set_max_osd(int m)
+{
+  int o = max_osd;
+  max_osd = m;
+  osd_state.resize(m);
+  osd_weight.resize(m);
+  for (; oclient_addr.resize(m);
+  osd_addrs->cluster_addr.resize(m);
+  osd_addrs->hb_back_addr.resize(m);
+  osd_addrs->hb_front_addr.resize(m);
+  osd_uuid->resize(m);
+  if (osd_primary_affinity)
+    osd_primary_affinity->resize(m, CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
+
+  calc_num_osds();
+}
+
+int OSDMap::calc_num_osds()
+{
+  num_osd = 0;
+  for (int i=0; i& ls) const
+{
+  for (int i=0; i& ls) const
+{
+  for (int i = 0; i < max_osd; i++) {
+    if (is_up(i))
+      ls.insert(i);
+  }
+}
+
+unsigned OSDMap::get_num_up_osds() const
+{
+  unsigned n = 0;
+  for (int i=0; i& st)
+{
+  unsigned t = state;
+  for (unsigned s = 1; t; s <<= 1) {
+    if (t & s) {
+      t &= ~s;
+      st.insert(ceph_osd_state_name(s));
+    }
+  }
+}
+
+void OSDMap::adjust_osd_weights(const map& weights, Incremental& inc) const
+{
+  float max = 0;
+  for (map::const_iterator p = weights.begin();
+       p != weights.end(); ++p) {
+    if (p->second > max)
+      max = p->second;
+  }
+
+  for (map::const_iterator p = weights.begin();
+       p != weights.end(); ++p) {
+    inc.new_weight[p->first] = (unsigned)((p->second / max) * CEPH_OSD_IN);
+  }
+}
+
+int OSDMap::identify_osd(const entity_addr_t& addr) const
+{
+  for (int i=0; ihas_nondefault_tunables())
+    features |= CEPH_FEATURE_CRUSH_TUNABLES;
+  if (crush->has_nondefault_tunables2())
+    features |= CEPH_FEATURE_CRUSH_TUNABLES2;
+  if (crush->has_nondefault_tunables3())
+    features |= CEPH_FEATURE_CRUSH_TUNABLES3;
+  mask |= CEPH_FEATURES_CRUSH;
+
+  for (map::const_iterator p = pools.begin(); p != pools.end(); ++p) {
+    if (p->second.flags & pg_pool_t::FLAG_HASHPSPOOL) {
+      features |= CEPH_FEATURE_OSDHASHPSPOOL;
+    }
+    if (p->second.is_erasure() &&
+	entity_type != CEPH_ENTITY_TYPE_CLIENT) { // not for clients
+      features |= CEPH_FEATURE_OSD_ERASURE_CODES;
+    }
+    if (!p->second.tiers.empty() ||
+	p->second.is_tier()) {
+      features |= CEPH_FEATURE_OSD_CACHEPOOL;
+    }
+    int ruleid = crush->find_rule(p->second.get_crush_ruleset(),
+				  p->second.get_type(),
+				  p->second.get_size());
+    if (ruleid >= 0) {
+      if (crush->is_v2_rule(ruleid))
+	features |= CEPH_FEATURE_CRUSH_V2;
+      if (crush->is_v3_rule(ruleid))
+	features |= CEPH_FEATURE_CRUSH_TUNABLES3;
+    }
+  }
+  mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL;
+  if (entity_type != CEPH_ENTITY_TYPE_CLIENT)
+    mask |= CEPH_FEATURE_OSD_ERASURE_CODES;
+
+  if (osd_primary_affinity) {
+    for (int i = 0; i < max_osd; ++i) {
+      if ((*osd_primary_affinity)[i] != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
+	features |= CEPH_FEATURE_OSD_PRIMARY_AFFINITY;
+	break;
+      }
+    }
+  }
+  mask |= CEPH_FEATURE_OSD_PRIMARY_AFFINITY;
+
+  if (pmask)
+    *pmask = mask;
+  return features;
+}
+
+uint64_t OSDMap::get_up_osd_features() const
+{
+  bool first = true;
+  uint64_t features = 0;
+  for (int osd = 0; osd < max_osd; ++osd) {
+    if (!is_up(osd))
+      continue;
+    const osd_xinfo_t &xi = get_xinfo(osd);
+    if (first) {
+      features = xi.features;
+      first = false;
+    } else {
+      features &= xi.features;
+    }
+  }
+  return features;
+}
+
+void OSDMap::dedup(const OSDMap *o, OSDMap *n)
+{
+  if (o->epoch == n->epoch)
+    return;
+
+  int diff = 0;
+
+  // do addrs match?
+  if (o->max_osd != n->max_osd)
+    diff++;
+  for (int i = 0; i < o->max_osd && i < n->max_osd; i++) {
+    if ( n->osd_addrs->client_addr[i] &&  o->osd_addrs->client_addr[i] &&
+	*n->osd_addrs->client_addr[i] == *o->osd_addrs->client_addr[i])
+      n->osd_addrs->client_addr[i] = o->osd_addrs->client_addr[i];
+    else
+      diff++;
+    if ( n->osd_addrs->cluster_addr[i] &&  o->osd_addrs->cluster_addr[i] &&
+	*n->osd_addrs->cluster_addr[i] == *o->osd_addrs->cluster_addr[i])
+      n->osd_addrs->cluster_addr[i] = o->osd_addrs->cluster_addr[i];
+    else
+      diff++;
+    if ( n->osd_addrs->hb_back_addr[i] &&  o->osd_addrs->hb_back_addr[i] &&
+	*n->osd_addrs->hb_back_addr[i] == *o->osd_addrs->hb_back_addr[i])
+      n->osd_addrs->hb_back_addr[i] = o->osd_addrs->hb_back_addr[i];
+    else
+      diff++;
+    if ( n->osd_addrs->hb_front_addr[i] &&  o->osd_addrs->hb_front_addr[i] &&
+	*n->osd_addrs->hb_front_addr[i] == *o->osd_addrs->hb_front_addr[i])
+      n->osd_addrs->hb_front_addr[i] = o->osd_addrs->hb_front_addr[i];
+    else
+      diff++;
+  }
+  if (diff == 0) {
+    // zoinks, no differences at all!
+    n->osd_addrs = o->osd_addrs;
+  }
+
+  // does crush match?
+  bufferlist oc, nc;
+  ::encode(*o->crush, oc);
+  ::encode(*n->crush, nc);
+  if (oc.contents_equal(nc)) {
+    n->crush = o->crush;
+  }
+
+  // does pg_temp match?
+  if (o->pg_temp->size() == n->pg_temp->size()) {
+    if (*o->pg_temp == *n->pg_temp)
+      n->pg_temp = o->pg_temp;
+  }
+
+  // does primary_temp match?
+  if (o->primary_temp->size() == n->primary_temp->size()) {
+    if (*o->primary_temp == *n->primary_temp)
+      n->primary_temp = o->primary_temp;
+  }
+
+  // do uuids match?
+  if (o->osd_uuid->size() == n->osd_uuid->size() &&
+      *o->osd_uuid == *n->osd_uuid)
+    n->osd_uuid = o->osd_uuid;
+}
+
+void OSDMap::remove_redundant_temporaries(CephContext *cct, const OSDMap& osdmap,
+					  OSDMap::Incremental *pending_inc)
+{
+  ldout(cct, 10) << "remove_redundant_temporaries" << dendl;
+
+  for (map >::iterator p = osdmap.pg_temp->begin();
+       p != osdmap.pg_temp->end();
+       ++p) {
+    if (pending_inc->new_pg_temp.count(p->first) == 0) {
+      vector raw_up;
+      int primary;
+      osdmap.pg_to_raw_up(p->first, &raw_up, &primary);
+      if (raw_up == p->second) {
+        ldout(cct, 10) << " removing unnecessary pg_temp " << p->first << " -> " << p->second << dendl;
+        pending_inc->new_pg_temp[p->first].clear();
+      }
+    }
+  }
+  if (!osdmap.primary_temp->empty()) {
+    OSDMap templess;
+    templess.deepish_copy_from(osdmap);
+    templess.primary_temp->clear();
+    for (map::iterator p = osdmap.primary_temp->begin();
+        p != osdmap.primary_temp->end();
+        ++p) {
+      if (pending_inc->new_primary_temp.count(p->first) == 0) {
+        vector real_up, templess_up;
+        int real_primary, templess_primary;
+        osdmap.pg_to_acting_osds(p->first, &real_up, &real_primary);
+        templess.pg_to_acting_osds(p->first, &templess_up, &templess_primary);
+        if (real_primary == templess_primary){
+          ldout(cct, 10) << " removing unnecessary primary_temp "
+                         << p->first << " -> " << p->second << dendl;
+          pending_inc->new_primary_temp[p->first] = -1;
+        }
+      }
+    }
+  }
+}
+
+void OSDMap::remove_down_temps(CephContext *cct,
+                               const OSDMap& osdmap, Incremental *pending_inc)
+{
+  ldout(cct, 10) << "remove_down_pg_temp" << dendl;
+  OSDMap tmpmap;
+  tmpmap.deepish_copy_from(osdmap);
+  tmpmap.apply_incremental(*pending_inc);
+
+  for (map >::iterator p = tmpmap.pg_temp->begin();
+       p != tmpmap.pg_temp->end();
+       ++p) {
+    unsigned num_up = 0;
+    for (vector::iterator i = p->second.begin();
+	 i != p->second.end();
+	 ++i) {
+      if (!tmpmap.is_down(*i))
+	++num_up;
+    }
+    if (num_up == 0)
+      pending_inc->new_pg_temp[p->first].clear();
+  }
+  for (map::iterator p = tmpmap.primary_temp->begin();
+      p != tmpmap.primary_temp->end();
+      ++p) {
+    if (tmpmap.is_down(p->second))
+      pending_inc->new_primary_temp[p->first] = -1;
+  }
+}
+
+int OSDMap::apply_incremental(const Incremental &inc)
+{
+  new_blacklist_entries = false;
+  if (inc.epoch == 1)
+    fsid = inc.fsid;
+  else if (inc.fsid != fsid)
+    return -EINVAL;
+  
+  assert(inc.epoch == epoch+1);
+  epoch++;
+  modified = inc.modified;
+
+  // full map?
+  if (inc.fullmap.length()) {
+    bufferlist bl(inc.fullmap);
+    decode(bl);
+    return 0;
+  }
+
+  // nope, incremental.
+  if (inc.new_flags >= 0)
+    flags = inc.new_flags;
+
+  if (inc.new_max_osd >= 0)
+    set_max_osd(inc.new_max_osd);
+
+  if (inc.new_pool_max != -1)
+    pool_max = inc.new_pool_max;
+
+  for (set::const_iterator p = inc.old_pools.begin();
+       p != inc.old_pools.end();
+       ++p) {
+    pools.erase(*p);
+    name_pool.erase(pool_name[*p]);
+    pool_name.erase(*p);
+  }
+  for (map::const_iterator p = inc.new_pools.begin();
+       p != inc.new_pools.end();
+       ++p) {
+    pools[p->first] = p->second;
+    pools[p->first].last_change = epoch;
+  }
+  for (map::const_iterator p = inc.new_pool_names.begin();
+       p != inc.new_pool_names.end();
+       ++p) {
+    if (pool_name.count(p->first))
+      name_pool.erase(pool_name[p->first]);
+    pool_name[p->first] = p->second;
+    name_pool[p->second] = p->first;
+  }
+
+  for (map::const_iterator i = inc.new_weight.begin();
+       i != inc.new_weight.end();
+       ++i) {
+    set_weight(i->first, i->second);
+
+    // if we are marking in, clear the AUTOOUT and NEW bits.
+    if (i->second)
+      osd_state[i->first] &= ~(CEPH_OSD_AUTOOUT | CEPH_OSD_NEW);
+  }
+
+  for (map::const_iterator i = inc.new_primary_affinity.begin();
+       i != inc.new_primary_affinity.end();
+       ++i) {
+    set_primary_affinity(i->first, i->second);
+  }
+
+  // erasure_code_profiles
+  for (map >::const_iterator i =
+	 inc.new_erasure_code_profiles.begin();
+       i != inc.new_erasure_code_profiles.end();
+       i++) {
+    set_erasure_code_profile(i->first, i->second);
+  }
+  
+  for (vector::const_iterator i = inc.old_erasure_code_profiles.begin();
+       i != inc.old_erasure_code_profiles.end();
+       i++)
+    erasure_code_profiles.erase(*i);
+  
+  // up/down
+  for (map::const_iterator i = inc.new_state.begin();
+       i != inc.new_state.end();
+       ++i) {
+    int s = i->second ? i->second : CEPH_OSD_UP;
+    if ((osd_state[i->first] & CEPH_OSD_UP) &&
+	(s & CEPH_OSD_UP)) {
+      osd_info[i->first].down_at = epoch;
+      osd_xinfo[i->first].down_stamp = modified;
+    }
+    if ((osd_state[i->first] & CEPH_OSD_EXISTS) &&
+	(s & CEPH_OSD_EXISTS))
+      (*osd_uuid)[i->first] = uuid_d();
+    osd_state[i->first] ^= s;
+  }
+  for (map::const_iterator i = inc.new_up_client.begin();
+       i != inc.new_up_client.end();
+       ++i) {
+    osd_state[i->first] |= CEPH_OSD_EXISTS | CEPH_OSD_UP;
+    osd_addrs->client_addr[i->first].reset(new entity_addr_t(i->second));
+    if (inc.new_hb_back_up.empty())
+      osd_addrs->hb_back_addr[i->first].reset(new entity_addr_t(i->second)); //this is a backward-compatibility hack
+    else
+      osd_addrs->hb_back_addr[i->first].reset(
+	new entity_addr_t(inc.new_hb_back_up.find(i->first)->second));
+    map::const_iterator j = inc.new_hb_front_up.find(i->first);
+    if (j != inc.new_hb_front_up.end())
+      osd_addrs->hb_front_addr[i->first].reset(new entity_addr_t(j->second));
+    else
+      osd_addrs->hb_front_addr[i->first].reset();
+
+    osd_info[i->first].up_from = epoch;
+  }
+  for (map::const_iterator i = inc.new_up_cluster.begin();
+       i != inc.new_up_cluster.end();
+       ++i)
+    osd_addrs->cluster_addr[i->first].reset(new entity_addr_t(i->second));
+
+  // info
+  for (map::const_iterator i = inc.new_up_thru.begin();
+       i != inc.new_up_thru.end();
+       ++i)
+    osd_info[i->first].up_thru = i->second;
+  for (map >::const_iterator i = inc.new_last_clean_interval.begin();
+       i != inc.new_last_clean_interval.end();
+       ++i) {
+    osd_info[i->first].last_clean_begin = i->second.first;
+    osd_info[i->first].last_clean_end = i->second.second;
+  }
+  for (map::const_iterator p = inc.new_lost.begin(); p != inc.new_lost.end(); ++p)
+    osd_info[p->first].lost_at = p->second;
+
+  // xinfo
+  for (map::const_iterator p = inc.new_xinfo.begin(); p != inc.new_xinfo.end(); ++p)
+    osd_xinfo[p->first] = p->second;
+
+  // uuid
+  for (map::const_iterator p = inc.new_uuid.begin(); p != inc.new_uuid.end(); ++p) 
+    (*osd_uuid)[p->first] = p->second;
+
+  // pg rebuild
+  for (map >::const_iterator p = inc.new_pg_temp.begin(); p != inc.new_pg_temp.end(); ++p) {
+    if (p->second.empty())
+      pg_temp->erase(p->first);
+    else
+      (*pg_temp)[p->first] = p->second;
+  }
+
+  for (map::const_iterator p = inc.new_primary_temp.begin();
+      p != inc.new_primary_temp.end();
+      ++p) {
+    if (p->second == -1)
+      primary_temp->erase(p->first);
+    else
+      (*primary_temp)[p->first] = p->second;
+  }
+
+  // blacklist
+  for (map::const_iterator p = inc.new_blacklist.begin();
+       p != inc.new_blacklist.end();
+       ++p) {
+    blacklist[p->first] = p->second;
+    new_blacklist_entries = true;
+  }
+  for (vector::const_iterator p = inc.old_blacklist.begin();
+       p != inc.old_blacklist.end();
+       ++p)
+    blacklist.erase(*p);
+
+  // cluster snapshot?
+  if (inc.cluster_snapshot.length()) {
+    cluster_snapshot = inc.cluster_snapshot;
+    cluster_snapshot_epoch = inc.epoch;
+  } else {
+    cluster_snapshot.clear();
+    cluster_snapshot_epoch = 0;
+  }
+
+  // do new crush map last (after up/down stuff)
+  if (inc.crush.length()) {
+    bufferlist bl(inc.crush);
+    bufferlist::iterator blp = bl.begin();
+    crush.reset(new CrushWrapper);
+    crush->decode(blp);
+  }
+
+  calc_num_osds();
+  return 0;
+}
+
+// mapping
+int OSDMap::object_locator_to_pg(
+	const object_t& oid,
+	const object_locator_t& loc,
+	pg_t &pg) const
+{
+  // calculate ps (placement seed)
+  const pg_pool_t *pool = get_pg_pool(loc.get_pool());
+  if (!pool)
+    return -ENOENT;
+  ps_t ps;
+  if (loc.hash >= 0) {
+    ps = loc.hash;
+  } else {
+    if (!loc.key.empty())
+      ps = pool->hash_key(loc.key, loc.nspace);
+    else
+      ps = pool->hash_key(oid.name, loc.nspace);
+  }
+  pg = pg_t(ps, loc.get_pool(), -1);
+  return 0;
+}
+
+ceph_object_layout OSDMap::make_object_layout(
+  object_t oid, int pg_pool, string nspace) const
+{
+  object_locator_t loc(pg_pool, nspace);
+
+  ceph_object_layout ol;
+  pg_t pgid = object_locator_to_pg(oid, loc);
+  ol.ol_pgid = pgid.get_old_pg().v;
+  ol.ol_stripe_unit = 0;
+  return ol;
+}
+
+void OSDMap::_remove_nonexistent_osds(const pg_pool_t& pool,
+				      vector& osds) const
+{
+  if (pool.can_shift_osds()) {
+    unsigned removed = 0;
+    for (unsigned i = 0; i < osds.size(); i++) {
+      if (!exists(osds[i])) {
+	removed++;
+	continue;
+      }
+      if (removed) {
+	osds[i - removed] = osds[i];
+      }
+    }
+    if (removed)
+      osds.resize(osds.size() - removed);
+  } else {
+    for (vector::iterator p = osds.begin(); p != osds.end(); ++p) {
+      if (!exists(*p))
+	*p = CRUSH_ITEM_NONE;
+    }
+  }
+}
+
+int OSDMap::_pg_to_osds(const pg_pool_t& pool, pg_t pg,
+                        vector *osds, int *primary,
+			ps_t *ppps) const
+{
+  // map to osds[]
+  ps_t pps = pool.raw_pg_to_pps(pg);  // placement ps
+  unsigned size = pool.get_size();
+
+  // what crush rule?
+  int ruleno = crush->find_rule(pool.get_crush_ruleset(), pool.get_type(), size);
+  if (ruleno >= 0)
+    crush->do_rule(ruleno, pps, *osds, size, osd_weight);
+
+  _remove_nonexistent_osds(pool, *osds);
+
+  *primary = -1;
+  for (unsigned i = 0; i < osds->size(); ++i) {
+    if ((*osds)[i] != CRUSH_ITEM_NONE) {
+      *primary = (*osds)[i];
+      break;
+    }
+  }
+  if (ppps)
+    *ppps = pps;
+
+  return osds->size();
+}
+
+// pg -> (up osd list)
+void OSDMap::_raw_to_up_osds(const pg_pool_t& pool, const vector& raw,
+                             vector *up, int *primary) const
+{
+  if (pool.can_shift_osds()) {
+    // shift left
+    up->clear();
+    for (unsigned i=0; ipush_back(raw[i]);
+    }
+    *primary = (up->empty() ? -1 : up->front());
+  } else {
+    // set down/dne devices to NONE
+    *primary = -1;
+    up->resize(raw.size());
+    for (int i = raw.size() - 1; i >= 0; --i) {
+      if (!exists(raw[i]) || is_down(raw[i])) {
+	(*up)[i] = CRUSH_ITEM_NONE;
+      } else {
+	*primary = (*up)[i] = raw[i];
+      }
+    }
+  }
+}
+
+void OSDMap::_apply_primary_affinity(ps_t seed,
+				     const pg_pool_t& pool,
+				     vector *osds,
+				     int *primary) const
+{
+  // do we have any non-default primary_affinity values for these osds?
+  if (!osd_primary_affinity)
+    return;
+
+  bool any = false;
+  for (vector::const_iterator p = osds->begin(); p != osds->end(); ++p) {
+    if (*p != CRUSH_ITEM_NONE &&
+	(*osd_primary_affinity)[*p] != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
+      any = true;
+    }
+  }
+  if (!any)
+    return;
+
+  // pick the primary.  feed both the seed (for the pg) and the osd
+  // into the hash/rng so that a proportional fraction of an osd's pgs
+  // get rejected as primary.
+  int pos = -1;
+  for (unsigned i = 0; i < osds->size(); ++i) {
+    int o = (*osds)[i];
+    if (o == CRUSH_ITEM_NONE)
+      continue;
+    unsigned a = (*osd_primary_affinity)[o];
+    if (a < CEPH_OSD_MAX_PRIMARY_AFFINITY &&
+	(crush_hash32_2(CRUSH_HASH_RJENKINS1,
+			seed, o) >> 16) >= a) {
+      // we chose not to use this primary.  note it anyway as a
+      // fallback in case we don't pick anyone else, but keep looking.
+      if (pos < 0)
+	pos = i;
+    } else {
+      pos = i;
+      break;
+    }
+  }
+  if (pos < 0)
+    return;
+
+  *primary = (*osds)[pos];
+
+  if (pool.can_shift_osds() && pos > 0) {
+    // move the new primary to the front.
+    for (int i = pos; i > 0; --i) {
+      (*osds)[i] = (*osds)[i-1];
+    }
+    (*osds)[0] = *primary;
+  }
+}
+
+void OSDMap::_get_temp_osds(const pg_pool_t& pool, pg_t pg,
+                            vector *temp_pg, int *temp_primary) const
+{
+  pg = pool.raw_pg_to_pg(pg);
+  map >::const_iterator p = pg_temp->find(pg);
+  temp_pg->clear();
+  if (p != pg_temp->end()) {
+    for (unsigned i=0; isecond.size(); i++) {
+      if (!exists(p->second[i]) || is_down(p->second[i])) {
+	if (pool.can_shift_osds()) {
+	  continue;
+	} else {
+	  temp_pg->push_back(CRUSH_ITEM_NONE);
+	}
+      } else {
+	temp_pg->push_back(p->second[i]);
+      }
+    }
+  }
+  map::const_iterator pp = primary_temp->find(pg);
+  *temp_primary = -1;
+  if (pp != primary_temp->end()) {
+    *temp_primary = pp->second;
+  } else if (!temp_pg->empty()) { // apply pg_temp's primary
+    for (unsigned i = 0; i < temp_pg->size(); ++i) {
+      if ((*temp_pg)[i] != CRUSH_ITEM_NONE) {
+	*temp_primary = (*temp_pg)[i];
+	break;
+      }
+    }
+  }
+}
+
+int OSDMap::pg_to_osds(pg_t pg, vector *raw, int *primary) const
+{
+  *primary = -1;
+  raw->clear();
+  const pg_pool_t *pool = get_pg_pool(pg.pool());
+  if (!pool)
+    return 0;
+  int r = _pg_to_osds(*pool, pg, raw, primary, NULL);
+  return r;
+}
+
+void OSDMap::pg_to_raw_up(pg_t pg, vector *up, int *primary) const
+{
+  const pg_pool_t *pool = get_pg_pool(pg.pool());
+  if (!pool) {
+    if (primary)
+      *primary = -1;
+    if (up)
+      up->clear();
+    return;
+  }
+  vector raw;
+  ps_t pps;
+  _pg_to_osds(*pool, pg, &raw, primary, &pps);
+  _raw_to_up_osds(*pool, raw, up, primary);
+  _apply_primary_affinity(pps, *pool, up, primary);
+}
+  
+void OSDMap::_pg_to_up_acting_osds(pg_t pg, vector *up, int *up_primary,
+                                   vector *acting, int *acting_primary) const
+{
+  const pg_pool_t *pool = get_pg_pool(pg.pool());
+  if (!pool) {
+    if (up)
+      up->clear();
+    if (up_primary)
+      *up_primary = -1;
+    if (acting)
+      acting->clear();
+    if (acting_primary)
+      *acting_primary = -1;
+    return;
+  }
+  vector raw;
+  vector _up;
+  vector _acting;
+  int _up_primary;
+  int _acting_primary;
+  ps_t pps;
+  _pg_to_osds(*pool, pg, &raw, &_up_primary, &pps);
+  _raw_to_up_osds(*pool, raw, &_up, &_up_primary);
+  _apply_primary_affinity(pps, *pool, &_up, &_up_primary);
+  _get_temp_osds(*pool, pg, &_acting, &_acting_primary);
+  if (_acting.empty()) {
+    _acting = _up;
+    if (_acting_primary == -1) {
+      _acting_primary = _up_primary;
+    }
+  }
+  if (up)
+    up->swap(_up);
+  if (up_primary)
+    *up_primary = _up_primary;
+  if (acting)
+    acting->swap(_acting);
+  if (acting_primary)
+    *acting_primary = _acting_primary;
+}
+
+int OSDMap::calc_pg_rank(int osd, const vector& acting, int nrep)
+{
+  if (!nrep)
+    nrep = acting.size();
+  for (int i=0; i& acting, int nrep)
+{
+  if (!nrep)
+    nrep = acting.size();
+  return calc_pg_rank(osd, acting, nrep);
+}
+
+bool OSDMap::primary_changed(
+  int oldprimary,
+  const vector &oldacting,
+  int newprimary,
+  const vector &newacting)
+{
+  if (oldacting.empty() && newacting.empty())
+    return false;    // both still empty
+  if (oldacting.empty() ^ newacting.empty())
+    return true;     // was empty, now not, or vice versa
+  if (oldprimary != newprimary)
+    return true;     // primary changed
+  if (calc_pg_rank(oldprimary, oldacting) !=
+      calc_pg_rank(newprimary, newacting))
+    return true;
+  return false;      // same primary (tho replicas may have changed)
+}
+
+
+// serialize, unserialize
+void OSDMap::encode_client_old(bufferlist& bl) const
+{
+  __u16 v = 5;
+  ::encode(v, bl);
+
+  // base
+  ::encode(fsid, bl);
+  ::encode(epoch, bl);
+  ::encode(created, bl);
+  ::encode(modified, bl);
+
+  // for ::encode(pools, bl);
+  __u32 n = pools.size();
+  ::encode(n, bl);
+  for (map::const_iterator p = pools.begin();
+       p != pools.end();
+       ++p) {
+    n = p->first;
+    ::encode(n, bl);
+    ::encode(p->second, bl, 0);
+  }
+  // for ::encode(pool_name, bl);
+  n = pool_name.size();
+  ::encode(n, bl);
+  for (map::const_iterator p = pool_name.begin();
+       p != pool_name.end();
+       ++p) {
+    n = p->first;
+    ::encode(n, bl);
+    ::encode(p->second, bl);
+  }
+  // for ::encode(pool_max, bl);
+  n = pool_max;
+  ::encode(n, bl);
+
+  ::encode(flags, bl);
+
+  ::encode(max_osd, bl);
+  ::encode(osd_state, bl);
+  ::encode(osd_weight, bl);
+  ::encode(osd_addrs->client_addr, bl);
+
+  // for ::encode(pg_temp, bl);
+  n = pg_temp->size();
+  ::encode(n, bl);
+  for (map >::const_iterator p = pg_temp->begin();
+       p != pg_temp->end();
+       ++p) {
+    old_pg_t opg = p->first.get_old_pg();
+    ::encode(opg, bl);
+    ::encode(p->second, bl);
+  }
+
+  // crush
+  bufferlist cbl;
+  crush->encode(cbl);
+  ::encode(cbl, bl);
+}
+
+void OSDMap::encode_classic(bufferlist& bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_PGID64) == 0) {
+    encode_client_old(bl);
+    return;
+  }
+
+  __u16 v = 6;
+  ::encode(v, bl);
+
+  // base
+  ::encode(fsid, bl);
+  ::encode(epoch, bl);
+  ::encode(created, bl);
+  ::encode(modified, bl);
+
+  ::encode(pools, bl, features);
+  ::encode(pool_name, bl);
+  ::encode(pool_max, bl);
+
+  ::encode(flags, bl);
+
+  ::encode(max_osd, bl);
+  ::encode(osd_state, bl);
+  ::encode(osd_weight, bl);
+  ::encode(osd_addrs->client_addr, bl);
+
+  ::encode(*pg_temp, bl);
+
+  // crush
+  bufferlist cbl;
+  crush->encode(cbl);
+  ::encode(cbl, bl);
+
+  // extended
+  __u16 ev = 10;
+  ::encode(ev, bl);
+  ::encode(osd_addrs->hb_back_addr, bl);
+  ::encode(osd_info, bl);
+  ::encode(blacklist, bl);
+  ::encode(osd_addrs->cluster_addr, bl);
+  ::encode(cluster_snapshot_epoch, bl);
+  ::encode(cluster_snapshot, bl);
+  ::encode(*osd_uuid, bl);
+  ::encode(osd_xinfo, bl);
+  ::encode(osd_addrs->hb_front_addr, bl);
+}
+
+void OSDMap::encode(bufferlist& bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_OSDMAP_ENC) == 0) {
+    encode_classic(bl, features);
+    return;
+  }
+  // meta-encoding: how we include client-used and osd-specific data
+  ENCODE_START(7, 7, bl);
+
+  {
+    ENCODE_START(3, 1, bl); // client-usable data
+    // base
+    ::encode(fsid, bl);
+    ::encode(epoch, bl);
+    ::encode(created, bl);
+    ::encode(modified, bl);
+
+    ::encode(pools, bl, features);
+    ::encode(pool_name, bl);
+    ::encode(pool_max, bl);
+
+    ::encode(flags, bl);
+
+    ::encode(max_osd, bl);
+    ::encode(osd_state, bl);
+    ::encode(osd_weight, bl);
+    ::encode(osd_addrs->client_addr, bl);
+
+    ::encode(*pg_temp, bl);
+    ::encode(*primary_temp, bl);
+    if (osd_primary_affinity) {
+      ::encode(*osd_primary_affinity, bl);
+    } else {
+      vector<__u32> v;
+      ::encode(v, bl);
+    }
+
+    // crush
+    bufferlist cbl;
+    crush->encode(cbl);
+    ::encode(cbl, bl);
+    ::encode(erasure_code_profiles, bl);
+    ENCODE_FINISH(bl); // client-usable data
+  }
+
+  {
+    ENCODE_START(1, 1, bl); // extended, osd-only data
+    ::encode(osd_addrs->hb_back_addr, bl);
+    ::encode(osd_info, bl);
+    {
+      // put this in a sorted, ordered map<> so that we encode in a
+      // deterministic order.
+      map blacklist_map;
+      for (ceph::unordered_map::const_iterator p =
+	     blacklist.begin(); p != blacklist.end(); ++p)
+	blacklist_map.insert(make_pair(p->first, p->second));
+      ::encode(blacklist_map, bl);
+    }
+    ::encode(osd_addrs->cluster_addr, bl);
+    ::encode(cluster_snapshot_epoch, bl);
+    ::encode(cluster_snapshot, bl);
+    ::encode(*osd_uuid, bl);
+    ::encode(osd_xinfo, bl);
+    ::encode(osd_addrs->hb_front_addr, bl);
+    ENCODE_FINISH(bl); // osd-only data
+  }
+
+  ENCODE_FINISH(bl); // meta-encoding wrapper
+}
+
+void OSDMap::decode(bufferlist& bl)
+{
+  bufferlist::iterator p = bl.begin();
+  decode(p);
+}
+
+void OSDMap::decode_classic(bufferlist::iterator& p)
+{
+  __u32 n, t;
+  __u16 v;
+  ::decode(v, p);
+
+  // base
+  ::decode(fsid, p);
+  ::decode(epoch, p);
+  ::decode(created, p);
+  ::decode(modified, p);
+
+  if (v < 6) {
+    if (v < 4) {
+      int32_t max_pools = 0;
+      ::decode(max_pools, p);
+      pool_max = max_pools;
+    }
+    pools.clear();
+    ::decode(n, p);
+    while (n--) {
+      ::decode(t, p);
+      ::decode(pools[t], p);
+    }
+    if (v == 4) {
+      ::decode(n, p);
+      pool_max = n;
+    } else if (v == 5) {
+      pool_name.clear();
+      ::decode(n, p);
+      while (n--) {
+	::decode(t, p);
+	::decode(pool_name[t], p);
+      }
+      ::decode(n, p);
+      pool_max = n;
+    }
+  } else {
+    ::decode(pools, p);
+    ::decode(pool_name, p);
+    ::decode(pool_max, p);
+  }
+  // kludge around some old bug that zeroed out pool_max (#2307)
+  if (pools.size() && pool_max < pools.rbegin()->first) {
+    pool_max = pools.rbegin()->first;
+  }
+
+  ::decode(flags, p);
+
+  ::decode(max_osd, p);
+  ::decode(osd_state, p);
+  ::decode(osd_weight, p);
+  ::decode(osd_addrs->client_addr, p);
+  if (v <= 5) {
+    pg_temp->clear();
+    ::decode(n, p);
+    while (n--) {
+      old_pg_t opg;
+      ::decode_raw(opg, p);
+      ::decode((*pg_temp)[pg_t(opg)], p);
+    }
+  } else {
+    ::decode(*pg_temp, p);
+  }
+
+  // crush
+  bufferlist cbl;
+  ::decode(cbl, p);
+  bufferlist::iterator cblp = cbl.begin();
+  crush->decode(cblp);
+
+  // extended
+  __u16 ev = 0;
+  if (v >= 5)
+    ::decode(ev, p);
+  ::decode(osd_addrs->hb_back_addr, p);
+  ::decode(osd_info, p);
+  if (v < 5)
+    ::decode(pool_name, p);
+
+  ::decode(blacklist, p);
+  if (ev >= 6)
+    ::decode(osd_addrs->cluster_addr, p);
+  else
+    osd_addrs->cluster_addr.resize(osd_addrs->client_addr.size());
+
+  if (ev >= 7) {
+    ::decode(cluster_snapshot_epoch, p);
+    ::decode(cluster_snapshot, p);
+  }
+
+  if (ev >= 8) {
+    ::decode(*osd_uuid, p);
+  } else {
+    osd_uuid->resize(max_osd);
+  }
+  if (ev >= 9)
+    ::decode(osd_xinfo, p);
+  else
+    osd_xinfo.resize(max_osd);
+
+  if (ev >= 10)
+    ::decode(osd_addrs->hb_front_addr, p);
+  else
+    osd_addrs->hb_front_addr.resize(osd_addrs->hb_back_addr.size());
+
+  osd_primary_affinity.reset();
+
+  post_decode();
+}
+
+void OSDMap::decode(bufferlist::iterator& bl)
+{
+  /**
+   * Older encodings of the OSDMap had a single struct_v which
+   * covered the whole encoding, and was prior to our modern
+   * stuff which includes a compatv and a size. So if we see
+   * a struct_v < 7, we must rewind to the beginning and use our
+   * classic decoder.
+   */
+  DECODE_START_LEGACY_COMPAT_LEN(7, 7, 7, bl); // wrapper
+  if (struct_v < 7) {
+    int struct_v_size = sizeof(struct_v);
+    bl.advance(-struct_v_size);
+    decode_classic(bl);
+    return;
+  }
+  /**
+   * Since we made it past that hurdle, we can use our normal paths.
+   */
+  {
+    DECODE_START(3, bl); // client-usable data
+    // base
+    ::decode(fsid, bl);
+    ::decode(epoch, bl);
+    ::decode(created, bl);
+    ::decode(modified, bl);
+
+    ::decode(pools, bl);
+    ::decode(pool_name, bl);
+    ::decode(pool_max, bl);
+
+    ::decode(flags, bl);
+
+    ::decode(max_osd, bl);
+    ::decode(osd_state, bl);
+    ::decode(osd_weight, bl);
+    ::decode(osd_addrs->client_addr, bl);
+
+    ::decode(*pg_temp, bl);
+    ::decode(*primary_temp, bl);
+    if (struct_v >= 2) {
+      osd_primary_affinity.reset(new vector<__u32>);
+      ::decode(*osd_primary_affinity, bl);
+      if (osd_primary_affinity->empty())
+	osd_primary_affinity.reset();
+    } else {
+      osd_primary_affinity.reset();
+    }
+
+    // crush
+    bufferlist cbl;
+    ::decode(cbl, bl);
+    bufferlist::iterator cblp = cbl.begin();
+    crush->decode(cblp);
+    if (struct_v >= 3) {
+      ::decode(erasure_code_profiles, bl);
+    } else {
+      erasure_code_profiles.clear();
+    }
+    DECODE_FINISH(bl); // client-usable data
+  }
+
+  {
+    DECODE_START(1, bl); // extended, osd-only data
+    ::decode(osd_addrs->hb_back_addr, bl);
+    ::decode(osd_info, bl);
+    ::decode(blacklist, bl);
+    ::decode(osd_addrs->cluster_addr, bl);
+    ::decode(cluster_snapshot_epoch, bl);
+    ::decode(cluster_snapshot, bl);
+    ::decode(*osd_uuid, bl);
+    ::decode(osd_xinfo, bl);
+    ::decode(osd_addrs->hb_front_addr, bl);
+    DECODE_FINISH(bl); // osd-only data
+  }
+
+  DECODE_FINISH(bl); // wrapper
+
+  post_decode();
+}
+
+void OSDMap::post_decode()
+{
+  // index pool names
+  name_pool.clear();
+  for (map::iterator i = pool_name.begin();
+       i != pool_name.end(); ++i) {
+    name_pool[i->second] = i->first;
+  }
+
+  calc_num_osds();
+}
+
+void OSDMap::dump_erasure_code_profiles(const map > &profiles,
+					Formatter *f)
+{
+  f->open_object_section("erasure_code_profiles");
+  for (map >::const_iterator i = profiles.begin();
+       i != profiles.end();
+       i++) {
+    f->open_object_section(i->first.c_str());
+    for (map::const_iterator j = i->second.begin();
+	 j != i->second.end();
+	 j++) {
+      f->dump_string(j->first.c_str(), j->second.c_str());
+    }
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void OSDMap::dump_json(ostream& out) const
+{
+  JSONFormatter jsf(true);
+  jsf.open_object_section("osdmap");
+  dump(&jsf);
+  jsf.close_section();
+  jsf.flush(out);
+}
+
+void OSDMap::dump(Formatter *f) const
+{
+  f->dump_int("epoch", get_epoch());
+  f->dump_stream("fsid") << get_fsid();
+  f->dump_stream("created") << get_created();
+  f->dump_stream("modified") << get_modified();
+  f->dump_string("flags", get_flag_string());
+  f->dump_string("cluster_snapshot", get_cluster_snapshot());
+  f->dump_int("pool_max", get_pool_max());
+  f->dump_int("max_osd", get_max_osd());
+
+  f->open_array_section("pools");
+  for (map::const_iterator p = pools.begin(); p != pools.end(); ++p) {
+    std::string name("");
+    map::const_iterator pni = pool_name.find(p->first);
+    if (pni != pool_name.end())
+      name = pni->second;
+    f->open_object_section("pool");
+    f->dump_int("pool", p->first);
+    f->dump_string("pool_name", name);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("osds");
+  for (int i=0; iopen_object_section("osd_info");
+      f->dump_int("osd", i);
+      f->dump_stream("uuid") << get_uuid(i);
+      f->dump_int("up", is_up(i));
+      f->dump_int("in", is_in(i));
+      f->dump_float("weight", get_weightf(i));
+      f->dump_float("primary_affinity", get_primary_affinityf(i));
+      get_info(i).dump(f);
+      f->dump_stream("public_addr") << get_addr(i);
+      f->dump_stream("cluster_addr") << get_cluster_addr(i);
+      f->dump_stream("heartbeat_back_addr") << get_hb_back_addr(i);
+      f->dump_stream("heartbeat_front_addr") << get_hb_front_addr(i);
+
+      set st;
+      get_state(i, st);
+      f->open_array_section("state");
+      for (set::iterator p = st.begin(); p != st.end(); ++p)
+	f->dump_string("state", *p);
+      f->close_section();
+
+      f->close_section();
+    }
+  f->close_section();
+
+  f->open_array_section("osd_xinfo");
+  for (int i=0; iopen_object_section("xinfo");
+      f->dump_int("osd", i);
+      osd_xinfo[i].dump(f);
+      f->close_section();
+    }
+  }
+  f->close_section();
+
+  f->open_array_section("pg_temp");
+  for (map >::const_iterator p = pg_temp->begin();
+       p != pg_temp->end();
+       ++p) {
+    f->open_object_section("osds");
+    f->dump_stream("pgid") << p->first;
+    f->open_array_section("osds");
+    for (vector::const_iterator q = p->second.begin(); q != p->second.end(); ++q)
+      f->dump_int("osd", *q);
+    f->close_section();
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("primary_temp");
+  for (map::const_iterator p = primary_temp->begin();
+      p != primary_temp->end();
+      ++p) {
+    f->dump_stream("pgid") << p->first;
+    f->dump_int("osd", p->second);
+  }
+  f->close_section(); // primary_temp
+
+  f->open_array_section("blacklist");
+  for (ceph::unordered_map::const_iterator p = blacklist.begin();
+       p != blacklist.end();
+       ++p) {
+    stringstream ss;
+    ss << p->first;
+    f->dump_stream(ss.str().c_str()) << p->second;
+  }
+  f->close_section();
+
+  dump_erasure_code_profiles(erasure_code_profiles, f);
+}
+
+void OSDMap::generate_test_instances(list& o)
+{
+  o.push_back(new OSDMap);
+
+  CephContext *cct = new CephContext(CODE_ENVIRONMENT_UTILITY);
+  o.push_back(new OSDMap);
+  uuid_d fsid;
+  o.back()->build_simple(cct, 1, fsid, 16, 7, 8);
+  o.back()->created = o.back()->modified = utime_t(1, 2);  // fix timestamp
+  o.back()->blacklist[entity_addr_t()] = utime_t(5, 6);
+  cct->put();
+}
+
+string OSDMap::get_flag_string(unsigned f)
+{
+  string s;
+  if ( f& CEPH_OSDMAP_NEARFULL)
+    s += ",nearfull";
+  if (f & CEPH_OSDMAP_FULL)
+    s += ",full";
+  if (f & CEPH_OSDMAP_PAUSERD)
+    s += ",pauserd";
+  if (f & CEPH_OSDMAP_PAUSEWR)
+    s += ",pausewr";
+  if (f & CEPH_OSDMAP_PAUSEREC)
+    s += ",pauserec";
+  if (f & CEPH_OSDMAP_NOUP)
+    s += ",noup";
+  if (f & CEPH_OSDMAP_NODOWN)
+    s += ",nodown";
+  if (f & CEPH_OSDMAP_NOOUT)
+    s += ",noout";
+  if (f & CEPH_OSDMAP_NOIN)
+    s += ",noin";
+  if (f & CEPH_OSDMAP_NOBACKFILL)
+    s += ",nobackfill";
+  if (f & CEPH_OSDMAP_NORECOVER)
+    s += ",norecover";
+  if (f & CEPH_OSDMAP_NOSCRUB)
+    s += ",noscrub";
+  if (f & CEPH_OSDMAP_NODEEP_SCRUB)
+    s += ",nodeep-scrub";
+  if (f & CEPH_OSDMAP_NOTIERAGENT)
+    s += ",notieragent";
+  if (s.length())
+    s = s.erase(0, 1);
+  return s;
+}
+
+string OSDMap::get_flag_string() const
+{
+  return get_flag_string(flags);
+}
+
+struct qi {
+  int item;
+  int depth;
+  float weight;
+  qi() : item(0), depth(0), weight(0) {}
+  qi(int i, int d, float w) : item(i), depth(d), weight(w) {}
+};
+
+void OSDMap::print(ostream& out) const
+{
+  out << "epoch " << get_epoch() << "\n"
+      << "fsid " << get_fsid() << "\n"
+      << "created " << get_created() << "\n"
+      << "modified " << get_modified() << "\n";
+
+  out << "flags " << get_flag_string() << "\n";
+  if (get_cluster_snapshot().length())
+    out << "cluster_snapshot " << get_cluster_snapshot() << "\n";
+  out << "\n";
+
+  for (map::const_iterator p = pools.begin(); p != pools.end(); ++p) {
+    std::string name("");
+    map::const_iterator pni = pool_name.find(p->first);
+    if (pni != pool_name.end())
+      name = pni->second;
+    out << "pool " << p->first
+	<< " '" << name
+	<< "' " << p->second << "\n";
+    for (map::const_iterator q = p->second.snaps.begin();
+	 q != p->second.snaps.end();
+	 ++q)
+      out << "\tsnap " << q->second.snapid << " '" << q->second.name << "' " << q->second.stamp << "\n";
+    if (!p->second.removed_snaps.empty())
+      out << "\tremoved_snaps " << p->second.removed_snaps << "\n";
+  }
+  out << std::endl;
+
+  out << "max_osd " << get_max_osd() << "\n";
+  for (int i=0; i st;
+      get_state(i, st);
+      out << " " << st;
+      if (!get_uuid(i).is_zero())
+	out << " " << get_uuid(i);
+      out << "\n";
+    }
+  }
+  out << std::endl;
+
+  for (map >::const_iterator p = pg_temp->begin();
+       p != pg_temp->end();
+       ++p)
+    out << "pg_temp " << p->first << " " << p->second << "\n";
+
+  for (map::const_iterator p = primary_temp->begin();
+      p != primary_temp->end();
+      ++p)
+    out << "primary_temp " << p->first << " " << p->second << "\n";
+
+  for (ceph::unordered_map::const_iterator p = blacklist.begin();
+       p != blacklist.end();
+       ++p)
+    out << "blacklist " << p->first << " expires " << p->second << "\n";
+
+  // ignore pg_swap_primary
+}
+
+void OSDMap::print_osd_line(int cur, ostream *out, Formatter *f) const
+{
+  if (f) {
+    f->dump_unsigned("id", cur);
+    f->dump_stream("name") << "osd." << cur;
+    f->dump_unsigned("exists", (int)exists(cur));
+    f->dump_string("type", crush->get_type_name(0));
+    f->dump_int("type_id", 0);
+  }
+  if (out)
+    *out << "osd." << cur << "\t";
+  if (!exists(cur)) {
+    if (out)
+      *out << "DNE\t\t";
+  } else {
+    if (is_up(cur)) {
+      if (out)
+	*out << "up\t";
+      if (f)
+	f->dump_string("status", "up");
+    } else {
+      if (out)
+	*out << "down\t";
+      if (f)
+	f->dump_string("status", "down");
+    }
+    if (out) {
+      std::streamsize p = out->precision();
+      *out << std::setprecision(4)
+	   << (exists(cur) ? get_weightf(cur) : 0)
+	   << std::setprecision(p)
+	   << "\t";
+    }
+    if (f) {
+      f->dump_float("reweight", get_weightf(cur));
+    }
+  }
+}
+
+void OSDMap::print_tree(ostream *out, Formatter *f) const
+{
+  if (out)
+    *out << "# id\tweight\ttype name\tup/down\treweight\n";
+  if (f)
+    f->open_array_section("nodes");
+  set touched;
+  set roots;
+  crush->find_roots(roots);
+  for (set::iterator p = roots.begin(); p != roots.end(); ++p) {
+    list q;
+    q.push_back(qi(*p, 0, crush->get_bucket_weight(*p) / (float)0x10000));
+    while (!q.empty()) {
+      int cur = q.front().item;
+      int depth = q.front().depth;
+      float weight = q.front().weight;
+      q.pop_front();
+
+      if (out) {
+	*out << cur << "\t";
+	int oldprecision = out->precision();
+	*out << std::setprecision(4) << weight << std::setprecision(oldprecision) << "\t";
+
+	for (int k=0; kopen_object_section("item");
+      }
+      if (cur >= 0) {
+	print_osd_line(cur, out, f);
+	if (out)
+	  *out << "\n";
+	if (f) {
+	  f->dump_float("crush_weight", weight);
+	  f->dump_unsigned("depth", depth);
+	  f->close_section();
+	}
+	touched.insert(cur);
+      }
+      if (cur >= 0) {
+	continue;
+      }
+
+      // queue bucket contents...
+      int type = crush->get_bucket_type(cur);
+      int s = crush->get_bucket_size(cur);
+      if (f) {
+	f->dump_int("id", cur);
+	f->dump_string("name", crush->get_item_name(cur));
+	f->dump_string("type", crush->get_type_name(type));
+	f->dump_int("type_id", type);
+	f->open_array_section("children");
+      }
+      for (int k=s-1; k>=0; k--) {
+	int item = crush->get_bucket_item(cur, k);
+	q.push_front(qi(item, depth+1, (float)crush->get_bucket_item_weight(cur, k) / (float)0x10000));
+	if (f)
+	  f->dump_int("child", item);
+      }
+      if (f)
+	f->close_section();
+
+      if (out)
+	*out << crush->get_type_name(type) << " " << crush->get_item_name(cur) << "\n";
+      if (f) {
+	f->close_section();
+      }
+
+    }
+  }
+  if (f) {
+    f->close_section();
+    f->open_array_section("stray");
+  }
+
+  set stray;
+  for (int i=0; iopen_object_section("osd");
+    for (set::iterator p = stray.begin(); p != stray.end(); ++p) {
+      if (out)
+	*out << *p << "\t0\t";
+      print_osd_line(*p, out, f);
+      if (out)
+	*out << "\n";
+    }
+    if (f)
+      f->close_section();
+  }
+  if (f)
+    f->close_section();
+}
+
+void OSDMap::print_summary(Formatter *f, ostream& out) const
+{
+  if (f) {
+    f->open_object_section("osdmap");
+    f->dump_int("epoch", get_epoch());
+    f->dump_int("num_osds", get_num_osds());
+    f->dump_int("num_up_osds", get_num_up_osds());
+    f->dump_int("num_in_osds", get_num_in_osds());
+    f->dump_bool("full", test_flag(CEPH_OSDMAP_FULL) ? true : false);
+    f->dump_bool("nearfull", test_flag(CEPH_OSDMAP_NEARFULL) ? true : false);
+    f->close_section();
+  } else {
+    out << "     osdmap e" << get_epoch() << ": "
+	<< get_num_osds() << " osds: "
+	<< get_num_up_osds() << " up, "
+	<< get_num_in_osds() << " in\n";
+    if (flags)
+      out << "            flags " << get_flag_string() << "\n";
+  }
+}
+
+void OSDMap::print_oneline_summary(ostream& out) const
+{
+  out << "e" << get_epoch() << ": "
+      << get_num_osds() << " osds: "
+      << get_num_up_osds() << " up, "
+      << get_num_in_osds() << " in";
+  if (test_flag(CEPH_OSDMAP_FULL))
+    out << " full";
+  else if (test_flag(CEPH_OSDMAP_NEARFULL))
+    out << " nearfull";
+}
+
+bool OSDMap::crush_ruleset_in_use(int ruleset) const
+{
+  for (map::const_iterator p = pools.begin(); p != pools.end(); ++p) {
+    if (p->second.crush_ruleset == ruleset)
+      return true;
+  }
+  return false;
+}
+
+int OSDMap::build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
+			  int nosd, int pg_bits, int pgp_bits)
+{
+  ldout(cct, 10) << "build_simple on " << num_osd
+		 << " osds with " << pg_bits << " pg bits per osd, "
+		 << dendl;
+  epoch = e;
+  set_fsid(fsid);
+  created = modified = ceph_clock_now(cct);
+
+  if (nosd >=  0) {
+    set_max_osd(nosd);
+  } else {
+    // count osds
+    int maxosd = 0, numosd = 0;
+    const md_config_t *conf = cct->_conf;
+    vector sections;
+    conf->get_all_sections(sections);
+    for (vector::iterator i = sections.begin(); i != sections.end(); ++i) {
+      if (i->find("osd.") != 0)
+	continue;
+
+      const char *begin = i->c_str() + 4;
+      char *end = (char*)begin;
+      int o = strtol(begin, &end, 10);
+      if (*end != '\0')
+	continue;
+
+      if (o > cct->_conf->mon_max_osd) {
+	lderr(cct) << "[osd." << o << "] in config has id > mon_max_osd " << cct->_conf->mon_max_osd << dendl;
+	return -ERANGE;
+      }
+      numosd++;
+      if (o > maxosd)
+	maxosd = o;
+    }
+
+    set_max_osd(maxosd + 1);
+  }
+
+  // pgp_num <= pg_num
+  if (pgp_bits > pg_bits)
+    pgp_bits = pg_bits;
+
+  vector pool_names;
+  pool_names.push_back("data");
+  pool_names.push_back("metadata");
+  pool_names.push_back("rbd");
+
+  stringstream ss;
+  int r;
+  if (nosd >= 0)
+    r = build_simple_crush_map(cct, *crush, nosd, &ss);
+  else
+    r = build_simple_crush_map_from_conf(cct, *crush, &ss);
+
+  int poolbase = get_max_osd() ? get_max_osd() : 1;
+
+  int const default_replicated_ruleset = crush->get_osd_pool_default_crush_replicated_ruleset(cct);
+  assert(default_replicated_ruleset >= 0);
+
+  for (vector::iterator p = pool_names.begin();
+       p != pool_names.end(); ++p) {
+    int64_t pool = ++pool_max;
+    pools[pool].type = pg_pool_t::TYPE_REPLICATED;
+    pools[pool].flags = cct->_conf->osd_pool_default_flags;
+    if (cct->_conf->osd_pool_default_flag_hashpspool)
+      pools[pool].flags |= pg_pool_t::FLAG_HASHPSPOOL;
+    pools[pool].size = cct->_conf->osd_pool_default_size;
+    pools[pool].min_size = cct->_conf->get_osd_pool_default_min_size();
+    pools[pool].crush_ruleset = default_replicated_ruleset;
+    pools[pool].object_hash = CEPH_STR_HASH_RJENKINS;
+    pools[pool].set_pg_num(poolbase << pg_bits);
+    pools[pool].set_pgp_num(poolbase << pgp_bits);
+    pools[pool].last_change = epoch;
+    if (*p == "data")
+      pools[pool].crash_replay_interval = cct->_conf->osd_default_data_pool_replay_window;
+    pool_name[pool] = *p;
+    name_pool[*p] = pool;
+  }
+
+  if (r < 0)
+    lderr(cct) << ss.str() << dendl;
+  
+  for (int i=0; i profile_map;
+  r = get_erasure_code_profile_default(cct, profile_map, &ss);
+  if (r < 0) {
+    lderr(cct) << ss.str() << dendl;
+    return r;
+  }
+  set_erasure_code_profile("default", profile_map);
+  return 0;
+}
+
+int OSDMap::get_erasure_code_profile_default(CephContext *cct,
+					     map &profile_map,
+					     ostream *ss)
+{
+  int r = get_str_map(cct->_conf->osd_pool_default_erasure_code_profile,
+		      *ss,
+		      &profile_map);
+  profile_map["directory"] =
+    cct->_conf->osd_pool_default_erasure_code_directory;
+  return r;
+}
+
+int OSDMap::_build_crush_types(CrushWrapper& crush)
+{
+  crush.set_type_name(0, "osd");
+  crush.set_type_name(1, "host");
+  crush.set_type_name(2, "chassis");
+  crush.set_type_name(3, "rack");
+  crush.set_type_name(4, "row");
+  crush.set_type_name(5, "pdu");
+  crush.set_type_name(6, "pod");
+  crush.set_type_name(7, "room");
+  crush.set_type_name(8, "datacenter");
+  crush.set_type_name(9, "region");
+  crush.set_type_name(10, "root");
+  return 10;
+}
+
+int OSDMap::build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
+				   int nosd, ostream *ss)
+{
+  crush.create();
+
+  // root
+  int root_type = _build_crush_types(crush);
+  int rootid;
+  int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT,
+			   root_type, 0, NULL, NULL, &rootid);
+  assert(r == 0);
+  crush.set_item_name(rootid, "default");
+
+  for (int o=0; o loc;
+    loc["host"] = "localhost";
+    loc["rack"] = "localrack";
+    loc["root"] = "default";
+    ldout(cct, 10) << " adding osd." << o << " at " << loc << dendl;
+    char name[8];
+    sprintf(name, "osd.%d", o);
+    crush.insert_item(cct, o, 1.0, name, loc);
+  }
+
+  build_simple_crush_rulesets(cct, crush, "default", ss);
+
+  crush.finalize();
+
+  return 0;
+}
+
+int OSDMap::build_simple_crush_map_from_conf(CephContext *cct,
+					     CrushWrapper& crush,
+					     ostream *ss)
+{
+  const md_config_t *conf = cct->_conf;
+
+  crush.create();
+
+  set hosts, racks;
+
+  // root
+  int root_type = _build_crush_types(crush);
+  int rootid;
+  int r = crush.add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_DEFAULT,
+			   root_type, 0, NULL, NULL, &rootid);
+  assert(r == 0);
+  crush.set_item_name(rootid, "default");
+
+  // add osds
+  vector sections;
+  conf->get_all_sections(sections);
+  for (vector::iterator i = sections.begin(); i != sections.end(); ++i) {
+    if (i->find("osd.") != 0)
+      continue;
+
+    const char *begin = i->c_str() + 4;
+    char *end = (char*)begin;
+    int o = strtol(begin, &end, 10);
+    if (*end != '\0')
+      continue;
+
+    string host, rack, row, room, dc, pool;
+    vector sections;
+    sections.push_back("osd");
+    sections.push_back(*i);
+    conf->get_val_from_conf_file(sections, "host", host, false);
+    conf->get_val_from_conf_file(sections, "rack", rack, false);
+    conf->get_val_from_conf_file(sections, "row", row, false);
+    conf->get_val_from_conf_file(sections, "room", room, false);
+    conf->get_val_from_conf_file(sections, "datacenter", dc, false);
+    conf->get_val_from_conf_file(sections, "root", pool, false);
+
+    if (host.length() == 0)
+      host = "unknownhost";
+    if (rack.length() == 0)
+      rack = "unknownrack";
+
+    hosts.insert(host);
+    racks.insert(rack);
+
+    map loc;
+    loc["host"] = host;
+    loc["rack"] = rack;
+    if (row.size())
+      loc["row"] = row;
+    if (room.size())
+      loc["room"] = room;
+    if (dc.size())
+      loc["datacenter"] = dc;
+    loc["root"] = "default";
+
+    ldout(cct, 5) << " adding osd." << o << " at " << loc << dendl;
+    crush.insert_item(cct, o, 1.0, *i, loc);
+  }
+
+  build_simple_crush_rulesets(cct, crush, "default", ss);
+
+  crush.finalize();
+
+  return 0;
+}
+
+
+int OSDMap::build_simple_crush_rulesets(CephContext *cct,
+					CrushWrapper& crush,
+					const string& root,
+					ostream *ss)
+{
+  string failure_domain =
+    crush.get_type_name(cct->_conf->osd_crush_chooseleaf_type);
+
+  int r;
+  r = crush.add_simple_ruleset("replicated_ruleset", root, failure_domain,
+			       "firstn", pg_pool_t::TYPE_REPLICATED, ss);
+  if (r < 0)
+    return r;
+  // do not add an erasure rule by default or else we will implicitly
+  // require the crush_v2 feature of clients
+  return 0;
+}
diff --git a/ceph/src/osd/OSDMap.h b/ceph/src/osd/OSDMap.h
new file mode 100644
index 00000000..a3475836
--- /dev/null
+++ b/ceph/src/osd/OSDMap.h
@@ -0,0 +1,843 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_OSDMAP_H
+#define CEPH_OSDMAP_H
+
+/*
+ * describe properties of the OSD cluster.
+ *   disks, disk groups, total # osds,
+ *
+ */
+#include "common/config.h"
+#include "include/types.h"
+#include "osd_types.h"
+#include "msg/Message.h"
+#include "common/Mutex.h"
+#include "common/Clock.h"
+
+#include "include/ceph_features.h"
+
+#include "crush/CrushWrapper.h"
+
+#include "include/interval_set.h"
+
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+using namespace std;
+
+#include "include/unordered_set.h"
+
+/*
+ * we track up to two intervals during which the osd was alive and
+ * healthy.  the most recent is [up_from,up_thru), where up_thru is
+ * the last epoch the osd is known to have _started_.  i.e., a lower
+ * bound on the actual osd death.  down_at (if it is > up_from) is an
+ * upper bound on the actual osd death.
+ *
+ * the second is the last_clean interval [first,last].  in that case,
+ * the last interval is the last epoch known to have been either
+ * _finished_, or during which the osd cleanly shut down.  when
+ * possible, we push this forward to the epoch the osd was eventually
+ * marked down.
+ *
+ * the lost_at is used to allow build_prior to proceed without waiting
+ * for an osd to recover.  In certain cases, progress may be blocked 
+ * because an osd is down that may contain updates (i.e., a pg may have
+ * gone rw during an interval).  If the osd can't be brought online, we
+ * can force things to proceed knowing that we _might_ be losing some
+ * acked writes.  If the osd comes back to life later, that's fine to,
+ * but those writes will still be lost (the divergent objects will be
+ * thrown out).
+ */
+struct osd_info_t {
+  epoch_t last_clean_begin;  // last interval that ended with a clean osd shutdown
+  epoch_t last_clean_end;
+  epoch_t up_from;   // epoch osd marked up
+  epoch_t up_thru;   // lower bound on actual osd death (if > up_from)
+  epoch_t down_at;   // upper bound on actual osd death (if > up_from)
+  epoch_t lost_at;   // last epoch we decided data was "lost"
+  
+  osd_info_t() : last_clean_begin(0), last_clean_end(0),
+		 up_from(0), up_thru(0), down_at(0), lost_at(0) {}
+
+  void dump(Formatter *f) const;
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(osd_info_t)
+
+ostream& operator<<(ostream& out, const osd_info_t& info);
+
+struct osd_xinfo_t {
+  utime_t down_stamp;      ///< timestamp when we were last marked down
+  float laggy_probability; ///< encoded as __u32: 0 = definitely not laggy, 0xffffffff definitely laggy
+  __u32 laggy_interval;    ///< average interval between being marked laggy and recovering
+  uint64_t features;       ///< features supported by this osd we should know about
+
+  osd_xinfo_t() : laggy_probability(0), laggy_interval(0),
+                  features(0) {}
+
+  void dump(Formatter *f) const;
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(osd_xinfo_t)
+
+ostream& operator<<(ostream& out, const osd_xinfo_t& xi);
+
+
+/** OSDMap
+ */
+class OSDMap {
+
+public:
+  class Incremental {
+  public:
+    /// feature bits we were encoded with.  the subsequent OSDMap
+    /// encoding should match.
+    uint64_t encode_features;
+    uuid_d fsid;
+    epoch_t epoch;   // new epoch; we are a diff from epoch-1 to epoch
+    utime_t modified;
+    int64_t new_pool_max; //incremented by the OSDMonitor on each pool create
+    int32_t new_flags;
+
+    // full (rare)
+    bufferlist fullmap;  // in leiu of below.
+    bufferlist crush;
+
+    // incremental
+    int32_t new_max_osd;
+    map new_pools;
+    map new_pool_names;
+    set old_pools;
+    map > new_erasure_code_profiles;
+    vector old_erasure_code_profiles;
+    map new_up_client;
+    map new_up_cluster;
+    map new_state;             // XORed onto previous state.
+    map new_weight;
+    map > new_pg_temp;     // [] to remove
+    map new_primary_temp;            // [-1] to remove
+    map new_primary_affinity;
+    map new_up_thru;
+    map > new_last_clean_interval;
+    map new_lost;
+    map new_uuid;
+    map new_xinfo;
+
+    map new_blacklist;
+    vector old_blacklist;
+    map new_hb_back_up;
+    map new_hb_front_up;
+
+    string cluster_snapshot;
+
+    int get_net_marked_out(const OSDMap *previous) const;
+    int get_net_marked_down(const OSDMap *previous) const;
+    int identify_osd(uuid_d u) const;
+
+    void encode_client_old(bufferlist& bl) const;
+    void encode_classic(bufferlist& bl, uint64_t features) const;
+    void encode(bufferlist& bl, uint64_t features=CEPH_FEATURES_ALL) const;
+    void decode_classic(bufferlist::iterator &p);
+    void decode(bufferlist::iterator &bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& o);
+
+    Incremental(epoch_t e=0) :
+      encode_features(0),
+      epoch(e), new_pool_max(-1), new_flags(-1), new_max_osd(-1) {
+      memset(&fsid, 0, sizeof(fsid));
+    }
+    Incremental(bufferlist &bl) {
+      bufferlist::iterator p = bl.begin();
+      decode(p);
+    }
+    Incremental(bufferlist::iterator &p) {
+      decode(p);
+    }
+
+    pg_pool_t *get_new_pool(int64_t pool, const pg_pool_t *orig) {
+      if (new_pools.count(pool) == 0)
+	new_pools[pool] = *orig;
+      return &new_pools[pool];
+    }
+    bool has_erasure_code_profile(const string &name) const {
+      map >::const_iterator i =
+	new_erasure_code_profiles.find(name);
+      return i != new_erasure_code_profiles.end();
+    }
+    void set_erasure_code_profile(const string &name,
+				  const map &profile) {
+      new_erasure_code_profiles[name] = profile;
+    }
+
+    /// propage update pools' snap metadata to any of their tiers
+    int propagate_snaps_to_tiers(CephContext *cct, const OSDMap &base);
+  };
+  
+private:
+  uuid_d fsid;
+  epoch_t epoch;        // what epoch of the osd cluster descriptor is this
+  utime_t created, modified; // epoch start time
+  int32_t pool_max;     // the largest pool num, ever
+
+  uint32_t flags;
+
+  int num_osd;         // not saved
+  int32_t max_osd;
+  vector osd_state;
+
+  struct addrs_s {
+    vector > client_addr;
+    vector > cluster_addr;
+    vector > hb_back_addr;
+    vector > hb_front_addr;
+    entity_addr_t blank;
+  };
+  ceph::shared_ptr osd_addrs;
+
+  vector<__u32>   osd_weight;   // 16.16 fixed point, 0x10000 = "in", 0 = "out"
+  vector osd_info;
+  ceph::shared_ptr< map > > pg_temp;  // temp pg mapping (e.g. while we rebuild)
+  ceph::shared_ptr< map > primary_temp;  // temp primary mapping (e.g. while we rebuild)
+  ceph::shared_ptr< vector<__u32> > osd_primary_affinity; ///< 16.16 fixed point, 0x10000 = baseline
+
+  map pools;
+  map pool_name;
+  map > erasure_code_profiles;
+  map name_pool;
+
+  ceph::shared_ptr< vector > osd_uuid;
+  vector osd_xinfo;
+
+  ceph::unordered_map blacklist;
+
+  epoch_t cluster_snapshot_epoch;
+  string cluster_snapshot;
+  bool new_blacklist_entries;
+
+ public:
+  ceph::shared_ptr crush;       // hierarchical map
+
+  friend class OSDMonitor;
+  friend class PGMonitor;
+  friend class MDS;
+
+ public:
+  OSDMap() : epoch(0), 
+	     pool_max(-1),
+	     flags(0),
+	     num_osd(0), max_osd(0),
+	     osd_addrs(new addrs_s),
+	     pg_temp(new map >),
+	     primary_temp(new map),
+	     osd_uuid(new vector),
+	     cluster_snapshot_epoch(0),
+	     new_blacklist_entries(false),
+	     crush(new CrushWrapper) {
+    memset(&fsid, 0, sizeof(fsid));
+  }
+
+  // no copying
+  /* oh, how i long for c++11...
+private:
+  OSDMap(const OSDMap& other) = default;
+  const OSDMap& operator=(const OSDMap& other) = default;
+public:
+  */
+
+  void deepish_copy_from(const OSDMap& o) {
+    *this = o;
+    primary_temp.reset(new map(*o.primary_temp));
+    pg_temp.reset(new map >(*o.pg_temp));
+    osd_uuid.reset(new vector(*o.osd_uuid));
+
+    // NOTE: this still references shared entity_addr_t's.
+    osd_addrs.reset(new addrs_s(*o.osd_addrs));
+
+    // NOTE: we do not copy crush.  note that apply_incremental will
+    // allocate a new CrushWrapper, though.
+  }
+
+  // map info
+  const uuid_d& get_fsid() const { return fsid; }
+  void set_fsid(uuid_d& f) { fsid = f; }
+
+  epoch_t get_epoch() const { return epoch; }
+  void inc_epoch() { epoch++; }
+
+  void set_epoch(epoch_t e);
+
+  /* stamps etc */
+  const utime_t& get_created() const { return created; }
+  const utime_t& get_modified() const { return modified; }
+
+  bool is_blacklisted(const entity_addr_t& a) const;
+  void get_blacklist(list > *bl) const;
+
+  string get_cluster_snapshot() const {
+    if (cluster_snapshot_epoch == epoch)
+      return cluster_snapshot;
+    return string();
+  }
+
+  /***** cluster state *****/
+  /* osds */
+  int get_max_osd() const { return max_osd; }
+  void set_max_osd(int m);
+
+  unsigned get_num_osds() const {
+    return num_osd;
+  }
+  int calc_num_osds();
+
+  void get_all_osds(set& ls) const;
+  void get_up_osds(set& ls) const;
+  unsigned get_num_up_osds() const;
+  unsigned get_num_in_osds() const;
+
+  int get_flags() const { return flags; }
+  int test_flag(int f) const { return flags & f; }
+  void set_flag(int f) { flags |= f; }
+  void clear_flag(int f) { flags &= ~f; }
+
+  static void calc_state_set(int state, set& st);
+
+  int get_state(int o) const {
+    assert(o < max_osd);
+    return osd_state[o];
+  }
+  int get_state(int o, set& st) const {
+    assert(o < max_osd);
+    unsigned t = osd_state[o];
+    calc_state_set(t, st);
+    return osd_state[o];
+  }
+  void set_state(int o, unsigned s) {
+    assert(o < max_osd);
+    osd_state[o] = s;
+  }
+  void set_weightf(int o, float w) {
+    set_weight(o, (int)((float)CEPH_OSD_IN * w));
+  }
+  void set_weight(int o, unsigned w) {
+    assert(o < max_osd);
+    osd_weight[o] = w;
+    if (w)
+      osd_state[o] |= CEPH_OSD_EXISTS;
+  }
+  unsigned get_weight(int o) const {
+    assert(o < max_osd);
+    return osd_weight[o];
+  }
+  float get_weightf(int o) const {
+    return (float)get_weight(o) / (float)CEPH_OSD_IN;
+  }
+  void adjust_osd_weights(const map& weights, Incremental& inc) const;
+
+  void set_primary_affinity(int o, int w) {
+    assert(o < max_osd);
+    if (!osd_primary_affinity)
+      osd_primary_affinity.reset(new vector<__u32>(max_osd,
+						   CEPH_OSD_DEFAULT_PRIMARY_AFFINITY));
+    (*osd_primary_affinity)[o] = w;
+  }
+  unsigned get_primary_affinity(int o) const {
+    assert(o < max_osd);
+    if (!osd_primary_affinity)
+      return CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;
+    return (*osd_primary_affinity)[o];
+  }
+  float get_primary_affinityf(int o) const {
+    return (float)get_primary_affinity(o) / (float)CEPH_OSD_MAX_PRIMARY_AFFINITY;
+  }
+
+  bool has_erasure_code_profile(const string &name) const {
+    map >::const_iterator i =
+      erasure_code_profiles.find(name);
+    return i != erasure_code_profiles.end();
+  }
+  int get_erasure_code_profile_default(CephContext *cct,
+				       map &profile_map,
+				       ostream *ss);
+  void set_erasure_code_profile(const string &name,
+				const map &profile) {
+    erasure_code_profiles[name] = profile;
+  }
+  const map &get_erasure_code_profile(const string &name) const {
+    map >::const_iterator i =
+      erasure_code_profiles.find(name);
+    static map empty;
+    if (i == erasure_code_profiles.end())
+      return empty;
+    else
+      return i->second;
+  }
+  map &get_erasure_code_profile(const string &name) {
+    return erasure_code_profiles[name];
+  }
+  const map > &get_erasure_code_profiles() const {
+    return erasure_code_profiles;
+  }
+
+  bool exists(int osd) const {
+    //assert(osd >= 0);
+    return osd >= 0 && osd < max_osd && (osd_state[osd] & CEPH_OSD_EXISTS);
+  }
+
+  bool is_up(int osd) const {
+    return exists(osd) && (osd_state[osd] & CEPH_OSD_UP);
+  }
+
+  bool is_down(int osd) const {
+    return !is_up(osd);
+  }
+
+  bool is_out(int osd) const {
+    return !exists(osd) || get_weight(osd) == CEPH_OSD_OUT;
+  }
+
+  bool is_in(int osd) const {
+    return !is_out(osd);
+  }
+
+  /**
+   * check if an entire crush subtre is down
+   */
+  bool subtree_is_down(int id, set *down_cache) const;
+  bool containing_subtree_is_down(CephContext *cct, int osd, int subtree_type, set *down_cache) const;
+  
+  int identify_osd(const entity_addr_t& addr) const;
+  int identify_osd(const uuid_d& u) const;
+
+  bool have_addr(const entity_addr_t& addr) const {
+    return identify_osd(addr) >= 0;
+  }
+  bool find_osd_on_ip(const entity_addr_t& ip) const;
+  bool have_inst(int osd) const {
+    return exists(osd) && is_up(osd); 
+  }
+  const entity_addr_t &get_addr(int osd) const {
+    assert(exists(osd));
+    return osd_addrs->client_addr[osd] ? *osd_addrs->client_addr[osd] : osd_addrs->blank;
+  }
+  const entity_addr_t &get_cluster_addr(int osd) const {
+    assert(exists(osd));
+    if (!osd_addrs->cluster_addr[osd] || *osd_addrs->cluster_addr[osd] == entity_addr_t())
+      return get_addr(osd);
+    return *osd_addrs->cluster_addr[osd];
+  }
+  const entity_addr_t &get_hb_back_addr(int osd) const {
+    assert(exists(osd));
+    return osd_addrs->hb_back_addr[osd] ? *osd_addrs->hb_back_addr[osd] : osd_addrs->blank;
+  }
+  const entity_addr_t &get_hb_front_addr(int osd) const {
+    assert(exists(osd));
+    return osd_addrs->hb_front_addr[osd] ? *osd_addrs->hb_front_addr[osd] : osd_addrs->blank;
+  }
+  entity_inst_t get_inst(int osd) const {
+    assert(is_up(osd));
+    return entity_inst_t(entity_name_t::OSD(osd), get_addr(osd));
+  }
+  entity_inst_t get_cluster_inst(int osd) const {
+    assert(is_up(osd));
+    return entity_inst_t(entity_name_t::OSD(osd), get_cluster_addr(osd));
+  }
+  entity_inst_t get_hb_back_inst(int osd) const {
+    assert(is_up(osd));
+    return entity_inst_t(entity_name_t::OSD(osd), get_hb_back_addr(osd));
+  }
+  entity_inst_t get_hb_front_inst(int osd) const {
+    assert(is_up(osd));
+    return entity_inst_t(entity_name_t::OSD(osd), get_hb_front_addr(osd));
+  }
+
+  const uuid_d& get_uuid(int osd) const {
+    assert(exists(osd));
+    return (*osd_uuid)[osd];
+  }
+
+  const epoch_t& get_up_from(int osd) const {
+    assert(exists(osd));
+    return osd_info[osd].up_from;
+  }
+  const epoch_t& get_up_thru(int osd) const {
+    assert(exists(osd));
+    return osd_info[osd].up_thru;
+  }
+  const epoch_t& get_down_at(int osd) const {
+    assert(exists(osd));
+    return osd_info[osd].down_at;
+  }
+  const osd_info_t& get_info(int osd) const {
+    assert(osd < max_osd);
+    return osd_info[osd];
+  }
+
+  const osd_xinfo_t& get_xinfo(int osd) const {
+    assert(osd < max_osd);
+    return osd_xinfo[osd];
+  }
+  
+  int get_any_up_osd() const {
+    for (int i=0; i= get_max_osd())
+	i = 0;
+      if (i == n)
+	break;
+      if (is_up(i))
+	return i;
+    }
+    return -1;
+  }
+
+  int get_previous_up_osd_before(int n) const {
+    for (int i = n - 1; i != n; --i) {
+      if (i < 0)
+	i = get_max_osd() - 1;
+      if (i == n)
+	break;
+      if (is_up(i))
+	return i;
+    }
+    return -1;
+  }
+
+  /**
+   * get feature bits required by the current structure
+   *
+   * @param entity_type [in] what entity type we are asking about
+   * @param mask [out] set of all possible map-related features we could set
+   * @return feature bits used by this map
+   */
+  uint64_t get_features(int entity_type, uint64_t *mask) const;
+
+  /**
+   * get intersection of features supported by up osds
+   */
+  uint64_t get_up_osd_features() const;
+
+  int apply_incremental(const Incremental &inc);
+
+  /// try to re-use/reference addrs in oldmap from newmap
+  static void dedup(const OSDMap *oldmap, OSDMap *newmap);
+
+  static void remove_redundant_temporaries(CephContext *cct, const OSDMap& osdmap,
+					   Incremental *pending_inc);
+  static void remove_down_temps(CephContext *cct, const OSDMap& osdmap,
+                                Incremental *pending_inc);
+
+  // serialize, unserialize
+private:
+  void encode_client_old(bufferlist& bl) const;
+  void encode_classic(bufferlist& bl, uint64_t features) const;
+  void decode_classic(bufferlist::iterator& p);
+  void post_decode();
+public:
+  void encode(bufferlist& bl, uint64_t features=CEPH_FEATURES_ALL) const;
+  void decode(bufferlist& bl);
+  void decode(bufferlist::iterator& bl);
+
+
+  /****   mapping facilities   ****/
+  int object_locator_to_pg(const object_t& oid, const object_locator_t& loc, pg_t &pg) const;
+  pg_t object_locator_to_pg(const object_t& oid, const object_locator_t& loc) const {
+    pg_t pg;
+    int ret = object_locator_to_pg(oid, loc, pg);
+    assert(ret == 0);
+    return pg;
+  }
+
+  static object_locator_t file_to_object_locator(const ceph_file_layout& layout) {
+    return object_locator_t(layout.fl_pg_pool);
+  }
+
+  // XXX: not used, mentioned in psim.cc comment
+  // oid -> pg
+  ceph_object_layout file_to_object_layout(object_t oid, ceph_file_layout& layout, string nspace) const {
+    return make_object_layout(oid, layout.fl_pg_pool, nspace);
+  }
+
+  ceph_object_layout make_object_layout(object_t oid, int pg_pool, string nspace) const;
+
+  int get_pg_num(int pg_pool) const
+  {
+    const pg_pool_t *pool = get_pg_pool(pg_pool);
+    return pool->get_pg_num();
+  }
+
+private:
+  /// pg -> (raw osd list)
+  int _pg_to_osds(const pg_pool_t& pool, pg_t pg,
+                  vector *osds, int *primary,
+		  ps_t *ppps) const;
+  void _remove_nonexistent_osds(const pg_pool_t& pool, vector& osds) const;
+
+  void _apply_primary_affinity(ps_t seed, const pg_pool_t& pool,
+			       vector *osds, int *primary) const;
+
+  /// pg -> (up osd list)
+  void _raw_to_up_osds(const pg_pool_t& pool, const vector& raw,
+                       vector *up, int *primary) const;
+
+  /**
+   * Get the pg and primary temp, if they are specified.
+   * @param temp_pg [out] Will be empty or contain the temp PG mapping on return
+   * @param temp_primary [out] Will be the value in primary_temp, or a value derived
+   * from the pg_temp (if specified), or -1 if you should use the calculated (up_)primary.
+   */
+  void _get_temp_osds(const pg_pool_t& pool, pg_t pg,
+                      vector *temp_pg, int *temp_primary) const;
+
+  /**
+   *  map to up and acting. Fills in whatever fields are non-NULL.
+   */
+  void _pg_to_up_acting_osds(pg_t pg, vector *up, int *up_primary,
+                             vector *acting, int *acting_primary) const;
+
+public:
+  /***
+   * This is suitable only for looking at raw CRUSH outputs. It skips
+   * applying the temp and up checks and should not be used
+   * by anybody for data mapping purposes.
+   * raw and primary must be non-NULL
+   */
+  int pg_to_osds(pg_t pg, vector *raw, int *primary) const;
+  /// map a pg to its acting set. @return acting set size
+  int pg_to_acting_osds(pg_t pg, vector *acting,
+                        int *acting_primary) const {
+    _pg_to_up_acting_osds(pg, NULL, NULL, acting, acting_primary);
+    return acting->size();
+  }
+  int pg_to_acting_osds(pg_t pg, vector& acting) const {
+    int primary;
+    int r = pg_to_acting_osds(pg, &acting, &primary);
+    return r;
+  }
+  /**
+   * This does not apply temp overrides and should not be used
+   * by anybody for data mapping purposes. Specify both pointers.
+   */
+  void pg_to_raw_up(pg_t pg, vector *up, int *primary) const;
+  /**
+   * map a pg to its acting set as well as its up set. You must use
+   * the acting set for data mapping purposes, but some users will
+   * also find the up set useful for things like deciding what to
+   * set as pg_temp.
+   * Each of these pointers must be non-NULL.
+   */
+  void pg_to_up_acting_osds(pg_t pg, vector *up, int *up_primary,
+                            vector *acting, int *acting_primary) const {
+    _pg_to_up_acting_osds(pg, up, up_primary, acting, acting_primary);
+  }
+  void pg_to_up_acting_osds(pg_t pg, vector& up, vector& acting) const {
+    int up_primary, acting_primary;
+    pg_to_up_acting_osds(pg, &up, &up_primary, &acting, &acting_primary);
+  }
+  bool pg_is_ec(pg_t pg) const {
+    map::const_iterator i = pools.find(pg.pool());
+    assert(i != pools.end());
+    return i->second.ec_pool();
+  }
+  bool get_primary_shard(pg_t pgid, spg_t *out) const {
+    map::const_iterator i = get_pools().find(pgid.pool());
+    if (i == get_pools().end()) {
+      return false;
+    }
+    int primary;
+    vector acting;
+    pg_to_acting_osds(pgid, &acting, &primary);
+    if (i->second.ec_pool()) {
+      for (shard_id_t i = 0; i < acting.size(); ++i) {
+	if (acting[i] == primary) {
+	  *out = spg_t(pgid, i);
+	  return true;
+	}
+      }
+    } else {
+      *out = spg_t(pgid);
+      return true;
+    }
+    return false;
+  }
+
+  int64_t lookup_pg_pool_name(const string& name) {
+    if (name_pool.count(name))
+      return name_pool[name];
+    return -ENOENT;
+  }
+
+  int64_t const_lookup_pg_pool_name(const char *name) const {
+    return const_cast(this)->lookup_pg_pool_name(name);
+  }
+
+  int64_t get_pool_max() const {
+    return pool_max;
+  }
+  const map& get_pools() const {
+    return pools;
+  }
+  const char *get_pool_name(int64_t p) const {
+    map::const_iterator i = pool_name.find(p);
+    if (i != pool_name.end())
+      return i->second.c_str();
+    return 0;
+  }
+  bool have_pg_pool(int64_t p) const {
+    return pools.count(p);
+  }
+  const pg_pool_t* get_pg_pool(int64_t p) const {
+    map::const_iterator i = pools.find(p);
+    if (i != pools.end())
+      return &i->second;
+    return NULL;
+  }
+  unsigned get_pg_size(pg_t pg) const {
+    map::const_iterator p = pools.find(pg.pool());
+    assert(p != pools.end());
+    return p->second.get_size();
+  }
+  int get_pg_type(pg_t pg) const {
+    assert(pools.count(pg.pool()));
+    return pools.find(pg.pool())->second.get_type();
+  }
+
+
+  pg_t raw_pg_to_pg(pg_t pg) const {
+    assert(pools.count(pg.pool()));
+    return pools.find(pg.pool())->second.raw_pg_to_pg(pg);
+  }
+
+  // pg -> acting primary osd
+  int get_pg_acting_primary(pg_t pg) const {
+    vector group;
+    int nrep = pg_to_acting_osds(pg, group);
+    if (nrep > 0)
+      return group[0];
+    return -1;  // we fail!
+  }
+  int get_pg_acting_tail(pg_t pg) const {
+    vector group;
+    int nrep = pg_to_acting_osds(pg, group);
+    if (nrep > 0)
+      return group[group.size()-1];
+    return -1;  // we fail!
+  }
+
+
+  /* what replica # is a given osd? 0 primary, -1 for none. */
+  static int calc_pg_rank(int osd, const vector& acting, int nrep=0);
+  static int calc_pg_role(int osd, const vector& acting, int nrep=0);
+  static bool primary_changed(
+    int oldprimary,
+    const vector &oldacting,
+    int newprimary,
+    const vector &newacting);
+  
+  /* rank is -1 (stray), 0 (primary), 1,2,3,... (replica) */
+  int get_pg_acting_rank(pg_t pg, int osd) const {
+    vector group;
+    int nrep = pg_to_acting_osds(pg, group);
+    return calc_pg_rank(osd, group, nrep);
+  }
+  /* role is -1 (stray), 0 (primary), 1 (replica) */
+  int get_pg_acting_role(pg_t pg, int osd) const {
+    vector group;
+    int nrep = pg_to_acting_osds(pg, group);
+    return calc_pg_role(osd, group, nrep);
+  }
+
+
+  /*
+   * handy helpers to build simple maps...
+   */
+  /**
+   * Build an OSD map suitable for basic usage. If **num_osd** is >= 0
+   * it will be initialized with the specified number of OSDs in a
+   * single host. If **num_osd** is < 0 the layout of the OSD map will 
+   * be built by reading the content of the configuration file.
+   *
+   * @param cct [in] in core ceph context 
+   * @param e [in] initial epoch
+   * @param fsid [in] id of the cluster
+   * @param num_osd [in] number of OSDs if >= 0 or read from conf if < 0
+   * @return **0** on success, negative errno on error.
+   */
+  int build_simple(CephContext *cct, epoch_t e, uuid_d &fsid,
+		   int num_osd, int pg_bits, int pgp_bits);
+  static int _build_crush_types(CrushWrapper& crush);
+  static int build_simple_crush_map(CephContext *cct, CrushWrapper& crush,
+				    int num_osd, ostream *ss);
+  static int build_simple_crush_map_from_conf(CephContext *cct,
+					      CrushWrapper& crush,
+					      ostream *ss);
+  static int build_simple_crush_rulesets(CephContext *cct, CrushWrapper& crush,
+					 const string& root,
+					 ostream *ss);
+
+  bool crush_ruleset_in_use(int ruleset) const;
+
+  void clear_temp() {
+    pg_temp->clear();
+    primary_temp->clear();
+  }
+
+private:
+  void print_osd_line(int cur, ostream *out, Formatter *f) const;
+public:
+  void print(ostream& out) const;
+  void print_summary(Formatter *f, ostream& out) const;
+  void print_oneline_summary(ostream& out) const;
+  void print_tree(ostream *out, Formatter *f) const;
+
+  string get_flag_string() const;
+  static string get_flag_string(unsigned flags);
+  static void dump_erasure_code_profiles(const map > &profiles,
+					 Formatter *f);
+  void dump_json(ostream& out) const;
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+  bool check_new_blacklist_entries() const { return new_blacklist_entries; }
+};
+WRITE_CLASS_ENCODER_FEATURES(OSDMap)
+WRITE_CLASS_ENCODER_FEATURES(OSDMap::Incremental)
+
+typedef ceph::shared_ptr OSDMapRef;
+
+inline ostream& operator<<(ostream& out, const OSDMap& m) {
+  m.print_oneline_summary(out);
+  return out;
+}
+
+
+#endif
diff --git a/ceph/src/osd/ObjectVersioner.h b/ceph/src/osd/ObjectVersioner.h
new file mode 100644
index 00000000..f7d75633
--- /dev/null
+++ b/ceph/src/osd/ObjectVersioner.h
@@ -0,0 +1,35 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_OSD_OBJECTVERSIONER_H
+#define CEPH_OSD_OBJECTVERSIONER_H
+
+class ObjectVersioner {
+ public:
+  pobject_t oid;
+
+  void get_versions(list& ls);
+  version_t head();      // newest
+  version_t committed(); // last committed
+  version_t tail();      // oldest
+
+  /* 
+   * prepare a new version, starting wit "raw" transaction t.
+   */
+  void prepare(ObjectStore::Transaction& t, version_t v);
+  void rollback_to(version_t v);
+  void commit_to(version_t v);
+};
+
+#endif
diff --git a/ceph/src/osd/OpRequest.cc b/ceph/src/osd/OpRequest.cc
new file mode 100644
index 00000000..bfa819dd
--- /dev/null
+++ b/ceph/src/osd/OpRequest.cc
@@ -0,0 +1,86 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+
+#include "OpRequest.h"
+#include "common/Formatter.h"
+#include 
+#include 
+#include "common/debug.h"
+#include "common/config.h"
+#include "msg/Message.h"
+#include "messages/MOSDOp.h"
+#include "messages/MOSDSubOp.h"
+#include "include/assert.h"
+#include "osd/osd_types.h"
+
+
+
+OpRequest::OpRequest(Message *req, OpTracker *tracker) :
+  TrackedOp(req, tracker),
+  rmw_flags(0),
+  hit_flag_points(0), latest_flag_point(0) {
+  if (req->get_priority() < tracker->cct->_conf->osd_client_op_priority) {
+    // don't warn as quickly for low priority ops
+    warn_interval_multiplier = tracker->cct->_conf->osd_recovery_op_warn_multiple;
+  }
+}
+
+void OpRequest::_dump(utime_t now, Formatter *f) const
+{
+  Message *m = request;
+  f->dump_string("flag_point", state_string());
+  if (m->get_orig_source().is_client()) {
+    f->open_object_section("client_info");
+    stringstream client_name;
+    client_name << m->get_orig_source();
+    f->dump_string("client", client_name.str());
+    f->dump_unsigned("tid", m->get_tid());
+    f->close_section(); // client_info
+  }
+  {
+    f->open_array_section("events");
+    for (list >::const_iterator i = events.begin();
+	 i != events.end();
+	 ++i) {
+      f->open_object_section("event");
+      f->dump_stream("time") << i->first;
+      f->dump_string("event", i->second);
+      f->close_section();
+    }
+    f->close_section();
+  }
+}
+
+void OpRequest::init_from_message()
+{
+  if (request->get_type() == CEPH_MSG_OSD_OP) {
+    reqid = static_cast(request)->get_reqid();
+  } else if (request->get_type() == MSG_OSD_SUBOP) {
+    reqid = static_cast(request)->reqid;
+  }
+}
+
+bool OpRequest::check_rmw(int flag) {
+  return rmw_flags & flag;
+}
+bool OpRequest::may_read() { return need_read_cap() || need_class_read_cap(); }
+bool OpRequest::may_write() { return need_write_cap() || need_class_write_cap(); }
+bool OpRequest::may_cache() { return check_rmw(CEPH_OSD_RMW_FLAG_CACHE); }
+bool OpRequest::includes_pg_op() { return check_rmw(CEPH_OSD_RMW_FLAG_PGOP); }
+bool OpRequest::need_read_cap() {
+  return check_rmw(CEPH_OSD_RMW_FLAG_READ);
+}
+bool OpRequest::need_write_cap() {
+  return check_rmw(CEPH_OSD_RMW_FLAG_WRITE);
+}
+bool OpRequest::need_class_read_cap() {
+  return check_rmw(CEPH_OSD_RMW_FLAG_CLASS_READ);
+}
+bool OpRequest::need_class_write_cap() {
+  return check_rmw(CEPH_OSD_RMW_FLAG_CLASS_WRITE);
+}
+void OpRequest::set_read() { rmw_flags |= CEPH_OSD_RMW_FLAG_READ; }
+void OpRequest::set_write() { rmw_flags |= CEPH_OSD_RMW_FLAG_WRITE; }
+void OpRequest::set_class_read() { rmw_flags |= CEPH_OSD_RMW_FLAG_CLASS_READ; }
+void OpRequest::set_class_write() { rmw_flags |= CEPH_OSD_RMW_FLAG_CLASS_WRITE; }
+void OpRequest::set_pg_op() { rmw_flags |= CEPH_OSD_RMW_FLAG_PGOP; }
+void OpRequest::set_cache() { rmw_flags |= CEPH_OSD_RMW_FLAG_CACHE; }
diff --git a/ceph/src/osd/OpRequest.h b/ceph/src/osd/OpRequest.h
new file mode 100644
index 00000000..b074bee0
--- /dev/null
+++ b/ceph/src/osd/OpRequest.h
@@ -0,0 +1,177 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 New Dream Network/Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef OPREQUEST_H_
+#define OPREQUEST_H_
+#include 
+#include 
+#include 
+
+#include 
+#include "common/Mutex.h"
+#include "include/xlist.h"
+#include "msg/Message.h"
+#include "include/memory.h"
+#include "common/TrackedOp.h"
+
+/**
+ * osd request identifier
+ *
+ * caller name + incarnation# + tid to unique identify this request.
+ */
+struct osd_reqid_t {
+  entity_name_t name; // who
+  ceph_tid_t         tid;
+  int32_t       inc;  // incarnation
+
+  osd_reqid_t()
+    : tid(0), inc(0) {}
+  osd_reqid_t(const entity_name_t& a, int i, ceph_tid_t t)
+    : name(a), tid(t), inc(i) {}
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(osd_reqid_t)
+
+/**
+ * The OpRequest takes in a Message* and takes over a single reference
+ * to it, which it puts() when destroyed.
+ */
+struct OpRequest : public TrackedOp {
+  friend class OpTracker;
+
+  // rmw flags
+  int rmw_flags;
+
+  bool check_rmw(int flag);
+  bool may_read();
+  bool may_write();
+  bool may_cache();
+  bool includes_pg_op();
+  bool need_read_cap();
+  bool need_write_cap();
+  bool need_class_read_cap();
+  bool need_class_write_cap();
+  void set_read();
+  void set_write();
+  void set_cache();
+  void set_class_read();
+  void set_class_write();
+  void set_pg_op();
+
+  void _dump(utime_t now, Formatter *f) const;
+
+  bool has_feature(uint64_t f) const {
+    return request->get_connection()->has_feature(f);
+  }
+
+private:
+  osd_reqid_t reqid;
+  uint8_t hit_flag_points;
+  uint8_t latest_flag_point;
+  utime_t dequeued_time;
+  static const uint8_t flag_queued_for_pg=1 << 0;
+  static const uint8_t flag_reached_pg =  1 << 1;
+  static const uint8_t flag_delayed =     1 << 2;
+  static const uint8_t flag_started =     1 << 3;
+  static const uint8_t flag_sub_op_sent = 1 << 4;
+  static const uint8_t flag_commit_sent = 1 << 5;
+
+  OpRequest(Message *req, OpTracker *tracker);
+
+public:
+  bool been_queued_for_pg() { return hit_flag_points & flag_queued_for_pg; }
+  bool been_reached_pg() { return hit_flag_points & flag_reached_pg; }
+  bool been_delayed() { return hit_flag_points & flag_delayed; }
+  bool been_started() { return hit_flag_points & flag_started; }
+  bool been_sub_op_sent() { return hit_flag_points & flag_sub_op_sent; }
+  bool been_commit_sent() { return hit_flag_points & flag_commit_sent; }
+  bool currently_queued_for_pg() { return latest_flag_point & flag_queued_for_pg; }
+  bool currently_reached_pg() { return latest_flag_point & flag_reached_pg; }
+  bool currently_delayed() { return latest_flag_point & flag_delayed; }
+  bool currently_started() { return latest_flag_point & flag_started; }
+  bool currently_sub_op_sent() { return latest_flag_point & flag_sub_op_sent; }
+  bool currently_commit_sent() { return latest_flag_point & flag_commit_sent; }
+
+  const char *state_string() const {
+    switch(latest_flag_point) {
+    case flag_queued_for_pg: return "queued for pg";
+    case flag_reached_pg: return "reached pg";
+    case flag_delayed: return "delayed";
+    case flag_started: return "started";
+    case flag_sub_op_sent: return "waiting for sub ops";
+    case flag_commit_sent: return "commit sent; apply or cleanup";
+    default: break;
+    }
+    return "no flag points reached";
+  }
+
+  void mark_queued_for_pg() {
+    mark_event("queued_for_pg");
+    current = "queued for pg";
+    hit_flag_points |= flag_queued_for_pg;
+    latest_flag_point = flag_queued_for_pg;
+  }
+  void mark_reached_pg() {
+    mark_event("reached_pg");
+    current = "reached pg";
+    hit_flag_points |= flag_reached_pg;
+    latest_flag_point = flag_reached_pg;
+  }
+  void mark_delayed(string s) {
+    mark_event(s);
+    current = s;
+    hit_flag_points |= flag_delayed;
+    latest_flag_point = flag_delayed;
+  }
+  void mark_started() {
+    mark_event("started");
+    current = "started";
+    hit_flag_points |= flag_started;
+    latest_flag_point = flag_started;
+  }
+  void mark_sub_op_sent(string s) {
+    mark_event(s);
+    current = s;
+    hit_flag_points |= flag_sub_op_sent;
+    latest_flag_point = flag_sub_op_sent;
+  }
+  void mark_commit_sent() {
+    mark_event("commit_sent");
+    current = "commit sent";
+    hit_flag_points |= flag_commit_sent;
+    latest_flag_point = flag_commit_sent;
+  }
+
+  utime_t get_dequeued_time() const {
+    return dequeued_time;
+  }
+  void set_dequeued_time(utime_t deq_time) {
+    dequeued_time = deq_time;
+  }
+
+  osd_reqid_t get_reqid() const {
+    return reqid;
+  }
+
+  void init_from_message();
+
+  typedef ceph::shared_ptr Ref;
+};
+
+typedef OpRequest::Ref OpRequestRef;
+
+#endif /* OPREQUEST_H_ */
diff --git a/ceph/src/osd/PG.cc b/ceph/src/osd/PG.cc
new file mode 100644
index 00000000..9356df49
--- /dev/null
+++ b/ceph/src/osd/PG.cc
@@ -0,0 +1,7697 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "PG.h"
+#include "common/errno.h"
+#include "common/config.h"
+#include "OSD.h"
+#include "OpRequest.h"
+
+#include "common/Timer.h"
+
+#include "messages/MOSDOp.h"
+#include "messages/MOSDPGNotify.h"
+#include "messages/MOSDPGLog.h"
+#include "messages/MOSDPGRemove.h"
+#include "messages/MOSDPGInfo.h"
+#include "messages/MOSDPGTrim.h"
+#include "messages/MOSDPGScan.h"
+#include "messages/MOSDPGBackfill.h"
+#include "messages/MBackfillReserve.h"
+#include "messages/MRecoveryReserve.h"
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPushReply.h"
+#include "messages/MOSDPGPull.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
+
+#include "messages/MOSDSubOp.h"
+#include "messages/MOSDSubOpReply.h"
+#include "common/BackTrace.h"
+
+#include 
+
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+template 
+static ostream& _prefix(std::ostream *_dout, T *t)
+{
+  return *_dout << t->gen_prefix();
+}
+
+void PG::get(const string &tag) 
+{
+  ref.inc();
+#ifdef PG_DEBUG_REFS
+  Mutex::Locker l(_ref_id_lock);
+  if (!_tag_counts.count(tag)) {
+    _tag_counts[tag] = 0;
+  }
+  _tag_counts[tag]++;
+#endif
+}
+
+void PG::put(const string &tag)
+{
+#ifdef PG_DEBUG_REFS
+  {
+    Mutex::Locker l(_ref_id_lock);
+    assert(_tag_counts.count(tag));
+    _tag_counts[tag]--;
+    if (_tag_counts[tag] == 0) {
+      _tag_counts.erase(tag);
+    }
+  }
+#endif
+  if (ref.dec() == 0)
+    delete this;
+}
+
+#ifdef PG_DEBUG_REFS
+uint64_t PG::get_with_id()
+{
+  ref.inc();
+  Mutex::Locker l(_ref_id_lock);
+  uint64_t id = ++_ref_id;
+  BackTrace bt(0);
+  stringstream ss;
+  bt.print(ss);
+  dout(20) << __func__ << ": " << info.pgid << " got id " << id << dendl;
+  assert(!_live_ids.count(id));
+  _live_ids.insert(make_pair(id, ss.str()));
+  return id;
+}
+
+void PG::put_with_id(uint64_t id)
+{
+  dout(20) << __func__ << ": " << info.pgid << " put id " << id << dendl;
+  {
+    Mutex::Locker l(_ref_id_lock);
+    assert(_live_ids.count(id));
+    _live_ids.erase(id);
+  }
+  if (ref.dec() == 0)
+    delete this;
+}
+
+void PG::dump_live_ids()
+{
+  Mutex::Locker l(_ref_id_lock);
+  dout(0) << "\t" << __func__ << ": " << info.pgid << " live ids:" << dendl;
+  for (map::iterator i = _live_ids.begin();
+       i != _live_ids.end();
+       ++i) {
+    dout(0) << "\t\tid: " << *i << dendl;
+  }
+  dout(0) << "\t" << __func__ << ": " << info.pgid << " live tags:" << dendl;
+  for (map::iterator i = _tag_counts.begin();
+       i != _tag_counts.end();
+       ++i) {
+    dout(0) << "\t\tid: " << *i << dendl;
+  }
+}
+#endif
+
+void PGPool::update(OSDMapRef map)
+{
+  const pg_pool_t *pi = map->get_pg_pool(id);
+  assert(pi);
+  info = *pi;
+  auid = pi->auid;
+  name = map->get_pool_name(id);
+  if (pi->get_snap_epoch() == map->get_epoch()) {
+    pi->build_removed_snaps(newly_removed_snaps);
+    newly_removed_snaps.subtract(cached_removed_snaps);
+    cached_removed_snaps.union_of(newly_removed_snaps);
+    snapc = pi->get_snap_context();
+  } else {
+    newly_removed_snaps.clear();
+  }
+  lgeneric_subdout(g_ceph_context, osd, 20)
+    << "PGPool::update cached_removed_snaps "
+    << cached_removed_snaps
+    << " newly_removed_snaps "
+    << newly_removed_snaps
+    << " snapc " << snapc
+    << (pi->get_snap_epoch() == map->get_epoch() ?
+	" (updated)":" (no change)")
+    << dendl;
+}
+
+PG::PG(OSDService *o, OSDMapRef curmap,
+       const PGPool &_pool, spg_t p, const hobject_t& loid,
+       const hobject_t& ioid) :
+  osd(o),
+  cct(o->cct),
+  osdriver(osd->store, coll_t(), OSD::make_snapmapper_oid()),
+  snap_mapper(
+    &osdriver,
+    p.ps(),
+    p.get_split_bits(curmap->get_pg_num(_pool.id)),
+    _pool.id,
+    p.shard),
+  map_lock("PG::map_lock"),
+  osdmap_ref(curmap), last_persisted_osdmap_ref(curmap), pool(_pool),
+  _lock("PG::_lock"),
+  ref(0),
+  #ifdef PG_DEBUG_REFS
+  _ref_id_lock("PG::_ref_id_lock"), _ref_id(0),
+  #endif
+  deleting(false), dirty_info(false), dirty_big_info(false),
+  info(p),
+  info_struct_v(0),
+  coll(p), pg_log(cct), log_oid(loid), biginfo_oid(ioid),
+  missing_loc(this),
+  recovery_item(this), scrub_item(this), scrub_finalize_item(this), snap_trim_item(this), stat_queue_item(this),
+  recovery_ops_active(0),
+  role(0),
+  state(0),
+  send_notify(false),
+  pg_whoami(osd->whoami, p.shard),
+  need_up_thru(false),
+  last_peering_reset(0),
+  heartbeat_peer_lock("PG::heartbeat_peer_lock"),
+  backfill_reserved(0),
+  backfill_reserving(0),
+  flushes_in_progress(0),
+  pg_stats_publish_lock("PG::pg_stats_publish_lock"),
+  pg_stats_publish_valid(false),
+  osr(osd->osr_registry.lookup_or_create(p, (stringify(p)))),
+  finish_sync_event(NULL),
+  scrub_after_recovery(false),
+  active_pushes(0),
+  recovery_state(this)
+{
+#ifdef PG_DEBUG_REFS
+  osd->add_pgid(p, this);
+#endif
+}
+
+PG::~PG()
+{
+#ifdef PG_DEBUG_REFS
+  osd->remove_pgid(info.pgid, this);
+#endif
+}
+
+void PG::lock_suspend_timeout(ThreadPool::TPHandle &handle)
+{
+  handle.suspend_tp_timeout();
+  lock();
+  handle.reset_tp_timeout();
+}
+
+void PG::lock(bool no_lockdep)
+{
+  _lock.Lock(no_lockdep);
+  // if we have unrecorded dirty state with the lock dropped, there is a bug
+  assert(!dirty_info);
+  assert(!dirty_big_info);
+
+  dout(30) << "lock" << dendl;
+}
+
+std::string PG::gen_prefix() const
+{
+  stringstream out;
+  OSDMapRef mapref = osdmap_ref;
+  if (_lock.is_locked_by_me()) {
+    out << "osd." << osd->whoami
+	<< " pg_epoch: " << (mapref ? mapref->get_epoch():0)
+	<< " " << *this << " ";
+  } else {
+    out << "osd." << osd->whoami
+	<< " pg_epoch: " << (mapref ? mapref->get_epoch():0)
+	<< " pg[" << info.pgid << "(unlocked)] ";
+  }
+  return out.str();
+}
+  
+/********* PG **********/
+
+void PG::proc_master_log(
+  ObjectStore::Transaction& t, pg_info_t &oinfo,
+  pg_log_t &olog, pg_missing_t& omissing, pg_shard_t from)
+{
+  dout(10) << "proc_master_log for osd." << from << ": "
+	   << olog << " " << omissing << dendl;
+  assert(!is_active() && is_primary());
+
+  // merge log into our own log to build master log.  no need to
+  // make any adjustments to their missing map; we are taking their
+  // log to be authoritative (i.e., their entries are by definitely
+  // non-divergent).
+  merge_log(t, oinfo, olog, from);
+  peer_info[from] = oinfo;
+  dout(10) << " peer osd." << from << " now " << oinfo << " " << omissing << dendl;
+  might_have_unfound.insert(from);
+
+  peer_missing[from].swap(omissing);
+}
+    
+void PG::proc_replica_log(
+  ObjectStore::Transaction& t,
+  pg_info_t &oinfo, pg_log_t &olog, pg_missing_t& omissing,
+  pg_shard_t from)
+{
+  dout(10) << "proc_replica_log for osd." << from << ": "
+	   << oinfo << " " << olog << " " << omissing << dendl;
+
+  pg_log.proc_replica_log(t, oinfo, olog, omissing, from);
+
+  peer_info[from] = oinfo;
+  dout(10) << " peer osd." << from << " now " << oinfo << " " << omissing << dendl;
+  might_have_unfound.insert(from);
+
+  for (map::iterator i = omissing.missing.begin();
+       i != omissing.missing.end();
+       ++i) {
+    dout(20) << " after missing " << i->first << " need " << i->second.need
+	     << " have " << i->second.have << dendl;
+  }
+  peer_missing[from].swap(omissing);
+}
+
+bool PG::proc_replica_info(pg_shard_t from, const pg_info_t &oinfo)
+{
+  map::iterator p = peer_info.find(from);
+  if (p != peer_info.end() && p->second.last_update == oinfo.last_update) {
+    dout(10) << " got dup osd." << from << " info " << oinfo << ", identical to ours" << dendl;
+    return false;
+  }
+
+  dout(10) << " got osd." << from << " " << oinfo << dendl;
+  assert(is_primary());
+  peer_info[from] = oinfo;
+  might_have_unfound.insert(from);
+  
+  unreg_next_scrub();
+  if (info.history.merge(oinfo.history))
+    dirty_info = true;
+  reg_next_scrub();
+  
+  // stray?
+  if (!is_up(from) && !is_acting(from)) {
+    dout(10) << " osd." << from << " has stray content: " << oinfo << dendl;
+    stray_set.insert(from);
+    if (is_clean()) {
+      purge_strays();
+    }
+  }
+
+  // was this a new info?  if so, update peers!
+  if (p == peer_info.end())
+    update_heartbeat_peers();
+
+  return true;
+}
+
+void PG::remove_snap_mapped_object(
+  ObjectStore::Transaction &t, const hobject_t &soid)
+{
+  t.remove(
+    coll,
+    ghobject_t(soid, ghobject_t::NO_GEN, pg_whoami.shard));
+  clear_object_snap_mapping(&t, soid);
+}
+
+void PG::clear_object_snap_mapping(
+  ObjectStore::Transaction *t, const hobject_t &soid)
+{
+  OSDriver::OSTransaction _t(osdriver.get_transaction(t));
+  if (soid.snap < CEPH_MAXSNAP) {
+    int r = snap_mapper.remove_oid(
+      soid,
+      &_t);
+    if (!(r == 0 || r == -ENOENT)) {
+      derr << __func__ << ": remove_oid returned " << cpp_strerror(r) << dendl;
+      assert(0);
+    }
+  }
+}
+
+void PG::update_object_snap_mapping(
+  ObjectStore::Transaction *t, const hobject_t &soid, const set &snaps)
+{
+  OSDriver::OSTransaction _t(osdriver.get_transaction(t));
+  assert(soid.snap < CEPH_MAXSNAP);
+  int r = snap_mapper.remove_oid(
+    soid,
+    &_t);
+  if (!(r == 0 || r == -ENOENT)) {
+    derr << __func__ << ": remove_oid returned " << cpp_strerror(r) << dendl;
+    assert(0);
+  }
+  snap_mapper.add_oid(
+    soid,
+    snaps,
+    &_t);
+}
+
+void PG::merge_log(
+  ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog, pg_shard_t from)
+{
+  PGLogEntryHandler rollbacker;
+  pg_log.merge_log(
+    t, oinfo, olog, from, info, &rollbacker, dirty_info, dirty_big_info);
+  rollbacker.apply(this, &t);
+}
+
+void PG::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead)
+{
+  PGLogEntryHandler rollbacker;
+  pg_log.rewind_divergent_log(
+    t, newhead, info, &rollbacker, dirty_info, dirty_big_info);
+  rollbacker.apply(this, &t);
+}
+
+/*
+ * Process information from a replica to determine if it could have any
+ * objects that i need.
+ *
+ * TODO: if the missing set becomes very large, this could get expensive.
+ * Instead, we probably want to just iterate over our unfound set.
+ */
+bool PG::search_for_missing(
+  const pg_info_t &oinfo, const pg_missing_t &omissing,
+  pg_shard_t from,
+  RecoveryCtx *ctx)
+{
+  unsigned num_unfound_before = missing_loc.num_unfound();
+  bool found_missing = missing_loc.add_source_info(
+    from, oinfo, omissing);
+  if (found_missing && num_unfound_before != missing_loc.num_unfound())
+    publish_stats_to_osd();
+  if (found_missing &&
+      (get_osdmap()->get_features(CEPH_ENTITY_TYPE_OSD, NULL) &
+       CEPH_FEATURE_OSD_ERASURE_CODES)) {
+    pg_info_t tinfo(oinfo);
+    tinfo.pgid.shard = pg_whoami.shard;
+    (*(ctx->info_map))[from.osd].push_back(
+      make_pair(
+	pg_notify_t(
+	  from.shard, pg_whoami.shard,
+	  get_osdmap()->get_epoch(),
+	  get_osdmap()->get_epoch(),
+	  tinfo),
+	past_intervals));
+  }
+  return found_missing;
+}
+
+bool PG::MissingLoc::readable_with_acting(
+  const hobject_t &hoid,
+  const set &acting) const {
+  if (!needs_recovery(hoid)) return true;
+  if (!missing_loc.count(hoid)) return false;
+  const set &locs = missing_loc.find(hoid)->second;
+  dout(10) << __func__ << ": locs:" << locs << dendl;
+  set have_acting;
+  for (set::const_iterator i = locs.begin();
+       i != locs.end();
+       ++i) {
+    if (acting.count(*i))
+      have_acting.insert(*i);
+  }
+  return (*is_readable)(have_acting);
+}
+
+bool PG::MissingLoc::add_source_info(
+  pg_shard_t fromosd,
+  const pg_info_t &oinfo,
+  const pg_missing_t &omissing)
+{
+  bool found_missing = false;;
+  // found items?
+  for (map::const_iterator p = needs_recovery_map.begin();
+       p != needs_recovery_map.end();
+       ++p) {
+    const hobject_t &soid(p->first);
+    eversion_t need = p->second.need;
+    if (oinfo.last_update < need) {
+      dout(10) << "search_for_missing " << soid << " " << need
+	       << " also missing on osd." << fromosd
+	       << " (last_update " << oinfo.last_update << " < needed " << need << ")"
+	       << dendl;
+      continue;
+    }
+    if (p->first >= oinfo.last_backfill) {
+      // FIXME: this is _probably_ true, although it could conceivably
+      // be in the undefined region!  Hmm!
+      dout(10) << "search_for_missing " << soid << " " << need
+	       << " also missing on osd." << fromosd
+	       << " (past last_backfill " << oinfo.last_backfill << ")"
+	       << dendl;
+      continue;
+    }
+    if (oinfo.last_complete < need) {
+      if (omissing.is_missing(soid)) {
+	dout(10) << "search_for_missing " << soid << " " << need
+		 << " also missing on osd." << fromosd << dendl;
+	continue;
+      }
+    }
+
+    dout(10) << "search_for_missing " << soid << " " << need
+	     << " is on osd." << fromosd << dendl;
+
+    missing_loc[soid].insert(fromosd);
+    missing_loc_sources.insert(fromosd);
+    found_missing = true;
+  }
+
+  dout(20) << "needs_recovery_map missing " << needs_recovery_map << dendl;
+  return found_missing;
+}
+
+void PG::discover_all_missing(map > &query_map)
+{
+  const pg_missing_t &missing = pg_log.get_missing();
+  assert(have_unfound());
+
+  dout(10) << __func__ << " "
+	   << missing.num_missing() << " missing, "
+	   << get_num_unfound() << " unfound"
+	   << dendl;
+
+  std::set::const_iterator m = might_have_unfound.begin();
+  std::set::const_iterator mend = might_have_unfound.end();
+  for (; m != mend; ++m) {
+    pg_shard_t peer(*m);
+    
+    if (!get_osdmap()->is_up(peer.osd)) {
+      dout(20) << __func__ << " skipping down osd." << peer << dendl;
+      continue;
+    }
+
+    map::const_iterator iter = peer_info.find(peer);
+    if (iter != peer_info.end() &&
+        (iter->second.is_empty() || iter->second.dne())) {
+      // ignore empty peers
+      continue;
+    }
+
+    // If we've requested any of this stuff, the pg_missing_t information
+    // should be on its way.
+    // TODO: coalsce requested_* into a single data structure
+    if (peer_missing.find(peer) != peer_missing.end()) {
+      dout(20) << __func__ << ": osd." << peer
+	       << ": we already have pg_missing_t" << dendl;
+      continue;
+    }
+    if (peer_log_requested.find(peer) != peer_log_requested.end()) {
+      dout(20) << __func__ << ": osd." << peer
+	       << ": in peer_log_requested" << dendl;
+      continue;
+    }
+    if (peer_missing_requested.find(peer) != peer_missing_requested.end()) {
+      dout(20) << __func__ << ": osd." << peer
+	       << ": in peer_missing_requested" << dendl;
+      continue;
+    }
+
+    // Request missing
+    dout(10) << __func__ << ": osd." << peer << ": requesting pg_missing_t"
+	     << dendl;
+    peer_missing_requested.insert(peer);
+    query_map[peer.osd][spg_t(info.pgid.pgid, peer.shard)] =
+      pg_query_t(
+	pg_query_t::FULLLOG,
+	peer.shard, pg_whoami.shard,
+	info.history, get_osdmap()->get_epoch());
+  }
+}
+
+/******* PG ***********/
+bool PG::needs_recovery() const
+{
+  assert(is_primary());
+
+  bool ret = false;
+
+  const pg_missing_t &missing = pg_log.get_missing();
+
+  if (missing.num_missing()) {
+    dout(10) << __func__ << " primary has " << missing.num_missing()
+      << " missing" << dendl;
+
+    ret = true;
+  }
+
+  assert(!actingbackfill.empty());
+  set::const_iterator end = actingbackfill.end();
+  set::const_iterator a = actingbackfill.begin();
+  assert(a != end);
+  for (; a != end; ++a) {
+    if (*a == get_primary()) continue;
+    pg_shard_t peer = *a;
+    map::const_iterator pm = peer_missing.find(peer);
+    if (pm == peer_missing.end()) {
+      dout(10) << __func__ << " osd." << peer << " doesn't have missing set"
+        << dendl;
+      ret = true;
+      continue;
+    }
+    if (pm->second.num_missing()) {
+      dout(10) << __func__ << " osd." << peer << " has "
+        << pm->second.num_missing() << " missing" << dendl;
+      ret = true;
+    }
+  }
+
+  if (!ret)
+    dout(10) << __func__ << " is recovered" << dendl;
+  return ret;
+}
+
+bool PG::needs_backfill() const
+{
+  assert(is_primary());
+
+  bool ret = false;
+
+  // We can assume that only possible osds that need backfill
+  // are on the backfill_targets vector nodes.
+  set::const_iterator end = backfill_targets.end();
+  set::const_iterator a = backfill_targets.begin();
+  for (; a != end; ++a) {
+    pg_shard_t peer = *a;
+    map::const_iterator pi = peer_info.find(peer);
+    if (!pi->second.last_backfill.is_max()) {
+      dout(10) << __func__ << " osd." << peer << " has last_backfill " << pi->second.last_backfill << dendl;
+      ret = true;
+    }
+  }
+
+  if (!ret)
+    dout(10) << __func__ << " does not need backfill" << dendl;
+  return ret;
+}
+
+bool PG::_calc_past_interval_range(epoch_t *start, epoch_t *end)
+{
+  *end = info.history.same_interval_since;
+
+  // Do we already have the intervals we want?
+  map::const_iterator pif = past_intervals.begin();
+  if (pif != past_intervals.end()) {
+    if (pif->first <= info.history.last_epoch_clean) {
+      dout(10) << __func__ << ": already have past intervals back to "
+	       << info.history.last_epoch_clean << dendl;
+      return false;
+    }
+    *end = past_intervals.begin()->first;
+  }
+
+  *start = MAX(MAX(info.history.epoch_created,
+		   info.history.last_epoch_clean),
+	       osd->get_superblock().oldest_map);
+  if (*start >= *end) {
+    dout(10) << __func__ << " start epoch " << *start << " >= end epoch " << *end
+	     << ", nothing to do" << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+
+void PG::generate_past_intervals()
+{
+  epoch_t cur_epoch, end_epoch;
+  if (!_calc_past_interval_range(&cur_epoch, &end_epoch)) {
+    return;
+  }
+
+  OSDMapRef last_map, cur_map;
+  int primary = -1;
+  int up_primary = -1;
+  vector acting, up, old_acting, old_up;
+
+  cur_map = osd->get_map(cur_epoch);
+  cur_map->pg_to_up_acting_osds(
+    get_pgid().pgid, &up, &up_primary, &acting, &primary);
+  epoch_t same_interval_since = cur_epoch;
+  dout(10) << __func__ << " over epochs " << cur_epoch << "-"
+	   << end_epoch << dendl;
+  ++cur_epoch;
+  for (; cur_epoch <= end_epoch; ++cur_epoch) {
+    int old_primary = primary;
+    int old_up_primary = up_primary;
+    last_map.swap(cur_map);
+    old_up.swap(up);
+    old_acting.swap(acting);
+
+    cur_map = osd->get_map(cur_epoch);
+    pg_t pgid = get_pgid().pgid;
+    if (cur_map->get_pools().count(pgid.pool()))
+      pgid = pgid.get_ancestor(cur_map->get_pg_num(pgid.pool()));
+    cur_map->pg_to_up_acting_osds(pgid, &up, &up_primary, &acting, &primary);
+
+    std::stringstream debug;
+    bool new_interval = pg_interval_t::check_new_interval(
+      old_primary,
+      primary,
+      old_acting,
+      acting,
+      old_up_primary,
+      up_primary,
+      old_up,
+      up,
+      same_interval_since,
+      info.history.last_epoch_clean,
+      cur_map,
+      last_map,
+      pgid.pool(),
+      pgid,
+      &past_intervals,
+      &debug);
+    if (new_interval) {
+      dout(10) << debug.str() << dendl;
+      same_interval_since = cur_epoch;
+    }
+  }
+
+  // record our work.
+  dirty_info = true;
+  dirty_big_info = true;
+}
+
+/*
+ * Trim past_intervals.
+ *
+ * This gets rid of all the past_intervals that happened before last_epoch_clean.
+ */
+void PG::trim_past_intervals()
+{
+  std::map::iterator pif = past_intervals.begin();
+  std::map::iterator end = past_intervals.end();
+  while (pif != end) {
+    if (pif->second.last >= info.history.last_epoch_clean)
+      return;
+    dout(10) << __func__ << ": trimming " << pif->second << dendl;
+    past_intervals.erase(pif++);
+    dirty_big_info = true;
+  }
+}
+
+
+bool PG::adjust_need_up_thru(const OSDMapRef osdmap)
+{
+  epoch_t up_thru = get_osdmap()->get_up_thru(osd->whoami);
+  if (need_up_thru &&
+      up_thru >= info.history.same_interval_since) {
+    dout(10) << "adjust_need_up_thru now " << up_thru << ", need_up_thru now false" << dendl;
+    need_up_thru = false;
+    return true;
+  }
+  return false;
+}
+
+void PG::remove_down_peer_info(const OSDMapRef osdmap)
+{
+  // Remove any downed osds from peer_info
+  bool removed = false;
+  map::iterator p = peer_info.begin();
+  while (p != peer_info.end()) {
+    if (!osdmap->is_up(p->first.osd)) {
+      dout(10) << " dropping down osd." << p->first << " info " << p->second << dendl;
+      peer_missing.erase(p->first);
+      peer_log_requested.erase(p->first);
+      peer_missing_requested.erase(p->first);
+      peer_info.erase(p++);
+      removed = true;
+    } else
+      ++p;
+  }
+
+  // if we removed anyone, update peers (which include peer_info)
+  if (removed)
+    update_heartbeat_peers();
+  check_recovery_sources(osdmap);
+}
+
+/*
+ * Returns true unless there is a non-lost OSD in might_have_unfound.
+ */
+bool PG::all_unfound_are_queried_or_lost(const OSDMapRef osdmap) const
+{
+  assert(is_primary());
+
+  set::const_iterator peer = might_have_unfound.begin();
+  set::const_iterator mend = might_have_unfound.end();
+  for (; peer != mend; ++peer) {
+    if (peer_missing.count(*peer))
+      continue;
+    map::const_iterator iter = peer_info.find(*peer);
+    if (iter != peer_info.end() &&
+        (iter->second.is_empty() || iter->second.dne()))
+      continue;
+    const osd_info_t &osd_info(osdmap->get_info(peer->osd));
+    if (osd_info.lost_at <= osd_info.up_from) {
+      // If there is even one OSD in might_have_unfound that isn't lost, we
+      // still might retrieve our unfound.
+      return false;
+    }
+  }
+  dout(10) << "all_unfound_are_queried_or_lost all of might_have_unfound " << might_have_unfound 
+	   << " have been queried or are marked lost" << dendl;
+  return true;
+}
+
+void PG::build_prior(std::auto_ptr &prior_set)
+{
+  if (1) {
+    // sanity check
+    for (map::iterator it = peer_info.begin();
+	 it != peer_info.end();
+	 ++it) {
+      assert(info.history.last_epoch_started >= it->second.history.last_epoch_started);
+    }
+  }
+  prior_set.reset(
+    new PriorSet(
+      pool.info.ec_pool(),
+      get_pgbackend()->get_is_recoverable_predicate(),
+      *get_osdmap(),
+      past_intervals,
+      up,
+      acting,
+      info,
+      this));
+  PriorSet &prior(*prior_set.get());
+				 
+  if (prior.pg_down) {
+    state_set(PG_STATE_DOWN);
+  }
+
+  if (get_osdmap()->get_up_thru(osd->whoami) < info.history.same_interval_since) {
+    dout(10) << "up_thru " << get_osdmap()->get_up_thru(osd->whoami)
+	     << " < same_since " << info.history.same_interval_since
+	     << ", must notify monitor" << dendl;
+    need_up_thru = true;
+  } else {
+    dout(10) << "up_thru " << get_osdmap()->get_up_thru(osd->whoami)
+	     << " >= same_since " << info.history.same_interval_since
+	     << ", all is well" << dendl;
+    need_up_thru = false;
+  }
+  set_probe_targets(prior_set->probe);
+}
+
+void PG::clear_primary_state()
+{
+  dout(10) << "clear_primary_state" << dendl;
+
+  // clear peering state
+  stray_set.clear();
+  peer_log_requested.clear();
+  peer_missing_requested.clear();
+  peer_info.clear();
+  peer_missing.clear();
+  need_up_thru = false;
+  peer_last_complete_ondisk.clear();
+  peer_activated.clear();
+  min_last_complete_ondisk = eversion_t();
+  pg_trim_to = eversion_t();
+  stray_purged.clear();
+  might_have_unfound.clear();
+
+  last_update_ondisk = eversion_t();
+
+  snap_trimq.clear();
+
+  finish_sync_event = 0;  // so that _finish_recvoery doesn't go off in another thread
+
+  missing_loc.clear();
+
+  pg_log.reset_recovery_pointers();
+
+  scrubber.reserved_peers.clear();
+  scrub_after_recovery = false;
+
+  osd->recovery_wq.dequeue(this);
+  osd->snap_trim_wq.dequeue(this);
+
+  agent_clear();
+
+  osd->remove_want_pg_temp(info.pgid.pgid);
+}
+
+/**
+ * find_best_info
+ *
+ * Returns an iterator to the best info in infos sorted by:
+ *  1) Prefer newer last_update
+ *  2) Prefer longer tail if it brings another info into contiguity
+ *  3) Prefer current primary
+ */
+map::const_iterator PG::find_best_info(
+  const map &infos) const
+{
+  eversion_t min_last_update_acceptable = eversion_t::max();
+  epoch_t max_last_epoch_started_found = 0;
+  for (map::const_iterator i = infos.begin();
+       i != infos.end();
+       ++i) {
+    if (max_last_epoch_started_found < i->second.history.last_epoch_started) {
+      min_last_update_acceptable = eversion_t::max();
+      max_last_epoch_started_found = i->second.history.last_epoch_started;
+    }
+    if (max_last_epoch_started_found < i->second.last_epoch_started) {
+      min_last_update_acceptable = eversion_t::max();
+      max_last_epoch_started_found = i->second.last_epoch_started;
+    }
+    if (max_last_epoch_started_found == i->second.last_epoch_started) {
+      if (min_last_update_acceptable > i->second.last_update)
+	min_last_update_acceptable = i->second.last_update;
+    }
+  }
+  if (min_last_update_acceptable == eversion_t::max())
+    return infos.end();
+
+  map::const_iterator best = infos.end();
+  // find osd with newest last_update (oldest for ec_pool).
+  // if there are multiples, prefer
+  //  - a longer tail, if it brings another peer into log contiguity
+  //  - the current primary
+  for (map::const_iterator p = infos.begin();
+       p != infos.end();
+       ++p) {
+    // Only consider peers with last_update >= min_last_update_acceptable
+    if (p->second.last_update < min_last_update_acceptable)
+      continue;
+    // Disquality anyone who is incomplete (not fully backfilled)
+    if (p->second.is_incomplete())
+      continue;
+    if (best == infos.end()) {
+      best = p;
+      continue;
+    }
+    // Prefer newer last_update
+    if (pool.info.require_rollback()) {
+      if (p->second.last_update > best->second.last_update)
+	continue;
+      if (p->second.last_update < best->second.last_update) {
+	best = p;
+	continue;
+      }
+    } else {
+      if (p->second.last_update < best->second.last_update)
+	continue;
+      if (p->second.last_update > best->second.last_update) {
+	best = p;
+	continue;
+      }
+    }
+
+    // Prefer longer tail
+    if (p->second.log_tail > best->second.log_tail) {
+      continue;
+    } else if (p->second.log_tail < best->second.log_tail) {
+      best = p;
+      continue;
+    }
+
+    // prefer current primary (usually the caller), all things being equal
+    if (p->first == pg_whoami) {
+      dout(10) << "calc_acting prefer osd." << p->first
+	       << " because it is current primary" << dendl;
+      best = p;
+      continue;
+    }
+  }
+  return best;
+}
+
+void PG::calc_ec_acting(
+  map::const_iterator auth_log_shard,
+  unsigned size,
+  const vector &acting,
+  pg_shard_t acting_primary,
+  const vector &up,
+  pg_shard_t up_primary,
+  const map &all_info,
+  bool compat_mode,
+  vector *_want,
+  set *backfill,
+  set *acting_backfill,
+  pg_shard_t *want_primary,
+  ostream &ss) {
+  vector want(size, CRUSH_ITEM_NONE);
+  map > all_info_by_shard;
+  unsigned usable = 0;
+  for(map::const_iterator i = all_info.begin();
+      i != all_info.end();
+      ++i) {
+    all_info_by_shard[i->first.shard].insert(i->first);
+  }
+  for (shard_id_t i = 0; i < want.size(); ++i) {
+    ss << "For position " << (unsigned)i << ": ";
+    if (up.size() > (unsigned)i && up[i] != CRUSH_ITEM_NONE &&
+	!all_info.find(pg_shard_t(up[i], i))->second.is_incomplete() &&
+	all_info.find(pg_shard_t(up[i], i))->second.last_update >=
+	auth_log_shard->second.log_tail) {
+      ss << " selecting up[i]: " << pg_shard_t(up[i], i) << std::endl;
+      want[i] = up[i];
+      ++usable;
+      continue;
+    }
+    if (up.size() > (unsigned)i && up[i] != CRUSH_ITEM_NONE) {
+      ss << " backfilling up[i]: " << pg_shard_t(up[i], i)
+	 << " and ";
+      backfill->insert(pg_shard_t(up[i], i));
+    }
+
+    if (acting.size() > (unsigned)i && acting[i] != CRUSH_ITEM_NONE &&
+	!all_info.find(pg_shard_t(acting[i], i))->second.is_incomplete() &&
+	all_info.find(pg_shard_t(acting[i], i))->second.last_update >=
+	auth_log_shard->second.log_tail) {
+      ss << " selecting acting[i]: " << pg_shard_t(acting[i], i) << std::endl;
+      want[i] = acting[i];
+      ++usable;
+    } else {
+      for (set::iterator j = all_info_by_shard[i].begin();
+	   j != all_info_by_shard[i].end();
+	   ++j) {
+	assert(j->shard == i);
+	if (!all_info.find(*j)->second.is_incomplete() &&
+	    all_info.find(*j)->second.last_update >=
+	    auth_log_shard->second.log_tail) {
+	  ss << " selecting stray: " << *j << std::endl;
+	  want[i] = j->osd;
+	  ++usable;
+	  break;
+	}
+      }
+      if (want[i] == CRUSH_ITEM_NONE)
+	ss << " failed to fill position " << i << std::endl;
+    }
+  }
+
+  bool found_primary = false;
+  for (shard_id_t i = 0; i < want.size(); ++i) {
+    if (want[i] != CRUSH_ITEM_NONE) {
+      acting_backfill->insert(pg_shard_t(want[i], i));
+      if (!found_primary) {
+	*want_primary = pg_shard_t(want[i], i);
+	found_primary = true;
+      }
+    }
+  }
+  acting_backfill->insert(backfill->begin(), backfill->end());
+  _want->swap(want);
+}
+
+/**
+ * calculate the desired acting set.
+ *
+ * Choose an appropriate acting set.  Prefer up[0], unless it is
+ * incomplete, or another osd has a longer tail that allows us to
+ * bring other up nodes up to date.
+ */
+void PG::calc_replicated_acting(
+  map::const_iterator auth_log_shard,
+  unsigned size,
+  const vector &acting,
+  pg_shard_t acting_primary,
+  const vector &up,
+  pg_shard_t up_primary,
+  const map &all_info,
+  bool compat_mode,
+  vector *want,
+  set *backfill,
+  set *acting_backfill,
+  pg_shard_t *want_primary,
+  ostream &ss)
+{
+  ss << "calc_acting newest update on osd." << auth_log_shard->first
+     << " with " << auth_log_shard->second << std::endl;
+  pg_shard_t auth_log_shard_id = auth_log_shard->first;
+  
+  // select primary
+  map::const_iterator primary;
+  if (up.size() &&
+      !all_info.find(up_primary)->second.is_incomplete() &&
+      all_info.find(up_primary)->second.last_update >=
+        auth_log_shard->second.log_tail) {
+    ss << "up_primary: " << up_primary << ") selected as primary" << std::endl;
+    primary = all_info.find(up_primary); // prefer up[0], all thing being equal
+  } else {
+    assert(!auth_log_shard->second.is_incomplete());
+    ss << "up[0] needs backfill, osd." << auth_log_shard_id
+       << " selected as primary instead" << std::endl;
+    primary = auth_log_shard;
+  }
+
+  ss << "calc_acting primary is osd." << primary->first
+     << " with " << primary->second << std::endl;
+  *want_primary = primary->first;
+  want->push_back(primary->first.osd);
+  acting_backfill->insert(primary->first);
+  unsigned usable = 1;
+
+  // select replicas that have log contiguity with primary.
+  // prefer up, then acting, then any peer_info osds 
+  for (vector::const_iterator i = up.begin();
+       i != up.end();
+       ++i) {
+    pg_shard_t up_cand = pg_shard_t(*i, ghobject_t::no_shard());
+    if (up_cand == primary->first)
+      continue;
+    const pg_info_t &cur_info = all_info.find(up_cand)->second;
+    if (cur_info.is_incomplete() ||
+      cur_info.last_update < MIN(
+	primary->second.log_tail,
+	auth_log_shard->second.log_tail)) {
+      /* We include auth_log_shard->second.log_tail because in GetLog,
+       * we will request logs back to the min last_update over our
+       * acting_backfill set, which will result in our log being extended
+       * as far backwards as necessary to pick up any peers which can
+       * be log recovered by auth_log_shard's log */
+      ss << " shard " << up_cand << " (up) backfill " << cur_info << std::endl;
+      if (compat_mode) {
+	if (backfill->empty()) {
+	  backfill->insert(up_cand);
+	  want->push_back(*i);
+	  acting_backfill->insert(up_cand);
+	}
+      } else {
+	backfill->insert(up_cand);
+	acting_backfill->insert(up_cand);
+      }
+    } else {
+      want->push_back(*i);
+      acting_backfill->insert(up_cand);
+      usable++;
+      ss << " osd." << *i << " (up) accepted " << cur_info << std::endl;
+    }
+  }
+
+  // This no longer has backfill OSDs, but they are covered above.
+  for (vector::const_iterator i = acting.begin();
+       i != acting.end();
+       ++i) {
+    pg_shard_t acting_cand(*i, ghobject_t::no_shard());
+    if (usable >= size)
+      break;
+
+    // skip up osds we already considered above
+    if (acting_cand == primary->first)
+      continue;
+    vector::const_iterator up_it = find(up.begin(), up.end(), acting_cand.osd);
+    if (up_it != up.end())
+      continue;
+
+    const pg_info_t &cur_info = all_info.find(acting_cand)->second;
+    if (cur_info.is_incomplete() ||
+	cur_info.last_update < primary->second.log_tail) {
+      ss << " shard " << acting_cand << " (stray) REJECTED "
+	       << cur_info << std::endl;
+    } else {
+      want->push_back(*i);
+      acting_backfill->insert(acting_cand);
+      ss << " shard " << acting_cand << " (stray) accepted "
+	 << cur_info << std::endl;
+      usable++;
+    }
+  }
+
+  for (map::const_iterator i = all_info.begin();
+       i != all_info.end();
+       ++i) {
+    if (usable >= size)
+      break;
+
+    // skip up osds we already considered above
+    if (i->first == primary->first)
+      continue;
+    vector::const_iterator up_it = find(up.begin(), up.end(), i->first.osd);
+    if (up_it != up.end())
+      continue;
+    vector::const_iterator acting_it = find(
+      acting.begin(), acting.end(), i->first.osd);
+    if (acting_it != acting.end())
+      continue;
+
+    if (i->second.is_incomplete() ||
+	i->second.last_update < primary->second.log_tail) {
+      ss << " shard " << i->first << " (stray) REJECTED "
+	 << i->second << std::endl;
+    } else {
+      want->push_back(i->first.osd);
+      acting_backfill->insert(i->first);
+      ss << " shard " << i->first << " (stray) accepted "
+	 << i->second << std::endl;
+      usable++;
+    }
+  }
+}
+
+/**
+ * choose acting
+ *
+ * calculate the desired acting, and request a change with the monitor
+ * if it differs from the current acting.
+ */
+bool PG::choose_acting(pg_shard_t &auth_log_shard_id)
+{
+  map all_info(peer_info.begin(), peer_info.end());
+  all_info[pg_whoami] = info;
+
+  for (map::iterator p = all_info.begin();
+       p != all_info.end();
+       ++p) {
+    dout(10) << "calc_acting osd." << p->first << " " << p->second << dendl;
+  }
+
+  map::const_iterator auth_log_shard =
+    find_best_info(all_info);
+
+  if (auth_log_shard == all_info.end()) {
+    if (up != acting) {
+      dout(10) << "choose_acting no suitable info found (incomplete backfills?),"
+	       << " reverting to up" << dendl;
+      want_acting = up;
+      vector empty;
+      osd->queue_want_pg_temp(info.pgid.pgid, empty);
+    } else {
+      dout(10) << "choose_acting failed" << dendl;
+      assert(want_acting.empty());
+    }
+    return false;
+  }
+
+  if ((up.size() &&
+      !all_info.find(up_primary)->second.is_incomplete() &&
+      all_info.find(up_primary)->second.last_update >=
+       auth_log_shard->second.log_tail) &&
+      auth_log_shard->second.is_incomplete()) {
+    map complete_infos;
+    for (map::const_iterator i = all_info.begin();
+	 i != all_info.end();
+	 ++i) {
+      if (!i->second.is_incomplete())
+	complete_infos.insert(*i);
+    }
+    map::const_iterator i = find_best_info(
+      complete_infos);
+    if (i != complete_infos.end()) {
+      auth_log_shard = all_info.find(i->first);
+    }
+  }
+
+  auth_log_shard_id = auth_log_shard->first;
+
+  // Determine if compatibility needed
+  bool compat_mode = !cct->_conf->osd_debug_override_acting_compat;
+  if (compat_mode) {
+    bool all_support = true;
+    OSDMapRef osdmap = get_osdmap();
+
+    for (map::iterator it = all_info.begin();
+	 it != all_info.end();
+	 ++it) {
+      pg_shard_t peer = it->first;
+
+      const osd_xinfo_t& xi = osdmap->get_xinfo(peer.osd);
+      if (!(xi.features & CEPH_FEATURE_OSD_ERASURE_CODES)) {
+	all_support = false;
+	break;
+      }
+    }
+    if (all_support)
+      compat_mode = false;
+  }
+
+  set want_backfill, want_acting_backfill;
+  vector want;
+  pg_shard_t want_primary;
+  stringstream ss;
+  if (!pool.info.ec_pool())
+    calc_replicated_acting(
+      auth_log_shard,
+      get_osdmap()->get_pg_size(info.pgid.pgid),
+      acting,
+      primary,
+      up,
+      up_primary,
+      all_info,
+      compat_mode,
+      &want,
+      &want_backfill,
+      &want_acting_backfill,
+      &want_primary,
+      ss);
+  else
+    calc_ec_acting(
+      auth_log_shard,
+      get_osdmap()->get_pg_size(info.pgid.pgid),
+      acting,
+      primary,
+      up,
+      up_primary,
+      all_info,
+      compat_mode,
+      &want,
+      &want_backfill,
+      &want_acting_backfill,
+      &want_primary,
+      ss);
+  dout(10) << ss.str() << dendl;
+
+  unsigned num_want_acting = 0;
+  for (vector::iterator i = want.begin();
+       i != want.end();
+       ++i) {
+    if (*i != CRUSH_ITEM_NONE)
+      ++num_want_acting;
+  }
+
+  // This is a bit of a problem, if we allow the pg to go active with
+  // want.size() < min_size, we won't consider the pg to have been
+  // maybe_went_rw in build_prior.
+  if (num_want_acting < pool.info.min_size) {
+    want_acting.clear();
+    return false;
+  }
+
+  /* Check whether we have enough acting shards to later perform recovery */
+  boost::scoped_ptr recoverable_predicate(
+    get_pgbackend()->get_is_recoverable_predicate());
+  set have;
+  for (int i = 0; i < (int)want.size(); ++i) {
+    if (want[i] != CRUSH_ITEM_NONE)
+      have.insert(
+	pg_shard_t(
+	  want[i],
+	  pool.info.ec_pool() ? i : ghobject_t::NO_SHARD));
+  }
+  if (!(*recoverable_predicate)(have)) {
+    want_acting.clear();
+    return false;
+  }
+
+  if (want != acting) {
+    dout(10) << "choose_acting want " << want << " != acting " << acting
+	     << ", requesting pg_temp change" << dendl;
+    want_acting = want;
+
+    if (want_acting == up) {
+      // There can't be any pending backfill if
+      // want is the same as crush map up OSDs.
+      assert(compat_mode || want_backfill.empty());
+      vector empty;
+      osd->queue_want_pg_temp(info.pgid.pgid, empty);
+    } else
+      osd->queue_want_pg_temp(info.pgid.pgid, want);
+    return false;
+  }
+  want_acting.clear();
+  actingbackfill = want_acting_backfill;
+  dout(10) << "actingbackfill is " << actingbackfill << dendl;
+  assert(backfill_targets.empty() || backfill_targets == want_backfill);
+  if (backfill_targets.empty()) {
+    // Caller is GetInfo
+    backfill_targets = want_backfill;
+    for (set::iterator i = backfill_targets.begin();
+	 i != backfill_targets.end();
+	 ++i) {
+      assert(!stray_set.count(*i));
+    }
+  } else {
+    // Will not change if already set because up would have had to change
+    assert(backfill_targets == want_backfill);
+    // Verify that nothing in backfill is in stray_set
+    for (set::iterator i = want_backfill.begin();
+	 i != want_backfill.end();
+	 ++i) {
+      assert(stray_set.find(*i) == stray_set.end());
+    }
+  }
+  dout(10) << "choose_acting want " << want << " (== acting) backfill_targets " 
+	   << want_backfill << dendl;
+  return true;
+}
+
+/* Build the might_have_unfound set.
+ *
+ * This is used by the primary OSD during recovery.
+ *
+ * This set tracks the OSDs which might have unfound objects that the primary
+ * OSD needs. As we receive pg_missing_t from each OSD in might_have_unfound, we
+ * will remove the OSD from the set.
+ */
+void PG::build_might_have_unfound()
+{
+  assert(might_have_unfound.empty());
+  assert(is_primary());
+
+  dout(10) << __func__ << dendl;
+
+  // Make sure that we have past intervals.
+  generate_past_intervals();
+
+  // We need to decide who might have unfound objects that we need
+  std::map::const_reverse_iterator p = past_intervals.rbegin();
+  std::map::const_reverse_iterator end = past_intervals.rend();
+  for (; p != end; ++p) {
+    const pg_interval_t &interval(p->second);
+    // We already have all the objects that exist at last_epoch_clean,
+    // so there's no need to look at earlier intervals.
+    if (interval.last < info.history.last_epoch_clean)
+      break;
+
+    // If nothing changed, we don't care about this interval.
+    if (!interval.maybe_went_rw)
+      continue;
+
+    int i = 0;
+    std::vector::const_iterator a = interval.acting.begin();
+    std::vector::const_iterator a_end = interval.acting.end();
+    for (; a != a_end; ++a, ++i) {
+      pg_shard_t shard(*a, pool.info.ec_pool() ? i : ghobject_t::NO_SHARD);
+      if (*a != CRUSH_ITEM_NONE && shard != pg_whoami)
+	might_have_unfound.insert(shard);
+    }
+  }
+
+  // include any (stray) peers
+  for (map::iterator p = peer_info.begin();
+       p != peer_info.end();
+       ++p)
+    might_have_unfound.insert(p->first);
+
+  dout(15) << __func__ << ": built " << might_have_unfound << dendl;
+}
+
+struct C_PG_ActivateCommitted : public Context {
+  PGRef pg;
+  epoch_t epoch;
+  C_PG_ActivateCommitted(PG *p, epoch_t e)
+    : pg(p), epoch(e) {}
+  void finish(int r) {
+    pg->_activate_committed(epoch);
+  }
+};
+
+void PG::activate(ObjectStore::Transaction& t,
+		  epoch_t query_epoch,
+		  list& tfin,
+		  map >& query_map,
+		  map > > *activator_map,
+                  RecoveryCtx *ctx)
+{
+  assert(!is_active());
+  assert(scrubber.callbacks.empty());
+  assert(callbacks_for_degraded_object.empty());
+
+  // -- crash recovery?
+  if (is_primary() &&
+      pool.info.crash_replay_interval > 0 &&
+      may_need_replay(get_osdmap())) {
+    replay_until = ceph_clock_now(cct);
+    replay_until += pool.info.crash_replay_interval;
+    dout(10) << "activate starting replay interval for " << pool.info.crash_replay_interval
+	     << " until " << replay_until << dendl;
+    state_set(PG_STATE_REPLAY);
+
+    // TODOSAM: osd->osd-> is no good
+    osd->osd->replay_queue_lock.Lock();
+    osd->osd->replay_queue.push_back(pair(
+	info.pgid, replay_until));
+    osd->osd->replay_queue_lock.Unlock();
+  }
+
+  // twiddle pg state
+  state_clear(PG_STATE_DOWN);
+
+  send_notify = false;
+
+  if (is_acting(pg_whoami))
+    info.last_epoch_started = query_epoch;
+
+  const pg_missing_t &missing = pg_log.get_missing();
+
+  if (is_primary()) {
+    last_update_ondisk = info.last_update;
+    min_last_complete_ondisk = eversion_t(0,0);  // we don't know (yet)!
+  }
+  last_update_applied = info.last_update;
+  last_rollback_info_trimmed_to_applied = pg_log.get_rollback_trimmed_to();
+
+  need_up_thru = false;
+
+  // write pg info, log
+  dirty_info = true;
+  dirty_big_info = true; // maybe
+
+  // find out when we commit
+  t.register_on_complete(new C_PG_ActivateCommitted(this, query_epoch));
+  
+  // initialize snap_trimq
+  if (is_primary()) {
+    dout(20) << "activate - purged_snaps " << info.purged_snaps
+	     << " cached_removed_snaps " << pool.cached_removed_snaps << dendl;
+    snap_trimq = pool.cached_removed_snaps;
+    snap_trimq.subtract(info.purged_snaps);
+    dout(10) << "activate - snap_trimq " << snap_trimq << dendl;
+    if (!snap_trimq.empty() && is_clean())
+      queue_snap_trim();
+  }
+
+  // init complete pointer
+  if (missing.num_missing() == 0) {
+    dout(10) << "activate - no missing, moving last_complete " << info.last_complete 
+	     << " -> " << info.last_update << dendl;
+    info.last_complete = info.last_update;
+    pg_log.reset_recovery_pointers();
+  } else {
+    dout(10) << "activate - not complete, " << missing << dendl;
+    pg_log.activate_not_complete(info);
+  }
+    
+  log_weirdness();
+
+  // if primary..
+  if (is_primary()) {
+    assert(ctx);
+    // start up replicas
+
+    assert(!actingbackfill.empty());
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == pg_whoami) continue;
+      pg_shard_t peer = *i;
+      assert(peer_info.count(peer));
+      pg_info_t& pi = peer_info[peer];
+
+      dout(10) << "activate peer osd." << peer << " " << pi << dendl;
+
+      MOSDPGLog *m = 0;
+      pg_missing_t& pm = peer_missing[peer];
+
+      bool needs_past_intervals = pi.dne();
+
+      if (pi.last_update == info.last_update) {
+        // empty log
+	if (!pi.is_empty() && activator_map) {
+	  dout(10) << "activate peer osd." << peer << " is up to date, queueing in pending_activators" << dendl;
+	  (*activator_map)[peer.osd].push_back(
+	    make_pair(
+	      pg_notify_t(
+		peer.shard, pg_whoami.shard,
+		get_osdmap()->get_epoch(),
+		get_osdmap()->get_epoch(),
+		info),
+	      past_intervals));
+	} else {
+	  dout(10) << "activate peer osd." << peer << " is up to date, but sending pg_log anyway" << dendl;
+	  m = new MOSDPGLog(
+	    i->shard, pg_whoami.shard,
+	    get_osdmap()->get_epoch(), info);
+	}
+      } else if (
+	pg_log.get_tail() > pi.last_update ||
+	pi.last_backfill == hobject_t() ||
+	(backfill_targets.count(*i) && pi.last_backfill.is_max())) {
+	/* This last case covers a situation where a replica is not contiguous
+	 * with the auth_log, but is contiguous with this replica.  Reshuffling
+	 * the active set to handle this would be tricky, so instead we just go
+	 * ahead and backfill it anyway.  This is probably preferrable in any
+	 * case since the replica in question would have to be significantly
+	 * behind.
+	 */
+	// backfill
+	osd->clog.info() << info.pgid << " restarting backfill on osd." << peer
+			 << " from (" << pi.log_tail << "," << pi.last_update << "] " << pi.last_backfill
+			 << " to " << info.last_update;
+
+	pi.last_update = info.last_update;
+	pi.last_complete = info.last_update;
+	pi.last_backfill = hobject_t();
+	pi.history = info.history;
+	pi.hit_set = info.hit_set;
+	pi.stats.stats.clear();
+
+	m = new MOSDPGLog(
+	  i->shard, pg_whoami.shard,
+	  get_osdmap()->get_epoch(), pi);
+
+	// send some recent log, so that op dup detection works well.
+	m->log.copy_up_to(pg_log.get_log(), cct->_conf->osd_min_pg_log_entries);
+	m->info.log_tail = m->log.tail;
+	pi.log_tail = m->log.tail;  // sigh...
+
+	pm.clear();
+      } else {
+	// catch up
+	assert(pg_log.get_tail() <= pi.last_update);
+	m = new MOSDPGLog(
+	  i->shard, pg_whoami.shard,
+	  get_osdmap()->get_epoch(), info);
+	// send new stuff to append to replicas log
+	m->log.copy_after(pg_log.get_log(), pi.last_update);
+      }
+
+      // share past_intervals if we are creating the pg on the replica
+      // based on whether our info for that peer was dne() *before*
+      // updating pi.history in the backfill block above.
+      if (needs_past_intervals)
+	m->past_intervals = past_intervals;
+
+      // update local version of peer's missing list!
+      if (m && pi.last_backfill != hobject_t()) {
+        for (list::iterator p = m->log.log.begin();
+             p != m->log.log.end();
+             ++p)
+	  if (p->soid <= pi.last_backfill)
+	    pm.add_next_event(*p);
+      }
+      
+      if (m) {
+	dout(10) << "activate peer osd." << peer << " sending " << m->log << dendl;
+	//m->log.print(cout);
+	osd->send_message_osd_cluster(peer.osd, m, get_osdmap()->get_epoch());
+      }
+
+      // peer now has 
+      pi.last_update = info.last_update;
+
+      // update our missing
+      if (pm.num_missing() == 0) {
+	pi.last_complete = pi.last_update;
+        dout(10) << "activate peer osd." << peer << " " << pi << " uptodate" << dendl;
+      } else {
+        dout(10) << "activate peer osd." << peer << " " << pi << " missing " << pm << dendl;
+      }
+    }
+
+    // Set up missing_loc
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == get_primary()) {
+	missing_loc.add_active_missing(pg_log.get_missing());
+      } else {
+	assert(peer_missing.count(*i));
+	missing_loc.add_active_missing(peer_missing[*i]);
+      }
+    }
+    // If necessary, create might_have_unfound to help us find our unfound objects.
+    // NOTE: It's important that we build might_have_unfound before trimming the
+    // past intervals.
+    might_have_unfound.clear();
+    if (needs_recovery()) {
+      missing_loc.add_source_info(pg_whoami, info, pg_log.get_missing());
+      for (set::iterator i = actingbackfill.begin();
+	   i != actingbackfill.end();
+	   ++i) {
+	if (*i == pg_whoami) continue;
+	dout(10) << __func__ << ": adding " << *i << " as a source" << dendl;
+	assert(peer_missing.count(*i));
+	assert(peer_info.count(*i));
+	missing_loc.add_source_info(
+	  *i,
+	  peer_info[*i],
+	  peer_missing[*i]);
+      }
+      for (map::iterator i = peer_missing.begin();
+	   i != peer_missing.end();
+	   ++i) {
+	if (is_actingbackfill(i->first))
+	  continue;
+	assert(peer_info.count(i->first));
+	search_for_missing(
+	  peer_info[i->first],
+	  i->second,
+	  i->first,
+	  ctx);
+      }
+
+      build_might_have_unfound();
+
+      dout(10) << "activate - starting recovery" << dendl;
+      osd->queue_for_recovery(this);
+      if (have_unfound())
+	discover_all_missing(query_map);
+    }
+
+    // degraded?
+    if (get_osdmap()->get_pg_size(info.pgid.pgid) > acting.size())
+      state_set(PG_STATE_DEGRADED);
+
+  }
+}
+
+bool PG::op_has_sufficient_caps(OpRequestRef op)
+{
+  // only check MOSDOp
+  if (op->get_req()->get_type() != CEPH_MSG_OSD_OP)
+    return true;
+
+  MOSDOp *req = static_cast(op->get_req());
+
+  OSD::Session *session = (OSD::Session *)req->get_connection()->get_priv();
+  if (!session) {
+    dout(0) << "op_has_sufficient_caps: no session for op " << *req << dendl;
+    return false;
+  }
+  OSDCap& caps = session->caps;
+  session->put();
+
+  string key = req->get_object_locator().key;
+  if (key.length() == 0)
+    key = req->get_oid().name;
+
+  bool cap = caps.is_capable(pool.name, req->get_object_locator().nspace,
+                             pool.auid, key,
+			     op->need_read_cap(),
+			     op->need_write_cap(),
+			     op->need_class_read_cap(),
+			     op->need_class_write_cap());
+
+  dout(20) << "op_has_sufficient_caps pool=" << pool.id << " (" << pool.name
+		   << " " << req->get_object_locator().nspace
+	   << ") owner=" << pool.auid
+	   << " need_read_cap=" << op->need_read_cap()
+	   << " need_write_cap=" << op->need_write_cap()
+	   << " need_class_read_cap=" << op->need_class_read_cap()
+	   << " need_class_write_cap=" << op->need_class_write_cap()
+	   << " -> " << (cap ? "yes" : "NO")
+	   << dendl;
+  return cap;
+}
+
+void PG::take_op_map_waiters()
+{
+  Mutex::Locker l(map_lock);
+  for (list::iterator i = waiting_for_map.begin();
+       i != waiting_for_map.end();
+       ) {
+    if (op_must_wait_for_map(get_osdmap_with_maplock(), *i)) {
+      break;
+    } else {
+      osd->op_wq.queue(make_pair(PGRef(this), *i));
+      waiting_for_map.erase(i++);
+    }
+  }
+}
+
+void PG::queue_op(OpRequestRef op)
+{
+  Mutex::Locker l(map_lock);
+  if (!waiting_for_map.empty()) {
+    // preserve ordering
+    waiting_for_map.push_back(op);
+    return;
+  }
+  if (op_must_wait_for_map(get_osdmap_with_maplock(), op)) {
+    waiting_for_map.push_back(op);
+    return;
+  }
+  osd->op_wq.queue(make_pair(PGRef(this), op));
+}
+
+void PG::replay_queued_ops()
+{
+  assert(is_replay());
+  eversion_t c = info.last_update;
+  list replay;
+  dout(10) << "replay_queued_ops" << dendl;
+  state_clear(PG_STATE_REPLAY);
+
+  for (map::iterator p = replay_queue.begin();
+       p != replay_queue.end();
+       ++p) {
+    if (p->first.version != c.version+1) {
+      dout(10) << "activate replay " << p->first
+	       << " skipping " << c.version+1 - p->first.version 
+	       << " ops"
+	       << dendl;      
+      c = p->first;
+    }
+    dout(10) << "activate replay " << p->first << " "
+             << *p->second->get_req() << dendl;
+    replay.push_back(p->second);
+  }
+  replay_queue.clear();
+  requeue_ops(replay);
+  requeue_ops(waiting_for_active);
+
+  publish_stats_to_osd();
+}
+
+void PG::_activate_committed(epoch_t e)
+{
+  lock();
+  if (pg_has_reset_since(e)) {
+    dout(10) << "_activate_committed " << e << ", that was an old interval" << dendl;
+  } else if (is_primary()) {
+    peer_activated.insert(pg_whoami);
+    dout(10) << "_activate_committed " << e << " peer_activated now " << peer_activated 
+	     << " last_epoch_started " << info.history.last_epoch_started
+	     << " same_interval_since " << info.history.same_interval_since << dendl;
+    assert(!actingbackfill.empty());
+    if (peer_activated.size() == actingbackfill.size())
+      all_activated_and_committed();
+  } else {
+    dout(10) << "_activate_committed " << e << " telling primary" << dendl;
+    MOSDPGInfo *m = new MOSDPGInfo(e);
+    pg_notify_t i = pg_notify_t(
+      get_primary().shard, pg_whoami.shard,
+      get_osdmap()->get_epoch(),
+      get_osdmap()->get_epoch(),
+      info);
+    i.info.history.last_epoch_started = e;
+    m->pg_list.push_back(make_pair(i, pg_interval_map_t()));
+    osd->send_message_osd_cluster(get_primary().osd, m, get_osdmap()->get_epoch());
+
+    state_set(PG_STATE_ACTIVE);
+    // waiters
+    if (flushes_in_progress == 0) {
+      requeue_ops(waiting_for_active);
+    }
+  }
+
+  if (dirty_info) {
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    write_if_dirty(*t);
+    int tr = osd->store->queue_transaction_and_cleanup(osr.get(), t);
+    assert(tr == 0);
+  }
+
+  unlock();
+}
+
+/*
+ * update info.history.last_epoch_started ONLY after we and all
+ * replicas have activated AND committed the activate transaction
+ * (i.e. the peering results are stable on disk).
+ */
+void PG::all_activated_and_committed()
+{
+  dout(10) << "all_activated_and_committed" << dendl;
+  assert(is_primary());
+  assert(peer_activated.size() == actingbackfill.size());
+  assert(!actingbackfill.empty());
+
+  // info.last_epoch_started is set during activate()
+  info.history.last_epoch_started = info.last_epoch_started;
+  state_clear(PG_STATE_CREATING);
+
+  share_pg_info();
+  publish_stats_to_osd();
+
+  queue_peering_event(
+    CephPeeringEvtRef(
+      new CephPeeringEvt(
+        get_osdmap()->get_epoch(),
+        get_osdmap()->get_epoch(),
+        AllReplicasActivated())));
+}
+
+void PG::queue_snap_trim()
+{
+  if (osd->queue_for_snap_trim(this))
+    dout(10) << "queue_snap_trim -- queuing" << dendl;
+  else
+    dout(10) << "queue_snap_trim -- already trimming" << dendl;
+}
+
+bool PG::queue_scrub()
+{
+  assert(_lock.is_locked());
+  if (is_scrubbing()) {
+    return false;
+  }
+  scrubber.must_scrub = false;
+  state_set(PG_STATE_SCRUBBING);
+  if (scrubber.must_deep_scrub) {
+    state_set(PG_STATE_DEEP_SCRUB);
+    scrubber.must_deep_scrub = false;
+  }
+  if (scrubber.must_repair) {
+    state_set(PG_STATE_REPAIR);
+    scrubber.must_repair = false;
+  }
+  osd->queue_for_scrub(this);
+  return true;
+}
+
+struct C_PG_FinishRecovery : public Context {
+  PGRef pg;
+  C_PG_FinishRecovery(PG *p) : pg(p) {}
+  void finish(int r) {
+    pg->_finish_recovery(this);
+  }
+};
+
+void PG::mark_clean()
+{
+  // only mark CLEAN if we have the desired number of replicas AND we
+  // are not remapped.
+  if (acting.size() == get_osdmap()->get_pg_size(info.pgid.pgid) &&
+      up == acting)
+    state_set(PG_STATE_CLEAN);
+
+  // NOTE: this is actually a bit premature: we haven't purged the
+  // strays yet.
+  info.history.last_epoch_clean = get_osdmap()->get_epoch();
+
+  trim_past_intervals();
+
+  if (is_clean() && !snap_trimq.empty())
+    queue_snap_trim();
+
+  dirty_info = true;
+}
+
+void PG::finish_recovery(list& tfin)
+{
+  dout(10) << "finish_recovery" << dendl;
+  assert(info.last_complete == info.last_update);
+
+  clear_recovery_state();
+
+  /*
+   * sync all this before purging strays.  but don't block!
+   */
+  finish_sync_event = new C_PG_FinishRecovery(this);
+  tfin.push_back(finish_sync_event);
+}
+
+void PG::_finish_recovery(Context *c)
+{
+  lock();
+  if (deleting) {
+    unlock();
+    return;
+  }
+  if (c == finish_sync_event) {
+    dout(10) << "_finish_recovery" << dendl;
+    finish_sync_event = 0;
+    purge_strays();
+
+    publish_stats_to_osd();
+
+    if (scrub_after_recovery) {
+      dout(10) << "_finish_recovery requeueing for scrub" << dendl;
+      scrub_after_recovery = false;
+      scrubber.must_deep_scrub = true;
+      queue_scrub();
+    }
+  } else {
+    dout(10) << "_finish_recovery -- stale" << dendl;
+  }
+  unlock();
+}
+
+void PG::start_recovery_op(const hobject_t& soid)
+{
+  dout(10) << "start_recovery_op " << soid
+#ifdef DEBUG_RECOVERY_OIDS
+	   << " (" << recovering_oids << ")"
+#endif
+	   << dendl;
+  assert(recovery_ops_active >= 0);
+  recovery_ops_active++;
+#ifdef DEBUG_RECOVERY_OIDS
+  assert(recovering_oids.count(soid) == 0);
+  recovering_oids.insert(soid);
+#endif
+  // TODOSAM: osd->osd-> not good
+  osd->osd->start_recovery_op(this, soid);
+}
+
+void PG::finish_recovery_op(const hobject_t& soid, bool dequeue)
+{
+  dout(10) << "finish_recovery_op " << soid
+#ifdef DEBUG_RECOVERY_OIDS
+	   << " (" << recovering_oids << ")" 
+#endif
+	   << dendl;
+  assert(recovery_ops_active > 0);
+  recovery_ops_active--;
+#ifdef DEBUG_RECOVERY_OIDS
+  assert(recovering_oids.count(soid));
+  recovering_oids.erase(soid);
+#endif
+  // TODOSAM: osd->osd-> not good
+  osd->osd->finish_recovery_op(this, soid, dequeue);
+}
+
+static void split_list(
+  list *from,
+  list *to,
+  unsigned match,
+  unsigned bits)
+{
+  for (list::iterator i = from->begin();
+       i != from->end();
+    ) {
+    if (PG::split_request(*i, match, bits)) {
+      to->push_back(*i);
+      from->erase(i++);
+    } else {
+      ++i;
+    }
+  }
+}
+
+static void split_replay_queue(
+  map *from,
+  map *to,
+  unsigned match,
+  unsigned bits)
+{
+  for (map::iterator i = from->begin();
+       i != from->end();
+       ) {
+    if (PG::split_request(i->second, match, bits)) {
+      to->insert(*i);
+      from->erase(i++);
+    } else {
+      ++i;
+    }
+  }
+}
+
+void PG::split_ops(PG *child, unsigned split_bits) {
+  unsigned match = child->info.pgid.ps();
+  assert(waiting_for_all_missing.empty());
+  assert(waiting_for_cache_not_full.empty());
+  assert(waiting_for_unreadable_object.empty());
+  assert(waiting_for_degraded_object.empty());
+  assert(waiting_for_ack.empty());
+  assert(waiting_for_ondisk.empty());
+  split_replay_queue(&replay_queue, &(child->replay_queue), match, split_bits);
+
+  osd->dequeue_pg(this, &waiting_for_active);
+  split_list(&waiting_for_active, &(child->waiting_for_active), match, split_bits);
+  {
+    Mutex::Locker l(map_lock); // to avoid a race with the osd dispatch
+    split_list(&waiting_for_map, &(child->waiting_for_map), match, split_bits);
+  }
+}
+
+void PG::split_into(pg_t child_pgid, PG *child, unsigned split_bits)
+{
+  child->update_snap_mapper_bits(split_bits);
+  child->update_osdmap_ref(get_osdmap());
+
+  child->pool = pool;
+
+  // Log
+  pg_log.split_into(child_pgid, split_bits, &(child->pg_log));
+  child->info.last_complete = info.last_complete;
+
+  info.last_update = pg_log.get_head();
+  child->info.last_update = child->pg_log.get_head();
+
+  child->info.last_user_version = info.last_user_version;
+
+  info.log_tail = pg_log.get_tail();
+  child->info.log_tail = child->pg_log.get_tail();
+
+  if (info.last_complete < pg_log.get_tail())
+    info.last_complete = pg_log.get_tail();
+  if (child->info.last_complete < child->pg_log.get_tail())
+    child->info.last_complete = child->pg_log.get_tail();
+
+  // Info
+  child->info.history = info.history;
+  child->info.purged_snaps = info.purged_snaps;
+  child->info.last_backfill = info.last_backfill;
+
+  child->info.stats = info.stats;
+  info.stats.stats_invalid = true;
+  child->info.stats.stats_invalid = true;
+  child->info.last_epoch_started = info.last_epoch_started;
+
+  child->snap_trimq = snap_trimq;
+
+  // There can't be recovery/backfill going on now
+  int primary, up_primary;
+  vector newup, newacting;
+  get_osdmap()->pg_to_up_acting_osds(
+    child->info.pgid.pgid, &newup, &up_primary, &newacting, &primary);
+  child->init_primary_up_acting(
+    newup,
+    newacting,
+    up_primary,
+    primary);
+  child->role = OSDMap::calc_pg_role(osd->whoami, child->acting);
+
+  // this comparison includes primary rank via pg_shard_t
+  if (get_primary() != child->get_primary())
+    child->info.history.same_primary_since = get_osdmap()->get_epoch();
+
+  // History
+  child->past_intervals = past_intervals;
+
+  split_ops(child, split_bits);
+  _split_into(child_pgid, child, split_bits);
+
+  child->dirty_info = true;
+  child->dirty_big_info = true;
+  dirty_info = true;
+  dirty_big_info = true;
+}
+
+void PG::clear_recovery_state() 
+{
+  dout(10) << "clear_recovery_state" << dendl;
+
+  pg_log.reset_recovery_pointers();
+  finish_sync_event = 0;
+
+  hobject_t soid;
+  while (recovery_ops_active > 0) {
+#ifdef DEBUG_RECOVERY_OIDS
+    soid = *recovering_oids.begin();
+#endif
+    finish_recovery_op(soid, true);
+  }
+
+  backfill_targets.clear();
+  backfill_info.clear();
+  peer_backfill_info.clear();
+  waiting_on_backfill.clear();
+  _clear_recovery_state();  // pg impl specific hook
+}
+
+void PG::cancel_recovery()
+{
+  dout(10) << "cancel_recovery" << dendl;
+  clear_recovery_state();
+}
+
+
+void PG::purge_strays()
+{
+  dout(10) << "purge_strays " << stray_set << dendl;
+  
+  bool removed = false;
+  for (set::iterator p = stray_set.begin();
+       p != stray_set.end();
+       ++p) {
+    assert(!is_actingbackfill(*p));
+    if (get_osdmap()->is_up(p->osd)) {
+      dout(10) << "sending PGRemove to osd." << *p << dendl;
+      vector to_remove;
+      to_remove.push_back(spg_t(info.pgid.pgid, p->shard));
+      MOSDPGRemove *m = new MOSDPGRemove(
+	get_osdmap()->get_epoch(),
+	to_remove);
+      osd->send_message_osd_cluster(p->osd, m, get_osdmap()->get_epoch());
+      stray_purged.insert(*p);
+    } else {
+      dout(10) << "not sending PGRemove to down osd." << *p << dendl;
+    }
+    peer_info.erase(*p);
+    peer_purged.insert(*p);
+    removed = true;
+  }
+
+  // if we removed anyone, update peers (which include peer_info)
+  if (removed)
+    update_heartbeat_peers();
+
+  stray_set.clear();
+
+  // clear _requested maps; we may have to peer() again if we discover
+  // (more) stray content
+  peer_log_requested.clear();
+  peer_missing_requested.clear();
+}
+
+void PG::set_probe_targets(const set &probe_set)
+{
+  Mutex::Locker l(heartbeat_peer_lock);
+  probe_targets.clear();
+  for (set::iterator i = probe_set.begin();
+       i != probe_set.end();
+       ++i) {
+    probe_targets.insert(i->osd);
+  }
+}
+
+void PG::clear_probe_targets()
+{
+  Mutex::Locker l(heartbeat_peer_lock);
+  probe_targets.clear();
+}
+
+void PG::update_heartbeat_peers()
+{
+  assert(is_locked());
+
+  set new_peers;
+  if (is_primary()) {
+    for (unsigned i=0; i::iterator p = peer_info.begin();
+	 p != peer_info.end();
+	 ++p)
+      new_peers.insert(p->first.osd);
+  }
+
+  bool need_update = false;
+  heartbeat_peer_lock.Lock();
+  if (new_peers == heartbeat_peers) {
+    dout(10) << "update_heartbeat_peers " << heartbeat_peers << " unchanged" << dendl;
+  } else {
+    dout(10) << "update_heartbeat_peers " << heartbeat_peers << " -> " << new_peers << dendl;
+    heartbeat_peers.swap(new_peers);
+    need_update = true;
+  }
+  heartbeat_peer_lock.Unlock();
+
+  if (need_update)
+    osd->need_heartbeat_peer_update();
+}
+
+void PG::_update_calc_stats()
+{
+  info.stats.version = info.last_update;
+  info.stats.created = info.history.epoch_created;
+  info.stats.last_scrub = info.history.last_scrub;
+  info.stats.last_scrub_stamp = info.history.last_scrub_stamp;
+  info.stats.last_deep_scrub = info.history.last_deep_scrub;
+  info.stats.last_deep_scrub_stamp = info.history.last_deep_scrub_stamp;
+  info.stats.last_clean_scrub_stamp = info.history.last_clean_scrub_stamp;
+  info.stats.last_epoch_clean = info.history.last_epoch_clean;
+
+  info.stats.log_size = pg_log.get_head().version - pg_log.get_tail().version;
+  info.stats.ondisk_log_size =
+    pg_log.get_head().version - pg_log.get_tail().version;
+  info.stats.log_start = pg_log.get_tail();
+  info.stats.ondisk_log_start = pg_log.get_tail();
+
+  // calc copies, degraded
+  unsigned target = MAX(
+    get_osdmap()->get_pg_size(info.pgid.pgid), actingbackfill.size());
+  info.stats.stats.calc_copies(target);
+  info.stats.stats.sum.num_objects_degraded = 0;
+  if ((is_degraded() || !is_clean()) && is_active()) {
+    // NOTE: we only generate copies, degraded, unfound values for
+    // the summation, not individual stat categories.
+    uint64_t num_objects = info.stats.stats.sum.num_objects;
+
+    uint64_t degraded = 0;
+
+    // if the actingbackfill set is smaller than we want, add in those missing replicas
+    if (actingbackfill.size() < target)
+      degraded += (target - actingbackfill.size()) * num_objects;
+
+    // missing on primary
+    info.stats.stats.sum.num_objects_missing_on_primary =
+      pg_log.get_missing().num_missing();
+    degraded += pg_log.get_missing().num_missing();
+
+    assert(!actingbackfill.empty());
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == pg_whoami) continue;
+      assert(peer_missing.count(*i));
+
+      // in missing set
+      degraded += peer_missing[*i].num_missing();
+
+      // not yet backfilled
+      degraded += num_objects - peer_info[*i].stats.stats.sum.num_objects;
+    }
+    info.stats.stats.sum.num_objects_degraded = degraded;
+    info.stats.stats.sum.num_objects_unfound = get_num_unfound();
+  }
+}
+
+void PG::publish_stats_to_osd()
+{
+  pg_stats_publish_lock.Lock();
+  if (is_primary()) {
+    // update our stat summary
+    info.stats.reported_epoch = get_osdmap()->get_epoch();
+    ++info.stats.reported_seq;
+
+    if (info.stats.stats.sum.num_scrub_errors)
+      state_set(PG_STATE_INCONSISTENT);
+    else
+      state_clear(PG_STATE_INCONSISTENT);
+
+    utime_t now = ceph_clock_now(cct);
+    info.stats.last_fresh = now;
+    if (info.stats.state != state) {
+      info.stats.state = state;
+      info.stats.last_change = now;
+      if ((state & PG_STATE_ACTIVE) &&
+	  !(info.stats.state & PG_STATE_ACTIVE))
+	info.stats.last_became_active = now;
+    }
+    if (info.stats.state & PG_STATE_CLEAN)
+      info.stats.last_clean = now;
+    if (info.stats.state & PG_STATE_ACTIVE)
+      info.stats.last_active = now;
+    info.stats.last_unstale = now;
+
+    _update_calc_stats();
+
+    pg_stats_publish_valid = true;
+    pg_stats_publish = info.stats;
+    pg_stats_publish.stats.add(unstable_stats);
+
+    dout(15) << "publish_stats_to_osd " << pg_stats_publish.reported_epoch
+	     << ":" << pg_stats_publish.reported_seq << dendl;
+  } else {
+    pg_stats_publish_valid = false;
+    dout(15) << "publish_stats_to_osd -- not primary" << dendl;
+  }
+  pg_stats_publish_lock.Unlock();
+
+  if (is_primary())
+    osd->pg_stat_queue_enqueue(this);
+}
+
+void PG::clear_publish_stats()
+{
+  dout(15) << "clear_stats" << dendl;
+  pg_stats_publish_lock.Lock();
+  pg_stats_publish_valid = false;
+  pg_stats_publish_lock.Unlock();
+
+  osd->pg_stat_queue_dequeue(this);
+}
+
+/**
+ * initialize a newly instantiated pg
+ *
+ * Initialize PG state, as when a PG is initially created, or when it
+ * is first instantiated on the current node.
+ *
+ * @param role our role/rank
+ * @param newup up set
+ * @param newacting acting set
+ * @param history pg history
+ * @param pi past_intervals
+ * @param backfill true if info should be marked as backfill
+ * @param t transaction to write out our new state in
+ */
+void PG::init(
+  int role,
+  vector& newup, int new_up_primary,
+  vector& newacting, int new_acting_primary,
+  pg_history_t& history,
+  pg_interval_map_t& pi,
+  bool backfill,
+  ObjectStore::Transaction *t)
+{
+  dout(10) << "init role " << role << " up " << newup << " acting " << newacting
+	   << " history " << history
+	   << " " << pi.size() << " past_intervals"
+	   << dendl;
+
+  set_role(role);
+  acting = newacting;
+  up = newup;
+  init_primary_up_acting(
+    newup,
+    newacting,
+    new_up_primary,
+    new_acting_primary);
+
+  info.history = history;
+  past_intervals.swap(pi);
+
+  info.stats.up = up;
+  info.stats.up_primary = new_up_primary;
+  info.stats.acting = acting;
+  info.stats.acting_primary = new_acting_primary;
+  info.stats.mapping_epoch = info.history.same_interval_since;
+
+  if (backfill) {
+    dout(10) << __func__ << ": Setting backfill" << dendl;
+    info.last_backfill = hobject_t();
+    info.last_complete = info.last_update;
+    pg_log.mark_log_for_rewrite();
+  }
+
+  reg_next_scrub();
+
+  dirty_info = true;
+  dirty_big_info = true;
+  write_if_dirty(*t);
+}
+
+void PG::upgrade(ObjectStore *store, const interval_set &snapcolls)
+{
+  unsigned removed = 0;
+  for (interval_set::const_iterator i = snapcolls.begin();
+       i != snapcolls.end();
+       ++i) {
+    for (snapid_t next_dir = i.get_start();
+	 next_dir != i.get_start() + i.get_len();
+	 ++next_dir) {
+      ++removed;
+      coll_t cid(info.pgid, next_dir);
+      dout(1) << "Removing collection " << cid
+	      << " (" << removed << "/" << snapcolls.size()
+	      << ")" << dendl;
+
+      hobject_t cur;
+      vector objects;
+      while (1) {
+	int r = get_pgbackend()->objects_list_partial(
+	  cur,
+	  store->get_ideal_list_min(),
+	  store->get_ideal_list_max(),
+	  0,
+	  &objects,
+	  &cur);
+	if (r != 0) {
+	  derr << __func__ << ": collection_list_partial returned "
+	       << cpp_strerror(r) << dendl;
+	  assert(0);
+	}
+	if (objects.empty()) {
+	  assert(cur.is_max());
+	  break;
+	}
+	ObjectStore::Transaction t;
+	for (vector::iterator j = objects.begin();
+	     j != objects.end();
+	     ++j) {
+	  t.remove(cid, *j);
+	}
+	r = store->apply_transaction(t);
+	if (r != 0) {
+	  derr << __func__ << ": apply_transaction returned "
+	       << cpp_strerror(r) << dendl;
+	  assert(0);
+	}
+	objects.clear();
+      }
+      ObjectStore::Transaction t;
+      t.remove_collection(cid);
+      int r = store->apply_transaction(t);
+      if (r != 0) {
+	derr << __func__ << ": apply_transaction returned "
+	     << cpp_strerror(r) << dendl;
+	assert(0);
+      }
+    }
+  }
+
+  hobject_t cur;
+  coll_t cid(info.pgid);
+  unsigned done = 0;
+  vector objects;
+  while (1) {
+    dout(1) << "Updating snap_mapper from main collection, "
+	    << done << " objects done" << dendl;
+    int r = get_pgbackend()->objects_list_partial(
+      cur,
+      store->get_ideal_list_min(),
+      store->get_ideal_list_max(),
+      0,
+      &objects,
+      &cur);
+    if (r != 0) {
+      derr << __func__ << ": collection_list_partial returned "
+	   << cpp_strerror(r) << dendl;
+      assert(0);
+    }
+    if (objects.empty()) {
+      assert(cur.is_max());
+      break;
+    }
+    done += objects.size();
+    ObjectStore::Transaction t;
+    for (vector::iterator j = objects.begin();
+	 j != objects.end();
+	 ++j) {
+      if (j->snap < CEPH_MAXSNAP) {
+	OSDriver::OSTransaction _t(osdriver.get_transaction(&t));
+	bufferlist bl;
+	r = get_pgbackend()->objects_get_attr(
+	  *j,
+	  OI_ATTR,
+	  &bl);
+	if (r < 0) {
+	  derr << __func__ << ": getattr returned "
+	       << cpp_strerror(r) << dendl;
+	  assert(0);
+	}
+	object_info_t oi(bl);
+	set oi_snaps(oi.snaps.begin(), oi.snaps.end());
+	set cur_snaps;
+	r = snap_mapper.get_snaps(*j, &cur_snaps);
+	if (r == 0) {
+	  assert(cur_snaps == oi_snaps);
+	} else if (r == -ENOENT) {
+	  snap_mapper.add_oid(*j, oi_snaps, &_t);
+	} else {
+	  derr << __func__ << ": get_snaps returned "
+	       << cpp_strerror(r) << dendl;
+	  assert(0);
+	}
+      }
+    }
+    r = store->apply_transaction(t);
+    if (r != 0) {
+      derr << __func__ << ": apply_transaction returned "
+	   << cpp_strerror(r) << dendl;
+      assert(0);
+    }
+    objects.clear();
+  }
+  ObjectStore::Transaction t;
+  snap_collections.clear();
+  dirty_info = true;
+  write_if_dirty(t);
+  int r = store->apply_transaction(t);
+  if (r != 0) {
+    derr << __func__ << ": apply_transaction returned "
+	 << cpp_strerror(r) << dendl;
+    assert(0);
+  }
+  assert(r == 0);
+}
+
+int PG::_write_info(ObjectStore::Transaction& t, epoch_t epoch,
+    pg_info_t &info, coll_t coll,
+    map &past_intervals,
+    interval_set &snap_collections,
+    hobject_t &infos_oid,
+    __u8 info_struct_v, bool dirty_big_info, bool force_ver)
+{
+  // pg state
+
+  if (info_struct_v > cur_struct_v)
+    return -EINVAL;
+
+  // Only need to write struct_v to attr when upgrading
+  if (force_ver || info_struct_v < cur_struct_v) {
+    bufferlist attrbl;
+    info_struct_v = cur_struct_v;
+    ::encode(info_struct_v, attrbl);
+    t.collection_setattr(coll, "info", attrbl);
+    dirty_big_info = true;
+  }
+
+  // info.  store purged_snaps separately.
+  interval_set purged_snaps;
+  map v;
+  ::encode(epoch, v[get_epoch_key(info.pgid)]);
+  purged_snaps.swap(info.purged_snaps);
+  ::encode(info, v[get_info_key(info.pgid)]);
+  purged_snaps.swap(info.purged_snaps);
+
+  if (dirty_big_info) {
+    // potentially big stuff
+    bufferlist& bigbl = v[get_biginfo_key(info.pgid)];
+    ::encode(past_intervals, bigbl);
+    ::encode(snap_collections, bigbl);
+    ::encode(info.purged_snaps, bigbl);
+    //dout(20) << "write_info bigbl " << bigbl.length() << dendl;
+  }
+
+  t.omap_setkeys(coll_t::META_COLL, infos_oid, v);
+
+  return 0;
+}
+
+void PG::write_info(ObjectStore::Transaction& t)
+{
+  info.stats.stats.add(unstable_stats);
+  unstable_stats.clear();
+
+  int ret = _write_info(t, get_osdmap()->get_epoch(), info, coll,
+     past_intervals, snap_collections, osd->infos_oid,
+     info_struct_v, dirty_big_info);
+  assert(ret == 0);
+  last_persisted_osdmap_ref = osdmap_ref;
+
+  dirty_info = false;
+  dirty_big_info = false;
+}
+
+epoch_t PG::peek_map_epoch(ObjectStore *store, coll_t coll, hobject_t &infos_oid, bufferlist *bl)
+{
+  assert(bl);
+  spg_t pgid;
+  snapid_t snap;
+  bool ok = coll.is_pg(pgid, snap);
+  assert(ok);
+  int r = store->collection_getattr(coll, "info", *bl);
+  assert(r > 0);
+  bufferlist::iterator bp = bl->begin();
+  __u8 struct_v = 0;
+  ::decode(struct_v, bp);
+  if (struct_v < 5)
+    return 0;
+  epoch_t cur_epoch = 0;
+  if (struct_v < 6) {
+    ::decode(cur_epoch, bp);
+  } else {
+    // get epoch out of leveldb
+    bufferlist tmpbl;
+    string ek = get_epoch_key(pgid);
+    set keys;
+    keys.insert(get_epoch_key(pgid));
+    map values;
+    store->omap_get_values(coll_t::META_COLL, infos_oid, keys, &values);
+    assert(values.size() == 1);
+    tmpbl = values[ek];
+    bufferlist::iterator p = tmpbl.begin();
+    ::decode(cur_epoch, p);
+  }
+  return cur_epoch;
+}
+
+void PG::write_if_dirty(ObjectStore::Transaction& t)
+{
+  if (dirty_big_info || dirty_info)
+    write_info(t);
+  pg_log.write_log(t, log_oid);
+}
+
+void PG::trim_peers()
+{
+  assert(is_primary());
+  calc_trim_to();
+  dout(10) << "trim_peers " << pg_trim_to << dendl;
+  if (pg_trim_to != eversion_t()) {
+    assert(!actingbackfill.empty());
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == pg_whoami) continue;
+      osd->send_message_osd_cluster(
+	i->osd,
+	new MOSDPGTrim(
+	  get_osdmap()->get_epoch(),
+	  spg_t(info.pgid.pgid, i->shard),
+	  pg_trim_to),
+	get_osdmap()->get_epoch());
+    }
+  }
+}
+
+void PG::add_log_entry(pg_log_entry_t& e, bufferlist& log_bl)
+{
+  // raise last_complete only if we were previously up to date
+  if (info.last_complete == info.last_update)
+    info.last_complete = e.version;
+  
+  // raise last_update.
+  assert(e.version > info.last_update);
+  info.last_update = e.version;
+
+  // raise user_version, if it increased (it may have not get bumped
+  // by all logged updates)
+  if (e.user_version > info.last_user_version)
+    info.last_user_version = e.user_version;
+
+  /**
+   * Make sure we don't keep around more than we need to in the
+   * in-memory log
+   */
+  e.mod_desc.trim_bl();
+
+  // log mutation
+  pg_log.add(e);
+  dout(10) << "add_log_entry " << e << dendl;
+
+  e.encode_with_checksum(log_bl);
+}
+
+
+void PG::append_log(
+  vector& logv,
+  eversion_t trim_to,
+  eversion_t trim_rollback_to,
+  ObjectStore::Transaction &t,
+  bool transaction_applied)
+{
+  if (transaction_applied)
+    update_snap_map(logv, t);
+  dout(10) << "append_log " << pg_log.get_log() << " " << logv << dendl;
+
+  map keys;
+  for (vector::iterator p = logv.begin();
+       p != logv.end();
+       ++p) {
+    p->offset = 0;
+    add_log_entry(*p, keys[p->get_key_name()]);
+  }
+
+  PGLogEntryHandler handler;
+  if (!transaction_applied) {
+    pg_log.clear_can_rollback_to(&handler);
+    t.register_on_applied(
+      new C_UpdateLastRollbackInfoTrimmedToApplied(
+	this,
+	get_osdmap()->get_epoch(),
+	info.last_update));
+  } else if (trim_rollback_to > pg_log.get_rollback_trimmed_to()) {
+    pg_log.trim_rollback_info(
+      trim_rollback_to,
+      &handler);
+    t.register_on_applied(
+      new C_UpdateLastRollbackInfoTrimmedToApplied(
+	this,
+	get_osdmap()->get_epoch(),
+	trim_rollback_to));
+  }
+
+  dout(10) << "append_log  adding " << keys.size() << " keys" << dendl;
+  t.omap_setkeys(coll_t::META_COLL, log_oid, keys);
+
+  pg_log.trim(&handler, trim_to, info);
+
+  dout(10) << __func__ << ": trimming to " << trim_rollback_to
+	   << " entries " << handler.to_trim << dendl;
+  handler.apply(this, &t);
+
+  // update the local pg, pg log
+  dirty_info = true;
+  write_if_dirty(t);
+}
+
+bool PG::check_log_for_corruption(ObjectStore *store)
+{
+  /// TODO: this method needs to work with the omap log
+  return true;
+}
+
+//! Get the name we're going to save our corrupt page log as
+std::string PG::get_corrupt_pg_log_name() const
+{
+  const int MAX_BUF = 512;
+  char buf[MAX_BUF];
+  struct tm tm_buf;
+  time_t my_time(time(NULL));
+  const struct tm *t = localtime_r(&my_time, &tm_buf);
+  int ret = strftime(buf, sizeof(buf), "corrupt_log_%Y-%m-%d_%k:%M_", t);
+  if (ret == 0) {
+    dout(0) << "strftime failed" << dendl;
+    return "corrupt_log_unknown_time";
+  }
+  string out(buf);
+  out += stringify(info.pgid);
+  return out;
+}
+
+int PG::read_info(
+  ObjectStore *store, const coll_t coll, bufferlist &bl,
+  pg_info_t &info, map &past_intervals,
+  hobject_t &biginfo_oid, hobject_t &infos_oid,
+  interval_set  &snap_collections, __u8 &struct_v)
+{
+  bufferlist::iterator p = bl.begin();
+  bufferlist lbl;
+
+  // info
+  ::decode(struct_v, p);
+  if (struct_v < 4)
+    ::decode(info, p);
+  if (struct_v < 2) {
+    ::decode(past_intervals, p);
+  
+    // snap_collections
+    store->collection_getattr(coll, "snap_collections", lbl);
+    p = lbl.begin();
+    ::decode(struct_v, p);
+  } else {
+    if (struct_v < 6) {
+      int r = store->read(coll_t::META_COLL, biginfo_oid, 0, 0, lbl);
+      if (r < 0)
+        return r;
+      p = lbl.begin();
+      ::decode(past_intervals, p);
+    } else {
+      // get info out of leveldb
+      string k = get_info_key(info.pgid);
+      string bk = get_biginfo_key(info.pgid);
+      set keys;
+      keys.insert(k);
+      keys.insert(bk);
+      map values;
+      store->omap_get_values(coll_t::META_COLL, infos_oid, keys, &values);
+      assert(values.size() == 2);
+      lbl = values[k];
+      p = lbl.begin();
+      ::decode(info, p);
+
+      lbl = values[bk];
+      p = lbl.begin();
+      ::decode(past_intervals, p);
+    }
+  }
+
+  if (struct_v < 3) {
+    set snap_collections_temp;
+    ::decode(snap_collections_temp, p);
+    snap_collections.clear();
+    for (set::iterator i = snap_collections_temp.begin();
+	 i != snap_collections_temp.end();
+	 ++i) {
+      snap_collections.insert(*i);
+    }
+  } else {
+    ::decode(snap_collections, p);
+    if (struct_v >= 4 && struct_v < 6)
+      ::decode(info, p);
+    else if (struct_v >= 6)
+      ::decode(info.purged_snaps, p);
+  }
+  return 0;
+}
+
+void PG::read_state(ObjectStore *store, bufferlist &bl)
+{
+  int r = read_info(store, coll, bl, info, past_intervals, biginfo_oid,
+    osd->infos_oid, snap_collections, info_struct_v);
+  assert(r >= 0);
+
+  ostringstream oss;
+  if (pg_log.read_log(
+      store, coll, log_oid, info,
+      oss)) {
+    /* We don't want to leave the old format around in case the next log
+     * write happens to be an append_log()
+     */
+    pg_log.mark_log_for_rewrite();
+    ObjectStore::Transaction t;
+    t.remove(coll_t(), log_oid); // remove old version
+    pg_log.write_log(t, log_oid);
+    int r = osd->store->apply_transaction(t);
+    assert(!r);
+  }
+  if (oss.str().length())
+    osd->clog.error() << oss;
+
+  // log any weirdness
+  log_weirdness();
+}
+
+void PG::log_weirdness()
+{
+  if (pg_log.get_tail() != info.log_tail)
+    osd->clog.error() << info.pgid
+		      << " info mismatch, log.tail " << pg_log.get_tail()
+		      << " != info.log_tail " << info.log_tail
+		      << "\n";
+  if (pg_log.get_head() != info.last_update)
+    osd->clog.error() << info.pgid
+		      << " info mismatch, log.head " << pg_log.get_head()
+		      << " != info.last_update " << info.last_update
+		      << "\n";
+
+  if (!pg_log.get_log().empty()) {
+    // sloppy check
+    if ((pg_log.get_log().log.begin()->version <= pg_log.get_tail()))
+      osd->clog.error() << info.pgid
+			<< " log bound mismatch, info (" << pg_log.get_tail() << ","
+			<< pg_log.get_head() << "]"
+			<< " actual ["
+			<< pg_log.get_log().log.begin()->version << ","
+			<< pg_log.get_log().log.rbegin()->version << "]"
+			<< "\n";
+  }
+  
+  if (pg_log.get_log().caller_ops.size() > pg_log.get_log().log.size()) {
+    osd->clog.error() << info.pgid
+		      << " caller_ops.size " << pg_log.get_log().caller_ops.size()
+		      << " > log size " << pg_log.get_log().log.size()
+		      << "\n";
+  }
+}
+
+void PG::update_snap_map(
+  vector &log_entries,
+  ObjectStore::Transaction &t)
+{
+  for (vector::iterator i = log_entries.begin();
+       i != log_entries.end();
+       ++i) {
+    OSDriver::OSTransaction _t(osdriver.get_transaction(&t));
+    if (i->soid.snap < CEPH_MAXSNAP) {
+      if (i->is_delete()) {
+	int r = snap_mapper.remove_oid(
+	  i->soid,
+	  &_t);
+	assert(r == 0);
+      } else {
+	assert(i->snaps.length() > 0);
+	vector snaps;
+	bufferlist::iterator p = i->snaps.begin();
+	try {
+	  ::decode(snaps, p);
+	} catch (...) {
+	  snaps.clear();
+	}
+	set _snaps(snaps.begin(), snaps.end());
+
+	if (i->is_clone() || i->is_promote()) {
+	  snap_mapper.add_oid(
+	    i->soid,
+	    _snaps,
+	    &_t);
+	} else if (i->is_modify()) {
+	  assert(i->is_modify());
+	  int r = snap_mapper.update_snaps(
+	    i->soid,
+	    _snaps,
+	    0,
+	    &_t);
+	  assert(r == 0);
+	} else {
+	  assert(i->is_clean());
+	}
+      }
+    }
+  }
+}
+
+/**
+ * filter trimming|trimmed snaps out of snapcontext
+ */
+void PG::filter_snapc(vector &snaps)
+{
+  bool filtering = false;
+  vector newsnaps;
+  for (vector::iterator p = snaps.begin();
+       p != snaps.end();
+       ++p) {
+    if (snap_trimq.contains(*p) || info.purged_snaps.contains(*p)) {
+      if (!filtering) {
+	// start building a new vector with what we've seen so far
+	dout(10) << "filter_snapc filtering " << snaps << dendl;
+	newsnaps.insert(newsnaps.begin(), snaps.begin(), p);
+	filtering = true;
+      }
+      dout(20) << "filter_snapc  removing trimq|purged snap " << *p << dendl;
+    } else {
+      if (filtering)
+	newsnaps.push_back(*p);  // continue building new vector
+    }
+  }
+  if (filtering) {
+    snaps.swap(newsnaps);
+    dout(10) << "filter_snapc  result " << snaps << dendl;
+  }
+}
+
+void PG::requeue_object_waiters(map >& m)
+{
+  for (map >::iterator it = m.begin();
+       it != m.end();
+       ++it)
+    requeue_ops(it->second);
+  m.clear();
+}
+
+void PG::requeue_op(OpRequestRef op)
+{
+  osd->op_wq.queue_front(make_pair(PGRef(this), op));
+}
+
+void PG::requeue_ops(list &ls)
+{
+  dout(15) << " requeue_ops " << ls << dendl;
+  for (list::reverse_iterator i = ls.rbegin();
+       i != ls.rend();
+       ++i) {
+    osd->op_wq.queue_front(make_pair(PGRef(this), *i));
+  }
+  ls.clear();
+}
+
+
+// ==========================================================================================
+// SCRUB
+
+/*
+ * when holding pg and sched_scrub_lock, then the states are:
+ *   scheduling:
+ *     scrubber.reserved = true
+ *     scrub_rserved_peers includes whoami
+ *     osd->scrub_pending++
+ *   scheduling, replica declined:
+ *     scrubber.reserved = true
+ *     scrubber.reserved_peers includes -1
+ *     osd->scrub_pending++
+ *   pending:
+ *     scrubber.reserved = true
+ *     scrubber.reserved_peers.size() == acting.size();
+ *     pg on scrub_wq
+ *     osd->scrub_pending++
+ *   scrubbing:
+ *     scrubber.reserved = false;
+ *     scrubber.reserved_peers empty
+ *     osd->scrubber.active++
+ */
+
+// returns true if a scrub has been newly kicked off
+bool PG::sched_scrub()
+{
+  assert(_lock.is_locked());
+  if (!(is_primary() && is_active() && is_clean() && !is_scrubbing())) {
+    return false;
+  }
+
+  bool time_for_deep = (ceph_clock_now(cct) >
+    info.history.last_deep_scrub_stamp + cct->_conf->osd_deep_scrub_interval);
+ 
+  //NODEEP_SCRUB so ignore time initiated deep-scrub
+  if (osd->osd->get_osdmap()->test_flag(CEPH_OSDMAP_NODEEP_SCRUB))
+    time_for_deep = false;
+
+  if (!scrubber.must_scrub) {
+    assert(!scrubber.must_deep_scrub);
+
+    //NOSCRUB so skip regular scrubs
+    if (osd->osd->get_osdmap()->test_flag(CEPH_OSDMAP_NOSCRUB) && !time_for_deep)
+      return false;
+  }
+
+  bool ret = true;
+  if (!scrubber.reserved) {
+    assert(scrubber.reserved_peers.empty());
+    if (osd->inc_scrubs_pending()) {
+      dout(20) << "sched_scrub: reserved locally, reserving replicas" << dendl;
+      scrubber.reserved = true;
+      scrubber.reserved_peers.insert(pg_whoami);
+      scrub_reserve_replicas();
+    } else {
+      dout(20) << "sched_scrub: failed to reserve locally" << dendl;
+      ret = false;
+    }
+  }
+  if (scrubber.reserved) {
+    if (scrubber.reserve_failed) {
+      dout(20) << "sched_scrub: failed, a peer declined" << dendl;
+      clear_scrub_reserved();
+      scrub_unreserve_replicas();
+      ret = false;
+    } else if (scrubber.reserved_peers.size() == acting.size()) {
+      dout(20) << "sched_scrub: success, reserved self and replicas" << dendl;
+      if (time_for_deep) {
+	dout(10) << "sched_scrub: scrub will be deep" << dendl;
+	state_set(PG_STATE_DEEP_SCRUB);
+      }
+      queue_scrub();
+    } else {
+      // none declined, since scrubber.reserved is set
+      dout(20) << "sched_scrub: reserved " << scrubber.reserved_peers << ", waiting for replicas" << dendl;
+    }
+  }
+
+  return ret;
+}
+
+void PG::reg_next_scrub()
+{
+  if (scrubber.must_scrub ||
+      (info.stats.stats_invalid && g_conf->osd_scrub_invalid_stats)) {
+    scrubber.scrub_reg_stamp = utime_t();
+  } else {
+    scrubber.scrub_reg_stamp = info.history.last_scrub_stamp;
+  }
+  if (is_primary())
+    osd->reg_last_pg_scrub(info.pgid, scrubber.scrub_reg_stamp);
+}
+
+void PG::unreg_next_scrub()
+{
+  if (is_primary())
+    osd->unreg_last_pg_scrub(info.pgid, scrubber.scrub_reg_stamp);
+}
+
+void PG::sub_op_scrub_map(OpRequestRef op)
+{
+  MOSDSubOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+  dout(7) << "sub_op_scrub_map" << dendl;
+
+  if (m->map_epoch < info.history.same_interval_since) {
+    dout(10) << "sub_op_scrub discarding old sub_op from "
+	     << m->map_epoch << " < " << info.history.same_interval_since << dendl;
+    return;
+  }
+
+  op->mark_started();
+
+  dout(10) << " got " << m->from << " scrub map" << dendl;
+  bufferlist::iterator p = m->get_data().begin();
+
+  if (scrubber.is_chunky) { // chunky scrub
+    scrubber.received_maps[m->from].decode(p, info.pgid.pool());
+    dout(10) << "map version is "
+	     << scrubber.received_maps[m->from].valid_through
+	     << dendl;
+  } else {               // classic scrub
+    if (scrubber.received_maps.count(m->from)) {
+      ScrubMap incoming;
+      incoming.decode(p, info.pgid.pool());
+      dout(10) << "from replica " << m->from << dendl;
+      dout(10) << "map version is " << incoming.valid_through << dendl;
+      scrubber.received_maps[m->from].merge_incr(incoming);
+    } else {
+      scrubber.received_maps[m->from].decode(p, info.pgid.pool());
+    }
+  }
+
+  --scrubber.waiting_on;
+  scrubber.waiting_on_whom.erase(m->from);
+
+  if (scrubber.waiting_on == 0) {
+    if (scrubber.is_chunky) { // chunky scrub
+      osd->scrub_wq.queue(this);
+    } else {                  // classic scrub
+      if (scrubber.finalizing) { // incremental lists received
+        osd->scrub_finalize_wq.queue(this);
+      } else {                // initial lists received
+        scrubber.block_writes = true;
+        if (last_update_applied == info.last_update) {
+          scrubber.finalizing = true;
+          scrub_gather_replica_maps();
+          ++scrubber.waiting_on;
+          scrubber.waiting_on_whom.insert(pg_whoami);
+          osd->scrub_wq.queue(this);
+        }
+      }
+    }
+  }
+}
+
+// send scrub v2-compatible messages (classic scrub)
+void PG::_request_scrub_map_classic(pg_shard_t replica, eversion_t version)
+{
+  assert(replica != pg_whoami);
+  dout(10) << "scrub  requesting scrubmap from osd." << replica << dendl;
+  MOSDRepScrub *repscrubop =
+    new MOSDRepScrub(
+      spg_t(info.pgid.pgid, replica.shard), version,
+      last_update_applied,
+      get_osdmap()->get_epoch());
+  osd->send_message_osd_cluster(
+    replica.osd, repscrubop, get_osdmap()->get_epoch());
+}
+
+// send scrub v3 messages (chunky scrub)
+void PG::_request_scrub_map(
+  pg_shard_t replica, eversion_t version,
+  hobject_t start, hobject_t end,
+  bool deep)
+{
+  assert(replica != pg_whoami);
+  dout(10) << "scrub  requesting scrubmap from osd." << replica << dendl;
+  MOSDRepScrub *repscrubop = new MOSDRepScrub(
+    spg_t(info.pgid.pgid, replica.shard), version,
+    get_osdmap()->get_epoch(),
+    start, end, deep);
+  osd->send_message_osd_cluster(
+    replica.osd, repscrubop, get_osdmap()->get_epoch());
+}
+
+void PG::sub_op_scrub_reserve(OpRequestRef op)
+{
+  MOSDSubOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+  dout(7) << "sub_op_scrub_reserve" << dendl;
+
+  if (scrubber.reserved) {
+    dout(10) << "Ignoring reserve request: Already reserved" << dendl;
+    return;
+  }
+
+  op->mark_started();
+
+  scrubber.reserved = osd->inc_scrubs_pending();
+
+  MOSDSubOpReply *reply = new MOSDSubOpReply(
+    m, pg_whoami, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+  ::encode(scrubber.reserved, reply->get_data());
+  osd->send_message_osd_cluster(reply, m->get_connection());
+}
+
+void PG::sub_op_scrub_reserve_reply(OpRequestRef op)
+{
+  MOSDSubOpReply *reply = static_cast(op->get_req());
+  assert(reply->get_header().type == MSG_OSD_SUBOPREPLY);
+  dout(7) << "sub_op_scrub_reserve_reply" << dendl;
+
+  if (!scrubber.reserved) {
+    dout(10) << "ignoring obsolete scrub reserve reply" << dendl;
+    return;
+  }
+
+  op->mark_started();
+
+  pg_shard_t from = reply->from;
+  bufferlist::iterator p = reply->get_data().begin();
+  bool reserved;
+  ::decode(reserved, p);
+
+  if (scrubber.reserved_peers.find(from) != scrubber.reserved_peers.end()) {
+    dout(10) << " already had osd." << from << " reserved" << dendl;
+  } else {
+    if (reserved) {
+      dout(10) << " osd." << from << " scrub reserve = success" << dendl;
+      scrubber.reserved_peers.insert(from);
+    } else {
+      /* One decline stops this pg from being scheduled for scrubbing. */
+      dout(10) << " osd." << from << " scrub reserve = fail" << dendl;
+      scrubber.reserve_failed = true;
+    }
+    sched_scrub();
+  }
+}
+
+void PG::sub_op_scrub_unreserve(OpRequestRef op)
+{
+  assert(op->get_req()->get_header().type == MSG_OSD_SUBOP);
+  dout(7) << "sub_op_scrub_unreserve" << dendl;
+
+  op->mark_started();
+
+  clear_scrub_reserved();
+}
+
+void PG::sub_op_scrub_stop(OpRequestRef op)
+{
+  op->mark_started();
+
+  MOSDSubOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+  dout(7) << "sub_op_scrub_stop" << dendl;
+
+  // see comment in sub_op_scrub_reserve
+  scrubber.reserved = false;
+
+  MOSDSubOpReply *reply = new MOSDSubOpReply(
+    m, pg_whoami, 0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+  osd->send_message_osd_cluster(reply, m->get_connection());
+}
+
+void PG::reject_reservation()
+{
+  osd->send_message_osd_cluster(
+    primary.osd,
+    new MBackfillReserve(
+      MBackfillReserve::REJECT,
+      spg_t(info.pgid.pgid, primary.shard),
+      get_osdmap()->get_epoch()),
+    get_osdmap()->get_epoch());
+}
+
+void PG::schedule_backfill_full_retry()
+{
+  Mutex::Locker lock(osd->backfill_request_lock);
+  osd->backfill_request_timer.add_event_after(
+    cct->_conf->osd_backfill_retry_interval,
+    new QueuePeeringEvt(
+      this, get_osdmap()->get_epoch(),
+      RequestBackfill()));
+}
+
+void PG::clear_scrub_reserved()
+{
+  osd->scrub_wq.dequeue(this);
+  scrubber.reserved_peers.clear();
+  scrubber.reserve_failed = false;
+
+  if (scrubber.reserved) {
+    scrubber.reserved = false;
+    osd->dec_scrubs_pending();
+  }
+}
+
+void PG::scrub_reserve_replicas()
+{
+  assert(backfill_targets.empty());
+  for (set::iterator i = actingbackfill.begin();
+       i != actingbackfill.end();
+       ++i) {
+    if (*i == pg_whoami) continue;
+    dout(10) << "scrub requesting reserve from osd." << *i << dendl;
+    vector scrub(1);
+    scrub[0].op.op = CEPH_OSD_OP_SCRUB_RESERVE;
+    hobject_t poid;
+    eversion_t v;
+    osd_reqid_t reqid;
+    MOSDSubOp *subop = new MOSDSubOp(
+      reqid, pg_whoami, spg_t(info.pgid.pgid, i->shard), poid, false, 0,
+      get_osdmap()->get_epoch(), osd->get_tid(), v);
+    subop->ops = scrub;
+    osd->send_message_osd_cluster(
+      i->osd, subop, get_osdmap()->get_epoch());
+  }
+}
+
+void PG::scrub_unreserve_replicas()
+{
+  assert(backfill_targets.empty());
+  for (set::iterator i = actingbackfill.begin();
+       i != actingbackfill.end();
+       ++i) {
+    if (*i == pg_whoami) continue;
+    dout(10) << "scrub requesting unreserve from osd." << *i << dendl;
+    vector scrub(1);
+    scrub[0].op.op = CEPH_OSD_OP_SCRUB_UNRESERVE;
+    hobject_t poid;
+    eversion_t v;
+    osd_reqid_t reqid;
+    MOSDSubOp *subop = new MOSDSubOp(
+      reqid, pg_whoami, spg_t(info.pgid.pgid, i->shard), poid, false, 0,
+      get_osdmap()->get_epoch(), osd->get_tid(), v);
+    subop->ops = scrub;
+    osd->send_message_osd_cluster(i->osd, subop, get_osdmap()->get_epoch());
+  }
+}
+
+void PG::_scan_rollback_obs(
+  const vector &rollback_obs,
+  ThreadPool::TPHandle &handle)
+{
+  ObjectStore::Transaction *t = NULL;
+  eversion_t trimmed_to = last_rollback_info_trimmed_to_applied;
+  for (vector::const_iterator i = rollback_obs.begin();
+       i != rollback_obs.end();
+       ++i) {
+    if (i->generation < trimmed_to.version) {
+      osd->clog.error() << "osd." << osd->whoami
+			<< " pg " << info.pgid
+			<< " found obsolete rollback obj "
+			<< *i << " generation < trimmed_to "
+			<< trimmed_to
+			<< "...repaired";
+      if (!t)
+	t = new ObjectStore::Transaction;
+      t->remove(coll, *i);
+    }
+  }
+  if (t) {
+    derr << __func__ << ": queueing trans to clean up obsolete rollback objs"
+	 << dendl;
+    osd->store->queue_transaction_and_cleanup(osr.get(), t);
+  }
+}
+
+void PG::_scan_snaps(ScrubMap &smap) 
+{
+  for (map::iterator i = smap.objects.begin();
+       i != smap.objects.end();
+       ++i) {
+    const hobject_t &hoid = i->first;
+    ScrubMap::object &o = i->second;
+
+    if (hoid.snap < CEPH_MAXSNAP) {
+      // fake nlinks for old primaries
+      bufferlist bl;
+      bl.push_back(o.attrs[OI_ATTR]);
+      object_info_t oi(bl);
+      if (oi.snaps.empty()) {
+	// Just head
+	o.nlinks = 1;
+      } else if (oi.snaps.size() == 1) {
+	// Just head + only snap
+	o.nlinks = 2;
+      } else {
+	// Just head + 1st and last snaps
+	o.nlinks = 3;
+      }
+
+      // check and if necessary fix snap_mapper
+      set oi_snaps(oi.snaps.begin(), oi.snaps.end());
+      set cur_snaps;
+      int r = snap_mapper.get_snaps(hoid, &cur_snaps);
+      if (r != 0 && r != -ENOENT) {
+	derr << __func__ << ": get_snaps returned " << cpp_strerror(r) << dendl;
+	assert(0);
+      }
+      if (r == -ENOENT || cur_snaps != oi_snaps) {
+	ObjectStore::Transaction t;
+	OSDriver::OSTransaction _t(osdriver.get_transaction(&t));
+	if (r == 0) {
+	  r = snap_mapper.remove_oid(hoid, &_t);
+	  if (r != 0) {
+	    derr << __func__ << ": remove_oid returned " << cpp_strerror(r)
+		 << dendl;
+	    assert(0);
+	  }
+	  osd->clog.error() << "osd." << osd->whoami
+			    << " found snap mapper error on pg "
+			    << info.pgid
+			    << " oid " << hoid << " snaps in mapper: "
+			    << cur_snaps << ", oi: "
+			    << oi_snaps
+			    << "...repaired";
+	} else {
+	  osd->clog.error() << "osd." << osd->whoami
+			    << " found snap mapper error on pg "
+			    << info.pgid
+			    << " oid " << hoid << " snaps missing in mapper"
+			    << ", should be: "
+			    << oi_snaps
+			    << "...repaired";
+	}
+	snap_mapper.add_oid(hoid, oi_snaps, &_t);
+	r = osd->store->apply_transaction(t);
+	if (r != 0) {
+	  derr << __func__ << ": apply_transaction got " << cpp_strerror(r)
+	       << dendl;
+	}
+      }
+    } else {
+      o.nlinks = 1;
+    }
+  }
+}
+
+/*
+ * build a scrub map over a chunk without releasing the lock
+ * only used by chunky scrub
+ */
+int PG::build_scrub_map_chunk(
+  ScrubMap &map,
+  hobject_t start, hobject_t end, bool deep,
+  ThreadPool::TPHandle &handle)
+{
+  dout(10) << "build_scrub_map" << dendl;
+  dout(20) << "scrub_map_chunk [" << start << "," << end << ")" << dendl;
+
+  map.valid_through = info.last_update;
+
+  // objects
+  vector ls;
+  vector rollback_obs;
+  int ret = get_pgbackend()->objects_list_range(
+    start,
+    end,
+    0,
+    &ls,
+    &rollback_obs);
+  if (ret < 0) {
+    dout(5) << "objects_list_range error: " << ret << dendl;
+    return ret;
+  }
+
+
+  get_pgbackend()->be_scan_list(map, ls, deep, handle);
+  _scan_rollback_obs(rollback_obs, handle);
+  _scan_snaps(map);
+
+  // pg attrs
+  osd->store->collection_getattrs(coll, map.attrs);
+  dout(10) << __func__ << " done." << dendl;
+
+  return 0;
+}
+
+/*
+ * build a (sorted) summary of pg content for purposes of scrubbing
+ * called while holding pg lock
+ */ 
+void PG::build_scrub_map(ScrubMap &map, ThreadPool::TPHandle &handle)
+{
+  dout(10) << "build_scrub_map" << dendl;
+
+  map.valid_through = info.last_update;
+  epoch_t epoch = get_osdmap()->get_epoch();
+
+  unlock();
+
+  // wait for any writes on our pg to flush to disk first.  this avoids races
+  // with scrub starting immediately after trim or recovery completion.
+  osr->flush();
+
+  // objects
+  vector ls;
+  osd->store->collection_list(coll, ls);
+
+  get_pgbackend()->be_scan_list(map, ls, false, handle);
+  lock();
+  _scan_snaps(map);
+
+  if (pg_has_reset_since(epoch)) {
+    dout(10) << "scrub  pg changed, aborting" << dendl;
+    return;
+  }
+
+
+  dout(10) << "PG relocked, finalizing" << dendl;
+
+  // pg attrs
+  osd->store->collection_getattrs(coll, map.attrs);
+
+  dout(10) << __func__ << " done." << dendl;
+}
+
+
+/* 
+ * build a summary of pg content changed starting after v
+ * called while holding pg lock
+ */
+void PG::build_inc_scrub_map(
+  ScrubMap &map, eversion_t v,
+  ThreadPool::TPHandle &handle)
+{
+  map.valid_through = last_update_applied;
+  map.incr_since = v;
+  vector ls;
+  list::const_iterator p;
+  if (v == pg_log.get_tail()) {
+    p = pg_log.get_log().log.begin();
+  } else if (v > pg_log.get_tail()) {
+    p = pg_log.get_log().find_entry(v);
+    ++p;
+  } else {
+    assert(0);
+  }
+  
+  for (; p != pg_log.get_log().log.end(); ++p) {
+    if (p->is_update()) {
+      ls.push_back(p->soid);
+      map.objects[p->soid].negative = false;
+    } else if (p->is_delete()) {
+      map.objects[p->soid].negative = true;
+    }
+  }
+
+  get_pgbackend()->be_scan_list(map, ls, false, handle);
+  // pg attrs
+  osd->store->collection_getattrs(coll, map.attrs);
+}
+
+void PG::repair_object(
+  const hobject_t& soid, ScrubMap::object *po,
+  pg_shard_t bad_peer, pg_shard_t ok_peer)
+{
+  dout(10) << "repair_object " << soid << " bad_peer osd."
+	   << bad_peer << " ok_peer osd." << ok_peer << dendl;
+  eversion_t v;
+  bufferlist bv;
+  bv.push_back(po->attrs[OI_ATTR]);
+  object_info_t oi(bv);
+  if (bad_peer != primary) {
+    peer_missing[bad_peer].add(soid, oi.version, eversion_t());
+  } else {
+    // We should only be scrubbing if the PG is clean.
+    assert(waiting_for_unreadable_object.empty());
+
+    pg_log.missing_add(soid, oi.version, eversion_t());
+    missing_loc.add_missing(soid, oi.version, eversion_t());
+    missing_loc.add_location(soid, ok_peer);
+
+    pg_log.set_last_requested(0);
+  }
+}
+
+/* replica_scrub
+ *
+ * Classic behavior:
+ *
+ * If msg->scrub_from is not set, replica_scrub calls build_scrubmap to
+ * build a complete map (with the pg lock dropped).
+ *
+ * If msg->scrub_from is set, replica_scrub sets scrubber.finalizing.
+ * Similarly to scrub, if last_update_applied is behind info.last_update
+ * replica_scrub returns to be requeued by sub_op_modify_applied.
+ * replica_scrub then builds an incremental scrub map with the 
+ * pg lock held.
+ *
+ * Chunky behavior:
+ *
+ * Wait for last_update_applied to match msg->scrub_to as above. Wait
+ * for pushes to complete in case of recent recovery. Build a single
+ * scrubmap of objects that are in the range [msg->start, msg->end).
+ */
+void PG::replica_scrub(
+  MOSDRepScrub *msg,
+  ThreadPool::TPHandle &handle)
+{
+  assert(!scrubber.active_rep_scrub);
+  dout(7) << "replica_scrub" << dendl;
+
+  if (msg->map_epoch < info.history.same_interval_since) {
+    if (scrubber.finalizing) {
+      dout(10) << "scrub  pg changed, aborting" << dendl;
+      scrubber.finalizing = 0;
+    } else {
+      dout(10) << "replica_scrub discarding old replica_scrub from "
+	       << msg->map_epoch << " < " << info.history.same_interval_since 
+	       << dendl;
+    }
+    return;
+  }
+
+  ScrubMap map;
+
+  if (msg->chunky) { // chunky scrub
+    if (last_update_applied < msg->scrub_to) {
+      dout(10) << "waiting for last_update_applied to catch up" << dendl;
+      scrubber.active_rep_scrub = msg;
+      msg->get();
+      return;
+    }
+
+    if (active_pushes > 0) {
+      dout(10) << "waiting for active pushes to finish" << dendl;
+      scrubber.active_rep_scrub = msg;
+      msg->get();
+      return;
+    }
+
+    build_scrub_map_chunk(
+      map, msg->start, msg->end, msg->deep,
+      handle);
+
+  } else {
+    if (msg->scrub_from > eversion_t()) {
+      if (scrubber.finalizing) {
+        assert(last_update_applied == info.last_update);
+        assert(last_update_applied == msg->scrub_to);
+      } else {
+        scrubber.finalizing = 1;
+        if (last_update_applied != msg->scrub_to) {
+          scrubber.active_rep_scrub = msg;
+          msg->get();
+          return;
+        }
+      }
+      build_inc_scrub_map(map, msg->scrub_from, handle);
+      scrubber.finalizing = 0;
+    } else {
+      build_scrub_map(map, handle);
+    }
+
+    if (msg->map_epoch < info.history.same_interval_since) {
+      dout(10) << "scrub  pg changed, aborting" << dendl;
+      return;
+    }
+  }
+
+  vector scrub(1);
+  scrub[0].op.op = CEPH_OSD_OP_SCRUB_MAP;
+  hobject_t poid;
+  eversion_t v;
+  osd_reqid_t reqid;
+  MOSDSubOp *subop = new MOSDSubOp(
+    reqid,
+    pg_whoami,
+    spg_t(info.pgid.pgid, get_primary().shard),
+    poid,
+    false,
+    0,
+    msg->map_epoch,
+    osd->get_tid(),
+    v);
+  ::encode(map, subop->get_data());
+  subop->ops = scrub;
+
+  osd->send_message_osd_cluster(subop, msg->get_connection());
+}
+
+/* Scrub:
+ * PG_STATE_SCRUBBING is set when the scrub is queued
+ * 
+ * scrub will be chunky if all OSDs in PG support chunky scrub
+ * scrub will fall back to classic in any other case
+ */
+void PG::scrub(ThreadPool::TPHandle &handle)
+{
+  lock();
+  if (g_conf->osd_scrub_sleep > 0 &&
+      (scrubber.state == PG::Scrubber::NEW_CHUNK ||
+       scrubber.state == PG::Scrubber::INACTIVE)) {
+    dout(20) << __func__ << " state is INACTIVE|NEW_CHUNK, sleeping" << dendl;
+    unlock();
+    utime_t t;
+    t.set_from_double(g_conf->osd_scrub_sleep);
+    t.sleep();
+    lock();
+    dout(20) << __func__ << " slept for " << t << dendl;
+  }
+  if (deleting) {
+    unlock();
+    return;
+  }
+
+  if (!is_primary() || !is_active() || !is_clean() || !is_scrubbing()) {
+    dout(10) << "scrub -- not primary or active or not clean" << dendl;
+    state_clear(PG_STATE_SCRUBBING);
+    state_clear(PG_STATE_REPAIR);
+    state_clear(PG_STATE_DEEP_SCRUB);
+    publish_stats_to_osd();
+    unlock();
+    return;
+  }
+
+  // when we're starting a scrub, we need to determine which type of scrub to do
+  if (!scrubber.active) {
+    OSDMapRef curmap = osd->get_osdmap();
+    scrubber.is_chunky = true;
+    assert(backfill_targets.empty());
+    for (unsigned i=0; iget_con_osd_cluster(acting[i], get_osdmap()->get_epoch());
+      if (!con)
+	continue;
+      if (!con->has_feature(CEPH_FEATURE_CHUNKY_SCRUB)) {
+        dout(20) << "OSD " << acting[i]
+                 << " does not support chunky scrubs, falling back to classic"
+                 << dendl;
+        scrubber.is_chunky = false;
+        break;
+      }
+    }
+
+    if (scrubber.is_chunky) {
+      scrubber.deep = state_test(PG_STATE_DEEP_SCRUB);
+    } else {
+      state_clear(PG_STATE_DEEP_SCRUB);
+    }
+
+    dout(10) << "starting a new " << (scrubber.is_chunky ? "chunky" : "classic") << " scrub" << dendl;
+  }
+
+  if (scrubber.is_chunky) {
+    chunky_scrub(handle);
+  } else {
+    classic_scrub(handle);
+  }
+
+  unlock();
+}
+
+/*
+ * Classic scrub is a two stage scrub: an initial scrub with writes enabled
+ * followed by a finalize with writes blocked.
+ *
+ * A request is sent out to all replicas for initial scrub maps. Once they reply
+ * (sub_op_scrub_map) writes are blocked for all objects in the PG.
+ *
+ * Finalize: Primaries and replicas wait for all writes in the log to be applied
+ * (op_applied), then builds an incremental scrub of all the changes since the
+ * beginning of the scrub.
+ *
+ * Once the primary has received all maps, it compares them and performs
+ * repairs.
+ *
+ * The initial stage of the scrub is handled by scrub_wq and the final stage by
+ * scrub_finalize_wq.
+ *
+ * Relevant variables:
+ *
+ * scrubber.waiting_on (int)
+ * scrubber.waiting_on_whom
+ *    Number of people who still need to build an initial/incremental scrub map.
+ *    This is decremented in sub_op_scrub_map.
+ *
+ * last_update_applied
+ *    The last update that's hit the disk. In the finalize stage, we block
+ *    writes and wait for all writes to flush by checking:
+ *
+ *      last_update_appied == info.last_update
+ *
+ *    This is checked in op_applied.
+ *
+ *  scrubber.block_writes
+ *    Flag to determine if writes are blocked.
+ *
+ *  finalizing scrub
+ *    Flag set when we're in the finalize stage.
+ *
+ */
+void PG::classic_scrub(ThreadPool::TPHandle &handle)
+{
+  assert(pool.info.type == pg_pool_t::TYPE_REPLICATED);
+  if (!scrubber.active) {
+    dout(10) << "scrub start" << dendl;
+    scrubber.active = true;
+    scrubber.classic = true;
+
+    publish_stats_to_osd();
+    scrubber.received_maps.clear();
+    scrubber.epoch_start = info.history.same_interval_since;
+
+    osd->inc_scrubs_active(scrubber.reserved);
+    if (scrubber.reserved) {
+      scrubber.reserved = false;
+      scrubber.reserved_peers.clear();
+    }
+
+    /* scrubber.waiting_on == 0 iff all replicas have sent the requested maps and
+     * the primary has done a final scrub (which in turn can only happen if
+     * last_update_applied == info.last_update)
+     */
+    scrubber.waiting_on = acting.size();
+    scrubber.waiting_on_whom.insert(
+      actingbackfill.begin(), actingbackfill.end());
+    scrubber.waiting_on_whom.erase(pg_whoami);
+
+    // request maps from replicas
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == pg_whoami) continue;
+      _request_scrub_map_classic(*i, eversion_t());
+    }
+
+    // Unlocks and relocks...
+    scrubber.primary_scrubmap = ScrubMap();
+    build_scrub_map(scrubber.primary_scrubmap, handle);
+
+    if (scrubber.epoch_start != info.history.same_interval_since) {
+      dout(10) << "scrub  pg changed, aborting" << dendl;
+      scrub_clear_state();
+      scrub_unreserve_replicas();
+      return;
+    }
+
+    --scrubber.waiting_on;
+    scrubber.waiting_on_whom.erase(pg_whoami);
+
+    if (scrubber.waiting_on == 0) {
+      // the replicas have completed their scrub map, so lock out writes
+      scrubber.block_writes = true;
+    } else {
+      dout(10) << "wait for replicas to build initial scrub map" << dendl;
+      return;
+    }
+
+    if (last_update_applied != info.last_update) {
+      dout(10) << "wait for cleanup" << dendl;
+      return;
+    }
+
+    // fall through if last_update_applied == info.last_update and scrubber.waiting_on == 0
+
+    // request incrementals from replicas
+    scrub_gather_replica_maps();
+    ++scrubber.waiting_on;
+    scrubber.waiting_on_whom.insert(pg_whoami);
+  }
+    
+  dout(10) << "clean up scrub" << dendl;
+  assert(last_update_applied == info.last_update);
+
+  scrubber.finalizing = true;
+
+  if (scrubber.epoch_start != info.history.same_interval_since) {
+    dout(10) << "scrub  pg changed, aborting" << dendl;
+    scrub_clear_state();
+    scrub_unreserve_replicas();
+    return;
+  }
+  
+  if (scrubber.primary_scrubmap.valid_through != pg_log.get_head()) {
+    ScrubMap incr;
+    build_inc_scrub_map(incr, scrubber.primary_scrubmap.valid_through, handle);
+    scrubber.primary_scrubmap.merge_incr(incr);
+  }
+  
+  --scrubber.waiting_on;
+  scrubber.waiting_on_whom.erase(pg_whoami);
+  if (scrubber.waiting_on == 0) {
+    assert(last_update_applied == info.last_update);
+    osd->scrub_finalize_wq.queue(this);
+  }
+}
+
+/*
+ * Chunky scrub scrubs objects one chunk at a time with writes blocked for that
+ * chunk.
+ *
+ * The object store is partitioned into chunks which end on hash boundaries. For
+ * each chunk, the following logic is performed:
+ *
+ *  (1) Block writes on the chunk
+ *  (2) Request maps from replicas
+ *  (3) Wait for pushes to be applied (after recovery)
+ *  (4) Wait for writes to flush on the chunk
+ *  (5) Wait for maps from replicas
+ *  (6) Compare / repair all scrub maps
+ *
+ * This logic is encoded in the very linear state machine:
+ *
+ *           +------------------+
+ *  _________v__________        |
+ * |                    |       |
+ * |      INACTIVE      |       |
+ * |____________________|       |
+ *           |                  |
+ *           |   +----------+   |
+ *  _________v___v______    |   |
+ * |                    |   |   |
+ * |      NEW_CHUNK     |   |   |
+ * |____________________|   |   |
+ *           |              |   |
+ *  _________v__________    |   |
+ * |                    |   |   |
+ * |     WAIT_PUSHES    |   |   |
+ * |____________________|   |   |
+ *           |              |   |
+ *  _________v__________    |   |
+ * |                    |   |   |
+ * |  WAIT_LAST_UPDATE  |   |   |
+ * |____________________|   |   |
+ *           |              |   |
+ *  _________v__________    |   |
+ * |                    |   |   |
+ * |      BUILD_MAP     |   |   |
+ * |____________________|   |   |
+ *           |              |   |
+ *  _________v__________    |   |
+ * |                    |   |   |
+ * |    WAIT_REPLICAS   |   |   |
+ * |____________________|   |   |
+ *           |              |   |
+ *  _________v__________    |   |
+ * |                    |   |   |
+ * |    COMPARE_MAPS    |   |   |
+ * |____________________|   |   |
+ *           |   |          |   |
+ *           |   +----------+   |
+ *  _________v__________        |
+ * |                    |       |
+ * |       FINISH       |       |
+ * |____________________|       |
+ *           |                  |
+ *           +------------------+
+ *
+ * The primary determines the last update from the subset by walking the log. If
+ * it sees a log entry pertaining to a file in the chunk, it tells the replicas
+ * to wait until that update is applied before building a scrub map. Both the
+ * primary and replicas will wait for any active pushes to be applied.
+ *
+ * In contrast to classic_scrub, chunky_scrub is entirely handled by scrub_wq.
+ *
+ * scrubber.state encodes the current state of the scrub (refer to state diagram
+ * for details).
+ */
+void PG::chunky_scrub(ThreadPool::TPHandle &handle)
+{
+  // check for map changes
+  if (scrubber.is_chunky_scrub_active()) {
+    if (scrubber.epoch_start != info.history.same_interval_since) {
+      dout(10) << "scrub  pg changed, aborting" << dendl;
+      scrub_clear_state();
+      scrub_unreserve_replicas();
+      return;
+    }
+  }
+
+  bool done = false;
+  int ret;
+
+  while (!done) {
+    dout(20) << "scrub state " << Scrubber::state_string(scrubber.state) << dendl;
+
+    switch (scrubber.state) {
+      case PG::Scrubber::INACTIVE:
+        dout(10) << "scrub start" << dendl;
+
+        publish_stats_to_osd();
+        scrubber.epoch_start = info.history.same_interval_since;
+        scrubber.active = true;
+
+	osd->inc_scrubs_active(scrubber.reserved);
+	if (scrubber.reserved) {
+	  scrubber.reserved = false;
+	  scrubber.reserved_peers.clear();
+	}
+
+        scrubber.start = hobject_t();
+        scrubber.state = PG::Scrubber::NEW_CHUNK;
+
+        break;
+
+      case PG::Scrubber::NEW_CHUNK:
+        scrubber.primary_scrubmap = ScrubMap();
+        scrubber.received_maps.clear();
+
+        {
+	  hobject_t candidate_end;
+
+          // get the start and end of our scrub chunk
+          //
+          // start and end need to lie on a hash boundary. We test for this by
+          // requesting a list and searching backward from the end looking for a
+          // boundary. If there's no boundary, we request a list after the first
+          // list, and so forth.
+
+          bool boundary_found = false;
+          hobject_t start = scrubber.start;
+          while (!boundary_found) {
+            vector objects;
+            ret = get_pgbackend()->objects_list_partial(
+	      start,
+	      cct->_conf->osd_scrub_chunk_min,
+	      cct->_conf->osd_scrub_chunk_max,
+	      0,
+	      &objects,
+	      &candidate_end);
+            assert(ret >= 0);
+
+            // in case we don't find a boundary: start again at the end
+            start = candidate_end;
+
+            // special case: reached end of file store, implicitly a boundary
+            if (objects.empty()) {
+              break;
+            }
+
+            // search backward from the end looking for a boundary
+            objects.push_back(candidate_end);
+            while (!boundary_found && objects.size() > 1) {
+              hobject_t end = objects.back().get_boundary();
+              objects.pop_back();
+
+              if (objects.back().get_filestore_key() != end.get_filestore_key()) {
+                candidate_end = end;
+                boundary_found = true;
+              }
+            }
+          }
+
+	  if (!_range_available_for_scrub(scrubber.start, candidate_end)) {
+	    // we'll be requeued by whatever made us unavailable for scrub
+	    dout(10) << __func__ << ": scrub blocked somewhere in range "
+		     << "[" << scrubber.start << ", " << candidate_end << ")"
+		     << dendl;
+	    done = true;
+	    break;
+	  }
+	  scrubber.end = candidate_end;
+        }
+        scrubber.block_writes = true;
+
+        // walk the log to find the latest update that affects our chunk
+        scrubber.subset_last_update = pg_log.get_tail();
+        for (list::const_iterator p = pg_log.get_log().log.begin();
+             p != pg_log.get_log().log.end();
+             ++p) {
+          if (p->soid >= scrubber.start && p->soid < scrubber.end)
+            scrubber.subset_last_update = p->version;
+        }
+
+        // ask replicas to wait until last_update_applied >= scrubber.subset_last_update and then scan
+        scrubber.waiting_on_whom.insert(pg_whoami);
+        ++scrubber.waiting_on;
+
+        // request maps from replicas
+	for (set::iterator i = actingbackfill.begin();
+	     i != actingbackfill.end();
+	     ++i) {
+	  if (*i == pg_whoami) continue;
+          _request_scrub_map(*i, scrubber.subset_last_update,
+                             scrubber.start, scrubber.end, scrubber.deep);
+          scrubber.waiting_on_whom.insert(*i);
+          ++scrubber.waiting_on;
+        }
+
+        scrubber.state = PG::Scrubber::WAIT_PUSHES;
+
+        break;
+
+      case PG::Scrubber::WAIT_PUSHES:
+        if (active_pushes == 0) {
+          scrubber.state = PG::Scrubber::WAIT_LAST_UPDATE;
+        } else {
+          dout(15) << "wait for pushes to apply" << dendl;
+          done = true;
+        }
+        break;
+
+      case PG::Scrubber::WAIT_LAST_UPDATE:
+        if (last_update_applied >= scrubber.subset_last_update) {
+          scrubber.state = PG::Scrubber::BUILD_MAP;
+        } else {
+          // will be requeued by op_applied
+          dout(15) << "wait for writes to flush" << dendl;
+          done = true;
+        }
+        break;
+
+      case PG::Scrubber::BUILD_MAP:
+        assert(last_update_applied >= scrubber.subset_last_update);
+
+        // build my own scrub map
+        ret = build_scrub_map_chunk(scrubber.primary_scrubmap,
+                                    scrubber.start, scrubber.end,
+                                    scrubber.deep,
+				    handle);
+        if (ret < 0) {
+          dout(5) << "error building scrub map: " << ret << ", aborting" << dendl;
+          scrub_clear_state();
+          scrub_unreserve_replicas();
+          return;
+        }
+
+        --scrubber.waiting_on;
+        scrubber.waiting_on_whom.erase(pg_whoami);
+
+        scrubber.state = PG::Scrubber::WAIT_REPLICAS;
+        break;
+
+      case PG::Scrubber::WAIT_REPLICAS:
+        if (scrubber.waiting_on > 0) {
+          // will be requeued by sub_op_scrub_map
+          dout(10) << "wait for replicas to build scrub map" << dendl;
+          done = true;
+        } else {
+          scrubber.state = PG::Scrubber::COMPARE_MAPS;
+        }
+        break;
+
+      case PG::Scrubber::COMPARE_MAPS:
+        assert(last_update_applied >= scrubber.subset_last_update);
+        assert(scrubber.waiting_on == 0);
+
+        scrub_compare_maps();
+        scrubber.block_writes = false;
+	scrubber.run_callbacks();
+
+        // requeue the writes from the chunk that just finished
+        requeue_ops(waiting_for_active);
+
+        if (scrubber.end < hobject_t::get_max()) {
+          // schedule another leg of the scrub
+          scrubber.start = scrubber.end;
+
+          scrubber.state = PG::Scrubber::NEW_CHUNK;
+          osd->scrub_wq.queue(this);
+          done = true;
+        } else {
+          scrubber.state = PG::Scrubber::FINISH;
+        }
+
+        break;
+
+      case PG::Scrubber::FINISH:
+        scrub_finish();
+        scrubber.state = PG::Scrubber::INACTIVE;
+        done = true;
+
+        break;
+
+      default:
+        assert(0);
+    }
+  }
+}
+
+void PG::scrub_clear_state()
+{
+  assert(_lock.is_locked());
+  state_clear(PG_STATE_SCRUBBING);
+  state_clear(PG_STATE_REPAIR);
+  state_clear(PG_STATE_DEEP_SCRUB);
+  publish_stats_to_osd();
+
+  // active -> nothing.
+  if (scrubber.active)
+    osd->dec_scrubs_active();
+
+  requeue_ops(waiting_for_active);
+
+  if (scrubber.queue_snap_trim) {
+    dout(10) << "scrub finished, requeuing snap_trimmer" << dendl;
+    queue_snap_trim();
+  }
+
+  scrubber.reset();
+
+  // type-specific state clear
+  _scrub_clear_state();
+}
+
+bool PG::scrub_gather_replica_maps()
+{
+  assert(scrubber.waiting_on == 0);
+  assert(_lock.is_locked());
+
+  for (map::iterator p = scrubber.received_maps.begin();
+       p != scrubber.received_maps.end();
+       ++p) {
+    
+    if (scrubber.received_maps[p->first].valid_through != pg_log.get_head()) {
+      scrubber.waiting_on++;
+      scrubber.waiting_on_whom.insert(p->first);
+      // Need to request another incremental map
+      _request_scrub_map_classic(p->first, p->second.valid_through);
+    }
+  }
+  
+  if (scrubber.waiting_on > 0) {
+    return false;
+  } else {
+    return true;
+  }
+}
+
+void PG::scrub_compare_maps() 
+{
+  dout(10) << "scrub_compare_maps has maps, analyzing" << dendl;
+
+  // construct authoritative scrub map for type specific scrubbing
+  ScrubMap authmap(scrubber.primary_scrubmap);
+
+  if (acting.size() > 1) {
+    dout(10) << "scrub  comparing replica scrub maps" << dendl;
+
+    stringstream ss;
+
+    // Map from object with errors to good peer
+    map authoritative;
+    map maps;
+
+    dout(2) << "scrub   osd." << acting[0] << " has " 
+	    << scrubber.primary_scrubmap.objects.size() << " items" << dendl;
+    maps[pg_whoami] = &scrubber.primary_scrubmap;
+
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == pg_whoami) continue;
+      dout(2) << "scrub replica " << *i << " has "
+	      << scrubber.received_maps[*i].objects.size()
+	      << " items" << dendl;
+      maps[*i] = &scrubber.received_maps[*i];
+    }
+
+    get_pgbackend()->be_compare_scrubmaps(
+      maps,
+      scrubber.missing,
+      scrubber.inconsistent,
+      authoritative,
+      scrubber.inconsistent_snapcolls,
+      scrubber.shallow_errors,
+      scrubber.deep_errors,
+      info.pgid, acting,
+      ss);
+    dout(2) << ss.str() << dendl;
+
+    if (!authoritative.empty() || !scrubber.inconsistent_snapcolls.empty()) {
+      osd->clog.error(ss);
+    }
+
+    for (map::iterator i = authoritative.begin();
+	 i != authoritative.end();
+	 ++i) {
+      scrubber.authoritative.insert(
+	make_pair(
+	  i->first,
+	  make_pair(maps[i->second]->objects[i->first], i->second)));
+    }
+
+    for (map::iterator i = authoritative.begin();
+	 i != authoritative.end();
+	 ++i) {
+      authmap.objects.erase(i->first);
+      authmap.objects.insert(*(maps[i->second]->objects.find(i->first)));
+    }
+  }
+
+  // ok, do the pg-type specific scrubbing
+  _scrub(authmap);
+}
+
+void PG::scrub_process_inconsistent()
+{
+  dout(10) << "process_inconsistent() checking authoritative" << dendl;
+  bool repair = state_test(PG_STATE_REPAIR);
+  bool deep_scrub = state_test(PG_STATE_DEEP_SCRUB);
+  const char *mode = (repair ? "repair": (deep_scrub ? "deep-scrub" : "scrub"));
+
+  if (!scrubber.authoritative.empty() || !scrubber.inconsistent.empty()) {
+    stringstream ss;
+    for (map >::iterator obj =
+	   scrubber.inconsistent_snapcolls.begin();
+	 obj != scrubber.inconsistent_snapcolls.end();
+	 ++obj) {
+      for (set::iterator j = obj->second.begin();
+	   j != obj->second.end();
+	   ++j) {
+	++scrubber.shallow_errors;
+	ss << info.pgid << " " << mode << " " << " object " << obj->first
+	   << " has inconsistent snapcolls on " << *j << std::endl;
+      }
+    }
+
+    ss << info.pgid << " " << mode << " "
+       << scrubber.missing.size() << " missing, "
+       << scrubber.inconsistent.size() << " inconsistent objects\n";
+    dout(2) << ss.str() << dendl;
+    osd->clog.error(ss);
+    if (repair) {
+      state_clear(PG_STATE_CLEAN);
+      for (map >::iterator i =
+	     scrubber.authoritative.begin();
+	   i != scrubber.authoritative.end();
+	   ++i) {
+	set::iterator j;
+	
+	if (scrubber.missing.count(i->first)) {
+	  for (j = scrubber.missing[i->first].begin();
+	       j != scrubber.missing[i->first].end(); 
+	       ++j) {
+	    repair_object(
+	      i->first,
+	      &(i->second.first),
+	      *j,
+	      i->second.second);
+	    ++scrubber.fixed;
+	  }
+	}
+	if (scrubber.inconsistent.count(i->first)) {
+	  for (j = scrubber.inconsistent[i->first].begin(); 
+	       j != scrubber.inconsistent[i->first].end(); 
+	       ++j) {
+	    repair_object(i->first, 
+	      &(i->second.first),
+	      *j,
+	      i->second.second);
+	    ++scrubber.fixed;
+	  }
+	}
+      }
+    }
+  }
+}
+
+void PG::scrub_finalize()
+{
+  lock();
+  if (deleting) {
+    unlock();
+    return;
+  }
+
+  assert(last_update_applied == info.last_update);
+
+  if (scrubber.epoch_start != info.history.same_interval_since) {
+    dout(10) << "scrub  pg changed, aborting" << dendl;
+    scrub_clear_state();
+    scrub_unreserve_replicas();
+    unlock();
+    return;
+  }
+
+  if (!scrub_gather_replica_maps()) {
+    dout(10) << "maps not yet up to date, sent out new requests" << dendl;
+    unlock();
+    return;
+  }
+
+  scrub_compare_maps();
+
+  scrub_finish();
+
+  dout(10) << "scrub done" << dendl;
+  unlock();
+}
+
+// the part that actually finalizes a scrub
+void PG::scrub_finish() 
+{
+  bool repair = state_test(PG_STATE_REPAIR);
+  bool deep_scrub = state_test(PG_STATE_DEEP_SCRUB);
+  const char *mode = (repair ? "repair": (deep_scrub ? "deep-scrub" : "scrub"));
+
+  // type-specific finish (can tally more errors)
+  _scrub_finish();
+
+  scrub_process_inconsistent();
+
+  {
+    stringstream oss;
+    oss << info.pgid.pgid << " " << mode << " ";
+    int total_errors = scrubber.shallow_errors + scrubber.deep_errors;
+    if (total_errors)
+      oss << total_errors << " errors";
+    else
+      oss << "ok";
+    if (!deep_scrub && info.stats.stats.sum.num_deep_scrub_errors)
+      oss << " ( " << info.stats.stats.sum.num_deep_scrub_errors
+          << " remaining deep scrub error(s) )";
+    if (repair)
+      oss << ", " << scrubber.fixed << " fixed";
+    oss << "\n";
+    if (total_errors)
+      osd->clog.error(oss);
+    else
+      osd->clog.info(oss);
+  }
+
+  // finish up
+  unreg_next_scrub();
+  utime_t now = ceph_clock_now(cct);
+  info.history.last_scrub = info.last_update;
+  info.history.last_scrub_stamp = now;
+  if (scrubber.deep) {
+    info.history.last_deep_scrub = info.last_update;
+    info.history.last_deep_scrub_stamp = now;
+  }
+  // Since we don't know which errors were fixed, we can only clear them
+  // when every one has been fixed.
+  if (repair) {
+    if (scrubber.fixed == scrubber.shallow_errors + scrubber.deep_errors) {
+      assert(deep_scrub);
+      scrubber.shallow_errors = scrubber.deep_errors = 0;
+    } else {
+      // Deep scrub in order to get corrected error counts
+      scrub_after_recovery = true;
+    }
+  }
+  if (deep_scrub) {
+    if ((scrubber.shallow_errors == 0) && (scrubber.deep_errors == 0))
+      info.history.last_clean_scrub_stamp = now;
+    info.stats.stats.sum.num_shallow_scrub_errors = scrubber.shallow_errors;
+    info.stats.stats.sum.num_deep_scrub_errors = scrubber.deep_errors;
+  } else {
+    info.stats.stats.sum.num_shallow_scrub_errors = scrubber.shallow_errors;
+    // XXX: last_clean_scrub_stamp doesn't mean the pg is not inconsistent
+    // because of deep-scrub errors
+    if (scrubber.shallow_errors == 0)
+      info.history.last_clean_scrub_stamp = now;
+  }
+  info.stats.stats.sum.num_scrub_errors = 
+    info.stats.stats.sum.num_shallow_scrub_errors +
+    info.stats.stats.sum.num_deep_scrub_errors;
+  reg_next_scrub();
+
+  {
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    dirty_info = true;
+    write_if_dirty(*t);
+    int tr = osd->store->queue_transaction_and_cleanup(osr.get(), t);
+    assert(tr == 0);
+  }
+
+
+  if (repair) {
+    queue_peering_event(
+      CephPeeringEvtRef(
+	new CephPeeringEvt(
+	  get_osdmap()->get_epoch(),
+	  get_osdmap()->get_epoch(),
+	  DoRecovery())));
+  }
+
+  scrub_clear_state();
+  scrub_unreserve_replicas();
+
+  if (is_active() && is_primary()) {
+    share_pg_info();
+  }
+}
+
+void PG::share_pg_info()
+{
+  dout(10) << "share_pg_info" << dendl;
+
+  // share new pg_info_t with replicas
+  assert(!actingbackfill.empty());
+  for (set::iterator i = actingbackfill.begin();
+       i != actingbackfill.end();
+       ++i) {
+    if (*i == pg_whoami) continue;
+    pg_shard_t peer = *i;
+    if (peer_info.count(peer)) {
+      peer_info[peer].last_epoch_started = info.last_epoch_started;
+      peer_info[peer].history.merge(info.history);
+    }
+    MOSDPGInfo *m = new MOSDPGInfo(get_osdmap()->get_epoch());
+    m->pg_list.push_back(
+      make_pair(
+	pg_notify_t(
+	  peer.shard, pg_whoami.shard,
+	  get_osdmap()->get_epoch(),
+	  get_osdmap()->get_epoch(),
+	  info),
+	pg_interval_map_t()));
+    osd->send_message_osd_cluster(peer.osd, m, get_osdmap()->get_epoch());
+  }
+}
+
+/*
+ * Share a new segment of this PG's log with some replicas, after PG is active.
+ *
+ * Updates peer_missing and peer_info.
+ */
+void PG::share_pg_log()
+{
+  dout(10) << __func__ << dendl;
+  assert(is_primary());
+
+  set::const_iterator a = actingbackfill.begin();
+  assert(a != actingbackfill.end());
+  set::const_iterator end = actingbackfill.end();
+  while (a != end) {
+    pg_shard_t peer(*a);
+    ++a;
+    if (peer == pg_whoami) continue;
+    pg_missing_t& pmissing(peer_missing[peer]);
+    pg_info_t& pinfo(peer_info[peer]);
+
+    MOSDPGLog *m = new MOSDPGLog(
+      peer.shard, pg_whoami.shard,
+      info.last_update.epoch, info);
+    m->log.copy_after(pg_log.get_log(), pinfo.last_update);
+
+    for (list::const_iterator i = m->log.log.begin();
+	 i != m->log.log.end();
+	 ++i) {
+      pmissing.add_next_event(*i);
+    }
+    pinfo.last_update = m->log.head;
+
+    osd->send_message_osd_cluster(peer.osd, m, get_osdmap()->get_epoch());
+  }
+}
+
+void PG::update_history_from_master(pg_history_t new_history)
+{
+  unreg_next_scrub();
+  info.history.merge(new_history);
+  reg_next_scrub();
+}
+
+void PG::fulfill_info(
+  pg_shard_t from, const pg_query_t &query,
+  pair ¬ify_info)
+{
+  assert(from == primary);
+  assert(query.type == pg_query_t::INFO);
+
+  // info
+  dout(10) << "sending info" << dendl;
+  notify_info = make_pair(from, info);
+}
+
+void PG::fulfill_log(
+  pg_shard_t from, const pg_query_t &query, epoch_t query_epoch)
+{
+  dout(10) << "log request from " << from << dendl;
+  assert(from == primary);
+  assert(query.type != pg_query_t::INFO);
+
+  MOSDPGLog *mlog = new MOSDPGLog(
+    from.shard, pg_whoami.shard,
+    get_osdmap()->get_epoch(),
+    info, query_epoch);
+  mlog->missing = pg_log.get_missing();
+
+  // primary -> other, when building master log
+  if (query.type == pg_query_t::LOG) {
+    dout(10) << " sending info+missing+log since " << query.since
+	     << dendl;
+    if (query.since != eversion_t() && query.since < pg_log.get_tail()) {
+      osd->clog.error() << info.pgid << " got broken pg_query_t::LOG since " << query.since
+			<< " when my log.tail is " << pg_log.get_tail()
+			<< ", sending full log instead\n";
+      mlog->log = pg_log.get_log();           // primary should not have requested this!!
+    } else
+      mlog->log.copy_after(pg_log.get_log(), query.since);
+  }
+  else if (query.type == pg_query_t::FULLLOG) {
+    dout(10) << " sending info+missing+full log" << dendl;
+    mlog->log = pg_log.get_log();
+  }
+
+  dout(10) << " sending " << mlog->log << " " << mlog->missing << dendl;
+
+  ConnectionRef con = osd->get_con_osd_cluster(
+    from.osd, get_osdmap()->get_epoch());
+  if (con) {
+    osd->osd->_share_map_outgoing(from.osd, con.get(), get_osdmap());
+    osd->send_message_osd_cluster(mlog, con.get());
+  } else {
+    mlog->put();
+  }
+}
+
+
+// true if all OSDs in prior intervals may have crashed, and we need to replay
+// false positives are okay, false negatives are not.
+bool PG::may_need_replay(const OSDMapRef osdmap) const
+{
+  bool crashed = false;
+
+  for (map::const_reverse_iterator p = past_intervals.rbegin();
+       p != past_intervals.rend();
+       ++p) {
+    const pg_interval_t &interval = p->second;
+    dout(10) << "may_need_replay " << interval << dendl;
+
+    if (interval.last < info.history.last_epoch_started)
+      break;  // we don't care
+
+    if (interval.acting.empty())
+      continue;
+
+    if (!interval.maybe_went_rw)
+      continue;
+
+    // look at whether any of the osds during this interval survived
+    // past the end of the interval (i.e., didn't crash and
+    // potentially fail to COMMIT a write that it ACKed).
+    bool any_survived_interval = false;
+
+    // consider ACTING osds
+    for (unsigned i=0; iexists(o))
+	pinfo = &osdmap->get_info(o);
+
+      // does this osd appear to have survived through the end of the
+      // interval?
+      if (pinfo) {
+	if (pinfo->up_from <= interval.first && pinfo->up_thru > interval.last) {
+	  dout(10) << "may_need_replay  osd." << o
+		   << " up_from " << pinfo->up_from << " up_thru " << pinfo->up_thru
+		   << " survived the interval" << dendl;
+	  any_survived_interval = true;
+	}
+	else if (pinfo->up_from <= interval.first &&
+		 (std::find(acting.begin(), acting.end(), o) != acting.end() ||
+		  std::find(up.begin(), up.end(), o) != up.end())) {
+	  dout(10) << "may_need_replay  osd." << o
+		   << " up_from " << pinfo->up_from << " and is in acting|up,"
+		   << " assumed to have survived the interval" << dendl;
+	  // (if it hasn't, we will rebuild PriorSet)
+	  any_survived_interval = true;
+	}
+	else if (pinfo->up_from > interval.last &&
+		 pinfo->last_clean_begin <= interval.first &&
+		 pinfo->last_clean_end > interval.last) {
+	  dout(10) << "may_need_replay  prior osd." << o
+		   << " up_from " << pinfo->up_from
+		   << " and last clean interval ["
+		   << pinfo->last_clean_begin << "," << pinfo->last_clean_end
+		   << ") survived the interval" << dendl;
+	  any_survived_interval = true;
+	}
+      }
+    }
+
+    if (!any_survived_interval) {
+      dout(3) << "may_need_replay  no known survivors of interval "
+	      << interval.first << "-" << interval.last
+	      << ", may need replay" << dendl;
+      crashed = true;
+      break;
+    }
+  }
+
+  return crashed;
+}
+
+bool PG::is_split(OSDMapRef lastmap, OSDMapRef nextmap)
+{
+  return info.pgid.is_split(
+    lastmap->get_pg_num(pool.id),
+    nextmap->get_pg_num(pool.id),
+    0);
+}
+
+bool PG::acting_up_affected(
+  int newupprimary,
+  int newactingprimary,
+  const vector& newup, const vector& newacting)
+{
+  if (newupprimary != up_primary.osd ||
+      newactingprimary != primary.osd ||
+      acting != newacting ||
+      up != newup) {
+    dout(20) << "acting_up_affected newup " << newup
+	     << " newacting " << newacting << dendl;
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool PG::old_peering_msg(epoch_t reply_epoch, epoch_t query_epoch)
+{
+  if (last_peering_reset > reply_epoch ||
+      last_peering_reset > query_epoch) {
+    dout(10) << "old_peering_msg reply_epoch " << reply_epoch << " query_epoch " << query_epoch
+	     << " last_peering_reset " << last_peering_reset
+	     << dendl;
+    return true;
+  }
+  return false;
+}
+
+void PG::set_last_peering_reset()
+{
+  dout(20) << "set_last_peering_reset " << get_osdmap()->get_epoch() << dendl;
+  last_peering_reset = get_osdmap()->get_epoch();
+}
+
+struct FlushState {
+  PGRef pg;
+  epoch_t epoch;
+  FlushState(PG *pg, epoch_t epoch) : pg(pg), epoch(epoch) {}
+  ~FlushState() {
+    pg->lock();
+    if (!pg->pg_has_reset_since(epoch))
+      pg->queue_flushed(epoch);
+    pg->unlock();
+  }
+};
+typedef ceph::shared_ptr FlushStateRef;
+
+void PG::start_flush(ObjectStore::Transaction *t,
+		     list *on_applied,
+		     list *on_safe)
+{
+  // flush in progress ops
+  FlushStateRef flush_trigger(
+    new FlushState(this, get_osdmap()->get_epoch()));
+  t->nop();
+  flushes_in_progress++;
+  on_applied->push_back(new ContainerContext(flush_trigger));
+  on_safe->push_back(new ContainerContext(flush_trigger));
+}
+
+void PG::reset_interval_flush()
+{
+  dout(10) << "Clearing blocked outgoing recovery messages" << dendl;
+  recovery_state.clear_blocked_outgoing();
+  
+  if (!osr->flush_commit(
+      new QueuePeeringEvt(
+	this, get_osdmap()->get_epoch(), IntervalFlush()))) {
+    dout(10) << "Beginning to block outgoing recovery messages" << dendl;
+    recovery_state.begin_block_outgoing();
+  } else {
+    dout(10) << "Not blocking outgoing recovery messages" << dendl;
+  }
+}
+
+/* Called before initializing peering during advance_map */
+void PG::start_peering_interval(
+  const OSDMapRef lastmap,
+  const vector& newup, int new_up_primary,
+  const vector& newacting, int new_acting_primary,
+  ObjectStore::Transaction *t)
+{
+  const OSDMapRef osdmap = get_osdmap();
+
+  set_last_peering_reset();
+  reset_interval_flush();
+
+  vector oldacting, oldup;
+  int oldrole = get_role();
+
+  unreg_next_scrub();
+
+  pg_shard_t old_acting_primary = get_primary();
+  pg_shard_t old_up_primary = up_primary;
+  bool was_old_primary = is_primary();
+
+  acting.swap(oldacting);
+  up.swap(oldup);
+  init_primary_up_acting(
+    newup,
+    newacting,
+    new_up_primary,
+    new_acting_primary);
+
+  if (info.stats.up != up ||
+      info.stats.acting != acting ||
+      info.stats.up_primary != new_up_primary ||
+      info.stats.acting_primary != new_acting_primary) {
+    info.stats.up = up;
+    info.stats.up_primary = new_up_primary;
+    info.stats.acting = acting;
+    info.stats.acting_primary = new_acting_primary;
+    info.stats.mapping_epoch = info.history.same_interval_since;
+  }
+
+  // This will now be remapped during a backfill in cases
+  // that it would not have been before.
+  if (up != acting)
+    state_set(PG_STATE_REMAPPED);
+  else
+    state_clear(PG_STATE_REMAPPED);
+
+  int role = osdmap->calc_pg_role(osd->whoami, acting, acting.size());
+  if (pool.info.is_replicated() || role == pg_whoami.shard)
+    set_role(role);
+  else
+    set_role(-1);
+
+  reg_next_scrub();
+
+  // set CREATING bit until we have peered for the first time.
+  if (is_primary() && info.history.last_epoch_started == 0)
+    state_set(PG_STATE_CREATING);
+  else
+    state_clear(PG_STATE_CREATING);
+
+  // did acting, up, primary|acker change?
+  if (!lastmap) {
+    dout(10) << " no lastmap" << dendl;
+    dirty_info = true;
+    dirty_big_info = true;
+  } else {
+    std::stringstream debug;
+    bool new_interval = pg_interval_t::check_new_interval(
+      old_acting_primary.osd,
+      new_acting_primary,
+      oldacting, newacting,
+      old_up_primary.osd,
+      new_up_primary,
+      oldup, newup,
+      info.history.same_interval_since,
+      info.history.last_epoch_clean,
+      osdmap,
+      lastmap,
+      info.pgid.pool(),
+      info.pgid.pgid,
+      &past_intervals,
+      &debug);
+    dout(10) << __func__ << ": check_new_interval output: "
+	     << debug.str() << dendl;
+    if (new_interval) {
+      dout(10) << " noting past " << past_intervals.rbegin()->second << dendl;
+      dirty_info = true;
+      dirty_big_info = true;
+    }
+  }
+
+  if (old_up_primary != up_primary ||
+      old_acting_primary != primary ||
+      oldacting != acting ||
+      oldup != up ||
+      is_split(lastmap, osdmap)) {
+    info.history.same_interval_since = osdmap->get_epoch();
+  }
+  if (old_up_primary != up_primary ||
+      oldup != up) {
+    info.history.same_up_since = osdmap->get_epoch();
+  }
+  // this comparison includes primary rank via pg_shard_t
+  if (old_acting_primary != get_primary()) {
+    info.history.same_primary_since = osdmap->get_epoch();
+  }
+
+  dout(10) << " up " << oldup << " -> " << up 
+	   << ", acting " << oldacting << " -> " << acting 
+	   << ", acting_primary " << old_acting_primary << " -> " << new_acting_primary
+	   << ", up_primary " << old_up_primary << " -> " << new_up_primary
+	   << ", role " << oldrole << " -> " << role << dendl; 
+
+  // deactivate.
+  state_clear(PG_STATE_ACTIVE);
+  state_clear(PG_STATE_DOWN);
+  state_clear(PG_STATE_RECOVERY_WAIT);
+  state_clear(PG_STATE_RECOVERING);
+
+  peer_missing.clear();
+  peer_purged.clear();
+  actingbackfill.clear();
+
+  // reset primary state?
+  if (was_old_primary || is_primary())
+    clear_primary_state();
+
+    
+  // pg->on_*
+  on_change(t);
+
+  assert(!deleting);
+
+  // should we tell the primary we are here?
+  send_notify = !is_primary();
+
+  if (role != oldrole ||
+      was_old_primary != is_primary()) {
+    // did primary change?
+    if (was_old_primary != is_primary()) {
+      state_clear(PG_STATE_CLEAN);
+      clear_publish_stats();
+	
+      // take replay queue waiters
+      list ls;
+      for (map::iterator it = replay_queue.begin();
+	   it != replay_queue.end();
+	   ++it)
+	ls.push_back(it->second);
+      replay_queue.clear();
+      requeue_ops(ls);
+    }
+
+    on_role_change();
+
+    // take active waiters
+    requeue_ops(waiting_for_active);
+
+  } else {
+    // no role change.
+    // did primary change?
+    if (get_primary() != old_acting_primary) {    
+      dout(10) << *this << " " << oldacting << " -> " << acting 
+	       << ", acting primary " 
+	       << old_acting_primary << " -> " << get_primary() 
+	       << dendl;
+    } else {
+      // primary is the same.
+      if (is_primary()) {
+	// i am (still) primary. but my replica set changed.
+	state_clear(PG_STATE_CLEAN);
+	  
+	dout(10) << oldacting << " -> " << acting
+		 << ", replicas changed" << dendl;
+      }
+    }
+  }
+  cancel_recovery();
+
+  if (acting.empty() && !up.empty() && up_primary == pg_whoami) {
+    dout(10) << " acting empty, but i am up[0], clearing pg_temp" << dendl;
+    osd->queue_want_pg_temp(info.pgid.pgid, acting);
+  }
+}
+
+void PG::proc_primary_info(ObjectStore::Transaction &t, const pg_info_t &oinfo)
+{
+  assert(!is_primary());
+
+  unreg_next_scrub();
+  if (info.history.merge(oinfo.history))
+    dirty_info = true;
+  reg_next_scrub();
+
+  if (last_complete_ondisk.epoch >= info.history.last_epoch_started) {
+    // DEBUG: verify that the snaps are empty in snap_mapper
+    if (cct->_conf->osd_debug_verify_snaps_on_info) {
+      interval_set p;
+      p.union_of(oinfo.purged_snaps, info.purged_snaps);
+      p.subtract(info.purged_snaps);
+      if (!p.empty()) {
+	for (interval_set::iterator i = p.begin();
+	     i != p.end();
+	     ++i) {
+	  for (snapid_t snap = i.get_start();
+	       snap != i.get_len() + i.get_start();
+	       ++snap) {
+	    hobject_t hoid;
+	    int r = snap_mapper.get_next_object_to_trim(snap, &hoid);
+	    if (r != 0 && r != -ENOENT) {
+	      derr << __func__ << ": snap_mapper get_next_object_to_trim returned "
+		   << cpp_strerror(r) << dendl;
+	      assert(0);
+	    } else if (r != -ENOENT) {
+	      derr << __func__ << ": snap_mapper get_next_object_to_trim returned "
+		   << cpp_strerror(r) << " for object "
+		   << hoid << " on snap " << snap
+		   << " which should have been fully trimmed " << dendl;
+	      assert(0);
+	    }
+	  }
+	}
+      }
+    }
+    info.purged_snaps = oinfo.purged_snaps;
+    dirty_info = true;
+    dirty_big_info = true;
+  }
+}
+
+ostream& operator<<(ostream& out, const PG& pg)
+{
+  out << "pg[" << pg.info
+      << " " << pg.up;
+  if (pg.acting != pg.up)
+    out << "/" << pg.acting;
+  out << " r=" << pg.get_role();
+  out << " lpr=" << pg.get_last_peering_reset();
+
+  if (!pg.past_intervals.empty()) {
+    out << " pi=" << pg.past_intervals.begin()->first << "-" << pg.past_intervals.rbegin()->second.last
+	<< "/" << pg.past_intervals.size();
+  }
+
+  if (pg.is_active() &&
+      pg.last_update_ondisk != pg.info.last_update)
+    out << " luod=" << pg.last_update_ondisk;
+
+  if (pg.recovery_ops_active)
+    out << " rops=" << pg.recovery_ops_active;
+
+  if (pg.pg_log.get_tail() != pg.info.log_tail ||
+      pg.pg_log.get_head() != pg.info.last_update)
+    out << " (info mismatch, " << pg.pg_log.get_log() << ")";
+
+  if (!pg.pg_log.get_log().empty()) {
+    if ((pg.pg_log.get_log().log.begin()->version <= pg.pg_log.get_tail())) {
+      out << " (log bound mismatch, actual=["
+	  << pg.pg_log.get_log().log.begin()->version << ","
+	  << pg.pg_log.get_log().log.rbegin()->version << "]";
+      out << ")";
+    }
+  }
+
+  if (!pg.backfill_targets.empty())
+    out << " bft=" << pg.backfill_targets;
+  out << " crt=" << pg.pg_log.get_log().can_rollback_to;
+
+  if (pg.last_complete_ondisk != pg.info.last_complete)
+    out << " lcod " << pg.last_complete_ondisk;
+
+  if (pg.is_primary()) {
+    out << " mlcod " << pg.min_last_complete_ondisk;
+  }
+
+  out << " " << pg_state_string(pg.get_state());
+  if (pg.should_send_notify())
+    out << " NOTIFY";
+
+  if (pg.scrubber.must_repair)
+    out << " MUST_REPAIR";
+  if (pg.scrubber.must_deep_scrub)
+    out << " MUST_DEEP_SCRUB";
+  if (pg.scrubber.must_scrub)
+    out << " MUST_SCRUB";
+
+  //out << " (" << pg.pg_log.get_tail() << "," << pg.pg_log.get_head() << "]";
+  if (pg.pg_log.get_missing().num_missing()) {
+    out << " m=" << pg.pg_log.get_missing().num_missing();
+    if (pg.is_primary()) {
+      int unfound = pg.get_num_unfound();
+      if (unfound)
+	out << " u=" << unfound;
+    }
+  }
+  if (pg.snap_trimq.size())
+    out << " snaptrimq=" << pg.snap_trimq;
+
+  out << "]";
+
+
+  return out;
+}
+
+bool PG::can_discard_op(OpRequestRef op)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  if (OSD::op_is_discardable(m)) {
+    dout(20) << " discard " << *m << dendl;
+    return true;
+  }
+
+  if (m->get_map_epoch() < info.history.same_primary_since) {
+    dout(7) << " changed after " << m->get_map_epoch()
+	    << ", dropping " << *m << dendl;
+    return true;
+  }
+
+  if (m->get_map_epoch() < pool.info.last_force_op_resend &&
+      m->get_connection()->has_feature(CEPH_FEATURE_OSD_POOLRESEND)) {
+    dout(7) << __func__ << " sent before last_force_op_resend "
+	    << pool.info.last_force_op_resend << ", dropping" << *m << dendl;
+    return true;
+  }
+
+  if ((m->get_flags() & (CEPH_OSD_FLAG_BALANCE_READS |
+			 CEPH_OSD_FLAG_LOCALIZE_READS)) &&
+      op->may_read() &&
+      !(op->may_write() || op->may_cache())) {
+    // balanced reads; any replica will do
+    if (!(is_primary() || is_replica())) {
+      osd->handle_misdirected_op(this, op);
+      return true;
+    }
+  } else {
+    // normal case; must be primary
+    if (!is_primary()) {
+      osd->handle_misdirected_op(this, op);
+      return true;
+    }
+  }
+  if (is_replay()) {
+    if (m->get_version().version > 0) {
+      dout(7) << " queueing replay at " << m->get_version()
+	      << " for " << *m << dendl;
+      replay_queue[m->get_version()] = op;
+      op->mark_delayed("waiting for replay");
+      return true;
+    }
+  }
+  return false;
+}
+
+template
+bool PG::can_discard_replica_op(OpRequestRef op)
+{
+  T *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSGTYPE);
+
+  /* Mostly, this overlaps with the old_peering_msg
+   * condition.  An important exception is pushes
+   * sent by replicas not in the acting set, since
+   * if such a replica goes down it does not cause
+   * a new interval. */
+  int from = m->get_source().num();
+  if (get_osdmap()->get_down_at(from) >= m->map_epoch)
+    return true;
+
+  // same pg?
+  //  if pg changes _at all_, we reset and repeer!
+  if (old_peering_msg(m->map_epoch, m->map_epoch)) {
+    dout(10) << "can_discard_replica_op pg changed " << info.history
+	     << " after " << m->map_epoch
+	     << ", dropping" << dendl;
+    return true;
+  }
+  return false;
+}
+
+bool PG::can_discard_scan(OpRequestRef op)
+{
+  MOSDPGScan *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_SCAN);
+
+  if (old_peering_msg(m->map_epoch, m->query_epoch)) {
+    dout(10) << " got old scan, ignoring" << dendl;
+    return true;
+  }
+  return false;
+}
+
+bool PG::can_discard_backfill(OpRequestRef op)
+{
+  MOSDPGBackfill *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_BACKFILL);
+
+  if (old_peering_msg(m->map_epoch, m->query_epoch)) {
+    dout(10) << " got old backfill, ignoring" << dendl;
+    return true;
+  }
+
+  return false;
+
+}
+
+bool PG::can_discard_request(OpRequestRef op)
+{
+  switch (op->get_req()->get_type()) {
+  case CEPH_MSG_OSD_OP:
+    return can_discard_op(op);
+  case MSG_OSD_SUBOP:
+    return can_discard_replica_op(op);
+  case MSG_OSD_PG_PUSH:
+    return can_discard_replica_op(op);
+  case MSG_OSD_PG_PULL:
+    return can_discard_replica_op(op);
+  case MSG_OSD_PG_PUSH_REPLY:
+    return can_discard_replica_op(op);
+  case MSG_OSD_SUBOPREPLY:
+    return can_discard_replica_op(op);
+
+  case MSG_OSD_EC_WRITE:
+    return can_discard_replica_op(op);
+  case MSG_OSD_EC_WRITE_REPLY:
+    return can_discard_replica_op(op);
+  case MSG_OSD_EC_READ:
+    return can_discard_replica_op(op);
+  case MSG_OSD_EC_READ_REPLY:
+    return can_discard_replica_op(op);
+
+  case MSG_OSD_PG_SCAN:
+    return can_discard_scan(op);
+  case MSG_OSD_PG_BACKFILL:
+    return can_discard_backfill(op);
+  }
+  return true;
+}
+
+bool PG::split_request(OpRequestRef op, unsigned match, unsigned bits)
+{
+  unsigned mask = ~((~0)<get_req()->get_type()) {
+  case CEPH_MSG_OSD_OP:
+    return (static_cast(op->get_req())->get_pg().m_seed & mask) == match;
+  }
+  return false;
+}
+
+bool PG::op_must_wait_for_map(OSDMapRef curmap, OpRequestRef op)
+{
+  switch (op->get_req()->get_type()) {
+  case CEPH_MSG_OSD_OP:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->get_map_epoch());
+
+  case MSG_OSD_SUBOP:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_SUBOPREPLY:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_PG_SCAN:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_PG_BACKFILL:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_PG_PUSH:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_PG_PULL:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_PG_PUSH_REPLY:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_EC_WRITE:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_EC_WRITE_REPLY:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_EC_READ:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+
+  case MSG_OSD_EC_READ_REPLY:
+    return !have_same_or_newer_map(
+      curmap,
+      static_cast(op->get_req())->map_epoch);
+  }
+  assert(0);
+  return false;
+}
+
+void PG::take_waiters()
+{
+  dout(10) << "take_waiters" << dendl;
+  take_op_map_waiters();
+  for (list::iterator i = peering_waiters.begin();
+       i != peering_waiters.end();
+       ++i) osd->queue_for_peering(this);
+  peering_queue.splice(peering_queue.begin(), peering_waiters,
+		       peering_waiters.begin(), peering_waiters.end());
+}
+
+void PG::handle_peering_event(CephPeeringEvtRef evt, RecoveryCtx *rctx)
+{
+  dout(10) << "handle_peering_event: " << evt->get_desc() << dendl;
+  if (!have_same_or_newer_map(evt->get_epoch_sent())) {
+    dout(10) << "deferring event " << evt->get_desc() << dendl;
+    peering_waiters.push_back(evt);
+    return;
+  }
+  if (old_peering_evt(evt))
+    return;
+  recovery_state.handle_event(evt, rctx);
+}
+
+void PG::queue_peering_event(CephPeeringEvtRef evt)
+{
+  if (old_peering_evt(evt))
+    return;
+  peering_queue.push_back(evt);
+  osd->queue_for_peering(this);
+}
+
+void PG::queue_notify(epoch_t msg_epoch,
+		      epoch_t query_epoch,
+		      pg_shard_t from, pg_notify_t& i)
+{
+  dout(10) << "notify " << i << " from replica " << from << dendl;
+  queue_peering_event(
+    CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
+					 MNotifyRec(from, i))));
+}
+
+void PG::queue_info(epoch_t msg_epoch,
+		     epoch_t query_epoch,
+		     pg_shard_t from, pg_info_t& i)
+{
+  dout(10) << "info " << i << " from replica " << from << dendl;
+  queue_peering_event(
+    CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
+					 MInfoRec(from, i, msg_epoch))));
+}
+
+void PG::queue_log(epoch_t msg_epoch,
+		   epoch_t query_epoch,
+		   pg_shard_t from,
+		   MOSDPGLog *msg)
+{
+  dout(10) << "log " << *msg << " from replica " << from << dendl;
+  queue_peering_event(
+    CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
+					 MLogRec(from, msg))));
+}
+
+void PG::queue_null(epoch_t msg_epoch,
+		    epoch_t query_epoch)
+{
+  dout(10) << "null" << dendl;
+  queue_peering_event(
+    CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
+					 NullEvt())));
+}
+
+void PG::queue_flushed(epoch_t e)
+{
+  dout(10) << "flushed" << dendl;
+  queue_peering_event(
+    CephPeeringEvtRef(new CephPeeringEvt(e, e,
+					 FlushedEvt())));
+}
+
+void PG::queue_query(epoch_t msg_epoch,
+		     epoch_t query_epoch,
+		     pg_shard_t from, const pg_query_t& q)
+{
+  dout(10) << "handle_query " << q << " from replica " << from << dendl;
+  queue_peering_event(
+    CephPeeringEvtRef(new CephPeeringEvt(msg_epoch, query_epoch,
+					 MQuery(from, q, query_epoch))));
+}
+
+void PG::handle_advance_map(
+  OSDMapRef osdmap, OSDMapRef lastmap,
+  vector& newup, int up_primary,
+  vector& newacting, int acting_primary,
+  RecoveryCtx *rctx)
+{
+  assert(lastmap->get_epoch() == osdmap_ref->get_epoch());
+  assert(lastmap == osdmap_ref);
+  dout(10) << "handle_advance_map "
+	   << newup << "/" << newacting
+	   << " -- " << up_primary << "/" << acting_primary
+	   << dendl;
+  update_osdmap_ref(osdmap);
+  pool.update(osdmap);
+  if (pool.info.last_change == osdmap_ref->get_epoch())
+    on_pool_change();
+  AdvMap evt(
+    osdmap, lastmap, newup, up_primary,
+    newacting, acting_primary);
+  recovery_state.handle_event(evt, rctx);
+}
+
+void PG::handle_activate_map(RecoveryCtx *rctx)
+{
+  dout(10) << "handle_activate_map " << dendl;
+  ActMap evt;
+  recovery_state.handle_event(evt, rctx);
+  if (osdmap_ref->get_epoch() - last_persisted_osdmap_ref->get_epoch() >
+    cct->_conf->osd_pg_epoch_persisted_max_stale) {
+    dout(20) << __func__ << ": Dirtying info: last_persisted is "
+	     << last_persisted_osdmap_ref->get_epoch()
+	     << " while current is " << osdmap_ref->get_epoch() << dendl;
+    dirty_info = true;
+  } else {
+    dout(20) << __func__ << ": Not dirtying info: last_persisted is "
+	     << last_persisted_osdmap_ref->get_epoch()
+	     << " while current is " << osdmap_ref->get_epoch() << dendl;
+  }
+  if (osdmap_ref->check_new_blacklist_entries()) check_blacklisted_watchers();
+}
+
+void PG::handle_loaded(RecoveryCtx *rctx)
+{
+  dout(10) << "handle_loaded" << dendl;
+  Load evt;
+  recovery_state.handle_event(evt, rctx);
+}
+
+void PG::handle_create(RecoveryCtx *rctx)
+{
+  dout(10) << "handle_create" << dendl;
+  Initialize evt;
+  recovery_state.handle_event(evt, rctx);
+  ActMap evt2;
+  recovery_state.handle_event(evt2, rctx);
+}
+
+void PG::handle_query_state(Formatter *f)
+{
+  dout(10) << "handle_query_state" << dendl;
+  QueryState q(f);
+  recovery_state.handle_event(q, 0);
+}
+
+
+
+std::ostream& operator<<(std::ostream& oss,
+			 const struct PG::PriorSet &prior)
+{
+  oss << "PriorSet[probe=" << prior.probe << " "
+      << "down=" << prior.down << " "
+      << "blocked_by=" << prior.blocked_by << "]";
+  return oss;
+}
+
+/*------------ Recovery State Machine----------------*/
+#undef dout_prefix
+#define dout_prefix (*_dout << context< RecoveryMachine >().pg->gen_prefix() \
+		     << "state<" << get_state_name() << ">: ")
+
+/*------Crashed-------*/
+PG::RecoveryState::Crashed::Crashed(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Crashed")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  assert(0 == "we got a bad state machine event");
+}
+
+
+/*------Initial-------*/
+PG::RecoveryState::Initial::Initial(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Initial")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+boost::statechart::result PG::RecoveryState::Initial::react(const Load& l)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+
+  // do we tell someone we're here?
+  pg->send_notify = (!pg->is_primary());
+
+  return transit< Reset >();
+}
+
+boost::statechart::result PG::RecoveryState::Initial::react(const MNotifyRec& notify)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->proc_replica_info(notify.from, notify.notify.info);
+  pg->update_heartbeat_peers();
+  pg->set_last_peering_reset();
+  return transit< Primary >();
+}
+
+boost::statechart::result PG::RecoveryState::Initial::react(const MInfoRec& i)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(!pg->is_primary());
+  post_event(i);
+  return transit< Stray >();
+}
+
+boost::statechart::result PG::RecoveryState::Initial::react(const MLogRec& i)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(!pg->is_primary());
+  post_event(i);
+  return transit< Stray >();
+}
+
+void PG::RecoveryState::Initial::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_initial_latency, dur);
+}
+
+/*------Started-------*/
+PG::RecoveryState::Started::Started(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+boost::statechart::result
+PG::RecoveryState::Started::react(const IntervalFlush&)
+{
+  dout(10) << "Ending blocked outgoing recovery messages" << dendl;
+  context< RecoveryMachine >().pg->recovery_state.end_block_outgoing();
+  return discard_event();
+}
+
+
+boost::statechart::result
+PG::RecoveryState::Started::react(const FlushedEvt&)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->on_flushed();
+  return discard_event();
+}
+
+
+boost::statechart::result PG::RecoveryState::Started::react(const AdvMap& advmap)
+{
+  dout(10) << "Started advmap" << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  if (pg->acting_up_affected(
+	advmap.up_primary,
+	advmap.acting_primary,
+	advmap.newup,
+	advmap.newacting) ||
+      pg->is_split(advmap.lastmap, advmap.osdmap)) {
+    dout(10) << "up or acting affected, transitioning to Reset" << dendl;
+    post_event(advmap);
+    return transit< Reset >();
+  }
+  pg->remove_down_peer_info(advmap.osdmap);
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Started::react(const QueryState& q)
+{
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+  q.f->close_section();
+  return discard_event();
+}
+
+void PG::RecoveryState::Started::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_started_latency, dur);
+}
+
+/*--------Reset---------*/
+PG::RecoveryState::Reset::Reset(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Reset")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+
+  pg->flushes_in_progress = 0;
+  pg->set_last_peering_reset();
+}
+
+boost::statechart::result
+PG::RecoveryState::Reset::react(const FlushedEvt&)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->on_flushed();
+  return discard_event();
+}
+
+boost::statechart::result
+PG::RecoveryState::Reset::react(const IntervalFlush&)
+{
+  dout(10) << "Ending blocked outgoing recovery messages" << dendl;
+  context< RecoveryMachine >().pg->recovery_state.end_block_outgoing();
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Reset::react(const AdvMap& advmap)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  dout(10) << "Reset advmap" << dendl;
+
+  // make sure we have past_intervals filled in.  hopefully this will happen
+  // _before_ we are active.
+  pg->generate_past_intervals();
+
+  if (pg->acting_up_affected(
+	advmap.up_primary,
+	advmap.acting_primary,
+	advmap.newup,
+	advmap.newacting) ||
+      pg->is_split(advmap.lastmap, advmap.osdmap)) {
+    dout(10) << "up or acting affected, calling start_peering_interval again"
+	     << dendl;
+    pg->start_peering_interval(
+      advmap.lastmap,
+      advmap.newup, advmap.up_primary,
+      advmap.newacting, advmap.acting_primary,
+      context< RecoveryMachine >().get_cur_transaction());
+  }
+  pg->remove_down_peer_info(advmap.osdmap);
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Reset::react(const ActMap&)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  if (pg->should_send_notify() && pg->get_primary().osd >= 0) {
+    context< RecoveryMachine >().send_notify(
+      pg->get_primary(),
+      pg_notify_t(
+	pg->get_primary().shard, pg->pg_whoami.shard,
+	pg->get_osdmap()->get_epoch(),
+	pg->get_osdmap()->get_epoch(),
+	pg->info),
+      pg->past_intervals);
+  }
+
+  pg->update_heartbeat_peers();
+  pg->take_waiters();
+
+  return transit< Started >();
+}
+
+boost::statechart::result PG::RecoveryState::Reset::react(const QueryState& q)
+{
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+  q.f->close_section();
+  return discard_event();
+}
+
+void PG::RecoveryState::Reset::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_reset_latency, dur);
+}
+
+/*-------Start---------*/
+PG::RecoveryState::Start::Start(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Start")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  if (pg->is_primary()) {
+    dout(1) << "transitioning to Primary" << dendl;
+    post_event(MakePrimary());
+  } else { //is_stray
+    dout(1) << "transitioning to Stray" << dendl; 
+    post_event(MakeStray());
+  }
+}
+
+void PG::RecoveryState::Start::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_start_latency, dur);
+}
+
+/*---------Primary--------*/
+PG::RecoveryState::Primary::Primary(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(pg->want_acting.empty());
+}
+
+boost::statechart::result PG::RecoveryState::Primary::react(const MNotifyRec& notevt)
+{
+  dout(7) << "handle_pg_notify from osd." << notevt.from << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  if (pg->peer_info.count(notevt.from) &&
+      pg->peer_info[notevt.from].last_update == notevt.notify.info.last_update) {
+    dout(10) << *pg << " got dup osd." << notevt.from << " info " << notevt.notify.info
+	     << ", identical to ours" << dendl;
+  } else {
+    pg->proc_replica_info(notevt.from, notevt.notify.info);
+  }
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Primary::react(const ActMap&)
+{
+  dout(7) << "handle ActMap primary" << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->publish_stats_to_osd();
+  pg->take_waiters();
+  return discard_event();
+}
+
+void PG::RecoveryState::Primary::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->want_acting.clear();
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_primary_latency, dur);
+}
+
+/*---------Peering--------*/
+PG::RecoveryState::Peering::Peering(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(!pg->is_active());
+  assert(!pg->is_peering());
+  assert(pg->is_primary());
+  pg->state_set(PG_STATE_PEERING);
+}
+
+boost::statechart::result PG::RecoveryState::Peering::react(const AdvMap& advmap) 
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  dout(10) << "Peering advmap" << dendl;
+  if (prior_set.get()->affected_by_map(advmap.osdmap, pg)) {
+    dout(1) << "Peering, affected_by_map, going to Reset" << dendl;
+    post_event(advmap);
+    return transit< Reset >();
+  }
+  
+  pg->adjust_need_up_thru(advmap.osdmap);
+  
+  return forward_event();
+}
+
+boost::statechart::result PG::RecoveryState::Peering::react(const QueryState& q)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+
+  q.f->open_array_section("past_intervals");
+  for (map::iterator p = pg->past_intervals.begin();
+       p != pg->past_intervals.end();
+       ++p) {
+    q.f->open_object_section("past_interval");
+    p->second.dump(q.f);
+    q.f->close_section();
+  }
+  q.f->close_section();
+
+  q.f->open_array_section("probing_osds");
+  for (set::iterator p = prior_set->probe.begin();
+       p != prior_set->probe.end();
+       ++p)
+    q.f->dump_stream("osd") << *p;
+  q.f->close_section();
+
+  if (prior_set->pg_down)
+    q.f->dump_string("blocked", "peering is blocked due to down osds");
+
+  q.f->open_array_section("down_osds_we_would_probe");
+  for (set::iterator p = prior_set->down.begin();
+       p != prior_set->down.end();
+       ++p)
+    q.f->dump_int("osd", *p);
+  q.f->close_section();
+
+  q.f->open_array_section("peering_blocked_by");
+  for (map::iterator p = prior_set->blocked_by.begin();
+       p != prior_set->blocked_by.end();
+       ++p) {
+    q.f->open_object_section("osd");
+    q.f->dump_int("osd", p->first);
+    q.f->dump_int("current_lost_at", p->second);
+    q.f->dump_string("comment", "starting or marking this osd lost may let us proceed");
+    q.f->close_section();
+  }
+  q.f->close_section();
+
+  q.f->close_section();
+  return forward_event();
+}
+
+void PG::RecoveryState::Peering::exit()
+{
+  dout(10) << "Leaving Peering" << dendl;
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_clear(PG_STATE_PEERING);
+  pg->clear_probe_targets();
+
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_peering_latency, dur);
+}
+
+
+/*------Backfilling-------*/
+PG::RecoveryState::Backfilling::Backfilling(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Backfilling")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->backfill_reserved = true;
+  pg->osd->queue_for_recovery(pg);
+  pg->state_clear(PG_STATE_BACKFILL_TOOFULL);
+  pg->state_clear(PG_STATE_BACKFILL_WAIT);
+  pg->state_set(PG_STATE_BACKFILL);
+}
+
+boost::statechart::result
+PG::RecoveryState::Backfilling::react(const RemoteReservationRejected &)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->local_reserver.cancel_reservation(pg->info.pgid);
+  pg->state_set(PG_STATE_BACKFILL_TOOFULL);
+
+  pg->osd->recovery_wq.dequeue(pg);
+
+  pg->schedule_backfill_full_retry();
+  return transit();
+}
+
+void PG::RecoveryState::Backfilling::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->backfill_reserved = false;
+  pg->backfill_reserving = false;
+  pg->state_clear(PG_STATE_BACKFILL);
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_backfilling_latency, dur);
+}
+
+/*--WaitRemoteBackfillReserved--*/
+
+PG::RecoveryState::WaitRemoteBackfillReserved::WaitRemoteBackfillReserved(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteBackfillReserved"),
+    backfill_osd_it(context< Active >().remote_shards_to_reserve_backfill.begin())
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_set(PG_STATE_BACKFILL_WAIT);
+  post_event(RemoteBackfillReserved());
+}
+
+boost::statechart::result
+PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteBackfillReserved &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+
+  if (backfill_osd_it != context< Active >().remote_shards_to_reserve_backfill.end()) {
+    //The primary never backfills itself
+    assert(*backfill_osd_it != pg->pg_whoami);
+    ConnectionRef con = pg->osd->get_con_osd_cluster(
+      backfill_osd_it->osd, pg->get_osdmap()->get_epoch());
+    if (con) {
+      if (con->has_feature(CEPH_FEATURE_BACKFILL_RESERVATION)) {
+        unsigned priority = pg->is_degraded() ? OSDService::BACKFILL_HIGH
+	  : OSDService::BACKFILL_LOW;
+        pg->osd->send_message_osd_cluster(
+          new MBackfillReserve(
+	  MBackfillReserve::REQUEST,
+	  spg_t(pg->info.pgid.pgid, backfill_osd_it->shard),
+	  pg->get_osdmap()->get_epoch(), priority),
+	con.get());
+      } else {
+        post_event(RemoteBackfillReserved());
+      }
+    }
+    ++backfill_osd_it;
+  } else {
+    post_event(AllBackfillsReserved());
+  }
+  return discard_event();
+}
+
+void PG::RecoveryState::WaitRemoteBackfillReserved::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_waitremotebackfillreserved_latency, dur);
+}
+
+boost::statechart::result
+PG::RecoveryState::WaitRemoteBackfillReserved::react(const RemoteReservationRejected &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->local_reserver.cancel_reservation(pg->info.pgid);
+
+  // Send REJECT to all previously acquired reservations
+  set::const_iterator it, begin, end, next;
+  begin = context< Active >().remote_shards_to_reserve_backfill.begin();
+  end = context< Active >().remote_shards_to_reserve_backfill.end();
+  assert(begin != end);
+  for (next = it = begin, ++next ; next != backfill_osd_it; ++it, ++next) {
+    //The primary never backfills itself
+    assert(*it != pg->pg_whoami);
+    ConnectionRef con = pg->osd->get_con_osd_cluster(
+      it->osd, pg->get_osdmap()->get_epoch());
+    if (con) {
+      if (con->has_feature(CEPH_FEATURE_BACKFILL_RESERVATION)) {
+        pg->osd->send_message_osd_cluster(
+          new MBackfillReserve(
+	  MBackfillReserve::REJECT,
+	  spg_t(pg->info.pgid.pgid, it->shard),
+	  pg->get_osdmap()->get_epoch()),
+	con.get());
+      }
+    }
+  }
+
+  pg->state_clear(PG_STATE_BACKFILL_WAIT);
+  pg->state_set(PG_STATE_BACKFILL_TOOFULL);
+
+  pg->schedule_backfill_full_retry();
+
+  return transit();
+}
+
+/*--WaitLocalBackfillReserved--*/
+PG::RecoveryState::WaitLocalBackfillReserved::WaitLocalBackfillReserved(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitLocalBackfillReserved")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_set(PG_STATE_BACKFILL_WAIT);
+  pg->osd->local_reserver.request_reservation(
+    pg->info.pgid,
+    new QueuePeeringEvt(
+      pg, pg->get_osdmap()->get_epoch(),
+      LocalBackfillReserved()), pg->is_degraded() ? OSDService::BACKFILL_HIGH
+	 : OSDService::BACKFILL_LOW);
+}
+
+void PG::RecoveryState::WaitLocalBackfillReserved::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_waitlocalbackfillreserved_latency, dur);
+}
+
+/*----NotBackfilling------*/
+PG::RecoveryState::NotBackfilling::NotBackfilling(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/NotBackfilling")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+boost::statechart::result
+PG::RecoveryState::NotBackfilling::react(const RemoteBackfillReserved &evt)
+{
+  return discard_event();
+}
+
+boost::statechart::result
+PG::RecoveryState::NotBackfilling::react(const RemoteReservationRejected &evt)
+{
+  return discard_event();
+}
+
+void PG::RecoveryState::NotBackfilling::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_notbackfilling_latency, dur);
+}
+
+/*---RepNotRecovering----*/
+PG::RecoveryState::RepNotRecovering::RepNotRecovering(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepNotRecovering")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+void PG::RecoveryState::RepNotRecovering::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_repnotrecovering_latency, dur);
+}
+
+/*---RepWaitRecoveryReserved--*/
+PG::RecoveryState::RepWaitRecoveryReserved::RepWaitRecoveryReserved(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepWaitRecoveryReserved")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+
+  pg->osd->remote_reserver.request_reservation(
+    pg->info.pgid,
+    new QueuePeeringEvt(
+      pg, pg->get_osdmap()->get_epoch(),
+      RemoteRecoveryReserved()), OSDService::RECOVERY);
+}
+
+boost::statechart::result
+PG::RecoveryState::RepWaitRecoveryReserved::react(const RemoteRecoveryReserved &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->send_message_osd_cluster(
+    pg->primary.osd,
+    new MRecoveryReserve(
+      MRecoveryReserve::GRANT,
+      spg_t(pg->info.pgid.pgid, pg->primary.shard),
+      pg->get_osdmap()->get_epoch()),
+    pg->get_osdmap()->get_epoch());
+  return transit();
+}
+
+void PG::RecoveryState::RepWaitRecoveryReserved::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_repwaitrecoveryreserved_latency, dur);
+}
+
+/*-RepWaitBackfillReserved*/
+PG::RecoveryState::RepWaitBackfillReserved::RepWaitBackfillReserved(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepWaitBackfillReserved")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+boost::statechart::result 
+PG::RecoveryState::RepNotRecovering::react(const RequestBackfillPrio &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  double ratio, max_ratio;
+
+  if (g_conf->osd_debug_reject_backfill_probability > 0 &&
+      (rand()%1000 < (g_conf->osd_debug_reject_backfill_probability*1000.0))) {
+    dout(10) << "backfill reservation rejected: failure injection" << dendl;
+    post_event(RemoteReservationRejected());
+  } else if (pg->osd->too_full_for_backfill(&ratio, &max_ratio) &&
+      !pg->cct->_conf->osd_debug_skip_full_check_in_backfill_reservation) {
+    dout(10) << "backfill reservation rejected: full ratio is "
+	     << ratio << ", which is greater than max allowed ratio "
+	     << max_ratio << dendl;
+    post_event(RemoteReservationRejected());
+  } else {
+    pg->osd->remote_reserver.request_reservation(
+      pg->info.pgid,
+      new QueuePeeringEvt(
+        pg, pg->get_osdmap()->get_epoch(),
+        RemoteBackfillReserved()), evt.priority);
+  }
+  return transit();
+}
+
+void PG::RecoveryState::RepWaitBackfillReserved::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_repwaitbackfillreserved_latency, dur);
+}
+
+boost::statechart::result
+PG::RecoveryState::RepWaitBackfillReserved::react(const RemoteBackfillReserved &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->send_message_osd_cluster(
+    pg->primary.osd,
+    new MBackfillReserve(
+      MBackfillReserve::GRANT,
+      spg_t(pg->info.pgid.pgid, pg->primary.shard),
+      pg->get_osdmap()->get_epoch()),
+    pg->get_osdmap()->get_epoch());
+  return transit();
+}
+
+boost::statechart::result
+PG::RecoveryState::RepWaitBackfillReserved::react(const RemoteReservationRejected &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->reject_reservation();
+  return transit();
+}
+
+/*---RepRecovering-------*/
+PG::RecoveryState::RepRecovering::RepRecovering(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepRecovering")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+boost::statechart::result
+PG::RecoveryState::RepRecovering::react(const BackfillTooFull &)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->reject_reservation();
+  return transit();
+}
+
+void PG::RecoveryState::RepRecovering::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->remote_reserver.cancel_reservation(pg->info.pgid);
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_RepRecovering_latency, dur);
+}
+
+/*------Activating--------*/
+PG::RecoveryState::Activating::Activating(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Activating")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+void PG::RecoveryState::Activating::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_activating_latency, dur);
+}
+
+PG::RecoveryState::WaitLocalRecoveryReserved::WaitLocalRecoveryReserved(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitLocalRecoveryReserved")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_set(PG_STATE_RECOVERY_WAIT);
+  pg->osd->local_reserver.request_reservation(
+    pg->info.pgid,
+    new QueuePeeringEvt(
+      pg, pg->get_osdmap()->get_epoch(),
+      LocalRecoveryReserved()), OSDService::RECOVERY);
+}
+
+void PG::RecoveryState::WaitLocalRecoveryReserved::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_waitlocalrecoveryreserved_latency, dur);
+}
+
+PG::RecoveryState::WaitRemoteRecoveryReserved::WaitRemoteRecoveryReserved(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteRecoveryReserved"),
+    remote_recovery_reservation_it(context< Active >().remote_shards_to_reserve_recovery.begin())
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  post_event(RemoteRecoveryReserved());
+}
+
+boost::statechart::result
+PG::RecoveryState::WaitRemoteRecoveryReserved::react(const RemoteRecoveryReserved &evt) {
+  PG *pg = context< RecoveryMachine >().pg;
+
+  if (remote_recovery_reservation_it != context< Active >().remote_shards_to_reserve_recovery.end()) {
+    assert(*remote_recovery_reservation_it != pg->pg_whoami);
+  }
+
+  if (remote_recovery_reservation_it != context< Active >().remote_shards_to_reserve_recovery.end()) {
+    ConnectionRef con = pg->osd->get_con_osd_cluster(
+      remote_recovery_reservation_it->osd, pg->get_osdmap()->get_epoch());
+    if (con) {
+      if (con->has_feature(CEPH_FEATURE_RECOVERY_RESERVATION)) {
+	pg->osd->send_message_osd_cluster(
+          new MRecoveryReserve(
+	    MRecoveryReserve::REQUEST,
+	    spg_t(pg->info.pgid.pgid, remote_recovery_reservation_it->shard),
+	    pg->get_osdmap()->get_epoch()),
+	  con.get());
+      } else {
+	post_event(RemoteRecoveryReserved());
+      }
+    }
+    ++remote_recovery_reservation_it;
+  } else {
+    post_event(AllRemotesReserved());
+  }
+  return discard_event();
+}
+
+void PG::RecoveryState::WaitRemoteRecoveryReserved::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_waitremoterecoveryreserved_latency, dur);
+}
+
+PG::RecoveryState::Recovering::Recovering(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Recovering")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_clear(PG_STATE_RECOVERY_WAIT);
+  pg->state_set(PG_STATE_RECOVERING);
+  pg->osd->queue_for_recovery(pg);
+}
+
+void PG::RecoveryState::Recovering::release_reservations()
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(!pg->pg_log.get_missing().have_missing());
+
+  // release remote reservations
+  for (set::const_iterator i =
+	 context< Active >().remote_shards_to_reserve_recovery.begin();
+        i != context< Active >().remote_shards_to_reserve_recovery.end();
+        ++i) {
+    if (*i == pg->pg_whoami) // skip myself
+      continue;
+    ConnectionRef con = pg->osd->get_con_osd_cluster(
+      i->osd, pg->get_osdmap()->get_epoch());
+    if (con) {
+      if (con->has_feature(CEPH_FEATURE_RECOVERY_RESERVATION)) {
+	pg->osd->send_message_osd_cluster(
+          new MRecoveryReserve(
+	    MRecoveryReserve::RELEASE,
+	    spg_t(pg->info.pgid.pgid, i->shard),
+	    pg->get_osdmap()->get_epoch()),
+	  con.get());
+      }
+    }
+  }
+}
+
+boost::statechart::result
+PG::RecoveryState::Recovering::react(const AllReplicasRecovered &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_clear(PG_STATE_RECOVERING);
+  release_reservations();
+  return transit();
+}
+
+boost::statechart::result
+PG::RecoveryState::Recovering::react(const RequestBackfill &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_clear(PG_STATE_RECOVERING);
+  release_reservations();
+  return transit();
+}
+
+void PG::RecoveryState::Recovering::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_recovering_latency, dur);
+}
+
+PG::RecoveryState::Recovered::Recovered(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Recovered")
+{
+  pg_shard_t auth_log_shard;
+
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->local_reserver.cancel_reservation(pg->info.pgid);
+
+  // if we finished backfill, all acting are active; recheck if
+  // DEGRADED is appropriate.
+  assert(!pg->actingbackfill.empty());
+  if (pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid) <=
+      pg->actingbackfill.size())
+    pg->state_clear(PG_STATE_DEGRADED);
+
+  // adjust acting set?  (e.g. because backfill completed...)
+  if (pg->acting != pg->up && !pg->choose_acting(auth_log_shard))
+    assert(pg->want_acting.size());
+
+  assert(!pg->needs_recovery());
+
+  if (context< Active >().all_replicas_activated)
+    post_event(GoClean());
+}
+
+void PG::RecoveryState::Recovered::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_recovered_latency, dur);
+}
+
+PG::RecoveryState::Clean::Clean(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Clean")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+
+  if (pg->info.last_complete != pg->info.last_update) {
+    assert(0);
+  }
+  pg->finish_recovery(*context< RecoveryMachine >().get_on_safe_context_list());
+  pg->mark_clean();
+
+  pg->share_pg_info();
+  pg->publish_stats_to_osd();
+
+}
+
+void PG::RecoveryState::Clean::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->state_clear(PG_STATE_CLEAN);
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_clean_latency, dur);
+}
+
+template 
+set unique_osd_shard_set(const pg_shard_t & skip, const T &in)
+{
+  set osds_found;
+  set out;
+  for (typename T::const_iterator i = in.begin();
+       i != in.end();
+       ++i) {
+    if (*i != skip && !osds_found.count(i->osd)) {
+      osds_found.insert(i->osd);
+      out.insert(*i);
+    }
+  }
+  return out;
+}
+
+/*---------Active---------*/
+PG::RecoveryState::Active::Active(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active"),
+    remote_shards_to_reserve_recovery(
+      unique_osd_shard_set(
+	context< RecoveryMachine >().pg->pg_whoami,
+	context< RecoveryMachine >().pg->actingbackfill)),
+    remote_shards_to_reserve_backfill(
+      unique_osd_shard_set(
+	context< RecoveryMachine >().pg->pg_whoami,
+	context< RecoveryMachine >().pg->backfill_targets)),
+    all_replicas_activated(false)
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+
+  assert(!pg->backfill_reserving);
+  assert(!pg->backfill_reserved);
+  assert(pg->is_primary());
+  dout(10) << "In Active, about to call activate" << dendl;
+  pg->start_flush(
+    context< RecoveryMachine >().get_cur_transaction(),
+    context< RecoveryMachine >().get_on_applied_context_list(),
+    context< RecoveryMachine >().get_on_safe_context_list());
+  pg->activate(*context< RecoveryMachine >().get_cur_transaction(),
+	       pg->get_osdmap()->get_epoch(),
+	       *context< RecoveryMachine >().get_on_safe_context_list(),
+	       *context< RecoveryMachine >().get_query_map(),
+	       context< RecoveryMachine >().get_info_map(),
+	       context< RecoveryMachine >().get_recovery_ctx());
+  dout(10) << "Activate Finished" << dendl;
+}
+
+boost::statechart::result PG::RecoveryState::Active::react(const AdvMap& advmap)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  dout(10) << "Active advmap" << dendl;
+  if (!pg->pool.newly_removed_snaps.empty()) {
+    pg->snap_trimq.union_of(pg->pool.newly_removed_snaps);
+    dout(10) << *pg << " snap_trimq now " << pg->snap_trimq << dendl;
+    pg->dirty_info = true;
+    pg->dirty_big_info = true;
+  }
+
+  for (vector::iterator p = pg->want_acting.begin();
+       p != pg->want_acting.end(); ++p) {
+    if (!advmap.osdmap->is_up(*p)) {
+      assert((std::find(pg->acting.begin(), pg->acting.end(), *p) !=
+	      pg->acting.end()) ||
+	     (std::find(pg->up.begin(), pg->up.end(), *p) !=
+	      pg->up.end()));
+    }
+  }
+
+  /* Check for changes in pool size (if the acting set changed as a result,
+   * this does not matter) */
+  if (advmap.lastmap->get_pg_size(pg->info.pgid.pgid) !=
+      pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid)) {
+    if (pg->get_osdmap()->get_pg_size(pg->info.pgid.pgid) <= pg->acting.size())
+      pg->state_clear(PG_STATE_DEGRADED);
+    else
+      pg->state_set(PG_STATE_DEGRADED);
+    pg->publish_stats_to_osd(); // degraded may have changed
+  }
+
+  // if we haven't reported our PG stats in a long time, do so now.
+  if (pg->info.stats.reported_epoch + pg->cct->_conf->osd_pg_stat_report_interval_max < advmap.osdmap->get_epoch()) {
+    dout(20) << "reporting stats to osd after " << (advmap.osdmap->get_epoch() - pg->info.stats.reported_epoch)
+	     << " epochs" << dendl;
+    pg->publish_stats_to_osd();
+  }
+
+  return forward_event();
+}
+    
+boost::statechart::result PG::RecoveryState::Active::react(const ActMap&)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  dout(10) << "Active: handling ActMap" << dendl;
+  assert(pg->is_primary());
+
+  if (pg->have_unfound()) {
+    // object may have become unfound
+    pg->discover_all_missing(*context< RecoveryMachine >().get_query_map());
+  }
+
+  if (pg->cct->_conf->osd_check_for_log_corruption)
+    pg->check_log_for_corruption(pg->osd->store);
+
+  int unfound = pg->missing_loc.num_unfound();
+  if (unfound > 0 &&
+      pg->all_unfound_are_queried_or_lost(pg->get_osdmap())) {
+    if (pg->cct->_conf->osd_auto_mark_unfound_lost) {
+      pg->osd->clog.error() << pg->info.pgid << " has " << unfound
+			    << " objects unfound and apparently lost, would automatically marking lost but NOT IMPLEMENTED\n";
+      //pg->mark_all_unfound_lost(*context< RecoveryMachine >().get_cur_transaction());
+    } else
+      pg->osd->clog.error() << pg->info.pgid << " has " << unfound << " objects unfound and apparently lost\n";
+  }
+
+  if (!pg->snap_trimq.empty() &&
+      pg->is_clean()) {
+    dout(10) << "Active: queuing snap trim" << dendl;
+    pg->queue_snap_trim();
+  }
+
+  if (!pg->is_clean() &&
+      !pg->get_osdmap()->test_flag(CEPH_OSDMAP_NOBACKFILL)) {
+    pg->osd->queue_for_recovery(pg);
+  }
+  return forward_event();
+}
+
+boost::statechart::result PG::RecoveryState::Active::react(const MNotifyRec& notevt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(pg->is_primary());
+  if (pg->peer_info.count(notevt.from)) {
+    dout(10) << "Active: got notify from " << notevt.from 
+	     << ", already have info from that osd, ignoring" 
+	     << dendl;
+  } else if (pg->peer_purged.count(notevt.from)) {
+    dout(10) << "Active: got notify from " << notevt.from
+	     << ", already purged that peer, ignoring"
+	     << dendl;
+  } else {
+    dout(10) << "Active: got notify from " << notevt.from 
+	     << ", calling proc_replica_info and discover_all_missing"
+	     << dendl;
+    pg->proc_replica_info(notevt.from, notevt.notify.info);
+    if (pg->have_unfound()) {
+      pg->discover_all_missing(*context< RecoveryMachine >().get_query_map());
+    }
+  }
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Active::react(const MInfoRec& infoevt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(pg->is_primary());
+
+  assert(!pg->actingbackfill.empty());
+  // don't update history (yet) if we are active and primary; the replica
+  // may be telling us they have activated (and committed) but we can't
+  // share that until _everyone_ does the same.
+  if (pg->is_actingbackfill(infoevt.from)) {
+    assert(pg->info.history.last_epoch_started < 
+	   pg->info.history.same_interval_since);
+    assert(infoevt.info.history.last_epoch_started >= 
+	   pg->info.history.same_interval_since);
+    dout(10) << " peer osd." << infoevt.from << " activated and committed" 
+	     << dendl;
+    pg->peer_activated.insert(infoevt.from);
+
+    if (pg->peer_activated.size() == pg->actingbackfill.size()) {
+      pg->all_activated_and_committed();
+    }
+  }
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Active::react(const MLogRec& logevt)
+{
+  dout(10) << "searching osd." << logevt.from
+           << " log for unfound items" << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->proc_replica_log(
+    *context().get_cur_transaction(),
+    logevt.msg->info, logevt.msg->log, logevt.msg->missing, logevt.from);
+  bool got_missing = pg->search_for_missing(
+    pg->peer_info[logevt.from],
+    pg->peer_missing[logevt.from],
+    logevt.from,
+    context< RecoveryMachine >().get_recovery_ctx());
+  if (got_missing)
+    pg->osd->queue_for_recovery(pg);
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Active::react(const QueryState& q)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+
+  {
+    q.f->open_array_section("might_have_unfound");
+    for (set::iterator p = pg->might_have_unfound.begin();
+	 p != pg->might_have_unfound.end();
+	 ++p) {
+      q.f->open_object_section("osd");
+      q.f->dump_stream("osd") << *p;
+      if (pg->peer_missing.count(*p)) {
+	q.f->dump_string("status", "already probed");
+      } else if (pg->peer_missing_requested.count(*p)) {
+	q.f->dump_string("status", "querying");
+      } else if (!pg->get_osdmap()->is_up(p->osd)) {
+	q.f->dump_string("status", "osd is down");
+      } else {
+	q.f->dump_string("status", "not queried");
+      }
+      q.f->close_section();
+    }
+    q.f->close_section();
+  }
+  {
+    q.f->open_object_section("recovery_progress");
+    pg->dump_recovery_info(q.f);
+    q.f->close_section();
+  }
+
+  {
+    q.f->open_object_section("scrub");
+    q.f->dump_stream("scrubber.epoch_start") << pg->scrubber.epoch_start;
+    q.f->dump_int("scrubber.active", pg->scrubber.active);
+    q.f->dump_int("scrubber.block_writes", pg->scrubber.block_writes);
+    q.f->dump_int("scrubber.finalizing", pg->scrubber.finalizing);
+    q.f->dump_int("scrubber.waiting_on", pg->scrubber.waiting_on);
+    {
+      q.f->open_array_section("scrubber.waiting_on_whom");
+      for (set::iterator p = pg->scrubber.waiting_on_whom.begin();
+	   p != pg->scrubber.waiting_on_whom.end();
+	   ++p) {
+	q.f->dump_stream("shard") << *p;
+      }
+      q.f->close_section();
+    }
+    q.f->close_section();
+  }
+
+  q.f->close_section();
+  return forward_event();
+}
+
+boost::statechart::result PG::RecoveryState::Active::react(const AllReplicasActivated &evt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  all_replicas_activated = true;
+
+  pg->state_set(PG_STATE_ACTIVE);
+
+  pg->check_local();
+
+  // waiters
+  if (!pg->is_replay() && pg->flushes_in_progress == 0) {
+    pg->requeue_ops(pg->waiting_for_active);
+  }
+
+  pg->on_activate();
+
+  return discard_event();
+}
+
+void PG::RecoveryState::Active::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->local_reserver.cancel_reservation(pg->info.pgid);
+
+  pg->backfill_reserved = false;
+  pg->backfill_reserving = false;
+  pg->state_clear(PG_STATE_DEGRADED);
+  pg->state_clear(PG_STATE_BACKFILL_TOOFULL);
+  pg->state_clear(PG_STATE_BACKFILL_WAIT);
+  pg->state_clear(PG_STATE_RECOVERY_WAIT);
+  pg->state_clear(PG_STATE_REPLAY);
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_active_latency, dur);
+  pg->agent_stop();
+}
+
+/*------ReplicaActive-----*/
+PG::RecoveryState::ReplicaActive::ReplicaActive(my_context ctx) 
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->start_flush(
+    context< RecoveryMachine >().get_cur_transaction(),
+    context< RecoveryMachine >().get_on_applied_context_list(),
+    context< RecoveryMachine >().get_on_safe_context_list());
+}
+
+
+boost::statechart::result PG::RecoveryState::ReplicaActive::react(
+  const Activate& actevt) {
+  dout(10) << "In ReplicaActive, about to call activate" << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  map > query_map;
+  pg->activate(*context< RecoveryMachine >().get_cur_transaction(),
+	       actevt.query_epoch,
+	       *context< RecoveryMachine >().get_on_safe_context_list(),
+	       query_map, NULL, NULL);
+  dout(10) << "Activate Finished" << dendl;
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::ReplicaActive::react(const MInfoRec& infoevt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->proc_primary_info(*context().get_cur_transaction(),
+			infoevt.info);
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::ReplicaActive::react(const MLogRec& logevt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  dout(10) << "received log from " << logevt.from << dendl;
+  ObjectStore::Transaction* t = context().get_cur_transaction();
+  pg->merge_log(*t,logevt.msg->info, logevt.msg->log, logevt.from);
+  assert(pg->pg_log.get_head() == pg->info.last_update);
+
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::ReplicaActive::react(const ActMap&)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  if (pg->should_send_notify() && pg->get_primary().osd >= 0) {
+    context< RecoveryMachine >().send_notify(
+      pg->get_primary(),
+      pg_notify_t(
+	pg->get_primary().shard, pg->pg_whoami.shard,
+	pg->get_osdmap()->get_epoch(),
+	pg->get_osdmap()->get_epoch(),
+	pg->info),
+      pg->past_intervals);
+  }
+  pg->take_waiters();
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::ReplicaActive::react(const MQuery& query)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  if (query.query.type == pg_query_t::MISSING) {
+    pg->update_history_from_master(query.query.history);
+    pg->fulfill_log(query.from, query.query, query.query_epoch);
+  } // else: from prior to activation, safe to ignore
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::ReplicaActive::react(const QueryState& q)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+  q.f->dump_int("scrubber.finalizing", pg->scrubber.finalizing);
+  q.f->close_section();
+  return forward_event();
+}
+
+void PG::RecoveryState::ReplicaActive::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->osd->remote_reserver.cancel_reservation(pg->info.pgid);
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_replicaactive_latency, dur);
+}
+
+/*-------Stray---*/
+PG::RecoveryState::Stray::Stray(my_context ctx) 
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Stray")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(!pg->is_active());
+  assert(!pg->is_peering());
+  assert(!pg->is_primary());
+  pg->start_flush(
+    context< RecoveryMachine >().get_cur_transaction(),
+    context< RecoveryMachine >().get_on_applied_context_list(),
+    context< RecoveryMachine >().get_on_safe_context_list());
+}
+
+boost::statechart::result PG::RecoveryState::Stray::react(const MLogRec& logevt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  MOSDPGLog *msg = logevt.msg.get();
+  dout(10) << "got info+log from osd." << logevt.from << " " << msg->info << " " << msg->log << dendl;
+
+  ObjectStore::Transaction* t = context().get_cur_transaction();
+  if (msg->info.last_backfill == hobject_t()) {
+    // restart backfill
+    pg->unreg_next_scrub();
+    pg->info = msg->info;
+    pg->reg_next_scrub();
+    pg->dirty_info = true;
+    pg->dirty_big_info = true;  // maybe.
+
+    PGLogEntryHandler rollbacker;
+    pg->pg_log.claim_log_and_clear_rollback_info(msg->log, &rollbacker);
+    rollbacker.apply(pg, t);
+
+    pg->pg_log.reset_backfill();
+  } else {
+    pg->merge_log(*t, msg->info, msg->log, logevt.from);
+  }
+
+  assert(pg->pg_log.get_head() == pg->info.last_update);
+
+  post_event(Activate(logevt.msg->get_epoch()));
+  return transit();
+}
+
+boost::statechart::result PG::RecoveryState::Stray::react(const MInfoRec& infoevt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  dout(10) << "got info from osd." << infoevt.from << " " << infoevt.info << dendl;
+
+  if (pg->info.last_update > infoevt.info.last_update) {
+    // rewind divergent log entries
+    ObjectStore::Transaction* t = context().get_cur_transaction();
+    pg->rewind_divergent_log(*t, infoevt.info.last_update);
+    pg->info.stats = infoevt.info.stats;
+    pg->info.hit_set = infoevt.info.hit_set;
+  }
+  
+  assert(infoevt.info.last_update == pg->info.last_update);
+  assert(pg->pg_log.get_head() == pg->info.last_update);
+
+  post_event(Activate(infoevt.msg_epoch));
+  return transit();
+}
+
+boost::statechart::result PG::RecoveryState::Stray::react(const MQuery& query)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  if (query.query.type == pg_query_t::INFO) {
+    pair notify_info;
+    pg->update_history_from_master(query.query.history);
+    pg->fulfill_info(query.from, query.query, notify_info);
+    context< RecoveryMachine >().send_notify(
+      notify_info.first,
+      pg_notify_t(
+	notify_info.first.shard, pg->pg_whoami.shard,
+	query.query_epoch,
+	pg->get_osdmap()->get_epoch(),
+	notify_info.second),
+      pg->past_intervals);
+  } else {
+    pg->fulfill_log(query.from, query.query, query.query_epoch);
+  }
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::Stray::react(const ActMap&)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  if (pg->should_send_notify() && pg->get_primary().osd >= 0) {
+    context< RecoveryMachine >().send_notify(
+      pg->get_primary(),
+      pg_notify_t(
+	pg->get_primary().shard, pg->pg_whoami.shard,
+	pg->get_osdmap()->get_epoch(),
+	pg->get_osdmap()->get_epoch(),
+	pg->info),
+      pg->past_intervals);
+  }
+  pg->take_waiters();
+  return discard_event();
+}
+
+void PG::RecoveryState::Stray::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_stray_latency, dur);
+}
+
+/*--------GetInfo---------*/
+PG::RecoveryState::GetInfo::GetInfo(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetInfo")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->generate_past_intervals();
+  auto_ptr &prior_set = context< Peering >().prior_set;
+
+  if (!prior_set.get())
+    pg->build_prior(prior_set);
+
+  pg->publish_stats_to_osd();
+
+  get_infos();
+  if (peer_info_requested.empty() && !prior_set->pg_down) {
+    post_event(GotInfo());
+  }
+}
+
+void PG::RecoveryState::GetInfo::get_infos()
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  auto_ptr &prior_set = context< Peering >().prior_set;
+
+  for (set::const_iterator it = prior_set->probe.begin();
+       it != prior_set->probe.end();
+       ++it) {
+    pg_shard_t peer = *it;
+    if (peer == pg->pg_whoami) {
+      continue;
+    }
+    if (pg->peer_info.count(peer)) {
+      dout(10) << " have osd." << peer << " info " << pg->peer_info[peer] << dendl;
+      continue;
+    }
+    if (peer_info_requested.count(peer)) {
+      dout(10) << " already requested info from osd." << peer << dendl;
+    } else if (!pg->get_osdmap()->is_up(peer.osd)) {
+      dout(10) << " not querying info from down osd." << peer << dendl;
+    } else {
+      dout(10) << " querying info from osd." << peer << dendl;
+      context< RecoveryMachine >().send_query(
+	peer, pg_query_t(pg_query_t::INFO,
+			 it->shard, pg->pg_whoami.shard,
+			 pg->info.history,
+			 pg->get_osdmap()->get_epoch()));
+      peer_info_requested.insert(peer);
+    }
+  }
+}
+
+boost::statechart::result PG::RecoveryState::GetInfo::react(const MNotifyRec& infoevt) 
+{
+  set::iterator p = peer_info_requested.find(infoevt.from);
+  if (p != peer_info_requested.end())
+    peer_info_requested.erase(p);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  epoch_t old_start = pg->info.history.last_epoch_started;
+  if (pg->proc_replica_info(infoevt.from, infoevt.notify.info)) {
+    // we got something new ...
+    auto_ptr &prior_set = context< Peering >().prior_set;
+    if (old_start < pg->info.history.last_epoch_started) {
+      dout(10) << " last_epoch_started moved forward, rebuilding prior" << dendl;
+      pg->build_prior(prior_set);
+
+      // filter out any osds that got dropped from the probe set from
+      // peer_info_requested.  this is less expensive than restarting
+      // peering (which would re-probe everyone).
+      set::iterator p = peer_info_requested.begin();
+      while (p != peer_info_requested.end()) {
+	if (prior_set->probe.count(*p) == 0) {
+	  dout(20) << " dropping osd." << *p << " from info_requested, no longer in probe set" << dendl;
+	  peer_info_requested.erase(p++);
+	} else {
+	  ++p;
+	}
+      }
+      get_infos();
+    }
+
+    // are we done getting everything?
+    if (peer_info_requested.empty() && !prior_set->pg_down) {
+      /*
+       * make sure we have at least one !incomplete() osd from the
+       * last rw interval.  the incomplete (backfilling) replicas
+       * get a copy of the log, but they don't get all the object
+       * updates, so they are insufficient to recover changes during
+       * that interval.
+       */
+      if (pg->info.history.last_epoch_started) {
+	for (map::reverse_iterator p = pg->past_intervals.rbegin();
+	     p != pg->past_intervals.rend();
+	     ++p) {
+	  if (p->first < pg->info.history.last_epoch_started)
+	    break;
+	  if (!p->second.maybe_went_rw)
+	    continue;
+	  pg_interval_t& interval = p->second;
+	  dout(10) << " last maybe_went_rw interval was " << interval << dendl;
+	  OSDMapRef osdmap = pg->get_osdmap();
+
+	  /*
+	   * this mirrors the PriorSet calculation: we wait if we
+	   * don't have an up (AND !incomplete) node AND there are
+	   * nodes down that might be usable.
+	   */
+	  bool any_up_complete_now = false;
+	  bool any_down_now = false;
+	  for (unsigned i=0; ipool.info.ec_pool() ? i : ghobject_t::NO_SHARD);
+	    if (!osdmap->exists(o) || osdmap->get_info(o).lost_at > interval.first)
+	      continue;  // dne or lost
+	    if (osdmap->is_up(o)) {
+	      pg_info_t *pinfo;
+	      if (so == pg->pg_whoami) {
+		pinfo = &pg->info;
+	      } else {
+		assert(pg->peer_info.count(so));
+		pinfo = &pg->peer_info[so];
+	      }
+	      if (!pinfo->is_incomplete())
+		any_up_complete_now = true;
+	    } else {
+	      any_down_now = true;
+	    }
+	  }
+	  if (!any_up_complete_now && any_down_now) {
+	    dout(10) << " no osds up+complete from interval " << interval << dendl;
+	    pg->state_set(PG_STATE_DOWN);
+	    return discard_event();
+	  }
+	  break;
+	}
+      }
+      post_event(GotInfo());
+    }
+  }
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::GetInfo::react(const QueryState& q)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+
+  q.f->open_array_section("requested_info_from");
+  for (set::iterator p = peer_info_requested.begin();
+       p != peer_info_requested.end();
+       ++p) {
+    q.f->open_object_section("osd");
+    q.f->dump_stream("osd") << *p;
+    if (pg->peer_info.count(*p)) {
+      q.f->open_object_section("got_info");
+      pg->peer_info[*p].dump(q.f);
+      q.f->close_section();
+    }
+    q.f->close_section();
+  }
+  q.f->close_section();
+
+  q.f->close_section();
+  return forward_event();
+}
+
+void PG::RecoveryState::GetInfo::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_getinfo_latency, dur);
+}
+
+/*------GetLog------------*/
+PG::RecoveryState::GetLog::GetLog(my_context ctx)
+  : my_base(ctx),
+    NamedState(
+      context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetLog"),
+    msg(0)
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+
+  // adjust acting?
+  if (!pg->choose_acting(auth_log_shard)) {
+    if (!pg->want_acting.empty()) {
+      post_event(NeedActingChange());
+    } else {
+      post_event(IsIncomplete());
+    }
+    return;
+  }
+
+  // am i the best?
+  if (auth_log_shard == pg->pg_whoami) {
+    post_event(GotLog());
+    return;
+  }
+
+  const pg_info_t& best = pg->peer_info[auth_log_shard];
+
+  // am i broken?
+  if (pg->info.last_update < best.log_tail) {
+    dout(10) << " not contiguous with osd." << auth_log_shard << ", down" << dendl;
+    post_event(IsIncomplete());
+    return;
+  }
+
+  // how much log to request?
+  eversion_t request_log_from = pg->info.last_update;
+  assert(!pg->actingbackfill.empty());
+  for (set::iterator p = pg->actingbackfill.begin();
+       p != pg->actingbackfill.end();
+       ++p) {
+    if (*p == pg->pg_whoami) continue;
+    pg_info_t& ri = pg->peer_info[*p];
+    if (ri.last_update >= best.log_tail && ri.last_update < request_log_from)
+      request_log_from = ri.last_update;
+  }
+
+  // how much?
+  dout(10) << " requesting log from osd." << auth_log_shard << dendl;
+  context().send_query(
+    auth_log_shard,
+    pg_query_t(
+      pg_query_t::LOG,
+      auth_log_shard.shard, pg->pg_whoami.shard,
+      request_log_from, pg->info.history,
+      pg->get_osdmap()->get_epoch()));
+}
+
+boost::statechart::result PG::RecoveryState::GetLog::react(const AdvMap& advmap)
+{
+  // make sure our log source didn't go down.  we need to check
+  // explicitly because it may not be part of the prior set, which
+  // means the Peering state check won't catch it going down.
+  if (!advmap.osdmap->is_up(auth_log_shard.osd)) {
+    dout(10) << "GetLog: auth_log_shard osd."
+	     << auth_log_shard.osd << " went down" << dendl;
+    post_event(advmap);
+    return transit< Reset >();
+  }
+
+  // let the Peering state do its checks.
+  return forward_event();
+}
+
+boost::statechart::result PG::RecoveryState::GetLog::react(const MLogRec& logevt)
+{
+  assert(!msg);
+  if (logevt.from != auth_log_shard) {
+    dout(10) << "GetLog: discarding log from "
+	     << "non-auth_log_shard osd." << logevt.from << dendl;
+    return discard_event();
+  }
+  dout(10) << "GetLog: received master log from osd"
+	   << logevt.from << dendl;
+  msg = logevt.msg;
+  post_event(GotLog());
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::GetLog::react(const GotLog&)
+{
+  dout(10) << "leaving GetLog" << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  if (msg) {
+    dout(10) << "processing master log" << dendl;
+    pg->proc_master_log(*context().get_cur_transaction(),
+			msg->info, msg->log, msg->missing, 
+			auth_log_shard);
+  }
+  pg->start_flush(
+    context< RecoveryMachine >().get_cur_transaction(),
+    context< RecoveryMachine >().get_on_applied_context_list(),
+    context< RecoveryMachine >().get_on_safe_context_list());
+  return transit< GetMissing >();
+}
+
+boost::statechart::result PG::RecoveryState::GetLog::react(const QueryState& q)
+{
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+  q.f->dump_stream("auth_log_shard") << auth_log_shard;
+  q.f->close_section();
+  return forward_event();
+}
+
+void PG::RecoveryState::GetLog::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_getlog_latency, dur);
+}
+
+/*------WaitActingChange--------*/
+PG::RecoveryState::WaitActingChange::WaitActingChange(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitActingChange")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+boost::statechart::result PG::RecoveryState::WaitActingChange::react(const AdvMap& advmap)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  OSDMapRef osdmap = advmap.osdmap;
+
+  dout(10) << "verifying no want_acting " << pg->want_acting << " targets didn't go down" << dendl;
+  for (vector::iterator p = pg->want_acting.begin(); p != pg->want_acting.end(); ++p) {
+    if (!osdmap->is_up(*p)) {
+      dout(10) << " want_acting target osd." << *p << " went down, resetting" << dendl;
+      post_event(advmap);
+      return transit< Reset >();
+    }
+  }
+  return forward_event();
+}
+
+boost::statechart::result PG::RecoveryState::WaitActingChange::react(const MLogRec& logevt)
+{
+  dout(10) << "In WaitActingChange, ignoring MLocRec" << dendl;
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::WaitActingChange::react(const MInfoRec& evt)
+{
+  dout(10) << "In WaitActingChange, ignoring MInfoRec" << dendl;
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::WaitActingChange::react(const MNotifyRec& evt)
+{
+  dout(10) << "In WaitActingChange, ignoring MNotifyRec" << dendl;
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::WaitActingChange::react(const QueryState& q)
+{
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+  q.f->dump_string("comment", "waiting for pg acting set to change");
+  q.f->close_section();
+  return forward_event();
+}
+
+void PG::RecoveryState::WaitActingChange::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_waitactingchange_latency, dur);
+}
+
+/*------Incomplete--------*/
+PG::RecoveryState::Incomplete::Incomplete(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/Incomplete")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+  PG *pg = context< RecoveryMachine >().pg;
+
+  pg->state_clear(PG_STATE_PEERING);
+  pg->state_set(PG_STATE_INCOMPLETE);
+  pg->publish_stats_to_osd();
+}
+
+boost::statechart::result PG::RecoveryState::Incomplete::react(const AdvMap &advmap) {
+  PG *pg = context< RecoveryMachine >().pg;
+  int64_t poolnum = pg->info.pgid.pool();
+
+  // Reset if min_size changed, pg might now be able to go active
+  if (advmap.lastmap->get_pools().find(poolnum)->second.min_size !=
+      advmap.osdmap->get_pools().find(poolnum)->second.min_size) {
+    post_event(advmap);
+    return transit< Reset >();
+  }
+
+  return forward_event();
+}
+
+boost::statechart::result PG::RecoveryState::Incomplete::react(const MNotifyRec& notevt) {
+  dout(7) << "handle_pg_notify from osd." << notevt.from << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  if (pg->peer_info.count(notevt.from) &&
+      pg->peer_info[notevt.from].last_update == notevt.notify.info.last_update) {
+    dout(10) << *pg << " got dup osd." << notevt.from << " info " << notevt.notify.info
+	     << ", identical to ours" << dendl;
+    return discard_event();
+  } else {
+    pg->proc_replica_info(notevt.from, notevt.notify.info);
+    // try again!
+    return transit< GetLog >();
+  }
+}
+
+void PG::RecoveryState::Incomplete::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+
+  pg->state_clear(PG_STATE_INCOMPLETE);
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_incomplete_latency, dur);
+}
+
+/*------GetMissing--------*/
+PG::RecoveryState::GetMissing::GetMissing(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetMissing")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+
+  PG *pg = context< RecoveryMachine >().pg;
+  assert(!pg->actingbackfill.empty());
+  for (set::iterator i = pg->actingbackfill.begin();
+       i != pg->actingbackfill.end();
+       ++i) {
+    if (*i == pg->get_primary()) continue;
+    const pg_info_t& pi = pg->peer_info[*i];
+
+    if (pi.is_empty())
+      continue;                                // no pg data, nothing divergent
+
+    if (pi.last_update < pg->pg_log.get_tail()) {
+      dout(10) << " osd." << *i << " is not contiguous, will restart backfill" << dendl;
+      pg->peer_missing[*i];
+      continue;
+    }
+    if (pi.last_backfill == hobject_t()) {
+      dout(10) << " osd." << *i << " will fully backfill; can infer empty missing set" << dendl;
+      pg->peer_missing[*i];
+      continue;
+    }
+
+    if (pi.last_update == pi.last_complete &&  // peer has no missing
+	pi.last_update == pg->info.last_update) {  // peer is up to date
+      // replica has no missing and identical log as us.  no need to
+      // pull anything.
+      // FIXME: we can do better here.  if last_update==last_complete we
+      //        can infer the rest!
+      dout(10) << " osd." << *i << " has no missing, identical log" << dendl;
+      pg->peer_missing[*i];
+      continue;
+    }
+
+    // We pull the log from the peer's last_epoch_started to ensure we
+    // get enough log to detect divergent updates.
+    eversion_t since(pi.last_epoch_started, 0);
+    assert(pi.last_update >= pg->info.log_tail);  // or else choose_acting() did a bad thing
+    if (pi.log_tail <= since) {
+      dout(10) << " requesting log+missing since " << since << " from osd." << *i << dendl;
+      context< RecoveryMachine >().send_query(
+	*i,
+	pg_query_t(
+	  pg_query_t::LOG,
+	  i->shard, pg->pg_whoami.shard,
+	  since, pg->info.history,
+	  pg->get_osdmap()->get_epoch()));
+    } else {
+      dout(10) << " requesting fulllog+missing from osd." << *i
+	       << " (want since " << since << " < log.tail " << pi.log_tail << ")"
+	       << dendl;
+      context< RecoveryMachine >().send_query(
+	*i, pg_query_t(
+	  pg_query_t::FULLLOG,
+	  i->shard, pg->pg_whoami.shard,
+	  pg->info.history, pg->get_osdmap()->get_epoch()));
+    }
+    peer_missing_requested.insert(*i);
+  }
+
+  if (peer_missing_requested.empty()) {
+    if (pg->need_up_thru) {
+      dout(10) << " still need up_thru update before going active" << dendl;
+      post_event(NeedUpThru());
+      return;
+    }
+
+    // all good!
+    post_event(Activate(pg->get_osdmap()->get_epoch()));
+  }
+}
+
+boost::statechart::result PG::RecoveryState::GetMissing::react(const MLogRec& logevt)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+
+  peer_missing_requested.erase(logevt.from);
+  pg->proc_replica_log(*context().get_cur_transaction(),
+		       logevt.msg->info, logevt.msg->log, logevt.msg->missing, logevt.from);
+  
+  if (peer_missing_requested.empty()) {
+    if (pg->need_up_thru) {
+      dout(10) << " still need up_thru update before going active" << dendl;
+      post_event(NeedUpThru());
+    } else {
+      dout(10) << "Got last missing, don't need missing "
+	       << "posting CheckRepops" << dendl;
+      post_event(Activate(pg->get_osdmap()->get_epoch()));
+    }
+  }
+  return discard_event();
+};
+
+boost::statechart::result PG::RecoveryState::GetMissing::react(const QueryState& q)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+
+  q.f->open_array_section("peer_missing_requested");
+  for (set::iterator p = peer_missing_requested.begin();
+       p != peer_missing_requested.end();
+       ++p) {
+    q.f->open_object_section("osd");
+    q.f->dump_stream("osd") << *p;
+    if (pg->peer_missing.count(*p)) {
+      q.f->open_object_section("got_missing");
+      pg->peer_missing[*p].dump(q.f);
+      q.f->close_section();
+    }
+    q.f->close_section();
+  }
+  q.f->close_section();
+
+  q.f->close_section();
+  return forward_event();
+}
+
+void PG::RecoveryState::GetMissing::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_getmissing_latency, dur);
+}
+
+/*------WaitUpThru--------*/
+PG::RecoveryState::WaitUpThru::WaitUpThru(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitUpThru")
+{
+  context< RecoveryMachine >().log_enter(state_name);
+}
+
+boost::statechart::result PG::RecoveryState::WaitUpThru::react(const ActMap& am)
+{
+  PG *pg = context< RecoveryMachine >().pg;
+  if (!pg->need_up_thru) {
+    post_event(Activate(pg->get_osdmap()->get_epoch()));
+  }
+  return forward_event();
+}
+
+boost::statechart::result PG::RecoveryState::WaitUpThru::react(const MLogRec& logevt)
+{
+  dout(10) << "Noting missing from osd." << logevt.from << dendl;
+  PG *pg = context< RecoveryMachine >().pg;
+  pg->peer_missing[logevt.from].swap(logevt.msg->missing);
+  pg->peer_info[logevt.from] = logevt.msg->info;
+  return discard_event();
+}
+
+boost::statechart::result PG::RecoveryState::WaitUpThru::react(const QueryState& q)
+{
+  q.f->open_object_section("state");
+  q.f->dump_string("name", state_name);
+  q.f->dump_stream("enter_time") << enter_time;
+  q.f->dump_string("comment", "waiting for osdmap to reflect a new up_thru for this osd");
+  q.f->close_section();
+  return forward_event();
+}
+
+void PG::RecoveryState::WaitUpThru::exit()
+{
+  context< RecoveryMachine >().log_exit(state_name, enter_time);
+  PG *pg = context< RecoveryMachine >().pg;
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  pg->osd->recoverystate_perf->tinc(rs_waitupthru_latency, dur);
+}
+
+/*----RecoveryState::RecoveryMachine Methods-----*/
+#undef dout_prefix
+#define dout_prefix *_dout << pg->gen_prefix() 
+
+void PG::RecoveryState::RecoveryMachine::log_enter(const char *state_name)
+{
+  dout(5) << "enter " << state_name << dendl;
+  pg->osd->pg_recovery_stats.log_enter(state_name);
+}
+
+void PG::RecoveryState::RecoveryMachine::log_exit(const char *state_name, utime_t enter_time)
+{
+  utime_t dur = ceph_clock_now(pg->cct) - enter_time;
+  dout(5) << "exit " << state_name << " " << dur << " " << event_count << " " << event_time << dendl;
+  pg->osd->pg_recovery_stats.log_exit(state_name, ceph_clock_now(pg->cct) - enter_time,
+				      event_count, event_time);
+  event_count = 0;
+  event_time = utime_t();
+}
+
+
+/*---------------------------------------------------*/
+#undef dout_prefix
+#define dout_prefix (*_dout << (debug_pg ? debug_pg->gen_prefix() : string()) << " PriorSet: ")
+
+PG::PriorSet::PriorSet(bool ec_pool,
+		       PGBackend::IsRecoverablePredicate *c,
+		       const OSDMap &osdmap,
+		       const map &past_intervals,
+		       const vector &up,
+		       const vector &acting,
+		       const pg_info_t &info,
+		       const PG *debug_pg)
+  : ec_pool(ec_pool), pg_down(false), pcontdec(c)
+{
+  /*
+   * We have to be careful to gracefully deal with situations like
+   * so. Say we have a power outage or something that takes out both
+   * OSDs, but the monitor doesn't mark them down in the same epoch.
+   * The history may look like
+   *
+   *  1: A B
+   *  2:   B
+   *  3:       let's say B dies for good, too (say, from the power spike) 
+   *  4: A
+   *
+   * which makes it look like B may have applied updates to the PG
+   * that we need in order to proceed.  This sucks...
+   *
+   * To minimize the risk of this happening, we CANNOT go active if
+   * _any_ OSDs in the prior set are down until we send an MOSDAlive
+   * to the monitor such that the OSDMap sets osd_up_thru to an epoch.
+   * Then, we have something like
+   *
+   *  1: A B
+   *  2:   B   up_thru[B]=0
+   *  3:
+   *  4: A
+   *
+   * -> we can ignore B, bc it couldn't have gone active (alive_thru
+   *    still 0).
+   *
+   * or,
+   *
+   *  1: A B
+   *  2:   B   up_thru[B]=0
+   *  3:   B   up_thru[B]=2
+   *  4:
+   *  5: A    
+   *
+   * -> we must wait for B, bc it was alive through 2, and could have
+   *    written to the pg.
+   *
+   * If B is really dead, then an administrator will need to manually
+   * intervene by marking the OSD as "lost."
+   */
+
+  // Include current acting and up nodes... not because they may
+  // contain old data (this interval hasn't gone active, obviously),
+  // but because we want their pg_info to inform choose_acting(), and
+  // so that we know what they do/do not have explicitly before
+  // sending them any new info/logs/whatever.
+  for (unsigned i=0; i::const_reverse_iterator p = past_intervals.rbegin();
+       p != past_intervals.rend();
+       ++p) {
+    const pg_interval_t &interval = p->second;
+    dout(10) << "build_prior " << interval << dendl;
+
+    if (interval.last < info.history.last_epoch_started)
+      break;  // we don't care
+
+    if (interval.acting.empty())
+      continue;
+
+    if (!interval.maybe_went_rw)
+      continue;
+
+    // look at candidate osds during this interval.  each falls into
+    // one of three categories: up, down (but potentially
+    // interesting), or lost (down, but we won't wait for it).
+    set up_now;
+    bool any_down_now = false;  // any candidates down now (that might have useful data)
+
+    // consider ACTING osds
+    for (unsigned i=0; ilost_at > interval.first) {
+	dout(10) << "build_prior  prior osd." << o << " is down, but lost_at " << pinfo->lost_at << dendl;
+	up_now.insert(so);
+	down.insert(o);
+      } else {
+	dout(10) << "build_prior  prior osd." << o << " is down" << dendl;
+	down.insert(o);
+	any_down_now = true;
+      }
+    }
+
+    // if not enough osds survived this interval, and we may have gone rw,
+    // then we need to wait for one of those osds to recover to
+    // ensure that we haven't lost any information.
+    if (!(*pcontdec)(up_now) && any_down_now) {
+      // fixme: how do we identify a "clean" shutdown anyway?
+      dout(10) << "build_prior  possibly went active+rw, insufficient up;"
+	       << " including down osds" << dendl;
+      for (vector::const_iterator i = interval.acting.begin();
+	   i != interval.acting.end();
+	   ++i) {
+	if (osdmap.exists(*i) &&   // if it doesn't exist, we already consider it lost.
+	    osdmap.is_down(*i)) {
+	  pg_down = true;
+
+	  // make note of when any down osd in the cur set was lost, so that
+	  // we can notice changes in prior_set_affected.
+	  blocked_by[*i] = osdmap.get_info(*i).lost_at;
+	}
+      }
+    }
+  }
+
+  dout(10) << "build_prior final: probe " << probe
+	   << " down " << down
+	   << " blocked_by " << blocked_by
+	   << (pg_down ? " pg_down":"")
+	   << dendl;
+}
+
+// true if the given map affects the prior set
+bool PG::PriorSet::affected_by_map(const OSDMapRef osdmap, const PG *debug_pg) const
+{
+  for (set::iterator p = probe.begin();
+       p != probe.end();
+       ++p) {
+    int o = p->osd;
+
+    // did someone in the prior set go down?
+    if (osdmap->is_down(o) && down.count(o) == 0) {
+      dout(10) << "affected_by_map osd." << o << " now down" << dendl;
+      return true;
+    }
+
+    // did a down osd in cur get (re)marked as lost?
+    map::const_iterator r = blocked_by.find(o);
+    if (r != blocked_by.end()) {
+      if (!osdmap->exists(o)) {
+	dout(10) << "affected_by_map osd." << o << " no longer exists" << dendl;
+	return true;
+      }
+      if (osdmap->get_info(o).lost_at != r->second) {
+	dout(10) << "affected_by_map osd." << o << " (re)marked as lost" << dendl;
+	return true;
+      }
+    }
+  }
+
+  // did someone in the prior down set go up?
+  for (set::const_iterator p = down.begin();
+       p != down.end();
+       ++p) {
+    int o = *p;
+
+    if (osdmap->is_up(o)) {
+      dout(10) << "affected_by_map osd." << *p << " now up" << dendl;
+      return true;
+    }
+
+    // did someone in the prior set get lost or destroyed?
+    if (!osdmap->exists(o)) {
+      dout(10) << "affected_by_map osd." << o << " no longer exists" << dendl;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void PG::RecoveryState::start_handle(RecoveryCtx *new_ctx) {
+  assert(!rctx);
+  assert(!orig_ctx);
+  orig_ctx = new_ctx;
+  if (new_ctx) {
+    if (messages_pending_flush) {
+      rctx = RecoveryCtx(*messages_pending_flush, *new_ctx);
+    } else {
+      rctx = *new_ctx;
+    }
+    rctx->start_time = ceph_clock_now(pg->cct);
+  }
+}
+
+void PG::RecoveryState::begin_block_outgoing() {
+  assert(!messages_pending_flush);
+  assert(orig_ctx);
+  assert(rctx);
+  messages_pending_flush = BufferedRecoveryMessages();
+  rctx = RecoveryCtx(*messages_pending_flush, *orig_ctx);
+}
+
+void PG::RecoveryState::clear_blocked_outgoing() {
+  assert(orig_ctx);
+  assert(rctx);
+  messages_pending_flush = boost::optional();
+}
+
+void PG::RecoveryState::end_block_outgoing() {
+  assert(messages_pending_flush);
+  assert(orig_ctx);
+  assert(rctx);
+
+  rctx = RecoveryCtx(*orig_ctx);
+  rctx->accept_buffered_messages(*messages_pending_flush);
+  messages_pending_flush = boost::optional();
+}
+
+void PG::RecoveryState::end_handle() {
+  if (rctx) {
+    utime_t dur = ceph_clock_now(pg->cct) - rctx->start_time;
+    machine.event_time += dur;
+  }
+
+  machine.event_count++;
+  rctx = boost::optional();
+  orig_ctx = NULL;
+}
+
+void intrusive_ptr_add_ref(PG *pg) { pg->get("intptr"); }
+void intrusive_ptr_release(PG *pg) { pg->put("intptr"); }
+
+#ifdef PG_DEBUG_REFS
+  uint64_t get_with_id(PG *pg) { return pg->get_with_id(); }
+  void put_with_id(PG *pg, uint64_t id) { return pg->put_with_id(id); }
+#endif
diff --git a/ceph/src/osd/PG.h b/ceph/src/osd/PG.h
new file mode 100644
index 00000000..1aadaf03
--- /dev/null
+++ b/ceph/src/osd/PG.h
@@ -0,0 +1,2252 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_PG_H
+#define CEPH_PG_H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+
+// re-include our assert to clobber boost's
+#include "include/assert.h" 
+
+#include "include/types.h"
+#include "include/stringify.h"
+#include "osd_types.h"
+#include "include/buffer.h"
+#include "include/xlist.h"
+#include "include/atomic.h"
+#include "SnapMapper.h"
+
+#include "PGLog.h"
+#include "OpRequest.h"
+#include "OSDMap.h"
+#include "os/ObjectStore.h"
+#include "msg/Messenger.h"
+#include "messages/MOSDRepScrub.h"
+#include "messages/MOSDPGLog.h"
+#include "common/cmdparse.h"
+#include "common/tracked_int_ptr.hpp"
+#include "common/WorkQueue.h"
+#include "common/ceph_context.h"
+#include "include/str_list.h"
+#include "PGBackend.h"
+
+#include 
+#include 
+#include 
+using namespace std;
+
+#include "include/unordered_map.h"
+#include "include/unordered_set.h"
+
+
+//#define DEBUG_RECOVERY_OIDS   // track set of recovering oids explicitly, to find counting bugs
+
+class OSD;
+class OSDService;
+class MOSDOp;
+class MOSDSubOp;
+class MOSDSubOpReply;
+class MOSDPGScan;
+class MOSDPGBackfill;
+class MOSDPGInfo;
+
+class PG;
+
+void intrusive_ptr_add_ref(PG *pg);
+void intrusive_ptr_release(PG *pg);
+
+#ifdef PG_DEBUG_REFS
+  uint64_t get_with_id(PG *pg);
+  void put_with_id(PG *pg, uint64_t id);
+  typedef TrackedIntPtr PGRef;
+#else
+  typedef boost::intrusive_ptr PGRef;
+#endif
+
+struct PGRecoveryStats {
+  struct per_state_info {
+    uint64_t enter, exit;     // enter/exit counts
+    uint64_t events;
+    utime_t event_time;       // time spent processing events
+    utime_t total_time;       // total time in state
+    utime_t min_time, max_time;
+
+    per_state_info() : enter(0), exit(0), events(0) {}
+  };
+  map info;
+  Mutex lock;
+
+  PGRecoveryStats() : lock("PGRecoverStats::lock") {}
+
+  void reset() {
+    Mutex::Locker l(lock);
+    info.clear();
+  }
+  void dump(ostream& out) {
+    Mutex::Locker l(lock);
+    for (map::iterator p = info.begin(); p != info.end(); ++p) {
+      per_state_info& i = p->second;
+      out << i.enter << "\t" << i.exit << "\t"
+	  << i.events << "\t" << i.event_time << "\t"
+	  << i.total_time << "\t"
+	  << i.min_time << "\t" << i.max_time << "\t"
+	  << p->first << "\n";
+    }
+  }
+
+  void dump_formatted(Formatter *f) {
+    Mutex::Locker l(lock);
+    f->open_array_section("pg_recovery_stats");
+    for (map::iterator p = info.begin();
+	 p != info.end(); ++p) {
+      per_state_info& i = p->second;
+      f->open_object_section("recovery_state");
+      f->dump_int("enter", i.enter);
+      f->dump_int("exit", i.exit);
+      f->dump_int("events", i.events);
+      f->dump_stream("event_time") << i.event_time;
+      f->dump_stream("total_time") << i.total_time;
+      f->dump_stream("min_time") << i.min_time;
+      f->dump_stream("max_time") << i.max_time;
+      vector states;
+      get_str_vec(p->first, "/", states);
+      f->open_array_section("nested_states");
+      for (vector::iterator st = states.begin();
+	   st != states.end(); ++st) {
+	f->dump_string("state", *st);
+      }
+      f->close_section();
+      f->close_section();
+    }
+    f->close_section();
+  }
+
+  void log_enter(const char *s) {
+    Mutex::Locker l(lock);
+    info[s].enter++;
+  }
+  void log_exit(const char *s, utime_t dur, uint64_t events, utime_t event_dur) {
+    Mutex::Locker l(lock);
+    per_state_info &i = info[s];
+    i.exit++;
+    i.total_time += dur;
+    if (dur > i.max_time)
+      i.max_time = dur;
+    if (dur < i.min_time || i.min_time == utime_t())
+      i.min_time = dur;
+    i.events += events;
+    i.event_time += event_dur;
+  }
+};
+
+struct PGPool {
+  int64_t id;
+  string name;
+  uint64_t auid;
+
+  pg_pool_t info;      
+  SnapContext snapc;   // the default pool snapc, ready to go.
+
+  interval_set cached_removed_snaps;      // current removed_snaps set
+  interval_set newly_removed_snaps;  // newly removed in the last epoch
+
+  PGPool(int64_t i, const char *_name, uint64_t au) :
+    id(i), auid(au) {
+    if (_name)
+      name = _name;
+  }
+
+  void update(OSDMapRef map);
+};
+
+/** PG - Replica Placement Group
+ *
+ */
+
+class PG {
+public:
+  std::string gen_prefix() const;
+
+  /*** PG ****/
+protected:
+  OSDService *osd;
+  CephContext *cct;
+  OSDriver osdriver;
+  SnapMapper snap_mapper;
+
+  virtual PGBackend *get_pgbackend() = 0;
+public:
+  void update_snap_mapper_bits(uint32_t bits) {
+    snap_mapper.update_bits(bits);
+  }
+protected:
+  // Ops waiting for map, should be queued at back
+  Mutex map_lock;
+  list waiting_for_map;
+  OSDMapRef osdmap_ref;
+  OSDMapRef last_persisted_osdmap_ref;
+  PGPool pool;
+
+  void queue_op(OpRequestRef op);
+  void take_op_map_waiters();
+
+  void update_osdmap_ref(OSDMapRef newmap) {
+    assert(_lock.is_locked_by_me());
+    Mutex::Locker l(map_lock);
+    osdmap_ref = newmap;
+  }
+
+  OSDMapRef get_osdmap_with_maplock() const {
+    assert(map_lock.is_locked());
+    assert(osdmap_ref);
+    return osdmap_ref;
+  }
+
+  OSDMapRef get_osdmap() const {
+    assert(is_locked());
+    assert(osdmap_ref);
+    return osdmap_ref;
+  }
+
+  /** locking and reference counting.
+   * I destroy myself when the reference count hits zero.
+   * lock() should be called before doing anything.
+   * get() should be called on pointer copy (to another thread, etc.).
+   * put() should be called on destruction of some previously copied pointer.
+   * put_unlock() when done with the current pointer (_most common_).
+   */  
+  Mutex _lock;
+  atomic_t ref;
+
+#ifdef PG_DEBUG_REFS
+  Mutex _ref_id_lock;
+  map _live_ids;
+  map _tag_counts;
+  uint64_t _ref_id;
+#endif
+
+public:
+  bool deleting;  // true while in removing or OSD is shutting down
+
+
+  void lock_suspend_timeout(ThreadPool::TPHandle &handle);
+  void lock(bool no_lockdep = false);
+  void unlock() {
+    //generic_dout(0) << this << " " << info.pgid << " unlock" << dendl;
+    assert(!dirty_info);
+    assert(!dirty_big_info);
+    _lock.Unlock();
+  }
+
+  void assert_locked() {
+    assert(_lock.is_locked());
+  }
+  bool is_locked() const {
+    return _lock.is_locked();
+  }
+
+#ifdef PG_DEBUG_REFS
+  uint64_t get_with_id();
+  void put_with_id(uint64_t);
+  void dump_live_ids();
+#endif
+  void get(const string &tag);
+  void put(const string &tag);
+
+  bool dirty_info, dirty_big_info;
+
+public:
+  bool is_ec_pg() const {
+    return pool.info.ec_pool();
+  }
+  // pg state
+  pg_info_t        info;
+  __u8 info_struct_v;
+  static const __u8 cur_struct_v = 7;
+  bool must_upgrade() {
+    return info_struct_v < 7;
+  }
+  void upgrade(
+    ObjectStore *store,
+    const interval_set &snapcolls);
+
+  const coll_t coll;
+  PGLog  pg_log;
+  static string get_info_key(spg_t pgid) {
+    return stringify(pgid) + "_info";
+  }
+  static string get_biginfo_key(spg_t pgid) {
+    return stringify(pgid) + "_biginfo";
+  }
+  static string get_epoch_key(spg_t pgid) {
+    return stringify(pgid) + "_epoch";
+  }
+  hobject_t    log_oid;
+  hobject_t    biginfo_oid;
+
+  class MissingLoc {
+    map needs_recovery_map;
+    map > missing_loc;
+    set missing_loc_sources;
+    PG *pg;
+    set empty_set;
+  public:
+    boost::scoped_ptr is_readable;
+    boost::scoped_ptr is_recoverable;
+    MissingLoc(PG *pg)
+      : pg(pg) {}
+    void set_backend_predicates(
+      PGBackend::IsReadablePredicate *_is_readable,
+      PGBackend::IsRecoverablePredicate *_is_recoverable) {
+      is_readable.reset(_is_readable);
+      is_recoverable.reset(_is_recoverable);
+    }
+    string gen_prefix() const { return pg->gen_prefix(); }
+    bool needs_recovery(
+      const hobject_t &hoid,
+      eversion_t *v = 0) const {
+      map::const_iterator i =
+	needs_recovery_map.find(hoid);
+      if (i == needs_recovery_map.end())
+	return false;
+      if (v)
+	*v = i->second.need;
+      return true;
+    }
+    bool is_unfound(const hobject_t &hoid) const {
+      return needs_recovery(hoid) && (
+	!missing_loc.count(hoid) ||
+	!(*is_recoverable)(missing_loc.find(hoid)->second));
+    }
+    bool readable_with_acting(
+      const hobject_t &hoid,
+      const set &acting) const;
+    uint64_t num_unfound() const {
+      uint64_t ret = 0;
+      for (map::const_iterator i =
+	     needs_recovery_map.begin();
+	   i != needs_recovery_map.end();
+	   ++i) {
+	if (is_unfound(i->first))
+	  ++ret;
+      }
+      return ret;
+    }
+
+    const map &get_all_missing() {
+      return needs_recovery_map;
+    }
+
+    void clear() {
+      needs_recovery_map.clear();
+      missing_loc.clear();
+      missing_loc_sources.clear();
+    }
+
+    void add_location(const hobject_t &hoid, pg_shard_t location) {
+      missing_loc[hoid].insert(location);
+    }
+    void remove_location(const hobject_t &hoid, pg_shard_t location) {
+      missing_loc[hoid].erase(location);
+    }
+    void add_active_missing(const pg_missing_t &missing) {
+      for (map::const_iterator i =
+	     missing.missing.begin();
+	   i != missing.missing.end();
+	   ++i) {
+	map::const_iterator j =
+	  needs_recovery_map.find(i->first);
+	if (j == needs_recovery_map.end()) {
+	  needs_recovery_map.insert(*i);
+	} else {
+	  assert(i->second.need == j->second.need);
+	}
+      }
+    }
+
+    void add_missing(const hobject_t &hoid, eversion_t need, eversion_t have) {
+      needs_recovery_map[hoid] = pg_missing_t::item(need, have);
+    }
+    void revise_need(const hobject_t &hoid, eversion_t need) {
+      assert(needs_recovery(hoid));
+      needs_recovery_map[hoid].need = need;
+    }
+
+    /// Adds info about a possible recovery source
+    bool add_source_info(
+      pg_shard_t source,           ///< [in] source
+      const pg_info_t &oinfo,      ///< [in] info
+      const pg_missing_t &omissing ///< [in] (optional) missing
+      ); ///< @return whether a new object location was discovered
+
+    /// Uses osdmap to update structures for now down sources
+    void check_recovery_sources(const OSDMapRef osdmap);
+
+    /// Call when hoid is no longer missing in acting set
+    void recovered(const hobject_t &hoid) {
+      needs_recovery_map.erase(hoid);
+      missing_loc.erase(hoid);
+    }
+
+    const set &get_locations(const hobject_t &hoid) const {
+      return missing_loc.count(hoid) ?
+	missing_loc.find(hoid)->second : empty_set;
+    }
+    const map > &get_missing_locs() const {
+      return missing_loc;
+    }
+    const map &get_needs_recovery() const {
+      return needs_recovery_map;
+    }
+  } missing_loc;
+  
+  interval_set snap_collections; // obsolete
+  map past_intervals;
+
+  interval_set snap_trimq;
+
+  /* You should not use these items without taking their respective queue locks
+   * (if they have one) */
+  xlist::item recovery_item, scrub_item, scrub_finalize_item, snap_trim_item, stat_queue_item;
+  int recovery_ops_active;
+  set waiting_on_backfill;
+#ifdef DEBUG_RECOVERY_OIDS
+  set recovering_oids;
+#endif
+
+  utime_t replay_until;
+
+protected:
+  int         role;    // 0 = primary, 1 = replica, -1=none.
+  unsigned    state;   // PG_STATE_*
+
+  bool send_notify;    ///< true if we are non-primary and should notify the primary
+
+public:
+  eversion_t  last_update_ondisk;    // last_update that has committed; ONLY DEFINED WHEN is_active()
+  eversion_t  last_complete_ondisk;  // last_complete that has committed.
+  eversion_t  last_update_applied;
+
+
+  struct C_UpdateLastRollbackInfoTrimmedToApplied : Context {
+    PGRef pg;
+    epoch_t e;
+    eversion_t v;
+    C_UpdateLastRollbackInfoTrimmedToApplied(PG *pg, epoch_t e, eversion_t v)
+      : pg(pg), e(e), v(v) {}
+    void finish(int) {
+      pg->lock();
+      if (!pg->pg_has_reset_since(e)) {
+	pg->last_rollback_info_trimmed_to_applied = v;
+      }
+      pg->unlock();
+    }
+  };
+  // entries <= last_rollback_info_trimmed_to_applied have been trimmed,
+  // and the transaction has applied
+  eversion_t  last_rollback_info_trimmed_to_applied;
+
+  // primary state
+ public:
+  pg_shard_t primary;
+  pg_shard_t pg_whoami;
+  pg_shard_t up_primary;
+  vector up, acting, want_acting;
+  set actingbackfill, actingset;
+  map peer_last_complete_ondisk;
+  eversion_t  min_last_complete_ondisk;  // up: min over last_complete_ondisk, peer_last_complete_ondisk
+  eversion_t  pg_trim_to;
+
+  // [primary only] content recovery state
+ protected:
+  struct PriorSet {
+    const bool ec_pool;
+    set probe; /// current+prior OSDs we need to probe.
+    set down;  /// down osds that would normally be in @a probe and might be interesting.
+    map blocked_by;  /// current lost_at values for any OSDs in cur set for which (re)marking them lost would affect cur set
+
+    bool pg_down;   /// some down osds are included in @a cur; the DOWN pg state bit should be set.
+    boost::scoped_ptr pcontdec;
+    PriorSet(bool ec_pool,
+	     PGBackend::IsRecoverablePredicate *c,
+	     const OSDMap &osdmap,
+	     const map &past_intervals,
+	     const vector &up,
+	     const vector &acting,
+	     const pg_info_t &info,
+	     const PG *debug_pg=NULL);
+
+    bool affected_by_map(const OSDMapRef osdmap, const PG *debug_pg=0) const;
+  };
+
+  friend std::ostream& operator<<(std::ostream& oss,
+				  const struct PriorSet &prior);
+
+  bool may_need_replay(const OSDMapRef osdmap) const;
+
+
+public:    
+  struct BufferedRecoveryMessages {
+    map > query_map;
+    map > > info_map;
+    map > > notify_list;
+  };
+
+  struct RecoveryCtx {
+    utime_t start_time;
+    map > *query_map;
+    map > > *info_map;
+    map > > *notify_list;
+    C_Contexts *on_applied;
+    C_Contexts *on_safe;
+    ObjectStore::Transaction *transaction;
+    RecoveryCtx(map > *query_map,
+		map > > *info_map,
+		map > > *notify_list,
+		C_Contexts *on_applied,
+		C_Contexts *on_safe,
+		ObjectStore::Transaction *transaction)
+      : query_map(query_map), info_map(info_map), 
+	notify_list(notify_list),
+	on_applied(on_applied),
+	on_safe(on_safe),
+	transaction(transaction) {}
+
+    RecoveryCtx(BufferedRecoveryMessages &buf, RecoveryCtx &rctx)
+      : query_map(&(buf.query_map)),
+	info_map(&(buf.info_map)),
+	notify_list(&(buf.notify_list)),
+	on_applied(rctx.on_applied),
+	on_safe(rctx.on_safe),
+	transaction(rctx.transaction) {}
+
+    void accept_buffered_messages(BufferedRecoveryMessages &m) {
+      assert(query_map);
+      assert(info_map);
+      assert(notify_list);
+      for (map >::iterator i = m.query_map.begin();
+	   i != m.query_map.end();
+	   ++i) {
+	map &omap = (*query_map)[i->first];
+	for (map::iterator j = i->second.begin();
+	     j != i->second.end();
+	     ++j) {
+	  omap[j->first] = j->second;
+	}
+      }
+      for (map > >::iterator i
+	     = m.info_map.begin();
+	   i != m.info_map.end();
+	   ++i) {
+	vector > &ovec =
+	  (*info_map)[i->first];
+	ovec.reserve(ovec.size() + i->second.size());
+	ovec.insert(ovec.end(), i->second.begin(), i->second.end());
+      }
+      for (map > >::iterator i
+	     = m.notify_list.begin();
+	   i != m.notify_list.end();
+	   ++i) {
+	vector > &ovec =
+	  (*notify_list)[i->first];
+	ovec.reserve(ovec.size() + i->second.size());
+	ovec.insert(ovec.end(), i->second.begin(), i->second.end());
+      }
+    }
+  };
+
+  struct NamedState {
+    const char *state_name;
+    utime_t enter_time;
+    const char *get_state_name() { return state_name; }
+    NamedState(CephContext *cct_, const char *state_name_)
+      : state_name(state_name_),
+        enter_time(ceph_clock_now(cct_)) {};
+    virtual ~NamedState() {}
+  };
+
+
+
+protected:
+
+  /*
+   * peer_info    -- projected (updates _before_ replicas ack)
+   * peer_missing -- committed (updates _after_ replicas ack)
+   */
+  
+  bool        need_up_thru;
+  set    stray_set;   // non-acting osds that have PG data.
+  eversion_t  oldest_update; // acting: lowest (valid) last_update in active set
+  map    peer_info;   // info from peers (stray or prior)
+  set peer_purged; // peers purged
+  map peer_missing;
+  set peer_log_requested;  // logs i've requested (and start stamps)
+  set peer_missing_requested;
+
+  // i deleted these strays; ignore racing PGInfo from them
+  set stray_purged;
+  set peer_activated;
+
+  // primary-only, recovery-only state
+  set might_have_unfound;  // These osds might have objects on them
+                                       // which are unfound on the primary
+  epoch_t last_peering_reset;
+
+
+  /* heartbeat peers */
+  void set_probe_targets(const set &probe_set);
+  void clear_probe_targets();
+public:
+  Mutex heartbeat_peer_lock;
+  set heartbeat_peers;
+  set probe_targets;
+
+protected:
+  /**
+   * BackfillInterval
+   *
+   * Represents the objects in a range [begin, end)
+   *
+   * Possible states:
+   * 1) begin == end == hobject_t() indicates the the interval is unpopulated
+   * 2) Else, objects contains all objects in [begin, end)
+   */
+  struct BackfillInterval {
+    // info about a backfill interval on a peer
+    eversion_t version; /// version at which the scan occurred
+    map objects;
+    hobject_t begin;
+    hobject_t end;
+    
+    /// clear content
+    void clear() {
+      *this = BackfillInterval();
+    }
+
+    void reset(hobject_t start) {
+      clear();
+      begin = end = start;
+    }
+
+    /// true if there are no objects in this interval
+    bool empty() const {
+      return objects.empty();
+    }
+
+    /// true if interval extends to the end of the range
+    bool extends_to_end() const {
+      return end.is_max();
+    }
+
+    /// removes items <= soid and adjusts begin to the first object
+    void trim_to(const hobject_t &soid) {
+      trim();
+      while (!objects.empty() && objects.begin()->first <= soid) {
+	pop_front();
+      }
+    }
+
+    /// Adjusts begin to the first object
+    void trim() {
+      if (!objects.empty())
+	begin = objects.begin()->first;
+      else
+	begin = end;
+    }
+
+    /// drop first entry, and adjust @begin accordingly
+    void pop_front() {
+      assert(!objects.empty());
+      objects.erase(objects.begin());
+      trim();
+    }
+
+    /// dump
+    void dump(Formatter *f) const {
+      f->dump_stream("begin") << begin;
+      f->dump_stream("end") << end;
+      f->open_array_section("objects");
+      for (map::const_iterator i = objects.begin();
+	   i != objects.end();
+	   ++i) {
+	f->open_object_section("object");
+	f->dump_stream("object") << i->first;
+	f->dump_stream("version") << i->second;
+	f->close_section();
+      }
+      f->close_section();
+    }
+  };
+  
+  BackfillInterval backfill_info;
+  map peer_backfill_info;
+  bool backfill_reserved;
+  bool backfill_reserving;
+
+  friend class OSD;
+
+public:
+  set backfill_targets;
+
+  bool is_backfill_targets(pg_shard_t osd) {
+    return backfill_targets.count(osd);
+  }
+
+protected:
+
+
+  // pg waiters
+  unsigned flushes_in_progress;
+
+  // Ops waiting on backfill_pos to change
+  list            waiting_for_active;
+  list            waiting_for_cache_not_full;
+  list            waiting_for_all_missing;
+  map > waiting_for_unreadable_object,
+			     waiting_for_degraded_object,
+			     waiting_for_blocked_object;
+  // Callbacks should assume pg (and nothing else) is locked
+  map > callbacks_for_degraded_object;
+  map > waiting_for_ack, waiting_for_ondisk;
+  map   replay_queue;
+  void split_ops(PG *child, unsigned split_bits);
+
+  void requeue_object_waiters(map >& m);
+  void requeue_op(OpRequestRef op);
+  void requeue_ops(list &l);
+
+  // stats that persist lazily
+  object_stat_collection_t unstable_stats;
+
+  // publish stats
+  Mutex pg_stats_publish_lock;
+  bool pg_stats_publish_valid;
+  pg_stat_t pg_stats_publish;
+
+  // for ordering writes
+  ceph::shared_ptr osr;
+
+  void _update_calc_stats();
+  void publish_stats_to_osd();
+  void clear_publish_stats();
+
+public:
+  void clear_primary_state();
+
+ public:
+  bool is_actingbackfill(pg_shard_t osd) const {
+    return actingbackfill.count(osd);
+  }
+  bool is_acting(pg_shard_t osd) const {
+    if (pool.info.ec_pool()) {
+      return acting.size() > osd.shard && acting[osd.shard] == osd.osd;
+    } else {
+      return std::find(acting.begin(), acting.end(), osd.osd) != acting.end();
+    }
+  }
+  bool is_up(pg_shard_t osd) const {
+    if (pool.info.ec_pool()) {
+      return up.size() > osd.shard && up[osd.shard] == osd.osd;
+    } else {
+      return std::find(up.begin(), up.end(), osd.osd) != up.end();
+    }
+  }
+  
+  bool needs_recovery() const;
+  bool needs_backfill() const;
+
+  void mark_clean();  ///< mark an active pg clean
+
+  bool _calc_past_interval_range(epoch_t *start, epoch_t *end);
+  void generate_past_intervals();
+  void trim_past_intervals();
+  void build_prior(std::auto_ptr &prior_set);
+
+  void remove_down_peer_info(const OSDMapRef osdmap);
+
+  bool adjust_need_up_thru(const OSDMapRef osdmap);
+
+  bool all_unfound_are_queried_or_lost(const OSDMapRef osdmap) const;
+  virtual void mark_all_unfound_lost(int how) = 0;
+  virtual void dump_recovery_info(Formatter *f) const = 0;
+
+  bool calc_min_last_complete_ondisk() {
+    eversion_t min = last_complete_ondisk;
+    assert(!actingbackfill.empty());
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == get_primary()) continue;
+      if (peer_last_complete_ondisk.count(*i) == 0)
+	return false;   // we don't have complete info
+      eversion_t a = peer_last_complete_ondisk[*i];
+      if (a < min)
+	min = a;
+    }
+    if (min == min_last_complete_ondisk)
+      return false;
+    min_last_complete_ondisk = min;
+    return true;
+  }
+
+  virtual void calc_trim_to() = 0;
+
+  void proc_replica_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog,
+			pg_missing_t& omissing, pg_shard_t from);
+  void proc_master_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog,
+		       pg_missing_t& omissing, pg_shard_t from);
+  bool proc_replica_info(pg_shard_t from, const pg_info_t &info);
+
+
+  struct LogEntryTrimmer : public ObjectModDesc::Visitor {
+    const hobject_t &soid;
+    PG *pg;
+    ObjectStore::Transaction *t;
+    LogEntryTrimmer(const hobject_t &soid, PG *pg, ObjectStore::Transaction *t)
+      : soid(soid), pg(pg), t(t) {}
+    void rmobject(version_t old_version) {
+      pg->get_pgbackend()->trim_stashed_object(
+	soid,
+	old_version,
+	t);
+    }
+  };
+
+  struct SnapRollBacker : public ObjectModDesc::Visitor {
+    const hobject_t &soid;
+    PG *pg;
+    ObjectStore::Transaction *t;
+    SnapRollBacker(const hobject_t &soid, PG *pg, ObjectStore::Transaction *t)
+      : soid(soid), pg(pg), t(t) {}
+    void update_snaps(set &snaps) {
+      pg->update_object_snap_mapping(t, soid, snaps);
+    }
+    void create() {
+      pg->clear_object_snap_mapping(
+	t,
+	soid);
+    }
+  };
+
+  struct PGLogEntryHandler : public PGLog::LogEntryHandler {
+    list to_rollback;
+    set to_remove;
+    list to_trim;
+    
+    // LogEntryHandler
+    void remove(const hobject_t &hoid) {
+      to_remove.insert(hoid);
+    }
+    void rollback(const pg_log_entry_t &entry) {
+      to_rollback.push_back(entry);
+    }
+    void trim(const pg_log_entry_t &entry) {
+      to_trim.push_back(entry);
+    }
+
+    void apply(PG *pg, ObjectStore::Transaction *t) {
+      for (list::iterator j = to_rollback.begin();
+	   j != to_rollback.end();
+	   ++j) {
+	assert(j->mod_desc.can_rollback());
+	pg->get_pgbackend()->rollback(j->soid, j->mod_desc, t);
+	SnapRollBacker rollbacker(j->soid, pg, t);
+	j->mod_desc.visit(&rollbacker);
+      }
+      for (set::iterator i = to_remove.begin();
+	   i != to_remove.end();
+	   ++i) {
+	pg->get_pgbackend()->rollback_create(*i, t);
+	pg->remove_snap_mapped_object(*t, *i);
+      }
+      for (list::reverse_iterator i = to_trim.rbegin();
+	   i != to_trim.rend();
+	   ++i) {
+	LogEntryTrimmer trimmer(i->soid, pg, t);
+	i->mod_desc.visit(&trimmer);
+      }
+    }
+  };
+  
+  friend struct SnapRollBacker;
+  friend struct PGLogEntryHandler;
+  friend struct LogEntryTrimmer;
+  void update_object_snap_mapping(
+    ObjectStore::Transaction *t, const hobject_t &soid,
+    const set &snaps);
+  void clear_object_snap_mapping(
+    ObjectStore::Transaction *t, const hobject_t &soid);
+  void remove_snap_mapped_object(
+    ObjectStore::Transaction& t, const hobject_t& soid);
+  void merge_log(
+    ObjectStore::Transaction& t, pg_info_t &oinfo,
+    pg_log_t &olog, pg_shard_t from);
+  void rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead);
+  bool search_for_missing(
+    const pg_info_t &oinfo, const pg_missing_t &omissing,
+    pg_shard_t fromosd,
+    RecoveryCtx*);
+
+  void check_for_lost_objects();
+  void forget_lost_objects();
+
+  void discover_all_missing(std::map > &query_map);
+  
+  void trim_write_ahead();
+
+  map::const_iterator find_best_info(
+    const map &infos) const;
+  static void calc_ec_acting(
+    map::const_iterator auth_log_shard,
+    unsigned size,
+    const vector &acting,
+    pg_shard_t acting_primary,
+    const vector &up,
+    pg_shard_t up_primary,
+    const map &all_info,
+    bool compat_mode,
+    vector *want,
+    set *backfill,
+    set *acting_backfill,
+    pg_shard_t *want_primary,
+    ostream &ss);
+  static void calc_replicated_acting(
+    map::const_iterator auth_log_shard,
+    unsigned size,
+    const vector &acting,
+    pg_shard_t acting_primary,
+    const vector &up,
+    pg_shard_t up_primary,
+    const map &all_info,
+    bool compat_mode,
+    vector *want,
+    set *backfill,
+    set *acting_backfill,
+    pg_shard_t *want_primary,
+    ostream &ss);
+  bool choose_acting(pg_shard_t &auth_log_shard);
+  void build_might_have_unfound();
+  void replay_queued_ops();
+  void activate(
+    ObjectStore::Transaction& t,
+    epoch_t query_epoch,
+    list& tfin,
+    map >& query_map,
+    map > > *activator_map,
+    RecoveryCtx *ctx);
+  void _activate_committed(epoch_t e);
+  void all_activated_and_committed();
+
+  void proc_primary_info(ObjectStore::Transaction &t, const pg_info_t &info);
+
+  bool have_unfound() const { 
+    return missing_loc.num_unfound();
+  }
+  int get_num_unfound() const {
+    return missing_loc.num_unfound();
+  }
+
+  virtual void check_local() = 0;
+
+  /**
+   * @param ops_begun returns how many recovery ops the function started
+   * @returns true if any useful work was accomplished; false otherwise
+   */
+  virtual bool start_recovery_ops(
+    int max, RecoveryCtx *prctx,
+    ThreadPool::TPHandle &handle,
+    int *ops_begun) = 0;
+
+  void purge_strays();
+
+  void update_heartbeat_peers();
+
+  Context *finish_sync_event;
+
+  void finish_recovery(list& tfin);
+  void _finish_recovery(Context *c);
+  void cancel_recovery();
+  void clear_recovery_state();
+  virtual void _clear_recovery_state() = 0;
+  virtual void check_recovery_sources(const OSDMapRef newmap) = 0;
+  void start_recovery_op(const hobject_t& soid);
+  void finish_recovery_op(const hobject_t& soid, bool dequeue=false);
+
+  void split_into(pg_t child_pgid, PG *child, unsigned split_bits);
+  virtual void _split_into(pg_t child_pgid, PG *child, unsigned split_bits) = 0;
+
+  loff_t get_log_write_pos() {
+    return 0;
+  }
+
+  friend class C_OSD_RepModify_Commit;
+
+
+  // -- scrub --
+  struct Scrubber {
+    Scrubber() :
+      reserved(false), reserve_failed(false),
+      epoch_start(0),
+      block_writes(false), active(false), queue_snap_trim(false),
+      waiting_on(0), shallow_errors(0), deep_errors(0), fixed(0),
+      active_rep_scrub(0),
+      must_scrub(false), must_deep_scrub(false), must_repair(false),
+      classic(false),
+      finalizing(false), is_chunky(false), state(INACTIVE),
+      deep(false)
+    {
+    }
+
+    // metadata
+    set reserved_peers;
+    bool reserved, reserve_failed;
+    epoch_t epoch_start;
+
+    // common to both scrubs
+    bool block_writes;
+    bool active;
+    bool queue_snap_trim;
+    int waiting_on;
+    set waiting_on_whom;
+    int shallow_errors;
+    int deep_errors;
+    int fixed;
+    ScrubMap primary_scrubmap;
+    map received_maps;
+    MOSDRepScrub *active_rep_scrub;
+    utime_t scrub_reg_stamp;  // stamp we registered for
+
+    // flags to indicate explicitly requested scrubs (by admin)
+    bool must_scrub, must_deep_scrub, must_repair;
+
+    // Maps from objects with errors to missing/inconsistent peers
+    map > missing;
+    map > inconsistent;
+    map > inconsistent_snapcolls;
+
+    // Map from object with errors to good peer
+    map > authoritative;
+
+    // classic scrub
+    bool classic;
+    bool finalizing;
+
+    // chunky scrub
+    bool is_chunky;
+    hobject_t start, end;
+    eversion_t subset_last_update;
+
+    // chunky scrub state
+    enum State {
+      INACTIVE,
+      NEW_CHUNK,
+      WAIT_PUSHES,
+      WAIT_LAST_UPDATE,
+      BUILD_MAP,
+      WAIT_REPLICAS,
+      COMPARE_MAPS,
+      FINISH,
+    } state;
+
+    // deep scrub
+    bool deep;
+
+    list callbacks;
+    void add_callback(Context *context) {
+      callbacks.push_back(context);
+    }
+    void run_callbacks() {
+      list to_run;
+      to_run.swap(callbacks);
+      for (list::iterator i = to_run.begin();
+	   i != to_run.end();
+	   ++i) {
+	(*i)->complete(0);
+      }
+    }
+
+    static const char *state_string(const PG::Scrubber::State& state) {
+      const char *ret = NULL;
+      switch( state )
+      {
+        case INACTIVE: ret = "INACTIVE"; break;
+        case NEW_CHUNK: ret = "NEW_CHUNK"; break;
+        case WAIT_PUSHES: ret = "WAIT_PUSHES"; break;
+        case WAIT_LAST_UPDATE: ret = "WAIT_LAST_UPDATE"; break;
+        case BUILD_MAP: ret = "BUILD_MAP"; break;
+        case WAIT_REPLICAS: ret = "WAIT_REPLICAS"; break;
+        case COMPARE_MAPS: ret = "COMPARE_MAPS"; break;
+        case FINISH: ret = "FINISH"; break;
+      }
+      return ret;
+    }
+
+    bool is_chunky_scrub_active() const { return state != INACTIVE; }
+
+    // classic (non chunk) scrubs block all writes
+    // chunky scrubs only block writes to a range
+    bool write_blocked_by_scrub(const hobject_t &soid) {
+      if (!block_writes)
+	return false;
+
+      if (!is_chunky)
+	return true;
+
+      if (soid >= start && soid < end)
+	return true;
+
+      return false;
+    }
+
+    // clear all state
+    void reset() {
+      classic = false;
+      finalizing = false;
+      block_writes = false;
+      active = false;
+      queue_snap_trim = false;
+      waiting_on = 0;
+      waiting_on_whom.clear();
+      if (active_rep_scrub) {
+        active_rep_scrub->put();
+        active_rep_scrub = NULL;
+      }
+      received_maps.clear();
+
+      must_scrub = false;
+      must_deep_scrub = false;
+      must_repair = false;
+
+      state = PG::Scrubber::INACTIVE;
+      start = hobject_t();
+      end = hobject_t();
+      subset_last_update = eversion_t();
+      shallow_errors = 0;
+      deep_errors = 0;
+      fixed = 0;
+      deep = false;
+      run_callbacks();
+      inconsistent.clear();
+      missing.clear();
+      authoritative.clear();
+    }
+
+  } scrubber;
+
+  bool scrub_after_recovery;
+
+  int active_pushes;
+
+  void repair_object(
+    const hobject_t& soid, ScrubMap::object *po,
+    pg_shard_t bad_peer,
+    pg_shard_t ok_peer);
+
+  void scrub(ThreadPool::TPHandle &handle);
+  void classic_scrub(ThreadPool::TPHandle &handle);
+  void chunky_scrub(ThreadPool::TPHandle &handle);
+  void scrub_compare_maps();
+  void scrub_process_inconsistent();
+  void scrub_finalize();
+  void scrub_finish();
+  void scrub_clear_state();
+  bool scrub_gather_replica_maps();
+  void _scan_snaps(ScrubMap &map);
+  void _scan_rollback_obs(
+    const vector &rollback_obs,
+    ThreadPool::TPHandle &handle);
+  void _request_scrub_map_classic(pg_shard_t replica, eversion_t version);
+  void _request_scrub_map(pg_shard_t replica, eversion_t version,
+                          hobject_t start, hobject_t end, bool deep);
+  int build_scrub_map_chunk(
+    ScrubMap &map,
+    hobject_t start, hobject_t end, bool deep,
+    ThreadPool::TPHandle &handle);
+  void build_scrub_map(ScrubMap &map, ThreadPool::TPHandle &handle);
+  void build_inc_scrub_map(
+    ScrubMap &map, eversion_t v, ThreadPool::TPHandle &handle);
+  /**
+   * returns true if [begin, end) is good to scrub at this time
+   * a false return value obliges the implementer to requeue scrub when the
+   * condition preventing scrub clears
+   */
+  virtual bool _range_available_for_scrub(
+    const hobject_t &begin, const hobject_t &end) = 0;
+  virtual void _scrub(ScrubMap &map) { }
+  virtual void _scrub_clear_state() { }
+  virtual void _scrub_finish() { }
+  virtual void get_colls(list *out) = 0;
+  virtual void split_colls(
+    spg_t child,
+    int split_bits,
+    int seed,
+    ObjectStore::Transaction *t) = 0;
+  virtual bool _report_snap_collection_errors(
+    const hobject_t &hoid,
+    const map &attrs,
+    pg_shard_t osd,
+    ostream &out) { return false; };
+  void clear_scrub_reserved();
+  void scrub_reserve_replicas();
+  void scrub_unreserve_replicas();
+  bool scrub_all_replicas_reserved() const;
+  bool sched_scrub();
+  void reg_next_scrub();
+  void unreg_next_scrub();
+
+  void replica_scrub(
+    struct MOSDRepScrub *op,
+    ThreadPool::TPHandle &handle);
+  void sub_op_scrub_map(OpRequestRef op);
+  void sub_op_scrub_reserve(OpRequestRef op);
+  void sub_op_scrub_reserve_reply(OpRequestRef op);
+  void sub_op_scrub_unreserve(OpRequestRef op);
+  void sub_op_scrub_stop(OpRequestRef op);
+
+  void reject_reservation();
+  void schedule_backfill_full_retry();
+
+  // -- recovery state --
+
+  template 
+  struct QueuePeeringEvt : Context {
+    PGRef pg;
+    epoch_t epoch;
+    EVT evt;
+    QueuePeeringEvt(PG *pg, epoch_t epoch, EVT evt) :
+      pg(pg), epoch(epoch), evt(evt) {}
+    void finish(int r) {
+      pg->lock();
+      pg->queue_peering_event(PG::CephPeeringEvtRef(
+				new PG::CephPeeringEvt(
+				  epoch,
+				  epoch,
+				  evt)));
+      pg->unlock();
+    }
+  };
+
+  class CephPeeringEvt {
+    epoch_t epoch_sent;
+    epoch_t epoch_requested;
+    boost::intrusive_ptr< const boost::statechart::event_base > evt;
+    string desc;
+  public:
+    template 
+    CephPeeringEvt(epoch_t epoch_sent,
+		   epoch_t epoch_requested,
+		   const T &evt_) :
+      epoch_sent(epoch_sent), epoch_requested(epoch_requested),
+      evt(evt_.intrusive_from_this()) {
+      stringstream out;
+      out << "epoch_sent: " << epoch_sent
+	  << " epoch_requested: " << epoch_requested << " ";
+      evt_.print(&out);
+      desc = out.str();
+    }
+    epoch_t get_epoch_sent() { return epoch_sent; }
+    epoch_t get_epoch_requested() { return epoch_requested; }
+    const boost::statechart::event_base &get_event() { return *evt; }
+    string get_desc() { return desc; }
+  };
+  typedef ceph::shared_ptr CephPeeringEvtRef;
+  list peering_queue;  // op queue
+  list peering_waiters;
+
+  struct QueryState : boost::statechart::event< QueryState > {
+    Formatter *f;
+    QueryState(Formatter *f) : f(f) {}
+    void print(std::ostream *out) const {
+      *out << "Query";
+    }
+  };
+
+  struct MInfoRec : boost::statechart::event< MInfoRec > {
+    pg_shard_t from;
+    pg_info_t info;
+    epoch_t msg_epoch;
+    MInfoRec(pg_shard_t from, pg_info_t &info, epoch_t msg_epoch) :
+      from(from), info(info), msg_epoch(msg_epoch) {}
+    void print(std::ostream *out) const {
+      *out << "MInfoRec from " << from << " info: " << info;
+    }
+  };
+
+  struct MLogRec : boost::statechart::event< MLogRec > {
+    pg_shard_t from;
+    boost::intrusive_ptr msg;
+    MLogRec(pg_shard_t from, MOSDPGLog *msg) :
+      from(from), msg(msg) {}
+    void print(std::ostream *out) const {
+      *out << "MLogRec from " << from;
+    }
+  };
+
+  struct MNotifyRec : boost::statechart::event< MNotifyRec > {
+    pg_shard_t from;
+    pg_notify_t notify;
+    MNotifyRec(pg_shard_t from, pg_notify_t ¬ify) :
+      from(from), notify(notify) {}
+    void print(std::ostream *out) const {
+      *out << "MNotifyRec from " << from << " notify: " << notify;
+    }
+  };
+
+  struct MQuery : boost::statechart::event< MQuery > {
+    pg_shard_t from;
+    pg_query_t query;
+    epoch_t query_epoch;
+    MQuery(pg_shard_t from, const pg_query_t &query, epoch_t query_epoch):
+      from(from), query(query), query_epoch(query_epoch) {}
+    void print(std::ostream *out) const {
+      *out << "MQuery from " << from
+	   << " query_epoch " << query_epoch
+	   << " query: " << query;
+    }
+  };
+
+  struct AdvMap : boost::statechart::event< AdvMap > {
+    OSDMapRef osdmap;
+    OSDMapRef lastmap;
+    vector newup, newacting;
+    int up_primary, acting_primary;
+    AdvMap(
+      OSDMapRef osdmap, OSDMapRef lastmap,
+      vector& newup, int up_primary,
+      vector& newacting, int acting_primary):
+      osdmap(osdmap), lastmap(lastmap),
+      newup(newup),
+      newacting(newacting),
+      up_primary(up_primary),
+      acting_primary(acting_primary) {}
+    void print(std::ostream *out) const {
+      *out << "AdvMap";
+    }
+  };
+
+  struct ActMap : boost::statechart::event< ActMap > {
+    ActMap() : boost::statechart::event< ActMap >() {}
+    void print(std::ostream *out) const {
+      *out << "ActMap";
+    }
+  };
+  struct Activate : boost::statechart::event< Activate > {
+    epoch_t query_epoch;
+    Activate(epoch_t q) : boost::statechart::event< Activate >(),
+			  query_epoch(q) {}
+    void print(std::ostream *out) const {
+      *out << "Activate from " << query_epoch;
+    }
+  };
+  struct RequestBackfillPrio : boost::statechart::event< RequestBackfillPrio > {
+    unsigned priority;
+    RequestBackfillPrio(unsigned prio) :
+              boost::statechart::event< RequestBackfillPrio >(),
+			  priority(prio) {}
+    void print(std::ostream *out) const {
+      *out << "RequestBackfillPrio: priority " << priority;
+    }
+  };
+#define TrivialEvent(T) struct T : boost::statechart::event< T > { \
+    T() : boost::statechart::event< T >() {}			   \
+    void print(std::ostream *out) const {			   \
+      *out << #T;						   \
+    }								   \
+  };
+  TrivialEvent(Initialize)
+  TrivialEvent(Load)
+  TrivialEvent(GotInfo)
+  TrivialEvent(NeedUpThru)
+  TrivialEvent(CheckRepops)
+  TrivialEvent(NullEvt)
+  TrivialEvent(FlushedEvt)
+  TrivialEvent(Backfilled)
+  TrivialEvent(LocalBackfillReserved)
+  TrivialEvent(RemoteBackfillReserved)
+  TrivialEvent(RemoteReservationRejected)
+  TrivialEvent(RequestBackfill)
+  TrivialEvent(RequestRecovery)
+  TrivialEvent(RecoveryDone)
+  TrivialEvent(BackfillTooFull)
+
+  TrivialEvent(AllReplicasRecovered)
+  TrivialEvent(DoRecovery)
+  TrivialEvent(LocalRecoveryReserved)
+  TrivialEvent(RemoteRecoveryReserved)
+  TrivialEvent(AllRemotesReserved)
+  TrivialEvent(AllBackfillsReserved)
+  TrivialEvent(Recovering)
+  TrivialEvent(GoClean)
+
+  TrivialEvent(AllReplicasActivated)
+
+  TrivialEvent(IntervalFlush)
+
+  /* Encapsulates PG recovery process */
+  class RecoveryState {
+    void start_handle(RecoveryCtx *new_ctx);
+    void end_handle();
+  public:
+    void begin_block_outgoing();
+    void end_block_outgoing();
+    void clear_blocked_outgoing();
+  private:
+
+    /* States */
+    struct Initial;
+    class RecoveryMachine : public boost::statechart::state_machine< RecoveryMachine, Initial > {
+      RecoveryState *state;
+    public:
+      PG *pg;
+
+      utime_t event_time;
+      uint64_t event_count;
+      
+      void clear_event_counters() {
+	event_time = utime_t();
+	event_count = 0;
+      }
+
+      void log_enter(const char *state_name);
+      void log_exit(const char *state_name, utime_t duration);
+
+      RecoveryMachine(RecoveryState *state, PG *pg) : state(state), pg(pg), event_count(0) {}
+
+      /* Accessor functions for state methods */
+      ObjectStore::Transaction* get_cur_transaction() {
+	assert(state->rctx);
+	assert(state->rctx->transaction);
+	return state->rctx->transaction;
+      }
+
+      void send_query(pg_shard_t to, const pg_query_t &query) {
+	assert(state->rctx);
+	assert(state->rctx->query_map);
+	(*state->rctx->query_map)[to.osd][spg_t(pg->info.pgid.pgid, to.shard)] =
+	  query;
+      }
+
+      map > *get_query_map() {
+	assert(state->rctx);
+	assert(state->rctx->query_map);
+	return state->rctx->query_map;
+      }
+
+      map > > *get_info_map() {
+	assert(state->rctx);
+	assert(state->rctx->info_map);
+	return state->rctx->info_map;
+      }
+
+      list< Context* > *get_on_safe_context_list() {
+	assert(state->rctx);
+	assert(state->rctx->on_safe);
+	return &(state->rctx->on_safe->contexts);
+      }
+
+      list< Context * > *get_on_applied_context_list() {
+	assert(state->rctx);
+	assert(state->rctx->on_applied);
+	return &(state->rctx->on_applied->contexts);
+      }
+
+      RecoveryCtx *get_recovery_ctx() { return &*(state->rctx); }
+
+      void send_notify(pg_shard_t to,
+		       const pg_notify_t &info, const pg_interval_map_t &pi) {
+	assert(state->rctx);
+	assert(state->rctx->notify_list);
+	(*state->rctx->notify_list)[to.osd].push_back(make_pair(info, pi));
+      }
+    };
+    friend class RecoveryMachine;
+
+    /* States */
+
+    struct Crashed : boost::statechart::state< Crashed, RecoveryMachine >, NamedState {
+      Crashed(my_context ctx);
+    };
+
+    struct Started;
+    struct Reset;
+
+    struct Initial : boost::statechart::state< Initial, RecoveryMachine >, NamedState {
+      Initial(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::transition< Initialize, Reset >,
+	boost::statechart::custom_reaction< Load >,
+	boost::statechart::custom_reaction< NullEvt >,
+	boost::statechart::transition< boost::statechart::event_base, Crashed >
+	> reactions;
+
+      boost::statechart::result react(const Load&);
+      boost::statechart::result react(const MNotifyRec&);
+      boost::statechart::result react(const MInfoRec&);
+      boost::statechart::result react(const MLogRec&);
+      boost::statechart::result react(const boost::statechart::event_base&) {
+	return discard_event();
+      }
+    };
+
+    struct Reset : boost::statechart::state< Reset, RecoveryMachine >, NamedState {
+      Reset(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< AdvMap >,
+	boost::statechart::custom_reaction< ActMap >,
+	boost::statechart::custom_reaction< NullEvt >,
+	boost::statechart::custom_reaction< FlushedEvt >,
+	boost::statechart::custom_reaction< IntervalFlush >,
+	boost::statechart::transition< boost::statechart::event_base, Crashed >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const AdvMap&);
+      boost::statechart::result react(const ActMap&);
+      boost::statechart::result react(const FlushedEvt&);
+      boost::statechart::result react(const IntervalFlush&);
+      boost::statechart::result react(const boost::statechart::event_base&) {
+	return discard_event();
+      }
+    };
+
+    struct Start;
+
+    struct Started : boost::statechart::state< Started, RecoveryMachine, Start >, NamedState {
+      Started(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< AdvMap >,
+	boost::statechart::custom_reaction< NullEvt >,
+	boost::statechart::custom_reaction< FlushedEvt >,
+	boost::statechart::custom_reaction< IntervalFlush >,
+	boost::statechart::transition< boost::statechart::event_base, Crashed >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const AdvMap&);
+      boost::statechart::result react(const FlushedEvt&);
+      boost::statechart::result react(const IntervalFlush&);
+      boost::statechart::result react(const boost::statechart::event_base&) {
+	return discard_event();
+      }
+    };
+
+    struct MakePrimary : boost::statechart::event< MakePrimary > {
+      MakePrimary() : boost::statechart::event< MakePrimary >() {}
+    };
+    struct MakeStray : boost::statechart::event< MakeStray > {
+      MakeStray() : boost::statechart::event< MakeStray >() {}
+    };
+    struct Primary;
+    struct Stray;
+
+    struct Start : boost::statechart::state< Start, Started >, NamedState {
+      Start(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::transition< MakePrimary, Primary >,
+	boost::statechart::transition< MakeStray, Stray >
+	> reactions;
+    };
+
+    struct Peering;
+    struct WaitActingChange;
+    struct NeedActingChange : boost::statechart::event< NeedActingChange > {
+      NeedActingChange() : boost::statechart::event< NeedActingChange >() {}
+    };
+    struct Incomplete;
+    struct IsIncomplete : boost::statechart::event< IsIncomplete > {
+      IsIncomplete() : boost::statechart::event< IsIncomplete >() {}
+    };
+
+    struct Primary : boost::statechart::state< Primary, Started, Peering >, NamedState {
+      Primary(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< ActMap >,
+	boost::statechart::custom_reaction< MNotifyRec >,
+	boost::statechart::transition< NeedActingChange, WaitActingChange >
+	> reactions;
+      boost::statechart::result react(const ActMap&);
+      boost::statechart::result react(const MNotifyRec&);
+    };
+
+    struct WaitActingChange : boost::statechart::state< WaitActingChange, Primary>,
+			      NamedState {
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< AdvMap >,
+	boost::statechart::custom_reaction< MLogRec >,
+	boost::statechart::custom_reaction< MInfoRec >,
+	boost::statechart::custom_reaction< MNotifyRec >
+	> reactions;
+      WaitActingChange(my_context ctx);
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const AdvMap&);
+      boost::statechart::result react(const MLogRec&);
+      boost::statechart::result react(const MInfoRec&);
+      boost::statechart::result react(const MNotifyRec&);
+      void exit();
+    };
+
+    struct GetInfo;
+    struct Active;
+
+    struct Peering : boost::statechart::state< Peering, Primary, GetInfo >, NamedState {
+      std::auto_ptr< PriorSet > prior_set;
+
+      Peering(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::transition< Activate, Active >,
+	boost::statechart::custom_reaction< AdvMap >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const AdvMap &advmap);
+    };
+
+    struct WaitLocalRecoveryReserved;
+    struct Activating;
+    struct Active : boost::statechart::state< Active, Primary, Activating >, NamedState {
+      Active(my_context ctx);
+      void exit();
+
+      const set remote_shards_to_reserve_recovery;
+      const set remote_shards_to_reserve_backfill;
+      bool all_replicas_activated;
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< ActMap >,
+	boost::statechart::custom_reaction< AdvMap >,
+	boost::statechart::custom_reaction< MInfoRec >,
+	boost::statechart::custom_reaction< MNotifyRec >,
+	boost::statechart::custom_reaction< MLogRec >,
+	boost::statechart::custom_reaction< Backfilled >,
+	boost::statechart::custom_reaction< AllReplicasActivated >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const ActMap&);
+      boost::statechart::result react(const AdvMap&);
+      boost::statechart::result react(const MInfoRec& infoevt);
+      boost::statechart::result react(const MNotifyRec& notevt);
+      boost::statechart::result react(const MLogRec& logevt);
+      boost::statechart::result react(const Backfilled&) {
+	return discard_event();
+      }
+      boost::statechart::result react(const AllReplicasActivated&);
+    };
+
+    struct Clean : boost::statechart::state< Clean, Active >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::transition< DoRecovery, WaitLocalRecoveryReserved >
+      > reactions;
+      Clean(my_context ctx);
+      void exit();
+    };
+
+    struct Recovered : boost::statechart::state< Recovered, Active >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::transition< GoClean, Clean >,
+	boost::statechart::custom_reaction< AllReplicasActivated >
+      > reactions;
+      Recovered(my_context ctx);
+      void exit();
+      boost::statechart::result react(const AllReplicasActivated&) {
+	post_event(GoClean());
+	return forward_event();
+      }
+    };
+
+    struct Backfilling : boost::statechart::state< Backfilling, Active >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::transition< Backfilled, Recovered >,
+	boost::statechart::custom_reaction< RemoteReservationRejected >
+	> reactions;
+      Backfilling(my_context ctx);
+      boost::statechart::result react(const RemoteReservationRejected& evt);
+      void exit();
+    };
+
+    struct WaitRemoteBackfillReserved : boost::statechart::state< WaitRemoteBackfillReserved, Active >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::custom_reaction< RemoteBackfillReserved >,
+	boost::statechart::custom_reaction< RemoteReservationRejected >,
+	boost::statechart::transition< AllBackfillsReserved, Backfilling >
+	> reactions;
+      set::const_iterator backfill_osd_it;
+      WaitRemoteBackfillReserved(my_context ctx);
+      void exit();
+      boost::statechart::result react(const RemoteBackfillReserved& evt);
+      boost::statechart::result react(const RemoteReservationRejected& evt);
+    };
+
+    struct WaitLocalBackfillReserved : boost::statechart::state< WaitLocalBackfillReserved, Active >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::transition< LocalBackfillReserved, WaitRemoteBackfillReserved >
+	> reactions;
+      WaitLocalBackfillReserved(my_context ctx);
+      void exit();
+    };
+
+    struct NotBackfilling : boost::statechart::state< NotBackfilling, Active>, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::transition< RequestBackfill, WaitLocalBackfillReserved>,
+	boost::statechart::custom_reaction< RemoteBackfillReserved >,
+	boost::statechart::custom_reaction< RemoteReservationRejected >
+	> reactions;
+      NotBackfilling(my_context ctx);
+      void exit();
+      boost::statechart::result react(const RemoteBackfillReserved& evt);
+      boost::statechart::result react(const RemoteReservationRejected& evt);
+    };
+
+    struct RepNotRecovering;
+    struct ReplicaActive : boost::statechart::state< ReplicaActive, Started, RepNotRecovering >, NamedState {
+      ReplicaActive(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< ActMap >,
+	boost::statechart::custom_reaction< MQuery >,
+	boost::statechart::custom_reaction< MInfoRec >,
+	boost::statechart::custom_reaction< MLogRec >,
+	boost::statechart::custom_reaction< Activate >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const MInfoRec& infoevt);
+      boost::statechart::result react(const MLogRec& logevt);
+      boost::statechart::result react(const ActMap&);
+      boost::statechart::result react(const MQuery&);
+      boost::statechart::result react(const Activate&);
+    };
+
+    struct RepRecovering : boost::statechart::state< RepRecovering, ReplicaActive >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::transition< RecoveryDone, RepNotRecovering >,
+	boost::statechart::transition< RemoteReservationRejected, RepNotRecovering >,
+	boost::statechart::custom_reaction< BackfillTooFull >
+	> reactions;
+      RepRecovering(my_context ctx);
+      boost::statechart::result react(const BackfillTooFull &evt);
+      void exit();
+    };
+
+    struct RepWaitBackfillReserved : boost::statechart::state< RepWaitBackfillReserved, ReplicaActive >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::custom_reaction< RemoteBackfillReserved >,
+	boost::statechart::custom_reaction< RemoteReservationRejected >
+	> reactions;
+      RepWaitBackfillReserved(my_context ctx);
+      void exit();
+      boost::statechart::result react(const RemoteBackfillReserved &evt);
+      boost::statechart::result react(const RemoteReservationRejected &evt);
+    };
+
+    struct RepWaitRecoveryReserved : boost::statechart::state< RepWaitRecoveryReserved, ReplicaActive >, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::custom_reaction< RemoteRecoveryReserved >
+	> reactions;
+      RepWaitRecoveryReserved(my_context ctx);
+      void exit();
+      boost::statechart::result react(const RemoteRecoveryReserved &evt);
+    };
+
+    struct RepNotRecovering : boost::statechart::state< RepNotRecovering, ReplicaActive>, NamedState {
+      typedef boost::mpl::list<
+	boost::statechart::custom_reaction< RequestBackfillPrio >,
+        boost::statechart::transition< RequestRecovery, RepWaitRecoveryReserved >,
+	boost::statechart::transition< RecoveryDone, RepNotRecovering >  // for compat with pre-reservation peers
+	> reactions;
+      RepNotRecovering(my_context ctx);
+      boost::statechart::result react(const RequestBackfillPrio &evt);
+      void exit();
+    };
+
+    struct Recovering : boost::statechart::state< Recovering, Active >, NamedState {
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< AllReplicasRecovered >,
+	boost::statechart::custom_reaction< RequestBackfill >
+	> reactions;
+      Recovering(my_context ctx);
+      void exit();
+      void release_reservations();
+      boost::statechart::result react(const AllReplicasRecovered &evt);
+      boost::statechart::result react(const RequestBackfill &evt);
+    };
+
+    struct WaitRemoteRecoveryReserved : boost::statechart::state< WaitRemoteRecoveryReserved, Active >, NamedState {
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< RemoteRecoveryReserved >,
+	boost::statechart::transition< AllRemotesReserved, Recovering >
+	> reactions;
+      set::const_iterator remote_recovery_reservation_it;
+      WaitRemoteRecoveryReserved(my_context ctx);
+      boost::statechart::result react(const RemoteRecoveryReserved &evt);
+      void exit();
+    };
+
+    struct WaitLocalRecoveryReserved : boost::statechart::state< WaitLocalRecoveryReserved, Active >, NamedState {
+      typedef boost::mpl::list <
+	boost::statechart::transition< LocalRecoveryReserved, WaitRemoteRecoveryReserved >
+	> reactions;
+      WaitLocalRecoveryReserved(my_context ctx);
+      void exit();
+    };
+
+    struct Activating : boost::statechart::state< Activating, Active >, NamedState {
+      typedef boost::mpl::list <
+	boost::statechart::transition< AllReplicasRecovered, Recovered >,
+	boost::statechart::transition< DoRecovery, WaitLocalRecoveryReserved >,
+	boost::statechart::transition< RequestBackfill, WaitLocalBackfillReserved >
+	> reactions;
+      Activating(my_context ctx);
+      void exit();
+    };
+
+    struct Stray : boost::statechart::state< Stray, Started >, NamedState {
+      map > pending_queries;
+
+      Stray(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< MQuery >,
+	boost::statechart::custom_reaction< MLogRec >,
+	boost::statechart::custom_reaction< MInfoRec >,
+	boost::statechart::custom_reaction< ActMap >,
+	boost::statechart::custom_reaction< RecoveryDone >
+	> reactions;
+      boost::statechart::result react(const MQuery& query);
+      boost::statechart::result react(const MLogRec& logevt);
+      boost::statechart::result react(const MInfoRec& infoevt);
+      boost::statechart::result react(const ActMap&);
+      boost::statechart::result react(const RecoveryDone&) {
+	return discard_event();
+      }
+    };
+
+    struct GetLog;
+
+    struct GetInfo : boost::statechart::state< GetInfo, Peering >, NamedState {
+      set peer_info_requested;
+
+      GetInfo(my_context ctx);
+      void exit();
+      void get_infos();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::transition< GotInfo, GetLog >,
+	boost::statechart::custom_reaction< MNotifyRec >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const MNotifyRec& infoevt);
+    };
+
+    struct GetMissing;
+    struct GotLog : boost::statechart::event< GotLog > {
+      GotLog() : boost::statechart::event< GotLog >() {}
+    };
+
+    struct GetLog : boost::statechart::state< GetLog, Peering >, NamedState {
+      pg_shard_t auth_log_shard;
+      boost::intrusive_ptr msg;
+
+      GetLog(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< MLogRec >,
+	boost::statechart::custom_reaction< GotLog >,
+	boost::statechart::custom_reaction< AdvMap >,
+	boost::statechart::transition< IsIncomplete, Incomplete >
+	> reactions;
+      boost::statechart::result react(const AdvMap&);
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const MLogRec& logevt);
+      boost::statechart::result react(const GotLog&);
+    };
+
+    struct WaitUpThru;
+
+    struct GetMissing : boost::statechart::state< GetMissing, Peering >, NamedState {
+      set peer_missing_requested;
+
+      GetMissing(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< MLogRec >,
+	boost::statechart::transition< NeedUpThru, WaitUpThru >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const MLogRec& logevt);
+    };
+
+    struct WaitUpThru : boost::statechart::state< WaitUpThru, Peering >, NamedState {
+      WaitUpThru(my_context ctx);
+      void exit();
+
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< QueryState >,
+	boost::statechart::custom_reaction< ActMap >,
+	boost::statechart::custom_reaction< MLogRec >
+	> reactions;
+      boost::statechart::result react(const QueryState& q);
+      boost::statechart::result react(const ActMap& am);
+      boost::statechart::result react(const MLogRec& logrec);
+    };
+
+    struct Incomplete : boost::statechart::state< Incomplete, Peering>, NamedState {
+      typedef boost::mpl::list <
+	boost::statechart::custom_reaction< AdvMap >,
+	boost::statechart::custom_reaction< MNotifyRec >
+	> reactions;
+      Incomplete(my_context ctx);
+      boost::statechart::result react(const AdvMap &advmap);
+      boost::statechart::result react(const MNotifyRec& infoevt);
+      void exit();
+    };
+
+
+    RecoveryMachine machine;
+    PG *pg;
+
+    /// context passed in by state machine caller
+    RecoveryCtx *orig_ctx;
+
+    /// populated if we are buffering messages pending a flush
+    boost::optional messages_pending_flush;
+
+    /**
+     * populated between start_handle() and end_handle(), points into
+     * the message lists for messages_pending_flush while blocking messages
+     * or into orig_ctx otherwise
+     */
+    boost::optional rctx;
+
+  public:
+    RecoveryState(PG *pg)
+      : machine(this, pg), pg(pg), orig_ctx(0) {
+      machine.initiate();
+    }
+
+    void handle_event(const boost::statechart::event_base &evt,
+		      RecoveryCtx *rctx) {
+      start_handle(rctx);
+      machine.process_event(evt);
+      end_handle();
+    }
+
+    void handle_event(CephPeeringEvtRef evt,
+		      RecoveryCtx *rctx) {
+      start_handle(rctx);
+      machine.process_event(evt->get_event());
+      end_handle();
+    }
+
+  } recovery_state;
+
+
+ public:
+  PG(OSDService *o, OSDMapRef curmap,
+     const PGPool &pool, spg_t p, const hobject_t& loid, const hobject_t& ioid);
+  virtual ~PG();
+
+ private:
+  // Prevent copying
+  PG(const PG& rhs);
+  PG& operator=(const PG& rhs);
+
+ public:
+  spg_t      get_pgid() const { return info.pgid; }
+  int        get_nrep() const { return acting.size(); }
+
+  void init_primary_up_acting(
+    const vector &newup,
+    const vector &newacting,
+    int new_up_primary,
+    int new_acting_primary) {
+    actingset.clear();
+    acting = newacting;
+    for (shard_id_t i = 0; i < acting.size(); ++i) {
+      if (acting[i] != CRUSH_ITEM_NONE)
+	actingset.insert(
+	  pg_shard_t(
+	    acting[i],
+	    pool.info.ec_pool() ? i : ghobject_t::NO_SHARD));
+    }
+    up = newup;
+    if (!pool.info.ec_pool()) {
+      up_primary = pg_shard_t(new_up_primary, ghobject_t::no_shard());
+      primary = pg_shard_t(new_acting_primary, ghobject_t::no_shard());
+      return;
+    }
+    up_primary = pg_shard_t();
+    primary = pg_shard_t();
+    for (shard_id_t i = 0; i < up.size(); ++i) {
+      if (up[i] == new_up_primary) {
+	up_primary = pg_shard_t(up[i], i);
+	break;
+      }
+    }
+    for (shard_id_t i = 0; i < acting.size(); ++i) {
+      if (acting[i] == new_acting_primary) {
+	primary = pg_shard_t(acting[i], i);
+	break;
+      }
+    }
+    assert(up_primary.osd == new_up_primary);
+    assert(primary.osd == new_acting_primary);
+  }
+  pg_shard_t get_primary() const { return primary; }
+  
+  int        get_role() const { return role; }
+  void       set_role(int r) { role = r; }
+
+  bool       is_primary() const { return pg_whoami == primary; }
+  bool       is_replica() const { return role > 0; }
+
+  epoch_t get_last_peering_reset() const { return last_peering_reset; }
+  
+  //int  get_state() const { return state; }
+  bool state_test(int m) const { return (state & m) != 0; }
+  void state_set(int m) { state |= m; }
+  void state_clear(int m) { state &= ~m; }
+
+  bool is_complete() const { return info.last_complete == info.last_update; }
+  bool should_send_notify() const { return send_notify; }
+
+  int get_state() const { return state; }
+  bool       is_active() const { return state_test(PG_STATE_ACTIVE); }
+  bool       is_peering() const { return state_test(PG_STATE_PEERING); }
+  bool       is_down() const { return state_test(PG_STATE_DOWN); }
+  bool       is_replay() const { return state_test(PG_STATE_REPLAY); }
+  bool       is_clean() const { return state_test(PG_STATE_CLEAN); }
+  bool       is_degraded() const { return state_test(PG_STATE_DEGRADED); }
+
+  bool       is_scrubbing() const { return state_test(PG_STATE_SCRUBBING); }
+
+  bool  is_empty() const { return info.last_update == eversion_t(0,0); }
+
+  void init(
+    int role,
+    vector& up,
+    int up_primary,
+    vector& acting,
+    int acting_primary,
+    pg_history_t& history,
+    pg_interval_map_t& pim,
+    bool backfill,
+    ObjectStore::Transaction *t);
+
+  // pg on-disk state
+  void do_pending_flush();
+
+private:
+  void write_info(ObjectStore::Transaction& t);
+
+public:
+  static int _write_info(ObjectStore::Transaction& t, epoch_t epoch,
+    pg_info_t &info, coll_t coll,
+    map &past_intervals,
+    interval_set &snap_collections,
+    hobject_t &infos_oid,
+    __u8 info_struct_v, bool dirty_big_info, bool force_ver = false);
+  void write_if_dirty(ObjectStore::Transaction& t);
+
+  eversion_t get_next_version() const {
+    eversion_t at_version(get_osdmap()->get_epoch(),
+			  pg_log.get_head().version+1);
+    assert(at_version > info.last_update);
+    assert(at_version > pg_log.get_head());
+    return at_version;
+  }
+
+  void add_log_entry(pg_log_entry_t& e, bufferlist& log_bl);
+  void append_log(
+    vector& logv,
+    eversion_t trim_to,
+    eversion_t trim_rollback_to,
+    ObjectStore::Transaction &t,
+    bool transaction_applied = true);
+  bool check_log_for_corruption(ObjectStore *store);
+  void trim_peers();
+
+  std::string get_corrupt_pg_log_name() const;
+  static int read_info(
+    ObjectStore *store, const coll_t coll,
+    bufferlist &bl, pg_info_t &info, map &past_intervals,
+    hobject_t &biginfo_oid, hobject_t &infos_oid,
+    interval_set  &snap_collections, __u8 &);
+  void read_state(ObjectStore *store, bufferlist &bl);
+  static epoch_t peek_map_epoch(ObjectStore *store, coll_t coll,
+                               hobject_t &infos_oid, bufferlist *bl);
+  void update_snap_map(
+    vector &log_entries,
+    ObjectStore::Transaction& t);
+
+  void filter_snapc(vector &snaps);
+
+  void log_weirdness();
+
+  void queue_snap_trim();
+  bool queue_scrub();
+
+  /// share pg info after a pg is active
+  void share_pg_info();
+  /// share new pg log entries after a pg is active
+  void share_pg_log();
+
+  void reset_interval_flush();
+  void start_peering_interval(
+    const OSDMapRef lastmap,
+    const vector& newup, int up_primary,
+    const vector& newacting, int acting_primary,
+    ObjectStore::Transaction *t);
+  void start_flush(ObjectStore::Transaction *t,
+		   list *on_applied,
+		   list *on_safe);
+  void set_last_peering_reset();
+  bool pg_has_reset_since(epoch_t e) {
+    assert(is_locked());
+    return deleting || e < get_last_peering_reset();
+  }
+
+  void update_history_from_master(pg_history_t new_history);
+  void fulfill_info(pg_shard_t from, const pg_query_t &query,
+		    pair ¬ify_info);
+  void fulfill_log(pg_shard_t from, const pg_query_t &query, epoch_t query_epoch);
+  bool is_split(OSDMapRef lastmap, OSDMapRef nextmap);
+  bool acting_up_affected(
+    int newupprimary, int newactingprimary,
+    const vector& newup, const vector& newacting);
+
+  // OpRequest queueing
+  bool can_discard_op(OpRequestRef op);
+  bool can_discard_scan(OpRequestRef op);
+  bool can_discard_backfill(OpRequestRef op);
+  bool can_discard_request(OpRequestRef op);
+
+  template
+  bool can_discard_replica_op(OpRequestRef op);
+
+  static bool op_must_wait_for_map(OSDMapRef curmap, OpRequestRef op);
+
+  static bool split_request(OpRequestRef op, unsigned match, unsigned bits);
+
+  bool old_peering_msg(epoch_t reply_epoch, epoch_t query_epoch);
+  bool old_peering_evt(CephPeeringEvtRef evt) {
+    return old_peering_msg(evt->get_epoch_sent(), evt->get_epoch_requested());
+  }
+  static bool have_same_or_newer_map(OSDMapRef osdmap, epoch_t e) {
+    return e <= osdmap->get_epoch();
+  }
+  bool have_same_or_newer_map(epoch_t e) {
+    return e <= get_osdmap()->get_epoch();
+  }
+
+  bool op_has_sufficient_caps(OpRequestRef op);
+
+
+  // recovery bits
+  void take_waiters();
+  void queue_peering_event(CephPeeringEvtRef evt);
+  void handle_peering_event(CephPeeringEvtRef evt, RecoveryCtx *rctx);
+  void queue_notify(epoch_t msg_epoch, epoch_t query_epoch,
+		    pg_shard_t from, pg_notify_t& i);
+  void queue_info(epoch_t msg_epoch, epoch_t query_epoch,
+		  pg_shard_t from, pg_info_t& i);
+  void queue_log(epoch_t msg_epoch, epoch_t query_epoch, pg_shard_t from,
+		 MOSDPGLog *msg);
+  void queue_query(epoch_t msg_epoch, epoch_t query_epoch,
+		   pg_shard_t from, const pg_query_t& q);
+  void queue_null(epoch_t msg_epoch, epoch_t query_epoch);
+  void queue_flushed(epoch_t started_at);
+  void handle_advance_map(
+    OSDMapRef osdmap, OSDMapRef lastmap,
+    vector& newup, int up_primary,
+    vector& newacting, int acting_primary,
+    RecoveryCtx *rctx);
+  void handle_activate_map(RecoveryCtx *rctx);
+  void handle_create(RecoveryCtx *rctx);
+  void handle_loaded(RecoveryCtx *rctx);
+  void handle_query_state(Formatter *f);
+
+  virtual void on_removal(ObjectStore::Transaction *t) = 0;
+
+
+  // abstract bits
+  virtual void do_request(
+    OpRequestRef op,
+    ThreadPool::TPHandle &handle
+  ) = 0;
+
+  virtual void do_op(OpRequestRef op) = 0;
+  virtual void do_sub_op(OpRequestRef op) = 0;
+  virtual void do_sub_op_reply(OpRequestRef op) = 0;
+  virtual void do_scan(
+    OpRequestRef op,
+    ThreadPool::TPHandle &handle
+  ) = 0;
+  virtual void do_backfill(OpRequestRef op) = 0;
+  virtual void snap_trimmer() = 0;
+
+  virtual int do_command(cmdmap_t cmdmap, ostream& ss,
+			 bufferlist& idata, bufferlist& odata) = 0;
+
+  virtual void on_role_change() = 0;
+  virtual void on_pool_change() = 0;
+  virtual void on_change(ObjectStore::Transaction *t) = 0;
+  virtual void on_activate() = 0;
+  virtual void on_flushed() = 0;
+  virtual void on_shutdown() = 0;
+  virtual void check_blacklisted_watchers() = 0;
+  virtual void get_watchers(std::list&) = 0;
+
+  virtual bool agent_work(int max) = 0;
+  virtual void agent_stop() = 0;
+  virtual void agent_delay() = 0;
+  virtual void agent_clear() = 0;
+  virtual void agent_choose_mode_restart() = 0;
+};
+
+ostream& operator<<(ostream& out, const PG& pg);
+
+#endif
diff --git a/ceph/src/osd/PGBackend.cc b/ceph/src/osd/PGBackend.cc
new file mode 100644
index 00000000..57faadd6
--- /dev/null
+++ b/ceph/src/osd/PGBackend.cc
@@ -0,0 +1,577 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013,2014 Inktank Storage, Inc.
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+
+#include "common/errno.h"
+#include "ReplicatedBackend.h"
+#include "ECBackend.h"
+#include "PGBackend.h"
+#include "OSD.h"
+#include "erasure-code/ErasureCodePlugin.h"
+
+#define dout_subsys ceph_subsys_osd
+#define DOUT_PREFIX_ARGS this
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+static ostream& _prefix(std::ostream *_dout, PGBackend *pgb) {
+  return *_dout << pgb->get_parent()->gen_dbg_prefix();
+}
+
+// -- ObjectModDesc --
+struct RollbackVisitor : public ObjectModDesc::Visitor {
+  const hobject_t &hoid;
+  PGBackend *pg;
+  ObjectStore::Transaction t;
+  RollbackVisitor(
+    const hobject_t &hoid,
+    PGBackend *pg) : hoid(hoid), pg(pg) {}
+  void append(uint64_t old_size) {
+    ObjectStore::Transaction temp;
+    pg->rollback_append(hoid, old_size, &temp);
+    temp.append(t);
+    temp.swap(t);
+  }
+  void setattrs(map > &attrs) {
+    ObjectStore::Transaction temp;
+    pg->rollback_setattrs(hoid, attrs, &temp);
+    temp.append(t);
+    temp.swap(t);
+  }
+  void rmobject(version_t old_version) {
+    ObjectStore::Transaction temp;
+    pg->rollback_stash(hoid, old_version, &temp);
+    temp.append(t);
+    temp.swap(t);
+  }
+  void create() {
+    ObjectStore::Transaction temp;
+    pg->rollback_create(hoid, &temp);
+    temp.append(t);
+    temp.swap(t);
+  }
+  void update_snaps(set &snaps) {
+    // pass
+  }
+};
+
+void PGBackend::rollback(
+  const hobject_t &hoid,
+  const ObjectModDesc &desc,
+  ObjectStore::Transaction *t)
+{
+  assert(desc.can_rollback());
+  RollbackVisitor vis(hoid, this);
+  desc.visit(&vis);
+  t->append(vis.t);
+}
+
+
+void PGBackend::on_change_cleanup(ObjectStore::Transaction *t)
+{
+  dout(10) << __func__ << dendl;
+  // clear temp
+  for (set::iterator i = temp_contents.begin();
+       i != temp_contents.end();
+       ++i) {
+    dout(10) << __func__ << ": Removing oid "
+	     << *i << " from the temp collection" << dendl;
+    t->remove(
+      get_temp_coll(t),
+      ghobject_t(*i, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+  }
+  temp_contents.clear();
+}
+
+coll_t PGBackend::get_temp_coll(ObjectStore::Transaction *t)
+{
+  if (temp_created)
+    return temp_coll;
+  if (!store->collection_exists(temp_coll))
+      t->create_collection(temp_coll);
+  temp_created = true;
+  return temp_coll;
+}
+
+int PGBackend::objects_list_partial(
+  const hobject_t &begin,
+  int min,
+  int max,
+  snapid_t seq,
+  vector *ls,
+  hobject_t *next)
+{
+  assert(ls);
+  // Starts with the smallest shard id and generation to
+  // make sure the result list has the marker object (
+  // it might have multiple generations though, which would
+  // be filtered).
+  ghobject_t _next(begin, 0, shard_id_t(0));
+  ls->reserve(max);
+  int r = 0;
+  while (!_next.is_max() && ls->size() < (unsigned)min) {
+    vector objects;
+    int r = store->collection_list_partial(
+      coll,
+      _next,
+      min - ls->size(),
+      max - ls->size(),
+      seq,
+      &objects,
+      &_next);
+    if (r != 0)
+      break;
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i) {
+      if (i->is_no_gen()) {
+	ls->push_back(i->hobj);
+      }
+    }
+  }
+  if (r == 0)
+    *next = _next.hobj;
+  return r;
+}
+
+int PGBackend::objects_list_range(
+  const hobject_t &start,
+  const hobject_t &end,
+  snapid_t seq,
+  vector *ls,
+  vector *gen_obs)
+{
+  assert(ls);
+  vector objects;
+  int r = store->collection_list_range(
+    coll,
+    start,
+    end,
+    seq,
+    &objects);
+  ls->reserve(objects.size());
+  for (vector::iterator i = objects.begin();
+       i != objects.end();
+       ++i) {
+    if (i->is_no_gen()) {
+      ls->push_back(i->hobj);
+    } else if (gen_obs) {
+      gen_obs->push_back(*i);
+    }
+  }
+  return r;
+}
+
+int PGBackend::objects_get_attr(
+  const hobject_t &hoid,
+  const string &attr,
+  bufferlist *out)
+{
+  bufferptr bp;
+  int r = store->getattr(
+    hoid.is_temp() ? temp_coll : coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+    attr.c_str(),
+    bp);
+  if (r >= 0 && out) {
+    out->clear();
+    out->push_back(bp);
+  }
+  return r;
+}
+
+int PGBackend::objects_get_attrs(
+  const hobject_t &hoid,
+  map *out)
+{
+  return store->getattrs(
+    hoid.is_temp() ? temp_coll : coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+    *out);
+}
+
+void PGBackend::rollback_setattrs(
+  const hobject_t &hoid,
+  map > &old_attrs,
+  ObjectStore::Transaction *t) {
+  map to_set;
+  set to_remove;
+  assert(!hoid.is_temp());
+  for (map >::iterator i = old_attrs.begin();
+       i != old_attrs.end();
+       ++i) {
+    if (i->second) {
+      to_set[i->first] = i->second.get();
+    } else {
+      t->rmattr(
+	coll,
+	ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+	i->first);
+    }
+  }
+  t->setattrs(
+    coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+    to_set);
+}
+
+void PGBackend::rollback_append(
+  const hobject_t &hoid,
+  uint64_t old_size,
+  ObjectStore::Transaction *t) {
+  assert(!hoid.is_temp());
+  t->truncate(
+    coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+    old_size);
+}
+
+void PGBackend::rollback_stash(
+  const hobject_t &hoid,
+  version_t old_version,
+  ObjectStore::Transaction *t) {
+  assert(!hoid.is_temp());
+  t->remove(
+    coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+  t->collection_move_rename(
+    coll,
+    ghobject_t(hoid, old_version, get_parent()->whoami_shard().shard),
+    coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+}
+
+void PGBackend::rollback_create(
+  const hobject_t &hoid,
+  ObjectStore::Transaction *t) {
+  assert(!hoid.is_temp());
+  t->remove(
+    coll,
+    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+}
+
+void PGBackend::trim_stashed_object(
+  const hobject_t &hoid,
+  version_t old_version,
+  ObjectStore::Transaction *t) {
+  assert(!hoid.is_temp());
+  t->remove(
+    coll, ghobject_t(hoid, old_version, get_parent()->whoami_shard().shard));
+}
+
+PGBackend *PGBackend::build_pg_backend(
+  const pg_pool_t &pool,
+  const OSDMapRef curmap,
+  Listener *l,
+  coll_t coll,
+  coll_t temp_coll,
+  ObjectStore *store,
+  CephContext *cct)
+{
+  switch (pool.type) {
+  case pg_pool_t::TYPE_REPLICATED: {
+    return new ReplicatedBackend(l, coll, temp_coll, store, cct);
+  }
+  case pg_pool_t::TYPE_ERASURE: {
+    ErasureCodeInterfaceRef ec_impl;
+    const map &profile = curmap->get_erasure_code_profile(pool.erasure_code_profile);
+    assert(profile.count("plugin"));
+    stringstream ss;
+    ceph::ErasureCodePluginRegistry::instance().factory(
+      profile.find("plugin")->second,
+      profile,
+      &ec_impl,
+      ss);
+    assert(ec_impl);
+    return new ECBackend(
+      l,
+      coll,
+      temp_coll,
+      store,
+      cct,
+      ec_impl,
+      pool.stripe_width);
+  }
+  default:
+    assert(0);
+    return NULL;
+  }
+}
+
+/*
+ * pg lock may or may not be held
+ */
+void PGBackend::be_scan_list(
+  ScrubMap &map, const vector &ls, bool deep,
+  ThreadPool::TPHandle &handle)
+{
+  dout(10) << "_scan_list scanning " << ls.size() << " objects"
+           << (deep ? " deeply" : "") << dendl;
+  int i = 0;
+  for (vector::const_iterator p = ls.begin();
+       p != ls.end();
+       ++p, i++) {
+    handle.reset_tp_timeout();
+    hobject_t poid = *p;
+
+    struct stat st;
+    int r = store->stat(
+      coll,
+      ghobject_t(
+	poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+      &st,
+      true);
+    if (r == 0) {
+      ScrubMap::object &o = map.objects[poid];
+      o.size = st.st_size;
+      assert(!o.negative);
+      store->getattrs(
+	coll,
+	ghobject_t(
+	  poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+	o.attrs);
+
+      // calculate the CRC32 on deep scrubs
+      if (deep) {
+	be_deep_scrub(*p, o, handle);
+      }
+
+      dout(25) << "_scan_list  " << poid << dendl;
+    } else if (r == -ENOENT) {
+      dout(25) << "_scan_list  " << poid << " got " << r << ", skipping" << dendl;
+    } else if (r == -EIO) {
+      dout(25) << "_scan_list  " << poid << " got " << r << ", read_error" << dendl;
+      ScrubMap::object &o = map.objects[poid];
+      o.read_error = true;
+    } else {
+      derr << "_scan_list got: " << cpp_strerror(r) << dendl;
+      assert(0);
+    }
+  }
+}
+
+enum scrub_error_type PGBackend::be_compare_scrub_objects(
+  const ScrubMap::object &auth,
+  const ScrubMap::object &candidate,
+  ostream &errorstream)
+{
+  enum scrub_error_type error = CLEAN;
+  if (candidate.read_error) {
+    // This can occur on stat() of a shallow scrub, but in that case size will
+    // be invalid, and this will be over-ridden below.
+    error = DEEP_ERROR;
+    errorstream << "candidate had a read error";
+  }
+  if (auth.digest_present && candidate.digest_present) {
+    if (auth.digest != candidate.digest) {
+      if (error != CLEAN)
+        errorstream << ", ";
+      error = DEEP_ERROR;
+
+      errorstream << "digest " << candidate.digest
+                  << " != known digest " << auth.digest;
+    }
+  }
+  if (auth.omap_digest_present && candidate.omap_digest_present) {
+    if (auth.omap_digest != candidate.omap_digest) {
+      if (error != CLEAN)
+        errorstream << ", ";
+      error = DEEP_ERROR;
+
+      errorstream << "omap_digest " << candidate.omap_digest
+                  << " != known omap_digest " << auth.omap_digest;
+    }
+  }
+  // Shallow error takes precendence because this will be seen by
+  // both types of scrubs.
+  if (auth.size != candidate.size) {
+    if (error != CLEAN)
+      errorstream << ", ";
+    error = SHALLOW_ERROR;
+    errorstream << "size " << candidate.size
+		<< " != known size " << auth.size;
+  }
+  for (map::const_iterator i = auth.attrs.begin();
+       i != auth.attrs.end();
+       ++i) {
+    if (!candidate.attrs.count(i->first)) {
+      if (error != CLEAN)
+        errorstream << ", ";
+      error = SHALLOW_ERROR;
+      errorstream << "missing attr " << i->first;
+    } else if (candidate.attrs.find(i->first)->second.cmp(i->second)) {
+      if (error != CLEAN)
+        errorstream << ", ";
+      error = SHALLOW_ERROR;
+      errorstream << "attr value mismatch " << i->first;
+    }
+  }
+  for (map::const_iterator i = candidate.attrs.begin();
+       i != candidate.attrs.end();
+       ++i) {
+    if (!auth.attrs.count(i->first)) {
+      if (error != CLEAN)
+        errorstream << ", ";
+      error = SHALLOW_ERROR;
+      errorstream << "extra attr " << i->first;
+    }
+  }
+  return error;
+}
+
+map::const_iterator
+  PGBackend::be_select_auth_object(
+  const hobject_t &obj,
+  const map &maps)
+{
+  map::const_iterator auth = maps.end();
+  for (map::const_iterator j = maps.begin();
+       j != maps.end();
+       ++j) {
+    map::iterator i =
+      j->second->objects.find(obj);
+    if (i == j->second->objects.end()) {
+      continue;
+    }
+    if (auth == maps.end()) {
+      // Something is better than nothing
+      // TODO: something is NOT better than nothing, do something like
+      // unfound_lost if no valid copies can be found, or just mark unfound
+      auth = j;
+      dout(10) << __func__ << ": selecting osd " << j->first
+	       << " for obj " << obj
+	       << ", auth == maps.end()"
+	       << dendl;
+      continue;
+    }
+    if (i->second.read_error) {
+      // scrub encountered read error, probably corrupt
+      dout(10) << __func__ << ": rejecting osd " << j->first
+	       << " for obj " << obj
+	       << ", read_error"
+	       << dendl;
+      continue;
+    }
+    map::iterator k = i->second.attrs.find(OI_ATTR);
+    if (k == i->second.attrs.end()) {
+      // no object info on object, probably corrupt
+      dout(10) << __func__ << ": rejecting osd " << j->first
+	       << " for obj " << obj
+	       << ", no oi attr"
+	       << dendl;
+      continue;
+    }
+
+    bufferlist bl;
+    bl.push_back(k->second);
+    object_info_t oi;
+    try {
+      bufferlist::iterator bliter = bl.begin();
+      ::decode(oi, bliter);
+    } catch (...) {
+      dout(10) << __func__ << ": rejecting osd " << j->first
+	       << " for obj " << obj
+	       << ", corrupt oi attr"
+	       << dendl;
+      // invalid object info, probably corrupt
+      continue;
+    }
+    uint64_t correct_size = be_get_ondisk_size(oi.size);
+    if (correct_size != i->second.size) {
+      // invalid size, probably corrupt
+      dout(10) << __func__ << ": rejecting osd " << j->first
+	       << " for obj " << obj
+	       << ", size mismatch"
+	       << dendl;
+      // invalid object info, probably corrupt
+      continue;
+    }
+    dout(10) << __func__ << ": selecting osd " << j->first
+	     << " for obj " << obj
+	     << dendl;
+    auth = j;
+  }
+  return auth;
+}
+
+void PGBackend::be_compare_scrubmaps(
+  const map &maps,
+  map > &missing,
+  map > &inconsistent,
+  map &authoritative,
+  map > &invalid_snapcolls,
+  int &shallow_errors, int &deep_errors,
+  const spg_t pgid,
+  const vector &acting,
+  ostream &errorstream)
+{
+  map::const_iterator i;
+  map::const_iterator j;
+  set master_set;
+
+  // Construct master set
+  for (j = maps.begin(); j != maps.end(); ++j) {
+    for (i = j->second->objects.begin(); i != j->second->objects.end(); ++i) {
+      master_set.insert(i->first);
+    }
+  }
+
+  // Check maps against master set and each other
+  for (set::const_iterator k = master_set.begin();
+       k != master_set.end();
+       ++k) {
+    map::const_iterator auth =
+      be_select_auth_object(*k, maps);
+    assert(auth != maps.end());
+    set cur_missing;
+    set cur_inconsistent;
+    for (j = maps.begin(); j != maps.end(); ++j) {
+      if (j == auth)
+	continue;
+      if (j->second->objects.count(*k)) {
+	// Compare
+	stringstream ss;
+	enum scrub_error_type error = be_compare_scrub_objects(auth->second->objects[*k],
+	    j->second->objects[*k],
+	    ss);
+        if (error != CLEAN) {
+	  cur_inconsistent.insert(j->first);
+          if (error == SHALLOW_ERROR)
+	    ++shallow_errors;
+          else
+	    ++deep_errors;
+	  errorstream << pgid << " shard " << j->first
+		      << ": soid " << *k << " " << ss.str() << std::endl;
+	}
+      } else {
+	cur_missing.insert(j->first);
+	++shallow_errors;
+	errorstream << pgid << " shard " << j->first
+		    << " missing " << *k << std::endl;
+      }
+    }
+    assert(auth != maps.end());
+    if (!cur_missing.empty()) {
+      missing[*k] = cur_missing;
+    }
+    if (!cur_inconsistent.empty()) {
+      inconsistent[*k] = cur_inconsistent;
+    }
+    if (!cur_inconsistent.empty() || !cur_missing.empty()) {
+      authoritative[*k] = auth->first;
+    }
+  }
+}
diff --git a/ceph/src/osd/PGBackend.h b/ceph/src/osd/PGBackend.h
new file mode 100644
index 00000000..40707529
--- /dev/null
+++ b/ceph/src/osd/PGBackend.h
@@ -0,0 +1,646 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013,2014 Inktank Storage, Inc.
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef PGBACKEND_H
+#define PGBACKEND_H
+
+#include "OSDMap.h"
+#include "PGLog.h"
+#include "osd_types.h"
+#include "common/WorkQueue.h"
+#include "osd_types.h"
+#include "include/Context.h"
+#include "os/ObjectStore.h"
+#include "common/LogClient.h"
+#include 
+
+ /**
+  * PGBackend
+  *
+  * PGBackend defines an interface for logic handling IO and
+  * replication on RADOS objects.  The PGBackend implementation
+  * is responsible for:
+  *
+  * 1) Handling client operations
+  * 2) Handling object recovery
+  * 3) Handling object access
+  * 4) Handling scrub, deep-scrub, repair
+  */
+ class PGBackend {
+ protected:
+   ObjectStore *store;
+   const coll_t coll;
+   const coll_t temp_coll;
+ public:	
+   /**
+    * Provides interfaces for PGBackend callbacks
+    *
+    * The intention is that the parent calls into the PGBackend
+    * implementation holding a lock and that the callbacks are
+    * called under the same locks.
+    */
+   class Listener {
+   public:
+     /// Recovery
+
+     virtual void on_local_recover_start(
+       const hobject_t &oid,
+       ObjectStore::Transaction *t) = 0;
+     /**
+      * Called with the transaction recovering oid
+      */
+     virtual void on_local_recover(
+       const hobject_t &oid,
+       const object_stat_sum_t &stat_diff,
+       const ObjectRecoveryInfo &recovery_info,
+       ObjectContextRef obc,
+       ObjectStore::Transaction *t
+       ) = 0;
+
+     /**
+      * Called when transaction recovering oid is durable and
+      * applied on all replicas
+      */
+     virtual void on_global_recover(const hobject_t &oid) = 0;
+
+     /**
+      * Called when peer is recovered
+      */
+     virtual void on_peer_recover(
+       pg_shard_t peer,
+       const hobject_t &oid,
+       const ObjectRecoveryInfo &recovery_info,
+       const object_stat_sum_t &stat
+       ) = 0;
+
+     virtual void begin_peer_recover(
+       pg_shard_t peer,
+       const hobject_t oid) = 0;
+
+     virtual void failed_push(pg_shard_t from, const hobject_t &soid) = 0;
+     
+     virtual void cancel_pull(const hobject_t &soid) = 0;
+
+     /**
+      * Bless a context
+      *
+      * Wraps a context in whatever outer layers the parent usually
+      * uses to call into the PGBackend
+      */
+     virtual Context *bless_context(Context *c) = 0;
+     virtual GenContext *bless_gencontext(
+       GenContext *c) = 0;
+
+     virtual void send_message(int to_osd, Message *m) = 0;
+     virtual void queue_transaction(
+       ObjectStore::Transaction *t,
+       OpRequestRef op = OpRequestRef()
+       ) = 0;
+     virtual epoch_t get_epoch() const = 0;
+
+     virtual const set &get_actingbackfill_shards() const = 0;
+     virtual const set &get_acting_shards() const = 0;
+     virtual const set &get_backfill_shards() const = 0;
+
+     virtual std::string gen_dbg_prefix() const = 0;
+
+     virtual const map > &get_missing_loc_shards()
+       const = 0;
+
+     virtual const pg_missing_t &get_local_missing() const = 0;
+     virtual const map &get_shard_missing()
+       const = 0;
+     virtual boost::optional maybe_get_shard_missing(
+       pg_shard_t peer) const {
+       if (peer == primary_shard()) {
+	 return get_local_missing();
+       } else {
+	 map::const_iterator i =
+	   get_shard_missing().find(peer);
+	 if (i == get_shard_missing().end()) {
+	   return boost::optional();
+	 } else {
+	   return i->second;
+	 }
+       }
+     }
+     virtual const pg_missing_t &get_shard_missing(pg_shard_t peer) const {
+       boost::optional m = maybe_get_shard_missing(peer);
+       assert(m);
+       return *m;
+     }
+
+     virtual const map &get_shard_info() const = 0;
+     virtual const pg_info_t &get_shard_info(pg_shard_t peer) const {
+       if (peer == primary_shard()) {
+	 return get_info();
+       } else {
+	 map::const_iterator i =
+	   get_shard_info().find(peer);
+	 assert(i != get_shard_info().end());
+	 return i->second;
+       }
+     }
+
+     virtual const PGLog &get_log() const = 0;
+     virtual bool pgb_is_primary() const = 0;
+     virtual OSDMapRef pgb_get_osdmap() const = 0;
+     virtual const pg_info_t &get_info() const = 0;
+     virtual const pg_pool_t &get_pool() const = 0;
+
+     virtual ObjectContextRef get_obc(
+       const hobject_t &hoid,
+       map &attrs) = 0;
+
+     virtual void op_applied(
+       const eversion_t &applied_version) = 0;
+
+     virtual bool should_send_op(
+       pg_shard_t peer,
+       const hobject_t &hoid) = 0;
+
+     virtual void log_operation(
+       vector &logv,
+       boost::optional &hset_history,
+       const eversion_t &trim_to,
+       const eversion_t &trim_rollback_to,
+       bool transaction_applied,
+       ObjectStore::Transaction *t) = 0;
+
+     virtual void update_peer_last_complete_ondisk(
+       pg_shard_t fromosd,
+       eversion_t lcod) = 0;
+
+     virtual void update_last_complete_ondisk(
+       eversion_t lcod) = 0;
+
+     virtual void update_stats(
+       const pg_stat_t &stat) = 0;
+
+     virtual void schedule_work(
+       GenContext *c) = 0;
+
+     virtual pg_shard_t whoami_shard() const = 0;
+     int whoami() const {
+       return whoami_shard().osd;
+     }
+     spg_t whoami_spg_t() const {
+       return get_info().pgid;
+     }
+
+     virtual spg_t primary_spg_t() const = 0;
+     virtual pg_shard_t primary_shard() const = 0;
+
+     virtual void send_message_osd_cluster(
+       int peer, Message *m, epoch_t from_epoch) = 0;
+     virtual void send_message_osd_cluster(
+       Message *m, Connection *con) = 0;
+     virtual void send_message_osd_cluster(
+       Message *m, const ConnectionRef& con) = 0;
+     virtual ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch) = 0;
+     virtual entity_name_t get_cluster_msgr_name() = 0;
+
+     virtual PerfCounters *get_logger() = 0;
+
+     virtual ceph_tid_t get_tid() = 0;
+
+     virtual LogClientTemp clog_error() = 0;
+
+     virtual ~Listener() {}
+   };
+   Listener *parent;
+   Listener *get_parent() const { return parent; }
+   PGBackend(Listener *l, ObjectStore *store, coll_t coll, coll_t temp_coll) :
+     store(store),
+     coll(coll),
+     temp_coll(temp_coll),
+     parent(l), temp_created(false) {}
+   bool is_primary() const { return get_parent()->pgb_is_primary(); }
+   OSDMapRef get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
+   const pg_info_t &get_info() { return get_parent()->get_info(); }
+
+   std::string gen_prefix() const {
+     return parent->gen_dbg_prefix();
+   }
+
+   /**
+    * RecoveryHandle
+    *
+    * We may want to recover multiple objects in the same set of
+    * messages.  RecoveryHandle is an interface for the opaque
+    * object used by the implementation to store the details of
+    * the pending recovery operations.
+    */
+   struct RecoveryHandle {
+     virtual ~RecoveryHandle() {}
+   };
+
+   /// Get a fresh recovery operation
+   virtual RecoveryHandle *open_recovery_op() = 0;
+
+   /// run_recovery_op: finish the operation represented by h
+   virtual void run_recovery_op(
+     RecoveryHandle *h,     ///< [in] op to finish
+     int priority           ///< [in] msg priority
+     ) = 0;
+
+   /**
+    * recover_object
+    *
+    * Triggers a recovery operation on the specified hobject_t
+    * onreadable must be called before onwriteable
+    *
+    * On each replica (primary included), get_parent()->on_not_missing()
+    * must be called when the transaction finalizing the recovery
+    * is queued.  Similarly, get_parent()->on_readable() must be called
+    * when the transaction is applied in the backing store.
+    *
+    * get_parent()->on_not_degraded() should be called on the primary
+    * when writes can resume on the object.
+    *
+    * obc may be NULL if the primary lacks the object.
+    *
+    * head may be NULL only if the head/snapdir is missing
+    *
+    * @param missing [in] set of info, missing pairs for queried nodes
+    * @param overlaps [in] mapping of object to file offset overlaps
+    */
+   virtual void recover_object(
+     const hobject_t &hoid, ///< [in] object to recover
+     eversion_t v,          ///< [in] version to recover
+     ObjectContextRef head,  ///< [in] context of the head/snapdir object
+     ObjectContextRef obc,  ///< [in] context of the object
+     RecoveryHandle *h      ///< [in,out] handle to attach recovery op to
+     ) = 0;
+
+   /**
+    * true if PGBackend can handle this message while inactive
+    *
+    * If it returns true, handle_message *must* also return true
+    */
+   virtual bool can_handle_while_inactive(OpRequestRef op) = 0;
+
+   /// gives PGBackend a crack at an incoming message
+   virtual bool handle_message(
+     OpRequestRef op ///< [in] message received
+     ) = 0; ///< @return true if the message was handled
+
+   virtual void check_recovery_sources(const OSDMapRef osdmap) = 0;
+
+
+   /**
+    * clean up any temporary on-disk state due to a pg interval change
+    */
+   void on_change_cleanup(ObjectStore::Transaction *t);
+   /**
+    * implementation should clear itself, contexts blessed prior to on_change
+    * won't be called after on_change()
+    */
+   virtual void on_change() = 0;
+   virtual void clear_state() = 0;
+
+   virtual void on_flushed() = 0;
+
+   class IsRecoverablePredicate {
+   public:
+     /**
+      * have encodes the shards available
+      */
+     virtual bool operator()(const set &have) const = 0;
+     virtual ~IsRecoverablePredicate() {}
+   };
+   virtual IsRecoverablePredicate *get_is_recoverable_predicate() = 0;
+
+   class IsReadablePredicate {
+   public:
+     /**
+      * have encodes the shards available
+      */
+     virtual bool operator()(const set &have) const = 0;
+     virtual ~IsReadablePredicate() {}
+   };
+   virtual IsReadablePredicate *get_is_readable_predicate() = 0;
+
+   void temp_colls(list *out) {
+     if (temp_created)
+       out->push_back(temp_coll);
+   }
+   void split_colls(
+     spg_t child,
+     int split_bits,
+     int seed,
+     ObjectStore::Transaction *t) {
+     coll_t target = coll_t::make_temp_coll(child);
+     if (!temp_created)
+       return;
+     t->create_collection(target);
+     t->split_collection(
+       temp_coll,
+       split_bits,
+       seed,
+       target);
+   }
+
+   virtual void dump_recovery_info(Formatter *f) const = 0;
+
+ private:
+   bool temp_created;
+   set temp_contents;
+ public:
+   coll_t get_temp_coll(ObjectStore::Transaction *t);
+   coll_t get_temp_coll() const {
+    return temp_coll;
+   }
+   bool have_temp_coll() const { return temp_created; }
+
+   // Track contents of temp collection, clear on reset
+   void add_temp_obj(const hobject_t &oid) {
+     temp_contents.insert(oid);
+   }
+   void add_temp_objs(const set &oids) {
+     temp_contents.insert(oids.begin(), oids.end());
+   }
+   void clear_temp_obj(const hobject_t &oid) {
+     temp_contents.erase(oid);
+   }
+   void clear_temp_objs(const set &oids) {
+     for (set::const_iterator i = oids.begin();
+	  i != oids.end();
+	  ++i) {
+       temp_contents.erase(*i);
+     }
+   }
+
+   virtual ~PGBackend() {}
+
+   /**
+    * Client IO Interface
+    */
+   class PGTransaction {
+   public:
+     /// Write
+     virtual void touch(
+       const hobject_t &hoid  ///< [in] obj to touch
+       ) = 0;
+     virtual void stash(
+       const hobject_t &hoid,   ///< [in] obj to remove
+       version_t former_version ///< [in] former object version
+       ) = 0;
+     virtual void remove(
+       const hobject_t &hoid ///< [in] obj to remove
+       ) = 0;
+     virtual void setattrs(
+       const hobject_t &hoid,         ///< [in] object to write
+       map &attrs ///< [in] attrs, may be cleared
+       ) = 0;
+     virtual void setattr(
+       const hobject_t &hoid,         ///< [in] object to write
+       const string &attrname,        ///< [in] attr to write
+       bufferlist &bl                 ///< [in] val to write, may be claimed
+       ) = 0;
+     virtual void rmattr(
+       const hobject_t &hoid,         ///< [in] object to write
+       const string &attrname         ///< [in] attr to remove
+       ) = 0;
+     virtual void clone(
+       const hobject_t &from,
+       const hobject_t &to
+       ) = 0;
+     virtual void rename(
+       const hobject_t &from,
+       const hobject_t &to
+       ) = 0;
+     virtual void set_alloc_hint(
+       const hobject_t &hoid,
+       uint64_t expected_object_size,
+       uint64_t expected_write_size
+       ) = 0;
+
+     /// Optional, not supported on ec-pool
+     virtual void write(
+       const hobject_t &hoid, ///< [in] object to write
+       uint64_t off,          ///< [in] off at which to write
+       uint64_t len,          ///< [in] len to write from bl
+       bufferlist &bl         ///< [in] bl to write will be claimed to len
+       ) { assert(0); }
+     virtual void omap_setkeys(
+       const hobject_t &hoid,         ///< [in] object to write
+       map &keys  ///< [in] omap keys, may be cleared
+       ) { assert(0); }
+     virtual void omap_rmkeys(
+       const hobject_t &hoid,         ///< [in] object to write
+       set &keys              ///< [in] omap keys, may be cleared
+       ) { assert(0); }
+     virtual void omap_clear(
+       const hobject_t &hoid          ///< [in] object to clear omap
+       ) { assert(0); }
+     virtual void omap_setheader(
+       const hobject_t &hoid,         ///< [in] object to write
+       bufferlist &header             ///< [in] header
+       ) { assert(0); }
+     virtual void clone_range(
+       const hobject_t &from,         ///< [in] from
+       const hobject_t &to,           ///< [in] to
+       uint64_t fromoff,              ///< [in] offset
+       uint64_t len,                  ///< [in] len
+       uint64_t tooff                 ///< [in] offset
+       ) { assert(0); }
+     virtual void truncate(
+       const hobject_t &hoid,
+       uint64_t off
+       ) { assert(0); }
+     virtual void zero(
+       const hobject_t &hoid,
+       uint64_t off,
+       uint64_t len
+       ) { assert(0); }
+
+     /// Supported on all backends
+
+     /// off must be the current object size
+     virtual void append(
+       const hobject_t &hoid, ///< [in] object to write
+       uint64_t off,          ///< [in] off at which to write
+       uint64_t len,          ///< [in] len to write from bl
+       bufferlist &bl         ///< [in] bl to write will be claimed to len
+       ) { write(hoid, off, len, bl); }
+
+     /// to_append *must* have come from the same PGBackend (same concrete type)
+     virtual void append(
+       PGTransaction *to_append ///< [in] trans to append, to_append is cleared
+       ) = 0;
+     virtual void nop() = 0;
+     virtual bool empty() const = 0;
+     virtual uint64_t get_bytes_written() const = 0;
+     virtual ~PGTransaction() {}
+   };
+   /// Get implementation specific empty transaction
+   virtual PGTransaction *get_transaction() = 0;
+
+   /// execute implementation specific transaction
+   virtual void submit_transaction(
+     const hobject_t &hoid,               ///< [in] object
+     const eversion_t &at_version,        ///< [in] version
+     PGTransaction *t,                    ///< [in] trans to execute
+     const eversion_t &trim_to,           ///< [in] trim log to here
+     const eversion_t &trim_rollback_to,  ///< [in] trim rollback info to here
+     vector &log_entries, ///< [in] log entries for t
+     /// [in] hitset history (if updated with this transaction)
+     boost::optional &hset_history,
+     Context *on_local_applied_sync,      ///< [in] called when applied locally
+     Context *on_all_applied,             ///< [in] called when all acked
+     Context *on_all_commit,              ///< [in] called when all commit
+     ceph_tid_t tid,                      ///< [in] tid
+     osd_reqid_t reqid,                   ///< [in] reqid
+     OpRequestRef op                      ///< [in] op
+     ) = 0;
+
+
+   void rollback(
+     const hobject_t &hoid,
+     const ObjectModDesc &desc,
+     ObjectStore::Transaction *t);
+
+   /// Reapply old attributes
+   void rollback_setattrs(
+     const hobject_t &hoid,
+     map > &old_attrs,
+     ObjectStore::Transaction *t);
+
+   /// Truncate object to rollback append
+   virtual void rollback_append(
+     const hobject_t &hoid,
+     uint64_t old_size,
+     ObjectStore::Transaction *t);
+
+   /// Unstash object to rollback stash
+   void rollback_stash(
+     const hobject_t &hoid,
+     version_t old_version,
+     ObjectStore::Transaction *t);
+
+   /// Delete object to rollback create
+   void rollback_create(
+     const hobject_t &hoid,
+     ObjectStore::Transaction *t);
+
+   /// Trim object stashed at stashed_version
+   void trim_stashed_object(
+     const hobject_t &hoid,
+     version_t stashed_version,
+     ObjectStore::Transaction *t);
+
+   /// List objects in collection
+   int objects_list_partial(
+     const hobject_t &begin,
+     int min,
+     int max,
+     snapid_t seq,
+     vector *ls,
+     hobject_t *next);
+
+   int objects_list_range(
+     const hobject_t &start,
+     const hobject_t &end,
+     snapid_t seq,
+     vector *ls,
+     vector *gen_obs=0);
+
+   int objects_get_attr(
+     const hobject_t &hoid,
+     const string &attr,
+     bufferlist *out);
+
+   virtual int objects_get_attrs(
+     const hobject_t &hoid,
+     map *out);
+
+   virtual int objects_read_sync(
+     const hobject_t &hoid,
+     uint64_t off,
+     uint64_t len,
+     bufferlist *bl) = 0;
+
+   virtual void objects_read_async(
+     const hobject_t &hoid,
+     const list,
+		pair > > &to_read,
+     Context *on_complete) = 0;
+
+   virtual bool scrub_supported() { return false; }
+   void be_scan_list(
+     ScrubMap &map, const vector &ls, bool deep,
+     ThreadPool::TPHandle &handle);
+   enum scrub_error_type be_compare_scrub_objects(
+     const ScrubMap::object &auth,
+     const ScrubMap::object &candidate,
+     ostream &errorstream);
+   map::const_iterator be_select_auth_object(
+     const hobject_t &obj,
+     const map &maps);
+   void be_compare_scrubmaps(
+     const map &maps,
+     map > &missing,
+     map > &inconsistent,
+     map &authoritative,
+     map > &invalid_snapcolls,
+     int &shallow_errors, int &deep_errors,
+     const spg_t pgid,
+     const vector &acting,
+     ostream &errorstream);
+   virtual uint64_t be_get_ondisk_size(
+     uint64_t logical_size) { assert(0); return 0; }
+   virtual void be_deep_scrub(
+     const hobject_t &poid,
+     ScrubMap::object &o,
+     ThreadPool::TPHandle &handle) { assert(0); }
+
+   static PGBackend *build_pg_backend(
+     const pg_pool_t &pool,
+     const OSDMapRef curmap,
+     Listener *l,
+     coll_t coll,
+     coll_t temp_coll,
+     ObjectStore *store,
+     CephContext *cct);
+ };
+
+struct PG_SendMessageOnConn: public Context {
+  PGBackend::Listener *pg;
+  Message *reply;
+  ConnectionRef conn;
+  PG_SendMessageOnConn(
+    PGBackend::Listener *pg,
+    Message *reply,
+    ConnectionRef conn) : pg(pg), reply(reply), conn(conn) {}
+  void finish(int) {
+    pg->send_message_osd_cluster(reply, conn.get());
+  }
+};
+
+struct PG_QueueAsync : public Context {
+  PGBackend::Listener *pg;
+  GenContext *c;
+  PG_QueueAsync(
+    PGBackend::Listener *pg,
+    GenContext *c) : pg(pg), c(c) {}
+  void finish(int) {
+    pg->schedule_work(c);
+  }
+};
+
+#endif
diff --git a/ceph/src/osd/PGLog.cc b/ceph/src/osd/PGLog.cc
new file mode 100644
index 00000000..3f61346d
--- /dev/null
+++ b/ceph/src/osd/PGLog.cc
@@ -0,0 +1,1054 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "PGLog.h"
+#include "PG.h"
+#include "SnapMapper.h"
+#include "../include/unordered_map.h"
+
+#define dout_subsys ceph_subsys_osd
+
+//////////////////// PGLog::IndexedLog ////////////////////
+
+void PGLog::IndexedLog::advance_rollback_info_trimmed_to(
+  eversion_t to,
+  LogEntryHandler *h)
+{
+  assert(to <= can_rollback_to);
+
+  if (to > rollback_info_trimmed_to)
+    rollback_info_trimmed_to = to;
+
+  while (rollback_info_trimmed_to_riter != log.rbegin()) {
+    --rollback_info_trimmed_to_riter;
+    if (rollback_info_trimmed_to_riter->version > rollback_info_trimmed_to) {
+      ++rollback_info_trimmed_to_riter;
+      break;
+    }
+    h->trim(*rollback_info_trimmed_to_riter);
+  }
+}
+
+void PGLog::IndexedLog::split_into(
+  pg_t child_pgid,
+  unsigned split_bits,
+  PGLog::IndexedLog *olog)
+{
+  list oldlog;
+  oldlog.swap(log);
+
+  eversion_t old_tail;
+  olog->head = head;
+  olog->tail = tail;
+  unsigned mask = ~((~0)<::iterator i = oldlog.begin();
+       i != oldlog.end();
+       ) {
+    if ((i->soid.hash & mask) == child_pgid.m_seed) {
+      olog->log.push_back(*i);
+    } else {
+      log.push_back(*i);
+    }
+    oldlog.erase(i++);
+  }
+
+
+  olog->can_rollback_to = can_rollback_to;
+
+  olog->index();
+  index();
+}
+
+void PGLog::IndexedLog::trim(
+  LogEntryHandler *handler,
+  eversion_t s,
+  set *trimmed)
+{
+  if (complete_to != log.end() &&
+      complete_to->version <= s) {
+    generic_dout(0) << " bad trim to " << s << " when complete_to is "
+		    << complete_to->version
+		    << " on " << *this << dendl;
+  }
+
+  if (s > can_rollback_to)
+    can_rollback_to = s;
+  advance_rollback_info_trimmed_to(s, handler);
+
+  while (!log.empty()) {
+    pg_log_entry_t &e = *log.begin();
+    if (e.version > s)
+      break;
+    generic_dout(20) << "trim " << e << dendl;
+    if (trimmed)
+      trimmed->insert(e.version);
+
+    unindex(e);         // remove from index,
+
+    if (rollback_info_trimmed_to_riter == log.rend() ||
+	e.version == rollback_info_trimmed_to_riter->version) {
+      log.pop_front();
+      rollback_info_trimmed_to_riter = log.rend();
+    } else {
+      log.pop_front();
+    }
+  }
+
+  // raise tail?
+  if (tail < s)
+    tail = s;
+}
+
+ostream& PGLog::IndexedLog::print(ostream& out) const 
+{
+  out << *this << std::endl;
+  for (list::const_iterator p = log.begin();
+       p != log.end();
+       ++p) {
+    out << *p << " " << (logged_object(p->soid) ? "indexed":"NOT INDEXED") << std::endl;
+    assert(!p->reqid_is_indexed() || logged_req(p->reqid));
+  }
+  return out;
+}
+
+//////////////////// PGLog ////////////////////
+
+void PGLog::reset_backfill()
+{
+  missing.clear();
+  divergent_priors.clear();
+  dirty_divergent_priors = true;
+}
+
+void PGLog::clear() {
+  divergent_priors.clear();
+  missing.clear();
+  log.clear();
+  log_keys_debug.clear();
+  undirty();
+}
+
+void PGLog::clear_info_log(
+  spg_t pgid,
+  const hobject_t &infos_oid,
+  const hobject_t &log_oid,
+  ObjectStore::Transaction *t) {
+
+  set keys_to_remove;
+  keys_to_remove.insert(PG::get_epoch_key(pgid));
+  keys_to_remove.insert(PG::get_biginfo_key(pgid));
+  keys_to_remove.insert(PG::get_info_key(pgid));
+
+  t->remove(coll_t::META_COLL, log_oid);
+  t->omap_rmkeys(coll_t::META_COLL, infos_oid, keys_to_remove);
+}
+
+void PGLog::trim(
+  LogEntryHandler *handler,
+  eversion_t trim_to,
+  pg_info_t &info)
+{
+  // trim?
+  if (trim_to > log.tail) {
+    /* If we are trimming, we must be complete up to trim_to, time
+     * to throw out any divergent_priors
+     */
+    divergent_priors.clear();
+    // We shouldn't be trimming the log past last_complete
+    assert(trim_to <= info.last_complete);
+
+    dout(10) << "trim " << log << " to " << trim_to << dendl;
+    log.trim(handler, trim_to, &trimmed);
+    info.log_tail = log.tail;
+  }
+}
+
+void PGLog::proc_replica_log(
+  ObjectStore::Transaction& t,
+  pg_info_t &oinfo, const pg_log_t &olog, pg_missing_t& omissing,
+  pg_shard_t from) const
+{
+  dout(10) << "proc_replica_log for osd." << from << ": "
+	   << oinfo << " " << olog << " " << omissing << dendl;
+
+  /*
+    basically what we're doing here is rewinding the remote log,
+    dropping divergent entries, until we find something that matches
+    our master log.  we then reset last_update to reflect the new
+    point up to which missing is accurate.
+
+    later, in activate(), missing will get wound forward again and
+    we will send the peer enough log to arrive at the same state.
+  */
+
+  for (map::iterator i = omissing.missing.begin();
+       i != omissing.missing.end();
+       ++i) {
+    dout(20) << " before missing " << i->first << " need " << i->second.need
+	     << " have " << i->second.have << dendl;
+  }
+
+  list::const_iterator fromiter = log.log.end();
+  eversion_t lower_bound = log.tail;
+  while (1) {
+    if (fromiter == log.log.begin())
+      break;
+    --fromiter;
+    if (fromiter->version <= olog.head) {
+      dout(20) << "merge_log cut point (usually last shared) is "
+	       << *fromiter << dendl;
+      lower_bound = fromiter->version;
+      ++fromiter;
+      break;
+    }
+  }
+
+  list divergent;
+  list::const_iterator pp = olog.log.end();
+  eversion_t lu(oinfo.last_update);
+  while (true) {
+    if (pp == olog.log.begin()) {
+      if (pp != olog.log.end())   // no last_update adjustment if we discard nothing!
+	lu = olog.tail;
+      break;
+    }
+    --pp;
+    const pg_log_entry_t& oe = *pp;
+
+    // don't continue past the tail of our log.
+    if (oe.version <= log.tail) {
+      lu = oe.version;
+      ++pp;
+      break;
+    }
+
+    if (oe.version <= lower_bound) {
+      lu = oe.version;
+      ++pp;
+      break;
+    }
+
+    divergent.push_front(oe);
+  }    
+
+
+  IndexedLog folog;
+  folog.log.insert(folog.log.begin(), olog.log.begin(), pp);
+  folog.index();
+  _merge_divergent_entries(
+    folog,
+    divergent,
+    oinfo,
+    olog.can_rollback_to,
+    omissing,
+    0,
+    0);
+
+  if (lu < oinfo.last_update) {
+    dout(10) << " peer osd." << from << " last_update now " << lu << dendl;
+    oinfo.last_update = lu;
+  }
+
+  if (omissing.have_missing()) {
+    eversion_t first_missing =
+      omissing.missing[omissing.rmissing.begin()->second].need;
+    oinfo.last_complete = eversion_t();
+    list::const_iterator i = olog.log.begin();
+    for (;
+	 i != olog.log.end();
+	 ++i) {
+      if (i->version < first_missing)
+	oinfo.last_complete = i->version;
+      else
+	break;
+    }
+  } else {
+    oinfo.last_complete = oinfo.last_update;
+  }
+}
+
+/**
+ * _merge_object_divergent_entries
+ *
+ * There are 5 distinct cases:
+ * 1) There is a more recent update: in this case we assume we adjusted the
+ *    store and missing during merge_log
+ * 2) The first entry in the divergent sequence is a create.  This might
+ *    either be because the object is a clone or because prior_version is
+ *    eversion_t().  In this case the object does not exist and we must
+ *    adjust missing and the store to match.
+ * 3) We are currently missing the object.  In this case, we adjust the
+ *    missing to our prior_version taking care to add a divergent_prior
+ *    if necessary
+ * 4) We can rollback all of the entries.  In this case, we do so using
+ *    the rollbacker and return -- the object does not go into missing.
+ * 5) We cannot rollback at least 1 of the entries.  In this case, we
+ *    clear the object out of the store and add a missing entry at
+ *    prior_version taking care to add a divergent_prior if
+ *    necessary.
+ */
+void PGLog::_merge_object_divergent_entries(
+  const IndexedLog &log,
+  const hobject_t &hoid,
+  const list &entries,
+  const pg_info_t &info,
+  eversion_t olog_can_rollback_to,
+  pg_missing_t &missing,
+  boost::optional > *new_divergent_prior,
+  LogEntryHandler *rollbacker
+  )
+{
+  dout(10) << __func__ << ": merging hoid " << hoid
+	   << " entries: " << entries << dendl;
+
+  if (hoid > info.last_backfill) {
+    dout(10) << __func__ << ": hoid " << hoid << " after last_backfill"
+	     << dendl;
+    return;
+  }
+
+  // entries is non-empty
+  assert(!entries.empty());
+  eversion_t last;
+  for (list::const_iterator i = entries.begin();
+       i != entries.end();
+       ++i) {
+    // all entries are on hoid
+    assert(i->soid == hoid);
+    if (i != entries.begin() && i->prior_version != eversion_t()) {
+      // in increasing order of version
+      assert(i->version > last);
+      // prior_version correct
+      assert(i->prior_version == last);
+    }
+    last = i->version;
+
+    if (rollbacker)
+      rollbacker->trim(*i);
+  }
+
+  const eversion_t prior_version = entries.begin()->prior_version;
+  const eversion_t first_divergent_update = entries.begin()->version;
+  const eversion_t last_divergent_update = entries.rbegin()->version;
+  const bool object_not_in_store =
+    !missing.is_missing(hoid) &&
+    entries.rbegin()->is_delete();
+  dout(10) << __func__ << ": hoid " << hoid
+	   << " prior_version: " << prior_version
+	   << " first_divergent_update: " << first_divergent_update
+	   << " last_divergent_update: " << last_divergent_update
+	   << dendl;
+
+  ceph::unordered_map::const_iterator objiter =
+    log.objects.find(hoid);
+  if (objiter != log.objects.end() &&
+      objiter->second->version >= first_divergent_update) {
+    /// Case 1)
+    assert(objiter->second->version > last_divergent_update);
+
+    dout(10) << __func__ << ": more recent entry found: "
+	     << *objiter->second << ", already merged" << dendl;
+
+    // ensure missing has been updated appropriately
+    if (objiter->second->is_update()) {
+      assert(missing.is_missing(hoid) &&
+	     missing.missing[hoid].need == objiter->second->version);
+    } else {
+      assert(!missing.is_missing(hoid));
+    }
+    missing.revise_have(hoid, eversion_t());
+    if (rollbacker && !object_not_in_store)
+      rollbacker->remove(hoid);
+    return;
+  }
+
+  dout(10) << __func__ << ": hoid " << hoid
+	   <<" has no more recent entries in log" << dendl;
+  if (prior_version == eversion_t() || entries.front().is_clone()) {
+    /// Case 2)
+    dout(10) << __func__ << ": hoid " << hoid
+	     << " prior_version or op type indicates creation, deleting"
+	     << dendl;
+    if (missing.is_missing(hoid))
+      missing.rm(missing.missing.find(hoid));
+    if (rollbacker && !object_not_in_store)
+      rollbacker->remove(hoid);
+    return;
+  }
+
+  if (missing.is_missing(hoid)) {
+    /// Case 3)
+    dout(10) << __func__ << ": hoid " << hoid
+	     << " missing, " << missing.missing[hoid]
+	     << " adjusting" << dendl;
+
+    if (missing.missing[hoid].have == prior_version) {
+      dout(10) << __func__ << ": hoid " << hoid
+	       << " missing.have is prior_version " << prior_version
+	       << " removing from missing" << dendl;
+      missing.rm(missing.missing.find(hoid));
+    } else {
+      dout(10) << __func__ << ": hoid " << hoid
+	       << " missing.have is " << missing.missing[hoid].have
+	       << ", adjusting" << dendl;
+      missing.revise_need(hoid, prior_version);
+      if (prior_version <= info.log_tail) {
+	dout(10) << __func__ << ": hoid " << hoid
+		 << " prior_version " << prior_version << " <= info.log_tail "
+		 << info.log_tail << dendl;
+	if (new_divergent_prior)
+	  *new_divergent_prior = make_pair(prior_version, hoid);
+      }
+    }
+    return;
+  }
+
+  dout(10) << __func__ << ": hoid " << hoid
+	   << " must be rolled back or recovered, attempting to rollback"
+	   << dendl;
+  bool can_rollback = true;
+  /// Distinguish between 4) and 5)
+  for (list::const_reverse_iterator i = entries.rbegin();
+       i != entries.rend();
+       ++i) {
+    if (!i->mod_desc.can_rollback() || i->version <= olog_can_rollback_to) {
+      dout(10) << __func__ << ": hoid " << hoid << " cannot rollback "
+	       << *i << dendl;
+      can_rollback = false;
+      break;
+    }
+  }
+
+  if (can_rollback) {
+    /// Case 4)
+    for (list::const_reverse_iterator i = entries.rbegin();
+	 i != entries.rend();
+	 ++i) {
+      assert(i->mod_desc.can_rollback() && i->version > olog_can_rollback_to);
+      dout(10) << __func__ << ": hoid " << hoid
+	       << " rolling back " << *i << dendl;
+      if (rollbacker)
+	rollbacker->rollback(*i);
+    }
+    dout(10) << __func__ << ": hoid " << hoid << " rolled back" << dendl;
+    return;
+  } else {
+    /// Case 5)
+    dout(10) << __func__ << ": hoid " << hoid << " cannot roll back, "
+	     << "removing and adding to missing" << dendl;
+    if (rollbacker && !object_not_in_store)
+      rollbacker->remove(hoid);
+    missing.add(hoid, prior_version, eversion_t());
+    if (prior_version <= info.log_tail) {
+      dout(10) << __func__ << ": hoid " << hoid
+	       << " prior_version " << prior_version << " <= info.log_tail "
+	       << info.log_tail << dendl;
+      if (new_divergent_prior)
+	*new_divergent_prior = make_pair(prior_version, hoid);
+    }
+  }
+}
+
+/**
+ * rewind divergent entries at the head of the log
+ *
+ * This rewinds entries off the head of our log that are divergent.
+ * This is used by replicas during activation.
+ *
+ * @param t transaction
+ * @param newhead new head to rewind to
+ */
+void PGLog::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead,
+				 pg_info_t &info, LogEntryHandler *rollbacker,
+				 bool &dirty_info, bool &dirty_big_info)
+{
+  dout(10) << "rewind_divergent_log truncate divergent future " << newhead << dendl;
+  assert(newhead >= log.tail);
+
+  list::iterator p = log.log.end();
+  list divergent;
+  while (true) {
+    if (p == log.log.begin()) {
+      // yikes, the whole thing is divergent!
+      divergent.swap(log.log);
+      break;
+    }
+    --p;
+    mark_dirty_from(p->version);
+    if (p->version <= newhead) {
+      ++p;
+      divergent.splice(divergent.begin(), log.log, p, log.log.end());
+      break;
+    }
+    assert(p->version > newhead);
+    dout(10) << "rewind_divergent_log future divergent " << *p << dendl;
+  }
+
+  log.head = newhead;
+  info.last_update = newhead;
+  if (info.last_complete > newhead)
+    info.last_complete = newhead;
+
+  log.index();
+
+  map new_priors;
+  _merge_divergent_entries(
+    log,
+    divergent,
+    info,
+    log.can_rollback_to,
+    missing,
+    &new_priors,
+    rollbacker);
+  for (map::iterator i = new_priors.begin();
+       i != new_priors.end();
+       ++i) {
+    add_divergent_prior(
+      i->first,
+      i->second);
+  }
+
+  if (info.last_update < log.can_rollback_to)
+    log.can_rollback_to = info.last_update;
+
+  dirty_info = true;
+  dirty_big_info = true;
+}
+
+void PGLog::merge_log(ObjectStore::Transaction& t,
+                      pg_info_t &oinfo, pg_log_t &olog, pg_shard_t fromosd,
+                      pg_info_t &info, LogEntryHandler *rollbacker,
+                      bool &dirty_info, bool &dirty_big_info)
+{
+  dout(10) << "merge_log " << olog << " from osd." << fromosd
+           << " into " << log << dendl;
+
+  // Check preconditions
+
+  // If our log is empty, the incoming log needs to have not been trimmed.
+  assert(!log.null() || olog.tail == eversion_t());
+  // The logs must overlap.
+  assert(log.head >= olog.tail && olog.head >= log.tail);
+
+  for (map::iterator i = missing.missing.begin();
+       i != missing.missing.end();
+       ++i) {
+    dout(20) << "pg_missing_t sobject: " << i->first << dendl;
+  }
+
+  bool changed = false;
+
+  // extend on tail?
+  //  this is just filling in history.  it does not affect our
+  //  missing set, as that should already be consistent with our
+  //  current log.
+  if (olog.tail < log.tail) {
+    mark_dirty_to(log.log.begin()->version); // last clean entry
+    dout(10) << "merge_log extending tail to " << olog.tail << dendl;
+    list::iterator from = olog.log.begin();
+    list::iterator to;
+    for (to = from;
+	 to != olog.log.end();
+	 ++to) {
+      if (to->version > log.tail)
+	break;
+      log.index(*to);
+      dout(15) << *to << dendl;
+    }
+      
+    // splice into our log.
+    log.log.splice(log.log.begin(),
+		   olog.log, from, to);
+      
+    info.log_tail = log.tail = olog.tail;
+    changed = true;
+  }
+
+  if (oinfo.stats.reported_seq < info.stats.reported_seq ||   // make sure reported always increases
+      oinfo.stats.reported_epoch < info.stats.reported_epoch) {
+    oinfo.stats.reported_seq = info.stats.reported_seq;
+    oinfo.stats.reported_epoch = info.stats.reported_epoch;
+  }
+  if (info.last_backfill.is_max())
+    info.stats = oinfo.stats;
+  info.hit_set = oinfo.hit_set;
+
+  // do we have divergent entries to throw out?
+  if (olog.head < log.head) {
+    rewind_divergent_log(t, olog.head, info, rollbacker, dirty_info, dirty_big_info);
+    changed = true;
+  }
+
+  // extend on head?
+  if (olog.head > log.head) {
+    dout(10) << "merge_log extending head to " << olog.head << dendl;
+      
+    // find start point in olog
+    list::iterator to = olog.log.end();
+    list::iterator from = olog.log.end();
+    eversion_t lower_bound = olog.tail;
+    while (1) {
+      if (from == olog.log.begin())
+	break;
+      --from;
+      dout(20) << "  ? " << *from << dendl;
+      if (from->version <= log.head) {
+	dout(20) << "merge_log cut point (usually last shared) is " << *from << dendl;
+	lower_bound = from->version;
+	++from;
+	break;
+      }
+    }
+    mark_dirty_from(lower_bound);
+
+    // index, update missing, delete deleted
+    for (list::iterator p = from; p != to; ++p) {
+      pg_log_entry_t &ne = *p;
+      dout(20) << "merge_log " << ne << dendl;
+      log.index(ne);
+      if (ne.soid <= info.last_backfill) {
+	missing.add_next_event(ne);
+	if (ne.is_delete())
+	  rollbacker->remove(ne.soid);
+      }
+    }
+      
+    // move aside divergent items
+    list divergent;
+    while (!log.empty()) {
+      pg_log_entry_t &oe = *log.log.rbegin();
+      /*
+       * look at eversion.version here.  we want to avoid a situation like:
+       *  our log: 100'10 (0'0) m 10000004d3a.00000000/head by client4225.1:18529
+       *  new log: 122'10 (0'0) m 10000004d3a.00000000/head by client4225.1:18529
+       *  lower_bound = 100'9
+       * i.e, same request, different version.  If the eversion.version is > the
+       * lower_bound, we it is divergent.
+       */
+      if (oe.version.version <= lower_bound.version)
+	break;
+      dout(10) << "merge_log divergent " << oe << dendl;
+      divergent.push_front(oe);
+      log.log.pop_back();
+    }
+
+    // splice
+    log.log.splice(log.log.end(), 
+		   olog.log, from, to);
+    log.index();   
+
+    info.last_update = log.head = olog.head;
+
+    info.last_user_version = oinfo.last_user_version;
+    info.purged_snaps = oinfo.purged_snaps;
+
+    map new_priors;
+    _merge_divergent_entries(
+      log,
+      divergent,
+      info,
+      log.can_rollback_to,
+      missing,
+      &new_priors,
+      rollbacker);
+    for (map::iterator i = new_priors.begin();
+	 i != new_priors.end();
+	 ++i) {
+      add_divergent_prior(
+	i->first,
+	i->second);
+    }
+
+    // We cannot rollback into the new log entries
+    log.can_rollback_to = log.head;
+
+    changed = true;
+  }
+  
+  dout(10) << "merge_log result " << log << " " << missing << " changed=" << changed << dendl;
+
+  if (changed) {
+    dirty_info = true;
+    dirty_big_info = true;
+  }
+}
+
+void PGLog::write_log(
+  ObjectStore::Transaction& t, const hobject_t &log_oid)
+{
+  if (is_dirty()) {
+    dout(10) << "write_log with: "
+	     << "dirty_to: " << dirty_to
+	     << ", dirty_from: " << dirty_from
+	     << ", dirty_divergent_priors: " << dirty_divergent_priors
+	     << ", writeout_from: " << writeout_from
+	     << ", trimmed: " << trimmed
+	     << dendl;
+    _write_log(
+      t, log, log_oid, divergent_priors,
+      dirty_to,
+      dirty_from,
+      writeout_from,
+      trimmed,
+      dirty_divergent_priors,
+      !touched_log,
+      (pg_log_debug ? &log_keys_debug : 0));
+    undirty();
+  } else {
+    dout(10) << "log is not dirty" << dendl;
+  }
+}
+
+void PGLog::write_log(ObjectStore::Transaction& t, pg_log_t &log,
+    const hobject_t &log_oid, map &divergent_priors)
+{
+  _write_log(
+    t, log, log_oid,
+    divergent_priors, eversion_t::max(), eversion_t(), eversion_t(),
+    set(),
+    true, true, 0);
+}
+
+void PGLog::_write_log(
+  ObjectStore::Transaction& t, pg_log_t &log,
+  const hobject_t &log_oid, map &divergent_priors,
+  eversion_t dirty_to,
+  eversion_t dirty_from,
+  eversion_t writeout_from,
+  const set &trimmed,
+  bool dirty_divergent_priors,
+  bool touch_log,
+  set *log_keys_debug
+  )
+{
+  set to_remove;
+  for (set::const_iterator i = trimmed.begin();
+       i != trimmed.end();
+       ++i) {
+    to_remove.insert(i->get_key_name());
+    if (log_keys_debug) {
+      assert(log_keys_debug->count(i->get_key_name()));
+      log_keys_debug->erase(i->get_key_name());
+    }
+  }
+
+//dout(10) << "write_log, clearing up to " << dirty_to << dendl;
+  if (touch_log)
+    t.touch(coll_t(), log_oid);
+  if (dirty_to != eversion_t()) {
+    t.omap_rmkeyrange(
+      coll_t(), log_oid,
+      eversion_t().get_key_name(), dirty_to.get_key_name());
+    clear_up_to(log_keys_debug, dirty_to.get_key_name());
+  }
+  if (dirty_to != eversion_t::max() && dirty_from != eversion_t::max()) {
+    //   dout(10) << "write_log, clearing from " << dirty_from << dendl;
+    t.omap_rmkeyrange(
+      coll_t(), log_oid,
+      dirty_from.get_key_name(), eversion_t::max().get_key_name());
+    clear_after(log_keys_debug, dirty_from.get_key_name());
+  }
+
+  map keys;
+  for (list::iterator p = log.log.begin();
+       p != log.log.end() && p->version < dirty_to;
+       ++p) {
+    bufferlist bl(sizeof(*p) * 2);
+    p->encode_with_checksum(bl);
+    keys[p->get_key_name()].claim(bl);
+  }
+
+  for (list::reverse_iterator p = log.log.rbegin();
+       p != log.log.rend() &&
+	 (p->version >= dirty_from || p->version >= writeout_from) &&
+	 p->version >= dirty_to;
+       ++p) {
+    bufferlist bl(sizeof(*p) * 2);
+    p->encode_with_checksum(bl);
+    keys[p->get_key_name()].claim(bl);
+  }
+
+  if (log_keys_debug) {
+    for (map::iterator i = keys.begin();
+	 i != keys.end();
+	 ++i) {
+      assert(!log_keys_debug->count(i->first));
+      log_keys_debug->insert(i->first);
+    }
+  }
+
+  if (dirty_divergent_priors) {
+    //dout(10) << "write_log: writing divergent_priors" << dendl;
+    ::encode(divergent_priors, keys["divergent_priors"]);
+  }
+  ::encode(log.can_rollback_to, keys["can_rollback_to"]);
+
+  t.omap_rmkeys(coll_t::META_COLL, log_oid, to_remove);
+  t.omap_setkeys(coll_t::META_COLL, log_oid, keys);
+}
+
+bool PGLog::read_log(ObjectStore *store, coll_t coll, hobject_t log_oid,
+  const pg_info_t &info, map &divergent_priors,
+  IndexedLog &log,
+  pg_missing_t &missing,
+  ostringstream &oss,
+  set *log_keys_debug)
+{
+  dout(10) << "read_log" << dendl;
+  bool rewrite_log = false;
+
+  // legacy?
+  struct stat st;
+  int r = store->stat(coll_t::META_COLL, log_oid, &st);
+  assert(r == 0);
+  if (st.st_size > 0) {
+    read_log_old(store, coll, log_oid, info, divergent_priors, log, missing, oss, log_keys_debug);
+    rewrite_log = true;
+  } else {
+    log.tail = info.log_tail;
+    // will get overridden below if it had been recorded
+    log.can_rollback_to = info.last_update;
+    ObjectMap::ObjectMapIterator p = store->get_omap_iterator(coll_t::META_COLL, log_oid);
+    if (p) for (p->seek_to_first(); p->valid() ; p->next()) {
+      bufferlist bl = p->value();//Copy bufferlist before creating iterator
+      bufferlist::iterator bp = bl.begin();
+      if (p->key() == "divergent_priors") {
+	::decode(divergent_priors, bp);
+	dout(20) << "read_log " << divergent_priors.size() << " divergent_priors" << dendl;
+      } else if (p->key() == "can_rollback_to") {
+	bufferlist bl = p->value();
+	bufferlist::iterator bp = bl.begin();
+	::decode(log.can_rollback_to, bp);
+      } else {
+	pg_log_entry_t e;
+	e.decode_with_checksum(bp);
+	dout(20) << "read_log " << e << dendl;
+	if (!log.log.empty()) {
+	  pg_log_entry_t last_e(log.log.back());
+	  assert(last_e.version.version < e.version.version);
+	  assert(last_e.version.epoch <= e.version.epoch);
+	}
+	log.log.push_back(e);
+	log.head = e.version;
+	if (log_keys_debug)
+	  log_keys_debug->insert(e.get_key_name());
+      }
+    }
+  }
+  log.head = info.last_update;
+  log.index();
+
+  // build missing
+  if (info.last_complete < info.last_update) {
+    dout(10) << "read_log checking for missing items over interval (" << info.last_complete
+	     << "," << info.last_update << "]" << dendl;
+
+    set did;
+    for (list::reverse_iterator i = log.log.rbegin();
+	 i != log.log.rend();
+	 ++i) {
+      if (i->version <= info.last_complete) break;
+      if (i->soid > info.last_backfill) continue;
+      if (did.count(i->soid)) continue;
+      did.insert(i->soid);
+      
+      if (i->is_delete()) continue;
+      
+      bufferlist bv;
+      int r = store->getattr(
+	coll,
+	ghobject_t(i->soid, ghobject_t::NO_GEN, info.pgid.shard),
+	OI_ATTR,
+	bv);
+      if (r >= 0) {
+	object_info_t oi(bv);
+	if (oi.version < i->version) {
+	  dout(15) << "read_log  missing " << *i << " (have " << oi.version << ")" << dendl;
+	  missing.add(i->soid, i->version, oi.version);
+	}
+      } else {
+	dout(15) << "read_log  missing " << *i << dendl;
+	missing.add(i->soid, i->version, eversion_t());
+      }
+    }
+    for (map::reverse_iterator i =
+	   divergent_priors.rbegin();
+	 i != divergent_priors.rend();
+	 ++i) {
+      if (i->first <= info.last_complete) break;
+      if (i->second > info.last_backfill) continue;
+      if (did.count(i->second)) continue;
+      did.insert(i->second);
+      bufferlist bv;
+      int r = store->getattr(
+	coll, 
+	ghobject_t(i->second, ghobject_t::NO_GEN, info.pgid.shard),
+	OI_ATTR,
+	bv);
+      if (r >= 0) {
+	object_info_t oi(bv);
+	/**
+	 * 1) we see this entry in the divergent priors mapping
+	 * 2) we didn't see an entry for this object in the log
+	 *
+	 * From 1 & 2 we know that either the object does not exist
+	 * or it is at the version specified in the divergent_priors
+	 * map since the object would have been deleted atomically
+	 * with the addition of the divergent_priors entry, an older
+	 * version would not have been recovered, and a newer version
+	 * would show up in the log above.
+	 */
+	assert(oi.version == i->first);
+      } else {
+	dout(15) << "read_log  missing " << *i << dendl;
+	missing.add(i->second, i->first, eversion_t());
+      }
+    }
+  }
+  dout(10) << "read_log done" << dendl;
+  return rewrite_log;
+}
+
+void PGLog::read_log_old(ObjectStore *store, coll_t coll, hobject_t log_oid,
+			 const pg_info_t &info, map &divergent_priors,
+			 IndexedLog &log,
+			 pg_missing_t &missing, ostringstream &oss,
+			 set *log_keys_debug)
+{
+  // load bounds, based on old OndiskLog encoding.
+  uint64_t ondisklog_tail = 0;
+  uint64_t ondisklog_head = 0;
+  uint64_t ondisklog_zero_to;
+  bool ondisklog_has_checksums;
+
+  bufferlist blb;
+  store->collection_getattr(coll, "ondisklog", blb);
+  {
+    bufferlist::iterator bl = blb.begin();
+    DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+    ondisklog_has_checksums = (struct_v >= 2);
+    ::decode(ondisklog_tail, bl);
+    ::decode(ondisklog_head, bl);
+    if (struct_v >= 4)
+      ::decode(ondisklog_zero_to, bl);
+    else
+      ondisklog_zero_to = 0;
+    if (struct_v >= 5)
+      ::decode(divergent_priors, bl);
+    DECODE_FINISH(bl);
+  }
+  uint64_t ondisklog_length = ondisklog_head - ondisklog_tail;
+  dout(10) << "read_log " << ondisklog_tail << "~" << ondisklog_length << dendl;
+ 
+  log.tail = info.log_tail;
+
+  if (ondisklog_head > 0) {
+    // read
+    bufferlist bl;
+    store->read(coll_t::META_COLL, log_oid, ondisklog_tail, ondisklog_length, bl);
+    if (bl.length() < ondisklog_length) {
+      std::ostringstream oss;
+      oss << "read_log got " << bl.length() << " bytes, expected "
+	  << ondisklog_head << "-" << ondisklog_tail << "="
+	  << ondisklog_length;
+      throw read_log_error(oss.str().c_str());
+    }
+    
+    pg_log_entry_t e;
+    bufferlist::iterator p = bl.begin();
+    assert(log.empty());
+    eversion_t last;
+    bool reorder = false;
+
+    while (!p.end()) {
+      uint64_t pos = ondisklog_tail + p.get_off();
+      if (ondisklog_has_checksums) {
+	bufferlist ebl;
+	::decode(ebl, p);
+	__u32 crc;
+	::decode(crc, p);
+	
+	__u32 got = ebl.crc32c(0);
+	if (crc == got) {
+	  bufferlist::iterator q = ebl.begin();
+	  ::decode(e, q);
+	} else {
+	  std::ostringstream oss;
+	  oss << "read_log " << pos << " bad crc got " << got << " expected" << crc;
+	  throw read_log_error(oss.str().c_str());
+	}
+      } else {
+	::decode(e, p);
+      }
+      dout(20) << "read_log " << pos << " " << e << dendl;
+
+      // [repair] in order?
+      if (e.version < last) {
+	dout(0) << "read_log " << pos << " out of order entry " << e << " follows " << last << dendl;
+	oss << info.pgid << " log has out of order entry "
+	      << e << " following " << last << "\n";
+	reorder = true;
+      }
+
+      if (e.version <= log.tail) {
+	dout(20) << "read_log  ignoring entry at " << pos << " below log.tail" << dendl;
+	continue;
+      }
+      if (last.version == e.version.version) {
+	dout(0) << "read_log  got dup " << e.version << " (last was " << last << ", dropping that one)" << dendl;
+	log.log.pop_back();
+	oss << info.pgid << " read_log got dup "
+	      << e.version << " after " << last << "\n";
+      }
+
+      assert(!e.invalid_hash);
+
+      if (e.invalid_pool) {
+	e.soid.pool = info.pgid.pool();
+      }
+
+      e.offset = pos;
+      uint64_t endpos = ondisklog_tail + p.get_off();
+      log.log.push_back(e);
+      if (log_keys_debug)
+	log_keys_debug->insert(e.get_key_name());
+      last = e.version;
+
+      // [repair] at end of log?
+      if (!p.end() && e.version == info.last_update) {
+	oss << info.pgid << " log has extra data at "
+	   << endpos << "~" << (ondisklog_head-endpos) << " after "
+	   << info.last_update << "\n";
+
+	dout(0) << "read_log " << endpos << " *** extra gunk at end of log, "
+	        << "adjusting ondisklog_head" << dendl;
+	ondisklog_head = endpos;
+	break;
+      }
+    }
+  
+    if (reorder) {
+      dout(0) << "read_log reordering log" << dendl;
+      map m;
+      for (list::iterator p = log.log.begin(); p != log.log.end(); ++p)
+	m[p->version] = *p;
+      log.log.clear();
+      for (map::iterator p = m.begin(); p != m.end(); ++p)
+	log.log.push_back(p->second);
+    }
+  }
+}
diff --git a/ceph/src/osd/PGLog.h b/ceph/src/osd/PGLog.h
new file mode 100644
index 00000000..1744cc8f
--- /dev/null
+++ b/ceph/src/osd/PGLog.h
@@ -0,0 +1,587 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#ifndef CEPH_PG_LOG_H
+#define CEPH_PG_LOG_H
+
+// re-include our assert to clobber boost's
+#include "include/assert.h" 
+#include "osd_types.h"
+#include "os/ObjectStore.h"
+#include "common/ceph_context.h"
+#include 
+using namespace std;
+
+struct PGLog {
+  ////////////////////////////// sub classes //////////////////////////////
+  struct LogEntryHandler {
+    virtual void rollback(
+      const pg_log_entry_t &entry) = 0;
+    virtual void remove(
+      const hobject_t &hoid) = 0;
+    virtual void trim(
+      const pg_log_entry_t &entry) = 0;
+    virtual ~LogEntryHandler() {}
+  };
+
+  /* Exceptions */
+  class read_log_error : public buffer::error {
+  public:
+    explicit read_log_error(const char *what) {
+      snprintf(buf, sizeof(buf), "read_log_error: %s", what);
+    }
+    const char *what() const throw () {
+      return buf;
+    }
+  private:
+    char buf[512];
+  };
+
+  /**
+   * IndexLog - adds in-memory index of the log, by oid.
+   * plus some methods to manipulate it all.
+   */
+  struct IndexedLog : public pg_log_t {
+    ceph::unordered_map objects;  // ptrs into log.  be careful!
+    ceph::unordered_map caller_ops;
+
+    // recovery pointers
+    list::iterator complete_to;  // not inclusive of referenced item
+    version_t last_requested;           // last object requested by primary
+
+    //
+  private:
+    /**
+     * rollback_info_trimmed_to_riter points to the first log entry <=
+     * rollback_info_trimmed_to
+     *
+     * It's a reverse_iterator because rend() is a natural representation for
+     * tail, and rbegin() works nicely for head.
+     */
+    list::reverse_iterator rollback_info_trimmed_to_riter;
+  public:
+    void advance_rollback_info_trimmed_to(eversion_t to, LogEntryHandler *h);
+
+    /****/
+    IndexedLog() :
+      complete_to(log.end()),
+      last_requested(0),
+      rollback_info_trimmed_to_riter(log.rbegin())
+      {}
+
+    void claim_log_and_clear_rollback_info(const pg_log_t& o) {
+      // we must have already trimmed the old entries
+      assert(rollback_info_trimmed_to == head);
+      assert(rollback_info_trimmed_to_riter == log.rbegin());
+
+      log = o.log;
+      head = o.head;
+      rollback_info_trimmed_to = head;
+      tail = o.tail;
+      index();
+    }
+
+    void split_into(
+      pg_t child_pgid,
+      unsigned split_bits,
+      IndexedLog *olog);
+
+    void zero() {
+      // we must have already trimmed the old entries
+      assert(rollback_info_trimmed_to == head);
+      assert(rollback_info_trimmed_to_riter == log.rbegin());
+
+      unindex();
+      pg_log_t::clear();
+      rollback_info_trimmed_to_riter = log.rbegin();
+      reset_recovery_pointers();
+    }
+    void clear() {
+      rollback_info_trimmed_to = head;
+      rollback_info_trimmed_to_riter = log.rbegin();
+      zero();
+    }
+    void reset_recovery_pointers() {
+      complete_to = log.end();
+      last_requested = 0;
+    }
+
+    bool logged_object(const hobject_t& oid) const {
+      return objects.count(oid);
+    }
+    bool logged_req(const osd_reqid_t &r) const {
+      return caller_ops.count(r);
+    }
+    const pg_log_entry_t *get_request(const osd_reqid_t &r) const {
+      ceph::unordered_map::const_iterator p = caller_ops.find(r);
+      if (p == caller_ops.end())
+	return NULL;
+      return p->second;
+    }
+
+    void index() {
+      objects.clear();
+      caller_ops.clear();
+      for (list::iterator i = log.begin();
+           i != log.end();
+           ++i) {
+        objects[i->soid] = &(*i);
+	if (i->reqid_is_indexed()) {
+	  //assert(caller_ops.count(i->reqid) == 0);  // divergent merge_log indexes new before unindexing old
+	  caller_ops[i->reqid] = &(*i);
+	}
+      }
+
+      rollback_info_trimmed_to_riter = log.rbegin();
+      while (rollback_info_trimmed_to_riter != log.rend() &&
+	     rollback_info_trimmed_to_riter->version > rollback_info_trimmed_to)
+	rollback_info_trimmed_to_riter++;
+    }
+
+    void index(pg_log_entry_t& e) {
+      if (objects.count(e.soid) == 0 || 
+          objects[e.soid]->version < e.version)
+        objects[e.soid] = &e;
+      if (e.reqid_is_indexed()) {
+	//assert(caller_ops.count(i->reqid) == 0);  // divergent merge_log indexes new before unindexing old
+	caller_ops[e.reqid] = &e;
+      }
+    }
+    void unindex() {
+      objects.clear();
+      caller_ops.clear();
+    }
+    void unindex(pg_log_entry_t& e) {
+      // NOTE: this only works if we remove from the _tail_ of the log!
+      if (objects.count(e.soid) && objects[e.soid]->version == e.version)
+        objects.erase(e.soid);
+      if (e.reqid_is_indexed() &&
+	  caller_ops.count(e.reqid) &&  // divergent merge_log indexes new before unindexing old
+	  caller_ops[e.reqid] == &e)
+	caller_ops.erase(e.reqid);
+    }
+
+    // actors
+    void add(pg_log_entry_t& e) {
+      // add to log
+      log.push_back(e);
+
+      // riter previously pointed to the previous entry
+      if (rollback_info_trimmed_to_riter == log.rbegin())
+	++rollback_info_trimmed_to_riter;
+
+      assert(e.version > head);
+      assert(head.version == 0 || e.version.version > head.version);
+      head = e.version;
+
+      // to our index
+      objects[e.soid] = &(log.back());
+      if (e.reqid_is_indexed())
+	caller_ops[e.reqid] = &(log.back());
+    }
+
+    void trim(
+      LogEntryHandler *handler,
+      eversion_t s,
+      set *trimmed);
+
+    ostream& print(ostream& out) const;
+  };
+
+
+protected:
+  //////////////////// data members ////////////////////
+  bool pg_log_debug;
+
+  map divergent_priors;
+  pg_missing_t     missing;
+  IndexedLog  log;
+
+  /// Log is clean on [dirty_to, dirty_from)
+  bool touched_log;
+  eversion_t dirty_to;         ///< must clear/writeout all keys up to dirty_to
+  eversion_t dirty_from;       ///< must clear/writeout all keys past dirty_from
+  eversion_t writeout_from;    ///< must writout keys past writeout_from
+  set trimmed;     ///< must clear keys in trimmed
+  bool dirty_divergent_priors;
+  CephContext *cct;
+
+  bool is_dirty() const {
+    return !touched_log ||
+      (dirty_to != eversion_t()) ||
+      (dirty_from != eversion_t::max()) ||
+      dirty_divergent_priors ||
+      (writeout_from != eversion_t::max()) ||
+      !(trimmed.empty());
+  }
+  void mark_dirty_to(eversion_t to) {
+    if (to > dirty_to)
+      dirty_to = to;
+  }
+  void mark_dirty_from(eversion_t from) {
+    if (from < dirty_from)
+      dirty_from = from;
+  }
+  void mark_writeout_from(eversion_t from) {
+    if (from < writeout_from)
+      writeout_from = from;
+  }
+  void add_divergent_prior(eversion_t version, hobject_t obj) {
+    divergent_priors.insert(make_pair(version, obj));
+    dirty_divergent_priors = true;
+  }
+public:
+  void mark_log_for_rewrite() {
+    mark_dirty_to(eversion_t::max());
+    mark_dirty_from(eversion_t());
+    touched_log = false;
+  }
+protected:
+
+  /// DEBUG
+  set log_keys_debug;
+  static void clear_after(set *log_keys_debug, const string &lb) {
+    if (!log_keys_debug)
+      return;
+    for (set::iterator i = log_keys_debug->lower_bound(lb);
+	 i != log_keys_debug->end();
+	 log_keys_debug->erase(i++));
+  }
+  static void clear_up_to(set *log_keys_debug, const string &ub) {
+    if (!log_keys_debug)
+      return;
+    for (set::iterator i = log_keys_debug->begin();
+	 i != log_keys_debug->end() && *i < ub;
+	 log_keys_debug->erase(i++));
+  }
+  void check() {
+    if (!pg_log_debug)
+      return;
+    assert(log.log.size() == log_keys_debug.size());
+    for (list::iterator i = log.log.begin();
+	 i != log.log.end();
+	 ++i) {
+      assert(log_keys_debug.count(i->get_key_name()));
+    }
+  }
+
+  void undirty() {
+    dirty_to = eversion_t();
+    dirty_from = eversion_t::max();
+    dirty_divergent_priors = false;
+    touched_log = true;
+    trimmed.clear();
+    writeout_from = eversion_t::max();
+    check();
+  }
+public:
+  PGLog(CephContext *cct = 0) :
+    pg_log_debug(!(cct && !(cct->_conf->osd_debug_pg_log_writeout))),
+    touched_log(false), dirty_from(eversion_t::max()),
+    writeout_from(eversion_t::max()),
+    dirty_divergent_priors(false), cct(cct) {}
+
+
+  void reset_backfill();
+
+  void clear();
+
+  //////////////////// get or set missing ////////////////////
+
+  const pg_missing_t& get_missing() const { return missing; }
+
+  void missing_got(map::const_iterator m) {
+    map::iterator p = missing.missing.find(m->first);
+    missing.got(p);
+  }
+
+  void revise_have(hobject_t oid, eversion_t have) {
+    missing.revise_have(oid, have);
+  }
+
+  void revise_need(hobject_t oid, eversion_t need) {
+    missing.revise_need(oid, need);
+  }
+
+  void missing_add(const hobject_t& oid, eversion_t need, eversion_t have) {
+    missing.add(oid, need, have);
+  }
+
+  void missing_rm(map::const_iterator m) {
+    map::iterator p = missing.missing.find(m->first);
+    missing.rm(p);
+  }
+
+  void missing_add_event(const pg_log_entry_t &e) {
+    missing.add_next_event(e);
+  }
+
+  //////////////////// get or set log ////////////////////
+
+  const IndexedLog &get_log() const { return log; }
+
+  const eversion_t &get_tail() const { return log.tail; }
+
+  void set_tail(eversion_t tail) { log.tail = tail; }
+
+  const eversion_t &get_head() const { return log.head; }
+
+  void set_head(eversion_t head) { log.head = head; }
+
+  void set_last_requested(version_t last_requested) {
+    log.last_requested = last_requested;
+  }
+
+  void index() { log.index(); }
+
+  void unindex() { log.unindex(); }
+
+  void add(pg_log_entry_t& e) {
+    mark_writeout_from(e.version);
+    log.add(e);
+  }
+
+  void reset_recovery_pointers() { log.reset_recovery_pointers(); }
+
+  static void clear_info_log(
+    spg_t pgid,
+    const hobject_t &infos_oid,
+    const hobject_t &log_oid,
+    ObjectStore::Transaction *t);
+
+  void trim(
+    LogEntryHandler *handler,
+    eversion_t trim_to,
+    pg_info_t &info);
+
+  void trim_rollback_info(
+    eversion_t trim_rollback_to,
+    LogEntryHandler *h) {
+    if (trim_rollback_to > log.can_rollback_to)
+      log.can_rollback_to = trim_rollback_to;
+    log.advance_rollback_info_trimmed_to(
+      trim_rollback_to,
+      h);
+  }
+
+  eversion_t get_rollback_trimmed_to() const {
+    return log.rollback_info_trimmed_to;
+  }
+
+  void clear_can_rollback_to(LogEntryHandler *h) {
+    log.can_rollback_to = log.head;
+    log.advance_rollback_info_trimmed_to(
+      log.head,
+      h);
+  }
+
+  //////////////////// get or set log & missing ////////////////////
+
+  void claim_log_and_clear_rollback_info(const pg_log_t &o, LogEntryHandler *h) {
+    log.can_rollback_to = log.head;
+    log.advance_rollback_info_trimmed_to(log.head, h);
+    log.claim_log_and_clear_rollback_info(o);
+    missing.clear();
+    mark_dirty_to(eversion_t::max());
+  }
+
+  void split_into(
+      pg_t child_pgid,
+      unsigned split_bits,
+      PGLog *opg_log) { 
+    log.split_into(child_pgid, split_bits, &(opg_log->log));
+    missing.split_into(child_pgid, split_bits, &(opg_log->missing));
+    opg_log->mark_dirty_to(eversion_t::max());
+    mark_dirty_to(eversion_t::max());
+  }
+
+  void recover_got(hobject_t oid, eversion_t v, pg_info_t &info) {
+    if (missing.is_missing(oid, v)) {
+      missing.got(oid, v);
+      
+      // raise last_complete?
+      if (missing.missing.empty()) {
+	log.complete_to = log.log.end();
+	info.last_complete = info.last_update;
+      }
+      while (log.complete_to != log.log.end()) {
+	if (missing.missing[missing.rmissing.begin()->second].need <=
+	    log.complete_to->version)
+	  break;
+	if (info.last_complete < log.complete_to->version)
+	  info.last_complete = log.complete_to->version;
+	++log.complete_to;
+      }
+    }
+
+    if (log.can_rollback_to < v)
+      log.can_rollback_to = v;
+  }
+
+  void activate_not_complete(pg_info_t &info) {
+    log.complete_to = log.log.begin();
+    while (log.complete_to->version <
+	   missing.missing[missing.rmissing.begin()->second].need)
+      ++log.complete_to;
+    assert(log.complete_to != log.log.end());
+    if (log.complete_to == log.log.begin()) {
+      info.last_complete = eversion_t();
+    } else {
+      --log.complete_to;
+      info.last_complete = log.complete_to->version;
+      ++log.complete_to;
+    }
+    log.last_requested = 0;
+  }
+
+  void proc_replica_log(ObjectStore::Transaction& t, pg_info_t &oinfo, const pg_log_t &olog,
+			pg_missing_t& omissing, pg_shard_t from) const;
+
+protected:
+  static void split_by_object(
+    list &entries,
+    map > *out_entries) {
+    while (!entries.empty()) {
+      list &out_list = (*out_entries)[entries.front().soid];
+      out_list.splice(out_list.end(), entries, entries.begin());
+    }
+  }
+
+  /**
+   * Merge complete list of divergent entries for an object
+   *
+   * @param new_divergent_prior [out] filled out for a new divergent prior
+   */
+  static void _merge_object_divergent_entries(
+    const IndexedLog &log,               ///< [in] log to merge against
+    const hobject_t &hoid,               ///< [in] object we are merging
+    const list &entries, ///< [in] entries for hoid to merge
+    const pg_info_t &oinfo,              ///< [in] info for merging entries
+    eversion_t olog_can_rollback_to,     ///< [in] rollback boundary
+    pg_missing_t &omissing,              ///< [in,out] missing to adjust, use
+    boost::optional > *new_divergent_prior,
+    LogEntryHandler *rollbacker          ///< [in] optional rollbacker object
+    );
+
+  /// Merge all entries using above
+  static void _merge_divergent_entries(
+    const IndexedLog &log,               ///< [in] log to merge against
+    list &entries,       ///< [in] entries to merge
+    const pg_info_t &oinfo,              ///< [in] info for merging entries
+    eversion_t olog_can_rollback_to,     ///< [in] rollback boundary
+    pg_missing_t &omissing,              ///< [in,out] missing to adjust, use
+    map *priors,  ///< [out] target for new priors
+    LogEntryHandler *rollbacker          ///< [in] optional rollbacker object
+    ) {
+    map > split;
+    split_by_object(entries, &split);
+    for (map >::iterator i = split.begin();
+	 i != split.end();
+	 ++i) {
+      boost::optional > new_divergent_prior;
+      _merge_object_divergent_entries(
+	log,
+	i->first,
+	i->second,
+	oinfo,
+	olog_can_rollback_to,
+	omissing,
+	&new_divergent_prior,
+	rollbacker);
+      if (priors && new_divergent_prior) {
+	(*priors)[new_divergent_prior->first] = new_divergent_prior->second;
+      }
+    }
+  }
+
+  /**
+   * Exists for use in TestPGLog for simply testing single divergent log
+   * cases
+   */
+  void merge_old_entry(
+    ObjectStore::Transaction& t,
+    const pg_log_entry_t& oe,
+    const pg_info_t& info,
+    LogEntryHandler *rollbacker) {
+    boost::optional > new_divergent_prior;
+    list entries;
+    entries.push_back(oe);
+    _merge_object_divergent_entries(
+      log,
+      oe.soid,
+      entries,
+      info,
+      log.can_rollback_to,
+      missing,
+      &new_divergent_prior,
+      rollbacker);
+    if (new_divergent_prior)
+      add_divergent_prior(
+	(*new_divergent_prior).first,
+	(*new_divergent_prior).second);
+  }
+public:
+  void rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead,
+                            pg_info_t &info, LogEntryHandler *rollbacker,
+                            bool &dirty_info, bool &dirty_big_info);
+
+  void merge_log(ObjectStore::Transaction& t, pg_info_t &oinfo, pg_log_t &olog,
+		 pg_shard_t from,
+		 pg_info_t &info, LogEntryHandler *rollbacker,
+		 bool &dirty_info, bool &dirty_big_info);
+
+  void write_log(ObjectStore::Transaction& t, const hobject_t &log_oid);
+
+  static void write_log(ObjectStore::Transaction& t, pg_log_t &log,
+    const hobject_t &log_oid, map &divergent_priors);
+
+  static void _write_log(
+    ObjectStore::Transaction& t, pg_log_t &log,
+    const hobject_t &log_oid, map &divergent_priors,
+    eversion_t dirty_to,
+    eversion_t dirty_from,
+    eversion_t writeout_from,
+    const set &trimmed,
+    bool dirty_divergent_priors,
+    bool touch_log,
+    set *log_keys_debug
+    );
+
+  bool read_log(ObjectStore *store, coll_t coll, hobject_t log_oid,
+		const pg_info_t &info, ostringstream &oss) {
+    return read_log(
+      store, coll, log_oid, info, divergent_priors,
+      log, missing, oss,
+      (pg_log_debug ? &log_keys_debug : 0));
+  }
+
+  /// return true if the log should be rewritten
+  static bool read_log(ObjectStore *store, coll_t coll, hobject_t log_oid,
+    const pg_info_t &info, map &divergent_priors,
+    IndexedLog &log,
+    pg_missing_t &missing, ostringstream &oss,
+    set *log_keys_debug = 0
+    );
+
+protected:
+  static void read_log_old(ObjectStore *store, coll_t coll, hobject_t log_oid,
+			   const pg_info_t &info, map &divergent_priors,
+			   IndexedLog &log,
+			   pg_missing_t &missing, ostringstream &oss,
+			   set *log_keys_debug);
+};
+  
+#endif // CEPH_PG_LOG_H
diff --git a/ceph/src/osd/ReplicatedBackend.cc b/ceph/src/osd/ReplicatedBackend.cc
new file mode 100644
index 00000000..4430b397
--- /dev/null
+++ b/ceph/src/osd/ReplicatedBackend.cc
@@ -0,0 +1,758 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "common/errno.h"
+#include "ReplicatedBackend.h"
+#include "messages/MOSDOp.h"
+#include "messages/MOSDSubOp.h"
+#include "messages/MOSDSubOpReply.h"
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPull.h"
+#include "messages/MOSDPGPushReply.h"
+
+#define dout_subsys ceph_subsys_osd
+#define DOUT_PREFIX_ARGS this
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+static ostream& _prefix(std::ostream *_dout, ReplicatedBackend *pgb) {
+  return *_dout << pgb->get_parent()->gen_dbg_prefix();
+}
+
+ReplicatedBackend::ReplicatedBackend(
+  PGBackend::Listener *pg,
+  coll_t coll,
+  coll_t temp_coll,
+  ObjectStore *store,
+  CephContext *cct) :
+  PGBackend(pg, store,
+	    coll, temp_coll),
+  cct(cct) {}
+
+void ReplicatedBackend::run_recovery_op(
+  PGBackend::RecoveryHandle *_h,
+  int priority)
+{
+  RPGHandle *h = static_cast(_h);
+  send_pushes(priority, h->pushes);
+  send_pulls(priority, h->pulls);
+  delete h;
+}
+
+void ReplicatedBackend::recover_object(
+  const hobject_t &hoid,
+  eversion_t v,
+  ObjectContextRef head,
+  ObjectContextRef obc,
+  RecoveryHandle *_h
+  )
+{
+  dout(10) << __func__ << ": " << hoid << dendl;
+  RPGHandle *h = static_cast(_h);
+  if (get_parent()->get_local_missing().is_missing(hoid)) {
+    assert(!obc);
+    // pull
+    prepare_pull(
+      v,
+      hoid,
+      head,
+      h);
+    return;
+  } else {
+    assert(obc);
+    int started = start_pushes(
+      hoid,
+      obc,
+      h);
+    assert(started > 0);
+  }
+}
+
+void ReplicatedBackend::check_recovery_sources(const OSDMapRef osdmap)
+{
+  for(map >::iterator i = pull_from_peer.begin();
+      i != pull_from_peer.end();
+      ) {
+    if (osdmap->is_down(i->first.osd)) {
+      dout(10) << "check_recovery_sources resetting pulls from osd." << i->first
+	       << ", osdmap has it marked down" << dendl;
+      for (set::iterator j = i->second.begin();
+	   j != i->second.end();
+	   ++j) {
+	assert(pulling.count(*j) == 1);
+	get_parent()->cancel_pull(*j);
+	pulling.erase(*j);
+      }
+      pull_from_peer.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+}
+
+bool ReplicatedBackend::can_handle_while_inactive(OpRequestRef op)
+{
+  dout(10) << __func__ << ": " << op << dendl;
+  switch (op->get_req()->get_type()) {
+  case MSG_OSD_PG_PULL:
+    return true;
+  case MSG_OSD_SUBOP: {
+    MOSDSubOp *m = static_cast(op->get_req());
+    if (m->ops.size() >= 1) {
+      OSDOp *first = &m->ops[0];
+      switch (first->op.op) {
+      case CEPH_OSD_OP_PULL:
+	return true;
+      default:
+	return false;
+      }
+    } else {
+      return false;
+    }
+  }
+  default:
+    return false;
+  }
+}
+
+bool ReplicatedBackend::handle_message(
+  OpRequestRef op
+  )
+{
+  dout(10) << __func__ << ": " << op << dendl;
+  switch (op->get_req()->get_type()) {
+  case MSG_OSD_PG_PUSH:
+    do_push(op);
+    return true;
+
+  case MSG_OSD_PG_PULL:
+    do_pull(op);
+    return true;
+
+  case MSG_OSD_PG_PUSH_REPLY:
+    do_push_reply(op);
+    return true;
+
+  case MSG_OSD_SUBOP: {
+    MOSDSubOp *m = static_cast(op->get_req());
+    if (m->ops.size() >= 1) {
+      OSDOp *first = &m->ops[0];
+      switch (first->op.op) {
+      case CEPH_OSD_OP_PULL:
+	sub_op_pull(op);
+	return true;
+      case CEPH_OSD_OP_PUSH:
+	sub_op_push(op);
+	return true;
+      default:
+	break;
+      }
+    } else {
+      sub_op_modify(op);
+    }
+    break;
+  }
+
+  case MSG_OSD_SUBOPREPLY: {
+    MOSDSubOpReply *r = static_cast(op->get_req());
+    if (r->ops.size() >= 1) {
+      OSDOp &first = r->ops[0];
+      switch (first.op.op) {
+      case CEPH_OSD_OP_PUSH:
+	// continue peer recovery
+	sub_op_push_reply(op);
+	return true;
+      }
+    } else {
+      sub_op_modify_reply(op);
+    }
+    break;
+  }
+
+  default:
+    break;
+  }
+  return false;
+}
+
+void ReplicatedBackend::clear_state()
+{
+  // clear pushing/pulling maps
+  pushing.clear();
+  pulling.clear();
+  pull_from_peer.clear();
+}
+
+void ReplicatedBackend::on_change()
+{
+  dout(10) << __func__ << dendl;
+  for (map::iterator i = in_progress_ops.begin();
+       i != in_progress_ops.end();
+       in_progress_ops.erase(i++)) {
+    if (i->second.on_commit)
+      delete i->second.on_commit;
+    if (i->second.on_applied)
+      delete i->second.on_applied;
+  }
+  clear_state();
+}
+
+void ReplicatedBackend::on_flushed()
+{
+  if (have_temp_coll() &&
+      !store->collection_empty(get_temp_coll())) {
+    vector objects;
+    store->collection_list(get_temp_coll(), objects);
+    derr << __func__ << ": found objects in the temp collection: "
+	 << objects << ", crashing now"
+	 << dendl;
+    assert(0 == "found garbage in the temp collection");
+  }
+}
+
+int ReplicatedBackend::objects_read_sync(
+  const hobject_t &hoid,
+  uint64_t off,
+  uint64_t len,
+  bufferlist *bl)
+{
+  return store->read(coll, hoid, off, len, *bl);
+}
+
+struct AsyncReadCallback : public GenContext {
+  int r;
+  Context *c;
+  AsyncReadCallback(int r, Context *c) : r(r), c(c) {}
+  void finish(ThreadPool::TPHandle&) {
+    c->complete(r);
+    c = NULL;
+  }
+  ~AsyncReadCallback() {
+    delete c;
+  }
+};
+void ReplicatedBackend::objects_read_async(
+  const hobject_t &hoid,
+  const list,
+		  pair > > &to_read,
+  Context *on_complete)
+{
+  int r = 0;
+  for (list,
+		 pair > >::const_iterator i =
+	   to_read.begin();
+       i != to_read.end() && r >= 0;
+       ++i) {
+    int _r = store->read(coll, hoid, i->first.first,
+			 i->first.second, *(i->second.first));
+    if (i->second.second) {
+      get_parent()->schedule_work(
+	get_parent()->bless_gencontext(
+	  new AsyncReadCallback(_r, i->second.second)));
+    }
+    if (_r < 0)
+      r = _r;
+  }
+  get_parent()->schedule_work(
+    get_parent()->bless_gencontext(
+      new AsyncReadCallback(r, on_complete)));
+}
+
+
+class RPGTransaction : public PGBackend::PGTransaction {
+  coll_t coll;
+  coll_t temp_coll;
+  set temp_added;
+  set temp_cleared;
+  ObjectStore::Transaction *t;
+  const coll_t &get_coll_ct(const hobject_t &hoid) {
+    if (hoid.is_temp()) {
+      temp_cleared.erase(hoid);
+      temp_added.insert(hoid);
+    }
+    return get_coll(hoid);
+  }
+  const coll_t &get_coll_rm(const hobject_t &hoid) {
+    if (hoid.is_temp()) {
+      temp_added.erase(hoid);
+      temp_cleared.insert(hoid);
+    }
+    return get_coll(hoid);
+  }
+  const coll_t &get_coll(const hobject_t &hoid) {
+    if (hoid.is_temp())
+      return temp_coll;
+    else
+      return coll;
+  }
+public:
+  RPGTransaction(coll_t coll, coll_t temp_coll)
+    : coll(coll), temp_coll(temp_coll), t(new ObjectStore::Transaction)
+    {}
+
+  /// Yields ownership of contained transaction
+  ObjectStore::Transaction *get_transaction() {
+    ObjectStore::Transaction *_t = t;
+    t = 0;
+    return _t;
+  }
+  const set &get_temp_added() {
+    return temp_added;
+  }
+  const set &get_temp_cleared() {
+    return temp_cleared;
+  }
+
+  void write(
+    const hobject_t &hoid,
+    uint64_t off,
+    uint64_t len,
+    bufferlist &bl
+    ) {
+    t->write(get_coll_ct(hoid), hoid, off, len, bl);
+  }
+  void remove(
+    const hobject_t &hoid
+    ) {
+    t->remove(get_coll_rm(hoid), hoid);
+  }
+  void stash(
+    const hobject_t &hoid,
+    version_t former_version) {
+    t->collection_move_rename(
+      coll, hoid, coll,
+      ghobject_t(hoid, former_version, ghobject_t::NO_SHARD));
+  }
+  void setattrs(
+    const hobject_t &hoid,
+    map &attrs
+    ) {
+    t->setattrs(get_coll(hoid), hoid, attrs);
+  }
+  void setattr(
+    const hobject_t &hoid,
+    const string &attrname,
+    bufferlist &bl
+    ) {
+    t->setattr(get_coll(hoid), hoid, attrname, bl);
+  }
+  void rmattr(
+    const hobject_t &hoid,
+    const string &attrname
+    ) {
+    t->rmattr(get_coll(hoid), hoid, attrname);
+  }
+  void omap_setkeys(
+    const hobject_t &hoid,
+    map &keys
+    ) {
+    return t->omap_setkeys(get_coll(hoid), hoid, keys);
+  }
+  void omap_rmkeys(
+    const hobject_t &hoid,
+    set &keys
+    ) {
+    t->omap_rmkeys(get_coll(hoid), hoid, keys);
+  }
+  void omap_clear(
+    const hobject_t &hoid
+    ) {
+    t->omap_clear(get_coll(hoid), hoid);
+  }
+  void omap_setheader(
+    const hobject_t &hoid,
+    bufferlist &header
+    ) {
+    t->omap_setheader(get_coll(hoid), hoid, header);
+  }
+  void clone_range(
+    const hobject_t &from,
+    const hobject_t &to,
+    uint64_t fromoff,
+    uint64_t len,
+    uint64_t tooff
+    ) {
+    assert(get_coll(from) == get_coll_ct(to)  && get_coll(from) == coll);
+    t->clone_range(coll, from, to, fromoff, len, tooff);
+  }
+  void clone(
+    const hobject_t &from,
+    const hobject_t &to
+    ) {
+    assert(get_coll(from) == get_coll_ct(to)  && get_coll(from) == coll);
+    t->clone(coll, from, to);
+  }
+  void rename(
+    const hobject_t &from,
+    const hobject_t &to
+    ) {
+    t->collection_move_rename(
+      get_coll_rm(from),
+      from,
+      get_coll_ct(to),
+      to);
+  }
+
+  void touch(
+    const hobject_t &hoid
+    ) {
+    t->touch(get_coll_ct(hoid), hoid);
+  }
+
+  void truncate(
+    const hobject_t &hoid,
+    uint64_t off
+    ) {
+    t->truncate(get_coll(hoid), hoid, off);
+  }
+  void zero(
+    const hobject_t &hoid,
+    uint64_t off,
+    uint64_t len
+    ) {
+    t->zero(get_coll(hoid), hoid, off, len);
+  }
+
+  void set_alloc_hint(
+    const hobject_t &hoid,
+    uint64_t expected_object_size,
+    uint64_t expected_write_size
+    ) {
+    t->set_alloc_hint(get_coll(hoid), hoid, expected_object_size,
+                      expected_write_size);
+  }
+
+  void append(
+    PGTransaction *_to_append
+    ) {
+    RPGTransaction *to_append = dynamic_cast(_to_append);
+    assert(to_append);
+    t->append(*(to_append->t));
+    for (set::iterator i = to_append->temp_added.begin();
+	 i != to_append->temp_added.end();
+	 ++i) {
+      temp_cleared.erase(*i);
+      temp_added.insert(*i);
+    }
+    for (set::iterator i = to_append->temp_cleared.begin();
+	 i != to_append->temp_cleared.end();
+	 ++i) {
+      temp_added.erase(*i);
+      temp_cleared.insert(*i);
+    }
+  }
+  void nop() {
+    t->nop();
+  }
+  bool empty() const {
+    return t->empty();
+  }
+  uint64_t get_bytes_written() const {
+    return t->get_encoded_bytes();
+  }
+  ~RPGTransaction() { delete t; }
+};
+
+PGBackend::PGTransaction *ReplicatedBackend::get_transaction()
+{
+  return new RPGTransaction(coll, get_temp_coll());
+}
+
+class C_OSD_OnOpCommit : public Context {
+  ReplicatedBackend *pg;
+  ReplicatedBackend::InProgressOp *op;
+public:
+  C_OSD_OnOpCommit(ReplicatedBackend *pg, ReplicatedBackend::InProgressOp *op) 
+    : pg(pg), op(op) {}
+  void finish(int) {
+    pg->op_commit(op);
+  }
+};
+
+class C_OSD_OnOpApplied : public Context {
+  ReplicatedBackend *pg;
+  ReplicatedBackend::InProgressOp *op;
+public:
+  C_OSD_OnOpApplied(ReplicatedBackend *pg, ReplicatedBackend::InProgressOp *op) 
+    : pg(pg), op(op) {}
+  void finish(int) {
+    pg->op_applied(op);
+  }
+};
+
+void ReplicatedBackend::submit_transaction(
+  const hobject_t &soid,
+  const eversion_t &at_version,
+  PGTransaction *_t,
+  const eversion_t &trim_to,
+  const eversion_t &trim_rollback_to,
+  vector &log_entries,
+  boost::optional &hset_history,
+  Context *on_local_applied_sync,
+  Context *on_all_acked,
+  Context *on_all_commit,
+  ceph_tid_t tid,
+  osd_reqid_t reqid,
+  OpRequestRef orig_op)
+{
+  RPGTransaction *t = dynamic_cast(_t);
+  assert(t);
+  ObjectStore::Transaction *op_t = t->get_transaction();
+
+  assert(t->get_temp_added().size() <= 1);
+  assert(t->get_temp_cleared().size() <= 1);
+
+  assert(!in_progress_ops.count(tid));
+  InProgressOp &op = in_progress_ops.insert(
+    make_pair(
+      tid,
+      InProgressOp(
+	tid, on_all_commit, on_all_acked,
+	orig_op, at_version)
+      )
+    ).first->second;
+
+  op.waiting_for_applied.insert(
+    parent->get_actingbackfill_shards().begin(),
+    parent->get_actingbackfill_shards().end());
+  op.waiting_for_commit.insert(
+    parent->get_actingbackfill_shards().begin(),
+    parent->get_actingbackfill_shards().end());
+
+
+  issue_op(
+    soid,
+    at_version,
+    tid,
+    reqid,
+    trim_to,
+    trim_rollback_to,
+    t->get_temp_added().size() ? *(t->get_temp_added().begin()) : hobject_t(),
+    t->get_temp_cleared().size() ?
+      *(t->get_temp_cleared().begin()) :hobject_t(),
+    log_entries,
+    hset_history,
+    &op,
+    op_t);
+
+  ObjectStore::Transaction local_t;
+  if (t->get_temp_added().size()) {
+    get_temp_coll(&local_t);
+    add_temp_objs(t->get_temp_added());
+  }
+  clear_temp_objs(t->get_temp_cleared());
+
+  parent->log_operation(
+    log_entries,
+    hset_history,
+    trim_to,
+    trim_rollback_to,
+    true,
+    &local_t);
+  local_t.append(*op_t);
+  local_t.swap(*op_t);
+  
+  op_t->register_on_applied_sync(on_local_applied_sync);
+  op_t->register_on_applied(
+    parent->bless_context(
+      new C_OSD_OnOpApplied(this, &op)));
+  op_t->register_on_applied(
+    new ObjectStore::C_DeleteTransaction(op_t));
+  op_t->register_on_commit(
+    parent->bless_context(
+      new C_OSD_OnOpCommit(this, &op)));
+      
+  parent->queue_transaction(op_t, op.op);
+  delete t;
+}
+
+void ReplicatedBackend::op_applied(
+  InProgressOp *op)
+{
+  dout(10) << __func__ << ": " << op->tid << dendl;
+  if (op->op)
+    op->op->mark_event("op_applied");
+
+  op->waiting_for_applied.erase(get_parent()->whoami_shard());
+  parent->op_applied(op->v);
+
+  if (op->waiting_for_applied.empty()) {
+    op->on_applied->complete(0);
+    op->on_applied = 0;
+  }
+  if (op->done()) {
+    assert(!op->on_commit && !op->on_applied);
+    in_progress_ops.erase(op->tid);
+  }
+}
+
+void ReplicatedBackend::op_commit(
+  InProgressOp *op)
+{
+  dout(10) << __func__ << ": " << op->tid << dendl;
+  if (op->op)
+    op->op->mark_event("op_commit");
+
+  op->waiting_for_commit.erase(get_parent()->whoami_shard());
+
+  if (op->waiting_for_commit.empty()) {
+    op->on_commit->complete(0);
+    op->on_commit = 0;
+  }
+  if (op->done()) {
+    assert(!op->on_commit && !op->on_applied);
+    in_progress_ops.erase(op->tid);
+  }
+}
+
+void ReplicatedBackend::sub_op_modify_reply(OpRequestRef op)
+{
+  MOSDSubOpReply *r = static_cast(op->get_req());
+  assert(r->get_header().type == MSG_OSD_SUBOPREPLY);
+
+  op->mark_started();
+
+  // must be replication.
+  ceph_tid_t rep_tid = r->get_tid();
+  pg_shard_t from = r->from;
+
+  if (in_progress_ops.count(rep_tid)) {
+    map::iterator iter =
+      in_progress_ops.find(rep_tid);
+    InProgressOp &ip_op = iter->second;
+    MOSDOp *m = NULL;
+    if (ip_op.op)
+      m = static_cast(ip_op.op->get_req());
+
+    if (m)
+      dout(7) << __func__ << ": tid " << ip_op.tid << " op " //<< *m
+	      << " ack_type " << (int)r->ack_type
+	      << " from " << from
+	      << dendl;
+    else
+      dout(7) << __func__ << ": tid " << ip_op.tid << " (no op) "
+	      << " ack_type " << (int)r->ack_type
+	      << " from " << from
+	      << dendl;
+
+    // oh, good.
+
+    if (r->ack_type & CEPH_OSD_FLAG_ONDISK) {
+      assert(ip_op.waiting_for_commit.count(from));
+      ip_op.waiting_for_commit.erase(from);
+      if (ip_op.op)
+	ip_op.op->mark_event("sub_op_commit_rec");
+    } else {
+      assert(ip_op.waiting_for_applied.count(from));
+      if (ip_op.op)
+	ip_op.op->mark_event("sub_op_applied_rec");
+    }
+    ip_op.waiting_for_applied.erase(from);
+
+    parent->update_peer_last_complete_ondisk(
+      from,
+      r->get_last_complete_ondisk());
+
+    if (ip_op.waiting_for_applied.empty() &&
+        ip_op.on_applied) {
+      ip_op.on_applied->complete(0);
+      ip_op.on_applied = 0;
+    }
+    if (ip_op.waiting_for_commit.empty() &&
+        ip_op.on_commit) {
+      ip_op.on_commit->complete(0);
+      ip_op.on_commit= 0;
+    }
+    if (ip_op.done()) {
+      assert(!ip_op.on_commit && !ip_op.on_applied);
+      in_progress_ops.erase(iter);
+    }
+  }
+}
+
+void ReplicatedBackend::be_deep_scrub(
+  const hobject_t &poid,
+  ScrubMap::object &o,
+  ThreadPool::TPHandle &handle) {
+  bufferhash h, oh;
+  bufferlist bl, hdrbl;
+  int r;
+  __u64 pos = 0;
+  while ( (r = store->read(
+	     coll,
+	     ghobject_t(
+	       poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+	     pos,
+	     cct->_conf->osd_deep_scrub_stride, bl,
+	     true)) > 0) {
+    handle.reset_tp_timeout();
+    h << bl;
+    pos += bl.length();
+    bl.clear();
+  }
+  if (r == -EIO) {
+    dout(25) << "_scan_list  " << poid << " got "
+	     << r << " on read, read_error" << dendl;
+    o.read_error = true;
+  }
+  o.digest = h.digest();
+  o.digest_present = true;
+
+  bl.clear();
+  r = store->omap_get_header(
+    coll,
+    ghobject_t(
+      poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
+    &hdrbl, true);
+  if (r == 0) {
+    dout(25) << "CRC header " << string(hdrbl.c_str(), hdrbl.length())
+             << dendl;
+    ::encode(hdrbl, bl);
+    oh << bl;
+    bl.clear();
+  } else if (r == -EIO) {
+    dout(25) << "_scan_list  " << poid << " got "
+	     << r << " on omap header read, read_error" << dendl;
+    o.read_error = true;
+  }
+
+  ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(
+    coll,
+    ghobject_t(
+      poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard));
+  assert(iter);
+  uint64_t keys_scanned = 0;
+  for (iter->seek_to_first(); iter->valid() ; iter->next()) {
+    if (cct->_conf->osd_scan_list_ping_tp_interval &&
+	(keys_scanned % cct->_conf->osd_scan_list_ping_tp_interval == 0)) {
+      handle.reset_tp_timeout();
+    }
+    ++keys_scanned;
+
+    dout(25) << "CRC key " << iter->key() << " value "
+	     << string(iter->value().c_str(), iter->value().length()) << dendl;
+
+    ::encode(iter->key(), bl);
+    ::encode(iter->value(), bl);
+    oh << bl;
+    bl.clear();
+  }
+  if (iter->status() == -EIO) {
+    dout(25) << "_scan_list  " << poid << " got "
+	     << r << " on omap scan, read_error" << dendl;
+    o.read_error = true;
+  }
+
+  //Store final calculated CRC32 of omap header & key/values
+  o.omap_digest = oh.digest();
+  o.omap_digest_present = true;
+}
diff --git a/ceph/src/osd/ReplicatedBackend.h b/ceph/src/osd/ReplicatedBackend.h
new file mode 100644
index 00000000..5e1f0ec1
--- /dev/null
+++ b/ceph/src/osd/ReplicatedBackend.h
@@ -0,0 +1,420 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef REPBACKEND_H
+#define REPBACKEND_H
+
+#include "OSD.h"
+#include "PGBackend.h"
+#include "osd_types.h"
+#include "../include/memory.h"
+
+struct C_ReplicatedBackend_OnPullComplete;
+class ReplicatedBackend : public PGBackend {
+  struct RPGHandle : public PGBackend::RecoveryHandle {
+    map > pushes;
+    map > pulls;
+  };
+  friend struct C_ReplicatedBackend_OnPullComplete;
+public:
+  CephContext *cct;
+
+  ReplicatedBackend(
+    PGBackend::Listener *pg,
+    coll_t coll,
+    coll_t temp_coll,
+    ObjectStore *store,
+    CephContext *cct);
+
+  /// @see PGBackend::open_recovery_op
+  RPGHandle *_open_recovery_op() {
+    return new RPGHandle();
+  }
+  PGBackend::RecoveryHandle *open_recovery_op() {
+    return _open_recovery_op();
+  }
+
+  /// @see PGBackend::run_recovery_op
+  void run_recovery_op(
+    PGBackend::RecoveryHandle *h,
+    int priority);
+
+  /// @see PGBackend::recover_object
+  void recover_object(
+    const hobject_t &hoid,
+    eversion_t v,
+    ObjectContextRef head,
+    ObjectContextRef obc,
+    RecoveryHandle *h
+    );
+
+  void check_recovery_sources(const OSDMapRef osdmap);
+
+  /// @see PGBackend::delay_message_until_active
+  bool can_handle_while_inactive(OpRequestRef op);
+
+  /// @see PGBackend::handle_message
+  bool handle_message(
+    OpRequestRef op
+    );
+
+  void on_change();
+  void clear_state();
+  void on_flushed();
+
+  class RPCRecPred : public IsRecoverablePredicate {
+  public:
+    bool operator()(const set &have) const {
+      return !have.empty();
+    }
+  };
+  IsRecoverablePredicate *get_is_recoverable_predicate() {
+    return new RPCRecPred;
+  }
+
+  class RPCReadPred : public IsReadablePredicate {
+    pg_shard_t whoami;
+  public:
+    RPCReadPred(pg_shard_t whoami) : whoami(whoami) {}
+    bool operator()(const set &have) const {
+      return have.count(whoami);
+    }
+  };
+  IsReadablePredicate *get_is_readable_predicate() {
+    return new RPCReadPred(get_parent()->whoami_shard());
+  }
+
+  virtual void dump_recovery_info(Formatter *f) const {
+    {
+      f->open_array_section("pull_from_peer");
+      for (map >::const_iterator i = pull_from_peer.begin();
+	   i != pull_from_peer.end();
+	   ++i) {
+	f->open_object_section("pulling_from");
+	f->dump_stream("pull_from") << i->first;
+	{
+	  f->open_array_section("pulls");
+	  for (set::const_iterator j = i->second.begin();
+	       j != i->second.end();
+	       ++j) {
+	    f->open_object_section("pull_info");
+	    assert(pulling.count(*j));
+	    pulling.find(*j)->second.dump(f);
+	    f->close_section();
+	  }
+	  f->close_section();
+	}
+	f->close_section();
+      }
+      f->close_section();
+    }
+    {
+      f->open_array_section("pushing");
+      for (map >::const_iterator i =
+	     pushing.begin();
+	   i != pushing.end();
+	   ++i) {
+	f->open_object_section("object");
+	f->dump_stream("pushing") << i->first;
+	{
+	  f->open_array_section("pushing_to");
+	  for (map::const_iterator j = i->second.begin();
+	       j != i->second.end();
+	       ++j) {
+	    f->open_object_section("push_progress");
+	    f->dump_stream("pushing_to") << j->first;
+	    {
+	      f->open_object_section("push_info");
+	      j->second.dump(f);
+	      f->close_section();
+	    }
+	    f->close_section();
+	  }
+	  f->close_section();
+	}
+	f->close_section();
+      }
+      f->close_section();
+    }
+  }
+
+  int objects_read_sync(
+    const hobject_t &hoid,
+    uint64_t off,
+    uint64_t len,
+    bufferlist *bl);
+
+  void objects_read_async(
+    const hobject_t &hoid,
+    const list,
+	       pair > > &to_read,
+    Context *on_complete);
+
+private:
+  // push
+  struct PushInfo {
+    ObjectRecoveryProgress recovery_progress;
+    ObjectRecoveryInfo recovery_info;
+    ObjectContextRef obc;
+    object_stat_sum_t stat;
+
+    void dump(Formatter *f) const {
+      {
+	f->open_object_section("recovery_progress");
+	recovery_progress.dump(f);
+	f->close_section();
+      }
+      {
+	f->open_object_section("recovery_info");
+	recovery_info.dump(f);
+	f->close_section();
+      }
+    }
+  };
+  map > pushing;
+
+  // pull
+  struct PullInfo {
+    ObjectRecoveryProgress recovery_progress;
+    ObjectRecoveryInfo recovery_info;
+    ObjectContextRef head_ctx;
+    ObjectContextRef obc;
+    object_stat_sum_t stat;
+
+    void dump(Formatter *f) const {
+      {
+	f->open_object_section("recovery_progress");
+	recovery_progress.dump(f);
+	f->close_section();
+      }
+      {
+	f->open_object_section("recovery_info");
+	recovery_info.dump(f);
+	f->close_section();
+      }
+    }
+
+    bool is_complete() const {
+      return recovery_progress.is_complete(recovery_info);
+    }
+  };
+
+  map pulling;
+
+  // Reverse mapping from osd peer to objects beging pulled from that peer
+  map > pull_from_peer;
+
+  void sub_op_push(OpRequestRef op);
+  void sub_op_push_reply(OpRequestRef op);
+  void sub_op_pull(OpRequestRef op);
+
+  void _do_push(OpRequestRef op);
+  void _do_pull_response(OpRequestRef op);
+  void do_push(OpRequestRef op) {
+    if (is_primary()) {
+      _do_pull_response(op);
+    } else {
+      _do_push(op);
+    }
+  }
+  void do_pull(OpRequestRef op);
+  void do_push_reply(OpRequestRef op);
+
+  bool handle_push_reply(pg_shard_t peer, PushReplyOp &op, PushOp *reply);
+  void handle_pull(pg_shard_t peer, PullOp &op, PushOp *reply);
+  bool handle_pull_response(
+    pg_shard_t from, PushOp &op, PullOp *response,
+    list *to_continue,
+    ObjectStore::Transaction *t);
+  void handle_push(pg_shard_t from, PushOp &op, PushReplyOp *response,
+		   ObjectStore::Transaction *t);
+
+  static void trim_pushed_data(const interval_set ©_subset,
+			       const interval_set &intervals_received,
+			       bufferlist data_received,
+			       interval_set *intervals_usable,
+			       bufferlist *data_usable);
+  void _failed_push(pg_shard_t from, const hobject_t &soid);
+
+  void send_pushes(int prio, map > &pushes);
+  void prep_push_op_blank(const hobject_t& soid, PushOp *op);
+  int send_push_op_legacy(int priority, pg_shard_t peer,
+			  PushOp &pop);
+  int send_pull_legacy(int priority, pg_shard_t peer,
+		       const ObjectRecoveryInfo& recovery_info,
+		       ObjectRecoveryProgress progress);
+  void send_pulls(
+    int priority,
+    map > &pulls);
+
+  int build_push_op(const ObjectRecoveryInfo &recovery_info,
+		    const ObjectRecoveryProgress &progress,
+		    ObjectRecoveryProgress *out_progress,
+		    PushOp *out_op,
+		    object_stat_sum_t *stat = 0);
+  void submit_push_data(ObjectRecoveryInfo &recovery_info,
+			bool first,
+			bool complete,
+			const interval_set &intervals_included,
+			bufferlist data_included,
+			bufferlist omap_header,
+			map &attrs,
+			map &omap_entries,
+			ObjectStore::Transaction *t);
+  void submit_push_complete(ObjectRecoveryInfo &recovery_info,
+			    ObjectStore::Transaction *t);
+
+  void calc_clone_subsets(
+    SnapSet& snapset, const hobject_t& poid, const pg_missing_t& missing,
+    const hobject_t &last_backfill,
+    interval_set& data_subset,
+    map >& clone_subsets);
+  void prepare_pull(
+    eversion_t v,
+    const hobject_t& soid,
+    ObjectContextRef headctx,
+    RPGHandle *h);
+  int start_pushes(
+    const hobject_t &soid,
+    ObjectContextRef obj,
+    RPGHandle *h);
+  void prep_push_to_replica(
+    ObjectContextRef obc, const hobject_t& soid, pg_shard_t peer,
+    PushOp *pop);
+  void prep_push(ObjectContextRef obc,
+		 const hobject_t& oid, pg_shard_t dest,
+		 PushOp *op);
+  void prep_push(ObjectContextRef obc,
+		 const hobject_t& soid, pg_shard_t peer,
+		 eversion_t version,
+		 interval_set &data_subset,
+		 map >& clone_subsets,
+		 PushOp *op);
+  void calc_head_subsets(ObjectContextRef obc, SnapSet& snapset, const hobject_t& head,
+			 const pg_missing_t& missing,
+			 const hobject_t &last_backfill,
+			 interval_set& data_subset,
+			 map >& clone_subsets);
+  ObjectRecoveryInfo recalc_subsets(
+    const ObjectRecoveryInfo& recovery_info,
+    SnapSetContext *ssc
+    );
+
+  /**
+   * Client IO
+   */
+  struct InProgressOp {
+    ceph_tid_t tid;
+    set waiting_for_commit;
+    set waiting_for_applied;
+    Context *on_commit;
+    Context *on_applied;
+    OpRequestRef op;
+    eversion_t v;
+    InProgressOp(
+      ceph_tid_t tid, Context *on_commit, Context *on_applied,
+      OpRequestRef op, eversion_t v)
+      : tid(tid), on_commit(on_commit), on_applied(on_applied),
+	op(op), v(v) {}
+    bool done() const {
+      return waiting_for_commit.empty() &&
+	waiting_for_applied.empty();
+    }
+  };
+  map in_progress_ops;
+public:
+  PGTransaction *get_transaction();
+  friend class C_OSD_OnOpCommit;
+  friend class C_OSD_OnOpApplied;
+  void submit_transaction(
+    const hobject_t &hoid,
+    const eversion_t &at_version,
+    PGTransaction *t,
+    const eversion_t &trim_to,
+    const eversion_t &trim_rollback_to,
+    vector &log_entries,
+    boost::optional &hset_history,
+    Context *on_local_applied_sync,
+    Context *on_all_applied,
+    Context *on_all_commit,
+    ceph_tid_t tid,
+    osd_reqid_t reqid,
+    OpRequestRef op
+    );
+
+private:
+  void issue_op(
+    const hobject_t &soid,
+    const eversion_t &at_version,
+    ceph_tid_t tid,
+    osd_reqid_t reqid,
+    eversion_t pg_trim_to,
+    eversion_t pg_trim_rollback_to,
+    hobject_t new_temp_oid,
+    hobject_t discard_temp_oid,
+    vector &log_entries,
+    boost::optional &hset_history,
+    InProgressOp *op,
+    ObjectStore::Transaction *op_t);
+  void op_applied(InProgressOp *op);
+  void op_commit(InProgressOp *op);
+  void sub_op_modify_reply(OpRequestRef op);
+  void sub_op_modify(OpRequestRef op);
+
+  struct RepModify {
+    OpRequestRef op;
+    bool applied, committed;
+    int ackerosd;
+    eversion_t last_complete;
+    epoch_t epoch_started;
+
+    uint64_t bytes_written;
+
+    ObjectStore::Transaction opt, localt;
+    
+    RepModify() : applied(false), committed(false), ackerosd(-1),
+		  epoch_started(0), bytes_written(0) {}
+  };
+  typedef ceph::shared_ptr RepModifyRef;
+
+  struct C_OSD_RepModifyApply : public Context {
+    ReplicatedBackend *pg;
+    RepModifyRef rm;
+    C_OSD_RepModifyApply(ReplicatedBackend *pg, RepModifyRef r)
+      : pg(pg), rm(r) {}
+    void finish(int r) {
+      pg->sub_op_modify_applied(rm);
+    }
+  };
+  struct C_OSD_RepModifyCommit : public Context {
+    ReplicatedBackend *pg;
+    RepModifyRef rm;
+    C_OSD_RepModifyCommit(ReplicatedBackend *pg, RepModifyRef r)
+      : pg(pg), rm(r) {}
+    void finish(int r) {
+      pg->sub_op_modify_commit(rm);
+    }
+  };
+  void sub_op_modify_applied(RepModifyRef rm);
+  void sub_op_modify_commit(RepModifyRef rm);
+  bool scrub_supported() { return true; }
+
+  void be_deep_scrub(
+    const hobject_t &obj,
+    ScrubMap::object &o,
+    ThreadPool::TPHandle &handle);
+  uint64_t be_get_ondisk_size(uint64_t logical_size) { return logical_size; }
+};
+
+#endif
diff --git a/ceph/src/osd/ReplicatedPG.cc b/ceph/src/osd/ReplicatedPG.cc
new file mode 100644
index 00000000..d23e6fc9
--- /dev/null
+++ b/ceph/src/osd/ReplicatedPG.cc
@@ -0,0 +1,12226 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "boost/tuple/tuple.hpp"
+#include "PG.h"
+#include "ReplicatedPG.h"
+#include "OSD.h"
+#include "OpRequest.h"
+
+#include "common/errno.h"
+#include "common/perf_counters.h"
+
+#include "messages/MOSDOp.h"
+#include "messages/MOSDOpReply.h"
+#include "messages/MOSDSubOp.h"
+#include "messages/MOSDSubOpReply.h"
+
+#include "messages/MOSDPGNotify.h"
+#include "messages/MOSDPGInfo.h"
+#include "messages/MOSDPGRemove.h"
+#include "messages/MOSDPGTrim.h"
+#include "messages/MOSDPGScan.h"
+#include "messages/MOSDPGBackfill.h"
+
+#include "messages/MOSDPing.h"
+#include "messages/MWatchNotify.h"
+
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPull.h"
+#include "messages/MOSDPGPushReply.h"
+
+#include "Watch.h"
+
+#include "mds/inode_backtrace.h" // Ugh
+
+#include "common/config.h"
+#include "include/compat.h"
+#include "common/cmdparse.h"
+
+#include "mon/MonClient.h"
+#include "osdc/Objecter.h"
+
+#include "json_spirit/json_spirit_value.h"
+#include "json_spirit/json_spirit_reader.h"
+#include "include/assert.h"  // json_spirit clobbers it
+
+#define dout_subsys ceph_subsys_osd
+#define DOUT_PREFIX_ARGS this, osd->whoami, get_osdmap()
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+template 
+static ostream& _prefix(std::ostream *_dout, T *pg) {
+  return *_dout << pg->gen_prefix();
+}
+
+
+#include 
+#include 
+
+#include 
+
+PGLSFilter::PGLSFilter()
+{
+}
+
+PGLSFilter::~PGLSFilter()
+{
+}
+
+static void log_subop_stats(
+  PerfCounters *logger,
+  OpRequestRef op, int tag_inb, int tag_lat)
+{
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t latency = now;
+  latency -= op->get_req()->get_recv_stamp();
+
+  uint64_t inb = op->get_req()->get_data().length();
+
+  logger->inc(l_osd_sop);
+
+  logger->inc(l_osd_sop_inb, inb);
+  logger->tinc(l_osd_sop_lat, latency);
+
+  if (tag_inb)
+    logger->inc(tag_inb, inb);
+  logger->tinc(tag_lat, latency);
+}
+
+struct OnReadComplete : public Context {
+  ReplicatedPG *pg;
+  ReplicatedPG::OpContext *opcontext;
+  OnReadComplete(
+    ReplicatedPG *pg,
+    ReplicatedPG::OpContext *ctx) : pg(pg), opcontext(ctx) {}
+  void finish(int r) {
+    if (r < 0)
+      opcontext->async_read_result = r;
+    opcontext->finish_read(pg);
+  }
+  ~OnReadComplete() {}
+};
+
+// OpContext
+void ReplicatedPG::OpContext::start_async_reads(ReplicatedPG *pg)
+{
+  inflightreads = 1;
+  pg->pgbackend->objects_read_async(
+    obc->obs.oi.soid,
+    pending_async_reads,
+    new OnReadComplete(pg, this));
+  pending_async_reads.clear();
+}
+void ReplicatedPG::OpContext::finish_read(ReplicatedPG *pg)
+{
+  assert(inflightreads > 0);
+  --inflightreads;
+  if (async_reads_complete()) {
+    assert(pg->in_progress_async_reads.size());
+    assert(pg->in_progress_async_reads.front().second == this);
+    pg->in_progress_async_reads.pop_front();
+    pg->complete_read_ctx(async_read_result, this);
+  }
+}
+
+class CopyFromCallback: public ReplicatedPG::CopyCallback {
+public:
+  ReplicatedPG::CopyResults *results;
+  int retval;
+  ReplicatedPG::OpContext *ctx;
+  CopyFromCallback(ReplicatedPG::OpContext *ctx_)
+    : results(NULL),
+      retval(0),
+      ctx(ctx_) {}
+  ~CopyFromCallback() {}
+
+  virtual void finish(ReplicatedPG::CopyCallbackResults results_) {
+    results = results_.get<1>();
+    int r = results_.get<0>();
+    retval = r;
+
+    // for finish_copyfrom
+    ctx->user_at_version = results->user_version;
+
+    if (r >= 0) {
+      ctx->pg->execute_ctx(ctx);
+    }
+    ctx->copy_cb = NULL;
+    if (r < 0) {
+      if (r != -ECANCELED) { // on cancel just toss it out; client resends
+	if (ctx->op)
+	  ctx->pg->osd->reply_op_error(ctx->op, r);
+      } else if (results->should_requeue) {
+	if (ctx->op)
+	  ctx->pg->requeue_op(ctx->op);
+      }
+      ctx->pg->close_op_ctx(ctx, r);
+    }
+  }
+
+  bool is_temp_obj_used() {
+    return results->started_temp_obj;
+  }
+  uint64_t get_data_size() {
+    return results->object_size;
+  }
+  int get_result() {
+    return retval;
+  }
+};
+
+// ======================
+// PGBackend::Listener
+
+
+void ReplicatedPG::on_local_recover_start(
+  const hobject_t &oid,
+  ObjectStore::Transaction *t)
+{
+  pg_log.revise_have(oid, eversion_t());
+  remove_snap_mapped_object(*t, oid);
+}
+
+void ReplicatedPG::on_local_recover(
+  const hobject_t &hoid,
+  const object_stat_sum_t &stat_diff,
+  const ObjectRecoveryInfo &_recovery_info,
+  ObjectContextRef obc,
+  ObjectStore::Transaction *t
+  )
+{
+  dout(10) << __func__ << ": " << hoid << dendl;
+  ObjectRecoveryInfo recovery_info(_recovery_info);
+  if (recovery_info.soid.snap < CEPH_NOSNAP) {
+    assert(recovery_info.oi.snaps.size());
+    OSDriver::OSTransaction _t(osdriver.get_transaction(t));
+    set snaps(
+      recovery_info.oi.snaps.begin(),
+      recovery_info.oi.snaps.end());
+    snap_mapper.add_oid(
+      recovery_info.soid,
+      snaps,
+      &_t);
+  }
+
+  if (pg_log.get_missing().is_missing(recovery_info.soid) &&
+      pg_log.get_missing().missing.find(recovery_info.soid)->second.need > recovery_info.version) {
+    assert(is_primary());
+    const pg_log_entry_t *latest = pg_log.get_log().objects.find(recovery_info.soid)->second;
+    if (latest->op == pg_log_entry_t::LOST_REVERT &&
+	latest->reverting_to == recovery_info.version) {
+      dout(10) << " got old revert version " << recovery_info.version
+	       << " for " << *latest << dendl;
+      recovery_info.version = latest->version;
+      // update the attr to the revert event version
+      recovery_info.oi.prior_version = recovery_info.oi.version;
+      recovery_info.oi.version = latest->version;
+      bufferlist bl;
+      ::encode(recovery_info.oi, bl);
+      t->setattr(coll, recovery_info.soid, OI_ATTR, bl);
+      if (obc)
+	obc->attr_cache[OI_ATTR] = bl;
+    }
+  }
+
+  // keep track of active pushes for scrub
+  ++active_pushes;
+
+  recover_got(recovery_info.soid, recovery_info.version);
+
+  if (is_primary()) {
+    info.stats.stats.sum.add(stat_diff);
+
+    assert(obc);
+    obc->obs.exists = true;
+    obc->ondisk_write_lock();
+    obc->obs.oi = recovery_info.oi;  // may have been updated above
+
+
+    t->register_on_applied(new C_OSD_AppliedRecoveredObject(this, obc));
+    t->register_on_applied_sync(new C_OSD_OndiskWriteUnlock(obc));
+
+    publish_stats_to_osd();
+    assert(missing_loc.needs_recovery(hoid));
+    missing_loc.add_location(hoid, pg_whoami);
+    if (!is_unreadable_object(hoid) &&
+        waiting_for_unreadable_object.count(hoid)) {
+      dout(20) << " kicking unreadable waiters on " << hoid << dendl;
+      requeue_ops(waiting_for_unreadable_object[hoid]);
+      waiting_for_unreadable_object.erase(hoid);
+    }
+    if (pg_log.get_missing().missing.size() == 0) {
+      requeue_ops(waiting_for_all_missing);
+      waiting_for_all_missing.clear();
+    }
+  } else {
+    t->register_on_applied(
+      new C_OSD_AppliedRecoveredObjectReplica(this));
+
+  }
+
+  t->register_on_commit(
+    new C_OSD_CommittedPushedObject(
+      this,
+      get_osdmap()->get_epoch(),
+      info.last_complete));
+
+  // update pg
+  dirty_info = true;
+  write_if_dirty(*t);
+
+}
+
+void ReplicatedPG::on_global_recover(
+  const hobject_t &soid)
+{
+  missing_loc.recovered(soid);
+  publish_stats_to_osd();
+  dout(10) << "pushed " << soid << " to all replicas" << dendl;
+  map::iterator i = recovering.find(soid);
+  assert(i != recovering.end());
+  if (backfills_in_flight.count(soid)) {
+    list requeue_list;
+    i->second->drop_backfill_read(&requeue_list);
+    requeue_ops(requeue_list);
+    backfills_in_flight.erase(soid);
+  }
+  recovering.erase(i);
+  finish_recovery_op(soid);
+  if (waiting_for_degraded_object.count(soid)) {
+    dout(20) << " kicking degraded waiters on " << soid << dendl;
+    requeue_ops(waiting_for_degraded_object[soid]);
+    waiting_for_degraded_object.erase(soid);
+  }
+  if (waiting_for_unreadable_object.count(soid)) {
+    dout(20) << " kicking unreadable waiters on " << soid << dendl;
+    requeue_ops(waiting_for_unreadable_object[soid]);
+    waiting_for_unreadable_object.erase(soid);
+  }
+  finish_degraded_object(soid);
+}
+
+void ReplicatedPG::on_peer_recover(
+  pg_shard_t peer,
+  const hobject_t &soid,
+  const ObjectRecoveryInfo &recovery_info,
+  const object_stat_sum_t &stat)
+{
+  info.stats.stats.sum.add(stat);
+  publish_stats_to_osd();
+  // done!
+  peer_missing[peer].got(soid, recovery_info.version);
+}
+
+void ReplicatedPG::begin_peer_recover(
+  pg_shard_t peer,
+  const hobject_t soid)
+{
+  peer_missing[peer].revise_have(soid, eversion_t());
+}
+
+void ReplicatedPG::schedule_work(
+  GenContext *c)
+{
+  osd->gen_wq.queue(c);
+}
+
+void ReplicatedPG::send_message_osd_cluster(
+  int peer, Message *m, epoch_t from_epoch)
+{
+  osd->send_message_osd_cluster(peer, m, from_epoch);
+}
+
+void ReplicatedPG::send_message_osd_cluster(
+  Message *m, Connection *con)
+{
+  osd->send_message_osd_cluster(m, con);
+}
+
+void ReplicatedPG::send_message_osd_cluster(
+  Message *m, const ConnectionRef& con)
+{
+  osd->send_message_osd_cluster(m, con);
+}
+
+ConnectionRef ReplicatedPG::get_con_osd_cluster(
+  int peer, epoch_t from_epoch)
+{
+  return osd->get_con_osd_cluster(peer, from_epoch);
+}
+
+PerfCounters *ReplicatedPG::get_logger()
+{
+  return osd->logger;
+}
+
+
+// ====================
+// missing objects
+
+bool ReplicatedPG::is_missing_object(const hobject_t& soid) const
+{
+  return pg_log.get_missing().missing.count(soid);
+}
+
+void ReplicatedPG::wait_for_unreadable_object(
+  const hobject_t& soid, OpRequestRef op)
+{
+  assert(is_unreadable_object(soid));
+
+  eversion_t v;
+  bool needs_recovery = missing_loc.needs_recovery(soid, &v);
+  assert(needs_recovery);
+
+  map::const_iterator p = recovering.find(soid);
+  if (p != recovering.end()) {
+    dout(7) << "missing " << soid << " v " << v << ", already recovering." << dendl;
+  } else if (missing_loc.is_unfound(soid)) {
+    dout(7) << "missing " << soid << " v " << v << ", is unfound." << dendl;
+  } else {
+    dout(7) << "missing " << soid << " v " << v << ", recovering." << dendl;
+    PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
+    recover_missing(soid, v, cct->_conf->osd_client_op_priority, h);
+    pgbackend->run_recovery_op(h, cct->_conf->osd_client_op_priority);
+  }
+  waiting_for_unreadable_object[soid].push_back(op);
+  op->mark_delayed("waiting for missing object");
+}
+
+void ReplicatedPG::wait_for_all_missing(OpRequestRef op)
+{
+  waiting_for_all_missing.push_back(op);
+}
+
+bool ReplicatedPG::is_degraded_object(const hobject_t& soid)
+{
+  if (pg_log.get_missing().missing.count(soid))
+    return true;
+  assert(actingbackfill.size() > 0);
+  for (set::iterator i = actingbackfill.begin();
+       i != actingbackfill.end();
+       ++i) {
+    if (*i == get_primary()) continue;
+    pg_shard_t peer = *i;
+    if (peer_missing.count(peer) &&
+	peer_missing[peer].missing.count(soid))
+      return true;
+
+    // Object is degraded if after last_backfill AND
+    // we are backfilling it
+    if (is_backfill_targets(peer) &&
+	peer_info[peer].last_backfill <= soid &&
+	last_backfill_started >= soid &&
+	backfills_in_flight.count(soid))
+      return true;
+  }
+  return false;
+}
+
+void ReplicatedPG::wait_for_degraded_object(const hobject_t& soid, OpRequestRef op)
+{
+  assert(is_degraded_object(soid));
+
+  // we don't have it (yet).
+  if (recovering.count(soid)) {
+    dout(7) << "degraded "
+	    << soid 
+	    << ", already recovering"
+	    << dendl;
+  } else if (missing_loc.is_unfound(soid)) {
+    dout(7) << "degraded "
+	    << soid
+	    << ", still unfound, waiting"
+	    << dendl;
+  } else {
+    dout(7) << "degraded " 
+	    << soid 
+	    << ", recovering"
+	    << dendl;
+    eversion_t v;
+    assert(actingbackfill.size() > 0);
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == get_primary()) continue;
+      pg_shard_t peer = *i;
+      if (peer_missing.count(peer) &&
+	  peer_missing[peer].missing.count(soid)) {
+	v = peer_missing[peer].missing[soid].need;
+	break;
+      }
+    }
+    PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
+    prep_object_replica_pushes(soid, v, h);
+    pgbackend->run_recovery_op(h, cct->_conf->osd_client_op_priority);
+  }
+  waiting_for_degraded_object[soid].push_back(op);
+  op->mark_delayed("waiting for degraded object");
+}
+
+bool ReplicatedPG::maybe_await_blocked_snapset(
+  const hobject_t &hoid,
+  OpRequestRef op)
+{
+  ObjectContextRef obc;
+  if (obc = object_contexts.lookup(hoid.get_head())) {
+    if (obc->is_blocked()) {
+      wait_for_blocked_object(obc->obs.oi.soid, op);
+      return true;
+    } else {
+      return false;
+    }
+  }
+  if (obc = object_contexts.lookup(hoid.get_snapdir())) {
+    if (obc->is_blocked()) {
+      wait_for_blocked_object(obc->obs.oi.soid, op);
+      return true;
+    } else {
+      return false;
+    }
+  }
+  return false;
+}
+
+void ReplicatedPG::wait_for_blocked_object(const hobject_t& soid, OpRequestRef op)
+{
+  dout(10) << __func__ << " " << soid << " " << op << dendl;
+  waiting_for_blocked_object[soid].push_back(op);
+  op->mark_delayed("waiting for blocked object");
+}
+
+bool PGLSParentFilter::filter(bufferlist& xattr_data, bufferlist& outdata)
+{
+  bufferlist::iterator iter = xattr_data.begin();
+  inode_backtrace_t bt;
+
+  generic_dout(0) << "PGLSParentFilter::filter" << dendl;
+
+  ::decode(bt, iter);
+
+  vector::iterator vi;
+  for (vi = bt.ancestors.begin(); vi != bt.ancestors.end(); ++vi) {
+    generic_dout(0) << "vi->dirino=" << vi->dirino << " parent_ino=" << parent_ino << dendl;
+    if ( vi->dirino == parent_ino) {
+      ::encode(*vi, outdata);
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool PGLSPlainFilter::filter(bufferlist& xattr_data, bufferlist& outdata)
+{
+  if (val.size() != xattr_data.length())
+    return false;
+
+  if (memcmp(val.c_str(), xattr_data.c_str(), val.size()))
+    return false;
+
+  return true;
+}
+
+bool ReplicatedPG::pgls_filter(PGLSFilter *filter, hobject_t& sobj, bufferlist& outdata)
+{
+  bufferlist bl;
+  int ret = pgbackend->objects_get_attr(
+    sobj,
+    filter->get_xattr(),
+    &bl);
+  dout(0) << "getattr (sobj=" << sobj << ", attr=" << filter->get_xattr() << ") returned " << ret << dendl;
+  if (ret < 0)
+    return false;
+
+  return filter->filter(bl, outdata);
+}
+
+int ReplicatedPG::get_pgls_filter(bufferlist::iterator& iter, PGLSFilter **pfilter)
+{
+  string type;
+  PGLSFilter *filter;
+
+  try {
+    ::decode(type, iter);
+  }
+  catch (buffer::error& e) {
+    return -EINVAL;
+  }
+
+  if (type.compare("parent") == 0) {
+    filter = new PGLSParentFilter(iter);
+  } else if (type.compare("plain") == 0) {
+    filter = new PGLSPlainFilter(iter);
+  } else {
+    return -EINVAL;
+  }
+
+  *pfilter = filter;
+
+  return  0;
+}
+
+
+// ==========================================================
+
+int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss,
+			     bufferlist& idata, bufferlist& odata)
+{
+  const pg_missing_t &missing = pg_log.get_missing();
+  string prefix;
+  string format;
+
+  cmd_getval(cct, cmdmap, "format", format);
+  boost::scoped_ptr f(new_formatter(format));
+  // demand that we have a formatter
+  if (!f)
+    f.reset(new_formatter("json"));
+
+  string command;
+  cmd_getval(cct, cmdmap, "cmd", command);
+  if (command == "query") {
+    f->open_object_section("pg");
+    f->dump_string("state", pg_state_string(get_state()));
+    f->dump_unsigned("epoch", get_osdmap()->get_epoch());
+    f->open_array_section("up");
+    for (vector::iterator p = up.begin(); p != up.end(); ++p)
+      f->dump_unsigned("osd", *p);
+    f->close_section();
+    f->open_array_section("acting");
+    for (vector::iterator p = acting.begin(); p != acting.end(); ++p)
+      f->dump_unsigned("osd", *p);
+    f->close_section();
+    if (backfill_targets.size() > 0) {
+      f->open_array_section("backfill_targets");
+      for (set::iterator p = backfill_targets.begin();
+	   p != backfill_targets.end();
+	   ++p)
+        f->dump_stream("shard") << *p;
+      f->close_section();
+    }
+    if (actingbackfill.size() > 0) {
+      f->open_array_section("actingbackfill");
+      for (set::iterator p = actingbackfill.begin();
+	   p != actingbackfill.end();
+	   ++p)
+        f->dump_stream("shard") << *p;
+      f->close_section();
+    }
+    f->open_object_section("info");
+    _update_calc_stats();
+    info.dump(f.get());
+    f->close_section();
+
+    f->open_array_section("peer_info");
+    for (map::iterator p = peer_info.begin();
+	 p != peer_info.end();
+	 ++p) {
+      f->open_object_section("info");
+      f->dump_stream("peer") << p->first;
+      p->second.dump(f.get());
+      f->close_section();
+    }
+    f->close_section();
+
+    f->open_array_section("recovery_state");
+    handle_query_state(f.get());
+    f->close_section();
+
+    f->open_object_section("agent_state");
+    if (agent_state)
+      agent_state->dump(f.get());
+    f->close_section();
+
+    f->close_section();
+    f->flush(odata);
+    return 0;
+  }
+  else if (command == "mark_unfound_lost") {
+    string mulcmd;
+    cmd_getval(cct, cmdmap, "mulcmd", mulcmd);
+    int mode = -1;
+    if (mulcmd == "revert") {
+      if (pool.info.ec_pool()) {
+	ss << "mode must be 'delete' for ec pool";
+	return -EINVAL;
+      }
+      mode = pg_log_entry_t::LOST_REVERT;
+    } else if (mulcmd == "delete") {
+      mode = pg_log_entry_t::LOST_DELETE;
+    } else {
+      ss << "mode must be 'revert' or 'delete'; mark not yet implemented";
+      return -EINVAL;
+    }
+    assert(mode == pg_log_entry_t::LOST_REVERT ||
+	   mode == pg_log_entry_t::LOST_DELETE);
+
+    if (!is_primary()) {
+      ss << "not primary";
+      return -EROFS;
+    }
+
+    int unfound = missing_loc.num_unfound();
+    if (!unfound) {
+      ss << "pg has no unfound objects";
+      return 0;  // make command idempotent
+    }
+
+    if (!all_unfound_are_queried_or_lost(get_osdmap())) {
+      ss << "pg has " << unfound
+	 << " unfound objects but we haven't probed all sources, not marking lost";
+      return -EINVAL;
+    }
+
+    ss << "pg has " << unfound
+       << " objects unfound and apparently lost, marking";
+    mark_all_unfound_lost(mode);
+    return 0;
+  }
+  else if (command == "list_missing") {
+    hobject_t offset;
+    string offset_json;
+    if (cmd_getval(cct, cmdmap, "offset", offset_json)) {
+      json_spirit::Value v;
+      try {
+	if (!json_spirit::read(offset_json, v))
+	  throw std::runtime_error("bad json");
+	offset.decode(v);
+      } catch (std::runtime_error& e) {
+	ss << "error parsing offset: " << e.what();
+	return -EINVAL;
+      }
+    }
+    f->open_object_section("missing");
+    {
+      f->open_object_section("offset");
+      offset.dump(f.get());
+      f->close_section();
+    }
+    f->dump_int("num_missing", missing.num_missing());
+    f->dump_int("num_unfound", get_num_unfound());
+    map::const_iterator p = missing.missing.upper_bound(offset);
+    {
+      f->open_array_section("objects");
+      int32_t num = 0;
+      bufferlist bl;
+      while (p != missing.missing.end() && num < cct->_conf->osd_command_max_records) {
+	f->open_object_section("object");
+	{
+	  f->open_object_section("oid");
+	  p->first.dump(f.get());
+	  f->close_section();
+	}
+	p->second.dump(f.get());  // have, need keys
+	{
+	  f->open_array_section("locations");
+	  if (missing_loc.needs_recovery(p->first)) {
+	    for (set::iterator r =
+		   missing_loc.get_locations(p->first).begin();
+		 r != missing_loc.get_locations(p->first).end();
+		 ++r)
+	      f->dump_stream("shard") << *r;
+	  }
+	  f->close_section();
+	}
+	f->close_section();
+	++p;
+	num++;
+      }
+      f->close_section();
+    }
+    f->dump_int("more", p != missing.missing.end());
+    f->close_section();
+    f->flush(odata);
+    return 0;
+  };
+
+  ss << "unknown pg command " << prefix;
+  return -EINVAL;
+}
+
+// ==========================================================
+
+bool ReplicatedPG::pg_op_must_wait(MOSDOp *op)
+{
+  if (pg_log.get_missing().missing.empty())
+    return false;
+  for (vector::iterator p = op->ops.begin(); p != op->ops.end(); ++p) {
+    if (p->op.op == CEPH_OSD_OP_PGLS) {
+      if (op->get_snapid() != CEPH_NOSNAP) {
+	return true;
+      }
+    }
+  }
+  return false;
+}
+
+void ReplicatedPG::do_pg_op(OpRequestRef op)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == CEPH_MSG_OSD_OP);
+  dout(10) << "do_pg_op " << *m << dendl;
+
+  op->mark_started();
+
+  int result = 0;
+  string cname, mname;
+  PGLSFilter *filter = NULL;
+  bufferlist filter_out;
+
+  snapid_t snapid = m->get_snapid();
+
+  vector ops = m->ops;
+
+  for (vector::iterator p = ops.begin(); p != ops.end(); ++p) {
+    OSDOp& osd_op = *p;
+    bufferlist::iterator bp = p->indata.begin();
+    switch (p->op.op) {
+    case CEPH_OSD_OP_PGLS_FILTER:
+      try {
+	::decode(cname, bp);
+	::decode(mname, bp);
+      }
+      catch (const buffer::error& e) {
+	dout(0) << "unable to decode PGLS_FILTER description in " << *m << dendl;
+	result = -EINVAL;
+	break;
+      }
+      result = get_pgls_filter(bp, &filter);
+      if (result < 0)
+        break;
+
+      assert(filter);
+
+      // fall through
+
+    case CEPH_OSD_OP_PGLS:
+      if (m->get_pg() != info.pgid.pgid) {
+        dout(10) << " pgls pg=" << m->get_pg() << " != " << info.pgid << dendl;
+	result = 0; // hmm?
+      } else {
+	unsigned list_size = MIN(cct->_conf->osd_max_pgls, p->op.pgls.count);
+
+        dout(10) << " pgls pg=" << m->get_pg() << " count " << list_size << dendl;
+	// read into a buffer
+        vector sentries;
+        pg_ls_response_t response;
+	try {
+	  ::decode(response.handle, bp);
+	}
+	catch (const buffer::error& e) {
+	  dout(0) << "unable to decode PGLS handle in " << *m << dendl;
+	  result = -EINVAL;
+	  break;
+	}
+
+	hobject_t next;
+	hobject_t current = response.handle;
+	osr->flush();
+	int r = pgbackend->objects_list_partial(
+	  current,
+	  list_size,
+	  list_size,
+	  snapid,
+	  &sentries,
+	  &next);
+	if (r != 0) {
+	  result = -EINVAL;
+	  break;
+	}
+
+	assert(snapid == CEPH_NOSNAP || pg_log.get_missing().missing.empty());
+	map::const_iterator missing_iter =
+	  pg_log.get_missing().missing.lower_bound(current);
+	vector::iterator ls_iter = sentries.begin();
+	hobject_t _max = hobject_t::get_max();
+	while (1) {
+	  const hobject_t &mcand =
+	    missing_iter == pg_log.get_missing().missing.end() ?
+	    _max :
+	    missing_iter->first;
+	  const hobject_t &lcand =
+	    ls_iter == sentries.end() ?
+	    _max :
+	    *ls_iter;
+
+	  hobject_t candidate;
+	  if (mcand == lcand) {
+	    candidate = mcand;
+	    if (!mcand.is_max()) {
+	      ++ls_iter;
+	      ++missing_iter;
+	    }
+	  } else if (mcand < lcand) {
+	    candidate = mcand;
+	    assert(!mcand.is_max());
+	    ++missing_iter;
+	  } else {
+	    candidate = lcand;
+	    assert(!lcand.is_max());
+	    ++ls_iter;
+	  }
+
+	  if (candidate >= next) {
+	    break;
+	  }
+	    
+	  if (response.entries.size() == list_size) {
+	    next = candidate;
+	    break;
+	  }
+
+	  // skip snapdir objects
+	  if (candidate.snap == CEPH_SNAPDIR)
+	    continue;
+
+	  if (candidate.snap < snapid)
+	    continue;
+
+	  if (snapid != CEPH_NOSNAP) {
+	    bufferlist bl;
+	    if (candidate.snap == CEPH_NOSNAP) {
+	      pgbackend->objects_get_attr(
+		candidate,
+		SS_ATTR,
+		&bl);
+	      SnapSet snapset(bl);
+	      if (snapid <= snapset.seq)
+		continue;
+	    } else {
+	      bufferlist attr_bl;
+	      pgbackend->objects_get_attr(
+		candidate, OI_ATTR, &attr_bl);
+	      object_info_t oi(attr_bl);
+	      vector::iterator i = find(oi.snaps.begin(),
+						  oi.snaps.end(),
+						  snapid);
+	      if (i == oi.snaps.end())
+		continue;
+	    }
+	  }
+
+	  // skip wrong namespace
+	  if (candidate.get_namespace() != m->get_object_locator().nspace)
+	    continue;
+
+	  if (filter && !pgls_filter(filter, candidate, filter_out))
+	    continue;
+
+	  response.entries.push_back(make_pair(candidate.oid,
+					       candidate.get_key()));
+	}
+	if (next.is_max() &&
+	    missing_iter == pg_log.get_missing().missing.end() &&
+	    ls_iter == sentries.end()) {
+	  result = 1;
+	}
+	response.handle = next;
+	::encode(response, osd_op.outdata);
+	if (filter)
+	  ::encode(filter_out, osd_op.outdata);
+	dout(10) << " pgls result=" << result << " outdata.length()="
+		 << osd_op.outdata.length() << dendl;
+      }
+      break;
+
+    case CEPH_OSD_OP_PG_HITSET_LS:
+      {
+	list< pair > ls;
+	for (list::const_iterator p = info.hit_set.history.begin();
+	     p != info.hit_set.history.end();
+	     ++p)
+	  ls.push_back(make_pair(p->begin, p->end));
+	if (info.hit_set.current_info.begin)
+	  ls.push_back(make_pair(info.hit_set.current_info.begin, utime_t()));
+	else if (hit_set)
+	  ls.push_back(make_pair(hit_set_start_stamp, utime_t()));
+	::encode(ls, osd_op.outdata);
+      }
+      break;
+
+    case CEPH_OSD_OP_PG_HITSET_GET:
+      {
+	utime_t stamp(osd_op.op.hit_set_get.stamp);
+	if ((info.hit_set.current_info.begin &&
+	     stamp >= info.hit_set.current_info.begin) ||
+	    stamp >= hit_set_start_stamp) {
+	  // read the current in-memory HitSet, not the version we've
+	  // checkpointed.
+	  if (!hit_set) {
+	    result= -ENOENT;
+	    break;
+	  }
+	  ::encode(*hit_set, osd_op.outdata);
+	  result = osd_op.outdata.length();
+	} else {
+	  // read an archived HitSet.
+	  hobject_t oid;
+	  for (list::const_iterator p = info.hit_set.history.begin();
+	       p != info.hit_set.history.end();
+	       ++p) {
+	    if (stamp >= p->begin && stamp <= p->end) {
+	      oid = get_hit_set_archive_object(p->begin, p->end);
+	      break;
+	    }
+	  }
+	  if (oid == hobject_t()) {
+	    result = -ENOENT;
+	    break;
+	  }
+	  if (!pool.info.is_replicated()) {
+	    // FIXME: EC not supported yet
+	    result = -EOPNOTSUPP;
+	    break;
+	  }
+	  if (is_unreadable_object(oid)) {
+	    wait_for_unreadable_object(oid, op);
+	    return;
+	  }
+	  result = osd->store->read(coll, oid, 0, 0, osd_op.outdata);
+	}
+      }
+      break;
+
+
+    default:
+      result = -EINVAL;
+      break;
+    }
+  }
+
+  // reply
+  MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(),
+				       CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK,
+				       false);
+  reply->claim_op_out_data(ops);
+  reply->set_result(result);
+  reply->set_reply_versions(info.last_update, info.last_user_version);
+  osd->send_message_osd_client(reply, m->get_connection());
+  delete filter;
+}
+
+void ReplicatedPG::calc_trim_to()
+{
+  if (is_scrubbing() && scrubber.classic) {
+    dout(10) << "calc_trim_to no trim during classic scrub" << dendl;
+    pg_trim_to = eversion_t();
+    return;
+  }
+
+  size_t target = cct->_conf->osd_min_pg_log_entries;
+  if (is_degraded() ||
+      state_test(PG_STATE_RECOVERING |
+		 PG_STATE_RECOVERY_WAIT |
+		 PG_STATE_BACKFILL |
+		 PG_STATE_BACKFILL_WAIT |
+		 PG_STATE_BACKFILL_TOOFULL)) {
+    target = cct->_conf->osd_max_pg_log_entries;
+  }
+
+  if (min_last_complete_ondisk != eversion_t() &&
+      min_last_complete_ondisk != pg_trim_to &&
+      pg_log.get_log().approx_size() > target) {
+    size_t num_to_trim = pg_log.get_log().approx_size() - target;
+    list::const_iterator it = pg_log.get_log().log.begin();
+    eversion_t new_trim_to;
+    for (size_t i = 0; i < num_to_trim; ++i) {
+      new_trim_to = it->version;
+      ++it;
+      if (new_trim_to > min_last_complete_ondisk) {
+	new_trim_to = min_last_complete_ondisk;
+	dout(10) << "calc_trim_to trimming to min_last_complete_ondisk" << dendl;
+	break;
+      }
+    }
+    dout(10) << "calc_trim_to " << pg_trim_to << " -> " << new_trim_to << dendl;
+    pg_trim_to = new_trim_to;
+    assert(pg_trim_to <= pg_log.get_head());
+    assert(pg_trim_to <= min_last_complete_ondisk);
+  }
+}
+
+ReplicatedPG::ReplicatedPG(OSDService *o, OSDMapRef curmap,
+			   const PGPool &_pool, spg_t p, const hobject_t& oid,
+			   const hobject_t& ioid) :
+  PG(o, curmap, _pool, p, oid, ioid),
+  pgbackend(
+    PGBackend::build_pg_backend(
+      _pool.info, curmap, this, coll_t(p), coll_t::make_temp_coll(p), o->store, cct)),
+  snapset_contexts_lock("ReplicatedPG::snapset_contexts"),
+  temp_seq(0),
+  snap_trimmer_machine(this)
+{ 
+  missing_loc.set_backend_predicates(
+    pgbackend->get_is_readable_predicate(),
+    pgbackend->get_is_recoverable_predicate());
+  snap_trimmer_machine.initiate();
+}
+
+void ReplicatedPG::get_src_oloc(const object_t& oid, const object_locator_t& oloc, object_locator_t& src_oloc)
+{
+  src_oloc = oloc;
+  if (oloc.key.empty())
+    src_oloc.key = oid.name;
+}
+
+void ReplicatedPG::do_request(
+  OpRequestRef op,
+  ThreadPool::TPHandle &handle)
+{
+  if (!op_has_sufficient_caps(op)) {
+    osd->reply_op_error(op, -EPERM);
+    return;
+  }
+  assert(!op_must_wait_for_map(get_osdmap(), op));
+  if (can_discard_request(op)) {
+    return;
+  }
+  if (flushes_in_progress > 0) {
+    dout(20) << flushes_in_progress
+	     << " flushes_in_progress pending "
+	     << "waiting for active on " << op << dendl;
+    waiting_for_active.push_back(op);
+    return;
+  }
+
+  if (!is_active()) {
+    // Delay unless PGBackend says it's ok
+    if (pgbackend->can_handle_while_inactive(op)) {
+      bool handled = pgbackend->handle_message(op);
+      assert(handled);
+      return;
+    } else {
+      waiting_for_active.push_back(op);
+      return;
+    }
+  }
+
+  assert(is_active() && flushes_in_progress == 0);
+  if (pgbackend->handle_message(op))
+    return;
+
+  switch (op->get_req()->get_type()) {
+  case CEPH_MSG_OSD_OP:
+    if (is_replay()) {
+      dout(20) << " replay, waiting for active on " << op << dendl;
+      waiting_for_active.push_back(op);
+      return;
+    }
+    // verify client features
+    if ((pool.info.has_tiers() || pool.info.is_tier()) &&
+	!op->has_feature(CEPH_FEATURE_OSD_CACHEPOOL)) {
+      osd->reply_op_error(op, -EOPNOTSUPP);
+      return;
+    }
+    do_op(op); // do it now
+    break;
+
+  case MSG_OSD_SUBOP:
+    do_sub_op(op);
+    break;
+
+  case MSG_OSD_SUBOPREPLY:
+    do_sub_op_reply(op);
+    break;
+
+  case MSG_OSD_PG_SCAN:
+    do_scan(op, handle);
+    break;
+
+  case MSG_OSD_PG_BACKFILL:
+    do_backfill(op);
+    break;
+
+  default:
+    assert(0 == "bad message type in do_request");
+  }
+}
+
+hobject_t ReplicatedPG::earliest_backfill() const
+{
+  hobject_t e = hobject_t::get_max();
+  for (set::iterator i = backfill_targets.begin();
+       i != backfill_targets.end();
+       ++i) {
+    pg_shard_t bt = *i;
+    map::const_iterator iter = peer_info.find(bt);
+    assert(iter != peer_info.end());
+    if (iter->second.last_backfill < e)
+      e = iter->second.last_backfill;
+  }
+  return e;
+}
+
+// if we have src_oids, we need to be careful of the target being
+// before and a src being after the last_backfill line, or else the
+// operation won't apply properly on the backfill_target.  (the
+// opposite is not a problem; if the target is after the line, we
+// don't apply on the backfill_target and it doesn't matter.)
+// With multi-backfill some backfill targets can be ahead of
+// last_backfill_started.  We consider each replica individually and
+// take the larger of last_backfill_started and the replicas last_backfill.
+bool ReplicatedPG::check_src_targ(const hobject_t& soid, const hobject_t& toid) const
+{
+  for (set::iterator i = actingbackfill.begin();
+       i != actingbackfill.end();
+       ++i) {
+    if (*i == get_primary()) continue;
+    pg_shard_t bt = *i;
+    map::const_iterator iter = peer_info.find(bt);
+    assert(iter != peer_info.end());
+
+    if (toid <= MAX(last_backfill_started, iter->second.last_backfill) &&
+	soid > MAX(last_backfill_started, iter->second.last_backfill))
+      return true;
+  }
+  return false;
+}
+
+/** do_op - do an op
+ * pg lock will be held (if multithreaded)
+ * osd_lock NOT held.
+ */
+void ReplicatedPG::do_op(OpRequestRef op)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == CEPH_MSG_OSD_OP);
+  if (op->includes_pg_op()) {
+    if (pg_op_must_wait(m)) {
+      wait_for_all_missing(op);
+      return;
+    }
+    return do_pg_op(op);
+  }
+
+  if (get_osdmap()->is_blacklisted(m->get_source_addr())) {
+    dout(10) << "do_op " << m->get_source_addr() << " is blacklisted" << dendl;
+    osd->reply_op_error(op, -EBLACKLISTED);
+    return;
+  }
+
+  // order this op as a write?
+  bool write_ordered =
+    op->may_write() ||
+    op->may_cache() ||
+    (m->get_flags() & CEPH_OSD_FLAG_RWORDERED);
+
+  dout(10) << "do_op " << *m
+	   << (op->may_write() ? " may_write" : "")
+	   << (op->may_read() ? " may_read" : "")
+	   << (op->may_cache() ? " may_cache" : "")
+	   << " -> " << (write_ordered ? "write-ordered" : "read-ordered")
+	   << " flags " << ceph_osd_flag_string(m->get_flags())
+	   << dendl;
+
+  hobject_t head(m->get_oid(), m->get_object_locator().key,
+		 CEPH_NOSNAP, m->get_pg().ps(),
+		 info.pgid.pool(), m->get_object_locator().nspace);
+
+
+  if (write_ordered && scrubber.write_blocked_by_scrub(head)) {
+    dout(20) << __func__ << ": waiting for scrub" << dendl;
+    waiting_for_active.push_back(op);
+    op->mark_delayed("waiting for scrub");
+    return;
+  }
+
+  // missing object?
+  if (is_unreadable_object(head)) {
+    wait_for_unreadable_object(head, op);
+    return;
+  }
+
+  // degraded object?
+  if (write_ordered && is_degraded_object(head)) {
+    wait_for_degraded_object(head, op);
+    return;
+  }
+
+  // missing snapdir?
+  hobject_t snapdir(m->get_oid(), m->get_object_locator().key,
+		    CEPH_SNAPDIR, m->get_pg().ps(), info.pgid.pool(),
+		    m->get_object_locator().nspace);
+  if (is_unreadable_object(snapdir)) {
+    wait_for_unreadable_object(snapdir, op);
+    return;
+  }
+
+  // degraded object?
+  if (write_ordered && is_degraded_object(snapdir)) {
+    wait_for_degraded_object(snapdir, op);
+    return;
+  }
+ 
+  // asking for SNAPDIR is only ok for reads
+  if (m->get_snapid() == CEPH_SNAPDIR && op->may_write()) {
+    osd->reply_op_error(op, -EINVAL);
+    return;
+  }
+
+  // dup/replay?
+  if (op->may_write() || op->may_cache()) {
+    const pg_log_entry_t *entry = pg_log.get_log().get_request(m->get_reqid());
+    if (entry) {
+      const eversion_t& oldv = entry->version;
+      dout(3) << __func__ << " dup " << m->get_reqid()
+	      << " was " << oldv << dendl;
+      if (already_complete(oldv)) {
+	osd->reply_op_error(op, 0, oldv, entry->user_version);
+      } else {
+	if (m->wants_ack()) {
+	  if (already_ack(oldv)) {
+	    MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, false);
+	    reply->add_flags(CEPH_OSD_FLAG_ACK);
+	    reply->set_reply_versions(oldv, entry->user_version);
+	    osd->send_message_osd_client(reply, m->get_connection());
+	  } else {
+	    dout(10) << " waiting for " << oldv << " to ack" << dendl;
+	    waiting_for_ack[oldv].push_back(op);
+	  }
+	}
+	dout(10) << " waiting for " << oldv << " to commit" << dendl;
+	waiting_for_ondisk[oldv].push_back(op);  // always queue ondisk waiters, so that we can requeue if needed
+	op->mark_delayed("waiting for ondisk");
+      }
+      return;
+    }
+  }
+
+  ObjectContextRef obc;
+  bool can_create = op->may_write() || op->may_cache();
+  hobject_t missing_oid;
+  hobject_t oid(m->get_oid(),
+		m->get_object_locator().key,
+		m->get_snapid(),
+		m->get_pg().ps(),
+		m->get_object_locator().get_pool(),
+		m->get_object_locator().nspace);
+
+  // io blocked on obc?
+  if (((m->get_flags() & CEPH_OSD_FLAG_FLUSH) == 0) &&
+      maybe_await_blocked_snapset(oid, op)) {
+    return;
+  }
+
+  int r = find_object_context(
+    oid, &obc, can_create,
+    m->get_flags() & CEPH_OSD_FLAG_MAP_SNAP_CLONE,
+    &missing_oid);
+
+  if (r == -EAGAIN) {
+    // If we're not the primary of this OSD, and we have
+    // CEPH_OSD_FLAG_LOCALIZE_READS set, we just return -EAGAIN. Otherwise,
+    // we have to wait for the object.
+    if (is_primary() ||
+	(!(m->get_flags() & CEPH_OSD_FLAG_BALANCE_READS) &&
+	 !(m->get_flags() & CEPH_OSD_FLAG_LOCALIZE_READS))) {
+      // missing the specific snap we need; requeue and wait.
+      assert(!op->may_write()); // only happens on a read/cache
+      wait_for_unreadable_object(missing_oid, op);
+      return;
+    }
+  } else if (r == 0) {
+    if (is_unreadable_object(obc->obs.oi.soid)) {
+      dout(10) << __func__ << ": clone " << obc->obs.oi.soid
+	       << " is unreadable, waiting" << dendl;
+      wait_for_unreadable_object(obc->obs.oi.soid, op);
+      return;
+    }
+
+    // degraded object?  (the check above was for head; this could be a clone)
+    if (write_ordered &&
+	obc->obs.oi.soid.snap != CEPH_NOSNAP &&
+	is_degraded_object(obc->obs.oi.soid)) {
+      dout(10) << __func__ << ": clone " << obc->obs.oi.soid
+	       << " is degraded, waiting" << dendl;
+      wait_for_degraded_object(obc->obs.oi.soid, op);
+      return;
+    }
+  }
+
+  if (hit_set) {
+    hit_set->insert(oid);
+    if (hit_set->is_full() ||
+	hit_set_start_stamp + pool.info.hit_set_period <= m->get_recv_stamp()) {
+      hit_set_persist();
+    }
+  }
+
+  if (agent_state) {
+    agent_choose_mode();
+  }
+
+  if ((m->get_flags() & CEPH_OSD_FLAG_IGNORE_CACHE) == 0 &&
+      maybe_handle_cache(op, write_ordered, obc, r, missing_oid, false))
+    return;
+
+  if (r) {
+    osd->reply_op_error(op, r);
+    return;
+  }
+
+  // make sure locator is consistent
+  object_locator_t oloc(obc->obs.oi.soid);
+  if (m->get_object_locator() != oloc) {
+    dout(10) << " provided locator " << m->get_object_locator() 
+	     << " != object's " << obc->obs.oi.soid << dendl;
+    osd->clog.warn() << "bad locator " << m->get_object_locator() 
+		     << " on object " << oloc
+		     << " op " << *m << "\n";
+  }
+
+  // io blocked on obc?
+  if (obc->is_blocked() &&
+      (m->get_flags() & CEPH_OSD_FLAG_FLUSH) == 0) {
+    wait_for_blocked_object(obc->obs.oi.soid, op);
+    return;
+  }
+
+  dout(25) << __func__ << " oi " << obc->obs.oi << dendl;
+
+  // are writes blocked by another object?
+  if (obc->blocked_by) {
+    dout(10) << "do_op writes for " << obc->obs.oi.soid << " blocked by "
+	     << obc->blocked_by->obs.oi.soid << dendl;
+    wait_for_degraded_object(obc->blocked_by->obs.oi.soid, op);
+    return;
+  }
+
+  // src_oids
+  map src_obc;
+  for (vector::iterator p = m->ops.begin(); p != m->ops.end(); ++p) {
+    OSDOp& osd_op = *p;
+
+    // make sure LIST_SNAPS is on CEPH_SNAPDIR and nothing else
+    if (osd_op.op.op == CEPH_OSD_OP_LIST_SNAPS &&
+	m->get_snapid() != CEPH_SNAPDIR) {
+      dout(10) << "LIST_SNAPS with incorrect context" << dendl;
+      osd->reply_op_error(op, -EINVAL);
+      return;
+    }
+
+    if (!ceph_osd_op_type_multi(osd_op.op.op))
+      continue;
+    if (osd_op.soid.oid.name.length()) {
+      object_locator_t src_oloc;
+      get_src_oloc(m->get_oid(), m->get_object_locator(), src_oloc);
+      hobject_t src_oid(osd_op.soid, src_oloc.key, m->get_pg().ps(),
+			info.pgid.pool(), m->get_object_locator().nspace);
+      if (!src_obc.count(src_oid)) {
+	ObjectContextRef sobc;
+	hobject_t wait_oid;
+	int r;
+
+	if (src_oid.is_head() && is_missing_object(src_oid)) {
+	  wait_for_unreadable_object(src_oid, op);
+	} else if ((r = find_object_context(
+		      src_oid, &sobc, false, false,
+		      &wait_oid)) == -EAGAIN) {
+	  // missing the specific snap we need; requeue and wait.
+	  wait_for_unreadable_object(wait_oid, op);
+	} else if (r) {
+	  if (!maybe_handle_cache(op, write_ordered, sobc, r, wait_oid, true))
+	    osd->reply_op_error(op, r);
+	} else if (sobc->obs.oi.is_whiteout()) {
+	  osd->reply_op_error(op, -ENOENT);
+	} else {
+	  if (sobc->obs.oi.soid.get_key() != obc->obs.oi.soid.get_key() &&
+		   sobc->obs.oi.soid.get_key() != obc->obs.oi.soid.oid.name &&
+		   sobc->obs.oi.soid.oid.name != obc->obs.oi.soid.get_key()) {
+	    dout(1) << " src_oid " << sobc->obs.oi.soid << " != "
+		  << obc->obs.oi.soid << dendl;
+	    osd->reply_op_error(op, -EINVAL);
+	  } else if (is_degraded_object(sobc->obs.oi.soid) ||
+		   (check_src_targ(sobc->obs.oi.soid, obc->obs.oi.soid))) {
+	    if (is_degraded_object(sobc->obs.oi.soid)) {
+	      wait_for_degraded_object(sobc->obs.oi.soid, op);
+	    } else {
+	      waiting_for_degraded_object[sobc->obs.oi.soid].push_back(op);
+	      op->mark_delayed("waiting for degraded object");
+	    }
+	    dout(10) << " writes for " << obc->obs.oi.soid << " now blocked by "
+		     << sobc->obs.oi.soid << dendl;
+	    obc->blocked_by = sobc;
+	    sobc->blocking.insert(obc);
+	  } else {
+	    dout(10) << " src_oid " << src_oid << " obc " << src_obc << dendl;
+	    src_obc[src_oid] = sobc;
+	    continue;
+	  }
+	}
+	// Error cleanup below
+      } else {
+	continue;
+      }
+      // Error cleanup below
+    } else {
+      dout(10) << "no src oid specified for multi op " << osd_op << dendl;
+      osd->reply_op_error(op, -EINVAL);
+    }
+    return;
+  }
+
+  // any SNAPDIR op needs to have all clones present.  treat them as
+  // src_obc's so that we track references properly and clean up later.
+  if (m->get_snapid() == CEPH_SNAPDIR) {
+    for (vector::iterator p = obc->ssc->snapset.clones.begin();
+	 p != obc->ssc->snapset.clones.end();
+	 ++p) {
+      hobject_t clone_oid = obc->obs.oi.soid;
+      clone_oid.snap = *p;
+      if (!src_obc.count(clone_oid)) {
+	if (is_unreadable_object(clone_oid)) {
+	  wait_for_unreadable_object(clone_oid, op);
+	  return;
+	}
+
+	ObjectContextRef sobc = get_object_context(clone_oid, false);
+	if (!sobc) {
+	  if (!maybe_handle_cache(op, write_ordered, sobc, -ENOENT, clone_oid, true))
+	    osd->reply_op_error(op, -ENOENT);
+	  return;
+	} else {
+	  dout(10) << " clone_oid " << clone_oid << " obc " << sobc << dendl;
+	  src_obc[clone_oid] = sobc;
+	  continue;
+	}
+	assert(0); // unreachable
+      } else {
+	continue;
+      }
+    }
+  }
+
+  OpContext *ctx = new OpContext(op, m->get_reqid(), m->ops,
+				 &obc->obs, obc->ssc, 
+				 this);
+  ctx->op_t = pgbackend->get_transaction();
+  ctx->obc = obc;
+
+  if (!obc->obs.exists)
+    ctx->snapset_obc = get_object_context(obc->obs.oi.soid.get_snapdir(), false);
+
+  if (m->get_flags() & CEPH_OSD_FLAG_SKIPRWLOCKS) {
+    dout(20) << __func__ << ": skipping rw locks" << dendl;
+  } else if (m->get_flags() & CEPH_OSD_FLAG_FLUSH) {
+    dout(20) << __func__ << ": part of flush, will ignore write lock" << dendl;
+
+    // verify there is in fact a flush in progress
+    // FIXME: we could make this a stronger test.
+    map::iterator p = flush_ops.find(obc->obs.oi.soid);
+    if (p == flush_ops.end()) {
+      dout(10) << __func__ << " no flush in progress, aborting" << dendl;
+      reply_ctx(ctx, -EINVAL);
+      return;
+    }
+  } else if (!get_rw_locks(ctx)) {
+    dout(20) << __func__ << " waiting for rw locks " << dendl;
+    op->mark_delayed("waiting for rw locks");
+    close_op_ctx(ctx, -EBUSY);
+    return;
+  }
+
+  if ((op->may_read()) && (obc->obs.oi.is_lost())) {
+    // This object is lost. Reading from it returns an error.
+    dout(20) << __func__ << ": object " << obc->obs.oi.soid
+	     << " is lost" << dendl;
+    reply_ctx(ctx, -ENFILE);
+    return;
+  }
+  if (!op->may_write() &&
+      !op->may_cache() &&
+      (!obc->obs.exists ||
+       ((m->get_snapid() != CEPH_SNAPDIR) &&
+	obc->obs.oi.is_whiteout()))) {
+    reply_ctx(ctx, -ENOENT);
+    return;
+  }
+
+  op->mark_started();
+  ctx->src_obc = src_obc;
+
+  execute_ctx(ctx);
+}
+
+bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
+				      bool write_ordered,
+				      ObjectContextRef obc,
+                                      int r, const hobject_t& missing_oid,
+				      bool must_promote)
+{
+  if (obc)
+    dout(25) << __func__ << " " << obc->obs.oi << " "
+	     << (obc->obs.exists ? "exists" : "DNE")
+	     << " missing_oid " << missing_oid
+	     << dendl;
+  else
+    dout(25) << __func__ << " (no obc)"
+	     << " missing_oid " << missing_oid
+	     << dendl;
+
+  if (obc.get() && obc->is_blocked()) {
+    // we're already doing something with this object
+    dout(20) << __func__ << " blocked on " << obc->obs.oi.soid << dendl;
+    return false;
+  }
+
+  if (r == -ENOENT && missing_oid == hobject_t()) {
+    // we know this object is logically absent (e.g., an undefined clone)
+    return false;
+  }
+
+  switch (pool.info.cache_mode) {
+  case pg_pool_t::CACHEMODE_NONE:
+    return false;
+
+  case pg_pool_t::CACHEMODE_WRITEBACK:
+    if (obc.get() && obc->obs.exists) {
+      return false;
+    }
+    if (agent_state &&
+	agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
+      if (!op->may_write() && !op->may_cache() && !write_ordered) {
+	dout(20) << __func__ << " cache pool full, redirecting read" << dendl;
+	do_cache_redirect(op, obc);
+	return true;
+      }
+      dout(20) << __func__ << " cache pool full, waiting" << dendl;
+      waiting_for_cache_not_full.push_back(op);
+      return true;
+    }
+    if (!must_promote && can_skip_promote(op, obc)) {
+      return false;
+    }
+    promote_object(op, obc, missing_oid);
+    return true;
+
+  case pg_pool_t::CACHEMODE_FORWARD:
+    if (obc.get() && obc->obs.exists) {
+      return false;
+    }
+    if (must_promote)
+      promote_object(op, obc, missing_oid);
+    else
+      do_cache_redirect(op, obc);
+    return true;
+
+  case pg_pool_t::CACHEMODE_READONLY:
+    // TODO: clean this case up
+    if (obc.get() && obc->obs.exists) {
+      return false;
+    }
+    if (!obc.get() && r == -ENOENT) {
+      // we don't have the object and op's a read
+      promote_object(op, obc, missing_oid);
+      return true;
+    }
+    if (!r) { // it must be a write
+      do_cache_redirect(op, obc);
+      return true;
+    }
+    // crap, there was a failure of some kind
+    return false;
+
+  default:
+    assert(0 == "unrecognized cache_mode");
+  }
+  return false;
+}
+
+bool ReplicatedPG::can_skip_promote(OpRequestRef op, ObjectContextRef obc)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  if (m->ops.empty())
+    return false;
+  // if we get a delete with FAILOK we can skip promote.  without
+  // FAILOK we still need to promote (or do something smarter) to
+  // determine whether to return ENOENT or 0.
+  if (m->ops[0].op.op == CEPH_OSD_OP_DELETE &&
+      (m->ops[0].op.flags & CEPH_OSD_OP_FLAG_FAILOK))
+    return true;
+  return false;
+}
+
+void ReplicatedPG::do_cache_redirect(OpRequestRef op, ObjectContextRef obc)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  int flags = m->get_flags() & (CEPH_OSD_FLAG_ACK|CEPH_OSD_FLAG_ONDISK);
+  MOSDOpReply *reply = new MOSDOpReply(m, -ENOENT,
+				       get_osdmap()->get_epoch(), flags, false);
+  request_redirect_t redir(m->get_object_locator(), pool.info.tier_of);
+  reply->set_redirect(redir);
+  dout(10) << "sending redirect to pool " << pool.info.tier_of << " for op "
+	   << op << dendl;
+  m->get_connection()->get_messenger()->send_message(reply, m->get_connection());
+  return;
+}
+
+class PromoteCallback: public ReplicatedPG::CopyCallback {
+  OpRequestRef op;
+  ObjectContextRef obc;
+  ReplicatedPG *pg;
+public:
+  PromoteCallback(OpRequestRef op_, ObjectContextRef obc_,
+		  ReplicatedPG *pg_)
+    : op(op_),
+      obc(obc_),
+      pg(pg_) {}
+
+  virtual void finish(ReplicatedPG::CopyCallbackResults results) {
+    ReplicatedPG::CopyResults *results_data = results.get<1>();
+    int r = results.get<0>();
+    pg->finish_promote(r, op, results_data, obc);
+  }
+};
+
+void ReplicatedPG::promote_object(OpRequestRef op, ObjectContextRef obc,
+				  const hobject_t& missing_oid)
+{
+  MOSDOp *m = static_cast(op->get_req());
+  if (!obc) { // we need to create an ObjectContext
+    assert(missing_oid != hobject_t());
+    obc = get_object_context(missing_oid, true);
+  }
+  dout(10) << __func__ << " " << obc->obs.oi.soid << dendl;
+
+  PromoteCallback *cb = new PromoteCallback(op, obc, this);
+  object_locator_t oloc(m->get_object_locator());
+  oloc.pool = pool.info.tier_of;
+  start_copy(cb, obc, obc->obs.oi.soid, oloc, 0,
+	     CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY |
+	     CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE |
+	     CEPH_OSD_COPY_FROM_FLAG_MAP_SNAP_CLONE,
+	     obc->obs.oi.soid.snap == CEPH_NOSNAP);
+
+  assert(obc->is_blocked());
+  wait_for_blocked_object(obc->obs.oi.soid, op);
+}
+
+void ReplicatedPG::execute_ctx(OpContext *ctx)
+{
+  dout(10) << __func__ << " " << ctx << dendl;
+  ctx->reset_obs(ctx->obc);
+  OpRequestRef op = ctx->op;
+  MOSDOp *m = static_cast(op->get_req());
+  ObjectContextRef obc = ctx->obc;
+  const hobject_t& soid = obc->obs.oi.soid;
+  map& src_obc = ctx->src_obc;
+
+  // this method must be idempotent since we may call it several times
+  // before we finally apply the resulting transaction.
+  delete ctx->op_t;
+  ctx->op_t = pgbackend->get_transaction();
+
+  if (op->may_write() || op->may_cache()) {
+    op->mark_started();
+
+    // snap
+    if (!(m->get_flags() & CEPH_OSD_FLAG_ENFORCE_SNAPC) &&
+	pool.info.is_pool_snaps_mode()) {
+      // use pool's snapc
+      ctx->snapc = pool.snapc;
+    } else {
+      // client specified snapc
+      ctx->snapc.seq = m->get_snap_seq();
+      ctx->snapc.snaps = m->get_snaps();
+    }
+    if ((m->get_flags() & CEPH_OSD_FLAG_ORDERSNAP) &&
+	ctx->snapc.seq < obc->ssc->snapset.seq) {
+      dout(10) << " ORDERSNAP flag set and snapc seq " << ctx->snapc.seq
+	       << " < snapset seq " << obc->ssc->snapset.seq
+	       << " on " << obc->obs.oi.soid << dendl;
+      reply_ctx(ctx, -EOLDSNAPC);
+      return;
+    }
+
+    // version
+    ctx->at_version = get_next_version();
+    ctx->mtime = m->get_mtime();
+
+    dout(10) << "do_op " << soid << " " << ctx->ops
+	     << " ov " << obc->obs.oi.version << " av " << ctx->at_version 
+	     << " snapc " << ctx->snapc
+	     << " snapset " << obc->ssc->snapset
+	     << dendl;  
+  } else {
+    dout(10) << "do_op " << soid << " " << ctx->ops
+	     << " ov " << obc->obs.oi.version
+	     << dendl;  
+  }
+
+  if (!ctx->user_at_version)
+    ctx->user_at_version = obc->obs.oi.user_version;
+  dout(30) << __func__ << " user_at_version " << ctx->user_at_version << dendl;
+
+  // note my stats
+  utime_t now = ceph_clock_now(cct);
+
+  if (op->may_read()) {
+    dout(10) << " taking ondisk_read_lock" << dendl;
+    obc->ondisk_read_lock();
+  }
+  for (map::iterator p = src_obc.begin(); p != src_obc.end(); ++p) {
+    dout(10) << " taking ondisk_read_lock for src " << p->first << dendl;
+    p->second->ondisk_read_lock();
+  }
+
+  int result = prepare_transaction(ctx);
+
+  if (op->may_read()) {
+    dout(10) << " dropping ondisk_read_lock" << dendl;
+    obc->ondisk_read_unlock();
+  }
+  for (map::iterator p = src_obc.begin(); p != src_obc.end(); ++p) {
+    dout(10) << " dropping ondisk_read_lock for src " << p->first << dendl;
+    p->second->ondisk_read_unlock();
+  }
+
+  if (result == -EINPROGRESS) {
+    // come back later.
+    return;
+  }
+
+  if (result == -EAGAIN) {
+    // clean up after the ctx
+    close_op_ctx(ctx, result);
+    return;
+  }
+
+  // check for full
+  if (ctx->delta_stats.num_bytes > 0 &&
+      pool.info.get_flags() & pg_pool_t::FLAG_FULL) {
+    reply_ctx(ctx, -ENOSPC);
+    return;
+  }
+
+  bool successful_write = !ctx->op_t->empty() && op->may_write() && result >= 0;
+  // prepare the reply
+  ctx->reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0,
+			       successful_write);
+
+  // Write operations aren't allowed to return a data payload because
+  // we can't do so reliably. If the client has to resend the request
+  // and it has already been applied, we will return 0 with no
+  // payload.  Non-deterministic behavior is no good.  However, it is
+  // possible to construct an operation that does a read, does a guard
+  // check (e.g., CMPXATTR), and then a write.  Then we either succeed
+  // with the write, or return a CMPXATTR and the read value.
+  if (successful_write) {
+    // write.  normalize the result code.
+    dout(20) << " zeroing write result code " << result << dendl;
+    result = 0;
+  }
+  ctx->reply->set_result(result);
+
+  // read or error?
+  if (ctx->op_t->empty() || result < 0) {
+    if (ctx->pending_async_reads.empty()) {
+      complete_read_ctx(result, ctx);
+    } else {
+      in_progress_async_reads.push_back(make_pair(op, ctx));
+      ctx->start_async_reads(this);
+    }
+    return;
+  }
+
+  ctx->reply->set_reply_versions(ctx->at_version, ctx->user_at_version);
+
+  assert(op->may_write() || op->may_cache());
+
+  // trim log?
+  calc_trim_to();
+
+  // verify that we are doing this in order?
+  if (cct->_conf->osd_debug_op_order && m->get_source().is_client() &&
+      !pool.info.is_tier() && !pool.info.has_tiers()) {
+    map& cm = debug_op_order[obc->obs.oi.soid];
+    ceph_tid_t t = m->get_tid();
+    client_t n = m->get_source().num();
+    map::iterator p = cm.find(n);
+    if (p == cm.end()) {
+      dout(20) << " op order client." << n << " tid " << t << " (first)" << dendl;
+      cm[n] = t;
+    } else {
+      dout(20) << " op order client." << n << " tid " << t << " last was " << p->second << dendl;
+      if (p->second > t) {
+	derr << "bad op order, already applied " << p->second << " > this " << t << dendl;;
+	assert(0 == "out of order op");
+      }
+      p->second = t;
+    }
+  }
+
+  // issue replica writes
+  ceph_tid_t rep_tid = osd->get_tid();
+  RepGather *repop = new_repop(ctx, obc, rep_tid);  // new repop claims our obc, src_obc refs
+  // note: repop now owns ctx AND ctx->op
+
+  repop->src_obc.swap(src_obc); // and src_obc.
+
+  issue_repop(repop, now);
+
+  eval_repop(repop);
+  repop->put();
+}
+
+void ReplicatedPG::reply_ctx(OpContext *ctx, int r)
+{
+  if (ctx->op)
+    osd->reply_op_error(ctx->op, r);
+  close_op_ctx(ctx, r);
+}
+
+void ReplicatedPG::reply_ctx(OpContext *ctx, int r, eversion_t v, version_t uv)
+{
+  if (ctx->op)
+    osd->reply_op_error(ctx->op, r, v, uv);
+  close_op_ctx(ctx, r);
+}
+
+void ReplicatedPG::log_op_stats(OpContext *ctx)
+{
+  OpRequestRef op = ctx->op;
+  MOSDOp *m = static_cast(op->get_req());
+
+  utime_t now = ceph_clock_now(cct);
+  utime_t latency = now;
+  latency -= ctx->op->get_req()->get_recv_stamp();
+  utime_t process_latency = now;
+  process_latency -= ctx->op->get_dequeued_time();
+
+  utime_t rlatency;
+  if (ctx->readable_stamp != utime_t()) {
+    rlatency = ctx->readable_stamp;
+    rlatency -= ctx->op->get_req()->get_recv_stamp();
+  }
+
+  uint64_t inb = ctx->bytes_written;
+  uint64_t outb = ctx->bytes_read;
+
+  osd->logger->inc(l_osd_op);
+
+  osd->logger->inc(l_osd_op_outb, outb);
+  osd->logger->inc(l_osd_op_inb, inb);
+  osd->logger->tinc(l_osd_op_lat, latency);
+  osd->logger->tinc(l_osd_op_process_lat, process_latency);
+
+  if (op->may_read() && op->may_write()) {
+    osd->logger->inc(l_osd_op_rw);
+    osd->logger->inc(l_osd_op_rw_inb, inb);
+    osd->logger->inc(l_osd_op_rw_outb, outb);
+    osd->logger->tinc(l_osd_op_rw_rlat, rlatency);
+    osd->logger->tinc(l_osd_op_rw_lat, latency);
+    osd->logger->tinc(l_osd_op_rw_process_lat, process_latency);
+  } else if (op->may_read()) {
+    osd->logger->inc(l_osd_op_r);
+    osd->logger->inc(l_osd_op_r_outb, outb);
+    osd->logger->tinc(l_osd_op_r_lat, latency);
+    osd->logger->tinc(l_osd_op_r_process_lat, process_latency);
+  } else if (op->may_write() || op->may_cache()) {
+    osd->logger->inc(l_osd_op_w);
+    osd->logger->inc(l_osd_op_w_inb, inb);
+    osd->logger->tinc(l_osd_op_w_rlat, rlatency);
+    osd->logger->tinc(l_osd_op_w_lat, latency);
+    osd->logger->tinc(l_osd_op_w_process_lat, process_latency);
+  } else
+    assert(0);
+
+  dout(15) << "log_op_stats " << *m
+	   << " inb " << inb
+	   << " outb " << outb
+	   << " rlat " << rlatency
+	   << " lat " << latency << dendl;
+}
+
+void ReplicatedPG::do_sub_op(OpRequestRef op)
+{
+  MOSDSubOp *m = static_cast(op->get_req());
+  assert(have_same_or_newer_map(m->map_epoch));
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+  dout(15) << "do_sub_op " << *op->get_req() << dendl;
+
+  OSDOp *first = NULL;
+  if (m->ops.size() >= 1) {
+    first = &m->ops[0];
+  }
+
+  if (!is_active()) {
+    waiting_for_active.push_back(op);
+    op->mark_delayed("waiting for active");
+    return;
+  }
+
+  if (first) {
+    switch (first->op.op) {
+    case CEPH_OSD_OP_DELETE:
+      sub_op_remove(op);
+      return;
+    case CEPH_OSD_OP_SCRUB_RESERVE:
+      sub_op_scrub_reserve(op);
+      return;
+    case CEPH_OSD_OP_SCRUB_UNRESERVE:
+      sub_op_scrub_unreserve(op);
+      return;
+    case CEPH_OSD_OP_SCRUB_STOP:
+      sub_op_scrub_stop(op);
+      return;
+    case CEPH_OSD_OP_SCRUB_MAP:
+      sub_op_scrub_map(op);
+      return;
+    }
+  }
+}
+
+void ReplicatedPG::do_sub_op_reply(OpRequestRef op)
+{
+  MOSDSubOpReply *r = static_cast(op->get_req());
+  assert(r->get_header().type == MSG_OSD_SUBOPREPLY);
+  if (r->ops.size() >= 1) {
+    OSDOp& first = r->ops[0];
+    switch (first.op.op) {
+    case CEPH_OSD_OP_SCRUB_RESERVE:
+      sub_op_scrub_reserve_reply(op);
+      return;
+    }
+  }
+}
+
+void ReplicatedPG::do_scan(
+  OpRequestRef op,
+  ThreadPool::TPHandle &handle)
+{
+  MOSDPGScan *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_SCAN);
+  dout(10) << "do_scan " << *m << dendl;
+
+  op->mark_started();
+
+  switch (m->op) {
+  case MOSDPGScan::OP_SCAN_GET_DIGEST:
+    {
+      double ratio, full_ratio;
+      if (osd->too_full_for_backfill(&ratio, &full_ratio)) {
+	dout(1) << __func__ << ": Canceling backfill, current usage is "
+		<< ratio << ", which exceeds " << full_ratio << dendl;
+	queue_peering_event(
+	  CephPeeringEvtRef(
+	    new CephPeeringEvt(
+	      get_osdmap()->get_epoch(),
+	      get_osdmap()->get_epoch(),
+	      BackfillTooFull())));
+	return;
+      }
+
+      BackfillInterval bi;
+      bi.begin = m->begin;
+      // No need to flush, there won't be any in progress writes occuring
+      // past m->begin
+      scan_range(
+	cct->_conf->osd_backfill_scan_min,
+	cct->_conf->osd_backfill_scan_max,
+	&bi,
+	handle);
+      MOSDPGScan *reply = new MOSDPGScan(
+	MOSDPGScan::OP_SCAN_DIGEST,
+	pg_whoami,
+	get_osdmap()->get_epoch(), m->query_epoch,
+	spg_t(info.pgid.pgid, get_primary().shard), bi.begin, bi.end);
+      ::encode(bi.objects, reply->get_data());
+      osd->send_message_osd_cluster(reply, m->get_connection());
+    }
+    break;
+
+  case MOSDPGScan::OP_SCAN_DIGEST:
+    {
+      pg_shard_t from = m->from;
+
+      // Check that from is in backfill_targets vector
+      assert(is_backfill_targets(from));
+
+      BackfillInterval bi;
+      bi.begin = m->begin;
+      bi.end = m->end;
+      bufferlist::iterator p = m->get_data().begin();
+      ::decode(bi.objects, p);
+
+      // handle hobject_t encoding change
+      if (bi.objects.size() && bi.objects.begin()->first.pool == -1) {
+	map tmp;
+	tmp.swap(bi.objects);
+	for (map::iterator i = tmp.begin();
+	     i != tmp.end();
+	     ++i) {
+	  hobject_t first(i->first);
+	  if (!first.is_max() && first.pool == -1)
+	    first.pool = info.pgid.pool();
+	  bi.objects[first] = i->second;
+	}
+      }
+      peer_backfill_info[from] = bi;
+
+      assert(waiting_on_backfill.find(from) != waiting_on_backfill.end());
+      waiting_on_backfill.erase(from);
+
+      if (waiting_on_backfill.empty()) {
+        assert(peer_backfill_info.size() == backfill_targets.size());
+        finish_recovery_op(hobject_t::get_max());
+      }
+    }
+    break;
+  }
+}
+
+void ReplicatedBackend::_do_push(OpRequestRef op)
+{
+  MOSDPGPush *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_PUSH);
+  pg_shard_t from = m->from;
+
+  vector replies;
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  for (vector::iterator i = m->pushes.begin();
+       i != m->pushes.end();
+       ++i) {
+    replies.push_back(PushReplyOp());
+    handle_push(from, *i, &(replies.back()), t);
+  }
+
+  MOSDPGPushReply *reply = new MOSDPGPushReply;
+  reply->from = get_parent()->whoami_shard();
+  reply->set_priority(m->get_priority());
+  reply->pgid = get_info().pgid;
+  reply->map_epoch = m->map_epoch;
+  reply->replies.swap(replies);
+  reply->compute_cost(cct);
+
+  t->register_on_complete(
+    new PG_SendMessageOnConn(
+      get_parent(), reply, m->get_connection()));
+
+  t->register_on_applied(
+    new ObjectStore::C_DeleteTransaction(t));
+  get_parent()->queue_transaction(t);
+}
+
+struct C_ReplicatedBackend_OnPullComplete : GenContext {
+  ReplicatedBackend *bc;
+  list to_continue;
+  int priority;
+  C_ReplicatedBackend_OnPullComplete(ReplicatedBackend *bc, int priority)
+    : bc(bc), priority(priority) {}
+
+  void finish(ThreadPool::TPHandle &handle) {
+    ReplicatedBackend::RPGHandle *h = bc->_open_recovery_op();
+    for (list::iterator i =
+	   to_continue.begin();
+	 i != to_continue.end();
+	 ++i) {
+      map::iterator j =
+	bc->pulling.find(*i);
+      assert(j != bc->pulling.end());
+      if (!bc->start_pushes(*i, j->second.obc, h)) {
+	bc->get_parent()->on_global_recover(
+	  *i);
+      }
+      bc->pulling.erase(*i);
+      handle.reset_tp_timeout();
+    }
+    bc->run_recovery_op(h, priority);
+  }
+};
+
+void ReplicatedBackend::_do_pull_response(OpRequestRef op)
+{
+  MOSDPGPush *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_PUSH);
+  pg_shard_t from = m->from;
+
+  vector replies(1);
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  list to_continue;
+  for (vector::iterator i = m->pushes.begin();
+       i != m->pushes.end();
+       ++i) {
+    bool more = handle_pull_response(from, *i, &(replies.back()), &to_continue, t);
+    if (more)
+      replies.push_back(PullOp());
+  }
+  if (!to_continue.empty()) {
+    C_ReplicatedBackend_OnPullComplete *c =
+      new C_ReplicatedBackend_OnPullComplete(
+	this,
+	m->get_priority());
+    c->to_continue.swap(to_continue);
+    t->register_on_complete(
+      new PG_QueueAsync(
+	get_parent(),
+	get_parent()->bless_gencontext(c)));
+  }
+  replies.erase(replies.end() - 1);
+
+  if (replies.size()) {
+    MOSDPGPull *reply = new MOSDPGPull;
+    reply->from = parent->whoami_shard();
+    reply->set_priority(m->get_priority());
+    reply->pgid = get_info().pgid;
+    reply->map_epoch = m->map_epoch;
+    reply->pulls.swap(replies);
+    reply->compute_cost(cct);
+
+    t->register_on_complete(
+      new PG_SendMessageOnConn(
+	get_parent(), reply, m->get_connection()));
+  }
+
+  t->register_on_applied(
+    new ObjectStore::C_DeleteTransaction(t));
+  get_parent()->queue_transaction(t);
+}
+
+void ReplicatedBackend::do_pull(OpRequestRef op)
+{
+  MOSDPGPull *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_PULL);
+  pg_shard_t from = m->from;
+
+  map > replies;
+  for (vector::iterator i = m->pulls.begin();
+       i != m->pulls.end();
+       ++i) {
+    replies[from].push_back(PushOp());
+    handle_pull(from, *i, &(replies[from].back()));
+  }
+  send_pushes(m->get_priority(), replies);
+}
+
+void ReplicatedBackend::do_push_reply(OpRequestRef op)
+{
+  MOSDPGPushReply *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_PUSH_REPLY);
+  pg_shard_t from = m->from;
+
+  vector replies(1);
+  for (vector::iterator i = m->replies.begin();
+       i != m->replies.end();
+       ++i) {
+    bool more = handle_push_reply(from, *i, &(replies.back()));
+    if (more)
+      replies.push_back(PushOp());
+  }
+  replies.erase(replies.end() - 1);
+
+  map > _replies;
+  _replies[from].swap(replies);
+  send_pushes(m->get_priority(), _replies);
+}
+
+void ReplicatedPG::do_backfill(OpRequestRef op)
+{
+  MOSDPGBackfill *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_PG_BACKFILL);
+  dout(10) << "do_backfill " << *m << dendl;
+
+  op->mark_started();
+
+  switch (m->op) {
+  case MOSDPGBackfill::OP_BACKFILL_FINISH:
+    {
+      assert(cct->_conf->osd_kill_backfill_at != 1);
+
+      MOSDPGBackfill *reply = new MOSDPGBackfill(
+	MOSDPGBackfill::OP_BACKFILL_FINISH_ACK,
+	get_osdmap()->get_epoch(),
+	m->query_epoch,
+	spg_t(info.pgid.pgid, primary.shard));
+      reply->set_priority(cct->_conf->osd_recovery_op_priority);
+      osd->send_message_osd_cluster(reply, m->get_connection());
+      queue_peering_event(
+	CephPeeringEvtRef(
+	  new CephPeeringEvt(
+	    get_osdmap()->get_epoch(),
+	    get_osdmap()->get_epoch(),
+	    RecoveryDone())));
+    }
+    // fall-thru
+
+  case MOSDPGBackfill::OP_BACKFILL_PROGRESS:
+    {
+      assert(cct->_conf->osd_kill_backfill_at != 2);
+
+      info.last_backfill = m->last_backfill;
+      if (m->compat_stat_sum) {
+	info.stats.stats = m->stats.stats; // Previously, we only sent sum
+      } else {
+	info.stats = m->stats;
+      }
+
+      ObjectStore::Transaction *t = new ObjectStore::Transaction;
+      dirty_info = true;
+      write_if_dirty(*t);
+      int tr = osd->store->queue_transaction_and_cleanup(osr.get(), t);
+      assert(tr == 0);
+    }
+    break;
+
+  case MOSDPGBackfill::OP_BACKFILL_FINISH_ACK:
+    {
+      assert(is_primary());
+      assert(cct->_conf->osd_kill_backfill_at != 3);
+      finish_recovery_op(hobject_t::get_max());
+    }
+    break;
+  }
+}
+
+ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid)
+{
+  // load clone info
+  bufferlist bl;
+  ObjectContextRef obc = get_object_context(coid, false, NULL);
+  if (!obc) {
+    derr << __func__ << "could not find coid " << coid << dendl;
+    assert(0);
+  }
+  assert(obc->ssc);
+
+  if (!obc->get_snaptrimmer_write()) {
+    dout(10) << __func__ << ": Unable to get a wlock on " << coid << dendl;
+    return NULL;
+  }
+
+  hobject_t snapoid(
+    coid.oid, coid.get_key(),
+    obc->ssc->snapset.head_exists ? CEPH_NOSNAP:CEPH_SNAPDIR, coid.hash,
+    info.pgid.pool(), coid.get_namespace());
+  ObjectContextRef snapset_obc = get_object_context(snapoid, false);
+
+  if (!snapset_obc->get_snaptrimmer_write()) {
+    dout(10) << __func__ << ": Unable to get a wlock on " << snapoid << dendl;
+    list to_wake;
+    bool requeue_recovery = false;
+    bool requeue_snaptrimmer = false;
+    obc->put_write(&to_wake, &requeue_recovery, &requeue_snaptrimmer);
+    assert(to_wake.empty());
+    assert(!requeue_recovery);
+    return NULL;
+  }
+
+  object_info_t &coi = obc->obs.oi;
+  set old_snaps(coi.snaps.begin(), coi.snaps.end());
+  assert(old_snaps.size());
+
+  SnapSet& snapset = obc->ssc->snapset;
+
+  dout(10) << coid << " old_snaps " << old_snaps
+	   << " old snapset " << snapset << dendl;
+  assert(snapset.seq);
+
+  RepGather *repop = simple_repop_create(obc);
+  OpContext *ctx = repop->ctx;
+  ctx->snapset_obc = snapset_obc;
+  ctx->lock_to_release = OpContext::W_LOCK;
+  ctx->release_snapset_obc = true;
+  ctx->at_version = get_next_version();
+
+  PGBackend::PGTransaction *t = ctx->op_t;
+  set new_snaps;
+  for (set::iterator i = old_snaps.begin();
+       i != old_snaps.end();
+       ++i) {
+    if (!pool.info.is_removed_snap(*i))
+      new_snaps.insert(*i);
+  }
+
+  if (new_snaps.empty()) {
+    // remove clone
+    dout(10) << coid << " snaps " << old_snaps << " -> "
+	     << new_snaps << " ... deleting" << dendl;
+
+    // ...from snapset
+    snapid_t last = coid.snap;
+    vector::iterator p;
+    for (p = snapset.clones.begin(); p != snapset.clones.end(); ++p)
+      if (*p == last)
+	break;
+    assert(p != snapset.clones.end());
+    object_stat_sum_t delta;
+    delta.num_bytes -= snapset.get_clone_bytes(last);
+
+    if (p != snapset.clones.begin()) {
+      // not the oldest... merge overlap into next older clone
+      vector::iterator n = p - 1;
+      hobject_t prev_coid = coid;
+      prev_coid.snap = *n;
+      bool adjust_prev_bytes = is_present_clone(prev_coid);
+
+      if (adjust_prev_bytes)
+	delta.num_bytes -= snapset.get_clone_bytes(*n);
+
+      snapset.clone_overlap[*n].intersection_of(
+	snapset.clone_overlap[*p]);
+
+      if (adjust_prev_bytes)
+	delta.num_bytes += snapset.get_clone_bytes(*n);
+    }
+    delta.num_objects--;
+    if (coi.is_dirty())
+      delta.num_objects_dirty--;
+    if (coi.is_omap())
+      delta.num_objects_omap--;
+    if (coi.is_whiteout()) {
+      dout(20) << __func__ << " trimming whiteout on " << coid << dendl;
+      delta.num_whiteouts--;
+    }
+    delta.num_object_clones--;
+    info.stats.stats.add(delta, obc->obs.oi.category);
+    obc->obs.exists = false;
+
+    snapset.clones.erase(p);
+    snapset.clone_overlap.erase(last);
+    snapset.clone_size.erase(last);
+	
+    ctx->log.push_back(
+      pg_log_entry_t(
+	pg_log_entry_t::DELETE,
+	coid,
+	ctx->at_version,
+	ctx->obs->oi.version,
+	0,
+	osd_reqid_t(),
+	ctx->mtime)
+      );
+    if (pool.info.require_rollback()) {
+      set snaps(
+	ctx->obc->obs.oi.snaps.begin(),
+	ctx->obc->obs.oi.snaps.end());
+      ctx->log.back().mod_desc.update_snaps(snaps);
+      if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+	t->stash(coid, ctx->at_version.version);
+      } else {
+	t->remove(coid);
+      }
+    } else {
+      t->remove(coid);
+      ctx->log.back().mod_desc.mark_unrollbackable();
+    }
+    ctx->at_version.version++;
+  } else {
+    // save adjusted snaps for this object
+    dout(10) << coid << " snaps " << old_snaps
+	     << " -> " << new_snaps << dendl;
+    coi.snaps = vector(new_snaps.rbegin(), new_snaps.rend());
+
+    coi.prior_version = coi.version;
+    coi.version = ctx->at_version;
+    bl.clear();
+    ::encode(coi, bl);
+    setattr_maybe_cache(ctx->obc, ctx, t, OI_ATTR, bl);
+
+    ctx->log.push_back(
+      pg_log_entry_t(
+	pg_log_entry_t::MODIFY,
+	coid,
+	coi.version,
+	coi.prior_version,
+	0,
+	osd_reqid_t(),
+	ctx->mtime)
+      );
+    if (pool.info.require_rollback()) {
+      set changing;
+      changing.insert(OI_ATTR);
+      ctx->obc->fill_in_setattrs(changing, &(ctx->log.back().mod_desc));
+      set snaps(
+	ctx->obc->obs.oi.snaps.begin(),
+	ctx->obc->obs.oi.snaps.end());
+      ctx->log.back().mod_desc.update_snaps(old_snaps);
+    } else {
+      ctx->log.back().mod_desc.mark_unrollbackable();
+    }
+    
+    ::encode(coi.snaps, ctx->log.back().snaps);
+    ctx->at_version.version++;
+  }
+
+  // save head snapset
+  dout(10) << coid << " new snapset " << snapset << dendl;
+
+  if (snapset.clones.empty() && !snapset.head_exists) {
+    dout(10) << coid << " removing " << snapoid << dendl;
+    ctx->log.push_back(
+      pg_log_entry_t(
+	pg_log_entry_t::DELETE,
+	snapoid,
+	ctx->at_version,
+	ctx->snapset_obc->obs.oi.version,
+	0,
+	osd_reqid_t(),
+	ctx->mtime)
+      );
+
+    ctx->snapset_obc->obs.exists = false;
+    
+    if (pool.info.require_rollback()) {
+      if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+	t->stash(snapoid, ctx->at_version.version);
+      } else {
+	t->remove(snapoid);
+      }
+    } else {
+      t->remove(snapoid);
+      ctx->log.back().mod_desc.mark_unrollbackable();
+    }
+  } else {
+    dout(10) << coid << " updating snapset on " << snapoid << dendl;
+    ctx->log.push_back(
+      pg_log_entry_t(
+	pg_log_entry_t::MODIFY,
+	snapoid,
+	ctx->at_version,
+	ctx->snapset_obc->obs.oi.version,
+	0,
+	osd_reqid_t(),
+	ctx->mtime)
+      );
+
+    ctx->snapset_obc->obs.oi.prior_version =
+      ctx->snapset_obc->obs.oi.version;
+    ctx->snapset_obc->obs.oi.version = ctx->at_version;
+
+    bl.clear();
+    ::encode(snapset, bl);
+    setattr_maybe_cache(ctx->snapset_obc, ctx, t, SS_ATTR, bl);
+
+    bl.clear();
+    ::encode(ctx->snapset_obc->obs.oi, bl);
+    setattr_maybe_cache(ctx->snapset_obc, ctx, t, OI_ATTR, bl);
+
+    if (pool.info.require_rollback()) {
+      set changing;
+      changing.insert(OI_ATTR);
+      changing.insert(SS_ATTR);
+      ctx->snapset_obc->fill_in_setattrs(changing, &(ctx->log.back().mod_desc));
+    } else {
+      ctx->log.back().mod_desc.mark_unrollbackable();
+    }
+  }
+
+  return repop;
+}
+
+void ReplicatedPG::snap_trimmer()
+{
+  if (g_conf->osd_snap_trim_sleep > 0) {
+    utime_t t;
+    t.set_from_double(g_conf->osd_snap_trim_sleep);
+    t.sleep();
+    lock();
+    dout(20) << __func__ << " slept for " << t << dendl;
+  } else {
+    lock();
+  }
+  if (deleting) {
+    unlock();
+    return;
+  }
+  dout(10) << "snap_trimmer entry" << dendl;
+  if (is_primary()) {
+    entity_inst_t nobody;
+    if (scrubber.active) {
+      dout(10) << " scrubbing, will requeue snap_trimmer after" << dendl;
+      scrubber.queue_snap_trim = true;
+      unlock();
+      return;
+    }
+
+    dout(10) << "snap_trimmer posting" << dendl;
+    snap_trimmer_machine.process_event(SnapTrim());
+
+    if (snap_trimmer_machine.need_share_pg_info) {
+      dout(10) << "snap_trimmer share_pg_info" << dendl;
+      snap_trimmer_machine.need_share_pg_info = false;
+      share_pg_info();
+    }
+  } else if (is_active() && 
+	     last_complete_ondisk.epoch > info.history.last_epoch_started) {
+    // replica collection trimming
+    snap_trimmer_machine.process_event(SnapTrim());
+  }
+  if (snap_trimmer_machine.requeue) {
+    dout(10) << "snap_trimmer requeue" << dendl;
+    queue_snap_trim();
+  }
+  unlock();
+  return;
+}
+
+int ReplicatedPG::do_xattr_cmp_u64(int op, __u64 v1, bufferlist& xattr)
+{
+  __u64 v2;
+  if (xattr.length())
+    v2 = atoll(xattr.c_str());
+  else
+    v2 = 0;
+
+  dout(20) << "do_xattr_cmp_u64 '" << v1 << "' vs '" << v2 << "' op " << op << dendl;
+
+  switch (op) {
+  case CEPH_OSD_CMPXATTR_OP_EQ:
+    return (v1 == v2);
+  case CEPH_OSD_CMPXATTR_OP_NE:
+    return (v1 != v2);
+  case CEPH_OSD_CMPXATTR_OP_GT:
+    return (v1 > v2);
+  case CEPH_OSD_CMPXATTR_OP_GTE:
+    return (v1 >= v2);
+  case CEPH_OSD_CMPXATTR_OP_LT:
+    return (v1 < v2);
+  case CEPH_OSD_CMPXATTR_OP_LTE:
+    return (v1 <= v2);
+  default:
+    return -EINVAL;
+  }
+}
+
+int ReplicatedPG::do_xattr_cmp_str(int op, string& v1s, bufferlist& xattr)
+{
+  string v2s(xattr.c_str(), xattr.length());
+
+  dout(20) << "do_xattr_cmp_str '" << v1s << "' vs '" << v2s << "' op " << op << dendl;
+
+  switch (op) {
+  case CEPH_OSD_CMPXATTR_OP_EQ:
+    return (v1s.compare(v2s) == 0);
+  case CEPH_OSD_CMPXATTR_OP_NE:
+    return (v1s.compare(v2s) != 0);
+  case CEPH_OSD_CMPXATTR_OP_GT:
+    return (v1s.compare(v2s) > 0);
+  case CEPH_OSD_CMPXATTR_OP_GTE:
+    return (v1s.compare(v2s) >= 0);
+  case CEPH_OSD_CMPXATTR_OP_LT:
+    return (v1s.compare(v2s) < 0);
+  case CEPH_OSD_CMPXATTR_OP_LTE:
+    return (v1s.compare(v2s) <= 0);
+  default:
+    return -EINVAL;
+  }
+}
+
+// ========================================================================
+// low level osd ops
+
+int ReplicatedPG::do_tmap2omap(OpContext *ctx, unsigned flags)
+{
+  dout(20) << " convert tmap to omap for " << ctx->new_obs.oi.soid << dendl;
+  bufferlist header, vals;
+  int r = _get_tmap(ctx, &header, &vals);
+  if (r < 0) {
+    if (r == -ENODATA && (flags & CEPH_OSD_TMAP2OMAP_NULLOK))
+      r = 0;
+    return r;
+  }
+
+  vector ops(3);
+
+  ops[0].op.op = CEPH_OSD_OP_TRUNCATE;
+  ops[0].op.extent.offset = 0;
+  ops[0].op.extent.length = 0;
+
+  ops[1].op.op = CEPH_OSD_OP_OMAPSETHEADER;
+  ops[1].indata.claim(header);
+
+  ops[2].op.op = CEPH_OSD_OP_OMAPSETVALS;
+  ops[2].indata.claim(vals);
+
+  return do_osd_ops(ctx, ops);
+}
+
+int ReplicatedPG::do_tmapup_slow(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd_op,
+				    bufferlist& bl)
+{
+  // decode
+  bufferlist header;
+  map m;
+  if (bl.length()) {
+    bufferlist::iterator p = bl.begin();
+    ::decode(header, p);
+    ::decode(m, p);
+    assert(p.end());
+  }
+
+  // do the update(s)
+  while (!bp.end()) {
+    __u8 op;
+    string key;
+    ::decode(op, bp);
+
+    switch (op) {
+    case CEPH_OSD_TMAP_SET: // insert key
+      {
+	::decode(key, bp);
+	bufferlist data;
+	::decode(data, bp);
+	m[key] = data;
+      }
+      break;
+    case CEPH_OSD_TMAP_RM: // remove key
+      ::decode(key, bp);
+      if (!m.count(key)) {
+	return -ENOENT;
+      }
+      m.erase(key);
+      break;
+    case CEPH_OSD_TMAP_RMSLOPPY: // remove key
+      ::decode(key, bp);
+      m.erase(key);
+      break;
+    case CEPH_OSD_TMAP_HDR: // update header
+      {
+	::decode(header, bp);
+      }
+      break;
+    default:
+      return -EINVAL;
+    }
+  }
+
+  // reencode
+  bufferlist obl;
+  ::encode(header, obl);
+  ::encode(m, obl);
+
+  // write it out
+  vector nops(1);
+  OSDOp& newop = nops[0];
+  newop.op.op = CEPH_OSD_OP_WRITEFULL;
+  newop.op.extent.offset = 0;
+  newop.op.extent.length = obl.length();
+  newop.indata = obl;
+  do_osd_ops(ctx, nops);
+  osd_op.outdata.claim(newop.outdata);
+  return 0;
+}
+
+int ReplicatedPG::do_tmapup(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd_op)
+{
+  bufferlist::iterator orig_bp = bp;
+  int result = 0;
+  if (bp.end()) {
+    dout(10) << "tmapup is a no-op" << dendl;
+  } else {
+    // read the whole object
+    vector nops(1);
+    OSDOp& newop = nops[0];
+    newop.op.op = CEPH_OSD_OP_READ;
+    newop.op.extent.offset = 0;
+    newop.op.extent.length = 0;
+    do_osd_ops(ctx, nops);
+
+    dout(10) << "tmapup read " << newop.outdata.length() << dendl;
+
+    dout(30) << " starting is \n";
+    newop.outdata.hexdump(*_dout);
+    *_dout << dendl;
+
+    bufferlist::iterator ip = newop.outdata.begin();
+    bufferlist obl;
+
+    dout(30) << "the update command is: \n";
+    osd_op.indata.hexdump(*_dout);
+    *_dout << dendl;
+
+    // header
+    bufferlist header;
+    __u32 nkeys = 0;
+    if (newop.outdata.length()) {
+      ::decode(header, ip);
+      ::decode(nkeys, ip);
+    }
+    dout(10) << "tmapup header " << header.length() << dendl;
+
+    if (!bp.end() && *bp == CEPH_OSD_TMAP_HDR) {
+      ++bp;
+      ::decode(header, bp);
+      dout(10) << "tmapup new header " << header.length() << dendl;
+    }
+
+    ::encode(header, obl);
+
+    dout(20) << "tmapup initial nkeys " << nkeys << dendl;
+
+    // update keys
+    bufferlist newkeydata;
+    string nextkey, last_in_key;
+    bufferlist nextval;
+    bool have_next = false;
+    string last_disk_key;
+    if (!ip.end()) {
+      have_next = true;
+      ::decode(nextkey, ip);
+      ::decode(nextval, ip);
+      if (nextkey < last_disk_key) {
+	dout(5) << "tmapup warning: key '" << nextkey << "' < previous key '" << last_disk_key
+		<< "', falling back to an inefficient (unsorted) update" << dendl;
+	bp = orig_bp;
+	return do_tmapup_slow(ctx, bp, osd_op, newop.outdata);
+      }
+      last_disk_key = nextkey;
+    }
+    result = 0;
+    while (!bp.end() && !result) {
+      __u8 op;
+      string key;
+      try {
+	::decode(op, bp);
+	::decode(key, bp);
+      }
+      catch (buffer::error& e) {
+	return -EINVAL;
+      }
+      if (key < last_in_key) {
+	dout(5) << "tmapup warning: key '" << key << "' < previous key '" << last_in_key
+		<< "', falling back to an inefficient (unsorted) update" << dendl;
+	bp = orig_bp;
+	return do_tmapup_slow(ctx, bp, osd_op, newop.outdata);
+      }
+      last_in_key = key;
+
+      dout(10) << "tmapup op " << (int)op << " key " << key << dendl;
+	  
+      // skip existing intervening keys
+      bool key_exists = false;
+      while (have_next && !key_exists) {
+	dout(20) << "  (have_next=" << have_next << " nextkey=" << nextkey << ")" << dendl;
+	if (nextkey > key)
+	  break;
+	if (nextkey < key) {
+	  // copy untouched.
+	  ::encode(nextkey, newkeydata);
+	  ::encode(nextval, newkeydata);
+	  dout(20) << "  keep " << nextkey << " " << nextval.length() << dendl;
+	} else {
+	  // don't copy; discard old value.  and stop.
+	  dout(20) << "  drop " << nextkey << " " << nextval.length() << dendl;
+	  key_exists = true;
+	  nkeys--;
+	}
+	if (!ip.end()) {
+	  ::decode(nextkey, ip);
+	  ::decode(nextval, ip);
+	} else {
+	  have_next = false;
+	}
+      }
+
+      if (op == CEPH_OSD_TMAP_SET) {
+	bufferlist val;
+	try {
+	  ::decode(val, bp);
+	}
+	catch (buffer::error& e) {
+	  return -EINVAL;
+	}
+	::encode(key, newkeydata);
+	::encode(val, newkeydata);
+	dout(20) << "   set " << key << " " << val.length() << dendl;
+	nkeys++;
+      } else if (op == CEPH_OSD_TMAP_CREATE) {
+	if (key_exists) {
+	  return -EEXIST;
+	}
+	bufferlist val;
+	try {
+	  ::decode(val, bp);
+	}
+	catch (buffer::error& e) {
+	  return -EINVAL;
+	}
+	::encode(key, newkeydata);
+	::encode(val, newkeydata);
+	dout(20) << "   create " << key << " " << val.length() << dendl;
+	nkeys++;
+      } else if (op == CEPH_OSD_TMAP_RM) {
+	// do nothing.
+	if (!key_exists) {
+	  return -ENOENT;
+	}
+      } else if (op == CEPH_OSD_TMAP_RMSLOPPY) {
+	// do nothing
+      } else {
+	dout(10) << "  invalid tmap op " << (int)op << dendl;
+	return -EINVAL;
+      }
+    }
+
+    // copy remaining
+    if (have_next) {
+      ::encode(nextkey, newkeydata);
+      ::encode(nextval, newkeydata);
+      dout(20) << "  keep " << nextkey << " " << nextval.length() << dendl;
+    }
+    if (!ip.end()) {
+      bufferlist rest;
+      rest.substr_of(newop.outdata, ip.get_off(), newop.outdata.length() - ip.get_off());
+      dout(20) << "  keep trailing " << rest.length()
+	       << " at " << newkeydata.length() << dendl;
+      newkeydata.claim_append(rest);
+    }
+
+    // encode final key count + key data
+    dout(20) << "tmapup final nkeys " << nkeys << dendl;
+    ::encode(nkeys, obl);
+    obl.claim_append(newkeydata);
+
+    if (0) {
+      dout(30) << " final is \n";
+      obl.hexdump(*_dout);
+      *_dout << dendl;
+
+      // sanity check
+      bufferlist::iterator tp = obl.begin();
+      bufferlist h;
+      ::decode(h, tp);
+      map d;
+      ::decode(d, tp);
+      assert(tp.end());
+      dout(0) << " **** debug sanity check, looks ok ****" << dendl;
+    }
+
+    // write it out
+    if (!result) {
+      dout(20) << "tmapput write " << obl.length() << dendl;
+      newop.op.op = CEPH_OSD_OP_WRITEFULL;
+      newop.op.extent.offset = 0;
+      newop.op.extent.length = obl.length();
+      newop.indata = obl;
+      do_osd_ops(ctx, nops);
+      osd_op.outdata.claim(newop.outdata);
+    }
+  }
+  return result;
+}
+
+static int check_offset_and_length(uint64_t offset, uint64_t length, uint64_t max)
+{
+  if (offset >= max ||
+      length > max ||
+      offset + length > max)
+    return -EFBIG;
+
+  return 0;
+}
+
+struct FillInExtent : public Context {
+  ceph_le64 *r;
+  FillInExtent(ceph_le64 *r) : r(r) {}
+  void finish(int _r) {
+    if (_r >= 0) {
+      *r = _r;
+    }
+  }
+};
+
+int ReplicatedPG::do_osd_ops(OpContext *ctx, vector& ops)
+{
+  int result = 0;
+  SnapSetContext *ssc = ctx->obc->ssc;
+  ObjectState& obs = ctx->new_obs;
+  object_info_t& oi = obs.oi;
+  const hobject_t& soid = oi.soid;
+
+  bool first_read = true;
+
+  PGBackend::PGTransaction* t = ctx->op_t;
+
+  dout(10) << "do_osd_op " << soid << " " << ops << dendl;
+
+  for (vector::iterator p = ops.begin(); p != ops.end(); ++p, ctx->current_osd_subop_num++) {
+    OSDOp& osd_op = *p;
+    ceph_osd_op& op = osd_op.op;
+ 
+    dout(10) << "do_osd_op  " << osd_op << dendl;
+
+    bufferlist::iterator bp = osd_op.indata.begin();
+
+    // user-visible modifcation?
+    switch (op.op) {
+      // non user-visible modifications
+    case CEPH_OSD_OP_WATCH:
+    case CEPH_OSD_OP_CACHE_EVICT:
+    case CEPH_OSD_OP_CACHE_FLUSH:
+    case CEPH_OSD_OP_CACHE_TRY_FLUSH:
+    case CEPH_OSD_OP_UNDIRTY:
+    case CEPH_OSD_OP_COPY_FROM:  // we handle user_version update explicitly
+      break;
+    default:
+      if (op.op & CEPH_OSD_OP_MODE_WR)
+	ctx->user_modify = true;
+    }
+
+    ObjectContextRef src_obc;
+    if (ceph_osd_op_type_multi(op.op)) {
+      MOSDOp *m = static_cast(ctx->op->get_req());
+      object_locator_t src_oloc;
+      get_src_oloc(soid.oid, m->get_object_locator(), src_oloc);
+      hobject_t src_oid(osd_op.soid, src_oloc.key, soid.hash,
+			info.pgid.pool(), src_oloc.nspace);
+      src_obc = ctx->src_obc[src_oid];
+      dout(10) << " src_oid " << src_oid << " obc " << src_obc << dendl;
+      assert(src_obc);
+    }
+
+    // munge -1 truncate to 0 truncate
+    if (op.extent.truncate_seq == 1 && op.extent.truncate_size == (-1ULL)) {
+      op.extent.truncate_size = 0;
+      op.extent.truncate_seq = 0;
+    }
+
+    // munge ZERO -> TRUNCATE?  (don't munge to DELETE or we risk hosing attributes)
+    if (op.op == CEPH_OSD_OP_ZERO &&
+	obs.exists &&
+	op.extent.offset < cct->_conf->osd_max_object_size &&
+	op.extent.length >= 1 &&
+	op.extent.length <= cct->_conf->osd_max_object_size &&
+	op.extent.offset + op.extent.length >= oi.size) {
+      if (op.extent.offset >= oi.size) {
+        // no-op
+	goto fail;
+      }
+      dout(10) << " munging ZERO " << op.extent.offset << "~" << op.extent.length
+	       << " -> TRUNCATE " << op.extent.offset << " (old size is " << oi.size << ")" << dendl;
+      op.op = CEPH_OSD_OP_TRUNCATE;
+    }
+
+    switch (op.op) {
+      
+      // --- READS ---
+
+    case CEPH_OSD_OP_SYNC_READ:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      // fall through
+    case CEPH_OSD_OP_READ:
+      ++ctx->num_read;
+      {
+	__u32 seq = oi.truncate_seq;
+	uint64_t size = oi.size;
+	bool trimmed_read = false;
+	// are we beyond truncate_size?
+	if ( (seq < op.extent.truncate_seq) &&
+	     (op.extent.offset + op.extent.length > op.extent.truncate_size) )
+	  size = op.extent.truncate_size;
+
+	if (op.extent.offset >= size) {
+	  op.extent.length = 0;
+	  trimmed_read = true;
+	} else if (op.extent.offset + op.extent.length > size) {
+	  op.extent.length = size - op.extent.offset;
+	  trimmed_read = true;
+	}
+
+	// read into a buffer
+	bufferlist bl;
+	if (trimmed_read && op.extent.length == 0) {
+	  // read size was trimmed to zero and it is expected to do nothing
+	  // a read operation of 0 bytes does *not* do nothing, this is why
+	  // the trimmed_read boolean is needed
+	} else if (pool.info.require_rollback()) {
+	  ctx->pending_async_reads.push_back(
+	    make_pair(
+	      make_pair(op.extent.offset, op.extent.length),
+	      make_pair(&osd_op.outdata, new FillInExtent(&op.extent.length))));
+	  dout(10) << " async_read noted for " << soid << dendl;
+	} else {
+	  int r = pgbackend->objects_read_sync(
+	    soid, op.extent.offset, op.extent.length, &osd_op.outdata);
+	  if (r >= 0)
+	    op.extent.length = r;
+	  else {
+	    result = r;
+	    op.extent.length = 0;
+	  }
+	  dout(10) << " read got " << r << " / " << op.extent.length
+		   << " bytes from obj " << soid << dendl;
+	}
+	if (first_read) {
+	  first_read = false;
+	  ctx->data_off = op.extent.offset;
+	}
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(op.extent.length, 10);
+	ctx->delta_stats.num_rd++;
+
+      }
+      break;
+
+    /* map extents */
+    case CEPH_OSD_OP_MAPEXT:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_read;
+      {
+	// read into a buffer
+	bufferlist bl;
+	int r = osd->store->fiemap(coll, soid, op.extent.offset, op.extent.length, bl);
+	osd_op.outdata.claim(bl);
+	if (r < 0)
+	  result = r;
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(op.extent.length, 10);
+	ctx->delta_stats.num_rd++;
+	dout(10) << " map_extents done on object " << soid << dendl;
+      }
+      break;
+
+    /* map extents */
+    case CEPH_OSD_OP_SPARSE_READ:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_read;
+      {
+        if (op.extent.truncate_seq) {
+          dout(0) << "sparse_read does not support truncation sequence " << dendl;
+          result = -EINVAL;
+          break;
+        }
+	// read into a buffer
+	bufferlist bl;
+        int total_read = 0;
+	int r = osd->store->fiemap(coll, soid, op.extent.offset, op.extent.length, bl);
+	if (r < 0)  {
+	  result = r;
+          break;
+	}
+        map m;
+        bufferlist::iterator iter = bl.begin();
+        ::decode(m, iter);
+        map::iterator miter;
+        bufferlist data_bl;
+	uint64_t last = op.extent.offset;
+        for (miter = m.begin(); miter != m.end(); ++miter) {
+	  // verify hole?
+	  if (cct->_conf->osd_verify_sparse_read_holes &&
+	      last < miter->first) {
+	    bufferlist t;
+	    uint64_t len = miter->first - last;
+	    r = pgbackend->objects_read_sync(
+	      soid, last, len, &t);
+	    if (!t.is_zero()) {
+	      osd->clog.error() << coll << " " << soid << " sparse-read found data in hole "
+				<< last << "~" << len << "\n";
+	    }
+	  }
+
+          bufferlist tmpbl;
+	  r = pgbackend->objects_read_sync(
+	    soid, miter->first, miter->second, &tmpbl);
+          if (r < 0)
+            break;
+
+          if (r < (int)miter->second) /* this is usually happen when we get extent that exceeds the actual file size */
+            miter->second = r;
+          total_read += r;
+          dout(10) << "sparse-read " << miter->first << "@" << miter->second << dendl;
+	  data_bl.claim_append(tmpbl);
+	  last = miter->first + r;
+        }
+
+	// verify trailing hole?
+	if (cct->_conf->osd_verify_sparse_read_holes) {
+	  uint64_t end = MIN(op.extent.offset + op.extent.length, oi.size);
+	  if (last < end) {
+	    bufferlist t;
+	    uint64_t len = end - last;
+	    r = pgbackend->objects_read_sync(
+	      soid, last, len, &t);
+	    if (!t.is_zero()) {
+	      osd->clog.error() << coll << " " << soid << " sparse-read found data in hole "
+				<< last << "~" << len << "\n";
+	    }
+	  }
+	}
+
+        if (r < 0) {
+          result = r;
+          break;
+        }
+
+        op.extent.length = total_read;
+
+        ::encode(m, osd_op.outdata);
+        ::encode(data_bl, osd_op.outdata);
+
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(op.extent.length, 10);
+	ctx->delta_stats.num_rd++;
+
+	dout(10) << " sparse_read got " << total_read << " bytes from object " << soid << dendl;
+      }
+      break;
+
+    case CEPH_OSD_OP_CALL:
+      {
+	string cname, mname;
+	bufferlist indata;
+	try {
+	  bp.copy(op.cls.class_len, cname);
+	  bp.copy(op.cls.method_len, mname);
+	  bp.copy(op.cls.indata_len, indata);
+	} catch (buffer::error& e) {
+	  dout(10) << "call unable to decode class + method + indata" << dendl;
+	  dout(30) << "in dump: ";
+	  osd_op.indata.hexdump(*_dout);
+	  *_dout << dendl;
+	  result = -EINVAL;
+	  break;
+	}
+
+	ClassHandler::ClassData *cls;
+	result = osd->class_handler->open_class(cname, &cls);
+	assert(result == 0);   // init_op_flags() already verified this works.
+
+	ClassHandler::ClassMethod *method = cls->get_method(mname.c_str());
+	if (!method) {
+	  dout(10) << "call method " << cname << "." << mname << " does not exist" << dendl;
+	  result = -EOPNOTSUPP;
+	  break;
+	}
+
+	int flags = method->get_flags();
+	if (flags & CLS_METHOD_WR)
+	  ctx->user_modify = true;
+
+	bufferlist outdata;
+	dout(10) << "call method " << cname << "." << mname << dendl;
+	int prev_rd = ctx->num_read;
+	int prev_wr = ctx->num_write;
+	result = method->exec((cls_method_context_t)&ctx, indata, outdata);
+
+	if (ctx->num_read > prev_rd && !(flags & CLS_METHOD_RD)) {
+	  derr << "method " << cname << "." << mname << " tried to read object but is not marked RD" << dendl;
+	  result = -EIO;
+	  break;
+	}
+	if (ctx->num_write > prev_wr && !(flags & CLS_METHOD_WR)) {
+	  derr << "method " << cname << "." << mname << " tried to update object but is not marked WR" << dendl;
+	  result = -EIO;
+	  break;
+	}
+
+	dout(10) << "method called response length=" << outdata.length() << dendl;
+	op.extent.length = outdata.length();
+	osd_op.outdata.claim_append(outdata);
+	dout(30) << "out dump: ";
+	osd_op.outdata.hexdump(*_dout);
+	*_dout << dendl;
+      }
+      break;
+
+    case CEPH_OSD_OP_STAT:
+      // note: stat does not require RD
+      {
+	if (obs.exists && !oi.is_whiteout()) {
+	  ::encode(oi.size, osd_op.outdata);
+	  ::encode(oi.mtime, osd_op.outdata);
+	  dout(10) << "stat oi has " << oi.size << " " << oi.mtime << dendl;
+	} else {
+	  result = -ENOENT;
+	  dout(10) << "stat oi object does not exist" << dendl;
+	}
+
+	ctx->delta_stats.num_rd++;
+      }
+      break;
+
+    case CEPH_OSD_OP_ISDIRTY:
+      ++ctx->num_read;
+      {
+	bool is_dirty = obs.oi.is_dirty();
+	::encode(is_dirty, osd_op.outdata);
+	ctx->delta_stats.num_rd++;
+	result = 0;
+      }
+      break;
+
+    case CEPH_OSD_OP_UNDIRTY:
+      ++ctx->num_write;
+      {
+	if (oi.is_dirty()) {
+	  ctx->undirty = true;  // see make_writeable()
+	  ctx->modify = true;
+	  ctx->delta_stats.num_wr++;
+	}
+	result = 0;
+      }
+      break;
+
+    case CEPH_OSD_OP_CACHE_TRY_FLUSH:
+      ++ctx->num_write;
+      {
+	if (ctx->lock_to_release != OpContext::NONE) {
+	  dout(10) << "cache-try-flush without SKIPRWLOCKS flag set" << dendl;
+	  result = -EINVAL;
+	  break;
+	}
+	if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE) {
+	  result = -EINVAL;
+	  break;
+	}
+	if (!obs.exists) {
+	  result = 0;
+	  break;
+	}
+	if (oi.is_dirty()) {
+	  result = start_flush(ctx->op, ctx->obc, false, NULL, NULL);
+	  if (result == -EINPROGRESS)
+	    result = -EAGAIN;
+	} else {
+	  result = 0;
+	}
+      }
+      break;
+
+    case CEPH_OSD_OP_CACHE_FLUSH:
+      ++ctx->num_write;
+      {
+	if (ctx->lock_to_release == OpContext::NONE) {
+	  dout(10) << "cache-flush with SKIPRWLOCKS flag set" << dendl;
+	  result = -EINVAL;
+	  break;
+	}
+	if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE) {
+	  result = -EINVAL;
+	  break;
+	}
+	if (!obs.exists) {
+	  result = 0;
+	  break;
+	}
+	hobject_t missing;
+	if (oi.is_dirty()) {
+	  result = start_flush(ctx->op, ctx->obc, true, &missing, NULL);
+	  if (result == -EINPROGRESS)
+	    result = -EAGAIN;
+	} else {
+	  result = 0;
+	}
+	// Check special return value which has set missing_return
+        if (result == -ENOENT) {
+          dout(10) << __func__ << " CEPH_OSD_OP_CACHE_FLUSH got ENOENT" << dendl;
+	  assert(!missing.is_min());
+	  wait_for_unreadable_object(missing, ctx->op);
+	  // Error code which is used elsewhere when wait_for_unreadable_object() is used
+	  result = -EAGAIN;
+	}
+      }
+      break;
+
+    case CEPH_OSD_OP_CACHE_EVICT:
+      ++ctx->num_write;
+      {
+	if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE) {
+	  result = -EINVAL;
+	  break;
+	}
+	if (oi.is_dirty()) {
+	  result = -EBUSY;
+	  break;
+	}
+	if (!oi.watchers.empty()) {
+	  result = -EBUSY;
+	  break;
+	}
+	if (soid.snap == CEPH_NOSNAP) {
+	  result = _verify_no_head_clones(soid, ssc->snapset);
+	  if (result < 0)
+	    break;
+	}
+	result = _delete_oid(ctx, true);
+	osd->logger->inc(l_osd_tier_evict);
+      }
+      break;
+
+    case CEPH_OSD_OP_GETXATTR:
+      ++ctx->num_read;
+      {
+	string aname;
+	bp.copy(op.xattr.name_len, aname);
+	string name = "_" + aname;
+	int r = getattr_maybe_cache(
+	  ctx->obc,
+	  name,
+	  &(osd_op.outdata));
+	if (r >= 0) {
+	  op.xattr.value_len = r;
+	  result = 0;
+	  ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(r, 10);
+	  ctx->delta_stats.num_rd++;
+	} else
+	  result = r;
+      }
+      break;
+
+   case CEPH_OSD_OP_GETXATTRS:
+      ++ctx->num_read;
+      {
+	map out;
+	result = getattrs_maybe_cache(
+	  ctx->obc,
+	  &out,
+	  true);
+        
+        bufferlist bl;
+        ::encode(out, bl);
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(bl.length(), 10);
+        ctx->delta_stats.num_rd++;
+        osd_op.outdata.claim_append(bl);
+      }
+      break;
+      
+    case CEPH_OSD_OP_CMPXATTR:
+    case CEPH_OSD_OP_SRC_CMPXATTR:
+      ++ctx->num_read;
+      {
+	string aname;
+	bp.copy(op.xattr.name_len, aname);
+	string name = "_" + aname;
+	name[op.xattr.name_len + 1] = 0;
+	
+	bufferlist xattr;
+	if (op.op == CEPH_OSD_OP_CMPXATTR)
+	  result = getattr_maybe_cache(
+	    ctx->obc,
+	    name,
+	    &xattr);
+	else
+	  result = getattr_maybe_cache(
+	    src_obc,
+	    name,
+	    &xattr);
+	if (result < 0 && result != -EEXIST && result != -ENODATA)
+	  break;
+	
+	ctx->delta_stats.num_rd++;
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(xattr.length(), 10);
+
+	switch (op.xattr.cmp_mode) {
+	case CEPH_OSD_CMPXATTR_MODE_STRING:
+	  {
+	    string val;
+	    bp.copy(op.xattr.value_len, val);
+	    val[op.xattr.value_len] = 0;
+	    dout(10) << "CEPH_OSD_OP_CMPXATTR name=" << name << " val=" << val
+		     << " op=" << (int)op.xattr.cmp_op << " mode=" << (int)op.xattr.cmp_mode << dendl;
+	    result = do_xattr_cmp_str(op.xattr.cmp_op, val, xattr);
+	  }
+	  break;
+
+        case CEPH_OSD_CMPXATTR_MODE_U64:
+	  {
+	    uint64_t u64val;
+	    try {
+	      ::decode(u64val, bp);
+	    }
+	    catch (buffer::error& e) {
+	      result = -EINVAL;
+	      goto fail;
+	    }
+	    dout(10) << "CEPH_OSD_OP_CMPXATTR name=" << name << " val=" << u64val
+		     << " op=" << (int)op.xattr.cmp_op << " mode=" << (int)op.xattr.cmp_mode << dendl;
+	    result = do_xattr_cmp_u64(op.xattr.cmp_op, u64val, xattr);
+	  }
+	  break;
+
+	default:
+	  dout(10) << "bad cmp mode " << (int)op.xattr.cmp_mode << dendl;
+	  result = -EINVAL;
+	}
+
+	if (!result) {
+	  dout(10) << "comparison returned false" << dendl;
+	  result = -ECANCELED;
+	  break;
+	}
+	if (result < 0) {
+	  dout(10) << "comparison returned " << result << " " << cpp_strerror(-result) << dendl;
+	  break;
+	}
+
+	dout(10) << "comparison returned true" << dendl;
+      }
+      break;
+
+    case CEPH_OSD_OP_ASSERT_VER:
+      ++ctx->num_read;
+      {
+	uint64_t ver = op.watch.ver;
+	if (!ver)
+	  result = -EINVAL;
+        else if (ver < oi.user_version)
+	  result = -ERANGE;
+	else if (ver > oi.user_version)
+	  result = -EOVERFLOW;
+      }
+      break;
+
+    case CEPH_OSD_OP_LIST_WATCHERS:
+      ++ctx->num_read;
+      {
+        obj_list_watch_response_t resp;
+
+        map, watch_info_t>::const_iterator oi_iter;
+        for (oi_iter = oi.watchers.begin(); oi_iter != oi.watchers.end();
+                                       ++oi_iter) {
+          dout(20) << "key cookie=" << oi_iter->first.first
+               << " entity=" << oi_iter->first.second << " "
+               << oi_iter->second << dendl;
+          assert(oi_iter->first.first == oi_iter->second.cookie);
+          assert(oi_iter->first.second.is_client());
+
+          watch_item_t wi(oi_iter->first.second, oi_iter->second.cookie,
+		 oi_iter->second.timeout_seconds, oi_iter->second.addr);
+          resp.entries.push_back(wi);
+        }
+
+        resp.encode(osd_op.outdata);
+        result = 0;
+
+        ctx->delta_stats.num_rd++;
+        break;
+      }
+
+    case CEPH_OSD_OP_LIST_SNAPS:
+      ++ctx->num_read;
+      {
+        obj_list_snap_response_t resp;
+
+        if (!ssc) {
+	  ssc = ctx->obc->ssc = get_snapset_context(soid, false);
+        }
+        assert(ssc);
+
+        int clonecount = ssc->snapset.clones.size();
+        if (ssc->snapset.head_exists)
+          clonecount++;
+        resp.clones.reserve(clonecount);
+        for (vector::const_iterator clone_iter = ssc->snapset.clones.begin();
+	     clone_iter != ssc->snapset.clones.end(); ++clone_iter) {
+          clone_info ci;
+          ci.cloneid = *clone_iter;
+
+	  hobject_t clone_oid = soid;
+	  clone_oid.snap = *clone_iter;
+	  ObjectContextRef clone_obc = ctx->src_obc[clone_oid];
+          assert(clone_obc);
+	  for (vector::reverse_iterator p = clone_obc->obs.oi.snaps.rbegin();
+	       p != clone_obc->obs.oi.snaps.rend();
+	       ++p) {
+	    ci.snaps.push_back(*p);
+	  }
+
+          dout(20) << " clone " << *clone_iter << " snaps " << ci.snaps << dendl;
+
+          map >::const_iterator coi;
+          coi = ssc->snapset.clone_overlap.find(ci.cloneid);
+          if (coi == ssc->snapset.clone_overlap.end()) {
+            osd->clog.error() << "osd." << osd->whoami << ": inconsistent clone_overlap found for oid "
+			      << soid << " clone " << *clone_iter;
+            result = -EINVAL;
+            break;
+          }
+          const interval_set &o = coi->second;
+          ci.overlap.reserve(o.num_intervals());
+          for (interval_set::const_iterator r = o.begin();
+               r != o.end(); ++r) {
+            ci.overlap.push_back(pair(r.get_start(), r.get_len()));
+          }
+
+          map::const_iterator si;
+          si = ssc->snapset.clone_size.find(ci.cloneid);
+          if (si == ssc->snapset.clone_size.end()) {
+            osd->clog.error() << "osd." << osd->whoami << ": inconsistent clone_size found for oid "
+			      << soid << " clone " << *clone_iter;
+            result = -EINVAL;
+            break;
+          }
+          ci.size = si->second;
+
+          resp.clones.push_back(ci);
+        }
+        if (ssc->snapset.head_exists &&
+	    !ctx->obc->obs.oi.is_whiteout()) {
+          assert(obs.exists);
+          clone_info ci;
+          ci.cloneid = CEPH_NOSNAP;
+
+          //Size for HEAD is oi.size
+          ci.size = oi.size;
+
+          resp.clones.push_back(ci);
+        }
+	resp.seq = ssc->snapset.seq;
+
+        resp.encode(osd_op.outdata);
+        result = 0;
+
+        ctx->delta_stats.num_rd++;
+        break;
+      }
+
+    case CEPH_OSD_OP_ASSERT_SRC_VERSION:
+      ++ctx->num_read;
+      {
+	uint64_t ver = op.assert_ver.ver;
+	if (!ver)
+	  result = -EINVAL;
+        else if (ver < src_obc->obs.oi.user_version)
+	  result = -ERANGE;
+	else if (ver > src_obc->obs.oi.user_version)
+	  result = -EOVERFLOW;
+	break;
+      }
+
+   case CEPH_OSD_OP_NOTIFY:
+      ++ctx->num_read;
+      {
+	uint32_t ver;
+	uint32_t timeout;
+        bufferlist bl;
+
+	try {
+          ::decode(ver, bp);
+	  ::decode(timeout, bp);
+          ::decode(bl, bp);
+	} catch (const buffer::error &e) {
+	  timeout = 0;
+	}
+	if (!timeout)
+	  timeout = cct->_conf->osd_default_notify_timeout;
+
+	notify_info_t n;
+	n.timeout = timeout;
+	n.cookie = op.watch.cookie;
+        n.bl = bl;
+	ctx->notifies.push_back(n);
+      }
+      break;
+
+    case CEPH_OSD_OP_NOTIFY_ACK:
+      ++ctx->num_read;
+      {
+	try {
+	  uint64_t notify_id = 0;
+	  uint64_t watch_cookie = 0;
+	  ::decode(notify_id, bp);
+	  ::decode(watch_cookie, bp);
+	  OpContext::NotifyAck ack(notify_id, watch_cookie);
+	  ctx->notify_acks.push_back(ack);
+	} catch (const buffer::error &e) {
+	  OpContext::NotifyAck ack(
+	    // op.watch.cookie is actually the notify_id for historical reasons
+	    op.watch.cookie
+	    );
+	  ctx->notify_acks.push_back(ack);
+	}
+      }
+      break;
+
+    case CEPH_OSD_OP_SETALLOCHINT:
+      ++ctx->num_write;
+      {
+        if (!obs.exists) {
+          ctx->mod_desc.create();
+          t->touch(soid);
+          ctx->delta_stats.num_objects++;
+          obs.exists = true;
+        }
+        t->set_alloc_hint(soid, op.alloc_hint.expected_object_size,
+                          op.alloc_hint.expected_write_size);
+        ctx->delta_stats.num_wr++;
+        result = 0;
+      }
+      break;
+
+
+      // --- WRITES ---
+
+      // -- object data --
+
+    case CEPH_OSD_OP_WRITE:
+      ++ctx->num_write;
+      { // write
+	if (op.extent.length != osd_op.indata.length()) {
+	  result = -EINVAL;
+	  break;
+	}
+
+	if (pool.info.requires_aligned_append() &&
+	    (op.extent.offset % pool.info.required_alignment() != 0)) {
+	  result = -EOPNOTSUPP;
+	  break;
+	}
+
+	if (!obs.exists) {
+	  ctx->mod_desc.create();
+	} else if (op.extent.offset == oi.size) {
+	  ctx->mod_desc.append(oi.size);
+	} else {
+	  ctx->mod_desc.mark_unrollbackable();
+	  if (pool.info.require_rollback()) {
+	    result = -EOPNOTSUPP;
+	    break;
+	  }
+	}
+
+        __u32 seq = oi.truncate_seq;
+        if (seq && (seq > op.extent.truncate_seq) &&
+            (op.extent.offset + op.extent.length > oi.size)) {
+	  // old write, arrived after trimtrunc
+	  op.extent.length = (op.extent.offset > oi.size ? 0 : oi.size - op.extent.offset);
+	  dout(10) << " old truncate_seq " << op.extent.truncate_seq << " < current " << seq
+		   << ", adjusting write length to " << op.extent.length << dendl;
+	  bufferlist t;
+	  t.substr_of(osd_op.indata, 0, op.extent.length);
+	  osd_op.indata.swap(t);
+        }
+	if (op.extent.truncate_seq > seq) {
+	  // write arrives before trimtrunc
+	  if (obs.exists && !oi.is_whiteout()) {
+	    dout(10) << " truncate_seq " << op.extent.truncate_seq << " > current " << seq
+		     << ", truncating to " << op.extent.truncate_size << dendl;
+	    t->truncate(soid, op.extent.truncate_size);
+	    oi.truncate_seq = op.extent.truncate_seq;
+	    oi.truncate_size = op.extent.truncate_size;
+	    if (op.extent.truncate_size != oi.size) {
+	      ctx->delta_stats.num_bytes -= oi.size;
+	      ctx->delta_stats.num_bytes += op.extent.truncate_size;
+	      oi.size = op.extent.truncate_size;
+	    }
+	  } else {
+	    dout(10) << " truncate_seq " << op.extent.truncate_seq << " > current " << seq
+		     << ", but object is new" << dendl;
+	    oi.truncate_seq = op.extent.truncate_seq;
+	    oi.truncate_size = op.extent.truncate_size;
+	  }
+	}
+	result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size);
+	if (result < 0)
+	  break;
+	if (pool.info.require_rollback()) {
+	  t->append(soid, op.extent.offset, op.extent.length, osd_op.indata);
+	} else {
+	  t->write(soid, op.extent.offset, op.extent.length, osd_op.indata);
+	}
+	write_update_size_and_usage(ctx->delta_stats, oi, ssc->snapset, ctx->modified_ranges,
+				    op.extent.offset, op.extent.length, true);
+	if (!obs.exists) {
+	  ctx->delta_stats.num_objects++;
+	  obs.exists = true;
+	}
+      }
+      break;
+      
+    case CEPH_OSD_OP_WRITEFULL:
+      ++ctx->num_write;
+      { // write full object
+	if (op.extent.length != osd_op.indata.length()) {
+	  result = -EINVAL;
+	  break;
+	}
+	result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size);
+	if (result < 0)
+	  break;
+
+	if (pool.info.require_rollback()) {
+	  if (obs.exists) {
+	    if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+	      t->stash(soid, ctx->at_version.version);
+	    } else {
+	      t->remove(soid);
+	    }
+	  }
+	  ctx->mod_desc.create();
+	  t->append(soid, op.extent.offset, op.extent.length, osd_op.indata);
+	  if (obs.exists) {
+	    map to_set = ctx->obc->attr_cache;
+	    map > &overlay =
+	      ctx->pending_attrs[ctx->obc];
+	    for (map >::iterator i =
+		   overlay.begin();
+		 i != overlay.end();
+		 ++i) {
+	      if (i->second) {
+		to_set[i->first] = *(i->second);
+	      } else {
+		to_set.erase(i->first);
+	      }
+	    }
+	    t->setattrs(soid, to_set);
+	  }
+	} else {
+	  ctx->mod_desc.mark_unrollbackable();
+	  if (obs.exists) {
+	    t->truncate(soid, 0);
+	  }
+	  t->write(soid, op.extent.offset, op.extent.length, osd_op.indata);
+	}
+	if (!obs.exists) {
+	  ctx->delta_stats.num_objects++;
+	  obs.exists = true;
+	}
+	interval_set ch;
+	if (oi.size > 0)
+	  ch.insert(0, oi.size);
+	ctx->modified_ranges.union_of(ch);
+	if (op.extent.length + op.extent.offset != oi.size) {
+	  ctx->delta_stats.num_bytes -= oi.size;
+	  oi.size = op.extent.length + op.extent.offset;
+	  ctx->delta_stats.num_bytes += oi.size;
+	}
+	ctx->delta_stats.num_wr++;
+	ctx->delta_stats.num_wr_kb += SHIFT_ROUND_UP(op.extent.length, 10);
+      }
+      break;
+
+    case CEPH_OSD_OP_ROLLBACK :
+      ++ctx->num_write;
+      result = _rollback_to(ctx, op);
+      break;
+
+    case CEPH_OSD_OP_ZERO:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_write;
+      { // zero
+	result = check_offset_and_length(op.extent.offset, op.extent.length, cct->_conf->osd_max_object_size);
+	if (result < 0)
+	  break;
+	assert(op.extent.length);
+	if (obs.exists && !oi.is_whiteout()) {
+	  ctx->mod_desc.mark_unrollbackable();
+	  t->zero(soid, op.extent.offset, op.extent.length);
+	  interval_set ch;
+	  ch.insert(op.extent.offset, op.extent.length);
+	  ctx->modified_ranges.union_of(ch);
+	  ctx->delta_stats.num_wr++;
+	} else {
+	  // no-op
+	}
+      }
+      break;
+    case CEPH_OSD_OP_CREATE:
+      ++ctx->num_write;
+      {
+        int flags = le32_to_cpu(op.flags);
+	if (obs.exists && !oi.is_whiteout() &&
+	    (flags & CEPH_OSD_OP_FLAG_EXCL)) {
+          result = -EEXIST; /* this is an exclusive create */
+	} else {
+	  if (osd_op.indata.length()) {
+	    bufferlist::iterator p = osd_op.indata.begin();
+	    string category;
+	    try {
+	      ::decode(category, p);
+	    }
+	    catch (buffer::error& e) {
+	      result = -EINVAL;
+	      goto fail;
+	    }
+	    if (category.size()) {
+	      if (obs.exists && !oi.is_whiteout()) {
+		if (obs.oi.category != category)
+		  result = -EEXIST;  // category cannot be reset
+	      } else {
+		obs.oi.category = category;
+	      }
+	    }
+	  }
+          if (result >= 0) {
+            if (!obs.exists)
+              ctx->mod_desc.create();
+            t->touch(soid);
+            if (!obs.exists) {
+              ctx->delta_stats.num_objects++;
+              obs.exists = true;
+            }
+          }
+	}
+      }
+      break;
+
+    case CEPH_OSD_OP_TRIMTRUNC:
+      op.extent.offset = op.extent.truncate_size;
+      // falling through
+
+    case CEPH_OSD_OP_TRUNCATE:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_write;
+      ctx->mod_desc.mark_unrollbackable();
+      {
+	// truncate
+	if (!obs.exists || oi.is_whiteout()) {
+	  dout(10) << " object dne, truncate is a no-op" << dendl;
+	  break;
+	}
+
+	if (op.extent.offset > cct->_conf->osd_max_object_size) {
+	  result = -EFBIG;
+	  break;
+	}
+
+	if (op.extent.truncate_seq) {
+	  assert(op.extent.offset == op.extent.truncate_size);
+	  if (op.extent.truncate_seq <= oi.truncate_seq) {
+	    dout(10) << " truncate seq " << op.extent.truncate_seq << " <= current " << oi.truncate_seq
+		     << ", no-op" << dendl;
+	    break; // old
+	  }
+	  dout(10) << " truncate seq " << op.extent.truncate_seq << " > current " << oi.truncate_seq
+		   << ", truncating" << dendl;
+	  oi.truncate_seq = op.extent.truncate_seq;
+	  oi.truncate_size = op.extent.truncate_size;
+	}
+
+	t->truncate(soid, op.extent.offset);
+	if (oi.size > op.extent.offset) {
+	  interval_set trim;
+	  trim.insert(op.extent.offset, oi.size-op.extent.offset);
+	  ctx->modified_ranges.union_of(trim);
+	}
+	if (op.extent.offset != oi.size) {
+	  ctx->delta_stats.num_bytes -= oi.size;
+	  ctx->delta_stats.num_bytes += op.extent.offset;
+	  oi.size = op.extent.offset;
+	}
+	ctx->delta_stats.num_wr++;
+	// do no set exists, or we will break above DELETE -> TRUNCATE munging.
+      }
+      break;
+    
+    case CEPH_OSD_OP_DELETE:
+      ++ctx->num_write;
+      if (ctx->obc->obs.oi.watchers.size()) {
+	// Cannot delete an object with watchers
+	result = -EBUSY;
+      } else {
+	result = _delete_oid(ctx, false);
+      }
+      break;
+
+    case CEPH_OSD_OP_CLONERANGE:
+      ctx->mod_desc.mark_unrollbackable();
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_read;
+      ++ctx->num_write;
+      {
+	if (!obs.exists) {
+	  t->touch(obs.oi.soid);
+	  ctx->delta_stats.num_objects++;
+	  obs.exists = true;
+	}
+	if (op.clonerange.src_offset + op.clonerange.length > src_obc->obs.oi.size) {
+	  dout(10) << " clonerange source " << osd_op.soid << " "
+		   << op.clonerange.src_offset << "~" << op.clonerange.length
+		   << " extends past size " << src_obc->obs.oi.size << dendl;
+	  result = -EINVAL;
+	  break;
+	}
+	t->clone_range(src_obc->obs.oi.soid,
+		      obs.oi.soid, op.clonerange.src_offset,
+		      op.clonerange.length, op.clonerange.offset);
+		      
+
+	write_update_size_and_usage(ctx->delta_stats, oi, ssc->snapset, ctx->modified_ranges,
+				    op.clonerange.offset, op.clonerange.length, false);
+      }
+      break;
+      
+    case CEPH_OSD_OP_WATCH:
+      ++ctx->num_write;
+      {
+	if (!obs.exists) {
+	  result = -ENOENT;
+	  break;
+	}
+        uint64_t cookie = op.watch.cookie;
+	bool do_watch = op.watch.flag & 1;
+        entity_name_t entity = ctx->reqid.name;
+	ObjectContextRef obc = ctx->obc;
+
+	dout(10) << "watch: ctx->obc=" << (void *)obc.get() << " cookie=" << cookie
+		 << " oi.version=" << oi.version.version << " ctx->at_version=" << ctx->at_version << dendl;
+	dout(10) << "watch: oi.user_version=" << oi.user_version<< dendl;
+	dout(10) << "watch: peer_addr="
+	  << ctx->op->get_req()->get_connection()->get_peer_addr() << dendl;
+
+	watch_info_t w(cookie, cct->_conf->osd_client_watch_timeout,
+	  ctx->op->get_req()->get_connection()->get_peer_addr());
+	if (do_watch) {
+	  if (oi.watchers.count(make_pair(cookie, entity))) {
+	    dout(10) << " found existing watch " << w << " by " << entity << dendl;
+	  } else {
+	    dout(10) << " registered new watch " << w << " by " << entity << dendl;
+	    oi.watchers[make_pair(cookie, entity)] = w;
+	    t->nop();  // make sure update the object_info on disk!
+	  }
+	  ctx->watch_connects.push_back(w);
+        } else {
+	  map, watch_info_t>::iterator oi_iter =
+	    oi.watchers.find(make_pair(cookie, entity));
+	  if (oi_iter != oi.watchers.end()) {
+	    dout(10) << " removed watch " << oi_iter->second << " by "
+		     << entity << dendl;
+            oi.watchers.erase(oi_iter);
+	    t->nop();  // update oi on disk
+	    ctx->watch_disconnects.push_back(w);
+	  } else {
+	    dout(10) << " can't remove: no watch by " << entity << dendl;
+	  }
+        }
+      }
+      break;
+
+
+      // -- object attrs --
+      
+    case CEPH_OSD_OP_SETXATTR:
+      ++ctx->num_write;
+      {
+	if (cct->_conf->osd_max_attr_size > 0 &&
+	    op.xattr.value_len > cct->_conf->osd_max_attr_size) {
+	  result = -EFBIG;
+	  break;
+	}
+	if (!obs.exists) {
+	  ctx->mod_desc.create();
+	  t->touch(soid);
+	  ctx->delta_stats.num_objects++;
+	  obs.exists = true;
+	}
+	string aname;
+	bp.copy(op.xattr.name_len, aname);
+	string name = "_" + aname;
+	if (pool.info.require_rollback()) {
+	  map > to_set;
+	  bufferlist old;
+	  int r = getattr_maybe_cache(ctx->obc, name, &old);
+	  if (r == 0) {
+	    to_set[name] = old;
+	  } else {
+	    to_set[name];
+	  }
+	  ctx->mod_desc.setattrs(to_set);
+	} else {
+	  ctx->mod_desc.mark_unrollbackable();
+	}
+	bufferlist bl;
+	bp.copy(op.xattr.value_len, bl);
+	setattr_maybe_cache(ctx->obc, ctx, t, name, bl);
+ 	ctx->delta_stats.num_wr++;
+      }
+      break;
+
+    case CEPH_OSD_OP_RMXATTR:
+      ++ctx->num_write;
+      {
+	string aname;
+	bp.copy(op.xattr.name_len, aname);
+	string name = "_" + aname;
+	if (pool.info.require_rollback()) {
+	  map > to_set;
+	  bufferlist old;
+	  int r = getattr_maybe_cache(ctx->obc, name, &old);
+	  if (r == 0) {
+	    to_set[name] = old;
+	  } else {
+	    to_set[name];
+	  }
+	  ctx->mod_desc.setattrs(to_set);
+	} else {
+	  ctx->mod_desc.mark_unrollbackable();
+	}
+	rmattr_maybe_cache(ctx->obc, ctx, t, name);
+ 	ctx->delta_stats.num_wr++;
+      }
+      break;
+    
+
+      // -- fancy writers --
+    case CEPH_OSD_OP_APPEND:
+      {
+	// just do it inline; this works because we are happy to execute
+	// fancy op on replicas as well.
+	vector nops(1);
+	OSDOp& newop = nops[0];
+	newop.op.op = CEPH_OSD_OP_WRITE;
+	newop.op.extent.offset = oi.size;
+	newop.op.extent.length = op.extent.length;
+	newop.op.extent.truncate_seq = oi.truncate_seq;
+        newop.indata = osd_op.indata;
+	result = do_osd_ops(ctx, nops);
+	osd_op.outdata.claim(newop.outdata);
+      }
+      break;
+
+    case CEPH_OSD_OP_STARTSYNC:
+      t->nop();
+      break;
+
+
+      // -- trivial map --
+    case CEPH_OSD_OP_TMAPGET:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_read;
+      {
+	vector nops(1);
+	OSDOp& newop = nops[0];
+	newop.op.op = CEPH_OSD_OP_SYNC_READ;
+	newop.op.extent.offset = 0;
+	newop.op.extent.length = 0;
+	do_osd_ops(ctx, nops);
+	osd_op.outdata.claim(newop.outdata);
+      }
+      break;
+
+    case CEPH_OSD_OP_TMAPPUT:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_write;
+      {
+	//_dout_lock.Lock();
+	//osd_op.data.hexdump(*_dout);
+	//_dout_lock.Unlock();
+
+	// verify sort order
+	bool unsorted = false;
+	if (true) {
+	  bufferlist header;
+	  ::decode(header, bp);
+	  uint32_t n;
+	  ::decode(n, bp);
+	  string last_key;
+	  while (n--) {
+	    string key;
+	    ::decode(key, bp);
+	    dout(10) << "tmapput key " << key << dendl;
+	    bufferlist val;
+	    ::decode(val, bp);
+	    if (key < last_key) {
+	      dout(10) << "TMAPPUT is unordered; resorting" << dendl;
+	      unsorted = true;
+	      break;
+	    }
+	    last_key = key;
+	  }
+	}
+
+	// write it
+	vector nops(1);
+	OSDOp& newop = nops[0];
+	newop.op.op = CEPH_OSD_OP_WRITEFULL;
+	newop.op.extent.offset = 0;
+	newop.op.extent.length = osd_op.indata.length();
+	newop.indata = osd_op.indata;
+
+	if (unsorted) {
+	  bp = osd_op.indata.begin();
+	  bufferlist header;
+	  map m;
+	  ::decode(header, bp);
+	  ::decode(m, bp);
+	  assert(bp.end());
+	  bufferlist newbl;
+	  ::encode(header, newbl);
+	  ::encode(m, newbl);
+	  newop.indata = newbl;
+	}
+	do_osd_ops(ctx, nops);
+      }
+      break;
+
+    case CEPH_OSD_OP_TMAPUP:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ++ctx->num_write;
+      result = do_tmapup(ctx, bp, osd_op);
+      break;
+
+    case CEPH_OSD_OP_TMAP2OMAP:
+      ++ctx->num_write;
+      result = do_tmap2omap(ctx, op.tmap2omap.flags);
+      break;
+
+      // OMAP Read ops
+    case CEPH_OSD_OP_OMAPGETKEYS:
+      ++ctx->num_read;
+      {
+	string start_after;
+	uint64_t max_return;
+	try {
+	  ::decode(start_after, bp);
+	  ::decode(max_return, bp);
+	}
+	catch (buffer::error& e) {
+	  result = -EINVAL;
+	  goto fail;
+	}
+	set out_set;
+
+	if (!pool.info.require_rollback()) {
+	  ObjectMap::ObjectMapIterator iter = osd->store->get_omap_iterator(
+	    coll, soid
+	    );
+	  assert(iter);
+	  iter->upper_bound(start_after);
+	  for (uint64_t i = 0;
+	       i < max_return && iter->valid();
+	       ++i, iter->next()) {
+	    out_set.insert(iter->key());
+	  }
+	} // else return empty out_set
+	::encode(out_set, osd_op.outdata);
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(osd_op.outdata.length(), 10);
+	ctx->delta_stats.num_rd++;
+      }
+      break;
+
+    case CEPH_OSD_OP_OMAPGETVALS:
+      ++ctx->num_read;
+      {
+	string start_after;
+	uint64_t max_return;
+	string filter_prefix;
+	try {
+	  ::decode(start_after, bp);
+	  ::decode(max_return, bp);
+	  ::decode(filter_prefix, bp);
+	}
+	catch (buffer::error& e) {
+	  result = -EINVAL;
+	  goto fail;
+	}
+	map out_set;
+
+	if (!pool.info.require_rollback()) {
+	  ObjectMap::ObjectMapIterator iter = osd->store->get_omap_iterator(
+	    coll, soid
+	    );
+          if (!iter) {
+            result = -ENOENT;
+            goto fail;
+          }
+	  iter->upper_bound(start_after);
+	  if (filter_prefix >= start_after) iter->lower_bound(filter_prefix);
+	  for (uint64_t i = 0;
+	       i < max_return && iter->valid() &&
+		 iter->key().substr(0, filter_prefix.size()) == filter_prefix;
+	       ++i, iter->next()) {
+	    dout(20) << "Found key " << iter->key() << dendl;
+	    out_set.insert(make_pair(iter->key(), iter->value()));
+	  }
+	} // else return empty out_set
+	::encode(out_set, osd_op.outdata);
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(osd_op.outdata.length(), 10);
+	ctx->delta_stats.num_rd++;
+      }
+      break;
+
+    case CEPH_OSD_OP_OMAPGETHEADER:
+      if (pool.info.require_rollback()) {
+	// return empty header
+	break;
+      }
+      ++ctx->num_read;
+      {
+	osd->store->omap_get_header(coll, soid, &osd_op.outdata);
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(osd_op.outdata.length(), 10);
+	ctx->delta_stats.num_rd++;
+      }
+      break;
+
+    case CEPH_OSD_OP_OMAPGETVALSBYKEYS:
+      ++ctx->num_read;
+      {
+	set keys_to_get;
+	try {
+	  ::decode(keys_to_get, bp);
+	}
+	catch (buffer::error& e) {
+	  result = -EINVAL;
+	  goto fail;
+	}
+	map out;
+	if (!pool.info.require_rollback()) {
+	  osd->store->omap_get_values(coll, soid, keys_to_get, &out);
+	} // else return empty omap entries
+	::encode(out, osd_op.outdata);
+	ctx->delta_stats.num_rd_kb += SHIFT_ROUND_UP(osd_op.outdata.length(), 10);
+	ctx->delta_stats.num_rd++;
+      }
+      break;
+
+    case CEPH_OSD_OP_OMAP_CMP:
+      ++ctx->num_read;
+      {
+	if (!obs.exists || oi.is_whiteout()) {
+	  result = -ENOENT;
+	  break;
+	}
+	map > assertions;
+	try {
+	  ::decode(assertions, bp);
+	}
+	catch (buffer::error& e) {
+	  result = -EINVAL;
+	  goto fail;
+	}
+	
+	map out;
+
+	if (!pool.info.require_rollback()) {
+	  set to_get;
+	  for (map >::iterator i = assertions.begin();
+	       i != assertions.end();
+	       ++i)
+	    to_get.insert(i->first);
+	  int r = osd->store->omap_get_values(coll, soid, to_get, &out);
+	  if (r < 0) {
+	    result = r;
+	    break;
+	  }
+	} // else leave out empty
+
+	//Should set num_rd_kb based on encode length of map
+	ctx->delta_stats.num_rd++;
+
+	int r = 0;
+	bufferlist empty;
+	for (map >::iterator i = assertions.begin();
+	     i != assertions.end();
+	     ++i) {
+	  bufferlist &bl = out.count(i->first) ? 
+	    out[i->first] : empty;
+	  switch (i->second.second) {
+	  case CEPH_OSD_CMPXATTR_OP_EQ:
+	    if (!(bl == i->second.first)) {
+	      r = -ECANCELED;
+	    }
+	    break;
+	  case CEPH_OSD_CMPXATTR_OP_LT:
+	    if (!(bl < i->second.first)) {
+	      r = -ECANCELED;
+	    }
+	    break;
+	  case CEPH_OSD_CMPXATTR_OP_GT:
+	    if (!(bl > i->second.first)) {
+	      r = -ECANCELED;
+	    }
+	    break;
+	  default:
+	    r = -EINVAL;
+	    break;
+	  }
+	  if (r < 0)
+	    break;
+	}
+	if (r < 0) {
+	  result = r;
+	}
+      }
+      break;
+
+      // OMAP Write ops
+    case CEPH_OSD_OP_OMAPSETVALS:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ctx->mod_desc.mark_unrollbackable();
+      ++ctx->num_write;
+      {
+	if (!obs.exists) {
+	  ctx->delta_stats.num_objects++;
+	  obs.exists = true;
+	}
+	t->touch(soid);
+	map to_set;
+	try {
+	  ::decode(to_set, bp);
+	}
+	catch (buffer::error& e) {
+	  result = -EINVAL;
+	  goto fail;
+	}
+	dout(20) << "setting vals: " << dendl;
+	for (map::iterator i = to_set.begin();
+	     i != to_set.end();
+	     ++i) {
+	  dout(20) << "\t" << i->first << dendl;
+	}
+	t->omap_setkeys(soid, to_set);
+	ctx->delta_stats.num_wr++;
+      }
+      obs.oi.set_flag(object_info_t::FLAG_OMAP);
+      break;
+
+    case CEPH_OSD_OP_OMAPSETHEADER:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ctx->mod_desc.mark_unrollbackable();
+      ++ctx->num_write;
+      {
+	if (!obs.exists) {
+	  ctx->delta_stats.num_objects++;
+	  obs.exists = true;
+	}
+	t->touch(soid);
+	t->omap_setheader(soid, osd_op.indata);
+	ctx->delta_stats.num_wr++;
+      }
+      obs.oi.set_flag(object_info_t::FLAG_OMAP);
+      break;
+
+    case CEPH_OSD_OP_OMAPCLEAR:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ctx->mod_desc.mark_unrollbackable();
+      ++ctx->num_write;
+      {
+	if (!obs.exists || oi.is_whiteout()) {
+	  result = -ENOENT;
+	  break;
+	}
+	t->touch(soid);
+	t->omap_clear(soid);
+	ctx->delta_stats.num_wr++;
+      }
+      obs.oi.set_flag(object_info_t::FLAG_OMAP);
+      break;
+
+    case CEPH_OSD_OP_OMAPRMKEYS:
+      if (pool.info.require_rollback()) {
+	result = -EOPNOTSUPP;
+	break;
+      }
+      ctx->mod_desc.mark_unrollbackable();
+      ++ctx->num_write;
+      {
+	if (!obs.exists || oi.is_whiteout()) {
+	  result = -ENOENT;
+	  break;
+	}
+	t->touch(soid);
+	set to_rm;
+	try {
+	  ::decode(to_rm, bp);
+	}
+	catch (buffer::error& e) {
+	  result = -EINVAL;
+	  goto fail;
+	}
+	t->omap_rmkeys(soid, to_rm);
+	ctx->delta_stats.num_wr++;
+      }
+      obs.oi.set_flag(object_info_t::FLAG_OMAP);
+      break;
+
+    case CEPH_OSD_OP_COPY_GET_CLASSIC:
+      ++ctx->num_read;
+      result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, true);
+      if (result == -EINVAL)
+	goto fail;
+      break;
+
+    case CEPH_OSD_OP_COPY_GET:
+      ++ctx->num_read;
+      result = fill_in_copy_get(ctx, bp, osd_op, ctx->obc, false);
+      if (result == -EINVAL)
+	goto fail;
+      break;
+
+    case CEPH_OSD_OP_COPY_FROM:
+      ++ctx->num_write;
+      {
+	object_t src_name;
+	object_locator_t src_oloc;
+	snapid_t src_snapid = (uint64_t)op.copy_from.snapid;
+	version_t src_version = op.copy_from.src_version;
+	try {
+	  ::decode(src_name, bp);
+	  ::decode(src_oloc, bp);
+	}
+	catch (buffer::error& e) {
+	  result = -EINVAL;
+	  goto fail;
+	}
+	if (!ctx->copy_cb) {
+	  // start
+	  pg_t raw_pg;
+	  get_osdmap()->object_locator_to_pg(src_name, src_oloc, raw_pg);
+	  hobject_t src(src_name, src_oloc.key, src_snapid,
+			raw_pg.ps(), raw_pg.pool(),
+			src_oloc.nspace);
+	  if (src == soid) {
+	    dout(20) << " copy from self is invalid" << dendl;
+	    result = -EINVAL;
+	    break;
+	  }
+	  CopyFromCallback *cb = new CopyFromCallback(ctx);
+	  ctx->copy_cb = cb;
+	  start_copy(cb, ctx->obc, src, src_oloc, src_version,
+		     op.copy_from.flags,
+		     false);
+	  result = -EINPROGRESS;
+	} else {
+	  // finish
+	  assert(ctx->copy_cb->get_result() >= 0);
+	  finish_copyfrom(ctx);
+	  result = 0;
+	}
+      }
+      break;
+
+    default:
+      dout(1) << "unrecognized osd op " << op.op
+	      << " " << ceph_osd_op_name(op.op)
+	      << dendl;
+      result = -EOPNOTSUPP;
+    }
+
+    ctx->bytes_read += osd_op.outdata.length();
+
+  fail:
+    osd_op.rval = result;
+    if (result < 0 && (op.flags & CEPH_OSD_OP_FLAG_FAILOK))
+      result = 0;
+
+    if (result < 0)
+      break;
+  }
+  return result;
+}
+
+int ReplicatedPG::_get_tmap(OpContext *ctx, bufferlist *header, bufferlist *vals)
+{
+  if (ctx->new_obs.oi.size == 0) {
+    dout(20) << "unable to get tmap for zero sized " << ctx->new_obs.oi.soid << dendl;
+    return -ENODATA;
+  }
+  vector nops(1);
+  OSDOp &newop = nops[0];
+  newop.op.op = CEPH_OSD_OP_TMAPGET;
+  do_osd_ops(ctx, nops);
+  try {
+    bufferlist::iterator i = newop.outdata.begin();
+    ::decode(*header, i);
+    (*vals).substr_of(newop.outdata, i.get_off(), i.get_remaining());
+  } catch (...) {
+    dout(20) << "unsuccessful at decoding tmap for " << ctx->new_obs.oi.soid
+	     << dendl;
+    return -EINVAL;
+  }
+  dout(20) << "successful at decoding tmap for " << ctx->new_obs.oi.soid
+	   << dendl;
+  return 0;
+}
+
+int ReplicatedPG::_verify_no_head_clones(const hobject_t& soid,
+					const SnapSet& ss)
+{
+  // verify that all clones have been evicted
+  dout(20) << __func__ << " verifying clones are absent "
+	   << ss << dendl;
+  for (vector::const_iterator p = ss.clones.begin();
+       p != ss.clones.end();
+       ++p) {
+    hobject_t clone_oid = soid;
+    clone_oid.snap = *p;
+    if (is_missing_object(clone_oid))
+      return -EBUSY;
+    ObjectContextRef clone_obc = get_object_context(clone_oid, false);
+    if (clone_obc && clone_obc->obs.exists) {
+      dout(10) << __func__ << " cannot evict head before clone "
+	       << clone_oid << dendl;
+      return -EBUSY;
+    }
+    if (copy_ops.count(clone_oid)) {
+      dout(10) << __func__ << " cannot evict head, pending promote on clone "
+	       << clone_oid << dendl;
+      return -EBUSY;
+    }
+  }
+  return 0;
+}
+
+inline int ReplicatedPG::_delete_oid(OpContext *ctx, bool no_whiteout)
+{
+  SnapSet& snapset = ctx->new_snapset;
+  ObjectState& obs = ctx->new_obs;
+  object_info_t& oi = obs.oi;
+  const hobject_t& soid = oi.soid;
+  PGBackend::PGTransaction* t = ctx->op_t;
+
+  if (!obs.exists || (obs.oi.is_whiteout() && !no_whiteout))
+    return -ENOENT;
+
+  if (pool.info.require_rollback()) {
+    if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+      t->stash(soid, ctx->at_version.version);
+    } else {
+      t->remove(soid);
+    }
+    map new_attrs;
+    replace_cached_attrs(ctx, ctx->obc, new_attrs);
+  } else {
+    ctx->mod_desc.mark_unrollbackable();
+    t->remove(soid);
+  }
+
+  if (oi.size > 0) {
+    interval_set ch;
+    ch.insert(0, oi.size);
+    ctx->modified_ranges.union_of(ch);
+  }
+
+  ctx->delta_stats.num_wr++;
+  if (soid.is_snap()) {
+    assert(ctx->obc->ssc->snapset.clone_overlap.count(soid.snap));
+    ctx->delta_stats.num_bytes -= ctx->obc->ssc->snapset.get_clone_bytes(soid.snap);
+  } else {
+    ctx->delta_stats.num_bytes -= oi.size;
+  }
+  oi.size = 0;
+
+  // cache: cache: set whiteout on delete?
+  if (pool.info.cache_mode != pg_pool_t::CACHEMODE_NONE && !no_whiteout) {
+    dout(20) << __func__ << " setting whiteout on " << soid << dendl;
+    oi.set_flag(object_info_t::FLAG_WHITEOUT);
+    ctx->delta_stats.num_whiteouts++;
+    t->touch(soid);
+    osd->logger->inc(l_osd_tier_whiteout);
+    return 0;
+  }
+
+  ctx->delta_stats.num_objects--;
+  if (soid.is_snap())
+    ctx->delta_stats.num_object_clones--;
+  if (oi.is_whiteout()) {
+    dout(20) << __func__ << " deleting whiteout on " << soid << dendl;
+    ctx->delta_stats.num_whiteouts--;
+  }
+  if (soid.is_head())
+    snapset.head_exists = false;
+  obs.exists = false;
+  return 0;
+}
+
+int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
+{
+  SnapSet& snapset = ctx->new_snapset;
+  ObjectState& obs = ctx->new_obs;
+  object_info_t& oi = obs.oi;
+  const hobject_t& soid = oi.soid;
+  PGBackend::PGTransaction* t = ctx->op_t;
+  snapid_t snapid = (uint64_t)op.snap.snapid;
+  hobject_t missing_oid;
+
+  dout(10) << "_rollback_to " << soid << " snapid " << snapid << dendl;
+
+  ObjectContextRef rollback_to;
+  int ret = find_object_context(
+    hobject_t(soid.oid, soid.get_key(), snapid, soid.hash, info.pgid.pool(),
+	      soid.get_namespace()),
+    &rollback_to, false, false, &missing_oid);
+  if (ret == -EAGAIN) {
+    /* clone must be missing */
+    assert(is_missing_object(missing_oid));
+    dout(20) << "_rollback_to attempted to roll back to a missing object "
+	     << missing_oid << " (requested snapid: ) " << snapid << dendl;
+    wait_for_unreadable_object(missing_oid, ctx->op);
+    return ret;
+  }
+  if (maybe_handle_cache(ctx->op, true, rollback_to, ret, missing_oid, true)) {
+    // promoting the rollback src, presumably
+    return -EAGAIN;
+  }
+  if (ret == -ENOENT || (rollback_to && rollback_to->obs.oi.is_whiteout())) {
+    // there's no snapshot here, or there's no object.
+    // if there's no snapshot, we delete the object; otherwise, do nothing.
+    dout(20) << "_rollback_to deleting head on " << soid.oid
+	     << " because got ENOENT|whiteout on find_object_context" << dendl;
+    if (ctx->obc->obs.oi.watchers.size()) {
+      // Cannot delete an object with watchers
+      ret = -EBUSY;
+    } else {
+      _delete_oid(ctx, false);
+      ret = 0;
+    }
+  } else if (ret) {
+    // ummm....huh? It *can't* return anything else at time of writing.
+    assert(0 == "unexpected error code in _rollback_to");
+  } else { //we got our context, let's use it to do the rollback!
+    hobject_t& rollback_to_sobject = rollback_to->obs.oi.soid;
+    if (is_degraded_object(rollback_to_sobject)) {
+      dout(20) << "_rollback_to attempted to roll back to a degraded object "
+	       << rollback_to_sobject << " (requested snapid: ) " << snapid << dendl;
+      wait_for_degraded_object(rollback_to_sobject, ctx->op);
+      ret = -EAGAIN;
+    } else if (rollback_to->obs.oi.soid.snap == CEPH_NOSNAP) {
+      // rolling back to the head; we just need to clone it.
+      ctx->modify = true;
+    } else {
+      /* 1) Delete current head
+       * 2) Clone correct snapshot into head
+       * 3) Calculate clone_overlaps by following overlaps
+       *    forward from rollback snapshot */
+      dout(10) << "_rollback_to deleting " << soid.oid
+	       << " and rolling back to old snap" << dendl;
+
+      if (pool.info.require_rollback()) {
+	if (obs.exists) {
+	  if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+	    t->stash(soid, ctx->at_version.version);
+	  } else {
+	    t->remove(soid);
+	  }
+	}
+	replace_cached_attrs(ctx, ctx->obc, rollback_to->attr_cache);
+      } else {
+	if (obs.exists) {
+	  ctx->mod_desc.mark_unrollbackable();
+	  t->remove(soid);
+	}
+      }
+      ctx->mod_desc.create();
+      t->clone(rollback_to_sobject, soid);
+      snapset.head_exists = true;
+
+      map >::iterator iter =
+	snapset.clone_overlap.lower_bound(snapid);
+      interval_set overlaps = iter->second;
+      assert(iter != snapset.clone_overlap.end());
+      for ( ;
+	    iter != snapset.clone_overlap.end();
+	    ++iter)
+	overlaps.intersection_of(iter->second);
+
+      if (obs.oi.size > 0) {
+	interval_set modified;
+	modified.insert(0, obs.oi.size);
+	overlaps.intersection_of(modified);
+	modified.subtract(overlaps);
+	ctx->modified_ranges.union_of(modified);
+      }
+
+      // Adjust the cached objectcontext
+      if (!obs.exists) {
+	obs.exists = true; //we're about to recreate it
+	ctx->delta_stats.num_objects++;
+      }
+      ctx->delta_stats.num_bytes -= obs.oi.size;
+      ctx->delta_stats.num_bytes += rollback_to->obs.oi.size;
+      obs.oi.size = rollback_to->obs.oi.size;
+      snapset.head_exists = true;
+    }
+  }
+  return ret;
+}
+
+void ReplicatedPG::_make_clone(
+  OpContext *ctx,
+  PGBackend::PGTransaction* t,
+  ObjectContextRef obc,
+  const hobject_t& head, const hobject_t& coid,
+  object_info_t *poi)
+{
+  bufferlist bv;
+  ::encode(*poi, bv);
+
+  t->clone(head, coid);
+  setattr_maybe_cache(obc, ctx, t, OI_ATTR, bv);
+  rmattr_maybe_cache(obc, ctx, t, SS_ATTR);
+}
+
+void ReplicatedPG::make_writeable(OpContext *ctx)
+{
+  const hobject_t& soid = ctx->obs->oi.soid;
+  SnapContext& snapc = ctx->snapc;
+  PGBackend::PGTransaction *t = pgbackend->get_transaction();
+
+  // clone?
+  assert(soid.snap == CEPH_NOSNAP);
+  dout(20) << "make_writeable " << soid << " snapset=" << ctx->snapset
+	   << "  snapc=" << snapc << dendl;;
+  
+  bool was_dirty = ctx->obc->obs.oi.is_dirty();
+  if (ctx->new_obs.exists) {
+    // we will mark the object dirty
+    if (ctx->undirty && was_dirty) {
+      dout(20) << " clearing DIRTY flag" << dendl;
+      assert(ctx->new_obs.oi.is_dirty());
+      ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
+      --ctx->delta_stats.num_objects_dirty;
+      osd->logger->inc(l_osd_tier_clean);
+    } else if (!was_dirty && !ctx->undirty) {
+      dout(20) << " setting DIRTY flag" << dendl;
+      ctx->new_obs.oi.set_flag(object_info_t::FLAG_DIRTY);
+      ++ctx->delta_stats.num_objects_dirty;
+      osd->logger->inc(l_osd_tier_dirty);
+    }
+  } else {
+    if (was_dirty) {
+      dout(20) << " deletion, decrementing num_dirty and clearing flag" << dendl;
+      ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
+      --ctx->delta_stats.num_objects_dirty;
+    }
+  }
+
+  if ((ctx->new_obs.exists &&
+       ctx->new_obs.oi.is_omap()) &&
+      (!ctx->obc->obs.exists ||
+       !ctx->obc->obs.oi.is_omap())) {
+    ++ctx->delta_stats.num_objects_omap;
+  }
+  if ((!ctx->new_obs.exists ||
+       !ctx->new_obs.oi.is_omap()) &&
+      (ctx->obc->obs.exists &&
+       ctx->obc->obs.oi.is_omap())) {
+    --ctx->delta_stats.num_objects_omap;
+  }
+
+  // use newer snapc?
+  if (ctx->new_snapset.seq > snapc.seq) {
+    snapc.seq = ctx->new_snapset.seq;
+    snapc.snaps = ctx->new_snapset.snaps;
+    dout(10) << " using newer snapc " << snapc << dendl;
+  }
+
+  if (ctx->obs->exists)
+    filter_snapc(snapc.snaps);
+  
+  if ((ctx->obs->exists && !ctx->obs->oi.is_whiteout()) && // head exist(ed)
+      snapc.snaps.size() &&                 // there are snaps
+      snapc.snaps[0] > ctx->new_snapset.seq) {  // existing object is old
+    // clone
+    hobject_t coid = soid;
+    coid.snap = snapc.seq;
+    
+    unsigned l;
+    for (l=1; l ctx->new_snapset.seq; l++) ;
+    
+    vector snaps(l);
+    for (unsigned i=0; iclone_obc = object_contexts.lookup_or_create(static_snap_oi.soid);
+      ctx->clone_obc->destructor_callback = new C_PG_ObjectContext(this, ctx->clone_obc.get());
+      ctx->clone_obc->obs.oi = static_snap_oi;
+      ctx->clone_obc->obs.exists = true;
+      ctx->clone_obc->ssc = ctx->obc->ssc;
+      ctx->clone_obc->ssc->ref++;
+      if (pool.info.require_rollback())
+	ctx->clone_obc->attr_cache = ctx->obc->attr_cache;
+      snap_oi = &ctx->clone_obc->obs.oi;
+      bool got = ctx->clone_obc->get_write_greedy(ctx->op);
+      assert(got);
+      dout(20) << " got greedy write on clone_obc " << *ctx->clone_obc << dendl;
+    } else {
+      snap_oi = &static_snap_oi;
+    }
+    snap_oi->version = ctx->at_version;
+    snap_oi->prior_version = ctx->obs->oi.version;
+    snap_oi->copy_user_bits(ctx->obs->oi);
+    snap_oi->snaps = snaps;
+    if (was_dirty)
+      snap_oi->set_flag(object_info_t::FLAG_DIRTY);
+    _make_clone(ctx, t, ctx->clone_obc, soid, coid, snap_oi);
+    
+    ctx->delta_stats.num_objects++;
+    if (snap_oi->is_dirty())
+      ctx->delta_stats.num_objects_dirty++;
+    if (snap_oi->is_whiteout()) {
+      dout(20) << __func__ << " cloning whiteout on " << soid << " to " << coid << dendl;
+      ctx->delta_stats.num_whiteouts++;
+    }
+    if (snap_oi->is_omap())
+      ctx->delta_stats.num_objects_omap++;
+    ctx->delta_stats.num_object_clones++;
+    ctx->new_snapset.clones.push_back(coid.snap);
+    ctx->new_snapset.clone_size[coid.snap] = ctx->obs->oi.size;
+
+    // clone_overlap should contain an entry for each clone 
+    // (an empty interval_set if there is no overlap)
+    ctx->new_snapset.clone_overlap[coid.snap];
+    if (ctx->obs->oi.size)
+      ctx->new_snapset.clone_overlap[coid.snap].insert(0, ctx->obs->oi.size);
+    
+    // log clone
+    dout(10) << " cloning v " << ctx->obs->oi.version
+	     << " to " << coid << " v " << ctx->at_version
+	     << " snaps=" << snaps << dendl;
+    ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::CLONE, coid, ctx->at_version,
+				      ctx->obs->oi.version,
+				      ctx->obs->oi.user_version,
+				      osd_reqid_t(), ctx->new_obs.oi.mtime));
+    ::encode(snaps, ctx->log.back().snaps);
+    ctx->log.back().mod_desc.create();
+
+    ctx->at_version.version++;
+  }
+
+  // update most recent clone_overlap and usage stats
+  if (ctx->new_snapset.clones.size() > 0) {
+    /* we need to check whether the most recent clone exists, if it's been evicted,
+     * it's not included in the stats */
+    hobject_t last_clone_oid = soid;
+    last_clone_oid.snap = ctx->new_snapset.clone_overlap.rbegin()->first;
+    if (is_present_clone(last_clone_oid)) {
+      interval_set &newest_overlap = ctx->new_snapset.clone_overlap.rbegin()->second;
+      ctx->modified_ranges.intersection_of(newest_overlap);
+      // modified_ranges is still in use by the clone
+      add_interval_usage(ctx->modified_ranges, ctx->delta_stats);
+      newest_overlap.subtract(ctx->modified_ranges);
+    }
+  }
+  
+  // prepend transaction to op_t
+  t->append(ctx->op_t);
+  delete ctx->op_t;
+  ctx->op_t = t;
+
+  // update snapset with latest snap context
+  ctx->new_snapset.seq = snapc.seq;
+  ctx->new_snapset.snaps = snapc.snaps;
+  ctx->new_snapset.head_exists = ctx->new_obs.exists;
+  dout(20) << "make_writeable " << soid << " done, snapset=" << ctx->new_snapset << dendl;
+}
+
+
+void ReplicatedPG::write_update_size_and_usage(object_stat_sum_t& delta_stats, object_info_t& oi,
+					       SnapSet& ss, interval_set& modified,
+					       uint64_t offset, uint64_t length, bool count_bytes)
+{
+  interval_set ch;
+  if (length)
+    ch.insert(offset, length);
+  modified.union_of(ch);
+  if (length && (offset + length > oi.size)) {
+    uint64_t new_size = offset + length;
+    delta_stats.num_bytes += new_size - oi.size;
+    oi.size = new_size;
+  }
+  delta_stats.num_wr++;
+  if (count_bytes)
+    delta_stats.num_wr_kb += SHIFT_ROUND_UP(length, 10);
+}
+
+void ReplicatedPG::add_interval_usage(interval_set& s, object_stat_sum_t& delta_stats)
+{
+  for (interval_set::const_iterator p = s.begin(); p != s.end(); ++p) {
+    delta_stats.num_bytes += p.get_len();
+  }
+}
+
+void ReplicatedPG::do_osd_op_effects(OpContext *ctx)
+{
+  ConnectionRef conn(ctx->op->get_req()->get_connection());
+  boost::intrusive_ptr session(
+    (OSD::Session *)conn->get_priv());
+  session->put();  // get_priv() takes a ref, and so does the intrusive_ptr
+  entity_name_t entity = ctx->reqid.name;
+
+  dout(15) << "do_osd_op_effects on session " << session.get() << dendl;
+
+  for (list::iterator i = ctx->watch_connects.begin();
+       i != ctx->watch_connects.end();
+       ++i) {
+    pair watcher(i->cookie, entity);
+    dout(15) << "do_osd_op_effects applying watch connect on session "
+	     << session.get() << " watcher " << watcher << dendl;
+    WatchRef watch;
+    if (ctx->obc->watchers.count(watcher)) {
+      dout(15) << "do_osd_op_effects found existing watch watcher " << watcher
+	       << dendl;
+      watch = ctx->obc->watchers[watcher];
+    } else {
+      dout(15) << "do_osd_op_effects new watcher " << watcher
+	       << dendl;
+      watch = Watch::makeWatchRef(
+	this, osd, ctx->obc, i->timeout_seconds,
+	i->cookie, entity, conn->get_peer_addr());
+      ctx->obc->watchers.insert(
+	make_pair(
+	  watcher,
+	  watch));
+    }
+    watch->connect(conn);
+  }
+
+  for (list::iterator i = ctx->watch_disconnects.begin();
+       i != ctx->watch_disconnects.end();
+       ++i) {
+    pair watcher(i->cookie, entity);
+    dout(15) << "do_osd_op_effects applying watch disconnect on session "
+	     << session.get() << " and watcher " << watcher << dendl;
+    if (ctx->obc->watchers.count(watcher)) {
+      WatchRef watch = ctx->obc->watchers[watcher];
+      dout(10) << "do_osd_op_effects applying disconnect found watcher "
+	       << watcher << dendl;
+      ctx->obc->watchers.erase(watcher);
+      watch->remove();
+    } else {
+      dout(10) << "do_osd_op_effects failed to find watcher "
+	       << watcher << dendl;
+    }
+  }
+
+  for (list::iterator p = ctx->notifies.begin();
+       p != ctx->notifies.end();
+       ++p) {
+    dout(10) << "do_osd_op_effects, notify " << *p << dendl;
+    NotifyRef notif(
+      Notify::makeNotifyRef(
+	conn,
+	ctx->obc->watchers.size(),
+	p->bl,
+	p->timeout,
+	p->cookie,
+	osd->get_next_id(get_osdmap()->get_epoch()),
+	ctx->obc->obs.oi.user_version,
+	osd));
+    for (map, WatchRef>::iterator i =
+	   ctx->obc->watchers.begin();
+	 i != ctx->obc->watchers.end();
+	 ++i) {
+      dout(10) << "starting notify on watch " << i->first << dendl;
+      i->second->start_notify(notif);
+    }
+    notif->init();
+  }
+
+  for (list::iterator p = ctx->notify_acks.begin();
+       p != ctx->notify_acks.end();
+       ++p) {
+    dout(10) << "notify_ack " << make_pair(p->watch_cookie, p->notify_id) << dendl;
+    for (map, WatchRef>::iterator i =
+	   ctx->obc->watchers.begin();
+	 i != ctx->obc->watchers.end();
+	 ++i) {
+      if (i->first.second != entity) continue;
+      if (p->watch_cookie &&
+	  p->watch_cookie.get() != i->first.first) continue;
+      dout(10) << "acking notify on watch " << i->first << dendl;
+      i->second->notify_ack(p->notify_id);
+    }
+  }
+}
+
+coll_t ReplicatedPG::get_temp_coll(ObjectStore::Transaction *t)
+{
+  return pgbackend->get_temp_coll(t);
+}
+
+hobject_t ReplicatedPG::generate_temp_object()
+{
+  ostringstream ss;
+  ss << "temp_" << info.pgid << "_" << get_role() << "_" << osd->monc->get_global_id() << "_" << (++temp_seq);
+  hobject_t hoid = hobject_t::make_temp(ss.str());
+  dout(20) << __func__ << " " << hoid << dendl;
+  return hoid;
+}
+
+int ReplicatedPG::prepare_transaction(OpContext *ctx)
+{
+  assert(!ctx->ops.empty());
+  
+  const hobject_t& soid = ctx->obs->oi.soid;
+
+  // valid snap context?
+  if (!ctx->snapc.is_valid()) {
+    dout(10) << " invalid snapc " << ctx->snapc << dendl;
+    return -EINVAL;
+  }
+
+  // prepare the actual mutation
+  int result = do_osd_ops(ctx, ctx->ops);
+  if (result < 0)
+    return result;
+
+  // finish side-effects
+  if (result == 0)
+    do_osd_op_effects(ctx);
+
+  // read-op?  done?
+  if (ctx->op_t->empty() && !ctx->modify) {
+    unstable_stats.add(ctx->delta_stats, ctx->obc->obs.oi.category);
+    return result;
+  }
+
+  // cache: clear whiteout?
+  if (pool.info.cache_mode != pg_pool_t::CACHEMODE_NONE) {
+    if (ctx->user_modify &&
+	ctx->obc->obs.oi.is_whiteout()) {
+      dout(10) << __func__ << " clearing whiteout on " << soid << dendl;
+      ctx->new_obs.oi.clear_flag(object_info_t::FLAG_WHITEOUT);
+      --ctx->delta_stats.num_whiteouts;
+    }
+  }
+
+  // clone, if necessary
+  if (soid.snap == CEPH_NOSNAP)
+    make_writeable(ctx);
+
+  finish_ctx(ctx,
+	     ctx->new_obs.exists ? pg_log_entry_t::MODIFY :
+	     pg_log_entry_t::DELETE);
+
+  return result;
+}
+
+void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type, bool maintain_ssc)
+{
+  const hobject_t& soid = ctx->obs->oi.soid;
+  dout(20) << __func__ << " " << soid << " " << ctx
+	   << " op " << pg_log_entry_t::get_op_name(log_op_type)
+	   << dendl;
+
+  // snapset
+  bufferlist bss;
+
+  if (soid.snap == CEPH_NOSNAP && maintain_ssc) {
+    ::encode(ctx->new_snapset, bss);
+    assert(ctx->new_obs.exists == ctx->new_snapset.head_exists);
+
+    if (ctx->new_obs.exists) {
+      if (!ctx->obs->exists) {
+	if (ctx->snapset_obc && ctx->snapset_obc->obs.exists) {
+	  hobject_t snapoid = soid.get_snapdir();
+	  ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::DELETE, snapoid,
+	      ctx->at_version,
+	      ctx->snapset_obc->obs.oi.version,
+	      0, osd_reqid_t(), ctx->mtime));
+	  if (pool.info.require_rollback()) {
+	    if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+	      ctx->op_t->stash(snapoid, ctx->at_version.version);
+	    } else {
+	      ctx->op_t->remove(snapoid);
+	    }
+	  } else {
+	    ctx->op_t->remove(snapoid);
+	    ctx->log.back().mod_desc.mark_unrollbackable();
+	  }
+	  dout(10) << " removing old " << snapoid << dendl;
+
+	  ctx->at_version.version++;
+
+	  ctx->snapset_obc->obs.exists = false;
+	}
+      }
+    } else if (ctx->new_snapset.clones.size()) {
+      // save snapset on _snap
+      hobject_t snapoid(soid.oid, soid.get_key(), CEPH_SNAPDIR, soid.hash,
+			info.pgid.pool(), soid.get_namespace());
+      dout(10) << " final snapset " << ctx->new_snapset
+	       << " in " << snapoid << dendl;
+      ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::MODIFY, snapoid,
+					ctx->at_version,
+	                                eversion_t(),
+					0, osd_reqid_t(), ctx->mtime));
+
+      ctx->snapset_obc = get_object_context(snapoid, true);
+      bool got = ctx->snapset_obc->get_write_greedy(ctx->op);
+      assert(got);
+      dout(20) << " got greedy write on snapset_obc " << *ctx->snapset_obc << dendl;
+      ctx->release_snapset_obc = true;
+      if (pool.info.require_rollback() && !ctx->snapset_obc->obs.exists) {
+	ctx->log.back().mod_desc.create();
+      } else if (!pool.info.require_rollback()) {
+	ctx->log.back().mod_desc.mark_unrollbackable();
+      }
+      ctx->snapset_obc->obs.exists = true;
+      ctx->snapset_obc->obs.oi.version = ctx->at_version;
+      ctx->snapset_obc->obs.oi.last_reqid = ctx->reqid;
+      ctx->snapset_obc->obs.oi.mtime = ctx->mtime;
+
+      bufferlist bv(sizeof(ctx->new_obs.oi));
+      ::encode(ctx->snapset_obc->obs.oi, bv);
+      ctx->op_t->touch(snapoid);
+      setattr_maybe_cache(ctx->snapset_obc, ctx, ctx->op_t, OI_ATTR, bv);
+      setattr_maybe_cache(ctx->snapset_obc, ctx, ctx->op_t, SS_ATTR, bss);
+      if (pool.info.require_rollback()) {
+	map > to_set;
+	to_set[SS_ATTR];
+	to_set[OI_ATTR];
+	ctx->log.back().mod_desc.setattrs(to_set);
+      } else {
+	ctx->log.back().mod_desc.mark_unrollbackable();
+      }
+      ctx->at_version.version++;
+    }
+  }
+
+  // finish and log the op.
+  if (ctx->user_modify) {
+    // update the user_version for any modify ops, except for the watch op
+    ctx->user_at_version = MAX(info.last_user_version, ctx->new_obs.oi.user_version) + 1;
+    /* In order for new clients and old clients to interoperate properly
+     * when exchanging versions, we need to lower bound the user_version
+     * (which our new clients pay proper attention to)
+     * by the at_version (which is all the old clients can ever see). */
+    if (ctx->at_version.version > ctx->user_at_version)
+      ctx->user_at_version = ctx->at_version.version;
+    ctx->new_obs.oi.user_version = ctx->user_at_version;
+  }
+  ctx->bytes_written = ctx->op_t->get_bytes_written();
+ 
+  if (ctx->new_obs.exists) {
+    // on the head object
+    ctx->new_obs.oi.version = ctx->at_version;
+    ctx->new_obs.oi.prior_version = ctx->obs->oi.version;
+    ctx->new_obs.oi.last_reqid = ctx->reqid;
+    if (ctx->mtime != utime_t()) {
+      ctx->new_obs.oi.mtime = ctx->mtime;
+      dout(10) << " set mtime to " << ctx->new_obs.oi.mtime << dendl;
+    } else {
+      dout(10) << " mtime unchanged at " << ctx->new_obs.oi.mtime << dendl;
+    }
+
+    bufferlist bv(sizeof(ctx->new_obs.oi));
+    ::encode(ctx->new_obs.oi, bv);
+    setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, OI_ATTR, bv);
+
+    if (soid.snap == CEPH_NOSNAP) {
+      dout(10) << " final snapset " << ctx->new_snapset
+	       << " in " << soid << dendl;
+      setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, SS_ATTR, bss);
+
+      if (pool.info.require_rollback()) {
+	set changing;
+	changing.insert(OI_ATTR);
+	changing.insert(SS_ATTR);
+	ctx->obc->fill_in_setattrs(changing, &(ctx->mod_desc));
+      } else {
+	// replicated pools are never rollbackable in this case
+	ctx->mod_desc.mark_unrollbackable();
+      }
+    } else {
+      dout(10) << " no snapset (this is a clone)" << dendl;
+    }
+  } else {
+    ctx->new_obs.oi = object_info_t(ctx->obc->obs.oi.soid);
+  }
+
+  // append to log
+  ctx->log.push_back(pg_log_entry_t(log_op_type, soid, ctx->at_version,
+				    ctx->obs->oi.version,
+				    ctx->user_at_version, ctx->reqid,
+				    ctx->mtime));
+  if (soid.snap < CEPH_NOSNAP) {
+    set _snaps(ctx->new_obs.oi.snaps.begin(),
+			 ctx->new_obs.oi.snaps.end());
+    switch (log_op_type) {
+    case pg_log_entry_t::MODIFY:
+    case pg_log_entry_t::PROMOTE:
+      dout(20) << __func__ << " encoding snaps " << ctx->new_obs.oi.snaps
+	       << dendl;
+      ::encode(ctx->new_obs.oi.snaps, ctx->log.back().snaps);
+      break;
+    case pg_log_entry_t::CLEAN:
+      dout(20) << __func__ << " encoding snaps " << ctx->new_obs.oi.snaps
+	       << dendl;
+      ::encode(ctx->new_obs.oi.snaps, ctx->log.back().snaps);
+      break;
+    default:
+      break;
+    }
+  }
+
+  ctx->log.back().mod_desc.claim(ctx->mod_desc);
+
+  // apply new object state.
+  ctx->obc->obs = ctx->new_obs;
+
+  if (!maintain_ssc && soid.is_head()) {
+    ctx->obc->ssc->exists = false;
+    ctx->obc->ssc->snapset = SnapSet();
+  } else {
+    ctx->obc->ssc->exists = true;
+    ctx->obc->ssc->snapset = ctx->new_snapset;
+  }
+
+  info.stats.stats.add(ctx->delta_stats, ctx->obs->oi.category);
+
+  for (set::iterator i = backfill_targets.begin();
+       i != backfill_targets.end();
+       ++i) {
+    pg_shard_t bt = *i;
+    pg_info_t& pinfo = peer_info[bt];
+    if (soid <= pinfo.last_backfill)
+      pinfo.stats.stats.add(ctx->delta_stats, ctx->obs->oi.category);
+    else if (soid <= last_backfill_started)
+      pending_backfill_updates[soid].stats.add(ctx->delta_stats,
+					       ctx->obs->oi.category);
+  }
+
+  if (scrubber.active && scrubber.is_chunky) {
+    assert(soid < scrubber.start || soid >= scrubber.end);
+    if (soid < scrubber.start)
+      scrub_cstat.add(ctx->delta_stats, ctx->obs->oi.category);
+  }
+}
+
+void ReplicatedPG::complete_read_ctx(int result, OpContext *ctx)
+{
+  MOSDOp *m = static_cast(ctx->op->get_req());
+  assert(ctx->async_reads_complete());
+  ctx->reply->claim_op_out_data(ctx->ops);
+  ctx->reply->get_header().data_off = ctx->data_off;
+
+  MOSDOpReply *reply = ctx->reply;
+  ctx->reply = NULL;
+
+  if (result >= 0) {
+    log_op_stats(ctx);
+    publish_stats_to_osd();
+
+    // on read, return the current object version
+    reply->set_reply_versions(eversion_t(), ctx->obs->oi.user_version);
+  } else if (result == -ENOENT) {
+    // on ENOENT, set a floor for what the next user version will be.
+    reply->set_enoent_reply_versions(info.last_update, info.last_user_version);
+  }
+
+  reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
+  osd->send_message_osd_client(reply, m->get_connection());
+  close_op_ctx(ctx, 0);
+}
+
+// ========================================================================
+// copyfrom
+
+struct C_Copyfrom : public Context {
+  ReplicatedPGRef pg;
+  hobject_t oid;
+  epoch_t last_peering_reset;
+  ceph_tid_t tid;
+  ReplicatedPG::CopyOpRef cop;
+  C_Copyfrom(ReplicatedPG *p, hobject_t o, epoch_t lpr,
+	     const ReplicatedPG::CopyOpRef& c)
+    : pg(p), oid(o), last_peering_reset(lpr),
+      tid(0), cop(c)
+  {}
+  void finish(int r) {
+    if (r == -ECANCELED)
+      return;
+    pg->lock();
+    if (last_peering_reset == pg->get_last_peering_reset()) {
+      pg->process_copy_chunk(oid, tid, r);
+    }
+    pg->unlock();
+  }
+};
+
+struct C_CopyFrom_AsyncReadCb : public Context {
+  OSDOp *osd_op;
+  object_copy_data_t reply_obj;
+  bool classic;
+  size_t len;
+  C_CopyFrom_AsyncReadCb(OSDOp *osd_op, bool classic) :
+    osd_op(osd_op), classic(classic), len(0) {}
+  void finish(int r) {
+    assert(len > 0);
+    assert(len <= reply_obj.data.length());
+    bufferlist bl;
+    bl.substr_of(reply_obj.data, 0, len);
+    reply_obj.data.swap(bl);
+    if (classic) {
+      reply_obj.encode_classic(osd_op->outdata);
+    } else {
+      ::encode(reply_obj, osd_op->outdata);
+    }
+  }
+};
+
+int ReplicatedPG::fill_in_copy_get(
+  OpContext *ctx,
+  bufferlist::iterator& bp,
+  OSDOp& osd_op,
+  ObjectContextRef &obc,
+  bool classic)
+{
+  object_info_t& oi = obc->obs.oi;
+  hobject_t& soid = oi.soid;
+  int result = 0;
+  object_copy_cursor_t cursor;
+  uint64_t out_max;
+  try {
+    ::decode(cursor, bp);
+    ::decode(out_max, bp);
+  }
+  catch (buffer::error& e) {
+    result = -EINVAL;
+    return result;
+  }
+
+  bool async_read_started = false;
+  object_copy_data_t _reply_obj;
+  C_CopyFrom_AsyncReadCb *cb = NULL;
+  if (pool.info.require_rollback()) {
+    cb = new C_CopyFrom_AsyncReadCb(&osd_op, classic);
+  }
+  object_copy_data_t &reply_obj = cb ? cb->reply_obj : _reply_obj;
+  // size, mtime
+  reply_obj.size = oi.size;
+  reply_obj.mtime = oi.mtime;
+  reply_obj.category = oi.category;
+  if (soid.snap < CEPH_NOSNAP) {
+    reply_obj.snaps = oi.snaps;
+  } else {
+    assert(obc->ssc);
+    reply_obj.snap_seq = obc->ssc->snapset.seq;
+  }
+
+  // attrs
+  map& out_attrs = reply_obj.attrs;
+  if (!cursor.attr_complete) {
+    result = getattrs_maybe_cache(
+      ctx->obc,
+      &out_attrs,
+      true);
+    if (result < 0) {
+      if (cb) {
+        delete cb;
+      }
+      return result;
+    }
+    cursor.attr_complete = true;
+    dout(20) << " got attrs" << dendl;
+  }
+
+  int64_t left = out_max - osd_op.outdata.length();
+
+  // data
+  bufferlist& bl = reply_obj.data;
+  if (left > 0 && !cursor.data_complete) {
+    if (cursor.data_offset < oi.size) {
+      if (cb) {
+	async_read_started = true;
+	ctx->pending_async_reads.push_back(
+	  make_pair(
+	    make_pair(cursor.data_offset, left),
+	    make_pair(&bl, cb)));
+	result = MIN(oi.size - cursor.data_offset, (uint64_t)left);
+	cb->len = result;
+      } else {
+	result = pgbackend->objects_read_sync(
+	  oi.soid, cursor.data_offset, left, &bl);
+	if (result < 0)
+	  return result;
+      }
+      assert(result <= left);
+      left -= result;
+      cursor.data_offset += result;
+    }
+    if (cursor.data_offset == oi.size) {
+      cursor.data_complete = true;
+      dout(20) << " got data" << dendl;
+    }
+    assert(cursor.data_offset <= oi.size);
+  }
+
+  // omap
+  std::map& out_omap = reply_obj.omap;
+  if (pool.info.require_rollback()) {
+    cursor.omap_complete = true;
+  } else {
+    if (left > 0 && !cursor.omap_complete) {
+      assert(cursor.data_complete);
+      if (cursor.omap_offset.empty()) {
+	osd->store->omap_get_header(coll, oi.soid, &reply_obj.omap_header);
+      }
+      ObjectMap::ObjectMapIterator iter =
+	osd->store->get_omap_iterator(coll, oi.soid);
+      assert(iter);
+      iter->upper_bound(cursor.omap_offset);
+      for (; iter->valid(); iter->next()) {
+	out_omap.insert(make_pair(iter->key(), iter->value()));
+	left -= iter->key().length() + 4 + iter->value().length() + 4;
+	if (left <= 0)
+	  break;
+      }
+      if (iter->valid()) {
+	cursor.omap_offset = iter->key();
+      } else {
+	cursor.omap_complete = true;
+	dout(20) << " got omap" << dendl;
+      }
+    }
+  }
+
+  dout(20) << " cursor.is_complete=" << cursor.is_complete()
+	   << " " << out_attrs.size() << " attrs"
+	   << " " << bl.length() << " bytes"
+	   << " " << reply_obj.omap_header.length() << " omap header bytes"
+	   << " " << out_omap.size() << " keys"
+	   << dendl;
+  reply_obj.cursor = cursor;
+  if (!async_read_started) {
+    if (classic) {
+      reply_obj.encode_classic(osd_op.outdata);
+    } else {
+      ::encode(reply_obj, osd_op.outdata);
+    }
+  }
+  if (cb && !async_read_started) {
+    delete cb;
+  }
+  result = 0;
+  return result;
+}
+
+void ReplicatedPG::start_copy(CopyCallback *cb, ObjectContextRef obc,
+			      hobject_t src, object_locator_t oloc,
+			      version_t version, unsigned flags,
+			      bool mirror_snapset)
+{
+  const hobject_t& dest = obc->obs.oi.soid;
+  dout(10) << __func__ << " " << dest
+	   << " from " << src << " " << oloc << " v" << version
+	   << " flags " << flags
+	   << (mirror_snapset ? " mirror_snapset" : "")
+	   << dendl;
+
+  assert(!mirror_snapset || (src.snap == CEPH_NOSNAP ||
+			     src.snap == CEPH_SNAPDIR));
+
+  // cancel a previous in-progress copy?
+  if (copy_ops.count(dest)) {
+    // FIXME: if the src etc match, we could avoid restarting from the
+    // beginning.
+    CopyOpRef cop = copy_ops[dest];
+    cancel_copy(cop, false);
+  }
+
+  CopyOpRef cop(new CopyOp(cb, obc, src, oloc, version, flags,
+			   mirror_snapset));
+  copy_ops[dest] = cop;
+  obc->start_block();
+
+  _copy_some(obc, cop);
+}
+
+void ReplicatedPG::_copy_some(ObjectContextRef obc, CopyOpRef cop)
+{
+  dout(10) << __func__ << " " << obc << " " << cop << dendl;
+
+  unsigned flags = 0;
+  if (cop->flags & CEPH_OSD_COPY_FROM_FLAG_FLUSH)
+    flags |= CEPH_OSD_FLAG_FLUSH;
+  if (cop->flags & CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE)
+    flags |= CEPH_OSD_FLAG_IGNORE_CACHE;
+  if (cop->flags & CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY)
+    flags |= CEPH_OSD_FLAG_IGNORE_OVERLAY;
+  if (cop->flags & CEPH_OSD_COPY_FROM_FLAG_MAP_SNAP_CLONE)
+    flags |= CEPH_OSD_FLAG_MAP_SNAP_CLONE;
+
+  C_GatherBuilder gather(g_ceph_context);
+
+  if (cop->cursor.is_initial() && cop->mirror_snapset) {
+    // list snaps too.
+    assert(cop->src.snap == CEPH_NOSNAP);
+    ObjectOperation op;
+    op.list_snaps(&cop->results.snapset, NULL);
+    osd->objecter_lock.Lock();
+    ceph_tid_t tid = osd->objecter->read(cop->src.oid, cop->oloc, op,
+				    CEPH_SNAPDIR, NULL,
+				    flags, gather.new_sub(), NULL);
+    cop->objecter_tid2 = tid;
+    osd->objecter_lock.Unlock();
+  }
+
+  ObjectOperation op;
+  if (cop->results.user_version) {
+    op.assert_version(cop->results.user_version);
+  } else {
+    // we should learn the version after the first chunk, if we didn't know
+    // it already!
+    assert(cop->cursor.is_initial());
+  }
+  op.copy_get(&cop->cursor, get_copy_chunk_size(),
+	      &cop->results.object_size, &cop->results.mtime,
+	      &cop->results.category,
+	      &cop->attrs, &cop->data, &cop->omap_header, &cop->omap,
+	      &cop->results.snaps, &cop->results.snap_seq,
+	      &cop->rval);
+
+  C_Copyfrom *fin = new C_Copyfrom(this, obc->obs.oi.soid,
+				   get_last_peering_reset(), cop);
+  gather.set_finisher(new C_OnFinisher(fin,
+				       &osd->objecter_finisher));
+
+  osd->objecter_lock.Lock();
+  ceph_tid_t tid = osd->objecter->read(cop->src.oid, cop->oloc, op,
+				  cop->src.snap, NULL,
+				  flags,
+				  gather.new_sub(),
+				  // discover the object version if we don't know it yet
+				  cop->results.user_version ? NULL : &cop->results.user_version);
+  fin->tid = tid;
+  cop->objecter_tid = tid;
+  gather.activate();
+  osd->objecter_lock.Unlock();
+}
+
+void ReplicatedPG::process_copy_chunk(hobject_t oid, ceph_tid_t tid, int r)
+{
+  dout(10) << __func__ << " " << oid << " tid " << tid
+	   << " " << cpp_strerror(r) << dendl;
+  map::iterator p = copy_ops.find(oid);
+  if (p == copy_ops.end()) {
+    dout(10) << __func__ << " no copy_op found" << dendl;
+    return;
+  }
+  CopyOpRef cop = p->second;
+  if (tid != cop->objecter_tid) {
+    dout(10) << __func__ << " tid " << tid << " != cop " << cop
+	     << " tid " << cop->objecter_tid << dendl;
+    return;
+  }
+
+  if (cop->omap.size())
+    cop->results.has_omap = true;
+
+  if (r >= 0 && pool.info.require_rollback() && cop->omap.size()) {
+    r = -EOPNOTSUPP;
+  }
+  cop->objecter_tid = 0;
+  cop->objecter_tid2 = 0;  // assume this ordered before us (if it happened)
+  ObjectContextRef& cobc = cop->obc;
+
+  if (r < 0)
+    goto out;
+
+  assert(cop->rval >= 0);
+
+  if (oid.snap < CEPH_NOSNAP && !cop->results.snaps.empty()) {
+    // verify snap hasn't been deleted
+    vector::iterator p = cop->results.snaps.begin();
+    while (p != cop->results.snaps.end()) {
+      if (pool.info.is_removed_snap(*p)) {
+	dout(10) << __func__ << " clone snap " << *p << " has been deleted"
+		 << dendl;
+	for (vector::iterator q = p + 1;
+	     q != cop->results.snaps.end();
+	     ++q)
+	  *(q - 1) = *q;
+	cop->results.snaps.resize(cop->results.snaps.size() - 1);
+      } else {
+	++p;
+      }
+    }
+    if (cop->results.snaps.empty()) {
+      dout(10) << __func__ << " no more snaps for " << oid << dendl;
+      r = -ENOENT;
+      goto out;
+    }
+  }
+
+  assert(cop->rval >= 0);
+
+  if (!cop->cursor.is_complete()) {
+    // write out what we have so far
+    if (cop->temp_cursor.is_initial()) {
+      assert(!cop->results.started_temp_obj);
+      cop->results.started_temp_obj = true;
+      cop->results.temp_oid = generate_temp_object();
+      dout(20) << __func__ << " using temp " << cop->results.temp_oid << dendl;
+    }
+    ObjectContextRef tempobc = get_object_context(cop->results.temp_oid, true);
+    RepGather *repop = simple_repop_create(tempobc);
+    if (cop->temp_cursor.is_initial()) {
+      repop->ctx->new_temp_oid = cop->results.temp_oid;
+    }
+    _write_copy_chunk(cop, repop->ctx->op_t);
+    simple_repop_submit(repop);
+    dout(10) << __func__ << " fetching more" << dendl;
+    _copy_some(cobc, cop);
+    return;
+  }
+
+  dout(20) << __func__ << " success; committing" << dendl;
+  cop->results.final_tx = pgbackend->get_transaction();
+  _build_finish_copy_transaction(cop, cop->results.final_tx);
+
+ out:
+  dout(20) << __func__ << " complete r = " << cpp_strerror(r) << dendl;
+  CopyCallbackResults results(r, &cop->results);
+  cop->cb->complete(results);
+
+  copy_ops.erase(cobc->obs.oi.soid);
+  cobc->stop_block();
+  kick_object_context_blocked(cobc);
+}
+
+void ReplicatedPG::_write_copy_chunk(CopyOpRef cop, PGBackend::PGTransaction *t)
+{
+  dout(20) << __func__ << " " << cop
+	   << " " << cop->attrs.size() << " attrs"
+	   << " " << cop->data.length() << " bytes"
+	   << " " << cop->omap.size() << " keys"
+	   << dendl;
+  if (!cop->temp_cursor.attr_complete) {
+    t->touch(cop->results.temp_oid);
+    for (map::iterator p = cop->attrs.begin();
+	 p != cop->attrs.end();
+	 ++p) {
+      cop->results.attrs[string("_") + p->first] = p->second;
+      t->setattr(
+	cop->results.temp_oid,
+	string("_") + p->first, p->second);
+    }
+    cop->attrs.clear();
+  }
+  if (!cop->temp_cursor.data_complete) {
+    assert(cop->data.length() + cop->temp_cursor.data_offset ==
+	   cop->cursor.data_offset);
+    if (pool.info.requires_aligned_append() &&
+	!cop->cursor.data_complete) {
+      /**
+       * Trim off the unaligned bit at the end, we'll adjust cursor.data_offset
+       * to pick it up on the next pass.
+       */
+      assert(cop->temp_cursor.data_offset %
+	     pool.info.required_alignment() == 0);
+      if (cop->data.length() % pool.info.required_alignment() != 0) {
+	uint64_t to_trim =
+	  cop->data.length() % pool.info.required_alignment();
+	bufferlist bl;
+	bl.substr_of(cop->data, 0, cop->data.length() - to_trim);
+	cop->data.swap(bl);
+	cop->cursor.data_offset -= to_trim;
+	assert(cop->data.length() + cop->temp_cursor.data_offset ==
+	       cop->cursor.data_offset);
+      }
+    }
+    t->append(
+      cop->results.temp_oid,
+      cop->temp_cursor.data_offset,
+      cop->data.length(),
+      cop->data);
+    cop->data.clear();
+  }
+  if (!pool.info.require_rollback()) {
+    if (!cop->temp_cursor.omap_complete) {
+      if (cop->omap_header.length()) {
+	t->omap_setheader(
+	  cop->results.temp_oid,
+	  cop->omap_header);
+	cop->omap_header.clear();
+      }
+      t->omap_setkeys(cop->results.temp_oid, cop->omap);
+      cop->omap.clear();
+    }
+  } else {
+    assert(cop->omap_header.length() == 0);
+    assert(cop->omap.empty());
+  }
+  cop->temp_cursor = cop->cursor;
+}
+
+void ReplicatedPG::_build_finish_copy_transaction(CopyOpRef cop,
+                                                  PGBackend::PGTransaction* t)
+{
+  ObjectState& obs = cop->obc->obs;
+  if (cop->temp_cursor.is_initial()) {
+    // write directly to final object
+    cop->results.temp_oid = obs.oi.soid;
+    _write_copy_chunk(cop, t);
+  } else {
+    // finish writing to temp object, then move into place
+    _write_copy_chunk(cop, t);
+    t->rename(cop->results.temp_oid, obs.oi.soid);
+  }
+}
+
+void ReplicatedPG::finish_copyfrom(OpContext *ctx)
+{
+  dout(20) << "finish_copyfrom on " << ctx->obs->oi.soid << dendl;
+  ObjectState& obs = ctx->new_obs;
+  CopyFromCallback *cb = static_cast(ctx->copy_cb);
+
+  if (pool.info.require_rollback()) {
+    if (obs.exists) {
+      if (ctx->mod_desc.rmobject(ctx->at_version.version)) {
+	ctx->op_t->stash(obs.oi.soid, ctx->at_version.version);
+      } else {
+	ctx->op_t->remove(obs.oi.soid);
+      }
+    }
+    ctx->mod_desc.create();
+    replace_cached_attrs(ctx, ctx->obc, cb->results->attrs);
+  } else {
+    if (obs.exists) {
+      ctx->op_t->remove(obs.oi.soid);
+    }
+    ctx->mod_desc.mark_unrollbackable();
+  }
+
+  if (!obs.exists) {
+    ctx->delta_stats.num_objects++;
+    obs.exists = true;
+  }
+  if (cb->is_temp_obj_used()) {
+    ctx->discard_temp_oid = cb->results->temp_oid;
+  }
+  ctx->op_t->append(cb->results->final_tx);
+  delete cb->results->final_tx;
+  cb->results->final_tx = NULL;
+
+  // CopyFromCallback fills this in for us
+  obs.oi.user_version = ctx->user_at_version;
+
+  // cache: clear whiteout?
+  if (obs.oi.is_whiteout()) {
+    dout(10) << __func__ << " clearing whiteout on " << obs.oi.soid << dendl;
+    obs.oi.clear_flag(object_info_t::FLAG_WHITEOUT);
+    --ctx->delta_stats.num_whiteouts;
+  }
+
+  if (cb->results->has_omap) {
+    dout(10) << __func__ << " setting omap flag on " << obs.oi.soid << dendl;
+    obs.oi.set_flag(object_info_t::FLAG_OMAP);
+  } else {
+    dout(10) << __func__ << " clearing omap flag on " << obs.oi.soid << dendl;
+    obs.oi.clear_flag(object_info_t::FLAG_OMAP);
+  }
+
+  interval_set ch;
+  if (obs.oi.size > 0)
+    ch.insert(0, obs.oi.size);
+  ctx->modified_ranges.union_of(ch);
+
+  if (cb->get_data_size() != obs.oi.size) {
+    ctx->delta_stats.num_bytes -= obs.oi.size;
+    obs.oi.size = cb->get_data_size();
+    ctx->delta_stats.num_bytes += obs.oi.size;
+  }
+  ctx->delta_stats.num_wr++;
+  ctx->delta_stats.num_wr_kb += SHIFT_ROUND_UP(obs.oi.size, 10);
+
+  osd->logger->inc(l_osd_copyfrom);
+}
+
+void ReplicatedPG::finish_promote(int r, OpRequestRef op,
+				  CopyResults *results, ObjectContextRef obc)
+{
+  const hobject_t& soid = obc->obs.oi.soid;
+  dout(10) << __func__ << " " << soid << " r=" << r
+	   << " uv" << results->user_version << dendl;
+
+  if (r == -ECANCELED) {
+    return;
+  }
+
+  if (r == -ENOENT && results->started_temp_obj) {
+    dout(10) << __func__ << " abort; will clean up partial work" << dendl;
+    ObjectContextRef tempobc = get_object_context(results->temp_oid, true);
+    RepGather *repop = simple_repop_create(tempobc);
+    repop->ctx->op_t->remove(results->temp_oid);
+    simple_repop_submit(repop);
+    results->started_temp_obj = false;
+  }
+
+  if (r == -ENOENT && soid.is_snap()) {
+    dout(10) << __func__
+	     << ": enoent while trying to promote clone, " << soid
+	     << " must have been trimmed, removing from snapset"
+	     << dendl;
+    hobject_t head(soid.get_head());
+    ObjectContextRef obc = get_object_context(head, false);
+    assert(obc);
+    RepGather *repop = simple_repop_create(obc);
+    OpContext *tctx = repop->ctx;
+    tctx->at_version = get_next_version();
+    filter_snapc(tctx->new_snapset.snaps);
+    vector new_clones(tctx->new_snapset.clones.size());
+    for (vector::iterator i = tctx->new_snapset.clones.begin();
+	 i != tctx->new_snapset.clones.end();
+	 ++i) {
+      if (*i != soid.snap)
+	new_clones.push_back(*i);
+    }
+    tctx->new_snapset.clones.swap(new_clones);
+    tctx->new_snapset.clone_overlap.erase(soid.snap);
+    tctx->new_snapset.clone_size.erase(soid.snap);
+
+    // take RWWRITE lock for duration of our local write.  ignore starvation.
+    if (!obc->rwstate.take_write_lock()) {
+      assert(0 == "problem!");
+    }
+    tctx->lock_to_release = OpContext::W_LOCK;
+    dout(20) << __func__ << " took lock on obc, " << obc->rwstate << dendl;
+
+    finish_ctx(tctx, pg_log_entry_t::PROMOTE);
+
+    simple_repop_submit(repop);
+    return;
+  }
+
+  bool whiteout = false;
+  if (r == -ENOENT &&
+      soid.snap == CEPH_NOSNAP &&
+      (pool.info.cache_mode == pg_pool_t::CACHEMODE_WRITEBACK ||
+       pool.info.cache_mode == pg_pool_t::CACHEMODE_READONLY)) {
+    dout(10) << __func__ << " whiteout " << soid << dendl;
+    whiteout = true;
+  }
+
+  if (r < 0 && !whiteout) {
+    // we need to get rid of the op in the blocked queue
+    map >::iterator blocked_iter =
+      waiting_for_blocked_object.find(soid);
+    assert(blocked_iter != waiting_for_blocked_object.end());
+    assert(blocked_iter->second.begin()->get() == op.get());
+    blocked_iter->second.pop_front();
+    if (blocked_iter->second.empty()) {
+      waiting_for_blocked_object.erase(blocked_iter);
+    }
+    osd->reply_op_error(op, r);
+    return;
+  }
+
+  RepGather *repop = simple_repop_create(obc);
+  OpContext *tctx = repop->ctx;
+  tctx->at_version = get_next_version();
+
+  ++tctx->delta_stats.num_objects;
+  if (soid.snap < CEPH_NOSNAP)
+    ++tctx->delta_stats.num_object_clones;
+  tctx->new_obs.exists = true;
+
+  if (whiteout) {
+    // create a whiteout
+    tctx->op_t->touch(soid);
+    tctx->new_obs.oi.set_flag(object_info_t::FLAG_WHITEOUT);
+    ++tctx->delta_stats.num_whiteouts;
+    dout(20) << __func__ << " creating whiteout on " << soid << dendl;
+    osd->logger->inc(l_osd_tier_whiteout);
+  } else {
+    if (results->has_omap) {
+      dout(10) << __func__ << " setting omap flag on " << soid << dendl;
+      tctx->new_obs.oi.set_flag(object_info_t::FLAG_OMAP);
+      ++tctx->delta_stats.num_objects_omap;
+    }
+
+    tctx->op_t->append(results->final_tx);
+    delete results->final_tx;
+    results->final_tx = NULL;
+    if (results->started_temp_obj) {
+      tctx->discard_temp_oid = results->temp_oid;
+    }
+    tctx->new_obs.oi.size = results->object_size;
+    tctx->new_obs.oi.category = results->category;
+    tctx->new_obs.oi.user_version = results->user_version;
+
+    if (soid.snap != CEPH_NOSNAP) {
+      if (!results->snaps.empty()) {
+	tctx->new_obs.oi.snaps = results->snaps;
+      } else {
+	// we must have read "snap" content from the head object in
+	// the base pool.  use snap_seq to construct what snaps should
+	// be for this clone (what is was before we evicted the clean
+	// clone from this pool, and what it will be when we flush and
+	// the clone eventually happens in the base pool).
+	SnapSet& snapset = obc->ssc->snapset;
+	vector::iterator p = snapset.snaps.begin();
+	while (p != snapset.snaps.end() && *p > soid.snap)
+	  ++p;
+	assert(p != snapset.snaps.end());
+	do {
+	  tctx->new_obs.oi.snaps.push_back(*p);
+	  ++p;
+	} while (p != snapset.snaps.end() && *p > results->snap_seq);
+      }
+      dout(20) << __func__ << " snaps " << tctx->new_obs.oi.snaps << dendl;
+      assert(!tctx->new_obs.oi.snaps.empty());
+      assert(obc->ssc->snapset.clone_size.count(soid.snap));
+      assert(obc->ssc->snapset.clone_size[soid.snap] ==
+	     results->object_size);
+      assert(obc->ssc->snapset.clone_overlap.count(soid.snap));
+
+      tctx->delta_stats.num_bytes += obc->ssc->snapset.get_clone_bytes(soid.snap);
+    } else {
+      tctx->delta_stats.num_bytes += results->object_size;
+    }
+  }
+
+  if (results->mirror_snapset) {
+    assert(tctx->new_obs.oi.soid.snap == CEPH_NOSNAP);
+    tctx->new_snapset.from_snap_set(results->snapset);
+  }
+  tctx->new_snapset.head_exists = true;
+  dout(20) << __func__ << " new_snapset " << tctx->new_snapset << dendl;
+
+  // take RWWRITE lock for duration of our local write.  ignore starvation.
+  if (!obc->rwstate.take_write_lock()) {
+    assert(0 == "problem!");
+  }
+  tctx->lock_to_release = OpContext::W_LOCK;
+  dout(20) << __func__ << " took lock on obc, " << obc->rwstate << dendl;
+
+  finish_ctx(tctx, pg_log_entry_t::PROMOTE);
+
+  simple_repop_submit(repop);
+
+  osd->logger->inc(l_osd_tier_promote);
+}
+
+void ReplicatedPG::cancel_copy(CopyOpRef cop, bool requeue)
+{
+  dout(10) << __func__ << " " << cop->obc->obs.oi.soid
+	   << " from " << cop->src << " " << cop->oloc
+	   << " v" << cop->results.user_version << dendl;
+
+  // cancel objecter op, if we can
+  if (cop->objecter_tid) {
+    Mutex::Locker l(osd->objecter_lock);
+    osd->objecter->op_cancel(cop->objecter_tid, -ECANCELED);
+    cop->objecter_tid = 0;
+    if (cop->objecter_tid2) {
+      osd->objecter->op_cancel(cop->objecter_tid2, -ECANCELED);
+      cop->objecter_tid2 = 0;
+    }
+  }
+
+  copy_ops.erase(cop->obc->obs.oi.soid);
+  cop->obc->stop_block();
+
+  kick_object_context_blocked(cop->obc);
+  cop->results.should_requeue = requeue;
+  CopyCallbackResults result(-ECANCELED, &cop->results);
+  cop->cb->complete(result);
+
+  // There may still be an objecter callback referencing this copy op.
+  // That callback will not need the obc since it's been canceled, and
+  // we need the obc reference to go away prior to flush.
+  cop->obc = ObjectContextRef();
+}
+
+void ReplicatedPG::cancel_copy_ops(bool requeue)
+{
+  dout(10) << __func__ << dendl;
+  map::iterator p = copy_ops.begin();
+  while (p != copy_ops.end()) {
+    // requeue this op? can I queue up all of them?
+    cancel_copy((p++)->second, requeue);
+  }
+}
+
+
+// ========================================================================
+// flush
+//
+// Flush a dirty object in the cache tier by writing it back to the
+// base tier.  The sequence looks like:
+//
+//  * send a copy-from operation to the base tier to copy the current
+//    version of the object
+//  * base tier will pull the object via (perhaps multiple) copy-get(s)
+//  * on completion, we check if the object has been modified.  if so,
+//    just reply with -EAGAIN.
+//  * try to take a write lock so we can clear the dirty flag.  if this
+//    fails, wait and retry
+//  * start a repop that clears the bit.
+//
+// If we have to wait, we will retry by coming back through the
+// start_flush method.  We check if a flush is already in progress
+// and, if so, try to finish it by rechecking the version and trying
+// to clear the dirty bit.
+//
+// In order for the cache-flush (a write op) to not block the copy-get
+// from reading the object, the client *must* set the SKIPRWLOCKS
+// flag.
+//
+// NOTE: normally writes are strictly ordered for the client, but
+// flushes are special in that they can be reordered with respect to
+// other writes.  In particular, we can't have a flush request block
+// an update to the cache pool object!
+
+struct C_Flush : public Context {
+  ReplicatedPGRef pg;
+  hobject_t oid;
+  epoch_t last_peering_reset;
+  ceph_tid_t tid;
+  C_Flush(ReplicatedPG *p, hobject_t o, epoch_t lpr)
+    : pg(p), oid(o), last_peering_reset(lpr),
+      tid(0)
+  {}
+  void finish(int r) {
+    if (r == -ECANCELED)
+      return;
+    pg->lock();
+    if (last_peering_reset == pg->get_last_peering_reset()) {
+      pg->finish_flush(oid, tid, r);
+    }
+    pg->unlock();
+  }
+};
+
+int ReplicatedPG::start_flush(
+  OpRequestRef op, ObjectContextRef obc,
+  bool blocking, hobject_t *pmissing,
+  Context *on_flush)
+{
+  const object_info_t& oi = obc->obs.oi;
+  const hobject_t& soid = oi.soid;
+  dout(10) << __func__ << " " << soid
+	   << " v" << oi.version
+	   << " uv" << oi.user_version
+	   << " " << (blocking ? "blocking" : "non-blocking/best-effort")
+	   << dendl;
+
+  // verify there are no (older) check for dirty clones
+  SnapSet& snapset = obc->ssc->snapset;
+  {
+    dout(20) << " snapset " << snapset << dendl;
+    vector::reverse_iterator p = snapset.clones.rbegin();
+    while (p != snapset.clones.rend() && *p >= soid.snap)
+      ++p;
+    if (p != snapset.clones.rend()) {
+      hobject_t next = soid;
+      next.snap = *p;
+      assert(next.snap < soid.snap);
+      if (pg_log.get_missing().is_missing(next)) {
+	dout(10) << __func__ << " missing clone is " << next << dendl;
+	if (pmissing)
+	  *pmissing = next;
+	return -ENOENT;
+      }
+      ObjectContextRef older_obc = get_object_context(next, false);
+      if (older_obc) {
+	dout(20) << __func__ << " next oldest clone is " << older_obc->obs.oi
+		 << dendl;
+	if (older_obc->obs.oi.is_dirty()) {
+	  dout(10) << __func__ << " next oldest clone is dirty: "
+		   << older_obc->obs.oi << dendl;
+	  return -EBUSY;
+	}
+      } else {
+	dout(20) << __func__ << " next oldest clone " << next
+		 << " is not present; implicitly clean" << dendl;
+      }
+    } else {
+      dout(20) << __func__ << " no older clones" << dendl;
+    }
+  }
+
+  if (blocking)
+    obc->start_block();
+
+  map::iterator p = flush_ops.find(soid);
+  if (p != flush_ops.end()) {
+    FlushOpRef fop = p->second;
+    if (fop->op == op) {
+      // we couldn't take the write lock on a cache-try-flush before;
+      // now we are trying again for the lock.
+      return try_flush_mark_clean(fop);
+    }
+    if (fop->flushed_version == obc->obs.oi.user_version &&
+	(fop->blocking || !blocking)) {
+      // nonblocking can join anything
+      // blocking can only join a blocking flush
+      dout(20) << __func__ << " piggybacking on existing flush " << dendl;
+      if (op)
+	fop->dup_ops.push_back(op);
+      return -EAGAIN;   // clean up this ctx; op will retry later
+    }
+
+    // cancel current flush since it will fail anyway, or because we
+    // are blocking and the existing flush is nonblocking.
+    dout(20) << __func__ << " canceling previous flush; it will fail" << dendl;
+    if (fop->op)
+      osd->reply_op_error(fop->op, -EBUSY);
+    while (!fop->dup_ops.empty()) {
+      osd->reply_op_error(fop->dup_ops.front(), -EBUSY);
+      fop->dup_ops.pop_front();
+    }
+    cancel_flush(fop, false);
+  }
+
+  /**
+   * In general, we need to send two deletes and a copyfrom.
+   * Consider snapc 10:[10, 9, 8, 4, 3, 2]:[10(10, 9), 4(4,3,2)]
+   * where 4 is marked as clean.  To flush 10, we have to:
+   * 1) delete 4:[4,3,2] -- ensure head is created at cloneid 4
+   * 2) delete (8-1):[4,3,2] -- ensure that the object does not exist at 8
+   * 3) copyfrom 8:[8,4,3,2] -- flush object excluding snap 8
+   *
+   * The second delete is required in case at some point in the past
+   * there had been a clone 7(7,6), which we had flushed.  Without
+   * the second delete, the object would appear in the base pool to
+   * have existed.
+   */
+
+  SnapContext snapc, dsnapc, dsnapc2;
+  if (snapset.seq != 0) {
+    if (soid.snap == CEPH_NOSNAP) {
+      snapc.seq = snapset.seq;
+      snapc.snaps = snapset.snaps;
+    } else {
+      snapid_t min_included_snap = oi.snaps.back();
+      snapc = snapset.get_ssc_as_of(min_included_snap - 1);
+    }
+
+    snapid_t prev_snapc = 0;
+    for (vector::reverse_iterator citer = snapset.clones.rbegin();
+	 citer != snapset.clones.rend();
+	 ++citer) {
+      if (*citer < soid.snap) {
+	prev_snapc = *citer;
+	break;
+      }
+    }
+
+    if (prev_snapc != snapc.seq) {
+      dsnapc = snapset.get_ssc_as_of(prev_snapc);
+      snapid_t first_snap_after_prev_snapc =
+	snapset.get_first_snap_after(prev_snapc, snapc.seq);
+      dsnapc2 = snapset.get_ssc_as_of(
+	first_snap_after_prev_snapc - 1);
+    }
+  }
+
+  object_locator_t base_oloc(soid);
+  base_oloc.pool = pool.info.tier_of;
+
+  if (dsnapc.seq > 0 && dsnapc.seq < snapc.seq) {
+    ObjectOperation o;
+    o.remove();
+    osd->objecter_lock.Lock();
+    osd->objecter->mutate(
+      soid.oid,
+      base_oloc,
+      o,
+      dsnapc,
+      oi.mtime,
+      (CEPH_OSD_FLAG_IGNORE_OVERLAY |
+       CEPH_OSD_FLAG_ORDERSNAP |
+       CEPH_OSD_FLAG_ENFORCE_SNAPC),
+      NULL,
+      NULL /* no callback, we'll rely on the ordering w.r.t the next op */);
+    osd->objecter_lock.Unlock();
+  }
+
+  if (dsnapc2.seq > dsnapc.seq && dsnapc2.seq < snapc.seq) {
+    ObjectOperation o;
+    o.remove();
+    osd->objecter_lock.Lock();
+    osd->objecter->mutate(
+      soid.oid,
+      base_oloc,
+      o,
+      dsnapc2,
+      oi.mtime,
+      (CEPH_OSD_FLAG_IGNORE_OVERLAY |
+       CEPH_OSD_FLAG_ORDERSNAP |
+       CEPH_OSD_FLAG_ENFORCE_SNAPC),
+      NULL,
+      NULL /* no callback, we'll rely on the ordering w.r.t the next op */);
+    osd->objecter_lock.Unlock();
+  }
+
+  FlushOpRef fop(new FlushOp);
+  fop->obc = obc;
+  fop->flushed_version = oi.user_version;
+  fop->blocking = blocking;
+  fop->on_flush = on_flush;
+  fop->op = op;
+
+  ObjectOperation o;
+  if (oi.is_whiteout()) {
+    fop->removal = true;
+    o.remove();
+  } else {
+    object_locator_t oloc(soid);
+    o.copy_from(soid.oid.name, soid.snap, oloc, oi.user_version,
+		CEPH_OSD_COPY_FROM_FLAG_FLUSH |
+		CEPH_OSD_COPY_FROM_FLAG_IGNORE_OVERLAY |
+		CEPH_OSD_COPY_FROM_FLAG_IGNORE_CACHE |
+		CEPH_OSD_COPY_FROM_FLAG_MAP_SNAP_CLONE);
+  }
+  C_Flush *fin = new C_Flush(this, soid, get_last_peering_reset());
+
+  osd->objecter_lock.Lock();
+  ceph_tid_t tid = osd->objecter->mutate(
+    soid.oid, base_oloc, o, snapc, oi.mtime,
+    CEPH_OSD_FLAG_IGNORE_OVERLAY | CEPH_OSD_FLAG_ENFORCE_SNAPC,
+    NULL,
+    new C_OnFinisher(fin,
+		     &osd->objecter_finisher));
+  fin->tid = tid;
+  fop->objecter_tid = tid;
+  osd->objecter_lock.Unlock();
+
+  flush_ops[soid] = fop;
+  return -EINPROGRESS;
+}
+
+void ReplicatedPG::finish_flush(hobject_t oid, ceph_tid_t tid, int r)
+{
+  dout(10) << __func__ << " " << oid << " tid " << tid
+	   << " " << cpp_strerror(r) << dendl;
+  map::iterator p = flush_ops.find(oid);
+  if (p == flush_ops.end()) {
+    dout(10) << __func__ << " no flush_op found" << dendl;
+    return;
+  }
+  FlushOpRef fop = p->second;
+  if (tid != fop->objecter_tid) {
+    dout(10) << __func__ << " tid " << tid << " != fop " << fop
+	     << " tid " << fop->objecter_tid << dendl;
+    return;
+  }
+  ObjectContextRef obc = fop->obc;
+  fop->objecter_tid = 0;
+
+  if (r < 0 && !(r == -ENOENT && fop->removal)) {
+    if (fop->op)
+      osd->reply_op_error(fop->op, -EBUSY);
+    if (!fop->dup_ops.empty()) {
+      dout(20) << __func__ << " requeueing dups" << dendl;
+      requeue_ops(fop->dup_ops);
+    }
+    if (fop->on_flush) {
+      Context *on_flush = fop->on_flush;
+      fop->on_flush = NULL;
+      on_flush->complete(-EBUSY);
+    }
+    flush_ops.erase(oid);
+    return;
+  }
+
+  r = try_flush_mark_clean(fop);
+  if (r == -EBUSY && fop->op) {
+    osd->reply_op_error(fop->op, r);
+  }
+}
+
+int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop)
+{
+  ObjectContextRef obc = fop->obc;
+  const hobject_t& oid = obc->obs.oi.soid;
+
+  if (fop->blocking) {
+    obc->stop_block();
+    kick_object_context_blocked(obc);
+  }
+
+  if (fop->flushed_version != obc->obs.oi.user_version ||
+      !obc->obs.exists) {
+    if (obc->obs.exists)
+      dout(10) << __func__ << " flushed_version " << fop->flushed_version
+	       << " != current " << obc->obs.oi.user_version
+	       << dendl;
+    else
+      dout(10) << __func__ << " object no longer exists" << dendl;
+
+    if (!fop->dup_ops.empty()) {
+      dout(20) << __func__ << " requeueing dups" << dendl;
+      requeue_ops(fop->dup_ops);
+    }
+    if (fop->on_flush) {
+      Context *on_flush = fop->on_flush;
+      fop->on_flush = NULL;
+      on_flush->complete(-EBUSY);
+    }
+    flush_ops.erase(oid);
+    if (fop->blocking)
+      osd->logger->inc(l_osd_tier_flush_fail);
+    else
+      osd->logger->inc(l_osd_tier_try_flush_fail);
+    return -EBUSY;
+  }
+
+  // successfully flushed; can we clear the dirty bit?
+  // try to take the lock manually, since we don't
+  // have a ctx yet.
+  if (obc->get_write(fop->op)) {
+    dout(20) << __func__ << " took write lock" << dendl;
+  } else if (fop->op) {
+    dout(10) << __func__ << " waiting on write lock" << dendl;
+    requeue_op(fop->op);
+    requeue_ops(fop->dup_ops);
+    return -EAGAIN;    // will retry
+  } else {
+    dout(10) << __func__ << " failed write lock, no op; failing" << dendl;
+    osd->logger->inc(l_osd_tier_try_flush_fail);
+    cancel_flush(fop, false);
+    return -ECANCELED;
+  }
+
+  dout(10) << __func__ << " clearing DIRTY flag for " << oid << dendl;
+  RepGather *repop = simple_repop_create(fop->obc);
+  OpContext *ctx = repop->ctx;
+
+  ctx->on_finish = fop->on_flush;
+  fop->on_flush = NULL;
+
+  ctx->lock_to_release = OpContext::W_LOCK;  // we took it above
+  ctx->at_version = get_next_version();
+
+  ctx->new_obs = obc->obs;
+  ctx->new_obs.oi.clear_flag(object_info_t::FLAG_DIRTY);
+  --ctx->delta_stats.num_objects_dirty;
+
+  finish_ctx(ctx, pg_log_entry_t::CLEAN);
+
+  osd->logger->inc(l_osd_tier_clean);
+
+  if (!fop->dup_ops.empty() || fop->op) {
+    dout(20) << __func__ << " requeueing for " << ctx->at_version << dendl;
+    list ls;
+    if (fop->op)
+      ls.push_back(fop->op);
+    ls.splice(ls.end(), fop->dup_ops);
+    requeue_ops(ls);
+  }
+
+  simple_repop_submit(repop);
+
+  flush_ops.erase(oid);
+
+  if (fop->blocking)
+    osd->logger->inc(l_osd_tier_flush);
+  else
+    osd->logger->inc(l_osd_tier_try_flush);
+
+  return -EINPROGRESS;
+}
+
+void ReplicatedPG::cancel_flush(FlushOpRef fop, bool requeue)
+{
+  dout(10) << __func__ << " " << fop->obc->obs.oi.soid << " tid "
+	   << fop->objecter_tid << dendl;
+  if (fop->objecter_tid) {
+    Mutex::Locker l(osd->objecter_lock);
+    osd->objecter->op_cancel(fop->objecter_tid, -ECANCELED);
+    fop->objecter_tid = 0;
+  }
+  if (requeue) {
+    if (fop->op)
+      requeue_op(fop->op);
+    requeue_ops(fop->dup_ops);
+  }
+  if (fop->blocking) {
+    fop->obc->stop_block();
+    kick_object_context_blocked(fop->obc);
+  }
+  if (fop->on_flush) {
+    Context *on_flush = fop->on_flush;
+    fop->on_flush = NULL;
+    on_flush->complete(-ECANCELED);
+  }
+  flush_ops.erase(fop->obc->obs.oi.soid);
+}
+
+void ReplicatedPG::cancel_flush_ops(bool requeue)
+{
+  dout(10) << __func__ << dendl;
+  map::iterator p = flush_ops.begin();
+  while (p != flush_ops.end()) {
+    cancel_flush((p++)->second, requeue);
+  }
+}
+
+bool ReplicatedPG::is_present_clone(hobject_t coid)
+{
+  if (!pool.info.allow_incomplete_clones())
+    return true;
+  if (is_missing_object(coid))
+    return true;
+  ObjectContextRef obc = get_object_context(coid, false);
+  return obc && obc->obs.exists;
+}
+
+// ========================================================================
+// rep op gather
+
+class C_OSD_RepopApplied : public Context {
+  ReplicatedPGRef pg;
+  boost::intrusive_ptr repop;
+public:
+  C_OSD_RepopApplied(ReplicatedPG *pg, ReplicatedPG::RepGather *repop)
+  : pg(pg), repop(repop) {}
+  void finish(int) {
+    pg->repop_all_applied(repop.get());
+  }
+};
+
+
+void ReplicatedPG::repop_all_applied(RepGather *repop)
+{
+  dout(10) << __func__ << ": repop tid " << repop->rep_tid << " all applied "
+	   << dendl;
+  repop->all_applied = true;
+  if (!repop->rep_aborted) {
+    eval_repop(repop);
+    if (repop->on_applied) {
+     repop->on_applied->complete(0);
+     repop->on_applied = NULL;
+    }
+  }
+}
+
+class C_OSD_RepopCommit : public Context {
+  ReplicatedPGRef pg;
+  boost::intrusive_ptr repop;
+public:
+  C_OSD_RepopCommit(ReplicatedPG *pg, ReplicatedPG::RepGather *repop)
+    : pg(pg), repop(repop) {}
+  void finish(int) {
+    pg->repop_all_committed(repop.get());
+  }
+};
+
+void ReplicatedPG::repop_all_committed(RepGather *repop)
+{
+  dout(10) << __func__ << ": repop tid " << repop->rep_tid << " all committed "
+	   << dendl;
+  repop->all_committed = true;
+
+  if (!repop->rep_aborted) {
+    if (repop->v != eversion_t()) {
+      last_update_ondisk = repop->v;
+      last_complete_ondisk = repop->pg_local_last_complete;
+    }
+    eval_repop(repop);
+  }
+}
+
+void ReplicatedPG::op_applied(const eversion_t &applied_version)
+{
+  dout(10) << "op_applied on primary on version " << applied_version << dendl;
+  if (applied_version == eversion_t())
+    return;
+  assert(applied_version > last_update_applied);
+  assert(applied_version <= info.last_update);
+  last_update_applied = applied_version;
+  if (is_primary()) {
+    if (scrubber.active && scrubber.is_chunky) {
+      if (last_update_applied == scrubber.subset_last_update) {
+        osd->scrub_wq.queue(this);
+      }
+    } else if (last_update_applied == info.last_update && scrubber.block_writes) {
+      dout(10) << "requeueing scrub for cleanup" << dendl;
+      scrubber.finalizing = true;
+      scrub_gather_replica_maps();
+      ++scrubber.waiting_on;
+      scrubber.waiting_on_whom.insert(pg_whoami);
+      osd->scrub_wq.queue(this);
+    }
+  } else {
+    dout(10) << "op_applied on replica on version " << applied_version << dendl;
+    if (scrubber.active_rep_scrub) {
+      if (last_update_applied == scrubber.active_rep_scrub->scrub_to) {
+	osd->rep_scrub_wq.queue(scrubber.active_rep_scrub);
+	scrubber.active_rep_scrub = 0;
+      }
+    }
+  }
+}
+
+void ReplicatedPG::eval_repop(RepGather *repop)
+{
+  MOSDOp *m = NULL;
+  if (repop->ctx->op)
+    m = static_cast(repop->ctx->op->get_req());
+
+  if (m)
+    dout(10) << "eval_repop " << *repop
+	     << " wants=" << (m->wants_ack() ? "a":"") << (m->wants_ondisk() ? "d":"")
+	     << (repop->rep_done ? " DONE" : "")
+	     << dendl;
+  else
+    dout(10) << "eval_repop " << *repop << " (no op)"
+	     << (repop->rep_done ? " DONE" : "")
+	     << dendl;
+
+  if (repop->rep_done)
+    return;
+
+  if (m) {
+
+    // an 'ondisk' reply implies 'ack'. so, prefer to send just one
+    // ondisk instead of ack followed by ondisk.
+
+    // ondisk?
+    if (repop->all_committed) {
+
+      log_op_stats(repop->ctx);
+      publish_stats_to_osd();
+
+      // send dup commits, in order
+      if (waiting_for_ondisk.count(repop->v)) {
+	assert(waiting_for_ondisk.begin()->first == repop->v);
+	for (list::iterator i = waiting_for_ondisk[repop->v].begin();
+	     i != waiting_for_ondisk[repop->v].end();
+	     ++i) {
+	  osd->reply_op_error(*i, 0, repop->ctx->at_version,
+			      repop->ctx->user_at_version);
+	}
+	waiting_for_ondisk.erase(repop->v);
+      }
+
+      // clear out acks, we sent the commits above
+      if (waiting_for_ack.count(repop->v)) {
+	assert(waiting_for_ack.begin()->first == repop->v);
+	waiting_for_ack.erase(repop->v);
+      }
+
+      if (m->wants_ondisk() && !repop->sent_disk) {
+	// send commit.
+	MOSDOpReply *reply = repop->ctx->reply;
+	if (reply)
+	  repop->ctx->reply = NULL;
+	else {
+	  reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, true);
+	  reply->set_reply_versions(repop->ctx->at_version,
+	                            repop->ctx->user_at_version);
+	}
+	reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
+	dout(10) << " sending commit on " << *repop << " " << reply << dendl;
+	osd->send_message_osd_client(reply, m->get_connection());
+	repop->sent_disk = true;
+	repop->ctx->op->mark_commit_sent();
+      }
+    }
+
+    // applied?
+    if (repop->all_applied) {
+
+      // send dup acks, in order
+      if (waiting_for_ack.count(repop->v)) {
+	assert(waiting_for_ack.begin()->first == repop->v);
+	for (list::iterator i = waiting_for_ack[repop->v].begin();
+	     i != waiting_for_ack[repop->v].end();
+	     ++i) {
+	  MOSDOp *m = (MOSDOp*)(*i)->get_req();
+	  MOSDOpReply *reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, true);
+	  reply->set_reply_versions(repop->ctx->at_version,
+	                            repop->ctx->user_at_version);
+	  reply->add_flags(CEPH_OSD_FLAG_ACK);
+	  osd->send_message_osd_client(reply, m->get_connection());
+	}
+	waiting_for_ack.erase(repop->v);
+      }
+
+      if (m->wants_ack() && !repop->sent_ack && !repop->sent_disk) {
+	// send ack
+	MOSDOpReply *reply = repop->ctx->reply;
+	if (reply)
+	  repop->ctx->reply = NULL;
+	else {
+	  reply = new MOSDOpReply(m, 0, get_osdmap()->get_epoch(), 0, true);
+	  reply->set_reply_versions(repop->ctx->at_version,
+	                            repop->ctx->user_at_version);
+	}
+	reply->add_flags(CEPH_OSD_FLAG_ACK);
+	dout(10) << " sending ack on " << *repop << " " << reply << dendl;
+        assert(entity_name_t::TYPE_OSD != m->get_connection()->peer_type);
+	osd->send_message_osd_client(reply, m->get_connection());
+	repop->sent_ack = true;
+      }
+
+      // note the write is now readable (for rlatency calc).  note
+      // that this will only be defined if the write is readable
+      // _prior_ to being committed; it will not get set with
+      // writeahead journaling, for instance.
+      if (repop->ctx->readable_stamp == utime_t())
+	repop->ctx->readable_stamp = ceph_clock_now(cct);
+    }
+  }
+
+  // done.
+  if (repop->all_applied && repop->all_committed) {
+    repop->rep_done = true;
+
+    release_op_ctx_locks(repop->ctx);
+
+    calc_min_last_complete_ondisk();
+
+    // kick snap_trimmer if necessary
+    if (repop->queue_snap_trimmer) {
+      queue_snap_trim();
+    }
+
+    dout(10) << " removing " << *repop << dendl;
+    assert(!repop_queue.empty());
+    dout(20) << "   q front is " << *repop_queue.front() << dendl; 
+    if (repop_queue.front() != repop) {
+      dout(0) << " removing " << *repop << dendl;
+      dout(0) << "   q front is " << *repop_queue.front() << dendl; 
+      assert(repop_queue.front() == repop);
+    }
+    repop_queue.pop_front();
+    remove_repop(repop);
+  }
+}
+
+void ReplicatedPG::issue_repop(RepGather *repop, utime_t now)
+{
+  OpContext *ctx = repop->ctx;
+  const hobject_t& soid = ctx->obs->oi.soid;
+  if (ctx->op &&
+    ((static_cast(
+	ctx->op->get_req()))->get_flags() & CEPH_OSD_FLAG_PARALLELEXEC)) {
+    // replicate original op for parallel execution on replica
+    assert(0 == "broken implementation, do not use");
+  }
+  dout(7) << "issue_repop rep_tid " << repop->rep_tid
+          << " o " << soid
+          << dendl;
+
+  repop->v = ctx->at_version;
+  if (ctx->at_version > eversion_t()) {
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == get_primary()) continue;
+      pg_info_t &pinfo = peer_info[*i];
+      // keep peer_info up to date
+      if (pinfo.last_complete == pinfo.last_update)
+	pinfo.last_complete = ctx->at_version;
+      pinfo.last_update = ctx->at_version;
+    }
+  }
+
+  repop->obc->ondisk_write_lock();
+  if (repop->ctx->clone_obc)
+    repop->ctx->clone_obc->ondisk_write_lock();
+
+  bool unlock_snapset_obc = false;
+  if (repop->ctx->snapset_obc && repop->ctx->snapset_obc->obs.oi.soid !=
+      repop->obc->obs.oi.soid) {
+    repop->ctx->snapset_obc->ondisk_write_lock();
+    unlock_snapset_obc = true;
+  }
+
+  repop->ctx->apply_pending_attrs();
+
+  if (pool.info.require_rollback()) {
+    for (vector::iterator i = repop->ctx->log.begin();
+	 i != repop->ctx->log.end();
+	 ++i) {
+      assert(i->mod_desc.can_rollback());
+      assert(!i->mod_desc.empty());
+    }
+  }
+
+  Context *on_all_commit = new C_OSD_RepopCommit(this, repop);
+  Context *on_all_applied = new C_OSD_RepopApplied(this, repop);
+  Context *onapplied_sync = new C_OSD_OndiskWriteUnlock(
+    repop->obc,
+    repop->ctx->clone_obc,
+    unlock_snapset_obc ? repop->ctx->snapset_obc : ObjectContextRef());
+  pgbackend->submit_transaction(
+    soid,
+    repop->ctx->at_version,
+    repop->ctx->op_t,
+    pg_trim_to,
+    min_last_complete_ondisk,
+    repop->ctx->log,
+    repop->ctx->updated_hset_history,
+    onapplied_sync,
+    on_all_applied,
+    on_all_commit,
+    repop->rep_tid,
+    repop->ctx->reqid,
+    repop->ctx->op);
+  repop->ctx->op_t = NULL;
+}
+    
+void ReplicatedBackend::issue_op(
+  const hobject_t &soid,
+  const eversion_t &at_version,
+  ceph_tid_t tid,
+  osd_reqid_t reqid,
+  eversion_t pg_trim_to,
+  eversion_t pg_trim_rollback_to,
+  hobject_t new_temp_oid,
+  hobject_t discard_temp_oid,
+  vector &log_entries,
+  boost::optional &hset_hist,
+  InProgressOp *op,
+  ObjectStore::Transaction *op_t)
+{
+  int acks_wanted = CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK;
+
+  if (parent->get_actingbackfill_shards().size() > 1) {
+    ostringstream ss;
+    set replicas = parent->get_actingbackfill_shards();
+    replicas.erase(parent->whoami_shard());
+    ss << "waiting for subops from " << replicas;
+    if (op->op)
+      op->op->mark_sub_op_sent(ss.str());
+  }
+  for (set::const_iterator i =
+	 parent->get_actingbackfill_shards().begin();
+       i != parent->get_actingbackfill_shards().end();
+       ++i) {
+    if (*i == parent->whoami_shard()) continue;
+    pg_shard_t peer = *i;
+    const pg_info_t &pinfo = parent->get_shard_info().find(peer)->second;
+
+    // forward the write/update/whatever
+    MOSDSubOp *wr = new MOSDSubOp(
+      reqid, parent->whoami_shard(),
+      spg_t(get_info().pgid.pgid, i->shard),
+      soid,
+      false, acks_wanted,
+      get_osdmap()->get_epoch(),
+      tid, at_version);
+
+    // ship resulting transaction, log entries, and pg_stats
+    if (!parent->should_send_op(peer, soid)) {
+      dout(10) << "issue_repop shipping empty opt to osd." << peer
+	       <<", object " << soid
+	       << " beyond MAX(last_backfill_started "
+	       << ", pinfo.last_backfill "
+	       << pinfo.last_backfill << ")" << dendl;
+      ObjectStore::Transaction t;
+      ::encode(t, wr->get_data());
+    } else {
+      ::encode(*op_t, wr->get_data());
+    }
+
+    ::encode(log_entries, wr->logbl);
+
+    if (pinfo.is_incomplete())
+      wr->pg_stats = pinfo.stats;  // reflects backfill progress
+    else
+      wr->pg_stats = get_info().stats;
+    
+    wr->pg_trim_to = pg_trim_to;
+    wr->pg_trim_rollback_to = pg_trim_rollback_to;
+
+    wr->new_temp_oid = new_temp_oid;
+    wr->discard_temp_oid = discard_temp_oid;
+    wr->updated_hit_set_history = hset_hist;
+
+    get_parent()->send_message_osd_cluster(
+      peer.osd, wr, get_osdmap()->get_epoch());
+  }
+}
+
+ReplicatedPG::RepGather *ReplicatedPG::new_repop(OpContext *ctx, ObjectContextRef obc,
+						 ceph_tid_t rep_tid)
+{
+  if (ctx->op)
+    dout(10) << "new_repop rep_tid " << rep_tid << " on " << *ctx->op->get_req() << dendl;
+  else
+    dout(10) << "new_repop rep_tid " << rep_tid << " (no op)" << dendl;
+
+  RepGather *repop = new RepGather(ctx, obc, rep_tid, info.last_complete);
+
+  repop->start = ceph_clock_now(cct);
+
+  repop_queue.push_back(&repop->queue_item);
+  repop_map[repop->rep_tid] = repop;
+  repop->get();
+
+  osd->logger->set(l_osd_op_wip, repop_map.size());
+
+  return repop;
+}
+ 
+void ReplicatedPG::remove_repop(RepGather *repop)
+{
+  dout(20) << __func__ << " " << *repop << dendl;
+  if (repop->ctx->obc)
+    dout(20) << " obc " << *repop->ctx->obc << dendl;
+  if (repop->ctx->clone_obc)
+    dout(20) << " clone_obc " << *repop->ctx->clone_obc << dendl;
+  if (repop->ctx->snapset_obc)
+    dout(20) << " snapset_obc " << *repop->ctx->snapset_obc << dendl;
+  release_op_ctx_locks(repop->ctx);
+  repop->ctx->finish(0);  // FIXME: return value here is sloppy
+  repop_map.erase(repop->rep_tid);
+  repop->put();
+
+  osd->logger->set(l_osd_op_wip, repop_map.size());
+}
+
+ReplicatedPG::RepGather *ReplicatedPG::simple_repop_create(ObjectContextRef obc)
+{
+  dout(20) << __func__ << " " << obc->obs.oi.soid << dendl;
+  vector ops;
+  ceph_tid_t rep_tid = osd->get_tid();
+  osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid);
+  OpContext *ctx = new OpContext(OpRequestRef(), reqid, ops,
+				 &obc->obs, obc->ssc, this);
+  ctx->op_t = pgbackend->get_transaction();
+  ctx->mtime = ceph_clock_now(g_ceph_context);
+  ctx->obc = obc;
+  RepGather *repop = new_repop(ctx, obc, rep_tid);
+  return repop;
+}
+
+void ReplicatedPG::simple_repop_submit(RepGather *repop)
+{
+  dout(20) << __func__ << " " << repop << dendl;
+  issue_repop(repop, repop->ctx->mtime);
+  eval_repop(repop);
+  repop->put();
+}
+
+// -------------------------------------------------------
+
+void ReplicatedPG::get_watchers(list &pg_watchers)
+{
+  pair i;
+  while (object_contexts.get_next(i.first, &i)) {
+    ObjectContextRef obc(i.second);
+    get_obc_watchers(obc, pg_watchers);
+  }
+}
+
+void ReplicatedPG::get_obc_watchers(ObjectContextRef obc, list &pg_watchers)
+{
+  for (map, WatchRef>::iterator j =
+	 obc->watchers.begin();
+	j != obc->watchers.end();
+	++j) {
+    obj_watch_item_t owi;
+
+    owi.obj = obc->obs.oi.soid;
+    owi.wi.addr = j->second->get_peer_addr();
+    owi.wi.name = j->second->get_entity();
+    owi.wi.cookie = j->second->get_cookie();
+    owi.wi.timeout_seconds = j->second->get_timeout();
+
+    dout(30) << "watch: Found oid=" << owi.obj << " addr=" << owi.wi.addr
+      << " name=" << owi.wi.name << " cookie=" << owi.wi.cookie << dendl;
+
+    pg_watchers.push_back(owi);
+  }
+}
+
+void ReplicatedPG::check_blacklisted_watchers()
+{
+  dout(20) << "ReplicatedPG::check_blacklisted_watchers for pg " << get_pgid() << dendl;
+  pair i;
+  while (object_contexts.get_next(i.first, &i))
+    check_blacklisted_obc_watchers(i.second);
+}
+
+void ReplicatedPG::check_blacklisted_obc_watchers(ObjectContextRef obc)
+{
+  dout(20) << "ReplicatedPG::check_blacklisted_obc_watchers for obc " << obc->obs.oi.soid << dendl;
+  for (map, WatchRef>::iterator k =
+	 obc->watchers.begin();
+	k != obc->watchers.end();
+	) {
+    //Advance iterator now so handle_watch_timeout() can erase element
+    map, WatchRef>::iterator j = k++;
+    dout(30) << "watch: Found " << j->second->get_entity() << " cookie " << j->second->get_cookie() << dendl;
+    entity_addr_t ea = j->second->get_peer_addr();
+    dout(30) << "watch: Check entity_addr_t " << ea << dendl;
+    if (get_osdmap()->is_blacklisted(ea)) {
+      dout(10) << "watch: Found blacklisted watcher for " << ea << dendl;
+      assert(j->second->get_pg() == this);
+      handle_watch_timeout(j->second);
+    }
+  }
+}
+
+void ReplicatedPG::populate_obc_watchers(ObjectContextRef obc)
+{
+  assert(is_active());
+  assert((recovering.count(obc->obs.oi.soid) ||
+	  !is_missing_object(obc->obs.oi.soid)) ||
+	 (pg_log.get_log().objects.count(obc->obs.oi.soid) && // or this is a revert... see recover_primary()
+	  pg_log.get_log().objects.find(obc->obs.oi.soid)->second->op ==
+	    pg_log_entry_t::LOST_REVERT &&
+	  pg_log.get_log().objects.find(obc->obs.oi.soid)->second->reverting_to ==
+	    obc->obs.oi.version));
+
+  dout(10) << "populate_obc_watchers " << obc->obs.oi.soid << dendl;
+  assert(obc->watchers.empty());
+  // populate unconnected_watchers
+  for (map, watch_info_t>::iterator p =
+	obc->obs.oi.watchers.begin();
+       p != obc->obs.oi.watchers.end();
+       ++p) {
+    utime_t expire = info.stats.last_became_active;
+    expire += p->second.timeout_seconds;
+    dout(10) << "  unconnected watcher " << p->first << " will expire " << expire << dendl;
+    WatchRef watch(
+      Watch::makeWatchRef(
+	this, osd, obc, p->second.timeout_seconds, p->first.first,
+	p->first.second, p->second.addr));
+    watch->disconnect();
+    obc->watchers.insert(
+      make_pair(
+	make_pair(p->first.first, p->first.second),
+	watch));
+  }
+  // Look for watchers from blacklisted clients and drop
+  check_blacklisted_obc_watchers(obc);
+}
+
+void ReplicatedPG::handle_watch_timeout(WatchRef watch)
+{
+  ObjectContextRef obc = watch->get_obc(); // handle_watch_timeout owns this ref
+  dout(10) << "handle_watch_timeout obc " << obc << dendl;
+
+  if (is_degraded_object(obc->obs.oi.soid)) {
+    callbacks_for_degraded_object[obc->obs.oi.soid].push_back(
+      watch->get_delayed_cb()
+      );
+    dout(10) << "handle_watch_timeout waiting for degraded on obj "
+	     << obc->obs.oi.soid
+	     << dendl;
+    return;
+  }
+
+  if (scrubber.write_blocked_by_scrub(obc->obs.oi.soid)) {
+    dout(10) << "handle_watch_timeout waiting for scrub on obj "
+	     << obc->obs.oi.soid
+	     << dendl;
+    scrubber.add_callback(
+      watch->get_delayed_cb() // This callback!
+      );
+    return;
+  }
+
+  obc->watchers.erase(make_pair(watch->get_cookie(), watch->get_entity()));
+  obc->obs.oi.watchers.erase(make_pair(watch->get_cookie(), watch->get_entity()));
+  watch->remove();
+
+  vector ops;
+  ceph_tid_t rep_tid = osd->get_tid();
+  osd_reqid_t reqid(osd->get_cluster_msgr_name(), 0, rep_tid);
+  OpContext *ctx = new OpContext(OpRequestRef(), reqid, ops,
+				 &obc->obs, obc->ssc, this);
+  ctx->op_t = pgbackend->get_transaction();
+  ctx->mtime = ceph_clock_now(cct);
+  ctx->at_version = get_next_version();
+
+  entity_inst_t nobody;
+
+  RepGather *repop = new_repop(ctx, obc, rep_tid);
+
+  PGBackend::PGTransaction *t = ctx->op_t;
+
+  ctx->log.push_back(pg_log_entry_t(pg_log_entry_t::MODIFY, obc->obs.oi.soid,
+				    ctx->at_version,
+				    obc->obs.oi.version,
+				    0,
+				    osd_reqid_t(), ctx->mtime));
+
+  obc->obs.oi.prior_version = repop->obc->obs.oi.version;
+  obc->obs.oi.version = ctx->at_version;
+  bufferlist bl;
+  ::encode(obc->obs.oi, bl);
+  setattr_maybe_cache(obc, repop->ctx, t, OI_ATTR, bl);
+
+  if (pool.info.require_rollback()) {
+    map > to_set;
+    to_set[OI_ATTR] = bl;
+    ctx->log.back().mod_desc.setattrs(to_set);
+  } else {
+    ctx->log.back().mod_desc.mark_unrollbackable();
+  }
+
+  // obc ref swallowed by repop!
+  issue_repop(repop, repop->ctx->mtime);
+  eval_repop(repop);
+  repop->put();
+}
+
+ObjectContextRef ReplicatedPG::create_object_context(const object_info_t& oi,
+						     SnapSetContext *ssc)
+{
+  ObjectContextRef obc(object_contexts.lookup_or_create(oi.soid));
+  assert(obc->destructor_callback == NULL);
+  obc->destructor_callback = new C_PG_ObjectContext(this, obc.get());  
+  obc->obs.oi = oi;
+  obc->obs.exists = false;
+  obc->ssc = ssc;
+  if (ssc)
+    register_snapset_context(ssc);
+  dout(10) << "create_object_context " << (void*)obc.get() << " " << oi.soid << " " << dendl;
+  populate_obc_watchers(obc);
+  return obc;
+}
+
+ObjectContextRef ReplicatedPG::get_object_context(const hobject_t& soid,
+						  bool can_create,
+						  map *attrs)
+{
+  assert(
+    attrs || !pg_log.get_missing().is_missing(soid) ||
+    // or this is a revert... see recover_primary()
+    (pg_log.get_log().objects.count(soid) &&
+      pg_log.get_log().objects.find(soid)->second->op ==
+      pg_log_entry_t::LOST_REVERT));
+  ObjectContextRef obc = object_contexts.lookup(soid);
+  if (obc) {
+    dout(10) << __func__ << ": found obc in cache: " << obc
+	     << dendl;
+  } else {
+    // check disk
+    bufferlist bv;
+    if (attrs) {
+      assert(attrs->count(OI_ATTR));
+      bv = attrs->find(OI_ATTR)->second;
+    } else {
+      int r = pgbackend->objects_get_attr(soid, OI_ATTR, &bv);
+      if (r < 0) {
+	if (!can_create) {
+	  dout(10) << __func__ << ": no obc for soid "
+		   << soid << " and !can_create"
+		   << dendl;
+	  return ObjectContextRef();   // -ENOENT!
+	}
+
+	dout(10) << __func__ << ": no obc for soid "
+		 << soid << " but can_create"
+		 << dendl;
+	// new object.
+	object_info_t oi(soid);
+	SnapSetContext *ssc = get_snapset_context(
+	  soid, true,
+	  soid.has_snapset() ? attrs : 0);
+	obc = create_object_context(oi, ssc);
+	dout(10) << __func__ << ": " << obc << " " << soid
+		 << " " << obc->rwstate
+		 << " oi: " << obc->obs.oi
+		 << " ssc: " << obc->ssc
+		 << " snapset: " << obc->ssc->snapset << dendl;
+	return obc;
+      }
+    }
+
+    object_info_t oi(bv);
+
+    assert(oi.soid.pool == (int64_t)info.pgid.pool());
+
+    obc = object_contexts.lookup_or_create(oi.soid);
+    obc->destructor_callback = new C_PG_ObjectContext(this, obc.get());
+    obc->obs.oi = oi;
+    obc->obs.exists = true;
+
+    obc->ssc = get_snapset_context(
+      soid, true,
+      soid.has_snapset() ? attrs : 0);
+    register_snapset_context(obc->ssc);
+
+    populate_obc_watchers(obc);
+
+    if (pool.info.require_rollback()) {
+      if (attrs) {
+	obc->attr_cache = *attrs;
+      } else {
+	int r = pgbackend->objects_get_attrs(
+	  soid,
+	  &obc->attr_cache);
+	assert(r == 0);
+      }
+    }
+
+    dout(10) << __func__ << ": creating obc from disk: " << obc
+	     << dendl;
+  }
+  assert(obc->ssc);
+  dout(10) << __func__ << ": " << obc << " " << soid
+	   << " " << obc->rwstate
+	   << " oi: " << obc->obs.oi
+	   << " ssc: " << obc->ssc
+	   << " snapset: " << obc->ssc->snapset << dendl;
+  return obc;
+}
+
+void ReplicatedPG::context_registry_on_change()
+{
+  pair i;
+  while (object_contexts.get_next(i.first, &i)) {
+    ObjectContextRef obc(i.second);
+    if (obc) {
+      for (map, WatchRef>::iterator j =
+	     obc->watchers.begin();
+	   j != obc->watchers.end();
+	   obc->watchers.erase(j++)) {
+	j->second->discard();
+      }
+    }
+  }
+}
+
+
+/*
+ * If we return an error, and set *pmissing, then promoting that
+ * object may help.
+ *
+ * If we return -EAGAIN, we will always set *pmissing to the missing
+ * object to wait for.
+ *
+ * If we return an error but do not set *pmissing, then we know the
+ * object does not exist.
+ */
+int ReplicatedPG::find_object_context(const hobject_t& oid,
+				      ObjectContextRef *pobc,
+				      bool can_create,
+				      bool map_snapid_to_clone,
+				      hobject_t *pmissing)
+{
+  hobject_t head(oid.oid, oid.get_key(), CEPH_NOSNAP, oid.hash,
+		 info.pgid.pool(), oid.get_namespace());
+  hobject_t snapdir(oid.oid, oid.get_key(), CEPH_SNAPDIR, oid.hash,
+		    info.pgid.pool(), oid.get_namespace());
+
+  // want the snapdir?
+  if (oid.snap == CEPH_SNAPDIR) {
+    // return head or snapdir, whichever exists.
+    ObjectContextRef obc = get_object_context(head, can_create);
+    if (!obc || !obc->obs.exists)
+      obc = get_object_context(snapdir, can_create);
+    if (!obc || !obc->obs.exists) {
+      // if we have neither, we would want to promote the head.
+      if (pmissing)
+	*pmissing = head;
+      return -ENOENT;
+    }
+    dout(10) << "find_object_context " << oid
+	     << " @" << oid.snap
+	     << " oi=" << obc->obs.oi
+	     << dendl;
+    *pobc = obc;
+
+    // always populate ssc for SNAPDIR...
+    if (!obc->ssc)
+      obc->ssc = get_snapset_context(
+	oid, true);
+    return 0;
+  }
+
+  // want the head?
+  if (oid.snap == CEPH_NOSNAP) {
+    ObjectContextRef obc = get_object_context(head, can_create);
+    if (!obc) {
+      if (pmissing)
+	*pmissing = head;
+      return -ENOENT;
+    }
+    dout(10) << "find_object_context " << oid
+	     << " @" << oid.snap
+	     << " oi=" << obc->obs.oi
+	     << dendl;
+    *pobc = obc;
+
+    if (can_create && !obc->ssc)
+      obc->ssc = get_snapset_context(oid, true);
+
+    return 0;
+  }
+
+  // we want a snap
+  if (!map_snapid_to_clone && pool.info.is_removed_snap(oid.snap)) {
+    dout(10) << __func__ << " snap " << oid.snap << " is removed" << dendl;
+    return -ENOENT;
+  }
+
+  SnapSetContext *ssc = get_snapset_context(oid, can_create);
+  if (!ssc || !(ssc->exists)) {
+    dout(20) << __func__ << " " << oid << " no snapset" << dendl;
+    if (pmissing)
+      *pmissing = head;  // start by getting the head
+    if (ssc)
+      put_snapset_context(ssc);
+    return -ENOENT;
+  }
+
+  if (map_snapid_to_clone) {
+    dout(10) << "find_object_context " << oid << " @" << oid.snap
+	     << " snapset " << ssc->snapset
+	     << " map_snapid_to_clone=true" << dendl;
+    if (oid.snap > ssc->snapset.seq) {
+      // already must be readable
+      ObjectContextRef obc = get_object_context(head, false);
+      dout(10) << "find_object_context " << oid << " @" << oid.snap
+	       << " snapset " << ssc->snapset
+	       << " maps to head" << dendl;
+      *pobc = obc;
+      put_snapset_context(ssc);
+      return (obc && obc->obs.exists) ? 0 : -ENOENT;
+    } else {
+      vector::const_iterator citer = std::find(
+	ssc->snapset.clones.begin(),
+	ssc->snapset.clones.end(),
+	oid.snap);
+      if (citer == ssc->snapset.clones.end()) {
+	dout(10) << "find_object_context " << oid << " @" << oid.snap
+		 << " snapset " << ssc->snapset
+		 << " maps to nothing" << dendl;
+	put_snapset_context(ssc);
+	return -ENOENT;
+      }
+
+      dout(10) << "find_object_context " << oid << " @" << oid.snap
+	       << " snapset " << ssc->snapset
+	       << " maps to " << oid << dendl;
+
+      if (pg_log.get_missing().is_missing(oid)) {
+	dout(10) << "find_object_context " << oid << " @" << oid.snap
+		 << " snapset " << ssc->snapset
+		 << " " << oid << " is missing" << dendl;
+	if (pmissing)
+	  *pmissing = oid;
+	put_snapset_context(ssc);
+	return -EAGAIN;
+      }
+
+      ObjectContextRef obc = get_object_context(oid, false);
+      if (!obc || !obc->obs.exists) {
+	dout(10) << "find_object_context " << oid << " @" << oid.snap
+		 << " snapset " << ssc->snapset
+		 << " " << oid << " is not present" << dendl;
+	if (pmissing)
+	  *pmissing = oid;
+	put_snapset_context(ssc);
+	return -ENOENT;
+      }
+      dout(10) << "find_object_context " << oid << " @" << oid.snap
+	       << " snapset " << ssc->snapset
+	       << " " << oid << " HIT" << dendl;
+      *pobc = obc;
+      put_snapset_context(ssc);
+      return 0;
+    }
+    assert(0); //unreachable
+  }
+
+  dout(10) << "find_object_context " << oid << " @" << oid.snap
+	   << " snapset " << ssc->snapset << dendl;
+ 
+  // head?
+  if (oid.snap > ssc->snapset.seq) {
+    if (ssc->snapset.head_exists) {
+      ObjectContextRef obc = get_object_context(head, false);
+      dout(10) << "find_object_context  " << head
+	       << " want " << oid.snap << " > snapset seq " << ssc->snapset.seq
+	       << " -- HIT " << obc->obs
+	       << dendl;
+      if (!obc->ssc)
+	obc->ssc = ssc;
+      else {
+	assert(ssc == obc->ssc);
+	put_snapset_context(ssc);
+      }
+      *pobc = obc;
+      return 0;
+    }
+    dout(10) << "find_object_context  " << head
+	     << " want " << oid.snap << " > snapset seq " << ssc->snapset.seq
+	     << " but head dne -- DNE"
+	     << dendl;
+    put_snapset_context(ssc);
+    return -ENOENT;
+  }
+
+  // which clone would it be?
+  unsigned k = 0;
+  while (k < ssc->snapset.clones.size() &&
+	 ssc->snapset.clones[k] < oid.snap)
+    k++;
+  if (k == ssc->snapset.clones.size()) {
+    dout(10) << "find_object_context  no clones with last >= oid.snap "
+	     << oid.snap << " -- DNE" << dendl;
+    put_snapset_context(ssc);
+    return -ENOENT;
+  }
+  hobject_t soid(oid.oid, oid.get_key(), ssc->snapset.clones[k], oid.hash,
+		 info.pgid.pool(), oid.get_namespace());
+
+  if (pg_log.get_missing().is_missing(soid)) {
+    dout(20) << "find_object_context  " << soid << " missing, try again later"
+	     << dendl;
+    if (pmissing)
+      *pmissing = soid;
+    put_snapset_context(ssc);
+    return -EAGAIN;
+  }
+
+  ObjectContextRef obc = get_object_context(soid, false);
+  if (!obc || !obc->obs.exists) {
+    dout(20) << __func__ << " missing clone " << soid << dendl;
+    if (pmissing)
+      *pmissing = soid;
+    put_snapset_context(ssc);
+    return -ENOENT;
+  }
+
+  if (!obc->ssc) {
+    obc->ssc = ssc;
+  } else {
+    assert(obc->ssc == ssc);
+    put_snapset_context(ssc);
+  }
+  ssc = 0;
+
+  // clone
+  dout(20) << "find_object_context  " << soid << " snaps " << obc->obs.oi.snaps
+	   << dendl;
+  snapid_t first = obc->obs.oi.snaps[obc->obs.oi.snaps.size()-1];
+  snapid_t last = obc->obs.oi.snaps[0];
+  if (first <= oid.snap) {
+    dout(20) << "find_object_context  " << soid << " [" << first << "," << last
+	     << "] contains " << oid.snap << " -- HIT " << obc->obs << dendl;
+    *pobc = obc;
+    return 0;
+  } else {
+    dout(20) << "find_object_context  " << soid << " [" << first << "," << last
+	     << "] does not contain " << oid.snap << " -- DNE" << dendl;
+    return -ENOENT;
+  }
+}
+
+void ReplicatedPG::object_context_destructor_callback(ObjectContext *obc)
+{
+  if (obc->ssc)
+    put_snapset_context(obc->ssc);
+}
+
+void ReplicatedPG::add_object_context_to_pg_stat(ObjectContextRef obc, pg_stat_t *pgstat)
+{
+  object_info_t& oi = obc->obs.oi;
+
+  dout(10) << "add_object_context_to_pg_stat " << oi.soid << dendl;
+  object_stat_sum_t stat;
+
+  stat.num_bytes += oi.size;
+
+  if (oi.soid.snap != CEPH_SNAPDIR)
+    stat.num_objects++;
+  if (oi.is_dirty())
+    stat.num_objects_dirty++;
+  if (oi.is_whiteout())
+    stat.num_whiteouts++;
+  if (oi.is_omap())
+    stat.num_objects_omap++;
+
+  if (oi.soid.snap && oi.soid.snap != CEPH_NOSNAP && oi.soid.snap != CEPH_SNAPDIR) {
+    stat.num_object_clones++;
+
+    if (!obc->ssc)
+      obc->ssc = get_snapset_context(oi.soid, false);
+    assert(obc->ssc);
+
+    // subtract off clone overlap
+    if (obc->ssc->snapset.clone_overlap.count(oi.soid.snap)) {
+      interval_set& o = obc->ssc->snapset.clone_overlap[oi.soid.snap];
+      for (interval_set::const_iterator r = o.begin();
+	   r != o.end();
+	   ++r) {
+	stat.num_bytes -= r.get_len();
+      }	  
+    }
+  }
+
+  // add it in
+  pgstat->stats.sum.add(stat);
+  if (oi.category.length())
+    pgstat->stats.cat_sum[oi.category].add(stat);
+}
+
+void ReplicatedPG::kick_object_context_blocked(ObjectContextRef obc)
+{
+  const hobject_t& soid = obc->obs.oi.soid;
+  map >::iterator p = waiting_for_blocked_object.find(soid);
+  if (p == waiting_for_blocked_object.end())
+    return;
+
+  if (obc->is_blocked()) {
+    dout(10) << __func__ << " " << soid << " still blocked" << dendl;
+    return;
+  }
+
+  list& ls = p->second;
+  dout(10) << __func__ << " " << soid << " requeuing " << ls.size() << " requests" << dendl;
+  requeue_ops(ls);
+  waiting_for_blocked_object.erase(p);
+
+  if (obc->requeue_scrub_on_unblock)
+    osd->queue_for_scrub(this);
+}
+
+SnapSetContext *ReplicatedPG::create_snapset_context(const hobject_t& oid)
+{
+  Mutex::Locker l(snapset_contexts_lock);
+  SnapSetContext *ssc = new SnapSetContext(oid.get_snapdir());
+  _register_snapset_context(ssc);
+  ssc->ref++;
+  return ssc;
+}
+
+SnapSetContext *ReplicatedPG::get_snapset_context(
+  const hobject_t& oid,
+  bool can_create,
+  map *attrs)
+{
+  Mutex::Locker l(snapset_contexts_lock);
+  SnapSetContext *ssc;
+  map::iterator p = snapset_contexts.find(
+    oid.get_snapdir());
+  if (p != snapset_contexts.end()) {
+    if (can_create || p->second->exists) {
+      ssc = p->second;
+      ssc->exists = true;
+    } else {
+      return NULL;
+    }
+  } else {
+    bufferlist bv;
+    if (!attrs) {
+      int r = pgbackend->objects_get_attr(oid.get_head(), SS_ATTR, &bv);
+      if (r < 0) {
+	// try _snapset
+	r = pgbackend->objects_get_attr(oid.get_snapdir(), SS_ATTR, &bv);
+	if (r < 0 && !can_create)
+	  return NULL;
+      }
+    } else {
+      assert(attrs->count(SS_ATTR));
+      bv = attrs->find(SS_ATTR)->second;
+    }
+    ssc = new SnapSetContext(oid.get_snapdir());
+    _register_snapset_context(ssc);
+    if (bv.length()) {
+      bufferlist::iterator bvp = bv.begin();
+      ssc->snapset.decode(bvp);
+    }
+  }
+  assert(ssc);
+  ssc->ref++;
+  return ssc;
+}
+
+void ReplicatedPG::put_snapset_context(SnapSetContext *ssc)
+{
+  Mutex::Locker l(snapset_contexts_lock);
+  --ssc->ref;
+  if (ssc->ref == 0) {
+    if (ssc->registered)
+      snapset_contexts.erase(ssc->oid);
+    delete ssc;
+  }
+}
+
+// sub op modify
+
+void ReplicatedBackend::sub_op_modify(OpRequestRef op)
+{
+  MOSDSubOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+
+  const hobject_t& soid = m->poid;
+
+  const char *opname;
+  if (m->noop)
+    opname = "no-op";
+  else if (m->ops.size())
+    opname = ceph_osd_op_name(m->ops[0].op.op);
+  else
+    opname = "trans";
+
+  dout(10) << "sub_op_modify " << opname 
+           << " " << soid 
+           << " v " << m->version
+	   << (m->noop ? " NOOP" : "")
+	   << (m->logbl.length() ? " (transaction)" : " (parallel exec")
+	   << " " << m->logbl.length()
+	   << dendl;  
+
+  // sanity checks
+  assert(m->map_epoch >= get_info().history.same_interval_since);
+  
+  // we better not be missing this.
+  assert(!parent->get_log().get_missing().is_missing(soid));
+
+  int ackerosd = m->get_source().num();
+  
+  op->mark_started();
+
+  RepModifyRef rm(new RepModify);
+  rm->op = op;
+  rm->ackerosd = ackerosd;
+  rm->last_complete = get_info().last_complete;
+  rm->epoch_started = get_osdmap()->get_epoch();
+
+  if (!m->noop) {
+    assert(m->logbl.length());
+    // shipped transaction and log entries
+    vector log;
+
+    bufferlist::iterator p = m->get_data().begin();
+    ::decode(rm->opt, p);
+    if (!(m->get_connection()->get_features() & CEPH_FEATURE_OSD_SNAPMAPPER))
+      rm->opt.set_tolerate_collection_add_enoent();
+
+    if (m->new_temp_oid != hobject_t()) {
+      dout(20) << __func__ << " start tracking temp " << m->new_temp_oid << dendl;
+      add_temp_obj(m->new_temp_oid);
+      get_temp_coll(&rm->localt);
+    }
+    if (m->discard_temp_oid != hobject_t()) {
+      dout(20) << __func__ << " stop tracking temp " << m->discard_temp_oid << dendl;
+      if (rm->opt.empty()) {
+	dout(10) << __func__ << ": removing object " << m->discard_temp_oid
+		 << " since we won't get the transaction" << dendl;
+	rm->localt.remove(temp_coll, m->discard_temp_oid);
+      }
+      clear_temp_obj(m->discard_temp_oid);
+    }
+
+    p = m->logbl.begin();
+    ::decode(log, p);
+    if (m->hobject_incorrect_pool) {
+      for (vector::iterator i = log.begin();
+	  i != log.end();
+	  ++i) {
+	if (!i->soid.is_max() && i->soid.pool == -1)
+	  i->soid.pool = get_info().pgid.pool();
+      }
+      rm->opt.set_pool_override(get_info().pgid.pool());
+    }
+    rm->opt.set_replica();
+
+    bool update_snaps = false;
+    if (!rm->opt.empty()) {
+      // If the opt is non-empty, we infer we are before
+      // last_backfill (according to the primary, not our
+      // not-quite-accurate value), and should update the
+      // collections now.  Otherwise, we do it later on push.
+      update_snaps = true;
+    }
+    parent->update_stats(m->pg_stats);
+    parent->log_operation(
+      log,
+      m->updated_hit_set_history,
+      m->pg_trim_to,
+      m->pg_trim_rollback_to,
+      update_snaps,
+      &(rm->localt));
+      
+    rm->bytes_written = rm->opt.get_encoded_bytes();
+
+  } else {
+    assert(0);
+    #if 0
+    // just trim the log
+    if (m->pg_trim_to != eversion_t()) {
+      pg_log.trim(m->pg_trim_to, info);
+      dirty_info = true;
+      write_if_dirty(rm->localt);
+    }
+    #endif
+  }
+  
+  op->mark_started();
+
+  rm->localt.append(rm->opt);
+  rm->localt.register_on_commit(
+    parent->bless_context(
+      new C_OSD_RepModifyCommit(this, rm)));
+  rm->localt.register_on_applied(
+    parent->bless_context(
+      new C_OSD_RepModifyApply(this, rm)));
+  parent->queue_transaction(&(rm->localt), op);
+  // op is cleaned up by oncommit/onapply when both are executed
+}
+
+void ReplicatedBackend::sub_op_modify_applied(RepModifyRef rm)
+{
+  rm->op->mark_event("sub_op_applied");
+  rm->applied = true;
+
+  dout(10) << "sub_op_modify_applied on " << rm << " op "
+	   << *rm->op->get_req() << dendl;
+  MOSDSubOp *m = static_cast(rm->op->get_req());
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+  
+  if (!rm->committed) {
+    // send ack to acker only if we haven't sent a commit already
+    MOSDSubOpReply *ack = new MOSDSubOpReply(
+      m, parent->whoami_shard(),
+      0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+    ack->set_priority(CEPH_MSG_PRIO_HIGH); // this better match commit priority!
+    get_parent()->send_message_osd_cluster(
+      rm->ackerosd, ack, get_osdmap()->get_epoch());
+  }
+  
+  parent->op_applied(m->version);
+}
+
+void ReplicatedBackend::sub_op_modify_commit(RepModifyRef rm)
+{
+  rm->op->mark_commit_sent();
+  rm->committed = true;
+
+  // send commit.
+  dout(10) << "sub_op_modify_commit on op " << *rm->op->get_req()
+	   << ", sending commit to osd." << rm->ackerosd
+	   << dendl;
+  
+  assert(get_osdmap()->is_up(rm->ackerosd));
+  get_parent()->update_last_complete_ondisk(rm->last_complete);
+  MOSDSubOpReply *commit = new MOSDSubOpReply(
+    static_cast(rm->op->get_req()),
+    get_parent()->whoami_shard(),
+    0, get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ONDISK);
+  commit->set_last_complete_ondisk(rm->last_complete);
+  commit->set_priority(CEPH_MSG_PRIO_HIGH); // this better match ack priority!
+  get_parent()->send_message_osd_cluster(
+    rm->ackerosd, commit, get_osdmap()->get_epoch());
+  
+  log_subop_stats(get_parent()->get_logger(), rm->op,
+		  l_osd_sop_w_inb, l_osd_sop_w_lat);
+}
+
+
+// ===========================================================
+
+void ReplicatedBackend::calc_head_subsets(
+  ObjectContextRef obc, SnapSet& snapset, const hobject_t& head,
+  const pg_missing_t& missing,
+  const hobject_t &last_backfill,
+  interval_set& data_subset,
+  map >& clone_subsets)
+{
+  dout(10) << "calc_head_subsets " << head
+	   << " clone_overlap " << snapset.clone_overlap << dendl;
+
+  uint64_t size = obc->obs.oi.size;
+  if (size)
+    data_subset.insert(0, size);
+
+  if (get_parent()->get_pool().allow_incomplete_clones()) {
+    dout(10) << __func__ << ": caching (was) enabled, skipping clone subsets" << dendl;
+    return;
+  }
+
+  if (!cct->_conf->osd_recover_clone_overlap) {
+    dout(10) << "calc_head_subsets " << head << " -- osd_recover_clone_overlap disabled" << dendl;
+    return;
+  }
+
+
+  interval_set cloning;
+  interval_set prev;
+  if (size)
+    prev.insert(0, size);    
+  
+  for (int j=snapset.clones.size()-1; j>=0; j--) {
+    hobject_t c = head;
+    c.snap = snapset.clones[j];
+    prev.intersection_of(snapset.clone_overlap[snapset.clones[j]]);
+    if (!missing.is_missing(c) && c < last_backfill) {
+      dout(10) << "calc_head_subsets " << head << " has prev " << c
+	       << " overlap " << prev << dendl;
+      clone_subsets[c] = prev;
+      cloning.union_of(prev);
+      break;
+    }
+    dout(10) << "calc_head_subsets " << head << " does not have prev " << c
+	     << " overlap " << prev << dendl;
+  }
+
+
+  if (cloning.num_intervals() > cct->_conf->osd_recover_clone_overlap_limit) {
+    dout(10) << "skipping clone, too many holes" << dendl;
+    clone_subsets.clear();
+    cloning.clear();
+  }
+
+  // what's left for us to push?
+  data_subset.subtract(cloning);
+
+  dout(10) << "calc_head_subsets " << head
+	   << "  data_subset " << data_subset
+	   << "  clone_subsets " << clone_subsets << dendl;
+}
+
+void ReplicatedBackend::calc_clone_subsets(
+  SnapSet& snapset, const hobject_t& soid,
+  const pg_missing_t& missing,
+  const hobject_t &last_backfill,
+  interval_set& data_subset,
+  map >& clone_subsets)
+{
+  dout(10) << "calc_clone_subsets " << soid
+	   << " clone_overlap " << snapset.clone_overlap << dendl;
+
+  uint64_t size = snapset.clone_size[soid.snap];
+  if (size)
+    data_subset.insert(0, size);
+
+  if (get_parent()->get_pool().allow_incomplete_clones()) {
+    dout(10) << __func__ << ": caching (was) enabled, skipping clone subsets" << dendl;
+    return;
+  }
+
+  if (!cct->_conf->osd_recover_clone_overlap) {
+    dout(10) << "calc_clone_subsets " << soid << " -- osd_recover_clone_overlap disabled" << dendl;
+    return;
+  }
+  
+  unsigned i;
+  for (i=0; i < snapset.clones.size(); i++)
+    if (snapset.clones[i] == soid.snap)
+      break;
+
+  // any overlap with next older clone?
+  interval_set cloning;
+  interval_set prev;
+  if (size)
+    prev.insert(0, size);    
+  for (int j=i-1; j>=0; j--) {
+    hobject_t c = soid;
+    c.snap = snapset.clones[j];
+    prev.intersection_of(snapset.clone_overlap[snapset.clones[j]]);
+    if (!missing.is_missing(c) && c < last_backfill) {
+      dout(10) << "calc_clone_subsets " << soid << " has prev " << c
+	       << " overlap " << prev << dendl;
+      clone_subsets[c] = prev;
+      cloning.union_of(prev);
+      break;
+    }
+    dout(10) << "calc_clone_subsets " << soid << " does not have prev " << c
+	     << " overlap " << prev << dendl;
+  }
+  
+  // overlap with next newest?
+  interval_set next;
+  if (size)
+    next.insert(0, size);    
+  for (unsigned j=i+1; j cct->_conf->osd_recover_clone_overlap_limit) {
+    dout(10) << "skipping clone, too many holes" << dendl;
+    clone_subsets.clear();
+    cloning.clear();
+  }
+
+  
+  // what's left for us to push?
+  data_subset.subtract(cloning);
+
+  dout(10) << "calc_clone_subsets " << soid
+	   << "  data_subset " << data_subset
+	   << "  clone_subsets " << clone_subsets << dendl;
+}
+
+
+/** pull - request object from a peer
+ */
+
+/*
+ * Return values:
+ *  NONE  - didn't pull anything
+ *  YES   - pulled what the caller wanted
+ *  OTHER - needed to pull something else first (_head or _snapdir)
+ */
+enum { PULL_NONE, PULL_OTHER, PULL_YES };
+
+void ReplicatedBackend::prepare_pull(
+  eversion_t v,
+  const hobject_t& soid,
+  ObjectContextRef headctx,
+  RPGHandle *h)
+{
+  assert(get_parent()->get_local_missing().missing.count(soid));
+  eversion_t _v = get_parent()->get_local_missing().missing.find(
+    soid)->second.need;
+  assert(_v == v);
+  const map > &missing_loc(
+    get_parent()->get_missing_loc_shards());
+  const map &peer_missing(
+    get_parent()->get_shard_missing());
+  map >::const_iterator q = missing_loc.find(soid);
+  assert(q != missing_loc.end());
+  assert(!q->second.empty());
+
+  // pick a pullee
+  vector shuffle(q->second.begin(), q->second.end());
+  random_shuffle(shuffle.begin(), shuffle.end());
+  vector::iterator p = shuffle.begin();
+  assert(get_osdmap()->is_up(p->osd));
+  pg_shard_t fromshard = *p;
+
+  dout(7) << "pull " << soid
+	  << " v " << v
+	  << " on osds " << *p
+	  << " from osd." << fromshard
+	  << dendl;
+
+  assert(peer_missing.count(fromshard));
+  const pg_missing_t &pmissing = peer_missing.find(fromshard)->second;
+  if (pmissing.is_missing(soid, v)) {
+    assert(pmissing.missing.find(soid)->second.have != v);
+    dout(10) << "pulling soid " << soid << " from osd " << fromshard
+	     << " at version " << pmissing.missing.find(soid)->second.have
+	     << " rather than at version " << v << dendl;
+    v = pmissing.missing.find(soid)->second.have;
+    assert(get_parent()->get_log().get_log().objects.count(soid) &&
+	   (get_parent()->get_log().get_log().objects.find(soid)->second->op ==
+	    pg_log_entry_t::LOST_REVERT) &&
+	   (get_parent()->get_log().get_log().objects.find(
+	     soid)->second->reverting_to ==
+	    v));
+  }
+  
+  ObjectRecoveryInfo recovery_info;
+
+  if (soid.is_snap()) {
+    assert(!get_parent()->get_local_missing().is_missing(
+	     soid.get_head()) ||
+	   !get_parent()->get_local_missing().is_missing(
+	     soid.get_snapdir()));
+    assert(headctx);
+    // check snapset
+    SnapSetContext *ssc = headctx->ssc;
+    assert(ssc);
+    dout(10) << " snapset " << ssc->snapset << dendl;
+    calc_clone_subsets(ssc->snapset, soid, get_parent()->get_local_missing(),
+		       get_info().last_backfill,
+		       recovery_info.copy_subset,
+		       recovery_info.clone_subset);
+    // FIXME: this may overestimate if we are pulling multiple clones in parallel...
+    dout(10) << " pulling " << recovery_info << dendl;
+  } else {
+    // pulling head or unversioned object.
+    // always pull the whole thing.
+    recovery_info.copy_subset.insert(0, (uint64_t)-1);
+    recovery_info.size = ((uint64_t)-1);
+  }
+
+  h->pulls[fromshard].push_back(PullOp());
+  PullOp &op = h->pulls[fromshard].back();
+  op.soid = soid;
+
+  op.recovery_info = recovery_info;
+  op.recovery_info.soid = soid;
+  op.recovery_info.version = v;
+  op.recovery_progress.data_complete = false;
+  op.recovery_progress.omap_complete = false;
+  op.recovery_progress.data_recovered_to = 0;
+  op.recovery_progress.first = true;
+
+  assert(!pulling.count(soid));
+  pull_from_peer[fromshard].insert(soid);
+  PullInfo &pi = pulling[soid];
+  pi.head_ctx = headctx;
+  pi.recovery_info = op.recovery_info;
+  pi.recovery_progress = op.recovery_progress;
+}
+
+int ReplicatedPG::recover_missing(
+  const hobject_t &soid, eversion_t v,
+  int priority,
+  PGBackend::RecoveryHandle *h)
+{
+  if (missing_loc.is_unfound(soid)) {
+    dout(7) << "pull " << soid
+	    << " v " << v 
+	    << " but it is unfound" << dendl;
+    return PULL_NONE;
+  }
+
+  // is this a snapped object?  if so, consult the snapset.. we may not need the entire object!
+  ObjectContextRef obc;
+  ObjectContextRef head_obc;
+  if (soid.snap && soid.snap < CEPH_NOSNAP) {
+    // do we have the head and/or snapdir?
+    hobject_t head = soid.get_head();
+    if (pg_log.get_missing().is_missing(head)) {
+      if (recovering.count(head)) {
+	dout(10) << " missing but already recovering head " << head << dendl;
+	return PULL_NONE;
+      } else {
+	int r = recover_missing(
+	  head, pg_log.get_missing().missing.find(head)->second.need, priority,
+	  h);
+	if (r != PULL_NONE)
+	  return PULL_OTHER;
+	return PULL_NONE;
+      }
+    }
+    head = soid.get_snapdir();
+    if (pg_log.get_missing().is_missing(head)) {
+      if (recovering.count(head)) {
+	dout(10) << " missing but already recovering snapdir " << head << dendl;
+	return PULL_NONE;
+      } else {
+	int r = recover_missing(
+	  head, pg_log.get_missing().missing.find(head)->second.need, priority,
+	  h);
+	if (r != PULL_NONE)
+	  return PULL_OTHER;
+	return PULL_NONE;
+      }
+    }
+
+    // we must have one or the other
+    head_obc = get_object_context(
+      soid.get_head(),
+      false,
+      0);
+    if (!head_obc)
+      head_obc = get_object_context(
+	soid.get_snapdir(),
+	false,
+	0);
+    assert(head_obc);
+  }
+  start_recovery_op(soid);
+  assert(!recovering.count(soid));
+  recovering.insert(make_pair(soid, obc));
+  pgbackend->recover_object(
+    soid,
+    v,
+    head_obc,
+    obc,
+    h);
+  return PULL_YES;
+}
+
+void ReplicatedPG::send_remove_op(
+  const hobject_t& oid, eversion_t v, pg_shard_t peer)
+{
+  ceph_tid_t tid = osd->get_tid();
+  osd_reqid_t rid(osd->get_cluster_msgr_name(), 0, tid);
+
+  dout(10) << "send_remove_op " << oid << " from osd." << peer
+	   << " tid " << tid << dendl;
+
+  MOSDSubOp *subop = new MOSDSubOp(
+    rid, pg_whoami, spg_t(info.pgid.pgid, peer.shard),
+    oid, false, CEPH_OSD_FLAG_ACK,
+    get_osdmap()->get_epoch(), tid, v);
+  subop->ops = vector(1);
+  subop->ops[0].op.op = CEPH_OSD_OP_DELETE;
+
+  osd->send_message_osd_cluster(peer.osd, subop, get_osdmap()->get_epoch());
+}
+
+/*
+ * intelligently push an object to a replica.  make use of existing
+ * clones/heads and dup data ranges where possible.
+ */
+void ReplicatedBackend::prep_push_to_replica(
+  ObjectContextRef obc, const hobject_t& soid, pg_shard_t peer,
+  PushOp *pop)
+{
+  const object_info_t& oi = obc->obs.oi;
+  uint64_t size = obc->obs.oi.size;
+
+  dout(10) << __func__ << ": " << soid << " v" << oi.version
+	   << " size " << size << " to osd." << peer << dendl;
+
+  map > clone_subsets;
+  interval_set data_subset;
+
+  // are we doing a clone on the replica?
+  if (soid.snap && soid.snap < CEPH_NOSNAP) {	
+    hobject_t head = soid;
+    head.snap = CEPH_NOSNAP;
+
+    // try to base push off of clones that succeed/preceed poid
+    // we need the head (and current SnapSet) locally to do that.
+    if (get_parent()->get_local_missing().is_missing(head)) {
+      dout(15) << "push_to_replica missing head " << head << ", pushing raw clone" << dendl;
+      return prep_push(obc, soid, peer, pop);
+    }
+    hobject_t snapdir = head;
+    snapdir.snap = CEPH_SNAPDIR;
+    if (get_parent()->get_local_missing().is_missing(snapdir)) {
+      dout(15) << "push_to_replica missing snapdir " << snapdir
+	       << ", pushing raw clone" << dendl;
+      return prep_push(obc, soid, peer, pop);
+    }
+    
+    SnapSetContext *ssc = obc->ssc;
+    assert(ssc);
+    dout(15) << "push_to_replica snapset is " << ssc->snapset << dendl;
+    map::const_iterator pm =
+      get_parent()->get_shard_missing().find(peer);
+    assert(pm != get_parent()->get_shard_missing().end());
+    map::const_iterator pi =
+      get_parent()->get_shard_info().find(peer);
+    assert(pi != get_parent()->get_shard_info().end());
+    calc_clone_subsets(ssc->snapset, soid,
+		       pm->second,
+		       pi->second.last_backfill,
+		       data_subset, clone_subsets);
+  } else if (soid.snap == CEPH_NOSNAP) {
+    // pushing head or unversioned object.
+    // base this on partially on replica's clones?
+    SnapSetContext *ssc = obc->ssc;
+    assert(ssc);
+    dout(15) << "push_to_replica snapset is " << ssc->snapset << dendl;
+    calc_head_subsets(
+      obc,
+      ssc->snapset, soid, get_parent()->get_shard_missing().find(peer)->second,
+      get_parent()->get_shard_info().find(peer)->second.last_backfill,
+      data_subset, clone_subsets);
+  }
+
+  prep_push(obc, soid, peer, oi.version, data_subset, clone_subsets, pop);
+}
+
+void ReplicatedBackend::prep_push(ObjectContextRef obc,
+			     const hobject_t& soid, pg_shard_t peer,
+			     PushOp *pop)
+{
+  interval_set data_subset;
+  if (obc->obs.oi.size)
+    data_subset.insert(0, obc->obs.oi.size);
+  map > clone_subsets;
+
+  prep_push(obc, soid, peer,
+	    obc->obs.oi.version, data_subset, clone_subsets,
+	    pop);
+}
+
+void ReplicatedBackend::prep_push(
+  ObjectContextRef obc,
+  const hobject_t& soid, pg_shard_t peer,
+  eversion_t version,
+  interval_set &data_subset,
+  map >& clone_subsets,
+  PushOp *pop)
+{
+  get_parent()->begin_peer_recover(peer, soid);
+  // take note.
+  PushInfo &pi = pushing[soid][peer];
+  pi.obc = obc;
+  pi.recovery_info.size = obc->obs.oi.size;
+  pi.recovery_info.copy_subset = data_subset;
+  pi.recovery_info.clone_subset = clone_subsets;
+  pi.recovery_info.soid = soid;
+  pi.recovery_info.oi = obc->obs.oi;
+  pi.recovery_info.version = version;
+  pi.recovery_progress.first = true;
+  pi.recovery_progress.data_recovered_to = 0;
+  pi.recovery_progress.data_complete = 0;
+  pi.recovery_progress.omap_complete = 0;
+
+  ObjectRecoveryProgress new_progress;
+  int r = build_push_op(pi.recovery_info,
+			pi.recovery_progress,
+			&new_progress,
+			pop,
+			&(pi.stat));
+  assert(r == 0);
+  pi.recovery_progress = new_progress;
+}
+
+int ReplicatedBackend::send_pull_legacy(int prio, pg_shard_t peer,
+					const ObjectRecoveryInfo &recovery_info,
+					ObjectRecoveryProgress progress)
+{
+  // send op
+  ceph_tid_t tid = get_parent()->get_tid();
+  osd_reqid_t rid(get_parent()->get_cluster_msgr_name(), 0, tid);
+
+  dout(10) << "send_pull_op " << recovery_info.soid << " "
+	   << recovery_info.version
+	   << " first=" << progress.first
+	   << " data " << recovery_info.copy_subset
+	   << " from osd." << peer
+	   << " tid " << tid << dendl;
+
+  MOSDSubOp *subop = new MOSDSubOp(
+    rid, parent->whoami_shard(),
+    get_info().pgid, recovery_info.soid,
+    false, CEPH_OSD_FLAG_ACK,
+    get_osdmap()->get_epoch(), tid,
+    recovery_info.version);
+  subop->set_priority(prio);
+  subop->ops = vector(1);
+  subop->ops[0].op.op = CEPH_OSD_OP_PULL;
+  subop->ops[0].op.extent.length = cct->_conf->osd_recovery_max_chunk;
+  subop->recovery_info = recovery_info;
+  subop->recovery_progress = progress;
+
+  get_parent()->send_message_osd_cluster(
+    peer.osd, subop, get_osdmap()->get_epoch());
+
+  get_parent()->get_logger()->inc(l_osd_pull);
+  return 0;
+}
+
+void ReplicatedBackend::submit_push_data(
+  ObjectRecoveryInfo &recovery_info,
+  bool first,
+  bool complete,
+  const interval_set &intervals_included,
+  bufferlist data_included,
+  bufferlist omap_header,
+  map &attrs,
+  map &omap_entries,
+  ObjectStore::Transaction *t)
+{
+  coll_t target_coll;
+  if (first && complete) {
+    target_coll = coll;
+  } else {
+    dout(10) << __func__ << ": Creating oid "
+	     << recovery_info.soid << " in the temp collection" << dendl;
+    add_temp_obj(recovery_info.soid);
+    target_coll = get_temp_coll(t);
+  }
+
+  if (first) {
+    get_parent()->on_local_recover_start(recovery_info.soid, t);
+    t->remove(get_temp_coll(t), recovery_info.soid);
+    t->touch(target_coll, recovery_info.soid);
+    t->omap_setheader(target_coll, recovery_info.soid, omap_header);
+  }
+  uint64_t off = 0;
+  for (interval_set::const_iterator p = intervals_included.begin();
+       p != intervals_included.end();
+       ++p) {
+    bufferlist bit;
+    bit.substr_of(data_included, off, p.get_len());
+    t->write(target_coll, recovery_info.soid,
+	     p.get_start(), p.get_len(), bit);
+    off += p.get_len();
+  }
+
+  t->omap_setkeys(target_coll, recovery_info.soid,
+		  omap_entries);
+  t->setattrs(target_coll, recovery_info.soid,
+	      attrs);
+
+  if (complete) {
+    if (!first) {
+      dout(10) << __func__ << ": Removing oid "
+	       << recovery_info.soid << " from the temp collection" << dendl;
+      clear_temp_obj(recovery_info.soid);
+      t->collection_move(coll, target_coll, recovery_info.soid);
+    }
+
+    submit_push_complete(recovery_info, t);
+  }
+}
+
+void ReplicatedBackend::submit_push_complete(ObjectRecoveryInfo &recovery_info,
+					     ObjectStore::Transaction *t)
+{
+  for (map >::const_iterator p =
+	 recovery_info.clone_subset.begin();
+       p != recovery_info.clone_subset.end();
+       ++p) {
+    for (interval_set::const_iterator q = p->second.begin();
+	 q != p->second.end();
+	 ++q) {
+      dout(15) << " clone_range " << p->first << " "
+	       << q.get_start() << "~" << q.get_len() << dendl;
+      t->clone_range(coll, p->first, recovery_info.soid,
+		     q.get_start(), q.get_len(), q.get_start());
+    }
+  }
+}
+
+ObjectRecoveryInfo ReplicatedBackend::recalc_subsets(
+  const ObjectRecoveryInfo& recovery_info,
+  SnapSetContext *ssc)
+{
+  if (!recovery_info.soid.snap || recovery_info.soid.snap >= CEPH_NOSNAP)
+    return recovery_info;
+  ObjectRecoveryInfo new_info = recovery_info;
+  new_info.copy_subset.clear();
+  new_info.clone_subset.clear();
+  assert(ssc);
+  calc_clone_subsets(ssc->snapset, new_info.soid, get_parent()->get_local_missing(),
+		     get_info().last_backfill,
+		     new_info.copy_subset, new_info.clone_subset);
+  return new_info;
+}
+
+bool ReplicatedBackend::handle_pull_response(
+  pg_shard_t from, PushOp &pop, PullOp *response,
+  list *to_continue,
+  ObjectStore::Transaction *t
+  )
+{
+  interval_set data_included = pop.data_included;
+  bufferlist data;
+  data.claim(pop.data);
+  dout(10) << "handle_pull_response "
+	   << pop.recovery_info
+	   << pop.after_progress
+	   << " data.size() is " << data.length()
+	   << " data_included: " << data_included
+	   << dendl;
+  if (pop.version == eversion_t()) {
+    // replica doesn't have it!
+    _failed_push(from, pop.soid);
+    return false;
+  }
+
+  hobject_t &hoid = pop.soid;
+  assert((data_included.empty() && data.length() == 0) ||
+	 (!data_included.empty() && data.length() > 0));
+
+  if (!pulling.count(hoid)) {
+    return false;
+  }
+
+  PullInfo &pi = pulling[hoid];
+  if (pi.recovery_info.size == (uint64_t(-1))) {
+    pi.recovery_info.size = pop.recovery_info.size;
+    pi.recovery_info.copy_subset.intersection_of(
+      pop.recovery_info.copy_subset);
+  }
+
+  bool first = pi.recovery_progress.first;
+  if (first) {
+    pi.obc = get_parent()->get_obc(pi.recovery_info.soid, pop.attrset);
+    pi.recovery_info.oi = pi.obc->obs.oi;
+    pi.recovery_info = recalc_subsets(pi.recovery_info, pi.obc->ssc);
+  }
+
+
+  interval_set usable_intervals;
+  bufferlist usable_data;
+  trim_pushed_data(pi.recovery_info.copy_subset,
+		   data_included,
+		   data,
+		   &usable_intervals,
+		   &usable_data);
+  data_included = usable_intervals;
+  data.claim(usable_data);
+
+
+  pi.recovery_progress = pop.after_progress;
+
+  pi.stat.num_bytes_recovered += data.length();
+
+  dout(10) << "new recovery_info " << pi.recovery_info
+	   << ", new progress " << pi.recovery_progress
+	   << dendl;
+
+  bool complete = pi.is_complete();
+
+  submit_push_data(pi.recovery_info, first,
+		   complete,
+		   data_included, data,
+		   pop.omap_header,
+		   pop.attrset,
+		   pop.omap_entries,
+		   t);
+
+  pi.stat.num_keys_recovered += pop.omap_entries.size();
+
+  if (complete) {
+    to_continue->push_back(hoid);
+    pi.stat.num_objects_recovered++;
+    get_parent()->on_local_recover(
+      hoid, pi.stat, pi.recovery_info, pi.obc, t);
+    pull_from_peer[from].erase(hoid);
+    if (pull_from_peer[from].empty())
+      pull_from_peer.erase(from);
+    return false;
+  } else {
+    response->soid = pop.soid;
+    response->recovery_info = pi.recovery_info;
+    response->recovery_progress = pi.recovery_progress;
+    return true;
+  }
+}
+
+struct C_OnPushCommit : public Context {
+  ReplicatedPG *pg;
+  OpRequestRef op;
+  C_OnPushCommit(ReplicatedPG *pg, OpRequestRef op) : pg(pg), op(op) {}
+  void finish(int) {
+    op->mark_event("committed");
+    log_subop_stats(pg->osd->logger, op, l_osd_push_inb, l_osd_sop_push_lat);
+  }
+};
+
+void ReplicatedBackend::handle_push(
+  pg_shard_t from, PushOp &pop, PushReplyOp *response,
+  ObjectStore::Transaction *t)
+{
+  dout(10) << "handle_push "
+	   << pop.recovery_info
+	   << pop.after_progress
+	   << dendl;
+  bufferlist data;
+  data.claim(pop.data);
+  bool first = pop.before_progress.first;
+  bool complete = pop.after_progress.data_complete &&
+    pop.after_progress.omap_complete;
+
+  response->soid = pop.recovery_info.soid;
+  submit_push_data(pop.recovery_info,
+		   first,
+		   complete,
+		   pop.data_included,
+		   data,
+		   pop.omap_header,
+		   pop.attrset,
+		   pop.omap_entries,
+		   t);
+
+  if (complete)
+    get_parent()->on_local_recover(
+      pop.recovery_info.soid,
+      object_stat_sum_t(),
+      pop.recovery_info,
+      ObjectContextRef(), // ok, is replica
+      t);
+}
+
+void ReplicatedBackend::send_pushes(int prio, map > &pushes)
+{
+  for (map >::iterator i = pushes.begin();
+       i != pushes.end();
+       ++i) {
+    ConnectionRef con = get_parent()->get_con_osd_cluster(
+      i->first.osd,
+      get_osdmap()->get_epoch());
+    if (!con)
+      continue;
+    if (!(con->get_features() & CEPH_FEATURE_OSD_PACKED_RECOVERY)) {
+      for (vector::iterator j = i->second.begin();
+	   j != i->second.end();
+	   ++j) {
+	dout(20) << __func__ << ": sending push (legacy) " << *j
+		 << " to osd." << i->first << dendl;
+	send_push_op_legacy(prio, i->first, *j);
+      }
+    } else {
+      vector::iterator j = i->second.begin();
+      while (j != i->second.end()) {
+	uint64_t cost = 0;
+	uint64_t pushes = 0;
+	MOSDPGPush *msg = new MOSDPGPush();
+	msg->from = get_parent()->whoami_shard();
+	msg->pgid = get_parent()->primary_spg_t();
+	msg->map_epoch = get_osdmap()->get_epoch();
+	msg->set_priority(prio);
+	for (;
+	     (j != i->second.end() &&
+	      cost < cct->_conf->osd_max_push_cost &&
+	      pushes < cct->_conf->osd_max_push_objects) ;
+	     ++j) {
+	  dout(20) << __func__ << ": sending push " << *j
+		   << " to osd." << i->first << dendl;
+	  cost += j->cost(cct);
+	  pushes += 1;
+	  msg->pushes.push_back(*j);
+	}
+	msg->compute_cost(cct);
+	get_parent()->send_message_osd_cluster(msg, con);
+      }
+    }
+  }
+}
+
+void ReplicatedBackend::send_pulls(int prio, map > &pulls)
+{
+  for (map >::iterator i = pulls.begin();
+       i != pulls.end();
+       ++i) {
+    ConnectionRef con = get_parent()->get_con_osd_cluster(
+      i->first.osd,
+      get_osdmap()->get_epoch());
+    if (!con)
+      continue;
+    if (!(con->get_features() & CEPH_FEATURE_OSD_PACKED_RECOVERY)) {
+      for (vector::iterator j = i->second.begin();
+	   j != i->second.end();
+	   ++j) {
+	dout(20) << __func__ << ": sending pull (legacy) " << *j
+		 << " to osd." << i->first << dendl;
+	send_pull_legacy(
+	  prio,
+	  i->first,
+	  j->recovery_info,
+	  j->recovery_progress);
+      }
+    } else {
+      dout(20) << __func__ << ": sending pulls " << i->second
+	       << " to osd." << i->first << dendl;
+      MOSDPGPull *msg = new MOSDPGPull();
+      msg->from = parent->whoami_shard();
+      msg->set_priority(prio);
+      msg->pgid = get_parent()->primary_spg_t();
+      msg->map_epoch = get_osdmap()->get_epoch();
+      msg->pulls.swap(i->second);
+      msg->compute_cost(cct);
+      get_parent()->send_message_osd_cluster(msg, con);
+    }
+  }
+}
+
+int ReplicatedBackend::build_push_op(const ObjectRecoveryInfo &recovery_info,
+				     const ObjectRecoveryProgress &progress,
+				     ObjectRecoveryProgress *out_progress,
+				     PushOp *out_op,
+				     object_stat_sum_t *stat)
+{
+  ObjectRecoveryProgress _new_progress;
+  if (!out_progress)
+    out_progress = &_new_progress;
+  ObjectRecoveryProgress &new_progress = *out_progress;
+  new_progress = progress;
+
+  dout(7) << "send_push_op " << recovery_info.soid
+	  << " v " << recovery_info.version
+	  << " size " << recovery_info.size
+	  << " recovery_info: " << recovery_info
+          << dendl;
+
+  if (progress.first) {
+    store->omap_get_header(coll, recovery_info.soid, &out_op->omap_header);
+    store->getattrs(coll, recovery_info.soid, out_op->attrset);
+
+    // Debug
+    bufferlist bv = out_op->attrset[OI_ATTR];
+    object_info_t oi(bv);
+
+    if (oi.version != recovery_info.version) {
+      get_parent()->clog_error() << get_info().pgid << " push "
+				 << recovery_info.soid << " v "
+				 << recovery_info.version
+				 << " failed because local copy is "
+				 << oi.version << "\n";
+      return -EINVAL;
+    }
+
+    new_progress.first = false;
+  }
+
+  uint64_t available = cct->_conf->osd_recovery_max_chunk;
+  if (!progress.omap_complete) {
+    ObjectMap::ObjectMapIterator iter =
+      store->get_omap_iterator(coll,
+			       recovery_info.soid);
+    for (iter->lower_bound(progress.omap_recovered_to);
+	 iter->valid();
+	 iter->next()) {
+      if (!out_op->omap_entries.empty() &&
+	  available <= (iter->key().size() + iter->value().length()))
+	break;
+      out_op->omap_entries.insert(make_pair(iter->key(), iter->value()));
+
+      if ((iter->key().size() + iter->value().length()) <= available)
+	available -= (iter->key().size() + iter->value().length());
+      else
+	available = 0;
+    }
+    if (!iter->valid())
+      new_progress.omap_complete = true;
+    else
+      new_progress.omap_recovered_to = iter->key();
+  }
+
+  if (available > 0) {
+    out_op->data_included.span_of(recovery_info.copy_subset,
+				 progress.data_recovered_to,
+				 available);
+  } else {
+    out_op->data_included.clear();
+  }
+
+  for (interval_set::iterator p = out_op->data_included.begin();
+       p != out_op->data_included.end();
+       ++p) {
+    bufferlist bit;
+    store->read(coll, recovery_info.soid,
+		     p.get_start(), p.get_len(), bit);
+    if (p.get_len() != bit.length()) {
+      dout(10) << " extent " << p.get_start() << "~" << p.get_len()
+	       << " is actually " << p.get_start() << "~" << bit.length()
+	       << dendl;
+      p.set_len(bit.length());
+      new_progress.data_complete = true;
+    }
+    out_op->data.claim_append(bit);
+  }
+
+  if (!out_op->data_included.empty())
+    new_progress.data_recovered_to = out_op->data_included.range_end();
+
+  if (new_progress.is_complete(recovery_info)) {
+    new_progress.data_complete = true;
+    if (stat)
+      stat->num_objects_recovered++;
+  }
+
+  if (stat) {
+    stat->num_keys_recovered += out_op->omap_entries.size();
+    stat->num_bytes_recovered += out_op->data.length();
+  }
+
+  get_parent()->get_logger()->inc(l_osd_push);
+  get_parent()->get_logger()->inc(l_osd_push_outb, out_op->data.length());
+  
+  // send
+  out_op->version = recovery_info.version;
+  out_op->soid = recovery_info.soid;
+  out_op->recovery_info = recovery_info;
+  out_op->after_progress = new_progress;
+  out_op->before_progress = progress;
+  return 0;
+}
+
+int ReplicatedBackend::send_push_op_legacy(int prio, pg_shard_t peer, PushOp &pop)
+{
+  ceph_tid_t tid = get_parent()->get_tid();
+  osd_reqid_t rid(get_parent()->get_cluster_msgr_name(), 0, tid);
+  MOSDSubOp *subop = new MOSDSubOp(
+    rid, parent->whoami_shard(),
+    spg_t(get_info().pgid.pgid, peer.shard), pop.soid,
+    false, 0, get_osdmap()->get_epoch(),
+    tid, pop.recovery_info.version);
+  subop->ops = vector(1);
+  subop->ops[0].op.op = CEPH_OSD_OP_PUSH;
+
+  subop->set_priority(prio);
+  subop->version = pop.version;
+  subop->ops[0].indata.claim(pop.data);
+  subop->data_included.swap(pop.data_included);
+  subop->omap_header.claim(pop.omap_header);
+  subop->omap_entries.swap(pop.omap_entries);
+  subop->attrset.swap(pop.attrset);
+  subop->recovery_info = pop.recovery_info;
+  subop->current_progress = pop.before_progress;
+  subop->recovery_progress = pop.after_progress;
+
+  get_parent()->send_message_osd_cluster(peer.osd, subop, get_osdmap()->get_epoch());
+  return 0;
+}
+
+void ReplicatedBackend::prep_push_op_blank(const hobject_t& soid, PushOp *op)
+{
+  op->recovery_info.version = eversion_t();
+  op->version = eversion_t();
+  op->soid = soid;
+}
+
+void ReplicatedBackend::sub_op_push_reply(OpRequestRef op)
+{
+  MOSDSubOpReply *reply = static_cast(op->get_req());
+  const hobject_t& soid = reply->get_poid();
+  assert(reply->get_header().type == MSG_OSD_SUBOPREPLY);
+  dout(10) << "sub_op_push_reply from " << reply->get_source() << " " << *reply << dendl;
+  pg_shard_t peer = reply->from;
+
+  op->mark_started();
+  
+  PushReplyOp rop;
+  rop.soid = soid;
+  PushOp pop;
+  bool more = handle_push_reply(peer, rop, &pop);
+  if (more)
+    send_push_op_legacy(op->get_req()->get_priority(), peer, pop);
+}
+
+bool ReplicatedBackend::handle_push_reply(pg_shard_t peer, PushReplyOp &op, PushOp *reply)
+{
+  const hobject_t &soid = op.soid;
+  if (pushing.count(soid) == 0) {
+    dout(10) << "huh, i wasn't pushing " << soid << " to osd." << peer
+	     << ", or anybody else"
+	     << dendl;
+    return false;
+  } else if (pushing[soid].count(peer) == 0) {
+    dout(10) << "huh, i wasn't pushing " << soid << " to osd." << peer
+	     << dendl;
+    return false;
+  } else {
+    PushInfo *pi = &pushing[soid][peer];
+
+    if (!pi->recovery_progress.data_complete) {
+      dout(10) << " pushing more from, "
+	       << pi->recovery_progress.data_recovered_to
+	       << " of " << pi->recovery_info.copy_subset << dendl;
+      ObjectRecoveryProgress new_progress;
+      int r = build_push_op(
+	pi->recovery_info,
+	pi->recovery_progress, &new_progress, reply,
+	&(pi->stat));
+      assert(r == 0);
+      pi->recovery_progress = new_progress;
+      return true;
+    } else {
+      // done!
+      get_parent()->on_peer_recover(
+	peer, soid, pi->recovery_info,
+	pi->stat);
+      
+      pushing[soid].erase(peer);
+      pi = NULL;
+      
+      
+      if (pushing[soid].empty()) {
+	get_parent()->on_global_recover(soid);
+	pushing.erase(soid);
+      } else {
+	dout(10) << "pushed " << soid << ", still waiting for push ack from " 
+		 << pushing[soid].size() << " others" << dendl;
+      }
+      return false;
+    }
+  }
+}
+
+void ReplicatedPG::finish_degraded_object(const hobject_t& oid)
+{
+  dout(10) << "finish_degraded_object " << oid << dendl;
+  ObjectContextRef obc(object_contexts.lookup(oid));
+  if (obc) {
+    for (set::iterator j = obc->blocking.begin();
+	 j != obc->blocking.end();
+	 obc->blocking.erase(j++)) {
+      dout(10) << " no longer blocking writes for " << (*j)->obs.oi.soid << dendl;
+      (*j)->blocked_by = ObjectContextRef();
+    }
+  }
+  if (callbacks_for_degraded_object.count(oid)) {
+    list contexts;
+    contexts.swap(callbacks_for_degraded_object[oid]);
+    callbacks_for_degraded_object.erase(oid);
+    for (list::iterator i = contexts.begin();
+	 i != contexts.end();
+	 ++i) {
+      (*i)->complete(0);
+    }
+  }
+}
+
+/** op_pull
+ * process request to pull an entire object.
+ * NOTE: called from opqueue.
+ */
+void ReplicatedBackend::sub_op_pull(OpRequestRef op)
+{
+  MOSDSubOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+
+  op->mark_started();
+
+  const hobject_t soid = m->poid;
+
+  dout(7) << "pull" << soid << " v " << m->version
+          << " from " << m->get_source()
+          << dendl;
+
+  assert(!is_primary());  // we should be a replica or stray.
+
+  PullOp pop;
+  pop.soid = soid;
+  pop.recovery_info = m->recovery_info;
+  pop.recovery_progress = m->recovery_progress;
+
+  PushOp reply;
+  handle_pull(m->from, pop, &reply);
+  send_push_op_legacy(
+    m->get_priority(),
+    m->from,
+    reply);
+
+  log_subop_stats(get_parent()->get_logger(), op, 0, l_osd_sop_pull_lat);
+}
+
+void ReplicatedBackend::handle_pull(pg_shard_t peer, PullOp &op, PushOp *reply)
+{
+  const hobject_t &soid = op.soid;
+  struct stat st;
+  int r = store->stat(coll, soid, &st);
+  if (r != 0) {
+    get_parent()->clog_error() << get_info().pgid << " "
+			       << peer << " tried to pull " << soid
+			       << " but got " << cpp_strerror(-r) << "\n";
+    prep_push_op_blank(soid, reply);
+  } else {
+    ObjectRecoveryInfo &recovery_info = op.recovery_info;
+    ObjectRecoveryProgress &progress = op.recovery_progress;
+    if (progress.first && recovery_info.size == ((uint64_t)-1)) {
+      // Adjust size and copy_subset
+      recovery_info.size = st.st_size;
+      recovery_info.copy_subset.clear();
+      if (st.st_size)
+	recovery_info.copy_subset.insert(0, st.st_size);
+      assert(recovery_info.clone_subset.empty());
+    }
+
+    r = build_push_op(recovery_info, progress, 0, reply);
+    if (r < 0)
+      prep_push_op_blank(soid, reply);
+  }
+}
+
+
+void ReplicatedPG::_committed_pushed_object(
+  epoch_t epoch, eversion_t last_complete)
+{
+  lock();
+  if (!pg_has_reset_since(epoch)) {
+    dout(10) << "_committed_pushed_object last_complete " << last_complete << " now ondisk" << dendl;
+    last_complete_ondisk = last_complete;
+
+    if (last_complete_ondisk == info.last_update) {
+      if (!is_primary()) {
+        // Either we are a replica or backfill target.
+	// we are fully up to date.  tell the primary!
+	osd->send_message_osd_cluster(
+	  get_primary().osd,
+	  new MOSDPGTrim(
+	    get_osdmap()->get_epoch(),
+	    spg_t(info.pgid.pgid, primary.shard),
+	    last_complete_ondisk),
+	  get_osdmap()->get_epoch());
+      } else {
+	// we are the primary.  tell replicas to trim?
+	if (calc_min_last_complete_ondisk())
+	  trim_peers();
+      }
+    }
+
+  } else {
+    dout(10) << "_committed_pushed_object pg has changed, not touching last_complete_ondisk" << dendl;
+  }
+
+  unlock();
+}
+
+void ReplicatedPG::_applied_recovered_object(ObjectContextRef obc)
+{
+  lock();
+  dout(10) << "_applied_recovered_object " << *obc << dendl;
+
+  assert(active_pushes >= 1);
+  --active_pushes;
+
+  // requeue an active chunky scrub waiting on recovery ops
+  if (!deleting && active_pushes == 0
+      && scrubber.is_chunky_scrub_active()) {
+    osd->scrub_wq.queue(this);
+  }
+
+  unlock();
+}
+
+void ReplicatedPG::_applied_recovered_object_replica()
+{
+  lock();
+  dout(10) << "_applied_recovered_object_replica" << dendl;
+
+  assert(active_pushes >= 1);
+  --active_pushes;
+
+  // requeue an active chunky scrub waiting on recovery ops
+  if (!deleting && active_pushes == 0 &&
+      scrubber.active_rep_scrub && scrubber.active_rep_scrub->chunky) {
+    osd->rep_scrub_wq.queue(scrubber.active_rep_scrub);
+    scrubber.active_rep_scrub = 0;
+  }
+
+  unlock();
+}
+
+void ReplicatedPG::recover_got(hobject_t oid, eversion_t v)
+{
+  dout(10) << "got missing " << oid << " v " << v << dendl;
+  pg_log.recover_got(oid, v, info);
+  if (pg_log.get_log().complete_to != pg_log.get_log().log.end()) {
+    dout(10) << "last_complete now " << info.last_complete
+	     << " log.complete_to " << pg_log.get_log().complete_to->version
+	     << dendl;
+  } else {
+    dout(10) << "last_complete now " << info.last_complete
+	     << " log.complete_to at end" << dendl;
+    //below is not true in the repair case.
+    //assert(missing.num_missing() == 0);  // otherwise, complete_to was wrong.
+    assert(info.last_complete == info.last_update);
+  }
+}
+
+
+/**
+ * trim received data to remove what we don't want
+ *
+ * @param copy_subset intervals we want
+ * @param data_included intervals we got
+ * @param data_recieved data we got
+ * @param intervals_usable intervals we want to keep
+ * @param data_usable matching data we want to keep
+ */
+void ReplicatedBackend::trim_pushed_data(
+  const interval_set ©_subset,
+  const interval_set &intervals_received,
+  bufferlist data_received,
+  interval_set *intervals_usable,
+  bufferlist *data_usable)
+{
+  if (intervals_received.subset_of(copy_subset)) {
+    *intervals_usable = intervals_received;
+    *data_usable = data_received;
+    return;
+  }
+
+  intervals_usable->intersection_of(copy_subset,
+				    intervals_received);
+
+  uint64_t off = 0;
+  for (interval_set::const_iterator p = intervals_received.begin();
+       p != intervals_received.end();
+       ++p) {
+    interval_set x;
+    x.insert(p.get_start(), p.get_len());
+    x.intersection_of(copy_subset);
+    for (interval_set::const_iterator q = x.begin();
+	 q != x.end();
+	 ++q) {
+      bufferlist sub;
+      uint64_t data_off = off + (q.get_start() - p.get_start());
+      sub.substr_of(data_received, data_off, q.get_len());
+      data_usable->claim_append(sub);
+    }
+    off += p.get_len();
+  }
+}
+
+/** op_push
+ * NOTE: called from opqueue.
+ */
+void ReplicatedBackend::sub_op_push(OpRequestRef op)
+{
+  op->mark_started();
+  MOSDSubOp *m = static_cast(op->get_req());
+
+  PushOp pop;
+  pop.soid = m->recovery_info.soid;
+  pop.version = m->version;
+  m->claim_data(pop.data);
+  pop.data_included.swap(m->data_included);
+  pop.omap_header.swap(m->omap_header);
+  pop.omap_entries.swap(m->omap_entries);
+  pop.attrset.swap(m->attrset);
+  pop.recovery_info = m->recovery_info;
+  pop.before_progress = m->current_progress;
+  pop.after_progress = m->recovery_progress;
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+
+  if (is_primary()) {
+    PullOp resp;
+    RPGHandle *h = _open_recovery_op();
+    list to_continue;
+    bool more = handle_pull_response(
+      m->from, pop, &resp,
+      &to_continue, t);
+    if (more) {
+      send_pull_legacy(
+	m->get_priority(),
+	m->from,
+	resp.recovery_info,
+	resp.recovery_progress);
+    } else {
+      C_ReplicatedBackend_OnPullComplete *c =
+	new C_ReplicatedBackend_OnPullComplete(
+	  this,
+	  op->get_req()->get_priority());
+      c->to_continue.swap(to_continue);
+      t->register_on_complete(
+	new PG_QueueAsync(
+	  get_parent(),
+	  get_parent()->bless_gencontext(c)));
+    }
+    run_recovery_op(h, op->get_req()->get_priority());
+  } else {
+    PushReplyOp resp;
+    MOSDSubOpReply *reply = new MOSDSubOpReply(
+      m, parent->whoami_shard(), 0,
+      get_osdmap()->get_epoch(), CEPH_OSD_FLAG_ACK);
+    reply->set_priority(m->get_priority());
+    assert(entity_name_t::TYPE_OSD == m->get_connection()->peer_type);
+    handle_push(m->from, pop, &resp, t);
+    t->register_on_complete(new PG_SendMessageOnConn(
+			      get_parent(), reply, m->get_connection()));
+  }
+  t->register_on_applied(
+    new ObjectStore::C_DeleteTransaction(t));
+  get_parent()->queue_transaction(t);
+  return;
+}
+
+void ReplicatedPG::failed_push(pg_shard_t from, const hobject_t &soid)
+{
+  assert(recovering.count(soid));
+  recovering.erase(soid);
+  missing_loc.remove_location(soid, from);
+  dout(0) << "_failed_push " << soid << " from shard " << from
+	  << ", reps on " << missing_loc.get_locations(soid)
+	  << " unfound? " << missing_loc.is_unfound(soid) << dendl;
+  finish_recovery_op(soid);  // close out this attempt,
+}
+
+void ReplicatedBackend::_failed_push(pg_shard_t from, const hobject_t &soid)
+{
+  get_parent()->failed_push(from, soid);
+  pull_from_peer[from].erase(soid);
+  if (pull_from_peer[from].empty())
+    pull_from_peer.erase(from);
+  pulling.erase(soid);
+}
+
+void ReplicatedPG::sub_op_remove(OpRequestRef op)
+{
+  MOSDSubOp *m = static_cast(op->get_req());
+  assert(m->get_header().type == MSG_OSD_SUBOP);
+  dout(7) << "sub_op_remove " << m->poid << dendl;
+
+  op->mark_started();
+
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  remove_snap_mapped_object(*t, m->poid);
+  int r = osd->store->queue_transaction_and_cleanup(osr.get(), t);
+  assert(r == 0);
+}
+
+
+eversion_t ReplicatedPG::pick_newest_available(const hobject_t& oid)
+{
+  eversion_t v;
+
+  assert(pg_log.get_missing().is_missing(oid));
+  v = pg_log.get_missing().missing.find(oid)->second.have;
+  dout(10) << "pick_newest_available " << oid << " " << v << " on osd." << osd->whoami << " (local)" << dendl;
+
+  assert(actingbackfill.size() > 0);
+  for (set::iterator i = actingbackfill.begin();
+       i != actingbackfill.end();
+       ++i) {
+    if (*i == get_primary()) continue;
+    pg_shard_t peer = *i;
+    if (!peer_missing[peer].is_missing(oid)) {
+      assert(is_backfill_targets(peer));
+      continue;
+    }
+    eversion_t h = peer_missing[peer].missing[oid].have;
+    dout(10) << "pick_newest_available " << oid << " " << h << " on osd." << peer << dendl;
+    if (h > v)
+      v = h;
+  }
+
+  dout(10) << "pick_newest_available " << oid << " " << v << " (newest)" << dendl;
+  return v;
+}
+
+
+/* Mark an object as lost
+ */
+ObjectContextRef ReplicatedPG::mark_object_lost(ObjectStore::Transaction *t,
+							    const hobject_t &oid, eversion_t version,
+							    utime_t mtime, int what)
+{
+  // Wake anyone waiting for this object. Now that it's been marked as lost,
+  // we will just return an error code.
+  map >::iterator wmo =
+    waiting_for_unreadable_object.find(oid);
+  if (wmo != waiting_for_unreadable_object.end()) {
+    requeue_ops(wmo->second);
+  }
+
+  // Add log entry
+  ++info.last_update.version;
+  pg_log_entry_t e(what, oid, info.last_update, version, 0, osd_reqid_t(), mtime);
+  pg_log.add(e);
+  
+  ObjectContextRef obc = get_object_context(oid, true);
+
+  obc->ondisk_write_lock();
+
+  obc->obs.oi.set_flag(object_info_t::FLAG_LOST);
+  obc->obs.oi.version = info.last_update;
+  obc->obs.oi.prior_version = version;
+
+  bufferlist b2;
+  obc->obs.oi.encode(b2);
+  t->setattr(coll, oid, OI_ATTR, b2);
+
+  return obc;
+}
+
+struct C_PG_MarkUnfoundLost : public Context {
+  ReplicatedPGRef pg;
+  list obcs;
+  C_PG_MarkUnfoundLost(ReplicatedPG *p) : pg(p) {}
+  void finish(int r) {
+    pg->_finish_mark_all_unfound_lost(obcs);
+  }
+};
+
+/* Mark all unfound objects as lost.
+ */
+void ReplicatedPG::mark_all_unfound_lost(int what)
+{
+  dout(3) << __func__ << " " << pg_log_entry_t::get_op_name(what) << dendl;
+
+  dout(30) << __func__ << ": log before:\n";
+  pg_log.get_log().print(*_dout);
+  *_dout << dendl;
+
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  C_PG_MarkUnfoundLost *c = new C_PG_MarkUnfoundLost(this);
+
+  utime_t mtime = ceph_clock_now(cct);
+  info.last_update.epoch = get_osdmap()->get_epoch();
+  const pg_missing_t &missing = pg_log.get_missing();
+  map::const_iterator m =
+    missing_loc.get_all_missing().begin();
+  map::const_iterator mend =
+    missing_loc.get_all_missing().end();
+  while (m != mend) {
+    const hobject_t &oid(m->first);
+    if (!missing_loc.is_unfound(oid)) {
+      // We only care about unfound objects
+      ++m;
+      continue;
+    }
+
+    ObjectContextRef obc;
+    eversion_t prev;
+
+    switch (what) {
+    case pg_log_entry_t::LOST_MARK:
+      obc = mark_object_lost(t, oid, m->second.need, mtime, pg_log_entry_t::LOST_MARK);
+      pg_log.missing_got(m++);
+      assert(0 == "actually, not implemented yet!");
+      // we need to be careful about how this is handled on the replica!
+      break;
+
+    case pg_log_entry_t::LOST_REVERT:
+      prev = pick_newest_available(oid);
+      if (prev > eversion_t()) {
+	// log it
+	++info.last_update.version;
+	pg_log_entry_t e(
+	  pg_log_entry_t::LOST_REVERT, oid, info.last_update,
+	  m->second.need, 0, osd_reqid_t(), mtime);
+	e.reverting_to = prev;
+	pg_log.add(e);
+	dout(10) << e << dendl;
+
+	// we are now missing the new version; recovery code will sort it out.
+	++m;
+	pg_log.revise_need(oid, info.last_update);
+	missing_loc.revise_need(oid, info.last_update);
+	break;
+      }
+      /** fall-thru **/
+
+    case pg_log_entry_t::LOST_DELETE:
+      {
+	// log it
+      	++info.last_update.version;
+	pg_log_entry_t e(pg_log_entry_t::LOST_DELETE, oid, info.last_update, m->second.need,
+		     0, osd_reqid_t(), mtime);
+	pg_log.add(e);
+	dout(10) << e << dendl;
+
+	t->remove(
+	  coll,
+	  ghobject_t(oid, ghobject_t::NO_GEN, pg_whoami.shard));
+	pg_log.missing_add_event(e);
+	++m;
+	missing_loc.recovered(oid);
+      }
+      break;
+
+    default:
+      assert(0);
+    }
+
+    if (obc)
+      c->obcs.push_back(obc);
+  }
+
+  dout(30) << __func__ << ": log after:\n";
+  pg_log.get_log().print(*_dout);
+  *_dout << dendl;
+
+  info.stats.stats_invalid = true;
+
+  if (missing.num_missing() == 0) {
+    // advance last_complete since nothing else is missing!
+    info.last_complete = info.last_update;
+  }
+
+  dirty_info = true;
+  write_if_dirty(*t);
+
+  t->register_on_complete(new ObjectStore::C_DeleteTransaction(t));
+
+  osd->store->queue_transaction(osr.get(), t, c, NULL, new C_OSD_OndiskWriteUnlockList(&c->obcs));
+	      
+  // Send out the PG log to all replicas
+  // So that they know what is lost
+  share_pg_log();
+
+  // queue ourselves so that we push the (now-lost) object_infos to replicas.
+  osd->queue_for_recovery(this);
+}
+
+void ReplicatedPG::_finish_mark_all_unfound_lost(list& obcs)
+{
+  lock();
+  dout(10) << "_finish_mark_all_unfound_lost " << dendl;
+
+  if (!deleting)
+    requeue_ops(waiting_for_all_missing);
+  waiting_for_all_missing.clear();
+
+  obcs.clear();
+  unlock();
+}
+
+void ReplicatedPG::_split_into(pg_t child_pgid, PG *child, unsigned split_bits)
+{
+  assert(repop_queue.empty());
+}
+
+/*
+ * pg status change notification
+ */
+
+void ReplicatedPG::apply_and_flush_repops(bool requeue)
+{
+  list rq;
+
+  // apply all repops
+  while (!repop_queue.empty()) {
+    RepGather *repop = repop_queue.front();
+    repop_queue.pop_front();
+    dout(10) << " canceling repop tid " << repop->rep_tid << dendl;
+    repop->rep_aborted = true;
+    if (repop->on_applied) {
+      delete repop->on_applied;
+      repop->on_applied = NULL;
+    }
+
+    if (requeue) {
+      if (repop->ctx->op) {
+	dout(10) << " requeuing " << *repop->ctx->op->get_req() << dendl;
+	rq.push_back(repop->ctx->op);
+	repop->ctx->op = OpRequestRef();
+      }
+
+      // also requeue any dups, interleaved into position
+      map >::iterator p = waiting_for_ondisk.find(repop->v);
+      if (p != waiting_for_ondisk.end()) {
+	dout(10) << " also requeuing ondisk waiters " << p->second << dendl;
+	rq.splice(rq.end(), p->second);
+	waiting_for_ondisk.erase(p);
+      }
+    }
+
+    remove_repop(repop);
+  }
+
+  if (requeue) {
+    requeue_ops(rq);
+    if (!waiting_for_ondisk.empty()) {
+      for (map >::iterator i =
+	     waiting_for_ondisk.begin();
+	   i != waiting_for_ondisk.end();
+	   ++i) {
+	for (list::iterator j = i->second.begin();
+	     j != i->second.end();
+	     ++j) {
+	  derr << __func__ << ": op " << *((*j)->get_req()) << " waiting on "
+	       << i->first << dendl;
+	}
+      }
+      assert(waiting_for_ondisk.empty());
+    }
+  }
+
+  waiting_for_ondisk.clear();
+  waiting_for_ack.clear();
+}
+
+void ReplicatedPG::on_flushed()
+{
+  assert(flushes_in_progress > 0);
+  flushes_in_progress--;
+  if (flushes_in_progress == 0) {
+    requeue_ops(waiting_for_active);
+  }
+  if (!is_active() || !is_primary()) {
+    pair i;
+    while (object_contexts.get_next(i.first, &i)) {
+      derr << "on_flushed: object " << i.first << " obc still alive" << dendl;
+    }
+    assert(object_contexts.empty());
+  }
+  pgbackend->on_flushed();
+}
+
+void ReplicatedPG::on_removal(ObjectStore::Transaction *t)
+{
+  dout(10) << "on_removal" << dendl;
+
+  // adjust info to backfill
+  info.last_backfill = hobject_t();
+  dirty_info = true;
+  write_if_dirty(*t);
+
+  on_shutdown();
+}
+
+void ReplicatedPG::on_shutdown()
+{
+  dout(10) << "on_shutdown" << dendl;
+
+  // remove from queues
+  osd->recovery_wq.dequeue(this);
+  osd->scrub_wq.dequeue(this);
+  osd->scrub_finalize_wq.dequeue(this);
+  osd->snap_trim_wq.dequeue(this);
+  osd->pg_stat_queue_dequeue(this);
+  osd->dequeue_pg(this, 0);
+  osd->peering_wq.dequeue(this);
+
+  // handles queue races
+  deleting = true;
+
+  unreg_next_scrub();
+  cancel_copy_ops(false);
+  cancel_flush_ops(false);
+  apply_and_flush_repops(false);
+
+  pgbackend->on_change();
+
+  context_registry_on_change();
+
+  osd->remote_reserver.cancel_reservation(info.pgid);
+  osd->local_reserver.cancel_reservation(info.pgid);
+
+  clear_primary_state();
+  cancel_recovery();
+}
+
+void ReplicatedPG::on_activate()
+{
+  // all clean?
+  if (needs_recovery()) {
+    dout(10) << "activate not all replicas are up-to-date, queueing recovery" << dendl;
+    queue_peering_event(
+      CephPeeringEvtRef(
+	new CephPeeringEvt(
+	  get_osdmap()->get_epoch(),
+	  get_osdmap()->get_epoch(),
+	  DoRecovery())));
+  } else if (needs_backfill()) {
+    dout(10) << "activate queueing backfill" << dendl;
+    queue_peering_event(
+      CephPeeringEvtRef(
+	new CephPeeringEvt(
+	  get_osdmap()->get_epoch(),
+	  get_osdmap()->get_epoch(),
+	  RequestBackfill())));
+  } else {
+    dout(10) << "activate all replicas clean, no recovery" << dendl;
+    queue_peering_event(
+      CephPeeringEvtRef(
+	new CephPeeringEvt(
+	  get_osdmap()->get_epoch(),
+	  get_osdmap()->get_epoch(),
+	  AllReplicasRecovered())));
+  }
+
+  publish_stats_to_osd();
+
+  if (!backfill_targets.empty()) {
+    last_backfill_started = earliest_backfill();
+    new_backfill = true;
+    assert(!last_backfill_started.is_max());
+    dout(5) << "on activate: bft=" << backfill_targets
+	   << " from " << last_backfill_started << dendl;
+    for (set::iterator i = backfill_targets.begin();
+	 i != backfill_targets.end();
+	 ++i) {
+      dout(5) << "target shard " << *i
+	     << " from " << peer_info[*i].last_backfill
+	     << dendl;
+    }
+  }
+
+  hit_set_setup();
+  agent_setup();
+}
+
+void ReplicatedPG::on_change(ObjectStore::Transaction *t)
+{
+  dout(10) << "on_change" << dendl;
+
+  if (hit_set && hit_set->insert_count() == 0) {
+    dout(20) << " discarding empty hit_set" << dendl;
+    hit_set_clear();
+  }
+
+  // requeue everything in the reverse order they should be
+  // reexamined.
+
+  clear_scrub_reserved();
+  scrub_clear_state();
+
+  context_registry_on_change();
+
+  for (list >::iterator i =
+         in_progress_async_reads.begin();
+       i != in_progress_async_reads.end();
+       in_progress_async_reads.erase(i++)) {
+    close_op_ctx(i->second, -ECANCELED);
+    requeue_op(i->first);
+  }
+
+  cancel_copy_ops(is_primary());
+  cancel_flush_ops(is_primary());
+
+  // requeue object waiters
+  if (is_primary()) {
+    requeue_object_waiters(waiting_for_unreadable_object);
+  } else {
+    waiting_for_unreadable_object.clear();
+  }
+  for (map >::iterator p = waiting_for_degraded_object.begin();
+       p != waiting_for_degraded_object.end();
+       waiting_for_degraded_object.erase(p++)) {
+    if (is_primary())
+      requeue_ops(p->second);
+    else
+      p->second.clear();
+    finish_degraded_object(p->first);
+  }
+  for (map >::iterator p = waiting_for_blocked_object.begin();
+       p != waiting_for_blocked_object.end();
+       waiting_for_blocked_object.erase(p++)) {
+    if (is_primary())
+      requeue_ops(p->second);
+    else
+      p->second.clear();
+  }
+
+  if (is_primary()) {
+    requeue_ops(waiting_for_cache_not_full);
+    requeue_ops(waiting_for_all_missing);
+  } else {
+    waiting_for_cache_not_full.clear();
+    waiting_for_all_missing.clear();
+  }
+
+  // this will requeue ops we were working on but didn't finish, and
+  // any dups
+  apply_and_flush_repops(is_primary());
+
+  pgbackend->on_change_cleanup(t);
+  pgbackend->on_change();
+
+  // clear snap_trimmer state
+  snap_trimmer_machine.process_event(Reset());
+
+  debug_op_order.clear();
+  unstable_stats.clear();
+}
+
+void ReplicatedPG::on_role_change()
+{
+  dout(10) << "on_role_change" << dendl;
+  if (get_role() != 0 && hit_set) {
+    dout(10) << " clearing hit set" << dendl;
+    hit_set_clear();
+  }
+}
+
+void ReplicatedPG::on_pool_change()
+{
+  dout(10) << __func__ << dendl;
+  // requeue cache full waiters just in case the cache_mode is
+  // changing away from writeback mode.  note that if we are not
+  // active the normal requeuing machinery is sufficient (and properly
+  // ordered).
+  if (is_active() &&
+      pool.info.cache_mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+      !waiting_for_cache_not_full.empty()) {
+    dout(10) << __func__ << " requeuing full waiters (not in writeback) "
+	     << dendl;
+    requeue_ops(waiting_for_cache_not_full);
+  }
+  hit_set_setup();
+  agent_setup();
+}
+
+// clear state.  called on recovery completion AND cancellation.
+void ReplicatedPG::_clear_recovery_state()
+{
+  missing_loc.clear();
+#ifdef DEBUG_RECOVERY_OIDS
+  recovering_oids.clear();
+#endif
+  last_backfill_started = hobject_t();
+  list blocked_ops;
+  set::iterator i = backfills_in_flight.begin();
+  while (i != backfills_in_flight.end()) {
+    assert(recovering.count(*i));
+    recovering[*i]->drop_backfill_read(&blocked_ops);
+    requeue_ops(blocked_ops);
+    backfills_in_flight.erase(i++);
+  }
+  assert(backfills_in_flight.empty());
+  pending_backfill_updates.clear();
+  recovering.clear();
+  pgbackend->clear_state();
+}
+
+void ReplicatedPG::cancel_pull(const hobject_t &soid)
+{
+  assert(recovering.count(soid));
+  recovering.erase(soid);
+  finish_recovery_op(soid);
+  if (is_missing_object(soid))
+    pg_log.set_last_requested(0); // get recover_primary to start over
+}
+
+void ReplicatedPG::check_recovery_sources(const OSDMapRef osdmap)
+{
+  /*
+   * check that any peers we are planning to (or currently) pulling
+   * objects from are dealt with.
+   */
+  missing_loc.check_recovery_sources(osdmap);
+  pgbackend->check_recovery_sources(osdmap);
+
+  for (set::iterator i = peer_log_requested.begin();
+       i != peer_log_requested.end();
+       ) {
+    if (!osdmap->is_up(i->osd)) {
+      dout(10) << "peer_log_requested removing " << *i << dendl;
+      peer_log_requested.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+
+  for (set::iterator i = peer_missing_requested.begin();
+       i != peer_missing_requested.end();
+       ) {
+    if (!osdmap->is_up(i->osd)) {
+      dout(10) << "peer_missing_requested removing " << *i << dendl;
+      peer_missing_requested.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+}
+
+void PG::MissingLoc::check_recovery_sources(const OSDMapRef osdmap)
+{
+  set now_down;
+  for (set::iterator p = missing_loc_sources.begin();
+       p != missing_loc_sources.end();
+       ) {
+    if (osdmap->is_up(p->osd)) {
+      ++p;
+      continue;
+    }
+    dout(10) << "check_recovery_sources source osd." << *p << " now down" << dendl;
+    now_down.insert(*p);
+    missing_loc_sources.erase(p++);
+  }
+
+  if (now_down.empty()) {
+    dout(10) << "check_recovery_sources no source osds (" << missing_loc_sources << ") went down" << dendl;
+  } else {
+    dout(10) << "check_recovery_sources sources osds " << now_down << " now down, remaining sources are "
+	     << missing_loc_sources << dendl;
+    
+    // filter missing_loc
+    map >::iterator p = missing_loc.begin();
+    while (p != missing_loc.end()) {
+      set::iterator q = p->second.begin();
+      while (q != p->second.end())
+	if (now_down.count(*q)) {
+	  p->second.erase(q++);
+	} else {
+	  ++q;
+	}
+      if (p->second.empty())
+	missing_loc.erase(p++);
+      else
+	++p;
+    }
+  }
+}
+  
+
+bool ReplicatedPG::start_recovery_ops(
+  int max, RecoveryCtx *prctx,
+  ThreadPool::TPHandle &handle,
+  int *ops_started)
+{
+  int& started = *ops_started;
+  started = 0;
+  bool work_in_progress = false;
+  assert(is_primary());
+
+  if (!state_test(PG_STATE_RECOVERING) &&
+      !state_test(PG_STATE_BACKFILL)) {
+    /* TODO: I think this case is broken and will make do_recovery()
+     * unhappy since we're returning false */
+    dout(10) << "recovery raced and were queued twice, ignoring!" << dendl;
+    return false;
+  }
+
+  const pg_missing_t &missing = pg_log.get_missing();
+
+  int num_missing = missing.num_missing();
+  int num_unfound = get_num_unfound();
+
+  if (num_missing == 0) {
+    info.last_complete = info.last_update;
+  }
+
+  if (num_missing == num_unfound) {
+    // All of the missing objects we have are unfound.
+    // Recover the replicas.
+    started = recover_replicas(max, handle);
+  }
+  if (!started) {
+    // We still have missing objects that we should grab from replicas.
+    started += recover_primary(max, handle);
+  }
+  if (!started && num_unfound != get_num_unfound()) {
+    // second chance to recovery replicas
+    started = recover_replicas(max, handle);
+  }
+
+  if (started)
+    work_in_progress = true;
+
+  bool deferred_backfill = false;
+  if (recovering.empty() &&
+      state_test(PG_STATE_BACKFILL) &&
+      !backfill_targets.empty() && started < max &&
+      missing.num_missing() == 0 &&
+      waiting_on_backfill.empty()) {
+    if (get_osdmap()->test_flag(CEPH_OSDMAP_NOBACKFILL)) {
+      dout(10) << "deferring backfill due to NOBACKFILL" << dendl;
+      deferred_backfill = true;
+    } else if (!backfill_reserved) {
+      dout(10) << "deferring backfill due to !backfill_reserved" << dendl;
+      if (!backfill_reserving) {
+	dout(10) << "queueing RequestBackfill" << dendl;
+	backfill_reserving = true;
+	queue_peering_event(
+	  CephPeeringEvtRef(
+	    new CephPeeringEvt(
+	      get_osdmap()->get_epoch(),
+	      get_osdmap()->get_epoch(),
+	      RequestBackfill())));
+      }
+      deferred_backfill = true;
+    } else {
+      started += recover_backfill(max - started, handle, &work_in_progress);
+    }
+  }
+
+  dout(10) << " started " << started << dendl;
+  osd->logger->inc(l_osd_rop, started);
+
+  if (!recovering.empty() ||
+      work_in_progress || recovery_ops_active > 0 || deferred_backfill)
+    return work_in_progress;
+
+  assert(recovering.empty());
+  assert(recovery_ops_active == 0);
+
+  dout(10) << __func__ << " needs_recovery: "
+	   << missing_loc.get_needs_recovery()
+	   << dendl;
+  dout(10) << __func__ << " missing_loc: "
+	   << missing_loc.get_missing_locs()
+	   << dendl;
+  int unfound = get_num_unfound();
+  if (unfound) {
+    dout(10) << " still have " << unfound << " unfound" << dendl;
+    return work_in_progress;
+  }
+
+  if (missing.num_missing() > 0) {
+    // this shouldn't happen!
+    osd->clog.error() << info.pgid << " recovery ending with " << missing.num_missing()
+		      << ": " << missing.missing << "\n";
+    return work_in_progress;
+  }
+
+  if (needs_recovery()) {
+    // this shouldn't happen!
+    // We already checked num_missing() so we must have missing replicas
+    osd->clog.error() << info.pgid << " recovery ending with missing replicas\n";
+    return work_in_progress;
+  }
+
+  if (state_test(PG_STATE_RECOVERING)) {
+    state_clear(PG_STATE_RECOVERING);
+    if (needs_backfill()) {
+      dout(10) << "recovery done, queuing backfill" << dendl;
+      queue_peering_event(
+        CephPeeringEvtRef(
+          new CephPeeringEvt(
+            get_osdmap()->get_epoch(),
+            get_osdmap()->get_epoch(),
+            RequestBackfill())));
+    } else {
+      dout(10) << "recovery done, no backfill" << dendl;
+      queue_peering_event(
+        CephPeeringEvtRef(
+          new CephPeeringEvt(
+            get_osdmap()->get_epoch(),
+            get_osdmap()->get_epoch(),
+            AllReplicasRecovered())));
+    }
+  } else { // backfilling
+    state_clear(PG_STATE_BACKFILL);
+    dout(10) << "recovery done, backfill done" << dendl;
+    queue_peering_event(
+      CephPeeringEvtRef(
+        new CephPeeringEvt(
+          get_osdmap()->get_epoch(),
+          get_osdmap()->get_epoch(),
+          Backfilled())));
+  }
+
+  return false;
+}
+
+/**
+ * do one recovery op.
+ * return true if done, false if nothing left to do.
+ */
+int ReplicatedPG::recover_primary(int max, ThreadPool::TPHandle &handle)
+{
+  assert(is_primary());
+
+  const pg_missing_t &missing = pg_log.get_missing();
+
+  dout(10) << "recover_primary recovering " << recovering.size()
+	   << " in pg" << dendl;
+  dout(10) << "recover_primary " << missing << dendl;
+  dout(25) << "recover_primary " << missing.missing << dendl;
+
+  // look at log!
+  pg_log_entry_t *latest = 0;
+  int started = 0;
+  int skipped = 0;
+
+  PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
+  map::const_iterator p =
+    missing.rmissing.lower_bound(pg_log.get_log().last_requested);
+  while (p != missing.rmissing.end()) {
+    handle.reset_tp_timeout();
+    hobject_t soid;
+    version_t v = p->first;
+
+    if (pg_log.get_log().objects.count(p->second)) {
+      latest = pg_log.get_log().objects.find(p->second)->second;
+      assert(latest->is_update());
+      soid = latest->soid;
+    } else {
+      latest = 0;
+      soid = p->second;
+    }
+    const pg_missing_t::item& item = missing.missing.find(p->second)->second;
+    ++p;
+
+    hobject_t head = soid;
+    head.snap = CEPH_NOSNAP;
+
+    eversion_t need = item.need;
+
+    bool unfound = missing_loc.is_unfound(soid);
+
+    dout(10) << "recover_primary "
+             << soid << " " << item.need
+	     << (unfound ? " (unfound)":"")
+	     << (missing.is_missing(soid) ? " (missing)":"")
+	     << (missing.is_missing(head) ? " (missing head)":"")
+             << (recovering.count(soid) ? " (recovering)":"")
+	     << (recovering.count(head) ? " (recovering head)":"")
+             << dendl;
+
+    if (latest) {
+      switch (latest->op) {
+      case pg_log_entry_t::CLONE:
+	/*
+	 * Handling for this special case removed for now, until we
+	 * can correctly construct an accurate SnapSet from the old
+	 * one.
+	 */
+	break;
+
+      case pg_log_entry_t::LOST_REVERT:
+	{
+	  if (item.have == latest->reverting_to) {
+	    ObjectContextRef obc = get_object_context(soid, true);
+	    
+	    if (obc->obs.oi.version == latest->version) {
+	      // I'm already reverting
+	      dout(10) << " already reverting " << soid << dendl;
+	    } else {
+	      dout(10) << " reverting " << soid << " to " << latest->prior_version << dendl;
+	      obc->ondisk_write_lock();
+	      obc->obs.oi.version = latest->version;
+
+	      ObjectStore::Transaction *t = new ObjectStore::Transaction;
+	      t->register_on_applied(new ObjectStore::C_DeleteTransaction(t));
+	      bufferlist b2;
+	      obc->obs.oi.encode(b2);
+	      t->setattr(coll, soid, OI_ATTR, b2);
+
+	      recover_got(soid, latest->version);
+	      missing_loc.add_location(soid, pg_whoami);
+
+	      ++active_pushes;
+
+	      osd->store->queue_transaction(osr.get(), t,
+					    new C_OSD_AppliedRecoveredObject(this, obc),
+					    new C_OSD_CommittedPushedObject(
+					      this,
+					      get_osdmap()->get_epoch(),
+					      info.last_complete),
+					    new C_OSD_OndiskWriteUnlock(obc));
+	      continue;
+	    }
+	  } else {
+	    /*
+	     * Pull the old version of the object.  Update missing_loc here to have the location
+	     * of the version we want.
+	     *
+	     * This doesn't use the usual missing_loc paths, but that's okay:
+	     *  - if we have it locally, we hit the case above, and go from there.
+	     *  - if we don't, we always pass through this case during recovery and set up the location
+	     *    properly.
+	     *  - this way we don't need to mangle the missing code to be general about needing an old
+	     *    version...
+	     */
+	    eversion_t alternate_need = latest->reverting_to;
+	    dout(10) << " need to pull prior_version " << alternate_need << " for revert " << item << dendl;
+
+	    for (map::iterator p = peer_missing.begin();
+		 p != peer_missing.end();
+		 ++p)
+	      if (p->second.is_missing(soid, need) &&
+		  p->second.missing[soid].have == alternate_need) {
+		missing_loc.add_location(soid, p->first);
+	      }
+	    dout(10) << " will pull " << alternate_need << " or " << need
+		     << " from one of " << missing_loc.get_locations(soid)
+		     << dendl;
+	    unfound = false;
+	  }
+	}
+	break;
+      }
+    }
+   
+    if (!recovering.count(soid)) {
+      if (recovering.count(head)) {
+	++skipped;
+      } else if (unfound) {
+	++skipped;
+      } else {
+	int r = recover_missing(
+	  soid, need, cct->_conf->osd_recovery_op_priority, h);
+	switch (r) {
+	case PULL_YES:
+	  ++started;
+	  break;
+	case PULL_OTHER:
+	  ++started;
+	case PULL_NONE:
+	  ++skipped;
+	  break;
+	default:
+	  assert(0);
+	}
+	if (started >= max)
+	  break;
+      }
+    }
+    
+    // only advance last_requested if we haven't skipped anything
+    if (!skipped)
+      pg_log.set_last_requested(v);
+  }
+ 
+  pgbackend->run_recovery_op(h, cct->_conf->osd_recovery_op_priority);
+  return started;
+}
+
+int ReplicatedPG::prep_object_replica_pushes(
+  const hobject_t& soid, eversion_t v,
+  PGBackend::RecoveryHandle *h)
+{
+  assert(is_primary());
+  dout(10) << __func__ << ": on " << soid << dendl;
+
+  // NOTE: we know we will get a valid oloc off of disk here.
+  ObjectContextRef obc = get_object_context(soid, false);
+  if (!obc) {
+    pg_log.missing_add(soid, v, eversion_t());
+    missing_loc.remove_location(soid, pg_whoami);
+    bool uhoh = true;
+    assert(actingbackfill.size() > 0);
+    for (set::iterator i = actingbackfill.begin();
+	 i != actingbackfill.end();
+	 ++i) {
+      if (*i == get_primary()) continue;
+      pg_shard_t peer = *i;
+      if (!peer_missing[peer].is_missing(soid, v)) {
+	missing_loc.add_location(soid, peer);
+	dout(10) << info.pgid << " unexpectedly missing " << soid << " v" << v
+		 << ", there should be a copy on shard " << peer << dendl;
+	uhoh = false;
+      }
+    }
+    if (uhoh)
+      osd->clog.error() << info.pgid << " missing primary copy of " << soid << ", unfound\n";
+    else
+      osd->clog.error() << info.pgid << " missing primary copy of " << soid
+			<< ", will try copies on " << missing_loc.get_locations(soid)
+			<< "\n";
+    return 0;
+  }
+
+  start_recovery_op(soid);
+  assert(!recovering.count(soid));
+  recovering.insert(make_pair(soid, obc));
+
+  /* We need this in case there is an in progress write on the object.  In fact,
+   * the only possible write is an update to the xattr due to a lost_revert --
+   * a client write would be blocked since the object is degraded.
+   * In almost all cases, therefore, this lock should be uncontended.
+   */
+  obc->ondisk_read_lock();
+  pgbackend->recover_object(
+    soid,
+    v,
+    ObjectContextRef(),
+    obc, // has snapset context
+    h);
+  obc->ondisk_read_unlock();
+  return 1;
+}
+
+int ReplicatedBackend::start_pushes(
+  const hobject_t &soid,
+  ObjectContextRef obc,
+  RPGHandle *h)
+{
+  int pushes = 0;
+  // who needs it?  
+  assert(get_parent()->get_actingbackfill_shards().size() > 0);
+  for (set::iterator i =
+	 get_parent()->get_actingbackfill_shards().begin();
+       i != get_parent()->get_actingbackfill_shards().end();
+       ++i) {
+    if (*i == get_parent()->whoami_shard()) continue;
+    pg_shard_t peer = *i;
+    map::const_iterator j =
+      get_parent()->get_shard_missing().find(peer);
+    assert(j != get_parent()->get_shard_missing().end());
+    if (j->second.is_missing(soid)) {
+      ++pushes;
+      h->pushes[peer].push_back(PushOp());
+      prep_push_to_replica(obc, soid, peer,
+			   &(h->pushes[peer].back())
+	);
+    }
+  }
+  return pushes;
+}
+
+int ReplicatedPG::recover_replicas(int max, ThreadPool::TPHandle &handle)
+{
+  dout(10) << __func__ << "(" << max << ")" << dendl;
+  int started = 0;
+
+  PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
+
+  // this is FAR from an optimal recovery order.  pretty lame, really.
+  assert(actingbackfill.size() > 0);
+  for (set::iterator i = actingbackfill.begin();
+       i != actingbackfill.end();
+       ++i) {
+    if (*i == get_primary()) continue;
+    pg_shard_t peer = *i;
+    map::const_iterator pm = peer_missing.find(peer);
+    assert(pm != peer_missing.end());
+    map::const_iterator pi = peer_info.find(peer);
+    assert(pi != peer_info.end());
+    size_t m_sz = pm->second.num_missing();
+
+    dout(10) << " peer osd." << peer << " missing " << m_sz << " objects." << dendl;
+    dout(20) << " peer osd." << peer << " missing " << pm->second.missing << dendl;
+
+    // oldest first!
+    const pg_missing_t &m(pm->second);
+    for (map::const_iterator p = m.rmissing.begin();
+	   p != m.rmissing.end() && started < max;
+	   ++p) {
+      handle.reset_tp_timeout();
+      const hobject_t soid(p->second);
+
+      if (soid > pi->second.last_backfill) {
+	if (!recovering.count(soid)) {
+	  derr << __func__ << ": object added to missing set for backfill, but "
+	       << "is not in recovering, error!" << dendl;
+	  assert(0);
+	}
+	continue;
+      }
+
+      if (recovering.count(soid)) {
+	dout(10) << __func__ << ": already recovering " << soid << dendl;
+	continue;
+      }
+
+      if (missing_loc.is_unfound(soid)) {
+	dout(10) << __func__ << ": " << soid << " still unfound" << dendl;
+	continue;
+      }
+
+      if (soid.is_snap() && pg_log.get_missing().is_missing(soid.get_head())) {
+	dout(10) << __func__ << ": " << soid.get_head()
+		 << " still missing on primary" << dendl;
+	continue;
+      }
+
+      if (soid.is_snap() && pg_log.get_missing().is_missing(soid.get_snapdir())) {
+	dout(10) << __func__ << ": " << soid.get_snapdir()
+		 << " still missing on primary" << dendl;
+	continue;
+      }
+
+      if (pg_log.get_missing().is_missing(soid)) {
+	dout(10) << __func__ << ": " << soid << " still missing on primary" << dendl;
+	continue;
+      }
+
+      dout(10) << __func__ << ": recover_object_replicas(" << soid << ")" << dendl;
+      map::const_iterator r = m.missing.find(soid);
+      started += prep_object_replica_pushes(soid, r->second.need,
+					    h);
+    }
+  }
+
+  pgbackend->run_recovery_op(h, cct->_conf->osd_recovery_op_priority);
+  return started;
+}
+
+hobject_t ReplicatedPG::earliest_peer_backfill() const
+{
+  hobject_t e = hobject_t::get_max();
+  for (set::const_iterator i = backfill_targets.begin();
+       i != backfill_targets.end();
+       ++i) {
+    pg_shard_t peer = *i;
+    map::const_iterator iter =
+      peer_backfill_info.find(peer);
+    assert(iter != peer_backfill_info.end());
+    if (iter->second.begin < e)
+      e = iter->second.begin;
+  }
+  return e;
+}
+
+bool ReplicatedPG::all_peer_done() const
+{
+  // Primary hasn't got any more objects
+  assert(backfill_info.empty());
+
+  for (set::const_iterator i = backfill_targets.begin();
+       i != backfill_targets.end();
+       ++i) {
+    pg_shard_t bt = *i;
+    map::const_iterator piter =
+      peer_backfill_info.find(bt);
+    assert(piter != peer_backfill_info.end());
+    const BackfillInterval& pbi = piter->second;
+    // See if peer has more to process
+    if (!pbi.extends_to_end() || !pbi.empty())
+	return false;
+  }
+  return true;
+}
+
+/**
+ * recover_backfill
+ *
+ * Invariants:
+ *
+ * backfilled: fully pushed to replica or present in replica's missing set (both
+ * our copy and theirs).
+ *
+ * All objects on a backfill_target in
+ * [MIN,peer_backfill_info[backfill_target].begin) are either
+ * not present or backfilled (all removed objects have been removed).
+ * There may be PG objects in this interval yet to be backfilled.
+ *
+ * All objects in PG in [MIN,backfill_info.begin) have been backfilled to all
+ * backfill_targets.  There may be objects on backfill_target(s) yet to be deleted.
+ *
+ * For a backfill target, all objects < MIN(peer_backfill_info[target].begin,
+ *     backfill_info.begin) in PG are backfilled.  No deleted objects in this
+ * interval remain on the backfill target.
+ *
+ * For a backfill target, all objects <= peer_info[target].last_backfill
+ * have been backfilled to target
+ *
+ * There *MAY* be objects between last_backfill_started and
+ * MIN(peer_backfill_info[*].begin, backfill_info.begin) in the event that client
+ * io created objects since the last scan.  For this reason, we call
+ * update_range() again before continuing backfill.
+ */
+int ReplicatedPG::recover_backfill(
+  int max,
+  ThreadPool::TPHandle &handle, bool *work_started)
+{
+  dout(10) << "recover_backfill (" << max << ")"
+           << " bft=" << backfill_targets
+	   << " last_backfill_started " << last_backfill_started << dendl;
+  assert(!backfill_targets.empty());
+
+  // Initialize from prior backfill state
+  if (new_backfill) {
+    // on_activate() was called prior to getting here
+    assert(last_backfill_started == earliest_backfill());
+    new_backfill = false;
+    for (set::iterator i = backfill_targets.begin();
+	 i != backfill_targets.end();
+	 ++i) {
+      peer_backfill_info[*i].reset(peer_info[*i].last_backfill);
+    }
+    backfill_info.reset(last_backfill_started);
+  }
+
+  for (set::iterator i = backfill_targets.begin();
+       i != backfill_targets.end();
+       ++i) {
+    dout(10) << "peer osd." << *i
+	   << " info " << peer_info[*i]
+	   << " interval " << peer_backfill_info[*i].begin
+	   << "-" << peer_backfill_info[*i].end
+	   << " " << peer_backfill_info[*i].objects.size() << " objects"
+	   << dendl;
+  }
+
+  // update our local interval to cope with recent changes
+  backfill_info.begin = last_backfill_started;
+  update_range(&backfill_info, handle);
+
+  int ops = 0;
+  vector > > to_push;
+  vector > to_remove;
+  set add_to_stat;
+
+  for (set::iterator i = backfill_targets.begin();
+       i != backfill_targets.end();
+       ++i) {
+    peer_backfill_info[*i].trim_to(last_backfill_started);
+  }
+  backfill_info.trim_to(last_backfill_started);
+
+  hobject_t backfill_pos = MIN(backfill_info.begin, earliest_peer_backfill());
+  while (ops < max) {
+    if (backfill_info.begin <= earliest_peer_backfill() &&
+	!backfill_info.extends_to_end() && backfill_info.empty()) {
+      hobject_t next = backfill_info.end;
+      backfill_info.clear();
+      backfill_info.begin = next;
+      backfill_info.end = hobject_t::get_max();
+      update_range(&backfill_info, handle);
+      backfill_info.trim();
+    }
+    backfill_pos = MIN(backfill_info.begin, earliest_peer_backfill());
+
+    dout(20) << "   my backfill interval " << backfill_info.begin << "-" << backfill_info.end
+	     << " " << backfill_info.objects.size() << " objects"
+	     << " " << backfill_info.objects
+	     << dendl;
+
+    bool sent_scan = false;
+    for (set::iterator i = backfill_targets.begin();
+	 i != backfill_targets.end();
+	 ++i) {
+      pg_shard_t bt = *i;
+      BackfillInterval& pbi = peer_backfill_info[bt];
+
+      dout(20) << " peer shard " << bt << " backfill " << pbi.begin << "-"
+	       << pbi.end << " " << pbi.objects << dendl;
+      if (pbi.begin <= backfill_info.begin &&
+	  !pbi.extends_to_end() && pbi.empty()) {
+	dout(10) << " scanning peer osd." << bt << " from " << pbi.end << dendl;
+	epoch_t e = get_osdmap()->get_epoch();
+	MOSDPGScan *m = new MOSDPGScan(
+	  MOSDPGScan::OP_SCAN_GET_DIGEST, pg_whoami, e, e,
+	  spg_t(info.pgid.pgid, bt.shard),
+	  pbi.end, hobject_t());
+	osd->send_message_osd_cluster(bt.osd, m, get_osdmap()->get_epoch());
+	assert(waiting_on_backfill.find(bt) == waiting_on_backfill.end());
+	waiting_on_backfill.insert(bt);
+        sent_scan = true;
+      }
+    }
+
+    // Count simultaneous scans as a single op and let those complete
+    if (sent_scan) {
+        ops++;
+	start_recovery_op(hobject_t::get_max()); // XXX: was pbi.end
+        break;
+    }
+
+    if (backfill_info.empty() && all_peer_done()) {
+      dout(10) << " reached end for both local and all peers" << dendl;
+      break;
+    }
+
+    // Get object within set of peers to operate on and
+    // the set of targets for which that object applies.
+    hobject_t check = earliest_peer_backfill();
+
+    if (check < backfill_info.begin) {
+
+      set check_targets;
+      for (set::iterator i = backfill_targets.begin();
+	   i != backfill_targets.end();
+	   ++i) {
+        pg_shard_t bt = *i;
+        BackfillInterval& pbi = peer_backfill_info[bt];
+        if (pbi.begin == check)
+          check_targets.insert(bt);
+      }
+      assert(!check_targets.empty());
+
+      dout(20) << " BACKFILL removing " << check
+	       << " from peers " << check_targets << dendl;
+      for (set::iterator i = check_targets.begin();
+	   i != check_targets.end();
+	   ++i) {
+        pg_shard_t bt = *i;
+        BackfillInterval& pbi = peer_backfill_info[bt];
+        assert(pbi.begin == check);
+
+        to_remove.push_back(boost::make_tuple(check, pbi.objects.begin()->second, bt));
+        pbi.pop_front();
+      }
+      last_backfill_started = check;
+      // Don't increment ops here because deletions
+      // are cheap and not replied to unlike real recovery_ops,
+      // and we can't increment ops without requeueing ourself
+      // for recovery.
+    } else {
+      eversion_t& obj_v = backfill_info.objects.begin()->second;
+
+      vector need_ver_targs, missing_targs, keep_ver_targs, skip_targs;
+      for (set::iterator i = backfill_targets.begin();
+	   i != backfill_targets.end();
+	   ++i) {
+	pg_shard_t bt = *i;
+	BackfillInterval& pbi = peer_backfill_info[bt];
+        // Find all check peers that have the wrong version
+	if (check == backfill_info.begin && check == pbi.begin) {
+	  if (pbi.objects.begin()->second != obj_v) {
+	    need_ver_targs.push_back(bt);
+	  } else {
+	    keep_ver_targs.push_back(bt);
+	  }
+        } else {
+	  pg_info_t& pinfo = peer_info[bt];
+
+          // Only include peers that we've caught up to their backfill line
+	  // otherwise, they only appear to be missing this object
+	  // because their pbi.begin > backfill_info.begin.
+          if (backfill_info.begin > pinfo.last_backfill)
+	    missing_targs.push_back(bt);
+	  else
+	    skip_targs.push_back(bt);
+	}
+      }
+
+      if (!keep_ver_targs.empty()) {
+        // These peers have version obj_v
+	dout(20) << " BACKFILL keeping " << check
+		 << " with ver " << obj_v
+		 << " on peers " << keep_ver_targs << dendl;
+	//assert(!waiting_for_degraded_object.count(check));
+      }
+      if (!need_ver_targs.empty() || !missing_targs.empty()) {
+	ObjectContextRef obc = get_object_context(backfill_info.begin, false);
+	assert(obc);
+	if (obc->get_backfill_read()) {
+	  if (!need_ver_targs.empty()) {
+	    dout(20) << " BACKFILL replacing " << check
+		   << " with ver " << obj_v
+		   << " to peers " << need_ver_targs << dendl;
+	  }
+	  if (!missing_targs.empty()) {
+	    dout(20) << " BACKFILL pushing " << backfill_info.begin
+	         << " with ver " << obj_v
+	         << " to peers " << missing_targs << dendl;
+	  }
+	  vector all_push = need_ver_targs;
+	  all_push.insert(all_push.end(), missing_targs.begin(), missing_targs.end());
+
+	  to_push.push_back(
+	    boost::tuple >
+	    (backfill_info.begin, obj_v, obc, all_push));
+	  // Count all simultaneous pushes of the same object as a single op
+	  ops++;
+	} else {
+	  *work_started = true;
+	  dout(20) << "backfill blocking on " << backfill_info.begin
+		   << "; could not get rw_manager lock" << dendl;
+	  break;
+	}
+      }
+      dout(20) << "need_ver_targs=" << need_ver_targs
+	       << " keep_ver_targs=" << keep_ver_targs << dendl;
+      dout(20) << "backfill_targets=" << backfill_targets
+	       << " missing_targs=" << missing_targs
+	       << " skip_targs=" << skip_targs << dendl;
+
+      last_backfill_started = backfill_info.begin;
+      add_to_stat.insert(backfill_info.begin); // XXX: Only one for all pushes?
+      backfill_info.pop_front();
+      vector check_targets = need_ver_targs;
+      check_targets.insert(check_targets.end(), keep_ver_targs.begin(), keep_ver_targs.end());
+      for (vector::iterator i = check_targets.begin();
+	   i != check_targets.end();
+	   ++i) {
+        pg_shard_t bt = *i;
+        BackfillInterval& pbi = peer_backfill_info[bt];
+        pbi.pop_front();
+      }
+    }
+  }
+  backfill_pos = MIN(backfill_info.begin, earliest_peer_backfill());
+
+  for (set::iterator i = add_to_stat.begin();
+       i != add_to_stat.end();
+       ++i) {
+    ObjectContextRef obc = get_object_context(*i, false);
+    pg_stat_t stat;
+    add_object_context_to_pg_stat(obc, &stat);
+    pending_backfill_updates[*i] = stat;
+  }
+  for (unsigned i = 0; i < to_remove.size(); ++i) {
+    handle.reset_tp_timeout();
+
+    // ordered before any subsequent updates
+    send_remove_op(to_remove[i].get<0>(), to_remove[i].get<1>(), to_remove[i].get<2>());
+
+    pending_backfill_updates[to_remove[i].get<0>()]; // add empty stat!
+  }
+
+  PGBackend::RecoveryHandle *h = pgbackend->open_recovery_op();
+  for (unsigned i = 0; i < to_push.size(); ++i) {
+    handle.reset_tp_timeout();
+    prep_backfill_object_push(to_push[i].get<0>(), to_push[i].get<1>(),
+	    to_push[i].get<2>(), to_push[i].get<3>(), h);
+  }
+  pgbackend->run_recovery_op(h, cct->_conf->osd_recovery_op_priority);
+
+  dout(5) << "backfill_pos is " << backfill_pos << dendl;
+  for (set::iterator i = backfills_in_flight.begin();
+       i != backfills_in_flight.end();
+       ++i) {
+    dout(20) << *i << " is still in flight" << dendl;
+  }
+
+  hobject_t next_backfill_to_complete = backfills_in_flight.size() ?
+    *(backfills_in_flight.begin()) : backfill_pos;
+  hobject_t new_last_backfill = earliest_backfill();
+  dout(10) << "starting new_last_backfill at " << new_last_backfill << dendl;
+  for (map::iterator i = pending_backfill_updates.begin();
+       i != pending_backfill_updates.end() &&
+	 i->first < next_backfill_to_complete;
+       pending_backfill_updates.erase(i++)) {
+    assert(i->first > new_last_backfill);
+    for (set::iterator j = backfill_targets.begin();
+	 j != backfill_targets.end();
+	 ++j) {
+      pg_shard_t bt = *j;
+      pg_info_t& pinfo = peer_info[bt];
+      //Add stats to all peers that were missing object
+      if (i->first > pinfo.last_backfill)
+        pinfo.stats.add(i->second);
+    }
+    new_last_backfill = i->first;
+  }
+  dout(10) << "possible new_last_backfill at " << new_last_backfill << dendl;
+
+  assert(!pending_backfill_updates.empty() ||
+	 new_last_backfill == last_backfill_started);
+  if (pending_backfill_updates.empty() &&
+      backfill_pos.is_max()) {
+    assert(backfills_in_flight.empty());
+    new_last_backfill = backfill_pos;
+    last_backfill_started = backfill_pos;
+  }
+  dout(10) << "final new_last_backfill at " << new_last_backfill << dendl;
+
+  // If new_last_backfill == MAX, then we will send OP_BACKFILL_FINISH to
+  // all the backfill targets.  Otherwise, we will move last_backfill up on
+  // those targets need it and send OP_BACKFILL_PROGRESS to them.
+  for (set::iterator i = backfill_targets.begin();
+       i != backfill_targets.end();
+       ++i) {
+    pg_shard_t bt = *i;
+    pg_info_t& pinfo = peer_info[bt];
+
+    if (new_last_backfill > pinfo.last_backfill) {
+      pinfo.last_backfill = new_last_backfill;
+      epoch_t e = get_osdmap()->get_epoch();
+      MOSDPGBackfill *m = NULL;
+      if (pinfo.last_backfill.is_max()) {
+        m = new MOSDPGBackfill(
+	  MOSDPGBackfill::OP_BACKFILL_FINISH,
+	  e,
+	  e,
+	  spg_t(info.pgid.pgid, bt.shard));
+        // Use default priority here, must match sub_op priority
+        /* pinfo.stats might be wrong if we did log-based recovery on the
+         * backfilled portion in addition to continuing backfill.
+         */
+        pinfo.stats = info.stats;
+        start_recovery_op(hobject_t::get_max());
+      } else {
+        m = new MOSDPGBackfill(
+	  MOSDPGBackfill::OP_BACKFILL_PROGRESS,
+	  e,
+	  e,
+	  spg_t(info.pgid.pgid, bt.shard));
+        // Use default priority here, must match sub_op priority
+      }
+      m->last_backfill = pinfo.last_backfill;
+      m->stats = pinfo.stats;
+      osd->send_message_osd_cluster(bt.osd, m, get_osdmap()->get_epoch());
+      dout(10) << " peer " << bt
+	       << " num_objects now " << pinfo.stats.stats.sum.num_objects
+	       << " / " << info.stats.stats.sum.num_objects << dendl;
+    }
+  }
+
+  if (ops)
+    *work_started = true;
+  return ops;
+}
+
+void ReplicatedPG::prep_backfill_object_push(
+  hobject_t oid, eversion_t v,
+  ObjectContextRef obc,
+  vector peers,
+  PGBackend::RecoveryHandle *h)
+{
+  dout(10) << "push_backfill_object " << oid << " v " << v << " to peers " << peers << dendl;
+  assert(!peers.empty());
+
+  backfills_in_flight.insert(oid);
+  for (unsigned int i = 0 ; i < peers.size(); ++i) {
+    map::iterator bpm = peer_missing.find(peers[i]);
+    assert(bpm != peer_missing.end());
+    bpm->second.add(oid, eversion_t(), eversion_t());
+  }
+
+  assert(!recovering.count(oid));
+
+  start_recovery_op(oid);
+  recovering.insert(make_pair(oid, obc));
+
+  // We need to take the read_lock here in order to flush in-progress writes
+  obc->ondisk_read_lock();
+  pgbackend->recover_object(
+    oid,
+    v,
+    ObjectContextRef(),
+    obc,
+    h);
+  obc->ondisk_read_unlock();
+}
+
+void ReplicatedPG::update_range(
+  BackfillInterval *bi,
+  ThreadPool::TPHandle &handle)
+{
+  int local_min = cct->_conf->osd_backfill_scan_min;
+  int local_max = cct->_conf->osd_backfill_scan_max;
+
+  if (bi->version < info.log_tail) {
+    dout(10) << __func__<< ": bi is old, rescanning local backfill_info"
+	     << dendl;
+    if (last_update_applied >= info.log_tail) {
+      bi->version = last_update_applied;
+    } else {
+      osr->flush();
+      bi->version = info.last_update;
+    }
+    scan_range(local_min, local_max, bi, handle);
+  }
+
+  if (bi->version >= info.last_update) {
+    dout(10) << __func__<< ": bi is current " << dendl;
+    assert(bi->version == info.last_update);
+  } else if (bi->version >= info.log_tail) {
+    if (pg_log.get_log().empty()) {
+      /* Because we don't move log_tail on split, the log might be
+       * empty even if log_tail != last_update.  However, the only
+       * way to get here with an empty log is if log_tail is actually
+       * eversion_t(), because otherwise the entry which changed
+       * last_update since the last scan would have to be present.
+       */
+      assert(bi->version == eversion_t());
+      return;
+    }
+    assert(!pg_log.get_log().empty());
+    dout(10) << __func__<< ": bi is old, (" << bi->version
+	     << ") can be updated with log" << dendl;
+    list::const_iterator i =
+      pg_log.get_log().log.end();
+    --i;
+    while (i != pg_log.get_log().log.begin() &&
+           i->version > bi->version) {
+      --i;
+    }
+    if (i->version == bi->version)
+      ++i;
+
+    assert(i != pg_log.get_log().log.end());
+    dout(10) << __func__ << ": updating from version " << i->version
+	     << dendl;
+    for (; i != pg_log.get_log().log.end(); ++i) {
+      const hobject_t &soid = i->soid;
+      if (soid >= bi->begin && soid < bi->end) {
+	if (i->is_update()) {
+	  dout(10) << __func__ << ": " << i->soid << " updated to version "
+		   << i->version << dendl;
+	  bi->objects.erase(i->soid);
+	  bi->objects.insert(
+	    make_pair(
+	      i->soid,
+	      i->version));
+	} else if (i->is_delete()) {
+	  dout(10) << __func__ << ": " << i->soid << " removed" << dendl;
+	  bi->objects.erase(i->soid);
+	}
+      }
+    }
+    bi->version = info.last_update;
+  } else {
+    assert(0 == "scan_range should have raised bi->version past log_tail");
+  }
+}
+
+void ReplicatedPG::scan_range(
+  int min, int max, BackfillInterval *bi,
+  ThreadPool::TPHandle &handle)
+{
+  assert(is_locked());
+  dout(10) << "scan_range from " << bi->begin << dendl;
+  bi->objects.clear();  // for good measure
+
+  vector ls;
+  ls.reserve(max);
+  int r = pgbackend->objects_list_partial(bi->begin, min, max, 0, &ls, &bi->end);
+  assert(r >= 0);
+  dout(10) << " got " << ls.size() << " items, next " << bi->end << dendl;
+  dout(20) << ls << dendl;
+
+  for (vector::iterator p = ls.begin(); p != ls.end(); ++p) {
+    handle.reset_tp_timeout();
+    ObjectContextRef obc;
+    if (is_primary())
+      obc = object_contexts.lookup(*p);
+    if (obc) {
+      bi->objects[*p] = obc->obs.oi.version;
+      dout(20) << "  " << *p << " " << obc->obs.oi.version << dendl;
+    } else {
+      bufferlist bl;
+      int r = pgbackend->objects_get_attr(*p, OI_ATTR, &bl);
+      assert(r >= 0);
+      object_info_t oi(bl);
+      bi->objects[*p] = oi.version;
+      dout(20) << "  " << *p << " " << oi.version << dendl;
+    }
+  }
+}
+
+
+/** check_local
+ * 
+ * verifies that stray objects have been deleted
+ */
+void ReplicatedPG::check_local()
+{
+  dout(10) << __func__ << dendl;
+
+  assert(info.last_update >= pg_log.get_tail());  // otherwise we need some help!
+
+  if (!cct->_conf->osd_debug_verify_stray_on_activate)
+    return;
+
+  // just scan the log.
+  set did;
+  for (list::const_reverse_iterator p = pg_log.get_log().log.rbegin();
+       p != pg_log.get_log().log.rend();
+       ++p) {
+    if (did.count(p->soid))
+      continue;
+    did.insert(p->soid);
+
+    if (p->is_delete()) {
+      dout(10) << " checking " << p->soid
+	       << " at " << p->version << dendl;
+      struct stat st;
+      int r = osd->store->stat(
+	coll,
+	ghobject_t(p->soid, ghobject_t::NO_GEN, pg_whoami.shard),
+	&st);
+      if (r != -ENOENT) {
+	derr << __func__ << " " << p->soid << " exists, but should have been "
+	     << "deleted" << dendl;
+	assert(0 == "erroneously present object");
+      }
+    } else {
+      // ignore old(+missing) objects
+    }
+  }
+}
+
+
+
+// ===========================
+// hit sets
+
+hobject_t ReplicatedPG::get_hit_set_current_object(utime_t stamp)
+{
+  ostringstream ss;
+  ss << "hit_set_" << info.pgid.pgid << "_current_" << stamp;
+  hobject_t hoid(sobject_t(ss.str(), CEPH_NOSNAP), "",
+		 info.pgid.ps(), info.pgid.pool(),
+		 cct->_conf->osd_hit_set_namespace);
+  dout(20) << __func__ << " " << hoid << dendl;
+  return hoid;
+}
+
+hobject_t ReplicatedPG::get_hit_set_archive_object(utime_t start, utime_t end)
+{
+  ostringstream ss;
+  ss << "hit_set_" << info.pgid.pgid << "_archive_" << start << "_" << end;
+  hobject_t hoid(sobject_t(ss.str(), CEPH_NOSNAP), "",
+		 info.pgid.ps(), info.pgid.pool(),
+		 cct->_conf->osd_hit_set_namespace);
+  dout(20) << __func__ << " " << hoid << dendl;
+  return hoid;
+}
+
+void ReplicatedPG::hit_set_clear()
+{
+  dout(20) << __func__ << dendl;
+  hit_set.reset();
+  hit_set_start_stamp = utime_t();
+  hit_set_flushing.clear();
+}
+
+void ReplicatedPG::hit_set_setup()
+{
+  if (!is_active() ||
+      !is_primary() ||
+      !pool.info.hit_set_count ||
+      !pool.info.hit_set_period ||
+      pool.info.hit_set_params.get_type() == HitSet::TYPE_NONE) {
+    hit_set_clear();
+    //hit_set_remove_all();  // FIXME: implement me soon
+    return;
+  }
+
+  // FIXME: discard any previous data for now
+  hit_set_create();
+
+  // include any writes we know about from the pg log.  this doesn't
+  // capture reads, but it is better than nothing!
+  hit_set_apply_log();
+}
+
+void ReplicatedPG::hit_set_create()
+{
+  utime_t now = ceph_clock_now(NULL);
+  // make a copy of the params to modify
+  HitSet::Params params(pool.info.hit_set_params);
+
+  dout(20) << __func__ << " " << params << dendl;
+  if (pool.info.hit_set_params.get_type() == HitSet::TYPE_BLOOM) {
+    BloomHitSet::Params *p =
+      static_cast(params.impl.get());
+
+    // convert false positive rate so it holds up across the full period
+    p->set_fpp(p->get_fpp() / pool.info.hit_set_count);
+    if (p->get_fpp() <= 0.0)
+      p->set_fpp(.01);  // fpp cannot be zero!
+
+    // if we don't have specified size, estimate target size based on the
+    // previous bin!
+    if (p->target_size == 0 && hit_set) {
+      utime_t dur = now - hit_set_start_stamp;
+      unsigned unique = hit_set->approx_unique_insert_count();
+      dout(20) << __func__ << " previous set had approx " << unique
+	       << " unique items over " << dur << " seconds" << dendl;
+      p->target_size = (double)unique * (double)pool.info.hit_set_period
+		     / (double)dur;
+    }
+    if (p->target_size < static_cast(g_conf->osd_hit_set_min_size))
+      p->target_size = g_conf->osd_hit_set_min_size;
+
+    if (p->target_size > static_cast(g_conf->osd_hit_set_max_size))
+      p->target_size = g_conf->osd_hit_set_max_size;
+
+    p->seed = now.sec();
+
+    dout(10) << __func__ << " target_size " << p->target_size
+	     << " fpp " << p->get_fpp() << dendl;
+  }
+  hit_set.reset(new HitSet(params));
+  hit_set_start_stamp = now;
+}
+
+/**
+ * apply log entries to set
+ *
+ * this would only happen after peering, to at least capture writes
+ * during an interval that was potentially lost.
+ */
+bool ReplicatedPG::hit_set_apply_log()
+{
+  if (!hit_set)
+    return false;
+
+  eversion_t to = info.last_update;
+  eversion_t from = info.hit_set.current_last_update;
+  if (to <= from) {
+    dout(20) << __func__ << " no update" << dendl;
+    return false;
+  }
+
+  dout(20) << __func__ << " " << to << " .. " << info.last_update << dendl;
+  list::const_reverse_iterator p = pg_log.get_log().log.rbegin();
+  while (p != pg_log.get_log().log.rend() && p->version > to)
+    ++p;
+  while (p != pg_log.get_log().log.rend() && p->version > from) {
+    hit_set->insert(p->soid);
+    ++p;
+  }
+
+  return true;
+}
+
+struct C_HitSetFlushing : public Context {
+  ReplicatedPGRef pg;
+  time_t hit_set_name;
+  C_HitSetFlushing(ReplicatedPG *p, time_t n) : pg(p), hit_set_name(n) { }
+  void finish(int r) {
+    pg->hit_set_flushing.erase(hit_set_name);
+  }
+};
+
+void ReplicatedPG::hit_set_persist()
+{
+  dout(10) << __func__  << dendl;
+  bufferlist bl;
+  unsigned max = pool.info.hit_set_count;
+
+  utime_t now = ceph_clock_now(cct);
+  RepGather *repop;
+  hobject_t oid;
+  time_t flush_time = 0;
+
+  // See what start is going to be used later
+  utime_t start = info.hit_set.current_info.begin;
+  if (!start)
+     start = hit_set_start_stamp;
+
+  // If any archives are degraded we skip this persist request
+  // account for the additional entry being added below
+  for (list::iterator p = info.hit_set.history.begin();
+       p != info.hit_set.history.end();
+       ++p) {
+    hobject_t aoid = get_hit_set_archive_object(p->begin, p->end);
+
+    // Once we hit a degraded object just skip further trim
+    if (is_degraded_object(aoid))
+      return;
+  }
+
+  oid = get_hit_set_archive_object(start, now);
+  // If the current object is degraded we skip this persist request
+  if (is_degraded_object(oid))
+    return;
+
+  // If backfill is in progress and we could possibly overlap with the
+  // hit_set_* objects, back off.  Since these all have
+  // hobject_t::hash set to pgid.ps(), and those sort first, we can
+  // look just at that.  This is necessary because our transactions
+  // may include a modify of the new hit_set *and* a delete of the
+  // old one, and this may span the backfill boundary.
+  for (set::iterator p = backfill_targets.begin();
+       p != backfill_targets.end();
+       ++p) {
+    assert(peer_info.count(*p));
+    const pg_info_t& pi = peer_info[*p];
+    if (pi.last_backfill == hobject_t() ||
+	pi.last_backfill.hash == info.pgid.ps()) {
+      dout(10) << __func__ << " backfill target osd." << *p
+	       << " last_backfill has not progressed past pgid ps"
+	       << dendl;
+      return;
+    }
+  }
+
+  if (!info.hit_set.current_info.begin)
+    info.hit_set.current_info.begin = hit_set_start_stamp;
+
+  hit_set->seal();
+  ::encode(*hit_set, bl);
+  info.hit_set.current_info.end = now;
+  dout(20) << __func__ << " archive " << oid << dendl;
+
+  if (agent_state)
+    agent_state->add_hit_set(info.hit_set.current_info.begin, hit_set);
+
+  // hold a ref until it is flushed to disk
+  hit_set_flushing[info.hit_set.current_info.begin] = hit_set;
+  flush_time = info.hit_set.current_info.begin;
+
+  ObjectContextRef obc = get_object_context(oid, true);
+  repop = simple_repop_create(obc);
+  if (flush_time != 0)
+    repop->on_applied = new C_HitSetFlushing(this, flush_time);
+  OpContext *ctx = repop->ctx;
+  ctx->at_version = get_next_version();
+  ctx->updated_hset_history = info.hit_set;
+  pg_hit_set_history_t &updated_hit_set_hist = *(ctx->updated_hset_history);
+
+  if (updated_hit_set_hist.current_last_stamp != utime_t()) {
+    // FIXME: we cheat slightly here by bundling in a remove on a object
+    // other the RepGather object.  we aren't carrying an ObjectContext for
+    // the deleted object over this period.
+    hobject_t old_obj =
+      get_hit_set_current_object(updated_hit_set_hist.current_last_stamp);
+    ctx->log.push_back(
+      pg_log_entry_t(pg_log_entry_t::DELETE,
+		     old_obj,
+		     ctx->at_version,
+		     updated_hit_set_hist.current_last_update,
+		     0,
+		     osd_reqid_t(),
+		     ctx->mtime));
+    if (pool.info.require_rollback()) {
+      if (ctx->log.back().mod_desc.rmobject(ctx->at_version.version)) {
+	ctx->op_t->stash(old_obj, ctx->at_version.version);
+      } else {
+	ctx->op_t->remove(old_obj);
+      }
+    } else {
+      ctx->op_t->remove(old_obj);
+      ctx->log.back().mod_desc.mark_unrollbackable();
+    }
+    ++ctx->at_version.version;
+
+    struct stat st;
+    int r = osd->store->stat(
+      coll,
+      ghobject_t(old_obj, ghobject_t::NO_GEN, pg_whoami.shard),
+      &st);
+    assert(r == 0);
+    --ctx->delta_stats.num_objects;
+    ctx->delta_stats.num_bytes -= st.st_size;
+  }
+
+  updated_hit_set_hist.current_last_update = info.last_update; // *after* above remove!
+  updated_hit_set_hist.current_info.version = ctx->at_version;
+
+  updated_hit_set_hist.history.push_back(updated_hit_set_hist.current_info);
+  hit_set_create();
+  updated_hit_set_hist.current_info = pg_hit_set_info_t();
+  updated_hit_set_hist.current_last_stamp = utime_t();
+
+  // fabricate an object_info_t and SnapSet
+  obc->obs.oi.version = ctx->at_version;
+  obc->obs.oi.mtime = now;
+  obc->obs.oi.size = bl.length();
+  obc->obs.exists = true;
+
+  ctx->new_obs = obc->obs;
+  ctx->new_snapset.head_exists = true;
+
+  ctx->delta_stats.num_objects++;
+  ctx->delta_stats.num_objects_hit_set_archive++;
+  ctx->delta_stats.num_bytes += bl.length();
+
+  bufferlist bss;
+  ::encode(ctx->new_snapset, bss);
+  bufferlist boi(sizeof(ctx->new_obs.oi));
+  ::encode(ctx->new_obs.oi, boi);
+
+  ctx->op_t->append(oid, 0, bl.length(), bl);
+  setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, OI_ATTR, boi);
+  setattr_maybe_cache(ctx->obc, ctx, ctx->op_t, SS_ATTR, bss);
+  ctx->log.push_back(
+    pg_log_entry_t(
+      pg_log_entry_t::MODIFY,
+      oid,
+      ctx->at_version,
+      ctx->obs->oi.version,
+      0,
+      osd_reqid_t(),
+      ctx->mtime)
+    );
+  if (pool.info.require_rollback()) {
+    ctx->log.back().mod_desc.create();
+  } else {
+    ctx->log.back().mod_desc.mark_unrollbackable();
+  }
+
+  hit_set_trim(repop, max);
+
+  info.stats.stats.add(ctx->delta_stats, string());
+
+  simple_repop_submit(repop);
+}
+
+void ReplicatedPG::hit_set_trim(RepGather *repop, unsigned max)
+{
+  assert(repop->ctx->updated_hset_history);
+  pg_hit_set_history_t &updated_hit_set_hist =
+    *(repop->ctx->updated_hset_history);
+  for (unsigned num = updated_hit_set_hist.history.size(); num > max; --num) {
+    list::iterator p = updated_hit_set_hist.history.begin();
+    assert(p != updated_hit_set_hist.history.end());
+    hobject_t oid = get_hit_set_archive_object(p->begin, p->end);
+
+    assert(!is_degraded_object(oid));
+
+    dout(20) << __func__ << " removing " << oid << dendl;
+    ++repop->ctx->at_version.version;
+    repop->ctx->log.push_back(
+        pg_log_entry_t(pg_log_entry_t::DELETE,
+		       oid,
+		       repop->ctx->at_version,
+		       p->version,
+		       0,
+		       osd_reqid_t(),
+		       repop->ctx->mtime));
+    if (pool.info.require_rollback()) {
+      if (repop->ctx->log.back().mod_desc.rmobject(
+	  repop->ctx->at_version.version)) {
+	repop->ctx->op_t->stash(oid, repop->ctx->at_version.version);
+      } else {
+	repop->ctx->op_t->remove(oid);
+      }
+    } else {
+      repop->ctx->op_t->remove(oid);
+      repop->ctx->log.back().mod_desc.mark_unrollbackable();
+    }
+    if (agent_state)
+      agent_state->remove_oldest_hit_set();
+    updated_hit_set_hist.history.pop_front();
+
+    ObjectContextRef obc = get_object_context(oid, false);
+    assert(obc);
+    --repop->ctx->delta_stats.num_objects;
+    --repop->ctx->delta_stats.num_objects_hit_set_archive;
+    repop->ctx->delta_stats.num_bytes -= obc->obs.oi.size;
+  }
+}
+
+
+// =======================================
+// cache agent
+
+void ReplicatedPG::agent_setup()
+{
+  assert(is_locked());
+  if (!is_active() ||
+      !is_primary() ||
+      pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE ||
+      pool.info.tier_of < 0 ||
+      !get_osdmap()->have_pg_pool(pool.info.tier_of)) {
+    agent_clear();
+    return;
+  }
+  if (!agent_state) {
+    agent_state.reset(new TierAgentState);
+
+    // choose random starting position
+    agent_state->position = hobject_t();
+    agent_state->position.pool = info.pgid.pool();
+    agent_state->position.hash = pool.info.get_random_pg_position(
+      info.pgid.pgid,
+      rand());
+    agent_state->start = agent_state->position;
+
+    dout(10) << __func__ << " allocated new state, position "
+	     << agent_state->position << dendl;
+  } else {
+    dout(10) << __func__ << " keeping existing state" << dendl;
+  }
+
+  if (info.stats.stats_invalid) {
+    osd->clog.warn() << "pg " << info.pgid << " has invalid (post-split) stats; must scrub before tier agent can activate";
+  }
+
+  agent_choose_mode();
+}
+
+void ReplicatedPG::agent_clear()
+{
+  agent_stop();
+  agent_state.reset(NULL);
+}
+
+// Return false if no objects operated on since start of object hash space
+bool ReplicatedPG::agent_work(int start_max)
+{
+  lock();
+  if (!agent_state) {
+    dout(10) << __func__ << " no agent state, stopping" << dendl;
+    unlock();
+    return true;
+  }
+
+  assert(!deleting);
+
+  if (agent_state->is_idle()) {
+    dout(10) << __func__ << " idle, stopping" << dendl;
+    unlock();
+    return true;
+  }
+
+  osd->logger->inc(l_osd_agent_wake);
+
+  dout(10) << __func__
+	   << " max " << start_max
+	   << ", flush " << agent_state->get_flush_mode_name()
+	   << ", evict " << agent_state->get_evict_mode_name()
+	   << ", pos " << agent_state->position
+	   << dendl;
+  assert(is_primary());
+  assert(is_active());
+
+  agent_load_hit_sets();
+
+  const pg_pool_t *base_pool = get_osdmap()->get_pg_pool(pool.info.tier_of);
+  assert(base_pool);
+
+  int ls_min = 1;
+  int ls_max = 10; // FIXME?
+
+  // list some objects.  this conveniently lists clones (oldest to
+  // newest) before heads... the same order we want to flush in.
+  //
+  // NOTE: do not flush the Sequencer.  we will assume that the
+  // listing we get back is imprecise.
+  vector ls;
+  hobject_t next;
+  int r = pgbackend->objects_list_partial(agent_state->position, ls_min, ls_max,
+					  0 /* no filtering by snapid */,
+					  &ls, &next);
+  assert(r >= 0);
+  dout(20) << __func__ << " got " << ls.size() << " objects" << dendl;
+  int started = 0;
+  for (vector::iterator p = ls.begin();
+       p != ls.end();
+       ++p) {
+    if (p->nspace == cct->_conf->osd_hit_set_namespace) {
+      dout(20) << __func__ << " skip (hit set) " << *p << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+    if (is_degraded_object(*p)) {
+      dout(20) << __func__ << " skip (degraded) " << *p << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+    if (is_missing_object(p->get_head())) {
+      dout(20) << __func__ << " skip (missing head) " << *p << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+    ObjectContextRef obc = get_object_context(*p, false, NULL);
+    if (!obc) {
+      // we didn't flush; we may miss something here.
+      dout(20) << __func__ << " skip (no obc) " << *p << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+    if (!obc->obs.exists) {
+      dout(20) << __func__ << " skip (dne) " << obc->obs.oi.soid << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+    if (scrubber.write_blocked_by_scrub(obc->obs.oi.soid)) {
+      dout(20) << __func__ << " skip (scrubbing) " << obc->obs.oi << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+    if (obc->is_blocked()) {
+      dout(20) << __func__ << " skip (blocked) " << obc->obs.oi << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+
+    // be careful flushing omap to an EC pool.
+    if (base_pool->is_erasure() &&
+	obc->obs.oi.test_flag(object_info_t::FLAG_OMAP)) {
+      dout(20) << __func__ << " skip (omap to EC) " << obc->obs.oi << dendl;
+      osd->logger->inc(l_osd_agent_skip);
+      continue;
+    }
+
+    if (agent_state->flush_mode != TierAgentState::FLUSH_MODE_IDLE &&
+	agent_maybe_flush(obc))
+      ++started;
+    if (agent_state->evict_mode != TierAgentState::EVICT_MODE_IDLE &&
+	agent_maybe_evict(obc))
+      ++started;
+    if (started >= start_max) {
+      // If finishing early, set "next" to the next object
+      if (++p != ls.end())
+	next = *p;
+      break;
+    }
+  }
+
+  if (++agent_state->hist_age > g_conf->osd_agent_hist_halflife) {
+    dout(20) << __func__ << " resetting atime and temp histograms" << dendl;
+    agent_state->hist_age = 0;
+    agent_state->atime_hist.decay();
+    agent_state->temp_hist.decay();
+  }
+
+  // Total objects operated on so far
+  int total_started = agent_state->started + started;
+  bool need_delay = false;
+
+  dout(20) << __func__ << " start pos " << agent_state->position
+    << " next start pos " << next
+    << " started " << total_started << dendl;
+
+  // See if we've made a full pass over the object hash space
+  // This might check at most ls_max objects a second time to notice that
+  // we've checked every objects at least once.
+  if (agent_state->position < agent_state->start && next >= agent_state->start) {
+    dout(20) << __func__ << " wrap around " << agent_state->start << dendl;
+    if (total_started == 0)
+      need_delay = true;
+    else
+      total_started = 0;
+    agent_state->start = next;
+  }
+  agent_state->started = total_started;
+
+  // See if we are starting from beginning
+  if (next.is_max())
+    agent_state->position = hobject_t();
+  else
+    agent_state->position = next;
+
+  if (need_delay) {
+    assert(agent_state->delaying == false);
+    agent_delay();
+    unlock();
+    return false;
+  }
+  agent_choose_mode();
+  unlock();
+  return true;
+}
+
+void ReplicatedPG::agent_load_hit_sets()
+{
+  if (agent_state->evict_mode == TierAgentState::EVICT_MODE_IDLE) {
+    agent_state->discard_hit_sets();
+    return;
+  }
+
+  if (agent_state->hit_set_map.size() < info.hit_set.history.size()) {
+    dout(10) << __func__ << dendl;
+    for (list::iterator p = info.hit_set.history.begin();
+	 p != info.hit_set.history.end(); ++p) {
+      if (agent_state->hit_set_map.count(p->begin.sec()) == 0) {
+	dout(10) << __func__ << " loading " << p->begin << "-"
+		 << p->end << dendl;
+	if (!pool.info.is_replicated()) {
+	  // FIXME: EC not supported here yet
+	  derr << __func__ << " on non-replicated pool" << dendl;
+	  break;
+	}
+
+	// check if it's still in flight
+	if (hit_set_flushing.count(p->begin)) {
+	  agent_state->add_hit_set(p->begin.sec(), hit_set_flushing[p->begin]);
+	  continue;
+	}
+
+	hobject_t oid = get_hit_set_archive_object(p->begin, p->end);
+	if (is_unreadable_object(oid)) {
+	  dout(10) << __func__ << " unreadable " << oid << ", waiting" << dendl;
+	  break;
+	}
+
+	ObjectContextRef obc = get_object_context(oid, false);
+	if (!obc) {
+	  derr << __func__ << ": could not load hitset " << oid << dendl;
+	  break;
+	}
+
+	bufferlist bl;
+	{
+	  obc->ondisk_read_lock();
+	  int r = osd->store->read(coll, oid, 0, 0, bl);
+	  assert(r >= 0);
+	  obc->ondisk_read_unlock();
+	}
+	HitSetRef hs(new HitSet);
+	bufferlist::iterator pbl = bl.begin();
+	::decode(*hs, pbl);
+	agent_state->add_hit_set(p->begin.sec(), hs);
+      }
+    }
+  }
+}
+
+struct C_AgentFlushStartStop : public Context {
+  ReplicatedPGRef pg;
+  hobject_t oid;
+  C_AgentFlushStartStop(ReplicatedPG *p, hobject_t o) : pg(p), oid(o) {
+    pg->osd->agent_start_op(oid);
+  }
+  void finish(int r) {
+    pg->osd->agent_finish_op(oid);
+  }
+};
+
+bool ReplicatedPG::agent_maybe_flush(ObjectContextRef& obc)
+{
+  if (!obc->obs.oi.is_dirty()) {
+    dout(20) << __func__ << " skip (clean) " << obc->obs.oi << dendl;
+    osd->logger->inc(l_osd_agent_skip);
+    return false;
+  }
+
+  utime_t now = ceph_clock_now(NULL);
+  if (obc->obs.oi.mtime + utime_t(pool.info.cache_min_flush_age, 0) > now) {
+    dout(20) << __func__ << " skip (too young) " << obc->obs.oi << dendl;
+    osd->logger->inc(l_osd_agent_skip);
+    return false;
+  }
+
+  if (osd->agent_is_active_oid(obc->obs.oi.soid)) {
+    dout(20) << __func__ << " skip (flushing) " << obc->obs.oi << dendl;
+    osd->logger->inc(l_osd_agent_skip);
+    return false;
+  }
+
+  dout(10) << __func__ << " flushing " << obc->obs.oi << dendl;
+
+  // FIXME: flush anything dirty, regardless of what distribution of
+  // ages we expect.
+
+  Context *on_flush = new C_AgentFlushStartStop(this, obc->obs.oi.soid);
+  int result = start_flush(
+    OpRequestRef(), obc, false, NULL,
+    on_flush);
+  if (result != -EINPROGRESS) {
+    on_flush->complete(result);
+    dout(10) << __func__ << " start_flush() failed " << obc->obs.oi
+      << " with " << result << dendl;
+    osd->logger->inc(l_osd_agent_skip);
+    return false;
+  }
+
+  osd->logger->inc(l_osd_agent_flush);
+  return true;
+}
+
+bool ReplicatedPG::agent_maybe_evict(ObjectContextRef& obc)
+{
+  const hobject_t& soid = obc->obs.oi.soid;
+  if (obc->obs.oi.is_dirty()) {
+    dout(20) << __func__ << " skip (dirty) " << obc->obs.oi << dendl;
+    return false;
+  }
+  if (!obc->obs.oi.watchers.empty()) {
+    dout(20) << __func__ << " skip (watchers) " << obc->obs.oi << dendl;
+    return false;
+  }
+
+  if (soid.snap == CEPH_NOSNAP) {
+    int result = _verify_no_head_clones(soid, obc->ssc->snapset);
+    if (result < 0) {
+      dout(20) << __func__ << " skip (clones) " << obc->obs.oi << dendl;
+      return false;
+    }
+  }
+
+  if (agent_state->evict_mode != TierAgentState::EVICT_MODE_FULL &&
+      hit_set) {
+    // is this object old and/or cold enough?
+    int atime = -1, temp = 0;
+    agent_estimate_atime_temp(soid, &atime, NULL /*FIXME &temp*/);
+
+    uint64_t atime_upper = 0, atime_lower = 0;
+    if (atime < 0 && obc->obs.oi.mtime != utime_t())
+      atime = ceph_clock_now(NULL).sec() - obc->obs.oi.mtime;
+    if (atime < 0)
+      atime = pool.info.hit_set_period * pool.info.hit_set_count; // "infinite"
+    if (atime >= 0) {
+      agent_state->atime_hist.add(atime);
+      agent_state->atime_hist.get_position_micro(atime, &atime_lower,
+						 &atime_upper);
+    }
+
+    unsigned temp_upper = 0, temp_lower = 0;
+    /*
+    // FIXME: bound atime based on creation time?
+    agent_state->temp_hist.add(atime);
+    agent_state->temp_hist.get_position_micro(temp, &temp_lower, &temp_upper);
+    */
+
+    dout(20) << __func__
+	     << " atime " << atime
+	     << " pos " << atime_lower << "-" << atime_upper
+	     << ", temp " << temp
+	     << " pos " << temp_lower << "-" << temp_upper
+	     << ", evict_effort " << agent_state->evict_effort
+	     << dendl;
+    dout(30) << "agent_state:\n";
+    Formatter *f = new_formatter("");
+    f->open_object_section("agent_state");
+    agent_state->dump(f);
+    f->close_section();
+    f->flush(*_dout);
+    delete f;
+    *_dout << dendl;
+
+    // FIXME: ignore temperature for now.
+
+    // KISS: if [lower,upper] spans our target effort, evict it.
+    if (atime_lower >= agent_state->evict_effort)
+      return false;
+  }
+
+  dout(10) << __func__ << " evicting " << obc->obs.oi << dendl;
+  RepGather *repop = simple_repop_create(obc);
+  OpContext *ctx = repop->ctx;
+  ctx->at_version = get_next_version();
+  assert(ctx->new_obs.exists);
+  int r = _delete_oid(ctx, true);
+  if (obc->obs.oi.is_omap())
+    ctx->delta_stats.num_objects_omap--;
+  assert(r == 0);
+  finish_ctx(ctx, pg_log_entry_t::DELETE, false);
+  simple_repop_submit(repop);
+  osd->logger->inc(l_osd_tier_evict);
+  osd->logger->inc(l_osd_agent_evict);
+  return true;
+}
+
+void ReplicatedPG::agent_stop()
+{
+  dout(20) << __func__ << dendl;
+  if (agent_state && !agent_state->is_idle()) {
+    agent_state->evict_mode = TierAgentState::EVICT_MODE_IDLE;
+    agent_state->flush_mode = TierAgentState::FLUSH_MODE_IDLE;
+    osd->agent_disable_pg(this, agent_state->evict_effort);
+  }
+}
+
+void ReplicatedPG::agent_delay()
+{
+  dout(20) << __func__ << dendl;
+  if (agent_state && !agent_state->is_idle()) {
+    assert(agent_state->delaying == false);
+    agent_state->delaying = true;
+    osd->agent_disable_pg(this, agent_state->evict_effort);
+  }
+}
+
+void ReplicatedPG::agent_choose_mode_restart()
+{
+  dout(20) << __func__ << dendl;
+  lock();
+  if (agent_state && agent_state->delaying) {
+    agent_state->delaying = false;
+    agent_choose_mode(true);
+  }
+  unlock();
+}
+
+void ReplicatedPG::agent_choose_mode(bool restart)
+{
+  // Let delay play out
+  if (agent_state->delaying) {
+    dout(20) << __func__ << this << " delaying, ignored" << dendl;
+    return;
+  }
+
+  uint64_t divisor = pool.info.get_pg_num_divisor(info.pgid.pgid);
+  assert(divisor > 0);
+
+  uint64_t num_user_objects = info.stats.stats.sum.num_objects;
+
+  // adjust (effective) user objects down based on the number
+  // of HitSet objects, which should not count toward our total since
+  // they cannot be flushed.
+  uint64_t unflushable = info.stats.stats.sum.num_objects_hit_set_archive;
+
+  // also exclude omap objects if ec backing pool
+  const pg_pool_t *base_pool = get_osdmap()->get_pg_pool(pool.info.tier_of);
+  assert(base_pool);
+  if (base_pool->is_erasure())
+    unflushable += info.stats.stats.sum.num_objects_omap;
+
+
+  if (num_user_objects > unflushable)
+    num_user_objects -= unflushable;
+  else
+    num_user_objects = 0;
+
+  // also reduce the num_dirty by num_objects_omap
+  int64_t num_dirty = info.stats.stats.sum.num_objects_dirty;
+  if (base_pool->is_erasure()) {
+    if (num_dirty > info.stats.stats.sum.num_objects_omap)
+      num_dirty -= info.stats.stats.sum.num_objects_omap;
+    else
+      num_dirty = 0;
+  }
+
+  dout(10) << __func__
+	   << " flush_mode: "
+	   << TierAgentState::get_flush_mode_name(agent_state->flush_mode)
+	   << " evict_mode: "
+	   << TierAgentState::get_evict_mode_name(agent_state->evict_mode)
+	   << " num_objects: " << info.stats.stats.sum.num_objects
+	   << " num_bytes: " << info.stats.stats.sum.num_bytes
+	   << " num_objects_dirty: " << info.stats.stats.sum.num_objects_dirty
+	   << " num_objects_omap: " << info.stats.stats.sum.num_objects_omap
+	   << " num_dirty: " << num_dirty
+	   << " num_user_objects: " << num_user_objects
+	   << " pool.info.target_max_bytes: " << pool.info.target_max_bytes
+	   << " pool.info.target_max_objects: " << pool.info.target_max_objects
+	   << dendl;
+
+  // get dirty, full ratios
+  uint64_t dirty_micro = 0;
+  uint64_t full_micro = 0;
+  if (pool.info.target_max_bytes && info.stats.stats.sum.num_objects > 0) {
+    uint64_t avg_size = info.stats.stats.sum.num_bytes /
+      info.stats.stats.sum.num_objects;
+    dirty_micro =
+      num_dirty * avg_size * 1000000 /
+      MAX(pool.info.target_max_bytes / divisor, 1);
+    full_micro =
+      num_user_objects * avg_size * 1000000 /
+      MAX(pool.info.target_max_bytes / divisor, 1);
+  }
+  if (pool.info.target_max_objects > 0) {
+    uint64_t dirty_objects_micro =
+      num_dirty * 1000000 /
+      MAX(pool.info.target_max_objects / divisor, 1);
+    if (dirty_objects_micro > dirty_micro)
+      dirty_micro = dirty_objects_micro;
+    uint64_t full_objects_micro =
+      num_user_objects * 1000000 /
+      MAX(pool.info.target_max_objects / divisor, 1);
+    if (full_objects_micro > full_micro)
+      full_micro = full_objects_micro;
+  }
+  dout(20) << __func__ << " dirty " << ((float)dirty_micro / 1000000.0)
+	   << " full " << ((float)full_micro / 1000000.0)
+	   << dendl;
+
+  // flush mode
+  TierAgentState::flush_mode_t flush_mode = TierAgentState::FLUSH_MODE_IDLE;
+  uint64_t flush_target = pool.info.cache_target_dirty_ratio_micro;
+  uint64_t flush_slop = (float)flush_target * g_conf->osd_agent_slop;
+  if (restart || agent_state->flush_mode == TierAgentState::FLUSH_MODE_IDLE)
+    flush_target += flush_slop;
+  else
+    flush_target -= MIN(flush_target, flush_slop);
+
+  if (info.stats.stats_invalid) {
+    // idle; stats can't be trusted until we scrub.
+    dout(20) << __func__ << " stats invalid (post-split), idle" << dendl;
+  } else if (dirty_micro > flush_target) {
+    flush_mode = TierAgentState::FLUSH_MODE_ACTIVE;
+  }
+
+  // evict mode
+  TierAgentState::evict_mode_t evict_mode = TierAgentState::EVICT_MODE_IDLE;
+  unsigned evict_effort = 0;
+  uint64_t evict_target = pool.info.cache_target_full_ratio_micro;
+  uint64_t evict_slop = (float)evict_target * g_conf->osd_agent_slop;
+  if (restart || agent_state->evict_mode == TierAgentState::EVICT_MODE_IDLE)
+    evict_target += evict_slop;
+  else
+    evict_target -= MIN(evict_target, evict_slop);
+
+  if (info.stats.stats_invalid) {
+    // idle; stats can't be trusted until we scrub.
+  } else if (full_micro > 1000000) {
+    // evict anything clean
+    evict_mode = TierAgentState::EVICT_MODE_FULL;
+    evict_effort = 1000000;
+  } else if (full_micro > evict_target) {
+    // set effort in [0..1] range based on where we are between
+    evict_mode = TierAgentState::EVICT_MODE_SOME;
+    uint64_t over = full_micro - evict_target;
+    uint64_t span;
+    if (evict_target >= 1000000)
+      span = 1;
+    else
+      span = 1000000 - evict_target;
+    evict_effort = MAX(over * 1000000 / span,
+		       (unsigned)(1000000.0 * g_conf->osd_agent_min_evict_effort));
+
+    // quantize effort to avoid too much reordering in the agent_queue.
+    uint64_t inc = g_conf->osd_agent_quantize_effort * 1000000;
+    assert(inc > 0);
+    uint64_t was = evict_effort;
+    evict_effort -= evict_effort % inc;
+    if (evict_effort < inc)
+      evict_effort = inc;
+    assert(evict_effort >= inc && evict_effort <= 1000000);
+    dout(30) << __func__ << " evict_effort " << was << " quantized by " << inc << " to " << evict_effort << dendl;
+  }
+
+  bool old_idle = agent_state->is_idle();
+  if (flush_mode != agent_state->flush_mode) {
+    dout(5) << __func__ << " flush_mode "
+	    << TierAgentState::get_flush_mode_name(agent_state->flush_mode)
+	    << " -> "
+	    << TierAgentState::get_flush_mode_name(flush_mode)
+	    << dendl;
+    agent_state->flush_mode = flush_mode;
+  }
+  if (evict_mode != agent_state->evict_mode) {
+    dout(5) << __func__ << " evict_mode "
+	    << TierAgentState::get_evict_mode_name(agent_state->evict_mode)
+	    << " -> "
+	    << TierAgentState::get_evict_mode_name(evict_mode)
+	    << dendl;
+    if (agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL &&
+	is_active()) {
+      requeue_ops(waiting_for_cache_not_full);
+      requeue_ops(waiting_for_active);
+    }
+    agent_state->evict_mode = evict_mode;
+  }
+  uint64_t old_effort = agent_state->evict_effort;
+  if (evict_effort != agent_state->evict_effort) {
+    dout(5) << __func__ << " evict_effort "
+	    << ((float)agent_state->evict_effort / 1000000.0)
+	    << " -> "
+	    << ((float)evict_effort / 1000000.0)
+	    << dendl;
+    agent_state->evict_effort = evict_effort;
+  }
+
+  // NOTE: we are using evict_effort as a proxy for *all* agent effort
+  // (including flush).  This is probably fine (they should be
+  // correlated) but it is not precisely correct.
+  if (agent_state->is_idle()) {
+    if (!restart && !old_idle) {
+      osd->agent_disable_pg(this, old_effort);
+    }
+  } else {
+    if (restart || old_idle) {
+      osd->agent_enable_pg(this, agent_state->evict_effort);
+    } else if (old_effort != agent_state->evict_effort) {
+      osd->agent_adjust_pg(this, old_effort, agent_state->evict_effort);
+    }
+  }
+}
+
+void ReplicatedPG::agent_estimate_atime_temp(const hobject_t& oid,
+					     int *atime, int *temp)
+{
+  assert(hit_set);
+  *atime = -1;
+  if (temp)
+    *temp = 0;
+  if (hit_set->contains(oid)) {
+    *atime = 0;
+    if (temp)
+      ++(*temp);
+    else
+      return;
+  }
+  time_t now = ceph_clock_now(NULL).sec();
+  for (map::iterator p = agent_state->hit_set_map.begin();
+       p != agent_state->hit_set_map.end();
+       ++p) {
+    if (p->second->contains(oid)) {
+      if (*atime < 0)
+	*atime = now - p->first;
+      if (temp)
+	++(*temp);
+      else
+	return;
+    }
+  }
+}
+
+
+// ==========================================================================================
+// SCRUB
+
+
+bool ReplicatedPG::_range_available_for_scrub(
+  const hobject_t &begin, const hobject_t &end)
+{
+  pair next;
+  next.second = object_contexts.lookup(begin);
+  next.first = begin;
+  bool more = true;
+  while (more && next.first < end) {
+    if (next.second && next.second->is_blocked()) {
+      next.second->requeue_scrub_on_unblock = true;
+      dout(10) << __func__ << ": scrub delayed, "
+	       << next.first << " is blocked"
+	       << dendl;
+      return false;
+    }
+    more = object_contexts.get_next(next.first, &next);
+  }
+  return true;
+}
+
+void ReplicatedPG::_scrub(ScrubMap& scrubmap)
+{
+  dout(10) << "_scrub" << dendl;
+
+  coll_t c(info.pgid);
+  bool repair = state_test(PG_STATE_REPAIR);
+  bool deep_scrub = state_test(PG_STATE_DEEP_SCRUB);
+  const char *mode = (repair ? "repair": (deep_scrub ? "deep-scrub" : "scrub"));
+
+  // traverse in reverse order.
+  hobject_t head;
+  SnapSet snapset;
+  vector::reverse_iterator curclone;
+  hobject_t next_clone;
+
+  bufferlist last_data;
+
+  for (map::reverse_iterator p = scrubmap.objects.rbegin(); 
+       p != scrubmap.objects.rend(); 
+       ++p) {
+    const hobject_t& soid = p->first;
+    object_stat_sum_t stat;
+    if (soid.snap != CEPH_SNAPDIR)
+      stat.num_objects++;
+
+    if (soid.nspace == cct->_conf->osd_hit_set_namespace)
+      stat.num_objects_hit_set_archive++;
+
+    // new snapset?
+    if (soid.snap == CEPH_SNAPDIR ||
+	soid.snap == CEPH_NOSNAP) {
+      if (p->second.attrs.count(SS_ATTR) == 0) {
+	osd->clog.error() << mode << " " << info.pgid << " " << soid
+			  << " no '" << SS_ATTR << "' attr";
+        ++scrubber.shallow_errors;
+	continue;
+      }
+      bufferlist bl;
+      bl.push_back(p->second.attrs[SS_ATTR]);
+      bufferlist::iterator blp = bl.begin();
+      ::decode(snapset, blp);
+
+      // did we finish the last oid?
+      if (head != hobject_t() &&
+	  !pool.info.allow_incomplete_clones()) {
+	osd->clog.error() << mode << " " << info.pgid << " " << head
+			  << " missing clones";
+        ++scrubber.shallow_errors;
+      }
+      
+      // what will be next?
+      if (snapset.clones.empty())
+	head = hobject_t();  // no clones.
+      else {
+	curclone = snapset.clones.rbegin();
+	head = p->first;
+	next_clone = hobject_t();
+	dout(20) << "  snapset " << snapset << dendl;
+      }
+    }
+
+    // basic checks.
+    if (p->second.attrs.count(OI_ATTR) == 0) {
+      osd->clog.error() << mode << " " << info.pgid << " " << soid
+			<< " no '" << OI_ATTR << "' attr";
+      ++scrubber.shallow_errors;
+      continue;
+    }
+    bufferlist bv;
+    bv.push_back(p->second.attrs[OI_ATTR]);
+    object_info_t oi(bv);
+
+    if (pgbackend->be_get_ondisk_size(oi.size) != p->second.size) {
+      osd->clog.error() << mode << " " << info.pgid << " " << soid
+			<< " on disk size (" << p->second.size
+			<< ") does not match object info size ("
+			<< oi.size << ") ajusted for ondisk to ("
+			<< pgbackend->be_get_ondisk_size(oi.size)
+			<< ")";
+      ++scrubber.shallow_errors;
+    }
+
+    dout(20) << mode << "  " << soid << " " << oi << dendl;
+
+    if (soid.is_snap()) {
+      stat.num_bytes += snapset.get_clone_bytes(soid.snap);
+    } else {
+      stat.num_bytes += oi.size;
+    }
+
+    if (!soid.is_snapdir()) {
+      if (oi.is_dirty())
+	++stat.num_objects_dirty;
+      if (oi.is_whiteout())
+	++stat.num_whiteouts;
+      if (oi.is_omap())
+	++stat.num_objects_omap;
+    }
+
+    //bufferlist data;
+    //osd->store->read(c, poid, 0, 0, data);
+    //assert(data.length() == p->size);
+    //
+
+    if (!next_clone.is_min() && next_clone != soid &&
+	pool.info.allow_incomplete_clones()) {
+      // it is okay to be missing one or more clones in a cache tier.
+      // skip higher-numbered clones in the list.
+      while (curclone != snapset.clones.rend() &&
+	     soid.snap < *curclone)
+	++curclone;
+      if (curclone != snapset.clones.rend() &&
+	  soid.snap == *curclone) {
+	dout(20) << __func__ << " skipped some clones in cache tier" << dendl;
+	next_clone.snap = *curclone;
+      }
+      if (curclone == snapset.clones.rend() ||
+	  soid.snap == CEPH_NOSNAP) {
+	dout(20) << __func__ << " skipped remaining clones in cache tier"
+		 << dendl;
+	next_clone = hobject_t();
+	head = hobject_t();
+      }
+    }
+    if (!next_clone.is_min() && next_clone != soid) {
+      osd->clog.error() << mode << " " << info.pgid << " " << soid
+			<< " expected clone " << next_clone;
+      ++scrubber.shallow_errors;
+    }
+
+    if (soid.snap == CEPH_NOSNAP || soid.snap == CEPH_SNAPDIR) {
+      if (soid.snap == CEPH_NOSNAP && !snapset.head_exists) {
+	osd->clog.error() << mode << " " << info.pgid << " " << soid
+			  << " snapset.head_exists=false, but head exists";
+        ++scrubber.shallow_errors;
+      }
+      if (soid.snap == CEPH_SNAPDIR && snapset.head_exists) {
+	osd->clog.error() << mode << " " << info.pgid << " " << soid
+			  << " snapset.head_exists=true, but snapdir exists";
+        ++scrubber.shallow_errors;
+      }
+      if (curclone == snapset.clones.rend()) {
+	next_clone = hobject_t();
+      } else {
+	next_clone = soid;
+	next_clone.snap = *curclone;
+      }
+    } else if (soid.snap) {
+      // it's a clone
+      stat.num_object_clones++;
+      
+      if (head == hobject_t()) {
+	osd->clog.error() << mode << " " << info.pgid << " " << soid
+			  << " found clone without head";
+	++scrubber.shallow_errors;
+	continue;
+      }
+
+      if (soid.snap != *curclone) {
+	continue; // we warn above.  we could do better here...
+      }
+
+      if (oi.size != snapset.clone_size[*curclone]) {
+	osd->clog.error() << mode << " " << info.pgid << " " << soid
+			  << " size " << oi.size << " != clone_size "
+			  << snapset.clone_size[*curclone];
+	++scrubber.shallow_errors;
+      }
+
+      // verify overlap?
+      // ...
+
+      // what's next?
+      if (curclone != snapset.clones.rend()) {
+	++curclone;
+      }
+      if (curclone == snapset.clones.rend()) {
+	head = hobject_t();
+	next_clone = hobject_t();
+      } else {
+	next_clone.snap = *curclone;
+      }
+
+    } else {
+      // it's unversioned.
+      next_clone = hobject_t();
+    }
+
+    string cat; // fixme
+    scrub_cstat.add(stat, cat);
+  }
+
+  if (!next_clone.is_min() &&
+      !pool.info.allow_incomplete_clones()) {
+    osd->clog.error() << mode << " " << info.pgid
+		      << " expected clone " << next_clone;
+    ++scrubber.shallow_errors;
+  }
+  
+  dout(10) << "_scrub (" << mode << ") finish" << dendl;
+}
+
+void ReplicatedPG::_scrub_clear_state()
+{
+  scrub_cstat = object_stat_collection_t();
+}
+
+void ReplicatedPG::_scrub_finish()
+{
+  bool repair = state_test(PG_STATE_REPAIR);
+  bool deep_scrub = state_test(PG_STATE_DEEP_SCRUB);
+  const char *mode = (repair ? "repair": (deep_scrub ? "deep-scrub" : "scrub"));
+
+  if (info.stats.stats_invalid) {
+    info.stats.stats = scrub_cstat;
+    info.stats.stats_invalid = false;
+
+    if (agent_state)
+      agent_choose_mode();
+  }
+
+  dout(10) << mode << " got "
+	   << scrub_cstat.sum.num_objects << "/" << info.stats.stats.sum.num_objects << " objects, "
+	   << scrub_cstat.sum.num_object_clones << "/" << info.stats.stats.sum.num_object_clones << " clones, "
+	   << scrub_cstat.sum.num_objects_dirty << "/" << info.stats.stats.sum.num_objects_dirty << " dirty, "
+	   << scrub_cstat.sum.num_objects_omap << "/" << info.stats.stats.sum.num_objects_omap << " omap, "
+	   << scrub_cstat.sum.num_objects_hit_set_archive << "/" << info.stats.stats.sum.num_objects_hit_set_archive << " hit_set_archive, "
+	   << scrub_cstat.sum.num_bytes << "/" << info.stats.stats.sum.num_bytes << " bytes."
+	   << dendl;
+
+  if (scrub_cstat.sum.num_objects != info.stats.stats.sum.num_objects ||
+      scrub_cstat.sum.num_object_clones != info.stats.stats.sum.num_object_clones ||
+      (scrub_cstat.sum.num_objects_dirty != info.stats.stats.sum.num_objects_dirty &&
+       !info.stats.dirty_stats_invalid) ||
+      (scrub_cstat.sum.num_objects_omap != info.stats.stats.sum.num_objects_omap &&
+       !info.stats.omap_stats_invalid) ||
+      (scrub_cstat.sum.num_objects_hit_set_archive != info.stats.stats.sum.num_objects_hit_set_archive &&
+       !info.stats.hitset_stats_invalid) ||
+      scrub_cstat.sum.num_whiteouts != info.stats.stats.sum.num_whiteouts ||
+      scrub_cstat.sum.num_bytes != info.stats.stats.sum.num_bytes) {
+    osd->clog.error() << info.pgid << " " << mode
+		      << " stat mismatch, got "
+		      << scrub_cstat.sum.num_objects << "/" << info.stats.stats.sum.num_objects << " objects, "
+		      << scrub_cstat.sum.num_object_clones << "/" << info.stats.stats.sum.num_object_clones << " clones, "
+		      << scrub_cstat.sum.num_objects_dirty << "/" << info.stats.stats.sum.num_objects_dirty << " dirty, "
+		      << scrub_cstat.sum.num_objects_omap << "/" << info.stats.stats.sum.num_objects_omap << " omap, "
+		      << scrub_cstat.sum.num_objects_hit_set_archive << "/" << info.stats.stats.sum.num_objects_hit_set_archive << " hit_set_archive, "
+		      << scrub_cstat.sum.num_whiteouts << "/" << info.stats.stats.sum.num_whiteouts << " whiteouts, "
+		      << scrub_cstat.sum.num_bytes << "/" << info.stats.stats.sum.num_bytes << " bytes.\n";
+    ++scrubber.shallow_errors;
+
+    if (repair) {
+      ++scrubber.fixed;
+      info.stats.stats = scrub_cstat;
+      info.stats.dirty_stats_invalid = false;
+      info.stats.omap_stats_invalid = false;
+      info.stats.hitset_stats_invalid = false;
+      publish_stats_to_osd();
+      share_pg_info();
+    }
+  }
+}
+
+/*---SnapTrimmer Logging---*/
+#undef dout_prefix
+#define dout_prefix *_dout << pg->gen_prefix() 
+
+ReplicatedPG::SnapTrimmer::~SnapTrimmer()
+{
+  while (!repops.empty()) {
+    (*repops.begin())->put();
+    repops.erase(repops.begin());
+  }
+}
+
+void ReplicatedPG::SnapTrimmer::log_enter(const char *state_name)
+{
+  dout(20) << "enter " << state_name << dendl;
+}
+
+void ReplicatedPG::SnapTrimmer::log_exit(const char *state_name, utime_t enter_time)
+{
+  dout(20) << "exit " << state_name << dendl;
+}
+
+/*---SnapTrimmer states---*/
+#undef dout_prefix
+#define dout_prefix (*_dout << context< SnapTrimmer >().pg->gen_prefix() \
+		     << "SnapTrimmer state<" << get_state_name() << ">: ")
+
+/* NotTrimming */
+ReplicatedPG::NotTrimming::NotTrimming(my_context ctx)
+  : my_base(ctx), 
+    NamedState(context< SnapTrimmer >().pg->cct, "NotTrimming")
+{
+  context< SnapTrimmer >().requeue = false;
+  context< SnapTrimmer >().log_enter(state_name);
+}
+
+void ReplicatedPG::NotTrimming::exit()
+{
+  context< SnapTrimmer >().requeue = true;
+  context< SnapTrimmer >().log_exit(state_name, enter_time);
+}
+
+boost::statechart::result ReplicatedPG::NotTrimming::react(const SnapTrim&)
+{
+  ReplicatedPG *pg = context< SnapTrimmer >().pg;
+  dout(10) << "NotTrimming react" << dendl;
+
+  if (!pg->is_primary() || !pg->is_active() || !pg->is_clean()) {
+    dout(10) << "NotTrimming not primary, active, clean" << dendl;
+    return discard_event();
+  } else if (pg->scrubber.active) {
+    dout(10) << "NotTrimming finalizing scrub" << dendl;
+    pg->queue_snap_trim();
+    return discard_event();
+  }
+
+  // Primary trimming
+  if (pg->snap_trimq.empty()) {
+    return discard_event();
+  } else {
+    context().snap_to_trim = pg->snap_trimq.range_start();
+    dout(10) << "NotTrimming: trimming "
+	     << pg->snap_trimq.range_start()
+	     << dendl;
+    post_event(SnapTrim());
+    return transit();
+  }
+}
+
+/* TrimmingObjects */
+ReplicatedPG::TrimmingObjects::TrimmingObjects(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< SnapTrimmer >().pg->cct, "Trimming/TrimmingObjects")
+{
+  context< SnapTrimmer >().log_enter(state_name);
+}
+
+void ReplicatedPG::TrimmingObjects::exit()
+{
+  context< SnapTrimmer >().log_exit(state_name, enter_time);
+  // Clean up repops in case of reset
+  set &repops = context().repops;
+  for (set::iterator i = repops.begin();
+       i != repops.end();
+       repops.erase(i++)) {
+    (*i)->put();
+  }
+}
+
+boost::statechart::result ReplicatedPG::TrimmingObjects::react(const SnapTrim&)
+{
+  dout(10) << "TrimmingObjects react" << dendl;
+  ReplicatedPG *pg = context< SnapTrimmer >().pg;
+  snapid_t snap_to_trim = context().snap_to_trim;
+  set &repops = context().repops;
+
+  dout(10) << "TrimmingObjects: trimming snap " << snap_to_trim << dendl;
+
+  // Get next
+  hobject_t old_pos = pos;
+  int r = pg->snap_mapper.get_next_object_to_trim(snap_to_trim, &pos);
+  if (r != 0 && r != -ENOENT) {
+    derr << __func__ << ": get_next returned " << cpp_strerror(r) << dendl;
+    assert(0);
+  } else if (r == -ENOENT) {
+    // Done!
+    dout(10) << "TrimmingObjects: got ENOENT" << dendl;
+    post_event(SnapTrim());
+    return transit< WaitingOnReplicas >();
+  }
+
+  dout(10) << "TrimmingObjects react trimming " << pos << dendl;
+  RepGather *repop = pg->trim_object(pos);
+  if (!repop) {
+    dout(10) << __func__ << " could not get write lock on obj "
+	     << pos << dendl;
+    pos = old_pos;
+    return discard_event();
+  }
+  assert(repop);
+  repop->queue_snap_trimmer = true;
+
+  repops.insert(repop->get());
+  pg->simple_repop_submit(repop);
+  return discard_event();
+}
+/* WaitingOnReplicasObjects */
+ReplicatedPG::WaitingOnReplicas::WaitingOnReplicas(my_context ctx)
+  : my_base(ctx),
+    NamedState(context< SnapTrimmer >().pg->cct, "Trimming/WaitingOnReplicas")
+{
+  context< SnapTrimmer >().log_enter(state_name);
+  context< SnapTrimmer >().requeue = false;
+}
+
+void ReplicatedPG::WaitingOnReplicas::exit()
+{
+  context< SnapTrimmer >().log_exit(state_name, enter_time);
+
+  // Clean up repops in case of reset
+  set &repops = context().repops;
+  for (set::iterator i = repops.begin();
+       i != repops.end();
+       repops.erase(i++)) {
+    (*i)->put();
+  }
+}
+
+boost::statechart::result ReplicatedPG::WaitingOnReplicas::react(const SnapTrim&)
+{
+  // Have all the repops applied?
+  dout(10) << "Waiting on Replicas react" << dendl;
+  ReplicatedPG *pg = context< SnapTrimmer >().pg;
+  set &repops = context().repops;
+  for (set::iterator i = repops.begin();
+       i != repops.end();
+       repops.erase(i++)) {
+    if (!(*i)->all_applied) {
+      return discard_event();
+    } else {
+      (*i)->put();
+    }
+  }
+
+  snapid_t &sn = context().snap_to_trim;
+  dout(10) << "WaitingOnReplicas: adding snap " << sn << " to purged_snaps"
+	   << dendl;
+
+  pg->info.purged_snaps.insert(sn);
+  pg->snap_trimq.erase(sn);
+  dout(10) << "purged_snaps now " << pg->info.purged_snaps << ", snap_trimq now " 
+	   << pg->snap_trimq << dendl;
+  
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  pg->dirty_big_info = true;
+  pg->write_if_dirty(*t);
+  int tr = pg->osd->store->queue_transaction_and_cleanup(pg->osr.get(), t);
+  assert(tr == 0);
+
+  context().need_share_pg_info = true;
+
+  // Back to the start
+  post_event(SnapTrim());
+  return transit< NotTrimming >();
+}
+
+void ReplicatedPG::replace_cached_attrs(
+  OpContext *ctx,
+  ObjectContextRef obc,
+  const map &new_attrs)
+{
+  ctx->pending_attrs[obc].clear();
+  for (map::iterator i = obc->attr_cache.begin();
+       i != obc->attr_cache.end();
+       ++i) {
+    ctx->pending_attrs[obc][i->first] = boost::optional();
+  }
+  for (map::const_iterator i = new_attrs.begin();
+       i != new_attrs.end();
+       ++i) {
+    ctx->pending_attrs[obc][i->first] = i->second;
+  }
+}
+
+void ReplicatedPG::setattr_maybe_cache(
+  ObjectContextRef obc,
+  OpContext *op,
+  PGBackend::PGTransaction *t,
+  const string &key,
+  bufferlist &val)
+{
+  if (pool.info.require_rollback()) {
+    op->pending_attrs[obc][key] = val;
+  }
+  t->setattr(obc->obs.oi.soid, key, val);
+}
+
+void ReplicatedPG::rmattr_maybe_cache(
+  ObjectContextRef obc,
+  OpContext *op,
+  PGBackend::PGTransaction *t,
+  const string &key)
+{
+  if (pool.info.require_rollback()) {
+    op->pending_attrs[obc][key] = boost::optional();
+  }
+  t->rmattr(obc->obs.oi.soid, key);
+}
+
+int ReplicatedPG::getattr_maybe_cache(
+  ObjectContextRef obc,
+  const string &key,
+  bufferlist *val)
+{
+  if (pool.info.require_rollback()) {
+    map::iterator i = obc->attr_cache.find(key);
+    if (i != obc->attr_cache.end()) {
+      if (val)
+	*val = i->second;
+      return 0;
+    } else {
+      return -ENODATA;
+    }
+  }
+  return pgbackend->objects_get_attr(obc->obs.oi.soid, key, val);
+}
+
+int ReplicatedPG::getattrs_maybe_cache(
+  ObjectContextRef obc,
+  map *out,
+  bool user_only)
+{
+  int r = 0;
+  if (pool.info.require_rollback()) {
+    if (out)
+      *out = obc->attr_cache;
+  } else {
+    r = pgbackend->objects_get_attrs(obc->obs.oi.soid, out);
+  }
+  if (out && user_only) {
+    map tmp;
+    for (map::iterator i = out->begin();
+	 i != out->end();
+	 ++i) {
+      if (i->first.size() > 1 && i->first[0] == '_')
+	tmp[i->first.substr(1, i->first.size())].claim(i->second);
+    }
+    tmp.swap(*out);
+  }
+  return r;
+}
+
+void intrusive_ptr_add_ref(ReplicatedPG *pg) { pg->get("intptr"); }
+void intrusive_ptr_release(ReplicatedPG *pg) { pg->put("intptr"); }
+
+#ifdef PG_DEBUG_REFS
+uint64_t get_with_id(ReplicatedPG *pg) { return pg->get_with_id(); }
+void put_with_id(ReplicatedPG *pg, uint64_t id) { return pg->put_with_id(id); }
+#endif
+
+void intrusive_ptr_add_ref(ReplicatedPG::RepGather *repop) { repop->get(); }
+void intrusive_ptr_release(ReplicatedPG::RepGather *repop) { repop->put(); }
diff --git a/ceph/src/osd/ReplicatedPG.h b/ceph/src/osd/ReplicatedPG.h
new file mode 100644
index 00000000..9ef131c1
--- /dev/null
+++ b/ceph/src/osd/ReplicatedPG.h
@@ -0,0 +1,1453 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_REPLICATEDPG_H
+#define CEPH_REPLICATEDPG_H
+
+#include 
+#include 
+
+#include "include/assert.h" 
+#include "common/cmdparse.h"
+
+#include "HitSet.h"
+#include "OSD.h"
+#include "PG.h"
+#include "Watch.h"
+#include "OpRequest.h"
+#include "TierAgentState.h"
+
+#include "messages/MOSDOp.h"
+#include "messages/MOSDOpReply.h"
+#include "messages/MOSDSubOp.h"
+
+#include "common/sharedptr_registry.hpp"
+
+#include "PGBackend.h"
+#include "ReplicatedBackend.h"
+#include "ECBackend.h"
+
+class MOSDSubOpReply;
+
+class CopyFromCallback;
+class PromoteCallback;
+
+class ReplicatedPG;
+void intrusive_ptr_add_ref(ReplicatedPG *pg);
+void intrusive_ptr_release(ReplicatedPG *pg);
+uint64_t get_with_id(ReplicatedPG *pg);
+void put_with_id(ReplicatedPG *pg, uint64_t id);
+
+#ifdef PG_DEBUG_REFS
+  typedef TrackedIntPtr ReplicatedPGRef;
+#else
+  typedef boost::intrusive_ptr ReplicatedPGRef;
+#endif
+
+class PGLSFilter {
+protected:
+  string xattr;
+public:
+  PGLSFilter();
+  virtual ~PGLSFilter();
+  virtual bool filter(bufferlist& xattr_data, bufferlist& outdata) = 0;
+  virtual string& get_xattr() { return xattr; }
+};
+
+class PGLSPlainFilter : public PGLSFilter {
+  string val;
+public:
+  PGLSPlainFilter(bufferlist::iterator& params) {
+    ::decode(xattr, params);
+    ::decode(val, params);
+  }
+  virtual ~PGLSPlainFilter() {}
+  virtual bool filter(bufferlist& xattr_data, bufferlist& outdata);
+};
+
+class PGLSParentFilter : public PGLSFilter {
+  inodeno_t parent_ino;
+public:
+  PGLSParentFilter(bufferlist::iterator& params) {
+    xattr = "_parent";
+    ::decode(parent_ino, params);
+    generic_dout(0) << "parent_ino=" << parent_ino << dendl;
+  }
+  virtual ~PGLSParentFilter() {}
+  virtual bool filter(bufferlist& xattr_data, bufferlist& outdata);
+};
+
+class ReplicatedPG : public PG, public PGBackend::Listener {
+  friend class OSD;
+  friend class Watch;
+
+public:
+
+  /*
+   * state associated with a copy operation
+   */
+  struct OpContext;
+  class CopyCallback;
+
+  /**
+   * CopyResults stores the object metadata of interest to a copy initiator.
+   */
+  struct CopyResults {
+    utime_t mtime; ///< the copy source's mtime
+    uint64_t object_size; ///< the copied object's size
+    bool started_temp_obj; ///< true if the callback needs to delete temp object
+    hobject_t temp_oid;    ///< temp object (if any)
+    /**
+     * Final transaction; if non-empty the callback must execute it before any
+     * other accesses to the object (in order to complete the copy).
+     */
+    PGBackend::PGTransaction *final_tx;
+    string category; ///< The copy source's category
+    version_t user_version; ///< The copy source's user version
+    bool should_requeue;  ///< op should be requeued on cancel
+    vector snaps;  ///< src's snaps (if clone)
+    snapid_t snap_seq;       ///< src's snap_seq (if head)
+    librados::snap_set_t snapset; ///< src snapset (if head)
+    bool mirror_snapset;
+    map attrs; ///< src user attrs
+    bool has_omap;
+    CopyResults() : object_size(0), started_temp_obj(false),
+		    final_tx(NULL), user_version(0), 
+		    should_requeue(false), mirror_snapset(false),
+		    has_omap(false) {}
+  };
+
+  struct CopyOp {
+    CopyCallback *cb;
+    ObjectContextRef obc;
+    hobject_t src;
+    object_locator_t oloc;
+    unsigned flags;
+    bool mirror_snapset;
+
+    CopyResults results;
+
+    ceph_tid_t objecter_tid;
+    ceph_tid_t objecter_tid2;
+
+    object_copy_cursor_t cursor;
+    map attrs;
+    bufferlist data;
+    bufferlist omap_header;
+    map omap;
+    int rval;
+
+    object_copy_cursor_t temp_cursor;
+
+    CopyOp(CopyCallback *cb_, ObjectContextRef _obc, hobject_t s,
+	   object_locator_t l,
+           version_t v,
+	   unsigned f,
+	   bool ms)
+      : cb(cb_), obc(_obc), src(s), oloc(l), flags(f),
+	mirror_snapset(ms),
+	objecter_tid(0),
+	objecter_tid2(0),
+	rval(-1)
+    {
+      results.user_version = v;
+      results.mirror_snapset = mirror_snapset;
+    }
+  };
+  typedef boost::shared_ptr CopyOpRef;
+
+  /**
+   * The CopyCallback class defines an interface for completions to the
+   * copy_start code. Users of the copy infrastructure must implement
+   * one and give an instance of the class to start_copy.
+   *
+   * The implementer is responsible for making sure that the CopyCallback
+   * can associate itself with the correct copy operation.
+   */
+  typedef boost::tuple CopyCallbackResults;
+  class CopyCallback : public GenContext {
+  protected:
+    CopyCallback() {}
+    /**
+     * results.get<0>() is the return code: 0 for success; -ECANCELLED if
+     * the operation was cancelled by the local OSD; -errno for other issues.
+     * results.get<1>() is a pointer to a CopyResults object, which you are
+     * responsible for deleting.
+     */
+    virtual void finish(CopyCallbackResults results_) = 0;
+
+  public:
+    /// Provide the final size of the copied object to the CopyCallback
+    virtual ~CopyCallback() {};
+  };
+
+  friend class CopyFromCallback;
+  friend class PromoteCallback;
+
+  struct FlushOp {
+    ObjectContextRef obc;       ///< obc we are flushing
+    OpRequestRef op;            ///< initiating op
+    list dup_ops; ///< bandwagon jumpers
+    version_t flushed_version;  ///< user version we are flushing
+    ceph_tid_t objecter_tid;    ///< copy-from request tid
+    int rval;                   ///< copy-from result
+    bool blocking;              ///< whether we are blocking updates
+    bool removal;               ///< we are removing the backend object
+    Context *on_flush;          ///< callback, may be null
+
+    FlushOp()
+      : objecter_tid(0), rval(0),
+	blocking(false), removal(false),
+	on_flush(NULL) {}
+    ~FlushOp() { assert(!on_flush); }
+  };
+  typedef boost::shared_ptr FlushOpRef;
+
+  boost::scoped_ptr pgbackend;
+  PGBackend *get_pgbackend() {
+    return pgbackend.get();
+  }
+
+  /// Listener methods
+  void on_local_recover_start(
+    const hobject_t &oid,
+    ObjectStore::Transaction *t);
+  void on_local_recover(
+    const hobject_t &oid,
+    const object_stat_sum_t &stat_diff,
+    const ObjectRecoveryInfo &recovery_info,
+    ObjectContextRef obc,
+    ObjectStore::Transaction *t
+    );
+  void on_peer_recover(
+    pg_shard_t peer,
+    const hobject_t &oid,
+    const ObjectRecoveryInfo &recovery_info,
+    const object_stat_sum_t &stat
+    );
+  void begin_peer_recover(
+    pg_shard_t peer,
+    const hobject_t oid);
+  void on_global_recover(
+    const hobject_t &oid);
+  void failed_push(pg_shard_t from, const hobject_t &soid);
+  void cancel_pull(const hobject_t &soid);
+
+  template 
+  class BlessedGenContext : public GenContext {
+    ReplicatedPGRef pg;
+    GenContext *c;
+    epoch_t e;
+  public:
+    BlessedGenContext(ReplicatedPG *pg, GenContext *c, epoch_t e)
+      : pg(pg), c(c), e(e) {}
+    void finish(T t) {
+      pg->lock();
+      if (pg->pg_has_reset_since(e))
+	delete c;
+      else
+	c->complete(t);
+      pg->unlock();
+    }
+  };
+  class BlessedContext : public Context {
+    ReplicatedPGRef pg;
+    Context *c;
+    epoch_t e;
+  public:
+    BlessedContext(ReplicatedPG *pg, Context *c, epoch_t e)
+      : pg(pg), c(c), e(e) {}
+    void finish(int r) {
+      pg->lock();
+      if (pg->pg_has_reset_since(e))
+	delete c;
+      else
+	c->complete(r);
+      pg->unlock();
+    }
+  };
+  Context *bless_context(Context *c) {
+    return new BlessedContext(this, c, get_osdmap()->get_epoch());
+  }
+  GenContext *bless_gencontext(
+    GenContext *c) {
+    return new BlessedGenContext(
+      this, c, get_osdmap()->get_epoch());
+  }
+    
+  void send_message(int to_osd, Message *m) {
+    osd->send_message_osd_cluster(to_osd, m, get_osdmap()->get_epoch());
+  }
+  void queue_transaction(ObjectStore::Transaction *t, OpRequestRef op) {
+    list tls;
+    tls.push_back(t);
+    osd->store->queue_transaction(osr.get(), t, 0, 0, 0, op);
+  }
+  epoch_t get_epoch() const {
+    return get_osdmap()->get_epoch();
+  }
+  const set &get_actingbackfill_shards() const {
+    return actingbackfill;
+  }
+  const set &get_acting_shards() const {
+    return actingset;
+  }
+  const set &get_backfill_shards() const {
+    return backfill_targets;
+  }
+
+  std::string gen_dbg_prefix() const { return gen_prefix(); }
+  
+  const map > &get_missing_loc_shards() const {
+    return missing_loc.get_missing_locs();
+  }
+  const map &get_shard_missing() const {
+    return peer_missing;
+  }
+  const map &get_shard_info() const {
+    return peer_info;
+  }
+  const pg_missing_t &get_local_missing() const {
+    return pg_log.get_missing();
+  }
+  const PGLog &get_log() const {
+    return pg_log;
+  }
+  bool pgb_is_primary() const {
+    return is_primary();
+  }
+  OSDMapRef pgb_get_osdmap() const {
+    return get_osdmap();
+  }
+  const pg_info_t &get_info() const {
+    return info;
+  }
+  const pg_pool_t &get_pool() const {
+    return pool.info;
+  }
+  ObjectContextRef get_obc(
+    const hobject_t &hoid,
+    map &attrs) {
+    return get_object_context(hoid, true, &attrs);
+  }
+  void log_operation(
+    vector &logv,
+    boost::optional &hset_history,
+    const eversion_t &trim_to,
+    const eversion_t &trim_rollback_to,
+    bool transaction_applied,
+    ObjectStore::Transaction *t) {
+    if (hset_history) {
+      info.hit_set = *hset_history;
+      dirty_info = true;
+    }
+    append_log(logv, trim_to, trim_rollback_to, *t, transaction_applied);
+  }
+
+  void op_applied(
+    const eversion_t &applied_version);
+
+  bool should_send_op(
+    pg_shard_t peer,
+    const hobject_t &hoid) {
+    if (peer == get_primary())
+      return true;
+    assert(peer_info.count(peer));
+    bool should_send = hoid.pool != (int64_t)info.pgid.pool() ||
+      hoid <= MAX(last_backfill_started, peer_info[peer].last_backfill);
+    if (!should_send)
+      assert(is_backfill_targets(peer));
+    return should_send;
+  }
+  
+  void update_peer_last_complete_ondisk(
+    pg_shard_t fromosd,
+    eversion_t lcod) {
+    peer_last_complete_ondisk[fromosd] = lcod;
+  }
+
+  void update_last_complete_ondisk(
+    eversion_t lcod) {
+    last_complete_ondisk = lcod;
+  }
+
+  void update_stats(
+    const pg_stat_t &stat) {
+    info.stats = stat;
+  }
+
+  void schedule_work(
+    GenContext *c);
+
+  pg_shard_t whoami_shard() const {
+    return pg_whoami;
+  }
+  spg_t primary_spg_t() const {
+    return spg_t(info.pgid.pgid, primary.shard);
+  }
+  pg_shard_t primary_shard() const {
+    return primary;
+  }
+
+  void send_message_osd_cluster(
+    int peer, Message *m, epoch_t from_epoch);
+  void send_message_osd_cluster(
+    Message *m, Connection *con);
+  void send_message_osd_cluster(
+    Message *m, const ConnectionRef& con);
+  ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch);
+  entity_name_t get_cluster_msgr_name() {
+    return osd->get_cluster_msgr_name();
+  }
+
+  PerfCounters *get_logger();
+
+  ceph_tid_t get_tid() { return osd->get_tid(); }
+
+  LogClientTemp clog_error() { return osd->clog.error(); }
+
+  /*
+   * Capture all object state associated with an in-progress read or write.
+   */
+  struct OpContext {
+    OpRequestRef op;
+    osd_reqid_t reqid;
+    vector &ops;
+
+    const ObjectState *obs; // Old objectstate
+    const SnapSet *snapset; // Old snapset
+
+    ObjectState new_obs;  // resulting ObjectState
+    SnapSet new_snapset;  // resulting SnapSet (in case of a write)
+    //pg_stat_t new_stats;  // resulting Stats
+    object_stat_sum_t delta_stats;
+
+    bool modify;          // (force) modification (even if op_t is empty)
+    bool user_modify;     // user-visible modification
+    bool undirty;         // user explicitly un-dirtying this object
+
+    // side effects
+    list watch_connects;
+    list watch_disconnects;
+    list notifies;
+    struct NotifyAck {
+      boost::optional watch_cookie;
+      uint64_t notify_id;
+      NotifyAck(uint64_t notify_id) : notify_id(notify_id) {}
+      NotifyAck(uint64_t notify_id, uint64_t cookie)
+	: watch_cookie(cookie), notify_id(notify_id) {}
+    };
+    list notify_acks;
+    
+    uint64_t bytes_written, bytes_read;
+
+    utime_t mtime;
+    SnapContext snapc;           // writer snap context
+    eversion_t at_version;       // pg's current version pointer
+    version_t user_at_version;   // pg's current user version pointer
+
+    int current_osd_subop_num;
+
+    PGBackend::PGTransaction *op_t;
+    vector log;
+    boost::optional updated_hset_history;
+
+    interval_set modified_ranges;
+    ObjectContextRef obc;
+    map src_obc;
+    ObjectContextRef clone_obc;    // if we created a clone
+    ObjectContextRef snapset_obc;  // if we created/deleted a snapdir
+
+    int data_off;        // FIXME: we may want to kill this msgr hint off at some point!
+
+    MOSDOpReply *reply;
+
+    utime_t readable_stamp;  // when applied on all replicas
+    ReplicatedPG *pg;
+
+    int num_read;    ///< count read ops
+    int num_write;   ///< count update ops
+
+    CopyFromCallback *copy_cb;
+
+    hobject_t new_temp_oid, discard_temp_oid;  ///< temp objects we should start/stop tracking
+
+    // pending xattr updates
+    map > > pending_attrs;
+    void apply_pending_attrs() {
+      for (map > >::iterator i =
+	     pending_attrs.begin();
+	   i != pending_attrs.end();
+	   ++i) {
+	if (i->first->obs.exists) {
+	  for (map >::iterator j =
+		 i->second.begin();
+	       j != i->second.end();
+	       ++j) {
+	    if (j->second)
+	      i->first->attr_cache[j->first] = j->second.get();
+	    else
+	      i->first->attr_cache.erase(j->first);
+	  }
+	} else {
+	  i->first->attr_cache.clear();
+	}
+      }
+      pending_attrs.clear();
+    }
+
+    // pending async reads  -> 
+    list,
+	      pair > > pending_async_reads;
+    int async_read_result;
+    unsigned inflightreads;
+    friend struct OnReadComplete;
+    void start_async_reads(ReplicatedPG *pg);
+    void finish_read(ReplicatedPG *pg);
+    bool async_reads_complete() {
+      return inflightreads == 0;
+    }
+
+    ObjectModDesc mod_desc;
+
+    enum { W_LOCK, R_LOCK, NONE } lock_to_release;
+
+    Context *on_finish;
+
+    OpContext(const OpContext& other);
+    const OpContext& operator=(const OpContext& other);
+
+    bool release_snapset_obc;
+
+    OpContext(OpRequestRef _op, osd_reqid_t _reqid, vector& _ops,
+	      ObjectState *_obs, SnapSetContext *_ssc,
+	      ReplicatedPG *_pg) :
+      op(_op), reqid(_reqid), ops(_ops), obs(_obs), snapset(0),
+      new_obs(_obs->oi, _obs->exists),
+      modify(false), user_modify(false), undirty(false),
+      bytes_written(0), bytes_read(0), user_at_version(0),
+      current_osd_subop_num(0),
+      op_t(NULL),
+      data_off(0), reply(NULL), pg(_pg),
+      num_read(0),
+      num_write(0),
+      copy_cb(NULL),
+      async_read_result(0),
+      inflightreads(0),
+      lock_to_release(NONE),
+      on_finish(NULL),
+      release_snapset_obc(false) {
+      if (_ssc) {
+	new_snapset = _ssc->snapset;
+	snapset = &_ssc->snapset;
+      }
+    }
+    void reset_obs(ObjectContextRef obc) {
+      new_obs = ObjectState(obc->obs.oi, obc->obs.exists);
+      if (obc->ssc) {
+	new_snapset = obc->ssc->snapset;
+	snapset = &obc->ssc->snapset;
+      }
+    }
+    ~OpContext() {
+      assert(!op_t);
+      assert(lock_to_release == NONE);
+      if (reply)
+	reply->put();
+      for (list,
+		     pair > >::iterator i =
+	     pending_async_reads.begin();
+	   i != pending_async_reads.end();
+	   pending_async_reads.erase(i++)) {
+	delete i->second.second;
+      }
+      assert(on_finish == NULL);
+    }
+    void finish(int r) {
+      if (on_finish) {
+	on_finish->complete(r);
+	on_finish = NULL;
+      }
+    }
+  };
+  friend struct OpContext;
+
+  /*
+   * State on the PG primary associated with the replicated mutation
+   */
+  class RepGather {
+  public:
+    xlist::item queue_item;
+    int nref;
+
+    eversion_t v;
+
+    OpContext *ctx;
+    ObjectContextRef obc;
+    map src_obc;
+
+    ceph_tid_t rep_tid;
+
+    bool rep_aborted, rep_done;
+
+    bool all_applied;
+    bool all_committed;
+    bool sent_ack;
+    //bool sent_nvram;
+    bool sent_disk;
+    
+    utime_t   start;
+    
+    eversion_t          pg_local_last_complete;
+
+    bool queue_snap_trimmer;
+
+    Context *on_applied;
+    
+    RepGather(OpContext *c, ObjectContextRef pi, ceph_tid_t rt,
+	      eversion_t lc) :
+      queue_item(this),
+      nref(1),
+      ctx(c), obc(pi),
+      rep_tid(rt), 
+      rep_aborted(false), rep_done(false),
+      all_applied(false), all_committed(false), sent_ack(false),
+      //sent_nvram(false),
+      sent_disk(false),
+      pg_local_last_complete(lc),
+      queue_snap_trimmer(false),
+      on_applied(NULL) { }
+
+    RepGather *get() {
+      nref++;
+      return this;
+    }
+    void put() {
+      assert(nref > 0);
+      if (--nref == 0) {
+	delete ctx; // must already be unlocked
+	assert(on_applied == NULL);
+	delete this;
+	//generic_dout(0) << "deleting " << this << dendl;
+      }
+    }
+  };
+
+
+protected:
+
+  /**
+   * Grabs locks for OpContext, should be cleaned up in close_op_ctx
+   *
+   * @param ctx [in,out] ctx to get locks for
+   * @return true on success, false if we are queued
+   */
+  bool get_rw_locks(OpContext *ctx) {
+    /* If snapset_obc, !obc->obs->exists and we will always take the
+     * snapdir lock *before* the head lock.  Since all callers will do
+     * this (read or write) if we get the first we will be guaranteed
+     * to get the second.
+     */
+    if (ctx->snapset_obc) {
+      assert(!ctx->obc->obs.exists);
+      if (ctx->op->may_write() || ctx->op->may_cache()) {
+	if (ctx->snapset_obc->get_write(ctx->op)) {
+	  ctx->release_snapset_obc = true;
+	  ctx->lock_to_release = OpContext::W_LOCK;
+	} else {
+	  return false;
+	}
+      } else {
+	assert(ctx->op->may_read());
+	if (ctx->snapset_obc->get_read(ctx->op)) {
+	  ctx->release_snapset_obc = true;
+	  ctx->lock_to_release = OpContext::R_LOCK;
+	} else {
+	  return false;
+	}
+      }
+    }
+    if (ctx->op->may_write() || ctx->op->may_cache()) {
+      if (ctx->obc->get_write(ctx->op)) {
+	ctx->lock_to_release = OpContext::W_LOCK;
+	return true;
+      } else {
+	assert(!ctx->snapset_obc);
+	return false;
+      }
+    } else {
+      assert(ctx->op->may_read());
+      if (ctx->obc->get_read(ctx->op)) {
+	ctx->lock_to_release = OpContext::R_LOCK;
+	return true;
+      } else {
+	assert(!ctx->snapset_obc);
+	return false;
+      }
+    }
+  }
+
+  /**
+   * Cleans up OpContext
+   *
+   * @param ctx [in] ctx to clean up
+   */
+  void close_op_ctx(OpContext *ctx, int r) {
+    release_op_ctx_locks(ctx);
+    delete ctx->op_t;
+    ctx->op_t = NULL;
+    ctx->finish(r);
+    delete ctx;
+  }
+
+  /**
+   * Releases ctx locks
+   *
+   * @param ctx [in] ctx to clean up
+   */
+  void release_op_ctx_locks(OpContext *ctx) {
+    list to_req;
+    bool requeue_recovery = false;
+    bool requeue_recovery_clone = false;
+    bool requeue_recovery_snapset = false;
+    bool requeue_snaptrimmer = false;
+    bool requeue_snaptrimmer_clone = false;
+    bool requeue_snaptrimmer_snapset = false;
+    switch (ctx->lock_to_release) {
+    case OpContext::W_LOCK:
+      if (ctx->snapset_obc && ctx->release_snapset_obc) {
+	ctx->snapset_obc->put_write(
+	  &to_req,
+	  &requeue_recovery_snapset,
+	  &requeue_snaptrimmer_snapset);
+	ctx->release_snapset_obc = false;
+      }
+      ctx->obc->put_write(
+	&to_req,
+	&requeue_recovery,
+	&requeue_snaptrimmer);
+      if (ctx->clone_obc)
+	ctx->clone_obc->put_write(
+	  &to_req,
+	  &requeue_recovery_clone,
+	  &requeue_snaptrimmer_clone);
+      break;
+    case OpContext::R_LOCK:
+      if (ctx->snapset_obc && ctx->release_snapset_obc) {
+	ctx->snapset_obc->put_read(&to_req);
+	ctx->release_snapset_obc = false;
+      }
+      ctx->obc->put_read(&to_req);
+      break;
+    case OpContext::NONE:
+      break;
+    default:
+      assert(0);
+    };
+    assert(ctx->release_snapset_obc == false);
+    ctx->lock_to_release = OpContext::NONE;
+    if (requeue_recovery || requeue_recovery_clone || requeue_recovery_snapset)
+      osd->recovery_wq.queue(this);
+    if (requeue_snaptrimmer ||
+	requeue_snaptrimmer_clone ||
+	requeue_snaptrimmer_snapset)
+      queue_snap_trim();
+    requeue_ops(to_req);
+  }
+
+  // replica ops
+  // [primary|tail]
+  xlist repop_queue;
+  map repop_map;
+
+  friend class C_OSD_RepopApplied;
+  friend class C_OSD_RepopCommit;
+  void repop_all_applied(RepGather *repop);
+  void repop_all_committed(RepGather *repop);
+  void eval_repop(RepGather*);
+  void issue_repop(RepGather *repop, utime_t now);
+  RepGather *new_repop(OpContext *ctx, ObjectContextRef obc, ceph_tid_t rep_tid);
+  void remove_repop(RepGather *repop);
+
+  RepGather *simple_repop_create(ObjectContextRef obc);
+  void simple_repop_submit(RepGather *repop);
+
+  // hot/cold tracking
+  HitSetRef hit_set;        ///< currently accumulating HitSet
+  utime_t hit_set_start_stamp;    ///< time the current HitSet started recording
+
+  map hit_set_flushing; ///< currently being written, not yet readable
+
+  void hit_set_clear();     ///< discard any HitSet state
+  void hit_set_setup();     ///< initialize HitSet state
+  void hit_set_create();    ///< create a new HitSet
+  void hit_set_persist();   ///< persist hit info
+  bool hit_set_apply_log(); ///< apply log entries to update in-memory HitSet
+  void hit_set_trim(RepGather *repop, unsigned max); ///< discard old HitSets
+
+  hobject_t get_hit_set_current_object(utime_t stamp);
+  hobject_t get_hit_set_archive_object(utime_t start, utime_t end);
+
+  // agent
+  boost::scoped_ptr agent_state;
+
+  friend class C_AgentFlushStartStop;
+  friend class C_HitSetFlushing;
+
+  void agent_setup();       ///< initialize agent state
+  bool agent_work(int max); ///< entry point to do some agent work
+  bool agent_maybe_flush(ObjectContextRef& obc);  ///< maybe flush
+  bool agent_maybe_evict(ObjectContextRef& obc);  ///< maybe evict
+
+  void agent_load_hit_sets();  ///< load HitSets, if needed
+
+  /// estimate object atime and temperature
+  ///
+  /// @param oid [in] object name
+  /// @param atime [out] seconds since last access (lower bound)
+  /// @param temperature [out] relative temperature (# hitset bins we appear in)
+  void agent_estimate_atime_temp(const hobject_t& oid,
+				 int *atime, int *temperature);
+
+  /// stop the agent
+  void agent_stop();
+  void agent_delay();
+
+  /// clear agent state
+  void agent_clear();
+
+  void agent_choose_mode(bool restart = false);  ///< choose (new) agent mode(s)
+  void agent_choose_mode_restart();
+
+  /// true if we can send an ondisk/commit for v
+  bool already_complete(eversion_t v) {
+    for (xlist::iterator i = repop_queue.begin();
+	 !i.end();
+	 ++i) {
+      // skip copy from temp object ops
+      if ((*i)->v == eversion_t())
+	continue;
+      if ((*i)->v > v)
+        break;
+      if (!(*i)->all_committed)
+	return false;
+    }
+    return true;
+  }
+  /// true if we can send an ack for v
+  bool already_ack(eversion_t v) {
+    for (xlist::iterator i = repop_queue.begin();
+	 !i.end();
+	 ++i) {
+      // skip copy from temp object ops
+      if ((*i)->v == eversion_t())
+	continue;
+      if ((*i)->v > v)
+        break;
+      if (!(*i)->all_applied)
+	return false;
+    }
+    return true;
+  }
+
+  friend struct C_OnPushCommit;
+
+  // projected object info
+  SharedPtrRegistry object_contexts;
+  // map from oid.snapdir() to SnapSetContext *
+  map snapset_contexts;
+  Mutex snapset_contexts_lock;
+
+  // debug order that client ops are applied
+  map > debug_op_order;
+
+  void populate_obc_watchers(ObjectContextRef obc);
+  void check_blacklisted_obc_watchers(ObjectContextRef obc);
+  void check_blacklisted_watchers();
+  void get_watchers(list &pg_watchers);
+  void get_obc_watchers(ObjectContextRef obc, list &pg_watchers);
+public:
+  void handle_watch_timeout(WatchRef watch);
+protected:
+
+  ObjectContextRef create_object_context(const object_info_t& oi, SnapSetContext *ssc);
+  ObjectContextRef get_object_context(
+    const hobject_t& soid,
+    bool can_create,
+    map *attrs = 0
+    );
+
+  void context_registry_on_change();
+  void object_context_destructor_callback(ObjectContext *obc);
+  struct C_PG_ObjectContext : public Context {
+    ReplicatedPGRef pg;
+    ObjectContext *obc;
+    C_PG_ObjectContext(ReplicatedPG *p, ObjectContext *o) :
+      pg(p), obc(o) {}
+    void finish(int r) {
+      pg->object_context_destructor_callback(obc);
+     }
+  };
+
+  int find_object_context(const hobject_t& oid,
+			  ObjectContextRef *pobc,
+			  bool can_create,
+			  bool map_snapid_to_clone=false,
+			  hobject_t *missing_oid=NULL);
+
+  void add_object_context_to_pg_stat(ObjectContextRef obc, pg_stat_t *stat);
+
+  void get_src_oloc(const object_t& oid, const object_locator_t& oloc, object_locator_t& src_oloc);
+
+  SnapSetContext *create_snapset_context(const hobject_t& oid);
+  SnapSetContext *get_snapset_context(
+    const hobject_t& oid,
+    bool can_create,
+    map *attrs = 0
+    );
+  void register_snapset_context(SnapSetContext *ssc) {
+    Mutex::Locker l(snapset_contexts_lock);
+    _register_snapset_context(ssc);
+  }
+  void _register_snapset_context(SnapSetContext *ssc) {
+    assert(snapset_contexts_lock.is_locked());
+    if (!ssc->registered) {
+      assert(snapset_contexts.count(ssc->oid) == 0);
+      ssc->registered = true;
+      snapset_contexts[ssc->oid] = ssc;
+    }
+  }
+  void put_snapset_context(SnapSetContext *ssc);
+
+  map recovering;
+
+  /*
+   * Backfill
+   *
+   * peer_info[backfill_target].last_backfill == info.last_backfill on the peer.
+   *
+   * objects prior to peer_info[backfill_target].last_backfill
+   *   - are on the peer
+   *   - are included in the peer stats
+   *
+   * objects \in (last_backfill, last_backfill_started]
+   *   - are on the peer or are in backfills_in_flight
+   *   - are not included in pg stats (yet)
+   *   - have their stats in pending_backfill_updates on the primary
+   */
+  set backfills_in_flight;
+  map pending_backfill_updates;
+
+  void dump_recovery_info(Formatter *f) const {
+    f->open_array_section("backfill_targets");
+    for (set::const_iterator p = backfill_targets.begin();
+        p != backfill_targets.end(); ++p)
+      f->dump_stream("replica") << *p;
+    f->close_section();
+    f->open_array_section("waiting_on_backfill");
+    for (set::const_iterator p = waiting_on_backfill.begin();
+        p != waiting_on_backfill.end(); ++p)
+      f->dump_stream("osd") << *p;
+    f->close_section();
+    f->dump_stream("last_backfill_started") << last_backfill_started;
+    {
+      f->open_object_section("backfill_info");
+      backfill_info.dump(f);
+      f->close_section();
+    }
+    {
+      f->open_array_section("peer_backfill_info");
+      for (map::const_iterator pbi =
+	     peer_backfill_info.begin();
+          pbi != peer_backfill_info.end(); ++pbi) {
+        f->dump_stream("osd") << pbi->first;
+        f->open_object_section("BackfillInterval");
+          pbi->second.dump(f);
+        f->close_section();
+      }
+      f->close_section();
+    }
+    {
+      f->open_array_section("backfills_in_flight");
+      for (set::const_iterator i = backfills_in_flight.begin();
+	   i != backfills_in_flight.end();
+	   ++i) {
+	f->dump_stream("object") << *i;
+      }
+      f->close_section();
+    }
+    {
+      f->open_array_section("recovering");
+      for (map::const_iterator i = recovering.begin();
+	   i != recovering.end();
+	   ++i) {
+	f->dump_stream("object") << i->first;
+      }
+      f->close_section();
+    }
+    {
+      f->open_object_section("pg_backend");
+      pgbackend->dump_recovery_info(f);
+      f->close_section();
+    }
+  }
+
+  /// last backfill operation started
+  hobject_t last_backfill_started;
+  bool new_backfill;
+
+  int prep_object_replica_pushes(const hobject_t& soid, eversion_t v,
+				 PGBackend::RecoveryHandle *h);
+
+  void finish_degraded_object(const hobject_t& oid);
+
+  // Cancels/resets pulls from peer
+  void check_recovery_sources(const OSDMapRef map);
+
+  int recover_missing(
+    const hobject_t& oid,
+    eversion_t v,
+    int priority,
+    PGBackend::RecoveryHandle *h);
+
+  // low level ops
+
+  void _make_clone(
+    OpContext *ctx,
+    PGBackend::PGTransaction* t,
+    ObjectContextRef obc,
+    const hobject_t& head, const hobject_t& coid,
+    object_info_t *poi);
+  void execute_ctx(OpContext *ctx);
+  void finish_ctx(OpContext *ctx, int log_op_type, bool maintain_ssc=true);
+  void reply_ctx(OpContext *ctx, int err);
+  void reply_ctx(OpContext *ctx, int err, eversion_t v, version_t uv);
+  void make_writeable(OpContext *ctx);
+  void log_op_stats(OpContext *ctx);
+
+  void write_update_size_and_usage(object_stat_sum_t& stats, object_info_t& oi,
+				   SnapSet& ss, interval_set& modified,
+				   uint64_t offset, uint64_t length, bool count_bytes);
+  void add_interval_usage(interval_set& s, object_stat_sum_t& st);
+
+  /**
+   * This helper function is called from do_op if the ObjectContext lookup fails.
+   * @returns true if the caching code is handling the Op, false otherwise.
+   */
+  inline bool maybe_handle_cache(OpRequestRef op,
+				 bool write_ordered,
+				 ObjectContextRef obc, int r,
+				 const hobject_t& missing_oid,
+				 bool must_promote);
+  /**
+   * This helper function tells the client to redirect their request elsewhere.
+   */
+  void do_cache_redirect(OpRequestRef op, ObjectContextRef obc);
+  /**
+   * This function starts up a copy from
+   */
+  void promote_object(OpRequestRef op, ObjectContextRef obc,
+		      const hobject_t& missing_object);
+
+  /**
+   * Check if the op is such that we can skip promote (e.g., DELETE)
+   */
+  bool can_skip_promote(OpRequestRef op, ObjectContextRef obc);
+
+  int prepare_transaction(OpContext *ctx);
+  list > in_progress_async_reads;
+  void complete_read_ctx(int result, OpContext *ctx);
+  
+  // pg on-disk content
+  void check_local();
+
+  void _clear_recovery_state();
+
+  void queue_for_recovery();
+  bool start_recovery_ops(
+    int max, RecoveryCtx *prctx,
+    ThreadPool::TPHandle &handle, int *started);
+
+  int recover_primary(int max, ThreadPool::TPHandle &handle);
+  int recover_replicas(int max, ThreadPool::TPHandle &handle);
+  hobject_t earliest_peer_backfill() const;
+  bool all_peer_done() const;
+  /**
+   * @param work_started will be set to true if recover_backfill got anywhere
+   * @returns the number of operations started
+   */
+  int recover_backfill(int max, ThreadPool::TPHandle &handle,
+                       bool *work_started);
+
+  /**
+   * scan a (hash) range of objects in the current pg
+   *
+   * @begin first item should be >= this value
+   * @min return at least this many items, unless we are done
+   * @max return no more than this many items
+   * @bi [out] resulting map of objects to eversion_t's
+   */
+  void scan_range(
+    int min, int max, BackfillInterval *bi,
+    ThreadPool::TPHandle &handle
+    );
+
+  /// Update a hash range to reflect changes since the last scan
+  void update_range(
+    BackfillInterval *bi,        ///< [in,out] interval to update
+    ThreadPool::TPHandle &handle ///< [in] tp handle
+    );
+
+  void prep_backfill_object_push(
+    hobject_t oid, eversion_t v, ObjectContextRef obc,
+    vector peers,
+    PGBackend::RecoveryHandle *h);
+  void send_remove_op(const hobject_t& oid, eversion_t v, pg_shard_t peer);
+
+
+  struct C_OSD_OndiskWriteUnlock : public Context {
+    ObjectContextRef obc, obc2, obc3;
+    C_OSD_OndiskWriteUnlock(
+      ObjectContextRef o,
+      ObjectContextRef o2 = ObjectContextRef(),
+      ObjectContextRef o3 = ObjectContextRef()) : obc(o), obc2(o2), obc3(o3) {}
+    void finish(int r) {
+      obc->ondisk_write_unlock();
+      if (obc2)
+	obc2->ondisk_write_unlock();
+      if (obc3)
+	obc3->ondisk_write_unlock();
+    }
+  };
+  struct C_OSD_OndiskWriteUnlockList : public Context {
+    list *pls;
+    C_OSD_OndiskWriteUnlockList(list *l) : pls(l) {}
+    void finish(int r) {
+      for (list::iterator p = pls->begin(); p != pls->end(); ++p)
+	(*p)->ondisk_write_unlock();
+    }
+  };
+  struct C_OSD_AppliedRecoveredObject : public Context {
+    ReplicatedPGRef pg;
+    ObjectContextRef obc;
+    C_OSD_AppliedRecoveredObject(ReplicatedPG *p, ObjectContextRef o) :
+      pg(p), obc(o) {}
+    void finish(int r) {
+      pg->_applied_recovered_object(obc);
+    }
+  };
+  struct C_OSD_CommittedPushedObject : public Context {
+    ReplicatedPGRef pg;
+    epoch_t epoch;
+    eversion_t last_complete;
+    C_OSD_CommittedPushedObject(
+      ReplicatedPG *p, epoch_t epoch, eversion_t lc) :
+      pg(p), epoch(epoch), last_complete(lc) {
+    }
+    void finish(int r) {
+      pg->_committed_pushed_object(epoch, last_complete);
+    }
+  };
+  struct C_OSD_AppliedRecoveredObjectReplica : public Context {
+    ReplicatedPGRef pg;
+    C_OSD_AppliedRecoveredObjectReplica(ReplicatedPG *p) :
+      pg(p) {}
+    void finish(int r) {
+      pg->_applied_recovered_object_replica();
+    }
+  };
+
+  void sub_op_remove(OpRequestRef op);
+
+  void _applied_recovered_object(ObjectContextRef obc);
+  void _applied_recovered_object_replica();
+  void _committed_pushed_object(epoch_t epoch, eversion_t lc);
+  void recover_got(hobject_t oid, eversion_t v);
+
+  // -- copyfrom --
+  map copy_ops;
+
+  int fill_in_copy_get(
+    OpContext *ctx,
+    bufferlist::iterator& bp,
+    OSDOp& op,
+    ObjectContextRef& obc,
+    bool classic);
+
+  /**
+   * To copy an object, call start_copy.
+   *
+   * @param cb: The CopyCallback to be activated when the copy is complete
+   * @param obc: The ObjectContext we are copying into
+   * @param src: The source object
+   * @param oloc: the source object locator
+   * @param version: the version of the source object to copy (0 for any)
+   * @param temp_dest_oid: the temporary object to use for large objects
+   */
+  void start_copy(CopyCallback *cb, ObjectContextRef obc, hobject_t src,
+		  object_locator_t oloc, version_t version, unsigned flags,
+		  bool mirror_snapset);
+  void process_copy_chunk(hobject_t oid, ceph_tid_t tid, int r);
+  void _write_copy_chunk(CopyOpRef cop, PGBackend::PGTransaction *t);
+  uint64_t get_copy_chunk_size() const {
+    uint64_t size = cct->_conf->osd_copyfrom_max_chunk;
+    if (pool.info.requires_aligned_append()) {
+      uint64_t alignment = pool.info.required_alignment();
+      if (size % alignment) {
+	size += alignment - (size % alignment);
+      }
+    }
+    return size;
+  }
+  void _copy_some(ObjectContextRef obc, CopyOpRef cop);
+  void _build_finish_copy_transaction(CopyOpRef cop,
+                                      PGBackend::PGTransaction *t);
+  void finish_copyfrom(OpContext *ctx);
+  void finish_promote(int r, OpRequestRef op,
+		      CopyResults *results, ObjectContextRef obc);
+  void cancel_copy(CopyOpRef cop, bool requeue);
+  void cancel_copy_ops(bool requeue);
+
+  friend struct C_Copyfrom;
+
+  // -- flush --
+  map flush_ops;
+
+  /// start_flush takes ownership of on_flush iff ret == -EINPROGRESS
+  int start_flush(
+    OpRequestRef op, ObjectContextRef obc,
+    bool blocking, hobject_t *pmissing,
+    Context *on_flush);
+  void finish_flush(hobject_t oid, ceph_tid_t tid, int r);
+  int try_flush_mark_clean(FlushOpRef fop);
+  void cancel_flush(FlushOpRef fop, bool requeue);
+  void cancel_flush_ops(bool requeue);
+
+  /// @return false if clone is has been evicted
+  bool is_present_clone(hobject_t coid);
+
+  friend struct C_Flush;
+
+  // -- scrub --
+  virtual bool _range_available_for_scrub(
+    const hobject_t &begin, const hobject_t &end);
+  virtual void _scrub(ScrubMap& map);
+  virtual void _scrub_clear_state();
+  virtual void _scrub_finish();
+  object_stat_collection_t scrub_cstat;
+
+  virtual void _split_into(pg_t child_pgid, PG *child, unsigned split_bits);
+  void apply_and_flush_repops(bool requeue);
+
+  void calc_trim_to();
+  int do_xattr_cmp_u64(int op, __u64 v1, bufferlist& xattr);
+  int do_xattr_cmp_str(int op, string& v1s, bufferlist& xattr);
+
+  bool pgls_filter(PGLSFilter *filter, hobject_t& sobj, bufferlist& outdata);
+  int get_pgls_filter(bufferlist::iterator& iter, PGLSFilter **pfilter);
+
+public:
+  ReplicatedPG(OSDService *o, OSDMapRef curmap,
+	       const PGPool &_pool, spg_t p, const hobject_t& oid,
+	       const hobject_t& ioid);
+  ~ReplicatedPG() {}
+
+  int do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata,
+		 bufferlist& odata);
+
+  void do_request(
+    OpRequestRef op,
+    ThreadPool::TPHandle &handle);
+  void do_op(OpRequestRef op);
+  bool pg_op_must_wait(MOSDOp *op);
+  void do_pg_op(OpRequestRef op);
+  void do_sub_op(OpRequestRef op);
+  void do_sub_op_reply(OpRequestRef op);
+  void do_scan(
+    OpRequestRef op,
+    ThreadPool::TPHandle &handle);
+  void do_backfill(OpRequestRef op);
+
+  RepGather *trim_object(const hobject_t &coid);
+  void snap_trimmer();
+  int do_osd_ops(OpContext *ctx, vector& ops);
+
+  int _get_tmap(OpContext *ctx, bufferlist *header, bufferlist *vals);
+  int do_tmap2omap(OpContext *ctx, unsigned flags);
+  int do_tmapup(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd_op);
+  int do_tmapup_slow(OpContext *ctx, bufferlist::iterator& bp, OSDOp& osd_op, bufferlist& bl);
+
+  void do_osd_op_effects(OpContext *ctx);
+private:
+  hobject_t earliest_backfill() const;
+  bool check_src_targ(const hobject_t& soid, const hobject_t& toid) const;
+  uint64_t temp_seq; ///< last id for naming temp objects
+  coll_t get_temp_coll(ObjectStore::Transaction *t);
+  hobject_t generate_temp_object();  ///< generate a new temp object name
+public:
+  void get_colls(list *out) {
+    out->push_back(coll);
+    return pgbackend->temp_colls(out);
+  }
+  void split_colls(
+    spg_t child,
+    int split_bits,
+    int seed,
+    ObjectStore::Transaction *t) {
+    coll_t target = coll_t(child);
+    t->create_collection(target);
+    t->split_collection(
+      coll,
+      split_bits,
+      seed,
+      target);
+    pgbackend->split_colls(child, split_bits, seed, t);
+  }
+private:
+  struct NotTrimming;
+  struct SnapTrim : boost::statechart::event< SnapTrim > {
+    SnapTrim() : boost::statechart::event < SnapTrim >() {}
+  };
+  struct Reset : boost::statechart::event< Reset > {
+    Reset() : boost::statechart::event< Reset >() {}
+  };
+  struct SnapTrimmer : public boost::statechart::state_machine< SnapTrimmer, NotTrimming > {
+    ReplicatedPG *pg;
+    set repops;
+    snapid_t snap_to_trim;
+    bool need_share_pg_info;
+    bool requeue;
+    SnapTrimmer(ReplicatedPG *pg) : pg(pg), need_share_pg_info(false), requeue(false) {}
+    ~SnapTrimmer();
+    void log_enter(const char *state_name);
+    void log_exit(const char *state_name, utime_t duration);
+  } snap_trimmer_machine;
+
+  /* SnapTrimmerStates */
+  struct TrimmingObjects : boost::statechart::state< TrimmingObjects, SnapTrimmer >, NamedState {
+    typedef boost::mpl::list <
+      boost::statechart::custom_reaction< SnapTrim >,
+      boost::statechart::transition< Reset, NotTrimming >
+      > reactions;
+    hobject_t pos;
+    TrimmingObjects(my_context ctx);
+    void exit();
+    boost::statechart::result react(const SnapTrim&);
+  };
+
+  struct WaitingOnReplicas : boost::statechart::state< WaitingOnReplicas, SnapTrimmer >, NamedState {
+    typedef boost::mpl::list <
+      boost::statechart::custom_reaction< SnapTrim >,
+      boost::statechart::transition< Reset, NotTrimming >
+      > reactions;
+    WaitingOnReplicas(my_context ctx);
+    void exit();
+    boost::statechart::result react(const SnapTrim&);
+  };
+  
+  struct NotTrimming : boost::statechart::state< NotTrimming, SnapTrimmer >, NamedState {
+    typedef boost::mpl::list <
+      boost::statechart::custom_reaction< SnapTrim >,
+      boost::statechart::transition< Reset, NotTrimming >
+      > reactions;
+    NotTrimming(my_context ctx);
+    void exit();
+    boost::statechart::result react(const SnapTrim&);
+  };
+
+  int _verify_no_head_clones(const hobject_t& soid,
+			     const SnapSet& ss);
+  int _delete_oid(OpContext *ctx, bool no_whiteout);
+  int _rollback_to(OpContext *ctx, ceph_osd_op& op);
+public:
+  bool is_missing_object(const hobject_t& oid) const;
+  bool is_unreadable_object(const hobject_t &oid) const {
+    return is_missing_object(oid) ||
+      !missing_loc.readable_with_acting(oid, actingset);
+  }
+  void wait_for_unreadable_object(const hobject_t& oid, OpRequestRef op);
+  void wait_for_all_missing(OpRequestRef op);
+
+  bool is_degraded_object(const hobject_t& oid);
+  void wait_for_degraded_object(const hobject_t& oid, OpRequestRef op);
+
+  bool maybe_await_blocked_snapset(const hobject_t &soid, OpRequestRef op);
+  void wait_for_blocked_object(const hobject_t& soid, OpRequestRef op);
+  void kick_object_context_blocked(ObjectContextRef obc);
+
+  void mark_all_unfound_lost(int what);
+  eversion_t pick_newest_available(const hobject_t& oid);
+  ObjectContextRef mark_object_lost(ObjectStore::Transaction *t,
+				  const hobject_t& oid, eversion_t version,
+				  utime_t mtime, int what);
+  void _finish_mark_all_unfound_lost(list& obcs);
+
+  void on_role_change();
+  void on_pool_change();
+  void on_change(ObjectStore::Transaction *t);
+  void on_activate();
+  void on_flushed();
+  void on_removal(ObjectStore::Transaction *t);
+  void on_shutdown();
+
+  // attr cache handling
+  void replace_cached_attrs(
+    OpContext *ctx,
+    ObjectContextRef obc,
+    const map &new_attrs);
+  void setattr_maybe_cache(
+    ObjectContextRef obc,
+    OpContext *op,
+    PGBackend::PGTransaction *t,
+    const string &key,
+    bufferlist &val);
+  void rmattr_maybe_cache(
+    ObjectContextRef obc,
+    OpContext *op,
+    PGBackend::PGTransaction *t,
+    const string &key);
+  int getattr_maybe_cache(
+    ObjectContextRef obc,
+    const string &key,
+    bufferlist *val);
+  int getattrs_maybe_cache(
+    ObjectContextRef obc,
+    map *out,
+    bool user_only = false);
+};
+
+inline ostream& operator<<(ostream& out, ReplicatedPG::RepGather& repop)
+{
+  out << "repgather(" << &repop
+      << " " << repop.v
+      << " rep_tid=" << repop.rep_tid 
+      << " committed?=" << repop.all_committed
+      << " applied?=" << repop.all_applied;
+  if (repop.ctx->lock_to_release != ReplicatedPG::OpContext::NONE)
+    out << " lock=" << (int)repop.ctx->lock_to_release;
+  if (repop.ctx->op)
+    out << " op=" << *(repop.ctx->op->get_req());
+  out << ")";
+  return out;
+}
+
+void intrusive_ptr_add_ref(ReplicatedPG::RepGather *repop);
+void intrusive_ptr_release(ReplicatedPG::RepGather *repop);
+
+
+#endif
diff --git a/ceph/src/osd/SnapMapper.cc b/ceph/src/osd/SnapMapper.cc
new file mode 100644
index 00000000..870fdd05
--- /dev/null
+++ b/ceph/src/osd/SnapMapper.cc
@@ -0,0 +1,307 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "SnapMapper.h"
+
+using std::string;
+
+const string SnapMapper::MAPPING_PREFIX = "MAP_";
+const string SnapMapper::OBJECT_PREFIX = "OBJ_";
+
+int OSDriver::get_keys(
+  const std::set &keys,
+  std::map *out)
+{
+  return os->omap_get_values(cid, hoid, keys, out);
+}
+
+int OSDriver::get_next(
+  const std::string &key,
+  pair *next)
+{
+  ObjectMap::ObjectMapIterator iter =
+    os->get_omap_iterator(cid, hoid);
+  if (!iter) {
+    assert(0);
+    return -EINVAL;
+  }
+  iter->upper_bound(key);
+  if (iter->valid()) {
+    if (next)
+      *next = make_pair(iter->key(), iter->value());
+    return 0;
+  } else {
+    return -ENOENT;
+  }
+}
+
+struct Mapping {
+  snapid_t snap;
+  hobject_t hoid;
+  Mapping(const pair &in)
+    : snap(in.first), hoid(in.second) {}
+  Mapping() : snap(0) {}
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(snap, bl);
+    ::encode(hoid, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    DECODE_START(1, bl);
+    ::decode(snap, bl);
+    ::decode(hoid, bl);
+    DECODE_FINISH(bl);
+  }
+};
+WRITE_CLASS_ENCODER(Mapping);
+
+string SnapMapper::get_prefix(snapid_t snap)
+{
+  char buf[100];
+  int len = snprintf(
+    buf, sizeof(buf),
+    "%.*X_", (int)(sizeof(snap)*2),
+    static_cast(snap));
+  return MAPPING_PREFIX + string(buf, len);
+}
+
+string SnapMapper::to_raw_key(
+  const pair &in)
+{
+  return get_prefix(in.first) + shard_prefix + in.second.to_str();
+}
+
+pair SnapMapper::to_raw(
+  const pair &in)
+{
+  bufferlist bl;
+  ::encode(Mapping(in), bl);
+  return make_pair(
+    to_raw_key(in),
+    bl);
+}
+
+pair SnapMapper::from_raw(
+  const pair &image)
+{
+  Mapping map;
+  bufferlist bl(image.second);
+  bufferlist::iterator bp(bl.begin());
+  ::decode(map, bp);
+  return make_pair(map.snap, map.hoid);
+}
+
+bool SnapMapper::is_mapping(const string &to_test)
+{
+  return to_test.substr(0, MAPPING_PREFIX.size()) == MAPPING_PREFIX;
+}
+
+string SnapMapper::to_object_key(const hobject_t &hoid)
+{
+  return OBJECT_PREFIX + shard_prefix + hoid.to_str();
+}
+
+void SnapMapper::object_snaps::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(oid, bl);
+  ::encode(snaps, bl);
+  ENCODE_FINISH(bl);
+}
+
+void SnapMapper::object_snaps::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(oid, bl);
+  ::decode(snaps, bl);
+  DECODE_FINISH(bl);
+}
+
+int SnapMapper::get_snaps(
+  const hobject_t &oid,
+  object_snaps *out)
+{
+  assert(check(oid));
+  set keys;
+  map got;
+  keys.insert(to_object_key(oid));
+  int r = backend.get_keys(keys, &got);
+  if (r < 0)
+    return r;
+  if (got.empty())
+    return -ENOENT;
+  if (out) {
+    bufferlist::iterator bp = got.begin()->second.begin();
+    ::decode(*out, bp);
+    assert(!out->snaps.empty());
+  }
+  return 0;
+}
+
+void SnapMapper::clear_snaps(
+  const hobject_t &oid,
+  MapCacher::Transaction *t)
+{
+  assert(check(oid));
+  set to_remove;
+  to_remove.insert(to_object_key(oid));
+  backend.remove_keys(to_remove, t);
+}
+
+void SnapMapper::set_snaps(
+  const hobject_t &oid,
+  const object_snaps &in,
+  MapCacher::Transaction *t)
+{
+  assert(check(oid));
+  map to_set;
+  bufferlist bl;
+  ::encode(in, bl);
+  to_set[to_object_key(oid)] = bl;
+  backend.set_keys(to_set, t);
+}
+
+int SnapMapper::update_snaps(
+  const hobject_t &oid,
+  const set &new_snaps,
+  const set *old_snaps_check,
+  MapCacher::Transaction *t)
+{
+  assert(check(oid));
+  if (new_snaps.empty())
+    return remove_oid(oid, t);
+
+  object_snaps out;
+  int r = get_snaps(oid, &out);
+  if (r < 0)
+    return r;
+  if (old_snaps_check)
+    assert(out.snaps == *old_snaps_check);
+
+  object_snaps in(oid, new_snaps);
+  set_snaps(oid, in, t);
+
+  set to_remove;
+  for (set::iterator i = out.snaps.begin();
+       i != out.snaps.end();
+       ++i) {
+    if (!new_snaps.count(*i)) {
+      to_remove.insert(to_raw_key(make_pair(*i, oid)));
+    }
+  }
+  backend.remove_keys(to_remove, t);
+  return 0;
+}
+
+void SnapMapper::add_oid(
+  const hobject_t &oid,
+  set snaps,
+  MapCacher::Transaction *t)
+{
+  assert(check(oid));
+  {
+    object_snaps out;
+    int r = get_snaps(oid, &out);
+    assert(r == -ENOENT);
+  }
+
+  object_snaps _snaps(oid, snaps);
+  set_snaps(oid, _snaps, t);
+
+  map to_add;
+  for (set::iterator i = snaps.begin();
+       i != snaps.end();
+       ++i) {
+    to_add.insert(to_raw(make_pair(*i, oid)));
+  }
+  backend.set_keys(to_add, t);
+}
+
+int SnapMapper::get_next_object_to_trim(
+  snapid_t snap,
+  hobject_t *hoid)
+{
+  for (set::iterator i = prefixes.begin();
+       i != prefixes.end();
+       ++i) {
+    string list_after(get_prefix(snap) + *i);
+
+    pair next;
+    int r = backend.get_next(list_after, &next);
+    if (r < 0) {
+      break; // Done
+    }
+
+    if (next.first.substr(0, list_after.size()) !=
+	list_after) {
+      continue; // Done with this prefix
+    }
+
+    assert(is_mapping(next.first));
+
+    pair next_decoded(from_raw(next));
+    assert(next_decoded.first == snap);
+    assert(check(next_decoded.second));
+
+    if (hoid)
+      *hoid = next_decoded.second;
+    return 0;
+  }
+  return -ENOENT;
+}
+
+
+int SnapMapper::remove_oid(
+  const hobject_t &oid,
+  MapCacher::Transaction *t)
+{
+  assert(check(oid));
+  return _remove_oid(oid, t);
+}
+
+int SnapMapper::_remove_oid(
+  const hobject_t &oid,
+  MapCacher::Transaction *t)
+{
+  object_snaps out;
+  int r = get_snaps(oid, &out);
+  if (r < 0)
+    return r;
+
+  clear_snaps(oid, t);
+
+  set to_remove;
+  for (set::iterator i = out.snaps.begin();
+       i != out.snaps.end();
+       ++i) {
+    to_remove.insert(to_raw_key(make_pair(*i, oid)));
+  }
+  backend.remove_keys(to_remove, t);
+  return 0;
+}
+
+int SnapMapper::get_snaps(
+  const hobject_t &oid,
+  std::set *snaps)
+{
+  assert(check(oid));
+  object_snaps out;
+  int r = get_snaps(oid, &out);
+  if (r < 0)
+    return r;
+  if (snaps)
+    snaps->swap(out.snaps);
+  return 0;
+}
diff --git a/ceph/src/osd/SnapMapper.h b/ceph/src/osd/SnapMapper.h
new file mode 100644
index 00000000..5565e175
--- /dev/null
+++ b/ceph/src/osd/SnapMapper.h
@@ -0,0 +1,233 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef SNAPMAPPER_H
+#define SNAPMAPPER_H
+
+#include 
+#include 
+#include 
+#include 
+
+#include "common/map_cacher.hpp"
+#include "common/hobject.h"
+#include "include/buffer.h"
+#include "include/encoding.h"
+#include "include/object.h"
+#include "os/ObjectStore.h"
+
+class OSDriver : public MapCacher::StoreDriver {
+  ObjectStore *os;
+  coll_t cid;
+  hobject_t hoid;
+
+public:
+  class OSTransaction : public MapCacher::Transaction {
+    friend class OSDriver;
+    coll_t cid;
+    hobject_t hoid;
+    ObjectStore::Transaction *t;
+    OSTransaction(
+      coll_t cid,
+      const hobject_t &hoid,
+      ObjectStore::Transaction *t)
+      : cid(cid), hoid(hoid), t(t) {}
+  public:
+    void set_keys(
+      const std::map &to_set) {
+      t->omap_setkeys(cid, hoid, to_set);
+    }
+    void remove_keys(
+      const std::set &to_remove) {
+      t->omap_rmkeys(cid, hoid, to_remove);
+    }
+    void add_callback(
+      Context *c) {
+      t->register_on_applied(c);
+    }
+  };
+
+  OSTransaction get_transaction(
+    ObjectStore::Transaction *t) {
+    return OSTransaction(cid, hoid, t);
+  }
+
+  OSDriver(ObjectStore *os, coll_t cid, const hobject_t &hoid) :
+    os(os), cid(cid), hoid(hoid) {}
+  int get_keys(
+    const std::set &keys,
+    std::map *out);
+  int get_next(
+    const std::string &key,
+    pair *next);
+};
+
+/**
+ * SnapMapper
+ *
+ * Manages two mappings:
+ *  1) hobject_t -> {snapid}
+ *  2) snapid -> {hobject_t}
+ *
+ * We accomplish this using two sets of keys:
+ *  1) OBJECT_PREFIX + obj.str() -> encoding of object_snaps
+ *  2) MAPPING_PREFIX + snapid_t + obj.str() -> encoding of pair
+ *
+ * The on disk strings and encodings are implemented in to_raw, to_raw_key,
+ * from_raw, to_object_key.
+ *
+ * The object -> {snapid} mapping is primarily included so that the
+ * SnapMapper state can be verified against the external PG state during
+ * scrub etc.
+ *
+ * The 2) mapping is arranged such that all objects in a particular
+ * snap will sort together, and so that all objects in a pg for a
+ * particular snap will group under up to 8 prefixes.
+ */
+class SnapMapper {
+public:
+  struct object_snaps {
+    hobject_t oid;
+    std::set snaps;
+    object_snaps(hobject_t oid, const std::set &snaps)
+      : oid(oid), snaps(snaps) {}
+    object_snaps() {}
+    void encode(bufferlist &bl) const;
+    void decode(bufferlist::iterator &bp);
+  };
+
+private:
+  MapCacher::MapCacher backend;
+
+  static const std::string MAPPING_PREFIX;
+  static const std::string OBJECT_PREFIX;
+
+  static std::string get_prefix(snapid_t snap);
+
+  std::string to_raw_key(
+    const std::pair &to_map);
+
+  std::pair to_raw(
+    const std::pair &to_map);
+
+  static bool is_mapping(const std::string &to_test);
+
+  std::pair from_raw(
+    const std::pair &image);
+
+  std::string to_object_key(const hobject_t &hoid);
+
+  int get_snaps(const hobject_t &oid, object_snaps *out);
+
+  void set_snaps(
+    const hobject_t &oid,
+    const object_snaps &out,
+    MapCacher::Transaction *t);
+
+  void clear_snaps(
+    const hobject_t &oid,
+    MapCacher::Transaction *t);
+
+  // True if hoid belongs in this mapping based on mask_bits and match
+  bool check(const hobject_t &hoid) const {
+    return hoid.match(mask_bits, match);
+  }
+
+  int _remove_oid(
+    const hobject_t &oid,    ///< [in] oid to remove
+    MapCacher::Transaction *t ///< [out] transaction
+    );
+
+public:
+  static string make_shard_prefix(shard_id_t shard) {
+    if (shard == ghobject_t::NO_SHARD)
+      return string();
+    char buf[20];
+    int r = snprintf(buf, sizeof(buf), ".%x", (int)shard);
+    assert(r < (int)sizeof(buf));
+    return string(buf, r) + '_';
+  }
+  uint32_t mask_bits;
+  const uint32_t match;
+  string last_key_checked;
+  const int64_t pool;
+  const shard_id_t shard;
+  const string shard_prefix;
+  SnapMapper(
+    MapCacher::StoreDriver *driver,
+    uint32_t match,  ///< [in] pgid
+    uint32_t bits,   ///< [in] current split bits
+    int64_t pool,    ///< [in] pool
+    shard_id_t shard ///< [in] shard
+    )
+    : backend(driver), mask_bits(bits), match(match), pool(pool),
+      shard(shard), shard_prefix(make_shard_prefix(shard)) {
+    update_bits(mask_bits);
+  }
+
+  set prefixes;
+  /// Update bits in case of pg split
+  void update_bits(
+    uint32_t new_bits  ///< [in] new split bits
+    ) {
+    assert(new_bits >= mask_bits);
+    mask_bits = new_bits;
+    set _prefixes = hobject_t::get_prefixes(
+      mask_bits,
+      match,
+      pool);
+    prefixes.clear();
+    for (set::iterator i = _prefixes.begin();
+	 i != _prefixes.end();
+	 ++i) {
+      prefixes.insert(shard_prefix + *i);
+    }
+  }
+
+  /// Update snaps for oid, empty new_snaps removes the mapping
+  int update_snaps(
+    const hobject_t &oid,       ///< [in] oid to update
+    const std::set &new_snaps, ///< [in] new snap set
+    const std::set *old_snaps, ///< [in] old snaps (for debugging)
+    MapCacher::Transaction *t ///< [out] transaction
+    ); ///@ return error, 0 on success
+
+  /// Add mapping for oid, must not already be mapped
+  void add_oid(
+    const hobject_t &oid,       ///< [in] oid to add
+    std::set new_snaps, ///< [in] snaps
+    MapCacher::Transaction *t ///< [out] transaction
+    );
+
+  /// Returns first object with snap as a snap
+  int get_next_object_to_trim(
+    snapid_t snap,              ///< [in] snap to check
+    hobject_t *hoid             ///< [out] next hoid to trim
+    );  ///< @return error, -ENOENT if no more objects
+
+  /// Remove mapping for oid
+  int remove_oid(
+    const hobject_t &oid,    ///< [in] oid to remove
+    MapCacher::Transaction *t ///< [out] transaction
+    ); ///< @return error, -ENOENT if the object is not mapped
+
+  /// Get snaps for oid
+  int get_snaps(
+    const hobject_t &oid,     ///< [in] oid to get snaps for
+    std::set *snaps ///< [out] snaps
+    ); ///< @return error, -ENOENT if oid is not recorded
+};
+WRITE_CLASS_ENCODER(SnapMapper::object_snaps)
+
+#endif
diff --git a/ceph/src/osd/TierAgentState.h b/ceph/src/osd/TierAgentState.h
new file mode 100644
index 00000000..e9c22b24
--- /dev/null
+++ b/ceph/src/osd/TierAgentState.h
@@ -0,0 +1,119 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_OSD_TIERAGENT_H
+#define CEPH_OSD_TIERAGENT_H
+
+struct TierAgentState {
+  /// current position iterating across pool
+  hobject_t position;
+  /// Count of agent_work since "start" position of object hash space
+  int started;
+  hobject_t start;
+  bool delaying;
+
+  /// histogram of ages we've encountered
+  pow2_hist_t atime_hist;
+  pow2_hist_t temp_hist;
+  int hist_age;
+
+  /// past HitSet(s) (not current)
+  map hit_set_map;
+
+  /// a few recent things we've seen that are clean
+  list recent_clean;
+
+  enum flush_mode_t {
+    FLUSH_MODE_IDLE,   // nothing to flush
+    FLUSH_MODE_ACTIVE, // flush what we can to bring down dirty count
+  } flush_mode;     ///< current flush behavior
+  static const char *get_flush_mode_name(flush_mode_t m) {
+    switch (m) {
+    case FLUSH_MODE_IDLE: return "idle";
+    case FLUSH_MODE_ACTIVE: return "active";
+    default: assert(0 == "bad flush mode");
+    }
+  }
+  const char *get_flush_mode_name() const {
+    return get_flush_mode_name(flush_mode);
+  }
+
+  enum evict_mode_t {
+    EVICT_MODE_IDLE,      // no need to evict anything
+    EVICT_MODE_SOME,      // evict some things as we are near the target
+    EVICT_MODE_FULL,      // evict anything
+  } evict_mode;     ///< current evict behavior
+  static const char *get_evict_mode_name(evict_mode_t m) {
+    switch (m) {
+    case EVICT_MODE_IDLE: return "idle";
+    case EVICT_MODE_SOME: return "some";
+    case EVICT_MODE_FULL: return "full";
+    default: assert(0 == "bad evict mode");
+    }
+  }
+  const char *get_evict_mode_name() const {
+    return get_evict_mode_name(evict_mode);
+  }
+
+  /// approximate ratio of objects (assuming they are uniformly
+  /// distributed) that i should aim to evict.
+  unsigned evict_effort;
+
+  TierAgentState()
+    : started(0),
+      delaying(false),
+      hist_age(0),
+      flush_mode(FLUSH_MODE_IDLE),
+      evict_mode(EVICT_MODE_IDLE),
+      evict_effort(0)
+  {}
+
+  /// false if we have any work to do
+  bool is_idle() const {
+    return
+      delaying ||
+      (flush_mode == FLUSH_MODE_IDLE &&
+      evict_mode == EVICT_MODE_IDLE);
+  }
+
+  /// add archived HitSet
+  void add_hit_set(time_t start, HitSetRef hs) {
+    hit_set_map.insert(make_pair(start, hs));
+  }
+
+  /// remove old/trimmed HitSet
+  void remove_oldest_hit_set() {
+    if (!hit_set_map.empty())
+      hit_set_map.erase(hit_set_map.begin());
+  }
+
+  /// discard all open hit sets
+  void discard_hit_sets() {
+    hit_set_map.clear();
+  }
+
+  void dump(Formatter *f) const {
+    f->dump_string("flush_mode", get_flush_mode_name());
+    f->dump_string("evict_mode", get_evict_mode_name());
+    f->dump_unsigned("evict_effort", evict_effort);
+    f->dump_stream("position") << position;
+    f->open_object_section("atime_hist");
+    atime_hist.dump(f);
+    f->close_section();
+    f->open_object_section("temp_hist");
+    temp_hist.dump(f);
+    f->close_section();
+  }
+};
+
+#endif
diff --git a/ceph/src/osd/Watch.cc b/ceph/src/osd/Watch.cc
new file mode 100644
index 00000000..4ef4bd13
--- /dev/null
+++ b/ceph/src/osd/Watch.cc
@@ -0,0 +1,457 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "PG.h"
+
+#include "include/types.h"
+#include "messages/MWatchNotify.h"
+
+#include 
+
+#include "OSD.h"
+#include "ReplicatedPG.h"
+#include "Watch.h"
+
+#include "common/config.h"
+
+struct CancelableContext : public Context {
+  virtual void cancel() = 0;
+};
+
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+
+static ostream& _prefix(
+  std::ostream* _dout,
+  Notify *notify) {
+  return *_dout << notify->gen_dbg_prefix();
+}
+
+Notify::Notify(
+  ConnectionRef client,
+  unsigned num_watchers,
+  bufferlist &payload,
+  uint32_t timeout,
+  uint64_t cookie,
+  uint64_t notify_id,
+  uint64_t version,
+  OSDService *osd)
+  : client(client),
+    in_progress_watchers(num_watchers),
+    complete(false),
+    discarded(false),
+    payload(payload),
+    timeout(timeout),
+    cookie(cookie),
+    notify_id(notify_id),
+    version(version),
+    osd(osd),
+    cb(NULL),
+    lock("Notify::lock") {}
+
+NotifyRef Notify::makeNotifyRef(
+  ConnectionRef client,
+  unsigned num_watchers,
+  bufferlist &payload,
+  uint32_t timeout,
+  uint64_t cookie,
+  uint64_t notify_id,
+  uint64_t version,
+  OSDService *osd) {
+  NotifyRef ret(
+    new Notify(
+      client, num_watchers,
+      payload, timeout,
+      cookie, notify_id,
+      version, osd));
+  ret->set_self(ret);
+  return ret;
+}
+
+class NotifyTimeoutCB : public CancelableContext {
+  NotifyRef notif;
+  bool canceled; // protected by notif lock
+public:
+  NotifyTimeoutCB(NotifyRef notif) : notif(notif), canceled(false) {}
+  void finish(int) {
+    notif->osd->watch_lock.Unlock();
+    notif->lock.Lock();
+    if (!canceled)
+      notif->do_timeout(); // drops lock
+    else
+      notif->lock.Unlock();
+    notif->osd->watch_lock.Lock();
+  }
+  void cancel() {
+    assert(notif->lock.is_locked_by_me());
+    canceled = true;
+  }
+};
+
+void Notify::do_timeout()
+{
+  assert(lock.is_locked_by_me());
+  dout(10) << "timeout" << dendl;
+  cb = NULL;
+  if (is_discarded()) {
+    lock.Unlock();
+    return;
+  }
+
+  in_progress_watchers = 0; // we give up TODO: we should return an error code
+  maybe_complete_notify();
+  assert(complete);
+  set _watchers;
+  _watchers.swap(watchers);
+  lock.Unlock();
+
+  for (set::iterator i = _watchers.begin();
+       i != _watchers.end();
+       ++i) {
+    boost::intrusive_ptr pg((*i)->get_pg());
+    pg->lock();
+    if (!(*i)->is_discarded()) {
+      (*i)->cancel_notify(self.lock());
+    }
+    pg->unlock();
+  }
+}
+
+void Notify::register_cb()
+{
+  assert(lock.is_locked_by_me());
+  {
+    osd->watch_lock.Lock();
+    cb = new NotifyTimeoutCB(self.lock());
+    osd->watch_timer.add_event_after(
+      timeout,
+      cb);
+    osd->watch_lock.Unlock();
+  }
+}
+
+void Notify::unregister_cb()
+{
+  assert(lock.is_locked_by_me());
+  if (!cb)
+    return;
+  cb->cancel();
+  {
+    osd->watch_lock.Lock();
+    osd->watch_timer.cancel_event(cb);
+    cb = NULL;
+    osd->watch_lock.Unlock();
+  }
+}
+
+void Notify::start_watcher(WatchRef watch)
+{
+  Mutex::Locker l(lock);
+  dout(10) << "start_watcher" << dendl;
+  watchers.insert(watch);
+}
+
+void Notify::complete_watcher(WatchRef watch)
+{
+  Mutex::Locker l(lock);
+  dout(10) << "complete_watcher" << dendl;
+  if (is_discarded())
+    return;
+  assert(in_progress_watchers > 0);
+  watchers.erase(watch);
+  --in_progress_watchers;
+  maybe_complete_notify();
+}
+
+void Notify::maybe_complete_notify()
+{
+  dout(10) << "maybe_complete_notify -- "
+	   << in_progress_watchers
+	   << " in progress watchers " << dendl;
+  if (!in_progress_watchers) {
+    MWatchNotify *reply(new MWatchNotify(cookie, version, notify_id,
+					 WATCH_NOTIFY, payload));
+    osd->send_message_osd_client(reply, client.get());
+    unregister_cb();
+    complete = true;
+  }
+}
+
+void Notify::discard()
+{
+  Mutex::Locker l(lock);
+  discarded = true;
+  unregister_cb();
+  watchers.clear();
+}
+
+void Notify::init()
+{
+  Mutex::Locker l(lock);
+  register_cb();
+  maybe_complete_notify();
+  assert(in_progress_watchers == watchers.size());
+}
+
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, watch.get())
+
+static ostream& _prefix(
+  std::ostream* _dout,
+  Watch *watch) {
+  return *_dout << watch->gen_dbg_prefix();
+}
+
+class HandleWatchTimeout : public CancelableContext {
+  WatchRef watch;
+public:
+  bool canceled; // protected by watch->pg->lock
+  HandleWatchTimeout(WatchRef watch) : watch(watch), canceled(false) {}
+  void cancel() {
+    canceled = true;
+  }
+  void finish(int) { assert(0); /* not used */ }
+  void complete(int) {
+    dout(10) << "HandleWatchTimeout" << dendl;
+    boost::intrusive_ptr pg(watch->pg);
+    OSDService *osd(watch->osd);
+    osd->watch_lock.Unlock();
+    pg->lock();
+    watch->cb = NULL;
+    if (!watch->is_discarded() && !canceled)
+      watch->pg->handle_watch_timeout(watch);
+    delete this; // ~Watch requires pg lock!
+    pg->unlock();
+    osd->watch_lock.Lock();
+  }
+};
+
+class HandleDelayedWatchTimeout : public CancelableContext {
+  WatchRef watch;
+public:
+  bool canceled;
+  HandleDelayedWatchTimeout(WatchRef watch) : watch(watch), canceled(false) {}
+  void cancel() {
+    canceled = true;
+  }
+  void finish(int) {
+    dout(10) << "HandleWatchTimeoutDelayed" << dendl;
+    assert(watch->pg->is_locked());
+    watch->cb = NULL;
+    if (!watch->is_discarded() && !canceled)
+      watch->pg->handle_watch_timeout(watch);
+  }
+};
+
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+
+string Watch::gen_dbg_prefix() {
+  stringstream ss;
+  ss << pg->gen_prefix() << " -- Watch(" 
+     << make_pair(cookie, entity) << ") ";
+  return ss.str();
+}
+
+Watch::Watch(
+  ReplicatedPG *pg,
+  OSDService *osd,
+  ObjectContextRef obc,
+  uint32_t timeout,
+  uint64_t cookie,
+  entity_name_t entity,
+  const entity_addr_t &addr)
+  : cb(NULL),
+    osd(osd),
+    pg(pg),
+    obc(obc),
+    timeout(timeout),
+    cookie(cookie),
+    addr(addr),
+    entity(entity),
+    discarded(false) {
+  dout(10) << "Watch()" << dendl;
+}
+
+Watch::~Watch() {
+  dout(10) << "~Watch" << dendl;
+  // users must have called remove() or discard() prior to this point
+  assert(!obc);
+  assert(!conn);
+}
+
+bool Watch::connected() { return conn; }
+
+Context *Watch::get_delayed_cb()
+{
+  assert(!cb);
+  cb = new HandleDelayedWatchTimeout(self.lock());
+  return cb;
+}
+
+void Watch::register_cb()
+{
+  Mutex::Locker l(osd->watch_lock);
+  dout(15) << "registering callback, timeout: " << timeout << dendl;
+  cb = new HandleWatchTimeout(self.lock());
+  osd->watch_timer.add_event_after(
+    timeout,
+    cb);
+}
+
+void Watch::unregister_cb()
+{
+  dout(15) << "unregister_cb" << dendl;
+  if (!cb)
+    return;
+  dout(15) << "actually registered, cancelling" << dendl;
+  cb->cancel();
+  {
+    Mutex::Locker l(osd->watch_lock);
+    osd->watch_timer.cancel_event(cb); // harmless if not registered with timer
+  }
+  cb = NULL;
+}
+
+void Watch::connect(ConnectionRef con)
+{
+  dout(10) << "connecting" << dendl;
+  conn = con;
+  OSD::Session* sessionref(static_cast(con->get_priv()));
+  sessionref->wstate.addWatch(self.lock());
+  sessionref->put();
+  for (map::iterator i = in_progress_notifies.begin();
+       i != in_progress_notifies.end();
+       ++i) {
+    send_notify(i->second);
+  }
+  unregister_cb();
+}
+
+void Watch::disconnect()
+{
+  dout(10) << "disconnect" << dendl;
+  conn = ConnectionRef();
+  register_cb();
+}
+
+void Watch::discard()
+{
+  dout(10) << "discard" << dendl;
+  for (map::iterator i = in_progress_notifies.begin();
+       i != in_progress_notifies.end();
+       ++i) {
+    i->second->discard();
+  }
+  discard_state();
+}
+
+void Watch::discard_state()
+{
+  assert(pg->is_locked());
+  assert(!discarded);
+  assert(obc);
+  in_progress_notifies.clear();
+  unregister_cb();
+  discarded = true;
+  if (conn) {
+    OSD::Session* sessionref(static_cast(conn->get_priv()));
+    sessionref->wstate.removeWatch(self.lock());
+    sessionref->put();
+    conn = ConnectionRef();
+  }
+  obc = ObjectContextRef();
+}
+
+bool Watch::is_discarded()
+{
+  return discarded;
+}
+
+void Watch::remove()
+{
+  dout(10) << "remove" << dendl;
+  for (map::iterator i = in_progress_notifies.begin();
+       i != in_progress_notifies.end();
+       ++i) {
+    i->second->complete_watcher(self.lock());
+  }
+  discard_state();
+}
+
+void Watch::start_notify(NotifyRef notif)
+{
+  dout(10) << "start_notify " << notif->notify_id << dendl;
+  assert(in_progress_notifies.find(notif->notify_id) ==
+	 in_progress_notifies.end());
+  in_progress_notifies[notif->notify_id] = notif;
+  notif->start_watcher(self.lock());
+  if (connected())
+    send_notify(notif);
+}
+
+void Watch::cancel_notify(NotifyRef notif)
+{
+  dout(10) << "cancel_notify " << notif->notify_id << dendl;
+  in_progress_notifies.erase(notif->notify_id);
+}
+
+void Watch::send_notify(NotifyRef notif)
+{
+  dout(10) << "send_notify" << dendl;
+  MWatchNotify *notify_msg = new MWatchNotify(
+    cookie, notif->version, notif->notify_id,
+    WATCH_NOTIFY, notif->payload);
+  osd->send_message_osd_client(notify_msg, conn.get());
+}
+
+void Watch::notify_ack(uint64_t notify_id)
+{
+  dout(10) << "notify_ack" << dendl;
+  map::iterator i = in_progress_notifies.find(notify_id);
+  if (i != in_progress_notifies.end()) {
+    i->second->complete_watcher(self.lock());
+    in_progress_notifies.erase(i);
+  }
+}
+
+WatchRef Watch::makeWatchRef(
+  ReplicatedPG *pg, OSDService *osd,
+  ObjectContextRef obc, uint32_t timeout, uint64_t cookie, entity_name_t entity, const entity_addr_t& addr)
+{
+  WatchRef ret(new Watch(pg, osd, obc, timeout, cookie, entity, addr));
+  ret->set_self(ret);
+  return ret;
+}
+
+void WatchConState::addWatch(WatchRef watch)
+{
+  Mutex::Locker l(lock);
+  watches.insert(watch);
+}
+
+void WatchConState::removeWatch(WatchRef watch)
+{
+  Mutex::Locker l(lock);
+  watches.erase(watch);
+}
+
+void WatchConState::reset()
+{
+  set _watches;
+  {
+    Mutex::Locker l(lock);
+    _watches.swap(watches);
+  }
+  for (set::iterator i = _watches.begin();
+       i != _watches.end();
+       ++i) {
+    boost::intrusive_ptr pg((*i)->get_pg());
+    pg->lock();
+    if (!(*i)->is_discarded()) {
+      (*i)->disconnect();
+    }
+    pg->unlock();
+  }
+}
diff --git a/ceph/src/osd/Watch.h b/ceph/src/osd/Watch.h
new file mode 100644
index 00000000..e2cbfc1b
--- /dev/null
+++ b/ceph/src/osd/Watch.h
@@ -0,0 +1,268 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#ifndef CEPH_WATCH_H
+#define CEPH_WATCH_H
+
+#include 
+#include "include/memory.h"
+#include 
+
+#include "msg/Messenger.h"
+#include "include/Context.h"
+#include "common/Mutex.h"
+
+enum WatcherState {
+  WATCHER_PENDING,
+  WATCHER_NOTIFIED,
+};
+
+class OSDService;
+class ReplicatedPG;
+void intrusive_ptr_add_ref(ReplicatedPG *pg);
+void intrusive_ptr_release(ReplicatedPG *pg);
+struct ObjectContext;
+class MWatchNotify;
+
+class Watch;
+typedef ceph::shared_ptr WatchRef;
+typedef ceph::weak_ptr WWatchRef;
+
+class Notify;
+typedef ceph::shared_ptr NotifyRef;
+typedef ceph::weak_ptr WNotifyRef;
+
+struct CancelableContext;
+
+/**
+ * Notify tracks the progress of a particular notify
+ *
+ * References are held by Watch and the timeout callback.
+ */
+class NotifyTimeoutCB;
+class Notify {
+  friend class NotifyTimeoutCB;
+  friend class Watch;
+  WNotifyRef self;
+  ConnectionRef client;
+  unsigned in_progress_watchers;
+  bool complete;
+  bool discarded;
+  set watchers;
+
+  bufferlist payload;
+  uint32_t timeout;
+  uint64_t cookie;
+  uint64_t notify_id;
+  uint64_t version;
+
+  OSDService *osd;
+  CancelableContext *cb;
+  Mutex lock;
+
+
+  /// true if this notify is being discarded
+  bool is_discarded() {
+    return discarded || complete;
+  }
+
+  /// Sends notify completion if in_progress_watchers == 0
+  void maybe_complete_notify();
+
+  /// Called on Notify timeout
+  void do_timeout();
+
+  Notify(
+    ConnectionRef client,
+    unsigned num_watchers,
+    bufferlist &payload,
+    uint32_t timeout,
+    uint64_t cookie,
+    uint64_t notify_id,
+    uint64_t version,
+    OSDService *osd);
+
+  /// registers a timeout callback with the watch_timer
+  void register_cb();
+
+  /// removes the timeout callback, called on completion or cancellation
+  void unregister_cb();
+public:
+  string gen_dbg_prefix() {
+    stringstream ss;
+    ss << "Notify(" << make_pair(cookie, notify_id) << " "
+       << " in_progress_watchers=" << in_progress_watchers
+       << ") ";
+    return ss.str();
+  }
+  void set_self(NotifyRef _self) {
+    self = _self;
+  }
+  static NotifyRef makeNotifyRef(
+    ConnectionRef client,
+    unsigned num_watchers,
+    bufferlist &payload,
+    uint32_t timeout,
+    uint64_t cookie,
+    uint64_t notify_id,
+    uint64_t version,
+    OSDService *osd);
+
+  /// Call after creation to initialize
+  void init();
+
+  /// Called once per watcher prior to init()
+  void start_watcher(
+    WatchRef watcher ///< [in] watcher to complete
+    );
+
+  /// Called once per NotifyAck
+  void complete_watcher(
+    WatchRef watcher ///< [in] watcher to complete
+    );
+
+  /// Called when the notify is canceled due to a new peering interval
+  void discard();
+};
+
+/**
+ * Watch is a mapping between a Connection and an ObjectContext
+ *
+ * References are held by ObjectContext and the timeout callback
+ */
+class HandleWatchTimeout;
+class HandleDelayedWatchTimeout;
+class Watch {
+  WWatchRef self;
+  friend class HandleWatchTimeout;
+  friend class HandleDelayedWatchTimeout;
+  ConnectionRef conn;
+  CancelableContext *cb;
+
+  OSDService *osd;
+  boost::intrusive_ptr pg;
+  ceph::shared_ptr obc;
+
+  std::map in_progress_notifies;
+
+  // Could have watch_info_t here, but this file includes osd_types.h
+  uint32_t timeout;
+  uint64_t cookie;
+  entity_addr_t addr;
+
+  entity_name_t entity;
+  bool discarded;
+
+  Watch(
+    ReplicatedPG *pg, OSDService *osd,
+    ceph::shared_ptr obc, uint32_t timeout,
+    uint64_t cookie, entity_name_t entity,
+    const entity_addr_t& addr);
+
+  /// Registers the timeout callback with watch_timer
+  void register_cb();
+
+  /// Unregisters the timeout callback
+  void unregister_cb();
+
+  /// send a Notify message when connected for notif
+  void send_notify(NotifyRef notif);
+
+  /// Cleans up state on discard or remove (including Connection state, obc)
+  void discard_state();
+public:
+  /// NOTE: must be called with pg lock held
+  ~Watch();
+
+  string gen_dbg_prefix();
+  static WatchRef makeWatchRef(
+    ReplicatedPG *pg, OSDService *osd,
+    ceph::shared_ptr obc, uint32_t timeout, uint64_t cookie, entity_name_t entity, const entity_addr_t &addr);
+  void set_self(WatchRef _self) {
+    self = _self;
+  }
+
+  /// Does not grant a ref count!
+  boost::intrusive_ptr get_pg() { return pg; }
+
+  ceph::shared_ptr get_obc() { return obc; }
+
+  uint64_t get_cookie() const { return cookie; }
+  entity_name_t get_entity() const { return entity; }
+  entity_addr_t get_peer_addr() const { return addr; }
+  uint32_t get_timeout() const { return timeout; }
+
+  /// Generates context for use if watch timeout is delayed by scrub or recovery
+  Context *get_delayed_cb();
+
+  /// True if currently connected
+  bool connected();
+
+  /// Transitions Watch to connected, unregister_cb, resends pending Notifies
+  void connect(
+    ConnectionRef con ///< [in] Reference to new connection
+    );
+
+  /// Transitions watch to disconnected, register_cb
+  void disconnect();
+
+  /// Called if Watch state is discarded due to new peering interval
+  void discard();
+
+  /// True if removed or discarded
+  bool is_discarded();
+
+  /// Called on unwatch
+  void remove();
+
+  /// Adds notif as in-progress notify
+  void start_notify(
+    NotifyRef notif ///< [in] Reference to new in-progress notify
+    );
+
+  /// Removes timed out notify
+  void cancel_notify(
+    NotifyRef notif ///< [in] notify which timed out
+    );
+
+  /// Call when notify_ack received on notify_id
+  void notify_ack(
+    uint64_t notify_id ///< [in] id of acked notify
+    );
+};
+
+/**
+ * Holds weak refs to Watch structures corresponding to a connection
+ * Lives in the OSD::Session object of an OSD connection
+ */
+class WatchConState {
+  Mutex lock;
+  std::set watches;
+public:
+  WatchConState() : lock("WatchConState") {}
+
+  /// Add a watch
+  void addWatch(
+    WatchRef watch ///< [in] Ref to new watch object
+    );
+
+  /// Remove a watch
+  void removeWatch(
+    WatchRef watch ///< [in] Ref to watch object to remove
+    );
+
+  /// Called on session reset, disconnects watchers
+  void reset();
+};
+
+#endif
diff --git a/ceph/src/osd/osd_types.cc b/ceph/src/osd/osd_types.cc
new file mode 100644
index 00000000..16bdbaf3
--- /dev/null
+++ b/ceph/src/osd/osd_types.cc
@@ -0,0 +1,4494 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "osd_types.h"
+#include "include/ceph_features.h"
+extern "C" {
+#include "crush/hash.h"
+}
+#include "PG.h"
+#include "OSDMap.h"
+#include "PGBackend.h"
+
+const char *ceph_osd_flag_name(unsigned flag)
+{
+  switch (flag) {
+  case CEPH_OSD_FLAG_ACK: return "ack";
+  case CEPH_OSD_FLAG_ONNVRAM: return "onnvram";
+  case CEPH_OSD_FLAG_ONDISK: return "ondisk";
+  case CEPH_OSD_FLAG_RETRY: return "retry";
+  case CEPH_OSD_FLAG_READ: return "read";
+  case CEPH_OSD_FLAG_WRITE: return "write";
+  case CEPH_OSD_FLAG_ORDERSNAP: return "ordersnap";
+  case CEPH_OSD_FLAG_PEERSTAT_OLD: return "peerstat_old";
+  case CEPH_OSD_FLAG_BALANCE_READS: return "balance_reads";
+  case CEPH_OSD_FLAG_PARALLELEXEC: return "parallelexec";
+  case CEPH_OSD_FLAG_PGOP: return "pgop";
+  case CEPH_OSD_FLAG_EXEC: return "exec";
+  case CEPH_OSD_FLAG_EXEC_PUBLIC: return "exec_public";
+  case CEPH_OSD_FLAG_LOCALIZE_READS: return "localize_reads";
+  case CEPH_OSD_FLAG_RWORDERED: return "rwordered";
+  case CEPH_OSD_FLAG_IGNORE_CACHE: return "ignore_cache";
+  case CEPH_OSD_FLAG_SKIPRWLOCKS: return "skiprwlocks";
+  case CEPH_OSD_FLAG_IGNORE_OVERLAY: return "ignore_overlay";
+  case CEPH_OSD_FLAG_FLUSH: return "flush";
+  case CEPH_OSD_FLAG_MAP_SNAP_CLONE: return "map_snap_clone";
+  case CEPH_OSD_FLAG_ENFORCE_SNAPC: return "enforce_snapc";
+  default: return "???";
+  }
+}
+
+string ceph_osd_flag_string(unsigned flags)
+{
+  string s;
+  for (unsigned i=0; i<32; ++i) {
+    if (flags & (1u<dump_stream("name") << name;
+  f->dump_int("inc", inc);
+  f->dump_unsigned("tid", tid);
+}
+
+void osd_reqid_t::generate_test_instances(list& o)
+{
+  o.push_back(new osd_reqid_t);
+  o.push_back(new osd_reqid_t(entity_name_t::CLIENT(123), 1, 45678));
+}
+
+// -- object_locator_t --
+
+void object_locator_t::encode(bufferlist& bl) const
+{
+  // verify that nobody's corrupted the locator
+  assert(hash == -1 || key.empty());
+  __u8 encode_compat = 3;
+  ENCODE_START(6, encode_compat, bl);
+  ::encode(pool, bl);
+  int32_t preferred = -1;  // tell old code there is no preferred osd (-1).
+  ::encode(preferred, bl);
+  ::encode(key, bl);
+  ::encode(nspace, bl);
+  ::encode(hash, bl);
+  if (hash != -1)
+    encode_compat = MAX(encode_compat, 6); // need to interpret the hash
+  ENCODE_FINISH_NEW_COMPAT(bl, encode_compat);
+}
+
+void object_locator_t::decode(bufferlist::iterator& p)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, p);
+  if (struct_v < 2) {
+    int32_t op;
+    ::decode(op, p);
+    pool = op;
+    int16_t pref;
+    ::decode(pref, p);
+  } else {
+    ::decode(pool, p);
+    int32_t preferred;
+    ::decode(preferred, p);
+  }
+  ::decode(key, p);
+  if (struct_v >= 5)
+    ::decode(nspace, p);
+  if (struct_v >= 6)
+    ::decode(hash, p);
+  else
+    hash = -1;
+  DECODE_FINISH(p);
+  // verify that nobody's corrupted the locator
+  assert(hash == -1 || key.empty());
+}
+
+void object_locator_t::dump(Formatter *f) const
+{
+  f->dump_int("pool", pool);
+  f->dump_string("key", key);
+  f->dump_string("namespace", nspace);
+  f->dump_int("hash", hash);
+}
+
+void object_locator_t::generate_test_instances(list& o)
+{
+  o.push_back(new object_locator_t);
+  o.push_back(new object_locator_t(123));
+  o.push_back(new object_locator_t(123, 876));
+  o.push_back(new object_locator_t(1, "n2"));
+  o.push_back(new object_locator_t(1234, "", "key"));
+  o.push_back(new object_locator_t(12, "n1", "key2"));
+}
+
+// -- request_redirect_t --
+void request_redirect_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(redirect_locator, bl);
+  ::encode(redirect_object, bl);
+  ::encode(osd_instructions, bl);
+  ENCODE_FINISH(bl);
+}
+
+void request_redirect_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START(1, bl);
+  ::decode(redirect_locator, bl);
+  ::decode(redirect_object, bl);
+  ::decode(osd_instructions, bl);
+  DECODE_FINISH(bl);
+}
+
+void request_redirect_t::dump(Formatter *f) const
+{
+  f->dump_string("object", redirect_object);
+  f->open_object_section("locator");
+  redirect_locator.dump(f);
+  f->close_section(); // locator
+}
+
+void request_redirect_t::generate_test_instances(list& o)
+{
+  object_locator_t loc(1, "redir_obj");
+  o.push_back(new request_redirect_t());
+  o.push_back(new request_redirect_t(loc, 0));
+  o.push_back(new request_redirect_t(loc, "redir_obj"));
+  o.push_back(new request_redirect_t(loc));
+}
+
+void objectstore_perf_stat_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("commit_latency_ms", filestore_commit_latency);
+  f->dump_unsigned("apply_latency_ms", filestore_apply_latency);
+}
+
+void objectstore_perf_stat_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(filestore_commit_latency, bl);
+  ::encode(filestore_apply_latency, bl);
+  ENCODE_FINISH(bl);
+}
+
+void objectstore_perf_stat_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(filestore_commit_latency, bl);
+  ::decode(filestore_apply_latency, bl);
+  DECODE_FINISH(bl);
+}
+
+void objectstore_perf_stat_t::generate_test_instances(std::list& o)
+{
+  o.push_back(new objectstore_perf_stat_t());
+  o.push_back(new objectstore_perf_stat_t());
+  o.back()->filestore_commit_latency = 20;
+  o.back()->filestore_apply_latency = 30;
+}
+
+// -- osd_stat_t --
+void osd_stat_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("kb", kb);
+  f->dump_unsigned("kb_used", kb_used);
+  f->dump_unsigned("kb_avail", kb_avail);
+  f->open_array_section("hb_in");
+  for (vector::const_iterator p = hb_in.begin(); p != hb_in.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+  f->open_array_section("hb_out");
+  for (vector::const_iterator p = hb_out.begin(); p != hb_out.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+  f->dump_int("snap_trim_queue_len", snap_trim_queue_len);
+  f->dump_int("num_snap_trimming", num_snap_trimming);
+  f->open_object_section("op_queue_age_hist");
+  op_queue_age_hist.dump(f);
+  f->close_section();
+  f->open_object_section("fs_perf_stat");
+  fs_perf_stat.dump(f);
+  f->close_section();
+}
+
+void osd_stat_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(4, 2, bl);
+  ::encode(kb, bl);
+  ::encode(kb_used, bl);
+  ::encode(kb_avail, bl);
+  ::encode(snap_trim_queue_len, bl);
+  ::encode(num_snap_trimming, bl);
+  ::encode(hb_in, bl);
+  ::encode(hb_out, bl);
+  ::encode(op_queue_age_hist, bl);
+  ::encode(fs_perf_stat, bl);
+  ENCODE_FINISH(bl);
+}
+
+void osd_stat_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(4, 2, 2, bl);
+  ::decode(kb, bl);
+  ::decode(kb_used, bl);
+  ::decode(kb_avail, bl);
+  ::decode(snap_trim_queue_len, bl);
+  ::decode(num_snap_trimming, bl);
+  ::decode(hb_in, bl);
+  ::decode(hb_out, bl);
+  if (struct_v >= 3)
+    ::decode(op_queue_age_hist, bl);
+  if (struct_v >= 4)
+    ::decode(fs_perf_stat, bl);
+  DECODE_FINISH(bl);
+}
+
+void osd_stat_t::generate_test_instances(std::list& o)
+{
+  o.push_back(new osd_stat_t);
+
+  o.push_back(new osd_stat_t);
+  o.back()->kb = 1;
+  o.back()->kb_used = 2;
+  o.back()->kb_avail = 3;
+  o.back()->hb_in.push_back(5);
+  o.back()->hb_in.push_back(6);
+  o.back()->hb_out = o.back()->hb_in;
+  o.back()->hb_out.push_back(7);
+  o.back()->snap_trim_queue_len = 8;
+  o.back()->num_snap_trimming = 99;
+}
+
+// -- pg_t --
+
+int pg_t::print(char *o, int maxlen) const
+{
+  if (preferred() >= 0)
+    return snprintf(o, maxlen, "%llu.%xp%d", (unsigned long long)pool(), ps(), preferred());
+  else
+    return snprintf(o, maxlen, "%llu.%x", (unsigned long long)pool(), ps());
+}
+
+bool pg_t::parse(const char *s)
+{
+  uint64_t ppool;
+  uint32_t pseed;
+  int32_t pref;
+  int r = sscanf(s, "%llu.%xp%d", (long long unsigned *)&ppool, &pseed, &pref);
+  if (r < 2)
+    return false;
+  m_pool = ppool;
+  m_seed = pseed;
+  if (r == 3)
+    m_preferred = pref;
+  else
+    m_preferred = -1;
+  return true;
+}
+
+bool spg_t::parse(const char *s)
+{
+  pgid.set_preferred(-1);
+  shard = ghobject_t::NO_SHARD;
+  uint64_t ppool;
+  uint32_t pseed;
+  int32_t pref;
+  uint32_t pshard;
+  int r = sscanf(s, "%llu.%x", (long long unsigned *)&ppool, &pseed);
+  if (r < 2)
+    return false;
+  pgid.set_pool(ppool);
+  pgid.set_ps(pseed);
+
+  const char *p = strchr(s, 'p');
+  if (p) {
+    r = sscanf(p, "p%d", &pref);
+    if (r == 1) {
+      pgid.set_preferred(pref);
+    } else {
+      return false;
+    }
+  }
+
+  p = strchr(s, 's');
+  if (p) {
+    r = sscanf(p, "s%d", &pshard);
+    if (r == 1) {
+      shard = pshard;
+    } else {
+      return false;
+    }
+  }
+  return true;
+}
+
+ostream& operator<<(ostream& out, const spg_t &pg)
+{
+  out << pg.pgid;
+  if (!pg.is_no_shard())
+    out << "s" << (unsigned)pg.shard;
+  return out;
+}
+
+pg_t pg_t::get_ancestor(unsigned old_pg_num) const
+{
+  int old_bits = pg_pool_t::calc_bits_of(old_pg_num);
+  int old_mask = (1 << old_bits) - 1;
+  pg_t ret = *this;
+  ret.m_seed = ceph_stable_mod(m_seed, old_pg_num, old_mask);
+  return ret;
+}
+
+bool pg_t::is_split(unsigned old_pg_num, unsigned new_pg_num, set *children) const
+{
+  assert(m_seed < old_pg_num);
+  if (new_pg_num <= old_pg_num)
+    return false;
+
+  bool split = false;
+  if (true) {
+    int old_bits = pg_pool_t::calc_bits_of(old_pg_num);
+    int old_mask = (1 << old_bits) - 1;
+    for (int n = 1; ; n++) {
+      int next_bit = (n << (old_bits-1));
+      unsigned s = next_bit | m_seed;
+
+      if (s < old_pg_num || s == m_seed)
+	continue;
+      if (s >= new_pg_num)
+	break;
+      if ((unsigned)ceph_stable_mod(s, old_pg_num, old_mask) == m_seed) {
+	split = true;
+	if (children)
+	  children->insert(pg_t(s, m_pool, m_preferred));
+      }
+    }
+  }
+  if (false) {
+    // brute force
+    int old_bits = pg_pool_t::calc_bits_of(old_pg_num);
+    int old_mask = (1 << old_bits) - 1;
+    for (unsigned x = old_pg_num; x < new_pg_num; ++x) {
+      unsigned o = ceph_stable_mod(x, old_pg_num, old_mask);
+      if (o == m_seed) {
+	split = true;
+	children->insert(pg_t(x, m_pool, m_preferred));
+      }
+    }
+  }
+  return split;
+}
+
+unsigned pg_t::get_split_bits(unsigned pg_num) const {
+  if (pg_num == 1)
+    return 0;
+  assert(pg_num > 1);
+
+  // Find unique p such that pg_num \in [2^(p-1), 2^p)
+  unsigned p = pg_pool_t::calc_bits_of(pg_num);
+  assert(p); // silence coverity #751330 
+
+  if ((m_seed % (1<<(p-1))) < (pg_num % (1<<(p-1))))
+    return p;
+  else
+    return p - 1;
+}
+
+pg_t pg_t::get_parent() const
+{
+  unsigned bits = pg_pool_t::calc_bits_of(m_seed);
+  assert(bits);
+  pg_t retval = *this;
+  retval.m_seed &= ~((~0)<<(bits - 1));
+  return retval;
+}
+
+void pg_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("pool", m_pool);
+  f->dump_unsigned("seed", m_seed);
+  f->dump_int("preferred_osd", m_preferred);
+}
+
+void pg_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_t);
+  o.push_back(new pg_t(1, 2, -1));
+  o.push_back(new pg_t(13123, 3, -1));
+  o.push_back(new pg_t(131223, 4, 23));
+}
+
+ostream& operator<<(ostream& out, const pg_t &pg)
+{
+  out << pg.pool() << '.';
+  out << hex << pg.ps() << dec;
+
+  if (pg.preferred() >= 0)
+    out << 'p' << pg.preferred();
+
+  //out << "=" << hex << (__uint64_t)pg << dec;
+  return out;
+}
+
+
+// -- coll_t --
+
+const coll_t coll_t::META_COLL("meta");
+
+bool coll_t::is_temp(spg_t& pgid) const
+{
+  const char *cstr(str.c_str());
+  if (!pgid.parse(cstr))
+    return false;
+  const char *tmp_start = strchr(cstr, '_');
+  if (!tmp_start)
+    return false;
+  if (strncmp(tmp_start, "_TEMP", 5) == 0)
+    return true;
+  return false;
+}
+
+bool coll_t::is_pg(spg_t& pgid, snapid_t& snap) const
+{
+  const char *cstr(str.c_str());
+
+  if (!pgid.parse(cstr))
+    return false;
+  const char *snap_start = strchr(cstr, '_');
+  if (!snap_start)
+    return false;
+  if (strncmp(snap_start, "_head", 5) == 0) {
+    snap = CEPH_NOSNAP;
+  } else {
+    errno = 0;
+    snap = strtoull(snap_start+1, 0, 16);
+    if (errno)
+      return false;
+  }
+  return true;
+}
+
+bool coll_t::is_pg_prefix(spg_t& pgid) const
+{
+  const char *cstr(str.c_str());
+
+  if (!pgid.parse(cstr))
+    return false;
+  const char *snap_start = strchr(cstr, '_');
+  if (!snap_start)
+    return false;
+  return true;
+}
+
+bool coll_t::is_removal(uint64_t *seq, spg_t *pgid) const
+{
+  if (str.substr(0, 11) != string("FORREMOVAL_"))
+    return false;
+
+  stringstream ss(str.substr(11));
+  ss >> *seq;
+  char sep;
+  ss >> sep;
+  assert(sep == '_');
+  string pgid_str;
+  ss >> pgid_str;
+  if (!pgid->parse(pgid_str.c_str())) {
+    assert(0);
+    return false;
+  }
+  return true;
+}
+
+void coll_t::encode(bufferlist& bl) const
+{
+  __u8 struct_v = 3;
+  ::encode(struct_v, bl);
+  ::encode(str, bl);
+}
+
+void coll_t::decode(bufferlist::iterator& bl)
+{
+  __u8 struct_v;
+  ::decode(struct_v, bl);
+  switch (struct_v) {
+  case 1: {
+    spg_t pgid;
+    snapid_t snap;
+
+    ::decode(pgid, bl);
+    ::decode(snap, bl);
+    // infer the type
+    if (pgid == spg_t() && snap == 0)
+      str = "meta";
+    else
+      str = pg_and_snap_to_str(pgid, snap);
+    break;
+  }
+
+  case 2: {
+    __u8 type;
+    spg_t pgid;
+    snapid_t snap;
+    
+    ::decode(type, bl);
+    ::decode(pgid, bl);
+    ::decode(snap, bl);
+    switch (type) {
+    case 0:
+      str = "meta";
+      break;
+    case 1:
+      str = "temp";
+      break;
+    case 2:
+      str = pg_and_snap_to_str(pgid, snap);
+      break;
+    default: {
+      ostringstream oss;
+      oss << "coll_t::decode(): can't understand type " << type;
+      throw std::domain_error(oss.str());
+    }
+    }
+    break;
+  }
+
+  case 3:
+    ::decode(str, bl);
+    break;
+    
+  default: {
+    ostringstream oss;
+    oss << "coll_t::decode(): don't know how to decode version "
+	<< struct_v;
+    throw std::domain_error(oss.str());
+  }
+  }
+}
+
+void coll_t::dump(Formatter *f) const
+{
+  f->dump_string("name", str);
+}
+
+void coll_t::generate_test_instances(list& o)
+{
+  o.push_back(new coll_t);
+  o.push_back(new coll_t("meta"));
+  o.push_back(new coll_t("temp"));
+  o.push_back(new coll_t("foo"));
+  o.push_back(new coll_t("bar"));
+}
+
+// ---
+
+std::string pg_state_string(int state)
+{
+  ostringstream oss;
+  if (state & PG_STATE_STALE)
+    oss << "stale+";
+  if (state & PG_STATE_CREATING)
+    oss << "creating+";
+  if (state & PG_STATE_ACTIVE)
+    oss << "active+";
+  if (state & PG_STATE_CLEAN)
+    oss << "clean+";
+  if (state & PG_STATE_RECOVERY_WAIT)
+    oss << "recovery_wait+";
+  if (state & PG_STATE_RECOVERING)
+    oss << "recovering+";
+  if (state & PG_STATE_DOWN)
+    oss << "down+";
+  if (state & PG_STATE_REPLAY)
+    oss << "replay+";
+  if (state & PG_STATE_SPLITTING)
+    oss << "splitting+";
+  if (state & PG_STATE_DEGRADED)
+    oss << "degraded+";
+  if (state & PG_STATE_REMAPPED)
+    oss << "remapped+";
+  if (state & PG_STATE_SCRUBBING)
+    oss << "scrubbing+";
+  if (state & PG_STATE_DEEP_SCRUB)
+    oss << "deep+";
+  if (state & PG_STATE_SCRUBQ)
+    oss << "scrubq+";
+  if (state & PG_STATE_INCONSISTENT)
+    oss << "inconsistent+";
+  if (state & PG_STATE_PEERING)
+    oss << "peering+";
+  if (state & PG_STATE_REPAIR)
+    oss << "repair+";
+  if ((state & PG_STATE_BACKFILL_WAIT) &&
+      !(state &PG_STATE_BACKFILL))
+    oss << "wait_backfill+";
+  if (state & PG_STATE_BACKFILL)
+    oss << "backfilling+";
+  if (state & PG_STATE_BACKFILL_TOOFULL)
+    oss << "backfill_toofull+";
+  if (state & PG_STATE_INCOMPLETE)
+    oss << "incomplete+";
+  string ret(oss.str());
+  if (ret.length() > 0)
+    ret.resize(ret.length() - 1);
+  else
+    ret = "inactive";
+  return ret;
+}
+
+
+// -- eversion_t --
+string eversion_t::get_key_name() const
+{
+  char key[40];
+  snprintf(
+    key, sizeof(key), "%010u.%020llu", epoch, (long long unsigned)version);
+  return string(key);
+}
+
+
+// -- pool_snap_info_t --
+void pool_snap_info_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("snapid", snapid);
+  f->dump_stream("stamp") << stamp;
+  f->dump_string("name", name);
+}
+
+void pool_snap_info_t::encode(bufferlist& bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_PGPOOL3) == 0) {
+    __u8 struct_v = 1;
+    ::encode(struct_v, bl);
+    ::encode(snapid, bl);
+    ::encode(stamp, bl);
+    ::encode(name, bl);
+    return;
+  }
+  ENCODE_START(2, 2, bl);
+  ::encode(snapid, bl);
+  ::encode(stamp, bl);
+  ::encode(name, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pool_snap_info_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(snapid, bl);
+  ::decode(stamp, bl);
+  ::decode(name, bl);
+  DECODE_FINISH(bl);
+}
+
+void pool_snap_info_t::generate_test_instances(list& o)
+{
+  o.push_back(new pool_snap_info_t);
+  o.push_back(new pool_snap_info_t);
+  o.back()->snapid = 1;
+  o.back()->stamp = utime_t(1, 2);
+  o.back()->name = "foo";
+}
+
+
+// -- pg_pool_t --
+
+void pg_pool_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("flags", get_flags());
+  f->dump_string("flags_names", get_flags_string());
+  f->dump_int("type", get_type());
+  f->dump_int("size", get_size());
+  f->dump_int("min_size", get_min_size());
+  f->dump_int("crush_ruleset", get_crush_ruleset());
+  f->dump_int("object_hash", get_object_hash());
+  f->dump_int("pg_num", get_pg_num());
+  f->dump_int("pg_placement_num", get_pgp_num());
+  f->dump_unsigned("crash_replay_interval", get_crash_replay_interval());
+  f->dump_stream("last_change") << get_last_change();
+  f->dump_stream("last_force_op_resend") << get_last_force_op_resend();
+  f->dump_unsigned("auid", get_auid());
+  f->dump_string("snap_mode", is_pool_snaps_mode() ? "pool" : "selfmanaged");
+  f->dump_unsigned("snap_seq", get_snap_seq());
+  f->dump_unsigned("snap_epoch", get_snap_epoch());
+  f->open_array_section("pool_snaps");
+  for (map::const_iterator p = snaps.begin(); p != snaps.end(); ++p) {
+    f->open_object_section("pool_snap_info");
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->dump_stream("removed_snaps") << removed_snaps;
+  f->dump_int("quota_max_bytes", quota_max_bytes);
+  f->dump_int("quota_max_objects", quota_max_objects);
+  f->open_array_section("tiers");
+  for (set::const_iterator p = tiers.begin(); p != tiers.end(); ++p)
+    f->dump_int("pool_id", *p);
+  f->close_section();
+  f->dump_int("tier_of", tier_of);
+  f->dump_int("read_tier", read_tier);
+  f->dump_int("write_tier", write_tier);
+  f->dump_string("cache_mode", get_cache_mode_name());
+  f->dump_unsigned("target_max_bytes", target_max_bytes);
+  f->dump_unsigned("target_max_objects", target_max_objects);
+  f->dump_unsigned("cache_target_dirty_ratio_micro",
+		   cache_target_dirty_ratio_micro);
+  f->dump_unsigned("cache_target_full_ratio_micro",
+		   cache_target_full_ratio_micro);
+  f->dump_unsigned("cache_min_flush_age", cache_min_flush_age);
+  f->dump_unsigned("cache_min_evict_age", cache_min_evict_age);
+  f->dump_string("erasure_code_profile", erasure_code_profile);
+  f->open_object_section("hit_set_params");
+  hit_set_params.dump(f);
+  f->close_section(); // hit_set_params
+  f->dump_unsigned("hit_set_period", hit_set_period);
+  f->dump_unsigned("hit_set_count", hit_set_count);
+  f->dump_unsigned("stripe_width", get_stripe_width());
+}
+
+
+int pg_pool_t::calc_bits_of(int t)
+{
+  int b = 0;
+  while (t > 0) {
+    t = t >> 1;
+    b++;
+  }
+  return b;
+}
+
+void pg_pool_t::calc_pg_masks()
+{
+  pg_num_mask = (1 << calc_bits_of(pg_num-1)) - 1;
+  pgp_num_mask = (1 << calc_bits_of(pgp_num-1)) - 1;
+}
+
+unsigned pg_pool_t::get_pg_num_divisor(pg_t pgid) const
+{
+  if (pg_num == pg_num_mask + 1)
+    return pg_num;                    // power-of-2 split
+  unsigned mask = pg_num_mask >> 1;
+  if ((pgid.ps() & mask) < (pg_num & mask))
+    return pg_num_mask + 1;           // smaller bin size (already split)
+  else
+    return (pg_num_mask + 1) >> 1;    // bigger bin (not yet split)
+}
+
+/*
+ * we have two snap modes:
+ *  - pool global snaps
+ *    - snap existence/non-existence defined by snaps[] and snap_seq
+ *  - user managed snaps
+ *    - removal governed by removed_snaps
+ *
+ * we know which mode we're using based on whether removed_snaps is empty.
+ */
+bool pg_pool_t::is_pool_snaps_mode() const
+{
+  return removed_snaps.empty() && get_snap_seq() > 0;
+}
+
+bool pg_pool_t::is_unmanaged_snaps_mode() const
+{
+  return removed_snaps.size() && get_snap_seq() > 0;
+}
+
+bool pg_pool_t::is_removed_snap(snapid_t s) const
+{
+  if (is_pool_snaps_mode())
+    return s <= get_snap_seq() && snaps.count(s) == 0;
+  else
+    return removed_snaps.contains(s);
+}
+
+/*
+ * build set of known-removed sets from either pool snaps or
+ * explicit removed_snaps set.
+ */
+void pg_pool_t::build_removed_snaps(interval_set& rs) const
+{
+  if (is_pool_snaps_mode()) {
+    rs.clear();
+    for (snapid_t s = 1; s <= get_snap_seq(); s = s + 1)
+      if (snaps.count(s) == 0)
+	rs.insert(s);
+  } else {
+    rs = removed_snaps;
+  }
+}
+
+snapid_t pg_pool_t::snap_exists(const char *s) const
+{
+  for (map::const_iterator p = snaps.begin();
+       p != snaps.end();
+       ++p)
+    if (p->second.name == s)
+      return p->second.snapid;
+  return 0;
+}
+
+void pg_pool_t::add_snap(const char *n, utime_t stamp)
+{
+  assert(!is_unmanaged_snaps_mode());
+  snapid_t s = get_snap_seq() + 1;
+  snap_seq = s;
+  snaps[s].snapid = s;
+  snaps[s].name = n;
+  snaps[s].stamp = stamp;
+}
+
+void pg_pool_t::add_unmanaged_snap(uint64_t& snapid)
+{
+  if (removed_snaps.empty()) {
+    assert(!is_pool_snaps_mode());
+    removed_snaps.insert(snapid_t(1));
+    snap_seq = 1;
+  }
+  snapid = snap_seq = snap_seq + 1;
+}
+
+void pg_pool_t::remove_snap(snapid_t s)
+{
+  assert(snaps.count(s));
+  snaps.erase(s);
+  snap_seq = snap_seq + 1;
+}
+
+void pg_pool_t::remove_unmanaged_snap(snapid_t s)
+{
+  assert(is_unmanaged_snaps_mode());
+  removed_snaps.insert(s);
+  snap_seq = snap_seq + 1;
+  removed_snaps.insert(get_snap_seq());
+}
+
+SnapContext pg_pool_t::get_snap_context() const
+{
+  vector s(snaps.size());
+  unsigned i = 0;
+  for (map::const_reverse_iterator p = snaps.rbegin();
+       p != snaps.rend();
+       ++p)
+    s[i++] = p->first;
+  return SnapContext(get_snap_seq(), s);
+}
+
+static string make_hash_str(const string &inkey, const string &nspace)
+{
+  if (nspace.empty())
+    return inkey;
+  return nspace + '\037' + inkey;
+}
+
+uint32_t pg_pool_t::hash_key(const string& key, const string& ns) const
+{
+  string n = make_hash_str(key, ns);
+  return ceph_str_hash(object_hash, n.c_str(), n.length());
+}
+
+uint32_t pg_pool_t::raw_hash_to_pg(uint32_t v) const
+{
+  return ceph_stable_mod(v, pg_num, pg_num_mask);
+}
+
+/*
+ * map a raw pg (with full precision ps) into an actual pg, for storage
+ */
+pg_t pg_pool_t::raw_pg_to_pg(pg_t pg) const
+{
+  pg.set_ps(ceph_stable_mod(pg.ps(), pg_num, pg_num_mask));
+  return pg;
+}
+  
+/*
+ * map raw pg (full precision ps) into a placement seed.  include
+ * pool id in that value so that different pools don't use the same
+ * seeds.
+ */
+ps_t pg_pool_t::raw_pg_to_pps(pg_t pg) const
+{
+  if (flags & FLAG_HASHPSPOOL) {
+    // Hash the pool id so that pool PGs do not overlap.
+    return
+      crush_hash32_2(CRUSH_HASH_RJENKINS1,
+		     ceph_stable_mod(pg.ps(), pgp_num, pgp_num_mask),
+		     pg.pool());
+  } else {
+    // Legacy behavior; add ps and pool together.  This is not a great
+    // idea because the PGs from each pool will essentially overlap on
+    // top of each other: 0.5 == 1.4 == 2.3 == ...
+    return
+      ceph_stable_mod(pg.ps(), pgp_num, pgp_num_mask) +
+      pg.pool();
+  }
+}
+
+uint32_t pg_pool_t::get_random_pg_position(pg_t pg, uint32_t seed) const
+{
+  uint32_t r = crush_hash32_2(CRUSH_HASH_RJENKINS1, seed, 123);
+  if (pg_num == pg_num_mask + 1) {
+    r &= ~pg_num_mask;
+  } else {
+    unsigned smaller_mask = pg_num_mask >> 1;
+    if ((pg.ps() & smaller_mask) < (pg_num & smaller_mask)) {
+      r &= ~pg_num_mask;
+    } else {
+      r &= ~smaller_mask;
+    }
+  }
+  r |= pg.ps();
+  return r;
+}
+
+void pg_pool_t::encode(bufferlist& bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_PGPOOL3) == 0) {
+    // this encoding matches the old struct ceph_pg_pool
+    __u8 struct_v = 2;
+    ::encode(struct_v, bl);
+    ::encode(type, bl);
+    ::encode(size, bl);
+    ::encode(crush_ruleset, bl);
+    ::encode(object_hash, bl);
+    ::encode(pg_num, bl);
+    ::encode(pgp_num, bl);
+    __u32 lpg_num = 0, lpgp_num = 0;  // tell old code that there are no localized pgs.
+    ::encode(lpg_num, bl);
+    ::encode(lpgp_num, bl);
+    ::encode(last_change, bl);
+    ::encode(snap_seq, bl);
+    ::encode(snap_epoch, bl);
+
+    __u32 n = snaps.size();
+    ::encode(n, bl);
+    n = removed_snaps.num_intervals();
+    ::encode(n, bl);
+
+    ::encode(auid, bl);
+
+    ::encode_nohead(snaps, bl, features);
+    removed_snaps.encode_nohead(bl);
+    return;
+  }
+
+  if ((features & CEPH_FEATURE_OSDENC) == 0) {
+    __u8 struct_v = 4;
+    ::encode(struct_v, bl);
+    ::encode(type, bl);
+    ::encode(size, bl);
+    ::encode(crush_ruleset, bl);
+    ::encode(object_hash, bl);
+    ::encode(pg_num, bl);
+    ::encode(pgp_num, bl);
+    __u32 lpg_num = 0, lpgp_num = 0;  // tell old code that there are no localized pgs.
+    ::encode(lpg_num, bl);
+    ::encode(lpgp_num, bl);
+    ::encode(last_change, bl);
+    ::encode(snap_seq, bl);
+    ::encode(snap_epoch, bl);
+    ::encode(snaps, bl, features);
+    ::encode(removed_snaps, bl);
+    ::encode(auid, bl);
+    ::encode(flags, bl);
+    ::encode(crash_replay_interval, bl);
+    return;
+  }
+
+  __u8 encode_compat = 5;
+  ENCODE_START(15, encode_compat, bl);
+  ::encode(type, bl);
+  ::encode(size, bl);
+  ::encode(crush_ruleset, bl);
+  ::encode(object_hash, bl);
+  ::encode(pg_num, bl);
+  ::encode(pgp_num, bl);
+  __u32 lpg_num = 0, lpgp_num = 0;  // tell old code that there are no localized pgs.
+  ::encode(lpg_num, bl);
+  ::encode(lpgp_num, bl);
+  ::encode(last_change, bl);
+  ::encode(snap_seq, bl);
+  ::encode(snap_epoch, bl);
+  ::encode(snaps, bl, features);
+  ::encode(removed_snaps, bl);
+  ::encode(auid, bl);
+  ::encode(flags, bl);
+  ::encode(crash_replay_interval, bl);
+  ::encode(min_size, bl);
+  ::encode(quota_max_bytes, bl);
+  ::encode(quota_max_objects, bl);
+  ::encode(tiers, bl);
+  ::encode(tier_of, bl);
+  __u8 c = cache_mode;
+  ::encode(c, bl);
+  ::encode(read_tier, bl);
+  ::encode(write_tier, bl);
+  ::encode(properties, bl);
+  ::encode(hit_set_params, bl);
+  ::encode(hit_set_period, bl);
+  ::encode(hit_set_count, bl);
+  ::encode(stripe_width, bl);
+  ::encode(target_max_bytes, bl);
+  ::encode(target_max_objects, bl);
+  ::encode(cache_target_dirty_ratio_micro, bl);
+  ::encode(cache_target_full_ratio_micro, bl);
+  ::encode(cache_min_flush_age, bl);
+  ::encode(cache_min_evict_age, bl);
+  ::encode(erasure_code_profile, bl);
+  ::encode(last_force_op_resend, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_pool_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(15, 5, 5, bl);
+  ::decode(type, bl);
+  ::decode(size, bl);
+  ::decode(crush_ruleset, bl);
+  ::decode(object_hash, bl);
+  ::decode(pg_num, bl);
+  ::decode(pgp_num, bl);
+  {
+    __u32 lpg_num, lpgp_num;
+    ::decode(lpg_num, bl);
+    ::decode(lpgp_num, bl);
+  }
+  ::decode(last_change, bl);
+  ::decode(snap_seq, bl);
+  ::decode(snap_epoch, bl);
+
+  if (struct_v >= 3) {
+    ::decode(snaps, bl);
+    ::decode(removed_snaps, bl);
+    ::decode(auid, bl);
+  } else {
+    __u32 n, m;
+    ::decode(n, bl);
+    ::decode(m, bl);
+    ::decode(auid, bl);
+    ::decode_nohead(n, snaps, bl);
+    removed_snaps.decode_nohead(m, bl);
+  }
+
+  if (struct_v >= 4) {
+    ::decode(flags, bl);
+    ::decode(crash_replay_interval, bl);
+  } else {
+    flags = 0;
+
+    // if this looks like the 'data' pool, set the
+    // crash_replay_interval appropriately.  unfortunately, we can't
+    // be precise here.  this should be good enough to preserve replay
+    // on the data pool for the majority of cluster upgrades, though.
+    if (crush_ruleset == 0 && auid == 0)
+      crash_replay_interval = 60;
+    else
+      crash_replay_interval = 0;
+  }
+  if (struct_v >= 7) {
+    ::decode(min_size, bl);
+  } else {
+    min_size = size - size/2;
+  }
+  if (struct_v >= 8) {
+    ::decode(quota_max_bytes, bl);
+    ::decode(quota_max_objects, bl);
+  }
+  if (struct_v >= 9) {
+    ::decode(tiers, bl);
+    ::decode(tier_of, bl);
+    __u8 v;
+    ::decode(v, bl);
+    cache_mode = (cache_mode_t)v;
+    ::decode(read_tier, bl);
+    ::decode(write_tier, bl);
+  }
+  if (struct_v >= 10) {
+    ::decode(properties, bl);
+  }
+  if (struct_v >= 11) {
+    ::decode(hit_set_params, bl);
+    ::decode(hit_set_period, bl);
+    ::decode(hit_set_count, bl);
+  } else {
+    pg_pool_t def;
+    hit_set_period = def.hit_set_period;
+    hit_set_count = def.hit_set_count;
+  }
+  if (struct_v >= 12) {
+    ::decode(stripe_width, bl);
+  } else {
+    set_stripe_width(0);
+  }
+  if (struct_v >= 13) {
+    ::decode(target_max_bytes, bl);
+    ::decode(target_max_objects, bl);
+    ::decode(cache_target_dirty_ratio_micro, bl);
+    ::decode(cache_target_full_ratio_micro, bl);
+    ::decode(cache_min_flush_age, bl);
+    ::decode(cache_min_evict_age, bl);
+  } else {
+    target_max_bytes = 0;
+    target_max_objects = 0;
+    cache_target_dirty_ratio_micro = 0;
+    cache_target_full_ratio_micro = 0;
+    cache_min_flush_age = 0;
+    cache_min_evict_age = 0;
+  }
+  if (struct_v >= 14) {
+    ::decode(erasure_code_profile, bl);
+  }
+  if (struct_v >= 15) {
+    ::decode(last_force_op_resend, bl);
+  } else {
+    last_force_op_resend = 0;
+  }
+  DECODE_FINISH(bl);
+  calc_pg_masks();
+}
+
+void pg_pool_t::generate_test_instances(list& o)
+{
+  pg_pool_t a;
+  o.push_back(new pg_pool_t(a));
+
+  a.type = TYPE_REPLICATED;
+  a.size = 2;
+  a.crush_ruleset = 3;
+  a.object_hash = 4;
+  a.pg_num = 6;
+  a.pgp_num = 5;
+  a.last_change = 9;
+  a.last_force_op_resend = 123823;
+  a.snap_seq = 10;
+  a.snap_epoch = 11;
+  a.auid = 12;
+  a.crash_replay_interval = 13;
+  a.quota_max_bytes = 473;
+  a.quota_max_objects = 474;
+  o.push_back(new pg_pool_t(a));
+
+  a.snaps[3].name = "asdf";
+  a.snaps[3].snapid = 3;
+  a.snaps[3].stamp = utime_t(123, 4);
+  a.snaps[6].name = "qwer";
+  a.snaps[6].snapid = 6;
+  a.snaps[6].stamp = utime_t(23423, 4);
+  o.push_back(new pg_pool_t(a));
+
+  a.removed_snaps.insert(2);   // not quite valid to combine with snaps!
+  a.quota_max_bytes = 2473;
+  a.quota_max_objects = 4374;
+  a.tiers.insert(0);
+  a.tiers.insert(1);
+  a.tier_of = 2;
+  a.cache_mode = CACHEMODE_WRITEBACK;
+  a.read_tier = 1;
+  a.write_tier = 1;
+  a.hit_set_params = HitSet::Params(new BloomHitSet::Params);
+  a.hit_set_period = 3600;
+  a.hit_set_count = 8;
+  a.set_stripe_width(12345);
+  a.target_max_bytes = 1238132132;
+  a.target_max_objects = 1232132;
+  a.cache_target_dirty_ratio_micro = 187232;
+  a.cache_target_full_ratio_micro = 987222;
+  a.cache_min_flush_age = 231;
+  a.cache_min_evict_age = 2321;
+  a.erasure_code_profile = "profile in osdmap";
+  o.push_back(new pg_pool_t(a));
+}
+
+ostream& operator<<(ostream& out, const pg_pool_t& p)
+{
+  out << p.get_type_name()
+      << " size " << p.get_size()
+      << " min_size " << p.get_min_size()
+      << " crush_ruleset " << p.get_crush_ruleset()
+      << " object_hash " << p.get_object_hash_name()
+      << " pg_num " << p.get_pg_num()
+      << " pgp_num " << p.get_pgp_num()
+      << " last_change " << p.get_last_change();
+  if (p.get_last_force_op_resend())
+    out << " lfor " << p.get_last_force_op_resend();
+  if (p.get_auid())
+    out << " owner " << p.get_auid();
+  if (p.flags)
+    out << " flags " << p.get_flags_string();
+  if (p.crash_replay_interval)
+    out << " crash_replay_interval " << p.crash_replay_interval;
+  if (p.quota_max_bytes)
+    out << " max_bytes " << p.quota_max_bytes;
+  if (p.quota_max_objects)
+    out << " max_objects " << p.quota_max_objects;
+  if (!p.tiers.empty())
+    out << " tiers " << p.tiers;
+  if (p.is_tier())
+    out << " tier_of " << p.tier_of;
+  if (p.has_read_tier())
+    out << " read_tier " << p.read_tier;
+  if (p.has_write_tier())
+    out << " write_tier " << p.write_tier;
+  if (p.cache_mode)
+    out << " cache_mode " << p.get_cache_mode_name();
+  if (p.target_max_bytes)
+    out << " target_bytes " << p.target_max_bytes;
+  if (p.target_max_objects)
+    out << " target_objects " << p.target_max_objects;
+  if (p.hit_set_params.get_type() != HitSet::TYPE_NONE) {
+    out << " hit_set " << p.hit_set_params
+	<< " " << p.hit_set_period << "s"
+	<< " x" << p.hit_set_count;
+  }
+  out << " stripe_width " << p.get_stripe_width();
+  return out;
+}
+
+
+// -- object_stat_sum_t --
+
+void object_stat_sum_t::dump(Formatter *f) const
+{
+  f->dump_int("num_bytes", num_bytes);
+  f->dump_int("num_objects", num_objects);
+  f->dump_int("num_object_clones", num_object_clones);
+  f->dump_int("num_object_copies", num_object_copies);
+  f->dump_int("num_objects_missing_on_primary", num_objects_missing_on_primary);
+  f->dump_int("num_objects_degraded", num_objects_degraded);
+  f->dump_int("num_objects_unfound", num_objects_unfound);
+  f->dump_int("num_objects_dirty", num_objects_dirty);
+  f->dump_int("num_whiteouts", num_whiteouts);
+  f->dump_int("num_read", num_rd);
+  f->dump_int("num_read_kb", num_rd_kb);
+  f->dump_int("num_write", num_wr);
+  f->dump_int("num_write_kb", num_wr_kb);
+  f->dump_int("num_scrub_errors", num_scrub_errors);
+  f->dump_int("num_shallow_scrub_errors", num_shallow_scrub_errors);
+  f->dump_int("num_deep_scrub_errors", num_deep_scrub_errors);
+  f->dump_int("num_objects_recovered", num_objects_recovered);
+  f->dump_int("num_bytes_recovered", num_bytes_recovered);
+  f->dump_int("num_keys_recovered", num_keys_recovered);
+  f->dump_int("num_objects_omap", num_objects_omap);
+  f->dump_int("num_objects_hit_set_archive", num_objects_hit_set_archive);
+}
+
+void object_stat_sum_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(9, 3, bl);
+  ::encode(num_bytes, bl);
+  ::encode(num_objects, bl);
+  ::encode(num_object_clones, bl);
+  ::encode(num_object_copies, bl);
+  ::encode(num_objects_missing_on_primary, bl);
+  ::encode(num_objects_degraded, bl);
+  ::encode(num_objects_unfound, bl);
+  ::encode(num_rd, bl);
+  ::encode(num_rd_kb, bl);
+  ::encode(num_wr, bl);
+  ::encode(num_wr_kb, bl);
+  ::encode(num_scrub_errors, bl);
+  ::encode(num_objects_recovered, bl);
+  ::encode(num_bytes_recovered, bl);
+  ::encode(num_keys_recovered, bl);
+  ::encode(num_shallow_scrub_errors, bl);
+  ::encode(num_deep_scrub_errors, bl);
+  ::encode(num_objects_dirty, bl);
+  ::encode(num_whiteouts, bl);
+  ::encode(num_objects_omap, bl);
+  ::encode(num_objects_hit_set_archive, bl);
+  ENCODE_FINISH(bl);
+}
+
+void object_stat_sum_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(9, 3, 3, bl);
+  ::decode(num_bytes, bl);
+  if (struct_v < 3) {
+    uint64_t num_kb;
+    ::decode(num_kb, bl);
+  }
+  ::decode(num_objects, bl);
+  ::decode(num_object_clones, bl);
+  ::decode(num_object_copies, bl);
+  ::decode(num_objects_missing_on_primary, bl);
+  ::decode(num_objects_degraded, bl);
+  if (struct_v >= 2)
+    ::decode(num_objects_unfound, bl);
+  ::decode(num_rd, bl);
+  ::decode(num_rd_kb, bl);
+  ::decode(num_wr, bl);
+  ::decode(num_wr_kb, bl);
+  if (struct_v >= 4)
+    ::decode(num_scrub_errors, bl);
+  else
+    num_scrub_errors = 0;
+  if (struct_v >= 5) {
+    ::decode(num_objects_recovered, bl);
+    ::decode(num_bytes_recovered, bl);
+    ::decode(num_keys_recovered, bl);
+  } else {
+    num_objects_recovered = 0;
+    num_bytes_recovered = 0;
+    num_keys_recovered = 0;
+  }
+  if (struct_v >= 6) {
+    ::decode(num_shallow_scrub_errors, bl);
+    ::decode(num_deep_scrub_errors, bl);
+  } else {
+    num_shallow_scrub_errors = 0;
+    num_deep_scrub_errors = 0;
+  }
+  if (struct_v >= 7) {
+    ::decode(num_objects_dirty, bl);
+    ::decode(num_whiteouts, bl);
+  } else {
+    num_objects_dirty = 0;
+    num_whiteouts = 0;
+  }
+  if (struct_v >= 8) {
+    ::decode(num_objects_omap, bl);
+  } else {
+    num_objects_omap = 0;
+  }
+  if (struct_v >= 9) {
+    ::decode(num_objects_hit_set_archive, bl);
+  } else {
+    num_objects_hit_set_archive = 0;
+  }
+  DECODE_FINISH(bl);
+}
+
+void object_stat_sum_t::generate_test_instances(list& o)
+{
+  object_stat_sum_t a;
+  o.push_back(new object_stat_sum_t(a));
+
+  a.num_bytes = 1;
+  a.num_objects = 3;
+  a.num_object_clones = 4;
+  a.num_object_copies = 5;
+  a.num_objects_missing_on_primary = 6;
+  a.num_objects_degraded = 7;
+  a.num_objects_unfound = 8;
+  a.num_rd = 9; a.num_rd_kb = 10;
+  a.num_wr = 11; a.num_wr_kb = 12;
+  a.num_objects_recovered = 14;
+  a.num_bytes_recovered = 15;
+  a.num_keys_recovered = 16;
+  a.num_deep_scrub_errors = 17;
+  a.num_shallow_scrub_errors = 18;
+  a.num_scrub_errors = a.num_deep_scrub_errors + a.num_shallow_scrub_errors;
+  a.num_objects_dirty = 21;
+  a.num_whiteouts = 22;
+  o.push_back(new object_stat_sum_t(a));
+}
+
+void object_stat_sum_t::add(const object_stat_sum_t& o)
+{
+  num_bytes += o.num_bytes;
+  num_objects += o.num_objects;
+  num_object_clones += o.num_object_clones;
+  num_object_copies += o.num_object_copies;
+  num_objects_missing_on_primary += o.num_objects_missing_on_primary;
+  num_objects_degraded += o.num_objects_degraded;
+  num_rd += o.num_rd;
+  num_rd_kb += o.num_rd_kb;
+  num_wr += o.num_wr;
+  num_wr_kb += o.num_wr_kb;
+  num_objects_unfound += o.num_objects_unfound;
+  num_scrub_errors += o.num_scrub_errors;
+  num_shallow_scrub_errors += o.num_shallow_scrub_errors;
+  num_deep_scrub_errors += o.num_deep_scrub_errors;
+  num_objects_recovered += o.num_objects_recovered;
+  num_bytes_recovered += o.num_bytes_recovered;
+  num_keys_recovered += o.num_keys_recovered;
+  num_objects_dirty += o.num_objects_dirty;
+  num_whiteouts += o.num_whiteouts;
+  num_objects_omap += o.num_objects_omap;
+  num_objects_hit_set_archive += o.num_objects_hit_set_archive;
+}
+
+void object_stat_sum_t::sub(const object_stat_sum_t& o)
+{
+  num_bytes -= o.num_bytes;
+  num_objects -= o.num_objects;
+  num_object_clones -= o.num_object_clones;
+  num_object_copies -= o.num_object_copies;
+  num_objects_missing_on_primary -= o.num_objects_missing_on_primary;
+  num_objects_degraded -= o.num_objects_degraded;
+  num_rd -= o.num_rd;
+  num_rd_kb -= o.num_rd_kb;
+  num_wr -= o.num_wr;
+  num_wr_kb -= o.num_wr_kb;
+  num_objects_unfound -= o.num_objects_unfound;
+  num_scrub_errors -= o.num_scrub_errors;
+  num_shallow_scrub_errors -= o.num_shallow_scrub_errors;
+  num_deep_scrub_errors -= o.num_deep_scrub_errors;
+  num_objects_recovered -= o.num_objects_recovered;
+  num_bytes_recovered -= o.num_bytes_recovered;
+  num_keys_recovered -= o.num_keys_recovered;
+  num_objects_dirty -= o.num_objects_dirty;
+  num_whiteouts -= o.num_whiteouts;
+  num_objects_omap -= o.num_objects_omap;
+  num_objects_hit_set_archive -= o.num_objects_hit_set_archive;
+}
+
+
+// -- object_stat_collection_t --
+
+void object_stat_collection_t::dump(Formatter *f) const
+{
+  f->open_object_section("stat_sum");
+  sum.dump(f);
+  f->close_section();
+  f->open_object_section("stat_cat_sum");
+  for (map::const_iterator p = cat_sum.begin(); p != cat_sum.end(); ++p) {
+    f->open_object_section(p->first.c_str());
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void object_stat_collection_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(sum, bl);
+  ::encode(cat_sum, bl);
+  ENCODE_FINISH(bl);
+}
+
+void object_stat_collection_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(sum, bl);
+  ::decode(cat_sum, bl);
+  DECODE_FINISH(bl);
+}
+
+void object_stat_collection_t::generate_test_instances(list& o)
+{
+  object_stat_collection_t a;
+  o.push_back(new object_stat_collection_t(a));
+  list l;
+  object_stat_sum_t::generate_test_instances(l);
+  char n[2] = { 'a', 0 };
+  for (list::iterator p = l.begin(); p != l.end(); ++p) {
+    a.add(**p, n);
+    n[0]++;
+    o.push_back(new object_stat_collection_t(a));
+  }
+}
+
+
+// -- pg_stat_t --
+
+void pg_stat_t::dump(Formatter *f) const
+{
+  f->dump_stream("version") << version;
+  f->dump_stream("reported_seq") << reported_seq;
+  f->dump_stream("reported_epoch") << reported_epoch;
+  f->dump_string("state", pg_state_string(state));
+  f->dump_stream("last_fresh") << last_fresh;
+  f->dump_stream("last_change") << last_change;
+  f->dump_stream("last_active") << last_active;
+  f->dump_stream("last_clean") << last_clean;
+  f->dump_stream("last_became_active") << last_became_active;
+  f->dump_stream("last_unstale") << last_unstale;
+  f->dump_unsigned("mapping_epoch", mapping_epoch);
+  f->dump_stream("log_start") << log_start;
+  f->dump_stream("ondisk_log_start") << ondisk_log_start;
+  f->dump_unsigned("created", created);
+  f->dump_unsigned("last_epoch_clean", last_epoch_clean);
+  f->dump_stream("parent") << parent;
+  f->dump_unsigned("parent_split_bits", parent_split_bits);
+  f->dump_stream("last_scrub") << last_scrub;
+  f->dump_stream("last_scrub_stamp") << last_scrub_stamp;
+  f->dump_stream("last_deep_scrub") << last_deep_scrub;
+  f->dump_stream("last_deep_scrub_stamp") << last_deep_scrub_stamp;
+  f->dump_stream("last_clean_scrub_stamp") << last_clean_scrub_stamp;
+  f->dump_unsigned("log_size", log_size);
+  f->dump_unsigned("ondisk_log_size", ondisk_log_size);
+  f->dump_stream("stats_invalid") << stats_invalid;
+  stats.dump(f);
+  f->open_array_section("up");
+  for (vector::const_iterator p = up.begin(); p != up.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+  f->open_array_section("acting");
+  for (vector::const_iterator p = acting.begin(); p != acting.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+  f->dump_int("up_primary", up_primary);
+  f->dump_int("acting_primary", acting_primary);
+}
+
+void pg_stat_t::dump_brief(Formatter *f) const
+{
+  f->dump_string("state", pg_state_string(state));
+  f->open_array_section("up");
+  for (vector::const_iterator p = up.begin(); p != up.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+  f->open_array_section("acting");
+  for (vector::const_iterator p = acting.begin(); p != acting.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+  f->dump_int("up_primary", up_primary);
+  f->dump_int("acting_primary", acting_primary);
+}
+
+void pg_stat_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(17, 8, bl);
+  ::encode(version, bl);
+  ::encode(reported_seq, bl);
+  ::encode(reported_epoch, bl);
+  ::encode(state, bl);
+  ::encode(log_start, bl);
+  ::encode(ondisk_log_start, bl);
+  ::encode(created, bl);
+  ::encode(last_epoch_clean, bl);
+  ::encode(parent, bl);
+  ::encode(parent_split_bits, bl);
+  ::encode(last_scrub, bl);
+  ::encode(last_scrub_stamp, bl);
+  ::encode(stats, bl);
+  ::encode(log_size, bl);
+  ::encode(ondisk_log_size, bl);
+  ::encode(up, bl);
+  ::encode(acting, bl);
+  ::encode(last_fresh, bl);
+  ::encode(last_change, bl);
+  ::encode(last_active, bl);
+  ::encode(last_clean, bl);
+  ::encode(last_unstale, bl);
+  ::encode(mapping_epoch, bl);
+  ::encode(last_deep_scrub, bl);
+  ::encode(last_deep_scrub_stamp, bl);
+  ::encode(stats_invalid, bl);
+  ::encode(last_clean_scrub_stamp, bl);
+  ::encode(last_became_active, bl);
+  ::encode(dirty_stats_invalid, bl);
+  ::encode(up_primary, bl);
+  ::encode(acting_primary, bl);
+  ::encode(omap_stats_invalid, bl);
+  ::encode(hitset_stats_invalid, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_stat_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(17, 8, 8, bl);
+  ::decode(version, bl);
+  ::decode(reported_seq, bl);
+  ::decode(reported_epoch, bl);
+  ::decode(state, bl);
+  ::decode(log_start, bl);
+  ::decode(ondisk_log_start, bl);
+  ::decode(created, bl);
+  if (struct_v >= 7)
+    ::decode(last_epoch_clean, bl);
+  else
+    last_epoch_clean = 0;
+  if (struct_v < 6) {
+    old_pg_t opgid;
+    ::decode(opgid, bl);
+    parent = opgid;
+  } else {
+    ::decode(parent, bl);
+  }
+  ::decode(parent_split_bits, bl);
+  ::decode(last_scrub, bl);
+  ::decode(last_scrub_stamp, bl);
+  if (struct_v <= 4) {
+    ::decode(stats.sum.num_bytes, bl);
+    uint64_t num_kb;
+    ::decode(num_kb, bl);
+    ::decode(stats.sum.num_objects, bl);
+    ::decode(stats.sum.num_object_clones, bl);
+    ::decode(stats.sum.num_object_copies, bl);
+    ::decode(stats.sum.num_objects_missing_on_primary, bl);
+    ::decode(stats.sum.num_objects_degraded, bl);
+    ::decode(log_size, bl);
+    ::decode(ondisk_log_size, bl);
+    if (struct_v >= 2) {
+      ::decode(stats.sum.num_rd, bl);
+      ::decode(stats.sum.num_rd_kb, bl);
+      ::decode(stats.sum.num_wr, bl);
+      ::decode(stats.sum.num_wr_kb, bl);
+    }
+    if (struct_v >= 3) {
+      ::decode(up, bl);
+    }
+    if (struct_v == 4) {
+      ::decode(stats.sum.num_objects_unfound, bl);  // sigh.
+    }
+    ::decode(acting, bl);
+  } else {
+    ::decode(stats, bl);
+    ::decode(log_size, bl);
+    ::decode(ondisk_log_size, bl);
+    ::decode(up, bl);
+    ::decode(acting, bl);
+    if (struct_v >= 9) {
+      ::decode(last_fresh, bl);
+      ::decode(last_change, bl);
+      ::decode(last_active, bl);
+      ::decode(last_clean, bl);
+      ::decode(last_unstale, bl);
+      ::decode(mapping_epoch, bl);
+      if (struct_v >= 10) {
+        ::decode(last_deep_scrub, bl);
+        ::decode(last_deep_scrub_stamp, bl);
+      }
+    }
+  }
+  if (struct_v < 11) {
+    stats_invalid = false;
+  } else {
+    ::decode(stats_invalid, bl);
+  }
+  if (struct_v >= 12) {
+    ::decode(last_clean_scrub_stamp, bl);
+  } else {
+    last_clean_scrub_stamp = utime_t();
+  }
+  if (struct_v >= 13) {
+    ::decode(last_became_active, bl);
+  } else {
+    last_became_active = last_active;
+  }
+  if (struct_v >= 14) {
+    ::decode(dirty_stats_invalid, bl);
+  } else {
+    // if we are decoding an old encoding of this object, then the
+    // encoder may not have supported num_objects_dirty accounting.
+    dirty_stats_invalid = true;
+  }
+  if (struct_v >= 15) {
+    ::decode(up_primary, bl);
+    ::decode(acting_primary, bl);
+  } else {
+    up_primary = up.size() ? up[0] : -1;
+    acting_primary = acting.size() ? acting[0] : -1;
+  }
+  if (struct_v >= 16) {
+    ::decode(omap_stats_invalid, bl);
+  } else {
+    // if we are decoding an old encoding of this object, then the
+    // encoder may not have supported num_objects_omap accounting.
+    omap_stats_invalid = true;
+  }
+  if (struct_v >= 17) {
+    ::decode(hitset_stats_invalid, bl);
+  } else {
+    // if we are decoding an old encoding of this object, then the
+    // encoder may not have supported num_objects_hit_set_archive accounting.
+    hitset_stats_invalid = true;
+  }
+  DECODE_FINISH(bl);
+}
+
+void pg_stat_t::generate_test_instances(list& o)
+{
+  pg_stat_t a;
+  o.push_back(new pg_stat_t(a));
+
+  a.version = eversion_t(1, 3);
+  a.reported_epoch = 1;
+  a.reported_seq = 2;
+  a.state = 123;
+  a.mapping_epoch = 998;
+  a.last_fresh = utime_t(1002, 1);
+  a.last_change = utime_t(1002, 2);
+  a.last_active = utime_t(1002, 3);
+  a.last_clean = utime_t(1002, 4);
+  a.last_unstale = utime_t(1002, 5);
+  a.log_start = eversion_t(1, 4);
+  a.ondisk_log_start = eversion_t(1, 5);
+  a.created = 6;
+  a.last_epoch_clean = 7;
+  a.parent = pg_t(1, 2, 3);
+  a.parent_split_bits = 12;
+  a.last_scrub = eversion_t(9, 10);
+  a.last_scrub_stamp = utime_t(11, 12);
+  a.last_deep_scrub = eversion_t(13, 14);
+  a.last_deep_scrub_stamp = utime_t(15, 16);
+  a.last_clean_scrub_stamp = utime_t(17, 18);
+  list l;
+  object_stat_collection_t::generate_test_instances(l);
+  a.stats = *l.back();
+  a.log_size = 99;
+  a.ondisk_log_size = 88;
+  a.up.push_back(123);
+  a.up_primary = 123;
+  a.acting.push_back(456);
+  a.acting_primary = 456;
+  o.push_back(new pg_stat_t(a));
+
+  a.up.push_back(124);
+  a.up_primary = 124;
+  a.acting.push_back(124);
+  a.acting_primary = 124;
+  o.push_back(new pg_stat_t(a));
+}
+
+
+// -- pool_stat_t --
+
+void pool_stat_t::dump(Formatter *f) const
+{
+  stats.dump(f);
+  f->dump_int("log_size", log_size);
+  f->dump_int("ondisk_log_size", ondisk_log_size);
+}
+
+void pool_stat_t::encode(bufferlist &bl, uint64_t features) const
+{
+  if ((features & CEPH_FEATURE_OSDENC) == 0) {
+    __u8 v = 4;
+    ::encode(v, bl);
+    ::encode(stats, bl);
+    ::encode(log_size, bl);
+    ::encode(ondisk_log_size, bl);
+    return;
+  }
+
+  ENCODE_START(5, 5, bl);
+  ::encode(stats, bl);
+  ::encode(log_size, bl);
+  ::encode(ondisk_log_size, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pool_stat_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(5, 5, 5, bl);
+  if (struct_v >= 4) {
+    ::decode(stats, bl);
+    ::decode(log_size, bl);
+    ::decode(ondisk_log_size, bl);
+  } else {
+    ::decode(stats.sum.num_bytes, bl);
+    uint64_t num_kb;
+    ::decode(num_kb, bl);
+    ::decode(stats.sum.num_objects, bl);
+    ::decode(stats.sum.num_object_clones, bl);
+    ::decode(stats.sum.num_object_copies, bl);
+    ::decode(stats.sum.num_objects_missing_on_primary, bl);
+    ::decode(stats.sum.num_objects_degraded, bl);
+    ::decode(log_size, bl);
+    ::decode(ondisk_log_size, bl);
+    if (struct_v >= 2) {
+      ::decode(stats.sum.num_rd, bl);
+      ::decode(stats.sum.num_rd_kb, bl);
+      ::decode(stats.sum.num_wr, bl);
+      ::decode(stats.sum.num_wr_kb, bl);
+    }
+    if (struct_v >= 3) {
+      ::decode(stats.sum.num_objects_unfound, bl);
+    }
+  }
+  DECODE_FINISH(bl);
+}
+
+void pool_stat_t::generate_test_instances(list& o)
+{
+  pool_stat_t a;
+  o.push_back(new pool_stat_t(a));
+
+  list l;
+  object_stat_collection_t::generate_test_instances(l);
+  a.stats = *l.back();
+  a.log_size = 123;
+  a.ondisk_log_size = 456;
+  o.push_back(new pool_stat_t(a));
+}
+
+
+// -- pg_history_t --
+
+void pg_history_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(6, 4, bl);
+  ::encode(epoch_created, bl);
+  ::encode(last_epoch_started, bl);
+  ::encode(last_epoch_clean, bl);
+  ::encode(last_epoch_split, bl);
+  ::encode(same_interval_since, bl);
+  ::encode(same_up_since, bl);
+  ::encode(same_primary_since, bl);
+  ::encode(last_scrub, bl);
+  ::encode(last_scrub_stamp, bl);
+  ::encode(last_deep_scrub, bl);
+  ::encode(last_deep_scrub_stamp, bl);
+  ::encode(last_clean_scrub_stamp, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_history_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(6, 4, 4, bl);
+  ::decode(epoch_created, bl);
+  ::decode(last_epoch_started, bl);
+  if (struct_v >= 3)
+    ::decode(last_epoch_clean, bl);
+  else
+    last_epoch_clean = last_epoch_started;  // careful, it's a lie!
+  ::decode(last_epoch_split, bl);
+  ::decode(same_interval_since, bl);
+  ::decode(same_up_since, bl);
+  ::decode(same_primary_since, bl);
+  if (struct_v >= 2) {
+    ::decode(last_scrub, bl);
+    ::decode(last_scrub_stamp, bl);
+  }
+  if (struct_v >= 5) {
+    ::decode(last_deep_scrub, bl);
+    ::decode(last_deep_scrub_stamp, bl);
+  }
+  if (struct_v >= 6) {
+    ::decode(last_clean_scrub_stamp, bl);
+  }
+  DECODE_FINISH(bl);
+}
+
+void pg_history_t::dump(Formatter *f) const
+{
+  f->dump_int("epoch_created", epoch_created);
+  f->dump_int("last_epoch_started", last_epoch_started);
+  f->dump_int("last_epoch_clean", last_epoch_clean);
+  f->dump_int("last_epoch_split", last_epoch_split);
+  f->dump_int("same_up_since", same_up_since);
+  f->dump_int("same_interval_since", same_interval_since);
+  f->dump_int("same_primary_since", same_primary_since);
+  f->dump_stream("last_scrub") << last_scrub;
+  f->dump_stream("last_scrub_stamp") << last_scrub_stamp;
+  f->dump_stream("last_deep_scrub") << last_deep_scrub;
+  f->dump_stream("last_deep_scrub_stamp") << last_deep_scrub_stamp;
+  f->dump_stream("last_clean_scrub_stamp") << last_clean_scrub_stamp;
+}
+
+void pg_history_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_history_t);
+  o.push_back(new pg_history_t);
+  o.back()->epoch_created = 1;
+  o.back()->last_epoch_started = 2;
+  o.back()->last_epoch_clean = 3;
+  o.back()->last_epoch_split = 4;
+  o.back()->same_up_since = 5;
+  o.back()->same_interval_since = 6;
+  o.back()->same_primary_since = 7;
+  o.back()->last_scrub = eversion_t(8, 9);
+  o.back()->last_scrub_stamp = utime_t(10, 11);
+  o.back()->last_deep_scrub = eversion_t(12, 13);
+  o.back()->last_deep_scrub_stamp = utime_t(14, 15);
+  o.back()->last_clean_scrub_stamp = utime_t(16, 17);
+}
+
+
+// -- pg_info_t --
+
+void pg_info_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(30, 26, bl);
+  ::encode(pgid.pgid, bl);
+  ::encode(last_update, bl);
+  ::encode(last_complete, bl);
+  ::encode(log_tail, bl);
+  ::encode(last_backfill, bl);
+  ::encode(stats, bl);
+  history.encode(bl);
+  ::encode(purged_snaps, bl);
+  ::encode(last_epoch_started, bl);
+  ::encode(last_user_version, bl);
+  ::encode(hit_set, bl);
+  ::encode(pgid.shard, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_info_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(29, 26, 26, bl);
+  if (struct_v < 23) {
+    old_pg_t opgid;
+    ::decode(opgid, bl);
+    pgid.pgid = opgid;
+  } else {
+    ::decode(pgid.pgid, bl);
+  }
+  ::decode(last_update, bl);
+  ::decode(last_complete, bl);
+  ::decode(log_tail, bl);
+  if (struct_v < 25) {
+    bool log_backlog;
+    ::decode(log_backlog, bl);
+  }
+  if (struct_v >= 24)
+    ::decode(last_backfill, bl);
+  ::decode(stats, bl);
+  history.decode(bl);
+  if (struct_v >= 22)
+    ::decode(purged_snaps, bl);
+  else {
+    set snap_trimq;
+    ::decode(snap_trimq, bl);
+  }
+  if (struct_v < 27) {
+    last_epoch_started = history.last_epoch_started;
+  } else {
+    ::decode(last_epoch_started, bl);
+  }
+  if (struct_v >= 28)
+    ::decode(last_user_version, bl);
+  else
+    last_user_version = last_update.version;
+  if (struct_v >= 29)
+    ::decode(hit_set, bl);
+  if (struct_v >= 30)
+    ::decode(pgid.shard, bl);
+  else
+    pgid.shard = ghobject_t::no_shard();
+  DECODE_FINISH(bl);
+}
+
+// -- pg_info_t --
+
+void pg_info_t::dump(Formatter *f) const
+{
+  f->dump_stream("pgid") << pgid;
+  f->dump_stream("last_update") << last_update;
+  f->dump_stream("last_complete") << last_complete;
+  f->dump_stream("log_tail") << log_tail;
+  f->dump_int("last_user_version", last_user_version);
+  f->dump_stream("last_backfill") << last_backfill;
+  f->dump_stream("purged_snaps") << purged_snaps;
+  f->open_object_section("history");
+  history.dump(f);
+  f->close_section();
+  f->open_object_section("stats");
+  stats.dump(f);
+  f->close_section();
+
+  f->dump_int("empty", is_empty());
+  f->dump_int("dne", dne());
+  f->dump_int("incomplete", is_incomplete());
+  f->dump_int("last_epoch_started", last_epoch_started);
+
+  f->open_object_section("hit_set_history");
+  hit_set.dump(f);
+  f->close_section();
+}
+
+void pg_info_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_info_t);
+  o.push_back(new pg_info_t);
+  list h;
+  pg_history_t::generate_test_instances(h);
+  o.back()->history = *h.back();
+  o.back()->pgid = spg_t(pg_t(1, 2, -1), ghobject_t::no_shard());
+  o.back()->last_update = eversion_t(3, 4);
+  o.back()->last_complete = eversion_t(5, 6);
+  o.back()->last_user_version = 2;
+  o.back()->log_tail = eversion_t(7, 8);
+  o.back()->last_backfill = hobject_t(object_t("objname"), "key", 123, 456, -1, "");
+  {
+    list s;
+    pg_stat_t::generate_test_instances(s);
+    o.back()->stats = *s.back();
+  }
+  {
+    list s;
+    pg_hit_set_history_t::generate_test_instances(s);
+    o.back()->hit_set = *s.back();
+  }
+}
+
+// -- pg_notify_t --
+void pg_notify_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 1, bl);
+  ::encode(query_epoch, bl);
+  ::encode(epoch_sent, bl);
+  ::encode(info, bl);
+  ::encode(to, bl);
+  ::encode(from, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_notify_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(2, bl);
+  ::decode(query_epoch, bl);
+  ::decode(epoch_sent, bl);
+  ::decode(info, bl);
+  if (struct_v >= 2) {
+    ::decode(to, bl);
+    ::decode(from, bl);
+  } else {
+    to = ghobject_t::NO_SHARD;
+    from = ghobject_t::NO_SHARD;
+  }
+  DECODE_FINISH(bl);
+}
+
+void pg_notify_t::dump(Formatter *f) const
+{
+  f->dump_int("from", from);
+  f->dump_int("to", to);
+  f->dump_unsigned("query_epoch", query_epoch);
+  f->dump_unsigned("epoch_sent", epoch_sent);
+  {
+    f->open_object_section("info");
+    info.dump(f);
+    f->close_section();
+  }
+}
+
+void pg_notify_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_notify_t(3, ghobject_t::NO_SHARD, 1 ,1 , pg_info_t()));
+  o.push_back(new pg_notify_t(0, 0, 3, 10, pg_info_t()));
+}
+
+ostream &operator<<(ostream &lhs, const pg_notify_t ¬ify)
+{
+  lhs << "(query_epoch:" << notify.query_epoch
+      << ", epoch_sent:" << notify.epoch_sent
+      << ", info:" << notify.info;
+  if (notify.from != ghobject_t::NO_SHARD ||
+      notify.to != ghobject_t::NO_SHARD)
+    lhs << " " << (unsigned)notify.from
+	<< "->" << (unsigned)notify.to;
+  return lhs << ")";
+}
+
+// -- pg_interval_t --
+
+void pg_interval_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(4, 2, bl);
+  ::encode(first, bl);
+  ::encode(last, bl);
+  ::encode(up, bl);
+  ::encode(acting, bl);
+  ::encode(maybe_went_rw, bl);
+  ::encode(primary, bl);
+  ::encode(up_primary, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_interval_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(4, 2, 2, bl);
+  ::decode(first, bl);
+  ::decode(last, bl);
+  ::decode(up, bl);
+  ::decode(acting, bl);
+  ::decode(maybe_went_rw, bl);
+  if (struct_v >= 3) {
+    ::decode(primary, bl);
+  } else {
+    if (acting.size())
+      primary = acting[0];
+  }
+  if (struct_v >= 4) {
+    ::decode(up_primary, bl);
+  } else {
+    if (up.size())
+      up_primary = up[0];
+  }
+  DECODE_FINISH(bl);
+}
+
+void pg_interval_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("first", first);
+  f->dump_unsigned("last", last);
+  f->dump_int("maybe_went_rw", maybe_went_rw ? 1 : 0);
+  f->open_array_section("up");
+  for (vector::const_iterator p = up.begin(); p != up.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+  f->open_array_section("acting");
+  for (vector::const_iterator p = acting.begin(); p != acting.end(); ++p)
+    f->dump_int("osd", *p);
+  f->dump_int("primary", primary);
+  f->dump_int("up_primary", up_primary);
+  f->close_section();
+}
+
+void pg_interval_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_interval_t);
+  o.push_back(new pg_interval_t);
+  o.back()->up.push_back(1);
+  o.back()->acting.push_back(2);
+  o.back()->acting.push_back(3);
+  o.back()->first = 4;
+  o.back()->last = 5;
+  o.back()->maybe_went_rw = true;
+}
+
+bool pg_interval_t::check_new_interval(
+  int old_acting_primary,
+  int new_acting_primary,
+  const vector &old_acting,
+  const vector &new_acting,
+  int old_up_primary,
+  int new_up_primary,
+  const vector &old_up,
+  const vector &new_up,
+  epoch_t same_interval_since,
+  epoch_t last_epoch_clean,
+  OSDMapRef osdmap,
+  OSDMapRef lastmap,
+  int64_t pool_id,
+  pg_t pgid,
+  map *past_intervals,
+  std::ostream *out)
+{
+  // remember past interval
+  //  NOTE: a change in the up set primary triggers an interval
+  //  change, even though the interval members in the pg_interval_t
+  //  do not change.
+  if (old_acting_primary != new_acting_primary ||
+      new_acting != old_acting ||
+      old_up_primary != new_up_primary ||
+      new_up != old_up ||
+      (!(lastmap->get_pools().count(pool_id))) ||
+      (lastmap->get_pools().find(pool_id)->second.min_size !=
+       osdmap->get_pools().find(pool_id)->second.min_size)  ||
+      pgid.is_split(lastmap->get_pg_num(pgid.pool()),
+        osdmap->get_pg_num(pgid.pool()), 0)) {
+    pg_interval_t& i = (*past_intervals)[same_interval_since];
+    i.first = same_interval_since;
+    i.last = osdmap->get_epoch() - 1;
+    i.acting = old_acting;
+    i.up = old_up;
+    i.primary = old_acting_primary;
+    i.up_primary = old_up_primary;
+
+    if (!i.acting.empty() && i.primary != -1 &&
+	i.acting.size() >=
+	lastmap->get_pools().find(pool_id)->second.min_size) {
+      if (out)
+	*out << "generate_past_intervals " << i
+	     << ": not rw,"
+	     << " up_thru " << lastmap->get_up_thru(i.primary)
+	     << " up_from " << lastmap->get_up_from(i.primary)
+	     << " last_epoch_clean " << last_epoch_clean
+	     << std::endl;
+      if (lastmap->get_up_thru(i.primary) >= i.first &&
+	  lastmap->get_up_from(i.primary) <= i.first) {
+	i.maybe_went_rw = true;
+	if (out)
+	  *out << "generate_past_intervals " << i
+	       << " : primary up " << lastmap->get_up_from(i.primary)
+	       << "-" << lastmap->get_up_thru(i.primary)
+	       << " includes interval"
+	       << std::endl;
+      } else if (last_epoch_clean >= i.first &&
+		 last_epoch_clean <= i.last) {
+	// If the last_epoch_clean is included in this interval, then
+	// the pg must have been rw (for recovery to have completed).
+	// This is important because we won't know the _real_
+	// first_epoch because we stop at last_epoch_clean, and we
+	// don't want the oldest interval to randomly have
+	// maybe_went_rw false depending on the relative up_thru vs
+	// last_epoch_clean timing.
+	i.maybe_went_rw = true;
+	if (out)
+	  *out << "generate_past_intervals " << i
+	       << " : includes last_epoch_clean " << last_epoch_clean
+	       << " and presumed to have been rw"
+	       << std::endl;
+      } else {
+	i.maybe_went_rw = false;
+	if (out)
+	  *out << "generate_past_intervals " << i
+	       << " : primary up " << lastmap->get_up_from(i.primary)
+	       << "-" << lastmap->get_up_thru(i.primary)
+	       << " does not include interval"
+	       << std::endl;
+      }
+    } else {
+      i.maybe_went_rw = false;
+      if (out)
+	*out << "generate_past_intervals " << i << " : acting set is too small" << std::endl;
+    }
+    return true;
+  } else {
+    return false;
+  }
+}
+
+ostream& operator<<(ostream& out, const pg_interval_t& i)
+{
+  out << "interval(" << i.first << "-" << i.last
+      << " up " << i.up << "(" << i.up_primary << ")"
+      << " acting " << i.acting << "(" << i.primary << ")";
+  if (i.maybe_went_rw)
+    out << " maybe_went_rw";
+  out << ")";
+  return out;
+}
+
+
+
+// -- pg_query_t --
+
+void pg_query_t::encode(bufferlist &bl, uint64_t features) const {
+  if (features & CEPH_FEATURE_QUERY_T) {
+    ENCODE_START(3, 2, bl);
+    ::encode(type, bl);
+    ::encode(since, bl);
+    history.encode(bl);
+    ::encode(epoch_sent, bl);
+    ::encode(to, bl);
+    ::encode(from, bl);
+    ENCODE_FINISH(bl);
+  } else {
+    ::encode(type, bl);
+    ::encode(since, bl);
+    history.encode(bl);
+  }
+}
+
+void pg_query_t::decode(bufferlist::iterator &bl) {
+  bufferlist::iterator bl2 = bl;
+  try {
+    DECODE_START(3, bl);
+    ::decode(type, bl);
+    ::decode(since, bl);
+    history.decode(bl);
+    ::decode(epoch_sent, bl);
+    if (struct_v >= 3) {
+      ::decode(to, bl);
+      ::decode(from, bl);
+    } else {
+      to = ghobject_t::NO_SHARD;
+      from = ghobject_t::NO_SHARD;
+    }
+    DECODE_FINISH(bl);
+  } catch (...) {
+    bl = bl2;
+    ::decode(type, bl);
+    ::decode(since, bl);
+    history.decode(bl);
+  }
+}
+
+void pg_query_t::dump(Formatter *f) const
+{
+  f->dump_int("from", from);
+  f->dump_int("to", to);
+  f->dump_string("type", get_type_name());
+  f->dump_stream("since") << since;
+  f->dump_stream("epoch_sent") << epoch_sent;
+  f->open_object_section("history");
+  history.dump(f);
+  f->close_section();
+}
+void pg_query_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_query_t());
+  list h;
+  pg_history_t::generate_test_instances(h);
+  o.push_back(new pg_query_t(pg_query_t::INFO, 1, 2, *h.back(), 4));
+  o.push_back(new pg_query_t(pg_query_t::MISSING, 2, 3, *h.back(), 4));
+  o.push_back(new pg_query_t(pg_query_t::LOG, 0, 0,
+			     eversion_t(4, 5), *h.back(), 4));
+  o.push_back(new pg_query_t(pg_query_t::FULLLOG,
+			     ghobject_t::NO_SHARD, ghobject_t::NO_SHARD,
+			     *h.back(), 5));
+}
+
+// -- ObjectModDesc --
+void ObjectModDesc::visit(Visitor *visitor) const
+{
+  bufferlist::iterator bp = bl.begin();
+  try {
+    while (!bp.end()) {
+      DECODE_START(1, bp);
+      uint8_t code;
+      ::decode(code, bp);
+      switch (code) {
+      case APPEND: {
+	uint64_t size;
+	::decode(size, bp);
+	visitor->append(size);
+	break;
+      }
+      case SETATTRS: {
+	map > attrs;
+	::decode(attrs, bp);
+	visitor->setattrs(attrs);
+	break;
+      }
+      case DELETE: {
+	version_t old_version;
+	::decode(old_version, bp);
+	visitor->rmobject(old_version);
+	break;
+      }
+      case CREATE: {
+	visitor->create();
+	break;
+      }
+      case UPDATE_SNAPS: {
+	set snaps;
+	::decode(snaps, bp);
+	visitor->update_snaps(snaps);
+	break;
+      }
+      default:
+	assert(0 == "Invalid rollback code");
+      }
+      DECODE_FINISH(bp);
+    }
+  } catch (...) {
+    assert(0 == "Invalid encoding");
+  }
+}
+
+struct DumpVisitor : public ObjectModDesc::Visitor {
+  Formatter *f;
+  DumpVisitor(Formatter *f) : f(f) {}
+  void append(uint64_t old_size) {
+    f->open_object_section("op");
+    f->dump_string("code", "APPEND");
+    f->dump_unsigned("old_size", old_size);
+    f->close_section();
+  }
+  void setattrs(map > &attrs) {
+    f->open_object_section("op");
+    f->dump_string("code", "SETATTRS");
+    f->open_array_section("attrs");
+    for (map >::iterator i = attrs.begin();
+	 i != attrs.end();
+	 ++i) {
+      f->dump_string("attr_name", i->first);
+    }
+    f->close_section();
+    f->close_section();
+  }
+  void rmobject(version_t old_version) {
+    f->open_object_section("op");
+    f->dump_string("code", "RMOBJECT");
+    f->dump_unsigned("old_version", old_version);
+    f->close_section();
+  }
+  void create() {
+    f->open_object_section("op");
+    f->dump_string("code", "CREATE");
+    f->close_section();
+  }
+  void update_snaps(set &snaps) {
+    f->open_object_section("op");
+    f->dump_string("code", "UPDATE_SNAPS");
+    f->dump_stream("snaps") << snaps;
+    f->close_section();
+  }
+};
+
+void ObjectModDesc::dump(Formatter *f) const
+{
+  f->open_object_section("object_mod_desc");
+  f->dump_bool("can_local_rollback", can_local_rollback);
+  f->dump_bool("rollback_info_completed", rollback_info_completed);
+  {
+    f->open_array_section("ops");
+    DumpVisitor vis(f);
+    visit(&vis);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ObjectModDesc::generate_test_instances(list& o)
+{
+  map > attrs;
+  attrs[OI_ATTR];
+  attrs[SS_ATTR];
+  attrs["asdf"];
+  o.push_back(new ObjectModDesc());
+  o.back()->append(100);
+  o.back()->setattrs(attrs);
+  o.push_back(new ObjectModDesc());
+  o.back()->rmobject(1001);
+  o.push_back(new ObjectModDesc());
+  o.back()->create();
+  o.back()->setattrs(attrs);
+  o.push_back(new ObjectModDesc());
+  o.back()->create();
+  o.back()->setattrs(attrs);
+  o.back()->mark_unrollbackable();
+  o.back()->append(1000);
+}
+
+void ObjectModDesc::encode(bufferlist &_bl) const
+{
+  ENCODE_START(1, 1, _bl);
+  ::encode(can_local_rollback, _bl);
+  ::encode(rollback_info_completed, _bl);
+  ::encode(bl, _bl);
+  ENCODE_FINISH(_bl);
+}
+void ObjectModDesc::decode(bufferlist::iterator &_bl)
+{
+  DECODE_START(1, _bl);
+  ::decode(can_local_rollback, _bl);
+  ::decode(rollback_info_completed, _bl);
+  ::decode(bl, _bl);
+  DECODE_FINISH(_bl);
+}
+
+// -- pg_log_entry_t --
+
+string pg_log_entry_t::get_key_name() const
+{
+  return version.get_key_name();
+}
+
+void pg_log_entry_t::encode_with_checksum(bufferlist& bl) const
+{
+  bufferlist ebl(sizeof(*this)*2);
+  encode(ebl);
+  __u32 crc = ebl.crc32c(0);
+  ::encode(ebl, bl);
+  ::encode(crc, bl);
+}
+
+void pg_log_entry_t::decode_with_checksum(bufferlist::iterator& p)
+{
+  bufferlist bl;
+  ::decode(bl, p);
+  __u32 crc;
+  ::decode(crc, p);
+  if (crc != bl.crc32c(0))
+    throw buffer::malformed_input("bad checksum on pg_log_entry_t");
+  bufferlist::iterator q = bl.begin();
+  decode(q);
+}
+
+void pg_log_entry_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(9, 4, bl);
+  ::encode(op, bl);
+  ::encode(soid, bl);
+  ::encode(version, bl);
+
+  /**
+   * Added with reverting_to:
+   * Previous code used prior_version to encode
+   * what we now call reverting_to.  This will
+   * allow older code to decode reverting_to
+   * into prior_version as expected.
+   */
+  if (op == LOST_REVERT)
+    ::encode(reverting_to, bl);
+  else
+    ::encode(prior_version, bl);
+
+  ::encode(reqid, bl);
+  ::encode(mtime, bl);
+  if (op == LOST_REVERT)
+    ::encode(prior_version, bl);
+  ::encode(snaps, bl);
+  ::encode(user_version, bl);
+  ::encode(mod_desc, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_log_entry_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(8, 4, 4, bl);
+  ::decode(op, bl);
+  if (struct_v < 2) {
+    sobject_t old_soid;
+    ::decode(old_soid, bl);
+    soid.oid = old_soid.oid;
+    soid.snap = old_soid.snap;
+    invalid_hash = true;
+  } else {
+    ::decode(soid, bl);
+  }
+  if (struct_v < 3)
+    invalid_hash = true;
+  ::decode(version, bl);
+
+  if (struct_v >= 6 && op == LOST_REVERT)
+    ::decode(reverting_to, bl);
+  else
+    ::decode(prior_version, bl);
+
+  ::decode(reqid, bl);
+  ::decode(mtime, bl);
+  if (struct_v < 5)
+    invalid_pool = true;
+
+  if (op == LOST_REVERT) {
+    if (struct_v >= 6) {
+      ::decode(prior_version, bl);
+    } else {
+      reverting_to = prior_version;
+    }
+  }
+  if (struct_v >= 7 ||  // for v >= 7, this is for all ops.
+      op == CLONE) {    // for v < 7, it's only present for CLONE.
+    ::decode(snaps, bl);
+  }
+
+  if (struct_v >= 8)
+    ::decode(user_version, bl);
+  else
+    user_version = version.version;
+
+  if (struct_v >= 9)
+    ::decode(mod_desc, bl);
+  else
+    mod_desc.mark_unrollbackable();
+
+  DECODE_FINISH(bl);
+}
+
+void pg_log_entry_t::dump(Formatter *f) const
+{
+  f->dump_string("op", get_op_name());
+  f->dump_stream("object") << soid;
+  f->dump_stream("version") << version;
+  f->dump_stream("prior_version") << version;
+  f->dump_stream("reqid") << reqid;
+  f->dump_stream("mtime") << mtime;
+  if (snaps.length() > 0) {
+    vector v;
+    bufferlist c = snaps;
+    bufferlist::iterator p = c.begin();
+    try {
+      ::decode(v, p);
+    } catch (...) {
+      v.clear();
+    }
+    f->open_object_section("snaps");
+    for (vector::iterator p = v.begin(); p != v.end(); ++p)
+      f->dump_unsigned("snap", *p);
+    f->close_section();
+  }
+  {
+    f->open_object_section("mod_desc");
+    mod_desc.dump(f);
+    f->close_section();
+  }
+}
+
+void pg_log_entry_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_log_entry_t());
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  o.push_back(new pg_log_entry_t(MODIFY, oid, eversion_t(1,2), eversion_t(3,4),
+				 1, osd_reqid_t(entity_name_t::CLIENT(777), 8, 999),
+				 utime_t(8,9)));
+}
+
+ostream& operator<<(ostream& out, const pg_log_entry_t& e)
+{
+  out << e.version << " (" << e.prior_version << ") "
+      << e.get_op_name() << ' ' << e.soid << " by " << e.reqid << " " << e.mtime;
+  if (e.snaps.length()) {
+    vector snaps;
+    bufferlist c = e.snaps;
+    bufferlist::iterator p = c.begin();
+    try {
+      ::decode(snaps, p);
+    } catch (...) {
+      snaps.clear();
+    }
+    out << " snaps " << snaps;
+  }
+  return out;
+}
+
+
+// -- pg_log_t --
+
+void pg_log_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(6, 3, bl);
+  ::encode(head, bl);
+  ::encode(tail, bl);
+  ::encode(log, bl);
+  ::encode(can_rollback_to, bl);
+  ::encode(rollback_info_trimmed_to, bl);
+  ENCODE_FINISH(bl);
+}
+ 
+void pg_log_t::decode(bufferlist::iterator &bl, int64_t pool)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
+  ::decode(head, bl);
+  ::decode(tail, bl);
+  if (struct_v < 2) {
+    bool backlog;
+    ::decode(backlog, bl);
+  }
+  ::decode(log, bl);
+  if (struct_v >= 5)
+    ::decode(can_rollback_to, bl);
+
+  if (struct_v >= 6)
+    ::decode(rollback_info_trimmed_to, bl);
+  else
+    rollback_info_trimmed_to = tail;
+  DECODE_FINISH(bl);
+
+  // handle hobject_t format change
+  if (struct_v < 4) {
+    for (list::iterator i = log.begin();
+	 i != log.end();
+	 ++i) {
+      if (!i->soid.is_max() && i->soid.pool == -1)
+	i->soid.pool = pool;
+    }
+  }
+}
+
+void pg_log_t::dump(Formatter *f) const
+{
+  f->dump_stream("head") << head;
+  f->dump_stream("tail") << head;
+  f->open_array_section("log");
+  for (list::const_iterator p = log.begin(); p != log.end(); ++p) {
+    f->open_object_section("entry");
+    p->dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void pg_log_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_log_t);
+
+  // this is nonsensical:
+  o.push_back(new pg_log_t);
+  o.back()->head = eversion_t(1,2);
+  o.back()->tail = eversion_t(3,4);
+  list e;
+  pg_log_entry_t::generate_test_instances(e);
+  for (list::iterator p = e.begin(); p != e.end(); ++p)
+    o.back()->log.push_back(**p);
+}
+
+void pg_log_t::copy_after(const pg_log_t &other, eversion_t v) 
+{
+  can_rollback_to = other.can_rollback_to;
+  head = other.head;
+  tail = other.tail;
+  for (list::const_reverse_iterator i = other.log.rbegin();
+       i != other.log.rend();
+       ++i) {
+    assert(i->version > other.tail);
+    if (i->version <= v) {
+      // make tail accurate.
+      tail = i->version;
+      break;
+    }
+    log.push_front(*i);
+  }
+}
+
+void pg_log_t::copy_range(const pg_log_t &other, eversion_t from, eversion_t to)
+{
+  can_rollback_to = other.can_rollback_to;
+  list::const_reverse_iterator i = other.log.rbegin();
+  assert(i != other.log.rend());
+  while (i->version > to) {
+    ++i;
+    assert(i != other.log.rend());
+  }
+  assert(i->version == to);
+  head = to;
+  for ( ; i != other.log.rend(); ++i) {
+    if (i->version <= from) {
+      tail = i->version;
+      break;
+    }
+    log.push_front(*i);
+  }
+}
+
+void pg_log_t::copy_up_to(const pg_log_t &other, int max)
+{
+  can_rollback_to = other.can_rollback_to;
+  int n = 0;
+  head = other.head;
+  tail = other.tail;
+  for (list::const_reverse_iterator i = other.log.rbegin();
+       i != other.log.rend();
+       ++i) {
+    if (n++ >= max) {
+      tail = i->version;
+      break;
+    }
+    log.push_front(*i);
+  }
+}
+
+ostream& pg_log_t::print(ostream& out) const 
+{
+  out << *this << std::endl;
+  for (list::const_iterator p = log.begin();
+       p != log.end();
+       ++p) 
+    out << *p << std::endl;
+  return out;
+}
+
+
+// -- pg_missing_t --
+
+void pg_missing_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(3, 2, bl);
+  ::encode(missing, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_missing_t::decode(bufferlist::iterator &bl, int64_t pool)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
+  ::decode(missing, bl);
+  DECODE_FINISH(bl);
+
+  if (struct_v < 3) {
+    // Handle hobject_t upgrade
+    map tmp;
+    for (map::iterator i = missing.begin();
+	 i != missing.end();
+      ) {
+      if (!i->first.is_max() && i->first.pool == -1) {
+	hobject_t to_insert(i->first);
+	to_insert.pool = pool;
+	tmp[to_insert] = i->second;
+	missing.erase(i++);
+      } else {
+	++i;
+      }
+    }
+    missing.insert(tmp.begin(), tmp.end());
+  }
+
+  for (map::iterator it = missing.begin();
+       it != missing.end();
+       ++it)
+    rmissing[it->second.need.version] = it->first;
+}
+
+void pg_missing_t::dump(Formatter *f) const
+{
+  f->open_array_section("missing");
+  for (map::const_iterator p = missing.begin(); p != missing.end(); ++p) {
+    f->open_object_section("item");
+    f->dump_stream("object") << p->first;
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void pg_missing_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_missing_t);
+  o.push_back(new pg_missing_t);
+  o.back()->add(hobject_t(object_t("foo"), "foo", 123, 456, 0, ""), eversion_t(5, 6), eversion_t(5, 1));
+}
+
+ostream& operator<<(ostream& out, const pg_missing_t::item& i) 
+{
+  out << i.need;
+  if (i.have != eversion_t())
+    out << "(" << i.have << ")";
+  return out;
+}
+
+ostream& operator<<(ostream& out, const pg_missing_t& missing) 
+{
+  out << "missing(" << missing.num_missing();
+  //if (missing.num_lost()) out << ", " << missing.num_lost() << " lost";
+  out << ")";
+  return out;
+}
+
+
+unsigned int pg_missing_t::num_missing() const
+{
+  return missing.size();
+}
+
+bool pg_missing_t::have_missing() const
+{
+  return !missing.empty();
+}
+
+void pg_missing_t::swap(pg_missing_t& o)
+{
+  missing.swap(o.missing);
+  rmissing.swap(o.rmissing);
+}
+
+bool pg_missing_t::is_missing(const hobject_t& oid) const
+{
+  return (missing.find(oid) != missing.end());
+}
+
+bool pg_missing_t::is_missing(const hobject_t& oid, eversion_t v) const
+{
+  map::const_iterator m = missing.find(oid);
+  if (m == missing.end())
+    return false;
+  const pg_missing_t::item &item(m->second);
+  if (item.need > v)
+    return false;
+  return true;
+}
+
+eversion_t pg_missing_t::have_old(const hobject_t& oid) const
+{
+  map::const_iterator m = missing.find(oid);
+  if (m == missing.end())
+    return eversion_t();
+  const pg_missing_t::item &item(m->second);
+  return item.have;
+}
+
+/*
+ * this needs to be called in log order as we extend the log.  it
+ * assumes missing is accurate up through the previous log entry.
+ */
+void pg_missing_t::add_next_event(const pg_log_entry_t& e)
+{
+  if (e.is_update()) {
+    if (e.prior_version == eversion_t() || e.is_clone()) {
+      // new object.
+      //assert(missing.count(e.soid) == 0);  // might already be missing divergent item.
+      if (missing.count(e.soid))  // already missing divergent item
+	rmissing.erase(missing[e.soid].need.version);
+      missing[e.soid] = item(e.version, eversion_t());  // .have = nil
+    } else if (missing.count(e.soid)) {
+      // already missing (prior).
+      //assert(missing[e.soid].need == e.prior_version);
+      rmissing.erase(missing[e.soid].need.version);
+      missing[e.soid].need = e.version;  // leave .have unchanged.
+    } else if (e.is_backlog()) {
+      // May not have prior version
+      assert(0 == "these don't exist anymore");
+    } else {
+      // not missing, we must have prior_version (if any)
+      missing[e.soid] = item(e.version, e.prior_version);
+    }
+    rmissing[e.version.version] = e.soid;
+  } else
+    rm(e.soid, e.version);
+}
+
+void pg_missing_t::revise_need(hobject_t oid, eversion_t need)
+{
+  if (missing.count(oid)) {
+    rmissing.erase(missing[oid].need.version);
+    missing[oid].need = need;            // no not adjust .have
+  } else {
+    missing[oid] = item(need, eversion_t());
+  }
+  rmissing[need.version] = oid;
+}
+
+void pg_missing_t::revise_have(hobject_t oid, eversion_t have)
+{
+  if (missing.count(oid)) {
+    missing[oid].have = have;
+  }
+}
+
+void pg_missing_t::add(const hobject_t& oid, eversion_t need, eversion_t have)
+{
+  missing[oid] = item(need, have);
+  rmissing[need.version] = oid;
+}
+
+void pg_missing_t::rm(const hobject_t& oid, eversion_t v)
+{
+  std::map::iterator p = missing.find(oid);
+  if (p != missing.end() && p->second.need <= v)
+    rm(p);
+}
+
+void pg_missing_t::rm(const std::map::iterator &m)
+{
+  rmissing.erase(m->second.need.version);
+  missing.erase(m);
+}
+
+void pg_missing_t::got(const hobject_t& oid, eversion_t v)
+{
+  std::map::iterator p = missing.find(oid);
+  assert(p != missing.end());
+  assert(p->second.need <= v);
+  got(p);
+}
+
+void pg_missing_t::got(const std::map::iterator &m)
+{
+  rmissing.erase(m->second.need.version);
+  missing.erase(m);
+}
+
+void pg_missing_t::split_into(
+  pg_t child_pgid,
+  unsigned split_bits,
+  pg_missing_t *omissing)
+{
+  unsigned mask = ~((~0)<::iterator i = missing.begin();
+       i != missing.end();
+       ) {
+    if ((i->first.hash & mask) == child_pgid.m_seed) {
+      omissing->add(i->first, i->second.need, i->second.have);
+      rm(i++);
+    } else {
+      ++i;
+    }
+  }
+}
+
+// -- object_copy_cursor_t --
+
+void object_copy_cursor_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(attr_complete, bl);
+  ::encode(data_offset, bl);
+  ::encode(data_complete, bl);
+  ::encode(omap_offset, bl);
+  ::encode(omap_complete, bl);
+  ENCODE_FINISH(bl);
+}
+
+void object_copy_cursor_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(attr_complete, bl);
+  ::decode(data_offset, bl);
+  ::decode(data_complete, bl);
+  ::decode(omap_offset, bl);
+  ::decode(omap_complete, bl);
+  DECODE_FINISH(bl);
+}
+
+void object_copy_cursor_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("attr_complete", (int)attr_complete);
+  f->dump_unsigned("data_offset", data_offset);
+  f->dump_unsigned("data_complete", (int)data_complete);
+  f->dump_string("omap_offset", omap_offset);
+  f->dump_unsigned("omap_complete", (int)omap_complete);
+}
+
+void object_copy_cursor_t::generate_test_instances(list& o)
+{
+  o.push_back(new object_copy_cursor_t);
+  o.push_back(new object_copy_cursor_t);
+  o.back()->attr_complete = true;
+  o.back()->data_offset = 123;
+  o.push_back(new object_copy_cursor_t);
+  o.back()->attr_complete = true;
+  o.back()->data_complete = true;
+  o.back()->omap_offset = "foo";
+  o.push_back(new object_copy_cursor_t);
+  o.back()->attr_complete = true;
+  o.back()->data_complete = true;
+  o.back()->omap_complete = true;
+}
+
+// -- object_copy_data_t --
+
+void object_copy_data_t::encode_classic(bufferlist& bl) const
+{
+  ::encode(size, bl);
+  ::encode(mtime, bl);
+  ::encode(attrs, bl);
+  ::encode(data, bl);
+  ::encode(omap, bl);
+  ::encode(cursor, bl);
+}
+
+void object_copy_data_t::decode_classic(bufferlist::iterator& bl)
+{
+  ::decode(size, bl);
+  ::decode(mtime, bl);
+  ::decode(attrs, bl);
+  ::decode(data, bl);
+  ::decode(omap, bl);
+  ::decode(cursor, bl);
+}
+
+void object_copy_data_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 1, bl);
+  ::encode(size, bl);
+  ::encode(mtime, bl);
+  ::encode(category, bl);
+  ::encode(attrs, bl);
+  ::encode(data, bl);
+  ::encode(omap, bl);
+  ::encode(cursor, bl);
+  ::encode(omap_header, bl);
+  ::encode(snaps, bl);
+  ::encode(snap_seq, bl);
+  ENCODE_FINISH(bl);
+}
+
+void object_copy_data_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START(2, bl);
+  ::decode(size, bl);
+  ::decode(mtime, bl);
+  ::decode(category, bl);
+  ::decode(attrs, bl);
+  ::decode(data, bl);
+  ::decode(omap, bl);
+  ::decode(cursor, bl);
+  if (struct_v >= 2)
+    ::decode(omap_header, bl);
+  if (struct_v >= 3) {
+    ::decode(snaps, bl);
+    ::decode(snap_seq, bl);
+  } else {
+    snaps.clear();
+    snap_seq = 0;
+  }
+  DECODE_FINISH(bl);
+}
+
+void object_copy_data_t::generate_test_instances(list& o)
+{
+  o.push_back(new object_copy_data_t());
+
+  list cursors;
+  object_copy_cursor_t::generate_test_instances(cursors);
+  list::iterator ci = cursors.begin();
+  o.back()->cursor = **(ci++);
+
+  o.push_back(new object_copy_data_t());
+  o.back()->cursor = **(ci++);
+
+  o.push_back(new object_copy_data_t());
+  o.back()->size = 1234;
+  o.back()->mtime.set_from_double(1234);
+  bufferptr bp("there", 5);
+  bufferlist bl;
+  bl.push_back(bp);
+  o.back()->attrs["hello"] = bl;
+  bufferptr bp2("not", 3);
+  bufferlist bl2;
+  bl2.push_back(bp2);
+  o.back()->omap["why"] = bl2;
+  bufferptr databp("iamsomedatatocontain", 20);
+  o.back()->data.push_back(databp);
+  o.back()->omap_header.append("this is an omap header");
+  o.back()->snaps.push_back(123);
+}
+
+void object_copy_data_t::dump(Formatter *f) const
+{
+  f->open_object_section("cursor");
+  cursor.dump(f);
+  f->close_section(); // cursor
+  f->dump_int("size", size);
+  f->dump_stream("mtime") << mtime;
+  /* we should really print out the attrs here, but bufferlist
+     const-correctness prents that */
+  f->dump_int("attrs_size", attrs.size());
+  f->dump_int("omap_size", omap.size());
+  f->dump_int("omap_header_length", omap_header.length());
+  f->dump_int("data_length", data.length());
+  f->open_array_section("snaps");
+  for (vector::const_iterator p = snaps.begin();
+       p != snaps.end(); ++p)
+    f->dump_unsigned("snap", *p);
+  f->close_section();
+}
+
+// -- pg_create_t --
+
+void pg_create_t::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(created, bl);
+  ::encode(parent, bl);
+  ::encode(split_bits, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_create_t::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(created, bl);
+  ::decode(parent, bl);
+  ::decode(split_bits, bl);
+  DECODE_FINISH(bl);
+}
+
+void pg_create_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("created", created);
+  f->dump_stream("parent") << parent;
+  f->dump_int("split_bits", split_bits);
+}
+
+void pg_create_t::generate_test_instances(list& o)
+{
+  o.push_back(new pg_create_t);
+  o.push_back(new pg_create_t(1, pg_t(3, 4, -1), 2));
+}
+
+
+// -- pg_hit_set_info_t --
+
+void pg_hit_set_info_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(begin, bl);
+  ::encode(end, bl);
+  ::encode(version, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_hit_set_info_t::decode(bufferlist::iterator& p)
+{
+  DECODE_START(1, p);
+  ::decode(begin, p);
+  ::decode(end, p);
+  ::decode(version, p);
+  DECODE_FINISH(p);
+}
+
+void pg_hit_set_info_t::dump(Formatter *f) const
+{
+  f->dump_stream("begin") << begin;
+  f->dump_stream("end") << end;
+  f->dump_stream("version") << version;
+}
+
+void pg_hit_set_info_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new pg_hit_set_info_t);
+  ls.push_back(new pg_hit_set_info_t);
+  ls.back()->begin = utime_t(1, 2);
+  ls.back()->end = utime_t(3, 4);
+}
+
+
+// -- pg_hit_set_history_t --
+
+void pg_hit_set_history_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(current_last_update, bl);
+  ::encode(current_last_stamp, bl);
+  ::encode(current_info, bl);
+  ::encode(history, bl);
+  ENCODE_FINISH(bl);
+}
+
+void pg_hit_set_history_t::decode(bufferlist::iterator& p)
+{
+  DECODE_START(1, p);
+  ::decode(current_last_update, p);
+  ::decode(current_last_stamp, p);
+  ::decode(current_info, p);
+  ::decode(history, p);
+  DECODE_FINISH(p);
+}
+
+void pg_hit_set_history_t::dump(Formatter *f) const
+{
+  f->dump_stream("current_last_update") << current_last_update;
+  f->dump_stream("current_last_stamp") << current_last_stamp;
+  f->open_object_section("current_info");
+  current_info.dump(f);
+  f->close_section();
+  f->open_array_section("history");
+  for (list::const_iterator p = history.begin();
+       p != history.end(); ++p) {
+    f->open_object_section("info");
+    p->dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void pg_hit_set_history_t::generate_test_instances(list& ls)
+{
+  ls.push_back(new pg_hit_set_history_t);
+  ls.push_back(new pg_hit_set_history_t);
+  ls.back()->current_last_update = eversion_t(1, 2);
+  ls.back()->current_last_stamp = utime_t(100, 123);
+  ls.back()->current_info.begin = utime_t(2, 4);
+  ls.back()->current_info.end = utime_t(62, 24);
+  ls.back()->history.push_back(ls.back()->current_info);
+  ls.back()->history.push_back(pg_hit_set_info_t());
+}
+
+// -- osd_peer_stat_t --
+
+void osd_peer_stat_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(stamp, bl);
+  ENCODE_FINISH(bl);
+}
+
+void osd_peer_stat_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START(1, bl);
+  ::decode(stamp, bl);
+  DECODE_FINISH(bl);
+}
+
+void osd_peer_stat_t::dump(Formatter *f) const
+{
+  f->dump_stream("stamp") << stamp;
+}
+
+void osd_peer_stat_t::generate_test_instances(list& o)
+{
+  o.push_back(new osd_peer_stat_t);
+  o.push_back(new osd_peer_stat_t);
+  o.back()->stamp = utime_t(1, 2);
+}
+
+ostream& operator<<(ostream& out, const osd_peer_stat_t &stat)
+{
+  return out << "stat(" << stat.stamp << ")";
+}
+
+
+// -- OSDSuperblock --
+
+void OSDSuperblock::encode(bufferlist &bl) const
+{
+  ENCODE_START(6, 5, bl);
+  ::encode(cluster_fsid, bl);
+  ::encode(whoami, bl);
+  ::encode(current_epoch, bl);
+  ::encode(oldest_map, bl);
+  ::encode(newest_map, bl);
+  ::encode(weight, bl);
+  compat_features.encode(bl);
+  ::encode(clean_thru, bl);
+  ::encode(mounted, bl);
+  ::encode(osd_fsid, bl);
+  ::encode(last_map_marked_full, bl);
+  ENCODE_FINISH(bl);
+}
+
+void OSDSuperblock::decode(bufferlist::iterator &bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, bl);
+  if (struct_v < 3) {
+    string magic;
+    ::decode(magic, bl);
+  }
+  ::decode(cluster_fsid, bl);
+  ::decode(whoami, bl);
+  ::decode(current_epoch, bl);
+  ::decode(oldest_map, bl);
+  ::decode(newest_map, bl);
+  ::decode(weight, bl);
+  if (struct_v >= 2) {
+    compat_features.decode(bl);
+  } else { //upgrade it!
+    compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE);
+  }
+  ::decode(clean_thru, bl);
+  ::decode(mounted, bl);
+  if (struct_v >= 4)
+    ::decode(osd_fsid, bl);
+  if (struct_v >= 6)
+    ::decode(last_map_marked_full, bl);
+  DECODE_FINISH(bl);
+}
+
+void OSDSuperblock::dump(Formatter *f) const
+{
+  f->dump_stream("cluster_fsid") << cluster_fsid;
+  f->dump_stream("osd_fsid") << osd_fsid;
+  f->dump_int("whoami", whoami);
+  f->dump_int("current_epoch", current_epoch);
+  f->dump_int("oldest_map", oldest_map);
+  f->dump_int("newest_map", newest_map);
+  f->dump_float("weight", weight);
+  f->open_object_section("compat");
+  compat_features.dump(f);
+  f->close_section();
+  f->dump_int("clean_thru", clean_thru);
+  f->dump_int("last_epoch_mounted", mounted);
+  f->dump_int("last_map_marked_full", last_map_marked_full);
+}
+
+void OSDSuperblock::generate_test_instances(list& o)
+{
+  OSDSuperblock z;
+  o.push_back(new OSDSuperblock(z));
+  memset(&z.cluster_fsid, 1, sizeof(z.cluster_fsid));
+  memset(&z.osd_fsid, 2, sizeof(z.osd_fsid));
+  z.whoami = 3;
+  z.current_epoch = 4;
+  z.oldest_map = 5;
+  z.newest_map = 9;
+  z.mounted = 8;
+  z.clean_thru = 7;
+  o.push_back(new OSDSuperblock(z));
+  z.last_map_marked_full = 7;
+  o.push_back(new OSDSuperblock(z));
+}
+
+// -- SnapSet --
+
+void SnapSet::encode(bufferlist& bl) const
+{
+  ENCODE_START(2, 2, bl);
+  ::encode(seq, bl);
+  ::encode(head_exists, bl);
+  ::encode(snaps, bl);
+  ::encode(clones, bl);
+  ::encode(clone_overlap, bl);
+  ::encode(clone_size, bl);
+  ENCODE_FINISH(bl);
+}
+
+void SnapSet::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+  ::decode(seq, bl);
+  ::decode(head_exists, bl);
+  ::decode(snaps, bl);
+  ::decode(clones, bl);
+  ::decode(clone_overlap, bl);
+  ::decode(clone_size, bl);
+  DECODE_FINISH(bl);
+}
+
+void SnapSet::dump(Formatter *f) const
+{
+  SnapContext sc(seq, snaps);
+  f->open_object_section("snap_context");
+  sc.dump(f);
+  f->close_section();
+  f->dump_int("head_exists", head_exists);
+  f->open_array_section("clones");
+  for (vector::const_iterator p = clones.begin(); p != clones.end(); ++p) {
+    f->open_object_section("clone");
+    f->dump_unsigned("snap", *p);
+    f->dump_unsigned("size", clone_size.find(*p)->second);
+    f->dump_stream("overlap") << clone_overlap.find(*p)->second;
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void SnapSet::generate_test_instances(list& o)
+{
+  o.push_back(new SnapSet);
+  o.push_back(new SnapSet);
+  o.back()->head_exists = true;
+  o.back()->seq = 123;
+  o.back()->snaps.push_back(123);
+  o.back()->snaps.push_back(12);
+  o.push_back(new SnapSet);
+  o.back()->head_exists = true;
+  o.back()->seq = 123;
+  o.back()->snaps.push_back(123);
+  o.back()->snaps.push_back(12);
+  o.back()->clones.push_back(12);
+  o.back()->clone_size[12] = 12345;
+  o.back()->clone_overlap[12];
+}
+
+ostream& operator<<(ostream& out, const SnapSet& cs)
+{
+  return out << cs.seq << "=" << cs.snaps << ":"
+	     << cs.clones
+	     << (cs.head_exists ? "+head":"");
+}
+
+void SnapSet::from_snap_set(const librados::snap_set_t& ss)
+{
+  // NOTE: our reconstruction of snaps (and the snapc) is not strictly
+  // correct: it will not include snaps that still logically exist
+  // but for which there was no clone that is defined.  For all
+  // practical purposes this doesn't matter, since we only use that
+  // information to clone on the OSD, and we have already moved
+  // forward past that part of the object history.
+
+  seq = ss.seq;
+  set _snaps;
+  set _clones;
+  head_exists = false;
+  for (vector::const_iterator p = ss.clones.begin();
+       p != ss.clones.end();
+       ++p) {
+    if (p->cloneid == librados::SNAP_HEAD) {
+      head_exists = true;
+    } else {
+      _clones.insert(p->cloneid);
+      _snaps.insert(p->snaps.begin(), p->snaps.end());
+      clone_size[p->cloneid] = p->size;
+      clone_overlap[p->cloneid];  // the entry must exist, even if it's empty.
+      for (vector >::const_iterator q =
+	     p->overlap.begin(); q != p->overlap.end(); ++q)
+	clone_overlap[p->cloneid].insert(q->first, q->second);
+    }
+  }
+
+  // ascending
+  clones.clear();
+  clones.reserve(_clones.size());
+  for (set::iterator p = _clones.begin(); p != _clones.end(); ++p)
+    clones.push_back(*p);
+
+  // descending
+  snaps.clear();
+  snaps.reserve(_snaps.size());
+  for (set::reverse_iterator p = _snaps.rbegin();
+       p != _snaps.rend(); ++p)
+    snaps.push_back(*p);
+}
+
+uint64_t SnapSet::get_clone_bytes(snapid_t clone) const
+{
+  assert(clone_size.count(clone));
+  uint64_t size = clone_size.find(clone)->second;
+  assert(clone_overlap.count(clone));
+  const interval_set &overlap = clone_overlap.find(clone)->second;
+  for (interval_set::const_iterator i = overlap.begin();
+       i != overlap.end();
+       ++i) {
+    assert(size >= i.get_len());
+    size -= i.get_len();
+  }
+  return size;
+}
+
+// -- watch_info_t --
+
+void watch_info_t::encode(bufferlist& bl) const
+{
+  ENCODE_START(4, 3, bl);
+  ::encode(cookie, bl);
+  ::encode(timeout_seconds, bl);
+  ::encode(addr, bl);
+  ENCODE_FINISH(bl);
+}
+
+void watch_info_t::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl);
+  ::decode(cookie, bl);
+  if (struct_v < 2) {
+    uint64_t ver;
+    ::decode(ver, bl);
+  }
+  ::decode(timeout_seconds, bl);
+  if (struct_v >= 4) {
+    ::decode(addr, bl);
+  }
+  DECODE_FINISH(bl);
+}
+
+void watch_info_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("cookie", cookie);
+  f->dump_unsigned("timeout_seconds", timeout_seconds);
+  f->open_object_section("addr");
+  addr.dump(f);
+  f->close_section();
+}
+
+void watch_info_t::generate_test_instances(list& o)
+{
+  o.push_back(new watch_info_t);
+  o.push_back(new watch_info_t);
+  o.back()->cookie = 123;
+  o.back()->timeout_seconds = 99;
+  entity_addr_t ea;
+  ea.set_nonce(1);
+  ea.set_family(AF_INET);
+  ea.set_in4_quad(0, 127);
+  ea.set_in4_quad(1, 0);
+  ea.set_in4_quad(2, 1);
+  ea.set_in4_quad(3, 2);
+  ea.set_port(2);
+  o.back()->addr = ea;
+}
+
+
+// -- object_info_t --
+
+void object_info_t::copy_user_bits(const object_info_t& other)
+{
+  // these bits are copied from head->clone.
+  size = other.size;
+  mtime = other.mtime;
+  last_reqid = other.last_reqid;
+  truncate_seq = other.truncate_seq;
+  truncate_size = other.truncate_size;
+  flags = other.flags;
+  category = other.category;
+  user_version = other.user_version;
+}
+
+ps_t object_info_t::legacy_object_locator_to_ps(const object_t &oid, 
+						const object_locator_t &loc) {
+  ps_t ps;
+  if (loc.key.length())
+    // Hack, we don't have the osd map, so we don't really know the hash...
+    ps = ceph_str_hash(CEPH_STR_HASH_RJENKINS, loc.key.c_str(), 
+		       loc.key.length());
+  else
+    ps = ceph_str_hash(CEPH_STR_HASH_RJENKINS, oid.name.c_str(),
+		       oid.name.length());
+  return ps;
+}
+
+void object_info_t::encode(bufferlist& bl) const
+{
+  object_locator_t myoloc(soid);
+  map old_watchers;
+  for (map, watch_info_t>::const_iterator i =
+	 watchers.begin();
+       i != watchers.end();
+       ++i) {
+    old_watchers.insert(make_pair(i->first.second, i->second));
+  }
+  ENCODE_START(13, 8, bl);
+  ::encode(soid, bl);
+  ::encode(myoloc, bl);	//Retained for compatibility
+  ::encode(category, bl);
+  ::encode(version, bl);
+  ::encode(prior_version, bl);
+  ::encode(last_reqid, bl);
+  ::encode(size, bl);
+  ::encode(mtime, bl);
+  if (soid.snap == CEPH_NOSNAP)
+    ::encode(wrlock_by, bl);
+  else
+    ::encode(snaps, bl);
+  ::encode(truncate_seq, bl);
+  ::encode(truncate_size, bl);
+  ::encode(is_lost(), bl);
+  ::encode(old_watchers, bl);
+  /* shenanigans to avoid breaking backwards compatibility in the disk format.
+   * When we can, switch this out for simply putting the version_t on disk. */
+  eversion_t user_eversion(0, user_version);
+  ::encode(user_eversion, bl);
+  ::encode(test_flag(FLAG_USES_TMAP), bl);
+  ::encode(watchers, bl);
+  __u32 _flags = flags;
+  ::encode(_flags, bl);
+  ENCODE_FINISH(bl);
+}
+
+void object_info_t::decode(bufferlist::iterator& bl)
+{
+  object_locator_t myoloc;
+  DECODE_START_LEGACY_COMPAT_LEN(13, 8, 8, bl);
+  map old_watchers;
+  if (struct_v >= 2 && struct_v <= 5) {
+    sobject_t obj;
+    ::decode(obj, bl);
+    ::decode(myoloc, bl);
+    soid = hobject_t(obj.oid, myoloc.key, obj.snap, 0, 0 , "");
+    soid.hash = legacy_object_locator_to_ps(soid.oid, myoloc);
+  } else if (struct_v >= 6) {
+    ::decode(soid, bl);
+    ::decode(myoloc, bl);
+    if (struct_v == 6) {
+      hobject_t hoid(soid.oid, myoloc.key, soid.snap, soid.hash, 0 , "");
+      soid = hoid;
+    }
+  }
+    
+  if (struct_v >= 5)
+    ::decode(category, bl);
+  ::decode(version, bl);
+  ::decode(prior_version, bl);
+  ::decode(last_reqid, bl);
+  ::decode(size, bl);
+  ::decode(mtime, bl);
+  if (soid.snap == CEPH_NOSNAP)
+    ::decode(wrlock_by, bl);
+  else
+    ::decode(snaps, bl);
+  ::decode(truncate_seq, bl);
+  ::decode(truncate_size, bl);
+  if (struct_v >= 3) {
+    // if this is struct_v >= 13, we will overwrite this
+    // below since this field is just here for backwards
+    // compatibility
+    __u8 lo;
+    ::decode(lo, bl);
+    flags = (flag_t)lo;
+  } else {
+    flags = (flag_t)0;
+  }
+  if (struct_v >= 4) {
+    ::decode(old_watchers, bl);
+    eversion_t user_eversion;
+    ::decode(user_eversion, bl);
+    user_version = user_eversion.version;
+  }
+  if (struct_v >= 9) {
+    bool uses_tmap = false;
+    ::decode(uses_tmap, bl);
+    if (uses_tmap)
+      set_flag(FLAG_USES_TMAP);
+  } else {
+    set_flag(FLAG_USES_TMAP);
+  }
+  if (struct_v < 10)
+    soid.pool = myoloc.pool;
+  if (struct_v >= 11) {
+    ::decode(watchers, bl);
+  } else {
+    for (map::iterator i = old_watchers.begin();
+	 i != old_watchers.end();
+	 ++i) {
+      watchers.insert(
+	make_pair(
+	  make_pair(i->second.cookie, i->first), i->second));
+    }
+  }
+  if (struct_v >= 13) {
+    __u32 _flags;
+    ::decode(_flags, bl);
+    flags = (flag_t)_flags;
+  }
+  DECODE_FINISH(bl);
+}
+
+void object_info_t::dump(Formatter *f) const
+{
+  f->open_object_section("oid");
+  soid.dump(f);
+  f->close_section();
+  f->dump_string("category", category);
+  f->dump_stream("version") << version;
+  f->dump_stream("prior_version") << prior_version;
+  f->dump_stream("last_reqid") << last_reqid;
+  f->dump_unsigned("user_version", user_version);
+  f->dump_unsigned("size", size);
+  f->dump_stream("mtime") << mtime;
+  f->dump_unsigned("lost", (int)is_lost());
+  f->dump_unsigned("flags", (int)flags);
+  f->dump_stream("wrlock_by") << wrlock_by;
+  f->open_array_section("snaps");
+  for (vector::const_iterator p = snaps.begin(); p != snaps.end(); ++p)
+    f->dump_unsigned("snap", *p);
+  f->close_section();
+  f->dump_unsigned("truncate_seq", truncate_seq);
+  f->dump_unsigned("truncate_size", truncate_size);
+  f->open_object_section("watchers");
+  for (map,watch_info_t>::const_iterator p =
+         watchers.begin(); p != watchers.end(); ++p) {
+    stringstream ss;
+    ss << p->first.second;
+    f->open_object_section(ss.str().c_str());
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void object_info_t::generate_test_instances(list& o)
+{
+  o.push_back(new object_info_t());
+  
+  // fixme
+}
+
+
+ostream& operator<<(ostream& out, const object_info_t& oi)
+{
+  out << oi.soid << "(" << oi.version
+      << " " << oi.last_reqid;
+  if (oi.soid.snap == CEPH_NOSNAP)
+    out << " wrlock_by=" << oi.wrlock_by;
+  else
+    out << " " << oi.snaps;
+  if (oi.flags)
+    out << " " << oi.get_flag_string();
+  out << " s " << oi.size;
+  out << " uv" << oi.user_version;
+  out << ")";
+  return out;
+}
+
+// -- ObjectRecovery --
+void ObjectRecoveryProgress::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(first, bl);
+  ::encode(data_complete, bl);
+  ::encode(data_recovered_to, bl);
+  ::encode(omap_recovered_to, bl);
+  ::encode(omap_complete, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ObjectRecoveryProgress::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(first, bl);
+  ::decode(data_complete, bl);
+  ::decode(data_recovered_to, bl);
+  ::decode(omap_recovered_to, bl);
+  ::decode(omap_complete, bl);
+  DECODE_FINISH(bl);
+}
+
+ostream &operator<<(ostream &out, const ObjectRecoveryProgress &prog)
+{
+  return prog.print(out);
+}
+
+void ObjectRecoveryProgress::generate_test_instances(
+  list& o)
+{
+  o.push_back(new ObjectRecoveryProgress);
+  o.back()->first = false;
+  o.back()->data_complete = true;
+  o.back()->omap_complete = true;
+  o.back()->data_recovered_to = 100;
+
+  o.push_back(new ObjectRecoveryProgress);
+  o.back()->first = true;
+  o.back()->data_complete = false;
+  o.back()->omap_complete = false;
+  o.back()->data_recovered_to = 0;
+}
+
+ostream &ObjectRecoveryProgress::print(ostream &out) const
+{
+  return out << "ObjectRecoveryProgress("
+	     << ( first ? "" : "!" ) << "first, "
+	     << "data_recovered_to:" << data_recovered_to
+	     << ", data_complete:" << ( data_complete ? "true" : "false" )
+	     << ", omap_recovered_to:" << omap_recovered_to
+	     << ", omap_complete:" << ( omap_complete ? "true" : "false" )
+	     << ")";
+}
+
+void ObjectRecoveryProgress::dump(Formatter *f) const
+{
+  f->dump_int("first?", first);
+  f->dump_int("data_complete?", data_complete);
+  f->dump_unsigned("data_recovered_to", data_recovered_to);
+  f->dump_int("omap_complete?", omap_complete);
+  f->dump_string("omap_recovered_to", omap_recovered_to);
+}
+
+void ObjectRecoveryInfo::encode(bufferlist &bl) const
+{
+  ENCODE_START(2, 1, bl);
+  ::encode(soid, bl);
+  ::encode(version, bl);
+  ::encode(size, bl);
+  ::encode(oi, bl);
+  ::encode(ss, bl);
+  ::encode(copy_subset, bl);
+  ::encode(clone_subset, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ObjectRecoveryInfo::decode(bufferlist::iterator &bl,
+				int64_t pool)
+{
+  DECODE_START(2, bl);
+  ::decode(soid, bl);
+  ::decode(version, bl);
+  ::decode(size, bl);
+  ::decode(oi, bl);
+  ::decode(ss, bl);
+  ::decode(copy_subset, bl);
+  ::decode(clone_subset, bl);
+  DECODE_FINISH(bl);
+
+  if (struct_v < 2) {
+    if (!soid.is_max() && soid.pool == -1)
+      soid.pool = pool;
+    map > tmp;
+    tmp.swap(clone_subset);
+    for (map >::iterator i = tmp.begin();
+	 i != tmp.end();
+	 ++i) {
+      hobject_t first(i->first);
+      if (!first.is_max() && first.pool == -1)
+	first.pool = pool;
+      clone_subset[first].swap(i->second);
+    }
+  }
+}
+
+void ObjectRecoveryInfo::generate_test_instances(
+  list& o)
+{
+  o.push_back(new ObjectRecoveryInfo);
+  o.back()->soid = hobject_t(sobject_t("key", CEPH_NOSNAP));
+  o.back()->version = eversion_t(0,0);
+  o.back()->size = 100;
+}
+
+
+void ObjectRecoveryInfo::dump(Formatter *f) const
+{
+  f->dump_stream("object") << soid;
+  f->dump_stream("at_version") << version;
+  f->dump_stream("size") << size;
+  {
+    f->open_object_section("object_info");
+    oi.dump(f);
+    f->close_section();
+  }
+  {
+    f->open_object_section("snapset");
+    ss.dump(f);
+    f->close_section();
+  }
+  f->dump_stream("copy_subset") << copy_subset;
+  f->dump_stream("clone_subset") << clone_subset;
+}
+
+ostream& operator<<(ostream& out, const ObjectRecoveryInfo &inf)
+{
+  return inf.print(out);
+}
+
+ostream &ObjectRecoveryInfo::print(ostream &out) const
+{
+  return out << "ObjectRecoveryInfo("
+	     << soid << "@" << version
+	     << ", copy_subset: " << copy_subset
+	     << ", clone_subset: " << clone_subset
+	     << ")";
+}
+
+// -- PushReplyOp --
+void PushReplyOp::generate_test_instances(list &o)
+{
+  o.push_back(new PushReplyOp);
+  o.push_back(new PushReplyOp);
+  o.back()->soid = hobject_t(sobject_t("asdf", 2));
+  o.push_back(new PushReplyOp);
+  o.back()->soid = hobject_t(sobject_t("asdf", CEPH_NOSNAP));
+}
+
+void PushReplyOp::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(soid, bl);
+  ENCODE_FINISH(bl);
+}
+
+void PushReplyOp::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(soid, bl);
+  DECODE_FINISH(bl);
+}
+
+void PushReplyOp::dump(Formatter *f) const
+{
+  f->dump_stream("soid") << soid;
+}
+
+ostream &PushReplyOp::print(ostream &out) const
+{
+  return out
+    << "PushReplyOp(" << soid
+    << ")";
+}
+
+ostream& operator<<(ostream& out, const PushReplyOp &op)
+{
+  return op.print(out);
+}
+
+uint64_t PushReplyOp::cost(CephContext *cct) const
+{
+
+  return cct->_conf->osd_push_per_object_cost +
+    cct->_conf->osd_recovery_max_chunk;
+}
+
+// -- PullOp --
+void PullOp::generate_test_instances(list &o)
+{
+  o.push_back(new PullOp);
+  o.push_back(new PullOp);
+  o.back()->soid = hobject_t(sobject_t("asdf", 2));
+  o.back()->recovery_info.version = eversion_t(3, 10);
+  o.push_back(new PullOp);
+  o.back()->soid = hobject_t(sobject_t("asdf", CEPH_NOSNAP));
+  o.back()->recovery_info.version = eversion_t(0, 0);
+}
+
+void PullOp::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(soid, bl);
+  ::encode(recovery_info, bl);
+  ::encode(recovery_progress, bl);
+  ENCODE_FINISH(bl);
+}
+
+void PullOp::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(soid, bl);
+  ::decode(recovery_info, bl);
+  ::decode(recovery_progress, bl);
+  DECODE_FINISH(bl);
+}
+
+void PullOp::dump(Formatter *f) const
+{
+  f->dump_stream("soid") << soid;
+  {
+    f->open_object_section("recovery_info");
+    recovery_info.dump(f);
+    f->close_section();
+  }
+  {
+    f->open_object_section("recovery_progress");
+    recovery_progress.dump(f);
+    f->close_section();
+  }
+}
+
+ostream &PullOp::print(ostream &out) const
+{
+  return out
+    << "PullOp(" << soid
+    << ", recovery_info: " << recovery_info
+    << ", recovery_progress: " << recovery_progress
+    << ")";
+}
+
+ostream& operator<<(ostream& out, const PullOp &op)
+{
+  return op.print(out);
+}
+
+uint64_t PullOp::cost(CephContext *cct) const
+{
+  return cct->_conf->osd_push_per_object_cost +
+    cct->_conf->osd_recovery_max_chunk;
+}
+
+// -- PushOp --
+void PushOp::generate_test_instances(list &o)
+{
+  o.push_back(new PushOp);
+  o.push_back(new PushOp);
+  o.back()->soid = hobject_t(sobject_t("asdf", 2));
+  o.back()->version = eversion_t(3, 10);
+  o.push_back(new PushOp);
+  o.back()->soid = hobject_t(sobject_t("asdf", CEPH_NOSNAP));
+  o.back()->version = eversion_t(0, 0);
+}
+
+void PushOp::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(soid, bl);
+  ::encode(version, bl);
+  ::encode(data, bl);
+  ::encode(data_included, bl);
+  ::encode(omap_header, bl);
+  ::encode(omap_entries, bl);
+  ::encode(attrset, bl);
+  ::encode(recovery_info, bl);
+  ::encode(after_progress, bl);
+  ::encode(before_progress, bl);
+  ENCODE_FINISH(bl);
+}
+
+void PushOp::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(soid, bl);
+  ::decode(version, bl);
+  ::decode(data, bl);
+  ::decode(data_included, bl);
+  ::decode(omap_header, bl);
+  ::decode(omap_entries, bl);
+  ::decode(attrset, bl);
+  ::decode(recovery_info, bl);
+  ::decode(after_progress, bl);
+  ::decode(before_progress, bl);
+  DECODE_FINISH(bl);
+}
+
+void PushOp::dump(Formatter *f) const
+{
+  f->dump_stream("soid") << soid;
+  f->dump_stream("version") << version;
+  f->dump_int("data_len", data.length());
+  f->dump_stream("data_included") << data_included;
+  f->dump_int("omap_header_len", omap_header.length());
+  f->dump_int("omap_entries_len", omap_entries.size());
+  f->dump_int("attrset_len", attrset.size());
+  {
+    f->open_object_section("recovery_info");
+    recovery_info.dump(f);
+    f->close_section();
+  }
+  {
+    f->open_object_section("after_progress");
+    after_progress.dump(f);
+    f->close_section();
+  }
+  {
+    f->open_object_section("before_progress");
+    before_progress.dump(f);
+    f->close_section();
+  }
+}
+
+ostream &PushOp::print(ostream &out) const
+{
+  return out
+    << "PushOp(" << soid
+    << ", version: " << version
+    << ", data_included: " << data_included
+    << ", data_size: " << data.length()
+    << ", omap_header_size: " << omap_header.length()
+    << ", omap_entries_size: " << omap_entries.size()
+    << ", attrset_size: " << attrset.size()
+    << ", recovery_info: " << recovery_info
+    << ", after_progress: " << after_progress
+    << ", before_progress: " << before_progress
+    << ")";
+}
+
+ostream& operator<<(ostream& out, const PushOp &op)
+{
+  return op.print(out);
+}
+
+uint64_t PushOp::cost(CephContext *cct) const
+{
+  uint64_t cost = data_included.size();
+  for (map::const_iterator i =
+	 omap_entries.begin();
+       i != omap_entries.end();
+       ++i) {
+    cost += i->second.length();
+  }
+  cost += cct->_conf->osd_push_per_object_cost;
+  return cost;
+}
+
+// -- ScrubMap --
+
+void ScrubMap::merge_incr(const ScrubMap &l)
+{
+  assert(valid_through == l.incr_since);
+  attrs = l.attrs;
+  valid_through = l.valid_through;
+
+  for (map::const_iterator p = l.objects.begin();
+       p != l.objects.end();
+       ++p){
+    if (p->second.negative) {
+      map::iterator q = objects.find(p->first);
+      if (q != objects.end()) {
+	objects.erase(q);
+      }
+    } else {
+      objects[p->first] = p->second;
+    }
+  }
+}          
+
+void ScrubMap::encode(bufferlist& bl) const
+{
+  ENCODE_START(3, 2, bl);
+  ::encode(objects, bl);
+  ::encode(attrs, bl);
+  bufferlist old_logbl;  // not used
+  ::encode(old_logbl, bl);
+  ::encode(valid_through, bl);
+  ::encode(incr_since, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ScrubMap::decode(bufferlist::iterator& bl, int64_t pool)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
+  ::decode(objects, bl);
+  ::decode(attrs, bl);
+  bufferlist old_logbl;   // not used
+  ::decode(old_logbl, bl);
+  ::decode(valid_through, bl);
+  ::decode(incr_since, bl);
+  DECODE_FINISH(bl);
+
+  // handle hobject_t upgrade
+  if (struct_v < 3) {
+    map tmp;
+    tmp.swap(objects);
+    for (map::iterator i = tmp.begin();
+	 i != tmp.end();
+	 ++i) {
+      hobject_t first(i->first);
+      if (!first.is_max() && first.pool == -1)
+	first.pool = pool;
+      objects[first] = i->second;
+    }
+  }
+}
+
+void ScrubMap::dump(Formatter *f) const
+{
+  f->dump_stream("valid_through") << valid_through;
+  f->dump_stream("incremental_since") << incr_since;
+  f->open_array_section("attrs");
+  for (map::const_iterator p = attrs.begin(); p != attrs.end(); ++p) {
+    f->open_object_section("attr");
+    f->dump_string("name", p->first);
+    f->dump_int("length", p->second.length());
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("objects");
+  for (map::const_iterator p = objects.begin(); p != objects.end(); ++p) {
+    f->open_object_section("object");
+    f->dump_string("name", p->first.oid.name);
+    f->dump_unsigned("hash", p->first.hash);
+    f->dump_string("key", p->first.get_key());
+    f->dump_int("snapid", p->first.snap);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ScrubMap::generate_test_instances(list& o)
+{
+  o.push_back(new ScrubMap);
+  o.push_back(new ScrubMap);
+  o.back()->valid_through = eversion_t(1, 2);
+  o.back()->incr_since = eversion_t(3, 4);
+  o.back()->attrs["foo"] = buffer::copy("foo", 3);
+  o.back()->attrs["bar"] = buffer::copy("barval", 6);
+  list obj;
+  object::generate_test_instances(obj);
+  o.back()->objects[hobject_t(object_t("foo"), "fookey", 123, 456, 0, "")] = *obj.back();
+  obj.pop_back();
+  o.back()->objects[hobject_t(object_t("bar"), string(), 123, 456, 0, "")] = *obj.back();
+}
+
+// -- ScrubMap::object --
+
+void ScrubMap::object::encode(bufferlist& bl) const
+{
+  ENCODE_START(6, 2, bl);
+  ::encode(size, bl);
+  ::encode(negative, bl);
+  ::encode(attrs, bl);
+  ::encode(digest, bl);
+  ::encode(digest_present, bl);
+  ::encode(nlinks, bl);
+  ::encode(snapcolls, bl);
+  ::encode(omap_digest, bl);
+  ::encode(omap_digest_present, bl);
+  ::encode(read_error, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ScrubMap::object::decode(bufferlist::iterator& bl)
+{
+  DECODE_START_LEGACY_COMPAT_LEN(5, 2, 2, bl);
+  ::decode(size, bl);
+  ::decode(negative, bl);
+  ::decode(attrs, bl);
+  if (struct_v >= 3) {
+    ::decode(digest, bl);
+    ::decode(digest_present, bl);
+  }
+  if (struct_v >= 4) {
+    ::decode(nlinks, bl);
+    ::decode(snapcolls, bl);
+  } else {
+    /* Indicates that encoder was not aware of this field since stat must
+     * return nlink >= 1 */
+    nlinks = 0;
+  }
+  if (struct_v >= 5) {
+    ::decode(omap_digest, bl);
+    ::decode(omap_digest_present, bl);
+  }
+  if (struct_v >= 6) {
+    ::decode(read_error, bl);
+  }
+  DECODE_FINISH(bl);
+}
+
+void ScrubMap::object::dump(Formatter *f) const
+{
+  f->dump_int("size", size);
+  f->dump_int("negative", negative);
+  f->open_array_section("attrs");
+  for (map::const_iterator p = attrs.begin(); p != attrs.end(); ++p) {
+    f->open_object_section("attr");
+    f->dump_string("name", p->first);
+    f->dump_int("length", p->second.length());
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ScrubMap::object::generate_test_instances(list& o)
+{
+  o.push_back(new object);
+  o.push_back(new object);
+  o.back()->negative = true;
+  o.push_back(new object);
+  o.back()->size = 123;
+  o.back()->attrs["foo"] = buffer::copy("foo", 3);
+  o.back()->attrs["bar"] = buffer::copy("barval", 6);
+}
+
+// -- OSDOp --
+
+ostream& operator<<(ostream& out, const OSDOp& op)
+{
+  out << ceph_osd_op_name(op.op.op);
+  if (ceph_osd_op_type_data(op.op.op)) {
+    // data extent
+    switch (op.op.op) {
+    case CEPH_OSD_OP_STAT:
+    case CEPH_OSD_OP_DELETE:
+    case CEPH_OSD_OP_LIST_WATCHERS:
+    case CEPH_OSD_OP_LIST_SNAPS:
+    case CEPH_OSD_OP_UNDIRTY:
+    case CEPH_OSD_OP_ISDIRTY:
+    case CEPH_OSD_OP_CACHE_FLUSH:
+    case CEPH_OSD_OP_CACHE_TRY_FLUSH:
+    case CEPH_OSD_OP_CACHE_EVICT:
+      break;
+    case CEPH_OSD_OP_ASSERT_VER:
+      out << " v" << op.op.assert_ver.ver;
+      break;
+    case CEPH_OSD_OP_TRUNCATE:
+      out << " " << op.op.extent.offset;
+      break;
+    case CEPH_OSD_OP_MASKTRUNC:
+    case CEPH_OSD_OP_TRIMTRUNC:
+      out << " " << op.op.extent.truncate_seq << "@" << (int64_t)op.op.extent.truncate_size;
+      break;
+    case CEPH_OSD_OP_ROLLBACK:
+      out << " " << snapid_t(op.op.snap.snapid);
+      break;
+    case CEPH_OSD_OP_WATCH:
+      out << (op.op.watch.flag ? " add":" remove")
+	  << " cookie " << op.op.watch.cookie << " ver " << op.op.watch.ver;
+      break;
+    case CEPH_OSD_OP_COPY_GET:
+    case CEPH_OSD_OP_COPY_GET_CLASSIC:
+      out << " max " << op.op.copy_get.max;
+      break;
+    case CEPH_OSD_OP_COPY_FROM:
+      out << " ver " << op.op.copy_from.src_version;
+      break;
+    case CEPH_OSD_OP_SETALLOCHINT:
+      out << " object_size " << op.op.alloc_hint.expected_object_size
+          << " write_size " << op.op.alloc_hint.expected_write_size;
+      break;
+    default:
+      out << " " << op.op.extent.offset << "~" << op.op.extent.length;
+      if (op.op.extent.truncate_seq)
+	out << " [" << op.op.extent.truncate_seq << "@" << (int64_t)op.op.extent.truncate_size << "]";
+    }
+  } else if (ceph_osd_op_type_attr(op.op.op)) {
+    // xattr name
+    if (op.op.xattr.name_len && op.indata.length()) {
+      out << " ";
+      op.indata.write(0, op.op.xattr.name_len, out);
+    }
+    if (op.op.xattr.value_len)
+      out << " (" << op.op.xattr.value_len << ")";
+    if (op.op.op == CEPH_OSD_OP_CMPXATTR)
+      out << " op " << (int)op.op.xattr.cmp_op << " mode " << (int)op.op.xattr.cmp_mode;
+  } else if (ceph_osd_op_type_exec(op.op.op)) {
+    // class.method
+    if (op.op.cls.class_len && op.indata.length()) {
+      out << " ";
+      op.indata.write(0, op.op.cls.class_len, out);
+      out << ".";
+      op.indata.write(op.op.cls.class_len, op.op.cls.method_len, out);
+    }
+  } else if (ceph_osd_op_type_pg(op.op.op)) {
+    switch (op.op.op) {
+    case CEPH_OSD_OP_PGLS:
+    case CEPH_OSD_OP_PGLS_FILTER:
+      out << " start_epoch " << op.op.pgls.start_epoch;
+      break;
+    case CEPH_OSD_OP_PG_HITSET_LS:
+      break;
+    case CEPH_OSD_OP_PG_HITSET_GET:
+      out << " " << utime_t(op.op.hit_set_get.stamp);
+      break;
+    }
+  } else if (ceph_osd_op_type_multi(op.op.op)) {
+    switch (op.op.op) {
+    case CEPH_OSD_OP_CLONERANGE:
+      out << " " << op.op.clonerange.offset << "~" << op.op.clonerange.length
+	  << " from " << op.soid
+	  << " offset " << op.op.clonerange.src_offset;
+      break;
+    case CEPH_OSD_OP_ASSERT_SRC_VERSION:
+      out << " v" << op.op.watch.ver
+	  << " of " << op.soid;
+      break;
+    case CEPH_OSD_OP_SRC_CMPXATTR:
+      out << " " << op.soid;
+      if (op.op.xattr.name_len && op.indata.length()) {
+	out << " ";
+	op.indata.write(0, op.op.xattr.name_len, out);
+      }
+      if (op.op.xattr.value_len)
+	out << " (" << op.op.xattr.value_len << ")";
+      if (op.op.op == CEPH_OSD_OP_CMPXATTR)
+	out << " op " << (int)op.op.xattr.cmp_op << " mode " << (int)op.op.xattr.cmp_mode;
+      break;
+    }
+  }
+  return out;
+}
+
+
+void OSDOp::split_osd_op_vector_in_data(vector& ops, bufferlist& in)
+{
+  bufferlist::iterator datap = in.begin();
+  for (unsigned i = 0; i < ops.size(); i++) {
+    if (ceph_osd_op_type_multi(ops[i].op.op)) {
+      ::decode(ops[i].soid, datap);
+    }
+    if (ops[i].op.payload_len) {
+      datap.copy(ops[i].op.payload_len, ops[i].indata);
+    }
+  }
+}
+
+void OSDOp::merge_osd_op_vector_in_data(vector& ops, bufferlist& out)
+{
+  for (unsigned i = 0; i < ops.size(); i++) {
+    if (ceph_osd_op_type_multi(ops[i].op.op)) {
+      ::encode(ops[i].soid, out);
+    }
+    if (ops[i].indata.length()) {
+      ops[i].op.payload_len = ops[i].indata.length();
+      out.append(ops[i].indata);
+    }
+  }
+}
+
+void OSDOp::split_osd_op_vector_out_data(vector& ops, bufferlist& in)
+{
+  bufferlist::iterator datap = in.begin();
+  for (unsigned i = 0; i < ops.size(); i++) {
+    if (ops[i].op.payload_len) {
+      datap.copy(ops[i].op.payload_len, ops[i].outdata);
+    }
+  }
+}
+
+void OSDOp::merge_osd_op_vector_out_data(vector& ops, bufferlist& out)
+{
+  for (unsigned i = 0; i < ops.size(); i++) {
+    if (ops[i].outdata.length()) {
+      ops[i].op.payload_len = ops[i].outdata.length();
+      out.append(ops[i].outdata);
+    }
+  }
+}
diff --git a/ceph/src/osd/osd_types.h b/ceph/src/osd/osd_types.h
new file mode 100644
index 00000000..8e9cf6f5
--- /dev/null
+++ b/ceph/src/osd/osd_types.h
@@ -0,0 +1,3452 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_OSD_TYPES_H
+#define CEPH_OSD_TYPES_H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/rados/rados_types.hpp"
+
+#include "msg/msg_types.h"
+#include "include/types.h"
+#include "include/utime.h"
+#include "include/CompatSet.h"
+#include "common/histogram.h"
+#include "include/interval_set.h"
+#include "common/Formatter.h"
+#include "common/bloom_filter.hpp"
+#include "common/hobject.h"
+#include "common/snap_types.h"
+#include "HitSet.h"
+#include "Watch.h"
+#include "OpRequest.h"
+#include "include/cmp.h"
+
+#define CEPH_OSD_ONDISK_MAGIC "ceph osd volume v026"
+
+#define CEPH_OSD_FEATURE_INCOMPAT_BASE CompatSet::Feature(1, "initial feature set(~v.18)")
+#define CEPH_OSD_FEATURE_INCOMPAT_PGINFO CompatSet::Feature(2, "pginfo object")
+#define CEPH_OSD_FEATURE_INCOMPAT_OLOC CompatSet::Feature(3, "object locator")
+#define CEPH_OSD_FEATURE_INCOMPAT_LEC  CompatSet::Feature(4, "last_epoch_clean")
+#define CEPH_OSD_FEATURE_INCOMPAT_CATEGORIES  CompatSet::Feature(5, "categories")
+#define CEPH_OSD_FEATURE_INCOMPAT_HOBJECTPOOL  CompatSet::Feature(6, "hobjectpool")
+#define CEPH_OSD_FEATURE_INCOMPAT_BIGINFO CompatSet::Feature(7, "biginfo")
+#define CEPH_OSD_FEATURE_INCOMPAT_LEVELDBINFO CompatSet::Feature(8, "leveldbinfo")
+#define CEPH_OSD_FEATURE_INCOMPAT_LEVELDBLOG CompatSet::Feature(9, "leveldblog")
+#define CEPH_OSD_FEATURE_INCOMPAT_SNAPMAPPER CompatSet::Feature(10, "snapmapper")
+#define CEPH_OSD_FEATURE_INCOMPAT_SHARDS CompatSet::Feature(11, "sharded objects")
+
+
+typedef hobject_t collection_list_handle_t;
+
+typedef uint8_t shard_id_t;
+
+/// convert a single CPEH_OSD_FLAG_* to a string
+const char *ceph_osd_flag_name(unsigned flag);
+
+/// convert CEPH_OSD_FLAG_* op flags to a string
+string ceph_osd_flag_string(unsigned flags);
+
+struct pg_shard_t {
+  int osd;
+  shard_id_t shard;
+  pg_shard_t() : osd(-1), shard(ghobject_t::NO_SHARD) {}
+  explicit pg_shard_t(int osd) : osd(osd), shard(ghobject_t::NO_SHARD) {}
+  pg_shard_t(int osd, shard_id_t shard) : osd(osd), shard(shard) {}
+  static pg_shard_t undefined_shard() {
+    return pg_shard_t(-1, ghobject_t::NO_SHARD);
+  }
+  bool is_undefined() const {
+    return osd == -1;
+  }
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+};
+WRITE_CLASS_ENCODER(pg_shard_t)
+WRITE_EQ_OPERATORS_2(pg_shard_t, osd, shard)
+WRITE_CMP_OPERATORS_2(pg_shard_t, osd, shard)
+ostream &operator<<(ostream &lhs, const pg_shard_t &rhs);
+
+inline ostream& operator<<(ostream& out, const osd_reqid_t& r) {
+  return out << r.name << "." << r.inc << ":" << r.tid;
+}
+
+inline bool operator==(const osd_reqid_t& l, const osd_reqid_t& r) {
+  return (l.name == r.name) && (l.inc == r.inc) && (l.tid == r.tid);
+}
+inline bool operator!=(const osd_reqid_t& l, const osd_reqid_t& r) {
+  return (l.name != r.name) || (l.inc != r.inc) || (l.tid != r.tid);
+}
+inline bool operator<(const osd_reqid_t& l, const osd_reqid_t& r) {
+  return (l.name < r.name) || (l.inc < r.inc) || 
+    (l.name == r.name && l.inc == r.inc && l.tid < r.tid);
+}
+inline bool operator<=(const osd_reqid_t& l, const osd_reqid_t& r) {
+  return (l.name < r.name) || (l.inc < r.inc) ||
+    (l.name == r.name && l.inc == r.inc && l.tid <= r.tid);
+}
+inline bool operator>(const osd_reqid_t& l, const osd_reqid_t& r) { return !(l <= r); }
+inline bool operator>=(const osd_reqid_t& l, const osd_reqid_t& r) { return !(l < r); }
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash {
+    size_t operator()(const osd_reqid_t &r) const { 
+      static hash H;
+      return H(r.name.num() ^ r.tid ^ r.inc);
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+
+// -----
+
+// a locator constrains the placement of an object.  mainly, which pool
+// does it go in.
+struct object_locator_t {
+  // You specify either the hash or the key -- not both
+  int64_t pool;     ///< pool id
+  string key;       ///< key string (if non-empty)
+  string nspace;    ///< namespace
+  int64_t hash;     ///< hash position (if >= 0)
+
+  explicit object_locator_t()
+    : pool(-1), hash(-1) {}
+  explicit object_locator_t(int64_t po)
+    : pool(po), hash(-1)  {}
+  explicit object_locator_t(int64_t po, int64_t ps)
+    : pool(po), hash(ps)  {}
+  explicit object_locator_t(int64_t po, string ns)
+    : pool(po), nspace(ns), hash(-1) {}
+  explicit object_locator_t(int64_t po, string ns, int64_t ps)
+    : pool(po), nspace(ns), hash(ps) {}
+  explicit object_locator_t(int64_t po, string ns, string s)
+    : pool(po), key(s), nspace(ns), hash(-1) {}
+  explicit object_locator_t(const hobject_t& soid)
+    : pool(soid.pool), key(soid.get_key()), nspace(soid.nspace), hash(-1) {}
+
+  int64_t get_pool() const {
+    return pool;
+  }
+
+  void clear() {
+    pool = -1;
+    key = "";
+    nspace = "";
+    hash = -1;
+  }
+
+  bool empty() const {
+    return pool == -1;
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& p);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(object_locator_t)
+
+inline bool operator==(const object_locator_t& l, const object_locator_t& r) {
+  return l.pool == r.pool && l.key == r.key && l.nspace == r.nspace && l.hash == r.hash;
+}
+inline bool operator!=(const object_locator_t& l, const object_locator_t& r) {
+  return !(l == r);
+}
+
+inline ostream& operator<<(ostream& out, const object_locator_t& loc)
+{
+  out << "@" << loc.pool;
+  if (loc.nspace.length())
+    out << ";" << loc.nspace;
+  if (loc.key.length())
+    out << ":" << loc.key;
+  return out;
+}
+
+struct request_redirect_t {
+private:
+  object_locator_t redirect_locator; ///< this is authoritative
+  string redirect_object; ///< If non-empty, the request goes to this object name
+  bufferlist osd_instructions; ///< a bufferlist for the OSDs, passed but not interpreted by clients
+
+  friend ostream& operator<<(ostream& out, const request_redirect_t& redir);
+public:
+
+  request_redirect_t() {}
+  explicit request_redirect_t(const object_locator_t& orig, int64_t rpool) :
+      redirect_locator(orig) { redirect_locator.pool = rpool; }
+  explicit request_redirect_t(const object_locator_t& rloc) :
+      redirect_locator(rloc) {}
+  explicit request_redirect_t(const object_locator_t& orig,
+                              const string& robj) :
+      redirect_locator(orig), redirect_object(robj) {}
+
+  void set_instructions(const bufferlist& bl) { osd_instructions = bl; }
+  const bufferlist& get_instructions() { return osd_instructions; }
+
+  bool empty() const { return redirect_locator.empty() &&
+			      redirect_object.empty(); }
+
+  void combine_with_locator(object_locator_t& orig, string& obj) const {
+    orig = redirect_locator;
+    if (!redirect_object.empty())
+      obj = redirect_object;
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(request_redirect_t)
+
+inline ostream& operator<<(ostream& out, const request_redirect_t& redir) {
+  out << "object " << redir.redirect_object << ", locator{" << redir.redirect_locator << "}";
+  return out;
+}
+
+// Internal OSD op flags - set by the OSD based on the op types
+enum {
+  CEPH_OSD_RMW_FLAG_READ        = (1 << 1),
+  CEPH_OSD_RMW_FLAG_WRITE       = (1 << 2),
+  CEPH_OSD_RMW_FLAG_CLASS_READ  = (1 << 3),
+  CEPH_OSD_RMW_FLAG_CLASS_WRITE = (1 << 4),
+  CEPH_OSD_RMW_FLAG_PGOP        = (1 << 5),
+  CEPH_OSD_RMW_FLAG_CACHE       = (1 << 6),
+};
+
+
+// pg stuff
+
+// object namespaces
+#define CEPH_METADATA_NS       1
+#define CEPH_DATA_NS           2
+#define CEPH_CAS_NS            3
+#define CEPH_OSDMETADATA_NS 0xff
+
+#define OSD_SUPERBLOCK_POBJECT hobject_t(sobject_t(object_t("osd_superblock"), 0))
+
+// placement seed (a hash value)
+typedef uint32_t ps_t;
+
+// old (v1) pg_t encoding (wrap old struct ceph_pg)
+struct old_pg_t {
+  ceph_pg v;
+  void encode(bufferlist& bl) const {
+    ::encode_raw(v, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode_raw(v, bl);
+  }
+};
+WRITE_CLASS_ENCODER(old_pg_t)
+
+// placement group id
+struct pg_t {
+  uint64_t m_pool;
+  uint32_t m_seed;
+  int32_t m_preferred;
+
+  pg_t() : m_pool(0), m_seed(0), m_preferred(-1) {}
+  pg_t(ps_t seed, uint64_t pool, int pref=-1) {
+    m_seed = seed;
+    m_pool = pool;
+    m_preferred = pref;
+  }
+
+  pg_t(const ceph_pg& cpg) {
+    m_pool = cpg.pool;
+    m_seed = cpg.ps;
+    m_preferred = (__s16)cpg.preferred;
+  }
+  old_pg_t get_old_pg() const {
+    old_pg_t o;
+    assert(m_pool < 0xffffffffull);
+    o.v.pool = m_pool;
+    o.v.ps = m_seed;
+    o.v.preferred = (__s16)m_preferred;
+    return o;
+  }
+  pg_t(const old_pg_t& opg) {
+    *this = opg.v;
+  }
+
+  ps_t ps() const {
+    return m_seed;
+  }
+  uint64_t pool() const {
+    return m_pool;
+  }
+  int32_t preferred() const {
+    return m_preferred;
+  }
+
+  void set_ps(ps_t p) {
+    m_seed = p;
+  }
+  void set_pool(uint64_t p) {
+    m_pool = p;
+  }
+  void set_preferred(int32_t osd) {
+    m_preferred = osd;
+  }
+
+  pg_t get_parent() const;
+  pg_t get_ancestor(unsigned old_pg_num) const;
+
+  int print(char *o, int maxlen) const;
+  bool parse(const char *s);
+
+  bool is_split(unsigned old_pg_num, unsigned new_pg_num, set *pchildren) const;
+
+  /**
+   * Returns b such that for all object o:
+   *   ~((~0)<& o);
+};
+WRITE_CLASS_ENCODER(pg_t)
+
+inline bool operator<(const pg_t& l, const pg_t& r) {
+  return l.pool() < r.pool() ||
+    (l.pool() == r.pool() && (l.preferred() < r.preferred() ||
+			      (l.preferred() == r.preferred() && (l.ps() < r.ps()))));
+}
+inline bool operator<=(const pg_t& l, const pg_t& r) {
+  return l.pool() < r.pool() ||
+    (l.pool() == r.pool() && (l.preferred() < r.preferred() ||
+			      (l.preferred() == r.preferred() && (l.ps() <= r.ps()))));
+}
+inline bool operator==(const pg_t& l, const pg_t& r) {
+  return l.pool() == r.pool() &&
+    l.preferred() == r.preferred() &&
+    l.ps() == r.ps();
+}
+inline bool operator!=(const pg_t& l, const pg_t& r) {
+  return l.pool() != r.pool() ||
+    l.preferred() != r.preferred() ||
+    l.ps() != r.ps();
+}
+inline bool operator>(const pg_t& l, const pg_t& r) {
+  return l.pool() > r.pool() ||
+    (l.pool() == r.pool() && (l.preferred() > r.preferred() ||
+			      (l.preferred() == r.preferred() && (l.ps() > r.ps()))));
+}
+inline bool operator>=(const pg_t& l, const pg_t& r) {
+  return l.pool() > r.pool() ||
+    (l.pool() == r.pool() && (l.preferred() > r.preferred() ||
+			      (l.preferred() == r.preferred() && (l.ps() >= r.ps()))));
+}
+
+ostream& operator<<(ostream& out, const pg_t &pg);
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash< pg_t >
+  {
+    size_t operator()( const pg_t& x ) const
+    {
+      static hash H;
+      return H((x.pool() & 0xffffffff) ^ (x.pool() >> 32) ^ x.ps() ^ x.preferred());
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+struct spg_t {
+  pg_t pgid;
+  shard_id_t shard;
+  spg_t() : shard(ghobject_t::NO_SHARD) {}
+  spg_t(pg_t pgid, shard_id_t shard) : pgid(pgid), shard(shard) {}
+  explicit spg_t(pg_t pgid) : pgid(pgid), shard(ghobject_t::NO_SHARD) {}
+  unsigned get_split_bits(unsigned pg_num) const {
+    return pgid.get_split_bits(pg_num);
+  }
+  spg_t get_parent() const {
+    return spg_t(pgid.get_parent(), shard);
+  }
+  ps_t ps() const {
+    return pgid.ps();
+  }
+  uint64_t pool() const {
+    return pgid.pool();
+  }
+  int32_t preferred() const {
+    return pgid.preferred();
+  }
+  bool parse(const char *s);
+  bool is_split(unsigned old_pg_num, unsigned new_pg_num,
+		set *pchildren) const {
+    set _children;
+    set *children = pchildren ? &_children : NULL;
+    bool is_split = pgid.is_split(old_pg_num, new_pg_num, children);
+    if (pchildren && is_split) {
+      for (set::iterator i = _children.begin();
+	   i != _children.end();
+	   ++i) {
+	pchildren->insert(spg_t(*i, shard));
+      }
+    }
+    return is_split;
+  }
+  bool is_no_shard() const {
+    return shard == ghobject_t::NO_SHARD;
+  }
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(pgid, bl);
+    ::encode(shard, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    DECODE_START(1, bl);
+    ::decode(pgid, bl);
+    ::decode(shard, bl);
+    DECODE_FINISH(bl);
+  }
+};
+WRITE_CLASS_ENCODER(spg_t)
+WRITE_EQ_OPERATORS_2(spg_t, pgid, shard)
+WRITE_CMP_OPERATORS_2(spg_t, pgid, shard)
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash< spg_t >
+  {
+    size_t operator()( const spg_t& x ) const
+      {
+      static hash H;
+      return H(hash()(x.pgid) ^ x.shard);
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+ostream& operator<<(ostream& out, const spg_t &pg);
+
+// ----------------------
+
+class coll_t {
+public:
+  const static coll_t META_COLL;
+
+  coll_t()
+    : str("meta")
+  { }
+
+  explicit coll_t(const std::string &str_)
+    : str(str_)
+  { }
+
+  explicit coll_t(spg_t pgid, snapid_t snap = CEPH_NOSNAP)
+    : str(pg_and_snap_to_str(pgid, snap))
+  { }
+
+  static coll_t make_temp_coll(spg_t pgid) {
+    return coll_t(pg_to_tmp_str(pgid));
+  }
+
+  static coll_t make_removal_coll(uint64_t seq, spg_t pgid) {
+    return coll_t(seq_to_removal_str(seq, pgid));
+  }
+
+  const std::string& to_str() const {
+    return str;
+  }
+
+  const char* c_str() const {
+    return str.c_str();
+  }
+
+  int operator<(const coll_t &rhs) const {
+    return str < rhs.str;
+  }
+
+  bool is_pg_prefix(spg_t& pgid) const;
+  bool is_pg(spg_t& pgid, snapid_t& snap) const;
+  bool is_temp(spg_t& pgid) const;
+  bool is_removal(uint64_t *seq, spg_t *pgid) const;
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  inline bool operator==(const coll_t& rhs) const {
+    return str == rhs.str;
+  }
+  inline bool operator!=(const coll_t& rhs) const {
+    return str != rhs.str;
+  }
+
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+private:
+  static std::string pg_and_snap_to_str(spg_t p, snapid_t s) {
+    std::ostringstream oss;
+    oss << p << "_" << s;
+    return oss.str();
+  }
+  static std::string pg_to_tmp_str(spg_t p) {
+    std::ostringstream oss;
+    oss << p << "_TEMP";
+    return oss.str();
+  }
+  static std::string seq_to_removal_str(uint64_t seq, spg_t pgid) {
+    std::ostringstream oss;
+    oss << "FORREMOVAL_" << seq << "_" << pgid;
+    return oss.str();
+  }
+
+  std::string str;
+};
+
+WRITE_CLASS_ENCODER(coll_t)
+
+inline ostream& operator<<(ostream& out, const coll_t& c) {
+  out << c.to_str();
+  return out;
+}
+
+CEPH_HASH_NAMESPACE_START
+  template<> struct hash {
+    size_t operator()(const coll_t &c) const { 
+      size_t h = 0;
+      string str(c.to_str());
+      std::string::const_iterator end(str.end());
+      for (std::string::const_iterator s = str.begin(); s != end; ++s) {
+	h += *s;
+	h += (h << 10);
+	h ^= (h >> 6);
+      }
+      h += (h << 3);
+      h ^= (h >> 11);
+      h += (h << 15);
+      return h;
+    }
+  };
+CEPH_HASH_NAMESPACE_END
+
+inline ostream& operator<<(ostream& out, const ceph_object_layout &ol)
+{
+  out << pg_t(ol.ol_pgid);
+  int su = ol.ol_stripe_unit;
+  if (su)
+    out << ".su=" << su;
+  return out;
+}
+
+
+
+// compound rados version type
+class eversion_t {
+public:
+  version_t version;
+  epoch_t epoch;
+  __u32 __pad;
+  eversion_t() : version(0), epoch(0), __pad(0) {}
+  eversion_t(epoch_t e, version_t v) : version(v), epoch(e), __pad(0) {}
+
+  eversion_t(const ceph_eversion& ce) : 
+    version(ce.version),
+    epoch(ce.epoch),
+    __pad(0) { }
+
+  eversion_t(bufferlist& bl) : __pad(0) { decode(bl); }
+
+  static eversion_t max() {
+    eversion_t max;
+    max.version -= 1;
+    max.epoch -= 1;
+    return max;
+  }
+
+  operator ceph_eversion() {
+    ceph_eversion c;
+    c.epoch = epoch;
+    c.version = version;
+    return c;
+  }
+
+  string get_key_name() const;
+
+  void encode(bufferlist &bl) const {
+    ::encode(version, bl);
+    ::encode(epoch, bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    ::decode(version, bl);
+    ::decode(epoch, bl);
+  }
+  void decode(bufferlist& bl) {
+    bufferlist::iterator p = bl.begin();
+    decode(p);
+  }
+};
+WRITE_CLASS_ENCODER(eversion_t)
+
+inline bool operator==(const eversion_t& l, const eversion_t& r) {
+  return (l.epoch == r.epoch) && (l.version == r.version);
+}
+inline bool operator!=(const eversion_t& l, const eversion_t& r) {
+  return (l.epoch != r.epoch) || (l.version != r.version);
+}
+inline bool operator<(const eversion_t& l, const eversion_t& r) {
+  return (l.epoch == r.epoch) ? (l.version < r.version):(l.epoch < r.epoch);
+}
+inline bool operator<=(const eversion_t& l, const eversion_t& r) {
+  return (l.epoch == r.epoch) ? (l.version <= r.version):(l.epoch <= r.epoch);
+}
+inline bool operator>(const eversion_t& l, const eversion_t& r) {
+  return (l.epoch == r.epoch) ? (l.version > r.version):(l.epoch > r.epoch);
+}
+inline bool operator>=(const eversion_t& l, const eversion_t& r) {
+  return (l.epoch == r.epoch) ? (l.version >= r.version):(l.epoch >= r.epoch);
+}
+inline ostream& operator<<(ostream& out, const eversion_t e) {
+  return out << e.epoch << "'" << e.version;
+}
+
+/**
+ * objectstore_perf_stat_t
+ *
+ * current perf information about the osd
+ */
+struct objectstore_perf_stat_t {
+  // cur_op_latency is in ms since double add/sub are not associative
+  uint32_t filestore_commit_latency;
+  uint32_t filestore_apply_latency;
+
+  objectstore_perf_stat_t() :
+    filestore_commit_latency(0), filestore_apply_latency(0) {}
+
+  void add(const objectstore_perf_stat_t &o) {
+    filestore_commit_latency += o.filestore_commit_latency;
+    filestore_apply_latency += o.filestore_apply_latency;
+  }
+  void sub(const objectstore_perf_stat_t &o) {
+    filestore_commit_latency -= o.filestore_commit_latency;
+    filestore_apply_latency -= o.filestore_apply_latency;
+  }
+  void dump(Formatter *f) const;
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  static void generate_test_instances(std::list& o);
+};
+WRITE_CLASS_ENCODER(objectstore_perf_stat_t)
+
+/** osd_stat
+ * aggregate stats for an osd
+ */
+struct osd_stat_t {
+  int64_t kb, kb_used, kb_avail;
+  vector hb_in, hb_out;
+  int32_t snap_trim_queue_len, num_snap_trimming;
+
+  pow2_hist_t op_queue_age_hist;
+
+  objectstore_perf_stat_t fs_perf_stat;
+
+  osd_stat_t() : kb(0), kb_used(0), kb_avail(0),
+		 snap_trim_queue_len(0), num_snap_trimming(0) {}
+
+  void add(const osd_stat_t& o) {
+    kb += o.kb;
+    kb_used += o.kb_used;
+    kb_avail += o.kb_avail;
+    snap_trim_queue_len += o.snap_trim_queue_len;
+    num_snap_trimming += o.num_snap_trimming;
+    op_queue_age_hist.add(o.op_queue_age_hist);
+    fs_perf_stat.add(o.fs_perf_stat);
+  }
+  void sub(const osd_stat_t& o) {
+    kb -= o.kb;
+    kb_used -= o.kb_used;
+    kb_avail -= o.kb_avail;
+    snap_trim_queue_len -= o.snap_trim_queue_len;
+    num_snap_trimming -= o.num_snap_trimming;
+    op_queue_age_hist.sub(o.op_queue_age_hist);
+    fs_perf_stat.sub(o.fs_perf_stat);
+  }
+
+  void dump(Formatter *f) const;
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  static void generate_test_instances(std::list& o);
+};
+WRITE_CLASS_ENCODER(osd_stat_t)
+
+inline bool operator==(const osd_stat_t& l, const osd_stat_t& r) {
+  return l.kb == r.kb &&
+    l.kb_used == r.kb_used &&
+    l.kb_avail == r.kb_avail &&
+    l.snap_trim_queue_len == r.snap_trim_queue_len &&
+    l.num_snap_trimming == r.num_snap_trimming &&
+    l.hb_in == r.hb_in &&
+    l.hb_out == r.hb_out;
+}
+inline bool operator!=(const osd_stat_t& l, const osd_stat_t& r) {
+  return !(l == r);
+}
+
+
+
+inline ostream& operator<<(ostream& out, const osd_stat_t& s) {
+  return out << "osd_stat(" << kb_t(s.kb_used) << " used, "
+	     << kb_t(s.kb_avail) << " avail, "
+	     << kb_t(s.kb) << " total, "
+	     << "peers " << s.hb_in << "/" << s.hb_out
+	     << " op hist " << s.op_queue_age_hist.h
+	     << ")";
+}
+
+
+/*
+ * pg states
+ */
+#define PG_STATE_CREATING     (1<<0)  // creating
+#define PG_STATE_ACTIVE       (1<<1)  // i am active.  (primary: replicas too)
+#define PG_STATE_CLEAN        (1<<2)  // peers are complete, clean of stray replicas.
+#define PG_STATE_DOWN         (1<<4)  // a needed replica is down, PG offline
+#define PG_STATE_REPLAY       (1<<5)  // crashed, waiting for replay
+//#define PG_STATE_STRAY      (1<<6)  // i must notify the primary i exist.
+#define PG_STATE_SPLITTING    (1<<7)  // i am splitting
+#define PG_STATE_SCRUBBING    (1<<8)  // scrubbing
+#define PG_STATE_SCRUBQ       (1<<9)  // queued for scrub
+#define PG_STATE_DEGRADED     (1<<10) // pg membership not complete
+#define PG_STATE_INCONSISTENT (1<<11) // pg replicas are inconsistent (but shouldn't be)
+#define PG_STATE_PEERING      (1<<12) // pg is (re)peering
+#define PG_STATE_REPAIR       (1<<13) // pg should repair on next scrub
+#define PG_STATE_RECOVERING   (1<<14) // pg is recovering/migrating objects
+#define PG_STATE_BACKFILL_WAIT     (1<<15) // [active] reserving backfill
+#define PG_STATE_INCOMPLETE   (1<<16) // incomplete content, peering failed.
+#define PG_STATE_STALE        (1<<17) // our state for this pg is stale, unknown.
+#define PG_STATE_REMAPPED     (1<<18) // pg is explicitly remapped to different OSDs than CRUSH
+#define PG_STATE_DEEP_SCRUB   (1<<19) // deep scrub: check CRC32 on files
+#define PG_STATE_BACKFILL  (1<<20) // [active] backfilling pg content
+#define PG_STATE_BACKFILL_TOOFULL (1<<21) // backfill can't proceed: too full
+#define PG_STATE_RECOVERY_WAIT (1<<22) // waiting for recovery reservations
+
+std::string pg_state_string(int state);
+
+
+/*
+ * pool_snap_info_t
+ *
+ * attributes for a single pool snapshot.  
+ */
+struct pool_snap_info_t {
+  snapid_t snapid;
+  utime_t stamp;
+  string name;
+
+  void dump(Formatter *f) const;
+  void encode(bufferlist& bl, uint64_t features) const;
+  void decode(bufferlist::iterator& bl);
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER_FEATURES(pool_snap_info_t)
+
+inline ostream& operator<<(ostream& out, const pool_snap_info_t& si) {
+  return out << si.snapid << '(' << si.name << ' ' << si.stamp << ')';
+}
+
+
+/*
+ * pg_pool
+ */
+struct pg_pool_t {
+  enum {
+    TYPE_REPLICATED = 1,     // replication
+    //TYPE_RAID4 = 2,   // raid4 (never implemented)
+    TYPE_ERASURE = 3,      // erasure-coded
+  };
+  static const char *get_type_name(int t) {
+    switch (t) {
+    case TYPE_REPLICATED: return "replicated";
+      //case TYPE_RAID4: return "raid4";
+    case TYPE_ERASURE: return "erasure";
+    default: return "???";
+    }
+  }
+  const char *get_type_name() const {
+    return get_type_name(type);
+  }
+  static const char* get_default_type() {
+    return "replicated";
+  }
+
+  enum {
+    FLAG_HASHPSPOOL = 1<<0, // hash pg seed and pool together (instead of adding)
+    FLAG_FULL       = 1<<1, // pool is full
+    FLAG_DEBUG_FAKE_EC_POOL = 1<<2, // require ReplicatedPG to act like an EC pg
+    FLAG_INCOMPLETE_CLONES = 1<<3, // may have incomplete clones (bc we are/were an overlay)
+  };
+
+  static const char *get_flag_name(int f) {
+    switch (f) {
+    case FLAG_HASHPSPOOL: return "hashpspool";
+    case FLAG_FULL: return "full";
+    case FLAG_DEBUG_FAKE_EC_POOL: return "require_local_rollback";
+    case FLAG_INCOMPLETE_CLONES: return "incomplete_clones";
+    default: return "???";
+    }
+  }
+  static string get_flags_string(uint64_t f) {
+    string s;
+    for (unsigned n=0; f && n<64; ++n) {
+      if (f & (1ull << n)) {
+	if (s.length())
+	  s += ",";
+	s += get_flag_name(1ull << n);
+      }
+    }
+    return s;
+  }
+  string get_flags_string() const {
+    return get_flags_string(flags);
+  }
+
+  typedef enum {
+    CACHEMODE_NONE = 0,                  ///< no caching
+    CACHEMODE_WRITEBACK = 1,             ///< write to cache, flush later
+    CACHEMODE_FORWARD = 2,               ///< forward if not in cache
+    CACHEMODE_READONLY = 3,              ///< handle reads, forward writes [not strongly consistent]
+  } cache_mode_t;
+  static const char *get_cache_mode_name(cache_mode_t m) {
+    switch (m) {
+    case CACHEMODE_NONE: return "none";
+    case CACHEMODE_WRITEBACK: return "writeback";
+    case CACHEMODE_FORWARD: return "forward";
+    case CACHEMODE_READONLY: return "readonly";
+    default: return "unknown";
+    }
+  }
+  static cache_mode_t get_cache_mode_from_str(const string& s) {
+    if (s == "none")
+      return CACHEMODE_NONE;
+    if (s == "writeback")
+      return CACHEMODE_WRITEBACK;
+    if (s == "forward")
+      return CACHEMODE_FORWARD;
+    if (s == "readonly")
+      return CACHEMODE_READONLY;
+    return (cache_mode_t)-1;
+  }
+  const char *get_cache_mode_name() const {
+    return get_cache_mode_name(cache_mode);
+  }
+  bool cache_mode_requires_hit_set() const {
+    switch (cache_mode) {
+    case CACHEMODE_NONE:
+    case CACHEMODE_FORWARD:
+    case CACHEMODE_READONLY:
+      return false;
+    case CACHEMODE_WRITEBACK:
+      return true;
+    default:
+      assert(0 == "implement me");
+    }
+  }
+
+  uint64_t flags;           ///< FLAG_*
+  __u8 type;                ///< TYPE_*
+  __u8 size, min_size;      ///< number of osds in each pg
+  __u8 crush_ruleset;       ///< crush placement rule set
+  __u8 object_hash;         ///< hash mapping object name to ps
+private:
+  __u32 pg_num, pgp_num;    ///< number of pgs
+
+
+public:
+  map properties;  ///< OBSOLETE
+  string erasure_code_profile; ///< name of the erasure code profile in OSDMap
+  epoch_t last_change;      ///< most recent epoch changed, exclusing snapshot changes
+  epoch_t last_force_op_resend; ///< last epoch that forced clients to resend
+  snapid_t snap_seq;        ///< seq for per-pool snapshot
+  epoch_t snap_epoch;       ///< osdmap epoch of last snap
+  uint64_t auid;            ///< who owns the pg
+  __u32 crash_replay_interval; ///< seconds to allow clients to replay ACKed but unCOMMITted requests
+
+  uint64_t quota_max_bytes; ///< maximum number of bytes for this pool
+  uint64_t quota_max_objects; ///< maximum number of objects for this pool
+
+  /*
+   * Pool snaps (global to this pool).  These define a SnapContext for
+   * the pool, unless the client manually specifies an alternate
+   * context.
+   */
+  map snaps;
+  /*
+   * Alternatively, if we are definining non-pool snaps (e.g. via the
+   * Ceph MDS), we must track @removed_snaps (since @snaps is not
+   * used).  Snaps and removed_snaps are to be used exclusive of each
+   * other!
+   */
+  interval_set removed_snaps;
+
+  unsigned pg_num_mask, pgp_num_mask;
+
+  set tiers;      ///< pools that are tiers of us
+  int64_t tier_of;         ///< pool for which we are a tier
+  // Note that write wins for read+write ops
+  int64_t read_tier;       ///< pool/tier for objecter to direct reads to
+  int64_t write_tier;      ///< pool/tier for objecter to direct writes to
+  cache_mode_t cache_mode;  ///< cache pool mode
+
+  bool is_tier() const { return tier_of >= 0; }
+  bool has_tiers() const { return !tiers.empty(); }
+  void clear_tier() {
+    tier_of = -1;
+    clear_read_tier();
+    clear_write_tier();
+    clear_tier_tunables();
+  }
+  bool has_read_tier() const { return read_tier >= 0; }
+  void clear_read_tier() { read_tier = -1; }
+  bool has_write_tier() const { return write_tier >= 0; }
+  void clear_write_tier() { write_tier = -1; }
+  void clear_tier_tunables() {
+    if (cache_mode != CACHEMODE_NONE)
+      flags |= FLAG_INCOMPLETE_CLONES;
+    cache_mode = CACHEMODE_NONE;
+
+    target_max_bytes = 0;
+    target_max_objects = 0;
+    cache_target_dirty_ratio_micro = 0;
+    cache_target_full_ratio_micro = 0;
+    hit_set_params = HitSet::Params();
+    hit_set_period = 0;
+    hit_set_count = 0;
+  }
+
+  uint64_t target_max_bytes;   ///< tiering: target max pool size
+  uint64_t target_max_objects; ///< tiering: target max pool size
+
+  uint32_t cache_target_dirty_ratio_micro; ///< cache: fraction of target to leave dirty
+  uint32_t cache_target_full_ratio_micro;  ///< cache: fraction of target to fill before we evict in earnest
+
+  uint32_t cache_min_flush_age;  ///< minimum age (seconds) before we can flush
+  uint32_t cache_min_evict_age;  ///< minimum age (seconds) before we can evict
+
+  HitSet::Params hit_set_params; ///< The HitSet params to use on this pool
+  uint32_t hit_set_period;      ///< periodicity of HitSet segments (seconds)
+  uint32_t hit_set_count;       ///< number of periods to retain
+
+  uint32_t stripe_width;        ///< erasure coded stripe size in bytes
+
+  pg_pool_t()
+    : flags(0), type(0), size(0), min_size(0),
+      crush_ruleset(0), object_hash(0),
+      pg_num(0), pgp_num(0),
+      last_change(0),
+      last_force_op_resend(0),
+      snap_seq(0), snap_epoch(0),
+      auid(0),
+      crash_replay_interval(0),
+      quota_max_bytes(0), quota_max_objects(0),
+      pg_num_mask(0), pgp_num_mask(0),
+      tier_of(-1), read_tier(-1), write_tier(-1),
+      cache_mode(CACHEMODE_NONE),
+      target_max_bytes(0), target_max_objects(0),
+      cache_target_dirty_ratio_micro(0),
+      cache_target_full_ratio_micro(0),
+      cache_min_flush_age(0),
+      cache_min_evict_age(0),
+      hit_set_params(),
+      hit_set_period(0),
+      hit_set_count(0),
+      stripe_width(0)
+  { }
+
+  void dump(Formatter *f) const;
+
+  uint64_t get_flags() const { return flags; }
+  bool has_flag(uint64_t f) const { return flags & f; }
+
+  /// This method will later return true for ec pools as well
+  bool ec_pool() const {
+    return type == TYPE_ERASURE;
+  }
+  bool require_rollback() const {
+    return ec_pool() || flags & FLAG_DEBUG_FAKE_EC_POOL;
+  }
+
+  /// true if incomplete clones may be present
+  bool allow_incomplete_clones() const {
+    return cache_mode != CACHEMODE_NONE || has_flag(FLAG_INCOMPLETE_CLONES);
+  }
+
+  unsigned get_type() const { return type; }
+  unsigned get_size() const { return size; }
+  unsigned get_min_size() const { return min_size; }
+  int get_crush_ruleset() const { return crush_ruleset; }
+  int get_object_hash() const { return object_hash; }
+  const char *get_object_hash_name() const {
+    return ceph_str_hash_name(get_object_hash());
+  }
+  epoch_t get_last_change() const { return last_change; }
+  epoch_t get_last_force_op_resend() const { return last_force_op_resend; }
+  epoch_t get_snap_epoch() const { return snap_epoch; }
+  snapid_t get_snap_seq() const { return snap_seq; }
+  uint64_t get_auid() const { return auid; }
+  unsigned get_crash_replay_interval() const { return crash_replay_interval; }
+
+  void set_snap_seq(snapid_t s) { snap_seq = s; }
+  void set_snap_epoch(epoch_t e) { snap_epoch = e; }
+
+  void set_stripe_width(uint32_t s) { stripe_width = s; }
+  uint32_t get_stripe_width() const { return stripe_width; }
+
+  bool is_replicated()   const { return get_type() == TYPE_REPLICATED; }
+  bool is_erasure() const { return get_type() == TYPE_ERASURE; }
+
+  bool requires_aligned_append() const { return is_erasure(); }
+  uint64_t required_alignment() const { return stripe_width; }
+
+  bool can_shift_osds() const {
+    switch (get_type()) {
+    case TYPE_REPLICATED:
+      return true;
+    case TYPE_ERASURE:
+      return false;
+    default:
+      assert(0 == "unhandled pool type");
+    }
+  }
+
+  unsigned get_pg_num() const { return pg_num; }
+  unsigned get_pgp_num() const { return pgp_num; }
+
+  unsigned get_pg_num_mask() const { return pg_num_mask; }
+  unsigned get_pgp_num_mask() const { return pgp_num_mask; }
+
+  // if pg_num is not a multiple of two, pgs are not equally sized.
+  // return, for a given pg, the fraction (denominator) of the total
+  // pool size that it represents.
+  unsigned get_pg_num_divisor(pg_t pgid) const;
+
+  void set_pg_num(int p) {
+    pg_num = p;
+    calc_pg_masks();
+  }
+  void set_pgp_num(int p) {
+    pgp_num = p;
+    calc_pg_masks();
+  }
+
+  void set_quota_max_bytes(uint64_t m) {
+    quota_max_bytes = m;
+  }
+  uint64_t get_quota_max_bytes() {
+    return quota_max_bytes;
+  }
+
+  void set_quota_max_objects(uint64_t m) {
+    quota_max_objects = m;
+  }
+  uint64_t get_quota_max_objects() {
+    return quota_max_objects;
+  }
+
+  static int calc_bits_of(int t);
+  void calc_pg_masks();
+
+  /*
+   * we have two snap modes:
+   *  - pool global snaps
+   *    - snap existence/non-existence defined by snaps[] and snap_seq
+   *  - user managed snaps
+   *    - removal governed by removed_snaps
+   *
+   * we know which mode we're using based on whether removed_snaps is empty.
+   * If nothing has been created, both functions report false.
+   */
+  bool is_pool_snaps_mode() const;
+  bool is_unmanaged_snaps_mode() const;
+  bool is_removed_snap(snapid_t s) const;
+
+  /*
+   * build set of known-removed sets from either pool snaps or
+   * explicit removed_snaps set.
+   */
+  void build_removed_snaps(interval_set& rs) const;
+  snapid_t snap_exists(const char *s) const;
+  void add_snap(const char *n, utime_t stamp);
+  void add_unmanaged_snap(uint64_t& snapid);
+  void remove_snap(snapid_t s);
+  void remove_unmanaged_snap(snapid_t s);
+
+  SnapContext get_snap_context() const;
+
+  /// hash a object name+namespace key to a hash position
+  uint32_t hash_key(const string& key, const string& ns) const;
+
+  /// round a hash position down to a pg num
+  uint32_t raw_hash_to_pg(uint32_t v) const;
+
+  /*
+   * map a raw pg (with full precision ps) into an actual pg, for storage
+   */
+  pg_t raw_pg_to_pg(pg_t pg) const;
+  
+  /*
+   * map raw pg (full precision ps) into a placement seed.  include
+   * pool id in that value so that different pools don't use the same
+   * seeds.
+   */
+  ps_t raw_pg_to_pps(pg_t pg) const;
+
+  /// choose a random hash position within a pg
+  uint32_t get_random_pg_position(pg_t pgid, uint32_t seed) const;
+
+  void encode(bufferlist& bl, uint64_t features) const;
+  void decode(bufferlist::iterator& bl);
+
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER_FEATURES(pg_pool_t)
+
+ostream& operator<<(ostream& out, const pg_pool_t& p);
+
+
+/**
+ * a summation of object stats
+ *
+ * This is just a container for object stats; we don't know what for.
+ */
+struct object_stat_sum_t {
+  int64_t num_bytes;    // in bytes
+  int64_t num_objects;
+  int64_t num_object_clones;
+  int64_t num_object_copies;  // num_objects * num_replicas
+  int64_t num_objects_missing_on_primary;
+  int64_t num_objects_degraded;
+  int64_t num_objects_unfound;
+  int64_t num_rd, num_rd_kb;
+  int64_t num_wr, num_wr_kb;
+  int64_t num_scrub_errors;	// total deep and shallow scrub errors
+  int64_t num_shallow_scrub_errors;
+  int64_t num_deep_scrub_errors;
+  int64_t num_objects_recovered;
+  int64_t num_bytes_recovered;
+  int64_t num_keys_recovered;
+  int64_t num_objects_dirty;
+  int64_t num_whiteouts;
+  int64_t num_objects_omap;
+  int64_t num_objects_hit_set_archive;
+
+  object_stat_sum_t()
+    : num_bytes(0),
+      num_objects(0), num_object_clones(0), num_object_copies(0),
+      num_objects_missing_on_primary(0), num_objects_degraded(0), num_objects_unfound(0),
+      num_rd(0), num_rd_kb(0), num_wr(0), num_wr_kb(0),
+      num_scrub_errors(0), num_shallow_scrub_errors(0),
+      num_deep_scrub_errors(0),
+      num_objects_recovered(0),
+      num_bytes_recovered(0),
+      num_keys_recovered(0),
+      num_objects_dirty(0),
+      num_whiteouts(0),
+      num_objects_omap(0),
+      num_objects_hit_set_archive(0)
+  {}
+
+  void floor(int64_t f) {
+#define FLOOR(x) if (x < f) x = f
+    FLOOR(num_bytes);
+    FLOOR(num_objects);
+    FLOOR(num_object_clones);
+    FLOOR(num_object_copies);
+    FLOOR(num_objects_missing_on_primary);
+    FLOOR(num_objects_degraded);
+    FLOOR(num_objects_unfound);
+    FLOOR(num_rd);
+    FLOOR(num_rd_kb);
+    FLOOR(num_wr);
+    FLOOR(num_wr_kb);
+    FLOOR(num_scrub_errors);
+    FLOOR(num_shallow_scrub_errors);
+    FLOOR(num_deep_scrub_errors);
+    FLOOR(num_objects_recovered);
+    FLOOR(num_bytes_recovered);
+    FLOOR(num_keys_recovered);
+    FLOOR(num_objects_dirty);
+    FLOOR(num_whiteouts);
+    FLOOR(num_objects_omap);
+    FLOOR(num_objects_hit_set_archive);
+#undef FLOOR
+  }
+
+  void split(vector &out) const {
+#define SPLIT(PARAM)                            \
+    for (unsigned i = 0; i < out.size(); ++i) { \
+      out[i].PARAM = PARAM / out.size();        \
+      if (i < (PARAM % out.size())) {           \
+	out[i].PARAM++;                         \
+      }                                         \
+    }                                           \
+
+    SPLIT(num_bytes);
+    SPLIT(num_objects);
+    SPLIT(num_object_clones);
+    SPLIT(num_object_copies);
+    SPLIT(num_objects_missing_on_primary);
+    SPLIT(num_objects_degraded);
+    SPLIT(num_objects_unfound);
+    SPLIT(num_rd);
+    SPLIT(num_rd_kb);
+    SPLIT(num_wr);
+    SPLIT(num_wr_kb);
+    SPLIT(num_scrub_errors);
+    SPLIT(num_shallow_scrub_errors);
+    SPLIT(num_deep_scrub_errors);
+    SPLIT(num_objects_recovered);
+    SPLIT(num_bytes_recovered);
+    SPLIT(num_keys_recovered);
+    SPLIT(num_objects_dirty);
+    SPLIT(num_whiteouts);
+    SPLIT(num_objects_omap);
+    SPLIT(num_objects_hit_set_archive);
+#undef SPLIT
+  }
+
+  void clear() {
+    memset(this, 0, sizeof(*this));
+  }
+
+  void calc_copies(int nrep) {
+    num_object_copies = nrep * num_objects;
+  }
+
+  bool is_zero() const {
+    object_stat_sum_t zero;
+    return memcmp(this, &zero, sizeof(zero)) == 0;
+  }
+
+  void add(const object_stat_sum_t& o);
+  void sub(const object_stat_sum_t& o);
+
+  void dump(Formatter *f) const;
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(object_stat_sum_t)
+
+/**
+ * a collection of object stat sums
+ *
+ * This is a collection of stat sums over different categories.
+ */
+struct object_stat_collection_t {
+  object_stat_sum_t sum;
+  map cat_sum;
+
+  void calc_copies(int nrep) {
+    sum.calc_copies(nrep);
+    for (map::iterator p = cat_sum.begin(); p != cat_sum.end(); ++p)
+      p->second.calc_copies(nrep);
+  }
+
+  void dump(Formatter *f) const;
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  static void generate_test_instances(list& o);
+
+  bool is_zero() const {
+    return (cat_sum.empty() && sum.is_zero());
+  }
+
+  void clear() {
+    sum.clear();
+    cat_sum.clear();
+  }
+
+  void floor(int64_t f) {
+    sum.floor(f);
+    for (map::iterator p = cat_sum.begin(); p != cat_sum.end(); ++p)
+      p->second.floor(f);
+  }
+
+  void add(const object_stat_sum_t& o, const string& cat) {
+    sum.add(o);
+    if (cat.length())
+      cat_sum[cat].add(o);
+  }
+
+  void add(const object_stat_collection_t& o) {
+    sum.add(o.sum);
+    for (map::const_iterator p = o.cat_sum.begin();
+	 p != o.cat_sum.end();
+	 ++p)
+      cat_sum[p->first].add(p->second);
+  }
+  void sub(const object_stat_collection_t& o) {
+    sum.sub(o.sum);
+    for (map::const_iterator p = o.cat_sum.begin();
+	 p != o.cat_sum.end();
+	 ++p) {
+      object_stat_sum_t& s = cat_sum[p->first];
+      s.sub(p->second);
+      if (s.is_zero())
+	cat_sum.erase(p->first);
+    }
+  }
+};
+WRITE_CLASS_ENCODER(object_stat_collection_t)
+
+/** pg_stat
+ * aggregate stats for a single PG.
+ */
+struct pg_stat_t {
+  eversion_t version;
+  version_t reported_seq;  // sequence number
+  epoch_t reported_epoch;  // epoch of this report
+  __u32 state;
+  utime_t last_fresh;   // last reported
+  utime_t last_change;  // new state != previous state
+  utime_t last_active;  // state & PG_STATE_ACTIVE
+  utime_t last_clean;   // state & PG_STATE_CLEAN
+  utime_t last_unstale; // (state & PG_STATE_STALE) == 0
+
+  eversion_t log_start;         // (log_start,version]
+  eversion_t ondisk_log_start;  // there may be more on disk
+
+  epoch_t created;
+  epoch_t last_epoch_clean;
+  pg_t parent;
+  __u32 parent_split_bits;
+
+  eversion_t last_scrub;
+  eversion_t last_deep_scrub;
+  utime_t last_scrub_stamp;
+  utime_t last_deep_scrub_stamp;
+  utime_t last_clean_scrub_stamp;
+
+  object_stat_collection_t stats;
+  bool stats_invalid;
+
+  int64_t log_size;
+  int64_t ondisk_log_size;    // >= active_log_size
+
+  vector up, acting;
+  epoch_t mapping_epoch;
+
+  utime_t last_became_active;
+
+  /// true if num_objects_dirty is not accurate (because it was not
+  /// maintained starting from pool creation)
+  bool dirty_stats_invalid;
+  bool omap_stats_invalid;
+  bool hitset_stats_invalid;
+
+  /// up, acting primaries
+  int up_primary;
+  int acting_primary;
+
+  pg_stat_t()
+    : reported_seq(0),
+      reported_epoch(0),
+      state(0),
+      created(0), last_epoch_clean(0),
+      parent_split_bits(0),
+      stats_invalid(false),
+      log_size(0), ondisk_log_size(0),
+      mapping_epoch(0),
+      dirty_stats_invalid(false),
+      omap_stats_invalid(false),
+      hitset_stats_invalid(false),
+      up_primary(-1),
+      acting_primary(-1)
+  { }
+
+  epoch_t get_effective_last_epoch_clean() const {
+    if (state & PG_STATE_CLEAN) {
+      // we are clean as of this report, and should thus take the
+      // reported epoch
+      return reported_epoch;
+    } else {
+      return last_epoch_clean;
+    }
+  }
+
+  pair get_version_pair() const {
+    return make_pair(reported_epoch, reported_seq);
+  }
+
+  void floor(int64_t f) {
+    stats.floor(f);
+    if (log_size < f)
+      log_size = f;
+    if (ondisk_log_size < f)
+      ondisk_log_size = f;
+  }
+
+  void add(const pg_stat_t& o) {
+    stats.add(o.stats);
+    log_size += o.log_size;
+    ondisk_log_size += o.ondisk_log_size;
+  }
+  void sub(const pg_stat_t& o) {
+    stats.sub(o.stats);
+    log_size -= o.log_size;
+    ondisk_log_size -= o.ondisk_log_size;
+  }
+
+  void dump(Formatter *f) const;
+  void dump_brief(Formatter *f) const;
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_stat_t)
+
+/*
+ * summation over an entire pool
+ */
+struct pool_stat_t {
+  object_stat_collection_t stats;
+  int64_t log_size;
+  int64_t ondisk_log_size;    // >= active_log_size
+
+  pool_stat_t() : log_size(0), ondisk_log_size(0)
+  { }
+
+  void floor(int64_t f) {
+    stats.floor(f);
+    if (log_size < f)
+      log_size = f;
+    if (ondisk_log_size < f)
+      ondisk_log_size = f;
+  }
+
+  void add(const pg_stat_t& o) {
+    stats.add(o.stats);
+    log_size += o.log_size;
+    ondisk_log_size += o.ondisk_log_size;
+  }
+  void sub(const pg_stat_t& o) {
+    stats.sub(o.stats);
+    log_size -= o.log_size;
+    ondisk_log_size -= o.ondisk_log_size;
+  }
+
+  bool is_zero() const {
+    return (stats.is_zero() &&
+	    log_size == 0 &&
+	    ondisk_log_size == 0);
+  }
+
+  void dump(Formatter *f) const;
+  void encode(bufferlist &bl, uint64_t features) const;
+  void decode(bufferlist::iterator &bl);
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER_FEATURES(pool_stat_t)
+
+
+// -----------------------------------------
+
+/**
+ * pg_hit_set_info_t - information about a single recorded HitSet
+ *
+ * Track basic metadata about a HitSet, like the nubmer of insertions
+ * and the time range it covers.
+ */
+struct pg_hit_set_info_t {
+  utime_t begin, end;   ///< time interval
+  eversion_t version;   ///< version this HitSet object was written
+
+  pg_hit_set_info_t() {}
+  pg_hit_set_info_t(utime_t b)
+    : begin(b) {}
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_hit_set_info_t)
+
+/**
+ * pg_hit_set_history_t - information about a history of hitsets
+ *
+ * Include information about the currently accumulating hit set as well
+ * as archived/historical ones.
+ */
+struct pg_hit_set_history_t {
+  eversion_t current_last_update;  ///< last version inserted into current set
+  utime_t current_last_stamp;      ///< timestamp of last insert
+  pg_hit_set_info_t current_info;  ///< metadata about the current set
+  list history; ///< archived sets, sorted oldest -> newest
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_hit_set_history_t)
+
+
+// -----------------------------------------
+
+/**
+ * pg_history_t - information about recent pg peering/mapping history
+ *
+ * This is aggressively shared between OSDs to bound the amount of past
+ * history they need to worry about.
+ */
+struct pg_history_t {
+  epoch_t epoch_created;       // epoch in which PG was created
+  epoch_t last_epoch_started;  // lower bound on last epoch started (anywhere, not necessarily locally)
+  epoch_t last_epoch_clean;    // lower bound on last epoch the PG was completely clean.
+  epoch_t last_epoch_split;    // as parent
+  
+  /**
+   * In the event of a map discontinuity, same_*_since may reflect the first
+   * map the osd has seen in the new map sequence rather than the actual start
+   * of the interval.  This is ok since a discontinuity at epoch e means there
+   * must have been a clean interval between e and now and that we cannot be
+   * in the active set during the interval containing e.
+   */
+  epoch_t same_up_since;       // same acting set since
+  epoch_t same_interval_since;   // same acting AND up set since
+  epoch_t same_primary_since;  // same primary at least back through this epoch.
+
+  eversion_t last_scrub;
+  eversion_t last_deep_scrub;
+  utime_t last_scrub_stamp;
+  utime_t last_deep_scrub_stamp;
+  utime_t last_clean_scrub_stamp;
+
+  pg_history_t()
+    : epoch_created(0),
+      last_epoch_started(0), last_epoch_clean(0), last_epoch_split(0),
+      same_up_since(0), same_interval_since(0), same_primary_since(0) {}
+  
+  bool merge(const pg_history_t &other) {
+    // Here, we only update the fields which cannot be calculated from the OSDmap.
+    bool modified = false;
+    if (epoch_created < other.epoch_created) {
+      epoch_created = other.epoch_created;
+      modified = true;
+    }
+    if (last_epoch_started < other.last_epoch_started) {
+      last_epoch_started = other.last_epoch_started;
+      modified = true;
+    }
+    if (last_epoch_clean < other.last_epoch_clean) {
+      last_epoch_clean = other.last_epoch_clean;
+      modified = true;
+    }
+    if (last_epoch_split < other.last_epoch_split) {
+      last_epoch_split = other.last_epoch_split; 
+      modified = true;
+    }
+    if (other.last_scrub > last_scrub) {
+      last_scrub = other.last_scrub;
+      modified = true;
+    }
+    if (other.last_scrub_stamp > last_scrub_stamp) {
+      last_scrub_stamp = other.last_scrub_stamp;
+      modified = true;
+    }
+    if (other.last_deep_scrub > last_deep_scrub) {
+      last_deep_scrub = other.last_deep_scrub;
+      modified = true;
+    }
+    if (other.last_deep_scrub_stamp > last_deep_scrub_stamp) {
+      last_deep_scrub_stamp = other.last_deep_scrub_stamp;
+      modified = true;
+    }
+    if (other.last_clean_scrub_stamp > last_clean_scrub_stamp) {
+      last_clean_scrub_stamp = other.last_clean_scrub_stamp;
+      modified = true;
+    }
+    return modified;
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& p);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_history_t)
+
+inline ostream& operator<<(ostream& out, const pg_history_t& h) {
+  return out << "ec=" << h.epoch_created
+	     << " les/c " << h.last_epoch_started << "/" << h.last_epoch_clean
+	     << " " << h.same_up_since << "/" << h.same_interval_since << "/" << h.same_primary_since;
+}
+
+
+/**
+ * pg_info_t - summary of PG statistics.
+ *
+ * some notes: 
+ *  - last_complete implies we have all objects that existed as of that
+ *    stamp, OR a newer object, OR have already applied a later delete.
+ *  - if last_complete >= log.bottom, then we know pg contents thru log.head.
+ *    otherwise, we have no idea what the pg is supposed to contain.
+ */
+struct pg_info_t {
+  spg_t pgid;
+  eversion_t last_update;    // last object version applied to store.
+  eversion_t last_complete;  // last version pg was complete through.
+  epoch_t last_epoch_started;// last epoch at which this pg started on this osd
+  
+  version_t last_user_version; // last user object version applied to store
+
+  eversion_t log_tail;     // oldest log entry.
+
+  hobject_t last_backfill;   // objects >= this and < last_complete may be missing
+
+  interval_set purged_snaps;
+
+  pg_stat_t stats;
+
+  pg_history_t history;
+  pg_hit_set_history_t hit_set;
+
+  pg_info_t()
+    : last_epoch_started(0), last_user_version(0),
+      last_backfill(hobject_t::get_max())
+  { }
+  pg_info_t(spg_t p)
+    : pgid(p),
+      last_epoch_started(0), last_user_version(0),
+      last_backfill(hobject_t::get_max())
+  { }
+  
+  bool is_empty() const { return last_update.version == 0; }
+  bool dne() const { return history.epoch_created == 0; }
+
+  bool is_incomplete() const { return !last_backfill.is_max(); }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& p);
+  void dump(Formatter *f) const;
+  bool overlaps_with(const pg_info_t &oinfo) const {
+    return last_update > oinfo.log_tail ?
+      oinfo.last_update >= log_tail :
+      last_update >= oinfo.log_tail;
+  }
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_info_t)
+
+inline ostream& operator<<(ostream& out, const pg_info_t& pgi) 
+{
+  out << pgi.pgid << "(";
+  if (pgi.dne())
+    out << " DNE";
+  if (pgi.is_empty())
+    out << " empty";
+  else {
+    out << " v " << pgi.last_update;
+    if (pgi.last_complete != pgi.last_update)
+      out << " lc " << pgi.last_complete;
+    out << " (" << pgi.log_tail << "," << pgi.last_update << "]";
+  }
+  if (pgi.is_incomplete())
+    out << " lb " << pgi.last_backfill;
+  //out << " c " << pgi.epoch_created;
+  out << " local-les=" << pgi.last_epoch_started;
+  out << " n=" << pgi.stats.stats.sum.num_objects;
+  out << " " << pgi.history
+      << ")";
+  return out;
+}
+
+struct pg_notify_t {
+  epoch_t query_epoch;
+  epoch_t epoch_sent;
+  pg_info_t info;
+  shard_id_t to;
+  shard_id_t from;
+  pg_notify_t() :
+    query_epoch(0), epoch_sent(0), to(ghobject_t::no_shard()),
+    from(ghobject_t::no_shard()) {}
+  pg_notify_t(
+    shard_id_t to,
+    shard_id_t from,
+    epoch_t query_epoch,
+    epoch_t epoch_sent,
+    const pg_info_t &info)
+    : query_epoch(query_epoch),
+      epoch_sent(epoch_sent),
+      info(info), to(to), from(from) {
+    assert(from == info.pgid.shard);
+  }
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &p);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list &o);
+};
+WRITE_CLASS_ENCODER(pg_notify_t)
+ostream &operator<<(ostream &lhs, const pg_notify_t ¬ify);
+
+
+/**
+ * pg_interval_t - information about a past interval
+ */
+class OSDMap;
+struct pg_interval_t {
+  vector up, acting;
+  epoch_t first, last;
+  bool maybe_went_rw;
+  int primary;
+  int up_primary;
+
+  pg_interval_t()
+    : first(0), last(0),
+      maybe_went_rw(false),
+      primary(-1),
+      up_primary(-1)
+  {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  /**
+   * Integrates a new map into *past_intervals, returns true
+   * if an interval was closed out.
+   */
+  static bool check_new_interval(
+    int old_acting_primary,                     ///< [in] primary as of lastmap
+    int new_acting_primary,                     ///< [in] primary as of lastmap
+    const vector &old_acting,              ///< [in] acting as of lastmap
+    const vector &new_acting,              ///< [in] acting as of osdmap
+    int old_up_primary,                         ///< [in] up primary of lastmap
+    int new_up_primary,                         ///< [in] up primary of osdmap
+    const vector &old_up,                  ///< [in] up as of lastmap
+    const vector &new_up,                  ///< [in] up as of osdmap
+    epoch_t same_interval_since,                ///< [in] as of osdmap
+    epoch_t last_epoch_clean,                   ///< [in] current
+    ceph::shared_ptr osdmap,  ///< [in] current map
+    ceph::shared_ptr lastmap, ///< [in] last map
+    int64_t poolid,                             ///< [in] pool for pg
+    pg_t pgid,                                  ///< [in] pgid for pg
+    map *past_intervals,///< [out] intervals
+    ostream *out = 0                            ///< [out] debug ostream
+    );
+};
+WRITE_CLASS_ENCODER(pg_interval_t)
+
+ostream& operator<<(ostream& out, const pg_interval_t& i);
+
+typedef map pg_interval_map_t;
+
+
+/** 
+ * pg_query_t - used to ask a peer for information about a pg.
+ *
+ * note: if version=0, type=LOG, then we just provide our full log.
+ */
+struct pg_query_t {
+  enum {
+    INFO = 0,
+    LOG = 1,
+    MISSING = 4,
+    FULLLOG = 5,
+  };
+  const char *get_type_name() const {
+    switch (type) {
+    case INFO: return "info";
+    case LOG: return "log";
+    case MISSING: return "missing";
+    case FULLLOG: return "fulllog";
+    default: return "???";
+    }
+  }
+
+  __s32 type;
+  eversion_t since;
+  pg_history_t history;
+  epoch_t epoch_sent;
+  shard_id_t to;
+  shard_id_t from;
+
+  pg_query_t() : type(-1), epoch_sent(0), to(ghobject_t::NO_SHARD),
+		 from(ghobject_t::NO_SHARD) {}
+  pg_query_t(
+    int t,
+    shard_id_t to,
+    shard_id_t from,
+    const pg_history_t& h,
+    epoch_t epoch_sent)
+    : type(t),
+      history(h),
+      epoch_sent(epoch_sent),
+      to(to), from(from) {
+    assert(t != LOG);
+  }
+  pg_query_t(
+    int t,
+    shard_id_t to,
+    shard_id_t from,
+    eversion_t s,
+    const pg_history_t& h,
+    epoch_t epoch_sent)
+    : type(t), since(s), history(h),
+      epoch_sent(epoch_sent), to(to), from(from) {
+    assert(t == LOG);
+  }
+  
+  void encode(bufferlist &bl, uint64_t features) const;
+  void decode(bufferlist::iterator &bl);
+
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER_FEATURES(pg_query_t)
+
+inline ostream& operator<<(ostream& out, const pg_query_t& q) {
+  out << "query(" << q.get_type_name() << " " << q.since;
+  if (q.type == pg_query_t::LOG)
+    out << " " << q.history;
+  out << ")";
+  return out;
+}
+
+class PGBackend;
+class ObjectModDesc {
+  bool can_local_rollback;
+  bool rollback_info_completed;
+public:
+  class Visitor {
+  public:
+    virtual void append(uint64_t old_offset) {}
+    virtual void setattrs(map > &attrs) {}
+    virtual void rmobject(version_t old_version) {}
+    virtual void create() {}
+    virtual void update_snaps(set &old_snaps) {}
+    virtual ~Visitor() {}
+  };
+  void visit(Visitor *visitor) const;
+  mutable bufferlist bl;
+  enum ModID {
+    APPEND = 1,
+    SETATTRS = 2,
+    DELETE = 3,
+    CREATE = 4,
+    UPDATE_SNAPS = 5
+  };
+  ObjectModDesc() : can_local_rollback(true), rollback_info_completed(false) {}
+  void claim(ObjectModDesc &other) {
+    bl.clear();
+    bl.claim(other.bl);
+    can_local_rollback = other.can_local_rollback;
+    rollback_info_completed = other.rollback_info_completed;
+  }
+  void claim_append(ObjectModDesc &other) {
+    if (!can_local_rollback || rollback_info_completed)
+      return;
+    if (!other.can_local_rollback) {
+      mark_unrollbackable();
+      return;
+    }
+    bl.claim_append(other.bl);
+    rollback_info_completed = other.rollback_info_completed;
+  }
+  void swap(ObjectModDesc &other) {
+    bl.swap(other.bl);
+
+    bool temp = other.can_local_rollback;
+    other.can_local_rollback = can_local_rollback;
+    can_local_rollback = temp;
+
+    temp = other.rollback_info_completed;
+    other.rollback_info_completed = rollback_info_completed;
+    rollback_info_completed = temp;
+  }
+  void append_id(ModID id) {
+    uint8_t _id(id);
+    ::encode(_id, bl);
+  }
+  void append(uint64_t old_size) {
+    if (!can_local_rollback || rollback_info_completed)
+      return;
+    ENCODE_START(1, 1, bl);
+    append_id(APPEND);
+    ::encode(old_size, bl);
+    ENCODE_FINISH(bl);
+  }
+  void setattrs(map > &old_attrs) {
+    if (!can_local_rollback || rollback_info_completed)
+      return;
+    ENCODE_START(1, 1, bl);
+    append_id(SETATTRS);
+    ::encode(old_attrs, bl);
+    ENCODE_FINISH(bl);
+  }
+  bool rmobject(version_t deletion_version) {
+    if (!can_local_rollback || rollback_info_completed)
+      return false;
+    ENCODE_START(1, 1, bl);
+    append_id(DELETE);
+    ::encode(deletion_version, bl);
+    ENCODE_FINISH(bl);
+    rollback_info_completed = true;
+    return true;
+  }
+  void create() {
+    if (!can_local_rollback || rollback_info_completed)
+      return;
+    rollback_info_completed = true;
+    ENCODE_START(1, 1, bl);
+    append_id(CREATE);
+    ENCODE_FINISH(bl);
+  }
+  void update_snaps(set &old_snaps) {
+    if (!can_local_rollback || rollback_info_completed)
+      return;
+    ENCODE_START(1, 1, bl);
+    append_id(UPDATE_SNAPS);
+    ::encode(old_snaps, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  // cannot be rolled back
+  void mark_unrollbackable() {
+    can_local_rollback = false;
+    bl.clear();
+  }
+  bool can_rollback() const {
+    return can_local_rollback;
+  }
+  bool empty() const {
+    return can_local_rollback && (bl.length() == 0);
+  }
+
+  /**
+   * Create fresh copy of bl bytes to avoid keeping large buffers around
+   * in the case that bl contains ptrs which point into a much larger
+   * message buffer
+   */
+  void trim_bl() {
+    if (bl.length() > 0)
+      bl.rebuild();
+  }
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ObjectModDesc)
+
+
+/**
+ * pg_log_entry_t - single entry/event in pg log
+ *
+ */
+struct pg_log_entry_t {
+  enum {
+    MODIFY = 1,   // some unspecified modification (but not *all* modifications)
+    CLONE = 2,    // cloned object from head
+    DELETE = 3,   // deleted object
+    BACKLOG = 4,  // event invented by generate_backlog [deprecated]
+    LOST_REVERT = 5, // lost new version, revert to an older version.
+    LOST_DELETE = 6, // lost new version, revert to no object (deleted).
+    LOST_MARK = 7,   // lost new version, now EIO
+    PROMOTE = 8,     // promoted object from another tier
+    CLEAN = 9,       // mark an object clean
+  };
+  static const char *get_op_name(int op) {
+    switch (op) {
+    case MODIFY:
+      return "modify  ";
+    case PROMOTE:
+      return "promote ";
+    case CLONE:
+      return "clone   ";
+    case DELETE:
+      return "delete  ";
+    case BACKLOG:
+      return "backlog ";
+    case LOST_REVERT:
+      return "l_revert";
+    case LOST_DELETE:
+      return "l_delete";
+    case LOST_MARK:
+      return "l_mark  ";
+    case CLEAN:
+      return "clean   ";
+    default:
+      return "unknown ";
+    }
+  }
+  const char *get_op_name() const {
+    return get_op_name(op);
+  }
+
+  __s32      op;
+  hobject_t  soid;
+  eversion_t version, prior_version, reverting_to;
+  version_t user_version; // the user version for this entry
+  osd_reqid_t reqid;  // caller+tid to uniquely identify request
+  utime_t     mtime;  // this is the _user_ mtime, mind you
+  bufferlist snaps;   // only for clone entries
+  bool invalid_hash; // only when decoding sobject_t based entries
+  bool invalid_pool; // only when decoding pool-less hobject based entries
+
+  uint64_t offset;   // [soft state] my offset on disk
+
+  /// describes state for a locally-rollbackable entry
+  ObjectModDesc mod_desc;
+      
+  pg_log_entry_t()
+    : op(0), user_version(0),
+      invalid_hash(false), invalid_pool(false), offset(0) {}
+  pg_log_entry_t(int _op, const hobject_t& _soid, 
+		 const eversion_t& v, const eversion_t& pv,
+		 version_t uv,
+		 const osd_reqid_t& rid, const utime_t& mt)
+    : op(_op), soid(_soid), version(v),
+      prior_version(pv), user_version(uv),
+      reqid(rid), mtime(mt), invalid_hash(false), invalid_pool(false),
+      offset(0) {}
+      
+  bool is_clone() const { return op == CLONE; }
+  bool is_modify() const { return op == MODIFY; }
+  bool is_promote() const { return op == PROMOTE; }
+  bool is_clean() const { return op == CLEAN; }
+  bool is_backlog() const { return op == BACKLOG; }
+  bool is_lost_revert() const { return op == LOST_REVERT; }
+  bool is_lost_delete() const { return op == LOST_DELETE; }
+  bool is_lost_mark() const { return op == LOST_MARK; }
+
+  bool is_update() const {
+    return
+      is_clone() || is_modify() || is_promote() || is_clean() ||
+      is_backlog() || is_lost_revert() || is_lost_mark();
+  }
+  bool is_delete() const {
+    return op == DELETE || op == LOST_DELETE;
+  }
+      
+  bool reqid_is_indexed() const {
+    return reqid != osd_reqid_t() && (op == MODIFY || op == DELETE);
+  }
+
+  string get_key_name() const;
+  void encode_with_checksum(bufferlist& bl) const;
+  void decode_with_checksum(bufferlist::iterator& p);
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+};
+WRITE_CLASS_ENCODER(pg_log_entry_t)
+
+ostream& operator<<(ostream& out, const pg_log_entry_t& e);
+
+
+
+/**
+ * pg_log_t - incremental log of recent pg changes.
+ *
+ *  serves as a recovery queue for recent changes.
+ */
+struct pg_log_t {
+  /*
+   *   head - newest entry (update|delete)
+   *   tail - entry previous to oldest (update|delete) for which we have
+   *          complete negative information.  
+   * i.e. we can infer pg contents for any store whose last_update >= tail.
+   */
+  eversion_t head;    // newest entry
+  eversion_t tail;    // version prior to oldest
+
+  // We can rollback rollback-able entries > can_rollback_to
+  eversion_t can_rollback_to;
+
+  // always <= can_rollback_to, indicates how far stashed rollback
+  // data can be found
+  eversion_t rollback_info_trimmed_to;
+
+  list log;  // the actual log.
+  
+  pg_log_t() {}
+
+  void clear() {
+    eversion_t z;
+    can_rollback_to = head = tail = z;
+    log.clear();
+  }
+
+  bool empty() const {
+    return log.empty();
+  }
+
+  bool null() const {
+    return head.version == 0 && head.epoch == 0;
+  }
+
+  size_t approx_size() const {
+    return head.version - tail.version;
+  }
+
+  list::const_iterator find_entry(eversion_t v) const {
+    int fromhead = head.version - v.version;
+    int fromtail = v.version - tail.version;
+    list::const_iterator p;
+    if (fromhead < fromtail) {
+      p = log.end();
+      --p;
+      while (p->version > v)
+	--p;
+      return p;
+    } else {
+      p = log.begin();
+      while (p->version < v)
+	++p;
+      return p;
+    }      
+  }
+
+  list::iterator find_entry(eversion_t v) {
+    int fromhead = head.version - v.version;
+    int fromtail = v.version - tail.version;
+    list::iterator p;
+    if (fromhead < fromtail) {
+      p = log.end();
+      --p;
+      while (p->version > v)
+	--p;
+      return p;
+    } else {
+      p = log.begin();
+      while (p->version < v)
+	++p;
+      return p;
+    }      
+  }
+
+  /**
+   * copy entries from the tail of another pg_log_t
+   *
+   * @param other pg_log_t to copy from
+   * @param from copy entries after this version
+   */
+  void copy_after(const pg_log_t &other, eversion_t from);
+
+  /**
+   * copy a range of entries from another pg_log_t
+   *
+   * @param other pg_log_t to copy from
+   * @param from copy entries after this version
+   * @parem to up to and including this version
+   */
+  void copy_range(const pg_log_t &other, eversion_t from, eversion_t to);
+
+  /**
+   * copy up to N entries
+   *
+   * @param o source log
+   * @param max max number of entreis to copy
+   */
+  void copy_up_to(const pg_log_t &other, int max);
+
+  ostream& print(ostream& out) const;
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl, int64_t pool = -1);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_log_t)
+
+inline ostream& operator<<(ostream& out, const pg_log_t& log) 
+{
+  out << "log((" << log.tail << "," << log.head << "], crt="
+      << log.can_rollback_to << ")";
+  return out;
+}
+
+
+/**
+ * pg_missing_t - summary of missing objects.
+ *
+ *  kept in memory, as a supplement to pg_log_t
+ *  also used to pass missing info in messages.
+ */
+struct pg_missing_t {
+  struct item {
+    eversion_t need, have;
+    item() {}
+    item(eversion_t n) : need(n) {}  // have no old version
+    item(eversion_t n, eversion_t h) : need(n), have(h) {}
+
+    void encode(bufferlist& bl) const {
+      ::encode(need, bl);
+      ::encode(have, bl);
+    }
+    void decode(bufferlist::iterator& bl) {
+      ::decode(need, bl);
+      ::decode(have, bl);
+    }
+    void dump(Formatter *f) const {
+      f->dump_stream("need") << need;
+      f->dump_stream("have") << have;
+    }
+    static void generate_test_instances(list& o) {
+      o.push_back(new item);
+      o.push_back(new item);
+      o.back()->need = eversion_t(1, 2);
+      o.back()->have = eversion_t(1, 1);
+    }
+  }; 
+  WRITE_CLASS_ENCODER(item)
+
+  map missing;         // oid -> (need v, have v)
+  map rmissing;  // v -> oid
+
+  unsigned int num_missing() const;
+  bool have_missing() const;
+  void swap(pg_missing_t& o);
+  bool is_missing(const hobject_t& oid) const;
+  bool is_missing(const hobject_t& oid, eversion_t v) const;
+  eversion_t have_old(const hobject_t& oid) const;
+  void add_next_event(const pg_log_entry_t& e);
+  void revise_need(hobject_t oid, eversion_t need);
+  void revise_have(hobject_t oid, eversion_t have);
+  void add(const hobject_t& oid, eversion_t need, eversion_t have);
+  void rm(const hobject_t& oid, eversion_t v);
+  void rm(const std::map::iterator &m);
+  void got(const hobject_t& oid, eversion_t v);
+  void got(const std::map::iterator &m);
+  void split_into(pg_t child_pgid, unsigned split_bits, pg_missing_t *omissing);
+
+  void clear() {
+    missing.clear();
+    rmissing.clear();
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl, int64_t pool = -1);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_missing_t::item)
+WRITE_CLASS_ENCODER(pg_missing_t)
+
+ostream& operator<<(ostream& out, const pg_missing_t::item& i);
+ostream& operator<<(ostream& out, const pg_missing_t& missing);
+
+/**
+ * pg list objects response format
+ *
+ */
+struct pg_ls_response_t {
+  collection_list_handle_t handle; 
+  list > entries;
+
+  void encode(bufferlist& bl) const {
+    __u8 v = 1;
+    ::encode(v, bl);
+    ::encode(handle, bl);
+    ::encode(entries, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    __u8 v;
+    ::decode(v, bl);
+    assert(v == 1);
+    ::decode(handle, bl);
+    ::decode(entries, bl);
+  }
+  void dump(Formatter *f) const {
+    f->dump_stream("handle") << handle;
+    f->open_array_section("entries");
+    for (list >::const_iterator p = entries.begin(); p != entries.end(); ++p) {
+      f->open_object_section("object");
+      f->dump_stream("object") << p->first;
+      f->dump_string("key", p->second);
+      f->close_section();
+    }
+    f->close_section();
+  }
+  static void generate_test_instances(list& o) {
+    o.push_back(new pg_ls_response_t);
+    o.push_back(new pg_ls_response_t);
+    o.back()->handle = hobject_t(object_t("hi"), "key", 1, 2, -1, "");
+    o.back()->entries.push_back(make_pair(object_t("one"), string()));
+    o.back()->entries.push_back(make_pair(object_t("two"), string("twokey")));
+  }
+};
+
+WRITE_CLASS_ENCODER(pg_ls_response_t)
+
+/**
+ * object_copy_cursor_t
+ */
+struct object_copy_cursor_t {
+  bool attr_complete;
+  uint64_t data_offset;
+  bool data_complete;
+  string omap_offset;
+  bool omap_complete;
+
+  object_copy_cursor_t()
+    : attr_complete(false),
+      data_offset(0),
+      data_complete(false),
+      omap_complete(false)
+  {}
+
+  bool is_initial() const {
+    return !attr_complete && data_offset == 0 && omap_offset.empty();
+  }
+  bool is_complete() const {
+    return attr_complete && data_complete && omap_complete;
+  }
+
+  static void generate_test_instances(list& o);
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(object_copy_cursor_t)
+
+/**
+ * object_copy_data_t
+ *
+ * Return data from a copy request. The semantics are a little strange
+ * as a result of the encoding's heritage.
+ *
+ * In particular, the sender unconditionally fills in the cursor (from what
+ * it receives and sends), the size, and the mtime, but is responsible for
+ * figuring out whether it should put any data in the attrs, data, or
+ * omap members (corresponding to xattrs, object data, and the omap entries)
+ * based on external data (the client includes a max amount to return with
+ * the copy request). The client then looks into the attrs, data, and/or omap
+ * based on the contents of the cursor.
+ */
+struct object_copy_data_t {
+  object_copy_cursor_t cursor;
+  uint64_t size;
+  utime_t mtime;
+  map attrs;
+  bufferlist data;
+  bufferlist omap_header;
+  map omap;
+  string category;
+
+  /// which snaps we are defined for (if a snap and not the head)
+  vector snaps;
+  ///< latest snap seq for the object (if head)
+  snapid_t snap_seq;
+public:
+  object_copy_data_t() : size((uint64_t)-1) {}
+
+  static void generate_test_instances(list& o);
+  void encode_classic(bufferlist& bl) const;
+  void decode_classic(bufferlist::iterator& bl);
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(object_copy_data_t)
+
+/**
+ * pg creation info
+ */
+struct pg_create_t {
+  epoch_t created;   // epoch pg created
+  pg_t parent;       // split from parent (if != pg_t())
+  __s32 split_bits;
+
+  pg_create_t()
+    : created(0), split_bits(0) {}
+  pg_create_t(unsigned c, pg_t p, int s)
+    : created(c), parent(p), split_bits(s) {}
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(pg_create_t)
+
+// -----------------------------------------
+
+struct osd_peer_stat_t {
+  utime_t stamp;
+
+  osd_peer_stat_t() { }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(osd_peer_stat_t)
+
+ostream& operator<<(ostream& out, const osd_peer_stat_t &stat);
+
+
+// -----------------------------------------
+
+class ObjectExtent {
+ public:
+  object_t    oid;       // object id
+  uint64_t    objectno;
+  uint64_t    offset;    // in object
+  uint64_t    length;    // in object
+  uint64_t    truncate_size;	// in object
+
+  object_locator_t oloc;   // object locator (pool etc)
+
+  vector >  buffer_extents;  // off -> len.  extents in buffer being mapped (may be fragmented bc of striping!)
+  
+  ObjectExtent() : objectno(0), offset(0), length(0), truncate_size(0) {}
+  ObjectExtent(object_t o, uint64_t ono, uint64_t off, uint64_t l, uint64_t ts) :
+    oid(o), objectno(ono), offset(off), length(l), truncate_size(ts) { }
+};
+
+inline ostream& operator<<(ostream& out, const ObjectExtent &ex)
+{
+  return out << "extent(" 
+             << ex.oid << " (" << ex.objectno << ") in " << ex.oloc
+             << " " << ex.offset << "~" << ex.length
+	     << " -> " << ex.buffer_extents
+             << ")";
+}
+
+
+
+
+
+
+// ---------------------------------------
+
+class OSDSuperblock {
+public:
+  uuid_d cluster_fsid, osd_fsid;
+  int32_t whoami;    // my role in this fs.
+  epoch_t current_epoch;             // most recent epoch
+  epoch_t oldest_map, newest_map;    // oldest/newest maps we have.
+  double weight;
+
+  CompatSet compat_features;
+
+  // last interval over which i mounted and was then active
+  epoch_t mounted;     // last epoch i mounted
+  epoch_t clean_thru;  // epoch i was active and clean thru
+  epoch_t last_map_marked_full; // last epoch osdmap was marked full
+
+  OSDSuperblock() : 
+    whoami(-1), 
+    current_epoch(0), oldest_map(0), newest_map(0), weight(0),
+    mounted(0), clean_thru(0), last_map_marked_full(0) {
+  }
+
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(OSDSuperblock)
+
+inline ostream& operator<<(ostream& out, const OSDSuperblock& sb)
+{
+  return out << "sb(" << sb.cluster_fsid
+             << " osd." << sb.whoami
+	     << " " << sb.osd_fsid
+             << " e" << sb.current_epoch
+             << " [" << sb.oldest_map << "," << sb.newest_map << "]"
+	     << " lci=[" << sb.mounted << "," << sb.clean_thru << "]"
+             << ")";
+}
+
+
+// -------
+
+WRITE_CLASS_ENCODER(interval_set)
+
+
+
+
+
+/*
+ * attached to object head.  describes most recent snap context, and
+ * set of existing clones.
+ */
+struct SnapSet {
+  snapid_t seq;
+  bool head_exists;
+  vector snaps;    // descending
+  vector clones;   // ascending
+  map > clone_overlap;  // overlap w/ next newest
+  map clone_size;
+
+  SnapSet() : seq(0), head_exists(false) {}
+  SnapSet(bufferlist& bl) {
+    bufferlist::iterator p = bl.begin();
+    decode(p);
+  }
+
+  /// populate SnapSet from a librados::snap_set_t
+  void from_snap_set(const librados::snap_set_t& ss);
+
+  /// get space accounted to clone
+  uint64_t get_clone_bytes(snapid_t clone) const;
+    
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);  
+
+  SnapContext get_ssc_as_of(snapid_t as_of) const {
+    SnapContext out;
+    out.seq = as_of;
+    for (vector::const_iterator i = snaps.begin();
+	 i != snaps.end();
+	 ++i) {
+      if (*i <= as_of)
+	out.snaps.push_back(*i);
+    }
+    return out;
+  }
+
+  // return min element of snaps > after, return max if no such element
+  snapid_t get_first_snap_after(snapid_t after, snapid_t max) const {
+    for (vector::const_reverse_iterator i = snaps.rbegin();
+	 i != snaps.rend();
+	 ++i) {
+      if (*i > after)
+	return *i;
+    }
+    return max;
+  }
+};
+WRITE_CLASS_ENCODER(SnapSet)
+
+ostream& operator<<(ostream& out, const SnapSet& cs);
+
+
+
+#define OI_ATTR "_"
+#define SS_ATTR "snapset"
+
+struct watch_info_t {
+  uint64_t cookie;
+  uint32_t timeout_seconds;
+  entity_addr_t addr;
+
+  watch_info_t() : cookie(0), timeout_seconds(0) { }
+  watch_info_t(uint64_t c, uint32_t t, const entity_addr_t& a) : cookie(c), timeout_seconds(t), addr(a) {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(watch_info_t)
+
+static inline bool operator==(const watch_info_t& l, const watch_info_t& r) {
+  return l.cookie == r.cookie && l.timeout_seconds == r.timeout_seconds
+	    && l.addr == r.addr;
+}
+
+static inline ostream& operator<<(ostream& out, const watch_info_t& w) {
+  return out << "watch(cookie " << w.cookie << " " << w.timeout_seconds << "s"
+    << " " << w.addr << ")";
+}
+
+struct notify_info_t {
+  uint64_t cookie;
+  uint32_t timeout;
+  bufferlist bl;
+};
+
+static inline ostream& operator<<(ostream& out, const notify_info_t& n) {
+  return out << "notify(cookie " << n.cookie << " " << n.timeout << "s)";
+}
+
+
+struct object_info_t {
+  hobject_t soid;
+  string category;
+
+  eversion_t version, prior_version;
+  version_t user_version;
+  osd_reqid_t last_reqid;
+
+  uint64_t size;
+  utime_t mtime;
+
+  // note: these are currently encoded into a total 16 bits; see
+  // encode()/decode() for the weirdness.
+  typedef enum {
+    FLAG_LOST     = 1<<0,
+    FLAG_WHITEOUT = 1<<1,  // object logically does not exist
+    FLAG_DIRTY    = 1<<2,  // object has been modified since last flushed or undirtied
+    FLAG_OMAP     = 1 << 3,  // has (or may have) some/any omap data
+    // ...
+    FLAG_USES_TMAP = 1<<8,  // deprecated; no longer used.
+  } flag_t;
+
+  flag_t flags;
+
+  static string get_flag_string(flag_t flags) {
+    string s;
+    if (flags & FLAG_LOST)
+      s += "|lost";
+    if (flags & FLAG_WHITEOUT)
+      s += "|whiteout";
+    if (flags & FLAG_DIRTY)
+      s += "|dirty";
+    if (flags & FLAG_USES_TMAP)
+      s += "|uses_tmap";
+    if (flags & FLAG_OMAP)
+      s += "|omap";
+    if (s.length())
+      return s.substr(1);
+    return s;
+  }
+  string get_flag_string() const {
+    return get_flag_string(flags);
+  }
+
+  osd_reqid_t wrlock_by;   // [head]
+  vector snaps;  // [clone]
+
+  uint64_t truncate_seq, truncate_size;
+
+  map, watch_info_t> watchers;
+
+  void copy_user_bits(const object_info_t& other);
+
+  static ps_t legacy_object_locator_to_ps(const object_t &oid, 
+					  const object_locator_t &loc);
+
+  bool test_flag(flag_t f) const {
+    return (flags & f) == f;
+  }
+  void set_flag(flag_t f) {
+    flags = (flag_t)(flags | f);
+  }
+  void clear_flag(flag_t f) {
+    flags = (flag_t)(flags & ~f);
+  }
+  bool is_lost() const {
+    return test_flag(FLAG_LOST);
+  }
+  bool is_whiteout() const {
+    return test_flag(FLAG_WHITEOUT);
+  }
+  bool is_dirty() const {
+    return test_flag(FLAG_DIRTY);
+  }
+  bool is_omap() const {
+    return test_flag(FLAG_OMAP);
+  }
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void decode(bufferlist& bl) {
+    bufferlist::iterator p = bl.begin();
+    decode(p);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  explicit object_info_t()
+    : user_version(0), size(0), flags((flag_t)0),
+      truncate_seq(0), truncate_size(0)
+  {}
+
+  object_info_t(const hobject_t& s)
+    : soid(s),
+      user_version(0), size(0), flags((flag_t)0),
+      truncate_seq(0), truncate_size(0) {}
+
+  object_info_t(bufferlist& bl) {
+    decode(bl);
+  }
+};
+WRITE_CLASS_ENCODER(object_info_t)
+
+struct ObjectState {
+  object_info_t oi;
+  bool exists;         ///< the stored object exists (i.e., we will remember the object_info_t)
+
+  ObjectState() : exists(false) {}
+
+  ObjectState(const object_info_t &oi_, bool exists_)
+    : oi(oi_), exists(exists_) {}
+};
+
+
+struct SnapSetContext {
+  hobject_t oid;
+  int ref;
+  bool registered;
+  SnapSet snapset;
+  bool exists;
+
+  SnapSetContext(const hobject_t& o) :
+    oid(o), ref(0), registered(false), exists(true) { }
+};
+
+
+/*
+  * keep tabs on object modifications that are in flight.
+  * we need to know the projected existence, size, snapset,
+  * etc., because we don't send writes down to disk until after
+  * replicas ack.
+  */
+
+struct ObjectContext;
+
+typedef ceph::shared_ptr ObjectContextRef;
+
+struct ObjectContext {
+  ObjectState obs;
+
+  SnapSetContext *ssc;  // may be null
+
+  Context *destructor_callback;
+
+private:
+  Mutex lock;
+public:
+  Cond cond;
+  int unstable_writes, readers, writers_waiting, readers_waiting;
+
+  /// in-progress copyfrom ops for this object
+  bool blocked;
+
+  // set if writes for this object are blocked on another objects recovery
+  ObjectContextRef blocked_by;      // object blocking our writes
+  set blocking;   // objects whose writes we block
+  bool requeue_scrub_on_unblock;    // true if we need to requeue scrub on unblock
+
+  // any entity in obs.oi.watchers MUST be in either watchers or unconnected_watchers.
+  map, WatchRef> watchers;
+
+  struct RWState {
+    enum State {
+      RWNONE,
+      RWREAD,
+      RWWRITE
+    };
+    static const char *get_state_name(State s) {
+      switch (s) {
+      case RWNONE: return "none";
+      case RWREAD: return "read";
+      case RWWRITE: return "write";
+      default: return "???";
+      }
+    }
+    const char *get_state_name() const {
+      return get_state_name(state);
+    }
+
+    State state;                 ///< rw state
+    uint64_t count;              ///< number of readers or writers
+    list waiters;  ///< ops waiting on state change
+
+    /// if set, restart backfill when we can get a read lock
+    bool backfill_read_marker;
+
+    /// if set, requeue snaptrim on lock release
+    bool snaptrimmer_write_marker;
+
+    RWState()
+      : state(RWNONE),
+	count(0),
+	backfill_read_marker(false),
+	snaptrimmer_write_marker(false)
+    {}
+    bool get_read(OpRequestRef op) {
+      if (get_read_lock()) {
+	return true;
+      } // else
+      waiters.push_back(op);
+      return false;
+    }
+    /// this function adjusts the counts if necessary
+    bool get_read_lock() {
+      // don't starve anybody!
+      if (!waiters.empty()) {
+	return false;
+      }
+      switch (state) {
+      case RWNONE:
+	assert(count == 0);
+	state = RWREAD;
+	// fall through
+      case RWREAD:
+	count++;
+	return true;
+      case RWWRITE:
+	return false;
+      default:
+	assert(0 == "unhandled case");
+	return false;
+      }
+    }
+
+    bool get_write(OpRequestRef op, bool greedy=false) {
+      if (get_write_lock(greedy)) {
+	return true;
+      } // else
+      if (op)
+	waiters.push_back(op);
+      return false;
+    }
+    bool get_write_lock(bool greedy=false) {
+      if (!greedy) {
+	// don't starve anybody!
+	if (!waiters.empty() ||
+	    backfill_read_marker) {
+	  return false;
+	}
+      }
+      switch (state) {
+      case RWNONE:
+	assert(count == 0);
+	state = RWWRITE;
+	// fall through
+      case RWWRITE:
+	count++;
+	return true;
+      case RWREAD:
+	return false;
+      default:
+	assert(0 == "unhandled case");
+	return false;
+      }
+    }
+    /// same as get_write_lock, but ignore starvation
+    bool take_write_lock() {
+      if (state == RWWRITE) {
+	count++;
+	return true;
+      }
+      return get_write_lock();
+    }
+    void dec(list *requeue) {
+      assert(count > 0);
+      assert(requeue);
+      count--;
+      if (count == 0) {
+	state = RWNONE;
+	requeue->splice(requeue->end(), waiters);
+      }
+    }
+    void put_read(list *requeue) {
+      assert(state == RWREAD);
+      dec(requeue);
+    }
+    void put_write(list *requeue) {
+      assert(state == RWWRITE);
+      dec(requeue);
+    }
+    bool empty() const { return state == RWNONE; }
+  } rwstate;
+
+  bool get_read(OpRequestRef op) {
+    return rwstate.get_read(op);
+  }
+  bool get_write(OpRequestRef op) {
+    return rwstate.get_write(op, false);
+  }
+  bool get_write_greedy(OpRequestRef op) {
+    return rwstate.get_write(op, true);
+  }
+  bool get_snaptrimmer_write() {
+    if (rwstate.get_write_lock()) {
+      return true;
+    } else {
+      rwstate.snaptrimmer_write_marker = true;
+      return false;
+    }
+  }
+  bool get_backfill_read() {
+    rwstate.backfill_read_marker = true;
+    if (rwstate.get_read_lock()) {
+      return true;
+    }
+    return false;
+  }
+  void drop_backfill_read(list *ls) {
+    assert(rwstate.backfill_read_marker);
+    rwstate.put_read(ls);
+    rwstate.backfill_read_marker = false;
+  }
+  void put_read(list *to_wake) {
+    rwstate.put_read(to_wake);
+  }
+  void put_write(list *to_wake,
+		 bool *requeue_recovery,
+		 bool *requeue_snaptrimmer) {
+    rwstate.put_write(to_wake);
+    if (rwstate.empty() && rwstate.backfill_read_marker) {
+      rwstate.backfill_read_marker = false;
+      *requeue_recovery = true;
+    }
+    if (rwstate.empty() && rwstate.snaptrimmer_write_marker) {
+      rwstate.snaptrimmer_write_marker = false;
+      *requeue_snaptrimmer = true;
+    }
+  }
+
+  ObjectContext()
+    : ssc(NULL),
+      destructor_callback(0),
+      lock("ReplicatedPG::ObjectContext::lock"),
+      unstable_writes(0), readers(0), writers_waiting(0), readers_waiting(0),
+      blocked(false), requeue_scrub_on_unblock(false) {}
+
+  ~ObjectContext() {
+    assert(rwstate.empty());
+    if (destructor_callback)
+      destructor_callback->complete(0);
+  }
+
+  void start_block() {
+    assert(!blocked);
+    blocked = true;
+  }
+  void stop_block() {
+    assert(blocked);
+    blocked = false;
+  }
+  bool is_blocked() const {
+    return blocked;
+  }
+
+  // do simple synchronous mutual exclusion, for now.  now waitqueues or anything fancy.
+  void ondisk_write_lock() {
+    lock.Lock();
+    writers_waiting++;
+    while (readers_waiting || readers)
+      cond.Wait(lock);
+    writers_waiting--;
+    unstable_writes++;
+    lock.Unlock();
+  }
+  void ondisk_write_unlock() {
+    lock.Lock();
+    assert(unstable_writes > 0);
+    unstable_writes--;
+    if (!unstable_writes && readers_waiting)
+      cond.Signal();
+    lock.Unlock();
+  }
+  void ondisk_read_lock() {
+    lock.Lock();
+    readers_waiting++;
+    while (unstable_writes)
+      cond.Wait(lock);
+    readers_waiting--;
+    readers++;
+    lock.Unlock();
+  }
+  void ondisk_read_unlock() {
+    lock.Lock();
+    assert(readers > 0);
+    readers--;
+    if (!readers && writers_waiting)
+      cond.Signal();
+    lock.Unlock();
+  }
+
+  // attr cache
+  map attr_cache;
+
+  void fill_in_setattrs(const set &changing, ObjectModDesc *mod) {
+    map > to_set;
+    for (set::const_iterator i = changing.begin();
+	 i != changing.end();
+	 ++i) {
+      map::iterator iter = attr_cache.find(*i);
+      if (iter != attr_cache.end()) {
+	to_set[*i] = iter->second;
+      } else {
+	to_set[*i];
+      }
+    }
+    mod->setattrs(to_set);
+  }
+};
+
+inline ostream& operator<<(ostream& out, const ObjectState& obs)
+{
+  out << obs.oi.soid;
+  if (!obs.exists)
+    out << "(dne)";
+  return out;
+}
+
+inline ostream& operator<<(ostream& out, const ObjectContext::RWState& rw)
+{
+  return out << "rwstate(" << rw.get_state_name()
+	     << " n=" << rw.count
+	     << " w=" << rw.waiters.size()
+	     << ")";
+}
+
+inline ostream& operator<<(ostream& out, const ObjectContext& obc)
+{
+  return out << "obc(" << obc.obs << " " << obc.rwstate << ")";
+}
+
+
+ostream& operator<<(ostream& out, const object_info_t& oi);
+
+
+
+// Object recovery
+struct ObjectRecoveryInfo {
+  hobject_t soid;
+  eversion_t version;
+  uint64_t size;
+  object_info_t oi;
+  SnapSet ss;
+  interval_set copy_subset;
+  map > clone_subset;
+
+  ObjectRecoveryInfo() : size(0) { }
+
+  static void generate_test_instances(list& o);
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl, int64_t pool = -1);
+  ostream &print(ostream &out) const;
+  void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(ObjectRecoveryInfo)
+ostream& operator<<(ostream& out, const ObjectRecoveryInfo &inf);
+
+struct ObjectRecoveryProgress {
+  bool first;
+  uint64_t data_recovered_to;
+  bool data_complete;
+  string omap_recovered_to;
+  bool omap_complete;
+
+  ObjectRecoveryProgress()
+    : first(true),
+      data_recovered_to(0),
+      data_complete(false), omap_complete(false) { }
+
+  bool is_complete(const ObjectRecoveryInfo& info) const {
+    return (data_recovered_to >= (
+      info.copy_subset.empty() ?
+      0 : info.copy_subset.range_end())) &&
+      omap_complete;
+  }
+
+  static void generate_test_instances(list& o);
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  ostream &print(ostream &out) const;
+  void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(ObjectRecoveryProgress)
+ostream& operator<<(ostream& out, const ObjectRecoveryProgress &prog);
+
+struct PushReplyOp {
+  hobject_t soid;
+
+  static void generate_test_instances(list& o);
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  ostream &print(ostream &out) const;
+  void dump(Formatter *f) const;
+
+  uint64_t cost(CephContext *cct) const;
+};
+WRITE_CLASS_ENCODER(PushReplyOp)
+ostream& operator<<(ostream& out, const PushReplyOp &op);
+
+struct PullOp {
+  hobject_t soid;
+
+  ObjectRecoveryInfo recovery_info;
+  ObjectRecoveryProgress recovery_progress;
+
+  static void generate_test_instances(list& o);
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  ostream &print(ostream &out) const;
+  void dump(Formatter *f) const;
+
+  uint64_t cost(CephContext *cct) const;
+};
+WRITE_CLASS_ENCODER(PullOp)
+ostream& operator<<(ostream& out, const PullOp &op);
+
+struct PushOp {
+  hobject_t soid;
+  eversion_t version;
+  bufferlist data;
+  interval_set data_included;
+  bufferlist omap_header;
+  map omap_entries;
+  map attrset;
+
+  ObjectRecoveryInfo recovery_info;
+  ObjectRecoveryProgress before_progress;
+  ObjectRecoveryProgress after_progress;
+
+  static void generate_test_instances(list& o);
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bl);
+  ostream &print(ostream &out) const;
+  void dump(Formatter *f) const;
+
+  uint64_t cost(CephContext *cct) const;
+};
+WRITE_CLASS_ENCODER(PushOp)
+ostream& operator<<(ostream& out, const PushOp &op);
+
+
+/*
+ * summarize pg contents for purposes of a scrub
+ */
+struct ScrubMap {
+  struct object {
+    uint64_t size;
+    bool negative;
+    map attrs;
+    __u32 digest;
+    bool digest_present;
+    uint32_t nlinks;
+    set snapcolls;
+    __u32 omap_digest;
+    bool omap_digest_present;
+    bool read_error;
+
+    object() :
+      // Init invalid size so it won't match if we get a stat EIO error
+      size(-1), negative(false), digest(0), digest_present(false),
+      nlinks(0), omap_digest(0), omap_digest_present(false),
+      read_error(false) {}
+
+    void encode(bufferlist& bl) const;
+    void decode(bufferlist::iterator& bl);
+    void dump(Formatter *f) const;
+    static void generate_test_instances(list& o);
+  };
+  WRITE_CLASS_ENCODER(object)
+
+  map objects;
+  map attrs;
+  eversion_t valid_through;
+  eversion_t incr_since;
+
+  void merge_incr(const ScrubMap &l);
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl, int64_t pool=-1);
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ScrubMap::object)
+WRITE_CLASS_ENCODER(ScrubMap)
+
+
+struct OSDOp {
+  ceph_osd_op op;
+  sobject_t soid;
+
+  bufferlist indata, outdata;
+  int32_t rval;
+
+  OSDOp() : rval(0) {
+    memset(&op, 0, sizeof(ceph_osd_op));
+  }
+
+  /**
+   * split a bufferlist into constituent indata nembers of a vector of OSDOps
+   *
+   * @param ops [out] vector of OSDOps
+   * @param in  [in] combined data buffer
+   */
+  static void split_osd_op_vector_in_data(vector& ops, bufferlist& in);
+
+  /**
+   * merge indata nembers of a vector of OSDOp into a single bufferlist
+   *
+   * Notably this also encodes certain other OSDOp data into the data
+   * buffer, including the sobject_t soid.
+   *
+   * @param ops [in] vector of OSDOps
+   * @param in  [out] combined data buffer
+   */
+  static void merge_osd_op_vector_in_data(vector& ops, bufferlist& out);
+
+  /**
+   * split a bufferlist into constituent outdata members of a vector of OSDOps
+   *
+   * @param ops [out] vector of OSDOps
+   * @param in  [in] combined data buffer
+   */
+  static void split_osd_op_vector_out_data(vector& ops, bufferlist& in);
+
+  /**
+   * merge outdata members of a vector of OSDOps into a single bufferlist
+   *
+   * @param ops [in] vector of OSDOps
+   * @param in  [out] combined data buffer
+   */
+  static void merge_osd_op_vector_out_data(vector& ops, bufferlist& out);
+};
+
+ostream& operator<<(ostream& out, const OSDOp& op);
+
+struct watch_item_t {
+  entity_name_t name;
+  uint64_t cookie;
+  uint32_t timeout_seconds;
+  entity_addr_t addr;
+
+  watch_item_t() : cookie(0), timeout_seconds(0) { }
+  watch_item_t(entity_name_t name, uint64_t cookie, uint32_t timeout,
+     const entity_addr_t& addr)
+    : name(name), cookie(cookie), timeout_seconds(timeout),
+    addr(addr) { }
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(2, 1, bl);
+    ::encode(name, bl);
+    ::encode(cookie, bl);
+    ::encode(timeout_seconds, bl);
+    ::encode(addr, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &bl) {
+    DECODE_START(2, bl);
+    ::decode(name, bl);
+    ::decode(cookie, bl);
+    ::decode(timeout_seconds, bl);
+    if (struct_v >= 2) {
+      ::decode(addr, bl);
+    }
+    DECODE_FINISH(bl);
+  }
+};
+WRITE_CLASS_ENCODER(watch_item_t)
+
+struct obj_watch_item_t {
+  hobject_t obj;
+  watch_item_t wi;
+};
+
+/**
+ * obj list watch response format
+ *
+ */
+struct obj_list_watch_response_t {
+  list entries;
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(entries, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(entries, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const {
+    f->open_array_section("entries");
+    for (list::const_iterator p = entries.begin(); p != entries.end(); ++p) {
+      f->open_object_section("watch");
+      f->dump_stream("watcher") << p->name;
+      f->dump_int("cookie", p->cookie);
+      f->dump_int("timeout", p->timeout_seconds);
+      f->open_object_section("addr");
+      p->addr.dump(f);
+      f->close_section();
+      f->close_section();
+    }
+    f->close_section();
+  }
+  static void generate_test_instances(list& o) {
+    entity_addr_t ea;
+    o.push_back(new obj_list_watch_response_t);
+    o.push_back(new obj_list_watch_response_t);
+    ea.set_nonce(1000);
+    ea.set_family(AF_INET);
+    ea.set_in4_quad(0, 127);
+    ea.set_in4_quad(1, 0);
+    ea.set_in4_quad(2, 0);
+    ea.set_in4_quad(3, 1);
+    ea.set_port(1024);
+    o.back()->entries.push_back(watch_item_t(entity_name_t(entity_name_t::TYPE_CLIENT, 1), 10, 30, ea));
+    ea.set_nonce(1001);
+    ea.set_in4_quad(3, 2);
+    ea.set_port(1025);
+    o.back()->entries.push_back(watch_item_t(entity_name_t(entity_name_t::TYPE_CLIENT, 2), 20, 60, ea));
+  }
+};
+
+WRITE_CLASS_ENCODER(obj_list_watch_response_t)
+
+struct clone_info {
+  snapid_t cloneid;
+  vector snaps;  // ascending
+  vector< pair > overlap;
+  uint64_t size;
+
+  clone_info() : cloneid(CEPH_NOSNAP), size(0) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(cloneid, bl);
+    ::encode(snaps, bl);
+    ::encode(overlap, bl);
+    ::encode(size, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(cloneid, bl);
+    ::decode(snaps, bl);
+    ::decode(overlap, bl);
+    ::decode(size, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const {
+    if (cloneid == CEPH_NOSNAP)
+      f->dump_string("cloneid", "HEAD");
+    else
+      f->dump_unsigned("cloneid", cloneid.val);
+    f->open_array_section("snapshots");
+    for (vector::const_iterator p = snaps.begin(); p != snaps.end(); ++p) {
+      f->open_object_section("snap");
+      f->dump_unsigned("id", p->val);
+      f->close_section();
+    }
+    f->close_section();
+    f->open_array_section("overlaps");
+    for (vector< pair >::const_iterator q = overlap.begin();
+          q != overlap.end(); ++q) {
+      f->open_object_section("overlap");
+      f->dump_unsigned("offset", q->first);
+      f->dump_unsigned("length", q->second);
+      f->close_section();
+    }
+    f->close_section();
+    f->dump_unsigned("size", size);
+  }
+  static void generate_test_instances(list& o) {
+    o.push_back(new clone_info);
+    o.push_back(new clone_info);
+    o.back()->cloneid = 1;
+    o.back()->snaps.push_back(1);
+    o.back()->overlap.push_back(pair(0,4096));
+    o.back()->overlap.push_back(pair(8192,4096));
+    o.back()->size = 16384;
+    o.push_back(new clone_info);
+    o.back()->cloneid = CEPH_NOSNAP;
+    o.back()->size = 32768;
+  }
+};
+WRITE_CLASS_ENCODER(clone_info)
+
+/**
+ * obj list snaps response format
+ *
+ */
+struct obj_list_snap_response_t {
+  vector clones;   // ascending
+  snapid_t seq;
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 1, bl);
+    ::encode(clones, bl);
+    ::encode(seq, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(2, bl);
+    ::decode(clones, bl);
+    if (struct_v >= 2)
+      ::decode(seq, bl);
+    else
+      seq = CEPH_NOSNAP;
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const {
+    f->open_array_section("clones");
+    for (vector::const_iterator p = clones.begin(); p != clones.end(); ++p) {
+      f->open_object_section("clone");
+      p->dump(f);
+      f->close_section();
+    }
+    f->dump_unsigned("seq", seq);
+    f->close_section();
+  }
+  static void generate_test_instances(list& o) {
+    o.push_back(new obj_list_snap_response_t);
+    o.push_back(new obj_list_snap_response_t);
+    clone_info cl;
+    cl.cloneid = 1;
+    cl.snaps.push_back(1);
+    cl.overlap.push_back(pair(0,4096));
+    cl.overlap.push_back(pair(8192,4096));
+    cl.size = 16384;
+    o.back()->clones.push_back(cl);
+    cl.cloneid = CEPH_NOSNAP;
+    cl.snaps.clear();
+    cl.overlap.clear();
+    cl.size = 32768;
+    o.back()->clones.push_back(cl);
+    o.back()->seq = 123;
+  }
+};
+
+WRITE_CLASS_ENCODER(obj_list_snap_response_t)
+
+enum scrub_error_type {
+  CLEAN,
+  DEEP_ERROR,
+  SHALLOW_ERROR
+};
+#endif
diff --git a/ceph/src/osdc/Blinker.h b/ceph/src/osdc/Blinker.h
new file mode 100644
index 00000000..48526c69
--- /dev/null
+++ b/ceph/src/osdc/Blinker.h
@@ -0,0 +1,92 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_BLINKER_H
+#define CEPH_BLINKER_H
+
+class Blinker {
+
+ public:
+
+  class Op {
+    int op;
+    static const int LOOKUP = 1;
+    static const int INSERT = 2;
+    static const int REMOVE = 3;
+    static const int CLEAR = 4;
+    Op(int o) : op(o) {}
+  };
+  
+  class OpLookup : public Op {
+  public:
+    bufferptr key;
+    OpLookup(bufferptr& k) : Op(Op::LOOKUP), key(k) {}
+  };
+
+  class OpInsert : public Op {
+    bufferptr key;
+    bufferlist val;
+    OpInsert(bufferptr& k, bufferlist& v) : Op(Op::INSERT), key(k), val(v) {}
+  };
+
+  class OpRemove : public Op {
+  public:
+    bufferptr key;
+    OpRemove(bufferptr& k) : Op(Op::REMOVE), key(k) {}
+  };
+
+  class OpClear : public Op {
+  public:
+    OpClear() : Op(Op::CLEAR) {}
+  };
+
+
+
+private:
+  Objecter *objecter;
+
+  // in-flight operations.
+
+
+  // cache information about tree structure.
+  
+
+
+public:
+  // public interface
+
+  // simple accessors
+  void lookup(inode_t& inode, bufferptr& key, bufferlist *pval, Context *onfinish);
+
+  // simple modifiers
+  void insert(inode_t& inode, bufferptr& key, bufferlist& val, Context *onack, Context *onsafe);
+  void remove(inode_t& inode, bufferptr& key, Context *onack, Context *onsafe);
+  void clear(inode_t& inode, Context *onack, Context *onsafe);
+
+  // these are dangerous: the table may be large.
+  void listkeys(inode_t& inode, list* pkeys, Context *onfinish);
+  void listvals(inode_t& inode, list* pkeys, list* pvals, Context *onfinish);
+
+  // fetch *at least* key, but also anything else that is convenient. 
+  // include lexical bounds for which this is a complete result.
+  //  (if *start and *end are empty, it's the entire table)
+  void prefetch(inode_t& inode, bufferptr& key, 
+		list* pkeys, list* pvals, 
+		bufferptr *start, bufferptr *end,
+		Context *onfinish);
+
+  
+};
+
+#endif
diff --git a/ceph/src/osdc/Filer.cc b/ceph/src/osdc/Filer.cc
new file mode 100644
index 00000000..8f94a97d
--- /dev/null
+++ b/ceph/src/osdc/Filer.cc
@@ -0,0 +1,304 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include "Filer.h"
+#include "osd/OSDMap.h"
+#include "Striper.h"
+
+#include "messages/MOSDOp.h"
+#include "messages/MOSDOpReply.h"
+#include "messages/MOSDMap.h"
+
+#include "msg/Messenger.h"
+
+#include "include/Context.h"
+
+#include "common/config.h"
+
+#define dout_subsys ceph_subsys_filer
+#undef dout_prefix
+#define dout_prefix *_dout << objecter->messenger->get_myname() << ".filer "
+
+class Filer::C_Probe : public Context {
+public:
+  Filer *filer;
+  Probe *probe;
+  object_t oid;
+  uint64_t size;
+  utime_t mtime;
+  C_Probe(Filer *f, Probe *p, object_t o) : filer(f), probe(p), oid(o), size(0) {}
+  void finish(int r) {
+    if (r == -ENOENT) {
+      r = 0;
+      assert(size == 0);
+    }
+
+    // TODO: handle this error.
+    if (r != 0)
+      probe->err = r;
+
+    filer->_probed(probe, oid, size, mtime);
+  }  
+};
+
+int Filer::probe(inodeno_t ino,
+		 ceph_file_layout *layout,
+		 snapid_t snapid,
+		 uint64_t start_from,
+		 uint64_t *end,           // LB, when !fwd
+		 utime_t *pmtime,
+		 bool fwd,
+		 int flags,
+		 Context *onfinish) 
+{
+  ldout(cct, 10) << "probe " << (fwd ? "fwd ":"bwd ")
+	   << hex << ino << dec
+	   << " starting from " << start_from
+	   << dendl;
+
+  assert(snapid);  // (until there is a non-NOSNAP write)
+
+  Probe *probe = new Probe(ino, *layout, snapid, start_from, end, pmtime, flags, fwd, onfinish);
+  
+  // period (bytes before we jump unto a new set of object(s))
+  uint64_t period = (uint64_t)layout->fl_stripe_count * (uint64_t)layout->fl_object_size;
+  
+  // start with 1+ periods.
+  probe->probing_len = period;
+  if (probe->fwd) {
+    if (start_from % period)
+      probe->probing_len += period - (start_from % period);
+  } else {
+    assert(start_from > *end);
+    if (start_from % period)
+      probe->probing_len -= period - (start_from % period);
+    probe->probing_off -= probe->probing_len;
+  }
+  
+  _probe(probe);
+  return 0;
+}
+
+
+void Filer::_probe(Probe *probe)
+{
+  ldout(cct, 10) << "_probe " << hex << probe->ino << dec 
+	   << " " << probe->probing_off << "~" << probe->probing_len 
+	   << dendl;
+  
+  // map range onto objects
+  probe->known_size.clear();
+  probe->probing.clear();
+  Striper::file_to_extents(cct, probe->ino, &probe->layout,
+			   probe->probing_off, probe->probing_len, 0, probe->probing);
+  
+  for (vector::iterator p = probe->probing.begin();
+       p != probe->probing.end();
+       ++p) {
+    ldout(cct, 10) << "_probe  probing " << p->oid << dendl;
+    C_Probe *c = new C_Probe(this, probe, p->oid);
+    objecter->stat(p->oid, p->oloc, probe->snapid, &c->size, &c->mtime, 
+		   probe->flags | CEPH_OSD_FLAG_RWORDERED, c);
+    probe->ops.insert(p->oid);
+  }
+}
+
+void Filer::_probed(Probe *probe, const object_t& oid, uint64_t size, utime_t mtime)
+{
+  ldout(cct, 10) << "_probed " << probe->ino << " object " << oid
+	   << " has size " << size << " mtime " << mtime << dendl;
+
+  probe->known_size[oid] = size;
+  if (mtime > probe->max_mtime)
+    probe->max_mtime = mtime;
+
+  assert(probe->ops.count(oid));
+  probe->ops.erase(oid);
+
+  if (!probe->ops.empty()) 
+    return;  // waiting for more!
+
+  if (probe->err) { // we hit an error, propagate back up
+    probe->onfinish->complete(probe->err);
+    delete probe;
+    return;
+  }
+
+  // analyze!
+  uint64_t end = 0;
+
+  if (!probe->fwd) {
+    // reverse
+    vector r;
+    for (vector::reverse_iterator p = probe->probing.rbegin();
+	 p != probe->probing.rend();
+	 ++p)
+      r.push_back(*p);
+    probe->probing.swap(r);
+  }
+
+  for (vector::iterator p = probe->probing.begin();
+       p != probe->probing.end();
+       ++p) {
+    uint64_t shouldbe = p->length + p->offset;
+    ldout(cct, 10) << "_probed  " << probe->ino << " object " << hex << p->oid << dec
+	     << " should be " << shouldbe
+	     << ", actual is " << probe->known_size[p->oid]
+	     << dendl;
+
+    if (!probe->found_size) {
+      assert(probe->known_size[p->oid] <= shouldbe);
+
+      if ((probe->fwd && probe->known_size[p->oid] == shouldbe) ||
+	  (!probe->fwd && probe->known_size[p->oid] == 0 && probe->probing_off > 0))
+	continue;  // keep going
+      
+      // aha, we found the end!
+      // calc offset into buffer_extent to get distance from probe->from.
+      uint64_t oleft = probe->known_size[p->oid] - p->offset;
+      for (vector >::iterator i = p->buffer_extents.begin();
+	   i != p->buffer_extents.end();
+	   ++i) {
+	if (oleft <= (uint64_t)i->second) {
+	  end = probe->probing_off + i->first + oleft;
+	  ldout(cct, 10) << "_probed  end is in buffer_extent " << i->first << "~" << i->second << " off " << oleft 
+		   << ", from was " << probe->probing_off << ", end is " << end 
+		   << dendl;
+	  
+	  probe->found_size = true;
+	  ldout(cct, 10) << "_probed found size at " << end << dendl;
+	  *probe->psize = end;
+	  
+	  if (!probe->pmtime)  // stop if we don't need mtime too
+	    break;
+	}
+	oleft -= i->second;
+      }
+    }
+    break;
+  }
+
+  if (!probe->found_size || (probe->probing_off && probe->pmtime)) {
+    // keep probing!
+    ldout(cct, 10) << "_probed probing further" << dendl;
+
+    uint64_t period = (uint64_t)probe->layout.fl_stripe_count * (uint64_t)probe->layout.fl_object_size;
+    if (probe->fwd) {
+      probe->probing_off += probe->probing_len;
+      assert(probe->probing_off % period == 0);
+      probe->probing_len = period;
+    } else {
+      // previous period.
+      assert(probe->probing_off % period == 0);
+      probe->probing_len = period;
+      probe->probing_off -= period;
+    }
+    _probe(probe);
+    return;
+  }
+
+  if (probe->pmtime) {
+    ldout(cct, 10) << "_probed found mtime " << probe->max_mtime << dendl;
+    *probe->pmtime = probe->max_mtime;
+  }
+
+  // done!  finish and clean up.
+  probe->onfinish->complete(probe->err);
+  delete probe;
+}
+
+
+// -----------------------
+
+struct PurgeRange {
+  inodeno_t ino;
+  ceph_file_layout layout;
+  SnapContext snapc;
+  uint64_t first, num;
+  utime_t mtime;
+  int flags;
+  Context *oncommit;
+  int uncommitted;
+};
+
+int Filer::purge_range(inodeno_t ino,
+		       ceph_file_layout *layout,
+		       const SnapContext& snapc,
+		       uint64_t first_obj, uint64_t num_obj,
+		       utime_t mtime,
+		       int flags,
+		       Context *oncommit) 
+{
+  assert(num_obj > 0);
+
+  // single object?  easy!
+  if (num_obj == 1) {
+    object_t oid = file_object_t(ino, first_obj);
+    object_locator_t oloc = objecter->osdmap->file_to_object_locator(*layout);
+    objecter->remove(oid, oloc, snapc, mtime, flags, NULL, oncommit);
+    return 0;
+  }
+
+  // lots!  let's do this in pieces.
+  PurgeRange *pr = new PurgeRange;
+  pr->ino = ino;
+  pr->layout = *layout;
+  pr->snapc = snapc;
+  pr->first = first_obj;
+  pr->num = num_obj;
+  pr->mtime = mtime;
+  pr->flags = flags;
+  pr->oncommit = oncommit;
+  pr->uncommitted = 0;
+
+  _do_purge_range(pr, 0);
+  return 0;
+}
+
+struct C_PurgeRange : public Context {
+  Filer *filer;
+  PurgeRange *pr;
+  C_PurgeRange(Filer *f, PurgeRange *p) : filer(f), pr(p) {}
+  void finish(int r) {
+    filer->_do_purge_range(pr, 1);
+  }
+};
+
+void Filer::_do_purge_range(PurgeRange *pr, int fin)
+{
+  pr->uncommitted -= fin;
+  ldout(cct, 10) << "_do_purge_range " << pr->ino << " objects " << pr->first << "~" << pr->num
+	   << " uncommitted " << pr->uncommitted << dendl;
+
+  if (pr->num == 0 && pr->uncommitted == 0) {
+    pr->oncommit->complete(0);
+    delete pr;
+    return;
+  }
+
+  int max = 10 - pr->uncommitted;
+  while (pr->num > 0 && max > 0) {
+    object_t oid = file_object_t(pr->ino, pr->first);
+    object_locator_t oloc = objecter->osdmap->file_to_object_locator(pr->layout);
+    objecter->remove(oid, oloc, pr->snapc, pr->mtime, pr->flags,
+		     NULL, new C_PurgeRange(this, pr));
+    pr->uncommitted++;
+    pr->first++;
+    pr->num--;
+    max--;
+  }
+}
+
+
diff --git a/ceph/src/osdc/Filer.h b/ceph/src/osdc/Filer.h
new file mode 100644
index 00000000..607dc7b3
--- /dev/null
+++ b/ceph/src/osdc/Filer.h
@@ -0,0 +1,290 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_FILER_H
+#define CEPH_FILER_H
+
+/*** Filer
+ *
+ * stripe file ranges onto objects.
+ * build list for the objecter or objectcacher.
+ *
+ * also, provide convenience methods that call objecter for you.
+ *
+ * "files" are identified by ino. 
+ */
+
+#include "include/types.h"
+
+#include "osd/OSDMap.h"
+#include "Objecter.h"
+#include "Striper.h"
+
+class Context;
+class Messenger;
+class OSDMap;
+
+
+
+/**** Filer interface ***/
+
+class Filer {
+  CephContext *cct;
+  Objecter   *objecter;
+  
+  // probes
+  struct Probe {
+    inodeno_t ino;
+    ceph_file_layout layout;
+    snapid_t snapid;
+
+    uint64_t *psize;
+    utime_t *pmtime;
+
+    int flags;
+
+    bool fwd;
+
+    Context *onfinish;
+    
+    vector probing;
+    uint64_t probing_off, probing_len;
+    
+    map known_size;
+    utime_t max_mtime;
+
+    set ops;
+
+    int err;
+    bool found_size;
+
+    Probe(inodeno_t i, ceph_file_layout &l, snapid_t sn,
+	  uint64_t f, uint64_t *e, utime_t *m, int fl, bool fw, Context *c) : 
+      ino(i), layout(l), snapid(sn),
+      psize(e), pmtime(m), flags(fl), fwd(fw), onfinish(c),
+      probing_off(f), probing_len(0),
+      err(0), found_size(false) {}
+  };
+  
+  class C_Probe;
+
+  void _probe(Probe *p);
+  void _probed(Probe *p, const object_t& oid, uint64_t size, utime_t mtime);
+
+ public:
+  Filer(const Filer& other);
+  const Filer operator=(const Filer& other);
+
+  Filer(Objecter *o) : cct(o->cct), objecter(o) {}
+  ~Filer() {}
+
+  bool is_active() {
+    return objecter->is_active(); // || (oc && oc->is_active());
+  }
+
+
+  /*** async file interface.  scatter/gather as needed. ***/
+
+  int read(inodeno_t ino,
+	   ceph_file_layout *layout,
+	   snapid_t snap,
+           uint64_t offset, 
+           uint64_t len, 
+           bufferlist *bl,   // ptr to data
+	   int flags,
+           Context *onfinish) {
+    assert(snap);  // (until there is a non-NOSNAP write)
+    vector extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
+    objecter->sg_read(extents, snap, bl, flags, onfinish);
+    return 0;
+  }
+
+  int read_trunc(inodeno_t ino,
+	   ceph_file_layout *layout,
+	   snapid_t snap,
+           uint64_t offset, 
+           uint64_t len, 
+           bufferlist *bl,   // ptr to data
+	   int flags,
+	   uint64_t truncate_size,
+	   __u32 truncate_seq,
+           Context *onfinish) {
+    assert(snap);  // (until there is a non-NOSNAP write)
+    vector extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, truncate_size, extents);
+    objecter->sg_read_trunc(extents, snap, bl, flags,
+			    truncate_size, truncate_seq, onfinish);
+    return 0;
+  }
+
+  int write(inodeno_t ino,
+	    ceph_file_layout *layout,
+	    const SnapContext& snapc,
+	    uint64_t offset, 
+            uint64_t len, 
+            bufferlist& bl,
+	    utime_t mtime,
+            int flags, 
+            Context *onack,
+            Context *oncommit) {
+    vector extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
+    objecter->sg_write(extents, snapc, bl, mtime, flags, onack, oncommit);
+    return 0;
+  }
+
+  int write_trunc(inodeno_t ino,
+	    ceph_file_layout *layout,
+	    const SnapContext& snapc,
+	    uint64_t offset, 
+            uint64_t len, 
+            bufferlist& bl,
+	    utime_t mtime,
+            int flags, 
+	   uint64_t truncate_size,
+	   __u32 truncate_seq,
+            Context *onack,
+            Context *oncommit) {
+    vector extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, truncate_size, extents);
+    objecter->sg_write_trunc(extents, snapc, bl, mtime, flags,
+		       truncate_size, truncate_seq, onack, oncommit);
+    return 0;
+  }
+
+  int truncate(inodeno_t ino,
+	       ceph_file_layout *layout,
+	       const SnapContext& snapc,
+	       uint64_t offset,
+	       uint64_t len,
+	       __u32 truncate_seq,
+	       utime_t mtime,
+	       int flags,
+	       Context *onack,
+	       Context *oncommit) {
+    vector extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
+    if (extents.size() == 1) {
+      vector ops(1);
+      ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+      ops[0].op.extent.truncate_seq = truncate_seq;
+      ops[0].op.extent.truncate_size = extents[0].offset;
+      objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc, flags, onack, oncommit);
+    } else {
+      C_GatherBuilder gack(cct, onack);
+      C_GatherBuilder gcom(cct, oncommit);
+      for (vector::iterator p = extents.begin(); p != extents.end(); ++p) {
+	vector ops(1);
+	ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+	ops[0].op.extent.truncate_size = p->offset;
+	ops[0].op.extent.truncate_seq = truncate_seq;
+	objecter->_modify(p->oid, p->oloc, ops, mtime, snapc, flags,
+			  onack ? gack.new_sub():0,
+			  oncommit ? gcom.new_sub():0);
+      }
+      gack.activate();
+      gcom.activate();
+    }
+    return 0;
+  }
+
+  int zero(inodeno_t ino,
+	   ceph_file_layout *layout,
+	   const SnapContext& snapc,
+	   uint64_t offset,
+           uint64_t len,
+	   utime_t mtime,
+	   int flags,
+	   bool keep_first,
+           Context *onack,
+           Context *oncommit) {
+    vector extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
+    if (extents.size() == 1) {
+      if (extents[0].offset == 0 && extents[0].length == layout->fl_object_size &&
+	  (!keep_first || extents[0].objectno != 0))
+	objecter->remove(extents[0].oid, extents[0].oloc, 
+			 snapc, mtime, flags, onack, oncommit);
+      else
+	objecter->zero(extents[0].oid, extents[0].oloc, extents[0].offset, extents[0].length, 
+		       snapc, mtime, flags, onack, oncommit);
+    } else {
+      C_GatherBuilder gack(cct, onack);
+      C_GatherBuilder gcom(cct, oncommit);
+      for (vector::iterator p = extents.begin(); p != extents.end(); ++p) {
+	if (p->offset == 0 && p->length == layout->fl_object_size &&
+	    (!keep_first || p->objectno != 0))
+	  objecter->remove(p->oid, p->oloc,
+			   snapc, mtime, flags,
+			   onack ? gack.new_sub():0,
+			   oncommit ? gcom.new_sub():0);
+	else
+	  objecter->zero(p->oid, p->oloc, p->offset, p->length, 
+			 snapc, mtime, flags,
+			 onack ? gack.new_sub():0,
+			 oncommit ? gcom.new_sub():0);
+      }
+      gack.activate();
+      gcom.activate();
+    }
+    return 0;
+  }
+
+  int zero(inodeno_t ino,
+	   ceph_file_layout *layout,
+	   const SnapContext& snapc,
+	   uint64_t offset,
+           uint64_t len,
+	   utime_t mtime,
+	   int flags,
+           Context *onack,
+           Context *oncommit) {
+
+    return zero(ino, layout,
+                snapc, offset,
+                len, mtime,
+                flags, false,
+                onack, oncommit);
+  }
+  // purge range of ino.### objects
+  int purge_range(inodeno_t ino,
+		  ceph_file_layout *layout,
+		  const SnapContext& snapc,
+		  uint64_t first_obj, uint64_t num_obj,
+		  utime_t mtime,
+		  int flags,
+		  Context *oncommit);
+  void _do_purge_range(struct PurgeRange *pr, int fin);
+
+  /*
+   * probe 
+   *  specify direction,
+   *  and whether we stop when we find data, or hole.
+   */
+  int probe(inodeno_t ino,
+	    ceph_file_layout *layout,
+	    snapid_t snapid,
+	    uint64_t start_from,
+	    uint64_t *end,
+	    utime_t *mtime,
+	    bool fwd,
+	    int flags,
+	    Context *onfinish);
+};
+
+
+
+#endif
diff --git a/ceph/src/osdc/Journaler.cc b/ceph/src/osdc/Journaler.cc
new file mode 100644
index 00000000..ba4ca8dc
--- /dev/null
+++ b/ceph/src/osdc/Journaler.cc
@@ -0,0 +1,1064 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "common/perf_counters.h"
+#include "common/dout.h"
+#include "include/Context.h"
+#include "msg/Messenger.h"
+#include "osdc/Journaler.h"
+#include "common/errno.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_journaler
+#undef dout_prefix
+#define dout_prefix *_dout << objecter->messenger->get_myname() << ".journaler" << (readonly ? "(ro) ":"(rw) ")
+
+
+void Journaler::set_readonly()
+{
+  ldout(cct, 1) << "set_readonly" << dendl;
+  readonly = true;
+}
+
+void Journaler::set_writeable()
+{
+  ldout(cct, 1) << "set_writeable" << dendl;
+  readonly = false;
+}
+
+void Journaler::create(ceph_file_layout *l)
+{
+  assert(!readonly);
+  ldout(cct, 1) << "create blank journal" << dendl;
+  state = STATE_ACTIVE;
+
+  set_layout(l);
+
+  prezeroing_pos = prezero_pos = write_pos = flush_pos = safe_pos =
+    read_pos = requested_pos = received_pos =
+    expire_pos = trimming_pos = trimmed_pos = layout.fl_stripe_count * layout.fl_object_size;
+}
+
+void Journaler::set_layout(ceph_file_layout *l)
+{
+  layout = *l;
+
+  assert(layout.fl_pg_pool == pg_pool);
+  last_written.layout = layout;
+  last_committed.layout = layout;
+
+  // prefetch intelligently.
+  // (watch out, this is big if you use big objects or weird striping)
+  uint64_t periods = cct->_conf->journaler_prefetch_periods;
+  if (periods < 2)
+    periods = 2;  // we need at least 2 periods to make progress.
+  fetch_len = layout.fl_stripe_count * layout.fl_object_size * periods;
+}
+
+
+/***************** HEADER *******************/
+
+ostream& operator<<(ostream& out, Journaler::Header &h) 
+{
+  return out << "loghead(trim " << h.trimmed_pos
+	     << ", expire " << h.expire_pos
+	     << ", write " << h.write_pos
+	     << ")";
+}
+
+class Journaler::C_ReadHead : public Context {
+  Journaler *ls;
+public:
+  bufferlist bl;
+  C_ReadHead(Journaler *l) : ls(l) {}
+  void finish(int r) {
+    ls->_finish_read_head(r, bl);
+  }
+};
+
+class Journaler::C_RereadHead : public Context {
+  Journaler *ls;
+  Context *onfinish;
+public:
+  bufferlist bl;
+  C_RereadHead(Journaler *l, Context *onfinish_) : ls (l), onfinish(onfinish_){}
+  void finish(int r) {
+    ls->_finish_reread_head(r, bl, onfinish);
+  }
+};
+
+class Journaler::C_ProbeEnd : public Context {
+  Journaler *ls;
+public:
+  uint64_t end;
+  C_ProbeEnd(Journaler *l) : ls(l), end(-1) {}
+  void finish(int r) {
+    ls->_finish_probe_end(r, end);
+  }
+};
+
+class Journaler::C_ReProbe : public Context {
+  Journaler *ls;
+  Context *onfinish;
+public:
+  uint64_t end;
+  C_ReProbe(Journaler *l, Context *onfinish_) :
+    ls(l), onfinish(onfinish_), end(0) {}
+  void finish(int r) {
+    ls->_finish_reprobe(r, end, onfinish);
+  }
+};
+
+void Journaler::recover(Context *onread) 
+{
+  ldout(cct, 1) << "recover start" << dendl;
+  assert(state != STATE_ACTIVE);
+  assert(readonly);
+
+  if (onread)
+    waitfor_recover.push_back(onread);
+  
+  if (state != STATE_UNDEF) {
+    ldout(cct, 1) << "recover - already recoverying" << dendl;
+    return;
+  }
+
+  ldout(cct, 1) << "read_head" << dendl;
+  state = STATE_READHEAD;
+  C_ReadHead *fin = new C_ReadHead(this);
+  read_head(fin, &fin->bl);
+}
+
+void Journaler::read_head(Context *on_finish, bufferlist *bl)
+{
+  assert(state == STATE_READHEAD || state == STATE_REREADHEAD);
+
+  object_t oid = file_object_t(ino, 0);
+  object_locator_t oloc(pg_pool);
+  objecter->read_full(oid, oloc, CEPH_NOSNAP, bl, 0, on_finish);
+}
+
+/**
+ * Re-read the head from disk, and set the write_pos, expire_pos, trimmed_pos
+ * from the on-disk header. This switches the state to STATE_REREADHEAD for
+ * the duration, and you shouldn't start a re-read while other operations are
+ * in-flight, nor start other operations while a re-read is in progress.
+ * Also, don't call this until the Journaler has finished its recovery and has
+ * gone STATE_ACTIVE!
+ */
+void Journaler::reread_head(Context *onfinish)
+{
+  ldout(cct, 10) << "reread_head" << dendl;
+  assert(state == STATE_ACTIVE);
+
+  state = STATE_REREADHEAD;
+  C_RereadHead *fin = new C_RereadHead(this, onfinish);
+  read_head(fin, &fin->bl);
+}
+
+void Journaler::_finish_reread_head(int r, bufferlist& bl, Context *finish)
+{
+  //read on-disk header into
+  assert(bl.length() || r < 0 );
+
+  // unpack header
+  Header h;
+  bufferlist::iterator p = bl.begin();
+  ::decode(h, p);
+  prezeroing_pos = prezero_pos = write_pos = flush_pos = safe_pos = h.write_pos;
+  expire_pos = h.expire_pos;
+  trimmed_pos = trimming_pos = h.trimmed_pos;
+  init_headers(h);
+  state = STATE_ACTIVE;
+  finish->complete(r);
+}
+
+void Journaler::_finish_read_head(int r, bufferlist& bl)
+{
+  assert(state == STATE_READHEAD);
+
+  if (r!=0) {
+    ldout(cct, 0) << "error getting journal off disk"
+                  << dendl;
+    list ls;
+    ls.swap(waitfor_recover);
+    finish_contexts(cct, ls, r);
+    return;
+  }
+
+  if (bl.length() == 0) {
+    ldout(cct, 1) << "_finish_read_head r=" << r << " read 0 bytes, assuming empty log" << dendl;    
+    state = STATE_ACTIVE;
+    list ls;
+    ls.swap(waitfor_recover);
+    finish_contexts(cct, ls, 0);
+    return;
+  } 
+
+  // unpack header
+  Header h;
+  bufferlist::iterator p = bl.begin();
+  ::decode(h, p);
+
+  if (h.magic != magic) {
+    ldout(cct, 0) << "on disk magic '" << h.magic << "' != my magic '"
+	    << magic << "'" << dendl;
+    list ls;
+    ls.swap(waitfor_recover);
+    finish_contexts(cct, ls, -EINVAL);
+    return;
+  }
+
+  prezeroing_pos = prezero_pos = write_pos = flush_pos = safe_pos = h.write_pos;
+  read_pos = requested_pos = received_pos = expire_pos = h.expire_pos;
+  trimmed_pos = trimming_pos = h.trimmed_pos;
+
+  init_headers(h);
+  set_layout(&h.layout);
+
+  ldout(cct, 1) << "_finish_read_head " << h << ".  probing for end of log (from " << write_pos << ")..." << dendl;
+  C_ProbeEnd *fin = new C_ProbeEnd(this);
+  state = STATE_PROBING;
+  probe(fin, &fin->end);
+}
+
+void Journaler::probe(Context *finish, uint64_t *end)
+{
+  ldout(cct, 1) << "probing for end of the log" << dendl;
+  assert(state == STATE_PROBING || state == STATE_REPROBING);
+  // probe the log
+  filer.probe(ino, &layout, CEPH_NOSNAP,
+	      write_pos, end, 0, true, 0, finish);
+}
+
+void Journaler::reprobe(Context *finish)
+{
+  ldout(cct, 10) << "reprobe" << dendl;
+  assert(state == STATE_ACTIVE);
+
+  state = STATE_REPROBING;
+  C_ReProbe *fin = new C_ReProbe(this, finish);
+  probe(fin, &fin->end);
+}
+
+
+void Journaler::_finish_reprobe(int r, uint64_t new_end, Context *onfinish) {
+  assert(new_end >= write_pos || r < 0);
+  ldout(cct, 1) << "_finish_reprobe new_end = " << new_end 
+	  << " (header had " << write_pos << ")."
+	  << dendl;
+  prezeroing_pos = prezero_pos = write_pos = flush_pos = safe_pos = new_end;
+  state = STATE_ACTIVE;
+  onfinish->complete(r);
+}
+
+void Journaler::_finish_probe_end(int r, uint64_t end)
+{
+  assert(state == STATE_PROBING);
+  if (r < 0) { // error in probing
+    goto out;
+  }
+  if (((int64_t)end) == -1) {
+    end = write_pos;
+    ldout(cct, 1) << "_finish_probe_end write_pos = " << end 
+	    << " (header had " << write_pos << "). log was empty. recovered."
+	    << dendl;
+    assert(0); // hrm.
+  } else {
+    assert(end >= write_pos);
+    ldout(cct, 1) << "_finish_probe_end write_pos = " << end 
+	    << " (header had " << write_pos << "). recovered."
+	    << dendl;
+  }
+
+  state = STATE_ACTIVE;
+
+  prezeroing_pos = prezero_pos = write_pos = flush_pos = safe_pos = end;
+  
+out:
+  // done.
+  list ls;
+  ls.swap(waitfor_recover);
+  finish_contexts(cct, ls, r);
+}
+
+class Journaler::C_RereadHeadProbe : public Context
+{
+  Journaler *ls;
+  Context *final_finish;
+public:
+  C_RereadHeadProbe(Journaler *l, Context *finish) :
+    ls(l), final_finish(finish) {}
+  void finish(int r) {
+    ls->_finish_reread_head_and_probe(r, final_finish);
+  }
+};
+
+void Journaler::reread_head_and_probe(Context *onfinish)
+{
+  assert(state == STATE_ACTIVE);
+  reread_head(new C_RereadHeadProbe(this, onfinish));
+}
+
+void Journaler::_finish_reread_head_and_probe(int r, Context *onfinish)
+{
+  assert(!r); //if we get an error, we're boned
+  reprobe(onfinish);
+}
+
+
+// WRITING
+
+class Journaler::C_WriteHead : public Context {
+public:
+  Journaler *ls;
+  Header h;
+  Context *oncommit;
+  C_WriteHead(Journaler *l, Header& h_, Context *c) : ls(l), h(h_), oncommit(c) {}
+  void finish(int r) {
+    ls->_finish_write_head(r, h, oncommit);
+  }
+};
+
+void Journaler::write_head(Context *oncommit)
+{
+  assert(!readonly);
+  assert(state == STATE_ACTIVE);
+  last_written.trimmed_pos = trimmed_pos;
+  last_written.expire_pos = expire_pos;
+  last_written.unused_field = expire_pos;
+  last_written.write_pos = safe_pos;
+  ldout(cct, 10) << "write_head " << last_written << dendl;
+  
+  last_wrote_head = ceph_clock_now(cct);
+
+  bufferlist bl;
+  ::encode(last_written, bl);
+  SnapContext snapc;
+  
+  object_t oid = file_object_t(ino, 0);
+  object_locator_t oloc(pg_pool);
+  objecter->write_full(oid, oloc, snapc, bl, ceph_clock_now(cct), 0, 
+		       NULL, 
+		       new C_WriteHead(this, last_written, oncommit));
+}
+
+void Journaler::_finish_write_head(int r, Header &wrote, Context *oncommit)
+{
+  if (r < 0) {
+    lderr(cct) << "_finish_write_head got " << cpp_strerror(r) << dendl;
+    handle_write_error(r);
+    return;
+  }
+  assert(!readonly);
+  ldout(cct, 10) << "_finish_write_head " << wrote << dendl;
+  last_committed = wrote;
+  if (oncommit) {
+    oncommit->complete(r);
+  }
+
+  trim();  // trim?
+}
+
+
+/***************** WRITING *******************/
+
+class Journaler::C_Flush : public Context {
+  Journaler *ls;
+  uint64_t start;
+  utime_t stamp;
+public:
+  C_Flush(Journaler *l, int64_t s, utime_t st) : ls(l), start(s), stamp(st) {}
+  void finish(int r) {
+    ls->_finish_flush(r, start, stamp);
+  }
+};
+
+void Journaler::_finish_flush(int r, uint64_t start, utime_t stamp)
+{
+  assert(!readonly);
+  if (r < 0) {
+    lderr(cct) << "_finish_flush got " << cpp_strerror(r) << dendl;
+    handle_write_error(r);
+    return;
+  }
+
+  assert(start >= safe_pos);
+  assert(start < flush_pos);
+
+  // calc latency?
+  if (logger) {
+    utime_t lat = ceph_clock_now(cct);
+    lat -= stamp;
+    logger->tinc(logger_key_lat, lat);
+  }
+
+  // adjust safe_pos
+  assert(pending_safe.count(start));
+  pending_safe.erase(start);
+  if (pending_safe.empty())
+    safe_pos = flush_pos;
+  else
+    safe_pos = *pending_safe.begin();
+
+  ldout(cct, 10) << "_finish_flush safe from " << start
+		 << ", pending_safe " << pending_safe
+		 << ", (prezeroing/prezero)/write/flush/safe positions now "
+		 << "(" << prezeroing_pos << "/" << prezero_pos << ")/" << write_pos
+		 << "/" << flush_pos << "/" << safe_pos
+		 << dendl;
+
+  // kick waiters <= safe_pos
+  while (!waitfor_safe.empty()) {
+    if (waitfor_safe.begin()->first > safe_pos)
+      break;
+    finish_contexts(cct, waitfor_safe.begin()->second);
+    waitfor_safe.erase(waitfor_safe.begin());
+  }
+}
+
+
+uint64_t Journaler::append_entry(bufferlist& bl)
+{
+  assert(!readonly);
+  uint32_t s = bl.length();
+
+  if (!cct->_conf->journaler_allow_split_entries) {
+    // will we span a stripe boundary?
+    int p = layout.fl_stripe_unit;
+    if (write_pos / p != (write_pos + (int64_t)(bl.length() + sizeof(s))) / p) {
+      // yes.
+      // move write_pos forward.
+      int64_t owp = write_pos;
+      write_pos += p;
+      write_pos -= (write_pos % p);
+      
+      // pad with zeros.
+      bufferptr bp(write_pos - owp);
+      bp.zero();
+      assert(bp.length() >= 4);
+      write_buf.push_back(bp);
+      
+      // now flush.
+      flush();
+      
+      ldout(cct, 12) << "append_entry skipped " << (write_pos-owp) << " bytes to " << write_pos << " to avoid spanning stripe boundary" << dendl;
+    }
+  }
+	
+  ldout(cct, 10) << "append_entry len " << bl.length() << " to " << write_pos << "~" << (bl.length() + sizeof(uint32_t)) << dendl;
+  
+  // append
+  ::encode(s, write_buf);
+  write_buf.claim_append(bl);
+  write_pos += sizeof(s) + s;
+
+  // flush previous object?
+  uint64_t su = get_layout_period();
+  assert(su > 0);
+  uint64_t write_off = write_pos % su;
+  uint64_t write_obj = write_pos / su;
+  uint64_t flush_obj = flush_pos / su;
+  if (write_obj != flush_obj) {
+    ldout(cct, 10) << " flushing completed object(s) (su " << su << " wro " << write_obj << " flo " << flush_obj << ")" << dendl;
+    _do_flush(write_buf.length() - write_off);
+  }
+
+  return write_pos;
+}
+
+
+void Journaler::_do_flush(unsigned amount)
+{
+  if (write_pos == flush_pos)
+    return;
+  assert(write_pos > flush_pos);
+  assert(!readonly);
+
+  // flush
+  unsigned len = write_pos - flush_pos;
+  assert(len == write_buf.length());
+  if (amount && amount < len)
+    len = amount;
+
+  // zero at least two full periods ahead.  this ensures
+  // that the next object will not exist.
+  uint64_t period = get_layout_period();
+  if (flush_pos + len + 2*period > prezero_pos) {
+    _issue_prezero();
+
+    int64_t newlen = prezero_pos - flush_pos - period;
+    if (newlen <= 0) {
+      ldout(cct, 10) << "_do_flush wanted to do " << flush_pos << "~" << len
+	       << " already too close to prezero_pos " << prezero_pos << ", zeroing first" << dendl;
+      waiting_for_zero = true;
+      return;
+    }
+    if (newlen < len) {
+      ldout(cct, 10) << "_do_flush wanted to do " << flush_pos << "~" << len << " but hit prezero_pos " << prezero_pos
+	       << ", will do " << flush_pos << "~" << newlen << dendl;
+      len = newlen;
+    } else {
+      waiting_for_zero = false;
+    }
+  } else {
+    waiting_for_zero = false;
+  }
+  ldout(cct, 10) << "_do_flush flushing " << flush_pos << "~" << len << dendl;
+  
+  // submit write for anything pending
+  // flush _start_ pos to _finish_flush
+  utime_t now = ceph_clock_now(cct);
+  SnapContext snapc;
+
+  Context *onsafe = new C_Flush(this, flush_pos, now);  // on COMMIT
+  pending_safe.insert(flush_pos);
+
+  bufferlist write_bl;
+
+  // adjust pointers
+  if (len == write_buf.length()) {
+    write_bl.swap(write_buf);
+  } else {
+    write_buf.splice(0, len, &write_bl);
+  }
+
+  filer.write(ino, &layout, snapc,
+	      flush_pos, len, write_bl, ceph_clock_now(cct),
+	      0,
+	      NULL, onsafe);
+
+  flush_pos += len;
+  assert(write_buf.length() == write_pos - flush_pos);
+    
+  ldout(cct, 10) << "_do_flush (prezeroing/prezero)/write/flush/safe pointers now at "
+	   << "(" << prezeroing_pos << "/" << prezero_pos << ")/" << write_pos << "/" << flush_pos << "/" << safe_pos << dendl;
+
+  _issue_prezero();
+}
+
+
+
+void Journaler::wait_for_flush(Context *onsafe)
+{
+  assert(!readonly);
+  
+  // all flushed and safe?
+  if (write_pos == safe_pos) {
+    assert(write_buf.length() == 0);
+    ldout(cct, 10) << "flush nothing to flush, (prezeroing/prezero)/write/flush/safe pointers at " 
+	     << "(" << prezeroing_pos << "/" << prezero_pos << ")/" << write_pos << "/" << flush_pos << "/" << safe_pos << dendl;
+    if (onsafe) {
+      onsafe->complete(0);
+      onsafe = 0;
+    }
+    return;
+  }
+
+  // queue waiter
+  if (onsafe) 
+    waitfor_safe[write_pos].push_back(onsafe);
+}  
+
+void Journaler::flush(Context *onsafe)
+{
+  assert(!readonly);
+
+  if (write_pos == flush_pos) {
+    assert(write_buf.length() == 0);
+    ldout(cct, 10) << "flush nothing to flush, (prezeroing/prezero)/write/flush/safe pointers at "
+	     << "(" << prezeroing_pos << "/" << prezero_pos << ")/" << write_pos << "/" << flush_pos << "/" << safe_pos << dendl;
+    if (onsafe) {
+      onsafe->complete(0);
+    }
+  } else {
+    if (1) {
+      // maybe buffer
+      if (write_buf.length() < cct->_conf->journaler_batch_max) {
+	// delay!  schedule an event.
+	ldout(cct, 20) << "flush delaying flush" << dendl;
+	if (delay_flush_event)
+	  timer->cancel_event(delay_flush_event);
+	delay_flush_event = new C_DelayFlush(this);
+	timer->add_event_after(cct->_conf->journaler_batch_interval, delay_flush_event);	
+      } else {
+	ldout(cct, 20) << "flush not delaying flush" << dendl;
+	_do_flush();
+      }
+    } else {
+      // always flush
+      _do_flush();
+    }
+    wait_for_flush(onsafe);
+  }
+
+  // write head?
+  if (last_wrote_head.sec() + cct->_conf->journaler_write_head_interval < ceph_clock_now(cct).sec()) {
+    write_head();
+  }
+}
+
+
+/*************** prezeroing ******************/
+
+struct C_Journaler_Prezero : public Context {
+  Journaler *journaler;
+  uint64_t from, len;
+  C_Journaler_Prezero(Journaler *j, uint64_t f, uint64_t l) : journaler(j), from(f), len(l) {}
+  void finish(int r) {
+    journaler->_prezeroed(r, from, len);
+  }
+};
+
+void Journaler::_issue_prezero()
+{
+  assert(prezeroing_pos >= flush_pos);
+
+  // we need to zero at least two periods, minimum, to ensure that we have a full
+  // empty object/period in front of us.
+  uint64_t num_periods = MAX(2, cct->_conf->journaler_prezero_periods);
+
+  /*
+   * issue zero requests based on write_pos, even though the invariant
+   * is that we zero ahead of flush_pos.
+   */
+  uint64_t period = get_layout_period();
+  uint64_t to = write_pos + period * num_periods  + period - 1;
+  to -= to % period;
+
+  if (prezeroing_pos >= to) {
+    ldout(cct, 20) << "_issue_prezero target " << to << " <= prezeroing_pos " << prezeroing_pos << dendl;
+    return;
+  }
+
+  while (prezeroing_pos < to) {
+    uint64_t len;
+    if (prezeroing_pos % period == 0) {
+      len = period;
+      ldout(cct, 10) << "_issue_prezero removing " << prezeroing_pos << "~" << period << " (full period)" << dendl;
+    } else {
+      len = period - (prezeroing_pos % period);
+      ldout(cct, 10) << "_issue_prezero zeroing " << prezeroing_pos << "~" << len << " (partial period)" << dendl;
+    }
+    SnapContext snapc;
+    Context *c = new C_Journaler_Prezero(this, prezeroing_pos, len);
+    filer.zero(ino, &layout, snapc, prezeroing_pos, len, ceph_clock_now(cct), 0, NULL, c);
+    prezeroing_pos += len;
+  }
+}
+
+void Journaler::_prezeroed(int r, uint64_t start, uint64_t len)
+{
+  ldout(cct, 10) << "_prezeroed to " << start << "~" << len
+	   << ", prezeroing/prezero was " << prezeroing_pos << "/" << prezero_pos
+	   << ", pending " << pending_zero
+	   << dendl;
+  if (r < 0 && r != -ENOENT) {
+    lderr(cct) << "_prezeroed got " << cpp_strerror(r) << dendl;
+    handle_write_error(r);
+    return;
+  }
+
+  assert(r == 0 || r == -ENOENT);
+
+  if (start == prezero_pos) {
+    prezero_pos += len;
+    while (!pending_zero.empty() &&
+	   pending_zero.begin().get_start() == prezero_pos) {
+      interval_set::iterator b(pending_zero.begin());
+      prezero_pos += b.get_len();
+      pending_zero.erase(b);
+    }
+
+    if (waiting_for_zero) {
+      _do_flush();
+    }
+  } else {
+    pending_zero.insert(start, len);
+  }
+  ldout(cct, 10) << "_prezeroed prezeroing/prezero now " << prezeroing_pos << "/" << prezero_pos
+	   << ", pending " << pending_zero
+	   << dendl;
+}
+
+
+
+/***************** READING *******************/
+
+
+class Journaler::C_Read : public Context {
+  Journaler *ls;
+  uint64_t offset;
+public:
+  bufferlist bl;
+  C_Read(Journaler *l, uint64_t o) : ls(l), offset(o) {}
+  void finish(int r) {
+    ls->_finish_read(r, offset, bl);
+  }
+};
+
+class Journaler::C_RetryRead : public Context {
+  Journaler *ls;
+public:
+  C_RetryRead(Journaler *l) : ls(l) {}
+  void finish(int r) {
+    // kickstart.
+    ls->_prefetch();
+  }  
+};
+
+void Journaler::_finish_read(int r, uint64_t offset, bufferlist& bl)
+{
+  if (r < 0) {
+    ldout(cct, 0) << "_finish_read got error " << r << dendl;
+    error = r;
+    if (on_readable) {
+      Context *f = on_readable;
+      on_readable = 0;
+      f->complete(r);
+    }
+    return;
+  }
+  assert(r>=0);
+
+  ldout(cct, 10) << "_finish_read got " << offset << "~" << bl.length() << dendl;
+  prefetch_buf[offset].swap(bl);
+
+  _assimilate_prefetch();
+  _prefetch();
+}
+
+void Journaler::_assimilate_prefetch()
+{
+  bool was_readable = _is_readable();
+
+  bool got_any = false;
+  while (!prefetch_buf.empty()) {
+    map::iterator p = prefetch_buf.begin();
+    if (p->first != received_pos) {
+      uint64_t gap = p->first - received_pos;
+      ldout(cct, 10) << "_assimilate_prefetch gap of " << gap << " from received_pos " << received_pos
+	       << " to first prefetched buffer " << p->first << dendl;
+      break;
+    }
+
+    ldout(cct, 10) << "_assimilate_prefetch " << p->first << "~" << p->second.length() << dendl;
+    received_pos += p->second.length();
+    read_buf.claim_append(p->second);
+    assert(received_pos <= requested_pos);
+    prefetch_buf.erase(p);
+    got_any = true;
+  }
+
+  if (got_any)
+    ldout(cct, 10) << "_assimilate_prefetch read_buf now " << read_pos << "~" << read_buf.length() 
+	     << ", read pointers " << read_pos << "/" << received_pos << "/" << requested_pos
+	     << dendl;
+
+  if ((got_any && !was_readable && _is_readable()) ||
+      read_pos == write_pos) {
+    // readable!
+    ldout(cct, 10) << "_finish_read now readable (or at journal end)" << dendl;
+    if (on_readable) {
+      Context *f = on_readable;
+      on_readable = 0;
+      f->complete(0);
+    }
+  }
+}
+
+void Journaler::_issue_read(uint64_t len)
+{
+  // make sure we're fully flushed
+  _do_flush();
+
+  // stuck at safe_pos?
+  //  (this is needed if we are reading the tail of a journal we are also writing to)
+  assert(requested_pos <= safe_pos);
+  if (requested_pos == safe_pos) {
+    ldout(cct, 10) << "_issue_read requested_pos = safe_pos = " << safe_pos << ", waiting" << dendl;
+    assert(write_pos > requested_pos);
+    if (flush_pos == safe_pos)
+      flush();
+    assert(flush_pos > safe_pos);
+    waitfor_safe[flush_pos].push_back(new C_RetryRead(this));
+    return;
+  }
+
+  // don't read too much
+  if (requested_pos + len > safe_pos) {
+    len = safe_pos - requested_pos;
+    ldout(cct, 10) << "_issue_read reading only up to safe_pos " << safe_pos << dendl;
+  }
+
+  // go.
+  ldout(cct, 10) << "_issue_read reading " << requested_pos << "~" << len 
+	   << ", read pointers " << read_pos << "/" << received_pos << "/" << (requested_pos+len)
+	   << dendl;
+  
+  // step by period (object).  _don't_ do a single big filer.read()
+  // here because it will wait for all object reads to complete before
+  // giving us back any data.  this way we can process whatever bits
+  // come in that are contiguous.
+  uint64_t period = get_layout_period();
+  while (len > 0) {
+    uint64_t e = requested_pos + period;
+    e -= e % period;
+    uint64_t l = e - requested_pos;
+    if (l > len)
+      l = len;
+    C_Read *c = new C_Read(this, requested_pos);
+    filer.read(ino, &layout, CEPH_NOSNAP, requested_pos, l, &c->bl, 0, c);
+    requested_pos += l;
+    len -= l;
+  }
+}
+
+void Journaler::_prefetch()
+{
+  ldout(cct, 10) << "_prefetch" << dendl;
+  // prefetch
+  uint64_t pf;
+  if (temp_fetch_len) {
+    ldout(cct, 10) << "_prefetch temp_fetch_len " << temp_fetch_len << dendl;
+    pf = temp_fetch_len;
+    temp_fetch_len = 0;
+  } else {
+    pf = fetch_len;
+  }
+
+  uint64_t raw_target = read_pos + pf;
+
+  // read full log segments, so increase if necessary
+  uint64_t period = get_layout_period();
+  uint64_t remainder = raw_target % period;
+  uint64_t adjustment = remainder ? period - remainder : 0;
+  uint64_t target = raw_target + adjustment;
+
+  // don't read past the log tail
+  if (target > write_pos)
+    target = write_pos;
+
+  if (requested_pos < target) {
+    uint64_t len = target - requested_pos;
+    ldout(cct, 10) << "_prefetch " << pf << " requested_pos " << requested_pos << " < target " << target
+	     << " (" << raw_target << "), prefetching " << len << dendl;
+    _issue_read(len);
+  }
+}
+
+/*
+ * _is_readable() - return true if next entry is ready.
+ */
+bool Journaler::_is_readable()
+{
+  // anything to read?
+  if (read_pos == write_pos)
+    return false;
+
+  // have enough for entry size?
+  uint32_t s = 0;
+  bufferlist::iterator p = read_buf.begin();
+  if (read_buf.length() >= sizeof(s))
+    ::decode(s, p);
+
+  // entry and payload?
+  if (read_buf.length() >= sizeof(s) &&
+      read_buf.length() >= sizeof(s) + s) 
+    return true;  // yep, next entry is ready.
+
+  ldout (cct, 10) << "_is_readable read_buf.length() == " << read_buf.length()
+		  << ", but need " << s + sizeof(s)
+		  << " for next entry; fetch_len is " << fetch_len << dendl;
+
+  // partial fragment at the end?
+  if (received_pos == write_pos) {
+    ldout(cct, 10) << "is_readable() detected partial entry at tail, adjusting write_pos to " << read_pos << dendl;
+
+    // adjust write_pos
+    prezeroing_pos = prezero_pos = write_pos = flush_pos = safe_pos = read_pos;
+    assert(write_buf.length() == 0);
+
+    // reset read state
+    requested_pos = received_pos = read_pos;
+    read_buf.clear();    
+    
+    // FIXME: truncate on disk?
+
+    return false;
+  }
+
+  uint64_t need = sizeof(s) + s;
+  if (need > fetch_len) {
+    temp_fetch_len = sizeof(s) + s;
+    ldout(cct, 10) << "_is_readable noting temp_fetch_len " << temp_fetch_len
+	     << " for len " << s << " entry" << dendl;
+  }
+
+  ldout(cct, 10) << "_is_readable: not readable, returning false" << dendl;
+  return false;
+}
+
+/*
+ * is_readable() - kickstart prefetch, too
+ */
+bool Journaler::is_readable() 
+{
+  bool r = _is_readable();
+  _prefetch();
+  return r;
+}
+
+
+/* try_read_entry(bl)
+ *  read entry into bl if it's ready.
+ *  otherwise, do nothing.  (well, we'll start fetching it for good measure.)
+ */
+bool Journaler::try_read_entry(bufferlist& bl)
+{
+  if (!is_readable()) {  // this may start a read. 
+    ldout(cct, 10) << "try_read_entry at " << read_pos << " not readable" << dendl;
+    return false;
+  }
+  
+  uint32_t s;
+  {
+    bufferlist::iterator p = read_buf.begin();
+    ::decode(s, p);
+  }
+  assert(read_buf.length() >= sizeof(s) + s);
+  
+  ldout(cct, 10) << "try_read_entry at " << read_pos << " reading " 
+	   << read_pos << "~" << (sizeof(s)+s) << " (have " << read_buf.length() << ")" << dendl;
+
+  if (s == 0) {
+    ldout(cct, 0) << "try_read_entry got 0 len entry at offset " << read_pos << dendl;
+    error = -EINVAL;
+    return false;
+  }
+
+  // do it
+  assert(bl.length() == 0);
+  read_buf.splice(0, sizeof(s));
+  read_buf.splice(0, s, &bl);
+  read_pos += sizeof(s) + s;
+
+  // prefetch?
+  _prefetch();
+  return true;
+}
+
+void Journaler::wait_for_readable(Context *onreadable)
+{
+  ldout(cct, 10) << "wait_for_readable at " << read_pos << " onreadable " << onreadable << dendl;
+  assert(!_is_readable());
+  assert(on_readable == 0);
+  on_readable = onreadable;
+}
+
+
+
+
+/***************** TRIMMING *******************/
+
+
+class Journaler::C_Trim : public Context {
+  Journaler *ls;
+  uint64_t to;
+public:
+  C_Trim(Journaler *l, int64_t t) : ls(l), to(t) {}
+  void finish(int r) {
+    ls->_trim_finish(r, to);
+  }
+};
+
+void Journaler::trim()
+{
+  assert(!readonly);
+  uint64_t period = get_layout_period();
+  uint64_t trim_to = last_committed.expire_pos;
+  trim_to -= trim_to % period;
+  ldout(cct, 10) << "trim last_commited head was " << last_committed
+	   << ", can trim to " << trim_to
+	   << dendl;
+  if (trim_to == 0 || trim_to == trimming_pos) {
+    ldout(cct, 10) << "trim already trimmed/trimming to " 
+	     << trimmed_pos << "/" << trimming_pos << dendl;
+    return;
+  }
+
+  if (trimming_pos > trimmed_pos) {
+    ldout(cct, 10) << "trim already trimming atm, try again later.  trimmed/trimming is " 
+	     << trimmed_pos << "/" << trimming_pos << dendl;
+    return;
+  }
+  
+  // trim
+  assert(trim_to <= write_pos);
+  assert(trim_to <= expire_pos);
+  assert(trim_to > trimming_pos);
+  ldout(cct, 10) << "trim trimming to " << trim_to 
+	   << ", trimmed/trimming/expire are " 
+	   << trimmed_pos << "/" << trimming_pos << "/" << expire_pos
+	   << dendl;
+
+  // delete range of objects
+  uint64_t first = trimming_pos / period;
+  uint64_t num = (trim_to - trimming_pos) / period;
+  SnapContext snapc;
+  filer.purge_range(ino, &layout, snapc, first, num, ceph_clock_now(cct), 0, 
+		    new C_Trim(this, trim_to));
+  trimming_pos = trim_to;  
+}
+
+void Journaler::_trim_finish(int r, uint64_t to)
+{
+  assert(!readonly);
+  ldout(cct, 10) << "_trim_finish trimmed_pos was " << trimmed_pos
+	   << ", trimmed/trimming/expire now "
+	   << to << "/" << trimming_pos << "/" << expire_pos
+	   << dendl;
+  if (r < 0 && r != -ENOENT) {
+    lderr(cct) << "_trim_finish got " << cpp_strerror(r) << dendl;
+    handle_write_error(r);
+    return;
+  }
+
+  assert(r >= 0 || r == -ENOENT);
+  
+  assert(to <= trimming_pos);
+  assert(to > trimmed_pos);
+  trimmed_pos = to;
+
+  // finishers?
+  while (!waitfor_trim.empty() &&
+	 waitfor_trim.begin()->first <= trimmed_pos) {
+    finish_contexts(cct, waitfor_trim.begin()->second, 0);
+    waitfor_trim.erase(waitfor_trim.begin());
+  }
+}
+
+void Journaler::handle_write_error(int r)
+{
+  lderr(cct) << "handle_write_error " << cpp_strerror(r) << dendl;
+  if (on_write_error) {
+    on_write_error->complete(r);
+    on_write_error = NULL;
+  } else {
+    assert(0 == "unhandled write error");
+  }
+}
+
+
+// eof.
diff --git a/ceph/src/osdc/Journaler.h b/ceph/src/osdc/Journaler.h
new file mode 100644
index 00000000..e3a57135
--- /dev/null
+++ b/ceph/src/osdc/Journaler.h
@@ -0,0 +1,400 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/* Journaler
+ *
+ * This class stripes a serial log over objects on the store.  Four logical pointers:
+ *
+ *  write_pos - where we're writing new entries
+ *   unused_field - where we're reading old entires
+ * expire_pos - what is deemed "old" by user
+ *   trimmed_pos - where we're expiring old items
+ *
+ *  trimmed_pos <= expire_pos <= unused_field <= write_pos.
+ *
+ * Often, unused_field <= write_pos (as with MDS log).  During recovery, write_pos is undefined
+ * until the end of the log is discovered.
+ *
+ * A "head" struct at the beginning of the log is used to store metadata at
+ * regular intervals.  The basic invariants include:
+ *
+ *   head.unused_field   <= unused_field   -- the head may "lag", since it's updated lazily.
+ *   head.write_pos  <= write_pos
+ *   head.expire_pos <= expire_pos
+ *   head.trimmed_pos   <= trimmed_pos
+ *
+ * More significantly,
+ *
+ *   head.expire_pos >= trimmed_pos -- this ensures we can find the "beginning" of the log
+ *                                  as last recorded, before it is trimmed.  trimming will
+ *                                  block until a sufficiently current expire_pos is committed.
+ *
+ * To recover log state, we simply start at the last write_pos in the head, and probe the
+ * object sequence sizes until we read the end.  
+ *
+ * Head struct is stored in the first object.  Actual journal starts after layout.period() bytes.
+ *
+ */
+
+#ifndef CEPH_JOURNALER_H
+#define CEPH_JOURNALER_H
+
+#include "Objecter.h"
+#include "Filer.h"
+
+#include 
+#include 
+
+class CephContext;
+class Context;
+class PerfCounters;
+
+class Journaler {
+public:
+  CephContext *cct;
+  // this goes at the head of the log "file".
+  struct Header {
+    uint64_t trimmed_pos;
+    uint64_t expire_pos;
+    uint64_t unused_field;
+    uint64_t write_pos;
+    string magic;
+    ceph_file_layout layout;
+
+    Header(const char *m="") :
+      trimmed_pos(0), expire_pos(0), unused_field(0), write_pos(0),
+      magic(m) {
+      memset(&layout, 0, sizeof(layout));
+    }
+
+    void encode(bufferlist &bl) const {
+      __u8 struct_v = 1;
+      ::encode(struct_v, bl);
+      ::encode(magic, bl);
+      ::encode(trimmed_pos, bl);
+      ::encode(expire_pos, bl);
+      ::encode(unused_field, bl);
+      ::encode(write_pos, bl);
+      ::encode(layout, bl);
+    }
+    void decode(bufferlist::iterator &bl) {
+      __u8 struct_v;
+      ::decode(struct_v, bl);
+      ::decode(magic, bl);
+      ::decode(trimmed_pos, bl);
+      ::decode(expire_pos, bl);
+      ::decode(unused_field, bl);
+      ::decode(write_pos, bl);
+      ::decode(layout, bl);
+    }
+
+    void dump(Formatter *f) const {
+      f->open_object_section("journal_header");
+      {
+	f->dump_string("magic", magic);
+	f->dump_unsigned("write_pos", write_pos);
+	f->dump_unsigned("expire_pos", expire_pos);
+	f->dump_unsigned("trimmed_pos", trimmed_pos);
+	f->open_object_section("layout");
+	{
+	  f->dump_unsigned("stripe_unit", layout.fl_stripe_unit);
+	  f->dump_unsigned("stripe_count", layout.fl_stripe_unit);
+	  f->dump_unsigned("object_size", layout.fl_stripe_unit);
+	  f->dump_unsigned("cas_hash", layout.fl_stripe_unit);
+	  f->dump_unsigned("object_stripe_unit", layout.fl_stripe_unit);
+	  f->dump_unsigned("pg_pool", layout.fl_stripe_unit);
+	}
+	f->close_section(); // layout
+      }
+      f->close_section(); // journal_header
+    }
+
+    static void generate_test_instances(list &ls)
+    {
+      ls.push_back(new Header());
+      ls.push_back(new Header());
+      ls.back()->trimmed_pos = 1;
+      ls.back()->expire_pos = 2;
+      ls.back()->unused_field = 3;
+      ls.back()->write_pos = 4;
+      ls.back()->magic = "magique";
+    }
+  } last_written, last_committed;
+  WRITE_CLASS_ENCODER(Header)
+
+private:
+  // me
+  inodeno_t ino;
+  int64_t pg_pool;
+  bool readonly;
+  ceph_file_layout layout;
+
+  const char *magic;
+  Objecter *objecter;
+  Filer filer;
+
+  PerfCounters *logger;
+  int logger_key_lat;
+
+  SafeTimer *timer;
+
+  class C_DelayFlush : public Context {
+    Journaler *journaler;
+  public:
+    C_DelayFlush(Journaler *j) : journaler(j) {}
+    void finish(int r) {
+      journaler->delay_flush_event = 0;
+      journaler->_do_flush();
+    }
+  } *delay_flush_event;
+
+
+  // my state
+  static const int STATE_UNDEF = 0;
+  static const int STATE_READHEAD = 1;
+  static const int STATE_PROBING = 2;
+  static const int STATE_ACTIVE = 3;
+  static const int STATE_REREADHEAD = 4;
+  static const int STATE_REPROBING = 5;
+
+  int state;
+  int error;
+
+  // header
+  utime_t last_wrote_head;
+  void _finish_write_head(int r, Header &wrote, Context *oncommit);
+  class C_WriteHead;
+  friend class C_WriteHead;
+
+  list waitfor_recover;
+  void read_head(Context *on_finish, bufferlist *bl);
+  void _finish_read_head(int r, bufferlist& bl);
+  void _finish_reread_head(int r, bufferlist& bl, Context *finish);
+  void probe(Context *finish, uint64_t *end);
+  void _finish_probe_end(int r, uint64_t end);
+  void _finish_reprobe(int r, uint64_t end, Context *onfinish);
+  void _finish_reread_head_and_probe(int r, Context *onfinish);
+  class C_ReadHead;
+  friend class C_ReadHead;
+  class C_ProbeEnd;
+  friend class C_ProbeEnd;
+  class C_RereadHead;
+  friend class C_RereadHead;
+  class C_ReProbe;
+  friend class C_ReProbe;
+  class C_RereadHeadProbe;
+  friend class C_RereadHeadProbe;
+
+
+
+  // writer
+  uint64_t prezeroing_pos;
+  uint64_t prezero_pos;     // we zero journal space ahead of write_pos to avoid problems with tail probing
+  uint64_t write_pos;       // logical write position, where next entry will go
+  uint64_t flush_pos;       // where we will flush. if write_pos>flush_pos, we're buffering writes.
+  uint64_t safe_pos;        // what has been committed safely to disk.
+  bufferlist write_buf;  // write buffer.  flush_pos + write_buf.length() == write_pos.
+
+  bool waiting_for_zero;
+  interval_set pending_zero;  // non-contig bits we've zeroed
+  std::set pending_safe;
+  std::map > waitfor_safe; // when safe through given offset
+
+  void _do_flush(unsigned amount=0);
+  void _finish_flush(int r, uint64_t start, utime_t stamp);
+  class C_Flush;
+  friend class C_Flush;
+
+  // reader
+  uint64_t read_pos;      // logical read position, where next entry starts.
+  uint64_t requested_pos; // what we've requested from OSD.
+  uint64_t received_pos;  // what we've received from OSD.
+  bufferlist read_buf; // read buffer.  unused_field + read_buf.length() == prefetch_pos.
+
+  map prefetch_buf;
+
+  uint64_t fetch_len;     // how much to read at a time
+  uint64_t temp_fetch_len;
+
+  // for wait_for_readable()
+  Context    *on_readable;
+
+  Context    *on_write_error;
+
+  void _finish_read(int r, uint64_t offset, bufferlist &bl); // read completion callback
+  void _assimilate_prefetch();
+  void _issue_read(uint64_t len);  // read some more
+  void _prefetch();             // maybe read ahead
+  class C_Read;
+  friend class C_Read;
+  class C_RetryRead;
+  friend class C_RetryRead;
+
+  // trimmer
+  uint64_t expire_pos;    // what we're allowed to trim to
+  uint64_t trimming_pos;      // what we've requested to trim through
+  uint64_t trimmed_pos;   // what has been trimmed
+  map > waitfor_trim;
+
+  void _trim_finish(int r, uint64_t to);
+  class C_Trim;
+  friend class C_Trim;
+
+  void _issue_prezero();
+  void _prezeroed(int r, uint64_t from, uint64_t len);
+  friend struct C_Journaler_Prezero;
+
+  // only init_headers when following or first reading off-disk
+  void init_headers(Header& h) {
+    assert(readonly ||
+           state == STATE_READHEAD ||
+           state == STATE_REREADHEAD);
+    last_written = last_committed = h;
+  }
+
+  /**
+   * handle a write error
+   *
+   * called when we get an objecter error on a write.
+   *
+   * @param r error code
+   */
+  void handle_write_error(int r);
+
+public:
+  Journaler(inodeno_t ino_, int64_t pool, const char *mag, Objecter *obj, PerfCounters *l, int lkey, SafeTimer *tim) : 
+    cct(obj->cct), last_written(mag), last_committed(mag),
+    ino(ino_), pg_pool(pool), readonly(true), magic(mag),
+    objecter(obj), filer(objecter), logger(l), logger_key_lat(lkey),
+    timer(tim), delay_flush_event(0),
+    state(STATE_UNDEF), error(0),
+    prezeroing_pos(0), prezero_pos(0), write_pos(0), flush_pos(0), safe_pos(0),
+    waiting_for_zero(false),
+    read_pos(0), requested_pos(0), received_pos(0),
+    fetch_len(0), temp_fetch_len(0),
+    on_readable(0), on_write_error(NULL),
+    expire_pos(0), trimming_pos(0), trimmed_pos(0) 
+  {
+    memset(&layout, 0, sizeof(layout));
+  }
+
+  void reset() {
+    assert(state == STATE_ACTIVE);
+    readonly = true;
+    delay_flush_event = 0;
+    state = STATE_UNDEF;
+    error = 0;
+    prezeroing_pos = 0;
+    prezero_pos = 0;
+    write_pos = 0;
+    flush_pos = 0;
+    safe_pos = 0;
+    read_pos = 0;
+    requested_pos = 0;
+    received_pos = 0;
+    fetch_len = 0;
+    assert(!on_readable);
+    expire_pos = 0;
+    trimming_pos = 0;
+    trimmed_pos = 0;
+    waiting_for_zero = false;
+  }
+
+  // me
+  //void open(Context *onopen);
+  //void claim(Context *onclaim, msg_addr_t from);
+
+  /* reset 
+   *  NOTE: we assume the caller knows/has ensured that any objects 
+   * in our sequence do not exist.. e.g. after a MKFS.  this is _not_
+   * an "erase" method.
+   */
+  void create(ceph_file_layout *layout);
+  void recover(Context *onfinish);
+  void reread_head(Context *onfinish);
+  void reprobe(Context *onfinish);
+  void reread_head_and_probe(Context *onfinish);
+  void write_head(Context *onsave=0);
+
+  void set_layout(ceph_file_layout *l);
+
+  void set_readonly();
+  void set_writeable();
+  bool is_readonly() { return readonly; }
+
+  bool is_active() { return state == STATE_ACTIVE; }
+  int get_error() { return error; }
+
+  uint64_t get_write_pos() const { return write_pos; }
+  uint64_t get_write_safe_pos() const { return safe_pos; }
+  uint64_t get_read_pos() const { return read_pos; }
+  uint64_t get_expire_pos() const { return expire_pos; }
+  uint64_t get_trimmed_pos() const { return trimmed_pos; }
+
+  uint64_t get_layout_period() const { return (uint64_t)layout.fl_stripe_count * (uint64_t)layout.fl_object_size; }
+  ceph_file_layout& get_layout() { return layout; }
+
+  // write
+  uint64_t append_entry(bufferlist& bl);
+  void wait_for_flush(Context *onsafe = 0);
+  void flush(Context *onsafe = 0);
+
+  // read
+  void set_read_pos(int64_t p) { 
+    assert(requested_pos == received_pos);  // we can't cope w/ in-progress read right now.
+    read_pos = requested_pos = received_pos = p;
+    read_buf.clear();
+  }
+
+  bool _is_readable();
+  bool is_readable();
+  bool try_read_entry(bufferlist& bl);
+  void wait_for_readable(Context *onfinish);
+  
+  void set_write_pos(int64_t p) { 
+    prezeroing_pos = prezero_pos = write_pos = flush_pos = safe_pos = p;
+  }
+
+  /**
+   * set write error callback
+   *
+   * Set a callback/context to trigger if we get a write error from
+   * the objecter.  This may be from an explicit request (e.g., flush)
+   * or something async the journaler did on its own (e.g., journal
+   * header update).
+   *
+   * It is only used once; if the caller continues to use the
+   * Journaler and wants to hear about errors, it needs to reset the
+   * error_handler.
+   *
+   * @param c callback/context to trigger on error
+   */
+  void set_write_error_handler(Context *c) {
+    assert(!on_write_error);
+    on_write_error = c;
+  }
+
+  // trim
+  void set_expire_pos(int64_t ep) { expire_pos = ep; }
+  void set_trimmed_pos(int64_t p) { trimming_pos = trimmed_pos = p; }
+
+  void trim();
+  void trim_tail() {
+    assert(!readonly);
+    _issue_prezero();
+  }
+};
+WRITE_CLASS_ENCODER(Journaler::Header)
+
+#endif
diff --git a/ceph/src/osdc/Makefile.am b/ceph/src/osdc/Makefile.am
new file mode 100644
index 00000000..3a8a2165
--- /dev/null
+++ b/ceph/src/osdc/Makefile.am
@@ -0,0 +1,17 @@
+libosdc_la_SOURCES = \
+	osdc/Objecter.cc \
+	osdc/ObjectCacher.cc \
+	osdc/Filer.cc \
+	osdc/Striper.cc \
+	osdc/Journaler.cc
+noinst_LTLIBRARIES += libosdc.la
+
+noinst_HEADERS += \
+	osdc/Blinker.h \
+	osdc/Filer.h \
+	osdc/Journaler.h \
+	osdc/ObjectCacher.h \
+	osdc/Objecter.h \
+	osdc/Striper.h \
+	osdc/WritebackHandler.h
+
diff --git a/ceph/src/osdc/ObjectCacher.cc b/ceph/src/osdc/ObjectCacher.cc
new file mode 100644
index 00000000..e1499b41
--- /dev/null
+++ b/ceph/src/osdc/ObjectCacher.cc
@@ -0,0 +1,2049 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#include 
+
+#include "msg/Messenger.h"
+#include "ObjectCacher.h"
+#include "WritebackHandler.h"
+#include "common/errno.h"
+#include "common/perf_counters.h"
+
+#include "include/assert.h"
+
+#define MAX_FLUSH_UNDER_LOCK 20  ///< max bh's we start writeback on while holding the lock
+
+/*** ObjectCacher::BufferHead ***/
+
+
+/*** ObjectCacher::Object ***/
+
+#define dout_subsys ceph_subsys_objectcacher
+#undef dout_prefix
+#define dout_prefix *_dout << "objectcacher.object(" << oid << ") "
+
+
+
+ObjectCacher::BufferHead *ObjectCacher::Object::split(BufferHead *left, loff_t off)
+{
+  assert(oc->lock.is_locked());
+  ldout(oc->cct, 20) << "split " << *left << " at " << off << dendl;
+  
+  // split off right
+  ObjectCacher::BufferHead *right = new BufferHead(this);
+  right->last_write_tid = left->last_write_tid;
+  right->last_read_tid = left->last_read_tid;
+  right->set_state(left->get_state());
+  right->snapc = left->snapc;
+
+  loff_t newleftlen = off - left->start();
+  right->set_start(off);
+  right->set_length(left->length() - newleftlen);
+  
+  // shorten left
+  oc->bh_stat_sub(left);
+  left->set_length(newleftlen);
+  oc->bh_stat_add(left);
+  
+  // add right
+  oc->bh_add(this, right);
+  
+  // split buffers too
+  bufferlist bl;
+  bl.claim(left->bl);
+  if (bl.length()) {
+    assert(bl.length() == (left->length() + right->length()));
+    right->bl.substr_of(bl, left->length(), right->length());
+    left->bl.substr_of(bl, 0, left->length());
+  }
+
+  // move read waiters
+  if (!left->waitfor_read.empty()) {
+    map >::iterator start_remove = left->waitfor_read.begin();
+    while (start_remove != left->waitfor_read.end() &&
+	   start_remove->first < right->start())
+      ++start_remove;
+    for (map >::iterator p = start_remove;
+	 p != left->waitfor_read.end(); ++p) {
+      ldout(oc->cct, 20) << "split  moving waiters at byte " << p->first << " to right bh" << dendl;
+      right->waitfor_read[p->first].swap( p->second );
+      assert(p->second.empty());
+    }
+    left->waitfor_read.erase(start_remove, left->waitfor_read.end());
+  }
+
+  ldout(oc->cct, 20) << "split    left is " << *left << dendl;
+  ldout(oc->cct, 20) << "split   right is " << *right << dendl;
+  return right;
+}
+
+
+void ObjectCacher::Object::merge_left(BufferHead *left, BufferHead *right)
+{
+  assert(oc->lock.is_locked());
+  assert(left->end() == right->start());
+  assert(left->get_state() == right->get_state());
+
+  ldout(oc->cct, 10) << "merge_left " << *left << " + " << *right << dendl;
+  oc->bh_remove(this, right);
+  oc->bh_stat_sub(left);
+  left->set_length(left->length() + right->length());
+  oc->bh_stat_add(left);
+
+  // data
+  left->bl.claim_append(right->bl);
+  
+  // version 
+  // note: this is sorta busted, but should only be used for dirty buffers
+  left->last_write_tid =  MAX( left->last_write_tid, right->last_write_tid );
+  left->last_write = MAX( left->last_write, right->last_write );
+
+  // waiters
+  for (map >::iterator p = right->waitfor_read.begin();
+       p != right->waitfor_read.end();
+       ++p) 
+    left->waitfor_read[p->first].splice( left->waitfor_read[p->first].begin(),
+                                         p->second );
+  
+  // hose right
+  delete right;
+
+  ldout(oc->cct, 10) << "merge_left result " << *left << dendl;
+}
+
+void ObjectCacher::Object::try_merge_bh(BufferHead *bh)
+{
+  assert(oc->lock.is_locked());
+  ldout(oc->cct, 10) << "try_merge_bh " << *bh << dendl;
+
+  // do not merge rx buffers; last_read_tid may not match
+  if (bh->is_rx())
+    return;
+
+  // to the left?
+  map::iterator p = data.find(bh->start());
+  assert(p->second == bh);
+  if (p != data.begin()) {
+    --p;
+    if (p->second->end() == bh->start() &&
+	p->second->get_state() == bh->get_state()) {
+      merge_left(p->second, bh);
+      bh = p->second;
+    } else {
+      ++p;
+    }
+  }
+  // to the right?
+  assert(p->second == bh);
+  ++p;
+  if (p != data.end() &&
+      p->second->start() == bh->end() &&
+      p->second->get_state() == bh->get_state())
+    merge_left(bh, p->second);
+}
+
+/*
+ * count bytes we have cached in given range
+ */
+bool ObjectCacher::Object::is_cached(loff_t cur, loff_t left)
+{
+  assert(oc->lock.is_locked());
+  map::iterator p = data_lower_bound(cur);
+  while (left > 0) {
+    if (p == data.end())
+      return false;
+
+    if (p->first <= cur) {
+      // have part of it
+      loff_t lenfromcur = MIN(p->second->end() - cur, left);
+      cur += lenfromcur;
+      left -= lenfromcur;
+      ++p;
+      continue;
+    } else if (p->first > cur) {
+      // gap
+      return false;
+    } else
+      assert(0);
+  }
+
+  return true;
+}
+
+/*
+ * map a range of bytes into buffer_heads.
+ * - create missing buffer_heads as necessary.
+ */
+int ObjectCacher::Object::map_read(OSDRead *rd,
+                                   map& hits,
+                                   map& missing,
+                                   map& rx,
+				   map& errors)
+{
+  assert(oc->lock.is_locked());
+  for (vector::iterator ex_it = rd->extents.begin();
+       ex_it != rd->extents.end();
+       ++ex_it) {
+    
+    if (ex_it->oid != oid.oid)
+      continue;
+
+    ldout(oc->cct, 10) << "map_read " << ex_it->oid 
+		       << " " << ex_it->offset << "~" << ex_it->length
+		       << dendl;
+    
+    loff_t cur = ex_it->offset;
+    loff_t left = ex_it->length;
+
+    map::iterator p = data_lower_bound(ex_it->offset);
+    while (left > 0) {
+      // at end?
+      if (p == data.end()) {
+        // rest is a miss.
+        BufferHead *n = new BufferHead(this);
+        n->set_start(cur);
+        n->set_length(left);
+        oc->bh_add(this, n);
+	if (complete) {
+	  oc->mark_zero(n);
+	  hits[cur] = n;
+	  ldout(oc->cct, 20) << "map_read miss+complete+zero " << left << " left, " << *n << dendl;
+	} else {
+	  missing[cur] = n;
+	  ldout(oc->cct, 20) << "map_read miss " << left << " left, " << *n << dendl;
+	}
+        cur += left;
+        left = 0;
+        assert(cur == (loff_t)ex_it->offset + (loff_t)ex_it->length);
+        break;  // no more.
+      }
+      
+      if (p->first <= cur) {
+        // have it (or part of it)
+        BufferHead *e = p->second;
+
+        if (e->is_clean() ||
+            e->is_dirty() ||
+            e->is_tx() ||
+	    e->is_zero()) {
+          hits[cur] = e;     // readable!
+          ldout(oc->cct, 20) << "map_read hit " << *e << dendl;
+        } else if (e->is_rx()) {
+          rx[cur] = e;       // missing, not readable.
+          ldout(oc->cct, 20) << "map_read rx " << *e << dendl;
+        } else if (e->is_error()) {
+	  errors[cur] = e;
+	  ldout(oc->cct, 20) << "map_read error " << *e << dendl;
+	} else {
+	  assert(0);
+	}
+        
+        loff_t lenfromcur = MIN(e->end() - cur, left);
+        cur += lenfromcur;
+        left -= lenfromcur;
+        ++p;
+        continue;  // more?
+        
+      } else if (p->first > cur) {
+        // gap.. miss
+        loff_t next = p->first;
+        BufferHead *n = new BufferHead(this);
+	loff_t len = MIN(next - cur, left);
+        n->set_start(cur);
+	n->set_length(len);
+        oc->bh_add(this,n);
+	if (complete) {
+	  oc->mark_zero(n);
+	  hits[cur] = n;
+	  ldout(oc->cct, 20) << "map_read gap+complete+zero " << *n << dendl;
+	} else {
+	  missing[cur] = n;
+	  ldout(oc->cct, 20) << "map_read gap " << *n << dendl;
+	}
+        cur += MIN(left, n->length());
+        left -= MIN(left, n->length());
+        continue;    // more?
+      } else {
+        assert(0);
+      }
+    }
+  }
+  return 0;
+}
+
+void ObjectCacher::Object::audit_buffers()
+{
+  loff_t offset = 0;
+  for (map::const_iterator it = data.begin();
+       it != data.end(); ++it) {
+    if (it->first != it->second->start()) {
+      lderr(oc->cct) << "AUDIT FAILURE: map position " << it->first
+		     << " does not match bh start position: "
+		     << *it->second << dendl;
+      assert(it->first == it->second->start());
+    }
+    if (it->first < offset) {
+      lderr(oc->cct) << "AUDIT FAILURE: " << it->first << " " << *it->second
+		     << " overlaps with previous bh " << *((--it)->second)
+		     << dendl;
+      assert(it->first >= offset);
+    }
+    BufferHead *bh = it->second;
+    map >::const_iterator w_it;
+    for (w_it = bh->waitfor_read.begin();
+	 w_it != bh->waitfor_read.end(); ++w_it) {
+      if (w_it->first < bh->start() ||
+	    w_it->first >= bh->start() + bh->length()) {
+	lderr(oc->cct) << "AUDIT FAILURE: waiter at " << w_it->first
+		       << " is not within bh " << *bh << dendl;
+	assert(w_it->first >= bh->start());
+	assert(w_it->first < bh->start() + bh->length());
+      }
+    }
+    offset = it->first + it->second->length();
+  }
+}
+
+/*
+ * map a range of extents on an object's buffer cache.
+ * - combine any bh's we're writing into one
+ * - break up bufferheads that don't fall completely within the range
+ * //no! - return a bh that includes the write.  may also include other dirty data to left and/or right.
+ */
+ObjectCacher::BufferHead *ObjectCacher::Object::map_write(OSDWrite *wr)
+{
+  assert(oc->lock.is_locked());
+  BufferHead *final = 0;
+
+  for (vector::iterator ex_it = wr->extents.begin();
+       ex_it != wr->extents.end();
+       ++ex_it) {
+
+    if (ex_it->oid != oid.oid) continue;
+
+    ldout(oc->cct, 10) << "map_write oex " << ex_it->oid
+		       << " " << ex_it->offset << "~" << ex_it->length << dendl;
+
+    loff_t cur = ex_it->offset;
+    loff_t left = ex_it->length;
+
+    map::iterator p = data_lower_bound(ex_it->offset);
+    while (left > 0) {
+      loff_t max = left;
+
+      // at end ?
+      if (p == data.end()) {
+        if (final == NULL) {
+          final = new BufferHead(this);
+          final->set_start( cur );
+          final->set_length( max );
+          oc->bh_add(this, final);
+          ldout(oc->cct, 10) << "map_write adding trailing bh " << *final << dendl;
+        } else {
+	  oc->bh_stat_sub(final);
+          final->set_length(final->length() + max);
+	  oc->bh_stat_add(final);
+        }
+        left -= max;
+        cur += max;
+        continue;
+      }
+      
+      ldout(oc->cct, 10) << "cur is " << cur << ", p is " << *p->second << dendl;
+      //oc->verify_stats();
+
+      if (p->first <= cur) {
+        BufferHead *bh = p->second;
+        ldout(oc->cct, 10) << "map_write bh " << *bh << " intersected" << dendl;
+        
+        if (p->first < cur) {
+          assert(final == 0);
+          if (cur + max >= p->first + p->second->length()) {
+            // we want right bit (one splice)
+            final = split(bh, cur);   // just split it, take right half.
+            ++p;
+            assert(p->second == final);
+          } else {
+            // we want middle bit (two splices)
+            final = split(bh, cur);
+            ++p;
+            assert(p->second == final);
+            split(final, cur+max);
+          }
+        } else {
+	  assert(p->first == cur);
+          if (p->second->length() <= max) {
+            // whole bufferhead, piece of cake.
+          } else {
+            // we want left bit (one splice)
+            split(bh, cur + max);        // just split
+          }
+          if (final) {
+	    oc->mark_dirty(bh);
+	    oc->mark_dirty(final);
+	    --p;  // move iterator back to final
+	    assert(p->second == final);
+            merge_left(final, bh);
+	  } else {
+            final = bh;
+	  }
+        }
+        
+        // keep going.
+        loff_t lenfromcur = final->end() - cur;
+        cur += lenfromcur;
+        left -= lenfromcur;
+        ++p;
+        continue; 
+      } else {
+        // gap!
+        loff_t next = p->first;
+        loff_t glen = MIN(next - cur, max);
+        ldout(oc->cct, 10) << "map_write gap " << cur << "~" << glen << dendl;
+        if (final) {
+	  oc->bh_stat_sub(final);
+          final->set_length(final->length() + glen);
+	  oc->bh_stat_add(final);
+        } else {
+          final = new BufferHead(this);
+          final->set_start( cur );
+          final->set_length( glen );
+          oc->bh_add(this, final);
+        }
+        
+        cur += glen;
+        left -= glen;
+        continue;    // more?
+      }
+    }
+  }
+  
+  // set versoin
+  assert(final);
+  ldout(oc->cct, 10) << "map_write final is " << *final << dendl;
+
+  return final;
+}
+
+void ObjectCacher::Object::truncate(loff_t s)
+{
+  assert(oc->lock.is_locked());
+  ldout(oc->cct, 10) << "truncate " << *this << " to " << s << dendl;
+
+  while (!data.empty()) {
+    BufferHead *bh = data.rbegin()->second;
+    if (bh->end() <= s) 
+      break;
+
+    // split bh at truncation point?
+    if (bh->start() < s) {
+      split(bh, s);
+      continue;
+    }
+
+    // remove bh entirely
+    assert(bh->start() >= s);
+    assert(bh->waitfor_read.empty());
+    oc->bh_remove(this, bh);
+    delete bh;
+  }
+}
+
+void ObjectCacher::Object::discard(loff_t off, loff_t len)
+{
+  assert(oc->lock.is_locked());
+  ldout(oc->cct, 10) << "discard " << *this << " " << off << "~" << len << dendl;
+
+  if (!exists) {
+    ldout(oc->cct, 10) << " setting exists on " << *this << dendl;
+    exists = true;
+  }
+  if (complete) {
+    ldout(oc->cct, 10) << " clearing complete on " << *this << dendl;
+    complete = false;
+  }
+
+  map::iterator p = data_lower_bound(off);
+  while (p != data.end()) {
+    BufferHead *bh = p->second;
+    if (bh->start() >= off + len)
+      break;
+
+    // split bh at truncation point?
+    if (bh->start() < off) {
+      split(bh, off);
+      ++p;
+      continue;
+    }
+
+    assert(bh->start() >= off);
+    if (bh->end() > off + len) {
+      split(bh, off + len);
+    }
+
+    ++p;
+    ldout(oc->cct, 10) << "discard " << *this << " bh " << *bh << dendl;
+    assert(bh->waitfor_read.empty());
+    oc->bh_remove(this, bh);
+    delete bh;
+  }
+}
+
+
+
+/*** ObjectCacher ***/
+
+#undef dout_prefix
+#define dout_prefix *_dout << "objectcacher "
+
+
+ObjectCacher::ObjectCacher(CephContext *cct_, string name, WritebackHandler& wb, Mutex& l,
+			   flush_set_callback_t flush_callback,
+			   void *flush_callback_arg,
+			   uint64_t max_bytes, uint64_t max_objects,
+			   uint64_t max_dirty, uint64_t target_dirty,
+			   double max_dirty_age, bool block_writes_upfront)
+  : perfcounter(NULL),
+    cct(cct_), writeback_handler(wb), name(name), lock(l),
+    max_dirty(max_dirty), target_dirty(target_dirty),
+    max_size(max_bytes), max_objects(max_objects),
+    block_writes_upfront(block_writes_upfront),
+    flush_set_callback(flush_callback), flush_set_callback_arg(flush_callback_arg),
+    last_read_tid(0),
+    flusher_stop(false), flusher_thread(this), finisher(cct),
+    stat_clean(0), stat_zero(0), stat_dirty(0), stat_rx(0), stat_tx(0), stat_missing(0),
+    stat_error(0), stat_dirty_waiting(0), reads_outstanding(0)
+{
+  this->max_dirty_age.set_from_double(max_dirty_age);
+  perf_start();
+  finisher.start();
+}
+
+ObjectCacher::~ObjectCacher()
+{
+  finisher.stop();
+  perf_stop();
+  // we should be empty.
+  for (vector >::iterator i = objects.begin();
+      i != objects.end();
+      ++i)
+    assert(i->empty());
+  assert(bh_lru_rest.lru_get_size() == 0);
+  assert(bh_lru_dirty.lru_get_size() == 0);
+  assert(ob_lru.lru_get_size() == 0);
+  assert(dirty_bh.empty());
+}
+
+void ObjectCacher::perf_start()
+{
+  string n = "objectcacher-" + name;
+  PerfCountersBuilder plb(cct, n, l_objectcacher_first, l_objectcacher_last);
+
+  plb.add_u64_counter(l_objectcacher_cache_ops_hit, "cache_ops_hit");
+  plb.add_u64_counter(l_objectcacher_cache_ops_miss, "cache_ops_miss");
+  plb.add_u64_counter(l_objectcacher_cache_bytes_hit, "cache_bytes_hit");
+  plb.add_u64_counter(l_objectcacher_cache_bytes_miss, "cache_bytes_miss");
+  plb.add_u64_counter(l_objectcacher_data_read, "data_read");
+  plb.add_u64_counter(l_objectcacher_data_written, "data_written");
+  plb.add_u64_counter(l_objectcacher_data_flushed, "data_flushed");
+  plb.add_u64_counter(l_objectcacher_overwritten_in_flush,
+                      "data_overwritten_while_flushing");
+  plb.add_u64_counter(l_objectcacher_write_ops_blocked, "write_ops_blocked");
+  plb.add_u64_counter(l_objectcacher_write_bytes_blocked, "write_bytes_blocked");
+  plb.add_time(l_objectcacher_write_time_blocked, "write_time_blocked");
+
+  perfcounter = plb.create_perf_counters();
+  cct->get_perfcounters_collection()->add(perfcounter);
+}
+
+void ObjectCacher::perf_stop()
+{
+  assert(perfcounter);
+  cct->get_perfcounters_collection()->remove(perfcounter);
+  delete perfcounter;
+}
+
+/* private */
+ObjectCacher::Object *ObjectCacher::get_object(sobject_t oid, ObjectSet *oset,
+					       object_locator_t &l,
+					       uint64_t truncate_size,
+					       uint64_t truncate_seq)
+{
+  // XXX: Add handling of nspace in object_locator_t in cache
+  assert(lock.is_locked());
+  // have it?
+  if ((uint32_t)l.pool < objects.size()) {
+    if (objects[l.pool].count(oid)) {
+      Object *o = objects[l.pool][oid];
+      o->truncate_size = truncate_size;
+      o->truncate_seq = truncate_seq;
+      return o;
+    }
+  } else {
+    objects.resize(l.pool+1);
+  }
+
+  // create it.
+  Object *o = new Object(this, oid, oset, l, truncate_size, truncate_seq);
+  objects[l.pool][oid] = o;
+  ob_lru.lru_insert_top(o);
+  return o;
+}
+
+void ObjectCacher::close_object(Object *ob) 
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "close_object " << *ob << dendl;
+  assert(ob->can_close());
+  
+  // ok!
+  ob_lru.lru_remove(ob);
+  objects[ob->oloc.pool].erase(ob->get_soid());
+  ob->set_item.remove_myself();
+  delete ob;
+}
+
+
+
+
+void ObjectCacher::bh_read(BufferHead *bh)
+{
+  assert(lock.is_locked());
+  ldout(cct, 7) << "bh_read on " << *bh << " outstanding reads "
+		<< reads_outstanding << dendl;
+
+  mark_rx(bh);
+  bh->last_read_tid = ++last_read_tid;
+
+  // finisher
+  C_ReadFinish *onfinish = new C_ReadFinish(this, bh->ob, bh->last_read_tid,
+					    bh->start(), bh->length());
+  // go
+  writeback_handler.read(bh->ob->get_oid(), bh->ob->get_oloc(),
+			 bh->start(), bh->length(), bh->ob->get_snap(),
+			 &onfinish->bl, bh->ob->truncate_size, bh->ob->truncate_seq,
+			 onfinish);
+
+  ++reads_outstanding;
+}
+
+void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, ceph_tid_t tid,
+				  loff_t start, uint64_t length,
+				  bufferlist &bl, int r,
+				  bool trust_enoent)
+{
+  assert(lock.is_locked());
+  ldout(cct, 7) << "bh_read_finish " 
+		<< oid
+		<< " tid " << tid
+		<< " " << start << "~" << length
+		<< " (bl is " << bl.length() << ")"
+		<< " returned " << r
+		<< " outstanding reads " << reads_outstanding
+		<< dendl;
+
+  if (bl.length() < length) {
+    bufferptr bp(length - bl.length());
+    bp.zero();
+    ldout(cct, 7) << "bh_read_finish " << oid << " padding " << start << "~" << length 
+	    << " with " << bp.length() << " bytes of zeroes" << dendl;
+    bl.push_back(bp);
+  }
+
+  list ls;
+  int err = 0;
+
+  if (objects[poolid].count(oid) == 0) {
+    ldout(cct, 7) << "bh_read_finish no object cache" << dendl;
+  } else {
+    Object *ob = objects[poolid][oid];
+    
+    if (r == -ENOENT && !ob->complete) {
+      // wake up *all* rx waiters, or else we risk reordering identical reads. e.g.
+      //   read 1~1
+      //   reply to unrelated 3~1 -> !exists
+      //   read 1~1 -> immediate ENOENT
+      //   reply to first 1~1 -> ooo ENOENT
+      bool allzero = true;
+      for (map::iterator p = ob->data.begin(); p != ob->data.end(); ++p) {
+	BufferHead *bh = p->second;
+	for (map >::iterator p = bh->waitfor_read.begin();
+	     p != bh->waitfor_read.end();
+	     ++p)
+	  ls.splice(ls.end(), p->second);
+	bh->waitfor_read.clear();
+	if (!bh->is_zero() && !bh->is_rx())
+	  allzero = false;
+      }
+
+      // just pass through and retry all waiters if we don't trust
+      // -ENOENT for this read
+      if (trust_enoent) {
+	ldout(cct, 7) << "bh_read_finish ENOENT, marking complete and !exists on " << *ob << dendl;
+	ob->complete = true;
+	ob->exists = false;
+
+	/* If all the bhs are effectively zero, get rid of them.  All
+	 * the waiters will be retried and get -ENOENT immediately, so
+	 * it's safe to clean up the unneeded bh's now. Since we know
+	 * it's safe to remove them now, do so, so they aren't hanging
+	 *around waiting for more -ENOENTs from rados while the cache
+	 * is being shut down.
+	 *
+	 * Only do this when all the bhs are rx or clean, to match the
+	 * condition in _readx(). If there are any non-rx or non-clean
+	 * bhs, _readx() will wait for the final result instead of
+	 * returning -ENOENT immediately.
+	 */
+	if (allzero) {
+	  ldout(cct, 10) << "bh_read_finish ENOENT and allzero, getting rid of "
+			 << "bhs for " << *ob << dendl;
+	  map::iterator p = ob->data.begin();
+	  while (p != ob->data.end()) {
+	    BufferHead *bh = p->second;
+	    // current iterator will be invalidated by bh_remove()
+	    ++p;
+	    bh_remove(ob, bh);
+	    delete bh;
+	  }
+	}
+      }
+    }
+
+    // apply to bh's!
+    loff_t opos = start;
+    while (true) {
+      map::iterator p = ob->data_lower_bound(opos);
+      if (p == ob->data.end())
+	break;
+      if (opos >= start+(loff_t)length) {
+	ldout(cct, 20) << "break due to opos " << opos << " >= start+length "
+		       << start << "+" << length << "=" << start+(loff_t)length
+		       << dendl;
+	break;
+      }
+
+      BufferHead *bh = p->second;
+      ldout(cct, 20) << "checking bh " << *bh << dendl;
+
+      // finishers?
+      for (map >::iterator it = bh->waitfor_read.begin();
+           it != bh->waitfor_read.end();
+           ++it)
+	ls.splice(ls.end(), it->second);
+      bh->waitfor_read.clear();
+
+      if (bh->start() > opos) {
+        ldout(cct, 1) << "bh_read_finish skipping gap "
+		      << opos << "~" << bh->start() - opos
+		      << dendl;
+        opos = bh->start();
+        continue;
+      }
+
+      if (!bh->is_rx()) {
+        ldout(cct, 10) << "bh_read_finish skipping non-rx " << *bh << dendl;
+        opos = bh->end();
+        continue;
+      }
+
+      if (bh->last_read_tid != tid) {
+	ldout(cct, 10) << "bh_read_finish bh->last_read_tid " << bh->last_read_tid
+		       << " != tid " << tid << ", skipping" << dendl;
+	opos = bh->end();
+	continue;
+      }
+
+      assert(opos >= bh->start());
+      assert(bh->start() == opos);   // we don't merge rx bh's... yet!
+      assert(bh->length() <= start+(loff_t)length-opos);
+
+      if (bh->error < 0)
+	err = bh->error;
+
+      loff_t oldpos = opos;
+      opos = bh->end();
+
+      if (r == -ENOENT) {
+	if (trust_enoent) {
+	  ldout(cct, 10) << "bh_read_finish removing " << *bh << dendl;
+	  bh_remove(ob, bh);
+	  delete bh;
+	} else {
+	  ldout(cct, 10) << "skipping unstrusted -ENOENT and will retry for "
+			 << *bh << dendl;
+	}
+	continue;
+      }
+
+      if (r < 0) {
+	bh->error = r;
+	mark_error(bh);
+      } else {
+	bh->bl.substr_of(bl,
+			 oldpos-bh->start(),
+			 bh->length());
+	mark_clean(bh);
+      }
+
+      ldout(cct, 10) << "bh_read_finish read " << *bh << dendl;
+
+      ob->try_merge_bh(bh);
+    }
+  }
+
+  // called with lock held.
+  ldout(cct, 20) << "finishing waiters " << ls << dendl;
+
+  finish_contexts(cct, ls, err);
+  --reads_outstanding;
+  read_cond.Signal();
+}
+
+
+void ObjectCacher::bh_write(BufferHead *bh)
+{
+  assert(lock.is_locked());
+  ldout(cct, 7) << "bh_write " << *bh << dendl;
+
+  bh->ob->get();
+
+  // finishers
+  C_WriteCommit *oncommit = new C_WriteCommit(this, bh->ob->oloc.pool,
+                                              bh->ob->get_soid(), bh->start(), bh->length());
+  // go
+  ceph_tid_t tid = writeback_handler.write(bh->ob->get_oid(), bh->ob->get_oloc(),
+				      bh->start(), bh->length(),
+				      bh->snapc, bh->bl, bh->last_write,
+				      bh->ob->truncate_size, bh->ob->truncate_seq,
+				      oncommit);
+  ldout(cct, 20) << " tid " << tid << " on " << bh->ob->get_oid() << dendl;
+
+  // set bh last_write_tid
+  oncommit->tid = tid;
+  bh->ob->last_write_tid = tid;
+  bh->last_write_tid = tid;
+
+  if (perfcounter) {
+    perfcounter->inc(l_objectcacher_data_flushed, bh->length());
+  }
+
+  mark_tx(bh);
+}
+
+void ObjectCacher::bh_write_commit(int64_t poolid, sobject_t oid, loff_t start,
+				   uint64_t length, ceph_tid_t tid, int r)
+{
+  assert(lock.is_locked());
+  ldout(cct, 7) << "bh_write_commit " 
+		<< oid 
+		<< " tid " << tid
+		<< " " << start << "~" << length
+		<< " returned " << r
+		<< dendl;
+
+  if (objects[poolid].count(oid) == 0) {
+    ldout(cct, 7) << "bh_write_commit no object cache" << dendl;
+  } else {
+    Object *ob = objects[poolid][oid];
+    int was_dirty_or_tx = ob->oset->dirty_or_tx;
+    
+    if (!ob->exists) {
+      ldout(cct, 10) << "bh_write_commit marking exists on " << *ob << dendl;
+      ob->exists = true;
+
+      if (writeback_handler.may_copy_on_write(ob->get_oid(), start, length, ob->get_snap())) {
+	ldout(cct, 10) << "bh_write_commit may copy on write, clearing complete on " << *ob << dendl;
+	ob->complete = false;
+      }
+    }
+
+    // apply to bh's!
+    for (map::iterator p = ob->data_lower_bound(start);
+         p != ob->data.end();
+         ++p) {
+      BufferHead *bh = p->second;
+      
+      if (bh->start() > start+(loff_t)length)
+	break;
+
+      if (bh->start() < start &&
+          bh->end() > start+(loff_t)length) {
+        ldout(cct, 20) << "bh_write_commit skipping " << *bh << dendl;
+        continue;
+      }
+      
+      // make sure bh is tx
+      if (!bh->is_tx()) {
+        ldout(cct, 10) << "bh_write_commit skipping non-tx " << *bh << dendl;
+        continue;
+      }
+      
+      // make sure bh tid matches
+      if (bh->last_write_tid != tid) {
+        assert(bh->last_write_tid > tid);
+        ldout(cct, 10) << "bh_write_commit newer tid on " << *bh << dendl;
+        continue;
+      }
+
+      if (r >= 0) {
+	// ok!  mark bh clean and error-free
+	mark_clean(bh);
+	ldout(cct, 10) << "bh_write_commit clean " << *bh << dendl;
+      } else {
+	mark_dirty(bh);
+	ldout(cct, 10) << "bh_write_commit marking dirty again due to error "
+		       << *bh << " r = " << r << " " << cpp_strerror(-r)
+		       << dendl;
+      }
+    }
+
+    // update last_commit.
+    assert(ob->last_commit_tid < tid);
+    ob->last_commit_tid = tid;
+
+    // waiters?
+    list ls;
+    if (ob->waitfor_commit.count(tid)) {
+      ls.splice(ls.begin(), ob->waitfor_commit[tid]);
+      ob->waitfor_commit.erase(tid);
+    }
+
+    // is the entire object set now clean and fully committed?
+    ObjectSet *oset = ob->oset;
+    ob->put();
+
+    if (flush_set_callback &&
+	was_dirty_or_tx > 0 &&
+	oset->dirty_or_tx == 0) {        // nothing dirty/tx
+      flush_set_callback(flush_set_callback_arg, oset);      
+    }
+
+    if (!ls.empty())
+      finish_contexts(cct, ls, r);
+  }
+}
+
+void ObjectCacher::flush(loff_t amount)
+{
+  assert(lock.is_locked());
+  utime_t cutoff = ceph_clock_now(cct);
+
+  ldout(cct, 10) << "flush " << amount << dendl;
+  
+  /*
+   * NOTE: we aren't actually pulling things off the LRU here, just looking at the
+   * tail item.  Then we call bh_write, which moves it to the other LRU, so that we
+   * can call lru_dirty.lru_get_next_expire() again.
+   */
+  loff_t did = 0;
+  while (amount == 0 || did < amount) {
+    BufferHead *bh = static_cast(bh_lru_dirty.lru_get_next_expire());
+    if (!bh) break;
+    if (bh->last_write > cutoff) break;
+
+    did += bh->length();
+    bh_write(bh);
+  }    
+}
+
+
+void ObjectCacher::trim()
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "trim  start: bytes: max " << max_size << "  clean " << get_stat_clean()
+		 << ", objects: max " << max_objects << " current " << ob_lru.lru_get_size()
+		 << dendl;
+
+  while (get_stat_clean() > 0 && (uint64_t) get_stat_clean() > max_size) {
+    BufferHead *bh = static_cast(bh_lru_rest.lru_expire());
+    if (!bh)
+      break;
+
+    ldout(cct, 10) << "trim trimming " << *bh << dendl;
+    assert(bh->is_clean() || bh->is_zero());
+
+    Object *ob = bh->ob;
+    bh_remove(ob, bh);
+    delete bh;
+
+    if (ob->complete) {
+      ldout(cct, 10) << "trim clearing complete on " << *ob << dendl;
+      ob->complete = false;
+    }
+  }
+
+  while (ob_lru.lru_get_size() > max_objects) {
+    Object *ob = static_cast(ob_lru.lru_expire());
+    if (!ob)
+      break;
+
+    ldout(cct, 10) << "trim trimming " << *ob << dendl;
+    close_object(ob);
+  }
+  
+  ldout(cct, 10) << "trim finish:  max " << max_size << "  clean " << get_stat_clean()
+		 << ", objects: max " << max_objects << " current " << ob_lru.lru_get_size()
+		 << dendl;
+}
+
+
+
+/* public */
+
+bool ObjectCacher::is_cached(ObjectSet *oset, vector& extents, snapid_t snapid)
+{
+  assert(lock.is_locked());
+  for (vector::iterator ex_it = extents.begin();
+       ex_it != extents.end();
+       ++ex_it) {
+    ldout(cct, 10) << "is_cached " << *ex_it << dendl;
+
+    // get Object cache
+    sobject_t soid(ex_it->oid, snapid);
+    Object *o = get_object_maybe(soid, ex_it->oloc);
+    if (!o)
+      return false;
+    if (!o->is_cached(ex_it->offset, ex_it->length))
+      return false;
+  }
+  return true;
+}
+
+
+/*
+ * returns # bytes read (if in cache).  onfinish is untouched (caller must delete it)
+ * returns 0 if doing async read
+ */
+int ObjectCacher::readx(OSDRead *rd, ObjectSet *oset, Context *onfinish)
+{
+  return _readx(rd, oset, onfinish, true);
+}
+
+int ObjectCacher::_readx(OSDRead *rd, ObjectSet *oset, Context *onfinish,
+			 bool external_call)
+{
+  assert(lock.is_locked());
+  bool success = true;
+  int error = 0;
+  list hit_ls;
+  uint64_t bytes_in_cache = 0;
+  uint64_t bytes_not_in_cache = 0;
+  uint64_t total_bytes_read = 0;
+  map stripe_map;  // final buffer offset -> substring
+
+  for (vector::iterator ex_it = rd->extents.begin();
+       ex_it != rd->extents.end();
+       ++ex_it) {
+    ldout(cct, 10) << "readx " << *ex_it << dendl;
+
+    total_bytes_read += ex_it->length;
+
+    // get Object cache
+    sobject_t soid(ex_it->oid, rd->snap);
+    Object *o = get_object(soid, oset, ex_it->oloc, ex_it->truncate_size, oset->truncate_seq);
+    touch_ob(o);
+
+    // does not exist and no hits?
+    if (oset->return_enoent && !o->exists) {
+      // WARNING: we can only meaningfully return ENOENT if the read request
+      // passed in a single ObjectExtent.  Any caller who wants ENOENT instead of
+      // zeroed buffers needs to feed single extents into readx().
+      assert(rd->extents.size() == 1);
+      ldout(cct, 10) << "readx  object !exists, 1 extent..." << dendl;
+
+      // should we worry about COW underneaeth us?
+      if (writeback_handler.may_copy_on_write(soid.oid, ex_it->offset, ex_it->length, soid.snap)) {
+	ldout(cct, 20) << "readx  may copy on write" << dendl;
+	bool wait = false;
+	for (map::iterator bh_it = o->data.begin();
+	     bh_it != o->data.end();
+	     ++bh_it) {
+	  BufferHead *bh = bh_it->second;
+	  if (bh->is_dirty() || bh->is_tx()) {
+	    ldout(cct, 10) << "readx  flushing " << *bh << dendl;
+	    wait = true;
+	    if (bh->is_dirty())
+	      bh_write(bh);
+	  }
+	}
+	if (wait) {
+	  ldout(cct, 10) << "readx  waiting on tid " << o->last_write_tid << " on " << *o << dendl;
+	  o->waitfor_commit[o->last_write_tid].push_back(new C_RetryRead(this, rd, oset, onfinish));
+	  // FIXME: perfcounter!
+	  return 0;
+	}
+      }
+
+      // can we return ENOENT?
+      bool allzero = true;
+      for (map::iterator bh_it = o->data.begin();
+	   bh_it != o->data.end();
+	   ++bh_it) {
+	ldout(cct, 20) << "readx  ob has bh " << *bh_it->second << dendl;
+	if (!bh_it->second->is_zero() && !bh_it->second->is_rx()) {
+	  allzero = false;
+	  break;
+	}
+      }
+      if (allzero) {
+	ldout(cct, 10) << "readx  ob has all zero|rx, returning ENOENT" << dendl;
+	delete rd;
+	return -ENOENT;
+      }
+    }
+
+    // map extent into bufferheads
+    map hits, missing, rx, errors;
+    o->map_read(rd, hits, missing, rx, errors);
+    if (external_call) {
+      // retry reading error buffers
+      missing.insert(errors.begin(), errors.end());
+    } else {
+      // some reads had errors, fail later so completions
+      // are cleaned up up properly
+      // TODO: make read path not call _readx for every completion
+      hits.insert(errors.begin(), errors.end());
+    }
+    
+    if (!missing.empty() || !rx.empty()) {
+      // read missing
+      for (map::iterator bh_it = missing.begin();
+           bh_it != missing.end();
+           ++bh_it) {
+        bh_read(bh_it->second);
+        if (success && onfinish) {
+          ldout(cct, 10) << "readx missed, waiting on " << *bh_it->second 
+                   << " off " << bh_it->first << dendl;
+	  bh_it->second->waitfor_read[bh_it->first].push_back( new C_RetryRead(this, rd, oset, onfinish) );
+        }
+        bytes_not_in_cache += bh_it->second->length();
+	success = false;
+      }
+
+      // bump rx
+      for (map::iterator bh_it = rx.begin();
+           bh_it != rx.end();
+           ++bh_it) {
+        touch_bh(bh_it->second);        // bump in lru, so we don't lose it.
+        if (success && onfinish) {
+          ldout(cct, 10) << "readx missed, waiting on " << *bh_it->second 
+                   << " off " << bh_it->first << dendl;
+	  bh_it->second->waitfor_read[bh_it->first].push_back( new C_RetryRead(this, rd, oset, onfinish) );
+        }
+        bytes_not_in_cache += bh_it->second->length();
+	success = false;
+      }      
+    } else {
+      assert(!hits.empty());
+
+      // make a plain list
+      for (map::iterator bh_it = hits.begin();
+           bh_it != hits.end();
+           ++bh_it) {
+	ldout(cct, 10) << "readx hit bh " << *bh_it->second << dendl;
+	if (bh_it->second->is_error() && bh_it->second->error)
+	  error = bh_it->second->error;
+        hit_ls.push_back(bh_it->second);
+        bytes_in_cache += bh_it->second->length();
+      }
+
+      // create reverse map of buffer offset -> object for the eventual result.
+      // this is over a single ObjectExtent, so we know that
+      //  - the bh's are contiguous
+      //  - the buffer frags need not be (and almost certainly aren't)
+      loff_t opos = ex_it->offset;
+      map::iterator bh_it = hits.begin();
+      assert(bh_it->second->start() <= opos);
+      uint64_t bhoff = opos - bh_it->second->start();
+      vector >::iterator f_it = ex_it->buffer_extents.begin();
+      uint64_t foff = 0;
+      while (1) {
+        BufferHead *bh = bh_it->second;
+        assert(opos == (loff_t)(bh->start() + bhoff));
+
+        uint64_t len = MIN(f_it->second - foff, bh->length() - bhoff);
+        ldout(cct, 10) << "readx rmap opos " << opos
+		       << ": " << *bh << " +" << bhoff
+		       << " frag " << f_it->first << "~" << f_it->second << " +" << foff << "~" << len
+		       << dendl;
+
+	bufferlist bit;  // put substr here first, since substr_of clobbers, and
+	                 // we may get multiple bh's at this stripe_map position
+	if (bh->is_zero()) {
+	  bufferptr bp(len);
+	  bp.zero();
+	  stripe_map[f_it->first].push_back(bp);
+	} else {
+	  bit.substr_of(bh->bl,
+			opos - bh->start(),
+			len);
+	  stripe_map[f_it->first].claim_append(bit);
+	}
+
+        opos += len;
+        bhoff += len;
+        foff += len;
+        if (opos == bh->end()) {
+          ++bh_it;
+          bhoff = 0;
+        }
+        if (foff == f_it->second) {
+          ++f_it;
+          foff = 0;
+        }
+        if (bh_it == hits.end()) break;
+        if (f_it == ex_it->buffer_extents.end())
+	  break;
+      }
+      assert(f_it == ex_it->buffer_extents.end());
+      assert(opos == (loff_t)ex_it->offset + (loff_t)ex_it->length);
+    }
+  }
+  
+  // bump hits in lru
+  for (list::iterator bhit = hit_ls.begin();
+       bhit != hit_ls.end();
+       ++bhit) 
+    touch_bh(*bhit);
+  
+  if (!success) {
+    if (perfcounter && external_call) {
+      perfcounter->inc(l_objectcacher_data_read, total_bytes_read);
+      perfcounter->inc(l_objectcacher_cache_bytes_miss, bytes_not_in_cache);
+      perfcounter->inc(l_objectcacher_cache_ops_miss);
+    }
+    if (onfinish) {
+      ldout(cct, 20) << "readx defer " << rd << dendl;
+    } else {
+      ldout(cct, 20) << "readx drop " << rd << " (no complete, but no waiter)" << dendl;
+      delete rd;
+    }
+    return 0;  // wait!
+  }
+  if (perfcounter && external_call) {
+    perfcounter->inc(l_objectcacher_data_read, total_bytes_read);
+    perfcounter->inc(l_objectcacher_cache_bytes_hit, bytes_in_cache);
+    perfcounter->inc(l_objectcacher_cache_ops_hit);
+  }
+
+  // no misses... success!  do the read.
+  assert(!hit_ls.empty());
+  ldout(cct, 10) << "readx has all buffers" << dendl;
+  
+  // ok, assemble into result buffer.
+  uint64_t pos = 0;
+  if (rd->bl && !error) {
+    rd->bl->clear();
+    for (map::iterator i = stripe_map.begin();
+	 i != stripe_map.end();
+	 ++i) {
+      assert(pos == i->first);
+      ldout(cct, 10) << "readx  adding buffer len " << i->second.length() << " at " << pos << dendl;
+      pos += i->second.length();
+      rd->bl->claim_append(i->second);
+      assert(rd->bl->length() == pos);
+    }
+    ldout(cct, 10) << "readx  result is " << rd->bl->length() << dendl;
+  } else {
+    ldout(cct, 10) << "readx  no bufferlist ptr (readahead?), done." << dendl;
+    map::reverse_iterator i = stripe_map.rbegin();
+    pos = i->first + i->second.length();
+  }
+
+  // done with read.
+  int ret = error ? error : pos;
+  ldout(cct, 20) << "readx done " << rd << " " << ret << dendl;
+  assert(pos <= (uint64_t) INT_MAX);
+
+  delete rd;
+
+  trim();
+
+  return ret;
+}
+
+
+int ObjectCacher::writex(OSDWrite *wr, ObjectSet *oset, Mutex& wait_on_lock,
+			 Context *onfreespace)
+{
+  assert(lock.is_locked());
+  utime_t now = ceph_clock_now(cct);
+  uint64_t bytes_written = 0;
+  uint64_t bytes_written_in_flush = 0;
+  
+  for (vector::iterator ex_it = wr->extents.begin();
+       ex_it != wr->extents.end();
+       ++ex_it) {
+    // get object cache
+    sobject_t soid(ex_it->oid, CEPH_NOSNAP);
+    Object *o = get_object(soid, oset, ex_it->oloc, ex_it->truncate_size, oset->truncate_seq);
+
+    // map it all into a single bufferhead.
+    BufferHead *bh = o->map_write(wr);
+    bh->snapc = wr->snapc;
+    
+    bytes_written += bh->length();
+    if (bh->is_tx()) {
+      bytes_written_in_flush += bh->length();
+    }
+
+    // adjust buffer pointers (ie "copy" data into my cache)
+    // this is over a single ObjectExtent, so we know that
+    //  - there is one contiguous bh
+    //  - the buffer frags need not be (and almost certainly aren't)
+    // note: i assume striping is monotonic... no jumps backwards, ever!
+    loff_t opos = ex_it->offset;
+    for (vector >::iterator f_it = ex_it->buffer_extents.begin();
+         f_it != ex_it->buffer_extents.end();
+         ++f_it) {
+      ldout(cct, 10) << "writex writing " << f_it->first << "~" << f_it->second << " into " << *bh << " at " << opos << dendl;
+      uint64_t bhoff = bh->start() - opos;
+      assert(f_it->second <= bh->length() - bhoff);
+
+      // get the frag we're mapping in
+      bufferlist frag; 
+      frag.substr_of(wr->bl, 
+                     f_it->first, f_it->second);
+
+      // keep anything left of bhoff
+      bufferlist newbl;
+      if (bhoff)
+	newbl.substr_of(bh->bl, 0, bhoff);
+      newbl.claim_append(frag);
+      bh->bl.swap(newbl);
+
+      opos += f_it->second;
+    }
+
+    // ok, now bh is dirty.
+    mark_dirty(bh);
+    touch_bh(bh);
+    bh->last_write = now;
+
+    o->try_merge_bh(bh);
+  }
+
+  if (perfcounter) {
+    perfcounter->inc(l_objectcacher_data_written, bytes_written);
+    if (bytes_written_in_flush) {
+      perfcounter->inc(l_objectcacher_overwritten_in_flush,
+                       bytes_written_in_flush);
+    }
+  }
+
+  int r = _wait_for_write(wr, bytes_written, oset, wait_on_lock, onfreespace);
+  delete wr;
+
+  //verify_stats();
+  trim();
+  return r;
+}
+
+void ObjectCacher::C_WaitForWrite::finish(int r)
+{
+  Mutex::Locker l(m_oc->lock);
+  m_oc->maybe_wait_for_writeback(m_len);
+  m_onfinish->complete(r);
+}
+
+void ObjectCacher::maybe_wait_for_writeback(uint64_t len)
+{
+  assert(lock.is_locked());
+  utime_t start = ceph_clock_now(cct);
+  int blocked = 0;
+  // wait for writeback?
+  //  - wait for dirty and tx bytes (relative to the max_dirty threshold)
+  //  - do not wait for bytes other waiters are waiting on.  this means that
+  //    threads do not wait for each other.  this effectively allows the cache
+  //    size to balloon proportional to the data that is in flight.
+  while (get_stat_dirty() + get_stat_tx() > 0 &&
+	 (uint64_t) (get_stat_dirty() + get_stat_tx()) >=
+	 max_dirty + get_stat_dirty_waiting()) {
+    ldout(cct, 10) << __func__ << " waiting for dirty|tx "
+		   << (get_stat_dirty() + get_stat_tx()) << " >= max "
+		   << max_dirty << " + dirty_waiting "
+		   << get_stat_dirty_waiting() << dendl;
+    flusher_cond.Signal();
+    stat_dirty_waiting += len;
+    stat_cond.Wait(lock);
+    stat_dirty_waiting -= len;
+    ++blocked;
+    ldout(cct, 10) << __func__ << " woke up" << dendl;
+  }
+  if (blocked && perfcounter) {
+    perfcounter->inc(l_objectcacher_write_ops_blocked);
+    perfcounter->inc(l_objectcacher_write_bytes_blocked, len);
+    utime_t blocked = ceph_clock_now(cct) - start;
+    perfcounter->tinc(l_objectcacher_write_time_blocked, blocked);
+  }
+}
+
+// blocking wait for write.
+int ObjectCacher::_wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, Mutex& lock, Context *onfreespace)
+{
+  assert(lock.is_locked());
+  int ret = 0;
+
+  if (max_dirty > 0) {
+    if (block_writes_upfront) {
+      maybe_wait_for_writeback(len);
+      if (onfreespace)
+	onfreespace->complete(0);
+    } else {
+      assert(onfreespace);
+      finisher.queue(new C_WaitForWrite(this, len, onfreespace));
+    }
+  } else {
+    // write-thru!  flush what we just wrote.
+    Cond cond;
+    bool done;
+    Context *fin = block_writes_upfront ?
+      new C_Cond(&cond, &done, &ret) : onfreespace;
+    assert(fin);
+    bool flushed = flush_set(oset, wr->extents, fin);
+    assert(!flushed);   // we just dirtied it, and didn't drop our lock!
+    ldout(cct, 10) << "wait_for_write waiting on write-thru of " << len << " bytes" << dendl;
+    if (block_writes_upfront) {
+      while (!done)
+	cond.Wait(lock);
+      ldout(cct, 10) << "wait_for_write woke up, ret " << ret << dendl;
+      if (onfreespace)
+	onfreespace->complete(ret);
+    }
+  }
+
+  // start writeback anyway?
+  if (get_stat_dirty() > 0 && (uint64_t) get_stat_dirty() > target_dirty) {
+    ldout(cct, 10) << "wait_for_write " << get_stat_dirty() << " > target "
+		   << target_dirty << ", nudging flusher" << dendl;
+    flusher_cond.Signal();
+  }
+  return ret;
+}
+
+void ObjectCacher::flusher_entry()
+{
+  ldout(cct, 10) << "flusher start" << dendl;
+  lock.Lock();
+  while (!flusher_stop) {
+    loff_t all = get_stat_tx() + get_stat_rx() + get_stat_clean() + get_stat_dirty();
+    ldout(cct, 11) << "flusher "
+		   << all << " / " << max_size << ":  "
+		   << get_stat_tx() << " tx, "
+		   << get_stat_rx() << " rx, "
+		   << get_stat_clean() << " clean, "
+		   << get_stat_dirty() << " dirty ("
+		   << target_dirty << " target, "
+		   << max_dirty << " max)"
+		   << dendl;
+    loff_t actual = get_stat_dirty() + get_stat_dirty_waiting();
+    if (actual > 0 && (uint64_t) actual > target_dirty) {
+      // flush some dirty pages
+      ldout(cct, 10) << "flusher " 
+		     << get_stat_dirty() << " dirty + " << get_stat_dirty_waiting()
+		     << " dirty_waiting > target "
+		     << target_dirty
+		     << ", flushing some dirty bhs" << dendl;
+      flush(actual - target_dirty);
+    } else {
+      // check tail of lru for old dirty items
+      utime_t cutoff = ceph_clock_now(cct);
+      cutoff -= max_dirty_age;
+      BufferHead *bh = 0;
+      int max = MAX_FLUSH_UNDER_LOCK;
+      while ((bh = static_cast(bh_lru_dirty.lru_get_next_expire())) != 0 &&
+	     bh->last_write < cutoff &&
+	     --max > 0) {
+	ldout(cct, 10) << "flusher flushing aged dirty bh " << *bh << dendl;
+	bh_write(bh);
+      }
+      if (!max) {
+	// back off the lock to avoid starving other threads
+	lock.Unlock();
+	lock.Lock();
+	continue;
+      }
+    }
+    if (flusher_stop)
+      break;
+    flusher_cond.WaitInterval(cct, lock, utime_t(1,0));
+  }
+
+  /* Wait for reads to finish. This is only possible if handling
+   * -ENOENT made some read completions finish before their rados read
+   * came back. If we don't wait for them, and destroy the cache, when
+   * the rados reads do come back their callback will try to access the
+   * no-longer-valid ObjectCacher.
+   */
+  while (reads_outstanding > 0) {
+    ldout(cct, 10) << "Waiting for all reads to complete. Number left: "
+		   << reads_outstanding << dendl;
+    read_cond.Wait(lock);
+  }
+
+  lock.Unlock();
+  ldout(cct, 10) << "flusher finish" << dendl;
+}
+
+
+// -------------------------------------------------
+
+bool ObjectCacher::set_is_empty(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  if (oset->objects.empty())
+    return true;
+
+  for (xlist::iterator p = oset->objects.begin(); !p.end(); ++p)
+    if (!(*p)->is_empty())
+      return false;
+
+  return true;
+}
+
+bool ObjectCacher::set_is_cached(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  if (oset->objects.empty())
+    return false;
+  
+  for (xlist::iterator p = oset->objects.begin();
+       !p.end(); ++p) {
+    Object *ob = *p;
+    for (map::iterator q = ob->data.begin();
+         q != ob->data.end();
+         ++q) {
+      BufferHead *bh = q->second;
+      if (!bh->is_dirty() && !bh->is_tx()) 
+        return true;
+    }
+  }
+
+  return false;
+}
+
+bool ObjectCacher::set_is_dirty_or_committing(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  if (oset->objects.empty())
+    return false;
+  
+  for (xlist::iterator i = oset->objects.begin();
+       !i.end(); ++i) {
+    Object *ob = *i;
+    
+    for (map::iterator p = ob->data.begin();
+         p != ob->data.end();
+         ++p) {
+      BufferHead *bh = p->second;
+      if (bh->is_dirty() || bh->is_tx()) 
+        return true;
+    }
+  }  
+  
+  return false;
+}
+
+
+// purge.  non-blocking.  violently removes dirty buffers from cache.
+void ObjectCacher::purge(Object *ob)
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "purge " << *ob << dendl;
+
+  ob->truncate(0);
+}
+
+
+// flush.  non-blocking.  no callback.
+// true if clean, already flushed.  
+// false if we wrote something.
+// be sloppy about the ranges and flush any buffer it touches
+bool ObjectCacher::flush(Object *ob, loff_t offset, loff_t length)
+{
+  assert(lock.is_locked());
+  bool clean = true;
+  ldout(cct, 10) << "flush " << *ob << " " << offset << "~" << length << dendl;
+  for (map::iterator p = ob->data_lower_bound(offset); p != ob->data.end(); ++p) {
+    BufferHead *bh = p->second;
+    ldout(cct, 20) << "flush  " << *bh << dendl;
+    if (length && bh->start() > offset+length) {
+      break;
+    }
+    if (bh->is_tx()) {
+      clean = false;
+      continue;
+    }
+    if (!bh->is_dirty()) {
+      continue;
+    }
+    bh_write(bh);
+    clean = false;
+  }
+  return clean;
+}
+
+bool ObjectCacher::_flush_set_finish(C_GatherBuilder *gather, Context *onfinish)
+{
+  assert(lock.is_locked());
+  if (gather->has_subs()) {
+    gather->set_finisher(onfinish);
+    gather->activate();
+    return false;
+  }
+
+  ldout(cct, 10) << "flush_set has no dirty|tx bhs" << dendl;
+  onfinish->complete(0);
+  return true;
+}
+
+// flush.  non-blocking, takes callback.
+// returns true if already flushed
+bool ObjectCacher::flush_set(ObjectSet *oset, Context *onfinish)
+{
+  assert(lock.is_locked());
+  assert(onfinish != NULL);
+  if (oset->objects.empty()) {
+    ldout(cct, 10) << "flush_set on " << oset << " dne" << dendl;
+    onfinish->complete(0);
+    return true;
+  }
+
+  ldout(cct, 10) << "flush_set " << oset << dendl;
+
+  // we'll need to wait for all objects to flush!
+  C_GatherBuilder gather(cct);
+
+  for (xlist::iterator i = oset->objects.begin();
+       !i.end(); ++i) {
+    Object *ob = *i;
+
+    if (ob->dirty_or_tx == 0)
+      continue;
+
+    if (!flush(ob, 0, 0)) {
+      // we'll need to gather...
+      ldout(cct, 10) << "flush_set " << oset << " will wait for ack tid " 
+               << ob->last_write_tid 
+               << " on " << *ob
+               << dendl;
+      ob->waitfor_commit[ob->last_write_tid].push_back(gather.new_sub());
+    }
+  }
+
+  return _flush_set_finish(&gather, onfinish);
+}
+
+// flush.  non-blocking, takes callback.
+// returns true if already flushed
+bool ObjectCacher::flush_set(ObjectSet *oset, vector& exv, Context *onfinish)
+{
+  assert(lock.is_locked());
+  assert(onfinish != NULL);
+  if (oset->objects.empty()) {
+    ldout(cct, 10) << "flush_set on " << oset << " dne" << dendl;
+    onfinish->complete(0);
+    return true;
+  }
+
+  ldout(cct, 10) << "flush_set " << oset << " on " << exv.size()
+		 << " ObjectExtents" << dendl;
+
+  // we'll need to wait for all objects to flush!
+  C_GatherBuilder gather(cct);
+
+  for (vector::iterator p = exv.begin();
+       p != exv.end();
+       ++p) {
+    ObjectExtent &ex = *p;
+    sobject_t soid(ex.oid, CEPH_NOSNAP);
+    if (objects[oset->poolid].count(soid) == 0)
+      continue;
+    Object *ob = objects[oset->poolid][soid];
+
+    ldout(cct, 20) << "flush_set " << oset << " ex " << ex << " ob " << soid << " " << ob << dendl;
+
+    if (!flush(ob, ex.offset, ex.length)) {
+      // we'll need to gather...
+      ldout(cct, 10) << "flush_set " << oset << " will wait for ack tid " 
+		     << ob->last_write_tid << " on " << *ob << dendl;
+      ob->waitfor_commit[ob->last_write_tid].push_back(gather.new_sub());
+    }
+  }
+
+  return _flush_set_finish(&gather, onfinish);
+}
+
+void ObjectCacher::purge_set(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  if (oset->objects.empty()) {
+    ldout(cct, 10) << "purge_set on " << oset << " dne" << dendl;
+    return;
+  }
+
+  ldout(cct, 10) << "purge_set " << oset << dendl;
+
+  for (xlist::iterator i = oset->objects.begin();
+       !i.end(); ++i) {
+    Object *ob = *i;
+	purge(ob);
+  }
+}
+
+
+loff_t ObjectCacher::release(Object *ob)
+{
+  assert(lock.is_locked());
+  list clean;
+  loff_t o_unclean = 0;
+
+  for (map::iterator p = ob->data.begin();
+       p != ob->data.end();
+       ++p) {
+    BufferHead *bh = p->second;
+    if (bh->is_clean() || bh->is_zero())
+      clean.push_back(bh);
+    else 
+      o_unclean += bh->length();
+  }
+
+  for (list::iterator p = clean.begin();
+       p != clean.end();
+       ++p) {
+    bh_remove(ob, *p);
+    delete *p;
+  }
+
+  if (ob->can_close()) {
+    ldout(cct, 10) << "release trimming " << *ob << dendl;
+    close_object(ob);
+    assert(o_unclean == 0);
+    return 0;
+  }
+
+  if (ob->complete) {
+    ldout(cct, 10) << "release clearing complete on " << *ob << dendl;
+    ob->complete = false;
+  }
+  if (!ob->exists) {
+    ldout(cct, 10) << "release setting exists on " << *ob << dendl;
+    ob->exists = true;
+  }
+
+  return o_unclean;
+}
+
+loff_t ObjectCacher::release_set(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  // return # bytes not clean (and thus not released).
+  loff_t unclean = 0;
+
+  if (oset->objects.empty()) {
+    ldout(cct, 10) << "release_set on " << oset << " dne" << dendl;
+    return 0;
+  }
+
+  ldout(cct, 10) << "release_set " << oset << dendl;
+
+  xlist::iterator q;
+  for (xlist::iterator p = oset->objects.begin();
+       !p.end(); ) {
+    q = p;
+    ++q;
+    Object *ob = *p;
+
+    loff_t o_unclean = release(ob);
+    unclean += o_unclean;
+
+    if (o_unclean) 
+      ldout(cct, 10) << "release_set " << oset << " " << *ob 
+               << " has " << o_unclean << " bytes left"
+               << dendl;
+    p = q;
+  }
+
+  if (unclean) {
+    ldout(cct, 10) << "release_set " << oset
+             << ", " << unclean << " bytes left" << dendl;
+  }
+
+  return unclean;
+}
+
+
+uint64_t ObjectCacher::release_all()
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "release_all" << dendl;
+  uint64_t unclean = 0;
+  
+  vector >::iterator i = objects.begin();
+  while (i != objects.end()) {
+    ceph::unordered_map::iterator p = i->begin();
+    while (p != i->end()) {
+      ceph::unordered_map::iterator n = p;
+      ++n;
+
+      Object *ob = p->second;
+
+      loff_t o_unclean = release(ob);
+      unclean += o_unclean;
+
+      if (o_unclean)
+        ldout(cct, 10) << "release_all " << *ob
+        << " has " << o_unclean << " bytes left"
+        << dendl;
+    p = n;
+    }
+    ++i;
+  }
+
+  if (unclean) {
+    ldout(cct, 10) << "release_all unclean " << unclean << " bytes left" << dendl;
+  }
+
+  return unclean;
+}
+
+void ObjectCacher::clear_nonexistence(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "clear_nonexistence() " << oset << dendl;
+
+  for (xlist::iterator p = oset->objects.begin();
+       !p.end(); ++p) {
+    Object *ob = *p;
+    if (!ob->exists) {
+      ldout(cct, 10) << " setting exists and complete on " << *ob << dendl;
+      ob->exists = true;
+      ob->complete = false;
+    }
+    for (xlist::iterator q = ob->reads.begin();
+	 !q.end(); ++q) {
+      C_ReadFinish *comp = *q;
+      comp->distrust_enoent();
+    }
+  }
+}
+
+/**
+ * discard object extents from an ObjectSet by removing the objects in exls from the in-memory oset.
+ */
+void ObjectCacher::discard_set(ObjectSet *oset, vector& exls)
+{
+  assert(lock.is_locked());
+  if (oset->objects.empty()) {
+    ldout(cct, 10) << "discard_set on " << oset << " dne" << dendl;
+    return;
+  }
+  
+  ldout(cct, 10) << "discard_set " << oset << dendl;
+
+  bool were_dirty = oset->dirty_or_tx > 0;
+
+  for (vector::iterator p = exls.begin();
+       p != exls.end();
+       ++p) {
+    ldout(cct, 10) << "discard_set " << oset << " ex " << *p << dendl;
+    ObjectExtent &ex = *p;
+    sobject_t soid(ex.oid, CEPH_NOSNAP);
+    if (objects[oset->poolid].count(soid) == 0)
+      continue;
+    Object *ob = objects[oset->poolid][soid];
+    
+    ob->discard(ex.offset, ex.length);
+  }
+
+  // did we truncate off dirty data?
+  if (flush_set_callback &&
+      were_dirty && oset->dirty_or_tx == 0)
+    flush_set_callback(flush_set_callback_arg, oset);
+}
+
+void ObjectCacher::verify_stats() const
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "verify_stats" << dendl;
+
+  loff_t clean = 0, zero = 0, dirty = 0, rx = 0, tx = 0, missing = 0, error = 0;
+  for (vector >::const_iterator i = objects.begin();
+      i != objects.end();
+      ++i) {
+    for (ceph::unordered_map::const_iterator p = i->begin();
+        p != i->end();
+        ++p) {
+      Object *ob = p->second;
+      for (map::const_iterator q = ob->data.begin();
+          q != ob->data.end();
+          ++q) {
+        BufferHead *bh = q->second;
+        switch (bh->get_state()) {
+        case BufferHead::STATE_MISSING:
+          missing += bh->length();
+          break;
+        case BufferHead::STATE_CLEAN:
+          clean += bh->length();
+          break;
+        case BufferHead::STATE_ZERO:
+          zero += bh->length();
+          break;
+        case BufferHead::STATE_DIRTY:
+          dirty += bh->length();
+          break;
+        case BufferHead::STATE_TX:
+          tx += bh->length();
+          break;
+        case BufferHead::STATE_RX:
+          rx += bh->length();
+          break;
+	case BufferHead::STATE_ERROR:
+	  error += bh->length();
+	  break;
+        default:
+          assert(0);
+        }
+      }
+    }
+  }
+
+  ldout(cct, 10) << " clean " << clean
+	   << " rx " << rx 
+	   << " tx " << tx
+	   << " dirty " << dirty
+	   << " missing " << missing
+	   << " error " << error
+	   << dendl;
+  assert(clean == stat_clean);
+  assert(rx == stat_rx);
+  assert(tx == stat_tx);
+  assert(dirty == stat_dirty);
+  assert(missing == stat_missing);
+  assert(zero == stat_zero);
+  assert(error == stat_error);
+}
+
+void ObjectCacher::bh_stat_add(BufferHead *bh)
+{
+  assert(lock.is_locked());
+  switch (bh->get_state()) {
+  case BufferHead::STATE_MISSING:
+    stat_missing += bh->length();
+    break;
+  case BufferHead::STATE_CLEAN:
+    stat_clean += bh->length();
+    break;
+  case BufferHead::STATE_ZERO:
+    stat_zero += bh->length();
+    break;
+  case BufferHead::STATE_DIRTY:
+    stat_dirty += bh->length();
+    bh->ob->dirty_or_tx += bh->length();
+    bh->ob->oset->dirty_or_tx += bh->length();
+    break;
+  case BufferHead::STATE_TX:
+    stat_tx += bh->length();
+    bh->ob->dirty_or_tx += bh->length();
+    bh->ob->oset->dirty_or_tx += bh->length();
+    break;
+  case BufferHead::STATE_RX:
+    stat_rx += bh->length();
+    break;
+  case BufferHead::STATE_ERROR:
+    stat_error += bh->length();
+    break;
+  default:
+    assert(0 == "bh_stat_add: invalid bufferhead state");
+  }
+  if (get_stat_dirty_waiting() > 0)
+    stat_cond.Signal();
+}
+
+void ObjectCacher::bh_stat_sub(BufferHead *bh)
+{
+  assert(lock.is_locked());
+  switch (bh->get_state()) {
+  case BufferHead::STATE_MISSING:
+    stat_missing -= bh->length();
+    break;
+  case BufferHead::STATE_CLEAN:
+    stat_clean -= bh->length();
+    break;
+  case BufferHead::STATE_ZERO:
+    stat_zero -= bh->length();
+    break;
+  case BufferHead::STATE_DIRTY:
+    stat_dirty -= bh->length();
+    bh->ob->dirty_or_tx -= bh->length();
+    bh->ob->oset->dirty_or_tx -= bh->length();
+    break;
+  case BufferHead::STATE_TX:
+    stat_tx -= bh->length();
+    bh->ob->dirty_or_tx -= bh->length();
+    bh->ob->oset->dirty_or_tx -= bh->length();
+    break;
+  case BufferHead::STATE_RX:
+    stat_rx -= bh->length();
+    break;
+  case BufferHead::STATE_ERROR:
+    stat_error -= bh->length();
+    break;
+  default:
+    assert(0 == "bh_stat_sub: invalid bufferhead state");
+  }
+}
+
+void ObjectCacher::bh_set_state(BufferHead *bh, int s)
+{
+  assert(lock.is_locked());
+  // move between lru lists?
+  if (s == BufferHead::STATE_DIRTY && bh->get_state() != BufferHead::STATE_DIRTY) {
+    bh_lru_rest.lru_remove(bh);
+    bh_lru_dirty.lru_insert_top(bh);
+    dirty_bh.insert(bh);
+  }
+  if (s != BufferHead::STATE_DIRTY && bh->get_state() == BufferHead::STATE_DIRTY) {
+    bh_lru_dirty.lru_remove(bh);
+    bh_lru_rest.lru_insert_top(bh);
+    dirty_bh.erase(bh);
+  }
+  if (s != BufferHead::STATE_ERROR && bh->get_state() == BufferHead::STATE_ERROR) {
+    bh->error = 0;
+  }
+
+  // set state
+  bh_stat_sub(bh);
+  bh->set_state(s);
+  bh_stat_add(bh);
+}
+
+void ObjectCacher::bh_add(Object *ob, BufferHead *bh)
+{
+  assert(lock.is_locked());
+  ldout(cct, 30) << "bh_add " << *ob << " " << *bh << dendl;
+  ob->add_bh(bh);
+  if (bh->is_dirty()) {
+    bh_lru_dirty.lru_insert_top(bh);
+    dirty_bh.insert(bh);
+  } else {
+    bh_lru_rest.lru_insert_top(bh);
+  }
+  bh_stat_add(bh);
+}
+
+void ObjectCacher::bh_remove(Object *ob, BufferHead *bh)
+{
+  assert(lock.is_locked());
+  ldout(cct, 30) << "bh_remove " << *ob << " " << *bh << dendl;
+  ob->remove_bh(bh);
+  if (bh->is_dirty()) {
+    bh_lru_dirty.lru_remove(bh);
+    dirty_bh.erase(bh);
+  } else {
+    bh_lru_rest.lru_remove(bh);
+  }
+  bh_stat_sub(bh);
+}
+
diff --git a/ceph/src/osdc/ObjectCacher.h b/ceph/src/osdc/ObjectCacher.h
new file mode 100644
index 00000000..d2aebe98
--- /dev/null
+++ b/ceph/src/osdc/ObjectCacher.h
@@ -0,0 +1,728 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_OBJECTCACHER_H
+#define CEPH_OBJECTCACHER_H
+
+#include "include/types.h"
+#include "include/lru.h"
+#include "include/Context.h"
+#include "include/xlist.h"
+
+#include "common/Cond.h"
+#include "common/Finisher.h"
+#include "common/Thread.h"
+
+#include "Objecter.h"
+#include "Striper.h"
+
+class CephContext;
+class WritebackHandler;
+class PerfCounters;
+
+enum {
+  l_objectcacher_first = 25000,
+
+  l_objectcacher_cache_ops_hit, // ops we satisfy completely from cache
+  l_objectcacher_cache_ops_miss, // ops we don't satisfy completely from cache
+
+  l_objectcacher_cache_bytes_hit, // bytes read directly from cache
+  l_objectcacher_cache_bytes_miss, // bytes we couldn't read directly from cache
+
+  l_objectcacher_data_read, // total bytes read out
+  l_objectcacher_data_written, // bytes written to cache
+  l_objectcacher_data_flushed, // bytes flushed to WritebackHandler
+  l_objectcacher_overwritten_in_flush, // bytes overwritten while flushing is in progress
+
+  l_objectcacher_write_ops_blocked, // total write ops we delayed due to dirty limits
+  l_objectcacher_write_bytes_blocked, // total number of write bytes we delayed due to dirty limits
+  l_objectcacher_write_time_blocked, // total time in seconds spent blocking a write due to dirty limits
+
+  l_objectcacher_last,
+};
+
+class ObjectCacher {
+  PerfCounters *perfcounter;
+ public:
+  CephContext *cct;
+  class Object;
+  struct ObjectSet;
+  class C_ReadFinish;
+
+  typedef void (*flush_set_callback_t) (void *p, ObjectSet *oset);
+
+  // read scatter/gather  
+  struct OSDRead {
+    vector extents;
+    snapid_t snap;
+    map read_data;  // bits of data as they come back
+    bufferlist *bl;
+    int flags;
+    OSDRead(snapid_t s, bufferlist *b, int f) : snap(s), bl(b), flags(f) {}
+  };
+
+  OSDRead *prepare_read(snapid_t snap, bufferlist *b, int f) {
+    return new OSDRead(snap, b, f);
+  }
+  
+  // write scatter/gather  
+  struct OSDWrite {
+    vector extents;
+    SnapContext snapc;
+    bufferlist bl;
+    utime_t mtime;
+    int flags;
+    OSDWrite(const SnapContext& sc, bufferlist& b, utime_t mt, int f) : snapc(sc), bl(b), mtime(mt), flags(f) {}
+  };
+
+  OSDWrite *prepare_write(const SnapContext& sc, bufferlist &b, utime_t mt, int f) { 
+    return new OSDWrite(sc, b, mt, f); 
+  }
+
+
+
+  // ******* BufferHead *********
+  class BufferHead : public LRUObject {
+  public:
+    // states
+    static const int STATE_MISSING = 0;
+    static const int STATE_CLEAN = 1;
+    static const int STATE_ZERO = 2;   // NOTE: these are *clean* zeros
+    static const int STATE_DIRTY = 3;
+    static const int STATE_RX = 4;
+    static const int STATE_TX = 5;
+    static const int STATE_ERROR = 6; // a read error occurred
+
+  private:
+    // my fields
+    int state;
+    int ref;
+    struct {
+      loff_t start, length;   // bh extent in object
+    } ex;
+        
+  public:
+    Object *ob;
+    bufferlist  bl;
+    ceph_tid_t last_write_tid;  // version of bh (if non-zero)
+    ceph_tid_t last_read_tid;   // tid of last read op (if any)
+    utime_t last_write;
+    SnapContext snapc;
+    int error; // holds return value for failed reads
+    
+    map< loff_t, list > waitfor_read;
+    
+    // cons
+    BufferHead(Object *o) : 
+      state(STATE_MISSING),
+      ref(0),
+      ob(o),
+      last_write_tid(0),
+      last_read_tid(0),
+      error(0) {
+      ex.start = ex.length = 0;
+    }
+  
+    // extent
+    loff_t start() const { return ex.start; }
+    void set_start(loff_t s) { ex.start = s; }
+    loff_t length() const { return ex.length; }
+    void set_length(loff_t l) { ex.length = l; }
+    loff_t end() const { return ex.start + ex.length; }
+    loff_t last() const { return end() - 1; }
+
+    // states
+    void set_state(int s) {
+      if (s == STATE_RX || s == STATE_TX) get();
+      if (state == STATE_RX || state == STATE_TX) put();
+      state = s;
+    }
+    int get_state() const { return state; }
+    
+    bool is_missing() { return state == STATE_MISSING; }
+    bool is_dirty() { return state == STATE_DIRTY; }
+    bool is_clean() { return state == STATE_CLEAN; }
+    bool is_zero() { return state == STATE_ZERO; }
+    bool is_tx() { return state == STATE_TX; }
+    bool is_rx() { return state == STATE_RX; }
+    bool is_error() { return state == STATE_ERROR; }
+    
+    // reference counting
+    int get() {
+      assert(ref >= 0);
+      if (ref == 0) lru_pin();
+      return ++ref;
+    }
+    int put() {
+      assert(ref > 0);
+      if (ref == 1) lru_unpin();
+      --ref;
+      return ref;
+    }
+  };
+
+  // ******* Object *********
+  class Object : public LRUObject {
+  private:
+    // ObjectCacher::Object fields
+    int ref;
+    ObjectCacher *oc;
+    sobject_t oid;
+    friend struct ObjectSet;
+
+  public:
+    ObjectSet *oset;
+    xlist::item set_item;
+    object_locator_t oloc;
+    uint64_t truncate_size, truncate_seq;
+    
+    bool complete;
+    bool exists;
+
+  public:
+    map     data;
+
+    ceph_tid_t last_write_tid;  // version of bh (if non-zero)
+    ceph_tid_t last_commit_tid; // last update commited.
+
+    int dirty_or_tx;
+
+    map< ceph_tid_t, list > waitfor_commit;
+    xlist reads;
+
+  public:
+    Object(const Object& other);
+    const Object& operator=(const Object& other);
+
+    Object(ObjectCacher *_oc, sobject_t o, ObjectSet *os, object_locator_t& l,
+	   uint64_t ts, uint64_t tq) :
+      ref(0),
+      oc(_oc),
+      oid(o), oset(os), set_item(this), oloc(l),
+      truncate_size(ts), truncate_seq(tq),
+      complete(false), exists(true),
+      last_write_tid(0), last_commit_tid(0),
+      dirty_or_tx(0) {
+      // add to set
+      os->objects.push_back(&set_item);
+    }
+    ~Object() {
+      reads.clear();
+      assert(ref == 0);
+      assert(data.empty());
+      assert(dirty_or_tx == 0);
+      set_item.remove_myself();
+    }
+
+    sobject_t get_soid() { return oid; }
+    object_t get_oid() { return oid.oid; }
+    snapid_t get_snap() { return oid.snap; }
+    ObjectSet *get_object_set() { return oset; }
+    string get_namespace() { return oloc.nspace; }
+    
+    object_locator_t& get_oloc() { return oloc; }
+    void set_object_locator(object_locator_t& l) { oloc = l; }
+
+    bool can_close() {
+      if (lru_is_expireable()) {
+	assert(data.empty());
+	assert(waitfor_commit.empty());
+	return true;
+      }
+      return false;
+    }
+
+    /**
+     * Check buffers and waiters for consistency
+     * - no overlapping buffers
+     * - index in map matches BH
+     * - waiters fall within BH
+     */
+    void audit_buffers();
+
+    /**
+     * find first buffer that includes or follows an offset
+     *
+     * @param offset object byte offset
+     * @return iterator pointing to buffer, or data.end()
+     */
+    map::iterator data_lower_bound(loff_t offset) {
+      map::iterator p = data.lower_bound(offset);
+      if (p != data.begin() &&
+	  (p == data.end() || p->first > offset)) {
+	--p;     // might overlap!
+	if (p->first + p->second->length() <= offset)
+	  ++p;   // doesn't overlap.
+      }
+      return p;
+    }
+
+    // bh
+    // add to my map
+    void add_bh(BufferHead *bh) {
+      if (data.empty())
+	get();
+      assert(data.count(bh->start()) == 0);
+      data[bh->start()] = bh;
+    }
+    void remove_bh(BufferHead *bh) {
+      assert(data.count(bh->start()));
+      data.erase(bh->start());
+      if (data.empty())
+	put();
+    }
+
+    bool is_empty() { return data.empty(); }
+
+    // mid-level
+    BufferHead *split(BufferHead *bh, loff_t off);
+    void merge_left(BufferHead *left, BufferHead *right);
+    void try_merge_bh(BufferHead *bh);
+
+    bool is_cached(loff_t off, loff_t len);
+    int map_read(OSDRead *rd,
+                 map& hits,
+                 map& missing,
+                 map& rx,
+		 map& errors);
+    BufferHead *map_write(OSDWrite *wr);
+    
+    void truncate(loff_t s);
+    void discard(loff_t off, loff_t len);
+
+    // reference counting
+    int get() {
+      assert(ref >= 0);
+      if (ref == 0) lru_pin();
+      return ++ref;
+    }
+    int put() {
+      assert(ref > 0);
+      if (ref == 1) lru_unpin();
+      --ref;
+      return ref;
+    }
+  };
+  
+
+  struct ObjectSet {
+    void *parent;
+
+    inodeno_t ino;
+    uint64_t truncate_seq, truncate_size;
+
+    int64_t poolid;
+    xlist objects;
+
+    int dirty_or_tx;
+    bool return_enoent;
+
+    ObjectSet(void *p, int64_t _poolid, inodeno_t i)
+      : parent(p), ino(i), truncate_seq(0),
+	truncate_size(0), poolid(_poolid), dirty_or_tx(0),
+	return_enoent(false) {}
+
+  };
+
+
+  // ******* ObjectCacher *********
+  // ObjectCacher fields
+ private:
+  WritebackHandler& writeback_handler;
+
+  string name;
+  Mutex& lock;
+  
+  uint64_t max_dirty, target_dirty, max_size, max_objects;
+  utime_t max_dirty_age;
+  bool block_writes_upfront;
+
+  flush_set_callback_t flush_set_callback;
+  void *flush_set_callback_arg;
+
+  vector > objects; // indexed by pool_id
+
+  ceph_tid_t last_read_tid;
+
+  set    dirty_bh;
+  LRU   bh_lru_dirty, bh_lru_rest;
+  LRU   ob_lru;
+
+  Cond flusher_cond;
+  bool flusher_stop;
+  void flusher_entry();
+  class FlusherThread : public Thread {
+    ObjectCacher *oc;
+  public:
+    FlusherThread(ObjectCacher *o) : oc(o) {}
+    void *entry() {
+      oc->flusher_entry();
+      return 0;
+    }
+  } flusher_thread;
+
+  Finisher finisher;
+
+  // objects
+  Object *get_object_maybe(sobject_t oid, object_locator_t &l) {
+    // have it?
+    if (((uint32_t)l.pool < objects.size()) &&
+        (objects[l.pool].count(oid)))
+      return objects[l.pool][oid];
+    return NULL;
+  }
+
+  Object *get_object(sobject_t oid, ObjectSet *oset, object_locator_t &l,
+		     uint64_t truncate_size, uint64_t truncate_seq);
+  void close_object(Object *ob);
+
+  // bh stats
+  Cond  stat_cond;
+
+  loff_t stat_clean;
+  loff_t stat_zero;
+  loff_t stat_dirty;
+  loff_t stat_rx;
+  loff_t stat_tx;
+  loff_t stat_missing;
+  loff_t stat_error;
+  loff_t stat_dirty_waiting;   // bytes that writers are waiting on to write
+
+  void verify_stats() const;
+
+  void bh_stat_add(BufferHead *bh);
+  void bh_stat_sub(BufferHead *bh);
+  loff_t get_stat_tx() { return stat_tx; }
+  loff_t get_stat_rx() { return stat_rx; }
+  loff_t get_stat_dirty() { return stat_dirty; }
+  loff_t get_stat_dirty_waiting() { return stat_dirty_waiting; }
+  loff_t get_stat_clean() { return stat_clean; }
+  loff_t get_stat_zero() { return stat_zero; }
+
+  void touch_bh(BufferHead *bh) {
+    if (bh->is_dirty())
+      bh_lru_dirty.lru_touch(bh);
+    else
+      bh_lru_rest.lru_touch(bh);
+    touch_ob(bh->ob);
+  }
+  void touch_ob(Object *ob) {
+    ob_lru.lru_touch(ob);
+  }
+
+  // bh states
+  void bh_set_state(BufferHead *bh, int s);
+  void copy_bh_state(BufferHead *bh1, BufferHead *bh2) { 
+    bh_set_state(bh2, bh1->get_state());
+  }
+  
+  void mark_missing(BufferHead *bh) { bh_set_state(bh, BufferHead::STATE_MISSING); };
+  void mark_clean(BufferHead *bh) { bh_set_state(bh, BufferHead::STATE_CLEAN); };
+  void mark_zero(BufferHead *bh) { bh_set_state(bh, BufferHead::STATE_ZERO); };
+  void mark_rx(BufferHead *bh) { bh_set_state(bh, BufferHead::STATE_RX); };
+  void mark_tx(BufferHead *bh) { bh_set_state(bh, BufferHead::STATE_TX); };
+  void mark_error(BufferHead *bh) { bh_set_state(bh, BufferHead::STATE_ERROR); };
+  void mark_dirty(BufferHead *bh) { 
+    bh_set_state(bh, BufferHead::STATE_DIRTY); 
+    bh_lru_dirty.lru_touch(bh);
+    //bh->set_dirty_stamp(ceph_clock_now(g_ceph_context));
+  };
+
+  void bh_add(Object *ob, BufferHead *bh);
+  void bh_remove(Object *ob, BufferHead *bh);
+
+  // io
+  void bh_read(BufferHead *bh);
+  void bh_write(BufferHead *bh);
+
+  void trim();
+  void flush(loff_t amount=0);
+
+  /**
+   * flush a range of buffers
+   *
+   * Flush any buffers that intersect the specified extent.  If len==0,
+   * flush *all* buffers for the object.
+   *
+   * @param o object
+   * @param off start offset
+   * @param len extent length, or 0 for entire object
+   * @return true if object was already clean/flushed.
+   */
+  bool flush(Object *o, loff_t off, loff_t len);
+  loff_t release(Object *o);
+  void purge(Object *o);
+
+  int64_t reads_outstanding;
+  Cond read_cond;
+
+  int _readx(OSDRead *rd, ObjectSet *oset, Context *onfinish,
+	     bool external_call);
+
+ public:
+  void bh_read_finish(int64_t poolid, sobject_t oid, ceph_tid_t tid,
+		      loff_t offset, uint64_t length,
+		      bufferlist &bl, int r,
+		      bool trust_enoent);
+  void bh_write_commit(int64_t poolid, sobject_t oid, loff_t offset,
+		       uint64_t length, ceph_tid_t t, int r);
+
+  class C_ReadFinish : public Context {
+    ObjectCacher *oc;
+    int64_t poolid;
+    sobject_t oid;
+    loff_t start;
+    uint64_t length;
+    xlist::item set_item;
+    bool trust_enoent;
+    ceph_tid_t tid;
+
+  public:
+    bufferlist bl;
+    C_ReadFinish(ObjectCacher *c, Object *ob, ceph_tid_t t, loff_t s, uint64_t l) :
+      oc(c), poolid(ob->oloc.pool), oid(ob->get_soid()), start(s), length(l),
+      set_item(this), trust_enoent(true),
+      tid(t) {
+      ob->reads.push_back(&set_item);
+    }
+
+    void finish(int r) {
+      oc->bh_read_finish(poolid, oid, tid, start, length, bl, r, trust_enoent);
+
+      // object destructor clears the list
+      if (set_item.is_on_list())
+	set_item.remove_myself();
+    }
+
+    void distrust_enoent() {
+      trust_enoent = false;
+    }
+  };
+
+  class C_WriteCommit : public Context {
+    ObjectCacher *oc;
+    int64_t poolid;
+    sobject_t oid;
+    loff_t start;
+    uint64_t length;
+  public:
+    ceph_tid_t tid;
+    C_WriteCommit(ObjectCacher *c, int64_t _poolid, sobject_t o, loff_t s, uint64_t l) :
+      oc(c), poolid(_poolid), oid(o), start(s), length(l), tid(0) {}
+    void finish(int r) {
+      oc->bh_write_commit(poolid, oid, start, length, tid, r);
+    }
+  };
+
+  class C_WaitForWrite : public Context {
+  public:
+    C_WaitForWrite(ObjectCacher *oc, uint64_t len, Context *onfinish) :
+      m_oc(oc), m_len(len), m_onfinish(onfinish) {}
+    void finish(int r);
+  private:
+    ObjectCacher *m_oc;
+    uint64_t m_len;
+    Context *m_onfinish;
+  };
+
+  void perf_start();
+  void perf_stop();
+
+
+
+  ObjectCacher(CephContext *cct_, string name, WritebackHandler& wb, Mutex& l,
+	       flush_set_callback_t flush_callback,
+	       void *flush_callback_arg,
+	       uint64_t max_bytes, uint64_t max_objects,
+	       uint64_t max_dirty, uint64_t target_dirty, double max_age,
+	       bool block_writes_upfront);
+  ~ObjectCacher();
+
+  void start() {
+    flusher_thread.create();
+  }
+  void stop() {
+    assert(flusher_thread.is_started());
+    lock.Lock();  // hmm.. watch out for deadlock!
+    flusher_stop = true;
+    flusher_cond.Signal();
+    lock.Unlock();
+    flusher_thread.join();
+  }
+
+
+  class C_RetryRead : public Context {
+    ObjectCacher *oc;
+    OSDRead *rd;
+    ObjectSet *oset;
+    Context *onfinish;
+  public:
+    C_RetryRead(ObjectCacher *_oc, OSDRead *r, ObjectSet *os, Context *c) : oc(_oc), rd(r), oset(os), onfinish(c) {}
+    void finish(int r) {
+      if (r < 0) {
+	if (onfinish)
+	  onfinish->complete(r);
+	return;
+      }
+      int ret = oc->_readx(rd, oset, onfinish, false);
+      if (ret != 0 && onfinish) {
+        onfinish->complete(ret);
+      }
+    }
+  };
+
+
+
+  // non-blocking.  async.
+
+  /**
+   * @note total read size must be <= INT_MAX, since
+   * the return value is total bytes read
+   */
+  int readx(OSDRead *rd, ObjectSet *oset, Context *onfinish);
+  int writex(OSDWrite *wr, ObjectSet *oset, Mutex& wait_on_lock,
+	     Context *onfreespace);
+  bool is_cached(ObjectSet *oset, vector& extents, snapid_t snapid);
+
+private:
+  // write blocking
+  int _wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, Mutex& lock,
+		      Context *onfreespace);
+  void maybe_wait_for_writeback(uint64_t len);
+  bool _flush_set_finish(C_GatherBuilder *gather, Context *onfinish);
+
+public:
+  bool set_is_empty(ObjectSet *oset);
+  bool set_is_cached(ObjectSet *oset);
+  bool set_is_dirty_or_committing(ObjectSet *oset);
+
+  bool flush_set(ObjectSet *oset, Context *onfinish=0);
+  bool flush_set(ObjectSet *oset, vector& ex, Context *onfinish=0);
+  void flush_all(Context *onfinish=0);
+
+  void purge_set(ObjectSet *oset);
+
+  loff_t release_set(ObjectSet *oset);  // returns # of bytes not released (ie non-clean)
+  uint64_t release_all();
+
+  void discard_set(ObjectSet *oset, vector& ex);
+
+  /**
+   * Retry any in-flight reads that get -ENOENT instead of marking
+   * them zero, and get rid of any cached -ENOENTs.
+   * After this is called and the cache's lock is unlocked,
+   * any new requests will treat -ENOENT normally.
+   */
+  void clear_nonexistence(ObjectSet *oset);
+
+
+  // cache sizes
+  void set_max_dirty(uint64_t v) {
+    max_dirty = v;
+  }
+  void set_target_dirty(int64_t v) {
+    target_dirty = v;
+  }
+  void set_max_size(int64_t v) {
+    max_size = v;
+  }
+  void set_max_dirty_age(double a) {
+    max_dirty_age.set_from_double(a);
+  }
+  void set_max_objects(int64_t v) {
+    max_objects = v;
+  }
+
+
+  // file functions
+
+  /*** async+caching (non-blocking) file interface ***/
+  int file_is_cached(ObjectSet *oset, ceph_file_layout *layout, snapid_t snapid,
+		     loff_t offset, uint64_t len) {
+    vector extents;
+    Striper::file_to_extents(cct, oset->ino, layout, offset, len, oset->truncate_size, extents);
+    return is_cached(oset, extents, snapid);
+  }
+
+  int file_read(ObjectSet *oset, ceph_file_layout *layout, snapid_t snapid,
+                loff_t offset, uint64_t len, 
+                bufferlist *bl,
+		int flags,
+                Context *onfinish) {
+    OSDRead *rd = prepare_read(snapid, bl, flags);
+    Striper::file_to_extents(cct, oset->ino, layout, offset, len, oset->truncate_size, rd->extents);
+    return readx(rd, oset, onfinish);
+  }
+
+  int file_write(ObjectSet *oset, ceph_file_layout *layout, const SnapContext& snapc,
+                 loff_t offset, uint64_t len, 
+                 bufferlist& bl, utime_t mtime, int flags,
+		 Mutex& wait_on_lock) {
+    OSDWrite *wr = prepare_write(snapc, bl, mtime, flags);
+    Striper::file_to_extents(cct, oset->ino, layout, offset, len, oset->truncate_size, wr->extents);
+    return writex(wr, oset, wait_on_lock, NULL);
+  }
+
+  bool file_flush(ObjectSet *oset, ceph_file_layout *layout, const SnapContext& snapc,
+                  loff_t offset, uint64_t len, Context *onfinish) {
+    vector extents;
+    Striper::file_to_extents(cct, oset->ino, layout, offset, len, oset->truncate_size, extents);
+    return flush_set(oset, extents, onfinish);
+  }
+};
+
+
+inline ostream& operator<<(ostream& out, ObjectCacher::BufferHead &bh)
+{
+  out << "bh[ " << &bh << " "
+      << bh.start() << "~" << bh.length()
+      << " " << bh.ob
+      << " (" << bh.bl.length() << ")"
+      << " v " << bh.last_write_tid;
+  if (bh.is_tx()) out << " tx";
+  if (bh.is_rx()) out << " rx";
+  if (bh.is_dirty()) out << " dirty";
+  if (bh.is_clean()) out << " clean";
+  if (bh.is_zero()) out << " zero";
+  if (bh.is_missing()) out << " missing";
+  if (bh.bl.length() > 0) out << " firstbyte=" << (int)bh.bl[0];
+  if (bh.error) out << " error=" << bh.error;
+  out << "]";
+  out << " waiters = {";
+  for (map >::const_iterator it = bh.waitfor_read.begin();
+       it != bh.waitfor_read.end(); ++it) {
+    out << " " << it->first << "->[";
+    for (list::const_iterator lit = it->second.begin();
+	 lit != it->second.end(); ++lit) {
+	 out << *lit << ", ";
+    }
+    out << "]";
+  }
+  out << "}";
+  return out;
+}
+
+inline ostream& operator<<(ostream& out, ObjectCacher::ObjectSet &os)
+{
+  return out << "objectset[" << os.ino
+	     << " ts " << os.truncate_seq << "/" << os.truncate_size
+	     << " objects " << os.objects.size()
+	     << " dirty_or_tx " << os.dirty_or_tx
+	     << "]";
+}
+
+inline ostream& operator<<(ostream& out, ObjectCacher::Object &ob)
+{
+  out << "object["
+      << ob.get_soid() << " oset " << ob.oset << dec
+      << " wr " << ob.last_write_tid << "/" << ob.last_commit_tid;
+
+  if (ob.complete)
+    out << " COMPLETE";
+  if (!ob.exists)
+    out << " !EXISTS";
+
+  out << "]";
+  return out;
+}
+
+#endif
diff --git a/ceph/src/osdc/Objecter.cc b/ceph/src/osdc/Objecter.cc
new file mode 100644
index 00000000..d82b3e12
--- /dev/null
+++ b/ceph/src/osdc/Objecter.cc
@@ -0,0 +1,2897 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "Objecter.h"
+#include "osd/OSDMap.h"
+#include "Filer.h"
+
+#include "mon/MonClient.h"
+
+#include "msg/Messenger.h"
+#include "msg/Message.h"
+
+#include "messages/MPing.h"
+#include "messages/MOSDOp.h"
+#include "messages/MOSDOpReply.h"
+#include "messages/MOSDMap.h"
+
+#include "messages/MPoolOp.h"
+#include "messages/MPoolOpReply.h"
+
+#include "messages/MGetPoolStats.h"
+#include "messages/MGetPoolStatsReply.h"
+#include "messages/MStatfs.h"
+#include "messages/MStatfsReply.h"
+
+#include "messages/MOSDFailure.h"
+#include "messages/MMonCommand.h"
+
+#include "messages/MCommand.h"
+#include "messages/MCommandReply.h"
+
+#include 
+
+#include "common/config.h"
+#include "common/perf_counters.h"
+#include "include/str_list.h"
+#include "common/errno.h"
+
+
+#define dout_subsys ceph_subsys_objecter
+#undef dout_prefix
+#define dout_prefix *_dout << messenger->get_myname() << ".objecter "
+
+
+enum {
+  l_osdc_first = 123200,
+  l_osdc_op_active,
+  l_osdc_op_laggy,
+  l_osdc_op_send,
+  l_osdc_op_send_bytes,
+  l_osdc_op_resend,
+  l_osdc_op_ack,
+  l_osdc_op_commit,
+
+  l_osdc_op,
+  l_osdc_op_r,
+  l_osdc_op_w,
+  l_osdc_op_rmw,
+  l_osdc_op_pg,
+
+  l_osdc_osdop_stat,
+  l_osdc_osdop_create,
+  l_osdc_osdop_read,
+  l_osdc_osdop_write,
+  l_osdc_osdop_writefull,
+  l_osdc_osdop_append,
+  l_osdc_osdop_zero,
+  l_osdc_osdop_truncate,
+  l_osdc_osdop_delete,
+  l_osdc_osdop_mapext,
+  l_osdc_osdop_sparse_read,
+  l_osdc_osdop_clonerange,
+  l_osdc_osdop_getxattr,
+  l_osdc_osdop_setxattr,
+  l_osdc_osdop_cmpxattr,
+  l_osdc_osdop_rmxattr,
+  l_osdc_osdop_resetxattrs,
+  l_osdc_osdop_tmap_up,
+  l_osdc_osdop_tmap_put,
+  l_osdc_osdop_tmap_get,
+  l_osdc_osdop_call,
+  l_osdc_osdop_watch,
+  l_osdc_osdop_notify,
+  l_osdc_osdop_src_cmpxattr,
+  l_osdc_osdop_pgls,
+  l_osdc_osdop_pgls_filter,
+  l_osdc_osdop_other,
+
+  l_osdc_linger_active,
+  l_osdc_linger_send,
+  l_osdc_linger_resend,
+
+  l_osdc_poolop_active,
+  l_osdc_poolop_send,
+  l_osdc_poolop_resend,
+
+  l_osdc_poolstat_active,
+  l_osdc_poolstat_send,
+  l_osdc_poolstat_resend,
+
+  l_osdc_statfs_active,
+  l_osdc_statfs_send,
+  l_osdc_statfs_resend,
+
+  l_osdc_command_active,
+  l_osdc_command_send,
+  l_osdc_command_resend,
+
+  l_osdc_map_epoch,
+  l_osdc_map_full,
+  l_osdc_map_inc,
+
+  l_osdc_osd_sessions,
+  l_osdc_osd_session_open,
+  l_osdc_osd_session_close,
+  l_osdc_osd_laggy,
+  l_osdc_last,
+};
+
+
+// config obs ----------------------------
+
+static const char *config_keys[] = {
+  "crush_location",
+  NULL
+};
+
+const char** Objecter::get_tracked_conf_keys() const
+{
+  return config_keys;
+}
+
+
+void Objecter::handle_conf_change(const struct md_config_t *conf,
+				  const std::set  &changed)
+{
+  if (changed.count("crush_location")) {
+    crush_location.clear();
+    vector lvec;
+    get_str_vec(cct->_conf->crush_location, ";, \t", lvec);
+    int r = CrushWrapper::parse_loc_multimap(lvec, &crush_location);
+    if (r < 0) {
+      lderr(cct) << "warning: crush_location '" << cct->_conf->crush_location
+		 << "' does not parse" << dendl;
+    }
+  }
+}
+
+
+// messages ------------------------------
+
+void Objecter::init_unlocked()
+{
+  assert(!initialized);
+
+  if (!logger) {
+    PerfCountersBuilder pcb(cct, "objecter", l_osdc_first, l_osdc_last);
+
+    pcb.add_u64(l_osdc_op_active, "op_active");
+    pcb.add_u64(l_osdc_op_laggy, "op_laggy");
+    pcb.add_u64_counter(l_osdc_op_send, "op_send");
+    pcb.add_u64_counter(l_osdc_op_send_bytes, "op_send_bytes");
+    pcb.add_u64_counter(l_osdc_op_resend, "op_resend");
+    pcb.add_u64_counter(l_osdc_op_ack, "op_ack");
+    pcb.add_u64_counter(l_osdc_op_commit, "op_commit");
+
+    pcb.add_u64_counter(l_osdc_op, "op");
+    pcb.add_u64_counter(l_osdc_op_r, "op_r");
+    pcb.add_u64_counter(l_osdc_op_w, "op_w");
+    pcb.add_u64_counter(l_osdc_op_rmw, "op_rmw");
+    pcb.add_u64_counter(l_osdc_op_pg, "op_pg");
+
+    pcb.add_u64_counter(l_osdc_osdop_stat, "osdop_stat");
+    pcb.add_u64_counter(l_osdc_osdop_create, "osdop_create");
+    pcb.add_u64_counter(l_osdc_osdop_read, "osdop_read");
+    pcb.add_u64_counter(l_osdc_osdop_write, "osdop_write");
+    pcb.add_u64_counter(l_osdc_osdop_writefull, "osdop_writefull");
+    pcb.add_u64_counter(l_osdc_osdop_append, "osdop_append");
+    pcb.add_u64_counter(l_osdc_osdop_zero, "osdop_zero");
+    pcb.add_u64_counter(l_osdc_osdop_truncate, "osdop_truncate");
+    pcb.add_u64_counter(l_osdc_osdop_delete, "osdop_delete");
+    pcb.add_u64_counter(l_osdc_osdop_mapext, "osdop_mapext");
+    pcb.add_u64_counter(l_osdc_osdop_sparse_read, "osdop_sparse_read");
+    pcb.add_u64_counter(l_osdc_osdop_clonerange, "osdop_clonerange");
+    pcb.add_u64_counter(l_osdc_osdop_getxattr, "osdop_getxattr");
+    pcb.add_u64_counter(l_osdc_osdop_setxattr, "osdop_setxattr");
+    pcb.add_u64_counter(l_osdc_osdop_cmpxattr, "osdop_cmpxattr");
+    pcb.add_u64_counter(l_osdc_osdop_rmxattr, "osdop_rmxattr");
+    pcb.add_u64_counter(l_osdc_osdop_resetxattrs, "osdop_resetxattrs");
+    pcb.add_u64_counter(l_osdc_osdop_tmap_up, "osdop_tmap_up");
+    pcb.add_u64_counter(l_osdc_osdop_tmap_put, "osdop_tmap_put");
+    pcb.add_u64_counter(l_osdc_osdop_tmap_get, "osdop_tmap_get");
+    pcb.add_u64_counter(l_osdc_osdop_call, "osdop_call");
+    pcb.add_u64_counter(l_osdc_osdop_watch, "osdop_watch");
+    pcb.add_u64_counter(l_osdc_osdop_notify, "osdop_notify");
+    pcb.add_u64_counter(l_osdc_osdop_src_cmpxattr, "osdop_src_cmpxattr");
+    pcb.add_u64_counter(l_osdc_osdop_pgls, "osdop_pgls");
+    pcb.add_u64_counter(l_osdc_osdop_pgls_filter, "osdop_pgls_filter");
+    pcb.add_u64_counter(l_osdc_osdop_other, "osdop_other");
+
+    pcb.add_u64(l_osdc_linger_active, "linger_active");
+    pcb.add_u64_counter(l_osdc_linger_send, "linger_send");
+    pcb.add_u64_counter(l_osdc_linger_resend, "linger_resend");
+
+    pcb.add_u64(l_osdc_poolop_active, "poolop_active");
+    pcb.add_u64_counter(l_osdc_poolop_send, "poolop_send");
+    pcb.add_u64_counter(l_osdc_poolop_resend, "poolop_resend");
+
+    pcb.add_u64(l_osdc_poolstat_active, "poolstat_active");
+    pcb.add_u64_counter(l_osdc_poolstat_send, "poolstat_send");
+    pcb.add_u64_counter(l_osdc_poolstat_resend, "poolstat_resend");
+
+    pcb.add_u64(l_osdc_statfs_active, "statfs_active");
+    pcb.add_u64_counter(l_osdc_statfs_send, "statfs_send");
+    pcb.add_u64_counter(l_osdc_statfs_resend, "statfs_resend");
+
+    pcb.add_u64(l_osdc_command_active, "command_active");
+    pcb.add_u64_counter(l_osdc_command_send, "command_send");
+    pcb.add_u64_counter(l_osdc_command_resend, "command_resend");
+
+    pcb.add_u64(l_osdc_map_epoch, "map_epoch");
+    pcb.add_u64_counter(l_osdc_map_full, "map_full");
+    pcb.add_u64_counter(l_osdc_map_inc, "map_inc");
+
+    pcb.add_u64(l_osdc_osd_sessions, "osd_sessions");  // open sessions
+    pcb.add_u64_counter(l_osdc_osd_session_open, "osd_session_open");
+    pcb.add_u64_counter(l_osdc_osd_session_close, "osd_session_close");
+    pcb.add_u64(l_osdc_osd_laggy, "osd_laggy");
+
+    logger = pcb.create_perf_counters();
+    cct->get_perfcounters_collection()->add(logger);
+  }
+
+  m_request_state_hook = new RequestStateHook(this);
+  AdminSocket* admin_socket = cct->get_admin_socket();
+  int ret = admin_socket->register_command("objecter_requests",
+					   "objecter_requests",
+					   m_request_state_hook,
+					   "show in-progress osd requests");
+  if (ret < 0) {
+    lderr(cct) << "error registering admin socket command: "
+	       << cpp_strerror(ret) << dendl;
+  }
+}
+
+void Objecter::init_locked()
+{
+  assert(client_lock.is_locked());
+  assert(!initialized);
+
+  schedule_tick();
+  if (osdmap->get_epoch() == 0)
+    maybe_request_map();
+
+  initialized = true;
+}
+
+void Objecter::shutdown_locked() 
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+  initialized = false;
+
+  map::iterator p;
+  while (!osd_sessions.empty()) {
+    p = osd_sessions.begin();
+    close_session(p->second);
+  }
+
+  if (tick_event) {
+    timer.cancel_event(tick_event);
+    tick_event = NULL;
+  }
+}
+
+void Objecter::shutdown_unlocked()
+{
+  if (m_request_state_hook) {
+    AdminSocket* admin_socket = cct->get_admin_socket();
+    admin_socket->unregister_command("objecter_requests");
+    delete m_request_state_hook;
+    m_request_state_hook = NULL;
+  }
+
+  if (logger) {
+    cct->get_perfcounters_collection()->remove(logger);
+    delete logger;
+    logger = NULL;
+  }
+}
+
+void Objecter::send_linger(LingerOp *info)
+{
+  ldout(cct, 15) << "send_linger " << info->linger_id << dendl;
+  vector opv = info->ops; // need to pass a copy to ops
+  Context *onack = (!info->registered && info->on_reg_ack) ? new C_Linger_Ack(this, info) : NULL;
+  Context *oncommit = new C_Linger_Commit(this, info);
+  Op *o = new Op(info->target.base_oid, info->target.base_oloc,
+		 opv, info->target.flags | CEPH_OSD_FLAG_READ,
+		 onack, oncommit,
+		 info->pobjver);
+  o->snapid = info->snap;
+  o->snapc = info->snapc;
+  o->mtime = info->mtime;
+
+  // do not resend this; we will send a new op to reregister
+  o->should_resend = false;
+
+  if (info->session) {
+    int r = recalc_op_target(o);
+    if (r == RECALC_OP_TARGET_POOL_DNE) {
+      _send_linger_map_check(info);
+    }
+  }
+
+  if (info->register_tid) {
+    // repeat send.  cancel old registeration op, if any.
+    if (ops.count(info->register_tid)) {
+      Op *o = ops[info->register_tid];
+      op_cancel_map_check(o);
+      cancel_linger_op(o);
+    }
+    info->register_tid = _op_submit(o);
+  } else {
+    // first send
+    // populate info->pgid and info->acting so we
+    // don't resend the linger op on the next osdmap update
+    recalc_linger_op_target(info);
+    info->register_tid = op_submit(o);
+  }
+
+  OSDSession *s = o->session;
+  if (info->session != s) {
+    info->session_item.remove_myself();
+    info->session = s;
+    if (info->session)
+      s->linger_ops.push_back(&info->session_item);
+  }
+
+  logger->inc(l_osdc_linger_send);
+}
+
+void Objecter::_linger_ack(LingerOp *info, int r) 
+{
+  ldout(cct, 10) << "_linger_ack " << info->linger_id << dendl;
+  if (info->on_reg_ack) {
+    info->on_reg_ack->complete(r);
+    info->on_reg_ack = NULL;
+  }
+}
+
+void Objecter::_linger_commit(LingerOp *info, int r) 
+{
+  ldout(cct, 10) << "_linger_commit " << info->linger_id << dendl;
+  if (info->on_reg_commit) {
+    info->on_reg_commit->complete(r);
+    info->on_reg_commit = NULL;
+  }
+
+  // only tell the user the first time we do this
+  info->registered = true;
+  info->pobjver = NULL;
+}
+
+void Objecter::unregister_linger(uint64_t linger_id)
+{
+  map::iterator iter = linger_ops.find(linger_id);
+  if (iter != linger_ops.end()) {
+    LingerOp *info = iter->second;
+    info->session_item.remove_myself();
+    linger_ops.erase(iter);
+    info->put();
+    logger->set(l_osdc_linger_active, linger_ops.size());
+  }
+}
+
+ceph_tid_t Objecter::linger_mutate(const object_t& oid, const object_locator_t& oloc,
+			      ObjectOperation& op,
+			      const SnapContext& snapc, utime_t mtime,
+			      bufferlist& inbl, int flags,
+			      Context *onack, Context *oncommit,
+			      version_t *objver)
+{
+  LingerOp *info = new LingerOp;
+  info->target.base_oid = oid;
+  info->target.base_oloc = oloc;
+  if (info->target.base_oloc.key == oid)
+    info->target.base_oloc.key.clear();
+  info->snapc = snapc;
+  info->mtime = mtime;
+  info->target.flags = flags | CEPH_OSD_FLAG_WRITE;
+  info->ops = op.ops;
+  info->inbl = inbl;
+  info->poutbl = NULL;
+  info->pobjver = objver;
+  info->on_reg_ack = onack;
+  info->on_reg_commit = oncommit;
+
+  info->linger_id = ++max_linger_id;
+  linger_ops[info->linger_id] = info;
+
+  logger->set(l_osdc_linger_active, linger_ops.size());
+
+  send_linger(info);
+
+  return info->linger_id;
+}
+
+ceph_tid_t Objecter::linger_read(const object_t& oid, const object_locator_t& oloc,
+			    ObjectOperation& op,
+			    snapid_t snap, bufferlist& inbl, bufferlist *poutbl, int flags,
+			    Context *onfinish,
+			    version_t *objver)
+{
+  LingerOp *info = new LingerOp;
+  info->target.base_oid = oid;
+  info->target.base_oloc = oloc;
+  if (info->target.base_oloc.key == oid)
+    info->target.base_oloc.key.clear();
+  info->snap = snap;
+  info->target.flags = flags;
+  info->ops = op.ops;
+  info->inbl = inbl;
+  info->poutbl = poutbl;
+  info->pobjver = objver;
+  info->on_reg_commit = onfinish;
+
+  info->linger_id = ++max_linger_id;
+  linger_ops[info->linger_id] = info;
+
+  logger->set(l_osdc_linger_active, linger_ops.size());
+
+  send_linger(info);
+
+  return info->linger_id;
+}
+
+void Objecter::dispatch(Message *m)
+{
+  switch (m->get_type()) {
+  case CEPH_MSG_OSD_OPREPLY:
+    handle_osd_op_reply(static_cast(m));
+    break;
+    
+  case CEPH_MSG_OSD_MAP:
+    handle_osd_map(static_cast(m));
+    break;
+
+  case MSG_GETPOOLSTATSREPLY:
+    handle_get_pool_stats_reply(static_cast(m));
+    break;
+
+  case CEPH_MSG_STATFS_REPLY:
+    handle_fs_stats_reply(static_cast(m));
+    break;
+
+  case CEPH_MSG_POOLOP_REPLY:
+    handle_pool_op_reply(static_cast(m));
+    break;
+
+  case MSG_COMMAND_REPLY:
+    handle_command_reply(static_cast(m));
+    break;
+
+  default:
+    ldout(cct, 0) << "don't know message type " << m->get_type() << dendl;
+    assert(0);
+  }
+}
+
+void Objecter::scan_requests(bool force_resend,
+			     bool force_resend_writes,
+			     map& need_resend,
+			     list& need_resend_linger,
+			     map& need_resend_command)
+{
+  // check for changed linger mappings (_before_ regular ops)
+  map::iterator lp = linger_ops.begin();
+  while (lp != linger_ops.end()) {
+    LingerOp *op = lp->second;
+    ++lp;   // check_linger_pool_dne() may touch linger_ops; prevent iterator invalidation
+    ldout(cct, 10) << " checking linger op " << op->linger_id << dendl;
+    int r = recalc_linger_op_target(op);
+    switch (r) {
+    case RECALC_OP_TARGET_NO_ACTION:
+      if (!force_resend && !force_resend_writes)
+	break;
+      // -- fall-thru --
+    case RECALC_OP_TARGET_NEED_RESEND:
+      need_resend_linger.push_back(op);
+      linger_cancel_map_check(op);
+      break;
+    case RECALC_OP_TARGET_POOL_DNE:
+      check_linger_pool_dne(op);
+      break;
+    }
+  }
+
+  // check for changed request mappings
+  map::iterator p = ops.begin();
+  while (p != ops.end()) {
+    Op *op = p->second;
+    ++p;   // check_op_pool_dne() may touch ops; prevent iterator invalidation
+    ldout(cct, 10) << " checking op " << op->tid << dendl;
+    int r = recalc_op_target(op);
+    switch (r) {
+    case RECALC_OP_TARGET_NO_ACTION:
+      if (!force_resend &&
+	  (!force_resend_writes || !(op->target.flags & CEPH_OSD_FLAG_WRITE)))
+	break;
+      // -- fall-thru --
+    case RECALC_OP_TARGET_NEED_RESEND:
+      need_resend[op->tid] = op;
+      op_cancel_map_check(op);
+      break;
+    case RECALC_OP_TARGET_POOL_DNE:
+      check_op_pool_dne(op);
+      break;
+    }
+  }
+
+  // commands
+  map::iterator cp = command_ops.begin();
+  while (cp != command_ops.end()) {
+    CommandOp *c = cp->second;
+    ++cp;
+    ldout(cct, 10) << " checking command " << c->tid << dendl;
+    int r = recalc_command_target(c);
+    switch (r) {
+    case RECALC_OP_TARGET_NO_ACTION:
+      // resend if skipped map; otherwise do nothing.
+      if (!force_resend && !force_resend_writes)
+	break;
+      // -- fall-thru --
+    case RECALC_OP_TARGET_NEED_RESEND:
+      need_resend_command[c->tid] = c;
+      command_cancel_map_check(c);
+      break;
+    case RECALC_OP_TARGET_POOL_DNE:
+    case RECALC_OP_TARGET_OSD_DNE:
+    case RECALC_OP_TARGET_OSD_DOWN:
+      check_command_map_dne(c);
+      break;
+    }     
+  }
+}
+
+void Objecter::handle_osd_map(MOSDMap *m)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+  assert(osdmap); 
+
+  if (m->fsid != monc->get_fsid()) {
+    ldout(cct, 0) << "handle_osd_map fsid " << m->fsid << " != " << monc->get_fsid() << dendl;
+    m->put();
+    return;
+  }
+
+  bool was_pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
+  bool was_full = osdmap_full_flag();
+  bool was_pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || was_full;
+
+  list need_resend_linger;
+  map need_resend;
+  map need_resend_command;
+
+  if (m->get_last() <= osdmap->get_epoch()) {
+    ldout(cct, 3) << "handle_osd_map ignoring epochs [" 
+            << m->get_first() << "," << m->get_last() 
+            << "] <= " << osdmap->get_epoch() << dendl;
+  } 
+  else {
+    ldout(cct, 3) << "handle_osd_map got epochs [" 
+            << m->get_first() << "," << m->get_last() 
+            << "] > " << osdmap->get_epoch()
+            << dendl;
+
+    if (osdmap->get_epoch()) {
+      bool skipped_map = false;
+      // we want incrementals
+      for (epoch_t e = osdmap->get_epoch() + 1;
+	   e <= m->get_last();
+	   e++) {
+ 
+	if (osdmap->get_epoch() == e-1 &&
+	    m->incremental_maps.count(e)) {
+	  ldout(cct, 3) << "handle_osd_map decoding incremental epoch " << e << dendl;
+	  OSDMap::Incremental inc(m->incremental_maps[e]);
+	  osdmap->apply_incremental(inc);
+	  logger->inc(l_osdc_map_inc);
+	}
+	else if (m->maps.count(e)) {
+	  ldout(cct, 3) << "handle_osd_map decoding full epoch " << e << dendl;
+	  osdmap->decode(m->maps[e]);
+	  logger->inc(l_osdc_map_full);
+	}
+	else {
+	  if (e && e > m->get_oldest()) {
+	    ldout(cct, 3) << "handle_osd_map requesting missing epoch " << osdmap->get_epoch()+1 << dendl;
+	    maybe_request_map();
+	    break;
+	  }
+	  ldout(cct, 3) << "handle_osd_map missing epoch " << osdmap->get_epoch()+1
+			<< ", jumping to " << m->get_oldest() << dendl;
+	  e = m->get_oldest() - 1;
+	  skipped_map = true;
+	  continue;
+	}
+	logger->set(l_osdc_map_epoch, osdmap->get_epoch());
+
+	was_full = was_full || osdmap_full_flag();
+	scan_requests(skipped_map, was_full, need_resend, need_resend_linger,
+		      need_resend_command);
+
+	// osd addr changes?
+	for (map::iterator p = osd_sessions.begin();
+	     p != osd_sessions.end(); ) {
+	  OSDSession *s = p->second;
+	  ++p;
+	  if (osdmap->is_up(s->osd)) {
+	    if (s->con && s->con->get_peer_addr() != osdmap->get_inst(s->osd).addr)
+	      close_session(s);
+	  } else {
+	    close_session(s);
+	  }
+	}
+
+	assert(e == osdmap->get_epoch());
+      }
+      
+    } else {
+      // first map.  we want the full thing.
+      if (m->maps.count(m->get_last())) {
+	ldout(cct, 3) << "handle_osd_map decoding full epoch " << m->get_last() << dendl;
+	osdmap->decode(m->maps[m->get_last()]);
+
+	scan_requests(false, false, need_resend, need_resend_linger,
+		      need_resend_command);
+      } else {
+	ldout(cct, 3) << "handle_osd_map hmm, i want a full map, requesting" << dendl;
+	monc->sub_want("osdmap", 0, CEPH_SUBSCRIBE_ONETIME);
+	monc->renew_subs();
+      }
+    }
+  }
+
+  bool pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
+  bool pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || osdmap_full_flag();
+
+  // was/is paused?
+  if (was_pauserd || was_pausewr || pauserd || pausewr)
+    maybe_request_map();
+
+  // resend requests
+  for (map::iterator p = need_resend.begin(); p != need_resend.end(); ++p) {
+    Op *op = p->second;
+    if (op->should_resend) {
+      if (op->session && !op->target.paused) {
+	logger->inc(l_osdc_op_resend);
+	send_op(op);
+      }
+    } else {
+      cancel_linger_op(op);
+    }
+  }
+  for (list::iterator p = need_resend_linger.begin(); p != need_resend_linger.end(); ++p) {
+    LingerOp *op = *p;
+    if (op->session) {
+      logger->inc(l_osdc_linger_resend);
+      send_linger(op);
+    }
+  }
+  for (map::iterator p = need_resend_command.begin(); p != need_resend_command.end(); ++p) {
+    CommandOp *c = p->second;
+    if (c->session) {
+      _send_command(c);
+    }
+  }
+
+  dump_active();
+  
+  // finish any Contexts that were waiting on a map update
+  map > >::iterator p =
+    waiting_for_map.begin();
+  while (p != waiting_for_map.end() &&
+	 p->first <= osdmap->get_epoch()) {
+    //go through the list and call the onfinish methods
+    for (list >::iterator i = p->second.begin();
+	 i != p->second.end(); ++i) {
+      i->first->complete(i->second);
+    }
+    waiting_for_map.erase(p++);
+  }
+
+  m->put();
+
+  monc->sub_got("osdmap", osdmap->get_epoch());
+
+  if (!waiting_for_map.empty())
+    maybe_request_map();
+}
+
+// op pool check
+
+void Objecter::C_Op_Map_Latest::finish(int r)
+{
+  if (r == -EAGAIN || r == -ECANCELED)
+    return;
+
+  lgeneric_subdout(objecter->cct, objecter, 10) << "op_map_latest r=" << r << " tid=" << tid
+						<< " latest " << latest << dendl;
+
+  Mutex::Locker l(objecter->client_lock);
+
+  map::iterator iter =
+    objecter->check_latest_map_ops.find(tid);
+  if (iter == objecter->check_latest_map_ops.end()) {
+    lgeneric_subdout(objecter->cct, objecter, 10) << "op_map_latest op " << tid << " not found" << dendl;
+    return;
+  }
+
+  Op *op = iter->second;
+  objecter->check_latest_map_ops.erase(iter);
+
+  lgeneric_subdout(objecter->cct, objecter, 20) << "op_map_latest op " << op << dendl;
+
+  if (op->map_dne_bound == 0)
+    op->map_dne_bound = latest;
+
+  objecter->check_op_pool_dne(op);
+}
+
+void Objecter::check_op_pool_dne(Op *op)
+{
+  ldout(cct, 10) << "check_op_pool_dne tid " << op->tid
+		 << " current " << osdmap->get_epoch()
+		 << " map_dne_bound " << op->map_dne_bound
+		 << dendl;
+  if (op->map_dne_bound > 0) {
+    if (osdmap->get_epoch() >= op->map_dne_bound) {
+      // we had a new enough map
+      ldout(cct, 10) << "check_op_pool_dne tid " << op->tid
+		     << " concluding pool " << op->target.base_pgid.pool() << " dne"
+		     << dendl;
+      if (op->onack) {
+	op->onack->complete(-ENOENT);
+      }
+      if (op->oncommit) {
+	op->oncommit->complete(-ENOENT);
+      }
+      op->session_item.remove_myself();
+      ops.erase(op->tid);
+      delete op;
+    }
+  } else {
+    _send_op_map_check(op);
+  }
+}
+
+void Objecter::_send_op_map_check(Op *op)
+{
+  assert(client_lock.is_locked());
+  // ask the monitor
+  if (check_latest_map_ops.count(op->tid) == 0) {
+    check_latest_map_ops[op->tid] = op;
+    C_Op_Map_Latest *c = new C_Op_Map_Latest(this, op->tid);
+    monc->get_version("osdmap", &c->latest, NULL, c);
+  }
+}
+
+void Objecter::op_cancel_map_check(Op *op)
+{
+  assert(client_lock.is_locked());
+  map::iterator iter =
+    check_latest_map_ops.find(op->tid);
+  if (iter != check_latest_map_ops.end()) {
+    check_latest_map_ops.erase(iter);
+  }
+}
+
+// linger pool check
+
+void Objecter::C_Linger_Map_Latest::finish(int r)
+{
+  if (r == -EAGAIN || r == -ECANCELED) {
+    // ignore callback; we will retry in resend_mon_ops()
+    return;
+  }
+
+  Mutex::Locker l(objecter->client_lock);
+
+  map::iterator iter =
+    objecter->check_latest_map_lingers.find(linger_id);
+  if (iter == objecter->check_latest_map_lingers.end()) {
+    return;
+  }
+
+  LingerOp *op = iter->second;
+  objecter->check_latest_map_lingers.erase(iter);
+  op->put();
+
+  if (op->map_dne_bound == 0)
+    op->map_dne_bound = latest;
+
+  objecter->check_linger_pool_dne(op);
+}
+
+void Objecter::check_linger_pool_dne(LingerOp *op)
+{
+  ldout(cct, 10) << "check_linger_pool_dne linger_id " << op->linger_id
+		 << " current " << osdmap->get_epoch()
+		 << " map_dne_bound " << op->map_dne_bound
+		 << dendl;
+  if (op->map_dne_bound > 0) {
+    if (osdmap->get_epoch() >= op->map_dne_bound) {
+      if (op->on_reg_ack) {
+	op->on_reg_ack->complete(-ENOENT);
+      }
+      if (op->on_reg_commit) {
+	op->on_reg_commit->complete(-ENOENT);
+      }
+      unregister_linger(op->linger_id);
+    }
+  } else {
+    _send_linger_map_check(op);
+  }
+}
+
+void Objecter::_send_linger_map_check(LingerOp *op)
+{
+  // ask the monitor
+  if (check_latest_map_lingers.count(op->linger_id) == 0) {
+    op->get();
+    check_latest_map_lingers[op->linger_id] = op;
+    C_Linger_Map_Latest *c = new C_Linger_Map_Latest(this, op->linger_id);
+    monc->get_version("osdmap", &c->latest, NULL, c);
+  }
+}
+
+void Objecter::linger_cancel_map_check(LingerOp *op)
+{
+  map::iterator iter =
+    check_latest_map_lingers.find(op->linger_id);
+  if (iter != check_latest_map_lingers.end()) {
+    LingerOp *op = iter->second;
+    op->put();
+    check_latest_map_lingers.erase(iter);
+  }
+}
+
+// command pool check
+
+void Objecter::C_Command_Map_Latest::finish(int r)
+{
+  if (r == -EAGAIN || r == -ECANCELED) {
+    // ignore callback; we will retry in resend_mon_ops()
+    return;
+  }
+
+  Mutex::Locker l(objecter->client_lock);
+
+  map::iterator iter =
+    objecter->check_latest_map_commands.find(tid);
+  if (iter == objecter->check_latest_map_commands.end()) {
+    return;
+  }
+
+  CommandOp *c = iter->second;
+  objecter->check_latest_map_commands.erase(iter);
+  c->put();
+
+  if (c->map_dne_bound == 0)
+    c->map_dne_bound = latest;
+
+  objecter->check_command_map_dne(c);
+}
+
+void Objecter::check_command_map_dne(CommandOp *c)
+{
+  ldout(cct, 10) << "check_command_map_dne tid " << c->tid
+		 << " current " << osdmap->get_epoch()
+		 << " map_dne_bound " << c->map_dne_bound
+		 << dendl;
+  if (c->map_dne_bound > 0) {
+    if (osdmap->get_epoch() >= c->map_dne_bound) {
+      _finish_command(c, c->map_check_error, c->map_check_error_str);
+    }
+  } else {
+    _send_command_map_check(c);
+  }
+}
+
+void Objecter::_send_command_map_check(CommandOp *c)
+{
+  // ask the monitor
+  if (check_latest_map_commands.count(c->tid) == 0) {
+    c->get();
+    check_latest_map_commands[c->tid] = c;
+    C_Command_Map_Latest *f = new C_Command_Map_Latest(this, c->tid);
+    monc->get_version("osdmap", &f->latest, NULL, f);
+  }
+}
+
+void Objecter::command_cancel_map_check(CommandOp *c)
+{
+  map::iterator iter =
+    check_latest_map_commands.find(c->tid);
+  if (iter != check_latest_map_commands.end()) {
+    CommandOp *c = iter->second;
+    c->put();
+    check_latest_map_commands.erase(iter);
+  }
+}
+
+
+
+Objecter::OSDSession *Objecter::get_session(int osd)
+{
+  map::iterator p = osd_sessions.find(osd);
+  if (p != osd_sessions.end())
+    return p->second;
+  OSDSession *s = new OSDSession(osd);
+  osd_sessions[osd] = s;
+  s->con = messenger->get_connection(osdmap->get_inst(osd));
+  logger->inc(l_osdc_osd_session_open);
+  logger->inc(l_osdc_osd_sessions, osd_sessions.size());
+  return s;
+}
+
+void Objecter::reopen_session(OSDSession *s)
+{
+  entity_inst_t inst = osdmap->get_inst(s->osd);
+  ldout(cct, 10) << "reopen_session osd." << s->osd << " session, addr now " << inst << dendl;
+  if (s->con) {
+    messenger->mark_down(s->con);
+    logger->inc(l_osdc_osd_session_close);
+  }
+  s->con = messenger->get_connection(inst);
+  s->incarnation++;
+  logger->inc(l_osdc_osd_session_open);
+}
+
+void Objecter::close_session(OSDSession *s)
+{
+  ldout(cct, 10) << "close_session for osd." << s->osd << dendl;
+  if (s->con) {
+    messenger->mark_down(s->con);
+    logger->inc(l_osdc_osd_session_close);
+  }
+  s->ops.clear();
+  s->linger_ops.clear();
+  s->command_ops.clear();
+  osd_sessions.erase(s->osd);
+  delete s;
+
+  logger->set(l_osdc_osd_sessions, osd_sessions.size());
+}
+
+void Objecter::wait_for_osd_map()
+{
+  if (osdmap->get_epoch()) return;
+  Mutex lock("");
+  Cond cond;
+  bool done;
+  lock.Lock();
+  C_SafeCond *context = new C_SafeCond(&lock, &cond, &done, NULL);
+  waiting_for_map[0].push_back(pair(context, 0));
+  while (!done)
+    cond.Wait(lock);
+  lock.Unlock();
+}
+
+struct C_Objecter_GetVersion : public Context {
+  Objecter *objecter;
+  uint64_t oldest, newest;
+  Context *fin;
+  C_Objecter_GetVersion(Objecter *o, Context *c)
+    : objecter(o), oldest(0), newest(0), fin(c) {}
+  void finish(int r) {
+    if (r >= 0)
+      objecter->_get_latest_version(oldest, newest, fin);
+    else if (r == -EAGAIN) { // try again as instructed
+      objecter->wait_for_latest_osdmap(fin);
+    } else {
+      // it doesn't return any other error codes!
+      assert(0);
+    }
+  }
+};
+
+void Objecter::wait_for_latest_osdmap(Context *fin)
+{
+  ldout(cct, 10) << __func__ << dendl;
+  C_Objecter_GetVersion *c = new C_Objecter_GetVersion(this, fin);
+  monc->get_version("osdmap", &c->newest, &c->oldest, c);
+}
+
+void Objecter::_get_latest_version(epoch_t oldest, epoch_t newest, Context *fin)
+{
+  if (osdmap->get_epoch() >= newest) {
+  ldout(cct, 10) << __func__ << " latest " << newest << ", have it" << dendl;
+    if (fin)
+      fin->complete(0);
+    return;
+  }
+
+  ldout(cct, 10) << __func__ << " latest " << newest << ", waiting" << dendl;
+  wait_for_new_map(fin, newest, 0);
+}
+
+void Objecter::maybe_request_map()
+{
+  int flag = 0;
+  if (osdmap_full_flag()) {
+    ldout(cct, 10) << "maybe_request_map subscribing (continuous) to next osd map (FULL flag is set)" << dendl;
+  } else {
+    ldout(cct, 10) << "maybe_request_map subscribing (onetime) to next osd map" << dendl;
+    flag = CEPH_SUBSCRIBE_ONETIME;
+  }
+  epoch_t epoch = osdmap->get_epoch() ? osdmap->get_epoch()+1 : 0;
+  if (monc->sub_want("osdmap", epoch, flag))
+    monc->renew_subs();
+}
+
+void Objecter::wait_for_new_map(Context *c, epoch_t epoch, int err)
+{
+  waiting_for_map[epoch].push_back(pair(c, err));
+  maybe_request_map();
+}
+
+void Objecter::kick_requests(OSDSession *session)
+{
+  ldout(cct, 10) << "kick_requests for osd." << session->osd << dendl;
+
+  // resend ops
+  map resend;  // resend in tid order
+  for (xlist::iterator p = session->ops.begin(); !p.end();) {
+    Op *op = *p;
+    ++p;
+    logger->inc(l_osdc_op_resend);
+    if (op->should_resend) {
+      if (!op->target.paused)
+	resend[op->tid] = op;
+    } else {
+      cancel_linger_op(op);
+    }
+  }
+  while (!resend.empty()) {
+    send_op(resend.begin()->second);
+    resend.erase(resend.begin());
+  }
+
+  // resend lingers
+  map lresend;  // resend in order
+  for (xlist::iterator j = session->linger_ops.begin(); !j.end(); ++j) {
+    logger->inc(l_osdc_linger_resend);
+    lresend[(*j)->linger_id] = *j;
+  }
+  while (!lresend.empty()) {
+    send_linger(lresend.begin()->second);
+    lresend.erase(lresend.begin());
+  }
+
+  // resend commands
+  map cresend;  // resend in order
+  for (xlist::iterator k = session->command_ops.begin(); !k.end(); ++k) {
+    logger->inc(l_osdc_command_resend);
+    cresend[(*k)->tid] = *k;
+  }
+  while (!cresend.empty()) {
+    _send_command(cresend.begin()->second);
+    cresend.erase(cresend.begin());
+  }
+}
+
+void Objecter::schedule_tick()
+{
+  assert(tick_event == NULL);
+  tick_event = new C_Tick(this);
+  timer.add_event_after(cct->_conf->objecter_tick_interval, tick_event);
+}
+
+void Objecter::tick()
+{
+  ldout(cct, 10) << "tick" << dendl;
+  assert(client_lock.is_locked());
+  assert(initialized);
+
+  // we are only called by C_Tick
+  assert(tick_event);
+  tick_event = NULL;
+
+  set toping;
+
+  // look for laggy requests
+  utime_t cutoff = ceph_clock_now(cct);
+  cutoff -= cct->_conf->objecter_timeout;  // timeout
+
+  unsigned laggy_ops = 0;
+  for (map::iterator p = ops.begin();
+       p != ops.end();
+       ++p) {
+    Op *op = p->second;
+    if (op->session && op->stamp < cutoff) {
+      ldout(cct, 2) << " tid " << p->first << " on osd." << op->session->osd << " is laggy" << dendl;
+      toping.insert(op->session);
+      ++laggy_ops;
+    }
+  }
+  for (map::iterator p = linger_ops.begin();
+       p != linger_ops.end();
+       ++p) {
+    LingerOp *op = p->second;
+    if (op->session) {
+      ldout(cct, 10) << " pinging osd that serves lingering tid " << p->first << " (osd." << op->session->osd << ")" << dendl;
+      toping.insert(op->session);
+    } else {
+      ldout(cct, 10) << " lingering tid " << p->first << " does not have session" << dendl;
+    }
+  }
+  for (map::iterator p = command_ops.begin();
+       p != command_ops.end();
+       ++p) {
+    CommandOp *op = p->second;
+    if (op->session) {
+      ldout(cct, 10) << " pinging osd that serves command tid " << p->first << " (osd." << op->session->osd << ")" << dendl;
+      toping.insert(op->session);
+    } else {
+      ldout(cct, 10) << " command tid " << p->first << " does not have session" << dendl;
+    }
+  }
+  logger->set(l_osdc_op_laggy, laggy_ops);
+  logger->set(l_osdc_osd_laggy, toping.size());
+
+  if (num_homeless_ops || !toping.empty())
+    maybe_request_map();
+
+  if (!toping.empty()) {
+    // send a ping to these osds, to ensure we detect any session resets
+    // (osd reply message policy is lossy)
+    for (set::iterator i = toping.begin();
+	 i != toping.end();
+	 ++i) {
+      messenger->send_message(new MPing, (*i)->con);
+    }
+  }
+    
+  // reschedule
+  schedule_tick();
+}
+
+void Objecter::resend_mon_ops()
+{
+  assert(client_lock.is_locked());
+  ldout(cct, 10) << "resend_mon_ops" << dendl;
+
+  for (map::iterator p = poolstat_ops.begin(); p!=poolstat_ops.end(); ++p) {
+    poolstat_submit(p->second);
+    logger->inc(l_osdc_poolstat_resend);
+  }
+
+  for (map::iterator p = statfs_ops.begin(); p!=statfs_ops.end(); ++p) {
+    fs_stats_submit(p->second);
+    logger->inc(l_osdc_statfs_resend);
+  }
+
+  for (map::iterator p = pool_ops.begin(); p!=pool_ops.end(); ++p) {
+    _pool_op_submit(p->second);
+    logger->inc(l_osdc_poolop_resend);
+  }
+
+  for (map::iterator p = check_latest_map_ops.begin();
+       p != check_latest_map_ops.end();
+       ++p) {
+    C_Op_Map_Latest *c = new C_Op_Map_Latest(this, p->second->tid);
+    monc->get_version("osdmap", &c->latest, NULL, c);
+  }
+
+  for (map::iterator p = check_latest_map_lingers.begin();
+       p != check_latest_map_lingers.end();
+       ++p) {
+    C_Linger_Map_Latest *c = new C_Linger_Map_Latest(this, p->second->linger_id);
+    monc->get_version("osdmap", &c->latest, NULL, c);
+  }
+
+  for (map::iterator p = check_latest_map_commands.begin();
+       p != check_latest_map_commands.end();
+       ++p) {
+    C_Command_Map_Latest *c = new C_Command_Map_Latest(this, p->second->tid);
+    monc->get_version("osdmap", &c->latest, NULL, c);
+  }
+}
+
+
+
+// read | write ---------------------------
+
+class C_CancelOp : public Context
+{
+  Objecter::Op *op;
+  Objecter *objecter;
+public:
+  C_CancelOp(Objecter::Op *op, Objecter *objecter) : op(op),
+						     objecter(objecter) {}
+  void finish(int r) {
+    // note that objecter lock == timer lock, and is already held
+    objecter->op_cancel(op->tid, -ETIMEDOUT);
+  }
+};
+
+ceph_tid_t Objecter::op_submit(Op *op)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+
+  assert(op->ops.size() == op->out_bl.size());
+  assert(op->ops.size() == op->out_rval.size());
+  assert(op->ops.size() == op->out_handler.size());
+
+  if (osd_timeout > 0) {
+    op->ontimeout = new C_CancelOp(op, this);
+    timer.add_event_after(osd_timeout, op->ontimeout);
+  }
+
+  // throttle.  before we look at any state, because
+  // take_op_budget() may drop our lock while it blocks.
+  take_op_budget(op);
+
+  return _op_submit(op);
+}
+
+ceph_tid_t Objecter::_op_submit(Op *op)
+{
+  // pick tid if we haven't got one yet
+  if (op->tid == ceph_tid_t(0)) {
+    ceph_tid_t mytid = ++last_tid;
+    op->tid = mytid;
+  }
+  assert(client_inc >= 0);
+
+  // pick target
+  num_homeless_ops++;  // initially; recalc_op_target() will decrement if it finds a target
+  int r = recalc_op_target(op);
+  bool check_for_latest_map = (r == RECALC_OP_TARGET_POOL_DNE);
+
+  // add to gather set(s)
+  if (op->onack) {
+    ++num_unacked;
+  } else {
+    ldout(cct, 20) << " note: not requesting ack" << dendl;
+  }
+  if (op->oncommit) {
+    ++num_uncommitted;
+  } else {
+    ldout(cct, 20) << " note: not requesting commit" << dendl;
+  }
+  ops[op->tid] = op;
+
+  logger->set(l_osdc_op_active, ops.size());
+
+  logger->inc(l_osdc_op);
+  if ((op->target.flags & (CEPH_OSD_FLAG_READ|CEPH_OSD_FLAG_WRITE)) == (CEPH_OSD_FLAG_READ|CEPH_OSD_FLAG_WRITE))
+    logger->inc(l_osdc_op_rmw);
+  else if (op->target.flags & CEPH_OSD_FLAG_WRITE)
+    logger->inc(l_osdc_op_w);
+  else if (op->target.flags & CEPH_OSD_FLAG_READ)
+    logger->inc(l_osdc_op_r);
+
+  if (op->target.flags & CEPH_OSD_FLAG_PGOP)
+    logger->inc(l_osdc_op_pg);
+
+  for (vector::iterator p = op->ops.begin(); p != op->ops.end(); ++p) {
+    int code = l_osdc_osdop_other;
+    switch (p->op.op) {
+    case CEPH_OSD_OP_STAT: code = l_osdc_osdop_stat; break;
+    case CEPH_OSD_OP_CREATE: code = l_osdc_osdop_create; break;
+    case CEPH_OSD_OP_READ: code = l_osdc_osdop_read; break;
+    case CEPH_OSD_OP_WRITE: code = l_osdc_osdop_write; break;
+    case CEPH_OSD_OP_WRITEFULL: code = l_osdc_osdop_writefull; break;
+    case CEPH_OSD_OP_APPEND: code = l_osdc_osdop_append; break;
+    case CEPH_OSD_OP_ZERO: code = l_osdc_osdop_zero; break;
+    case CEPH_OSD_OP_TRUNCATE: code = l_osdc_osdop_truncate; break;
+    case CEPH_OSD_OP_DELETE: code = l_osdc_osdop_delete; break;
+    case CEPH_OSD_OP_MAPEXT: code = l_osdc_osdop_mapext; break;
+    case CEPH_OSD_OP_SPARSE_READ: code = l_osdc_osdop_sparse_read; break;
+    case CEPH_OSD_OP_CLONERANGE: code = l_osdc_osdop_clonerange; break;
+    case CEPH_OSD_OP_GETXATTR: code = l_osdc_osdop_getxattr; break;
+    case CEPH_OSD_OP_SETXATTR: code = l_osdc_osdop_setxattr; break;
+    case CEPH_OSD_OP_CMPXATTR: code = l_osdc_osdop_cmpxattr; break;
+    case CEPH_OSD_OP_RMXATTR: code = l_osdc_osdop_rmxattr; break;
+    case CEPH_OSD_OP_RESETXATTRS: code = l_osdc_osdop_resetxattrs; break;
+    case CEPH_OSD_OP_TMAPUP: code = l_osdc_osdop_tmap_up; break;
+    case CEPH_OSD_OP_TMAPPUT: code = l_osdc_osdop_tmap_put; break;
+    case CEPH_OSD_OP_TMAPGET: code = l_osdc_osdop_tmap_get; break;
+    case CEPH_OSD_OP_CALL: code = l_osdc_osdop_call; break;
+    case CEPH_OSD_OP_WATCH: code = l_osdc_osdop_watch; break;
+    case CEPH_OSD_OP_NOTIFY: code = l_osdc_osdop_notify; break;
+    case CEPH_OSD_OP_SRC_CMPXATTR: code = l_osdc_osdop_src_cmpxattr; break;
+    }
+    if (code)
+      logger->inc(code);
+  }
+
+  // send?
+  ldout(cct, 10) << "op_submit oid " << op->target.base_oid
+           << " " << op->target.base_oloc << " " << op->target.target_oloc
+	   << " " << op->ops << " tid " << op->tid
+           << " osd." << (op->session ? op->session->osd : -1)
+           << dendl;
+
+  assert(op->target.flags & (CEPH_OSD_FLAG_READ|CEPH_OSD_FLAG_WRITE));
+
+  if ((op->target.flags & CEPH_OSD_FLAG_WRITE) &&
+      osdmap->test_flag(CEPH_OSDMAP_PAUSEWR)) {
+    ldout(cct, 10) << " paused modify " << op << " tid " << last_tid << dendl;
+    op->target.paused = true;
+    maybe_request_map();
+  } else if ((op->target.flags & CEPH_OSD_FLAG_READ) &&
+	     osdmap->test_flag(CEPH_OSDMAP_PAUSERD)) {
+    ldout(cct, 10) << " paused read " << op << " tid " << last_tid << dendl;
+    op->target.paused = true;
+    maybe_request_map();
+  } else if ((op->target.flags & CEPH_OSD_FLAG_WRITE) && osdmap_full_flag()) {
+    ldout(cct, 0) << " FULL, paused modify " << op << " tid " << last_tid << dendl;
+    op->target.paused = true;
+    maybe_request_map();
+  } else if (op->session) {
+    send_op(op);
+  } else {
+    maybe_request_map();
+  }
+
+  if (check_for_latest_map) {
+    _send_op_map_check(op);
+  }
+
+  ldout(cct, 5) << num_unacked << " unacked, " << num_uncommitted << " uncommitted" << dendl;
+  
+  return op->tid;
+}
+
+int Objecter::op_cancel(ceph_tid_t tid, int r)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+
+  map::iterator p = ops.find(tid);
+  if (p == ops.end()) {
+    ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+    return -ENOENT;
+  }
+
+  ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+  Op *op = p->second;
+  if (op->con) {
+    ldout(cct, 20) << " revoking rx buffer for " << tid
+		   << " on " << op->con << dendl;
+    op->con->revoke_rx_buffer(tid);
+  }
+  if (op->onack) {
+    op->onack->complete(r);
+    op->onack = NULL;
+  }
+  if (op->oncommit) {
+    op->oncommit->complete(r);
+    op->oncommit = NULL;
+  }
+  op_cancel_map_check(op);
+  finish_op(op);
+  return 0;
+}
+
+bool Objecter::is_pg_changed(
+  int oldprimary,
+  const vector& oldacting,
+  int newprimary,
+  const vector& newacting,
+  bool any_change)
+{
+  if (OSDMap::primary_changed(
+	oldprimary,
+	oldacting,
+	newprimary,
+	newacting))
+    return true;
+  if (any_change && oldacting != newacting)
+    return true;
+  return false;      // same primary (tho replicas may have changed)
+}
+
+bool Objecter::target_should_be_paused(op_target_t *t)
+{
+  bool pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
+  bool pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || osdmap_full_flag();
+
+  return (t->flags & CEPH_OSD_FLAG_READ && pauserd) ||
+         (t->flags & CEPH_OSD_FLAG_WRITE && pausewr);
+}
+
+
+/**
+ * Wrapper around osdmap->test_flag for special handling of the FULL flag.
+ */
+bool Objecter::osdmap_full_flag() const
+{
+  // Ignore the FULL flag if we are working on behalf of an MDS, in order to permit
+  // MDS journal writes for file deletions.
+  return osdmap->test_flag(CEPH_OSDMAP_FULL) && (messenger->get_myname().type() != entity_name_t::TYPE_MDS);
+}
+
+
+int64_t Objecter::get_object_hash_position(int64_t pool, const string& key,
+					   const string& ns)
+{
+  const pg_pool_t *p = osdmap->get_pg_pool(pool);
+  if (!p)
+    return -ENOENT;
+  return p->hash_key(key, ns);
+}
+
+int64_t Objecter::get_object_pg_hash_position(int64_t pool, const string& key,
+					      const string& ns)
+{
+  const pg_pool_t *p = osdmap->get_pg_pool(pool);
+  if (!p)
+    return -ENOENT;
+  return p->raw_hash_to_pg(p->hash_key(key, ns));
+}
+
+int Objecter::calc_target(op_target_t *t, bool any_change)
+{
+  bool is_read = t->flags & CEPH_OSD_FLAG_READ;
+  bool is_write = t->flags & CEPH_OSD_FLAG_WRITE;
+
+  const pg_pool_t *pi = osdmap->get_pg_pool(t->base_oloc.pool);
+  bool force_resend = false;
+  bool need_check_tiering = false;
+  if (pi && osdmap->get_epoch() == pi->last_force_op_resend) {
+    force_resend = true;
+  }
+  if (t->target_oid.name.empty() || force_resend) {
+    t->target_oid = t->base_oid;
+    need_check_tiering = true;
+  }
+  if (t->target_oloc.empty() || force_resend) {
+    t->target_oloc = t->base_oloc;
+    need_check_tiering = true;
+  }
+  
+  if (need_check_tiering &&
+      (t->flags & CEPH_OSD_FLAG_IGNORE_OVERLAY) == 0) {
+    if (pi) {
+      if (is_read && pi->has_read_tier())
+	t->target_oloc.pool = pi->read_tier;
+      if (is_write && pi->has_write_tier())
+	t->target_oloc.pool = pi->write_tier;
+    }
+  }
+
+  pg_t pgid;
+  if (t->precalc_pgid) {
+    assert(t->base_oid.name.empty()); // make sure this is a listing op
+    ldout(cct, 10) << __func__ << " have " << t->base_pgid << " pool "
+		   << osdmap->have_pg_pool(t->base_pgid.pool()) << dendl;
+    if (!osdmap->have_pg_pool(t->base_pgid.pool()))
+      return RECALC_OP_TARGET_POOL_DNE;
+    pgid = osdmap->raw_pg_to_pg(t->base_pgid);
+  } else {
+    int ret = osdmap->object_locator_to_pg(t->target_oid, t->target_oloc,
+					   pgid);
+    if (ret == -ENOENT)
+      return RECALC_OP_TARGET_POOL_DNE;
+  }
+  int primary;
+  vector acting;
+  osdmap->pg_to_acting_osds(pgid, &acting, &primary);
+
+  bool need_resend = false;
+
+  bool paused = target_should_be_paused(t);
+  if (!paused && paused != t->paused) {
+    t->paused = false;
+    need_resend = true;
+  }
+
+  if (t->pgid != pgid ||
+      is_pg_changed(
+	t->primary, t->acting, primary, acting, t->used_replica || any_change) ||
+      force_resend) {
+    t->pgid = pgid;
+    t->acting = acting;
+    t->primary = primary;
+    ldout(cct, 10) << __func__ << " pgid " << pgid
+		   << " acting " << acting << dendl;
+    t->used_replica = false;
+    if (primary == -1) {
+      t->osd = -1;
+    } else {
+      int osd;
+      bool read = is_read && !is_write;
+      if (read && (t->flags & CEPH_OSD_FLAG_BALANCE_READS)) {
+	int p = rand() % acting.size();
+	if (p)
+	  t->used_replica = true;
+	osd = acting[p];
+	ldout(cct, 10) << " chose random osd." << osd << " of " << acting << dendl;
+      } else if (read && (t->flags & CEPH_OSD_FLAG_LOCALIZE_READS) &&
+		 acting.size() > 1) {
+	// look for a local replica.  prefer the primary if the
+	// distance is the same.
+	int best = -1;
+	int best_locality;
+	for (unsigned i = 0; i < acting.size(); ++i) {
+	  int locality = osdmap->crush->get_common_ancestor_distance(
+		 cct, acting[i], crush_location);
+	  ldout(cct, 20) << __func__ << " localize: rank " << i
+			 << " osd." << acting[i]
+			 << " locality " << locality << dendl;
+	  if (i == 0 ||
+	      (locality >= 0 && best_locality >= 0 &&
+	       locality < best_locality) ||
+	      (best_locality < 0 && locality >= 0)) {
+	    best = i;
+	    best_locality = locality;
+	    if (i)
+	      t->used_replica = true;
+	  }
+	}
+	assert(best >= 0);
+	osd = acting[best];
+      } else {
+	osd = primary;
+      }
+      t->osd = osd;
+    }
+    need_resend = true;
+  }
+  if (need_resend) {
+    return RECALC_OP_TARGET_NEED_RESEND;
+  }
+  return RECALC_OP_TARGET_NO_ACTION;
+}
+
+int Objecter::recalc_op_target(Op *op)
+{
+  int r = calc_target(&op->target);
+  if (r == RECALC_OP_TARGET_NEED_RESEND) {
+    OSDSession *s = NULL;
+    if (op->target.osd >= 0)
+      s = get_session(op->target.osd);
+    if (op->session != s) {
+      if (!op->session)
+	num_homeless_ops--;
+      op->session_item.remove_myself();
+      op->session = s;
+      if (s)
+	s->ops.push_back(&op->session_item);
+      else
+	num_homeless_ops++;
+    }
+  }
+  return r;
+}
+
+bool Objecter::recalc_linger_op_target(LingerOp *linger_op)
+{
+  int r = calc_target(&linger_op->target, true);
+  if (r == RECALC_OP_TARGET_NEED_RESEND) {
+    ldout(cct, 10) << "recalc_linger_op_target tid " << linger_op->linger_id
+		   << " pgid " << linger_op->target.pgid
+		   << " acting " << linger_op->target.acting << dendl;
+    
+    OSDSession *s = linger_op->target.osd != -1 ?
+      get_session(linger_op->target.osd) : NULL;
+    if (linger_op->session != s) {
+      linger_op->session_item.remove_myself();
+      linger_op->session = s;
+      if (s)
+	s->linger_ops.push_back(&linger_op->session_item);
+    }
+  }
+  return r;
+}
+
+void Objecter::cancel_linger_op(Op *op)
+{
+  ldout(cct, 15) << "cancel_op " << op->tid << dendl;
+
+  assert(!op->should_resend);
+  delete op->onack;
+  delete op->oncommit;
+
+  finish_op(op);
+}
+
+void Objecter::finish_op(Op *op)
+{
+  ldout(cct, 15) << "finish_op " << op->tid << dendl;
+
+  op->session_item.remove_myself();
+  if (op->budgeted)
+    put_op_budget(op);
+
+  ops.erase(op->tid);
+  logger->set(l_osdc_op_active, ops.size());
+  assert(check_latest_map_ops.find(op->tid) == check_latest_map_ops.end());
+
+  if (op->ontimeout)
+    timer.cancel_event(op->ontimeout);
+
+  delete op;
+}
+
+void Objecter::send_op(Op *op)
+{
+  ldout(cct, 15) << "send_op " << op->tid << " to osd." << op->session->osd << dendl;
+
+  int flags = op->target.flags;
+  if (op->oncommit)
+    flags |= CEPH_OSD_FLAG_ONDISK;
+  if (op->onack)
+    flags |= CEPH_OSD_FLAG_ACK;
+
+  assert(op->session->con);
+
+  // preallocated rx buffer?
+  if (op->con) {
+    ldout(cct, 20) << " revoking rx buffer for " << op->tid << " on " << op->con << dendl;
+    op->con->revoke_rx_buffer(op->tid);
+  }
+  if (op->outbl && op->outbl->length()) {
+    ldout(cct, 20) << " posting rx buffer for " << op->tid << " on " << op->session->con << dendl;
+    op->con = op->session->con;
+    op->con->post_rx_buffer(op->tid, *op->outbl);
+  }
+
+  op->target.paused = false;
+  op->incarnation = op->session->incarnation;
+  op->stamp = ceph_clock_now(cct);
+
+  MOSDOp *m = new MOSDOp(client_inc, op->tid, 
+			 op->target.target_oid, op->target.target_oloc,
+			 op->target.pgid,
+			 osdmap->get_epoch(),
+			 flags);
+
+  m->set_snapid(op->snapid);
+  m->set_snap_seq(op->snapc.seq);
+  m->set_snaps(op->snapc.snaps);
+
+  m->ops = op->ops;
+  m->set_mtime(op->mtime);
+  m->set_retry_attempt(op->attempts++);
+
+  if (op->replay_version != eversion_t())
+    m->set_version(op->replay_version);  // we're replaying this op!
+
+  if (op->priority)
+    m->set_priority(op->priority);
+  else
+    m->set_priority(cct->_conf->osd_client_op_priority);
+
+  logger->inc(l_osdc_op_send);
+  logger->inc(l_osdc_op_send_bytes, m->get_data().length());
+
+  messenger->send_message(m, op->session->con);
+}
+
+int Objecter::calc_op_budget(Op *op)
+{
+  int op_budget = 0;
+  for (vector::iterator i = op->ops.begin();
+       i != op->ops.end();
+       ++i) {
+    if (i->op.op & CEPH_OSD_OP_MODE_WR) {
+      op_budget += i->indata.length();
+    } else if (ceph_osd_op_mode_read(i->op.op)) {
+      if (ceph_osd_op_type_data(i->op.op)) {
+        if ((int64_t)i->op.extent.length > 0)
+	  op_budget += (int64_t)i->op.extent.length;
+      } else if (ceph_osd_op_type_attr(i->op.op)) {
+        op_budget += i->op.xattr.name_len + i->op.xattr.value_len;
+      }
+    }
+  }
+  return op_budget;
+}
+
+void Objecter::throttle_op(Op *op, int op_budget)
+{
+  if (!op_budget)
+    op_budget = calc_op_budget(op);
+  if (!op_throttle_bytes.get_or_fail(op_budget)) { //couldn't take right now
+    client_lock.Unlock();
+    op_throttle_bytes.get(op_budget);
+    client_lock.Lock();
+  }
+  if (!op_throttle_ops.get_or_fail(1)) { //couldn't take right now
+    client_lock.Unlock();
+    op_throttle_ops.get(1);
+    client_lock.Lock();
+  }
+}
+
+void Objecter::unregister_op(Op *op)
+{
+  if (op->onack)
+    num_unacked--;
+  if (op->oncommit)
+    num_uncommitted--;
+  ops.erase(op->tid);
+}
+
+/* This function DOES put the passed message before returning */
+void Objecter::handle_osd_op_reply(MOSDOpReply *m)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+  ldout(cct, 10) << "in handle_osd_op_reply" << dendl;
+
+  // get pio
+  ceph_tid_t tid = m->get_tid();
+
+  if (ops.count(tid) == 0) {
+    ldout(cct, 7) << "handle_osd_op_reply " << tid
+	    << (m->is_ondisk() ? " ondisk":(m->is_onnvram() ? " onnvram":" ack"))
+	    << " ... stray" << dendl;
+    m->put();
+    return;
+  }
+
+  ldout(cct, 7) << "handle_osd_op_reply " << tid
+		<< (m->is_ondisk() ? " ondisk":(m->is_onnvram() ? " onnvram":" ack"))
+		<< " v " << m->get_replay_version() << " uv " << m->get_user_version()
+		<< " in " << m->get_pg()
+		<< " attempt " << m->get_retry_attempt()
+		<< dendl;
+  Op *op = ops[tid];
+
+  if (m->get_retry_attempt() >= 0) {
+    if (m->get_retry_attempt() != (op->attempts - 1)) {
+      ldout(cct, 7) << " ignoring reply from attempt " << m->get_retry_attempt()
+		    << " from " << m->get_source_inst()
+		    << "; last attempt " << (op->attempts - 1) << " sent to "
+		    << op->session->con->get_peer_addr() << dendl;
+      m->put();
+      return;
+    }
+  } else {
+    // we don't know the request attempt because the server is old, so
+    // just accept this one.  we may do ACK callbacks we shouldn't
+    // have, but that is better than doing callbacks out of order.
+  }
+
+  Context *onack = 0;
+  Context *oncommit = 0;
+
+  int rc = m->get_result();
+
+  if (m->is_redirect_reply()) {
+    ldout(cct, 5) << " got redirect reply; redirecting" << dendl;
+    unregister_op(op);
+    m->get_redirect().combine_with_locator(op->target.target_oloc,
+					   op->target.target_oid.name);
+    _op_submit(op);
+    m->put();
+    return;
+  }
+
+  if (rc == -EAGAIN) {
+    ldout(cct, 7) << " got -EAGAIN, resubmitting" << dendl;
+    unregister_op(op);
+    _op_submit(op);
+    m->put();
+    return;
+  }
+
+  if (op->objver)
+    *op->objver = m->get_user_version();
+  if (op->reply_epoch)
+    *op->reply_epoch = m->get_map_epoch();
+
+  // per-op result demuxing
+  vector out_ops;
+  m->claim_ops(out_ops);
+  
+  if (out_ops.size() != op->ops.size())
+    ldout(cct, 0) << "WARNING: tid " << op->tid << " reply ops " << out_ops
+		  << " != request ops " << op->ops
+		  << " from " << m->get_source_inst() << dendl;
+
+  vector::iterator pb = op->out_bl.begin();
+  vector::iterator pr = op->out_rval.begin();
+  vector::iterator ph = op->out_handler.begin();
+  assert(op->out_bl.size() == op->out_rval.size());
+  assert(op->out_bl.size() == op->out_handler.size());
+  vector::iterator p = out_ops.begin();
+  for (unsigned i = 0;
+       p != out_ops.end() && pb != op->out_bl.end();
+       ++i, ++p, ++pb, ++pr, ++ph) {
+    ldout(cct, 10) << " op " << i << " rval " << p->rval
+		   << " len " << p->outdata.length() << dendl;
+    if (*pb)
+      **pb = p->outdata;
+    // set rval before running handlers so that handlers
+    // can change it if e.g. decoding fails
+    if (*pr)
+      **pr = p->rval;
+    if (*ph) {
+      ldout(cct, 10) << " op " << i << " handler " << *ph << dendl;
+      (*ph)->complete(p->rval);
+      *ph = NULL;
+    }
+  }
+
+  // ack|commit -> ack
+  if (op->onack) {
+    ldout(cct, 15) << "handle_osd_op_reply ack" << dendl;
+    op->replay_version = m->get_replay_version();
+    onack = op->onack;
+    op->onack = 0;  // only do callback once
+    num_unacked--;
+    logger->inc(l_osdc_op_ack);
+  }
+  if (op->oncommit && (m->is_ondisk() || rc)) {
+    ldout(cct, 15) << "handle_osd_op_reply safe" << dendl;
+    oncommit = op->oncommit;
+    op->oncommit = 0;
+    num_uncommitted--;
+    logger->inc(l_osdc_op_commit);
+  }
+
+  // got data?
+  if (op->outbl) {
+    if (op->con)
+      op->con->revoke_rx_buffer(op->tid);
+    m->claim_data(*op->outbl);
+    op->outbl = 0;
+  }
+
+  // done with this tid?
+  if (!op->onack && !op->oncommit) {
+    ldout(cct, 15) << "handle_osd_op_reply completed tid " << tid << dendl;
+    finish_op(op);
+  }
+  
+  ldout(cct, 5) << num_unacked << " unacked, " << num_uncommitted << " uncommitted" << dendl;
+
+  // do callbacks
+  if (onack) {
+    onack->complete(rc);
+  }
+  if (oncommit) {
+    oncommit->complete(rc);
+  }
+
+  m->put();
+}
+
+
+uint32_t Objecter::list_objects_seek(ListContext *list_context,
+				     uint32_t pos)
+{
+  assert(client_lock.is_locked());
+  pg_t actual = osdmap->raw_pg_to_pg(pg_t(pos, list_context->pool_id));
+  ldout(cct, 10) << "list_objects_seek " << list_context
+		 << " pos " << pos << " -> " << actual << dendl;
+  list_context->current_pg = actual.ps();
+  list_context->cookie = collection_list_handle_t();
+  list_context->at_end_of_pg = false;
+  list_context->at_end_of_pool = false;
+  list_context->current_pg_epoch = 0;
+  return list_context->current_pg;
+}
+
+void Objecter::list_objects(ListContext *list_context, Context *onfinish)
+{
+  assert(client_lock.is_locked());
+  ldout(cct, 10) << "list_objects" << dendl;
+  ldout(cct, 20) << " pool_id " << list_context->pool_id
+	   << " pool_snap_seq " << list_context->pool_snap_seq
+	   << " max_entries " << list_context->max_entries
+	   << " list_context " << list_context
+	   << " onfinish " << onfinish
+	   << " list_context->current_pg " << list_context->current_pg
+	   << " list_context->cookie " << list_context->cookie << dendl;
+
+  if (list_context->at_end_of_pg) {
+    list_context->at_end_of_pg = false;
+    ++list_context->current_pg;
+    list_context->current_pg_epoch = 0;
+    list_context->cookie = collection_list_handle_t();
+    if (list_context->current_pg >= list_context->starting_pg_num) {
+      list_context->at_end_of_pool = true;
+      ldout(cct, 20) << " no more pgs; reached end of pool" << dendl;
+    } else {
+      ldout(cct, 20) << " move to next pg " << list_context->current_pg << dendl;
+    }
+  }
+  if (list_context->at_end_of_pool) {
+    onfinish->complete(0);
+    return;
+  }
+
+  const pg_pool_t *pool = osdmap->get_pg_pool(list_context->pool_id);
+  int pg_num = pool->get_pg_num();
+
+  if (list_context->starting_pg_num == 0) {     // there can't be zero pgs!
+    list_context->starting_pg_num = pg_num;
+    ldout(cct, 20) << pg_num << " placement groups" << dendl;
+  }
+  if (list_context->starting_pg_num != pg_num) {
+    // start reading from the beginning; the pgs have changed
+    ldout(cct, 10) << " pg_num changed; restarting with " << pg_num << dendl;
+    list_context->current_pg = 0;
+    list_context->cookie = collection_list_handle_t();
+    list_context->current_pg_epoch = 0;
+    list_context->starting_pg_num = pg_num;
+  }
+  assert(list_context->current_pg <= pg_num);
+
+  ObjectOperation op;
+  op.pg_ls(list_context->max_entries, list_context->filter, list_context->cookie,
+	   list_context->current_pg_epoch);
+  list_context->bl.clear();
+  C_List *onack = new C_List(list_context, onfinish, this);
+  object_locator_t oloc(list_context->pool_id, list_context->nspace);
+  pg_read(list_context->current_pg, oloc, op,
+	  &list_context->bl, 0, onack, &onack->epoch);
+}
+
+void Objecter::_list_reply(ListContext *list_context, int r,
+			   Context *final_finish, epoch_t reply_epoch)
+{
+  ldout(cct, 10) << "_list_reply" << dendl;
+
+  bufferlist::iterator iter = list_context->bl.begin();
+  pg_ls_response_t response;
+  bufferlist extra_info;
+  ::decode(response, iter);
+  if (!iter.end()) {
+    ::decode(extra_info, iter);
+  }
+  list_context->cookie = response.handle;
+  if (!list_context->current_pg_epoch) {
+    // first pgls result, set epoch marker
+    ldout(cct, 20) << " first pgls piece, reply_epoch is "
+		   << reply_epoch << dendl;
+    list_context->current_pg_epoch = reply_epoch;
+  }
+
+  int response_size = response.entries.size();
+  ldout(cct, 20) << " response.entries.size " << response_size
+		 << ", response.entries " << response.entries << dendl;
+  list_context->extra_info.append(extra_info);
+  if (response_size) {
+    list_context->list.merge(response.entries);
+  }
+
+  // if the osd returns 1 (newer code), or no entries, it means we
+  // hit the end of the pg.
+  if (response_size == 0 || r == 1) {
+    ldout(cct, 20) << " at end of pg" << dendl;
+    list_context->at_end_of_pg = true;
+  } else {
+    // there is more for this pg; get it?
+    if (response_size < list_context->max_entries) {
+      list_context->max_entries -= response_size;
+      list_objects(list_context, final_finish);
+      return;
+    }
+  }
+  if (!list_context->list.empty()) {
+    ldout(cct, 20) << " returning results so far" << dendl;
+    final_finish->complete(0);
+    return;
+  }
+
+  // continue!
+  list_objects(list_context, final_finish);
+}
+
+
+
+//snapshots
+
+int Objecter::create_pool_snap(int64_t pool, string& snap_name, Context *onfinish)
+{
+  ldout(cct, 10) << "create_pool_snap; pool: " << pool << "; snap: " << snap_name << dendl;
+
+  const pg_pool_t *p = osdmap->get_pg_pool(pool);
+  if (!p)
+    return -EINVAL;
+  if (p->snap_exists(snap_name.c_str()))
+    return -EEXIST;
+
+  PoolOp *op = new PoolOp;
+  if (!op)
+    return -ENOMEM;
+  op->tid = ++last_tid;
+  op->pool = pool;
+  op->name = snap_name;
+  op->onfinish = onfinish;
+  op->pool_op = POOL_OP_CREATE_SNAP;
+  pool_ops[op->tid] = op;
+
+  pool_op_submit(op);
+
+  return 0;
+}
+
+struct C_SelfmanagedSnap : public Context {
+  bufferlist bl;
+  snapid_t *psnapid;
+  Context *fin;
+  C_SelfmanagedSnap(snapid_t *ps, Context *f) : psnapid(ps), fin(f) {}
+  void finish(int r) {
+    if (r == 0) {
+      bufferlist::iterator p = bl.begin();
+      ::decode(*psnapid, p);
+    }
+    fin->complete(r);
+  }
+};
+
+int Objecter::allocate_selfmanaged_snap(int64_t pool, snapid_t *psnapid,
+					Context *onfinish)
+{
+  ldout(cct, 10) << "allocate_selfmanaged_snap; pool: " << pool << dendl;
+  PoolOp *op = new PoolOp;
+  if (!op) return -ENOMEM;
+  op->tid = ++last_tid;
+  op->pool = pool;
+  C_SelfmanagedSnap *fin = new C_SelfmanagedSnap(psnapid, onfinish);
+  op->onfinish = fin;
+  op->blp = &fin->bl;
+  op->pool_op = POOL_OP_CREATE_UNMANAGED_SNAP;
+  pool_ops[op->tid] = op;
+
+  pool_op_submit(op);
+  return 0;
+}
+
+int Objecter::delete_pool_snap(int64_t pool, string& snap_name, Context *onfinish)
+{
+  ldout(cct, 10) << "delete_pool_snap; pool: " << pool << "; snap: " << snap_name << dendl;
+
+  const pg_pool_t *p = osdmap->get_pg_pool(pool);
+  if (!p)
+    return -EINVAL;
+  if (!p->snap_exists(snap_name.c_str()))
+    return -ENOENT;
+
+  PoolOp *op = new PoolOp;
+  if (!op)
+    return -ENOMEM;
+  op->tid = ++last_tid;
+  op->pool = pool;
+  op->name = snap_name;
+  op->onfinish = onfinish;
+  op->pool_op = POOL_OP_DELETE_SNAP;
+  pool_ops[op->tid] = op;
+  
+  pool_op_submit(op);
+  
+  return 0;
+}
+
+int Objecter::delete_selfmanaged_snap(int64_t pool, snapid_t snap,
+				      Context *onfinish) {
+  ldout(cct, 10) << "delete_selfmanaged_snap; pool: " << pool << "; snap: " 
+	   << snap << dendl;
+  PoolOp *op = new PoolOp;
+  if (!op) return -ENOMEM;
+  op->tid = ++last_tid;
+  op->pool = pool;
+  op->onfinish = onfinish;
+  op->pool_op = POOL_OP_DELETE_UNMANAGED_SNAP;
+  op->snapid = snap;
+  pool_ops[op->tid] = op;
+
+  pool_op_submit(op);
+
+  return 0;
+}
+
+int Objecter::create_pool(string& name, Context *onfinish, uint64_t auid,
+			  int crush_rule)
+{
+  ldout(cct, 10) << "create_pool name=" << name << dendl;
+
+  if (osdmap->lookup_pg_pool_name(name.c_str()) >= 0)
+    return -EEXIST;
+
+  PoolOp *op = new PoolOp;
+  if (!op)
+    return -ENOMEM;
+  op->tid = ++last_tid;
+  op->pool = 0;
+  op->name = name;
+  op->onfinish = onfinish;
+  op->pool_op = POOL_OP_CREATE;
+  pool_ops[op->tid] = op;
+  op->auid = auid;
+  op->crush_rule = crush_rule;
+
+  pool_op_submit(op);
+
+  return 0;
+}
+
+int Objecter::delete_pool(int64_t pool, Context *onfinish)
+{
+  ldout(cct, 10) << "delete_pool " << pool << dendl;
+
+  if (!osdmap->have_pg_pool(pool))
+    return -ENOENT;
+
+  PoolOp *op = new PoolOp;
+  if (!op) return -ENOMEM;
+  op->tid = ++last_tid;
+  op->pool = pool;
+  op->name = "delete";
+  op->onfinish = onfinish;
+  op->pool_op = POOL_OP_DELETE;
+  pool_ops[op->tid] = op;
+
+  pool_op_submit(op);
+
+  return 0;
+}
+
+/**
+ * change the auid owner of a pool by contacting the monitor.
+ * This requires the current connection to have write permissions
+ * on both the pool's current auid and the new (parameter) auid.
+ * Uses the standard Context callback when done.
+ */
+int Objecter::change_pool_auid(int64_t pool, Context *onfinish, uint64_t auid)
+{
+  ldout(cct, 10) << "change_pool_auid " << pool << " to " << auid << dendl;
+  PoolOp *op = new PoolOp;
+  if (!op) return -ENOMEM;
+  op->tid = ++last_tid;
+  op->pool = pool;
+  op->name = "change_pool_auid";
+  op->onfinish = onfinish;
+  op->pool_op = POOL_OP_AUID_CHANGE;
+  op->auid = auid;
+  pool_ops[op->tid] = op;
+
+  logger->set(l_osdc_poolop_active, pool_ops.size());
+
+  pool_op_submit(op);
+  return 0;
+}
+
+class C_CancelPoolOp : public Context
+{
+  ceph_tid_t tid;
+  Objecter *objecter;
+public:
+  C_CancelPoolOp(ceph_tid_t tid, Objecter *objecter) : tid(tid),
+						       objecter(objecter) {}
+  void finish(int r) {
+    // note that objecter lock == timer lock, and is already held
+    objecter->pool_op_cancel(tid, -ETIMEDOUT);
+  }
+};
+
+void Objecter::pool_op_submit(PoolOp *op)
+{
+  if (mon_timeout > 0) {
+    op->ontimeout = new C_CancelPoolOp(op->tid, this);
+    timer.add_event_after(mon_timeout, op->ontimeout);
+  }
+  _pool_op_submit(op);
+}
+
+void Objecter::_pool_op_submit(PoolOp *op)
+{
+  ldout(cct, 10) << "pool_op_submit " << op->tid << dendl;
+  MPoolOp *m = new MPoolOp(monc->get_fsid(), op->tid, op->pool,
+			   op->name, op->pool_op,
+			   op->auid, last_seen_osdmap_version);
+  if (op->snapid) m->snapid = op->snapid;
+  if (op->crush_rule) m->crush_rule = op->crush_rule;
+  monc->send_mon_message(m);
+  op->last_submit = ceph_clock_now(cct);
+
+  logger->inc(l_osdc_poolop_send);
+}
+
+/**
+ * Handle a reply to a PoolOp message. Check that we sent the message
+ * and give the caller responsibility for the returned bufferlist.
+ * Then either call the finisher or stash the PoolOp, depending on if we
+ * have a new enough map.
+ * Lastly, clean up the message and PoolOp.
+ */
+void Objecter::handle_pool_op_reply(MPoolOpReply *m)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+  ldout(cct, 10) << "handle_pool_op_reply " << *m << dendl;
+  ceph_tid_t tid = m->get_tid();
+  if (pool_ops.count(tid)) {
+    PoolOp *op = pool_ops[tid];
+    ldout(cct, 10) << "have request " << tid << " at " << op << " Op: " << ceph_pool_op_name(op->pool_op) << dendl;
+    if (op->blp)
+      op->blp->claim(m->response_data);
+    if (m->version > last_seen_osdmap_version)
+      last_seen_osdmap_version = m->version;
+    if (osdmap->get_epoch() < m->epoch) {
+      ldout(cct, 20) << "waiting for client to reach epoch " << m->epoch << " before calling back" << dendl;
+      wait_for_new_map(op->onfinish, m->epoch, m->replyCode);
+    }
+    else {
+      op->onfinish->complete(m->replyCode);
+    }
+    op->onfinish = NULL;
+    finish_pool_op(op);
+  } else {
+    ldout(cct, 10) << "unknown request " << tid << dendl;
+  }
+  ldout(cct, 10) << "done" << dendl;
+  m->put();
+}
+
+int Objecter::pool_op_cancel(ceph_tid_t tid, int r)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+
+  map::iterator it = pool_ops.find(tid);
+  if (it == pool_ops.end()) {
+    ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+    return -ENOENT;
+  }
+
+  ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+  PoolOp *op = it->second;
+  if (op->onfinish)
+    op->onfinish->complete(r);
+  finish_pool_op(op);
+  return 0;
+}
+
+void Objecter::finish_pool_op(PoolOp *op)
+{
+  pool_ops.erase(op->tid);
+  logger->set(l_osdc_poolop_active, pool_ops.size());
+
+  if (op->ontimeout)
+    timer.cancel_event(op->ontimeout);
+
+  delete op;
+}
+
+// pool stats
+
+class C_CancelPoolStatOp : public Context
+{
+  ceph_tid_t tid;
+  Objecter *objecter;
+public:
+  C_CancelPoolStatOp(ceph_tid_t tid, Objecter *objecter) : tid(tid),
+						           objecter(objecter) {}
+  void finish(int r) {
+    // note that objecter lock == timer lock, and is already held
+    objecter->pool_stat_op_cancel(tid, -ETIMEDOUT);
+  }
+};
+
+void Objecter::get_pool_stats(list& pools, map *result,
+			      Context *onfinish)
+{
+  ldout(cct, 10) << "get_pool_stats " << pools << dendl;
+
+  PoolStatOp *op = new PoolStatOp;
+  op->tid = ++last_tid;
+  op->pools = pools;
+  op->pool_stats = result;
+  op->onfinish = onfinish;
+  op->ontimeout = NULL;
+  if (mon_timeout > 0) {
+    op->ontimeout = new C_CancelPoolStatOp(op->tid, this);
+    timer.add_event_after(mon_timeout, op->ontimeout);
+  }
+  poolstat_ops[op->tid] = op;
+
+  logger->set(l_osdc_poolstat_active, poolstat_ops.size());
+
+  poolstat_submit(op);
+}
+
+void Objecter::poolstat_submit(PoolStatOp *op)
+{
+  ldout(cct, 10) << "poolstat_submit " << op->tid << dendl;
+  monc->send_mon_message(new MGetPoolStats(monc->get_fsid(), op->tid, op->pools, last_seen_pgmap_version));
+  op->last_submit = ceph_clock_now(cct);
+
+  logger->inc(l_osdc_poolstat_send);
+}
+
+void Objecter::handle_get_pool_stats_reply(MGetPoolStatsReply *m)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+  ldout(cct, 10) << "handle_get_pool_stats_reply " << *m << dendl;
+  ceph_tid_t tid = m->get_tid();
+
+  if (poolstat_ops.count(tid)) {
+    PoolStatOp *op = poolstat_ops[tid];
+    ldout(cct, 10) << "have request " << tid << " at " << op << dendl;
+    *op->pool_stats = m->pool_stats;
+    if (m->version > last_seen_pgmap_version)
+      last_seen_pgmap_version = m->version;
+    op->onfinish->complete(0);
+    finish_pool_stat_op(op);
+  } else {
+    ldout(cct, 10) << "unknown request " << tid << dendl;
+  } 
+  ldout(cct, 10) << "done" << dendl;
+  m->put();
+}
+
+int Objecter::pool_stat_op_cancel(ceph_tid_t tid, int r)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+
+  map::iterator it = poolstat_ops.find(tid);
+  if (it == poolstat_ops.end()) {
+    ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+    return -ENOENT;
+  }
+
+  ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+  PoolStatOp *op = it->second;
+  if (op->onfinish)
+    op->onfinish->complete(r);
+  finish_pool_stat_op(op);
+  return 0;
+}
+
+void Objecter::finish_pool_stat_op(PoolStatOp *op)
+{
+  poolstat_ops.erase(op->tid);
+  logger->set(l_osdc_poolstat_active, poolstat_ops.size());
+
+  if (op->ontimeout)
+    timer.cancel_event(op->ontimeout);
+
+  delete op;
+}
+
+class C_CancelStatfsOp : public Context
+{
+  ceph_tid_t tid;
+  Objecter *objecter;
+public:
+  C_CancelStatfsOp(ceph_tid_t tid, Objecter *objecter) : tid(tid),
+							 objecter(objecter) {}
+  void finish(int r) {
+    // note that objecter lock == timer lock, and is already held
+    objecter->statfs_op_cancel(tid, -ETIMEDOUT);
+  }
+};
+
+void Objecter::get_fs_stats(ceph_statfs& result, Context *onfinish)
+{
+  ldout(cct, 10) << "get_fs_stats" << dendl;
+
+  StatfsOp *op = new StatfsOp;
+  op->tid = ++last_tid;
+  op->stats = &result;
+  op->onfinish = onfinish;
+  op->ontimeout = NULL;
+  if (mon_timeout > 0) {
+    op->ontimeout = new C_CancelStatfsOp(op->tid, this);
+    timer.add_event_after(mon_timeout, op->ontimeout);
+  }
+  statfs_ops[op->tid] = op;
+
+  logger->set(l_osdc_statfs_active, statfs_ops.size());
+
+  fs_stats_submit(op);
+}
+
+void Objecter::fs_stats_submit(StatfsOp *op)
+{
+  ldout(cct, 10) << "fs_stats_submit" << op->tid << dendl;
+  monc->send_mon_message(new MStatfs(monc->get_fsid(), op->tid, last_seen_pgmap_version));
+  op->last_submit = ceph_clock_now(cct);
+
+  logger->inc(l_osdc_statfs_send);
+}
+
+void Objecter::handle_fs_stats_reply(MStatfsReply *m)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+  ldout(cct, 10) << "handle_fs_stats_reply " << *m << dendl;
+  ceph_tid_t tid = m->get_tid();
+
+  if (statfs_ops.count(tid)) {
+    StatfsOp *op = statfs_ops[tid];
+    ldout(cct, 10) << "have request " << tid << " at " << op << dendl;
+    *(op->stats) = m->h.st;
+    if (m->h.version > last_seen_pgmap_version)
+      last_seen_pgmap_version = m->h.version;
+    op->onfinish->complete(0);
+    finish_statfs_op(op);
+  } else {
+    ldout(cct, 10) << "unknown request " << tid << dendl;
+  }
+  ldout(cct, 10) << "done" << dendl;
+  m->put();
+}
+
+int Objecter::statfs_op_cancel(ceph_tid_t tid, int r)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+
+  map::iterator it = statfs_ops.find(tid);
+  if (it == statfs_ops.end()) {
+    ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+    return -ENOENT;
+  }
+
+  ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+  StatfsOp *op = it->second;
+  if (op->onfinish)
+    op->onfinish->complete(r);
+  finish_statfs_op(op);
+  return 0;
+}
+
+void Objecter::finish_statfs_op(StatfsOp *op)
+{
+  statfs_ops.erase(op->tid);
+  logger->set(l_osdc_statfs_active, statfs_ops.size());
+
+  if (op->ontimeout)
+    timer.cancel_event(op->ontimeout);
+
+  delete op;
+}
+
+// scatter/gather
+
+void Objecter::_sg_read_finish(vector& extents, vector& resultbl, 
+			       bufferlist *bl, Context *onfinish)
+{
+  // all done
+  ldout(cct, 15) << "_sg_read_finish" << dendl;
+
+  if (extents.size() > 1) {
+    Striper::StripedReadResult r;
+    vector::iterator bit = resultbl.begin();
+    for (vector::iterator eit = extents.begin();
+	 eit != extents.end();
+	 ++eit, ++bit) {
+      r.add_partial_result(cct, *bit, eit->buffer_extents);
+    }
+    bl->clear();
+    r.assemble_result(cct, *bl, false);
+  } else {
+    ldout(cct, 15) << "  only one frag" << dendl;
+    bl->claim(resultbl[0]);
+  }
+
+  // done
+  uint64_t bytes_read = bl->length();
+  ldout(cct, 7) << "_sg_read_finish " << bytes_read << " bytes" << dendl;
+
+  if (onfinish) {
+    onfinish->complete(bytes_read);// > 0 ? bytes_read:m->get_result());
+  }
+}
+
+
+void Objecter::ms_handle_connect(Connection *con)
+{
+  ldout(cct, 10) << "ms_handle_connect " << con << dendl;
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON)
+    resend_mon_ops();
+}
+
+void Objecter::ms_handle_reset(Connection *con)
+{
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_OSD) {
+    //
+    int osd = osdmap->identify_osd(con->get_peer_addr());
+    if (osd >= 0) {
+      ldout(cct, 1) << "ms_handle_reset on osd." << osd << dendl;
+      map::iterator p = osd_sessions.find(osd);
+      if (p != osd_sessions.end()) {
+	OSDSession *session = p->second;
+	reopen_session(session);
+	kick_requests(session);
+	maybe_request_map();
+      }
+    } else {
+      ldout(cct, 10) << "ms_handle_reset on unknown osd addr " << con->get_peer_addr() << dendl;
+    }
+  }
+}
+
+void Objecter::ms_handle_remote_reset(Connection *con)
+{
+  /*
+   * treat these the same.
+   */
+  ms_handle_reset(con);
+}
+
+
+void Objecter::op_target_t::dump(Formatter *f) const
+{
+  f->dump_stream("pg") << pgid;
+  f->dump_int("osd", osd);
+  f->dump_stream("object_id") << base_oid;
+  f->dump_stream("object_locator") << base_oloc;
+  f->dump_stream("target_object_id") << target_oid;
+  f->dump_stream("target_object_locator") << target_oloc;
+  f->dump_int("paused", (int)paused);
+  f->dump_int("used_replica", (int)used_replica);
+  f->dump_int("precalc_pgid", (int)precalc_pgid);
+}
+
+void Objecter::dump_active()
+{
+  ldout(cct, 20) << "dump_active .. " << num_homeless_ops << " homeless" << dendl;
+  for (map::iterator p = ops.begin(); p != ops.end(); ++p) {
+    Op *op = p->second;
+    ldout(cct, 20) << op->tid << "\t" << op->target.pgid
+		   << "\tosd." << (op->session ? op->session->osd : -1)
+		   << "\t" << op->target.base_oid
+		   << "\t" << op->ops << dendl;
+  }
+}
+
+void Objecter::dump_requests(Formatter *fmt) const
+{
+  assert(client_lock.is_locked());
+
+  fmt->open_object_section("requests");
+  dump_ops(fmt);
+  dump_linger_ops(fmt);
+  dump_pool_ops(fmt);
+  dump_pool_stat_ops(fmt);
+  dump_statfs_ops(fmt);
+  dump_command_ops(fmt);
+  fmt->close_section(); // requests object
+}
+
+void Objecter::dump_ops(Formatter *fmt) const
+{
+  fmt->open_array_section("ops");
+  for (map::const_iterator p = ops.begin();
+       p != ops.end();
+       ++p) {
+    Op *op = p->second;
+    fmt->open_object_section("op");
+    fmt->dump_unsigned("tid", op->tid);
+    op->target.dump(fmt);
+    fmt->dump_stream("last_sent") << op->stamp;
+    fmt->dump_int("attempts", op->attempts);
+    fmt->dump_stream("snapid") << op->snapid;
+    fmt->dump_stream("snap_context") << op->snapc;
+    fmt->dump_stream("mtime") << op->mtime;
+
+    fmt->open_array_section("osd_ops");
+    for (vector::const_iterator it = op->ops.begin();
+	 it != op->ops.end();
+	 ++it) {
+      fmt->dump_stream("osd_op") << *it;
+    }
+    fmt->close_section(); // osd_ops array
+
+    fmt->close_section(); // op object
+  }
+  fmt->close_section(); // ops array
+}
+
+void Objecter::dump_linger_ops(Formatter *fmt) const
+{
+  fmt->open_array_section("linger_ops");
+  for (map::const_iterator p = linger_ops.begin();
+       p != linger_ops.end();
+       ++p) {
+    LingerOp *op = p->second;
+    fmt->open_object_section("linger_op");
+    fmt->dump_unsigned("linger_id", op->linger_id);
+    op->target.dump(fmt);
+    fmt->dump_stream("snapid") << op->snap;
+    fmt->dump_stream("registered") << op->registered;
+    fmt->close_section(); // linger_op object
+  }
+  fmt->close_section(); // linger_ops array
+}
+
+void Objecter::dump_command_ops(Formatter *fmt) const
+{
+  fmt->open_array_section("command_ops");
+  for (map::const_iterator p = command_ops.begin();
+       p != command_ops.end();
+       ++p) {
+    CommandOp *op = p->second;
+    fmt->open_object_section("command_op");
+    fmt->dump_unsigned("command_id", op->tid);
+    fmt->dump_int("osd", op->session ? op->session->osd : -1);
+    fmt->open_array_section("command");
+    for (vector::const_iterator q = op->cmd.begin(); q != op->cmd.end(); ++q)
+      fmt->dump_string("word", *q);
+    fmt->close_section();
+    if (op->target_osd >= 0)
+      fmt->dump_int("target_osd", op->target_osd);
+    else
+      fmt->dump_stream("target_pg") << op->target_pg;
+    fmt->close_section(); // command_op object
+  }
+  fmt->close_section(); // command_ops array
+}
+
+void Objecter::dump_pool_ops(Formatter *fmt) const
+{
+  fmt->open_array_section("pool_ops");
+  for (map::const_iterator p = pool_ops.begin();
+       p != pool_ops.end();
+       ++p) {
+    PoolOp *op = p->second;
+    fmt->open_object_section("pool_op");
+    fmt->dump_unsigned("tid", op->tid);
+    fmt->dump_int("pool", op->pool);
+    fmt->dump_string("name", op->name);
+    fmt->dump_int("operation_type", op->pool_op);
+    fmt->dump_unsigned("auid", op->auid);
+    fmt->dump_unsigned("crush_rule", op->crush_rule);
+    fmt->dump_stream("snapid") << op->snapid;
+    fmt->dump_stream("last_sent") << op->last_submit;
+    fmt->close_section(); // pool_op object
+  }
+  fmt->close_section(); // pool_ops array
+}
+
+void Objecter::dump_pool_stat_ops(Formatter *fmt) const
+{
+  fmt->open_array_section("pool_stat_ops");
+  for (map::const_iterator p = poolstat_ops.begin();
+       p != poolstat_ops.end();
+       ++p) {
+    PoolStatOp *op = p->second;
+    fmt->open_object_section("pool_stat_op");
+    fmt->dump_unsigned("tid", op->tid);
+    fmt->dump_stream("last_sent") << op->last_submit;
+
+    fmt->open_array_section("pools");
+    for (list::const_iterator it = op->pools.begin();
+	 it != op->pools.end();
+	 ++it) {
+      fmt->dump_string("pool", *it);
+    }
+    fmt->close_section(); // pool_op object
+
+    fmt->close_section(); // pool_stat_op object
+  }
+  fmt->close_section(); // pool_stat_ops array
+}
+
+void Objecter::dump_statfs_ops(Formatter *fmt) const
+{
+  fmt->open_array_section("statfs_ops");
+  for (map::const_iterator p = statfs_ops.begin();
+       p != statfs_ops.end();
+       ++p) {
+    StatfsOp *op = p->second;
+    fmt->open_object_section("statfs_op");
+    fmt->dump_unsigned("tid", op->tid);
+    fmt->dump_stream("last_sent") << op->last_submit;
+    fmt->close_section(); // pool_stat_op object
+  }
+  fmt->close_section(); // pool_stat_ops array
+}
+
+Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) :
+  m_objecter(objecter)
+{
+}
+
+bool Objecter::RequestStateHook::call(std::string command, cmdmap_t& cmdmap,
+				      std::string format, bufferlist& out)
+{
+  Formatter *f = new_formatter(format);
+  if (!f)
+    f = new_formatter("json-pretty");
+  m_objecter->client_lock.Lock();
+  m_objecter->dump_requests(f);
+  m_objecter->client_lock.Unlock();
+  f->flush(out);
+  delete f;
+  return true;
+}
+
+void Objecter::blacklist_self(bool set)
+{
+  ldout(cct, 10) << "blacklist_self " << (set ? "add" : "rm") << dendl;
+
+  vector cmd;
+  cmd.push_back("{\"prefix\":\"osd blacklist\", ");
+  if (set)
+    cmd.push_back("\"blacklistop\":\"add\","); 
+  else
+    cmd.push_back("\"blacklistop\":\"rm\",");
+  stringstream ss;
+  ss << messenger->get_myaddr();
+  cmd.push_back("\"addr\":\"" + ss.str() + "\"");
+
+  MMonCommand *m = new MMonCommand(monc->get_fsid());
+  m->cmd = cmd;
+
+  monc->send_mon_message(m);
+}
+
+// commands
+
+void Objecter::handle_command_reply(MCommandReply *m)
+{
+  map::iterator p = command_ops.find(m->get_tid());
+  if (p == command_ops.end()) {
+    ldout(cct, 10) << "handle_command_reply tid " << m->get_tid() << " not found" << dendl;
+    m->put();
+    return;
+  }
+
+  CommandOp *c = p->second;
+  if (!c->session ||
+      m->get_connection() != c->session->con) {
+    ldout(cct, 10) << "handle_command_reply tid " << m->get_tid() << " got reply from wrong connection "
+		   << m->get_connection() << " " << m->get_source_inst() << dendl;
+    m->put();
+    return;
+  }
+  if (c->poutbl)
+    c->poutbl->claim(m->get_data());
+  _finish_command(c, m->r, m->rs);
+  m->put();
+}
+
+class C_CancelCommandOp : public Context
+{
+  ceph_tid_t tid;
+  Objecter *objecter;
+public:
+  C_CancelCommandOp(ceph_tid_t tid, Objecter *objecter) : tid(tid),
+						          objecter(objecter) {}
+  void finish(int r) {
+    // note that objecter lock == timer lock, and is already held
+    objecter->command_op_cancel(tid, -ETIMEDOUT);
+  }
+};
+
+int Objecter::_submit_command(CommandOp *c, ceph_tid_t *ptid)
+{
+  ceph_tid_t tid = ++last_tid;
+  ldout(cct, 10) << "_submit_command " << tid << " " << c->cmd << dendl;
+  c->tid = tid;
+  if (osd_timeout > 0) {
+    c->ontimeout = new C_CancelCommandOp(tid, this);
+    timer.add_event_after(osd_timeout, c->ontimeout);
+  }
+  command_ops[tid] = c;
+  num_homeless_ops++;
+  (void)recalc_command_target(c);
+
+  if (c->session)
+    _send_command(c);
+  else
+    maybe_request_map();
+  if (c->map_check_error)
+    _send_command_map_check(c);
+  *ptid = tid;
+
+  logger->set(l_osdc_command_active, command_ops.size());
+  return 0;
+}
+
+int Objecter::recalc_command_target(CommandOp *c)
+{
+  OSDSession *s = NULL;
+  c->map_check_error = 0;
+  if (c->target_osd >= 0) {
+    if (!osdmap->exists(c->target_osd)) {
+      c->map_check_error = -ENOENT;
+      c->map_check_error_str = "osd dne";
+      return RECALC_OP_TARGET_OSD_DNE;
+    }
+    if (osdmap->is_down(c->target_osd)) {
+      c->map_check_error = -ENXIO;
+      c->map_check_error_str = "osd down";
+      return RECALC_OP_TARGET_OSD_DOWN;
+    }
+    s = get_session(c->target_osd);
+  } else {
+    if (!osdmap->have_pg_pool(c->target_pg.pool())) {
+      c->map_check_error = -ENOENT;
+      c->map_check_error_str = "pool dne";
+      return RECALC_OP_TARGET_POOL_DNE;
+    }
+    int primary;
+    vector acting;
+    osdmap->pg_to_acting_osds(c->target_pg, &acting, &primary);
+    if (primary != -1)
+      s = get_session(primary);
+  }
+  if (c->session != s) {
+    ldout(cct, 10) << "recalc_command_target " << c->tid << " now " << c->session << dendl;
+    if (s) {
+      if (!c->session)
+	num_homeless_ops--;
+      c->session = s;
+      s->command_ops.push_back(&c->session_item);
+    } else {
+      c->session = NULL;
+      num_homeless_ops++;
+    }
+    return RECALC_OP_TARGET_NEED_RESEND;
+  }
+  ldout(cct, 20) << "recalc_command_target " << c->tid << " no change, " << c->session << dendl;
+  return RECALC_OP_TARGET_NO_ACTION;
+}
+
+void Objecter::_send_command(CommandOp *c)
+{
+  ldout(cct, 10) << "_send_command " << c->tid << dendl;
+  assert(c->session);
+  assert(c->session->con);
+  MCommand *m = new MCommand(monc->monmap.fsid);
+  m->cmd = c->cmd;
+  m->set_data(c->inbl);
+  m->set_tid(c->tid);
+  messenger->send_message(m, c->session->con);
+  logger->inc(l_osdc_command_send);
+}
+
+int Objecter::command_op_cancel(ceph_tid_t tid, int r)
+{
+  assert(client_lock.is_locked());
+  assert(initialized);
+
+  map::iterator it = command_ops.find(tid);
+  if (it == command_ops.end()) {
+    ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+    return -ENOENT;
+  }
+
+  ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+  CommandOp *op = it->second;
+  command_cancel_map_check(op);
+  _finish_command(op, -ETIMEDOUT, "");
+  return 0;
+}
+
+void Objecter::_finish_command(CommandOp *c, int r, string rs)
+{
+  ldout(cct, 10) << "_finish_command " << c->tid << " = " << r << " " << rs << dendl;
+  c->session_item.remove_myself();
+  if (c->prs)
+    *c->prs = rs;
+  if (c->onfinish)
+    c->onfinish->complete(r);
+  command_ops.erase(c->tid);
+  if (c->ontimeout)
+    timer.cancel_event(c->ontimeout);
+  c->put();
+
+  logger->set(l_osdc_command_active, command_ops.size());
+}
diff --git a/ceph/src/osdc/Objecter.h b/ceph/src/osdc/Objecter.h
new file mode 100644
index 00000000..2ede888a
--- /dev/null
+++ b/ceph/src/osdc/Objecter.h
@@ -0,0 +1,2193 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_OBJECTER_H
+#define CEPH_OBJECTER_H
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "include/xlist.h"
+
+#include "osd/OSDMap.h"
+#include "messages/MOSDOp.h"
+
+#include "common/admin_socket.h"
+#include "common/Timer.h"
+#include "include/rados/rados_types.h"
+#include "include/rados/rados_types.hpp"
+
+#include 
+#include 
+#include 
+#include 
+using namespace std;
+
+class Context;
+class Messenger;
+class OSDMap;
+class MonClient;
+class Message;
+
+class MPoolOpReply;
+
+class MGetPoolStatsReply;
+class MStatfsReply;
+class MCommandReply;
+
+class PerfCounters;
+
+// -----------------------------------------
+
+struct ObjectOperation {
+  vector ops;
+  int flags;
+  int priority;
+
+  vector out_bl;
+  vector out_handler;
+  vector out_rval;
+
+  ObjectOperation() : flags(0), priority(0) {}
+  ~ObjectOperation() {
+    while (!out_handler.empty()) {
+      delete out_handler.back();
+      out_handler.pop_back();
+    }
+  }
+
+  size_t size() {
+    return ops.size();
+  }
+
+  void set_last_op_flags(int flags) {
+    assert(!ops.empty());
+    ops.rbegin()->op.flags = flags;
+  }
+
+  /**
+   * This is a more limited form of C_Contexts, but that requires
+   * a ceph_context which we don't have here.
+   */
+  class C_TwoContexts : public Context {
+    Context *first;
+    Context *second;
+  public:
+    C_TwoContexts(Context *first, Context *second) : first(first),
+						     second(second) {}
+    void finish(int r) {
+      first->complete(r);
+      second->complete(r);
+    }
+  };
+
+  /**
+   * Add a callback to run when this operation completes,
+   * after any other callbacks for it.
+   */
+  void add_handler(Context *extra) {
+    size_t last = out_handler.size() - 1;
+    Context *orig = out_handler[last];
+    if (orig) {
+      Context *wrapper = new C_TwoContexts(orig, extra);
+      out_handler[last] = wrapper;
+    } else {
+      out_handler[last] = extra;
+    }
+  }
+
+  OSDOp& add_op(int op) {
+    int s = ops.size();
+    ops.resize(s+1);
+    ops[s].op.op = op;
+    out_bl.resize(s+1);
+    out_bl[s] = NULL;
+    out_handler.resize(s+1);
+    out_handler[s] = NULL;
+    out_rval.resize(s+1);
+    out_rval[s] = NULL;
+    return ops[s];
+  }
+  void add_data(int op, uint64_t off, uint64_t len, bufferlist& bl) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.extent.offset = off;
+    osd_op.op.extent.length = len;
+    osd_op.indata.claim_append(bl);
+  }
+  void add_clone_range(int op, uint64_t off, uint64_t len, const object_t& srcoid, uint64_t srcoff, snapid_t srcsnapid) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.clonerange.offset = off;
+    osd_op.op.clonerange.length = len;
+    osd_op.op.clonerange.src_offset = srcoff;
+    osd_op.soid = sobject_t(srcoid, srcsnapid);
+  }
+  void add_xattr(int op, const char *name, const bufferlist& data) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.xattr.name_len = (name ? strlen(name) : 0);
+    osd_op.op.xattr.value_len = data.length();
+    if (name)
+      osd_op.indata.append(name);
+    osd_op.indata.append(data);
+  }
+  void add_xattr_cmp(int op, const char *name, uint8_t cmp_op, uint8_t cmp_mode, const bufferlist& data) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.xattr.name_len = (name ? strlen(name) : 0);
+    osd_op.op.xattr.value_len = data.length();
+    osd_op.op.xattr.cmp_op = cmp_op;
+    osd_op.op.xattr.cmp_mode = cmp_mode;
+    if (name)
+      osd_op.indata.append(name);
+    osd_op.indata.append(data);
+  }
+  void add_call(int op, const char *cname, const char *method, bufferlist &indata,
+                bufferlist *outbl, Context *ctx, int *prval) {
+    OSDOp& osd_op = add_op(op);
+
+    unsigned p = ops.size() - 1;
+    out_handler[p] = ctx;
+    out_bl[p] = outbl;
+    out_rval[p] = prval;
+
+    osd_op.op.op = op;
+    osd_op.op.cls.class_len = strlen(cname);
+    osd_op.op.cls.method_len = strlen(method);
+    osd_op.op.cls.indata_len = indata.length();
+    osd_op.indata.append(cname, osd_op.op.cls.class_len);
+    osd_op.indata.append(method, osd_op.op.cls.method_len);
+    osd_op.indata.append(indata);
+  }
+  void add_watch(int op, uint64_t cookie, uint64_t ver, uint8_t flag, bufferlist& inbl) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.watch.cookie = cookie;
+    osd_op.op.watch.ver = ver;
+    osd_op.op.watch.flag = flag;
+    osd_op.indata.append(inbl);
+  }
+  void add_pgls(int op, uint64_t count, collection_list_handle_t cookie, epoch_t start_epoch) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.pgls.count = count;
+    osd_op.op.pgls.start_epoch = start_epoch;
+    ::encode(cookie, osd_op.indata);
+  }
+  void add_pgls_filter(int op, uint64_t count, bufferlist& filter, collection_list_handle_t cookie, epoch_t start_epoch) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.pgls.count = count;
+    osd_op.op.pgls.start_epoch = start_epoch;
+    string cname = "pg";
+    string mname = "filter";
+    ::encode(cname, osd_op.indata);
+    ::encode(mname, osd_op.indata);
+    ::encode(cookie, osd_op.indata);
+    osd_op.indata.append(filter);
+  }
+  void add_alloc_hint(int op, uint64_t expected_object_size,
+                      uint64_t expected_write_size) {
+    OSDOp& osd_op = add_op(op);
+    osd_op.op.op = op;
+    osd_op.op.alloc_hint.expected_object_size = expected_object_size;
+    osd_op.op.alloc_hint.expected_write_size = expected_write_size;
+  }
+
+  // ------
+
+  // pg
+  void pg_ls(uint64_t count, bufferlist& filter, collection_list_handle_t cookie, epoch_t start_epoch) {
+    if (filter.length() == 0)
+      add_pgls(CEPH_OSD_OP_PGLS, count, cookie, start_epoch);
+    else
+      add_pgls_filter(CEPH_OSD_OP_PGLS_FILTER, count, filter, cookie, start_epoch);
+    flags |= CEPH_OSD_FLAG_PGOP;
+  }
+
+  void create(bool excl) {
+    OSDOp& o = add_op(CEPH_OSD_OP_CREATE);
+    o.op.flags = (excl ? CEPH_OSD_OP_FLAG_EXCL : 0);
+  }
+  void create(bool excl, const string& category) {
+    OSDOp& o = add_op(CEPH_OSD_OP_CREATE);
+    o.op.flags = (excl ? CEPH_OSD_OP_FLAG_EXCL : 0);
+    ::encode(category, o.indata);
+  }
+
+  struct C_ObjectOperation_stat : public Context {
+    bufferlist bl;
+    uint64_t *psize;
+    utime_t *pmtime;
+    time_t *ptime;
+    int *prval;
+    C_ObjectOperation_stat(uint64_t *ps, utime_t *pm, time_t *pt, int *prval)
+      : psize(ps), pmtime(pm), ptime(pt), prval(prval) {}
+    void finish(int r) {
+      if (r >= 0) {
+	bufferlist::iterator p = bl.begin();
+	try {
+	  uint64_t size;
+	  utime_t mtime;
+	  ::decode(size, p);
+	  ::decode(mtime, p);
+	  if (psize)
+	    *psize = size;
+	  if (pmtime)
+	    *pmtime = mtime;
+	  if (ptime)
+	    *ptime = mtime.sec();
+	} catch (buffer::error& e) {
+	  if (prval)
+	    *prval = -EIO;
+	}
+      }
+    }
+  };
+  void stat(uint64_t *psize, utime_t *pmtime, int *prval) {
+    add_op(CEPH_OSD_OP_STAT);
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_stat *h = new C_ObjectOperation_stat(psize, pmtime, NULL,
+							   prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+    out_rval[p] = prval;
+  }
+  void stat(uint64_t *psize, time_t *ptime, int *prval) {
+    add_op(CEPH_OSD_OP_STAT);
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_stat *h = new C_ObjectOperation_stat(psize, NULL, ptime,
+							   prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+    out_rval[p] = prval;
+  }
+
+  // object data
+  void read(uint64_t off, uint64_t len, bufferlist *pbl, int *prval,
+	    Context* ctx) {
+    bufferlist bl;
+    add_data(CEPH_OSD_OP_READ, off, len, bl);
+    unsigned p = ops.size() - 1;
+    out_bl[p] = pbl;
+    out_rval[p] = prval;
+    out_handler[p] = ctx;
+  }
+
+  struct C_ObjectOperation_sparse_read : public Context {
+    bufferlist bl;
+    bufferlist *data_bl;
+    std::map *extents;
+    int *prval;
+    C_ObjectOperation_sparse_read(bufferlist *data_bl,
+				  std::map *extents,
+				  int *prval)
+      : data_bl(data_bl), extents(extents), prval(prval) {}
+    void finish(int r) {
+      bufferlist::iterator iter = bl.begin();
+      if (r >= 0) {
+	try {
+	  ::decode(*extents, iter);
+	  ::decode(*data_bl, iter);
+	} catch (buffer::error& e) {
+	  if (prval)
+	    *prval = -EIO;
+	}
+      }
+    }
+  };
+  void sparse_read(uint64_t off, uint64_t len, std::map *m,
+		   bufferlist *data_bl, int *prval) {
+    bufferlist bl;
+    add_data(CEPH_OSD_OP_SPARSE_READ, off, len, bl);
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_sparse_read *h =
+      new C_ObjectOperation_sparse_read(data_bl, m, prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+    out_rval[p] = prval;
+  }
+  void write(uint64_t off, bufferlist& bl,
+             uint64_t truncate_size,
+             uint32_t truncate_seq) {
+    add_data(CEPH_OSD_OP_WRITE, off, bl.length(), bl);
+    OSDOp& o = *ops.rbegin();
+    o.op.extent.truncate_size = truncate_size;
+    o.op.extent.truncate_seq = truncate_seq;
+  }
+  void write(uint64_t off, bufferlist& bl) {
+    write(off, bl, 0, 0);
+  }
+  void write_full(bufferlist& bl) {
+    add_data(CEPH_OSD_OP_WRITEFULL, 0, bl.length(), bl);
+  }
+  void append(bufferlist& bl) {
+    add_data(CEPH_OSD_OP_APPEND, 0, bl.length(), bl);
+  }
+  void zero(uint64_t off, uint64_t len) {
+    bufferlist bl;
+    add_data(CEPH_OSD_OP_ZERO, off, len, bl);
+  }
+  void truncate(uint64_t off) {
+    bufferlist bl;
+    add_data(CEPH_OSD_OP_TRUNCATE, off, 0, bl);
+  }
+  void remove() {
+    bufferlist bl;
+    add_data(CEPH_OSD_OP_DELETE, 0, 0, bl);
+  }
+  void mapext(uint64_t off, uint64_t len) {
+    bufferlist bl;
+    add_data(CEPH_OSD_OP_MAPEXT, off, len, bl);
+  }
+  void sparse_read(uint64_t off, uint64_t len) {
+    bufferlist bl;
+    add_data(CEPH_OSD_OP_SPARSE_READ, off, len, bl);
+  }
+
+  void clone_range(const object_t& src_oid, uint64_t src_offset, uint64_t len, uint64_t dst_offset) {
+    add_clone_range(CEPH_OSD_OP_CLONERANGE, dst_offset, len, src_oid, src_offset, CEPH_NOSNAP);
+  }
+
+  // object attrs
+  void getxattr(const char *name, bufferlist *pbl, int *prval) {
+    bufferlist bl;
+    add_xattr(CEPH_OSD_OP_GETXATTR, name, bl);
+    unsigned p = ops.size() - 1;
+    out_bl[p] = pbl;
+    out_rval[p] = prval;
+  }
+  struct C_ObjectOperation_decodevals : public Context {
+    bufferlist bl;
+    std::map *pattrs;
+    int *prval;
+    C_ObjectOperation_decodevals(std::map *pa, int *pr)
+      : pattrs(pa), prval(pr) {}
+    void finish(int r) {
+      if (r >= 0) {
+	bufferlist::iterator p = bl.begin();
+	try {
+	  if (pattrs)
+	    ::decode(*pattrs, p);
+	}
+	catch (buffer::error& e) {
+	  if (prval)
+	    *prval = -EIO;
+	}
+      }
+    }
+  };
+  struct C_ObjectOperation_decodekeys : public Context {
+    bufferlist bl;
+    std::set *pattrs;
+    int *prval;
+    C_ObjectOperation_decodekeys(std::set *pa, int *pr)
+      : pattrs(pa), prval(pr) {}
+    void finish(int r) {
+      if (r >= 0) {
+	bufferlist::iterator p = bl.begin();
+	try {
+	  if (pattrs)
+	    ::decode(*pattrs, p);
+	}
+	catch (buffer::error& e) {
+	  if (prval)
+	    *prval = -EIO;
+	}
+      }	
+    }
+  };
+  struct C_ObjectOperation_decodewatchers : public Context {
+    bufferlist bl;
+    list *pwatchers;
+    int *prval;
+    C_ObjectOperation_decodewatchers(list *pw, int *pr)
+      : pwatchers(pw), prval(pr) {}
+    void finish(int r) {
+      if (r >= 0) {
+	bufferlist::iterator p = bl.begin();
+	try {
+          obj_list_watch_response_t resp;
+	  ::decode(resp, p);
+	  if (pwatchers) {
+            for (list::iterator i = resp.entries.begin() ;
+                    i != resp.entries.end() ; ++i) {
+              obj_watch_t ow;
+	      ostringstream sa;
+	      sa << i->addr;
+	      strncpy(ow.addr, sa.str().c_str(), 256);
+              ow.watcher_id = i->name.num();
+              ow.cookie = i->cookie;
+              ow.timeout_seconds = i->timeout_seconds;
+              pwatchers->push_back(ow);
+            }
+          }
+	}
+	catch (buffer::error& e) {
+	  if (prval)
+	    *prval = -EIO;
+	}
+      }	
+    }
+  };
+  struct C_ObjectOperation_decodesnaps : public Context {
+    bufferlist bl;
+    librados::snap_set_t *psnaps;
+    int *prval;
+    C_ObjectOperation_decodesnaps(librados::snap_set_t *ps, int *pr)
+      : psnaps(ps), prval(pr) {}
+    void finish(int r) {
+      if (r >= 0) {
+	bufferlist::iterator p = bl.begin();
+	try {
+          obj_list_snap_response_t resp;
+	  ::decode(resp, p);
+	  if (psnaps) {
+            psnaps->clones.clear();
+            for (vector::iterator ci = resp.clones.begin(); 
+		 ci != resp.clones.end(); 
+		 ++ci) {
+              librados::clone_info_t clone;
+
+              clone.cloneid = ci->cloneid;
+              clone.snaps.reserve(ci->snaps.size());
+              clone.snaps.insert(clone.snaps.end(), ci->snaps.begin(), ci->snaps.end());
+              clone.overlap = ci->overlap;
+              clone.size = ci->size;
+
+              psnaps->clones.push_back(clone);
+            }
+	    psnaps->seq = resp.seq;
+          }
+	}
+	catch (buffer::error& e) {
+	  if (prval)
+	    *prval = -EIO;
+	}
+      }
+    }
+  };
+  void getxattrs(std::map *pattrs, int *prval) {
+    add_op(CEPH_OSD_OP_GETXATTRS);
+    if (pattrs || prval) {
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodevals *h = new C_ObjectOperation_decodevals(pattrs, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
+    }
+  }
+  void setxattr(const char *name, const bufferlist& bl) {
+    add_xattr(CEPH_OSD_OP_SETXATTR, name, bl);
+  }
+  void setxattr(const char *name, const string& s) {
+    bufferlist bl;
+    bl.append(s);
+    add_xattr(CEPH_OSD_OP_SETXATTR, name, bl);
+  }
+  void cmpxattr(const char *name, uint8_t cmp_op, uint8_t cmp_mode, const bufferlist& bl) {
+    add_xattr_cmp(CEPH_OSD_OP_CMPXATTR, name, cmp_op, cmp_mode, bl);
+  }
+  void rmxattr(const char *name) {
+    bufferlist bl;
+    add_xattr(CEPH_OSD_OP_RMXATTR, name, bl);
+  }
+  void setxattrs(map& attrs) {
+    bufferlist bl;
+    ::encode(attrs, bl);
+    add_xattr(CEPH_OSD_OP_RESETXATTRS, 0, bl.length());
+  }
+  void resetxattrs(const char *prefix, map& attrs) {
+    bufferlist bl;
+    ::encode(attrs, bl);
+    add_xattr(CEPH_OSD_OP_RESETXATTRS, prefix, bl);
+  }
+  
+  // trivialmap
+  void tmap_update(bufferlist& bl) {
+    add_data(CEPH_OSD_OP_TMAPUP, 0, 0, bl);
+  }
+  void tmap_put(bufferlist& bl) {
+    add_data(CEPH_OSD_OP_TMAPPUT, 0, bl.length(), bl);
+  }
+  void tmap_get(bufferlist *pbl, int *prval) {
+    add_op(CEPH_OSD_OP_TMAPGET);
+    unsigned p = ops.size() - 1;
+    out_bl[p] = pbl;
+    out_rval[p] = prval;
+  }
+  void tmap_get() {
+    add_op(CEPH_OSD_OP_TMAPGET);
+  }
+  void tmap_to_omap(bool nullok=false) {
+     OSDOp& osd_op = add_op(CEPH_OSD_OP_TMAP2OMAP);
+     osd_op.op.op = CEPH_OSD_OP_TMAP2OMAP;
+     if (nullok)
+       osd_op.op.tmap2omap.flags = CEPH_OSD_TMAP2OMAP_NULLOK;
+  }
+
+  // objectmap
+  void omap_get_keys(const string &start_after,
+		     uint64_t max_to_get,
+		     std::set *out_set,
+		     int *prval) {
+    OSDOp &op = add_op(CEPH_OSD_OP_OMAPGETKEYS);
+    bufferlist bl;
+    ::encode(start_after, bl);
+    ::encode(max_to_get, bl);
+    op.op.extent.offset = 0;
+    op.op.extent.length = bl.length();
+    op.indata.claim_append(bl);
+    if (prval || out_set) {
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodekeys *h =
+	new C_ObjectOperation_decodekeys(out_set, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
+    }
+  }
+
+  void omap_get_vals(const string &start_after,
+		     const string &filter_prefix,
+		     uint64_t max_to_get,
+		     std::map *out_set,
+		     int *prval) {
+    OSDOp &op = add_op(CEPH_OSD_OP_OMAPGETVALS);
+    bufferlist bl;
+    ::encode(start_after, bl);
+    ::encode(max_to_get, bl);
+    ::encode(filter_prefix, bl);
+    op.op.extent.offset = 0;
+    op.op.extent.length = bl.length();
+    op.indata.claim_append(bl);
+    if (prval || out_set) {
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodevals *h =
+	new C_ObjectOperation_decodevals(out_set, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
+    }
+  }
+
+  void omap_get_vals_by_keys(const std::set &to_get,
+			    std::map *out_set,
+			    int *prval) {
+    OSDOp &op = add_op(CEPH_OSD_OP_OMAPGETVALSBYKEYS);
+    bufferlist bl;
+    ::encode(to_get, bl);
+    op.op.extent.offset = 0;
+    op.op.extent.length = bl.length();
+    op.indata.claim_append(bl);
+    if (prval || out_set) {
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodevals *h =
+	new C_ObjectOperation_decodevals(out_set, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
+    }
+  }
+
+  void omap_cmp(const std::map > &assertions,
+		int *prval) {
+    OSDOp &op = add_op(CEPH_OSD_OP_OMAP_CMP);
+    bufferlist bl;
+    ::encode(assertions, bl);
+    op.op.extent.offset = 0;
+    op.op.extent.length = bl.length();
+    op.indata.claim_append(bl);
+    if (prval) {
+      unsigned p = ops.size() - 1;
+      out_rval[p] = prval;
+      out_bl[p] = NULL;
+      out_handler[p] = NULL;
+    }
+  }
+
+  struct C_ObjectOperation_copyget : public Context {
+    bufferlist bl;
+    object_copy_cursor_t *cursor;
+    uint64_t *out_size;
+    utime_t *out_mtime;
+    string *out_category;
+    std::map *out_attrs;
+    bufferlist *out_data, *out_omap_header;
+    std::map *out_omap;
+    vector *out_snaps;
+    snapid_t *out_snap_seq;
+    int *prval;
+    C_ObjectOperation_copyget(object_copy_cursor_t *c,
+			      uint64_t *s,
+			      utime_t *m,
+			      string *cat,
+			      std::map *a,
+			      bufferlist *d, bufferlist *oh,
+			      std::map *o,
+			      std::vector *osnaps,
+			      snapid_t *osnap_seq,
+			      int *r)
+      : cursor(c),
+	out_size(s), out_mtime(m), out_category(cat),
+	out_attrs(a), out_data(d), out_omap_header(oh),
+	out_omap(o), out_snaps(osnaps), out_snap_seq(osnap_seq),
+	prval(r) {}
+    void finish(int r) {
+      if (r < 0)
+	return;
+      try {
+	bufferlist::iterator p = bl.begin();
+	object_copy_data_t copy_reply;
+	::decode(copy_reply, p);
+	if (out_size)
+	  *out_size = copy_reply.size;
+	if (out_mtime)
+	  *out_mtime = copy_reply.mtime;
+	if (out_category)
+	  *out_category = copy_reply.category;
+	if (out_attrs)
+	  *out_attrs = copy_reply.attrs;
+	if (out_data)
+	  out_data->claim_append(copy_reply.data);
+	if (out_omap_header)
+	  out_omap_header->claim_append(copy_reply.omap_header);
+	if (out_omap)
+	  *out_omap = copy_reply.omap;
+	if (out_snaps)
+	  *out_snaps = copy_reply.snaps;
+	if (out_snap_seq)
+	  *out_snap_seq = copy_reply.snap_seq;
+	*cursor = copy_reply.cursor;
+      } catch (buffer::error& e) {
+	if (prval)
+	  *prval = -EIO;
+      }
+    }
+  };
+
+  void copy_get(object_copy_cursor_t *cursor,
+		uint64_t max,
+		uint64_t *out_size,
+		utime_t *out_mtime,
+		string *out_category,
+		std::map *out_attrs,
+		bufferlist *out_data,
+		bufferlist *out_omap_header,
+		std::map *out_omap,
+		vector *out_snaps,
+		snapid_t *out_snap_seq,
+		int *prval) {
+    OSDOp& osd_op = add_op(CEPH_OSD_OP_COPY_GET);
+    osd_op.op.copy_get.max = max;
+    ::encode(*cursor, osd_op.indata);
+    ::encode(max, osd_op.indata);
+    unsigned p = ops.size() - 1;
+    out_rval[p] = prval;
+    C_ObjectOperation_copyget *h =
+      new C_ObjectOperation_copyget(cursor, out_size, out_mtime, out_category,
+                                    out_attrs, out_data, out_omap_header,
+				    out_omap, out_snaps, out_snap_seq, prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+  }
+
+  void undirty() {
+    add_op(CEPH_OSD_OP_UNDIRTY);
+  }
+
+  struct C_ObjectOperation_isdirty : public Context {
+    bufferlist bl;
+    bool *pisdirty;
+    int *prval;
+    C_ObjectOperation_isdirty(bool *p, int *r)
+      : pisdirty(p), prval(r) {}
+    void finish(int r) {
+      if (r < 0)
+	return;
+      try {
+	bufferlist::iterator p = bl.begin();
+	bool isdirty;
+	::decode(isdirty, p);
+	if (pisdirty)
+	  *pisdirty = isdirty;
+      } catch (buffer::error& e) {
+	if (prval)
+	  *prval = -EIO;
+      }
+    }
+  };
+
+  void is_dirty(bool *pisdirty, int *prval) {
+    add_op(CEPH_OSD_OP_ISDIRTY);
+    unsigned p = ops.size() - 1;
+    out_rval[p] = prval;
+    C_ObjectOperation_isdirty *h =
+      new C_ObjectOperation_isdirty(pisdirty, prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+  }
+
+  struct C_ObjectOperation_hit_set_ls : public Context {
+    bufferlist bl;
+    std::list< std::pair > *ptls;
+    std::list< std::pair > *putls;
+    int *prval;
+    C_ObjectOperation_hit_set_ls(std::list< std::pair > *t,
+				 std::list< std::pair > *ut,
+				 int *r)
+      : ptls(t), putls(ut), prval(r) {}
+    void finish(int r) {
+      if (r < 0)
+	return;
+      try {
+	bufferlist::iterator p = bl.begin();
+	std::list< std::pair > ls;
+	::decode(ls, p);
+	if (ptls) {
+	  ptls->clear();
+	  for (list< pair >::iterator p = ls.begin(); p != ls.end(); ++p)
+	    // round initial timestamp up to the next full second to keep this a valid interval.
+	    ptls->push_back(make_pair(p->first.usec() ? p->first.sec() + 1 : p->first.sec(), p->second.sec()));
+	}
+	if (putls)
+	  putls->swap(ls);
+      } catch (buffer::error& e) {
+	r = -EIO;
+      }
+      if (prval)
+	*prval = r;
+    }
+  };
+
+  /**
+   * list available HitSets.
+   *
+   * We will get back a list of time intervals.  Note that the most recent range may have
+   * an empty end timestamp if it is still accumulating.
+   *
+   * @param pls [out] list of time intervals
+   * @param prval [out] return value
+   */
+  void hit_set_ls(std::list< std::pair > *pls, int *prval) {
+    add_op(CEPH_OSD_OP_PG_HITSET_LS);
+    unsigned p = ops.size() - 1;
+    out_rval[p] = prval;
+    C_ObjectOperation_hit_set_ls *h =
+      new C_ObjectOperation_hit_set_ls(pls, NULL, prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+  }
+  void hit_set_ls(std::list< std::pair > *pls, int *prval) {
+    add_op(CEPH_OSD_OP_PG_HITSET_LS);
+    unsigned p = ops.size() - 1;
+    out_rval[p] = prval;
+    C_ObjectOperation_hit_set_ls *h =
+      new C_ObjectOperation_hit_set_ls(NULL, pls, prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+  }
+
+  /**
+   * get HitSet
+   *
+   * Return an encoded HitSet that includes the provided time
+   * interval.
+   *
+   * @param stamp [in] timestamp
+   * @param pbl [out] target buffer for encoded HitSet
+   * @param prval [out] return value
+   */
+  void hit_set_get(utime_t stamp, bufferlist *pbl, int *prval) {
+    OSDOp& op = add_op(CEPH_OSD_OP_PG_HITSET_GET);
+    op.op.hit_set_get.stamp.tv_sec = stamp.sec();
+    op.op.hit_set_get.stamp.tv_nsec = stamp.nsec();
+    unsigned p = ops.size() - 1;
+    out_rval[p] = prval;
+    out_bl[p] = pbl;
+  }
+
+  void omap_get_header(bufferlist *bl, int *prval) {
+    add_op(CEPH_OSD_OP_OMAPGETHEADER);
+    unsigned p = ops.size() - 1;
+    out_bl[p] = bl;
+    out_rval[p] = prval;
+  }
+
+  void omap_set(const map &map) {
+    bufferlist bl;
+    ::encode(map, bl);
+    add_data(CEPH_OSD_OP_OMAPSETVALS, 0, bl.length(), bl);
+  }
+
+  void omap_set_header(bufferlist &bl) {
+    add_data(CEPH_OSD_OP_OMAPSETHEADER, 0, bl.length(), bl);
+  }
+
+  void omap_clear() {
+    add_op(CEPH_OSD_OP_OMAPCLEAR);
+  }
+
+  void omap_rm_keys(const std::set &to_remove) {
+    bufferlist bl;
+    ::encode(to_remove, bl);
+    add_data(CEPH_OSD_OP_OMAPRMKEYS, 0, bl.length(), bl);
+  }
+
+  // object classes
+  void call(const char *cname, const char *method, bufferlist &indata) {
+    add_call(CEPH_OSD_OP_CALL, cname, method, indata, NULL, NULL, NULL);
+  }
+
+  void call(const char *cname, const char *method, bufferlist &indata, bufferlist *outdata,
+	    Context *ctx, int *prval) {
+    add_call(CEPH_OSD_OP_CALL, cname, method, indata, outdata, ctx, prval);
+  }
+
+  // watch/notify
+  void watch(uint64_t cookie, uint64_t ver, bool set) {
+    bufferlist inbl;
+    add_watch(CEPH_OSD_OP_WATCH, cookie, ver, (set ? 1 : 0), inbl);
+  }
+
+  void notify(uint64_t cookie, uint64_t ver, bufferlist& inbl) {
+    add_watch(CEPH_OSD_OP_NOTIFY, cookie, ver, 1, inbl); 
+  }
+
+  void notify_ack(uint64_t notify_id, uint64_t ver, uint64_t cookie) {
+    bufferlist bl;
+    ::encode(notify_id, bl);
+    ::encode(cookie, bl);
+    add_watch(CEPH_OSD_OP_NOTIFY_ACK, notify_id, ver, 0, bl);
+  }
+
+  void list_watchers(list *out,
+		     int *prval) {
+    (void)add_op(CEPH_OSD_OP_LIST_WATCHERS);
+    if (prval || out) {
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodewatchers *h =
+	new C_ObjectOperation_decodewatchers(out, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
+    }
+  }
+
+  void list_snaps(librados::snap_set_t *out, int *prval) {
+    (void)add_op(CEPH_OSD_OP_LIST_SNAPS);
+    if (prval || out) {
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodesnaps *h =
+	new C_ObjectOperation_decodesnaps(out, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
+    }
+  }
+
+  void assert_version(uint64_t ver) {
+    OSDOp& osd_op = add_op(CEPH_OSD_OP_ASSERT_VER);
+    osd_op.op.assert_ver.ver = ver;
+  }
+  void assert_src_version(const object_t& srcoid, snapid_t srcsnapid, uint64_t ver) {
+    bufferlist bl;
+    add_watch(CEPH_OSD_OP_ASSERT_SRC_VERSION, 0, ver, 0, bl);
+    ops.rbegin()->soid = sobject_t(srcoid, srcsnapid);
+  }
+
+  void cmpxattr(const char *name, const bufferlist& val,
+		int op, int mode) {
+    add_xattr(CEPH_OSD_OP_CMPXATTR, name, val);
+    OSDOp& o = *ops.rbegin();
+    o.op.xattr.cmp_op = op;
+    o.op.xattr.cmp_mode = mode;
+  }
+  void src_cmpxattr(const object_t& srcoid, snapid_t srcsnapid,
+		    const char *name, const bufferlist& val,
+		    int op, int mode) {
+    add_xattr(CEPH_OSD_OP_SRC_CMPXATTR, name, val);
+    OSDOp& o = *ops.rbegin();
+    o.soid = sobject_t(srcoid, srcsnapid);
+    o.op.xattr.cmp_op = op;
+    o.op.xattr.cmp_mode = mode;
+  }
+
+  void rollback(uint64_t snapid) {
+    OSDOp& osd_op = add_op(CEPH_OSD_OP_ROLLBACK);
+    osd_op.op.snap.snapid = snapid;
+  }
+
+  void copy_from(object_t src, snapid_t snapid, object_locator_t src_oloc,
+		 version_t src_version, unsigned flags) {
+    OSDOp& osd_op = add_op(CEPH_OSD_OP_COPY_FROM);
+    osd_op.op.copy_from.snapid = snapid;
+    osd_op.op.copy_from.src_version = src_version;
+    osd_op.op.copy_from.flags = flags;
+    ::encode(src, osd_op.indata);
+    ::encode(src_oloc, osd_op.indata);
+  }
+
+  /**
+   * writeback content to backing tier
+   *
+   * If object is marked dirty in the cache tier, write back content
+   * to backing tier. If the object is clean this is a no-op.
+   *
+   * If writeback races with an update, the update will block.
+   *
+   * use with IGNORE_CACHE to avoid triggering promote.
+   */
+  void cache_flush() {
+    add_op(CEPH_OSD_OP_CACHE_FLUSH);
+  }
+
+  /**
+   * writeback content to backing tier
+   *
+   * If object is marked dirty in the cache tier, write back content
+   * to backing tier. If the object is clean this is a no-op.
+   *
+   * If writeback races with an update, return EAGAIN.  Requires that
+   * the SKIPRWLOCKS flag be set.
+   *
+   * use with IGNORE_CACHE to avoid triggering promote.
+   */
+  void cache_try_flush() {
+    add_op(CEPH_OSD_OP_CACHE_TRY_FLUSH);
+  }
+
+  /**
+   * evict object from cache tier
+   *
+   * If object is marked clean, remove the object from the cache tier.
+   * Otherwise, return EBUSY.
+   *
+   * use with IGNORE_CACHE to avoid triggering promote.
+   */
+  void cache_evict() {
+    add_op(CEPH_OSD_OP_CACHE_EVICT);
+  }
+
+  void set_alloc_hint(uint64_t expected_object_size,
+                      uint64_t expected_write_size ) {
+    add_alloc_hint(CEPH_OSD_OP_SETALLOCHINT, expected_object_size,
+                   expected_write_size);
+
+    // CEPH_OSD_OP_SETALLOCHINT op is advisory and therefore deemed
+    // not worth a feature bit.  Set FAILOK per-op flag to make
+    // sure older osds don't trip over an unsupported opcode.
+    set_last_op_flags(CEPH_OSD_OP_FLAG_FAILOK);
+  }
+};
+
+
+// ----------------
+
+
+class Objecter : public md_config_obs_t {
+public:
+  // config observer bits
+  virtual const char** get_tracked_conf_keys() const;
+  virtual void handle_conf_change(const struct md_config_t *conf,
+				  const std::set  &changed);
+
+public:
+  Messenger *messenger;
+  MonClient *monc;
+  OSDMap    *osdmap;
+  CephContext *cct;
+  std::multimap crush_location;
+
+  bool initialized;
+
+private:
+  ceph_tid_t last_tid;
+  int client_inc;
+  uint64_t max_linger_id;
+  int num_unacked;
+  int num_uncommitted;
+  int global_op_flags; // flags which are applied to each IO op
+  bool keep_balanced_budget;
+  bool honor_osdmap_full;
+
+public:
+  void maybe_request_map();
+private:
+
+  version_t last_seen_osdmap_version;
+  version_t last_seen_pgmap_version;
+
+  Mutex &client_lock;
+  SafeTimer &timer;
+
+  PerfCounters *logger;
+  
+  class C_Tick : public Context {
+    Objecter *ob;
+  public:
+    C_Tick(Objecter *o) : ob(o) {}
+    void finish(int r) { ob->tick(); }
+  } *tick_event;
+
+  void schedule_tick();
+  void tick();
+
+  class RequestStateHook : public AdminSocketHook {
+    Objecter *m_objecter;
+  public:
+    RequestStateHook(Objecter *objecter);
+    bool call(std::string command, cmdmap_t& cmdmap, std::string format,
+	      bufferlist& out);
+  };
+
+  RequestStateHook *m_request_state_hook;
+
+public:
+  /*** track pending operations ***/
+  // read
+ public:
+
+  struct OSDSession;
+
+  struct op_target_t {
+    int flags;
+    object_t base_oid;
+    object_locator_t base_oloc;
+    object_t target_oid;
+    object_locator_t target_oloc;
+
+    bool precalc_pgid;   ///< true if we are directed at base_pgid, not base_oid
+    pg_t base_pgid;      ///< explciti pg target, if any
+
+    pg_t pgid;           ///< last pg we mapped to
+    vector acting;  ///< acting for last pg we mapped to
+    int primary;         ///< primary for last pg we mapped to
+
+    bool used_replica;
+    bool paused;
+
+    int osd;      ///< the final target osd, or -1
+
+    op_target_t(object_t oid, object_locator_t oloc, int flags)
+      : flags(flags),
+	base_oid(oid),
+	base_oloc(oloc),
+	precalc_pgid(false),
+	primary(-1),
+	used_replica(false),
+	paused(false),
+	osd(-1)
+    {}
+
+    void dump(Formatter *f) const;
+  };
+
+  struct Op {
+    OSDSession *session;
+    xlist::item session_item;
+    int incarnation;
+
+    op_target_t target;
+
+    ConnectionRef con;  // for rx buffer only
+
+    vector ops;
+
+    snapid_t snapid;
+    SnapContext snapc;
+    utime_t mtime;
+
+    bufferlist *outbl;
+    vector out_bl;
+    vector out_handler;
+    vector out_rval;
+
+    int priority;
+    Context *onack, *oncommit, *ontimeout;
+
+    ceph_tid_t tid;
+    eversion_t replay_version;        // for op replay
+    int attempts;
+
+    version_t *objver;
+    epoch_t *reply_epoch;
+
+    utime_t stamp;
+
+    epoch_t map_dne_bound;
+
+    bool budgeted;
+
+    /// true if we should resend this message on failure
+    bool should_resend;
+
+    Op(const object_t& o, const object_locator_t& ol, vector& op,
+       int f, Context *ac, Context *co, version_t *ov) :
+      session(NULL), session_item(this), incarnation(0),
+      target(o, ol, f),
+      con(NULL),
+      snapid(CEPH_NOSNAP),
+      outbl(NULL),
+      priority(0), onack(ac), oncommit(co),
+      ontimeout(NULL),
+      tid(0), attempts(0),
+      objver(ov), reply_epoch(NULL),
+      map_dne_bound(0),
+      budgeted(false),
+      should_resend(true) {
+      ops.swap(op);
+      
+      /* initialize out_* to match op vector */
+      out_bl.resize(ops.size());
+      out_rval.resize(ops.size());
+      out_handler.resize(ops.size());
+      for (unsigned i = 0; i < ops.size(); i++) {
+	out_bl[i] = NULL;
+	out_handler[i] = NULL;
+	out_rval[i] = NULL;
+      }
+
+      if (target.base_oloc.key == o)
+	target.base_oloc.key.clear();
+    }
+    ~Op() {
+      while (!out_handler.empty()) {
+	delete out_handler.back();
+	out_handler.pop_back();
+      }
+    }
+
+    bool operator<(const Op& other) const {
+      return tid < other.tid;
+    }
+  };
+
+  struct C_Op_Map_Latest : public Context {
+    Objecter *objecter;
+    ceph_tid_t tid;
+    version_t latest;
+    C_Op_Map_Latest(Objecter *o, ceph_tid_t t) : objecter(o), tid(t), latest(0) {}
+    void finish(int r);
+  };
+
+  struct C_Command_Map_Latest : public Context {
+    Objecter *objecter;
+    uint64_t tid;
+    version_t latest;
+    C_Command_Map_Latest(Objecter *o, ceph_tid_t t) :  objecter(o), tid(t), latest(0) {}
+    void finish(int r);
+  };
+
+  struct C_Stat : public Context {
+    bufferlist bl;
+    uint64_t *psize;
+    utime_t *pmtime;
+    Context *fin;
+    C_Stat(uint64_t *ps, utime_t *pm, Context *c) :
+      psize(ps), pmtime(pm), fin(c) {}
+    void finish(int r) {
+      if (r >= 0) {
+	bufferlist::iterator p = bl.begin();
+	uint64_t s;
+	utime_t m;
+	::decode(s, p);
+	::decode(m, p);
+	if (psize)
+	  *psize = s;
+	if (pmtime)
+	  *pmtime = m;
+      }
+      fin->complete(r);
+    }
+  };
+
+  struct C_GetAttrs : public Context {
+    bufferlist bl;
+    map& attrset;
+    Context *fin;
+    C_GetAttrs(map& set, Context *c) : attrset(set), fin(c) {}
+    void finish(int r) {
+      if (r >= 0) {
+	bufferlist::iterator p = bl.begin();
+	::decode(attrset, p);
+      }
+      fin->complete(r);
+    }
+  };
+
+
+  // Pools and statistics 
+  struct ListContext {
+    int current_pg;
+    collection_list_handle_t cookie;
+    epoch_t current_pg_epoch;
+    int starting_pg_num;
+    bool at_end_of_pool;
+    bool at_end_of_pg;
+
+    int64_t pool_id;
+    int pool_snap_seq;
+    int max_entries;
+    string nspace;
+
+    bufferlist bl;   // raw data read to here
+    std::list > list;
+
+    bufferlist filter;
+
+    bufferlist extra_info;
+
+    ListContext() : current_pg(0), current_pg_epoch(0), starting_pg_num(0),
+		    at_end_of_pool(false),
+		    at_end_of_pg(false),
+		    pool_id(0),
+		    pool_snap_seq(0), max_entries(0) {}
+
+    bool at_end() const {
+      return at_end_of_pool;
+    }
+
+    uint32_t get_pg_hash_position() const {
+      return current_pg;
+    }
+  };
+
+  struct C_List : public Context {
+    ListContext *list_context;
+    Context *final_finish;
+    Objecter *objecter;
+    epoch_t epoch;
+    C_List(ListContext *lc, Context * finish, Objecter *ob) :
+      list_context(lc), final_finish(finish), objecter(ob), epoch(0) {}
+    void finish(int r) {
+      if (r >= 0) {
+        objecter->_list_reply(list_context, r, final_finish, epoch);
+      } else {
+        final_finish->complete(r);
+      }
+    }
+  };
+  
+  struct PoolStatOp {
+    ceph_tid_t tid;
+    list pools;
+
+    map *pool_stats;
+    Context *onfinish, *ontimeout;
+
+    utime_t last_submit;
+  };
+
+  struct StatfsOp {
+    ceph_tid_t tid;
+    struct ceph_statfs *stats;
+    Context *onfinish, *ontimeout;
+
+    utime_t last_submit;
+  };
+
+  struct PoolOp {
+    ceph_tid_t tid;
+    int64_t pool;
+    string name;
+    Context *onfinish, *ontimeout;
+    int pool_op;
+    uint64_t auid;
+    __u8 crush_rule;
+    snapid_t snapid;
+    bufferlist *blp;
+
+    utime_t last_submit;
+    PoolOp() : tid(0), pool(0), onfinish(NULL), ontimeout(NULL), pool_op(0),
+	       auid(0), crush_rule(0), snapid(0), blp(NULL) {}
+  };
+
+  // -- osd commands --
+  struct CommandOp : public RefCountedObject {
+    xlist::item session_item;
+    OSDSession *session;
+    ceph_tid_t tid;
+    vector cmd;
+    bufferlist inbl;
+    bufferlist *poutbl;
+    string *prs;
+    int target_osd;
+    pg_t target_pg;
+    epoch_t map_dne_bound;
+    int map_check_error;           // error to return if map check fails
+    const char *map_check_error_str;
+    Context *onfinish, *ontimeout;
+    utime_t last_submit;
+
+    CommandOp()
+      : session_item(this), session(NULL),
+	tid(0), poutbl(NULL), prs(NULL), target_osd(-1),
+	map_dne_bound(0),
+	map_check_error(0),
+	map_check_error_str(NULL),
+	onfinish(NULL), ontimeout(NULL) {}
+  };
+
+  int _submit_command(CommandOp *c, ceph_tid_t *ptid);
+  int recalc_command_target(CommandOp *c);
+  void _send_command(CommandOp *c);
+  int command_op_cancel(ceph_tid_t tid, int r);
+  void _finish_command(CommandOp *c, int r, string rs);
+  void handle_command_reply(MCommandReply *m);
+
+
+  // -- lingering ops --
+
+  struct LingerOp : public RefCountedObject {
+    uint64_t linger_id;
+
+    op_target_t target;
+
+    snapid_t snap;
+    SnapContext snapc;
+    utime_t mtime;
+
+    vector ops;
+    bufferlist inbl;
+    bufferlist *poutbl;
+    version_t *pobjver;
+
+    bool registered;
+    Context *on_reg_ack, *on_reg_commit;
+
+    OSDSession *session;
+    xlist::item session_item;
+
+    ceph_tid_t register_tid;
+    epoch_t map_dne_bound;
+
+    LingerOp() : linger_id(0),
+		 target(object_t(), object_locator_t(), 0),
+		 snap(CEPH_NOSNAP),
+		 poutbl(NULL), pobjver(NULL),
+		 registered(false),
+		 on_reg_ack(NULL), on_reg_commit(NULL),
+		 session(NULL), session_item(this),
+		 register_tid(0),
+		 map_dne_bound(0) {}
+
+    // no copy!
+    const LingerOp &operator=(const LingerOp& r);
+    LingerOp(const LingerOp& o);
+  private:
+    ~LingerOp() {}
+  };
+
+  struct C_Linger_Ack : public Context {
+    Objecter *objecter;
+    LingerOp *info;
+    C_Linger_Ack(Objecter *o, LingerOp *l) : objecter(o), info(l) {
+      info->get();
+    }
+    ~C_Linger_Ack() {
+      info->put();
+    }
+    void finish(int r) {
+      objecter->_linger_ack(info, r);
+    }
+  };
+  
+  struct C_Linger_Commit : public Context {
+    Objecter *objecter;
+    LingerOp *info;
+    C_Linger_Commit(Objecter *o, LingerOp *l) : objecter(o), info(l) {
+      info->get();
+    }
+    ~C_Linger_Commit() {
+      info->put();
+    }
+    void finish(int r) {
+      objecter->_linger_commit(info, r);
+    }
+  };
+
+  struct C_Linger_Map_Latest : public Context {
+    Objecter *objecter;
+    uint64_t linger_id;
+    version_t latest;
+    C_Linger_Map_Latest(Objecter *o, uint64_t id) :
+      objecter(o), linger_id(id), latest(0) {}
+    void finish(int r);
+  };
+
+  // -- osd sessions --
+  struct OSDSession {
+    xlist ops;
+    xlist linger_ops;
+    xlist command_ops;
+    int osd;
+    int incarnation;
+    ConnectionRef con;
+
+    OSDSession(int o) : osd(o), incarnation(0), con(NULL) {}
+  };
+  map osd_sessions;
+
+
+ private:
+  // pending ops
+  map       ops;
+  int                       num_homeless_ops;
+  map  linger_ops;
+  map    poolstat_ops;
+  map      statfs_ops;
+  map        pool_ops;
+  map     command_ops;
+
+  // ops waiting for an osdmap with a new pool or confirmation that
+  // the pool does not exist (may be expanded to other uses later)
+  map       check_latest_map_lingers;
+  map           check_latest_map_ops;
+  map    check_latest_map_commands;
+
+  map > > waiting_for_map;
+
+  double mon_timeout, osd_timeout;
+
+  void send_op(Op *op);
+  void cancel_linger_op(Op *op);
+  void finish_op(Op *op);
+  static bool is_pg_changed(
+    int oldprimary,
+    const vector& oldacting,
+    int newprimary,
+    const vector& newacting,
+    bool any_change=false);
+  enum recalc_op_target_result {
+    RECALC_OP_TARGET_NO_ACTION = 0,
+    RECALC_OP_TARGET_NEED_RESEND,
+    RECALC_OP_TARGET_POOL_DNE,
+    RECALC_OP_TARGET_OSD_DNE,
+    RECALC_OP_TARGET_OSD_DOWN,
+  };
+  bool osdmap_full_flag() const;
+  bool target_should_be_paused(op_target_t *op);
+
+  int calc_target(op_target_t *t, bool any_change=false);
+  int recalc_op_target(Op *op);
+  bool recalc_linger_op_target(LingerOp *op);
+
+  void send_linger(LingerOp *info);
+  void _linger_ack(LingerOp *info, int r);
+  void _linger_commit(LingerOp *info, int r);
+
+  void check_op_pool_dne(Op *op);
+  void _send_op_map_check(Op *op);
+  void op_cancel_map_check(Op *op);
+  void check_linger_pool_dne(LingerOp *op);
+  void _send_linger_map_check(LingerOp *op);
+  void linger_cancel_map_check(LingerOp *op);
+  void check_command_map_dne(CommandOp *op);
+  void _send_command_map_check(CommandOp *op);
+  void command_cancel_map_check(CommandOp *op);
+
+  void kick_requests(OSDSession *session);
+
+  OSDSession *get_session(int osd);
+  void reopen_session(OSDSession *session);
+  void close_session(OSDSession *session);
+  
+  void _list_reply(ListContext *list_context, int r, Context *final_finish,
+		   epoch_t reply_epoch);
+
+  void resend_mon_ops();
+
+  /**
+   * handle a budget for in-flight ops
+   * budget is taken whenever an op goes into the ops map
+   * and returned whenever an op is removed from the map
+   * If throttle_op needs to throttle it will unlock client_lock.
+   */
+  int calc_op_budget(Op *op);
+  void throttle_op(Op *op, int op_size=0);
+  void take_op_budget(Op *op) {
+    int op_budget = calc_op_budget(op);
+    if (keep_balanced_budget) {
+      throttle_op(op, op_budget);
+    } else {
+      op_throttle_bytes.take(op_budget);
+      op_throttle_ops.take(1);
+    }
+    op->budgeted = true;
+  }
+  void put_op_budget(Op *op) {
+    assert(op->budgeted);
+    int op_budget = calc_op_budget(op);
+    op_throttle_bytes.put(op_budget);
+    op_throttle_ops.put(1);
+  }
+  Throttle op_throttle_bytes, op_throttle_ops;
+
+ public:
+  Objecter(CephContext *cct_, Messenger *m, MonClient *mc,
+	   OSDMap *om, Mutex& l, SafeTimer& t, double mon_timeout,
+	   double osd_timeout) :
+    messenger(m), monc(mc), osdmap(om), cct(cct_),
+    initialized(false),
+    last_tid(0), client_inc(-1), max_linger_id(0),
+    num_unacked(0), num_uncommitted(0),
+    global_op_flags(0),
+    keep_balanced_budget(false), honor_osdmap_full(true),
+    last_seen_osdmap_version(0),
+    last_seen_pgmap_version(0),
+    client_lock(l), timer(t),
+    logger(NULL), tick_event(NULL),
+    m_request_state_hook(NULL),
+    num_homeless_ops(0),
+    mon_timeout(mon_timeout),
+    osd_timeout(osd_timeout),
+    op_throttle_bytes(cct, "objecter_bytes", cct->_conf->objecter_inflight_op_bytes),
+    op_throttle_ops(cct, "objecter_ops", cct->_conf->objecter_inflight_ops)
+  { }
+  ~Objecter() {
+    assert(!tick_event);
+    assert(!m_request_state_hook);
+    assert(!logger);
+  }
+
+  void init_unlocked();
+  void init_locked();
+  void shutdown_locked();
+  void shutdown_unlocked();
+
+  /**
+   * Tell the objecter to throttle outgoing ops according to its
+   * budget (in _conf). If you do this, ops can block, in
+   * which case it will unlock client_lock and sleep until
+   * incoming messages reduce the used budget low enough for
+   * the ops to continue going; then it will lock client_lock again.
+   */
+  void set_balanced_budget() { keep_balanced_budget = true; }
+  void unset_balanced_budget() { keep_balanced_budget = false; }
+
+  void set_honor_osdmap_full() { honor_osdmap_full = true; }
+  void unset_honor_osdmap_full() { honor_osdmap_full = false; }
+
+  void scan_requests(bool force_resend,
+		     bool force_resend_writes,
+		     map& need_resend,
+		     list& need_resend_linger,
+		     map& need_resend_command);
+
+  int64_t get_object_hash_position(int64_t pool, const string& key, const string& ns);
+  int64_t get_object_pg_hash_position(int64_t pool, const string& key, const string& ns);
+
+  // messages
+ public:
+  void dispatch(Message *m);
+  void handle_osd_op_reply(class MOSDOpReply *m);
+  void handle_osd_map(class MOSDMap *m);
+  void wait_for_osd_map();
+
+private:
+  // low-level
+  ceph_tid_t _op_submit(Op *op);
+  inline void unregister_op(Op *op);
+
+  // public interface
+public:
+  ceph_tid_t op_submit(Op *op);
+  bool is_active() {
+    return !(ops.empty() && linger_ops.empty() && poolstat_ops.empty() && statfs_ops.empty());
+  }
+
+  /**
+   * Output in-flight requests
+   */
+  void dump_active();
+  void dump_requests(Formatter *fmt) const;
+  void dump_ops(Formatter *fmt) const;
+  void dump_linger_ops(Formatter *fmt) const;
+  void dump_command_ops(Formatter *fmt) const;
+  void dump_pool_ops(Formatter *fmt) const;
+  void dump_pool_stat_ops(Formatter *fmt) const;
+  void dump_statfs_ops(Formatter *fmt) const;
+
+  int get_client_incarnation() const { return client_inc; }
+  void set_client_incarnation(int inc) { client_inc = inc; }
+
+  void wait_for_new_map(Context *c, epoch_t epoch, int err=0);
+  void wait_for_latest_osdmap(Context *fin);
+  void _get_latest_version(epoch_t oldest, epoch_t neweset, Context *fin);
+
+  /** Get the current set of global op flags */
+  int get_global_op_flags() { return global_op_flags; }
+  /** Add a flag to the global op flags */
+  void add_global_op_flags(int flag) { global_op_flags |= flag; }
+  /** Clear the passed flags from the global op flag set */
+  void clear_global_op_flag(int flags) { global_op_flags &= ~flags; }
+
+  /// cancel an in-progress request with the given return code
+  int op_cancel(ceph_tid_t tid, int r);
+
+  // commands
+  int osd_command(int osd, vector& cmd,
+		  const bufferlist& inbl, ceph_tid_t *ptid,
+		  bufferlist *poutbl, string *prs, Context *onfinish) {
+    assert(osd >= 0);
+    CommandOp *c = new CommandOp;
+    c->cmd = cmd;
+    c->inbl = inbl;
+    c->poutbl = poutbl;
+    c->prs = prs;
+    c->onfinish = onfinish;
+    c->target_osd = osd;
+    return _submit_command(c, ptid);
+  }
+  int pg_command(pg_t pgid, vector& cmd,
+		 const bufferlist& inbl, ceph_tid_t *ptid,
+		 bufferlist *poutbl, string *prs, Context *onfinish) {
+    CommandOp *c = new CommandOp;
+    c->cmd = cmd;
+    c->inbl = inbl;
+    c->poutbl = poutbl;
+    c->prs = prs;
+    c->onfinish = onfinish;
+    c->target_pg = pgid;
+    return _submit_command(c, ptid);
+  }
+
+  // mid-level helpers
+  Op *prepare_mutate_op(const object_t& oid, const object_locator_t& oloc,
+	       ObjectOperation& op,
+	       const SnapContext& snapc, utime_t mtime, int flags,
+	       Context *onack, Context *oncommit, version_t *objver = NULL) {
+    Op *o = new Op(oid, oloc, op.ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->priority = op.priority;
+    o->mtime = mtime;
+    o->snapc = snapc;
+    o->out_rval.swap(op.out_rval);
+    return o;
+  }
+  ceph_tid_t mutate(const object_t& oid, const object_locator_t& oloc,
+	       ObjectOperation& op,
+	       const SnapContext& snapc, utime_t mtime, int flags,
+	       Context *onack, Context *oncommit, version_t *objver = NULL) {
+    Op *o = prepare_mutate_op(oid, oloc, op, snapc, mtime, flags, onack, oncommit, objver);
+    return op_submit(o);
+  }
+  Op *prepare_read_op(const object_t& oid, const object_locator_t& oloc,
+	     ObjectOperation& op,
+	     snapid_t snapid, bufferlist *pbl, int flags,
+	     Context *onack, version_t *objver = NULL) {
+    Op *o = new Op(oid, oloc, op.ops, flags | global_op_flags | CEPH_OSD_FLAG_READ, onack, NULL, objver);
+    o->priority = op.priority;
+    o->snapid = snapid;
+    o->outbl = pbl;
+    o->out_bl.swap(op.out_bl);
+    o->out_handler.swap(op.out_handler);
+    o->out_rval.swap(op.out_rval);
+    return o;
+  }
+  ceph_tid_t read(const object_t& oid, const object_locator_t& oloc,
+	     ObjectOperation& op,
+	     snapid_t snapid, bufferlist *pbl, int flags,
+	     Context *onack, version_t *objver = NULL) {
+    Op *o = prepare_read_op(oid, oloc, op, snapid, pbl, flags, onack, objver);
+    return op_submit(o);
+  }
+  ceph_tid_t pg_read(uint32_t hash, object_locator_t oloc,
+		ObjectOperation& op,
+		bufferlist *pbl, int flags,
+		Context *onack,
+		epoch_t *reply_epoch) {
+    Op *o = new Op(object_t(), oloc,
+		   op.ops, flags | global_op_flags | CEPH_OSD_FLAG_READ,
+		   onack, NULL, NULL);
+    o->target.precalc_pgid = true;
+    o->target.base_pgid = pg_t(hash, oloc.pool);
+    o->priority = op.priority;
+    o->snapid = CEPH_NOSNAP;
+    o->outbl = pbl;
+    o->out_bl.swap(op.out_bl);
+    o->out_handler.swap(op.out_handler);
+    o->out_rval.swap(op.out_rval);
+    o->reply_epoch = reply_epoch;
+    return op_submit(o);
+  }
+  ceph_tid_t linger_mutate(const object_t& oid, const object_locator_t& oloc,
+		      ObjectOperation& op,
+		      const SnapContext& snapc, utime_t mtime,
+		      bufferlist& inbl, int flags,
+		      Context *onack, Context *onfinish,
+		      version_t *objver);
+  ceph_tid_t linger_read(const object_t& oid, const object_locator_t& oloc,
+		    ObjectOperation& op,
+		    snapid_t snap, bufferlist& inbl, bufferlist *poutbl, int flags,
+		    Context *onack,
+		    version_t *objver);
+  void unregister_linger(uint64_t linger_id);
+
+  /**
+   * set up initial ops in the op vector, and allocate a final op slot.
+   *
+   * The caller is responsible for filling in the final ops_count ops.
+   *
+   * @param ops op vector
+   * @param ops_count number of final ops the caller will fill in
+   * @param extra_ops pointer to [array of] initial op[s]
+   * @return index of final op (for caller to fill in)
+   */
+  int init_ops(vector& ops, int ops_count, ObjectOperation *extra_ops) {
+    int i;
+    int extra = 0;
+
+    if (extra_ops)
+      extra = extra_ops->ops.size();
+
+    ops.resize(ops_count + extra);
+
+    for (i=0; iops[i];
+    }
+
+    return i;
+  }
+
+
+  // high-level helpers
+  ceph_tid_t stat(const object_t& oid, const object_locator_t& oloc, snapid_t snap,
+	     uint64_t *psize, utime_t *pmtime, int flags, 
+	     Context *onfinish,
+	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_STAT;
+    C_Stat *fin = new C_Stat(psize, pmtime, onfinish);
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_READ, fin, 0, objver);
+    o->snapid = snap;
+    o->outbl = &fin->bl;
+    return op_submit(o);
+  }
+
+  ceph_tid_t read(const object_t& oid, const object_locator_t& oloc,
+	     uint64_t off, uint64_t len, snapid_t snap, bufferlist *pbl, int flags,
+	     Context *onfinish,
+	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_READ;
+    ops[i].op.extent.offset = off;
+    ops[i].op.extent.length = len;
+    ops[i].op.extent.truncate_size = 0;
+    ops[i].op.extent.truncate_seq = 0;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_READ, onfinish, 0, objver);
+    o->snapid = snap;
+    o->outbl = pbl;
+    return op_submit(o);
+  }
+
+  ceph_tid_t read_trunc(const object_t& oid, const object_locator_t& oloc,
+	     uint64_t off, uint64_t len, snapid_t snap, bufferlist *pbl, int flags,
+	     uint64_t trunc_size, __u32 trunc_seq,
+	     Context *onfinish,
+	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_READ;
+    ops[i].op.extent.offset = off;
+    ops[i].op.extent.length = len;
+    ops[i].op.extent.truncate_size = trunc_size;
+    ops[i].op.extent.truncate_seq = trunc_seq;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_READ, onfinish, 0, objver);
+    o->snapid = snap;
+    o->outbl = pbl;
+    return op_submit(o);
+  }
+  ceph_tid_t mapext(const object_t& oid, const object_locator_t& oloc,
+	     uint64_t off, uint64_t len, snapid_t snap, bufferlist *pbl, int flags,
+	     Context *onfinish,
+	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_MAPEXT;
+    ops[i].op.extent.offset = off;
+    ops[i].op.extent.length = len;
+    ops[i].op.extent.truncate_size = 0;
+    ops[i].op.extent.truncate_seq = 0;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_READ, onfinish, 0, objver);
+    o->snapid = snap;
+    o->outbl = pbl;
+    return op_submit(o);
+  }
+  ceph_tid_t getxattr(const object_t& oid, const object_locator_t& oloc,
+	     const char *name, snapid_t snap, bufferlist *pbl, int flags,
+	     Context *onfinish,
+	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_GETXATTR;
+    ops[i].op.xattr.name_len = (name ? strlen(name) : 0);
+    ops[i].op.xattr.value_len = 0;
+    if (name)
+      ops[i].indata.append(name);
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_READ, onfinish, 0, objver);
+    o->snapid = snap;
+    o->outbl = pbl;
+    return op_submit(o);
+  }
+
+  ceph_tid_t getxattrs(const object_t& oid, const object_locator_t& oloc, snapid_t snap,
+		  map& attrset,
+		  int flags, Context *onfinish,
+		  version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_GETXATTRS;
+    C_GetAttrs *fin = new C_GetAttrs(attrset, onfinish);
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_READ, fin, 0, objver);
+    o->snapid = snap;
+    o->outbl = &fin->bl;
+    return op_submit(o);
+  }
+
+  ceph_tid_t read_full(const object_t& oid, const object_locator_t& oloc,
+		  snapid_t snap, bufferlist *pbl, int flags,
+		  Context *onfinish,
+	          version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    return read(oid, oloc, 0, 0, snap, pbl, flags | global_op_flags | CEPH_OSD_FLAG_READ, onfinish, objver);
+  }
+
+     
+  // writes
+  ceph_tid_t _modify(const object_t& oid, const object_locator_t& oloc,
+		vector& ops, utime_t mtime,
+		const SnapContext& snapc, int flags,
+	        Context *onack, Context *oncommit,
+	        version_t *objver = NULL) {
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t write(const object_t& oid, const object_locator_t& oloc,
+	      uint64_t off, uint64_t len, const SnapContext& snapc, const bufferlist &bl,
+	      utime_t mtime, int flags,
+	      Context *onack, Context *oncommit,
+	      version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_WRITE;
+    ops[i].op.extent.offset = off;
+    ops[i].op.extent.length = len;
+    ops[i].op.extent.truncate_size = 0;
+    ops[i].op.extent.truncate_seq = 0;
+    ops[i].indata = bl;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t append(const object_t& oid, const object_locator_t& oloc,
+	       uint64_t len, const SnapContext& snapc, const bufferlist &bl,
+	       utime_t mtime, int flags,
+	       Context *onack, Context *oncommit,
+	       version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_APPEND;
+    ops[i].op.extent.offset = 0;
+    ops[i].op.extent.length = len;
+    ops[i].op.extent.truncate_size = 0;
+    ops[i].op.extent.truncate_seq = 0;
+    ops[i].indata = bl;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t write_trunc(const object_t& oid, const object_locator_t& oloc,
+	      uint64_t off, uint64_t len, const SnapContext& snapc, const bufferlist &bl,
+	      utime_t mtime, int flags,
+	      uint64_t trunc_size, __u32 trunc_seq,
+	      Context *onack, Context *oncommit,
+	      version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_WRITE;
+    ops[i].op.extent.offset = off;
+    ops[i].op.extent.length = len;
+    ops[i].op.extent.truncate_size = trunc_size;
+    ops[i].op.extent.truncate_seq = trunc_seq;
+    ops[i].indata = bl;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t write_full(const object_t& oid, const object_locator_t& oloc,
+		   const SnapContext& snapc, const bufferlist &bl, utime_t mtime, int flags,
+		   Context *onack, Context *oncommit,
+		   version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_WRITEFULL;
+    ops[i].op.extent.offset = 0;
+    ops[i].op.extent.length = bl.length();
+    ops[i].indata = bl;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t trunc(const object_t& oid, const object_locator_t& oloc,
+	      const SnapContext& snapc,
+	      utime_t mtime, int flags,
+	      uint64_t trunc_size, __u32 trunc_seq,
+              Context *onack, Context *oncommit,
+	      version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_TRUNCATE;
+    ops[i].op.extent.offset = trunc_size;
+    ops[i].op.extent.truncate_size = trunc_size;
+    ops[i].op.extent.truncate_seq = trunc_seq;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t zero(const object_t& oid, const object_locator_t& oloc,
+	     uint64_t off, uint64_t len, const SnapContext& snapc, utime_t mtime, int flags,
+             Context *onack, Context *oncommit,
+	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_ZERO;
+    ops[i].op.extent.offset = off;
+    ops[i].op.extent.length = len;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t rollback_object(const object_t& oid, const object_locator_t& oloc,
+		 const SnapContext& snapc, snapid_t snapid,
+		 utime_t mtime, Context *onack, Context *oncommit,
+		 version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_ROLLBACK;
+    ops[i].op.snap.snapid = snapid;
+    Op *o = new Op(oid, oloc, ops, CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t create(const object_t& oid, const object_locator_t& oloc,
+	     const SnapContext& snapc, utime_t mtime,
+             int global_flags, int create_flags,
+             Context *onack, Context *oncommit,
+             version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_CREATE;
+    ops[i].op.flags = create_flags;
+    Op *o = new Op(oid, oloc, ops, global_flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t remove(const object_t& oid, const object_locator_t& oloc,
+	       const SnapContext& snapc, utime_t mtime, int flags,
+	       Context *onack, Context *oncommit,
+	       version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_DELETE;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+
+  ceph_tid_t lock(const object_t& oid, const object_locator_t& oloc, int op, int flags,
+	     Context *onack, Context *oncommit, version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    SnapContext snapc;  // no snapc for lock ops
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = op;
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t setxattr(const object_t& oid, const object_locator_t& oloc,
+	      const char *name, const SnapContext& snapc, const bufferlist &bl,
+	      utime_t mtime, int flags,
+	      Context *onack, Context *oncommit,
+	      version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_SETXATTR;
+    ops[i].op.xattr.name_len = (name ? strlen(name) : 0);
+    ops[i].op.xattr.value_len = bl.length();
+    if (name)
+      ops[i].indata.append(name);
+    ops[i].indata.append(bl);
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+  ceph_tid_t removexattr(const object_t& oid, const object_locator_t& oloc,
+	      const char *name, const SnapContext& snapc,
+	      utime_t mtime, int flags,
+	      Context *onack, Context *oncommit,
+	      version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
+    vector ops;
+    int i = init_ops(ops, 1, extra_ops);
+    ops[i].op.op = CEPH_OSD_OP_RMXATTR;
+    ops[i].op.xattr.name_len = (name ? strlen(name) : 0);
+    ops[i].op.xattr.value_len = 0;
+    if (name)
+      ops[i].indata.append(name);
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags | CEPH_OSD_FLAG_WRITE, onack, oncommit, objver);
+    o->mtime = mtime;
+    o->snapc = snapc;
+    return op_submit(o);
+  }
+
+  void list_objects(ListContext *p, Context *onfinish);
+  uint32_t list_objects_seek(ListContext *p, uint32_t pos);
+
+  // -------------------------
+  // pool ops
+private:
+  void pool_op_submit(PoolOp *op);
+  void _pool_op_submit(PoolOp *op);
+public:
+  int create_pool_snap(int64_t pool, string& snapName, Context *onfinish);
+  int allocate_selfmanaged_snap(int64_t pool, snapid_t *psnapid, Context *onfinish);
+  int delete_pool_snap(int64_t pool, string& snapName, Context *onfinish);
+  int delete_selfmanaged_snap(int64_t pool, snapid_t snap, Context *onfinish);
+
+  int create_pool(string& name, Context *onfinish, uint64_t auid=0,
+		  int crush_rule=-1);
+  int delete_pool(int64_t pool, Context *onfinish);
+  int change_pool_auid(int64_t pool, Context *onfinish, uint64_t auid);
+
+  void handle_pool_op_reply(MPoolOpReply *m);
+  int pool_op_cancel(ceph_tid_t tid, int r);
+  void finish_pool_op(PoolOp *op);
+
+  // --------------------------
+  // pool stats
+private:
+  void poolstat_submit(PoolStatOp *op);
+public:
+  void handle_get_pool_stats_reply(MGetPoolStatsReply *m);
+  void get_pool_stats(list& pools, map *result,
+		      Context *onfinish);
+  int pool_stat_op_cancel(ceph_tid_t tid, int r);
+  void finish_pool_stat_op(PoolStatOp *op);
+
+  // ---------------------------
+  // df stats
+private:
+  void fs_stats_submit(StatfsOp *op);
+public:
+  void handle_fs_stats_reply(MStatfsReply *m);
+  void get_fs_stats(struct ceph_statfs& result, Context *onfinish);
+  int statfs_op_cancel(ceph_tid_t tid, int r);
+  void finish_statfs_op(StatfsOp *op);
+
+  // ---------------------------
+  // some scatter/gather hackery
+
+  void _sg_read_finish(vector& extents, vector& resultbl, 
+		       bufferlist *bl, Context *onfinish);
+
+  struct C_SGRead : public Context {
+    Objecter *objecter;
+    vector extents;
+    vector resultbl;
+    bufferlist *bl;
+    Context *onfinish;
+    C_SGRead(Objecter *ob, 
+	     vector& e, vector& r, bufferlist *b, Context *c) :
+      objecter(ob), bl(b), onfinish(c) {
+      extents.swap(e);
+      resultbl.swap(r);
+    }
+    void finish(int r) {
+      objecter->_sg_read_finish(extents, resultbl, bl, onfinish);
+    }      
+  };
+
+  void sg_read_trunc(vector& extents, snapid_t snap, bufferlist *bl, int flags,
+		uint64_t trunc_size, __u32 trunc_seq, Context *onfinish) {
+    if (extents.size() == 1) {
+      read_trunc(extents[0].oid, extents[0].oloc, extents[0].offset, extents[0].length,
+	   snap, bl, flags, extents[0].truncate_size, trunc_seq, onfinish);
+    } else {
+      C_GatherBuilder gather(cct);
+      vector resultbl(extents.size());
+      int i=0;
+      for (vector::iterator p = extents.begin(); p != extents.end(); ++p) {
+	read_trunc(p->oid, p->oloc, p->offset, p->length,
+	     snap, &resultbl[i++], flags, p->truncate_size, trunc_seq, gather.new_sub());
+      }
+      gather.set_finisher(new C_SGRead(this, extents, resultbl, bl, onfinish));
+      gather.activate();
+    }
+  }
+
+  void sg_read(vector& extents, snapid_t snap, bufferlist *bl, int flags, Context *onfinish) {
+    sg_read_trunc(extents, snap, bl, flags, 0, 0, onfinish);
+  }
+
+  void sg_write_trunc(vector& extents, const SnapContext& snapc, const bufferlist& bl, utime_t mtime,
+		int flags, uint64_t trunc_size, __u32 trunc_seq,
+		Context *onack, Context *oncommit) {
+    if (extents.size() == 1) {
+      write_trunc(extents[0].oid, extents[0].oloc, extents[0].offset, extents[0].length,
+	    snapc, bl, mtime, flags, extents[0].truncate_size, trunc_seq, onack, oncommit);
+    } else {
+      C_GatherBuilder gack(cct, onack);
+      C_GatherBuilder gcom(cct, oncommit);
+      for (vector::iterator p = extents.begin(); p != extents.end(); ++p) {
+	bufferlist cur;
+	for (vector >::iterator bit = p->buffer_extents.begin();
+	     bit != p->buffer_extents.end();
+	     ++bit)
+	  bl.copy(bit->first, bit->second, cur);
+	assert(cur.length() == p->length);
+	write_trunc(p->oid, p->oloc, p->offset, p->length, 
+	      snapc, cur, mtime, flags, p->truncate_size, trunc_seq,
+	      onack ? gack.new_sub():0,
+	      oncommit ? gcom.new_sub():0);
+      }
+      gack.activate();
+      gcom.activate();
+    }
+  }
+
+  void sg_write(vector& extents, const SnapContext& snapc, const bufferlist& bl, utime_t mtime,
+		int flags, Context *onack, Context *oncommit) {
+    sg_write_trunc(extents, snapc, bl, mtime, flags, 0, 0, onack, oncommit);
+  }
+
+  void ms_handle_connect(Connection *con);
+  void ms_handle_reset(Connection *con);
+  void ms_handle_remote_reset(Connection *con);
+  void blacklist_self(bool set);
+};
+
+#endif
diff --git a/ceph/src/osdc/Striper.cc b/ceph/src/osdc/Striper.cc
new file mode 100644
index 00000000..bde32846
--- /dev/null
+++ b/ceph/src/osdc/Striper.cc
@@ -0,0 +1,340 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "Striper.h"
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "osd/OSDMap.h"
+
+#include "common/config.h"
+#include "common/debug.h"
+
+#define dout_subsys ceph_subsys_striper
+#undef dout_prefix
+#define dout_prefix *_dout << "striper "
+
+
+void Striper::file_to_extents(CephContext *cct, const char *object_format,
+			    ceph_file_layout *layout,
+			    uint64_t offset, uint64_t len, uint64_t trunc_size,
+			    vector& extents,
+			    uint64_t buffer_offset)
+{
+  map > object_extents;
+  file_to_extents(cct, object_format, layout, offset, len, trunc_size,
+		  object_extents, buffer_offset);
+  assimilate_extents(object_extents, extents);
+}
+
+void Striper::file_to_extents(CephContext *cct, const char *object_format,
+			      ceph_file_layout *layout,
+			      uint64_t offset, uint64_t len, uint64_t trunc_size,
+			      map >& object_extents,
+			      uint64_t buffer_offset)
+{
+  ldout(cct, 10) << "file_to_extents " << offset << "~" << len 
+		 << " format " << object_format
+		 << dendl;
+  assert(len > 0);
+
+  /*
+   * we want only one extent per object!
+   * this means that each extent we read may map into different bits of the 
+   * final read buffer.. hence OSDExtent.buffer_extents
+   */
+  
+  __u32 object_size = layout->fl_object_size;
+  __u32 su = layout->fl_stripe_unit;
+  __u32 stripe_count = layout->fl_stripe_count;
+  assert(object_size >= su);
+  uint64_t stripes_per_object = object_size / su;
+  ldout(cct, 20) << " su " << su << " sc " << stripe_count << " os " << object_size
+		 << " stripes_per_object " << stripes_per_object << dendl;
+
+  uint64_t cur = offset;
+  uint64_t left = len;
+  while (left > 0) {
+    // layout into objects
+    uint64_t blockno = cur / su;          // which block
+    uint64_t stripeno = blockno / stripe_count;    // which horizontal stripe        (Y)
+    uint64_t stripepos = blockno % stripe_count;   // which object in the object set (X)
+    uint64_t objectsetno = stripeno / stripes_per_object;       // which object set
+    uint64_t objectno = objectsetno * stripe_count + stripepos;  // object id
+
+    // find oid, extent
+    char buf[strlen(object_format) + 32];
+    snprintf(buf, sizeof(buf), object_format, (long long unsigned)objectno);
+    object_t oid = buf;
+
+    // map range into object
+    uint64_t block_start = (stripeno % stripes_per_object) * su;
+    uint64_t block_off = cur % su;
+    uint64_t max = su - block_off;
+
+    uint64_t x_offset = block_start + block_off;
+    uint64_t x_len;
+    if (left > max)
+      x_len = max;
+    else
+      x_len = left;
+
+    ldout(cct, 20) << " off " << cur << " blockno " << blockno << " stripeno " << stripeno
+		   << " stripepos " << stripepos << " objectsetno " << objectsetno
+		   << " objectno " << objectno
+		   << " block_start " << block_start
+		   << " block_off " << block_off
+		   << " " << x_offset << "~" << x_len
+		   << dendl;
+
+    ObjectExtent *ex = 0;
+    vector& exv = object_extents[oid];
+    if (exv.empty() || exv.back().offset + exv.back().length != x_offset) {
+      exv.resize(exv.size() + 1);
+      ex = &exv.back();
+      ex->oid = oid;
+      ex->objectno = objectno;
+      ex->oloc = OSDMap::file_to_object_locator(*layout);
+
+      ex->offset = x_offset;
+      ex->length = x_len;
+      ex->truncate_size = object_truncate_size(cct, layout, objectno, trunc_size);
+
+      ldout(cct, 20) << " added new " << *ex << dendl;
+    } else {
+      // add to extent
+      ex = &exv.back();
+      ldout(cct, 20) << " adding in to " << *ex << dendl;
+      ex->length += x_len;
+    }
+    ex->buffer_extents.push_back(make_pair(cur - offset + buffer_offset, x_len));
+        
+    ldout(cct, 15) << "file_to_extents  " << *ex << " in " << ex->oloc << dendl;
+    //ldout(cct, 0) << "map: ino " << ino << " oid " << ex.oid << " osd " << ex.osd << " offset " << ex.offset << " len " << ex.len << " ... left " << left << dendl;
+    
+    left -= x_len;
+    cur += x_len;
+  }
+}
+
+void Striper::assimilate_extents(map >& object_extents,
+				 vector& extents)
+{
+  // make final list
+  for (map >::iterator it = object_extents.begin();
+       it != object_extents.end();
+       ++it) {
+    for (vector::iterator p = it->second.begin(); p != it->second.end(); ++p) {
+      extents.push_back(*p);
+    }
+  }
+}
+
+void Striper::extent_to_file(CephContext *cct, ceph_file_layout *layout,
+			   uint64_t objectno, uint64_t off, uint64_t len,
+			   vector >& extents)
+{
+  ldout(cct, 10) << "extent_to_file " << objectno << " " << off << "~" << len << dendl;
+
+  __u32 object_size = layout->fl_object_size;
+  __u32 su = layout->fl_stripe_unit;
+  __u32 stripe_count = layout->fl_stripe_count;
+  assert(object_size >= su);
+  uint64_t stripes_per_object = object_size / su;
+  ldout(cct, 20) << " stripes_per_object " << stripes_per_object << dendl;
+
+  uint64_t off_in_block = off % su;
+
+  extents.reserve(len / su + 1);
+
+  while (len > 0) {
+    uint64_t stripepos = objectno % stripe_count;
+    uint64_t objectsetno = objectno / stripe_count;
+    uint64_t stripeno = off / su + objectsetno * stripes_per_object;
+    uint64_t blockno = stripeno * stripe_count + stripepos;
+    uint64_t extent_off = blockno * su + off_in_block;
+    uint64_t extent_len = MIN(len, su - off_in_block);
+    extents.push_back(make_pair(extent_off, extent_len));
+
+    ldout(cct, 20) << " object " << off << "~" << extent_len
+		   << " -> file " << extent_off << "~" << extent_len
+		   << dendl;
+
+    off_in_block = 0;
+    off += extent_len;
+    len -= extent_len;
+  }
+}
+
+uint64_t Striper::object_truncate_size(CephContext *cct, ceph_file_layout *layout,
+				       uint64_t objectno, uint64_t trunc_size)
+{
+  uint64_t obj_trunc_size;
+  if (trunc_size == 0 || trunc_size == (uint64_t)-1) {
+    obj_trunc_size = trunc_size;
+  } else {
+    __u32 object_size = layout->fl_object_size;
+    __u32 su = layout->fl_stripe_unit;
+    __u32 stripe_count = layout->fl_stripe_count;
+    assert(object_size >= su);
+    uint64_t stripes_per_object = object_size / su;
+
+    uint64_t objectsetno = objectno / stripe_count;
+    uint64_t trunc_objectsetno = trunc_size / object_size / stripe_count;
+    if (objectsetno > trunc_objectsetno)
+      obj_trunc_size = 0;
+    else if (objectsetno < trunc_objectsetno)
+      obj_trunc_size = object_size;
+    else {
+      uint64_t trunc_blockno = trunc_size / su;
+      uint64_t trunc_stripeno = trunc_blockno / stripe_count;
+      uint64_t trunc_stripepos = trunc_blockno % stripe_count;
+      uint64_t trunc_objectno = trunc_objectsetno * stripe_count + trunc_stripepos;
+      if (objectno < trunc_objectno)
+	obj_trunc_size = ((trunc_stripeno % stripes_per_object) + 1) * su;
+      else if (objectno > trunc_objectno)
+	obj_trunc_size = (trunc_stripeno % stripes_per_object) * su;
+      else
+	obj_trunc_size = (trunc_stripeno % stripes_per_object) * su + (trunc_size % su);
+    }
+  }
+  ldout(cct, 20) << "object_truncate_size " << objectno << " "
+		 << trunc_size << "->" << obj_trunc_size << dendl;
+  return obj_trunc_size;
+}
+
+// StripedReadResult
+
+void Striper::StripedReadResult::add_partial_result(CephContext *cct,
+						    bufferlist& bl,
+						    const vector >& buffer_extents)
+{
+  ldout(cct, 10) << "add_partial_result(" << this << ") " << bl.length() << " to " << buffer_extents << dendl;
+  for (vector >::const_iterator p = buffer_extents.begin();
+       p != buffer_extents.end();
+       ++p) {
+    pair& r = partial[p->first];
+    size_t actual = MIN(bl.length(), p->second);
+    bl.splice(0, actual, &r.first);
+    r.second = p->second;
+  }
+}
+
+void Striper::StripedReadResult::add_partial_sparse_result(CephContext *cct,
+							 bufferlist& bl, const map& bl_map,
+							 uint64_t bl_off,
+							 const vector >& buffer_extents)
+{
+  ldout(cct, 10) << "add_partial_sparse_result(" << this << ") " << bl.length()
+		 << " covering " << bl_map << " (offset " << bl_off << ")"
+		 << " to " << buffer_extents << dendl;
+  map::const_iterator s = bl_map.begin();
+  for (vector >::const_iterator p = buffer_extents.begin();
+       p != buffer_extents.end();
+       ++p) {
+    uint64_t tofs = p->first;
+    uint64_t tlen = p->second;
+    ldout(cct, 30) << " be " << tofs << "~" << tlen << dendl;
+    while (tlen > 0) {
+      ldout(cct, 20) << "  t " << tofs << "~" << tlen
+		     << " bl has " << bl.length()
+		     << " off " << bl_off
+		     << dendl;
+      if (s == bl_map.end()) {
+	ldout(cct, 20) << "  s at end" << dendl;
+	pair& r = partial[tofs];
+	r.second = tlen;
+	break;
+      }
+
+      ldout(cct, 30) << "  s " << s->first << "~" << s->second << dendl;
+
+      // skip zero-length extent
+      if (s->second == 0) {
+	ldout(cct, 30) << "  s len 0, skipping" << dendl;
+	++s;
+	continue;
+      }
+
+      if (s->first > bl_off) {
+	// gap in sparse read result
+	pair& r = partial[tofs];
+	size_t gap = MIN(s->first - bl_off, tlen);
+	ldout(cct, 20) << "  s gap " << gap << ", skipping" << dendl;
+	r.second = gap;
+	bl_off += gap;
+	tofs += gap;
+	tlen -= gap;
+	if (tlen == 0) {
+	  continue;
+	}
+      }
+
+      assert(s->first <= bl_off);
+      size_t left = (s->first + s->second) - bl_off;
+      size_t actual = MIN(left, tlen);
+
+      if (actual > 0) {
+	ldout(cct, 20) << "  s has " << actual << ", copying" << dendl;
+	pair& r = partial[tofs];
+	bl.splice(0, actual, &r.first);
+	r.second = actual;
+	bl_off += actual;
+	tofs += actual;
+	tlen -= actual;
+      }
+      if (actual == left) {
+	ldout(cct, 30) << "  s advancing" << dendl;
+	++s;
+      }
+    }
+  }
+}
+
+void Striper::StripedReadResult::assemble_result(CephContext *cct, bufferlist& bl, bool zero_tail)
+{
+  ldout(cct, 10) << "assemble_result(" << this << ") zero_tail=" << zero_tail << dendl;
+
+  // go backwards, so that we can efficiently discard zeros
+  map >::reverse_iterator p = partial.rbegin();
+  if (p == partial.rend())
+    return;
+
+  uint64_t end = p->first + p->second.second;
+  while (p != partial.rend()) {
+    // sanity check
+    ldout(cct, 20) << "assemble_result(" << this << ") " << p->first << "~" << p->second.second
+		   << " " << p->second.first.length() << " bytes"
+		   << dendl;
+    assert(p->first == end - p->second.second);
+    end = p->first;
+
+    size_t len = p->second.first.length();
+    if (len < p->second.second) {
+      if (zero_tail || bl.length()) {
+	bufferptr bp(p->second.second - len);
+	bp.zero();
+	bl.push_front(bp);
+	bl.claim_prepend(p->second.first);
+      } else {
+	bl.claim_prepend(p->second.first);
+      }
+    } else {
+      bl.claim_prepend(p->second.first);
+    }
+    ++p;
+  }
+  partial.clear();
+}
+
diff --git a/ceph/src/osdc/Striper.h b/ceph/src/osdc/Striper.h
new file mode 100644
index 00000000..bd61fc8f
--- /dev/null
+++ b/ceph/src/osdc/Striper.h
@@ -0,0 +1,97 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_STRIPER_H
+#define CEPH_STRIPER_H
+
+#include "include/types.h"
+#include "osd/osd_types.h"
+
+class CephContext;
+
+//namespace ceph {
+
+  class Striper {
+  public:
+    /*
+     * map (ino, layout, offset, len) to a (list of) OSDExtents (byte
+     * ranges in objects on (primary) osds)
+     */
+    static void file_to_extents(CephContext *cct, const char *object_format,
+				ceph_file_layout *layout,
+				uint64_t offset, uint64_t len, uint64_t trunc_size,
+				map >& extents,
+				uint64_t buffer_offset=0);
+
+    static void file_to_extents(CephContext *cct, const char *object_format,
+				ceph_file_layout *layout,
+				uint64_t offset, uint64_t len, uint64_t trunc_size,
+				vector& extents,
+				uint64_t buffer_offset=0);
+
+    static void file_to_extents(CephContext *cct, inodeno_t ino,
+				ceph_file_layout *layout,
+				uint64_t offset, uint64_t len, uint64_t trunc_size,
+				vector& extents) {
+      // generate prefix/format
+      char buf[32];
+      snprintf(buf, sizeof(buf), "%llx.%%08llx", (long long unsigned)ino);
+
+      file_to_extents(cct, buf, layout, offset, len, trunc_size, extents);
+    }
+
+    static void assimilate_extents(map >& object_extents,
+				   vector& extents);
+
+    /**
+     * reverse map an object extent to file extents
+     */
+    static void extent_to_file(CephContext *cct, ceph_file_layout *layout,
+			       uint64_t objectno, uint64_t off, uint64_t len,
+			       vector >& extents);
+
+    static uint64_t object_truncate_size(CephContext *cct, ceph_file_layout *layout,
+					 uint64_t objectno, uint64_t trunc_size);
+
+    /*
+     * helper to assemble a striped result
+     */
+    class StripedReadResult {
+      map > partial;  // offset -> (data, intended length)
+
+    public:
+      void add_partial_result(CephContext *cct,
+			      bufferlist& bl,
+			      const vector >& buffer_extents);
+      /**
+       * add sparse read into results
+       *
+       * @param bl buffer
+       * @param bl_map map of which logical source extents this covers
+       * @param bl_off logical buffer offset (e.g., first bl_map key if the buffer is not sparse)
+       * @param buffer_extents output buffer extents the data maps to
+       */
+      void add_partial_sparse_result(CephContext *cct,
+				     bufferlist& bl, const map& bl_map,
+				     uint64_t bl_off,
+				     const vector >& buffer_extents);
+
+      void assemble_result(CephContext *cct, bufferlist& bl, bool zero_tail);
+    };
+
+  };
+
+//};
+
+#endif
diff --git a/ceph/src/osdc/WritebackHandler.h b/ceph/src/osdc/WritebackHandler.h
new file mode 100644
index 00000000..48698373
--- /dev/null
+++ b/ceph/src/osdc/WritebackHandler.h
@@ -0,0 +1,42 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_OSDC_WRITEBACKHANDLER_H
+#define CEPH_OSDC_WRITEBACKHANDLER_H
+
+#include "include/Context.h"
+#include "include/types.h"
+#include "osd/osd_types.h"
+
+class WritebackHandler {
+ public:
+  WritebackHandler() {}
+  virtual ~WritebackHandler() {}
+
+  virtual void read(const object_t& oid, const object_locator_t& oloc,
+		    uint64_t off, uint64_t len, snapid_t snapid,
+		    bufferlist *pbl, uint64_t trunc_size,  __u32 trunc_seq,
+		    Context *onfinish) = 0;
+  /**
+   * check if a given extent read result may change due to a write
+   *
+   * Check if the content we see at the given read offset may change due to a write to
+   * this object.
+   *
+   * @param oid object
+   * @param read_off read offset
+   * @param read_len read length
+   * @param snapid read snapid
+   */
+  virtual bool may_copy_on_write(const object_t& oid, uint64_t read_off, uint64_t read_len, snapid_t snapid) = 0;
+  virtual ceph_tid_t write(const object_t& oid, const object_locator_t& oloc,
+			   uint64_t off, uint64_t len, const SnapContext& snapc,
+			   const bufferlist &bl, utime_t mtime,
+			   uint64_t trunc_size, __u32 trunc_seq,
+			   Context *oncommit) = 0;
+  virtual ceph_tid_t lock(const object_t& oid, const object_locator_t& oloc,
+			  int op, int flags, Context *onack, Context *oncommit) {
+    assert(0 == "this WritebackHandler does not support the lock operation");
+  }
+};
+
+#endif
diff --git a/ceph/src/perfglue/Makefile.am b/ceph/src/perfglue/Makefile.am
new file mode 100644
index 00000000..f2b8d503
--- /dev/null
+++ b/ceph/src/perfglue/Makefile.am
@@ -0,0 +1,23 @@
+libperfglue_la_SOURCES =
+
+if WITH_TCMALLOC
+libperfglue_la_SOURCES += perfglue/heap_profiler.cc
+libperfglue_la_LIBADD = -ltcmalloc
+AM_CFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
+AM_CXXFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
+else
+libperfglue_la_SOURCES += perfglue/disabled_heap_profiler.cc
+endif # WITH_TCMALLOC
+
+if WITH_PROFILER
+libperfglue_la_SOURCES += perfglue/cpu_profiler.cc
+else
+libperfglue_la_SOURCES += perfglue/disabled_stubs.cc
+endif # WITH_PROFILER
+
+noinst_LTLIBRARIES += libperfglue.la
+
+noinst_HEADERS += \
+	perfglue/cpu_profiler.h \
+	perfglue/heap_profiler.h
+
diff --git a/ceph/src/perfglue/cpu_profiler.cc b/ceph/src/perfglue/cpu_profiler.cc
new file mode 100644
index 00000000..8ba15b0a
--- /dev/null
+++ b/ceph/src/perfglue/cpu_profiler.cc
@@ -0,0 +1,39 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/LogClient.h"
+#include "perfglue/cpu_profiler.h"
+
+#include 
+
+void cpu_profiler_handle_command(const std::vector &cmd,
+				 ostream& out)
+{
+  if (cmd[1] == "status") {
+    ProfilerState st;
+    ProfilerGetCurrentState(&st);
+    out << "cpu_profiler " << (st.enabled ? "enabled":"not enabled")
+	<< " start_time " << st.start_time
+	<< " profile_name " << st.profile_name
+	<< " samples " << st.samples_gathered;
+  }
+  else if (cmd[1] == "flush") {
+    ProfilerFlush();
+    out << "cpu_profiler: flushed";
+  }
+  else {
+    out << "cpu_profiler: unrecognized command " << cmd
+	<< "; expected one of status, flush.";
+  }
+}
diff --git a/ceph/src/perfglue/cpu_profiler.h b/ceph/src/perfglue/cpu_profiler.h
new file mode 100644
index 00000000..5f892adf
--- /dev/null
+++ b/ceph/src/perfglue/cpu_profiler.h
@@ -0,0 +1,25 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_PERFGLUE_CPU_PROFILER
+
+/*
+ * Ceph glue for the Google Perftools CPU profiler
+ */
+#include 
+#include 
+
+void cpu_profiler_handle_command(const std::vector &cmd,
+				 ostream& out);
+
+#endif
diff --git a/ceph/src/perfglue/disabled_heap_profiler.cc b/ceph/src/perfglue/disabled_heap_profiler.cc
new file mode 100644
index 00000000..d2d4cb73
--- /dev/null
+++ b/ceph/src/perfglue/disabled_heap_profiler.cc
@@ -0,0 +1,33 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network/Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include "heap_profiler.h"
+
+bool ceph_using_tcmalloc() { return false; }
+
+void ceph_heap_profiler_init() { return; }
+
+void ceph_heap_profiler_stats(char *buf, int length) { return; }
+
+void ceph_heap_release_free_memory() { return; }
+
+bool ceph_heap_profiler_running() { return false; }
+
+void ceph_heap_profiler_start() { return; }
+
+void ceph_heap_profiler_stop() { return; }
+
+void ceph_heap_profiler_dump(const char *reason) { return; }
+
+void ceph_heap_profiler_handle_command(const std::vector& cmd,
+                                       ostream& out) { return; }
diff --git a/ceph/src/perfglue/disabled_stubs.cc b/ceph/src/perfglue/disabled_stubs.cc
new file mode 100644
index 00000000..dfbc0e66
--- /dev/null
+++ b/ceph/src/perfglue/disabled_stubs.cc
@@ -0,0 +1,25 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/LogClient.h"
+#include "perfglue/cpu_profiler.h"
+
+#include 
+#include 
+
+void cpu_profiler_handle_command(const std::vector &cmd,
+				 ostream& out)
+{
+  out << "cpu_profiler support not linked in";
+}
diff --git a/ceph/src/perfglue/heap_profiler.cc b/ceph/src/perfglue/heap_profiler.cc
new file mode 100644
index 00000000..6b079b86
--- /dev/null
+++ b/ceph/src/perfglue/heap_profiler.cc
@@ -0,0 +1,118 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network/Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include "heap_profiler.h"
+#include "common/environment.h"
+#include "common/LogClient.h"
+#include "global/global_context.h"
+#include "common/debug.h"
+
+bool ceph_using_tcmalloc()
+{
+  return true;
+}
+
+void ceph_heap_profiler_init()
+{
+  // Two other interesting environment variables to set are:
+  // HEAP_PROFILE_ALLOCATION_INTERVAL, HEAP_PROFILE_INUSE_INTERVAL
+  if (get_env_bool("CEPH_HEAP_PROFILER_INIT")) {
+    ceph_heap_profiler_start();
+  }
+}
+
+void ceph_heap_profiler_stats(char *buf, int length)
+{
+  MallocExtension::instance()->GetStats(buf, length);
+}
+
+void ceph_heap_release_free_memory()
+{
+  MallocExtension::instance()->ReleaseFreeMemory();
+}
+
+bool ceph_heap_profiler_running()
+{
+  return IsHeapProfilerRunning();
+}
+
+static void get_profile_name(char *profile_name, int profile_name_len)
+{
+  char path[PATH_MAX];
+  snprintf(path, sizeof(path), "%s", g_conf->log_file.c_str());
+  char *last_slash = rindex(path, '/');
+
+  if (last_slash == NULL) {
+    snprintf(profile_name, profile_name_len, "./%s.profile",
+	     g_conf->name.to_cstr());
+  }
+  else {
+    last_slash[1] = '\0';
+    snprintf(profile_name, profile_name_len, "%s/%s.profile",
+	     path, g_conf->name.to_cstr());
+  }
+}
+
+void ceph_heap_profiler_start()
+{
+  char profile_name[PATH_MAX];
+  get_profile_name(profile_name, sizeof(profile_name)); 
+  generic_dout(0) << "turning on heap profiler with prefix "
+		  << profile_name << dendl;
+  HeapProfilerStart(profile_name);
+}
+
+void ceph_heap_profiler_stop()
+{
+  HeapProfilerStop();
+}
+
+void ceph_heap_profiler_dump(const char *reason)
+{
+  HeapProfilerDump(reason);
+}
+
+void ceph_heap_profiler_handle_command(const std::vector& cmd,
+                                       ostream& out)
+{
+  if (cmd.size() == 1 && cmd[0] == "dump") {
+    if (!ceph_heap_profiler_running()) {
+      out << "heap profiler not running; can't dump";
+      return;
+    }
+    char *heap_stats = new char[1024];
+    ceph_heap_profiler_stats(heap_stats, 1024);
+    out << g_conf->name << "dumping heap profile now.\n"
+	<< heap_stats;
+    ceph_heap_profiler_dump("admin request");
+  } else if (cmd.size() == 1 && cmd[0] == "start_profiler") {
+    ceph_heap_profiler_start();
+    out << g_conf->name << " started profiler";
+  } else if (cmd.size() == 1 && cmd[0] == "stop_profiler") {
+    ceph_heap_profiler_stop();
+    out << g_conf->name << " stopped profiler";
+  } else if (cmd.size() == 1 && cmd[0] == "release") {
+    ceph_heap_release_free_memory();
+    out << g_conf->name << " releasing free RAM back to system.";
+  } else if (cmd.size() == 1 && cmd[0] == "stats") {
+    char *heap_stats = new char[1024];
+    ceph_heap_profiler_stats(heap_stats, 1024);
+    out << g_conf->name << "tcmalloc heap stats:"
+	<< heap_stats;
+  } else {
+    out << "unknown command " << cmd;
+  }
+}
diff --git a/ceph/src/perfglue/heap_profiler.h b/ceph/src/perfglue/heap_profiler.h
new file mode 100644
index 00000000..dac20d4f
--- /dev/null
+++ b/ceph/src/perfglue/heap_profiler.h
@@ -0,0 +1,49 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network/Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+#ifndef HEAP_PROFILER_H_
+#define HEAP_PROFILER_H_
+
+#include 
+#include 
+#include "common/config.h"
+
+class LogClient;
+
+/*
+ * Ceph glue for the Google perftools heap profiler, included
+ * as part of tcmalloc. This replaces ugly function pointers
+ * and #ifdef hacks!
+ */
+bool ceph_using_tcmalloc();
+
+/*
+ * Configure the heap profiler
+ */
+void ceph_heap_profiler_init();
+
+void ceph_heap_profiler_stats(char *buf, int length);
+
+void ceph_heap_release_free_memory();
+
+bool ceph_heap_profiler_running();
+
+void ceph_heap_profiler_start();
+
+void ceph_heap_profiler_stop();
+
+void ceph_heap_profiler_dump(const char *reason);
+
+void ceph_heap_profiler_handle_command(const std::vector &cmd,
+                                       ostream& out);
+
+#endif /* HEAP_PROFILER_H_ */
diff --git a/ceph/src/pybind/ceph_argparse.py b/ceph/src/pybind/ceph_argparse.py
new file mode 100644
index 00000000..8f2eb8f2
--- /dev/null
+++ b/ceph/src/pybind/ceph_argparse.py
@@ -0,0 +1,1118 @@
+"""
+Types and routines used by the ceph CLI as well as the RESTful
+interface.  These have to do with querying the daemons for
+command-description information, validating user command input against
+those descriptions, and submitting the command to the appropriate
+daemon.
+
+Copyright (C) 2013 Inktank Storage, Inc.
+
+LGPL2.  See file COPYING.
+"""
+import copy
+import json
+import os
+import pprint
+import re
+import socket
+import stat
+import sys
+import types
+import uuid
+
+class ArgumentError(Exception):
+    """
+    Something wrong with arguments
+    """
+    pass
+
+class ArgumentNumber(ArgumentError):
+    """
+    Wrong number of a repeated argument
+    """
+    pass
+
+class ArgumentFormat(ArgumentError):
+    """
+    Argument value has wrong format
+    """
+    pass
+
+class ArgumentValid(ArgumentError):
+    """
+    Argument value is otherwise invalid (doesn't match choices, for instance)
+    """
+    pass
+
+class ArgumentTooFew(ArgumentError):
+    """
+    Fewer arguments than descriptors in signature; may mean to continue
+    the search, so gets a special exception type
+    """
+
+class ArgumentPrefix(ArgumentError):
+    """
+    Special for mismatched prefix; less severe, don't report by default
+    """
+    pass
+
+class JsonFormat(Exception):
+    """
+    some syntactic or semantic issue with the JSON
+    """
+    pass
+
+class CephArgtype(object):
+    """
+    Base class for all Ceph argument types
+
+    Instantiating an object sets any validation parameters
+    (allowable strings, numeric ranges, etc.).  The 'valid'
+    method validates a string against that initialized instance,
+    throwing ArgumentError if there's a problem.
+    """
+    def __init__(self, **kwargs):
+        """
+        set any per-instance validation parameters here
+        from kwargs (fixed string sets, integer ranges, etc)
+        """
+        pass
+
+    def valid(self, s, partial=False):
+        """
+        Run validation against given string s (generally one word);
+        partial means to accept partial string matches (begins-with).
+        If cool, set self.val to the value that should be returned
+        (a copy of the input string, or a numeric or boolean interpretation
+        thereof, for example)
+        if not, throw ArgumentError(msg-as-to-why)
+        """
+        self.val = s
+
+    def __repr__(self):
+        """
+        return string representation of description of type.  Note,
+        this is not a representation of the actual value.  Subclasses
+        probably also override __str__() to give a more user-friendly
+        'name/type' description for use in command format help messages.
+        """
+        a = ''
+        if hasattr(self, 'typeargs'):
+            a = self.typeargs
+        return '{0}(\'{1}\')'.format(self.__class__.__name__, a)
+
+    def __str__(self):
+        """
+        where __repr__ (ideally) returns a string that could be used to
+        reproduce the object, __str__ returns one you'd like to see in
+        print messages.  Use __str__ to format the argtype descriptor
+        as it would be useful in a command usage message.
+        """
+        return '<{0}>'.format(self.__class__.__name__)
+
+class CephInt(CephArgtype):
+    """
+    range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+
+    range: list of 1 or 2 ints, [min] or [min,max]
+    """
+    def __init__(self, range=''):
+        if range == '':
+            self.range = list()
+        else:
+            self.range = list(range.split('|'))
+            self.range = map(long, self.range)
+
+    def valid(self, s, partial=False):
+        try:
+            val = long(s)
+        except ValueError:
+            raise ArgumentValid("{0} doesn't represent an int".format(s))
+        if len(self.range) == 2:
+            if val < self.range[0] or val > self.range[1]:
+                raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+        elif len(self.range) == 1:
+            if val < self.range[0]:
+                raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+        self.val = val
+
+    def __str__(self):
+        r = ''
+        if len(self.range) == 1:
+            r = '[{0}-]'.format(self.range[0])
+        if len(self.range) == 2:
+            r = '[{0}-{1}]'.format(self.range[0], self.range[1])
+
+        return ''.format(r)
+
+
+class CephFloat(CephArgtype):
+    """
+    range-limited float type
+    range: list of 1 or 2 floats, [min] or [min, max]
+    """
+    def __init__(self, range=''):
+        if range == '':
+            self.range = list()
+        else:
+            self.range = list(range.split('|'))
+            self.range = map(float, self.range)
+
+    def valid(self, s, partial=False):
+        try:
+            val = float(s)
+        except ValueError:
+            raise ArgumentValid("{0} doesn't represent a float".format(s))
+        if len(self.range) == 2:
+            if val < self.range[0] or val > self.range[1]:
+                raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+        elif len(self.range) == 1:
+            if val < self.range[0]:
+                raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+        self.val = val
+
+    def __str__(self):
+        r = ''
+        if len(self.range) == 1:
+            r = '[{0}-]'.format(self.range[0])
+        if len(self.range) == 2:
+            r = '[{0}-{1}]'.format(self.range[0], self.range[1])
+        return ''.format(r)
+
+class CephString(CephArgtype):
+    """
+    String; pretty generic.  goodchars is a RE char class of valid chars
+    """
+    def __init__(self, goodchars=''):
+        from string import printable
+        try:
+            re.compile(goodchars)
+        except:
+            raise ValueError('CephString(): "{0}" is not a valid RE'.\
+                format(goodchars))
+        self.goodchars = goodchars
+        self.goodset = frozenset(
+            [c for c in printable if re.match(goodchars, c)]
+        )
+
+    def valid(self, s, partial=False):
+        sset = set(s)
+        if self.goodset and not sset <= self.goodset:
+            raise ArgumentFormat("invalid chars {0} in {1}".\
+                format(''.join(sset - self.goodset), s))
+        self.val = s
+
+    def __str__(self):
+        b = ''
+        if self.goodchars:
+            b += '(goodchars {0})'.format(self.goodchars)
+        return ''.format(b)
+
+class CephSocketpath(CephArgtype):
+    """
+    Admin socket path; check that it's readable and S_ISSOCK
+    """
+    def valid(self, s, partial=False):
+        mode = os.stat(s).st_mode
+        if not stat.S_ISSOCK(mode):
+            raise ArgumentValid('socket path {0} is not a socket'.format(s))
+        self.val = s
+
+    def __str__(self):
+        return ''
+
+class CephIPAddr(CephArgtype):
+    """
+    IP address (v4 or v6) with optional port
+    """
+    def valid(self, s, partial=False):
+        # parse off port, use socket to validate addr
+        type = 6
+        if s.startswith('['):
+            type = 6
+        elif s.find('.') != -1:
+            type = 4
+        if type == 4:
+            port = s.find(':')
+            if (port != -1):
+                a = s[:port]
+                p = s[port+1:]
+                if int(p) > 65535:
+                    raise ArgumentValid('{0}: invalid IPv4 port'.format(p))
+            else:
+                a = s
+                p = None
+            try:
+                socket.inet_pton(socket.AF_INET, a)
+            except:
+                raise ArgumentValid('{0}: invalid IPv4 address'.format(a))
+        else:
+            # v6
+            if s.startswith('['):
+                end = s.find(']')
+                if end == -1:
+                    raise ArgumentFormat('{0} missing terminating ]'.format(s))
+                if s[end+1] == ':':
+                    try:
+                        p = int(s[end+2])
+                    except:
+                        raise ArgumentValid('{0}: bad port number'.format(s))
+                a = s[1:end]
+            else:
+                a = s
+                p = None
+            try:
+                socket.inet_pton(socket.AF_INET6, a)
+            except:
+                raise ArgumentValid('{0} not valid IPv6 address'.format(s))
+        if p is not None and long(p) > 65535:
+            raise ArgumentValid("{0} not a valid port number".format(p))
+        self.val = s
+        self.addr = a
+        self.port = p
+
+    def __str__(self):
+        return ''
+
+class CephEntityAddr(CephIPAddr):
+    """
+    EntityAddress, that is, IP address[/nonce]
+    """
+    def valid(self, s, partial=False):
+        nonce = None
+        if '/' in s:
+            ip, nonce = s.split('/')
+        else:
+            ip = s
+        super(self.__class__, self).valid(ip)
+        if nonce:
+            nonce_long = None
+            try:
+                nonce_long = long(nonce)
+            except ValueError:
+                pass
+            if nonce_long is None or nonce_long < 0:
+                raise ArgumentValid(
+                    '{0}: invalid entity, nonce {1} not integer > 0'.\
+                    format(s, nonce)
+                )
+        self.val = s
+
+    def __str__(self):
+        return ''
+
+class CephPoolname(CephArgtype):
+    """
+    Pool name; very little utility
+    """
+    def __str__(self):
+        return ''
+
+class CephObjectname(CephArgtype):
+    """
+    Object name.  Maybe should be combined with Pool name as they're always
+    present in pairs, and then could be checked for presence
+    """
+    def __str__(self):
+        return ''
+
+class CephPgid(CephArgtype):
+    """
+    pgid, in form N.xxx (N = pool number, xxx = hex pgnum)
+    """
+    def valid(self, s, partial=False):
+        if s.find('.') == -1:
+            raise ArgumentFormat('pgid has no .')
+        poolid, pgnum = s.split('.')
+        if poolid < 0:
+            raise ArgumentFormat('pool {0} < 0'.format(poolid))
+        try:
+            pgnum = int(pgnum, 16)
+        except:
+            raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum))
+        self.val = s
+
+    def __str__(self):
+        return ''
+
+class CephName(CephArgtype):
+    """
+    Name (type.id) where:
+    type is osd|mon|client|mds
+    id is a base10 int, if type == osd, or a string otherwise
+
+    Also accept '*'
+    """
+    def __init__(self):
+        self.nametype = None
+        self.nameid = None
+
+    def valid(self, s, partial=False):
+        if s == '*':
+            self.val = s
+            return
+        if s.find('.') == -1:
+            raise ArgumentFormat('CephName: no . in {0}'.format(s))
+        else:
+            t, i = s.split('.')
+            if not t in ('osd', 'mon', 'client', 'mds'):
+                raise ArgumentValid('unknown type ' + t)
+            if t == 'osd':
+                if i != '*':
+                    try:
+                        i = int(i)
+                    except:
+                        raise ArgumentFormat('osd id ' + i + ' not integer')
+            self.nametype = t
+        self.val = s
+        self.nameid = i
+
+    def __str__(self):
+        return ''
+
+class CephOsdName(CephArgtype):
+    """
+    Like CephName, but specific to osds: allow  alone
+
+    osd., or , or *, where id is a base10 int
+    """
+    def __init__(self):
+        self.nametype = None
+        self.nameid = None
+
+    def valid(self, s, partial=False):
+        if s == '*':
+            self.val = s
+            return
+        if s.find('.') != -1:
+            t, i = s.split('.')
+            if t != 'osd':
+                raise ArgumentValid('unknown type ' + t)
+        else:
+            t = 'osd'
+            i = s
+        try:
+            i = int(i)
+        except:
+            raise ArgumentFormat('osd id ' + i + ' not integer')
+        self.nametype = t
+        self.nameid = i
+        self.val = i
+
+    def __str__(self):
+        return ''
+
+class CephChoices(CephArgtype):
+    """
+    Set of string literals; init with valid choices
+    """
+    def __init__(self, strings='', **kwargs):
+        self.strings = strings.split('|')
+
+    def valid(self, s, partial=False):
+        if not partial:
+            if not s in self.strings:
+                # show as __str__ does: {s1|s2..}
+                raise ArgumentValid("{0} not in {1}".format(s, self))
+            self.val = s
+            return
+
+        # partial
+        for t in self.strings:
+            if t.startswith(s):
+                self.val = s
+                return
+        raise ArgumentValid("{0} not in {1}".  format(s, self))
+
+    def __str__(self):
+        if len(self.strings) == 1:
+            return '{0}'.format(self.strings[0])
+        else:
+            return '{0}'.format('|'.join(self.strings))
+
+class CephFilepath(CephArgtype):
+    """
+    Openable file
+    """
+    def valid(self, s, partial=False):
+        try:
+            f = open(s, 'a+')
+        except Exception as e:
+            raise ArgumentValid('can\'t open {0}: {1}'.format(s, e))
+        f.close()
+        self.val = s
+
+    def __str__(self):
+        return ''
+
+class CephFragment(CephArgtype):
+    """
+    'Fragment' ??? XXX
+    """
+    def valid(self, s, partial=False):
+        if s.find('/') == -1:
+            raise ArgumentFormat('{0}: no /'.format(s))
+        val, bits = s.split('/')
+        # XXX is this right?
+        if not val.startswith('0x'):
+            raise ArgumentFormat("{0} not a hex integer".format(val))
+        try:
+            long(val)
+        except:
+            raise ArgumentFormat('can\'t convert {0} to integer'.format(val))
+        try:
+            long(bits)
+        except:
+            raise ArgumentFormat('can\'t convert {0} to integer'.format(bits))
+        self.val = s
+
+    def __str__(self):
+        return ""
+
+
+class CephUUID(CephArgtype):
+    """
+    CephUUID: pretty self-explanatory
+    """
+    def valid(self, s, partial=False):
+        try:
+            uuid.UUID(s)
+        except Exception as e:
+            raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e))
+        self.val = s
+
+    def __str__(self):
+        return ''
+
+
+class CephPrefix(CephArgtype):
+    """
+    CephPrefix: magic type for "all the first n fixed strings"
+    """
+    def __init__(self, prefix=''):
+        self.prefix = prefix
+
+    def valid(self, s, partial=False):
+        if partial:
+            if self.prefix.startswith(s):
+                self.val = s
+                return
+        else:
+            if (s == self.prefix):
+                self.val = s
+                return
+
+        raise ArgumentPrefix("no match for {0}".format(s))
+
+    def __str__(self):
+        return self.prefix
+
+
+class argdesc(object):
+    """
+    argdesc(typename, name='name', n=numallowed|N,
+            req=False, helptext=helptext, **kwargs (type-specific))
+
+    validation rules:
+    typename: type(**kwargs) will be constructed
+    later, type.valid(w) will be called with a word in that position
+
+    name is used for parse errors and for constructing JSON output
+    n is a numeric literal or 'n|N', meaning "at least one, but maybe more"
+    req=False means the argument need not be present in the list
+    helptext is the associated help for the command
+    anything else are arguments to pass to the type constructor.
+
+    self.instance is an instance of type t constructed with typeargs.
+
+    valid() will later be called with input to validate against it,
+    and will store the validated value in self.instance.val for extraction.
+    """
+    def __init__(self, t, name=None, n=1, req=True, **kwargs):
+        if isinstance(t, types.StringTypes):
+            self.t = CephPrefix
+            self.typeargs = {'prefix':t}
+            self.req = True
+        else:
+            self.t = t
+            self.typeargs = kwargs
+            self.req = bool(req == True or req == 'True')
+
+        self.name = name
+        self.N = (n in ['n', 'N'])
+        if self.N:
+            self.n = 1
+        else:
+            self.n = int(n)
+        self.instance = self.t(**self.typeargs)
+
+    def __repr__(self):
+        r = 'argdesc(' + str(self.t) + ', '
+        internals = ['N', 'typeargs', 'instance', 't']
+        for (k, v) in self.__dict__.iteritems():
+            if k.startswith('__') or k in internals:
+                pass
+            else:
+                # undo modification from __init__
+                if k == 'n' and self.N:
+                    v = 'N'
+                r += '{0}={1}, '.format(k, v)
+        for (k, v) in self.typeargs.iteritems():
+            r += '{0}={1}, '.format(k, v)
+        return r[:-2] + ')'
+
+    def __str__(self):
+        if ((self.t == CephChoices and len(self.instance.strings) == 1)
+            or (self.t == CephPrefix)):
+            s = str(self.instance)
+        else:
+            s = '{0}({1})'.format(self.name, str(self.instance))
+            if self.N:
+                s += ' [' + str(self.instance) + '...]'
+        if not self.req:
+            s = '{' + s + '}'
+        return s
+
+    def helpstr(self):
+        """
+        like str(), but omit parameter names (except for CephString,
+        which really needs them)
+        """
+        if self.t == CephString:
+            chunk = '<{0}>'.format(self.name)
+        else:
+            chunk = str(self.instance)
+        s = chunk
+        if self.N:
+            s += ' [' + chunk + '...]'
+        if not self.req:
+            s = '{' + s + '}'
+        return s
+
+def concise_sig(sig):
+    """
+    Return string representation of sig useful for syntax reference in help
+    """
+    return ' '.join([d.helpstr() for d in sig])
+
+def descsort(sh1, sh2):
+    """
+    sort descriptors by prefixes, defined as the concatenation of all simple
+    strings in the descriptor; this works out to just the leading strings.
+    """
+    return cmp(concise_sig(sh1['sig']), concise_sig(sh2['sig']))
+
+def parse_funcsig(sig):
+    """
+    parse a single descriptor (array of strings or dicts) into a
+    dict of function descriptor/validators (objects of CephXXX type)
+    """
+    newsig = []
+    argnum = 0
+    for desc in sig:
+        argnum += 1
+        if isinstance(desc, types.StringTypes):
+            t = CephPrefix
+            desc = {'type':t, 'name':'prefix', 'prefix':desc}
+        else:
+            # not a simple string, must be dict
+            if not 'type' in desc:
+                s = 'JSON descriptor {0} has no type'.format(sig)
+                raise JsonFormat(s)
+            # look up type string in our globals() dict; if it's an
+            # object of type types.TypeType, it must be a
+            # locally-defined class. otherwise, we haven't a clue.
+            if desc['type'] in globals():
+                t = globals()[desc['type']]
+                if type(t) != types.TypeType:
+                    s = 'unknown type {0}'.format(desc['type'])
+                    raise JsonFormat(s)
+            else:
+                s = 'unknown type {0}'.format(desc['type'])
+                raise JsonFormat(s)
+
+        kwargs = dict()
+        for key, val in desc.items():
+            if key not in ['type', 'name', 'n', 'req']:
+                kwargs[key] = val
+        newsig.append(argdesc(t,
+                              name=desc.get('name', None),
+                              n=desc.get('n', 1),
+                              req=desc.get('req', True),
+                              **kwargs))
+    return newsig
+
+
+def parse_json_funcsigs(s, consumer):
+    """
+    A function signature is mostly an array of argdesc; it's represented
+    in JSON as
+    {
+      "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false ], "help":helptext, "module":modulename, "perm":perms, "avail":availability}
+       .
+       .
+       .
+      ]
+
+    A set of sigs is in an dict mapped by a unique number:
+    {
+      "cmd1": {
+         "sig": ["type.. ], "help":helptext...
+      }
+      "cmd2"{
+         "sig": [.. ], "help":helptext...
+      }
+    }
+
+    Parse the string s and return a dict of dicts, keyed by opcode;
+    each dict contains 'sig' with the array of descriptors, and 'help'
+    with the helptext, 'module' with the module name, 'perm' with a
+    string representing required permissions in that module to execute
+    this command (and also whether it is a read or write command from
+    the cluster state perspective), and 'avail' as a hint for
+    whether the command should be advertised by CLI, REST, or both.
+    If avail does not contain 'consumer', don't include the command
+    in the returned dict.
+    """
+    try:
+        overall = json.loads(s)
+    except Exception as e:
+        print >> sys.stderr, "Couldn't parse JSON {0}: {1}".format(s, e)
+        raise e
+    sigdict = {}
+    for cmdtag, cmd in overall.iteritems():
+        if not 'sig' in cmd:
+            s = "JSON descriptor {0} has no 'sig'".format(cmdtag)
+            raise JsonFormat(s)
+        # check 'avail' and possibly ignore this command
+        if 'avail' in cmd:
+            if not consumer in cmd['avail']:
+                continue
+        # rewrite the 'sig' item with the argdesc-ized version, and...
+        cmd['sig'] = parse_funcsig(cmd['sig'])
+        # just take everything else as given
+        sigdict[cmdtag] = cmd
+    return sigdict
+
+def validate_one(word, desc, partial=False):
+    """
+    validate_one(word, desc, partial=False)
+
+    validate word against the constructed instance of the type
+    in desc.  May raise exception.  If it returns false (and doesn't
+    raise an exception), desc.instance.val will
+    contain the validated value (in the appropriate type).
+    """
+    desc.instance.valid(word, partial)
+    desc.numseen += 1
+    if desc.N:
+        desc.n = desc.numseen + 1
+
+def matchnum(args, signature, partial=False):
+    """
+    matchnum(s, signature, partial=False)
+
+    Returns number of arguments matched in s against signature.
+    Can be used to determine most-likely command for full or partial
+    matches (partial applies to string matches).
+    """
+    words = args[:]
+    mysig = copy.deepcopy(signature)
+    matchcnt = 0
+    for desc in mysig:
+        setattr(desc, 'numseen', 0)
+        while desc.numseen < desc.n:
+            # if there are no more arguments, return
+            if not words:
+                return matchcnt
+            word = words.pop(0)
+
+            try:
+                validate_one(word, desc, partial)
+                valid = True
+            except ArgumentError:
+                # matchnum doesn't care about type of error
+                valid = False
+
+            if not valid:
+                if not desc.req:
+                    # this wasn't required, so word may match the next desc
+                    words.insert(0, word)
+                    break
+                else:
+                    # it was required, and didn't match, return
+                    return matchcnt
+        if desc.req:
+            matchcnt += 1
+    return matchcnt
+
+def get_next_arg(desc, args):
+    '''
+    Get either the value matching key 'desc.name' or the next arg in
+    the non-dict list.  Return None if args are exhausted.  Used in
+    validate() below.
+    '''
+    arg = None
+    if isinstance(args, dict):
+        arg = args.pop(desc.name, None)
+        # allow 'param=param' to be expressed as 'param'
+        if arg == '':
+            arg = desc.name
+        # Hack, or clever?  If value is a list, keep the first element,
+        # push rest back onto myargs for later processing.
+        # Could process list directly, but nesting here is already bad
+        if arg and isinstance(arg, list):
+            args[desc.name] = arg[1:]
+            arg = arg[0]
+    elif args:
+        arg = args.pop(0)
+        if arg and isinstance(arg, list):
+            args = arg[1:] + args
+            arg = arg[0]
+    return arg
+
+def store_arg(desc, d):
+    '''
+    Store argument described by, and held in, thanks to valid(),
+    desc into the dictionary d, keyed by desc.name.  Three cases:
+
+    1) desc.N is set: value in d is a list
+    2) prefix: multiple args are joined with ' ' into one d{} item
+    3) single prefix or other arg: store as simple value
+
+    Used in validate() below.
+    '''
+    if desc.N:
+        # value should be a list
+        if desc.name in d:
+            d[desc.name] += [desc.instance.val]
+        else:
+            d[desc.name] = [desc.instance.val]
+    elif (desc.t == CephPrefix) and (desc.name in d):
+        # prefixes' values should be a space-joined concatenation
+        d[desc.name] += ' ' + desc.instance.val
+    else:
+        # if first CephPrefix or any other type, just set it
+        d[desc.name] = desc.instance.val
+
+def validate(args, signature, partial=False):
+    """
+    validate(args, signature, partial=False)
+
+    args is a list of either words or k,v pairs representing a possible
+    command input following format of signature.  Runs a validation; no
+    exception means it's OK.  Return a dict containing all arguments keyed
+    by their descriptor name, with duplicate args per name accumulated
+    into a list (or space-separated value for CephPrefix).
+
+    Mismatches of prefix are non-fatal, as this probably just means the
+    search hasn't hit the correct command.  Mismatches of non-prefix
+    arguments are treated as fatal, and an exception raised.
+
+    This matching is modified if partial is set: allow partial matching
+    (with partial dict returned); in this case, there are no exceptions
+    raised.
+    """
+
+    myargs = copy.deepcopy(args)
+    mysig = copy.deepcopy(signature)
+    reqsiglen = len([desc for desc in mysig if desc.req])
+    matchcnt = 0
+    d = dict()
+    for desc in mysig:
+        setattr(desc, 'numseen', 0)
+        while desc.numseen < desc.n:
+            myarg = get_next_arg(desc, myargs)
+
+            # no arg, but not required?  Continue consuming mysig
+            # in case there are later required args
+            if not myarg and not desc.req:
+                break
+
+            # out of arguments for a required param?
+            # Either return (if partial validation) or raise 
+            if not myarg and desc.req:
+                if desc.N and desc.numseen < 1:
+                    # wanted N, didn't even get 1
+                    if partial:
+                        return d
+                    raise ArgumentNumber(
+                        'saw {0} of {1}, expected at least 1'.\
+                         format(desc.numseen, desc)
+                    )
+                elif not desc.N and desc.numseen < desc.n:
+                    # wanted n, got too few
+                    if partial:
+                        return d
+                    # special-case the "0 expected 1" case
+                    if desc.numseen == 0 and desc.n == 1:
+                        raise ArgumentNumber(
+                            'missing required parameter {0}'.format(desc)
+                        )
+                    raise ArgumentNumber(
+                        'saw {0} of {1}, expected {2}'.\
+                        format(desc.numseen, desc, desc.n)
+                    )
+                break
+
+            # Have an arg; validate it
+            try:
+                validate_one(myarg, desc)
+                valid = True
+            except ArgumentError as e:
+                valid = False
+            if not valid:
+                # argument mismatch
+                if not desc.req:
+                    # if not required, just push back; it might match
+                    # the next arg
+                    print >> sys.stderr, myarg, 'not valid: ', str(e)
+                    myargs.insert(0, myarg)
+                    break
+                else:
+                    # hm, it was required, so time to return/raise
+                    if partial:
+                        return d
+                    raise e
+
+            # Whew, valid arg acquired.  Store in dict
+            matchcnt += 1
+            store_arg(desc, d)
+
+    # Done with entire list of argdescs
+    if matchcnt < reqsiglen:
+        raise ArgumentTooFew("not enough arguments given")
+
+    if myargs and not partial:
+        raise ArgumentError("unused arguments: " + str(myargs))
+
+    # Finally, success
+    return d
+
+def cmdsiglen(sig):
+    sigdict = sig.values()
+    assert len(sigdict) == 1
+    return len(sig.values()[0]['sig'])
+
+def validate_command(sigdict, args, verbose=False):
+    """
+    turn args into a valid dictionary ready to be sent off as JSON,
+    validated against sigdict.
+    """
+    found = []
+    valid_dict = {}
+    if args:
+        # look for best match, accumulate possibles in bestcmds
+        # (so we can maybe give a more-useful error message)
+        best_match_cnt = 0
+        bestcmds = []
+        for cmdtag, cmd in sigdict.iteritems():
+            sig = cmd['sig']
+            matched = matchnum(args, sig, partial=True)
+            if (matched > best_match_cnt):
+                if verbose:
+                    print >> sys.stderr, \
+                        "better match: {0} > {1}: {2}:{3} ".format(matched,
+                                      best_match_cnt, cmdtag, concise_sig(sig))
+                best_match_cnt = matched
+                bestcmds = [{cmdtag:cmd}]
+            elif matched == best_match_cnt:
+                if verbose:
+                    print >> sys.stderr, \
+                        "equal match: {0} > {1}: {2}:{3} ".format(matched,
+                                      best_match_cnt, cmdtag, concise_sig(sig))
+                bestcmds.append({cmdtag:cmd})
+
+        # Sort bestcmds by number of args so we can try shortest first
+        # (relies on a cmdsig being key,val where val is a list of len 1)
+        bestcmds_sorted = sorted(bestcmds,
+                                 cmp=lambda x,y:cmp(cmdsiglen(x), cmdsiglen(y)))
+
+        if verbose:
+            print >> sys.stderr, "bestcmds_sorted: "
+            pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted)
+
+        # for everything in bestcmds, look for a true match
+        for cmdsig in bestcmds_sorted:
+            for cmd in cmdsig.itervalues():
+                sig = cmd['sig']
+                try:
+                    valid_dict = validate(args, sig)
+                    found = cmd
+                    break
+                except ArgumentPrefix:
+                    # ignore prefix mismatches; we just haven't found
+                    # the right command yet
+                    pass
+                except ArgumentTooFew:
+                    # It looked like this matched the beginning, but it
+                    # didn't have enough args supplied.  If we're out of
+                    # cmdsigs we'll fall out unfound; if we're not, maybe
+                    # the next one matches completely.  Whine, but pass.
+                    if verbose:
+                        print >> sys.stderr, 'Not enough args supplied for ', \
+                                              concise_sig(sig)
+                except ArgumentError as e:
+                    # Solid mismatch on an arg (type, range, etc.)
+                    # Stop now, because we have the right command but
+                    # some other input is invalid
+                    print >> sys.stderr, "Invalid command: ", str(e)
+                    print >> sys.stderr, concise_sig(sig), ': ', cmd['help']
+                    return {}
+            if found:
+                break
+
+        if not found:
+            print >> sys.stderr, 'no valid command found; 10 closest matches:'
+            for cmdsig in bestcmds[:10]:
+                for (cmdtag, cmd) in cmdsig.iteritems():
+                    print >> sys.stderr, concise_sig(cmd['sig'])
+            return None
+
+        return valid_dict
+
+def find_cmd_target(childargs):
+    """
+    Using a minimal validation, figure out whether the command
+    should be sent to a monitor or an osd.  We do this before even
+    asking for the 'real' set of command signatures, so we can ask the
+    right daemon.
+    Returns ('osd', osdid), ('pg', pgid), or ('mon', '')
+    """
+    sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}])
+    try:
+        valid_dict = validate(childargs, sig, partial=True)
+    except ArgumentError:
+        pass
+    else:
+        if len(valid_dict) == 2:
+            # revalidate to isolate type and id
+            name = CephName()
+            # if this fails, something is horribly wrong, as it just
+            # validated successfully above
+            name.valid(valid_dict['target'])
+            return name.nametype, name.nameid
+
+    sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}])
+    try:
+        valid_dict = validate(childargs, sig, partial=True)
+    except ArgumentError:
+        pass
+    else:
+        if len(valid_dict) == 2:
+            # pg doesn't need revalidation; the string is fine
+            return 'pg', valid_dict['pgid']
+
+    sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}])
+    try:
+        valid_dict = validate(childargs, sig, partial=True)
+    except ArgumentError:
+        pass
+    else:
+        if len(valid_dict) == 2:
+            return 'pg', valid_dict['pgid']
+
+    return 'mon', ''
+
+def send_command(cluster, target=('mon', ''), cmd=None, inbuf='', timeout=0, 
+                 verbose=False):
+    """
+    Send a command to a daemon using librados's
+    mon_command, osd_command, or pg_command.  Any bulk input data
+    comes in inbuf.
+
+    Returns (ret, outbuf, outs); ret is the return code, outbuf is
+    the outbl "bulk useful output" buffer, and outs is any status
+    or error message (intended for stderr).
+
+    If target is osd.N, send command to that osd (except for pgid cmds)
+    """
+    cmd = cmd or []
+    try:
+        if target[0] == 'osd':
+            osdid = target[1]
+
+            if verbose:
+                print >> sys.stderr, 'submit {0} to osd.{1}'.\
+                    format(cmd, osdid)
+            ret, outbuf, outs = \
+                cluster.osd_command(osdid, cmd, inbuf, timeout)
+
+        elif target[0] == 'pg':
+            pgid = target[1]
+            # pgid will already be in the command for the pg 
+            # form, but for tell , we need to put it in
+            if cmd:
+                cmddict = json.loads(cmd[0])
+                cmddict['pgid'] = pgid
+            else:
+                cmddict = dict(pgid=pgid)
+            cmd = [json.dumps(cmddict)]
+            if verbose:
+                print >> sys.stderr, 'submit {0} for pgid {1}'.\
+                    format(cmd, pgid)
+            ret, outbuf, outs = \
+                cluster.pg_command(pgid, cmd, inbuf, timeout)
+
+        elif target[0] == 'mon':
+            if verbose:
+                print >> sys.stderr, '{0} to {1}'.\
+                    format(cmd, target[0])
+            if target[1] == '':
+                ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout)
+            else:
+                ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout, target[1])
+        else:
+            raise ArgumentValid("Bad target type '{0}'".format(target[0]))
+
+    except Exception as e:
+        if not isinstance(e, ArgumentError):
+            raise RuntimeError('"{0}": exception {1}'.format(cmd, e))
+        else:
+            raise
+
+    return ret, outbuf, outs
+
+def json_command(cluster, target=('mon', ''), prefix=None, argdict=None,
+                 inbuf='', timeout=0, verbose=False):
+    """
+    Format up a JSON command and send it with send_command() above.
+    Prefix may be supplied separately or in argdict.  Any bulk input
+    data comes in inbuf.
+
+    If target is osd.N, send command to that osd (except for pgid cmds)
+    """
+    cmddict = {}
+    if prefix:
+        cmddict.update({'prefix':prefix})
+    if argdict:
+        cmddict.update(argdict)
+
+    # grab prefix for error messages
+    prefix = cmddict['prefix']
+
+    try:
+        if target[0] == 'osd':
+            osdtarg = CephName()
+            osdtarget = '{0}.{1}'.format(*target)
+            # prefer target from cmddict if present and valid
+            if 'target' in cmddict:
+                osdtarget = cmddict.pop('target')
+            try:
+                osdtarg.valid(osdtarget)
+                target = ('osd', osdtarg.nameid)
+            except:
+                # use the target we were originally given
+                pass
+
+        ret, outbuf, outs = send_command(cluster, target, [json.dumps(cmddict)],
+                                         inbuf, timeout, verbose)
+
+    except Exception as e:
+        if not isinstance(e, ArgumentError):
+            raise RuntimeError('"{0}": exception {1}'.format(cmd, e))
+        else:
+            raise
+
+    return ret, outbuf, outs
+
+
diff --git a/ceph/src/pybind/ceph_rest_api.py b/ceph/src/pybind/ceph_rest_api.py
new file mode 100755
index 00000000..d940db6d
--- /dev/null
+++ b/ceph/src/pybind/ceph_rest_api.py
@@ -0,0 +1,499 @@
+# vim: ts=4 sw=4 smarttab expandtab
+
+import errno
+import json
+import logging
+import logging.handlers
+import os
+import rados
+import textwrap
+import xml.etree.ElementTree
+import xml.sax.saxutils
+
+import flask
+from ceph_argparse import \
+    ArgumentError, CephPgid, CephOsdName, CephChoices, CephPrefix, \
+    concise_sig, descsort, parse_funcsig, parse_json_funcsigs, \
+    validate, json_command
+
+#
+# Globals and defaults
+#
+
+DEFAULT_ADDR = '0.0.0.0'
+DEFAULT_PORT = '5000'
+DEFAULT_ID = 'restapi'
+
+DEFAULT_BASEURL = '/api/v0.1'
+DEFAULT_LOG_LEVEL = 'warning'
+DEFAULT_LOGDIR = '/var/log/ceph'
+# default client name will be 'client.'
+
+# 'app' must be global for decorators, etc.
+APPNAME = '__main__'
+app = flask.Flask(APPNAME)
+
+LOGLEVELS = {
+    'critical':logging.CRITICAL,
+    'error':logging.ERROR,
+    'warning':logging.WARNING,
+    'info':logging.INFO,
+    'debug':logging.DEBUG,
+}
+
+def find_up_osd(app):
+    '''
+    Find an up OSD.  Return the last one that's up.
+    Returns id as an int.
+    '''
+    ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump",
+                                     argdict=dict(format='json'))
+    if ret:
+        raise EnvironmentError(ret, 'Can\'t get osd dump output')
+    try:
+        osddump = json.loads(outbuf)
+    except:
+        raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump')
+    osds = [osd['osd'] for osd in osddump['osds'] if osd['up']]
+    if not osds:
+        return None
+    return int(osds[-1])
+
+
+METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']}
+
+def api_setup(app, conf, cluster, clientname, clientid, args):
+    '''
+    This is done globally, and cluster connection kept open for
+    the lifetime of the daemon.  librados should assure that even
+    if the cluster goes away and comes back, our connection remains.
+
+    Initialize the running instance.  Open the cluster, get the command
+    signatures, module, perms, and help; stuff them away in the app.ceph_urls
+    dict.  Also save app.ceph_sigdict for help() handling.
+    '''
+    def get_command_descriptions(cluster, target=('mon','')):
+        ret, outbuf, outs = json_command(cluster, target,
+                                         prefix='get_command_descriptions',
+                                         timeout=30)
+        if ret:
+            err = "Can't get command descriptions: {0}".format(outs)
+            app.logger.error(err)
+            raise EnvironmentError(ret, err)
+
+        try:
+            sigdict = parse_json_funcsigs(outbuf, 'rest')
+        except Exception as e:
+            err = "Can't parse command descriptions: {}".format(e)
+            app.logger.error(err)
+            raise EnvironmentError(err)
+        return sigdict
+
+    app.ceph_cluster = cluster or 'ceph'
+    app.ceph_urls = {}
+    app.ceph_sigdict = {}
+    app.ceph_baseurl = ''
+
+    conf = conf or ''
+    cluster = cluster or 'ceph'
+    clientid = clientid or DEFAULT_ID
+    clientname = clientname or 'client.' + clientid
+
+    app.ceph_cluster = rados.Rados(name=clientname, conffile=conf)
+    app.ceph_cluster.conf_parse_argv(args)
+    app.ceph_cluster.connect()
+
+    app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \
+         or DEFAULT_BASEURL
+    if app.ceph_baseurl.endswith('/'):
+        app.ceph_baseurl = app.ceph_baseurl[:-1]
+    addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR
+
+    # remove any nonce from the conf value
+    addr = addr.split('/')[0]
+    addr, port = addr.rsplit(':', 1)
+    addr = addr or DEFAULT_ADDR
+    port = port or DEFAULT_PORT
+    port = int(port)
+
+    loglevel = app.ceph_cluster.conf_get('restapi_log_level') \
+        or DEFAULT_LOG_LEVEL
+    # ceph has a default log file for daemons only; clients (like this)
+    # default to "".  Override that for this particular client.
+    logfile = app.ceph_cluster.conf_get('log_file')
+    if not logfile:
+        logfile = os.path.join(
+            DEFAULT_LOGDIR,
+            '{cluster}-{clientname}.{pid}.log'.format(
+                cluster=cluster,
+                clientname=clientname,
+                pid=os.getpid()
+            )
+        )
+    app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile))
+    app.logger.setLevel(LOGLEVELS[loglevel.lower()])
+    for h in app.logger.handlers:
+        h.setFormatter(logging.Formatter(
+            '%(asctime)s %(name)s %(levelname)s: %(message)s'))
+
+    app.ceph_sigdict = get_command_descriptions(app.ceph_cluster)
+
+    osdid = find_up_osd(app)
+    if osdid is not None:
+        osd_sigdict = get_command_descriptions(app.ceph_cluster,
+                                               target=('osd', int(osdid)))
+
+        # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict
+        maxkey = sorted(app.ceph_sigdict.keys())[-1]
+        maxkey = int(maxkey.replace('cmd', ''))
+        osdkey = maxkey + 1
+        for k, v in osd_sigdict.iteritems():
+            newv = v
+            newv['flavor'] = 'tell'
+            globk = 'cmd' + str(osdkey)
+            app.ceph_sigdict[globk] = newv
+            osdkey += 1
+
+    # app.ceph_sigdict maps "cmdNNN" to a dict containing:
+    # 'sig', an array of argdescs
+    # 'help', the helptext
+    # 'module', the Ceph module this command relates to
+    # 'perm', a 'rwx*' string representing required permissions, and also
+    #    a hint as to whether this is a GET or POST/PUT operation
+    # 'avail', a comma-separated list of strings of consumers that should
+    #    display this command (filtered by parse_json_funcsigs() above)
+    app.ceph_urls = {}
+    for cmdnum, cmddict in app.ceph_sigdict.iteritems():
+        cmdsig = cmddict['sig']
+        flavor = cmddict.get('flavor', 'mon')
+        url, params = generate_url_and_params(app, cmdsig, flavor)
+        perm = cmddict['perm']
+        for k in METHOD_DICT.iterkeys():
+            if k in perm:
+                methods = METHOD_DICT[k]
+        urldict = {'paramsig':params,
+                   'help':cmddict['help'],
+                   'module':cmddict['module'],
+                   'perm':perm,
+                   'flavor':flavor,
+                   'methods':methods,
+                  }
+
+        # app.ceph_urls contains a list of urldicts (usually only one long)
+        if url not in app.ceph_urls:
+            app.ceph_urls[url] = [urldict]
+        else:
+            # If more than one, need to make union of methods of all.
+            # Method must be checked in handler
+            methodset = set(methods)
+            for old_urldict in app.ceph_urls[url]:
+                methodset |= set(old_urldict['methods'])
+            methods = list(methodset)
+            app.ceph_urls[url].append(urldict)
+
+        # add, or re-add, rule with all methods and urldicts
+        app.add_url_rule(url, url, handler, methods=methods)
+        url += '.'
+        app.add_url_rule(url, url, handler, methods=methods)
+
+    app.logger.debug("urls added: %d", len(app.ceph_urls))
+
+    app.add_url_rule('/', '/',
+                     handler, methods=['GET', 'PUT'])
+    return addr, port
+
+
+def generate_url_and_params(app, sig, flavor):
+    '''
+    Digest command signature from cluster; generate an absolute
+    (including app.ceph_baseurl) endpoint from all the prefix words,
+    and a list of non-prefix param descs
+    '''
+
+    url = ''
+    params = []
+    # the OSD command descriptors don't include the 'tell ', so
+    # tack it onto the front of sig
+    if flavor == 'tell':
+        tellsig = parse_funcsig(['tell',
+                                {'name':'target', 'type':'CephOsdName'}])
+        sig = tellsig + sig
+
+    for desc in sig:
+        # prefixes go in the URL path
+        if desc.t == CephPrefix:
+            url += '/' + desc.instance.prefix
+        # CephChoices with 1 required string (not --) do too, unless
+        # we've already started collecting params, in which case they
+        # too are params
+        elif desc.t == CephChoices and \
+             len(desc.instance.strings) == 1 and \
+             desc.req and \
+             not str(desc.instance).startswith('--') and \
+             not params:
+            url += '/' + str(desc.instance)
+        else:
+            # tell/ is a weird case; the URL includes what
+            # would everywhere else be a parameter
+            if flavor == 'tell' and  \
+              (desc.t, desc.name) == (CephOsdName, 'target'):
+                url += '/'
+            else:
+                params.append(desc)
+
+    return app.ceph_baseurl + url, params
+
+
+#
+# end setup (import-time) functions, begin request-time functions
+#
+
+def concise_sig_for_uri(sig, flavor):
+    '''
+    Return a generic description of how one would send a REST request for sig
+    '''
+    prefix = []
+    args = []
+    ret = ''
+    if flavor == 'tell':
+        ret = 'tell//'
+    for d in sig:
+        if d.t == CephPrefix:
+            prefix.append(d.instance.prefix)
+        else:
+            args.append(d.name + '=' + str(d))
+    ret += '/'.join(prefix)
+    if args:
+        ret += '?' + '&'.join(args)
+    return ret
+
+def show_human_help(prefix):
+    '''
+    Dump table showing commands matching prefix
+    '''
+    # XXX There ought to be a better discovery mechanism than an HTML table
+    s = ''
+
+    permmap = {'r':'GET', 'rw':'PUT', 'rx':'GET', 'rwx':'PUT'}
+    line = ''
+    for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort):
+        concise = concise_sig(cmdsig['sig'])
+        flavor = cmdsig.get('flavor', 'mon')
+        if flavor == 'tell':
+            concise = 'tell//' + concise
+        if concise.startswith(prefix):
+            line = ['\n')
+            s += ''.join(line)
+
+    s += '
Possible commands:MethodDescription
']
+            wrapped_sig = textwrap.wrap(
+                concise_sig_for_uri(cmdsig['sig'], flavor), 40
+            )
+            for sigline in wrapped_sig:
+                line.append(flask.escape(sigline) + '\n')
+            line.append('')
+            line.append(permmap[cmdsig['perm']])
+            line.append('')
+            line.append(flask.escape(cmdsig['help']))
+            line.append('
'
+    if line:
+        return s
+    else:
+        return ''
+
+@app.before_request
+def log_request():
+    '''
+    For every request, log it.  XXX Probably overkill for production
+    '''
+    app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string)
+    app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values())
+
+@app.route('/')
+def root_redir():
+    return flask.redirect(app.ceph_baseurl)
+
+def make_response(fmt, output, statusmsg, errorcode):
+    '''
+    If formatted output, cobble up a response object that contains the
+    output and status wrapped in enclosing objects; if nonformatted, just
+    use output+status.  Return HTTP status errorcode in any event.
+    '''
+    response = output
+    if fmt:
+        if 'json' in fmt:
+            try:
+                native_output = json.loads(output or '[]')
+                response = json.dumps({"output":native_output,
+                                       "status":statusmsg})
+            except:
+                return flask.make_response("Error decoding JSON from " +
+                                           output, 500)
+        elif 'xml' in fmt:
+            # XXX
+            # one is tempted to do this with xml.etree, but figuring out how
+            # to 'un-XML' the XML-dumped output so it can be reassembled into
+            # a piece of the tree here is beyond me right now.
+            #ET = xml.etree.ElementTree
+            #resp_elem = ET.Element('response')
+            #o = ET.SubElement(resp_elem, 'output')
+            #o.text = output
+            #s = ET.SubElement(resp_elem, 'status')
+            #s.text = statusmsg
+            #response = ET.tostring(resp_elem)
+            response = '''
+
+  
+    {0}
+  
+  
+    {1}
+  
+'''.format(response, xml.sax.saxutils.escape(statusmsg))
+    else:
+        if not 200 <= errorcode < 300:
+            response = response + '\n' + statusmsg + '\n'
+
+    return flask.make_response(response, errorcode)
+
+def handler(catchall_path=None, fmt=None, target=None):
+    '''
+    Main endpoint handler; generic for every endpoint, including catchall.
+    Handles the catchall, anything with <.fmt>, anything with embedded
+    .  Partial match or ?help cause the HTML-table
+    "show_human_help" output.  
+    '''
+
+    ep = catchall_path or flask.request.endpoint
+    ep = ep.replace('.', '')
+
+    if ep[0] != '/':
+        ep = '/' + ep
+
+    # demand that endpoint begin with app.ceph_baseurl
+    if not ep.startswith(app.ceph_baseurl):
+        return make_response(fmt, '', 'Page not found', 404)
+
+    rel_ep = ep[len(app.ceph_baseurl)+1:]
+
+    # Extensions override Accept: headers override defaults
+    if not fmt:
+        if 'application/json' in flask.request.accept_mimetypes.values():
+            fmt = 'json'
+        elif 'application/xml' in flask.request.accept_mimetypes.values():
+            fmt = 'xml'
+
+    prefix = ''
+    pgid = None
+    cmdtarget = 'mon', ''
+
+    if target:
+        # got tell/; validate osdid or pgid
+        name = CephOsdName()
+        pgidobj = CephPgid()
+        try:
+            name.valid(target)
+        except ArgumentError:
+            # try pgid
+            try:
+                pgidobj.valid(target)
+            except ArgumentError:
+                return flask.make_response("invalid osdid or pgid", 400)
+            else:
+                # it's a pgid
+                pgid = pgidobj.val
+                cmdtarget = 'pg', pgid
+        else:
+            # it's an osd
+            cmdtarget = name.nametype, name.nameid
+
+        # prefix does not include tell//
+        prefix = ' '.join(rel_ep.split('/')[2:]).strip()
+    else:
+        # non-target command: prefix is entire path
+        prefix = ' '.join(rel_ep.split('/')).strip()
+
+    # show "match as much as you gave me" help for unknown endpoints
+    if not ep in app.ceph_urls:
+        helptext = show_human_help(prefix)
+        if helptext:
+            resp = flask.make_response(helptext, 400)
+            resp.headers['Content-Type'] = 'text/html'
+            return resp
+        else:
+            return make_response(fmt, '', 'Invalid endpoint ' + ep, 400)
+
+    found = None
+    exc = ''
+    for urldict in app.ceph_urls[ep]:
+        if flask.request.method not in urldict['methods']:
+            continue
+        paramsig = urldict['paramsig']
+
+        # allow '?help' for any specifically-known endpoint
+        if 'help' in flask.request.args:
+            response = flask.make_response('{0}: {1}'.\
+                format(prefix + concise_sig(paramsig), urldict['help']))
+            response.headers['Content-Type'] = 'text/plain'
+            return response
+
+        # if there are parameters for this endpoint, process them
+        if paramsig:
+            args = {}
+            for k, l in flask.request.args.iterlists():
+                if len(l) == 1:
+                    args[k] = l[0]
+                else:
+                    args[k] = l
+
+            # is this a valid set of params?
+            try:
+                argdict = validate(args, paramsig)
+                found = urldict
+                break
+            except Exception as e:
+                exc += str(e)
+                continue
+        else:
+            if flask.request.args:
+                continue
+            found = urldict
+            argdict = {}
+            break
+
+    if not found:
+        return make_response(fmt, '', exc + '\n', 400)
+
+    argdict['format'] = fmt or 'plain'
+    argdict['module'] = found['module']
+    argdict['perm'] = found['perm']
+    if pgid:
+        argdict['pgid'] = pgid
+
+    if not cmdtarget:
+        cmdtarget = ('mon', '')
+
+    app.logger.debug('sending command prefix %s argdict %s', prefix, argdict)
+    ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix,
+                                     target=cmdtarget,
+                                     inbuf=flask.request.data, argdict=argdict)
+    if ret:
+        return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400)
+
+    response = make_response(fmt, outbuf, outs or 'OK', 200)
+    if fmt:
+        contenttype = 'application/' + fmt.replace('-pretty','')
+    else:
+        contenttype = 'text/plain'
+    response.headers['Content-Type'] = contenttype
+    return response
+
+#
+# Main entry point from wrapper/WSGI server: call with cmdline args,
+# get back the WSGI app entry point
+#
+def generate_app(conf, cluster, clientname, clientid, args):
+    addr, port = api_setup(app, conf, cluster, clientname, clientid, args)
+    app.ceph_addr = addr
+    app.ceph_port = port
+    return app
diff --git a/ceph/src/pybind/cephfs.py b/ceph/src/pybind/cephfs.py
new file mode 100644
index 00000000..13a82625
--- /dev/null
+++ b/ceph/src/pybind/cephfs.py
@@ -0,0 +1,356 @@
+"""
+This module is a thin wrapper around libcephfs.
+"""
+from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_int, c_long, c_uint, c_ulong, \
+    create_string_buffer, byref, Structure
+from ctypes.util import find_library
+import errno
+
+class Error(Exception):
+    pass
+
+class PermissionError(Error):
+    pass
+
+class ObjectNotFound(Error):
+    pass
+
+class NoData(Error):
+    pass
+
+class ObjectExists(Error):
+    pass
+
+class IOError(Error):
+    pass
+
+class NoSpace(Error):
+    pass
+
+class IncompleteWriteError(Error):
+    pass
+
+class LibCephFSStateError(Error):
+    pass
+
+def make_ex(ret, msg):
+    """
+    Translate a libcephfs return code into an exception.
+
+    :param ret: the return code
+    :type ret: int
+    :param msg: the error message to use
+    :type msg: str
+    :returns: a subclass of :class:`Error`
+    """
+
+    errors = {
+        errno.EPERM     : PermissionError,
+        errno.ENOENT    : ObjectNotFound,
+        errno.EIO       : IOError,
+        errno.ENOSPC    : NoSpace,
+        errno.EEXIST    : ObjectExists,
+        errno.ENODATA   : NoData
+        }
+    ret = abs(ret)
+    if ret in errors:
+        return errors[ret](msg)
+    else:
+        return Error(msg + (": error code %d" % ret))
+
+class cephfs_statvfs(Structure):
+    _fields_ = [("f_bsize", c_uint),
+                ("f_frsize", c_uint),
+                ("f_blocks", c_uint),
+                ("f_bfree", c_uint),
+                ("f_bavail", c_uint),
+                ("f_files", c_uint),
+                ("f_ffree", c_uint),
+                ("f_favail", c_uint),
+                ("f_fsid", c_uint),
+                ("f_flag", c_uint),
+                ("f_namemax", c_uint)]
+
+# struct timespec {
+#   long int tv_sec;
+#   long int tv_nsec;
+# }
+class cephfs_timespec(Structure):
+    _fields_ = [('tv_sec', c_long),
+                ('tv_nsec', c_long)]
+
+# struct stat {
+#   unsigned long st_dev;
+#   unsigned long st_ino;
+#   unsigned long st_nlink;
+#   unsigned int st_mode;
+#   unsigned int st_uid;
+#   unsigned int st_gid;
+#   int __pad0;
+#   unsigned long st_rdev;
+#   long int st_size;
+#   long int st_blksize;
+#   long int st_blocks;
+#   struct timespec st_atim;
+#   struct timespec st_mtim;
+#   struct timespec st_ctim;
+#   long int __unused[3];
+# };
+class cephfs_stat(Structure):
+    _fields_ = [('st_dev', c_ulong), # ID of device containing file
+                ('st_ino', c_ulong), # inode number
+                ('st_nlink', c_ulong), # number of hard links
+                ('st_mode', c_uint), # protection
+                ('st_uid', c_uint), # user ID of owner
+                ('st_gid', c_uint), # group ID of owner
+                ('__pad0', c_int),
+                ('st_rdev', c_ulong), # device ID (if special file)
+                ('st_size', c_long), # total size, in bytes
+                ('st_blksize', c_long), # blocksize for file system I/O
+                ('st_blocks', c_long), # number of 512B blocks allocated
+                ('st_atime', cephfs_timespec), # time of last access
+                ('st_mtime', cephfs_timespec), # time of last modification
+                ('st_ctime', cephfs_timespec), # time of last status change
+                ('__unused1', c_long),
+                ('__unused2', c_long),
+                ('__unused3', c_long) ]
+
+def load_libcephfs():
+    """
+    Load the libcephfs shared library.
+    """
+    libcephfs_path = find_library('cephfs')
+    if libcephfs_path:
+        return CDLL(libcephfs_path)
+
+    # try harder, find_library() doesn't search LD_LIBRARY_PATH
+    # in addition, it doesn't seem work on centos 6.4 (see e46d2ca067b5)
+    try:
+        return CDLL('libcephfs.so.1')
+    except OSError as e:
+        raise EnvironmentError("Unable to load libcephfs: %s" % e)
+
+class LibCephFS(object):
+    """libcephfs python wrapper"""
+    def require_state(self, *args):
+        for a in args:
+            if self.state == a:
+                return
+        raise LibCephFSStateError("You cannot perform that operation on a "
+                                  "CephFS object in state %s." % (self.state))
+
+    def __init__(self, conf=None, conffile=None):
+        self.libcephfs = load_libcephfs()
+        self.cluster = c_void_p()
+
+        if conffile is not None and not isinstance(conffile, str):
+            raise TypeError('conffile must be a string or None')
+        ret = self.libcephfs.ceph_create(byref(self.cluster), c_char_p(0))
+        if ret != 0:
+            raise Error("libcephfs_initialize failed with error code: %d" %ret)
+        self.state = "configuring"
+        if conffile is not None:
+            # read the default conf file when '' is given
+            if conffile == '':
+                conffile = None
+            self.conf_read_file(conffile)
+        if conf is not None:
+            for key, value in conf.iteritems():
+                self.conf_set(key, value)
+
+    def conf_read_file(self, conffile=None):
+        if conffile is not None and not isinstance(conffile, str):
+            raise TypeError('conffile param must be a string')
+        ret = self.libcephfs.ceph_conf_read_file(self.cluster, c_char_p(conffile))
+        if ret != 0:
+            raise make_ex(ret, "error calling conf_read_file")
+
+    def shutdown(self):
+        """
+        Unmount and destroy the ceph mount handle. 
+        """
+        if self.state != "shutdown":
+            self.libcephfs.ceph_shutdown(self.cluster)
+            self.state = "shutdown"
+
+    def __enter__(self):
+        self.mount()
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        self.shutdown()
+        return False
+
+    def __del__(self):
+        self.shutdown()
+
+    def version(self):
+        """
+        Get the version number of the ``libcephfs`` C library.
+
+        :returns: a tuple of ``(major, minor, extra)`` components of the
+                  libcephfs version
+        """
+        major = c_int(0)
+        minor = c_int(0)
+        extra = c_int(0)
+        self.libcephfs.ceph_version(byref(major), byref(minor), byref(extra))
+        return (major.value, minor.value, extra.value)
+
+    def conf_get(self, option):
+        self.require_state("configuring", "connected")
+        if not isinstance(option, str):
+            raise TypeError('option must be a string')
+        length = 20
+        while True:
+            ret_buf = create_string_buffer(length)
+            ret = self.libcephfs.ceph_conf_get(self.cluster, option,
+                                                ret_buf, c_size_t(length))
+            if ret == 0:
+                return ret_buf.value
+            elif ret == -errno.ENAMETOOLONG:
+                length = length * 2
+            elif ret == -errno.ENOENT:
+                return None
+            else:
+                raise make_ex(ret, "error calling conf_get")
+
+    def conf_set(self, option, val):
+        self.require_state("configuring", "connected")
+        if not isinstance(option, str):
+            raise TypeError('option must be a string')
+        if not isinstance(val, str):
+            raise TypeError('val must be a string')
+        ret = self.libcephfs.ceph_conf_set(self.cluster, c_char_p(option),
+                                            c_char_p(val))
+        if ret != 0:
+            raise make_ex(ret, "error calling conf_set")
+
+    def mount(self):
+        self.require_state("configuring")
+        ret = self.libcephfs.ceph_mount(self.cluster, "/")
+        if ret != 0:
+            raise make_ex(ret, "error calling ceph_mount")
+        self.state = "mounted"
+
+    def statfs(self, path):
+        self.require_state("mounted")
+        statbuf = cephfs_statvfs()
+        ret = self.libcephfs.ceph_statfs(self.cluster, c_char_p(path), byref(statbuf))
+        if ret < 0:
+            raise make_ex(ret, "statfs failed: %s" % path)
+        return {'f_bsize': statbuf.f_bsize,
+                'f_frsize': statbuf.f_frsize,
+                'f_blocks': statbuf.f_blocks,
+                'f_bfree': statbuf.f_bfree,
+                'f_bavail': statbuf.f_bavail,
+                'f_files': statbuf.f_files,
+                'f_ffree': statbuf.f_ffree,
+                'f_favail': statbuf.f_favail,
+                'f_fsid': statbuf.f_fsid,
+                'f_flag': statbuf.f_flag,
+                'f_namemax': statbuf.f_namemax }
+
+    def sync_fs(self):
+        self.require_state("mounted")
+        ret = self.libcephfs.ceph_sync_fs(self.cluster)
+        if ret < 0:
+            raise make_ex(ret, "sync_fs failed")
+
+    def getcwd(self):
+        self.require_state("mounted")
+        return self.libcephfs.ceph_getcwd(self.cluster)
+
+    def chdir(self, path):
+        self.require_state("mounted")
+        ret = self.libcephfs.ceph_chdir(self.cluster, c_char_p(path))
+        if ret < 0:
+            raise make_ex(ret, "chdir failed")
+
+    def mkdir(self, path, mode):
+        self.require_state("mounted")
+        if not isinstance(path, str):
+            raise TypeError('path must be a string')
+        ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode))
+        if ret < 0:
+            raise make_ex(ret, "error in mkdir '%s'" % path)
+
+    def mkdirs(self, path, mode):
+        self.require_state("mounted")
+        if not isinstance(path, str):
+            raise TypeError('path must be a string')
+        if not isinstance(mode, int):
+            raise TypeError('mode must be an int')
+        ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode))
+        if ret < 0:
+            raise make_ex(ret, "error in mkdirs '%s'" % path)
+
+    def open(self, path, flags, mode):
+        self.require_state("mounted")
+        if not isinstance(path, str):
+            raise TypeError('path must be a string')
+        if not isinstance(mode, int):
+            raise TypeError('mode must be an int')
+        if not isinstance(flags, int):
+            raise TypeError('flags must be an int')
+        ret = self.libcephfs.ceph_open(self.cluster, c_char_p(path), c_int(flags), c_int(mode))
+        if ret < 0:
+            raise make_ex(ret, "error in open '%s'" % path)
+        return ret
+
+    def close(self, fd):
+        self.require_state("mounted")
+        ret = self.libcephfs.ceph_close(self.cluster, c_int(fd))
+        if ret < 0:
+            raise make_ex(ret, "error in close")
+
+    def setxattr(self, path, name, value, flags):
+        if not isinstance(path, str):
+            raise TypeError('path must be a string')
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        if not isinstance(value, str):
+            raise TypeError('value must be a string')
+        self.require_state("mounted")
+        ret = self.libcephfs.ceph_setxattr(
+                    self.cluster,
+                    c_char_p(path),
+                    c_char_p(name),
+                    c_void_p(value),
+                    c_size_t(len(value)),
+                    c_int(flags))
+        if ret < 0:
+            raise make_ex(ret, "error in setxattr")
+
+    def stat(self, path):
+        self.require_state("mounted")
+        if not isinstance(path, str):
+            raise TypeError('path must be a string')
+        statbuf = cephfs_stat()
+        ret = self.libcephfs.ceph_stat(
+                self.cluster,
+                c_char_p(path),
+                byref(statbuf))
+        if ret < 0:
+            raise make_ex(ret, "error in stat: %s" % path)
+        return {'st_dev': statbuf.st_dev,
+                'st_ino': statbuf.st_ino,
+                'st_mode': statbuf.st_mode,
+                'st_nlink': statbuf.st_nlink,
+                'st_uid': statbuf.st_uid,
+                'st_gid': statbuf.st_gid,
+                'st_rdev': statbuf.st_rdev,
+                'st_size': statbuf.st_size,
+                'st_blksize': statbuf.st_blksize,
+                'st_blocks': statbuf.st_blocks,
+                'st_atime': statbuf.st_atime,
+                'st_mtime': statbuf.st_mtime,
+                'st_ctime': statbuf.st_ctime }
+
+    def unlink(self, path):
+        self.require_state("mounted")
+        ret = self.libcephfs.ceph_unlink(
+            self.cluster,
+            c_char_p(path))
+        if ret < 0:
+            raise make_ex(ret, "error in unlink: %s" % path)
diff --git a/ceph/src/pybind/rados.py b/ceph/src/pybind/rados.py
new file mode 100644
index 00000000..0fbd10e6
--- /dev/null
+++ b/ceph/src/pybind/rados.py
@@ -0,0 +1,1771 @@
+"""
+This module is a thin wrapper around librados.
+
+Copyright 2011, Hannu Valtonen 
+"""
+from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_char, c_int, c_long, \
+    c_ulong, create_string_buffer, byref, Structure, c_uint64, c_ubyte, \
+    pointer, CFUNCTYPE
+from ctypes.util import find_library
+import ctypes
+import errno
+import threading
+import time
+from datetime import datetime
+
+ANONYMOUS_AUID = 0xffffffffffffffff
+ADMIN_AUID = 0
+
+class Error(Exception):
+    """ `Error` class, derived from `Exception` """
+    pass
+
+class InterruptedOrTimeoutError(Error):
+    """ `InterruptedOrTimeoutError` class, derived from `Error` """
+    pass
+
+class PermissionError(Error):
+    """ `PermissionError` class, derived from `Error` """
+    pass
+
+class ObjectNotFound(Error):
+    """ `ObjectNotFound` class, derived from `Error` """
+    pass
+
+class NoData(Error):
+    """ `NoData` class, derived from `Error` """
+    pass
+
+class ObjectExists(Error):
+    """ `ObjectExists` class, derived from `Error` """
+    pass
+
+class IOError(Error):
+    """ `IOError` class, derived from `Error` """
+    pass
+
+class NoSpace(Error):
+    """ `NoSpace` class, derived from `Error` """
+    pass
+
+class IncompleteWriteError(Error):
+    """ `IncompleteWriteError` class, derived from `Error` """
+    pass
+
+class RadosStateError(Error):
+    """ `RadosStateError` class, derived from `Error` """
+    pass
+
+class IoctxStateError(Error):
+    """ `IoctxStateError` class, derived from `Error` """
+    pass
+
+class ObjectStateError(Error):
+    """ `ObjectStateError` class, derived from `Error` """
+    pass
+
+class LogicError(Error):
+    """ `` class, derived from `Error` """
+    pass
+
+class TimedOut(Error):
+    """ `TimedOut` class, derived from `Error` """
+    pass
+
+def make_ex(ret, msg):
+    """
+    Translate a librados return code into an exception.
+
+    :param ret: the return code
+    :type ret: int
+    :param msg: the error message to use
+    :type msg: str
+    :returns: a subclass of :class:`Error`
+    """
+
+    errors = {
+        errno.EPERM     : PermissionError,
+        errno.ENOENT    : ObjectNotFound,
+        errno.EIO       : IOError,
+        errno.ENOSPC    : NoSpace,
+        errno.EEXIST    : ObjectExists,
+        errno.ENODATA   : NoData,
+        errno.EINTR     : InterruptedOrTimeoutError,
+        errno.ETIMEDOUT : TimedOut
+        }
+    ret = abs(ret)
+    if ret in errors:
+        return errors[ret](msg)
+    else:
+        return Error(msg + (": errno %s" % errno.errorcode[ret]))
+
+class rados_pool_stat_t(Structure):
+    """ Usage information for a pool """
+    _fields_ = [("num_bytes", c_uint64),
+                ("num_kb", c_uint64),
+                ("num_objects", c_uint64),
+                ("num_object_clones", c_uint64),
+                ("num_object_copies", c_uint64),
+                ("num_objects_missing_on_primary", c_uint64),
+                ("num_objects_unfound", c_uint64),
+                ("num_objects_degraded", c_uint64),
+                ("num_rd", c_uint64),
+                ("num_rd_kb", c_uint64),
+                ("num_wr", c_uint64),
+                ("num_wr_kb", c_uint64)]
+
+class rados_cluster_stat_t(Structure):
+    """ Cluster-wide usage information """
+    _fields_ = [("kb", c_uint64),
+                ("kb_used", c_uint64),
+                ("kb_avail", c_uint64),
+                ("num_objects", c_uint64)]
+
+class Version(object):
+    """ Version information """
+    def __init__(self, major, minor, extra):
+        self.major = major
+        self.minor = minor
+        self.extra = extra
+
+    def __str__(self):
+        return "%d.%d.%d" % (self.major, self.minor, self.extra)
+
+class RadosThread(threading.Thread):
+    def __init__(self, target, args=None):
+        self.args = args
+        self.target = target
+        threading.Thread.__init__(self)
+
+    def run(self):
+        self.retval = self.target(*self.args)
+
+# time in seconds between each call to t.join() for child thread
+POLL_TIME_INCR = 0.5
+
+def run_in_thread(target, args, timeout=0):
+    import sys
+    interrupt = False
+
+    countdown = timeout
+    t = RadosThread(target, args)
+
+    # allow the main thread to exit (presumably, avoid a join() on this
+    # subthread) before this thread terminates.  This allows SIGINT
+    # exit of a blocked call.  See below.
+    t.daemon = True
+
+    t.start()
+    try:
+        # poll for thread exit
+        while t.is_alive():
+            t.join(POLL_TIME_INCR)
+            if timeout:
+                countdown = countdown - POLL_TIME_INCR
+                if countdown <= 0:
+                    raise KeyboardInterrupt
+
+        t.join()        # in case t exits before reaching the join() above
+    except KeyboardInterrupt:
+        # ..but allow SIGINT to terminate the waiting.  Note: this
+        # relies on the Linux kernel behavior of delivering the signal
+        # to the main thread in preference to any subthread (all that's
+        # strictly guaranteed is that *some* thread that has the signal
+        # unblocked will receive it).  But there doesn't seem to be
+        # any interface to create t with SIGINT blocked.
+        interrupt = True
+
+    if interrupt:
+        t.retval = -errno.EINTR
+    return t.retval
+
+class Rados(object):
+    """librados python wrapper"""
+    def require_state(self, *args):
+        """ 
+        Checks if the Rados object is in a special state
+
+        :raises: RadosStateError
+        """
+        for a in args:
+            if self.state == a:
+                return
+        raise RadosStateError("You cannot perform that operation on a \
+Rados object in state %s." % (self.state))
+
+    def __init__(self, rados_id=None, name=None, clustername=None,
+                 conf_defaults=None, conffile=None, conf=None, flags=0):
+        librados_path = find_library('rados')
+        if not librados_path:
+            #maybe find_library can not find it correctly on all platforms.
+            try:
+                self.librados = CDLL('librados.so.2')
+            except OSError as e:
+                    raise EnvironmentError("Unable to load librados: %s" % e)
+            except:
+                raise Error("Unexpected error")
+        else:
+            self.librados = CDLL(librados_path)
+        self.cluster = c_void_p()
+        self.rados_id = rados_id
+        if rados_id is not None and not isinstance(rados_id, str):
+            raise TypeError('rados_id must be a string or None')
+        if conffile is not None and not isinstance(conffile, str):
+            raise TypeError('conffile must be a string or None')
+        if name is not None and not isinstance(name, str):
+            raise TypeError('name must be a string or None')
+        if clustername is not None and not isinstance(clustername, str):
+            raise TypeError('clustername must be a string or None')
+        if rados_id and name:
+            raise Error("Rados(): can't supply both rados_id and name")
+        if rados_id:
+            name = 'client.' +  rados_id
+        if name is None:
+            name = 'client.admin'
+        if clustername is None:
+            clustername = 'ceph'
+        ret = run_in_thread(self.librados.rados_create2,
+                            (byref(self.cluster), c_char_p(clustername),
+                            c_char_p(name), c_uint64(flags)))
+
+        if ret != 0:
+            raise Error("rados_initialize failed with error code: %d" % ret)
+        self.state = "configuring"
+        # order is important: conf_defaults, then conffile, then conf
+        if conf_defaults:
+            for key, value in conf_defaults.iteritems():
+                self.conf_set(key, value)
+        if conffile is not None:
+            # read the default conf file when '' is given
+            if conffile == '':
+                conffile = None
+            self.conf_read_file(conffile)
+        if conf:
+            for key, value in conf.iteritems():
+                self.conf_set(key, value)
+
+    def shutdown(self):
+        """
+        Disconnects from the cluster.
+        """
+        if (self.__dict__.has_key("state") and self.state != "shutdown"):
+            run_in_thread(self.librados.rados_shutdown, (self.cluster,))
+            self.state = "shutdown"
+
+    def __enter__(self):
+        self.connect()
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        self.shutdown()
+        return False
+
+    def __del__(self):
+        self.shutdown()
+
+    def version(self):
+        """
+        Get the version number of the ``librados`` C library.
+    
+        :returns: a tuple of ``(major, minor, extra)`` components of the
+                  librados version
+        """
+        major = c_int(0)
+        minor = c_int(0)
+        extra = c_int(0)
+        run_in_thread(self.librados.rados_version,
+                      (byref(major), byref(minor), byref(extra)))
+        return Version(major.value, minor.value, extra.value)
+
+    def conf_read_file(self, path=None):
+        """
+        Configure the cluster handle using a Ceph config file.
+
+        :param path: path to the config file
+        :type path: str
+        """
+        self.require_state("configuring", "connected")
+        if path is not None and not isinstance(path, str):
+            raise TypeError('path must be a string')
+        ret = run_in_thread(self.librados.rados_conf_read_file,
+                            (self.cluster, c_char_p(path)))
+        if (ret != 0):
+            raise make_ex(ret, "error calling conf_read_file")
+
+    def conf_parse_argv(self, args):
+        """
+        Parse known arguments from args, and remove; returned
+        args contain only those unknown to ceph
+        """
+        self.require_state("configuring", "connected")
+        if not args:
+            return
+        # create instances of arrays of c_char_p's, both len(args) long
+        # cretargs will always be a subset of cargs (perhaps identical)
+        cargs = (c_char_p * len(args))(*args)
+        cretargs = (c_char_p * len(args))()
+        ret = run_in_thread(self.librados.rados_conf_parse_argv_remainder,
+                            (self.cluster, len(args), cargs, cretargs))
+        if ret:
+            raise make_ex(ret, "error calling conf_parse_argv_remainder")
+
+        # cretargs was allocated with fixed length; collapse return
+        # list to eliminate any missing args
+
+        retargs = [a for a in cretargs if a is not None]
+        return retargs
+
+    def conf_parse_env(self, var='CEPH_ARGS'):
+        """
+        Parse known arguments from an environment variable, normally
+        CEPH_ARGS.
+        """
+        self.require_state("configuring", "connected")
+        if not var:
+            return
+        ret = run_in_thread(self.librados.rados_conf_parse_env,
+                            (self.cluster, c_char_p(var)))
+        if (ret != 0):
+            raise make_ex(ret, "error calling conf_parse_env")
+
+    def conf_get(self, option):
+        """
+        Get the value of a configuration option
+
+        :param option: which option to read
+        :type option: str
+
+        :returns: str - value of the option or None
+        :raises: :class:`TypeError`
+        """
+        self.require_state("configuring", "connected")
+        if not isinstance(option, str):
+            raise TypeError('option must be a string')
+        length = 20
+        while True:
+            ret_buf = create_string_buffer(length)
+            ret = run_in_thread(self.librados.rados_conf_get,
+                                (self.cluster, c_char_p(option), ret_buf,
+                                c_size_t(length)))
+            if (ret == 0):
+                return ret_buf.value
+            elif (ret == -errno.ENAMETOOLONG):
+                length = length * 2
+            elif (ret == -errno.ENOENT):
+                return None
+            else:
+                raise make_ex(ret, "error calling conf_get")
+
+    def conf_set(self, option, val):
+        """
+        Set the value of a configuration option
+
+        :param option: which option to set
+        :type option: str
+        :param option: value of the option
+        :type option: str
+
+        :raises: :class:`TypeError`, :class:`ObjectNotFound`
+        """
+        self.require_state("configuring", "connected")
+        if not isinstance(option, str):
+            raise TypeError('option must be a string')
+        if not isinstance(val, str):
+            raise TypeError('val must be a string')
+        ret = run_in_thread(self.librados.rados_conf_set,
+                            (self.cluster, c_char_p(option), c_char_p(val)))
+        if (ret != 0):
+            raise make_ex(ret, "error calling conf_set")
+
+
+    def ping_monitor(self, mon_id):
+      """
+      Ping a monitor to assess liveness
+
+      May be used as a simply way to assess liveness, or to obtain
+      information about the monitor in a simple way even in the
+      absence of quorum.
+
+      :param mon_id: the ID portion of the monitor's name (i.e., mon.)
+      :type mon_id: str
+      :returns: the string reply from the monitor
+      """
+
+      self.require_state("configuring", "connected")
+
+      outstrp = pointer(pointer(c_char()))
+      outstrlen = c_long()
+
+      ret = run_in_thread(self.librados.rados_ping_monitor,
+                          (self.cluster, c_char_p(mon_id),
+                           outstrp, byref(outstrlen)))
+
+      my_outstr = outstrp.contents[:(outstrlen.value)]
+      if outstrlen.value:
+        run_in_thread(self.librados.rados_buffer_free, (outstrp.contents,))
+
+      if ret != 0:
+        raise make_ex(ret, "error calling ping_monitor")
+      return my_outstr
+
+    def connect(self, timeout=0):
+        """
+        Connect to the cluster.
+        """
+        self.require_state("configuring")
+        ret = run_in_thread(self.librados.rados_connect, (self.cluster,),
+                            timeout)
+        if (ret != 0):
+            raise make_ex(ret, "error calling connect")
+        self.state = "connected"
+
+    def get_cluster_stats(self):
+        """
+        Read usage info about the cluster
+        
+        This tells you total space, space used, space available, and number
+        of objects. These are not updated immediately when data is written,
+        they are eventually consistent.
+
+        :returns: dict - contains the following keys:
+
+            - ``kb`` (int) - total space 
+
+            - ``kb_used`` (int) - space used
+
+            - ``kb_avail`` (int) - free space available
+
+            - ``num_objects`` (int) - number of objects
+
+        """
+        stats = rados_cluster_stat_t()
+        ret = run_in_thread(self.librados.rados_cluster_stat,
+                            (self.cluster, byref(stats)))
+        if ret < 0:
+            raise make_ex(
+                ret, "Rados.get_cluster_stats(%s): get_stats failed" % self.rados_id)
+        return {'kb': stats.kb,
+                'kb_used': stats.kb_used,
+                'kb_avail': stats.kb_avail,
+                'num_objects': stats.num_objects}
+
+    def pool_exists(self, pool_name):
+        """
+        Checks if a given pool exists.
+
+        :param pool_name: name of the pool to check
+        :type pool_name: str
+
+        :raises: :class:`TypeError`, :class:`Error`
+        :returns: bool - whether the pool exists, false otherwise.
+        """
+        self.require_state("connected")
+        if not isinstance(pool_name, str):
+            raise TypeError('pool_name must be a string')
+        ret = run_in_thread(self.librados.rados_pool_lookup,
+                            (self.cluster, c_char_p(pool_name)))
+        if (ret >= 0):
+            return True
+        elif (ret == -errno.ENOENT):
+            return False
+        else:
+            raise make_ex(ret, "error looking up pool '%s'" % pool_name)
+
+    def create_pool(self, pool_name, auid=None, crush_rule=None):
+        """
+        Create a pool:
+        - with default settings: if auid=None and crush_rule=None
+        - owned by a specific auid: auid given and crush_rule=None
+        - with a specific CRUSH rule: if auid=None and crush_rule given
+        - with a specific CRUSH rule and auid: if auid and crush_rule given
+
+        :param pool_name: name of the pool to create
+        :type pool_name: str
+        :param auid: the id of the owner of the new pool
+        :type auid: int
+        :param crush_rule: rule to use for placement in the new pool
+        :type crush_rule: str
+
+        :raises: :class:`TypeError`, :class:`Error`
+        """
+        self.require_state("connected")
+        if not isinstance(pool_name, str):
+            raise TypeError('pool_name must be a string')
+        if crush_rule is not None and not isinstance(crush_rule, str):
+            raise TypeError('cruse_rule must be a string')
+        if (auid == None):
+            if (crush_rule == None):
+                ret = run_in_thread(self.librados.rados_pool_create,
+                                    (self.cluster, c_char_p(pool_name)))
+            else:
+                ret = run_in_thread(self.librados.\
+                                    rados_pool_create_with_crush_rule,
+                                    (self.cluster, c_char_p(pool_name),
+                                    c_ubyte(crush_rule)))
+
+        elif (crush_rule == None):
+            ret = run_in_thread(self.librados.rados_pool_create_with_auid,
+                                (self.cluster, c_char_p(pool_name),
+                                c_uint64(auid)))
+        else:
+            ret = run_in_thread(self.librados.rados_pool_create_with_all,
+                                (self.cluster, c_char_p(pool_name),
+                                c_uint64(auid), c_ubyte(crush_rule)))
+        if ret < 0:
+            raise make_ex(ret, "error creating pool '%s'" % pool_name)
+
+    def delete_pool(self, pool_name):
+        """
+        Delete a pool and all data inside it.
+
+        The pool is removed from the cluster immediately,
+        but the actual data is deleted in the background.
+
+        :param pool_name: name of the pool to delete 
+        :type pool_name: str
+
+        :raises: :class:`TypeError`, :class:`Error`
+        """
+        self.require_state("connected")
+        if not isinstance(pool_name, str):
+            raise TypeError('pool_name must be a string')
+        ret = run_in_thread(self.librados.rados_pool_delete,
+                            (self.cluster, c_char_p(pool_name)))
+        if ret < 0:
+            raise make_ex(ret, "error deleting pool '%s'" % pool_name)
+
+    def list_pools(self):
+        """
+        Gets a list of pool names. 
+
+        :returns: list - of pool names.
+        """
+        self.require_state("connected")
+        size = c_size_t(512)
+        while True:
+            c_names = create_string_buffer(size.value)
+            ret = run_in_thread(self.librados.rados_pool_list,
+                                (self.cluster, byref(c_names), size))
+            if ret > size.value:
+                size = c_size_t(ret)
+            else:
+                break
+        return filter(lambda name: name != '', c_names.raw.split('\0'))
+
+    def get_fsid(self):
+        """
+        Get the fsid of the cluster as a hexadecimal string.
+
+        :raises: :class:`Error`
+        :returns: str - cluster fsid
+        """
+        self.require_state("connected")
+        buf_len = 37
+        fsid = create_string_buffer(buf_len)
+        ret = run_in_thread(self.librados.rados_cluster_fsid,
+                            (self.cluster, byref(fsid), c_size_t(buf_len)))
+        if ret < 0:
+            raise make_ex(ret, "error getting cluster fsid")
+        return fsid.value
+
+    def open_ioctx(self, ioctx_name):
+        """
+        Create an io context
+
+        The io context allows you to perform operations within a particular
+        pool. 
+
+        :param ioctx_name: name of the pool 
+        :type ioctx_name: str
+
+        :raises: :class:`TypeError`, :class:`Error`
+        :returns: Ioctx - Rados Ioctx object 
+        """
+        self.require_state("connected")
+        if not isinstance(ioctx_name, str):
+            raise TypeError('ioctx_name must be a string')
+        ioctx = c_void_p()
+        ret = run_in_thread(self.librados.rados_ioctx_create,
+                            (self.cluster, c_char_p(ioctx_name), byref(ioctx)))
+        if ret < 0:
+            raise make_ex(ret, "error opening ioctx '%s'" % ioctx_name)
+        return Ioctx(ioctx_name, self.librados, ioctx)
+
+    def mon_command(self, cmd, inbuf, timeout=0, target=None):
+        """
+        mon_command[_target](cmd, inbuf, outbuf, outbuflen, outs, outslen)
+        returns (int ret, string outbuf, string outs)
+        """
+        import sys
+        self.require_state("connected")
+        outbufp = pointer(pointer(c_char()))
+        outbuflen = c_long()
+        outsp = pointer(pointer(c_char()))
+        outslen = c_long()
+        cmdarr = (c_char_p * len(cmd))(*cmd)
+
+        if target:
+            ret = run_in_thread(self.librados.rados_mon_command_target,
+                                (self.cluster, c_char_p(target), cmdarr,
+                                 len(cmd), c_char_p(inbuf), len(inbuf),
+                                 outbufp, byref(outbuflen), outsp,
+                                 byref(outslen)), timeout)
+        else:
+            ret = run_in_thread(self.librados.rados_mon_command,
+                                (self.cluster, cmdarr, len(cmd),
+                                 c_char_p(inbuf), len(inbuf),
+                                 outbufp, byref(outbuflen), outsp, byref(outslen)),
+                                timeout)
+
+        # copy returned memory (ctypes makes a copy, not a reference)
+        my_outbuf = outbufp.contents[:(outbuflen.value)]
+        my_outs = outsp.contents[:(outslen.value)]
+
+        # free callee's allocations
+        if outbuflen.value:
+            run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,))
+        if outslen.value:
+            run_in_thread(self.librados.rados_buffer_free, (outsp.contents,))
+
+        return (ret, my_outbuf, my_outs)
+
+    def osd_command(self, osdid, cmd, inbuf, timeout=0):
+        """
+        osd_command(osdid, cmd, inbuf, outbuf, outbuflen, outs, outslen)
+        returns (int ret, string outbuf, string outs)
+        """
+        import sys
+        self.require_state("connected")
+        outbufp = pointer(pointer(c_char()))
+        outbuflen = c_long()
+        outsp = pointer(pointer(c_char()))
+        outslen = c_long()
+        cmdarr = (c_char_p * len(cmd))(*cmd)
+        ret = run_in_thread(self.librados.rados_osd_command,
+                            (self.cluster, osdid, cmdarr, len(cmd),
+                            c_char_p(inbuf), len(inbuf),
+                            outbufp, byref(outbuflen), outsp, byref(outslen)),
+                            timeout)
+
+        # copy returned memory (ctypes makes a copy, not a reference)
+        my_outbuf = outbufp.contents[:(outbuflen.value)]
+        my_outs = outsp.contents[:(outslen.value)]
+
+        # free callee's allocations
+        if outbuflen.value:
+            run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,))
+        if outslen.value:
+            run_in_thread(self.librados.rados_buffer_free, (outsp.contents,))
+
+        return (ret, my_outbuf, my_outs)
+
+    def pg_command(self, pgid, cmd, inbuf, timeout=0):
+        """
+        pg_command(pgid, cmd, inbuf, outbuf, outbuflen, outs, outslen)
+        returns (int ret, string outbuf, string outs)
+        """
+        import sys
+        self.require_state("connected")
+        outbufp = pointer(pointer(c_char()))
+        outbuflen = c_long()
+        outsp = pointer(pointer(c_char()))
+        outslen = c_long()
+        cmdarr = (c_char_p * len(cmd))(*cmd)
+        ret = run_in_thread(self.librados.rados_pg_command,
+                            (self.cluster, c_char_p(pgid), cmdarr, len(cmd),
+                            c_char_p(inbuf), len(inbuf),
+                            outbufp, byref(outbuflen), outsp, byref(outslen)),
+                            timeout)
+
+        # copy returned memory (ctypes makes a copy, not a reference)
+        my_outbuf = outbufp.contents[:(outbuflen.value)]
+        my_outs = outsp.contents[:(outslen.value)]
+
+        # free callee's allocations
+        if outbuflen.value:
+            run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,))
+        if outslen.value:
+            run_in_thread(self.librados.rados_buffer_free, (outsp.contents,))
+
+        return (ret, my_outbuf, my_outs)
+
+class ObjectIterator(object):
+    """rados.Ioctx Object iterator"""
+    def __init__(self, ioctx):
+        self.ioctx = ioctx
+        self.ctx = c_void_p()
+        ret = run_in_thread(self.ioctx.librados.rados_objects_list_open,
+                            (self.ioctx.io, byref(self.ctx)))
+        if ret < 0:
+            raise make_ex(ret, "error iterating over the objects in ioctx '%s'" \
+                % self.ioctx.name)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        """
+        Get the next object name and locator in the pool
+
+        :raises: StopIteration
+        :returns: next rados.Ioctx Object
+        """
+        key = c_char_p()
+        locator = c_char_p()
+        ret = run_in_thread(self.ioctx.librados.rados_objects_list_next,
+                            (self.ctx, byref(key), byref(locator)))
+        if ret < 0:
+            raise StopIteration()
+        return Object(self.ioctx, key.value, locator.value)
+
+    def __del__(self):
+        run_in_thread(self.ioctx.librados.rados_objects_list_close, (self.ctx,))
+
+class XattrIterator(object):
+    """Extended attribute iterator"""
+    def __init__(self, ioctx, it, oid):
+        self.ioctx = ioctx
+        self.it = it
+        self.oid = oid
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        """
+        Get the next xattr on the object
+
+        :raises: StopIteration
+        :returns: pair - of name and value of the next Xattr
+        """
+        name_ = c_char_p(0)
+        val_ = c_char_p(0)
+        len_ = c_int(0)
+        ret = run_in_thread(self.ioctx.librados.rados_getxattrs_next,
+                            (self.it, byref(name_), byref(val_), byref(len_)))
+        if (ret != 0):
+            raise make_ex(ret, "error iterating over the extended attributes \
+in '%s'" % self.oid)
+        if name_.value == None:
+            raise StopIteration()
+        name = ctypes.string_at(name_)
+        val = ctypes.string_at(val_, len_)
+        return (name, val)
+
+    def __del__(self):
+        run_in_thread(self.ioctx.librados.rados_getxattrs_end, (self.it,))
+
+class SnapIterator(object):
+    """Snapshot iterator"""
+    def __init__(self, ioctx):
+        self.ioctx = ioctx
+        # We don't know how big a buffer we need until we've called the
+        # function. So use the exponential doubling strategy.
+        num_snaps = 10
+        while True:
+            self.snaps = (ctypes.c_uint64 * num_snaps)()
+            ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_list,
+                                (self.ioctx.io, self.snaps, c_int(num_snaps)))
+            if (ret >= 0):
+                self.max_snap = ret
+                break
+            elif (ret != -errno.ERANGE):
+                raise make_ex(ret, "error calling rados_snap_list for \
+ioctx '%s'" % self.ioctx.name)
+            num_snaps = num_snaps * 2
+        self.cur_snap = 0
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        """
+        Get the next Snapshot
+
+        :raises: :class:`Error`, StopIteration 
+        :returns: Snap - next snapshot
+        """
+        if (self.cur_snap >= self.max_snap):
+            raise StopIteration
+        snap_id = self.snaps[self.cur_snap]
+        name_len = 10
+        while True:
+            name = create_string_buffer(name_len)
+            ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_get_name,
+                                (self.ioctx.io, c_uint64(snap_id), byref(name),
+                                 c_int(name_len)))
+            if (ret == 0):
+                name_len = ret
+                break
+            elif (ret != -errno.ERANGE):
+                raise make_ex(ret, "rados_snap_get_name error")
+            name_len = name_len * 2
+        snap = Snap(self.ioctx, name.value, snap_id)
+        self.cur_snap = self.cur_snap + 1
+        return snap
+
+class Snap(object):
+    """Snapshot object"""
+    def __init__(self, ioctx, name, snap_id):
+        self.ioctx = ioctx
+        self.name = name
+        self.snap_id = snap_id
+
+    def __str__(self):
+        return "rados.Snap(ioctx=%s,name=%s,snap_id=%d)" \
+            % (str(self.ioctx), self.name, self.snap_id)
+
+    def get_timestamp(self):
+        """
+        Find when a snapshot in the current pool occurred
+
+        :raises: :class:`Error`
+        :returns: datetime - the data and time the snapshot was created
+        """
+        snap_time = c_long(0)
+        ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_get_stamp,
+                            (self.ioctx.io, self.snap_id, byref(snap_time)))
+        if (ret != 0):
+            raise make_ex(ret, "rados_ioctx_snap_get_stamp error")
+        return datetime.fromtimestamp(snap_time.value)
+
+class Completion(object):
+    """completion object"""
+    def __init__(self, ioctx, rados_comp, oncomplete, onsafe):
+        self.rados_comp = rados_comp
+        self.oncomplete = oncomplete
+        self.onsafe = onsafe
+        self.ioctx = ioctx
+
+    def wait_for_safe(self):
+        """
+        Is an asynchronous operation safe?
+
+        This does not imply that the safe callback has finished.
+
+        :returns: whether the operation is safe
+        """
+        return run_in_thread(self.ioctx.librados.rados_aio_is_safe,
+                             (self.rados_comp,))
+
+    def wait_for_complete(self):
+        """
+        Has an asynchronous operation completed?
+
+        This does not imply that the safe callback has finished.
+
+        :returns:  whether the operation is completed
+        """
+        return run_in_thread(self.ioctx.librados.rados_aio_is_complete,
+                             (self.rados_comp,))
+
+    def get_return_value(self):
+        """
+        Get the return value of an asychronous operation
+
+        The return value is set when the operation is complete or safe,
+        whichever comes first.
+
+        :returns: int - return value of the operation
+        """
+        return run_in_thread(self.ioctx.librados.rados_aio_get_return_value,
+                             (self.rados_comp,))
+
+    def __del__(self):
+        """ 
+        Release a completion 
+
+        Call this when you no longer need the completion. It may not be
+        freed immediately if the operation is not acked and committed.
+        """
+        run_in_thread(self.ioctx.librados.rados_aio_release,
+                      (self.rados_comp,))
+
+class Ioctx(object):
+    """rados.Ioctx object"""
+    def __init__(self, name, librados, io):
+        self.name = name
+        self.librados = librados
+        self.io = io
+        self.state = "open"
+        self.locator_key = ""
+        self.safe_cbs = {}
+        self.complete_cbs = {}
+        RADOS_CB = CFUNCTYPE(c_int, c_void_p, c_void_p)
+        self.__aio_safe_cb_c = RADOS_CB(self.__aio_safe_cb)
+        self.__aio_complete_cb_c = RADOS_CB(self.__aio_complete_cb)
+        self.lock = threading.Lock()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        self.close()
+        return False
+
+    def __del__(self):
+        self.close()
+
+    def __aio_safe_cb(self, completion, _):
+        """ 
+        Callback to onsafe() for asynchronous operations 
+        """
+        cb = None
+        with self.lock:
+            cb = self.safe_cbs[completion]
+            del self.safe_cbs[completion]
+        cb.onsafe(cb)
+        return 0
+
+    def __aio_complete_cb(self, completion, _):
+        """ 
+        Callback to oncomplete() for asynchronous operations 
+        """
+        cb = None
+        with self.lock:
+            cb = self.complete_cbs[completion]
+            del self.complete_cbs[completion]
+        cb.oncomplete(cb)
+        return 0
+
+    def __get_completion(self, oncomplete, onsafe):
+        """
+        Constructs a completion to use with asynchronous operations
+
+        :param oncomplete: what to do when the write is safe and complete in memory
+            on all replicas
+        :type oncomplete: completion
+        :param onsafe:  what to do when the write is safe and complete on storage
+            on all replicas
+        :type onsafe: completion
+
+        :raises: :class:`Error`
+        :returns: completion object 
+        """
+        completion = c_void_p(0)
+        complete_cb = None
+        safe_cb = None
+        if oncomplete:
+            complete_cb = self.__aio_complete_cb_c
+        if onsafe:
+            safe_cb = self.__aio_safe_cb_c
+        ret = run_in_thread(self.librados.rados_aio_create_completion,
+                            (c_void_p(0), complete_cb, safe_cb,
+                            byref(completion)))
+        if ret < 0:
+            raise make_ex(ret, "error getting a completion")
+        with self.lock:
+            completion_obj = Completion(self, completion, oncomplete, onsafe)
+            if oncomplete:
+                self.complete_cbs[completion.value] = completion_obj
+            if onsafe:
+                self.safe_cbs[completion.value] = completion_obj
+        return completion_obj
+
+    def aio_write(self, object_name, to_write, offset=0,
+                  oncomplete=None, onsafe=None):
+        """
+        Write data to an object asynchronously
+
+        Queues the write and returns.
+
+        :param object_name: name of the object
+        :type object_name: str
+        :param to_write: data to write
+        :type to_write: str
+        :param offset: byte offset in the object to begin writing at
+        :type offset: int
+        :param oncomplete: what to do when the write is safe and complete in memory
+            on all replicas
+        :type oncomplete: completion
+        :param onsafe:  what to do when the write is safe and complete on storage
+            on all replicas
+        :type onsafe: completion
+
+        :raises: :class:`Error`
+        :returns: completion object 
+        """
+        completion = self.__get_completion(oncomplete, onsafe)
+        ret = run_in_thread(self.librados.rados_aio_write,
+                            (self.io, c_char_p(object_name),
+                            completion.rados_comp, c_char_p(to_write),
+                            c_size_t(len(to_write)), c_uint64(offset)))
+        if ret < 0:
+            raise make_ex(ret, "error writing object %s" % object_name)
+        return completion
+
+    def aio_write_full(self, object_name, to_write,
+                       oncomplete=None, onsafe=None):
+        """
+        Asychronously write an entire object
+
+        The object is filled with the provided data. If the object exists,
+        it is atomically truncated and then written.
+        Queues the write and returns.
+
+        :param object_name: name of the object
+        :type object_name: str
+        :param to_write: data to write
+        :type to_write: str
+        :param oncomplete: what to do when the write is safe and complete in memory
+            on all replicas
+        :type oncomplete: completion
+        :param onsafe:  what to do when the write is safe and complete on storage
+            on all replicas
+        :type onsafe: completion
+
+        :raises: :class:`Error`
+        :returns: completion object 
+        """
+        completion = self.__get_completion(oncomplete, onsafe)
+        ret = run_in_thread(self.librados.rados_aio_write_full,
+                            (self.io, c_char_p(object_name),
+                            completion.rados_comp, c_char_p(to_write),
+                            c_size_t(len(to_write))))
+        if ret < 0:
+            raise make_ex(ret, "error writing object %s" % object_name)
+        return completion
+
+    def aio_append(self, object_name, to_append, oncomplete=None, onsafe=None):
+        """
+        Asychronously append data to an object
+
+        Queues the write and returns.
+
+        :param object_name: name of the object
+        :type object_name: str
+        :param to_append: data to append
+        :type to_append: str
+        :param offset: byte offset in the object to begin writing at
+        :type offset: int
+        :param oncomplete: what to do when the write is safe and complete in memory
+            on all replicas
+        :type oncomplete: completion
+        :param onsafe:  what to do when the write is safe and complete on storage
+            on all replicas
+        :type onsafe: completion
+
+        :raises: :class:`Error`
+        :returns: completion object 
+        """
+        completion = self.__get_completion(oncomplete, onsafe)
+        ret = run_in_thread(self.librados.rados_aio_append,
+                            (self.io, c_char_p(object_name),
+                            completion.rados_comp, c_char_p(to_append),
+                            c_size_t(len(to_append))))
+        if ret < 0:
+            raise make_ex(ret, "error appending to object %s" % object_name)
+        return completion
+
+    def aio_flush(self):
+        """
+        Block until all pending writes in an io context are safe
+
+        :raises: :class:`Error`
+        """
+        ret = run_in_thread(self.librados.rados_aio_flush, (self.io,))
+        if ret < 0:
+            raise make_ex(ret, "error flushing")
+
+    def aio_read(self, object_name, length, offset, oncomplete):
+        """
+        Asychronously read data from an object
+
+        oncomplete will be called with the returned read value as
+        well as the completion:
+
+        oncomplete(completion, data_read)
+
+        :param object_name: name of the object to read from
+        :type object_name: str
+        :param length: the number of bytes to read
+        :type length: int
+        :param offset: byte offset in the object to begin reading from
+        :type offset: int
+        :param oncomplete: what to do when the read is complete
+        :type oncomplete: completion
+
+        :raises: :class:`Error`
+        :returns: completion object
+        """
+        buf = create_string_buffer(length)
+        def oncomplete_(completion_v):
+            return_value = completion_v.get_return_value()
+            return oncomplete(completion_v,
+                              ctypes.string_at(buf, return_value) if return_value >= 0 else None)
+
+        completion = self.__get_completion(oncomplete_, None)
+        ret = run_in_thread(self.librados.rados_aio_read,
+                            (self.io, c_char_p(object_name),
+                            completion.rados_comp, buf, c_size_t(length),
+                            c_uint64(offset)))
+        if ret < 0:
+            raise make_ex(ret, "error reading %s" % object_name)
+        return completion
+
+    def aio_remove(self, object_name, oncomplete=None, onsafe=None):
+        """
+        Asychronously remove an object
+
+        :param object_name: name of the object to remove
+        :type object_name: str
+        :param oncomplete: what to do when the remove is safe and complete in memory
+            on all replicas
+        :type oncomplete: completion
+        :param onsafe:  what to do when the remove is safe and complete on storage
+            on all replicas
+        :type onsafe: completion
+
+        :raises: :class:`Error`
+        :returns: completion object
+        """
+        completion = self.__get_completion(oncomplete, onsafe)
+        ret = run_in_thread(self.librados.rados_aio_remove,
+                            (self.io, c_char_p(object_name),
+                            completion.rados_comp))
+        if ret < 0:
+            raise make_ex(ret, "error removing %s" % object_name)
+        return completion
+
+    def require_ioctx_open(self):
+        """
+        Checks if the rados.Ioctx object state is 'open'
+
+        :raises: IoctxStateError
+        """
+        if self.state != "open":
+            raise IoctxStateError("The pool is %s" % self.state)
+
+    def change_auid(self, auid):
+        """
+        Attempt to change an io context's associated auid "owner."
+
+        Requires that you have write permission on both the current and new
+        auid.
+
+        :raises: :class:`Error`
+        """
+        self.require_ioctx_open()
+        ret = run_in_thread(self.librados.rados_ioctx_pool_set_auid,
+                            (self.io, ctypes.c_uint64(auid)))
+        if ret < 0:
+            raise make_ex(ret, "error changing auid of '%s' to %d" %\
+                (self.name, auid))
+
+    def set_locator_key(self, loc_key):
+        """
+        Set the key for mapping objects to pgs within an io context.
+
+        The key is used instead of the object name to determine which
+        placement groups an object is put in. This affects all subsequent
+        operations of the io context - until a different locator key is
+        set, all objects in this io context will be placed in the same pg.
+
+        :param loc_key: the key to use as the object locator, or NULL to discard
+            any previously set key
+        :type loc_key: str
+
+        :raises: :class:`TypeError`
+        """
+        self.require_ioctx_open()
+        if not isinstance(loc_key, str):
+            raise TypeError('loc_key must be a string')
+        run_in_thread(self.librados.rados_ioctx_locator_set_key,
+                     (self.io, c_char_p(loc_key)))
+        self.locator_key = loc_key
+
+    def get_locator_key(self):
+        """
+        Get the locator_key of context
+
+        :returns: locator_key
+        """
+        return self.locator_key
+
+    def close(self):
+        """
+        Close a rados.Ioctx object.
+
+        This just tells librados that you no longer need to use the io context.
+        It may not be freed immediately if there are pending asynchronous
+        requests on it, but you should not use an io context again after
+        calling this function on it.
+        """
+        if self.state == "open":
+            self.require_ioctx_open()
+            run_in_thread(self.librados.rados_ioctx_destroy, (self.io,))
+            self.state = "closed"
+
+    def write(self, key, data, offset=0):
+        """
+        Write data to an object synchronously
+
+        :param key: name of the object
+        :type key: str
+        :param data: data to write
+        :type data: str
+        :param offset: byte offset in the object to begin writing at
+        :type offset: int
+
+        :raises: :class:`TypeError`
+        :raises: :class:`LogicError`
+        :returns: int - number of bytes written 
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        if not isinstance(data, str):
+            raise TypeError('data must be a string')
+        length = len(data)
+        ret = run_in_thread(self.librados.rados_write,
+                            (self.io, c_char_p(key), c_char_p(data),
+                            c_size_t(length), c_uint64(offset)))
+        if ret == 0:
+            return ret
+        elif ret < 0:
+            raise make_ex(ret, "Ioctx.write(%s): failed to write %s" % \
+                (self.name, key))
+        else:
+            raise LogicError("Ioctx.write(%s): rados_write \
+returned %d, but should return zero on success." % (self.name, ret))
+
+    def write_full(self, key, data):
+        """
+        Write an entire object synchronously.
+
+        The object is filled with the provided data. If the object exists,
+        it is atomically truncated and then written.
+
+        :param key: name of the object
+        :type key: str
+        :param data: data to write
+        :type data: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: int - 0 on success
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        if not isinstance(data, str):
+            raise TypeError('data must be a string')
+        length = len(data)
+        ret = run_in_thread(self.librados.rados_write_full,
+                            (self.io, c_char_p(key), c_char_p(data),
+                            c_size_t(length)))
+        if ret == 0:
+            return ret
+        elif ret < 0:
+            raise make_ex(ret, "Ioctx.write_full(%s): failed to write %s" % \
+                (self.name, key))
+        else:
+            raise LogicError("Ioctx.write_full(%s): rados_write_full \
+returned %d, but should return zero on success." % (self.name, ret))
+
+    def append(self, key, data):
+        """
+        Append data to an object synchronously
+
+        :param key: name of the object
+        :type key: str
+        :param data: data to write
+        :type data: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`LogicError`
+        :returns: int - number of bytes written
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        if not isinstance(data, str):
+            raise TypeError('data must be a string')
+        length = len(data)
+        ret = run_in_thread(self.librados.rados_append,
+                            (self.io, c_char_p(key), c_char_p(data),
+                            c_size_t(length)))
+        if ret == 0:
+            return ret
+        elif ret < 0:
+            raise make_ex(ret, "Ioctx.append(%s): failed to append %s" % \
+                (self.name, key))
+        else:
+            raise LogicError("Ioctx.append(%s): rados_append \
+returned %d, but should return zero on success." % (self.name, ret))
+
+    def read(self, key, length=8192, offset=0):
+        """
+        Read data from an object synchronously
+
+        :param key: name of the object
+        :type key: str
+        :param length: the number of bytes to read (default=8192)
+        :type length: int 
+        :param offset: byte offset in the object to begin reading at
+        :type offset: int
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: str - data read from object
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        ret_buf = create_string_buffer(length)
+        ret = run_in_thread(self.librados.rados_read,
+                            (self.io, c_char_p(key), ret_buf, c_size_t(length),
+                            c_uint64(offset)))
+        if ret < 0:
+            raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key))
+        return ctypes.string_at(ret_buf, ret)
+
+    def get_stats(self):
+        """
+        Get pool usage statistics
+
+        :returns: dict - contains the following keys:
+
+            - ``num_bytes`` (int) - size of pool in bytes
+
+            - ``num_kb`` (int) - size of pool in kbytes
+
+            - ``num_objects`` (int) - number of objects in the pool
+
+            - ``num_object_clones`` (int) - number of object clones
+
+            - ``num_object_copies`` (int) - number of object copies
+
+            - ``num_objects_missing_on_primary`` (int) - number of objets
+                missing on primary
+
+            - ``num_objects_unfound`` (int) - number of unfound objects
+
+            - ``num_objects_degraded`` (int) - number of degraded objects
+
+            - ``num_rd`` (int) - bytes read
+
+            - ``num_rd_kb`` (int) - kbytes read
+
+            - ``num_wr`` (int) - bytes written
+
+            - ``num_wr_kb`` (int) - kbytes written
+        """
+        self.require_ioctx_open()
+        stats = rados_pool_stat_t()
+        ret = run_in_thread(self.librados.rados_ioctx_pool_stat,
+                            (self.io, byref(stats)))
+        if ret < 0:
+            raise make_ex(ret, "Ioctx.get_stats(%s): get_stats failed" % self.name)
+        return {'num_bytes': stats.num_bytes,
+                'num_kb': stats.num_kb,
+                'num_objects': stats.num_objects,
+                'num_object_clones': stats.num_object_clones,
+                'num_object_copies': stats.num_object_copies,
+                "num_objects_missing_on_primary": stats.num_objects_missing_on_primary,
+                "num_objects_unfound": stats.num_objects_unfound,
+                "num_objects_degraded": stats.num_objects_degraded,
+                "num_rd": stats.num_rd,
+                "num_rd_kb": stats.num_rd_kb,
+                "num_wr": stats.num_wr,
+                "num_wr_kb": stats.num_wr_kb }
+
+    def remove_object(self, key):
+        """
+        Delete an object
+
+        This does not delete any snapshots of the object.
+
+        :param key: the name of the object to delete
+        :type key: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: bool - True on success
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        ret = run_in_thread(self.librados.rados_remove,
+                            (self.io, c_char_p(key)))
+        if ret < 0:
+            raise make_ex(ret, "Failed to remove '%s'" % key)
+        return True
+
+    def trunc(self, key, size):
+        """
+        Resize an object
+
+        If this enlarges the object, the new area is logically filled with
+        zeroes. If this shrinks the object, the excess data is removed.
+
+        :param key: the name of the object to resize
+        :type key: str
+        :param size: the new size of the object in bytes
+        :type size: int
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: int - 0 on success, otherwise raises error
+        """
+        
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        ret = run_in_thread(self.librados.rados_trunc,
+                            (self.io, c_char_p(key), c_uint64(size)))
+        if ret < 0:
+            raise make_ex(ret, "Ioctx.trunc(%s): failed to truncate %s" % (self.name, key))
+        return ret
+
+    def stat(self, key):
+        """
+        Get object stats (size/mtime)
+        
+        :param key: the name of the object to get stats from
+        :type key: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: (size,timestamp)
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        psize = c_uint64()
+        pmtime = c_uint64()
+
+        ret = run_in_thread(self.librados.rados_stat,
+                            (self.io, c_char_p(key), pointer(psize),
+                            pointer(pmtime)))
+        if ret < 0:
+            raise make_ex(ret, "Failed to stat %r" % key)
+        return psize.value, time.localtime(pmtime.value)
+
+    def get_xattr(self, key, xattr_name):
+        """
+        Get the value of an extended attribute on an object.
+        
+        :param key: the name of the object to get xattr from
+        :type key: str
+        :param xattr_name: which extended attribute to read
+        :type xattr_name: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: str - value of the xattr
+        """
+        self.require_ioctx_open()
+        if not isinstance(xattr_name, str):
+            raise TypeError('xattr_name must be a string')
+        ret_length = 4096
+        while ret_length < 4096 * 1024 * 1024:
+            ret_buf = create_string_buffer(ret_length)
+            ret = run_in_thread(self.librados.rados_getxattr,
+                                (self.io, c_char_p(key), c_char_p(xattr_name),
+                                 ret_buf, c_size_t(ret_length)))
+            if (ret == -errno.ERANGE):
+                ret_length *= 2
+            elif ret < 0:
+                raise make_ex(ret, "Failed to get xattr %r" % xattr_name)
+            else:
+                break
+        return ctypes.string_at(ret_buf, ret)
+
+    def get_xattrs(self, oid):
+        """
+        Start iterating over xattrs on an object.
+        
+        :param oid: the name of the object to get xattrs from
+        :type key: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: XattrIterator
+        """
+        self.require_ioctx_open()
+        if not isinstance(oid, str):
+            raise TypeError('oid must be a string')
+        it = c_void_p(0)
+        ret = run_in_thread(self.librados.rados_getxattrs,
+                            (self.io, oid, byref(it)))
+        if ret != 0:
+            raise make_ex(ret, "Failed to get rados xattrs for object %r" % oid)
+        return XattrIterator(self, it, oid)
+
+    def set_xattr(self, key, xattr_name, xattr_value):
+        """
+        Set an extended attribute on an object.
+        
+        :param key: the name of the object to set xattr to
+        :type key: str
+        :param xattr_name: which extended attribute to set
+        :type xattr_name: str
+        :param xattr_value: the value of the  extended attribute
+        :type xattr_value: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: bool - True on success, otherwise raise an error
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        if not isinstance(xattr_name, str):
+            raise TypeError('xattr_name must be a string')
+        if not isinstance(xattr_value, str):
+            raise TypeError('xattr_value must be a string')
+        ret = run_in_thread(self.librados.rados_setxattr,
+                            (self.io, c_char_p(key), c_char_p(xattr_name),
+                            c_char_p(xattr_value), c_size_t(len(xattr_value))))
+        if ret < 0:
+            raise make_ex(ret, "Failed to set xattr %r" % xattr_name)
+        return True
+
+    def rm_xattr(self, key, xattr_name):
+        """
+        Removes an extended attribute on from an object.
+        
+        :param key: the name of the object to remove xattr from
+        :type key: str
+        :param xattr_name: which extended attribute to remove
+        :type xattr_name: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: bool - True on success, otherwise raise an error
+        """
+        self.require_ioctx_open()
+        if not isinstance(key, str):
+            raise TypeError('key must be a string')
+        if not isinstance(xattr_name, str):
+            raise TypeError('xattr_name must be a string')
+        ret = run_in_thread(self.librados.rados_rmxattr,
+                            (self.io, c_char_p(key), c_char_p(xattr_name)))
+        if ret < 0:
+            raise make_ex(ret, "Failed to delete key %r xattr %r" %
+                (key, xattr_name))
+        return True
+
+    def list_objects(self):
+        """ 
+        Get ObjectIterator on rados.Ioctx object.
+
+        :returns: ObjectIterator
+        """
+        self.require_ioctx_open()
+        return ObjectIterator(self)
+
+    def list_snaps(self):
+        """ 
+        Get SnapIterator on rados.Ioctx object.
+
+        :returns: SnapIterator
+        """
+        self.require_ioctx_open()
+        return SnapIterator(self)
+
+    def create_snap(self, snap_name):
+        """
+        Create a pool-wide snapshot
+
+        :param snap_name: the name of the snapshot
+        :type snap_name: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        """
+        self.require_ioctx_open()
+        if not isinstance(snap_name, str):
+            raise TypeError('snap_name must be a string')
+        ret = run_in_thread(self.librados.rados_ioctx_snap_create,
+                            (self.io, c_char_p(snap_name)))
+        if (ret != 0):
+            raise make_ex(ret, "Failed to create snap %s" % snap_name)
+
+    def remove_snap(self, snap_name):
+        """
+        Removes a pool-wide snapshot
+
+        :param snap_name: the name of the snapshot
+        :type snap_name: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        """
+        self.require_ioctx_open()
+        if not isinstance(snap_name, str):
+            raise TypeError('snap_name must be a string')
+        ret = run_in_thread(self.librados.rados_ioctx_snap_remove,
+                            (self.io, c_char_p(snap_name)))
+        if (ret != 0):
+            raise make_ex(ret, "Failed to remove snap %s" % snap_name)
+
+    def lookup_snap(self, snap_name):
+        """
+        Get the id of a pool snapshot
+
+        :param snap_name: the name of the snapshot to lookop
+        :type snap_name: str
+
+        :raises: :class:`TypeError`
+        :raises: :class:`Error`
+        :returns: Snap - on success
+        """
+        self.require_ioctx_open()
+        if not isinstance(snap_name, str):
+            raise TypeError('snap_name must be a string')
+        snap_id = c_uint64()
+        ret = run_in_thread(self.librados.rados_ioctx_snap_lookup,
+                           (self.io, c_char_p(snap_name), byref(snap_id)))
+        if (ret != 0):
+            raise make_ex(ret, "Failed to lookup snap %s" % snap_name)
+        return Snap(self, snap_name, snap_id)
+
+    def get_last_version(self):
+        """
+        Return the version of the last object read or written to.
+
+        This exposes the internal version number of the last object read or
+        written via this io context
+
+        :returns: version of the last object used
+        """
+        self.require_ioctx_open()
+        return run_in_thread(self.librados.rados_get_last_version, (self.io,))
+
+def set_object_locator(func):
+    def retfunc(self, *args, **kwargs):
+        if self.locator_key is not None:
+            old_locator = self.ioctx.get_locator_key()
+            self.ioctx.set_locator_key(self.locator_key)
+            retval = func(self, *args, **kwargs)
+            self.ioctx.set_locator_key(old_locator)
+            return retval
+        else:
+            return func(self, *args, **kwargs)
+    return retfunc
+
+class Object(object):
+    """Rados object wrapper, makes the object look like a file"""
+    def __init__(self, ioctx, key, locator_key=None):
+        self.key = key
+        self.ioctx = ioctx
+        self.offset = 0
+        self.state = "exists"
+        self.locator_key = locator_key
+
+    def __str__(self):
+        return "rados.Object(ioctx=%s,key=%s)" % (str(self.ioctx), self.key)
+
+    def require_object_exists(self):
+        if self.state != "exists":
+            raise ObjectStateError("The object is %s" % self.state)
+
+    @set_object_locator
+    def read(self, length = 1024*1024):
+        self.require_object_exists()
+        ret = self.ioctx.read(self.key, length, self.offset)
+        self.offset += len(ret)
+        return ret
+
+    @set_object_locator
+    def write(self, string_to_write):
+        self.require_object_exists()
+        ret = self.ioctx.write(self.key, string_to_write, self.offset)
+        self.offset += ret
+        return ret
+
+    @set_object_locator
+    def remove(self):
+        self.require_object_exists()
+        self.ioctx.remove_object(self.key)
+        self.state = "removed"
+
+    @set_object_locator
+    def stat(self):
+        self.require_object_exists()
+        return self.ioctx.stat(self.key)
+
+    def seek(self, position):
+        self.require_object_exists()
+        self.offset = position
+
+    @set_object_locator
+    def get_xattr(self, xattr_name):
+        self.require_object_exists()
+        return self.ioctx.get_xattr(self.key, xattr_name)
+
+    @set_object_locator
+    def get_xattrs(self):
+        self.require_object_exists()
+        return self.ioctx.get_xattrs(self.key)
+
+    @set_object_locator
+    def set_xattr(self, xattr_name, xattr_value):
+        self.require_object_exists()
+        return self.ioctx.set_xattr(self.key, xattr_name, xattr_value)
+
+    @set_object_locator
+    def rm_xattr(self, xattr_name):
+        self.require_object_exists()
+        return self.ioctx.rm_xattr(self.key, xattr_name)
+
+MONITOR_LEVELS = [
+   "debug",
+   "info",
+   "warn", "warning",
+   "err", "error", 
+   "sec",
+   ]
+
+
+class MonitorLog(object):
+    """
+    For watching cluster log messages.  Instantiate an object and keep
+    it around while callback is periodically called.  Construct with
+    'level' to monitor 'level' messages (one of MONITOR_LEVELS).
+    arg will be passed to the callback.
+
+    callback will be called with:
+        arg (given to __init__)
+        line (the full line, including timestamp, who, level, msg)
+        who (which entity issued the log message)
+        timestamp_sec (sec of a struct timespec)
+        timestamp_nsec (sec of a struct timespec)
+        seq (sequence number)
+        level (string representing the level of the log message)
+        msg (the message itself)
+    callback's return value is ignored
+    """
+
+    def monitor_log_callback(self, arg, line, who, sec, nsec, seq, level, msg):
+        """
+        Local callback wrapper, in case we decide to do something
+        """
+        self.callback(arg, line, who, sec, nsec, seq, level, msg)
+        return 0
+
+    def __init__(self, cluster, level, callback, arg):
+        if level not in MONITOR_LEVELS:
+            raise LogicError("invalid monitor level " + level)
+        if not callable(callback):
+            raise LogicError("callback must be a callable function")
+        self.level = level
+        self.callback = callback
+        self.arg = arg
+        callback_factory = CFUNCTYPE(c_int,    # return type (really void)
+                                     c_void_p, # arg
+                                     c_char_p, # line
+                                     c_char_p, # who
+                                     c_uint64, # timestamp_sec
+                                     c_uint64, # timestamp_nsec
+                                     c_ulong,  # seq
+                                     c_char_p, # level
+                                     c_char_p) # msg
+        self.internal_callback = callback_factory(self.monitor_log_callback)
+
+        r = run_in_thread(cluster.librados.rados_monitor_log,
+                          (cluster.cluster, level, self.internal_callback, arg))
+        if r:
+            raise make_ex(r, 'error calling rados_monitor_log')
diff --git a/ceph/src/pybind/rbd.py b/ceph/src/pybind/rbd.py
new file mode 100644
index 00000000..ab093ce3
--- /dev/null
+++ b/ceph/src/pybind/rbd.py
@@ -0,0 +1,962 @@
+"""
+This module is a thin wrapper around librbd.
+
+It currently provides all the synchronous methods of librbd that do
+not use callbacks.
+
+Error codes from librbd are turned into exceptions that subclass
+:class:`Error`. Almost all methods may raise :class:`Error`
+(the base class of all rbd exceptions), :class:`PermissionError`
+and :class:`IOError`, in addition to those documented for the
+method.
+
+A number of methods have string arguments, which must not be unicode
+to interact correctly with librbd. If unicode is passed to these
+methods, a :class:`TypeError` will be raised.
+"""
+# Copyright 2011 Josh Durgin
+from ctypes import CDLL, c_char, c_char_p, c_size_t, c_void_p, c_int, \
+    create_string_buffer, byref, Structure, c_uint64, c_int64, c_uint8, \
+    CFUNCTYPE
+from ctypes.util import find_library
+import ctypes
+import errno
+
+ANONYMOUS_AUID = 0xffffffffffffffff
+ADMIN_AUID = 0
+
+RBD_FEATURE_LAYERING = 1
+RBD_FEATURE_STRIPINGV2 = 2
+
+class Error(Exception):
+    pass
+
+class PermissionError(Error):
+    pass
+
+class ImageNotFound(Error):
+    pass
+
+class ImageExists(Error):
+    pass
+
+class IOError(Error):
+    pass
+
+class NoSpace(Error):
+    pass
+
+class IncompleteWriteError(Error):
+    pass
+
+class InvalidArgument(Error):
+    pass
+
+class LogicError(Error):
+    pass
+
+class ReadOnlyImage(Error):
+    pass
+
+class ImageBusy(Error):
+    pass
+
+class ImageHasSnapshots(Error):
+    pass
+
+class FunctionNotSupported(Error):
+    pass
+
+class ArgumentOutOfRange(Error):
+    pass
+
+class ConnectionShutdown(Error):
+    pass
+
+class Timeout(Error):
+    pass
+
+def make_ex(ret, msg):
+    """
+    Translate a librbd return code into an exception.
+
+    :param ret: the return code
+    :type ret: int
+    :param msg: the error message to use
+    :type msg: str
+    :returns: a subclass of :class:`Error`
+    """
+    errors = {
+        errno.EPERM     : PermissionError,
+        errno.ENOENT    : ImageNotFound,
+        errno.EIO       : IOError,
+        errno.ENOSPC    : NoSpace,
+        errno.EEXIST    : ImageExists,
+        errno.EINVAL    : InvalidArgument,
+        errno.EROFS     : ReadOnlyImage,
+        errno.EBUSY     : ImageBusy,
+        errno.ENOTEMPTY : ImageHasSnapshots,
+        errno.ENOSYS    : FunctionNotSupported,
+        errno.EDOM      : ArgumentOutOfRange,
+        errno.ESHUTDOWN : ConnectionShutdown,
+        errno.ETIMEDOUT : Timeout,
+        }
+    ret = abs(ret)
+    if ret in errors:
+        return errors[ret](msg)
+    else:
+        return Error(msg + (": error code %d" % ret))
+
+class rbd_image_info_t(Structure):
+    _fields_ = [("size", c_uint64),
+                ("obj_size", c_uint64),
+                ("num_objs", c_uint64),
+                ("order", c_int),
+                ("block_name_prefix", c_char * 24),
+                ("parent_pool", c_int64),
+                ("parent_name", c_char * 96)]
+
+class rbd_snap_info_t(Structure):
+    _fields_ = [("id", c_uint64),
+                ("size", c_uint64),
+                ("name", c_char_p)]
+
+def load_librbd():
+    """
+    Load the librbd shared library.
+    """
+    librbd_path = find_library('rbd')
+    if librbd_path:
+        return CDLL(librbd_path)
+
+    # try harder, find_library() doesn't search LD_LIBRARY_PATH
+    # in addition, it doesn't seem work on centos 6.4 (see e46d2ca067b5)
+    try:
+        return CDLL('librbd.so.1')
+    except OSError as e:
+        raise EnvironmentError("Unable to load librbd: %s" % e)
+
+class RBD(object):
+    """
+    This class wraps librbd CRUD functions.
+    """
+    def __init__(self):
+        self.librbd = load_librbd()
+
+    def version(self):
+        """
+        Get the version number of the ``librbd`` C library.
+
+        :returns: a tuple of ``(major, minor, extra)`` components of the
+                  librbd version
+        """
+        major = c_int(0)
+        minor = c_int(0)
+        extra = c_int(0)
+        self.librbd.rbd_version(byref(major), byref(minor), byref(extra))
+        return (major.value, minor.value, extra.value)
+
+    def create(self, ioctx, name, size, order=None, old_format=True,
+               features=0, stripe_unit=0, stripe_count=0):
+        """
+        Create an rbd image.
+
+        :param ioctx: the context in which to create the image
+        :type ioctx: :class:`rados.Ioctx`
+        :param name: what the image is called
+        :type name: str
+        :param size: how big the image is in bytes
+        :type size: int
+        :param order: the image is split into (2**order) byte objects
+        :type order: int
+        :param old_format: whether to create an old-style image that
+                           is accessible by old clients, but can't
+                           use more advanced features like layering.
+        :type old_format: bool
+        :param features: bitmask of features to enable
+        :type features: int
+        :param stripe_unit: stripe unit in bytes (default 0 for object size)
+        :type stripe_unit: int
+        :param stripe_count: objects to stripe over before looping
+        :type stripe_count: int
+        :raises: :class:`ImageExists`
+        :raises: :class:`TypeError`
+        :raises: :class:`InvalidArgument`
+        :raises: :class:`FunctionNotSupported`
+        """
+        if order is None:
+            order = 0
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        if old_format:
+            if features != 0 or stripe_unit != 0 or stripe_count != 0:
+                raise InvalidArgument('format 1 images do not support feature'
+                                      ' masks or non-default striping')
+            ret = self.librbd.rbd_create(ioctx.io, c_char_p(name),
+                                         c_uint64(size),
+                                         byref(c_int(order)))
+        else:
+            if not hasattr(self.librbd, 'rbd_create2'):
+                raise FunctionNotSupported('installed version of librbd does'
+                                           ' not support format 2 images')
+            has_create3 = hasattr(self.librbd, 'rbd_create3')
+            if (stripe_unit != 0 or stripe_count != 0) and not has_create3:
+                raise FunctionNotSupported('installed version of librbd does'
+                                           ' not support stripe unit or count')
+            if has_create3:
+                ret = self.librbd.rbd_create3(ioctx.io, c_char_p(name),
+                                              c_uint64(size),
+                                              c_uint64(features),
+                                              byref(c_int(order)),
+                                              c_uint64(stripe_unit),
+                                              c_uint64(stripe_count))
+            else:
+                ret = self.librbd.rbd_create2(ioctx.io, c_char_p(name),
+                                              c_uint64(size),
+                                              c_uint64(features),
+                                              byref(c_int(order)))
+        if ret < 0:
+            raise make_ex(ret, 'error creating image')
+
+    def clone(self, p_ioctx, p_name, p_snapname, c_ioctx, c_name,
+              features=0, order=None):
+        """
+        Clone a parent rbd snapshot into a COW sparse child.
+
+        :param p_ioctx: the parent context that represents the parent snap
+        :type ioctx: :class:`rados.Ioctx`
+        :param p_name: the parent image name
+        :type name: str
+        :param p_snapname: the parent image snapshot name
+        :type name: str
+        :param c_ioctx: the child context that represents the new clone
+        :type ioctx: :class:`rados.Ioctx`
+        :param c_name: the clone (child) name
+        :type name: str
+        :param features: bitmask of features to enable; if set, must include layering
+        :type features: int
+        :param order: the image is split into (2**order) byte objects
+        :type order: int
+        :raises: :class:`TypeError`
+        :raises: :class:`InvalidArgument`
+        :raises: :class:`ImageExists`
+        :raises: :class:`FunctionNotSupported`
+        :raises: :class:`ArgumentOutOfRange`
+        """
+        if order is None:
+            order = 0
+        if not isinstance(p_snapname, str) or not isinstance(p_name, str):
+            raise TypeError('parent name and snapname must be strings')
+        if not isinstance(c_name, str):
+            raise TypeError('child name must be a string')
+
+        ret = self.librbd.rbd_clone(p_ioctx.io, c_char_p(p_name),
+                                    c_char_p(p_snapname),
+                                    c_ioctx.io, c_char_p(c_name),
+                                          c_uint64(features),
+                                          byref(c_int(order)))
+        if ret < 0:
+            raise make_ex(ret, 'error creating clone')
+
+    def list(self, ioctx):
+        """
+        List image names.
+
+        :param ioctx: determines which RADOS pool is read
+        :type ioctx: :class:`rados.Ioctx`
+        :returns: list -- a list of image names
+        """
+        size = c_size_t(512)
+        while True:
+            c_names = create_string_buffer(size.value)
+            ret = self.librbd.rbd_list(ioctx.io, byref(c_names), byref(size))
+            if ret >= 0:
+                break
+            elif ret != -errno.ERANGE:
+                raise make_ex(ret, 'error listing images')
+        return filter(lambda name: name != '', c_names.raw.split('\0'))
+
+    def remove(self, ioctx, name):
+        """
+        Delete an RBD image. This may take a long time, since it does
+        not return until every object that comprises the image has
+        been deleted. Note that all snapshots must be deleted before
+        the image can be removed. If there are snapshots left,
+        :class:`ImageHasSnapshots` is raised. If the image is still
+        open, or the watch from a crashed client has not expired,
+        :class:`ImageBusy` is raised.
+
+        :param ioctx: determines which RADOS pool the image is in
+        :type ioctx: :class:`rados.Ioctx`
+        :param name: the name of the image to remove
+        :type name: str
+        :raises: :class:`ImageNotFound`, :class:`ImageBusy`,
+                 :class:`ImageHasSnapshots`
+        """
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        ret = self.librbd.rbd_remove(ioctx.io, c_char_p(name))
+        if ret != 0:
+            raise make_ex(ret, 'error removing image')
+
+    def rename(self, ioctx, src, dest):
+        """
+        Rename an RBD image.
+
+        :param ioctx: determines which RADOS pool the image is in
+        :type ioctx: :class:`rados.Ioctx`
+        :param src: the current name of the image
+        :type src: str
+        :param dest: the new name of the image
+        :type dest: str
+        :raises: :class:`ImageNotFound`, :class:`ImageExists`
+        """
+        if not isinstance(src, str) or not isinstance(dest, str):
+            raise TypeError('src and dest must be strings')
+        ret = self.librbd.rbd_rename(ioctx.io, c_char_p(src), c_char_p(dest))
+        if ret != 0:
+            raise make_ex(ret, 'error renaming image')
+
+class Image(object):
+    """
+    This class represents an RBD image. It is used to perform I/O on
+    the image and interact with snapshots.
+
+    **Note**: Any method of this class may raise :class:`ImageNotFound`
+    if the image has been deleted.
+    """
+
+    def __init__(self, ioctx, name, snapshot=None, read_only=False):
+        """
+        Open the image at the given snapshot.
+        If a snapshot is specified, the image will be read-only, unless
+        :func:`Image.set_snap` is called later.
+
+        If read-only mode is used, metadata for the :class:`Image`
+        object (such as which snapshots exist) may become obsolete. See
+        the C api for more details.
+
+        To clean up from opening the image, :func:`Image.close` should
+        be called.  For ease of use, this is done automatically when
+        an :class:`Image` is used as a context manager (see :pep:`343`).
+
+        :param ioctx: determines which RADOS pool the image is in
+        :type ioctx: :class:`rados.Ioctx`
+        :param name: the name of the image
+        :type name: str
+        :param snapshot: which snapshot to read from
+        :type snaphshot: str
+        :param read_only: whether to open the image in read-only mode
+        :type read_only: bool
+        """
+        self.closed = True
+        self.librbd = load_librbd()
+        self.image = c_void_p()
+        self.name = name
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        if snapshot is not None and not isinstance(snapshot, str):
+            raise TypeError('snapshot must be a string or None')
+        if read_only:
+            if not hasattr(self.librbd, 'rbd_open_read_only'):
+                raise FunctionNotSupported('installed version of librbd does '
+                                           'not support open in read-only mode')
+            ret = self.librbd.rbd_open_read_only(ioctx.io, c_char_p(name),
+                                                 byref(self.image),
+                                                 c_char_p(snapshot))
+        else:
+            ret = self.librbd.rbd_open(ioctx.io, c_char_p(name),
+                                       byref(self.image), c_char_p(snapshot))
+        if ret != 0:
+            raise make_ex(ret, 'error opening image %s at snapshot %s' % (name, snapshot))
+        self.closed = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        """
+        Closes the image. See :func:`close`
+        """
+        self.close()
+        return False
+
+    def close(self):
+        """
+        Release the resources used by this image object.
+
+        After this is called, this object should not be used.
+        """
+        if not self.closed:
+            self.closed = True
+            self.librbd.rbd_close(self.image)
+
+    def __del__(self):
+        self.close()
+
+    def __str__(self):
+        s = "rbd.Image(" + dict.__repr__(self.__dict__) + ")"
+        return s
+
+    def resize(self, size):
+        """
+        Change the size of the image.
+
+        :param size: the new size of the image
+        :type size: int
+        """
+        ret = self.librbd.rbd_resize(self.image, c_uint64(size))
+        if ret < 0:
+            raise make_ex(ret, 'error resizing image %s' % (self.name,))
+
+    def stat(self):
+        """
+        Get information about the image. Currently parent pool and
+        parent name are always -1 and ''.
+
+        :returns: dict - contains the following keys:
+
+            * ``size`` (int) - the size of the image in bytes
+
+            * ``obj_size`` (int) - the size of each object that comprises the
+              image
+
+            * ``num_objs`` (int) - the number of objects in the image
+
+            * ``order`` (int) - log_2(object_size)
+
+            * ``block_name_prefix`` (str) - the prefix of the RADOS objects used
+              to store the image
+
+            * ``parent_pool`` (int) - deprecated
+
+            * ``parent_name``  (str) - deprecated
+
+            See also :meth:`format` and :meth:`features`.
+
+        """
+        info = rbd_image_info_t()
+        ret = self.librbd.rbd_stat(self.image, byref(info), ctypes.sizeof(info))
+        if ret != 0:
+            raise make_ex(ret, 'error getting info for image %s' % (self.name,))
+        return {
+            'size'              : info.size,
+            'obj_size'          : info.obj_size,
+            'num_objs'          : info.num_objs,
+            'order'             : info.order,
+            'block_name_prefix' : info.block_name_prefix,
+            'parent_pool'       : info.parent_pool,
+            'parent_name'       : info.parent_name
+            }
+
+    def parent_info(self):
+        ret = -errno.ERANGE
+        size = 8
+        while ret == -errno.ERANGE and size <= 4096:
+            pool = create_string_buffer(size)
+            name = create_string_buffer(size)
+            snapname = create_string_buffer(size)
+            ret = self.librbd.rbd_get_parent_info(self.image, pool, len(pool), 
+                name, len(name), snapname, len(snapname))
+            if ret == -errno.ERANGE:
+                size *= 2
+
+        if (ret != 0):
+            raise make_ex(ret, 'error getting parent info for image %s' % (self.name,))
+        return (pool.value, name.value, snapname.value)
+
+    def old_format(self):
+        old = c_uint8()
+        ret = self.librbd.rbd_get_old_format(self.image, byref(old))
+        if (ret != 0):
+            raise make_ex(ret, 'error getting old_format for image' % (self.name))
+        return old.value != 0
+
+    def size(self):
+        """
+        Get the size of the image. If open to a snapshot, returns the
+        size of that snapshot.
+
+        :returns: the size of the image in bytes
+        """
+        image_size = c_uint64()
+        ret = self.librbd.rbd_get_size(self.image, byref(image_size))
+        if (ret != 0):
+            raise make_ex(ret, 'error getting size for image' % (self.name))
+        return image_size.value
+
+    def features(self):
+        features = c_uint64()
+        ret = self.librbd.rbd_get_features(self.image, byref(features))
+        if (ret != 0):
+            raise make_ex(ret, 'error getting features for image' % (self.name))
+        return features.value
+
+    def overlap(self):
+        overlap = c_uint64()
+        ret = self.librbd.rbd_get_overlap(self.image, byref(overlap))
+        if (ret != 0):
+            raise make_ex(ret, 'error getting overlap for image' % (self.name))
+        return overlap.value
+
+    def copy(self, dest_ioctx, dest_name):
+        """
+        Copy the image to another location.
+
+        :param dest_ioctx: determines which pool to copy into
+        :type dest_ioctx: :class:`rados.Ioctx`
+        :param dest_name: the name of the copy
+        :type dest_name: str
+        :raises: :class:`ImageExists`
+        """
+        if not isinstance(dest_name, str):
+            raise TypeError('dest_name must be a string')
+        ret = self.librbd.rbd_copy(self.image, dest_ioctx.io, c_char_p(dest_name))
+        if ret < 0:
+            raise make_ex(ret, 'error copying image %s to %s' % (self.name, dest_name))
+
+    def list_snaps(self):
+        """
+        Iterate over the snapshots of an image.
+
+        :returns: :class:`SnapIterator`
+        """
+        return SnapIterator(self)
+
+    def create_snap(self, name):
+        """
+        Create a snapshot of the image.
+
+        :param name: the name of the snapshot
+        :type name: str
+        :raises: :class:`ImageExists`
+        """
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        ret = self.librbd.rbd_snap_create(self.image, c_char_p(name))
+        if ret != 0:
+            raise make_ex(ret, 'error creating snapshot %s from %s' % (name, self.name))
+
+    def remove_snap(self, name):
+        """
+        Delete a snapshot of the image.
+
+        :param name: the name of the snapshot
+        :type name: str
+        :raises: :class:`IOError`, :class:`ImageBusy`
+        """
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        ret = self.librbd.rbd_snap_remove(self.image, c_char_p(name))
+        if ret != 0:
+            raise make_ex(ret, 'error removing snapshot %s from %s' % (name, self.name))
+
+    def rollback_to_snap(self, name):
+        """
+        Revert the image to its contents at a snapshot. This is a
+        potentially expensive operation, since it rolls back each
+        object individually.
+
+        :param name: the snapshot to rollback to
+        :type name: str
+        :raises: :class:`IOError`
+        """
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        ret = self.librbd.rbd_snap_rollback(self.image, c_char_p(name))
+        if ret != 0:
+            raise make_ex(ret, 'error rolling back image %s to snapshot %s' % (self.name, name))
+
+    def protect_snap(self, name):
+        """
+        Mark a snapshot as protected. This means it can't be deleted
+        until it is unprotected.
+
+        :param name: the snapshot to protect
+        :type name: str
+        :raises: :class:`IOError`, :class:`ImageNotFound`
+        """
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        ret = self.librbd.rbd_snap_protect(self.image, c_char_p(name))
+        if ret != 0:
+            raise make_ex(ret, 'error protecting snapshot %s@%s' % (self.name, name))
+
+    def unprotect_snap(self, name):
+        """
+        Mark a snapshot unprotected. This allows it to be deleted if
+        it was protected.
+
+        :param name: the snapshot to unprotect
+        :type name: str
+        :raises: :class:`IOError`, :class:`ImageNotFound`
+        """
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        ret = self.librbd.rbd_snap_unprotect(self.image, c_char_p(name))
+        if ret != 0:
+            raise make_ex(ret, 'error unprotecting snapshot %s@%s' % (self.name, name))
+
+    def is_protected_snap(self, name):
+        """
+        Find out whether a snapshot is protected from deletion.
+
+        :param name: the snapshot to check
+        :type name: str
+        :returns: bool - whether the snapshot is protected
+        :raises: :class:`IOError`, :class:`ImageNotFound`
+        """
+        if not isinstance(name, str):
+            raise TypeError('name must be a string')
+        is_protected = c_int()
+        ret = self.librbd.rbd_snap_is_protected(self.image, c_char_p(name),
+                                                byref(is_protected))
+        if ret != 0:
+            raise make_ex(ret, 'error checking if snapshot %s@%s is protected' % (self.name, name))
+        return is_protected.value == 1
+
+    def set_snap(self, name):
+        """
+        Set the snapshot to read from. Writes will raise ReadOnlyImage
+        while a snapshot is set. Pass None to unset the snapshot
+        (reads come from the current image) , and allow writing again.
+
+        :param name: the snapshot to read from, or None to unset the snapshot
+        :type name: str or None
+        """
+        if name is not None and not isinstance(name, str):
+            raise TypeError('name must be a string')
+        ret = self.librbd.rbd_snap_set(self.image, c_char_p(name))
+        if ret != 0:
+            raise make_ex(ret, 'error setting image %s to snapshot %s' % (self.name, name))
+
+    def read(self, offset, length):
+        """
+        Read data from the image. Raises :class:`InvalidArgument` if
+        part of the range specified is outside the image.
+
+        :param offset: the offset to start reading at
+        :type offset: int
+        :param length: how many bytes to read
+        :type length: int
+        :returns: str - the data read
+        :raises: :class:`InvalidArgument`, :class:`IOError`
+        """
+        ret_buf = create_string_buffer(length)
+        ret = self.librbd.rbd_read(self.image, c_uint64(offset),
+                                   c_size_t(length), byref(ret_buf))
+        if ret < 0:
+            raise make_ex(ret, 'error reading %s %ld~%ld' % (self.image, offset, length))
+        return ctypes.string_at(ret_buf, ret)
+
+    def diff_iterate(self, offset, length, from_snapshot, iterate_cb):
+        """
+        Iterate over the changed extents of an image.
+
+        This will call iterate_cb with three arguments:
+
+        (offset, length, exists)
+
+        where the changed extent starts at offset bytes, continues for
+        length bytes, and is full of data (if exists is True) or zeroes
+        (if exists is False).
+
+        If from_snapshot is None, it is interpreted as the beginning
+        of time and this generates all allocated extents.
+
+        The end version is whatever is currently selected (via set_snap)
+        for the image.
+
+        Raises :class:`InvalidArgument` if from_snapshot is after
+        the currently set snapshot.
+
+        Raises :class:`ImageNotFound` if from_snapshot is not the name
+        of a snapshot of the image.
+
+        :param offset: start offset in bytes
+        :type offset: int
+        :param length: size of region to report on, in bytes
+        :type length: int
+        :param from_snapshot: starting snapshot name, or None
+        :type from_snapshot: str or None
+        :param iterate_cb: function to call for each extent
+        :type iterate_cb: function acception arguments for offset,
+                           length, and exists
+        :raises: :class:`InvalidArgument`, :class:`IOError`,
+                 :class:`ImageNotFound`
+        """
+        if from_snapshot is not None and not isinstance(from_snapshot, str):
+            raise TypeError('client must be a string')
+
+        RBD_DIFF_CB = CFUNCTYPE(c_int, c_uint64, c_size_t, c_int, c_void_p)
+        cb_holder = DiffIterateCB(iterate_cb)
+        cb = RBD_DIFF_CB(cb_holder.callback)
+        ret = self.librbd.rbd_diff_iterate(self.image,
+                                           c_char_p(from_snapshot),
+                                           c_uint64(offset),
+                                           c_uint64(length),
+                                           cb,
+                                           c_void_p(None))
+        if ret < 0:
+            msg = 'error generating diff from snapshot %s' % from_snapshot
+            raise make_ex(ret, msg)
+
+    def write(self, data, offset):
+        """
+        Write data to the image. Raises :class:`InvalidArgument` if
+        part of the write would fall outside the image.
+
+        :param data: the data to be written
+        :type data: str
+        :param offset: where to start writing data
+        :type offset: int
+        :returns: int - the number of bytes written
+        :raises: :class:`IncompleteWriteError`, :class:`LogicError`,
+                 :class:`InvalidArgument`, :class:`IOError`
+        """
+        if not isinstance(data, str):
+            raise TypeError('data must be a string')
+        length = len(data)
+        ret = self.librbd.rbd_write(self.image, c_uint64(offset),
+                                    c_size_t(length), c_char_p(data))
+        if ret == length:
+            return ret
+        elif ret < 0:
+            raise make_ex(ret, "error writing to %s" % (self.name,))
+        elif ret < length:
+            raise IncompleteWriteError("Wrote only %ld out of %ld bytes" % (ret, length))
+        else:
+            raise LogicError("logic error: rbd_write(%s) \
+returned %d, but %d was the maximum number of bytes it could have \
+written." % (self.name, ret, length))
+
+    def discard(self, offset, length):
+        """
+        Trim the range from the image. It will be logically filled
+        with zeroes.
+        """
+        ret = self.librbd.rbd_discard(self.image,
+                                      c_uint64(offset),
+                                      c_uint64(length))
+        if ret < 0:
+            msg = 'error discarding region %d~%d' % (offset, length)
+            raise make_ex(ret, msg)
+
+    def flush(self):
+        """
+        Block until all writes are fully flushed if caching is enabled.
+        """
+        ret = self.librbd.rbd_flush(self.image)
+        if ret < 0:
+            raise make_ex(ret, 'error flushing image')
+
+    def invalidate_cache(self):
+        """
+        Drop any cached data for the image.
+        """
+        ret = self.librbd.rbd_invalidate_cache(self.image)
+        if ret < 0:
+            raise make_ex(ret, 'error invalidating cache')
+
+    def stripe_unit(self):
+        """
+        Returns the stripe unit used for the image.
+        """
+        stripe_unit = c_uint64()
+        ret = self.librbd.rbd_get_stripe_unit(self.image, byref(stripe_unit))
+        if ret != 0:
+            raise make_ex(ret, 'error getting stripe unit for image' % (self.name))
+        return stripe_unit.value
+
+    def stripe_count(self):
+        """
+        Returns the stripe count used for the image.
+        """
+        stripe_count = c_uint64()
+        ret = self.librbd.rbd_get_stripe_count(self.image, byref(stripe_count))
+        if ret != 0:
+            raise make_ex(ret, 'error getting stripe count for image' % (self.name))
+        return stripe_count.value
+
+    def flatten(self):
+        """
+        Flatten clone image (copy all blocks from parent to child)
+        """
+        ret = self.librbd.rbd_flatten(self.image)
+        if (ret < 0):
+            raise make_ex(ret, "error flattening %s" % self.name)
+
+    def list_children(self):
+        """
+        List children of the currently set snapshot (set via set_snap()).
+
+        :returns: list - a list of (pool name, image name) tuples
+        """
+        pools_size = c_size_t(512)
+        images_size = c_size_t(512)
+        while True:
+            c_pools = create_string_buffer(pools_size.value)
+            c_images = create_string_buffer(images_size.value)
+            ret = self.librbd.rbd_list_children(self.image,
+                                                byref(c_pools),
+                                                byref(pools_size),
+                                                byref(c_images),
+                                                byref(images_size))
+            if ret >= 0:
+                break
+            elif ret != -errno.ERANGE:
+                raise make_ex(ret, 'error listing images')
+        if ret == 0:
+            return []
+        pools = c_pools.raw[:pools_size.value - 1].split('\0')
+        images = c_images.raw[:images_size.value - 1].split('\0')
+        return zip(pools, images)
+
+    def list_lockers(self):
+        """
+        List clients that have locked the image and information
+        about the lock.
+
+        :returns: dict - contains the following keys:
+
+                  * ``tag`` - the tag associated with the lock (every
+                    additional locker must use the same tag)
+                  * ``exclusive`` - boolean indicating whether the
+                     lock is exclusive or shared
+                  * ``lockers`` - a list of (client, cookie, address)
+                    tuples
+        """
+        clients_size = c_size_t(512)
+        cookies_size = c_size_t(512)
+        addrs_size = c_size_t(512)
+        tag_size = c_size_t(512)
+        exclusive = c_int(0)
+
+        while True:
+            c_clients = create_string_buffer(clients_size.value)
+            c_cookies = create_string_buffer(cookies_size.value)
+            c_addrs = create_string_buffer(addrs_size.value)
+            c_tag = create_string_buffer(tag_size.value)
+            ret = self.librbd.rbd_list_lockers(self.image,
+                                               byref(exclusive),
+                                               byref(c_tag),
+                                               byref(tag_size),
+                                               byref(c_clients),
+                                               byref(clients_size),
+                                               byref(c_cookies),
+                                               byref(cookies_size),
+                                               byref(c_addrs),
+                                               byref(addrs_size))
+            if ret >= 0:
+                break
+            elif ret != -errno.ERANGE:
+                raise make_ex(ret, 'error listing images')
+        if ret == 0:
+            return []
+        clients = c_clients.raw[:clients_size.value - 1].split('\0')
+        cookies = c_cookies.raw[:cookies_size.value - 1].split('\0')
+        addrs = c_addrs.raw[:addrs_size.value - 1].split('\0')
+        return {
+            'tag'       : c_tag.value,
+            'exclusive' : exclusive.value == 1,
+            'lockers'   : zip(clients, cookies, addrs),
+            }
+
+    def lock_exclusive(self, cookie):
+        """
+        Take an exclusive lock on the image.
+
+        :raises: :class:`ImageBusy` if a different client or cookie locked it
+                 :class:`ImageExists` if the same client and cookie locked it
+        """
+        if not isinstance(cookie, str):
+            raise TypeError('cookie must be a string')
+        ret = self.librbd.rbd_lock_exclusive(self.image, c_char_p(cookie))
+        if ret < 0:
+            raise make_ex(ret, 'error acquiring exclusive lock on image')
+
+    def lock_shared(self, cookie, tag):
+        """
+        Take a shared lock on the image. The tag must match
+        that of the existing lockers, if any.
+
+        :raises: :class:`ImageBusy` if a different client or cookie locked it
+                 :class:`ImageExists` if the same client and cookie locked it
+        """
+        if not isinstance(cookie, str):
+            raise TypeError('cookie must be a string')
+        if not isinstance(tag, str):
+            raise TypeError('tag must be a string')
+        ret = self.librbd.rbd_lock_shared(self.image, c_char_p(cookie),
+                                          c_char_p(tag))
+        if ret < 0:
+            raise make_ex(ret, 'error acquiring shared lock on image')
+
+    def unlock(self, cookie):
+        """
+        Release a lock on the image that was locked by this rados client.
+        """
+        if not isinstance(cookie, str):
+            raise TypeError('cookie must be a string')
+        ret = self.librbd.rbd_unlock(self.image, c_char_p(cookie))
+        if ret < 0:
+            raise make_ex(ret, 'error unlocking image')
+
+    def break_lock(self, client, cookie):
+        """
+        Release a lock held by another rados client.
+        """
+        if not isinstance(client, str):
+            raise TypeError('client must be a string')
+        if not isinstance(cookie, str):
+            raise TypeError('cookie must be a string')
+        ret = self.librbd.rbd_break_lock(self.image, c_char_p(client),
+                                         c_char_p(cookie))
+        if ret < 0:
+            raise make_ex(ret, 'error unlocking image')
+
+class DiffIterateCB(object):
+    def __init__(self, cb):
+        self.cb = cb
+
+    def callback(self, offset, length, exists, unused):
+        self.cb(offset, length, exists == 1)
+        return 0
+
+class SnapIterator(object):
+    """
+    Iterator over snapshot info for an image.
+
+    Yields a dictionary containing information about a snapshot.
+
+    Keys are:
+
+    * ``id`` (int) - numeric identifier of the snapshot
+
+    * ``size`` (int) - size of the image at the time of snapshot (in bytes)
+
+    * ``name`` (str) - name of the snapshot
+    """
+    def __init__(self, image):
+        self.librbd = image.librbd
+        num_snaps = c_int(10)
+        while True:
+            self.snaps = (rbd_snap_info_t * num_snaps.value)()
+            ret = self.librbd.rbd_snap_list(image.image, byref(self.snaps),
+                                            byref(num_snaps))
+            if ret >= 0:
+                self.num_snaps = ret
+                break
+            elif ret != -errno.ERANGE:
+                raise make_ex(ret, 'error listing snapshots for image %s' % (image.name,))
+
+    def __iter__(self):
+        for i in xrange(self.num_snaps):
+            yield {
+                'id'   : self.snaps[i].id,
+                'size' : self.snaps[i].size,
+                'name' : self.snaps[i].name,
+                }
+
+    def __del__(self):
+        self.librbd.rbd_snap_list_end(self.snaps)
diff --git a/ceph/src/rbd.cc b/ceph/src/rbd.cc
new file mode 100644
index 00000000..f8c43070
--- /dev/null
+++ b/ceph/src/rbd.cc
@@ -0,0 +1,3132 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2012 Sage Weil  and others
+ *
+ * LGPL2.  See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+
+#include "mon/MonClient.h"
+#include "mon/MonMap.h"
+#include "common/config.h"
+
+#include "auth/KeyRing.h"
+#include "common/errno.h"
+#include "common/ceph_argparse.h"
+#include "common/strtol.h"
+#include "global/global_init.h"
+#include "common/safe_io.h"
+#include "common/secret.h"
+#include "include/stringify.h"
+#include "include/rados/librados.hpp"
+#include "include/rbd/librbd.hpp"
+#include "include/byteorder.h"
+
+#include "include/intarith.h"
+
+#include "include/compat.h"
+#include "common/blkdev.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+#include 
+
+#include "include/rbd_types.h"
+#include "common/TextTable.h"
+#include "include/util.h"
+
+#include "common/Formatter.h"
+
+#if defined(__linux__)
+#include 
+#endif
+
+#if defined(__FreeBSD__)
+#include 
+#endif
+
+#include 
+
+#define MAX_SECRET_LEN 1000
+#define MAX_POOL_NAME_SIZE 128
+
+#define RBD_DIFF_BANNER "rbd diff v1\n"
+
+static string dir_oid = RBD_DIRECTORY;
+static string dir_info_oid = RBD_INFO;
+
+bool udevadm_settle = true;
+bool progress = true;
+bool resize_allow_shrink = false;
+
+map map_options; // -o / --options map
+
+#define dout_subsys ceph_subsys_rbd
+
+void usage()
+{
+  cout <<
+"usage: rbd [-n ] [OPTIONS]  ...\n"
+"where 'pool' is a rados pool name (default is 'rbd') and 'cmd' is one of:\n"
+"  (ls | list) [-l | --long ] [pool-name] list rbd images\n"
+"                                              (-l includes snapshots/clones)\n"
+"  info                            show information about image size,\n"
+"                                              striping, etc.\n"
+"  create [--order ] --size    create an empty image\n"
+"  clone [--order ]  \n"
+"                                              clone a snapshot into a COW\n"
+"                                              child image\n"
+"  children                         display children of snapshot\n"
+"  flatten                         fill clone with parent data\n"
+"                                              (make it independent)\n"
+"  resize --size               resize (expand or contract) image\n"
+"  rm                              delete an image\n"
+"  export                    export image to file\n"
+"                                              \"-\" for stdout\n"
+"  import                    import image from file\n"
+"                                              (dest defaults\n"
+"                                               as the filename part of file)\n"
+"                                              \"-\" for stdin\n"
+"  diff  [--from-snap ] print extents that differ since\n"
+"                                              a previous snap, or image creation\n"
+"  export-diff  [--from-snap ] \n"
+"                                              export an incremental diff to\n"
+"                                              path, or \"-\" for stdout\n"
+"  import-diff               import an incremental diff from\n"
+"                                              path or \"-\" for stdin\n"
+"  (cp | copy)                      copy src image to dest\n"
+"  (mv | rename)                    rename src image to dest\n"
+"  snap ls                         dump list of image snapshots\n"
+"  snap create                      create a snapshot\n"
+"  snap rollback                    rollback image to snapshot\n"
+"  snap rm                          deletes a snapshot\n"
+"  snap purge                      deletes all snapshots\n"
+"  snap protect                     prevent a snapshot from being deleted\n"
+"  snap unprotect                   allow a snapshot to be deleted\n"
+"  watch                           watch events on image\n"
+"  map                             map image to a block device\n"
+"                                              using the kernel\n"
+"  unmap                               unmap a rbd device that was\n"
+"                                              mapped by the kernel\n"
+"  showmapped                                  show the rbd images mapped\n"
+"                                              by the kernel\n"
+"  lock list                       show locks held on an image\n"
+"  lock add   [--shared ] take a lock called id on an image\n"
+"  lock remove         release a lock on an image\n"
+"  bench-write                     simple write benchmark\n"
+"                 --io-size               write size\n"
+"                 --io-threads              ios in flight\n"
+"                 --io-total              total bytes to write\n"
+"                 --io-pattern         write pattern\n"
+"\n"
+",  are [pool/]name[@snap], or you may specify\n"
+"individual pieces of names with -p/--pool, --image, and/or --snap.\n"
+"\n"
+"Other input options:\n"
+"  -p, --pool                   source pool name\n"
+"  --image                image name\n"
+"  --dest                 destination [pool and] image name\n"
+"  --snap                  snapshot name\n"
+"  --dest-pool                  destination pool name\n"
+"  --path                  path name for import/export\n"
+"  --size                 size of image for create and resize\n"
+"  --order                      the object size in bits; object size will be\n"
+"                                     (1 << order) bytes. Default is 22 (4 MB).\n"
+"  --image-format      format to use when creating an image\n"
+"                                     format 1 is the original format (default)\n"
+"                                     format 2 supports cloning\n"
+"  --id                     rados user (without 'client.'prefix) to\n"
+"                                     authenticate as\n"
+"  --keyfile                    file containing secret key for use with cephx\n"
+"  --shared                      take a shared (rather than exclusive) lock\n"
+"  --format            output format (default: plain, json, xml)\n"
+"  --pretty-format                    make json or xml output more readable\n"
+"  --no-settle                        do not wait for udevadm to settle on map/unmap\n"
+"  --no-progress                      do not show progress for long-running commands\n"
+"  -o, --options         options to use when mapping an image\n"
+"  --read-only                        set device readonly when mapping image\n"
+"  --allow-shrink                     allow shrinking of an image when resizing\n";
+}
+
+static string feature_str(uint64_t feature)
+{
+  switch (feature) {
+  case RBD_FEATURE_LAYERING:
+    return "layering";
+  case RBD_FEATURE_STRIPINGV2:
+    return "striping";
+  default:
+    return "";
+  }
+}
+
+static string features_str(uint64_t features)
+{
+  string s = "";
+
+  for (uint64_t feature = 1; feature <= RBD_FEATURE_STRIPINGV2;
+       feature <<= 1) {
+    if (feature & features) {
+      if (s.size())
+	s += ", ";
+      s += feature_str(feature);
+    }
+  }
+  return s;
+}
+
+static void format_features(Formatter *f, uint64_t features)
+{
+  f->open_array_section("features");
+  for (uint64_t feature = 1; feature <= RBD_FEATURE_STRIPINGV2;
+       feature <<= 1) {
+    f->dump_string("feature", feature_str(feature));
+  }
+  f->close_section();
+}
+
+struct MyProgressContext : public librbd::ProgressContext {
+  const char *operation;
+  int last_pc;
+
+  MyProgressContext(const char *o) : operation(o), last_pc(0) {
+  }
+
+  int update_progress(uint64_t offset, uint64_t total) {
+    if (progress) {
+      int pc = total ? (offset * 100ull / total) : 0;
+      if (pc != last_pc) {
+	cerr << "\r" << operation << ": "
+	  //	   << offset << " / " << total << " "
+	     << pc << "% complete...";
+	cerr.flush();
+	last_pc = pc;
+      }
+    }
+    return 0;
+  }
+  void finish() {
+    if (progress) {
+      cerr << "\r" << operation << ": 100% complete...done." << std::endl;
+    }
+  }
+  void fail() {
+    if (progress) {
+      cerr << "\r" << operation << ": " << last_pc << "% complete...failed."
+	   << std::endl;
+    }
+  }
+};
+
+static int get_outfmt(const char *output_format,
+		      bool pretty,
+		      boost::scoped_ptr *f)
+{
+  if (!strcmp(output_format, "json")) {
+    f->reset(new JSONFormatter(pretty));
+  } else if (!strcmp(output_format, "xml")) {
+    f->reset(new XMLFormatter(pretty));
+  } else if (strcmp(output_format, "plain")) {
+    cerr << "rbd: unknown format '" << output_format << "'" << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+static int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag,
+		   Formatter *f)
+{
+  std::vector names;
+  int r = rbd.list(io_ctx, names);
+  if (r == -ENOENT)
+    r = 0;
+  if (r < 0)
+    return r;
+
+  if (!lflag) {
+    if (f)
+      f->open_array_section("images");
+    for (std::vector::const_iterator i = names.begin();
+       i != names.end(); ++i) {
+       if (f)
+	 f->dump_string("name", *i);
+       else
+	 cout << *i << std::endl;
+    }
+    if (f) {
+      f->close_section();
+      f->flush(cout);
+    }
+    return 0;
+  }
+
+  TextTable tbl;
+
+  if (f) {
+    f->open_array_section("images");
+  } else {
+    tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("SIZE", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("PARENT", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("FMT", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("PROT", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("LOCK", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  string pool, image, snap, parent;
+
+  for (std::vector::const_iterator i = names.begin();
+       i != names.end(); ++i) {
+    librbd::image_info_t info;
+    librbd::Image im;
+
+    r = rbd.open_read_only(io_ctx, im, i->c_str(), NULL);
+    // image might disappear between rbd.list() and rbd.open(); ignore
+    // that, warn about other possible errors (EPERM, say, for opening
+    // an old-format image, because you need execute permission for the
+    // class method)
+    if (r < 0) {
+      if (r != -ENOENT) {
+	cerr << "rbd: error opening " << *i << ": " << cpp_strerror(r)
+	     << std::endl;
+      }
+      // in any event, continue to next image
+      continue;
+    }
+
+    // handle second-nth trips through loop
+    parent.clear();
+    r = im.parent_info(&pool, &image, &snap);
+    if (r < 0 && r != -ENOENT)
+      return r;
+
+    bool has_parent = false;
+    if (r != -ENOENT) {
+      parent = pool + "/" + image + "@" + snap;
+      has_parent = true;
+    }
+
+    if (im.stat(info, sizeof(info)) < 0)
+      return -EINVAL;
+
+    uint8_t old_format;
+    im.old_format(&old_format);
+
+    list lockers;
+    bool exclusive;
+    r = im.list_lockers(&lockers, &exclusive, NULL);
+    if (r < 0)
+      return r;
+    string lockstr;
+    if (!lockers.empty()) {
+      lockstr = (exclusive) ? "excl" : "shr";
+    }
+
+    if (f) {
+      f->open_object_section("image");
+      f->dump_string("image", *i);
+      f->dump_unsigned("size", info.size);
+      if (has_parent) {
+	f->open_object_section("parent");
+	f->dump_string("pool", pool);
+	f->dump_string("image", image);
+	f->dump_string("snapshot", snap);
+	f->close_section();
+      }
+      f->dump_int("format", old_format ? 1 : 2);
+      if (!lockers.empty())
+	f->dump_string("lock_type", exclusive ? "exclusive" : "shared");
+      f->close_section();
+    } else {
+      tbl << *i
+	  << stringify(si_t(info.size))
+	  << parent
+	  << ((old_format) ? '1' : '2')
+	  << ""				// protect doesn't apply to images
+	  << lockstr
+	  << TextTable::endrow;
+    }
+
+    vector snaplist;
+    if (im.snap_list(snaplist) >= 0 && !snaplist.empty()) {
+      for (std::vector::iterator s = snaplist.begin();
+	   s != snaplist.end(); ++s) {
+	bool is_protected;
+	bool has_parent = false;
+	parent.clear();
+	im.snap_set(s->name.c_str());
+	r = im.snap_is_protected(s->name.c_str(), &is_protected);
+	if (r < 0)
+	  return r;
+	if (im.parent_info(&pool, &image, &snap) >= 0) {
+	  parent = pool + "/" + image + "@" + snap;
+	  has_parent = true;
+	}
+	if (f) {
+	  f->open_object_section("snapshot");
+	  f->dump_string("image", *i);
+	  f->dump_string("snapshot", s->name);
+	  f->dump_unsigned("size", s->size);
+	  if (has_parent) {
+	    f->open_object_section("parent");
+	    f->dump_string("pool", pool);
+	    f->dump_string("image", image);
+	    f->dump_string("snapshot", snap);
+	    f->close_section();
+	  }
+	  f->dump_int("format", old_format ? 1 : 2);
+	  f->dump_string("protected", is_protected ? "true" : "false");
+	  f->close_section();
+	} else {
+	  tbl << *i + "@" + s->name
+	      << stringify(si_t(s->size))
+	      << parent
+	      << ((old_format) ? '1' : '2')
+	      << (is_protected ? "yes" : "")
+	      << "" 			// locks don't apply to snaps
+	      << TextTable::endrow;
+	}
+      }
+    }
+  }
+  if (f) {
+    f->close_section();
+    f->flush(cout);
+  } else if (!names.empty()) {
+    cout << tbl;
+  }
+
+  return 0;
+}
+
+static int do_create(librbd::RBD &rbd, librados::IoCtx& io_ctx,
+		     const char *imgname, uint64_t size, int *order,
+		     int format, uint64_t features,
+		     uint64_t stripe_unit, uint64_t stripe_count)
+{
+  int r;
+
+  if (format == 1) {
+    // weird striping not allowed with format 1!
+    if ((stripe_unit || stripe_count) &&
+	(stripe_unit != (1ull << *order) && stripe_count != 1)) {
+      cerr << "non-default striping not allowed with format 1; use --format 2"
+	   << std::endl;
+      return -EINVAL;
+    }
+    r = rbd.create(io_ctx, imgname, size, order);
+  } else {
+    if (features == 0) {
+      features = RBD_FEATURE_LAYERING;
+    }
+    if ((stripe_unit || stripe_count) &&
+	(stripe_unit != (1ull << *order) && stripe_count != 1)) {
+      features |= RBD_FEATURE_STRIPINGV2;
+    }
+    r = rbd.create3(io_ctx, imgname, size, features, order,
+		    stripe_unit, stripe_count);
+  }
+  if (r < 0)
+    return r;
+  return 0;
+}
+
+static int do_clone(librbd::RBD &rbd, librados::IoCtx &p_ioctx,
+		    const char *p_name, const char *p_snapname,
+		    librados::IoCtx &c_ioctx, const char *c_name,
+		    uint64_t features, int *c_order)
+{
+  if (features == 0)
+    features = RBD_FEATURES_ALL;
+  else if ((features & RBD_FEATURE_LAYERING) != RBD_FEATURE_LAYERING)
+    return -EINVAL;
+
+  return rbd.clone(p_ioctx, p_name, p_snapname, c_ioctx, c_name, features,
+		    c_order);
+}
+
+static int do_flatten(librbd::Image& image)
+{
+  MyProgressContext pc("Image flatten");
+  int r = image.flatten_with_progress(pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+static int do_rename(librbd::RBD &rbd, librados::IoCtx& io_ctx,
+		     const char *imgname, const char *destname)
+{
+  int r = rbd.rename(io_ctx, imgname, destname);
+  if (r < 0)
+    return r;
+  return 0;
+}
+
+static int do_show_info(const char *imgname, librbd::Image& image,
+			const char *snapname, Formatter *f)
+{
+  librbd::image_info_t info;
+  string parent_pool, parent_name, parent_snapname;
+  uint8_t old_format;
+  uint64_t overlap, features;
+  bool snap_protected;
+  int r;
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  r = image.old_format(&old_format);
+  if (r < 0)
+    return r;
+
+  r = image.overlap(&overlap);
+  if (r < 0)
+    return r;
+
+  r = image.features(&features);
+  if (r < 0)
+    return r;
+
+  if (snapname) {
+    r = image.snap_is_protected(snapname, &snap_protected);
+    if (r < 0)
+      return r;
+  }
+
+  char prefix[RBD_MAX_BLOCK_NAME_SIZE + 1];
+  strncpy(prefix, info.block_name_prefix, RBD_MAX_BLOCK_NAME_SIZE);
+  prefix[RBD_MAX_BLOCK_NAME_SIZE] = '\0';
+
+  if (f) {
+    f->open_object_section("image");
+    f->dump_string("name", imgname);
+    f->dump_unsigned("size", info.size);
+    f->dump_unsigned("objects", info.num_objs);
+    f->dump_int("order", info.order);
+    f->dump_unsigned("object_size", info.obj_size);
+    f->dump_string("block_name_prefix", prefix);
+    f->dump_int("format", (old_format ? 1 : 2));
+  } else {
+    cout << "rbd image '" << imgname << "':\n"
+	 << "\tsize " << prettybyte_t(info.size) << " in "
+	 << info.num_objs << " objects"
+	 << std::endl
+	 << "\torder " << info.order
+	 << " (" << prettybyte_t(info.obj_size) << " objects)"
+	 << std::endl
+	 << "\tblock_name_prefix: " << prefix
+	 << std::endl
+	 << "\tformat: " << (old_format ? "1" : "2")
+	 << std::endl;
+  }
+
+  if (!old_format) {
+    if (f)
+      format_features(f, features);
+    else
+      cout << "\tfeatures: " << features_str(features) << std::endl;
+  }
+
+  // snapshot info, if present
+  if (snapname) {
+    if (f) {
+      f->dump_string("protected", snap_protected ? "true" : "false");
+    } else {
+      cout << "\tprotected: " << (snap_protected ? "True" : "False")
+	   << std::endl;
+    }
+  }
+
+  // parent info, if present
+  if ((image.parent_info(&parent_pool, &parent_name, &parent_snapname) == 0) &&
+      parent_name.length() > 0) {
+    if (f) {
+      f->open_object_section("parent");
+      f->dump_string("pool", parent_pool);
+      f->dump_string("image", parent_name);
+      f->dump_string("snapshot", parent_snapname);
+      f->dump_unsigned("overlap", overlap);
+      f->close_section();
+    } else {
+      cout << "\tparent: " << parent_pool << "/" << parent_name
+	   << "@" << parent_snapname << std::endl;
+      cout << "\toverlap: " << prettybyte_t(overlap) << std::endl;
+    }
+  }
+
+  // striping info, if feature is set
+  if (features & RBD_FEATURE_STRIPINGV2) {
+    if (f) {
+      f->dump_unsigned("stripe_unit", image.get_stripe_unit());
+      f->dump_unsigned("stripe_count", image.get_stripe_count());
+    } else {
+      cout << "\tstripe unit: " << prettybyte_t(image.get_stripe_unit())
+	   << std::endl
+	   << "\tstripe count: " << image.get_stripe_count() << std::endl;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(cout);
+  }
+
+  return 0;
+}
+
+static int do_delete(librbd::RBD &rbd, librados::IoCtx& io_ctx,
+		     const char *imgname)
+{
+  MyProgressContext pc("Removing image");
+  int r = rbd.remove_with_progress(io_ctx, imgname, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+static int do_resize(librbd::Image& image, uint64_t size)
+{
+  MyProgressContext pc("Resizing image");
+  int r = image.resize_with_progress(size, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+static int do_list_snaps(librbd::Image& image, Formatter *f)
+{
+  std::vector snaps;
+  TextTable t;
+  int r;
+
+  r = image.snap_list(snaps);
+  if (r < 0)
+    return r;
+
+  if (f) {
+    f->open_array_section("snapshots");
+  } else {
+    t.define_column("SNAPID", TextTable::RIGHT, TextTable::RIGHT);
+    t.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    t.define_column("SIZE", TextTable::RIGHT, TextTable::RIGHT);
+  }
+
+  for (std::vector::iterator s = snaps.begin();
+       s != snaps.end(); ++s) {
+    if (f) {
+      f->open_object_section("snapshot");
+      f->dump_unsigned("id", s->id);
+      f->dump_string("name", s->name);
+      f->dump_unsigned("size", s->size);
+      f->close_section();
+    } else {
+      t << s->id << s->name << stringify(prettybyte_t(s->size))
+	<< TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(cout);
+  } else if (snaps.size()) {
+    cout << t;
+  }
+
+  return 0;
+}
+
+static int do_add_snap(librbd::Image& image, const char *snapname)
+{
+  int r = image.snap_create(snapname);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+static int do_remove_snap(librbd::Image& image, const char *snapname)
+{
+  int r = image.snap_remove(snapname);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+static int do_rollback_snap(librbd::Image& image, const char *snapname)
+{
+  MyProgressContext pc("Rolling back to snapshot");
+  int r = image.snap_rollback_with_progress(snapname, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+static int do_purge_snaps(librbd::Image& image)
+{
+  MyProgressContext pc("Removing all snapshots");
+  std::vector snaps;
+  int r = image.snap_list(snaps);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+
+  for (size_t i = 0; i < snaps.size(); ++i) {
+    r = image.snap_remove(snaps[i].name.c_str());
+    if (r < 0) {
+      pc.fail();
+      return r;
+    }
+    pc.update_progress(i + 1, snaps.size());
+  }
+
+  pc.finish();
+  return 0;
+}
+
+static int do_protect_snap(librbd::Image& image, const char *snapname)
+{
+  int r = image.snap_protect(snapname);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+static int do_unprotect_snap(librbd::Image& image, const char *snapname)
+{
+  int r = image.snap_unprotect(snapname);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+static int do_list_children(librbd::Image &image, Formatter *f)
+{
+  set > children;
+  int r;
+
+  r = image.list_children(&children);
+  if (r < 0)
+    return r;
+
+  if (f)
+    f->open_array_section("children");
+
+  for (set >::const_iterator child_it = children.begin();
+       child_it != children.end(); child_it++) {
+    if (f) {
+      f->open_object_section("child");
+      f->dump_string("pool", child_it->first);
+      f->dump_string("image", child_it->second);
+      f->close_section();
+    } else {
+      cout << child_it->first << "/" << child_it->second << std::endl;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(cout);
+  }
+
+  return 0;
+}
+
+static int do_lock_list(librbd::Image& image, Formatter *f)
+{
+  list lockers;
+  bool exclusive;
+  string tag;
+  TextTable tbl;
+  int r;
+
+  r = image.list_lockers(&lockers, &exclusive, &tag);
+  if (r < 0)
+    return r;
+
+  if (f) {
+    f->open_object_section("locks");
+  } else {
+    tbl.define_column("Locker", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("ID", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Address", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  if (lockers.size()) {
+    bool one = (lockers.size() == 1);
+
+    if (!f) {
+      cout << "There " << (one ? "is " : "are ") << lockers.size()
+	   << (exclusive ? " exclusive" : " shared")
+	   << " lock" << (one ? "" : "s") << " on this image.\n";
+      if (!exclusive)
+	cout << "Lock tag: " << tag << "\n";
+    }
+
+    for (list::const_iterator it = lockers.begin();
+	 it != lockers.end(); ++it) {
+      if (f) {
+	f->open_object_section(it->cookie.c_str());
+	f->dump_string("locker", it->client);
+	f->dump_string("address", it->address);
+	f->close_section();
+      } else {
+	tbl << it->client << it->cookie << it->address << TextTable::endrow;
+      }
+    }
+    if (!f)
+      cout << tbl;
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(cout);
+  }
+  return 0;
+}
+
+static int do_lock_add(librbd::Image& image, const char *cookie,
+		       const char *tag)
+{
+  if (tag)
+    return image.lock_shared(cookie, tag);
+  else
+    return image.lock_exclusive(cookie);
+}
+
+static int do_lock_remove(librbd::Image& image, const char *client,
+			  const char *cookie)
+{
+  return image.break_lock(client, cookie);
+}
+
+static void rbd_bencher_completion(void *c, void *pc);
+
+struct rbd_bencher;
+
+struct rbd_bencher {
+  librbd::Image *image;
+  Mutex lock;
+  Cond cond;
+  int in_flight;
+
+  rbd_bencher(librbd::Image *i)
+    : image(i),
+      lock("rbd_bencher::lock"),
+      in_flight(0)
+  { }
+
+  bool start_write(int max, uint64_t off, uint64_t len, bufferlist& bl)
+  {
+    {
+      Mutex::Locker l(lock);
+      if (in_flight >= max)
+	return false;
+      in_flight++;
+    }
+    librbd::RBD::AioCompletion *c =
+      new librbd::RBD::AioCompletion((void *)this, rbd_bencher_completion);
+    image->aio_write(off, len, bl, c);
+    //cout << "start " << c << " at " << off << "~" << len << std::endl;
+    return true;
+  }
+
+  void wait_for(int max) {
+    Mutex::Locker l(lock);
+    while (in_flight > max) {
+      utime_t dur;
+      dur.set_from_double(.2);
+      cond.WaitInterval(g_ceph_context, lock, dur);
+    }
+  }
+
+};
+
+void rbd_bencher_completion(void *vc, void *pc)
+{
+  librbd::RBD::AioCompletion *c = (librbd::RBD::AioCompletion *)vc;
+  rbd_bencher *b = static_cast(pc);
+  //cout << "complete " << c << std::endl;
+  int ret = c->get_return_value();
+  if (ret != 0) {
+    cout << "write error: " << cpp_strerror(ret) << std::endl;
+    assert(0 == ret);
+  }
+  b->lock.Lock();
+  b->in_flight--;
+  b->cond.Signal();
+  b->lock.Unlock();
+  c->release();
+}
+
+static int do_bench_write(librbd::Image& image, uint64_t io_size,
+			  uint64_t io_threads, uint64_t io_bytes,
+			  string pattern)
+{
+  rbd_bencher b(&image);
+
+  cout << "bench-write "
+       << " io_size " << io_size
+       << " io_threads " << io_threads
+       << " bytes " << io_bytes
+       << " pattern " << pattern
+       << std::endl;
+
+  if (pattern != "rand" && pattern != "seq")
+    return -EINVAL;
+
+  srand(time(NULL) % (unsigned long) -1);
+
+  bufferptr bp(io_size);
+  memset(bp.c_str(), rand() & 0xff, io_size);
+  bufferlist bl;
+  bl.push_back(bp);
+
+  utime_t start = ceph_clock_now(NULL);
+  utime_t last;
+  unsigned ios = 0;
+
+  uint64_t size = 0;
+  image.size(&size);
+
+  vector thread_offset;
+  uint64_t i;
+  uint64_t start_pos;
+
+  // disturb all thread's offset, used by seq write
+  for (i = 0; i < io_threads; i++) {
+    start_pos = (rand() % (size / io_size)) * io_size;
+    thread_offset.push_back(start_pos);
+  }
+
+  printf("  SEC       OPS   OPS/SEC   BYTES/SEC\n");
+  uint64_t off;
+  for (off = 0; off < io_bytes; off += io_size) {
+    b.wait_for(io_threads - 1);
+    i = 0;
+    while (i < io_threads && off < io_bytes &&
+	   b.start_write(io_threads, thread_offset[i], io_size, bl)) {
+      ++i;
+      ++ios;
+      off += io_size;
+
+      if (pattern == "rand") {
+        thread_offset[i] = (rand() % (size / io_size)) * io_size;
+      } else {
+        thread_offset[i] += io_size;
+        if (thread_offset[i] + io_size > size)
+          thread_offset[i] = 0;
+      }
+    }
+
+    utime_t now = ceph_clock_now(NULL);
+    utime_t elapsed = now - start;
+    if (elapsed.sec() != last.sec()) {
+      printf("%5d  %8d  %8.2lf  %8.2lf\n",
+	     (int)elapsed,
+	     (int)(ios - io_threads),
+	     (double)(ios - io_threads) / elapsed,
+	     (double)(off - io_threads * io_size) / elapsed);
+      last = elapsed;
+    }
+  }
+  b.wait_for(0);
+  int r = image.flush();
+  if (r < 0) {
+    cerr << "Error flushing data at the end: " << cpp_strerror(r) << std::endl;
+  }
+
+  utime_t now = ceph_clock_now(NULL);
+  double elapsed = now - start;
+
+  printf("elapsed: %5d  ops: %8d  ops/sec: %8.2lf  bytes/sec: %8.2lf\n",
+	 (int)elapsed, ios, (double)ios / elapsed, (double)off / elapsed);
+
+  return 0;
+}
+
+struct ExportContext {
+  librbd::Image *image;
+  int fd;
+  uint64_t totalsize;
+  MyProgressContext pc;
+
+  ExportContext(librbd::Image *i, int f, uint64_t t) :
+    image(i),
+    fd(f),
+    totalsize(t),
+    pc("Exporting image")
+  {}
+};
+
+static int export_read_cb(uint64_t ofs, size_t len, const char *buf, void *arg)
+{
+  ssize_t ret;
+  ExportContext *ec = static_cast(arg);
+  int fd = ec->fd;
+  static char *localbuf = NULL;
+  static size_t maplen = 0;
+
+  if (fd == 1) {
+    if (!buf) {
+      // can't seek stdout; need actual data to write
+      if (maplen < len) {
+	// never mapped, or need to map larger
+	int r;
+	if (localbuf != NULL){
+	  if ((r = munmap(localbuf, len)) < 0) {
+	    cerr << "rbd: error " << r << "munmap'ing buffer" << std::endl;
+	    return errno;
+	  }
+	}
+
+	maplen = len;
+	localbuf = (char *)mmap(NULL, maplen, PROT_READ,
+			        MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	if (localbuf == MAP_FAILED) {
+	  cerr << "rbd: MAP_FAILED mmap'ing buffer for zero writes"
+	       << std::endl;
+	  return -ENOMEM;
+	}
+      }
+      ret = write(fd, localbuf, len);
+    } else {
+      ret = write(fd, buf, len);
+    }
+  } else {		// not stdout
+    if (!buf || buf_is_zero(buf, len)) {
+      /* a hole */
+      return 0;
+    }
+
+    ret = lseek64(fd, ofs, SEEK_SET);
+    if (ret < 0)
+      return -errno;
+    ret = write(fd, buf, len);
+    ec->pc.update_progress(ofs, ec->totalsize);
+  }
+
+  if (ret < 0)
+    return -errno;
+
+  return 0;
+}
+
+static int do_export(librbd::Image& image, const char *path)
+{
+  int64_t r;
+  librbd::image_info_t info;
+  int fd;
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  if (strcmp(path, "-") == 0)
+    fd = 1;
+  else
+    fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+  if (fd < 0)
+    return -errno;
+
+  ExportContext ec(&image, fd, info.size);
+  r = image.read_iterate2(0, info.size, export_read_cb, (void *)&ec);
+  if (r < 0)
+    goto out;
+
+  if (fd != 1)
+    r = ftruncate(fd, info.size);
+  if (r < 0)
+    goto out;
+
+ out:
+  close(fd);
+  if (r < 0)
+    ec.pc.fail();
+  else
+    ec.pc.finish();
+  return r;
+}
+
+static int export_diff_cb(uint64_t ofs, size_t _len, int exists, void *arg)
+{
+  ExportContext *ec = static_cast(arg);
+  int r;
+
+  // extent
+  bufferlist bl;
+  __u8 tag = exists ? 'w' : 'z';
+  ::encode(tag, bl);
+  ::encode(ofs, bl);
+  uint64_t len = _len;
+  ::encode(len, bl);
+  r = bl.write_fd(ec->fd);
+  if (r < 0)
+    return r;
+
+  if (exists) {
+    // read block
+    bl.clear();
+    r = ec->image->read(ofs, len, bl);
+    if (r < 0)
+      return r;
+    r = bl.write_fd(ec->fd);
+    if (r < 0)
+      return r;
+  }
+
+  ec->pc.update_progress(ofs, ec->totalsize);
+
+  return 0;
+}
+
+static int do_export_diff(librbd::Image& image, const char *fromsnapname,
+			  const char *endsnapname,
+			  const char *path)
+{
+  int r;
+  librbd::image_info_t info;
+  int fd;
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  if (strcmp(path, "-") == 0)
+    fd = 1;
+  else
+    fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+  if (fd < 0)
+    return -errno;
+
+  {
+    // header
+    bufferlist bl;
+    bl.append(RBD_DIFF_BANNER, strlen(RBD_DIFF_BANNER));
+
+    __u8 tag;
+    if (fromsnapname) {
+      tag = 'f';
+      ::encode(tag, bl);
+      string from(fromsnapname);
+      ::encode(from, bl);
+    }
+
+    if (endsnapname) {
+      tag = 't';
+      ::encode(tag, bl);
+      string to(endsnapname);
+      ::encode(to, bl);
+    }
+
+    tag = 's';
+    ::encode(tag, bl);
+    uint64_t endsize = info.size;
+    ::encode(endsize, bl);
+
+    r = bl.write_fd(fd);
+    if (r < 0) {
+      close(fd);
+      return r;
+    }
+  }
+
+  ExportContext ec(&image, fd, info.size);
+  r = image.diff_iterate(fromsnapname, 0, info.size, export_diff_cb, (void *)&ec);
+  if (r < 0)
+    goto out;
+
+  {
+    __u8 tag = 'e';
+    bufferlist bl;
+    ::encode(tag, bl);
+    r = bl.write_fd(fd);
+  }
+
+ out:
+  close(fd);
+  if (r < 0)
+    ec.pc.fail();
+  else
+    ec.pc.finish();
+  return r;
+}
+
+struct output_method {
+  output_method() : f(NULL), t(NULL), empty(true) {}
+  Formatter *f;
+  TextTable *t;
+  bool empty;
+};
+
+static int diff_cb(uint64_t ofs, size_t len, int exists, void *arg)
+{
+  output_method *om = static_cast(arg);
+  om->empty = false;
+  if (om->f) {
+    om->f->open_object_section("extent");
+    om->f->dump_unsigned("offset", ofs);
+    om->f->dump_unsigned("length", len);
+    om->f->dump_string("exists", exists ? "true" : "false");
+    om->f->close_section();
+  } else {
+    assert(om->t);
+    *(om->t) << ofs << len << (exists ? "data" : "zero") << TextTable::endrow;
+  }
+  return 0;
+}
+
+static int do_diff(librbd::Image& image, const char *fromsnapname,
+		   Formatter *f)
+{
+  int r;
+  librbd::image_info_t info;
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  output_method om;
+  if (f) {
+    om.f = f;
+    f->open_array_section("extents");
+  } else {
+    om.t = new TextTable();
+    om.t->define_column("Offset", TextTable::LEFT, TextTable::LEFT);
+    om.t->define_column("Length", TextTable::LEFT, TextTable::LEFT);
+    om.t->define_column("Type", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  r = image.diff_iterate(fromsnapname, 0, info.size, diff_cb, &om);
+  if (f) {
+    f->close_section();
+    f->flush(cout);
+  } else {
+    if (!om.empty)
+      cout << *om.t;
+    delete om.t;
+  }
+  return r;
+}
+
+static const char *imgname_from_path(const char *path)
+{
+  const char *imgname;
+
+  imgname = strrchr(path, '/');
+  if (imgname)
+    imgname++;
+  else
+    imgname = path;
+
+  return imgname;
+}
+
+static void update_snap_name(char *imgname, char **snap)
+{
+  char *s;
+
+  s = strrchr(imgname, '@');
+  if (!s)
+    return;
+
+  *s = '\0';
+
+  if (!snap)
+    return;
+
+  s++;
+  if (*s)
+    *snap = s;
+}
+
+static void set_pool_image_name(const char *orig_pool, const char *orig_img,
+                                char **new_pool, char **new_img, char **snap)
+{
+  const char *sep;
+
+  if (!orig_img)
+    return;
+
+  sep = strchr(orig_img, '/');
+  if (!sep) {
+    *new_img = strdup(orig_img);
+    goto done_img;
+  }
+
+  *new_pool =  strdup(orig_img);
+  sep = strchr(*new_pool, '/');
+  assert (sep);
+
+  *(char *)sep = '\0';
+  *new_img = strdup(sep + 1);
+
+done_img:
+  update_snap_name(*new_img, snap);
+}
+
+static int do_import(librbd::RBD &rbd, librados::IoCtx& io_ctx,
+		     const char *imgname, int *order, const char *path,
+		     int format, uint64_t features, uint64_t size)
+{
+  int fd, r;
+  struct stat stat_buf;
+  MyProgressContext pc("Importing image");
+
+  assert(imgname);
+
+  // default order as usual
+  if (*order == 0)
+    *order = 22;
+
+  // try to fill whole imgblklen blocks for sparsification
+  uint64_t image_pos = 0;
+  size_t imgblklen = 1 << *order;
+  char *p = new char[imgblklen];
+  size_t reqlen = imgblklen;	// amount requested from read
+  ssize_t readlen;		// amount received from one read
+  size_t blklen = 0;		// amount accumulated from reads to fill blk
+  librbd::Image image;
+
+  bool from_stdin = !strcmp(path, "-");
+  if (from_stdin) {
+    fd = 0;
+    size = 1ULL << *order;
+  } else {
+    if ((fd = open(path, O_RDONLY)) < 0) {
+      r = -errno;
+      cerr << "rbd: error opening " << path << std::endl;
+      goto done2;
+    }
+
+    if ((fstat(fd, &stat_buf)) < 0) {
+      r = -errno;
+      cerr << "rbd: stat error " << path << std::endl;
+      goto done;
+    }
+    if (S_ISDIR(stat_buf.st_mode)) {
+      r = -EISDIR;
+      cerr << "rbd: cannot import a directory" << std::endl;
+      goto done;
+    }
+    if (stat_buf.st_size)
+      size = (uint64_t)stat_buf.st_size;
+
+    if (!size) {
+      int64_t bdev_size = 0;
+      r = get_block_device_size(fd, &bdev_size);
+      if (r < 0) {
+	cerr << "rbd: unable to get size of file/block device" << std::endl;
+	goto done;
+      }
+      assert(bdev_size >= 0);
+      size = (uint64_t) bdev_size;
+    }
+  }
+  r = do_create(rbd, io_ctx, imgname, size, order, format, features, 0, 0);
+  if (r < 0) {
+    cerr << "rbd: image creation failed" << std::endl;
+    goto done;
+  }
+  r = rbd.open(io_ctx, image, imgname);
+  if (r < 0) {
+    cerr << "rbd: failed to open image" << std::endl;
+    goto done;
+  }
+
+  // loop body handles 0 return, as we may have a block to flush
+  while ((readlen = ::read(fd, p + blklen, reqlen)) >= 0) {
+    blklen += readlen;
+    // if read was short, try again to fill the block before writing
+    if (readlen && ((size_t)readlen < reqlen)) {
+      reqlen -= readlen;
+      continue;
+    }
+    if (!from_stdin)
+      pc.update_progress(image_pos, size);
+
+    bufferlist bl(blklen);
+    bl.append(p, blklen);
+    // resize output image by binary expansion as we go for stdin
+    if (from_stdin && (image_pos + (size_t)blklen) > size) {
+      size *= 2;
+      r = image.resize(size);
+      if (r < 0) {
+	cerr << "rbd: can't resize image during import" << std::endl;
+	goto done;
+      }
+    }
+
+    // write as much as we got; perhaps less than imgblklen
+    // but skip writing zeros to create sparse images
+    if (!bl.is_zero()) {
+      r = image.write(image_pos, blklen, bl);
+      if (r < 0) {
+	cerr << "rbd: error writing to image position " << image_pos
+	     << std::endl;
+	goto done;
+      }
+    }
+    // done with whole block, whether written or not
+    image_pos += blklen;
+    // if read had returned 0, we're at EOF and should quit
+    if (readlen == 0)
+      break;
+    blklen = 0;
+    reqlen = imgblklen;
+  }
+  if (from_stdin) {
+    r = image.resize(image_pos);
+    if (r < 0) {
+      cerr << "rbd: final image resize failed" << std::endl;
+      goto done;
+    }
+  }
+
+  r = 0;
+
+ done:
+  if (!from_stdin) {
+    if (r < 0)
+      pc.fail();
+    else
+      pc.finish();
+    close(fd);
+  }
+ done2:
+  delete[] p;
+  return r;
+}
+
+static int read_string(int fd, unsigned max, string *out)
+{
+  char buf[4];
+
+  int r = safe_read_exact(fd, buf, 4);
+  if (r < 0)
+    return r;
+
+  bufferlist bl;
+  bl.append(buf, 4);
+  bufferlist::iterator p = bl.begin();
+  uint32_t len;
+  ::decode(len, p);
+  if (len > max)
+    return -EINVAL;
+
+  char sbuf[len];
+  r = safe_read_exact(fd, sbuf, len);
+  if (r < 0)
+    return r;
+  out->assign(sbuf, len);
+  return len;
+}
+
+static int do_import_diff(librbd::Image &image, const char *path)
+{
+  int fd, r;
+  struct stat stat_buf;
+  MyProgressContext pc("Importing image diff");
+  uint64_t size = 0;
+  uint64_t off = 0;
+  string from, to;
+
+  bool from_stdin = !strcmp(path, "-");
+  if (from_stdin) {
+    fd = 0;
+  } else {
+    fd = open(path, O_RDONLY);
+    if (fd < 0) {
+      r = -errno;
+      cerr << "rbd: error opening " << path << std::endl;
+      return r;
+    }
+    r = ::fstat(fd, &stat_buf);
+    if (r < 0)
+      goto done;
+    size = (uint64_t)stat_buf.st_size;
+  }
+
+  char buf[strlen(RBD_DIFF_BANNER) + 1];
+  r = safe_read_exact(fd, buf, strlen(RBD_DIFF_BANNER));
+  if (r < 0)
+    goto done;
+  buf[strlen(RBD_DIFF_BANNER)] = '\0';
+  if (strcmp(buf, RBD_DIFF_BANNER)) {
+    cerr << "invalid banner '" << buf << "', expected '" << RBD_DIFF_BANNER << "'" << std::endl;
+    r = -EINVAL;
+    goto done;
+  }
+
+  while (true) {
+    __u8 tag;
+    r = safe_read_exact(fd, &tag, 1);
+    if (r < 0) {
+      goto done;
+    }
+
+    if (tag == 'e') {
+      dout(2) << " end diff" << dendl;
+      break;
+    } else if (tag == 'f') {
+      r = read_string(fd, 4096, &from);   // 4k limit to make sure we don't get a garbage string
+      if (r < 0)
+	goto done;
+      dout(2) << " from snap " << from << dendl;
+
+      if (!image.snap_exists(from.c_str())) {
+	cerr << "start snapshot '" << from << "' does not exist in the image, aborting" << std::endl;
+	r = -EINVAL;
+	goto done;
+      }
+    }
+    else if (tag == 't') {
+      r = read_string(fd, 4096, &to);   // 4k limit to make sure we don't get a garbage string
+      if (r < 0)
+	goto done;
+      dout(2) << "   to snap " << to << dendl;
+
+      // verify this snap isn't already present
+      if (image.snap_exists(to.c_str())) {
+	cerr << "end snapshot '" << to << "' already exists, aborting" << std::endl;
+	r = -EEXIST;
+	goto done;
+      }
+    } else if (tag == 's') {
+      uint64_t end_size;
+      char buf[8];
+      r = safe_read_exact(fd, buf, 8);
+      if (r < 0)
+	goto done;
+      bufferlist bl;
+      bl.append(buf, 8);
+      bufferlist::iterator p = bl.begin();
+      ::decode(end_size, p);
+      uint64_t cur_size;
+      image.size(&cur_size);
+      if (cur_size != end_size) {
+	dout(2) << "resize " << cur_size << " -> " << end_size << dendl;
+	image.resize(end_size);
+      } else {
+	dout(2) << "size " << end_size << " (no change)" << dendl;
+      }
+      if (from_stdin)
+	size = end_size;
+    } else if (tag == 'w' || tag == 'z') {
+      uint64_t len;
+      char buf[16];
+      r = safe_read_exact(fd, buf, 16);
+      if (r < 0)
+	goto done;
+      bufferlist bl;
+      bl.append(buf, 16);
+      bufferlist::iterator p = bl.begin();
+      ::decode(off, p);
+      ::decode(len, p);
+
+      if (tag == 'w') {
+	bufferptr bp = buffer::create(len);
+	r = safe_read_exact(fd, bp.c_str(), len);
+	if (r < 0)
+	  goto done;
+	bufferlist data;
+	data.append(bp);
+	dout(2) << " write " << off << "~" << len << dendl;
+	image.write(off, len, data);
+      } else {
+	dout(2) << " zero " << off << "~" << len << dendl;
+	image.discard(off, len);
+      }
+    } else {
+      cerr << "unrecognized tag byte " << (int)tag << " in stream; aborting" << std::endl;
+      r = -EINVAL;
+      goto done;
+    }
+    if (!from_stdin) {
+      // progress through input
+      uint64_t off = lseek64(fd, 0, SEEK_CUR);
+      pc.update_progress(off, size);
+    } else if (size) {
+      // progress through image offsets.  this may jitter if blocks
+      // aren't in order, but it is better than nothing.
+      pc.update_progress(off, size);
+    }
+  }
+
+  // take final snap
+  if (to.length()) {
+    dout(2) << " create end snap " << to << dendl;
+    r = image.snap_create(to.c_str());
+  }
+
+ done:
+  if (r < 0)
+    pc.fail();
+  else
+    pc.finish();
+  if (!from_stdin)
+    close(fd);
+  return r;
+}
+
+static int do_copy(librbd::Image &src, librados::IoCtx& dest_pp,
+		   const char *destname)
+{
+  MyProgressContext pc("Image copy");
+  int r = src.copy_with_progress(dest_pp, destname, pc);
+  if (r < 0){
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+class RbdWatchCtx : public librados::WatchCtx {
+  string name;
+public:
+  RbdWatchCtx(const char *imgname) : name(imgname) {}
+  virtual ~RbdWatchCtx() {}
+  virtual void notify(uint8_t opcode, uint64_t ver, bufferlist& bl) {
+    cout << name << " got notification opcode=" << (int)opcode << " ver="
+	 << ver << " bl.length=" << bl.length() << std::endl;
+  }
+};
+
+static int do_watch(librados::IoCtx& pp, const char *imgname)
+{
+  string md_oid, dest_md_oid;
+  uint64_t cookie;
+  RbdWatchCtx ctx(imgname);
+
+  string old_header_oid = imgname;
+  old_header_oid += RBD_SUFFIX;
+  string new_header_oid = RBD_HEADER_PREFIX;
+  new_header_oid += imgname;
+  bool old_format = true;
+
+  int r = pp.stat(old_header_oid, NULL, NULL);
+  if (r < 0) {
+    r = pp.stat(new_header_oid, NULL, NULL);
+    if (r < 0)
+      return r;
+    old_format = false;
+  }
+
+  r = pp.watch(old_format ? old_header_oid : new_header_oid,
+	       0, &cookie, &ctx);
+  if (r < 0) {
+    cerr << "rbd: watch failed" << std::endl;
+    return r;
+  }
+
+  cout << "press enter to exit..." << std::endl;
+  getchar();
+
+  return 0;
+}
+
+static int do_kernel_add(const char *poolname, const char *imgname,
+			 const char *snapname)
+{
+  MonMap monmap;
+  int r = monmap.build_initial(g_ceph_context, cerr);
+  if (r < 0)
+    return r;
+
+  map::const_iterator it = monmap.mon_addr.begin();
+  ostringstream oss;
+  for (size_t i = 0; i < monmap.mon_addr.size(); ++i, ++it) {
+    oss << it->second.addr;
+    if (i + 1 < monmap.mon_addr.size())
+      oss << ",";
+  }
+
+  const char *user = g_conf->name.get_id().c_str();
+  oss << " name=" << user;
+
+  char key_name[strlen(user) + strlen("client.") + 1];
+  snprintf(key_name, sizeof(key_name), "client.%s", user);
+
+  KeyRing keyring;
+  r = keyring.from_ceph_context(g_ceph_context);
+  if (r == -ENOENT && !(g_conf->keyfile.length() ||
+			g_conf->key.length()))
+    r = 0;
+  if (r < 0) {
+    cerr << "rbd: failed to get secret: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  CryptoKey secret;
+  if (keyring.get_secret(g_conf->name, secret)) {
+    string secret_str;
+    secret.encode_base64(secret_str);
+
+    r = set_kernel_secret(secret_str.c_str(), key_name);
+    if (r >= 0) {
+      if (r == 0)
+	cerr << "rbd: warning: secret has length 0" << std::endl;
+      oss << ",key=" << key_name;
+    } else if (r == -ENODEV || r == -ENOSYS) {
+      /* running against older kernel; fall back to secret= in options */
+      oss << ",secret=" << secret_str;
+    } else {
+      cerr << "rbd: failed to add ceph secret key '" << key_name
+	   << "' to kernel: " << cpp_strerror(r) << std::endl;
+      return r;
+    }
+  } else if (is_kernel_secret(key_name)) {
+    oss << ",key=" << key_name;
+  }
+
+  for (map::const_iterator it = map_options.begin();
+       it != map_options.end();
+       ++it) {
+    // for compatibility with < 3.7 kernels, assume that rw is on by
+    // default and omit it even if it was specified by the user
+    // (see ceph.git commit fb0f1986449b)
+    if (it->first == "rw" && it->second == "rw")
+      continue;
+
+    oss << "," << it->second;
+  }
+
+  oss << " " << poolname << " " << imgname;
+
+  if (snapname) {
+    oss << " " << snapname;
+  }
+
+  // modprobe the rbd module if /sys/bus/rbd doesn't exist
+  struct stat sb;
+  if ((stat("/sys/bus/rbd", &sb) < 0) || (!S_ISDIR(sb.st_mode))) {
+    // turn on single-major device number allocation scheme if the
+    // kernel supports it
+    const char *cmd = "/sbin/modprobe rbd";
+    r = system("/sbin/modinfo -F parm rbd | /bin/grep -q ^single_major:");
+    if (r == 0) {
+      cmd = "/sbin/modprobe rbd single_major=Y";
+    } else if (r < 0) {
+      cerr << "rbd: error executing modinfo as shell command!" << std::endl;
+    }
+
+    r = system(cmd);
+    if (r) {
+      if (r < 0)
+        cerr << "rbd: error executing modprobe as shell command!" << std::endl;
+      else
+        cerr << "rbd: modprobe rbd failed! (" << r << ")" < device_dir(opendir(devices_path), do_closedir);
+  if (!device_dir.get()) {
+    r = -errno;
+    cerr << "rbd: could not open " << devices_path << ": "
+	 << cpp_strerror(-r) << std::endl;
+    return r;
+  }
+
+  struct dirent *dent;
+  dent = readdir(device_dir.get());
+  if (!dent) {
+    r = -errno;
+    cerr << "rbd: error reading " << devices_path << ": "
+	 << cpp_strerror(-r) << std::endl;
+    return r;
+  }
+
+  if (f) {
+    f->open_object_section("devices");
+  } else {
+    tbl.define_column("id", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("pool", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("image", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("snap", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("device", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  do {
+    if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)
+      continue;
+
+    char fn[PATH_MAX];
+
+    char dev[PATH_MAX];
+    snprintf(dev, sizeof(dev), "/dev/rbd%s", dent->d_name);
+
+    char name[RBD_MAX_IMAGE_NAME_SIZE];
+    snprintf(fn, sizeof(fn), "%s/%s/name", devices_path, dent->d_name);
+    r = read_file(fn, name, sizeof(name));
+    if (r < 0) {
+      cerr << "rbd: could not read name from " << fn << ": "
+	   << cpp_strerror(-r) << std::endl;
+      continue;
+    }
+
+    char pool[4096];
+    snprintf(fn, sizeof(fn), "%s/%s/pool", devices_path, dent->d_name);
+    r = read_file(fn, pool, sizeof(pool));
+    if (r < 0) {
+      cerr << "rbd: could not read name from " << fn << ": "
+	   << cpp_strerror(-r) << std::endl;
+      continue;
+    }
+
+    char snap[4096];
+    snprintf(fn, sizeof(fn), "%s/%s/current_snap", devices_path, dent->d_name);
+    r = read_file(fn, snap, sizeof(snap));
+    if (r < 0) {
+      cerr << "rbd: could not read name from " << fn << ": "
+	   << cpp_strerror(-r) << std::endl;
+      continue;
+    }
+
+    if (f) {
+      f->open_object_section(dent->d_name);
+      f->dump_string("pool", pool);
+      f->dump_string("name", name);
+      f->dump_string("snap", snap);
+      f->dump_string("device", dev);
+      f->close_section();
+    } else {
+      tbl << dent->d_name << pool << name << snap << dev << TextTable::endrow;
+    }
+    have_output = true;
+
+  } while ((dent = readdir(device_dir.get())));
+
+  if (f) {
+    f->close_section();
+    f->flush(cout);
+  } else {
+    if (have_output)
+      cout << tbl;
+  }
+
+  return 0;
+}
+
+static int get_rbd_seq(dev_t devno, string &seq)
+{
+  // convert devno, which might be a partition major:minor pair, into
+  // a whole disk major:minor pair
+  dev_t wholediskno;
+  int r = blkid_devno_to_wholedisk(devno, NULL, 0, &wholediskno);
+  if (r) {
+    cerr << "rbd: could not compute wholediskno: " << r << std::endl;
+    // ignore the error: devno == wholediskno most of the time, and if
+    // it turns out it's not we will fail with -ENOENT later anyway
+    wholediskno = devno;
+  }
+
+  const char *devices_path = "/sys/bus/rbd/devices";
+  DIR *device_dir = opendir(devices_path);
+  if (!device_dir) {
+    r = -errno;
+    cerr << "rbd: could not open " << devices_path << ": " << cpp_strerror(-r)
+	 << std::endl;
+    return r;
+  }
+
+  struct dirent *dent;
+  dent = readdir(device_dir);
+  if (!dent) {
+    r = -errno;
+    cerr << "Error reading " << devices_path << ": " << cpp_strerror(-r)
+	 << std::endl;
+    closedir(device_dir);
+    return r;
+  }
+
+  int match_minor = -1;
+  do {
+    char fn[strlen(devices_path) + strlen(dent->d_name) + strlen("//major") + 1];
+    char buf[32];
+
+    if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)
+      continue;
+
+    snprintf(fn, sizeof(fn), "%s/%s/major", devices_path, dent->d_name);
+    r = read_file(fn, buf, sizeof(buf));
+    if (r < 0) {
+      cerr << "rbd: could not read major number from " << fn << ": "
+	   << cpp_strerror(-r) << std::endl;
+      continue;
+    }
+    string err;
+    int cur_major = strict_strtol(buf, 10, &err);
+    if (!err.empty()) {
+      cerr << err << std::endl;
+      cerr << "rbd: could not parse major number read from " << fn << ": "
+           << cpp_strerror(-r) << std::endl;
+      continue;
+    }
+    if (cur_major != (int)major(wholediskno))
+      continue;
+
+    if (match_minor == -1) {
+      // matching minors in addition to majors is not necessary unless
+      // single-major scheme is turned on, but, if the kernel supports
+      // it, do it anyway (blkid stuff above ensures that we always have
+      // the correct minor to match with)
+      struct stat sbuf;
+      snprintf(fn, sizeof(fn), "%s/%s/minor", devices_path, dent->d_name);
+      match_minor = (stat(fn, &sbuf) == 0);
+    }
+
+    if (match_minor == 1) {
+      snprintf(fn, sizeof(fn), "%s/%s/minor", devices_path, dent->d_name);
+      r = read_file(fn, buf, sizeof(buf));
+      if (r < 0) {
+        cerr << "rbd: could not read minor number from " << fn << ": "
+             << cpp_strerror(-r) << std::endl;
+        continue;
+      }
+      int cur_minor = strict_strtol(buf, 10, &err);
+      if (!err.empty()) {
+        cerr << err << std::endl;
+        cerr << "rbd: could not parse minor number read from " << fn << ": "
+             << cpp_strerror(-r) << std::endl;
+        continue;
+      }
+      if (cur_minor != (int)minor(wholediskno))
+        continue;
+    } else {
+      assert(match_minor == 0);
+    }
+
+    seq = string(dent->d_name);
+    closedir(device_dir);
+    return 0;
+  } while ((dent = readdir(device_dir)));
+
+  closedir(device_dir);
+  return -ENOENT;
+}
+
+static int do_kernel_rm(const char *dev)
+{
+  struct stat sbuf;
+  if (stat(dev, &sbuf) || !S_ISBLK(sbuf.st_mode)) {
+    cerr << "rbd: " << dev << " is not a block device" << std::endl;
+    return -EINVAL;
+  }
+
+  string seq_num;
+  int r = get_rbd_seq(sbuf.st_rdev, seq_num);
+  if (r == -ENOENT) {
+    cerr << "rbd: " << dev << " is not an rbd device" << std::endl;
+    return -EINVAL;
+  }
+  if (r < 0)
+    return r;
+
+  // let udevadm do its job *before* we try to unmap
+  if (udevadm_settle) {
+    int r = system("/sbin/udevadm settle");
+    if (r) {
+      if (r < 0)
+        cerr << "rbd: error executing udevadm as shell command!" << std::endl;
+      else
+        cerr << "rbd: '/sbin/udevadm settle' failed! (" << r << ")" <::const_iterator it = map_options.find(key);
+  if (it != map_options.end()) {
+    cerr << "rbd: warning: redefining map option " << key << ": '"
+         << it->second << "' -> '" << val << "'" << std::endl;
+  }
+  map_options[key] = val;
+}
+
+static int put_map_option_value(const string opt, const char *value_char,
+                                string (*parse_cb)(const char *))
+{
+  if (!value_char || *value_char == '\0') {
+    cerr << "rbd: " << opt << " option requires a value" << std::endl;
+    return 1;
+  }
+
+  string value = parse_cb(value_char);
+  if (value.empty()) {
+    cerr << "rbd: invalid " << opt << " value '" << value_char << "'"
+         << std::endl;
+    return 1;
+  }
+
+  put_map_option(opt, opt + "=" + value);
+  return 0;
+}
+
+static int parse_map_options(char *options)
+{
+  for (char *this_char = strtok(options, ", ");
+       this_char != NULL;
+       this_char = strtok(NULL, ",")) {
+    char *value_char;
+
+    if ((value_char = strchr(this_char, '=')) != NULL)
+      *value_char++ = '\0';
+
+    if (!strcmp(this_char, "fsid")) {
+      if (put_map_option_value("fsid", value_char, map_option_uuid_cb))
+        return 1;
+    } else if (!strcmp(this_char, "ip")) {
+      if (put_map_option_value("ip", value_char, map_option_ip_cb))
+        return 1;
+    } else if (!strcmp(this_char, "share") || !strcmp(this_char, "noshare")) {
+      put_map_option("share", this_char);
+    } else if (!strcmp(this_char, "crc") || !strcmp(this_char, "nocrc")) {
+      put_map_option("crc", this_char);
+    } else if (!strcmp(this_char, "mount_timeout")) {
+      if (put_map_option_value("mount_timeout", value_char, map_option_int_cb))
+        return 1;
+    } else if (!strcmp(this_char, "osdkeepalive")) {
+      if (put_map_option_value("osdkeepalive", value_char, map_option_int_cb))
+        return 1;
+    } else if (!strcmp(this_char, "osd_idle_ttl")) {
+      if (put_map_option_value("osd_idle_ttl", value_char, map_option_int_cb))
+        return 1;
+    } else if (!strcmp(this_char, "rw") || !strcmp(this_char, "ro")) {
+      put_map_option("rw", this_char);
+    } else {
+      cerr << "rbd: unknown map option '" << this_char << "'" << std::endl;
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+enum {
+  OPT_NO_CMD = 0,
+  OPT_LIST,
+  OPT_INFO,
+  OPT_CREATE,
+  OPT_CLONE,
+  OPT_FLATTEN,
+  OPT_CHILDREN,
+  OPT_RESIZE,
+  OPT_RM,
+  OPT_EXPORT,
+  OPT_EXPORT_DIFF,
+  OPT_DIFF,
+  OPT_IMPORT,
+  OPT_IMPORT_DIFF,
+  OPT_COPY,
+  OPT_RENAME,
+  OPT_SNAP_CREATE,
+  OPT_SNAP_ROLLBACK,
+  OPT_SNAP_REMOVE,
+  OPT_SNAP_LIST,
+  OPT_SNAP_PURGE,
+  OPT_SNAP_PROTECT,
+  OPT_SNAP_UNPROTECT,
+  OPT_WATCH,
+  OPT_MAP,
+  OPT_UNMAP,
+  OPT_SHOWMAPPED,
+  OPT_LOCK_LIST,
+  OPT_LOCK_ADD,
+  OPT_LOCK_REMOVE,
+  OPT_BENCH_WRITE,
+};
+
+static int get_cmd(const char *cmd, bool snapcmd, bool lockcmd)
+{
+  if (!snapcmd && !lockcmd) {
+    if (strcmp(cmd, "ls") == 0 ||
+        strcmp(cmd, "list") == 0)
+      return OPT_LIST;
+    if (strcmp(cmd, "info") == 0)
+      return OPT_INFO;
+    if (strcmp(cmd, "create") == 0)
+      return OPT_CREATE;
+    if (strcmp(cmd, "clone") == 0)
+      return OPT_CLONE;
+    if (strcmp(cmd, "flatten") == 0)
+      return OPT_FLATTEN;
+    if (strcmp(cmd, "children") == 0)
+      return OPT_CHILDREN;
+    if (strcmp(cmd, "resize") == 0)
+      return OPT_RESIZE;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_RM;
+    if (strcmp(cmd, "export") == 0)
+      return OPT_EXPORT;
+    if (strcmp(cmd, "export-diff") == 0)
+      return OPT_EXPORT_DIFF;
+    if (strcmp(cmd, "diff") == 0)
+      return OPT_DIFF;
+    if (strcmp(cmd, "import") == 0)
+      return OPT_IMPORT;
+    if (strcmp(cmd, "import-diff") == 0)
+      return OPT_IMPORT_DIFF;
+    if (strcmp(cmd, "copy") == 0 ||
+        strcmp(cmd, "cp") == 0)
+      return OPT_COPY;
+    if (strcmp(cmd, "rename") == 0 ||
+        strcmp(cmd, "mv") == 0)
+      return OPT_RENAME;
+    if (strcmp(cmd, "watch") == 0)
+      return OPT_WATCH;
+    if (strcmp(cmd, "map") == 0)
+      return OPT_MAP;
+    if (strcmp(cmd, "showmapped") == 0)
+      return OPT_SHOWMAPPED;
+    if (strcmp(cmd, "unmap") == 0)
+      return OPT_UNMAP;
+    if (strcmp(cmd, "bench-write") == 0)
+      return OPT_BENCH_WRITE;
+  } else if (snapcmd) {
+    if (strcmp(cmd, "create") == 0 ||
+        strcmp(cmd, "add") == 0)
+      return OPT_SNAP_CREATE;
+    if (strcmp(cmd, "rollback") == 0 ||
+        strcmp(cmd, "revert") == 0)
+      return OPT_SNAP_ROLLBACK;
+    if (strcmp(cmd, "remove") == 0 ||
+        strcmp(cmd, "rm") == 0)
+      return OPT_SNAP_REMOVE;
+    if (strcmp(cmd, "ls") == 0 ||
+        strcmp(cmd, "list") == 0)
+      return OPT_SNAP_LIST;
+    if (strcmp(cmd, "purge") == 0)
+      return OPT_SNAP_PURGE;
+    if (strcmp(cmd, "protect") == 0)
+      return OPT_SNAP_PROTECT;
+    if (strcmp(cmd, "unprotect") == 0)
+      return OPT_SNAP_UNPROTECT;
+  } else {
+    if (strcmp(cmd, "ls") == 0 ||
+        strcmp(cmd, "list") == 0)
+      return OPT_LOCK_LIST;
+    if (strcmp(cmd, "add") == 0)
+      return OPT_LOCK_ADD;
+    if (strcmp(cmd, "remove") == 0 ||
+	strcmp(cmd, "rm") == 0)
+      return OPT_LOCK_REMOVE;
+  }
+
+  return OPT_NO_CMD;
+}
+
+/*
+ * Called 1-N times depending on how many args the command needs.  If
+ * the positional varN is already set, set the next one; this handles
+ * both --args above and unadorned args below.  Calling with all args
+ * filled is an error.
+ */
+static bool set_conf_param(const char *param, const char **var1,
+			   const char **var2, const char **var3)
+{
+  if (!*var1)
+    *var1 = param;
+  else if (var2 && !*var2)
+    *var2 = param;
+  else if (var3 && !*var3)
+    *var3 = param;
+  else
+    return false;
+  return true;
+}
+
+bool size_set;
+
+int main(int argc, const char **argv)
+{
+  librados::Rados rados;
+  librbd::RBD rbd;
+  librados::IoCtx io_ctx, dest_io_ctx;
+  librbd::Image image;
+
+  vector args;
+
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  int opt_cmd = OPT_NO_CMD;
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+
+  const char *poolname = NULL;
+  uint64_t size = 0;  // in bytes
+  int order = 0;
+  bool format_specified = false, output_format_specified = false;
+  int format = 1;
+  uint64_t features = RBD_FEATURE_LAYERING;
+  const char *imgname = NULL, *snapname = NULL, *destname = NULL,
+    *dest_poolname = NULL, *dest_snapname = NULL, *path = NULL,
+    *devpath = NULL, *lock_cookie = NULL, *lock_client = NULL,
+    *lock_tag = NULL, *output_format = "plain",
+    *fromsnapname = NULL;
+  bool lflag = false;
+  int pretty_format = 0;
+  long long stripe_unit = 0, stripe_count = 0;
+  long long bench_io_size = 4096, bench_io_threads = 16, bench_bytes = 1 << 30;
+  string bench_pattern = "seq";
+
+  std::string val;
+  std::ostringstream err;
+  long long sizell = 0;
+  std::vector::iterator i;
+  for (i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_witharg(args, i, &val, "--secret", (char*)NULL)) {
+      int r = g_conf->set_val("keyfile", val.c_str());
+      assert(r == 0);
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+      return 0;
+    } else if (ceph_argparse_flag(args, i, "--new-format", (char*)NULL)) {
+      format = 2;
+      format_specified = true;
+    } else if (ceph_argparse_withint(args, i, &format, &err, "--image-format",
+				     (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << "rbd: " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+      format_specified = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "-p", "--pool", (char*)NULL)) {
+      poolname = strdup(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--dest-pool", (char*)NULL)) {
+      dest_poolname = strdup(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--snap", (char*)NULL)) {
+      snapname = strdup(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--from-snap", (char*)NULL)) {
+      fromsnapname = strdup(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "-i", "--image", (char*)NULL)) {
+      imgname = strdup(val.c_str());
+    } else if (ceph_argparse_withlonglong(args, i, &sizell, &err, "-s", "--size", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << "rbd: " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+      if (sizell < 0) {
+	cerr << "rbd: size must be >= 0" << std::endl;
+	return EXIT_FAILURE;
+      }
+      size = sizell << 20;   // bytes to MB
+      size_set = true;
+    } else if (ceph_argparse_flag(args, i, "-l", "--long", (char*)NULL)) {
+      lflag = true;
+    } else if (ceph_argparse_withlonglong(args, i, &stripe_unit, &err, "--stripe-unit", (char*)NULL)) {
+    } else if (ceph_argparse_withlonglong(args, i, &stripe_count, &err, "--stripe-count", (char*)NULL)) {
+    } else if (ceph_argparse_withint(args, i, &order, &err, "--order", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << "rbd: " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_withlonglong(args, i, &bench_io_size, &err, "--io-size", (char*)NULL)) {
+    } else if (ceph_argparse_withlonglong(args, i, &bench_io_threads, &err, "--io-threads", (char*)NULL)) {
+    } else if (ceph_argparse_withlonglong(args, i, &bench_bytes, &err, "--io-total", (char*)NULL)) {
+    } else if (ceph_argparse_witharg(args, i, &bench_pattern, &err, "--io-pattern", (char*)NULL)) {
+    } else if (ceph_argparse_withlonglong(args, i, &stripe_count, &err, "--stripe-count", (char*)NULL)) {
+    } else if (ceph_argparse_witharg(args, i, &val, "--path", (char*)NULL)) {
+      path = strdup(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--dest", (char*)NULL)) {
+      destname = strdup(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--parent", (char *)NULL)) {
+      imgname = strdup(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--shared", (char *)NULL)) {
+      lock_tag = strdup(val.c_str());
+    } else if (ceph_argparse_flag(args, i, "--no-settle", (char *)NULL)) {
+      udevadm_settle = false;
+    } else if (ceph_argparse_witharg(args, i, &val, "-o", "--options", (char*)NULL)) {
+      char *map_options = strdup(val.c_str());
+      if (parse_map_options(map_options)) {
+        cerr << "rbd: couldn't parse map options" << std::endl;
+        return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_flag(args, i, "--read-only", (char *)NULL)) {
+      // --read-only is equivalent to -o ro
+      put_map_option("rw", "ro");
+    } else if (ceph_argparse_flag(args, i, "--no-progress", (char *)NULL)) {
+      progress = false;
+    } else if (ceph_argparse_flag(args, i , "--allow-shrink", (char *)NULL)) {
+      resize_allow_shrink = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--format", (char *) NULL)) {
+      std::string err;
+      long long ret = strict_strtoll(val.c_str(), 10, &err);
+      if (err.empty()) {
+	format = ret;
+	format_specified = true;
+	cerr << "rbd: using --format for specifying the rbd image format is"
+	     << " deprecated, use --image-format instead"
+	     << std::endl;
+      } else {
+	output_format = strdup(val.c_str());
+	output_format_specified = true;
+      }
+    } else if (ceph_argparse_binary_flag(args, i, &pretty_format, NULL, "--pretty-format", (char*)NULL)) {
+    } else {
+      ++i;
+    }
+  }
+
+  common_init_finish(g_ceph_context);
+
+  i = args.begin();
+  if (i == args.end()) {
+    cerr << "rbd: you must specify a command." << std::endl;
+    return EXIT_FAILURE;
+  } else if (strcmp(*i, "snap") == 0) {
+    i = args.erase(i);
+    if (i == args.end()) {
+      cerr << "rbd: which snap command do you want?" << std::endl;
+      return EXIT_FAILURE;
+    }
+    opt_cmd = get_cmd(*i, true, false);
+  } else if (strcmp(*i, "lock") == 0) {
+    i = args.erase(i);
+    if (i == args.end()) {
+      cerr << "rbd: which lock command do you want?" << std::endl;
+      return EXIT_FAILURE;
+    }
+    opt_cmd = get_cmd(*i, false, true);
+  } else {
+    opt_cmd = get_cmd(*i, false, false);
+  }
+  if (opt_cmd == OPT_NO_CMD) {
+    cerr << "rbd: error parsing command '" << *i << "'; -h or --help for usage" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  // loop across all remaining arguments; by command, accumulate any
+  // that are still missing into the appropriate variables, one at a
+  // time (i.e. SET_CONF_PARAM will be called N times for N remaining
+  // arguments).
+
+#define SET_CONF_PARAM(v, p1, p2, p3) \
+if (!set_conf_param(v, p1, p2, p3)) { \
+  cerr << "rbd: extraneous parameter " << v << std::endl; \
+  return EXIT_FAILURE; \
+}
+
+  for (i = args.erase(i); i != args.end(); ++i) {
+    const char *v = *i;
+    switch (opt_cmd) {
+      case OPT_LIST:
+	SET_CONF_PARAM(v, &poolname, NULL, NULL);
+	break;
+      case OPT_INFO:
+      case OPT_CREATE:
+      case OPT_FLATTEN:
+      case OPT_RESIZE:
+      case OPT_RM:
+      case OPT_SNAP_CREATE:
+      case OPT_SNAP_ROLLBACK:
+      case OPT_SNAP_REMOVE:
+      case OPT_SNAP_LIST:
+      case OPT_SNAP_PURGE:
+      case OPT_SNAP_PROTECT:
+      case OPT_SNAP_UNPROTECT:
+      case OPT_WATCH:
+      case OPT_MAP:
+      case OPT_BENCH_WRITE:
+      case OPT_LOCK_LIST:
+      case OPT_DIFF:
+	SET_CONF_PARAM(v, &imgname, NULL, NULL);
+	break;
+      case OPT_UNMAP:
+	SET_CONF_PARAM(v, &devpath, NULL, NULL);
+	break;
+      case OPT_EXPORT:
+      case OPT_EXPORT_DIFF:
+	SET_CONF_PARAM(v, &imgname, &path, NULL);
+	break;
+      case OPT_IMPORT:
+      case OPT_IMPORT_DIFF:
+	SET_CONF_PARAM(v, &path, &imgname, NULL);
+	break;
+      case OPT_COPY:
+      case OPT_RENAME:
+      case OPT_CLONE:
+	SET_CONF_PARAM(v, &imgname, &destname, NULL);
+	break;
+      case OPT_SHOWMAPPED:
+	cerr << "rbd: showmapped takes no parameters" << std::endl;
+	return EXIT_FAILURE;
+      case OPT_CHILDREN:
+	SET_CONF_PARAM(v, &imgname, NULL, NULL);
+	break;
+      case OPT_LOCK_ADD:
+	SET_CONF_PARAM(v, &imgname, &lock_cookie, NULL);
+	break;
+      case OPT_LOCK_REMOVE:
+	SET_CONF_PARAM(v, &imgname, &lock_client, &lock_cookie);
+	break;
+    default:
+	assert(0);
+	break;
+    }
+  }
+
+  if (format_specified && opt_cmd != OPT_IMPORT && opt_cmd != OPT_CREATE) {
+    cerr << "rbd: image format can only be set when "
+	 << "creating or importing an image" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (pretty_format && !strcmp(output_format, "plain")) {
+    cerr << "rbd: --pretty-format only works when --format is json or xml"
+	 << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  boost::scoped_ptr formatter;
+  if (output_format_specified && opt_cmd != OPT_SHOWMAPPED &&
+      opt_cmd != OPT_INFO && opt_cmd != OPT_LIST &&
+      opt_cmd != OPT_SNAP_LIST && opt_cmd != OPT_LOCK_LIST &&
+      opt_cmd != OPT_CHILDREN && opt_cmd != OPT_DIFF) {
+    cerr << "rbd: command doesn't use output formatting"
+	 << std::endl;
+    return EXIT_FAILURE;
+  } else if (get_outfmt(output_format, pretty_format, &formatter) < 0) {
+    return EXIT_FAILURE;
+  }
+
+  if (format_specified) {
+    if (format < 1 || format > 2) {
+      cerr << "rbd: image format must be 1 or 2" << std::endl;
+      return EXIT_FAILURE;
+    }
+  }
+
+  if (opt_cmd == OPT_EXPORT && !imgname) {
+    cerr << "rbd: image name was not specified" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if ((opt_cmd == OPT_IMPORT || opt_cmd == OPT_IMPORT_DIFF) && !path) {
+    cerr << "rbd: path was not specified" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (opt_cmd == OPT_IMPORT && !destname) {
+    destname = imgname;
+    if (!destname)
+      destname = imgname_from_path(path);
+    imgname = NULL;
+  }
+
+  if (opt_cmd != OPT_LOCK_ADD && lock_tag) {
+    cerr << "rbd: only the lock add command uses the --shared option"
+	 << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if ((opt_cmd == OPT_LOCK_ADD || opt_cmd == OPT_LOCK_REMOVE) &&
+      !lock_cookie) {
+    cerr << "rbd: lock id was not specified" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (opt_cmd != OPT_LIST &&
+      opt_cmd != OPT_IMPORT &&
+      opt_cmd != OPT_IMPORT_DIFF &&
+      opt_cmd != OPT_UNMAP &&
+      opt_cmd != OPT_SHOWMAPPED && !imgname) {
+    cerr << "rbd: image name was not specified" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (opt_cmd == OPT_UNMAP && !devpath) {
+    cerr << "rbd: device path was not specified" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  // do this unconditionally so we can parse pool/image@snapshot into
+  // the relevant parts
+  set_pool_image_name(poolname, imgname, (char **)&poolname,
+		      (char **)&imgname, (char **)&snapname);
+  if (snapname && opt_cmd != OPT_SNAP_CREATE && opt_cmd != OPT_SNAP_ROLLBACK &&
+      opt_cmd != OPT_SNAP_REMOVE && opt_cmd != OPT_INFO &&
+      opt_cmd != OPT_EXPORT && opt_cmd != OPT_EXPORT_DIFF && opt_cmd != OPT_DIFF && opt_cmd != OPT_COPY &&
+      opt_cmd != OPT_MAP && opt_cmd != OPT_CLONE &&
+      opt_cmd != OPT_SNAP_PROTECT && opt_cmd != OPT_SNAP_UNPROTECT &&
+      opt_cmd != OPT_CHILDREN) {
+    cerr << "rbd: snapname specified for a command that doesn't use it"
+	 << std::endl;
+    return EXIT_FAILURE;
+  }
+  if ((opt_cmd == OPT_SNAP_CREATE || opt_cmd == OPT_SNAP_ROLLBACK ||
+       opt_cmd == OPT_SNAP_REMOVE || opt_cmd == OPT_CLONE ||
+       opt_cmd == OPT_SNAP_PROTECT || opt_cmd == OPT_SNAP_UNPROTECT ||
+       opt_cmd == OPT_CHILDREN) && !snapname) {
+    cerr << "rbd: snap name was not specified" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  set_pool_image_name(dest_poolname, destname, (char **)&dest_poolname,
+		      (char **)&destname, (char **)&dest_snapname);
+
+  if (opt_cmd == OPT_IMPORT) {
+    if (poolname && dest_poolname) {
+      cerr << "rbd: source and destination pool both specified" << std::endl;
+      return EXIT_FAILURE;
+    }
+    if (imgname && destname) {
+      cerr << "rbd: source and destination image both specified" << std::endl;
+      return EXIT_FAILURE;
+    }
+    if (poolname)
+      dest_poolname = poolname;
+  }
+
+  if (!poolname)
+    poolname = "rbd";
+
+  if (!dest_poolname)
+    dest_poolname = "rbd";
+
+  if (opt_cmd == OPT_EXPORT && !path)
+    path = imgname;
+
+  if ((opt_cmd == OPT_COPY || opt_cmd == OPT_CLONE || opt_cmd == OPT_RENAME) &&
+      !destname ) {
+    cerr << "rbd: destination image name was not specified" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if ((opt_cmd == OPT_CLONE) && dest_snapname) {
+    cerr << "rbd: cannot clone to a snapshot" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if ((opt_cmd == OPT_CLONE) && size) {
+    cerr << "rbd: clone must begin at size of parent" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if ((opt_cmd == OPT_RENAME) && (strcmp(poolname, dest_poolname) != 0)) {
+    cerr << "rbd: mv/rename across pools not supported" << std::endl;
+    cerr << "source pool: " << poolname << " dest pool: " << dest_poolname
+      << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  bool talk_to_cluster = (opt_cmd != OPT_MAP &&
+			  opt_cmd != OPT_UNMAP &&
+			  opt_cmd != OPT_SHOWMAPPED);
+  if (talk_to_cluster && rados.init_with_context(g_ceph_context) < 0) {
+    cerr << "rbd: couldn't initialize rados!" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (talk_to_cluster && rados.connect() < 0) {
+    cerr << "rbd: couldn't connect to the cluster!" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  int r;
+  if (talk_to_cluster && opt_cmd != OPT_IMPORT) {
+    r = rados.ioctx_create(poolname, io_ctx);
+    if (r < 0) {
+      cerr << "rbd: error opening pool " << poolname << ": "
+	   << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+  }
+
+  if (imgname && talk_to_cluster &&
+      (opt_cmd == OPT_RESIZE || opt_cmd == OPT_SNAP_CREATE ||
+       opt_cmd == OPT_SNAP_ROLLBACK || opt_cmd == OPT_SNAP_REMOVE ||
+       opt_cmd == OPT_SNAP_PURGE || opt_cmd == OPT_SNAP_PROTECT ||
+       opt_cmd == OPT_SNAP_UNPROTECT || opt_cmd == OPT_WATCH ||
+       opt_cmd == OPT_FLATTEN || opt_cmd == OPT_LOCK_ADD ||
+       opt_cmd == OPT_LOCK_REMOVE || opt_cmd == OPT_BENCH_WRITE ||
+       opt_cmd == OPT_INFO || opt_cmd == OPT_SNAP_LIST ||
+       opt_cmd == OPT_IMPORT_DIFF ||
+       opt_cmd == OPT_EXPORT || opt_cmd == OPT_EXPORT_DIFF || opt_cmd == OPT_COPY ||
+       opt_cmd == OPT_DIFF ||
+       opt_cmd == OPT_CHILDREN || opt_cmd == OPT_LOCK_LIST)) {
+
+    if (opt_cmd == OPT_INFO || opt_cmd == OPT_SNAP_LIST ||
+	opt_cmd == OPT_EXPORT || opt_cmd == OPT_EXPORT || opt_cmd == OPT_COPY ||
+	opt_cmd == OPT_CHILDREN || opt_cmd == OPT_LOCK_LIST) {
+      r = rbd.open_read_only(io_ctx, image, imgname, NULL);
+    } else {
+      r = rbd.open(io_ctx, image, imgname);
+    }
+    if (r < 0) {
+      cerr << "rbd: error opening image " << imgname << ": "
+	   << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+  }
+
+  if (snapname && talk_to_cluster &&
+      (opt_cmd == OPT_INFO ||
+       opt_cmd == OPT_EXPORT ||
+       opt_cmd == OPT_EXPORT_DIFF ||
+       opt_cmd == OPT_DIFF ||
+       opt_cmd == OPT_COPY ||
+       opt_cmd == OPT_CHILDREN)) {
+    r = image.snap_set(snapname);
+    if (r < 0) {
+      cerr << "rbd: error setting snapshot context: " << cpp_strerror(-r)
+	   << std::endl;
+      return -r;
+    }
+  }
+
+  if (opt_cmd == OPT_COPY || opt_cmd == OPT_IMPORT || opt_cmd == OPT_CLONE) {
+    r = rados.ioctx_create(dest_poolname, dest_io_ctx);
+    if (r < 0) {
+      cerr << "rbd: error opening pool " << dest_poolname << ": "
+	   << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+  }
+
+  if (opt_cmd == OPT_CREATE || opt_cmd == OPT_RESIZE) {
+    if (!size_set) {
+      cerr << "rbd: must specify --size " << std::endl;
+      return EINVAL;
+    }
+  }
+
+  switch (opt_cmd) {
+  case OPT_LIST:
+    r = do_list(rbd, io_ctx, lflag, formatter.get());
+    if (r < 0) {
+      cerr << "rbd: list: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_CREATE:
+    if (order && (order < 12 || order > 25)) {
+      cerr << "rbd: order must be between 12 (4 KB) and 25 (32 MB)"
+	   << std::endl;
+      return EINVAL;
+    }
+    if ((stripe_unit && !stripe_count) || (!stripe_unit && stripe_count)) {
+      cerr << "must specify both (or neither) of stripe-unit and stripe-count"
+	   << std::endl;
+      usage();
+      return EINVAL;
+    }
+    r = do_create(rbd, io_ctx, imgname, size, &order, format, features,
+		  stripe_unit, stripe_count);
+    if (r < 0) {
+      cerr << "rbd: create error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_CLONE:
+    if (order && (order < 12 || order > 25)) {
+      cerr << "rbd: order must be between 12 (4 KB) and 25 (32 MB)"
+	   << std::endl;
+      return EINVAL;
+    }
+
+    r = do_clone(rbd, io_ctx, imgname, snapname, dest_io_ctx, destname,
+		 features, &order);
+    if (r < 0) {
+      cerr << "rbd: clone error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_FLATTEN:
+    r = do_flatten(image);
+    if (r < 0) {
+      cerr << "rbd: flatten error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_RENAME:
+    r = do_rename(rbd, io_ctx, imgname, destname);
+    if (r < 0) {
+      cerr << "rbd: rename error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_INFO:
+    r = do_show_info(imgname, image, snapname, formatter.get());
+    if (r < 0) {
+      cerr << "rbd: info: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_RM:
+    r = do_delete(rbd, io_ctx, imgname);
+    if (r < 0) {
+      if (r == -ENOTEMPTY) {
+	cerr << "rbd: image has snapshots - these must be deleted"
+	     << " with 'rbd snap purge' before the image can be removed."
+	     << std::endl;
+      } else if (r == -EBUSY) {
+	cerr << "rbd: error: image still has watchers"
+	     << std::endl
+	     << "This means the image is still open or the client using "
+	     << "it crashed. Try again after closing/unmapping it or "
+	     << "waiting 30s for the crashed client to timeout."
+	     << std::endl;
+      } else {
+	cerr << "rbd: delete error: " << cpp_strerror(-r) << std::endl;
+      }
+      return -r ;
+    }
+    break;
+
+  case OPT_RESIZE:
+    librbd::image_info_t info;
+    r = image.stat(info, sizeof(info));
+    if (r < 0) {
+      cerr << "rbd: resize error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+
+    if (info.size > size && !resize_allow_shrink) {
+      cerr << "rbd: shrinking an image is only allowed with the --allow-shrink flag" << std::endl;
+      return EINVAL;
+    }
+
+    r = do_resize(image, size);
+    if (r < 0) {
+      cerr << "rbd: resize error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_SNAP_LIST:
+    if (!imgname) {
+      cerr << "rbd: snap list requires an image parameter" << std::endl;
+      return EXIT_FAILURE;
+    }
+    r = do_list_snaps(image, formatter.get());
+    if (r < 0) {
+      cerr << "rbd: failed to list snapshots: " << cpp_strerror(-r)
+	   << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_SNAP_CREATE:
+    if (!imgname || !snapname) {
+      cerr << "rbd: snap create requires image and snapname" << std::endl;
+      return EINVAL;
+    }
+    r = do_add_snap(image, snapname);
+    if (r < 0) {
+      cerr << "rbd: failed to create snapshot: " << cpp_strerror(-r)
+	   << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_SNAP_ROLLBACK:
+    if (!imgname) {
+      cerr << "rbd: snap rollback requires image name" << std::endl;
+      return EINVAL;
+    }
+    r = do_rollback_snap(image, snapname);
+    if (r < 0) {
+      cerr << "rbd: rollback failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_SNAP_REMOVE:
+    if (!imgname) {
+      cerr << "rbd: snap remove requires image name" << std::endl;
+      return EINVAL;
+    }
+    r = do_remove_snap(image, snapname);
+    if (r < 0) {
+      if (r == -EBUSY) {
+        cerr << "rbd: snapshot '" << snapname << "' is protected from removal."
+             << std::endl;
+      } else {
+        cerr << "rbd: failed to remove snapshot: " << cpp_strerror(-r)
+             << std::endl;
+      }
+      return -r;
+    }
+    break;
+
+  case OPT_SNAP_PURGE:
+    if (!imgname) {
+      cerr << "rbd: snap purge requires image name" << std::endl;
+      return EINVAL;
+    }
+    r = do_purge_snaps(image);
+    if (r < 0) {
+      cerr << "rbd: removing snaps failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_SNAP_PROTECT:
+    if (!imgname) {
+      cerr << "rbd: snap protect requires image name" << std::endl;
+      return EINVAL;
+    }
+    r = do_protect_snap(image, snapname);
+    if (r < 0) {
+      cerr << "rbd: protecting snap failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_SNAP_UNPROTECT:
+    if (!imgname) {
+      cerr << "rbd: snap unprotect requires image name" << std::endl;
+      return EINVAL;
+    }
+    r = do_unprotect_snap(image, snapname);
+    if (r < 0) {
+      cerr << "rbd: unprotecting snap failed: " << cpp_strerror(-r)
+	   << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_CHILDREN:
+    r = do_list_children(image, formatter.get());
+    if (r < 0) {
+      cerr << "rbd: listing children failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_EXPORT:
+    if (!path) {
+      cerr << "rbd: export requires pathname" << std::endl;
+      return EINVAL;
+    }
+    r = do_export(image, path);
+    if (r < 0) {
+      cerr << "rbd: export error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_DIFF:
+    r = do_diff(image, fromsnapname, formatter.get());
+    if (r < 0) {
+      cerr << "rbd: diff error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_EXPORT_DIFF:
+    if (!path) {
+      cerr << "rbd: export-diff requires pathname" << std::endl;
+      return EINVAL;
+    }
+    r = do_export_diff(image, fromsnapname, snapname, path);
+    if (r < 0) {
+      cerr << "rbd: export-diff error: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_IMPORT:
+    if (!path) {
+      cerr << "rbd: import requires pathname" << std::endl;
+      return EINVAL;
+    }
+    r = do_import(rbd, dest_io_ctx, destname, &order, path,
+		  format, features, size);
+    if (r < 0) {
+      cerr << "rbd: import failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_IMPORT_DIFF:
+    assert(path);
+    r = do_import_diff(image, path);
+    if (r < 0) {
+      cerr << "rbd: import-diff failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_COPY:
+    r = do_copy(image, dest_io_ctx, destname);
+    if (r < 0) {
+      cerr << "rbd: copy failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_WATCH:
+    r = do_watch(io_ctx, imgname);
+    if (r < 0) {
+      cerr << "rbd: watch failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_MAP:
+    r = do_kernel_add(poolname, imgname, snapname);
+    if (r < 0) {
+      cerr << "rbd: add failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_UNMAP:
+    r = do_kernel_rm(devpath);
+    if (r < 0) {
+      cerr << "rbd: remove failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_SHOWMAPPED:
+    r = do_kernel_showmapped(formatter.get());
+    if (r < 0) {
+      cerr << "rbd: showmapped failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_LOCK_LIST:
+    r = do_lock_list(image, formatter.get());
+    if (r < 0) {
+      cerr << "rbd: listing locks failed: " << cpp_strerror(r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_LOCK_ADD:
+    r = do_lock_add(image, lock_cookie, lock_tag);
+    if (r < 0) {
+      if (r == -EBUSY || r == -EEXIST) {
+	if (lock_tag) {
+	  cerr << "rbd: lock is alrady held by someone else"
+	       << " with a different tag" << std::endl;
+	} else {
+	  cerr << "rbd: lock is already held by someone else" << std::endl;
+	}
+      } else {
+	cerr << "rbd: taking lock failed: " << cpp_strerror(r) << std::endl;
+      }
+      return -r;
+    }
+    break;
+
+  case OPT_LOCK_REMOVE:
+    r = do_lock_remove(image, lock_cookie, lock_client);
+    if (r < 0) {
+      cerr << "rbd: releasing lock failed: " << cpp_strerror(r) << std::endl;
+      return -r;
+    }
+    break;
+
+  case OPT_BENCH_WRITE:
+    r = do_bench_write(image, bench_io_size, bench_io_threads, bench_bytes, bench_pattern);
+    if (r < 0) {
+      cerr << "bench-write failed: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+    break;
+  }
+
+  return 0;
+}
diff --git a/ceph/src/rbd_fuse/rbd-fuse.c b/ceph/src/rbd_fuse/rbd-fuse.c
new file mode 100644
index 00000000..a13ca44e
--- /dev/null
+++ b/ceph/src/rbd_fuse/rbd-fuse.c
@@ -0,0 +1,764 @@
+/*
+ * rbd-fuse
+ */
+#define FUSE_USE_VERSION 30
+
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/rbd/librbd.h"
+
+static int gotrados = 0;
+char *pool_name;
+rados_t cluster;
+rados_ioctx_t ioctx;
+
+static pthread_mutex_t readdir_lock;
+
+struct rbd_stat {
+	u_char valid;
+	rbd_image_info_t rbd_info;
+};
+
+struct rbd_options {
+	char *ceph_config;
+	char *pool_name;
+};
+
+struct rbd_image {
+	char *image_name;
+	struct rbd_image *next;
+};
+struct rbd_image *rbd_images;
+
+struct rbd_openimage {
+	char *image_name;
+	rbd_image_t image;
+	struct rbd_stat rbd_stat;
+};
+#define MAX_RBD_IMAGES		128
+struct rbd_openimage opentbl[MAX_RBD_IMAGES];
+
+struct rbd_options rbd_options = {"/etc/ceph/ceph.conf", "rbd"};
+
+#define rbdsize(fd)	opentbl[fd].rbd_stat.rbd_info.size
+#define rbdblksize(fd)	opentbl[fd].rbd_stat.rbd_info.obj_size
+#define rbdblkcnt(fd)	opentbl[fd].rbd_stat.rbd_info.num_objs
+
+uint64_t imagesize = 1024ULL * 1024 * 1024;
+uint64_t imageorder = 22ULL;
+uint64_t imagefeatures = 1ULL;
+
+// Minimize calls to rbd_list: marks bracketing of opendir//releasedir
+int in_opendir;
+
+/* prototypes */
+int connect_to_cluster(rados_t *pcluster);
+void enumerate_images(struct rbd_image **head);
+int open_rbd_image(const char *image_name);
+int find_openrbd(const char *path);
+
+void simple_err(const char *msg, int err);
+
+void
+enumerate_images(struct rbd_image **head)
+{
+	char *ibuf = NULL;
+	size_t ibuf_len = 0;
+	struct rbd_image *im, *next;
+	char *ip;
+	int ret;
+
+	if (*head != NULL) {
+		for (im = *head; im != NULL;) {
+			next = im->next;
+			free(im);
+			im = next;
+		}
+		*head = NULL;
+	}
+
+	ret = rbd_list(ioctx, ibuf, &ibuf_len);
+	if (ret == -ERANGE) {
+		assert(ibuf_len > 0);
+		ibuf = malloc(ibuf_len);
+		if (!ibuf) {
+			simple_err("Failed to get ibuf", -ENOMEM);
+			return;
+		}
+	} else if (ret < 0) {
+		simple_err("Failed to get ibuf_len", ret);
+		return;
+	}
+
+	ret = rbd_list(ioctx, ibuf, &ibuf_len);
+	if (ret < 0) {
+		simple_err("Failed to populate ibuf", ret);
+		free(ibuf);
+		return;
+	}
+	assert(ret == (int)ibuf_len);
+
+	fprintf(stderr, "pool %s: ", pool_name);
+	for (ip = ibuf; ip < &ibuf[ibuf_len]; ip += strlen(ip) + 1)  {
+		fprintf(stderr, "%s, ", ip);
+		im = malloc(sizeof(*im));
+		im->image_name = ip;
+		im->next = *head;
+		*head = im;
+	}
+	fprintf(stderr, "\n");
+}
+
+int
+find_openrbd(const char *path)
+{
+	int i;
+
+	/* find in opentbl[] entry if already open */
+	for (i = 0; i < MAX_RBD_IMAGES; i++) {
+		if ((opentbl[i].image_name != NULL) &&
+		    (strcmp(opentbl[i].image_name, path) == 0)) {
+			return i;
+		}
+	}
+	return -1;
+}
+
+int
+open_rbd_image(const char *image_name)
+{
+	struct rbd_image *im;
+	struct rbd_openimage *rbd;
+	int fd;
+
+	if (image_name == (char *)NULL) 
+		return -1;
+
+	// relies on caller to keep rbd_images up to date
+	for (im = rbd_images; im != NULL; im = im->next) {
+		if (strcmp(im->image_name, image_name) == 0) {
+			break;
+		}
+	}
+	if (im == NULL)
+		return -1;
+
+	/* find in opentbl[] entry if already open */
+	if ((fd = find_openrbd(image_name)) != -1) {
+		rbd = &opentbl[fd];
+	} else {
+		int i;
+		// allocate an opentbl[] and open the image
+		for (i = 0; i < MAX_RBD_IMAGES; i++) {
+			if (opentbl[i].image == NULL) {
+				fd = i;
+				rbd = &opentbl[fd];
+				rbd->image_name = strdup(image_name);
+				break;
+			}
+		}
+		if (i == MAX_RBD_IMAGES)
+			return -1;
+		int ret = rbd_open(ioctx, rbd->image_name, &(rbd->image), NULL);
+		if (ret < 0) {
+			simple_err("open_rbd_image: can't open: ", ret);
+			return ret;
+		}
+	}
+	rbd_stat(rbd->image, &(rbd->rbd_stat.rbd_info),
+		 sizeof(rbd_image_info_t));
+	rbd->rbd_stat.valid = 1;
+	return fd;
+}
+
+static void
+iter_images(void *cookie,
+	    void (*iter)(void *cookie, const char *image))
+{
+	struct rbd_image *im;
+
+	pthread_mutex_lock(&readdir_lock);
+
+	for (im = rbd_images; im != NULL; im = im->next)
+		iter(cookie, im->image_name);
+	pthread_mutex_unlock(&readdir_lock);
+}
+
+static void count_images_cb(void *cookie, const char *image)
+{
+	(*((unsigned int *)cookie))++;
+}
+
+static int count_images(void)
+{
+	unsigned int count = 0;
+
+	pthread_mutex_lock(&readdir_lock);
+	enumerate_images(&rbd_images);
+	pthread_mutex_unlock(&readdir_lock);
+
+	iter_images(&count, count_images_cb);
+	return count;
+}
+
+static int rbdfs_getattr(const char *path, struct stat *stbuf)
+{
+	int fd;
+	time_t now;
+
+	if (!gotrados)
+		return -ENXIO;
+
+	if (path[0] == 0)
+		return -ENOENT;
+
+	memset(stbuf, 0, sizeof(struct stat));
+
+	if (strcmp(path, "/") == 0) {
+
+		now = time(NULL);
+		stbuf->st_mode = S_IFDIR + 0755;
+		stbuf->st_nlink = 2+count_images();
+		stbuf->st_uid = getuid();
+		stbuf->st_gid = getgid();
+		stbuf->st_size = 1024;
+		stbuf->st_blksize = 1024;
+		stbuf->st_blocks = 1;
+		stbuf->st_atime = now;
+		stbuf->st_mtime = now;
+		stbuf->st_ctime = now;
+
+		return 0;
+	}
+
+	if (!in_opendir) {
+		pthread_mutex_lock(&readdir_lock);
+		enumerate_images(&rbd_images);
+		pthread_mutex_unlock(&readdir_lock);
+	}
+	fd = open_rbd_image(path + 1);
+	if (fd < 0)
+		return -ENOENT;
+
+	now = time(NULL);
+	stbuf->st_mode = S_IFREG | 0666;
+	stbuf->st_nlink = 1;
+	stbuf->st_uid = getuid();
+	stbuf->st_gid = getgid();
+	stbuf->st_size = rbdsize(fd);
+	stbuf->st_blksize = rbdblksize(fd);
+	stbuf->st_blocks = rbdblkcnt(fd);
+	stbuf->st_atime = now;
+	stbuf->st_mtime = now;
+	stbuf->st_ctime = now;
+
+	return 0;
+}
+
+
+static int rbdfs_open(const char *path, struct fuse_file_info *fi)
+{
+	int fd;
+
+	if (!gotrados)
+		return -ENXIO;
+
+	if (path[0] == 0)
+		return -ENOENT;
+
+	pthread_mutex_lock(&readdir_lock);
+	enumerate_images(&rbd_images);
+	pthread_mutex_unlock(&readdir_lock);
+	fd = open_rbd_image(path + 1);
+	if (fd < 0)
+		return -ENOENT;
+
+	fi->fh = fd;
+	return 0;
+}
+
+static int rbdfs_read(const char *path, char *buf, size_t size,
+			off_t offset, struct fuse_file_info *fi)
+{
+	size_t numread;
+	struct rbd_openimage *rbd;
+
+	if (!gotrados)
+		return -ENXIO;
+
+	rbd = &opentbl[fi->fh];
+	numread = 0;
+	while (size > 0) {
+		ssize_t ret;
+
+		ret = rbd_read(rbd->image, offset, size, buf);
+
+		if (ret <= 0)
+			break;
+		buf += ret;
+		size -= ret;
+		offset += ret;
+		numread += ret;
+	}
+
+	return numread;
+}
+
+static int rbdfs_write(const char *path, const char *buf, size_t size,
+			 off_t offset, struct fuse_file_info *fi)
+{
+	size_t numwritten;
+	struct rbd_openimage *rbd;
+
+	if (!gotrados)
+		return -ENXIO;
+
+	rbd = &opentbl[fi->fh];
+	numwritten = 0;
+	while (size > 0) {
+		ssize_t ret;
+
+		if (offset + size > rbdsize(fi->fh)) {
+			int r;
+			fprintf(stderr, "rbdfs_write resizing %s to 0x%"PRIxMAX"\n",
+				path, offset+size);
+			r = rbd_resize(rbd->image, offset+size);
+			if (r < 0)
+				return r;
+
+			r = rbd_stat(rbd->image, &(rbd->rbd_stat.rbd_info),
+				 sizeof(rbd_image_info_t));
+			if (r < 0)
+				return r;
+		}
+		ret = rbd_write(rbd->image, offset, size, buf);
+
+		if (ret < 0)
+			break;
+		buf += ret;
+		size -= ret;
+		offset += ret;
+		numwritten += ret;
+	}
+
+	return numwritten;
+}
+
+static void rbdfs_statfs_image_cb(void *num, const char *image)
+{
+	int	fd;
+
+	((uint64_t *)num)[0]++;
+
+	fd = open_rbd_image(image);
+	if (fd >= 0)
+		((uint64_t *)num)[1] += rbdsize(fd);
+}
+
+static int rbdfs_statfs(const char *path, struct statvfs *buf)
+{
+	uint64_t num[2];
+
+	if (!gotrados)
+		return -ENXIO;
+
+	num[0] = 1;
+	num[1] = 0;
+	pthread_mutex_lock(&readdir_lock);
+	enumerate_images(&rbd_images);
+	pthread_mutex_unlock(&readdir_lock);
+	iter_images(num, rbdfs_statfs_image_cb);
+
+#define	RBDFS_BSIZE	4096
+	buf->f_bsize = RBDFS_BSIZE;
+	buf->f_frsize = RBDFS_BSIZE;
+	buf->f_blocks = num[1] / RBDFS_BSIZE;
+	buf->f_bfree = 0;
+	buf->f_bavail = 0;
+	buf->f_files = num[0];
+	buf->f_ffree = 0;
+	buf->f_favail = 0;
+	buf->f_fsid = 0;
+	buf->f_flag = 0;
+	buf->f_namemax = PATH_MAX;
+
+	return 0;
+}
+
+static int rbdfs_fsync(const char *path, int datasync,
+			 struct fuse_file_info *fi)
+{
+	if (!gotrados)
+		return -ENXIO;
+	rbd_flush(opentbl[fi->fh].image);
+	return 0;
+}
+
+static int rbdfs_opendir(const char *path, struct fuse_file_info *fi)
+{
+	// only one directory, so global "in_opendir" flag should be fine
+	pthread_mutex_lock(&readdir_lock);
+	in_opendir++;
+	enumerate_images(&rbd_images);
+	pthread_mutex_unlock(&readdir_lock);
+	return 0;
+}
+
+struct rbdfs_readdir_info {
+	void *buf;
+	fuse_fill_dir_t filler;
+};
+
+static void rbdfs_readdir_cb(void *_info, const char *name)
+{
+	struct rbdfs_readdir_info *info = _info;
+
+	info->filler(info->buf, name, NULL, 0);
+}
+
+static int rbdfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
+			   off_t offset, struct fuse_file_info *fi)
+{
+	struct rbdfs_readdir_info info = { buf, filler };
+
+	if (!gotrados)
+		return -ENXIO;
+	if (!in_opendir)
+		fprintf(stderr, "in readdir, but not inside opendir?\n");
+
+	if (strcmp(path, "/") != 0)
+		return -ENOENT;
+
+	filler(buf, ".", NULL, 0);
+	filler(buf, "..", NULL, 0);
+	iter_images(&info, rbdfs_readdir_cb);
+
+	return 0;
+}
+static int rbdfs_releasedir(const char *path, struct fuse_file_info *fi)
+{
+	// see opendir comments
+	pthread_mutex_lock(&readdir_lock);
+	in_opendir--;
+	pthread_mutex_unlock(&readdir_lock);
+	return 0;
+}
+
+void *
+rbdfs_init(struct fuse_conn_info *conn)
+{
+	int ret;
+
+	// init cannot fail, so if we fail here, gotrados remains at 0,
+	// causing other operations to fail immediately with ENXIO
+
+	ret = connect_to_cluster(&cluster);
+	if (ret < 0)
+		exit(90);
+
+	pool_name = rbd_options.pool_name;
+	ret = rados_ioctx_create(cluster, pool_name, &ioctx);
+	if (ret < 0)
+		exit(91);
+#if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8)
+	conn->want |= FUSE_CAP_BIG_WRITES;
+#endif
+	gotrados = 1;
+
+	// init's return value shows up in fuse_context.private_data,
+	// also to void (*destroy)(void *); useful?
+	return NULL;
+}
+
+// return -errno on error.  fi->fh is not set until open time
+
+int
+rbdfs_create(const char *path, mode_t mode, struct fuse_file_info *fi)
+{
+	int r;
+	int order = imageorder;
+
+	r = rbd_create2(ioctx, path+1, imagesize, imagefeatures, &order);
+	return r;
+}
+
+int
+rbdfs_utime(const char *path, struct utimbuf *utime)
+{
+	// called on create; not relevant
+	return 0;
+}
+
+int
+rbdfs_unlink(const char *path)
+{
+	int fd = find_openrbd(path);
+	if (fd != -1) {
+		struct rbd_openimage *rbd = &opentbl[fd];
+		rbd_close(rbd->image);
+		rbd->image = 0;
+		free(rbd->image_name);
+		rbd->rbd_stat.valid = 0;
+	}
+	return rbd_remove(ioctx, path+1);
+}
+
+
+int
+rbdfs_truncate(const char *path, off_t size)
+{
+	int fd;
+	int r;
+	struct rbd_openimage *rbd;
+
+	if ((fd = open_rbd_image(path+1)) < 0)
+		return -ENOENT;
+
+	rbd = &opentbl[fd];
+	fprintf(stderr, "truncate %s to %"PRIdMAX" (0x%"PRIxMAX")\n", path, size, size);
+	r = rbd_resize(rbd->image, size);
+	if (r < 0)
+		return r;
+
+	r = rbd_stat(rbd->image, &(rbd->rbd_stat.rbd_info),
+		 sizeof(rbd_image_info_t));
+	if (r < 0)
+		return r;
+	return 0;
+}
+
+/**
+ * set an xattr on path, with name/value, length size.
+ * Presumably flags are from Linux, as in XATTR_CREATE or 
+ * XATTR_REPLACE (both "set", but fail if exist vs fail if not exist.
+ *
+ * We accept xattrs only on the root node.
+ *
+ * All values converted with strtoull, so can be expressed in any base
+ */
+
+struct rbdfuse_attr {
+	char *attrname;
+	uint64_t *attrvalp;
+} attrs[] = {
+	{ "user.rbdfuse.imagesize", &imagesize },
+	{ "user.rbdfuse.imageorder", &imageorder },
+	{ "user.rbdfuse.imagefeatures", &imagefeatures },
+	{ NULL }
+};
+
+int
+rbdfs_setxattr(const char *path, const char *name, const char *value,
+		 size_t size, int flags)
+{
+	struct rbdfuse_attr *ap;
+	if (strcmp(path, "/") != 0)
+		return -EINVAL;
+
+	for (ap = attrs; ap->attrname != NULL; ap++) {
+		if (strcmp(name, ap->attrname) == 0) {
+			*ap->attrvalp = strtoull(value, NULL, 0);
+			fprintf(stderr, "rbd-fuse: %s set to 0x%"PRIx64"\n",
+				ap->attrname, *ap->attrvalp);
+			return 0;
+		}
+	}
+	return -EINVAL;
+}
+
+int
+rbdfs_getxattr(const char *path, const char *name, char *value,
+		 size_t size)
+{
+	struct rbdfuse_attr *ap;
+	char buf[128];
+	// allow gets on other files; ls likes to ask for things like
+	// security.*
+
+	for (ap = attrs; ap->attrname != NULL; ap++) {
+		if (strcmp(name, ap->attrname) == 0) {
+			sprintf(buf, "%"PRIu64, *ap->attrvalp);
+			if (value != NULL && size >= strlen(buf))
+				strcpy(value, buf);
+			fprintf(stderr, "rbd-fuse: get %s\n", ap->attrname);
+			return (strlen(buf));
+		}
+	}
+	return 0;
+}
+
+int
+rbdfs_listxattr(const char *path, char *list, size_t len)
+{
+	struct rbdfuse_attr *ap;
+	size_t required_len = 0;
+
+	if (strcmp(path, "/") != 0)
+		return -EINVAL;
+
+	for (ap = attrs; ap->attrname != NULL; ap++)
+		required_len += strlen(ap->attrname) + 1;
+	if (len >= required_len) {
+		for (ap = attrs; ap->attrname != NULL; ap++) {
+			sprintf(list, "%s", ap->attrname);
+			list += strlen(ap->attrname) + 1;
+		}
+	}
+	return required_len;
+}
+
+static struct fuse_operations rbdfs_oper = {
+	.create		= rbdfs_create,
+	.fsync		= rbdfs_fsync,
+	.getattr	= rbdfs_getattr,
+	.getxattr	= rbdfs_getxattr,
+	.init		= rbdfs_init,
+	.listxattr	= rbdfs_listxattr,
+	.open		= rbdfs_open,
+	.opendir	= rbdfs_opendir,
+	.read		= rbdfs_read,
+	.readdir	= rbdfs_readdir,
+	.releasedir	= rbdfs_releasedir,
+	.setxattr	= rbdfs_setxattr,
+	.statfs		= rbdfs_statfs,
+	.truncate	= rbdfs_truncate,
+	.unlink		= rbdfs_unlink,
+	.utime		= rbdfs_utime,
+	.write		= rbdfs_write,
+};
+
+enum {
+	KEY_HELP,
+	KEY_VERSION,
+	KEY_CEPH_CONFIG,
+	KEY_CEPH_CONFIG_LONG,
+	KEY_RADOS_POOLNAME,
+	KEY_RADOS_POOLNAME_LONG
+};
+
+static struct fuse_opt rbdfs_opts[] = {
+	FUSE_OPT_KEY("-h", KEY_HELP),
+	FUSE_OPT_KEY("--help", KEY_HELP),
+	FUSE_OPT_KEY("-V", KEY_VERSION),
+	FUSE_OPT_KEY("--version", KEY_VERSION),
+	{"-c %s", offsetof(struct rbd_options, ceph_config), KEY_CEPH_CONFIG},
+	{"--configfile=%s", offsetof(struct rbd_options, ceph_config),
+	 KEY_CEPH_CONFIG_LONG},
+	{"-p %s", offsetof(struct rbd_options, pool_name), KEY_RADOS_POOLNAME},
+	{"--poolname=%s", offsetof(struct rbd_options, pool_name),
+	 KEY_RADOS_POOLNAME_LONG},
+};
+
+static void usage(const char *progname)
+{
+	fprintf(stderr,
+"Usage: %s mountpoint [options]\n"
+"\n"
+"General options:\n"
+"    -h   --help            print help\n"
+"    -V   --version         print version\n"
+"    -c   --configfile      ceph configuration file [/etc/ceph/ceph.conf]\n"
+"    -p   --poolname        rados pool name [rbd]\n"
+"\n", progname);
+}
+
+static int rbdfs_opt_proc(void *data, const char *arg, int key,
+			    struct fuse_args *outargs)
+{
+	if (key == KEY_HELP) {
+		usage(outargs->argv[0]);
+		fuse_opt_add_arg(outargs, "-ho");
+		fuse_main(outargs->argc, outargs->argv, &rbdfs_oper, NULL);
+		exit(1);
+	}
+
+	if (key == KEY_VERSION) {
+		fuse_opt_add_arg(outargs, "--version");
+		fuse_main(outargs->argc, outargs->argv, &rbdfs_oper, NULL);
+		exit(0);
+	}
+
+	if (key == KEY_CEPH_CONFIG) {
+		if (rbd_options.ceph_config != NULL) {
+			free(rbd_options.ceph_config);
+			rbd_options.ceph_config = NULL;
+		}
+		rbd_options.ceph_config = strdup(arg+2);
+		return 0;
+	}
+
+	if (key == KEY_RADOS_POOLNAME) {
+		if (rbd_options.pool_name != NULL) {
+			free(rbd_options.pool_name);
+			rbd_options.pool_name = NULL;
+		}
+		rbd_options.pool_name = strdup(arg+2);
+		return 0;
+	}
+
+	return 1;
+}
+
+void
+simple_err(const char *msg, int err)
+{
+    fprintf(stderr, "%s: %s\n", msg, strerror(-err));
+    return;
+}
+
+int
+connect_to_cluster(rados_t *pcluster)
+{
+	int r;
+
+	r = rados_create(pcluster, NULL);
+	if (r < 0) {
+		simple_err("Could not create cluster handle", r);
+		return r;
+	}
+	rados_conf_parse_env(*pcluster, NULL);
+	r = rados_conf_read_file(*pcluster, rbd_options.ceph_config);
+	if (r < 0) {
+		simple_err("Error reading Ceph config file", r);
+		goto failed_shutdown;
+	}
+	r = rados_connect(*pcluster);
+	if (r < 0) {
+		simple_err("Error connecting to cluster", r);
+		goto failed_shutdown;
+	}
+
+	return 0;
+
+failed_shutdown:
+	rados_shutdown(*pcluster);
+	return r;
+}
+
+int main(int argc, char *argv[])
+{
+	struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
+
+	if (fuse_opt_parse(&args, &rbd_options, rbdfs_opts, rbdfs_opt_proc)
+	    == -1) {
+		exit(1);
+	}
+
+	pthread_mutex_init(&readdir_lock, NULL);
+
+	return fuse_main(args.argc, args.argv, &rbdfs_oper, NULL);
+}
diff --git a/ceph/src/rbdmap b/ceph/src/rbdmap
new file mode 100644
index 00000000..b0784cfd
--- /dev/null
+++ b/ceph/src/rbdmap
@@ -0,0 +1,2 @@
+# RbdDevice		Parameters
+#poolname/imagename	id=client,keyring=/etc/ceph/ceph.client.keyring
diff --git a/ceph/src/rgw/Makefile.am b/ceph/src/rgw/Makefile.am
new file mode 100644
index 00000000..3d6886d1
--- /dev/null
+++ b/ceph/src/rgw/Makefile.am
@@ -0,0 +1,164 @@
+if WITH_RADOSGW
+librgw_la_SOURCES =  \
+	rgw/librgw.cc \
+	rgw/rgw_acl.cc \
+	rgw/rgw_acl_s3.cc \
+	rgw/rgw_acl_swift.cc \
+	rgw/rgw_client_io.cc \
+	rgw/rgw_fcgi.cc \
+	rgw/rgw_xml.cc \
+	rgw/rgw_usage.cc \
+	rgw/rgw_json_enc.cc \
+	rgw/rgw_user.cc \
+	rgw/rgw_bucket.cc\
+	rgw/rgw_tools.cc \
+	rgw/rgw_rados.cc \
+	rgw/rgw_http_client.cc \
+	rgw/rgw_rest_client.cc \
+	rgw/rgw_rest_conn.cc \
+	rgw/rgw_op.cc \
+	rgw/rgw_common.cc \
+	rgw/rgw_cache.cc \
+	rgw/rgw_formats.cc \
+	rgw/rgw_log.cc \
+	rgw/rgw_multi.cc \
+	rgw/rgw_policy_s3.cc \
+	rgw/rgw_gc.cc \
+	rgw/rgw_multi_del.cc \
+	rgw/rgw_env.cc \
+	rgw/rgw_cors.cc \
+	rgw/rgw_cors_s3.cc \
+	rgw/rgw_auth_s3.cc \
+	rgw/rgw_metadata.cc \
+	rgw/rgw_replica_log.cc \
+	rgw/rgw_keystone.cc \
+	rgw/rgw_quota.cc \
+	rgw/rgw_dencoder.cc
+librgw_la_CXXFLAGS = -Woverloaded-virtual ${AM_CXXFLAGS}
+noinst_LTLIBRARIES += librgw.la
+
+LIBRGW_DEPS += \
+	$(LIBRADOS) \
+	libcls_rgw_client.la \
+	libcls_log_client.a \
+	libcls_statelog_client.a \
+	libcls_user_client.a \
+	libcls_replica_log_client.a \
+	libcls_lock_client.la \
+	libcls_refcount_client.la \
+	libcls_version_client.a \
+	-lcurl \
+	-lexpat \
+	-lm \
+	-lfcgi \
+	-ldl
+
+radosgw_SOURCES = \
+	rgw/rgw_resolve.cc \
+	rgw/rgw_rest.cc \
+	rgw/rgw_rest_swift.cc \
+	rgw/rgw_rest_s3.cc \
+	rgw/rgw_rest_usage.cc \
+	rgw/rgw_rest_user.cc \
+	rgw/rgw_rest_bucket.cc \
+	rgw/rgw_rest_metadata.cc \
+	rgw/rgw_replica_log.cc \
+	rgw/rgw_rest_log.cc \
+	rgw/rgw_rest_opstate.cc \
+	rgw/rgw_rest_replica_log.cc \
+	rgw/rgw_rest_config.cc \
+	rgw/rgw_http_client.cc \
+	rgw/rgw_swift.cc \
+	rgw/rgw_swift_auth.cc \
+	rgw/rgw_loadgen.cc \
+	rgw/rgw_civetweb.cc \
+	civetweb/src/civetweb.c \
+	rgw/rgw_main.cc
+radosgw_CFLAGS = -Icivetweb/include
+radosgw_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(RESOLV_LIBS) $(CEPH_GLOBAL)
+bin_PROGRAMS += radosgw
+
+radosgw_admin_SOURCES = rgw/rgw_admin.cc
+radosgw_admin_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL)
+bin_PROGRAMS += radosgw-admin
+
+ceph_rgw_multiparser_SOURCES = rgw/rgw_multiparser.cc
+ceph_rgw_multiparser_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_rgw_multiparser
+
+ceph_rgw_jsonparser_SOURCES = \
+	rgw/rgw_jsonparser.cc \
+	rgw/rgw_common.cc \
+	rgw/rgw_env.cc \
+	rgw/rgw_json_enc.cc
+ceph_rgw_jsonparser_LDADD = $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_rgw_jsonparser
+
+# inject rgw stuff in the decoder testcase
+DENCODER_SOURCES += \
+	rgw/rgw_dencoder.cc \
+	rgw/rgw_acl.cc \
+	rgw/rgw_common.cc \
+	rgw/rgw_env.cc \
+	rgw/rgw_json_enc.cc
+
+
+endif # WITH_RADOSGW
+
+
+noinst_HEADERS += \
+	rgw/logrotate.conf \
+	rgw/rgw_acl.h \
+	rgw/rgw_acl_s3.h \
+	rgw/rgw_acl_swift.h \
+	rgw/rgw_client_io.h \
+	rgw/rgw_fcgi.h \
+	rgw/rgw_xml.h \
+	rgw/rgw_cache.h \
+	rgw/rgw_common.h \
+	rgw/rgw_cors.h \
+	rgw/rgw_cors_s3.h \
+	rgw/rgw_cors_swift.h \
+	rgw/rgw_string.h \
+	rgw/rgw_formats.h \
+	rgw/rgw_http_errors.h \
+	rgw/rgw_log.h \
+	rgw/rgw_loadgen.h \
+	rgw/rgw_multi.h \
+	rgw/rgw_policy_s3.h \
+	rgw/rgw_gc.h \
+	rgw/rgw_metadata.h \
+	rgw/rgw_multi_del.h \
+	rgw/rgw_op.h \
+	rgw/rgw_http_client.h \
+	rgw/rgw_swift.h \
+	rgw/rgw_swift_auth.h \
+	rgw/rgw_quota.h \
+	rgw/rgw_rados.h \
+	rgw/rgw_replica_log.h \
+	rgw/rgw_resolve.h \
+	rgw/rgw_rest.h \
+	rgw/rgw_rest_swift.h \
+	rgw/rgw_rest_s3.h \
+	rgw/rgw_auth_s3.h \
+	rgw/rgw_rest_admin.h \
+	rgw/rgw_rest_usage.h \
+	rgw/rgw_rest_user.h \
+	rgw/rgw_rest_bucket.h \
+	rgw/rgw_rest_client.h \
+	rgw/rgw_rest_conn.h \
+	rgw/rgw_tools.h \
+	rgw/rgw_rest_metadata.h \
+	rgw/rgw_rest_log.h \
+	rgw/rgw_rest_opstate.h \
+	rgw/rgw_rest_replica_log.h \
+	rgw/rgw_rest_config.h \
+	rgw/rgw_usage.h \
+	rgw/rgw_user.h \
+	rgw/rgw_bucket.h \
+	rgw/rgw_keystone.h \
+	rgw/rgw_civetweb.h \
+	civetweb/civetweb.h \
+	civetweb/include/civetweb.h \
+	civetweb/src/md5.h
+
diff --git a/ceph/src/rgw/librgw.cc b/ceph/src/rgw/librgw.cc
new file mode 100644
index 00000000..556f57f3
--- /dev/null
+++ b/ceph/src/rgw/librgw.cc
@@ -0,0 +1,134 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/types.h"
+#include "include/rados/librgw.h"
+#include "rgw/rgw_acl_s3.h"
+#include "rgw_acl.h"
+#include "common/ceph_argparse.h"
+#include "common/ceph_context.h"
+#include "common/common_init.h"
+#include "common/dout.h"
+
+#include 
+#include 
+#include 
+
+#define dout_subsys ceph_subsys_rgw
+
+int librgw_create(librgw_t *rgw, const char * const id)
+{
+  CephInitParameters iparams(CEPH_ENTITY_TYPE_CLIENT);
+  if (id) {
+    iparams.name.set(CEPH_ENTITY_TYPE_CLIENT, id);
+  }
+  CephContext *cct = common_preinit(iparams, CODE_ENVIRONMENT_LIBRARY, 0);
+  cct->_conf->set_val("log_to_stderr", "false"); // quiet by default
+  cct->_conf->set_val("err_to_stderr", "true"); // quiet by default
+  cct->_conf->parse_env(); // environment variables override
+  cct->_conf->apply_changes(NULL);
+
+  common_init_finish(cct);
+  *rgw = cct;
+  return 0;
+}
+
+int librgw_acl_bin2xml(librgw_t rgw, const char *bin, int bin_len, char **xml)
+{
+  try {
+    // convert to bufferlist
+    bufferlist bl;
+    bl.append(bin, bin_len);
+
+    // convert to RGWAccessControlPolicy
+    RGWAccessControlPolicy_S3 acl((CephContext *)rgw);
+    bufferlist::iterator bli(bl.begin());
+    acl.decode(bli);
+
+    // convert to XML stringstream
+    stringstream ss;
+    acl.to_xml(ss);
+
+    // convert to XML C string
+    *xml = strdup(ss.str().c_str());
+    if (!*xml)
+      return -ENOBUFS;
+    return 0;
+  }
+  catch (const std::exception &e) {
+    lderr(rgw) << "librgw_acl_bin2xml: caught exception " << e.what() << dendl;
+    return -2000;
+  }
+  catch (...) {
+    lderr(rgw) << "librgw_acl_bin2xml: caught unknown exception " << dendl;
+    return -2000;
+  }
+}
+
+void librgw_free_xml(librgw_t rgw, char *xml)
+{
+  free(xml);
+}
+
+int librgw_acl_xml2bin(librgw_t rgw, const char *xml, char **bin, int *bin_len)
+{
+  char *bin_ = NULL;
+  try {
+    RGWACLXMLParser_S3 parser((CephContext *)rgw);
+    if (!parser.init()) {
+      return -1000;
+    }
+    if (!parser.parse(xml, strlen(xml), true)) {
+      return -EINVAL;
+    }
+    RGWAccessControlPolicy_S3 *policy =
+      (RGWAccessControlPolicy_S3 *)parser.find_first("AccessControlPolicy");
+    if (!policy) {
+      return -1001;
+    }
+    bufferlist bl;
+    policy->encode(bl);
+
+    bin_ = (char*)malloc(bl.length());
+    if (!bin_) {
+      return -ENOBUFS;
+    }
+    int bin_len_ = bl.length();
+    bl.copy(0, bin_len_, bin_);
+
+    *bin = bin_;
+    *bin_len = bin_len_;
+    return 0;
+  }
+  catch (const std::exception &e) {
+    lderr(rgw) << "librgw_acl_bin2xml: caught exception " << e.what() << dendl;
+  }
+  catch (...) {
+    lderr(rgw) << "librgw_acl_bin2xml: caught unknown exception " << dendl;
+  }
+  if (!bin_)
+    free(bin_);
+  bin_ = NULL;
+  return -2000;
+}
+
+void librgw_free_bin(librgw_t rgw, char *bin)
+{
+  free(bin);
+}
+
+void librgw_shutdown(librgw_t rgw)
+{
+  rgw->put();
+}
diff --git a/ceph/src/rgw/logrotate.conf b/ceph/src/rgw/logrotate.conf
new file mode 100644
index 00000000..ec47f008
--- /dev/null
+++ b/ceph/src/rgw/logrotate.conf
@@ -0,0 +1,26 @@
+/var/log/radosgw/*.log {
+    rotate 7
+    daily
+    compress
+    sharedscripts
+    postrotate
+        if which invoke-rc.d > /dev/null 2>&1 && [ -x `which invoke-rc.d` ]; then
+            invoke-rc.d radosgw reload >/dev/null
+        elif which service > /dev/null 2>&1 && [ -x `which service` ]; then
+            service radosgw reload >/dev/null
+        fi
+        # Possibly reload twice, but depending on ceph.conf the reload above may be a no-op
+        if which initctl > /dev/null 2>&1 && [ -x `which initctl` ]; then
+          find -L /var/lib/ceph/radosgw/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+          | while read f; do
+            if [ -e "/var/lib/ceph/radosgw/$f/done" ]; then
+                cluster="${f%%-*}"
+                id="${f#*-}"
+                initctl reload radosgw cluster="$cluster" id="$id" 2>/dev/null || :
+            fi
+          done
+        fi
+    endscript
+    missingok
+    notifempty
+}
diff --git a/ceph/src/rgw/rgw_acl.cc b/ceph/src/rgw/rgw_acl.cc
new file mode 100644
index 00000000..868bd65e
--- /dev/null
+++ b/ceph/src/rgw/rgw_acl.cc
@@ -0,0 +1,117 @@
+#include 
+
+#include 
+#include 
+
+#include "include/types.h"
+
+#include "common/Formatter.h"
+
+#include "rgw_acl.h"
+#include "rgw_user.h"
+
+#include "rgw_acl_s3.h" // required for backward compatibility
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+void RGWAccessControlList::_add_grant(ACLGrant *grant)
+{
+  ACLPermission& perm = grant->get_permission();
+  ACLGranteeType& type = grant->get_type();
+  switch (type.get_type()) {
+  case ACL_TYPE_GROUP:
+    acl_group_map[grant->get_group()] |= perm.get_permissions();
+    break;
+  default:
+    {
+      string id;
+      if (!grant->get_id(id)) {
+        ldout(cct, 0) << "ERROR: grant->get_id() failed" << dendl;
+      }
+      acl_user_map[id] |= perm.get_permissions();
+    }
+  }
+}
+
+void RGWAccessControlList::add_grant(ACLGrant *grant)
+{
+  string id;
+  grant->get_id(id); // not that this will return false for groups, but that's ok, we won't search groups
+  grant_map.insert(pair(id, *grant));
+  _add_grant(grant);
+}
+
+int RGWAccessControlList::get_perm(string& id, int perm_mask) {
+  ldout(cct, 5) << "Searching permissions for uid=" << id << " mask=" << perm_mask << dendl;
+  map::iterator iter = acl_user_map.find(id);
+  if (iter != acl_user_map.end()) {
+    ldout(cct, 5) << "Found permission: " << iter->second << dendl;
+    return iter->second & perm_mask;
+  }
+  ldout(cct, 5) << "Permissions for user not found" << dendl;
+  return 0;
+}
+
+int RGWAccessControlList::get_group_perm(ACLGroupTypeEnum group, int perm_mask) {
+  ldout(cct, 5) << "Searching permissions for group=" << (int)group << " mask=" << perm_mask << dendl;
+  map::iterator iter = acl_group_map.find((uint32_t)group);
+  if (iter != acl_group_map.end()) {
+    ldout(cct, 5) << "Found permission: " << iter->second << dendl;
+    return iter->second & perm_mask;
+  }
+  ldout(cct, 5) << "Permissions for group not found" << dendl;
+  return 0;
+}
+
+int RGWAccessControlPolicy::get_perm(string& id, int perm_mask) {
+  int perm = acl.get_perm(id, perm_mask);
+
+  if (id.compare(owner.get_id()) == 0) {
+    perm |= perm_mask & (RGW_PERM_READ_ACP | RGW_PERM_WRITE_ACP);
+  }
+
+  if (perm == perm_mask)
+    return perm;
+
+  /* should we continue looking up? */
+  if ((perm & perm_mask) != perm_mask) {
+    perm |= acl.get_group_perm(ACL_GROUP_ALL_USERS, perm_mask);
+
+    if (!compare_group_name(id, ACL_GROUP_ALL_USERS)) {
+      /* this is not the anonymous user */
+      perm |= acl.get_group_perm(ACL_GROUP_AUTHENTICATED_USERS, perm_mask);
+    }
+  }
+
+  ldout(cct, 5) << "Getting permissions id=" << id << " owner=" << owner.get_id() << " perm=" << perm << dendl;
+
+  return perm;
+}
+
+bool RGWAccessControlPolicy::verify_permission(string& uid, int user_perm_mask, int perm)
+{
+  int test_perm = perm | RGW_PERM_READ_OBJS | RGW_PERM_WRITE_OBJS;
+
+  int policy_perm = get_perm(uid, test_perm);
+
+  /* the swift WRITE_OBJS perm is equivalent to the WRITE obj, just
+     convert those bits. Note that these bits will only be set on
+     buckets, so the swift READ permission on bucket will allow listing
+     the bucket content */
+  if (policy_perm & RGW_PERM_WRITE_OBJS) {
+    policy_perm |= (RGW_PERM_WRITE | RGW_PERM_WRITE_ACP);
+  }
+  if (policy_perm & RGW_PERM_READ_OBJS) {
+    policy_perm |= (RGW_PERM_READ | RGW_PERM_READ_ACP);
+  }
+   
+  int acl_perm = policy_perm & perm & user_perm_mask;
+
+  ldout(cct, 10) << " uid=" << uid << " requested perm (type)=" << perm << ", policy perm=" << policy_perm << ", user_perm_mask=" << user_perm_mask << ", acl perm=" << acl_perm << dendl;
+
+  return (perm == acl_perm);
+}
+
+
diff --git a/ceph/src/rgw/rgw_acl.h b/ceph/src/rgw/rgw_acl.h
new file mode 100644
index 00000000..c06e9eb3
--- /dev/null
+++ b/ceph/src/rgw/rgw_acl.h
@@ -0,0 +1,334 @@
+#ifndef CEPH_RGW_ACL_H
+#define CEPH_RGW_ACL_H
+
+#include 
+#include 
+#include 
+#include 
+
+#include "common/debug.h"
+
+using namespace std;
+
+
+#define RGW_PERM_READ            0x01
+#define RGW_PERM_WRITE           0x02
+#define RGW_PERM_READ_ACP        0x04
+#define RGW_PERM_WRITE_ACP       0x08
+#define RGW_PERM_READ_OBJS       0x10
+#define RGW_PERM_WRITE_OBJS      0x20
+#define RGW_PERM_FULL_CONTROL    ( RGW_PERM_READ | RGW_PERM_WRITE | \
+                                  RGW_PERM_READ_ACP | RGW_PERM_WRITE_ACP )
+#define RGW_PERM_ALL_S3          RGW_PERM_FULL_CONTROL
+
+enum ACLGranteeTypeEnum {
+/* numbers are encoded, should not change */
+  ACL_TYPE_CANON_USER = 0,
+  ACL_TYPE_EMAIL_USER = 1,
+  ACL_TYPE_GROUP      = 2,
+  ACL_TYPE_UNKNOWN    = 3,
+};
+
+enum ACLGroupTypeEnum {
+/* numbers are encoded should not change */
+  ACL_GROUP_NONE                = 0,
+  ACL_GROUP_ALL_USERS           = 1,
+  ACL_GROUP_AUTHENTICATED_USERS = 2,
+};
+
+class ACLPermission
+{
+protected:
+  int flags;
+public:
+  ACLPermission() : flags(0) {}
+  ~ACLPermission() {}
+  int get_permissions() { return flags; }
+  void set_permissions(int perm) { flags = perm; }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(flags, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    ::decode(flags, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ACLPermission)
+
+class ACLGranteeType
+{
+protected:
+  __u32 type;
+public:
+  ACLGranteeType() : type(ACL_TYPE_UNKNOWN) {}
+  virtual ~ACLGranteeType() {}
+//  virtual const char *to_string() = 0;
+  ACLGranteeTypeEnum get_type() { return (ACLGranteeTypeEnum)type; }
+  void set(ACLGranteeTypeEnum t) { type = t; }
+//  virtual void set(const char *s) = 0;
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(type, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    ::decode(type, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ACLGranteeType)
+
+class ACLGrantee
+{
+public:
+  ACLGrantee() {}
+  ~ACLGrantee() {}
+};
+
+
+class ACLGrant
+{
+protected:
+  ACLGranteeType type;
+  string id;
+  string email;
+  ACLPermission permission;
+  string name;
+  ACLGroupTypeEnum group;
+
+public:
+  ACLGrant() : group(ACL_GROUP_NONE) {}
+  virtual ~ACLGrant() {}
+
+  /* there's an assumption here that email/uri/id encodings are
+     different and there can't be any overlap */
+  bool get_id(string& _id) {
+    switch(type.get_type()) {
+    case ACL_TYPE_EMAIL_USER:
+      _id = email;
+      return true;
+    case ACL_TYPE_GROUP:
+      return false;
+    default:
+      _id = id;
+      return true;
+    }
+  }
+  ACLGranteeType& get_type() { return type; }
+  ACLPermission& get_permission() { return permission; }
+  ACLGroupTypeEnum get_group() { return group; }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(3, 3, bl);
+    ::encode(type, bl);
+    ::encode(id, bl);
+    string uri;
+    ::encode(uri, bl);
+    ::encode(email, bl);
+    ::encode(permission, bl);
+    ::encode(name, bl);
+    __u32 g = (__u32)group;
+    ::encode(g, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+    ::decode(type, bl);
+    ::decode(id, bl);
+    string uri;
+    ::decode(uri, bl);
+    ::decode(email, bl);
+    ::decode(permission, bl);
+    ::decode(name, bl);
+    if (struct_v > 1) {
+      __u32 g;
+      ::decode(g, bl);
+      group = (ACLGroupTypeEnum)g;
+    } else {
+      group = uri_to_group(uri);
+    }
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  ACLGroupTypeEnum uri_to_group(string& uri);
+  
+  void set_canon(string& _id, string& _name, int perm) {
+    type.set(ACL_TYPE_CANON_USER);
+    id = _id;
+    name = _name;
+    permission.set_permissions(perm);
+  }
+  void set_group(ACLGroupTypeEnum _group, int perm) {
+    type.set(ACL_TYPE_GROUP);
+    group = _group;
+    permission.set_permissions(perm);
+  }
+};
+WRITE_CLASS_ENCODER(ACLGrant)
+
+class RGWAccessControlList
+{
+protected:
+  CephContext *cct;
+  map acl_user_map;
+  map acl_group_map;
+  multimap grant_map;
+  void _add_grant(ACLGrant *grant);
+public:
+  RGWAccessControlList(CephContext *_cct) : cct(_cct) {}
+  RGWAccessControlList() : cct(NULL) {}
+
+  void set_ctx(CephContext *ctx) {
+    cct = ctx;
+  }
+
+  virtual ~RGWAccessControlList() {}
+
+  int get_perm(string& id, int perm_mask);
+  int get_group_perm(ACLGroupTypeEnum group, int perm_mask);
+  void encode(bufferlist& bl) const {
+    ENCODE_START(3, 3, bl);
+    bool maps_initialized = true;
+    ::encode(maps_initialized, bl);
+    ::encode(acl_user_map, bl);
+    ::encode(grant_map, bl);
+    ::encode(acl_group_map, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+    bool maps_initialized;
+    ::decode(maps_initialized, bl);
+    ::decode(acl_user_map, bl);
+    ::decode(grant_map, bl);
+    if (struct_v >= 2) {
+      ::decode(acl_group_map, bl);
+    } else if (!maps_initialized) {
+      multimap::iterator iter;
+      for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
+        ACLGrant& grant = iter->second;
+        _add_grant(&grant);
+      }
+    }
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  void add_grant(ACLGrant *grant);
+
+  multimap& get_grant_map() { return grant_map; }
+
+  void create_default(string id, string name) {
+    acl_user_map.clear();
+    acl_group_map.clear();
+
+    ACLGrant grant;
+    grant.set_canon(id, name, RGW_PERM_FULL_CONTROL);
+    add_grant(&grant);
+  }
+};
+WRITE_CLASS_ENCODER(RGWAccessControlList)
+
+class ACLOwner
+{
+protected:
+  string id;
+  string display_name;
+public:
+  ACLOwner() {}
+  ~ACLOwner() {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(id, bl);
+    ::encode(display_name, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    ::decode(id, bl);
+    ::decode(display_name, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+  void set_id(const string& _id) { id = _id; }
+  void set_name(string& name) { display_name = name; }
+
+  string& get_id() { return id; }
+  string& get_display_name() { return display_name; }
+};
+WRITE_CLASS_ENCODER(ACLOwner)
+
+class RGWAccessControlPolicy
+{
+protected:
+  CephContext *cct;
+  RGWAccessControlList acl;
+  ACLOwner owner;
+
+public:
+  RGWAccessControlPolicy(CephContext *_cct) : cct(_cct), acl(_cct) {}
+  RGWAccessControlPolicy() : cct(NULL), acl(NULL) {}
+  virtual ~RGWAccessControlPolicy() {}
+
+  void set_ctx(CephContext *ctx) {
+    cct = ctx;
+    acl.set_ctx(ctx);
+  }
+
+  int get_perm(string& id, int perm_mask);
+  int get_group_perm(ACLGroupTypeEnum group, int perm_mask);
+  bool verify_permission(string& uid, int user_perm_mask, int perm);
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(owner, bl);
+    ::encode(acl, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    ::decode(owner, bl);
+    ::decode(acl, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+  void decode_owner(bufferlist::iterator& bl) { // sometimes we only need that, should be faster
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    ::decode(owner, bl);
+    DECODE_FINISH(bl);
+  }
+
+  void set_owner(ACLOwner& o) { owner = o; }
+  ACLOwner& get_owner() {
+    return owner;
+  }
+
+  void create_default(string& id, string& name) {
+    acl.create_default(id, name);
+    owner.set_id(id);
+    owner.set_name(name);
+  }
+  RGWAccessControlList& get_acl() {
+    return acl;
+  }
+
+  virtual bool compare_group_name(string& id, ACLGroupTypeEnum group) { return false; }
+};
+WRITE_CLASS_ENCODER(RGWAccessControlPolicy)
+
+#endif
diff --git a/ceph/src/rgw/rgw_acl_s3.cc b/ceph/src/rgw/rgw_acl_s3.cc
new file mode 100644
index 00000000..b47c14df
--- /dev/null
+++ b/ceph/src/rgw/rgw_acl_s3.cc
@@ -0,0 +1,578 @@
+#include 
+
+#include 
+#include 
+
+#include "include/types.h"
+
+#include "rgw_acl_s3.h"
+#include "rgw_user.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+
+#define RGW_URI_ALL_USERS	"http://acs.amazonaws.com/groups/global/AllUsers"
+#define RGW_URI_AUTH_USERS	"http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+
+static string rgw_uri_all_users = RGW_URI_ALL_USERS;
+static string rgw_uri_auth_users = RGW_URI_AUTH_USERS;
+
+void ACLPermission_S3::to_xml(ostream& out)
+{
+  if ((flags & RGW_PERM_FULL_CONTROL) == RGW_PERM_FULL_CONTROL) {
+   out << "FULL_CONTROL";
+  } else {
+    if (flags & RGW_PERM_READ)
+      out << "READ";
+    if (flags & RGW_PERM_WRITE)
+      out << "WRITE";
+    if (flags & RGW_PERM_READ_ACP)
+      out << "READ_ACP";
+    if (flags & RGW_PERM_WRITE_ACP)
+      out << "WRITE_ACP";
+  }
+}
+
+bool ACLPermission_S3::
+xml_end(const char *el)
+{
+  const char *s = data.c_str();
+  if (strcasecmp(s, "READ") == 0) {
+    flags |= RGW_PERM_READ;
+    return true;
+  } else if (strcasecmp(s, "WRITE") == 0) {
+    flags |= RGW_PERM_WRITE;
+    return true;
+  } else if (strcasecmp(s, "READ_ACP") == 0) {
+    flags |= RGW_PERM_READ_ACP;
+    return true;
+  } else if (strcasecmp(s, "WRITE_ACP") == 0) {
+    flags |= RGW_PERM_WRITE_ACP;
+    return true;
+  } else if (strcasecmp(s, "FULL_CONTROL") == 0) {
+    flags |= RGW_PERM_FULL_CONTROL;
+    return true;
+  }
+  return false;
+}
+
+
+class ACLGranteeType_S3 {
+public:
+  static const char *to_string(ACLGranteeType& type) {
+    switch (type.get_type()) {
+    case ACL_TYPE_CANON_USER:
+      return "CanonicalUser";
+    case ACL_TYPE_EMAIL_USER:
+      return "AmazonCustomerByEmail";
+    case ACL_TYPE_GROUP:
+      return "Group";
+     default:
+      return "unknown";
+    }
+  }
+
+  static void set(const char *s, ACLGranteeType& type) {
+    if (!s) {
+      type.set(ACL_TYPE_UNKNOWN);
+      return;
+    }
+    if (strcmp(s, "CanonicalUser") == 0)
+      type.set(ACL_TYPE_CANON_USER);
+    else if (strcmp(s, "AmazonCustomerByEmail") == 0)
+      type.set(ACL_TYPE_EMAIL_USER);
+    else if (strcmp(s, "Group") == 0)
+      type.set(ACL_TYPE_GROUP);
+    else
+      type.set(ACL_TYPE_UNKNOWN);
+  }
+};
+
+class ACLID_S3 : public XMLObj
+{
+public:
+  ACLID_S3() {}
+  ~ACLID_S3() {}
+  string& to_str() { return data; }
+};
+
+class ACLURI_S3 : public XMLObj
+{
+public:
+  ACLURI_S3() {}
+  ~ACLURI_S3() {}
+};
+
+class ACLEmail_S3 : public XMLObj
+{
+public:
+  ACLEmail_S3() {}
+  ~ACLEmail_S3() {}
+};
+
+class ACLDisplayName_S3 : public XMLObj
+{
+public:
+ ACLDisplayName_S3() {}
+ ~ACLDisplayName_S3() {}
+};
+
+bool ACLOwner_S3::xml_end(const char *el) {
+  ACLID_S3 *acl_id = static_cast(find_first("ID"));
+  ACLID_S3 *acl_name = static_cast(find_first("DisplayName"));
+
+  // ID is mandatory
+  if (!acl_id)
+    return false;
+  id = acl_id->get_data();
+
+  // DisplayName is optional
+  if (acl_name)
+    display_name = acl_name->get_data();
+  else
+    display_name = "";
+
+  return true;
+}
+
+bool ACLGrant_S3::xml_end(const char *el) {
+  ACLGrantee_S3 *acl_grantee;
+  ACLID_S3 *acl_id;
+  ACLURI_S3 *acl_uri;
+  ACLEmail_S3 *acl_email;
+  ACLPermission_S3 *acl_permission;
+  ACLDisplayName_S3 *acl_name;
+  string uri;
+
+  acl_grantee = static_cast(find_first("Grantee"));
+  if (!acl_grantee)
+    return false;
+  string type_str;
+  if (!acl_grantee->get_attr("xsi:type", type_str))
+    return false;
+  ACLGranteeType_S3::set(type_str.c_str(), type);
+  
+  acl_permission = static_cast(find_first("Permission"));
+  if (!acl_permission)
+    return false;
+
+  permission = *acl_permission;
+
+  id.clear();
+  name.clear();
+  email.clear();
+
+  switch (type.get_type()) {
+  case ACL_TYPE_CANON_USER:
+    acl_id = static_cast(acl_grantee->find_first("ID"));
+    if (!acl_id)
+      return false;
+    id = acl_id->to_str();
+    acl_name = static_cast(acl_grantee->find_first("DisplayName"));
+    if (acl_name)
+      name = acl_name->get_data();
+    break;
+  case ACL_TYPE_GROUP:
+    acl_uri = static_cast(acl_grantee->find_first("URI"));
+    if (!acl_uri)
+      return false;
+    uri = acl_uri->get_data();
+    group = uri_to_group(uri);
+    break;
+  case ACL_TYPE_EMAIL_USER:
+    acl_email = static_cast(acl_grantee->find_first("EmailAddress"));
+    if (!acl_email)
+      return false;
+    email = acl_email->get_data();
+    break;
+  default:
+    // unknown user type
+    return false;
+  };
+  return true;
+}
+
+void ACLGrant_S3::to_xml(CephContext *cct, ostream& out) {
+  ACLPermission_S3& perm = static_cast(permission);
+
+  /* only show s3 compatible permissions */
+  if (!(perm.get_permissions() & RGW_PERM_ALL_S3))
+    return;
+
+  string uri;
+
+  out << "" <<
+         "";
+  switch (type.get_type()) {
+  case ACL_TYPE_CANON_USER:
+    out << "" << id << "";
+    if (name.size()) {
+      out << "" << name << "";
+    }
+    break;
+  case ACL_TYPE_EMAIL_USER:
+    out << "" << email << "";
+    break;
+  case ACL_TYPE_GROUP:
+    if (!group_to_uri(group, uri)) {
+      ldout(cct, 0) << "ERROR: group_to_uri failed with group=" << (int)group << dendl;
+      break;
+    }
+    out << "" << uri << "";
+    break;
+  default:
+    break;
+  }
+  out << "";
+  perm.to_xml(out);
+  out << "";
+}
+
+bool ACLGrant_S3::group_to_uri(ACLGroupTypeEnum group, string& uri)
+{
+  switch (group) {
+  case ACL_GROUP_ALL_USERS:
+    uri = rgw_uri_all_users;
+    return true;
+  case ACL_GROUP_AUTHENTICATED_USERS:
+    uri = rgw_uri_auth_users;
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool RGWAccessControlList_S3::xml_end(const char *el) {
+  XMLObjIter iter = find("Grant");
+  ACLGrant_S3 *grant = static_cast(iter.get_next());
+  while (grant) {
+    add_grant(grant);
+    grant = static_cast(iter.get_next());
+  }
+  return true;
+}
+
+struct s3_acl_header {
+  int rgw_perm;
+  const char *http_header;
+};
+
+static const char *get_acl_header(RGWEnv *env,
+        const struct s3_acl_header *perm)
+{
+  const char *header = perm->http_header;
+
+  return env->get(header, NULL);
+}
+
+static int parse_grantee_str(RGWRados *store, string& grantee_str,
+        const struct s3_acl_header *perm, ACLGrant& grant)
+{
+  string id_type, id_val_quoted;
+  int rgw_perm = perm->rgw_perm;
+  int ret;
+
+  RGWUserInfo info;
+
+  ret = parse_key_value(grantee_str, id_type, id_val_quoted);
+  if (ret < 0)
+    return ret;
+
+  string id_val = rgw_trim_quotes(id_val_quoted);
+
+  if (strcasecmp(id_type.c_str(), "emailAddress") == 0) {
+    ret = rgw_get_user_info_by_email(store, id_val, info);
+    if (ret < 0)
+      return ret;
+
+    grant.set_canon(info.user_id, info.display_name, rgw_perm);
+  } else if (strcasecmp(id_type.c_str(), "id") == 0) {
+    ret = rgw_get_user_info_by_uid(store, id_val, info);
+    if (ret < 0)
+      return ret;
+
+    grant.set_canon(info.user_id, info.display_name, rgw_perm);
+  } else if (strcasecmp(id_type.c_str(), "uri") == 0) {
+    ACLGroupTypeEnum gid = grant.uri_to_group(id_val);
+    if (gid == ACL_GROUP_NONE)
+      return -EINVAL;
+
+    grant.set_group(gid, rgw_perm);
+  } else {
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+static int parse_acl_header(RGWRados *store, RGWEnv *env,
+         const struct s3_acl_header *perm, std::list& _grants)
+{
+  std::list grantees;
+  std::string hacl_str;
+
+  const char *hacl = get_acl_header(env, perm);
+  if (hacl == NULL)
+    return 0;
+
+  hacl_str = hacl;
+  get_str_list(hacl_str, ",", grantees);
+
+  for (list::iterator it = grantees.begin(); it != grantees.end(); ++it) {
+    ACLGrant grant;
+    int ret = parse_grantee_str(store, *it, perm, grant);
+    if (ret < 0)
+      return ret;
+
+    _grants.push_back(grant);
+  }
+
+  return 0;
+}
+
+int RGWAccessControlList_S3::create_canned(ACLOwner& owner, ACLOwner& bucket_owner, const string& canned_acl)
+{
+  acl_user_map.clear();
+  grant_map.clear();
+
+  ACLGrant owner_grant;
+
+  string bid = bucket_owner.get_id();
+  string bname = bucket_owner.get_display_name();
+
+  /* owner gets full control */
+  owner_grant.set_canon(owner.get_id(), owner.get_display_name(), RGW_PERM_FULL_CONTROL);
+  add_grant(&owner_grant);
+
+  if (canned_acl.size() == 0 || canned_acl.compare("private") == 0) {
+    return 0;
+  }
+
+  ACLGrant bucket_owner_grant;
+  ACLGrant group_grant;
+  if (canned_acl.compare("public-read") == 0) {
+    group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
+    add_grant(&group_grant);
+  } else if (canned_acl.compare("public-read-write") == 0) {
+    group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
+    add_grant(&group_grant);
+    group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_WRITE);
+    add_grant(&group_grant);
+  } else if (canned_acl.compare("authenticated-read") == 0) {
+    group_grant.set_group(ACL_GROUP_AUTHENTICATED_USERS, RGW_PERM_READ);
+    add_grant(&group_grant);
+  } else if (canned_acl.compare("bucket-owner-read") == 0) {
+    bucket_owner_grant.set_canon(bid, bname, RGW_PERM_READ);
+    if (bid.compare(owner.get_id()) != 0)
+      add_grant(&bucket_owner_grant);
+  } else if (canned_acl.compare("bucket-owner-full-control") == 0) {
+    bucket_owner_grant.set_canon(bid, bname, RGW_PERM_FULL_CONTROL);
+    if (bid.compare(owner.get_id()) != 0)
+      add_grant(&bucket_owner_grant);
+  } else {
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+int RGWAccessControlList_S3::create_from_grants(std::list& grants)
+{
+  if (grants.empty())
+    return -EINVAL;
+
+  acl_user_map.clear();
+  grant_map.clear();
+
+  for (std::list::iterator it = grants.begin(); it != grants.end(); ++it) {
+    ACLGrant g = *it;
+    add_grant(&g);
+  }
+
+  return 0;
+}
+
+bool RGWAccessControlPolicy_S3::xml_end(const char *el) {
+  RGWAccessControlList_S3 *s3acl =
+      static_cast(find_first("AccessControlList"));
+  if (!s3acl)
+    return false;
+
+  acl = *s3acl;
+
+  ACLOwner *owner_p = static_cast(find_first("Owner"));
+  if (!owner_p)
+    return false;
+  owner = *owner_p;
+  return true;
+}
+
+static const s3_acl_header acl_header_perms[] = {
+  {RGW_PERM_READ, "HTTP_X_AMZ_GRANT_READ"},
+  {RGW_PERM_WRITE, "HTTP_X_AMZ_GRANT_WRITE"},
+  {RGW_PERM_READ_ACP,"HTTP_X_AMZ_GRANT_READ_ACP"},
+  {RGW_PERM_WRITE_ACP, "HTTP_X_AMZ_GRANT_WRITE_ACP"},
+  {RGW_PERM_FULL_CONTROL, "HTTP_X_AMZ_GRANT_FULL_CONTROL"},
+  {0, NULL}
+};
+
+int RGWAccessControlPolicy_S3::create_from_headers(RGWRados *store, RGWEnv *env, ACLOwner& _owner)
+{
+  std::list grants;
+
+  for (const struct s3_acl_header *p = acl_header_perms; p->rgw_perm; p++) {
+    if (parse_acl_header(store, env, p, grants) < 0)
+      return false;
+  }
+
+  RGWAccessControlList_S3& _acl = static_cast(acl);
+  int r = _acl.create_from_grants(grants);
+
+  owner = _owner;
+
+  return r;
+}
+
+/*
+  can only be called on object that was parsed
+ */
+int RGWAccessControlPolicy_S3::rebuild(RGWRados *store, ACLOwner *owner, RGWAccessControlPolicy& dest)
+{
+  if (!owner)
+    return -EINVAL;
+
+  ACLOwner *requested_owner = static_cast(find_first("Owner"));
+  if (requested_owner) {
+    const string& requested_id = requested_owner->get_id();
+    if (!requested_id.empty() && requested_id.compare(owner->get_id()) != 0)
+      return -EPERM;
+  }
+
+  RGWUserInfo owner_info;
+  if (rgw_get_user_info_by_uid(store, owner->get_id(), owner_info) < 0) {
+    ldout(cct, 10) << "owner info does not exist" << dendl;
+    return -EINVAL;
+  }
+  ACLOwner& dest_owner = dest.get_owner();
+  dest_owner.set_id(owner->get_id());
+  dest_owner.set_name(owner_info.display_name);
+
+  ldout(cct, 20) << "owner id=" << owner->get_id() << dendl;
+  ldout(cct, 20) << "dest owner id=" << dest.get_owner().get_id() << dendl;
+
+  RGWAccessControlList& dst_acl = dest.get_acl();
+
+  multimap& grant_map = acl.get_grant_map();
+  multimap::iterator iter;
+  for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
+    ACLGrant& src_grant = iter->second;
+    ACLGranteeType& type = src_grant.get_type();
+    ACLGrant new_grant;
+    bool grant_ok = false;
+    string uid;
+    RGWUserInfo grant_user;
+    switch (type.get_type()) {
+    case ACL_TYPE_EMAIL_USER:
+      {
+        string email;
+        if (!src_grant.get_id(email)) {
+          ldout(cct, 0) << "ERROR: src_grant.get_id() failed" << dendl;
+          return -EINVAL;
+        }
+        ldout(cct, 10) << "grant user email=" << email << dendl;
+        if (rgw_get_user_info_by_email(store, email, grant_user) < 0) {
+          ldout(cct, 10) << "grant user email not found or other error" << dendl;
+          return -ERR_UNRESOLVABLE_EMAIL;
+        }
+        uid = grant_user.user_id;
+      }
+    case ACL_TYPE_CANON_USER:
+      {
+        if (type.get_type() == ACL_TYPE_CANON_USER) {
+          if (!src_grant.get_id(uid)) {
+            ldout(cct, 0) << "ERROR: src_grant.get_id() failed" << dendl;
+            return -EINVAL;
+          }
+        }
+    
+        if (grant_user.user_id.empty() && rgw_get_user_info_by_uid(store, uid, grant_user) < 0) {
+          ldout(cct, 10) << "grant user does not exist:" << uid << dendl;
+          return -EINVAL;
+        } else {
+          ACLPermission& perm = src_grant.get_permission();
+          new_grant.set_canon(uid, grant_user.display_name, perm.get_permissions());
+          grant_ok = true;
+          string new_id;
+          new_grant.get_id(new_id);
+          ldout(cct, 10) << "new grant: " << new_id << ":" << grant_user.display_name << dendl;
+        }
+      }
+      break;
+    case ACL_TYPE_GROUP:
+      {
+        string uri;
+        if (ACLGrant_S3::group_to_uri(src_grant.get_group(), uri)) {
+          new_grant = src_grant;
+          grant_ok = true;
+          ldout(cct, 10) << "new grant: " << uri << dendl;
+        } else {
+          ldout(cct, 10) << "bad grant group:" << (int)src_grant.get_group() << dendl;
+          return -EINVAL;
+        }
+      }
+    default:
+      break;
+    }
+    if (grant_ok) {
+      dst_acl.add_grant(&new_grant);
+    }
+  }
+
+  return 0; 
+}
+
+bool RGWAccessControlPolicy_S3::compare_group_name(string& id, ACLGroupTypeEnum group)
+{
+  switch (group) {
+  case ACL_GROUP_ALL_USERS:
+    return (id.compare(rgw_uri_all_users) == 0);
+  case ACL_GROUP_AUTHENTICATED_USERS:
+    return (id.compare(rgw_uri_auth_users) == 0);
+  default:
+    return id.empty();
+  }
+
+  // shouldn't get here
+  return false;
+}
+
+XMLObj *RGWACLXMLParser_S3::alloc_obj(const char *el)
+{
+  XMLObj * obj = NULL;
+  if (strcmp(el, "AccessControlPolicy") == 0) {
+    obj = new RGWAccessControlPolicy_S3(cct);
+  } else if (strcmp(el, "Owner") == 0) {
+    obj = new ACLOwner_S3();
+  } else if (strcmp(el, "AccessControlList") == 0) {
+    obj = new RGWAccessControlList_S3(cct);
+  } else if (strcmp(el, "ID") == 0) {
+    obj = new ACLID_S3();
+  } else if (strcmp(el, "DisplayName") == 0) {
+    obj = new ACLDisplayName_S3();
+  } else if (strcmp(el, "Grant") == 0) {
+    obj = new ACLGrant_S3();
+  } else if (strcmp(el, "Grantee") == 0) {
+    obj = new ACLGrantee_S3();
+  } else if (strcmp(el, "Permission") == 0) {
+    obj = new ACLPermission_S3();
+  } else if (strcmp(el, "URI") == 0) {
+    obj = new ACLURI_S3();
+  } else if (strcmp(el, "EmailAddress") == 0) {
+    obj = new ACLEmail_S3();
+  }
+
+  return obj;
+}
+
diff --git a/ceph/src/rgw/rgw_acl_s3.h b/ceph/src/rgw/rgw_acl_s3.h
new file mode 100644
index 00000000..e0340d8f
--- /dev/null
+++ b/ceph/src/rgw/rgw_acl_s3.h
@@ -0,0 +1,135 @@
+#ifndef CEPH_RGW_ACL_S3_H
+#define CEPH_RGW_ACL_S3_H
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "include/str_list.h"
+#include "rgw_xml.h"
+#include "rgw_acl.h"
+
+
+using namespace std;
+
+class RGWRados;
+
+class ACLPermission_S3 : public ACLPermission, public XMLObj
+{
+public:
+  ACLPermission_S3() {}
+  ~ACLPermission_S3() {}
+
+  bool xml_end(const char *el);
+  void to_xml(ostream& out);
+};
+
+class ACLGrantee_S3 : public ACLGrantee, public XMLObj
+{
+public:
+  ACLGrantee_S3() {}
+  ~ACLGrantee_S3() {}
+
+  bool xml_start(const char *el, const char **attr);
+};
+
+
+class ACLGrant_S3 : public ACLGrant, public XMLObj
+{
+public:
+  ACLGrant_S3() {}
+  ~ACLGrant_S3() {}
+
+  void to_xml(CephContext *cct, ostream& out);
+  bool xml_end(const char *el);
+  bool xml_start(const char *el, const char **attr);
+
+  static ACLGroupTypeEnum uri_to_group(string& uri);
+  static bool group_to_uri(ACLGroupTypeEnum group, string& uri);
+};
+
+class RGWAccessControlList_S3 : public RGWAccessControlList, public XMLObj
+{
+public:
+  RGWAccessControlList_S3(CephContext *_cct) : RGWAccessControlList(_cct) {}
+  ~RGWAccessControlList_S3() {}
+
+  bool xml_end(const char *el);
+  void to_xml(ostream& out) {
+    multimap::iterator iter;
+    out << "";
+    for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
+      ACLGrant_S3& grant = static_cast(iter->second);
+      grant.to_xml(cct, out);
+    }
+    out << "";
+  }
+
+  int create_canned(ACLOwner& owner, ACLOwner& bucket_owner, const string& canned_acl);
+  int create_from_grants(std::list& grants);
+};
+
+class ACLOwner_S3 : public ACLOwner, public XMLObj
+{
+public:
+  ACLOwner_S3() {}
+  ~ACLOwner_S3() {}
+
+  bool xml_end(const char *el);
+  void to_xml(ostream& out) {
+    if (id.empty())
+      return;
+    out << "" << "" << id << "";
+    if (!display_name.empty())
+      out << "" << display_name << "";
+    out << "";
+  }
+};
+
+class RGWEnv;
+
+class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
+{
+public:
+  RGWAccessControlPolicy_S3(CephContext *_cct) : RGWAccessControlPolicy(_cct) {}
+  ~RGWAccessControlPolicy_S3() {}
+
+  bool xml_end(const char *el);
+
+  void to_xml(ostream& out) {
+    out << "";
+    ACLOwner_S3& _owner = static_cast(owner);
+    RGWAccessControlList_S3& _acl = static_cast(acl);
+    _owner.to_xml(out);
+    _acl.to_xml(out);
+    out << "";
+  }
+  int rebuild(RGWRados *store, ACLOwner *owner, RGWAccessControlPolicy& dest);
+  bool compare_group_name(string& id, ACLGroupTypeEnum group);
+
+  virtual int create_canned(ACLOwner& _owner, ACLOwner& bucket_owner, string canned_acl) {
+    RGWAccessControlList_S3& _acl = static_cast(acl);
+    int ret = _acl.create_canned(_owner, bucket_owner, canned_acl);
+    owner = _owner;
+    return ret;
+  }
+  int create_from_headers(RGWRados *store, RGWEnv *env, ACLOwner& _owner);
+};
+
+/**
+ * Interfaces with the webserver's XML handling code
+ * to parse it in a way that makes sense for the rgw.
+ */
+class RGWACLXMLParser_S3 : public RGWXMLParser
+{
+  CephContext *cct;
+
+  XMLObj *alloc_obj(const char *el);
+public:
+  RGWACLXMLParser_S3(CephContext *_cct) : cct(_cct) {}
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_acl_swift.cc b/ceph/src/rgw/rgw_acl_swift.cc
new file mode 100644
index 00000000..b02ce90f
--- /dev/null
+++ b/ceph/src/rgw/rgw_acl_swift.cc
@@ -0,0 +1,131 @@
+
+#include 
+
+#include 
+
+#include "rgw_common.h"
+#include "rgw_user.h"
+#include "rgw_acl_swift.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+#define SWIFT_PERM_READ  RGW_PERM_READ_OBJS
+#define SWIFT_PERM_WRITE RGW_PERM_WRITE_OBJS
+
+#define SWIFT_GROUP_ALL_USERS ".r:*"
+
+static int parse_list(string& uid_list, vector& uids)
+{
+  char *s = strdup(uid_list.c_str());
+  if (!s)
+    return -ENOMEM;
+
+  const char *p = strtok(s, " ,");
+  while (p) {
+    if (*p) {
+      string acl = p;
+      uids.push_back(acl);
+    }
+    p = strtok(NULL, " ,");
+  }
+  free(s);
+  return 0;
+}
+
+static bool uid_is_public(string& uid)
+{
+  if (uid[0] != '.' || uid[1] != 'r')
+    return false;
+
+  int pos = uid.find(':');
+  if (pos < 0 || pos == (int)uid.size())
+    return false;
+
+  string sub = uid.substr(0, pos);
+  string after = uid.substr(pos + 1);
+
+  if (after.compare("*") != 0)
+    return false;
+
+  return sub.compare(".r") == 0 ||
+         sub.compare(".referer") == 0 ||
+         sub.compare(".referrer") == 0;
+}
+
+void RGWAccessControlPolicy_SWIFT::add_grants(RGWRados *store, vector& uids, int perm)
+{
+  vector::iterator iter;
+  for (iter = uids.begin(); iter != uids.end(); ++iter ) {
+    ACLGrant grant;
+    RGWUserInfo grant_user;
+    string& uid = *iter;
+    if (uid_is_public(uid)) {
+      grant.set_group(ACL_GROUP_ALL_USERS, perm);
+      acl.add_grant(&grant);
+    } else if (rgw_get_user_info_by_uid(store, uid, grant_user) < 0) {
+      ldout(cct, 10) << "grant user does not exist:" << uid << dendl;
+      /* skipping silently */
+    } else {
+      grant.set_canon(uid, grant_user.display_name, perm);
+      acl.add_grant(&grant);
+    }
+  }
+}
+
+bool RGWAccessControlPolicy_SWIFT::create(RGWRados *store, string& id, string& name, string& read_list, string& write_list)
+{
+  acl.create_default(id, name);
+  owner.set_id(id);
+  owner.set_name(name);
+
+  if (read_list.size()) {
+    vector uids;
+    int r = parse_list(read_list, uids);
+    if (r < 0) {
+      ldout(cct, 0) << "ERROR: parse_list returned r=" << r << dendl;
+      return false;
+    }
+
+    add_grants(store, uids, SWIFT_PERM_READ);
+  }
+  if (write_list.size()) {
+    vector uids;
+    int r = parse_list(write_list, uids);
+    if (r < 0) {
+      ldout(cct, 0) << "ERROR: parse_list returned r=" << r << dendl;
+      return false;
+    }
+
+    add_grants(store, uids, SWIFT_PERM_WRITE);
+  }
+  return true;
+}
+
+void RGWAccessControlPolicy_SWIFT::to_str(string& read, string& write)
+{
+  multimap& m = acl.get_grant_map();
+  multimap::iterator iter;
+
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    ACLGrant& grant = iter->second;
+    int perm = grant.get_permission().get_permissions();
+    string id;
+    if (!grant.get_id(id)) {
+      if (grant.get_group() != ACL_GROUP_ALL_USERS)
+        continue;
+      id = SWIFT_GROUP_ALL_USERS;
+    }
+    if (perm & SWIFT_PERM_READ) {
+      if (!read.empty())
+        read.append(", ");
+      read.append(id);
+    } else if (perm & SWIFT_PERM_WRITE) {
+      if (!write.empty())
+        write.append(", ");
+      write.append(id);
+    }
+  }
+}
+
diff --git a/ceph/src/rgw/rgw_acl_swift.h b/ceph/src/rgw/rgw_acl_swift.h
new file mode 100644
index 00000000..be64f580
--- /dev/null
+++ b/ceph/src/rgw/rgw_acl_swift.h
@@ -0,0 +1,25 @@
+#ifndef CEPH_RGW_ACL_SWIFT_H
+#define CEPH_RGW_ACL_SWIFT3_H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "rgw_acl.h"
+
+using namespace std;
+
+class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
+{
+public:
+  RGWAccessControlPolicy_SWIFT(CephContext *_cct) : RGWAccessControlPolicy(_cct) {}
+  ~RGWAccessControlPolicy_SWIFT() {}
+
+  void add_grants(RGWRados *store, vector& uids, int perm);
+  bool create(RGWRados *store, string& id, string& name, string& read_list, string& write_list);
+  void to_str(string& read, string& write);
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_admin.cc b/ceph/src/rgw/rgw_admin.cc
new file mode 100644
index 00000000..e31a28a6
--- /dev/null
+++ b/ceph/src/rgw/rgw_admin.cc
@@ -0,0 +1,2430 @@
+#include 
+
+#include 
+#include 
+#include 
+
+using namespace std;
+
+#include "common/ceph_json.h"
+
+#include "common/config.h"
+#include "common/ceph_argparse.h"
+#include "common/Formatter.h"
+#include "common/ceph_json.h"
+#include "global/global_init.h"
+#include "common/errno.h"
+#include "include/utime.h"
+#include "include/str_list.h"
+
+#include "common/armor.h"
+#include "rgw_user.h"
+#include "rgw_bucket.h"
+#include "rgw_rados.h"
+#include "rgw_acl.h"
+#include "rgw_acl_s3.h"
+#include "rgw_log.h"
+#include "rgw_formats.h"
+#include "rgw_usage.h"
+#include "rgw_replica_log.h"
+#include "auth/Crypto.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+#define SECRET_KEY_LEN 40
+#define PUBLIC_ID_LEN 20
+
+static RGWRados *store = NULL;
+
+void _usage() 
+{
+  cerr << "usage: radosgw-admin  [options...]" << std::endl;
+  cerr << "commands:\n";
+  cerr << "  user create                create a new user\n" ;
+  cerr << "  user modify                modify user\n";
+  cerr << "  user info                  get user info\n";
+  cerr << "  user rm                    remove user\n";
+  cerr << "  user suspend               suspend a user\n";
+  cerr << "  user enable                re-enable user after suspension\n";
+  cerr << "  user check                 check user info\n";
+  cerr << "  user stats                 show user stats as accounted by quota subsystem\n";
+  cerr << "  caps add                   add user capabilities\n";
+  cerr << "  caps rm                    remove user capabilities\n";
+  cerr << "  subuser create             create a new subuser\n" ;
+  cerr << "  subuser modify             modify subuser\n";
+  cerr << "  subuser rm                 remove subuser\n";
+  cerr << "  key create                 create access key\n";
+  cerr << "  key rm                     remove access key\n";
+  cerr << "  bucket list                list buckets\n";
+  cerr << "  bucket link                link bucket to specified user\n";
+  cerr << "  bucket unlink              unlink bucket from specified user\n";
+  cerr << "  bucket stats               returns bucket statistics\n";
+  cerr << "  bucket rm                  remove bucket\n";
+  cerr << "  bucket check               check bucket index\n";
+  cerr << "  object rm                  remove object\n";
+  cerr << "  object unlink              unlink object from bucket index\n";
+  cerr << "  quota set                  set quota params\n";
+  cerr << "  quota enable               enable quota\n";
+  cerr << "  quota disable              disable quota\n";
+  cerr << "  region get                 show region info\n";
+  cerr << "  regions list               list all regions set on this cluster\n";
+  cerr << "  region set                 set region info (requires infile)\n";
+  cerr << "  region default             set default region\n";
+  cerr << "  region-map get             show region-map\n";
+  cerr << "  region-map set             set region-map (requires infile)\n";
+  cerr << "  zone get                   show zone cluster params\n";
+  cerr << "  zone set                   set zone cluster params (requires infile)\n";
+  cerr << "  zone list                  list all zones set on this cluster\n";
+  cerr << "  pool add                   add an existing pool for data placement\n";
+  cerr << "  pool rm                    remove an existing pool from data placement set\n";
+  cerr << "  pools list                 list placement active set\n";
+  cerr << "  policy                     read bucket/object policy\n";
+  cerr << "  log list                   list log objects\n";
+  cerr << "  log show                   dump a log from specific object or (bucket + date\n";
+  cerr << "                             + bucket-id)\n";
+  cerr << "  log rm                     remove log object\n";
+  cerr << "  usage show                 show usage (by user, date range)\n";
+  cerr << "  usage trim                 trim usage (by user, date range)\n";
+  cerr << "  temp remove                remove temporary objects that were created up to\n";
+  cerr << "                             specified date (and optional time)\n";
+  cerr << "  gc list                    dump expired garbage collection objects (specify\n";
+  cerr << "                             --include-all to list all entries, including unexpired)\n";
+  cerr << "  gc process                 manually process garbage\n";
+  cerr << "  metadata get               get metadata info\n";
+  cerr << "  metadata put               put metadata info\n";
+  cerr << "  metadata rm                remove metadata info\n";
+  cerr << "  metadata list              list metadata info\n";
+  cerr << "  mdlog list                 list metadata log\n";
+  cerr << "  mdlog trim                 trim metadata log\n";
+  cerr << "  bilog list                 list bucket index log\n";
+  cerr << "  bilog trim                 trim bucket index log (use start-marker, end-marker)\n";
+  cerr << "  datalog list               list data log\n";
+  cerr << "  datalog trim               trim data log\n";
+  cerr << "  opstate list               list stateful operations entries (use client_id,\n";
+  cerr << "                             op_id, object)\n";
+  cerr << "  opstate set                set state on an entry (use client_id, op_id, object, state)\n";
+  cerr << "  opstate renew              renew state on an entry (use client_id, op_id, object)\n";
+  cerr << "  opstate rm                 remove entry (use client_id, op_id, object)\n";
+  cerr << "  replicalog get             get replica metadata log entry\n";
+  cerr << "  replicalog delete          delete replica metadata log entry\n";
+  cerr << "options:\n";
+  cerr << "   --uid=                user id\n";
+  cerr << "   --subuser=          subuser name\n";
+  cerr << "   --access-key=        S3 access key\n";
+  cerr << "   --email=\n";
+  cerr << "   --secret=            specify secret key\n";
+  cerr << "   --gen-access-key          generate random access key (for S3)\n";
+  cerr << "   --gen-secret              generate random secret key\n";
+  cerr << "   --key-type=         key type, options are: swift, s3\n";
+  cerr << "   --temp-url-key[-2]=  temp url key\n";
+  cerr << "   --access=         Set access permissions for sub-user, should be one\n";
+  cerr << "                             of read, write, readwrite, full\n";
+  cerr << "   --display-name=\n";
+  cerr << "   --system                  set the system flag on the user\n";
+  cerr << "   --bucket=\n";
+  cerr << "   --pool=\n";
+  cerr << "   --object=\n";
+  cerr << "   --date=\n";
+  cerr << "   --start-date=\n";
+  cerr << "   --end-date=\n";
+  cerr << "   --bucket-id=\n";
+  cerr << "   --shard-id=     optional for mdlog list\n";
+  cerr << "                             required for: \n";
+  cerr << "                               mdlog trim\n";
+  cerr << "                               replica mdlog get/delete\n";
+  cerr << "                               replica datalog get/delete\n";
+  cerr << "   --metadata-key=      key to retrieve metadata from with metadata get\n";
+  cerr << "   --rgw-region=     region in which radosgw is running\n";
+  cerr << "   --rgw-zone=         zone in which radosgw is running\n";
+  cerr << "   --fix                     besides checking bucket index, will also fix it\n";
+  cerr << "   --check-objects           bucket check: rebuilds bucket index according to\n";
+  cerr << "                             actual objects state\n";
+  cerr << "   --format=         specify output format for certain operations: xml,\n";
+  cerr << "                             json\n";
+  cerr << "   --purge-data              when specified, user removal will also purge all the\n";
+  cerr << "                             user data\n";
+  cerr << "   --purge-keys              when specified, subuser removal will also purge all the\n";
+  cerr << "                             subuser keys\n";
+  cerr << "   --purge-objects           remove a bucket's objects before deleting it\n";
+  cerr << "                             (NOTE: required to delete a non-empty bucket)\n";
+  cerr << "   --sync-stats              option to 'user stats', update user stats with current\n";
+  cerr << "                             stats reported by user's buckets indexes\n";
+  cerr << "   --show-log-entries= enable/disable dump of log entries on log show\n";
+  cerr << "   --show-log-sum=     enable/disable dump of log summation on log show\n";
+  cerr << "   --skip-zero-entries       log show only dumps entries that don't have zero value\n";
+  cerr << "                             in one of the numeric field\n";
+  cerr << "   --infile                  specify a file to read in when setting data\n";
+  cerr << "   --state=    specify a state for the opstate set command\n";
+  cerr << "   --replica-log-type        replica log type (metadata, data, bucket), required for\n";
+  cerr << "                             replica log operations\n";
+  cerr << "   --categories=       comma separated list of categories, used in usage show\n";
+  cerr << "   --caps=             list of caps (e.g., \"usage=read, write; user=read\"\n";
+  cerr << "   --yes-i-really-mean-it    required for certain operations\n";
+  cerr << "\n";
+  cerr << " := \"YYYY-MM-DD[ hh:mm:ss]\"\n";
+  cerr << "\nQuota options:\n";
+  cerr << "   --bucket                  specified bucket for quota command\n";
+  cerr << "   --max-objects             specify max objects (negative value to disable)\n";
+  cerr << "   --max-size                specify max size (in bytes, negative value to disable)\n";
+  cerr << "   --quota-scope             scope of quota (bucket, user)\n";
+  cerr << "\n";
+  generic_client_usage();
+}
+
+int usage()
+{
+  _usage();
+  return 1;
+}
+
+void usage_exit()
+{
+  _usage();
+  exit(1);
+}
+
+enum {
+  OPT_NO_CMD = 0,
+  OPT_USER_CREATE,
+  OPT_USER_INFO,
+  OPT_USER_MODIFY,
+  OPT_USER_RM,
+  OPT_USER_SUSPEND,
+  OPT_USER_ENABLE,
+  OPT_USER_CHECK,
+  OPT_USER_STATS,
+  OPT_SUBUSER_CREATE,
+  OPT_SUBUSER_MODIFY,
+  OPT_SUBUSER_RM,
+  OPT_KEY_CREATE,
+  OPT_KEY_RM,
+  OPT_BUCKETS_LIST,
+  OPT_BUCKET_LINK,
+  OPT_BUCKET_UNLINK,
+  OPT_BUCKET_STATS,
+  OPT_BUCKET_CHECK,
+  OPT_BUCKET_RM,
+  OPT_POLICY,
+  OPT_POOL_ADD,
+  OPT_POOL_RM,
+  OPT_POOLS_LIST,
+  OPT_LOG_LIST,
+  OPT_LOG_SHOW,
+  OPT_LOG_RM,
+  OPT_USAGE_SHOW,
+  OPT_USAGE_TRIM,
+  OPT_TEMP_REMOVE,
+  OPT_OBJECT_RM,
+  OPT_OBJECT_UNLINK,
+  OPT_OBJECT_STAT,
+  OPT_QUOTA_SET,
+  OPT_QUOTA_ENABLE,
+  OPT_QUOTA_DISABLE,
+  OPT_GC_LIST,
+  OPT_GC_PROCESS,
+  OPT_REGION_GET,
+  OPT_REGION_LIST,
+  OPT_REGION_SET,
+  OPT_REGION_DEFAULT,
+  OPT_REGIONMAP_GET,
+  OPT_REGIONMAP_SET,
+  OPT_REGIONMAP_UPDATE,
+  OPT_ZONE_GET,
+  OPT_ZONE_SET,
+  OPT_ZONE_LIST,
+  OPT_CAPS_ADD,
+  OPT_CAPS_RM,
+  OPT_METADATA_GET,
+  OPT_METADATA_PUT,
+  OPT_METADATA_RM,
+  OPT_METADATA_LIST,
+  OPT_MDLOG_LIST,
+  OPT_MDLOG_TRIM,
+  OPT_BILOG_LIST,
+  OPT_BILOG_TRIM,
+  OPT_DATALOG_LIST,
+  OPT_DATALOG_TRIM,
+  OPT_OPSTATE_LIST,
+  OPT_OPSTATE_SET,
+  OPT_OPSTATE_RENEW,
+  OPT_OPSTATE_RM,
+  OPT_REPLICALOG_GET,
+  OPT_REPLICALOG_DELETE,
+};
+
+static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
+{
+  *need_more = false;
+  // NOTE: please keep the checks in alphabetical order !!!
+  if (strcmp(cmd, "bilog") == 0 ||
+      strcmp(cmd, "bucket") == 0 ||
+      strcmp(cmd, "buckets") == 0 ||
+      strcmp(cmd, "caps") == 0 ||
+      strcmp(cmd, "datalog") == 0 ||
+      strcmp(cmd, "gc") == 0 || 
+      strcmp(cmd, "key") == 0 ||
+      strcmp(cmd, "log") == 0 ||
+      strcmp(cmd, "mdlog") == 0 ||
+      strcmp(cmd, "metadata") == 0 ||
+      strcmp(cmd, "object") == 0 ||
+      strcmp(cmd, "opstate") == 0 ||
+      strcmp(cmd, "pool") == 0 ||
+      strcmp(cmd, "pools") == 0 ||
+      strcmp(cmd, "quota") == 0 ||
+      strcmp(cmd, "region") == 0 ||
+      strcmp(cmd, "regions") == 0 ||
+      strcmp(cmd, "region-map") == 0 ||
+      strcmp(cmd, "regionmap") == 0 ||
+      strcmp(cmd, "replicalog") == 0 ||
+      strcmp(cmd, "subuser") == 0 ||
+      strcmp(cmd, "temp") == 0 ||
+      strcmp(cmd, "usage") == 0 ||
+      strcmp(cmd, "user") == 0 ||
+      strcmp(cmd, "zone") == 0) {
+    *need_more = true;
+    return 0;
+  }
+
+  if (strcmp(cmd, "policy") == 0)
+    return OPT_POLICY;
+
+  if (!prev_cmd)
+    return -EINVAL;
+
+  if (strcmp(prev_cmd, "user") == 0) {
+    if (strcmp(cmd, "create") == 0)
+      return OPT_USER_CREATE;
+    if (strcmp(cmd, "info") == 0)
+      return OPT_USER_INFO;
+    if (strcmp(cmd, "modify") == 0)
+      return OPT_USER_MODIFY;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_USER_RM;
+    if (strcmp(cmd, "suspend") == 0)
+      return OPT_USER_SUSPEND;
+    if (strcmp(cmd, "enable") == 0)
+      return OPT_USER_ENABLE;
+    if (strcmp(cmd, "check") == 0)
+      return OPT_USER_CHECK;
+    if (strcmp(cmd, "stats") == 0)
+      return OPT_USER_STATS;
+  } else if (strcmp(prev_cmd, "subuser") == 0) {
+    if (strcmp(cmd, "create") == 0)
+      return OPT_SUBUSER_CREATE;
+    if (strcmp(cmd, "modify") == 0)
+      return OPT_SUBUSER_MODIFY;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_SUBUSER_RM;
+  } else if (strcmp(prev_cmd, "key") == 0) {
+    if (strcmp(cmd, "create") == 0)
+      return OPT_KEY_CREATE;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_KEY_RM;
+  } else if (strcmp(prev_cmd, "buckets") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_BUCKETS_LIST;
+  } else if (strcmp(prev_cmd, "bucket") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_BUCKETS_LIST;
+    if (strcmp(cmd, "link") == 0)
+      return OPT_BUCKET_LINK;
+    if (strcmp(cmd, "unlink") == 0)
+      return OPT_BUCKET_UNLINK;
+    if (strcmp(cmd, "stats") == 0)
+      return OPT_BUCKET_STATS;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_BUCKET_RM;
+    if (strcmp(cmd, "check") == 0)
+      return OPT_BUCKET_CHECK;
+  } else if (strcmp(prev_cmd, "log") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_LOG_LIST;
+    if (strcmp(cmd, "show") == 0)
+      return OPT_LOG_SHOW;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_LOG_RM;
+  } else if (strcmp(prev_cmd, "usage") == 0) {
+    if (strcmp(cmd, "show") == 0)
+      return OPT_USAGE_SHOW;
+    if (strcmp(cmd, "trim") == 0)
+      return OPT_USAGE_TRIM;
+  } else if (strcmp(prev_cmd, "temp") == 0) {
+    if (strcmp(cmd, "remove") == 0)
+      return OPT_TEMP_REMOVE;
+  } else if (strcmp(prev_cmd, "caps") == 0) {
+    if (strcmp(cmd, "add") == 0)
+      return OPT_CAPS_ADD;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_CAPS_RM;
+  } else if (strcmp(prev_cmd, "pool") == 0) {
+    if (strcmp(cmd, "add") == 0)
+      return OPT_POOL_ADD;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_POOL_RM;
+    if (strcmp(cmd, "list") == 0)
+      return OPT_POOLS_LIST;
+  } else if (strcmp(prev_cmd, "pools") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_POOLS_LIST;
+  } else if (strcmp(prev_cmd, "object") == 0) {
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_OBJECT_RM;
+    if (strcmp(cmd, "unlink") == 0)
+      return OPT_OBJECT_UNLINK;
+    if (strcmp(cmd, "stat") == 0)
+      return OPT_OBJECT_STAT;
+  } else if (strcmp(prev_cmd, "region") == 0) {
+    if (strcmp(cmd, "get") == 0)
+      return OPT_REGION_GET;
+    if (strcmp(cmd, "list") == 0)
+      return OPT_REGION_LIST;
+    if (strcmp(cmd, "set") == 0)
+      return OPT_REGION_SET;
+    if (strcmp(cmd, "default") == 0)
+      return OPT_REGION_DEFAULT;
+  } else if (strcmp(prev_cmd, "quota") == 0) {
+    if (strcmp(cmd, "set") == 0)
+      return OPT_QUOTA_SET;
+    if (strcmp(cmd, "enable") == 0)
+      return OPT_QUOTA_ENABLE;
+    if (strcmp(cmd, "disable") == 0)
+      return OPT_QUOTA_DISABLE;
+  } else if (strcmp(prev_cmd, "regions") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_REGION_LIST;
+  } else if (strcmp(prev_cmd, "region-map") == 0 ||
+             strcmp(prev_cmd, "regionmap") == 0) {
+    if (strcmp(cmd, "get") == 0)
+      return OPT_REGIONMAP_GET;
+    if (strcmp(cmd, "set") == 0)
+      return OPT_REGIONMAP_SET;
+    if (strcmp(cmd, "update") == 0)
+      return OPT_REGIONMAP_UPDATE;
+  } else if (strcmp(prev_cmd, "zone") == 0) {
+    if (strcmp(cmd, "get") == 0)
+      return OPT_ZONE_GET;
+    if (strcmp(cmd, "set") == 0)
+      return OPT_ZONE_SET;
+    if (strcmp(cmd, "list") == 0)
+      return OPT_ZONE_LIST;
+  } else if (strcmp(prev_cmd, "zones") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_ZONE_LIST;
+  } else if (strcmp(prev_cmd, "gc") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_GC_LIST;
+    if (strcmp(cmd, "process") == 0)
+      return OPT_GC_PROCESS;
+  } else if (strcmp(prev_cmd, "metadata") == 0) {
+    if (strcmp(cmd, "get") == 0)
+      return OPT_METADATA_GET;
+    if (strcmp(cmd, "put") == 0)
+      return OPT_METADATA_PUT;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_METADATA_RM;
+    if (strcmp(cmd, "list") == 0)
+      return OPT_METADATA_LIST;
+  } else if (strcmp(prev_cmd, "mdlog") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_MDLOG_LIST;
+    if (strcmp(cmd, "trim") == 0)
+      return OPT_MDLOG_TRIM;
+  } else if (strcmp(prev_cmd, "bilog") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_BILOG_LIST;
+    if (strcmp(cmd, "trim") == 0)
+      return OPT_BILOG_TRIM;
+  } else if (strcmp(prev_cmd, "datalog") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_DATALOG_LIST;
+    if (strcmp(cmd, "trim") == 0)
+      return OPT_DATALOG_TRIM;
+  } else if (strcmp(prev_cmd, "opstate") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_OPSTATE_LIST;
+    if (strcmp(cmd, "set") == 0)
+      return OPT_OPSTATE_SET;
+    if (strcmp(cmd, "renew") == 0)
+      return OPT_OPSTATE_RENEW;
+    if (strcmp(cmd, "rm") == 0)
+      return OPT_OPSTATE_RM;
+  } else if (strcmp(prev_cmd, "replicalog") == 0) {
+    if (strcmp(cmd, "get") == 0)
+      return OPT_REPLICALOG_GET;
+    if (strcmp(cmd, "delete") == 0)
+      return OPT_REPLICALOG_DELETE;
+  }
+
+  return -EINVAL;
+}
+
+enum ReplicaLogType {
+  ReplicaLog_Invalid = 0,
+  ReplicaLog_Metadata,
+  ReplicaLog_Data,
+  ReplicaLog_Bucket,
+};
+
+ReplicaLogType get_replicalog_type(const string& name) {
+  if (name == "md" || name == "meta" || name == "metadata")
+    return ReplicaLog_Metadata;
+  if (name == "data")
+    return ReplicaLog_Data;
+  if (name == "bucket")
+    return ReplicaLog_Bucket;
+
+  return ReplicaLog_Invalid;
+}
+
+static void show_user_info(RGWUserInfo& info, Formatter *formatter)
+{
+  encode_json("user_info", info, formatter);
+  formatter->flush(cout);
+  cout << std::endl;
+}
+
+static void dump_bucket_usage(map& stats, Formatter *formatter)
+{
+  map::iterator iter;
+
+  formatter->open_object_section("usage");
+  for (iter = stats.begin(); iter != stats.end(); ++iter) {
+    RGWStorageStats& s = iter->second;
+    const char *cat_name = rgw_obj_category_name(iter->first);
+    formatter->open_object_section(cat_name);
+    formatter->dump_int("size_kb", s.num_kb);
+    formatter->dump_int("size_kb_actual", s.num_kb_rounded);
+    formatter->dump_int("num_objects", s.num_objects);
+    formatter->close_section();
+    formatter->flush(cout);
+  }
+  formatter->close_section();
+}
+
+int bucket_stats(rgw_bucket& bucket, Formatter *formatter)
+{
+  RGWBucketInfo bucket_info;
+  time_t mtime;
+  int r = store->get_bucket_info(NULL, bucket.name, bucket_info, &mtime);
+  if (r < 0)
+    return r;
+
+  map stats;
+  uint64_t bucket_ver, master_ver;
+  string max_marker;
+  int ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, &max_marker);
+  if (ret < 0) {
+    cerr << "error getting bucket stats ret=" << ret << std::endl;
+    return ret;
+  }
+  formatter->open_object_section("stats");
+  formatter->dump_string("bucket", bucket.name);
+  formatter->dump_string("pool", bucket.data_pool);
+  formatter->dump_string("index_pool", bucket.index_pool);
+  
+  formatter->dump_string("id", bucket.bucket_id);
+  formatter->dump_string("marker", bucket.marker);
+  formatter->dump_string("owner", bucket_info.owner);
+  formatter->dump_int("mtime", mtime);
+  formatter->dump_int("ver", bucket_ver);
+  formatter->dump_int("master_ver", master_ver);
+  formatter->dump_string("max_marker", max_marker);
+  dump_bucket_usage(stats, formatter);
+  formatter->close_section();
+
+  return 0;
+}
+
+class StoreDestructor {
+  RGWRados *store;
+public:
+  StoreDestructor(RGWRados *_s) : store(_s) {}
+  ~StoreDestructor() {
+    RGWStoreManager::close_storage(store);
+  }
+};
+
+static int init_bucket(string& bucket_name, RGWBucketInfo& bucket_info, rgw_bucket& bucket)
+{
+  if (!bucket_name.empty()) {
+    int r = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL);
+    if (r < 0) {
+      cerr << "could not get bucket info for bucket=" << bucket_name << std::endl;
+      return r;
+    }
+    bucket = bucket_info.bucket;
+  }
+  return 0;
+}
+
+static int read_input(const string& infile, bufferlist& bl)
+{
+  int fd = 0;
+  if (infile.size()) {
+    fd = open(infile.c_str(), O_RDONLY);
+    if (fd < 0) {
+      int err = -errno;
+      cerr << "error reading input file " << infile << std::endl;
+      return err;
+    }
+  }
+
+#define READ_CHUNK 8196
+  int r;
+  int err;
+
+  do {
+    char buf[READ_CHUNK];
+
+    r = read(fd, buf, READ_CHUNK);
+    if (r < 0) {
+      err = -errno;
+      cerr << "error while reading input" << std::endl;
+      goto out;
+    }
+    bl.append(buf, r);
+  } while (r > 0);
+  err = 0;
+
+ out:
+  if (infile.size()) {
+    close(fd);
+  }
+  return err;
+}
+
+template 
+static int read_decode_json(const string& infile, T& t)
+{
+  bufferlist bl;
+  int ret = read_input(infile, bl);
+  if (ret < 0) {
+    cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
+    return ret;
+  }
+  JSONParser p;
+  ret = p.parse(bl.c_str(), bl.length());
+  if (ret < 0) {
+    cout << "failed to parse JSON" << std::endl;
+    return ret;
+  }
+
+  try {
+    t.decode_json(&p);
+  } catch (JSONDecoder::err& e) {
+    cout << "failed to decode JSON input: " << e.message << std::endl;
+    return -EINVAL;
+  }
+  return 0;
+}
+    
+static int parse_date_str(const string& date_str, utime_t& ut)
+{
+  uint64_t epoch = 0;
+  uint64_t nsec = 0;
+
+  if (!date_str.empty()) {
+    int ret = utime_t::parse_date(date_str, &epoch, &nsec);
+    if (ret < 0) {
+      cerr << "ERROR: failed to parse date: " << date_str << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  ut = utime_t(epoch, nsec);
+
+  return 0;
+}
+
+template 
+static bool decode_dump(const char *field_name, bufferlist& bl, Formatter *f)
+{
+  T t;
+
+  bufferlist::iterator iter = bl.begin();
+
+  try {
+    ::decode(t, iter);
+  } catch (buffer::error& err) {
+    return false;
+  }
+
+  encode_json(field_name, t, f);
+
+  return true;
+}
+
+static bool dump_string(const char *field_name, bufferlist& bl, Formatter *f)
+{
+  string val;
+  if (bl.length() > 0) {
+    val.assign(bl.c_str(), bl.length());
+  }
+  f->dump_string(field_name, val);
+
+  return true;
+}
+
+void set_quota_info(RGWQuotaInfo& quota, int opt_cmd, int64_t max_size, int64_t max_objects,
+                    bool have_max_size, bool have_max_objects)
+{
+  switch (opt_cmd) {
+    case OPT_QUOTA_ENABLE:
+      quota.enabled = true;
+
+      // falling through on purpose
+
+    case OPT_QUOTA_SET:
+      if (have_max_objects) {
+        quota.max_objects = max_objects;
+      }
+      if (have_max_size) {
+        quota.max_size_kb = rgw_rounded_kb(max_size);
+      }
+      break;
+    case OPT_QUOTA_DISABLE:
+      quota.enabled = false;
+      break;
+  }
+}
+
+int set_bucket_quota(RGWRados *store, int opt_cmd, string& bucket_name, int64_t max_size, int64_t max_objects,
+                     bool have_max_size, bool have_max_objects)
+{
+  RGWBucketInfo bucket_info;
+  map attrs;
+  int r = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL, &attrs);
+  if (r < 0) {
+    cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
+    return -r;
+  }
+
+  set_quota_info(bucket_info.quota, opt_cmd, max_size, max_objects, have_max_size, have_max_objects);
+
+   r = store->put_bucket_instance_info(bucket_info, false, 0, &attrs);
+  if (r < 0) {
+    cerr << "ERROR: failed writing bucket instance info: " << cpp_strerror(-r) << std::endl;
+    return -r;
+  }
+  return 0;
+}
+
+int set_user_bucket_quota(int opt_cmd, RGWUser& user, RGWUserAdminOpState& op_state, int64_t max_size, int64_t max_objects,
+                          bool have_max_size, bool have_max_objects)
+{
+  RGWUserInfo& user_info = op_state.get_user_info();
+
+  set_quota_info(user_info.bucket_quota, opt_cmd, max_size, max_objects, have_max_size, have_max_objects);
+
+  op_state.set_bucket_quota(user_info.bucket_quota);
+
+  string err;
+  int r = user.modify(op_state, &err);
+  if (r < 0) {
+    cerr << "ERROR: failed updating user info: " << cpp_strerror(-r) << ": " << err << std::endl;
+    return -r;
+  }
+  return 0;
+}
+
+int set_user_quota(int opt_cmd, RGWUser& user, RGWUserAdminOpState& op_state, int64_t max_size, int64_t max_objects,
+                   bool have_max_size, bool have_max_objects)
+{
+  RGWUserInfo& user_info = op_state.get_user_info();
+
+  set_quota_info(user_info.user_quota, opt_cmd, max_size, max_objects, have_max_size, have_max_objects);
+
+  op_state.set_user_quota(user_info.user_quota);
+
+  string err;
+  int r = user.modify(op_state, &err);
+  if (r < 0) {
+    cerr << "ERROR: failed updating user info: " << cpp_strerror(-r) << ": " << err << std::endl;
+    return -r;
+  }
+  return 0;
+}
+
+int main(int argc, char **argv) 
+{
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  std::string user_id, access_key, secret_key, user_email, display_name;
+  std::string bucket_name, pool_name, object;
+  std::string date, subuser, access, format;
+  std::string start_date, end_date;
+  std::string key_type_str;
+  int key_type = KEY_TYPE_UNDEFINED;
+  rgw_bucket bucket;
+  uint32_t perm_mask = 0;
+  RGWUserInfo info;
+  int opt_cmd = OPT_NO_CMD;
+  bool need_more;
+  int gen_access_key = 0;
+  int gen_secret_key = 0;
+  bool set_perm = false;
+  bool set_temp_url_key = false;
+  map temp_url_keys;
+  string bucket_id;
+  Formatter *formatter = NULL;
+  int purge_data = false;
+  RGWBucketInfo bucket_info;
+  int pretty_format = false;
+  int show_log_entries = true;
+  int show_log_sum = true;
+  int skip_zero_entries = false;  // log show
+  int purge_keys = false;
+  int yes_i_really_mean_it = false;
+  int delete_child_objects = false;
+  int fix = false;
+  int max_buckets = -1;
+  map categories;
+  string caps;
+  int check_objects = false;
+  RGWUserAdminOpState user_op;
+  RGWBucketAdminOpState bucket_op;
+  string infile;
+  string metadata_key;
+  RGWObjVersionTracker objv_tracker;
+  string marker;
+  string start_marker;
+  string end_marker;
+  int max_entries = -1;
+  int system = false;
+  bool system_specified = false;
+  int shard_id = -1;
+  bool specified_shard_id = false;
+  string daemon_id;
+  bool specified_daemon_id = false;
+  string client_id;
+  string op_id;
+  string state_str;
+  string replica_log_type_str;
+  ReplicaLogType replica_log_type = ReplicaLog_Invalid;
+  string op_mask_str;
+  string quota_scope;
+
+  int64_t max_objects = -1;
+  int64_t max_size = -1;
+  bool have_max_objects = false;
+  bool have_max_size = false;
+  int include_all = false;
+
+  int sync_stats = false;
+
+  std::string val;
+  std::ostringstream errs;
+  string err;
+  long long tmp = 0;
+  for (std::vector::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+      return 0;
+    } else if (ceph_argparse_witharg(args, i, &val, "-i", "--uid", (char*)NULL)) {
+      user_id = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--access-key", (char*)NULL)) {
+      access_key = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--subuser", (char*)NULL)) {
+      subuser = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--secret", (char*)NULL)) {
+      secret_key = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-e", "--email", (char*)NULL)) {
+      user_email = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-n", "--display-name", (char*)NULL)) {
+      display_name = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-b", "--bucket", (char*)NULL)) {
+      bucket_name = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-p", "--pool", (char*)NULL)) {
+      pool_name = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-o", "--object", (char*)NULL)) {
+      object = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--client-id", (char*)NULL)) {
+      client_id = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--op-id", (char*)NULL)) {
+      op_id = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--state", (char*)NULL)) {
+      state_str = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--op-mask", (char*)NULL)) {
+      op_mask_str = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--key-type", (char*)NULL)) {
+      key_type_str = val;
+      if (key_type_str.compare("swift") == 0) {
+        key_type = KEY_TYPE_SWIFT;
+      } else if (key_type_str.compare("s3") == 0) {
+        key_type = KEY_TYPE_S3;
+      } else {
+        cerr << "bad key type: " << key_type_str << std::endl;
+        return usage();
+      }
+    } else if (ceph_argparse_binary_flag(args, i, &gen_access_key, NULL, "--gen-access-key", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &gen_secret_key, NULL, "--gen-secret", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &show_log_entries, NULL, "--show_log_entries", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &show_log_sum, NULL, "--show_log_sum", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &skip_zero_entries, NULL, "--skip_zero_entries", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &system, NULL, "--system", (char*)NULL)) {
+      system_specified = true;
+    } else if (ceph_argparse_withlonglong(args, i, &tmp, &errs, "-a", "--auth-uid", (char*)NULL)) {
+      if (!errs.str().empty()) {
+	cerr << errs.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-buckets", (char*)NULL)) {
+      max_buckets = atoi(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-entries", (char*)NULL)) {
+      max_entries = atoi(val.c_str());
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-size", (char*)NULL)) {
+      max_size = (int64_t)strict_strtoll(val.c_str(), 10, &err);
+      if (!err.empty()) {
+        cerr << "ERROR: failed to parse max size: " << err << std::endl;
+        return EINVAL;
+      }
+      have_max_size = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-objects", (char*)NULL)) {
+      max_objects = (int64_t)strict_strtoll(val.c_str(), 10, &err);
+      if (!err.empty()) {
+        cerr << "ERROR: failed to parse max objects: " << err << std::endl;
+        return EINVAL;
+      }
+      have_max_objects = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--date", "--time", (char*)NULL)) {
+      date = val;
+      if (end_date.empty())
+        end_date = date;
+    } else if (ceph_argparse_witharg(args, i, &val, "--start-date", "--start-time", (char*)NULL)) {
+      start_date = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--end-date", "--end-time", (char*)NULL)) {
+      end_date = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--shard-id", (char*)NULL)) {
+      shard_id = atoi(val.c_str());
+      specified_shard_id = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--daemon-id", (char*)NULL)) {
+      daemon_id = val;
+      specified_daemon_id = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--access", (char*)NULL)) {
+      access = val;
+      perm_mask = rgw_str_to_perm(access.c_str());
+      set_perm = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--temp-url-key", (char*)NULL)) {
+      temp_url_keys[0] = val;
+      set_temp_url_key = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--temp-url-key2", "--temp-url-key-2", (char*)NULL)) {
+      temp_url_keys[1] = val;
+      set_temp_url_key = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--bucket-id", (char*)NULL)) {
+      bucket_id = val;
+      if (bucket_id.empty()) {
+        cerr << "bad bucket-id" << std::endl;
+        return usage();
+      }
+    } else if (ceph_argparse_witharg(args, i, &val, "--format", (char*)NULL)) {
+      format = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--categories", (char*)NULL)) {
+      string cat_str = val;
+      list cat_list;
+      list::iterator iter;
+      get_str_list(cat_str, cat_list);
+      for (iter = cat_list.begin(); iter != cat_list.end(); ++iter) {
+	categories[*iter] = true;
+      }
+    } else if (ceph_argparse_binary_flag(args, i, &delete_child_objects, NULL, "--purge-objects", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &pretty_format, NULL, "--pretty-format", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &purge_data, NULL, "--purge-data", (char*)NULL)) {
+      delete_child_objects = purge_data;
+    } else if (ceph_argparse_binary_flag(args, i, &purge_keys, NULL, "--purge-keys", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &yes_i_really_mean_it, NULL, "--yes-i-really-mean-it", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &fix, NULL, "--fix", (char*)NULL)) {
+      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &check_objects, NULL, "--check-objects", (char*)NULL)) {
+     // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) {
+     // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &include_all, NULL, "--include-all", (char*)NULL)) {
+     // do nothing
+    } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
+      caps = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) {
+      infile = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--metadata-key", (char*)NULL)) {
+      metadata_key = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--marker", (char*)NULL)) {
+      marker = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--start-marker", (char*)NULL)) {
+      start_marker = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--end-marker", (char*)NULL)) {
+      end_marker = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--quota-scope", (char*)NULL)) {
+      quota_scope = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--replica-log-type", (char*)NULL)) {
+      replica_log_type_str = val;
+      replica_log_type = get_replicalog_type(replica_log_type_str);
+      if (replica_log_type == ReplicaLog_Invalid) {
+        cerr << "ERROR: invalid replica log type" << std::endl;
+        return EINVAL;
+      }
+    } else if (strncmp(*i, "-", 1) == 0) {
+      cerr << "ERROR: invalid flag " << *i << std::endl;
+      return EINVAL;
+    } else {
+      ++i;
+    }
+  }
+
+  if (args.empty()) {
+    return usage();
+  }
+  else {
+    const char *prev_cmd = NULL;
+    std::vector::iterator i ;
+    for (i = args.begin(); i != args.end(); ++i) {
+      opt_cmd = get_cmd(*i, prev_cmd, &need_more);
+      if (opt_cmd < 0) {
+	cerr << "unrecognized arg " << *i << std::endl;
+	return usage();
+      }
+      if (!need_more) {
+	++i;
+	break;
+      }
+      prev_cmd = *i;
+    }
+
+    if (opt_cmd == OPT_NO_CMD)
+      return usage();
+
+    /* some commands may have an optional extra param */
+    if (i != args.end()) {
+      switch (opt_cmd) {
+        case OPT_METADATA_GET:
+        case OPT_METADATA_PUT:
+        case OPT_METADATA_RM:
+        case OPT_METADATA_LIST:
+          metadata_key = *i;
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  // default to pretty json
+  if (format.empty()) {
+    format = "json";
+    pretty_format = true;
+  }
+
+  if (format ==  "xml")
+    formatter = new XMLFormatter(pretty_format);
+  else if (format == "json")
+    formatter = new JSONFormatter(pretty_format);
+  else {
+    cerr << "unrecognized format: " << format << std::endl;
+    return usage();
+  }
+
+  RGWStreamFlusher f(formatter, cout);
+
+  bool raw_storage_op = (opt_cmd == OPT_REGION_GET || opt_cmd == OPT_REGION_LIST ||
+                         opt_cmd == OPT_REGION_SET || opt_cmd == OPT_REGION_DEFAULT ||
+                         opt_cmd == OPT_REGIONMAP_GET || opt_cmd == OPT_REGIONMAP_SET ||
+                         opt_cmd == OPT_REGIONMAP_UPDATE ||
+                         opt_cmd == OPT_ZONE_GET || opt_cmd == OPT_ZONE_SET ||
+                         opt_cmd == OPT_ZONE_LIST);
+
+
+  if (raw_storage_op) {
+    store = RGWStoreManager::get_raw_storage(g_ceph_context);
+  } else {
+    store = RGWStoreManager::get_storage(g_ceph_context, false, false);
+  }
+  if (!store) {
+    cerr << "couldn't init storage provider" << std::endl;
+    return 5; //EIO
+  }
+
+  rgw_user_init(store->meta_mgr);
+  rgw_bucket_init(store->meta_mgr);
+
+  StoreDestructor store_destructor(store);
+
+  if (raw_storage_op) {
+    if (opt_cmd == OPT_REGION_GET) {
+      RGWRegion region;
+      int ret = region.init(g_ceph_context, store);
+      if (ret < 0) {
+        cerr << "failed to init region: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+
+      encode_json("region", region, formatter);
+      formatter->flush(cout);
+      cout << std::endl;
+    }
+    if (opt_cmd == OPT_REGION_LIST) {
+      RGWRegion region;
+      int ret = region.init(g_ceph_context, store, false);
+      if (ret < 0) {
+	cerr << "failed to init region: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+
+      list regions;
+      ret = store->list_regions(regions);
+      if (ret < 0) {
+        cerr << "failed to list regions: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+      RGWDefaultRegionInfo default_region;
+      ret = region.read_default(default_region);
+      if (ret < 0 && ret != -ENOENT) {
+	cerr << "could not determine default region: " << cpp_strerror(-ret) << std::endl;
+      }
+      formatter->open_object_section("regions_list");
+      encode_json("default_info", default_region, formatter);
+      encode_json("regions", regions, formatter);
+      formatter->close_section();
+      formatter->flush(cout);
+      cout << std::endl;
+    }
+    if (opt_cmd == OPT_REGION_SET) {
+      RGWRegion region;
+      int ret = region.init(g_ceph_context, store, false);
+      if (ret < 0) {
+	cerr << "failed to init region: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+      ret = read_decode_json(infile, region);
+      if (ret < 0) {
+        return 1;
+      }
+
+      ret = region.store_info(false);
+      if (ret < 0) {
+        cerr << "ERROR: couldn't store zone info: " << cpp_strerror(-ret) << std::endl;
+        return 1;
+      }
+
+      encode_json("region", region, formatter);
+      formatter->flush(cout);
+    }
+    if (opt_cmd == OPT_REGION_DEFAULT) {
+      RGWRegion region;
+      int ret = region.init(g_ceph_context, store);
+      if (ret < 0) {
+	cerr << "failed to init region: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+
+      ret = region.set_as_default();
+      if (ret < 0) {
+	cerr << "failed to set region as default: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+    }
+
+    if (opt_cmd == OPT_REGIONMAP_GET) {
+      RGWRegionMap regionmap;
+      int ret = regionmap.read(g_ceph_context, store);
+      if (ret < 0) {
+	cerr << "failed to read region map: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+      encode_json("region-map", regionmap, formatter);
+      formatter->flush(cout);
+    }
+
+    if (opt_cmd == OPT_REGIONMAP_SET) {
+      RGWRegionMap regionmap;
+      int ret = read_decode_json(infile, regionmap);
+      if (ret < 0) {
+        return 1;
+      }
+
+      ret = regionmap.store(g_ceph_context, store);
+      if (ret < 0) {
+        cerr << "ERROR: couldn't store region map info: " << cpp_strerror(-ret) << std::endl;
+        return 1;
+      }
+
+      encode_json("region-map", regionmap, formatter);
+      formatter->flush(cout);
+    }
+
+    if (opt_cmd == OPT_REGIONMAP_UPDATE) {
+      RGWRegionMap regionmap;
+      int ret = regionmap.read(g_ceph_context, store);
+      if (ret < 0 && ret != -ENOENT) {
+	cerr << "failed to read region map: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+
+      RGWRegion region;
+      ret = region.init(g_ceph_context, store, false);
+      if (ret < 0) {
+	cerr << "failed to init region: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+
+      list regions;
+      ret = store->list_regions(regions);
+      if (ret < 0) {
+        cerr << "failed to list regions: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+
+      for (list::iterator iter = regions.begin(); iter != regions.end(); ++iter) {
+        ret = region.read_info(*iter);
+        if (ret < 0) {
+        cerr << "failed to read region info (name=" << *iter << "): " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+        }
+        regionmap.update(region);
+      }
+
+      ret = regionmap.store(g_ceph_context, store);
+      if (ret < 0) {
+        cerr << "ERROR: couldn't store region map info: " << cpp_strerror(-ret) << std::endl;
+        return 1;
+      }
+
+      encode_json("region-map", regionmap, formatter);
+      formatter->flush(cout);
+    }
+
+    if (opt_cmd == OPT_ZONE_GET) {
+      RGWRegion region;
+      int ret = region.init(g_ceph_context, store);
+      if (ret < 0) {
+        cerr << "WARNING: failed to initialize region" << std::endl;
+      }
+      RGWZoneParams zone;
+      ret = zone.init(g_ceph_context, store, region);
+      if (ret < 0) {
+	cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+      encode_json("zone", zone, formatter);
+      formatter->flush(cout);
+    }
+
+    if (opt_cmd == OPT_ZONE_SET) {
+      RGWRegion region;
+      int ret = region.init(g_ceph_context, store);
+      if (ret < 0) {
+        cerr << "WARNING: failed to initialize region" << std::endl;
+      }
+      RGWZoneParams zone;
+      zone.init_default(store);
+      ret = read_decode_json(infile, zone);
+      if (ret < 0) {
+        return 1;
+      }
+
+      ret = zone.store_info(g_ceph_context, store, region);
+      if (ret < 0) {
+        cerr << "ERROR: couldn't store zone info: " << cpp_strerror(-ret) << std::endl;
+        return 1;
+      }
+
+      encode_json("zone", zone, formatter);
+      formatter->flush(cout);
+    }
+    if (opt_cmd == OPT_ZONE_LIST) {
+      list zones;
+      int ret = store->list_zones(zones);
+      if (ret < 0) {
+        cerr << "failed to list zones: " << cpp_strerror(-ret) << std::endl;
+	return -ret;
+      }
+      formatter->open_object_section("zones_list");
+      encode_json("zones", zones, formatter);
+      formatter->close_section();
+      formatter->flush(cout);
+      cout << std::endl;
+    }
+    return 0;
+  }
+
+  if (!user_id.empty()) {
+    user_op.set_user_id(user_id);
+    bucket_op.set_user_id(user_id);
+  }
+
+  if (!display_name.empty())
+    user_op.set_display_name(display_name);
+
+  if (!user_email.empty())
+    user_op.set_user_email(user_email);
+
+  if (!access_key.empty())
+    user_op.set_access_key(access_key);
+
+  if (!secret_key.empty())
+    user_op.set_secret_key(secret_key);
+
+  if (!subuser.empty())
+    user_op.set_subuser(subuser);
+
+  if (!caps.empty())
+    user_op.set_caps(caps);
+
+  user_op.set_purge_data(purge_data);
+
+  if (purge_keys)
+    user_op.set_purge_keys();
+
+  if (gen_access_key)
+    user_op.set_generate_key();
+
+  if (gen_secret_key)
+    user_op.set_gen_secret(); // assume that a key pair should be created
+
+  if (max_buckets >= 0)
+    user_op.set_max_buckets(max_buckets);
+
+  if (system_specified)
+    user_op.set_system(system);
+
+  if (set_perm)
+    user_op.set_perm(perm_mask);
+
+  if (set_temp_url_key) {
+    map::iterator iter = temp_url_keys.begin();
+    for (; iter != temp_url_keys.end(); ++iter) {
+      user_op.set_temp_url_key(iter->second, iter->first);
+    }
+  }
+
+  if (!op_mask_str.empty()) {
+    uint32_t op_mask;
+    int ret = rgw_parse_op_type_list(op_mask_str, &op_mask);
+    if (ret < 0) {
+      cerr << "failed to parse op_mask: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+
+    user_op.set_op_mask(op_mask);
+  }
+
+  if (key_type != KEY_TYPE_UNDEFINED)
+    user_op.set_key_type(key_type);
+
+  // set suspension operation parameters
+  if (opt_cmd == OPT_USER_ENABLE)
+    user_op.set_suspension(false);
+  else if (opt_cmd == OPT_USER_SUSPEND)
+    user_op.set_suspension(true);
+
+  // RGWUser to use for user operations
+  RGWUser user;
+  int ret = 0;
+  if (!user_id.empty() || !subuser.empty()) {
+    ret = user.init(store, user_op);
+    if (ret < 0) {
+      cerr << "user.init failed: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+
+  /* populate bucket operation */
+  bucket_op.set_bucket_name(bucket_name);
+  bucket_op.set_object(object);
+  bucket_op.set_check_objects(check_objects);
+  bucket_op.set_delete_children(delete_child_objects);
+
+  // required to gather errors from operations
+  std::string err_msg;
+
+  bool output_user_info = true;
+
+  switch (opt_cmd) {
+  case OPT_USER_INFO:
+    break;
+  case OPT_USER_CREATE:
+    if (!user_op.has_existing_user()) {
+      user_op.set_generate_key(); // generate a new key by default
+    }
+    ret = user.add(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not create user: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  case OPT_USER_RM:
+    ret = user.remove(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not remove user: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    output_user_info = false;
+    break;
+  case OPT_USER_ENABLE:
+  case OPT_USER_SUSPEND:
+  case OPT_USER_MODIFY:
+    ret = user.modify(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not modify user: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  case OPT_SUBUSER_CREATE:
+    ret = user.subusers.add(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not create subuser: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  case OPT_SUBUSER_MODIFY:
+    ret = user.subusers.modify(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not modify subuser: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    ret = user.info(info, &err_msg);
+    if (ret < 0) {
+      cerr << "could not fetch user info: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    show_user_info(info, formatter);
+
+    break;
+  case OPT_SUBUSER_RM:
+    ret = user.subusers.remove(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not remove subuser: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  case OPT_CAPS_ADD:
+    ret = user.caps.add(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not add caps: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  case OPT_CAPS_RM:
+    ret = user.caps.remove(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not add remove caps: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  case OPT_KEY_CREATE:
+    ret = user.keys.add(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not create key: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  case OPT_KEY_RM:
+    ret = user.keys.remove(user_op, &err_msg);
+    if (ret < 0) {
+      cerr << "could not remove key: " << err_msg << std::endl;
+      return -ret;
+    }
+
+    break;
+  default:
+    output_user_info = false;
+  }
+
+  // output the result of a user operation
+  if (output_user_info) {
+    ret = user.info(info, &err_msg);
+    if (ret < 0) {
+      cerr << "could not fetch user info: " << err_msg << std::endl;
+      return -ret;
+    }
+    show_user_info(info, formatter);
+  }
+
+  if (opt_cmd == OPT_POLICY) {
+    int ret = RGWBucketAdminOp::get_policy(store, bucket_op, cout);
+    if (ret >= 0) {
+      cout << std::endl;
+    }
+  }
+
+  if (opt_cmd == OPT_BUCKETS_LIST) {
+    if (bucket_name.empty()) {
+      RGWBucketAdminOp::info(store, bucket_op, f);
+    } else {
+      RGWBucketInfo bucket_info;
+      int ret = init_bucket(bucket_name, bucket_info, bucket);
+      if (ret < 0) {
+        cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+      formatter->open_array_section("entries");
+      bool truncated;
+      int count = 0;
+      if (max_entries < 0)
+        max_entries = 1000;
+
+      string prefix;
+      string delim;
+      vector result;
+      map common_prefixes;
+      string ns;
+      
+      do {
+        list entries;
+        ret = store->list_objects(bucket, max_entries - count, prefix, delim,
+                                  marker, NULL, result, common_prefixes, true,
+                                  ns, false, &truncated, NULL);
+        if (ret < 0) {
+          cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl;
+          return -ret;
+        }
+
+        count += result.size();
+
+        for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) {
+          RGWObjEnt& entry = *iter;
+          encode_json("entry", entry, formatter);
+
+          marker = entry.name;
+        }
+        formatter->flush(cout);
+      } while (truncated && count < max_entries);
+
+      formatter->close_section();
+      formatter->flush(cout);
+    }
+  }
+
+  if (opt_cmd == OPT_BUCKET_STATS) {
+    bucket_op.set_fetch_stats(true);
+
+    RGWBucketAdminOp::info(store, bucket_op, f);
+  }
+
+  if (opt_cmd == OPT_BUCKET_LINK) {
+    int r = RGWBucketAdminOp::link(store, bucket_op);
+    if (r < 0) {
+      cerr << "failure: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+  }
+
+  if (opt_cmd == OPT_BUCKET_UNLINK) {
+    int r = RGWBucketAdminOp::unlink(store, bucket_op);
+    if (r < 0) {
+      cerr << "failure: " << cpp_strerror(-r) << std::endl;
+      return -r;
+    }
+  }
+
+  if (opt_cmd == OPT_TEMP_REMOVE) {
+    if (date.empty()) {
+      cerr << "date wasn't specified" << std::endl;
+      return usage();
+    }
+    string parsed_date, parsed_time;
+    int r = utime_t::parse_date(date, NULL, NULL, &parsed_date, &parsed_time);
+    if (r < 0) {
+      cerr << "failure parsing date: " << cpp_strerror(r) << std::endl;
+      return 1;
+    }
+    r = store->remove_temp_objects(parsed_date, parsed_time);
+    if (r < 0) {
+      cerr << "failure removing temp objects: " << cpp_strerror(r) << std::endl;
+      return 1;
+    }
+  }
+
+  if (opt_cmd == OPT_LOG_LIST) {
+    // filter by date?
+    if (date.size() && date.size() != 10) {
+      cerr << "bad date format for '" << date << "', expect YYYY-MM-DD" << std::endl;
+      return -EINVAL;
+    }
+
+    formatter->reset();
+    formatter->open_array_section("logs");
+    RGWAccessHandle h;
+    int r = store->log_list_init(date, &h);
+    if (r == -ENOENT) {
+      // no logs.
+    } else {
+      if (r < 0) {
+	cerr << "log list: error " << r << std::endl;
+	return r;
+      }
+      while (true) {
+	string name;
+	int r = store->log_list_next(h, &name);
+	if (r == -ENOENT)
+	  break;
+	if (r < 0) {
+	  cerr << "log list: error " << r << std::endl;
+	  return r;
+	}
+	formatter->dump_string("object", name);
+      }
+    }
+    formatter->close_section();
+    formatter->flush(cout);
+    cout << std::endl;
+  }
+
+  if (opt_cmd == OPT_LOG_SHOW || opt_cmd == OPT_LOG_RM) {
+    if (object.empty() && (date.empty() || bucket_name.empty() || bucket_id.empty())) {
+      cerr << "specify an object or a date, bucket and bucket-id" << std::endl;
+      return usage();
+    }
+
+    string oid;
+    if (!object.empty()) {
+      oid = object;
+    } else {
+      oid = date;
+      oid += "-";
+      oid += bucket_id;
+      oid += "-";
+      oid += string(bucket.name);
+    }
+
+    if (opt_cmd == OPT_LOG_SHOW) {
+      RGWAccessHandle h;
+
+      int r = store->log_show_init(oid, &h);
+      if (r < 0) {
+	cerr << "error opening log " << oid << ": " << cpp_strerror(-r) << std::endl;
+	return -r;
+      }
+
+      formatter->reset();
+      formatter->open_object_section("log");
+
+      struct rgw_log_entry entry;
+      
+      // peek at first entry to get bucket metadata
+      r = store->log_show_next(h, &entry);
+      if (r < 0) {
+	cerr << "error reading log " << oid << ": " << cpp_strerror(-r) << std::endl;
+	return -r;
+      }
+      formatter->dump_string("bucket_id", entry.bucket_id);
+      formatter->dump_string("bucket_owner", entry.bucket_owner);
+      formatter->dump_string("bucket", entry.bucket);
+
+      uint64_t agg_time = 0;
+      uint64_t agg_bytes_sent = 0;
+      uint64_t agg_bytes_received = 0;
+      uint64_t total_entries = 0;
+
+      if (show_log_entries)
+        formatter->open_array_section("log_entries");
+
+      do {
+	uint64_t total_time =  entry.total_time.sec() * 1000000LL * entry.total_time.usec();
+
+        agg_time += total_time;
+        agg_bytes_sent += entry.bytes_sent;
+        agg_bytes_received += entry.bytes_received;
+        total_entries++;
+
+        if (skip_zero_entries && entry.bytes_sent == 0 &&
+            entry.bytes_received == 0)
+          goto next;
+
+        if (show_log_entries) {
+
+	  rgw_format_ops_log_entry(entry, formatter);
+	  formatter->flush(cout);
+        }
+next:
+	r = store->log_show_next(h, &entry);
+      } while (r > 0);
+
+      if (r < 0) {
+      	cerr << "error reading log " << oid << ": " << cpp_strerror(-r) << std::endl;
+	return -r;
+      }
+      if (show_log_entries)
+        formatter->close_section();
+
+      if (show_log_sum) {
+        formatter->open_object_section("log_sum");
+	formatter->dump_int("bytes_sent", agg_bytes_sent);
+	formatter->dump_int("bytes_received", agg_bytes_received);
+	formatter->dump_int("total_time", agg_time);
+	formatter->dump_int("total_entries", total_entries);
+        formatter->close_section();
+      }
+      formatter->close_section();
+      formatter->flush(cout);
+      cout << std::endl;
+    }
+    if (opt_cmd == OPT_LOG_RM) {
+      int r = store->log_remove(oid);
+      if (r < 0) {
+	cerr << "error removing log " << oid << ": " << cpp_strerror(-r) << std::endl;
+	return -r;
+      }
+    }
+  }
+  
+  if (opt_cmd == OPT_POOL_ADD) {
+    if (pool_name.empty()) {
+      cerr << "need to specify pool to add!" << std::endl;
+      return usage();
+    }
+
+    int ret = store->add_bucket_placement(pool_name);
+    if (ret < 0)
+      cerr << "failed to add bucket placement: " << cpp_strerror(-ret) << std::endl;
+  }
+
+  if (opt_cmd == OPT_POOL_RM) {
+    if (pool_name.empty()) {
+      cerr << "need to specify pool to remove!" << std::endl;
+      return usage();
+    }
+
+    int ret = store->remove_bucket_placement(pool_name);
+    if (ret < 0)
+      cerr << "failed to remove bucket placement: " << cpp_strerror(-ret) << std::endl;
+  }
+
+  if (opt_cmd == OPT_POOLS_LIST) {
+    set pools;
+    int ret = store->list_placement_set(pools);
+    if (ret < 0) {
+      cerr << "could not list placement set: " << cpp_strerror(-ret) << std::endl;
+      return ret;
+    }
+    formatter->reset();
+    formatter->open_array_section("pools");
+    set::iterator siter;
+    for (siter = pools.begin(); siter != pools.end(); ++siter) {
+      formatter->open_object_section("pool");
+      formatter->dump_string("name",  *siter);
+      formatter->close_section();
+    }
+    formatter->close_section();
+    formatter->flush(cout);
+    cout << std::endl;
+  }
+
+  if (opt_cmd == OPT_USAGE_SHOW) {
+    uint64_t start_epoch = 0;
+    uint64_t end_epoch = (uint64_t)-1;
+
+    int ret;
+    
+    if (!start_date.empty()) {
+      ret = utime_t::parse_date(start_date, &start_epoch, NULL);
+      if (ret < 0) {
+        cerr << "ERROR: failed to parse start date" << std::endl;
+        return 1;
+      }
+    }
+    if (!end_date.empty()) {
+      ret = utime_t::parse_date(end_date, &end_epoch, NULL);
+      if (ret < 0) {
+        cerr << "ERROR: failed to parse end date" << std::endl;
+        return 1;
+      }
+    }
+
+
+    ret = RGWUsage::show(store, user_id, start_epoch, end_epoch,
+			 show_log_entries, show_log_sum, &categories,
+			 f);
+    if (ret < 0) {
+      cerr << "ERROR: failed to show usage" << std::endl;
+      return 1;
+    }
+  }
+
+  if (opt_cmd == OPT_USAGE_TRIM) {
+    if (user_id.empty() && !yes_i_really_mean_it) {
+      cerr << "usage trim without user specified will remove *all* users data" << std::endl;
+      cerr << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl;
+      return 1;
+    }
+    int ret;
+    uint64_t start_epoch = 0;
+    uint64_t end_epoch = (uint64_t)-1;
+
+
+    if (!start_date.empty()) {
+      ret = utime_t::parse_date(start_date, &start_epoch, NULL);
+      if (ret < 0) {
+        cerr << "ERROR: failed to parse start date" << std::endl;
+        return 1;
+      }
+    }
+
+    if (!end_date.empty()) {
+      ret = utime_t::parse_date(end_date, &end_epoch, NULL);
+      if (ret < 0) {
+        cerr << "ERROR: failed to parse end date" << std::endl;
+        return 1;
+      }
+    }
+
+    ret = RGWUsage::trim(store, user_id, start_epoch, end_epoch);
+    if (ret < 0) {
+      cerr << "ERROR: read_usage() returned ret=" << ret << std::endl;
+      return 1;
+    }   
+  }
+
+  if (opt_cmd == OPT_OBJECT_RM) {
+    RGWBucketInfo bucket_info;
+    int ret = init_bucket(bucket_name, bucket_info, bucket);
+    if (ret < 0) {
+      cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+    ret = rgw_remove_object(store, bucket_info.owner, bucket, object);
+
+    if (ret < 0) {
+      cerr << "ERROR: object remove returned: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+
+  if (opt_cmd == OPT_OBJECT_UNLINK) {
+    RGWBucketInfo bucket_info;
+    int ret = init_bucket(bucket_name, bucket_info, bucket);
+    if (ret < 0) {
+      cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+    list oid_list;
+    oid_list.push_back(object);
+    ret = store->remove_objs_from_index(bucket, oid_list);
+    if (ret < 0) {
+      cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
+      return 1;
+    }
+  }
+
+  if (opt_cmd == OPT_OBJECT_STAT) {
+    RGWBucketInfo bucket_info;
+    int ret = init_bucket(bucket_name, bucket_info, bucket);
+    if (ret < 0) {
+      cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+    rgw_obj obj(bucket, object);
+
+    void *handle;
+    uint64_t obj_size;
+    map attrs;
+    void *obj_ctx = store->create_context(NULL);
+    ret = store->prepare_get_obj(obj_ctx, obj, NULL, NULL, &attrs, NULL,
+                                 NULL, NULL, NULL, NULL, NULL, &obj_size, NULL, &handle, NULL);
+    store->finish_get_obj(&handle);
+    store->destroy_context(obj_ctx);
+    if (ret < 0) {
+      cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl;
+      return 1;
+    }
+    formatter->open_object_section("object_metadata");
+    formatter->dump_string("name", object);
+    formatter->dump_unsigned("size", obj_size);
+
+    map::iterator iter;
+    map other_attrs;
+    for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+      bufferlist& bl = iter->second;
+      bool handled = false;
+      if (iter->first == RGW_ATTR_MANIFEST) {
+        handled = decode_dump("manifest", bl, formatter);
+      } else if (iter->first == RGW_ATTR_ACL) {
+        handled = decode_dump("policy", bl, formatter);
+      } else if (iter->first == RGW_ATTR_ID_TAG) {
+        handled = dump_string("tag", bl, formatter);
+      } else if (iter->first == RGW_ATTR_ETAG) {
+        handled = dump_string("etag", bl, formatter);
+      }
+
+      if (!handled)
+        other_attrs[iter->first] = bl;
+    }
+
+    formatter->open_object_section("attrs");
+    for (iter = other_attrs.begin(); iter != other_attrs.end(); ++iter) {
+      dump_string(iter->first.c_str(), iter->second, formatter);
+    }
+    formatter->close_section();
+    formatter->close_section();
+    formatter->flush(cout);
+  }
+
+  if (opt_cmd == OPT_BUCKET_CHECK) {
+    RGWBucketAdminOp::check_index(store, bucket_op, f);
+  }
+
+  if (opt_cmd == OPT_BUCKET_RM) {
+    RGWBucketAdminOp::remove_bucket(store, bucket_op);
+  }
+
+  if (opt_cmd == OPT_GC_LIST) {
+    int index = 0;
+    bool truncated;
+    formatter->open_array_section("entries");
+
+    do {
+      list result;
+      int ret = store->list_gc_objs(&index, marker, 1000, !include_all, result, &truncated);
+      if (ret < 0) {
+	cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl;
+	return 1;
+      }
+
+
+      list::iterator iter;
+      for (iter = result.begin(); iter != result.end(); ++iter) {
+	cls_rgw_gc_obj_info& info = *iter;
+	formatter->open_object_section("chain_info");
+	formatter->dump_string("tag", info.tag);
+	formatter->dump_stream("time") << info.time;
+	formatter->open_array_section("objs");
+        list::iterator liter;
+	cls_rgw_obj_chain& chain = info.chain;
+	for (liter = chain.objs.begin(); liter != chain.objs.end(); ++liter) {
+	  cls_rgw_obj& obj = *liter;
+          encode_json("obj", obj, formatter);
+	}
+	formatter->close_section(); // objs
+	formatter->close_section(); // obj_chain
+	formatter->flush(cout);
+      }
+    } while (truncated);
+    formatter->close_section();
+    formatter->flush(cout);
+  }
+
+  if (opt_cmd == OPT_GC_PROCESS) {
+    int ret = store->process_gc();
+    if (ret < 0) {
+      cerr << "ERROR: gc processing returned error: " << cpp_strerror(-ret) << std::endl;
+      return 1;
+    }
+  }
+
+  if (opt_cmd == OPT_USER_CHECK) {
+    check_bad_user_bucket_mapping(store, user_id, fix);
+  }
+
+  if (opt_cmd == OPT_USER_STATS) {
+    if (sync_stats) {
+      if (!bucket_name.empty()) {
+        int ret = rgw_bucket_sync_user_stats(store, bucket_name);
+        if (ret < 0) {
+          cerr << "ERROR: could not sync bucket stats: " << cpp_strerror(-ret) << std::endl;
+          return -ret;
+        }
+      } else {
+        int ret = rgw_user_sync_all_stats(store, user_id);
+        if (ret < 0) {
+          cerr << "ERROR: failed to sync user stats: " << cpp_strerror(-ret) << std::endl;
+          return -ret;
+        }
+      }
+    }
+
+    if (user_id.empty()) {
+      cerr << "ERROR: uid not specified" << std::endl;
+      return EINVAL;
+    }
+    cls_user_header header;
+    int ret = store->cls_user_get_header(user_id, &header);
+    if (ret < 0) {
+      cerr << "ERROR: can't read user header: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+
+    encode_json("header", header, formatter);
+    formatter->flush(cout);
+  }
+
+  if (opt_cmd == OPT_METADATA_GET) {
+    int ret = store->meta_mgr->get(metadata_key, formatter);
+    if (ret < 0) {
+      cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+
+    formatter->flush(cout);
+  }
+
+  if (opt_cmd == OPT_METADATA_PUT) {
+    bufferlist bl;
+    int ret = read_input(infile, bl);
+    if (ret < 0) {
+      cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
+      return ret;
+    }
+    ret = store->meta_mgr->put(metadata_key, bl, RGWMetadataHandler::APPLY_ALWAYS);
+    if (ret < 0) {
+      cerr << "ERROR: can't put key: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+
+  if (opt_cmd == OPT_METADATA_RM) {
+    int ret = store->meta_mgr->remove(metadata_key);
+    if (ret < 0) {
+      cerr << "ERROR: can't remove key: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+
+  if (opt_cmd == OPT_METADATA_LIST) {
+    void *handle;
+    int max = 1000;
+    int ret = store->meta_mgr->list_keys_init(metadata_key, &handle);
+    if (ret < 0) {
+      cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+
+    bool truncated;
+
+    formatter->open_array_section("keys");
+
+    do {
+      list keys;
+      ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated);
+      if (ret < 0) {
+        cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+
+      for (list::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
+	formatter->dump_string("key", *iter);
+      }
+      formatter->flush(cout);
+
+    } while (truncated);
+
+    formatter->close_section();
+    formatter->flush(cout);
+
+    store->meta_mgr->list_keys_complete(handle);
+  }
+
+  if (opt_cmd == OPT_MDLOG_LIST) {
+    utime_t start_time, end_time;
+
+    int ret = parse_date_str(start_date, start_time);
+    if (ret < 0)
+      return -ret;
+
+    ret = parse_date_str(end_date, end_time);
+    if (ret < 0)
+      return -ret;
+
+    int i = (specified_shard_id ? shard_id : 0);
+
+    formatter->open_array_section("entries");
+    for (; i < g_ceph_context->_conf->rgw_md_log_max_shards; i++) {
+      RGWMetadataLog *meta_log = store->meta_mgr->get_log();
+      void *handle;
+      list entries;
+
+
+      meta_log->init_list_entries(i, start_time, end_time, marker, &handle);
+
+      bool truncated;
+      do {
+	  int ret = meta_log->list_entries(handle, 1000, entries, NULL, &truncated);
+        if (ret < 0) {
+          cerr << "ERROR: meta_log->list_entries(): " << cpp_strerror(-ret) << std::endl;
+          return -ret;
+        }
+
+        for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
+          cls_log_entry& entry = *iter;
+          store->meta_mgr->dump_log_entry(entry, formatter);
+        }
+        formatter->flush(cout);
+      } while (truncated);
+
+      meta_log->complete_list_entries(handle);
+
+      if (specified_shard_id)
+        break;
+    }
+  
+
+    formatter->close_section();
+    formatter->flush(cout);
+  }
+
+  if (opt_cmd == OPT_MDLOG_TRIM) {
+    utime_t start_time, end_time;
+
+    if (!specified_shard_id) {
+      cerr << "ERROR: shard-id must be specified for trim operation" << std::endl;
+      return EINVAL;
+    }
+
+    int ret = parse_date_str(start_date, start_time);
+    if (ret < 0)
+      return -ret;
+
+    ret = parse_date_str(end_date, end_time);
+    if (ret < 0)
+      return -ret;
+
+    RGWMetadataLog *meta_log = store->meta_mgr->get_log();
+
+    ret = meta_log->trim(shard_id, start_time, end_time, start_marker, end_marker);
+    if (ret < 0) {
+      cerr << "ERROR: meta_log->trim(): " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+  
+  if (opt_cmd == OPT_BILOG_LIST) {
+    if (bucket_name.empty()) {
+      cerr << "ERROR: bucket not specified" << std::endl;
+      return -EINVAL;
+    }
+    RGWBucketInfo bucket_info;
+    int ret = init_bucket(bucket_name, bucket_info, bucket);
+    if (ret < 0) {
+      cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+    formatter->open_array_section("entries");
+    bool truncated;
+    int count = 0;
+    if (max_entries < 0)
+      max_entries = 1000;
+
+    do {
+      list entries;
+      ret = store->list_bi_log_entries(bucket, marker, max_entries - count, entries, &truncated);
+      if (ret < 0) {
+        cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+
+      count += entries.size();
+
+      for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
+        rgw_bi_log_entry& entry = *iter;
+        encode_json("entry", entry, formatter);
+
+        marker = entry.id;
+      }
+      formatter->flush(cout);
+    } while (truncated && count < max_entries);
+
+    formatter->close_section();
+    formatter->flush(cout);
+  }
+
+  if (opt_cmd == OPT_BILOG_TRIM) {
+    if (bucket_name.empty()) {
+      cerr << "ERROR: bucket not specified" << std::endl;
+      return -EINVAL;
+    }
+    RGWBucketInfo bucket_info;
+    int ret = init_bucket(bucket_name, bucket_info, bucket);
+    if (ret < 0) {
+      cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+    ret = store->trim_bi_log_entries(bucket, start_marker, end_marker);
+    if (ret < 0) {
+      cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+
+  if (opt_cmd == OPT_DATALOG_LIST) {
+    formatter->open_array_section("entries");
+    bool truncated;
+    int count = 0;
+    if (max_entries < 0)
+      max_entries = 1000;
+
+    utime_t start_time, end_time;
+
+    int ret = parse_date_str(start_date, start_time);
+    if (ret < 0)
+      return -ret;
+
+    ret = parse_date_str(end_date, end_time);
+    if (ret < 0)
+      return -ret;
+
+    RGWDataChangesLog *log = store->data_log;
+    RGWDataChangesLog::LogMarker marker;
+
+    do {
+      list entries;
+      ret = log->list_entries(start_time, end_time, max_entries - count, entries, marker, &truncated);
+      if (ret < 0) {
+        cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+
+      count += entries.size();
+
+      for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
+        rgw_data_change& entry = *iter;
+        encode_json("entry", entry, formatter);
+      }
+      formatter->flush(cout);
+    } while (truncated && count < max_entries);
+
+    formatter->close_section();
+    formatter->flush(cout);
+  }
+  
+  if (opt_cmd == OPT_DATALOG_TRIM) {
+    utime_t start_time, end_time;
+
+    int ret = parse_date_str(start_date, start_time);
+    if (ret < 0)
+      return -ret;
+
+    ret = parse_date_str(end_date, end_time);
+    if (ret < 0)
+      return -ret;
+
+    RGWDataChangesLog *log = store->data_log;
+    ret = log->trim_entries(start_time, end_time, start_marker, end_marker);
+    if (ret < 0) {
+      cerr << "ERROR: trim_entries(): " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+
+  if (opt_cmd == OPT_OPSTATE_LIST) {
+    RGWOpState oc(store);
+
+    int max = 1000;
+
+    void *handle;
+    oc.init_list_entries(client_id, op_id, object, &handle);
+    list entries;
+    bool done;
+    formatter->open_array_section("entries");
+    do {
+      int ret = oc.list_entries(handle, max, entries, &done);
+      if (ret < 0) {
+        cerr << "oc.list_entries returned " << cpp_strerror(-ret) << std::endl;
+        oc.finish_list_entries(handle);
+        return -ret;
+      }
+
+      for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
+        oc.dump_entry(*iter, formatter);
+      }
+
+      formatter->flush(cout);
+    } while (!done);
+    formatter->close_section();
+    formatter->flush(cout);
+    oc.finish_list_entries(handle);
+  }
+
+  if (opt_cmd == OPT_OPSTATE_SET || opt_cmd == OPT_OPSTATE_RENEW) {
+    RGWOpState oc(store);
+
+    RGWOpState::OpState state;
+    if (object.empty() || client_id.empty() || op_id.empty()) {
+      cerr << "ERROR: need to specify client_id, op_id, and object" << std::endl;
+      return EINVAL;
+    }
+    if (state_str.empty()) {
+      cerr << "ERROR: state was not specified" << std::endl;
+      return EINVAL;
+    }
+    int ret = oc.state_from_str(state_str, &state);
+    if (ret < 0) {
+      cerr << "ERROR: invalid state: " << state_str << std::endl;
+      return -ret;
+    }
+
+    if (opt_cmd == OPT_OPSTATE_SET) {
+      ret = oc.set_state(client_id, op_id, object, state);
+      if (ret < 0) {
+        cerr << "ERROR: failed to set state: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+    } else {
+      ret = oc.renew_state(client_id, op_id, object, state);
+      if (ret < 0) {
+        cerr << "ERROR: failed to renew state: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+    }
+  }
+  if (opt_cmd == OPT_OPSTATE_RM) {
+    RGWOpState oc(store);
+
+    if (object.empty() || client_id.empty() || op_id.empty()) {
+      cerr << "ERROR: need to specify client_id, op_id, and object" << std::endl;
+      return EINVAL;
+    }
+    ret = oc.remove_entry(client_id, op_id, object);
+    if (ret < 0) {
+      cerr << "ERROR: failed to set state: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+  }
+
+  if (opt_cmd == OPT_REPLICALOG_GET || opt_cmd == OPT_REPLICALOG_DELETE) {
+    if (replica_log_type_str.empty()) {
+      cerr << "ERROR: need to specify --replica-log-type=" << std::endl;
+      return EINVAL;
+    }
+  }
+
+  if (opt_cmd == OPT_REPLICALOG_GET) {
+    RGWReplicaBounds bounds;
+    if (replica_log_type == ReplicaLog_Metadata) {
+      if (!specified_shard_id) {
+        cerr << "ERROR: shard-id must be specified for get operation" << std::endl;
+        return EINVAL;
+      }
+
+      RGWReplicaObjectLogger logger(store, pool_name, META_REPLICA_LOG_OBJ_PREFIX);
+      int ret = logger.get_bounds(shard_id, bounds);
+      if (ret < 0)
+        return -ret;
+    } else if (replica_log_type == ReplicaLog_Data) {
+      if (!specified_shard_id) {
+        cerr << "ERROR: shard-id must be specified for get operation" << std::endl;
+        return EINVAL;
+      }
+      RGWReplicaObjectLogger logger(store, pool_name, DATA_REPLICA_LOG_OBJ_PREFIX);
+      int ret = logger.get_bounds(shard_id, bounds);
+      if (ret < 0)
+        return -ret;
+    } else if (replica_log_type == ReplicaLog_Bucket) {
+      if (bucket_name.empty()) {
+        cerr << "ERROR: bucket not specified" << std::endl;
+        return -EINVAL;
+      }
+      RGWBucketInfo bucket_info;
+      int ret = init_bucket(bucket_name, bucket_info, bucket);
+      if (ret < 0) {
+        cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+
+      RGWReplicaBucketLogger logger(store);
+      ret = logger.get_bounds(bucket, bounds);
+      if (ret < 0)
+        return -ret;
+    } else { // shouldn't get here
+      assert(0);
+    }
+    encode_json("bounds", bounds, formatter);
+    formatter->flush(cout);
+    cout << std::endl;
+  }
+
+  if (opt_cmd == OPT_REPLICALOG_DELETE) {
+    if (replica_log_type == ReplicaLog_Metadata) {
+      if (!specified_shard_id) {
+        cerr << "ERROR: shard-id must be specified for delete operation" << std::endl;
+        return EINVAL;
+      }
+      if (!specified_daemon_id) {
+        cerr << "ERROR: daemon-id must be specified for delete operation" << std::endl;
+        return EINVAL;
+      }
+      RGWReplicaObjectLogger logger(store, pool_name, META_REPLICA_LOG_OBJ_PREFIX);
+      int ret = logger.delete_bound(shard_id, daemon_id);
+      if (ret < 0)
+        return -ret;
+    } else if (replica_log_type == ReplicaLog_Data) {
+      if (!specified_shard_id) {
+        cerr << "ERROR: shard-id must be specified for delete operation" << std::endl;
+        return EINVAL;
+      }
+      if (!specified_daemon_id) {
+        cerr << "ERROR: daemon-id must be specified for delete operation" << std::endl;
+        return EINVAL;
+      }
+      RGWReplicaObjectLogger logger(store, pool_name, DATA_REPLICA_LOG_OBJ_PREFIX);
+      int ret = logger.delete_bound(shard_id, daemon_id);
+      if (ret < 0)
+        return -ret;
+    } else if (replica_log_type == ReplicaLog_Bucket) {
+      if (bucket_name.empty()) {
+        cerr << "ERROR: bucket not specified" << std::endl;
+        return -EINVAL;
+      }
+      RGWBucketInfo bucket_info;
+      int ret = init_bucket(bucket_name, bucket_info, bucket);
+      if (ret < 0) {
+        cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+
+      RGWReplicaBucketLogger logger(store);
+      ret = logger.delete_bound(bucket, daemon_id);
+      if (ret < 0)
+        return -ret;
+    }
+  }
+
+  bool quota_op = (opt_cmd == OPT_QUOTA_SET || opt_cmd == OPT_QUOTA_ENABLE || opt_cmd == OPT_QUOTA_DISABLE);
+
+  if (quota_op) {
+    if (bucket_name.empty() && user_id.empty()) {
+      cerr << "ERROR: bucket name or uid is required for quota operation" << std::endl;
+      return EINVAL;
+    }
+
+    if (!bucket_name.empty()) {
+      if (!quota_scope.empty() && quota_scope != "bucket") {
+        cerr << "ERROR: invalid quota scope specification." << std::endl;
+        return EINVAL;
+      }
+      set_bucket_quota(store, opt_cmd, bucket_name, max_size, max_objects, have_max_size, have_max_objects);
+    } else if (!user_id.empty()) {
+      if (quota_scope == "bucket") {
+        set_user_bucket_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
+      } else if (quota_scope == "user") {
+        set_user_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
+      } else {
+        cerr << "ERROR: invalid quota scope specification. Please specify either --quota-scope=bucket, or --quota-scope=user" << std::endl;
+        return EINVAL;
+      }
+    }
+  }
+
+  return 0;
+}
diff --git a/ceph/src/rgw/rgw_auth_s3.cc b/ceph/src/rgw/rgw_auth_s3.cc
new file mode 100644
index 00000000..b8246b78
--- /dev/null
+++ b/ceph/src/rgw/rgw_auth_s3.cc
@@ -0,0 +1,211 @@
+
+#include "common/armor.h"
+#include "rgw_common.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+static const char *signed_subresources[] = {
+  "acl",
+  "cors",
+  "delete",
+  "lifecycle",
+  "location",
+  "logging",
+  "notification",
+  "partNumber",
+  "policy",
+  "requestPayment",
+  "response-cache-control",
+  "response-content-disposition",
+  "response-content-encoding",
+  "response-content-language",
+  "response-content-type",
+  "response-expires",
+  "torrent",
+  "uploadId",
+  "uploads",
+  "versionId",
+  "versioning",
+  "versions",
+  "website",
+  NULL
+};
+
+/*
+ * ?get the canonical amazon-style header for something?
+ */
+
+static void get_canon_amz_hdr(map& meta_map, string& dest)
+{
+  dest = "";
+  map::iterator iter;
+  for (iter = meta_map.begin(); iter != meta_map.end(); ++iter) {
+    dest.append(iter->first);
+    dest.append(":");
+    dest.append(iter->second);
+    dest.append("\n");
+  }
+}
+
+/*
+ * ?get the canonical representation of the object's location
+ */
+static void get_canon_resource(const char *request_uri, map& sub_resources, string& dest)
+{
+  string s;
+
+  if (request_uri)
+    s.append(request_uri);
+
+  string append_str;
+
+  const char **p = signed_subresources;
+
+  for (; *p; ++p) {
+    map::iterator iter = sub_resources.find(*p);
+    if (iter == sub_resources.end())
+      continue;
+    
+    if (append_str.empty())
+      append_str.append("?");
+    else
+      append_str.append("&");     
+    append_str.append(iter->first);
+    if (!iter->second.empty()) {
+      append_str.append("=");
+      append_str.append(iter->second);
+    }
+  }
+  if (!append_str.empty()) {
+    s.append(append_str);
+  }
+  dest = s;
+
+  dout(10) << "get_canon_resource(): dest=" << dest << dendl;
+}
+
+/*
+ * get the header authentication  information required to
+ * compute a request's signature
+ */
+void rgw_create_s3_canonical_header(const char *method, const char *content_md5, const char *content_type, const char *date,
+                            map& meta_map, const char *request_uri, map& sub_resources,
+                            string& dest_str)
+{
+  string dest;
+
+  if (method)
+    dest = method;
+  dest.append("\n");
+  
+  if (content_md5) {
+    dest.append(content_md5);
+  }
+  dest.append("\n");
+
+  if (content_type)
+    dest.append(content_type);
+  dest.append("\n");
+
+  if (date)
+    dest.append(date);
+  dest.append("\n");
+
+  string canon_amz_hdr;
+  get_canon_amz_hdr(meta_map, canon_amz_hdr);
+  dest.append(canon_amz_hdr);
+
+  string canon_resource;
+  get_canon_resource(request_uri, sub_resources, canon_resource);
+
+  dest.append(canon_resource);
+
+  dest_str = dest;
+}
+
+int rgw_get_s3_header_digest(const string& auth_hdr, const string& key, string& dest)
+{
+  char hmac_sha1[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+  calc_hmac_sha1(key.c_str(), key.size(), auth_hdr.c_str(), auth_hdr.size(), hmac_sha1);
+
+  char b64[64]; /* 64 is really enough */
+  int ret = ceph_armor(b64, b64 + 64, hmac_sha1,
+		       hmac_sha1 + CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+  if (ret < 0) {
+    dout(10) << "ceph_armor failed" << dendl;
+    return ret;
+  }
+  b64[ret] = '\0';
+
+  dest = b64;
+
+  return 0;
+}
+
+static inline bool is_base64_for_content_md5(unsigned char c) {
+  return (isalnum(c) || isspace(c) || (c == '+') || (c == '/') || (c == '='));
+}
+
+/*
+ * get the header authentication  information required to
+ * compute a request's signature
+ */
+bool rgw_create_s3_canonical_header(req_info& info, utime_t *header_time, string& dest, bool qsr)
+{
+  const char *content_md5 = info.env->get("HTTP_CONTENT_MD5");
+  if (content_md5) {
+    for (const char *p = content_md5; *p; p++) {
+      if (!is_base64_for_content_md5(*p)) {
+        dout(0) << "NOTICE: bad content-md5 provided (not base64), aborting request p=" << *p << " " << (int)*p << dendl;
+        return false;
+      }
+    }
+  }
+
+  const char *content_type = info.env->get("CONTENT_TYPE");
+
+  string date;
+  if (qsr) {
+    date = info.args.get("Expires");
+  } else {
+    const char *str = info.env->get("HTTP_DATE");
+    const char *req_date = str;
+    if (str) {
+      date = str;
+    } else {
+      req_date = info.env->get("HTTP_X_AMZ_DATE");
+      if (!req_date) {
+        dout(0) << "NOTICE: missing date for auth header" << dendl;
+        return false;
+      }
+    }
+
+    if (header_time) {
+      struct tm t;
+      if (!parse_rfc2616(req_date, &t)) {
+        dout(0) << "NOTICE: failed to parse date for auth header" << dendl;
+        return false;
+      }
+      if (t.tm_year < 70) {
+        dout(0) << "NOTICE: bad date (predates epoch): " << req_date << dendl;
+        return false;
+      }
+      *header_time = utime_t(timegm(&t), 0);
+    }
+  }
+
+  map& meta_map = info.x_meta_map;
+  map& sub_resources = info.args.get_sub_resources();
+
+  string request_uri;
+  if (info.effective_uri.empty())
+    request_uri = info.request_uri;
+  else
+    request_uri = info.effective_uri;
+
+  rgw_create_s3_canonical_header(info.method, content_md5, content_type, date.c_str(),
+                            meta_map, request_uri.c_str(), sub_resources,
+                            dest);
+
+  return true;
+}
diff --git a/ceph/src/rgw/rgw_auth_s3.h b/ceph/src/rgw/rgw_auth_s3.h
new file mode 100644
index 00000000..33aa2841
--- /dev/null
+++ b/ceph/src/rgw/rgw_auth_s3.h
@@ -0,0 +1,15 @@
+#ifndef CEPH_RGW_AUTH_S3_H
+#define CEPH_RGW_AUTH_S3_H
+
+
+#include "rgw_common.h"
+
+void rgw_create_s3_canonical_header(const char *method, const char *content_md5, const char *content_type, const char *date,
+                            map& meta_map, const char *request_uri, map& sub_resources,
+                            string& dest_str);
+bool rgw_create_s3_canonical_header(req_info& info, utime_t *header_time, string& dest, bool qsr);
+int rgw_get_s3_header_digest(const string& auth_hdr, const string& key, string& dest);
+
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_bucket.cc b/ceph/src/rgw/rgw_bucket.cc
new file mode 100644
index 00000000..3c4f66c9
--- /dev/null
+++ b/ceph/src/rgw/rgw_bucket.cc
@@ -0,0 +1,1778 @@
+#include 
+
+#include 
+#include 
+
+#include "common/errno.h"
+#include "common/ceph_json.h"
+#include "rgw_rados.h"
+#include "rgw_acl.h"
+#include "rgw_acl_s3.h"
+
+#include "include/types.h"
+#include "rgw_bucket.h"
+#include "rgw_user.h"
+#include "rgw_string.h"
+
+// until everything is moved from rgw_common
+#include "rgw_common.h"
+
+#include "cls/user/cls_user_types.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+#define BUCKET_TAG_TIMEOUT 30
+
+using namespace std;
+
+static RGWMetadataHandler *bucket_meta_handler = NULL;
+static RGWMetadataHandler *bucket_instance_meta_handler = NULL;
+
+// define as static when RGWBucket implementation compete
+void rgw_get_buckets_obj(const string& user_id, string& buckets_obj_id)
+{
+  buckets_obj_id = user_id;
+  buckets_obj_id += RGW_BUCKETS_OBJ_SUFFIX;
+}
+
+/**
+ * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
+ * Returns: 0 on success, -ERR# on failure.
+ */
+int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
+                          const string& marker, uint64_t max, bool need_stats)
+{
+  int ret;
+  buckets.clear();
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  bufferlist bl;
+  rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+  bufferlist header;
+  list entries;
+
+  bool truncated = false;
+  string m = marker;
+
+  uint64_t total = 0;
+
+  do {
+    ret = store->cls_user_list_buckets(obj, m, max - total, entries, &m, &truncated);
+    if (ret == -ENOENT)
+      ret = 0;
+
+    if (ret < 0)
+      return ret;
+
+    for (list::iterator q = entries.begin(); q != entries.end(); ++q) {
+      RGWBucketEnt e(*q);
+      buckets.add(e);
+      total++;
+    }
+
+  } while (truncated && total < max);
+
+  if (need_stats) {
+    map& m = buckets.get_buckets();
+    int r = store->update_containers_stats(m);
+    if (r < 0) {
+      ldout(store->ctx(), 0) << "ERROR: could not get stats for buckets" << dendl;
+    }
+  }
+  return 0;
+}
+
+int rgw_bucket_sync_user_stats(RGWRados *store, const string& user_id, rgw_bucket& bucket)
+{
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+
+  return store->cls_user_sync_bucket_stats(obj, bucket);
+}
+
+int rgw_bucket_sync_user_stats(RGWRados *store, const string& bucket_name)
+{
+  RGWBucketInfo bucket_info;
+  int ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: could not fetch bucket info: ret=" << ret << dendl;
+    return ret;
+  }
+
+  ret = rgw_bucket_sync_user_stats(store, bucket_info.owner, bucket_info.bucket);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: could not sync user stats for bucket " << bucket_name << ": ret=" << ret << dendl;
+    return ret;
+  }
+
+  return 0;
+}
+
+int rgw_link_bucket(RGWRados *store, string user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint)
+{
+  int ret;
+  string& bucket_name = bucket.name;
+
+  cls_user_bucket_entry new_bucket;
+
+  RGWBucketEntryPoint ep;
+  RGWObjVersionTracker ot;
+
+  bucket.convert(&new_bucket.bucket);
+  new_bucket.size = 0;
+  if (!creation_time)
+    time(&new_bucket.creation_time);
+  else
+    new_bucket.creation_time = creation_time;
+
+  map attrs;
+
+  if (update_entrypoint) {
+    ret = store->get_bucket_entrypoint_info(NULL, bucket_name, ep, &ot, NULL, &attrs);
+    if (ret < 0 && ret != -ENOENT) {
+      ldout(store->ctx(), 0) << "ERROR: store->get_bucket_entrypoint_info() returned " << ret << dendl;
+    } else if (ret >= 0 && ep.linked && ep.owner != user_id) {
+      ldout(store->ctx(), 0) << "can't link bucket, already linked to a different user: " << ep.owner << dendl;
+      return -EINVAL;
+    }
+  }
+
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+
+  rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+  ret = store->cls_user_add_bucket(obj, new_bucket);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: error adding bucket to directory: "
+        << cpp_strerror(-ret)<< dendl;
+    goto done_err;
+  }
+
+  if (!update_entrypoint)
+    return 0;
+
+  ep.linked = true;
+  ep.owner = user_id;
+  ret = store->put_bucket_entrypoint_info(bucket_name, ep, false, ot, 0, &attrs);
+  if (ret < 0)
+    goto done_err;
+
+  return 0;
+done_err:
+  int r = rgw_unlink_bucket(store, user_id, bucket.name);
+  if (r < 0) {
+    ldout(store->ctx(), 0) << "ERROR: failed unlinking bucket on error cleanup: " << cpp_strerror(-r) << dendl;
+  }
+  return ret;
+}
+
+int rgw_unlink_bucket(RGWRados *store, string user_id, const string& bucket_name, bool update_entrypoint)
+{
+  int ret;
+
+  bufferlist bl;
+
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+
+  cls_user_bucket bucket;
+  bucket.name = bucket_name;
+  rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+  ret = store->cls_user_remove_bucket(obj, bucket);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: error removing bucket from directory: "
+        << cpp_strerror(-ret)<< dendl;
+  }
+
+  if (!update_entrypoint)
+    return 0;
+
+  RGWBucketEntryPoint ep;
+  RGWObjVersionTracker ot;
+  map attrs;
+  ret = store->get_bucket_entrypoint_info(NULL, bucket_name, ep, &ot, NULL, &attrs);
+  if (ret == -ENOENT)
+    return 0;
+  if (ret < 0)
+    return ret;
+
+  if (!ep.linked)
+    return 0;
+
+  if (ep.owner != user_id) {
+    ldout(store->ctx(), 0) << "bucket entry point user mismatch, can't unlink bucket: " << ep.owner << " != " << user_id << dendl;
+    return -EINVAL;
+  }
+
+  ep.linked = false;
+  ret = store->put_bucket_entrypoint_info(bucket_name, ep, false, ot, 0, &attrs);
+  if (ret < 0)
+    return ret;
+
+  return ret;
+}
+
+int rgw_bucket_store_info(RGWRados *store, const string& bucket_name, bufferlist& bl, bool exclusive,
+                          map *pattrs, RGWObjVersionTracker *objv_tracker,
+                          time_t mtime) {
+  return store->meta_mgr->put_entry(bucket_meta_handler, bucket_name, bl, exclusive, objv_tracker, mtime, pattrs);
+}
+
+int rgw_bucket_instance_store_info(RGWRados *store, string& entry, bufferlist& bl, bool exclusive,
+                          map *pattrs, RGWObjVersionTracker *objv_tracker,
+                          time_t mtime) {
+  return store->meta_mgr->put_entry(bucket_instance_meta_handler, entry, bl, exclusive, objv_tracker, mtime, pattrs);
+}
+
+int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersionTracker *objv_tracker) {
+  return store->meta_mgr->remove_entry(bucket_instance_meta_handler, entry, objv_tracker);
+}
+
+int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info,
+                         map& attrs,
+                         map* rmattrs,
+                         RGWObjVersionTracker *objv_tracker)
+{
+  rgw_bucket& bucket = bucket_info.bucket;
+
+  if (!bucket_info.has_instance_obj) {
+    /* an old bucket object, need to convert it */
+    int ret = store->convert_old_bucket_info(NULL, bucket.name);
+    if (ret < 0) {
+      ldout(store->ctx(), 0) << "ERROR: failed converting old bucket info: " << ret << dendl;
+      return ret;
+    }
+  }
+  string oid;
+  store->get_bucket_meta_oid(bucket, oid);
+  rgw_obj obj(store->zone.domain_root, oid);
+
+  string key;
+  store->get_bucket_instance_entry(bucket, key); /* we want the bucket instance name without
+						    the oid prefix cruft */
+  return store->meta_mgr->set_attrs(bucket_instance_meta_handler, key,
+                                    obj, attrs, rmattrs, objv_tracker);
+}
+
+static void dump_mulipart_index_results(list& objs_to_unlink,
+        Formatter *f)
+{
+  // make sure that an appropiately titled header has been opened previously
+  list::iterator oiter = objs_to_unlink.begin();
+
+  f->open_array_section("invalid_multipart_entries");
+
+  for ( ; oiter != objs_to_unlink.end(); ++oiter) {
+    f->dump_string("object",  *oiter);
+  }
+
+  f->close_section();
+}
+
+void check_bad_user_bucket_mapping(RGWRados *store, const string& user_id, bool fix)
+{
+  RGWUserBuckets user_buckets;
+  bool done;
+  string marker;
+
+  CephContext *cct = store->ctx();
+
+  size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+
+  do {
+    int ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, max_entries, false);
+    if (ret < 0) {
+      ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl;
+      return;
+    }
+
+    map& buckets = user_buckets.get_buckets();
+    for (map::iterator i = buckets.begin();
+         i != buckets.end();
+         ++i) {
+      marker = i->first;
+
+      RGWBucketEnt& bucket_ent = i->second;
+      rgw_bucket& bucket = bucket_ent.bucket;
+
+      RGWBucketInfo bucket_info;
+      time_t mtime;
+      int r = store->get_bucket_info(NULL, bucket.name, bucket_info, &mtime);
+      if (r < 0) {
+        ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl;
+        continue;
+      }
+
+      rgw_bucket& actual_bucket = bucket_info.bucket;
+
+      if (actual_bucket.name.compare(bucket.name) != 0 ||
+          actual_bucket.data_pool.compare(bucket.data_pool) != 0 ||
+          actual_bucket.index_pool.compare(bucket.index_pool) != 0 ||
+          actual_bucket.marker.compare(bucket.marker) != 0 ||
+          actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
+        cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
+        if (fix) {
+          cout << "fixing" << std::endl;
+          r = rgw_link_bucket(store, user_id, actual_bucket, bucket_info.creation_time);
+          if (r < 0) {
+            cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
+          }
+        }
+      }
+    }
+    done = (buckets.size() < max_entries);
+  } while (!done);
+}
+
+static bool bucket_object_check_filter(const string& name)
+{
+  string ns;
+  string obj = name;
+  return rgw_obj::translate_raw_obj_to_obj_in_ns(obj, ns);
+}
+
+int rgw_remove_object(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, std::string& object)
+{
+  RGWRadosCtx rctx(store);
+
+  rgw_obj obj(bucket, object);
+
+  int ret = store->delete_obj((void *)&rctx, bucket_owner, obj);
+
+  return ret;
+}
+
+int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, bool delete_children)
+{
+  int ret;
+  map stats;
+  std::vector objs;
+  std::string prefix, delim, marker, ns;
+  map common_prefixes;
+  rgw_obj obj;
+  RGWBucketInfo info;
+  bufferlist bl;
+
+  uint64_t bucket_ver, master_ver;
+
+  ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, NULL);
+  if (ret < 0)
+    return ret;
+
+  obj.bucket = bucket;
+
+  ret = store->get_bucket_info(NULL, bucket.name, info, NULL);
+  if (ret < 0)
+    return ret;
+
+  if (delete_children) {
+    int max = 1000;
+    ret = store->list_objects(bucket, max, prefix, delim, marker, NULL,
+            objs, common_prefixes,
+            false, ns, true, NULL, NULL);
+
+    if (ret < 0)
+      return ret;
+
+    while (!objs.empty()) {
+      std::vector::iterator it = objs.begin();
+      for (it = objs.begin(); it != objs.end(); ++it) {
+        ret = rgw_remove_object(store, bucket_owner, bucket, (*it).name);
+        if (ret < 0)
+          return ret;
+      }
+      objs.clear();
+
+      ret = store->list_objects(bucket, max, prefix, delim, marker, NULL, objs, common_prefixes,
+                                false, ns, true, NULL, NULL);
+      if (ret < 0)
+        return ret;
+    }
+  }
+
+  RGWObjVersionTracker objv_tracker;
+
+  ret = store->delete_bucket(bucket, objv_tracker);
+  if (ret < 0) {
+    lderr(store->ctx()) << "ERROR: could not remove bucket " << bucket.name << dendl;
+    return ret;
+  }
+
+  ret = rgw_unlink_bucket(store, info.owner, bucket.name);
+  if (ret < 0) {
+    lderr(store->ctx()) << "ERROR: unable to remove user bucket information" << dendl;
+  }
+
+  return ret;
+}
+
+int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker)
+{
+  return store->meta_mgr->remove_entry(bucket_meta_handler, bucket_name, &objv_tracker);
+}
+
+static void set_err_msg(std::string *sink, std::string msg)
+{
+  if (sink && !msg.empty())
+    *sink = msg;
+}
+
+int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state)
+{
+  if (!storage)
+    return -EINVAL;
+
+  store = storage;
+
+  string user_id = op_state.get_user_id();
+  bucket_name = op_state.get_bucket_name();
+  RGWUserBuckets user_buckets;
+
+  if (bucket_name.empty() && user_id.empty())
+    return -EINVAL;
+
+  if (!bucket_name.empty()) {
+    int r = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL);
+    if (r < 0) {
+      ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket_name << dendl;
+      return r;
+    }
+
+    op_state.set_bucket(bucket_info.bucket);
+  }
+
+  if (!user_id.empty()) {
+    int r = rgw_get_user_info_by_uid(store, user_id, user_info);
+    if (r < 0)
+      return r;
+
+    op_state.display_name = user_info.display_name;
+  }
+
+  clear_failure();
+  return 0;
+}
+
+int RGWBucket::link(RGWBucketAdminOpState& op_state, std::string *err_msg)
+{
+  if (!op_state.is_user_op()) {
+    set_err_msg(err_msg, "empty user id");
+    return -EINVAL;
+  }
+
+  std::string no_oid;
+
+  std::string display_name = op_state.get_user_display_name();
+  rgw_bucket bucket = op_state.get_bucket();
+
+  rgw_obj obj(bucket, no_oid);
+  RGWObjVersionTracker objv_tracker;
+
+  map attrs;
+  RGWBucketInfo bucket_info;
+
+  int r = store->get_bucket_info(NULL, bucket.name, bucket_info, NULL, &attrs);
+  if (r < 0) {
+    return r;
+  }
+
+  map::iterator aiter = attrs.find(RGW_ATTR_ACL);
+  if (aiter != attrs.end()) {
+    bufferlist aclbl = aiter->second;
+    RGWAccessControlPolicy policy;
+    ACLOwner owner;
+    try {
+     bufferlist::iterator iter = aclbl.begin();
+     ::decode(policy, iter);
+     owner = policy.get_owner();
+    } catch (buffer::error& err) {
+      set_err_msg(err_msg, "couldn't decode policy");
+      return -EIO;
+    }
+
+    r = rgw_unlink_bucket(store, owner.get_id(), bucket.name);
+    if (r < 0) {
+      set_err_msg(err_msg, "could not unlink policy from user " + owner.get_id());
+      return r;
+    }
+
+    // now update the user for the bucket...
+    if (display_name.empty()) {
+      ldout(store->ctx(), 0) << "WARNING: user " << user_info.user_id << " has no display name set" << dendl;
+    }
+    policy.create_default(user_info.user_id, display_name);
+
+    owner = policy.get_owner();
+    r = store->set_bucket_owner(bucket, owner);
+    if (r < 0) {
+      set_err_msg(err_msg, "failed to set bucket owner: " + cpp_strerror(-r));
+      return r;
+    }
+
+    // ...and encode the acl
+    aclbl.clear();
+    policy.encode(aclbl);
+
+    r = store->set_attr(NULL, obj, RGW_ATTR_ACL, aclbl, &objv_tracker);
+    if (r < 0)
+      return r;
+
+    r = rgw_link_bucket(store, user_info.user_id, bucket, 0);
+    if (r < 0)
+      return r;
+  }
+
+  return 0;
+}
+
+int RGWBucket::unlink(RGWBucketAdminOpState& op_state, std::string *err_msg)
+{
+  rgw_bucket bucket = op_state.get_bucket();
+
+  if (!op_state.is_user_op()) {
+    set_err_msg(err_msg, "could not fetch user or user bucket info");
+    return -EINVAL;
+  }
+
+  int r = rgw_unlink_bucket(store, user_info.user_id, bucket.name);
+  if (r < 0) {
+    set_err_msg(err_msg, "error unlinking bucket" + cpp_strerror(-r));
+  }
+
+  return r;
+}
+
+int RGWBucket::remove(RGWBucketAdminOpState& op_state, std::string *err_msg)
+{
+  bool delete_children = op_state.will_delete_children();
+  rgw_bucket bucket = op_state.get_bucket();
+
+  int ret = rgw_remove_bucket(store, bucket_info.owner, bucket, delete_children);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to remove bucket" + cpp_strerror(-ret));
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWBucket::remove_object(RGWBucketAdminOpState& op_state, std::string *err_msg)
+{
+  rgw_bucket bucket = op_state.get_bucket();
+  std::string object_name = op_state.get_object_name();
+
+  int ret = rgw_remove_object(store, bucket_info.owner, bucket, object_name);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to remove object" + cpp_strerror(-ret));
+    return ret;
+  }
+
+  return 0;
+}
+
+static void dump_bucket_index(map result,  Formatter *f)
+{
+  map::iterator iter;
+  for (iter = result.begin(); iter != result.end(); ++iter) {
+    f->dump_string("object", iter->first);
+   }
+}
+
+static void dump_bucket_usage(map& stats, Formatter *formatter)
+{
+  map::iterator iter;
+
+  formatter->open_object_section("usage");
+  for (iter = stats.begin(); iter != stats.end(); ++iter) {
+    RGWStorageStats& s = iter->second;
+    const char *cat_name = rgw_obj_category_name(iter->first);
+    formatter->open_object_section(cat_name);
+    formatter->dump_int("size_kb", s.num_kb);
+    formatter->dump_int("size_kb_actual", s.num_kb_rounded);
+    formatter->dump_int("num_objects", s.num_objects);
+    formatter->close_section();
+  }
+  formatter->close_section();
+}
+
+static void dump_index_check(map existing_stats,
+        map calculated_stats,
+        Formatter *formatter)
+{
+  formatter->open_object_section("check_result");
+  formatter->open_object_section("existing_header");
+  dump_bucket_usage(existing_stats, formatter);
+  formatter->close_section();
+  formatter->open_object_section("calculated_header");
+  dump_bucket_usage(calculated_stats, formatter);
+  formatter->close_section();
+  formatter->close_section();
+}
+
+int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state,
+        list& objs_to_unlink, std::string *err_msg)
+{
+  bool fix_index = op_state.will_fix_index();
+  rgw_bucket bucket = op_state.get_bucket();
+
+  int max = 1000;
+  string prefix;
+  string marker;
+  string delim;
+
+  map common_prefixes;
+  string ns = "multipart";
+
+  bool is_truncated;
+  map meta_objs;
+  map all_objs;
+
+  do {
+    vector result;
+    int r = store->list_objects(bucket, max, prefix, delim, marker, NULL,
+                                result, common_prefixes, false,
+                                ns, true,
+                                &is_truncated, NULL);
+
+    if (r < 0) {
+      set_err_msg(err_msg, "failed to list objects in bucket=" + bucket.name +
+              " err=" +  cpp_strerror(-r));
+
+      return r;
+    }
+
+    vector::iterator iter;
+    for (iter = result.begin(); iter != result.end(); ++iter) {
+      RGWObjEnt& ent = *iter;
+
+      rgw_obj obj(bucket, ent.name);
+      obj.set_ns(ns);
+
+      string& oid = obj.object;
+      marker = oid;
+
+      int pos = oid.find_last_of('.');
+      if (pos < 0)
+        continue;
+
+      string name = oid.substr(0, pos);
+      string suffix = oid.substr(pos + 1);
+
+      if (suffix.compare("meta") == 0) {
+        meta_objs[name] = true;
+      } else {
+        all_objs[oid] = name;
+      }
+    }
+
+  } while (is_truncated);
+
+  map::iterator aiter;
+  for (aiter = all_objs.begin(); aiter != all_objs.end(); ++aiter) {
+    string& name = aiter->second;
+
+    if (meta_objs.find(name) == meta_objs.end()) {
+      objs_to_unlink.push_back(aiter->first);
+    }
+  }
+
+  if (objs_to_unlink.empty())
+    return 0;
+
+  if (fix_index) {
+    int r = store->remove_objs_from_index(bucket, objs_to_unlink);
+    if (r < 0) {
+      set_err_msg(err_msg, "ERROR: remove_obj_from_index() returned error: " +
+              cpp_strerror(-r));
+
+      return r;
+    }
+  }
+
+  return 0;
+}
+
+int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
+        map result, std::string *err_msg)
+{
+
+  bool fix_index = op_state.will_fix_index();
+  rgw_bucket bucket = op_state.get_bucket();
+
+  if (!fix_index) {
+    set_err_msg(err_msg, "check-objects flag requires fix index enabled");
+    return -EINVAL;
+  }
+
+/*
+  dout(0) << "Checking objects, decreasing bucket 2-phase commit timeout.\n"\
+	  << "** Note that timeout will reset only when operation completes successfully **" << dendl;
+*/
+  store->cls_obj_set_bucket_tag_timeout(bucket, BUCKET_TAG_TIMEOUT);
+
+  string prefix;
+  string marker;
+  bool is_truncated = true;
+
+  while (is_truncated) {
+    map result;
+
+    int r = store->cls_bucket_list(bucket, marker, prefix, 1000, result,
+             &is_truncated, &marker,
+             bucket_object_check_filter);
+
+    if (r == -ENOENT) {
+      break;
+    } else if (r < 0 && r != -ENOENT) {
+      set_err_msg(err_msg, "ERROR: failed operation r=" + cpp_strerror(-r));
+    }
+  }
+
+  store->cls_obj_set_bucket_tag_timeout(bucket, 0);
+
+  return 0;
+}
+
+
+int RGWBucket::check_index(RGWBucketAdminOpState& op_state,
+        map& existing_stats,
+        map& calculated_stats,
+        std::string *err_msg)
+{
+  rgw_bucket bucket = op_state.get_bucket();
+  bool fix_index = op_state.will_fix_index();
+
+  int r = store->bucket_check_index(bucket, &existing_stats, &calculated_stats);
+  if (r < 0) {
+    set_err_msg(err_msg, "failed to check index error=" + cpp_strerror(-r));
+    return r;
+  }
+
+  if (fix_index) {
+    r = store->bucket_rebuild_index(bucket);
+    if (r < 0) {
+      set_err_msg(err_msg, "failed to rebuild index err=" + cpp_strerror(-r));
+      return r;
+    }
+  }
+
+  return 0;
+}
+
+
+int RGWBucket::policy_bl_to_stream(bufferlist& bl, ostream& o)
+{
+  RGWAccessControlPolicy_S3 policy(g_ceph_context);
+  bufferlist::iterator iter = bl.begin();
+  try {
+    policy.decode(iter);
+  } catch (buffer::error& err) {
+    dout(0) << "ERROR: caught buffer::error, could not decode policy" << dendl;
+    return -EIO;
+  }
+  policy.to_xml(o);
+  return 0;
+}
+
+int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, ostream& o)
+{
+  std::string object_name = op_state.get_object_name();
+  rgw_bucket bucket = op_state.get_bucket();
+
+  if (!object_name.empty()) {
+    bufferlist bl;
+    rgw_obj obj(bucket, object_name);
+    int ret = store->get_attr(NULL, obj, RGW_ATTR_ACL, bl);
+    if (ret < 0)
+      return ret;
+
+    return policy_bl_to_stream(bl, o);
+  }
+
+
+  RGWBucketInfo bucket_info;
+  map attrs;
+  int ret = store->get_bucket_info(NULL, bucket.name, bucket_info, NULL, &attrs);
+  if (ret < 0) {
+    return ret;
+  }
+
+  map::iterator aiter = attrs.find(RGW_ATTR_ACL);
+  if (aiter == attrs.end()) {
+    return -ENOENT;
+  }
+
+  return policy_bl_to_stream(aiter->second, o);
+}
+
+
+int RGWBucketAdminOp::get_policy(RGWRados *store, RGWBucketAdminOpState& op_state,
+                  ostream& os)
+{
+   RGWBucket bucket;
+
+  int ret = bucket.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  ret = bucket.get_policy(op_state, os);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+/* Wrappers to facilitate RESTful interface */
+
+
+int RGWBucketAdminOp::get_policy(RGWRados *store, RGWBucketAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  std::ostringstream policy_stream;
+
+  int ret = get_policy(store, op_state, policy_stream);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  flusher.start(0);
+
+  formatter->dump_string("policy", policy_stream.str());
+
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWBucketAdminOp::unlink(RGWRados *store, RGWBucketAdminOpState& op_state)
+{
+  RGWBucket bucket;
+
+  int ret = bucket.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  return bucket.unlink(op_state);
+}
+
+int RGWBucketAdminOp::link(RGWRados *store, RGWBucketAdminOpState& op_state)
+{
+  RGWBucket bucket;
+
+  int ret = bucket.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  return bucket.link(op_state);
+
+}
+
+int RGWBucketAdminOp::check_index(RGWRados *store, RGWBucketAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  int ret;
+  map result;
+  map existing_stats;
+  map calculated_stats;
+  list objs_to_unlink;
+
+  RGWBucket bucket;
+
+  ret = bucket.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+  flusher.start(0);
+
+  ret = bucket.check_bad_index_multipart(op_state, objs_to_unlink);
+  if (ret < 0)
+    return ret;
+
+  dump_mulipart_index_results(objs_to_unlink, formatter);
+  flusher.flush();
+
+  ret = bucket.check_object_index(op_state, result);
+  if (ret < 0)
+    return ret;
+
+  dump_bucket_index(result,  formatter);
+  flusher.flush();
+
+  ret = bucket.check_index(op_state, existing_stats, calculated_stats);
+  if (ret < 0)
+    return ret;
+
+  dump_index_check(existing_stats, calculated_stats, formatter);
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWBucketAdminOp::remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state)
+{
+  RGWBucket bucket;
+
+  int ret = bucket.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  return bucket.remove(op_state);
+}
+
+int RGWBucketAdminOp::remove_object(RGWRados *store, RGWBucketAdminOpState& op_state)
+{
+  RGWBucket bucket;
+
+  int ret = bucket.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  return bucket.remove_object(op_state);
+}
+
+static int bucket_stats(RGWRados *store, std::string&  bucket_name, Formatter *formatter)
+{
+  RGWBucketInfo bucket_info;
+  rgw_bucket bucket;
+  map stats;
+
+  time_t mtime;
+  int r = store->get_bucket_info(NULL, bucket_name, bucket_info, &mtime);
+  if (r < 0)
+    return r;
+
+  bucket = bucket_info.bucket;
+
+  uint64_t bucket_ver, master_ver;
+  string max_marker;
+  int ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, &max_marker);
+  if (ret < 0) {
+    cerr << "error getting bucket stats ret=" << ret << std::endl;
+    return ret;
+  }
+
+  formatter->open_object_section("stats");
+  formatter->dump_string("bucket", bucket.name);
+  formatter->dump_string("pool", bucket.data_pool);
+  formatter->dump_string("index_pool", bucket.index_pool);
+  formatter->dump_string("id", bucket.bucket_id);
+  formatter->dump_string("marker", bucket.marker);
+  formatter->dump_string("owner", bucket_info.owner);
+  formatter->dump_int("ver", bucket_ver);
+  formatter->dump_int("master_ver", master_ver);
+  formatter->dump_int("mtime", mtime);
+  formatter->dump_string("max_marker", max_marker);
+  dump_bucket_usage(stats, formatter);
+  encode_json("bucket_quota", bucket_info.quota, formatter);
+  formatter->close_section();
+
+  return 0;
+}
+
+
+int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWBucket bucket;
+  int ret;
+
+  string bucket_name = op_state.get_bucket_name();
+
+  if (!bucket_name.empty()) {
+    ret = bucket.init(store, op_state);
+    if (ret < 0)
+      return ret;
+  }
+
+  Formatter *formatter = flusher.get_formatter();
+  flusher.start(0);
+
+  CephContext *cct = store->ctx();
+
+  size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+
+  bool show_stats = op_state.will_fetch_stats();
+  if (op_state.is_user_op()) {
+    formatter->open_array_section("buckets");
+
+    RGWUserBuckets buckets;
+    string marker;
+    bool done;
+
+    do {
+      ret = rgw_read_user_buckets(store, op_state.get_user_id(), buckets, marker, max_entries, false);
+      if (ret < 0)
+        return ret;
+
+      map& m = buckets.get_buckets();
+      map::iterator iter;
+
+      for (iter = m.begin(); iter != m.end(); ++iter) {
+        std::string  obj_name = iter->first;
+        if (show_stats)
+          bucket_stats(store, obj_name, formatter);
+        else
+          formatter->dump_string("bucket", obj_name);
+
+        marker = obj_name;
+      }
+
+      flusher.flush();
+      done = (m.size() < max_entries);
+    } while (!done);
+
+    formatter->close_section();
+  } else if (!bucket_name.empty()) {
+    bucket_stats(store, bucket_name, formatter);
+  } else {
+    RGWAccessHandle handle;
+
+    formatter->open_array_section("buckets");
+    if (store->list_buckets_init(&handle) >= 0) {
+      RGWObjEnt obj;
+      while (store->list_buckets_next(obj, &handle) >= 0) {
+	formatter->dump_string("bucket", obj.name);
+        if (show_stats)
+          bucket_stats(store, obj.name, formatter);
+      }
+    }
+
+    formatter->close_section();
+  }
+
+  flusher.flush();
+
+  return 0;
+}
+
+
+void rgw_data_change::dump(Formatter *f) const
+{
+  string type;
+  switch (entity_type) {
+    case ENTITY_TYPE_BUCKET:
+      type = "bucket";
+      break;
+    default:
+      type = "unknown";
+  }
+  encode_json("entity_type", type, f);
+  encode_json("key", key, f);
+  encode_json("timestamp", timestamp, f);
+}
+
+
+int RGWDataChangesLog::choose_oid(rgw_bucket& bucket) {
+    string& name = bucket.name;
+    uint32_t r = ceph_str_hash_linux(name.c_str(), name.size()) % num_shards;
+
+    return (int)r;
+}
+
+int RGWDataChangesLog::renew_entries()
+{
+  if (!store->need_to_log_data())
+    return 0;
+
+  /* we can't keep the bucket name as part of the cls_log_entry, and we need
+   * it later, so we keep two lists under the map */
+  map, list > > m;
+
+  lock.Lock();
+  map entries;
+  entries.swap(cur_cycle);
+  lock.Unlock();
+
+  map::iterator iter;
+  string section;
+  utime_t ut = ceph_clock_now(cct);
+  for (iter = entries.begin(); iter != entries.end(); ++iter) {
+    rgw_bucket& bucket = iter->second;
+    int index = choose_oid(bucket);
+
+    cls_log_entry entry;
+
+    rgw_data_change change;
+    bufferlist bl;
+    change.entity_type = ENTITY_TYPE_BUCKET;
+    change.key = bucket.name + ":" + bucket.bucket_id;
+    change.timestamp = ut;
+    ::encode(change, bl);
+
+    store->time_log_prepare_entry(entry, ut, section, bucket.name, bl);
+
+    m[index].first.push_back(bucket.name);
+    m[index].second.push_back(entry);
+  }
+
+  map, list > >::iterator miter;
+  for (miter = m.begin(); miter != m.end(); ++miter) {
+    list& entries = miter->second.second;
+
+    utime_t now = ceph_clock_now(cct);
+
+    int ret = store->time_log_add(oids[miter->first], entries);
+    if (ret < 0) {
+      /* we don't really need to have a special handling for failed cases here,
+       * as this is just an optimization. */
+      lderr(cct) << "ERROR: store->time_log_add() returned " << ret << dendl;
+      return ret;
+    }
+
+    utime_t expiration = now;
+    expiration += utime_t(cct->_conf->rgw_data_log_window, 0);
+
+    list& buckets = miter->second.first;
+    list::iterator liter;
+    for (liter = buckets.begin(); liter != buckets.end(); ++liter) {
+      update_renewed(*liter, expiration);
+    }
+  }
+
+  return 0;
+}
+
+void RGWDataChangesLog::_get_change(string& bucket_name, ChangeStatusPtr& status)
+{
+  assert(lock.is_locked());
+  if (!changes.find(bucket_name, status)) {
+    status = ChangeStatusPtr(new ChangeStatus);
+    changes.add(bucket_name, status);
+  }
+}
+
+void RGWDataChangesLog::register_renew(rgw_bucket& bucket)
+{
+  Mutex::Locker l(lock);
+  cur_cycle[bucket.name] = bucket;
+}
+
+void RGWDataChangesLog::update_renewed(string& bucket_name, utime_t& expiration)
+{
+  Mutex::Locker l(lock);
+  ChangeStatusPtr status;
+  _get_change(bucket_name, status);
+
+  ldout(cct, 20) << "RGWDataChangesLog::update_renewd() bucket_name=" << bucket_name << " expiration=" << expiration << dendl;
+  status->cur_expiration = expiration;
+}
+
+int RGWDataChangesLog::add_entry(rgw_bucket& bucket) {
+  if (!store->need_to_log_data())
+    return 0;
+
+  lock.Lock();
+
+  ChangeStatusPtr status;
+  _get_change(bucket.name, status);
+
+  lock.Unlock();
+
+  utime_t now = ceph_clock_now(cct);
+
+  status->lock->Lock();
+
+  ldout(cct, 20) << "RGWDataChangesLog::add_entry() bucket.name=" << bucket.name << " now=" << now << " cur_expiration=" << status->cur_expiration << dendl;
+
+  if (now < status->cur_expiration) {
+    /* no need to send, recently completed */
+    status->lock->Unlock();
+
+    register_renew(bucket);
+    return 0;
+  }
+
+  RefCountedCond *cond;
+
+  if (status->pending) {
+    cond = status->cond;
+
+    assert(cond);
+
+    status->cond->get();
+    status->lock->Unlock();
+
+    int ret = cond->wait();
+    cond->put();
+    if (!ret) {
+      register_renew(bucket);
+    }
+    return ret;
+  }
+
+  status->cond = new RefCountedCond;
+  status->pending = true;
+
+  string& oid = oids[choose_oid(bucket)];
+  utime_t expiration;
+
+  int ret;
+
+  do {
+    status->cur_sent = now;
+
+    expiration = now;
+    expiration += utime_t(cct->_conf->rgw_data_log_window, 0);
+
+    status->lock->Unlock();
+  
+    bufferlist bl;
+    rgw_data_change change;
+    change.entity_type = ENTITY_TYPE_BUCKET;
+    change.key = bucket.name + ":" + bucket.bucket_id;
+    change.timestamp = now;
+    ::encode(change, bl);
+    string section;
+
+    ldout(cct, 20) << "RGWDataChangesLog::add_entry() sending update with now=" << now << " cur_expiration=" << expiration << dendl;
+
+    ret = store->time_log_add(oid, now, section, change.key, bl);
+
+    now = ceph_clock_now(cct);
+
+    status->lock->Lock();
+
+  } while (!ret && ceph_clock_now(cct) > expiration);
+
+  cond = status->cond;
+
+  status->pending = false;
+  status->cur_expiration = status->cur_sent; /* time of when operation started, not completed */
+  status->cur_expiration += utime_t(cct->_conf->rgw_data_log_window, 0);
+  status->cond = NULL;
+  status->lock->Unlock();
+
+  cond->done(ret);
+  cond->put();
+
+  return ret;
+}
+
+int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
+				    list& entries,
+				    const string& marker,
+				    string *out_marker,
+				    bool *truncated) {
+
+  list log_entries;
+
+  int ret = store->time_log_list(oids[shard], start_time, end_time,
+				 max_entries, log_entries, marker,
+				 out_marker, truncated);
+  if (ret < 0)
+    return ret;
+
+  list::iterator iter;
+  for (iter = log_entries.begin(); iter != log_entries.end(); ++iter) {
+    rgw_data_change entry;
+    bufferlist::iterator liter = iter->data.begin();
+    try {
+      ::decode(entry, liter);
+    } catch (buffer::error& err) {
+      lderr(cct) << "ERROR: failed to decode data changes log entry" << dendl;
+      return -EIO;
+    }
+    entries.push_back(entry);
+  }
+
+  return 0;
+}
+
+int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int max_entries,
+             list& entries, LogMarker& marker, bool *ptruncated) {
+  bool truncated;
+  entries.clear();
+
+  for (; marker.shard < num_shards && (int)entries.size() < max_entries;
+       marker.shard++, marker.marker.clear()) {
+    int ret = list_entries(marker.shard, start_time, end_time, max_entries - entries.size(), entries,
+			   marker.marker, NULL, &truncated);
+    if (ret == -ENOENT) {
+      continue;
+    }
+    if (ret < 0) {
+      return ret;
+    }
+    if (truncated) {
+      *ptruncated = true;
+      return 0;
+    }
+  }
+
+  *ptruncated = (marker.shard < num_shards);
+
+  return 0;
+}
+
+int RGWDataChangesLog::get_info(int shard_id, RGWDataChangesLogInfo *info)
+{
+  if (shard_id > num_shards)
+    return -EINVAL;
+
+  string oid = oids[shard_id];
+
+  cls_log_header header;
+
+  int ret = store->time_log_info(oid, &header);
+  if ((ret < 0) && (ret != -ENOENT))
+    return ret;
+
+  info->marker = header.max_marker;
+  info->last_update = header.max_time;
+
+  return 0;
+}
+
+int RGWDataChangesLog::trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time,
+                                    const string& start_marker, const string& end_marker)
+{
+  int ret;
+
+  if (shard_id > num_shards)
+    return -EINVAL;
+
+  ret = store->time_log_trim(oids[shard_id], start_time, end_time, start_marker, end_marker);
+
+  if (ret == -ENOENT)
+    ret = 0;
+
+  return ret;
+}
+
+int RGWDataChangesLog::trim_entries(const utime_t& start_time, const utime_t& end_time,
+                                    const string& start_marker, const string& end_marker)
+{
+  for (int shard = 0; shard < num_shards; shard++) {
+    int ret = store->time_log_trim(oids[shard], start_time, end_time, start_marker, end_marker);
+    if (ret == -ENOENT) {
+      continue;
+    }
+    if (ret < 0)
+      return ret;
+  }
+
+  return 0;
+}
+
+bool RGWDataChangesLog::going_down()
+{
+  return (down_flag.read() != 0);
+}
+
+RGWDataChangesLog::~RGWDataChangesLog() {
+  down_flag.set(1);
+  renew_thread->stop();
+  renew_thread->join();
+  delete renew_thread;
+  delete[] oids;
+}
+
+void *RGWDataChangesLog::ChangesRenewThread::entry() {
+  do {
+    dout(2) << "RGWDataChangesLog::ChangesRenewThread: start" << dendl;
+    int r = log->renew_entries();
+    if (r < 0) {
+      dout(0) << "ERROR: RGWDataChangesLog::renew_entries returned error r=" << r << dendl;
+    }
+
+    if (log->going_down())
+      break;
+
+    int interval = cct->_conf->rgw_data_log_window * 3 / 4;
+    lock.Lock();
+    cond.WaitInterval(cct, lock, utime_t(interval, 0));
+    lock.Unlock();
+  } while (!log->going_down());
+
+  return NULL;
+}
+
+void RGWDataChangesLog::ChangesRenewThread::stop()
+{
+  Mutex::Locker l(lock);
+  cond.Signal();
+}
+
+struct RGWBucketCompleteInfo {
+  RGWBucketInfo info;
+  map attrs;
+
+ void dump(Formatter *f) const {
+    encode_json("bucket_info", info, f);
+    encode_json("attrs", attrs, f);
+  }
+
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("bucket_info", info, obj);
+    JSONDecoder::decode_json("attrs", attrs, obj);
+  }
+};
+
+class RGWBucketEntryMetadataObject : public RGWMetadataObject {
+  RGWBucketEntryPoint ep;
+public:
+  RGWBucketEntryMetadataObject(RGWBucketEntryPoint& _ep, obj_version& v, time_t m) : ep(_ep) {
+    objv = v;
+    mtime = m;
+  }
+
+  void dump(Formatter *f) const {
+    ep.dump(f);
+  }
+};
+
+class RGWBucketInstanceMetadataObject : public RGWMetadataObject {
+  RGWBucketCompleteInfo info;
+public:
+  RGWBucketInstanceMetadataObject(RGWBucketCompleteInfo& i, obj_version& v, time_t m) : info(i) {
+    objv = v;
+    mtime = m;
+  }
+
+  void dump(Formatter *f) const {
+    info.dump(f);
+  }
+};
+
+class RGWBucketMetadataHandler : public RGWMetadataHandler {
+
+  int init_bucket(RGWRados *store, string& bucket_name, rgw_bucket& bucket, RGWObjVersionTracker *objv_tracker) {
+    RGWBucketInfo bucket_info;
+    int r = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL);
+    if (r < 0) {
+      cerr << "could not get bucket info for bucket=" << bucket_name << std::endl;
+      return r;
+    }
+    bucket = bucket_info.bucket;
+
+    return 0;
+  }
+
+public:
+  string get_type() { return "bucket"; }
+
+  int get(RGWRados *store, string& entry, RGWMetadataObject **obj) {
+    RGWObjVersionTracker ot;
+    RGWBucketEntryPoint be;
+
+    time_t mtime;
+    map attrs;
+
+    int ret = store->get_bucket_entrypoint_info(NULL, entry, be, &ot, &mtime, &attrs);
+    if (ret < 0)
+      return ret;
+
+    RGWBucketEntryMetadataObject *mdo = new RGWBucketEntryMetadataObject(be, ot.read_version, mtime);
+
+    *obj = mdo;
+
+    return 0;
+  }
+
+  int put(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker,
+          time_t mtime, JSONObj *obj, sync_type_t sync_type) {
+    RGWBucketEntryPoint be, old_be;
+    decode_json_obj(be, obj);
+
+    time_t orig_mtime;
+    map attrs;
+
+    RGWObjVersionTracker old_ot;
+
+    int ret = store->get_bucket_entrypoint_info(NULL, entry, old_be, &old_ot, &orig_mtime, &attrs);
+    if (ret < 0 && ret != -ENOENT)
+      return ret;
+
+    // are we actually going to perform this put, or is it too old?
+    if (ret != -ENOENT &&
+        !check_versions(old_ot.read_version, orig_mtime,
+			objv_tracker.write_version, mtime, sync_type)) {
+      return STATUS_NO_APPLY;
+    }
+
+    objv_tracker.read_version = old_ot.read_version; /* maintain the obj version we just read */
+
+    ret = store->put_bucket_entrypoint_info(entry, be, false, objv_tracker, mtime, &attrs);
+    if (ret < 0)
+      return ret;
+
+    /* link bucket */
+    if (be.linked) {
+      ret = rgw_link_bucket(store, be.owner, be.bucket, be.creation_time, false);
+    } else {
+      ret = rgw_unlink_bucket(store, be.owner, be.bucket.name, false);
+    }
+
+    return ret;
+  }
+
+  struct list_keys_info {
+    RGWRados *store;
+    RGWListRawObjsCtx ctx;
+  };
+
+  int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) {
+    RGWBucketEntryPoint be;
+
+    int ret = store->get_bucket_entrypoint_info(NULL, entry, be, &objv_tracker, NULL, NULL);
+    if (ret < 0)
+      return ret;
+
+    /*
+     * We're unlinking the bucket but we don't want to update the entrypoint here — we're removing
+     * it immediately and don't want to invalidate our cached objv_version or the bucket obj removal
+     * will incorrectly fail.
+     */
+    ret = rgw_unlink_bucket(store, be.owner, entry, false);
+    if (ret < 0) {
+      lderr(store->ctx()) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl;
+    }
+
+    ret = rgw_bucket_delete_bucket_obj(store, entry, objv_tracker);
+    if (ret < 0) {
+      lderr(store->ctx()) << "could not delete bucket=" << entry << dendl;
+    }
+    /* idempotent */
+    return 0;
+  }
+
+  void get_pool_and_oid(RGWRados *store, const string& key, rgw_bucket& bucket, string& oid) {
+    oid = key;
+    bucket = store->zone.domain_root;
+  }
+
+  int list_keys_init(RGWRados *store, void **phandle)
+  {
+    list_keys_info *info = new list_keys_info;
+
+    info->store = store;
+
+    *phandle = (void *)info;
+
+    return 0;
+  }
+
+  int list_keys_next(void *handle, int max, list& keys, bool *truncated) {
+    list_keys_info *info = static_cast(handle);
+
+    string no_filter;
+
+    keys.clear();
+
+    RGWRados *store = info->store;
+
+    list unfiltered_keys;
+
+    int ret = store->list_raw_objects(store->zone.domain_root, no_filter,
+                                      max, info->ctx, unfiltered_keys, truncated);
+    if (ret < 0 && ret != -ENOENT)
+      return ret;
+    if (ret == -ENOENT) {
+      if (truncated)
+        *truncated = false;
+      return 0;
+    }
+
+    // now filter out the system entries
+    list::iterator iter;
+    for (iter = unfiltered_keys.begin(); iter != unfiltered_keys.end(); ++iter) {
+      string& k = *iter;
+
+      if (k[0] != '.') {
+        keys.push_back(k);
+      }
+    }
+
+    return 0;
+  }
+
+  void list_keys_complete(void *handle) {
+    list_keys_info *info = static_cast(handle);
+    delete info;
+  }
+};
+
+class RGWBucketInstanceMetadataHandler : public RGWMetadataHandler {
+
+  int init_bucket(RGWRados *store, string& bucket_name, rgw_bucket& bucket, RGWObjVersionTracker *objv_tracker) {
+    RGWBucketInfo bucket_info;
+    int r = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL);
+    if (r < 0) {
+      cerr << "could not get bucket info for bucket=" << bucket_name << std::endl;
+      return r;
+    }
+    bucket = bucket_info.bucket;
+
+    return 0;
+  }
+
+public:
+  string get_type() { return "bucket.instance"; }
+
+  int get(RGWRados *store, string& oid, RGWMetadataObject **obj) {
+    RGWBucketCompleteInfo bci;
+
+    time_t mtime;
+
+    int ret = store->get_bucket_instance_info(NULL, oid, bci.info, &mtime, &bci.attrs);
+    if (ret < 0)
+      return ret;
+
+    RGWBucketInstanceMetadataObject *mdo = new RGWBucketInstanceMetadataObject(bci, bci.info.objv_tracker.read_version, mtime);
+
+    *obj = mdo;
+
+    return 0;
+  }
+
+  int put(RGWRados *store, string& oid, RGWObjVersionTracker& objv_tracker,
+          time_t mtime, JSONObj *obj, sync_type_t sync_type) {
+    RGWBucketCompleteInfo bci, old_bci;
+    decode_json_obj(bci, obj);
+
+    time_t orig_mtime;
+
+    int ret = store->get_bucket_instance_info(NULL, oid, old_bci.info, &orig_mtime, &old_bci.attrs);
+    bool exists = (ret != -ENOENT);
+    if (ret < 0 && exists)
+      return ret;
+
+
+    if (!exists || old_bci.info.bucket.bucket_id != bci.info.bucket.bucket_id) {
+      /* a new bucket, we need to select a new bucket placement for it */
+      rgw_bucket bucket;
+      ret = store->set_bucket_location_by_rule(bci.info.placement_rule, oid, bucket);
+      if (ret < 0) {
+        ldout(store->ctx(), 0) << "ERROR: select_bucket_placement() returned " << ret << dendl;
+        return ret;
+      }
+      bci.info.bucket.data_pool = bucket.data_pool;
+      bci.info.bucket.index_pool = bucket.index_pool;
+    } else {
+      /* existing bucket, keep its placement pools */
+      bci.info.bucket.data_pool = old_bci.info.bucket.data_pool;
+      bci.info.bucket.index_pool = old_bci.info.bucket.index_pool;
+    }
+
+    // are we actually going to perform this put, or is it too old?
+    if (exists &&
+        !check_versions(old_bci.info.objv_tracker.read_version, orig_mtime,
+			objv_tracker.write_version, mtime, sync_type)) {
+      objv_tracker.read_version = old_bci.info.objv_tracker.read_version;
+      return STATUS_NO_APPLY;
+    }
+
+    /* record the read version (if any), store the new version */
+    bci.info.objv_tracker.read_version = old_bci.info.objv_tracker.read_version;
+    bci.info.objv_tracker.write_version = objv_tracker.write_version;
+
+    ret = store->put_bucket_instance_info(bci.info, false, mtime, &bci.attrs);
+    if (ret < 0)
+      return ret;
+
+    objv_tracker = bci.info.objv_tracker;
+
+    ret = store->init_bucket_index(bci.info.bucket);
+    if (ret < 0)
+      return ret;
+
+    return STATUS_APPLIED;
+  }
+
+  struct list_keys_info {
+    RGWRados *store;
+    RGWListRawObjsCtx ctx;
+  };
+
+  int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) {
+    RGWBucketInfo info;
+
+    int ret = store->get_bucket_instance_info(NULL, entry, info, NULL, NULL);
+    if (ret < 0 && ret != -ENOENT)
+      return ret;
+
+    return rgw_bucket_instance_remove_entry(store, entry, &info.objv_tracker);
+  }
+
+  void get_pool_and_oid(RGWRados *store, const string& key, rgw_bucket& bucket, string& oid) {
+    oid = RGW_BUCKET_INSTANCE_MD_PREFIX + key;
+    bucket = store->zone.domain_root;
+  }
+
+  int list_keys_init(RGWRados *store, void **phandle)
+  {
+    list_keys_info *info = new list_keys_info;
+
+    info->store = store;
+
+    *phandle = (void *)info;
+
+    return 0;
+  }
+
+  int list_keys_next(void *handle, int max, list& keys, bool *truncated) {
+    list_keys_info *info = static_cast(handle);
+
+    string no_filter;
+
+    keys.clear();
+
+    RGWRados *store = info->store;
+
+    list unfiltered_keys;
+
+    int ret = store->list_raw_objects(store->zone.domain_root, no_filter,
+                                      max, info->ctx, unfiltered_keys, truncated);
+    if (ret < 0 && ret != -ENOENT)
+      return ret;
+    if (ret == -ENOENT) {
+      if (truncated)
+        *truncated = false;
+      return 0;
+    }
+
+    int prefix_size = sizeof(RGW_BUCKET_INSTANCE_MD_PREFIX) - 1;
+    // now filter in the relevant entries
+    list::iterator iter;
+    for (iter = unfiltered_keys.begin(); iter != unfiltered_keys.end(); ++iter) {
+      string& k = *iter;
+
+      if (k.compare(0, prefix_size, RGW_BUCKET_INSTANCE_MD_PREFIX) == 0) {
+        keys.push_back(k.substr(prefix_size));
+      }
+    }
+
+    return 0;
+  }
+
+  void list_keys_complete(void *handle) {
+    list_keys_info *info = static_cast(handle);
+    delete info;
+  }
+
+  /*
+   * hash entry for mdlog placement. Use the same hash key we'd have for the bucket entry
+   * point, so that the log entries end up at the same log shard, so that we process them
+   * in order
+   */
+  virtual void get_hash_key(const string& section, const string& key, string& hash_key) {
+    string k;
+    int pos = key.find(':');
+    if (pos < 0)
+      k = key;
+    else
+      k = key.substr(0, pos);
+    hash_key = "bucket:" + k;
+  }
+};
+
+void rgw_bucket_init(RGWMetadataManager *mm)
+{
+  bucket_meta_handler = new RGWBucketMetadataHandler;
+  mm->register_handler(bucket_meta_handler);
+  bucket_instance_meta_handler = new RGWBucketInstanceMetadataHandler;
+  mm->register_handler(bucket_instance_meta_handler);
+}
diff --git a/ceph/src/rgw/rgw_bucket.h b/ceph/src/rgw/rgw_bucket.h
new file mode 100644
index 00000000..f113590f
--- /dev/null
+++ b/ceph/src/rgw/rgw_bucket.h
@@ -0,0 +1,389 @@
+#ifndef CEPH_RGW_BUCKET_H
+#define CEPH_RGW_BUCKET_H
+
+#include 
+#include 
+
+#include "include/types.h"
+#include "rgw_common.h"
+#include "rgw_tools.h"
+
+#include "rgw_rados.h"
+
+#include "rgw_string.h"
+
+#include "common/Formatter.h"
+#include "common/lru_map.h"
+#include "rgw_formats.h"
+
+
+using namespace std;
+
+// define as static when RGWBucket implementation compete
+extern void rgw_get_buckets_obj(const string& user_id, string& buckets_obj_id);
+
+extern int rgw_bucket_store_info(RGWRados *store, const string& bucket_name, bufferlist& bl, bool exclusive,
+                                 map *pattrs, RGWObjVersionTracker *objv_tracker,
+                                 time_t mtime);
+extern int rgw_bucket_instance_store_info(RGWRados *store, string& oid, bufferlist& bl, bool exclusive,
+                                 map *pattrs, RGWObjVersionTracker *objv_tracker,
+                                 time_t mtime);
+
+extern int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersionTracker *objv_tracker);
+
+extern int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker);
+
+extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& user_id, rgw_bucket& bucket);
+extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& bucket_name);
+
+/**
+ * Store a list of the user's buckets, with associated functinos.
+ */
+class RGWUserBuckets
+{
+  map buckets;
+
+public:
+  RGWUserBuckets() {}
+  void encode(bufferlist& bl) const {
+    ::encode(buckets, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(buckets, bl);
+  }
+  /**
+   * Check if the user owns a bucket by the given name.
+   */
+  bool owns(string& name) {
+    map::iterator iter;
+    iter = buckets.find(name);
+    return (iter != buckets.end());
+  }
+
+  /**
+   * Add a (created) bucket to the user's bucket list.
+   */
+  void add(RGWBucketEnt& bucket) {
+    buckets[bucket.bucket.name] = bucket;
+  }
+
+  /**
+   * Remove a bucket from the user's list by name.
+   */
+  void remove(string& name) {
+    map::iterator iter;
+    iter = buckets.find(name);
+    if (iter != buckets.end()) {
+      buckets.erase(iter);
+    }
+  }
+
+  /**
+   * Get the user's buckets as a map.
+   */
+  map& get_buckets() { return buckets; }
+
+  /**
+   * Cleanup data structure
+   */
+  void clear() { buckets.clear(); }
+
+  size_t count() { return buckets.size(); }
+};
+WRITE_CLASS_ENCODER(RGWUserBuckets)
+
+class RGWMetadataManager;
+
+extern void rgw_bucket_init(RGWMetadataManager *mm);
+/**
+ * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
+ * Returns: 0 on success, -ERR# on failure.
+ */
+extern int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
+                                 const string& marker, uint64_t max, bool need_stats);
+
+extern int rgw_link_bucket(RGWRados *store, string user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true);
+extern int rgw_unlink_bucket(RGWRados *store, string user_id, const string& bucket_name, bool update_entrypoint = true);
+
+extern int rgw_remove_object(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, std::string& object);
+extern int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, bool delete_children);
+
+extern int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info,
+                                map& attrs,
+                                map* rmattrs,
+                                RGWObjVersionTracker *objv_tracker);
+
+extern void check_bad_user_bucket_mapping(RGWRados *store, const string& user_id, bool fix);
+
+struct RGWBucketAdminOpState {
+  std::string uid;
+  std::string display_name;
+  std::string bucket_name;
+  std::string bucket_id;
+  std::string object_name;
+
+  bool list_buckets;
+  bool stat_buckets;
+  bool check_objects;
+  bool fix_index;
+  bool delete_child_objects;
+  bool bucket_stored;
+
+  rgw_bucket bucket;
+
+  void set_fetch_stats(bool value) { stat_buckets = value; }
+  void set_check_objects(bool value) { check_objects = value; }
+  void set_fix_index(bool value) { fix_index = value; }
+  void set_delete_children(bool value) { delete_child_objects = value; }
+
+  void set_user_id(std::string& user_id) {
+    if (!user_id.empty())
+      uid = user_id;
+  }
+  void set_bucket_name(std::string& bucket_str) {
+    bucket_name = bucket_str; 
+  }
+  void set_object(std::string& object_str) {
+    object_name = object_str;
+  }
+
+  std::string& get_user_id() { return uid; };
+  std::string& get_user_display_name() { return display_name; };
+  std::string& get_bucket_name() { return bucket_name; };
+  std::string& get_object_name() { return object_name; };
+
+  rgw_bucket& get_bucket() { return bucket; };
+  void set_bucket(rgw_bucket& _bucket) {
+    bucket = _bucket; 
+    bucket_stored = true;
+  }
+
+  bool will_fetch_stats() { return stat_buckets; };
+  bool will_fix_index() { return fix_index; };
+  bool will_delete_children() { return delete_child_objects; };
+  bool will_check_objects() { return check_objects; };
+  bool is_user_op() { return !uid.empty(); };
+  bool is_system_op() { return uid.empty(); }; 
+  bool has_bucket_stored() { return bucket_stored; };
+
+  RGWBucketAdminOpState() : list_buckets(false), stat_buckets(false), check_objects(false), 
+                            fix_index(false), delete_child_objects(false),
+                            bucket_stored(false)  {}
+};
+
+/*
+ * A simple wrapper class for administrative bucket operations
+ */
+
+class RGWBucket
+{
+  RGWUserBuckets buckets;
+  RGWRados *store;
+  RGWAccessHandle handle;
+
+  RGWUserInfo user_info;
+  std::string bucket_name;
+
+  bool failure;
+
+  RGWBucketInfo bucket_info;
+
+private:
+
+public:
+  RGWBucket() : store(NULL), handle(NULL), failure(false) {}
+  int init(RGWRados *storage, RGWBucketAdminOpState& op_state);
+
+  int check_bad_index_multipart(RGWBucketAdminOpState& op_state,
+          list& objs_to_unlink, std::string *err_msg = NULL);
+
+  int check_object_index(RGWBucketAdminOpState& op_state,
+          map result, std::string *err_msg = NULL);
+
+  int check_index(RGWBucketAdminOpState& op_state,
+          map& existing_stats,
+          map& calculated_stats,
+          std::string *err_msg = NULL);
+
+  int remove(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
+  int link(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
+  int unlink(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
+
+  int remove_object(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
+  int policy_bl_to_stream(bufferlist& bl, ostream& o);
+  int get_policy(RGWBucketAdminOpState& op_state, ostream& o);
+
+  void clear_failure() { failure = false; };
+};
+
+class RGWBucketAdminOp
+{
+public:
+  static int get_policy(RGWRados *store, RGWBucketAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher);
+  static int get_policy(RGWRados *store, RGWBucketAdminOpState& op_state,
+                  ostream& os);
+
+
+  static int unlink(RGWRados *store, RGWBucketAdminOpState& op_state);
+  static int link(RGWRados *store, RGWBucketAdminOpState& op_state);
+
+  static int check_index(RGWRados *store, RGWBucketAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher);
+
+  static int remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state);
+  static int remove_object(RGWRados *store, RGWBucketAdminOpState& op_state);
+  static int info(RGWRados *store, RGWBucketAdminOpState& op_state, RGWFormatterFlusher& flusher);
+};
+
+
+enum DataLogEntityType {
+  ENTITY_TYPE_BUCKET = 1,
+};
+
+struct rgw_data_change {
+  DataLogEntityType entity_type;
+  string key;
+  utime_t timestamp;
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    uint8_t t = (uint8_t)entity_type;
+    ::encode(t, bl);
+    ::encode(key, bl);
+    ::encode(timestamp, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START(1, bl);
+     uint8_t t;
+     ::decode(t, bl);
+     entity_type = (DataLogEntityType)t;
+     ::decode(key, bl);
+     ::decode(timestamp, bl);
+     DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(rgw_data_change)
+
+struct RGWDataChangesLogInfo {
+  string marker;
+  utime_t last_update;
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+
+class RGWDataChangesLog {
+  CephContext *cct;
+  RGWRados *store;
+
+  int num_shards;
+  string *oids;
+
+  Mutex lock;
+
+  atomic_t down_flag;
+
+  struct ChangeStatus {
+    utime_t cur_expiration;
+    utime_t cur_sent;
+    bool pending;
+    RefCountedCond *cond;
+    Mutex *lock;
+
+    ChangeStatus() : pending(false), cond(NULL) {
+      lock = new Mutex("RGWDataChangesLog::ChangeStatus");
+    }
+
+    ~ChangeStatus() {
+      delete lock;
+    }
+  };
+
+  typedef ceph::shared_ptr ChangeStatusPtr;
+
+  lru_map changes;
+
+  map cur_cycle;
+
+  void _get_change(string& bucket_name, ChangeStatusPtr& status);
+  void register_renew(rgw_bucket& bucket);
+  void update_renewed(string& bucket_name, utime_t& expiration);
+
+  class ChangesRenewThread : public Thread {
+    CephContext *cct;
+    RGWDataChangesLog *log;
+    Mutex lock;
+    Cond cond;
+
+  public:
+    ChangesRenewThread(CephContext *_cct, RGWDataChangesLog *_log) : cct(_cct), log(_log), lock("ChangesRenewThread") {}
+    void *entry();
+    void stop();
+  };
+
+  ChangesRenewThread *renew_thread;
+
+public:
+
+  RGWDataChangesLog(CephContext *_cct, RGWRados *_store) : cct(_cct), store(_store), lock("RGWDataChangesLog"),
+                                                           changes(cct->_conf->rgw_data_log_changes_size) {
+    num_shards = cct->_conf->rgw_data_log_num_shards;
+
+    oids = new string[num_shards];
+
+    string prefix = cct->_conf->rgw_data_log_obj_prefix;
+
+    if (prefix.empty()) {
+      prefix = "data_log";
+    }
+
+    for (int i = 0; i < num_shards; i++) {
+      char buf[16];
+      snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i);
+      oids[i] = buf;
+    }
+
+    renew_thread = new ChangesRenewThread(cct, this);
+    renew_thread->create();
+  }
+
+  ~RGWDataChangesLog();
+
+  int choose_oid(rgw_bucket& bucket);
+  int add_entry(rgw_bucket& bucket);
+  int renew_entries();
+  int list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
+		   list& entries,
+		   const string& marker,
+		   string *out_marker,
+		   bool *truncated);
+  int trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time,
+                   const string& start_marker, const string& end_marker);
+  int trim_entries(const utime_t& start_time, const utime_t& end_time,
+                   const string& start_marker, const string& end_marker);
+  int get_info(int shard_id, RGWDataChangesLogInfo *info);
+  int lock_exclusive(int shard_id, utime_t& duration, string& zone_id, string& owner_id) {
+    return store->lock_exclusive(store->zone.log_pool, oids[shard_id], duration, zone_id, owner_id);
+  }
+  int unlock(int shard_id, string& zone_id, string& owner_id) {
+    return store->unlock(store->zone.log_pool, oids[shard_id], zone_id, owner_id);
+  }
+  struct LogMarker {
+    int shard;
+    string marker;
+
+    LogMarker() : shard(0) {}
+  };
+  int list_entries(utime_t& start_time, utime_t& end_time, int max_entries,
+               list& entries, LogMarker& marker, bool *ptruncated);
+
+  bool going_down();
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_cache.cc b/ceph/src/rgw/rgw_cache.cc
new file mode 100644
index 00000000..94b3d040
--- /dev/null
+++ b/ceph/src/rgw/rgw_cache.cc
@@ -0,0 +1,165 @@
+#include "rgw_cache.h"
+
+#include 
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask)
+{
+  RWLock::RLocker l(lock);
+
+  map::iterator iter = cache_map.find(name);
+  if (iter == cache_map.end()) {
+    ldout(cct, 10) << "cache get: name=" << name << " : miss" << dendl;
+    if(perfcounter) perfcounter->inc(l_rgw_cache_miss);
+    return -ENOENT;
+  }
+
+  ObjectCacheEntry& entry = iter->second;
+
+  if (lru_counter - entry.lru_promotion_ts > lru_window) {
+    ldout(cct, 20) << "cache get: touching lru, lru_counter=" << lru_counter << " promotion_ts=" << entry.lru_promotion_ts << dendl;
+    lock.unlock();
+    lock.get_write(); /* promote lock to writer */
+
+    /* check again, we might have lost a race here */
+    if (lru_counter - entry.lru_promotion_ts > lru_window) {
+      touch_lru(name, entry, iter->second.lru_iter);
+    }
+  }
+
+  ObjectCacheInfo& src = iter->second.info;
+  if ((src.flags & mask) != mask) {
+    ldout(cct, 10) << "cache get: name=" << name << " : type miss (requested=" << mask << ", cached=" << src.flags << ")" << dendl;
+    if(perfcounter) perfcounter->inc(l_rgw_cache_miss);
+    return -ENOENT;
+  }
+  ldout(cct, 10) << "cache get: name=" << name << " : hit" << dendl;
+
+  info = src;
+  if(perfcounter) perfcounter->inc(l_rgw_cache_hit);
+
+  return 0;
+}
+
+void ObjectCache::put(string& name, ObjectCacheInfo& info)
+{
+  RWLock::WLocker l(lock);
+
+  ldout(cct, 10) << "cache put: name=" << name << dendl;
+  map::iterator iter = cache_map.find(name);
+  if (iter == cache_map.end()) {
+    ObjectCacheEntry entry;
+    entry.lru_iter = lru.end();
+    cache_map.insert(pair(name, entry));
+    iter = cache_map.find(name);
+  }
+  ObjectCacheEntry& entry = iter->second;
+  ObjectCacheInfo& target = entry.info;
+
+  touch_lru(name, entry, entry.lru_iter);
+
+  target.status = info.status;
+
+  if (info.status < 0) {
+    target.flags = 0;
+    target.xattrs.clear();
+    target.data.clear();
+    return;
+  }
+
+  target.flags |= info.flags;
+
+  if (info.flags & CACHE_FLAG_META)
+    target.meta = info.meta;
+  else if (!(info.flags & CACHE_FLAG_MODIFY_XATTRS))
+    target.flags &= ~CACHE_FLAG_META; // non-meta change should reset meta
+
+  if (info.flags & CACHE_FLAG_XATTRS) {
+    target.xattrs = info.xattrs;
+    map::iterator iter;
+    for (iter = target.xattrs.begin(); iter != target.xattrs.end(); ++iter) {
+      ldout(cct, 10) << "updating xattr: name=" << iter->first << " bl.length()=" << iter->second.length() << dendl;
+    }
+  } else if (info.flags & CACHE_FLAG_MODIFY_XATTRS) {
+    map::iterator iter;
+    for (iter = info.rm_xattrs.begin(); iter != info.rm_xattrs.end(); ++iter) {
+      ldout(cct, 10) << "removing xattr: name=" << iter->first << dendl;
+      target.xattrs.erase(iter->first);
+    }
+    for (iter = info.xattrs.begin(); iter != info.xattrs.end(); ++iter) {
+      ldout(cct, 10) << "appending xattr: name=" << iter->first << " bl.length()=" << iter->second.length() << dendl;
+      target.xattrs[iter->first] = iter->second;
+    }
+  }
+
+  if (info.flags & CACHE_FLAG_DATA)
+    target.data = info.data;
+
+  if (info.flags & CACHE_FLAG_OBJV)
+    target.version = info.version;
+}
+
+void ObjectCache::remove(string& name)
+{
+  RWLock::WLocker l(lock);
+
+  map::iterator iter = cache_map.find(name);
+  if (iter == cache_map.end())
+    return;
+
+  ldout(cct, 10) << "removing " << name << " from cache" << dendl;
+
+  remove_lru(name, iter->second.lru_iter);
+  cache_map.erase(iter);
+}
+
+void ObjectCache::touch_lru(string& name, ObjectCacheEntry& entry, std::list::iterator& lru_iter)
+{
+  while (lru_size > (size_t)cct->_conf->rgw_cache_lru_size) {
+    list::iterator iter = lru.begin();
+    if ((*iter).compare(name) == 0) {
+      /*
+       * if the entry we're touching happens to be at the lru end, don't remove it,
+       * lru shrinking can wait for next time
+       */
+      break;
+    }
+    map::iterator map_iter = cache_map.find(*iter);
+    ldout(cct, 10) << "removing entry: name=" << *iter << " from cache LRU" << dendl;
+    if (map_iter != cache_map.end())
+      cache_map.erase(map_iter);
+    lru.pop_front();
+    lru_size--;
+  }
+
+  if (lru_iter == lru.end()) {
+    lru.push_back(name);
+    lru_size++;
+    lru_iter--;
+    ldout(cct, 10) << "adding " << name << " to cache LRU end" << dendl;
+  } else {
+    ldout(cct, 10) << "moving " << name << " to cache LRU end" << dendl;
+    lru.erase(lru_iter);
+    lru.push_back(name);
+    lru_iter = lru.end();
+    --lru_iter;
+  }
+
+  lru_counter++;
+  entry.lru_promotion_ts = lru_counter;
+}
+
+void ObjectCache::remove_lru(string& name, std::list::iterator& lru_iter)
+{
+  if (lru_iter == lru.end())
+    return;
+
+  lru.erase(lru_iter);
+  lru_size--;
+  lru_iter = lru.end();
+}
+
+
diff --git a/ceph/src/rgw/rgw_cache.h b/ceph/src/rgw/rgw_cache.h
new file mode 100644
index 00000000..3b793f14
--- /dev/null
+++ b/ceph/src/rgw/rgw_cache.h
@@ -0,0 +1,580 @@
+#ifndef CEPH_RGWCACHE_H
+#define CEPH_RGWCACHE_H
+
+#include "rgw_rados.h"
+#include 
+#include 
+#include "include/types.h"
+#include "include/utime.h"
+#include "include/assert.h"
+#include "common/RWLock.h"
+
+enum {
+  UPDATE_OBJ,
+  REMOVE_OBJ,
+};
+
+#define CACHE_FLAG_DATA           0x01
+#define CACHE_FLAG_XATTRS         0x02
+#define CACHE_FLAG_META           0x04
+#define CACHE_FLAG_MODIFY_XATTRS  0x08
+#define CACHE_FLAG_OBJV           0x10
+
+#define mydout(v) lsubdout(T::cct, rgw, v)
+
+struct ObjectMetaInfo {
+  uint64_t size;
+  time_t mtime;
+
+  ObjectMetaInfo() : size(0), mtime(0) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(size, bl);
+    utime_t t(mtime, 0);
+    ::encode(t, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+    ::decode(size, bl);
+    utime_t t;
+    ::decode(t, bl);
+    mtime = t.sec();
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ObjectMetaInfo)
+
+struct ObjectCacheInfo {
+  int status;
+  uint32_t flags;
+  uint64_t epoch;
+  bufferlist data;
+  map xattrs;
+  map rm_xattrs;
+  ObjectMetaInfo meta;
+  obj_version version;
+
+  ObjectCacheInfo() : status(0), flags(0), epoch(0), version() {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(5, 3, bl);
+    ::encode(status, bl);
+    ::encode(flags, bl);
+    ::encode(data, bl);
+    ::encode(xattrs, bl);
+    ::encode(meta, bl);
+    ::encode(rm_xattrs, bl);
+    ::encode(epoch, bl);
+    ::encode(version, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
+    ::decode(status, bl);
+    ::decode(flags, bl);
+    ::decode(data, bl);
+    ::decode(xattrs, bl);
+    ::decode(meta, bl);
+    if (struct_v >= 2)
+      ::decode(rm_xattrs, bl);
+    if (struct_v >= 4)
+      ::decode(epoch, bl);
+    if (struct_v >= 5)
+      ::decode(version, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(ObjectCacheInfo)
+
+struct RGWCacheNotifyInfo {
+  uint32_t op;
+  rgw_obj obj;
+  ObjectCacheInfo obj_info;
+  off_t ofs;
+  string ns;
+
+  RGWCacheNotifyInfo() : op(0), ofs(0) {}
+
+  void encode(bufferlist& obl) const {
+    ENCODE_START(2, 2, obl);
+    ::encode(op, obl);
+    ::encode(obj, obl);
+    ::encode(obj_info, obl);
+    ::encode(ofs, obl);
+    ::encode(ns, obl);
+    ENCODE_FINISH(obl);
+  }
+  void decode(bufferlist::iterator& ibl) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, ibl);
+    ::decode(op, ibl);
+    ::decode(obj, ibl);
+    ::decode(obj_info, ibl);
+    ::decode(ofs, ibl);
+    ::decode(ns, ibl);
+    DECODE_FINISH(ibl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(RGWCacheNotifyInfo)
+
+struct ObjectCacheEntry {
+  ObjectCacheInfo info;
+  std::list::iterator lru_iter;
+  uint64_t lru_promotion_ts;
+
+  ObjectCacheEntry() : lru_promotion_ts(0) {}
+};
+
+class ObjectCache {
+  std::map cache_map;
+  std::list lru;
+  unsigned long lru_size;
+  unsigned long lru_counter;
+  unsigned long lru_window;
+  RWLock lock;
+  CephContext *cct;
+
+  void touch_lru(string& name, ObjectCacheEntry& entry, std::list::iterator& lru_iter);
+  void remove_lru(string& name, std::list::iterator& lru_iter);
+public:
+  ObjectCache() : lru_size(0), lru_counter(0), lru_window(0), lock("ObjectCache"), cct(NULL) { }
+  int get(std::string& name, ObjectCacheInfo& bl, uint32_t mask);
+  void put(std::string& name, ObjectCacheInfo& bl);
+  void remove(std::string& name);
+  void set_ctx(CephContext *_cct) {
+    cct = _cct;
+    lru_window = cct->_conf->rgw_cache_lru_size / 2;
+  }
+};
+
+template 
+class RGWCache  : public T
+{
+  ObjectCache cache;
+
+  int list_objects_raw_init(rgw_bucket& bucket, RGWAccessHandle *handle) {
+    return T::list_objects_raw_init(bucket, handle);
+  }
+  int list_objects_raw_next(RGWObjEnt& obj, RGWAccessHandle *handle) {
+    return T::list_objects_raw_next(obj, handle);
+  }
+
+  string normal_name(rgw_bucket& bucket, std::string& oid) {
+    string& bucket_name = bucket.name;
+    char buf[bucket_name.size() + 1 + oid.size() + 1];
+    const char *bucket_str = bucket_name.c_str();
+    const char *oid_str = oid.c_str();
+    sprintf(buf, "%s+%s", bucket_str, oid_str);
+    return string(buf);
+  }
+
+  void normalize_bucket_and_obj(rgw_bucket& src_bucket, string& src_obj, rgw_bucket& dst_bucket, string& dst_obj);
+  string normal_name(rgw_obj& obj) {
+    return normal_name(obj.bucket, obj.object);
+  }
+
+  int init_rados() {
+    int ret;
+    cache.set_ctx(T::cct);
+    ret = T::init_rados();
+    if (ret < 0)
+      return ret;
+
+    return 0;
+  }
+
+  bool need_watch_notify() {
+    return true;
+  }
+
+  int distribute_cache(const string& normal_name, rgw_obj& obj, ObjectCacheInfo& obj_info, int op);
+  int watch_cb(int opcode, uint64_t ver, bufferlist& bl);
+public:
+  RGWCache() {}
+
+  int set_attr(void *ctx, rgw_obj& obj, const char *name, bufferlist& bl, RGWObjVersionTracker *objv_tracker);
+  int set_attrs(void *ctx, rgw_obj& obj, 
+                map& attrs,
+                map* rmattrs,
+                RGWObjVersionTracker *objv_tracker);
+  int put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size, time_t *mtime,
+                   map& attrs, RGWObjCategory category, int flags,
+                   map* rmattrs, const bufferlist *data,
+                   RGWObjManifest *manifest, const string *ptag, list *remove_objs,
+                   bool modify_version, RGWObjVersionTracker *objv_tracker, time_t set_mtime,
+                   const string& owner);
+  int put_obj_data(void *ctx, rgw_obj& obj, const char *data,
+              off_t ofs, size_t len, bool exclusive);
+
+  int get_obj(void *ctx, RGWObjVersionTracker *objv_tracker, void **handle, rgw_obj& obj, bufferlist& bl, off_t ofs, off_t end);
+
+  int obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime, uint64_t *epoch, map *attrs,
+               bufferlist *first_chunk, RGWObjVersionTracker *objv_tracker);
+
+  int delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker);
+};
+
+template 
+void RGWCache::normalize_bucket_and_obj(rgw_bucket& src_bucket, string& src_obj, rgw_bucket& dst_bucket, string& dst_obj)
+{
+  if (src_obj.size()) {
+    dst_bucket = src_bucket;
+    dst_obj = src_obj;
+  } else {
+    dst_bucket = T::zone.domain_root;
+    dst_obj = src_bucket.name;
+  }
+}
+
+template 
+int RGWCache::delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+{
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
+  if (bucket.name[0] != '.')
+    return T::delete_obj_impl(ctx, bucket_owner, obj, objv_tracker);
+
+  string name = normal_name(obj);
+  cache.remove(name);
+
+  ObjectCacheInfo info;
+  distribute_cache(name, obj, info, REMOVE_OBJ);
+
+  return T::delete_obj_impl(ctx, bucket_owner, obj, objv_tracker);
+}
+
+template 
+int RGWCache::get_obj(void *ctx, RGWObjVersionTracker *objv_tracker, void **handle, rgw_obj& obj, bufferlist& obl, off_t ofs, off_t end)
+{
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
+  if (bucket.name[0] != '.' || ofs != 0)
+    return T::get_obj(ctx, objv_tracker, handle, obj, obl, ofs, end);
+
+  string name = normal_name(obj.bucket, oid);
+
+  ObjectCacheInfo info;
+
+  uint32_t flags = CACHE_FLAG_DATA;
+  if (objv_tracker)
+    flags |= CACHE_FLAG_OBJV;
+  
+  if (cache.get(name, info, flags) == 0) {
+    if (info.status < 0)
+      return info.status;
+
+    bufferlist& bl = info.data;
+
+    bufferlist::iterator i = bl.begin();
+
+    obl.clear();
+
+    i.copy_all(obl);
+    if (objv_tracker)
+      objv_tracker->read_version = info.version;
+    return bl.length();
+  }
+  int r = T::get_obj(ctx, objv_tracker, handle, obj, obl, ofs, end);
+  if (r < 0) {
+    if (r == -ENOENT) { // only update ENOENT, we'd rather retry other errors
+      info.status = r;
+      cache.put(name, info);
+    }
+    return r;
+  }
+
+  if (obl.length() == end + 1) {
+    /* in this case, most likely object contains more data, we can't cache it */
+    return r;
+  }
+
+  bufferptr p(r);
+  bufferlist& bl = info.data;
+  bl.clear();
+  bufferlist::iterator o = obl.begin();
+  o.copy_all(bl);
+  info.status = 0;
+  info.flags = flags;
+  if (objv_tracker) {
+    info.version = objv_tracker->read_version;
+  }
+  cache.put(name, info);
+  return r;
+}
+
+template 
+int RGWCache::set_attr(void *ctx, rgw_obj& obj, const char *attr_name, bufferlist& bl, RGWObjVersionTracker *objv_tracker)
+{
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
+  ObjectCacheInfo info;
+  bool cacheable = false;
+  if (bucket.name[0] == '.') {
+    cacheable = true;
+    info.xattrs[attr_name] = bl;
+    info.status = 0;
+    info.flags = CACHE_FLAG_MODIFY_XATTRS;
+    if (objv_tracker) {
+      info.version = objv_tracker->write_version;
+      info.flags |= CACHE_FLAG_OBJV;
+    }
+  }
+  int ret = T::set_attr(ctx, obj, attr_name, bl, objv_tracker);
+  if (cacheable) {
+    string name = normal_name(bucket, oid);
+    if (ret >= 0) {
+      cache.put(name, info);
+      int r = distribute_cache(name, obj, info, UPDATE_OBJ);
+      if (r < 0)
+        mydout(0) << "ERROR: failed to distribute cache for " << obj << dendl;
+    } else {
+     cache.remove(name);
+    }
+  }
+
+  return ret;
+}
+
+template 
+int RGWCache::set_attrs(void *ctx, rgw_obj& obj, 
+                           map& attrs,
+                           map* rmattrs,
+                           RGWObjVersionTracker *objv_tracker) 
+{
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
+  ObjectCacheInfo info;
+  bool cacheable = false;
+  if (bucket.name[0] == '.') {
+    cacheable = true;
+    info.xattrs = attrs;
+    if (rmattrs)
+      info.rm_xattrs = *rmattrs;
+    info.status = 0;
+    info.flags = CACHE_FLAG_MODIFY_XATTRS;
+    if (objv_tracker) {
+      info.version = objv_tracker->write_version;
+      info.flags |= CACHE_FLAG_OBJV;
+    }
+  }
+  int ret = T::set_attrs(ctx, obj, attrs, rmattrs, objv_tracker);
+  if (cacheable) {
+    string name = normal_name(bucket, oid);
+    if (ret >= 0) {
+      cache.put(name, info);
+      int r = distribute_cache(name, obj, info, UPDATE_OBJ);
+      if (r < 0)
+        mydout(0) << "ERROR: failed to distribute cache for " << obj << dendl;
+    } else {
+     cache.remove(name);
+    }
+  }
+
+  return ret;
+}
+
+template 
+int RGWCache::put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size, time_t *mtime,
+                              map& attrs, RGWObjCategory category, int flags,
+                              map* rmattrs, const bufferlist *data,
+                              RGWObjManifest *manifest, const string *ptag, list *remove_objs,
+                              bool modify_version, RGWObjVersionTracker *objv_tracker, time_t set_mtime,
+                              const string& owner)
+{
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
+  ObjectCacheInfo info;
+  bool cacheable = false;
+  if (bucket.name[0] == '.') {
+    cacheable = true;
+    info.xattrs = attrs;
+    info.status = 0;
+    info.flags = CACHE_FLAG_XATTRS;
+    if (data) {
+      info.data = *data;
+      info.flags |= CACHE_FLAG_DATA;
+    }
+    if (objv_tracker) {
+      info.version = objv_tracker->write_version;
+      info.flags |= CACHE_FLAG_OBJV;
+    }
+  }
+  int ret = T::put_obj_meta_impl(ctx, obj, size, mtime, attrs, category, flags, rmattrs, data, manifest, ptag, remove_objs,
+                                 modify_version, objv_tracker, set_mtime, owner);
+  if (cacheable) {
+    string name = normal_name(bucket, oid);
+    if (ret >= 0) {
+      cache.put(name, info);
+      int r = distribute_cache(name, obj, info, UPDATE_OBJ);
+      if (r < 0)
+        mydout(0) << "ERROR: failed to distribute cache for " << obj << dendl;
+    } else {
+     cache.remove(name);
+    }
+  }
+
+  return ret;
+}
+
+template 
+int RGWCache::put_obj_data(void *ctx, rgw_obj& obj, const char *data,
+              off_t ofs, size_t len, bool exclusive)
+{
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
+  ObjectCacheInfo info;
+  bool cacheable = false;
+  if ((bucket.name[0] == '.') && ((ofs == 0) || (ofs == -1))) {
+    cacheable = true;
+    bufferptr p(len);
+    memcpy(p.c_str(), data, len);
+    bufferlist& bl = info.data;
+    bl.append(p);
+    info.meta.size = bl.length();
+    info.status = 0;
+    info.flags = CACHE_FLAG_DATA;
+  }
+  int ret = T::put_obj_data(ctx, obj, data, ofs, len, exclusive);
+  if (cacheable) {
+    string name = normal_name(bucket, oid);
+    if (ret >= 0) {
+      cache.put(name, info);
+      int r = distribute_cache(name, obj, info, UPDATE_OBJ);
+      if (r < 0)
+        mydout(0) << "ERROR: failed to distribute cache for " << obj << dendl;
+    } else {
+     cache.remove(name);
+    }
+  }
+
+  return ret;
+}
+
+template 
+int RGWCache::obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime,
+                          uint64_t *pepoch, map *attrs,
+                          bufferlist *first_chunk, RGWObjVersionTracker *objv_tracker)
+{
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(obj.bucket, obj.object, bucket, oid);
+  if (bucket.name[0] != '.')
+    return T::obj_stat(ctx, obj, psize, pmtime, pepoch, attrs, first_chunk, objv_tracker);
+
+  string name = normal_name(bucket, oid);
+
+  uint64_t size;
+  time_t mtime;
+  uint64_t epoch;
+
+  ObjectCacheInfo info;
+  uint32_t flags = CACHE_FLAG_META | CACHE_FLAG_XATTRS;
+  if (objv_tracker)
+    flags |= CACHE_FLAG_OBJV;
+  int r = cache.get(name, info, flags);
+  if (r == 0) {
+    if (info.status < 0)
+      return info.status;
+
+    size = info.meta.size;
+    mtime = info.meta.mtime;
+    epoch = info.epoch;
+    if (objv_tracker)
+      objv_tracker->read_version = info.version;
+    goto done;
+  }
+  r = T::obj_stat(ctx, obj, &size, &mtime, &epoch, &info.xattrs, first_chunk, objv_tracker);
+  if (r < 0) {
+    if (r == -ENOENT) {
+      info.status = r;
+      cache.put(name, info);
+    }
+    return r;
+  }
+  info.status = 0;
+  info.epoch = epoch;
+  info.meta.mtime = mtime;
+  info.meta.size = size;
+  info.flags = CACHE_FLAG_META | CACHE_FLAG_XATTRS;
+  if (objv_tracker) {
+    info.flags |= CACHE_FLAG_OBJV;
+    info.version = objv_tracker->read_version;
+  }
+  cache.put(name, info);
+done:
+  if (psize)
+    *psize = size;
+  if (pmtime)
+    *pmtime = mtime;
+  if (pepoch)
+    *pepoch = epoch;
+  if (attrs)
+    *attrs = info.xattrs;
+  return 0;
+}
+
+template 
+int RGWCache::distribute_cache(const string& normal_name, rgw_obj& obj, ObjectCacheInfo& obj_info, int op)
+{
+  RGWCacheNotifyInfo info;
+
+  info.op = op;
+
+  info.obj_info = obj_info;
+  info.obj = obj;
+  bufferlist bl;
+  ::encode(info, bl);
+  int ret = T::distribute(normal_name, bl);
+  return ret;
+}
+
+template 
+int RGWCache::watch_cb(int opcode, uint64_t ver, bufferlist& bl)
+{
+  RGWCacheNotifyInfo info;
+
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(info, iter);
+  } catch (buffer::end_of_buffer& err) {
+    mydout(0) << "ERROR: got bad notification" << dendl;
+    return -EIO;
+  } catch (buffer::error& err) {
+    mydout(0) << "ERROR: buffer::error" << dendl;
+    return -EIO;
+  }
+
+  rgw_bucket bucket;
+  string oid;
+  normalize_bucket_and_obj(info.obj.bucket, info.obj.object, bucket, oid);
+  string name = normal_name(bucket, oid);
+  
+  switch (info.op) {
+  case UPDATE_OBJ:
+    cache.put(name, info.obj_info);
+    break;
+  case REMOVE_OBJ:
+    cache.remove(name);
+    break;
+  default:
+    mydout(0) << "WARNING: got unknown notification op: " << info.op << dendl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+#endif
diff --git a/ceph/src/rgw/rgw_civetweb.cc b/ceph/src/rgw/rgw_civetweb.cc
new file mode 100644
index 00000000..a31177fb
--- /dev/null
+++ b/ceph/src/rgw/rgw_civetweb.cc
@@ -0,0 +1,172 @@
+
+#include 
+
+#include "civetweb/civetweb.h"
+#include "rgw_civetweb.h"
+
+
+#define dout_subsys ceph_subsys_rgw
+
+int RGWMongoose::write_data(const char *buf, int len)
+{
+  if (!header_done) {
+    header_data.append(buf, len);
+    return 0;
+  }
+  if (!sent_header) {
+    data.append(buf, len);
+    return 0;
+  }
+  return mg_write(conn, buf, len);
+}
+
+RGWMongoose::RGWMongoose(mg_connection *_conn, int _port) : conn(_conn), port(_port), header_done(false), sent_header(false), has_content_length(false),
+                                                 explicit_keepalive(false)
+{
+}
+
+int RGWMongoose::read_data(char *buf, int len)
+{
+  return mg_read(conn, buf, len);
+}
+
+void RGWMongoose::flush()
+{
+}
+
+int RGWMongoose::complete_request()
+{
+  if (!sent_header) {
+    if (!has_content_length) {
+      header_done = false; /* let's go back to writing the header */
+
+      if (0 && data.length() == 0) {
+        has_content_length = true;
+        print("Transfer-Enconding: %s\n", "chunked");
+        data.append("0\r\n\r\n", sizeof("0\r\n\r\n")-1);
+      } else {
+        int r = send_content_length(data.length());
+        if (r < 0)
+	  return r;
+      }
+    }
+
+    complete_header();
+  }
+
+  if (data.length()) {
+    int r = write_data(data.c_str(), data.length());
+    if (r < 0)
+      return r;
+    data.clear();
+  }
+
+  return 0;
+}
+
+void RGWMongoose::init_env(CephContext *cct)
+{
+  env.init(cct);
+  struct mg_request_info *info = mg_get_request_info(conn);
+  if (!info)
+    return;
+
+  for (int i = 0; i < info->num_headers; i++) {
+    struct mg_request_info::mg_header *header = &info->http_headers[i];
+
+    if (strcasecmp(header->name, "content-length") == 0) {
+      env.set("CONTENT_LENGTH", header->value);
+      continue;
+    }
+
+    if (strcasecmp(header->name, "content-type") == 0) {
+      env.set("CONTENT_TYPE", header->value);
+      continue;
+    }
+
+    if (strcasecmp(header->name, "connection") == 0) {
+      explicit_keepalive = (strcasecmp(header->value, "keep-alive") == 0);
+    }
+
+    int len = strlen(header->name) + 5; /* HTTP_ prepended */
+    char buf[len + 1];
+    memcpy(buf, "HTTP_", 5);
+    const char *src = header->name;
+    char *dest = &buf[5];
+    for (; *src; src++, dest++) {
+      char c = *src;
+      switch (c) {
+       case '-':
+         c = '_';
+         break;
+       default:
+         c = toupper(c);
+         break;
+      }
+      *dest = c;
+    }
+    *dest = '\0';
+    
+    env.set(buf, header->value);
+  }
+
+  env.set("REQUEST_METHOD", info->request_method);
+  env.set("REQUEST_URI", info->uri);
+  env.set("QUERY_STRING", info->query_string);
+  env.set("REMOTE_USER", info->remote_user);
+  env.set("SCRIPT_URI", info->uri); /* FIXME */
+
+  char port_buf[16];
+  snprintf(port_buf, sizeof(port_buf), "%d", port);
+  env.set("SERVER_PORT", port_buf);
+}
+
+int RGWMongoose::send_status(const char *status, const char *status_name)
+{
+  char buf[128];
+
+  if (!status_name)
+    status_name = "";
+
+  snprintf(buf, sizeof(buf), "HTTP/1.1 %s %s\n", status, status_name);
+
+  bufferlist bl;
+  bl.append(buf);
+  bl.append(header_data);
+  header_data = bl;
+
+  return 0;
+}
+
+int RGWMongoose::send_100_continue()
+{
+  char buf[] = "HTTP/1.1 100 CONTINUE\r\n\r\n";
+
+  return mg_write(conn, buf, sizeof(buf) - 1);
+}
+
+int RGWMongoose::complete_header()
+{
+  header_done = true;
+
+  if (!has_content_length) {
+    return 0;
+  }
+
+  if (explicit_keepalive)
+    header_data.append("Connection: Keep-Alive\r\n");
+
+  header_data.append("\r\n");
+
+  sent_header = true;
+
+  return write_data(header_data.c_str(), header_data.length());
+}
+
+int RGWMongoose::send_content_length(uint64_t len)
+{
+  has_content_length = true;
+  char buf[21];
+  snprintf(buf, sizeof(buf), "%"PRIu64, len);
+  return print("Content-Length: %s\n", buf);
+}
diff --git a/ceph/src/rgw/rgw_civetweb.h b/ceph/src/rgw/rgw_civetweb.h
new file mode 100644
index 00000000..c1df9f08
--- /dev/null
+++ b/ceph/src/rgw/rgw_civetweb.h
@@ -0,0 +1,41 @@
+#ifndef CEPH_RGW_MONGOOSE_H
+#define CEPH_RGW_MONGOOSE_H
+
+#include "rgw_client_io.h"
+
+
+struct mg_connection;
+
+
+class RGWMongoose : public RGWClientIO
+{
+  mg_connection *conn;
+
+  bufferlist header_data;
+  bufferlist data;
+
+  int port;
+
+  bool header_done;
+  bool sent_header;
+  bool has_content_length;
+  bool explicit_keepalive;
+
+public:
+  void init_env(CephContext *cct);
+
+  int write_data(const char *buf, int len);
+  int read_data(char *buf, int len);
+
+  int send_status(const char *status, const char *status_name);
+  int send_100_continue();
+  int complete_header();
+  int complete_request();
+  int send_content_length(uint64_t len);
+
+  RGWMongoose(mg_connection *_conn, int _port);
+  void flush();
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_client_io.cc b/ceph/src/rgw/rgw_client_io.cc
new file mode 100644
index 00000000..193f44e9
--- /dev/null
+++ b/ceph/src/rgw/rgw_client_io.cc
@@ -0,0 +1,75 @@
+
+#include 
+#include 
+#include 
+
+#include "rgw_client_io.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+void RGWClientIO::init(CephContext *cct) {
+  init_env(cct);
+
+  if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
+    std::map& env_map = env.get_map();
+    std::map::iterator iter = env_map.begin();
+
+    for (iter = env_map.begin(); iter != env_map.end(); ++iter) {
+      ldout(cct, 20) << iter->first << "=" << iter->second << dendl;
+    }
+  }
+}
+
+
+int RGWClientIO::print(const char *format, ...)
+{
+#define LARGE_ENOUGH 128
+  int size = LARGE_ENOUGH;
+
+  va_list ap;
+
+  while(1) {
+    char buf[size];
+    va_start(ap, format);
+    int ret = vsnprintf(buf, size, format, ap);
+    va_end(ap);
+
+    if (ret >= 0 && ret < size) {
+      return write(buf, ret);
+    }
+
+    if (ret >= 0)
+      size = ret + 1;
+    else
+      size *= 2;
+  }
+
+  /* not reachable */
+}
+
+int RGWClientIO::write(const char *buf, int len)
+{
+  int ret = write_data(buf, len);
+  if (ret < 0)
+    return ret;
+
+  if (account)
+    bytes_sent += len;
+
+  return 0;
+}
+
+
+int RGWClientIO::read(char *buf, int max, int *actual)
+{
+  int ret = read_data(buf, max);
+  if (ret < 0)
+    return ret;
+
+  *actual = ret;
+
+  bytes_received += *actual;
+
+  return 0;
+}
+
diff --git a/ceph/src/rgw/rgw_client_io.h b/ceph/src/rgw/rgw_client_io.h
new file mode 100644
index 00000000..2e8720e5
--- /dev/null
+++ b/ceph/src/rgw/rgw_client_io.h
@@ -0,0 +1,50 @@
+#ifndef CEPH_RGW_CLIENT_IO_H
+#define CEPH_RGW_CLIENT_IO_H
+
+#include 
+
+#include "include/types.h"
+
+#include "rgw_common.h"
+
+class RGWClientIO {
+  bool account;
+
+  size_t bytes_sent;
+  size_t bytes_received;
+
+protected:
+  RGWEnv env;
+
+  virtual void init_env(CephContext *cct) = 0;
+
+  virtual int write_data(const char *buf, int len) = 0;
+  virtual int read_data(char *buf, int max) = 0;
+
+public:
+  virtual ~RGWClientIO() {}
+  RGWClientIO() : account(false), bytes_sent(0), bytes_received(0) {}
+
+  void init(CephContext *cct);
+  int print(const char *format, ...);
+  int write(const char *buf, int len);
+  virtual void flush() = 0;
+  int read(char *buf, int max, int *actual);
+
+  virtual int send_status(const char *status, const char *status_name) = 0;
+  virtual int send_100_continue() = 0;
+  virtual int complete_header() = 0;
+  virtual int complete_request() = 0;
+  virtual int send_content_length(uint64_t len) = 0;
+
+  RGWEnv& get_env() { return env; }
+
+  void set_account(bool _account) {
+    account = _account;
+  }
+
+  uint64_t get_bytes_sent() { return bytes_sent; }
+  uint64_t get_bytes_received() { return bytes_received; }
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_common.cc b/ceph/src/rgw/rgw_common.cc
new file mode 100644
index 00000000..5a1043f3
--- /dev/null
+++ b/ceph/src/rgw/rgw_common.cc
@@ -0,0 +1,1047 @@
+#include 
+
+#include "json_spirit/json_spirit.h"
+#include "common/ceph_json.h"
+
+#include "rgw_common.h"
+#include "rgw_acl.h"
+#include "rgw_string.h"
+
+#include "common/ceph_crypto.h"
+#include "common/armor.h"
+#include "common/errno.h"
+#include "common/Clock.h"
+#include "common/Formatter.h"
+#include "common/perf_counters.h"
+#include "common/strtol.h"
+#include "include/str_list.h"
+#include "auth/Crypto.h"
+
+#include 
+
+#define dout_subsys ceph_subsys_rgw
+
+PerfCounters *perfcounter = NULL;
+
+int rgw_perf_start(CephContext *cct)
+{
+  PerfCountersBuilder plb(cct, cct->_conf->name.to_str(), l_rgw_first, l_rgw_last);
+
+  plb.add_u64_counter(l_rgw_req, "req");
+  plb.add_u64_counter(l_rgw_failed_req, "failed_req");
+
+  plb.add_u64_counter(l_rgw_get, "get");
+  plb.add_u64_counter(l_rgw_get_b, "get_b");
+  plb.add_time_avg(l_rgw_get_lat, "get_initial_lat");
+  plb.add_u64_counter(l_rgw_put, "put");
+  plb.add_u64_counter(l_rgw_put_b, "put_b");
+  plb.add_time_avg(l_rgw_put_lat, "put_initial_lat");
+
+  plb.add_u64(l_rgw_qlen, "qlen");
+  plb.add_u64(l_rgw_qactive, "qactive");
+
+  plb.add_u64_counter(l_rgw_cache_hit, "cache_hit");
+  plb.add_u64_counter(l_rgw_cache_miss, "cache_miss");
+
+  plb.add_u64_counter(l_rgw_keystone_token_cache_hit, "keystone_token_cache_hit");
+  plb.add_u64_counter(l_rgw_keystone_token_cache_miss, "keystone_token_cache_miss");
+
+  perfcounter = plb.create_perf_counters();
+  cct->get_perfcounters_collection()->add(perfcounter);
+  return 0;
+}
+
+void rgw_perf_stop(CephContext *cct)
+{
+  assert(perfcounter);
+  cct->get_perfcounters_collection()->remove(perfcounter);
+  delete perfcounter;
+}
+
+using namespace ceph::crypto;
+
+rgw_err::
+rgw_err()
+{
+  clear();
+}
+
+rgw_err::
+rgw_err(int http, const std::string& s3)
+    : http_ret(http), ret(0), s3_code(s3)
+{
+}
+
+void rgw_err::
+clear()
+{
+  http_ret = 200;
+  ret = 0;
+  s3_code.clear();
+}
+
+bool rgw_err::
+is_clear() const
+{
+  return (http_ret == 200);
+}
+
+bool rgw_err::
+is_err() const
+{
+  return !(http_ret >= 200 && http_ret <= 399);
+}
+
+
+req_info::req_info(CephContext *cct, class RGWEnv *e) : env(e) {
+  method = env->get("REQUEST_METHOD");
+  script_uri = env->get("SCRIPT_URI", cct->_conf->rgw_script_uri.c_str());
+  request_uri = env->get("REQUEST_URI", cct->_conf->rgw_request_uri.c_str());
+  int pos = request_uri.find('?');
+  if (pos >= 0) {
+    request_params = request_uri.substr(pos + 1);
+    request_uri = request_uri.substr(0, pos);
+  } else {
+    request_params = env->get("QUERY_STRING", "");
+  }
+  host = env->get("HTTP_HOST");
+}
+
+void req_info::rebuild_from(req_info& src)
+{
+  method = src.method;
+  script_uri = src.script_uri;
+  if (src.effective_uri.empty()) {
+    request_uri = src.request_uri;
+  } else {
+    request_uri = src.effective_uri;
+  }
+  effective_uri.clear();
+  host = src.host;
+
+  x_meta_map = src.x_meta_map;
+  x_meta_map.erase("x-amz-date");
+}
+
+
+req_state::req_state(CephContext *_cct, class RGWEnv *e) : cct(_cct), cio(NULL), op(OP_UNKNOWN),
+							   has_acl_header(false),
+                                                           os_auth_token(NULL), info(_cct, e)
+{
+  enable_ops_log = e->conf->enable_ops_log;
+  enable_usage_log = e->conf->enable_usage_log;
+  defer_to_bucket_acls = e->conf->defer_to_bucket_acls;
+  content_started = false;
+  format = 0;
+  formatter = NULL;
+  bucket_acl = NULL;
+  object_acl = NULL;
+  expect_cont = false;
+
+  object = NULL;
+
+  header_ended = false;
+  obj_size = 0;
+  prot_flags = 0;
+
+  system_request = false;
+
+  os_auth_token = NULL;
+  time = ceph_clock_now(cct);
+  perm_mask = 0;
+  content_length = 0;
+  object = NULL;
+  bucket_exists = false;
+  has_bad_meta = false;
+  length = NULL;
+  copy_source = NULL;
+  http_auth = NULL;
+  local_source = false;
+
+  obj_ctx = NULL;
+}
+
+req_state::~req_state() {
+  delete formatter;
+  delete bucket_acl;
+  delete object_acl;
+  free((void *)object);
+}
+
+struct str_len {
+  const char *str;
+  int len;
+};
+
+#define STR_LEN_ENTRY(s) { s, sizeof(s) - 1 }
+
+struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ"),
+                                   STR_LEN_ENTRY("HTTP_X_GOOG"),
+                                   STR_LEN_ENTRY("HTTP_X_DHO"),
+                                   STR_LEN_ENTRY("HTTP_X_RGW"),
+                                   STR_LEN_ENTRY("HTTP_X_OBJECT"),
+                                   STR_LEN_ENTRY("HTTP_X_CONTAINER"),
+                                   {NULL, 0} };
+
+
+void req_info::init_meta_info(bool *found_bad_meta)
+{
+  x_meta_map.clear();
+
+  map& m = env->get_map();
+  map::iterator iter;
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    const char *prefix;
+    const string& header_name = iter->first;
+    const string& val = iter->second;
+    for (int prefix_num = 0; (prefix = meta_prefixes[prefix_num].str) != NULL; prefix_num++) {
+      int len = meta_prefixes[prefix_num].len;
+      const char *p = header_name.c_str();
+      if (strncmp(p, prefix, len) == 0) {
+        dout(10) << "meta>> " << p << dendl;
+        const char *name = p+len; /* skip the prefix */
+        int name_len = header_name.size() - len;
+
+        if (found_bad_meta && strncmp(name, "_META_", name_len) == 0)
+          *found_bad_meta = true;
+
+        char name_low[meta_prefixes[0].len + name_len + 1];
+        snprintf(name_low, meta_prefixes[0].len - 5 + name_len + 1, "%s%s", meta_prefixes[0].str + 5 /* skip HTTP_ */, name); // normalize meta prefix
+        int j;
+        for (j = 0; name_low[j]; j++) {
+          if (name_low[j] != '_')
+            name_low[j] = tolower(name_low[j]);
+          else
+            name_low[j] = '-';
+        }
+        name_low[j] = 0;
+
+        map::iterator iter;
+        iter = x_meta_map.find(name_low);
+        if (iter != x_meta_map.end()) {
+          string old = iter->second;
+          int pos = old.find_last_not_of(" \t"); /* get rid of any whitespaces after the value */
+          old = old.substr(0, pos + 1);
+          old.append(",");
+          old.append(val);
+          x_meta_map[name_low] = old;
+        } else {
+          x_meta_map[name_low] = val;
+        }
+      }
+    }
+  }
+  for (iter = x_meta_map.begin(); iter != x_meta_map.end(); ++iter) {
+    dout(10) << "x>> " << iter->first << ":" << iter->second << dendl;
+  }
+}
+
+std::ostream& operator<<(std::ostream& oss, const rgw_err &err)
+{
+  oss << "rgw_err(http_ret=" << err.http_ret << ", s3='" << err.s3_code << "') ";
+  return oss;
+}
+
+string rgw_string_unquote(const string& s)
+{
+  if (s[0] != '"' || s.size() < 2)
+    return s;
+
+  int len;
+  for (len = s.size(); len > 2; --len) {
+    if (s[len - 1] != ' ')
+      break;
+  }
+
+  if (s[len-1] != '"')
+    return s;
+
+  return s.substr(1, len - 2);
+}
+
+static void trim_whitespace(const string& src, string& dst)
+{
+  const char *spacestr = " \t\n\r\f\v";
+  int start = src.find_first_not_of(spacestr);
+  if (start < 0)
+    return;
+
+  int end = src.find_last_not_of(spacestr);
+  dst = src.substr(start, end - start + 1);
+}
+
+static bool check_str_end(const char *s)
+{
+  if (!s)
+    return false;
+
+  while (*s) {
+    if (!isspace(*s))
+      return false;
+    s++;
+  }
+  return true;
+}
+
+static bool check_gmt_end(const char *s)
+{
+  if (!s || !*s)
+    return false;
+
+  while (isspace(*s)) {
+    ++s;
+  }
+
+  /* check for correct timezone */
+  if ((strncmp(s, "GMT", 3) != 0) &&
+      (strncmp(s, "UTC", 3) != 0)) {
+    return false;
+  }
+
+  return true;
+}
+
+static bool parse_rfc850(const char *s, struct tm *t)
+{
+  memset(t, 0, sizeof(*t));
+  return check_gmt_end(strptime(s, "%A, %d-%b-%y %H:%M:%S ", t));
+}
+
+static bool parse_asctime(const char *s, struct tm *t)
+{
+  memset(t, 0, sizeof(*t));
+  return check_str_end(strptime(s, "%a %b %d %H:%M:%S %Y", t));
+}
+
+static bool parse_rfc1123(const char *s, struct tm *t)
+{
+  memset(t, 0, sizeof(*t));
+  return check_gmt_end(strptime(s, "%a, %d %b %Y %H:%M:%S ", t));
+}
+
+static bool parse_rfc1123_alt(const char *s, struct tm *t)
+{
+  memset(t, 0, sizeof(*t));
+  return check_str_end(strptime(s, "%a, %d %b %Y %H:%M:%S %z", t));
+}
+
+bool parse_rfc2616(const char *s, struct tm *t)
+{
+  return parse_rfc850(s, t) || parse_asctime(s, t) || parse_rfc1123(s, t) || parse_rfc1123_alt(s,t);
+}
+
+bool parse_iso8601(const char *s, struct tm *t)
+{
+  memset(t, 0, sizeof(*t));
+  const char *p = strptime(s, "%Y-%m-%dT%T", t);
+  if (!p) {
+    dout(0) << "parse_iso8601 failed" << dendl;
+    return false;
+  }
+  string str;
+  trim_whitespace(p, str);
+  if (str.size() == 1 && str[0] == 'Z')
+    return true;
+
+  if (str.size() != 5) {
+    return false;
+  }
+  if (str[0] != '.' ||
+      str[str.size() - 1] != 'Z')
+    return false;
+
+  uint32_t ms;
+  int r = stringtoul(str.substr(1, 3), &ms);
+  if (r < 0)
+    return false;
+
+  return true;
+}
+
+int parse_key_value(string& in_str, const char *delim, string& key, string& val)
+{
+  if (delim == NULL)
+    return -EINVAL;
+
+  int pos = in_str.find(delim);
+  if (pos < 0)
+    return -EINVAL;
+
+  trim_whitespace(in_str.substr(0, pos), key);
+  pos++;
+
+  trim_whitespace(in_str.substr(pos), val);
+
+  return 0;
+}
+
+int parse_key_value(string& in_str, string& key, string& val)
+{
+  return parse_key_value(in_str, "=", key,val);
+}
+
+int parse_time(const char *time_str, time_t *time)
+{
+  struct tm tm;
+
+  if (!parse_rfc2616(time_str, &tm))
+    return -EINVAL;
+
+  *time = timegm(&tm);
+
+  return 0;
+}
+
+/*
+ * calculate the sha1 value of a given msg and key
+ */
+void calc_hmac_sha1(const char *key, int key_len,
+                    const char *msg, int msg_len, char *dest)
+/* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
+{
+  HMACSHA1 hmac((const unsigned char *)key, key_len);
+  hmac.Update((const unsigned char *)msg, msg_len);
+  hmac.Final((unsigned char *)dest);
+  
+  char hex_str[(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2) + 1];
+  buf_to_hex((unsigned char *)dest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE, hex_str);
+}
+
+int gen_rand_base64(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
+{
+  char buf[size];
+  char tmp_dest[size + 4]; /* so that there's space for the extra '=' characters, and some */
+  int ret;
+
+  ret = get_random_bytes(buf, sizeof(buf));
+  if (ret < 0) {
+    lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
+    return -1;
+  }
+
+  ret = ceph_armor(tmp_dest, &tmp_dest[sizeof(tmp_dest)],
+		   (const char *)buf, ((const char *)buf) + ((size - 1) * 3 + 4 - 1) / 4);
+  if (ret < 0) {
+    lderr(cct) << "ceph_armor failed" << dendl;
+    return -1;
+  }
+  tmp_dest[ret] = '\0';
+  memcpy(dest, tmp_dest, size);
+  dest[size] = '\0';
+
+  return 0;
+}
+
+static const char alphanum_upper_table[]="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+int gen_rand_alphanumeric_upper(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
+{
+  int ret = get_random_bytes(dest, size);
+  if (ret < 0) {
+    lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
+    return -1;
+  }
+
+  int i;
+  for (i=0; i= 0) {
+      string& name = nv.get_name();
+      string& val = nv.get_val();
+
+      if (name.compare(0, sizeof(RGW_SYS_PARAM_PREFIX) - 1, RGW_SYS_PARAM_PREFIX) == 0) {
+        sys_val_map[name] = val;
+      } else {
+        val_map[name] = val;
+      }
+
+      if ((name.compare("acl") == 0) ||
+          (name.compare("cors") == 0) ||
+          (name.compare("location") == 0) ||
+          (name.compare("logging") == 0) ||
+          (name.compare("delete") == 0) ||
+          (name.compare("uploads") == 0) ||
+          (name.compare("partNumber") == 0) ||
+          (name.compare("uploadId") == 0) ||
+          (name.compare("versionId") == 0) ||
+          (name.compare("torrent") == 0)) {
+        sub_resources[name] = val;
+      } else if (name[0] == 'r') { // root of all evil
+        if ((name.compare("response-content-type") == 0) ||
+           (name.compare("response-content-language") == 0) ||
+           (name.compare("response-expires") == 0) ||
+           (name.compare("response-cache-control") == 0) ||
+           (name.compare("response-content-disposition") == 0) ||
+           (name.compare("response-content-encoding") == 0)) {
+          sub_resources[name] = val;
+          has_resp_modifier = true;
+        }
+      } else if  ((name.compare("subuser") == 0) ||
+          (name.compare("key") == 0) ||
+          (name.compare("caps") == 0) ||
+          (name.compare("index") == 0) ||
+          (name.compare("policy") == 0) ||
+          (name.compare("quota") == 0) ||
+          (name.compare("object") == 0)) {
+
+        if (!admin_subresource_added) {
+          sub_resources[name] = "";
+          admin_subresource_added = true;
+        }
+      }
+    }
+
+    pos = fpos + 1;  
+  }
+
+  return 0;
+}
+
+string& XMLArgs::get(const string& name, bool *exists)
+{
+  map::iterator iter;
+  iter = val_map.find(name);
+  bool e = (iter != val_map.end());
+  if (exists)
+    *exists = e;
+  if (e)
+    return iter->second;
+  return empty_str;
+}
+
+string& XMLArgs::get(const char *name, bool *exists)
+{
+  string s(name);
+  return get(s, exists);
+}
+
+
+int XMLArgs::get_bool(const string& name, bool *val, bool *exists)
+{
+  map::iterator iter;
+  iter = val_map.find(name);
+  bool e = (iter != val_map.end());
+  if (exists)
+    *exists = e;
+
+  if (e) {
+    const char *s = iter->second.c_str();
+
+    if (strcasecmp(s, "false") == 0) {
+      *val = false;
+    } else if (strcasecmp(s, "true") == 0) {
+      *val = true;
+    } else {
+      return -EINVAL;
+    }
+  }
+
+  return 0;
+}
+
+int XMLArgs::get_bool(const char *name, bool *val, bool *exists)
+{
+  string s(name);
+  return get_bool(s, val, exists);
+}
+
+bool verify_bucket_permission(struct req_state *s, int perm)
+{
+  if (!s->bucket_acl)
+    return false;
+
+  if ((perm & (int)s->perm_mask) != perm)
+    return false;
+
+  return s->bucket_acl->verify_permission(s->user.user_id, perm, perm);
+}
+
+static inline bool check_deferred_bucket_acl(struct req_state *s, uint8_t deferred_check, int perm)
+{
+  return (s->defer_to_bucket_acls == deferred_check && verify_bucket_permission(s, perm));
+}
+
+bool verify_object_permission(struct req_state *s, RGWAccessControlPolicy *bucket_acl, RGWAccessControlPolicy *object_acl, int perm)
+{
+  if (check_deferred_bucket_acl(s, RGW_DEFER_TO_BUCKET_ACLS_RECURSE, perm) ||
+      check_deferred_bucket_acl(s, RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL, RGW_PERM_FULL_CONTROL)) {
+    return true;
+  }
+
+  if (!object_acl)
+    return false;
+
+  bool ret = object_acl->verify_permission(s->user.user_id, s->perm_mask, perm);
+  if (ret)
+    return true;
+
+  if (!s->cct->_conf->rgw_enforce_swift_acls)
+    return ret;
+
+  if ((perm & (int)s->perm_mask) != perm)
+    return false;
+
+  int swift_perm = 0;
+  if (perm & (RGW_PERM_READ | RGW_PERM_READ_ACP))
+    swift_perm |= RGW_PERM_READ_OBJS;
+  if (perm & RGW_PERM_WRITE)
+    swift_perm |= RGW_PERM_WRITE_OBJS;
+
+  if (!swift_perm)
+    return false;
+  /* we already verified the user mask above, so we pass swift_perm as the mask here,
+     otherwise the mask might not cover the swift permissions bits */
+  return bucket_acl->verify_permission(s->user.user_id, swift_perm, swift_perm);
+}
+
+bool verify_object_permission(struct req_state *s, int perm)
+{
+  return verify_object_permission(s, s->bucket_acl, s->object_acl, perm);
+}
+
+class HexTable
+{
+  char table[256];
+
+public:
+  HexTable() {
+    memset(table, -1, sizeof(table));
+    int i;
+    for (i = '0'; i<='9'; i++)
+      table[i] = i - '0';
+    for (i = 'A'; i<='F'; i++)
+      table[i] = i - 'A' + 0xa;
+    for (i = 'a'; i<='f'; i++)
+      table[i] = i - 'a' + 0xa;
+  }
+
+  char to_num(char c) {
+    return table[(int)c];
+  }
+};
+
+static char hex_to_num(char c)
+{
+  static HexTable hex_table;
+  return hex_table.to_num(c);
+}
+
+bool url_decode(string& src_str, string& dest_str)
+{
+  const char *src = src_str.c_str();
+  char dest[src_str.size() + 1];
+  int pos = 0;
+  char c;
+
+  bool in_query = false;
+  while (*src) {
+    if (*src != '%') {
+      if (!in_query || *src != '+') {
+        if (*src == '?') in_query = true;
+        dest[pos++] = *src++;
+      } else {
+        dest[pos++] = ' ';
+        ++src;
+      }
+    } else {
+      src++;
+      if (!*src)
+        break;
+      char c1 = hex_to_num(*src++);
+      if (!*src)
+        break;
+      c = c1 << 4;
+      if (c1 < 0)
+        return false;
+      c1 = hex_to_num(*src++);
+      if (c1 < 0)
+        return false;
+      c |= c1;
+      dest[pos++] = c;
+    }
+  }
+  dest[pos] = 0;
+  dest_str = dest;
+
+  return true;
+}
+
+static void escape_char(char c, string& dst)
+{
+  char buf[16];
+  snprintf(buf, sizeof(buf), "%%%.2X", (int)(unsigned char)c);
+  dst.append(buf);
+}
+
+static bool char_needs_url_encoding(char c)
+{
+  if (c <= 0x20 || c >= 0x7f)
+    return true;
+
+  switch (c) {
+    case 0x22:
+    case 0x23:
+    case 0x25:
+    case 0x26:
+    case 0x2B:
+    case 0x2C:
+    case 0x2F:
+    case 0x3A:
+    case 0x3B:
+    case 0x3C:
+    case 0x3E:
+    case 0x3D:
+    case 0x3F:
+    case 0x40:
+    case 0x5B:
+    case 0x5D:
+    case 0x5C:
+    case 0x5E:
+    case 0x60:
+    case 0x7B:
+    case 0x7D:
+      return true;
+  }
+  return false;
+}
+
+void url_encode(const string& src, string& dst)
+{
+  const char *p = src.c_str();
+  for (unsigned i = 0; i < src.size(); i++, p++) {
+    if (char_needs_url_encoding(*p)) {
+      escape_char(*p, dst);
+      continue;
+    }
+
+    dst.append(p, 1);
+  }
+}
+
+string rgw_trim_whitespace(const string& src)
+{
+  if (src.empty()) {
+    return string();
+  }
+
+  int start = 0;
+  for (; start != (int)src.size(); start++) {
+    if (!isspace(src[start]))
+      break;
+  }
+
+  int end = src.size() - 1;
+  if (end < start) {
+    return string();
+  }
+
+  for (; end > start; end--) {
+    if (!isspace(src[end]))
+      break;
+  }
+
+  return src.substr(start, end - start + 1);
+}
+
+string rgw_trim_quotes(const string& val)
+{
+  string s = rgw_trim_whitespace(val);
+  if (s.size() < 2)
+    return s;
+
+  int start = 0;
+  int end = s.size() - 1;
+  int quotes_count = 0;
+
+  if (s[start] == '"') {
+    start++;
+    quotes_count++;
+  }
+  if (s[end] == '"') {
+    end--;
+    quotes_count++;
+  }
+  if (quotes_count == 2) {
+    return s.substr(start, end - start + 1);
+  }
+  return s;
+}
+
+struct rgw_name_to_flag {
+  const char *type_name;
+  uint32_t flag;
+};
+
+static int parse_list_of_flags(struct rgw_name_to_flag *mapping,
+                               const string& str, uint32_t *perm)
+{
+  list strs;
+  get_str_list(str, strs);
+  list::iterator iter;
+  uint32_t v = 0;
+  for (iter = strs.begin(); iter != strs.end(); ++iter) {
+    string& s = *iter;
+    for (int i = 0; mapping[i].type_name; i++) {
+      if (s.compare(mapping[i].type_name) == 0)
+        v |= mapping[i].flag;
+    }
+  }
+
+  *perm = v;
+  return 0;
+}
+
+static struct rgw_name_to_flag cap_names[] = { {"*",     RGW_CAP_ALL},
+                  {"read",  RGW_CAP_READ},
+		  {"write", RGW_CAP_WRITE},
+		  {NULL, 0} };
+
+int RGWUserCaps::parse_cap_perm(const string& str, uint32_t *perm)
+{
+  return parse_list_of_flags(cap_names, str, perm);
+}
+
+int RGWUserCaps::get_cap(const string& cap, string& type, uint32_t *pperm)
+{
+  int pos = cap.find('=');
+  if (pos >= 0) {
+    trim_whitespace(cap.substr(0, pos), type);
+  }
+
+  if (type.size() == 0)
+    return -EINVAL;
+
+  string cap_perm;
+  uint32_t perm = 0;
+  if (pos < (int)cap.size() - 1) {
+    cap_perm = cap.substr(pos + 1);
+    int r = RGWUserCaps::parse_cap_perm(cap_perm, &perm);
+    if (r < 0)
+      return r;
+  }
+
+  *pperm = perm;
+
+  return 0;
+}
+
+int RGWUserCaps::add_cap(const string& cap)
+{
+  uint32_t perm;
+  string type;
+
+  int r = get_cap(cap, type, &perm);
+  if (r < 0)
+    return r;
+
+  caps[type] |= perm;
+
+  return 0;
+}
+
+int RGWUserCaps::remove_cap(const string& cap)
+{
+  uint32_t perm;
+  string type;
+
+  int r = get_cap(cap, type, &perm);
+  if (r < 0)
+    return r;
+
+  map::iterator iter = caps.find(type);
+  if (iter == caps.end())
+    return 0;
+
+  uint32_t& old_perm = iter->second;
+  old_perm &= ~perm;
+  if (!old_perm)
+    caps.erase(iter);
+
+  return 0;
+}
+
+int RGWUserCaps::add_from_string(const string& str)
+{
+  int start = 0;
+  do {
+    int end = str.find(';', start);
+    if (end < 0)
+      end = str.size();
+
+    int r = add_cap(str.substr(start, end - start));
+    if (r < 0)
+      return r;
+
+    start = end + 1;
+  } while (start < (int)str.size());
+
+  return 0;
+}
+
+int RGWUserCaps::remove_from_string(const string& str)
+{
+  int start = 0;
+  do {
+    int end = str.find(';', start);
+    if (end < 0)
+      end = str.size();
+
+    int r = remove_cap(str.substr(start, end - start));
+    if (r < 0)
+      return r;
+
+    start = end + 1;
+  } while (start < (int)str.size());
+
+  return 0;
+}
+
+void RGWUserCaps::dump(Formatter *f) const
+{
+  dump(f, "caps");
+}
+
+void RGWUserCaps::dump(Formatter *f, const char *name) const
+{
+  f->open_array_section(name);
+  map::const_iterator iter;
+  for (iter = caps.begin(); iter != caps.end(); ++iter)
+  {
+    f->open_object_section("cap");
+    f->dump_string("type", iter->first);
+    uint32_t perm = iter->second;
+    string perm_str;
+    for (int i=0; cap_names[i].type_name; i++) {
+      if ((perm & cap_names[i].flag) == cap_names[i].flag) {
+	if (perm_str.size())
+	  perm_str.append(", ");
+
+	perm_str.append(cap_names[i].type_name);
+	perm &= ~cap_names[i].flag;
+      }
+    }
+    if (perm_str.empty())
+      perm_str = "";
+
+    f->dump_string("perm", perm_str);
+    f->close_section();
+  }
+
+  f->close_section();
+}
+
+struct RGWUserCap {
+  string type;
+  uint32_t perm;
+
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("type", type, obj);
+    string perm_str;
+    JSONDecoder::decode_json("perm", perm_str, obj);
+    if (RGWUserCaps::parse_cap_perm(perm_str, &perm) < 0) {
+      throw JSONDecoder::err("failed to parse permissions");
+    }
+  }
+};
+
+void RGWUserCaps::decode_json(JSONObj *obj)
+{
+  list caps_list;
+  decode_json_obj(caps_list, obj);
+
+  list::iterator iter;
+  for (iter = caps_list.begin(); iter != caps_list.end(); ++iter) {
+    RGWUserCap& cap = *iter;
+    caps[cap.type] = cap.perm;
+  }
+}
+
+int RGWUserCaps::check_cap(const string& cap, uint32_t perm)
+{
+  map::iterator iter = caps.find(cap);
+
+  if ((iter == caps.end()) ||
+      (iter->second & perm) != perm) {
+    return -EPERM;
+  }
+
+  return 0;
+}
+
+
+static struct rgw_name_to_flag op_type_mapping[] = { {"*",  RGW_OP_TYPE_ALL},
+                  {"read",  RGW_OP_TYPE_READ},
+		  {"write", RGW_OP_TYPE_WRITE},
+		  {"delete", RGW_OP_TYPE_DELETE},
+		  {NULL, 0} };
+
+
+int rgw_parse_op_type_list(const string& str, uint32_t *perm)
+{
+  return parse_list_of_flags(op_type_mapping, str, perm);
+}
+
diff --git a/ceph/src/rgw/rgw_common.h b/ceph/src/rgw/rgw_common.h
new file mode 100644
index 00000000..4d7a118f
--- /dev/null
+++ b/ceph/src/rgw/rgw_common.h
@@ -0,0 +1,1355 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2009 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_RGW_COMMON_H
+#define CEPH_RGW_COMMON_H
+
+#include "common/ceph_crypto.h"
+#include "common/debug.h"
+#include "common/perf_counters.h"
+
+#include "acconfig.h"
+
+#include 
+#include 
+#include 
+#include 
+#include "include/types.h"
+#include "include/utime.h"
+#include "rgw_acl.h"
+#include "rgw_cors.h"
+#include "rgw_quota.h"
+#include "rgw_string.h"
+#include "cls/version/cls_version_types.h"
+#include "cls/user/cls_user_types.h"
+#include "include/rados/librados.hpp"
+
+using namespace std;
+
+namespace ceph {
+  class Formatter;
+}
+
+using ceph::crypto::MD5;
+
+
+#define RGW_ATTR_PREFIX  "user.rgw."
+
+#define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_"
+#define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-"
+
+#define RGW_AMZ_META_PREFIX "x-amz-meta-"
+
+#define RGW_SYS_PARAM_PREFIX "rgwx-"
+
+#define RGW_ATTR_ACL		RGW_ATTR_PREFIX "acl"
+#define RGW_ATTR_CORS		RGW_ATTR_PREFIX "cors"
+#define RGW_ATTR_ETAG    	RGW_ATTR_PREFIX "etag"
+#define RGW_ATTR_BUCKETS	RGW_ATTR_PREFIX "buckets"
+#define RGW_ATTR_META_PREFIX	RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
+#define RGW_ATTR_CONTENT_TYPE	RGW_ATTR_PREFIX "content_type"
+#define RGW_ATTR_CACHE_CONTROL	RGW_ATTR_PREFIX "cache_control"
+#define RGW_ATTR_CONTENT_DISP	RGW_ATTR_PREFIX "content_disposition"
+#define RGW_ATTR_CONTENT_ENC	RGW_ATTR_PREFIX "content_encoding"
+#define RGW_ATTR_CONTENT_LANG	RGW_ATTR_PREFIX "content_language"
+#define RGW_ATTR_EXPIRES	RGW_ATTR_PREFIX "expires"
+#define RGW_ATTR_ID_TAG    	RGW_ATTR_PREFIX "idtag"
+#define RGW_ATTR_SHADOW_OBJ    	RGW_ATTR_PREFIX "shadow_name"
+#define RGW_ATTR_MANIFEST    	RGW_ATTR_PREFIX "manifest"
+#define RGW_ATTR_USER_MANIFEST  RGW_ATTR_PREFIX "user_manifest"
+
+#define RGW_BUCKETS_OBJ_SUFFIX ".buckets"
+
+#define RGW_MAX_PENDING_CHUNKS  16
+#define RGW_MAX_PUT_SIZE        (5ULL*1024*1024*1024)
+#define RGW_MIN_MULTIPART_SIZE (5ULL*1024*1024)
+
+#define RGW_FORMAT_PLAIN        0
+#define RGW_FORMAT_XML          1
+#define RGW_FORMAT_JSON         2
+
+#define RGW_CAP_READ            0x1
+#define RGW_CAP_WRITE           0x2
+#define RGW_CAP_ALL             (RGW_CAP_READ | RGW_CAP_WRITE)
+
+#define RGW_REST_SWIFT          0x1
+#define RGW_REST_SWIFT_AUTH     0x2
+
+#define RGW_SUSPENDED_USER_AUID (uint64_t)-2
+
+#define RGW_OP_TYPE_READ         0x01
+#define RGW_OP_TYPE_WRITE        0x02
+#define RGW_OP_TYPE_DELETE       0x04
+
+#define RGW_OP_TYPE_MODIFY       (RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
+#define RGW_OP_TYPE_ALL          (RGW_OP_TYPE_READ | RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
+
+#define RGW_DEFAULT_MAX_BUCKETS 1000
+
+#define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1
+#define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2
+
+#define STATUS_CREATED           1900
+#define STATUS_ACCEPTED          1901
+#define STATUS_NO_CONTENT        1902
+#define STATUS_PARTIAL_CONTENT   1903
+#define STATUS_REDIRECT          1904
+#define STATUS_NO_APPLY          1905
+#define STATUS_APPLIED           1906
+
+#define ERR_INVALID_BUCKET_NAME  2000
+#define ERR_INVALID_OBJECT_NAME  2001
+#define ERR_NO_SUCH_BUCKET       2002
+#define ERR_METHOD_NOT_ALLOWED   2003
+#define ERR_INVALID_DIGEST       2004
+#define ERR_BAD_DIGEST           2005
+#define ERR_UNRESOLVABLE_EMAIL   2006
+#define ERR_INVALID_PART         2007
+#define ERR_INVALID_PART_ORDER   2008
+#define ERR_NO_SUCH_UPLOAD       2009
+#define ERR_REQUEST_TIMEOUT      2010
+#define ERR_LENGTH_REQUIRED      2011
+#define ERR_REQUEST_TIME_SKEWED  2012
+#define ERR_BUCKET_EXISTS        2013
+#define ERR_BAD_URL              2014
+#define ERR_PRECONDITION_FAILED  2015
+#define ERR_NOT_MODIFIED         2016
+#define ERR_INVALID_UTF8         2017
+#define ERR_UNPROCESSABLE_ENTITY 2018
+#define ERR_TOO_LARGE            2019
+#define ERR_TOO_MANY_BUCKETS     2020
+#define ERR_INVALID_REQUEST      2021
+#define ERR_TOO_SMALL            2022
+#define ERR_NOT_FOUND            2023
+#define ERR_PERMANENT_REDIRECT   2024
+#define ERR_LOCKED               2025
+#define ERR_QUOTA_EXCEEDED       2026
+#define ERR_USER_SUSPENDED       2100
+#define ERR_INTERNAL_ERROR       2200
+
+typedef void *RGWAccessHandle;
+
+
+/* perf counter */
+
+extern PerfCounters *perfcounter;
+
+extern int rgw_perf_start(CephContext *cct);
+extern void rgw_perf_stop(CephContext *cct);
+
+enum {
+  l_rgw_first = 15000,
+  l_rgw_req,
+  l_rgw_failed_req,
+
+  l_rgw_get,
+  l_rgw_get_b,
+  l_rgw_get_lat,
+
+  l_rgw_put,
+  l_rgw_put_b,
+  l_rgw_put_lat,
+
+  l_rgw_qlen,
+  l_rgw_qactive,
+
+  l_rgw_cache_hit,
+  l_rgw_cache_miss,
+
+  l_rgw_keystone_token_cache_hit,
+  l_rgw_keystone_token_cache_miss,
+
+  l_rgw_last,
+};
+
+
+ /* size should be the required string size + 1 */
+extern int gen_rand_base64(CephContext *cct, char *dest, int size);
+extern int gen_rand_alphanumeric(CephContext *cct, char *dest, int size);
+extern int gen_rand_alphanumeric_upper(CephContext *cct, char *dest, int size);
+
+enum RGWIntentEvent {
+  DEL_OBJ = 0,
+  DEL_DIR = 1,
+};
+
+enum RGWObjCategory {
+  RGW_OBJ_CATEGORY_NONE      = 0,
+  RGW_OBJ_CATEGORY_MAIN      = 1,
+  RGW_OBJ_CATEGORY_SHADOW    = 2,
+  RGW_OBJ_CATEGORY_MULTIMETA = 3,
+};
+
+/** Store error returns for output at a different point in the program */
+struct rgw_err {
+  rgw_err();
+  rgw_err(int http, const std::string &s3);
+  void clear();
+  bool is_clear() const;
+  bool is_err() const;
+  friend std::ostream& operator<<(std::ostream& oss, const rgw_err &err);
+
+  int http_ret;
+  int ret;
+  std::string s3_code;
+  std::string message;
+};
+
+/* Helper class used for XMLArgs parsing */
+class NameVal
+{
+   string str;
+   string name;
+   string val;
+ public:
+    NameVal(string nv) : str(nv) {}
+
+    int parse();
+
+    string& get_name() { return name; }
+    string& get_val() { return val; }
+};
+
+/** Stores the XML arguments associated with the HTTP request in req_state*/
+class XMLArgs
+{
+  string str, empty_str;
+  map val_map;
+  map sys_val_map;
+  map sub_resources;
+
+  bool has_resp_modifier;
+ public:
+  XMLArgs() : has_resp_modifier(false) {}
+  /** Set the arguments; as received */
+  void set(string s) {
+    has_resp_modifier = false;
+    val_map.clear();
+    sub_resources.clear();
+    str = s;
+  }
+  /** parse the received arguments */
+  int parse();
+  /** Get the value for a specific argument parameter */
+  string& get(const string& name, bool *exists = NULL);
+  string& get(const char *name, bool *exists = NULL);
+  int get_bool(const string& name, bool *val, bool *exists);
+  int get_bool(const char *name, bool *val, bool *exists);
+
+  /** see if a parameter is contained in this XMLArgs */
+  bool exists(const char *name) {
+    map::iterator iter = val_map.find(name);
+    return (iter != val_map.end());
+  }
+  bool sub_resource_exists(const char *name) {
+    map::iterator iter = sub_resources.find(name);
+    return (iter != sub_resources.end());
+  }
+  map& get_params() {
+    return val_map;
+  }
+  map& get_sub_resources() { return sub_resources; }
+  unsigned get_num_params() {
+    return val_map.size();
+  }
+  bool has_response_modifier() {
+    return has_resp_modifier;
+  }
+  void set_system() { /* make all system params visible */
+    map::iterator iter;
+    for (iter = sys_val_map.begin(); iter != sys_val_map.end(); ++iter) {
+      val_map[iter->first] = iter->second;
+    }
+  }
+};
+
+class RGWConf;
+
+class RGWEnv {
+  std::map env_map;
+public:
+  RGWConf *conf; 
+
+  RGWEnv();
+  ~RGWEnv();
+  void init(CephContext *cct);
+  void init(CephContext *cct, char **envp);
+  void set(const char *name, const char *val);
+  const char *get(const char *name, const char *def_val = NULL);
+  int get_int(const char *name, int def_val = 0);
+  bool get_bool(const char *name, bool def_val = 0);
+  size_t get_size(const char *name, size_t def_val = 0);
+  bool exists(const char *name);
+  bool exists_prefix(const char *prefix);
+
+  void remove(const char *name);
+
+  std::map& get_map() { return env_map; }
+};
+
+class RGWConf {
+  friend class RGWEnv;
+protected:
+  void init(CephContext *cct, RGWEnv * env);
+public:
+  RGWConf() :
+    enable_ops_log(1), enable_usage_log(1), defer_to_bucket_acls(0) {}
+
+  int enable_ops_log;
+  int enable_usage_log;
+  uint8_t defer_to_bucket_acls;
+};
+
+enum http_op {
+  OP_GET,
+  OP_PUT,
+  OP_DELETE,
+  OP_HEAD,
+  OP_POST,
+  OP_COPY,
+  OP_OPTIONS,
+  OP_UNKNOWN,
+};
+
+class RGWAccessControlPolicy;
+class JSONObj;
+
+struct RGWAccessKey {
+  string id; // AccessKey
+  string key; // SecretKey
+  string subuser;
+
+  RGWAccessKey() {}
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(id, bl);
+    ::encode(key, bl);
+    ::encode(subuser, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
+     ::decode(id, bl);
+     ::decode(key, bl);
+     ::decode(subuser, bl);
+     DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  void dump_plain(Formatter *f) const;
+  void dump(Formatter *f, const string& user, bool swift) const;
+  static void generate_test_instances(list& o);
+
+  void decode_json(JSONObj *obj);
+  void decode_json(JSONObj *obj, bool swift);
+};
+WRITE_CLASS_ENCODER(RGWAccessKey);
+
+struct RGWSubUser {
+  string name;
+  uint32_t perm_mask;
+
+  RGWSubUser() : perm_mask(0) {}
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(name, bl);
+    ::encode(perm_mask, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
+     ::decode(name, bl);
+     ::decode(perm_mask, bl);
+     DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  void dump(Formatter *f, const string& user) const;
+  static void generate_test_instances(list& o);
+
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWSubUser);
+
+class RGWUserCaps
+{
+  map caps;
+
+  int get_cap(const string& cap, string& type, uint32_t *perm);
+  int add_cap(const string& cap);
+  int remove_cap(const string& cap);
+public:
+  static int parse_cap_perm(const string& str, uint32_t *perm);
+  int add_from_string(const string& str);
+  int remove_from_string(const string& str);
+
+  void encode(bufferlist& bl) const {
+     ENCODE_START(1, 1, bl);
+     ::encode(caps, bl);
+     ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START(1, bl);
+     ::decode(caps, bl);
+     DECODE_FINISH(bl);
+  }
+  int check_cap(const string& cap, uint32_t perm);
+  void dump(Formatter *f) const;
+  void dump(Formatter *f, const char *name) const;
+
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWUserCaps);
+
+void encode_json(const char *name, const obj_version& v, Formatter *f);
+void encode_json(const char *name, const RGWUserCaps& val, Formatter *f);
+
+void decode_json_obj(obj_version& v, JSONObj *obj);
+
+struct RGWUserInfo
+{
+  uint64_t auid;
+  string user_id;
+  string display_name;
+  string user_email;
+  map access_keys;
+  map swift_keys;
+  map subusers;
+  __u8 suspended;
+  uint32_t max_buckets;
+  uint32_t op_mask;
+  RGWUserCaps caps;
+  __u8 system;
+  string default_placement;
+  list placement_tags;
+  RGWQuotaInfo bucket_quota;
+  map temp_url_keys;
+  RGWQuotaInfo user_quota;
+
+  RGWUserInfo() : auid(0), suspended(0), max_buckets(RGW_DEFAULT_MAX_BUCKETS), op_mask(RGW_OP_TYPE_ALL), system(0) {}
+
+  void encode(bufferlist& bl) const {
+     ENCODE_START(16, 9, bl);
+     ::encode(auid, bl);
+     string access_key;
+     string secret_key;
+     if (!access_keys.empty()) {
+       map::const_iterator iter = access_keys.begin();
+       const RGWAccessKey& k = iter->second;
+       access_key = k.id;
+       secret_key = k.key;
+     }
+     ::encode(access_key, bl);
+     ::encode(secret_key, bl);
+     ::encode(display_name, bl);
+     ::encode(user_email, bl);
+     string swift_name;
+     string swift_key;
+     if (!swift_keys.empty()) {
+       map::const_iterator iter = swift_keys.begin();
+       const RGWAccessKey& k = iter->second;
+       swift_name = k.id;
+       swift_key = k.key;
+     }
+     ::encode(swift_name, bl);
+     ::encode(swift_key, bl);
+     ::encode(user_id, bl);
+     ::encode(access_keys, bl);
+     ::encode(subusers, bl);
+     ::encode(suspended, bl);
+     ::encode(swift_keys, bl);
+     ::encode(max_buckets, bl);
+     ::encode(caps, bl);
+     ::encode(op_mask, bl);
+     ::encode(system, bl);
+     ::encode(default_placement, bl);
+     ::encode(placement_tags, bl);
+     ::encode(bucket_quota, bl);
+     ::encode(temp_url_keys, bl);
+     ::encode(user_quota, bl);
+     ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START_LEGACY_COMPAT_LEN_32(16, 9, 9, bl);
+     if (struct_v >= 2) ::decode(auid, bl);
+     else auid = CEPH_AUTH_UID_DEFAULT;
+     string access_key;
+     string secret_key;
+    ::decode(access_key, bl);
+    ::decode(secret_key, bl);
+    if (struct_v < 6) {
+      RGWAccessKey k;
+      k.id = access_key;
+      k.key = secret_key;
+      access_keys[access_key] = k;
+    }
+    ::decode(display_name, bl);
+    ::decode(user_email, bl);
+    string swift_name;
+    string swift_key;
+    if (struct_v >= 3) ::decode(swift_name, bl);
+    if (struct_v >= 4) ::decode(swift_key, bl);
+    if (struct_v >= 5)
+      ::decode(user_id, bl);
+    else
+      user_id = access_key;
+    if (struct_v >= 6) {
+      ::decode(access_keys, bl);
+      ::decode(subusers, bl);
+    }
+    suspended = 0;
+    if (struct_v >= 7) {
+      ::decode(suspended, bl);
+    }
+    if (struct_v >= 8) {
+      ::decode(swift_keys, bl);
+    }
+    if (struct_v >= 10) {
+      ::decode(max_buckets, bl);
+    } else {
+      max_buckets = RGW_DEFAULT_MAX_BUCKETS;
+    }
+    if (struct_v >= 11) {
+      ::decode(caps, bl);
+    }
+    if (struct_v >= 12) {
+      ::decode(op_mask, bl);
+    } else {
+      op_mask = RGW_OP_TYPE_ALL;
+    }
+    system = 0;
+    if (struct_v >= 13) {
+      ::decode(system, bl);
+      ::decode(default_placement, bl);
+      ::decode(placement_tags, bl); /* tags of allowed placement rules */
+    }
+    if (struct_v >= 14) {
+      ::decode(bucket_quota, bl);
+    }
+    if (struct_v >= 15) {
+     ::decode(temp_url_keys, bl);
+    }
+    if (struct_v >= 16) {
+      ::decode(user_quota, bl);
+    }
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  void decode_json(JSONObj *obj);
+
+  void clear() {
+    user_id.clear();
+    display_name.clear();
+    user_email.clear();
+    auid = CEPH_AUTH_UID_DEFAULT;
+    access_keys.clear();
+    suspended = 0;
+  }
+};
+WRITE_CLASS_ENCODER(RGWUserInfo)
+
+struct rgw_bucket {
+  std::string name;
+  std::string data_pool;
+  std::string data_extra_pool; /* if not set, then we should use data_pool instead */
+  std::string index_pool;
+  std::string marker;
+  std::string bucket_id;
+
+  std::string oid; /*
+                    * runtime in-memory only info. If not empty, points to the bucket instance object
+                    */
+
+  rgw_bucket() { }
+  rgw_bucket(const cls_user_bucket& b) {
+    name = b.name;
+    data_pool = b.data_pool;
+    data_extra_pool = b.data_extra_pool;
+    index_pool = b.index_pool;
+    marker = b.marker;
+    bucket_id = b.bucket_id;
+  }
+  rgw_bucket(const char *n) : name(n) {
+    assert(*n == '.'); // only rgw private buckets should be initialized without pool
+    data_pool = index_pool = n;
+    marker = "";
+  }
+  rgw_bucket(const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) :
+    name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {}
+
+  void convert(cls_user_bucket *b) {
+    b->name = name;
+    b->data_pool = data_pool;
+    b->data_extra_pool = data_extra_pool;
+    b->index_pool = index_pool;
+    b->marker = marker;
+    b->bucket_id = bucket_id;
+  }
+
+  void encode(bufferlist& bl) const {
+     ENCODE_START(7, 3, bl);
+    ::encode(name, bl);
+    ::encode(data_pool, bl);
+    ::encode(marker, bl);
+    ::encode(bucket_id, bl);
+    ::encode(index_pool, bl);
+    ::encode(data_extra_pool, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(7, 3, 3, bl);
+    ::decode(name, bl);
+    ::decode(data_pool, bl);
+    if (struct_v >= 2) {
+      ::decode(marker, bl);
+      if (struct_v <= 3) {
+        uint64_t id;
+        ::decode(id, bl);
+        char buf[16];
+        snprintf(buf, sizeof(buf), "%llu", (long long)id);
+        bucket_id = buf;
+      } else {
+        ::decode(bucket_id, bl);
+      }
+    }
+    if (struct_v >= 5) {
+      ::decode(index_pool, bl);
+    } else {
+      index_pool = data_pool;
+    }
+    if (struct_v >= 7) {
+      ::decode(data_extra_pool, bl);
+    }
+    DECODE_FINISH(bl);
+  }
+
+  const string& get_data_extra_pool() {
+    if (data_extra_pool.empty()) {
+      return data_pool;
+    }
+    return data_extra_pool;
+  }
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+  static void generate_test_instances(list& o);
+
+  bool operator<(const rgw_bucket& b) const {
+    return name.compare(b.name) < 0;
+  }
+};
+WRITE_CLASS_ENCODER(rgw_bucket)
+
+inline ostream& operator<<(ostream& out, const rgw_bucket &b) {
+  out << b.name;
+  if (b.name.compare(b.data_pool)) {
+    out << "(@";
+    string s;
+    if (!b.index_pool.empty() && b.data_pool.compare(b.index_pool))
+      s = "i=" + b.index_pool;
+    if (!b.data_extra_pool.empty() && b.data_pool.compare(b.data_extra_pool)) {
+      if (!s.empty()) {
+        s += ",";
+      }
+      s += "e=" + b.data_extra_pool;
+    }
+    if (!s.empty()) {
+      out << "{"  << s << "}";
+    }
+
+    out << b.data_pool << "[" << b.marker << "])";
+  }
+  return out;
+}
+
+struct RGWObjVersionTracker {
+  obj_version read_version;
+  obj_version write_version;
+
+  obj_version *version_for_read() {
+    return &read_version;
+  }
+
+  obj_version *version_for_write() {
+    if (write_version.ver == 0)
+      return NULL;
+
+    return &write_version;
+  }
+
+  obj_version *version_for_check() {
+    if (read_version.ver == 0)
+      return NULL;
+
+    return &read_version;
+  }
+
+  void prepare_op_for_read(librados::ObjectReadOperation *op);
+  void prepare_op_for_write(librados::ObjectWriteOperation *op);
+
+  void apply_write() {
+    read_version = write_version;
+    write_version = obj_version();
+  }
+
+  void clear() {
+    read_version = obj_version();
+    write_version = obj_version();
+  }
+
+  void generate_new_write_ver(CephContext *cct);
+};
+
+enum RGWBucketFlags {
+  BUCKET_SUSPENDED = 0x1,
+};
+
+struct RGWBucketInfo
+{
+  rgw_bucket bucket;
+  string owner;
+  uint32_t flags;
+  string region;
+  time_t creation_time;
+  string placement_rule;
+  bool has_instance_obj;
+  RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */
+  obj_version ep_objv; /* entry point object version, for runtime tracking only */
+  RGWQuotaInfo quota;
+
+  void encode(bufferlist& bl) const {
+     ENCODE_START(9, 4, bl);
+     ::encode(bucket, bl);
+     ::encode(owner, bl);
+     ::encode(flags, bl);
+     ::encode(region, bl);
+     uint64_t ct = (uint64_t)creation_time;
+     ::encode(ct, bl);
+     ::encode(placement_rule, bl);
+     ::encode(has_instance_obj, bl);
+     ::encode(quota, bl);
+     ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN_32(6, 4, 4, bl);
+     ::decode(bucket, bl);
+     if (struct_v >= 2)
+       ::decode(owner, bl);
+     if (struct_v >= 3)
+       ::decode(flags, bl);
+     if (struct_v >= 5)
+       ::decode(region, bl);
+     if (struct_v >= 6) {
+       uint64_t ct;
+       ::decode(ct, bl);
+       creation_time = (time_t)ct;
+     }
+     if (struct_v >= 7)
+       ::decode(placement_rule, bl);
+     if (struct_v >= 8)
+       ::decode(has_instance_obj, bl);
+     if (struct_v >= 9)
+       ::decode(quota, bl);
+     DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  void decode_json(JSONObj *obj);
+
+  RGWBucketInfo() : flags(0), creation_time(0), has_instance_obj(false) {}
+};
+WRITE_CLASS_ENCODER(RGWBucketInfo)
+
+struct RGWBucketEntryPoint
+{
+  rgw_bucket bucket;
+  string owner;
+  time_t creation_time;
+  bool linked;
+
+  bool has_bucket_info;
+  RGWBucketInfo old_bucket_info;
+
+  RGWBucketEntryPoint() : creation_time(0), linked(false), has_bucket_info(false) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(8, 8, bl);
+    ::encode(bucket, bl);
+    ::encode(owner, bl);
+    ::encode(linked, bl);
+    uint64_t ctime = (uint64_t)creation_time;
+    ::encode(ctime, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    bufferlist::iterator orig_iter = bl;
+    DECODE_START_LEGACY_COMPAT_LEN_32(8, 4, 4, bl);
+    if (struct_v < 8) {
+      /* ouch, old entry, contains the bucket info itself */
+      old_bucket_info.decode(orig_iter);
+      has_bucket_info = true;
+      return;
+    }
+    has_bucket_info = false;
+    ::decode(bucket, bl);
+    ::decode(owner, bl);
+    ::decode(linked, bl);
+    uint64_t ctime;
+    ::decode(ctime, bl);
+    creation_time = (uint64_t)ctime;
+    DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWBucketEntryPoint)
+
+struct RGWStorageStats
+{
+  RGWObjCategory category;
+  uint64_t num_kb;
+  uint64_t num_kb_rounded;
+  uint64_t num_objects;
+
+  RGWStorageStats() : category(RGW_OBJ_CATEGORY_NONE), num_kb(0), num_kb_rounded(0), num_objects(0) {}
+};
+
+struct req_state;
+
+class RGWEnv;
+
+class RGWClientIO;
+
+struct req_info {
+  RGWEnv *env;
+  XMLArgs args;
+  map x_meta_map;
+
+  const char *host;
+  const char *method;
+  string script_uri;
+  string request_uri;
+  string effective_uri;
+  string request_params;
+
+  req_info(CephContext *cct, RGWEnv *_env);
+  void rebuild_from(req_info& src);
+  void init_meta_info(bool *found_nad_meta);
+};
+
+/** Store all the state necessary to complete and respond to an HTTP request*/
+struct req_state {
+   CephContext *cct;
+   RGWClientIO *cio;
+   http_op op;
+   bool content_started;
+   int format;
+   ceph::Formatter *formatter;
+   string decoded_uri;
+   string relative_uri;
+   const char *length;
+   uint64_t content_length;
+   map generic_attrs;
+   struct rgw_err err;
+   bool expect_cont;
+   bool header_ended;
+   uint64_t obj_size;
+   bool enable_ops_log;
+   bool enable_usage_log;
+   uint8_t defer_to_bucket_acls;
+   uint32_t perm_mask;
+   utime_t header_time;
+
+   const char *object;
+
+   rgw_bucket bucket;
+   string bucket_name_str;
+   string object_str;
+   string src_bucket_name;
+   string src_object;
+   ACLOwner bucket_owner;
+   ACLOwner owner;
+
+   string bucket_instance_id;
+
+   RGWBucketInfo bucket_info;
+   map bucket_attrs;
+   bool bucket_exists;
+
+   bool has_bad_meta;
+
+   RGWUserInfo user; 
+   RGWAccessControlPolicy *bucket_acl;
+   RGWAccessControlPolicy *object_acl;
+
+   bool system_request;
+
+   string canned_acl;
+   bool has_acl_header;
+   const char *copy_source;
+   const char *http_auth;
+   bool local_source; /* source is local */
+
+   int prot_flags;
+
+   const char *os_auth_token;
+   string swift_user;
+   string swift_groups;
+
+   utime_t time;
+
+   void *obj_ctx;
+
+   string dialect;
+
+   string req_id;
+
+   req_info info;
+
+   req_state(CephContext *_cct, class RGWEnv *e);
+   ~req_state();
+};
+
+/** Store basic data on an object */
+struct RGWObjEnt {
+  std::string name;
+  std::string ns;
+  std::string owner;
+  std::string owner_display_name;
+  uint64_t size;
+  utime_t mtime;
+  string etag;
+  string content_type;
+  string tag;
+
+  RGWObjEnt() : size(0) {}
+
+  void dump(Formatter *f) const;
+};
+
+/** Store basic data on bucket */
+struct RGWBucketEnt {
+  rgw_bucket bucket;
+  size_t size;
+  size_t size_rounded;
+  time_t creation_time;
+  uint64_t count;
+
+  RGWBucketEnt() : size(0), size_rounded(0), creation_time(0), count(0) {}
+
+  RGWBucketEnt(const cls_user_bucket_entry& e) {
+    bucket = e.bucket;
+    size = e.size;
+    size_rounded = e.size_rounded;
+    creation_time = e.creation_time;
+    count = e.count;
+  }
+
+  void convert(cls_user_bucket_entry *b) {
+    bucket.convert(&b->bucket);
+    b->size = size;
+    b->size_rounded = size_rounded;
+    b->creation_time = creation_time;
+    b->count = count;
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(5, 5, bl);
+    uint64_t s = size;
+    __u32 mt = creation_time;
+    string empty_str;  // originally had the bucket name here, but we encode bucket later
+    ::encode(empty_str, bl);
+    ::encode(s, bl);
+    ::encode(mt, bl);
+    ::encode(count, bl);
+    ::encode(bucket, bl);
+    s = size_rounded;
+    ::encode(s, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(5, 5, 5, bl);
+    __u32 mt;
+    uint64_t s;
+    string empty_str;  // backward compatibility
+    ::decode(empty_str, bl);
+    ::decode(s, bl);
+    ::decode(mt, bl);
+    size = s;
+    creation_time = mt;
+    if (struct_v >= 2)
+      ::decode(count, bl);
+    if (struct_v >= 3)
+      ::decode(bucket, bl);
+    if (struct_v >= 4)
+      ::decode(s, bl);
+    size_rounded = s;
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(RGWBucketEnt)
+
+class rgw_obj {
+  std::string orig_obj;
+  std::string orig_key;
+public:
+  rgw_bucket bucket;
+  std::string key;
+  std::string ns;
+  std::string object;
+
+  bool in_extra_data; /* in-memory only member, does not serialize */
+
+  rgw_obj() : in_extra_data(false) {}
+  rgw_obj(const char *b, const char *o) : in_extra_data(false) {
+    rgw_bucket _b(b);
+    std::string _o(o);
+    init(_b, _o);
+  }
+  rgw_obj(rgw_bucket& b, const char *o) : in_extra_data(false) {
+    std::string _o(o);
+    init(b, _o);
+  }
+  rgw_obj(rgw_bucket& b, const std::string& o) : in_extra_data(false) {
+    init(b, o);
+  }
+  rgw_obj(rgw_bucket& b, const std::string& o, const std::string& k) : in_extra_data(false) {
+    init(b, o, k);
+  }
+  rgw_obj(rgw_bucket& b, const std::string& o, const std::string& k, const std::string& n) : in_extra_data(false) {
+    init(b, o, k, n);
+  }
+  void init(rgw_bucket& b, const std::string& o, const std::string& k, const std::string& n) {
+    bucket = b;
+    set_ns(n);
+    set_obj(o);
+    set_key(k);
+  }
+  void init(rgw_bucket& b, const std::string& o, const std::string& k) {
+    bucket = b;
+    set_obj(o);
+    set_key(k);
+  }
+  void init(rgw_bucket& b, const std::string& o) {
+    bucket = b;
+    set_obj(o);
+    orig_key = key = o;
+  }
+  void init_ns(rgw_bucket& b, const std::string& o, const std::string& n) {
+    bucket = b;
+    set_ns(n);
+    set_obj(o);
+    reset_key();
+  }
+  int set_ns(const char *n) {
+    if (!n)
+      return -EINVAL;
+    string ns_str(n);
+    return set_ns(ns_str);
+  }
+  int set_ns(const string& n) {
+    if (n[0] == '_')
+      return -EINVAL;
+    ns = n;
+    set_obj(orig_obj);
+    return 0;
+  }
+
+  void set_key(const string& k) {
+    orig_key = k;
+    key = k;
+  }
+
+  void reset_key() {
+    orig_key.clear();
+    key.clear();
+  }
+
+  void set_obj(const string& o) {
+    orig_obj = o;
+    if (ns.empty()) {
+      if (o.empty())
+        return;
+      if (o.size() < 1 || o[0] != '_') {
+        object = o;
+        return;
+      }
+      object = "_";
+      object.append(o);
+    } else {
+      object = "_";
+      object.append(ns);
+      object.append("_");
+      object.append(o);
+    }
+    if (orig_key.size())
+      set_key(orig_key);
+    else
+      set_key(orig_obj);
+  }
+
+  string loc() {
+    if (orig_key.empty())
+      return orig_obj;
+    else
+      return orig_key;
+  }
+
+  /**
+   * Translate a namespace-mangled object name to the user-facing name
+   * existing in the given namespace.
+   *
+   * If the object is part of the given namespace, it returns true
+   * and cuts down the name to the unmangled version. If it is not
+   * part of the given namespace, it returns false.
+   */
+  static bool translate_raw_obj_to_obj_in_ns(string& obj, string& ns) {
+    if (ns.empty()) {
+      if (obj[0] != '_')
+        return true;
+
+      if (obj.size() >= 2 && obj[1] == '_') {
+        obj = obj.substr(1);
+        return true;
+      }
+
+      return false;
+    }
+
+    if (obj[0] != '_' || obj.size() < 3) // for namespace, min size would be 3: _x_
+      return false;
+
+    int pos = obj.find('_', 1);
+    if (pos <= 1) // if it starts with __, it's not in our namespace
+      return false;
+
+    string obj_ns = obj.substr(1, pos - 1);
+    if (obj_ns.compare(ns) != 0)
+        return false;
+
+    obj = obj.substr(pos + 1);
+    return true;
+  }
+
+  /**
+   * Given a mangled object name and an empty namespace string, this
+   * function extracts the namespace into the string and sets the object
+   * name to be the unmangled version.
+   *
+   * It returns true after successfully doing so, or
+   * false if it fails.
+   */
+  static bool strip_namespace_from_object(string& obj, string& ns) {
+    ns.clear();
+    if (obj[0] != '_') {
+      return true;
+    }
+
+    size_t pos = obj.find('_', 1);
+    if (pos == string::npos) {
+      return false;
+    }
+
+    size_t period_pos = obj.find('.');
+    if (period_pos < pos) {
+      return false;
+    }
+
+    ns = obj.substr(1, pos-1);
+    obj = obj.substr(pos+1, string::npos);
+    return true;
+  }
+
+  void set_in_extra_data(bool val) {
+    in_extra_data = val;
+  }
+
+  bool is_in_extra_data() const {
+    return in_extra_data;
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(3, 3, bl);
+    ::encode(bucket.name, bl);
+    ::encode(key, bl);
+    ::encode(ns, bl);
+    ::encode(object, bl);
+    ::encode(bucket, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+    ::decode(bucket.name, bl);
+    ::decode(key, bl);
+    ::decode(ns, bl);
+    ::decode(object, bl);
+    if (struct_v >= 2)
+      ::decode(bucket, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  bool operator==(const rgw_obj& o) const {
+    return (object.compare(o.object) == 0) &&
+           (bucket.name.compare(o.bucket.name) == 0) &&
+           (ns.compare(o.ns) == 0);
+  }
+  bool operator<(const rgw_obj& o) const {
+    int r = bucket.name.compare(o.bucket.name);
+    if (r == 0) {
+     r = object.compare(o.object);
+     if (r == 0) {
+       r = ns.compare(o.ns);
+     }
+    }
+
+    return (r < 0);
+  }
+};
+WRITE_CLASS_ENCODER(rgw_obj)
+
+inline ostream& operator<<(ostream& out, const rgw_obj &o) {
+  return out << o.bucket.name << ":" << o.object;
+}
+
+static inline bool str_startswith(const string& str, const string& prefix)
+{
+  return (str.compare(0, prefix.size(), prefix) == 0);
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+static inline int hexdigit(char c)
+{
+  if (c >= '0' && c <= '9')
+    return (c - '0');
+  c = toupper(c);
+  if (c >= 'A' && c <= 'F')
+    return c - 'A' + 0xa;
+  return -EINVAL;
+}
+
+static inline int hex_to_buf(const char *hex, char *buf, int len)
+{
+  int i = 0;
+  const char *p = hex;
+  while (*p) {
+    if (i >= len)
+      return -EINVAL;
+    buf[i] = 0;
+    int d = hexdigit(*p);
+    if (d < 0)
+      return d;
+    buf[i] = d << 4;
+    p++;
+    if (!*p)
+      return -EINVAL;
+    d = hexdigit(*p);
+    if (d < 0)
+      return -d;
+    buf[i] += d;
+    i++;
+    p++;
+  }
+  return i;
+}
+
+static inline int rgw_str_to_bool(const char *s, int def_val)
+{
+  if (!s)
+    return def_val;
+
+  return (strcasecmp(s, "on") == 0 ||
+          strcasecmp(s, "yes") == 0 ||
+          strcasecmp(s, "1") == 0);
+}
+
+static inline void append_rand_alpha(CephContext *cct, string& src, string& dest, int len)
+{
+  dest = src;
+  char buf[len + 1];
+  gen_rand_alphanumeric(cct, buf, len);
+  dest.append("_");
+  dest.append(buf);
+}
+
+static inline const char *rgw_obj_category_name(RGWObjCategory category)
+{
+  switch (category) {
+  case RGW_OBJ_CATEGORY_NONE:
+    return "rgw.none";
+  case RGW_OBJ_CATEGORY_MAIN:
+    return "rgw.main";
+  case RGW_OBJ_CATEGORY_SHADOW:
+    return "rgw.shadow";
+  case RGW_OBJ_CATEGORY_MULTIMETA:
+    return "rgw.multimeta";
+  }
+
+  return "unknown";
+}
+
+static inline uint64_t rgw_rounded_kb(uint64_t bytes)
+{
+  return (bytes + 1023) / 1024;
+}
+
+static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
+{
+  return ((bytes + 4095) & ~4095) / 1024;
+}
+
+extern string rgw_string_unquote(const string& s);
+extern void parse_csv_string(const string& ival, vector& ovals);
+extern int parse_key_value(string& in_str, string& key, string& val);
+extern int parse_key_value(string& in_str, const char *delim, string& key, string& val);
+/** time parsing */
+extern int parse_time(const char *time_str, time_t *time);
+extern bool parse_rfc2616(const char *s, struct tm *t);
+extern bool parse_iso8601(const char *s, struct tm *t);
+extern string rgw_trim_whitespace(const string& src);
+extern string rgw_trim_quotes(const string& val);
+
+
+/** Check if the req_state's user has the necessary permissions
+ * to do the requested action */
+extern bool verify_bucket_permission(struct req_state *s, int perm);
+extern bool verify_object_permission(struct req_state *s, RGWAccessControlPolicy *bucket_acl, RGWAccessControlPolicy *object_acl, int perm);
+extern bool verify_object_permission(struct req_state *s, int perm);
+/** Convert an input URL into a sane object name
+ * by converting %-escaped strings into characters, etc*/
+extern bool url_decode(string& src_str, string& dest_str);
+extern void url_encode(const string& src, string& dst);
+
+extern void calc_hmac_sha1(const char *key, int key_len,
+                          const char *msg, int msg_len, char *dest);
+/* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
+
+extern int rgw_parse_op_type_list(const string& str, uint32_t *perm);
+
+#endif
diff --git a/ceph/src/rgw/rgw_cors.cc b/ceph/src/rgw/rgw_cors.cc
new file mode 100644
index 00000000..a120a686
--- /dev/null
+++ b/ceph/src/rgw/rgw_cors.cc
@@ -0,0 +1,182 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include 
+
+#include 
+#include 
+
+#include  
+
+#include "include/types.h"
+#include "common/debug.h"
+#include "include/str_list.h"
+#include "common/Formatter.h"
+
+#include "rgw_cors.h"
+
+#define dout_subsys ceph_subsys_rgw
+using namespace std;
+
+void RGWCORSRule::dump_origins() {
+  unsigned num_origins = allowed_origins.size();
+  dout(10) << "Allowed origins : " << num_origins << dendl;
+  for(set::iterator it = allowed_origins.begin();
+      it != allowed_origins.end(); 
+      ++it) {
+    dout(10) << *it << "," << dendl;
+  }
+}
+
+void RGWCORSRule::erase_origin_if_present(string& origin, bool *rule_empty) {
+  set::iterator it = allowed_origins.find(origin);
+  if (!rule_empty)
+    return;
+  *rule_empty = false;
+  if (it != allowed_origins.end()) {
+    dout(10) << "Found origin " << origin << ", set size:" << 
+        allowed_origins.size() << dendl;
+    allowed_origins.erase(it);
+    *rule_empty = (allowed_origins.empty());
+  }
+}
+
+/*
+ * make attrs look-like-this
+ * does not convert underscores or dashes
+ *
+ * Per CORS specification, section 3:
+ * ===
+ * "Converting a string to ASCII lowercase" means replacing all characters in the
+ * range U+0041 LATIN CAPITAL LETTER A to U+005A LATIN CAPITAL LETTER Z with
+ * the corresponding characters in the range U+0061 LATIN SMALL LETTER A to
+ * U+007A LATIN SMALL LETTER Z).
+ * ===
+ *
+ * @todo When UTF-8 is allowed in HTTP headers, this function will need to change
+ */
+string lowercase_http_attr(const string& orig)
+{
+  const char *s = orig.c_str();
+  char buf[orig.size() + 1];
+  buf[orig.size()] = '\0';
+
+  for (size_t i = 0; i < orig.size(); ++i, ++s) {
+	buf[i] = tolower(*s);
+  }
+  return string(buf);
+}
+
+
+static bool is_string_in_set(set& s, string h) {
+  if ((s.find("*") != s.end()) || 
+          (s.find(h) != s.end())) {
+    return true;
+  }
+  /* The header can be Content-*-type, or Content-* */
+  for(set::iterator it = s.begin();
+      it != s.end(); ++it) {
+    size_t off;
+    if ((off = (*it).find("*"))!=string::npos) {
+      list ssplit;
+      unsigned flen = 0;
+      
+      get_str_list((*it), "* \t", ssplit);
+      if (off != 0) {
+        string sl = ssplit.front();
+        flen = sl.length();
+        dout(10) << "Finding " << sl << ", in " << h << ", at offset 0" << dendl;
+        if (!boost::algorithm::starts_with(h,sl))
+          continue;
+        ssplit.pop_front();
+      }
+      if (off != ((*it).length() - 1)) {
+        string sl = ssplit.front();
+        dout(10) << "Finding " << sl << ", in " << h 
+          << ", at offset not less than " << flen << dendl;
+        if (h.compare((h.size() - sl.size()), sl.size(), sl) != 0)
+          continue;
+        ssplit.pop_front();
+      }
+      if (!ssplit.empty())
+        continue;
+      return true;
+    }
+  }
+  return false;
+}
+
+bool RGWCORSRule::is_origin_present(const char *o) {
+  string origin = o;
+  return is_string_in_set(allowed_origins, origin);
+}
+
+bool RGWCORSRule::is_header_allowed(const char *h, size_t len) {
+  string hdr(h, len);
+  if(lowercase_allowed_hdrs.empty()) {
+    set::iterator iter;
+    for (iter = allowed_hdrs.begin(); iter != allowed_hdrs.end(); ++iter) {
+      lowercase_allowed_hdrs.insert(lowercase_http_attr(*iter));
+    }
+  }
+  return is_string_in_set(lowercase_allowed_hdrs, lowercase_http_attr(hdr));
+}
+
+void RGWCORSRule::format_exp_headers(string& s) {
+  s = "";
+  for(list::iterator it = exposable_hdrs.begin();
+      it != exposable_hdrs.end(); ++it) {
+      if (s.length() > 0)
+        s.append(",");
+      s.append((*it));
+  }
+}
+
+RGWCORSRule * RGWCORSConfiguration::host_name_rule(const char *origin) {
+  for(list::iterator it_r = rules.begin(); 
+      it_r != rules.end(); ++it_r) {
+    RGWCORSRule& r = (*it_r);
+    if (r.is_origin_present(origin))
+      return &r;
+  }
+  return NULL;
+}
+
+void RGWCORSConfiguration::erase_host_name_rule(string& origin) {
+  bool rule_empty;
+  unsigned loop = 0;
+  /*Erase the host name from that rule*/
+  dout(10) << "Num of rules : " << rules.size() << dendl;
+  for(list::iterator it_r = rules.begin(); 
+      it_r != rules.end(); ++it_r, loop++) {
+    RGWCORSRule& r = (*it_r);
+    r.erase_origin_if_present(origin, &rule_empty);
+    dout(10) << "Origin:" << origin << ", rule num:" 
+      << loop << ", emptying now:" << rule_empty << dendl;
+    if (rule_empty) {
+      rules.erase(it_r);
+      break;
+    }
+  }
+}
+
+void RGWCORSConfiguration::dump() {
+  unsigned loop = 1;
+  unsigned num_rules = rules.size();
+  dout(10) << "Number of rules: " << num_rules << dendl;
+  for(list::iterator it = rules.begin();
+      it!= rules.end(); ++it, loop++) {
+    dout(10) << " <<<<<<< Rule " << loop << " >>>>>>> " << dendl;
+    (*it).dump_origins();
+  }
+}
diff --git a/ceph/src/rgw/rgw_cors.h b/ceph/src/rgw/rgw_cors.h
new file mode 100644
index 00000000..124ebf92
--- /dev/null
+++ b/ceph/src/rgw/rgw_cors.h
@@ -0,0 +1,135 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef CEPH_RGW_CORS_H
+#define CEPH_RGW_CORS_H
+
+#include 
+#include 
+#include 
+#include 
+
+#define RGW_CORS_GET    0x1
+#define RGW_CORS_PUT    0x2
+#define RGW_CORS_HEAD   0x4
+#define RGW_CORS_POST   0x8
+#define RGW_CORS_DELETE 0x10
+#define RGW_CORS_COPY   0x20
+#define RGW_CORS_ALL    (RGW_CORS_GET    |  \
+                         RGW_CORS_PUT    |  \
+                         RGW_CORS_HEAD   |  \
+                         RGW_CORS_POST   |  \
+                         RGW_CORS_DELETE |  \
+                         RGW_CORS_COPY)
+
+#define CORS_MAX_AGE_INVALID ((uint32_t)-1)
+
+class RGWCORSRule
+{
+protected:
+  uint32_t       max_age;
+  uint8_t        allowed_methods;
+  std::string         id;
+  std::set allowed_hdrs; /* If you change this, you need to discard lowercase_allowed_hdrs */
+  std::set lowercase_allowed_hdrs; /* Not built until needed in RGWCORSRule::is_header_allowed */
+  std::set allowed_origins;
+  std::list exposable_hdrs;
+
+public:
+  RGWCORSRule() : max_age(CORS_MAX_AGE_INVALID),allowed_methods(0) {}
+  RGWCORSRule(std::set& o, std::set& h, 
+              std::list& e, uint8_t f, uint32_t a)
+      :max_age(a),
+       allowed_methods(f),
+       allowed_hdrs(h),
+       allowed_origins(o),
+       exposable_hdrs(e) {}
+  virtual ~RGWCORSRule() {}
+
+  std::string& get_id() { return id; }
+  uint32_t get_max_age() { return max_age; }
+  uint8_t get_allowed_methods() { return allowed_methods; }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(max_age, bl);
+    ::encode(allowed_methods, bl);
+    ::encode(id, bl);
+    ::encode(allowed_hdrs, bl);
+    ::encode(allowed_origins, bl);
+    ::encode(exposable_hdrs, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(max_age, bl);
+    ::decode(allowed_methods, bl);
+    ::decode(id, bl);
+    ::decode(allowed_hdrs, bl);
+    ::decode(allowed_origins, bl);
+    ::decode(exposable_hdrs, bl);
+    DECODE_FINISH(bl);
+  }
+  bool is_origin_present(const char *o);
+  void format_exp_headers(std::string& s);
+  void erase_origin_if_present(std::string& origin, bool *rule_empty);
+  void dump_origins(); 
+  void dump(Formatter *f) const;
+  bool is_header_allowed(const char *hdr, size_t len);
+};
+WRITE_CLASS_ENCODER(RGWCORSRule)
+
+class RGWCORSConfiguration
+{
+  protected:
+    std::list rules;
+  public:
+    RGWCORSConfiguration() {}
+    ~RGWCORSConfiguration() {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(rules, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(rules, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  std::list& get_rules() {
+    return rules;
+  }
+  bool is_empty() {
+    return rules.empty();
+  }
+  void get_origins_list(const char *origin, std::list& origins);
+  RGWCORSRule * host_name_rule(const char *origin);
+  void erase_host_name_rule(std::string& origin);
+  void dump();
+  void stack_rule(RGWCORSRule& r) {
+    rules.push_front(r);    
+  }
+};
+WRITE_CLASS_ENCODER(RGWCORSConfiguration)
+
+static inline int validate_name_string(string& o) {
+  if (o.length() == 0)
+    return -1;
+  if (o.find_first_of("*") != o.find_last_of("*"))
+    return -1;
+  return 0;
+}
+#endif /*CEPH_RGW_CORS_H*/
diff --git a/ceph/src/rgw/rgw_cors_s3.cc b/ceph/src/rgw/rgw_cors_s3.cc
new file mode 100644
index 00000000..01150a9e
--- /dev/null
+++ b/ceph/src/rgw/rgw_cors_s3.cc
@@ -0,0 +1,239 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include 
+#include 
+
+#include 
+#include 
+
+#include "include/types.h"
+
+#include "rgw_cors_s3.h"
+#include "rgw_user.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+void RGWCORSRule_S3::to_xml(XMLFormatter& f) {
+
+  f.open_object_section("CORSRule");
+  /*ID if present*/
+  if (id.length() > 0) {
+    f.dump_string("ID", id);;
+  }
+  /*AllowedMethods*/
+  if (allowed_methods & RGW_CORS_GET)
+    f.dump_string("AllowedMethod", "GET");
+  if (allowed_methods & RGW_CORS_PUT)
+    f.dump_string("AllowedMethod", "PUT");
+  if (allowed_methods & RGW_CORS_DELETE)
+    f.dump_string("AllowedMethod", "DELETE");
+  if (allowed_methods & RGW_CORS_HEAD)
+    f.dump_string("AllowedMethod", "HEAD");
+  if (allowed_methods & RGW_CORS_POST)
+    f.dump_string("AllowedMethod", "POST");
+  if (allowed_methods & RGW_CORS_COPY)
+    f.dump_string("AllowedMethod", "COPY");
+  /*AllowedOrigins*/
+  for(set::iterator it = allowed_origins.begin(); 
+      it != allowed_origins.end(); 
+      ++it) {
+    string host = *it;
+    f.dump_string("AllowedOrigin", host);
+  }
+  /*AllowedHeader*/
+  for(set::iterator it = allowed_hdrs.begin(); 
+      it != allowed_hdrs.end(); ++it) {
+    f.dump_string("AllowedHeader", *it);
+  }
+  /*MaxAgeSeconds*/
+  if (max_age != CORS_MAX_AGE_INVALID) {
+    f.dump_unsigned("MaxAgeSeconds", max_age);
+  }
+  /*ExposeHeader*/
+  for(list::iterator it = exposable_hdrs.begin(); 
+      it != exposable_hdrs.end(); ++it) {
+    f.dump_string("ExposeHeader", *it);
+  }
+  f.close_section();
+}
+
+bool RGWCORSRule_S3::xml_end(const char *el) {
+  XMLObjIter iter = find("AllowedMethod");
+  XMLObj *obj;
+  /*Check all the allowedmethods*/
+  obj = iter.get_next();
+  if (obj) {
+    for( ; obj; obj = iter.get_next()) {
+      const char *s = obj->get_data().c_str();
+      dout(10) << "RGWCORSRule::xml_end, el : " << el << ", data : " << s << dendl;
+      if (strcasecmp(s, "GET") == 0) {
+        allowed_methods |= RGW_CORS_GET;
+      } else if (strcasecmp(s, "POST") == 0) {
+        allowed_methods |= RGW_CORS_POST;
+      } else if (strcasecmp(s, "DELETE") == 0) {
+        allowed_methods |= RGW_CORS_DELETE;
+      } else if (strcasecmp(s, "HEAD") == 0) {
+        allowed_methods |= RGW_CORS_HEAD;
+      } else if (strcasecmp(s, "PUT") == 0) {
+        allowed_methods |= RGW_CORS_PUT;
+      } else if (strcasecmp(s, "COPY") == 0) {
+        allowed_methods |= RGW_CORS_COPY;
+      } else {
+        return false;
+      }
+    }
+  } 
+  /*Check the id's len, it should be less than 255*/
+  XMLObj *xml_id = find_first("ID");
+  if (xml_id != NULL) {
+    string data = xml_id->get_data();
+    if (data.length() > 255) {
+      dout(0) << "RGWCORSRule has id of length greater than 255" << dendl;
+      return false;
+    }
+    dout(10) << "RGWCORRule id : " << data << dendl;  
+    id = data;
+  }
+  /*Check if there is atleast one AllowedOrigin*/
+  iter = find("AllowedOrigin");
+  if (!(obj = iter.get_next())) {
+    dout(0) << "RGWCORSRule does not have even one AllowedOrigin" << dendl;
+    return false;
+  }
+  for( ; obj; obj = iter.get_next()) {
+    dout(10) << "RGWCORSRule - origin : " << obj->get_data() << dendl;
+    /*Just take the hostname*/
+    string host = obj->get_data();
+    if (validate_name_string(host) != 0)
+      return false;
+    allowed_origins.insert(allowed_origins.end(), host);
+  }
+  /*Check of max_age*/
+  iter = find("MaxAgeSeconds");
+  if ((obj = iter.get_next())) {
+    char *end = NULL;
+
+    unsigned long long ull = strtoull(obj->get_data().c_str(), &end, 10);
+    if (ull >= 0x100000000ull) {
+      max_age = CORS_MAX_AGE_INVALID;
+    } else  {
+      max_age = (uint32_t)ull;
+    }
+    dout(10) << "RGWCORSRule : max_age : " << max_age << dendl;
+  }
+  /*Check and update ExposeHeader*/
+  iter = find("ExposeHeader");
+  if ((obj = iter.get_next())) {
+    for(; obj; obj = iter.get_next()) {
+      dout(10) << "RGWCORSRule - exp_hdr : " << obj->get_data() << dendl;
+      exposable_hdrs.push_back(obj->get_data());
+    }
+  }
+  /*Check and update AllowedHeader*/
+  iter = find("AllowedHeader");
+  if ((obj = iter.get_next())) {
+    for(; obj; obj = iter.get_next()) {
+      dout(10) << "RGWCORSRule - allowed_hdr : " << obj->get_data() << dendl;
+      string s = obj->get_data();
+      if (validate_name_string(s) != 0)
+         return false;
+      allowed_hdrs.insert(allowed_hdrs.end(), s);
+    }
+  }
+  return true;
+}
+
+void RGWCORSConfiguration_S3::to_xml(ostream& out) {
+  XMLFormatter f;
+  f.open_object_section("CORSConfiguration");
+  for(list::iterator it = rules.begin();
+      it != rules.end(); ++it) {
+    (static_cast(*it)).to_xml(f);
+  }
+  f.close_section();
+  f.flush(out);
+}
+
+bool RGWCORSConfiguration_S3::xml_end(const char *el) {
+  XMLObjIter iter = find("CORSRule");
+  RGWCORSRule_S3 *obj;
+  if (!(obj = static_cast(iter.get_next()))) {
+    dout(0) << "CORSConfiguration should have atleast one CORSRule" << dendl;
+    return false;
+  }
+  for(; obj; obj = static_cast(iter.get_next())) {
+    rules.push_back(*obj);
+  }
+  return true;
+}
+
+class CORSRuleID_S3 : public XMLObj {
+  public:
+    CORSRuleID_S3() {}
+    ~CORSRuleID_S3() {}
+};
+
+class CORSRuleAllowedOrigin_S3 : public XMLObj {
+  public:
+    CORSRuleAllowedOrigin_S3() {}
+    ~CORSRuleAllowedOrigin_S3() {}
+};
+
+class CORSRuleAllowedMethod_S3 : public XMLObj {
+  public:
+    CORSRuleAllowedMethod_S3() {}
+    ~CORSRuleAllowedMethod_S3() {}
+};
+
+class CORSRuleAllowedHeader_S3 : public XMLObj {
+  public:
+    CORSRuleAllowedHeader_S3() {}
+    ~CORSRuleAllowedHeader_S3() {}
+};
+
+class CORSRuleMaxAgeSeconds_S3 : public XMLObj {
+  public:
+    CORSRuleMaxAgeSeconds_S3() {}
+    ~CORSRuleMaxAgeSeconds_S3() {}
+};
+
+class CORSRuleExposeHeader_S3 : public XMLObj {
+  public:
+    CORSRuleExposeHeader_S3() {}
+    ~CORSRuleExposeHeader_S3() {}
+};
+
+XMLObj *RGWCORSXMLParser_S3::alloc_obj(const char *el) {
+  if (strcmp(el, "CORSConfiguration") == 0) {
+    return new RGWCORSConfiguration_S3;
+  } else if (strcmp(el, "CORSRule") == 0) {
+    return new RGWCORSRule_S3;
+  } else if (strcmp(el, "ID") == 0) {
+    return new CORSRuleID_S3;
+  } else if (strcmp(el, "AllowedOrigin") == 0) {
+    return new CORSRuleAllowedOrigin_S3;
+  } else if (strcmp(el, "AllowedMethod") == 0) {
+    return new CORSRuleAllowedMethod_S3;
+  } else if (strcmp(el, "AllowedHeader") == 0) {
+    return new CORSRuleAllowedHeader_S3;
+  } else if (strcmp(el, "MaxAgeSeconds") == 0) {
+    return new CORSRuleMaxAgeSeconds_S3;
+  } else if (strcmp(el, "ExposeHeader")  == 0) {
+    return new CORSRuleExposeHeader_S3;
+  }
+  return NULL;
+}
+
diff --git a/ceph/src/rgw/rgw_cors_s3.h b/ceph/src/rgw/rgw_cors_s3.h
new file mode 100644
index 00000000..0db03c3e
--- /dev/null
+++ b/ceph/src/rgw/rgw_cors_s3.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef CEPH_RGW_CORS_S3_H
+#define CEPH_RGW_CORS_S3_H
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include "rgw_xml.h"
+#include "rgw_cors.h"
+
+using namespace std;
+
+class RGWCORSRule_S3 : public RGWCORSRule, public XMLObj
+{
+  public:
+    RGWCORSRule_S3() {}
+    ~RGWCORSRule_S3() {}
+    
+    bool xml_end(const char *el);
+    void to_xml(XMLFormatter& f);
+};
+
+class RGWCORSConfiguration_S3 : public RGWCORSConfiguration, public XMLObj
+{
+  public:
+    RGWCORSConfiguration_S3() {}
+    ~RGWCORSConfiguration_S3() {}
+
+    bool xml_end(const char *el);
+    void to_xml(ostream& out);
+};
+
+class RGWCORSXMLParser_S3 : public RGWXMLParser
+{
+  CephContext *cct;
+
+  XMLObj *alloc_obj(const char *el);
+public:
+  RGWCORSXMLParser_S3(CephContext *_cct) : cct(_cct) {}
+};
+#endif /*CEPH_RGW_CORS_S3_H*/
diff --git a/ceph/src/rgw/rgw_cors_swift.h b/ceph/src/rgw/rgw_cors_swift.h
new file mode 100644
index 00000000..8037b4f5
--- /dev/null
+++ b/ceph/src/rgw/rgw_cors_swift.h
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef CEPH_RGW_CORS_SWIFT3_H
+#define CEPH_RGW_CORS_SWIFT3_H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "rgw_cors.h"
+
+using namespace std;
+
+class RGWCORSConfiguration_SWIFT : public RGWCORSConfiguration
+{
+  public:
+    RGWCORSConfiguration_SWIFT() {}
+    ~RGWCORSConfiguration_SWIFT() {}
+    int create_update(const char *allow_origins, const char *allow_headers, 
+                  const char *expose_headers, const char *max_age) {
+      set o, h, oc;
+      list e;
+      unsigned long a = CORS_MAX_AGE_INVALID;
+      uint8_t flags = RGW_CORS_ALL;
+
+      string ao = allow_origins;
+      get_str_set(ao, oc);
+      if (oc.empty())
+        return -EINVAL;
+      for(set::iterator it = oc.begin(); it != oc.end(); ++it) {
+        string host = *it;
+        if (validate_name_string(host) != 0)
+          return -EINVAL;
+        o.insert(o.end(), host);
+      }
+      if (allow_headers) {
+        string ah = allow_headers;
+        get_str_set(ah, h);
+        for(set::iterator it = h.begin();
+            it != h.end(); ++it) {
+          string s = (*it);
+          if (validate_name_string(s) != 0)
+            return -EINVAL;
+        }
+      }
+
+      if (expose_headers) {
+        string eh = expose_headers;
+        get_str_list(eh, e);
+      }
+      if (max_age) {
+        char *end = NULL;
+        a = strtoul(max_age, &end, 10);
+        if (a == ULONG_MAX)
+          a = CORS_MAX_AGE_INVALID;
+      }
+
+      RGWCORSRule rule(o, h, e, flags, a);
+      stack_rule(rule);
+      return 0;
+    }
+};
+#endif /*CEPH_RGW_CORS_SWIFT3_H*/
diff --git a/ceph/src/rgw/rgw_dencoder.cc b/ceph/src/rgw/rgw_dencoder.cc
new file mode 100644
index 00000000..e6dc2baf
--- /dev/null
+++ b/ceph/src/rgw/rgw_dencoder.cc
@@ -0,0 +1,458 @@
+
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "rgw_log.h"
+#include "rgw_acl.h"
+#include "rgw_acl_s3.h"
+#include "rgw_cache.h"
+
+#include "common/Formatter.h"
+
+static string shadow_ns = RGW_OBJ_NS_SHADOW;
+
+void RGWObjManifestPart::generate_test_instances(std::list& o)
+{
+  o.push_back(new RGWObjManifestPart);
+
+  RGWObjManifestPart *p = new RGWObjManifestPart;
+  rgw_bucket b("bucket", ".pool", ".index_pool", "marker_", "12", "region");
+  p->loc = rgw_obj(b, "object");
+  p->loc_ofs = 512 * 1024;
+  p->size = 128 * 1024;
+  o.push_back(p);
+}
+
+void RGWObjManifest::obj_iterator::seek(uint64_t o)
+{
+  ofs = o;
+  if (manifest->explicit_objs) {
+    explicit_iter = manifest->objs.upper_bound(ofs);
+    if (explicit_iter != manifest->objs.begin()) {
+      --explicit_iter;
+    }
+    if (ofs >= manifest->obj_size) {
+      ofs = manifest->obj_size;
+      return;
+    }
+    update_explicit_pos();
+    update_location();
+    return;
+  }
+  if (o < manifest->get_head_size()) {
+    rule_iter = manifest->rules.begin();
+    stripe_ofs = 0;
+    stripe_size = manifest->get_head_size();
+    if (rule_iter != manifest->rules.end()) {
+      cur_part_id = rule_iter->second.start_part_num;
+      cur_override_prefix = rule_iter->second.override_prefix;
+    }
+    update_location();
+    return;
+  }
+
+  rule_iter = manifest->rules.upper_bound(ofs);
+  next_rule_iter = rule_iter;
+  if (rule_iter != manifest->rules.begin()) {
+    --rule_iter;
+  }
+
+  if (rule_iter == manifest->rules.end()) {
+    update_location();
+    return;
+  }
+
+  RGWObjManifestRule& rule = rule_iter->second;
+
+  if (rule.part_size > 0) {
+    cur_part_id = rule.start_part_num + (ofs - rule.start_ofs) / rule.part_size;
+  } else {
+    cur_part_id = rule.start_part_num;
+  }
+  part_ofs = rule.start_ofs + (cur_part_id - rule.start_part_num) * rule.part_size;
+
+  if (rule.stripe_max_size > 0) {
+    cur_stripe = (ofs - part_ofs) / rule.stripe_max_size;
+
+    stripe_ofs = part_ofs + cur_stripe * rule.stripe_max_size;
+    if (!cur_part_id && manifest->get_head_size() > 0) {
+      cur_stripe++;
+    }
+  } else {
+    cur_stripe = 0;
+    stripe_ofs = part_ofs;
+  }
+
+  if (!rule.part_size) {
+    stripe_size = rule.stripe_max_size;
+    stripe_size = MIN(manifest->get_obj_size() - stripe_ofs, stripe_size);
+  } else {
+    uint64_t next = MIN(stripe_ofs + rule.stripe_max_size, part_ofs + rule.part_size);
+    stripe_size = next - stripe_ofs;
+  }
+
+  cur_override_prefix = rule.override_prefix;
+
+  update_location();
+}
+
+void RGWObjManifest::obj_iterator::update_location()
+{
+  if (manifest->explicit_objs) {
+    location = explicit_iter->second.loc;
+    return;
+  }
+
+  const rgw_obj& head = manifest->get_head();
+
+  if (ofs < manifest->get_head_size()) {
+    location = head;
+    return;
+  }
+
+  manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_override_prefix, &location);
+}
+
+void RGWObjManifest::obj_iterator::update_explicit_pos()
+{
+  ofs = explicit_iter->first;
+  stripe_ofs = ofs;
+
+  map::iterator next_iter = explicit_iter;
+  ++next_iter;
+  if (next_iter != manifest->objs.end()) {
+    stripe_size = next_iter->first - ofs;
+  } else {
+    stripe_size = manifest->obj_size - ofs;
+  }
+}
+
+void RGWObjManifest::generate_test_instances(std::list& o)
+{
+  RGWObjManifest *m = new RGWObjManifest;
+  for (int i = 0; i<10; i++) {
+    RGWObjManifestPart p;
+    rgw_bucket b("bucket", ".pool", ".index_pool", "marker_", "12", "region");
+    p.loc = rgw_obj(b, "object");
+    p.loc_ofs = 0;
+    p.size = 512 * 1024;
+    m->objs[(uint64_t)i * 512 * 1024] = p;
+  }
+  m->obj_size = 5 * 1024 * 1024;
+
+  o.push_back(m);
+
+  o.push_back(new RGWObjManifest);
+}
+
+void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj *location)
+{
+  string oid;
+  if (!override_prefix || override_prefix->empty()) {
+    oid = prefix;
+  } else {
+    oid = *override_prefix;
+  }
+  string ns;
+
+  if (!cur_part_id) {
+    if (ofs < max_head_size) {
+      *location = head_obj;
+      return;
+    } else {
+      char buf[16];
+      snprintf(buf, sizeof(buf), "%d", (int)cur_stripe);
+      oid += buf;
+      ns = shadow_ns;
+    }
+  } else {
+    char buf[32];
+    if (cur_stripe == 0) {
+      snprintf(buf, sizeof(buf), ".%d", (int)cur_part_id);
+      oid += buf;
+      ns= RGW_OBJ_NS_MULTIPART;
+    } else {
+      snprintf(buf, sizeof(buf), ".%d_%d", (int)cur_part_id, (int)cur_stripe);
+      oid += buf;
+      ns = shadow_ns;
+    }
+  }
+
+  rgw_bucket *bucket;
+
+  if (!tail_bucket.name.empty()) {
+    bucket = &tail_bucket;
+  } else {
+    bucket = &head_obj.bucket;
+  }
+
+  location->init_ns(*bucket, oid, ns);
+}
+
+
+
+void rgw_log_entry::generate_test_instances(list& o)
+{
+  rgw_log_entry *e = new rgw_log_entry;
+  e->object_owner = "object_owner";
+  e->bucket_owner = "bucket_owner";
+  e->bucket = "bucket";
+  e->remote_addr = "1.2.3.4";
+  e->user = "user";
+  e->obj = "obj";
+  e->uri = "http://uri/bucket/obj";
+  e->http_status = "200";
+  e->error_code = "error_code";
+  e->bytes_sent = 1024;
+  e->bytes_received = 512;
+  e->obj_size = 2048;
+  e->user_agent = "user_agent";
+  e->referrer = "referrer";
+  e->bucket_id = "10";
+  o.push_back(e);
+  o.push_back(new rgw_log_entry);
+}
+
+void rgw_intent_log_entry::generate_test_instances(list& o)
+{
+  rgw_intent_log_entry *e = new rgw_intent_log_entry;
+  rgw_bucket b("bucket", "pool", ".index_pool", "marker", "10", "region");
+  e->obj = rgw_obj(b, "object");
+  e->intent = DEL_OBJ;
+  o.push_back(e);
+  o.push_back(new rgw_intent_log_entry);
+}
+
+void ACLPermission::generate_test_instances(list& o)
+{
+  ACLPermission *p = new ACLPermission;
+  p->set_permissions(RGW_PERM_WRITE_ACP);
+  o.push_back(p);
+  o.push_back(new ACLPermission);
+}
+
+void ACLGranteeType::generate_test_instances(list& o)
+{
+  ACLGranteeType *t = new ACLGranteeType;
+  t->set(ACL_TYPE_CANON_USER);
+  o.push_back(t);
+  o.push_back(new ACLGranteeType);
+}
+
+/* the following is copied here from rgw_acl_s3.cc, to avoid having to have excessive linking
+   with everything it needs */
+
+#define RGW_URI_ALL_USERS	"http://acs.amazonaws.com/groups/global/AllUsers"
+#define RGW_URI_AUTH_USERS	"http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+
+static string rgw_uri_all_users = RGW_URI_ALL_USERS;
+static string rgw_uri_auth_users = RGW_URI_AUTH_USERS;
+
+ACLGroupTypeEnum ACLGrant::uri_to_group(string& uri)
+{
+  // this is required for backward compatibility
+  return ACLGrant_S3::uri_to_group(uri);
+}
+
+ACLGroupTypeEnum ACLGrant_S3::uri_to_group(string& uri)
+{
+  if (uri.compare(rgw_uri_all_users) == 0)
+    return ACL_GROUP_ALL_USERS;
+  else if (uri.compare(rgw_uri_auth_users) == 0)
+    return ACL_GROUP_AUTHENTICATED_USERS;
+
+  return ACL_GROUP_NONE;
+}
+
+void ACLGrant::generate_test_instances(list& o)
+{
+  string id, name, email;
+  id = "rgw";
+  name = "Mr. RGW";
+  email = "r@gw";
+
+  ACLGrant *g1 = new ACLGrant;
+  g1->set_canon(id, name, RGW_PERM_READ);
+  g1->email = email;
+  o.push_back(g1);
+
+  ACLGrant *g2 = new ACLGrant;
+  g1->set_group(ACL_GROUP_AUTHENTICATED_USERS, RGW_PERM_WRITE);
+  o.push_back(g2);
+
+  o.push_back(new ACLGrant);
+}
+
+void RGWAccessControlList::generate_test_instances(list& o)
+{
+  RGWAccessControlList *acl = new RGWAccessControlList(NULL);
+
+  list glist;
+  list::iterator iter;
+
+  ACLGrant::generate_test_instances(glist);
+  for (iter = glist.begin(); iter != glist.end(); ++iter) {
+    ACLGrant *grant = *iter;
+    acl->add_grant(grant);
+
+    delete grant;
+  }
+  o.push_back(acl);
+  o.push_back(new RGWAccessControlList(NULL));
+}
+
+void ACLOwner::generate_test_instances(list& o)
+{
+  ACLOwner *owner = new ACLOwner;
+  owner->id = "rgw";
+  owner->display_name = "Mr. RGW";
+  o.push_back(owner);
+  o.push_back(new ACLOwner);
+}
+
+void RGWAccessControlPolicy::generate_test_instances(list& o)
+{
+  list acl_list;
+  list::iterator iter;
+  for (iter = acl_list.begin(); iter != acl_list.end(); ++iter) {
+    RGWAccessControlList::generate_test_instances(acl_list);
+    iter = acl_list.begin();
+
+    RGWAccessControlPolicy *p = new RGWAccessControlPolicy(NULL);
+    RGWAccessControlList *l = *iter;
+    p->acl = *l;
+
+    string name = "radosgw";
+    string id = "rgw";
+    p->owner.set_name(name);
+    p->owner.set_id(id);
+
+    o.push_back(p);
+
+    delete l;
+  }
+
+  o.push_back(new RGWAccessControlPolicy(NULL));
+}
+
+
+void ObjectMetaInfo::generate_test_instances(list& o)
+{
+  ObjectMetaInfo *m = new ObjectMetaInfo;
+  m->size = 1024 * 1024;
+  o.push_back(m);
+  o.push_back(new ObjectMetaInfo);
+}
+
+void ObjectCacheInfo::generate_test_instances(list& o)
+{
+  ObjectCacheInfo *i = new ObjectCacheInfo;
+  i->status = 0;
+  i->flags = CACHE_FLAG_MODIFY_XATTRS;
+  string s = "this is a string";
+  string s2 = "this is a another string";
+  bufferlist data, data2;
+  ::encode(s, data);
+  ::encode(s2, data2);
+  i->data = data;
+  i->xattrs["x1"] = data;
+  i->xattrs["x2"] = data2;
+  i->rm_xattrs["r2"] = data2;
+  i->rm_xattrs["r3"] = data;
+  i->meta.size = 512 * 1024;
+  o.push_back(i);
+  o.push_back(new ObjectCacheInfo);
+}
+
+void RGWCacheNotifyInfo::generate_test_instances(list& o)
+{
+  o.push_back(new RGWCacheNotifyInfo);
+}
+
+void RGWAccessKey::generate_test_instances(list& o)
+{
+  RGWAccessKey *k = new RGWAccessKey;
+  k->id = "id";
+  k->key = "key";
+  k->subuser = "subuser";
+  o.push_back(k);
+  o.push_back(new RGWAccessKey);
+}
+
+void RGWSubUser::generate_test_instances(list& o)
+{
+  RGWSubUser *u = new RGWSubUser;
+  u->name = "name";
+  u->perm_mask = 0xf;
+  o.push_back(u);
+  o.push_back(new RGWSubUser);
+}
+
+void RGWUserInfo::generate_test_instances(list& o)
+{
+  RGWUserInfo *i = new RGWUserInfo;
+  i->auid = 1;
+  i->user_id = "user_id";
+  i->display_name =  "display_name";
+  i->user_email = "user@email";
+  RGWAccessKey k1, k2;
+  k1.id = "id1";
+  k1.key = "key1";
+  k2.id = "id2";
+  k2.subuser = "subuser";
+  RGWSubUser u;
+  u.name = "id2";
+  u.perm_mask = 0x1;
+  i->access_keys[k1.id] = k1;
+  i->swift_keys[k2.id] = k2;
+  i->subusers[u.name] = u;
+  o.push_back(i);
+
+  o.push_back(new RGWUserInfo);
+}
+
+void rgw_bucket::generate_test_instances(list& o)
+{
+  rgw_bucket *b = new rgw_bucket("name", "pool", ".index_pool", "marker", "123", "region");
+  o.push_back(b);
+  o.push_back(new rgw_bucket);
+}
+
+void RGWBucketInfo::generate_test_instances(list& o)
+{
+  RGWBucketInfo *i = new RGWBucketInfo;
+  i->bucket = rgw_bucket("bucket", "pool", ".index_pool", "marker", "10", "region");
+  i->owner = "owner";
+  i->flags = BUCKET_SUSPENDED;
+  o.push_back(i);
+  o.push_back(new RGWBucketInfo);
+}
+
+void RGWBucketEnt::generate_test_instances(list& o)
+{
+  RGWBucketEnt *e = new RGWBucketEnt;
+  e->bucket = rgw_bucket("bucket", "pool", ".index_pool", "marker", "10", "region");
+  e->size = 1024;
+  e->size_rounded = 4096;
+  e->count = 1;
+  o.push_back(e);
+  o.push_back(new RGWBucketEnt);
+}
+
+void RGWUploadPartInfo::generate_test_instances(list& o)
+{
+  RGWUploadPartInfo *i = new RGWUploadPartInfo;
+  i->num = 1;
+  i->size = 10 * 1024 * 1024;
+  i->etag = "etag";
+  o.push_back(i);
+  o.push_back(new RGWUploadPartInfo);
+}
+
+void rgw_obj::generate_test_instances(list& o)
+{
+  rgw_bucket b = rgw_bucket("bucket", "pool", ".index_pool", "marker", "10", "region");
+  rgw_obj *obj = new rgw_obj(b, "object");
+  o.push_back(obj);
+  o.push_back(new rgw_obj);
+}
+
diff --git a/ceph/src/rgw/rgw_env.cc b/ceph/src/rgw/rgw_env.cc
new file mode 100644
index 00000000..faa8d8f5
--- /dev/null
+++ b/ceph/src/rgw/rgw_env.cc
@@ -0,0 +1,127 @@
+#include "rgw_common.h"
+#include "rgw_log.h"
+
+#include 
+#include 
+
+#define dout_subsys ceph_subsys_rgw
+
+RGWEnv::RGWEnv()
+{
+  conf = new RGWConf;
+}
+
+RGWEnv::~RGWEnv()
+{
+  delete conf;
+}
+
+void RGWEnv::init(CephContext *cct)
+{
+  conf->init(cct, this);
+}
+
+void RGWEnv::set(const char *name, const char *val)
+{
+  if (!val)
+    val = "";
+  env_map[name] = val;
+
+  dout(20) << "RGWEnv::set(): " << name << ": " << val << dendl;
+}
+
+void RGWEnv::init(CephContext *cct, char **envp)
+{
+  const char *p;
+
+  env_map.clear();
+
+  for (int i=0; (p = envp[i]); ++i) {
+    string s(p);
+    int pos = s.find('=');
+    if (pos <= 0) // should never be 0
+      continue;
+    string name = s.substr(0, pos);
+    string val = s.substr(pos + 1);
+    env_map[name] = val;
+  }
+
+  init(cct);
+}
+
+const char *RGWEnv::get(const char *name, const char *def_val)
+{
+  map::iterator iter = env_map.find(name);
+  if (iter == env_map.end())
+    return def_val;
+
+  return iter->second.c_str();
+}
+
+int RGWEnv::get_int(const char *name, int def_val)
+{
+  map::iterator iter = env_map.find(name);
+  if (iter == env_map.end())
+    return def_val;
+
+  const char *s = iter->second.c_str();
+  return atoi(s);  
+}
+
+bool RGWEnv::get_bool(const char *name, bool def_val)
+{
+  map::iterator iter = env_map.find(name);
+  if (iter == env_map.end())
+    return def_val;
+
+  const char *s = iter->second.c_str();
+  return rgw_str_to_bool(s, def_val);
+}
+
+size_t RGWEnv::get_size(const char *name, size_t def_val)
+{
+  map::iterator iter = env_map.find(name);
+  if (iter == env_map.end())
+    return def_val;
+
+  const char *s = iter->second.c_str();
+  return atoll(s);  
+}
+
+bool RGWEnv::exists(const char *name)
+{
+  map::iterator iter = env_map.find(name);
+  return (iter != env_map.end());
+}
+
+bool RGWEnv::exists_prefix(const char *prefix)
+{
+  if (env_map.empty() || prefix == NULL)
+    return false;
+
+  map::iterator iter = env_map.lower_bound(prefix);
+  if (iter == env_map.end())
+    return false;
+
+  return (strncmp(iter->first.c_str(), prefix, strlen(prefix)) == 0);
+}
+
+void RGWEnv::remove(const char *name)
+{
+  map::iterator iter = env_map.find(name);
+  if (iter != env_map.end())
+    env_map.erase(iter);
+}
+
+void RGWConf::init(CephContext *cct, RGWEnv *env)
+{
+  enable_ops_log = cct->_conf->rgw_enable_ops_log;
+  enable_usage_log = cct->_conf->rgw_enable_usage_log;
+
+  defer_to_bucket_acls = 0;  // default
+  if (cct->_conf->rgw_defer_to_bucket_acls == "recurse") {
+    defer_to_bucket_acls = RGW_DEFER_TO_BUCKET_ACLS_RECURSE;
+  } else if (cct->_conf->rgw_defer_to_bucket_acls == "full_control") {
+    defer_to_bucket_acls = RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL;
+  }
+}
diff --git a/ceph/src/rgw/rgw_fcgi.cc b/ceph/src/rgw/rgw_fcgi.cc
new file mode 100644
index 00000000..4b24dabe
--- /dev/null
+++ b/ceph/src/rgw/rgw_fcgi.cc
@@ -0,0 +1,58 @@
+
+
+#include "rgw_fcgi.h"
+
+#include "acconfig.h"
+#ifdef FASTCGI_INCLUDE_DIR
+# include "fastcgi/fcgiapp.h"
+#else
+# include "fcgiapp.h"
+#endif
+
+
+int RGWFCGX::write_data(const char *buf, int len)
+{
+  return FCGX_PutStr(buf, len, fcgx->out);
+}
+
+int RGWFCGX::read_data(char *buf, int len)
+{
+  return FCGX_GetStr(buf, len, fcgx->in);
+}
+
+void RGWFCGX::flush()
+{
+  FCGX_FFlush(fcgx->out);
+}
+
+void RGWFCGX::init_env(CephContext *cct)
+{
+  env.init(cct, (char **)fcgx->envp);
+}
+
+int RGWFCGX::send_status(const char *status, const char *status_name)
+{
+  return print("Status: %s\n", status);
+}
+
+int RGWFCGX::send_100_continue()
+{
+  int r = send_status("100", "Continue");
+  if (r >= 0) {
+    flush();
+  }
+  return r;
+}
+
+int RGWFCGX::send_content_length(uint64_t len)
+{
+  char buf[21];
+  snprintf(buf, sizeof(buf), "%"PRIu64, len);
+  return print("Content-Length: %s\n", buf);
+}
+
+int RGWFCGX::complete_header()
+{
+  return print("\r\n");
+}
+
diff --git a/ceph/src/rgw/rgw_fcgi.h b/ceph/src/rgw/rgw_fcgi.h
new file mode 100644
index 00000000..fabc77a0
--- /dev/null
+++ b/ceph/src/rgw/rgw_fcgi.h
@@ -0,0 +1,29 @@
+#ifndef CEPH_RGW_FCGI_H
+#define CEPH_RGW_FCGI_H
+
+#include "rgw_client_io.h"
+
+
+struct FCGX_Request;
+
+
+class RGWFCGX : public RGWClientIO
+{
+  FCGX_Request *fcgx;
+protected:
+  void init_env(CephContext *cct);
+  int write_data(const char *buf, int len);
+  int read_data(char *buf, int len);
+
+  int send_status(const char *status, const char *status_name);
+  int send_100_continue();
+  int complete_header();
+  int complete_request() { return 0; }
+  int send_content_length(uint64_t len);
+public:
+  RGWFCGX(FCGX_Request *_fcgx) : fcgx(_fcgx) {}
+  void flush();
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_formats.cc b/ceph/src/rgw/rgw_formats.cc
new file mode 100644
index 00000000..56fc7e7d
--- /dev/null
+++ b/ceph/src/rgw/rgw_formats.cc
@@ -0,0 +1,250 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/escape.h"
+#include "common/Formatter.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_formats.h"
+
+#define LARGE_SIZE 8192
+
+#define dout_subsys ceph_subsys_rgw
+
+RGWFormatter_Plain::RGWFormatter_Plain()
+  : buf(NULL), len(0), max_len(0), min_stack_level(0)
+{
+}
+
+RGWFormatter_Plain::~RGWFormatter_Plain()
+{
+  free(buf);
+}
+
+void RGWFormatter_Plain::flush(ostream& os)
+{
+  if (!buf)
+    return;
+
+  if (len) {
+    os << buf << "\n";
+    os.flush();
+  }
+
+  reset_buf();
+}
+
+void RGWFormatter_Plain::reset_buf()
+{
+  free(buf);
+  buf = NULL;
+  len = 0;
+  max_len = 0;
+}
+
+void RGWFormatter_Plain::reset()
+{
+  reset_buf();
+  stack.clear();
+  min_stack_level = 0;
+}
+
+void RGWFormatter_Plain::open_array_section(const char *name)
+{
+  struct plain_stack_entry new_entry;
+  new_entry.is_array = true;
+  new_entry.size = 0;
+  stack.push_back(new_entry);
+}
+
+void RGWFormatter_Plain::open_array_section_in_ns(const char *name, const char *ns)
+{
+  ostringstream oss;
+  oss << name << " " << ns;
+  open_array_section(oss.str().c_str());
+}
+
+void RGWFormatter_Plain::open_object_section(const char *name)
+{
+  struct plain_stack_entry new_entry;
+  new_entry.is_array = false;
+  new_entry.size = 0;
+  stack.push_back(new_entry);
+}
+
+void RGWFormatter_Plain::open_object_section_in_ns(const char *name,
+						   const char *ns)
+{
+  ostringstream oss;
+  oss << name << " " << ns;
+  open_object_section(oss.str().c_str());
+}
+
+void RGWFormatter_Plain::close_section()
+{
+  stack.pop_back();
+}
+
+void RGWFormatter_Plain::dump_unsigned(const char *name, uint64_t u)
+{
+  dump_value_int(name, "%"PRIu64, u);
+}
+
+void RGWFormatter_Plain::dump_int(const char *name, int64_t u)
+{
+  dump_value_int(name, "%"PRId64, u);
+}
+
+void RGWFormatter_Plain::dump_float(const char *name, double d)
+{
+  dump_value_int(name, "%f", d);
+}
+
+void RGWFormatter_Plain::dump_string(const char *name, std::string s)
+{
+  dump_format(name, "%s", s.c_str());
+}
+
+std::ostream& RGWFormatter_Plain::dump_stream(const char *name)
+{
+  // TODO: implement this!
+  assert(0);
+}
+
+void RGWFormatter_Plain::dump_format(const char *name, const char *fmt, ...)
+{
+  char buf[LARGE_SIZE];
+  va_list ap;
+  const char *format;
+
+  struct plain_stack_entry& entry = stack.back();
+
+  if (!min_stack_level)
+    min_stack_level = stack.size();
+
+  bool should_print = (stack.size() == min_stack_level && !entry.size);
+
+  entry.size++;
+
+  if (!should_print)
+    return;
+
+  va_start(ap, fmt);
+  vsnprintf(buf, LARGE_SIZE, fmt, ap);
+  va_end(ap);
+  if (len)
+    format = "\n%s";
+  else
+    format = "%s";
+
+  write_data(format, buf);
+}
+
+int RGWFormatter_Plain::get_len() const
+{
+  // don't include null termination in length
+  return (len ? len - 1 : 0);
+}
+
+void RGWFormatter_Plain::write_raw_data(const char *data)
+{
+  write_data("%s", data);
+}
+
+void RGWFormatter_Plain::write_data(const char *fmt, ...)
+{
+#define LARGE_ENOUGH_LEN 128
+  int n, size = LARGE_ENOUGH_LEN;
+  char s[size + 8];
+  char *p, *np;
+  bool p_on_stack;
+  va_list ap;
+  int pos;
+
+  p = s;
+  p_on_stack = true;
+
+  while (1) {
+    va_start(ap, fmt);
+    n = vsnprintf(p, size, fmt, ap);
+    va_end(ap);
+
+    if (n > -1 && n < size)
+      goto done;
+    /* Else try again with more space. */
+    if (n > -1)    /* glibc 2.1 */
+      size = n+1; /* precisely what is needed */
+    else           /* glibc 2.0 */
+      size *= 2;  /* twice the old size */
+    if (p_on_stack)
+      np = (char *)malloc(size + 8);
+    else
+      np = (char *)realloc(p, size + 8);
+    if (!np)
+      goto done_free;
+    p = np;
+    p_on_stack = false;
+  }
+done:
+#define LARGE_ENOUGH_BUF 4096
+  if (!buf) {
+    max_len = max(LARGE_ENOUGH_BUF, size);
+    buf = (char *)malloc(max_len);
+  }
+
+  if (len + size > max_len) {
+    max_len = len + size + LARGE_ENOUGH_BUF;
+    buf = (char *)realloc(buf, max_len);
+  }
+  if (!buf) {
+    cerr << "ERROR: RGWFormatter_Plain::write_data: failed allocating " << max_len << " bytes" << std::endl;
+    goto done_free;
+  }
+  pos = len;
+  if (len)
+    pos--; // squash null termination
+  strcpy(buf + pos, p);
+  len = pos + strlen(p) + 1;
+done_free:
+  if (!p_on_stack)
+    free(p);
+}
+
+void RGWFormatter_Plain::dump_value_int(const char *name, const char *fmt, ...)
+{
+  char buf[LARGE_SIZE];
+  va_list ap;
+
+  if (!min_stack_level)
+    min_stack_level = stack.size();
+
+  struct plain_stack_entry& entry = stack.back();
+  bool should_print = (stack.size() == min_stack_level && !entry.size);
+
+  entry.size++;
+
+  if (!should_print)
+    return;
+
+  va_start(ap, fmt);
+  vsnprintf(buf, LARGE_SIZE, fmt, ap);
+  va_end(ap);
+
+  const char *eol;
+  if (len)
+    eol = "\n";
+  else
+    eol = "";
+
+  write_data("%s%s", eol, buf);
+}
diff --git a/ceph/src/rgw/rgw_formats.h b/ceph/src/rgw/rgw_formats.h
new file mode 100644
index 00000000..3653ef4f
--- /dev/null
+++ b/ceph/src/rgw/rgw_formats.h
@@ -0,0 +1,97 @@
+#ifndef CEPH_RGW_FORMATS_H
+#define CEPH_RGW_FORMATS_H
+
+#include "common/Formatter.h"
+
+#include 
+#include 
+#include 
+
+struct plain_stack_entry {
+  int size;
+  bool is_array;
+};
+
+/* FIXME: this class is mis-named.
+ * FIXME: This was a hack to send certain swift messages.
+ * There is a much better way to do this.
+ */
+class RGWFormatter_Plain : public Formatter {
+  void reset_buf();
+public:
+  RGWFormatter_Plain();
+  virtual ~RGWFormatter_Plain();
+
+  virtual void flush(ostream& os);
+  virtual void reset();
+
+  virtual void open_array_section(const char *name);
+  virtual void open_array_section_in_ns(const char *name, const char *ns);
+  virtual void open_object_section(const char *name);
+  virtual void open_object_section_in_ns(const char *name, const char *ns);
+  virtual void close_section();
+  virtual void dump_unsigned(const char *name, uint64_t u);
+  virtual void dump_int(const char *name, int64_t u);
+  virtual void dump_float(const char *name, double d);
+  virtual void dump_string(const char *name, std::string s);
+  virtual std::ostream& dump_stream(const char *name);
+  virtual void dump_format(const char *name, const char *fmt, ...);
+  virtual void dump_format_unquoted(const char *name, const char *fmt, ...) {
+    assert(0 == "not implemented");
+  }
+  virtual int get_len() const;
+  virtual void write_raw_data(const char *data);
+
+private:
+  void write_data(const char *fmt, ...);
+  void dump_value_int(const char *name, const char *fmt, ...);
+
+  char *buf;
+  int len;
+  int max_len;
+
+  std::list stack;
+  size_t min_stack_level;
+};
+
+class RGWFormatterFlusher {
+protected:
+  Formatter *formatter;
+  bool flushed;
+  bool started;
+  virtual void do_flush() = 0;
+  virtual void do_start(int ret) {}
+  void set_formatter(Formatter *f) {
+    formatter = f;
+  }
+public:
+  RGWFormatterFlusher(Formatter *f) : formatter(f), flushed(false), started(false) {}
+  virtual ~RGWFormatterFlusher() {}
+
+  void flush() {
+    do_flush();
+    flushed = true;
+  }
+
+  virtual void start(int client_ret) {
+    if (!started)
+      do_start(client_ret);
+    started = true;
+  }
+
+  Formatter *get_formatter() { return formatter; }
+  bool did_flush() { return flushed; }
+  bool did_start() { return started; }
+};
+
+class RGWStreamFlusher : public RGWFormatterFlusher {
+  ostream& os;
+protected:
+  virtual void do_flush() {
+    formatter->flush(os);
+  }
+public:
+  RGWStreamFlusher(Formatter *f, ostream& _os) : RGWFormatterFlusher(f), os(_os) {}
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_gc.cc b/ceph/src/rgw/rgw_gc.cc
new file mode 100644
index 00000000..dab07f34
--- /dev/null
+++ b/ceph/src/rgw/rgw_gc.cc
@@ -0,0 +1,308 @@
+
+
+#include "rgw_gc.h"
+#include "include/rados/librados.hpp"
+#include "cls/rgw/cls_rgw_client.h"
+#include "cls/refcount/cls_refcount_client.h"
+#include "cls/lock/cls_lock_client.h"
+#include "auth/Crypto.h"
+
+#include 
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+using namespace librados;
+
+static string gc_oid_prefix = "gc";
+static string gc_index_lock_name = "gc_process";
+
+
+#define HASH_PRIME 7877
+
+void RGWGC::initialize(CephContext *_cct, RGWRados *_store) {
+  cct = _cct;
+  store = _store;
+
+  max_objs = cct->_conf->rgw_gc_max_objs;
+  if (max_objs > HASH_PRIME)
+    max_objs = HASH_PRIME;
+
+  obj_names = new string[max_objs];
+
+  for (int i = 0; i < max_objs; i++) {
+    obj_names[i] = gc_oid_prefix;
+    char buf[32];
+    snprintf(buf, 32, ".%d", i);
+    obj_names[i].append(buf);
+  }
+}
+
+void RGWGC::finalize()
+{
+  delete[] obj_names;
+}
+
+int RGWGC::tag_index(const string& tag)
+{
+  return ceph_str_hash_linux(tag.c_str(), tag.size()) % HASH_PRIME % max_objs;
+}
+
+void RGWGC::add_chain(ObjectWriteOperation& op, cls_rgw_obj_chain& chain, const string& tag)
+{
+  cls_rgw_gc_obj_info info;
+  info.chain = chain;
+  info.tag = tag;
+
+  cls_rgw_gc_set_entry(op, cct->_conf->rgw_gc_obj_min_wait, info);
+}
+
+int RGWGC::send_chain(cls_rgw_obj_chain& chain, const string& tag, bool sync)
+{
+  ObjectWriteOperation op;
+  add_chain(op, chain, tag);
+
+  int i = tag_index(tag);
+
+  if (sync)
+    return store->gc_operate(obj_names[i], &op);
+
+  return store->gc_aio_operate(obj_names[i], &op);
+}
+
+int RGWGC::defer_chain(const string& tag, bool sync)
+{
+  ObjectWriteOperation op;
+  cls_rgw_gc_defer_entry(op, cct->_conf->rgw_gc_obj_min_wait, tag);
+
+  int i = tag_index(tag);
+
+  if (sync)
+    return store->gc_operate(obj_names[i], &op);
+
+  return store->gc_aio_operate(obj_names[i], &op);
+}
+
+int RGWGC::remove(int index, const std::list& tags)
+{
+  ObjectWriteOperation op;
+  cls_rgw_gc_remove(op, tags);
+  return store->gc_operate(obj_names[index], &op);
+}
+
+int RGWGC::list(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated)
+{
+  result.clear();
+
+  for (; *index < cct->_conf->rgw_gc_max_objs && result.size() < max; (*index)++, marker.clear()) {
+    std::list entries;
+    int ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[*index], marker, max - result.size(), expired_only, entries, truncated);
+    if (ret == -ENOENT)
+      continue;
+    if (ret < 0)
+      return ret;
+
+    std::list::iterator iter;
+    for (iter = entries.begin(); iter != entries.end(); ++iter) {
+      result.push_back(*iter);
+    }
+
+    if (*index == cct->_conf->rgw_gc_max_objs - 1) {
+      /* we cut short here, truncated will hold the correct value */
+      return 0;
+    }
+
+    if (result.size() == max) {
+      /* close approximation, it might be that the next of the objects don't hold
+       * anything, in this case truncated should have been false, but we can find
+       * that out on the next iteration
+       */
+      *truncated = true;
+      return 0;
+    }
+
+  }
+  *truncated = false;
+
+  return 0;
+}
+
+int RGWGC::process(int index, int max_secs)
+{
+  rados::cls::lock::Lock l(gc_index_lock_name);
+  utime_t end = ceph_clock_now(g_ceph_context);
+  std::list remove_tags;
+
+  /* max_secs should be greater than zero. We don't want a zero max_secs
+   * to be translated as no timeout, since we'd then need to break the
+   * lock and that would require a manual intervention. In this case
+   * we can just wait it out. */
+  if (max_secs <= 0)
+    return -EAGAIN;
+
+  end += max_secs;
+  utime_t time(max_secs, 0);
+  l.set_duration(time);
+
+  int ret = l.lock_exclusive(&store->gc_pool_ctx, obj_names[index]);
+  if (ret == -EBUSY) { /* already locked by another gc processor */
+    dout(0) << "RGWGC::process() failed to acquire lock on " << obj_names[index] << dendl;
+    return 0;
+  }
+  if (ret < 0)
+    return ret;
+
+  string marker;
+  bool truncated;
+  IoCtx *ctx = NULL;
+  do {
+    int max = 100;
+    std::list entries;
+    ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[index], marker, max, true, entries, &truncated);
+    if (ret == -ENOENT) {
+      ret = 0;
+      goto done;
+    }
+    if (ret < 0)
+      goto done;
+
+    string last_pool;
+    ctx = new IoCtx;
+    std::list::iterator iter;
+    for (iter = entries.begin(); iter != entries.end(); ++iter) {
+      bool remove_tag;
+      cls_rgw_gc_obj_info& info = *iter;
+      std::list::iterator liter;
+      cls_rgw_obj_chain& chain = info.chain;
+
+      utime_t now = ceph_clock_now(g_ceph_context);
+      if (now >= end)
+        goto done;
+
+      remove_tag = true;
+      for (liter = chain.objs.begin(); liter != chain.objs.end(); ++liter) {
+        cls_rgw_obj& obj = *liter;
+
+        if (obj.pool != last_pool) {
+          delete ctx;
+          ctx = new IoCtx;
+	  ret = store->rados->ioctx_create(obj.pool.c_str(), *ctx);
+	  if (ret < 0) {
+	    dout(0) << "ERROR: failed to create ioctx pool=" << obj.pool << dendl;
+	    continue;
+	  }
+          last_pool = obj.pool;
+        }
+
+        ctx->locator_set_key(obj.key);
+	dout(0) << "gc::process: removing " << obj.pool << ":" << obj.oid << dendl;
+	ObjectWriteOperation op;
+	cls_refcount_put(op, info.tag, true);
+        ret = ctx->operate(obj.oid, &op);
+	if (ret == -ENOENT)
+	  ret = 0;
+        if (ret < 0) {
+          remove_tag = false;
+          dout(0) << "failed to remove " << obj.pool << ":" << obj.oid << "@" << obj.key << dendl;
+        }
+
+        if (going_down()) // leave early, even if tag isn't removed, it's ok
+          goto done;
+      }
+      if (remove_tag) {
+        remove_tags.push_back(info.tag);
+#define MAX_REMOVE_CHUNK 16
+        if (remove_tags.size() > MAX_REMOVE_CHUNK) {
+          remove(index, remove_tags);
+          remove_tags.clear();
+        }
+      }
+    }
+  } while (truncated);
+
+done:
+  if (!remove_tags.empty())
+    remove(index, remove_tags);
+  l.unlock(&store->gc_pool_ctx, obj_names[index]);
+  delete ctx;
+  return 0;
+}
+
+int RGWGC::process()
+{
+  int max_objs = cct->_conf->rgw_gc_max_objs;
+  int max_secs = cct->_conf->rgw_gc_processor_max_time;
+
+  unsigned start;
+  int ret = get_random_bytes((char *)&start, sizeof(start));
+  if (ret < 0)
+    return ret;
+
+  for (int i = 0; i < max_objs; i++) {
+    int index = (i + start) % max_objs;
+    ret = process(index, max_secs);
+    if (ret < 0)
+      return ret;
+  }
+
+  return 0;
+}
+
+bool RGWGC::going_down()
+{
+  return (down_flag.read() != 0);
+}
+
+void RGWGC::start_processor()
+{
+  worker = new GCWorker(cct, this);
+  worker->create();
+}
+
+void RGWGC::stop_processor()
+{
+  down_flag.set(1);
+  if (worker) {
+    worker->stop();
+    worker->join();
+  }
+  delete worker;
+  worker = NULL;
+}
+
+void *RGWGC::GCWorker::entry() {
+  do {
+    utime_t start = ceph_clock_now(cct);
+    dout(2) << "garbage collection: start" << dendl;
+    int r = gc->process();
+    if (r < 0) {
+      dout(0) << "ERROR: garbage collection process() returned error r=" << r << dendl;
+    }
+    dout(2) << "garbage collection: stop" << dendl;
+
+    if (gc->going_down())
+      break;
+
+    utime_t end = ceph_clock_now(cct);
+    end -= start;
+    int secs = cct->_conf->rgw_gc_processor_period;
+
+    if (secs <= end.sec())
+      continue; // next round
+
+    secs -= end.sec();
+
+    lock.Lock();
+    cond.WaitInterval(cct, lock, utime_t(secs, 0));
+    lock.Unlock();
+  } while (!gc->going_down());
+
+  return NULL;
+}
+
+void RGWGC::GCWorker::stop()
+{
+  Mutex::Locker l(lock);
+  cond.Signal();
+}
+
diff --git a/ceph/src/rgw/rgw_gc.h b/ceph/src/rgw/rgw_gc.h
new file mode 100644
index 00000000..afaead49
--- /dev/null
+++ b/ceph/src/rgw/rgw_gc.h
@@ -0,0 +1,63 @@
+#ifndef CEPH_RGW_GC_H
+#define CEPH_RGW_GC_H
+
+
+#include "include/types.h"
+#include "include/atomic.h"
+#include "include/rados/librados.hpp"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/Thread.h"
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "cls/rgw/cls_rgw_types.h"
+
+class RGWGC {
+  CephContext *cct;
+  RGWRados *store;
+  int max_objs;
+  string *obj_names;
+  atomic_t down_flag;
+
+  int tag_index(const string& tag);
+
+  class GCWorker : public Thread {
+    CephContext *cct;
+    RGWGC *gc;
+    Mutex lock;
+    Cond cond;
+
+  public:
+    GCWorker(CephContext *_cct, RGWGC *_gc) : cct(_cct), gc(_gc), lock("GCWorker") {}
+    void *entry();
+    void stop();
+  };
+
+  GCWorker *worker;
+public:
+  RGWGC() : cct(NULL), store(NULL), max_objs(0), obj_names(NULL), worker(NULL) {}
+  ~RGWGC() {
+    stop_processor();
+    finalize();
+  }
+
+  void add_chain(librados::ObjectWriteOperation& op, cls_rgw_obj_chain& chain, const string& tag);
+  int send_chain(cls_rgw_obj_chain& chain, const string& tag, bool sync);
+  int defer_chain(const string& tag, bool sync);
+  int remove(int index, const std::list& tags);
+
+  void initialize(CephContext *_cct, RGWRados *_store);
+  void finalize();
+
+  int list(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated);
+  void list_init(int *index) { *index = 0; }
+  int process(int index, int process_max_secs);
+  int process();
+
+  bool going_down();
+  void start_processor();
+  void stop_processor();
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_http_client.cc b/ceph/src/rgw/rgw_http_client.cc
new file mode 100644
index 00000000..1c6b6d4d
--- /dev/null
+++ b/ceph/src/rgw/rgw_http_client.cc
@@ -0,0 +1,289 @@
+#include 
+#include 
+#include 
+
+#include "rgw_common.h"
+#include "rgw_http_client.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+static size_t receive_http_header(void *ptr, size_t size, size_t nmemb, void *_info)
+{
+  RGWHTTPClient *client = static_cast(_info);
+  size_t len = size * nmemb;
+  int ret = client->receive_header(ptr, size * nmemb);
+  if (ret < 0) {
+    dout(0) << "WARNING: client->receive_header() returned ret=" << ret << dendl;
+  }
+
+  return len;
+}
+
+static size_t receive_http_data(void *ptr, size_t size, size_t nmemb, void *_info)
+{
+  RGWHTTPClient *client = static_cast(_info);
+  size_t len = size * nmemb;
+  int ret = client->receive_data(ptr, size * nmemb);
+  if (ret < 0) {
+    dout(0) << "WARNING: client->receive_data() returned ret=" << ret << dendl;
+  }
+
+  return len;
+}
+
+static size_t send_http_data(void *ptr, size_t size, size_t nmemb, void *_info)
+{
+  RGWHTTPClient *client = static_cast(_info);
+  int ret = client->send_data(ptr, size * nmemb);
+  if (ret < 0) {
+    dout(0) << "WARNING: client->receive_data() returned ret=" << ret << dendl;
+  }
+
+  return ret;
+}
+
+int RGWHTTPClient::process(const char *method, const char *url)
+{
+  int ret = 0;
+  CURL *curl_handle;
+
+  char error_buf[CURL_ERROR_SIZE];
+
+  curl_handle = curl_easy_init();
+
+  dout(20) << "sending request to " << url << dendl;
+
+  curl_slist *h = NULL;
+
+  list >::iterator iter;
+  for (iter = headers.begin(); iter != headers.end(); ++iter) {
+    pair& p = *iter;
+    string val = p.first;
+
+    if (strncmp(val.c_str(), "HTTP_", 5) == 0) {
+      val = val.substr(5);
+    }
+    val.append(": ");
+    val.append(p.second);
+    h = curl_slist_append(h, val.c_str());
+  }
+
+  curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, method);
+  curl_easy_setopt(curl_handle, CURLOPT_URL, url);
+  curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L);
+  curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);
+  curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, receive_http_header);
+  curl_easy_setopt(curl_handle, CURLOPT_WRITEHEADER, (void *)this);
+  curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, receive_http_data);
+  curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)this);
+  curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, (void *)error_buf);
+  if (h) {
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, (void *)h);
+  }
+  curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, send_http_data);
+  curl_easy_setopt(curl_handle, CURLOPT_READDATA, (void *)this);
+  curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L); 
+  if (has_send_len) {
+    curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE, (void *)send_len); 
+  }
+  CURLcode status = curl_easy_perform(curl_handle);
+  if (status) {
+    dout(0) << "curl_easy_performed returned error: " << error_buf << dendl;
+    ret = -EINVAL;
+  }
+  curl_easy_cleanup(curl_handle);
+  curl_slist_free_all(h);
+
+  return ret;
+}
+
+struct multi_req_data {
+  CURL *easy_handle;
+  CURLM *multi_handle;
+  curl_slist *h;
+
+  multi_req_data() : easy_handle(NULL), multi_handle(NULL), h(NULL) {}
+  ~multi_req_data() {
+    if (multi_handle)
+      curl_multi_cleanup(multi_handle);
+
+    if (easy_handle)
+      curl_easy_cleanup(easy_handle);
+
+    if (h)
+      curl_slist_free_all(h);
+  }
+};
+
+int RGWHTTPClient::init_async(const char *method, const char *url, void **handle)
+{
+  CURL *easy_handle;
+  CURLM *multi_handle;
+  multi_req_data *req_data = new multi_req_data;
+  *handle = (void *)req_data;
+
+  char error_buf[CURL_ERROR_SIZE];
+
+  multi_handle = curl_multi_init();
+  easy_handle = curl_easy_init();
+
+  req_data->multi_handle = multi_handle;
+  req_data->easy_handle = easy_handle;
+
+  CURLMcode mstatus = curl_multi_add_handle(multi_handle, easy_handle);
+  if (mstatus) {
+    dout(0) << "ERROR: failed on curl_multi_add_handle, status=" << mstatus << dendl;
+    delete req_data;
+    return -EIO;
+  }
+
+  dout(20) << "sending request to " << url << dendl;
+
+  curl_slist *h = NULL;
+
+  list >::iterator iter;
+  for (iter = headers.begin(); iter != headers.end(); ++iter) {
+    pair& p = *iter;
+    string val = p.first;
+
+    if (strncmp(val.c_str(), "HTTP_", 5) == 0) {
+      val = val.substr(5);
+    }
+    val.append(": ");
+    val.append(p.second);
+    h = curl_slist_append(h, val.c_str());
+  }
+
+  req_data->h = h;
+
+  curl_easy_setopt(easy_handle, CURLOPT_CUSTOMREQUEST, method);
+  curl_easy_setopt(easy_handle, CURLOPT_URL, url);
+  curl_easy_setopt(easy_handle, CURLOPT_NOPROGRESS, 1L);
+  curl_easy_setopt(easy_handle, CURLOPT_NOSIGNAL, 1L);
+  curl_easy_setopt(easy_handle, CURLOPT_HEADERFUNCTION, receive_http_header);
+  curl_easy_setopt(easy_handle, CURLOPT_WRITEHEADER, (void *)this);
+  curl_easy_setopt(easy_handle, CURLOPT_WRITEFUNCTION, receive_http_data);
+  curl_easy_setopt(easy_handle, CURLOPT_WRITEDATA, (void *)this);
+  curl_easy_setopt(easy_handle, CURLOPT_ERRORBUFFER, (void *)error_buf);
+  if (h) {
+    curl_easy_setopt(easy_handle, CURLOPT_HTTPHEADER, (void *)h);
+  }
+  curl_easy_setopt(easy_handle, CURLOPT_READFUNCTION, send_http_data);
+  curl_easy_setopt(easy_handle, CURLOPT_READDATA, (void *)this);
+  curl_easy_setopt(easy_handle, CURLOPT_UPLOAD, 1L); 
+  if (has_send_len) {
+    curl_easy_setopt(easy_handle, CURLOPT_INFILESIZE, (void *)send_len); 
+  }
+
+  return 0;
+}
+
+#if HAVE_CURL_MULTI_WAIT
+
+static int do_curl_wait(CephContext *cct, CURLM *handle)
+{
+  int num_fds;
+  int ret = curl_multi_wait(handle, NULL, 0, cct->_conf->rgw_curl_wait_timeout_ms, &num_fds);
+  if (ret) {
+    dout(0) << "ERROR: curl_multi_wait() returned " << ret << dendl;
+    return -EIO;
+  }
+  return 0;
+}
+
+#else
+
+static int do_curl_wait(CephContext *cct, CURLM *handle)
+{
+  fd_set fdread;
+  fd_set fdwrite;
+  fd_set fdexcep;
+  int maxfd = -1;
+ 
+  FD_ZERO(&fdread);
+  FD_ZERO(&fdwrite);
+  FD_ZERO(&fdexcep);
+
+  /* get file descriptors from the transfers */ 
+  int ret = curl_multi_fdset(handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+  if (ret) {
+    dout(0) << "ERROR: curl_multi_fdset returned " << ret << dendl;
+    return -EIO;
+  }
+
+  /* forcing a strict timeout, as the returned fdsets might not reference all fds we wait on */
+  uint64_t to = cct->_conf->rgw_curl_wait_timeout_ms;
+#define RGW_CURL_TIMEOUT 1000
+  if (!to)
+    to = RGW_CURL_TIMEOUT;
+  struct timeval timeout;
+  timeout.tv_sec = to / 1000;
+  timeout.tv_usec = to % 1000;
+
+  ret = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
+  if (ret < 0) {
+    ret = -errno;
+    dout(0) << "ERROR: select returned " << ret << dendl;
+    return ret;
+  }
+
+  return 0;
+}
+
+#endif
+
+int RGWHTTPClient::process_request(void *handle, bool wait_for_data, bool *done)
+{
+  multi_req_data *req_data = static_cast(handle);
+  int still_running;
+  int mstatus;
+
+  do {
+    if (wait_for_data) {
+      int ret = do_curl_wait(cct, req_data->multi_handle);
+      if (ret < 0) {
+        return ret;
+      }
+    }
+
+    mstatus = curl_multi_perform(req_data->multi_handle, &still_running);
+    dout(20) << "curl_multi_perform returned: " << mstatus << dendl;
+    switch (mstatus) {
+      case CURLM_OK:
+      case CURLM_CALL_MULTI_PERFORM:
+        break;
+      default:
+        return -EINVAL;
+    }
+    int msgs_left;
+    CURLMsg *msg;
+    while ((msg = curl_multi_info_read(req_data->multi_handle, &msgs_left))) {
+      if (msg->msg == CURLMSG_DONE) {
+        switch (msg->data.result) {
+          case CURLE_OK:
+            break;
+          default:
+            dout(20) << "ERROR: msg->data.result=" << msg->data.result << dendl;
+            return -EIO;
+        }
+      }
+    }
+  } while (mstatus == CURLM_CALL_MULTI_PERFORM);
+
+  *done = (still_running == 0);
+
+  return 0;
+}
+
+int RGWHTTPClient::complete_request(void *handle)
+{
+  bool done;
+  int ret;
+  do {
+    ret = process_request(handle, true, &done);
+  } while (!done && !ret);
+  multi_req_data *req_data = static_cast(handle);
+  delete req_data;
+
+  return ret;
+}
diff --git a/ceph/src/rgw/rgw_http_client.h b/ceph/src/rgw/rgw_http_client.h
new file mode 100644
index 00000000..3c004042
--- /dev/null
+++ b/ceph/src/rgw/rgw_http_client.h
@@ -0,0 +1,41 @@
+#ifndef CEPH_RGW_HTTP_CLIENT_H
+#define CEPH_RGW_HTTP_CLIENT_H
+
+#include "rgw_common.h"
+
+class RGWHTTPClient
+{
+  bufferlist send_bl;
+  bufferlist::iterator send_iter;
+  size_t send_len;
+  bool has_send_len;
+protected:
+  CephContext *cct;
+
+  list > headers;
+public:
+  virtual ~RGWHTTPClient() {}
+  RGWHTTPClient(CephContext *_cct): send_len (0), has_send_len(false), cct(_cct) {}
+
+  void append_header(const string& name, const string& val) {
+    headers.push_back(pair(name, val));
+  }
+
+  virtual int receive_header(void *ptr, size_t len) = 0;
+  virtual int receive_data(void *ptr, size_t len) = 0;
+  virtual int send_data(void *ptr, size_t len) = 0;
+
+  void set_send_length(size_t len) {
+    send_len = len;
+    has_send_len = true;
+  }
+
+  int process(const char *method, const char *url);
+  int process(const char *url) { return process("GET", url); }
+
+  int init_async(const char *method, const char *url, void **handle);
+  int process_request(void *handle, bool wait_for_data, bool *done);
+  int complete_request(void *handle);
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_http_errors.h b/ceph/src/rgw/rgw_http_errors.h
new file mode 100644
index 00000000..0ab19add
--- /dev/null
+++ b/ceph/src/rgw/rgw_http_errors.h
@@ -0,0 +1,142 @@
+#ifndef RGW_HTTP_ERRORS_H_
+#define RGW_HTTP_ERRORS_H_
+
+#include "rgw_common.h"
+
+struct rgw_http_errors {
+  int err_no;
+  int http_ret;
+  const char *s3_code;
+};
+
+const static struct rgw_http_errors RGW_HTTP_ERRORS[] = {
+    { 0, 200, "" },
+    { STATUS_CREATED, 201, "Created" },
+    { STATUS_ACCEPTED, 202, "Accepted" },
+    { STATUS_NO_CONTENT, 204, "NoContent" },
+    { STATUS_PARTIAL_CONTENT, 206, "" },
+    { ERR_PERMANENT_REDIRECT, 301, "PermanentRedirect" },
+    { STATUS_REDIRECT, 303, "" },
+    { ERR_NOT_MODIFIED, 304, "NotModified" },
+    { EINVAL, 400, "InvalidArgument" },
+    { ERR_INVALID_REQUEST, 400, "InvalidRequest" },
+    { ERR_INVALID_DIGEST, 400, "InvalidDigest" },
+    { ERR_BAD_DIGEST, 400, "BadDigest" },
+    { ERR_INVALID_BUCKET_NAME, 400, "InvalidBucketName" },
+    { ERR_INVALID_OBJECT_NAME, 400, "InvalidObjectName" },
+    { ERR_UNRESOLVABLE_EMAIL, 400, "UnresolvableGrantByEmailAddress" },
+    { ERR_INVALID_PART, 400, "InvalidPart" },
+    { ERR_INVALID_PART_ORDER, 400, "InvalidPartOrder" },
+    { ERR_REQUEST_TIMEOUT, 400, "RequestTimeout" },
+    { ERR_TOO_LARGE, 400, "EntityTooLarge" },
+    { ERR_TOO_SMALL, 400, "EntityTooSmall" },
+    { ERR_TOO_MANY_BUCKETS, 400, "TooManyBuckets" },
+    { ERR_LENGTH_REQUIRED, 411, "MissingContentLength" },
+    { EACCES, 403, "AccessDenied" },
+    { EPERM, 403, "AccessDenied" },
+    { ERR_USER_SUSPENDED, 403, "UserSuspended" },
+    { ERR_REQUEST_TIME_SKEWED, 403, "RequestTimeTooSkewed" },
+    { ERR_QUOTA_EXCEEDED, 403, "QuotaExceeded" },
+    { ENOENT, 404, "NoSuchKey" },
+    { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket" },
+    { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload" },
+    { ERR_NOT_FOUND, 404, "Not Found"},
+    { ERR_METHOD_NOT_ALLOWED, 405, "MethodNotAllowed" },
+    { ETIMEDOUT, 408, "RequestTimeout" },
+    { EEXIST, 409, "BucketAlreadyExists" },
+    { ENOTEMPTY, 409, "BucketNotEmpty" },
+    { ERR_PRECONDITION_FAILED, 412, "PreconditionFailed" },
+    { ERANGE, 416, "InvalidRange" },
+    { ERR_UNPROCESSABLE_ENTITY, 422, "UnprocessableEntity" },
+    { ERR_LOCKED, 423, "Locked" },
+    { ERR_INTERNAL_ERROR, 500, "InternalError" },
+};
+
+const static struct rgw_http_errors RGW_HTTP_SWIFT_ERRORS[] = {
+    { EACCES, 401, "AccessDenied" },
+    { EPERM, 401, "AccessDenied" },
+    { ERR_USER_SUSPENDED, 401, "UserSuspended" },
+    { ERR_INVALID_UTF8, 412, "Invalid UTF8" },
+    { ERR_BAD_URL, 412, "Bad URL" },
+};
+
+struct rgw_http_status_code {
+  int code;
+  const char *name;
+};
+
+const static struct rgw_http_status_code http_codes[] = {
+  { 100, "Continue" },
+  { 200, "OK" },
+  { 201, "Created" },
+  { 202, "Accepted" },
+  { 204, "No Content" },
+  { 205, "Reset Content" },
+  { 206, "Partial Content" },
+  { 207, "Multi Status" },
+  { 208, "Already Reported" },
+  { 300, "Multiple Choices" },
+  { 302, "Found" },
+  { 303, "See Other" },
+  { 304, "Not Modified" },
+  { 305, "User Proxy" },
+  { 306, "Switch Proxy" },
+  { 307, "Temporary Redirect" },
+  { 308, "Permanent Redirect" },
+  { 400, "Bad Request" },
+  { 401, "Unauthorized" },
+  { 402, "Payment Required" },
+  { 403, "Forbidden" },
+  { 404, "Not Found" },
+  { 405, "Method Not Allowed" },
+  { 406, "Not Acceptable" },
+  { 407, "Proxy Authentication Required" },
+  { 408, "Request Timeout" },
+  { 409, "Conflict" },
+  { 410, "Gone" },
+  { 411, "Length Required" },
+  { 412, "Precondition Failed" },
+  { 413, "Request Entity Too Large" },
+  { 414, "Request-URI Too Long" },
+  { 415, "Unsupported Media Type" },
+  { 416, "Requested Range Not Satisfiable" },
+  { 417, "Expectation Failed" },
+  { 422, "Unprocessable Entity" },
+  { 500, "Internal Server Error" },
+  { 0, NULL },
+};
+
+#define ARRAY_LEN(arr) (sizeof(arr) / sizeof(arr[0]))
+
+static inline const struct rgw_http_errors *search_err(int err_no, const struct rgw_http_errors *errs, int len)
+{
+  for (int i = 0; i < len; ++i, ++errs) {
+    if (err_no == errs->err_no)
+      return errs;
+  }
+  return NULL;
+}
+
+
+static inline int rgw_http_error_to_errno(int http_err)
+{
+  if (http_err >= 200 && http_err <= 299)
+    return 0;
+  switch (http_err) {
+    case 400:
+      return -EINVAL;
+    case 401:
+      return -EPERM;
+    case 403:
+        return -EACCES;
+    case 404:
+        return -ENOENT;
+    default:
+        return -EIO;
+  }
+
+  return 0; /* unreachable */
+}
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_json_enc.cc b/ceph/src/rgw/rgw_json_enc.cc
new file mode 100644
index 00000000..a1986782
--- /dev/null
+++ b/ceph/src/rgw/rgw_json_enc.cc
@@ -0,0 +1,819 @@
+
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "rgw_log.h"
+#include "rgw_acl.h"
+#include "rgw_acl_s3.h"
+#include "rgw_cache.h"
+#include "rgw_bucket.h"
+#include "rgw_keystone.h"
+
+#include "common/ceph_json.h"
+#include "common/Formatter.h"
+
+void encode_json(const char *name, const obj_version& v, Formatter *f)
+{
+  f->open_object_section(name);
+  f->dump_string("tag", v.tag);
+  f->dump_unsigned("ver", v.ver);
+  f->close_section();
+}
+
+void decode_json_obj(obj_version& v, JSONObj *obj)
+{
+  JSONDecoder::decode_json("tag", v.tag, obj);
+  JSONDecoder::decode_json("ver", v.ver, obj);
+}
+
+void encode_json(const char *name, const RGWUserCaps& val, Formatter *f)
+{
+  val.dump(f, name);
+}
+
+
+void RGWObjManifestPart::dump(Formatter *f) const
+{
+  f->open_object_section("loc");
+  loc.dump(f);
+  f->close_section();
+  f->dump_unsigned("loc_ofs", loc_ofs);
+  f->dump_unsigned("size", size);
+}
+
+void RGWObjManifestRule::dump(Formatter *f) const
+{
+  encode_json("start_part_num", start_part_num, f);
+  encode_json("start_ofs", start_ofs, f);
+  encode_json("part_size", part_size, f);
+  encode_json("stripe_max_size", stripe_max_size, f);
+  encode_json("override_prefix", override_prefix, f);
+}
+
+void RGWObjManifest::dump(Formatter *f) const
+{
+  map::const_iterator iter = objs.begin();
+  f->open_array_section("objs");
+  for (; iter != objs.end(); ++iter) {
+    f->dump_unsigned("ofs", iter->first);
+    f->open_object_section("part");
+    iter->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->dump_unsigned("obj_size", obj_size);
+  ::encode_json("explicit_objs", explicit_objs, f);
+  ::encode_json("head_obj", head_obj, f);
+  ::encode_json("head_size", head_size, f);
+  ::encode_json("max_head_size", max_head_size, f);
+  ::encode_json("prefix", prefix, f);
+  ::encode_json("tail_bucket", tail_bucket, f);
+  ::encode_json("rules", rules, f);
+}
+
+void rgw_log_entry::dump(Formatter *f) const
+{
+  f->dump_string("object_owner", object_owner);
+  f->dump_string("bucket_owner", bucket_owner);
+  f->dump_string("bucket", bucket);
+  f->dump_stream("time") << time;
+  f->dump_string("remote_addr", remote_addr);
+  f->dump_string("user", user);
+  f->dump_string("obj", obj);
+  f->dump_string("op", op);
+  f->dump_string("uri", uri);
+  f->dump_string("http_status", http_status);
+  f->dump_string("error_code", error_code);
+  f->dump_unsigned("bytes_sent", bytes_sent);
+  f->dump_unsigned("bytes_received", bytes_received);
+  f->dump_unsigned("obj_size", obj_size);
+  f->dump_stream("total_time") << total_time;
+  f->dump_string("user_agent", user_agent);
+  f->dump_string("referrer", referrer);
+  f->dump_string("bucket_id", bucket_id);
+}
+
+void rgw_intent_log_entry::dump(Formatter *f) const
+{
+  f->open_object_section("obj");
+  obj.dump(f);
+  f->close_section();
+  f->dump_stream("op_time") << op_time;
+  f->dump_unsigned("intent", intent);
+}
+
+
+void ACLPermission::dump(Formatter *f) const
+{
+  f->dump_int("flags", flags);
+}
+
+void ACLGranteeType::dump(Formatter *f) const
+{
+  f->dump_unsigned("type", type);
+}
+
+void ACLGrant::dump(Formatter *f) const
+{
+  f->open_object_section("type");
+  type.dump(f);
+  f->close_section();
+
+  f->dump_string("id", id);
+  f->dump_string("email", email);
+
+  f->open_object_section("permission");
+  permission.dump(f);
+  f->close_section();
+
+  f->dump_string("name", name);
+  f->dump_int("group", (int)group);
+}
+
+void RGWAccessControlList::dump(Formatter *f) const
+{
+  map::const_iterator acl_user_iter = acl_user_map.begin();
+  f->open_array_section("acl_user_map");
+  for (; acl_user_iter != acl_user_map.end(); ++acl_user_iter) {
+    f->open_object_section("entry");
+    f->dump_string("user", acl_user_iter->first);
+    f->dump_int("acl", acl_user_iter->second);
+    f->close_section();
+  }
+  f->close_section();
+
+  map::const_iterator acl_group_iter = acl_group_map.begin();
+  f->open_array_section("acl_group_map");
+  for (; acl_group_iter != acl_group_map.end(); ++acl_group_iter) {
+    f->open_object_section("entry");
+    f->dump_unsigned("group", acl_group_iter->first);
+    f->dump_int("acl", acl_group_iter->second);
+    f->close_section();
+  }
+  f->close_section();
+
+  multimap::const_iterator giter = grant_map.begin();
+  f->open_array_section("grant_map");
+  for (; giter != grant_map.end(); ++giter) {
+    f->open_object_section("entry");
+    f->dump_string("id", giter->first);
+    f->open_object_section("grant");
+    giter->second.dump(f);
+    f->close_section();
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ACLOwner::dump(Formatter *f) const
+{
+  encode_json("id", id, f);
+  encode_json("display_name", display_name, f);
+}
+
+void RGWAccessControlPolicy::dump(Formatter *f) const
+{
+  encode_json("acl", acl, f);
+  encode_json("owner", owner, f);
+}
+
+void ObjectMetaInfo::dump(Formatter *f) const
+{
+  encode_json("size", size, f);
+  encode_json("mtime", mtime, f);
+}
+
+void ObjectCacheInfo::dump(Formatter *f) const
+{
+  encode_json("status", status, f);
+  encode_json("flags", flags, f);
+  encode_json("data", data, f);
+  encode_json_map("xattrs", "name", "value", "length", xattrs, f);
+  encode_json_map("rm_xattrs", "name", "value", "length", rm_xattrs, f);
+  encode_json("meta", meta, f);
+
+}
+
+void RGWCacheNotifyInfo::dump(Formatter *f) const
+{
+  encode_json("op", op, f);
+  encode_json("obj", obj, f);
+  encode_json("obj_info", obj_info, f);
+  encode_json("ofs", ofs, f);
+  encode_json("ns", ns, f);
+}
+
+void RGWAccessKey::dump(Formatter *f) const
+{
+  encode_json("access_key", id, f);
+  encode_json("secret_key", key, f);
+  encode_json("subuser", subuser, f);
+}
+
+void RGWAccessKey::dump_plain(Formatter *f) const
+{
+  encode_json("access_key", id, f);
+  encode_json("secret_key", key, f);
+}
+
+void encode_json_plain(const char *name, const RGWAccessKey& val, Formatter *f)
+{
+  f->open_object_section(name);
+  val.dump_plain(f);
+  f->close_section();
+}
+
+void RGWAccessKey::dump(Formatter *f, const string& user, bool swift) const
+{
+  string u = user;
+  if (!subuser.empty()) {
+    u.append(":");
+    u.append(subuser);
+  }
+  encode_json("user", u, f);
+  if (!swift) {
+    encode_json("access_key", id, f);
+  }
+  encode_json("secret_key", key, f);
+}
+
+void RGWAccessKey::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("access_key", id, obj, true);
+  JSONDecoder::decode_json("secret_key", key, obj, true);
+  if (!JSONDecoder::decode_json("subuser", subuser, obj)) {
+    string user;
+    JSONDecoder::decode_json("user", user, obj);
+    int pos = user.find(':');
+    if (pos >= 0) {
+      subuser = user.substr(pos + 1);
+    }
+  }
+}
+
+void RGWAccessKey::decode_json(JSONObj *obj, bool swift) {
+  if (!swift) {
+    decode_json(obj);
+    return;
+  }
+
+  if (!JSONDecoder::decode_json("subuser", subuser, obj)) {
+    JSONDecoder::decode_json("user", id, obj, true);
+    int pos = id.find(':');
+    if (pos >= 0) {
+      subuser = id.substr(pos + 1);
+    }
+  }
+  JSONDecoder::decode_json("secret_key", key, obj, true);
+}
+
+struct rgw_flags_desc {
+  uint32_t mask;
+  const char *str;
+};
+
+static struct rgw_flags_desc rgw_perms[] = {
+ { RGW_PERM_FULL_CONTROL, "full-control" },
+ { RGW_PERM_READ | RGW_PERM_WRITE, "read-write" },
+ { RGW_PERM_READ, "read" },
+ { RGW_PERM_WRITE, "write" },
+ { RGW_PERM_READ_ACP, "read-acp" },
+ { RGW_PERM_WRITE_ACP, "read-acp" },
+ { 0, NULL }
+};
+
+static void mask_to_str(rgw_flags_desc *mask_list, uint32_t mask, char *buf, int len)
+{
+  const char *sep = "";
+  int pos = 0;
+  if (!mask) {
+    snprintf(buf, len, "");
+    return;
+  }
+  while (mask) {
+    uint32_t orig_mask = mask;
+    for (int i = 0; mask_list[i].mask; i++) {
+      struct rgw_flags_desc *desc = &mask_list[i];
+      if ((mask & desc->mask) == desc->mask) {
+        pos += snprintf(buf + pos, len - pos, "%s%s", sep, desc->str);
+        if (pos == len)
+          return;
+        sep = ", ";
+        mask &= ~desc->mask;
+        if (!mask)
+          return;
+      }
+    }
+    if (mask == orig_mask) // no change
+      break;
+  }
+}
+
+static void perm_to_str(uint32_t mask, char *buf, int len)
+{
+  return mask_to_str(rgw_perms, mask, buf, len);
+}
+
+static struct rgw_flags_desc op_type_flags[] = {
+ { RGW_OP_TYPE_READ, "read" },
+ { RGW_OP_TYPE_WRITE, "write" },
+ { RGW_OP_TYPE_DELETE, "delete" },
+ { 0, NULL }
+};
+
+static void op_type_to_str(uint32_t mask, char *buf, int len)
+{
+  return mask_to_str(op_type_flags, mask, buf, len);
+}
+
+void RGWSubUser::dump(Formatter *f) const
+{
+  encode_json("id", name, f);
+  char buf[256];
+  perm_to_str(perm_mask, buf, sizeof(buf));
+  encode_json("permissions", (const char *)buf, f);
+}
+
+void RGWSubUser::dump(Formatter *f, const string& user) const
+{
+  string s = user;
+  s.append(":");
+  s.append(name);
+  encode_json("id", s, f);
+  char buf[256];
+  perm_to_str(perm_mask, buf, sizeof(buf));
+  encode_json("permissions", (const char *)buf, f);
+}
+
+static uint32_t str_to_perm(const string& s)
+{
+  if (s.compare("read") == 0)
+    return RGW_PERM_READ;
+  else if (s.compare("write") == 0)
+    return RGW_PERM_WRITE;
+  else if (s.compare("read-write") == 0)
+    return RGW_PERM_READ | RGW_PERM_WRITE;
+  else if (s.compare("full-control") == 0)
+    return RGW_PERM_FULL_CONTROL;
+  return 0;
+}
+
+void RGWSubUser::decode_json(JSONObj *obj)
+{
+  string uid;
+  JSONDecoder::decode_json("id", uid, obj);
+  int pos = uid.find(':');
+  if (pos >= 0)
+    name = uid.substr(pos + 1);
+  string perm_str;
+  JSONDecoder::decode_json("permissions", perm_str, obj);
+  perm_mask = str_to_perm(perm_str);
+}
+
+static void user_info_dump_subuser(const char *name, const RGWSubUser& subuser, Formatter *f, void *parent)
+{
+  RGWUserInfo *info = static_cast(parent);
+  subuser.dump(f, info->user_id);
+}
+
+static void user_info_dump_key(const char *name, const RGWAccessKey& key, Formatter *f, void *parent)
+{
+  RGWUserInfo *info = static_cast(parent);
+  key.dump(f, info->user_id, false);
+}
+
+static void user_info_dump_swift_key(const char *name, const RGWAccessKey& key, Formatter *f, void *parent)
+{
+  RGWUserInfo *info = static_cast(parent);
+  key.dump(f, info->user_id, true);
+}
+
+void RGWUserInfo::dump(Formatter *f) const
+{
+
+  encode_json("user_id", user_id, f);
+  encode_json("display_name", display_name, f);
+  encode_json("email", user_email, f);
+  encode_json("suspended", (int)suspended, f);
+  encode_json("max_buckets", (int)max_buckets, f);
+
+  encode_json("auid", auid, f);
+
+  encode_json_map("subusers", NULL, "subuser", NULL, user_info_dump_subuser,(void *)this, subusers, f);
+  encode_json_map("keys", NULL, "key", NULL, user_info_dump_key,(void *)this, access_keys, f);
+  encode_json_map("swift_keys", NULL, "key", NULL, user_info_dump_swift_key,(void *)this, swift_keys, f);
+
+  encode_json("caps", caps, f);
+
+  char buf[256];
+  op_type_to_str(op_mask, buf, sizeof(buf));
+  encode_json("op_mask", (const char *)buf, f);
+
+  if (system) { /* no need to show it for every user */
+    encode_json("system", (bool)system, f);
+  }
+  encode_json("default_placement", default_placement, f);
+  encode_json("placement_tags", placement_tags, f);
+  encode_json("bucket_quota", bucket_quota, f);
+  encode_json("user_quota", user_quota, f);
+  encode_json("temp_url_keys", temp_url_keys, f);
+}
+
+
+static void decode_access_keys(map& m, JSONObj *o)
+{
+  RGWAccessKey k;
+  k.decode_json(o);
+  m[k.id] = k;
+}
+
+static void decode_swift_keys(map& m, JSONObj *o)
+{
+  RGWAccessKey k;
+  k.decode_json(o, true);
+  m[k.subuser] = k;
+}
+
+static void decode_subusers(map& m, JSONObj *o)
+{
+  RGWSubUser u;
+  u.decode_json(o);
+  m[u.name] = u;
+}
+
+void RGWUserInfo::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("user_id", user_id, obj, true);
+  JSONDecoder::decode_json("display_name", display_name, obj);
+  JSONDecoder::decode_json("email", user_email, obj);
+  bool susp = false;
+  JSONDecoder::decode_json("suspended", susp, obj);
+  suspended = (__u8)susp;
+  JSONDecoder::decode_json("max_buckets", max_buckets, obj);
+  JSONDecoder::decode_json("auid", auid, obj);
+
+  JSONDecoder::decode_json("keys", access_keys, decode_access_keys, obj);
+  JSONDecoder::decode_json("swift_keys", swift_keys, decode_swift_keys, obj);
+  JSONDecoder::decode_json("subusers", subusers, decode_subusers, obj);
+
+  JSONDecoder::decode_json("caps", caps, obj);
+
+  string mask_str;
+  JSONDecoder::decode_json("op_mask", mask_str, obj);
+  rgw_parse_op_type_list(mask_str, &op_mask);
+
+  bool sys = false;
+  JSONDecoder::decode_json("system", sys, obj);
+  system = (__u8)sys;
+  JSONDecoder::decode_json("default_placement", default_placement, obj);
+  JSONDecoder::decode_json("placement_tags", placement_tags, obj);
+  JSONDecoder::decode_json("bucket_quota", bucket_quota, obj);
+  JSONDecoder::decode_json("user_quota", user_quota, obj);
+  JSONDecoder::decode_json("temp_url_keys", temp_url_keys, obj);
+}
+
+void RGWQuotaInfo::dump(Formatter *f) const
+{
+  f->dump_bool("enabled", enabled);
+  f->dump_int("max_size_kb", max_size_kb);
+  f->dump_int("max_objects", max_objects);
+}
+
+void RGWQuotaInfo::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("max_size_kb", max_size_kb, obj);
+  JSONDecoder::decode_json("max_objects", max_objects, obj);
+  JSONDecoder::decode_json("enabled", enabled, obj);
+}
+
+void rgw_bucket::dump(Formatter *f) const
+{
+  encode_json("name", name, f);
+  encode_json("pool", data_pool, f);
+  encode_json("data_extra_pool", data_extra_pool, f);
+  encode_json("index_pool", index_pool, f);
+  encode_json("marker", marker, f);
+  encode_json("bucket_id", bucket_id, f);
+}
+
+void rgw_bucket::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("name", name, obj);
+  JSONDecoder::decode_json("pool", data_pool, obj);
+  JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj);
+  JSONDecoder::decode_json("index_pool", index_pool, obj);
+  JSONDecoder::decode_json("marker", marker, obj);
+  JSONDecoder::decode_json("bucket_id", bucket_id, obj);
+}
+
+void RGWBucketEntryPoint::dump(Formatter *f) const
+{
+  encode_json("bucket", bucket, f);
+  encode_json("owner", owner, f);
+  encode_json("creation_time", creation_time, f);
+  encode_json("linked", linked, f);
+  encode_json("has_bucket_info", has_bucket_info, f);
+  if (has_bucket_info) {
+    encode_json("old_bucket_info", old_bucket_info, f);
+  }
+}
+
+void RGWBucketEntryPoint::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("bucket", bucket, obj);
+  JSONDecoder::decode_json("owner", owner, obj);
+  JSONDecoder::decode_json("creation_time", creation_time, obj);
+  JSONDecoder::decode_json("linked", linked, obj);
+  JSONDecoder::decode_json("has_bucket_info", has_bucket_info, obj);
+  if (has_bucket_info) {
+    JSONDecoder::decode_json("old_bucket_info", old_bucket_info, obj);
+  }
+}
+
+void RGWBucketInfo::dump(Formatter *f) const
+{
+  encode_json("bucket", bucket, f);
+  encode_json("creation_time", creation_time, f);
+  encode_json("owner", owner, f);
+  encode_json("flags", flags, f);
+  encode_json("region", region, f);
+  encode_json("placement_rule", placement_rule, f);
+  encode_json("has_instance_obj", has_instance_obj, f);
+  encode_json("quota", quota, f);
+}
+
+void RGWBucketInfo::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("bucket", bucket, obj);
+  JSONDecoder::decode_json("creation_time", creation_time, obj);
+  JSONDecoder::decode_json("owner", owner, obj);
+  JSONDecoder::decode_json("flags", flags, obj);
+  JSONDecoder::decode_json("region", region, obj);
+  JSONDecoder::decode_json("placement_rule", placement_rule, obj);
+  JSONDecoder::decode_json("has_instance_obj", has_instance_obj, obj);
+  JSONDecoder::decode_json("quota", quota, obj);
+}
+
+void RGWObjEnt::dump(Formatter *f) const
+{
+  encode_json("name", name, f);
+  encode_json("namespace", ns, f);
+  encode_json("owner", owner, f);
+  encode_json("owner_display_name", owner_display_name, f);
+  encode_json("size", size, f);
+  encode_json("mtime", mtime, f);
+  encode_json("etag", etag, f);
+  encode_json("content_type", content_type, f);
+  encode_json("tag", tag, f);
+}
+
+void RGWBucketEnt::dump(Formatter *f) const
+{
+  encode_json("bucket", bucket, f);
+  encode_json("size", size, f);
+  encode_json("size_rounded", size_rounded, f);
+  encode_json("mtime", creation_time, f); /* mtime / creation time discrepency needed for backward compatibility */
+  encode_json("count", count, f);
+}
+
+void RGWUploadPartInfo::dump(Formatter *f) const
+{
+  encode_json("num", num, f);
+  encode_json("size", size, f);
+  encode_json("etag", etag, f);
+  encode_json("modified", modified, f);
+}
+
+void rgw_obj::dump(Formatter *f) const
+{
+  encode_json("bucket", bucket, f);
+  encode_json("key", key, f);
+  encode_json("ns", ns, f);
+  encode_json("object", object, f);
+}
+
+void RGWZoneParams::dump(Formatter *f) const
+{
+  encode_json("domain_root", domain_root.data_pool, f);
+  encode_json("control_pool", control_pool.data_pool, f);
+  encode_json("gc_pool", gc_pool.data_pool, f);
+  encode_json("log_pool", log_pool.data_pool, f);
+  encode_json("intent_log_pool", intent_log_pool.data_pool, f);
+  encode_json("usage_log_pool", usage_log_pool.data_pool, f);
+  encode_json("user_keys_pool", user_keys_pool.data_pool, f);
+  encode_json("user_email_pool", user_email_pool.data_pool, f);
+  encode_json("user_swift_pool", user_swift_pool.data_pool, f);
+  encode_json("user_uid_pool", user_uid_pool.data_pool, f);
+  encode_json_plain("system_key", system_key, f);
+  encode_json("placement_pools", placement_pools, f);
+}
+
+static void decode_json(const char *field, rgw_bucket& bucket, JSONObj *obj)
+{
+  string pool;
+  JSONDecoder::decode_json(field, pool, obj);
+  if (pool[0] != '.') {
+    pool = string(".") + pool;
+  }
+  bucket = rgw_bucket(pool.c_str());
+}
+
+void RGWZonePlacementInfo::dump(Formatter *f) const
+{
+  encode_json("index_pool", index_pool, f);
+  encode_json("data_pool", data_pool, f);
+  encode_json("data_extra_pool", data_extra_pool, f);
+}
+
+void RGWZonePlacementInfo::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("index_pool", index_pool, obj);
+  JSONDecoder::decode_json("data_pool", data_pool, obj);
+  JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj);
+}
+
+void RGWZoneParams::decode_json(JSONObj *obj)
+{
+  ::decode_json("domain_root", domain_root, obj);
+  ::decode_json("control_pool", control_pool, obj);
+  ::decode_json("gc_pool", gc_pool, obj);
+  ::decode_json("log_pool", log_pool, obj);
+  ::decode_json("intent_log_pool", intent_log_pool, obj);
+  ::decode_json("usage_log_pool", usage_log_pool, obj);
+  ::decode_json("user_keys_pool", user_keys_pool, obj);
+  ::decode_json("user_email_pool", user_email_pool, obj);
+  ::decode_json("user_uid_pool", user_uid_pool, obj);
+  ::decode_json("user_swift_pool", user_swift_pool, obj);
+  JSONDecoder::decode_json("system_key", system_key, obj);
+  JSONDecoder::decode_json("placement_pools", placement_pools, obj);
+}
+
+void RGWZone::dump(Formatter *f) const
+{
+  encode_json("name", name, f);
+  encode_json("endpoints", endpoints, f);
+  encode_json("log_meta", log_meta, f);
+  encode_json("log_data", log_data, f);
+}
+
+void RGWZone::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("name", name, obj);
+  JSONDecoder::decode_json("endpoints", endpoints, obj);
+  JSONDecoder::decode_json("log_meta", log_meta, obj);
+  JSONDecoder::decode_json("log_data", log_data, obj);
+}
+
+void RGWRegionPlacementTarget::dump(Formatter *f) const
+{
+  encode_json("name", name, f);
+  encode_json("tags", tags, f);
+}
+
+void RGWRegionPlacementTarget::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("name", name, obj);
+  JSONDecoder::decode_json("tags", tags, obj);
+}
+
+void RGWRegion::dump(Formatter *f) const
+{
+  encode_json("name", name, f);
+  encode_json("api_name", api_name, f);
+  encode_json("is_master", is_master, f);
+  encode_json("endpoints", endpoints, f);
+  encode_json("master_zone", master_zone, f);
+  encode_json_map("zones", zones, f); /* more friendly representation */
+  encode_json_map("placement_targets", placement_targets, f); /* more friendly representation */
+  encode_json("default_placement", default_placement, f);
+}
+
+static void decode_zones(map& zones, JSONObj *o)
+{
+  RGWZone z;
+  z.decode_json(o);
+  zones[z.name] = z;
+}
+
+static void decode_placement_targets(map& targets, JSONObj *o)
+{
+  RGWRegionPlacementTarget t;
+  t.decode_json(o);
+  targets[t.name] = t;
+}
+
+
+void RGWRegion::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("name", name, obj);
+  JSONDecoder::decode_json("api_name", api_name, obj);
+  JSONDecoder::decode_json("is_master", is_master, obj);
+  JSONDecoder::decode_json("endpoints", endpoints, obj);
+  JSONDecoder::decode_json("master_zone", master_zone, obj);
+  JSONDecoder::decode_json("zones", zones, decode_zones, obj);
+  JSONDecoder::decode_json("placement_targets", placement_targets, decode_placement_targets, obj);
+  JSONDecoder::decode_json("default_placement", default_placement, obj);
+}
+
+
+void RGWRegionMap::dump(Formatter *f) const
+{
+  encode_json("regions", regions, f);
+  encode_json("master_region", master_region, f);
+  encode_json("bucket_quota", bucket_quota, f);
+  encode_json("user_quota", user_quota, f);
+}
+
+void RGWRegionMap::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("regions", regions, obj);
+  JSONDecoder::decode_json("master_region", master_region, obj);
+  JSONDecoder::decode_json("user_quota", user_quota, obj);
+}
+
+void RGWMetadataLogInfo::dump(Formatter *f) const
+{
+  encode_json("marker", marker, f);
+  encode_json("last_update", last_update, f);
+}
+
+void RGWMetadataLogInfo::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("marker", marker, obj);
+  JSONDecoder::decode_json("last_update", last_update, obj);
+}
+
+void RGWDataChangesLogInfo::dump(Formatter *f) const
+{
+  encode_json("marker", marker, f);
+  encode_json("last_update", last_update, f);
+}
+
+void RGWDataChangesLogInfo::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("marker", marker, obj);
+  JSONDecoder::decode_json("last_update", last_update, obj);
+}
+
+void KeystoneToken::Metadata::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("is_admin", is_admin, obj);
+}
+
+void KeystoneToken::Service::Endpoint::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("id", id, obj);
+  JSONDecoder::decode_json("adminURL", admin_url, obj);
+  JSONDecoder::decode_json("publicURL", public_url, obj);
+  JSONDecoder::decode_json("internalURL", internal_url, obj);
+  JSONDecoder::decode_json("region", region, obj);
+}
+
+void KeystoneToken::Service::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("type", type, obj, true);
+  JSONDecoder::decode_json("name", name, obj, true);
+  JSONDecoder::decode_json("endpoints", endpoints, obj);
+}
+
+void KeystoneToken::Token::Tenant::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("id", id, obj, true);
+  JSONDecoder::decode_json("name", name, obj, true);
+  JSONDecoder::decode_json("description", description, obj);
+  JSONDecoder::decode_json("enabled", enabled, obj);
+}
+
+void KeystoneToken::Token::decode_json(JSONObj *obj)
+{
+  string expires_iso8601;
+  struct tm t;
+
+  JSONDecoder::decode_json("id", id, obj, true);
+  JSONDecoder::decode_json("tenant", tenant, obj, true);
+  JSONDecoder::decode_json("expires", expires_iso8601, obj, true);
+
+  if (parse_iso8601(expires_iso8601.c_str(), &t)) {
+    expires = timegm(&t);
+  } else {
+    expires = 0;
+    throw JSONDecoder::err("Failed to parse ISO8601 expiration date from Keystone response.");
+  }
+}
+
+void KeystoneToken::User::Role::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("id", id, obj);
+  JSONDecoder::decode_json("name", name, obj);
+}
+
+void KeystoneToken::User::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("id", id, obj, true);
+  JSONDecoder::decode_json("name", name, obj);
+  JSONDecoder::decode_json("username", user_name, obj, true);
+  JSONDecoder::decode_json("roles", roles, obj);
+}
+
+void KeystoneToken::decode_json(JSONObj *access_obj)
+{
+  JSONDecoder::decode_json("metadata", metadata, access_obj);
+  JSONDecoder::decode_json("token", token, access_obj, true);
+  JSONDecoder::decode_json("user", user, access_obj, true);
+  JSONDecoder::decode_json("serviceCatalog", service_catalog, access_obj);
+}
diff --git a/ceph/src/rgw/rgw_jsonparser.cc b/ceph/src/rgw/rgw_jsonparser.cc
new file mode 100644
index 00000000..ec1c4ce2
--- /dev/null
+++ b/ceph/src/rgw/rgw_jsonparser.cc
@@ -0,0 +1,130 @@
+#include 
+#include 
+
+#include 
+#include 
+
+#include "include/types.h"
+
+#include "common/Formatter.h"
+#include "common/ceph_json.h"
+
+#include "rgw_common.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+void dump_array(JSONObj *obj)
+{
+
+  JSONObjIter iter = obj->find_first();
+
+  for (; !iter.end(); ++iter) { 
+    JSONObj *o = *iter;
+    cout << "data=" << o->get_data() << std::endl;
+  }
+
+}
+                                  
+struct Key {
+  string user;
+  string access_key;
+  string secret_key;
+
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("user", user, obj);
+    JSONDecoder::decode_json("access_key", access_key, obj);
+    JSONDecoder::decode_json("secret_key", secret_key, obj);
+  }
+};
+
+struct UserInfo {
+  string uid;
+  string display_name;
+  int max_buckets;
+  list keys;
+
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("user_id", uid, obj);
+    JSONDecoder::decode_json("display_name", display_name, obj);
+    JSONDecoder::decode_json("max_buckets", max_buckets, obj);
+    JSONDecoder::decode_json("keys", keys, obj);
+  }
+};
+
+
+int main(int argc, char **argv) {
+  JSONParser parser;
+
+  char buf[1024];
+  bufferlist bl;
+
+  for (;;) {
+    int done;
+    int len;
+
+    len = fread(buf, 1, sizeof(buf), stdin);
+    if (ferror(stdin)) {
+      cerr << "read error" << std::endl;
+      exit(-1);
+    }
+    done = feof(stdin);
+
+    bool ret = parser.parse(buf, len);
+    if (!ret)
+      cerr << "parse error" << std::endl;
+
+    if (done) {
+      bl.append(buf, len);
+      break;
+    }
+  }
+
+  JSONObjIter iter = parser.find_first();
+
+  for (; !iter.end(); ++iter) { 
+    JSONObj *obj = *iter;
+    cout << "is_object=" << obj->is_object() << std::endl;
+    cout << "is_array=" << obj->is_array() << std::endl;
+    cout << "name=" << obj->get_name() << std::endl;
+    cout << "data=" << obj->get_data() << std::endl;
+  }
+
+  iter = parser.find_first("conditions");
+  if (!iter.end()) {
+    JSONObj *obj = *iter;
+
+    JSONObjIter iter2 = obj->find_first();
+    for (; !iter2.end(); ++iter2) {
+      JSONObj *child = *iter2;
+      cout << "is_object=" << child->is_object() << std::endl;
+      cout << "is_array=" << child->is_array() << std::endl;
+      if (child->is_array()) {
+        dump_array(child);
+      }
+      cout << "name=" << child->get_name() <
+#include 
+
+#include "common/errno.h"
+#include "common/ceph_json.h"
+#include "include/types.h"
+#include "include/str_list.h"
+
+#include "rgw_common.h"
+#include "rgw_keystone.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+bool KeystoneToken::User::has_role(const string& r) {
+  list::iterator iter;
+  for (iter = roles.begin(); iter != roles.end(); ++iter) {
+      if (fnmatch(r.c_str(), ((*iter).name.c_str()), 0) == 0) {
+        return true;
+      }
+  }
+  return false;
+}
+
+int KeystoneToken::parse(CephContext *cct, bufferlist& bl)
+{
+  JSONParser parser;
+  if (!parser.parse(bl.c_str(), bl.length())) {
+    ldout(cct, 0) << "Keystone token parse error: malformed json" << dendl;
+    return -EINVAL;
+  }
+
+  try {
+    JSONDecoder::decode_json("access", *this, &parser);
+  } catch (JSONDecoder::err& err) {
+    ldout(cct, 0) << "Keystone token parse error: " << err.message << dendl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+bool RGWKeystoneTokenCache::find(const string& token_id, KeystoneToken& token)
+{
+  lock.Lock();
+  map::iterator iter = tokens.find(token_id);
+  if (iter == tokens.end()) {
+    lock.Unlock();
+    if (perfcounter) perfcounter->inc(l_rgw_keystone_token_cache_miss);
+    return false;
+  }
+
+  token_entry& entry = iter->second;
+  tokens_lru.erase(entry.lru_iter);
+
+  if (entry.token.expired()) {
+    tokens.erase(iter);
+    lock.Unlock();
+    if (perfcounter) perfcounter->inc(l_rgw_keystone_token_cache_hit);
+    return false;
+  }
+  token = entry.token;
+
+  tokens_lru.push_front(token_id);
+  entry.lru_iter = tokens_lru.begin();
+
+  lock.Unlock();
+  if (perfcounter) perfcounter->inc(l_rgw_keystone_token_cache_hit);
+
+  return true;
+}
+
+void RGWKeystoneTokenCache::add(const string& token_id, KeystoneToken& token)
+{
+  lock.Lock();
+  map::iterator iter = tokens.find(token_id);
+  if (iter != tokens.end()) {
+    token_entry& e = iter->second;
+    tokens_lru.erase(e.lru_iter);
+  }
+
+  tokens_lru.push_front(token_id);
+  token_entry& entry = tokens[token_id];
+  entry.token = token;
+  entry.lru_iter = tokens_lru.begin();
+
+  while (tokens_lru.size() > max) {
+    list::reverse_iterator riter = tokens_lru.rbegin();
+    iter = tokens.find(*riter);
+    assert(iter != tokens.end());
+    tokens.erase(iter);
+    tokens_lru.pop_back();
+  }
+
+  lock.Unlock();
+}
+
+void RGWKeystoneTokenCache::invalidate(const string& token_id)
+{
+  Mutex::Locker l(lock);
+  map::iterator iter = tokens.find(token_id);
+  if (iter == tokens.end())
+    return;
+
+  ldout(cct, 20) << "invalidating revoked token id=" << token_id << dendl;
+  token_entry& e = iter->second;
+  tokens_lru.erase(e.lru_iter);
+  tokens.erase(iter);
+}
diff --git a/ceph/src/rgw/rgw_keystone.h b/ceph/src/rgw/rgw_keystone.h
new file mode 100644
index 00000000..05199eef
--- /dev/null
+++ b/ceph/src/rgw/rgw_keystone.h
@@ -0,0 +1,106 @@
+#ifndef CEPH_RGW_KEYSTONE_H
+#define CEPH_RGW_KEYSTONE_H
+
+#include "rgw_common.h"
+
+class KeystoneToken {
+public:
+  class Metadata {
+  public:
+    Metadata() : is_admin(false) { };
+    bool is_admin;
+    void decode_json(JSONObj *obj);
+  };
+
+  class Service {
+  public:
+    class Endpoint {
+    public:
+      string id;
+      string admin_url;
+      string public_url;
+      string internal_url;
+      string region;
+      void decode_json(JSONObj *obj);
+    };
+    string type;
+    string name;
+    list endpoints;
+    void decode_json(JSONObj *obj);
+  };
+
+  class Token {
+  public:
+    Token() : expires(0) { };
+    class Tenant {
+    public:
+      Tenant() : enabled(false) { };
+      string id;
+      string name;
+      string description;
+      bool enabled;
+      void decode_json(JSONObj *obj);
+    };
+    string id;
+    time_t expires;
+    Tenant tenant;
+    void decode_json(JSONObj *obj);
+  };
+
+  class User {
+  public:
+    class Role {
+    public:
+      string id;
+      string name;
+      void decode_json(JSONObj *obj);
+    };
+    string id;
+    string name;
+    string user_name;
+    list roles;
+    void decode_json(JSONObj *obj);
+    bool has_role(const string& r);
+  };
+
+  Metadata metadata;
+  list service_catalog;
+  Token token;
+  User user;
+
+public:
+  int parse(CephContext *cct, bufferlist& bl);
+
+  bool expired() {
+    uint64_t now = ceph_clock_now(NULL).sec();
+    return (now >= (uint64_t)token.expires);
+  }
+
+  void decode_json(JSONObj *access_obj);
+};
+
+struct token_entry {
+  KeystoneToken token;
+  list::iterator lru_iter;
+};
+
+class RGWKeystoneTokenCache {
+  CephContext *cct;
+
+  map tokens;
+  list tokens_lru;
+
+  Mutex lock;
+
+  size_t max;
+
+public:
+  RGWKeystoneTokenCache(CephContext *_cct, int _max) : cct(_cct), lock("RGWKeystoneTokenCache"), max(_max) {}
+
+  bool find(const string& token_id, KeystoneToken& token);
+  void add(const string& token_id, KeystoneToken& token);
+  void invalidate(const string& token_id);
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_loadgen.cc b/ceph/src/rgw/rgw_loadgen.cc
new file mode 100644
index 00000000..01cf1a35
--- /dev/null
+++ b/ceph/src/rgw/rgw_loadgen.cc
@@ -0,0 +1,111 @@
+
+#include 
+
+#include "rgw_loadgen.h"
+#include "rgw_auth_s3.h"
+
+
+#define dout_subsys ceph_subsys_rgw
+
+void RGWLoadGenRequestEnv::set_date(utime_t& tm)
+{
+  stringstream s;
+  tm.asctime(s);
+  date_str = s.str();
+}
+
+int RGWLoadGenRequestEnv::sign(RGWAccessKey& access_key)
+{
+  map meta_map;
+  map sub_resources;
+
+  string canonical_header;
+  string digest;
+
+  rgw_create_s3_canonical_header(request_method.c_str(),
+                                 NULL, /* const char *content_md5 */
+                                 content_type.c_str(),
+                                 date_str.c_str(),
+                                 meta_map,
+                                 uri.c_str(),
+                                 sub_resources,
+                                 canonical_header);
+
+  int ret = rgw_get_s3_header_digest(canonical_header, access_key.key, digest);
+  if (ret < 0) {
+    return ret;
+  }
+
+  headers["HTTP_DATE"] = date_str;
+  headers["HTTP_AUTHORIZATION"] = string("AWS ") + access_key.id + ":" + digest;
+
+  return 0;
+}
+
+int RGWLoadGenIO::write_data(const char *buf, int len)
+{
+  return len;
+}
+
+int RGWLoadGenIO::read_data(char *buf, int len)
+{
+  int read_len = MIN(left_to_read, (uint64_t)len);
+  left_to_read -= read_len;
+  return read_len;
+}
+
+void RGWLoadGenIO::flush()
+{
+}
+
+int RGWLoadGenIO::complete_request()
+{
+  return 0;
+}
+
+void RGWLoadGenIO::init_env(CephContext *cct)
+{
+  env.init(cct);
+
+  left_to_read = req->content_length;
+
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%lld", (long long)req->content_length);
+  env.set("CONTENT_LENGTH", buf);
+
+  env.set("CONTENT_TYPE", req->content_type.c_str());
+  env.set("HTTP_DATE", req->date_str.c_str());
+
+  for (map::iterator iter = req->headers.begin(); iter != req->headers.end(); ++iter) {
+    env.set(iter->first.c_str(), iter->second.c_str());
+  }
+
+  env.set("REQUEST_METHOD", req->request_method.c_str());
+  env.set("REQUEST_URI", req->uri.c_str());
+  env.set("QUERY_STRING", req->query_string.c_str());
+  env.set("SCRIPT_URI", req->uri.c_str());
+
+  char port_buf[16];
+  snprintf(port_buf, sizeof(port_buf), "%d", req->port);
+  env.set("SERVER_PORT", port_buf);
+}
+
+int RGWLoadGenIO::send_status(const char *status, const char *status_name)
+{
+  return 0;
+}
+
+int RGWLoadGenIO::send_100_continue()
+{
+  return 0;
+}
+
+int RGWLoadGenIO::complete_header()
+{
+  return 0;
+}
+
+int RGWLoadGenIO::send_content_length(uint64_t len)
+{
+  return 0;
+}
diff --git a/ceph/src/rgw/rgw_loadgen.h b/ceph/src/rgw/rgw_loadgen.h
new file mode 100644
index 00000000..5459330c
--- /dev/null
+++ b/ceph/src/rgw/rgw_loadgen.h
@@ -0,0 +1,45 @@
+#ifndef CEPH_RGW_LOADGEN_H
+#define CEPH_RGW_LOADGEN_H
+
+#include "rgw_client_io.h"
+
+
+struct RGWLoadGenRequestEnv {
+  int port;
+  uint64_t content_length;
+  string content_type;
+  string request_method;
+  string uri;
+  string query_string;
+  string date_str;
+
+  map headers;
+
+  RGWLoadGenRequestEnv() : port(0), content_length(0) {}
+
+  void set_date(utime_t& tm);
+  int sign(RGWAccessKey& access_key);
+};
+
+class RGWLoadGenIO : public RGWClientIO
+{
+  uint64_t left_to_read;
+  RGWLoadGenRequestEnv *req;
+public:
+  void init_env(CephContext *cct);
+
+  int write_data(const char *buf, int len);
+  int read_data(char *buf, int len);
+
+  int send_status(const char *status, const char *status_name);
+  int send_100_continue();
+  int complete_header();
+  int complete_request();
+  int send_content_length(uint64_t len);
+
+  RGWLoadGenIO(RGWLoadGenRequestEnv *_re) : left_to_read(0), req(_re) {}
+  void flush();
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_log.cc b/ceph/src/rgw/rgw_log.cc
new file mode 100644
index 00000000..26737614
--- /dev/null
+++ b/ceph/src/rgw/rgw_log.cc
@@ -0,0 +1,412 @@
+#include "common/Clock.h"
+#include "common/Timer.h"
+#include "common/utf8.h"
+#include "common/OutputDataSocket.h"
+#include "common/Formatter.h"
+
+#include "rgw_log.h"
+#include "rgw_acl.h"
+#include "rgw_rados.h"
+#include "rgw_client_io.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+static void set_param_str(struct req_state *s, const char *name, string& str)
+{
+  const char *p = s->info.env->get(name);
+  if (p)
+    str = p;
+}
+
+string render_log_object_name(const string& format,
+			      struct tm *dt, string& bucket_id, const string& bucket_name)
+{
+  string o;
+  for (unsigned i=0; itm_year + 1900);
+	break;
+      case 'y':
+	sprintf(buf, "%.2d", dt->tm_year % 100);
+	break;
+      case 'm':
+	sprintf(buf, "%.2d", dt->tm_mon + 1);
+	break;
+      case 'd':
+	sprintf(buf, "%.2d", dt->tm_mday);
+	break;
+      case 'H':
+	sprintf(buf, "%.2d", dt->tm_hour);
+	break;
+      case 'I':
+	sprintf(buf, "%.2d", (dt->tm_hour % 12) + 1);
+	break;
+      case 'k':
+	sprintf(buf, "%d", dt->tm_hour);
+	break;
+      case 'l':
+	sprintf(buf, "%d", (dt->tm_hour % 12) + 1);
+	break;
+      case 'M':
+	sprintf(buf, "%.2d", dt->tm_min);
+	break;
+
+      case 'i':
+	o += bucket_id;
+	continue;
+      case 'n':
+	o += bucket_name;
+	continue;
+      default:
+	// unknown code
+	sprintf(buf, "%%%c", format[i]);
+	break;
+      }
+      o += buf;
+      continue;
+    }
+    o += format[i];
+  }
+  return o;
+}
+
+/* usage logger */
+class UsageLogger {
+  CephContext *cct;
+  RGWRados *store;
+  map usage_map;
+  Mutex lock;
+  int32_t num_entries;
+  Mutex timer_lock;
+  SafeTimer timer;
+  utime_t round_timestamp;
+
+  class C_UsageLogTimeout : public Context {
+    UsageLogger *logger;
+  public:
+    C_UsageLogTimeout(UsageLogger *_l) : logger(_l) {}
+    void finish(int r) {
+      logger->flush();
+      logger->set_timer();
+    }
+  };
+
+  void set_timer() {
+    timer.add_event_after(cct->_conf->rgw_usage_log_tick_interval, new C_UsageLogTimeout(this));
+  }
+public:
+
+  UsageLogger(CephContext *_cct, RGWRados *_store) : cct(_cct), store(_store), lock("UsageLogger"), num_entries(0), timer_lock("UsageLogger::timer_lock"), timer(cct, timer_lock) {
+    timer.init();
+    Mutex::Locker l(timer_lock);
+    set_timer();
+    utime_t ts = ceph_clock_now(cct);
+    recalc_round_timestamp(ts);
+  }
+
+  ~UsageLogger() {
+    Mutex::Locker l(timer_lock);
+    flush();
+    timer.cancel_all_events();
+    timer.shutdown();
+  }
+
+  void recalc_round_timestamp(utime_t& ts) {
+    round_timestamp = ts.round_to_hour();
+  }
+
+  void insert(utime_t& timestamp, rgw_usage_log_entry& entry) {
+    lock.Lock();
+    if (timestamp.sec() > round_timestamp + 3600)
+      recalc_round_timestamp(timestamp);
+    entry.epoch = round_timestamp.sec();
+    bool account;
+    rgw_user_bucket ub(entry.owner, entry.bucket);
+    usage_map[ub].insert(round_timestamp, entry, &account);
+    if (account)
+      num_entries++;
+    bool need_flush = (num_entries > cct->_conf->rgw_usage_log_flush_threshold);
+    lock.Unlock();
+    if (need_flush) {
+      Mutex::Locker l(timer_lock);
+      flush();
+    }
+  }
+
+  void flush() {
+    map old_map;
+    lock.Lock();
+    old_map.swap(usage_map);
+    num_entries = 0;
+    lock.Unlock();
+
+    store->log_usage(old_map);
+  }
+};
+
+static UsageLogger *usage_logger = NULL;
+
+void rgw_log_usage_init(CephContext *cct, RGWRados *store)
+{
+  usage_logger = new UsageLogger(cct, store);
+}
+
+void rgw_log_usage_finalize()
+{
+  delete usage_logger;
+  usage_logger = NULL;
+}
+
+static void log_usage(struct req_state *s, const string& op_name)
+{
+  if (s->system_request) /* don't log system user operations */
+    return;
+
+  if (!usage_logger)
+    return;
+
+  string user;
+
+  if (!s->bucket_name_str.empty())
+    user = s->bucket_owner.get_id();
+  else
+    user = s->user.user_id;
+
+  rgw_usage_log_entry entry(user, s->bucket.name);
+
+  uint64_t bytes_sent = s->cio->get_bytes_sent();
+  uint64_t bytes_received = s->cio->get_bytes_received();
+
+  rgw_usage_data data(bytes_sent, bytes_received);
+
+  data.ops = 1;
+  if (!s->err.is_err())
+    data.successful_ops = 1;
+
+  entry.add(op_name, data);
+
+  utime_t ts = ceph_clock_now(s->cct);
+
+  usage_logger->insert(ts, entry);
+}
+
+void rgw_format_ops_log_entry(struct rgw_log_entry& entry, Formatter *formatter)
+{
+  formatter->open_object_section("log_entry");
+  formatter->dump_string("bucket", entry.bucket);
+  entry.time.gmtime(formatter->dump_stream("time"));      // UTC
+  entry.time.localtime(formatter->dump_stream("time_local"));
+  formatter->dump_string("remote_addr", entry.remote_addr);
+  if (entry.object_owner.length())
+    formatter->dump_string("object_owner", entry.object_owner);
+  formatter->dump_string("user", entry.user);
+  formatter->dump_string("operation", entry.op);
+  formatter->dump_string("uri", entry.uri);
+  formatter->dump_string("http_status", entry.http_status);
+  formatter->dump_string("error_code", entry.error_code);
+  formatter->dump_int("bytes_sent", entry.bytes_sent);
+  formatter->dump_int("bytes_received", entry.bytes_received);
+  formatter->dump_int("object_size", entry.obj_size);
+  uint64_t total_time =  entry.total_time.sec() * 1000000LL * entry.total_time.usec();
+
+  formatter->dump_int("total_time", total_time);
+  formatter->dump_string("user_agent",  entry.user_agent);
+  formatter->dump_string("referrer",  entry.referrer);
+  formatter->close_section();
+}
+
+void OpsLogSocket::formatter_to_bl(bufferlist& bl)
+{
+  stringstream ss;
+  formatter->flush(ss);
+  const string& s = ss.str();
+
+  bl.append(s);
+}
+
+void OpsLogSocket::init_connection(bufferlist& bl)
+{
+  bl.append("[");
+}
+
+OpsLogSocket::OpsLogSocket(CephContext *cct, uint64_t _backlog) : OutputDataSocket(cct, _backlog), lock("OpsLogSocket")
+{
+  formatter = new JSONFormatter;
+  delim.append(",\n");
+}
+
+OpsLogSocket::~OpsLogSocket()
+{
+  delete formatter;
+}
+
+void OpsLogSocket::log(struct rgw_log_entry& entry)
+{
+  bufferlist bl;
+
+  lock.Lock();
+  rgw_format_ops_log_entry(entry, formatter);
+  formatter_to_bl(bl);
+  lock.Unlock();
+
+  append_output(bl);
+}
+
+int rgw_log_op(RGWRados *store, struct req_state *s, const string& op_name, OpsLogSocket *olog)
+{
+  struct rgw_log_entry entry;
+  string bucket_id;
+
+  if (s->enable_usage_log)
+    log_usage(s, op_name);
+
+  if (!s->enable_ops_log)
+    return 0;
+
+  if (s->bucket_name_str.empty()) {
+    ldout(s->cct, 5) << "nothing to log for operation" << dendl;
+    return -EINVAL;
+  }
+  if (s->err.ret == -ERR_NO_SUCH_BUCKET) {
+    if (!s->cct->_conf->rgw_log_nonexistent_bucket) {
+      ldout(s->cct, 5) << "bucket " << s->bucket << " doesn't exist, not logging" << dendl;
+      return 0;
+    }
+    bucket_id = "";
+  } else {
+    bucket_id = s->bucket.bucket_id;
+  }
+  entry.bucket = s->bucket_name_str;
+
+  if (check_utf8(s->bucket_name_str.c_str(), entry.bucket.size()) != 0) {
+    ldout(s->cct, 5) << "not logging op on bucket with non-utf8 name" << dendl;
+    return 0;
+  }
+
+  if (s->object)
+    entry.obj = s->object;
+  else
+    entry.obj = "-";
+
+  entry.obj_size = s->obj_size;
+
+  if (s->cct->_conf->rgw_remote_addr_param.length())
+    set_param_str(s, s->cct->_conf->rgw_remote_addr_param.c_str(), entry.remote_addr);
+  else
+    set_param_str(s, "REMOTE_ADDR", entry.remote_addr);    
+  set_param_str(s, "HTTP_USER_AGENT", entry.user_agent);
+  set_param_str(s, "HTTP_REFERRER", entry.referrer);
+  set_param_str(s, "REQUEST_URI", entry.uri);
+  set_param_str(s, "REQUEST_METHOD", entry.op);
+
+  entry.user = s->user.user_id;
+  if (s->object_acl)
+    entry.object_owner = s->object_acl->get_owner().get_id();
+  entry.bucket_owner = s->bucket_owner.get_id();
+
+
+  uint64_t bytes_sent = s->cio->get_bytes_sent();
+  uint64_t bytes_received = s->cio->get_bytes_received();
+
+  entry.time = s->time;
+  entry.total_time = ceph_clock_now(s->cct) - s->time;
+  entry.bytes_sent = bytes_sent;
+  entry.bytes_received = bytes_received;
+  if (s->err.http_ret) {
+    char buf[16];
+    snprintf(buf, sizeof(buf), "%d", s->err.http_ret);
+    entry.http_status = buf;
+  } else
+    entry.http_status = "200"; // default
+
+  entry.error_code = s->err.s3_code;
+  entry.bucket_id = bucket_id;
+
+  bufferlist bl;
+  ::encode(entry, bl);
+
+  struct tm bdt;
+  time_t t = entry.time.sec();
+  if (s->cct->_conf->rgw_log_object_name_utc)
+    gmtime_r(&t, &bdt);
+  else
+    localtime_r(&t, &bdt);
+
+  int ret = 0;
+
+  if (s->cct->_conf->rgw_ops_log_rados) {
+    string oid = render_log_object_name(s->cct->_conf->rgw_log_object_name, &bdt,
+				        s->bucket.bucket_id, entry.bucket);
+
+    rgw_obj obj(store->zone.log_pool, oid);
+
+    ret = store->append_async(obj, bl.length(), bl);
+    if (ret == -ENOENT) {
+      ret = store->create_pool(store->zone.log_pool);
+      if (ret < 0)
+        goto done;
+      // retry
+      ret = store->append_async(obj, bl.length(), bl);
+    }
+  }
+
+  if (olog) {
+    olog->log(entry);
+  }
+done:
+  if (ret < 0)
+    ldout(s->cct, 0) << "ERROR: failed to log entry" << dendl;
+
+  return ret;
+}
+
+int rgw_log_intent(RGWRados *store, rgw_obj& obj, RGWIntentEvent intent, const utime_t& timestamp, bool utc)
+{
+  rgw_bucket intent_log_bucket(store->zone.intent_log_pool);
+
+  rgw_intent_log_entry entry;
+  entry.obj = obj;
+  entry.intent = (uint32_t)intent;
+  entry.op_time = timestamp;
+
+  struct tm bdt;
+  time_t t = timestamp.sec();
+  if (utc)
+    gmtime_r(&t, &bdt);
+  else
+    localtime_r(&t, &bdt);
+
+  struct rgw_bucket& bucket = obj.bucket;
+
+  char buf[bucket.name.size() + bucket.bucket_id.size() + 16];
+  sprintf(buf, "%.4d-%.2d-%.2d-%s-%s", (bdt.tm_year+1900), (bdt.tm_mon+1), bdt.tm_mday,
+          bucket.bucket_id.c_str(), obj.bucket.name.c_str());
+  string oid(buf);
+  rgw_obj log_obj(intent_log_bucket, oid);
+
+  bufferlist bl;
+  ::encode(entry, bl);
+
+  int ret = store->append_async(log_obj, bl.length(), bl);
+  if (ret == -ENOENT) {
+    ret = store->create_pool(intent_log_bucket);
+    if (ret < 0)
+      goto done;
+    ret = store->append_async(log_obj, bl.length(), bl);
+  }
+
+done:
+  return ret;
+}
+
+int rgw_log_intent(RGWRados *store, struct req_state *s, rgw_obj& obj, RGWIntentEvent intent)
+{
+  return rgw_log_intent(store, obj, intent, s->time, s->cct->_conf->rgw_intent_log_object_name_utc);
+}
diff --git a/ceph/src/rgw/rgw_log.h b/ceph/src/rgw/rgw_log.h
new file mode 100644
index 00000000..37e387d4
--- /dev/null
+++ b/ceph/src/rgw/rgw_log.h
@@ -0,0 +1,144 @@
+#ifndef CEPH_RGW_LOG_H
+#define CEPH_RGW_LOG_H
+
+#include "rgw_common.h"
+#include "include/utime.h"
+#include "common/Formatter.h"
+#include "common/OutputDataSocket.h"
+
+class RGWRados;
+
+struct rgw_log_entry {
+  string object_owner;
+  string bucket_owner;
+  string bucket;
+  utime_t time;
+  string remote_addr;
+  string user;
+  string obj;
+  string op;
+  string uri;
+  string http_status;
+  string error_code;
+  uint64_t bytes_sent;
+  uint64_t bytes_received;
+  uint64_t obj_size;
+  utime_t total_time;
+  string user_agent;
+  string referrer;
+  string bucket_id;
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(6, 5, bl);
+    ::encode(object_owner, bl);
+    ::encode(bucket_owner, bl);
+    ::encode(bucket, bl);
+    ::encode(time, bl);
+    ::encode(remote_addr, bl);
+    ::encode(user, bl);
+    ::encode(obj, bl);
+    ::encode(op, bl);
+    ::encode(uri, bl);
+    ::encode(http_status, bl);
+    ::encode(error_code, bl);
+    ::encode(bytes_sent, bl);
+    ::encode(obj_size, bl);
+    ::encode(total_time, bl);
+    ::encode(user_agent, bl);
+    ::encode(referrer, bl);
+    ::encode(bytes_received, bl);
+    ::encode(bucket_id, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &p) {
+    DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, p);
+    ::decode(object_owner, p);
+    if (struct_v > 3)
+      ::decode(bucket_owner, p);
+    ::decode(bucket, p);
+    ::decode(time, p);
+    ::decode(remote_addr, p);
+    ::decode(user, p);
+    ::decode(obj, p);
+    ::decode(op, p);
+    ::decode(uri, p);
+    ::decode(http_status, p);
+    ::decode(error_code, p);
+    ::decode(bytes_sent, p);
+    ::decode(obj_size, p);
+    ::decode(total_time, p);
+    ::decode(user_agent, p);
+    ::decode(referrer, p);
+    if (struct_v >= 2)
+      ::decode(bytes_received, p);
+    else
+      bytes_received = 0;
+
+    if (struct_v >= 3) {
+      if (struct_v <= 5) {
+        uint64_t id;
+        ::decode(id, p);
+        char buf[32];
+        snprintf(buf, sizeof(buf), "%llu", (long long)id);
+        bucket_id = buf;
+      } else {
+        ::decode(bucket_id, p);
+      }
+    } else
+      bucket_id = "";
+    DECODE_FINISH(p);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(rgw_log_entry)
+
+struct rgw_intent_log_entry {
+  rgw_obj obj;
+  utime_t op_time;
+  uint32_t intent;
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(obj, bl);
+    ::encode(op_time, bl);
+    ::encode(intent, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator &p) {
+    DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, p);
+    ::decode(obj, p);
+    ::decode(op_time, p);
+    ::decode(intent, p);
+    DECODE_FINISH(p);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(rgw_intent_log_entry)
+
+class OpsLogSocket : public OutputDataSocket {
+  Formatter *formatter;
+  Mutex lock;
+
+  void formatter_to_bl(bufferlist& bl);
+
+protected:
+  void init_connection(bufferlist& bl);
+
+public:
+  OpsLogSocket(CephContext *cct, uint64_t _backlog);
+  ~OpsLogSocket();
+
+  void log(struct rgw_log_entry& entry);
+};
+
+int rgw_log_op(RGWRados *store, struct req_state *s, const string& op_name, OpsLogSocket *olog);
+int rgw_log_intent(RGWRados *store, rgw_obj& obj, RGWIntentEvent intent, const utime_t& timestamp, bool utc);
+int rgw_log_intent(RGWRados *store, struct req_state *s, rgw_obj& obj, RGWIntentEvent intent);
+void rgw_log_usage_init(CephContext *cct, RGWRados *store);
+void rgw_log_usage_finalize();
+void rgw_format_ops_log_entry(struct rgw_log_entry& entry, Formatter *formatter);
+
+#endif
+
diff --git a/ceph/src/rgw/rgw_main.cc b/ceph/src/rgw/rgw_main.cc
new file mode 100644
index 00000000..9614b078
--- /dev/null
+++ b/ceph/src/rgw/rgw_main.cc
@@ -0,0 +1,1205 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "acconfig.h"
+#ifdef FASTCGI_INCLUDE_DIR
+# include "fastcgi/fcgiapp.h"
+#else
+# include "fcgiapp.h"
+#endif
+
+#include "rgw_fcgi.h"
+
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/signal_handler.h"
+#include "common/config.h"
+#include "common/errno.h"
+#include "common/WorkQueue.h"
+#include "common/Timer.h"
+#include "common/Throttle.h"
+#include "common/safe_io.h"
+#include "include/str_list.h"
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "rgw_acl.h"
+#include "rgw_user.h"
+#include "rgw_op.h"
+#include "rgw_rest.h"
+#include "rgw_rest_s3.h"
+#include "rgw_rest_swift.h"
+#include "rgw_rest_admin.h"
+#include "rgw_rest_usage.h"
+#include "rgw_rest_user.h"
+#include "rgw_rest_bucket.h"
+#include "rgw_rest_metadata.h"
+#include "rgw_rest_log.h"
+#include "rgw_rest_opstate.h"
+#include "rgw_replica_log.h"
+#include "rgw_rest_replica_log.h"
+#include "rgw_rest_config.h"
+#include "rgw_swift_auth.h"
+#include "rgw_swift.h"
+#include "rgw_log.h"
+#include "rgw_tools.h"
+#include "rgw_resolve.h"
+#include "rgw_loadgen.h"
+#include "rgw_civetweb.h"
+
+#include "civetweb/civetweb.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/types.h"
+#include "common/BackTrace.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+static sig_t sighandler_alrm;
+
+class RGWProcess;
+
+static int signal_fd[2] = {0, 0};
+static atomic_t disable_signal_fd;
+
+static void signal_shutdown();
+
+
+#define SOCKET_BACKLOG 1024
+
+struct RGWRequest
+{
+  uint64_t id;
+  struct req_state *s;
+  string req_str;
+  RGWOp *op;
+  utime_t ts;
+
+  RGWRequest() : id(0), s(NULL), op(NULL) {
+  }
+
+  void init_state(req_state *_s) {
+    s = _s;
+  }
+
+  void log_format(struct req_state *s, const char *fmt, ...)
+  {
+#define LARGE_SIZE 1024
+    char buf[LARGE_SIZE];
+    va_list ap;
+
+    va_start(ap, fmt);
+    vsnprintf(buf, sizeof(buf), fmt, ap);
+    va_end(ap);
+
+    log(s, buf);
+  }
+
+  void log_init() {
+    ts = ceph_clock_now(g_ceph_context);
+  }
+
+  void log(struct req_state *s, const char *msg) {
+    if (s->info.method && req_str.size() == 0) {
+      req_str = s->info.method;
+      req_str.append(" ");
+      req_str.append(s->info.request_uri);
+    }
+    utime_t t = ceph_clock_now(g_ceph_context) - ts;
+    dout(2) << "req " << id << ":" << t << ":" << s->dialect << ":" << req_str << ":" << (op ? op->name() : "") << ":" << msg << dendl;
+  }
+};
+
+class RGWFrontendConfig {
+  string config;
+  map config_map;
+  int parse_config(const string& config, map& config_map);
+  string framework;
+public:
+  RGWFrontendConfig(const string& _conf) : config(_conf) {}
+  int init() {
+    int ret = parse_config(config, config_map);
+    if (ret < 0)
+      return ret;
+    return 0;
+  }
+  bool get_val(const string& key, const string& def_val, string *out);
+  bool get_val(const string& key, int def_val, int *out);
+
+  string get_framework() { return framework; }
+};
+
+
+struct RGWFCGXRequest : public RGWRequest {
+  FCGX_Request fcgx;
+};
+
+struct RGWProcessEnv {
+  RGWRados *store;
+  RGWREST *rest;
+  OpsLogSocket *olog;
+  int port;
+};
+
+class RGWProcess {
+  deque m_req_queue;
+protected:
+  RGWRados *store;
+  OpsLogSocket *olog;
+  ThreadPool m_tp;
+  Throttle req_throttle;
+  RGWREST *rest;
+  RGWFrontendConfig *conf;
+  int sock_fd;
+
+  struct RGWWQ : public ThreadPool::WorkQueue {
+    RGWProcess *process;
+    RGWWQ(RGWProcess *p, time_t timeout, time_t suicide_timeout, ThreadPool *tp)
+      : ThreadPool::WorkQueue("RGWWQ", timeout, suicide_timeout, tp), process(p) {}
+
+    bool _enqueue(RGWRequest *req) {
+      process->m_req_queue.push_back(req);
+      perfcounter->inc(l_rgw_qlen);
+      dout(20) << "enqueued request req=" << hex << req << dec << dendl;
+      _dump_queue();
+      return true;
+    }
+    void _dequeue(RGWRequest *req) {
+      assert(0);
+    }
+    bool _empty() {
+      return process->m_req_queue.empty();
+    }
+    RGWRequest *_dequeue() {
+      if (process->m_req_queue.empty())
+	return NULL;
+      RGWRequest *req = process->m_req_queue.front();
+      process->m_req_queue.pop_front();
+      dout(20) << "dequeued request req=" << hex << req << dec << dendl;
+      _dump_queue();
+      perfcounter->inc(l_rgw_qlen, -1);
+      return req;
+    }
+    void _process(RGWRequest *req) {
+      perfcounter->inc(l_rgw_qactive);
+      process->handle_request(req);
+      process->req_throttle.put(1);
+      perfcounter->inc(l_rgw_qactive, -1);
+    }
+    void _dump_queue() {
+      deque::iterator iter;
+      if (process->m_req_queue.empty()) {
+        dout(20) << "RGWWQ: empty" << dendl;
+        return;
+      }
+      dout(20) << "RGWWQ:" << dendl;
+      for (iter = process->m_req_queue.begin(); iter != process->m_req_queue.end(); ++iter) {
+        dout(20) << "req: " << hex << *iter << dec << dendl;
+      }
+    }
+    void _clear() {
+      assert(process->m_req_queue.empty());
+    }
+  } req_wq;
+
+  uint64_t max_req_id;
+
+public:
+  RGWProcess(CephContext *cct, RGWProcessEnv *pe, int num_threads, RGWFrontendConfig *_conf)
+    : store(pe->store), olog(pe->olog), m_tp(cct, "RGWProcess::m_tp", num_threads),
+      req_throttle(cct, "rgw_ops", num_threads * 2),
+      rest(pe->rest),
+      conf(_conf),
+      sock_fd(-1),
+      req_wq(this, g_conf->rgw_op_thread_timeout,
+	     g_conf->rgw_op_thread_suicide_timeout, &m_tp),
+      max_req_id(0) {}
+  virtual ~RGWProcess() {}
+  virtual void run() = 0;
+  virtual void handle_request(RGWRequest *req) = 0;
+
+  void close_fd() {
+    if (sock_fd >= 0) {
+      ::close(sock_fd);
+      sock_fd = -1;
+    }
+  }
+};
+
+
+class RGWFCGXProcess : public RGWProcess {
+public:
+  RGWFCGXProcess(CephContext *cct, RGWProcessEnv *pe, int num_threads, RGWFrontendConfig *_conf) :
+    RGWProcess(cct, pe, num_threads, _conf) {}
+  void run();
+  void handle_request(RGWRequest *req);
+};
+
+void RGWFCGXProcess::run()
+{
+  string socket_path;
+  string socket_port;
+  string socket_host;
+
+  conf->get_val("socket_path", g_conf->rgw_socket_path, &socket_path);
+  conf->get_val("socket_port", g_conf->rgw_port, &socket_port);
+  conf->get_val("socket_host", g_conf->rgw_host, &socket_host);
+
+
+  if (!socket_path.empty()) {
+    string path_str = socket_path;
+
+    /* this is necessary, as FCGX_OpenSocket might not return an error, but rather ungracefully exit */
+    int fd = open(path_str.c_str(), O_CREAT, 0644);
+    if (fd < 0) {
+      int err = errno;
+      /* ENXIO is actually expected, we'll get that if we try to open a unix domain socket */
+      if (err != ENXIO) {
+        dout(0) << "ERROR: cannot create socket: path=" << path_str << " error=" << cpp_strerror(err) << dendl;
+        return;
+      }
+    } else {
+      close(fd);
+    }
+
+    const char *path = path_str.c_str();
+    sock_fd = FCGX_OpenSocket(path, SOCKET_BACKLOG);
+    if (sock_fd < 0) {
+      dout(0) << "ERROR: FCGX_OpenSocket (" << path << ") returned " << sock_fd << dendl;
+      return;
+    }
+    if (chmod(path, 0777) < 0) {
+      dout(0) << "WARNING: couldn't set permissions on unix domain socket" << dendl;
+    }
+  } else if (!socket_port.empty()) {
+    string bind = socket_host + ":" + socket_port;
+    sock_fd = FCGX_OpenSocket(bind.c_str(), SOCKET_BACKLOG);
+    if (sock_fd < 0) {
+      dout(0) << "ERROR: FCGX_OpenSocket (" << bind.c_str() << ") returned " << sock_fd << dendl;
+      return;
+    }
+  }
+
+  m_tp.start();
+
+  for (;;) {
+    RGWFCGXRequest *req = new RGWFCGXRequest;
+    req->id = ++max_req_id;
+    dout(10) << "allocated request req=" << hex << req << dec << dendl;
+    FCGX_InitRequest(&req->fcgx, sock_fd, 0);
+    req_throttle.get(1);
+    int ret = FCGX_Accept_r(&req->fcgx);
+    if (ret < 0) {
+      delete req;
+      dout(0) << "ERROR: FCGX_Accept_r returned " << ret << dendl;
+      req_throttle.put(1);
+      break;
+    }
+
+    req_wq.queue(req);
+  }
+
+  m_tp.drain(&req_wq);
+  m_tp.stop();
+}
+
+struct RGWLoadGenRequest : public RGWRequest {
+  string method;
+  string resource;
+  int content_length;
+  atomic_t *fail_flag;
+
+
+  RGWLoadGenRequest(const string& _m, const  string& _r, int _cl,
+                    atomic_t *ff) : method(_m), resource(_r), content_length(_cl), fail_flag(ff) {}
+};
+
+class RGWLoadGenProcess : public RGWProcess {
+  RGWAccessKey access_key;
+public:
+  RGWLoadGenProcess(CephContext *cct, RGWProcessEnv *pe, int num_threads, RGWFrontendConfig *_conf) :
+    RGWProcess(cct, pe, num_threads, _conf) {}
+  void run();
+  void checkpoint();
+  void handle_request(RGWRequest *req);
+  void gen_request(const string& method, const string& resource, int content_length, atomic_t *fail_flag);
+
+  void set_access_key(RGWAccessKey& key) { access_key = key; }
+};
+
+void RGWLoadGenProcess::checkpoint()
+{
+  m_tp.drain(&req_wq);
+}
+
+void RGWLoadGenProcess::run()
+{
+  m_tp.start(); /* start thread pool */
+
+  int i;
+
+  int num_objs;
+
+  conf->get_val("num_objs", 1000, &num_objs);
+
+  int num_buckets;
+  conf->get_val("num_buckets", 1, &num_buckets);
+
+  string buckets[num_buckets];
+
+  atomic_t failed;
+
+  for (i = 0; i < num_buckets; i++) {
+    buckets[i] = "/loadgen";
+    string& bucket = buckets[i];
+    append_rand_alpha(NULL, bucket, bucket, 16);
+
+    /* first create a bucket */
+    gen_request("PUT", bucket, 0, &failed);
+    checkpoint();
+  }
+
+  string *objs = new string[num_objs];
+
+  if (failed.read()) {
+    derr << "ERROR: bucket creation failed" << dendl;
+    goto done;
+  }
+
+  for (i = 0; i < num_objs; i++) {
+    char buf[16 + 1];
+    gen_rand_alphanumeric(NULL, buf, sizeof(buf));
+    buf[16] = '\0';
+    objs[i] = buckets[i % num_buckets] + "/" + buf;
+  }
+
+  for (i = 0; i < num_objs; i++) {
+    gen_request("PUT", objs[i], 4096, &failed);
+  }
+
+  checkpoint();
+
+  if (failed.read()) {
+    derr << "ERROR: bucket creation failed" << dendl;
+    goto done;
+  }
+
+  for (i = 0; i < num_objs; i++) {
+    gen_request("GET", objs[i], 4096, NULL);
+  }
+
+  checkpoint();
+
+  for (i = 0; i < num_objs; i++) {
+    gen_request("DELETE", objs[i], 0, NULL);
+  }
+
+  checkpoint();
+
+  for (i = 0; i < num_buckets; i++) {
+    gen_request("DELETE", buckets[i], 0, NULL);
+  }
+
+done:
+  checkpoint();
+
+  m_tp.stop();
+
+  delete[] objs;
+
+  signal_shutdown();
+}
+
+void RGWLoadGenProcess::gen_request(const string& method, const string& resource, int content_length, atomic_t *fail_flag)
+{
+  RGWLoadGenRequest *req = new RGWLoadGenRequest(method, resource, content_length, fail_flag);
+  req->id = ++max_req_id;
+  dout(10) << "allocated request req=" << hex << req << dec << dendl;
+  req_throttle.get(1);
+  req_wq.queue(req);
+}
+
+static void signal_shutdown()
+{
+  if (!disable_signal_fd.read()) {
+    int val = 0;
+    int ret = write(signal_fd[0], (char *)&val, sizeof(val));
+    if (ret < 0) {
+      int err = -errno;
+      derr << "ERROR: " << __func__ << ": write() returned " << cpp_strerror(-err) << dendl;
+    }
+  }
+}
+
+static void wait_shutdown()
+{
+  int val;
+  int r = safe_read_exact(signal_fd[1], &val, sizeof(val));
+  if (r < 0) {
+    derr << "safe_read_exact returned with error" << dendl;
+  }
+}
+
+int signal_fd_init()
+{
+  return socketpair(AF_UNIX, SOCK_STREAM, 0, signal_fd);
+}
+
+static void signal_fd_finalize()
+{
+  close(signal_fd[0]);
+  close(signal_fd[1]);
+}
+
+static void handle_sigterm(int signum)
+{
+  dout(1) << __func__ << dendl;
+  FCGX_ShutdownPending();
+
+  // send a signal to make fcgi's accept(2) wake up.  unfortunately the
+  // initial signal often isn't sufficient because we race with accept's
+  // check of the flag wet by ShutdownPending() above.
+  if (signum != SIGUSR1) {
+    signal_shutdown();
+
+    // safety net in case we get stuck doing an orderly shutdown.
+    uint64_t secs = g_ceph_context->_conf->rgw_exit_timeout_secs;
+    if (secs)
+      alarm(secs);
+    dout(1) << __func__ << " set alarm for " << secs << dendl;
+  }
+
+}
+
+static void godown_alarm(int signum)
+{
+  _exit(0);
+}
+
+static int call_log_intent(RGWRados *store, void *ctx, rgw_obj& obj, RGWIntentEvent intent)
+{
+  struct req_state *s = (struct req_state *)ctx;
+  return rgw_log_intent(store, s, obj, intent);
+}
+
+
+static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWClientIO *client_io, OpsLogSocket *olog)
+{
+  int ret = 0;
+
+  client_io->init(g_ceph_context);
+
+  req->log_init();
+
+  dout(1) << "====== starting new request req=" << hex << req << dec << " =====" << dendl;
+  perfcounter->inc(l_rgw_req);
+
+  RGWEnv& rgw_env = client_io->get_env();
+
+  struct req_state rstate(g_ceph_context, &rgw_env);
+
+  struct req_state *s = &rstate;
+
+  RGWRadosCtx rados_ctx(store, s);
+  s->obj_ctx = &rados_ctx;
+  store->set_intent_cb(s->obj_ctx, call_log_intent);
+
+  s->req_id = store->unique_id(req->id);
+
+  req->log(s, "initializing");
+
+  RGWOp *op = NULL;
+  int init_error = 0;
+  bool should_log = false;
+  RGWRESTMgr *mgr;
+  RGWHandler *handler = rest->get_handler(store, s, client_io, &mgr, &init_error);
+  if (init_error != 0) {
+    abort_early(s, NULL, init_error);
+    goto done;
+  }
+
+  should_log = mgr->get_logging();
+
+  req->log(s, "getting op");
+  op = handler->get_op(store);
+  if (!op) {
+    abort_early(s, NULL, -ERR_METHOD_NOT_ALLOWED);
+    goto done;
+  }
+  req->op = op;
+
+  req->log(s, "authorizing");
+  ret = handler->authorize();
+  if (ret < 0) {
+    dout(10) << "failed to authorize request" << dendl;
+    abort_early(s, op, ret);
+    goto done;
+  }
+
+  if (s->user.suspended) {
+    dout(10) << "user is suspended, uid=" << s->user.user_id << dendl;
+    abort_early(s, op, -ERR_USER_SUSPENDED);
+    goto done;
+  }
+  req->log(s, "reading permissions");
+  ret = handler->read_permissions(op);
+  if (ret < 0) {
+    abort_early(s, op, ret);
+    goto done;
+  }
+
+  req->log(s, "init op");
+  ret = op->init_processing();
+  if (ret < 0) {
+    abort_early(s, op, ret);
+    goto done;
+  }
+
+  req->log(s, "verifying op mask");
+  ret = op->verify_op_mask();
+  if (ret < 0) {
+    abort_early(s, op, ret);
+    goto done;
+  }
+
+  req->log(s, "verifying op permissions");
+  ret = op->verify_permission();
+  if (ret < 0) {
+    if (s->system_request) {
+      dout(2) << "overriding permissions due to system operation" << dendl;
+    } else {
+      abort_early(s, op, ret);
+      goto done;
+    }
+  }
+
+  req->log(s, "verifying op params");
+  ret = op->verify_params();
+  if (ret < 0) {
+    abort_early(s, op, ret);
+    goto done;
+  }
+
+  req->log(s, "executing");
+  op->pre_exec();
+  op->execute();
+  op->complete();
+done:
+  int r = client_io->complete_request();
+  if (r < 0) {
+    dout(0) << "ERROR: client_io->complete_request() returned " << r << dendl;
+  }
+  if (should_log) {
+    rgw_log_op(store, s, (op ? op->name() : "unknown"), olog);
+  }
+
+  int http_ret = s->err.http_ret;
+
+  req->log_format(s, "http status=%d", http_ret);
+
+  if (handler)
+    handler->put_op(op);
+  rest->put_handler(handler);
+
+  dout(1) << "====== req done req=" << hex << req << dec << " http_status=" << http_ret << " ======" << dendl;
+
+  return (ret < 0 ? ret : s->err.ret);
+}
+
+void RGWFCGXProcess::handle_request(RGWRequest *r)
+{
+  RGWFCGXRequest *req = static_cast(r);
+  FCGX_Request *fcgx = &req->fcgx;
+  RGWFCGX client_io(fcgx);
+
+ 
+  int ret = process_request(store, rest, req, &client_io, olog);
+  if (ret < 0) {
+    /* we don't really care about return code */
+    dout(20) << "process_request() returned " << ret << dendl;
+  }
+
+  FCGX_Finish_r(fcgx);
+
+  delete req;
+}
+
+void RGWLoadGenProcess::handle_request(RGWRequest *r)
+{
+  RGWLoadGenRequest *req = static_cast(r);
+
+  RGWLoadGenRequestEnv env;
+
+  utime_t tm = ceph_clock_now(NULL);
+
+  env.port = 80;
+  env.content_length = req->content_length;
+  env.content_type = "binary/octet-stream";
+  env.request_method = req->method;
+  env.uri = req->resource;
+  env.set_date(tm);
+  env.sign(access_key);
+
+  RGWLoadGenIO client_io(&env);
+
+  int ret = process_request(store, rest, req, &client_io, olog);
+  if (ret < 0) {
+    /* we don't really care about return code */
+    dout(20) << "process_request() returned " << ret << dendl;
+
+    if (req->fail_flag) {
+      req->fail_flag->inc();
+    }
+  }
+
+  delete req;
+}
+
+
+static int civetweb_callback(struct mg_connection *conn) {
+  struct mg_request_info *req_info = mg_get_request_info(conn);
+  RGWProcessEnv *pe = (RGWProcessEnv *)req_info->user_data;
+  RGWRados *store = pe->store;
+  RGWREST *rest = pe->rest;
+  OpsLogSocket *olog = pe->olog;
+
+  RGWRequest *req = new RGWRequest;
+  RGWMongoose client_io(conn, pe->port);
+
+  client_io.init(g_ceph_context);
+
+
+  int ret = process_request(store, rest, req, &client_io, olog);
+  if (ret < 0) {
+    /* we don't really care about return code */
+    dout(20) << "process_request() returned " << ret << dendl;
+  }
+
+  delete req;
+
+// Mark as processed
+  return 1;
+}
+
+#ifdef HAVE_CURL_MULTI_WAIT
+static void check_curl()
+{
+}
+#else
+static void check_curl()
+{
+  derr << "WARNING: libcurl doesn't support curl_multi_wait()" << dendl;
+  derr << "WARNING: cross zone / region transfer performance may be affected" << dendl;
+}
+#endif
+
+class C_InitTimeout : public Context {
+public:
+  C_InitTimeout() {}
+  void finish(int r) {
+    derr << "Initialization timeout, failed to initialize" << dendl;
+    exit(1);
+  }
+};
+
+int usage()
+{
+  cerr << "usage: radosgw [options...]" << std::endl;
+  cerr << "options:\n";
+  cerr << "   --rgw-region=     region in which radosgw runs\n";
+  cerr << "   --rgw-zone=         zone in which radosgw runs\n";
+  generic_server_usage();
+  return 0;
+}
+
+static RGWRESTMgr *set_logging(RGWRESTMgr *mgr)
+{
+  mgr->set_logging(true);
+  return mgr;
+}
+
+
+int RGWFrontendConfig::parse_config(const string& config, map& config_map)
+{
+  list config_list;
+  get_str_list(config, " ", config_list);
+
+  list::iterator iter;
+  for (iter = config_list.begin(); iter != config_list.end(); ++iter) {
+    string& entry = *iter;
+    string key;
+    string val;
+
+    if (framework.empty()) {
+      framework = entry;
+      dout(0) << "framework: " << framework << dendl;
+      continue;
+    }
+
+    ssize_t pos = entry.find('=');
+    if (pos < 0) {
+      dout(0) << "framework conf key: " << entry << dendl;
+      config_map[entry] = "";
+      continue;
+    }
+
+    int ret = parse_key_value(entry, key, val);
+    if (ret < 0) {
+      cerr << "ERROR: can't parse " << entry << std::endl;
+      return ret;
+    }
+
+    dout(0) << "framework conf key: " << key << ", val: " << val << dendl;
+    config_map[key] = val;
+  }
+
+  return 0;
+}
+
+
+bool RGWFrontendConfig::get_val(const string& key, const string& def_val, string *out)
+{
+ map::iterator iter = config_map.find(key);
+ if (iter == config_map.end()) {
+   *out = def_val;
+   return false;
+ }
+
+ *out = iter->second;
+ return true;
+}
+
+
+bool RGWFrontendConfig::get_val(const string& key, int def_val, int *out)
+{
+  string str;
+  bool found = get_val(key, "", &str);
+  if (!found) {
+    *out = def_val;
+    return false;
+  }
+  string err;
+  *out = strict_strtol(str.c_str(), 10, &err);
+  if (!err.empty()) {
+    cerr << "error parsing int: " << str << ": " << err << std::endl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
+class RGWFrontend {
+public:
+  virtual ~RGWFrontend() {}
+
+  virtual int init() = 0;
+
+  virtual int run() = 0;
+  virtual void stop() = 0;
+  virtual void join() = 0;
+};
+
+class RGWProcessControlThread : public Thread {
+  RGWProcess *pprocess;
+public:
+  RGWProcessControlThread(RGWProcess *_pprocess) : pprocess(_pprocess) {}
+
+  void *entry() {
+    pprocess->run();
+    return NULL;
+  };
+};
+
+class RGWProcessFrontend : public RGWFrontend {
+protected:
+  RGWFrontendConfig *conf;
+  RGWProcess *pprocess;
+  RGWProcessEnv env;
+  RGWProcessControlThread *thread;
+
+public:
+  RGWProcessFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : conf(_conf), pprocess(NULL), env(pe), thread(NULL) {
+  }
+
+  ~RGWProcessFrontend() {
+    delete thread;
+    delete pprocess;
+  }
+
+  int run() {
+    assert(pprocess); /* should have initialized by init() */
+    thread = new RGWProcessControlThread(pprocess);
+    thread->create();
+    return 0;
+  }
+
+  void stop() {
+    pprocess->close_fd();
+    thread->kill(SIGUSR1);
+  }
+
+  void join() {
+    thread->join();
+  }
+};
+
+class RGWFCGXFrontend : public RGWProcessFrontend {
+public:
+  RGWFCGXFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : RGWProcessFrontend(pe, _conf) {}
+
+  int init() {
+    pprocess = new RGWFCGXProcess(g_ceph_context, &env, g_conf->rgw_thread_pool_size, conf);
+    return 0;
+  }
+};
+
+class RGWLoadGenFrontend : public RGWProcessFrontend {
+public:
+  RGWLoadGenFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : RGWProcessFrontend(pe, _conf) {}
+
+  int init() {
+    int num_threads;
+    conf->get_val("num_threads", g_conf->rgw_thread_pool_size, &num_threads);
+    RGWLoadGenProcess *pp = new RGWLoadGenProcess(g_ceph_context, &env, num_threads, conf);
+
+    pprocess = pp;
+
+    string uid;
+    conf->get_val("uid", "", &uid);
+    if (uid.empty()) {
+      derr << "ERROR: uid param must be specified for loadgen frontend" << dendl;
+      return EINVAL;
+    }
+
+    RGWUserInfo user_info;
+    int ret = rgw_get_user_info_by_uid(env.store, uid, user_info, NULL);
+    if (ret < 0) {
+      derr << "ERROR: failed reading user info: uid=" << uid << " ret=" << ret << dendl;
+      return ret;
+    }
+
+    map::iterator aiter = user_info.access_keys.begin();
+    if (aiter == user_info.access_keys.end()) {
+      derr << "ERROR: user has no S3 access keys set" << dendl;
+      return -EINVAL;
+    }
+
+    pp->set_access_key(aiter->second);
+
+    return 0;
+  }
+};
+
+class RGWMongooseFrontend : public RGWFrontend {
+  RGWFrontendConfig *conf;
+  struct mg_context *ctx;
+  RGWProcessEnv env;
+
+public:
+  RGWMongooseFrontend(RGWProcessEnv& pe, RGWFrontendConfig *_conf) : conf(_conf), ctx(NULL), env(pe) {
+  }
+
+  int init() {
+    return 0;
+  }
+
+  int run() {
+    char thread_pool_buf[32];
+    snprintf(thread_pool_buf, sizeof(thread_pool_buf), "%d", (int)g_conf->rgw_thread_pool_size);
+    string port_str;
+    conf->get_val("port", "80", &port_str);
+    const char *options[] = {"listening_ports", port_str.c_str(), "enable_keep_alive", "yes", "num_threads", thread_pool_buf, NULL};
+
+    struct mg_callbacks cb;
+    memset((void *)&cb, 0, sizeof(cb));
+    cb.begin_request = civetweb_callback;
+    ctx = mg_start(&cb, &env, (const char **)&options);
+
+    if (!ctx) {
+      return -EIO;
+    }
+
+    return 0;
+  }
+
+  void stop() {
+    if (ctx) {
+      mg_stop(ctx);
+    }
+  }
+
+  void join() {
+  }
+};
+
+/*
+ * start up the RADOS connection and then handle HTTP messages as they come in
+ */
+int main(int argc, const char **argv)
+{
+  // dout() messages will be sent to stderr, but FCGX wants messages on stdout
+  // Redirect stderr to stdout.
+  TEMP_FAILURE_RETRY(close(STDERR_FILENO));
+  if (TEMP_FAILURE_RETRY(dup2(STDOUT_FILENO, STDERR_FILENO) < 0)) {
+    int err = errno;
+    cout << "failed to redirect stderr to stdout: " << cpp_strerror(err)
+	 << std::endl;
+    return ENOSYS;
+  }
+
+  /* alternative default for module */
+  vector def_args;
+  def_args.push_back("--debug-rgw=1/5");
+  def_args.push_back("--keyring=$rgw_data/keyring");
+  def_args.push_back("--log-file=/var/log/radosgw/$cluster-$name");
+
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+  global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_DAEMON,
+	      CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS);
+
+  for (std::vector::iterator i = args.begin(); i != args.end(); ++i) {
+    if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+      return 0;
+    }
+  }
+
+  check_curl();
+
+  if (g_conf->daemonize) {
+    global_init_daemonize(g_ceph_context, 0);
+  }
+  Mutex mutex("main");
+  SafeTimer init_timer(g_ceph_context, mutex);
+  init_timer.init();
+  mutex.Lock();
+  init_timer.add_event_after(g_conf->rgw_init_timeout, new C_InitTimeout);
+  mutex.Unlock();
+
+  common_init_finish(g_ceph_context);
+
+  rgw_tools_init(g_ceph_context);
+
+  rgw_init_resolver();
+  rgw_rest_init(g_ceph_context);
+  
+  curl_global_init(CURL_GLOBAL_ALL);
+  
+  FCGX_Init();
+
+  int r = 0;
+  RGWRados *store = RGWStoreManager::get_storage(g_ceph_context, true, true);
+  if (!store) {
+    derr << "Couldn't init storage provider (RADOS)" << dendl;
+    r = EIO;
+  }
+  if (!r)
+    r = rgw_perf_start(g_ceph_context);
+
+  mutex.Lock();
+  init_timer.cancel_all_events();
+  init_timer.shutdown();
+  mutex.Unlock();
+
+  if (r) 
+    return 1;
+
+  rgw_user_init(store->meta_mgr);
+  rgw_bucket_init(store->meta_mgr);
+  rgw_log_usage_init(g_ceph_context, store);
+
+  RGWREST rest;
+
+  list apis;
+  bool do_swift = false;
+
+  get_str_list(g_conf->rgw_enable_apis, apis);
+
+  map apis_map;
+  for (list::iterator li = apis.begin(); li != apis.end(); ++li) {
+    apis_map[*li] = true;
+  }
+
+  if (apis_map.count("s3") > 0)
+    rest.register_default_mgr(set_logging(new RGWRESTMgr_S3));
+
+  if (apis_map.count("swift") > 0) {
+    do_swift = true;
+    swift_init(g_ceph_context);
+    rest.register_resource(g_conf->rgw_swift_url_prefix, set_logging(new RGWRESTMgr_SWIFT));
+  }
+
+  if (apis_map.count("swift_auth") > 0)
+    rest.register_resource(g_conf->rgw_swift_auth_entry, set_logging(new RGWRESTMgr_SWIFT_Auth));
+
+  if (apis_map.count("admin") > 0) {
+    RGWRESTMgr_Admin *admin_resource = new RGWRESTMgr_Admin;
+    admin_resource->register_resource("usage", new RGWRESTMgr_Usage);
+    admin_resource->register_resource("user", new RGWRESTMgr_User);
+    admin_resource->register_resource("bucket", new RGWRESTMgr_Bucket);
+  
+    /*Registering resource for /admin/metadata */
+    admin_resource->register_resource("metadata", new RGWRESTMgr_Metadata);
+    admin_resource->register_resource("log", new RGWRESTMgr_Log);
+    admin_resource->register_resource("opstate", new RGWRESTMgr_Opstate);
+    admin_resource->register_resource("replica_log", new RGWRESTMgr_ReplicaLog);
+    admin_resource->register_resource("config", new RGWRESTMgr_Config);
+    rest.register_resource(g_conf->rgw_admin_entry, admin_resource);
+  }
+
+  OpsLogSocket *olog = NULL;
+
+  if (!g_conf->rgw_ops_log_socket_path.empty()) {
+    olog = new OpsLogSocket(g_ceph_context, g_conf->rgw_ops_log_data_backlog);
+    olog->init(g_conf->rgw_ops_log_socket_path);
+  }
+
+  r = signal_fd_init();
+  if (r < 0) {
+    derr << "ERROR: unable to initialize signal fds" << dendl;
+    exit(1);
+  }
+
+  init_async_signal_handler();
+  register_async_signal_handler(SIGHUP, sighup_handler);
+  register_async_signal_handler(SIGTERM, handle_sigterm);
+  register_async_signal_handler(SIGINT, handle_sigterm);
+  register_async_signal_handler(SIGUSR1, handle_sigterm);
+  sighandler_alrm = signal(SIGALRM, godown_alarm);
+
+  string frontend_frameworks = g_conf->rgw_frontends;
+
+  list frontends;
+
+  get_str_list(g_conf->rgw_frontends, ",", frontends);
+
+  multimap fe_map;
+  list configs;
+  if (frontends.empty()) {
+    frontends.push_back("fastcgi");
+  }
+  for (list::iterator iter = frontends.begin(); iter != frontends.end(); ++iter) {
+    string& f = *iter;
+
+    RGWFrontendConfig *config = new RGWFrontendConfig(f);
+    int r = config->init();
+    if (r < 0) {
+      cerr << "ERROR: failed to init config: " << f << std::endl;
+      return EINVAL;
+    }
+
+    configs.push_back(config);
+
+    string framework = config->get_framework();
+    fe_map.insert(make_pair(framework, config));
+  }
+
+  list fes;
+
+  for (multimap::iterator fiter = fe_map.begin(); fiter != fe_map.end(); ++fiter) {
+    RGWFrontendConfig *config = fiter->second;
+    string framework = config->get_framework();
+    RGWFrontend *fe;
+    if (framework == "fastcgi" || framework == "fcgi") {
+      RGWProcessEnv fcgi_pe = { store, &rest, olog, 0 };
+
+      fe = new RGWFCGXFrontend(fcgi_pe, config);
+    } else if (framework == "civetweb" || framework == "mongoose") {
+      string err;
+
+      int port;
+      config->get_val("port", 80, &port);
+
+      RGWProcessEnv env = { store, &rest, olog, port };
+
+      fe = new RGWMongooseFrontend(env, config);
+    } else if (framework == "loadgen") {
+      int port;
+      config->get_val("port", 80, &port);
+
+      RGWProcessEnv env = { store, &rest, olog, port };
+
+      fe = new RGWLoadGenFrontend(env, config);
+    } else {
+      dout(0) << "WARNING: skipping unknown framework: " << framework << dendl;
+      continue;
+    }
+    dout(0) << "starting handler: " << fiter->first << dendl;
+    int r = fe->init();
+    if (r < 0) {
+      derr << "ERROR: failed initializing frontend" << dendl;
+      return -r;
+    }
+    fe->run();
+
+    fes.push_back(fe);
+  }
+
+  wait_shutdown();
+
+  derr << "shutting down" << dendl;
+
+  for (list::iterator liter = fes.begin(); liter != fes.end(); ++liter) {
+    RGWFrontend *fe = *liter;
+    fe->stop();
+  }
+
+  for (list::iterator liter = fes.begin(); liter != fes.end(); ++liter) {
+    RGWFrontend *fe = *liter;
+    fe->join();
+    delete fe;
+  }
+
+  for (list::iterator liter = configs.begin(); liter != configs.end(); ++liter) {
+    RGWFrontendConfig *fec = *liter;
+    delete fec;
+  }
+
+  unregister_async_signal_handler(SIGHUP, sighup_handler);
+  unregister_async_signal_handler(SIGTERM, handle_sigterm);
+  unregister_async_signal_handler(SIGINT, handle_sigterm);
+  unregister_async_signal_handler(SIGUSR1, handle_sigterm);
+  shutdown_async_signal_handler();
+
+  if (do_swift) {
+    swift_finalize();
+  }
+
+  rgw_log_usage_finalize();
+
+  delete olog;
+
+  rgw_perf_stop(g_ceph_context);
+
+  RGWStoreManager::close_storage(store);
+
+  rgw_tools_cleanup();
+  rgw_shutdown_resolver();
+  curl_global_cleanup();
+
+  dout(1) << "final shutdown" << dendl;
+  g_ceph_context->put();
+
+  ceph::crypto::shutdown();
+
+  signal_fd_finalize();
+
+  return 0;
+}
+
diff --git a/ceph/src/rgw/rgw_metadata.cc b/ceph/src/rgw/rgw_metadata.cc
new file mode 100644
index 00000000..4307ce9c
--- /dev/null
+++ b/ceph/src/rgw/rgw_metadata.cc
@@ -0,0 +1,632 @@
+
+
+#include "common/ceph_json.h"
+#include "rgw_metadata.h"
+#include "cls/version/cls_version_types.h"
+
+#include "rgw_rados.h"
+#include "rgw_tools.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+struct LogStatusDump {
+  RGWMDLogStatus status;
+
+  LogStatusDump(RGWMDLogStatus _status) : status(_status) {}
+  void dump(Formatter *f) const {
+    string s;
+    switch (status) {
+      case MDLOG_STATUS_WRITE:
+        s = "write";
+        break;
+      case MDLOG_STATUS_SETATTRS:
+        s = "set_attrs";
+        break;
+      case MDLOG_STATUS_REMOVE:
+        s = "remove";
+        break;
+      case MDLOG_STATUS_COMPLETE:
+        s = "complete";
+        break;
+      case MDLOG_STATUS_ABORT:
+        s = "abort";
+        break;
+      default:
+        s = "unknown";
+        break;
+    }
+    encode_json("status", s, f);
+  }
+};
+
+struct RGWMetadataLogData {
+  obj_version read_version;
+  obj_version write_version;
+  RGWMDLogStatus status;
+  
+  RGWMetadataLogData() : status(MDLOG_STATUS_UNKNOWN) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(read_version, bl);
+    ::encode(write_version, bl);
+    uint32_t s = (uint32_t)status;
+    ::encode(s, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START(1, bl);
+     ::decode(read_version, bl);
+     ::decode(write_version, bl);
+     uint32_t s;
+     ::decode(s, bl);
+     status = (RGWMDLogStatus)s;
+     DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const {
+    encode_json("read_version", read_version, f);
+    encode_json("write_version", write_version, f);
+    encode_json("status", LogStatusDump(status), f);
+  };
+};
+WRITE_CLASS_ENCODER(RGWMetadataLogData);
+
+
+int RGWMetadataLog::add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl) {
+  if (!store->need_to_log_metadata())
+    return 0;
+
+  string oid;
+
+  string hash_key;
+  handler->get_hash_key(section, key, hash_key);
+
+  store->shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid);
+  utime_t now = ceph_clock_now(cct);
+  return store->time_log_add(oid, now, section, key, bl);
+}
+
+void RGWMetadataLog::init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, 
+                                       string& marker, void **handle)
+{
+  LogListCtx *ctx = new LogListCtx();
+
+  ctx->cur_shard = shard_id;
+  ctx->from_time = from_time;
+  ctx->end_time  = end_time;
+  ctx->marker    = marker;
+
+  get_shard_oid(ctx->cur_shard, ctx->cur_oid);
+
+  *handle = (void *)ctx;
+}
+
+void RGWMetadataLog::complete_list_entries(void *handle) {
+  LogListCtx *ctx = static_cast(handle);
+  delete ctx;
+}
+
+int RGWMetadataLog::list_entries(void *handle,
+				 int max_entries,
+				 list& entries,
+				 string *last_marker,
+				 bool *truncated) {
+  LogListCtx *ctx = static_cast(handle);
+
+  if (!max_entries) {
+    *truncated = false;
+    return 0;
+  }
+
+  int ret = store->time_log_list(ctx->cur_oid, ctx->from_time, ctx->end_time,
+				 max_entries, entries, ctx->marker,
+				 last_marker, truncated);
+  if ((ret < 0) && (ret != -ENOENT))
+    return ret;
+
+  if (ret == -ENOENT)
+    *truncated = false;
+
+  return 0;
+}
+
+int RGWMetadataLog::get_info(int shard_id, RGWMetadataLogInfo *info)
+{
+  string oid;
+  get_shard_oid(shard_id, oid);
+
+  cls_log_header header;
+
+  int ret = store->time_log_info(oid, &header);
+  if ((ret < 0) && (ret != -ENOENT))
+    return ret;
+
+  info->marker = header.max_marker;
+  info->last_update = header.max_time;
+
+  return 0;
+}
+
+int RGWMetadataLog::trim(int shard_id, const utime_t& from_time, const utime_t& end_time,
+                         const string& start_marker, const string& end_marker)
+{
+  string oid;
+  get_shard_oid(shard_id, oid);
+
+  int ret;
+
+  ret = store->time_log_trim(oid, from_time, end_time, start_marker, end_marker);
+
+  if (ret == -ENOENT)
+    ret = 0;
+
+  return ret;
+}
+  
+int RGWMetadataLog::lock_exclusive(int shard_id, utime_t& duration, string& zone_id, string& owner_id) {
+  string oid;
+  get_shard_oid(shard_id, oid);
+
+  return store->lock_exclusive(store->zone.log_pool, oid, duration, zone_id, owner_id);
+}
+
+int RGWMetadataLog::unlock(int shard_id, string& zone_id, string& owner_id) {
+  string oid;
+  get_shard_oid(shard_id, oid);
+
+  return store->unlock(store->zone.log_pool, oid, zone_id, owner_id);
+}
+
+obj_version& RGWMetadataObject::get_version()
+{
+  return objv;
+}
+
+class RGWMetadataTopHandler : public RGWMetadataHandler {
+  struct iter_data {
+    list sections;
+    list::iterator iter;
+  };
+
+public:
+  RGWMetadataTopHandler() {}
+
+  virtual string get_type() { return string(); }
+
+  virtual int get(RGWRados *store, string& entry, RGWMetadataObject **obj) { return -ENOTSUP; }
+  virtual int put(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker,
+                  time_t mtime, JSONObj *obj, sync_type_t sync_type) { return -ENOTSUP; }
+
+  virtual void get_pool_and_oid(RGWRados *store, const string& key, rgw_bucket& bucket, string& oid) {}
+
+  virtual int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) { return -ENOTSUP; }
+
+  virtual int list_keys_init(RGWRados *store, void **phandle) {
+    iter_data *data = new iter_data;
+    store->meta_mgr->get_sections(data->sections);
+    data->iter = data->sections.begin();
+
+    *phandle = data;
+
+    return 0;
+  }
+  virtual int list_keys_next(void *handle, int max, list& keys, bool *truncated)  {
+    iter_data *data = static_cast(handle);
+    for (int i = 0; i < max && data->iter != data->sections.end(); ++i, ++(data->iter)) {
+      keys.push_back(*data->iter);
+    }
+
+    *truncated = (data->iter != data->sections.end());
+
+    return 0;
+  }
+  virtual void list_keys_complete(void *handle) {
+    iter_data *data = static_cast(handle);
+
+    delete data;
+  }
+};
+
+static RGWMetadataTopHandler md_top_handler;
+
+RGWMetadataManager::RGWMetadataManager(CephContext *_cct, RGWRados *_store) : cct(_cct), store(_store)
+{
+  md_log = new RGWMetadataLog(_cct, _store);
+}
+
+RGWMetadataManager::~RGWMetadataManager()
+{
+  map::iterator iter;
+
+  for (iter = handlers.begin(); iter != handlers.end(); ++iter) {
+    delete iter->second;
+  }
+
+  handlers.clear();
+  delete md_log;
+}
+
+int RGWMetadataManager::register_handler(RGWMetadataHandler *handler)
+{
+  string type = handler->get_type();
+
+  if (handlers.find(type) != handlers.end())
+    return -EINVAL;
+
+  handlers[type] = handler;
+
+  return 0;
+}
+
+RGWMetadataHandler *RGWMetadataManager::get_handler(const char *type)
+{
+  map::iterator iter = handlers.find(type);
+  if (iter == handlers.end())
+    return NULL;
+
+  return iter->second;
+}
+
+void RGWMetadataManager::parse_metadata_key(const string& metadata_key, string& type, string& entry)
+{
+  int pos = metadata_key.find(':');
+  if (pos < 0) {
+    type = metadata_key;
+  }
+
+  type = metadata_key.substr(0, pos);
+  entry = metadata_key.substr(pos + 1);
+}
+
+int RGWMetadataManager::find_handler(const string& metadata_key, RGWMetadataHandler **handler, string& entry)
+{
+  string type;
+
+  parse_metadata_key(metadata_key, type, entry);
+
+  if (type.empty()) {
+    *handler = &md_top_handler;
+    return 0;
+  }
+
+  map::iterator iter = handlers.find(type);
+  if (iter == handlers.end())
+    return -ENOENT;
+
+  *handler = iter->second;
+
+  return 0;
+
+}
+
+int RGWMetadataManager::get(string& metadata_key, Formatter *f)
+{
+  RGWMetadataHandler *handler;
+  string entry;
+  int ret = find_handler(metadata_key, &handler, entry);
+  if (ret < 0) {
+    return ret;
+  }
+
+  RGWMetadataObject *obj;
+
+  ret = handler->get(store, entry, &obj);
+  if (ret < 0) {
+    return ret;
+  }
+
+  f->open_object_section("metadata_info");
+  encode_json("key", metadata_key, f);
+  encode_json("ver", obj->get_version(), f);
+  time_t mtime = obj->get_mtime();
+  if (mtime > 0) {
+    encode_json("mtime", mtime, f);
+  }
+  encode_json("data", *obj, f);
+  f->close_section();
+
+  delete obj;
+
+  return 0;
+}
+
+int RGWMetadataManager::put(string& metadata_key, bufferlist& bl,
+                            RGWMetadataHandler::sync_type_t sync_type,
+                            obj_version *existing_version)
+{
+  RGWMetadataHandler *handler;
+  string entry;
+
+  int ret = find_handler(metadata_key, &handler, entry);
+  if (ret < 0)
+    return ret;
+
+  JSONParser parser;
+  if (!parser.parse(bl.c_str(), bl.length())) {
+    return -EINVAL;
+  }
+
+  RGWObjVersionTracker objv_tracker;
+
+  obj_version *objv = &objv_tracker.write_version;
+
+  time_t mtime = 0;
+
+  JSONDecoder::decode_json("key", metadata_key, &parser);
+  JSONDecoder::decode_json("ver", *objv, &parser);
+  JSONDecoder::decode_json("mtime", mtime, &parser);
+
+  JSONObj *jo = parser.find_obj("data");
+  if (!jo) {
+    return -EINVAL;
+  }
+
+  ret = handler->put(store, entry, objv_tracker, mtime, jo, sync_type);
+  if (existing_version) {
+    *existing_version = objv_tracker.read_version;
+  }
+  return ret;
+}
+
+int RGWMetadataManager::remove(string& metadata_key)
+{
+  RGWMetadataHandler *handler;
+  string entry;
+
+  int ret = find_handler(metadata_key, &handler, entry);
+  if (ret < 0)
+    return ret;
+
+  RGWMetadataObject *obj;
+
+  ret = handler->get(store, entry, &obj);
+  if (ret < 0) {
+    return ret;
+  }
+
+  RGWObjVersionTracker objv_tracker;
+
+  objv_tracker.read_version = obj->get_version();
+
+  delete obj;
+
+  return handler->remove(store, entry, objv_tracker);
+}
+
+int RGWMetadataManager::lock_exclusive(string& metadata_key, utime_t duration, string& owner_id) {
+  RGWMetadataHandler *handler;
+  string entry;
+  string zone_id;
+
+  int ret = find_handler(metadata_key, &handler, entry);
+  if (ret < 0) 
+    return ret;
+
+  rgw_bucket pool;
+  string oid;
+
+  handler->get_pool_and_oid(store, entry, pool, oid);
+
+  return store->lock_exclusive(pool, oid, duration, zone_id, owner_id);  
+}
+
+int RGWMetadataManager::unlock(string& metadata_key, string& owner_id) {
+  librados::IoCtx io_ctx;
+  RGWMetadataHandler *handler;
+  string entry;
+  string zone_id;
+
+  int ret = find_handler(metadata_key, &handler, entry);
+  if (ret < 0) 
+    return ret;
+
+  rgw_bucket pool;
+  string oid;
+
+  handler->get_pool_and_oid(store, entry, pool, oid);
+
+  return store->unlock(pool, oid, zone_id, owner_id);  
+}
+
+struct list_keys_handle {
+  void *handle;
+  RGWMetadataHandler *handler;
+};
+
+
+int RGWMetadataManager::list_keys_init(string& section, void **handle)
+{
+  string entry;
+  RGWMetadataHandler *handler;
+
+  int ret;
+
+  ret = find_handler(section, &handler, entry);
+  if (ret < 0) {
+    return -ENOENT;
+  }
+
+  list_keys_handle *h = new list_keys_handle;
+  h->handler = handler;
+  ret = handler->list_keys_init(store, &h->handle);
+  if (ret < 0) {
+    delete h;
+    return ret;
+  }
+
+  *handle = (void *)h;
+
+  return 0;
+}
+
+int RGWMetadataManager::list_keys_next(void *handle, int max, list& keys, bool *truncated)
+{
+  list_keys_handle *h = static_cast(handle);
+
+  RGWMetadataHandler *handler = h->handler;
+
+  return handler->list_keys_next(h->handle, max, keys, truncated);
+}
+
+
+void RGWMetadataManager::list_keys_complete(void *handle)
+{
+  list_keys_handle *h = static_cast(handle);
+
+  RGWMetadataHandler *handler = h->handler;
+
+  handler->list_keys_complete(h->handle);
+  delete h;
+}
+
+void RGWMetadataManager::dump_log_entry(cls_log_entry& entry, Formatter *f)
+{
+  f->open_object_section("entry");
+  f->dump_string("id", entry.id);
+  f->dump_string("section", entry.section);
+  f->dump_string("name", entry.name);
+  entry.timestamp.gmtime(f->dump_stream("timestamp"));
+
+  try {
+    RGWMetadataLogData log_data;
+    bufferlist::iterator iter = entry.data.begin();
+    ::decode(log_data, iter);
+
+    encode_json("data", log_data, f);
+  } catch (buffer::error& err) {
+    lderr(cct) << "failed to decode log entry: " << entry.section << ":" << entry.name<< " ts=" << entry.timestamp << dendl;
+  }
+  f->close_section();
+}
+
+void RGWMetadataManager::get_sections(list& sections)
+{
+  for (map::iterator iter = handlers.begin(); iter != handlers.end(); ++iter) {
+    sections.push_back(iter->first);
+  }
+}
+
+int RGWMetadataManager::pre_modify(RGWMetadataHandler *handler, string& section, const string& key,
+                                   RGWMetadataLogData& log_data, RGWObjVersionTracker *objv_tracker,
+                                   RGWMDLogStatus op_type)
+{
+  section = handler->get_type();
+
+  /* if write version has not been set, and there's a read version, set it so that we can
+   * log it
+   */
+  if (objv_tracker) {
+    if (objv_tracker->read_version.ver && !objv_tracker->write_version.ver) {
+      objv_tracker->write_version = objv_tracker->read_version;
+      objv_tracker->write_version.ver++;
+    }
+    log_data.read_version = objv_tracker->read_version;
+    log_data.write_version = objv_tracker->write_version;
+  }
+
+  log_data.status = op_type;
+
+  bufferlist logbl;
+  ::encode(log_data, logbl);
+
+  int ret = md_log->add_entry(store, handler, section, key, logbl);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWMetadataManager::post_modify(RGWMetadataHandler *handler, const string& section, const string& key, RGWMetadataLogData& log_data,
+                                    RGWObjVersionTracker *objv_tracker, int ret)
+{
+  if (ret >= 0)
+    log_data.status = MDLOG_STATUS_COMPLETE;
+  else 
+    log_data.status = MDLOG_STATUS_ABORT;
+
+  bufferlist logbl;
+  ::encode(log_data, logbl);
+
+  int r = md_log->add_entry(store, handler, section, key, logbl);
+  if (ret < 0)
+    return ret;
+
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWMetadataManager::put_entry(RGWMetadataHandler *handler, const string& key, bufferlist& bl, bool exclusive,
+                                  RGWObjVersionTracker *objv_tracker, time_t mtime, map *pattrs)
+{
+  string section;
+  RGWMetadataLogData log_data;
+  int ret = pre_modify(handler, section, key, log_data, objv_tracker, MDLOG_STATUS_WRITE);
+  if (ret < 0)
+    return ret;
+
+  string oid;
+  rgw_bucket bucket;
+
+  handler->get_pool_and_oid(store, key, bucket, oid);
+
+  ret = rgw_put_system_obj(store, bucket, oid,
+                           bl.c_str(), bl.length(), exclusive,
+                           objv_tracker, mtime, pattrs);
+  /* cascading ret into post_modify() */
+
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWMetadataManager::remove_entry(RGWMetadataHandler *handler, string& key, RGWObjVersionTracker *objv_tracker)
+{
+  string section;
+  RGWMetadataLogData log_data;
+  int ret = pre_modify(handler, section, key, log_data, objv_tracker, MDLOG_STATUS_REMOVE);
+  if (ret < 0)
+    return ret;
+
+  string oid;
+  rgw_bucket bucket;
+
+  handler->get_pool_and_oid(store, key, bucket, oid);
+
+  rgw_obj obj(bucket, oid);
+
+  ret = store->delete_system_obj(NULL, obj, objv_tracker);
+  /* cascading ret into post_modify() */
+
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWMetadataManager::set_attrs(RGWMetadataHandler *handler, string& key,
+                                  rgw_obj& obj, map& attrs,
+                                  map* rmattrs,
+                                  RGWObjVersionTracker *objv_tracker)
+{
+  string section;
+  RGWMetadataLogData log_data;
+  int ret = pre_modify(handler, section, key, log_data, objv_tracker, MDLOG_STATUS_SETATTRS);
+  if (ret < 0)
+    return ret;
+
+  ret = store->set_attrs(NULL, obj, attrs, rmattrs, objv_tracker);
+  /* cascading ret into post_modify() */
+
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
diff --git a/ceph/src/rgw/rgw_metadata.h b/ceph/src/rgw/rgw_metadata.h
new file mode 100644
index 00000000..0145208d
--- /dev/null
+++ b/ceph/src/rgw/rgw_metadata.h
@@ -0,0 +1,216 @@
+#ifndef CEPH_RGW_METADATA_H
+#define CEPH_RGW_METADATA_H
+
+#include 
+
+#include "include/types.h"
+#include "rgw_common.h"
+#include "cls/version/cls_version_types.h"
+#include "cls/log/cls_log_types.h"
+
+
+class RGWRados;
+class JSONObj;
+struct RGWObjVersionTracker;
+
+struct obj_version;
+
+
+enum RGWMDLogStatus {
+  MDLOG_STATUS_UNKNOWN,
+  MDLOG_STATUS_WRITE,
+  MDLOG_STATUS_SETATTRS,
+  MDLOG_STATUS_REMOVE,
+  MDLOG_STATUS_COMPLETE,
+  MDLOG_STATUS_ABORT,
+};
+
+class RGWMetadataObject {
+protected:
+  obj_version objv;
+  time_t mtime;
+  
+public:
+  RGWMetadataObject() : mtime(0) {}
+  virtual ~RGWMetadataObject() {}
+  obj_version& get_version();
+  time_t get_mtime() { return mtime; }
+
+  virtual void dump(Formatter *f) const = 0;
+};
+
+class RGWMetadataManager;
+
+class RGWMetadataHandler {
+  friend class RGWMetadataManager;
+
+public:
+  enum sync_type_t {
+    APPLY_ALWAYS,
+    APPLY_UPDATES,
+    APPLY_NEWER
+  };
+  static bool string_to_sync_type(const string& sync_string,
+                                  sync_type_t& type) {
+    if (sync_string.compare("update-by-version") == 0)
+      type = APPLY_UPDATES;
+    else if (sync_string.compare("update-by-timestamp") == 0)
+      type = APPLY_NEWER;
+    else if (sync_string.compare("always") == 0)
+      type = APPLY_ALWAYS;
+    else
+      return false;
+    return true;
+  }
+  virtual ~RGWMetadataHandler() {}
+  virtual string get_type() = 0;
+
+  virtual int get(RGWRados *store, string& entry, RGWMetadataObject **obj) = 0;
+  virtual int put(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker,
+                  time_t mtime, JSONObj *obj, sync_type_t type) = 0;
+  virtual int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) = 0;
+
+  virtual int list_keys_init(RGWRados *store, void **phandle) = 0;
+  virtual int list_keys_next(void *handle, int max, list& keys, bool *truncated) = 0;
+  virtual void list_keys_complete(void *handle) = 0;
+
+  /* key to use for hashing entries for log shard placement */
+  virtual void get_hash_key(const string& section, const string& key, string& hash_key) {
+    hash_key = section + ":" + key;
+  }
+
+protected:
+  virtual void get_pool_and_oid(RGWRados *store, const string& key, rgw_bucket& bucket, string& oid) = 0;
+  /**
+   * Compare an incoming versus on-disk tag/version+mtime combo against
+   * the sync mode to see if the new one should replace the on-disk one.
+   *
+   * @return true if the update should proceed, false otherwise.
+   */
+  bool check_versions(const obj_version& ondisk, const time_t& ondisk_time,
+                      const obj_version& incoming, const time_t& incoming_time,
+                      sync_type_t sync_mode) {
+    switch (sync_mode) {
+    case APPLY_UPDATES:
+      if ((ondisk.tag != incoming.tag) ||
+	  (ondisk.ver >= incoming.ver))
+	return false;
+      break;
+    case APPLY_NEWER:
+      if (ondisk_time >= incoming_time)
+	return false;
+      break;
+    case APPLY_ALWAYS: //deliberate fall-thru -- we always apply!
+    default: break;
+    }
+    return true;
+  }
+};
+
+#define META_LOG_OBJ_PREFIX "meta.log."
+
+struct RGWMetadataLogInfo {
+  string marker;
+  utime_t last_update;
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+
+class RGWMetadataLog {
+  CephContext *cct;
+  RGWRados *store;
+  string prefix;
+
+  void get_shard_oid(int id, string& oid) {
+    char buf[16];
+    snprintf(buf, sizeof(buf), "%d", id);
+    oid = prefix + buf;
+  }
+
+public:
+  RGWMetadataLog(CephContext *_cct, RGWRados *_store) : cct(_cct), store(_store), prefix(META_LOG_OBJ_PREFIX) {}
+
+  int add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl);
+
+  struct LogListCtx {
+    int cur_shard;
+    string marker;
+    utime_t from_time;
+    utime_t end_time;
+
+    string cur_oid;
+
+    bool done;
+
+    LogListCtx() : cur_shard(0), done(false) {}
+  };
+
+  void init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, string& marker, void **handle);
+  void complete_list_entries(void *handle);
+  int list_entries(void *handle,
+                   int max_entries,
+                   list& entries,
+		   string *out_marker,
+		   bool *truncated);
+
+  int trim(int shard_id, const utime_t& from_time, const utime_t& end_time, const string& start_marker, const string& end_marker);
+  int get_info(int shard_id, RGWMetadataLogInfo *info);
+  int lock_exclusive(int shard_id, utime_t& duration, string&zone_id, string& owner_id);
+  int unlock(int shard_id, string& zone_id, string& owner_id);
+};
+
+struct RGWMetadataLogData;
+
+class RGWMetadataManager {
+  map handlers;
+  CephContext *cct;
+  RGWRados *store;
+  RGWMetadataLog *md_log;
+
+  void parse_metadata_key(const string& metadata_key, string& type, string& entry);
+
+  int find_handler(const string& metadata_key, RGWMetadataHandler **handler, string& entry);
+  int pre_modify(RGWMetadataHandler *handler, string& section, const string& key,
+                 RGWMetadataLogData& log_data, RGWObjVersionTracker *objv_tracker,
+                 RGWMDLogStatus op_type);
+  int post_modify(RGWMetadataHandler *handler, const string& section, const string& key, RGWMetadataLogData& log_data,
+                 RGWObjVersionTracker *objv_tracker, int ret);
+
+public:
+  RGWMetadataManager(CephContext *_cct, RGWRados *_store);
+  ~RGWMetadataManager();
+
+  int register_handler(RGWMetadataHandler *handler);
+
+  RGWMetadataHandler *get_handler(const char *type);
+
+  int put_entry(RGWMetadataHandler *handler, const string& key, bufferlist& bl, bool exclusive,
+                RGWObjVersionTracker *objv_tracker, time_t mtime, map *pattrs = NULL);
+  int remove_entry(RGWMetadataHandler *handler, string& key, RGWObjVersionTracker *objv_tracker);
+  int set_attr(RGWMetadataHandler *handler, string& key, rgw_obj& obj, string& attr, bufferlist& bl,
+               RGWObjVersionTracker *objv_tracker);
+  int set_attrs(RGWMetadataHandler *handler, string& key,
+                rgw_obj& obj, map& attrs,
+                map* rmattrs,
+                RGWObjVersionTracker *objv_tracker);
+  int get(string& metadata_key, Formatter *f);
+  int put(string& metadata_key, bufferlist& bl,
+          RGWMetadataHandler::sync_type_t sync_mode,
+          obj_version *existing_version = NULL);
+  int remove(string& metadata_key);
+
+  int list_keys_init(string& section, void **phandle);
+  int list_keys_next(void *handle, int max, list& keys, bool *truncated);
+  void list_keys_complete(void *handle);
+
+  void dump_log_entry(cls_log_entry& entry, Formatter *f);
+
+  void get_sections(list& sections);
+  int lock_exclusive(string& metadata_key, utime_t duration, string& owner_id);
+  int unlock(string& metadata_key, string& owner_id);
+
+  RGWMetadataLog *get_log() { return md_log; }
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_multi.cc b/ceph/src/rgw/rgw_multi.cc
new file mode 100644
index 00000000..74d18ba7
--- /dev/null
+++ b/ceph/src/rgw/rgw_multi.cc
@@ -0,0 +1,64 @@
+#include 
+
+#include 
+#include 
+
+#include "include/types.h"
+
+#include "rgw_xml.h"
+#include "rgw_multi.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+
+bool RGWMultiPart::xml_end(const char *el)
+{
+  RGWMultiPartNumber *num_obj = static_cast(find_first("PartNumber"));
+  RGWMultiETag *etag_obj = static_cast(find_first("ETag"));
+
+  if (!num_obj || !etag_obj)
+    return false;
+
+  string s = num_obj->get_data();
+  if (s.empty())
+    return false;
+
+  num = atoi(s.c_str());
+
+  s = etag_obj->get_data();
+  etag = s;
+
+  return true;
+}
+
+bool RGWMultiCompleteUpload::xml_end(const char *el) {
+  XMLObjIter iter = find("Part");
+  RGWMultiPart *part = static_cast(iter.get_next());
+  while (part) {
+    int num = part->get_num();
+    string etag = part->get_etag();
+    parts[num] = etag;
+    part = static_cast(iter.get_next());
+  }
+  return true;
+}
+
+
+XMLObj *RGWMultiXMLParser::alloc_obj(const char *el) {
+  XMLObj *obj = NULL;
+  if (strcmp(el, "CompleteMultipartUpload") == 0 ||
+      strcmp(el, "MultipartUpload") == 0) {
+    obj = new RGWMultiCompleteUpload();
+  } else if (strcmp(el, "Part") == 0) {
+    obj = new RGWMultiPart();
+  } else if (strcmp(el, "PartNumber") == 0) {
+    obj = new RGWMultiPartNumber();
+  } else if (strcmp(el, "ETag") == 0) {
+    obj = new RGWMultiETag();
+  }
+
+  return obj;
+}
+
diff --git a/ceph/src/rgw/rgw_multi.h b/ceph/src/rgw/rgw_multi.h
new file mode 100644
index 00000000..f24f71d9
--- /dev/null
+++ b/ceph/src/rgw/rgw_multi.h
@@ -0,0 +1,52 @@
+#ifndef CEPH_RGW_MULTI_H
+#define CEPH_RGW_MULTI_H
+
+#include 
+#include "rgw_xml.h"
+
+class RGWMultiCompleteUpload : public XMLObj
+{
+public:
+  RGWMultiCompleteUpload() {}
+  ~RGWMultiCompleteUpload() {}
+  bool xml_end(const char *el);
+
+  std::map parts;
+};
+
+class RGWMultiPart : public XMLObj
+{
+  string etag;
+  int num;
+public:
+  RGWMultiPart() : num(0) {}
+  ~RGWMultiPart() {}
+  bool xml_end(const char *el);
+
+  string& get_etag() { return etag; }
+  int get_num() { return num; }
+};
+
+class RGWMultiPartNumber : public XMLObj
+{
+public:
+  RGWMultiPartNumber() {}
+  ~RGWMultiPartNumber() {}
+};
+
+class RGWMultiETag : public XMLObj
+{
+public:
+  RGWMultiETag() {}
+  ~RGWMultiETag() {}
+};
+
+class RGWMultiXMLParser : public RGWXMLParser
+{
+  XMLObj *alloc_obj(const char *el);
+public:
+  RGWMultiXMLParser() {}
+  ~RGWMultiXMLParser() {}
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_multi_del.cc b/ceph/src/rgw/rgw_multi_del.cc
new file mode 100644
index 00000000..502212f0
--- /dev/null
+++ b/ceph/src/rgw/rgw_multi_del.cc
@@ -0,0 +1,64 @@
+#include 
+
+#include 
+
+#include "include/types.h"
+
+#include "rgw_xml.h"
+#include "rgw_multi_del.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+
+bool RGWMultiDelObject::xml_end(const char *el)
+{
+  RGWMultiDelKey *key_obj = static_cast(find_first("Key"));
+
+  if (!key_obj)
+    return false;
+
+  string s = key_obj->get_data();
+  if (s.empty())
+    return false;
+
+  key = s;
+
+  return true;
+}
+
+bool RGWMultiDelDelete::xml_end(const char *el) {
+  RGWMultiDelQuiet *quiet_set = static_cast(find_first("Quiet"));
+  if (quiet_set) {
+    string quiet_val = quiet_set->get_data();
+    quiet = (strcasecmp(quiet_val.c_str(), "true") == 0);
+  }
+
+  XMLObjIter iter = find("Object");
+  RGWMultiDelObject *object = static_cast(iter.get_next());
+  while (object) {
+    string key = object->get_key();
+    objects.push_back(key);
+    object = static_cast(iter.get_next());
+  }
+  return true;
+}
+
+XMLObj *RGWMultiDelXMLParser::alloc_obj(const char *el) {
+  XMLObj *obj = NULL;
+  if (strcmp(el, "Delete") == 0) {
+    obj = new RGWMultiDelDelete();
+  } else if (strcmp(el, "Quiet") == 0) {
+    obj = new RGWMultiDelQuiet();
+  } else if (strcmp(el, "Object") == 0) {
+    obj = new RGWMultiDelObject ();
+  } else if (strcmp(el, "Key") == 0) {
+    obj = new RGWMultiDelKey();
+  } else if (strcmp(el, "VersionID") == 0) {
+    /*do nothing*/
+  }
+
+  return obj;
+}
+
diff --git a/ceph/src/rgw/rgw_multi_del.h b/ceph/src/rgw/rgw_multi_del.h
new file mode 100644
index 00000000..c590a1ef
--- /dev/null
+++ b/ceph/src/rgw/rgw_multi_del.h
@@ -0,0 +1,54 @@
+#ifndef RGW_MULTI_DELETE_H_
+#define RGW_MULTI_DELETE_H_
+
+#include 
+#include "rgw_xml.h"
+
+class RGWMultiDelDelete : public XMLObj
+{
+public:
+  RGWMultiDelDelete() :quiet(false) {}
+  ~RGWMultiDelDelete() {}
+  bool xml_end(const char *el);
+
+  std::vector objects;
+  bool quiet;
+  bool is_quiet() { return quiet; };
+};
+
+class RGWMultiDelQuiet : public XMLObj
+{
+public:
+  RGWMultiDelQuiet() {}
+  ~RGWMultiDelQuiet() {}
+};
+
+class RGWMultiDelObject : public XMLObj
+{
+  string key;
+  string versionID;
+public:
+  RGWMultiDelObject() {}
+  ~RGWMultiDelObject() {}
+  bool xml_end(const char *el);
+
+  string get_key() { return key; }
+};
+
+class RGWMultiDelKey : public XMLObj
+{
+public:
+  RGWMultiDelKey() {}
+  ~RGWMultiDelKey() {}
+};
+
+class RGWMultiDelXMLParser : public RGWXMLParser
+{
+  XMLObj *alloc_obj(const char *el);
+public:
+  RGWMultiDelXMLParser() {}
+  ~RGWMultiDelXMLParser() {}
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_multiparser.cc b/ceph/src/rgw/rgw_multiparser.cc
new file mode 100644
index 00000000..2f4d8d94
--- /dev/null
+++ b/ceph/src/rgw/rgw_multiparser.cc
@@ -0,0 +1,44 @@
+#include 
+
+#include 
+#include 
+
+#include "include/types.h"
+
+#include "rgw_multi.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+                                  
+int main(int argc, char **argv) {
+  RGWMultiXMLParser parser;
+
+  if (!parser.init())
+    exit(1);
+
+  char buf[1024];
+
+  for (;;) {
+    int done;
+    int len;
+
+    len = fread(buf, 1, sizeof(buf), stdin);
+    if (ferror(stdin)) {
+      fprintf(stderr, "Read error\n");
+      exit(-1);
+    }
+    done = feof(stdin);
+
+    bool result = parser.parse(buf, len, done);
+    if (!result) {
+      cerr << "failed to parse!" << std::endl;
+    }
+
+    if (done)
+      break;
+  }
+
+  exit(0);
+}
+
diff --git a/ceph/src/rgw/rgw_op.cc b/ceph/src/rgw/rgw_op.cc
new file mode 100644
index 00000000..76947480
--- /dev/null
+++ b/ceph/src/rgw/rgw_op.cc
@@ -0,0 +1,3170 @@
+
+#include 
+#include 
+
+#include 
+
+#include "common/Clock.h"
+#include "common/armor.h"
+#include "common/mime.h"
+#include "common/utf8.h"
+#include "common/ceph_json.h"
+
+#include "rgw_rados.h"
+#include "rgw_op.h"
+#include "rgw_rest.h"
+#include "rgw_acl.h"
+#include "rgw_acl_s3.h"
+#include "rgw_user.h"
+#include "rgw_bucket.h"
+#include "rgw_log.h"
+#include "rgw_multi.h"
+#include "rgw_multi_del.h"
+#include "rgw_cors.h"
+#include "rgw_cors_s3.h"
+
+#include "rgw_client_io.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+using ceph::crypto::MD5;
+
+static string mp_ns = RGW_OBJ_NS_MULTIPART;
+static string shadow_ns = RGW_OBJ_NS_SHADOW;
+
+#define MULTIPART_UPLOAD_ID_PREFIX "2/" // must contain a unique char that may not come up in gen_rand_alpha()
+
+class MultipartMetaFilter : public RGWAccessListFilter {
+public:
+  MultipartMetaFilter() {}
+  bool filter(string& name, string& key) {
+    int len = name.size();
+    if (len < 6)
+      return false;
+
+    int pos = name.find(MP_META_SUFFIX, len - 5);
+    if (pos <= 0)
+      return false;
+
+    pos = name.rfind('.', pos - 1);
+    if (pos < 0)
+      return false;
+
+    key = name.substr(0, pos);
+
+    return true;
+  }
+};
+
+static MultipartMetaFilter mp_filter;
+
+static int parse_range(const char *range, off_t& ofs, off_t& end, bool *partial_content)
+{
+  int r = -ERANGE;
+  string s(range);
+  string ofs_str;
+  string end_str;
+
+  *partial_content = false;
+
+  int pos = s.find("bytes=");
+  if (pos < 0) {
+    pos = 0;
+    while (isspace(s[pos]))
+      pos++;
+    int end = pos;
+    while (isalpha(s[end]))
+      end++;
+    if (strncasecmp(s.c_str(), "bytes", end - pos) != 0)
+      return 0;
+    while (isspace(s[end]))
+      end++;
+    if (s[end] != '=')
+      return 0;
+    s = s.substr(end + 1);
+  } else {
+    s = s.substr(pos + 6); /* size of("bytes=")  */
+  }
+  pos = s.find('-');
+  if (pos < 0)
+    goto done;
+
+  *partial_content = true;
+
+  ofs_str = s.substr(0, pos);
+  end_str = s.substr(pos + 1);
+  if (end_str.length()) {
+    end = atoll(end_str.c_str());
+    if (end < 0)
+      goto done;
+  }
+
+  if (ofs_str.length()) {
+    ofs = atoll(ofs_str.c_str());
+  } else { // RFC2616 suffix-byte-range-spec
+    ofs = -end;
+    end = -1;
+  }
+
+  if (end >= 0 && end < ofs)
+    goto done;
+
+  r = 0;
+done:
+  return r;
+}
+
+static void format_xattr(std::string &xattr)
+{
+  /* If the extended attribute is not valid UTF-8, we encode it using quoted-printable
+   * encoding.
+   */
+  if ((check_utf8(xattr.c_str(), xattr.length()) != 0) ||
+      (check_for_control_characters(xattr.c_str(), xattr.length()) != 0)) {
+    static const char MIME_PREFIX_STR[] = "=?UTF-8?Q?";
+    static const int MIME_PREFIX_LEN = sizeof(MIME_PREFIX_STR) - 1;
+    static const char MIME_SUFFIX_STR[] = "?=";
+    static const int MIME_SUFFIX_LEN = sizeof(MIME_SUFFIX_STR) - 1;
+    int mlen = mime_encode_as_qp(xattr.c_str(), NULL, 0);
+    char *mime = new char[MIME_PREFIX_LEN + mlen + MIME_SUFFIX_LEN + 1];
+    strcpy(mime, MIME_PREFIX_STR);
+    mime_encode_as_qp(xattr.c_str(), mime + MIME_PREFIX_LEN, mlen);
+    strcpy(mime + MIME_PREFIX_LEN + (mlen - 1), MIME_SUFFIX_STR);
+    xattr.assign(mime);
+    delete [] mime;
+  }
+}
+
+/**
+ * Get the HTTP request metadata out of the req_state as a
+ * map(, where attr_name is RGW_ATTR_PREFIX.HTTP_NAME)
+ * s: The request state
+ * attrs: will be filled up with attrs mapped as 
+ *
+ */
+static void rgw_get_request_metadata(CephContext *cct, struct req_info& info, map& attrs)
+{
+  map::iterator iter;
+  for (iter = info.x_meta_map.begin(); iter != info.x_meta_map.end(); ++iter) {
+    const string &name(iter->first);
+    string &xattr(iter->second);
+    ldout(cct, 10) << "x>> " << name << ":" << xattr << dendl;
+    format_xattr(xattr);
+    string attr_name(RGW_ATTR_PREFIX);
+    attr_name.append(name);
+    map::value_type v(attr_name, bufferlist());
+    std::pair < map::iterator, bool > rval(attrs.insert(v));
+    bufferlist& bl(rval.first->second);
+    bl.append(xattr.c_str(), xattr.size() + 1);
+  }
+}
+
+static int decode_policy(CephContext *cct, bufferlist& bl, RGWAccessControlPolicy *policy)
+{
+  bufferlist::iterator iter = bl.begin();
+  try {
+    policy->decode(iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+    return -EIO;
+  }
+  if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
+    RGWAccessControlPolicy_S3 *s3policy = static_cast(policy);
+    ldout(cct, 15) << "Read AccessControlPolicy";
+    s3policy->to_xml(*_dout);
+    *_dout << dendl;
+  }
+  return 0;
+}
+
+static int get_bucket_policy_from_attr(CephContext *cct, RGWRados *store, void *ctx,
+                                       RGWBucketInfo& bucket_info, map& bucket_attrs,
+                                       RGWAccessControlPolicy *policy, rgw_obj& obj)
+{
+  map::iterator aiter = bucket_attrs.find(RGW_ATTR_ACL);
+
+  if (aiter != bucket_attrs.end()) {
+    int ret = decode_policy(cct, aiter->second, policy);
+    if (ret < 0)
+      return ret;
+  } else {
+    ldout(cct, 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl;
+    RGWUserInfo uinfo;
+    /* object exists, but policy is broken */
+    int r = rgw_get_user_info_by_uid(store, bucket_info.owner, uinfo);
+    if (r < 0)
+      return r;
+
+    policy->create_default(bucket_info.owner, uinfo.display_name);
+  }
+  return 0;
+}
+
+static int get_obj_policy_from_attr(CephContext *cct, RGWRados *store, void *ctx,
+                                    RGWBucketInfo& bucket_info, map& bucket_attrs,
+                                    RGWAccessControlPolicy *policy, rgw_obj& obj)
+{
+  bufferlist bl;
+  int ret = 0;
+
+  ret = store->get_attr(ctx, obj, RGW_ATTR_ACL, bl);
+  if (ret >= 0) {
+    ret = decode_policy(cct, bl, policy);
+    if (ret < 0)
+      return ret;
+  } else if (ret == -ENODATA) {
+    /* object exists, but policy is broken */
+    ldout(cct, 0) << "WARNING: couldn't find acl header for object, generating default" << dendl;
+    RGWUserInfo uinfo;
+    ret = rgw_get_user_info_by_uid(store, bucket_info.owner, uinfo);
+    if (ret < 0)
+      return ret;
+
+    policy->create_default(bucket_info.owner, uinfo.display_name);
+  }
+  return ret;
+}
+
+
+/**
+ * Get the AccessControlPolicy for an object off of disk.
+ * policy: must point to a valid RGWACL, and will be filled upon return.
+ * bucket: name of the bucket containing the object.
+ * object: name of the object to get the ACL for.
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+static int get_policy_from_attr(CephContext *cct, RGWRados *store, void *ctx,
+                                RGWBucketInfo& bucket_info, map& bucket_attrs,
+                                RGWAccessControlPolicy *policy, rgw_obj& obj)
+{
+  if (obj.bucket.name.empty()) {
+    return 0;
+  }
+
+  if (obj.object.empty()) {
+    rgw_obj instance_obj;
+    store->get_bucket_instance_obj(bucket_info.bucket, instance_obj);
+    return get_bucket_policy_from_attr(cct, store, ctx, bucket_info, bucket_attrs,
+                                       policy, instance_obj);
+  }
+  return get_obj_policy_from_attr(cct, store, ctx, bucket_info, bucket_attrs,
+                                  policy, obj);
+}
+
+static int get_obj_attrs(RGWRados *store, struct req_state *s, rgw_obj& obj, map& attrs,
+                         uint64_t *obj_size, RGWObjVersionTracker *objv_tracker)
+{
+  void *handle;
+  int ret = store->prepare_get_obj(s->obj_ctx, obj, NULL, NULL, &attrs, NULL,
+                                      NULL, NULL, NULL, NULL, NULL, obj_size, objv_tracker, &handle, &s->err);
+  store->finish_get_obj(&handle);
+  return ret;
+}
+
+static int read_policy(RGWRados *store, struct req_state *s,
+                       RGWBucketInfo& bucket_info, map& bucket_attrs,
+                       RGWAccessControlPolicy *policy, rgw_bucket& bucket, string& object)
+{
+  string upload_id;
+  upload_id = s->info.args.get("uploadId");
+  string oid = object;
+  rgw_obj obj;
+
+  if (!s->system_request && bucket_info.flags & BUCKET_SUSPENDED) {
+    ldout(s->cct, 0) << "NOTICE: bucket " << bucket_info.bucket.name << " is suspended" << dendl;
+    return -ERR_USER_SUSPENDED;
+  }
+
+  if (!oid.empty() && !upload_id.empty()) {
+    RGWMPObj mp(oid, upload_id);
+    oid = mp.get_meta();
+    obj.init_ns(bucket, oid, mp_ns);
+    obj.set_in_extra_data(true);
+  } else {
+    obj.init(bucket, oid);
+  }
+  int ret = get_policy_from_attr(s->cct, store, s->obj_ctx, bucket_info, bucket_attrs, policy, obj);
+  if (ret == -ENOENT && object.size()) {
+    /* object does not exist checking the bucket's ACL to make sure
+       that we send a proper error code */
+    RGWAccessControlPolicy bucket_policy(s->cct);
+    string no_object;
+    rgw_obj no_obj(bucket, no_object);
+    ret = get_policy_from_attr(s->cct, store, s->obj_ctx, bucket_info, bucket_attrs, &bucket_policy, no_obj);
+    if (ret < 0)
+      return ret;
+    string& owner = bucket_policy.get_owner().get_id();
+    if (!s->system_request && owner.compare(s->user.user_id) != 0 &&
+        !bucket_policy.verify_permission(s->user.user_id, s->perm_mask, RGW_PERM_READ))
+      ret = -EACCES;
+    else
+      ret = -ENOENT;
+
+  } else if (ret == -ENOENT) {
+      ret = -ERR_NO_SUCH_BUCKET;
+  }
+
+  return ret;
+}
+
+/**
+ * Get the AccessControlPolicy for a bucket or object off of disk.
+ * s: The req_state to draw information from.
+ * only_bucket: If true, reads the bucket ACL rather than the object ACL.
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bucket, bool prefetch_data)
+{
+  int ret = 0;
+  string obj_str;
+  RGWUserInfo bucket_owner_info;
+
+  s->bucket_instance_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "bucket-instance");
+
+  s->bucket_acl = new RGWAccessControlPolicy(s->cct);
+
+  if (s->copy_source) { /* check if copy source is within the current domain */
+    const char *src = s->copy_source;
+    if (*src == '/')
+      ++src;
+    string copy_source_str(src);
+
+    int pos = copy_source_str.find('/');
+    if (pos > 0)
+      copy_source_str = copy_source_str.substr(0, pos);
+
+    RGWBucketInfo source_info;
+
+    ret = store->get_bucket_info(s->obj_ctx, copy_source_str, source_info, NULL);
+    if (ret == 0) {
+      string& region = source_info.region;
+      s->local_source = store->region.equals(region);
+    }
+  }
+
+  if (!s->bucket_name_str.empty()) {
+    s->bucket_exists = true;
+    if (s->bucket_instance_id.empty()) {
+      ret = store->get_bucket_info(s->obj_ctx, s->bucket_name_str, s->bucket_info, NULL, &s->bucket_attrs);
+    } else {
+      ret = store->get_bucket_instance_info(s->obj_ctx, s->bucket_instance_id, s->bucket_info, NULL, &s->bucket_attrs);
+    }
+    if (ret < 0) {
+      if (ret != -ENOENT) {
+        ldout(s->cct, 0) << "NOTICE: couldn't get bucket from bucket_name (name=" << s->bucket_name_str << ")" << dendl;
+        return ret;
+      }
+      s->bucket_exists = false;
+    }
+    s->bucket = s->bucket_info.bucket;
+
+    if (s->bucket_exists) {
+      string no_obj;
+      ret = read_policy(store, s, s->bucket_info, s->bucket_attrs, s->bucket_acl, s->bucket, no_obj);
+    } else {
+      s->bucket_acl->create_default(s->user.user_id, s->user.display_name);
+      ret = -ERR_NO_SUCH_BUCKET;
+    }
+
+    s->bucket_owner = s->bucket_acl->get_owner();
+
+    string& region = s->bucket_info.region;
+    if (s->bucket_exists && !store->region.equals(region)) {
+      ldout(s->cct, 0) << "NOTICE: request for data in a different region (" << region << " != " << store->region.name << ")" << dendl;
+      /* we now need to make sure that the operation actually requires copy source, that is
+       * it's a copy operation
+       */
+      if (store->region.is_master && s->op == OP_DELETE && s->system_request) {
+        /*If the operation is delete and if this is the master, don't redirect*/
+      } else if (!s->local_source ||
+          (s->op != OP_PUT && s->op != OP_COPY) ||
+          s->object_str.empty()) {
+        return -ERR_PERMANENT_REDIRECT;
+      }
+    }
+  }
+
+  /* we're passed only_bucket = true when we specifically need the bucket's
+     acls, that happens on write operations */
+  if (!only_bucket && !s->object_str.empty()) {
+    if (!s->bucket_exists) {
+      return -ERR_NO_SUCH_BUCKET;
+    }
+    s->object_acl = new RGWAccessControlPolicy(s->cct);
+
+    obj_str = s->object_str;
+    rgw_obj obj(s->bucket, obj_str);
+    store->set_atomic(s->obj_ctx, obj);
+    if (prefetch_data) {
+      store->set_prefetch_data(s->obj_ctx, obj);
+    }
+    ret = read_policy(store, s, s->bucket_info, s->bucket_attrs, s->object_acl, s->bucket, obj_str);
+  }
+
+  return ret;
+}
+
+static void rgw_bucket_object_pre_exec(struct req_state *s)
+{
+  if (s->expect_cont)
+    dump_continue(s);
+
+  dump_bucket_from_state(s);
+}
+
+int RGWGetObj::verify_permission()
+{
+  obj.init(s->bucket, s->object_str);
+  store->set_atomic(s->obj_ctx, obj);
+  store->set_prefetch_data(s->obj_ctx, obj);
+
+  if (!verify_object_permission(s, RGW_PERM_READ))
+    return -EACCES;
+
+  return 0;
+}
+
+
+int RGWOp::verify_op_mask()
+{
+  uint32_t required_mask = op_mask();
+
+  ldout(s->cct, 20) << "required_mask= " << required_mask << " user.op_mask=" << s->user.op_mask << dendl;
+
+  if ((s->user.op_mask & required_mask) != required_mask) {
+    return -EPERM;
+  }
+
+  if (!s->system_request && (required_mask & RGW_OP_TYPE_MODIFY) && !store->zone.is_master)  {
+    ldout(s->cct, 5) << "NOTICE: modify request to a non-master zone by a non-system user, permission denied"  << dendl;
+    return -EPERM;
+  }
+
+  return 0;
+}
+
+int RGWOp::init_quota()
+{
+  /* no quota enforcement for system requests */
+  if (s->system_request)
+    return 0;
+
+  /* init quota related stuff */
+  if (!(s->user.op_mask & RGW_OP_TYPE_MODIFY)) {
+    return 0;
+  }
+
+  /* only interested in object related ops */
+  if (s->object_str.empty()) {
+    return 0;
+  }
+
+  RGWUserInfo owner_info;
+  RGWUserInfo *uinfo;
+
+  if (s->user.user_id == s->bucket_owner.get_id()) {
+    uinfo = &s->user;
+  } else {
+    int r = rgw_get_user_info_by_uid(store, s->bucket_info.owner, owner_info);
+    if (r < 0)
+      return r;
+    uinfo = &owner_info;
+  }
+
+  if (s->bucket_info.quota.enabled) {
+    bucket_quota = s->bucket_info.quota;
+  } else if (uinfo->bucket_quota.enabled) {
+    bucket_quota = uinfo->bucket_quota;
+  } else {
+    bucket_quota = store->region_map.bucket_quota;
+  }
+
+  if (uinfo->user_quota.enabled) {
+    user_quota = uinfo->user_quota;
+  } else {
+    user_quota = store->region_map.user_quota;
+  }
+
+  return 0;
+}
+
+static bool validate_cors_rule_method(RGWCORSRule *rule, const char *req_meth) {
+  uint8_t flags = 0;
+  if (strcmp(req_meth, "GET") == 0) flags = RGW_CORS_GET;
+  else if (strcmp(req_meth, "POST") == 0) flags = RGW_CORS_POST;
+  else if (strcmp(req_meth, "PUT") == 0) flags = RGW_CORS_PUT;
+  else if (strcmp(req_meth, "DELETE") == 0) flags = RGW_CORS_DELETE;
+  else if (strcmp(req_meth, "HEAD") == 0) flags = RGW_CORS_HEAD;
+
+  if ((rule->get_allowed_methods() & flags) == flags) {
+    dout(10) << "Method " << req_meth << " is supported" << dendl;
+  } else {
+    dout(5) << "Method " << req_meth << " is not supported" << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+int RGWOp::read_bucket_cors()
+{
+  bufferlist bl;
+
+  map::iterator aiter = s->bucket_attrs.find(RGW_ATTR_CORS);
+  if (aiter == s->bucket_attrs.end()) {
+    ldout(s->cct, 20) << "no CORS configuration attr found" << dendl;
+    cors_exist = false;
+    return 0; /* no CORS configuration found */
+  }
+
+  cors_exist = true;
+
+  bl = aiter->second;
+
+  bufferlist::iterator iter = bl.begin();
+  try {
+    bucket_cors.decode(iter);
+  } catch (buffer::error& err) {
+    ldout(s->cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+    return -EIO;
+  }
+  if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
+    RGWCORSConfiguration_S3 *s3cors = static_cast(&bucket_cors);
+    ldout(s->cct, 15) << "Read RGWCORSConfiguration";
+    s3cors->to_xml(*_dout);
+    *_dout << dendl;
+  }
+  return 0;
+}
+
+/** CORS 6.2.6.
+ * If any of the header field-names is not a ASCII case-insensitive match for
+ * any of the values in list of headers do not set any additional headers and
+ * terminate this set of steps.
+ * */
+static void get_cors_response_headers(RGWCORSRule *rule, const char *req_hdrs, string& hdrs, string& exp_hdrs, unsigned *max_age) {
+  if (req_hdrs) {
+    list hl;
+    get_str_list(req_hdrs, hl);
+    for(list::iterator it = hl.begin(); it != hl.end(); ++it) {
+      if (!rule->is_header_allowed((*it).c_str(), (*it).length())) {
+        dout(5) << "Header " << (*it) << " is not registered in this rule" << dendl;
+      } else {
+        if (hdrs.length() > 0) hdrs.append(",");
+        hdrs.append((*it));
+      }
+    }
+  }
+  rule->format_exp_headers(exp_hdrs);
+  *max_age = rule->get_max_age();
+}
+
+/**
+ * Generate the CORS header response
+ *
+ * This is described in the CORS standard, section 6.2.
+ */
+bool RGWOp::generate_cors_headers(string& origin, string& method, string& headers, string& exp_headers, unsigned *max_age)
+{
+  /* CORS 6.2.1. */
+  const char *orig = s->info.env->get("HTTP_ORIGIN");
+  if (!orig) {
+    return false;
+  }
+
+  /* Custom: */
+  origin = orig;
+  int ret = read_bucket_cors();
+  if (ret < 0) {
+    return false;
+  }
+
+  if (!cors_exist) {
+    dout(2) << "No CORS configuration set yet for this bucket" << dendl;
+    return false;
+  }
+
+  /* CORS 6.2.2. */
+  RGWCORSRule *rule = bucket_cors.host_name_rule(orig);
+  if (!rule)
+    return false;
+
+  /* CORS 6.2.3. */
+  const char *req_meth = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_METHOD");
+  if (!req_meth) {
+    req_meth = s->info.method;
+  }
+
+  if (req_meth)
+    method = req_meth;
+  /* CORS 6.2.5. */
+  if (!validate_cors_rule_method(rule, req_meth)) {
+    return false;
+  }
+
+  /* CORS 6.2.4. */
+  const char *req_hdrs = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS");
+
+  /* CORS 6.2.6. */
+  get_cors_response_headers(rule, req_hdrs, headers, exp_headers, max_age);
+
+  return true;
+}
+
+int RGWGetObj::read_user_manifest_part(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs)
+{
+  ldout(s->cct, 0) << "user manifest obj=" << ent.name << dendl;
+
+  void *handle = NULL;
+  off_t cur_ofs = start_ofs;
+  off_t cur_end = end_ofs;
+  utime_t start_time = s->time;
+
+  rgw_obj part(bucket, ent.name);
+
+  map attrs;
+
+  uint64_t obj_size;
+  void *obj_ctx = store->create_context(s);
+  RGWAccessControlPolicy obj_policy(s->cct);
+
+  ldout(s->cct, 20) << "reading obj=" << part << " ofs=" << cur_ofs << " end=" << cur_end << dendl;
+
+  store->set_atomic(obj_ctx, part);
+  store->set_prefetch_data(obj_ctx, part);
+  ret = store->prepare_get_obj(obj_ctx, part, &cur_ofs, &cur_end, &attrs, NULL,
+                                  NULL, NULL, NULL, NULL, NULL, &obj_size, NULL, &handle, &s->err);
+  if (ret < 0)
+    goto done_err;
+
+  if (obj_size != ent.size) {
+    // hmm.. something wrong, object not as expected, abort!
+    ldout(s->cct, 0) << "ERROR: expected obj_size=" << obj_size << ", actual read size=" << ent.size << dendl;
+    ret = -EIO;
+    goto done_err;
+  }
+
+  ret = rgw_policy_from_attrset(s->cct, attrs, &obj_policy);
+  if (ret < 0)
+    goto done_err;
+
+  if (!verify_object_permission(s, bucket_policy, &obj_policy, RGW_PERM_READ)) {
+    ret = -EPERM;
+    goto done_err;
+  }
+
+  perfcounter->inc(l_rgw_get_b, cur_end - cur_ofs);
+  while (cur_ofs <= cur_end) {
+    bufferlist bl;
+    ret = store->get_obj(obj_ctx, NULL, &handle, part, bl, cur_ofs, cur_end);
+    if (ret < 0)
+      goto done_err;
+
+    off_t len = bl.length();
+    cur_ofs += len;
+    ofs += len;
+    ret = 0;
+    perfcounter->tinc(l_rgw_get_lat,
+                      (ceph_clock_now(s->cct) - start_time));
+    send_response_data(bl, 0, len);
+
+    start_time = ceph_clock_now(s->cct);
+  }
+
+  store->destroy_context(obj_ctx);
+  obj_ctx = NULL;
+
+  store->finish_get_obj(&handle);
+
+  return 0;
+
+done_err:
+  if (obj_ctx)
+    store->destroy_context(obj_ctx);
+  return ret;
+}
+
+static int iterate_user_manifest_parts(CephContext *cct, RGWRados *store, off_t ofs, off_t end,
+                                       rgw_bucket& bucket, string& obj_prefix, RGWAccessControlPolicy *bucket_policy,
+                                       uint64_t *ptotal_len,
+                                       int (*cb)(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy,
+                                                 off_t start_ofs, off_t end_ofs, void *param), void *cb_param)
+{
+  uint64_t obj_ofs = 0, len_count = 0;
+  bool found_start = false, found_end = false;
+  string delim;
+  string marker;
+  bool is_truncated;
+  string no_ns;
+  map common_prefixes;
+  vector objs;
+
+  utime_t start_time = ceph_clock_now(cct);
+
+  do {
+#define MAX_LIST_OBJS 100
+    int r = store->list_objects(bucket, MAX_LIST_OBJS, obj_prefix, delim, marker, NULL,
+                                objs, common_prefixes,
+                                true, no_ns, true, &is_truncated, NULL);
+    if (r < 0)
+      return r;
+
+    vector::iterator viter;
+
+    for (viter = objs.begin(); viter != objs.end() && !found_end; ++viter) {
+      RGWObjEnt& ent = *viter;
+      uint64_t cur_total_len = obj_ofs;
+      uint64_t start_ofs = 0, end_ofs = ent.size;
+
+      if (!found_start && cur_total_len + ent.size > (uint64_t)ofs) {
+	start_ofs = ofs - obj_ofs;
+	found_start = true;
+      }
+
+      obj_ofs += ent.size;
+
+      if (!found_end && obj_ofs > (uint64_t)end) {
+	end_ofs = end - cur_total_len + 1;
+	found_end = true;
+      }
+
+      perfcounter->tinc(l_rgw_get_lat,
+                       (ceph_clock_now(cct) - start_time));
+
+      if (found_start) {
+        len_count += end_ofs - start_ofs;
+
+        if (cb) {
+          r = cb(bucket, ent, bucket_policy, start_ofs, end_ofs, cb_param);
+          if (r < 0)
+            return r;
+        }
+      }
+      marker = ent.name;
+
+      start_time = ceph_clock_now(cct);
+    }
+  } while (is_truncated && !found_end);
+
+  if (ptotal_len)
+    *ptotal_len = len_count;
+
+  return 0;
+}
+
+static int get_obj_user_manifest_iterate_cb(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs,
+                                       void *param)
+{
+  RGWGetObj *op = (RGWGetObj *)param;
+  return op->read_user_manifest_part(bucket, ent, bucket_policy, start_ofs, end_ofs);
+}
+
+int RGWGetObj::handle_user_manifest(const char *prefix)
+{
+  ldout(s->cct, 2) << "RGWGetObj::handle_user_manifest() prefix=" << prefix << dendl;
+
+  string prefix_str = prefix;
+  int pos = prefix_str.find('/');
+  if (pos < 0)
+    return -EINVAL;
+
+  string bucket_name = prefix_str.substr(0, pos);
+  string obj_prefix = prefix_str.substr(pos + 1);
+
+  rgw_bucket bucket;
+
+  RGWAccessControlPolicy _bucket_policy(s->cct);
+  RGWAccessControlPolicy *bucket_policy;
+
+  if (bucket_name.compare(s->bucket.name) != 0) {
+    RGWBucketInfo bucket_info;
+    map bucket_attrs;
+    int r = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL, &bucket_attrs);
+    if (r < 0) {
+      ldout(s->cct, 0) << "could not get bucket info for bucket=" << bucket_name << dendl;
+      return r;
+    }
+    bucket = bucket_info.bucket;
+    string no_obj;
+    bucket_policy = &_bucket_policy;
+    r = read_policy(store, s, bucket_info, bucket_attrs, bucket_policy, bucket, no_obj);
+    if (r < 0) {
+      ldout(s->cct, 0) << "failed to read bucket policy" << dendl;
+      return r;
+    }
+  } else {
+    bucket = s->bucket;
+    bucket_policy = s->bucket_acl;
+  }
+
+  /* dry run to find out total length */
+  int r = iterate_user_manifest_parts(s->cct, store, ofs, end, bucket, obj_prefix, bucket_policy, &total_len, NULL, NULL);
+  if (r < 0)
+    return r;
+
+  s->obj_size = total_len;
+
+  r = iterate_user_manifest_parts(s->cct, store, ofs, end, bucket, obj_prefix, bucket_policy, NULL, get_obj_user_manifest_iterate_cb, (void *)this);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+class RGWGetObj_CB : public RGWGetDataCB
+{
+  RGWGetObj *op;
+public:
+  RGWGetObj_CB(RGWGetObj *_op) : op(_op) {}
+  virtual ~RGWGetObj_CB() {}
+
+  int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) {
+    return op->get_data_cb(bl, bl_ofs, bl_len);
+  }
+};
+
+int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len)
+{
+  /* garbage collection related handling */
+  utime_t start_time = ceph_clock_now(s->cct);
+  if (start_time > gc_invalidate_time) {
+    int r = store->defer_gc(s->obj_ctx, obj);
+    if (r < 0) {
+      dout(0) << "WARNING: could not defer gc entry for obj" << dendl;
+    }
+    gc_invalidate_time = start_time;
+    gc_invalidate_time += (s->cct->_conf->rgw_gc_obj_min_wait / 2);
+  }
+  return send_response_data(bl, bl_ofs, bl_len);
+}
+
+void RGWGetObj::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWGetObj::execute()
+{
+  void *handle = NULL;
+  utime_t start_time = s->time;
+  bufferlist bl;
+  gc_invalidate_time = ceph_clock_now(s->cct);
+  gc_invalidate_time += (s->cct->_conf->rgw_gc_obj_min_wait / 2);
+
+  RGWGetObj_CB cb(this);
+
+  map::iterator attr_iter;
+
+  perfcounter->inc(l_rgw_get);
+  off_t new_ofs, new_end;
+
+  ret = get_params();
+  if (ret < 0)
+    goto done_err;
+
+  ret = init_common();
+  if (ret < 0)
+    goto done_err;
+
+  new_ofs = ofs;
+  new_end = end;
+
+  ret = store->prepare_get_obj(s->obj_ctx, obj, &new_ofs, &new_end, &attrs, mod_ptr,
+                               unmod_ptr, &lastmod, if_match, if_nomatch, &total_len, &s->obj_size, NULL, &handle, &s->err);
+  if (ret < 0)
+    goto done_err;
+
+  attr_iter = attrs.find(RGW_ATTR_USER_MANIFEST);
+  if (attr_iter != attrs.end()) {
+    ret = handle_user_manifest(attr_iter->second.c_str());
+    if (ret < 0) {
+      ldout(s->cct, 0) << "ERROR: failed to handle user manifest ret=" << ret << dendl;
+    }
+    return;
+  }
+
+  ofs = new_ofs;
+  end = new_end;
+
+  start = ofs;
+
+  if (!get_data || ofs > end)
+    goto done_err;
+
+  perfcounter->inc(l_rgw_get_b, end - ofs);
+
+  ret = store->get_obj_iterate(s->obj_ctx, &handle, obj, ofs, end, &cb);
+
+  perfcounter->tinc(l_rgw_get_lat,
+                   (ceph_clock_now(s->cct) - start_time));
+  if (ret < 0) {
+    goto done_err;
+  }
+
+  store->finish_get_obj(&handle);
+
+done_err:
+  send_response_data(bl, 0, 0);
+  store->finish_get_obj(&handle);
+}
+
+int RGWGetObj::init_common()
+{
+  if (range_str) {
+    int r = parse_range(range_str, ofs, end, &partial_content);
+    if (r < 0)
+      return r;
+  }
+  if (if_mod) {
+    if (parse_time(if_mod, &mod_time) < 0)
+      return -EINVAL;
+    mod_ptr = &mod_time;
+  }
+
+  if (if_unmod) {
+    if (parse_time(if_unmod, &unmod_time) < 0)
+      return -EINVAL;
+    unmod_ptr = &unmod_time;
+  }
+
+  return 0;
+}
+
+int RGWListBuckets::verify_permission()
+{
+  return 0;
+}
+
+void RGWListBuckets::execute()
+{
+  bool done;
+  bool started = false;
+  uint64_t total_count = 0;
+
+  uint64_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
+
+  ret = get_params();
+  if (ret < 0)
+    goto send_end;
+
+  do {
+    RGWUserBuckets buckets;
+    uint64_t read_count;
+    if (limit > 0)
+      read_count = min(limit - total_count, (uint64_t)max_buckets);
+    else
+      read_count = max_buckets;
+
+    ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
+                                marker, read_count, should_get_stats());
+
+    if (!started) {
+      send_response_begin(buckets.count() > 0);
+      started = true;
+    }
+
+    if (ret < 0) {
+      /* hmm.. something wrong here.. the user was authenticated, so it
+         should exist */
+      ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+      break;
+    }
+    map& m = buckets.get_buckets();
+
+    total_count += m.size();
+
+    done = (m.size() < read_count || (limit > 0 && total_count == limit));
+
+    if (!m.empty()) {
+      send_response_data(buckets);
+
+      map::reverse_iterator riter = m.rbegin();
+      marker = riter->first;
+    }
+  } while (!done);
+
+send_end:
+  if (!started) {
+    send_response_begin(false);
+  }
+  send_response_end();
+}
+
+int RGWStatAccount::verify_permission()
+{
+  return 0;
+}
+
+void RGWStatAccount::execute()
+{
+  string marker;
+  bool done;
+  uint64_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
+
+  do {
+    RGWUserBuckets buckets;
+
+    ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, max_buckets, true);
+    if (ret < 0) {
+      /* hmm.. something wrong here.. the user was authenticated, so it
+         should exist */
+      ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+      break;
+    } else {
+      map& m = buckets.get_buckets();
+      map::iterator iter;
+      for (iter = m.begin(); iter != m.end(); ++iter) {
+        RGWBucketEnt& bucket = iter->second;
+        buckets_size += bucket.size;
+        buckets_size_rounded += bucket.size_rounded;
+        buckets_objcount += bucket.count;
+
+        marker = iter->first;
+      }
+      buckets_count += m.size();
+
+      done = (m.size() < max_buckets);
+    }
+  } while (!done);
+}
+
+int RGWStatBucket::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_READ))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWStatBucket::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWStatBucket::execute()
+{
+  RGWUserBuckets buckets;
+  bucket.bucket = s->bucket;
+  buckets.add(bucket);
+  map& m = buckets.get_buckets();
+  ret = store->update_containers_stats(m);
+  if (!ret)
+    ret = -EEXIST;
+  if (ret > 0) {
+    ret = 0;
+    map::iterator iter = m.find(bucket.bucket.name);
+    if (iter != m.end()) {
+      bucket = iter->second;
+    } else {
+      ret = -EINVAL;
+    }
+  }
+}
+
+int RGWListBucket::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_READ))
+    return -EACCES;
+
+  return 0;
+}
+
+int RGWListBucket::parse_max_keys()
+{
+  if (!max_keys.empty()) {
+    char *endptr;
+    max = strtol(max_keys.c_str(), &endptr, 10);
+    if (endptr) {
+      while (*endptr && isspace(*endptr)) // ignore white space
+        endptr++;
+      if (*endptr) {
+        return -EINVAL;
+      }
+    }
+  } else {
+    max = default_max;
+  }
+
+  return 0;
+}
+
+void RGWListBucket::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWListBucket::execute()
+{
+  string no_ns;
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  string *pnext_marker = (delimiter.empty() ? NULL : &next_marker);
+
+  ret = store->list_objects(s->bucket, max, prefix, delimiter, marker, pnext_marker, objs, common_prefixes,
+                               !!(s->prot_flags & RGW_REST_SWIFT), no_ns, true, &is_truncated, NULL);
+}
+
+int RGWGetBucketLogging::verify_permission()
+{
+  if (s->user.user_id.compare(s->bucket_owner.get_id()) != 0)
+    return -EACCES;
+
+  return 0;
+}
+
+int RGWCreateBucket::verify_permission()
+{
+  if (!rgw_user_is_authenticated(s->user))
+    return -EACCES;
+
+  if (s->user.max_buckets) {
+    RGWUserBuckets buckets;
+    string marker;
+    int ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, s->user.max_buckets, false);
+    if (ret < 0)
+      return ret;
+
+    map& m = buckets.get_buckets();
+    if (m.size() >= s->user.max_buckets) {
+      return -ERR_TOO_MANY_BUCKETS;
+    }
+  }
+
+  return 0;
+}
+
+static int forward_request_to_master(struct req_state *s, obj_version *objv, RGWRados *store, bufferlist& in_data, JSONParser *jp)
+{
+  if (!store->rest_master_conn) {
+    ldout(s->cct, 0) << "rest connection is invalid" << dendl;
+    return -EINVAL;
+  }
+  ldout(s->cct, 0) << "sending create_bucket request to master region" << dendl;
+  bufferlist response;
+#define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response
+  int ret = store->rest_master_conn->forward(s->user.user_id, s->info, objv, MAX_REST_RESPONSE, &in_data, &response);
+  if (ret < 0)
+    return ret;
+
+  ldout(s->cct, 20) << "response: " << response.c_str() << dendl;
+  ret = jp->parse(response.c_str(), response.length());
+  if (ret < 0) {
+    ldout(s->cct, 0) << "failed parsing response from master region" << dendl;
+    return ret;
+  }
+
+  return 0;
+}
+
+void RGWCreateBucket::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWCreateBucket::execute()
+{
+  RGWAccessControlPolicy old_policy(s->cct);
+  map attrs;
+  bufferlist aclbl;
+  bufferlist corsbl;
+  bool existed;
+  int r;
+  rgw_obj obj(store->zone.domain_root, s->bucket_name_str);
+  obj_version objv, *pobjv = NULL;
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  if (!store->region.is_master &&
+      store->region.api_name != location_constraint) {
+    ldout(s->cct, 0) << "location constraint (" << location_constraint << ") doesn't match region" << " (" << store->region.api_name << ")" << dendl;
+    ret = -EINVAL;
+    return;
+  }
+
+  /* we need to make sure we read bucket info, it's not read before for this specific request */
+  ret = store->get_bucket_info(s->obj_ctx, s->bucket_name_str, s->bucket_info, NULL, &s->bucket_attrs);
+  if (ret < 0 && ret != -ENOENT)
+    return;
+  s->bucket_exists = (ret != -ENOENT);
+
+  s->bucket_owner.set_id(s->user.user_id);
+  s->bucket_owner.set_name(s->user.display_name);
+  if (s->bucket_exists) {
+    r = get_policy_from_attr(s->cct, store, s->obj_ctx, s->bucket_info, s->bucket_attrs,
+                             &old_policy, obj);
+    if (r >= 0)  {
+      if (old_policy.get_owner().get_id().compare(s->user.user_id) != 0) {
+        ret = -EEXIST;
+        return;
+      }
+    }
+  }
+
+  RGWBucketInfo master_info;
+  rgw_bucket *pmaster_bucket;
+  time_t creation_time;
+
+  if (!store->region.is_master) {
+    JSONParser jp;
+    ret = forward_request_to_master(s, NULL, store, in_data, &jp);
+    if (ret < 0)
+      return;
+
+    JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp);
+    JSONDecoder::decode_json("object_ver", objv, &jp);
+    JSONDecoder::decode_json("bucket_info", master_info, &jp);
+    ldout(s->cct, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl;
+    ldout(s->cct, 20) << "got creation time: << " << master_info.creation_time << dendl;
+    pmaster_bucket= &master_info.bucket;
+    creation_time = master_info.creation_time;
+    pobjv = &objv;
+  } else {
+    pmaster_bucket = NULL;
+    creation_time = 0;
+  }
+
+  string region_name;
+
+  if (s->system_request) {
+    region_name = s->info.args.get(RGW_SYS_PARAM_PREFIX "region");
+    if (region_name.empty()) {
+      region_name = store->region.name;
+    }
+  } else {
+    region_name = store->region.name;
+  }
+
+  policy.encode(aclbl);
+
+  attrs[RGW_ATTR_ACL] = aclbl;
+
+  if (has_cors) {
+    cors_config.encode(corsbl);
+    attrs[RGW_ATTR_CORS] = corsbl;
+  }
+  s->bucket.name = s->bucket_name_str;
+  ret = store->create_bucket(s->user, s->bucket, region_name, placement_rule, attrs, info, pobjv,
+                             &ep_objv, creation_time, pmaster_bucket, true);
+  /* continue if EEXIST and create_bucket will fail below.  this way we can recover
+   * from a partial create by retrying it. */
+  ldout(s->cct, 20) << "rgw_create_bucket returned ret=" << ret << " bucket=" << s->bucket << dendl;
+
+  if (ret && ret != -EEXIST)
+    return;
+
+  existed = (ret == -EEXIST);
+
+  if (existed) {
+    /* bucket already existed, might have raced with another bucket creation, or
+     * might be partial bucket creation that never completed. Read existing bucket
+     * info, verify that the reported bucket owner is the current user.
+     * If all is ok then update the user's list of buckets
+     */
+    if (info.owner.compare(s->user.user_id) != 0) {
+      ret = -ERR_BUCKET_EXISTS;
+      return;
+    }
+    s->bucket = info.bucket;
+  }
+
+  ret = rgw_link_bucket(store, s->user.user_id, s->bucket, info.creation_time, false);
+  if (ret && !existed && ret != -EEXIST) {  /* if it exists (or previously existed), don't remove it! */
+    ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.name);
+    if (ret < 0) {
+      ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl;
+    }
+  }
+
+  if (ret == -EEXIST)
+    ret = -ERR_BUCKET_EXISTS;
+}
+
+int RGWDeleteBucket::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWDeleteBucket::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWDeleteBucket::execute()
+{
+  ret = -EINVAL;
+
+  if (s->bucket_name_str.empty())
+    return;
+
+  RGWObjVersionTracker ot;
+  ot.read_version = s->bucket_info.ep_objv;
+
+  if (s->system_request) {
+    string tag = s->info.args.get(RGW_SYS_PARAM_PREFIX "tag");
+    string ver_str = s->info.args.get(RGW_SYS_PARAM_PREFIX "ver");
+    if (!tag.empty()) {
+      ot.read_version.tag = tag;
+      uint64_t ver;
+      string err;
+      ver = strict_strtol(ver_str.c_str(), 10, &err);
+      if (!err.empty()) {
+        ldout(s->cct, 0) << "failed to parse ver param" << dendl;
+        ret = -EINVAL;
+        return;
+      }
+      ot.read_version.ver = ver;
+    }
+  }
+
+  ret = store->delete_bucket(s->bucket, ot);
+
+  if (ret == 0) {
+    ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.name, false);
+    if (ret < 0) {
+      ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl;
+    }
+  }
+
+  if (ret < 0) {
+    return;
+  }
+
+  if (!store->region.is_master) {
+    bufferlist in_data;
+    JSONParser jp;
+    ret = forward_request_to_master(s, &ot.read_version, store, in_data, &jp);
+    if (ret < 0) {
+      if (ret == -ENOENT) { /* adjust error,
+                               we want to return with NoSuchBucket and not NoSuchKey */
+        ret = -ERR_NO_SUCH_BUCKET;
+      }
+      return;
+    }
+  }
+
+}
+
+int RGWPutObj::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+    return -EACCES;
+
+  return 0;
+}
+
+class RGWPutObjProcessor_Multipart : public RGWPutObjProcessor_Atomic
+{
+  string part_num;
+  RGWMPObj mp;
+  req_state *s;
+  string upload_id;
+
+protected:
+  int prepare(RGWRados *store, void *obj_ctx, string *oid_rand);
+  int do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs);
+
+public:
+  bool immutable_head() { return true; }
+  RGWPutObjProcessor_Multipart(const string& bucket_owner, uint64_t _p, req_state *_s) :
+                   RGWPutObjProcessor_Atomic(bucket_owner, _s->bucket, _s->object_str, _p, _s->req_id), s(_s) {}
+};
+
+int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, void *obj_ctx, string *oid_rand)
+{
+  int r = prepare_init(store, obj_ctx, NULL);
+  if (r < 0) {
+    return r;
+  }
+
+  string oid = obj_str;
+  upload_id = s->info.args.get("uploadId");
+  if (!oid_rand) {
+    mp.init(oid, upload_id);
+  } else {
+    mp.init(oid, upload_id, *oid_rand);
+  }
+
+  part_num = s->info.args.get("partNumber");
+  if (part_num.empty()) {
+    ldout(s->cct, 10) << "part number is empty" << dendl;
+    return -EINVAL;
+  }
+
+  string err;
+  uint64_t num = (uint64_t)strict_strtol(part_num.c_str(), 10, &err);
+
+  if (!err.empty()) {
+    ldout(s->cct, 10) << "bad part number: " << part_num << ": " << err << dendl;
+    return -EINVAL;
+  }
+
+  string upload_prefix = oid + ".";
+
+  if (!oid_rand) {
+    upload_prefix.append(upload_id);
+  } else {
+    upload_prefix.append(*oid_rand);
+  }
+
+  rgw_obj target_obj;
+  target_obj.init(bucket, oid);
+
+  manifest.set_prefix(upload_prefix);
+
+  manifest.set_multipart_part_rule(store->ctx()->_conf->rgw_obj_stripe_size, num);
+
+  r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, target_obj);
+  if (r < 0) {
+    return r;
+  }
+
+  head_obj = manifest_gen.get_cur_obj();
+  cur_obj = head_obj;
+  add_obj(cur_obj);
+
+  return 0;
+}
+
+static bool is_v2_upload_id(const string& upload_id)
+{
+  const char *uid = upload_id.c_str();
+
+  return (strncmp(uid, MULTIPART_UPLOAD_ID_PREFIX, sizeof(MULTIPART_UPLOAD_ID_PREFIX) - 1) == 0);
+}
+
+int RGWPutObjProcessor_Multipart::do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs)
+{
+  complete_writing_data();
+
+  RGWRados::PutObjMetaExtraParams params;
+  params.set_mtime = set_mtime;
+  params.mtime = mtime;
+  params.owner = s->owner.get_id();
+
+  int r = store->put_obj_meta(obj_ctx, head_obj, s->obj_size, attrs, RGW_OBJ_CATEGORY_MAIN, 0, params);
+  if (r < 0)
+    return r;
+
+  bufferlist bl;
+  RGWUploadPartInfo info;
+  string p = "part.";
+  bool sorted_omap = is_v2_upload_id(upload_id);
+
+  if (sorted_omap) {
+    string err;
+    int part_num_int = strict_strtol(part_num.c_str(), 10, &err);
+    if (!err.empty()) {
+      dout(10) << "bad part number specified: " << part_num << dendl;
+      return -EINVAL;
+    }
+    char buf[32];
+    snprintf(buf, sizeof(buf), "%08d", part_num_int);
+    p.append(buf);
+  } else {
+    p.append(part_num);
+  }
+  info.num = atoi(part_num.c_str());
+  info.etag = etag;
+  info.size = s->obj_size;
+  info.modified = ceph_clock_now(store->ctx());
+  info.manifest = manifest;
+  ::encode(info, bl);
+
+  string multipart_meta_obj = mp.get_meta();
+
+  rgw_obj meta_obj;
+  meta_obj.init_ns(bucket, multipart_meta_obj, mp_ns);
+  meta_obj.set_in_extra_data(true);
+
+  r = store->omap_set(meta_obj, p, bl);
+
+  return r;
+}
+
+
+RGWPutObjProcessor *RGWPutObj::select_processor(bool *is_multipart)
+{
+  RGWPutObjProcessor *processor;
+
+  bool multipart = s->info.args.exists("uploadId");
+
+  uint64_t part_size = s->cct->_conf->rgw_obj_stripe_size;
+
+  const string& bucket_owner = s->bucket_owner.get_id();
+
+  if (!multipart) {
+    processor = new RGWPutObjProcessor_Atomic(bucket_owner, s->bucket, s->object_str, part_size, s->req_id);
+  } else {
+    processor = new RGWPutObjProcessor_Multipart(bucket_owner, part_size, s);
+  }
+
+  if (is_multipart) {
+    *is_multipart = multipart;
+  }
+
+  return processor;
+}
+
+void RGWPutObj::dispose_processor(RGWPutObjProcessor *processor)
+{
+  delete processor;
+}
+
+void RGWPutObj::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+static int put_obj_user_manifest_iterate_cb(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs,
+                                       void *param)
+{
+  RGWPutObj *op = (RGWPutObj *)param;
+  return op->user_manifest_iterate_cb(bucket, ent, bucket_policy, start_ofs, end_ofs);
+}
+
+int RGWPutObj::user_manifest_iterate_cb(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs)
+{
+  rgw_obj part(bucket, ent.name);
+
+  map attrs;
+
+  int ret = get_obj_attrs(store, s, part, attrs, NULL, NULL);
+  if (ret < 0) {
+    return ret;
+  }
+  map::iterator iter = attrs.find(RGW_ATTR_ETAG);
+  if (iter == attrs.end()) {
+    return 0;
+  }
+  bufferlist& bl = iter->second;
+  const char *buf = bl.c_str();
+  int len = bl.length();
+  while (len > 0 && buf[len - 1] == '\0') {
+    len--;
+  }
+  if (len > 0) {
+    user_manifest_parts_hash->Update((const byte *)bl.c_str(), len);
+  }
+
+  if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
+    string e(bl.c_str(), bl.length());
+    ldout(s->cct, 20) << __func__ << ": appending user manifest etag: " << e << dendl;
+  }
+
+  return 0;
+}
+
+static int put_data_and_throttle(RGWPutObjProcessor *processor, bufferlist& data, off_t ofs,
+                                 MD5 *hash, bool need_to_wait)
+{
+  const unsigned char *data_ptr = (hash ? (const unsigned char *)data.c_str() : NULL);
+  bool again;
+  uint64_t len = data.length();
+
+  do {
+    void *handle;
+
+    int ret = processor->handle_data(data, ofs, &handle, &again);
+    if (ret < 0)
+      return ret;
+
+    if (hash) {
+      hash->Update(data_ptr, len);
+      hash = NULL; /* only calculate hash once */
+    }
+
+    ret = processor->throttle_data(handle, need_to_wait);
+    if (ret < 0)
+      return ret;
+
+    need_to_wait = false; /* the need to wait only applies to the first iteration */
+  } while (again);
+
+  return 0;
+}
+
+
+void RGWPutObj::execute()
+{
+  RGWPutObjProcessor *processor = NULL;
+  char supplied_md5_bin[CEPH_CRYPTO_MD5_DIGESTSIZE + 1];
+  char supplied_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
+  char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
+  unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  MD5 hash;
+  bufferlist bl, aclbl;
+  map attrs;
+  int len;
+  map::iterator iter;
+  bool multipart;
+
+  bool need_calc_md5 = (obj_manifest == NULL);
+
+
+  perfcounter->inc(l_rgw_put);
+  ret = -EINVAL;
+  if (!s->object) {
+    goto done;
+  }
+
+  ret = get_params();
+  if (ret < 0)
+    goto done;
+
+  if (supplied_md5_b64) {
+    need_calc_md5 = true;
+
+    ldout(s->cct, 15) << "supplied_md5_b64=" << supplied_md5_b64 << dendl;
+    ret = ceph_unarmor(supplied_md5_bin, &supplied_md5_bin[CEPH_CRYPTO_MD5_DIGESTSIZE + 1],
+                       supplied_md5_b64, supplied_md5_b64 + strlen(supplied_md5_b64));
+    ldout(s->cct, 15) << "ceph_armor ret=" << ret << dendl;
+    if (ret != CEPH_CRYPTO_MD5_DIGESTSIZE) {
+      ret = -ERR_INVALID_DIGEST;
+      goto done;
+    }
+
+    buf_to_hex((const unsigned char *)supplied_md5_bin, CEPH_CRYPTO_MD5_DIGESTSIZE, supplied_md5);
+    ldout(s->cct, 15) << "supplied_md5=" << supplied_md5 << dendl;
+  }
+
+  if (!chunked_upload) { /* with chunked upload we don't know how big is the upload.
+                            we also check sizes at the end anyway */
+    ret = store->check_quota(s->bucket_owner.get_id(), s->bucket,
+                             user_quota, bucket_quota, s->content_length);
+    if (ret < 0) {
+      goto done;
+    }
+  }
+
+  if (supplied_etag) {
+    strncpy(supplied_md5, supplied_etag, sizeof(supplied_md5) - 1);
+    supplied_md5[sizeof(supplied_md5) - 1] = '\0';
+  }
+
+  processor = select_processor(&multipart);
+
+  ret = processor->prepare(store, s->obj_ctx, NULL);
+  if (ret < 0)
+    goto done;
+
+  do {
+    bufferlist data;
+    len = get_data(data);
+    if (len < 0) {
+      ret = len;
+      goto done;
+    }
+    if (!len)
+      break;
+
+    /* do we need this operation to be synchronous? if we're dealing with an object with immutable
+     * head, e.g., multipart object we need to make sure we're the first one writing to this object
+     */
+    bool need_to_wait = (ofs == 0) && multipart;
+
+    ret = put_data_and_throttle(processor, data, ofs, (need_calc_md5 ? &hash : NULL), need_to_wait);
+    if (ret < 0) {
+      if (!need_to_wait || ret != -EEXIST) {
+        ldout(s->cct, 20) << "processor->thottle_data() returned ret=" << ret << dendl;
+        goto done;
+      }
+
+      ldout(s->cct, 5) << "NOTICE: processor->throttle_data() returned -EEXIST, need to restart write" << dendl;
+
+      /* restart processing with different oid suffix */
+
+      dispose_processor(processor);
+      processor = select_processor(&multipart);
+
+      string oid_rand;
+      char buf[33];
+      gen_rand_alphanumeric(store->ctx(), buf, sizeof(buf) - 1);
+      oid_rand.append(buf);
+
+      ret = processor->prepare(store, s->obj_ctx, &oid_rand);
+      if (ret < 0) {
+        ldout(s->cct, 0) << "ERROR: processor->prepare() returned " << ret << dendl;
+        goto done;
+      }
+
+      ret = put_data_and_throttle(processor, data, ofs, NULL, false);
+      if (ret < 0) {
+        goto done;
+      }
+    }
+
+    ofs += len;
+  } while (len > 0);
+
+  if (!chunked_upload && (uint64_t)ofs != s->content_length) {
+    ret = -ERR_REQUEST_TIMEOUT;
+    goto done;
+  }
+  s->obj_size = ofs;
+  perfcounter->inc(l_rgw_put_b, s->obj_size);
+
+  ret = store->check_quota(s->bucket_owner.get_id(), s->bucket,
+                           user_quota, bucket_quota, s->obj_size);
+  if (ret < 0) {
+    goto done;
+  }
+
+  if (need_calc_md5) {
+    hash.Final(m);
+
+    buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
+    etag = calc_md5;
+
+    if (supplied_md5_b64 && strcmp(calc_md5, supplied_md5)) {
+      ret = -ERR_BAD_DIGEST;
+      goto done;
+    }
+  }
+
+  policy.encode(aclbl);
+
+  attrs[RGW_ATTR_ACL] = aclbl;
+  if (obj_manifest) {
+    bufferlist manifest_bl;
+    string manifest_obj_prefix;
+    string manifest_bucket;
+    RGWBucketInfo bucket_info;
+
+    char etag_buf[CEPH_CRYPTO_MD5_DIGESTSIZE];
+    char etag_buf_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16];
+
+    manifest_bl.append(obj_manifest, strlen(obj_manifest) + 1);
+    attrs[RGW_ATTR_USER_MANIFEST] = manifest_bl;
+    user_manifest_parts_hash = &hash;
+    string prefix_str = obj_manifest;
+    int pos = prefix_str.find('/');
+    if (pos < 0) {
+      ldout(s->cct, 0) << "bad user manifest, missing slash separator: " << obj_manifest << dendl;
+      goto done;
+    }
+
+    manifest_bucket = prefix_str.substr(0, pos);
+    manifest_obj_prefix = prefix_str.substr(pos + 1);
+
+    ret = store->get_bucket_info(NULL, manifest_bucket, bucket_info, NULL, NULL);
+    if (ret < 0) {
+      ldout(s->cct, 0) << "could not get bucket info for bucket=" << manifest_bucket << dendl;
+    }
+    ret = iterate_user_manifest_parts(s->cct, store, 0, -1, bucket_info.bucket, manifest_obj_prefix,
+                                      NULL, NULL, put_obj_user_manifest_iterate_cb, (void *)this);
+    if (ret < 0) {
+      goto done;
+    }
+
+    hash.Final((byte *)etag_buf);
+    buf_to_hex((const unsigned char *)etag_buf, CEPH_CRYPTO_MD5_DIGESTSIZE, etag_buf_str);
+
+    ldout(s->cct, 0) << __func__ << ": calculated md5 for user manifest: " << etag_buf_str << dendl;
+
+    etag = etag_buf_str;
+  }
+  if (supplied_etag && etag.compare(supplied_etag) != 0) {
+    ret = -ERR_UNPROCESSABLE_ENTITY;
+    goto done;
+  }
+  bl.append(etag.c_str(), etag.size() + 1);
+  attrs[RGW_ATTR_ETAG] = bl;
+
+  for (iter = s->generic_attrs.begin(); iter != s->generic_attrs.end(); ++iter) {
+    bufferlist& attrbl = attrs[iter->first];
+    const string& val = iter->second;
+    attrbl.append(val.c_str(), val.size() + 1);
+  }
+
+  rgw_get_request_metadata(s->cct, s->info, attrs);
+
+  ret = processor->complete(etag, &mtime, 0, attrs);
+done:
+  dispose_processor(processor);
+  perfcounter->tinc(l_rgw_put_lat,
+                   (ceph_clock_now(s->cct) - s->time));
+}
+
+int RGWPostObj::verify_permission()
+{
+  return 0;
+}
+
+RGWPutObjProcessor *RGWPostObj::select_processor()
+{
+  RGWPutObjProcessor *processor;
+
+  uint64_t part_size = s->cct->_conf->rgw_obj_stripe_size;
+
+  processor = new RGWPutObjProcessor_Atomic(s->bucket_owner.get_id(), s->bucket, s->object_str, part_size, s->req_id);
+
+  return processor;
+}
+
+void RGWPostObj::dispose_processor(RGWPutObjProcessor *processor)
+{
+  delete processor;
+}
+
+void RGWPostObj::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWPostObj::execute()
+{
+  RGWPutObjProcessor *processor = NULL;
+  char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
+  unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  MD5 hash;
+  bufferlist bl, aclbl;
+  int len = 0;
+
+  // read in the data from the POST form
+  ret = get_params();
+  if (ret < 0)
+    goto done;
+
+  ret = verify_params();
+  if (ret < 0)
+    goto done;
+
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE)) {
+    ret = -EACCES;
+    goto done;
+  }
+
+  processor = select_processor();
+
+  ret = processor->prepare(store, s->obj_ctx, NULL);
+  if (ret < 0)
+    goto done;
+
+  while (data_pending) {
+     bufferlist data;
+     len = get_data(data);
+
+     if (len < 0) {
+       ret = len;
+       goto done;
+     }
+
+     if (!len)
+       break;
+
+     ret = put_data_and_throttle(processor, data, ofs, &hash, false);
+
+     ofs += len;
+
+     if (ofs > max_len) {
+       ret = -ERR_TOO_LARGE;
+       goto done;
+     }
+   }
+
+  if (len < min_len) {
+    ret = -ERR_TOO_SMALL;
+    goto done;
+  }
+
+  s->obj_size = ofs;
+
+  hash.Final(m);
+  buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
+
+  policy.encode(aclbl);
+  etag = calc_md5;
+
+  bl.append(etag.c_str(), etag.size() + 1);
+  attrs[RGW_ATTR_ETAG] = bl;
+  attrs[RGW_ATTR_ACL] = aclbl;
+
+  if (content_type.size()) {
+    bufferlist ct_bl;
+    ct_bl.append(content_type.c_str(), content_type.size() + 1);
+    attrs[RGW_ATTR_CONTENT_TYPE] = ct_bl;
+  }
+
+  ret = processor->complete(etag, NULL, 0, attrs);
+
+done:
+  dispose_processor(processor);
+}
+
+
+int RGWPutMetadata::verify_permission()
+{
+  if (s->object) {
+    if (!verify_object_permission(s, RGW_PERM_WRITE))
+      return -EACCES;
+  } else {
+    if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+      return -EACCES;
+  }
+
+  return 0;
+}
+
+void RGWPutMetadata::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWPutMetadata::execute()
+{
+  const char *meta_prefix = RGW_ATTR_META_PREFIX;
+  int meta_prefix_len = sizeof(RGW_ATTR_META_PREFIX) - 1;
+  map attrs, orig_attrs, rmattrs;
+  map::iterator iter;
+  bufferlist bl, cors_bl;
+
+  rgw_obj obj(s->bucket, s->object_str);
+
+  store->set_atomic(s->obj_ctx, obj);
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  rgw_get_request_metadata(s->cct, s->info, attrs);
+
+  /* no need to track object versioning, need it for bucket's data only */
+  RGWObjVersionTracker *ptracker = (s->object ? NULL : &s->bucket_info.objv_tracker);
+
+  /* check if obj exists, read orig attrs */
+  ret = get_obj_attrs(store, s, obj, orig_attrs, NULL, ptracker);
+  if (ret < 0)
+    return;
+
+  /* only remove meta attrs */
+  for (iter = orig_attrs.begin(); iter != orig_attrs.end(); ++iter) {
+    const string& name = iter->first;
+    if (name.compare(0, meta_prefix_len, meta_prefix) == 0) {
+      rmattrs[name] = iter->second;
+    } else if (attrs.find(name) == attrs.end()) {
+      attrs[name] = iter->second;
+    }
+  }
+
+  map::iterator giter;
+  for (giter = s->generic_attrs.begin(); giter != s->generic_attrs.end(); ++giter) {
+    bufferlist& attrbl = attrs[giter->first];
+    const string& val = giter->second;
+    attrbl.append(val.c_str(), val.size() + 1);
+  }
+
+  if (has_policy) {
+    policy.encode(bl);
+    attrs[RGW_ATTR_ACL] = bl;
+  }
+  if (has_cors) {
+    cors_config.encode(cors_bl);
+    attrs[RGW_ATTR_CORS] = cors_bl;
+  }
+  if (s->object) {
+    ret = store->set_attrs(s->obj_ctx, obj, attrs, &rmattrs, ptracker);
+  } else {
+    ret = rgw_bucket_set_attrs(store, s->bucket_info, attrs, &rmattrs, ptracker);
+  }
+}
+
+int RGWSetTempUrl::verify_permission()
+{
+  if (s->perm_mask != RGW_PERM_FULL_CONTROL)
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWSetTempUrl::execute()
+{
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  RGWUserAdminOpState user_op;
+  user_op.set_user_id(s->user.user_id);
+  map::iterator iter;
+  for (iter = temp_url_keys.begin(); iter != temp_url_keys.end(); ++iter) {
+    user_op.set_temp_url_key(iter->second, iter->first);
+  }
+
+  RGWUser user;
+  ret = user.init(store, user_op);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: could not init user ret=" << ret << dendl;
+    return;
+  }
+  string err_msg;
+  ret = user.modify(user_op, &err_msg);
+  if (ret < 0) {
+    ldout(store->ctx(), 10) << "user.modify() returned " << ret << ": " << err_msg << dendl;
+    return;
+  }
+}
+
+
+int RGWDeleteObj::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWDeleteObj::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWDeleteObj::execute()
+{
+  ret = -EINVAL;
+  rgw_obj obj(s->bucket, s->object_str);
+  if (s->object) {
+    store->set_atomic(s->obj_ctx, obj);
+    ret = store->delete_obj(s->obj_ctx, s->bucket_owner.get_id(), obj);
+  }
+}
+
+bool RGWCopyObj::parse_copy_location(const char *src, string& bucket_name, string& object)
+{
+  string url_src(src);
+  string dec_src;
+
+  url_decode(url_src, dec_src);
+  src = dec_src.c_str();
+
+  if (*src == '/') ++src;
+
+  string str(src);
+
+  int pos = str.find("/");
+  if (pos <= 0)
+    return false;
+
+  bucket_name = str.substr(0, pos);
+  object = str.substr(pos + 1);
+
+  if (object.size() == 0)
+    return false;
+
+  return true;
+}
+
+int RGWCopyObj::verify_permission()
+{
+  string empty_str;
+  RGWAccessControlPolicy src_policy(s->cct);
+  ret = get_params();
+  if (ret < 0)
+    return ret;
+
+  map src_attrs;
+
+  ret = store->get_bucket_info(s->obj_ctx, src_bucket_name, src_bucket_info, NULL, &src_attrs);
+  if (ret < 0)
+    return ret;
+
+  src_bucket = src_bucket_info.bucket;
+
+  /* get buckets info (source and dest) */
+  if (s->local_source &&  source_zone.empty()) {
+    rgw_obj src_obj(src_bucket, src_object);
+    store->set_atomic(s->obj_ctx, src_obj);
+    store->set_prefetch_data(s->obj_ctx, src_obj);
+
+    /* check source object permissions */
+    ret = read_policy(store, s, src_bucket_info, src_attrs, &src_policy, src_bucket, src_object);
+    if (ret < 0)
+      return ret;
+
+    if (!s->system_request && /* system request overrides permission checks */
+        !src_policy.verify_permission(s->user.user_id, s->perm_mask, RGW_PERM_READ))
+      return -EACCES;
+  }
+
+  RGWAccessControlPolicy dest_bucket_policy(s->cct);
+  map dest_attrs;
+
+  if (src_bucket_name.compare(dest_bucket_name) == 0) { /* will only happen if s->local_source */
+    dest_bucket_info = src_bucket_info;
+  } else {
+    ret = store->get_bucket_info(s->obj_ctx, dest_bucket_name, dest_bucket_info, NULL, &dest_attrs);
+    if (ret < 0)
+      return ret;
+  }
+
+  dest_bucket = dest_bucket_info.bucket;
+
+  rgw_obj dest_obj(dest_bucket, dest_object);
+  store->set_atomic(s->obj_ctx, dest_obj);
+
+  /* check dest bucket permissions */
+  ret = read_policy(store, s, dest_bucket_info, dest_attrs, &dest_bucket_policy, dest_bucket, empty_str);
+  if (ret < 0)
+    return ret;
+
+  if (!s->system_request && /* system request overrides permission checks */
+      !dest_bucket_policy.verify_permission(s->user.user_id, s->perm_mask, RGW_PERM_WRITE))
+    return -EACCES;
+
+  ret = init_dest_policy();
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+
+int RGWCopyObj::init_common()
+{
+  if (if_mod) {
+    if (parse_time(if_mod, &mod_time) < 0) {
+      ret = -EINVAL;
+      return ret;
+    }
+    mod_ptr = &mod_time;
+  }
+
+  if (if_unmod) {
+    if (parse_time(if_unmod, &unmod_time) < 0) {
+      ret = -EINVAL;
+      return ret;
+    }
+    unmod_ptr = &unmod_time;
+  }
+
+  bufferlist aclbl;
+  dest_policy.encode(aclbl);
+
+  attrs[RGW_ATTR_ACL] = aclbl;
+  rgw_get_request_metadata(s->cct, s->info, attrs);
+
+  map::iterator iter;
+  for (iter = s->generic_attrs.begin(); iter != s->generic_attrs.end(); ++iter) {
+    bufferlist& attrbl = attrs[iter->first];
+    const string& val = iter->second;
+    attrbl.append(val.c_str(), val.size() + 1);
+  }
+  return 0;
+}
+
+static void copy_obj_progress_cb(off_t ofs, void *param)
+{
+  RGWCopyObj *op = static_cast(param);
+  op->progress_cb(ofs);
+}
+
+void RGWCopyObj::progress_cb(off_t ofs)
+{
+  if (!s->cct->_conf->rgw_copy_obj_progress)
+    return;
+
+  if (ofs - last_ofs < s->cct->_conf->rgw_copy_obj_progress_every_bytes)
+    return;
+
+  send_partial_response(ofs);
+
+  last_ofs = ofs;
+}
+
+void RGWCopyObj::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWCopyObj::execute()
+{
+  rgw_obj src_obj, dst_obj;
+
+  if (init_common() < 0)
+    return;
+
+  src_obj.init(src_bucket, src_object);
+  dst_obj.init(dest_bucket, dest_object);
+  store->set_atomic(s->obj_ctx, src_obj);
+
+  store->set_atomic(s->obj_ctx, dst_obj);
+
+  ret = store->copy_obj(s->obj_ctx,
+                        s->user.user_id,
+                        client_id,
+                        op_id,
+                        &s->info,
+                        source_zone,
+                        dst_obj,
+                        src_obj,
+                        dest_bucket_info,
+                        src_bucket_info,
+                        &mtime,
+                        mod_ptr,
+                        unmod_ptr,
+                        if_match,
+                        if_nomatch,
+                        replace_attrs,
+                        attrs, RGW_OBJ_CATEGORY_MAIN,
+                        &s->req_id, /* use req_id as tag */
+                        &s->err,
+                        copy_obj_progress_cb, (void *)this
+                        );
+}
+
+int RGWGetACLs::verify_permission()
+{
+  bool perm;
+  if (s->object) {
+    perm = verify_object_permission(s, RGW_PERM_READ_ACP);
+  } else {
+    perm = verify_bucket_permission(s, RGW_PERM_READ_ACP);
+  }
+  if (!perm)
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWGetACLs::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWGetACLs::execute()
+{
+  stringstream ss;
+  RGWAccessControlPolicy *acl = (s->object ? s->object_acl : s->bucket_acl);
+  RGWAccessControlPolicy_S3 *s3policy = static_cast(acl);
+  s3policy->to_xml(ss);
+  acls = ss.str();
+}
+
+
+
+int RGWPutACLs::verify_permission()
+{
+  bool perm;
+  if (s->object) {
+    perm = verify_object_permission(s, RGW_PERM_WRITE_ACP);
+  } else {
+    perm = verify_bucket_permission(s, RGW_PERM_WRITE_ACP);
+  }
+  if (!perm)
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWPutACLs::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWPutACLs::execute()
+{
+  bufferlist bl;
+
+  RGWAccessControlPolicy_S3 *policy = NULL;
+  RGWACLXMLParser_S3 parser(s->cct);
+  RGWAccessControlPolicy_S3 new_policy(s->cct);
+  stringstream ss;
+  char *new_data = NULL;
+  ACLOwner owner;
+  rgw_obj obj;
+
+  ret = 0;
+
+  if (!parser.init()) {
+    ret = -EINVAL;
+    return;
+  }
+
+  owner.set_id(s->user.user_id);
+  owner.set_name(s->user.display_name);
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  ldout(s->cct, 15) << "read len=" << len << " data=" << (data ? data : "") << dendl;
+
+  if (!s->canned_acl.empty() && len) {
+    ret = -EINVAL;
+    return;
+  }
+
+  if (!s->canned_acl.empty() || s->has_acl_header) {
+    ret = get_policy_from_state(store, s, ss);
+    if (ret < 0)
+      return;
+
+    new_data = strdup(ss.str().c_str());
+    free(data);
+    data = new_data;
+    len = ss.str().size();
+  }
+
+  if (!parser.parse(data, len, 1)) {
+    ret = -EACCES;
+    return;
+  }
+  policy = static_cast(parser.find_first("AccessControlPolicy"));
+  if (!policy) {
+    ret = -EINVAL;
+    return;
+  }
+
+  if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
+    ldout(s->cct, 15) << "Old AccessControlPolicy";
+    policy->to_xml(*_dout);
+    *_dout << dendl;
+  }
+
+  ret = policy->rebuild(store, &owner, new_policy);
+  if (ret < 0)
+    return;
+
+  if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
+    ldout(s->cct, 15) << "New AccessControlPolicy:";
+    new_policy.to_xml(*_dout);
+    *_dout << dendl;
+  }
+
+  RGWObjVersionTracker *ptracker = (s->object ? NULL : &s->bucket_info.objv_tracker);
+
+  new_policy.encode(bl);
+  obj.init(s->bucket, s->object_str);
+  map attrs;
+  attrs[RGW_ATTR_ACL] = bl;
+  store->set_atomic(s->obj_ctx, obj);
+  if (s->object) {
+    ret = store->set_attrs(s->obj_ctx, obj, attrs, NULL, ptracker);
+  } else {
+    ret = rgw_bucket_set_attrs(store, s->bucket_info, attrs, NULL, ptracker);
+  }
+}
+
+int RGWGetCORS::verify_permission()
+{
+  if (s->user.user_id.compare(s->bucket_owner.get_id()) != 0)
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWGetCORS::execute()
+{
+  ret = read_bucket_cors();
+  if (ret < 0)
+    return ;
+
+  if (!cors_exist) {
+    dout(2) << "No CORS configuration set yet for this bucket" << dendl;
+    ret = -ENOENT;
+    return;
+  }
+}
+
+int RGWPutCORS::verify_permission()
+{
+  if (s->user.user_id.compare(s->bucket_owner.get_id()) != 0)
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWPutCORS::execute()
+{
+  rgw_obj obj;
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  RGWObjVersionTracker *ptracker = (s->object ? NULL : &s->bucket_info.objv_tracker);
+
+  store->get_bucket_instance_obj(s->bucket, obj);
+  store->set_atomic(s->obj_ctx, obj);
+  ret = store->set_attr(s->obj_ctx, obj, RGW_ATTR_CORS, cors_bl, ptracker);
+}
+
+int RGWDeleteCORS::verify_permission()
+{
+  if (s->user.user_id.compare(s->bucket_owner.get_id()) != 0)
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWDeleteCORS::execute()
+{
+  ret = read_bucket_cors();
+  if (ret < 0)
+    return;
+
+  bufferlist bl;
+  rgw_obj obj;
+  if (!cors_exist) {
+    dout(2) << "No CORS configuration set yet for this bucket" << dendl;
+    ret = -ENOENT;
+    return;
+  }
+  store->get_bucket_instance_obj(s->bucket, obj);
+  store->set_atomic(s->obj_ctx, obj);
+  map orig_attrs, attrs, rmattrs;
+  map::iterator iter;
+
+  RGWObjVersionTracker *ptracker = (s->object ? NULL : &s->bucket_info.objv_tracker);
+
+  /* check if obj exists, read orig attrs */
+  ret = get_obj_attrs(store, s, obj, orig_attrs, NULL, ptracker);
+  if (ret < 0)
+    return;
+
+  /* only remove meta attrs */
+  for (iter = orig_attrs.begin(); iter != orig_attrs.end(); ++iter) {
+    const string& name = iter->first;
+    dout(10) << "DeleteCORS : attr: " << name << dendl;
+    if (name.compare(0, (sizeof(RGW_ATTR_CORS) - 1), RGW_ATTR_CORS) == 0) {
+      rmattrs[name] = iter->second;
+    } else if (attrs.find(name) == attrs.end()) {
+      attrs[name] = iter->second;
+    }
+  }
+  ret = store->set_attrs(s->obj_ctx, obj, attrs, &rmattrs, ptracker);
+}
+
+void RGWOptionsCORS::get_response_params(string& hdrs, string& exp_hdrs, unsigned *max_age) {
+  get_cors_response_headers(rule, req_hdrs, hdrs, exp_hdrs, max_age);
+}
+
+int RGWOptionsCORS::validate_cors_request(RGWCORSConfiguration *cc) {
+  rule = cc->host_name_rule(origin);
+  if (!rule) {
+    dout(10) << "There is no cors rule present for " << origin << dendl;
+    return -ENOENT;
+  }
+
+  if (!validate_cors_rule_method(rule, req_meth)) {
+    return -ENOENT;
+  }
+  return 0;
+}
+
+void RGWOptionsCORS::execute()
+{
+  ret = read_bucket_cors();
+  if (ret < 0)
+    return;
+
+  origin = s->info.env->get("HTTP_ORIGIN");
+  if (!origin) {
+    dout(0) <<
+    "Preflight request without mandatory Origin header"
+    << dendl;
+    ret = -EINVAL;
+    return;
+  }
+  req_meth = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_METHOD");
+  if (!req_meth) {
+    dout(0) <<
+    "Preflight request without mandatory Access-control-request-method header"
+    << dendl;
+    ret = -EINVAL;
+    return;
+  }
+  if (!cors_exist) {
+    dout(2) << "No CORS configuration set yet for this bucket" << dendl;
+    ret = -ENOENT;
+    return;
+  }
+  req_hdrs = s->info.env->get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS");
+  ret = validate_cors_request(&bucket_cors);
+  if (!rule) {
+    origin = req_meth = NULL;
+    return;
+  }
+  return;
+}
+
+int RGWInitMultipart::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWInitMultipart::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWInitMultipart::execute()
+{
+  bufferlist aclbl;
+  map attrs;
+  rgw_obj obj;
+  map::iterator iter;
+
+  if (get_params() < 0)
+    return;
+  ret = -EINVAL;
+  if (!s->object)
+    return;
+
+  policy.encode(aclbl);
+
+  attrs[RGW_ATTR_ACL] = aclbl;
+
+  for (iter = s->generic_attrs.begin(); iter != s->generic_attrs.end(); ++iter) {
+    bufferlist& attrbl = attrs[iter->first];
+    const string& val = iter->second;
+    attrbl.append(val.c_str(), val.size() + 1);
+  }
+
+  rgw_get_request_metadata(s->cct, s->info, attrs);
+
+  do {
+    char buf[33];
+    gen_rand_alphanumeric(s->cct, buf, sizeof(buf) - 1);
+    upload_id = "2/"; /* v2 upload id */
+    upload_id.append(buf);
+
+    string tmp_obj_name;
+    RGWMPObj mp(s->object_str, upload_id);
+    tmp_obj_name = mp.get_meta();
+
+    obj.init_ns(s->bucket, tmp_obj_name, mp_ns);
+    // the meta object will be indexed with 0 size, we c
+    obj.set_in_extra_data(true);
+    ret = store->put_obj_meta(s->obj_ctx, obj, 0, NULL, attrs, RGW_OBJ_CATEGORY_MULTIMETA, PUT_OBJ_CREATE_EXCL, s->owner.get_id());
+  } while (ret == -EEXIST);
+}
+
+static int get_multipart_info(RGWRados *store, struct req_state *s, string& meta_oid,
+                              RGWAccessControlPolicy *policy, map& attrs)
+{
+  map parts_map;
+  map::iterator iter;
+  bufferlist header;
+
+  rgw_obj obj;
+  obj.init_ns(s->bucket, meta_oid, mp_ns);
+  obj.set_in_extra_data(true);
+
+  int ret = get_obj_attrs(store, s, obj, attrs, NULL, NULL);
+  if (ret < 0)
+    return ret;
+
+  if (policy) {
+    for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+      string name = iter->first;
+      if (name.compare(RGW_ATTR_ACL) == 0) {
+        bufferlist& bl = iter->second;
+        bufferlist::iterator bli = bl.begin();
+        try {
+          ::decode(*policy, bli);
+        } catch (buffer::error& err) {
+          ldout(s->cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+          return -EIO;
+        }
+        break;
+      }
+    }
+  }
+
+  return 0;
+}
+
+static int list_multipart_parts(RGWRados *store, struct req_state *s,
+                                const string& upload_id,
+                                string& meta_oid, int num_parts,
+                                int marker, map& parts,
+                                int *next_marker, bool *truncated,
+                                bool assume_unsorted = false)
+{
+  map parts_map;
+  map::iterator iter;
+  bufferlist header;
+
+  rgw_obj obj;
+  obj.init_ns(s->bucket, meta_oid, mp_ns);
+  obj.set_in_extra_data(true);
+
+  bool sorted_omap = is_v2_upload_id(upload_id) && !assume_unsorted;
+
+  int ret;
+
+  parts.clear();
+  
+  if (sorted_omap) {
+    string p;
+    p = "part.";
+    char buf[32];
+
+    snprintf(buf, sizeof(buf), "%08d", marker);
+    p.append(buf);
+
+    ret = store->omap_get_vals(obj, header, p, num_parts + 1, parts_map);
+  } else {
+    ret = store->omap_get_all(obj, header, parts_map);
+  }
+  if (ret < 0)
+    return ret;
+
+  int i;
+  int last_num = 0;
+
+  uint32_t expected_next = marker + 1;
+
+  for (i = 0, iter = parts_map.begin(); (i < num_parts || !sorted_omap) && iter != parts_map.end(); ++iter, ++i) {
+    bufferlist& bl = iter->second;
+    bufferlist::iterator bli = bl.begin();
+    RGWUploadPartInfo info;
+    try {
+      ::decode(info, bli);
+    } catch (buffer::error& err) {
+      ldout(s->cct, 0) << "ERROR: could not part info, caught buffer::error" << dendl;
+      return -EIO;
+    }
+    if (sorted_omap) {
+      if (info.num != expected_next) {
+        /* ouch, we expected a specific part num here, but we got a different one. Either
+         * a part is missing, or it could be a case of mixed rgw versions working on the same
+         * upload, where one gateway doesn't support correctly sorted omap keys for multipart
+         * upload just assume data is unsorted.
+         */
+        return list_multipart_parts(store, s, upload_id, meta_oid, num_parts, marker, parts, next_marker, truncated, true);
+      }
+      expected_next++;
+    }
+    if (sorted_omap ||
+      (int)info.num > marker) {
+      parts[info.num] = info;
+      last_num = info.num;
+    }
+  }
+
+  if (sorted_omap) {
+    if (truncated)
+      *truncated = (iter != parts_map.end());
+  } else {
+    /* rebuild a map with only num_parts entries */
+
+    map new_parts;
+    map::iterator piter;
+
+    for (i = 0, piter = parts.begin(); i < num_parts && piter != parts.end(); ++i, ++piter) {
+      new_parts[piter->first] = piter->second;
+      last_num = piter->first;
+    }
+
+    if (truncated)
+      *truncated = (piter != parts.end());
+
+    parts.swap(new_parts);
+  }
+
+  if (next_marker) {
+    *next_marker = last_num;
+  }
+
+  return 0;
+}
+
+int RGWCompleteMultipart::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWCompleteMultipart::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWCompleteMultipart::execute()
+{
+  RGWMultiCompleteUpload *parts;
+  map::iterator iter;
+  RGWMultiXMLParser parser;
+  string meta_oid;
+  map obj_parts;
+  map::iterator obj_iter;
+  map attrs;
+  off_t ofs = 0;
+  MD5 hash;
+  char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16];
+  bufferlist etag_bl;
+  rgw_obj meta_obj;
+  rgw_obj target_obj;
+  RGWMPObj mp;
+  RGWObjManifest manifest;
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  if (!data) {
+    ret = -EINVAL;
+    return;
+  }
+
+  if (!parser.init()) {
+    ret = -EINVAL;
+    return;
+  }
+
+  if (!parser.parse(data, len, 1)) {
+    ret = -EINVAL;
+    return;
+  }
+
+  parts = static_cast(parser.find_first("CompleteMultipartUpload"));
+  if (!parts) {
+    ret = -EINVAL;
+    return;
+  }
+
+  mp.init(s->object_str, upload_id);
+  meta_oid = mp.get_meta();
+
+  int total_parts = 0;
+  int handled_parts = 0;
+  int max_parts = 1000;
+  int marker = 0;
+  bool truncated;
+
+  uint64_t min_part_size = s->cct->_conf->rgw_multipart_min_part_size;
+
+  list remove_objs; /* objects to be removed from index listing */
+
+  iter = parts->parts.begin();
+
+  meta_obj.init_ns(s->bucket, meta_oid, mp_ns);
+  meta_obj.set_in_extra_data(true);
+
+  ret = get_obj_attrs(store, s, meta_obj, attrs, NULL, NULL);
+  if (ret < 0) {
+    ldout(s->cct, 0) << "ERROR: failed to get obj attrs, obj=" << meta_obj << " ret=" << ret << dendl;
+    return;
+  }
+
+  do {
+    ret = list_multipart_parts(store, s, upload_id, meta_oid, max_parts, marker, obj_parts, &marker, &truncated);
+    if (ret == -ENOENT) {
+      ret = -ERR_NO_SUCH_UPLOAD;
+    }
+    if (ret < 0)
+      return;
+
+    total_parts += obj_parts.size();
+    if (!truncated && total_parts != (int)parts->parts.size()) {
+      ret = -ERR_INVALID_PART;
+      return;
+    }
+
+    for (obj_iter = obj_parts.begin(); iter != parts->parts.end() && obj_iter != obj_parts.end(); ++iter, ++obj_iter, ++handled_parts) {
+      uint64_t part_size = obj_iter->second.size;
+      if (handled_parts < (int)parts->parts.size() - 1 &&
+          part_size < min_part_size) {
+        ret = -ERR_TOO_SMALL;
+        return;
+      }
+
+      char petag[CEPH_CRYPTO_MD5_DIGESTSIZE];
+      if (iter->first != (int)obj_iter->first) {
+        ldout(s->cct, 0) << "NOTICE: parts num mismatch: next requested: " << iter->first << " next uploaded: " << obj_iter->first << dendl;
+        ret = -ERR_INVALID_PART;
+        return;
+      }
+      string part_etag = rgw_string_unquote(iter->second);
+      if (part_etag.compare(obj_iter->second.etag) != 0) {
+        ldout(s->cct, 0) << "NOTICE: etag mismatch: part: " << iter->first << " etag: " << iter->second << dendl;
+        ret = -ERR_INVALID_PART;
+        return;
+      }
+
+      hex_to_buf(obj_iter->second.etag.c_str(), petag, CEPH_CRYPTO_MD5_DIGESTSIZE);
+      hash.Update((const byte *)petag, sizeof(petag));
+
+      RGWUploadPartInfo& obj_part = obj_iter->second;
+
+      /* update manifest for part */
+      string oid = mp.get_part(obj_iter->second.num);
+      rgw_obj src_obj;
+      src_obj.init_ns(s->bucket, oid, mp_ns);
+
+      if (obj_part.manifest.empty()) {
+        ldout(s->cct, 0) << "ERROR: empty manifest for object part: obj=" << src_obj << dendl;
+        ret = -ERR_INVALID_PART;
+        return;
+      } else {
+        manifest.append(obj_part.manifest);
+      }
+
+      remove_objs.push_back(src_obj.object);
+
+      ofs += obj_part.size;
+    }
+  } while (truncated);
+  hash.Final((byte *)final_etag);
+
+  buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str);
+  snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2],  sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2,
+           "-%lld", (long long)parts->parts.size());
+  etag = final_etag_str;
+  ldout(s->cct, 10) << "calculated etag: " << final_etag_str << dendl;
+
+  etag_bl.append(final_etag_str, strlen(final_etag_str) + 1);
+
+  attrs[RGW_ATTR_ETAG] = etag_bl;
+
+  target_obj.init(s->bucket, s->object_str);
+
+  store->set_atomic(s->obj_ctx, target_obj);
+
+  RGWRados::PutObjMetaExtraParams extra_params;
+
+  extra_params.manifest = &manifest;
+  extra_params.remove_objs = &remove_objs;
+
+  extra_params.ptag = &s->req_id; /* use req_id as operation tag */
+  extra_params.owner = s->owner.get_id();
+
+  ret = store->put_obj_meta(s->obj_ctx, target_obj, ofs, attrs,
+                            RGW_OBJ_CATEGORY_MAIN, PUT_OBJ_CREATE,
+			    extra_params);
+  if (ret < 0)
+    return;
+
+  // remove the upload obj
+  store->delete_obj(s->obj_ctx, s->bucket_owner.get_id(), meta_obj);
+}
+
+int RGWAbortMultipart::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWAbortMultipart::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWAbortMultipart::execute()
+{
+  ret = -EINVAL;
+  string upload_id;
+  string meta_oid;
+  upload_id = s->info.args.get("uploadId");
+  map obj_parts;
+  map::iterator obj_iter;
+  map attrs;
+  rgw_obj meta_obj;
+  RGWMPObj mp;
+  const string& owner = s->bucket_owner.get_id();
+
+  if (upload_id.empty() || s->object_str.empty())
+    return;
+
+  mp.init(s->object_str, upload_id);
+  meta_oid = mp.get_meta();
+
+  ret = get_multipart_info(store, s, meta_oid, NULL, attrs);
+  if (ret < 0)
+    return;
+
+  bool truncated;
+  int marker = 0;
+  int max_parts = 1000;
+
+  do {
+    ret = list_multipart_parts(store, s, upload_id, meta_oid, max_parts, marker, obj_parts, &marker, &truncated);
+    if (ret < 0)
+      return;
+
+    for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {
+      RGWUploadPartInfo& obj_part = obj_iter->second;
+
+      if (obj_part.manifest.empty()) {
+        string oid = mp.get_part(obj_iter->second.num);
+        rgw_obj obj;
+        obj.init_ns(s->bucket, oid, mp_ns);
+        ret = store->delete_obj(s->obj_ctx, owner, obj);
+        if (ret < 0 && ret != -ENOENT)
+          return;
+      } else {
+        RGWObjManifest& manifest = obj_part.manifest;
+        RGWObjManifest::obj_iterator oiter;
+        for (oiter = manifest.obj_begin(); oiter != manifest.obj_end(); ++oiter) {
+          rgw_obj loc = oiter.get_location();
+          ret = store->delete_obj(s->obj_ctx, owner, loc);
+          if (ret < 0 && ret != -ENOENT)
+            return;
+        }
+      }
+    }
+  } while (truncated);
+
+  // and also remove the metadata obj
+  meta_obj.init_ns(s->bucket, meta_oid, mp_ns);
+  meta_obj.set_in_extra_data(true);
+  ret = store->delete_obj(s->obj_ctx, owner, meta_obj);
+  if (ret == -ENOENT) {
+    ret = -ERR_NO_SUCH_BUCKET;
+  }
+}
+
+int RGWListMultipart::verify_permission()
+{
+  if (!verify_object_permission(s, RGW_PERM_READ))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWListMultipart::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWListMultipart::execute()
+{
+  map xattrs;
+  string meta_oid;
+  RGWMPObj mp;
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  mp.init(s->object_str, upload_id);
+  meta_oid = mp.get_meta();
+
+  ret = get_multipart_info(store, s, meta_oid, &policy, xattrs);
+  if (ret < 0)
+    return;
+
+  ret = list_multipart_parts(store, s, upload_id, meta_oid, max_parts, marker, parts, NULL, &truncated);
+}
+
+int RGWListBucketMultiparts::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_READ))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWListBucketMultiparts::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWListBucketMultiparts::execute()
+{
+  vector objs;
+  string marker_meta;
+
+  ret = get_params();
+  if (ret < 0)
+    return;
+
+  if (s->prot_flags & RGW_REST_SWIFT) {
+    string path_args;
+    path_args = s->info.args.get("path");
+    if (!path_args.empty()) {
+      if (!delimiter.empty() || !prefix.empty()) {
+        ret = -EINVAL;
+        return;
+      }
+      prefix = path_args;
+      delimiter="/";
+    }
+  }
+  marker_meta = marker.get_meta();
+  ret = store->list_objects(s->bucket, max_uploads, prefix, delimiter, marker_meta, NULL, objs, common_prefixes,
+                               !!(s->prot_flags & RGW_REST_SWIFT), mp_ns, true, &is_truncated, &mp_filter);
+  if (!objs.empty()) {
+    vector::iterator iter;
+    RGWMultipartUploadEntry entry;
+    for (iter = objs.begin(); iter != objs.end(); ++iter) {
+      string name = iter->name;
+      if (!entry.mp.from_meta(name))
+        continue;
+      entry.obj = *iter;
+      uploads.push_back(entry);
+    }
+    next_marker = entry;
+  }
+}
+
+int RGWDeleteMultiObj::verify_permission()
+{
+  if (!verify_bucket_permission(s, RGW_PERM_WRITE))
+    return -EACCES;
+
+  return 0;
+}
+
+void RGWDeleteMultiObj::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWDeleteMultiObj::execute()
+{
+  RGWMultiDelDelete *multi_delete;
+  vector::iterator iter;
+  RGWMultiDelXMLParser parser;
+  pair result;
+  int num_processed = 0;
+
+  ret = get_params();
+  if (ret < 0) {
+    goto error;
+  }
+
+  if (!data) {
+    ret = -EINVAL;
+    goto error;
+  }
+
+  if (!parser.init()) {
+    ret = -EINVAL;
+    goto error;
+  }
+
+  if (!parser.parse(data, len, 1)) {
+    ret = -EINVAL;
+    goto error;
+  }
+
+  multi_delete = static_cast(parser.find_first("Delete"));
+  if (!multi_delete) {
+    ret = -EINVAL;
+    goto error;
+  }
+
+  if (multi_delete->is_quiet())
+    quiet = true;
+
+  begin_response();
+  if (multi_delete->objects.empty()) {
+    goto done;
+  }
+
+  for (iter = multi_delete->objects.begin();
+        iter != multi_delete->objects.end() && num_processed < max_to_delete;
+        ++iter, num_processed++) {
+
+    rgw_obj obj(bucket,(*iter));
+    store->set_atomic(s->obj_ctx, obj);
+    ret = store->delete_obj(s->obj_ctx, s->bucket_owner.get_id(), obj);
+    if (ret == -ENOENT) {
+      ret = 0;
+    }
+    result = make_pair(*iter, ret);
+
+    send_partial_response(result);
+  }
+
+  /*  set the return code to zero, errors at this point will be
+  dumped to the response */
+  ret = 0;
+
+done:
+  // will likely segfault if begin_response() has not been called
+  end_response();
+  free(data);
+  return;
+
+error:
+  send_status();
+  free(data);
+  return;
+
+}
+
+RGWHandler::~RGWHandler()
+{
+}
+
+int RGWHandler::init(RGWRados *_store, struct req_state *_s, RGWClientIO *cio)
+{
+  store = _store;
+  s = _s;
+
+  return 0;
+}
+
+int RGWHandler::do_read_permissions(RGWOp *op, bool only_bucket)
+{
+  int ret = rgw_build_policies(store, s, only_bucket, op->prefetch_data());
+
+  if (ret < 0) {
+    ldout(s->cct, 10) << "read_permissions on " << s->bucket << ":" <object_str << " only_bucket=" << only_bucket << " ret=" << ret << dendl;
+    if (ret == -ENODATA)
+      ret = -EACCES;
+  }
+
+  return ret;
+}
+
+
+RGWOp *RGWHandler::get_op(RGWRados *store)
+{
+  RGWOp *op;
+  switch (s->op) {
+   case OP_GET:
+     op = op_get();
+     break;
+   case OP_PUT:
+     op = op_put();
+     break;
+   case OP_DELETE:
+     op = op_delete();
+     break;
+   case OP_HEAD:
+     op = op_head();
+     break;
+   case OP_POST:
+     op = op_post();
+     break;
+   case OP_COPY:
+     op = op_copy();
+     break;
+   case OP_OPTIONS:
+     op = op_options();
+     break;
+   default:
+     return NULL;
+  }
+
+  if (op) {
+    op->init(store, s, this);
+  }
+  return op;
+}
+
+void RGWHandler::put_op(RGWOp *op)
+{
+  delete op;
+}
+
diff --git a/ceph/src/rgw/rgw_op.h b/ceph/src/rgw/rgw_op.h
new file mode 100644
index 00000000..b141ed5b
--- /dev/null
+++ b/ceph/src/rgw/rgw_op.h
@@ -0,0 +1,919 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/**
+ * All operations via the rados gateway are carried out by
+ * small classes known as RGWOps. This class contains a req_state
+ * and each possible command is a subclass of this with a defined
+ * execute() method that does whatever the subclass name implies.
+ * These subclasses must be further subclassed (by interface type)
+ * to provide additional virtual methods such as send_response or get_params.
+ */
+#ifndef CEPH_RGW_OP_H
+#define CEPH_RGW_OP_H
+
+#include 
+
+#include 
+#include 
+
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "rgw_user.h"
+#include "rgw_bucket.h"
+#include "rgw_acl.h"
+#include "rgw_cors.h"
+#include "rgw_quota.h"
+
+using namespace std;
+
+struct req_state;
+class RGWHandler;
+
+/**
+ * Provide the base class for all ops.
+ */
+class RGWOp {
+protected:
+  struct req_state *s;
+  RGWHandler *dialect_handler;
+  RGWRados *store;
+  RGWCORSConfiguration bucket_cors;
+  bool cors_exist;
+  RGWQuotaInfo bucket_quota;
+  RGWQuotaInfo user_quota;
+
+  virtual int init_quota();
+public:
+  RGWOp() : s(NULL), dialect_handler(NULL), store(NULL), cors_exist(false) {}
+  virtual ~RGWOp() {}
+
+  virtual int init_processing() {
+    int ret = init_quota();
+    if (ret < 0)
+      return ret;
+
+    return 0;
+  }
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *dialect_handler) {
+    this->store = store;
+    this->s = s;
+    this->dialect_handler = dialect_handler;
+  }
+  int read_bucket_cors();
+  bool generate_cors_headers(string& origin, string& method, string& headers, string& exp_headers, unsigned *max_age);
+
+  virtual int verify_params() { return 0; }
+  virtual bool prefetch_data() { return false; }
+  virtual int verify_permission() = 0;
+  virtual int verify_op_mask();
+  virtual void pre_exec() {}
+  virtual void execute() = 0;
+  virtual void send_response() {}
+  virtual void complete() {
+    send_response();
+  }
+  virtual const string name() = 0;
+
+  virtual uint32_t op_mask() { return 0; }
+};
+
+class RGWGetObj : public RGWOp {
+protected:
+  const char *range_str;
+  const char *if_mod;
+  const char *if_unmod;
+  const char *if_match;
+  const char *if_nomatch;
+  off_t ofs;
+  uint64_t total_len;
+  off_t start;
+  off_t end;
+  time_t mod_time;
+  time_t lastmod;
+  time_t unmod_time;
+  time_t *mod_ptr;
+  time_t *unmod_ptr;
+  map attrs;
+  int ret;
+  bool get_data;
+  bool partial_content;
+  rgw_obj obj;
+  utime_t gc_invalidate_time;
+
+  int init_common();
+public:
+  RGWGetObj() {
+    range_str = NULL;
+    if_mod = NULL;
+    if_unmod = NULL;
+    if_match = NULL;
+    if_nomatch = NULL;
+    start = 0;
+    ofs = 0;
+    total_len = 0;
+    end = -1;
+    mod_time = 0;
+    lastmod = 0;
+    unmod_time = 0;
+    mod_ptr = NULL;
+    unmod_ptr = NULL;
+    get_data = false;
+    partial_content = false;
+    ret = 0;
+ }
+
+  virtual bool prefetch_data() { return true; }
+
+  void set_get_data(bool get_data) {
+    this->get_data = get_data;
+  }
+  int verify_permission();
+  void pre_exec();
+  void execute();
+  int read_user_manifest_part(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs);
+  int handle_user_manifest(const char *prefix);
+
+  int get_data_cb(bufferlist& bl, off_t ofs, off_t len);
+
+  virtual int get_params() = 0;
+  virtual int send_response_data(bufferlist& bl, off_t ofs, off_t len) = 0;
+
+  virtual const string name() { return "get_obj"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+#define RGW_LIST_BUCKETS_LIMIT_MAX 10000
+
+class RGWListBuckets : public RGWOp {
+protected:
+  int ret;
+  bool sent_data;
+  string marker;
+  uint64_t limit;
+  uint64_t limit_max;
+
+public:
+  RGWListBuckets() : ret(0), sent_data(false) {
+    limit = limit_max = RGW_LIST_BUCKETS_LIMIT_MAX;
+  }
+
+  int verify_permission();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response_begin(bool has_buckets) = 0;
+  virtual void send_response_data(RGWUserBuckets& buckets) = 0;
+  virtual void send_response_end() = 0;
+  virtual void send_response() {}
+
+  virtual bool should_get_stats() { return false; }
+
+  virtual const string name() { return "list_buckets"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWStatAccount : public RGWOp {
+protected:
+  int ret;
+  uint32_t buckets_count;
+  uint64_t buckets_objcount;
+  uint64_t buckets_size;
+  uint64_t buckets_size_rounded;
+
+public:
+  RGWStatAccount() {
+    ret = 0;
+    buckets_count = 0;
+    buckets_objcount = 0;
+    buckets_size = 0;
+    buckets_size_rounded = 0;
+  }
+
+  int verify_permission();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "stat_account"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWListBucket : public RGWOp {
+protected:
+  string prefix;
+  string marker; 
+  string next_marker; 
+  string max_keys;
+  string delimiter;
+  int max;
+  int ret;
+  vector objs;
+  map common_prefixes;
+
+  int default_max;
+  bool is_truncated;
+
+  int parse_max_keys();
+
+public:
+  RGWListBucket() {
+    max = 0;
+    ret = 0;
+    default_max = 0;
+    is_truncated = false;
+  }
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "list_bucket"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWGetBucketLogging : public RGWOp {
+public:
+  RGWGetBucketLogging() {}
+  int verify_permission();
+  void execute() {}
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "get_bucket_logging"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWStatBucket : public RGWOp {
+protected:
+  int ret;
+  RGWBucketEnt bucket;
+
+public:
+  RGWStatBucket() : ret(0) {}
+  ~RGWStatBucket() {}
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "stat_bucket"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWCreateBucket : public RGWOp {
+protected:
+  int ret;
+  RGWAccessControlPolicy policy;
+  string location_constraint;
+  string placement_rule;
+  RGWBucketInfo info;
+  obj_version ep_objv;
+  bool has_cors;
+  RGWCORSConfiguration cors_config;
+
+  bufferlist in_data;
+
+public:
+  RGWCreateBucket() : ret(0), has_cors(false) {}
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    policy.set_ctx(s->cct);
+  }
+  virtual int get_params() { return 0; }
+  virtual void send_response() = 0;
+  virtual const string name() { return "create_bucket"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWDeleteBucket : public RGWOp {
+protected:
+  int ret;
+
+  RGWObjVersionTracker objv_tracker;
+
+public:
+  RGWDeleteBucket() : ret(0) {}
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "delete_bucket"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; }
+};
+
+class RGWPutObj : public RGWOp {
+
+  friend class RGWPutObjProcessor;
+
+protected:
+  int ret;
+  off_t ofs;
+  const char *supplied_md5_b64;
+  const char *supplied_etag;
+  string etag;
+  bool chunked_upload;
+  RGWAccessControlPolicy policy;
+  const char *obj_manifest;
+  time_t mtime;
+
+  MD5 *user_manifest_parts_hash;
+
+public:
+  RGWPutObj() {
+    ret = 0;
+    ofs = 0;
+    supplied_md5_b64 = NULL;
+    supplied_etag = NULL;
+    chunked_upload = false;
+    obj_manifest = NULL;
+    mtime = 0;
+    user_manifest_parts_hash = NULL;
+  }
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    policy.set_ctx(s->cct);
+  }
+
+  RGWPutObjProcessor *select_processor(bool *is_multipart);
+  void dispose_processor(RGWPutObjProcessor *processor);
+
+  int user_manifest_iterate_cb(rgw_bucket& bucket, RGWObjEnt& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs);
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual int get_data(bufferlist& bl) = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "put_obj"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWPostObj : public RGWOp {
+
+  friend class RGWPutObjProcessor;
+
+protected:
+  off_t min_len;
+  off_t max_len;
+  int ret;
+  int len;
+  off_t ofs;
+  const char *supplied_md5_b64;
+  const char *supplied_etag;
+  string etag;
+  string boundary;
+  bool data_pending;
+  string content_type;
+  RGWAccessControlPolicy policy;
+  map attrs;
+
+public:
+  RGWPostObj() : min_len(0), max_len(LLONG_MAX), ret(0), len(0), ofs(0),
+		 supplied_md5_b64(NULL), supplied_etag(NULL),
+		 data_pending(false) {}
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    policy.set_ctx(s->cct);
+  }
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  RGWPutObjProcessor *select_processor();
+  void dispose_processor(RGWPutObjProcessor *processor);
+
+  virtual int get_params() = 0;
+  virtual int get_data(bufferlist& bl) = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "post_obj"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWPutMetadata : public RGWOp {
+protected:
+  int ret;
+  map attrs;
+  bool has_policy, has_cors;
+  RGWAccessControlPolicy policy;
+  RGWCORSConfiguration cors_config;
+
+public:
+  RGWPutMetadata() {
+    has_cors = false;
+    has_policy = false;
+    ret = 0;
+  }
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    policy.set_ctx(s->cct);
+  }
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "put_obj_metadata"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWSetTempUrl : public RGWOp {
+protected:
+  int ret;
+  map temp_url_keys;
+public:
+  RGWSetTempUrl() : ret(0) {}
+
+  int verify_permission();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "set_temp_url"; }
+};
+
+class RGWDeleteObj : public RGWOp {
+protected:
+  int ret;
+
+public:
+  RGWDeleteObj() : ret(0) {}
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "delete_obj"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; }
+};
+
+class RGWCopyObj : public RGWOp {
+protected:
+  RGWAccessControlPolicy dest_policy;
+  const char *if_mod;
+  const char *if_unmod;
+  const char *if_match;
+  const char *if_nomatch;
+  off_t ofs;
+  off_t len;
+  off_t end;
+  time_t mod_time;
+  time_t unmod_time;
+  time_t *mod_ptr;
+  time_t *unmod_ptr;
+  int ret;
+  map attrs;
+  string src_bucket_name;
+  rgw_bucket src_bucket;
+  string src_object;
+  string dest_bucket_name;
+  rgw_bucket dest_bucket;
+  string dest_object;
+  time_t mtime;
+  bool replace_attrs;
+  RGWBucketInfo src_bucket_info;
+  RGWBucketInfo dest_bucket_info;
+  string source_zone;
+  string client_id;
+  string op_id;
+
+  off_t last_ofs;
+
+
+  int init_common();
+
+public:
+  RGWCopyObj() {
+    if_mod = NULL;
+    if_unmod = NULL;
+    if_match = NULL;
+    if_nomatch = NULL;
+    ofs = 0;
+    len = 0;
+    end = -1;
+    mod_time = 0;
+    unmod_time = 0;
+    mod_ptr = NULL;
+    unmod_ptr = NULL;
+    ret = 0;
+    mtime = 0;
+    replace_attrs = false;
+    last_ofs = 0;
+  }
+
+  static bool parse_copy_location(const char *src, string& bucket_name, string& object);
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    dest_policy.set_ctx(s->cct);
+  }
+  int verify_permission();
+  void pre_exec();
+  void execute();
+  void progress_cb(off_t ofs);
+
+  virtual int init_dest_policy() { return 0; }
+  virtual int get_params() = 0;
+  virtual void send_partial_response(off_t ofs) {}
+  virtual void send_response() = 0;
+  virtual const string name() { return "copy_obj"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWGetACLs : public RGWOp {
+protected:
+  int ret;
+  string acls;
+
+public:
+  RGWGetACLs() : ret(0) {}
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "get_acls"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWPutACLs : public RGWOp {
+protected:
+  int ret;
+  size_t len;
+  char *data;
+
+public:
+  RGWPutACLs() {
+    ret = 0;
+    len = 0;
+    data = NULL;
+  }
+  virtual ~RGWPutACLs() {
+    free(data);
+  }
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_policy_from_state(RGWRados *store, struct req_state *s, stringstream& ss) { return 0; }
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "put_acls"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWGetCORS : public RGWOp {
+protected:
+  int ret;
+
+public:
+  RGWGetCORS() : ret(0) {}
+
+  int verify_permission();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "get_cors"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWPutCORS : public RGWOp {
+protected:
+  int ret;
+  bufferlist cors_bl;
+
+public:
+  RGWPutCORS() {
+    ret = 0;
+  }
+  virtual ~RGWPutCORS() { }
+
+  int verify_permission();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "put_cors"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWDeleteCORS : public RGWOp {
+protected:
+  int ret;
+
+public:
+  RGWDeleteCORS() : ret(0) {}
+
+  int verify_permission();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "delete_cors"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWOptionsCORS : public RGWOp {
+protected:
+  int ret;
+  RGWCORSRule *rule;
+  const char *origin, *req_hdrs, *req_meth;
+
+public:
+  RGWOptionsCORS() : ret(0), rule(NULL), origin(NULL),
+                     req_hdrs(NULL), req_meth(NULL) {
+  }
+
+  int verify_permission() {return 0;}
+  int validate_cors_request(RGWCORSConfiguration *cc);
+  void execute();
+  void get_response_params(string& allowed_hdrs, string& exp_hdrs, unsigned *max_age);
+  virtual void send_response() = 0;
+  virtual const string name() { return "options_cors"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWInitMultipart : public RGWOp {
+protected:
+  int ret;
+  string upload_id;
+  RGWAccessControlPolicy policy;
+
+public:
+  RGWInitMultipart() {
+    ret = 0;
+  }
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    policy.set_ctx(s->cct);
+  }
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "init_multipart"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWCompleteMultipart : public RGWOp {
+protected:
+  int ret;
+  string upload_id;
+  string etag;
+  char *data;
+  int len;
+
+public:
+  RGWCompleteMultipart() {
+    ret = 0;
+    data = NULL;
+    len = 0;
+  }
+  virtual ~RGWCompleteMultipart() {
+    free(data);
+  }
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "complete_multipart"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
+class RGWAbortMultipart : public RGWOp {
+protected:
+  int ret;
+
+public:
+  RGWAbortMultipart() : ret(0) {}
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual void send_response() = 0;
+  virtual const string name() { return "abort_multipart"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; }
+};
+
+class RGWListMultipart : public RGWOp {
+protected:
+  int ret;
+  string upload_id;
+  map parts;
+  int max_parts;
+  int marker;
+  RGWAccessControlPolicy policy;
+  bool truncated;
+
+public:
+  RGWListMultipart() {
+    ret = 0;
+    max_parts = 1000;
+    marker = 0;
+    truncated = false;
+  }
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    policy = RGWAccessControlPolicy(s->cct);
+  }
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "list_multipart"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+#define MP_META_SUFFIX ".meta"
+
+class RGWMPObj {
+  string oid;
+  string prefix;
+  string meta;
+  string upload_id;
+public:
+  RGWMPObj() {}
+  RGWMPObj(const string& _oid, const string& _upload_id) {
+    init(_oid, _upload_id, _upload_id);
+  }
+  void init(const string& _oid, const string& _upload_id) {
+    init(_oid, _upload_id, _upload_id);
+  }
+  void init(const string& _oid, const string& _upload_id, const string& part_unique_str) {
+    if (_oid.empty()) {
+      clear();
+      return;
+    }
+    oid = _oid;
+    upload_id = _upload_id;
+    prefix = oid + ".";
+    meta = prefix + upload_id + MP_META_SUFFIX;
+    prefix.append(part_unique_str);
+  }
+  string& get_meta() { return meta; }
+  string get_part(int num) {
+    char buf[16];
+    snprintf(buf, 16, ".%d", num);
+    string s = prefix;
+    s.append(buf);
+    return s;
+  }
+  string get_part(string& part) {
+    string s = prefix;
+    s.append(".");
+    s.append(part);
+    return s;
+  }
+  string& get_upload_id() {
+    return upload_id;
+  }
+  string& get_key() {
+    return oid;
+  }
+  bool from_meta(string& meta) {
+    int end_pos = meta.rfind('.'); // search for ".meta"
+    if (end_pos < 0)
+      return false;
+    int mid_pos = meta.rfind('.', end_pos - 1); // .
+    if (mid_pos < 0)
+      return false;
+    oid = meta.substr(0, mid_pos);
+    upload_id = meta.substr(mid_pos + 1, end_pos - mid_pos - 1);
+    init(oid, upload_id, upload_id);
+    return true;
+  }
+  void clear() {
+    oid = "";
+    prefix = "";
+    meta = "";
+    upload_id = "";
+  }
+};
+
+struct RGWMultipartUploadEntry {
+  RGWObjEnt obj;
+  RGWMPObj mp;
+};
+
+class RGWListBucketMultiparts : public RGWOp {
+protected:
+  string prefix;
+  RGWMPObj marker; 
+  RGWMultipartUploadEntry next_marker; 
+  int max_uploads;
+  string delimiter;
+  int ret;
+  vector uploads;
+  map common_prefixes;
+  bool is_truncated;
+  int default_max;
+
+public:
+  RGWListBucketMultiparts() {
+    max_uploads = 0;
+    ret = 0;
+    is_truncated = false;
+    default_max = 0;
+  }
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWOp::init(store, s, h);
+    max_uploads = default_max;
+  }
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "list_bucket_multiparts"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
+};
+
+class RGWDeleteMultiObj : public RGWOp {
+protected:
+  int ret;
+  int max_to_delete;
+  size_t len;
+  char *data;
+  string bucket_name;
+  rgw_bucket bucket;
+  bool quiet;
+  bool status_dumped;
+
+
+public:
+  RGWDeleteMultiObj() {
+    ret = 0;
+    max_to_delete = 1000;
+    len = 0;
+    data = NULL;
+    quiet = false;
+    status_dumped = false;
+  }
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_status() = 0;
+  virtual void begin_response() = 0;
+  virtual void send_partial_response(pair& result) = 0;
+  virtual void end_response() = 0;
+  virtual const string name() { return "multi_object_delete"; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; }
+};
+
+
+class RGWHandler {
+protected:
+  RGWRados *store;
+  struct req_state *s;
+
+  int do_read_permissions(RGWOp *op, bool only_bucket);
+
+  virtual RGWOp *op_get() { return NULL; }
+  virtual RGWOp *op_put() { return NULL; }
+  virtual RGWOp *op_delete() { return NULL; }
+  virtual RGWOp *op_head() { return NULL; }
+  virtual RGWOp *op_post() { return NULL; }
+  virtual RGWOp *op_copy() { return NULL; }
+  virtual RGWOp *op_options() { return NULL; }
+public:
+  RGWHandler() : store(NULL), s(NULL) {}
+  virtual ~RGWHandler();
+  virtual int init(RGWRados *store, struct req_state *_s, RGWClientIO *cio);
+
+  virtual RGWOp *get_op(RGWRados *store);
+  virtual void put_op(RGWOp *op);
+  virtual int read_permissions(RGWOp *op) = 0;
+  virtual int authorize() = 0;
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_policy_s3.cc b/ceph/src/rgw/rgw_policy_s3.cc
new file mode 100644
index 00000000..fac05f18
--- /dev/null
+++ b/ceph/src/rgw/rgw_policy_s3.cc
@@ -0,0 +1,295 @@
+
+#include 
+
+#include "common/ceph_json.h"
+#include "rgw_policy_s3.h"
+#include "rgw_common.h"
+
+
+#define dout_subsys ceph_subsys_rgw
+
+class RGWPolicyCondition {
+protected:
+  string v1;
+  string v2;
+
+  virtual bool check(const string& first, const string& second, string& err_msg) = 0;
+
+public:
+  virtual ~RGWPolicyCondition() {}
+
+  void set_vals(const string& _v1, const string& _v2) {
+    v1 = _v1;
+    v2 = _v2;
+  }
+
+  bool check(RGWPolicyEnv *env, map& checked_vars, string& err_msg) {
+     string first, second;
+     env->get_value(v1, first, checked_vars);
+     env->get_value(v2, second, checked_vars);
+
+     dout(1) << "policy condition check " << v1 << " [" << first << "] " << v2 << " [" << second << "]" << dendl;
+     bool ret = check(first, second, err_msg);
+     if (!ret) {
+       err_msg.append(": ");
+       err_msg.append(v1);
+       err_msg.append(", ");
+       err_msg.append(v2);
+     }
+     return ret;
+  }
+
+};
+
+
+class RGWPolicyCondition_StrEqual : public RGWPolicyCondition {
+protected:
+  bool check(const string& first, const string& second, string& msg) {
+    bool ret = first.compare(second) == 0;
+    if (!ret) {
+      msg = "Policy condition failed: eq";
+    }
+    return ret;
+  }
+};
+
+class RGWPolicyCondition_StrStartsWith : public RGWPolicyCondition {
+protected:
+  bool check(const string& first, const string& second, string& msg) {
+    bool ret = first.compare(0, second.size(), second) == 0;
+    if (!ret) {
+      msg = "Policy condition failed: starts-with";
+    }
+    return ret;
+  }
+};
+
+void RGWPolicyEnv::add_var(const string& name, const string& value)
+{
+  vars[name] = value;
+}
+
+bool RGWPolicyEnv::get_var(const string& name, string& val)
+{
+  map::iterator iter = vars.find(name);
+  if (iter == vars.end())
+    return false;
+
+  val = iter->second;
+
+  return true;
+}
+
+bool RGWPolicyEnv::get_value(const string& s, string& val, map& checked_vars)
+{
+  if (s.empty() || s[0] != '$') {
+    val = s;
+    return true;
+  }
+
+  const string& var = s.substr(1);
+  checked_vars[var] = true;
+
+  return get_var(var, val);
+}
+
+
+bool RGWPolicyEnv::match_policy_vars(map& policy_vars, string& err_msg)
+{
+  map::iterator iter;
+  string ignore_prefix = "x-ignore-";
+  for (iter = vars.begin(); iter != vars.end(); ++iter) {
+    const string& var = iter->first;
+    if (strncasecmp(ignore_prefix.c_str(), var.c_str(), ignore_prefix.size()) == 0)
+      continue;
+    if (policy_vars.count(var) == 0) {
+      err_msg = "Policy missing condition: ";
+      err_msg.append(iter->first);
+      dout(1) << "env var missing in policy: " << iter->first << dendl;
+      return false;
+    }
+  }
+  return true;
+}
+
+RGWPolicy::~RGWPolicy()
+{
+  list::iterator citer;
+  for (citer = conditions.begin(); citer != conditions.end(); ++citer) {
+    RGWPolicyCondition *cond = *citer;
+    delete cond;
+  }
+}
+
+int RGWPolicy::set_expires(const string& e)
+{
+  struct tm t;
+  if (!parse_iso8601(e.c_str(), &t))
+      return -EINVAL;
+
+  expires = timegm(&t);
+
+  return 0;
+}
+
+int RGWPolicy::add_condition(const string& op, const string& first, const string& second, string& err_msg)
+{
+  RGWPolicyCondition *cond = NULL;
+  if (stringcasecmp(op, "eq") == 0) {
+    cond = new RGWPolicyCondition_StrEqual;
+  } else if (stringcasecmp(op, "starts-with") == 0) {
+    cond = new RGWPolicyCondition_StrStartsWith;
+  } else if (stringcasecmp(op, "content-length-range") == 0) {
+    off_t min, max;
+    int r = stringtoll(first, &min);
+    if (r < 0) {
+      err_msg = "Bad content-length-range param";
+      dout(0) << "bad content-length-range param: " << first << dendl;
+      return r;
+    }
+
+    r = stringtoll(second, &max);
+    if (r < 0) {
+      err_msg = "Bad content-length-range param";
+      dout(0) << "bad content-length-range param: " << second << dendl;
+      return r;
+    }
+
+    if (min > min_length)
+      min_length = min;
+
+    if (max < max_length)
+      max_length = max;
+
+    return 0;
+  }
+
+  if (!cond) {
+    err_msg = "Invalid condition: ";
+    err_msg.append(op);
+    dout(0) << "invalid condition: " << op << dendl;
+    return -EINVAL;
+  }
+
+  cond->set_vals(first, second);
+  
+  conditions.push_back(cond);
+
+  return 0;
+}
+
+int RGWPolicy::check(RGWPolicyEnv *env, string& err_msg)
+{
+  uint64_t now = ceph_clock_now(NULL).sec();
+  if (expires <= now) {
+    dout(0) << "NOTICE: policy calculated as expired: " << expiration_str << dendl;
+    err_msg = "Policy expired";
+    return -EACCES; // change to condition about expired policy following S3
+  }
+
+  list >::iterator viter;
+  for (viter = var_checks.begin(); viter != var_checks.end(); ++viter) {
+    pair& p = *viter;
+    const string& name = p.first;
+    const string& check_val = p.second;
+    string val;
+    if (!env->get_var(name, val)) {
+      dout(20) << " policy check failed, variable not found: '" << name << "'" << dendl;
+      err_msg = "Policy check failed, variable not found: ";
+      err_msg.append(name);
+      return -EACCES;
+    }
+
+    set_var_checked(name);
+
+    dout(20) << "comparing " << name << " [" << val << "], " << check_val << dendl;
+    if (val.compare(check_val) != 0) {
+      err_msg = "Policy check failed, variable not met condition: ";
+      err_msg.append(name);
+      dout(1) << "policy check failed, val=" << val << " != " << check_val << dendl;
+      return -EACCES;
+    }
+  }
+
+  list::iterator citer;
+  for (citer = conditions.begin(); citer != conditions.end(); ++citer) {
+    RGWPolicyCondition *cond = *citer;
+    if (!cond->check(env, checked_vars, err_msg)) {
+      return -EACCES;
+    }
+  }
+
+  if (!env->match_policy_vars(checked_vars, err_msg)) {
+    dout(1) << "missing policy condition" << dendl;
+    return -EACCES;
+  }
+  return 0;
+}
+
+
+int RGWPolicy::from_json(bufferlist& bl, string& err_msg)
+{
+  JSONParser parser;
+
+  if (!parser.parse(bl.c_str(), bl.length())) {
+    err_msg = "Malformed JSON";
+    dout(0) << "malformed json" << dendl;
+    return -EINVAL;
+  }
+
+  // as no time was included in the request, we hope that the user has included a short timeout
+  JSONObjIter iter = parser.find_first("expiration");
+  if (iter.end()) {
+    err_msg = "Policy missing expiration";
+    dout(0) << "expiration not found" << dendl;
+    return -EINVAL; // change to a "no expiration" error following S3
+  }
+
+  JSONObj *obj = *iter;
+  expiration_str = obj->get_data();
+  int r = set_expires(expiration_str);
+  if (r < 0) {
+    err_msg = "Failed to parse policy expiration";
+    return r;
+  }
+
+  iter = parser.find_first("conditions");
+  if (iter.end()) {
+    err_msg = "Policy missing conditions";
+    dout(0) << "conditions not found" << dendl;
+    return -EINVAL; // change to a "no conditions" error following S3
+  }
+
+  obj = *iter;
+
+  iter = obj->find_first();
+  for (; !iter.end(); ++iter) {
+    JSONObj *child = *iter;
+    dout(20) << "data=" << child->get_data() << dendl;
+    dout(20) << "is_object=" << child->is_object() << dendl;
+    dout(20) << "is_array=" << child->is_array() << dendl;
+    JSONObjIter citer = child->find_first();
+    if (child->is_array()) {
+      vector v;
+      int i;
+      for (i = 0; !citer.end() && i < 3; ++citer, ++i) {
+	JSONObj *o = *citer;
+        v.push_back(o->get_data());
+      }
+      if (i != 3 || !citer.end()) { /* we expect exactly 3 arguments here */
+        err_msg = "Bad condition array, expecting 3 arguments";
+        return -EINVAL;
+      }
+
+      int r = add_condition(v[0], v[1], v[2], err_msg);
+      if (r < 0)
+        return r;
+    } else {
+      JSONObj *c = *citer;
+      dout(0) << "adding simple_check: " << c->get_name() << " : " << c->get_data() << dendl;
+
+      add_simple_check(c->get_name(), c->get_data());
+    }
+  }
+  return 0;
+}
diff --git a/ceph/src/rgw/rgw_policy_s3.h b/ceph/src/rgw/rgw_policy_s3.h
new file mode 100644
index 00000000..84a2ee71
--- /dev/null
+++ b/ceph/src/rgw/rgw_policy_s3.h
@@ -0,0 +1,56 @@
+#ifndef CEPH_RGW_POLICY_H
+#define CEPH_RGW_POLICY_H
+
+#include 
+
+#include 
+#include 
+#include 
+
+#include "include/utime.h"
+
+#include "rgw_string.h"
+
+
+class RGWPolicyEnv {
+  std::map vars;
+
+public:
+  void add_var(const string& name, const string& value);
+  bool get_var(const string& name, string& val);
+  bool get_value(const string& s, string& val, std::map& checked_vars);
+  bool match_policy_vars(map& policy_vars, string& err_msg);
+};
+
+class RGWPolicyCondition;
+
+
+class RGWPolicy {
+  uint64_t expires;
+  string expiration_str;
+  std::list conditions;
+  std::list > var_checks;
+  std::map checked_vars;
+
+public:
+  off_t min_length;
+  off_t max_length;
+
+  RGWPolicy() : expires(0), min_length(0), max_length(LLONG_MAX) {}
+  ~RGWPolicy();
+
+  int set_expires(const string& e);
+
+  void set_var_checked(const std::string& var) {
+    checked_vars[var] = true;
+  }
+
+  int add_condition(const std::string& op, const std::string& first, const std::string& second, string& err_msg);
+  void add_simple_check(const std::string& var, const std::string& value) {
+    var_checks.push_back(pair(var, value));
+  }
+
+  int check(RGWPolicyEnv *env, string& err_msg);
+  int from_json(bufferlist& bl, string& err_msg);
+};
+#endif
diff --git a/ceph/src/rgw/rgw_quota.cc b/ceph/src/rgw/rgw_quota.cc
new file mode 100644
index 00000000..eab7e37a
--- /dev/null
+++ b/ceph/src/rgw/rgw_quota.cc
@@ -0,0 +1,751 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+
+#include "include/utime.h"
+#include "common/lru_map.h"
+#include "common/RefCountedObj.h"
+#include "common/Thread.h"
+#include "common/Mutex.h"
+#include "common/RWLock.h"
+
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "rgw_quota.h"
+#include "rgw_bucket.h"
+#include "rgw_user.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+
+struct RGWQuotaCacheStats {
+  RGWStorageStats stats;
+  utime_t expiration;
+  utime_t async_refresh_time;
+};
+
+template
+class RGWQuotaCache {
+protected:
+  RGWRados *store;
+  lru_map stats_map;
+  RefCountedWaitObject *async_refcount;
+
+  class StatsAsyncTestSet : public lru_map::UpdateContext {
+    int objs_delta;
+    uint64_t added_bytes;
+    uint64_t removed_bytes;
+  public:
+    StatsAsyncTestSet() {}
+    bool update(RGWQuotaCacheStats *entry) {
+      if (entry->async_refresh_time.sec() == 0)
+        return false;
+
+      entry->async_refresh_time = utime_t(0, 0);
+
+      return true;
+    }
+  };
+
+  virtual int fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats) = 0;
+
+  virtual bool map_find(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) = 0;
+
+  virtual bool map_find_and_update(const string& user, rgw_bucket& bucket, typename lru_map::UpdateContext *ctx) = 0;
+  virtual void map_add(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) = 0;
+
+  virtual void data_modified(const string& user, rgw_bucket& bucket) {}
+public:
+  RGWQuotaCache(RGWRados *_store, int size) : store(_store), stats_map(size) {
+    async_refcount = new RefCountedWaitObject;
+  }
+  virtual ~RGWQuotaCache() {
+    async_refcount->put_wait(); /* wait for all pending async requests to complete */
+  }
+
+  int get_stats(const string& user, rgw_bucket& bucket, RGWStorageStats& stats, RGWQuotaInfo& quota);
+  void adjust_stats(const string& user, rgw_bucket& bucket, int objs_delta, uint64_t added_bytes, uint64_t removed_bytes);
+
+  virtual bool can_use_cached_stats(RGWQuotaInfo& quota, RGWStorageStats& stats);
+
+  void set_stats(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs, RGWStorageStats& stats);
+  int async_refresh(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs);
+  void async_refresh_response(const string& user, rgw_bucket& bucket, RGWStorageStats& stats);
+
+  class AsyncRefreshHandler {
+  protected:
+    RGWRados *store;
+    RGWQuotaCache *cache;
+  public:
+    AsyncRefreshHandler(RGWRados *_store, RGWQuotaCache *_cache) : store(_store), cache(_cache) {}
+    virtual ~AsyncRefreshHandler() {}
+
+    virtual int init_fetch() = 0;
+    virtual void drop_reference() = 0;
+  };
+
+  virtual AsyncRefreshHandler *allocate_refresh_handler(const string& user, rgw_bucket& bucket) = 0;
+};
+
+template
+bool RGWQuotaCache::can_use_cached_stats(RGWQuotaInfo& quota, RGWStorageStats& cached_stats)
+{
+  if (quota.max_size_kb >= 0) {
+    if (quota.max_size_soft_threshold < 0) {
+      quota.max_size_soft_threshold = quota.max_size_kb * store->ctx()->_conf->rgw_bucket_quota_soft_threshold;
+    }
+
+    if (cached_stats.num_kb_rounded >= (uint64_t)quota.max_size_soft_threshold) {
+      ldout(store->ctx(), 20) << "quota: can't use cached stats, exceeded soft threshold (size): "
+        << cached_stats.num_kb_rounded << " >= " << quota.max_size_soft_threshold << dendl;
+      return false;
+    }
+  }
+
+  if (quota.max_objects >= 0) {
+    if (quota.max_objs_soft_threshold < 0) {
+      quota.max_objs_soft_threshold = quota.max_objects * store->ctx()->_conf->rgw_bucket_quota_soft_threshold;
+    }
+
+    if (cached_stats.num_objects >= (uint64_t)quota.max_objs_soft_threshold) {
+      ldout(store->ctx(), 20) << "quota: can't use cached stats, exceeded soft threshold (num objs): "
+        << cached_stats.num_objects << " >= " << quota.max_objs_soft_threshold << dendl;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+template
+int RGWQuotaCache::async_refresh(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs)
+{
+  /* protect against multiple updates */
+  StatsAsyncTestSet test_update;
+  if (!map_find_and_update(user, bucket, &test_update)) {
+    /* most likely we just raced with another update */
+    return 0;
+  }
+
+  async_refcount->get();
+
+
+  AsyncRefreshHandler *handler = allocate_refresh_handler(user, bucket);
+
+  int ret = handler->init_fetch();
+  if (ret < 0) {
+    async_refcount->put();
+    handler->drop_reference();
+    return ret;
+  }
+
+  return 0;
+}
+
+template
+void RGWQuotaCache::async_refresh_response(const string& user, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+  ldout(store->ctx(), 20) << "async stats refresh response for bucket=" << bucket << dendl;
+
+  RGWQuotaCacheStats qs;
+
+  map_find(user, bucket, qs);
+
+  set_stats(user, bucket, qs, stats);
+
+  async_refcount->put();
+}
+
+template
+void RGWQuotaCache::set_stats(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs, RGWStorageStats& stats)
+{
+  qs.stats = stats;
+  qs.expiration = ceph_clock_now(store->ctx());
+  qs.async_refresh_time = qs.expiration;
+  qs.expiration += store->ctx()->_conf->rgw_bucket_quota_ttl;
+  qs.async_refresh_time += store->ctx()->_conf->rgw_bucket_quota_ttl / 2;
+
+  map_add(user, bucket, qs);
+}
+
+template
+int RGWQuotaCache::get_stats(const string& user, rgw_bucket& bucket, RGWStorageStats& stats, RGWQuotaInfo& quota) {
+  RGWQuotaCacheStats qs;
+  utime_t now = ceph_clock_now(store->ctx());
+  if (map_find(user, bucket, qs)) {
+    if (qs.async_refresh_time.sec() > 0 && now >= qs.async_refresh_time) {
+      int r = async_refresh(user, bucket, qs);
+      if (r < 0) {
+        ldout(store->ctx(), 0) << "ERROR: quota async refresh returned ret=" << r << dendl;
+
+        /* continue processing, might be a transient error, async refresh is just optimization */
+      }
+    }
+
+    if (can_use_cached_stats(quota, qs.stats) && qs.expiration > ceph_clock_now(store->ctx())) {
+      stats = qs.stats;
+      return 0;
+    }
+  }
+
+  int ret = fetch_stats_from_storage(user, bucket, stats);
+  if (ret < 0 && ret != -ENOENT)
+    return ret;
+
+  set_stats(user, bucket, qs, stats);
+
+  return 0;
+}
+
+
+template
+class RGWQuotaStatsUpdate : public lru_map::UpdateContext {
+  int objs_delta;
+  uint64_t added_bytes;
+  uint64_t removed_bytes;
+public:
+  RGWQuotaStatsUpdate(int _objs_delta, uint64_t _added_bytes, uint64_t _removed_bytes) : 
+                    objs_delta(_objs_delta), added_bytes(_added_bytes), removed_bytes(_removed_bytes) {}
+  bool update(RGWQuotaCacheStats *entry) {
+    uint64_t rounded_kb_added = rgw_rounded_objsize_kb(added_bytes);
+    uint64_t rounded_kb_removed = rgw_rounded_objsize_kb(removed_bytes);
+
+    entry->stats.num_kb_rounded += (rounded_kb_added - rounded_kb_removed);
+    entry->stats.num_kb += (added_bytes - removed_bytes) / 1024;
+    entry->stats.num_objects += objs_delta;
+
+    return true;
+  }
+};
+
+
+template
+void RGWQuotaCache::adjust_stats(const string& user, rgw_bucket& bucket, int objs_delta,
+                                 uint64_t added_bytes, uint64_t removed_bytes)
+{
+  RGWQuotaStatsUpdate update(objs_delta, added_bytes, removed_bytes);
+  map_find_and_update(user, bucket, &update);
+
+  data_modified(user, bucket);
+}
+
+class BucketAsyncRefreshHandler : public RGWQuotaCache::AsyncRefreshHandler,
+                                  public RGWGetBucketStats_CB {
+  string user;
+public:
+  BucketAsyncRefreshHandler(RGWRados *_store, RGWQuotaCache *_cache,
+                            const string& _user, rgw_bucket& _bucket) :
+                                      RGWQuotaCache::AsyncRefreshHandler(_store, _cache),
+                                      RGWGetBucketStats_CB(_bucket), user(_user) {}
+
+  void drop_reference() { put(); }
+  void handle_response(int r);
+  int init_fetch();
+};
+
+int BucketAsyncRefreshHandler::init_fetch()
+{
+  ldout(store->ctx(), 20) << "initiating async quota refresh for bucket=" << bucket << dendl;
+
+  int r = store->get_bucket_stats_async(bucket, this);
+  if (r < 0) {
+    ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
+
+    /* get_bucket_stats_async() dropped our reference already */
+    return r;
+  }
+
+  return 0;
+}
+
+void BucketAsyncRefreshHandler::handle_response(int r)
+{
+  if (r < 0) {
+    ldout(store->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
+    return; /* nothing to do here */
+  }
+
+  RGWStorageStats bs;
+
+  map::iterator iter;
+  for (iter = stats->begin(); iter != stats->end(); ++iter) {
+    RGWStorageStats& s = iter->second;
+    bs.num_kb += s.num_kb;
+    bs.num_kb_rounded += s.num_kb_rounded;
+    bs.num_objects += s.num_objects;
+  }
+
+  cache->async_refresh_response(user, bucket, bs);
+}
+
+class RGWBucketStatsCache : public RGWQuotaCache {
+protected:
+  bool map_find(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+    return stats_map.find(bucket, qs);
+  }
+
+  bool map_find_and_update(const string& user, rgw_bucket& bucket, lru_map::UpdateContext *ctx) {
+    return stats_map.find_and_update(bucket, NULL, ctx);
+  }
+
+  void map_add(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+    stats_map.add(bucket, qs);
+  }
+
+  int fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats);
+
+public:
+  RGWBucketStatsCache(RGWRados *_store) : RGWQuotaCache(_store, _store->ctx()->_conf->rgw_bucket_quota_cache_size) {
+  }
+
+  AsyncRefreshHandler *allocate_refresh_handler(const string& user, rgw_bucket& bucket) {
+    return new BucketAsyncRefreshHandler(store, this, user, bucket);
+  }
+};
+
+int RGWBucketStatsCache::fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+  RGWBucketInfo bucket_info;
+
+  uint64_t bucket_ver;
+  uint64_t master_ver;
+
+  map bucket_stats;
+  int r = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, bucket_stats, NULL);
+  if (r < 0) {
+    ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
+    return r;
+  }
+
+  stats = RGWStorageStats();
+
+  map::iterator iter;
+  for (iter = bucket_stats.begin(); iter != bucket_stats.end(); ++iter) {
+    RGWStorageStats& s = iter->second;
+    stats.num_kb += s.num_kb;
+    stats.num_kb_rounded += s.num_kb_rounded;
+    stats.num_objects += s.num_objects;
+  }
+
+  return 0;
+}
+
+class UserAsyncRefreshHandler : public RGWQuotaCache::AsyncRefreshHandler,
+                                public RGWGetUserStats_CB {
+  rgw_bucket bucket;
+public:
+  UserAsyncRefreshHandler(RGWRados *_store, RGWQuotaCache *_cache,
+                          const string& _user, rgw_bucket& _bucket) :
+                          RGWQuotaCache::AsyncRefreshHandler(_store, _cache),
+                          RGWGetUserStats_CB(_user),
+                          bucket(_bucket) {}
+
+  void drop_reference() { put(); }
+  int init_fetch();
+  void handle_response(int r);
+};
+
+int UserAsyncRefreshHandler::init_fetch()
+{
+  ldout(store->ctx(), 20) << "initiating async quota refresh for user=" << user << dendl;
+  int r = store->get_user_stats_async(user, this);
+  if (r < 0) {
+    ldout(store->ctx(), 0) << "could not get bucket info for user=" << user << dendl;
+
+    /* get_bucket_stats_async() dropped our reference already */
+    return r;
+  }
+
+  return 0;
+}
+
+void UserAsyncRefreshHandler::handle_response(int r)
+{
+  if (r < 0) {
+    ldout(store->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
+    return; /* nothing to do here */
+  }
+
+  cache->async_refresh_response(user, bucket, stats);
+}
+
+class RGWUserStatsCache : public RGWQuotaCache {
+  atomic_t down_flag;
+  RWLock rwlock;
+  map modified_buckets;
+
+  /* thread, sync recent modified buckets info */
+  class BucketsSyncThread : public Thread {
+    CephContext *cct;
+    RGWUserStatsCache *stats;
+
+    Mutex lock;
+    Cond cond;
+  public:
+
+    BucketsSyncThread(CephContext *_cct, RGWUserStatsCache *_s) : cct(_cct), stats(_s), lock("RGWUserStatsCache::BucketsSyncThread") {}
+
+    void *entry() {
+      ldout(cct, 20) << "BucketsSyncThread: start" << dendl;
+      do {
+        map buckets;
+
+        stats->swap_modified_buckets(buckets);
+
+        for (map::iterator iter = buckets.begin(); iter != buckets.end(); ++iter) {
+          rgw_bucket bucket = iter->first;
+          string& user = iter->second;
+          ldout(cct, 20) << "BucketsSyncThread: sync user=" << user << " bucket=" << bucket << dendl;
+          int r = stats->sync_bucket(user, bucket);
+          if (r < 0) {
+            ldout(cct, 0) << "WARNING: sync_bucket() returned r=" << r << dendl;
+          }
+        }
+
+        if (stats->going_down())
+          break;
+
+        lock.Lock();
+        cond.WaitInterval(cct, lock, utime_t(cct->_conf->rgw_user_quota_bucket_sync_interval, 0));
+        lock.Unlock();
+      } while (!stats->going_down());
+      ldout(cct, 20) << "BucketsSyncThread: done" << dendl;
+
+      return NULL;
+    }
+
+    void stop() {
+      Mutex::Locker l(lock);
+      cond.Signal();
+    }
+  };
+
+  /*
+   * thread, full sync all users stats periodically
+   *
+   * only sync non idle users or ones that never got synced before, this is needed so that
+   * users that didn't have quota turned on before (or existed before the user objclass
+   * tracked stats) need to get their backend stats up to date.
+   */
+  class UserSyncThread : public Thread {
+    CephContext *cct;
+    RGWUserStatsCache *stats;
+
+    Mutex lock;
+    Cond cond;
+  public:
+
+    UserSyncThread(CephContext *_cct, RGWUserStatsCache *_s) : cct(_cct), stats(_s), lock("RGWUserStatsCache::UserSyncThread") {}
+
+    void *entry() {
+      ldout(cct, 20) << "UserSyncThread: start" << dendl;
+      do {
+
+        string key = "user";
+
+        int ret = stats->sync_all_users();
+        if (ret < 0) {
+          ldout(cct, 0) << "ERROR: sync_all_users() returned ret=" << ret << dendl;
+        }
+
+        lock.Lock();
+        cond.WaitInterval(cct, lock, utime_t(cct->_conf->rgw_user_quota_sync_interval, 0));
+        lock.Unlock();
+      } while (!stats->going_down());
+      ldout(cct, 20) << "UserSyncThread: done" << dendl;
+
+      return NULL;
+    }
+
+    void stop() {
+      Mutex::Locker l(lock);
+      cond.Signal();
+    }
+  };
+
+  BucketsSyncThread *buckets_sync_thread;
+  UserSyncThread *user_sync_thread;
+protected:
+  bool map_find(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+    return stats_map.find(user, qs);
+  }
+
+  bool map_find_and_update(const string& user, rgw_bucket& bucket, lru_map::UpdateContext *ctx) {
+    return stats_map.find_and_update(user, NULL, ctx);
+  }
+
+  void map_add(const string& user, rgw_bucket& bucket, RGWQuotaCacheStats& qs) {
+    stats_map.add(user, qs);
+  }
+
+  int fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats);
+  int sync_bucket(const string& user, rgw_bucket& bucket);
+  int sync_user(const string& user);
+  int sync_all_users();
+
+  void data_modified(const string& user, rgw_bucket& bucket);
+
+  void swap_modified_buckets(map& out) {
+    rwlock.get_write();
+    modified_buckets.swap(out);
+    rwlock.unlock();
+  }
+
+  template /* easier doing it as a template, Thread doesn't have ->stop() */
+  void stop_thread(T **pthr) {
+    T *thread = *pthr;
+    if (!thread)
+      return;
+
+    thread->stop();
+    thread->join();
+    delete thread;
+    *pthr = NULL;
+  }
+
+public:
+  RGWUserStatsCache(RGWRados *_store, bool quota_threads) : RGWQuotaCache(_store, _store->ctx()->_conf->rgw_bucket_quota_cache_size),
+                                        rwlock("RGWUserStatsCache::rwlock") {
+    if (quota_threads) {
+      buckets_sync_thread = new BucketsSyncThread(store->ctx(), this);
+      buckets_sync_thread->create();
+      user_sync_thread = new UserSyncThread(store->ctx(), this);
+      user_sync_thread->create();
+    } else {
+      buckets_sync_thread = NULL;
+      user_sync_thread = NULL;
+    }
+  }
+  ~RGWUserStatsCache() {
+    stop();
+  }
+
+  AsyncRefreshHandler *allocate_refresh_handler(const string& user, rgw_bucket& bucket) {
+    return new UserAsyncRefreshHandler(store, this, user, bucket);
+  }
+
+  bool can_use_cached_stats(RGWQuotaInfo& quota, RGWStorageStats& stats) {
+    /* in the user case, the cached stats may contain a better estimation of the totals, as
+     * the backend is only periodically getting updated.
+     */
+    return true;
+  }
+
+  bool going_down() {
+    return (down_flag.read() != 0);
+  }
+
+  void stop() {
+    down_flag.set(1);
+    rwlock.get_write();
+    stop_thread(&buckets_sync_thread);
+    rwlock.unlock();
+    stop_thread(&user_sync_thread);
+  }
+};
+
+int RGWUserStatsCache::fetch_stats_from_storage(const string& user, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+  int r = store->get_user_stats(user, stats);
+  if (r < 0) {
+    ldout(store->ctx(), 0) << "could not get user stats for user=" << user << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWUserStatsCache::sync_bucket(const string& user, rgw_bucket& bucket)
+{
+  int r = rgw_bucket_sync_user_stats(store, user, bucket);
+  if (r < 0) {
+    ldout(store->ctx(), 0) << "ERROR: rgw_bucket_sync_user_stats() for user=" << user << ", bucket=" << bucket << " returned " << r << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWUserStatsCache::sync_user(const string& user)
+{
+  cls_user_header header;
+  int ret = store->cls_user_get_header(user, &header);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: can't read user header: ret=" << ret << dendl;
+    return ret;
+  }
+
+  if (!store->ctx()->_conf->rgw_user_quota_sync_idle_users &&
+      header.last_stats_update < header.last_stats_sync) {
+    ldout(store->ctx(), 20) << "user is idle, not doing a full sync (user=" << user << ")" << dendl;
+    return 0;
+  }
+
+  utime_t when_need_full_sync = header.last_stats_sync;
+  when_need_full_sync += store->ctx()->_conf->rgw_user_quota_sync_wait_time;
+  
+  // check if enough time passed since last full sync
+
+  ret = rgw_user_sync_all_stats(store, user);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: failed user stats sync, ret=" << ret << dendl;
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWUserStatsCache::sync_all_users()
+{
+  string key = "user";
+  void *handle;
+
+  int ret = store->meta_mgr->list_keys_init(key, &handle);
+  if (ret < 0) {
+    ldout(store->ctx(), 0) << "ERROR: can't get key: ret=" << ret << dendl;
+    return ret;
+  }
+
+  bool truncated;
+  int max = 1000;
+
+  do {
+    list keys;
+    ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated);
+    if (ret < 0) {
+      ldout(store->ctx(), 0) << "ERROR: lists_keys_next(): ret=" << ret << dendl;
+      goto done;
+    }
+    for (list::iterator iter = keys.begin();
+         iter != keys.end() && !going_down(); 
+         ++iter) {
+      string& user = *iter;
+      ldout(store->ctx(), 20) << "RGWUserStatsCache: sync user=" << user << dendl;
+      int ret = sync_user(user);
+      if (ret < 0) {
+        ldout(store->ctx(), 0) << "ERROR: sync_user() failed, user=" << user << " ret=" << ret << dendl;
+
+        /* continuing to next user */
+        continue;
+      }
+    }
+  } while (truncated);
+
+  ret = 0;
+done:
+  store->meta_mgr->list_keys_complete(handle);
+  return ret;
+}
+
+void RGWUserStatsCache::data_modified(const string& user, rgw_bucket& bucket)
+{
+  /* racy, but it's ok */
+  rwlock.get_read();
+  bool need_update = modified_buckets.find(bucket) == modified_buckets.end();
+  rwlock.unlock();
+
+  if (need_update) {
+    rwlock.get_write();
+    modified_buckets[bucket] = user;
+    rwlock.unlock();
+  }
+}
+
+
+class RGWQuotaHandlerImpl : public RGWQuotaHandler {
+  RGWRados *store;
+  RGWBucketStatsCache bucket_stats_cache;
+  RGWUserStatsCache user_stats_cache;
+
+  int check_quota(const char *entity, RGWQuotaInfo& quota, RGWStorageStats& stats,
+                  uint64_t num_objs, uint64_t size_kb) {
+    ldout(store->ctx(), 20) << entity << " quota: max_objects=" << quota.max_objects
+                            << " max_size_kb=" << quota.max_size_kb << dendl;
+
+    if (quota.max_objects >= 0 &&
+        stats.num_objects + num_objs > (uint64_t)quota.max_objects) {
+      ldout(store->ctx(), 10) << "quota exceeded: stats.num_objects=" << stats.num_objects
+                              << " " << entity << "_quota.max_objects=" << quota.max_objects << dendl;
+
+      return -ERR_QUOTA_EXCEEDED;
+    }
+    if (quota.max_size_kb >= 0 &&
+               stats.num_kb_rounded + size_kb > (uint64_t)quota.max_size_kb) {
+      ldout(store->ctx(), 10) << "quota exceeded: stats.num_kb_rounded=" << stats.num_kb_rounded << " size_kb=" << size_kb
+                              << " " << entity << "_quota.max_size_kb=" << quota.max_size_kb << dendl;
+      return -ERR_QUOTA_EXCEEDED;
+    }
+
+    return 0;
+  }
+public:
+  RGWQuotaHandlerImpl(RGWRados *_store, bool quota_threads) : store(_store), bucket_stats_cache(_store), user_stats_cache(_store, quota_threads) {}
+  virtual int check_quota(const string& user, rgw_bucket& bucket,
+                          RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota,
+			  uint64_t num_objs, uint64_t size) {
+
+    if (!bucket_quota.enabled && !user_quota.enabled)
+      return 0;
+
+    uint64_t size_kb = rgw_rounded_objsize_kb(size);
+
+    RGWStorageStats bucket_stats;
+
+    /*
+     * we need to fetch bucket stats if the user quota is enabled, because the whole system relies
+     * on us periodically updating the user's bucket stats in the user's header, this happens in
+     * get_stats() if we actually fetch that info and not rely on cached data
+     */
+
+    int ret = bucket_stats_cache.get_stats(user, bucket, bucket_stats, bucket_quota);
+    if (ret < 0)
+      return ret;
+
+    if (bucket_quota.enabled) {
+      ret = check_quota("bucket", bucket_quota, bucket_stats, num_objs, size_kb);
+      if (ret < 0)
+        return ret;
+    }
+
+    if (user_quota.enabled) {
+      RGWStorageStats user_stats;
+
+      ret = user_stats_cache.get_stats(user, bucket, user_stats, user_quota);
+      if (ret < 0)
+        return ret;
+
+      ret = check_quota("user", user_quota, user_stats, num_objs, size_kb);
+      if (ret < 0)
+        return ret;
+    }
+
+    return 0;
+  }
+
+  virtual void update_stats(const string& user, rgw_bucket& bucket, int obj_delta, uint64_t added_bytes, uint64_t removed_bytes) {
+    bucket_stats_cache.adjust_stats(user, bucket, obj_delta, added_bytes, removed_bytes);
+    user_stats_cache.adjust_stats(user, bucket, obj_delta, added_bytes, removed_bytes);
+  };
+};
+
+
+RGWQuotaHandler *RGWQuotaHandler::generate_handler(RGWRados *store, bool quota_threads)
+{
+  return new RGWQuotaHandlerImpl(store, quota_threads);
+};
+
+void RGWQuotaHandler::free_handler(RGWQuotaHandler *handler)
+{
+  delete handler;
+}
diff --git a/ceph/src/rgw/rgw_quota.h b/ceph/src/rgw/rgw_quota.h
new file mode 100644
index 00000000..abdb62ed
--- /dev/null
+++ b/ceph/src/rgw/rgw_quota.h
@@ -0,0 +1,75 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_RGW_QUOTA_H
+#define CEPH_RGW_QUOTA_H
+
+
+#include "include/utime.h"
+#include "include/atomic.h"
+#include "common/lru_map.h"
+
+class RGWRados;
+class JSONObj;
+
+struct RGWQuotaInfo {
+  int64_t max_size_kb;
+  int64_t max_objects;
+  bool enabled;
+  int64_t max_size_soft_threshold;
+  int64_t max_objs_soft_threshold;
+
+  RGWQuotaInfo() : max_size_kb(-1), max_objects(-1), enabled(false),
+                   max_size_soft_threshold(-1), max_objs_soft_threshold(-1) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(max_size_kb, bl);
+    ::encode(max_objects, bl);
+    ::encode(enabled, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(max_size_kb, bl);
+    ::decode(max_objects, bl);
+    ::decode(enabled, bl);
+    DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+
+  void decode_json(JSONObj *obj);
+
+};
+WRITE_CLASS_ENCODER(RGWQuotaInfo)
+
+class rgw_bucket;
+
+class RGWQuotaHandler {
+public:
+  RGWQuotaHandler() {}
+  virtual ~RGWQuotaHandler() {
+  }
+  virtual int check_quota(const string& bucket_owner, rgw_bucket& bucket,
+                          RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota,
+			  uint64_t num_objs, uint64_t size) = 0;
+
+  virtual void update_stats(const string& bucket_owner, rgw_bucket& bucket, int obj_delta, uint64_t added_bytes, uint64_t removed_bytes) = 0;
+
+  static RGWQuotaHandler *generate_handler(RGWRados *store, bool quota_threads);
+  static void free_handler(RGWQuotaHandler *handler);
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_rados.cc b/ceph/src/rgw/rgw_rados.cc
new file mode 100644
index 00000000..e22bef0b
--- /dev/null
+++ b/ceph/src/rgw/rgw_rados.cc
@@ -0,0 +1,6924 @@
+#include 
+#include 
+#include 
+
+#include "common/ceph_json.h"
+#include "common/utf8.h"
+
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/Throttle.h"
+
+#include "rgw_rados.h"
+#include "rgw_cache.h"
+#include "rgw_acl.h"
+#include "rgw_acl_s3.h" /* for dumping s3policy in debug log */
+#include "rgw_metadata.h"
+#include "rgw_bucket.h"
+
+#include "cls/rgw/cls_rgw_types.h"
+#include "cls/rgw/cls_rgw_client.h"
+#include "cls/refcount/cls_refcount_client.h"
+#include "cls/version/cls_version_client.h"
+#include "cls/log/cls_log_client.h"
+#include "cls/statelog/cls_statelog_client.h"
+#include "cls/lock/cls_lock_client.h"
+#include "cls/user/cls_user_client.h"
+
+#include "rgw_tools.h"
+
+#include "common/Clock.h"
+
+#include "include/rados/librados.hpp"
+using namespace librados;
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "auth/Crypto.h" // get_random_bytes()
+
+#include "rgw_log.h"
+
+#include "rgw_gc.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+static RGWCache cached_rados_provider;
+static RGWRados rados_provider;
+
+static string notify_oid_prefix = "notify";
+static string *notify_oids = NULL;
+static string shadow_ns = "shadow";
+static string dir_oid_prefix = ".dir.";
+static string default_storage_pool = ".rgw.buckets";
+static string default_storage_extra_pool = ".rgw.buckets.extra";
+static string avail_pools = ".pools.avail";
+
+static string zone_info_oid_prefix = "zone_info.";
+static string region_info_oid_prefix = "region_info.";
+
+static string default_region_info_oid = "default.region";
+static string region_map_oid = "region_map";
+static string log_lock_name = "rgw_log_lock";
+
+static RGWObjCategory main_category = RGW_OBJ_CATEGORY_MAIN;
+
+#define RGW_USAGE_OBJ_PREFIX "usage."
+
+#define RGW_DEFAULT_ZONE_ROOT_POOL ".rgw.root"
+#define RGW_DEFAULT_REGION_ROOT_POOL ".rgw.root"
+
+#define RGW_STATELOG_OBJ_PREFIX "statelog."
+
+
+#define dout_subsys ceph_subsys_rgw
+
+void RGWDefaultRegionInfo::dump(Formatter *f) const {
+  encode_json("default_region", default_region, f);
+}
+
+void RGWDefaultRegionInfo::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("default_region", default_region, obj);
+}
+
+int RGWRegion::get_pool_name(CephContext *cct, string *pool_name)
+{
+  *pool_name = cct->_conf->rgw_region_root_pool;
+  if (pool_name->empty()) {
+    *pool_name = RGW_DEFAULT_REGION_ROOT_POOL;
+  } else if ((*pool_name)[0] != '.') {
+    derr << "ERROR: region root pool name must start with a period" << dendl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
+int RGWRegion::read_default(RGWDefaultRegionInfo& default_info)
+{
+  string pool_name;
+
+  int ret = get_pool_name(cct, &pool_name);
+  if (ret < 0) {
+    return ret;
+  }
+
+  string oid = cct->_conf->rgw_default_region_info_oid;
+  if (oid.empty()) {
+    oid = default_region_info_oid;
+  }
+
+  rgw_bucket pool(pool_name.c_str());
+  bufferlist bl;
+  ret = rgw_get_system_obj(store, NULL, pool, oid, bl, NULL, NULL);
+  if (ret < 0)
+    return ret;
+
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(default_info, iter);
+  } catch (buffer::error& err) {
+    derr << "error decoding data from " << pool << ":" << oid << dendl;
+    return -EIO;
+  }
+
+  name = default_info.default_region;
+
+  return 0;
+}
+
+int RGWRegion::set_as_default()
+{
+  string pool_name;
+  int ret = get_pool_name(cct, &pool_name);
+  if (ret < 0)
+    return ret;
+
+  string oid = cct->_conf->rgw_default_region_info_oid;
+  if (oid.empty()) {
+    oid = default_region_info_oid;
+  }
+
+  rgw_bucket pool(pool_name.c_str());
+  bufferlist bl;
+
+  RGWDefaultRegionInfo default_info;
+  default_info.default_region = name;
+
+  ::encode(default_info, bl);
+
+  ret = rgw_put_system_obj(store, pool, oid, bl.c_str(), bl.length(), false, NULL, 0, NULL);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWRegion::init(CephContext *_cct, RGWRados *_store, bool setup_region)
+{
+  cct = _cct;
+  store = _store;
+
+  if (!setup_region)
+    return 0;
+
+  string region_name = cct->_conf->rgw_region;
+
+  if (region_name.empty()) {
+    RGWDefaultRegionInfo default_info;
+    int r = read_default(default_info);
+    if (r == -ENOENT) {
+      r = create_default();
+      if (r == -EEXIST) { /* we may have raced with another region creation,
+                             make sure we can read the region info and continue
+                             as usual to make sure region creation is complete */
+        ldout(cct, 0) << "create_default() returned -EEXIST, we raced with another region creation" << dendl;
+        r = read_info(name);
+      }
+      if (r < 0)
+        return r;
+      r = set_as_default(); /* set this as default even if we weren't the creators */
+      if (r < 0)
+        return r;
+      /*Re attempt to read region info from newly created default region */
+      r = read_default(default_info);
+      if (r < 0)
+	return r;
+    } else if (r < 0) {
+      lderr(cct) << "failed reading default region info: " << cpp_strerror(-r) << dendl;
+      return r;
+    }
+    region_name = default_info.default_region;
+  }
+
+  return read_info(region_name);
+}
+
+int RGWRegion::read_info(const string& region_name)
+{
+  string pool_name;
+  int ret = get_pool_name(cct, &pool_name);
+  if (ret < 0)
+    return ret;
+
+  rgw_bucket pool(pool_name.c_str());
+  bufferlist bl;
+
+  name = region_name;
+
+  string oid = region_info_oid_prefix + name;
+
+  ret = rgw_get_system_obj(store, NULL, pool, oid, bl, NULL, NULL);
+  if (ret < 0) {
+    lderr(cct) << "failed reading region info from " << pool << ":" << oid << ": " << cpp_strerror(-ret) << dendl;
+    return ret;
+  }
+
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(*this, iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: failed to decode region from " << pool << ":" << oid << dendl;
+    return -EIO;
+  }
+
+  return 0;
+}
+
+int RGWRegion::create_default()
+{
+  name = "default";
+  string zone_name = "default";
+
+  is_master = true;
+
+  RGWRegionPlacementTarget placement_target;
+  placement_target.name = "default-placement";
+  placement_targets[placement_target.name] = placement_target;
+  default_placement = "default-placement";
+
+  RGWZone& default_zone = zones[zone_name];
+  default_zone.name = zone_name;
+
+  RGWZoneParams zone_params;
+  zone_params.name = zone_name;
+  zone_params.init_default(store);
+
+  int r = zone_params.store_info(cct, store, *this);
+  if (r < 0) {
+    derr << "error storing zone params: " << cpp_strerror(-r) << dendl;
+    return r;
+  }
+
+  r = store_info(true);
+  if (r < 0) {
+    derr << "error storing region info: " << cpp_strerror(-r) << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWRegion::store_info(bool exclusive)
+{
+  string pool_name;
+  int ret = get_pool_name(cct, &pool_name);
+  if (ret < 0)
+    return ret;
+
+  rgw_bucket pool(pool_name.c_str());
+
+  string oid = region_info_oid_prefix + name;
+
+  bufferlist bl;
+  ::encode(*this, bl);
+  ret = rgw_put_system_obj(store, pool, oid, bl.c_str(), bl.length(), exclusive, NULL, 0, NULL);
+
+  return ret;
+}
+
+int RGWRegion::equals(const string& other_region)
+{
+  if (is_master && other_region.empty())
+    return true;
+
+  return (name == other_region);
+}
+
+void RGWZoneParams::init_default(RGWRados *store)
+{
+  domain_root = ".rgw";
+  control_pool = ".rgw.control";
+  gc_pool = ".rgw.gc";
+  log_pool = ".log";
+  intent_log_pool = ".intent-log";
+  usage_log_pool = ".usage";
+  user_keys_pool = ".users";
+  user_email_pool = ".users.email";
+  user_swift_pool = ".users.swift";
+  user_uid_pool = ".users.uid";
+
+  /* check for old pools config */
+  rgw_obj obj(domain_root, avail_pools);
+  int r =  store->obj_stat(NULL, obj, NULL, NULL, NULL, NULL, NULL, NULL);
+  if (r < 0) {
+    ldout(store->ctx(), 0) << "couldn't find old data placement pools config, setting up new ones for the zone" << dendl;
+    /* a new system, let's set new placement info */
+    RGWZonePlacementInfo default_placement;
+    default_placement.index_pool = ".rgw.buckets.index";
+    default_placement.data_pool = ".rgw.buckets";
+    default_placement.data_extra_pool = ".rgw.buckets.extra";
+    placement_pools["default-placement"] = default_placement;
+  }
+}
+
+int RGWZoneParams::get_pool_name(CephContext *cct, string *pool_name)
+{
+  *pool_name = cct->_conf->rgw_zone_root_pool;
+  if (pool_name->empty()) {
+    *pool_name = RGW_DEFAULT_ZONE_ROOT_POOL;
+  } else if ((*pool_name)[0] != '.') {
+    derr << "ERROR: zone root pool name must start with a period" << dendl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+void RGWZoneParams::init_name(CephContext *cct, RGWRegion& region)
+{
+  name = cct->_conf->rgw_zone;
+
+  if (name.empty()) {
+    name = region.master_zone;
+
+    if (name.empty()) {
+      name = "default";
+    }
+  }
+}
+
+int RGWZoneParams::init(CephContext *cct, RGWRados *store, RGWRegion& region)
+{
+  init_name(cct, region);
+
+  string pool_name;
+  int ret = get_pool_name(cct, &pool_name);
+  if (ret < 0)
+    return ret;
+
+  rgw_bucket pool(pool_name.c_str());
+  bufferlist bl;
+
+  string oid = zone_info_oid_prefix + name;
+  ret = rgw_get_system_obj(store, NULL, pool, oid, bl, NULL, NULL);
+  if (ret < 0)
+    return ret;
+
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(*this, iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: failed to decode zone info from " << pool << ":" << oid << dendl;
+    return -EIO;
+  }
+
+  is_master = (name == region.master_zone) || (region.master_zone.empty() && name == "default");
+
+  ldout(cct, 2) << "zone " << name << " is " << (is_master ? "" : "NOT ") << "master" << dendl;
+
+  return 0;
+}
+
+int RGWZoneParams::store_info(CephContext *cct, RGWRados *store, RGWRegion& region)
+{
+  init_name(cct, region);
+
+  string pool_name;
+  int ret = get_pool_name(cct, &pool_name);
+  if (ret < 0)
+    return ret;
+
+  rgw_bucket pool(pool_name.c_str());
+  string oid = zone_info_oid_prefix + name;
+
+  bufferlist bl;
+  ::encode(*this, bl);
+  ret = rgw_put_system_obj(store, pool, oid, bl.c_str(), bl.length(), false, NULL, 0, NULL);
+
+  return ret;
+}
+
+void RGWRegionMap::encode(bufferlist& bl) const {
+  ENCODE_START(3, 1, bl);
+  ::encode(regions, bl);
+  ::encode(master_region, bl);
+  ::encode(bucket_quota, bl);
+  ::encode(user_quota, bl);
+  ENCODE_FINISH(bl);
+}
+
+void RGWRegionMap::decode(bufferlist::iterator& bl) {
+  DECODE_START(3, bl);
+  ::decode(regions, bl);
+  ::decode(master_region, bl);
+
+  if (struct_v >= 2)
+    ::decode(bucket_quota, bl);
+  if (struct_v >= 3)
+    ::decode(user_quota, bl);
+  DECODE_FINISH(bl);
+
+  regions_by_api.clear();
+  for (map::iterator iter = regions.begin();
+       iter != regions.end(); ++iter) {
+    RGWRegion& region = iter->second;
+    regions_by_api[region.api_name] = region;
+    if (region.is_master) {
+      master_region = region.name;
+    }
+  }
+}
+
+void RGWRegionMap::get_params(CephContext *cct, string& pool_name, string& oid)
+{
+  pool_name = cct->_conf->rgw_zone_root_pool;
+  if (pool_name.empty()) {
+    pool_name = RGW_DEFAULT_ZONE_ROOT_POOL;
+  }
+  oid = region_map_oid;
+}
+
+int RGWRegionMap::read(CephContext *cct, RGWRados *store)
+{
+  string pool_name, oid;
+
+  get_params(cct, pool_name, oid);
+
+  rgw_bucket pool(pool_name.c_str());
+
+  bufferlist bl;
+  int ret = rgw_get_system_obj(store, NULL, pool, oid, bl, NULL, NULL);
+  if (ret < 0)
+    return ret;
+
+
+  Mutex::Locker l(lock);
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(*this, iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: failed to decode region map info from " << pool << ":" << oid << dendl;
+    return -EIO;
+  }
+
+  return 0;
+}
+
+int RGWRegionMap::store(CephContext *cct, RGWRados *store)
+{
+  string pool_name, oid;
+
+  get_params(cct, pool_name, oid);
+
+  rgw_bucket pool(pool_name.c_str());
+
+  Mutex::Locker l(lock);
+
+  bufferlist bl;
+  ::encode(*this, bl);
+  int ret = rgw_put_system_obj(store, pool, oid, bl.c_str(), bl.length(), false, NULL, 0, NULL);
+
+  return ret;
+}
+
+int RGWRegionMap::update(RGWRegion& region)
+{
+  Mutex::Locker l(lock);
+
+  if (region.is_master && !region.equals(master_region)) {
+    derr << "cannot update region map, master_region conflict" << dendl;
+    return -EINVAL;
+  }
+  map::iterator iter = regions.find(region.name);
+  if (iter != regions.end()) {
+    RGWRegion& old_region = iter->second;
+    if (!old_region.api_name.empty()) {
+      regions_by_api.erase(old_region.api_name);
+    }
+  }
+  regions[region.name] = region;
+
+  if (!region.api_name.empty()) {
+    regions_by_api[region.api_name] = region;
+  }
+
+  if (region.is_master) {
+    master_region = region.name;
+  }
+  return 0;
+}
+
+
+void RGWObjVersionTracker::prepare_op_for_read(ObjectReadOperation *op)
+{
+  obj_version *check_objv = version_for_check();
+
+  if (check_objv) {
+    cls_version_check(*op, *check_objv, VER_COND_EQ);
+  }
+
+  cls_version_read(*op, &read_version);
+}
+
+void RGWObjVersionTracker::prepare_op_for_write(ObjectWriteOperation *op)
+{
+  obj_version *check_objv = version_for_check();
+  obj_version *modify_version = version_for_write();
+
+  if (check_objv) {
+    cls_version_check(*op, *check_objv, VER_COND_EQ);
+  }
+
+  if (modify_version) {
+    cls_version_set(*op, *modify_version);
+  } else {
+    cls_version_inc(*op);
+  }
+}
+
+void RGWObjManifest::obj_iterator::operator++()
+{
+  if (manifest->explicit_objs) {
+    ++explicit_iter;
+
+    if (explicit_iter == manifest->objs.end()) {
+      ofs = manifest->obj_size;
+      return;
+    }
+
+    update_explicit_pos();
+
+    update_location();
+    return;
+  }
+
+  uint64_t obj_size = manifest->get_obj_size();
+  uint64_t head_size = manifest->get_head_size();
+
+  if (ofs == obj_size) {
+    return;
+  }
+
+  if (manifest->rules.empty()) {
+    return;
+  }
+
+  /* are we still pointing at the head? */
+  if (ofs < head_size) {
+    rule_iter = manifest->rules.begin();
+    RGWObjManifestRule *rule = &rule_iter->second;
+    ofs = MIN(head_size, obj_size);
+    stripe_ofs = ofs;
+    cur_stripe = 1;
+    stripe_size = MIN(obj_size - ofs, rule->stripe_max_size);
+    if (rule->part_size > 0) {
+      stripe_size = MIN(stripe_size, rule->part_size);
+    }
+    update_location();
+    return;
+  }
+
+  RGWObjManifestRule *rule = &rule_iter->second;
+
+  stripe_ofs += rule->stripe_max_size;
+  cur_stripe++;
+  dout(20) << "RGWObjManifest::operator++(): rule->part_size=" << rule->part_size << " rules.size()=" << manifest->rules.size() << dendl;
+
+  if (rule->part_size > 0) {
+    /* multi part, multi stripes object */
+
+    dout(20) << "RGWObjManifest::operator++(): stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
+
+    if (stripe_ofs >= part_ofs + rule->part_size) {
+      /* moved to the next part */
+      cur_stripe = 0;
+      part_ofs += rule->part_size;
+      stripe_ofs = part_ofs;
+
+      bool last_rule = (next_rule_iter == manifest->rules.end());
+      /* move to the next rule? */
+      if (!last_rule && stripe_ofs >= next_rule_iter->second.start_ofs) {
+        rule_iter = next_rule_iter;
+        last_rule = (next_rule_iter == manifest->rules.end());
+        if (!last_rule) {
+          ++next_rule_iter;
+        }
+        cur_part_id = rule_iter->second.start_part_num;
+      } else {
+        cur_part_id++;
+      }
+
+      rule = &rule_iter->second;
+    }
+
+    stripe_size = MIN(rule->part_size - (stripe_ofs - part_ofs), rule->stripe_max_size);
+  }
+
+  cur_override_prefix = rule->override_prefix;
+
+  ofs = stripe_ofs;
+  if (ofs > obj_size) {
+    ofs = obj_size;
+    stripe_ofs = ofs;
+    stripe_size = 0;
+  }
+
+  dout(0) << "RGWObjManifest::operator++(): result: ofs=" << ofs << " stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
+  update_location();
+}
+
+int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m, rgw_bucket& _b, rgw_obj& _h)
+{
+  manifest = _m;
+
+  bucket = _b;
+  manifest->set_tail_bucket(_b);
+  manifest->set_head(_h);
+  last_ofs = 0;
+
+  if (manifest->get_prefix().empty()) {
+    char buf[33];
+    gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1);
+
+    string oid_prefix = ".";
+    oid_prefix.append(buf);
+    oid_prefix.append("_");
+
+    manifest->set_prefix(oid_prefix);
+  }
+
+  bool found = manifest->get_rule(0, &rule);
+  if (!found) {
+    derr << "ERROR: manifest->get_rule() could not find rule" << dendl;
+    return -EIO;
+  }
+
+  uint64_t head_size = manifest->get_head_size();
+
+  if (head_size > 0) {
+    cur_stripe_size = head_size;
+  } else {
+    cur_stripe_size = rule.stripe_max_size;
+  }
+  
+  cur_part_id = rule.start_part_num;
+
+  manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj);
+
+  manifest->update_iterators();
+
+  return 0;
+}
+
+int RGWObjManifest::generator::create_next(uint64_t ofs)
+{
+  if (ofs < last_ofs) /* only going forward */
+    return -EINVAL;
+
+  string obj_name = manifest->prefix;
+
+  uint64_t max_head_size = manifest->get_max_head_size();
+
+  if (ofs <= max_head_size) {
+    manifest->set_head_size(ofs);
+  }
+
+  if (ofs >= max_head_size) {
+    manifest->set_head_size(max_head_size);
+    cur_stripe = (ofs - max_head_size) / rule.stripe_max_size;
+    cur_stripe_size =  rule.stripe_max_size;
+
+    if (cur_part_id == 0 && max_head_size > 0) {
+      cur_stripe++;
+    }
+  }
+
+  last_ofs = ofs;
+  manifest->set_obj_size(ofs);
+
+
+  manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj);
+
+  manifest->update_iterators();
+
+  return 0;
+}
+
+const RGWObjManifest::obj_iterator& RGWObjManifest::obj_begin()
+{
+  return begin_iter;
+}
+
+const RGWObjManifest::obj_iterator& RGWObjManifest::obj_end()
+{
+  return end_iter;
+}
+
+RGWObjManifest::obj_iterator RGWObjManifest::obj_find(uint64_t ofs)
+{
+  if (ofs > obj_size) {
+    ofs = obj_size;
+  }
+  RGWObjManifest::obj_iterator iter(this);
+  iter.seek(ofs);
+  return iter;
+}
+
+int RGWObjManifest::append(RGWObjManifest& m)
+{
+  if (explicit_objs || m.explicit_objs) {
+    return append_explicit(m);
+  }
+
+  if (rules.empty()) {
+    *this = m;
+    return 0;
+  }
+
+  string override_prefix;
+
+  if (prefix.empty()) {
+    prefix = m.prefix;
+  }
+
+  if (prefix != m.prefix) {
+    override_prefix = m.prefix;
+  }
+
+  map::iterator miter = m.rules.begin();
+  if (miter == m.rules.end()) {
+    return append_explicit(m);
+  }
+
+  for (; miter != m.rules.end(); ++miter) {
+    map::reverse_iterator last_rule = rules.rbegin();
+
+    RGWObjManifestRule& rule = last_rule->second;
+
+    if (rule.part_size == 0) {
+      rule.part_size = obj_size - rule.start_ofs;
+    }
+
+    RGWObjManifestRule& next_rule = miter->second;
+    if (!next_rule.part_size) {
+      next_rule.part_size = m.obj_size - next_rule.start_ofs;
+    }
+
+    if (override_prefix != rule.override_prefix) {
+      append_rules(m, miter, &override_prefix);
+      break;
+    }
+
+    if (rule.part_size != next_rule.part_size ||
+        rule.stripe_max_size != next_rule.stripe_max_size ||
+        rule.override_prefix != next_rule.override_prefix) {
+      append_rules(m, miter, NULL);
+      break;
+    }
+
+    uint64_t expected_part_num = rule.start_part_num + 1;
+    if (rule.part_size > 0) {
+      expected_part_num = rule.start_part_num + (obj_size + next_rule.start_ofs - rule.start_ofs) / rule.part_size;
+    }
+
+    if (expected_part_num != next_rule.start_part_num) {
+      append_rules(m, miter, NULL);
+      break;
+    }
+  }
+
+  set_obj_size(obj_size + m.obj_size);
+
+  return 0;
+}
+
+void RGWObjManifest::append_rules(RGWObjManifest& m, map::iterator& miter,
+                                  string *override_prefix)
+{
+  for (; miter != m.rules.end(); ++miter) {
+    RGWObjManifestRule rule = miter->second;
+    rule.start_ofs += obj_size;
+    if (override_prefix)
+      rule.override_prefix = *override_prefix;
+    rules[rule.start_ofs] = rule;
+  }
+}
+
+void RGWObjManifest::convert_to_explicit()
+{
+  if (explicit_objs) {
+    return;
+  }
+  obj_iterator iter = obj_begin();
+
+  while (iter != obj_end()) {
+    RGWObjManifestPart& part = objs[iter.get_stripe_ofs()];
+    part.loc = iter.get_location();
+    part.loc_ofs = 0;
+
+    uint64_t ofs = iter.get_stripe_ofs();
+    ++iter;
+    uint64_t next_ofs = iter.get_stripe_ofs();
+
+    part.size = next_ofs - ofs;
+  }
+
+  explicit_objs = true;
+  rules.clear();
+  prefix.clear();
+}
+
+int RGWObjManifest::append_explicit(RGWObjManifest& m)
+{
+  if (!explicit_objs) {
+    convert_to_explicit();
+  }
+  if (!m.explicit_objs) {
+    m.convert_to_explicit();
+  }
+  map::iterator iter;
+  uint64_t base = obj_size;
+  for (iter = m.objs.begin(); iter != m.objs.end(); ++iter) {
+    RGWObjManifestPart& part = iter->second;
+    objs[base + iter->first] = part;
+  }
+  obj_size += m.obj_size;
+
+  return 0;
+}
+
+bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule)
+{
+  if (rules.empty()) {
+    return false;
+  }
+
+  map::iterator iter = rules.upper_bound(ofs);
+  if (iter != rules.begin()) {
+    --iter;
+  }
+
+  *rule = iter->second;
+
+  return true;
+}
+
+void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct)
+{
+  write_version.ver = 1;
+#define TAG_LEN 24
+
+  write_version.tag.clear();
+  append_rand_alpha(cct, write_version.tag, write_version.tag, TAG_LEN);
+}
+
+int RGWPutObjProcessor::complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs)
+{
+  int r = do_complete(etag, mtime, set_mtime, attrs);
+  if (r < 0)
+    return r;
+
+  is_complete = true;
+  return 0;
+}
+
+RGWPutObjProcessor::~RGWPutObjProcessor()
+{
+  if (is_complete)
+    return;
+
+  list::iterator iter;
+  for (iter = objs.begin(); iter != objs.end(); ++iter) {
+    rgw_obj& obj = *iter;
+    int r = store->delete_obj(obj_ctx, bucket_owner, obj);
+    if (r < 0 && r != -ENOENT) {
+      ldout(store->ctx(), 0) << "WARNING: failed to remove obj (" << obj << "), leaked" << dendl;
+    }
+  }
+}
+
+int RGWPutObjProcessor_Plain::prepare(RGWRados *store, void *obj_ctx, string *oid_rand)
+{
+  RGWPutObjProcessor::prepare(store, obj_ctx, oid_rand);
+
+  obj.init(bucket, obj_str);
+
+  return 0;
+};
+
+int RGWPutObjProcessor_Plain::handle_data(bufferlist& bl, off_t _ofs, void **phandle, bool *again)
+{
+  *again = false;
+
+  if (ofs != _ofs)
+    return -EINVAL;
+
+  data.append(bl);
+  ofs += bl.length();
+
+  return 0;
+}
+
+int RGWPutObjProcessor_Plain::do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs)
+{
+  RGWRados::PutObjMetaExtraParams params;
+  params.set_mtime = set_mtime;
+  params.mtime = mtime;
+  params.data = &data;
+  params.owner = bucket_owner;
+
+  int r = store->put_obj_meta(obj_ctx, obj, data.length(), attrs,
+                              RGW_OBJ_CATEGORY_MAIN, PUT_OBJ_CREATE,
+                              params);
+  return r;
+}
+
+
+int RGWPutObjProcessor_Aio::handle_obj_data(rgw_obj& obj, bufferlist& bl, off_t ofs, off_t abs_ofs, void **phandle, bool exclusive)
+{
+  if ((uint64_t)abs_ofs + bl.length() > obj_len)
+    obj_len = abs_ofs + bl.length();
+
+  // For the first call pass -1 as the offset to
+  // do a write_full.
+  int r = store->aio_put_obj_data(NULL, obj,
+                                     bl,
+                                     ((ofs != 0) ? ofs : -1),
+                                     exclusive, phandle);
+
+  return r;
+}
+
+struct put_obj_aio_info RGWPutObjProcessor_Aio::pop_pending()
+{
+  struct put_obj_aio_info info;
+  info = pending.front();
+  pending.pop_front();
+  return info;
+}
+
+int RGWPutObjProcessor_Aio::wait_pending_front()
+{
+  if (pending.empty()) {
+    return 0;
+  }
+  struct put_obj_aio_info info = pop_pending();
+  int ret = store->aio_wait(info.handle);
+  return ret;
+}
+
+bool RGWPutObjProcessor_Aio::pending_has_completed()
+{
+  if (pending.empty())
+    return false;
+
+  struct put_obj_aio_info& info = pending.front();
+  return store->aio_completed(info.handle);
+}
+
+int RGWPutObjProcessor_Aio::drain_pending()
+{
+  int ret = 0;
+  while (!pending.empty()) {
+    int r = wait_pending_front();
+    if (r < 0)
+      ret = r;
+  }
+  return ret;
+}
+
+int RGWPutObjProcessor_Aio::throttle_data(void *handle, bool need_to_wait)
+{
+  if (handle) {
+    struct put_obj_aio_info info;
+    info.handle = handle;
+    pending.push_back(info);
+  }
+  size_t orig_size = pending.size();
+
+  /* first drain complete IOs */
+  while (pending_has_completed()) {
+    int r = wait_pending_front();
+    if (r < 0)
+      return r;
+
+    need_to_wait = false;
+  }
+
+  /* resize window in case messages are draining too fast */
+  if (orig_size - pending.size() >= max_chunks) {
+    max_chunks++;
+  }
+
+  /* now throttle. Note that need_to_wait should only affect the first IO operation */
+  if (pending.size() > max_chunks ||
+      need_to_wait) {
+    int r = wait_pending_front();
+    if (r < 0)
+      return r;
+
+    need_to_wait = false;
+  }
+  return 0;
+}
+
+int RGWPutObjProcessor_Atomic::write_data(bufferlist& bl, off_t ofs, void **phandle, bool exclusive)
+{
+  if (ofs >= next_part_ofs) {
+    int r = prepare_next_part(ofs);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return RGWPutObjProcessor_Aio::handle_obj_data(cur_obj, bl, ofs - cur_part_ofs, ofs, phandle, exclusive);
+}
+
+int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, void **phandle, bool *again)
+{
+  *again = false;
+
+  *phandle = NULL;
+  if (extra_data_len) {
+    size_t extra_len = bl.length();
+    if (extra_len > extra_data_len)
+      extra_len = extra_data_len;
+
+    bufferlist extra;
+    bl.splice(0, extra_len, &extra);
+    extra_data_bl.append(extra);
+
+    extra_data_len -= extra_len;
+    if (bl.length() == 0) {
+      return 0;
+    }
+  }
+
+  uint64_t max_write_size = MIN(max_chunk_size, (uint64_t)next_part_ofs - data_ofs);
+
+  pending_data_bl.claim_append(bl);
+  if (pending_data_bl.length() < max_write_size)
+    return 0;
+
+  pending_data_bl.splice(0, max_write_size, &bl);
+
+  /* do we have enough data pending accumulated that needs to be written? */
+  *again = (pending_data_bl.length() >= max_chunk_size);
+
+  if (!data_ofs && !immutable_head()) {
+    first_chunk.claim(bl);
+    obj_len = (uint64_t)first_chunk.length();
+    int r = prepare_next_part(first_chunk.length());
+    if (r < 0) {
+      return r;
+    }
+    data_ofs = obj_len;
+    return 0;
+  }
+  off_t write_ofs = data_ofs;
+  data_ofs = write_ofs + bl.length();
+  bool exclusive = (!write_ofs && immutable_head()); /* immutable head object, need to verify nothing exists there
+                                                        we could be racing with another upload, to the same
+                                                        object and cleanup can be messy */
+  return write_data(bl, write_ofs, phandle, exclusive);
+}
+
+
+int RGWPutObjProcessor_Atomic::prepare_init(RGWRados *store, void *obj_ctx, string *oid_rand)
+{
+  RGWPutObjProcessor::prepare(store, obj_ctx, oid_rand);
+
+  int r = store->get_max_chunk_size(bucket, &max_chunk_size);
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWPutObjProcessor_Atomic::prepare(RGWRados *store, void *obj_ctx, string *oid_rand)
+{
+  int r = prepare_init(store, obj_ctx, oid_rand);
+  if (r < 0) {
+    return r;
+  }
+  head_obj.init(bucket, obj_str);
+
+  manifest.set_trivial_rule(max_chunk_size, store->ctx()->_conf->rgw_obj_stripe_size);
+
+  r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, head_obj);
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWPutObjProcessor_Atomic::prepare_next_part(off_t ofs) {
+
+  int ret = manifest_gen.create_next(ofs);
+  if (ret < 0) {
+    lderr(store->ctx()) << "ERROR: manifest_gen.create_next() returned ret=" << ret << dendl;
+    return ret;
+  }
+  cur_part_ofs = ofs;
+  next_part_ofs = ofs + manifest_gen.cur_stripe_max_size();
+  cur_obj = manifest_gen.get_cur_obj();
+  add_obj(cur_obj);
+
+  return 0;
+};
+
+int RGWPutObjProcessor_Atomic::complete_parts()
+{
+  if (obj_len > (uint64_t)cur_part_ofs) {
+    return prepare_next_part(obj_len);
+  }
+  return 0;
+}
+
+int RGWPutObjProcessor_Atomic::complete_writing_data()
+{
+  if (!data_ofs && !immutable_head()) {
+    first_chunk.claim(pending_data_bl);
+    obj_len = (uint64_t)first_chunk.length();
+  }
+  if (pending_data_bl.length()) {
+    void *handle;
+    int r = write_data(pending_data_bl, data_ofs, &handle, false);
+    if (r < 0) {
+      ldout(store->ctx(), 0) << "ERROR: write_data() returned " << r << dendl;
+      return r;
+    }
+    r = throttle_data(handle, false);
+    if (r < 0) {
+      ldout(store->ctx(), 0) << "ERROR: throttle_data() returned " << r << dendl;
+      return r;
+    }
+  }
+  int r = complete_parts();
+  if (r < 0) {
+    return r;
+  }
+
+  r = drain_pending();
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWPutObjProcessor_Atomic::do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs) {
+  int r = complete_writing_data();
+  if (r < 0)
+    return r;
+
+  store->set_atomic(obj_ctx, head_obj);
+
+  RGWRados::PutObjMetaExtraParams extra_params;
+
+  extra_params.data = &first_chunk;
+  extra_params.manifest = &manifest;
+  extra_params.ptag = &unique_tag; /* use req_id as operation tag */
+  extra_params.mtime = mtime;
+  extra_params.set_mtime = set_mtime;
+  extra_params.owner = bucket_owner;
+
+  r = store->put_obj_meta(obj_ctx, head_obj, obj_len, attrs,
+                          RGW_OBJ_CATEGORY_MAIN, PUT_OBJ_CREATE,
+                          extra_params);
+  return r;
+}
+
+class RGWWatcher : public librados::WatchCtx {
+  RGWRados *rados;
+public:
+  RGWWatcher(RGWRados *r) : rados(r) {}
+  void notify(uint8_t opcode, uint64_t ver, bufferlist& bl) {
+    ldout(rados->ctx(), 10) << "RGWWatcher::notify() opcode=" << (int)opcode << " ver=" << ver << " bl.length()=" << bl.length() << dendl;
+    rados->watch_cb(opcode, ver, bl);
+  }
+};
+
+RGWObjState *RGWRadosCtx::get_state(rgw_obj& obj) {
+  if (obj.object.size()) {
+    return &objs_state[obj];
+  } else {
+    rgw_obj new_obj(store->zone.domain_root, obj.bucket.name);
+    return &objs_state[new_obj];
+  }
+}
+
+void RGWRadosCtx::set_atomic(rgw_obj& obj) {
+  if (obj.object.size()) {
+    objs_state[obj].is_atomic = true;
+  } else {
+    rgw_obj new_obj(store->zone.domain_root, obj.bucket.name);
+    objs_state[new_obj].is_atomic = true;
+  }
+}
+
+void RGWRadosCtx::set_prefetch_data(rgw_obj& obj) {
+  if (obj.object.size()) {
+    objs_state[obj].prefetch_data = true;
+  } else {
+    rgw_obj new_obj(store->zone.domain_root, obj.bucket.name);
+    objs_state[new_obj].prefetch_data = true;
+  }
+}
+
+int RGWRados::get_required_alignment(rgw_bucket& bucket, uint64_t *alignment)
+{
+  IoCtx ioctx;
+  int r = open_bucket_data_ctx(bucket, ioctx);
+  if (r < 0) {
+    ldout(cct, 0) << "ERROR: open_bucket_data_ctx() returned " << r << dendl;
+    return r;
+  }
+
+  *alignment = ioctx.pool_required_alignment();
+  return 0;
+}
+
+int RGWRados::get_max_chunk_size(rgw_bucket& bucket, uint64_t *max_chunk_size)
+{
+  uint64_t alignment;
+  int r = get_required_alignment(bucket, &alignment);
+  if (r < 0) {
+    return r;
+  }
+
+  uint64_t config_chunk_size = cct->_conf->rgw_max_chunk_size;
+
+  if (alignment == 0) {
+    *max_chunk_size = config_chunk_size;
+    return 0;
+  }
+
+  if (config_chunk_size <= alignment) {
+    *max_chunk_size = alignment;
+    return 0;
+  }
+
+  *max_chunk_size = config_chunk_size - (config_chunk_size % alignment);
+
+  return 0;
+}
+
+void RGWRados::finalize()
+{
+  if (need_watch_notify()) {
+    finalize_watch();
+  }
+  delete meta_mgr;
+  delete data_log;
+  if (use_gc_thread) {
+    gc->stop_processor();
+    delete gc;
+    gc = NULL;
+  }
+  delete rest_master_conn;
+
+  map::iterator iter;
+  for (iter = zone_conn_map.begin(); iter != zone_conn_map.end(); ++iter) {
+    RGWRESTConn *conn = iter->second;
+    delete conn;
+  }
+
+  for (iter = region_conn_map.begin(); iter != region_conn_map.end(); ++iter) {
+    RGWRESTConn *conn = iter->second;
+    delete conn;
+  }
+  RGWQuotaHandler::free_handler(quota_handler);
+}
+
+/** 
+ * Initialize the RADOS instance and prepare to do other ops
+ * Returns 0 on success, -ERR# on failure.
+ */
+int RGWRados::init_rados()
+{
+  int ret;
+
+  rados = new Rados();
+  if (!rados)
+    return -ENOMEM;
+
+  ret = rados->init_with_context(cct);
+  if (ret < 0)
+   return ret;
+
+  ret = rados->connect();
+  if (ret < 0)
+   return ret;
+
+  meta_mgr = new RGWMetadataManager(cct, this);
+  data_log = new RGWDataChangesLog(cct, this);
+
+  return ret;
+}
+
+/** 
+ * Initialize the RADOS instance and prepare to do other ops
+ * Returns 0 on success, -ERR# on failure.
+ */
+int RGWRados::init_complete()
+{
+  int ret;
+
+  ret = region.init(cct, this);
+  if (ret < 0)
+    return ret;
+
+  ret = zone.init(cct, this, region);
+  if (ret < 0)
+    return ret;
+
+  ret = region_map.read(cct, this);
+  if (ret < 0) {
+    if (ret != -ENOENT) {
+      ldout(cct, 0) << "WARNING: cannot read region map" << dendl;
+    }
+    ret = region_map.update(region);
+    if (ret < 0) {
+      ldout(cct, 0) << "ERROR: failed to update regionmap with local region info" << dendl;
+      return -EIO;
+    }
+  } else {
+    string master_region = region_map.master_region;
+    if (master_region.empty()) {
+      lderr(cct) << "ERROR: region map does not specify master region" << dendl;
+      return -EINVAL;
+    }
+    map::iterator iter = region_map.regions.find(master_region);
+    if (iter == region_map.regions.end()) {
+      lderr(cct) << "ERROR: bad region map: inconsistent master region" << dendl;
+      return -EINVAL;
+    }
+    RGWRegion& region = iter->second;
+    rest_master_conn = new RGWRESTConn(cct, this, region.endpoints);
+
+    for (iter = region_map.regions.begin(); iter != region_map.regions.end(); ++iter) {
+      RGWRegion& region = iter->second;
+
+      region_conn_map[region.name] = new RGWRESTConn(cct, this, region.endpoints);
+    }
+  }
+
+  if (need_watch_notify()) {
+    ret = init_watch();
+    if (ret < 0) {
+      lderr(cct) << "ERROR: failed to initialize watch" << dendl;
+      return ret;
+    }
+  }
+
+  map::iterator ziter;
+  for (ziter = region.zones.begin(); ziter != region.zones.end(); ++ziter) {
+    const string& name = ziter->first;
+    RGWZone& z = ziter->second;
+    if (name != zone.name) {
+      ldout(cct, 20) << "generating connection object for zone " << name << dendl;
+      zone_conn_map[name] = new RGWRESTConn(cct, this, z.endpoints);
+    } else {
+      zone_public_config = z;
+    }
+  }
+
+  ret = open_root_pool_ctx();
+  if (ret < 0)
+    return ret;
+
+  ret = open_gc_pool_ctx();
+  if (ret < 0)
+    return ret;
+
+  pools_initialized = true;
+
+  gc = new RGWGC();
+  gc->initialize(cct, this);
+
+  if (use_gc_thread)
+    gc->start_processor();
+
+  quota_handler = RGWQuotaHandler::generate_handler(this, quota_threads);
+
+  return ret;
+}
+
+/** 
+ * Initialize the RADOS instance and prepare to do other ops
+ * Returns 0 on success, -ERR# on failure.
+ */
+int RGWRados::initialize()
+{
+  int ret;
+
+  ret = init_rados();
+  if (ret < 0)
+    return ret;
+
+  ret = init_complete();
+
+  return ret;
+}
+
+void RGWRados::finalize_watch()
+{
+  for (int i = 0; i < num_watchers; i++) {
+    string& notify_oid = notify_oids[i];
+    if (notify_oid.empty())
+      continue;
+    uint64_t watch_handle = watch_handles[i];
+    control_pool_ctx.unwatch(notify_oid, watch_handle);
+
+    RGWWatcher *watcher = watchers[i];
+    delete watcher;
+  }
+
+  delete[] notify_oids;
+  delete[] watch_handles;
+  delete[] watchers;
+}
+
+int RGWRados::list_raw_prefixed_objs(string pool_name, const string& prefix, list& result)
+{
+  rgw_bucket pool(pool_name.c_str());
+  bool is_truncated;
+  RGWListRawObjsCtx ctx;
+  do {
+    list oids;
+    int r = list_raw_objects(pool, prefix, 1000,
+			     ctx, oids, &is_truncated);
+    if (r < 0) {
+      return r;
+    }
+    list::iterator iter;
+    for (iter = oids.begin(); iter != oids.end(); ++iter) {
+      string& val = *iter;
+      if (val.size() > prefix.size())
+        result.push_back(val.substr(prefix.size()));
+    }
+  } while (is_truncated);
+
+  return 0;
+}
+
+int RGWRados::list_regions(list& regions)
+{
+  string pool_name;
+  int ret = RGWRegion::get_pool_name(cct, &pool_name);
+  if (ret < 0)
+    return ret;
+
+  return list_raw_prefixed_objs(pool_name, region_info_oid_prefix, regions);
+}
+
+int RGWRados::list_zones(list& zones)
+{
+  string pool_name;
+  int ret = RGWZoneParams::get_pool_name(cct, &pool_name);
+  if (ret < 0)
+    return ret;
+
+  return list_raw_prefixed_objs(pool_name, zone_info_oid_prefix, zones);
+}
+
+/**
+ * Open the pool used as root for this gateway
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+int RGWRados::open_root_pool_ctx()
+{
+  const string& pool = zone.domain_root.name;
+  const char *pool_str = pool.c_str();
+  int r = rados->ioctx_create(pool_str, root_pool_ctx);
+  if (r == -ENOENT) {
+    r = rados->pool_create(pool_str);
+    if (r == -EEXIST)
+      r = 0;
+    if (r < 0)
+      return r;
+
+    r = rados->ioctx_create(pool_str, root_pool_ctx);
+  }
+
+  return r;
+}
+
+int RGWRados::open_gc_pool_ctx()
+{
+  const char *gc_pool = zone.gc_pool.name.c_str();
+  int r = rados->ioctx_create(gc_pool, gc_pool_ctx);
+  if (r == -ENOENT) {
+    r = rados->pool_create(gc_pool);
+    if (r == -EEXIST)
+      r = 0;
+    if (r < 0)
+      return r;
+
+    r = rados->ioctx_create(gc_pool, gc_pool_ctx);
+  }
+
+  return r;
+}
+
+int RGWRados::init_watch()
+{
+  const char *control_pool = zone.control_pool.name.c_str();
+  int r = rados->ioctx_create(control_pool, control_pool_ctx);
+  if (r == -ENOENT) {
+    r = rados->pool_create(control_pool);
+    if (r == -EEXIST)
+      r = 0;
+    if (r < 0)
+      return r;
+
+    r = rados->ioctx_create(control_pool, control_pool_ctx);
+    if (r < 0)
+      return r;
+  }
+
+  num_watchers = cct->_conf->rgw_num_control_oids;
+
+  bool compat_oid = (num_watchers == 0);
+
+  if (num_watchers <= 0)
+    num_watchers = 1;
+
+  notify_oids = new string[num_watchers];
+  watchers = new RGWWatcher *[num_watchers];
+  watch_handles = new uint64_t[num_watchers];
+
+  for (int i=0; i < num_watchers; i++) {
+    string& notify_oid = notify_oids[i];
+    notify_oid = notify_oid_prefix;
+    if (!compat_oid) {
+      char buf[16];
+      snprintf(buf, sizeof(buf), ".%d", i);
+      notify_oid.append(buf);
+    }
+    r = control_pool_ctx.create(notify_oid, false);
+    if (r < 0 && r != -EEXIST)
+      return r;
+
+    RGWWatcher *watcher = new RGWWatcher(this);
+    watchers[i] = watcher;
+
+    r = control_pool_ctx.watch(notify_oid, 0, &watch_handles[i], watcher);
+    if (r < 0)
+      return r;
+  }
+
+  watch_initialized = true;
+
+  return 0;
+}
+
+void RGWRados::pick_control_oid(const string& key, string& notify_oid)
+{
+  uint32_t r = ceph_str_hash_linux(key.c_str(), key.size());
+
+  int i = r % num_watchers;
+  char buf[16];
+  snprintf(buf, sizeof(buf), ".%d", i);
+
+  notify_oid = notify_oid_prefix;
+  notify_oid.append(buf);
+}
+
+int RGWRados::open_bucket_pool_ctx(const string& bucket_name, const string& pool, librados::IoCtx&  io_ctx)
+{
+  int r = rados->ioctx_create(pool.c_str(), io_ctx);
+  if (r != -ENOENT)
+    return r;
+
+  if (!pools_initialized)
+    return r;
+
+  r = rados->pool_create(pool.c_str());
+  if (r < 0 && r != -EEXIST)
+    return r;
+
+  r = rados->ioctx_create(pool.c_str(), io_ctx);
+
+  return r;
+}
+
+int RGWRados::open_bucket_data_ctx(rgw_bucket& bucket, librados::IoCtx& data_ctx)
+{
+  int r = open_bucket_pool_ctx(bucket.name, bucket.data_pool, data_ctx);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::open_bucket_data_extra_ctx(rgw_bucket& bucket, librados::IoCtx& data_ctx)
+{
+  string& pool = (!bucket.data_extra_pool.empty() ? bucket.data_extra_pool : bucket.data_pool);
+  int r = open_bucket_pool_ctx(bucket.name, pool, data_ctx);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx& index_ctx)
+{
+  int r = open_bucket_pool_ctx(bucket.name, bucket.index_pool, index_ctx);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+/**
+ * set up a bucket listing.
+ * handle is filled in.
+ * Returns 0 on success, -ERR# otherwise.
+ */
+int RGWRados::list_buckets_init(RGWAccessHandle *handle)
+{
+  librados::ObjectIterator *state = new librados::ObjectIterator(root_pool_ctx.objects_begin());
+  *handle = (RGWAccessHandle)state;
+  return 0;
+}
+
+/** 
+ * get the next bucket in the listing.
+ * obj is filled in,
+ * handle is updated.
+ * returns 0 on success, -ERR# otherwise.
+ */
+int RGWRados::list_buckets_next(RGWObjEnt& obj, RGWAccessHandle *handle)
+{
+  librados::ObjectIterator *state = (librados::ObjectIterator *)*handle;
+
+  do {
+    if (*state == root_pool_ctx.objects_end()) {
+      delete state;
+      return -ENOENT;
+    }
+
+    obj.name = (*state)->first;
+    (*state)++;
+  } while (obj.name[0] == '.'); /* skip all entries starting with '.' */
+
+  return 0;
+}
+
+
+/**** logs ****/
+
+struct log_list_state {
+  string prefix;
+  librados::IoCtx io_ctx;
+  librados::ObjectIterator obit;
+};
+
+int RGWRados::log_list_init(const string& prefix, RGWAccessHandle *handle)
+{
+  log_list_state *state = new log_list_state;
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, state->io_ctx);
+  if (r < 0) {
+    delete state;
+    return r;
+  }
+  state->prefix = prefix;
+  state->obit = state->io_ctx.objects_begin();
+  *handle = (RGWAccessHandle)state;
+  return 0;
+}
+
+int RGWRados::log_list_next(RGWAccessHandle handle, string *name)
+{
+  log_list_state *state = static_cast(handle);
+  while (true) {
+    if (state->obit == state->io_ctx.objects_end()) {
+      delete state;
+      return -ENOENT;
+    }
+    if (state->prefix.length() &&
+	state->obit->first.find(state->prefix) != 0) {
+      state->obit++;
+      continue;
+    }
+    *name = state->obit->first;
+    state->obit++;
+    break;
+  }
+  return 0;
+}
+
+int RGWRados::log_remove(const string& name)
+{
+  librados::IoCtx io_ctx;
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, io_ctx);
+  if (r < 0)
+    return r;
+  return io_ctx.remove(name);
+}
+
+struct log_show_state {
+  librados::IoCtx io_ctx;
+  bufferlist bl;
+  bufferlist::iterator p;
+  string name;
+  uint64_t pos;
+  bool eof;
+  log_show_state() : pos(0), eof(false) {}
+};
+
+int RGWRados::log_show_init(const string& name, RGWAccessHandle *handle)
+{
+  log_show_state *state = new log_show_state;
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, state->io_ctx);
+  if (r < 0) {
+    delete state;
+    return r;
+  }
+  state->name = name;
+  *handle = (RGWAccessHandle)state;
+  return 0;
+}
+
+int RGWRados::log_show_next(RGWAccessHandle handle, rgw_log_entry *entry)
+{
+  log_show_state *state = static_cast(handle);
+  off_t off = state->p.get_off();
+
+  ldout(cct, 10) << "log_show_next pos " << state->pos << " bl " << state->bl.length()
+	   << " off " << off
+	   << " eof " << (int)state->eof
+	   << dendl;
+  // read some?
+  unsigned chunk = 1024*1024;
+  if ((state->bl.length() - off) < chunk/2 && !state->eof) {
+    bufferlist more;
+    int r = state->io_ctx.read(state->name, more, chunk, state->pos);
+    if (r < 0)
+      return r;
+    state->pos += r;
+    bufferlist old;
+    try {
+      old.substr_of(state->bl, off, state->bl.length() - off);
+    } catch (buffer::error& err) {
+      return -EINVAL;
+    }
+    state->bl.clear();
+    state->bl.claim(old);
+    state->bl.claim_append(more);
+    state->p = state->bl.begin();
+    if ((unsigned)r < chunk)
+      state->eof = true;
+    ldout(cct, 10) << " read " << r << dendl;
+  }
+
+  if (state->p.end())
+    return 0;  // end of file
+  try {
+    ::decode(*entry, state->p);
+  }
+  catch (const buffer::error &e) {
+    return -EINVAL;
+  }
+  return 1;
+}
+
+/**
+ * usage_log_hash: get usage log key hash, based on name and index
+ *
+ * Get the usage object name. Since a user may have more than 1
+ * object holding that info (multiple shards), we use index to
+ * specify that shard number. Once index exceeds max shards it
+ * wraps.
+ * If name is not being set, results for all users will be returned
+ * and index will wrap only after total shards number.
+ *
+ * @param cct [in] ceph context
+ * @param name [in] user name
+ * @param hash [out] hash value
+ * @param index [in] shard index number 
+ */
+static void usage_log_hash(CephContext *cct, const string& name, string& hash, uint32_t index)
+{
+  uint32_t val = index;
+
+  if (!name.empty()) {
+    int max_user_shards = max(cct->_conf->rgw_usage_max_user_shards, 1);
+    val %= max_user_shards;
+    val += ceph_str_hash_linux(name.c_str(), name.size());
+  }
+  char buf[16];
+  int max_shards = max(cct->_conf->rgw_usage_max_shards, 1);
+  snprintf(buf, sizeof(buf), RGW_USAGE_OBJ_PREFIX "%u", (unsigned)(val % max_shards));
+  hash = buf;
+}
+
+int RGWRados::log_usage(map& usage_info)
+{
+  uint32_t index = 0;
+
+  map log_objs;
+
+  string hash;
+  string last_user;
+
+  /* restructure usage map, zone by object hash */
+  map::iterator iter;
+  for (iter = usage_info.begin(); iter != usage_info.end(); ++iter) {
+    const rgw_user_bucket& ub = iter->first;
+    RGWUsageBatch& info = iter->second;
+
+    if (ub.user.empty()) {
+      ldout(cct, 0) << "WARNING: RGWRados::log_usage(): user name empty (bucket=" << ub.bucket << "), skipping" << dendl;
+      continue;
+    }
+
+    if (ub.user != last_user) {
+      /* index *should* be random, but why waste extra cycles
+         in most cases max user shards is not going to exceed 1,
+         so just incrementing it */
+      usage_log_hash(cct, ub.user, hash, index++);
+    }
+    last_user = ub.user;
+    vector& v = log_objs[hash].entries;
+
+    map::iterator miter;
+    for (miter = info.m.begin(); miter != info.m.end(); ++miter) {
+      v.push_back(miter->second);
+    }
+  }
+
+  map::iterator liter;
+
+  for (liter = log_objs.begin(); liter != log_objs.end(); ++liter) {
+    int r = cls_obj_usage_log_add(liter->first, liter->second);
+    if (r < 0)
+      return r;
+  }
+  return 0;
+}
+
+int RGWRados::read_usage(string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
+                         bool *is_truncated, RGWUsageIter& usage_iter, map& usage)
+{
+  uint32_t num = max_entries;
+  string hash, first_hash;
+  usage_log_hash(cct, user, first_hash, 0);
+
+  if (usage_iter.index) {
+    usage_log_hash(cct, user, hash, usage_iter.index);
+  } else {
+    hash = first_hash;
+  }
+
+  usage.clear();
+
+  do {
+    map ret_usage;
+    map::iterator iter;
+
+    int ret =  cls_obj_usage_log_read(hash, user, start_epoch, end_epoch, num,
+                                    usage_iter.read_iter, ret_usage, is_truncated);
+    if (ret == -ENOENT)
+      goto next;
+
+    if (ret < 0)
+      return ret;
+
+    num -= ret_usage.size();
+
+    for (iter = ret_usage.begin(); iter != ret_usage.end(); ++iter) {
+      usage[iter->first].aggregate(iter->second);
+    }
+
+next:
+    if (!*is_truncated) {
+      usage_iter.read_iter.clear();
+      usage_log_hash(cct, user, hash, ++usage_iter.index);
+    }
+  } while (num && !*is_truncated && hash != first_hash);
+  return 0;
+}
+
+int RGWRados::trim_usage(string& user, uint64_t start_epoch, uint64_t end_epoch)
+{
+  uint32_t index = 0;
+  string hash, first_hash;
+  usage_log_hash(cct, user, first_hash, index);
+
+  hash = first_hash;
+
+  do {
+    int ret =  cls_obj_usage_log_trim(hash, user, start_epoch, end_epoch);
+    if (ret == -ENOENT)
+      goto next;
+
+    if (ret < 0)
+      return ret;
+
+next:
+    usage_log_hash(cct, user, hash, ++index);
+  } while (hash != first_hash);
+
+  return 0;
+}
+
+void RGWRados::shard_name(const string& prefix, unsigned max_shards, const string& key, string& name)
+{
+  uint32_t val = ceph_str_hash_linux(key.c_str(), key.size());
+  char buf[16];
+  snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards));
+  name = prefix + buf;
+}
+
+void RGWRados::shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name)
+{
+  uint32_t val = ceph_str_hash_linux(key.c_str(), key.size());
+  val ^= ceph_str_hash_linux(section.c_str(), section.size());
+  char buf[16];
+  snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards));
+  name = prefix + buf;
+}
+
+void RGWRados::time_log_prepare_entry(cls_log_entry& entry, const utime_t& ut, string& section, string& key, bufferlist& bl)
+{
+  cls_log_add_prepare_entry(entry, ut, section, key, bl);
+}
+
+int RGWRados::time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl)
+{
+  librados::IoCtx io_ctx;
+
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, io_ctx);
+  if (r == -ENOENT) {
+    rgw_bucket pool(log_pool);
+    r = create_pool(pool);
+    if (r < 0)
+      return r;
+ 
+    // retry
+    r = rados->ioctx_create(log_pool, io_ctx);
+  }
+  if (r < 0)
+    return r;
+
+  ObjectWriteOperation op;
+  cls_log_add(op, ut, section, key, bl);
+
+  r = io_ctx.operate(oid, &op);
+  return r;
+}
+
+int RGWRados::time_log_add(const string& oid, list& entries)
+{
+  librados::IoCtx io_ctx;
+
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, io_ctx);
+  if (r == -ENOENT) {
+    rgw_bucket pool(log_pool);
+    r = create_pool(pool);
+    if (r < 0)
+      return r;
+ 
+    // retry
+    r = rados->ioctx_create(log_pool, io_ctx);
+  }
+  if (r < 0)
+    return r;
+
+  ObjectWriteOperation op;
+  cls_log_add(op, entries);
+
+  r = io_ctx.operate(oid, &op);
+  return r;
+}
+
+int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
+                            int max_entries, list& entries,
+			    const string& marker,
+			    string *out_marker,
+			    bool *truncated)
+{
+  librados::IoCtx io_ctx;
+
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, io_ctx);
+  if (r < 0)
+    return r;
+  librados::ObjectReadOperation op;
+
+  cls_log_list(op, start_time, end_time, marker, max_entries, entries,
+	       out_marker, truncated);
+
+  bufferlist obl;
+
+  int ret = io_ctx.operate(oid, &op, &obl);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWRados::time_log_info(const string& oid, cls_log_header *header)
+{
+  librados::IoCtx io_ctx;
+
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, io_ctx);
+  if (r < 0)
+    return r;
+  librados::ObjectReadOperation op;
+
+  cls_log_info(op, header);
+
+  bufferlist obl;
+
+  int ret = io_ctx.operate(oid, &op, &obl);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWRados::time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time,
+			    const string& from_marker, const string& to_marker)
+{
+  librados::IoCtx io_ctx;
+
+  const char *log_pool = zone.log_pool.name.c_str();
+  int r = rados->ioctx_create(log_pool, io_ctx);
+  if (r < 0)
+    return r;
+
+  return cls_log_trim(io_ctx, oid, start_time, end_time, from_marker, to_marker);
+}
+
+
+int RGWRados::lock_exclusive(rgw_bucket& pool, const string& oid, utime_t& duration, 
+                             string& zone_id, string& owner_id) {
+  librados::IoCtx io_ctx;
+
+  const char *pool_name = pool.name.c_str();
+  
+  int r = rados->ioctx_create(pool_name, io_ctx);
+  if (r < 0)
+    return r;
+  
+  rados::cls::lock::Lock l(log_lock_name);
+  l.set_duration(duration);
+  l.set_cookie(owner_id);
+  l.set_tag(zone_id);
+  l.set_renew(true);
+  
+  return l.lock_exclusive(&io_ctx, oid);
+}
+
+int RGWRados::unlock(rgw_bucket& pool, const string& oid, string& zone_id, string& owner_id) {
+  librados::IoCtx io_ctx;
+
+  const char *pool_name = pool.name.c_str();
+
+  int r = rados->ioctx_create(pool_name, io_ctx);
+  if (r < 0)
+    return r;
+  
+  rados::cls::lock::Lock l(log_lock_name);
+  l.set_tag(zone_id);
+  l.set_cookie(owner_id);
+  
+  return l.unlock(&io_ctx, oid);
+}
+
+int RGWRados::decode_policy(bufferlist& bl, ACLOwner *owner)
+{
+  bufferlist::iterator i = bl.begin();
+  RGWAccessControlPolicy policy(cct);
+  try {
+    policy.decode_owner(i);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+    return -EIO;
+  }
+  *owner = policy.get_owner();
+  return 0;
+}
+
+int rgw_policy_from_attrset(CephContext *cct, map& attrset, RGWAccessControlPolicy *policy)
+{
+  map::iterator aiter = attrset.find(RGW_ATTR_ACL);
+  if (aiter == attrset.end())
+    return -EIO;
+
+  bufferlist& bl = aiter->second;
+  bufferlist::iterator iter = bl.begin();
+  try {
+    policy->decode(iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+    return -EIO;
+  }
+  if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
+    RGWAccessControlPolicy_S3 *s3policy = static_cast(policy);
+    ldout(cct, 15) << "Read AccessControlPolicy";
+    s3policy->to_xml(*_dout);
+    *_dout << dendl;
+  }
+  return 0;
+}
+
+/** 
+ * get listing of the objects in a bucket.
+ * bucket: bucket to list contents of
+ * max: maximum number of results to return
+ * prefix: only return results that match this prefix
+ * delim: do not include results that match this string.
+ *     Any skipped results will have the matching portion of their name
+ *     inserted in common_prefixes with a "true" mark.
+ * marker: if filled in, begin the listing with this object.
+ * result: the objects are put in here.
+ * common_prefixes: if delim is filled in, any matching prefixes are placed
+ *     here.
+ */
+int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string& delim,
+			   string& marker, string *next_marker, vector& result,
+                           map& common_prefixes,
+			   bool get_content_type, string& ns, bool enforce_ns,
+                           bool *is_truncated, RGWAccessListFilter *filter)
+{
+  int count = 0;
+  bool truncated = true;
+
+  if (bucket_is_system(bucket)) {
+    return -EINVAL;
+  }
+  result.clear();
+
+  rgw_obj marker_obj, prefix_obj;
+  marker_obj.set_ns(ns);
+  marker_obj.set_obj(marker);
+  string cur_marker = marker_obj.object;
+
+  prefix_obj.set_ns(ns);
+  prefix_obj.set_obj(prefix);
+  string cur_prefix = prefix_obj.object;
+
+  string bigger_than_delim;
+
+  if (!delim.empty()) {
+    unsigned long val = decode_utf8((unsigned char *)delim.c_str(), delim.size());
+    char buf[delim.size() + 16];
+    int r = encode_utf8(val + 1, (unsigned char *)buf);
+    if (r < 0) {
+      ldout(cct,0) << "ERROR: encode_utf8() failed" << dendl;
+      return -EINVAL;
+    }
+    buf[r] = '\0';
+
+    bigger_than_delim = buf;
+  }
+
+  string skip_after_delim;
+
+  /* if marker points at a common prefix, fast forward it into its upperbound string */
+  if (!delim.empty()) {
+    int delim_pos = cur_marker.find(delim, prefix.size());
+    if (delim_pos >= 0) {
+      cur_marker = cur_marker.substr(0, delim_pos);
+      cur_marker.append(bigger_than_delim);
+    }
+  }
+
+  while (truncated && count <= max) {
+    if (skip_after_delim > cur_marker) {
+      cur_marker = skip_after_delim;
+      ldout(cct, 20) << "setting cur_marker=" << cur_marker << dendl;
+    }
+    std::map ent_map;
+    int r = cls_bucket_list(bucket, cur_marker, cur_prefix, max + 1 - count, ent_map,
+                            &truncated, &cur_marker);
+    if (r < 0)
+      return r;
+
+    std::map::iterator eiter;
+    for (eiter = ent_map.begin(); eiter != ent_map.end(); ++eiter) {
+      string obj = eiter->first;
+      string key = obj;
+
+      bool check_ns = rgw_obj::translate_raw_obj_to_obj_in_ns(obj, ns);
+
+      if (enforce_ns && !check_ns) {
+        if (!ns.empty()) {
+          /* we've iterated past the namespace we're searching -- done now */
+          truncated = false;
+          goto done;
+        }
+
+        /* we're not looking at the namespace this object is in, next! */
+        continue;
+      }
+
+      if (next_marker && count < max) {
+        *next_marker = obj;
+      }
+
+      if (filter && !filter->filter(obj, key))
+        continue;
+
+      if (prefix.size() &&  ((obj).compare(0, prefix.size(), prefix) != 0))
+        continue;
+
+      if (!delim.empty()) {
+        int delim_pos = obj.find(delim, prefix.size());
+
+        if (delim_pos >= 0) {
+          string prefix_key = obj.substr(0, delim_pos + 1);
+
+          if (common_prefixes.find(prefix_key) == common_prefixes.end()) {
+            if (count >= max) {
+              truncated = true;
+              goto done;
+            }
+            if (next_marker) {
+              *next_marker = prefix_key;
+            }
+            common_prefixes[prefix_key] = true;
+
+            skip_after_delim = obj.substr(0, delim_pos);
+            skip_after_delim.append(bigger_than_delim);
+
+            ldout(cct, 20) << "skip_after_delim=" << skip_after_delim << dendl;
+
+            count++;
+          }
+
+          continue;
+        }
+      }
+
+      if (count >= max) {
+        truncated = true;
+        goto done;
+      }
+
+      RGWObjEnt ent = eiter->second;
+      ent.name = obj;
+      ent.ns = ns;
+      result.push_back(ent);
+      count++;
+    }
+  }
+
+done:
+  if (is_truncated)
+    *is_truncated = truncated;
+
+  return 0;
+}
+
+/**
+ * create a rados pool, associated meta info
+ * returns 0 on success, -ERR# otherwise.
+ */
+int RGWRados::create_pool(rgw_bucket& bucket) 
+{
+  int ret = 0;
+
+  string pool = bucket.index_pool;
+
+  ret = rados->pool_create(pool.c_str(), 0);
+  if (ret == -EEXIST)
+    ret = 0;
+  if (ret < 0)
+    return ret;
+
+  if (bucket.data_pool != pool) {
+    ret = rados->pool_create(bucket.data_pool.c_str(), 0);
+    if (ret == -EEXIST)
+      ret = 0;
+    if (ret < 0)
+      return ret;
+  }
+
+  return 0;
+}
+
+int RGWRados::init_bucket_index(rgw_bucket& bucket)
+{
+  librados::IoCtx index_ctx; // context for new bucket
+
+  int r = open_bucket_index_ctx(bucket, index_ctx);
+  if (r < 0)
+    return r;
+
+  string dir_oid =  dir_oid_prefix;
+  dir_oid.append(bucket.marker);
+
+  librados::ObjectWriteOperation op;
+  op.create(true);
+  r = cls_rgw_init_index(index_ctx, op, dir_oid);
+  if (r < 0 && r != -EEXIST)
+    return r;
+
+  return 0;
+}
+
+/**
+ * create a bucket with name bucket and the given list of attrs
+ * returns 0 on success, -ERR# otherwise.
+ */
+int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
+                            const string& region_name,
+                            const string& placement_rule,
+			    map& attrs,
+                            RGWBucketInfo& info,
+                            obj_version *pobjv,
+                            obj_version *pep_objv,
+                            time_t creation_time,
+                            rgw_bucket *pmaster_bucket,
+			    bool exclusive)
+{
+#define MAX_CREATE_RETRIES 20 /* need to bound retries */
+  string selected_placement_rule;
+  for (int i = 0; i < MAX_CREATE_RETRIES; i++) {
+    int ret = 0;
+    ret = select_bucket_placement(owner, region_name, placement_rule, bucket.name, bucket, &selected_placement_rule);
+    if (ret < 0)
+      return ret;
+    bufferlist bl;
+    uint32_t nop = 0;
+    ::encode(nop, bl);
+
+    const string& pool = zone.domain_root.name;
+    const char *pool_str = pool.c_str();
+    librados::IoCtx id_io_ctx;
+    int r = rados->ioctx_create(pool_str, id_io_ctx);
+    if (r < 0)
+      return r;
+
+    if (!pmaster_bucket) {
+      uint64_t iid = instance_id();
+      uint64_t bid = next_bucket_id();
+      char buf[zone.name.size() + 48];
+      snprintf(buf, sizeof(buf), "%s.%llu.%llu", zone.name.c_str(), (long long)iid, (long long)bid);
+      bucket.marker = buf;
+      bucket.bucket_id = bucket.marker;
+    } else {
+      bucket.marker = pmaster_bucket->marker;
+      bucket.bucket_id = pmaster_bucket->bucket_id;
+    }
+
+    string dir_oid =  dir_oid_prefix;
+    dir_oid.append(bucket.marker);
+
+    r = init_bucket_index(bucket);
+    if (r < 0)
+      return r;
+
+    RGWObjVersionTracker& objv_tracker = info.objv_tracker;
+
+    if (pobjv) {
+      objv_tracker.write_version = *pobjv;
+    } else {
+      objv_tracker.generate_new_write_ver(cct);
+    }
+
+    info.bucket = bucket;
+    info.owner = owner.user_id;
+    info.region = region_name;
+    info.placement_rule = selected_placement_rule;
+    if (!creation_time)
+      time(&info.creation_time);
+    else
+      info.creation_time = creation_time;
+    ret = put_linked_bucket_info(info, exclusive, 0, pep_objv, &attrs, true);
+    if (ret == -EEXIST) {
+       /* we need to reread the info and return it, caller will have a use for it */
+      info.objv_tracker.clear();
+      r = get_bucket_info(NULL, bucket.name, info, NULL, NULL);
+      if (r < 0) {
+        if (r == -ENOENT) {
+          continue;
+        }
+        ldout(cct, 0) << "get_bucket_info returned " << r << dendl;
+        return r;
+      }
+
+      /* only remove it if it's a different bucket instance */
+      if (info.bucket.bucket_id != bucket.bucket_id) {
+        /* remove bucket meta instance */
+        string entry;
+        get_bucket_instance_entry(bucket, entry);
+        r = rgw_bucket_instance_remove_entry(this, entry, &info.objv_tracker);
+        if (r < 0)
+          return r;
+
+        /* remove bucket index */
+        librados::IoCtx index_ctx; // context for new bucket
+        int r = open_bucket_index_ctx(bucket, index_ctx);
+        if (r < 0)
+          return r;
+
+        index_ctx.remove(dir_oid);
+      }
+      /* ret == -ENOENT here */
+    }
+    return ret;
+  }
+
+  /* this is highly unlikely */
+  ldout(cct, 0) << "ERROR: could not create bucket, continuously raced with bucket creation and removal" << dendl;
+  return -ENOENT;
+}
+
+int RGWRados::select_new_bucket_location(RGWUserInfo& user_info, const string& region_name, const string& request_rule,
+                                         const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
+{
+  /* first check that rule exists within the specific region */
+  map::iterator riter = region_map.regions.find(region_name);
+  if (riter == region_map.regions.end()) {
+    ldout(cct, 0) << "could not find region " << region_name << " in region map" << dendl;
+    return -EINVAL;
+  }
+  /* now check that tag exists within region */
+  RGWRegion& region = riter->second;
+
+  /* find placement rule. Hierarchy: request rule > user default rule > region default rule */
+  string rule = request_rule;
+  if (rule.empty()) {
+    rule = user_info.default_placement;
+    if (rule.empty())
+      rule = region.default_placement;
+  }
+
+  if (rule.empty()) {
+    ldout(cct, 0) << "misconfiguration, should not have an empty placement rule name" << dendl;
+    return -EIO;
+  }
+
+  if (!rule.empty()) {
+    map::iterator titer = region.placement_targets.find(rule);
+    if (titer == region.placement_targets.end()) {
+      ldout(cct, 0) << "could not find placement rule " << rule << " within region " << dendl;
+      return -EINVAL;
+    }
+
+    /* now check tag for the rule, whether user is permitted to use rule */
+    RGWRegionPlacementTarget& target_rule = titer->second;
+    if (!target_rule.user_permitted(user_info.placement_tags)) {
+      ldout(cct, 0) << "user not permitted to use placement rule" << dendl;
+      return -EPERM;
+    }
+  }
+
+  if (pselected_rule)
+    *pselected_rule = rule;
+  
+  return set_bucket_location_by_rule(rule, bucket_name, bucket);
+}
+
+int RGWRados::set_bucket_location_by_rule(const string& location_rule, const std::string& bucket_name, rgw_bucket& bucket)
+{
+  bucket.name = bucket_name;
+
+  if (location_rule.empty()) {
+    /* we can only reach here if we're trying to set a bucket location from a bucket
+     * created on a different zone, using a legacy / default pool configuration
+     */
+    return select_legacy_bucket_placement(bucket_name, bucket);
+  }
+
+  /*
+   * make sure that zone has this rule configured. We're
+   * checking it for the local zone, because that's where this bucket object is going to
+   * reside.
+   */
+  map::iterator piter = zone.placement_pools.find(location_rule);
+  if (piter == zone.placement_pools.end()) {
+    /* couldn't find, means we cannot really place data for this bucket in this zone */
+    if (region.equals(region_name)) {
+      /* that's a configuration error, zone should have that rule, as we're within the requested
+       * region */
+      return -EINVAL;
+    } else {
+      /* oh, well, data is not going to be placed here, bucket object is just a placeholder */
+      return 0;
+    }
+  }
+
+  RGWZonePlacementInfo& placement_info = piter->second;
+
+  bucket.data_pool = placement_info.data_pool;
+  bucket.data_extra_pool = placement_info.data_extra_pool;
+  bucket.index_pool = placement_info.index_pool;
+
+  return 0;
+
+}
+
+int RGWRados::select_bucket_placement(RGWUserInfo& user_info, const string& region_name, const string& placement_rule,
+                                      const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
+{
+  if (!zone.placement_pools.empty()) {
+    return select_new_bucket_location(user_info, region_name, placement_rule, bucket_name, bucket, pselected_rule);
+  }
+
+  if (pselected_rule)
+    pselected_rule->clear();
+
+  return select_legacy_bucket_placement(bucket_name, bucket);
+}
+
+int RGWRados::select_legacy_bucket_placement(const string& bucket_name, rgw_bucket& bucket)
+{
+  bufferlist map_bl;
+  map m;
+  string pool_name;
+  bool write_map = false;
+
+  rgw_obj obj(zone.domain_root, avail_pools);
+
+  int ret = rgw_get_system_obj(this, NULL, zone.domain_root, avail_pools, map_bl, NULL, NULL);
+  if (ret < 0) {
+    goto read_omap;
+  }
+
+  try {
+    bufferlist::iterator iter = map_bl.begin();
+    ::decode(m, iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: couldn't decode avail_pools" << dendl;
+  }
+
+read_omap:
+  if (m.empty()) {
+    bufferlist header;
+    ret = omap_get_all(obj, header, m);
+
+    write_map = true;
+  }
+
+  if (ret < 0 || m.empty()) {
+    vector names;
+    names.push_back(default_storage_pool);
+    vector retcodes;
+    bufferlist bl;
+    ret = create_pools(names, retcodes);
+    if (ret < 0)
+      return ret;
+    ret = omap_set(obj, default_storage_pool, bl);
+    if (ret < 0)
+      return ret;
+    m[default_storage_pool] = bl;
+  }
+
+  if (write_map) {
+    bufferlist new_bl;
+    ::encode(m, new_bl);
+    ret = put_obj_data(NULL, obj, new_bl.c_str(), -1, new_bl.length(), false);
+    if (ret < 0) {
+      ldout(cct, 0) << "WARNING: could not save avail pools map info ret=" << ret << dendl;
+    }
+  }
+
+  map::iterator miter;
+  if (m.size() > 1) {
+    vector v;
+    for (miter = m.begin(); miter != m.end(); ++miter) {
+      v.push_back(miter->first);
+    }
+
+    uint32_t r;
+    ret = get_random_bytes((char *)&r, sizeof(r));
+    if (ret < 0)
+      return ret;
+
+    int i = r % v.size();
+    pool_name = v[i];
+  } else {
+    miter = m.begin();
+    pool_name = miter->first;
+  }
+  bucket.data_pool = pool_name;
+  bucket.index_pool = pool_name;
+  bucket.name = bucket_name;
+
+  return 0;
+
+}
+
+int RGWRados::update_placement_map()
+{
+  bufferlist header;
+  map m;
+  rgw_obj obj(zone.domain_root, avail_pools);
+  int ret = omap_get_all(obj, header, m);
+  if (ret < 0)
+    return ret;
+
+  bufferlist new_bl;
+  ::encode(m, new_bl);
+  ret = put_obj_data(NULL, obj, new_bl.c_str(), -1, new_bl.length(), false);
+  if (ret < 0) {
+    ldout(cct, 0) << "WARNING: could not save avail pools map info ret=" << ret << dendl;
+  }
+
+  return ret;
+}
+
+int RGWRados::add_bucket_placement(std::string& new_pool)
+{
+  int ret = rados->pool_lookup(new_pool.c_str());
+  if (ret < 0) // DNE, or something
+    return ret;
+
+  rgw_obj obj(zone.domain_root, avail_pools);
+  bufferlist empty_bl;
+  ret = omap_set(obj, new_pool, empty_bl);
+
+  // don't care about return value
+  update_placement_map();
+
+  return ret;
+}
+
+int RGWRados::remove_bucket_placement(std::string& old_pool)
+{
+  rgw_obj obj(zone.domain_root, avail_pools);
+  int ret = omap_del(obj, old_pool);
+
+  // don't care about return value
+  update_placement_map();
+
+  return ret;
+}
+
+int RGWRados::list_placement_set(set& names)
+{
+  bufferlist header;
+  map m;
+
+  rgw_obj obj(zone.domain_root, avail_pools);
+  int ret = omap_get_all(obj, header, m);
+  if (ret < 0)
+    return ret;
+
+  names.clear();
+  map::iterator miter;
+  for (miter = m.begin(); miter != m.end(); ++miter) {
+    names.insert(miter->first);
+  }
+
+  return names.size();
+}
+
+int RGWRados::create_pools(vector& names, vector& retcodes)
+{
+  vector::iterator iter;
+  vector completions;
+  vector rets;
+
+  for (iter = names.begin(); iter != names.end(); ++iter) {
+    librados::PoolAsyncCompletion *c = librados::Rados::pool_async_create_completion();
+    completions.push_back(c);
+    string& name = *iter;
+    int ret = rados->pool_create_async(name.c_str(), c);
+    rets.push_back(ret);
+  }
+
+  vector::iterator riter;
+  vector::iterator citer;
+
+  assert(rets.size() == completions.size());
+  for (riter = rets.begin(), citer = completions.begin(); riter != rets.end(); ++riter, ++citer) {
+    int r = *riter;
+    PoolAsyncCompletion *c = *citer;
+    if (r == 0) {
+      c->wait();
+      r = c->get_return_value();
+      if (r < 0) {
+        ldout(cct, 0) << "WARNING: async pool_create returned " << r << dendl;
+      }
+    }
+    c->release();
+    retcodes.push_back(r);
+  }
+  return 0;
+}
+
+
+int RGWRados::get_obj_ioctx(const rgw_obj& obj, librados::IoCtx *ioctx)
+{
+  rgw_bucket bucket;
+  string oid, key;
+  get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+
+  int r;
+
+  if (!obj.is_in_extra_data()) {
+    r = open_bucket_data_ctx(bucket, *ioctx);
+  } else {
+    r = open_bucket_data_extra_ctx(bucket, *ioctx);
+  }
+  if (r < 0)
+    return r;
+
+  ioctx->locator_set_key(key);
+
+  return 0;
+}
+
+int RGWRados::get_obj_ref(const rgw_obj& obj, rgw_rados_ref *ref, rgw_bucket *bucket, bool ref_system_obj)
+{
+  get_obj_bucket_and_oid_key(obj, *bucket, ref->oid, ref->key);
+
+  int r;
+
+  if (ref_system_obj && ref->oid.empty()) {
+    ref->oid = bucket->name;
+    *bucket = zone.domain_root;
+
+    r = open_bucket_data_ctx(*bucket, ref->ioctx);
+  } else if (!obj.is_in_extra_data()) {
+    r = open_bucket_data_ctx(*bucket, ref->ioctx);
+  } else {
+    r = open_bucket_data_extra_ctx(*bucket, ref->ioctx);
+  }
+  if (r < 0)
+    return r;
+
+  ref->ioctx.locator_set_key(ref->key);
+
+  return 0;
+}
+
+/**
+ * Write/overwrite an object to the bucket storage.
+ * bucket: the bucket to store the object in
+ * obj: the object name/key
+ * data: the object contents/value
+ * size: the amount of data to write (data must be this long)
+ * mtime: if non-NULL, writes the given mtime to the bucket storage
+ * attrs: all the given attrs are written to bucket storage for the given object
+ * exclusive: create object exclusively
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+int RGWRados::put_obj_meta_impl(void *ctx, rgw_obj& obj,  uint64_t size,
+                  time_t *mtime, map& attrs,
+                  RGWObjCategory category, int flags,
+                  map* rmattrs,
+                  const bufferlist *data,
+                  RGWObjManifest *manifest,
+		  const string *ptag,
+                  list *remove_objs,
+                  bool modify_version,
+                  RGWObjVersionTracker *objv_tracker,
+                  time_t set_mtime,
+                  const string& bucket_owner)
+{
+  rgw_bucket bucket;
+  rgw_rados_ref ref;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0)
+    return r;
+
+  RGWRadosCtx *rctx = static_cast(ctx);
+
+  ObjectWriteOperation op;
+
+  RGWObjState *state = NULL;
+
+  if (flags & PUT_OBJ_EXCL) {
+    if (!(flags & PUT_OBJ_CREATE))
+	return -EINVAL;
+    op.create(true); // exclusive create
+  } else {
+    bool reset_obj = (flags & PUT_OBJ_CREATE) != 0;
+    r = prepare_atomic_for_write(rctx, obj, op, &state, reset_obj, ptag);
+    if (r < 0)
+      return r;
+  }
+
+  if (objv_tracker) {
+    objv_tracker->prepare_op_for_write(&op);
+  }
+
+  utime_t ut;
+  if (set_mtime) {
+    ut = utime_t(set_mtime, 0);
+  } else {
+    ut = ceph_clock_now(0);
+    set_mtime = ut.sec();
+  }
+
+  op.mtime(&set_mtime);
+
+  if (data) {
+    /* if we want to overwrite the data, we also want to overwrite the
+       xattrs, so just remove the object */
+    op.write_full(*data);
+  }
+
+  string etag;
+  string content_type;
+  bufferlist acl_bl;
+
+  map::iterator iter;
+  if (rmattrs) {
+    for (iter = rmattrs->begin(); iter != rmattrs->end(); ++iter) {
+      const string& name = iter->first;
+      op.rmxattr(name.c_str());
+    }
+  }
+
+  if (manifest) {
+    /* remove existing manifest attr */
+    iter = attrs.find(RGW_ATTR_MANIFEST);
+    if (iter != attrs.end())
+      attrs.erase(iter);
+
+    bufferlist bl;
+    ::encode(*manifest, bl);
+    op.setxattr(RGW_ATTR_MANIFEST, bl);
+  }
+
+  for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+    const string& name = iter->first;
+    bufferlist& bl = iter->second;
+
+    if (!bl.length())
+      continue;
+
+    op.setxattr(name.c_str(), bl);
+
+    if (name.compare(RGW_ATTR_ETAG) == 0) {
+      etag = bl.c_str();
+    } else if (name.compare(RGW_ATTR_CONTENT_TYPE) == 0) {
+      content_type = bl.c_str();
+    } else if (name.compare(RGW_ATTR_ACL) == 0) {
+      acl_bl = bl;
+    }
+  }
+
+  if (!op.size())
+    return 0;
+
+  string index_tag;
+  uint64_t epoch;
+  int64_t poolid;
+
+  if (state) {
+    index_tag = state->write_tag;
+  }
+
+  r = prepare_update_index(NULL, bucket, CLS_RGW_OP_ADD, obj, index_tag);
+  if (r < 0)
+    return r;
+
+  r = ref.ioctx.operate(ref.oid, &op);
+  if (r < 0) /* we can expect to get -ECANCELED if object was replaced under,
+                or -ENOENT if was removed, or -EEXIST if it did not exist
+                before and now it does */
+    goto done_cancel;
+
+  if (objv_tracker) {
+    objv_tracker->apply_write();
+  }
+
+  epoch = ref.ioctx.get_last_version();
+  poolid = ref.ioctx.get_id();
+
+  r = complete_atomic_overwrite(rctx, state, obj);
+  if (r < 0) {
+    ldout(cct, 0) << "ERROR: complete_atomic_overwrite returned r=" << r << dendl;
+  }
+
+  r = complete_update_index(bucket, obj.object, index_tag, poolid, epoch, size,
+                            ut, etag, content_type, &acl_bl, category, remove_objs);
+  if (r < 0)
+    goto done_cancel;
+
+  if (mtime) {
+    *mtime = set_mtime;
+  }
+
+  if (state) {
+    /* update quota cache */
+    quota_handler->update_stats(bucket_owner, bucket, (state->exists ? 0 : 1), size, state->size);
+  }
+
+  return 0;
+
+done_cancel:
+  int ret = complete_update_index_cancel(bucket, obj.object, index_tag);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: complete_update_index_cancel() returned ret=" << ret << dendl;
+  }
+  /* we lost in a race. There are a few options:
+   * - existing object was rewritten (ECANCELED)
+   * - non existing object was created (EEXIST)
+   * - object was removed (ENOENT)
+   * should treat it as a success
+   */
+  if ((r == -ECANCELED || r == -ENOENT) ||
+      (!(flags & PUT_OBJ_EXCL) && r == -EEXIST)) {
+    r = 0;
+  }
+
+  return r;
+}
+
+/**
+ * Write/overwrite an object to the bucket storage.
+ * bucket: the bucket to store the object in
+ * obj: the object name/key
+ * data: the object contents/value
+ * offset: the offet to write to in the object
+ *         If this is -1, we will overwrite the whole object.
+ * size: the amount of data to write (data must be this long)
+ * attrs: all the given attrs are written to bucket storage for the given object
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+int RGWRados::put_obj_data(void *ctx, rgw_obj& obj,
+			   const char *data, off_t ofs, size_t len, bool exclusive)
+{
+  void *handle;
+  bufferlist bl;
+  bl.append(data, len);
+  int r = aio_put_obj_data(ctx, obj, bl, ofs, exclusive, &handle);
+  if (r < 0)
+    return r;
+  return aio_wait(handle);
+}
+
+int RGWRados::aio_put_obj_data(void *ctx, rgw_obj& obj, bufferlist& bl,
+			       off_t ofs, bool exclusive,
+                               void **handle)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
+  *handle = c;
+  
+  ObjectWriteOperation op;
+
+  if (exclusive)
+    op.create(true);
+
+  if (ofs == -1) {
+    op.write_full(bl);
+  } else {
+    op.write(ofs, bl);
+  }
+  r = ref.ioctx.aio_operate(ref.oid, c, &op);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::aio_wait(void *handle)
+{
+  AioCompletion *c = (AioCompletion *)handle;
+  c->wait_for_complete();
+  int ret = c->get_return_value();
+  c->release();
+  return ret;
+}
+
+bool RGWRados::aio_completed(void *handle)
+{
+  AioCompletion *c = (AioCompletion *)handle;
+  return c->is_complete();
+}
+
+class RGWRadosPutObj : public RGWGetDataCB
+{
+  rgw_obj obj;
+  RGWPutObjProcessor_Atomic *processor;
+  RGWOpStateSingleOp *opstate;
+  void (*progress_cb)(off_t, void *);
+  void *progress_data;
+public:
+  RGWRadosPutObj(RGWPutObjProcessor_Atomic *p, RGWOpStateSingleOp *_ops,
+                 void (*_progress_cb)(off_t, void *), void *_progress_data) : processor(p), opstate(_ops),
+                                                                       progress_cb(_progress_cb),
+                                                                       progress_data(_progress_data) {}
+  int handle_data(bufferlist& bl, off_t ofs, off_t len) {
+    progress_cb(ofs, progress_data);
+
+    bool again;
+
+    bool need_opstate = true;
+
+    do {
+      void *handle;
+      int ret = processor->handle_data(bl, ofs, &handle, &again);
+      if (ret < 0)
+        return ret;
+
+      if (need_opstate && opstate) {
+        /* need to update opstate repository with new state. This is ratelimited, so we're not
+         * really doing it every time
+         */
+        ret = opstate->renew_state();
+        if (ret < 0) {
+          /* could not renew state! might have been marked as cancelled */
+          return ret;
+        }
+
+        need_opstate = false;
+      }
+
+      ret = processor->throttle_data(handle, false);
+      if (ret < 0)
+        return ret;
+    } while (again);
+
+    return 0;
+  }
+
+  void set_extra_data_len(uint64_t len) {
+    RGWGetDataCB::set_extra_data_len(len);
+    processor->set_extra_data_len(len);
+  }
+
+  int complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs) {
+    return processor->complete(etag, mtime, set_mtime, attrs);
+  }
+};
+
+/*
+ * prepare attrset, either replace it with new attrs, or keep it (other than acls).
+ */
+static void set_copy_attrs(map& src_attrs, map& attrs, bool replace_attrs, bool intra_region)
+{
+  if (replace_attrs) {
+    if (!attrs[RGW_ATTR_ETAG].length())
+      attrs[RGW_ATTR_ETAG] = src_attrs[RGW_ATTR_ETAG];
+
+    src_attrs = attrs;
+  } else {
+    /* copying attrs from source, however acls should only be copied if it's intra-region operation */
+    if (!intra_region)
+      src_attrs[RGW_ATTR_ACL] = attrs[RGW_ATTR_ACL];
+  }
+}
+
+class GetObjHandleDestructor {
+  RGWRados *store;
+  void **handle;
+
+public:
+    GetObjHandleDestructor(RGWRados *_store) : store(_store), handle(NULL) {}
+    ~GetObjHandleDestructor() {
+      if (handle) {
+        store->finish_get_obj(handle);
+      }
+    }
+    void set_handle(void **_h) {
+      handle = _h;
+    }
+};
+
+/**
+ * Copy an object.
+ * dest_obj: the object to copy into
+ * src_obj: the object to copy from
+ * attrs: if replace_attrs is set then these are placed on the new object
+ * err: stores any errors resulting from the get of the original object
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+int RGWRados::copy_obj(void *ctx,
+               const string& user_id,
+               const string& client_id,
+               const string& op_id,
+               req_info *info,
+               const string& source_zone,
+               rgw_obj& dest_obj,
+               rgw_obj& src_obj,
+               RGWBucketInfo& dest_bucket_info,
+               RGWBucketInfo& src_bucket_info,
+               time_t *mtime,
+               const time_t *mod_ptr,
+               const time_t *unmod_ptr,
+               const char *if_match,
+               const char *if_nomatch,
+               bool replace_attrs,
+               map& attrs,
+               RGWObjCategory category,
+               string *ptag,
+               struct rgw_err *err,
+               void (*progress_cb)(off_t, void *),
+               void *progress_data)
+{
+  int ret;
+  uint64_t total_len, obj_size;
+  time_t lastmod;
+  rgw_obj shadow_obj = dest_obj;
+  string shadow_oid;
+
+  bool remote_src;
+  bool remote_dest;
+
+  append_rand_alpha(cct, dest_obj.object, shadow_oid, 32);
+  shadow_obj.init_ns(dest_obj.bucket, shadow_oid, shadow_ns);
+
+  remote_dest = !region.equals(dest_bucket_info.region);
+  remote_src = !region.equals(src_bucket_info.region);
+
+  if (remote_src && remote_dest) {
+    ldout(cct, 0) << "ERROR: can't copy object when both src and dest buckets are remote" << dendl;
+    return -EINVAL;
+  }
+
+  ldout(cct, 5) << "Copy object " << src_obj.bucket << ":" << src_obj.object << " => " << dest_obj.bucket << ":" << dest_obj.object << dendl;
+
+  void *handle = NULL;
+  GetObjHandleDestructor handle_destructor(this);
+
+  map src_attrs;
+  off_t ofs = 0;
+  off_t end = -1;
+  if (!remote_src && source_zone.empty()) {
+    ret = prepare_get_obj(ctx, src_obj, &ofs, &end, &src_attrs,
+                  mod_ptr, unmod_ptr, &lastmod, if_match, if_nomatch, &total_len, &obj_size, NULL, &handle, err);
+    if (ret < 0)
+      return ret;
+
+    handle_destructor.set_handle(&handle);
+  } else {
+    /* source is in a different region, copy it there */
+
+    RGWRESTStreamReadRequest *in_stream_req;
+    string tag;
+    append_rand_alpha(cct, tag, tag, 32);
+
+    RGWPutObjProcessor_Atomic processor(dest_bucket_info.owner, dest_obj.bucket, dest_obj.object,
+                                        cct->_conf->rgw_obj_stripe_size, tag);
+    ret = processor.prepare(this, ctx, NULL);
+    if (ret < 0)
+      return ret;
+
+    RGWRESTConn *conn;
+    if (source_zone.empty()) {
+      if (dest_bucket_info.region.empty()) {
+        /* source is in the master region */
+        conn = rest_master_conn;
+      } else {
+        map::iterator iter = region_conn_map.find(src_bucket_info.region);
+        if (iter == region_conn_map.end()) {
+          ldout(cct, 0) << "could not find region connection to region: " << source_zone << dendl;
+          return -ENOENT;
+        }
+        conn = iter->second;
+      }
+    } else {
+      map::iterator iter = zone_conn_map.find(source_zone);
+      if (iter == zone_conn_map.end()) {
+        ldout(cct, 0) << "could not find zone connection to zone: " << source_zone << dendl;
+        return -ENOENT;
+      }
+      conn = iter->second;
+    }
+
+    string obj_name = dest_obj.bucket.name + "/" + dest_obj.object;
+
+    RGWOpStateSingleOp opstate(this, client_id, op_id, obj_name);
+
+    int ret = opstate.set_state(RGWOpState::OPSTATE_IN_PROGRESS);
+    if (ret < 0) {
+      ldout(cct, 0) << "ERROR: failed to set opstate ret=" << ret << dendl;
+      return ret;
+    }
+    RGWRadosPutObj cb(&processor, &opstate, progress_cb, progress_data);
+    string etag;
+    map req_headers;
+    time_t set_mtime;
+   
+    ret = conn->get_obj(user_id, info, src_obj, true, &cb, &in_stream_req);
+    if (ret < 0)
+      goto set_err_state;
+
+    ret = conn->complete_request(in_stream_req, etag, &set_mtime, req_headers);
+    if (ret < 0)
+      goto set_err_state;
+
+    { /* opening scope so that we can do goto, sorry */
+      bufferlist& extra_data_bl = processor.get_extra_data();
+      if (extra_data_bl.length()) {
+        JSONParser jp;
+        if (!jp.parse(extra_data_bl.c_str(), extra_data_bl.length())) {
+          ldout(cct, 0) << "failed to parse response extra data. len=" << extra_data_bl.length() << " data=" << extra_data_bl.c_str() << dendl;
+          goto set_err_state;
+        }
+
+        JSONDecoder::decode_json("attrs", src_attrs, &jp);
+
+        src_attrs.erase(RGW_ATTR_MANIFEST); // not interested in original object layout
+      }
+    }
+
+    set_copy_attrs(src_attrs, attrs, replace_attrs, !source_zone.empty());
+
+    ret = cb.complete(etag, mtime, set_mtime, src_attrs);
+    if (ret < 0)
+      goto set_err_state;
+
+    ret = opstate.set_state(RGWOpState::OPSTATE_COMPLETE);
+    if (ret < 0) {
+      ldout(cct, 0) << "ERROR: failed to set opstate ret=" << ret << dendl;
+    }
+
+    return 0;
+set_err_state:
+    int r = opstate.set_state(RGWOpState::OPSTATE_ERROR);
+    if (r < 0) {
+      ldout(cct, 0) << "ERROR: failed to set opstate r=" << ret << dendl;
+    }
+    return ret;
+  }
+  set_copy_attrs(src_attrs, attrs, replace_attrs, false);
+  src_attrs.erase(RGW_ATTR_ID_TAG);
+
+  RGWObjManifest manifest;
+  RGWObjState *astate = NULL;
+  RGWRadosCtx *rctx = static_cast(ctx);
+  ret = get_obj_state(rctx, src_obj, &astate, NULL);
+  if (ret < 0)
+    return ret;
+
+  vector ref_objs;
+
+  if (remote_dest) {
+    /* dest is in a different region, copy it there */
+
+    string etag;
+
+    RGWRESTStreamWriteRequest *out_stream_req;
+
+    int ret = rest_master_conn->put_obj_init(user_id, dest_obj, astate->size, src_attrs, &out_stream_req);
+    if (ret < 0)
+      return ret;
+
+    ret = get_obj_iterate(ctx, &handle, src_obj, 0, astate->size - 1, out_stream_req->get_out_cb());
+    if (ret < 0)
+      return ret;
+
+    ret = rest_master_conn->complete_request(out_stream_req, etag, mtime);
+    if (ret < 0)
+      return ret;
+
+    return 0;
+  }
+  
+  uint64_t max_chunk_size;
+
+  ret = get_max_chunk_size(dest_obj.bucket, &max_chunk_size);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: failed to get max_chunk_size() for bucket " << dest_obj.bucket << dendl;
+    return ret;
+  }
+
+  bool copy_data = !astate->has_manifest;
+  bool copy_first = false;
+  if (astate->has_manifest) {
+    if (!astate->manifest.has_tail()) {
+      copy_data = true;
+    } else {
+      uint64_t head_size = astate->manifest.get_head_size();
+
+      if (head_size > 0) {
+	if (head_size > max_chunk_size)
+	  copy_data = true;
+	else
+          copy_first = true;
+      }
+    }
+  }
+
+  if (copy_data) { /* refcounting tail wouldn't work here, just copy the data */
+    return copy_obj_data(ctx, dest_bucket_info.owner, &handle, end, dest_obj, src_obj, max_chunk_size, mtime, src_attrs, category, ptag, err);
+  }
+
+  RGWObjManifest::obj_iterator miter = astate->manifest.obj_begin();
+
+  if (copy_first) // we need to copy first chunk, not increase refcount
+    ++miter;
+
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  ret = get_obj_ref(miter.get_location(), &ref, &bucket);
+  if (ret < 0) {
+    return ret;
+  }
+  PutObjMetaExtraParams ep;
+
+  bufferlist first_chunk;
+
+  bool copy_itself = (dest_obj == src_obj);
+  RGWObjManifest *pmanifest; 
+  ldout(cct, 0) << "dest_obj=" << dest_obj << " src_obj=" << src_obj << " copy_itself=" << (int)copy_itself << dendl;
+
+
+  string tag;
+
+  if (ptag)
+    tag = *ptag;
+
+  if (tag.empty()) {
+    append_rand_alpha(cct, tag, tag, 32);
+  }
+
+  if (!copy_itself) {
+    manifest = astate->manifest;
+    rgw_bucket& tail_bucket = manifest.get_tail_bucket();
+    if (tail_bucket.name.empty()) {
+      manifest.set_tail_bucket(src_obj.bucket);
+    }
+    string oid, key;
+    for (; miter != astate->manifest.obj_end(); ++miter) {
+      ObjectWriteOperation op;
+      cls_refcount_get(op, tag, true);
+      const rgw_obj& loc = miter.get_location();
+      get_obj_bucket_and_oid_key(loc, bucket, oid, key);
+      ref.ioctx.locator_set_key(key);
+
+      ret = ref.ioctx.operate(oid, &op);
+      if (ret < 0)
+        goto done_ret;
+
+      ref_objs.push_back(loc);
+    }
+
+    pmanifest = &manifest;
+  } else {
+    pmanifest = &astate->manifest;
+    /* don't send the object's tail for garbage collection */
+    astate->keep_tail = true;
+  }
+
+  if (copy_first) {
+    ret = get_obj(ctx, NULL, &handle, src_obj, first_chunk, 0, max_chunk_size);
+    if (ret < 0)
+      goto done_ret;
+
+    pmanifest->set_head(dest_obj);
+    pmanifest->set_head_size(first_chunk.length());
+  }
+
+  ep.data = &first_chunk;
+  ep.manifest = pmanifest;
+  ep.ptag = &tag;
+  ep.owner = dest_bucket_info.owner;
+
+  ret = put_obj_meta(ctx, dest_obj, end + 1, src_attrs, category, PUT_OBJ_CREATE, ep);
+
+  if (mtime)
+    obj_stat(ctx, dest_obj, NULL, mtime, NULL, NULL, NULL, NULL);
+
+  return 0;
+
+done_ret:
+  if (!copy_itself) {
+    vector::iterator riter;
+
+    string oid, key;
+
+    /* rollback reference */
+    for (riter = ref_objs.begin(); riter != ref_objs.end(); ++riter) {
+      ObjectWriteOperation op;
+      cls_refcount_put(op, tag, true);
+
+      get_obj_bucket_and_oid_key(*riter, bucket, oid, key);
+      ref.ioctx.locator_set_key(key);
+
+      int r = ref.ioctx.operate(oid, &op);
+      if (r < 0) {
+        ldout(cct, 0) << "ERROR: cleanup after error failed to drop reference on obj=" << *riter << dendl;
+      }
+    }
+  }
+  return ret;
+}
+
+
+int RGWRados::copy_obj_data(void *ctx,
+               const string& owner,
+	       void **handle, off_t end,
+               rgw_obj& dest_obj,
+               rgw_obj& src_obj,
+               uint64_t max_chunk_size,
+	       time_t *mtime,
+               map& attrs,
+               RGWObjCategory category,
+               string *ptag,
+               struct rgw_err *err)
+{
+  bufferlist first_chunk;
+  RGWObjManifest manifest;
+  map objs;
+  RGWObjManifestPart *first_part;
+  map::iterator iter;
+
+  rgw_obj shadow_obj = dest_obj;
+  string shadow_oid;
+
+  append_rand_alpha(cct, dest_obj.object, shadow_oid, 32);
+  shadow_obj.init_ns(dest_obj.bucket, shadow_oid, shadow_ns);
+
+  int ret, r;
+  off_t ofs = 0;
+  PutObjMetaExtraParams ep;
+
+  do {
+    bufferlist bl;
+    ret = get_obj(ctx, NULL, handle, src_obj, bl, ofs, end);
+    if (ret < 0)
+      return ret;
+
+    const char *data = bl.c_str();
+
+    if ((uint64_t)ofs < max_chunk_size) {
+      uint64_t len = min(max_chunk_size - ofs, (uint64_t)ret);
+      first_chunk.append(data, len);
+      ofs += len;
+      ret -= len;
+      data += len;
+    }
+
+    // In the first call to put_obj_data, we pass ofs == -1 so that it will do
+    // a write_full, wiping out whatever was in the object before this
+    r = 0;
+    if (ret > 0) {
+      r = put_obj_data(ctx, shadow_obj, data, ((ofs == 0) ? -1 : ofs), ret, false);
+    }
+    if (r < 0)
+      goto done_err;
+
+    ofs += ret;
+  } while (ofs <= end);
+
+  first_part = &objs[0];
+  first_part->loc = dest_obj;
+  first_part->loc_ofs = 0;
+  first_part->size = first_chunk.length();
+
+  if ((uint64_t)ofs > max_chunk_size) {
+    RGWObjManifestPart& tail = objs[max_chunk_size];
+    tail.loc = shadow_obj;
+    tail.loc_ofs = max_chunk_size;
+    tail.size = ofs - max_chunk_size;
+  }
+
+  manifest.set_explicit(ofs, objs);
+
+  ep.data = &first_chunk;
+  ep.manifest = &manifest;
+  ep.ptag = ptag;
+  ep.owner = owner;
+
+  ret = put_obj_meta(ctx, dest_obj, end + 1, attrs, category, PUT_OBJ_CREATE, ep);
+  if (mtime)
+    obj_stat(ctx, dest_obj, NULL, mtime, NULL, NULL, NULL, NULL);
+
+  return ret;
+done_err:
+  delete_obj(ctx, owner, shadow_obj);
+  return r;
+}
+
+/**
+ * Delete a bucket.
+ * bucket: the name of the bucket to delete
+ * Returns 0 on success, -ERR# otherwise.
+ */
+int RGWRados::delete_bucket(rgw_bucket& bucket, RGWObjVersionTracker& objv_tracker)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  std::map ent_map;
+  string marker, prefix;
+  bool is_truncated;
+
+  do {
+#define NUM_ENTRIES 1000
+    r = cls_bucket_list(bucket, marker, prefix, NUM_ENTRIES, ent_map,
+                        &is_truncated, &marker);
+    if (r < 0)
+      return r;
+
+    string ns;
+    std::map::iterator eiter;
+    string obj;
+    for (eiter = ent_map.begin(); eiter != ent_map.end(); ++eiter) {
+      obj = eiter->first;
+
+      if (rgw_obj::translate_raw_obj_to_obj_in_ns(obj, ns))
+        return -ENOTEMPTY;
+    }
+  } while (is_truncated);
+
+  r = rgw_bucket_delete_bucket_obj(this, bucket.name, objv_tracker);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+
+int RGWRados::set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner)
+{
+  RGWBucketInfo info;
+  map attrs;
+  int r = get_bucket_info(NULL, bucket.name, info, NULL, &attrs);
+  if (r < 0) {
+    ldout(cct, 0) << "NOTICE: get_bucket_info on bucket=" << bucket.name << " returned err=" << r << dendl;
+    return r;
+  }
+
+  info.owner = owner.get_id();
+
+  r = put_bucket_instance_info(info, false, 0, &attrs);
+  if (r < 0) {
+    ldout(cct, 0) << "NOTICE: put_bucket_info on bucket=" << bucket.name << " returned err=" << r << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
+
+int RGWRados::set_buckets_enabled(vector& buckets, bool enabled)
+{
+  int ret = 0;
+
+  vector::iterator iter;
+
+  for (iter = buckets.begin(); iter != buckets.end(); ++iter) {
+    rgw_bucket& bucket = *iter;
+    if (enabled)
+      ldout(cct, 20) << "enabling bucket name=" << bucket.name << dendl;
+    else
+      ldout(cct, 20) << "disabling bucket name=" << bucket.name << dendl;
+
+    RGWBucketInfo info;
+    map attrs;
+    int r = get_bucket_info(NULL, bucket.name, info, NULL, &attrs);
+    if (r < 0) {
+      ldout(cct, 0) << "NOTICE: get_bucket_info on bucket=" << bucket.name << " returned err=" << r << ", skipping bucket" << dendl;
+      ret = r;
+      continue;
+    }
+    if (enabled) {
+      info.flags &= ~BUCKET_SUSPENDED;
+    } else {
+      info.flags |= BUCKET_SUSPENDED;
+    }
+
+    r = put_bucket_instance_info(info, false, 0, &attrs);
+    if (r < 0) {
+      ldout(cct, 0) << "NOTICE: put_bucket_info on bucket=" << bucket.name << " returned err=" << r << ", skipping bucket" << dendl;
+      ret = r;
+      continue;
+    }
+  }
+  return ret;
+}
+
+int RGWRados::bucket_suspended(rgw_bucket& bucket, bool *suspended)
+{
+  RGWBucketInfo bucket_info;
+  int ret = get_bucket_info(NULL, bucket.name, bucket_info, NULL);
+  if (ret < 0) {
+    return ret;
+  }
+
+  *suspended = ((bucket_info.flags & BUCKET_SUSPENDED) != 0);
+  return 0;
+}
+
+int RGWRados::complete_atomic_overwrite(RGWRadosCtx *rctx, RGWObjState *state, rgw_obj& obj)
+{
+  if (!state || !state->has_manifest || state->keep_tail)
+    return 0;
+
+  cls_rgw_obj_chain chain;
+  RGWObjManifest::obj_iterator iter;
+  for (iter = state->manifest.obj_begin(); iter != state->manifest.obj_end(); ++iter) {
+    const rgw_obj& mobj = iter.get_location();
+    if (mobj == obj)
+      continue;
+    string oid, key;
+    rgw_bucket bucket;
+    get_obj_bucket_and_oid_key(mobj, bucket, oid, key);
+    chain.push_obj(bucket.data_pool, oid, key);
+  }
+
+  string tag = state->obj_tag.c_str();
+  int ret = gc->send_chain(chain, tag, false);  // do it async
+
+  return ret;
+}
+
+int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, string& bucket_oid)
+{
+  if (bucket_is_system(bucket))
+    return -EINVAL;
+
+  int r = open_bucket_index_ctx(bucket, index_ctx);
+  if (r < 0)
+    return r;
+
+  if (bucket.marker.empty()) {
+    ldout(cct, 0) << "ERROR: empty marker for bucket operation" << dendl;
+    return -EIO;
+  }
+
+  bucket_oid = dir_oid_prefix;
+  bucket_oid.append(bucket.marker);
+
+  return 0;
+}
+
+static void translate_raw_stats(rgw_bucket_dir_header& header, map& stats)
+{
+  map::iterator iter = header.stats.begin();
+  for (; iter != header.stats.end(); ++iter) {
+    RGWObjCategory category = (RGWObjCategory)iter->first;
+    RGWStorageStats& s = stats[category];
+    struct rgw_bucket_category_stats& header_stats = iter->second;
+    s.category = (RGWObjCategory)iter->first;
+    s.num_kb = ((header_stats.total_size + 1023) / 1024);
+    s.num_kb_rounded = ((header_stats.total_size_rounded + 1023) / 1024);
+    s.num_objects = header_stats.num_entries;
+  }
+}
+
+int RGWRados::bucket_check_index(rgw_bucket& bucket,
+				 map *existing_stats,
+				 map *calculated_stats)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+
+  int ret = open_bucket_index(bucket, index_ctx, oid);
+  if (ret < 0)
+    return ret;
+
+  rgw_bucket_dir_header existing_header;
+  rgw_bucket_dir_header calculated_header;
+
+  ret = cls_rgw_bucket_check_index_op(index_ctx, oid, &existing_header, &calculated_header);
+  if (ret < 0)
+    return ret;
+
+  translate_raw_stats(existing_header, *existing_stats);
+  translate_raw_stats(calculated_header, *calculated_stats);
+
+  return 0;
+}
+
+int RGWRados::bucket_rebuild_index(rgw_bucket& bucket)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+
+  int ret = open_bucket_index(bucket, index_ctx, oid);
+  if (ret < 0)
+    return ret;
+
+  return cls_rgw_bucket_rebuild_index_op(index_ctx, oid);
+}
+
+
+int RGWRados::defer_gc(void *ctx, rgw_obj& obj)
+{
+  RGWRadosCtx *rctx = static_cast(ctx);
+  rgw_bucket bucket;
+  std::string oid, key;
+  get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+  if (!rctx)
+    return 0;
+
+  RGWObjState *state = NULL;
+
+  int r = get_obj_state(rctx, obj, &state, NULL);
+  if (r < 0)
+    return r;
+
+  if (!state->is_atomic) {
+    ldout(cct, 20) << "state for obj=" << obj << " is not atomic, not deferring gc operation" << dendl;
+    return -EINVAL;
+  }
+
+  if (state->obj_tag.length() == 0) {// check for backward compatibility
+    ldout(cct, 20) << "state->obj_tag is empty, not deferring gc operation" << dendl;
+    return -EINVAL;
+  }
+
+  string tag = state->obj_tag.c_str();
+
+  ldout(cct, 0) << "defer chain tag=" << tag << dendl;
+
+  return gc->defer_chain(tag, false);
+}
+
+
+/**
+ * Delete an object.
+ * bucket: name of the bucket storing the object
+ * obj: name of the object to delete
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+int RGWRados::delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+  RGWRadosCtx *rctx = static_cast(ctx);
+
+  ObjectWriteOperation op;
+
+  RGWObjState *state;
+  r = prepare_atomic_for_write(rctx, obj, op, &state, false, NULL);
+  if (r < 0)
+    return r;
+
+  bool ret_not_existed = (state && !state->exists);
+
+  string tag;
+  r = prepare_update_index(state, bucket, CLS_RGW_OP_DEL, obj, tag);
+  if (r < 0)
+    return r;
+
+  if (objv_tracker) {
+    objv_tracker->prepare_op_for_write(&op);
+  }
+
+  cls_refcount_put(op, tag, true);
+  r = ref.ioctx.operate(ref.oid, &op);
+  bool removed = (r >= 0);
+
+  int64_t poolid = ref.ioctx.get_id();
+  if (r >= 0 || r == -ENOENT) {
+    uint64_t epoch = ref.ioctx.get_last_version();
+    r = complete_update_index_del(bucket, obj.object, tag, poolid, epoch);
+  } else {
+    int ret = complete_update_index_cancel(bucket, obj.object, tag);
+    if (ret < 0) {
+      ldout(cct, 0) << "ERROR: complete_update_index_cancel returned ret=" << ret << dendl;
+    }
+  }
+  if (removed) {
+    int ret = complete_atomic_overwrite(rctx, state, obj);
+    if (ret < 0) {
+      ldout(cct, 0) << "ERROR: complete_atomic_removal returned ret=" << ret << dendl;
+    }
+    /* other than that, no need to propagate error */
+  }
+
+  atomic_write_finish(state, r);
+
+  if (r < 0)
+    return r;
+
+  if (ret_not_existed)
+    return -ENOENT;
+
+  if (state) {
+    /* update quota cache */
+    quota_handler->update_stats(bucket_owner, bucket, -1, 0, state->size);
+  }
+
+  return 0;
+}
+
+int RGWRados::delete_obj(void *ctx, const string& bucket_owner, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+{
+  int r;
+
+  r = delete_obj_impl(ctx, bucket_owner, obj, objv_tracker);
+  if (r == -ECANCELED)
+    r = 0;
+
+  return r;
+}
+
+int RGWRados::delete_system_obj(void *ctx, rgw_obj& obj, RGWObjVersionTracker *objv_tracker)
+{
+  int r;
+
+  string no_owner;
+  r = delete_obj_impl(ctx, no_owner, obj, objv_tracker);
+  if (r == -ECANCELED)
+    r = 0;
+
+  return r;
+}
+
+int RGWRados::delete_obj_index(rgw_obj& obj)
+{
+  rgw_bucket bucket;
+  std::string oid, key;
+  get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+
+  string tag;
+  int r = complete_update_index_del(bucket, obj.object, tag, -1 /* pool */, 0);
+
+  return r;
+}
+
+static void generate_fake_tag(CephContext *cct, map& attrset, RGWObjManifest& manifest, bufferlist& manifest_bl, bufferlist& tag_bl)
+{
+  string tag;
+
+  RGWObjManifest::obj_iterator mi = manifest.obj_begin();
+  if (mi != manifest.obj_end()) {
+    if (manifest.has_tail()) // first object usually points at the head, let's skip to a more unique part
+      ++mi;
+    tag = mi.get_location().object;
+    tag.append("_");
+  }
+
+  unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  char md5_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
+  MD5 hash;
+  hash.Update((const byte *)manifest_bl.c_str(), manifest_bl.length());
+
+  map::iterator iter = attrset.find(RGW_ATTR_ETAG);
+  if (iter != attrset.end()) {
+    bufferlist& bl = iter->second;
+    hash.Update((const byte *)bl.c_str(), bl.length());
+  }
+
+  hash.Final(md5);
+  buf_to_hex(md5, CEPH_CRYPTO_MD5_DIGESTSIZE, md5_str);
+  tag.append(md5_str);
+
+  ldout(cct, 10) << "generate_fake_tag new tag=" << tag << dendl;
+
+  tag_bl.append(tag.c_str(), tag.size() + 1);
+}
+
+int RGWRados::get_obj_state(RGWRadosCtx *rctx, rgw_obj& obj, RGWObjState **state, RGWObjVersionTracker *objv_tracker)
+{
+  RGWObjState *s = rctx->get_state(obj);
+  ldout(cct, 20) << "get_obj_state: rctx=" << (void *)rctx << " obj=" << obj << " state=" << (void *)s << " s->prefetch_data=" << s->prefetch_data << dendl;
+  *state = s;
+  if (s->has_attrs)
+    return 0;
+
+  int r = obj_stat(rctx, obj, &s->size, &s->mtime, &s->epoch, &s->attrset, (s->prefetch_data ? &s->data : NULL), objv_tracker);
+  if (r == -ENOENT) {
+    s->exists = false;
+    s->has_attrs = true;
+    s->mtime = 0;
+    return 0;
+  }
+  if (r < 0)
+    return r;
+
+  s->exists = true;
+  s->has_attrs = true;
+  map::iterator iter = s->attrset.find(RGW_ATTR_SHADOW_OBJ);
+  if (iter != s->attrset.end()) {
+    bufferlist bl = iter->second;
+    bufferlist::iterator it = bl.begin();
+    it.copy(bl.length(), s->shadow_obj);
+    s->shadow_obj[bl.length()] = '\0';
+  }
+  s->obj_tag = s->attrset[RGW_ATTR_ID_TAG];
+  bufferlist manifest_bl = s->attrset[RGW_ATTR_MANIFEST];
+  if (manifest_bl.length()) {
+    bufferlist::iterator miter = manifest_bl.begin();
+    try {
+      ::decode(s->manifest, miter);
+      s->has_manifest = true;
+      s->size = s->manifest.get_obj_size();
+    } catch (buffer::error& err) {
+      ldout(cct, 20) << "ERROR: couldn't decode manifest" << dendl;
+      return -EIO;
+    }
+    ldout(cct, 10) << "manifest: total_size = " << s->manifest.get_obj_size() << dendl;
+    if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20) && s->manifest.has_explicit_objs()) {
+      RGWObjManifest::obj_iterator mi;
+      for (mi = s->manifest.obj_begin(); mi != s->manifest.obj_end(); ++mi) {
+        ldout(cct, 20) << "manifest: ofs=" << mi.get_ofs() << " loc=" << mi.get_location() << dendl;
+      }
+    }
+
+    if (!s->obj_tag.length()) {
+      /*
+       * Uh oh, something's wrong, object with manifest should have tag. Let's
+       * create one out of the manifest, would be unique
+       */
+      generate_fake_tag(cct, s->attrset, s->manifest, manifest_bl, s->obj_tag);
+      s->fake_tag = true;
+    }
+  }
+  if (s->obj_tag.length())
+    ldout(cct, 20) << "get_obj_state: setting s->obj_tag to " << s->obj_tag.c_str() << dendl;
+  else
+    ldout(cct, 20) << "get_obj_state: s->obj_tag was set empty" << dendl;
+  return 0;
+}
+
+/**
+ * Get the attributes for an object.
+ * bucket: name of the bucket holding the object.
+ * obj: name of the object
+ * name: name of the attr to retrieve
+ * dest: bufferlist to store the result in
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+int RGWRados::get_attr(void *ctx, rgw_obj& obj, const char *name, bufferlist& dest)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket, true);
+  if (r < 0) {
+    return r;
+  }
+  RGWRadosCtx *rctx = static_cast(ctx);
+
+  if (rctx) {
+    RGWObjState *state;
+    r = get_obj_state(rctx, obj, &state, NULL);
+    if (r < 0)
+      return r;
+    if (!state->exists)
+      return -ENOENT;
+    if (state->get_attr(name, dest))
+      return 0;
+    return -ENODATA;
+  }
+
+  ObjectReadOperation op;
+
+  int rval;
+  op.getxattr(name, &dest, &rval);
+  
+  r = ref.ioctx.operate(ref.oid, &op, NULL);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::append_atomic_test(RGWRadosCtx *rctx, rgw_obj& obj,
+                            ObjectOperation& op, RGWObjState **pstate)
+{
+  if (!rctx)
+    return 0;
+
+  int r = get_obj_state(rctx, obj, pstate, NULL);
+  if (r < 0)
+    return r;
+
+  RGWObjState *state = *pstate;
+
+  if (!state->is_atomic) {
+    ldout(cct, 20) << "state for obj=" << obj << " is not atomic, not appending atomic test" << dendl;
+    return 0;
+  }
+
+  if (state->obj_tag.length() > 0 && !state->fake_tag) {// check for backward compatibility
+    op.cmpxattr(RGW_ATTR_ID_TAG, LIBRADOS_CMPXATTR_OP_EQ, state->obj_tag);
+  } else {
+    ldout(cct, 20) << "state->obj_tag is empty, not appending atomic test" << dendl;
+  }
+  return 0;
+}
+
+int RGWRados::prepare_atomic_for_write_impl(RGWRadosCtx *rctx, rgw_obj& obj,
+                            ObjectWriteOperation& op, RGWObjState **pstate,
+			    bool reset_obj, const string *ptag)
+{
+  int r = get_obj_state(rctx, obj, pstate, NULL);
+  if (r < 0)
+    return r;
+
+  RGWObjState *state = *pstate;
+
+  bool need_guard = (state->has_manifest || (state->obj_tag.length() != 0)) && (!state->fake_tag);
+
+  if (!state->is_atomic) {
+    ldout(cct, 20) << "prepare_atomic_for_write_impl: state is not atomic. state=" << (void *)state << dendl;
+
+    if (reset_obj) {
+      op.create(false);
+      op.remove(); // we're not dropping reference here, actually removing object
+    }
+
+    return 0;
+  }
+
+  if (need_guard) {
+    /* first verify that the object wasn't replaced under */
+    op.cmpxattr(RGW_ATTR_ID_TAG, LIBRADOS_CMPXATTR_OP_EQ, state->obj_tag);
+    // FIXME: need to add FAIL_NOTEXIST_OK for racing deletion
+  }
+
+  if (reset_obj) {
+    if (state->exists) {
+      op.create(false);
+      op.remove();
+    } else {
+      op.create(true);
+    }
+  }
+
+  if (ptag) {
+    state->write_tag = *ptag;
+  } else {
+    append_rand_alpha(cct, state->write_tag, state->write_tag, 32);
+  }
+  bufferlist bl;
+  bl.append(state->write_tag.c_str(), state->write_tag.size() + 1);
+
+  ldout(cct, 10) << "setting object write_tag=" << state->write_tag << dendl;
+
+  op.setxattr(RGW_ATTR_ID_TAG, bl);
+
+  return 0;
+}
+
+int RGWRados::prepare_atomic_for_write(RGWRadosCtx *rctx, rgw_obj& obj,
+                            ObjectWriteOperation& op, RGWObjState **pstate,
+			    bool reset_obj, const string *ptag)
+{
+  if (!rctx) {
+    *pstate = NULL;
+    return 0;
+  }
+
+  int r;
+  r = prepare_atomic_for_write_impl(rctx, obj, op, pstate, reset_obj, ptag);
+
+  return r;
+}
+
+/**
+ * Set an attr on an object.
+ * bucket: name of the bucket holding the object
+ * obj: name of the object to set the attr on
+ * name: the attr to set
+ * bl: the contents of the attr
+ * Returns: 0 on success, -ERR# otherwise.
+ */
+int RGWRados::set_attr(void *ctx, rgw_obj& obj, const char *name, bufferlist& bl, RGWObjVersionTracker *objv_tracker)
+{
+  map attrs;
+  attrs[name] = bl;
+  return set_attrs(ctx, obj, attrs, NULL, objv_tracker);
+}
+
+int RGWRados::set_attrs(void *ctx, rgw_obj& obj,
+                        map& attrs,
+                        map* rmattrs,
+                        RGWObjVersionTracker *objv_tracker)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket, true);
+  if (r < 0) {
+    return r;
+  }
+  RGWRadosCtx *rctx = static_cast(ctx);
+
+  ObjectWriteOperation op;
+  RGWObjState *state = NULL;
+
+  r = append_atomic_test(rctx, obj, op, &state);
+  if (r < 0)
+    return r;
+
+  if (objv_tracker) {
+    objv_tracker->prepare_op_for_write(&op);
+  }
+
+  map::iterator iter;
+  if (rmattrs) {
+    for (iter = rmattrs->begin(); iter != rmattrs->end(); ++iter) {
+      const string& name = iter->first;
+      op.rmxattr(name.c_str());
+    }
+  }
+
+  for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+    const string& name = iter->first;
+    bufferlist& bl = iter->second;
+
+    if (!bl.length())
+      continue;
+
+    op.setxattr(name.c_str(), bl);
+  }
+
+  if (!op.size())
+    return 0;
+
+  r = ref.ioctx.operate(ref.oid, &op);
+  if (r < 0)
+    return r;
+
+  if (state) {
+    if (rmattrs) {
+      for (iter = rmattrs->begin(); iter != rmattrs->end(); ++iter) {
+        state->attrset.erase(iter->first);
+      }
+    }
+    for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+      state->attrset[iter->first] = iter->second;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Get data about an object out of RADOS and into memory.
+ * bucket: name of the bucket the object is in.
+ * obj: name/key of the object to read
+ * data: if get_data==true, this pointer will be set
+ *    to an address containing the object's data/value
+ * ofs: the offset of the object to read from
+ * end: the point in the object to stop reading
+ * attrs: if non-NULL, the pointed-to map will contain
+ *    all the attrs of the object when this function returns
+ * mod_ptr: if non-NULL, compares the object's mtime to *mod_ptr,
+ *    and if mtime is smaller it fails.
+ * unmod_ptr: if non-NULL, compares the object's mtime to *unmod_ptr,
+ *    and if mtime is >= it fails.
+ * if_match/nomatch: if non-NULL, compares the object's etag attr
+ *    to the string and, if it doesn't/does match, fails out.
+ * get_data: if true, the object's data/value will be read out, otherwise not
+ * err: Many errors will result in this structure being filled
+ *    with extra informatin on the error.
+ * Returns: -ERR# on failure, otherwise
+ *          (if get_data==true) length of read data,
+ *          (if get_data==false) length of the object
+ */
+int RGWRados::prepare_get_obj(void *ctx, rgw_obj& obj,
+            off_t *pofs, off_t *pend,
+            map *attrs,
+            const time_t *mod_ptr,
+            const time_t *unmod_ptr,
+            time_t *lastmod,
+            const char *if_match,
+            const char *if_nomatch,
+            uint64_t *total_size,
+            uint64_t *obj_size,
+            RGWObjVersionTracker *objv_tracker,
+            void **handle,
+            struct rgw_err *err)
+{
+  bufferlist etag;
+  time_t ctime;
+  RGWRadosCtx *rctx = static_cast(ctx);
+  RGWRadosCtx *new_ctx = NULL;
+  RGWObjState *astate = NULL;
+  off_t ofs = 0;
+  off_t end = -1;
+
+  map::iterator iter;
+
+  *handle = NULL;
+
+  GetObjState *state = new GetObjState;
+  if (!state)
+    return -ENOMEM;
+
+  *handle = state;
+
+  int r = get_obj_ioctx(obj, &state->io_ctx);
+  if (r < 0) {
+    delete state;
+    return r;
+  }
+
+  if (!rctx) {
+    new_ctx = new RGWRadosCtx(this);
+    rctx = new_ctx;
+  }
+
+  r = get_obj_state(rctx, obj, &astate, objv_tracker);
+  if (r < 0)
+    goto done_err;
+
+  if (!astate->exists) {
+    r = -ENOENT;
+    goto done_err;
+  }
+
+  if (attrs) {
+    *attrs = astate->attrset;
+    if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
+      for (iter = attrs->begin(); iter != attrs->end(); ++iter) {
+        ldout(cct, 20) << "Read xattr: " << iter->first << dendl;
+      }
+    }
+    if (r < 0)
+      goto done_err;
+  }
+
+  /* Convert all times go GMT to make them compatible */
+  if (mod_ptr || unmod_ptr) {
+    ctime = astate->mtime;
+
+    if (mod_ptr) {
+      ldout(cct, 10) << "If-Modified-Since: " << *mod_ptr << " Last-Modified: " << ctime << dendl;
+      if (ctime < *mod_ptr) {
+        r = -ERR_NOT_MODIFIED;
+        goto done_err;
+      }
+    }
+
+    if (unmod_ptr) {
+      ldout(cct, 10) << "If-UnModified-Since: " << *unmod_ptr << " Last-Modified: " << ctime << dendl;
+      if (ctime > *unmod_ptr) {
+        r = -ERR_PRECONDITION_FAILED;
+        goto done_err;
+      }
+    }
+  }
+  if (if_match || if_nomatch) {
+    r = get_attr(rctx, obj, RGW_ATTR_ETAG, etag);
+    if (r < 0)
+      goto done_err;
+
+    if (if_match) {
+      string if_match_str = rgw_string_unquote(if_match);
+      ldout(cct, 10) << "ETag: " << etag.c_str() << " " << " If-Match: " << if_match_str << dendl;
+      if (if_match_str.compare(etag.c_str()) != 0) {
+        r = -ERR_PRECONDITION_FAILED;
+        goto done_err;
+      }
+    }
+
+    if (if_nomatch) {
+      string if_nomatch_str = rgw_string_unquote(if_nomatch);
+      ldout(cct, 10) << "ETag: " << etag.c_str() << " " << " If-NoMatch: " << if_nomatch_str << dendl;
+      if (if_nomatch_str.compare(etag.c_str()) == 0) {
+        r = -ERR_NOT_MODIFIED;
+        goto done_err;
+      }
+    }
+  }
+
+  if (pofs)
+    ofs = *pofs;
+  if (pend)
+    end = *pend;
+
+  if (ofs < 0) {
+    ofs += astate->size;
+    if (ofs < 0)
+      ofs = 0;
+    end = astate->size - 1;
+  } else if (end < 0) {
+    end = astate->size - 1;
+  }
+
+  if (astate->size > 0) {
+    if (ofs >= (off_t)astate->size) {
+      r = -ERANGE;
+      goto done_err;
+    }
+    if (end >= (off_t)astate->size) {
+      end = astate->size - 1;
+    }
+  }
+
+  if (pofs)
+    *pofs = ofs;
+  if (pend)
+    *pend = end;
+  if (total_size)
+    *total_size = (ofs <= end ? end + 1 - ofs : 0);
+  if (obj_size)
+    *obj_size = astate->size;
+  if (lastmod)
+    *lastmod = astate->mtime;
+
+  delete new_ctx;
+
+  return 0;
+
+done_err:
+  delete new_ctx;
+  finish_get_obj(handle);
+  return r;
+}
+
+int RGWRados::prepare_update_index(RGWObjState *state, rgw_bucket& bucket,
+                                   RGWModifyOp op, rgw_obj& obj, string& tag)
+{
+  if (bucket_is_system(bucket))
+    return 0;
+
+  int ret = data_log->add_entry(obj.bucket);
+  if (ret < 0) {
+    lderr(cct) << "ERROR: failed writing data log" << dendl;
+    return ret;
+  }
+
+  if (state && state->obj_tag.length()) {
+    int len = state->obj_tag.length();
+    char buf[len + 1];
+    memcpy(buf, state->obj_tag.c_str(), len);
+    buf[len] = '\0';
+    tag = buf;
+  } else {
+    if (tag.empty()) {
+      append_rand_alpha(cct, tag, tag, 32);
+    }
+  }
+  ret = cls_obj_prepare_op(bucket, op, tag,
+                               obj.object, obj.key);
+
+  return ret;
+}
+
+int RGWRados::complete_update_index(rgw_bucket& bucket, string& oid, string& tag, int64_t poolid, uint64_t epoch, uint64_t size,
+                                    utime_t& ut, string& etag, string& content_type, bufferlist *acl_bl, RGWObjCategory category,
+                                    list *remove_objs)
+{
+  if (bucket_is_system(bucket))
+    return 0;
+
+  RGWObjEnt ent;
+  ent.name = oid;
+  ent.size = size;
+  ent.mtime = ut;
+  ent.etag = etag;
+  ACLOwner owner;
+  if (acl_bl && acl_bl->length()) {
+    int ret = decode_policy(*acl_bl, &owner);
+    if (ret < 0) {
+      ldout(cct, 0) << "WARNING: could not decode policy ret=" << ret << dendl;
+    }
+  }
+  ent.owner = owner.get_id();
+  ent.owner_display_name = owner.get_display_name();
+  ent.content_type = content_type;
+
+  int ret = cls_obj_complete_add(bucket, tag, poolid, epoch, ent, category, remove_objs);
+
+  return ret;
+}
+
+
+int RGWRados::clone_objs_impl(void *ctx, rgw_obj& dst_obj,
+                        vector& ranges,
+                        map attrs,
+                        RGWObjCategory category,
+                        time_t *pmtime,
+                        bool truncate_dest,
+                        bool exclusive,
+                        pair *xattr_cond)
+{
+  rgw_bucket bucket;
+  std::string dst_oid, dst_key;
+  get_obj_bucket_and_oid_key(dst_obj, bucket, dst_oid, dst_key);
+  librados::IoCtx io_ctx;
+  RGWRadosCtx *rctx = static_cast(ctx);
+  uint64_t size = 0;
+  string etag;
+  string content_type;
+  bufferlist acl_bl;
+  bool update_index = (category == RGW_OBJ_CATEGORY_MAIN ||
+                       category == RGW_OBJ_CATEGORY_MULTIMETA);
+
+  int r = open_bucket_data_ctx(bucket, io_ctx);
+  if (r < 0)
+    return r;
+  io_ctx.locator_set_key(dst_key);
+  ObjectWriteOperation op;
+  if (truncate_dest) {
+    op.remove();
+    op.set_op_flags(OP_FAILOK); // don't fail if object didn't exist
+  }
+
+  op.create(exclusive);
+
+
+  map::iterator iter;
+  for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+    const string& name = iter->first;
+    bufferlist& bl = iter->second;
+    op.setxattr(name.c_str(), bl);
+
+    if (name.compare(RGW_ATTR_ETAG) == 0) {
+      etag = bl.c_str();
+    } else if (name.compare(RGW_ATTR_CONTENT_TYPE) == 0) {
+      content_type = bl.c_str();
+    } else if (name.compare(RGW_ATTR_ACL) == 0) {
+      acl_bl = bl;
+    }
+  }
+  RGWObjState *state;
+  r = prepare_atomic_for_write(rctx, dst_obj, op, &state, true, NULL);
+  if (r < 0)
+    return r;
+
+  vector::iterator range_iter;
+  for (range_iter = ranges.begin(); range_iter != ranges.end(); ++range_iter) {
+    RGWCloneRangeInfo range = *range_iter;
+    vector::iterator next_iter = range_iter;
+
+    // merge ranges
+    while (++next_iter !=  ranges.end()) {
+      RGWCloneRangeInfo& next = *next_iter;
+      if (range.src_ofs + (int64_t)range.len != next.src_ofs ||
+          range.dst_ofs + (int64_t)range.len != next.dst_ofs)
+        break;
+      range_iter = next_iter;
+      range.len += next.len;
+    }
+    if (range.len) {
+      ldout(cct, 20) << "calling op.clone_range(dst_ofs=" << range.dst_ofs << ", src.object=" <<  range.src.object << " range.src_ofs=" << range.src_ofs << " range.len=" << range.len << dendl;
+      if (xattr_cond) {
+        string src_cmp_obj, src_cmp_key;
+        get_obj_bucket_and_oid_key(range.src, bucket, src_cmp_obj, src_cmp_key);
+        op.src_cmpxattr(src_cmp_obj, xattr_cond->first.c_str(),
+                        LIBRADOS_CMPXATTR_OP_EQ, xattr_cond->second);
+      }
+      string src_oid, src_key;
+      get_obj_bucket_and_oid_key(range.src, bucket, src_oid, src_key);
+      if (range.dst_ofs + range.len > size)
+        size = range.dst_ofs + range.len;
+      op.clone_range(range.dst_ofs, src_oid, range.src_ofs, range.len);
+    }
+  }
+  time_t mt;
+  utime_t ut;
+  if (pmtime) {
+    op.mtime(pmtime);
+    ut = utime_t(*pmtime, 0);
+  } else {
+    ut = ceph_clock_now(cct);
+    mt = ut.sec();
+    op.mtime(&mt);
+  }
+
+  string tag;
+  uint64_t epoch = 0;
+  int64_t poolid = io_ctx.get_id();
+  int ret;
+
+  if (update_index) {
+    ret = prepare_update_index(state, bucket, CLS_RGW_OP_ADD, dst_obj, tag);
+    if (ret < 0)
+      goto done;
+  }
+
+  ret = io_ctx.operate(dst_oid, &op);
+
+  epoch = io_ctx.get_last_version();
+
+done:
+  atomic_write_finish(state, ret);
+
+  if (update_index) {
+    if (ret >= 0) {
+      ret = complete_update_index(bucket, dst_obj.object, tag, poolid, epoch, size,
+                                  ut, etag, content_type, &acl_bl, category, NULL);
+    } else {
+      int r = complete_update_index_cancel(bucket, dst_obj.object, tag);
+      if (r < 0) {
+        ldout(cct, 0) << "ERROR: comlete_update_index_cancel() returned r=" << r << dendl;
+      }
+    }
+  }
+
+  return ret;
+}
+
+int RGWRados::clone_objs(void *ctx, rgw_obj& dst_obj,
+                        vector& ranges,
+                        map attrs,
+                        RGWObjCategory category,
+                        time_t *pmtime,
+                        bool truncate_dest,
+                        bool exclusive,
+                        pair *xattr_cond)
+{
+  int r;
+
+  r = clone_objs_impl(ctx, dst_obj, ranges, attrs, category, pmtime, truncate_dest, exclusive, xattr_cond);
+  if (r == -ECANCELED)
+    r = 0;
+
+  return r;
+}
+
+
+int RGWRados::get_obj(void *ctx, RGWObjVersionTracker *objv_tracker, void **handle, rgw_obj& obj,
+                      bufferlist& bl, off_t ofs, off_t end)
+{
+  rgw_bucket bucket;
+  std::string oid, key;
+  rgw_obj read_obj = obj;
+  uint64_t read_ofs = ofs;
+  uint64_t len, read_len;
+  RGWRadosCtx *rctx = static_cast(ctx);
+  RGWRadosCtx *new_ctx = NULL;
+  bool reading_from_head = true;
+  ObjectReadOperation op;
+
+  GetObjState *state = *(GetObjState **)handle;
+  RGWObjState *astate = NULL;
+
+  bool merge_bl = false;
+  bufferlist *pbl = &bl;
+  bufferlist read_bl;
+  uint64_t max_chunk_size;
+
+
+  get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+
+  if (!rctx) {
+    new_ctx = new RGWRadosCtx(this);
+    rctx = new_ctx;
+  }
+
+  int r = get_obj_state(rctx, obj, &astate, NULL);
+  if (r < 0)
+    goto done_ret;
+
+  if (end < 0)
+    len = 0;
+  else
+    len = end - ofs + 1;
+
+  if (astate->has_manifest && astate->manifest.has_tail()) {
+    /* now get the relevant object part */
+    RGWObjManifest::obj_iterator iter = astate->manifest.obj_find(ofs);
+
+    uint64_t stripe_ofs = iter.get_stripe_ofs();
+    read_obj = iter.get_location();
+    len = min(len, iter.get_stripe_size() - (ofs - stripe_ofs));
+    read_ofs = iter.location_ofs() + (ofs - stripe_ofs);
+    reading_from_head = (read_obj == obj);
+
+    if (!reading_from_head) {
+      get_obj_bucket_and_oid_key(read_obj, bucket, oid, key);
+    }
+  }
+
+  r = get_max_chunk_size(bucket, &max_chunk_size);
+  if (r < 0) {
+    ldout(cct, 0) << "ERROR: failed to get max_chunk_size() for bucket " << bucket << dendl;
+    goto done_ret;
+  }
+
+  if (len > max_chunk_size)
+    len = max_chunk_size;
+
+
+  state->io_ctx.locator_set_key(key);
+
+  read_len = len;
+
+  if (reading_from_head) {
+    /* only when reading from the head object do we need to do the atomic test */
+    r = append_atomic_test(rctx, read_obj, op, &astate);
+    if (r < 0)
+      goto done_ret;
+
+    if (astate && astate->prefetch_data) {
+      if (!ofs && astate->data.length() >= len) {
+        bl = astate->data;
+        goto done;
+      }
+
+      if (ofs < astate->data.length()) {
+        unsigned copy_len = min((uint64_t)astate->data.length() - ofs, len);
+        astate->data.copy(ofs, copy_len, bl);
+        read_len -= copy_len;
+        read_ofs += copy_len;
+        if (!read_len)
+	  goto done;
+
+        merge_bl = true;
+        pbl = &read_bl;
+      }
+    }
+  }
+
+  if (objv_tracker) {
+    objv_tracker->prepare_op_for_read(&op);
+  }
+
+
+  ldout(cct, 20) << "rados->read obj-ofs=" << ofs << " read_ofs=" << read_ofs << " read_len=" << read_len << dendl;
+  op.read(read_ofs, read_len, pbl, NULL);
+
+  r = state->io_ctx.operate(oid, &op, NULL);
+  ldout(cct, 20) << "rados->read r=" << r << " bl.length=" << bl.length() << dendl;
+
+  if (merge_bl)
+    bl.append(read_bl);
+
+done:
+  if (bl.length() > 0) {
+    r = bl.length();
+  }
+  if (r < 0 || !len || ((off_t)(ofs + len - 1) == end)) {
+    finish_get_obj(handle);
+  }
+
+done_ret:
+  delete new_ctx;
+
+  return r;
+}
+
+struct get_obj_data;
+
+struct get_obj_aio_data {
+  struct get_obj_data *op_data;
+  off_t ofs;
+  off_t len;
+};
+
+struct get_obj_io {
+  off_t len;
+  bufferlist bl;
+};
+
+static void _get_obj_aio_completion_cb(completion_t cb, void *arg);
+
+struct get_obj_data : public RefCountedObject {
+  CephContext *cct;
+  RGWRados *rados;
+  void *ctx;
+  IoCtx io_ctx;
+  map io_map;
+  map completion_map;
+  uint64_t total_read;
+  Mutex lock;
+  Mutex data_lock;
+  list aio_data;
+  RGWGetDataCB *client_cb;
+  atomic_t cancelled;
+  atomic_t err_code;
+  Throttle throttle;
+  list read_list;
+
+  get_obj_data(CephContext *_cct)
+    : cct(_cct),
+      rados(NULL), ctx(NULL),
+      total_read(0), lock("get_obj_data"), data_lock("get_obj_data::data_lock"),
+      client_cb(NULL),
+      throttle(cct, "get_obj_data", cct->_conf->rgw_get_obj_window_size, false) {}
+  virtual ~get_obj_data() { } 
+  void set_cancelled(int r) {
+    cancelled.set(1);
+    err_code.set(r);
+  }
+
+  bool is_cancelled() {
+    return cancelled.read() == 1;
+  }
+
+  int get_err_code() {
+    return err_code.read();
+  }
+
+  int wait_next_io(bool *done) {
+    lock.Lock();
+    map::iterator iter = completion_map.begin();
+    if (iter == completion_map.end()) {
+      *done = true;
+      lock.Unlock();
+      return 0;
+    }
+    off_t cur_ofs = iter->first;
+    librados::AioCompletion *c = iter->second;
+    lock.Unlock();
+
+    c->wait_for_complete_and_cb();
+    int r = c->get_return_value();
+
+    lock.Lock();
+    completion_map.erase(cur_ofs);
+
+    if (completion_map.empty()) {
+      *done = true;
+    }
+    lock.Unlock();
+
+    c->release();
+    
+    return r;
+  }
+
+  void add_io(off_t ofs, off_t len, bufferlist **pbl, AioCompletion **pc) {
+    Mutex::Locker l(lock);
+
+    get_obj_io& io = io_map[ofs];
+    *pbl = &io.bl;
+
+    struct get_obj_aio_data aio;
+    aio.ofs = ofs;
+    aio.len = len;
+    aio.op_data = this;
+
+    aio_data.push_back(aio);
+
+    struct get_obj_aio_data *paio_data =  &aio_data.back(); /* last element */
+
+    librados::AioCompletion *c = librados::Rados::aio_create_completion((void *)paio_data, _get_obj_aio_completion_cb, NULL);
+    completion_map[ofs] = c;
+
+    *pc = c;
+
+    /* we have a reference per IO, plus one reference for the calling function.
+     * reference is dropped for each callback, plus when we're done iterating
+     * over the parts */
+    get();
+  }
+
+  void cancel_io(off_t ofs) {
+    ldout(cct, 20) << "get_obj_data::cancel_io() ofs=" << ofs << dendl;
+    lock.Lock();
+    map::iterator iter = completion_map.find(ofs);
+    if (iter != completion_map.end()) {
+      AioCompletion *c = iter->second;
+      c->release();
+      completion_map.erase(ofs);
+      io_map.erase(ofs);
+    }
+    lock.Unlock();
+
+    /* we don't drop a reference here -- e.g., not calling d->put(), because we still
+     * need IoCtx to live, as io callback may still be called
+     */
+  }
+
+  void cancel_all_io() {
+    ldout(cct, 20) << "get_obj_data::cancel_all_io()" << dendl;
+    Mutex::Locker l(lock);
+    for (map::iterator iter = completion_map.begin();
+         iter != completion_map.end(); ++iter) {
+      librados::AioCompletion  *c = iter->second;
+      c->release();
+    }
+  }
+
+  int get_complete_ios(off_t ofs, list& bl_list) {
+    Mutex::Locker l(lock);
+
+    map::iterator liter = io_map.begin();
+
+    if (liter == io_map.end() ||
+        liter->first != ofs) {
+      return 0;
+    }
+
+    map::iterator aiter;
+    aiter = completion_map.find(ofs);
+    if (aiter == completion_map.end()) {
+    /* completion map does not hold this io, it was cancelled */
+      return 0;
+    }
+
+    AioCompletion *completion = aiter->second;
+    int r = completion->get_return_value();
+    if (r < 0)
+      return r;
+
+    for (; aiter != completion_map.end(); ++aiter) {
+      completion = aiter->second;
+      if (!completion->is_complete()) {
+        /* reached a request that is not yet complete, stop */
+        break;
+      }
+
+      r = completion->get_return_value();
+      if (r < 0) {
+        set_cancelled(r); /* mark it as cancelled, so that we don't continue processing next operations */
+        return r;
+      }
+
+      total_read += r;
+
+      map::iterator old_liter = liter++;
+      bl_list.push_back(old_liter->second.bl);
+      io_map.erase(old_liter);
+    }
+
+    return 0;
+  }
+};
+
+static int _get_obj_iterate_cb(rgw_obj& obj, off_t obj_ofs, off_t read_ofs, off_t len, bool is_head_obj, RGWObjState *astate, void *arg)
+{
+  struct get_obj_data *d = (struct get_obj_data *)arg;
+
+  return d->rados->get_obj_iterate_cb(d->ctx, astate, obj, obj_ofs, read_ofs, len, is_head_obj, arg);
+}
+
+static void _get_obj_aio_completion_cb(completion_t cb, void *arg)
+{
+  struct get_obj_aio_data *aio_data = (struct get_obj_aio_data *)arg;
+  struct get_obj_data *d = aio_data->op_data;
+
+  d->rados->get_obj_aio_completion_cb(cb, arg);
+}
+
+
+void RGWRados::get_obj_aio_completion_cb(completion_t c, void *arg)
+{
+  struct get_obj_aio_data *aio_data = (struct get_obj_aio_data *)arg;
+  struct get_obj_data *d = aio_data->op_data;
+  off_t ofs = aio_data->ofs;
+  off_t len = aio_data->len;
+
+  list bl_list;
+  list::iterator iter;
+  int r;
+
+  ldout(cct, 20) << "get_obj_aio_completion_cb: io completion ofs=" << ofs << " len=" << len << dendl;
+  d->throttle.put(len);
+
+  r = rados_aio_get_return_value(c);
+  if (r < 0) {
+    ldout(cct, 0) << "ERROR: got unexpected error when trying to read object: " << r << dendl;
+    d->set_cancelled(r);
+    goto done;
+  }
+
+  if (d->is_cancelled()) {
+    goto done;
+  }
+
+  d->data_lock.Lock();
+
+  r = d->get_complete_ios(ofs, bl_list);
+  if (r < 0) {
+    goto done_unlock;
+  }
+
+  d->read_list.splice(d->read_list.end(), bl_list);
+
+done_unlock:
+  d->data_lock.Unlock();
+done:
+  d->put();
+  return;
+}
+
+int RGWRados::flush_read_list(struct get_obj_data *d)
+{
+  d->data_lock.Lock();
+  list l;
+  l.swap(d->read_list);
+  d->get();
+  d->read_list.clear();
+
+  d->data_lock.Unlock();
+
+  int r = 0;
+
+  list::iterator iter;
+  for (iter = l.begin(); iter != l.end(); ++iter) {
+    bufferlist& bl = *iter;
+    r = d->client_cb->handle_data(bl, 0, bl.length());
+    if (r < 0) {
+      dout(0) << "ERROR: flush_read_list(): d->client_c->handle_data() returned " << r << dendl;
+      break;
+    }
+  }
+
+  d->data_lock.Lock();
+  d->put();
+  if (r < 0) {
+    d->set_cancelled(r);
+  }
+  d->data_lock.Unlock();
+  return r;
+}
+
+int RGWRados::get_obj_iterate_cb(void *ctx, RGWObjState *astate,
+		         rgw_obj& obj,
+			 off_t obj_ofs,
+                         off_t read_ofs, off_t len,
+                         bool is_head_obj, void *arg)
+{
+  RGWRadosCtx *rctx = static_cast(ctx);
+  ObjectReadOperation op;
+  struct get_obj_data *d = (struct get_obj_data *)arg;
+  string oid, key;
+  rgw_bucket bucket;
+  bufferlist *pbl;
+  AioCompletion *c;
+
+  int r;
+
+  if (is_head_obj) {
+    /* only when reading from the head object do we need to do the atomic test */
+    r = append_atomic_test(rctx, obj, op, &astate);
+    if (r < 0)
+      return r;
+
+    if (astate &&
+        obj_ofs < astate->data.length()) {
+      unsigned chunk_len = min((uint64_t)astate->data.length() - obj_ofs, (uint64_t)len);
+
+      d->data_lock.Lock();
+      r = d->client_cb->handle_data(astate->data, obj_ofs, chunk_len);
+      d->data_lock.Unlock();
+      if (r < 0)
+        return r;
+
+      d->lock.Lock();
+      d->total_read += chunk_len;
+      d->lock.Unlock();
+	
+      len -= chunk_len;
+      read_ofs += chunk_len;
+      obj_ofs += chunk_len;
+      if (!len)
+	  return 0;
+    }
+  }
+
+  r = flush_read_list(d);
+  if (r < 0)
+    return r;
+
+  get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+
+  d->throttle.get(len);
+  if (d->is_cancelled()) {
+    return d->get_err_code();
+  }
+
+  /* add io after we check that we're not cancelled, otherwise we're going to have trouble
+   * cleaning up
+   */
+  d->add_io(obj_ofs, len, &pbl, &c);
+
+  ldout(cct, 20) << "rados->get_obj_iterate_cb oid=" << oid << " obj-ofs=" << obj_ofs << " read_ofs=" << read_ofs << " len=" << len << dendl;
+  op.read(read_ofs, len, pbl, NULL);
+
+  librados::IoCtx io_ctx(d->io_ctx);
+  io_ctx.locator_set_key(key);
+
+  r = io_ctx.aio_operate(oid, c, &op, NULL);
+  ldout(cct, 20) << "rados->aio_operate r=" << r << " bl.length=" << pbl->length() << dendl;
+  if (r < 0)
+    goto done_err;
+
+  return 0;
+
+done_err:
+  ldout(cct, 20) << "cancelling io r=" << r << " obj_ofs=" << obj_ofs << dendl;
+  d->set_cancelled(r);
+  d->cancel_io(obj_ofs);
+
+  return r;
+}
+
+int RGWRados::get_obj_iterate(void *ctx, void **handle, rgw_obj& obj,
+                              off_t ofs, off_t end,
+			      RGWGetDataCB *cb)
+{
+  struct get_obj_data *data = new get_obj_data(cct);
+  bool done = false;
+
+  GetObjState *state = *(GetObjState **)handle;
+
+  data->rados = this;
+  data->ctx = ctx;
+  data->io_ctx.dup(state->io_ctx);
+  data->client_cb = cb;
+
+  int r = iterate_obj(ctx, obj, ofs, end, cct->_conf->rgw_get_obj_max_req_size, _get_obj_iterate_cb, (void *)data);
+  if (r < 0) {
+    data->cancel_all_io();
+    goto done;
+  }
+
+  while (!done) {
+    r = data->wait_next_io(&done);
+    if (r < 0) {
+      dout(10) << "get_obj_iterate() r=" << r << ", canceling all io" << dendl;
+      data->cancel_all_io();
+      break;
+    }
+    r = flush_read_list(data);
+    if (r < 0) {
+      dout(10) << "get_obj_iterate() r=" << r << ", canceling all io" << dendl;
+      data->cancel_all_io();
+      break;
+    }
+  }
+
+done:
+  data->put();
+  return r;
+}
+
+void RGWRados::finish_get_obj(void **handle)
+{
+  if (*handle) {
+    GetObjState *state = *(GetObjState **)handle;
+    delete state;
+    *handle = NULL;
+  }
+}
+
+int RGWRados::iterate_obj(void *ctx, rgw_obj& obj,
+                          off_t ofs, off_t end,
+			  uint64_t max_chunk_size,
+			  int (*iterate_obj_cb)(rgw_obj&, off_t, off_t, off_t, bool, RGWObjState *, void *),
+	                  void *arg)
+{
+  rgw_bucket bucket;
+  rgw_obj read_obj = obj;
+  uint64_t read_ofs = ofs;
+  uint64_t len;
+  RGWRadosCtx *rctx = static_cast(ctx);
+  RGWRadosCtx *new_ctx = NULL;
+  bool reading_from_head = true;
+  RGWObjState *astate = NULL;
+
+  if (!rctx) {
+    new_ctx = new RGWRadosCtx(this);
+    rctx = new_ctx;
+  }
+
+  int r = get_obj_state(rctx, obj, &astate, NULL);
+  if (r < 0)
+    goto done_err;
+
+  if (end < 0)
+    len = 0;
+  else
+    len = end - ofs + 1;
+
+  if (astate->has_manifest) {
+    /* now get the relevant object stripe */
+    RGWObjManifest::obj_iterator iter = astate->manifest.obj_find(ofs);
+
+    RGWObjManifest::obj_iterator obj_end = astate->manifest.obj_end();
+
+    for (; iter != obj_end && ofs <= end; ++iter) {
+      off_t stripe_ofs = iter.get_stripe_ofs();
+      off_t next_stripe_ofs = stripe_ofs + iter.get_stripe_size();
+
+      while (ofs < next_stripe_ofs && ofs <= end) {
+        read_obj = iter.get_location();
+        uint64_t read_len = min(len, iter.get_stripe_size() - (ofs - stripe_ofs));
+        read_ofs = iter.location_ofs() + (ofs - stripe_ofs);
+
+        if (read_len > max_chunk_size) {
+          read_len = max_chunk_size;
+        }
+
+        reading_from_head = (read_obj == obj);
+        r = iterate_obj_cb(read_obj, ofs, read_ofs, read_len, reading_from_head, astate, arg);
+	if (r < 0)
+	  goto done_err;
+
+	len -= read_len;
+        ofs += read_len;
+      }
+    }
+  } else {
+    while (ofs <= end) {
+      uint64_t read_len = min(len, max_chunk_size);
+
+      r = iterate_obj_cb(obj, ofs, ofs, read_len, reading_from_head, astate, arg);
+      if (r < 0)
+	goto done_err;
+
+      len -= read_len;
+      ofs += read_len;
+    }
+  }
+
+  return 0;
+
+done_err:
+  delete new_ctx;
+  return r;
+}
+
+/* a simple object read */
+int RGWRados::read(void *ctx, rgw_obj& obj, off_t ofs, size_t size, bufferlist& bl)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+  RGWRadosCtx *rctx = static_cast(ctx);
+  RGWObjState *astate = NULL;
+
+  ObjectReadOperation op;
+
+  r = append_atomic_test(rctx, obj, op, &astate);
+  if (r < 0)
+    return r;
+
+  op.read(ofs, size, &bl, NULL);
+
+  return ref.ioctx.operate(ref.oid, &op, NULL);
+}
+
+int RGWRados::obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime, uint64_t *epoch, map *attrs, bufferlist *first_chunk,
+                       RGWObjVersionTracker *objv_tracker)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  map unfiltered_attrset;
+  uint64_t size = 0;
+  time_t mtime = 0;
+
+  ObjectReadOperation op;
+  if (objv_tracker) {
+    objv_tracker->prepare_op_for_read(&op);
+  }
+  op.getxattrs(&unfiltered_attrset, NULL);
+  op.stat(&size, &mtime, NULL);
+  if (first_chunk) {
+    op.read(0, cct->_conf->rgw_max_chunk_size, first_chunk, NULL);
+  }
+  bufferlist outbl;
+  r = ref.ioctx.operate(ref.oid, &op, &outbl);
+
+  if (epoch)
+    *epoch = ref.ioctx.get_last_version();
+
+  if (r < 0)
+    return r;
+
+  map attrset;
+  map::iterator iter;
+  string check_prefix = RGW_ATTR_PREFIX;
+  for (iter = unfiltered_attrset.lower_bound(check_prefix);
+       iter != unfiltered_attrset.end(); ++iter) {
+    if (!str_startswith(iter->first, check_prefix))
+      break;
+    attrset[iter->first] = iter->second;
+  }
+
+  if (psize)
+    *psize = size;
+  if (pmtime)
+    *pmtime = mtime;
+  if (attrs)
+    *attrs = attrset;
+
+  return 0;
+}
+
+int RGWRados::get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map& stats,
+                               string *max_marker)
+{
+  rgw_bucket_dir_header header;
+  int r = cls_bucket_head(bucket, header);
+  if (r < 0)
+    return r;
+
+  stats.clear();
+
+  translate_raw_stats(header, stats);
+
+  *bucket_ver = header.ver;
+  *master_ver = header.master_ver;
+
+  if (max_marker)
+    *max_marker = header.max_marker;
+
+  return 0;
+}
+
+class RGWGetBucketStatsContext : public RGWGetDirHeader_CB {
+  RGWGetBucketStats_CB *cb;
+
+public:
+  RGWGetBucketStatsContext(RGWGetBucketStats_CB *_cb) : cb(_cb) {}
+  void handle_response(int r, rgw_bucket_dir_header& header) {
+    map stats;
+
+    if (r >= 0) {
+      translate_raw_stats(header, stats);
+      cb->set_response(header.ver, header.master_ver, &stats, header.max_marker);
+    }
+
+    cb->handle_response(r);
+
+    cb->put();
+  }
+};
+
+int RGWRados::get_bucket_stats_async(rgw_bucket& bucket, RGWGetBucketStats_CB *ctx)
+{
+  RGWGetBucketStatsContext *get_ctx = new RGWGetBucketStatsContext(ctx);
+  int r = cls_bucket_head_async(bucket, get_ctx);
+  if (r < 0) {
+    ctx->put();
+    delete get_ctx;
+    return r;
+  }
+
+  return 0;
+}
+
+class RGWGetUserStatsContext : public RGWGetUserHeader_CB {
+  RGWGetUserStats_CB *cb;
+
+public:
+  RGWGetUserStatsContext(RGWGetUserStats_CB *_cb) : cb(_cb) {}
+  void handle_response(int r, cls_user_header& header) {
+    cls_user_stats& hs = header.stats;
+    if (r >= 0) {
+      RGWStorageStats stats;
+
+      stats.num_kb = (hs.total_bytes + 1023) / 1024;
+      stats.num_kb_rounded = (hs.total_bytes_rounded + 1023) / 1024;
+      stats.num_objects = hs.total_entries;
+
+      cb->set_response(stats);
+    }
+
+    cb->handle_response(r);
+
+    cb->put();
+  }
+};
+
+int RGWRados::get_user_stats(const string& user, RGWStorageStats& stats)
+{
+  cls_user_header header;
+  int r = cls_user_get_header(user, &header);
+  if (r < 0)
+    return r;
+
+  cls_user_stats& hs = header.stats;
+
+  stats.num_kb = (hs.total_bytes + 1023) / 1024;
+  stats.num_kb_rounded = (hs.total_bytes_rounded + 1023) / 1024;
+  stats.num_objects = hs.total_entries;
+
+  return 0;
+}
+
+int RGWRados::get_user_stats_async(const string& user, RGWGetUserStats_CB *ctx)
+{
+  RGWGetUserStatsContext *get_ctx = new RGWGetUserStatsContext(ctx);
+  int r = cls_user_get_header_async(user, get_ctx);
+  if (r < 0) {
+    ctx->put();
+    delete get_ctx;
+    return r;
+  }
+
+  return 0;
+}
+
+void RGWRados::get_bucket_instance_entry(rgw_bucket& bucket, string& entry)
+{
+  entry = bucket.name + ":" + bucket.bucket_id;
+}
+
+void RGWRados::get_bucket_meta_oid(rgw_bucket& bucket, string& oid)
+{
+  string entry;
+  get_bucket_instance_entry(bucket, entry);
+  oid = RGW_BUCKET_INSTANCE_MD_PREFIX + entry;
+}
+
+void RGWRados::get_bucket_instance_obj(rgw_bucket& bucket, rgw_obj& obj)
+{
+  if (!bucket.oid.empty()) {
+    obj.init(zone.domain_root, bucket.oid);
+  } else {
+    string oid;
+    get_bucket_meta_oid(bucket, oid);
+    obj.init(zone.domain_root, oid);
+  }
+}
+
+int RGWRados::get_bucket_instance_info(void *ctx, const string& meta_key, RGWBucketInfo& info,
+                                       time_t *pmtime, map *pattrs)
+{
+  int pos = meta_key.find(':');
+  if (pos < 0) {
+    return -EINVAL;
+  }
+  string oid = RGW_BUCKET_INSTANCE_MD_PREFIX + meta_key;
+
+  return get_bucket_instance_from_oid(ctx, oid, info, pmtime, pattrs);
+}
+
+int RGWRados::get_bucket_instance_info(void *ctx, rgw_bucket& bucket, RGWBucketInfo& info,
+                                       time_t *pmtime, map *pattrs)
+{
+  string oid;
+  if (!bucket.oid.empty()) {
+    get_bucket_meta_oid(bucket, oid);
+  } else {
+    oid = bucket.oid;
+  }
+
+  return get_bucket_instance_from_oid(ctx, oid, info, pmtime, pattrs);
+}
+
+int RGWRados::get_bucket_instance_from_oid(void *ctx, string& oid, RGWBucketInfo& info,
+                                           time_t *pmtime, map *pattrs)
+{
+  ldout(cct, 20) << "reading from " << zone.domain_root << ":" << oid << dendl;
+
+  bufferlist epbl;
+
+  int ret = rgw_get_system_obj(this, ctx, zone.domain_root, oid, epbl, &info.objv_tracker, pmtime, pattrs);
+  if (ret < 0) {
+    return ret;
+  }
+
+  bufferlist::iterator iter = epbl.begin();
+  try {
+    ::decode(info, iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: could not decode buffer info, caught buffer::error" << dendl;
+    return -EIO;
+  }
+  info.bucket.oid = oid;
+  return 0;
+}
+
+int RGWRados::get_bucket_entrypoint_info(void *ctx, const string& bucket_name,
+                                         RGWBucketEntryPoint& entry_point,
+                                         RGWObjVersionTracker *objv_tracker,
+                                         time_t *pmtime,
+                                         map *pattrs)
+{
+  bufferlist bl;
+
+  int ret = rgw_get_system_obj(this, ctx, zone.domain_root, bucket_name, bl, objv_tracker, pmtime, pattrs);
+  if (ret < 0) {
+    return ret;
+  }
+
+  bufferlist::iterator iter = bl.begin();
+  try {
+    ::decode(entry_point, iter);
+  } catch (buffer::error& err) {
+    ldout(cct, 0) << "ERROR: could not decode buffer info, caught buffer::error" << dendl;
+    return -EIO;
+  }
+  return 0;
+}
+
+int RGWRados::convert_old_bucket_info(void *ctx, string& bucket_name)
+{
+  RGWBucketEntryPoint entry_point;
+  time_t ep_mtime;
+  RGWObjVersionTracker ot;
+  map attrs;
+  RGWBucketInfo info;
+
+  ldout(cct, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket_name << dendl;
+
+  int ret = get_bucket_entrypoint_info(ctx, bucket_name, entry_point, &ot, &ep_mtime, &attrs);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: get_bucket_entrypont_info() returned " << ret << " bucket=" << bucket_name << dendl;
+    return ret;
+  }
+
+  if (!entry_point.has_bucket_info) {
+    /* already converted! */
+    return 0;
+  }
+
+  info = entry_point.old_bucket_info;
+  info.bucket.oid = bucket_name;
+  info.ep_objv = ot.read_version;
+
+  ot.generate_new_write_ver(cct);
+
+  ret = put_linked_bucket_info(info, false, ep_mtime, &ot.write_version, &attrs, true);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret << dendl;
+  }
+
+  return 0;
+}
+
+int RGWRados::get_bucket_info(void *ctx, const string& bucket_name, RGWBucketInfo& info,
+                              time_t *pmtime, map *pattrs)
+{
+  bufferlist bl;
+
+  RGWBucketEntryPoint entry_point;
+  time_t ep_mtime;
+  RGWObjVersionTracker ot;
+  int ret = get_bucket_entrypoint_info(ctx, bucket_name, entry_point, &ot, &ep_mtime, pattrs);
+  if (ret < 0) {
+    info.bucket.name = bucket_name; /* only init this field */
+    return ret;
+  }
+
+  if (entry_point.has_bucket_info) {
+    info = entry_point.old_bucket_info;
+    info.bucket.oid = bucket_name;
+    info.ep_objv = ot.read_version;
+    ldout(cct, 20) << "rgw_get_bucket_info: old bucket info, bucket=" << info.bucket << " owner " << info.owner << dendl;
+    return 0;
+  }
+
+  /* data is in the bucket instance object, we need to get attributes from there, clear everything
+   * that we got
+   */
+  if (pattrs) {
+    pattrs->clear();
+  }
+
+  ldout(cct, 20) << "rgw_get_bucket_info: bucket instance: " << entry_point.bucket << dendl;
+
+  if (pattrs)
+    pattrs->clear();
+
+  /* read bucket instance info */
+
+  string oid;
+  get_bucket_meta_oid(entry_point.bucket, oid);
+
+  ret = get_bucket_instance_from_oid(ctx, oid, info, pmtime, pattrs);
+  info.ep_objv = ot.read_version;
+  if (ret < 0) {
+    info.bucket.name = bucket_name;
+    return ret;
+  }
+  return 0;
+}
+
+int RGWRados::put_bucket_entrypoint_info(const string& bucket_name, RGWBucketEntryPoint& entry_point,
+                                         bool exclusive, RGWObjVersionTracker& objv_tracker, time_t mtime,
+                                         map *pattrs)
+{
+  bufferlist epbl;
+  ::encode(entry_point, epbl);
+  return rgw_bucket_store_info(this, bucket_name, epbl, exclusive, pattrs, &objv_tracker, mtime);
+}
+
+int RGWRados::put_bucket_instance_info(RGWBucketInfo& info, bool exclusive,
+                              time_t mtime, map *pattrs)
+{
+  info.has_instance_obj = true;
+  bufferlist bl;
+
+  ::encode(info, bl);
+
+  string key;
+  get_bucket_instance_entry(info.bucket, key); /* when we go through meta api, we don't use oid directly */
+  return rgw_bucket_instance_store_info(this, key, bl, exclusive, pattrs, &info.objv_tracker, mtime);
+}
+
+int RGWRados::put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, time_t mtime, obj_version *pep_objv,
+                                     map *pattrs, bool create_entry_point)
+{
+  bufferlist bl;
+
+  bool create_head = !info.has_instance_obj || create_entry_point;
+
+  int ret = put_bucket_instance_info(info, exclusive, mtime, pattrs);
+  if (ret < 0) {
+    return ret;
+  }
+
+  if (!create_head)
+    return 0; /* done! */
+
+  RGWBucketEntryPoint entry_point;
+  entry_point.bucket = info.bucket;
+  entry_point.owner = info.owner;
+  entry_point.creation_time = info.creation_time;
+  entry_point.linked = true;
+  RGWObjVersionTracker ot;
+  if (pep_objv && !pep_objv->tag.empty()) {
+    ot.write_version = *pep_objv;
+  } else {
+    ot.generate_new_write_ver(cct);
+    if (pep_objv) {
+      *pep_objv = ot.write_version;
+    }
+  }
+  ret = put_bucket_entrypoint_info(info.bucket.name, entry_point, exclusive, ot, mtime, NULL); 
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWRados::omap_get_vals(rgw_obj& obj, bufferlist& header, const string& marker, uint64_t count, std::map& m)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  r = ref.ioctx.omap_get_vals(ref.oid, marker, count, &m);
+  if (r < 0)
+    return r;
+
+  return 0;
+ 
+}
+
+int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map& m)
+{
+  string start_after;
+
+  return omap_get_vals(obj, header, start_after, (uint64_t)-1, m);
+}
+
+int RGWRados::omap_set(rgw_obj& obj, std::string& key, bufferlist& bl)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+  ldout(cct, 15) << "omap_set bucket=" << bucket << " oid=" << ref.oid << " key=" << key << dendl;
+
+  map m;
+  m[key] = bl;
+
+  r = ref.ioctx.omap_set(ref.oid, m);
+
+  return r;
+}
+
+int RGWRados::omap_set(rgw_obj& obj, std::map& m)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  r = ref.ioctx.omap_set(ref.oid, m);
+
+  return r;
+}
+
+int RGWRados::omap_del(rgw_obj& obj, const std::string& key)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  set k;
+  k.insert(key);
+
+  r = ref.ioctx.omap_rm_keys(ref.oid, k);
+  return r;
+}
+
+int RGWRados::update_containers_stats(map& m)
+{
+  map::iterator iter;
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    RGWBucketEnt& ent = iter->second;
+    rgw_bucket& bucket = ent.bucket;
+
+    rgw_bucket_dir_header header;
+    int r = cls_bucket_head(bucket, header);
+    if (r < 0)
+      return r;
+
+    ent.count = 0;
+    ent.size = 0;
+
+    RGWObjCategory category = main_category;
+    map::iterator iter = header.stats.find((uint8_t)category);
+    if (iter != header.stats.end()) {
+      struct rgw_bucket_category_stats& stats = iter->second;
+      ent.count = stats.num_entries;
+      ent.size = stats.total_size;
+      ent.size_rounded = stats.total_size_rounded;
+    }
+  }
+
+  return m.size();
+}
+
+int RGWRados::append_async(rgw_obj& obj, size_t size, bufferlist& bl)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+  librados::AioCompletion *completion = rados->aio_create_completion(NULL, NULL, NULL);
+
+  r = ref.ioctx.aio_append(ref.oid, completion, bl, size);
+  completion->release();
+  return r;
+}
+
+int RGWRados::distribute(const string& key, bufferlist& bl)
+{
+  /*
+   * we were called before watch was initialized. This can only happen if we're updating some system
+   * config object (e.g., zone info) during init. Don't try to distribute the cache info for these
+   * objects, they're currently only read on startup anyway.
+   */
+  if (!watch_initialized)
+    return 0;
+
+  string notify_oid;
+  pick_control_oid(key, notify_oid);
+
+  ldout(cct, 10) << "distributing notification oid=" << notify_oid << " bl.length()=" << bl.length() << dendl;
+  int r = control_pool_ctx.notify(notify_oid, 0, bl);
+  return r;
+}
+
+int RGWRados::pool_iterate_begin(rgw_bucket& bucket, RGWPoolIterCtx& ctx)
+{
+  librados::IoCtx& io_ctx = ctx.io_ctx;
+  librados::ObjectIterator& iter = ctx.iter;
+
+  int r = open_bucket_data_ctx(bucket, io_ctx);
+  if (r < 0)
+    return r;
+
+  iter = io_ctx.objects_begin();
+
+  return 0;
+}
+
+int RGWRados::pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector& objs,
+                           bool *is_truncated, RGWAccessListFilter *filter)
+{
+  librados::IoCtx& io_ctx = ctx.io_ctx;
+  librados::ObjectIterator& iter = ctx.iter;
+
+  if (iter == io_ctx.objects_end())
+    return -ENOENT;
+
+  uint32_t i;
+
+  for (i = 0; i < num && iter != io_ctx.objects_end(); ++i, ++iter) {
+    RGWObjEnt e;
+
+    string oid = iter->first;
+    ldout(cct, 20) << "RGWRados::pool_iterate: got " << oid << dendl;
+
+    // fill it in with initial values; we may correct later
+    if (filter && !filter->filter(oid, oid))
+      continue;
+
+    e.name = oid;
+    objs.push_back(e);
+  }
+
+  if (is_truncated)
+    *is_truncated = (iter != io_ctx.objects_end());
+
+  return objs.size();
+}
+struct RGWAccessListFilterPrefix : public RGWAccessListFilter {
+  string prefix;
+
+  RGWAccessListFilterPrefix(const string& _prefix) : prefix(_prefix) {}
+  virtual bool filter(string& name, string& key) {
+    return (prefix.compare(key.substr(0, prefix.size())) == 0);
+  }
+};
+
+int RGWRados::list_raw_objects(rgw_bucket& pool, const string& prefix_filter,
+			       int max, RGWListRawObjsCtx& ctx, list& oids,
+			       bool *is_truncated)
+{
+  RGWAccessListFilterPrefix filter(prefix_filter);
+
+  if (!ctx.initialized) {
+    int r = pool_iterate_begin(pool, ctx.iter_ctx);
+    if (r < 0) {
+      lderr(cct) << "failed to list objects pool_iterate_begin() returned r=" << r << dendl;
+      return r;
+    }
+    ctx.initialized = true;
+  }
+
+  vector objs;
+  int r = pool_iterate(ctx.iter_ctx, max, objs, is_truncated, &filter);
+  if (r < 0) {
+    lderr(cct) << "failed to list objects pool_iterate returned r=" << r << dendl;
+    return r;
+  }
+
+  vector::iterator iter;
+  for (iter = objs.begin(); iter != objs.end(); ++iter) {
+    oids.push_back(iter->name);
+  }
+
+  return oids.size();
+}
+
+int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t max,
+                                  std::list& result, bool *truncated)
+{
+  result.clear();
+
+  librados::IoCtx index_ctx;
+  string oid;
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  std::list entries;
+  int ret = cls_rgw_bi_log_list(index_ctx, oid, marker, max - result.size(), entries, truncated);
+  if (ret < 0)
+    return ret;
+
+  std::list::iterator iter;
+  for (iter = entries.begin(); iter != entries.end(); ++iter) {
+    result.push_back(*iter);
+  }
+
+  return 0;
+}
+
+int RGWRados::trim_bi_log_entries(rgw_bucket& bucket, string& start_marker, string& end_marker)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  int ret = cls_rgw_bi_log_trim(index_ctx, oid, start_marker, end_marker);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWRados::gc_operate(string& oid, librados::ObjectWriteOperation *op)
+{
+  return gc_pool_ctx.operate(oid, op);
+}
+
+int RGWRados::gc_aio_operate(string& oid, librados::ObjectWriteOperation *op)
+{
+  AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
+  int r = gc_pool_ctx.aio_operate(oid, c, op);
+  c->release();
+  return r;
+}
+
+int RGWRados::gc_operate(string& oid, librados::ObjectReadOperation *op, bufferlist *pbl)
+{
+  return gc_pool_ctx.operate(oid, op, pbl);
+}
+
+int RGWRados::list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated)
+{
+  return gc->list(index, marker, max, expired_only, result, truncated);
+}
+
+int RGWRados::process_gc()
+{
+  return gc->process();
+}
+
+int RGWRados::cls_rgw_init_index(librados::IoCtx& index_ctx, librados::ObjectWriteOperation& op, string& oid)
+{
+  bufferlist in;
+  cls_rgw_bucket_init(op);
+  int r = index_ctx.operate(oid, &op);
+  return r;
+}
+
+int RGWRados::cls_obj_prepare_op(rgw_bucket& bucket, RGWModifyOp op, string& tag,
+                                 string& name, string& locator)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  ObjectWriteOperation o;
+  cls_rgw_bucket_prepare_op(o, op, tag, name, locator, zone_public_config.log_data);
+  r = index_ctx.operate(oid, &o);
+  return r;
+}
+
+int RGWRados::cls_obj_complete_op(rgw_bucket& bucket, RGWModifyOp op, string& tag,
+                                  int64_t pool, uint64_t epoch,
+                                  RGWObjEnt& ent, RGWObjCategory category,
+				  list *remove_objs)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  ObjectWriteOperation o;
+  rgw_bucket_dir_entry_meta dir_meta;
+  dir_meta.size = ent.size;
+  dir_meta.mtime = utime_t(ent.mtime, 0);
+  dir_meta.etag = ent.etag;
+  dir_meta.owner = ent.owner;
+  dir_meta.owner_display_name = ent.owner_display_name;
+  dir_meta.content_type = ent.content_type;
+  dir_meta.category = category;
+
+  rgw_bucket_entry_ver ver;
+  ver.pool = pool;
+  ver.epoch = epoch;
+  cls_rgw_bucket_complete_op(o, op, tag, ver, ent.name, dir_meta, remove_objs, zone_public_config.log_data);
+
+  AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
+  r = index_ctx.aio_operate(oid, c, &o);
+  c->release();
+  return r;
+}
+
+int RGWRados::cls_obj_complete_add(rgw_bucket& bucket, string& tag,
+                                   int64_t pool, uint64_t epoch,
+                                   RGWObjEnt& ent, RGWObjCategory category,
+                                   list *remove_objs)
+{
+  return cls_obj_complete_op(bucket, CLS_RGW_OP_ADD, tag, pool, epoch, ent, category, remove_objs);
+}
+
+int RGWRados::cls_obj_complete_del(rgw_bucket& bucket, string& tag,
+                                   int64_t pool, uint64_t epoch,
+                                   string& name)
+{
+  RGWObjEnt ent;
+  ent.name = name;
+  return cls_obj_complete_op(bucket, CLS_RGW_OP_DEL, tag, pool, epoch, ent, RGW_OBJ_CATEGORY_NONE, NULL);
+}
+
+int RGWRados::cls_obj_complete_cancel(rgw_bucket& bucket, string& tag, string& name)
+{
+  RGWObjEnt ent;
+  ent.name = name;
+  return cls_obj_complete_op(bucket, CLS_RGW_OP_ADD, tag, -1 /* pool id */, 0, ent, RGW_OBJ_CATEGORY_NONE, NULL);
+}
+
+int RGWRados::cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  ObjectWriteOperation o;
+  cls_rgw_bucket_set_tag_timeout(o, timeout);
+
+  r = index_ctx.operate(oid, &o);
+
+  return r;
+}
+
+int RGWRados::cls_bucket_list(rgw_bucket& bucket, string start, string prefix,
+		              uint32_t num, map& m,
+			      bool *is_truncated, string *last_entry,
+			      bool (*force_check_filter)(const string&  name))
+{
+  ldout(cct, 10) << "cls_bucket_list " << bucket << " start " << start << " num " << num << dendl;
+
+  librados::IoCtx index_ctx;
+  string oid;
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  struct rgw_bucket_dir dir;
+  r = cls_rgw_list_op(index_ctx, oid, start, prefix, num, &dir, is_truncated);
+  if (r < 0)
+    return r;
+
+  map::iterator miter;
+  bufferlist updates;
+  for (miter = dir.m.begin(); miter != dir.m.end(); ++miter) {
+    RGWObjEnt e;
+    rgw_bucket_dir_entry& dirent = miter->second;
+
+    // fill it in with initial values; we may correct later
+    e.name = dirent.name;
+    e.size = dirent.meta.size;
+    e.mtime = dirent.meta.mtime;
+    e.etag = dirent.meta.etag;
+    e.owner = dirent.meta.owner;
+    e.owner_display_name = dirent.meta.owner_display_name;
+    e.content_type = dirent.meta.content_type;
+    e.tag = dirent.tag;
+
+    /* oh, that shouldn't happen! */
+    if (e.name.empty()) {
+      ldout(cct, 0) << "WARNING: got empty dirent name, skipping" << dendl;
+      continue;
+    }
+
+    bool force_check = force_check_filter && force_check_filter(dirent.name);
+
+    if (!dirent.exists || !dirent.pending_map.empty() || force_check) {
+      /* there are uncommitted ops. We need to check the current state,
+       * and if the tags are old we need to do cleanup as well. */
+      librados::IoCtx sub_ctx;
+      sub_ctx.dup(index_ctx);
+      r = check_disk_state(sub_ctx, bucket, dirent, e, updates);
+      if (r < 0) {
+        if (r == -ENOENT)
+          continue;
+        else
+          return r;
+      }
+    }
+    m[e.name] = e;
+    ldout(cct, 10) << "RGWRados::cls_bucket_list: got " << e.name << dendl;
+  }
+
+  if (dir.m.size()) {
+    *last_entry = dir.m.rbegin()->first;
+  }
+
+  if (updates.length()) {
+    ObjectWriteOperation o;
+    cls_rgw_suggest_changes(o, updates);
+    // we don't care if we lose suggested updates, send them off blindly
+    AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
+    r = index_ctx.aio_operate(oid, c, &o);
+    c->release();
+  }
+  return m.size();
+}
+
+int RGWRados::cls_obj_usage_log_add(const string& oid, rgw_usage_log_info& info)
+{
+  librados::IoCtx io_ctx;
+
+  const char *usage_log_pool = zone.usage_log_pool.name.c_str();
+  int r = rados->ioctx_create(usage_log_pool, io_ctx);
+  if (r == -ENOENT) {
+    rgw_bucket pool(usage_log_pool);
+    r = create_pool(pool);
+    if (r < 0)
+      return r;
+ 
+    // retry
+    r = rados->ioctx_create(usage_log_pool, io_ctx);
+  }
+  if (r < 0)
+    return r;
+
+  ObjectWriteOperation op;
+  cls_rgw_usage_log_add(op, info);
+
+  r = io_ctx.operate(oid, &op);
+  return r;
+}
+
+int RGWRados::cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
+                                     string& read_iter, map& usage, bool *is_truncated)
+{
+  librados::IoCtx io_ctx;
+
+  *is_truncated = false;
+
+  const char *usage_log_pool = zone.usage_log_pool.name.c_str();
+  int r = rados->ioctx_create(usage_log_pool, io_ctx);
+  if (r < 0)
+    return r;
+
+  r = cls_rgw_usage_log_read(io_ctx, oid, user, start_epoch, end_epoch,
+			     max_entries, read_iter, usage, is_truncated);
+
+  return r;
+}
+
+int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch)
+{
+  librados::IoCtx io_ctx;
+
+  const char *usage_log_pool = zone.usage_log_pool.name.c_str();
+  int r = rados->ioctx_create(usage_log_pool, io_ctx);
+  if (r < 0)
+    return r;
+
+  ObjectWriteOperation op;
+  cls_rgw_usage_log_trim(op, user, start_epoch, end_epoch);
+
+  r = io_ctx.operate(oid, &op);
+  return r;
+}
+
+int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list& oid_list)
+{
+  librados::IoCtx index_ctx;
+  string dir_oid;
+
+  int r = open_bucket_index(bucket, index_ctx, dir_oid);
+  if (r < 0)
+    return r;
+
+  bufferlist updates;
+
+  list::iterator iter;
+
+  for (iter = oid_list.begin(); iter != oid_list.end(); ++iter) {
+    string& oid = *iter;
+    dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket << " oid=" << oid << dendl;
+    rgw_bucket_dir_entry entry;
+    entry.ver.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
+    entry.name = oid;
+    updates.append(CEPH_RGW_REMOVE);
+    ::encode(entry, updates);
+  }
+
+  bufferlist out;
+
+  r = index_ctx.exec(dir_oid, "rgw", "dir_suggest_changes", updates, out);
+
+  return r;
+}
+
+int RGWRados::check_disk_state(librados::IoCtx io_ctx,
+                               rgw_bucket& bucket,
+                               rgw_bucket_dir_entry& list_state,
+                               RGWObjEnt& object,
+                               bufferlist& suggested_updates)
+{
+  rgw_obj obj;
+  std::string oid, key, ns;
+  oid = list_state.name;
+  if (!rgw_obj::strip_namespace_from_object(oid, ns)) {
+    // well crap
+    assert(0 == "got bad object name off disk");
+  }
+  obj.init(bucket, oid, list_state.locator, ns);
+  get_obj_bucket_and_oid_key(obj, bucket, oid, key);
+  io_ctx.locator_set_key(key);
+
+  RGWObjState *astate = NULL;
+  RGWRadosCtx rctx(this);
+  int r = get_obj_state(&rctx, obj, &astate, NULL);
+  if (r < 0)
+    return r;
+
+  list_state.pending_map.clear(); // we don't need this and it inflates size
+  if (!astate->exists) {
+      /* object doesn't exist right now -- hopefully because it's
+       * marked as !exists and got deleted */
+    if (list_state.exists) {
+      /* FIXME: what should happen now? Work out if there are any
+       * non-bad ways this could happen (there probably are, but annoying
+       * to handle!) */
+    }
+    // encode a suggested removal of that key
+    list_state.ver.epoch = io_ctx.get_last_version();
+    list_state.ver.pool = io_ctx.get_id();
+    cls_rgw_encode_suggestion(CEPH_RGW_REMOVE, list_state, suggested_updates);
+    return -ENOENT;
+  }
+
+  string etag;
+  string content_type;
+  ACLOwner owner;
+
+  object.size = astate->size;
+  object.mtime = utime_t(astate->mtime, 0);
+
+  map::iterator iter = astate->attrset.find(RGW_ATTR_ETAG);
+  if (iter != astate->attrset.end()) {
+    etag = iter->second.c_str();
+  }
+  iter = astate->attrset.find(RGW_ATTR_CONTENT_TYPE);
+  if (iter != astate->attrset.end()) {
+    content_type = iter->second.c_str();
+  }
+  iter = astate->attrset.find(RGW_ATTR_ACL);
+  if (iter != astate->attrset.end()) {
+    r = decode_policy(iter->second, &owner);
+    if (r < 0) {
+      dout(0) << "WARNING: could not decode policy for object: " << obj << dendl;
+    }
+  }
+
+  if (astate->has_manifest) {
+    RGWObjManifest::obj_iterator miter;
+    RGWObjManifest& manifest = astate->manifest;
+    for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
+      rgw_obj loc = miter.get_location();
+
+      if (loc.ns == RGW_OBJ_NS_MULTIPART) {
+	dout(10) << "check_disk_state(): removing manifest part from index: " << loc << dendl;
+	r = delete_obj_index(loc);
+	if (r < 0) {
+	  dout(0) << "WARNING: delete_obj_index() returned r=" << r << dendl;
+	}
+      }
+    }
+  }
+
+  object.etag = etag;
+  object.content_type = content_type;
+  object.owner = owner.get_id();
+  object.owner_display_name = owner.get_display_name();
+
+  // encode suggested updates
+  list_state.ver.pool = io_ctx.get_id();
+  list_state.ver.epoch = astate->epoch;
+  list_state.meta.size = object.size;
+  list_state.meta.mtime.set_from_double(double(object.mtime));
+  list_state.meta.category = main_category;
+  list_state.meta.etag = etag;
+  list_state.meta.content_type = content_type;
+  if (astate->obj_tag.length() > 0)
+    list_state.tag = astate->obj_tag.c_str();
+  list_state.meta.owner = owner.get_id();
+  list_state.meta.owner_display_name = owner.get_display_name();
+
+  list_state.exists = true;
+  cls_rgw_encode_suggestion(CEPH_RGW_UPDATE, list_state, suggested_updates);
+  return 0;
+}
+
+int RGWRados::cls_bucket_head(rgw_bucket& bucket, struct rgw_bucket_dir_header& header)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  r = cls_rgw_get_dir_header(index_ctx, oid, &header);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::cls_bucket_head_async(rgw_bucket& bucket, RGWGetDirHeader_CB *ctx)
+{
+  librados::IoCtx index_ctx;
+  string oid;
+  int r = open_bucket_index(bucket, index_ctx, oid);
+  if (r < 0)
+    return r;
+
+  r = cls_rgw_get_dir_header_async(index_ctx, oid, ctx);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::cls_user_get_header(const string& user_id, cls_user_header *header)
+{
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectReadOperation op;
+  int rc;
+  ::cls_user_get_header(op, header, &rc);
+  bufferlist ibl;
+  r = ref.ioctx.operate(ref.oid, &op, &ibl);
+  if (r < 0)
+    return r;
+  if (rc < 0)
+    return rc;
+
+  return 0;
+}
+
+int RGWRados::cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx)
+{
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  r = ::cls_user_get_header_async(ref.ioctx, ref.oid, ctx);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket)
+{
+  rgw_bucket_dir_header header;
+  int r = cls_bucket_head(bucket, header);
+  if (r < 0) {
+    ldout(cct, 20) << "cls_bucket_header() returned " << r << dendl;
+    return r;
+  }
+
+  cls_user_bucket_entry entry;
+
+  bucket.convert(&entry.bucket);
+
+  map::iterator iter = header.stats.begin();
+  for (; iter != header.stats.end(); ++iter) {
+    struct rgw_bucket_category_stats& header_stats = iter->second;
+    entry.size += header_stats.total_size;
+    entry.size_rounded += header_stats.total_size_rounded;
+    entry.count += header_stats.num_entries;
+  }
+
+  list entries;
+  entries.push_back(entry);
+
+  r = cls_user_update_buckets(user_obj, entries, false);
+  if (r < 0) {
+    ldout(cct, 20) << "cls_user_update_buckets() returned " << r << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats)
+{
+  cls_user_bucket_entry entry;
+
+  entry.size = stats.num_kb * 1024;
+  entry.size_rounded = stats.num_kb_rounded * 1024;
+  entry.count += stats.num_objects;
+
+  list entries;
+  entries.push_back(entry);
+
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+
+  int r = cls_user_update_buckets(obj, entries, false);
+  if (r < 0) {
+    ldout(cct, 20) << "cls_user_update_buckets() returned " << r << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWRados::cls_user_list_buckets(rgw_obj& obj,
+                                    const string& in_marker, int max_entries,
+                                    list& entries,
+                                    string *out_marker, bool *truncated)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectReadOperation op;
+  int rc;
+
+  cls_user_bucket_list(op, in_marker, max_entries, entries, out_marker, truncated, &rc);
+  bufferlist ibl;
+  r = ref.ioctx.operate(ref.oid, &op, &ibl);
+  if (r < 0)
+    return r;
+  if (rc < 0)
+    return rc;
+
+  return 0;
+}
+
+int RGWRados::cls_user_update_buckets(rgw_obj& obj, list& entries, bool add)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectWriteOperation op;
+  cls_user_set_buckets(op, entries, add);
+  r = ref.ioctx.operate(ref.oid, &op);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::complete_sync_user_stats(const string& user_id)
+{
+  string buckets_obj_id;
+  rgw_get_buckets_obj(user_id, buckets_obj_id);
+  rgw_obj obj(zone.user_uid_pool, buckets_obj_id);
+  return cls_user_complete_stats_sync(obj);
+}
+
+int RGWRados::cls_user_complete_stats_sync(rgw_obj& obj)
+{
+  rgw_rados_ref ref;
+  rgw_bucket bucket;
+  int r = get_obj_ref(obj, &ref, &bucket);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectWriteOperation op;
+  ::cls_user_complete_stats_sync(op);
+  r = ref.ioctx.operate(ref.oid, &op);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry)
+{
+  list l;
+  l.push_back(entry);
+
+  return cls_user_update_buckets(obj, l, true);
+}
+
+int RGWRados::cls_user_remove_bucket(rgw_obj& obj, const cls_user_bucket& bucket)
+{
+  rgw_bucket b;
+  rgw_rados_ref ref;
+  int r = get_obj_ref(obj, &ref, &b);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectWriteOperation op;
+  ::cls_user_remove_bucket(op, bucket);
+  r = ref.ioctx.operate(ref.oid, &op);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::check_quota(const string& bucket_owner, rgw_bucket& bucket,
+                          RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size)
+{
+  return quota_handler->check_quota(bucket_owner, bucket, user_quota, bucket_quota, 1, obj_size);
+}
+
+class IntentLogNameFilter : public RGWAccessListFilter
+{
+  string prefix;
+  bool filter_exact_date;
+public:
+  IntentLogNameFilter(const char *date, struct tm *tm) : prefix(date) {
+    filter_exact_date = !(tm->tm_hour || tm->tm_min || tm->tm_sec); /* if time was specified and is not 00:00:00
+                                                                       we should look at objects from that date */
+  }
+  bool filter(string& name, string& key) {
+    if (filter_exact_date)
+      return name.compare(prefix) < 0;
+    else
+      return name.compare(0, prefix.size(), prefix) <= 0;
+  }
+};
+
+enum IntentFlags { // bitmask
+  I_DEL_OBJ = 1,
+  I_DEL_DIR = 2,
+};
+
+
+int RGWRados::remove_temp_objects(string date, string time)
+{
+  struct tm tm;
+  
+  string format = "%Y-%m-%d";
+  string datetime = date;
+  if (datetime.size() != 10) {
+    cerr << "bad date format" << std::endl;
+    return -EINVAL;
+  }
+
+  if (!time.empty()) {
+    if (time.size() != 5 && time.size() != 8) {
+      cerr << "bad time format" << std::endl;
+      return -EINVAL;
+    }
+    format.append(" %H:%M:%S");
+    datetime.append(time.c_str());
+  }
+  memset(&tm, 0, sizeof(tm));
+  const char *s = strptime(datetime.c_str(), format.c_str(), &tm);
+  if (s && *s) {
+    cerr << "failed to parse date/time" << std::endl;
+    return -EINVAL;
+  }
+  time_t epoch = mktime(&tm);
+
+  vector objs;
+  
+  int max = 1000;
+  bool is_truncated;
+  IntentLogNameFilter filter(date.c_str(), &tm);
+  RGWPoolIterCtx iter_ctx;
+  int r = pool_iterate_begin(zone.intent_log_pool, iter_ctx);
+  if (r < 0) {
+    cerr << "failed to list objects" << std::endl;
+    return r;
+  }
+  do {
+    objs.clear();
+    r = pool_iterate(iter_ctx, max, objs, &is_truncated, &filter);
+    if (r == -ENOENT)
+      break;
+    if (r < 0) {
+      cerr << "failed to list objects" << std::endl;
+    }
+    vector::iterator iter;
+    for (iter = objs.begin(); iter != objs.end(); ++iter) {
+      process_intent_log(zone.intent_log_pool, (*iter).name, epoch, I_DEL_OBJ | I_DEL_DIR, true);
+    }
+  } while (is_truncated);
+
+  return 0;
+}
+
+int RGWRados::process_intent_log(rgw_bucket& bucket, string& oid,
+				 time_t epoch, int flags, bool purge)
+{
+  cout << "processing intent log " << oid << std::endl;
+  rgw_obj obj(bucket, oid);
+
+  unsigned chunk = 1024 * 1024;
+  off_t pos = 0;
+  bool eof = false;
+  bool complete = true;
+  int ret = 0;
+  int r;
+
+  bufferlist bl;
+  bufferlist::iterator iter;
+  off_t off;
+
+  string no_owner;
+
+  while (!eof || !iter.end()) {
+    off = iter.get_off();
+    if (!eof && (bl.length() - off) < chunk / 2) {
+      bufferlist more;
+      r = read(NULL, obj, pos, chunk, more);
+      if (r < 0) {
+        cerr << "error while reading from " <<  bucket << ":" << oid
+	   << " " << cpp_strerror(-r) << std::endl;
+        return -r;
+      }
+      eof = (more.length() < (off_t)chunk);
+      pos += more.length();
+      bufferlist old;
+      old.substr_of(bl, off, bl.length() - off);
+      bl.clear();
+      bl.claim(old);
+      bl.claim_append(more);
+      iter = bl.begin();
+    }
+    
+    struct rgw_intent_log_entry entry;
+    try {
+      ::decode(entry, iter);
+    } catch (buffer::error& err) {
+      cerr << "failed to decode intent log entry in " << bucket << ":" << oid << std::endl;
+      cerr << "skipping log" << std::endl; // no use to continue
+      ret = -EIO;
+      complete = false;
+      break;
+    }
+    if (entry.op_time.sec() > epoch) {
+      cerr << "skipping entry for obj=" << obj << " entry.op_time=" << entry.op_time.sec() << " requested epoch=" << epoch << std::endl;
+      cerr << "skipping log" << std::endl; // no use to continue
+      complete = false;
+      break;
+    }
+    switch (entry.intent) {
+    case DEL_OBJ:
+      if (!(flags & I_DEL_OBJ)) {
+        complete = false;
+        break;
+      }
+      r = delete_obj(NULL, no_owner, entry.obj);
+      if (r < 0 && r != -ENOENT) {
+        cerr << "failed to remove obj: " << entry.obj << std::endl;
+        complete = false;
+      }
+      break;
+    case DEL_DIR:
+      if (!(flags & I_DEL_DIR)) {
+        complete = false;
+        break;
+      } else {
+        librados::IoCtx index_ctx;
+        string oid;
+        int r = open_bucket_index(entry.obj.bucket, index_ctx, oid);
+        if (r < 0)
+          return r;
+        ObjectWriteOperation op;
+        op.remove();
+        oid.append(entry.obj.bucket.marker);
+        librados::AioCompletion *completion = rados->aio_create_completion(NULL, NULL, NULL);
+        r = index_ctx.aio_operate(oid, completion, &op);
+        completion->release();
+        if (r < 0 && r != -ENOENT) {
+          cerr << "failed to remove bucket: " << entry.obj.bucket << std::endl;
+          complete = false;
+        }
+      }
+      break;
+    default:
+      complete = false;
+    }
+  }
+
+  if (complete) {
+    rgw_obj obj(bucket, oid);
+    cout << "completed intent log: " << obj << (purge ? ", purging it" : "") << std::endl;
+    if (purge) {
+      r = delete_system_obj(NULL, obj);
+      if (r < 0)
+        cerr << "failed to remove obj: " << obj << std::endl;
+    }
+  }
+
+  return ret;
+}
+
+
+void RGWStateLog::oid_str(int shard, string& oid) {
+  oid = RGW_STATELOG_OBJ_PREFIX + module_name + ".";
+  char buf[16];
+  snprintf(buf, sizeof(buf), "%d", shard);
+  oid += buf;
+}
+
+int RGWStateLog::get_shard_num(const string& object) {
+  uint32_t val = ceph_str_hash_linux(object.c_str(), object.length());
+  return val % num_shards;
+}
+
+string RGWStateLog::get_oid(const string& object) {
+  int shard = get_shard_num(object);
+  string oid;
+  oid_str(shard, oid);
+  return oid;
+}
+
+int RGWStateLog::open_ioctx(librados::IoCtx& ioctx) {
+  string pool_name;
+  store->get_log_pool_name(pool_name);
+  int r = store->rados->ioctx_create(pool_name.c_str(), ioctx);
+  if (r < 0) {
+    lderr(store->ctx()) << "ERROR: could not open rados pool" << dendl;
+    return r;
+  }
+  return 0;
+}
+
+int RGWStateLog::store_entry(const string& client_id, const string& op_id, const string& object,
+                  uint32_t state, bufferlist *bl, uint32_t *check_state)
+{
+  if (client_id.empty() ||
+      op_id.empty() ||
+      object.empty()) {
+    ldout(store->ctx(), 0) << "client_id / op_id / object is empty" << dendl;
+  }
+
+  librados::IoCtx ioctx;
+  int r = open_ioctx(ioctx);
+  if (r < 0)
+    return r;
+
+  string oid = get_oid(object);
+
+  librados::ObjectWriteOperation op;
+  if (check_state) {
+    cls_statelog_check_state(op, client_id, op_id, object, *check_state);
+  }
+  utime_t ts = ceph_clock_now(store->ctx());
+  bufferlist nobl;
+  cls_statelog_add(op, client_id, op_id, object, ts, state, (bl ? *bl : nobl));
+  r = ioctx.operate(oid, &op);
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+int RGWStateLog::remove_entry(const string& client_id, const string& op_id, const string& object)
+{
+  if (client_id.empty() ||
+      op_id.empty() ||
+      object.empty()) {
+    ldout(store->ctx(), 0) << "client_id / op_id / object is empty" << dendl;
+  }
+
+  librados::IoCtx ioctx;
+  int r = open_ioctx(ioctx);
+  if (r < 0)
+    return r;
+
+  string oid = get_oid(object);
+
+  librados::ObjectWriteOperation op;
+  cls_statelog_remove_by_object(op, object, op_id);
+  r = ioctx.operate(oid, &op);
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+void RGWStateLog::init_list_entries(const string& client_id, const string& op_id, const string& object,
+                                    void **handle)
+{
+  list_state *state = new list_state;
+  state->client_id = client_id;
+  state->op_id = op_id;
+  state->object = object;
+  if (object.empty()) {
+    state->cur_shard = 0;
+    state->max_shard = num_shards - 1;
+  } else {
+    state->cur_shard = state->max_shard = get_shard_num(object);
+  }
+  *handle = (void *)state;
+}
+
+int RGWStateLog::list_entries(void *handle, int max_entries,
+                              list& entries,
+                              bool *done)
+{
+  list_state *state = static_cast(handle);
+
+  librados::IoCtx ioctx;
+  int r = open_ioctx(ioctx);
+  if (r < 0)
+    return r;
+
+  entries.clear();
+
+  for (; state->cur_shard <= state->max_shard && max_entries > 0; ++state->cur_shard) {
+    string oid;
+    oid_str(state->cur_shard, oid);
+
+    librados::ObjectReadOperation op;
+    list ents;
+    bool truncated;
+    cls_statelog_list(op, state->client_id, state->op_id, state->object, state->marker,
+                      max_entries, ents, &state->marker, &truncated);
+    bufferlist ibl;
+    r = ioctx.operate(oid, &op, &ibl);
+    if (r == -ENOENT) {
+      truncated = false;
+      r = 0;
+    }
+    if (r < 0) {
+      ldout(store->ctx(), 0) << "cls_statelog_list returned " << r << dendl;
+      return r;
+    }
+
+    if (!truncated) {
+      state->marker.clear();
+    }
+
+    max_entries -= ents.size();
+
+    entries.splice(entries.end(), ents);
+
+    if (truncated)
+      break;
+  }
+
+  *done = (state->cur_shard > state->max_shard);
+
+  return 0;
+}
+
+void RGWStateLog::finish_list_entries(void *handle)
+{
+  list_state *state = static_cast(handle);
+  delete state;
+}
+
+void RGWStateLog::dump_entry(const cls_statelog_entry& entry, Formatter *f)
+{
+  f->open_object_section("statelog_entry");
+  f->dump_string("client_id", entry.client_id);
+  f->dump_string("op_id", entry.op_id);
+  f->dump_string("object", entry.object);
+  entry.timestamp.gmtime(f->dump_stream("timestamp"));
+  if (!dump_entry_internal(entry, f)) {
+    f->dump_int("state", entry.state);
+  }
+  f->close_section();
+}
+
+RGWOpState::RGWOpState(RGWRados *_store) : RGWStateLog(_store, _store->ctx()->_conf->rgw_num_zone_opstate_shards, string("obj_opstate"))
+{
+}
+
+bool RGWOpState::dump_entry_internal(const cls_statelog_entry& entry, Formatter *f)
+{
+  string s;
+  switch ((OpState)entry.state) {
+    case OPSTATE_UNKNOWN:
+      s = "unknown";
+      break;
+    case OPSTATE_IN_PROGRESS:
+      s = "in-progress";
+      break;
+    case OPSTATE_COMPLETE:
+      s = "complete";
+      break;
+    case OPSTATE_ERROR:
+      s = "error";
+      break;
+    case OPSTATE_ABORT:
+      s = "abort";
+      break;
+    case OPSTATE_CANCELLED:
+      s = "cancelled";
+      break;
+    default:
+      s = "invalid";
+  }
+  f->dump_string("state", s);
+  return true;
+}
+
+int RGWOpState::state_from_str(const string& s, OpState *state)
+{
+  if (s == "unknown") {
+    *state = OPSTATE_UNKNOWN;
+  } else if (s == "in-progress") {
+    *state = OPSTATE_IN_PROGRESS;
+  } else if (s == "complete") {
+    *state = OPSTATE_COMPLETE;
+  } else if (s == "error") {
+    *state = OPSTATE_ERROR;
+  } else if (s == "abort") {
+    *state = OPSTATE_ABORT;
+  } else if (s == "cancelled") {
+    *state = OPSTATE_CANCELLED;
+  } else {
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+int RGWOpState::set_state(const string& client_id, const string& op_id, const string& object, OpState state)
+{
+  uint32_t s = (uint32_t)state;
+  return store_entry(client_id, op_id, object, s, NULL, NULL);
+}
+
+int RGWOpState::renew_state(const string& client_id, const string& op_id, const string& object, OpState state)
+{
+  uint32_t s = (uint32_t)state;
+  return store_entry(client_id, op_id, object, s, NULL, &s);
+}
+
+RGWOpStateSingleOp::RGWOpStateSingleOp(RGWRados *store, const string& cid, const string& oid,
+                                       const string& obj) : os(store), client_id(cid), op_id(oid), object(obj)
+{
+  cct = store->ctx();
+  cur_state = RGWOpState::OPSTATE_UNKNOWN;
+}
+
+int RGWOpStateSingleOp::set_state(RGWOpState::OpState state) {
+  last_update = ceph_clock_now(cct);
+  cur_state = state;
+  return os.set_state(client_id, op_id, object, state);
+}
+
+int RGWOpStateSingleOp::renew_state() {
+  utime_t now = ceph_clock_now(cct);
+
+  int rate_limit_sec = cct->_conf->rgw_opstate_ratelimit_sec;
+
+  if (rate_limit_sec && now - last_update < rate_limit_sec) {
+    return 0;
+  }
+
+  last_update = now;
+  return os.renew_state(client_id, op_id, object, cur_state);
+}
+
+
+uint64_t RGWRados::instance_id()
+{
+  return rados->get_instance_id();
+}
+
+uint64_t RGWRados::next_bucket_id()
+{
+  Mutex::Locker l(bucket_id_lock);
+  return ++max_bucket_id;
+}
+
+RGWRados *RGWStoreManager::init_storage_provider(CephContext *cct, bool use_gc_thread, bool quota_threads)
+{
+  int use_cache = cct->_conf->rgw_cache_enabled;
+  RGWRados *store = NULL;
+  if (!use_cache) {
+    store = new RGWRados;
+  } else {
+    store = new RGWCache; 
+  }
+
+  if (store->initialize(cct, use_gc_thread, quota_threads) < 0) {
+    delete store;
+    return NULL;
+  }
+
+  return store;
+}
+
+RGWRados *RGWStoreManager::init_raw_storage_provider(CephContext *cct)
+{
+  RGWRados *store = NULL;
+  store = new RGWRados;
+
+  store->set_context(cct);
+
+  if (store->init_rados() < 0) {
+    delete store;
+    return NULL;
+  }
+
+  return store;
+}
+
+void RGWStoreManager::close_storage(RGWRados *store)
+{
+  if (!store)
+    return;
+
+  store->finalize();
+
+  delete store;
+}
+
diff --git a/ceph/src/rgw/rgw_rados.h b/ceph/src/rgw/rgw_rados.h
new file mode 100644
index 00000000..d811b499
--- /dev/null
+++ b/ceph/src/rgw/rgw_rados.h
@@ -0,0 +1,1927 @@
+#ifndef CEPH_RGWRADOS_H
+#define CEPH_RGWRADOS_H
+
+#include "include/rados/librados.hpp"
+#include "include/Context.h"
+#include "common/RefCountedObj.h"
+#include "rgw_common.h"
+#include "cls/rgw/cls_rgw_types.h"
+#include "cls/version/cls_version_types.h"
+#include "cls/log/cls_log_types.h"
+#include "cls/statelog/cls_statelog_types.h"
+#include "rgw_log.h"
+#include "rgw_metadata.h"
+#include "rgw_rest_conn.h"
+
+class RGWWatcher;
+class SafeTimer;
+class ACLOwner;
+class RGWGC;
+
+/* flags for put_obj_meta() */
+#define PUT_OBJ_CREATE      0x01
+#define PUT_OBJ_EXCL        0x02
+#define PUT_OBJ_CREATE_EXCL (PUT_OBJ_CREATE | PUT_OBJ_EXCL)
+
+#define RGW_OBJ_NS_MULTIPART "multipart"
+#define RGW_OBJ_NS_SHADOW    "shadow"
+
+#define RGW_BUCKET_INSTANCE_MD_PREFIX ".bucket.meta."
+
+static inline void prepend_bucket_marker(rgw_bucket& bucket, const string& orig_oid, string& oid)
+{
+  if (bucket.marker.empty() || orig_oid.empty()) {
+    oid = orig_oid;
+  } else {
+    oid = bucket.marker;
+    oid.append("_");
+    oid.append(orig_oid);
+  }
+}
+
+static inline void get_obj_bucket_and_oid_key(const rgw_obj& obj, rgw_bucket& bucket, string& oid, string& key)
+{
+  bucket = obj.bucket;
+  prepend_bucket_marker(bucket, obj.object, oid);
+  prepend_bucket_marker(bucket, obj.key, key);
+}
+
+int rgw_policy_from_attrset(CephContext *cct, map& attrset, RGWAccessControlPolicy *policy);
+
+struct RGWUsageBatch {
+  map m;
+
+  void insert(utime_t& t, rgw_usage_log_entry& entry, bool *account) {
+    bool exists = m.find(t) != m.end();
+    *account = !exists;
+    m[t].aggregate(entry);
+  }
+};
+
+struct RGWUsageIter {
+  string read_iter;
+  uint32_t index;
+
+  RGWUsageIter() : index(0) {}
+};
+
+class RGWGetDataCB {
+protected:
+  uint64_t extra_data_len;
+public:
+  virtual int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) = 0;
+  RGWGetDataCB() : extra_data_len(0) {}
+  virtual ~RGWGetDataCB() {}
+  virtual void set_extra_data_len(uint64_t len) {
+    extra_data_len = len;
+  }
+};
+
+class RGWAccessListFilter {
+public:
+  virtual ~RGWAccessListFilter() {}
+  virtual bool filter(string& name, string& key) = 0;
+};
+
+struct RGWCloneRangeInfo {
+  rgw_obj src;
+  off_t src_ofs;
+  off_t dst_ofs;
+  uint64_t len;
+};
+
+struct RGWObjManifestPart {
+  rgw_obj loc;       /* the object where the data is located */
+  uint64_t loc_ofs;  /* the offset at that object where the data is located */
+  uint64_t size;     /* the part size */
+
+  RGWObjManifestPart() : loc_ofs(0), size(0) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(loc, bl);
+    ::encode(loc_ofs, bl);
+    ::encode(size, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
+     ::decode(loc, bl);
+     ::decode(loc_ofs, bl);
+     ::decode(size, bl);
+     DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(RGWObjManifestPart);
+
+/*
+ The manifest defines a set of rules for structuring the object parts.
+ There are a few terms to note:
+     - head: the head part of the object, which is the part that contains
+       the first chunk of data. An object might not have a head (as in the
+       case of multipart-part objects).
+     - stripe: data portion of a single rgw object that resides on a single
+       rados object.
+     - part: a collection of stripes that make a contiguous part of an
+       object. A regular object will only have one part (although might have
+       many stripes), a multipart object might have many parts. Each part
+       has a fixed stripe size, although the last stripe of a part might
+       be smaller than that. Consecutive parts may be merged if their stripe
+       value is the same.
+*/
+
+struct RGWObjManifestRule {
+  uint32_t start_part_num;
+  uint64_t start_ofs;
+  uint64_t part_size; /* each part size, 0 if there's no part size, meaning it's unlimited */
+  uint64_t stripe_max_size; /* underlying obj max size */
+  string override_prefix;
+
+  RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {}
+  RGWObjManifestRule(uint32_t _start_part_num, uint64_t _start_ofs, uint64_t _part_size, uint64_t _stripe_max_size) :
+                       start_part_num(_start_part_num), start_ofs(_start_ofs), part_size(_part_size), stripe_max_size(_stripe_max_size) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 1, bl);
+    ::encode(start_part_num, bl);
+    ::encode(start_ofs, bl);
+    ::encode(part_size, bl);
+    ::encode(stripe_max_size, bl);
+    ::encode(override_prefix, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(2, bl);
+    ::decode(start_part_num, bl);
+    ::decode(start_ofs, bl);
+    ::decode(part_size, bl);
+    ::decode(stripe_max_size, bl);
+    if (struct_v >= 2)
+      ::decode(override_prefix, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(RGWObjManifestRule);
+
+class RGWObjManifest {
+protected:
+  bool explicit_objs; /* old manifest? */
+  map objs;
+
+  uint64_t obj_size;
+
+  rgw_obj head_obj;
+  uint64_t head_size;
+
+  uint64_t max_head_size;
+  string prefix;
+  rgw_bucket tail_bucket; /* might be different than the original bucket,
+                             as object might have been copied across buckets */
+  map rules;
+
+  void convert_to_explicit();
+  int append_explicit(RGWObjManifest& m);
+  void append_rules(RGWObjManifest& m, map::iterator& iter, string *override_prefix);
+
+  void update_iterators() {
+    begin_iter.seek(0);
+    end_iter.seek(obj_size);
+  }
+public:
+
+  RGWObjManifest() : explicit_objs(false), obj_size(0), head_size(0), max_head_size(0),
+                     begin_iter(this), end_iter(this) {}
+  RGWObjManifest(const RGWObjManifest& rhs) {
+    *this = rhs;
+  }
+  RGWObjManifest& operator=(const RGWObjManifest& rhs) {
+    explicit_objs = rhs.explicit_objs;
+    objs = rhs.objs;
+    obj_size = rhs.obj_size;
+    head_obj = rhs.head_obj;
+    head_size = rhs.head_size;
+    max_head_size = rhs.max_head_size;
+    prefix = rhs.prefix;
+    tail_bucket = rhs.tail_bucket;
+    rules = rhs.rules;
+
+    begin_iter.set_manifest(this);
+    end_iter.set_manifest(this);
+
+    begin_iter.seek(rhs.begin_iter.get_ofs());
+    end_iter.seek(rhs.end_iter.get_ofs());
+
+    return *this;
+  }
+
+
+  void set_explicit(uint64_t _size, map& _objs) {
+    explicit_objs = true;
+    obj_size = _size;
+    objs.swap(_objs);
+  }
+
+  void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj *location);
+
+  void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) {
+    RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size);
+    rules[0] = rule;
+    max_head_size = tail_ofs;
+  }
+
+  void set_multipart_part_rule(uint64_t stripe_max_size, uint64_t part_num) {
+    RGWObjManifestRule rule(0, 0, 0, stripe_max_size);
+    rule.start_part_num = part_num;
+    rules[0] = rule;
+    max_head_size = 0;
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(4, 3, bl);
+    ::encode(obj_size, bl);
+    ::encode(objs, bl);
+    ::encode(explicit_objs, bl);
+    ::encode(head_obj, bl);
+    ::encode(head_size, bl);
+    ::encode(max_head_size, bl);
+    ::encode(prefix, bl);
+    ::encode(rules, bl);
+    ::encode(tail_bucket, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN_32(4, 2, 2, bl);
+    ::decode(obj_size, bl);
+    ::decode(objs, bl);
+    if (struct_v >= 3) {
+      ::decode(explicit_objs, bl);
+      ::decode(head_obj, bl);
+      ::decode(head_size, bl);
+      ::decode(max_head_size, bl);
+      ::decode(prefix, bl);
+      ::decode(rules, bl);
+    } else {
+      explicit_objs = true;
+    }
+
+    if (struct_v >= 4) {
+      ::decode(tail_bucket, bl);
+    }
+
+    update_iterators();
+    DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+
+  int append(RGWObjManifest& m);
+
+  bool get_rule(uint64_t ofs, RGWObjManifestRule *rule);
+
+  bool empty() {
+    if (explicit_objs)
+      return objs.empty();
+    return rules.empty();
+  }
+
+  bool has_explicit_objs() {
+    return explicit_objs;
+  }
+
+  bool has_tail() {
+    if (explicit_objs) {
+      return (objs.size() >= 2);
+    }
+    return (obj_size > head_size);
+  }
+
+  void set_head(const rgw_obj& _o) {
+    head_obj = _o;
+  }
+
+  const rgw_obj& get_head() {
+    return head_obj;
+  }
+
+  void set_tail_bucket(const rgw_bucket& _b) {
+    tail_bucket = _b;
+  }
+
+  rgw_bucket& get_tail_bucket() {
+    return tail_bucket;
+  }
+
+  void set_prefix(const string& _p) {
+    prefix = _p;
+  }
+
+  const string& get_prefix() {
+    return prefix;
+  }
+
+  void set_head_size(uint64_t _s) {
+    head_size = _s;
+  }
+
+  void set_obj_size(uint64_t s) {
+    obj_size = s;
+
+    update_iterators();
+  }
+
+  uint64_t get_obj_size() {
+    return obj_size;
+  }
+
+  uint64_t get_head_size() {
+    return head_size;
+  }
+
+  void set_max_head_size(uint64_t s) {
+    max_head_size = s;
+  }
+
+  uint64_t get_max_head_size() {
+    return max_head_size;
+  }
+
+  class obj_iterator {
+    RGWObjManifest *manifest;
+    uint64_t part_ofs; /* where current part starts */
+    uint64_t stripe_ofs; /* where current stripe starts */
+    uint64_t ofs;       /* current position within the object */
+    uint64_t stripe_size;      /* current part size */
+
+    int cur_part_id;
+    int cur_stripe;
+    string cur_override_prefix;
+
+    rgw_obj location;
+
+    map::iterator rule_iter;
+    map::iterator next_rule_iter;
+
+    map::iterator explicit_iter;
+
+    void init() {
+      part_ofs = 0;
+      stripe_ofs = 0;
+      ofs = 0;
+      stripe_size = 0;
+      cur_part_id = 0;
+      cur_stripe = 0;
+    }
+
+    void update_explicit_pos();
+
+
+  protected:
+
+    void set_manifest(RGWObjManifest *m) {
+      manifest = m;
+    }
+
+  public:
+    obj_iterator() : manifest(NULL) {
+      init();
+    }
+    obj_iterator(RGWObjManifest *_m) : manifest(_m) {
+      init();
+      seek(0);
+    }
+    obj_iterator(RGWObjManifest *_m, uint64_t _ofs) : manifest(_m) {
+      init();
+      seek(_ofs);
+    }
+    void seek(uint64_t ofs);
+
+    void operator++();
+    bool operator==(const obj_iterator& rhs) {
+      return (ofs == rhs.ofs);
+    }
+    bool operator!=(const obj_iterator& rhs) {
+      return (ofs != rhs.ofs);
+    }
+    const rgw_obj& get_location() {
+      return location;
+    }
+
+    /* start of current stripe */
+    uint64_t get_stripe_ofs() {
+      if (manifest->explicit_objs) {
+        return explicit_iter->first;
+      }
+      return stripe_ofs;
+    }
+
+    /* current ofs relative to start of rgw object */
+    uint64_t get_ofs() const {
+      return ofs;
+    }
+
+    /* current stripe size */
+    uint64_t get_stripe_size() {
+      if (manifest->explicit_objs) {
+        return explicit_iter->second.size;
+      }
+      return stripe_size;
+    }
+
+    /* offset where data starts within current stripe */
+    uint64_t location_ofs() {
+      if (manifest->explicit_objs) {
+        return explicit_iter->second.loc_ofs;
+      }
+      return 0; /* all stripes start at zero offset */
+    }
+
+    void update_location();
+
+    friend class RGWObjManifest;
+  };
+
+  const obj_iterator& obj_begin();
+  const obj_iterator& obj_end();
+  obj_iterator obj_find(uint64_t ofs);
+
+  obj_iterator begin_iter;
+  obj_iterator end_iter;
+
+  /*
+   * simple object generator. Using a simple single rule manifest.
+   */
+  class generator {
+    RGWObjManifest *manifest;
+    uint64_t last_ofs;
+    uint64_t cur_part_ofs;
+    int cur_part_id;
+    int cur_stripe;
+    uint64_t cur_stripe_size;
+    string cur_oid;
+    
+    string oid_prefix;
+
+    rgw_obj cur_obj;
+    rgw_bucket bucket;
+
+
+    RGWObjManifestRule rule;
+
+  public:
+    generator() : manifest(NULL), last_ofs(0), cur_part_ofs(0), cur_part_id(0), 
+		  cur_stripe(0), cur_stripe_size(0) {}
+    int create_begin(CephContext *cct, RGWObjManifest *manifest, rgw_bucket& bucket, rgw_obj& head);
+
+    int create_next(uint64_t ofs);
+
+    const rgw_obj& get_cur_obj() { return cur_obj; }
+
+    /* total max size of current stripe (including head obj) */
+    uint64_t cur_stripe_max_size() {
+      return cur_stripe_size;
+    }
+  };
+};
+WRITE_CLASS_ENCODER(RGWObjManifest);
+
+struct RGWUploadPartInfo {
+  uint32_t num;
+  uint64_t size;
+  string etag;
+  utime_t modified;
+  RGWObjManifest manifest;
+
+  RGWUploadPartInfo() : num(0), size(0) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(3, 2, bl);
+    ::encode(num, bl);
+    ::encode(size, bl);
+    ::encode(etag, bl);
+    ::encode(modified, bl);
+    ::encode(manifest, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
+    ::decode(num, bl);
+    ::decode(size, bl);
+    ::decode(etag, bl);
+    ::decode(modified, bl);
+    if (struct_v >= 3)
+      ::decode(manifest, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list& o);
+};
+WRITE_CLASS_ENCODER(RGWUploadPartInfo)
+
+class RGWPutObjProcessor
+{
+protected:
+  RGWRados *store;
+  void *obj_ctx;
+  bool is_complete;
+  string bucket_owner;
+
+  virtual int do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs) = 0;
+
+  list objs;
+
+  void add_obj(const rgw_obj& obj) {
+    objs.push_back(obj);
+  }
+public:
+  RGWPutObjProcessor(const string& _bo) : store(NULL), obj_ctx(NULL), is_complete(false), bucket_owner(_bo) {}
+  virtual ~RGWPutObjProcessor();
+  virtual int prepare(RGWRados *_store, void *_o, string *oid_rand) {
+    store = _store;
+    obj_ctx = _o;
+    return 0;
+  };
+  virtual int handle_data(bufferlist& bl, off_t ofs, void **phandle, bool *again) = 0;
+  virtual int throttle_data(void *handle, bool need_to_wait) = 0;
+  virtual int complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs);
+};
+
+class RGWPutObjProcessor_Plain : public RGWPutObjProcessor
+{
+  rgw_bucket bucket;
+  string obj_str;
+
+  bufferlist data;
+  rgw_obj obj;
+  off_t ofs;
+
+protected:
+  int prepare(RGWRados *store, void *obj_ctx, string *oid_rand);
+  int handle_data(bufferlist& bl, off_t ofs, void **phandle, bool *again);
+  int do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs);
+
+public:
+  int throttle_data(void *handle, bool need_to_wait) { return 0; }
+  RGWPutObjProcessor_Plain(const string& bucket_owner, rgw_bucket& b, const string& o) : RGWPutObjProcessor(bucket_owner),
+                                                                                         bucket(b), obj_str(o), ofs(0) {}
+};
+
+struct put_obj_aio_info {
+  void *handle;
+};
+
+class RGWPutObjProcessor_Aio : public RGWPutObjProcessor
+{
+  list pending;
+  size_t max_chunks;
+
+  struct put_obj_aio_info pop_pending();
+  int wait_pending_front();
+  bool pending_has_completed();
+
+protected:
+  uint64_t obj_len;
+
+  int drain_pending();
+  int handle_obj_data(rgw_obj& obj, bufferlist& bl, off_t ofs, off_t abs_ofs, void **phandle, bool exclusive);
+
+public:
+  int throttle_data(void *handle, bool need_to_wait);
+
+  RGWPutObjProcessor_Aio(const string& bucket_owner) : RGWPutObjProcessor(bucket_owner), max_chunks(RGW_MAX_PENDING_CHUNKS), obj_len(0) {}
+  virtual ~RGWPutObjProcessor_Aio() {
+    drain_pending();
+  }
+};
+
+class RGWPutObjProcessor_Atomic : public RGWPutObjProcessor_Aio
+{
+  bufferlist first_chunk;
+  uint64_t part_size;
+  off_t cur_part_ofs;
+  off_t next_part_ofs;
+  int cur_part_id;
+  off_t data_ofs;
+
+  uint64_t extra_data_len;
+  bufferlist extra_data_bl;
+  bufferlist pending_data_bl;
+  uint64_t max_chunk_size;
+
+protected:
+  rgw_bucket bucket;
+  string obj_str;
+
+  string unique_tag;
+
+  rgw_obj head_obj;
+  rgw_obj cur_obj;
+  RGWObjManifest manifest;
+  RGWObjManifest::generator manifest_gen;
+
+  int write_data(bufferlist& bl, off_t ofs, void **phandle, bool exclusive);
+  virtual int do_complete(string& etag, time_t *mtime, time_t set_mtime, map& attrs);
+
+  int prepare_next_part(off_t ofs);
+  int complete_parts();
+  int complete_writing_data();
+
+  int prepare_init(RGWRados *store, void *obj_ctx, string *oid_rand);
+
+public:
+  ~RGWPutObjProcessor_Atomic() {}
+  RGWPutObjProcessor_Atomic(const string& bucket_owner, rgw_bucket& _b, const string& _o, uint64_t _p, const string& _t) :
+                                RGWPutObjProcessor_Aio(bucket_owner),
+                                part_size(_p),
+                                cur_part_ofs(0),
+                                next_part_ofs(_p),
+                                cur_part_id(0),
+                                data_ofs(0),
+                                extra_data_len(0),
+                                max_chunk_size(0),
+                                bucket(_b),
+                                obj_str(_o),
+                                unique_tag(_t) {}
+  int prepare(RGWRados *store, void *obj_ctx, string *oid_rand);
+  virtual bool immutable_head() { return false; }
+  void set_extra_data_len(uint64_t len) {
+    extra_data_len = len;
+  }
+  virtual int handle_data(bufferlist& bl, off_t ofs, void **phandle, bool *again);
+  bufferlist& get_extra_data() { return extra_data_bl; }
+};
+
+
+struct RGWObjState {
+  bool is_atomic;
+  bool has_attrs;
+  bool exists;
+  uint64_t size;
+  time_t mtime;
+  uint64_t epoch;
+  bufferlist obj_tag;
+  string write_tag;
+  bool fake_tag;
+  RGWObjManifest manifest;
+  bool has_manifest;
+  string shadow_obj;
+  bool has_data;
+  bufferlist data;
+  bool prefetch_data;
+  bool keep_tail;
+  RGWObjVersionTracker objv_tracker;
+
+  map attrset;
+  RGWObjState() : is_atomic(false), has_attrs(0), exists(false),
+                  size(0), mtime(0), epoch(0), fake_tag(false), has_manifest(false),
+                  has_data(false), prefetch_data(false), keep_tail(false) {}
+
+  bool get_attr(string name, bufferlist& dest) {
+    map::iterator iter = attrset.find(name);
+    if (iter != attrset.end()) {
+      dest = iter->second;
+      return true;
+    }
+    return false;
+  }
+
+  void clear() {
+    has_attrs = false;
+    exists = false;
+    fake_tag = false;
+    epoch = 0;
+    size = 0;
+    mtime = 0;
+    obj_tag.clear();
+    shadow_obj.clear();
+    attrset.clear();
+    data.clear();
+  }
+};
+
+struct RGWRadosCtx {
+  RGWRados *store;
+  map objs_state;
+  int (*intent_cb)(RGWRados *store, void *user_ctx, rgw_obj& obj, RGWIntentEvent intent);
+  void *user_ctx;
+
+  RGWRadosCtx(RGWRados *_store) : store(_store), intent_cb(NULL), user_ctx(NULL) { }
+  RGWRadosCtx(RGWRados *_store, void *_user_ctx) : store(_store), intent_cb(NULL), user_ctx(_user_ctx) { }
+
+  RGWObjState *get_state(rgw_obj& obj);
+  void set_atomic(rgw_obj& obj);
+  void set_prefetch_data(rgw_obj& obj);
+
+  void set_intent_cb(int (*cb)(RGWRados *store, void *user_ctx, rgw_obj& obj, RGWIntentEvent intent)) {
+    intent_cb = cb;
+  }
+
+  int notify_intent(RGWRados *store, rgw_obj& obj, RGWIntentEvent intent) {
+    if (intent_cb) {
+      return intent_cb(store, user_ctx, obj, intent);
+    }
+    return 0;
+  }
+};
+
+struct RGWPoolIterCtx {
+  librados::IoCtx io_ctx;
+  librados::ObjectIterator iter;
+};
+
+struct RGWListRawObjsCtx {
+  bool initialized;
+  RGWPoolIterCtx iter_ctx;
+
+  RGWListRawObjsCtx() : initialized(false) {}
+};
+
+struct RGWRegion;
+
+
+struct RGWZonePlacementInfo {
+  string index_pool;
+  string data_pool;
+  string data_extra_pool; /* if not set we should use data_pool */
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(4, 1, bl);
+    ::encode(index_pool, bl);
+    ::encode(data_pool, bl);
+    ::encode(data_extra_pool, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(4, bl);
+    ::decode(index_pool, bl);
+    ::decode(data_pool, bl);
+    if (struct_v >= 4) {
+      ::decode(data_extra_pool, bl);
+    }
+    DECODE_FINISH(bl);
+  }
+  const string& get_data_extra_pool() {
+    if (data_extra_pool.empty()) {
+      return data_pool;
+    }
+    return data_extra_pool;
+  }
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWZonePlacementInfo);
+
+struct RGWZoneParams {
+  rgw_bucket domain_root;
+  rgw_bucket control_pool;
+  rgw_bucket gc_pool;
+  rgw_bucket log_pool;
+  rgw_bucket intent_log_pool;
+  rgw_bucket usage_log_pool;
+
+  rgw_bucket user_keys_pool;
+  rgw_bucket user_email_pool;
+  rgw_bucket user_swift_pool;
+  rgw_bucket user_uid_pool;
+
+  string name;
+  bool is_master;
+
+  RGWAccessKey system_key;
+
+  map placement_pools;
+
+  RGWZoneParams() : is_master(false) {}
+
+  static int get_pool_name(CephContext *cct, string *pool_name);
+  void init_name(CephContext *cct, RGWRegion& region);
+  int init(CephContext *cct, RGWRados *store, RGWRegion& region);
+  void init_default(RGWRados *store);
+  int store_info(CephContext *cct, RGWRados *store, RGWRegion& region);
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(4, 1, bl);
+    ::encode(domain_root, bl);
+    ::encode(control_pool, bl);
+    ::encode(gc_pool, bl);
+    ::encode(log_pool, bl);
+    ::encode(intent_log_pool, bl);
+    ::encode(usage_log_pool, bl);
+    ::encode(user_keys_pool, bl);
+    ::encode(user_email_pool, bl);
+    ::encode(user_swift_pool, bl);
+    ::encode(user_uid_pool, bl);
+    ::encode(name, bl);
+    ::encode(system_key, bl);
+    ::encode(placement_pools, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+     DECODE_START(4, bl);
+    ::decode(domain_root, bl);
+    ::decode(control_pool, bl);
+    ::decode(gc_pool, bl);
+    ::decode(log_pool, bl);
+    ::decode(intent_log_pool, bl);
+    ::decode(usage_log_pool, bl);
+    ::decode(user_keys_pool, bl);
+    ::decode(user_email_pool, bl);
+    ::decode(user_swift_pool, bl);
+    ::decode(user_uid_pool, bl);
+    if (struct_v >= 2)
+      ::decode(name, bl);
+    if (struct_v >= 3)
+      ::decode(system_key, bl);
+    if (struct_v >= 4)
+      ::decode(placement_pools, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWZoneParams);
+
+struct RGWZone {
+  string name;
+  list endpoints;
+  bool log_meta;
+  bool log_data;
+
+  RGWZone() : log_meta(false), log_data(false) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 1, bl);
+    ::encode(name, bl);
+    ::encode(endpoints, bl);
+    ::encode(log_meta, bl);
+    ::encode(log_data, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(2, bl);
+    ::decode(name, bl);
+    ::decode(endpoints, bl);
+    if (struct_v >= 2) {
+      ::decode(log_meta, bl);
+      ::decode(log_data, bl);
+    }
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWZone);
+
+struct RGWDefaultRegionInfo {
+  string default_region;
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(default_region, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(default_region, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWDefaultRegionInfo);
+
+struct RGWRegionPlacementTarget {
+  string name;
+  list tags;
+
+  bool user_permitted(list& user_tags) {
+    if (tags.empty()) {
+      return true;
+    }
+    for (list::iterator uiter = user_tags.begin(); uiter != user_tags.end(); ++uiter) { /* we don't expect many of either, so we can handle this kind of lookup */
+      string& rule = *uiter;
+      for (list::iterator iter = tags.begin(); iter != tags.end(); ++iter) {
+        if (rule == *iter) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(name, bl);
+    ::encode(tags, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(name, bl);
+    ::decode(tags, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWRegionPlacementTarget);
+
+
+struct RGWRegion {
+  string name;
+  string api_name;
+  list endpoints;
+  bool is_master;
+
+  string master_zone;
+  map zones;
+
+  map placement_targets;
+  string default_placement;
+
+  CephContext *cct;
+  RGWRados *store;
+
+  RGWRegion() : is_master(false), cct(NULL), store(NULL) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(name, bl);
+    ::encode(api_name, bl);
+    ::encode(is_master, bl);
+    ::encode(endpoints, bl);
+    ::encode(master_zone, bl);
+    ::encode(zones, bl);
+    ::encode(placement_targets, bl);
+    ::encode(default_placement, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(name, bl);
+    ::decode(api_name, bl);
+    ::decode(is_master, bl);
+    ::decode(endpoints, bl);
+    ::decode(master_zone, bl);
+    ::decode(zones, bl);
+    ::decode(placement_targets, bl);
+    ::decode(default_placement, bl);
+    DECODE_FINISH(bl);
+  }
+
+  int init(CephContext *_cct, RGWRados *_store, bool setup_region = true);
+  int create_default();
+  int store_info(bool exclusive);
+  int read_info(const string& region_name);
+  int read_default(RGWDefaultRegionInfo& default_region);
+  int set_as_default();
+  int equals(const string& other_region);
+
+  static int get_pool_name(CephContext *cct, string *pool_name);
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWRegion);
+
+struct RGWRegionMap {
+  Mutex lock;
+  map regions;
+  map regions_by_api;
+
+  string master_region;
+
+  RGWQuotaInfo bucket_quota;
+  RGWQuotaInfo user_quota;
+
+  RGWRegionMap() : lock("RGWRegionMap") {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+
+  void get_params(CephContext *cct, string& pool_name, string& oid);
+  int read(CephContext *cct, RGWRados *store);
+  int store(CephContext *cct, RGWRados *store);
+
+  int update(RGWRegion& region);
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWRegionMap);
+
+class RGWDataChangesLog;
+class RGWReplicaLogger;
+  
+class RGWStateLog {
+  RGWRados *store;
+  int num_shards;
+  string module_name;
+
+  void oid_str(int shard, string& oid);
+  int get_shard_num(const string& object);
+  string get_oid(const string& object);
+  int open_ioctx(librados::IoCtx& ioctx);
+
+  struct list_state {
+    int cur_shard;
+    int max_shard;
+    string marker;
+    string client_id;
+    string op_id;
+    string object;
+
+    list_state() : cur_shard(0), max_shard(0) {}
+  };
+
+protected:
+  virtual bool dump_entry_internal(const cls_statelog_entry& entry, Formatter *f) {
+    return false;
+  }
+
+public:
+  RGWStateLog(RGWRados *_store, int _num_shards, const string& _module_name) :
+              store(_store), num_shards(_num_shards), module_name(_module_name) {}
+  virtual ~RGWStateLog() {}
+
+  int store_entry(const string& client_id, const string& op_id, const string& object,
+                  uint32_t state, bufferlist *bl, uint32_t *check_state);
+
+  int remove_entry(const string& client_id, const string& op_id, const string& object);
+
+  void init_list_entries(const string& client_id, const string& op_id, const string& object,
+                         void **handle);
+
+  int list_entries(void *handle, int max_entries, list& entries, bool *done);
+
+  void finish_list_entries(void *handle);
+
+  virtual void dump_entry(const cls_statelog_entry& entry, Formatter *f);
+};
+
+/*
+ * state transitions:
+ *
+ * unknown -> in-progress -> complete
+ *                        -> error
+ *
+ * user can try setting the 'abort' state, and it can only succeed if state is
+ * in-progress.
+ *
+ * state renewal cannot switch state (stays in the same state)
+ *
+ * rgw can switch from in-progress to complete
+ * rgw can switch from in-progress to error
+ *
+ * rgw can switch from abort to cancelled
+ *
+ */
+
+class RGWOpState : public RGWStateLog {
+protected:
+  bool dump_entry_internal(const cls_statelog_entry& entry, Formatter *f);
+public:
+
+  enum OpState {
+    OPSTATE_UNKNOWN     = 0,
+    OPSTATE_IN_PROGRESS = 1,
+    OPSTATE_COMPLETE    = 2,
+    OPSTATE_ERROR       = 3,
+    OPSTATE_ABORT       = 4,
+    OPSTATE_CANCELLED   = 5,
+  };
+
+  RGWOpState(RGWRados *_store);
+
+  int state_from_str(const string& s, OpState *state);
+  int set_state(const string& client_id, const string& op_id, const string& object, OpState state);
+  int renew_state(const string& client_id, const string& op_id, const string& object, OpState state);
+};
+
+class RGWOpStateSingleOp
+{
+  RGWOpState os;
+  string client_id;
+  string op_id;
+  string object;
+
+  CephContext *cct;
+
+  RGWOpState::OpState cur_state;
+  utime_t last_update;
+
+public:
+  RGWOpStateSingleOp(RGWRados *store, const string& cid, const string& oid, const string& obj);
+
+  int set_state(RGWOpState::OpState state);
+  int renew_state();
+};
+
+class RGWGetBucketStats_CB : public RefCountedObject {
+protected:
+  rgw_bucket bucket;
+  uint64_t bucket_ver;
+  uint64_t master_ver;
+  map *stats;
+  string max_marker;
+public:
+  RGWGetBucketStats_CB(rgw_bucket& _bucket) : bucket(_bucket), stats(NULL) {}
+  virtual ~RGWGetBucketStats_CB() {}
+  virtual void handle_response(int r) = 0;
+  virtual void set_response(uint64_t _bucket_ver, uint64_t _master_ver,
+                            map *_stats,
+                            const string &_max_marker) {
+    bucket_ver = _bucket_ver;
+    master_ver = _master_ver;
+    stats = _stats;
+    max_marker = _max_marker;
+  }
+};
+
+class RGWGetUserStats_CB : public RefCountedObject {
+protected:
+  string user;
+  RGWStorageStats stats;
+public:
+  RGWGetUserStats_CB(const string& _user) : user(_user) {}
+  virtual ~RGWGetUserStats_CB() {}
+  virtual void handle_response(int r) = 0;
+  virtual void set_response(RGWStorageStats& _stats) {
+    stats = _stats;
+  }
+};
+
+class RGWGetDirHeader_CB;
+class RGWGetUserHeader_CB;
+
+struct rgw_rados_ref {
+  string oid;
+  string key;
+  librados::IoCtx ioctx;
+};
+
+
+class RGWRados
+{
+  friend class RGWGC;
+  friend class RGWStateLog;
+  friend class RGWReplicaLogger;
+
+  /** Open the pool used as root for this gateway */
+  int open_root_pool_ctx();
+  int open_gc_pool_ctx();
+
+  int open_bucket_pool_ctx(const string& bucket_name, const string& pool, librados::IoCtx&  io_ctx);
+  int open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx&  index_ctx);
+  int open_bucket_data_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx);
+  int open_bucket_data_extra_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx);
+  int open_bucket_index(rgw_bucket& bucket, librados::IoCtx&  index_ctx, string& bucket_oid);
+
+  struct GetObjState {
+    librados::IoCtx io_ctx;
+    bool sent_data;
+
+    GetObjState() : sent_data(false) {}
+  };
+
+  Mutex lock;
+  SafeTimer *timer;
+
+  class C_Tick : public Context {
+    RGWRados *rados;
+  public:
+    C_Tick(RGWRados *_r) : rados(_r) {}
+    void finish(int r) {
+      rados->tick();
+    }
+  };
+
+  RGWGC *gc;
+  bool use_gc_thread;
+  bool quota_threads;
+
+  int num_watchers;
+  RGWWatcher **watchers;
+  uint64_t *watch_handles;
+  librados::IoCtx root_pool_ctx;      // .rgw
+  librados::IoCtx control_pool_ctx;   // .rgw.control
+  bool watch_initialized;
+
+  Mutex bucket_id_lock;
+
+  int get_obj_ioctx(const rgw_obj& obj, librados::IoCtx *ioctx);
+  int get_obj_ref(const rgw_obj& obj, rgw_rados_ref *ref, rgw_bucket *bucket, bool ref_system_obj = false);
+  uint64_t max_bucket_id;
+
+  int get_obj_state(RGWRadosCtx *rctx, rgw_obj& obj, RGWObjState **state, RGWObjVersionTracker *objv_tracker);
+  int append_atomic_test(RGWRadosCtx *rctx, rgw_obj& obj,
+                         librados::ObjectOperation& op, RGWObjState **state);
+  int prepare_atomic_for_write_impl(RGWRadosCtx *rctx, rgw_obj& obj,
+                         librados::ObjectWriteOperation& op, RGWObjState **pstate,
+			 bool reset_obj, const string *ptag);
+  int prepare_atomic_for_write(RGWRadosCtx *rctx, rgw_obj& obj,
+                         librados::ObjectWriteOperation& op, RGWObjState **pstate,
+			 bool reset_obj, const string *ptag);
+
+  void atomic_write_finish(RGWObjState *state, int r) {
+    if (state && r == -ECANCELED) {
+      state->clear();
+    }
+  }
+
+  int clone_objs_impl(void *ctx, rgw_obj& dst_obj, 
+                 vector& ranges,
+                 map attrs,
+                 RGWObjCategory category,
+                 time_t *pmtime,
+                 bool truncate_dest,
+                 bool exclusive,
+                 pair *cmp_xattr);
+
+  virtual int clone_obj(void *ctx, rgw_obj& dst_obj, off_t dst_ofs,
+                          rgw_obj& src_obj, off_t src_ofs,
+                          uint64_t size, time_t *pmtime,
+                          map attrs,
+                          RGWObjCategory category) {
+    RGWCloneRangeInfo info;
+    vector v;
+    info.src = src_obj;
+    info.src_ofs = src_ofs;
+    info.dst_ofs = dst_ofs;
+    info.len = size;
+    v.push_back(info);
+    return clone_objs(ctx, dst_obj, v, attrs, category, pmtime, true, false);
+  }
+  int complete_atomic_overwrite(RGWRadosCtx *rctx, RGWObjState *state, rgw_obj& obj);
+
+  int update_placement_map();
+  int store_bucket_info(RGWBucketInfo& info, map *pattrs, RGWObjVersionTracker *objv_tracker, bool exclusive);
+
+protected:
+  virtual int delete_obj_impl(void *ctx, const string& bucket_owner, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker);
+
+  CephContext *cct;
+  librados::Rados *rados;
+  librados::IoCtx gc_pool_ctx;        // .rgw.gc
+
+  bool pools_initialized;
+
+  string region_name;
+  string zone_name;
+
+  RGWQuotaHandler *quota_handler;
+
+public:
+  RGWRados() : lock("rados_timer_lock"), timer(NULL),
+               gc(NULL), use_gc_thread(false), quota_threads(false),
+               num_watchers(0), watchers(NULL), watch_handles(NULL),
+               watch_initialized(false),
+               bucket_id_lock("rados_bucket_id"), max_bucket_id(0),
+               cct(NULL), rados(NULL),
+               pools_initialized(false),
+               quota_handler(NULL),
+               rest_master_conn(NULL),
+               meta_mgr(NULL), data_log(NULL) {}
+
+  void set_context(CephContext *_cct) {
+    cct = _cct;
+  }
+
+  void set_region(const string& name) {
+    region_name = name;
+  }
+
+  void set_zone(const string& name) {
+    zone_name = name;
+  }
+
+  RGWRegion region;
+  RGWZoneParams zone; /* internal zone params, e.g., rados pools */
+  RGWZone zone_public_config; /* external zone params, e.g., entrypoints, log flags, etc. */
+  RGWRegionMap region_map;
+  RGWRESTConn *rest_master_conn;
+  map zone_conn_map;
+  map region_conn_map;
+
+  RGWMetadataManager *meta_mgr;
+
+  RGWDataChangesLog *data_log;
+
+  virtual ~RGWRados() {
+    if (rados) {
+      rados->shutdown();
+      delete rados;
+    }
+  }
+
+  int get_required_alignment(rgw_bucket& bucket, uint64_t *alignment);
+  int get_max_chunk_size(rgw_bucket& bucket, uint64_t *max_chunk_size);
+
+  int list_raw_objects(rgw_bucket& pool, const string& prefix_filter, int max,
+                       RGWListRawObjsCtx& ctx, list& oids,
+                       bool *is_truncated);
+
+  int list_raw_prefixed_objs(string pool_name, const string& prefix, list& result);
+  int list_regions(list& regions);
+  int list_zones(list& zones);
+
+  void tick();
+
+  CephContext *ctx() { return cct; }
+  /** do all necessary setup of the storage device */
+  int initialize(CephContext *_cct, bool _use_gc_thread, bool _quota_threads) {
+    set_context(_cct);
+    use_gc_thread = _use_gc_thread;
+    quota_threads = _quota_threads;
+    return initialize();
+  }
+  /** Initialize the RADOS instance and prepare to do other ops */
+  virtual int init_rados();
+  int init_complete();
+  virtual int initialize();
+  virtual void finalize();
+
+  /** set up a bucket listing. handle is filled in. */
+  virtual int list_buckets_init(RGWAccessHandle *handle);
+  /** 
+   * get the next bucket in the listing. obj is filled in,
+   * handle is updated.
+   */
+  virtual int list_buckets_next(RGWObjEnt& obj, RGWAccessHandle *handle);
+
+  /// list logs
+  int log_list_init(const string& prefix, RGWAccessHandle *handle);
+  int log_list_next(RGWAccessHandle handle, string *name);
+
+  /// remove log
+  int log_remove(const string& name);
+
+  /// show log
+  int log_show_init(const string& name, RGWAccessHandle *handle);
+  int log_show_next(RGWAccessHandle handle, rgw_log_entry *entry);
+
+  // log bandwidth info
+  int log_usage(map& usage_info);
+  int read_usage(string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
+                 bool *is_truncated, RGWUsageIter& read_iter, map& usage);
+  int trim_usage(string& user, uint64_t start_epoch, uint64_t end_epoch);
+
+  /**
+   * get listing of the objects in a bucket.
+   * bucket: bucket to list contents of
+   * max: maximum number of results to return
+   * prefix: only return results that match this prefix
+   * delim: do not include results that match this string.
+   *     Any skipped results will have the matching portion of their name
+   *     inserted in common_prefixes with a "true" mark.
+   * marker: if filled in, begin the listing with this object.
+   * result: the objects are put in here.
+   * common_prefixes: if delim is filled in, any matching prefixes are placed
+   *     here.
+   */
+  virtual int list_objects(rgw_bucket& bucket, int max, std::string& prefix, std::string& delim,
+                   std::string& marker, std::string *next_marker, std::vector& result,
+                   map& common_prefixes, bool get_content_type, string& ns, bool enforce_ns,
+                   bool *is_truncated, RGWAccessListFilter *filter);
+
+  virtual int create_pool(rgw_bucket& bucket);
+
+  /**
+   * create a bucket with name bucket and the given list of attrs
+   * returns 0 on success, -ERR# otherwise.
+   */
+  virtual int init_bucket_index(rgw_bucket& bucket);
+  int select_bucket_placement(RGWUserInfo& user_info, const string& region_name, const std::string& rule,
+                              const std::string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
+  int select_legacy_bucket_placement(const string& bucket_name, rgw_bucket& bucket);
+  int select_new_bucket_location(RGWUserInfo& user_info, const string& region_name, const string& rule,
+                                 const std::string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
+  int set_bucket_location_by_rule(const string& location_rule, const std::string& bucket_name, rgw_bucket& bucket);
+  virtual int create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
+                            const string& region_name,
+                            const string& placement_rule,
+                            map& attrs,
+                            RGWBucketInfo& bucket_info,
+                            obj_version *pobjv,
+                            obj_version *pep_objv,
+                            time_t creation_time,
+                            rgw_bucket *master_bucket,
+                            bool exclusive = true);
+  virtual int add_bucket_placement(std::string& new_pool);
+  virtual int remove_bucket_placement(std::string& new_pool);
+  virtual int list_placement_set(set& names);
+  virtual int create_pools(vector& names, vector& retcodes);
+
+  struct PutObjMetaExtraParams {
+    time_t *mtime;
+    map* rmattrs;
+    const bufferlist *data;
+    RGWObjManifest *manifest;
+    const string *ptag;
+    list *remove_objs;
+    bool modify_version;
+    RGWObjVersionTracker *objv_tracker;
+    time_t set_mtime;
+    string owner;
+
+    PutObjMetaExtraParams() : mtime(NULL), rmattrs(NULL),
+                     data(NULL), manifest(NULL), ptag(NULL),
+                     remove_objs(NULL), modify_version(false),
+                     objv_tracker(NULL), set_mtime(0) {}
+  };
+
+  /** Write/overwrite an object to the bucket storage. */
+  virtual int put_obj_meta_impl(void *ctx, rgw_obj& obj, uint64_t size, time_t *mtime,
+              map& attrs, RGWObjCategory category, int flags,
+              map* rmattrs, const bufferlist *data,
+              RGWObjManifest *manifest, const string *ptag, list *remove_objs,
+              bool modify_version, RGWObjVersionTracker *objv_tracker,
+              time_t set_mtime /* 0 for don't set */,
+              const string& owner);
+
+  virtual int put_obj_meta(void *ctx, rgw_obj& obj, uint64_t size, time_t *mtime,
+              map& attrs, RGWObjCategory category, int flags,
+              const string& owner, const bufferlist *data = NULL) {
+    return put_obj_meta_impl(ctx, obj, size, mtime, attrs, category, flags,
+                        NULL, data, NULL, NULL, NULL,
+                        false, NULL, 0, owner);
+  }
+
+  virtual int put_obj_meta(void *ctx, rgw_obj& obj, uint64_t size,  map& attrs,
+                           RGWObjCategory category, int flags, PutObjMetaExtraParams& params) {
+    return put_obj_meta_impl(ctx, obj, size, params.mtime, attrs, category, flags,
+                        params.rmattrs, params.data, params.manifest, params.ptag, params.remove_objs,
+                        params.modify_version, params.objv_tracker, params.set_mtime, params.owner);
+  }
+
+  virtual int put_obj_data(void *ctx, rgw_obj& obj, const char *data,
+              off_t ofs, size_t len, bool exclusive);
+  virtual int aio_put_obj_data(void *ctx, rgw_obj& obj, bufferlist& bl,
+                               off_t ofs, bool exclusive, void **handle);
+  /* note that put_obj doesn't set category on an object, only use it for none user objects */
+  int put_system_obj(void *ctx, rgw_obj& obj, const char *data, size_t len, bool exclusive,
+              time_t *mtime, map& attrs, RGWObjVersionTracker *objv_tracker,
+              time_t set_mtime) {
+    bufferlist bl;
+    bl.append(data, len);
+    int flags = PUT_OBJ_CREATE;
+    if (exclusive)
+      flags |= PUT_OBJ_EXCL;
+
+    PutObjMetaExtraParams ep;
+    ep.mtime = mtime;
+    ep.data = &bl;
+    ep.modify_version = true;
+    ep.objv_tracker = objv_tracker;
+    ep.set_mtime = set_mtime;
+
+    int ret = put_obj_meta(ctx, obj, len, attrs, RGW_OBJ_CATEGORY_NONE, flags, ep);
+    return ret;
+  }
+  virtual int aio_wait(void *handle);
+  virtual bool aio_completed(void *handle);
+  virtual int clone_objs(void *ctx, rgw_obj& dst_obj, 
+                         vector& ranges,
+                         map attrs,
+                         RGWObjCategory category,
+                         time_t *pmtime, bool truncate_dest, bool exclusive) {
+    return clone_objs(ctx, dst_obj, ranges, attrs, category, pmtime, truncate_dest, exclusive, NULL);
+  }
+
+  int clone_objs(void *ctx, rgw_obj& dst_obj, 
+                 vector& ranges,
+                 map attrs,
+                 RGWObjCategory category,
+                 time_t *pmtime,
+                 bool truncate_dest,
+                 bool exclusive,
+                 pair *cmp_xattr);
+
+  int clone_obj_cond(void *ctx, rgw_obj& dst_obj, off_t dst_ofs,
+                rgw_obj& src_obj, off_t src_ofs,
+                uint64_t size, map attrs,
+                RGWObjCategory category,
+                time_t *pmtime,
+                bool truncate_dest,
+                bool exclusive,
+                pair *xattr_cond) {
+    RGWCloneRangeInfo info;
+    vector v;
+    info.src = src_obj;
+    info.src_ofs = src_ofs;
+    info.dst_ofs = dst_ofs;
+    info.len = size;
+    v.push_back(info);
+    return clone_objs(ctx, dst_obj, v, attrs, category, pmtime, truncate_dest, exclusive, xattr_cond);
+  }
+
+  /**
+   * Copy an object.
+   * dest_obj: the object to copy into
+   * src_obj: the object to copy from
+   * attrs: if replace_attrs is set then these are placed on the new object
+   * err: stores any errors resulting from the get of the original object
+   * Returns: 0 on success, -ERR# otherwise.
+   */
+  virtual int copy_obj(void *ctx,
+               const string& user_id,
+               const string& client_id,
+               const string& op_id,
+               req_info *info,
+               const string& source_zone,
+               rgw_obj& dest_obj,
+               rgw_obj& src_obj,
+               RGWBucketInfo& dest_bucket_info,
+               RGWBucketInfo& src_bucket_info,
+               time_t *mtime,
+               const time_t *mod_ptr,
+               const time_t *unmod_ptr,
+               const char *if_match,
+               const char *if_nomatch,
+               bool replace_attrs,
+               map& attrs,
+               RGWObjCategory category,
+               string *ptag,
+               struct rgw_err *err,
+               void (*progress_cb)(off_t, void *),
+               void *progress_data);
+
+  int copy_obj_data(void *ctx,
+               const string& owner,
+	       void **handle, off_t end,
+               rgw_obj& dest_obj,
+               rgw_obj& src_obj,
+               uint64_t max_chunk_size,
+	       time_t *mtime,
+               map& attrs,
+               RGWObjCategory category,
+               string *ptag,
+               struct rgw_err *err);
+  /**
+   * Delete a bucket.
+   * bucket: the name of the bucket to delete
+   * Returns 0 on success, -ERR# otherwise.
+   */
+  virtual int delete_bucket(rgw_bucket& bucket, RGWObjVersionTracker& objv_tracker);
+
+  int set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner);
+  int set_buckets_enabled(std::vector& buckets, bool enabled);
+  int bucket_suspended(rgw_bucket& bucket, bool *suspended);
+
+  /** Delete an object.*/
+  virtual int delete_obj(void *ctx, const string& bucket_owner, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker = NULL);
+  virtual int delete_system_obj(void *ctx, rgw_obj& src_obj, RGWObjVersionTracker *objv_tracker = NULL);
+
+  /** Remove an object from the bucket index */
+  int delete_obj_index(rgw_obj& obj);
+
+  /**
+   * Get the attributes for an object.
+   * bucket: name of the bucket holding the object.
+   * obj: name of the object
+   * name: name of the attr to retrieve
+   * dest: bufferlist to store the result in
+   * Returns: 0 on success, -ERR# otherwise.
+   */
+  virtual int get_attr(void *ctx, rgw_obj& obj, const char *name, bufferlist& dest);
+
+  /**
+   * Set an attr on an object.
+   * bucket: name of the bucket holding the object
+   * obj: name of the object to set the attr on
+   * name: the attr to set
+   * bl: the contents of the attr
+   * Returns: 0 on success, -ERR# otherwise.
+   */
+  virtual int set_attr(void *ctx, rgw_obj& obj, const char *name, bufferlist& bl,
+                       RGWObjVersionTracker *objv_tracker);
+
+  virtual int set_attrs(void *ctx, rgw_obj& obj,
+                        map& attrs,
+                        map* rmattrs,
+                        RGWObjVersionTracker *objv_tracker);
+
+/**
+ * Get data about an object out of RADOS and into memory.
+ * bucket: name of the bucket the object is in.
+ * obj: name/key of the object to read
+ * data: if get_data==true, this pointer will be set
+ *    to an address containing the object's data/value
+ * ofs: the offset of the object to read from
+ * end: the point in the object to stop reading
+ * attrs: if non-NULL, the pointed-to map will contain
+ *    all the attrs of the object when this function returns
+ * mod_ptr: if non-NULL, compares the object's mtime to *mod_ptr,
+ *    and if mtime is smaller it fails.
+ * unmod_ptr: if non-NULL, compares the object's mtime to *unmod_ptr,
+ *    and if mtime is >= it fails.
+ * if_match/nomatch: if non-NULL, compares the object's etag attr
+ *    to the string and, if it doesn't/does match, fails out.
+ * err: Many errors will result in this structure being filled
+ *    with extra informatin on the error.
+ * Returns: -ERR# on failure, otherwise
+ *          (if get_data==true) length of read data,
+ *          (if get_data==false) length of the object
+ */
+  virtual int prepare_get_obj(void *ctx, rgw_obj& obj,
+            off_t *ofs, off_t *end,
+            map *attrs,
+            const time_t *mod_ptr,
+            const time_t *unmod_ptr,
+            time_t *lastmod,
+            const char *if_match,
+            const char *if_nomatch,
+            uint64_t *total_size,
+            uint64_t *obj_size,
+            RGWObjVersionTracker *objv_tracker,
+            void **handle,
+            struct rgw_err *err);
+
+  virtual int get_obj(void *ctx, RGWObjVersionTracker *objv_tracker, void **handle, rgw_obj& obj,
+                      bufferlist& bl, off_t ofs, off_t end);
+
+  virtual void finish_get_obj(void **handle);
+
+  int iterate_obj(void *ctx, rgw_obj& obj,
+                  off_t ofs, off_t end,
+                  uint64_t max_chunk_size,
+                  int (*iterate_obj_cb)(rgw_obj&, off_t, off_t, off_t, bool, RGWObjState *, void *),
+                  void *arg);
+
+  int get_obj_iterate(void *ctx, void **handle, rgw_obj& obj,
+                      off_t ofs, off_t end,
+	              RGWGetDataCB *cb);
+
+  int flush_read_list(struct get_obj_data *d);
+
+  int get_obj_iterate_cb(void *ctx, RGWObjState *astate,
+                         rgw_obj& obj,
+                         off_t obj_ofs, off_t read_ofs, off_t len,
+                         bool is_head_obj, void *arg);
+
+  void get_obj_aio_completion_cb(librados::completion_t cb, void *arg);
+
+  /**
+   * a simple object read without keeping state
+   */
+  virtual int read(void *ctx, rgw_obj& obj, off_t ofs, size_t size, bufferlist& bl);
+
+  virtual int obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime,
+                       uint64_t *epoch, map *attrs, bufferlist *first_chunk,
+                       RGWObjVersionTracker *objv_tracker);
+
+  virtual bool supports_omap() { return true; }
+  int omap_get_vals(rgw_obj& obj, bufferlist& header, const std::string& marker, uint64_t count, std::map& m);
+  virtual int omap_get_all(rgw_obj& obj, bufferlist& header, std::map& m);
+  virtual int omap_set(rgw_obj& obj, std::string& key, bufferlist& bl);
+  virtual int omap_set(rgw_obj& obj, map& m);
+  virtual int omap_del(rgw_obj& obj, const std::string& key);
+  virtual int update_containers_stats(map& m);
+  virtual int append_async(rgw_obj& obj, size_t size, bufferlist& bl);
+
+  virtual bool need_watch_notify() { return false; }
+  virtual int init_watch();
+  virtual void finalize_watch();
+  virtual int distribute(const string& key, bufferlist& bl);
+  virtual int watch_cb(int opcode, uint64_t ver, bufferlist& bl) { return 0; }
+  void pick_control_oid(const string& key, string& notify_oid);
+
+  void *create_context(void *user_ctx) {
+    RGWRadosCtx *rctx = new RGWRadosCtx(this);
+    rctx->user_ctx = user_ctx;
+    return rctx;
+  }
+  void destroy_context(void *ctx) {
+    delete static_cast(ctx);
+  }
+  void set_atomic(void *ctx, rgw_obj& obj) {
+    RGWRadosCtx *rctx = static_cast(ctx);
+    rctx->set_atomic(obj);
+  }
+  void set_prefetch_data(void *ctx, rgw_obj& obj) {
+    RGWRadosCtx *rctx = static_cast(ctx);
+    rctx->set_prefetch_data(obj);
+  }
+  // to notify upper layer that we need to do some operation on an object, and it's up to
+  // the upper layer to schedule this operation.. e.g., log intent in intent log
+  void set_intent_cb(void *ctx, int (*cb)(RGWRados *store, void *user_ctx, rgw_obj& obj, RGWIntentEvent intent)) {
+    RGWRadosCtx *rctx = static_cast(ctx);
+    rctx->set_intent_cb(cb);
+  }
+
+  int decode_policy(bufferlist& bl, ACLOwner *owner);
+  int get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map& stats,
+                       string *max_marker);
+  int get_bucket_stats_async(rgw_bucket& bucket, RGWGetBucketStats_CB *cb);
+  int get_user_stats(const string& user, RGWStorageStats& stats);
+  int get_user_stats_async(const string& user, RGWGetUserStats_CB *cb);
+  void get_bucket_instance_obj(rgw_bucket& bucket, rgw_obj& obj);
+  void get_bucket_instance_entry(rgw_bucket& bucket, string& entry);
+  void get_bucket_meta_oid(rgw_bucket& bucket, string& oid);
+
+  int put_bucket_entrypoint_info(const string& bucket_name, RGWBucketEntryPoint& entry_point, bool exclusive, RGWObjVersionTracker& objv_tracker, time_t mtime,
+                                 map *pattrs);
+  int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, time_t mtime, map *pattrs);
+  int get_bucket_entrypoint_info(void *ctx, const string& bucket_name, RGWBucketEntryPoint& entry_point, RGWObjVersionTracker *objv_tracker, time_t *pmtime,
+                                 map *pattrs);
+  int get_bucket_instance_info(void *ctx, const string& meta_key, RGWBucketInfo& info, time_t *pmtime, map *pattrs);
+  int get_bucket_instance_info(void *ctx, rgw_bucket& bucket, RGWBucketInfo& info, time_t *pmtime, map *pattrs);
+  int get_bucket_instance_from_oid(void *ctx, string& oid, RGWBucketInfo& info, time_t *pmtime, map *pattrs);
+
+  int convert_old_bucket_info(void *ctx, string& bucket_name);
+  virtual int get_bucket_info(void *ctx, const string& bucket_name, RGWBucketInfo& info,
+                              time_t *pmtime, map *pattrs = NULL);
+  virtual int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, time_t mtime, obj_version *pep_objv,
+                                     map *pattrs, bool create_entry_point);
+
+  int cls_rgw_init_index(librados::IoCtx& io_ctx, librados::ObjectWriteOperation& op, string& oid);
+  int cls_obj_prepare_op(rgw_bucket& bucket, RGWModifyOp op, string& tag,
+                         string& name, string& locator);
+  int cls_obj_complete_op(rgw_bucket& bucket, RGWModifyOp op, string& tag, int64_t pool, uint64_t epoch,
+                          RGWObjEnt& ent, RGWObjCategory category, list *remove_objs);
+  int cls_obj_complete_add(rgw_bucket& bucket, string& tag, int64_t pool, uint64_t epoch, RGWObjEnt& ent, RGWObjCategory category, list *remove_objs);
+  int cls_obj_complete_del(rgw_bucket& bucket, string& tag, int64_t pool, uint64_t epoch, string& name);
+  int cls_obj_complete_cancel(rgw_bucket& bucket, string& tag, string& name);
+  int cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout);
+  int cls_bucket_list(rgw_bucket& bucket, string start, string prefix, uint32_t num,
+                      map& m, bool *is_truncated,
+                      string *last_entry, bool (*force_check_filter)(const string&  name) = NULL);
+  int cls_bucket_head(rgw_bucket& bucket, struct rgw_bucket_dir_header& header);
+  int cls_bucket_head_async(rgw_bucket& bucket, RGWGetDirHeader_CB *ctx);
+  int prepare_update_index(RGWObjState *state, rgw_bucket& bucket,
+                           RGWModifyOp op, rgw_obj& oid, string& tag);
+  int complete_update_index(rgw_bucket& bucket, string& oid, string& tag, int64_t poolid, uint64_t epoch, uint64_t size,
+                            utime_t& ut, string& etag, string& content_type, bufferlist *acl_bl, RGWObjCategory category,
+			    list *remove_objs);
+  int complete_update_index_del(rgw_bucket& bucket, string& oid, string& tag, int64_t pool, uint64_t epoch) {
+    if (bucket_is_system(bucket))
+      return 0;
+
+    return cls_obj_complete_del(bucket, tag, pool, epoch, oid);
+  }
+  int complete_update_index_cancel(rgw_bucket& bucket, string& oid, string& tag) {
+    if (bucket_is_system(bucket))
+      return 0;
+
+    return cls_obj_complete_cancel(bucket, tag, oid);
+  }
+  int list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t max, std::list& result, bool *truncated);
+  int trim_bi_log_entries(rgw_bucket& bucket, string& marker, string& end_marker);
+
+  int cls_obj_usage_log_add(const string& oid, rgw_usage_log_info& info);
+  int cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
+                             string& read_iter, map& usage, bool *is_truncated);
+  int cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch);
+
+  void shard_name(const string& prefix, unsigned max_shards, const string& key, string& name);
+  void shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name);
+  void time_log_prepare_entry(cls_log_entry& entry, const utime_t& ut, string& section, string& key, bufferlist& bl);
+  int time_log_add(const string& oid, list& entries);
+  int time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl);
+  int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
+                    int max_entries, list& entries,
+		    const string& marker, string *out_marker, bool *truncated);
+  int time_log_info(const string& oid, cls_log_header *header);
+  int time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time,
+                    const string& from_marker, const string& to_marker);
+  int lock_exclusive(rgw_bucket& pool, const string& oid, utime_t& duration, string& zone_id, string& owner_id);
+  int unlock(rgw_bucket& pool, const string& oid, string& zone_id, string& owner_id);
+
+  /// clean up/process any temporary objects older than given date[/time]
+  int remove_temp_objects(string date, string time);
+
+  int gc_operate(string& oid, librados::ObjectWriteOperation *op);
+  int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op);
+  int gc_operate(string& oid, librados::ObjectReadOperation *op, bufferlist *pbl);
+
+  int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated);
+  int process_gc();
+  int defer_gc(void *ctx, rgw_obj& obj);
+
+  int bucket_check_index(rgw_bucket& bucket,
+                         map *existing_stats,
+                         map *calculated_stats);
+  int bucket_rebuild_index(rgw_bucket& bucket);
+  int remove_objs_from_index(rgw_bucket& bucket, list& oid_list);
+
+  int cls_user_get_header(const string& user_id, cls_user_header *header);
+  int cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx);
+  int cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket);
+  int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
+  int cls_user_list_buckets(rgw_obj& obj,
+                            const string& in_marker, int max_entries,
+                            list& entries,
+                            string *out_marker, bool *truncated);
+  int cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry);
+  int cls_user_update_buckets(rgw_obj& obj, list& entries, bool add);
+  int cls_user_complete_stats_sync(rgw_obj& obj);
+  int complete_sync_user_stats(const string& user_id);
+  int cls_user_add_bucket(rgw_obj& obj, list& entries);
+  int cls_user_remove_bucket(rgw_obj& obj, const cls_user_bucket& bucket);
+
+  int check_quota(const string& bucket_owner, rgw_bucket& bucket,
+                  RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size);
+
+  string unique_id(uint64_t unique_num) {
+    char buf[32];
+    snprintf(buf, sizeof(buf), ".%llu.%llu", (unsigned long long)instance_id(), (unsigned long long)unique_num);
+    string s = zone.name + buf;
+    return s;
+  }
+
+
+  void get_log_pool_name(string& name) {
+    name = zone.log_pool.name;
+  }
+
+  bool need_to_log_data() {
+    return zone_public_config.log_data;
+  }
+
+  bool need_to_log_metadata() {
+    return zone_public_config.log_meta;
+  }
+
+ private:
+  int process_intent_log(rgw_bucket& bucket, string& oid,
+			 time_t epoch, int flags, bool purge);
+  /**
+   * Check the actual on-disk state of the object specified
+   * by list_state, and fill in the time and size of object.
+   * Then append any changes to suggested_updates for
+   * the rgw class' dir_suggest_changes function.
+   *
+   * Note that this can maul list_state; don't use it afterwards. Also
+   * it expects object to already be filled in from list_state; it only
+   * sets the size and mtime.
+   *
+   * Returns 0 on success, -ENOENT if the object doesn't exist on disk,
+   * and -errno on other failures. (-ENOENT is not a failure, and it
+   * will encode that info as a suggested update.)
+   */
+  int check_disk_state(librados::IoCtx io_ctx,
+                       rgw_bucket& bucket,
+                       rgw_bucket_dir_entry& list_state,
+                       RGWObjEnt& object,
+                       bufferlist& suggested_updates);
+
+  bool bucket_is_system(rgw_bucket& bucket) {
+    return (bucket.name[0] == '.');
+  }
+
+  /**
+   * Init pool iteration
+   * bucket: pool name in a bucket object
+   * ctx: context object to use for the iteration
+   * Returns: 0 on success, -ERR# otherwise.
+   */
+  int pool_iterate_begin(rgw_bucket& bucket, RGWPoolIterCtx& ctx);
+  /**
+   * Iterate over pool return object names, use optional filter
+   * ctx: iteration context, initialized with pool_iterate_begin()
+   * num: max number of objects to return
+   * objs: a vector that the results will append into
+   * is_truncated: if not NULL, will hold true iff iteration is complete
+   * filter: if not NULL, will be used to filter returned objects
+   * Returns: 0 on success, -ERR# otherwise.
+   */
+  int pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector& objs,
+                   bool *is_truncated, RGWAccessListFilter *filter);
+
+  uint64_t instance_id();
+  uint64_t next_bucket_id();
+};
+
+class RGWStoreManager {
+public:
+  RGWStoreManager() {}
+  static RGWRados *get_storage(CephContext *cct, bool use_gc_thread, bool quota_threads) {
+    RGWRados *store = init_storage_provider(cct, use_gc_thread, quota_threads);
+    return store;
+  }
+  static RGWRados *get_raw_storage(CephContext *cct) {
+    RGWRados *store = init_raw_storage_provider(cct);
+    return store;
+  }
+  static RGWRados *init_storage_provider(CephContext *cct, bool use_gc_thread, bool quota_threads);
+  static RGWRados *init_raw_storage_provider(CephContext *cct);
+  static void close_storage(RGWRados *store);
+
+};
+
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_replica_log.cc b/ceph/src/rgw/rgw_replica_log.cc
new file mode 100644
index 00000000..f80ebf88
--- /dev/null
+++ b/ceph/src/rgw/rgw_replica_log.cc
@@ -0,0 +1,132 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ * Copyright 2013 Inktank
+ */
+
+#include "common/ceph_json.h"
+
+#include "rgw_replica_log.h"
+#include "cls/replica_log/cls_replica_log_client.h"
+#include "rgw_rados.h"
+
+
+void RGWReplicaBounds::dump(Formatter *f) const
+{
+  encode_json("marker", marker, f);
+  encode_json("oldest_time", oldest_time, f);
+  encode_json("markers", markers, f);
+}
+
+void RGWReplicaBounds::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("marker", marker, obj);
+  JSONDecoder::decode_json("oldest_time", oldest_time, obj);
+  JSONDecoder::decode_json("markers", markers, obj);
+}
+
+RGWReplicaLogger::RGWReplicaLogger(RGWRados *_store) :
+    cct(_store->cct), store(_store) {}
+
+int RGWReplicaLogger::open_ioctx(librados::IoCtx& ctx, const string& pool)
+{
+  int r = store->rados->ioctx_create(pool.c_str(), ctx);
+  if (r == -ENOENT) {
+    rgw_bucket p(pool.c_str());
+    r = store->create_pool(p);
+    if (r < 0)
+      return r;
+
+    // retry
+    r = store->rados->ioctx_create(pool.c_str(), ctx);
+  }
+  if (r < 0) {
+    lderr(cct) << "ERROR: could not open rados pool " << pool << dendl;
+  }
+  return r;
+}
+
+int RGWReplicaLogger::update_bound(const string& oid, const string& pool,
+                                   const string& daemon_id,
+                                   const string& marker, const utime_t& time,
+                                   const list *entries)
+{
+  cls_replica_log_progress_marker progress;
+  progress.entity_id = daemon_id;
+  progress.position_marker = marker;
+  progress.position_time = time;
+  progress.items = *entries;
+
+  librados::IoCtx ioctx;
+  int r = open_ioctx(ioctx, pool);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectWriteOperation opw;
+  cls_replica_log_update_bound(opw, progress);
+  return ioctx.operate(oid, &opw);
+}
+
+int RGWReplicaLogger::delete_bound(const string& oid, const string& pool,
+                                   const string& daemon_id)
+{
+  librados::IoCtx ioctx;
+  int r = open_ioctx(ioctx, pool);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectWriteOperation opw;
+  cls_replica_log_delete_bound(opw, daemon_id);
+  return ioctx.operate(oid, &opw);
+}
+
+int RGWReplicaLogger::get_bounds(const string& oid, const string& pool,
+                                 RGWReplicaBounds& bounds)
+{
+  librados::IoCtx ioctx;
+  int r = open_ioctx(ioctx, pool);
+  if (r < 0) {
+    return r;
+  }
+
+  return cls_replica_log_get_bounds(ioctx, oid, bounds.marker, bounds.oldest_time, bounds.markers);
+}
+
+RGWReplicaObjectLogger::
+RGWReplicaObjectLogger(RGWRados *_store,
+                       const string& _pool,
+                       const string& _prefix) : RGWReplicaLogger(_store),
+                       pool(_pool), prefix(_prefix) {
+  if (pool.empty())
+    store->get_log_pool_name(pool);
+}
+
+int RGWReplicaObjectLogger::create_log_objects(int shards)
+{
+  librados::IoCtx ioctx;
+  int r = open_ioctx(ioctx, pool);
+  if (r < 0) {
+    return r;
+  }
+  for (int i = 0; i < shards; ++i) {
+    string oid;
+    get_shard_oid(i, oid);
+    r = ioctx.create(oid, false);
+    if (r < 0)
+      return r;
+  }
+  return r;
+}
+
+RGWReplicaBucketLogger::RGWReplicaBucketLogger(RGWRados *_store) :
+  RGWReplicaLogger(_store)
+{
+  store->get_log_pool_name(pool);
+  prefix = _store->ctx()->_conf->rgw_replica_log_obj_prefix;
+  prefix.append(".");
+}
diff --git a/ceph/src/rgw/rgw_replica_log.h b/ceph/src/rgw/rgw_replica_log.h
new file mode 100644
index 00000000..f02fa423
--- /dev/null
+++ b/ceph/src/rgw/rgw_replica_log.h
@@ -0,0 +1,116 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef RGW_REPLICA_LOG_H_
+#define RGW_REPLICA_LOG_H_
+
+#include 
+#include "cls/replica_log/cls_replica_log_types.h"
+#include "include/types.h"
+#include "include/utime.h"
+#include "include/rados/librados.hpp"
+#include "rgw_common.h"
+
+class RGWRados;
+class CephContext;
+
+using namespace std;
+
+#define META_REPLICA_LOG_OBJ_PREFIX "meta.replicalog."
+#define DATA_REPLICA_LOG_OBJ_PREFIX "data.replicalog."
+
+typedef cls_replica_log_item_marker RGWReplicaItemMarker;
+typedef cls_replica_log_progress_marker RGWReplicaProgressMarker;
+
+struct RGWReplicaBounds {
+  string marker;
+  utime_t oldest_time;
+  list markers;
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+
+class RGWReplicaLogger {
+protected:
+  CephContext *cct;
+  RGWRados *store;
+  int open_ioctx(librados::IoCtx& ctx, const string& pool);
+
+  RGWReplicaLogger(RGWRados *_store);
+
+  int update_bound(const string& oid, const string& pool,
+                   const string& daemon_id, const string& marker,
+                   const utime_t& time,
+                   const list *entries);
+  int delete_bound(const string& oid, const string& pool,
+                   const string& daemon_id);
+  int get_bounds(const string& oid, const string& pool,
+                 RGWReplicaBounds& bounds);
+};
+
+class RGWReplicaObjectLogger : private RGWReplicaLogger {
+  string pool;
+  string prefix;
+
+  void get_shard_oid(int id, string& oid) {
+    char buf[16];
+    snprintf(buf, sizeof(buf), "%d", id);
+    oid = prefix + buf;
+  }
+
+public:
+  RGWReplicaObjectLogger(RGWRados *_store,
+                const string& _pool,
+                const string& _prefix);
+
+  int create_log_objects(int shards);
+  int update_bound(int shard, const string& daemon_id, const string& marker,
+                   const utime_t& time,
+                   const list *entries) {
+    string oid;
+    get_shard_oid(shard, oid);
+    return RGWReplicaLogger::update_bound(oid, pool,
+                                          daemon_id, marker, time, entries);
+  }
+  int delete_bound(int shard, const string& daemon_id) {
+    string oid;
+    get_shard_oid(shard, oid);
+    return RGWReplicaLogger::delete_bound(oid, pool,
+                                          daemon_id);
+  }
+  int get_bounds(int shard, RGWReplicaBounds& bounds) {
+    string oid;
+    get_shard_oid(shard, oid);
+    return RGWReplicaLogger::get_bounds(oid, pool, bounds);
+  }
+};
+
+class RGWReplicaBucketLogger : private RGWReplicaLogger {
+  string pool;
+  string prefix;
+public:
+  RGWReplicaBucketLogger(RGWRados *_store);
+  int update_bound(const rgw_bucket& bucket, const string& daemon_id,
+                   const string& marker, const utime_t& time,
+                   const list *entries) {
+    return RGWReplicaLogger::update_bound(prefix+bucket.name, pool,
+                                          daemon_id, marker, time, entries);
+  }
+  int delete_bound(const rgw_bucket& bucket, const string& daemon_id) {
+    return RGWReplicaLogger::delete_bound(prefix+bucket.name, pool,
+                                          daemon_id);
+  }
+  int get_bounds(const rgw_bucket& bucket, RGWReplicaBounds& bounds) {
+    return RGWReplicaLogger::get_bounds(prefix+bucket.name, pool,
+                                        bounds);
+  }
+};
+
+#endif /* RGW_REPLICA_LOG_H_ */
diff --git a/ceph/src/rgw/rgw_resolve.cc b/ceph/src/rgw/rgw_resolve.cc
new file mode 100644
index 00000000..471ac3f1
--- /dev/null
+++ b/ceph/src/rgw/rgw_resolve.cc
@@ -0,0 +1,183 @@
+#include 
+#include 
+#include 
+#include 
+
+#include "acconfig.h"
+
+#ifdef HAVE_ARPA_NAMESER_COMPAT_H
+#include 
+#endif
+
+#include "rgw_common.h"
+#include "rgw_resolve.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+class RGWDNSResolver {
+  list states;
+  Mutex lock;
+
+  int get_state(res_state *ps);
+  void put_state(res_state s);
+
+
+public:
+  ~RGWDNSResolver();
+  RGWDNSResolver() : lock("RGWDNSResolver") {}
+  int resolve_cname(const string& hostname, string& cname, bool *found);
+};
+
+RGWDNSResolver::~RGWDNSResolver()
+{
+  list::iterator iter;
+  for (iter = states.begin(); iter != states.end(); ++iter) {
+    struct __res_state *s = *iter;
+    delete s;
+  }
+}
+
+
+int RGWDNSResolver::get_state(res_state *ps)
+{
+  lock.Lock();
+  if (!states.empty()) {
+    res_state s = states.front();
+    states.pop_front();
+    lock.Unlock();
+    *ps = s;
+    return 0;
+  }
+  lock.Unlock();
+  struct __res_state *s = new struct __res_state;
+  s->options = 0;
+  if (res_ninit(s) < 0) {
+    delete s;
+    dout(0) << "ERROR: failed to call res_ninit()" << dendl;
+    return -EINVAL;
+  }
+  *ps = s;
+  return 0;
+}
+
+void RGWDNSResolver::put_state(res_state s)
+{
+  Mutex::Locker l(lock);
+  states.push_back(s);
+}
+
+
+int RGWDNSResolver::resolve_cname(const string& hostname, string& cname, bool *found)
+{
+  res_state res;
+
+  *found = false;
+
+  int r = get_state(&res);
+  if (r < 0) {
+    return r;
+  }
+
+  int ret;
+
+#define LARGE_ENOUGH_DNS_BUFSIZE 1024
+  unsigned char buf[LARGE_ENOUGH_DNS_BUFSIZE];
+
+#define MAX_FQDN_SIZE 255
+  char host[MAX_FQDN_SIZE + 1];
+  const char *origname = hostname.c_str();
+  unsigned char *pt, *answer;
+  unsigned char *answend;
+  int len = res_nquery(res, origname, C_IN, T_CNAME, buf, sizeof(buf));
+  if (len < 0) {
+    dout(20) << "res_query() failed" << dendl;
+    ret = 0;
+    goto done;
+  }
+
+  answer = buf;
+  pt = answer + sizeof(HEADER);
+  answend = answer + len;
+
+  /* read query */
+  if ((len = dn_expand(answer, answend, pt, host, sizeof(host))) < 0) {
+    dout(0) << "ERROR: dn_expand() failed" << dendl;
+    ret = -EINVAL;
+    goto done;
+  }
+  pt += len;
+
+  if (pt + 4 > answend) {
+    dout(0) << "ERROR: bad reply" << dendl;
+    ret = -EIO;
+    goto done;
+  }
+
+  int type;
+  GETSHORT(type, pt);
+
+  if (type != T_CNAME) {
+    dout(0) << "ERROR: failed response type: type=%d (was expecting " << T_CNAME << ")" << dendl;
+    ret = -EIO;
+    goto done;
+  }
+
+  pt += INT16SZ; /* class */
+
+  /* read answer */
+
+  if ((len = dn_expand(answer, answend, pt, host, sizeof(host))) < 0) {
+    ret = 0;
+    goto done;
+  }
+  pt += len;
+  dout(20) << "name=" << host << dendl;
+
+  if (pt + 10 > answend) {
+    dout(0) << "ERROR: bad reply" << dendl;
+    ret = -EIO;
+    goto done;
+  }
+
+  GETSHORT(type, pt);
+  pt += INT16SZ; /* class */
+  pt += INT32SZ; /* ttl */
+  pt += INT16SZ; /* size */
+
+  if ((len = dn_expand(answer, answend, pt, host, sizeof(host))) < 0) {
+    ret = 0;
+    goto done;
+  }
+  dout(20) << "cname host=" << host << dendl;
+  cname = host;
+
+  *found = true;
+  ret = 0;
+done:
+  put_state(res);
+  return ret;
+}
+
+RGWResolver::~RGWResolver() {
+  delete resolver;
+}
+RGWResolver::RGWResolver() {
+  resolver = new RGWDNSResolver;
+}
+
+int RGWResolver::resolve_cname(const string& hostname, string& cname, bool *found) {
+  return resolver->resolve_cname(hostname, cname, found);
+};
+
+RGWResolver *rgw_resolver;
+
+
+void rgw_init_resolver()
+{
+  rgw_resolver = new RGWResolver();
+}
+
+void rgw_shutdown_resolver()
+{
+  delete rgw_resolver;
+}
diff --git a/ceph/src/rgw/rgw_resolve.h b/ceph/src/rgw/rgw_resolve.h
new file mode 100644
index 00000000..3b29ba58
--- /dev/null
+++ b/ceph/src/rgw/rgw_resolve.h
@@ -0,0 +1,22 @@
+#ifndef CEPH_RGW_RESOLVE_H
+#define CEPH_RGW_RESOLVE_H
+
+#include "rgw_common.h"
+
+class RGWDNSResolver;
+
+class RGWResolver {
+  RGWDNSResolver *resolver;
+
+public:
+  ~RGWResolver();
+  RGWResolver();
+  int resolve_cname(const string& hostname, string& cname, bool *found);
+};
+
+
+extern void rgw_init_resolver(void);
+extern void rgw_shutdown_resolver(void);
+extern RGWResolver *rgw_resolver;
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest.cc b/ceph/src/rgw/rgw_rest.cc
new file mode 100644
index 00000000..b74002de
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest.cc
@@ -0,0 +1,1285 @@
+#include 
+#include 
+
+#include "common/Formatter.h"
+#include "common/utf8.h"
+#include "include/str_list.h"
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "rgw_formats.h"
+#include "rgw_op.h"
+#include "rgw_rest.h"
+#include "rgw_rest_swift.h"
+#include "rgw_rest_s3.h"
+#include "rgw_swift_auth.h"
+#include "rgw_cors_s3.h"
+#include "rgw_http_errors.h"
+
+#include "rgw_client_io.h"
+#include "rgw_resolve.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+
+struct rgw_http_attr {
+  const char *rgw_attr;
+  const char *http_attr;
+};
+
+/*
+ * mapping between rgw object attrs and output http fields
+ */
+static struct rgw_http_attr rgw_to_http_attr_list[] = {
+  { RGW_ATTR_CONTENT_TYPE, "Content-Type"},
+  { RGW_ATTR_CONTENT_LANG, "Content-Language"},
+  { RGW_ATTR_EXPIRES, "Expires"},
+  { RGW_ATTR_CACHE_CONTROL, "Cache-Control"},
+  { RGW_ATTR_CONTENT_DISP, "Content-Disposition"},
+  { RGW_ATTR_CONTENT_ENC, "Content-Encoding"},
+  { RGW_ATTR_USER_MANIFEST, "X-Object-Manifest"},
+  { NULL, NULL},
+};
+
+
+struct generic_attr {
+  const char *http_header;
+  const char *rgw_attr;
+};
+
+/*
+ * mapping between http env fields and rgw object attrs
+ */
+struct generic_attr generic_attrs[] = {
+  { "CONTENT_TYPE", RGW_ATTR_CONTENT_TYPE },
+  { "HTTP_CONTENT_LANGUAGE", RGW_ATTR_CONTENT_LANG },
+  { "HTTP_EXPIRES", RGW_ATTR_EXPIRES },
+  { "HTTP_CACHE_CONTROL", RGW_ATTR_CACHE_CONTROL },
+  { "HTTP_CONTENT_DISPOSITION", RGW_ATTR_CONTENT_DISP },
+  { "HTTP_CONTENT_ENCODING", RGW_ATTR_CONTENT_ENC },
+  { NULL, NULL },
+};
+
+map rgw_to_http_attrs;
+static map generic_attrs_map;
+map http_status_names;
+
+/*
+ * make attrs look_like_this
+ * converts dashes to underscores
+ */
+string lowercase_underscore_http_attr(const string& orig)
+{
+  const char *s = orig.c_str();
+  char buf[orig.size() + 1];
+  buf[orig.size()] = '\0';
+
+  for (size_t i = 0; i < orig.size(); ++i, ++s) {
+    switch (*s) {
+      case '-':
+	buf[i] = '_';
+	break;
+      default:
+	buf[i] = tolower(*s);
+    }
+  }
+  return string(buf);
+}
+
+/*
+ * make attrs LOOK_LIKE_THIS
+ * converts dashes to underscores
+ */
+string uppercase_underscore_http_attr(const string& orig)
+{
+  const char *s = orig.c_str();
+  char buf[orig.size() + 1];
+  buf[orig.size()] = '\0';
+
+  for (size_t i = 0; i < orig.size(); ++i, ++s) {
+    switch (*s) {
+      case '-':
+	buf[i] = '_';
+	break;
+      default:
+	buf[i] = toupper(*s);
+    }
+  }
+  return string(buf);
+}
+
+/*
+ * make attrs Look-Like-This
+ * converts underscores to dashes
+ */
+string camelcase_dash_http_attr(const string& orig)
+{
+  const char *s = orig.c_str();
+  char buf[orig.size() + 1];
+  buf[orig.size()] = '\0';
+
+  bool last_sep = true;
+
+  for (size_t i = 0; i < orig.size(); ++i, ++s) {
+    switch (*s) {
+      case '_':
+	buf[i] = '-';
+	last_sep = true;
+	break;
+      default:
+	if (last_sep)
+	  buf[i] = toupper(*s);
+	else
+	  buf[i] = tolower(*s);
+	last_sep = false;
+    }
+  }
+  return string(buf);
+}
+
+void rgw_rest_init(CephContext *cct)
+{
+  for (struct rgw_http_attr *attr = rgw_to_http_attr_list; attr->rgw_attr; attr++) {
+    rgw_to_http_attrs[attr->rgw_attr] = attr->http_attr;
+  }
+
+  for (struct generic_attr *gen_attr = generic_attrs; gen_attr->http_header; gen_attr++) {
+    generic_attrs_map[gen_attr->http_header] = gen_attr->rgw_attr;
+  }
+
+  list extended_http_attrs;
+  get_str_list(cct->_conf->rgw_extended_http_attrs, extended_http_attrs);
+
+  list::iterator iter;
+  for (iter = extended_http_attrs.begin(); iter != extended_http_attrs.end(); ++iter) {
+    string rgw_attr = RGW_ATTR_PREFIX;
+    rgw_attr.append(lowercase_underscore_http_attr(*iter));
+
+    rgw_to_http_attrs[rgw_attr] = camelcase_dash_http_attr(*iter);
+
+    string http_header = "HTTP_";
+    http_header.append(uppercase_underscore_http_attr(*iter));
+
+    generic_attrs_map[http_header] = rgw_attr;
+  }
+
+  for (const struct rgw_http_status_code *h = http_codes; h->code; h++) {
+    http_status_names[h->code] = h->name;
+  }
+}
+
+static void dump_status(struct req_state *s, const char *status, const char *status_name)
+{
+  int r = s->cio->send_status(status, status_name);
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->send_status() returned err=" << r << dendl;
+  }
+}
+
+void rgw_flush_formatter_and_reset(struct req_state *s, Formatter *formatter)
+{
+  std::ostringstream oss;
+  formatter->flush(oss);
+  std::string outs(oss.str());
+  if (!outs.empty() && s->op != OP_HEAD) {
+    s->cio->write(outs.c_str(), outs.size());
+  }
+
+  s->formatter->reset();
+}
+
+void rgw_flush_formatter(struct req_state *s, Formatter *formatter)
+{
+  std::ostringstream oss;
+  formatter->flush(oss);
+  std::string outs(oss.str());
+  if (!outs.empty() && s->op != OP_HEAD) {
+    s->cio->write(outs.c_str(), outs.size());
+  }
+}
+
+void set_req_state_err(struct req_state *s, int err_no)
+{
+  const struct rgw_http_errors *r;
+
+  if (err_no < 0)
+    err_no = -err_no;
+  s->err.ret = -err_no;
+  if (s->prot_flags & RGW_REST_SWIFT) {
+    r = search_err(err_no, RGW_HTTP_SWIFT_ERRORS, ARRAY_LEN(RGW_HTTP_SWIFT_ERRORS));
+    if (r) {
+      s->err.http_ret = r->http_ret;
+      s->err.s3_code = r->s3_code;
+      return;
+    }
+  }
+  r = search_err(err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS));
+  if (r) {
+    s->err.http_ret = r->http_ret;
+    s->err.s3_code = r->s3_code;
+    return;
+  }
+  dout(0) << "WARNING: set_req_state_err err_no=" << err_no << " resorting to 500" << dendl;
+
+  s->err.http_ret = 500;
+  s->err.s3_code = "UnknownError";
+}
+
+void dump_errno(struct req_state *s)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%d", s->err.http_ret);
+  dump_status(s, buf, http_status_names[s->err.http_ret]);
+}
+
+void dump_errno(struct req_state *s, int err)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%d", err);
+  dump_status(s, buf, http_status_names[s->err.http_ret]);
+}
+
+void dump_content_length(struct req_state *s, uint64_t len)
+{
+  int r = s->cio->send_content_length(len);
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+  }
+  r = s->cio->print("Accept-Ranges: %s\n", "bytes");
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+  }
+}
+
+void dump_etag(struct req_state *s, const char *etag)
+{
+  int r;
+  if (s->prot_flags & RGW_REST_SWIFT)
+    r = s->cio->print("etag: %s\n", etag);
+  else
+    r = s->cio->print("ETag: \"%s\"\n", etag);
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+  }
+}
+
+void dump_pair(struct req_state *s, const char *key, const char *value)
+{
+  if ( (strlen(key) > 0) && (strlen(value) > 0))
+    s->cio->print("%s: %s\n", key, value);
+}
+
+void dump_bucket_from_state(struct req_state *s)
+{
+  int expose_bucket = g_conf->rgw_expose_bucket;
+  if (expose_bucket) {
+    if (!s->bucket_name_str.empty())
+      s->cio->print("Bucket: \"%s\"\n", s->bucket_name_str.c_str());
+  }
+}
+
+void dump_object_from_state(struct req_state *s)
+{
+  if (!s->object_str.empty())
+    s->cio->print("Key: \"%s\"\n", s->object_str.c_str());
+}
+
+void dump_uri_from_state(struct req_state *s)
+{
+  if (strcmp(s->info.request_uri.c_str(), "/") == 0) {
+
+    string location = "http://";
+    string server = s->info.env->get("SERVER_NAME", "");
+    location.append(server);
+    location += "/";
+    if (!s->bucket_name_str.empty()) {
+      location += s->bucket_name_str;
+      location += "/";
+      if (!s->object_str.empty()) {
+        location += s->object_str;
+        s->cio->print("Location: %s\n", location.c_str());
+      }
+    }
+  }
+  else {
+    s->cio->print("Location: \"%s\"\n", s->info.request_uri.c_str());
+  }
+}
+
+void dump_redirect(struct req_state *s, const string& redirect)
+{
+  if (redirect.empty())
+    return;
+
+  s->cio->print("Location: %s\n", redirect.c_str());
+}
+
+static void dump_time_header(struct req_state *s, const char *name, time_t t)
+{
+
+  char timestr[TIME_BUF_SIZE];
+  struct tm result;
+  struct tm *tmp = gmtime_r(&t, &result);
+  if (tmp == NULL)
+    return;
+
+  if (strftime(timestr, sizeof(timestr), "%a, %d %b %Y %H:%M:%S %Z", tmp) == 0)
+    return;
+
+  int r = s->cio->print("%s: %s\n", name, timestr);
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+  }
+}
+
+void dump_last_modified(struct req_state *s, time_t t)
+{
+  dump_time_header(s, "Last-Modified", t);
+}
+
+void dump_epoch_header(struct req_state *s, const char *name, time_t t)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%lld", (long long)t);
+
+  int r = s->cio->print("%s: %s\n", name, buf);
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+  }
+}
+
+void dump_time(struct req_state *s, const char *name, time_t *t)
+{
+  char buf[TIME_BUF_SIZE];
+  struct tm result;
+  struct tm *tmp = gmtime_r(t, &result);
+  if (tmp == NULL)
+    return;
+
+  if (strftime(buf, sizeof(buf), "%Y-%m-%dT%T.000Z", tmp) == 0)
+    return;
+
+  s->formatter->dump_string(name, buf);
+}
+
+void dump_owner(struct req_state *s, string& id, string& name, const char *section)
+{
+  if (!section)
+    section = "Owner";
+  s->formatter->open_object_section(section);
+  s->formatter->dump_string("ID", id);
+  s->formatter->dump_string("DisplayName", name);
+  s->formatter->close_section();
+}
+
+void dump_access_control(struct req_state *s, const char *origin, const char *meth,
+                         const char *hdr, const char *exp_hdr, uint32_t max_age) {
+  if (origin && (origin[0] != '\0')) {
+    s->cio->print("Access-Control-Allow-Origin: %s\n", origin);
+    if (meth && (meth[0] != '\0'))
+      s->cio->print("Access-Control-Allow-Methods: %s\n", meth);
+    if (hdr && (hdr[0] != '\0'))
+      s->cio->print("Access-Control-Allow-Headers: %s\n", hdr);
+    if (exp_hdr && (exp_hdr[0] != '\0')) {
+      s->cio->print("Access-Control-Expose-Headers: %s\n", exp_hdr);
+    }
+    if (max_age != CORS_MAX_AGE_INVALID) {
+      s->cio->print("Access-Control-Max-Age: %d\n", max_age);
+    }
+  }
+}
+
+void dump_access_control(req_state *s, RGWOp *op)
+{
+  string origin;
+  string method;
+  string header;
+  string exp_header;
+  unsigned max_age = CORS_MAX_AGE_INVALID;
+
+  if (!op->generate_cors_headers(origin, method, header, exp_header, &max_age))
+    return;
+
+  dump_access_control(s, origin.c_str(), method.c_str(), header.c_str(), exp_header.c_str(), max_age);
+}
+
+void dump_start(struct req_state *s)
+{
+  if (!s->content_started) {
+    if (s->format == RGW_FORMAT_XML)
+      s->formatter->write_raw_data(XMLFormatter::XML_1_DTD);
+    s->content_started = true;
+  }
+}
+
+void end_header(struct req_state *s, RGWOp *op, const char *content_type)
+{
+  string ctype;
+
+  if (op) {
+    dump_access_control(s, op);
+  }
+
+  if (!content_type || s->err.is_err()) {
+    switch (s->format) {
+    case RGW_FORMAT_XML:
+      ctype = "application/xml";
+      break;
+    case RGW_FORMAT_JSON:
+      ctype = "application/json";
+      break;
+    default:
+      ctype = "text/plain";
+      break;
+    }
+    if (s->prot_flags & RGW_REST_SWIFT)
+      ctype.append("; charset=utf-8");
+    content_type = ctype.c_str();
+  }
+  if (s->err.is_err()) {
+    dump_start(s);
+    s->formatter->open_object_section("Error");
+    if (!s->err.s3_code.empty())
+      s->formatter->dump_string("Code", s->err.s3_code);
+    if (!s->err.message.empty())
+      s->formatter->dump_string("Message", s->err.message);
+    s->formatter->close_section();
+    dump_content_length(s, s->formatter->get_len());
+  }
+  int r = s->cio->print("Content-type: %s\r\n", content_type);
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+  }
+  r = s->cio->complete_header();
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->complete_header() returned err=" << r << dendl;
+  }
+
+  s->cio->set_account(true);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+void abort_early(struct req_state *s, RGWOp *op, int err_no)
+{
+  if (!s->formatter) {
+    s->formatter = new JSONFormatter;
+    s->format = RGW_FORMAT_JSON;
+  }
+  set_req_state_err(s, err_no);
+  dump_errno(s);
+  dump_bucket_from_state(s);
+  end_header(s, op);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+  perfcounter->inc(l_rgw_failed_req);
+}
+
+void dump_continue(struct req_state *s)
+{
+  s->cio->send_100_continue();
+}
+
+void dump_range(struct req_state *s, uint64_t ofs, uint64_t end, uint64_t total)
+{
+  char range_buf[128];
+
+  /* dumping range into temp buffer first, as libfcgi will fail to digest %lld */
+  snprintf(range_buf, sizeof(range_buf), "%lld-%lld/%lld", (long long)ofs, (long long)end, (long long)total);
+  int r = s->cio->print("Content-Range: bytes %s\n", range_buf);
+  if (r < 0) {
+    ldout(s->cct, 0) << "ERROR: s->cio->print() returned err=" << r << dendl;
+  }
+}
+
+int RGWGetObj_ObjStore::get_params()
+{
+  range_str = s->info.env->get("HTTP_RANGE");
+  if_mod = s->info.env->get("HTTP_IF_MODIFIED_SINCE");
+  if_unmod = s->info.env->get("HTTP_IF_UNMODIFIED_SINCE");
+  if_match = s->info.env->get("HTTP_IF_MATCH");
+  if_nomatch = s->info.env->get("HTTP_IF_NONE_MATCH");
+
+  return 0;
+}
+
+int RESTArgs::get_string(struct req_state *s, const string& name, const string& def_val, string *val, bool *existed)
+{
+  bool exists;
+  *val = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *val = def_val;
+    return 0;
+  }
+
+  return 0;
+}
+
+int RESTArgs::get_uint64(struct req_state *s, const string& name, uint64_t def_val, uint64_t *val, bool *existed)
+{
+  bool exists;
+  string sval = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *val = def_val;
+    return 0;
+  }
+
+  int r = stringtoull(sval, val);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RESTArgs::get_int64(struct req_state *s, const string& name, int64_t def_val, int64_t *val, bool *existed)
+{
+  bool exists;
+  string sval = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *val = def_val;
+    return 0;
+  }
+
+  int r = stringtoll(sval, val);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RESTArgs::get_uint32(struct req_state *s, const string& name, uint32_t def_val, uint32_t *val, bool *existed)
+{
+  bool exists;
+  string sval = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *val = def_val;
+    return 0;
+  }
+
+  int r = stringtoul(sval, val);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RESTArgs::get_int32(struct req_state *s, const string& name, int32_t def_val, int32_t *val, bool *existed)
+{
+  bool exists;
+  string sval = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *val = def_val;
+    return 0;
+  }
+
+  int r = stringtol(sval, val);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RESTArgs::get_time(struct req_state *s, const string& name, const utime_t& def_val, utime_t *val, bool *existed)
+{
+  bool exists;
+  string sval = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *val = def_val;
+    return 0;
+  }
+
+  uint64_t epoch, nsec;
+
+  int r = utime_t::parse_date(sval, &epoch, &nsec);
+  if (r < 0)
+    return r;
+
+  *val = utime_t(epoch, nsec);
+
+  return 0;
+}
+
+int RESTArgs::get_epoch(struct req_state *s, const string& name, uint64_t def_val, uint64_t *epoch, bool *existed)
+{
+  bool exists;
+  string date = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *epoch = def_val;
+    return 0;
+  }
+
+  int r = utime_t::parse_date(date, epoch, NULL);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RESTArgs::get_bool(struct req_state *s, const string& name, bool def_val, bool *val, bool *existed)
+{
+  bool exists;
+  string sval = s->info.args.get(name, &exists);
+
+  if (existed)
+    *existed = exists;
+
+  if (!exists) {
+    *val = def_val;
+    return 0;
+  }
+
+  const char *str = sval.c_str();
+
+  if (sval.empty() ||
+      strcasecmp(str, "true") == 0 ||
+      sval.compare("1") == 0) {
+    *val = true;
+    return 0;
+  }
+
+  if (strcasecmp(str, "false") != 0 &&
+      sval.compare("0") != 0) {
+    *val = def_val;
+    return -EINVAL;
+  }
+
+  *val = false;
+  return 0;
+}
+
+
+void RGWRESTFlusher::do_start(int ret)
+{
+  set_req_state_err(s, ret); /* no going back from here */
+  dump_errno(s);
+  dump_start(s);
+  end_header(s, op);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+void RGWRESTFlusher::do_flush()
+{
+  rgw_flush_formatter(s, s->formatter);
+}
+
+int RGWPutObj_ObjStore::verify_params()
+{
+  if (s->length) {
+    off_t len = atoll(s->length);
+    if (len > (off_t)RGW_MAX_PUT_SIZE) {
+      return -ERR_TOO_LARGE;
+    }
+  }
+
+  return 0;
+}
+
+int RGWPutObj_ObjStore::get_params()
+{
+  supplied_md5_b64 = s->info.env->get("HTTP_CONTENT_MD5");
+
+  return 0;
+}
+
+int RGWPutObj_ObjStore::get_data(bufferlist& bl)
+{
+  size_t cl;
+  uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
+  if (s->length) {
+    cl = atoll(s->length) - ofs;
+    if (cl > chunk_size)
+      cl = chunk_size;
+  } else {
+    cl = chunk_size;
+  }
+
+  int len = 0;
+  if (cl) {
+    bufferptr bp(cl);
+
+    int read_len; /* cio->read() expects int * */
+    int r = s->cio->read(bp.c_str(), cl, &read_len);
+    len = read_len;
+    if (r < 0)
+      return r;
+    bl.append(bp, 0, len);
+  }
+
+  if ((uint64_t)ofs + len > RGW_MAX_PUT_SIZE) {
+    return -ERR_TOO_LARGE;
+  }
+
+  if (!ofs)
+    supplied_md5_b64 = s->info.env->get("HTTP_CONTENT_MD5");
+
+  return len;
+}
+
+int RGWPostObj_ObjStore::verify_params()
+{
+  /*  check that we have enough memory to store the object
+  note that this test isn't exact and may fail unintentionally
+  for large requests is */
+  if (!s->length) {
+    return -ERR_LENGTH_REQUIRED;
+  }
+  off_t len = atoll(s->length);
+  if (len > (off_t)RGW_MAX_PUT_SIZE) {
+    return -ERR_TOO_LARGE;
+  }
+
+  return 0;
+}
+
+int RGWPutACLs_ObjStore::get_params()
+{
+  size_t cl = 0;
+  if (s->length)
+    cl = atoll(s->length);
+  if (cl) {
+    data = (char *)malloc(cl + 1);
+    if (!data) {
+       ret = -ENOMEM;
+       return ret;
+    }
+    int read_len;
+    int r = s->cio->read(data, cl, &read_len);
+    len = read_len;
+    if (r < 0)
+      return r;
+    data[len] = '\0';
+  } else {
+    len = 0;
+  }
+
+  return ret;
+}
+
+static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max_read)
+{
+#define READ_CHUNK 4096
+#define MAX_READ_CHUNK (128 * 1024)
+  int need_to_read = READ_CHUNK;
+  int total = need_to_read;
+  char *data = (char *)malloc(total + 1);
+  if (!data)
+    return -ENOMEM;
+
+  int read_len = 0, len = 0;
+  do {
+    int r = s->cio->read(data + len, need_to_read, &read_len);
+    if (r < 0) {
+      free(data);
+      return r;
+    }
+
+    len += read_len;
+
+    if (read_len == need_to_read) {
+      if (need_to_read < MAX_READ_CHUNK)
+	need_to_read *= 2;
+
+      if (total > max_read) {
+        free(data);
+        return -ERANGE;
+      }
+      total += need_to_read;
+
+      void *p = realloc(data, total + 1);
+      if (!p) {
+        free(data);
+        return -ENOMEM;
+      }
+      data = (char *)p;
+    } else {
+      break;
+    }
+
+  } while (true);
+  data[len] = '\0';
+
+  *pdata = data;
+  *plen = len;
+
+  return 0;
+}
+
+int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen, int max_len)
+{
+  size_t cl = 0;
+  int len = 0;
+  char *data = NULL;
+
+  if (s->length)
+    cl = atoll(s->length);
+  if (cl) {
+    if (cl > (size_t)max_len) {
+      return -ERANGE;
+    }
+    data = (char *)malloc(cl + 1);
+    if (!data) {
+       return -ENOMEM;
+    }
+    int ret = s->cio->read(data, cl, &len);
+    if (ret < 0) {
+      free(data);
+      return ret;
+    }
+    data[len] = '\0';
+  } else if (!s->length) {
+    const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
+    if (!encoding || strcmp(encoding, "chunked") != 0)
+      return -ERR_LENGTH_REQUIRED;
+
+    int ret = read_all_chunked_input(s, &data, &len, max_len);
+    if (ret < 0)
+      return ret;
+  }
+
+  *plen = len;
+  *pdata = data;
+
+  return 0;
+}
+
+
+int RGWCompleteMultipart_ObjStore::get_params()
+{
+  upload_id = s->info.args.get("uploadId");
+
+  if (upload_id.empty()) {
+    ret = -ENOTSUP;
+    return ret;
+  }
+
+#define COMPLETE_MULTIPART_MAX_LEN (1024 * 1024) /* api defines max 10,000 parts, this should be enough */
+  ret = rgw_rest_read_all_input(s, &data, &len, COMPLETE_MULTIPART_MAX_LEN);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWListMultipart_ObjStore::get_params()
+{
+  upload_id = s->info.args.get("uploadId");
+
+  if (upload_id.empty()) {
+    ret = -ENOTSUP;
+  }
+  string marker_str = s->info.args.get("part-number-marker");
+
+  if (!marker_str.empty()) {
+    string err;
+    marker = strict_strtol(marker_str.c_str(), 10, &err);
+    if (!err.empty()) {
+      ldout(s->cct, 20) << "bad marker: "  << marker << dendl;
+      ret = -EINVAL;
+      return ret;
+    }
+  }
+  
+  string str = s->info.args.get("max-parts");
+  if (!str.empty())
+    max_parts = atoi(str.c_str());
+
+  return ret;
+}
+
+int RGWListBucketMultiparts_ObjStore::get_params()
+{
+  delimiter = s->info.args.get("delimiter");
+  prefix = s->info.args.get("prefix");
+  string str = s->info.args.get("max-parts");
+  if (!str.empty())
+    max_uploads = atoi(str.c_str());
+  else
+    max_uploads = default_max;
+
+  string key_marker = s->info.args.get("key-marker");
+  string upload_id_marker = s->info.args.get("upload-id-marker");
+  if (!key_marker.empty())
+    marker.init(key_marker, upload_id_marker);
+
+  return 0;
+}
+
+int RGWDeleteMultiObj_ObjStore::get_params()
+{
+  bucket_name = s->bucket_name_str;
+
+  if (bucket_name.empty()) {
+    ret = -EINVAL;
+    return ret;
+  }
+
+  // everything is probably fine, set the bucket
+  bucket = s->bucket;
+
+  size_t cl = 0;
+
+  if (s->length)
+    cl = atoll(s->length);
+  if (cl) {
+    data = (char *)malloc(cl + 1);
+    if (!data) {
+      ret = -ENOMEM;
+      return ret;
+    }
+    int read_len;
+    ret = s->cio->read(data, cl, &read_len);
+    len = read_len;
+    if (ret < 0)
+      return ret;
+    data[len] = '\0';
+  } else {
+    return -EINVAL;
+  }
+
+  return ret;
+}
+
+
+void RGWRESTOp::send_response()
+{
+  if (!flusher.did_start()) {
+    set_req_state_err(s, http_ret);
+    dump_errno(s);
+    end_header(s, this);
+  }
+  flusher.flush();
+}
+
+int RGWRESTOp::verify_permission()
+{
+  return check_caps(s->user.caps);
+}
+
+
+int RGWHandler_ObjStore::allocate_formatter(struct req_state *s, int default_type, bool configurable)
+{
+  s->format = default_type;
+  if (configurable) {
+    string format_str = s->info.args.get("format");
+    if (format_str.compare("xml") == 0) {
+      s->format = RGW_FORMAT_XML;
+    } else if (format_str.compare("json") == 0) {
+      s->format = RGW_FORMAT_JSON;
+    }
+  }
+
+  switch (s->format) {
+    case RGW_FORMAT_PLAIN:
+      s->formatter = new RGWFormatter_Plain;
+      break;
+    case RGW_FORMAT_XML:
+      s->formatter = new XMLFormatter(false);
+      break;
+    case RGW_FORMAT_JSON:
+      s->formatter = new JSONFormatter(false);
+      break;
+    default:
+      return -EINVAL;
+
+  };
+  s->formatter->reset();
+
+  return 0;
+}
+
+// This function enforces Amazon's spec for bucket names.
+// (The requirements, not the recommendations.)
+int RGWHandler_ObjStore::validate_bucket_name(const string& bucket)
+{
+  int len = bucket.size();
+  if (len < 3) {
+    if (len == 0) {
+      // This request doesn't specify a bucket at all
+      return 0;
+    }
+    // Name too short
+    return -ERR_INVALID_BUCKET_NAME;
+  }
+  else if (len > 255) {
+    // Name too long
+    return -ERR_INVALID_BUCKET_NAME;
+  }
+
+  return 0;
+}
+
+// "The name for a key is a sequence of Unicode characters whose UTF-8 encoding
+// is at most 1024 bytes long."
+// However, we can still have control characters and other nasties in there.
+// Just as long as they're utf-8 nasties.
+int RGWHandler_ObjStore::validate_object_name(const string& object)
+{
+  int len = object.size();
+  if (len > 1024) {
+    // Name too long
+    return -ERR_INVALID_OBJECT_NAME;
+  }
+
+  if (check_utf8(object.c_str(), len)) {
+    // Object names must be valid UTF-8.
+    return -ERR_INVALID_OBJECT_NAME;
+  }
+  return 0;
+}
+
+static http_op op_from_method(const char *method)
+{
+  if (!method)
+    return OP_UNKNOWN;
+  if (strcmp(method, "GET") == 0)
+    return OP_GET;
+  if (strcmp(method, "PUT") == 0)
+    return OP_PUT;
+  if (strcmp(method, "DELETE") == 0)
+    return OP_DELETE;
+  if (strcmp(method, "HEAD") == 0)
+    return OP_HEAD;
+  if (strcmp(method, "POST") == 0)
+    return OP_POST;
+  if (strcmp(method, "COPY") == 0)
+    return OP_COPY;
+  if (strcmp(method, "OPTIONS") == 0)
+    return OP_OPTIONS;
+
+  return OP_UNKNOWN;
+}
+
+int RGWHandler_ObjStore::read_permissions(RGWOp *op_obj)
+{
+  bool only_bucket;
+
+  switch (s->op) {
+  case OP_HEAD:
+  case OP_GET:
+    only_bucket = false;
+    break;
+  case OP_PUT:
+  case OP_POST:
+  case OP_COPY:
+    /* is it a 'multi-object delete' request? */
+    if (s->info.request_params == "delete") {
+      only_bucket = true;
+      break;
+    }
+    if (is_obj_update_op()) {
+      only_bucket = false;
+      break;
+    }
+    /* is it a 'create bucket' request? */
+    if ((s->op == OP_PUT) && s->object_str.size() == 0)
+      return 0;
+    only_bucket = true;
+    break;
+  case OP_DELETE:
+    only_bucket = true;
+    break;
+  case OP_OPTIONS:
+    only_bucket = true;
+    break;
+  default:
+    return -EINVAL;
+  }
+
+  return do_read_permissions(op_obj, only_bucket);
+}
+
+void RGWRESTMgr::register_resource(string resource, RGWRESTMgr *mgr)
+{
+  string r = "/";
+  r.append(resource);
+
+  /* do we have a resource manager registered for this entry point? */
+  map::iterator iter = resource_mgrs.find(r);
+  if (iter != resource_mgrs.end()) {
+    delete iter->second;
+  }
+  resource_mgrs[r] = mgr;
+  resources_by_size.insert(pair(r.size(), r));
+
+  /* now build default resource managers for the path (instead of nested entry points)
+   * e.g., if the entry point is /auth/v1.0/ then we'd want to create a default
+   * manager for /auth/
+   */
+
+  size_t pos = r.find('/', 1);
+
+  while (pos != r.size() - 1 && pos != string::npos) {
+    string s = r.substr(0, pos);
+
+    iter = resource_mgrs.find(s);
+    if (iter == resource_mgrs.end()) { /* only register it if one does not exist */
+      resource_mgrs[s] = new RGWRESTMgr; /* a default do-nothing manager */
+      resources_by_size.insert(pair(s.size(), s));
+    }
+
+    pos = r.find('/', pos + 1);
+  }
+}
+
+void RGWRESTMgr::register_default_mgr(RGWRESTMgr *mgr)
+{
+  delete default_mgr;
+  default_mgr = mgr;
+}
+
+RGWRESTMgr *RGWRESTMgr::get_resource_mgr(struct req_state *s, const string& uri, string *out_uri)
+{
+  *out_uri = uri;
+
+  if (resources_by_size.empty())
+    return this;
+
+  multimap::reverse_iterator iter;
+
+  for (iter = resources_by_size.rbegin(); iter != resources_by_size.rend(); ++iter) {
+    string& resource = iter->second;
+    if (uri.compare(0, iter->first, resource) == 0 &&
+	(uri.size() == iter->first ||
+	 uri[iter->first] == '/')) {
+      string suffix = uri.substr(iter->first);
+      return resource_mgrs[resource]->get_resource_mgr(s, suffix, out_uri);
+    }
+  }
+
+  if (default_mgr)
+    return default_mgr;
+
+  return this;
+}
+
+RGWRESTMgr::~RGWRESTMgr()
+{
+  map::iterator iter;
+  for (iter = resource_mgrs.begin(); iter != resource_mgrs.end(); ++iter) {
+    delete iter->second;
+  }
+  delete default_mgr;
+}
+
+int RGWREST::preprocess(struct req_state *s, RGWClientIO *cio)
+{
+  req_info& info = s->info;
+
+  s->cio = cio;
+  if (g_conf->rgw_dns_name.length() && info.host) {
+    string h(s->info.host);
+
+    ldout(s->cct, 10) << "host=" << s->info.host << " rgw_dns_name=" << g_conf->rgw_dns_name << dendl;
+    int pos = h.find(g_conf->rgw_dns_name);
+
+    if (g_conf->rgw_resolve_cname && pos < 0) {
+      string cname;
+      bool found;
+      int r = rgw_resolver->resolve_cname(h, cname, &found);
+      if (r < 0) {
+	dout(0) << "WARNING: rgw_resolver->resolve_cname() returned r=" << r << dendl;
+      }
+      if (found) {
+        dout(0) << "resolved host cname " << h << " -> " << cname << dendl;
+	h = cname;
+        pos = h.find(g_conf->rgw_dns_name);
+      }
+    }
+
+    if (pos > 0 && h[pos - 1] == '.') {
+      string encoded_bucket = "/";
+      encoded_bucket.append(h.substr(0, pos-1));
+      if (s->info.request_uri[0] != '/')
+	encoded_bucket.append("/'");
+      encoded_bucket.append(s->info.request_uri);
+      s->info.request_uri = encoded_bucket;
+    }
+  }
+
+  url_decode(s->info.request_uri, s->decoded_uri);
+  s->length = info.env->get("CONTENT_LENGTH");
+  if (s->length) {
+    if (*s->length == '\0')
+      s->content_length = 0;
+    else
+      s->content_length = atoll(s->length);
+  }
+
+  map::iterator giter;
+  for (giter = generic_attrs_map.begin(); giter != generic_attrs_map.end(); ++giter) {
+    const char *env = info.env->get(giter->first.c_str());
+    if (env) {
+      s->generic_attrs[giter->second] = env;
+    }
+  }
+
+  s->http_auth = info.env->get("HTTP_AUTHORIZATION");
+
+  if (g_conf->rgw_print_continue) {
+    const char *expect = info.env->get("HTTP_EXPECT");
+    s->expect_cont = (expect && !strcasecmp(expect, "100-continue"));
+  }
+  s->op = op_from_method(info.method);
+
+  info.init_meta_info(&s->has_bad_meta);
+
+  return 0;
+}
+
+RGWHandler *RGWREST::get_handler(RGWRados *store, struct req_state *s, RGWClientIO *cio,
+				 RGWRESTMgr **pmgr, int *init_error)
+{
+  RGWHandler *handler;
+
+  *init_error = preprocess(s, cio);
+  if (*init_error < 0)
+    return NULL;
+
+  RGWRESTMgr *m = mgr.get_resource_mgr(s, s->decoded_uri, &s->relative_uri);
+  if (!m) {
+    *init_error = -ERR_METHOD_NOT_ALLOWED;
+    return NULL;
+  }
+
+  if (pmgr)
+    *pmgr = m;
+
+  handler = m->get_handler(s);
+  if (!handler) {
+    *init_error = -ERR_METHOD_NOT_ALLOWED;
+    return NULL;
+  }
+  *init_error = handler->init(store, s, cio);
+  if (*init_error < 0) {
+    m->put_handler(handler);
+    return NULL;
+  }
+
+  return handler;
+}
+
diff --git a/ceph/src/rgw/rgw_rest.h b/ceph/src/rgw/rgw_rest.h
new file mode 100644
index 00000000..38ffd8c3
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest.h
@@ -0,0 +1,382 @@
+#ifndef CEPH_RGW_REST_H
+#define CEPH_RGW_REST_H
+#define TIME_BUF_SIZE 128
+
+#include "common/ceph_json.h"
+#include "include/assert.h" /* needed because of common/ceph_json.h */
+#include "rgw_op.h"
+#include "rgw_formats.h"
+
+
+extern std::map rgw_to_http_attrs;
+
+extern void rgw_rest_init(CephContext *cct);
+
+extern void rgw_flush_formatter_and_reset(struct req_state *s,
+					 ceph::Formatter *formatter);
+
+extern void rgw_flush_formatter(struct req_state *s,
+                                         ceph::Formatter *formatter);
+
+extern int rgw_rest_read_all_input(struct req_state *s, char **data, int *plen, int max_len);
+
+template 
+int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out, int max_len, bool *empty)
+{
+  int rv, data_len;
+  char *data;
+
+  if (empty)
+    *empty = false;
+
+  if ((rv = rgw_rest_read_all_input(s, &data, &data_len, max_len)) < 0) {
+    return rv;
+  }
+
+  if (!data_len) {
+    if (empty) {
+      *empty = true;
+    }
+
+    return -EINVAL;
+  }
+
+  JSONParser parser;
+
+  if (!parser.parse(data, data_len)) {
+    free(data);
+    return -EINVAL;
+  }
+
+  decode_json_obj(out, &parser);
+
+  free(data);
+  return 0;
+}
+
+
+class RESTArgs {
+public:
+  static int get_string(struct req_state *s, const string& name, const string& def_val, string *val, bool *existed = NULL);
+  static int get_uint64(struct req_state *s, const string& name, uint64_t def_val, uint64_t *val, bool *existed = NULL);
+  static int get_int64(struct req_state *s, const string& name, int64_t def_val, int64_t *val, bool *existed = NULL);
+  static int get_uint32(struct req_state *s, const string& name, uint32_t def_val, uint32_t *val, bool *existed = NULL);
+  static int get_int32(struct req_state *s, const string& name, int32_t def_val, int32_t *val, bool *existed = NULL);
+  static int get_time(struct req_state *s, const string& name, const utime_t& def_val, utime_t *val, bool *existed = NULL);
+  static int get_epoch(struct req_state *s, const string& name, uint64_t def_val, uint64_t *epoch, bool *existed = NULL);
+  static int get_bool(struct req_state *s, const string& name, bool def_val, bool *val, bool *existed = NULL);
+};
+
+
+class RGWRESTFlusher : public RGWFormatterFlusher {
+  struct req_state *s;
+  RGWOp *op;
+protected:
+  virtual void do_flush();
+  virtual void do_start(int ret);
+public:
+  RGWRESTFlusher(struct req_state *_s, RGWOp *_op) : RGWFormatterFlusher(_s->formatter), s(_s), op(_op) {}
+  RGWRESTFlusher() : RGWFormatterFlusher(NULL), s(NULL), op(NULL) {}
+
+  void init(struct req_state *_s, RGWOp *_op) {
+    s = _s;
+    op = _op;
+    set_formatter(s->formatter);
+  }
+};
+
+class RGWClientIO;
+
+class RGWGetObj_ObjStore : public RGWGetObj
+{
+protected:
+  bool sent_header;
+public:
+  RGWGetObj_ObjStore() : sent_header(false) {}
+
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
+    RGWGetObj::init(store, s, h);
+    sent_header = false;
+  }
+
+  int get_params();
+};
+
+class RGWListBuckets_ObjStore : public RGWListBuckets {
+public:
+  RGWListBuckets_ObjStore() {}
+  ~RGWListBuckets_ObjStore() {}
+};
+
+class RGWListBucket_ObjStore : public RGWListBucket {
+public:
+  RGWListBucket_ObjStore() {}
+  ~RGWListBucket_ObjStore() {}
+};
+
+class RGWStatAccount_ObjStore : public RGWStatAccount {
+public:
+  RGWStatAccount_ObjStore() {}
+  ~RGWStatAccount_ObjStore() {}
+};
+
+class RGWStatBucket_ObjStore : public RGWStatBucket {
+public:
+  RGWStatBucket_ObjStore() {}
+  ~RGWStatBucket_ObjStore() {}
+};
+
+class RGWCreateBucket_ObjStore : public RGWCreateBucket {
+public:
+  RGWCreateBucket_ObjStore() {}
+  ~RGWCreateBucket_ObjStore() {}
+};
+
+class RGWDeleteBucket_ObjStore : public RGWDeleteBucket {
+public:
+  RGWDeleteBucket_ObjStore() {}
+  ~RGWDeleteBucket_ObjStore() {}
+};
+
+class RGWPutObj_ObjStore : public RGWPutObj
+{
+public:
+  RGWPutObj_ObjStore() {}
+  ~RGWPutObj_ObjStore() {}
+
+  virtual int verify_params();
+  virtual int get_params();
+  int get_data(bufferlist& bl);
+};
+
+class RGWPostObj_ObjStore : public RGWPostObj
+{
+public:
+  RGWPostObj_ObjStore() {}
+  ~RGWPostObj_ObjStore() {}
+
+  virtual int verify_params();
+};
+
+class RGWPutMetadata_ObjStore : public RGWPutMetadata
+{
+public:
+  RGWPutMetadata_ObjStore() {}
+  ~RGWPutMetadata_ObjStore() {}
+};
+
+class RGWSetTempUrl_ObjStore : public RGWSetTempUrl {
+public:
+  RGWSetTempUrl_ObjStore() {}
+  ~RGWSetTempUrl_ObjStore() {}
+};
+
+class RGWDeleteObj_ObjStore : public RGWDeleteObj {
+public:
+  RGWDeleteObj_ObjStore() {}
+  ~RGWDeleteObj_ObjStore() {}
+};
+
+class RGWCopyObj_ObjStore : public RGWCopyObj {
+public:
+  RGWCopyObj_ObjStore() {}
+  ~RGWCopyObj_ObjStore() {}
+};
+
+class RGWGetACLs_ObjStore : public RGWGetACLs {
+public:
+  RGWGetACLs_ObjStore() {}
+  ~RGWGetACLs_ObjStore() {}
+};
+
+class RGWPutACLs_ObjStore : public RGWPutACLs {
+public:
+  RGWPutACLs_ObjStore() {}
+  ~RGWPutACLs_ObjStore() {}
+
+  int get_params();
+};
+
+class RGWGetCORS_ObjStore : public RGWGetCORS {
+public:
+  RGWGetCORS_ObjStore() {}
+  ~RGWGetCORS_ObjStore() {}
+};
+
+class RGWPutCORS_ObjStore : public RGWPutCORS {
+public:
+  RGWPutCORS_ObjStore() {}
+  ~RGWPutCORS_ObjStore() {}
+};
+
+class RGWDeleteCORS_ObjStore : public RGWDeleteCORS {
+public:
+  RGWDeleteCORS_ObjStore() {}
+  ~RGWDeleteCORS_ObjStore() {}
+};
+
+class RGWOptionsCORS_ObjStore : public RGWOptionsCORS {
+public:
+  RGWOptionsCORS_ObjStore() {}
+  ~RGWOptionsCORS_ObjStore() {}
+};
+
+class RGWInitMultipart_ObjStore : public RGWInitMultipart {
+public:
+  RGWInitMultipart_ObjStore() {}
+  ~RGWInitMultipart_ObjStore() {}
+};
+
+class RGWCompleteMultipart_ObjStore : public RGWCompleteMultipart {
+public:
+  RGWCompleteMultipart_ObjStore() {}
+  ~RGWCompleteMultipart_ObjStore() {}
+
+  int get_params();
+};
+
+class RGWAbortMultipart_ObjStore : public RGWAbortMultipart {
+public:
+  RGWAbortMultipart_ObjStore() {}
+  ~RGWAbortMultipart_ObjStore() {}
+};
+
+class RGWListMultipart_ObjStore : public RGWListMultipart {
+public:
+  RGWListMultipart_ObjStore() {}
+  ~RGWListMultipart_ObjStore() {}
+
+  int get_params();
+};
+
+class RGWListBucketMultiparts_ObjStore : public RGWListBucketMultiparts {
+public:
+  RGWListBucketMultiparts_ObjStore() {}
+  ~RGWListBucketMultiparts_ObjStore() {}
+
+  int get_params();
+};
+
+class RGWDeleteMultiObj_ObjStore : public RGWDeleteMultiObj {
+public:
+  RGWDeleteMultiObj_ObjStore() {}
+  ~RGWDeleteMultiObj_ObjStore() {}
+
+  int get_params();
+};
+
+class RGWRESTOp : public RGWOp {
+protected:
+  int http_ret;
+  RGWRESTFlusher flusher;
+public:
+  RGWRESTOp() : http_ret(0) {}
+  virtual void init(RGWRados *store, struct req_state *s, RGWHandler *dialect_handler) {
+    RGWOp::init(store, s, dialect_handler);
+    flusher.init(s, this);
+  }
+  virtual void send_response();
+  virtual int check_caps(RGWUserCaps& caps) { return -EPERM; } /* should to be implemented! */
+  virtual int verify_permission();
+};
+
+class RGWHandler_ObjStore : public RGWHandler {
+protected:
+  virtual bool is_obj_update_op() { return false; }
+  virtual RGWOp *op_get() { return NULL; }
+  virtual RGWOp *op_put() { return NULL; }
+  virtual RGWOp *op_delete() { return NULL; }
+  virtual RGWOp *op_head() { return NULL; }
+  virtual RGWOp *op_post() { return NULL; }
+  virtual RGWOp *op_copy() { return NULL; }
+  virtual RGWOp *op_options() { return NULL; }
+
+  virtual int validate_bucket_name(const string& bucket);
+  virtual int validate_object_name(const string& object);
+
+  static int allocate_formatter(struct req_state *s, int default_formatter, bool configurable);
+public:
+  RGWHandler_ObjStore() {}
+  virtual ~RGWHandler_ObjStore() {}
+  int read_permissions(RGWOp *op);
+
+  virtual int authorize() = 0;
+};
+
+class RGWHandler_ObjStore_SWIFT;
+class RGWHandler_SWIFT_Auth;
+class RGWHandler_ObjStore_S3;
+
+class RGWRESTMgr {
+  bool should_log;
+protected:
+  map resource_mgrs;
+  multimap resources_by_size;
+  RGWRESTMgr *default_mgr;
+
+public:
+  RGWRESTMgr() : should_log(false), default_mgr(NULL) {}
+  virtual ~RGWRESTMgr();
+
+  void register_resource(string resource, RGWRESTMgr *mgr);
+  void register_default_mgr(RGWRESTMgr *mgr);
+
+  virtual RGWRESTMgr *get_resource_mgr(struct req_state *s, const string& uri, string *out_uri);
+  virtual RGWHandler *get_handler(struct req_state *s) { return NULL; }
+  virtual void put_handler(RGWHandler *handler) { delete handler; }
+
+  void set_logging(bool _should_log) { should_log = _should_log; }
+  bool get_logging() { return should_log; }
+};
+
+class RGWREST {
+  RGWRESTMgr mgr;
+
+  static int preprocess(struct req_state *s, RGWClientIO *cio);
+public:
+  RGWREST() {}
+  RGWHandler *get_handler(RGWRados *store, struct req_state *s, RGWClientIO *cio,
+			  RGWRESTMgr **pmgr, int *init_error);
+  void put_handler(RGWHandler *handler) {
+    mgr.put_handler(handler);
+  }
+
+  void register_resource(string resource, RGWRESTMgr *m, bool register_empty = false) {
+    if (!register_empty && resource.empty())
+      return;
+
+    mgr.register_resource(resource, m);
+  }
+  void register_default_mgr(RGWRESTMgr *m) {
+    mgr.register_default_mgr(m);
+  }
+};
+
+extern void set_req_state_err(struct req_state *s, int err_no);
+extern void dump_errno(struct req_state *s);
+extern void dump_errno(struct req_state *s, int ret);
+extern void end_header(struct req_state *s, RGWOp *op = NULL, const char *content_type = NULL);
+extern void dump_start(struct req_state *s);
+extern void list_all_buckets_start(struct req_state *s);
+extern void dump_owner(struct req_state *s, string& id, string& name, const char *section = NULL);
+extern void dump_content_length(struct req_state *s, uint64_t len);
+extern void dump_etag(struct req_state *s, const char *etag);
+extern void dump_epoch_header(struct req_state *s, const char *name, time_t t);
+extern void dump_last_modified(struct req_state *s, time_t t);
+extern void abort_early(struct req_state *s, RGWOp *op, int err);
+extern void dump_range(struct req_state *s, uint64_t ofs, uint64_t end, uint64_t total_size);
+extern void dump_continue(struct req_state *s);
+extern void list_all_buckets_end(struct req_state *s);
+extern void dump_time(struct req_state *s, const char *name, time_t *t);
+extern void dump_bucket_from_state(struct req_state *s);
+extern void dump_object_from_state(struct req_state *s);
+extern void dump_uri_from_state(struct req_state *s);
+extern void dump_redirect(struct req_state *s, const string& redirect);
+extern void dump_pair(struct req_state *s, const char *key, const char *value);
+extern bool is_valid_url(const char *url);
+extern void dump_access_control(struct req_state *s, const char *origin, const char *meth,
+                         const char *hdr, const char *exp_hdr, uint32_t max_age);
+extern void dump_access_control(req_state *s, RGWOp *op);
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_admin.h b/ceph/src/rgw/rgw_rest_admin.h
new file mode 100644
index 00000000..25470f9f
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_admin.h
@@ -0,0 +1,12 @@
+#ifndef CEPH_RGW_REST_ADMIN_H
+#define CEPH_RGW_REST_ADMIN_H
+
+
+class RGWRESTMgr_Admin : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Admin() {}
+  virtual ~RGWRESTMgr_Admin() {}
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_bucket.cc b/ceph/src/rgw/rgw_rest_bucket.cc
new file mode 100644
index 00000000..e7068b43
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_bucket.cc
@@ -0,0 +1,257 @@
+#include "rgw_op.h"
+#include "rgw_bucket.h"
+#include "rgw_rest_bucket.h"
+
+#include "include/str_list.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+class RGWOp_Bucket_Info : public RGWRESTOp {
+
+public:
+  RGWOp_Bucket_Info() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("buckets", RGW_CAP_READ);
+  }
+
+  void execute();
+
+  virtual const string name() { return "get_bucket_info"; }
+};
+
+void RGWOp_Bucket_Info::execute()
+{
+  RGWBucketAdminOpState op_state;
+
+  bool fetch_stats;
+
+  std::string uid;
+  std::string bucket;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "bucket", bucket, &bucket);
+  RESTArgs::get_bool(s, "stats", false, &fetch_stats);
+
+
+  op_state.set_user_id(uid);
+  op_state.set_bucket_name(bucket);
+  op_state.set_fetch_stats(fetch_stats);
+
+  http_ret = RGWBucketAdminOp::info(store, op_state, flusher);
+}
+
+class RGWOp_Get_Policy : public RGWRESTOp {
+
+public:
+  RGWOp_Get_Policy() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("buckets", RGW_CAP_READ);
+  }
+
+  void execute();
+
+  virtual const string name() { return "get_policy"; }
+};
+
+void RGWOp_Get_Policy::execute()
+{
+  RGWBucketAdminOpState op_state;
+
+  std::string bucket;
+  std::string object;
+
+  RESTArgs::get_string(s, "bucket", bucket, &bucket);
+  RESTArgs::get_string(s, "object", object, &object);
+
+  op_state.set_bucket_name(bucket);
+  op_state.set_object(object);
+
+  http_ret = RGWBucketAdminOp::get_policy(store, op_state, flusher);
+}
+
+class RGWOp_Check_Bucket_Index : public RGWRESTOp {
+
+public:
+  RGWOp_Check_Bucket_Index() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("buckets", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "check_bucket_index"; }
+};
+
+void RGWOp_Check_Bucket_Index::execute()
+{
+  std::string bucket;
+
+  bool fix_index;
+  bool check_objects;
+
+  RGWBucketAdminOpState op_state;
+
+  RESTArgs::get_string(s, "bucket", bucket, &bucket);
+  RESTArgs::get_bool(s, "fix", false, &fix_index);
+  RESTArgs::get_bool(s, "check-objects", false, &check_objects);
+
+  op_state.set_bucket_name(bucket);
+  op_state.set_fix_index(fix_index);
+  op_state.set_check_objects(check_objects);
+
+  http_ret = RGWBucketAdminOp::check_index(store, op_state, flusher);
+}
+
+class RGWOp_Bucket_Link : public RGWRESTOp {
+
+public:
+  RGWOp_Bucket_Link() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("buckets", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "link_bucket"; }
+};
+
+void RGWOp_Bucket_Link::execute()
+{
+  std::string uid;
+  std::string bucket;
+
+  RGWBucketAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "bucket", bucket, &bucket);
+
+  op_state.set_user_id(uid);
+  op_state.set_bucket_name(bucket);
+
+  http_ret = RGWBucketAdminOp::link(store, op_state);
+}
+
+class RGWOp_Bucket_Unlink : public RGWRESTOp {
+
+public:
+  RGWOp_Bucket_Unlink() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("buckets", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "unlink_bucket"; }
+};
+
+void RGWOp_Bucket_Unlink::execute()
+{
+  std::string uid;
+  std::string bucket;
+
+  RGWBucketAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "bucket", bucket, &bucket);
+
+  op_state.set_user_id(uid);
+  op_state.set_bucket_name(bucket);
+
+  http_ret = RGWBucketAdminOp::unlink(store, op_state);
+}
+
+class RGWOp_Bucket_Remove : public RGWRESTOp {
+
+public:
+  RGWOp_Bucket_Remove() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("buckets", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "remove_bucket"; }
+};
+
+void RGWOp_Bucket_Remove::execute()
+{
+  std::string bucket;
+  bool delete_children;
+
+  RGWBucketAdminOpState op_state;
+
+  RESTArgs::get_string(s, "bucket", bucket, &bucket);
+  RESTArgs::get_bool(s, "purge-objects", false, &delete_children);
+
+  op_state.set_bucket_name(bucket);
+  op_state.set_delete_children(delete_children);
+
+  http_ret = RGWBucketAdminOp::remove_bucket(store, op_state);
+}
+
+class RGWOp_Object_Remove: public RGWRESTOp {
+
+public:
+  RGWOp_Object_Remove() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("buckets", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "remove_object"; }
+};
+
+void RGWOp_Object_Remove::execute()
+{
+  std::string bucket;
+  std::string object;
+
+  RGWBucketAdminOpState op_state;
+
+  RESTArgs::get_string(s, "bucket", bucket, &bucket);
+  RESTArgs::get_string(s, "object", object, &object);
+
+  op_state.set_bucket_name(bucket);
+  op_state.set_object(object);
+
+  http_ret = RGWBucketAdminOp::remove_object(store, op_state);
+}
+
+RGWOp *RGWHandler_Bucket::op_get()
+{
+
+  if (s->info.args.sub_resource_exists("policy"))
+    return new RGWOp_Get_Policy;
+
+  if (s->info.args.sub_resource_exists("index"))
+    return new RGWOp_Check_Bucket_Index;
+
+  return new RGWOp_Bucket_Info;
+};
+
+RGWOp *RGWHandler_Bucket::op_put()
+{
+  return new RGWOp_Bucket_Link;
+};
+
+RGWOp *RGWHandler_Bucket::op_post()
+{
+  return new RGWOp_Bucket_Unlink;
+};
+
+RGWOp *RGWHandler_Bucket::op_delete()
+{
+  if (s->info.args.sub_resource_exists("object"))
+    return new RGWOp_Object_Remove;
+
+  return new RGWOp_Bucket_Remove;
+};
+
diff --git a/ceph/src/rgw/rgw_rest_bucket.h b/ceph/src/rgw/rgw_rest_bucket.h
new file mode 100644
index 00000000..bb706436
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_bucket.h
@@ -0,0 +1,33 @@
+#ifndef CEPH_RGW_REST_BUCKET_H
+#define CEPH_RGW_REST_BUCKET_H
+
+#include "rgw_rest.h"
+#include "rgw_rest_s3.h"
+
+
+class RGWHandler_Bucket : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_put();
+  RGWOp *op_post();
+  RGWOp *op_delete();
+public:
+  RGWHandler_Bucket() {}
+  virtual ~RGWHandler_Bucket() {}
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+};
+
+class RGWRESTMgr_Bucket : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Bucket() {}
+  virtual ~RGWRESTMgr_Bucket() {}
+
+  RGWHandler *get_handler(struct req_state *s) {
+    return new RGWHandler_Bucket;
+  }
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_client.cc b/ceph/src/rgw/rgw_rest_client.cc
new file mode 100644
index 00000000..256db77a
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_client.cc
@@ -0,0 +1,671 @@
+#include "rgw_common.h"
+#include "rgw_rest_client.h"
+#include "rgw_auth_s3.h"
+#include "rgw_http_errors.h"
+#include "rgw_rados.h"
+
+#include "common/ceph_crypto_cms.h"
+#include "common/armor.h"
+#include "common/strtol.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+int RGWRESTSimpleRequest::receive_header(void *ptr, size_t len)
+{
+  char line[len + 1];
+
+  char *s = (char *)ptr, *end = (char *)ptr + len;
+  char *p = line;
+  ldout(cct, 10) << "receive_http_header" << dendl;
+
+  while (s != end) {
+    if (*s == '\r') {
+      s++;
+      continue;
+    }
+    if (*s == '\n') {
+      *p = '\0';
+      ldout(cct, 10) << "received header:" << line << dendl;
+      // TODO: fill whatever data required here
+      char *l = line;
+      char *tok = strsep(&l, " \t:");
+      if (tok && l) {
+        while (*l == ' ')
+          l++;
+ 
+        if (strcmp(tok, "HTTP") == 0 || strncmp(tok, "HTTP/", 5) == 0) {
+          http_status = atoi(l);
+          if (http_status == 100) /* 100-continue response */
+            continue;
+          status = rgw_http_error_to_errno(http_status);
+        } else {
+          /* convert header field name to upper case  */
+          char *src = tok;
+          char buf[len + 1];
+          size_t i;
+          for (i = 0; i < len && *src; ++i, ++src) {
+            switch (*src) {
+              case '-':
+                buf[i] = '_';
+                break;
+              default:
+                buf[i] = toupper(*src);
+            }
+          }
+          buf[i] = '\0';
+          out_headers[buf] = l;
+          int r = handle_header(buf, l);
+          if (r < 0)
+            return r;
+        }
+      }
+    }
+    if (s != end)
+      *p++ = *s++;
+  }
+  return 0;
+}
+
+static void get_new_date_str(CephContext *cct, string& date_str)
+{
+  utime_t tm = ceph_clock_now(cct);
+  stringstream s;
+  tm.asctime(s);
+  date_str = s.str();
+}
+
+int RGWRESTSimpleRequest::execute(RGWAccessKey& key, const char *method, const char *resource)
+{
+  string new_url = url;
+  string new_resource = resource;
+
+  if (new_url[new_url.size() - 1] == '/' && resource[0] == '/') {
+    new_url = new_url.substr(0, new_url.size() - 1);
+  } else if (resource[0] != '/') {
+    new_resource = "/";
+    new_resource.append(resource);
+  }
+  new_url.append(new_resource);
+
+  string date_str;
+  get_new_date_str(cct, date_str);
+  headers.push_back(make_pair("HTTP_DATE", date_str));
+
+  string canonical_header;
+  map meta_map;
+  map sub_resources;
+  rgw_create_s3_canonical_header(method, NULL, NULL, date_str.c_str(),
+                            meta_map, new_url.c_str(), sub_resources,
+                            canonical_header);
+
+  string digest;
+  int ret = rgw_get_s3_header_digest(canonical_header, key.key, digest);
+  if (ret < 0) {
+    return ret;
+  }
+
+  string auth_hdr = "AWS " + key.id + ":" + digest;
+
+  ldout(cct, 15) << "generated auth header: " << auth_hdr << dendl;
+
+  headers.push_back(make_pair("AUTHORIZATION", auth_hdr));
+  int r = process(method, new_url.c_str());
+  if (r < 0)
+    return r;
+
+  return status;
+}
+
+int RGWRESTSimpleRequest::send_data(void *ptr, size_t len)
+{
+  if (!send_iter)
+    return 0;
+
+  if (len > send_iter->get_remaining())
+    len = send_iter->get_remaining();
+
+  send_iter->copy(len, (char *)ptr);
+
+  return len;
+}
+
+int RGWRESTSimpleRequest::receive_data(void *ptr, size_t len)
+{
+  if (response.length() > max_response)
+    return 0; /* don't read extra data */
+
+  bufferptr p((char *)ptr, len);
+
+  response.append(p);
+
+  return 0;
+
+}
+
+void RGWRESTSimpleRequest::append_param(string& dest, const string& name, const string& val)
+{
+  if (dest.empty()) {
+    dest.append("?");
+  } else {
+    dest.append("&");
+  }
+  dest.append(name);
+
+  if (!val.empty()) {
+    dest.append("=");
+    dest.append(val);
+  }
+}
+
+void RGWRESTSimpleRequest::get_params_str(map& extra_args, string& dest)
+{
+  map::iterator miter;
+  for (miter = extra_args.begin(); miter != extra_args.end(); ++miter) {
+    append_param(dest, miter->first, miter->second);
+  }
+  list >::iterator iter;
+  for (iter = params.begin(); iter != params.end(); ++iter) {
+    append_param(dest, iter->first, iter->second);
+  }
+}
+
+int RGWRESTSimpleRequest::sign_request(RGWAccessKey& key, RGWEnv& env, req_info& info)
+{
+  map& m = env.get_map();
+
+  map::iterator i;
+  for (i = m.begin(); i != m.end(); ++i) {
+    ldout(cct, 0) << "> " << i->first << " -> " << i->second << dendl;
+  }
+
+  string canonical_header;
+  if (!rgw_create_s3_canonical_header(info, NULL, canonical_header, false)) {
+    ldout(cct, 0) << "failed to create canonical s3 header" << dendl;
+    return -EINVAL;
+  }
+
+  ldout(cct, 10) << "generated canonical header: " << canonical_header << dendl;
+
+  string digest;
+  int ret = rgw_get_s3_header_digest(canonical_header, key.key, digest);
+  if (ret < 0) {
+    return ret;
+  }
+
+  string auth_hdr = "AWS " + key.id + ":" + digest;
+  ldout(cct, 15) << "generated auth header: " << auth_hdr << dendl;
+  
+  m["AUTHORIZATION"] = auth_hdr;
+
+  return 0;
+}
+
+int RGWRESTSimpleRequest::forward_request(RGWAccessKey& key, req_info& info, size_t max_response, bufferlist *inbl, bufferlist *outbl)
+{
+
+  string date_str;
+  get_new_date_str(cct, date_str);
+
+  RGWEnv new_env;
+  req_info new_info(cct, &new_env);
+  new_info.rebuild_from(info);
+
+  new_env.set("HTTP_DATE", date_str.c_str());
+
+  int ret = sign_request(key, new_env, new_info);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: failed to sign request" << dendl;
+    return ret;
+  }
+
+  map& m = new_env.get_map();
+  map::iterator iter;
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    headers.push_back(make_pair(iter->first, iter->second));
+  }
+
+  map& meta_map = new_info.x_meta_map;
+  for (iter = meta_map.begin(); iter != meta_map.end(); ++iter) {
+    headers.push_back(make_pair(iter->first, iter->second));
+  }
+
+  string params_str;
+  map& args = new_info.args.get_params();
+  get_params_str(args, params_str);
+
+  string new_url = url;
+  string& resource = new_info.request_uri;
+  string new_resource = resource;
+  if (new_url[new_url.size() - 1] == '/' && resource[0] == '/') {
+    new_url = new_url.substr(0, new_url.size() - 1);
+  } else if (resource[0] != '/') {
+    new_resource = "/";
+    new_resource.append(resource);
+  }
+  new_url.append(new_resource + params_str);
+
+  bufferlist::iterator bliter;
+
+  if (inbl) {
+    bliter = inbl->begin();
+    send_iter = &bliter;
+
+    set_send_length(inbl->length());
+  }
+
+  int r = process(new_info.method, new_url.c_str());
+  if (r < 0)
+    return r;
+
+  response.append((char)0); /* NULL terminate response */
+
+  if (outbl) {
+    outbl->claim(response);
+  }
+
+  return status;
+}
+
+class RGWRESTStreamOutCB : public RGWGetDataCB {
+  RGWRESTStreamWriteRequest *req;
+public:
+  RGWRESTStreamOutCB(RGWRESTStreamWriteRequest *_req) : req(_req) {}
+  int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len); /* callback for object iteration when sending data */
+};
+
+int RGWRESTStreamOutCB::handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len)
+{
+  dout(20) << "RGWRESTStreamOutCB::handle_data bl.length()=" << bl.length() << " bl_ofs=" << bl_ofs << " bl_len=" << bl_len << dendl;
+  if (!bl_ofs && bl_len == bl.length()) {
+    return req->add_output_data(bl);
+  }
+
+  bufferptr bp(bl.c_str() + bl_ofs, bl_len);
+  bufferlist new_bl;
+  new_bl.push_back(bp);
+
+  return req->add_output_data(new_bl);
+}
+
+RGWRESTStreamWriteRequest::~RGWRESTStreamWriteRequest()
+{
+  delete cb;
+}
+
+int RGWRESTStreamWriteRequest::add_output_data(bufferlist& bl)
+{
+  lock.Lock();
+  if (status < 0) {
+    int ret = status;
+    lock.Unlock();
+    return ret;
+  }
+  pending_send.push_back(bl);
+  lock.Unlock();
+
+  bool done;
+  return process_request(handle, false, &done);
+}
+
+static void grants_by_type_add_one_grant(map& grants_by_type, int perm, ACLGrant& grant)
+{
+  string& s = grants_by_type[perm];
+
+  if (!s.empty())
+    s.append(", ");
+
+  string id_type_str;
+  ACLGranteeType& type = grant.get_type();
+  switch (type.get_type()) {
+    case ACL_TYPE_GROUP:
+      id_type_str = "uri";
+      break;
+    case ACL_TYPE_EMAIL_USER:
+      id_type_str = "emailAddress";
+      break;
+    default:
+      id_type_str = "id";
+  }
+  string id;
+  grant.get_id(id);
+  s.append(id_type_str + "=\"" + id + "\"");
+}
+
+struct grant_type_to_header {
+  int type;
+  const char *header;
+};
+
+struct grant_type_to_header grants_headers_def[] = {
+  { RGW_PERM_FULL_CONTROL, "x-amz-grant-full-control"},
+  { RGW_PERM_READ,         "x-amz-grant-read"},
+  { RGW_PERM_WRITE,        "x-amz-grant-write"},
+  { RGW_PERM_READ_ACP,     "x-amz-grant-read-acp"},
+  { RGW_PERM_WRITE_ACP,    "x-amz-grant-write-acp"},
+  { 0, NULL}
+};
+
+static bool grants_by_type_check_perm(map& grants_by_type, int perm, ACLGrant& grant, int check_perm)
+{
+  if ((perm & check_perm) == perm) {
+    grants_by_type_add_one_grant(grants_by_type, check_perm, grant);
+    return true;
+  }
+  return false;
+}
+
+static void grants_by_type_add_perm(map& grants_by_type, int perm, ACLGrant& grant)
+{
+  struct grant_type_to_header *t;
+
+  for (t = grants_headers_def; t->header; t++) {
+    if (grants_by_type_check_perm(grants_by_type, perm, grant, t->type))
+      return;
+  }
+}
+
+static void add_grants_headers(map& grants, map& attrs, map& meta_map)
+{
+  struct grant_type_to_header *t;
+
+  for (t = grants_headers_def; t->header; t++) {
+    map::iterator iter = grants.find(t->type);
+    if (iter != grants.end()) {
+      attrs[t->header] = iter->second;
+      meta_map[t->header] = iter->second;
+    }
+  }
+}
+
+int RGWRESTStreamWriteRequest::put_obj_init(RGWAccessKey& key, rgw_obj& obj, uint64_t obj_size, map& attrs)
+{
+  string resource = obj.bucket.name + "/" + obj.object;
+  string new_url = url;
+  if (new_url[new_url.size() - 1] != '/')
+    new_url.append("/");
+
+  string date_str;
+  get_new_date_str(cct, date_str);
+
+  RGWEnv new_env;
+  req_info new_info(cct, &new_env);
+  
+  string params_str;
+  map& args = new_info.args.get_params();
+  get_params_str(args, params_str);
+
+  new_url.append(resource + params_str);
+
+  new_env.set("HTTP_DATE", date_str.c_str());
+
+  new_info.method = "PUT";
+
+  new_info.script_uri = "/";
+  new_info.script_uri.append(resource);
+  new_info.request_uri = new_info.script_uri;
+  new_info.effective_uri = new_info.effective_uri;
+
+  map& m = new_env.get_map();
+  map::iterator bliter;
+
+  /* merge send headers */
+  for (bliter = attrs.begin(); bliter != attrs.end(); ++bliter) {
+    bufferlist& bl = bliter->second;
+    const string& name = bliter->first;
+    string val = bl.c_str();
+    if (name.compare(0, sizeof(RGW_ATTR_META_PREFIX) - 1, RGW_ATTR_META_PREFIX) == 0) {
+      string header_name = RGW_AMZ_META_PREFIX;
+      header_name.append(name.substr(sizeof(RGW_ATTR_META_PREFIX) - 1));
+      m[header_name] = val;
+      new_info.x_meta_map[header_name] = val;
+    }
+  }
+  RGWAccessControlPolicy policy;
+  int ret = rgw_policy_from_attrset(cct, attrs, &policy);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: couldn't get policy ret=" << ret << dendl;
+    return ret;
+  }
+
+  /* update acl headers */
+  RGWAccessControlList& acl = policy.get_acl();
+  multimap& grant_map = acl.get_grant_map();
+  multimap::iterator giter;
+  map grants_by_type;
+  for (giter = grant_map.begin(); giter != grant_map.end(); ++giter) {
+    ACLGrant& grant = giter->second;
+    ACLPermission& perm = grant.get_permission();
+    grants_by_type_add_perm(grants_by_type, perm.get_permissions(), grant);
+  }
+  add_grants_headers(grants_by_type, m, new_info.x_meta_map);
+  ret = sign_request(key, new_env, new_info);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: failed to sign request" << dendl;
+    return ret;
+  }
+
+  map::iterator iter;
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    headers.push_back(make_pair(iter->first, iter->second));
+  }
+
+  cb = new RGWRESTStreamOutCB(this);
+
+  set_send_length(obj_size);
+
+  int r = init_async(new_info.method, new_url.c_str(), &handle);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRESTStreamWriteRequest::send_data(void *ptr, size_t len)
+{
+  uint64_t sent = 0;
+
+  dout(20) << "RGWRESTStreamWriteRequest::send_data()" << dendl;
+  lock.Lock();
+  if (pending_send.empty() || status < 0) {
+    lock.Unlock();
+    return status;
+  }
+
+  list::iterator iter = pending_send.begin();
+  while (iter != pending_send.end() && len > 0) {
+    bufferlist& bl = *iter;
+    
+    list::iterator next_iter = iter;
+    ++next_iter;
+    lock.Unlock();
+
+    uint64_t send_len = min(len, (size_t)bl.length());
+
+    memcpy(ptr, bl.c_str(), send_len);
+
+    ptr = (char *)ptr + send_len;
+    len -= send_len;
+    sent += send_len;
+
+    lock.Lock();
+
+    bufferlist new_bl;
+    if (bl.length() > send_len) {
+      bufferptr bp(bl.c_str() + send_len, bl.length() - send_len);
+      new_bl.append(bp);
+    }
+    pending_send.pop_front(); /* need to do this after we copy data from bl */
+    if (new_bl.length()) {
+      pending_send.push_front(new_bl);
+    }
+    iter = next_iter;
+  }
+  lock.Unlock();
+
+  return sent;
+}
+
+
+void set_str_from_headers(map& out_headers, const string& header_name, string& str)
+{
+  map::iterator iter = out_headers.find(header_name);
+  if (iter != out_headers.end()) {
+    str = iter->second;
+  } else {
+    str.clear();
+  }
+}
+
+int RGWRESTStreamWriteRequest::complete(string& etag, time_t *mtime)
+{
+  int ret = complete_request(handle);
+  if (ret < 0)
+    return ret;
+
+  set_str_from_headers(out_headers, "ETAG", etag);
+  if (mtime) {
+    string mtime_str;
+    set_str_from_headers(out_headers, "RGWX_MTIME", mtime_str);
+    string err;
+    long t = strict_strtol(mtime_str.c_str(), 10, &err);
+    if (!err.empty()) {
+      ldout(cct, 0) << "ERROR: failed converting mtime (" << mtime_str << ") to int " << dendl;
+      return -EINVAL;
+    }
+    *mtime = (time_t)t;
+  }
+
+  return status;
+}
+
+int RGWRESTStreamReadRequest::get_obj(RGWAccessKey& key, map& extra_headers, rgw_obj& obj)
+{
+  string urlsafe_bucket, urlsafe_object;
+  url_encode(obj.bucket.name, urlsafe_bucket);
+  url_encode(obj.object, urlsafe_object);
+  string resource = urlsafe_bucket + "/" + urlsafe_object;
+  string new_url = url;
+  if (new_url[new_url.size() - 1] != '/')
+    new_url.append("/");
+
+  string date_str;
+  get_new_date_str(cct, date_str);
+
+  RGWEnv new_env;
+  req_info new_info(cct, &new_env);
+  
+  string params_str;
+  map& args = new_info.args.get_params();
+  get_params_str(args, params_str);
+
+  new_url.append(resource + params_str);
+
+  new_env.set("HTTP_DATE", date_str.c_str());
+
+  for (map::iterator iter = extra_headers.begin();
+       iter != extra_headers.end(); ++iter) {
+    new_env.set(iter->first.c_str(), iter->second.c_str());
+  }
+
+  new_info.method = "GET";
+
+  new_info.script_uri = "/";
+  new_info.script_uri.append(resource);
+  new_info.request_uri = new_info.script_uri;
+  new_info.effective_uri = new_info.effective_uri;
+
+  new_info.init_meta_info(NULL);
+
+  int ret = sign_request(key, new_env, new_info);
+  if (ret < 0) {
+    ldout(cct, 0) << "ERROR: failed to sign request" << dendl;
+    return ret;
+  }
+
+  map& m = new_env.get_map();
+  map::iterator iter;
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    headers.push_back(make_pair(iter->first, iter->second));
+  }
+
+  int r = process(new_info.method, new_url.c_str());
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRESTStreamReadRequest::complete(string& etag, time_t *mtime, map& attrs)
+{
+  set_str_from_headers(out_headers, "ETAG", etag);
+  if (mtime) {
+    string mtime_str;
+    set_str_from_headers(out_headers, "RGWX_MTIME", mtime_str);
+    if (!mtime_str.empty()) {
+      string err;
+      long t = strict_strtol(mtime_str.c_str(), 10, &err);
+      if (!err.empty()) {
+        ldout(cct, 0) << "ERROR: failed converting mtime (" << mtime_str << ") to int " << dendl;
+        return -EINVAL;
+      }
+      *mtime = (time_t)t;
+    }
+  }
+
+  map::iterator iter;
+  for (iter = out_headers.begin(); iter != out_headers.end(); ++iter) {
+    const string& attr_name = iter->first;
+    if (attr_name.compare(0, sizeof(RGW_HTTP_RGWX_ATTR_PREFIX) - 1, RGW_HTTP_RGWX_ATTR_PREFIX) == 0) {
+      string name = attr_name.substr(sizeof(RGW_HTTP_RGWX_ATTR_PREFIX) - 1);
+      const char *src = name.c_str();
+      char buf[name.size() + 1];
+      char *dest = buf;
+      for (; *src; ++src, ++dest) {
+        switch(*src) {
+          case '_':
+            *dest = '-';
+            break;
+          default:
+            *dest = tolower(*src);
+        }
+      }
+      *dest = '\0';
+      attrs[buf] = iter->second;
+    }
+  }
+  return status;
+}
+
+int RGWRESTStreamReadRequest::handle_header(const string& name, const string& val)
+{
+  if (name == "RGWX_EMBEDDED_METADATA_LEN") {
+    string err;
+    long len = strict_strtol(val.c_str(), 10, &err);
+    if (!err.empty()) {
+      ldout(cct, 0) << "ERROR: failed converting embedded metadata len (" << val << ") to int " << dendl;
+      return -EINVAL;
+    }
+
+    cb->set_extra_data_len(len);
+  }
+  return 0;
+}
+
+int RGWRESTStreamReadRequest::receive_data(void *ptr, size_t len)
+{
+  bufferptr bp((const char *)ptr, len);
+  bufferlist bl;
+  bl.append(bp);
+  int ret = cb->handle_data(bl, ofs, len);
+  if (ret < 0)
+    return ret;
+  ofs += len;
+  return len;
+}
+
+int RGWRESTStreamReadRequest::send_data(void *ptr, size_t len)
+{
+  /* not sending any data */
+  return 0;
+}
+
diff --git a/ceph/src/rgw/rgw_rest_client.h b/ceph/src/rgw/rgw_rest_client.h
new file mode 100644
index 00000000..a12c3dd7
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_client.h
@@ -0,0 +1,98 @@
+#ifndef CEPH_RGW_REST_CLIENT_H
+#define CEPH_RGW_REST_CLIENT_H
+
+#include 
+
+#include "rgw_http_client.h"
+
+class RGWGetDataCB;
+
+class RGWRESTSimpleRequest : public RGWHTTPClient {
+protected:
+  int http_status;
+  int status;
+
+  string url;
+
+  map out_headers;
+  list > params;
+
+  bufferlist::iterator *send_iter;
+
+  size_t max_response; /* we need this as we don't stream out response */
+  bufferlist response;
+
+  virtual int handle_header(const string& name, const string& val) { return 0; }
+  void append_param(string& dest, const string& name, const string& val);
+  void get_params_str(map& extra_args, string& dest);
+
+  int sign_request(RGWAccessKey& key, RGWEnv& env, req_info& info);
+public:
+  RGWRESTSimpleRequest(CephContext *_cct, string& _url, list > *_headers,
+                list > *_params) : RGWHTTPClient(_cct), http_status(0), status(0),
+                url(_url), send_iter(NULL),
+                max_response(0) {
+    if (_headers)
+      headers = *_headers;
+
+    if (_params)
+      params = *_params;
+  }
+
+  int receive_header(void *ptr, size_t len);
+  virtual int receive_data(void *ptr, size_t len);
+  virtual int send_data(void *ptr, size_t len);
+
+  bufferlist& get_response() { return response; }
+
+  int execute(RGWAccessKey& key, const char *method, const char *resource);
+  int forward_request(RGWAccessKey& key, req_info& info, size_t max_response, bufferlist *inbl, bufferlist *outbl);
+
+  map& get_out_headers() { return out_headers; }
+};
+
+
+class RGWRESTStreamWriteRequest : public RGWRESTSimpleRequest {
+  Mutex lock;
+  list pending_send;
+  void *handle;
+  RGWGetDataCB *cb;
+public:
+  int add_output_data(bufferlist& bl);
+  int send_data(void *ptr, size_t len);
+
+  RGWRESTStreamWriteRequest(CephContext *_cct, string& _url, list > *_headers,
+                list > *_params) : RGWRESTSimpleRequest(_cct, _url, _headers, _params),
+                lock("RGWRESTStreamWriteRequest"), handle(NULL), cb(NULL) {}
+  ~RGWRESTStreamWriteRequest();
+  int put_obj_init(RGWAccessKey& key, rgw_obj& obj, uint64_t obj_size, map& attrs);
+  int complete(string& etag, time_t *mtime);
+
+  RGWGetDataCB *get_out_cb() { return cb; }
+};
+
+class RGWRESTStreamReadRequest : public RGWRESTSimpleRequest {
+  Mutex lock;
+  RGWGetDataCB *cb;
+  bufferlist in_data;
+  size_t chunk_ofs;
+  size_t ofs;
+protected:
+  int handle_header(const string& name, const string& val);
+public:
+  int send_data(void *ptr, size_t len);
+  int receive_data(void *ptr, size_t len);
+
+  RGWRESTStreamReadRequest(CephContext *_cct, string& _url, RGWGetDataCB *_cb, list > *_headers,
+                list > *_params) : RGWRESTSimpleRequest(_cct, _url, _headers, _params),
+                lock("RGWRESTStreamReadRequest"), cb(_cb),
+                chunk_ofs(0), ofs(0) {}
+  ~RGWRESTStreamReadRequest() {}
+  int get_obj(RGWAccessKey& key, map& extra_headers, rgw_obj& obj);
+  int complete(string& etag, time_t *mtime, map& attrs);
+
+  void set_in_cb(RGWGetDataCB *_cb) { cb = _cb; }
+};
+
+#endif
+
diff --git a/ceph/src/rgw/rgw_rest_config.cc b/ceph/src/rgw/rgw_rest_config.cc
new file mode 100644
index 00000000..27cc9897
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_config.cc
@@ -0,0 +1,47 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "common/ceph_json.h"
+#include "common/strtol.h"
+#include "rgw_rest.h"
+#include "rgw_op.h"
+#include "rgw_rados.h"
+#include "rgw_rest_s3.h"
+#include "rgw_rest_config.h"
+#include "rgw_client_io.h"
+#include "common/errno.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+void RGWOp_RegionMap_Get::execute() {
+  http_ret = regionmap.read(g_ceph_context, store);
+  if (http_ret < 0) {
+    dout(5) << "failed to read region map" << dendl;
+  }
+}
+
+void RGWOp_RegionMap_Get::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (http_ret < 0)
+    return;
+  
+  encode_json("region-map", regionmap, s->formatter);
+  flusher.flush(); 
+}
+
+RGWOp* RGWHandler_Config::op_get() {
+  return new RGWOp_RegionMap_Get;
+}
diff --git a/ceph/src/rgw/rgw_rest_config.h b/ceph/src/rgw/rgw_rest_config.h
new file mode 100644
index 00000000..2e0408af
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_config.h
@@ -0,0 +1,55 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#ifndef CEPH_RGW_REST_CONFIG_H
+#define CEPH_RGW_REST_CONFIG_H
+
+class RGWOp_RegionMap_Get : public RGWRESTOp {
+  RGWRegionMap regionmap;
+public:
+  RGWOp_RegionMap_Get() {}
+  ~RGWOp_RegionMap_Get() {}
+
+  int verify_permission() {
+    return 0; 
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "get_region_map";
+  }
+};
+
+class RGWHandler_Config : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get();
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+public:
+  RGWHandler_Config() : RGWHandler_Auth_S3() {}
+  virtual ~RGWHandler_Config() {}
+};
+
+class RGWRESTMgr_Config : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Config() {}
+  virtual ~RGWRESTMgr_Config() {}
+
+  virtual RGWHandler *get_handler(struct req_state *s){
+    return new RGWHandler_Config;
+  }
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_conn.cc b/ceph/src/rgw/rgw_rest_conn.cc
new file mode 100644
index 00000000..32bf0eaa
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_conn.cc
@@ -0,0 +1,119 @@
+#include "rgw_rados.h"
+#include "rgw_rest_conn.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+RGWRESTConn::RGWRESTConn(CephContext *_cct, RGWRados *store, list& remote_endpoints) : cct(_cct)
+{
+  list::iterator iter;
+  int i;
+  for (i = 0, iter = remote_endpoints.begin(); iter != remote_endpoints.end(); ++iter, ++i) {
+    endpoints[i] = *iter;
+  }
+  key = store->zone.system_key;
+  region = store->region.name;
+}
+
+int RGWRESTConn::get_url(string& endpoint)
+{
+  if (endpoints.empty()) {
+    ldout(cct, 0) << "ERROR: endpoints not configured for upstream zone" << dendl;
+    return -EIO;
+  }
+
+  int i = counter.inc();
+  endpoint = endpoints[i % endpoints.size()];
+
+  return 0;
+}
+
+int RGWRESTConn::forward(const string& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl)
+{
+  string url;
+  int ret = get_url(url);
+  if (ret < 0)
+    return ret;
+  list > params;
+  params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "uid", uid));
+  params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "region", region));
+  if (objv) {
+    params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "tag", objv->tag));
+    char buf[16];
+    snprintf(buf, sizeof(buf), "%lld", (long long)objv->ver);
+    params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "ver", buf));
+  }
+  RGWRESTSimpleRequest req(cct, url, NULL, ¶ms);
+  return req.forward_request(key, info, max_response, inbl, outbl);
+}
+
+class StreamObjData : public RGWGetDataCB {
+  rgw_obj obj;
+public:
+    StreamObjData(rgw_obj& _obj) : obj(_obj) {}
+};
+
+int RGWRESTConn::put_obj_init(const string& uid, rgw_obj& obj, uint64_t obj_size,
+                                      map& attrs, RGWRESTStreamWriteRequest **req)
+{
+  string url;
+  int ret = get_url(url);
+  if (ret < 0)
+    return ret;
+
+  list > params;
+  params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "uid", uid));
+  params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "region", region));
+  *req = new RGWRESTStreamWriteRequest(cct, url, NULL, ¶ms);
+  return (*req)->put_obj_init(key, obj, obj_size, attrs);
+}
+
+int RGWRESTConn::complete_request(RGWRESTStreamWriteRequest *req, string& etag, time_t *mtime)
+{
+  int ret = req->complete(etag, mtime);
+  delete req;
+
+  return ret;
+}
+
+int RGWRESTConn::get_obj(const string& uid, req_info *info /* optional */, rgw_obj& obj, bool prepend_metadata,
+                                 RGWGetDataCB *cb, RGWRESTStreamReadRequest **req)
+{
+  string url;
+  int ret = get_url(url);
+  if (ret < 0)
+    return ret;
+
+  list > params;
+  params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "uid", uid));
+  params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "region", region));
+  if (prepend_metadata) {
+    params.push_back(make_pair(RGW_SYS_PARAM_PREFIX "prepend-metadata", region));
+  }
+  *req = new RGWRESTStreamReadRequest(cct, url, cb, NULL, ¶ms);
+  map extra_headers;
+  if (info) {
+    map& orig_map = info->env->get_map();
+
+    /* add original headers that start with HTTP_X_AMZ_ */
+#define SEARCH_AMZ_PREFIX "HTTP_X_AMZ_"
+    for (map::iterator iter = orig_map.lower_bound(SEARCH_AMZ_PREFIX); iter != orig_map.end(); ++iter) {
+      const string& name = iter->first;
+      if (name == "HTTP_X_AMZ_DATE") /* dont forward date from original request */
+        continue;
+      if (name.compare(0, sizeof(SEARCH_AMZ_PREFIX) - 1, "HTTP_X_AMZ_") != 0)
+        break;
+      extra_headers[iter->first] = iter->second;
+    }
+  }
+  return (*req)->get_obj(key, extra_headers, obj);
+}
+
+int RGWRESTConn::complete_request(RGWRESTStreamReadRequest *req, string& etag, time_t *mtime, map& attrs)
+{
+  int ret = req->complete(etag, mtime, attrs);
+  delete req;
+
+  return ret;
+}
+
+
diff --git a/ceph/src/rgw/rgw_rest_conn.h b/ceph/src/rgw/rgw_rest_conn.h
new file mode 100644
index 00000000..4a0b6087
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_conn.h
@@ -0,0 +1,34 @@
+#ifndef CEPH_RGW_REST_CONN_H
+#define CEPH_RGW_REST_CONN_H
+
+#include "rgw_rest_client.h"
+
+class CephContext;
+class RGWRados;
+class RGWGetObjData;
+
+class RGWRESTConn
+{
+  CephContext *cct;
+  map endpoints;
+  RGWAccessKey key;
+  string region;
+  atomic_t counter;
+public:
+
+  RGWRESTConn(CephContext *_cct, RGWRados *store, list& endpoints);
+  int get_url(string& endpoint);
+
+  /* sync request */
+  int forward(const string& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl);
+
+  /* async request */
+  int put_obj_init(const string& uid, rgw_obj& obj, uint64_t obj_size,
+                   map& attrs, RGWRESTStreamWriteRequest **req);
+  int complete_request(RGWRESTStreamWriteRequest *req, string& etag, time_t *mtime);
+
+  int get_obj(const string& uid, req_info *info /* optional */, rgw_obj& obj, bool prepend_metadata, RGWGetDataCB *cb, RGWRESTStreamReadRequest **req);
+  int complete_request(RGWRESTStreamReadRequest *req, string& etag, time_t *mtime, map& attrs);
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_log.cc b/ceph/src/rgw/rgw_rest_log.cc
new file mode 100644
index 00000000..9f32fc9e
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_log.cc
@@ -0,0 +1,742 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "common/ceph_json.h"
+#include "common/strtol.h"
+#include "rgw_rest.h"
+#include "rgw_op.h"
+#include "rgw_rest_s3.h"
+#include "rgw_rest_log.h"
+#include "rgw_client_io.h"
+#include "common/errno.h"
+
+#define LOG_CLASS_LIST_MAX_ENTRIES (1000)
+#define dout_subsys ceph_subsys_rgw
+
+static int parse_date_str(string& in, utime_t& out) {
+  uint64_t epoch = 0;
+  uint64_t nsec = 0;
+
+  if (!in.empty()) {
+    if (utime_t::parse_date(in, &epoch, &nsec) < 0) {
+      dout(5) << "Error parsing date " << in << dendl;
+      return -EINVAL;
+    }
+  }
+  out = utime_t(epoch, nsec);
+  return 0;
+}
+
+void RGWOp_MDLog_List::execute() {
+  string   shard = s->info.args.get("id");
+  string   max_entries_str = s->info.args.get("max-entries");
+  string   st = s->info.args.get("start-time"),
+           et = s->info.args.get("end-time"),
+           marker = s->info.args.get("marker"),
+           err;
+  utime_t  ut_st, 
+           ut_et;
+  void    *handle;
+  unsigned shard_id, max_entries = LOG_CLASS_LIST_MAX_ENTRIES;
+
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id " << shard << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(st, ut_st) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(et, ut_et) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (!max_entries_str.empty()) {
+    max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err);
+    if (!err.empty()) {
+      dout(5) << "Error parsing max-entries " << max_entries_str << dendl;
+      http_ret = -EINVAL;
+      return;
+    }
+  } 
+  
+  RGWMetadataLog *meta_log = store->meta_mgr->get_log();
+
+  meta_log->init_list_entries(shard_id, ut_st, ut_et, marker, &handle);
+
+  do {
+    http_ret = meta_log->list_entries(handle, max_entries, entries,
+				      &last_marker, &truncated);
+    if (http_ret < 0) 
+      break;
+
+    if (!max_entries_str.empty()) 
+      max_entries -= entries.size();
+  } while (truncated && (max_entries > 0));
+
+  meta_log->complete_list_entries(handle);
+}
+
+void RGWOp_MDLog_List::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (http_ret < 0)
+    return;
+
+  s->formatter->open_object_section("log_entries");
+  s->formatter->dump_string("marker", last_marker);
+  s->formatter->dump_bool("truncated", truncated);
+  {
+    s->formatter->open_array_section("entries");
+    for (list::iterator iter = entries.begin();
+	 iter != entries.end(); ++iter) {
+      cls_log_entry& entry = *iter;
+      store->meta_mgr->dump_log_entry(entry, s->formatter);
+      flusher.flush();
+    }
+    s->formatter->close_section();
+  }
+  s->formatter->close_section();
+  flusher.flush();
+}
+
+void RGWOp_MDLog_Info::execute() {
+  num_objects = s->cct->_conf->rgw_md_log_max_shards;
+  http_ret = 0;
+}
+
+void RGWOp_MDLog_Info::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  s->formatter->open_object_section("num_objects");
+  s->formatter->dump_unsigned("num_objects", num_objects);
+  s->formatter->close_section();
+  flusher.flush();
+}
+
+void RGWOp_MDLog_ShardInfo::execute() {
+  string shard = s->info.args.get("id");
+  string err;
+
+  unsigned shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id " << shard << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  RGWMetadataLog *meta_log = store->meta_mgr->get_log();
+
+  http_ret = meta_log->get_info(shard_id, &info);
+}
+
+void RGWOp_MDLog_ShardInfo::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  encode_json("info", info, s->formatter);
+  flusher.flush();
+}
+
+void RGWOp_MDLog_Delete::execute() {
+  string   st = s->info.args.get("start-time"),
+           et = s->info.args.get("end-time"),
+           start_marker = s->info.args.get("start-marker"),
+           end_marker = s->info.args.get("end-marker"),
+           shard = s->info.args.get("id"),
+           err;
+  utime_t  ut_st, 
+           ut_et;
+  unsigned shard_id;
+
+  http_ret = 0;
+
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id " << shard << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  if (et.empty() && end_marker.empty()) { /* bounding end */
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(st, ut_st) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(et, ut_et) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+  RGWMetadataLog *meta_log = store->meta_mgr->get_log();
+
+  http_ret = meta_log->trim(shard_id, ut_st, ut_et, start_marker, end_marker);
+}
+
+void RGWOp_MDLog_Lock::execute() {
+  string shard_id_str, duration_str, locker_id, zone_id;
+  unsigned shard_id;
+
+  http_ret = 0;
+
+  shard_id_str = s->info.args.get("id");
+  duration_str = s->info.args.get("length");
+  locker_id    = s->info.args.get("locker-id");
+  zone_id      = s->info.args.get("zone-id");
+
+  if (shard_id_str.empty() ||
+      (duration_str.empty()) ||
+      locker_id.empty() ||
+      zone_id.empty()) {
+    dout(5) << "Error invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  string err;
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  RGWMetadataLog *meta_log = store->meta_mgr->get_log();
+  unsigned dur;
+  dur = (unsigned)strict_strtol(duration_str.c_str(), 10, &err);
+  if (!err.empty() || dur <= 0) {
+    dout(5) << "invalid length param " << duration_str << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  utime_t time(dur, 0);
+  http_ret = meta_log->lock_exclusive(shard_id, time, zone_id, locker_id);
+  if (http_ret == -EBUSY)
+    http_ret = -ERR_LOCKED;
+}
+
+void RGWOp_MDLog_Unlock::execute() {
+  string shard_id_str, locker_id, zone_id;
+  unsigned shard_id;
+
+  http_ret = 0;
+
+  shard_id_str = s->info.args.get("id");
+  locker_id    = s->info.args.get("locker-id");
+  zone_id      = s->info.args.get("zone-id");
+
+  if (shard_id_str.empty() ||
+      locker_id.empty() ||
+      zone_id.empty()) {
+    dout(5) << "Error invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  string err;
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  RGWMetadataLog *meta_log = store->meta_mgr->get_log();
+  http_ret = meta_log->unlock(shard_id, zone_id, locker_id);
+}
+
+void RGWOp_BILog_List::execute() {
+  string bucket_name = s->info.args.get("bucket"),
+         marker = s->info.args.get("marker"),
+         max_entries_str = s->info.args.get("max-entries"),
+         bucket_instance = s->info.args.get("bucket-instance");
+  RGWBucketInfo bucket_info;
+  unsigned max_entries;
+
+  if (bucket_name.empty() && bucket_instance.empty()) {
+    dout(5) << "ERROR: neither bucket nor bucket instance specified" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (!bucket_instance.empty()) {
+    http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket instance info for bucket instance id=" << bucket_instance << dendl;
+      return;
+    }
+  } else { /* !bucket_name.empty() */
+    http_ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl;
+      return;
+    }
+  }
+
+  bool truncated;
+  unsigned count = 0;
+  string err;
+
+  max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err);
+  if (!err.empty())
+    max_entries = LOG_CLASS_LIST_MAX_ENTRIES;
+
+  send_response();
+  do {
+    list entries;
+    int ret = store->list_bi_log_entries(bucket_info.bucket,
+                                          marker, max_entries - count, 
+                                          entries, &truncated);
+    if (ret < 0) {
+      dout(5) << "ERROR: list_bi_log_entries()" << dendl;
+      return;
+    }
+
+    count += entries.size();
+
+    send_response(entries, marker);
+  } while (truncated && count < max_entries);
+
+  send_response_end();
+}
+
+void RGWOp_BILog_List::send_response() {
+  if (sent_header)
+    return;
+
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  sent_header = true;
+
+  if (http_ret < 0)
+    return;
+
+  s->formatter->open_array_section("entries");
+}
+
+void RGWOp_BILog_List::send_response(list& entries, string& marker)
+{
+  for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
+    rgw_bi_log_entry& entry = *iter;
+    encode_json("entry", entry, s->formatter);
+
+    marker = entry.id;
+    flusher.flush();
+  }
+}
+
+void RGWOp_BILog_List::send_response_end() {
+  s->formatter->close_section();
+  flusher.flush();
+}
+      
+void RGWOp_BILog_Info::execute() {
+  string bucket_name = s->info.args.get("bucket"),
+         bucket_instance = s->info.args.get("bucket-instance");
+  RGWBucketInfo bucket_info;
+
+  if (bucket_name.empty() && bucket_instance.empty()) {
+    dout(5) << "ERROR: neither bucket nor bucket instance specified" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (!bucket_instance.empty()) {
+    http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket instance info for bucket instance id=" << bucket_instance << dendl;
+      return;
+    }
+  } else { /* !bucket_name.empty() */
+    http_ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl;
+      return;
+    }
+  }
+  map stats;
+  int ret =  store->get_bucket_stats(bucket_info.bucket, &bucket_ver, &master_ver, stats, &max_marker);
+  if (ret < 0 && ret != -ENOENT) {
+    http_ret = ret;
+    return;
+  }
+}
+
+void RGWOp_BILog_Info::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (http_ret < 0)
+    return;
+
+  s->formatter->open_object_section("info");
+  encode_json("bucket_ver", bucket_ver, s->formatter);
+  encode_json("master_ver", master_ver, s->formatter);
+  encode_json("max_marker", max_marker, s->formatter);
+  s->formatter->close_section();
+
+  flusher.flush();
+}
+
+void RGWOp_BILog_Delete::execute() {
+  string bucket_name = s->info.args.get("bucket"),
+         start_marker = s->info.args.get("start-marker"),
+         end_marker = s->info.args.get("end-marker"),
+         bucket_instance = s->info.args.get("bucket-instance");
+
+  RGWBucketInfo bucket_info;
+
+  http_ret = 0;
+  if ((bucket_name.empty() && bucket_instance.empty()) ||
+      end_marker.empty()) {
+    dout(5) << "ERROR: one of bucket and bucket instance, and also end-marker is mandatory" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  if (!bucket_instance.empty()) {
+    http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket instance info for bucket instance id=" << bucket_instance << dendl;
+      return;
+    }
+  } else { /* !bucket_name.empty() */
+    http_ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl;
+      return;
+    }
+  }
+  http_ret = store->trim_bi_log_entries(bucket_info.bucket, start_marker, end_marker);
+  if (http_ret < 0) {
+    dout(5) << "ERROR: trim_bi_log_entries() " << dendl;
+  }
+  return;
+}
+
+void RGWOp_DATALog_List::execute() {
+  string   shard = s->info.args.get("id");
+
+  string   st = s->info.args.get("start-time"),
+           et = s->info.args.get("end-time"),
+           max_entries_str = s->info.args.get("max-entries"),
+           marker = s->info.args.get("marker"),
+           err;
+  utime_t  ut_st, 
+           ut_et;
+  unsigned shard_id, max_entries = LOG_CLASS_LIST_MAX_ENTRIES;
+
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id " << shard << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(st, ut_st) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(et, ut_et) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (!max_entries_str.empty()) {
+    max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err);
+    if (!err.empty()) {
+      dout(5) << "Error parsing max-entries " << max_entries_str << dendl;
+      http_ret = -EINVAL;
+      return;
+    }
+  } 
+  
+  do {
+    // Note that last_marker is updated to be the marker of the last
+    // entry listed
+    http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, 
+					     max_entries, entries, marker,
+					     &last_marker, &truncated);
+    if (http_ret < 0) 
+      break;
+
+    if (!max_entries_str.empty()) 
+      max_entries -= entries.size();
+  } while (truncated && (max_entries > 0));
+}
+
+void RGWOp_DATALog_List::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (http_ret < 0)
+    return;
+
+  s->formatter->open_object_section("log_entries");
+  s->formatter->dump_string("marker", last_marker);
+  s->formatter->dump_bool("truncated", truncated);
+  {
+    s->formatter->open_array_section("entries");
+    for (list::iterator iter = entries.begin();
+	 iter != entries.end(); ++iter) {
+      rgw_data_change& entry = *iter;
+      encode_json("entry", entry, s->formatter);
+      flusher.flush();
+    }
+    s->formatter->close_section();
+  }
+  s->formatter->close_section();
+  flusher.flush();
+}
+
+
+void RGWOp_DATALog_Info::execute() {
+  num_objects = s->cct->_conf->rgw_data_log_num_shards;
+  http_ret = 0;
+}
+
+void RGWOp_DATALog_Info::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  s->formatter->open_object_section("num_objects");
+  s->formatter->dump_unsigned("num_objects", num_objects);
+  s->formatter->close_section();
+  flusher.flush();
+}
+
+void RGWOp_DATALog_ShardInfo::execute() {
+  string shard = s->info.args.get("id");
+  string err;
+
+  unsigned shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id " << shard << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  http_ret = store->data_log->get_info(shard_id, &info);
+}
+
+void RGWOp_DATALog_ShardInfo::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  encode_json("info", info, s->formatter);
+  flusher.flush();
+}
+
+void RGWOp_DATALog_Lock::execute() {
+  string shard_id_str, duration_str, locker_id, zone_id;
+  unsigned shard_id;
+
+  http_ret = 0;
+
+  shard_id_str = s->info.args.get("id");
+  duration_str = s->info.args.get("length");
+  locker_id    = s->info.args.get("locker-id");
+  zone_id      = s->info.args.get("zone-id");
+
+  if (shard_id_str.empty() ||
+      (duration_str.empty()) ||
+      locker_id.empty() ||
+      zone_id.empty()) {
+    dout(5) << "Error invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  string err;
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  unsigned dur;
+  dur = (unsigned)strict_strtol(duration_str.c_str(), 10, &err);
+  if (!err.empty() || dur <= 0) {
+    dout(5) << "invalid length param " << duration_str << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  utime_t time(dur, 0);
+  http_ret = store->data_log->lock_exclusive(shard_id, time, zone_id, locker_id);
+  if (http_ret == -EBUSY)
+    http_ret = -ERR_LOCKED;
+}
+
+void RGWOp_DATALog_Unlock::execute() {
+  string shard_id_str, locker_id, zone_id;
+  unsigned shard_id;
+
+  http_ret = 0;
+
+  shard_id_str = s->info.args.get("id");
+  locker_id    = s->info.args.get("locker-id");
+  zone_id      = s->info.args.get("zone-id");
+
+  if (shard_id_str.empty() ||
+      locker_id.empty() ||
+      zone_id.empty()) {
+    dout(5) << "Error invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  string err;
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  http_ret = store->data_log->unlock(shard_id, zone_id, locker_id);
+}
+
+void RGWOp_DATALog_Delete::execute() {
+  string   st = s->info.args.get("start-time"),
+           et = s->info.args.get("end-time"),
+           start_marker = s->info.args.get("start-marker"),
+           end_marker = s->info.args.get("end-marker"),
+           shard = s->info.args.get("id"),
+           err;
+  utime_t  ut_st, 
+           ut_et;
+  unsigned shard_id;
+
+  http_ret = 0;
+
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing shard_id " << shard << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  if (et.empty() && end_marker.empty()) { /* bounding end */
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(st, ut_st) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (parse_date_str(et, ut_et) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  http_ret = store->data_log->trim_entries(shard_id, ut_st, ut_et, start_marker, end_marker);
+}
+
+RGWOp *RGWHandler_Log::op_get() {
+  bool exists;
+  string type = s->info.args.get("type", &exists);
+
+  if (!exists) {
+    return NULL;
+  }
+
+  if (type.compare("metadata") == 0) {
+    if (s->info.args.exists("id")) {
+      if (s->info.args.exists("info")) {
+        return new RGWOp_MDLog_ShardInfo;
+      } else {
+        return new RGWOp_MDLog_List;
+      }
+    } else {
+      return new RGWOp_MDLog_Info;
+    }
+  } else if (type.compare("bucket-index") == 0) {
+    if (s->info.args.exists("info")) {
+      return new RGWOp_BILog_Info;
+    } else {
+      return new RGWOp_BILog_List;
+    }
+  } else if (type.compare("data") == 0) {
+    if (s->info.args.exists("id")) {
+      if (s->info.args.exists("info")) {
+        return new RGWOp_DATALog_ShardInfo;
+      } else {
+        return new RGWOp_DATALog_List;
+      }
+    } else {
+      return new RGWOp_DATALog_Info;
+    }
+  }
+  return NULL;
+}
+
+RGWOp *RGWHandler_Log::op_delete() {
+  bool exists;
+  string type = s->info.args.get("type", &exists);
+
+  if (!exists) {
+    return NULL;
+  }
+
+  if (type.compare("metadata") == 0)
+    return new RGWOp_MDLog_Delete;
+  else if (type.compare("bucket-index") == 0) 
+    return new RGWOp_BILog_Delete;
+  else if (type.compare("data") == 0)
+    return new RGWOp_DATALog_Delete;
+  return NULL;
+}
+
+RGWOp *RGWHandler_Log::op_post() {
+  bool exists;
+  string type = s->info.args.get("type", &exists);
+
+  if (!exists) {
+    return NULL;
+  }
+
+  if (type.compare("metadata") == 0) {
+    if (s->info.args.exists("lock"))
+      return new RGWOp_MDLog_Lock;
+    else if (s->info.args.exists("unlock"))
+      return new RGWOp_MDLog_Unlock;
+  } else if (type.compare("data") == 0) {
+    if (s->info.args.exists("lock"))
+      return new RGWOp_DATALog_Lock;
+    else if (s->info.args.exists("unlock"))
+      return new RGWOp_DATALog_Unlock;
+  }
+  return NULL;
+}
+
diff --git a/ceph/src/rgw/rgw_rest_log.h b/ceph/src/rgw/rgw_rest_log.h
new file mode 100644
index 00000000..ff1bf346
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_log.h
@@ -0,0 +1,306 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#ifndef CEPH_RGW_REST_LOG_H
+#define CEPH_RGW_REST_LOG_H
+
+#include "rgw_metadata.h"
+
+class RGWOp_BILog_List : public RGWRESTOp {
+  bool sent_header;
+public:
+  RGWOp_BILog_List() : sent_header(false) {}
+  ~RGWOp_BILog_List() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("bilog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  virtual void send_response();
+  virtual void send_response(list& entries, string& marker);
+  virtual void send_response_end();
+  void execute();
+  virtual const string name() {
+    return "list_bucket_index_log";
+  }
+};
+
+class RGWOp_BILog_Info : public RGWRESTOp {
+  uint64_t bucket_ver;
+  uint64_t master_ver;
+  string max_marker;
+public:
+  RGWOp_BILog_Info() : bucket_ver(0), master_ver(0) {}
+  ~RGWOp_BILog_Info() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("bilog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  virtual void send_response();
+  void execute();
+  virtual const string name() {
+    return "bucket_index_log_info";
+  }
+};
+
+class RGWOp_BILog_Delete : public RGWRESTOp {
+public:
+  RGWOp_BILog_Delete() {}
+  ~RGWOp_BILog_Delete() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("bilog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "trim_bucket_index_log";
+  }
+};
+
+class RGWOp_MDLog_List : public RGWRESTOp {
+  list entries;
+  string last_marker;
+  bool truncated;
+  int http_ret;
+public:
+  RGWOp_MDLog_List() : truncated(false), http_ret(0) {}
+  ~RGWOp_MDLog_List() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("mdlog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "list_metadata_log";
+  }
+};
+
+class RGWOp_MDLog_Info : public RGWRESTOp {
+  unsigned num_objects;
+  int http_ret;
+public:
+  RGWOp_MDLog_Info() : num_objects(0), http_ret(0) {}
+  ~RGWOp_MDLog_Info() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("mdlog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "get_metadata_log_info";
+  }
+};
+
+class RGWOp_MDLog_ShardInfo : public RGWRESTOp {
+  RGWMetadataLogInfo info;
+public:
+  RGWOp_MDLog_ShardInfo() {}
+  ~RGWOp_MDLog_ShardInfo() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("mdlog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "get_metadata_log_shard_info";
+  }
+};
+
+class RGWOp_MDLog_Lock : public RGWRESTOp {
+public:
+  RGWOp_MDLog_Lock() {}
+  ~RGWOp_MDLog_Lock() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("mdlog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "lock_mdlog_object";
+  }
+};
+
+class RGWOp_MDLog_Unlock : public RGWRESTOp {
+public:
+  RGWOp_MDLog_Unlock() {}
+  ~RGWOp_MDLog_Unlock() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("mdlog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "unlock_mdlog_object";
+  }
+};
+
+class RGWOp_MDLog_Delete : public RGWRESTOp {
+public:
+  RGWOp_MDLog_Delete() {}
+  ~RGWOp_MDLog_Delete() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("mdlog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "trim_metadata_log";
+  }
+};
+
+class RGWOp_DATALog_List : public RGWRESTOp {
+  list entries;
+  string last_marker;
+  bool truncated;
+  int http_ret;
+public:
+  RGWOp_DATALog_List() : truncated(false), http_ret(0) {}
+  ~RGWOp_DATALog_List() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("datalog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "list_data_changes_log";
+  }
+};
+
+class RGWOp_DATALog_Info : public RGWRESTOp {
+  unsigned num_objects;
+  int http_ret;
+public:
+  RGWOp_DATALog_Info() : num_objects(0), http_ret(0) {}
+  ~RGWOp_DATALog_Info() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("datalog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "get_data_changes_log_info";
+  }
+};
+
+class RGWOp_DATALog_ShardInfo : public RGWRESTOp {
+  RGWDataChangesLogInfo info;
+public:
+  RGWOp_DATALog_ShardInfo() {}
+  ~RGWOp_DATALog_ShardInfo() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("datalog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "get_data_changes_log_shard_info";
+  }
+};
+
+class RGWOp_DATALog_Lock : public RGWRESTOp {
+public:
+  RGWOp_DATALog_Lock() {}
+  ~RGWOp_DATALog_Lock() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("datalog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "lock_datalog_object";
+  }
+};
+
+class RGWOp_DATALog_Unlock : public RGWRESTOp {
+public:
+  RGWOp_DATALog_Unlock() {}
+  ~RGWOp_DATALog_Unlock() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("datalog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "unlock_datalog_object";
+  }
+};
+
+class RGWOp_DATALog_Delete : public RGWRESTOp {
+public:
+  RGWOp_DATALog_Delete() {}
+  ~RGWOp_DATALog_Delete() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("datalog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "trim_data_changes_log";
+  }
+};
+
+class RGWHandler_Log : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_delete();
+  RGWOp *op_post();
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+public:
+  RGWHandler_Log() : RGWHandler_Auth_S3() {}
+  virtual ~RGWHandler_Log() {}
+};
+
+class RGWRESTMgr_Log : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Log() {}
+  virtual ~RGWRESTMgr_Log() {}
+
+  virtual RGWHandler *get_handler(struct req_state *s){
+    return new RGWHandler_Log;
+  }
+};
+
+#endif
+
diff --git a/ceph/src/rgw/rgw_rest_metadata.cc b/ceph/src/rgw/rgw_rest_metadata.cc
new file mode 100644
index 00000000..afd5c7b6
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_metadata.cc
@@ -0,0 +1,291 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "include/page.h"
+
+#include "rgw_rest.h"
+#include "rgw_op.h"
+#include "rgw_rest_s3.h"
+#include "rgw_rest_metadata.h"
+#include "rgw_client_io.h"
+#include "common/errno.h"
+#include "common/strtol.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+const string RGWOp_Metadata_Get::name() {
+  return "get_metadata";
+}
+
+static inline void frame_metadata_key(req_state *s, string& out) {
+  bool exists;
+  string key = s->info.args.get("key", &exists);
+
+  string section;
+  if (!s->bucket_name_str.empty()) {
+    section = s->bucket_name_str;
+  } else {
+    section = key;
+    key.clear();
+  }
+
+  out = section;
+
+  if (!key.empty()) {
+    out += string(":") + key;
+  }
+}
+
+void RGWOp_Metadata_Get::execute() {
+  string metadata_key;
+
+  frame_metadata_key(s, metadata_key);
+
+  /* Get keys */
+  http_ret = store->meta_mgr->get(metadata_key, s->formatter);
+  if (http_ret < 0) {
+    dout(5) << "ERROR: can't get key: " << cpp_strerror(http_ret) << dendl;
+    return;
+  }
+
+  http_ret = 0;
+}
+
+const string RGWOp_Metadata_List::name() {
+  return "list_metadata";
+}
+
+void RGWOp_Metadata_List::execute() {
+  string metadata_key;
+
+  frame_metadata_key(s, metadata_key);
+  /* List keys */
+  void *handle;
+  int max = 1000;
+
+  http_ret = store->meta_mgr->list_keys_init(metadata_key, &handle);
+  if (http_ret < 0) {
+    dout(5) << "ERROR: can't get key: " << cpp_strerror(http_ret) << dendl;
+    return;
+  }
+
+  bool truncated;
+
+  s->formatter->open_array_section("keys");
+
+  do {
+    list keys;
+    http_ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated);
+    if (http_ret < 0) {
+      dout(5) << "ERROR: lists_keys_next(): " << cpp_strerror(http_ret) << dendl;
+      return;
+    }
+
+    for (list::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
+      s->formatter->dump_string("key", *iter);
+    }
+
+  } while (truncated);
+
+  s->formatter->close_section();
+
+  store->meta_mgr->list_keys_complete(handle);
+
+  http_ret = 0;
+}
+
+int RGWOp_Metadata_Put::get_data(bufferlist& bl) {
+  size_t cl = 0;
+  char *data;
+  int read_len;
+
+  if (s->length)
+    cl = atoll(s->length);
+  if (cl) {
+    data = (char *)malloc(cl + 1);
+    if (!data) {
+       return -ENOMEM;
+    }
+    int r = s->cio->read(data, cl, &read_len);
+    if (cl != (size_t)read_len) {
+      dout(10) << "cio->read incomplete" << dendl;
+    }
+    if (r < 0) {
+      free(data);
+      return r;
+    }
+    bl.append(data, read_len);
+  } else {
+    int chunk_size = CEPH_PAGE_SIZE; 
+    const char *enc = s->info.env->get("HTTP_TRANSFER_ENCODING");
+    if (!enc || strcmp(enc, "chunked")) {
+      return -ERR_LENGTH_REQUIRED;
+    }
+    data = (char *)malloc(chunk_size);
+    if (!data) {
+      return -ENOMEM;
+    }
+    do {
+      int r = s->cio->read(data, chunk_size, &read_len);
+      if (r < 0) {
+        free(data);
+        return r;
+      }
+      bl.append(data, read_len);
+    } while ((read_len == chunk_size));
+  }
+
+  free(data);
+  return 0;
+}
+
+void RGWOp_Metadata_Put::execute() {
+  bufferlist bl;
+  string metadata_key;
+
+  http_ret = get_data(bl);
+  if (http_ret < 0) {
+    return;
+  }
+  
+  frame_metadata_key(s, metadata_key);
+  
+  RGWMetadataHandler::sync_type_t sync_type = RGWMetadataHandler::APPLY_ALWAYS;
+
+  bool mode_exists = false;
+  string mode_string = s->info.args.get("update-type", &mode_exists);
+  if (mode_exists) {
+    bool parsed = RGWMetadataHandler::string_to_sync_type(mode_string,
+                                                          sync_type);
+    if (!parsed) {
+      http_ret = -EINVAL;
+      return;
+    }
+  }
+
+  http_ret = store->meta_mgr->put(metadata_key, bl, sync_type, &ondisk_version);
+  if (http_ret < 0) {
+    dout(5) << "ERROR: can't put key: " << cpp_strerror(http_ret) << dendl;
+    return;
+  }
+  // translate internal codes into return header
+  if (http_ret == STATUS_NO_APPLY)
+    update_status = "skipped";
+  else if (http_ret == STATUS_APPLIED)
+    update_status = "applied";
+}
+
+void RGWOp_Metadata_Put::send_response() {
+  int http_return_code = http_ret;
+  if ((http_ret == STATUS_NO_APPLY) || (http_ret == STATUS_APPLIED))
+    http_return_code = STATUS_NO_CONTENT;
+  set_req_state_err(s, http_return_code);
+  dump_errno(s);
+  stringstream ver_stream;
+  ver_stream << "ver:" << ondisk_version.ver
+	     <<",tag:" << ondisk_version.tag;
+  dump_pair(s, "RGWX_UPDATE_STATUS", update_status.c_str());
+  dump_pair(s, "RGWX_UPDATE_VERSION", ver_stream.str().c_str());
+  end_header(s);
+}
+
+void RGWOp_Metadata_Delete::execute() {
+  string metadata_key;
+
+  frame_metadata_key(s, metadata_key);
+  http_ret = store->meta_mgr->remove(metadata_key);
+  if (http_ret < 0) {
+    dout(5) << "ERROR: can't remove key: " << cpp_strerror(http_ret) << dendl;
+    return;
+  }
+  http_ret = 0;
+}
+
+void RGWOp_Metadata_Lock::execute() {
+  string duration_str, lock_id;
+  string metadata_key;
+
+  frame_metadata_key(s, metadata_key);
+
+  http_ret = 0;
+
+  duration_str = s->info.args.get("length");
+  lock_id      = s->info.args.get("lock_id");
+
+  if ((!s->info.args.exists("key")) || 
+      (duration_str.empty()) ||
+      lock_id.empty()) {
+    dout(5) << "Error invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  int dur;
+  string err;
+
+  dur = strict_strtol(duration_str.c_str(), 10, &err);
+  if (!err.empty() || dur <= 0) {
+    dout(5) << "invalid length param " << duration_str << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  utime_t time(dur, 0);
+  http_ret = store->meta_mgr->lock_exclusive(metadata_key, time, lock_id);
+  if (http_ret == -EBUSY)
+    http_ret = -ERR_LOCKED;
+}
+
+void RGWOp_Metadata_Unlock::execute() {
+  string lock_id;
+  string metadata_key;
+
+  frame_metadata_key(s, metadata_key);
+
+  http_ret = 0;
+
+  lock_id = s->info.args.get("lock_id");
+
+  if ((!s->info.args.exists("key")) || 
+      lock_id.empty()) {
+    dout(5) << "Error invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  http_ret = store->meta_mgr->unlock(metadata_key, lock_id);
+}
+
+RGWOp *RGWHandler_Metadata::op_get() {
+  if (s->info.args.exists("key"))
+    return new RGWOp_Metadata_Get;
+  else
+    return new RGWOp_Metadata_List;
+}
+
+RGWOp *RGWHandler_Metadata::op_put() {
+  return new RGWOp_Metadata_Put;
+}
+
+RGWOp *RGWHandler_Metadata::op_delete() {
+  return new RGWOp_Metadata_Delete;
+}
+
+RGWOp *RGWHandler_Metadata::op_post() {
+  if (s->info.args.exists("lock"))
+    return new RGWOp_Metadata_Lock;
+  else if (s->info.args.exists("unlock"))
+    return new RGWOp_Metadata_Unlock;
+
+  return NULL;
+}
+
diff --git a/ceph/src/rgw/rgw_rest_metadata.h b/ceph/src/rgw/rgw_rest_metadata.h
new file mode 100644
index 00000000..7f3cf1f2
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_metadata.h
@@ -0,0 +1,123 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#ifndef CEPH_RGW_REST_METADATA_H
+#define CEPH_RGW_REST_METADATA_H
+
+class RGWOp_Metadata_List : public RGWRESTOp {
+public:
+  RGWOp_Metadata_List() {}
+  ~RGWOp_Metadata_List() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_READ);
+  }
+  void execute();
+  virtual const string name();
+};
+
+class RGWOp_Metadata_Get : public RGWRESTOp {
+public:
+  RGWOp_Metadata_Get() {}
+  ~RGWOp_Metadata_Get() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_READ);
+  }
+  void execute();
+  virtual const string name();
+};
+
+class RGWOp_Metadata_Put : public RGWRESTOp {
+  int get_data(bufferlist& bl);
+  string update_status;
+  obj_version ondisk_version;
+public:
+  RGWOp_Metadata_Put() {}
+  ~RGWOp_Metadata_Put() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_WRITE);
+  }
+  void execute();
+  void send_response();
+  virtual const string name() { return "set_metadata"; }
+};
+
+class RGWOp_Metadata_Delete : public RGWRESTOp {
+public:
+  RGWOp_Metadata_Delete() {}
+  ~RGWOp_Metadata_Delete() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() { return "remove_metadata"; }
+};
+
+class RGWOp_Metadata_Lock : public RGWRESTOp {
+public:
+  RGWOp_Metadata_Lock() {}
+  ~RGWOp_Metadata_Lock() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "lock_metadata_object";
+  }
+};
+
+class RGWOp_Metadata_Unlock : public RGWRESTOp {
+public:
+  RGWOp_Metadata_Unlock() {}
+  ~RGWOp_Metadata_Unlock() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "unlock_metadata_object";
+  }
+};
+
+class RGWHandler_Metadata : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_put();
+  RGWOp *op_delete();
+  RGWOp *op_post();
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+public:
+  RGWHandler_Metadata() : RGWHandler_Auth_S3() {}
+  virtual ~RGWHandler_Metadata() {}
+};
+
+class RGWRESTMgr_Metadata : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Metadata() {}
+  virtual ~RGWRESTMgr_Metadata() {}
+
+  virtual RGWHandler *get_handler(struct req_state *s){
+    return new RGWHandler_Metadata;
+  }
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_opstate.cc b/ceph/src/rgw/rgw_rest_opstate.cc
new file mode 100644
index 00000000..2b215a71
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_opstate.cc
@@ -0,0 +1,189 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "common/ceph_json.h"
+#include "common/strtol.h"
+#include "rgw_rest.h"
+#include "rgw_op.h"
+#include "rgw_rest_s3.h"
+#include "rgw_rest_opstate.h"
+#include "rgw_client_io.h"
+#include "common/errno.h"
+
+#define OPSTATE_LIST_MAX_ENTRIES 1000
+#define dout_subsys ceph_subsys_rgw
+
+void RGWOp_Opstate_List::execute() {
+  string client_id = s->info.args.get("client-id"),
+         op_id = s->info.args.get("op-id"),
+         object = s->info.args.get("object"),
+         max_entries_str = s->info.args.get("max-entries");
+  unsigned max_entries = OPSTATE_LIST_MAX_ENTRIES;
+  
+  if (!max_entries_str.empty()) {
+    string err;
+    max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err);
+    if (!err.empty()) {
+      dout(5) << "Error parsing max-entries " << max_entries_str << dendl;
+      http_ret = -EINVAL;
+      return;
+    }
+  } 
+  
+  RGWOpState oc = RGWOpState(store);
+  void *handle;
+  bool done;
+  list entries;
+
+  oc.init_list_entries(client_id, op_id, object, &handle);
+
+  http_ret = 0;
+  send_response();
+  do {
+    int ret = oc.list_entries(handle, max_entries, entries, &done);
+
+    if (ret < 0) {
+      dout(5) << "oc.list_entries failed with error " << ret << dendl;
+      http_ret = ret;
+      oc.finish_list_entries(handle);
+      return;
+    }
+
+    if (!max_entries_str.empty()) 
+      max_entries -= entries.size();
+
+    send_response(entries);
+  } while (!done && max_entries > 0);
+
+  oc.finish_list_entries(handle);
+  send_response_end();
+}
+
+void RGWOp_Opstate_List::send_response() {
+  if (sent_header)
+    return;
+
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  sent_header = true;
+
+  if (http_ret < 0)
+    return;
+
+  s->formatter->open_array_section("entries");
+}
+
+void RGWOp_Opstate_List::send_response(list entries) {
+  RGWOpState oc(store);
+  for (list::iterator it = entries.begin();
+       it != entries.end(); ++it) {
+    oc.dump_entry(*it, s->formatter);
+    flusher.flush();
+  }
+}
+
+void RGWOp_Opstate_List::send_response_end() {
+  s->formatter->close_section();
+  flusher.flush();
+}
+
+void RGWOp_Opstate_Set::execute() {
+  string client_id = s->info.args.get("client-id"),
+         op_id = s->info.args.get("op-id"),
+         object = s->info.args.get("object"),
+         state_str = s->info.args.get("state");
+
+  if (client_id.empty() ||
+      op_id.empty() ||
+      object.empty() ||
+      state_str.empty()) {
+    dout(5) << "Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  RGWOpState oc(store);
+  RGWOpState::OpState state;
+
+  http_ret = oc.state_from_str(state_str, &state);
+  if (http_ret < 0) {
+    dout(5) << "Error - invalid state" << dendl;
+    return;
+  }
+
+  http_ret = oc.set_state(client_id, op_id, object, state);
+  if (http_ret < 0) {
+    dout(5) << "Error - Unable to set state" << dendl;
+    return;
+  }
+}
+
+void RGWOp_Opstate_Renew::execute() {
+  string client_id = s->info.args.get("client-id"),
+         op_id = s->info.args.get("op-id"),
+         object = s->info.args.get("object"),
+         state_str = s->info.args.get("state");
+
+  if (client_id.empty() ||
+      op_id.empty() ||
+      object.empty() ||
+      state_str.empty()) {
+    dout(5) << "Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  RGWOpState oc(store);
+  RGWOpState::OpState state;
+
+  http_ret = oc.state_from_str(state_str, &state);
+  if (http_ret < 0) {
+    dout(5) << "Error - invalid state" << dendl;
+    return;
+  }
+
+  http_ret = oc.renew_state(client_id, op_id, object, state);
+  if (http_ret < 0) {
+    dout(5) << "Error - Unable to renew state" << dendl;
+    return;
+  }
+}
+
+void RGWOp_Opstate_Delete::execute() {
+  string client_id = s->info.args.get("client-id"),
+         op_id = s->info.args.get("op-id"),
+         object = s->info.args.get("object");
+
+  if (client_id.empty() ||
+      op_id.empty() ||
+      object.empty()) {
+    dout(5) << "Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  
+  RGWOpState oc(store);
+
+  http_ret = oc.remove_entry(client_id, op_id, object);
+  if (http_ret < 0) {
+    dout(5) << "Error - Unable to remove entry" << dendl;
+  }
+}
+
+RGWOp *RGWHandler_Opstate::op_post() {
+  if (s->info.args.exists("renew")) {
+    return new RGWOp_Opstate_Renew;
+  } 
+  return new RGWOp_Opstate_Set;
+}
diff --git a/ceph/src/rgw/rgw_rest_opstate.h b/ceph/src/rgw/rgw_rest_opstate.h
new file mode 100644
index 00000000..de13dde6
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_opstate.h
@@ -0,0 +1,109 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#ifndef CEPH_RGW_REST_OPSTATE_H
+#define CEPH_RGW_REST_OPSTATE_H
+
+class RGWOp_Opstate_List : public RGWRESTOp {
+  bool sent_header;
+public:
+  RGWOp_Opstate_List() : sent_header(false) {}
+  ~RGWOp_Opstate_List() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("opstate", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual void send_response(list entries);
+  virtual void send_response_end();
+  virtual const string name() {
+    return "opstate_list";
+  }
+};
+
+class RGWOp_Opstate_Set : public RGWRESTOp {
+public:
+  RGWOp_Opstate_Set() {}
+  ~RGWOp_Opstate_Set() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("opstate", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "set_opstate";
+  }
+};
+
+class RGWOp_Opstate_Renew : public RGWRESTOp {
+public:
+  RGWOp_Opstate_Renew() {}
+  ~RGWOp_Opstate_Renew() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("opstate", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "renew_opstate";
+  }
+};
+
+class RGWOp_Opstate_Delete : public RGWRESTOp {
+public:
+  RGWOp_Opstate_Delete() {}
+  ~RGWOp_Opstate_Delete() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("opstate", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "delete_opstate";
+  }
+};
+
+class RGWHandler_Opstate : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get() {
+    return new RGWOp_Opstate_List;
+  }
+  RGWOp *op_delete() {
+    return new RGWOp_Opstate_Delete;
+  }
+  RGWOp *op_post();
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+public:
+  RGWHandler_Opstate() : RGWHandler_Auth_S3() {}
+  virtual ~RGWHandler_Opstate() {}
+};
+
+class RGWRESTMgr_Opstate : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Opstate() {}
+  virtual ~RGWRESTMgr_Opstate() {}
+
+  virtual RGWHandler *get_handler(struct req_state *s){
+    return new RGWHandler_Opstate;
+  }
+};
+
+#endif /*!CEPH_RGW_REST_OPSTATE_H*/
+
diff --git a/ceph/src/rgw/rgw_rest_replica_log.cc b/ceph/src/rgw/rgw_rest_replica_log.cc
new file mode 100644
index 00000000..e7dd962f
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_replica_log.cc
@@ -0,0 +1,301 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "common/ceph_json.h"
+#include "common/strtol.h"
+#include "rgw_rest.h"
+#include "rgw_op.h"
+#include "rgw_rest_s3.h"
+#include "rgw_replica_log.h"
+#include "rgw_metadata.h"
+#include "rgw_bucket.h"
+#include "rgw_rest_replica_log.h"
+#include "rgw_client_io.h"
+#include "common/errno.h"
+
+#define dout_subsys ceph_subsys_rgw
+#define REPLICA_INPUT_MAX_LEN (512*1024)
+
+static int parse_to_utime(string& in, utime_t& out) {
+  uint64_t sec = 0;
+  uint64_t nsec = 0;
+  int ret = utime_t::parse_date(in.c_str(), &sec, &nsec);
+  if (ret < 0)
+    return ret;
+
+  out = utime_t(sec, nsec);
+  return 0;
+}
+
+
+void RGWOp_OBJLog_SetBounds::execute() {
+  string id_str = s->info.args.get("id"),
+         marker = s->info.args.get("marker"),
+         time = s->info.args.get("time"),
+         daemon_id = s->info.args.get("daemon_id");
+
+  if (id_str.empty() ||
+      marker.empty() ||
+      time.empty() ||
+      daemon_id.empty()) {
+    dout(5) << "Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  
+  int shard;
+  string err;
+  utime_t ut;
+
+  shard = (int)strict_strtol(id_str.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing id parameter - " << id_str << ", err " << err << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  
+  if (parse_to_utime(time, ut) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  string pool;
+  RGWReplicaObjectLogger rl(store, pool, prefix);
+  bufferlist bl;
+  list markers;
+
+  if ((http_ret = rgw_rest_get_json_input(store->ctx(), s, markers, REPLICA_INPUT_MAX_LEN, NULL)) < 0) {
+    dout(5) << "Error - retrieving input data - " << http_ret << dendl;
+    return;
+  }
+
+  http_ret = rl.update_bound(shard, daemon_id, marker, ut, &markers);
+}
+
+void RGWOp_OBJLog_GetBounds::execute() {
+  string id = s->info.args.get("id");
+
+  if (id.empty()) {
+    dout(5) << " Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  int shard;
+  string err;
+
+  shard = (int)strict_strtol(id.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing id parameter - " << id << ", err " << err << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+ 
+  string pool;
+  RGWReplicaObjectLogger rl(store, pool, prefix);
+  http_ret = rl.get_bounds(shard, bounds);
+}
+
+void RGWOp_OBJLog_GetBounds::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (http_ret < 0)
+    return;
+
+  encode_json("bounds", bounds, s->formatter);
+  flusher.flush();
+}
+
+void RGWOp_OBJLog_DeleteBounds::execute() {
+  string id = s->info.args.get("id"),
+         daemon_id = s->info.args.get("daemon_id");
+
+  if (id.empty() ||
+      daemon_id.empty()) {
+    dout(5) << "Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  
+  int shard;
+  string err;
+
+  shard = (int)strict_strtol(id.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "Error parsing id parameter - " << id << ", err " << err << dendl;
+    http_ret = -EINVAL;
+  }
+  
+  string pool;
+  RGWReplicaObjectLogger rl(store, pool, prefix);
+  http_ret = rl.delete_bound(shard, daemon_id);
+}
+
+static int bucket_instance_to_bucket(RGWRados *store, string& bucket_instance, rgw_bucket& bucket) {
+  RGWBucketInfo bucket_info;
+  time_t mtime;
+  
+  int r = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, &mtime, NULL);
+  if (r < 0) {
+    dout(5) << "could not get bucket instance info for bucket=" << bucket_instance << ": " << cpp_strerror(r) << dendl;
+    if (r == -ENOENT)
+      return r;
+    return -EINVAL;
+  }
+
+  bucket = bucket_info.bucket;
+  return 0;
+}
+
+void RGWOp_BILog_SetBounds::execute() {
+  string bucket_instance = s->info.args.get("bucket-instance"),
+         marker = s->info.args.get("marker"),
+         time = s->info.args.get("time"),
+         daemon_id = s->info.args.get("daemon_id");
+
+  if (bucket_instance.empty() ||
+      marker.empty() ||
+      time.empty() ||
+      daemon_id.empty()) {
+    dout(5) << "Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  
+  utime_t ut;
+  
+  if (parse_to_utime(time, ut) < 0) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  rgw_bucket bucket;
+  if ((http_ret = bucket_instance_to_bucket(store, bucket_instance, bucket)) < 0) 
+    return;
+
+  RGWReplicaBucketLogger rl(store);
+  bufferlist bl;
+  list markers;
+
+  if ((http_ret = rgw_rest_get_json_input(store->ctx(), s, markers, REPLICA_INPUT_MAX_LEN, NULL)) < 0) {
+    dout(5) << "Error - retrieving input data - " << http_ret << dendl;
+    return;
+  }
+
+  http_ret = rl.update_bound(bucket, daemon_id, marker, ut, &markers);
+}
+
+void RGWOp_BILog_GetBounds::execute() {
+  string bucket_instance = s->info.args.get("bucket-instance");
+
+  if (bucket_instance.empty()) {
+    dout(5) << " Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  rgw_bucket bucket;
+  if ((http_ret = bucket_instance_to_bucket(store, bucket_instance, bucket)) < 0) 
+    return;
+
+  RGWReplicaBucketLogger rl(store);
+  http_ret = rl.get_bounds(bucket, bounds);
+}
+
+void RGWOp_BILog_GetBounds::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (http_ret < 0)
+    return;
+
+  encode_json("bounds", bounds, s->formatter);
+  flusher.flush();
+}
+
+void RGWOp_BILog_DeleteBounds::execute() {
+  string bucket_instance = s->info.args.get("bucket-instance"),
+         daemon_id = s->info.args.get("daemon_id");
+
+  if (bucket_instance.empty() ||
+      daemon_id.empty()) {
+    dout(5) << "Error - invalid parameter list" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+  
+  rgw_bucket bucket;
+  if ((http_ret = bucket_instance_to_bucket(store, bucket_instance, bucket)) < 0) 
+    return;
+  
+  RGWReplicaBucketLogger rl(store);
+  http_ret = rl.delete_bound(bucket, daemon_id);
+}
+
+RGWOp *RGWHandler_ReplicaLog::op_get() {
+  bool exists;
+  string type = s->info.args.get("type", &exists);
+
+  if (!exists) {
+    return NULL;
+  }
+
+  if (type.compare("metadata") == 0) {
+    return new RGWOp_OBJLog_GetBounds(META_REPLICA_LOG_OBJ_PREFIX, "mdlog");
+  } else if (type.compare("bucket-index") == 0) {
+    return new RGWOp_BILog_GetBounds;
+  } else if (type.compare("data") == 0) {
+    return new RGWOp_OBJLog_GetBounds(DATA_REPLICA_LOG_OBJ_PREFIX, "datalog");
+  }
+  return NULL;
+}
+
+RGWOp *RGWHandler_ReplicaLog::op_delete() {
+  bool exists;
+  string type = s->info.args.get("type", &exists);
+
+  if (!exists) {
+    return NULL;
+  }
+
+  if (type.compare("metadata") == 0)
+    return new RGWOp_OBJLog_DeleteBounds(META_REPLICA_LOG_OBJ_PREFIX, "mdlog");
+  else if (type.compare("bucket-index") == 0) 
+    return new RGWOp_BILog_DeleteBounds;
+  else if (type.compare("data") == 0)
+    return new RGWOp_OBJLog_DeleteBounds(DATA_REPLICA_LOG_OBJ_PREFIX, "datalog");
+  
+  return NULL;
+}
+
+RGWOp *RGWHandler_ReplicaLog::op_post() {
+  bool exists;
+  string type = s->info.args.get("type", &exists);
+
+  if (!exists) {
+    return NULL;
+  }
+
+  if (type.compare("metadata") == 0) {
+    return new RGWOp_OBJLog_SetBounds(META_REPLICA_LOG_OBJ_PREFIX, "mdlog");
+  } else if (type.compare("bucket-index") == 0) {
+    return new RGWOp_BILog_SetBounds;
+  } else if (type.compare("data") == 0) {
+    return new RGWOp_OBJLog_SetBounds(DATA_REPLICA_LOG_OBJ_PREFIX, "datalog");
+  }
+  return NULL;
+}
+
diff --git a/ceph/src/rgw/rgw_rest_replica_log.h b/ceph/src/rgw/rgw_rest_replica_log.h
new file mode 100644
index 00000000..c879150c
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_replica_log.h
@@ -0,0 +1,154 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#ifndef CEPH_RGW_REST_REPLICA_LOG_H
+#define CEPH_RGW_REST_REPLICA_LOG_H
+
+class RGWOp_OBJLog_GetBounds : public RGWRESTOp {
+  string prefix;
+  string obj_type;
+  RGWReplicaBounds bounds;
+
+public:
+  RGWOp_OBJLog_GetBounds(const char *_prefix, const char *type) 
+    : prefix(_prefix), obj_type(type){}
+  ~RGWOp_OBJLog_GetBounds() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap(obj_type.c_str(), RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    string s = "replica";
+    s.append(obj_type);
+    s.append("_getbounds");
+    return s;
+  }
+};
+
+class RGWOp_OBJLog_SetBounds : public RGWRESTOp {
+  string prefix;
+  string obj_type;
+public:
+  RGWOp_OBJLog_SetBounds(const char *_prefix, const char *type) 
+    : prefix(_prefix), obj_type(type){}
+  ~RGWOp_OBJLog_SetBounds() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap(obj_type.c_str(), RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    string s = "replica";
+    s.append(obj_type);
+    s.append("_updatebounds");
+    return s;
+  }
+};
+
+class RGWOp_OBJLog_DeleteBounds : public RGWRESTOp {
+  string prefix;
+  string obj_type;
+public:
+  RGWOp_OBJLog_DeleteBounds(const char *_prefix, const char *type) 
+    : prefix(_prefix), obj_type(type){}
+  ~RGWOp_OBJLog_DeleteBounds() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap(obj_type.c_str(), RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    string s = "replica";
+    s.append(obj_type);
+    s.append("_deletebound");
+    return s;
+  }
+};
+
+class RGWOp_BILog_GetBounds : public RGWRESTOp {
+  RGWReplicaBounds bounds;
+public:
+  RGWOp_BILog_GetBounds() {}
+  ~RGWOp_BILog_GetBounds() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("bilog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  void execute();
+  virtual void send_response();
+  virtual const string name() {
+    return "replicabilog_getbounds";
+  }
+};
+
+class RGWOp_BILog_SetBounds : public RGWRESTOp {
+public:
+  RGWOp_BILog_SetBounds() {}
+  ~RGWOp_BILog_SetBounds() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("bilog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "replicabilog_updatebounds";
+  }
+};
+
+class RGWOp_BILog_DeleteBounds : public RGWRESTOp {
+public:
+  RGWOp_BILog_DeleteBounds() {}
+  ~RGWOp_BILog_DeleteBounds() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("bilog", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const string name() {
+    return "replicabilog_deletebound";
+  }
+};
+
+class RGWHandler_ReplicaLog : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_delete();
+  RGWOp *op_post();
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+public:
+  RGWHandler_ReplicaLog() : RGWHandler_Auth_S3() {}
+  virtual ~RGWHandler_ReplicaLog() {}
+};
+
+class RGWRESTMgr_ReplicaLog : public RGWRESTMgr {
+public:
+  RGWRESTMgr_ReplicaLog() {}
+  virtual ~RGWRESTMgr_ReplicaLog() {}
+
+  virtual RGWHandler *get_handler(struct req_state *s){
+    return new RGWHandler_ReplicaLog;
+  }
+};
+
+#endif /*!CEPH_RGW_REST_REPLICA_LOG_H*/
diff --git a/ceph/src/rgw/rgw_rest_s3.cc b/ceph/src/rgw/rgw_rest_s3.cc
new file mode 100644
index 00000000..c7961f4a
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_s3.cc
@@ -0,0 +1,2237 @@
+#include 
+#include 
+
+#include "common/ceph_crypto.h"
+#include "common/Formatter.h"
+#include "common/utf8.h"
+#include "common/ceph_json.h"
+
+#include "rgw_rest.h"
+#include "rgw_rest_s3.h"
+#include "rgw_auth_s3.h"
+#include "rgw_acl.h"
+#include "rgw_policy_s3.h"
+#include "rgw_user.h"
+#include "rgw_cors.h"
+#include "rgw_cors_s3.h"
+
+#include "rgw_client_io.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace ceph::crypto;
+
+void list_all_buckets_start(struct req_state *s)
+{
+  s->formatter->open_array_section_in_ns("ListAllMyBucketsResult",
+			      "http://s3.amazonaws.com/doc/2006-03-01/");
+}
+
+void list_all_buckets_end(struct req_state *s)
+{
+  s->formatter->close_section();
+}
+
+void dump_bucket(struct req_state *s, RGWBucketEnt& obj)
+{
+  s->formatter->open_object_section("Bucket");
+  s->formatter->dump_string("Name", obj.bucket.name);
+  dump_time(s, "CreationDate", &obj.creation_time);
+  s->formatter->close_section();
+}
+
+void rgw_get_errno_s3(rgw_http_errors *e , int err_no)
+{
+  const struct rgw_http_errors *r;
+  r = search_err(err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS));
+
+  if (r) {
+    e->http_ret = r->http_ret;
+    e->s3_code = r->s3_code;
+  } else {
+    e->http_ret = 500;
+    e->s3_code = "UnknownError";
+  }
+}
+
+struct response_attr_param {
+  const char *param;
+  const char *http_attr;
+};
+
+static struct response_attr_param resp_attr_params[] = {
+  {"response-content-type", "Content-Type"},
+  {"response-content-language", "Content-Language"},
+  {"response-expires", "Expires"},
+  {"response-cache-control", "Cache-Control"},
+  {"response-content-disposition", "Content-Disposition"},
+  {"response-content-encoding", "Content-Encoding"},
+  {NULL, NULL},
+};
+
+int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len)
+{
+  const char *content_type = NULL;
+  string content_type_str;
+  map response_attrs;
+  map::iterator riter;
+  bufferlist metadata_bl;
+
+  if (ret)
+    goto done;
+
+  if (sent_header)
+    goto send_data;
+
+  if (range_str)
+    dump_range(s, start, end, s->obj_size);
+
+  if (s->system_request &&
+      s->info.args.exists(RGW_SYS_PARAM_PREFIX "prepend-metadata")) {
+
+    /* JSON encode object metadata */
+    JSONFormatter jf;
+    jf.open_object_section("obj_metadata");
+    encode_json("attrs", attrs, &jf);
+    encode_json("mtime", lastmod, &jf);
+    jf.close_section();
+    stringstream ss;
+    jf.flush(ss);
+    metadata_bl.append(ss.str());
+    s->cio->print("Rgwx-Embedded-Metadata-Len: %lld\r\n", (long long)metadata_bl.length());
+    total_len += metadata_bl.length();
+  }
+
+  if (s->system_request && lastmod) {
+    /* we end up dumping mtime in two different methods, a bit redundant */
+    dump_epoch_header(s, "Rgwx-Mtime", lastmod);
+  }
+
+  dump_content_length(s, total_len);
+  dump_last_modified(s, lastmod);
+
+  if (!ret) {
+    map::iterator iter = attrs.find(RGW_ATTR_ETAG);
+    if (iter != attrs.end()) {
+      bufferlist& bl = iter->second;
+      if (bl.length()) {
+        char *etag = bl.c_str();
+        dump_etag(s, etag);
+      }
+    }
+
+    for (struct response_attr_param *p = resp_attr_params; p->param; p++) {
+      bool exists;
+      string val = s->info.args.get(p->param, &exists);
+      if (exists) {
+	if (strcmp(p->param, "response-content-type") != 0) {
+	  response_attrs[p->http_attr] = val;
+	} else {
+	  content_type_str = val;
+	  content_type = content_type_str.c_str();
+	}
+      }
+    }
+
+    for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+      const char *name = iter->first.c_str();
+      map::iterator aiter = rgw_to_http_attrs.find(name);
+      if (aiter != rgw_to_http_attrs.end()) {
+	if (response_attrs.count(aiter->second) > 0) // was already overridden by a response param
+	  continue;
+
+	if (aiter->first.compare(RGW_ATTR_CONTENT_TYPE) == 0) { // special handling for content_type
+	  if (!content_type)
+	    content_type = iter->second.c_str();
+	  continue;
+        }
+	response_attrs[aiter->second] = iter->second.c_str();
+      } else {
+        if (strncmp(name, RGW_ATTR_META_PREFIX, sizeof(RGW_ATTR_META_PREFIX)-1) == 0) {
+          name += sizeof(RGW_ATTR_PREFIX) - 1;
+          s->cio->print("%s: %s\r\n", name, iter->second.c_str());
+        }
+      }
+    }
+  }
+
+done:
+  set_req_state_err(s, (partial_content && !ret) ? STATUS_PARTIAL_CONTENT : ret);
+
+  dump_errno(s);
+
+  for (riter = response_attrs.begin(); riter != response_attrs.end(); ++riter) {
+    s->cio->print("%s: %s\n", riter->first.c_str(), riter->second.c_str());
+  }
+
+  if (!content_type)
+    content_type = "binary/octet-stream";
+
+  end_header(s, this, content_type);
+
+  if (metadata_bl.length()) {
+    s->cio->write(metadata_bl.c_str(), metadata_bl.length());
+  }
+  sent_header = true;
+
+send_data:
+  if (get_data && !ret) {
+    int r = s->cio->write(bl.c_str() + bl_ofs, bl_len);
+    if (r < 0)
+      return r;
+  }
+
+  return 0;
+}
+
+void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets)
+{
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  dump_start(s);
+  end_header(s, NULL, "application/xml");
+
+  if (!ret) {
+    list_all_buckets_start(s);
+    dump_owner(s, s->user.user_id, s->user.display_name);
+    s->formatter->open_array_section("Buckets");
+    sent_data = true;
+  }
+}
+
+void RGWListBuckets_ObjStore_S3::send_response_data(RGWUserBuckets& buckets)
+{
+  if (!sent_data)
+    return;
+
+  map& m = buckets.get_buckets();
+  map::iterator iter;
+
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    RGWBucketEnt obj = iter->second;
+    dump_bucket(s, obj);
+  }
+  rgw_flush_formatter(s, s->formatter);
+}
+
+void RGWListBuckets_ObjStore_S3::send_response_end()
+{
+  if (sent_data) {
+    s->formatter->close_section();
+    list_all_buckets_end(s);
+    rgw_flush_formatter_and_reset(s, s->formatter);
+  }
+}
+
+int RGWListBucket_ObjStore_S3::get_params()
+{
+  prefix = s->info.args.get("prefix");
+  marker = s->info.args.get("marker");
+  max_keys = s->info.args.get("max-keys");
+  ret = parse_max_keys();
+  if (ret < 0) {
+    return ret;
+  }
+  delimiter = s->info.args.get("delimiter");
+  return 0;
+}
+
+void RGWListBucket_ObjStore_S3::send_response()
+{
+  if (ret < 0)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+
+  end_header(s, this, "application/xml");
+  dump_start(s);
+  if (ret < 0)
+    return;
+
+  s->formatter->open_object_section_in_ns("ListBucketResult",
+					  "http://s3.amazonaws.com/doc/2006-03-01/");
+  s->formatter->dump_string("Name", s->bucket_name_str);
+  s->formatter->dump_string("Prefix", prefix);
+  s->formatter->dump_string("Marker", marker);
+  if (is_truncated && !next_marker.empty())
+    s->formatter->dump_string("NextMarker", next_marker);
+  s->formatter->dump_int("MaxKeys", max);
+  if (!delimiter.empty())
+    s->formatter->dump_string("Delimiter", delimiter);
+
+  s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true" : "false"));
+
+  if (ret >= 0) {
+    vector::iterator iter;
+    for (iter = objs.begin(); iter != objs.end(); ++iter) {
+      s->formatter->open_array_section("Contents");
+      s->formatter->dump_string("Key", iter->name);
+      time_t mtime = iter->mtime.sec();
+      dump_time(s, "LastModified", &mtime);
+      s->formatter->dump_format("ETag", "\"%s\"", iter->etag.c_str());
+      s->formatter->dump_int("Size", iter->size);
+      s->formatter->dump_string("StorageClass", "STANDARD");
+      dump_owner(s, iter->owner, iter->owner_display_name);
+      s->formatter->close_section();
+    }
+    if (common_prefixes.size() > 0) {
+      map::iterator pref_iter;
+      for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) {
+        s->formatter->open_array_section("CommonPrefixes");
+        s->formatter->dump_string("Prefix", pref_iter->first);
+        s->formatter->close_section();
+      }
+    }
+  }
+  s->formatter->close_section();
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+void RGWGetBucketLogging_ObjStore_S3::send_response()
+{
+  dump_errno(s);
+  end_header(s, this, "application/xml");
+  dump_start(s);
+
+  s->formatter->open_object_section_in_ns("BucketLoggingStatus",
+					  "http://doc.s3.amazonaws.com/doc/2006-03-01/");
+  s->formatter->close_section();
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+static void dump_bucket_metadata(struct req_state *s, RGWBucketEnt& bucket)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%lld", (long long)bucket.count);
+  s->cio->print("X-RGW-Object-Count: %s\n", buf);
+  snprintf(buf, sizeof(buf), "%lld", (long long)bucket.size);
+  s->cio->print("X-RGW-Bytes-Used: %s\n", buf);
+}
+
+void RGWStatBucket_ObjStore_S3::send_response()
+{
+  if (ret >= 0) {
+    dump_bucket_metadata(s, bucket);
+  }
+
+  set_req_state_err(s, ret);
+  dump_errno(s);
+
+  end_header(s, this);
+  dump_start(s);
+}
+
+static int create_s3_policy(struct req_state *s, RGWRados *store, RGWAccessControlPolicy_S3& s3policy)
+{
+  if (s->has_acl_header) {
+    if (!s->canned_acl.empty())
+      return -ERR_INVALID_REQUEST;
+
+    return s3policy.create_from_headers(store, s->info.env, s->owner);
+  }
+
+  return s3policy.create_canned(s->owner, s->bucket_owner, s->canned_acl);
+}
+
+class RGWLocationConstraint : public XMLObj
+{
+public:
+  RGWLocationConstraint() {}
+  ~RGWLocationConstraint() {}
+  bool xml_end(const char *el) {
+    if (!el)
+      return false;
+
+    location_constraint = get_data();
+
+    return true;
+  }
+
+  string location_constraint;
+};
+
+class RGWCreateBucketConfig : public XMLObj
+{
+public:
+  RGWCreateBucketConfig() {}
+  ~RGWCreateBucketConfig() {}
+};
+
+class RGWCreateBucketParser : public RGWXMLParser
+{
+  XMLObj *alloc_obj(const char *el) {
+    return new XMLObj;
+  }
+
+public:
+  RGWCreateBucketParser() {}
+  ~RGWCreateBucketParser() {}
+
+  bool get_location_constraint(string& region) {
+    XMLObj *config = find_first("CreateBucketConfiguration");
+    if (!config)
+      return false;
+
+    XMLObj *constraint = config->find_first("LocationConstraint");
+    if (!constraint)
+      return false;
+
+    region = constraint->get_data();
+
+    return true;
+  }
+};
+
+int RGWCreateBucket_ObjStore_S3::get_params()
+{
+  RGWAccessControlPolicy_S3 s3policy(s->cct);
+
+  int r = create_s3_policy(s, store, s3policy);
+  if (r < 0)
+    return r;
+
+  policy = s3policy;
+
+  int len = 0;
+  char *data;
+#define CREATE_BUCKET_MAX_REQ_LEN (512 * 1024) /* this is way more than enough */
+  ret = rgw_rest_read_all_input(s, &data, &len, CREATE_BUCKET_MAX_REQ_LEN);
+  if ((ret < 0) && (ret != -ERR_LENGTH_REQUIRED))
+    return ret;
+
+  bufferptr in_ptr(data, len);
+  in_data.append(in_ptr);
+
+  if (len) {
+    RGWCreateBucketParser parser;
+
+    if (!parser.init()) {
+      ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
+      return -EIO;
+    }
+
+    bool success = parser.parse(data, len, 1);
+    ldout(s->cct, 20) << "create bucket input data=" << data << dendl;
+
+    if (!success) {
+      ldout(s->cct, 0) << "failed to parse input: " << data << dendl;
+      free(data);
+      return -EINVAL;
+    }
+    free(data);
+
+    if (!parser.get_location_constraint(location_constraint)) {
+      ldout(s->cct, 0) << "provided input did not specify location constraint correctly" << dendl;
+      return -EINVAL;
+    }
+
+    ldout(s->cct, 10) << "create bucket location constraint: " << location_constraint << dendl;
+  }
+
+  int pos = location_constraint.find(':');
+  if (pos >= 0) {
+    placement_rule = location_constraint.substr(pos + 1);
+    location_constraint = location_constraint.substr(0, pos);
+  }
+
+  return 0;
+}
+
+void RGWCreateBucket_ObjStore_S3::send_response()
+{
+  if (ret == -ERR_BUCKET_EXISTS)
+    ret = 0;
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (ret < 0)
+    return;
+
+  if (s->system_request) {
+    JSONFormatter f; /* use json formatter for system requests output */
+
+    f.open_object_section("info");
+    encode_json("entry_point_object_ver", ep_objv, &f);
+    encode_json("object_ver", info.objv_tracker.read_version, &f);
+    encode_json("bucket_info", info, &f);
+    f.close_section();
+    rgw_flush_formatter_and_reset(s, &f);
+  }
+}
+
+void RGWDeleteBucket_ObjStore_S3::send_response()
+{
+  int r = ret;
+  if (!r)
+    r = STATUS_NO_CONTENT;
+
+  set_req_state_err(s, r);
+  dump_errno(s);
+  end_header(s, this);
+
+  if (s->system_request) {
+    JSONFormatter f; /* use json formatter for system requests output */
+
+    f.open_object_section("info");
+    encode_json("object_ver", objv_tracker.read_version, &f);
+    f.close_section();
+    rgw_flush_formatter_and_reset(s, &f);
+  }
+}
+
+int RGWPutObj_ObjStore_S3::get_params()
+{
+  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  if (!s->length)
+    return -ERR_LENGTH_REQUIRED;
+
+  int r = create_s3_policy(s, store, s3policy);
+  if (r < 0)
+    return r;
+
+  policy = s3policy;
+
+  return RGWPutObj_ObjStore::get_params();
+}
+
+static int get_success_retcode(int code)
+{
+  switch (code) {
+    case 201:
+      return STATUS_CREATED;
+    case 204:
+      return STATUS_NO_CONTENT;
+  }
+  return 0;
+}
+
+void RGWPutObj_ObjStore_S3::send_response()
+{
+  if (ret) {
+    set_req_state_err(s, ret);
+  } else {
+    if (s->cct->_conf->rgw_s3_success_create_obj_status) {
+      ret = get_success_retcode(s->cct->_conf->rgw_s3_success_create_obj_status);
+      set_req_state_err(s, ret);
+    }
+    dump_etag(s, etag.c_str());
+    dump_content_length(s, 0);
+  }
+  if (s->system_request && mtime) {
+    dump_epoch_header(s, "Rgwx-Mtime", mtime);
+  }
+  dump_errno(s);
+  end_header(s, this);
+}
+
+/*
+ * parses params in the format: 'first; param1=foo; param2=bar'
+ */
+static void parse_params(const string& params_str, string& first, map& params)
+{
+  int pos = params_str.find(';');
+  if (pos < 0) {
+    first = rgw_trim_whitespace(params_str);
+    return;
+  }
+
+  first = rgw_trim_whitespace(params_str.substr(0, pos));
+
+  pos++;
+
+  while (pos < (int)params_str.size()) {
+    ssize_t end = params_str.find(';', pos);
+    if (end < 0)
+      end = params_str.size();
+
+    string param = params_str.substr(pos, end - pos);
+
+    int eqpos = param.find('=');
+    if (eqpos > 0) {
+      string param_name = rgw_trim_whitespace(param.substr(0, eqpos));
+      string val = rgw_trim_quotes(param.substr(eqpos + 1));
+      params[param_name] = val;
+    } else {
+      params[rgw_trim_whitespace(param)] = "";
+    }
+
+    pos = end + 1;
+  }
+}
+
+static int parse_part_field(const string& line, string& field_name, struct post_part_field& field)
+{
+  int pos = line.find(':');
+  if (pos < 0)
+    return -EINVAL;
+
+  field_name = line.substr(0, pos);
+  if (pos >= (int)line.size() - 1)
+    return 0;
+
+  parse_params(line.substr(pos + 1), field.val, field.params);
+
+  return 0;
+}
+
+bool is_crlf(const char *s)
+{
+  return (*s == '\r' && *(s + 1) == '\n');
+}
+
+/*
+ * find the index of the boundary, if exists, or optionally the next end of line
+ * also returns how many bytes to skip
+ */
+static int index_of(bufferlist& bl, int max_len, const string& str, bool check_crlf,
+                    bool *reached_boundary, int *skip)
+{
+  *reached_boundary = false;
+  *skip = 0;
+
+  if (str.size() < 2) // we assume boundary is at least 2 chars (makes it easier with crlf checks)
+    return -EINVAL;
+
+  if (bl.length() < str.size())
+    return -1;
+
+  const char *buf = bl.c_str();
+  const char *s = str.c_str();
+
+  if (max_len > (int)bl.length())
+    max_len = bl.length();
+
+  int i;
+  for (i = 0; i < max_len; i++, buf++) {
+    if (check_crlf &&
+        i >= 1 &&
+        is_crlf(buf - 1)) {
+        return i + 1; // skip the crlf
+    }
+    if ((i < max_len - (int)str.size() + 1) &&
+        (buf[0] == s[0] && buf[1] == s[1]) &&
+        (strncmp(buf, s, str.size()) == 0)) {
+      *reached_boundary = true;
+      *skip = str.size();
+
+      /* oh, great, now we need to swallow the preceding crlf
+       * if exists
+       */
+      if ((i >= 2) &&
+        is_crlf(buf - 2)) {
+        i -= 2;
+        *skip += 2;
+      }
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+int RGWPostObj_ObjStore_S3::read_with_boundary(bufferlist& bl, uint64_t max, bool check_crlf,
+                                               bool *reached_boundary, bool *done)
+{
+  uint64_t cl = max + 2 + boundary.size();
+
+  if (max > in_data.length()) {
+    uint64_t need_to_read = cl - in_data.length();
+
+    bufferptr bp(need_to_read);
+
+    int read_len;
+    s->cio->read(bp.c_str(), need_to_read, &read_len);
+
+    in_data.append(bp, 0, read_len);
+  }
+
+  *done = false;
+  int skip;
+  int index = index_of(in_data, cl, boundary, check_crlf, reached_boundary, &skip);
+  if (index >= 0)
+    max = index;
+
+  if (max > in_data.length())
+    max = in_data.length();
+
+  bl.substr_of(in_data, 0, max);
+
+  bufferlist new_read_data;
+
+  /*
+   * now we need to skip boundary for next time, also skip any crlf, or
+   * check to see if it's the last final boundary (marked with "--" at the end
+   */
+  if (*reached_boundary) {
+    int left = in_data.length() - max;
+    if (left < skip + 2) {
+      int need = skip + 2 - left;
+      bufferptr boundary_bp(need);
+      int actual;
+      s->cio->read(boundary_bp.c_str(), need, &actual);
+      in_data.append(boundary_bp);
+    }
+    max += skip; // skip boundary for next time
+    if (in_data.length() >= max + 2) {
+      const char *data = in_data.c_str();
+      if (is_crlf(data + max)) {
+	max += 2;
+      } else {
+        if (*(data + max) == '-' &&
+            *(data + max + 1) == '-') {
+          *done = true;
+	  max += 2;
+	}
+      }
+    }
+  }
+
+  new_read_data.substr_of(in_data, max, in_data.length() - max);
+  in_data = new_read_data;
+
+  return 0;
+}
+
+int RGWPostObj_ObjStore_S3::read_line(bufferlist& bl, uint64_t max,
+				  bool *reached_boundary, bool *done)
+{
+  return read_with_boundary(bl, max, true, reached_boundary, done);
+}
+
+int RGWPostObj_ObjStore_S3::read_data(bufferlist& bl, uint64_t max,
+				  bool *reached_boundary, bool *done)
+{
+  return read_with_boundary(bl, max, false, reached_boundary, done);
+}
+
+
+int RGWPostObj_ObjStore_S3::read_form_part_header(struct post_form_part *part,
+                                              bool *done)
+{
+  bufferlist bl;
+  bool reached_boundary;
+  uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
+  int r = read_line(bl, chunk_size, &reached_boundary, done);
+  if (r < 0)
+    return r;
+
+  if (*done) {
+    return 0;
+  }
+
+  if (reached_boundary) { // skip the first boundary
+    r = read_line(bl, chunk_size, &reached_boundary, done);
+    if (r < 0)
+      return r;
+    if (*done)
+      return 0;
+  }
+
+  while (true) {
+  /*
+   * iterate through fields
+   */
+    string line = rgw_trim_whitespace(string(bl.c_str(), bl.length()));
+
+    if (line.empty())
+      break;
+
+    struct post_part_field field;
+
+    string field_name;
+    r = parse_part_field(line, field_name, field);
+    if (r < 0)
+      return r;
+
+    part->fields[field_name] = field;
+
+    if (stringcasecmp(field_name, "Content-Disposition") == 0) {
+      part->name = field.params["name"];
+    }
+
+    if (reached_boundary)
+      break;
+
+    r = read_line(bl, chunk_size, &reached_boundary, done);
+  }
+
+  return 0;
+}
+
+bool RGWPostObj_ObjStore_S3::part_str(const string& name, string *val)
+{
+  map::iterator iter = parts.find(name);
+  if (iter == parts.end())
+    return false;
+
+  bufferlist& data = iter->second.data;
+  string str = string(data.c_str(), data.length());
+  *val = rgw_trim_whitespace(str);
+  return true;
+}
+
+bool RGWPostObj_ObjStore_S3::part_bl(const string& name, bufferlist *pbl)
+{
+  map::iterator iter = parts.find(name);
+  if (iter == parts.end())
+    return false;
+
+  *pbl = iter->second.data;
+  return true;
+}
+
+void RGWPostObj_ObjStore_S3::rebuild_key(string& key)
+{
+  static string var = "${filename}";
+  int pos = key.find(var);
+  if (pos < 0)
+    return;
+
+  string new_key = key.substr(0, pos);
+  new_key.append(filename);
+  new_key.append(key.substr(pos + var.size()));
+
+  key = new_key;
+}
+
+int RGWPostObj_ObjStore_S3::get_params()
+{
+  // get the part boundary
+  string req_content_type_str = s->info.env->get("CONTENT_TYPE", "");
+  string req_content_type;
+  map params;
+
+  if (s->expect_cont) {
+    /* ok, here it really gets ugly. With POST, the params are embedded in the
+     * request body, so we need to continue before being able to actually look
+     * at them. This diverts from the usual request flow.
+     */
+    dump_continue(s);
+    s->expect_cont = false;
+  }
+
+  parse_params(req_content_type_str, req_content_type, params);
+
+  if (req_content_type.compare("multipart/form-data") != 0) {
+    err_msg = "Request Content-Type is not multipart/form-data";
+    return -EINVAL;
+  }
+
+  if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
+    ldout(s->cct, 20) << "request content_type_str=" << req_content_type_str << dendl;
+    ldout(s->cct, 20) << "request content_type params:" << dendl;
+    map::iterator iter;
+    for (iter = params.begin(); iter != params.end(); ++iter) {
+      ldout(s->cct, 20) << " " << iter->first << " -> " << iter->second << dendl;
+    }
+  }
+
+  ldout(s->cct, 20) << "adding bucket to policy env: " << s->bucket.name << dendl;
+  env.add_var("bucket", s->bucket.name);
+
+  map::iterator iter = params.find("boundary");
+  if (iter == params.end()) {
+    err_msg = "Missing multipart boundary specification";
+    return -EINVAL;
+  }
+
+  // create the boundary
+  boundary = "--";
+  boundary.append(iter->second);
+
+  bool done;
+  do {
+    struct post_form_part part;
+    int r = read_form_part_header(&part, &done);
+    if (r < 0)
+      return r;
+    
+    if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
+      map::iterator piter;
+      for (piter = part.fields.begin(); piter != part.fields.end(); ++piter) {
+        ldout(s->cct, 20) << "read part header: name=" << part.name << " content_type=" << part.content_type << dendl;
+        ldout(s->cct, 20) << "name=" << piter->first << dendl;
+        ldout(s->cct, 20) << "val=" << piter->second.val << dendl;
+        ldout(s->cct, 20) << "params:" << dendl;
+        map& params = piter->second.params;
+        for (iter = params.begin(); iter != params.end(); ++iter) {
+          ldout(s->cct, 20) << " " << iter->first << " -> " << iter->second << dendl;
+        }
+      }
+    }
+
+    if (done) { /* unexpected here */
+      err_msg = "Malformed request";
+      return -EINVAL;
+    }
+
+    if (stringcasecmp(part.name, "file") == 0) { /* beginning of data transfer */
+      struct post_part_field& field = part.fields["Content-Disposition"];
+      map::iterator iter = field.params.find("filename");
+      if (iter != field.params.end()) {
+        filename = iter->second;
+      }
+      parts[part.name] = part;
+      data_pending = true;
+      break;
+    }
+
+    bool boundary;
+    uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
+    r = read_data(part.data, chunk_size, &boundary, &done);
+    if (!boundary) {
+      err_msg = "Couldn't find boundary";
+      return -EINVAL;
+    }
+    parts[part.name] = part;
+    string part_str(part.data.c_str(), part.data.length());
+    env.add_var(part.name, part_str);
+  } while (!done);
+
+  if (!part_str("key", &s->object_str)) {
+    err_msg = "Key not specified";
+    return -EINVAL;
+  }
+
+  rebuild_key(s->object_str);
+
+  if (s->object_str.empty()) {
+    err_msg = "Empty object name";
+    return -EINVAL;
+  }
+
+  env.add_var("key", s->object_str);
+
+  part_str("Content-Type", &content_type);
+  env.add_var("Content-Type", content_type);
+
+  map::iterator piter = parts.upper_bound(RGW_AMZ_META_PREFIX);
+  for (; piter != parts.end(); ++piter) {
+    string n = piter->first;
+    if (strncasecmp(n.c_str(), RGW_AMZ_META_PREFIX, sizeof(RGW_AMZ_META_PREFIX) - 1) != 0)
+      break;
+
+    string attr_name = RGW_ATTR_PREFIX;
+    attr_name.append(n);
+
+    /* need to null terminate it */
+    bufferlist& data = piter->second.data;
+    string str = string(data.c_str(), data.length());
+
+    bufferlist attr_bl;
+    attr_bl.append(str.c_str(), str.size() + 1);
+
+    attrs[attr_name] = attr_bl;
+  }
+
+  int r = get_policy();
+  if (r < 0)
+    return r;
+
+  min_len = post_policy.min_length;
+  max_len = post_policy.max_length;
+
+  return 0;
+}
+
+int RGWPostObj_ObjStore_S3::get_policy()
+{
+  bufferlist encoded_policy;
+
+  if (part_bl("policy", &encoded_policy)) {
+
+    // check that the signature matches the encoded policy
+    string s3_access_key;
+    if (!part_str("AWSAccessKeyId", &s3_access_key)) {
+      ldout(s->cct, 0) << "No S3 access key found!" << dendl;
+      err_msg = "Missing access key";
+      return -EINVAL;
+    }
+    string received_signature_str;
+    if (!part_str("signature", &received_signature_str)) {
+      ldout(s->cct, 0) << "No signature found!" << dendl;
+      err_msg = "Missing signature";
+      return -EINVAL;
+    }
+
+    RGWUserInfo user_info;
+
+    ret = rgw_get_user_info_by_access_key(store, s3_access_key, user_info);
+    if (ret < 0) {
+      ldout(s->cct, 0) << "User lookup failed!" << dendl;
+      err_msg = "Bad access key / signature";
+      return -EACCES;
+    }
+
+    map access_keys  = user_info.access_keys;
+
+    map::const_iterator iter = access_keys.find(s3_access_key);
+    // We know the key must exist, since the user was returned by
+    // rgw_get_user_info_by_access_key, but it doesn't hurt to check!
+    if (iter == access_keys.end()) {
+      ldout(s->cct, 0) << "Secret key lookup failed!" << dendl;
+      err_msg = "No secret key for matching access key";
+      return -EACCES;
+    }
+    string s3_secret_key = (iter->second).key;
+
+    char expected_signature_char[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+
+    calc_hmac_sha1(s3_secret_key.c_str(), s3_secret_key.size(), encoded_policy.c_str(), encoded_policy.length(), expected_signature_char);
+    bufferlist expected_signature_hmac_raw;
+    bufferlist expected_signature_hmac_encoded;
+    expected_signature_hmac_raw.append(expected_signature_char, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+    expected_signature_hmac_raw.encode_base64(expected_signature_hmac_encoded);
+    expected_signature_hmac_encoded.append((char)0); /* null terminate */
+
+    if (received_signature_str.compare(expected_signature_hmac_encoded.c_str()) != 0) {
+      ldout(s->cct, 0) << "Signature verification failed!" << dendl;
+      ldout(s->cct, 0) << "received: " << received_signature_str.c_str() << dendl;
+      ldout(s->cct, 0) << "expected: " << expected_signature_hmac_encoded.c_str() << dendl;
+      err_msg = "Bad access key / signature";
+      return -EACCES;
+    }
+    ldout(s->cct, 0) << "Successful Signature Verification!" << dendl;
+    bufferlist decoded_policy;
+    try {
+      decoded_policy.decode_base64(encoded_policy);
+    } catch (buffer::error& err) {
+      ldout(s->cct, 0) << "failed to decode_base64 policy" << dendl;
+      err_msg = "Could not decode policy";
+      return -EINVAL;
+    }
+
+    decoded_policy.append('\0'); // NULL terminate
+
+    ldout(s->cct, 0) << "POST policy: " << decoded_policy.c_str() << dendl;
+
+    int r = post_policy.from_json(decoded_policy, err_msg);
+    if (r < 0) {
+      if (err_msg.empty()) {
+	err_msg = "Failed to parse policy";
+      }
+      ldout(s->cct, 0) << "failed to parse policy" << dendl;
+      return -EINVAL;
+    }
+
+    post_policy.set_var_checked("AWSAccessKeyId");
+    post_policy.set_var_checked("policy");
+    post_policy.set_var_checked("signature");
+
+    r = post_policy.check(&env, err_msg);
+    if (r < 0) {
+      if (err_msg.empty()) {
+	err_msg = "Policy check failed";
+      }
+      ldout(s->cct, 0) << "policy check failed" << dendl;
+      return r;
+    }
+
+    s->user = user_info;
+    s->owner.set_id(user_info.user_id);
+    s->owner.set_name(user_info.display_name);
+  } else {
+    ldout(s->cct, 0) << "No attached policy found!" << dendl;
+  }
+
+  string canned_acl;
+  part_str("acl", &canned_acl);
+
+  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  ldout(s->cct, 20) << "canned_acl=" << canned_acl << dendl;
+  if (s3policy.create_canned(s->owner, s->bucket_owner, canned_acl) < 0) {
+    err_msg = "Bad canned ACLs";
+    return -EINVAL;
+  }
+
+  policy = s3policy;
+
+  return 0;
+}
+
+int RGWPostObj_ObjStore_S3::complete_get_params()
+{
+  bool done;
+  do {
+    struct post_form_part part;
+    int r = read_form_part_header(&part, &done);
+    if (r < 0)
+      return r;
+    
+    bufferlist part_data;
+    bool boundary;
+    uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
+    r = read_data(part.data, chunk_size, &boundary, &done);
+    if (!boundary) {
+      return -EINVAL;
+    }
+
+    parts[part.name] = part;
+  } while (!done);
+
+  return 0;
+}
+
+int RGWPostObj_ObjStore_S3::get_data(bufferlist& bl)
+{
+  bool boundary;
+  bool done;
+
+  uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
+  int r = read_data(bl, chunk_size, &boundary, &done);
+  if (r < 0)
+    return r;
+
+  if (boundary) {
+    data_pending = false;
+
+    if (!done) {  /* reached end of data, let's drain the rest of the params */
+      r = complete_get_params();
+      if (r < 0)
+        return r;
+    }
+  }
+
+  return bl.length();
+}
+
+void RGWPostObj_ObjStore_S3::send_response()
+{
+  if (ret == 0 && parts.count("success_action_redirect")) {
+    string redirect;
+
+    part_str("success_action_redirect", &redirect);
+
+    string bucket;
+    string key;
+    string etag_str = "\"";
+
+    etag_str.append(etag);
+    etag_str.append("\"");
+
+    string etag_url;
+
+    url_encode(s->bucket_name_str, bucket);
+    url_encode(s->object_str, key);
+    url_encode(etag_str, etag_url);
+
+    redirect.append("?bucket=");
+    redirect.append(bucket);
+    redirect.append("&key=");
+    redirect.append(key);
+    redirect.append("&etag=");
+    redirect.append(etag_url);
+
+    int r = check_utf8(redirect.c_str(), redirect.size());
+    if (r < 0) {
+      ret = r;
+      goto done;
+    }
+    dump_redirect(s, redirect);
+    ret = STATUS_REDIRECT;
+  } else if (ret == 0 && parts.count("success_action_status")) {
+    string status_string;
+    uint32_t status_int;
+
+    part_str("success_action_status", &status_string);
+
+    int r = stringtoul(status_string, &status_int);
+    if (r < 0) {
+      ret = r;
+      goto done;
+    }
+
+    switch (status_int) {
+      case 200:
+	break;
+      case 201:
+	ret = STATUS_CREATED;
+	break;
+      default:
+	ret = STATUS_NO_CONTENT;
+	break;
+    }
+  } else if (!ret) {
+    ret = STATUS_NO_CONTENT;
+  }
+
+done:
+  if (ret == STATUS_CREATED) {
+    s->formatter->open_object_section("PostResponse");
+    if (g_conf->rgw_dns_name.length())
+      s->formatter->dump_format("Location", "%s/%s", s->info.script_uri.c_str(), s->object_str.c_str());
+    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    s->formatter->dump_string("Key", s->object_str);
+    s->formatter->close_section();
+  }
+  s->err.message = err_msg;
+  set_req_state_err(s, ret);
+  dump_errno(s);
+  if (ret >= 0) {
+    dump_content_length(s, s->formatter->get_len());
+  }
+  end_header(s, this);
+  if (ret != STATUS_CREATED)
+    return;
+
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+
+void RGWDeleteObj_ObjStore_S3::send_response()
+{
+  int r = ret;
+  if (r == -ENOENT)
+    r = 0;
+  if (!r)
+    r = STATUS_NO_CONTENT;
+
+  set_req_state_err(s, r);
+  dump_errno(s);
+  end_header(s, this);
+}
+
+int RGWCopyObj_ObjStore_S3::init_dest_policy()
+{
+  RGWAccessControlPolicy_S3 s3policy(s->cct);
+
+  /* build a policy for the target object */
+  int r = create_s3_policy(s, store, s3policy);
+  if (r < 0)
+    return r;
+
+  dest_policy = s3policy;
+
+  return 0;
+}
+
+int RGWCopyObj_ObjStore_S3::get_params()
+{
+  if_mod = s->info.env->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE");
+  if_unmod = s->info.env->get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE");
+  if_match = s->info.env->get("HTTP_X_AMZ_COPY_IF_MATCH");
+  if_nomatch = s->info.env->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");
+
+  src_bucket_name = s->src_bucket_name;
+  src_object = s->src_object;
+  dest_bucket_name = s->bucket.name;
+  dest_object = s->object_str;
+
+  if (s->system_request) {
+    source_zone = s->info.args.get(RGW_SYS_PARAM_PREFIX "source-zone");
+    if (!source_zone.empty()) {
+      client_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "client-id");
+      op_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "op-id");
+
+      if (client_id.empty() || op_id.empty()) {
+        ldout(s->cct, 0) << RGW_SYS_PARAM_PREFIX "client-id or " RGW_SYS_PARAM_PREFIX "op-id were not provided, required for intra-region copy" << dendl;
+        return -EINVAL;
+      }
+    }
+  }
+
+  const char *md_directive = s->info.env->get("HTTP_X_AMZ_METADATA_DIRECTIVE");
+  if (md_directive) {
+    if (strcasecmp(md_directive, "COPY") == 0) {
+      replace_attrs = false;
+    } else if (strcasecmp(md_directive, "REPLACE") == 0) {
+      replace_attrs = true;
+    } else if (!source_zone.empty()) {
+      replace_attrs = false; // default for intra-region copy
+    } else {
+      ldout(s->cct, 0) << "invalid metadata directive" << dendl;
+      return -EINVAL;
+    }
+  }
+
+  if (source_zone.empty() &&
+      (dest_bucket_name.compare(src_bucket_name) == 0) &&
+      (dest_object.compare(src_object) == 0) &&
+      !replace_attrs) {
+    /* can only copy object into itself if replacing attrs */
+    ldout(s->cct, 0) << "can't copy object into itself if not replacing attrs" << dendl;
+    return -ERR_INVALID_REQUEST;
+  }
+  return 0;
+}
+
+void RGWCopyObj_ObjStore_S3::send_partial_response(off_t ofs)
+{
+  if (!sent_header) {
+    if (ret)
+    set_req_state_err(s, ret);
+    dump_errno(s);
+
+    end_header(s, this, "binary/octet-stream");
+    if (ret == 0) {
+      s->formatter->open_object_section("CopyObjectResult");
+    }
+    sent_header = true;
+  } else {
+    /* Send progress field. Note that this diverge from the original S3
+     * spec. We do this in order to keep connection alive.
+     */
+    s->formatter->dump_int("Progress", (uint64_t)ofs);
+  }
+  rgw_flush_formatter(s, s->formatter);
+}
+
+void RGWCopyObj_ObjStore_S3::send_response()
+{
+  if (!sent_header)
+    send_partial_response(0);
+
+  if (ret == 0) {
+    dump_time(s, "LastModified", &mtime);
+    map::iterator iter = attrs.find(RGW_ATTR_ETAG);
+    if (iter != attrs.end()) {
+      bufferlist& bl = iter->second;
+      if (bl.length()) {
+        char *etag = bl.c_str();
+        s->formatter->dump_string("ETag", etag);
+      }
+    }
+    s->formatter->close_section();
+    rgw_flush_formatter_and_reset(s, s->formatter);
+  }
+}
+
+void RGWGetACLs_ObjStore_S3::send_response()
+{
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this, "application/xml");
+  dump_start(s);
+  s->cio->write(acls.c_str(), acls.size());
+}
+
+int RGWPutACLs_ObjStore_S3::get_policy_from_state(RGWRados *store, struct req_state *s, stringstream& ss)
+{
+  RGWAccessControlPolicy_S3 s3policy(s->cct);
+
+  // bucket-* canned acls do not apply to bucket
+  if (s->object_str.empty()) {
+    if (s->canned_acl.find("bucket") != string::npos)
+      s->canned_acl.clear();
+  }
+
+  int r = create_s3_policy(s, store, s3policy);
+  if (r < 0)
+    return r;
+
+  s3policy.to_xml(ss);
+
+  return 0;
+}
+
+void RGWPutACLs_ObjStore_S3::send_response()
+{
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this, "application/xml");
+  dump_start(s);
+}
+
+void RGWGetCORS_ObjStore_S3::send_response()
+{
+  if (ret) {
+    if (ret == -ENOENT) 
+      set_req_state_err(s, ERR_NOT_FOUND);
+    else 
+      set_req_state_err(s, ret);
+  }
+  dump_errno(s);
+  end_header(s, NULL, "application/xml");
+  dump_start(s);
+  if (!ret) {
+    string cors;
+    RGWCORSConfiguration_S3 *s3cors = static_cast(&bucket_cors);
+    stringstream ss;
+
+    s3cors->to_xml(ss);
+    cors = ss.str();
+    s->cio->write(cors.c_str(), cors.size());
+  }
+}
+
+int RGWPutCORS_ObjStore_S3::get_params()
+{
+  int r;
+  char *data = NULL;
+  int len = 0;
+  size_t cl = 0;
+  RGWCORSXMLParser_S3 parser(s->cct);
+  RGWCORSConfiguration_S3 *cors_config;
+
+  if (s->length)
+    cl = atoll(s->length);
+  if (cl) {
+    data = (char *)malloc(cl + 1);
+    if (!data) {
+       r = -ENOMEM;
+       goto done_err;
+    }
+    int read_len;
+    r = s->cio->read(data, cl, &read_len);
+    len = read_len;
+    if (r < 0)
+      goto done_err;
+    data[len] = '\0';
+  } else {
+    len = 0;
+  }
+
+  if (!parser.init()) {
+    r = -EINVAL;
+    goto done_err;
+  }
+
+  if (!data || !parser.parse(data, len, 1)) {
+    r = -EINVAL;
+    goto done_err;
+  }
+  cors_config = static_cast(parser.find_first("CORSConfiguration"));
+  if (!cors_config) {
+    r = -EINVAL;
+    goto done_err;
+  }
+
+  if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
+    ldout(s->cct, 15) << "CORSConfiguration";
+    cors_config->to_xml(*_dout);
+    *_dout << dendl;
+  }
+
+  cors_config->encode(cors_bl);
+
+  free(data);
+  return 0;
+done_err:
+  free(data);
+  return r;
+}
+
+void RGWPutCORS_ObjStore_S3::send_response()
+{
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, NULL, "application/xml");
+  dump_start(s);
+}
+
+void RGWDeleteCORS_ObjStore_S3::send_response()
+{
+  int r = ret;
+  if (!r || r == -ENOENT)
+    r = STATUS_NO_CONTENT;
+
+  set_req_state_err(s, r);
+  dump_errno(s);
+  end_header(s, NULL);
+}
+
+void RGWOptionsCORS_ObjStore_S3::send_response()
+{
+  string hdrs, exp_hdrs;
+  uint32_t max_age = CORS_MAX_AGE_INVALID;
+  /*EACCES means, there is no CORS registered yet for the bucket
+   *ENOENT means, there is no match of the Origin in the list of CORSRule
+   */
+  if (ret == -ENOENT)
+    ret = -EACCES;
+  if (ret < 0) {
+    set_req_state_err(s, ret);
+    dump_errno(s);
+    end_header(s, NULL);
+    return;
+  }
+  get_response_params(hdrs, exp_hdrs, &max_age);
+
+  dump_errno(s);
+  dump_access_control(s, origin, req_meth, hdrs.c_str(), exp_hdrs.c_str(), max_age); 
+  end_header(s, NULL);
+}
+
+int RGWInitMultipart_ObjStore_S3::get_params()
+{
+  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  ret = create_s3_policy(s, store, s3policy);
+  if (ret < 0)
+    return ret;
+
+  policy = s3policy;
+
+  return 0;
+}
+
+void RGWInitMultipart_ObjStore_S3::send_response()
+{
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this, "application/xml");
+  if (ret == 0) { 
+    dump_start(s);
+    s->formatter->open_object_section_in_ns("InitiateMultipartUploadResult",
+		  "http://s3.amazonaws.com/doc/2006-03-01/");
+    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    s->formatter->dump_string("Key", s->object);
+    s->formatter->dump_string("UploadId", upload_id);
+    s->formatter->close_section();
+    rgw_flush_formatter_and_reset(s, s->formatter);
+  }
+}
+
+void RGWCompleteMultipart_ObjStore_S3::send_response()
+{
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this, "application/xml");
+  if (ret == 0) { 
+    dump_start(s);
+    s->formatter->open_object_section_in_ns("CompleteMultipartUploadResult",
+			  "http://s3.amazonaws.com/doc/2006-03-01/");
+    if (g_conf->rgw_dns_name.length())
+      s->formatter->dump_format("Location", "%s.%s", s->bucket_name_str.c_str(), g_conf->rgw_dns_name.c_str());
+    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    s->formatter->dump_string("Key", s->object);
+    s->formatter->dump_string("ETag", etag);
+    s->formatter->close_section();
+    rgw_flush_formatter_and_reset(s, s->formatter);
+  }
+}
+
+void RGWAbortMultipart_ObjStore_S3::send_response()
+{
+  int r = ret;
+  if (!r)
+    r = STATUS_NO_CONTENT;
+
+  set_req_state_err(s, r);
+  dump_errno(s);
+  end_header(s, this);
+}
+
+void RGWListMultipart_ObjStore_S3::send_response()
+{
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this, "application/xml");
+
+  if (ret == 0) { 
+    dump_start(s);
+    s->formatter->open_object_section_in_ns("ListMultipartUploadResult",
+		    "http://s3.amazonaws.com/doc/2006-03-01/");
+    map::iterator iter;
+    map::reverse_iterator test_iter;
+    int cur_max = 0;
+
+    iter = parts.begin();
+    test_iter = parts.rbegin();
+    if (test_iter != parts.rend()) {
+      cur_max = test_iter->first;
+    }
+    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    s->formatter->dump_string("Key", s->object);
+    s->formatter->dump_string("UploadId", upload_id);
+    s->formatter->dump_string("StorageClass", "STANDARD");
+    s->formatter->dump_int("PartNumberMarker", marker);
+    s->formatter->dump_int("NextPartNumberMarker", cur_max);
+    s->formatter->dump_int("MaxParts", max_parts);
+    s->formatter->dump_string("IsTruncated", (truncated ? "true" : "false"));
+
+    ACLOwner& owner = policy.get_owner();
+    dump_owner(s, owner.get_id(), owner.get_display_name());
+
+    for (; iter != parts.end(); ++iter) {
+      RGWUploadPartInfo& info = iter->second;
+
+      time_t sec = info.modified.sec();
+      struct tm tmp;
+      gmtime_r(&sec, &tmp);
+      char buf[TIME_BUF_SIZE];
+
+      s->formatter->open_object_section("Part");
+
+      if (strftime(buf, sizeof(buf), "%Y-%m-%dT%T.000Z", &tmp) > 0) {
+        s->formatter->dump_string("LastModified", buf);
+      }
+
+      s->formatter->dump_unsigned("PartNumber", info.num);
+      s->formatter->dump_string("ETag", info.etag);
+      s->formatter->dump_unsigned("Size", info.size);
+      s->formatter->close_section();
+    }
+    s->formatter->close_section();
+    rgw_flush_formatter_and_reset(s, s->formatter);
+  }
+}
+
+void RGWListBucketMultiparts_ObjStore_S3::send_response()
+{
+  if (ret < 0)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+
+  end_header(s, this, "application/xml");
+  dump_start(s);
+  if (ret < 0)
+    return;
+
+  s->formatter->open_object_section("ListMultipartUploadsResult");
+  s->formatter->dump_string("Bucket", s->bucket_name_str);
+  if (!prefix.empty())
+    s->formatter->dump_string("ListMultipartUploadsResult.Prefix", prefix);
+  string& key_marker = marker.get_key();
+  if (!key_marker.empty())
+    s->formatter->dump_string("KeyMarker", key_marker);
+  string& upload_id_marker = marker.get_upload_id();
+  if (!upload_id_marker.empty())
+    s->formatter->dump_string("UploadIdMarker", upload_id_marker);
+  string next_key = next_marker.mp.get_key();
+  if (!next_key.empty())
+    s->formatter->dump_string("NextKeyMarker", next_key);
+  string next_upload_id = next_marker.mp.get_upload_id();
+  if (!next_upload_id.empty())
+    s->formatter->dump_string("NextUploadIdMarker", next_upload_id);
+  s->formatter->dump_int("MaxUploads", max_uploads);
+  if (!delimiter.empty())
+    s->formatter->dump_string("Delimiter", delimiter);
+  s->formatter->dump_string("IsTruncated", (is_truncated ? "true" : "false"));
+
+  if (ret >= 0) {
+    vector::iterator iter;
+    for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
+      RGWMPObj& mp = iter->mp;
+      s->formatter->open_array_section("Upload");
+      s->formatter->dump_string("Key", mp.get_key());
+      s->formatter->dump_string("UploadId", mp.get_upload_id());
+      dump_owner(s, s->user.user_id, s->user.display_name, "Initiator");
+      dump_owner(s, s->user.user_id, s->user.display_name);
+      s->formatter->dump_string("StorageClass", "STANDARD");
+      time_t mtime = iter->obj.mtime.sec();
+      dump_time(s, "Initiated", &mtime);
+      s->formatter->close_section();
+    }
+    if (common_prefixes.size() > 0) {
+      s->formatter->open_array_section("CommonPrefixes");
+      map::iterator pref_iter;
+      for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) {
+        s->formatter->dump_string("CommonPrefixes.Prefix", pref_iter->first);
+      }
+      s->formatter->close_section();
+    }
+  }
+  s->formatter->close_section();
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+void RGWDeleteMultiObj_ObjStore_S3::send_status()
+{
+  if (!status_dumped) {
+    if (ret < 0)
+      set_req_state_err(s, ret);
+    dump_errno(s);
+    status_dumped = true;
+  }
+}
+
+void RGWDeleteMultiObj_ObjStore_S3::begin_response()
+{
+
+  if (!status_dumped) {
+    send_status();
+  }
+
+  dump_start(s);
+  end_header(s, this, "application/xml");
+  s->formatter->open_object_section_in_ns("DeleteResult",
+                                            "http://s3.amazonaws.com/doc/2006-03-01/");
+
+  rgw_flush_formatter(s, s->formatter);
+}
+
+void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(pair& result)
+{
+  if (!result.first.empty()) {
+    if (result.second == 0 && !quiet) {
+      s->formatter->open_object_section("Deleted");
+      s->formatter->dump_string("Key", result.first);
+      s->formatter->close_section();
+    } else if (result.second < 0) {
+      struct rgw_http_errors r;
+      int err_no;
+
+      s->formatter->open_object_section("Error");
+
+      err_no = -(result.second);
+      rgw_get_errno_s3(&r, err_no);
+
+      s->formatter->dump_string("Key", result.first);
+      s->formatter->dump_int("Code", r.http_ret);
+      s->formatter->dump_string("Message", r.s3_code);
+      s->formatter->close_section();
+    }
+
+    rgw_flush_formatter(s, s->formatter);
+  }
+}
+
+void RGWDeleteMultiObj_ObjStore_S3::end_response()
+{
+
+  s->formatter->close_section();
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+RGWOp *RGWHandler_ObjStore_Service_S3::op_get()
+{
+  return new RGWListBuckets_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Service_S3::op_head()
+{
+  return new RGWListBuckets_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_S3::get_obj_op(bool get_data)
+{
+  if (get_data)
+    return new RGWListBucket_ObjStore_S3;
+  else
+    return new RGWStatBucket_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_S3::op_get()
+{
+  if (s->info.args.sub_resource_exists("logging"))
+    return new RGWGetBucketLogging_ObjStore_S3;
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_S3;
+  } else if (is_cors_op()) {
+    return new RGWGetCORS_ObjStore_S3;
+  } else if (s->info.args.exists("uploads")) {
+    return new RGWListBucketMultiparts_ObjStore_S3;
+  }
+  return get_obj_op(true);
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_S3::op_head()
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_S3;
+  } else if (s->info.args.exists("uploads")) {
+    return new RGWListBucketMultiparts_ObjStore_S3;
+  }
+  return get_obj_op(false);
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_S3::op_put()
+{
+  if (s->info.args.sub_resource_exists("logging"))
+    return NULL;
+  if (is_acl_op()) {
+    return new RGWPutACLs_ObjStore_S3;
+  } else if (is_cors_op()) {
+    return new RGWPutCORS_ObjStore_S3;
+  } 
+  return new RGWCreateBucket_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_S3::op_delete()
+{
+  if (is_cors_op()) {
+    return new RGWDeleteCORS_ObjStore_S3;
+  }
+  return new RGWDeleteBucket_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_S3::op_post()
+{
+  if ( s->info.request_params == "delete" ) {
+    return new RGWDeleteMultiObj_ObjStore_S3;
+  }
+
+  return new RGWPostObj_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_S3::op_options()
+{
+  return new RGWOptionsCORS_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_S3::get_obj_op(bool get_data)
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_S3;
+  }
+  RGWGetObj_ObjStore_S3 *get_obj_op = new RGWGetObj_ObjStore_S3;
+  get_obj_op->set_get_data(get_data);
+  return get_obj_op;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_S3::op_get()
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_S3;
+  } else if (s->info.args.exists("uploadId")) {
+    return new RGWListMultipart_ObjStore_S3;
+  }
+  return get_obj_op(true);
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_S3::op_head()
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_S3;
+  } else if (s->info.args.exists("uploadId")) {
+    return new RGWListMultipart_ObjStore_S3;
+  }
+  return get_obj_op(false);
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_S3::op_put()
+{
+  if (is_acl_op()) {
+    return new RGWPutACLs_ObjStore_S3;
+  }
+  if (!s->copy_source)
+    return new RGWPutObj_ObjStore_S3;
+  else
+    return new RGWCopyObj_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_S3::op_delete()
+{
+  string upload_id = s->info.args.get("uploadId");
+
+  if (upload_id.empty())
+    return new RGWDeleteObj_ObjStore_S3;
+  else
+    return new RGWAbortMultipart_ObjStore_S3;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_S3::op_post()
+{
+  if (s->info.args.exists("uploadId"))
+    return new RGWCompleteMultipart_ObjStore_S3;
+
+  if (s->info.args.exists("uploads"))
+    return new RGWInitMultipart_ObjStore_S3;
+
+  return NULL;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_S3::op_options()
+{
+  return new RGWOptionsCORS_ObjStore_S3;
+}
+
+int RGWHandler_ObjStore_S3::init_from_header(struct req_state *s, int default_formatter, bool configurable_format)
+{
+  string req;
+  string first;
+
+  const char *req_name = s->relative_uri.c_str();
+  const char *p;
+
+  if (*req_name == '?') {
+    p = req_name;
+  } else {
+    p = s->info.request_params.c_str();
+  }
+
+  s->info.args.set(p);
+  s->info.args.parse();
+
+  /* must be called after the args parsing */
+  int ret = allocate_formatter(s, default_formatter, configurable_format);
+  if (ret < 0)
+    return ret;
+
+  if (*req_name != '/')
+    return 0;
+
+  req_name++;
+
+  if (!*req_name)
+    return 0;
+
+  req = req_name;
+  int pos = req.find('/');
+  if (pos >= 0) {
+    first = req.substr(0, pos);
+  } else {
+    first = req;
+  }
+
+  if (s->bucket_name_str.empty()) {
+    s->bucket_name_str = first;
+
+    if (pos >= 0) {
+      string encoded_obj_str = req.substr(pos+1);
+      s->object_str = encoded_obj_str;
+
+      if (s->object_str.size() > 0) {
+        s->object = strdup(s->object_str.c_str());
+      }
+    }
+  } else {
+    s->object_str = req_name;
+    s->object = strdup(s->object_str.c_str());
+  }
+  return 0;
+}
+
+static bool looks_like_ip_address(const char *bucket)
+{
+  int num_periods = 0;
+  bool expect_period = false;
+  for (const char *b = bucket; *b; ++b) {
+    if (*b == '.') {
+      if (!expect_period)
+	return false;
+      ++num_periods;
+      if (num_periods > 3)
+	return false;
+      expect_period = false;
+    }
+    else if (isdigit(*b)) {
+      expect_period = true;
+    }
+    else {
+      return false;
+    }
+  }
+  return (num_periods == 3);
+}
+
+int RGWHandler_ObjStore_S3::validate_bucket_name(const string& bucket, bool relaxed_names)
+{
+  int ret = RGWHandler_ObjStore::validate_bucket_name(bucket);
+  if (ret < 0)
+    return ret;
+
+  if (bucket.size() == 0)
+    return 0;
+
+  // bucket names must start with a number, letter, or underscore
+  if (!(isalpha(bucket[0]) || isdigit(bucket[0]))) {
+    if (!relaxed_names)
+      return -ERR_INVALID_BUCKET_NAME;
+    else if (!(bucket[0] == '_' || bucket[0] == '.' || bucket[0] == '-'))
+      return -ERR_INVALID_BUCKET_NAME;
+  } 
+
+  for (const char *s = bucket.c_str(); *s; ++s) {
+    char c = *s;
+    if (isdigit(c) || (c == '.'))
+      continue;
+    if (isalpha(c))
+      continue;
+    if ((c == '-') || (c == '_'))
+      continue;
+    // Invalid character
+    return -ERR_INVALID_BUCKET_NAME;
+  }
+
+  if (looks_like_ip_address(bucket.c_str()))
+    return -ERR_INVALID_BUCKET_NAME;
+
+  return 0;
+}
+
+int RGWHandler_ObjStore_S3::init(RGWRados *store, struct req_state *s, RGWClientIO *cio)
+{
+  dout(10) << "s->object=" << (s->object ? s->object : "") << " s->bucket=" << (!s->bucket_name_str.empty() ? s->bucket_name_str : "") << dendl;
+
+  bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
+  int ret = validate_bucket_name(s->bucket_name_str, relaxed_names);
+  if (ret)
+    return ret;
+  ret = validate_object_name(s->object_str);
+  if (ret)
+    return ret;
+
+  const char *cacl = s->info.env->get("HTTP_X_AMZ_ACL");
+  if (cacl)
+    s->canned_acl = cacl;
+
+  s->has_acl_header = s->info.env->exists_prefix("HTTP_X_AMZ_GRANT");
+
+  s->copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
+  if (s->copy_source) {
+    ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object);
+    if (!ret) {
+      ldout(s->cct, 0) << "failed to parse copy location" << dendl;
+      return -EINVAL;
+    }
+  }
+
+  s->dialect = "s3";
+
+  return RGWHandler_ObjStore::init(store, s, cio);
+}
+
+
+/*
+ * Try to validate S3 auth against keystone s3token interface
+ */
+int RGW_Auth_S3_Keystone_ValidateToken::validate_s3token(const string& auth_id, const string& auth_token, const string& auth_sign) {
+  /* prepare keystone url */
+  string keystone_url = cct->_conf->rgw_keystone_url;
+  if (keystone_url[keystone_url.size() - 1] != '/')
+    keystone_url.append("/");
+  keystone_url.append("v2.0/s3tokens");
+
+  /* set required headers for keystone request */
+  append_header("X-Auth-Token", cct->_conf->rgw_keystone_admin_token);
+  append_header("Content-Type", "application/json");
+
+  /* encode token */
+  bufferlist token_buff;
+  bufferlist token_encoded;
+  token_buff.append(auth_token);
+  token_buff.encode_base64(token_encoded);
+  token_encoded.append((char)0);
+
+  /* create json credentials request body */
+  JSONFormatter credentials(false);
+  credentials.open_object_section("");
+  credentials.open_object_section("credentials");
+  credentials.dump_string("access", auth_id);
+  credentials.dump_string("token", token_encoded.c_str());
+  credentials.dump_string("signature", auth_sign);
+  credentials.close_section();
+  credentials.close_section();
+
+  std::stringstream os;
+  credentials.flush(os);
+  set_tx_buffer(os.str());
+
+  /* send request */
+  int ret = process("POST", keystone_url.c_str());
+  if (ret < 0) {
+    dout(2) << "s3 keystone: token validation ERROR: " << rx_buffer.c_str() << dendl;
+    return -EPERM;
+  }
+
+  /* now parse response */
+  if (response.parse(cct, rx_buffer) < 0) {
+    dout(2) << "s3 keystone: token parsing failed" << dendl;
+    return -EPERM;
+  }
+
+  /* check if we have a valid role */
+  bool found = false;
+  list::iterator iter;
+  for (iter = roles_list.begin(); iter != roles_list.end(); ++iter) {
+    if ((found=response.user.has_role(*iter))==true)
+      break;
+  }
+
+  if (!found) {
+    ldout(cct, 5) << "s3 keystone: user does not hold a matching role; required roles: " << cct->_conf->rgw_keystone_accepted_roles << dendl;
+    return -EPERM;
+  }
+
+  /* everything seems fine, continue with this user */
+  ldout(cct, 5) << "s3 keystone: validated token: " << response.token.tenant.name << ":" << response.user.name << " expires: " << response.token.expires << dendl;
+  return 0;
+}
+
+/*
+ * verify that a signed request comes from the keyholder
+ * by checking the signature against our locally-computed version
+ */
+int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s)
+{
+  bool qsr = false;
+  string auth_id;
+  string auth_sign;
+
+  time_t now;
+  time(&now);
+
+  /* neither keystone and rados enabled; warn and exit! */
+  if (!store->ctx()->_conf->rgw_s3_auth_use_rados
+      && !store->ctx()->_conf->rgw_s3_auth_use_keystone) {
+    dout(0) << "WARNING: no authorization backend enabled! Users will never authenticate." << dendl;
+    return -EPERM;
+  }
+
+  if (!s->http_auth || !(*s->http_auth)) {
+    auth_id = s->info.args.get("AWSAccessKeyId");
+    if (auth_id.size()) {
+      auth_sign = s->info.args.get("Signature");
+
+      string date = s->info.args.get("Expires");
+      time_t exp = atoll(date.c_str());
+      if (now >= exp)
+        return -EPERM;
+
+      qsr = true;
+    } else {
+      /* anonymous access */
+      rgw_get_anon_user(s->user);
+      s->perm_mask = RGW_PERM_FULL_CONTROL;
+      return 0;
+    }
+  } else {
+    if (strncmp(s->http_auth, "AWS ", 4))
+      return -EINVAL;
+    string auth_str(s->http_auth + 4);
+    int pos = auth_str.find(':');
+    if (pos < 0)
+      return -EINVAL;
+
+    auth_id = auth_str.substr(0, pos);
+    auth_sign = auth_str.substr(pos + 1);
+  }
+
+  /* try keystone auth first */
+  int keystone_result = -EINVAL;
+  if (store->ctx()->_conf->rgw_s3_auth_use_keystone
+      && !store->ctx()->_conf->rgw_keystone_url.empty()) {
+    dout(20) << "s3 keystone: trying keystone auth" << dendl;
+
+    RGW_Auth_S3_Keystone_ValidateToken keystone_validator(store->ctx());
+    string token;
+
+    if (!rgw_create_s3_canonical_header(s->info, &s->header_time, token, qsr)) {
+        dout(10) << "failed to create auth header\n" << token << dendl;
+    } else {
+      keystone_result = keystone_validator.validate_s3token(auth_id, token, auth_sign);
+      if (keystone_result == 0) {
+        s->user.user_id = keystone_validator.response.token.tenant.id;
+        s->user.display_name = keystone_validator.response.token.tenant.name; // wow.
+
+        /* try to store user if it not already exists */
+        if (rgw_get_user_info_by_uid(store, keystone_validator.response.token.tenant.id, s->user) < 0) {
+          int ret = rgw_store_user_info(store, s->user, NULL, NULL, 0, true);
+          if (ret < 0)
+            dout(10) << "NOTICE: failed to store new user's info: ret=" << ret << dendl;
+        }
+
+        s->perm_mask = RGW_PERM_FULL_CONTROL;
+      }
+    }
+  }
+
+  /* keystone failed (or not enabled); check if we want to use rados backend */
+  if (!store->ctx()->_conf->rgw_s3_auth_use_rados
+      && keystone_result < 0)
+    return keystone_result;
+
+  /* now try rados backend, but only if keystone did not succeed */
+  if (keystone_result < 0) {
+    /* get the user info */
+    if (rgw_get_user_info_by_access_key(store, auth_id, s->user) < 0) {
+      dout(5) << "error reading user info, uid=" << auth_id << " can't authenticate" << dendl;
+      return -EPERM;
+    }
+
+    /* now verify signature */
+
+    string auth_hdr;
+    if (!rgw_create_s3_canonical_header(s->info, &s->header_time, auth_hdr, qsr)) {
+      dout(10) << "failed to create auth header\n" << auth_hdr << dendl;
+      return -EPERM;
+    }
+    dout(10) << "auth_hdr:\n" << auth_hdr << dendl;
+
+    time_t req_sec = s->header_time.sec();
+    if ((req_sec < now - RGW_AUTH_GRACE_MINS * 60 ||
+        req_sec > now + RGW_AUTH_GRACE_MINS * 60) && !qsr) {
+      dout(10) << "req_sec=" << req_sec << " now=" << now << "; now - RGW_AUTH_GRACE_MINS=" << now - RGW_AUTH_GRACE_MINS * 60 << "; now + RGW_AUTH_GRACE_MINS=" << now + RGW_AUTH_GRACE_MINS * 60 << dendl;
+      dout(0) << "NOTICE: request time skew too big now=" << utime_t(now, 0) << " req_time=" << s->header_time << dendl;
+      return -ERR_REQUEST_TIME_SKEWED;
+    }
+
+    map::iterator iter = s->user.access_keys.find(auth_id);
+    if (iter == s->user.access_keys.end()) {
+      dout(0) << "ERROR: access key not encoded in user info" << dendl;
+      return -EPERM;
+    }
+    RGWAccessKey& k = iter->second;
+
+    if (!k.subuser.empty()) {
+      map::iterator uiter = s->user.subusers.find(k.subuser);
+      if (uiter == s->user.subusers.end()) {
+        dout(0) << "NOTICE: could not find subuser: " << k.subuser << dendl;
+        return -EPERM;
+      }
+      RGWSubUser& subuser = uiter->second;
+      s->perm_mask = subuser.perm_mask;
+    } else
+      s->perm_mask = RGW_PERM_FULL_CONTROL;
+
+    string digest;
+    int ret = rgw_get_s3_header_digest(auth_hdr, k.key, digest);
+    if (ret < 0) {
+      return -EPERM;
+    }
+
+    dout(15) << "calculated digest=" << digest << dendl;
+    dout(15) << "auth_sign=" << auth_sign << dendl;
+    dout(15) << "compare=" << auth_sign.compare(digest) << dendl;
+
+    if (auth_sign != digest)
+      return -EPERM;
+
+    if (s->user.system) {
+      s->system_request = true;
+      dout(20) << "system request" << dendl;
+      s->info.args.set_system();
+      string effective_uid = s->info.args.get(RGW_SYS_PARAM_PREFIX "uid");
+      RGWUserInfo effective_user;
+      if (!effective_uid.empty()) {
+        ret = rgw_get_user_info_by_uid(store, effective_uid, effective_user);
+        if (ret < 0) {
+          ldout(s->cct, 0) << "User lookup failed!" << dendl;
+          return -ENOENT;
+        }
+        s->user = effective_user;
+      }
+    }
+
+  } /* if keystone_result < 0 */
+
+  // populate the owner info
+  s->owner.set_id(s->user.user_id);
+  s->owner.set_name(s->user.display_name);
+
+
+  return  0;
+}
+
+int RGWHandler_Auth_S3::init(RGWRados *store, struct req_state *state, RGWClientIO *cio)
+{
+  int ret = RGWHandler_ObjStore_S3::init_from_header(state, RGW_FORMAT_JSON, true);
+  if (ret < 0)
+    return ret;
+
+  return RGWHandler_ObjStore::init(store, state, cio);
+}
+
+RGWHandler *RGWRESTMgr_S3::get_handler(struct req_state *s)
+{
+  int ret = RGWHandler_ObjStore_S3::init_from_header(s, RGW_FORMAT_XML, false);
+  if (ret < 0)
+    return NULL;
+
+  if (s->bucket_name_str.empty())
+    return new RGWHandler_ObjStore_Service_S3;
+
+  if (!s->object)
+    return new RGWHandler_ObjStore_Bucket_S3;
+
+  return new RGWHandler_ObjStore_Obj_S3;
+}
diff --git a/ceph/src/rgw/rgw_rest_s3.h b/ceph/src/rgw/rgw_rest_s3.h
new file mode 100644
index 00000000..e62334b9
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_s3.h
@@ -0,0 +1,421 @@
+#ifndef CEPH_RGW_REST_S3_H
+#define CEPH_RGW_REST_S3_H
+#define TIME_BUF_SIZE 128
+
+#include "rgw_op.h"
+#include "rgw_http_errors.h"
+#include "rgw_acl_s3.h"
+#include "rgw_policy_s3.h"
+#include "rgw_keystone.h"
+
+#define RGW_AUTH_GRACE_MINS 15
+
+void rgw_get_errno_s3(struct rgw_http_errors *e, int err_no);
+
+class RGWGetObj_ObjStore_S3 : public RGWGetObj_ObjStore
+{
+public:
+  RGWGetObj_ObjStore_S3() {}
+  ~RGWGetObj_ObjStore_S3() {}
+
+  int send_response_data(bufferlist& bl, off_t ofs, off_t len);
+};
+
+class RGWListBuckets_ObjStore_S3 : public RGWListBuckets_ObjStore {
+public:
+  RGWListBuckets_ObjStore_S3() {}
+  ~RGWListBuckets_ObjStore_S3() {}
+
+  int get_params() {
+    limit = 0; /* no limit */
+    return 0;
+  }
+  virtual void send_response_begin(bool has_buckets);
+  virtual void send_response_data(RGWUserBuckets& buckets);
+  virtual void send_response_end();
+};
+
+class RGWListBucket_ObjStore_S3 : public RGWListBucket_ObjStore {
+public:
+  RGWListBucket_ObjStore_S3() {
+    default_max = 1000;
+  }
+  ~RGWListBucket_ObjStore_S3() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWGetBucketLogging_ObjStore_S3 : public RGWGetBucketLogging {
+public:
+  RGWGetBucketLogging_ObjStore_S3() {}
+  ~RGWGetBucketLogging_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWStatBucket_ObjStore_S3 : public RGWStatBucket_ObjStore {
+public:
+  RGWStatBucket_ObjStore_S3() {}
+  ~RGWStatBucket_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWCreateBucket_ObjStore_S3 : public RGWCreateBucket_ObjStore {
+public:
+  RGWCreateBucket_ObjStore_S3() {}
+  ~RGWCreateBucket_ObjStore_S3() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWDeleteBucket_ObjStore_S3 : public RGWDeleteBucket_ObjStore {
+public:
+  RGWDeleteBucket_ObjStore_S3() {}
+  ~RGWDeleteBucket_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWPutObj_ObjStore_S3 : public RGWPutObj_ObjStore {
+public:
+  RGWPutObj_ObjStore_S3() {}
+  ~RGWPutObj_ObjStore_S3() {}
+
+  int get_params();
+  void send_response();
+};
+
+struct post_part_field {
+  string val;
+  map params;
+};
+
+struct post_form_part {
+  string name;
+  string content_type;
+  map fields;
+  bufferlist data;
+};
+
+class RGWPostObj_ObjStore_S3 : public RGWPostObj_ObjStore {
+  string boundary;
+  string filename;
+  bufferlist in_data;
+  map parts;  
+  RGWPolicyEnv env;
+  RGWPolicy post_policy;
+  string err_msg;
+
+  int read_with_boundary(bufferlist& bl, uint64_t max, bool check_eol,
+                         bool *reached_boundary,
+			 bool *done);
+
+  int read_line(bufferlist& bl, uint64_t max,
+                bool *reached_boundary, bool *done);
+
+  int read_data(bufferlist& bl, uint64_t max, bool *reached_boundary, bool *done);
+
+  int read_form_part_header(struct post_form_part *part,
+                            bool *done);
+  bool part_str(const string& name, string *val);
+  bool part_bl(const string& name, bufferlist *pbl);
+
+  int get_policy();
+  void rebuild_key(string& key);
+public:
+  RGWPostObj_ObjStore_S3() {}
+  ~RGWPostObj_ObjStore_S3() {}
+
+  int get_params();
+  int complete_get_params();
+  void send_response();
+  int get_data(bufferlist& bl);
+};
+
+class RGWDeleteObj_ObjStore_S3 : public RGWDeleteObj_ObjStore {
+public:
+  RGWDeleteObj_ObjStore_S3() {}
+  ~RGWDeleteObj_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWCopyObj_ObjStore_S3 : public RGWCopyObj_ObjStore {
+  bool sent_header;
+public:
+  RGWCopyObj_ObjStore_S3() : sent_header(false) {}
+  ~RGWCopyObj_ObjStore_S3() {}
+
+  int init_dest_policy();
+  int get_params();
+  void send_partial_response(off_t ofs);
+  void send_response();
+};
+
+class RGWGetACLs_ObjStore_S3 : public RGWGetACLs_ObjStore {
+public:
+  RGWGetACLs_ObjStore_S3() {}
+  ~RGWGetACLs_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWPutACLs_ObjStore_S3 : public RGWPutACLs_ObjStore {
+public:
+  RGWPutACLs_ObjStore_S3() {}
+  ~RGWPutACLs_ObjStore_S3() {}
+
+  int get_policy_from_state(RGWRados *store, struct req_state *s, stringstream& ss);
+  void send_response();
+};
+
+class RGWGetCORS_ObjStore_S3 : public RGWGetCORS_ObjStore {
+public:
+  RGWGetCORS_ObjStore_S3() {}
+  ~RGWGetCORS_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWPutCORS_ObjStore_S3 : public RGWPutCORS_ObjStore {
+public:
+  RGWPutCORS_ObjStore_S3() {}
+  ~RGWPutCORS_ObjStore_S3() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWDeleteCORS_ObjStore_S3 : public RGWDeleteCORS_ObjStore {
+public:
+  RGWDeleteCORS_ObjStore_S3() {}
+  ~RGWDeleteCORS_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWOptionsCORS_ObjStore_S3 : public RGWOptionsCORS_ObjStore {
+public:
+  RGWOptionsCORS_ObjStore_S3() {}
+  ~RGWOptionsCORS_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWInitMultipart_ObjStore_S3 : public RGWInitMultipart_ObjStore {
+public:
+  RGWInitMultipart_ObjStore_S3() {}
+  ~RGWInitMultipart_ObjStore_S3() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWCompleteMultipart_ObjStore_S3 : public RGWCompleteMultipart_ObjStore {
+public:
+  RGWCompleteMultipart_ObjStore_S3() {}
+  ~RGWCompleteMultipart_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWAbortMultipart_ObjStore_S3 : public RGWAbortMultipart_ObjStore {
+public:
+  RGWAbortMultipart_ObjStore_S3() {}
+  ~RGWAbortMultipart_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWListMultipart_ObjStore_S3 : public RGWListMultipart_ObjStore {
+public:
+  RGWListMultipart_ObjStore_S3() {}
+  ~RGWListMultipart_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWListBucketMultiparts_ObjStore_S3 : public RGWListBucketMultiparts_ObjStore {
+public:
+  RGWListBucketMultiparts_ObjStore_S3() {
+    default_max = 1000;
+  }
+  ~RGWListBucketMultiparts_ObjStore_S3() {}
+
+  void send_response();
+};
+
+class RGWDeleteMultiObj_ObjStore_S3 : public RGWDeleteMultiObj_ObjStore {
+public:
+  RGWDeleteMultiObj_ObjStore_S3() {}
+  ~RGWDeleteMultiObj_ObjStore_S3() {}
+
+  void send_status();
+  void begin_response();
+  void send_partial_response(pair& result);
+  void end_response();
+};
+
+class RGW_Auth_S3_Keystone_ValidateToken : public RGWHTTPClient {
+private:
+  bufferlist rx_buffer;
+  bufferlist tx_buffer;
+  bufferlist::iterator tx_buffer_it;
+  list roles_list;
+
+public:
+  KeystoneToken response;
+
+private:
+  void set_tx_buffer(const string& d) {
+    tx_buffer.clear();
+    tx_buffer.append(d);
+    tx_buffer_it = tx_buffer.begin();
+    set_send_length(tx_buffer.length());
+  }
+
+public:
+  RGW_Auth_S3_Keystone_ValidateToken(CephContext *_cct)
+      : RGWHTTPClient(_cct) {
+    get_str_list(cct->_conf->rgw_keystone_accepted_roles, roles_list);
+  }
+
+  int receive_header(void *ptr, size_t len) {
+    return 0;
+  }
+  int receive_data(void *ptr, size_t len) {
+    rx_buffer.append((char *)ptr, len);
+    return 0;
+  }
+
+  int send_data(void *ptr, size_t len) {
+    if (!tx_buffer_it.get_remaining())
+      return 0; // nothing left to send
+
+    int l = MIN(tx_buffer_it.get_remaining(), len);
+    memcpy(ptr, tx_buffer_it.get_current_ptr().c_str(), l);
+    try {
+      tx_buffer_it.advance(l);
+    } catch (buffer::end_of_buffer &e) {
+      assert(0);
+    }
+
+    return l;
+  }
+
+  int validate_s3token(const string& auth_id, const string& auth_token, const string& auth_sign);
+
+};
+
+class RGW_Auth_S3 {
+public:
+  static int authorize(RGWRados *store, struct req_state *s);
+};
+
+class RGWHandler_Auth_S3 : public RGWHandler_ObjStore {
+  friend class RGWRESTMgr_S3;
+public:
+  RGWHandler_Auth_S3() : RGWHandler_ObjStore() {}
+  virtual ~RGWHandler_Auth_S3() {}
+
+  virtual int validate_bucket_name(const string& bucket) {
+    return 0;
+  }
+
+  virtual int validate_object_name(const string& bucket) { return 0; }
+
+  virtual int init(RGWRados *store, struct req_state *state, RGWClientIO *cio);
+  virtual int authorize() {
+    return RGW_Auth_S3::authorize(store, s);
+  }
+};
+
+class RGWHandler_ObjStore_S3 : public RGWHandler_ObjStore {
+  friend class RGWRESTMgr_S3;
+public:
+  static int init_from_header(struct req_state *s, int default_formatter, bool configurable_format);
+
+  RGWHandler_ObjStore_S3() : RGWHandler_ObjStore() {}
+  virtual ~RGWHandler_ObjStore_S3() {}
+
+  int validate_bucket_name(const string& bucket, bool relaxed_names);
+
+  virtual int init(RGWRados *store, struct req_state *state, RGWClientIO *cio);
+  virtual int authorize() {
+    return RGW_Auth_S3::authorize(store, s);
+  }
+};
+
+class RGWHandler_ObjStore_Service_S3 : public RGWHandler_ObjStore_S3 {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_head();
+public:
+  RGWHandler_ObjStore_Service_S3() {}
+  virtual ~RGWHandler_ObjStore_Service_S3() {}
+};
+
+class RGWHandler_ObjStore_Bucket_S3 : public RGWHandler_ObjStore_S3 {
+protected:
+  bool is_acl_op() {
+    return s->info.args.exists("acl");
+  }
+  bool is_cors_op() {
+      return s->info.args.exists("cors");
+  }
+  bool is_obj_update_op() {
+    return is_acl_op() || is_cors_op();
+  }
+  RGWOp *get_obj_op(bool get_data);
+
+  RGWOp *op_get();
+  RGWOp *op_head();
+  RGWOp *op_put();
+  RGWOp *op_delete();
+  RGWOp *op_post();
+  RGWOp *op_options();
+public:
+  RGWHandler_ObjStore_Bucket_S3() {}
+  virtual ~RGWHandler_ObjStore_Bucket_S3() {}
+};
+
+class RGWHandler_ObjStore_Obj_S3 : public RGWHandler_ObjStore_S3 {
+protected:
+  bool is_acl_op() {
+    return s->info.args.exists("acl");
+  }
+  bool is_cors_op() {
+      return s->info.args.exists("cors");
+  }
+  bool is_obj_update_op() {
+    return is_acl_op();
+  }
+  RGWOp *get_obj_op(bool get_data);
+
+  RGWOp *op_get();
+  RGWOp *op_head();
+  RGWOp *op_put();
+  RGWOp *op_delete();
+  RGWOp *op_post();
+  RGWOp *op_options();
+public:
+  RGWHandler_ObjStore_Obj_S3() {}
+  virtual ~RGWHandler_ObjStore_Obj_S3() {}
+};
+
+class RGWRESTMgr_S3 : public RGWRESTMgr {
+public:
+  RGWRESTMgr_S3() {}
+  virtual ~RGWRESTMgr_S3() {}
+
+  virtual RGWRESTMgr *get_resource_mgr(struct req_state *s, const string& uri) {
+    return this;
+  }
+  virtual RGWHandler *get_handler(struct req_state *s);
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_swift.cc b/ceph/src/rgw/rgw_rest_swift.cc
new file mode 100644
index 00000000..b5620793
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_swift.cc
@@ -0,0 +1,992 @@
+
+#include "common/Formatter.h"
+#include "common/utf8.h"
+#include "rgw_swift.h"
+#include "rgw_rest_swift.h"
+#include "rgw_acl_swift.h"
+#include "rgw_cors_swift.h"
+#include "rgw_formats.h"
+#include "rgw_client_io.h"
+
+#include 
+
+#define dout_subsys ceph_subsys_rgw
+
+int RGWListBuckets_ObjStore_SWIFT::get_params()
+{
+  marker = s->info.args.get("marker");
+  string limit_str;
+  limit_str = s->info.args.get("limit");
+  long l = strtol(limit_str.c_str(), NULL, 10);
+  if (l > (long)limit_max || l < 0)
+    return -ERR_PRECONDITION_FAILED;
+
+  limit = (uint64_t)l;
+
+  if (limit == 0)
+    limit = limit_max;
+
+  need_stats = (s->format != RGW_FORMAT_PLAIN);
+
+  if (need_stats) {
+    bool stats, exists;
+    int r = s->info.args.get_bool("stats", &stats, &exists);
+
+    if (r < 0)
+      return r;
+
+    if (exists) {
+      need_stats = stats;
+    }
+  }
+
+  return 0;
+}
+
+void RGWListBuckets_ObjStore_SWIFT::send_response_begin(bool has_buckets)
+{
+  if (ret) {
+    set_req_state_err(s, ret);
+  } else if (!has_buckets && s->format == RGW_FORMAT_PLAIN) {
+    ret = STATUS_NO_CONTENT;
+    set_req_state_err(s, ret);
+  }
+  dump_errno(s);
+  end_header(s, NULL);
+
+  if (!ret) {
+    dump_start(s);
+    s->formatter->open_array_section("account");
+
+    sent_data = true;
+  }
+}
+
+void RGWListBuckets_ObjStore_SWIFT::send_response_data(RGWUserBuckets& buckets)
+{
+  map& m = buckets.get_buckets();
+  map::iterator iter;
+
+  if (!sent_data)
+    return;
+
+  for (iter = m.begin(); iter != m.end(); ++iter) {
+    RGWBucketEnt obj = iter->second;
+    s->formatter->open_object_section("container");
+    s->formatter->dump_string("name", obj.bucket.name);
+    if (need_stats) {
+      s->formatter->dump_int("count", obj.count);
+      s->formatter->dump_int("bytes", obj.size);
+    }
+    s->formatter->close_section();
+    rgw_flush_formatter(s, s->formatter);
+  }
+}
+
+void RGWListBuckets_ObjStore_SWIFT::send_response_end()
+{
+  if (sent_data) {
+    s->formatter->close_section();
+    rgw_flush_formatter_and_reset(s, s->formatter);
+  }
+}
+
+int RGWListBucket_ObjStore_SWIFT::get_params()
+{
+  prefix = s->info.args.get("prefix");
+  marker = s->info.args.get("marker");
+  max_keys = s->info.args.get("limit");
+  ret = parse_max_keys();
+  if (ret < 0) {
+    return ret;
+  }
+  if (max > default_max)
+    return -ERR_PRECONDITION_FAILED;
+
+  delimiter = s->info.args.get("delimiter");
+
+  string path_args;
+  if (s->info.args.exists("path")) { // should handle empty path
+    path_args = s->info.args.get("path");
+    if (!delimiter.empty() || !prefix.empty()) {
+      return -EINVAL;
+    }
+    prefix = path_args;
+    delimiter="/";
+
+    path = prefix;
+    if (path.size() && path[path.size() - 1] != '/')
+      path.append("/");
+  }
+
+  int len = prefix.size();
+  int delim_size = delimiter.size();
+  if (len >= delim_size) {
+    if (prefix.substr(len - delim_size).compare(delimiter) != 0)
+      prefix.append(delimiter);
+  }
+
+  return 0;
+}
+
+void RGWListBucket_ObjStore_SWIFT::send_response()
+{
+  vector::iterator iter = objs.begin();
+  map::iterator pref_iter = common_prefixes.begin();
+
+  dump_start(s);
+
+  s->formatter->open_array_section_with_attrs("container", FormatterAttrs("name", s->bucket.name.c_str(), NULL));
+
+  while (iter != objs.end() || pref_iter != common_prefixes.end()) {
+    bool do_pref = false;
+    bool do_objs = false;
+    if (pref_iter == common_prefixes.end())
+      do_objs = true;
+    else if (iter == objs.end())
+      do_pref = true;
+    else if (iter->name.compare(pref_iter->first) == 0) {
+      do_objs = true;
+      ++pref_iter;
+    } else if (iter->name.compare(pref_iter->first) <= 0)
+      do_objs = true;
+    else
+      do_pref = true;
+
+    if (do_objs && (marker.empty() || iter->name.compare(marker) > 0)) {
+      if (iter->name.compare(path) == 0)
+        goto next;
+
+      s->formatter->open_object_section("object");
+      s->formatter->dump_string("name", iter->name);
+      s->formatter->dump_string("hash", iter->etag);
+      s->formatter->dump_int("bytes", iter->size);
+      string single_content_type = iter->content_type;
+      if (iter->content_type.size()) {
+        // content type might hold multiple values, just dump the last one
+        ssize_t pos = iter->content_type.rfind(',');
+        if (pos > 0) {
+          ++pos;
+          while (single_content_type[pos] == ' ')
+            ++pos;
+          single_content_type = single_content_type.substr(pos);
+        }
+        s->formatter->dump_string("content_type", single_content_type);
+      }
+      time_t mtime = iter->mtime.sec();
+      dump_time(s, "last_modified", &mtime);
+      s->formatter->close_section();
+    }
+
+    if (do_pref &&  (marker.empty() || pref_iter->first.compare(marker) > 0)) {
+      const string& name = pref_iter->first;
+      if (name.compare(delimiter) == 0)
+        goto next;
+
+        s->formatter->open_object_section_with_attrs("subdir", FormatterAttrs("name", name.c_str(), NULL));
+
+        /* swift is a bit inconsistent here */
+        switch (s->format) {
+          case RGW_FORMAT_XML:
+            s->formatter->dump_string("name", name);
+            break;
+          default:
+            s->formatter->dump_string("subdir", name);
+        }
+        s->formatter->close_section();
+    }
+next:
+    if (do_objs)
+      ++iter;
+    else
+      ++pref_iter;
+  }
+
+  s->formatter->close_section();
+
+  if (!ret && s->formatter->get_len() == 0)
+    ret = STATUS_NO_CONTENT;
+  else if (ret > 0)
+    ret = 0;
+
+  set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this);
+  if (ret < 0) {
+    return;
+  }
+
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+static void dump_container_metadata(struct req_state *s, RGWBucketEnt& bucket)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%lld", (long long)bucket.count);
+  s->cio->print("X-Container-Object-Count: %s\n", buf);
+  snprintf(buf, sizeof(buf), "%lld", (long long)bucket.size);
+  s->cio->print("X-Container-Bytes-Used: %s\n", buf);
+  snprintf(buf, sizeof(buf), "%lld", (long long)bucket.size_rounded);
+  s->cio->print("X-Container-Bytes-Used-Actual: %s\n", buf);
+
+  if (!s->object) {
+    RGWAccessControlPolicy_SWIFT *swift_policy = static_cast(s->bucket_acl);
+    string read_acl, write_acl;
+    swift_policy->to_str(read_acl, write_acl);
+    if (read_acl.size()) {
+      s->cio->print("X-Container-Read: %s\r\n", read_acl.c_str());
+    }
+    if (write_acl.size()) {
+      s->cio->print("X-Container-Write: %s\r\n", write_acl.c_str());
+    }
+  }
+}
+
+static void dump_account_metadata(struct req_state *s, uint32_t buckets_count,
+                                  uint64_t buckets_object_count, uint64_t buckets_size, uint64_t buckets_size_rounded)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%lld", (long long)buckets_count);
+  s->cio->print("X-Account-Container-Count: %s\n", buf);
+  snprintf(buf, sizeof(buf), "%lld", (long long)buckets_object_count);
+  s->cio->print("X-Account-Object-Count: %s\n", buf);
+  snprintf(buf, sizeof(buf), "%lld", (long long)buckets_size);
+  s->cio->print("X-Account-Bytes-Used: %s\n", buf);
+  snprintf(buf, sizeof(buf), "%lld", (long long)buckets_size_rounded);
+  s->cio->print("X-Account-Bytes-Used-Actual: %s\n", buf);
+}
+
+void RGWStatAccount_ObjStore_SWIFT::send_response()
+{
+  if (ret >= 0) {
+    ret = STATUS_NO_CONTENT;
+    dump_account_metadata(s, buckets_count, buckets_objcount, buckets_size, buckets_size_rounded);
+  }
+
+  set_req_state_err(s, ret);
+  dump_errno(s);
+
+  end_header(s, NULL);
+  dump_start(s);
+}
+
+void RGWStatBucket_ObjStore_SWIFT::send_response()
+{
+  if (ret >= 0) {
+    ret = STATUS_NO_CONTENT;
+    dump_container_metadata(s, bucket);
+  }
+
+  set_req_state_err(s, ret);
+  dump_errno(s);
+
+  end_header(s, this);
+  dump_start(s);
+}
+
+static int get_swift_container_settings(req_state *s, RGWRados *store, RGWAccessControlPolicy *policy, bool *has_policy,
+                                        RGWCORSConfiguration *cors_config, bool *has_cors)
+{
+  string read_list, write_list;
+
+  const char *read_attr = s->info.env->get("HTTP_X_CONTAINER_READ");
+  if (read_attr) {
+    read_list = read_attr;
+  }
+  const char *write_attr = s->info.env->get("HTTP_X_CONTAINER_WRITE");
+  if (write_attr) {
+    write_list = write_attr;
+  }
+
+  *has_policy = false;
+
+  if (read_attr || write_attr) {
+    RGWAccessControlPolicy_SWIFT swift_policy(s->cct);
+    int r = swift_policy.create(store, s->user.user_id, s->user.display_name, read_list, write_list);
+    if (r < 0)
+      return r;
+
+    *policy = swift_policy;
+    *has_policy = true;
+  }
+
+  *has_cors = false;
+
+  /*Check and update CORS configuration*/
+  const char *allow_origins = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_ALLOW_ORIGIN");
+  const char *allow_headers = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_ALLOW_HEADERS");
+  const char *expose_headers = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_EXPOSE_HEADERS");
+  const char *max_age = s->info.env->get("HTTP_X_CONTAINER_META_ACCESS_CONTROL_MAX_AGE");
+  if (allow_origins) {
+    RGWCORSConfiguration_SWIFT *swift_cors = new RGWCORSConfiguration_SWIFT;
+    int r = swift_cors->create_update(allow_origins, allow_headers, expose_headers, max_age);
+    if (r < 0) {
+      dout(0) << "Error creating/updating the cors configuration" << dendl;
+      delete swift_cors;
+      return r;
+    }
+    *has_cors = true;
+    *cors_config = *swift_cors;
+    cors_config->dump();
+    delete swift_cors;
+  }
+
+  return 0;
+}
+
+int RGWCreateBucket_ObjStore_SWIFT::get_params()
+{
+  bool has_policy;
+
+  int r = get_swift_container_settings(s, store, &policy, &has_policy, &cors_config, &has_cors);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!has_policy) {
+    policy.create_default(s->user.user_id, s->user.display_name);
+  }
+
+  location_constraint = store->region.api_name;
+
+  return 0;
+}
+
+void RGWCreateBucket_ObjStore_SWIFT::send_response()
+{
+  if (!ret)
+    ret = STATUS_CREATED;
+  else if (ret == -ERR_BUCKET_EXISTS)
+    ret = STATUS_ACCEPTED;
+  set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, NULL);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+void RGWDeleteBucket_ObjStore_SWIFT::send_response()
+{
+  int r = ret;
+  if (!r)
+    r = STATUS_NO_CONTENT;
+
+  set_req_state_err(s, r);
+  dump_errno(s);
+  end_header(s, this);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+int RGWPutObj_ObjStore_SWIFT::get_params()
+{
+  if (s->has_bad_meta)
+    return -EINVAL;
+
+  if (!s->length) {
+    const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
+    if (!encoding || strcmp(encoding, "chunked") != 0)
+      return -ERR_LENGTH_REQUIRED;
+
+    chunked_upload = true;
+  }
+
+  supplied_etag = s->info.env->get("HTTP_ETAG");
+
+  if (!s->generic_attrs.count(RGW_ATTR_CONTENT_TYPE)) {
+    dout(5) << "content type wasn't provided, trying to guess" << dendl;
+    const char *suffix = strrchr(s->object, '.');
+    if (suffix) {
+      suffix++;
+      if (*suffix) {
+        string suffix_str(suffix);
+	const char *mime = rgw_find_mime_by_ext(suffix_str);
+	if (mime) {
+          s->generic_attrs[RGW_ATTR_CONTENT_TYPE] = mime;
+	}
+      }
+    }
+  }
+
+  policy.create_default(s->user.user_id, s->user.display_name);
+
+  obj_manifest = s->info.env->get("HTTP_X_OBJECT_MANIFEST");
+
+  return RGWPutObj_ObjStore::get_params();
+}
+
+void RGWPutObj_ObjStore_SWIFT::send_response()
+{
+  if (!ret)
+    ret = STATUS_CREATED;
+  dump_etag(s, etag.c_str());
+  set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+int RGWPutMetadata_ObjStore_SWIFT::get_params()
+{
+  if (s->has_bad_meta)
+    return -EINVAL;
+
+  if (!s->object) {
+    int r = get_swift_container_settings(s, store, &policy, &has_policy, &cors_config, &has_cors);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return 0;
+}
+
+void RGWPutMetadata_ObjStore_SWIFT::send_response()
+{
+  if (!ret)
+    ret = STATUS_ACCEPTED;
+  set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s, this);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+int RGWSetTempUrl_ObjStore_SWIFT::get_params()
+{
+  const char *temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY");
+  if (temp_url) {
+    temp_url_keys[0] = temp_url;
+  }
+
+  temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY_2");
+  if (temp_url) {
+    temp_url_keys[1] = temp_url;
+  }
+
+  if (temp_url_keys.size() == 0)
+    return -EINVAL;
+
+  return 0;
+}
+
+void RGWSetTempUrl_ObjStore_SWIFT::send_response()
+{
+  int r = ret;
+  if (!r)
+    r = STATUS_NO_CONTENT;
+
+  set_req_state_err(s, r);
+  dump_errno(s);
+  end_header(s, this);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+void RGWDeleteObj_ObjStore_SWIFT::send_response()
+{
+  int r = ret;
+  if (!r)
+    r = STATUS_NO_CONTENT;
+
+  set_req_state_err(s, r);
+  dump_errno(s);
+  end_header(s, this);
+  rgw_flush_formatter_and_reset(s, s->formatter);
+}
+
+int RGWCopyObj_ObjStore_SWIFT::init_dest_policy()
+{
+  dest_policy.create_default(s->user.user_id, s->user.display_name);
+
+  return 0;
+}
+
+int RGWCopyObj_ObjStore_SWIFT::get_params()
+{
+  if_mod = s->info.env->get("HTTP_IF_MODIFIED_SINCE");
+  if_unmod = s->info.env->get("HTTP_IF_UNMODIFIED_SINCE");
+  if_match = s->info.env->get("HTTP_COPY_IF_MATCH");
+  if_nomatch = s->info.env->get("HTTP_COPY_IF_NONE_MATCH");
+
+  src_bucket_name = s->src_bucket_name;
+  src_object = s->src_object;
+  dest_bucket_name = s->bucket_name_str;
+  dest_object = s->object_str;
+
+  return 0;
+}
+
+void RGWCopyObj_ObjStore_SWIFT::send_partial_response(off_t ofs)
+{
+  if (!sent_header) {
+    if (!ret)
+      ret = STATUS_CREATED;
+    set_req_state_err(s, ret);
+    dump_errno(s);
+    end_header(s, this);
+
+    /* Send progress information. Note that this diverge from the original swift
+     * spec. We do this in order to keep connection alive.
+     */
+    if (ret == 0) {
+      s->formatter->open_array_section("progress");
+    }
+    sent_header = true;
+  } else {
+    s->formatter->dump_int("ofs", (uint64_t)ofs);
+  }
+  rgw_flush_formatter(s, s->formatter);
+}
+
+void RGWCopyObj_ObjStore_SWIFT::send_response()
+{
+  if (!sent_header) {
+   if (!ret)
+      ret = STATUS_CREATED;
+    set_req_state_err(s, ret);
+    dump_errno(s);
+    end_header(s, this);
+  } else {
+    s->formatter->close_section();
+    rgw_flush_formatter(s, s->formatter);
+  }
+}
+
+int RGWGetObj_ObjStore_SWIFT::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len)
+{
+  const char *content_type = NULL;
+  int orig_ret = ret;
+  map response_attrs;
+  map::iterator riter;
+
+  if (sent_header)
+    goto send_data;
+
+  if (range_str)
+    dump_range(s, ofs, end, s->obj_size);
+
+  dump_content_length(s, total_len);
+  dump_last_modified(s, lastmod);
+
+  if (!ret) {
+    map::iterator iter = attrs.find(RGW_ATTR_ETAG);
+    if (iter != attrs.end()) {
+      bufferlist& bl = iter->second;
+      if (bl.length()) {
+        char *etag = bl.c_str();
+        dump_etag(s, etag);
+      }
+    }
+
+    for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
+      const char *name = iter->first.c_str();
+      map::iterator aiter = rgw_to_http_attrs.find(name);
+      if (aiter != rgw_to_http_attrs.end()) {
+	if (aiter->first.compare(RGW_ATTR_CONTENT_TYPE) == 0) { // special handling for content_type
+	  content_type = iter->second.c_str();
+	  continue;
+        }
+        response_attrs[aiter->second] = iter->second.c_str();
+      } else {
+        if (strncmp(name, RGW_ATTR_META_PREFIX, sizeof(RGW_ATTR_META_PREFIX)-1) == 0) {
+          name += sizeof(RGW_ATTR_META_PREFIX) - 1;
+          s->cio->print("X-%s-Meta-%s: %s\r\n", (s->object ? "Object" : "Container"), name, iter->second.c_str());
+        }
+      }
+    }
+  }
+
+  if (partial_content && !ret)
+    ret = -STATUS_PARTIAL_CONTENT;
+
+  if (ret)
+    set_req_state_err(s, ret);
+  dump_errno(s);
+
+  for (riter = response_attrs.begin(); riter != response_attrs.end(); ++riter) {
+    s->cio->print("%s: %s\n", riter->first.c_str(), riter->second.c_str());
+  }
+
+  if (!content_type)
+    content_type = "binary/octet-stream";
+  end_header(s, this, content_type);
+
+  sent_header = true;
+
+send_data:
+  if (get_data && !orig_ret) {
+    int r = s->cio->write(bl.c_str() + bl_ofs, bl_len);
+    if (r < 0)
+      return r;
+  }
+  rgw_flush_formatter_and_reset(s, s->formatter);
+
+  return 0;
+}
+
+void RGWOptionsCORS_ObjStore_SWIFT::send_response()
+{
+  string hdrs, exp_hdrs;
+  uint32_t max_age = CORS_MAX_AGE_INVALID;
+  /*EACCES means, there is no CORS registered yet for the bucket
+   *ENOENT means, there is no match of the Origin in the list of CORSRule
+   */
+  if (ret == -ENOENT)
+    ret = -EACCES;
+  if (ret < 0) {
+    set_req_state_err(s, ret);
+    dump_errno(s);
+    end_header(s, NULL);
+    return;
+  }
+  get_response_params(hdrs, exp_hdrs, &max_age);
+  dump_errno(s);
+  dump_access_control(s, origin, req_meth, hdrs.c_str(), exp_hdrs.c_str(), max_age); 
+  end_header(s, NULL);
+}
+
+RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_get()
+{
+  return new RGWListBuckets_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_head()
+{
+  return new RGWStatAccount_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_post()
+{
+  const char *temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY");
+  if (temp_url) {
+    return new RGWSetTempUrl_ObjStore_SWIFT;
+  }
+  temp_url = s->info.env->get("HTTP_X_ACCOUNT_META_TEMP_URL_KEY_2");
+  if (temp_url) {
+    return new RGWSetTempUrl_ObjStore_SWIFT;
+  }
+  return NULL;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::get_obj_op(bool get_data)
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_SWIFT;
+  }
+
+  if (get_data)
+    return new RGWListBucket_ObjStore_SWIFT;
+  else
+    return new RGWStatBucket_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::op_get()
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_SWIFT;
+  }
+  return get_obj_op(true);
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::op_head()
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_SWIFT;
+  }
+  return get_obj_op(false);
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::op_put()
+{
+  if (is_acl_op()) {
+    return new RGWPutACLs_ObjStore_SWIFT;
+  }
+  return new RGWCreateBucket_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::op_delete()
+{
+  return new RGWDeleteBucket_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::op_post()
+{
+  return new RGWPutMetadata_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::op_options()
+{
+  return new RGWOptionsCORS_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::get_obj_op(bool get_data)
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_SWIFT;
+  }
+
+  RGWGetObj_ObjStore_SWIFT *get_obj_op = new RGWGetObj_ObjStore_SWIFT;
+  get_obj_op->set_get_data(get_data);
+  return get_obj_op;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_get()
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_SWIFT;
+  }
+  return get_obj_op(true);
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_head()
+{
+  if (is_acl_op()) {
+    return new RGWGetACLs_ObjStore_SWIFT;
+  }
+  return get_obj_op(false);
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_put()
+{
+  if (is_acl_op()) {
+    return new RGWPutACLs_ObjStore_SWIFT;
+  }
+  if (s->src_bucket_name.empty())
+    return new RGWPutObj_ObjStore_SWIFT;
+  else
+    return new RGWCopyObj_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_delete()
+{
+  return new RGWDeleteObj_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_post()
+{
+  return new RGWPutMetadata_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_copy()
+{
+  return new RGWCopyObj_ObjStore_SWIFT;
+}
+
+RGWOp *RGWHandler_ObjStore_Obj_SWIFT::op_options()
+{
+  return new RGWOptionsCORS_ObjStore_SWIFT;
+}
+
+int RGWHandler_ObjStore_SWIFT::authorize()
+{
+  if (!s->os_auth_token && s->info.args.get("temp_url_sig").empty()) {
+    /* anonymous access */
+    rgw_get_anon_user(s->user);
+    s->perm_mask = RGW_PERM_FULL_CONTROL;
+    return 0;
+  }
+
+  bool authorized = rgw_swift->verify_swift_token(store, s);
+  if (!authorized)
+    return -EPERM;
+
+  s->perm_mask = RGW_PERM_FULL_CONTROL;
+
+  return 0;
+}
+
+int RGWHandler_ObjStore_SWIFT::validate_bucket_name(const string& bucket)
+{
+  int ret = RGWHandler_ObjStore::validate_bucket_name(bucket);
+  if (ret < 0)
+    return ret;
+
+  int len = bucket.size();
+
+  if (len == 0)
+    return 0;
+
+  if (bucket[0] == '.')
+    return -ERR_INVALID_BUCKET_NAME;
+
+  if (check_utf8(bucket.c_str(), len))
+    return -ERR_INVALID_UTF8;
+
+  const char *s = bucket.c_str();
+
+  for (int i = 0; i < len; ++i, ++s) {
+    if (*(unsigned char *)s == 0xff)
+      return -ERR_INVALID_BUCKET_NAME;
+  }
+
+  return 0;
+}
+
+static void next_tok(string& str, string& tok, char delim)
+{
+  if (str.size() == 0) {
+    tok = "";
+    return;
+  }
+  tok = str;
+  int pos = str.find(delim);
+  if (pos > 0) {
+    tok = str.substr(0, pos);
+    str = str.substr(pos + 1);
+  } else {
+    str = "";
+  }
+}
+
+int RGWHandler_ObjStore_SWIFT::init_from_header(struct req_state *s)
+{
+  string req;
+  string first;
+
+  s->prot_flags |= RGW_REST_SWIFT;
+
+  const char *req_name = s->decoded_uri.c_str();
+  const char *p;
+
+  if (*req_name == '?') {
+    p = req_name;
+  } else {
+    p = s->info.request_params.c_str();
+  }
+
+  s->info.args.set(p);
+  s->info.args.parse();
+
+  if (*req_name != '/')
+    return 0;
+
+  req_name++;
+
+  if (!*req_name)
+    return 0;
+
+  req = req_name;
+
+  int pos = req.find('/');
+  if (pos >= 0) {
+    bool cut_url = g_conf->rgw_swift_url_prefix.length();
+    first = req.substr(0, pos);
+    if (first.compare(g_conf->rgw_swift_url_prefix) == 0) {
+      if (cut_url) {
+        next_tok(req, first, '/');
+      }
+    }
+  } else {
+    if (req.compare(g_conf->rgw_swift_url_prefix) == 0) {
+      s->formatter = new RGWFormatter_Plain;
+      return -ERR_BAD_URL;
+    }
+    first = req;
+  }
+
+  string tenant_path;
+  if (!g_conf->rgw_swift_tenant_name.empty()) {
+    tenant_path = "/AUTH_";
+    tenant_path.append(g_conf->rgw_swift_tenant_name);
+  }
+
+  /* verify that the request_uri conforms with what's expected */
+  char buf[g_conf->rgw_swift_url_prefix.length() + 16 + tenant_path.length()];
+  int blen = sprintf(buf, "/%s/v1%s", g_conf->rgw_swift_url_prefix.c_str(), tenant_path.c_str());
+  if (s->decoded_uri[0] != '/' ||
+    s->decoded_uri.compare(0, blen, buf) !=  0) {
+    return -ENOENT;
+  }
+
+  int ret = allocate_formatter(s, RGW_FORMAT_PLAIN, true);
+  if (ret < 0)
+    return ret;
+
+  string ver;
+
+  next_tok(req, ver, '/');
+
+  string tenant;
+  if (!tenant_path.empty()) {
+    next_tok(req, tenant, '/');
+  }
+
+  s->os_auth_token = s->info.env->get("HTTP_X_AUTH_TOKEN");
+  next_tok(req, first, '/');
+
+  dout(10) << "ver=" << ver << " first=" << first << " req=" << req << dendl;
+  if (first.size() == 0)
+    return 0;
+
+  s->bucket_name_str = first;
+   
+  s->info.effective_uri = "/" + s->bucket_name_str;
+
+  if (req.size()) {
+    s->object_str = req;
+    s->object = strdup(s->object_str.c_str());
+    s->info.effective_uri.append("/" + s->object_str);
+  }
+
+  return 0;
+}
+
+int RGWHandler_ObjStore_SWIFT::init(RGWRados *store, struct req_state *s, RGWClientIO *cio)
+{
+  dout(10) << "s->object=" << (s->object ? s->object : "") << " s->bucket=" << (!s->bucket_name_str.empty() ? s->bucket_name_str : "") << dendl;
+
+  int ret = validate_bucket_name(s->bucket_name_str.c_str());
+  if (ret)
+    return ret;
+  ret = validate_object_name(s->object_str);
+  if (ret)
+    return ret;
+
+  s->copy_source = s->info.env->get("HTTP_X_COPY_FROM");
+  if (s->copy_source) {
+    bool result = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object);
+    if (!result)
+       return -ERR_BAD_URL;
+  }
+
+  s->dialect = "swift";
+
+  if (s->op == OP_COPY) {
+    const char *req_dest = s->info.env->get("HTTP_DESTINATION");
+    if (!req_dest)
+      return -ERR_BAD_URL;
+
+    string dest_bucket_name;
+    string dest_object;
+    bool result = RGWCopyObj::parse_copy_location(req_dest, dest_bucket_name, dest_object);
+    if (!result)
+       return -ERR_BAD_URL;
+
+    if (dest_bucket_name != s->bucket_name_str) {
+      ret = validate_bucket_name(dest_bucket_name.c_str());
+      if (ret < 0)
+        return ret;
+    }
+
+    /* convert COPY operation into PUT */
+    s->src_bucket_name = s->bucket_name_str;
+    s->src_object = s->object_str;
+    s->bucket_name_str = dest_bucket_name;
+    s->object_str = dest_object;
+    s->op = OP_PUT;
+  }
+
+  return RGWHandler_ObjStore::init(store, s, cio);
+}
+
+
+RGWHandler *RGWRESTMgr_SWIFT::get_handler(struct req_state *s)
+{
+  int ret = RGWHandler_ObjStore_SWIFT::init_from_header(s);
+  if (ret < 0)
+    return NULL;
+
+  if (s->bucket_name_str.empty())
+    return new RGWHandler_ObjStore_Service_SWIFT;
+  if (!s->object)
+    return new RGWHandler_ObjStore_Bucket_SWIFT;
+
+  return new RGWHandler_ObjStore_Obj_SWIFT;
+}
diff --git a/ceph/src/rgw/rgw_rest_swift.h b/ceph/src/rgw/rgw_rest_swift.h
new file mode 100644
index 00000000..27a31aaa
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_swift.h
@@ -0,0 +1,226 @@
+#ifndef CEPH_RGW_REST_SWIFT_H
+#define CEPH_RGW_REST_SWIFT_H
+#define TIME_BUF_SIZE 128
+
+#include "rgw_op.h"
+#include "rgw_rest.h"
+
+class RGWGetObj_ObjStore_SWIFT : public RGWGetObj_ObjStore {
+public:
+  RGWGetObj_ObjStore_SWIFT() {}
+  ~RGWGetObj_ObjStore_SWIFT() {}
+
+  int send_response_data(bufferlist& bl, off_t ofs, off_t len);
+};
+
+class RGWListBuckets_ObjStore_SWIFT : public RGWListBuckets_ObjStore {
+  bool need_stats;
+public:
+  RGWListBuckets_ObjStore_SWIFT() : need_stats(true) {}
+  ~RGWListBuckets_ObjStore_SWIFT() {}
+
+  int get_params();
+  void send_response_begin(bool has_buckets);
+  void send_response_data(RGWUserBuckets& buckets);
+  void send_response_end();
+
+  bool should_get_stats() { return need_stats; }
+};
+
+class RGWListBucket_ObjStore_SWIFT : public RGWListBucket_ObjStore {
+  string path;
+public:
+  RGWListBucket_ObjStore_SWIFT() {
+    default_max = 10000;
+  }
+  ~RGWListBucket_ObjStore_SWIFT() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWStatAccount_ObjStore_SWIFT : public RGWStatAccount_ObjStore {
+public:
+  RGWStatAccount_ObjStore_SWIFT() {
+  }
+  ~RGWStatAccount_ObjStore_SWIFT() {}
+
+  void send_response();
+};
+
+class RGWStatBucket_ObjStore_SWIFT : public RGWStatBucket_ObjStore {
+public:
+  RGWStatBucket_ObjStore_SWIFT() {}
+  ~RGWStatBucket_ObjStore_SWIFT() {}
+
+  void send_response();
+};
+
+class RGWCreateBucket_ObjStore_SWIFT : public RGWCreateBucket_ObjStore {
+public:
+  RGWCreateBucket_ObjStore_SWIFT() {}
+  ~RGWCreateBucket_ObjStore_SWIFT() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWDeleteBucket_ObjStore_SWIFT : public RGWDeleteBucket_ObjStore {
+public:
+  RGWDeleteBucket_ObjStore_SWIFT() {}
+  ~RGWDeleteBucket_ObjStore_SWIFT() {}
+
+  void send_response();
+};
+
+class RGWPutObj_ObjStore_SWIFT : public RGWPutObj_ObjStore {
+public:
+  RGWPutObj_ObjStore_SWIFT() {}
+  ~RGWPutObj_ObjStore_SWIFT() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWPutMetadata_ObjStore_SWIFT : public RGWPutMetadata_ObjStore {
+public:
+  RGWPutMetadata_ObjStore_SWIFT() {}
+  ~RGWPutMetadata_ObjStore_SWIFT() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWSetTempUrl_ObjStore_SWIFT : public RGWSetTempUrl_ObjStore {
+public:
+  RGWSetTempUrl_ObjStore_SWIFT() {}
+  ~RGWSetTempUrl_ObjStore_SWIFT() {}
+
+  int get_params();
+  void send_response();
+};
+
+class RGWDeleteObj_ObjStore_SWIFT : public RGWDeleteObj_ObjStore {
+public:
+  RGWDeleteObj_ObjStore_SWIFT() {}
+  ~RGWDeleteObj_ObjStore_SWIFT() {}
+
+  void send_response();
+};
+
+class RGWCopyObj_ObjStore_SWIFT : public RGWCopyObj_ObjStore {
+  bool sent_header;
+public:
+  RGWCopyObj_ObjStore_SWIFT() : sent_header(false) {}
+  ~RGWCopyObj_ObjStore_SWIFT() {}
+
+  int init_dest_policy();
+  int get_params();
+  void send_response();
+  void send_partial_response(off_t ofs);
+};
+
+class RGWGetACLs_ObjStore_SWIFT : public RGWGetACLs_ObjStore {
+public:
+  RGWGetACLs_ObjStore_SWIFT() {}
+  ~RGWGetACLs_ObjStore_SWIFT() {}
+
+  void send_response() {}
+};
+
+class RGWPutACLs_ObjStore_SWIFT : public RGWPutACLs_ObjStore {
+public:
+  RGWPutACLs_ObjStore_SWIFT() : RGWPutACLs_ObjStore() {}
+  virtual ~RGWPutACLs_ObjStore_SWIFT() {}
+
+  void send_response() {}
+};
+
+class RGWOptionsCORS_ObjStore_SWIFT : public RGWOptionsCORS_ObjStore {
+public:
+  RGWOptionsCORS_ObjStore_SWIFT() {}
+  ~RGWOptionsCORS_ObjStore_SWIFT() {}
+
+  void send_response();
+};
+
+class RGWHandler_ObjStore_SWIFT : public RGWHandler_ObjStore {
+  friend class RGWRESTMgr_SWIFT;
+protected:
+  virtual bool is_acl_op() {
+    return false;
+  }
+
+  static int init_from_header(struct req_state *s);
+public:
+  RGWHandler_ObjStore_SWIFT() {}
+  virtual ~RGWHandler_ObjStore_SWIFT() {}
+
+  int validate_bucket_name(const string& bucket);
+
+  int init(RGWRados *store, struct req_state *state, RGWClientIO *cio);
+  int authorize();
+
+  RGWAccessControlPolicy *alloc_policy() { return NULL; /* return new RGWAccessControlPolicy_SWIFT; */ }
+  void free_policy(RGWAccessControlPolicy *policy) { delete policy; }
+};
+
+class RGWHandler_ObjStore_Service_SWIFT : public RGWHandler_ObjStore_SWIFT {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_head();
+  RGWOp *op_post();
+public:
+  RGWHandler_ObjStore_Service_SWIFT() {}
+  virtual ~RGWHandler_ObjStore_Service_SWIFT() {}
+};
+
+class RGWHandler_ObjStore_Bucket_SWIFT : public RGWHandler_ObjStore_SWIFT {
+protected:
+  bool is_obj_update_op() {
+    return s->op == OP_POST;
+  }
+
+  RGWOp *get_obj_op(bool get_data);
+  RGWOp *op_get();
+  RGWOp *op_head();
+  RGWOp *op_put();
+  RGWOp *op_delete();
+  RGWOp *op_post();
+  RGWOp *op_options();
+public:
+  RGWHandler_ObjStore_Bucket_SWIFT() {}
+  virtual ~RGWHandler_ObjStore_Bucket_SWIFT() {}
+};
+
+class RGWHandler_ObjStore_Obj_SWIFT : public RGWHandler_ObjStore_SWIFT {
+protected:
+  bool is_obj_update_op() {
+    return s->op == OP_POST;
+  }
+
+  RGWOp *get_obj_op(bool get_data);
+  RGWOp *op_get();
+  RGWOp *op_head();
+  RGWOp *op_put();
+  RGWOp *op_delete();
+  RGWOp *op_post();
+  RGWOp *op_copy();
+  RGWOp *op_options();
+public:
+  RGWHandler_ObjStore_Obj_SWIFT() {}
+  virtual ~RGWHandler_ObjStore_Obj_SWIFT() {}
+};
+
+class RGWRESTMgr_SWIFT : public RGWRESTMgr {
+public:
+  RGWRESTMgr_SWIFT() {}
+  virtual ~RGWRESTMgr_SWIFT() {}
+
+  virtual RGWRESTMgr *get_resource_mgr(struct req_state *s, const string& uri) {
+    return this;
+  }
+  virtual RGWHandler *get_handler(struct req_state *s);
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_usage.cc b/ceph/src/rgw/rgw_rest_usage.cc
new file mode 100644
index 00000000..1124d2b2
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_usage.cc
@@ -0,0 +1,96 @@
+#include "rgw_op.h"
+#include "rgw_usage.h"
+#include "rgw_rest_usage.h"
+
+#include "include/str_list.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+class RGWOp_Usage_Get : public RGWRESTOp {
+
+public:
+  RGWOp_Usage_Get() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("usage", RGW_CAP_READ);
+  }
+  void execute();
+
+  virtual const string name() { return "get_usage"; }
+};
+
+void RGWOp_Usage_Get::execute() {
+  map categories;
+
+  string uid;
+  uint64_t start, end;
+  bool show_entries;
+  bool show_summary;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_epoch(s, "start", 0, &start);
+  RESTArgs::get_epoch(s, "end", (uint64_t)-1, &end);
+  RESTArgs::get_bool(s, "show-entries", true, &show_entries);
+  RESTArgs::get_bool(s, "show-summary", true, &show_summary);
+
+  string cat_str;
+  RESTArgs::get_string(s, "categories", cat_str, &cat_str);
+
+  if (!cat_str.empty()) {
+    list cat_list;
+    list::iterator iter;
+    get_str_list(cat_str, cat_list);
+    for (iter = cat_list.begin(); iter != cat_list.end(); ++iter) {
+      categories[*iter] = true;
+    }
+  }
+
+  http_ret = RGWUsage::show(store, uid, start, end, show_entries, show_summary, &categories, flusher);
+}
+
+class RGWOp_Usage_Delete : public RGWRESTOp {
+
+public:
+  RGWOp_Usage_Delete() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("usage", RGW_CAP_WRITE);
+  }
+  void execute();
+
+  virtual const string name() { return "trim_usage"; }
+};
+
+void RGWOp_Usage_Delete::execute() {
+  string uid;
+  uint64_t start, end;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_epoch(s, "start", 0, &start);
+  RESTArgs::get_epoch(s, "end", (uint64_t)-1, &end);
+
+  if (uid.empty() &&
+      !start &&
+      end == (uint64_t)-1) {
+    bool remove_all;
+    RESTArgs::get_bool(s, "remove-all", false, &remove_all);
+    if (!remove_all) {
+      http_ret = -EINVAL;
+      return;
+    }
+  }
+
+  http_ret = RGWUsage::trim(store, uid, start, end);
+}
+
+RGWOp *RGWHandler_Usage::op_get()
+{
+  return new RGWOp_Usage_Get;
+};
+
+RGWOp *RGWHandler_Usage::op_delete()
+{
+  return new RGWOp_Usage_Delete;
+};
+
+
diff --git a/ceph/src/rgw/rgw_rest_usage.h b/ceph/src/rgw/rgw_rest_usage.h
new file mode 100644
index 00000000..a6eb3d57
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_usage.h
@@ -0,0 +1,32 @@
+#ifndef CEPH_RGW_REST_USAGE_H
+#define CEPH_RGW_REST_USAGE_H
+
+#include "rgw_rest.h"
+#include "rgw_rest_s3.h"
+
+
+class RGWHandler_Usage : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_delete();
+public:
+  RGWHandler_Usage() {}
+  virtual ~RGWHandler_Usage() {}
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+};
+
+class RGWRESTMgr_Usage : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Usage() {}
+  virtual ~RGWRESTMgr_Usage() {}
+
+  RGWHandler *get_handler(struct req_state *s) {
+    return new RGWHandler_Usage;
+  }
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_rest_user.cc b/ceph/src/rgw/rgw_rest_user.cc
new file mode 100644
index 00000000..30f46f05
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_user.cc
@@ -0,0 +1,917 @@
+#include "common/ceph_json.h"
+
+#include "rgw_op.h"
+#include "rgw_user.h"
+#include "rgw_rest_user.h"
+
+#include "include/str_list.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+class RGWOp_User_Info : public RGWRESTOp {
+
+public:
+  RGWOp_User_Info() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_READ);
+  }
+
+  void execute();
+
+  virtual const string name() { return "get_user_info"; }
+};
+
+void RGWOp_User_Info::execute()
+{
+  RGWUserAdminOpState op_state;
+
+  std::string uid;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+
+  op_state.set_user_id(uid);
+
+  http_ret = RGWUserAdminOp_User::info(store, op_state, flusher);
+}
+
+class RGWOp_User_Create : public RGWRESTOp {
+
+public:
+  RGWOp_User_Create() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "create_user"; }
+};
+
+void RGWOp_User_Create::execute()
+{
+  std::string uid;
+  std::string display_name;
+  std::string email;
+  std::string access_key;
+  std::string secret_key;
+  std::string key_type_str;
+  std::string caps;
+
+  bool gen_key;
+  bool suspended;
+  bool system;
+
+  uint32_t max_buckets;
+  int32_t key_type = KEY_TYPE_UNDEFINED;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "display-name", display_name, &display_name);
+  RESTArgs::get_string(s, "email", email, &email);
+  RESTArgs::get_string(s, "access-key", access_key, &access_key);
+  RESTArgs::get_string(s, "secret-key", secret_key, &secret_key);
+  RESTArgs::get_string(s, "key-type", key_type_str, &key_type_str);
+  RESTArgs::get_string(s, "user-caps", caps, &caps);
+  RESTArgs::get_bool(s, "generate-key", true, &gen_key);
+  RESTArgs::get_bool(s, "suspended", false, &suspended);
+  RESTArgs::get_uint32(s, "max-buckets", RGW_DEFAULT_MAX_BUCKETS, &max_buckets);
+  RESTArgs::get_bool(s, "system", false, &system);
+
+  if (!s->user.system && system) {
+    ldout(s->cct, 0) << "cannot set system flag by non-system user" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  // FIXME: don't do double argument checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!display_name.empty())
+    op_state.set_display_name(display_name);
+
+  if (!email.empty())
+    op_state.set_user_email(email);
+
+  if (!caps.empty())
+    op_state.set_caps(caps);
+
+  if (!access_key.empty())
+    op_state.set_access_key(access_key);
+
+  if (!secret_key.empty())
+    op_state.set_secret_key(secret_key);
+
+  if (!key_type_str.empty()) {
+    if (key_type_str.compare("swift") == 0)
+      key_type = KEY_TYPE_SWIFT;
+    else if (key_type_str.compare("s3") == 0)
+      key_type = KEY_TYPE_S3;
+
+    op_state.set_key_type(key_type);
+  }
+
+  if (max_buckets != RGW_DEFAULT_MAX_BUCKETS)
+    op_state.set_max_buckets(max_buckets);
+
+  if (s->info.args.exists("suspended"))
+    op_state.set_suspension(suspended);
+
+  if (s->info.args.exists("system"))
+    op_state.set_system(system);
+
+  if (gen_key)
+    op_state.set_generate_key();
+
+  http_ret = RGWUserAdminOp_User::create(store, op_state, flusher);
+}
+
+class RGWOp_User_Modify : public RGWRESTOp {
+
+public:
+  RGWOp_User_Modify() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "modify_user"; }
+};
+
+void RGWOp_User_Modify::execute()
+{
+  std::string uid;
+  std::string display_name;
+  std::string email;
+  std::string access_key;
+  std::string secret_key;
+  std::string key_type_str;
+  std::string caps;
+
+  bool gen_key;
+  bool suspended;
+  bool system;
+
+  uint32_t max_buckets;
+  int32_t key_type = KEY_TYPE_UNDEFINED;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "display-name", display_name, &display_name);
+  RESTArgs::get_string(s, "email", email, &email);
+  RESTArgs::get_string(s, "access-key", access_key, &access_key);
+  RESTArgs::get_string(s, "secret-key", secret_key, &secret_key);
+  RESTArgs::get_string(s, "user-caps", caps, &caps);
+  RESTArgs::get_bool(s, "generate-key", false, &gen_key);
+  RESTArgs::get_bool(s, "suspended", false, &suspended);
+  RESTArgs::get_uint32(s, "max-buckets", RGW_DEFAULT_MAX_BUCKETS, &max_buckets);
+  RESTArgs::get_string(s, "key-type", key_type_str, &key_type_str);
+
+  RESTArgs::get_bool(s, "system", false, &system);
+
+  if (!s->user.system && system) {
+    ldout(s->cct, 0) << "cannot set system flag by non-system user" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!display_name.empty())
+    op_state.set_display_name(display_name);
+
+  if (!email.empty())
+    op_state.set_user_email(email);
+
+  if (!caps.empty())
+    op_state.set_caps(caps);
+
+  if (!access_key.empty())
+    op_state.set_access_key(access_key);
+
+  if (!secret_key.empty())
+    op_state.set_secret_key(secret_key);
+
+  if (max_buckets != RGW_DEFAULT_MAX_BUCKETS)
+    op_state.set_max_buckets(max_buckets);
+
+  if (gen_key)
+    op_state.set_generate_key();
+
+  if (!key_type_str.empty()) {
+    if (key_type_str.compare("swift") == 0)
+      key_type = KEY_TYPE_SWIFT;
+    else if (key_type_str.compare("s3") == 0)
+      key_type = KEY_TYPE_S3;
+
+    op_state.set_key_type(key_type);
+  }
+
+  if (s->info.args.exists("suspended"))
+    op_state.set_suspension(suspended);
+
+  if (s->info.args.exists("system"))
+    op_state.set_system(system);
+
+  http_ret = RGWUserAdminOp_User::modify(store, op_state, flusher);
+}
+
+class RGWOp_User_Remove : public RGWRESTOp {
+
+public:
+  RGWOp_User_Remove() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "remove_user"; }
+};
+
+void RGWOp_User_Remove::execute()
+{
+  std::string uid;
+  bool purge_data;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_bool(s, "purge-data", false, &purge_data);
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  op_state.set_purge_data(purge_data);
+
+  http_ret = RGWUserAdminOp_User::remove(store, op_state, flusher);
+}
+
+class RGWOp_Subuser_Create : public RGWRESTOp {
+
+public:
+  RGWOp_Subuser_Create() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "create_subuser"; }
+};
+
+void RGWOp_Subuser_Create::execute()
+{
+  std::string uid;
+  std::string subuser;
+  std::string secret_key;
+  std::string perm_str;
+  std::string key_type_str;
+
+  bool gen_subuser = false; // FIXME placeholder
+  bool gen_secret;
+
+  uint32_t perm_mask = 0;
+  int32_t key_type = KEY_TYPE_SWIFT;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "subuser", subuser, &subuser);
+  RESTArgs::get_string(s, "secret-key", secret_key, &secret_key);
+  RESTArgs::get_string(s, "access", perm_str, &perm_str);
+  RESTArgs::get_string(s, "key-type", key_type_str, &key_type_str);
+  //RESTArgs::get_bool(s, "generate-subuser", false, &gen_subuser);
+  RESTArgs::get_bool(s, "generate-secret", false, &gen_secret);
+
+  perm_mask = rgw_str_to_perm(perm_str.c_str());
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!subuser.empty())
+    op_state.set_subuser(subuser);
+
+  if (!secret_key.empty())
+    op_state.set_secret_key(secret_key);
+
+  if (perm_mask != 0)
+    op_state.set_perm(perm_mask);
+
+  op_state.set_generate_subuser(gen_subuser);
+
+  if (gen_secret)
+    op_state.set_gen_secret();
+
+  if (!key_type_str.empty()) {
+    if (key_type_str.compare("swift") == 0)
+      key_type = KEY_TYPE_SWIFT;
+    else if (key_type_str.compare("s3") == 0)
+      key_type = KEY_TYPE_S3;
+  }
+  op_state.set_key_type(key_type);
+
+  http_ret = RGWUserAdminOp_Subuser::create(store, op_state, flusher);
+}
+
+class RGWOp_Subuser_Modify : public RGWRESTOp {
+
+public:
+  RGWOp_Subuser_Modify() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "modify_subuser"; }
+};
+
+void RGWOp_Subuser_Modify::execute()
+{
+  std::string uid;
+  std::string subuser;
+  std::string secret_key;
+  std::string key_type_str;
+  std::string perm_str;
+
+  RGWUserAdminOpState op_state;
+
+  uint32_t perm_mask;
+  int32_t key_type = KEY_TYPE_SWIFT;
+
+  bool gen_secret;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "subuser", subuser, &subuser);
+  RESTArgs::get_string(s, "secret-key", secret_key, &secret_key);
+  RESTArgs::get_string(s, "access", perm_str, &perm_str);
+  RESTArgs::get_string(s, "key-type", key_type_str, &key_type_str);
+  RESTArgs::get_bool(s, "generate-secret", false, &gen_secret);
+
+  perm_mask = rgw_str_to_perm(perm_str.c_str());
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!subuser.empty())
+    op_state.set_subuser(subuser);
+
+  if (!secret_key.empty())
+    op_state.set_secret_key(secret_key);
+
+  if (gen_secret)
+    op_state.set_gen_secret();
+
+  if (perm_mask != 0)
+    op_state.set_perm(perm_mask);
+
+  if (!key_type_str.empty()) {
+    if (key_type_str.compare("swift") == 0)
+      key_type = KEY_TYPE_SWIFT;
+    else if (key_type_str.compare("s3") == 0)
+      key_type = KEY_TYPE_S3;
+  }
+  op_state.set_key_type(key_type);
+
+  http_ret = RGWUserAdminOp_Subuser::modify(store, op_state, flusher);
+}
+
+class RGWOp_Subuser_Remove : public RGWRESTOp {
+
+public:
+  RGWOp_Subuser_Remove() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "remove_subuser"; }
+};
+
+void RGWOp_Subuser_Remove::execute()
+{
+  std::string uid;
+  std::string subuser;
+  bool purge_keys;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "subuser", subuser, &subuser);
+  RESTArgs::get_bool(s, "purge-keys", true, &purge_keys);
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!subuser.empty())
+    op_state.set_subuser(subuser);
+
+  if (purge_keys)
+    op_state.set_purge_keys();
+
+  http_ret = RGWUserAdminOp_Subuser::remove(store, op_state, flusher);
+}
+
+class RGWOp_Key_Create : public RGWRESTOp {
+
+public:
+  RGWOp_Key_Create() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "create_access_key"; }
+};
+
+void RGWOp_Key_Create::execute()
+{
+  std::string uid;
+  std::string subuser;
+  std::string access_key;
+  std::string secret_key;
+  std::string key_type_str;
+
+  int32_t key_type = KEY_TYPE_UNDEFINED;
+  bool gen_key;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "subuser", subuser, &subuser);
+  RESTArgs::get_string(s, "access-key", access_key, &access_key);
+  RESTArgs::get_string(s, "secret-key", secret_key, &secret_key);
+  RESTArgs::get_string(s, "key-type", key_type_str, &key_type_str);
+  RESTArgs::get_bool(s, "generate-key", true, &gen_key);
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!subuser.empty())
+    op_state.set_subuser(subuser);
+
+  if (!access_key.empty())
+    op_state.set_access_key(access_key);
+
+  if (!secret_key.empty())
+    op_state.set_secret_key(secret_key);
+
+  if (gen_key)
+    op_state.set_generate_key();
+
+  if (!key_type_str.empty()) {
+    if (key_type_str.compare("swift") == 0)
+      key_type = KEY_TYPE_SWIFT;
+    else if (key_type_str.compare("s3") == 0)
+      key_type = KEY_TYPE_S3;
+
+    op_state.set_key_type(key_type);
+  }
+
+  http_ret = RGWUserAdminOp_Key::create(store, op_state, flusher);
+}
+
+class RGWOp_Key_Remove : public RGWRESTOp {
+
+public:
+  RGWOp_Key_Remove() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "remove_access_key"; }
+};
+
+void RGWOp_Key_Remove::execute()
+{
+  std::string uid;
+  std::string subuser;
+  std::string access_key;
+  std::string key_type_str;
+
+  int32_t key_type = KEY_TYPE_UNDEFINED;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "subuser", subuser, &subuser);
+  RESTArgs::get_string(s, "access-key", access_key, &access_key);
+  RESTArgs::get_string(s, "key-type", key_type_str, &key_type_str);
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!subuser.empty())
+    op_state.set_subuser(subuser);
+
+  if (!access_key.empty())
+    op_state.set_access_key(access_key);
+
+  if (!key_type_str.empty()) {
+    if (key_type_str.compare("swift") == 0)
+      key_type = KEY_TYPE_SWIFT;
+    else if (key_type_str.compare("s3") == 0)
+      key_type = KEY_TYPE_S3;
+
+    op_state.set_key_type(key_type);
+  }
+
+  http_ret = RGWUserAdminOp_Key::remove(store, op_state, flusher);
+}
+
+class RGWOp_Caps_Add : public RGWRESTOp {
+
+public:
+  RGWOp_Caps_Add() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "add_user_caps"; }
+};
+
+void RGWOp_Caps_Add::execute()
+{
+  std::string uid;
+  std::string caps;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "user-caps", caps, &caps);
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!caps.empty())
+    op_state.set_caps(caps);
+
+  http_ret = RGWUserAdminOp_Caps::add(store, op_state, flusher);
+}
+
+class RGWOp_Caps_Remove : public RGWRESTOp {
+
+public:
+  RGWOp_Caps_Remove() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "remove_user_caps"; }
+};
+
+void RGWOp_Caps_Remove::execute()
+{
+  std::string uid;
+  std::string caps;
+
+  RGWUserAdminOpState op_state;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "user-caps", caps, &caps);
+
+  // FIXME: no double checking
+  if (!uid.empty())
+    op_state.set_user_id(uid);
+
+  if (!caps.empty())
+    op_state.set_caps(caps);
+
+  http_ret = RGWUserAdminOp_Caps::remove(store, op_state, flusher);
+}
+
+struct UserQuotas {
+  RGWQuotaInfo bucket_quota;
+  RGWQuotaInfo user_quota;
+
+  UserQuotas() {}
+
+  UserQuotas(RGWUserInfo& info) {
+    bucket_quota = info.bucket_quota;
+    user_quota = info.user_quota;
+  }
+  void dump(Formatter *f) const {
+    encode_json("bucket_quota", bucket_quota, f);
+    encode_json("user_quota", user_quota, f);
+  }
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("bucket_quota", bucket_quota, obj);
+    JSONDecoder::decode_json("user_quota", user_quota, obj);
+  }
+};
+
+class RGWOp_Quota_Info : public RGWRESTOp {
+
+public:
+  RGWOp_Quota_Info() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_READ);
+  }
+
+  void execute();
+
+  virtual const string name() { return "get_quota_info"; }
+};
+
+
+void RGWOp_Quota_Info::execute()
+{
+  RGWUserAdminOpState op_state;
+
+  std::string uid;
+  std::string quota_type;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "quota-type", quota_type, "a_type);
+
+  if (uid.empty()) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  bool show_all = quota_type.empty();
+  bool show_bucket = show_all || (quota_type == "bucket");
+  bool show_user = show_all || (quota_type == "user");
+
+  if (!(show_all || show_bucket || show_user)) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  op_state.set_user_id(uid);
+
+  RGWUser user;
+  http_ret = user.init(store, op_state);
+  if (http_ret < 0)
+    return;
+
+  RGWUserInfo info;
+  string err_msg;
+  http_ret = user.info(info, &err_msg);
+  if (http_ret < 0)
+    return;
+
+  if (show_all) {
+    UserQuotas quotas(info);
+    encode_json("quota", quotas, s->formatter);
+  } else if (show_user) {
+    encode_json("user_quota", info.user_quota, s->formatter);
+  } else {
+    encode_json("bucket_quota", info.bucket_quota, s->formatter);
+  }
+
+  flusher.flush();
+}
+
+class RGWOp_Quota_Set : public RGWRESTOp {
+
+public:
+  RGWOp_Quota_Set() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("users", RGW_CAP_WRITE);
+  }
+
+  void execute();
+
+  virtual const string name() { return "set_quota_info"; }
+};
+
+/**
+ * set quota
+ *
+ * two different ways to set the quota info: as json struct in the message body or via http params.
+ *
+ * as json:
+ *
+ * PUT /admin/user?uid=["a-type=]
+ *
+ * whereas quota-type is optional and is either user, or bucket
+ *
+ * if quota-type is not specified then we expect to get a structure that contains both quotas,
+ * otherwise we'll only get the relevant configuration.
+ *
+ * E.g., if quota type not specified:
+ * {
+ *    "user_quota" : {
+ *      "max_size_kb" : 4096,
+ *      "max_objects" : -1,
+ *      "enabled" : false
+ *    },
+ *    "bucket_quota" : {
+ *      "max_size_kb" : 1024,
+ *      "max_objects" : -1,
+ *      "enabled" : true
+ *    }
+ * }
+ *
+ *
+ * or if quota type is specified:
+ * {
+ *   "max_size_kb" : 4096,
+ *   "max_objects" : -1,
+ *   "enabled" : false
+ * }
+ *
+ * Another option is not to pass any body and set the following http params:
+ *
+ *
+ * max-size-kb=
+ * max-objects=
+ * enabled[={true,false}]
+ *
+ * all params are optionals and default to the current settings. With this type of configuration the
+ * quota-type param is mandatory.
+ *
+ */
+
+void RGWOp_Quota_Set::execute()
+{
+  RGWUserAdminOpState op_state;
+
+  std::string uid;
+  std::string quota_type;
+
+  RESTArgs::get_string(s, "uid", uid, &uid);
+  RESTArgs::get_string(s, "quota-type", quota_type, "a_type);
+
+  if (uid.empty()) {
+    http_ret = -EINVAL;
+    return;
+  }
+
+  bool set_all = quota_type.empty();
+  bool set_bucket = set_all || (quota_type == "bucket");
+  bool set_user = set_all || (quota_type == "user");
+
+  if (!(set_all || set_bucket || set_user)) {
+    ldout(store->ctx(), 20) << "invalid quota type" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  bool use_http_params;
+
+  if (s->content_length > 0) {
+    use_http_params = false;
+  } else {
+    const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
+    use_http_params = (!encoding || strcmp(encoding, "chunked") != 0);
+  }
+
+  if (use_http_params && set_all) {
+    ldout(store->ctx(), 20) << "quota type was not specified, can't set all quotas via http headers" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  op_state.set_user_id(uid);
+
+  RGWUser user;
+  http_ret = user.init(store, op_state);
+  if (http_ret < 0) {
+    ldout(store->ctx(), 20) << "failed initializing user info: " << http_ret << dendl;
+    return;
+  }
+
+#define QUOTA_INPUT_MAX_LEN 1024
+  if (set_all) {
+    UserQuotas quotas;
+
+    if ((http_ret = rgw_rest_get_json_input(store->ctx(), s, quotas, QUOTA_INPUT_MAX_LEN, NULL)) < 0) {
+      ldout(store->ctx(), 20) << "failed to retrieve input" << dendl;
+      return;
+    }
+
+    op_state.set_user_quota(quotas.user_quota);
+    op_state.set_bucket_quota(quotas.bucket_quota);
+  } else {
+    RGWQuotaInfo quota;
+
+    if (!use_http_params) {
+      bool empty;
+      http_ret = rgw_rest_get_json_input(store->ctx(), s, quota, QUOTA_INPUT_MAX_LEN, &empty);
+      if (http_ret < 0) {
+        ldout(store->ctx(), 20) << "failed to retrieve input" << dendl;
+        if (!empty)
+          return;
+
+        /* was probably chunked input, but no content provided, configure via http params */
+        use_http_params = true;
+      }
+    }
+
+    if (use_http_params) {
+      RGWUserInfo info;
+      string err_msg;
+      http_ret = user.info(info, &err_msg);
+      if (http_ret < 0) {
+        ldout(store->ctx(), 20) << "failed to get user info: " << http_ret << dendl;
+        return;
+      }
+      RGWQuotaInfo *old_quota;
+      if (set_user) {
+        old_quota = &info.user_quota;
+      } else {
+        old_quota = &info.bucket_quota;
+      }
+
+      RESTArgs::get_int64(s, "max-objects", old_quota->max_objects, "a.max_objects);
+      RESTArgs::get_int64(s, "max-size-kb", old_quota->max_size_kb, "a.max_size_kb);
+      RESTArgs::get_bool(s, "enabled", old_quota->enabled, "a.enabled);
+    }
+
+    if (set_user) {
+      op_state.set_user_quota(quota);
+    } else {
+      op_state.set_bucket_quota(quota);
+    }
+  }
+
+  string err;
+  http_ret = user.modify(op_state, &err);
+  if (http_ret < 0) {
+    ldout(store->ctx(), 20) << "failed updating user info: " << http_ret << ": " << err << dendl;
+    return;
+  }
+}
+
+RGWOp *RGWHandler_User::op_get()
+{
+  if (s->info.args.sub_resource_exists("quota"))
+    return new RGWOp_Quota_Info;
+
+  return new RGWOp_User_Info;
+};
+
+RGWOp *RGWHandler_User::op_put()
+{
+  if (s->info.args.sub_resource_exists("subuser"))
+    return new RGWOp_Subuser_Create;
+
+  if (s->info.args.sub_resource_exists("key"))
+    return new RGWOp_Key_Create;
+
+  if (s->info.args.sub_resource_exists("caps"))
+    return new RGWOp_Caps_Add;
+
+  if (s->info.args.sub_resource_exists("quota"))
+    return new RGWOp_Quota_Set;
+
+  return new RGWOp_User_Create;
+};
+
+RGWOp *RGWHandler_User::op_post()
+{
+  if (s->info.args.sub_resource_exists("subuser"))
+    return new RGWOp_Subuser_Modify;
+
+  return new RGWOp_User_Modify;
+};
+
+RGWOp *RGWHandler_User::op_delete()
+{
+  if (s->info.args.sub_resource_exists("subuser"))
+    return new RGWOp_Subuser_Remove;
+
+  if (s->info.args.sub_resource_exists("key"))
+    return new RGWOp_Key_Remove;
+
+  if (s->info.args.sub_resource_exists("caps"))
+    return new RGWOp_Caps_Remove;
+
+  return new RGWOp_User_Remove;
+};
+
diff --git a/ceph/src/rgw/rgw_rest_user.h b/ceph/src/rgw/rgw_rest_user.h
new file mode 100644
index 00000000..1fe29d65
--- /dev/null
+++ b/ceph/src/rgw/rgw_rest_user.h
@@ -0,0 +1,33 @@
+#ifndef CEPH_RGW_REST_USER_H
+#define CEPH_RGW_REST_USER_H
+
+#include "rgw_rest.h"
+#include "rgw_rest_s3.h"
+
+
+class RGWHandler_User : public RGWHandler_Auth_S3 {
+protected:
+  RGWOp *op_get();
+  RGWOp *op_put();
+  RGWOp *op_post();
+  RGWOp *op_delete();
+public:
+  RGWHandler_User() {}
+  virtual ~RGWHandler_User() {}
+
+  int read_permissions(RGWOp*) {
+    return 0;
+  }
+};
+
+class RGWRESTMgr_User : public RGWRESTMgr {
+public:
+  RGWRESTMgr_User() {}
+  virtual ~RGWRESTMgr_User() {}
+
+  RGWHandler *get_handler(struct req_state *s) {
+    return new RGWHandler_User;
+  }
+};
+
+#endif
diff --git a/ceph/src/rgw/rgw_string.h b/ceph/src/rgw/rgw_string.h
new file mode 100644
index 00000000..3c881a10
--- /dev/null
+++ b/ceph/src/rgw/rgw_string.h
@@ -0,0 +1,94 @@
+#ifndef CEPH_RGW_STRING_H
+#define CEPH_RGW_STRING_H
+
+#include 
+#include 
+
+struct ltstr_nocase
+{
+  bool operator()(const string& s1, const string& s2) const
+  {
+    return strcasecmp(s1.c_str(), s2.c_str()) < 0;
+  }
+};
+
+static inline int stringcasecmp(const string& s1, const string& s2)
+{
+  return strcasecmp(s1.c_str(), s2.c_str());
+}
+
+static inline int stringcasecmp(const string& s1, const char *s2)
+{
+  return strcasecmp(s1.c_str(), s2);
+}
+
+static inline int stringcasecmp(const string& s1, int ofs, int size, const string& s2)
+{
+  return strncasecmp(s1.c_str() + ofs, s2.c_str(), size);
+}
+
+static inline int stringtoll(const string& s, int64_t *val)
+{
+  char *end;
+
+  long long result = strtoll(s.c_str(), &end, 10);
+  if (result == LLONG_MAX)
+    return -EINVAL;
+
+  if (*end)
+    return -EINVAL;
+
+  *val = (int64_t)result;
+
+  return 0;
+}
+
+static inline int stringtoull(const string& s, uint64_t *val)
+{
+  char *end;
+
+  unsigned long long result = strtoull(s.c_str(), &end, 10);
+  if (result == ULLONG_MAX)
+    return -EINVAL;
+
+  if (*end)
+    return -EINVAL;
+
+  *val = (uint64_t)result;
+
+  return 0;
+}
+
+static inline int stringtol(const string& s, int32_t *val)
+{
+  char *end;
+
+  long result = strtol(s.c_str(), &end, 10);
+  if (result == LONG_MAX)
+    return -EINVAL;
+
+  if (*end)
+    return -EINVAL;
+
+  *val = (int32_t)result;
+
+  return 0;
+}
+
+static inline int stringtoul(const string& s, uint32_t *val)
+{
+  char *end;
+
+  unsigned long result = strtoul(s.c_str(), &end, 10);
+  if (result == ULONG_MAX)
+    return -EINVAL;
+
+  if (*end)
+    return -EINVAL;
+
+  *val = (uint32_t)result;
+
+  return 0;
+}
+
+#endif
diff --git a/ceph/src/rgw/rgw_swift.cc b/ceph/src/rgw/rgw_swift.cc
new file mode 100644
index 00000000..d9654a76
--- /dev/null
+++ b/ceph/src/rgw/rgw_swift.cc
@@ -0,0 +1,740 @@
+#include 
+#include 
+#include 
+
+#include "common/ceph_json.h"
+#include "rgw_common.h"
+#include "rgw_swift.h"
+#include "rgw_swift_auth.h"
+#include "rgw_user.h"
+#include "rgw_http_client.h"
+#include "rgw_keystone.h"
+
+#include "include/str_list.h"
+
+#include "common/ceph_crypto_cms.h"
+#include "common/armor.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+static list roles_list;
+
+class RGWValidateSwiftToken : public RGWHTTPClient {
+  struct rgw_swift_auth_info *info;
+
+protected:
+  RGWValidateSwiftToken() : RGWHTTPClient(NULL), info(NULL) {}
+public:
+  RGWValidateSwiftToken(CephContext *_cct, struct rgw_swift_auth_info *_info) : RGWHTTPClient(_cct), info(_info) {}
+
+  int receive_header(void *ptr, size_t len);
+  int receive_data(void *ptr, size_t len) {
+    return 0;
+  }
+  int send_data(void *ptr, size_t len) {
+    return 0;
+  }
+
+  friend class RGWKeystoneTokenCache;
+};
+
+int RGWValidateSwiftToken::receive_header(void *ptr, size_t len)
+{
+  char line[len + 1];
+
+  char *s = (char *)ptr, *end = (char *)ptr + len;
+  char *p = line;
+  ldout(cct, 10) << "read_http_header" << dendl;
+
+  while (s != end) {
+    if (*s == '\r') {
+      s++;
+      continue;
+    }
+    if (*s == '\n') {
+      *p = '\0';
+      ldout(cct, 10) << "os_auth:" << line << dendl;
+      // TODO: fill whatever data required here
+      char *l = line;
+      char *tok = strsep(&l, " \t:");
+      if (tok) {
+        while (l && *l == ' ')
+          l++;
+ 
+        if (strcmp(tok, "HTTP") == 0) {
+          info->status = atoi(l);
+        } else if (strcasecmp(tok, "X-Auth-Groups") == 0) {
+          info->auth_groups = l;
+          char *s = strchr(l, ',');
+          if (s) {
+            *s = '\0';
+            info->user = l;
+          }
+        } else if (strcasecmp(tok, "X-Auth-Ttl") == 0) {
+          info->ttl = atoll(l);
+        }
+      }
+    }
+    if (s != end)
+      *p++ = *s++;
+  }
+  return 0;
+}
+
+int RGWSwift::validate_token(const char *token, struct rgw_swift_auth_info *info)
+{
+  if (g_conf->rgw_swift_auth_url.empty())
+    return -EINVAL;
+
+  string auth_url = g_conf->rgw_swift_auth_url;
+  if (auth_url[auth_url.size() - 1] != '/')
+    auth_url.append("/");
+  auth_url.append("token");
+  char url_buf[auth_url.size() + 1 + strlen(token) + 1];
+  sprintf(url_buf, "%s/%s", auth_url.c_str(), token);
+
+  RGWValidateSwiftToken validate(cct, info);
+
+  ldout(cct, 10) << "rgw_swift_validate_token url=" << url_buf << dendl;
+
+  int ret = validate.process(url_buf);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+class RGWPostHTTPData : public RGWHTTPClient {
+  bufferlist *bl;
+  std::string post_data;
+  size_t post_data_index;
+public:
+  RGWPostHTTPData(CephContext *_cct, bufferlist *_bl) : RGWHTTPClient(_cct), bl(_bl), post_data_index(0) {}
+
+  void set_post_data(const std::string& _post_data) {
+    this->post_data = _post_data;
+  }
+
+  int send_data(void* ptr, size_t len) {
+    int length_to_copy = 0;
+    if (post_data_index < post_data.length()) {
+      length_to_copy = min(post_data.length() - post_data_index, len);
+      memcpy(ptr, post_data.data() + post_data_index, length_to_copy);
+      post_data_index += length_to_copy;
+    }
+    return length_to_copy;
+  }
+
+  int receive_data(void *ptr, size_t len) {
+    bl->append((char *)ptr, len);
+    return 0;
+  }
+
+  int receive_header(void *ptr, size_t len) {
+    return 0;
+  }
+};
+
+typedef RGWPostHTTPData RGWGetKeystoneAdminToken;
+typedef RGWPostHTTPData RGWGetRevokedTokens;
+
+class RGWValidateKeystoneToken : public RGWHTTPClient {
+  bufferlist *bl;
+public:
+  RGWValidateKeystoneToken(CephContext *_cct, bufferlist *_bl) : RGWHTTPClient(_cct), bl(_bl) {}
+
+  int receive_data(void *ptr, size_t len) {
+    bl->append((char *)ptr, len);
+    return 0;
+  }
+  int receive_header(void *ptr, size_t len) {
+    return 0;
+  }
+  int send_data(void *ptr, size_t len) {
+    return 0;
+  }
+
+};
+
+static RGWKeystoneTokenCache *keystone_token_cache = NULL;
+
+static int open_cms_envelope(CephContext *cct, string& src, string& dst)
+{
+#define BEGIN_CMS "-----BEGIN CMS-----"
+#define END_CMS "-----END CMS-----"
+
+  int start = src.find(BEGIN_CMS);
+  if (start < 0) {
+    ldout(cct, 0) << "failed to find " << BEGIN_CMS << " in response" << dendl;
+    return -EINVAL;
+  }
+  start += sizeof(BEGIN_CMS) - 1;
+
+  int end = src.find(END_CMS);
+  if (end < 0) {
+    ldout(cct, 0) << "failed to find " << END_CMS << " in response" << dendl;
+    return -EINVAL;
+  }
+
+  string s = src.substr(start, end - start);
+
+  int pos = 0;
+
+  do {
+    int next = s.find('\n', pos);
+    if (next < 0) {
+      dst.append(s.substr(pos));
+      break;
+    } else {
+      dst.append(s.substr(pos, next - pos));
+    }
+    pos = next + 1;
+  } while (pos < (int)s.size());
+
+  return 0;
+}
+
+static int decode_b64_cms(CephContext *cct, const string& signed_b64, bufferlist& bl)
+{
+  bufferptr signed_ber(signed_b64.size() * 2);
+  char *dest = signed_ber.c_str();
+  const char *src = signed_b64.c_str();
+  size_t len = signed_b64.size();
+  char buf[len + 1];
+  buf[len] = '\0';
+  for (size_t i = 0; i < len; i++, src++) {
+    if (*src != '-')
+      buf[i] = *src;
+    else
+      buf[i] = '/';
+  }
+  int ret = ceph_unarmor(dest, dest + signed_ber.length(), buf, buf + signed_b64.size());
+  if (ret < 0) {
+    ldout(cct, 0) << "ceph_unarmor() failed, ret=" << ret << dendl;
+    return ret;
+  }
+
+  bufferlist signed_ber_bl;
+  signed_ber_bl.append(signed_ber);
+
+  ret = ceph_decode_cms(cct, signed_ber_bl, bl);
+  if (ret < 0) {
+    ldout(cct, 0) << "ceph_decode_cms returned " << ret << dendl;
+    return ret;
+  }
+
+  return 0;
+}
+
+int	RGWSwift::get_keystone_url(std::string& url)
+{
+  bufferlist bl;
+  RGWGetRevokedTokens req(cct, &bl);
+
+  url = cct->_conf->rgw_keystone_url;
+  if (url.empty()) {
+    ldout(cct, 0) << "ERROR: keystone url is not configured" << dendl;
+    return -EINVAL;
+  }
+  if (url[url.size() - 1] != '/')
+    url.append("/");
+  return 0;
+}
+
+int	RGWSwift::get_keystone_admin_token(std::string& token)
+{
+  std::string token_url;
+
+  if (get_keystone_url(token_url) < 0)
+    return -EINVAL;
+  if (cct->_conf->rgw_keystone_admin_token.empty()) {
+    token_url.append("v2.0/tokens");
+    KeystoneToken t;
+    bufferlist token_bl;
+    RGWGetKeystoneAdminToken token_req(cct, &token_bl);
+    token_req.append_header("Content-Type", "application/json");
+    JSONFormatter jf;
+    jf.open_object_section("token_request");
+    jf.open_object_section("auth");
+    jf.open_object_section("passwordCredentials");
+    encode_json("username", cct->_conf->rgw_keystone_admin_user, &jf);
+    encode_json("password", cct->_conf->rgw_keystone_admin_password, &jf);
+    jf.close_section();
+    encode_json("tenantName", cct->_conf->rgw_keystone_admin_tenant, &jf);
+    jf.close_section();
+    jf.close_section();
+    std::stringstream ss;
+    jf.flush(ss);
+    token_req.set_post_data(ss.str());
+    int ret = token_req.process("POST", token_url.c_str());
+    if (ret < 0)
+      return ret;
+    if (t.parse(cct, token_bl) != 0)
+      return -EINVAL;
+    token = t.token.id;
+  } else {
+    token = cct->_conf->rgw_keystone_admin_token;
+  }
+  return 0; 
+}
+
+
+int RGWSwift::check_revoked()
+{
+  string url;
+  string token;
+
+  bufferlist bl;
+  RGWGetRevokedTokens req(cct, &bl);
+
+  if (get_keystone_admin_token(token) < 0)
+    return -EINVAL;
+  if (get_keystone_url(url) < 0)
+    return -EINVAL;
+  url.append("v2.0/tokens/revoked");
+  req.append_header("X-Auth-Token", token);
+  int ret = req.process(url.c_str());
+  if (ret < 0)
+    return ret;
+
+  bl.append((char)0); // NULL terminate for debug output
+
+  ldout(cct, 10) << "request returned " << bl.c_str() << dendl;
+
+  JSONParser parser;
+
+  if (!parser.parse(bl.c_str(), bl.length())) {
+    ldout(cct, 0) << "malformed json" << dendl;
+    return -EINVAL;
+  }
+
+  JSONObjIter iter = parser.find_first("signed");
+  if (iter.end()) {
+    ldout(cct, 0) << "revoked tokens response is missing signed section" << dendl;
+    return -EINVAL;
+  }  
+
+  JSONObj *signed_obj = *iter;
+
+  string signed_str = signed_obj->get_data();
+
+  ldout(cct, 10) << "signed=" << signed_str << dendl;
+
+  string signed_b64;
+  ret = open_cms_envelope(cct, signed_str, signed_b64);
+  if (ret < 0)
+    return ret;
+
+  ldout(cct, 10) << "content=" << signed_b64 << dendl;
+  
+  bufferlist json;
+  ret = decode_b64_cms(cct, signed_b64, json);
+  if (ret < 0) {
+    return ret;
+  }
+
+  ldout(cct, 10) << "ceph_decode_cms: decoded: " << json.c_str() << dendl;
+
+  JSONParser list_parser;
+  if (!list_parser.parse(json.c_str(), json.length())) {
+    ldout(cct, 0) << "malformed json" << dendl;
+    return -EINVAL;
+  }
+
+  JSONObjIter revoked_iter = list_parser.find_first("revoked");
+  if (revoked_iter.end()) {
+    ldout(cct, 0) << "no revoked section in json" << dendl;
+    return -EINVAL;
+  }
+
+  JSONObj *revoked_obj = *revoked_iter;
+
+  JSONObjIter tokens_iter = revoked_obj->find_first();
+  for (; !tokens_iter.end(); ++tokens_iter) {
+    JSONObj *o = *tokens_iter;
+
+    JSONObj *token = o->find_obj("id");
+    if (!token) {
+      ldout(cct, 0) << "bad token in array, missing id" << dendl;
+      continue;
+    }
+
+    string token_id = token->get_data();
+    keystone_token_cache->invalidate(token_id);
+  }
+  
+  return 0;
+}
+
+static void rgw_set_keystone_token_auth_info(KeystoneToken& token, struct rgw_swift_auth_info *info)
+{
+  info->user = token.token.tenant.id;
+  info->display_name = token.token.tenant.name;
+  info->status = 200;
+}
+
+int RGWSwift::parse_keystone_token_response(const string& token, bufferlist& bl, struct rgw_swift_auth_info *info, KeystoneToken& t)
+{
+  int ret = t.parse(cct, bl);
+  if (ret < 0)
+    return ret;
+
+  bool found = false;
+  list::iterator iter;
+  for (iter = roles_list.begin(); iter != roles_list.end(); ++iter) {
+    const string& role = *iter;
+    if ((found=t.user.has_role(role))==true)
+      break;
+  }
+
+  if (!found) {
+    ldout(cct, 0) << "user does not hold a matching role; required roles: " << g_conf->rgw_keystone_accepted_roles << dendl;
+    return -EPERM;
+  }
+
+  ldout(cct, 0) << "validated token: " << t.token.tenant.name << ":" << t.user.name << " expires: " << t.token.expires << dendl;
+
+  rgw_set_keystone_token_auth_info(t, info);
+
+  return 0;
+}
+
+int RGWSwift::update_user_info(RGWRados *store, struct rgw_swift_auth_info *info, RGWUserInfo& user_info)
+{
+  if (rgw_get_user_info_by_uid(store, info->user, user_info) < 0) {
+    ldout(cct, 0) << "NOTICE: couldn't map swift user" << dendl;
+    user_info.user_id = info->user;
+    user_info.display_name = info->display_name;
+
+    int ret = rgw_store_user_info(store, user_info, NULL, NULL, 0, true);
+    if (ret < 0) {
+      ldout(cct, 0) << "ERROR: failed to store new user's info: ret=" << ret << dendl;
+      return ret;
+    }
+  }
+  return 0;
+}
+
+#define PKI_ANS1_PREFIX "MII"
+
+static bool is_pki_token(const string& token)
+{
+  return token.compare(0, sizeof(PKI_ANS1_PREFIX) - 1, PKI_ANS1_PREFIX) == 0;
+}
+
+static void get_token_id(const string& token, string& token_id)
+{
+  if (!is_pki_token(token)) {
+    token_id = token;
+    return;
+  }
+
+  unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
+
+  MD5 hash;
+  hash.Update((const byte *)token.c_str(), token.size());
+  hash.Final(m);
+
+
+  char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
+  buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
+  token_id = calc_md5;
+}
+
+static bool decode_pki_token(CephContext *cct, const string& token, bufferlist& bl)
+{
+  if (!is_pki_token(token))
+    return false;
+
+  int ret = decode_b64_cms(cct, token, bl);
+  if (ret < 0)
+    return false;
+
+  ldout(cct, 20) << "successfully decoded pki token" << dendl;
+
+  return true;
+}
+
+int RGWSwift::validate_keystone_token(RGWRados *store, const string& token, struct rgw_swift_auth_info *info,
+				      RGWUserInfo& rgw_user)
+{
+  KeystoneToken t;
+
+  string token_id;
+  get_token_id(token, token_id);
+
+  ldout(cct, 20) << "token_id=" << token_id << dendl;
+
+  /* check cache first */
+  if (keystone_token_cache->find(token_id, t)) {
+    rgw_set_keystone_token_auth_info(t, info);
+
+    ldout(cct, 20) << "cached token.tenant.id=" << t.token.tenant.id << dendl;
+
+    int ret = update_user_info(store, info, rgw_user);
+    if (ret < 0)
+      return ret;
+
+    return 0;
+  }
+
+  bufferlist bl;
+
+  /* check if that's a self signed token that we can decode */
+  if (!decode_pki_token(cct, token, bl)) {
+
+    /* can't decode, just go to the keystone server for validation */
+
+    RGWValidateKeystoneToken validate(cct, &bl);
+
+    string url = g_conf->rgw_keystone_url;
+    if (url.empty()) {
+      ldout(cct, 0) << "ERROR: keystone url is not configured" << dendl;
+      return -EINVAL;
+    }
+    if (url[url.size() - 1] != '/')
+      url.append("/");
+    std::string admin_token;
+    if (get_keystone_admin_token(admin_token) < 0)
+      return -EINVAL;
+    if (get_keystone_url(url) < 0)
+      return -EINVAL;
+
+    url.append("v2.0/tokens/");
+    url.append(token);
+
+    validate.append_header("X-Auth-Token", admin_token);
+
+    int ret = validate.process(url.c_str());
+    if (ret < 0)
+      return ret;
+  }
+
+  bl.append((char)0); // NULL terminate for debug output
+
+  ldout(cct, 20) << "received response: " << bl.c_str() << dendl;
+
+  int ret = parse_keystone_token_response(token, bl, info, t);
+  if (ret < 0)
+    return ret;
+
+  ret = update_user_info(store, info, rgw_user);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int authenticate_temp_url(RGWRados *store, req_state *s)
+{
+  /* temp url requires bucket and object specified in the requets */
+  if (s->bucket_name_str.empty())
+    return -EPERM;
+
+  if (s->object_str.empty())
+    return -EPERM;
+
+  string temp_url_sig = s->info.args.get("temp_url_sig");
+  if (temp_url_sig.empty())
+    return -EPERM;
+
+  string temp_url_expires = s->info.args.get("temp_url_expires");
+  if (temp_url_expires.empty())
+    return -EPERM;
+
+  /* need to get user info of bucket owner */
+  RGWBucketInfo bucket_info;
+
+  int ret = store->get_bucket_info(NULL, s->bucket_name_str, bucket_info, NULL);
+  if (ret < 0)
+    return -EPERM;
+
+  dout(20) << "temp url user (bucket owner): " << bucket_info.owner << dendl;
+  if (rgw_get_user_info_by_uid(store, bucket_info.owner, s->user) < 0) {
+    return -EPERM;
+  }
+
+  if (s->user.temp_url_keys.empty()) {
+    dout(5) << "user does not have temp url key set, aborting" << dendl;
+    return -EPERM;
+  }
+
+  if (!s->info.method)
+    return -EPERM;
+
+  utime_t now = ceph_clock_now(g_ceph_context);
+
+  string err;
+  uint64_t expiration = (uint64_t)strict_strtoll(temp_url_expires.c_str(), 10, &err);
+  if (!err.empty()) {
+    dout(5) << "failed to parse temp_url_expires: " << err << dendl;
+    return -EPERM;
+  }
+  if (expiration <= (uint64_t)now.sec()) {
+    dout(5) << "temp url expired: " << expiration << " <= " << now.sec() << dendl;
+    return -EPERM;
+  }
+
+  /* strip the swift prefix from the uri */
+  int pos = g_conf->rgw_swift_url_prefix.find_last_not_of('/') + 1;
+  string object_path = s->info.request_uri.substr(pos + 1);
+  string str = string(s->info.method) + "\n" + temp_url_expires + "\n" + object_path;
+
+  dout(20) << "temp url signature (plain text): " << str << dendl;
+
+  map::iterator iter;
+  for (iter = s->user.temp_url_keys.begin(); iter != s->user.temp_url_keys.end(); ++iter) {
+    string& temp_url_key = iter->second;
+
+    if (temp_url_key.empty())
+      continue;
+
+    char dest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+    calc_hmac_sha1(temp_url_key.c_str(), temp_url_key.size(),
+                   str.c_str(), str.size(), dest);
+
+    char dest_str[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2 + 1];
+    buf_to_hex((const unsigned char *)dest, sizeof(dest), dest_str);
+    dout(20) << "temp url signature [" << iter->first << "] (calculated): " << dest_str << dendl;
+
+    if (dest_str != temp_url_sig) {
+      dout(5) << "temp url signature mismatch: " << dest_str << " != " << temp_url_sig << dendl;
+    } else {
+      return 0;
+    }
+  }
+
+  return -EPERM;
+}
+
+bool RGWSwift::verify_swift_token(RGWRados *store, req_state *s)
+{
+  if (!s->os_auth_token) {
+    int ret = authenticate_temp_url(store, s);
+    return (ret >= 0);
+  }
+
+  if (strncmp(s->os_auth_token, "AUTH_rgwtk", 10) == 0) {
+    int ret = rgw_swift_verify_signed_token(s->cct, store, s->os_auth_token, s->user);
+    if (ret < 0)
+      return false;
+
+    return  true;
+  }
+
+  struct rgw_swift_auth_info info;
+
+  info.status = 401; // start with access denied, validate_token might change that
+
+  int ret;
+
+  if (supports_keystone()) {
+    ret = validate_keystone_token(store, s->os_auth_token, &info, s->user);
+    return (ret >= 0);
+  }
+
+  ret = validate_token(s->os_auth_token, &info);
+  if (ret < 0)
+    return false;
+
+  if (info.user.empty()) {
+    ldout(cct, 5) << "swift auth didn't authorize a user" << dendl;
+    return false;
+  }
+
+  s->swift_user = info.user;
+  s->swift_groups = info.auth_groups;
+
+  string swift_user = s->swift_user;
+
+  ldout(cct, 10) << "swift user=" << s->swift_user << dendl;
+
+  if (rgw_get_user_info_by_swift(store, swift_user, s->user) < 0) {
+    ldout(cct, 0) << "NOTICE: couldn't map swift user" << dendl;
+    return false;
+  }
+
+  ldout(cct, 10) << "user_id=" << s->user.user_id << dendl;
+
+  return true;
+}
+
+void RGWSwift::init()
+{
+  get_str_list(cct->_conf->rgw_keystone_accepted_roles, roles_list);
+  if (supports_keystone())
+      init_keystone();
+}
+
+
+void RGWSwift::init_keystone()
+{
+  keystone_token_cache = new RGWKeystoneTokenCache(cct, cct->_conf->rgw_keystone_token_cache_size);
+
+  keystone_revoke_thread = new KeystoneRevokeThread(cct, this);
+  keystone_revoke_thread->create();
+}
+
+
+void RGWSwift::finalize()
+{
+  if (supports_keystone())
+    finalize_keystone();
+}
+
+void RGWSwift::finalize_keystone()
+{
+  delete keystone_token_cache;
+  keystone_token_cache = NULL;
+
+  down_flag.set(1);
+  if (keystone_revoke_thread) {
+    keystone_revoke_thread->stop();
+    keystone_revoke_thread->join();
+  }
+  delete keystone_revoke_thread;
+  keystone_revoke_thread = NULL;
+}
+
+RGWSwift *rgw_swift = NULL;
+
+void swift_init(CephContext *cct)
+{
+  rgw_swift = new RGWSwift(cct);
+}
+
+void swift_finalize()
+{
+  delete rgw_swift;
+}
+
+bool RGWSwift::going_down()
+{
+  return (down_flag.read() != 0);
+}
+
+void *RGWSwift::KeystoneRevokeThread::entry() {
+  do {
+    dout(2) << "keystone revoke thread: start" << dendl;
+    int r = swift->check_revoked();
+    if (r < 0) {
+      dout(0) << "ERROR: keystone revocation processing returned error r=" << r << dendl;
+    }
+
+    if (swift->going_down())
+      break;
+
+    lock.Lock();
+    cond.WaitInterval(cct, lock, utime_t(cct->_conf->rgw_keystone_revocation_interval, 0));
+    lock.Unlock();
+  } while (!swift->going_down());
+
+  return NULL;
+}
+
+void RGWSwift::KeystoneRevokeThread::stop()
+{
+  Mutex::Locker l(lock);
+  cond.Signal();
+}
+
diff --git a/ceph/src/rgw/rgw_swift.h b/ceph/src/rgw/rgw_swift.h
new file mode 100644
index 00000000..97347e80
--- /dev/null
+++ b/ceph/src/rgw/rgw_swift.h
@@ -0,0 +1,76 @@
+
+#ifndef CEPH_RGW_SWIFT_H
+#define CEPH_RGW_SWIFT_H
+
+#include "rgw_common.h"
+#include "common/Cond.h"
+
+class RGWRados;
+class KeystoneToken;
+
+struct rgw_swift_auth_info {
+  int status;
+  string auth_groups;
+  string user;
+  string display_name;
+  long long ttl;
+
+  rgw_swift_auth_info() : status(0), ttl(0) {}
+};
+
+class RGWSwift {
+  CephContext *cct;
+  atomic_t down_flag;
+
+  int validate_token(const char *token, struct rgw_swift_auth_info *info);
+  int validate_keystone_token(RGWRados *store, const string& token, struct rgw_swift_auth_info *info,
+			      RGWUserInfo& rgw_user);
+
+  int parse_keystone_token_response(const string& token, bufferlist& bl, struct rgw_swift_auth_info *info,
+		                    KeystoneToken& t);
+  int update_user_info(RGWRados *store, struct rgw_swift_auth_info *info, RGWUserInfo& user_info);
+  int get_keystone_url(std::string& url);
+  int get_keystone_admin_token(std::string& token);
+
+  class KeystoneRevokeThread : public Thread {
+    CephContext *cct;
+    RGWSwift *swift;
+    Mutex lock;
+    Cond cond;
+
+  public:
+    KeystoneRevokeThread(CephContext *_cct, RGWSwift *_swift) : cct(_cct), swift(_swift), lock("KeystoneRevokeThread") {}
+    void *entry();
+    void stop();
+  };
+
+  KeystoneRevokeThread *keystone_revoke_thread;
+
+  void init();
+  void finalize();
+  void init_keystone();
+  void finalize_keystone();
+  bool supports_keystone() {
+    return !cct->_conf->rgw_keystone_url.empty();
+  }
+protected:
+  int check_revoked();
+public:
+
+  RGWSwift(CephContext *_cct) : cct(_cct), keystone_revoke_thread(NULL) {
+    init();
+  }
+  ~RGWSwift() {
+    finalize();
+  }
+
+  bool verify_swift_token(RGWRados *store, req_state *s);
+  bool going_down();
+};
+
+extern RGWSwift *rgw_swift;
+void swift_init(CephContext *cct);
+void swift_finalize();
+
+#endif
+
diff --git a/ceph/src/rgw/rgw_swift_auth.cc b/ceph/src/rgw/rgw_swift_auth.cc
new file mode 100644
index 00000000..9c800c4c
--- /dev/null
+++ b/ceph/src/rgw/rgw_swift_auth.cc
@@ -0,0 +1,248 @@
+#include "rgw_swift_auth.h"
+#include "rgw_rest.h"
+
+#include "common/ceph_crypto.h"
+#include "common/Clock.h"
+
+#include "auth/Crypto.h"
+
+#include "rgw_client_io.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+#define DEFAULT_SWIFT_PREFIX "swift"
+
+using namespace ceph::crypto;
+
+static int build_token(string& swift_user, string& key, uint64_t nonce, utime_t& expiration, bufferlist& bl)
+{
+  ::encode(swift_user, bl);
+  ::encode(nonce, bl);
+  ::encode(expiration, bl);
+
+  bufferptr p(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+
+  char buf[bl.length() * 2 + 1];
+  buf_to_hex((const unsigned char *)bl.c_str(), bl.length(), buf);
+  dout(20) << "build_token token=" << buf << dendl;
+
+  char k[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+  memset(k, 0, sizeof(k));
+  const char *s = key.c_str();
+  for (int i = 0; i < (int)key.length(); i++, s++) {
+    k[i % CEPH_CRYPTO_HMACSHA1_DIGESTSIZE] |= *s;
+  }
+  calc_hmac_sha1(k, sizeof(k), bl.c_str(), bl.length(), p.c_str());
+
+  bl.append(p);
+
+  return 0;
+
+}
+
+static int encode_token(CephContext *cct, string& swift_user, string& key, bufferlist& bl)
+{
+  uint64_t nonce;
+
+  int ret = get_random_bytes((char *)&nonce, sizeof(nonce));
+  if (ret < 0)
+    return ret;
+
+  utime_t expiration = ceph_clock_now(cct);
+  expiration += cct->_conf->rgw_swift_token_expiration;
+
+  ret = build_token(swift_user, key, nonce, expiration, bl);
+
+  return ret;
+}
+
+int rgw_swift_verify_signed_token(CephContext *cct, RGWRados *store, const char *token, RGWUserInfo& info)
+{
+  if (strncmp(token, "AUTH_rgwtk", 10) != 0)
+    return -EINVAL;
+
+  token += 10;
+
+  int len = strlen(token);
+  if (len & 1) {
+    dout(0) << "NOTICE: failed to verify token: invalid token length len=" << len << dendl;
+    return -EINVAL;
+  }
+
+  bufferptr p(len/2);
+  int ret = hex_to_buf(token, p.c_str(), len);
+  if (ret < 0)
+    return ret;
+
+  bufferlist bl;
+  bl.append(p);
+
+  bufferlist::iterator iter = bl.begin();
+
+  uint64_t nonce;
+  utime_t expiration;
+  string swift_user;
+
+  try {
+    ::decode(swift_user, iter);
+    ::decode(nonce, iter);
+    ::decode(expiration, iter);
+  } catch (buffer::error& err) {
+    dout(0) << "NOTICE: failed to decode token: caught exception" << dendl;
+    return -EINVAL;
+  }
+  utime_t now = ceph_clock_now(cct);
+  if (expiration < now) {
+    dout(0) << "NOTICE: old timed out token was used now=" << now << " token.expiration=" << expiration << dendl;
+    return -EPERM;
+  }
+
+  if ((ret = rgw_get_user_info_by_swift(store, swift_user, info)) < 0)
+    return ret;
+
+  dout(10) << "swift_user=" << swift_user << dendl;
+
+  map::iterator siter = info.swift_keys.find(swift_user);
+  if (siter == info.swift_keys.end())
+    return -EPERM;
+  RGWAccessKey& swift_key = siter->second;
+
+  bufferlist tok;
+  ret = build_token(swift_user, swift_key.key, nonce, expiration, tok);
+  if (ret < 0)
+    return ret;
+
+  if (tok.length() != bl.length()) {
+    dout(0) << "NOTICE: tokens length mismatch: bl.length()=" << bl.length() << " tok.length()=" << tok.length() << dendl;
+    return -EPERM;
+  }
+
+  if (memcmp(tok.c_str(), bl.c_str(), tok.length()) != 0) {
+    char buf[tok.length() * 2 + 1];
+    buf_to_hex((const unsigned char *)tok.c_str(), tok.length(), buf);
+    dout(0) << "NOTICE: tokens mismatch tok=" << buf << dendl;
+    return -EPERM;
+  }
+
+  return 0;
+}
+
+void RGW_SWIFT_Auth_Get::execute()
+{
+  int ret = -EPERM;
+
+  const char *key = s->info.env->get("HTTP_X_AUTH_KEY");
+  const char *user = s->info.env->get("HTTP_X_AUTH_USER");
+
+  string user_str;
+  RGWUserInfo info;
+  bufferlist bl;
+  RGWAccessKey *swift_key;
+  map::iterator siter;
+
+  string swift_url = g_conf->rgw_swift_url;
+  string swift_prefix = g_conf->rgw_swift_url_prefix;
+  string tenant_path;
+
+  if (swift_prefix.size() == 0) {
+    swift_prefix = DEFAULT_SWIFT_PREFIX;
+  }
+
+  if (swift_url.size() == 0) {
+    bool add_port = false;
+    const char *server_port = s->info.env->get("SERVER_PORT_SECURE");
+    const char *protocol;
+    if (server_port) {
+      add_port = (strcmp(server_port, "443") != 0);
+      protocol = "https";
+    } else {
+      server_port = s->info.env->get("SERVER_PORT");
+      add_port = (strcmp(server_port, "80") != 0);
+      protocol = "http";
+    }
+    const char *host = s->info.env->get("HTTP_HOST");
+    if (!host) {
+      dout(0) << "NOTICE: server is misconfigured, missing rgw_swift_url_prefix or rgw_swift_url, HTTP_HOST is not set" << dendl;
+      ret = -EINVAL;
+      goto done;
+    }
+    swift_url = protocol;
+    swift_url.append("://");
+    swift_url.append(host);
+    if (add_port && !strchr(host, ':')) {
+      swift_url.append(":");
+      swift_url.append(server_port);
+    }
+  }
+
+
+  if (!key || !user)
+    goto done;
+
+  user_str = user;
+
+  if ((ret = rgw_get_user_info_by_swift(store, user_str, info)) < 0)
+  {
+    ret = -EACCES;
+    goto done;
+  }
+
+  siter = info.swift_keys.find(user_str);
+  if (siter == info.swift_keys.end()) {
+    ret = -EPERM;
+    goto done;
+  }
+  swift_key = &siter->second;
+
+  if (swift_key->key.compare(key) != 0) {
+    dout(0) << "NOTICE: RGW_SWIFT_Auth_Get::execute(): bad swift key" << dendl;
+    ret = -EPERM;
+    goto done;
+  }
+
+  if (!g_conf->rgw_swift_tenant_name.empty()) {
+    tenant_path = "/AUTH_";
+    tenant_path.append(g_conf->rgw_swift_tenant_name);
+  }
+
+  s->cio->print("X-Storage-Url: %s/%s/v1%s\n", swift_url.c_str(),
+	        swift_prefix.c_str(), tenant_path.c_str());
+
+  if ((ret = encode_token(s->cct, swift_key->id, swift_key->key, bl)) < 0)
+    goto done;
+
+  {
+    char buf[bl.length() * 2 + 1];
+    buf_to_hex((const unsigned char *)bl.c_str(), bl.length(), buf);
+
+    s->cio->print("X-Storage-Token: AUTH_rgwtk%s\n", buf);
+    s->cio->print("X-Auth-Token: AUTH_rgwtk%s\n", buf);
+  }
+
+  ret = STATUS_NO_CONTENT;
+
+done:
+  set_req_state_err(s, ret);
+  dump_errno(s);
+  end_header(s);
+}
+
+int RGWHandler_SWIFT_Auth::init(RGWRados *store, struct req_state *state, RGWClientIO *cio)
+{
+  state->dialect = "swift-auth";
+  state->formatter = new JSONFormatter;
+  state->format = RGW_FORMAT_JSON;
+
+  return RGWHandler::init(store, state, cio);
+}
+
+int RGWHandler_SWIFT_Auth::authorize()
+{
+  return 0;
+}
+
+RGWOp *RGWHandler_SWIFT_Auth::op_get()
+{
+  return new RGW_SWIFT_Auth_Get;
+}
+
diff --git a/ceph/src/rgw/rgw_swift_auth.h b/ceph/src/rgw/rgw_swift_auth.h
new file mode 100644
index 00000000..6d50d945
--- /dev/null
+++ b/ceph/src/rgw/rgw_swift_auth.h
@@ -0,0 +1,49 @@
+#ifndef CEPH_RGW_SWIFT_AUTH_H
+#define CEPH_RGW_SWIFT_AUTH_H
+
+#include "rgw_op.h"
+#include "rgw_rest.h"
+
+#define RGW_SWIFT_TOKEN_EXPIRATION (15 * 60)
+
+extern int rgw_swift_verify_signed_token(CephContext *cct, RGWRados *store, const char *token, RGWUserInfo& info);
+
+class RGW_SWIFT_Auth_Get : public RGWOp {
+public:
+  RGW_SWIFT_Auth_Get() {}
+  ~RGW_SWIFT_Auth_Get() {}
+
+  int verify_permission() { return 0; }
+  void execute();
+  virtual const string name() { return "swift_auth_get"; }
+};
+
+class RGWHandler_SWIFT_Auth : public RGWHandler {
+public:
+  RGWHandler_SWIFT_Auth() {}
+  ~RGWHandler_SWIFT_Auth() {}
+  RGWOp *op_get();
+
+  int init(RGWRados *store, struct req_state *state, RGWClientIO *cio);
+  int authorize();
+  int read_permissions(RGWOp *op) { return 0; }
+
+  virtual RGWAccessControlPolicy *alloc_policy() { return NULL; }
+  virtual void free_policy(RGWAccessControlPolicy *policy) {}
+};
+
+class RGWRESTMgr_SWIFT_Auth : public RGWRESTMgr {
+public:
+  RGWRESTMgr_SWIFT_Auth() {}
+  virtual ~RGWRESTMgr_SWIFT_Auth() {}
+
+  virtual RGWRESTMgr *get_resource_mgr(struct req_state *s, const string& uri, string *out_uri) {
+    return this;
+  }
+  virtual RGWHandler *get_handler(struct req_state *s) {
+    return new RGWHandler_SWIFT_Auth;
+  }
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_tools.cc b/ceph/src/rgw/rgw_tools.cc
new file mode 100644
index 00000000..49894eae
--- /dev/null
+++ b/ceph/src/rgw/rgw_tools.cc
@@ -0,0 +1,170 @@
+#include 
+
+#include "common/errno.h"
+
+#include "include/types.h"
+
+#include "rgw_common.h"
+#include "rgw_rados.h"
+#include "rgw_tools.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+#define READ_CHUNK_LEN (512 * 1024)
+
+static map ext_mime_map;
+
+int rgw_put_system_obj(RGWRados *rgwstore, rgw_bucket& bucket, string& oid, const char *data, size_t size, bool exclusive,
+                       RGWObjVersionTracker *objv_tracker, time_t set_mtime, map *pattrs)
+{
+  map no_attrs;
+  if (!pattrs)
+    pattrs = &no_attrs;
+
+  rgw_obj obj(bucket, oid);
+
+  int ret = rgwstore->put_system_obj(NULL, obj, data, size, exclusive, NULL, *pattrs, objv_tracker, set_mtime);
+
+  if (ret == -ENOENT) {
+    ret = rgwstore->create_pool(bucket);
+    if (ret >= 0)
+      ret = rgwstore->put_system_obj(NULL, obj, data, size, exclusive, NULL, *pattrs, objv_tracker, set_mtime);
+  }
+
+  return ret;
+}
+
+int rgw_get_system_obj(RGWRados *rgwstore, void *ctx, rgw_bucket& bucket, const string& key, bufferlist& bl,
+                       RGWObjVersionTracker *objv_tracker, time_t *pmtime, map *pattrs)
+{
+  struct rgw_err err;
+  void *handle = NULL;
+  bufferlist::iterator iter;
+  int request_len = READ_CHUNK_LEN;
+  rgw_obj obj(bucket, key);
+
+  do {
+    int ret = rgwstore->prepare_get_obj(ctx, obj, NULL, NULL, pattrs, NULL,
+                                        NULL, pmtime, NULL, NULL, NULL, NULL,
+                                        objv_tracker, &handle, &err);
+    if (ret < 0)
+      return ret;
+
+    ret = rgwstore->get_obj(ctx, objv_tracker, &handle, obj, bl, 0, request_len - 1);
+    rgwstore->finish_get_obj(&handle);
+    if (ret < 0)
+      return ret;
+
+    if (ret < request_len)
+      break;
+    bl.clear();
+    request_len *= 2;
+  } while (true);
+
+  return 0;
+}
+
+void parse_mime_map_line(const char *start, const char *end)
+{
+  char line[end - start + 1];
+  strncpy(line, start, end - start);
+  line[end - start] = '\0';
+  char *l = line;
+#define DELIMS " \t\n\r"
+
+  while (isspace(*l))
+    l++;
+
+  char *mime = strsep(&l, DELIMS);
+  if (!mime)
+    return;
+
+  char *ext;
+  do {
+    ext = strsep(&l, DELIMS);
+    if (ext && *ext) {
+      ext_mime_map[ext] = mime;
+    }
+  } while (ext);
+}
+
+
+void parse_mime_map(const char *buf)
+{
+  const char *start = buf, *end = buf;
+  while (*end) {
+    while (*end && *end != '\n') {
+      end++;
+    }
+    parse_mime_map_line(start, end);
+    end++;
+    start = end;
+  }
+}
+
+static int ext_mime_map_init(CephContext *cct, const char *ext_map)
+{
+  int fd = open(ext_map, O_RDONLY);
+  char *buf = NULL;
+  int ret;
+  if (fd < 0) {
+    ret = -errno;
+    ldout(cct, 0) << "ext_mime_map_init(): failed to open file=" << ext_map << " ret=" << ret << dendl;
+    return ret;
+  }
+
+  struct stat st;
+  ret = fstat(fd, &st);
+  if (ret < 0) {
+    ret = -errno;
+    ldout(cct, 0) << "ext_mime_map_init(): failed to stat file=" << ext_map << " ret=" << ret << dendl;
+    goto done;
+  }
+
+  buf = (char *)malloc(st.st_size + 1);
+  if (!buf) {
+    ret = -ENOMEM;
+    ldout(cct, 0) << "ext_mime_map_init(): failed to allocate buf" << dendl;
+    goto done;
+  }
+
+  ret = read(fd, buf, st.st_size + 1);
+  if (ret != st.st_size) {
+    // huh? file size has changed, what are the odds?
+    ldout(cct, 0) << "ext_mime_map_init(): raced! will retry.." << dendl;
+    free(buf);
+    close(fd);
+    return ext_mime_map_init(cct, ext_map);
+  }
+  buf[st.st_size] = '\0';
+
+  parse_mime_map(buf);
+  ret = 0;
+done:
+  free(buf);
+  close(fd);
+  return ret;
+}
+
+const char *rgw_find_mime_by_ext(string& ext)
+{
+  map::iterator iter = ext_mime_map.find(ext);
+  if (iter == ext_mime_map.end())
+    return NULL;
+
+  return iter->second.c_str();
+}
+
+int rgw_tools_init(CephContext *cct)
+{
+  int ret = ext_mime_map_init(cct, cct->_conf->rgw_mime_types_file.c_str());
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+void rgw_tools_cleanup()
+{
+  ext_mime_map.clear();
+}
diff --git a/ceph/src/rgw/rgw_tools.h b/ceph/src/rgw/rgw_tools.h
new file mode 100644
index 00000000..9adaee48
--- /dev/null
+++ b/ceph/src/rgw/rgw_tools.h
@@ -0,0 +1,23 @@
+#ifndef CEPH_RGW_TOOLS_H
+#define CEPH_RGW_TOOLS_H
+
+#include 
+
+#include "include/types.h"
+#include "rgw_common.h"
+
+class RGWRados;
+struct RGWObjVersionTracker;
+
+struct obj_version;
+
+int rgw_put_system_obj(RGWRados *rgwstore, rgw_bucket& bucket, string& oid, const char *data, size_t size, bool exclusive,
+                       RGWObjVersionTracker *objv_tracker, time_t set_mtime, map *pattrs = NULL);
+int rgw_get_system_obj(RGWRados *rgwstore, void *ctx, rgw_bucket& bucket, const string& key, bufferlist& bl,
+                       RGWObjVersionTracker *objv_tracker, time_t *pmtime, map *pattrs = NULL);
+
+int rgw_tools_init(CephContext *cct);
+void rgw_tools_cleanup();
+const char *rgw_find_mime_by_ext(string& ext);
+
+#endif
diff --git a/ceph/src/rgw/rgw_usage.cc b/ceph/src/rgw/rgw_usage.cc
new file mode 100644
index 00000000..f5016913
--- /dev/null
+++ b/ceph/src/rgw/rgw_usage.cc
@@ -0,0 +1,140 @@
+
+#include 
+#include 
+
+#include "rgw_rados.h"
+#include "rgw_usage.h"
+#include "rgw_formats.h"
+
+using namespace std;
+
+
+static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log_entry& entry, map *categories)
+{
+  formatter->open_array_section("categories");
+  map::const_iterator uiter;
+  for (uiter = entry.usage_map.begin(); uiter != entry.usage_map.end(); ++uiter) {
+    if (categories && !categories->empty() && !categories->count(uiter->first))
+      continue;
+    const rgw_usage_data& usage = uiter->second;
+    formatter->open_object_section("entry");
+    formatter->dump_string("category", uiter->first);
+    formatter->dump_int("bytes_sent", usage.bytes_sent);
+    formatter->dump_int("bytes_received", usage.bytes_received);
+    formatter->dump_int("ops", usage.ops);
+    formatter->dump_int("successful_ops", usage.successful_ops);
+    formatter->close_section(); // entry
+  }
+  formatter->close_section(); // categories
+}
+
+int RGWUsage::show(RGWRados *store, string& uid, uint64_t start_epoch,
+		   uint64_t end_epoch, bool show_log_entries, bool show_log_sum,
+		   map *categories,
+		   RGWFormatterFlusher& flusher)
+{
+  uint32_t max_entries = 1000;
+
+  bool is_truncated = true;
+
+  RGWUsageIter usage_iter;
+  Formatter *formatter = flusher.get_formatter();
+
+  map usage;
+
+  flusher.start(0);
+
+  formatter->open_object_section("usage");
+  if (show_log_entries) {
+    formatter->open_array_section("entries");
+  }
+  string last_owner;
+  bool user_section_open = false;
+  map summary_map;
+  while (is_truncated) {
+    int ret = store->read_usage(uid, start_epoch, end_epoch, max_entries,
+                                &is_truncated, usage_iter, usage);
+
+    if (ret == -ENOENT) {
+      ret = 0;
+      is_truncated = false;
+    }
+
+    if (ret < 0) {
+      return ret;
+    }
+
+    map::iterator iter;
+    for (iter = usage.begin(); iter != usage.end(); ++iter) {
+      const rgw_user_bucket& ub = iter->first;
+      const rgw_usage_log_entry& entry = iter->second;
+
+      if (show_log_entries) {
+        if (ub.user.compare(last_owner) != 0) {
+          if (user_section_open) {
+            formatter->close_section();
+            formatter->close_section();
+          }
+          formatter->open_object_section("user");
+          formatter->dump_string("owner", ub.user);
+          formatter->open_array_section("buckets");
+          user_section_open = true;
+          last_owner = ub.user;
+        }
+        formatter->open_object_section("bucket");
+        formatter->dump_string("bucket", ub.bucket);
+        utime_t ut(entry.epoch, 0);
+        ut.gmtime(formatter->dump_stream("time"));
+        formatter->dump_int("epoch", entry.epoch);
+        dump_usage_categories_info(formatter, entry, categories);
+        formatter->close_section(); // bucket
+        flusher.flush();
+      }
+
+      summary_map[ub.user].aggregate(entry, categories);
+    }
+  }
+  if (show_log_entries) {
+    if (user_section_open) {
+      formatter->close_section(); // buckets
+      formatter->close_section(); //user
+    }
+    formatter->close_section(); // entries
+  }
+
+  if (show_log_sum) {
+    formatter->open_array_section("summary");
+    map::iterator siter;
+    for (siter = summary_map.begin(); siter != summary_map.end(); ++siter) {
+      const rgw_usage_log_entry& entry = siter->second;
+      formatter->open_object_section("user");
+      formatter->dump_string("user", siter->first);
+      dump_usage_categories_info(formatter, entry, categories);
+      rgw_usage_data total_usage;
+      entry.sum(total_usage, *categories);
+      formatter->open_object_section("total");
+      formatter->dump_int("bytes_sent", total_usage.bytes_sent);
+      formatter->dump_int("bytes_received", total_usage.bytes_received);
+      formatter->dump_int("ops", total_usage.ops);
+      formatter->dump_int("successful_ops", total_usage.successful_ops);
+      formatter->close_section(); // total
+
+      formatter->close_section(); // user
+
+      flusher.flush();
+    }
+
+    formatter->close_section(); // summary
+  }
+
+  formatter->close_section(); // usage
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWUsage::trim(RGWRados *store, string& uid, uint64_t start_epoch,
+		   uint64_t end_epoch)
+{
+  return store->trim_usage(uid, start_epoch, end_epoch);
+}
diff --git a/ceph/src/rgw/rgw_usage.h b/ceph/src/rgw/rgw_usage.h
new file mode 100644
index 00000000..76ae0f54
--- /dev/null
+++ b/ceph/src/rgw/rgw_usage.h
@@ -0,0 +1,26 @@
+#ifndef CEPH_RGW_USAGE_H
+#define CEPH_RGW_USAGE_H
+
+#include 
+#include 
+
+#include "common/Formatter.h"
+#include "rgw_formats.h"
+
+class RGWRados;
+
+
+class RGWUsage
+{
+public:
+  static int show(RGWRados *store, std::string& uid, uint64_t start_epoch,
+	          uint64_t end_epoch, bool show_log_entries, bool show_log_sum,
+		  std::map *categories,
+	          RGWFormatterFlusher& flusher);
+
+  static int trim(RGWRados *store, std::string& uid, uint64_t start_epoch,
+	          uint64_t end_epoch);
+};
+
+
+#endif
diff --git a/ceph/src/rgw/rgw_user.cc b/ceph/src/rgw/rgw_user.cc
new file mode 100644
index 00000000..23575d8a
--- /dev/null
+++ b/ceph/src/rgw/rgw_user.cc
@@ -0,0 +1,2479 @@
+#include 
+
+#include 
+#include 
+
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/ceph_json.h"
+#include "rgw_rados.h"
+#include "rgw_acl.h"
+
+#include "include/types.h"
+#include "rgw_user.h"
+#include "rgw_string.h"
+
+// until everything is moved from rgw_common
+#include "rgw_common.h"
+
+#include "rgw_bucket.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+
+static RGWMetadataHandler *user_meta_handler = NULL;
+
+
+/**
+ * Get the anonymous (ie, unauthenticated) user info.
+ */
+void rgw_get_anon_user(RGWUserInfo& info)
+{
+  info.user_id = RGW_USER_ANON_ID;
+  info.display_name.clear();
+  info.access_keys.clear();
+}
+
+bool rgw_user_is_authenticated(RGWUserInfo& info)
+{
+  return (info.user_id != RGW_USER_ANON_ID);
+}
+
+int rgw_user_sync_all_stats(RGWRados *store, const string& user_id)
+{
+  CephContext *cct = store->ctx();
+  size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+  bool done;
+  string marker;
+  int ret;
+
+  do {
+    RGWUserBuckets user_buckets;
+    ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, max_entries, false);
+    if (ret < 0) {
+      ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl;
+      return ret;
+    }
+    map& buckets = user_buckets.get_buckets();
+    for (map::iterator i = buckets.begin();
+         i != buckets.end();
+         ++i) {
+      marker = i->first;
+
+      RGWBucketEnt& bucket_ent = i->second;
+      ret = rgw_bucket_sync_user_stats(store, user_id, bucket_ent.bucket);
+      if (ret < 0) {
+        ldout(cct, 0) << "ERROR: could not sync bucket stats: ret=" << ret << dendl;
+        return ret;
+      }
+    }
+    done = (buckets.size() < max_entries);
+  } while (!done);
+
+  ret = store->complete_sync_user_stats(user_id);
+  if (ret < 0) {
+    cerr << "ERROR: failed to complete syncing user stats: ret=" << ret << std::endl;
+    return ret;
+  }
+
+  return 0;
+}
+
+/**
+ * Save the given user information to storage.
+ * Returns: 0 on success, -ERR# on failure.
+ */
+int rgw_store_user_info(RGWRados *store, RGWUserInfo& info, RGWUserInfo *old_info,
+                        RGWObjVersionTracker *objv_tracker, time_t mtime, bool exclusive)
+{
+  bufferlist bl;
+  info.encode(bl);
+  int ret;
+  RGWObjVersionTracker ot;
+
+  if (objv_tracker) {
+    ot = *objv_tracker;
+  }
+
+  if (ot.write_version.tag.empty()) {
+    if (ot.read_version.tag.empty()) {
+      ot.generate_new_write_ver(store->ctx());
+    } else {
+      ot.write_version = ot.read_version;
+      ot.write_version.ver++;
+    }
+  }
+
+  map::iterator iter;
+  for (iter = info.swift_keys.begin(); iter != info.swift_keys.end(); ++iter) {
+    if (old_info && old_info->swift_keys.count(iter->first) != 0)
+      continue;
+    RGWAccessKey& k = iter->second;
+    /* check if swift mapping exists */
+    RGWUserInfo inf;
+    int r = rgw_get_user_info_by_swift(store, k.id, inf);
+    if (r >= 0 && inf.user_id.compare(info.user_id) != 0) {
+      ldout(store->ctx(), 0) << "WARNING: can't store user info, swift id (" << k.id
+        << ") already mapped to another user (" << info.user_id << ")" << dendl;
+      return -EEXIST;
+    }
+  }
+
+  if (!info.access_keys.empty()) {
+    /* check if access keys already exist */
+    RGWUserInfo inf;
+    map::iterator iter = info.access_keys.begin();
+    for (; iter != info.access_keys.end(); ++iter) {
+      RGWAccessKey& k = iter->second;
+      if (old_info && old_info->access_keys.count(iter->first) != 0)
+        continue;
+      int r = rgw_get_user_info_by_access_key(store, k.id, inf);
+      if (r >= 0 && inf.user_id.compare(info.user_id) != 0) {
+        ldout(store->ctx(), 0) << "WARNING: can't store user info, access key already mapped to another user" << dendl;
+        return -EEXIST;
+      }
+    }
+  }
+
+  RGWUID ui;
+  ui.user_id = info.user_id;
+
+  bufferlist link_bl;
+  ::encode(ui, link_bl);
+
+  bufferlist data_bl;
+  ::encode(ui, data_bl);
+  ::encode(info, data_bl);
+
+  ret = store->meta_mgr->put_entry(user_meta_handler, info.user_id, data_bl, exclusive, &ot, mtime);
+  if (ret < 0)
+    return ret;
+
+  if (!info.user_email.empty()) {
+    if (!old_info ||
+        old_info->user_email.compare(info.user_email) != 0) { /* only if new index changed */
+      ret = rgw_put_system_obj(store, store->zone.user_email_pool, info.user_email,
+                               link_bl.c_str(), link_bl.length(), exclusive, NULL, 0);
+      if (ret < 0)
+        return ret;
+    }
+  }
+
+  if (!info.access_keys.empty()) {
+    map::iterator iter = info.access_keys.begin();
+    for (; iter != info.access_keys.end(); ++iter) {
+      RGWAccessKey& k = iter->second;
+      if (old_info && old_info->access_keys.count(iter->first) != 0)
+	continue;
+
+      ret = rgw_put_system_obj(store, store->zone.user_keys_pool, k.id,
+                               link_bl.c_str(), link_bl.length(), exclusive,
+                               NULL, 0);
+      if (ret < 0)
+        return ret;
+    }
+  }
+
+  map::iterator siter;
+  for (siter = info.swift_keys.begin(); siter != info.swift_keys.end(); ++siter) {
+    RGWAccessKey& k = siter->second;
+    if (old_info && old_info->swift_keys.count(siter->first) != 0)
+      continue;
+
+    ret = rgw_put_system_obj(store, store->zone.user_swift_pool, k.id,
+                             link_bl.c_str(), link_bl.length(), exclusive,
+                             NULL, 0);
+    if (ret < 0)
+      return ret;
+  }
+
+  return ret;
+}
+
+int rgw_get_user_info_from_index(RGWRados *store, string& key, rgw_bucket& bucket, RGWUserInfo& info,
+                                 RGWObjVersionTracker *objv_tracker, time_t *pmtime)
+{
+  bufferlist bl;
+  RGWUID uid;
+
+  int ret = rgw_get_system_obj(store, NULL, bucket, key, bl, NULL, pmtime);
+  if (ret < 0)
+    return ret;
+
+  bufferlist::iterator iter = bl.begin();
+  try {
+    ::decode(uid, iter);
+    return rgw_get_user_info_by_uid(store, uid.user_id, info, objv_tracker);
+  } catch (buffer::error& err) {
+    ldout(store->ctx(), 0) << "ERROR: failed to decode user info, caught buffer::error" << dendl;
+    return -EIO;
+  }
+
+  return 0;
+}
+
+/**
+ * Given a uid, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+int rgw_get_user_info_by_uid(RGWRados *store, string& uid, RGWUserInfo& info,
+                             RGWObjVersionTracker *objv_tracker, time_t *pmtime)
+{
+  bufferlist bl;
+  RGWUID user_id;
+
+  int ret = rgw_get_system_obj(store, NULL, store->zone.user_uid_pool, uid, bl, objv_tracker, pmtime);
+  if (ret < 0)
+    return ret;
+
+  bufferlist::iterator iter = bl.begin();
+  try {
+    ::decode(user_id, iter);
+    if (user_id.user_id.compare(uid) != 0) {
+      lderr(store->ctx())  << "ERROR: rgw_get_user_info_by_uid(): user id mismatch: " << user_id.user_id << " != " << uid << dendl;
+      return -EIO;
+    }
+    if (!iter.end()) {
+      ::decode(info, iter);
+    }
+  } catch (buffer::error& err) {
+    ldout(store->ctx(), 0) << "ERROR: failed to decode user info, caught buffer::error" << dendl;
+    return -EIO;
+  }
+
+  return 0;
+}
+
+/**
+ * Given an email, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+int rgw_get_user_info_by_email(RGWRados *store, string& email, RGWUserInfo& info,
+                               RGWObjVersionTracker *objv_tracker, time_t *pmtime)
+{
+  return rgw_get_user_info_from_index(store, email, store->zone.user_email_pool, info, objv_tracker, pmtime);
+}
+
+/**
+ * Given an swift username, finds the user_info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+extern int rgw_get_user_info_by_swift(RGWRados *store, string& swift_name, RGWUserInfo& info,
+                                      RGWObjVersionTracker *objv_tracker, time_t *pmtime)
+{
+  return rgw_get_user_info_from_index(store, swift_name, store->zone.user_swift_pool, info, objv_tracker, pmtime);
+}
+
+/**
+ * Given an access key, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+extern int rgw_get_user_info_by_access_key(RGWRados *store, string& access_key, RGWUserInfo& info,
+                                           RGWObjVersionTracker *objv_tracker, time_t *pmtime)
+{
+  return rgw_get_user_info_from_index(store, access_key, store->zone.user_keys_pool, info, objv_tracker, pmtime);
+}
+
+int rgw_remove_key_index(RGWRados *store, RGWAccessKey& access_key)
+{
+  rgw_obj obj(store->zone.user_keys_pool, access_key.id);
+  int ret = store->delete_system_obj(NULL, obj);
+  return ret;
+}
+
+int rgw_remove_uid_index(RGWRados *store, string& uid)
+{
+  RGWObjVersionTracker objv_tracker;
+  RGWUserInfo info;
+  int ret = rgw_get_user_info_by_uid(store, uid, info, &objv_tracker, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = store->meta_mgr->remove_entry(user_meta_handler, uid, &objv_tracker);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int rgw_remove_email_index(RGWRados *store, string& email)
+{
+  rgw_obj obj(store->zone.user_email_pool, email);
+  int ret = store->delete_system_obj(NULL, obj);
+  return ret;
+}
+
+int rgw_remove_swift_name_index(RGWRados *store, string& swift_name)
+{
+  rgw_obj obj(store->zone.user_swift_pool, swift_name);
+  int ret = store->delete_system_obj(NULL, obj);
+  return ret;
+}
+
+/**
+ * delete a user's presence from the RGW system.
+ * First remove their bucket ACLs, then delete them
+ * from the user and user email pools. This leaves the pools
+ * themselves alone, as well as any ACLs embedded in object xattrs.
+ */
+int rgw_delete_user(RGWRados *store, RGWUserInfo& info, RGWObjVersionTracker& objv_tracker) {
+  string marker;
+  vector buckets_vec;
+
+  bool done;
+  int ret;
+  CephContext *cct = store->ctx();
+  size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+
+  do {
+    RGWUserBuckets user_buckets;
+    ret = rgw_read_user_buckets(store, info.user_id, user_buckets, marker, max_buckets, false);
+    if (ret < 0)
+      return ret;
+
+    map& buckets = user_buckets.get_buckets();
+    for (map::iterator i = buckets.begin();
+        i != buckets.end();
+        ++i) {
+      RGWBucketEnt& bucket = i->second;
+      buckets_vec.push_back(bucket.bucket);
+
+      marker = i->first;
+    }
+
+    done = (buckets.size() < max_buckets);
+  } while (!done);
+
+  map::iterator kiter = info.access_keys.begin();
+  for (; kiter != info.access_keys.end(); ++kiter) {
+    ldout(store->ctx(), 10) << "removing key index: " << kiter->first << dendl;
+    ret = rgw_remove_key_index(store, kiter->second);
+    if (ret < 0 && ret != -ENOENT) {
+      ldout(store->ctx(), 0) << "ERROR: could not remove " << kiter->first << " (access key object), should be fixed (err=" << ret << ")" << dendl;
+      return ret;
+    }
+  }
+
+  map::iterator siter = info.swift_keys.begin();
+  for (; siter != info.swift_keys.end(); ++siter) {
+    RGWAccessKey& k = siter->second;
+    ldout(store->ctx(), 10) << "removing swift subuser index: " << k.id << dendl;
+    /* check if swift mapping exists */
+    ret = rgw_remove_swift_name_index(store, k.id);
+    if (ret < 0 && ret != -ENOENT) {
+      ldout(store->ctx(), 0) << "ERROR: could not remove " << k.id << " (swift name object), should be fixed (err=" << ret << ")" << dendl;
+      return ret;
+    }
+  }
+
+  rgw_obj email_obj(store->zone.user_email_pool, info.user_email);
+  ldout(store->ctx(), 10) << "removing email index: " << info.user_email << dendl;
+  ret = store->delete_system_obj(NULL, email_obj);
+  if (ret < 0 && ret != -ENOENT) {
+    ldout(store->ctx(), 0) << "ERROR: could not remove " << info.user_id << ":" << email_obj << ", should be fixed (err=" << ret << ")" << dendl;
+    return ret;
+  }
+
+  string buckets_obj_id;
+  rgw_get_buckets_obj(info.user_id, buckets_obj_id);
+  rgw_obj uid_bucks(store->zone.user_uid_pool, buckets_obj_id);
+  ldout(store->ctx(), 10) << "removing user buckets index" << dendl;
+  ret = store->delete_system_obj(NULL, uid_bucks);
+  if (ret < 0 && ret != -ENOENT) {
+    ldout(store->ctx(), 0) << "ERROR: could not remove " << info.user_id << ":" << uid_bucks << ", should be fixed (err=" << ret << ")" << dendl;
+    return ret;
+  }
+  
+  rgw_obj uid_obj(store->zone.user_uid_pool, info.user_id);
+  ldout(store->ctx(), 10) << "removing user index: " << info.user_id << dendl;
+  ret = store->meta_mgr->remove_entry(user_meta_handler, info.user_id, &objv_tracker);
+  if (ret < 0 && ret != -ENOENT) {
+    ldout(store->ctx(), 0) << "ERROR: could not remove " << info.user_id << ":" << uid_obj << ", should be fixed (err=" << ret << ")" << dendl;
+    return ret;
+  }
+
+  return 0;
+}
+
+static bool char_is_unreserved_url(char c)
+{
+  if (isalnum(c))
+    return true;
+
+  switch (c) {
+  case '-':
+  case '.':
+  case '_':
+  case '~':
+    return true;
+  default:
+    return false;
+  }
+}
+
+struct rgw_flags_desc {
+  uint32_t mask;
+  const char *str;
+};
+
+static struct rgw_flags_desc rgw_perms[] = {
+ { RGW_PERM_FULL_CONTROL, "full-control" },
+ { RGW_PERM_READ | RGW_PERM_WRITE, "read-write" },
+ { RGW_PERM_READ, "read" },
+ { RGW_PERM_WRITE, "write" },
+ { RGW_PERM_READ_ACP, "read-acp" },
+ { RGW_PERM_WRITE_ACP, "read-acp" },
+ { 0, NULL }
+};
+
+void rgw_perm_to_str(uint32_t mask, char *buf, int len)
+{
+  const char *sep = "";
+  int pos = 0;
+  if (!mask) {
+    snprintf(buf, len, "");
+    return;
+  }
+  while (mask) {
+    uint32_t orig_mask = mask;
+    for (int i = 0; rgw_perms[i].mask; i++) {
+      struct rgw_flags_desc *desc = &rgw_perms[i];
+      if ((mask & desc->mask) == desc->mask) {
+        pos += snprintf(buf + pos, len - pos, "%s%s", sep, desc->str);
+        if (pos == len)
+          return;
+        sep = ", ";
+        mask &= ~desc->mask;
+        if (!mask)
+          return;
+      }
+    }
+    if (mask == orig_mask) // no change
+      break;
+  }
+}
+
+uint32_t rgw_str_to_perm(const char *str)
+{
+  if (strcasecmp(str, "read") == 0)
+    return RGW_PERM_READ;
+  else if (strcasecmp(str, "write") == 0)
+    return RGW_PERM_WRITE;
+  else if (strcasecmp(str, "readwrite") == 0)
+    return RGW_PERM_READ | RGW_PERM_WRITE;
+  else if (strcasecmp(str, "full") == 0)
+    return RGW_PERM_FULL_CONTROL;
+
+  return 0; // better to return no permission
+}
+
+static bool validate_access_key(string& key)
+{
+  const char *p = key.c_str();
+  while (*p) {
+    if (!char_is_unreserved_url(*p))
+      return false;
+    p++;
+  }
+  return true;
+}
+
+static void set_err_msg(std::string *sink, std::string msg)
+{
+  if (sink && !msg.empty())
+    *sink = msg;
+}
+
+static bool remove_old_indexes(RGWRados *store,
+         RGWUserInfo& old_info, RGWUserInfo& new_info, std::string *err_msg)
+{
+  int ret;
+  bool success = true;
+
+  if (!old_info.user_id.empty() && old_info.user_id.compare(new_info.user_id) != 0) {
+    ret = rgw_remove_uid_index(store, old_info.user_id);
+    if (ret < 0 && ret != -ENOENT) {
+      set_err_msg(err_msg, "ERROR: could not remove index for uid " + old_info.user_id);
+      success = false;
+    }
+  }
+
+  if (!old_info.user_email.empty() &&
+      old_info.user_email.compare(new_info.user_email) != 0) {
+    ret = rgw_remove_email_index(store, old_info.user_email);
+  if (ret < 0 && ret != -ENOENT) {
+      set_err_msg(err_msg, "ERROR: could not remove index for email " + old_info.user_email);
+      success = false;
+    }
+  }
+
+  map::iterator old_iter;
+  for (old_iter = old_info.swift_keys.begin(); old_iter != old_info.swift_keys.end(); ++old_iter) {
+    RGWAccessKey& swift_key = old_iter->second;
+    map::iterator new_iter = new_info.swift_keys.find(swift_key.id);
+    if (new_iter == new_info.swift_keys.end()) {
+      ret = rgw_remove_swift_name_index(store, swift_key.id);
+      if (ret < 0 && ret != -ENOENT) {
+        set_err_msg(err_msg, "ERROR: could not remove index for swift_name " + swift_key.id);
+        success = false;
+      }
+    }
+  }
+
+  return success;
+}
+
+/*
+ * Dump either the full user info or a subset to a formatter.
+ *
+ * NOTE: It is the caller's respnsibility to ensure that the
+ * formatter is flushed at the correct time.
+ */
+
+static void dump_subusers_info(Formatter *f, RGWUserInfo &info)
+{
+  map::iterator uiter;
+
+  f->open_array_section("subusers");
+  for (uiter = info.subusers.begin(); uiter != info.subusers.end(); ++uiter) {
+    RGWSubUser& u = uiter->second;
+    f->open_object_section("user");
+    f->dump_format("id", "%s:%s", info.user_id.c_str(), u.name.c_str());
+    char buf[256];
+    rgw_perm_to_str(u.perm_mask, buf, sizeof(buf));
+    f->dump_string("permissions", buf);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+static void dump_access_keys_info(Formatter *f, RGWUserInfo &info)
+{
+  map::iterator kiter;
+  f->open_array_section("keys");
+  for (kiter = info.access_keys.begin(); kiter != info.access_keys.end(); ++kiter) {
+    RGWAccessKey& k = kiter->second;
+    const char *sep = (k.subuser.empty() ? "" : ":");
+    const char *subuser = (k.subuser.empty() ? "" : k.subuser.c_str());
+    f->open_object_section("key");
+    f->dump_format("user", "%s%s%s", info.user_id.c_str(), sep, subuser);
+    f->dump_string("access_key", k.id);
+    f->dump_string("secret_key", k.key);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+static void dump_swift_keys_info(Formatter *f, RGWUserInfo &info)
+{
+  map::iterator kiter;
+  f->open_array_section("swift_keys");
+  for (kiter = info.swift_keys.begin(); kiter != info.swift_keys.end(); ++kiter) {
+    RGWAccessKey& k = kiter->second;
+    const char *sep = (k.subuser.empty() ? "" : ":");
+    const char *subuser = (k.subuser.empty() ? "" : k.subuser.c_str());
+    f->open_object_section("key");
+    f->dump_format("user", "%s%s%s", info.user_id.c_str(), sep, subuser);
+    f->dump_string("secret_key", k.key);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+static void dump_user_info(Formatter *f, RGWUserInfo &info)
+{
+  f->open_object_section("user_info");
+
+  f->dump_string("user_id", info.user_id);
+  f->dump_string("display_name", info.display_name);
+  f->dump_string("email", info.user_email);
+  f->dump_int("suspended", (int)info.suspended);
+  f->dump_int("max_buckets", (int)info.max_buckets);
+
+  dump_subusers_info(f, info);
+  dump_access_keys_info(f, info);
+  dump_swift_keys_info(f, info);
+  info.caps.dump(f);
+
+  f->close_section();
+}
+
+
+RGWAccessKeyPool::RGWAccessKeyPool(RGWUser* usr)
+{
+  user = usr;
+  swift_keys = NULL;
+  access_keys = NULL;
+
+  if (!user) {
+    keys_allowed = false;
+    store = NULL;
+    return;
+  }
+
+  keys_allowed = true;
+
+  store = user->get_store();
+}
+
+RGWAccessKeyPool::~RGWAccessKeyPool()
+{
+
+}
+
+int RGWAccessKeyPool::init(RGWUserAdminOpState& op_state)
+{
+  if (!op_state.is_initialized()) {
+    keys_allowed = false;
+    return -EINVAL;
+  }
+
+  std::string uid = op_state.get_user_id();
+  if (uid.compare(RGW_USER_ANON_ID) == 0) {
+    keys_allowed = false;
+    return -EACCES;
+  }
+
+  swift_keys = op_state.get_swift_keys();
+  access_keys = op_state.get_access_keys();
+
+  keys_allowed = true;
+
+  return 0;
+}
+
+/*
+ * Do a fairly exhaustive search for an existing key matching the parameters
+ * given. Also handles the case where no key type was specified and updates
+ * the operation state if needed.
+ */
+
+bool RGWAccessKeyPool::check_existing_key(RGWUserAdminOpState& op_state)
+{
+  bool existing_key = false;
+
+  int key_type = op_state.get_key_type();
+  std::string kid = op_state.get_access_key();
+  std::map::iterator kiter;
+  std::string swift_kid = op_state.build_default_swift_kid();
+
+  RGWUserInfo dup_info;
+
+  if (kid.empty() && swift_kid.empty())
+    return false;
+
+  switch (key_type) {
+  case KEY_TYPE_SWIFT:
+    kiter = swift_keys->find(kid);
+
+    existing_key = (kiter != swift_keys->end());
+    if (existing_key)
+      break;
+
+    if (swift_kid.empty())
+      return false;
+
+    kiter = swift_keys->find(swift_kid);
+
+    existing_key = (kiter != swift_keys->end());
+    if (existing_key)
+      op_state.set_access_key(swift_kid);
+
+    break;
+  case KEY_TYPE_S3:
+    kiter = access_keys->find(kid);
+    existing_key = (kiter != access_keys->end());
+
+    break;
+  default:
+    kiter = access_keys->find(kid);
+
+    existing_key = (kiter != access_keys->end());
+    if (existing_key) {
+      op_state.set_key_type(KEY_TYPE_S3);
+      break;
+    }
+
+    kiter = swift_keys->find(kid);
+
+    existing_key = (kiter != swift_keys->end());
+    if (existing_key) {
+      op_state.set_key_type(KEY_TYPE_SWIFT);
+      break;
+    }
+
+    // handle the case where the access key was not provided in user:key format
+    if (swift_kid.empty())
+      return false;
+
+    kiter = swift_keys->find(swift_kid);
+
+    existing_key = (kiter != swift_keys->end());
+    if (existing_key) {
+      op_state.set_access_key(swift_kid);
+      op_state.set_key_type(KEY_TYPE_SWIFT);
+    }
+  }
+
+  op_state.set_existing_key(existing_key);
+
+  return existing_key;
+}
+
+int RGWAccessKeyPool::check_op(RGWUserAdminOpState& op_state,
+     std::string *err_msg)
+{
+  RGWUserInfo dup_info;
+
+  if (!op_state.is_populated()) {
+    set_err_msg(err_msg, "user info was not populated");
+    return -EINVAL;
+  }
+
+  if (!keys_allowed) {
+    set_err_msg(err_msg, "keys not allowed for this user");
+    return -EACCES;
+  }
+
+  std::string access_key = op_state.get_access_key();
+  std::string secret_key = op_state.get_secret_key();
+
+  int32_t key_type = op_state.get_key_type();
+
+  // if a key type wasn't specified set it to s3
+  if (key_type < 0)
+    key_type = KEY_TYPE_S3;
+
+  op_state.set_key_type(key_type);
+
+  /* see if the access key or secret key was specified */
+  if (key_type == KEY_TYPE_S3 && !op_state.will_gen_access() && access_key.empty()) {
+    set_err_msg(err_msg, "empty access key");
+    return -EINVAL;
+  }
+
+  // don't check for secret key because we may be doing a removal
+
+  check_existing_key(op_state);
+
+  return 0;
+}
+
+// Generate a new random key
+int RGWAccessKeyPool::generate_key(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  std::string id;
+  std::string key;
+
+  std::pair key_pair;
+  RGWAccessKey new_key;
+  RGWUserInfo duplicate_check;
+
+  int ret = 0;
+  int key_type = op_state.get_key_type();
+  bool gen_access = op_state.will_gen_access();
+  bool gen_secret = op_state.will_gen_secret();
+  std::string subuser = op_state.get_subuser();
+
+  if (!keys_allowed) {
+    set_err_msg(err_msg, "access keys not allowed for this user");
+    return -EACCES;
+  }
+
+  if (op_state.has_existing_key()) {
+    set_err_msg(err_msg, "cannot create existing key");
+    return -EEXIST;
+  }
+
+  if (!gen_access) {
+    id = op_state.get_access_key();
+  }
+
+  if (!id.empty()) {
+    switch (key_type) {
+    case KEY_TYPE_SWIFT:
+      if (rgw_get_user_info_by_swift(store, id, duplicate_check) >= 0) {
+        set_err_msg(err_msg, "existing swift key in RGW system:" + id);
+        return -EEXIST;
+      }
+      break;
+    case KEY_TYPE_S3:
+      if (rgw_get_user_info_by_access_key(store, id, duplicate_check) >= 0) {
+        set_err_msg(err_msg, "existing S3 key in RGW system:" + id);
+        return -EEXIST;
+      }
+    }
+  }
+
+  if (op_state.has_subuser())
+    new_key.subuser = op_state.get_subuser();
+
+  if (!gen_secret) {
+    key = op_state.get_secret_key();
+  } else if (gen_secret) {
+    char secret_key_buf[SECRET_KEY_LEN + 1];
+
+    ret = gen_rand_base64(g_ceph_context, secret_key_buf, sizeof(secret_key_buf));
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to generate secret key");
+      return ret;
+    }
+
+    key = secret_key_buf;
+  }
+
+  // Generate the access key
+  if (key_type == KEY_TYPE_S3 && gen_access) {
+    char public_id_buf[PUBLIC_ID_LEN + 1];
+
+    do {
+      int id_buf_size = sizeof(public_id_buf);
+      ret = gen_rand_alphanumeric_upper(g_ceph_context,
+               public_id_buf, id_buf_size);
+
+      if (ret < 0) {
+        set_err_msg(err_msg, "unable to generate access key");
+        return ret;
+      }
+
+      id = public_id_buf;
+      if (!validate_access_key(id))
+        continue;
+
+    } while (!rgw_get_user_info_by_access_key(store, id, duplicate_check));
+  }
+
+  if (key_type == KEY_TYPE_SWIFT && gen_access) {
+    id = op_state.build_default_swift_kid();
+    if (id.empty()) {
+      set_err_msg(err_msg, "empty swift access key");
+      return -EINVAL;
+    }
+
+    // check that the access key doesn't exist
+    if (rgw_get_user_info_by_swift(store, id, duplicate_check) >= 0) {
+      set_err_msg(err_msg, "cannot create existing swift key");
+      return -EEXIST;
+    }
+  }
+
+  // finally create the new key
+  new_key.id = id;
+  new_key.key = key;
+
+  key_pair.first = id;
+  key_pair.second = new_key;
+
+  if (key_type == KEY_TYPE_S3) {
+    access_keys->insert(key_pair);
+  } else if (key_type == KEY_TYPE_SWIFT) {
+    swift_keys->insert(key_pair);
+  }
+
+  return 0;
+}
+
+// modify an existing key
+int RGWAccessKeyPool::modify_key(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  std::string id;
+  std::string key = op_state.get_secret_key();
+  int key_type = op_state.get_key_type();
+
+  RGWAccessKey modify_key;
+
+  pair key_pair;
+  map::iterator kiter;
+
+  switch (key_type) {
+  case KEY_TYPE_S3:
+    id = op_state.get_access_key();
+    if (id.empty()) {
+      set_err_msg(err_msg, "no access key specified");
+      return -EINVAL;
+    }
+    break;
+  case KEY_TYPE_SWIFT:
+    id = op_state.build_default_swift_kid();
+    if (id.empty()) {
+      set_err_msg(err_msg, "no subuser specified");
+      return -EINVAL;
+    }
+    break;
+  default:
+    set_err_msg(err_msg, "invalid key type");
+    return -EINVAL;
+  }
+
+  if (!op_state.has_existing_key()) {
+    set_err_msg(err_msg, "key does not exist");
+    return -EINVAL;
+  }
+
+  key_pair.first = id;
+
+  if (key_type == KEY_TYPE_SWIFT) {
+    modify_key.id = id;
+    modify_key.subuser = op_state.get_subuser();
+  } else if (key_type == KEY_TYPE_S3) {
+    kiter = access_keys->find(id);
+    if (kiter != access_keys->end()) {
+      modify_key = kiter->second;
+    }
+  }
+
+  if (op_state.will_gen_secret()) {
+    char secret_key_buf[SECRET_KEY_LEN + 1];
+
+    int ret;
+    int key_buf_size = sizeof(secret_key_buf);
+    ret  = gen_rand_base64(g_ceph_context, secret_key_buf, key_buf_size);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to generate secret key");
+      return ret;
+    }
+
+    key = secret_key_buf;
+  }
+
+  if (key.empty()) {
+      set_err_msg(err_msg, "empty secret key");
+      return  -EINVAL;
+  }
+
+  // update the access key with the new secret key
+  modify_key.key = key;
+
+  key_pair.second = modify_key;
+
+
+  if (key_type == KEY_TYPE_S3) {
+    (*access_keys)[id] = modify_key;
+  } else if (key_type == KEY_TYPE_SWIFT) {
+    (*swift_keys)[id] = modify_key;
+  }
+
+  return 0;
+}
+
+int RGWAccessKeyPool::execute_add(RGWUserAdminOpState& op_state,
+         std::string *err_msg, bool defer_user_update)
+{
+  int ret = 0;
+
+  std::string subprocess_msg;
+  int key_op = GENERATE_KEY;
+
+  // set the op
+  if (op_state.has_existing_key())
+    key_op = MODIFY_KEY;
+
+  switch (key_op) {
+  case GENERATE_KEY:
+    ret = generate_key(op_state, &subprocess_msg);
+    break;
+  case MODIFY_KEY:
+    ret = modify_key(op_state, &subprocess_msg);
+    break;
+  }
+
+  if (ret < 0) {
+    set_err_msg(err_msg, subprocess_msg);
+    return ret;
+  }
+
+  // store the updated info
+  if (!defer_user_update)
+    ret = user->update(op_state, err_msg);
+
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWAccessKeyPool::add(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  return add(op_state, err_msg, false);
+}
+
+int RGWAccessKeyPool::add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update)
+{
+  int ret; 
+  std::string subprocess_msg;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse request, " + subprocess_msg);
+    return ret;
+  }
+
+  ret = execute_add(op_state, &subprocess_msg, defer_user_update);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to add access key, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWAccessKeyPool::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update)
+{
+  int ret = 0;
+
+  int key_type = op_state.get_key_type();
+  std::string id = op_state.get_access_key();
+  map::iterator kiter;
+  map *keys_map;
+
+  if (!op_state.has_existing_key()) {
+    set_err_msg(err_msg, "unable to find access key");
+    return -EINVAL;
+  }
+
+  if (key_type == KEY_TYPE_S3) {
+    keys_map = access_keys;
+  } else if (key_type == KEY_TYPE_SWIFT) {
+    keys_map = swift_keys;
+  } else {
+    keys_map = NULL;
+    set_err_msg(err_msg, "invalid access key");
+    return -EINVAL;
+  }
+
+  kiter = keys_map->find(id);
+  if (kiter == keys_map->end()) {
+    set_err_msg(err_msg, "key not found");
+    return -EINVAL;
+  }
+
+  rgw_remove_key_index(store, kiter->second);
+  keys_map->erase(kiter);
+
+  if (!defer_user_update)
+    ret = user->update(op_state, err_msg);
+
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWAccessKeyPool::remove(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  return remove(op_state, err_msg, false);
+}
+
+int RGWAccessKeyPool::remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update)
+{
+  int ret;
+
+  std::string subprocess_msg;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse request, " + subprocess_msg);
+    return ret;
+  }
+
+  ret = execute_remove(op_state, &subprocess_msg, defer_user_update);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to remove access key, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+RGWSubUserPool::RGWSubUserPool(RGWUser *usr)
+{
+  subusers_allowed = (usr != NULL);
+  if (usr)
+    store = usr->get_store();
+  else
+    store = NULL;
+  user = usr;
+  subuser_map = NULL;
+}
+
+RGWSubUserPool::~RGWSubUserPool()
+{
+
+}
+
+int RGWSubUserPool::init(RGWUserAdminOpState& op_state)
+{
+  if (!op_state.is_initialized()) {
+    subusers_allowed = false;
+    return -EINVAL;
+  }
+
+  std::string uid = op_state.get_user_id();
+  if (uid.compare(RGW_USER_ANON_ID) == 0) {
+    subusers_allowed = false;
+    return -EACCES;
+  }
+
+  subuser_map = op_state.get_subusers();
+  if (subuser_map == NULL) {
+    subusers_allowed = false;
+    return -EINVAL;
+  }
+
+  subusers_allowed = true;
+
+  return 0;
+}
+
+bool RGWSubUserPool::exists(std::string subuser)
+{
+  if (subuser.empty())
+    return false;
+
+  if (!subuser_map)
+    return false;
+
+  if (subuser_map->count(subuser))
+    return true;
+
+  return false;
+}
+
+int RGWSubUserPool::check_op(RGWUserAdminOpState& op_state,
+        std::string *err_msg)
+{
+  bool existing = false;
+  std::string subuser = op_state.get_subuser();
+
+  if (!op_state.is_populated()) {
+    set_err_msg(err_msg, "user info was not populated");
+    return -EINVAL;
+  }
+
+  if (!subusers_allowed) {
+    set_err_msg(err_msg, "subusers not allowed for this user");
+    return -EACCES;
+  }
+
+  if (subuser.empty() && !op_state.will_gen_subuser()) {
+    set_err_msg(err_msg, "empty subuser name");
+    return -EINVAL;
+  }
+
+  // check if the subuser exists
+  if (!subuser.empty())
+    existing = exists(subuser);
+
+  op_state.set_existing_subuser(existing);
+
+  return 0;
+}
+
+int RGWSubUserPool::execute_add(RGWUserAdminOpState& op_state,
+        std::string *err_msg, bool defer_user_update)
+{
+  int ret = 0;
+  std::string subprocess_msg;
+
+  RGWSubUser subuser;
+  std::pair subuser_pair;
+  std::string subuser_str = op_state.get_subuser();
+
+  subuser_pair.first = subuser_str;
+
+  // assumes key should be created
+  if (op_state.has_key_op()) {
+    ret = user->keys.add(op_state, &subprocess_msg, true);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to create subuser key, " + subprocess_msg);
+      return ret;
+    }
+  }
+
+  // create the subuser
+  subuser.name = subuser_str;
+
+  if (op_state.has_subuser_perm())
+    subuser.perm_mask = op_state.get_subuser_perm();
+
+  // insert the subuser into user info
+  subuser_pair.second = subuser;
+  subuser_map->insert(subuser_pair);
+
+  // attempt to save the subuser
+  if (!defer_user_update)
+    ret = user->update(op_state, err_msg);
+
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWSubUserPool::add(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  return add(op_state, err_msg, false);
+}
+
+int RGWSubUserPool::add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update)
+{
+  std::string subprocess_msg;
+  int ret;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse request, " + subprocess_msg);
+    return ret;
+  }
+
+  if (op_state.get_secret_key().empty()) {
+    op_state.set_gen_access();
+  }
+
+  ret = execute_add(op_state, &subprocess_msg, defer_user_update);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to create subuser, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWSubUserPool::execute_remove(RGWUserAdminOpState& op_state,
+        std::string *err_msg, bool defer_user_update)
+{
+  int ret = 0;
+  std::string subprocess_msg;
+
+  std::string subuser_str = op_state.get_subuser();
+
+  map::iterator siter;
+  siter = subuser_map->find(subuser_str);
+
+  if (!op_state.has_existing_subuser()) {
+    set_err_msg(err_msg, "subuser not found: " + subuser_str);
+    return -EINVAL;
+  }
+
+  if (op_state.will_purge_keys()) {
+    // error would be non-existance so don't check
+    user->keys.remove(op_state, &subprocess_msg, true);
+  }
+
+  //remove the subuser from the user info
+  subuser_map->erase(siter);
+
+  // attempt to save the subuser
+  if (!defer_user_update)
+    ret = user->update(op_state, err_msg);
+
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWSubUserPool::remove(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  return remove(op_state, err_msg, false);
+}
+
+int RGWSubUserPool::remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update)
+{
+  std::string subprocess_msg;
+  int ret;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse request, " + subprocess_msg);
+    return ret;
+  }
+
+  ret = execute_remove(op_state, &subprocess_msg, defer_user_update);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to remove subuser, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWSubUserPool::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update)
+{
+  int ret = 0;
+  std::string subprocess_msg;
+  std::map::iterator siter;
+  std::pair subuser_pair;
+
+  std::string subuser_str = op_state.get_subuser();
+  RGWSubUser subuser;
+
+  if (!op_state.has_existing_subuser()) {
+    set_err_msg(err_msg, "subuser does not exist");
+    return -EINVAL;
+  }
+
+  subuser_pair.first = subuser_str;
+
+  siter = subuser_map->find(subuser_str);
+  subuser = siter->second;
+
+  if (op_state.has_key_op()) {
+    ret = user->keys.add(op_state, &subprocess_msg, true);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to create subuser keys, " + subprocess_msg);
+      return ret;
+    }
+  }
+
+  if (op_state.has_subuser_perm())
+    subuser.perm_mask = op_state.get_subuser_perm();
+
+  subuser_pair.second = subuser;
+
+  subuser_map->erase(siter);
+  subuser_map->insert(subuser_pair);
+
+  // attempt to save the subuser
+  if (!defer_user_update)
+    ret = user->update(op_state, err_msg);
+
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWSubUserPool::modify(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  return RGWSubUserPool::modify(op_state, err_msg, false);
+}
+
+int RGWSubUserPool::modify(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update)
+{
+  std::string subprocess_msg;
+  int ret;
+
+  RGWSubUser subuser;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse request, " + subprocess_msg);
+    return ret;
+  }
+
+  ret = execute_modify(op_state, &subprocess_msg, defer_user_update);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to modify subuser, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+RGWUserCapPool::RGWUserCapPool(RGWUser *usr)
+{
+  user = usr;
+  caps = NULL;
+  caps_allowed = (user != NULL);
+}
+
+RGWUserCapPool::~RGWUserCapPool()
+{
+
+}
+
+int RGWUserCapPool::init(RGWUserAdminOpState& op_state)
+{
+  if (!op_state.is_initialized()) {
+    caps_allowed = false;
+    return -EINVAL;
+  }
+
+  std::string uid = op_state.get_user_id();
+  if (uid == RGW_USER_ANON_ID) {
+    caps_allowed = false;
+    return -EACCES;
+  }
+
+  caps = op_state.get_caps_obj();
+  if (!caps) {
+    caps_allowed = false;
+    return -EINVAL;
+  }
+
+  caps_allowed = true;
+
+  return 0;
+}
+
+int RGWUserCapPool::add(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  return add(op_state, err_msg, false);
+}
+
+int RGWUserCapPool::add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save)
+{
+  int ret = 0;
+  std::string caps_str = op_state.get_caps();
+
+  if (!op_state.is_populated()) {
+    set_err_msg(err_msg, "user info was not populated");
+    return -EINVAL;
+  }
+
+  if (!caps_allowed) {
+    set_err_msg(err_msg, "caps not allowed for this user");
+    return -EACCES;
+  }
+
+  if (caps_str.empty()) {
+    set_err_msg(err_msg, "empty user caps");
+    return -EINVAL;
+  }
+
+  int r = caps->add_from_string(caps_str);
+  if (r < 0) {
+    set_err_msg(err_msg, "unable to add caps: " + caps_str);
+    return r;
+  }
+
+  if (!defer_save)
+    ret = user->update(op_state, err_msg);
+
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWUserCapPool::remove(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  return remove(op_state, err_msg, false);
+}
+
+int RGWUserCapPool::remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save)
+{
+  int ret = 0;
+
+  std::string caps_str = op_state.get_caps();
+
+  if (!op_state.is_populated()) {
+    set_err_msg(err_msg, "user info was not populated");
+    return -EINVAL;
+  }
+
+  if (!caps_allowed) {
+    set_err_msg(err_msg, "caps not allowed for this user");
+    return -EACCES;
+  }
+
+  if (caps_str.empty()) {
+    set_err_msg(err_msg, "empty user caps");
+    return -EINVAL;
+  }
+
+  int r = caps->remove_from_string(caps_str);
+  if (r < 0) {
+    set_err_msg(err_msg, "unable to remove caps: " + caps_str);
+    return r;
+  }
+
+  if (!defer_save)
+    ret = user->update(op_state, err_msg);
+
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+RGWUser::RGWUser() : store(NULL), info_stored(false), caps(this), keys(this), subusers(this)
+{
+  init_default();
+}
+
+int RGWUser::init(RGWRados *storage, RGWUserAdminOpState& op_state)
+{
+  init_default();
+  int ret = init_storage(storage);
+  if (ret < 0)
+    return ret;
+
+  ret = init(op_state);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+RGWUser::~RGWUser()
+{
+}
+
+void RGWUser::init_default()
+{
+  // use anonymous user info as a placeholder
+  rgw_get_anon_user(old_info);
+  user_id = RGW_USER_ANON_ID;
+
+  clear_populated();
+}
+
+int RGWUser::init_storage(RGWRados *storage)
+{
+  if (!storage) {
+    return -EINVAL;
+  }
+
+  store = storage;
+
+  clear_populated();
+
+  /* API wrappers */
+  keys = RGWAccessKeyPool(this);
+  caps = RGWUserCapPool(this);
+  subusers = RGWSubUserPool(this);
+
+  return 0;
+}
+
+int RGWUser::init(RGWUserAdminOpState& op_state)
+{
+  bool found = false;
+  std::string swift_user;
+  std::string uid = op_state.get_user_id();
+  std::string user_email = op_state.get_user_email();
+  std::string access_key = op_state.get_access_key();
+  std::string subuser = op_state.get_subuser();
+
+  int key_type = op_state.get_key_type();
+  if (key_type == KEY_TYPE_SWIFT) {
+    swift_user = op_state.get_access_key();
+    access_key.clear();
+  }
+
+  RGWUserInfo user_info;
+
+  clear_populated();
+
+  if (uid.empty() && !subuser.empty()) {
+    size_t pos = subuser.find(':');
+    if (pos != string::npos) {
+      uid = subuser.substr(0, pos);
+      op_state.set_user_id(uid);
+    }
+  }
+
+  if (!uid.empty() && (uid.compare(RGW_USER_ANON_ID) != 0))
+    found = (rgw_get_user_info_by_uid(store, uid, user_info, &op_state.objv) >= 0);
+
+  if (!user_email.empty() && !found)
+    found = (rgw_get_user_info_by_email(store, user_email, user_info, &op_state.objv) >= 0);
+
+  if (!swift_user.empty() && !found)
+    found = (rgw_get_user_info_by_swift(store, swift_user, user_info, &op_state.objv) >= 0);
+
+  if (!access_key.empty() && !found)
+    found = (rgw_get_user_info_by_access_key(store, access_key, user_info, &op_state.objv) >= 0);
+
+  op_state.set_existing_user(found);
+  if (found) {
+    op_state.set_user_info(user_info);
+    op_state.set_populated();
+
+    old_info = user_info;
+    set_populated();
+  }
+
+  user_id = user_info.user_id;
+  op_state.set_initialized();
+
+  // this may have been called by a helper object
+  int ret = init_members(op_state);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWUser::init_members(RGWUserAdminOpState& op_state)
+{
+  int ret = 0;
+
+  ret = keys.init(op_state);
+  if (ret < 0)
+    return ret;
+
+  ret = subusers.init(op_state);
+  if (ret < 0)
+    return ret;
+
+  ret = caps.init(op_state);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWUser::update(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  int ret;
+  std::string subprocess_msg;
+  RGWUserInfo user_info = op_state.get_user_info();
+
+  if (!store) {
+    set_err_msg(err_msg, "couldn't initialize storage");
+    return -EINVAL;
+  }
+
+  if (is_populated()) {
+    ret = rgw_store_user_info(store, user_info, &old_info, &op_state.objv, 0, false);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to store user info");
+      return ret;
+    }
+
+    ret = remove_old_indexes(store, old_info, user_info, &subprocess_msg);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to remove old user info, " + subprocess_msg);
+      return ret;
+    }
+  } else {
+    ret = rgw_store_user_info(store, user_info, NULL, &op_state.objv, 0, false);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to store user info");
+      return ret;
+    }
+  }
+
+  old_info = user_info;
+  set_populated();
+
+  return 0;
+}
+
+int RGWUser::check_op(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  bool same_id;
+  bool populated;
+  //bool existing_email = false; // this check causes a fault
+  std::string op_id = op_state.get_user_id();
+  std::string op_email = op_state.get_user_email();
+
+  RGWUserInfo user_info;
+
+  same_id = (user_id.compare(op_id) == 0);
+  populated = is_populated();
+
+  if (op_id.compare(RGW_USER_ANON_ID) == 0) {
+    set_err_msg(err_msg, "unable to perform operations on the anonymous user");
+    return -EINVAL;
+  }
+
+  if (populated && !same_id) {
+    set_err_msg(err_msg, "user id mismatch, operation id: " + op_id\
+            + " does not match: " + user_id);
+
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+int RGWUser::execute_add(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  std::string subprocess_msg;
+  int ret = 0;
+  bool defer_user_update = true;
+
+  RGWUserInfo user_info;
+
+  std::string uid = op_state.get_user_id();
+  std::string user_email = op_state.get_user_email();
+  std::string display_name = op_state.get_display_name();
+
+  // fail if the user exists already
+  if (op_state.has_existing_user()) {
+    if ((user_email.empty() || old_info.user_email == user_email) &&
+        old_info.display_name == display_name) {
+      return execute_modify(op_state, err_msg);
+    }
+
+    set_err_msg(err_msg, "user: " + op_state.user_id + " exists");
+
+    return -EEXIST;
+  }
+
+  // fail if the user_info has already been populated
+  if (op_state.is_populated()) {
+    set_err_msg(err_msg, "cannot overwrite already populated user");
+    return -EEXIST;
+  }
+
+  // fail if the display name was not included
+  if (display_name.empty()) {
+    set_err_msg(err_msg, "no display name specified");
+    return -EINVAL;
+  }
+
+  // fail if the user email is a duplicate
+  if (op_state.has_existing_email()) {
+    set_err_msg(err_msg, "duplicate email provided");
+    return -EEXIST;
+  }
+
+  // set the user info
+  user_id = uid;
+  user_info.user_id = user_id;
+  user_info.display_name = display_name;
+
+  if (!user_email.empty())
+    user_info.user_email = user_email;
+
+  user_info.max_buckets = op_state.get_max_buckets();
+  user_info.suspended = op_state.get_suspension_status();
+  user_info.system = op_state.system;
+
+  if (op_state.op_mask_specified)
+    user_info.op_mask = op_state.get_op_mask();
+
+  if (op_state.has_bucket_quota())
+    user_info.bucket_quota = op_state.get_bucket_quota();
+
+  if (op_state.temp_url_key_specified) {
+    map::iterator iter;
+    for (iter = op_state.temp_url_keys.begin();
+         iter != op_state.temp_url_keys.end(); ++iter) {
+      user_info.temp_url_keys[iter->first] = iter->second;
+    }
+  }
+
+  if (op_state.has_user_quota())
+    user_info.user_quota = op_state.get_user_quota();
+
+  // update the request
+  op_state.set_user_info(user_info);
+  op_state.set_populated();
+
+  // update the helper objects
+  ret = init_members(op_state);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to initialize user");
+    return ret;
+  }
+
+  // see if we need to add an access key
+  if (op_state.has_key_op()) {
+    ret = keys.add(op_state, &subprocess_msg, defer_user_update);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to create access key, " + subprocess_msg);
+      return ret;
+    }
+  }
+
+  // see if we need to add some caps
+  if (op_state.has_caps_op()) {
+    ret = caps.add(op_state, &subprocess_msg, defer_user_update);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to add user capabilities, " + subprocess_msg);
+      return ret;
+    }
+  }
+
+  ret = update(op_state, err_msg);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWUser::add(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  std::string subprocess_msg;
+  int ret;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg);
+    return ret;
+  }
+
+  ret = execute_add(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to create user, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  int ret;
+
+  bool purge_data = op_state.will_purge_data();
+  std::string uid = op_state.get_user_id();
+  RGWUserInfo user_info = op_state.get_user_info();
+
+  if (!op_state.has_existing_user()) {
+    set_err_msg(err_msg, "user does not exist");
+    return -EINVAL;
+  }
+
+  bool done;
+  string marker;
+  CephContext *cct = store->ctx();
+  size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+  do {
+    RGWUserBuckets buckets;
+    ret = rgw_read_user_buckets(store, uid, buckets, marker, max_buckets, false);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to read user bucket info");
+      return ret;
+    }
+
+    map& m = buckets.get_buckets();
+    if (!m.empty() && !purge_data) {
+      set_err_msg(err_msg, "must specify purge data to remove user with buckets");
+      return -EEXIST; // change to code that maps to 409: conflict
+    }
+
+    std::map::iterator it;
+    for (it = m.begin(); it != m.end(); ++it) {
+      ret = rgw_remove_bucket(store, uid, ((*it).second).bucket, true);
+      if (ret < 0) {
+        set_err_msg(err_msg, "unable to delete user data");
+        return ret;
+      }
+
+      marker = it->first;
+    }
+
+    done = (m.size() < max_buckets);
+  } while (!done);
+
+  ret = rgw_delete_user(store, user_info, op_state.objv);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to remove user from RADOS");
+    return ret;
+  }
+
+  op_state.clear_populated();
+  clear_populated();
+
+  return 0;
+}
+
+int RGWUser::remove(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  std::string subprocess_msg;
+  int ret;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg);
+    return ret;
+  }
+
+  ret = execute_remove(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to remove user, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  bool populated = op_state.is_populated();
+  int ret = 0;
+  std::string subprocess_msg;
+  std::string op_email = op_state.get_user_email();
+  std::string display_name = op_state.get_display_name();
+
+  RGWUserInfo user_info;
+  RGWUserInfo duplicate_check;
+
+  // ensure that the user info has been populated or is populate-able
+  if (!op_state.has_existing_user() && !populated) {
+    set_err_msg(err_msg, "user not found");
+    return -ENOENT;
+  }
+
+  // if the user hasn't already been populated...attempt to
+  if (!populated) {
+    ret = init(op_state);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to retrieve user info");
+      return ret;
+    }
+  }
+
+  // ensure that we can modify the user's attributes
+  if (user_id == RGW_USER_ANON_ID) {
+    set_err_msg(err_msg, "unable to modify anonymous user's info");
+    return -EACCES;
+  }
+
+  user_info = old_info;
+
+  std::string old_email = old_info.user_email;
+  if (!op_email.empty()) {
+    bool same_email = false;
+    same_email = (old_email.compare(op_email) == 0);
+
+    // make sure we are not adding a duplicate email
+    if (!same_email) {
+      ret = rgw_get_user_info_by_email(store, op_email, duplicate_check);
+      if (ret >= 0 && duplicate_check.user_id != user_id) {
+        set_err_msg(err_msg, "cannot add duplicate email");
+        return -EEXIST;
+      }
+    }
+    user_info.user_email = op_email;
+  }
+
+
+  // update the remaining user info
+  if (!display_name.empty())
+    user_info.display_name = display_name;
+
+  // will be set to RGW_DEFAULT_MAX_BUCKETS by default
+  uint32_t max_buckets = op_state.get_max_buckets();
+
+  ldout(store->ctx(), 0) << "max_buckets=" << max_buckets << " specified=" << op_state.max_buckets_specified << dendl;
+
+  if (op_state.max_buckets_specified)
+    user_info.max_buckets = max_buckets;
+
+  if (op_state.system_specified)
+    user_info.system = op_state.system;
+
+  if (op_state.temp_url_key_specified) {
+    map::iterator iter;
+    for (iter = op_state.temp_url_keys.begin();
+         iter != op_state.temp_url_keys.end(); ++iter) {
+      user_info.temp_url_keys[iter->first] = iter->second;
+    }
+  }
+
+  if (op_state.op_mask_specified)
+    user_info.op_mask = op_state.get_op_mask();
+
+  if (op_state.has_bucket_quota())
+    user_info.bucket_quota = op_state.get_bucket_quota();
+
+  if (op_state.has_user_quota())
+    user_info.user_quota = op_state.get_user_quota();
+
+  if (op_state.has_suspension_op()) {
+    __u8 suspended = op_state.get_suspension_status();
+    user_info.suspended = suspended;
+
+    RGWUserBuckets buckets;
+
+    if (user_id.empty()) {
+      set_err_msg(err_msg, "empty user id passed...aborting");
+      return -EINVAL;
+    }
+
+    bool done;
+    string marker;
+    CephContext *cct = store->ctx();
+    size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+    do {
+      ret = rgw_read_user_buckets(store, user_id, buckets, marker, max_buckets, false);
+      if (ret < 0) {
+        set_err_msg(err_msg, "could not get buckets for uid:  " + user_id);
+        return ret;
+      }
+
+      map& m = buckets.get_buckets();
+      map::iterator iter;
+
+      vector bucket_names;
+      for (iter = m.begin(); iter != m.end(); ++iter) {
+        RGWBucketEnt obj = iter->second;
+        bucket_names.push_back(obj.bucket);
+
+        marker = iter->first;
+      }
+
+      ret = store->set_buckets_enabled(bucket_names, !suspended);
+      if (ret < 0) {
+        set_err_msg(err_msg, "failed to modify bucket");
+        return ret;
+      }
+
+      done = (m.size() < max_buckets);
+    } while (!done);
+  }
+  op_state.set_user_info(user_info);
+
+  // if we're supposed to modify keys, do so
+  if (op_state.has_key_op()) {
+    ret = keys.add(op_state, &subprocess_msg, true);
+    if (ret < 0) {
+      set_err_msg(err_msg, "unable to create or modify keys, " + subprocess_msg);
+      return ret;
+    }
+  }
+
+  ret = update(op_state, err_msg);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWUser::modify(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+  std::string subprocess_msg;
+  int ret;
+
+  ret = check_op(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg);
+    return ret;
+  }
+
+  ret = execute_modify(op_state, &subprocess_msg);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to modify user, " + subprocess_msg);
+    return ret;
+  }
+
+  return 0;
+}
+
+int RGWUser::info(RGWUserAdminOpState& op_state, RGWUserInfo& fetched_info, std::string *err_msg)
+{
+  int ret = init(op_state);
+  if (ret < 0) {
+    set_err_msg(err_msg, "unable to fetch user info");
+    return ret;
+  }
+
+  fetched_info = op_state.get_user_info();
+
+  return 0;
+}
+
+int RGWUser::info(RGWUserInfo& fetched_info, std::string *err_msg)
+{
+  if (!is_populated()) {
+    set_err_msg(err_msg, "no user info saved");
+    return -EINVAL;
+  }
+
+  fetched_info = old_info;
+
+  return 0;
+}
+
+int RGWUserAdminOp_User::info(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  if (!op_state.has_existing_user())
+    return -ENOENT;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+
+  flusher.start(0);
+
+  dump_user_info(formatter, info);
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWUserAdminOp_User::create(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.add(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+
+  flusher.start(0);
+
+  dump_user_info(formatter, info);
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWUserAdminOp_User::modify(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.modify(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+
+  flusher.start(0);
+
+  dump_user_info(formatter, info);
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWUserAdminOp_User::remove(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+
+  ret = user.remove(op_state, NULL);
+
+  return ret;
+}
+
+int RGWUserAdminOp_Subuser::create(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.subusers.add(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+
+  flusher.start(0);
+
+  dump_subusers_info(formatter, info);
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWUserAdminOp_Subuser::modify(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.subusers.modify(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+  
+  flusher.start(0);
+
+  dump_subusers_info(formatter, info);
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWUserAdminOp_Subuser::remove(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+
+  ret = user.subusers.remove(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWUserAdminOp_Key::create(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.keys.add(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+
+  flusher.start(0);
+
+  int key_type = op_state.get_key_type();
+
+  if (key_type == KEY_TYPE_SWIFT)
+    dump_swift_keys_info(formatter, info);
+
+  else if (key_type == KEY_TYPE_S3)
+    dump_access_keys_info(formatter, info);
+
+  flusher.flush();
+
+  return 0;
+}
+
+int RGWUserAdminOp_Key::remove(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+
+  ret = user.keys.remove(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+int RGWUserAdminOp_Caps::add(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.caps.add(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+
+  flusher.start(0);
+
+  info.caps.dump(formatter);
+  flusher.flush();
+
+  return 0;
+}
+
+
+int RGWUserAdminOp_Caps::remove(RGWRados *store, RGWUserAdminOpState& op_state,
+                  RGWFormatterFlusher& flusher)
+{
+  RGWUserInfo info;
+  RGWUser user;
+  int ret = user.init(store, op_state);
+  if (ret < 0)
+    return ret;
+
+  Formatter *formatter = flusher.get_formatter();
+
+  ret = user.caps.remove(op_state, NULL);
+  if (ret < 0)
+    return ret;
+
+  ret = user.info(info, NULL);
+  if (ret < 0)
+    return ret;
+
+  flusher.start(0);
+
+  info.caps.dump(formatter);
+  flusher.flush();
+
+  return 0;
+}
+
+class RGWUserMetadataObject : public RGWMetadataObject {
+  RGWUserInfo info;
+public:
+  RGWUserMetadataObject(RGWUserInfo& i, obj_version& v, time_t m) : info(i) {
+    objv = v;
+    mtime = m;
+  }
+
+  void dump(Formatter *f) const {
+    info.dump(f);
+  }
+};
+
+class RGWUserMetadataHandler : public RGWMetadataHandler {
+public:
+  string get_type() { return "user"; }
+
+  int get(RGWRados *store, string& entry, RGWMetadataObject **obj) {
+    RGWUserInfo info;
+
+    RGWObjVersionTracker objv_tracker;
+    time_t mtime;
+
+    int ret = rgw_get_user_info_by_uid(store, entry, info, &objv_tracker, &mtime);
+    if (ret < 0)
+      return ret;
+
+    RGWUserMetadataObject *mdo = new RGWUserMetadataObject(info, objv_tracker.read_version, mtime);
+
+    *obj = mdo;
+
+    return 0;
+  }
+
+  int put(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker,
+          time_t mtime, JSONObj *obj, sync_type_t sync_mode) {
+    RGWUserInfo info;
+
+    decode_json_obj(info, obj);
+
+    RGWUserInfo old_info;
+    time_t orig_mtime;
+    int ret = rgw_get_user_info_by_uid(store, entry, old_info, &objv_tracker, &orig_mtime);
+    if (ret < 0 && ret != -ENOENT)
+      return ret;
+
+    // are we actually going to perform this put, or is it too old?
+    if (ret != -ENOENT &&
+        !check_versions(objv_tracker.read_version, orig_mtime,
+			objv_tracker.write_version, mtime, sync_mode)) {
+      return STATUS_NO_APPLY;
+    }
+
+    ret = rgw_store_user_info(store, info, &old_info, &objv_tracker, mtime, false);
+    if (ret < 0)
+      return ret;
+
+    return STATUS_APPLIED;
+  }
+
+  struct list_keys_info {
+    RGWRados *store;
+    RGWListRawObjsCtx ctx;
+  };
+
+  int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) {
+    RGWUserInfo info;
+    int ret = rgw_get_user_info_by_uid(store, entry, info, &objv_tracker);
+    if (ret < 0)
+      return ret;
+
+    return rgw_delete_user(store, info, objv_tracker);
+  }
+
+  void get_pool_and_oid(RGWRados *store, const string& key, rgw_bucket& bucket, string& oid) {
+    oid = key;
+    bucket = store->zone.user_uid_pool;
+  }
+
+  int list_keys_init(RGWRados *store, void **phandle)
+  {
+    list_keys_info *info = new list_keys_info;
+
+    info->store = store;
+
+    *phandle = (void *)info;
+
+    return 0;
+  }
+
+  int list_keys_next(void *handle, int max, list& keys, bool *truncated) {
+    list_keys_info *info = static_cast(handle);
+
+    string no_filter;
+
+    keys.clear();
+
+    RGWRados *store = info->store;
+
+    list unfiltered_keys;
+
+    int ret = store->list_raw_objects(store->zone.user_uid_pool, no_filter,
+                                      max, info->ctx, unfiltered_keys, truncated);
+    if (ret < 0)
+      return ret;
+
+    // now filter out the buckets entries
+    list::iterator iter;
+    for (iter = unfiltered_keys.begin(); iter != unfiltered_keys.end(); ++iter) {
+      string& k = *iter;
+
+      if (k.find(".buckets") == string::npos) {
+        keys.push_back(k);
+      }
+    }
+
+    return 0;
+  }
+
+  void list_keys_complete(void *handle) {
+    list_keys_info *info = static_cast(handle);
+    delete info;
+  }
+};
+
+void rgw_user_init(RGWMetadataManager *mm)
+{
+  user_meta_handler = new RGWUserMetadataHandler;
+  mm->register_handler(user_meta_handler);
+}
diff --git a/ceph/src/rgw/rgw_user.h b/ceph/src/rgw/rgw_user.h
new file mode 100644
index 00000000..af99ed58
--- /dev/null
+++ b/ceph/src/rgw/rgw_user.h
@@ -0,0 +1,661 @@
+#ifndef CEPH_RGW_USER_H
+#define CEPH_RGW_USER_H
+
+#include 
+
+#include "include/types.h"
+#include "rgw_common.h"
+#include "rgw_tools.h"
+
+#include "rgw_rados.h"
+
+#include "rgw_string.h"
+
+#include "common/Formatter.h"
+#include "rgw_formats.h"
+
+using namespace std;
+
+#define RGW_USER_ANON_ID "anonymous"
+
+#define SECRET_KEY_LEN 40
+#define PUBLIC_ID_LEN 20
+#define RAND_SUBUSER_LEN 5
+
+/**
+ * A string wrapper that includes encode/decode functions
+ * for easily accessing a UID in all forms
+ */
+struct RGWUID
+{
+  string user_id;
+  void encode(bufferlist& bl) const {
+    ::encode(user_id, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(user_id, bl);
+  }
+};
+WRITE_CLASS_ENCODER(RGWUID)
+
+extern int rgw_user_sync_all_stats(RGWRados *store, const string& user_id);
+/**
+ * Get the anonymous (ie, unauthenticated) user info.
+ */
+extern void rgw_get_anon_user(RGWUserInfo& info);
+
+/**
+ * verify that user is an actual user, and not the anonymous user
+ */
+extern bool rgw_user_is_authenticated(RGWUserInfo& info);
+/**
+ * Save the given user information to storage.
+ * Returns: 0 on success, -ERR# on failure.
+ */
+extern int rgw_store_user_info(RGWRados *store, RGWUserInfo& info, RGWUserInfo *old_info,
+                               RGWObjVersionTracker *objv_tracker, time_t mtime, bool exclusive);
+/**
+ * Given an email, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+extern int rgw_get_user_info_by_uid(RGWRados *store, string& user_id, RGWUserInfo& info,
+                                    RGWObjVersionTracker *objv_tracker = NULL, time_t *pmtime = NULL);
+/**
+ * Given an swift username, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+extern int rgw_get_user_info_by_email(RGWRados *store, string& email, RGWUserInfo& info,
+                                      RGWObjVersionTracker *objv_tracker = NULL, time_t *pmtime = NULL);
+/**
+ * Given an swift username, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+extern int rgw_get_user_info_by_swift(RGWRados *store, string& swift_name, RGWUserInfo& info,
+                                      RGWObjVersionTracker *objv_tracker = NULL, time_t *pmtime = NULL);
+/**
+ * Given an access key, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+extern int rgw_get_user_info_by_access_key(RGWRados *store, string& access_key, RGWUserInfo& info,
+                                           RGWObjVersionTracker *objv_tracker = NULL, time_t *pmtime = NULL);
+/**
+ * Given an RGWUserInfo, deletes the user and its bucket ACLs.
+ */
+extern int rgw_delete_user(RGWRados *store, RGWUserInfo& user, RGWObjVersionTracker& objv_tracker);
+/**
+ * Store a list of the user's buckets, with associated functinos.
+ */
+
+/*
+ * remove the different indexes
+ */
+extern int rgw_remove_key_index(RGWRados *store, RGWAccessKey& access_key);
+extern int rgw_remove_uid_index(RGWRados *store, string& uid);
+extern int rgw_remove_email_index(RGWRados *store, string& email);
+extern int rgw_remove_swift_name_index(RGWRados *store, string& swift_name);
+
+/*
+ * An RGWUser class along with supporting classes created
+ * to support the creation of an RESTful administrative API
+ */
+
+extern void rgw_perm_to_str(uint32_t mask, char *buf, int len);
+extern uint32_t rgw_str_to_perm(const char *str);
+
+enum ObjectKeyType {
+  KEY_TYPE_SWIFT,
+  KEY_TYPE_S3,
+  KEY_TYPE_UNDEFINED
+};
+
+enum RGWKeyPoolOp {
+  GENERATE_KEY,
+  MODIFY_KEY
+};
+
+enum RGWUserId {
+  RGW_USER_ID,
+  RGW_SWIFT_USERNAME,
+  RGW_USER_EMAIL,
+  RGW_ACCESS_KEY,
+};
+
+struct RGWUserAdminOpState {
+  // user attributes
+  RGWUserInfo info;
+  std::string user_id;
+  std::string user_email;
+  std::string display_name;
+  uint32_t max_buckets;
+  __u8 suspended;
+  __u8 system;
+  std::string caps;
+  RGWObjVersionTracker objv;
+  uint32_t op_mask;
+  map temp_url_keys;
+
+  // subuser attributes
+  std::string subuser;
+  uint32_t perm_mask;
+
+  // key_attributes
+  std::string id; // access key
+  std::string key; // secret key
+  int32_t key_type;
+
+  // operation attributes
+  bool existing_user;
+  bool existing_key;
+  bool existing_subuser;
+  bool existing_email;
+  bool subuser_specified;
+  bool gen_secret;
+  bool gen_access;
+  bool gen_subuser;
+  bool id_specified;
+  bool key_specified;
+  bool type_specified;
+  bool purge_data;
+  bool purge_keys;
+  bool display_name_specified;
+  bool user_email_specified;
+  bool max_buckets_specified;
+  bool perm_specified;
+  bool op_mask_specified;
+  bool caps_specified;
+  bool suspension_op;
+  bool system_specified;
+  bool key_op;
+  bool temp_url_key_specified;
+
+  // req parameters
+  bool populated;
+  bool initialized;
+  bool key_params_checked;
+  bool subuser_params_checked;
+  bool user_params_checked;
+
+  bool bucket_quota_specified;
+  bool user_quota_specified;
+
+  RGWQuotaInfo bucket_quota;
+  RGWQuotaInfo user_quota;
+
+  void set_access_key(std::string& access_key) {
+    if (access_key.empty())
+      return;
+
+    id = access_key;
+    id_specified = true;
+    gen_access = false;
+    key_op = true;
+  }
+  void set_secret_key(std::string& secret_key) {
+    if (secret_key.empty())
+      return;
+
+    key = secret_key;
+    key_specified = true;
+    gen_secret = false;
+    key_op = true;
+  }
+  void set_user_id(std::string& id) {
+    if (id.empty())
+      return;
+
+    user_id = id;
+  }
+  void set_user_email(std::string& email) {
+    if (email.empty())
+      return;
+
+    user_email = email;
+    user_email_specified = true;
+  }
+  void set_display_name(std::string& name) {
+    if (name.empty())
+      return;
+
+    display_name = name;
+    display_name_specified = true;
+  }
+  void set_subuser(std::string& _subuser) {
+    if (_subuser.empty())
+      return;
+
+    size_t pos = _subuser.find(":");
+
+    if (pos != string::npos) {
+      user_id = _subuser.substr(0, pos);
+      subuser = _subuser.substr(pos+1);
+    } else {
+      subuser = _subuser;
+    }
+
+    subuser_specified = true;
+  }
+  void set_caps(std::string& _caps) {
+    if (_caps.empty())
+      return;
+
+    caps = _caps;
+    caps_specified = true;
+  }
+  void set_perm(uint32_t perm) {
+    perm_mask = perm;
+    perm_specified = true;
+  }
+  void set_op_mask(uint32_t mask) {
+    op_mask = mask;
+    op_mask_specified = true;
+  }
+  void set_temp_url_key(const string& key, int index) {
+    temp_url_keys[index] = key;
+    temp_url_key_specified = true;
+  }
+  void set_key_type(int32_t type) {
+    key_type = type;
+    type_specified = true;
+  }
+  void set_suspension(__u8 is_suspended) {
+    suspended = is_suspended;
+    suspension_op = true;
+  }
+  void set_system(__u8 is_system) {
+    system = is_system;
+    system_specified = true;
+  }
+  void set_user_info(RGWUserInfo& user_info) {
+    user_id = user_info.user_id;
+    info = user_info;
+  }
+  void set_max_buckets(uint32_t mb) {
+    max_buckets = mb;
+    max_buckets_specified = true;
+  }
+  void set_gen_access() {
+    gen_access = true;
+    key_op = true;
+  }
+  void set_gen_secret() {
+    gen_secret = true;
+    key_op = true;
+  }
+  void set_generate_key() {
+    if (id.empty())
+      gen_access = true;
+    if (key.empty())
+      gen_secret = true;
+    key_op = true;
+  }
+  void clear_generate_key() {
+    gen_access = false;
+    gen_secret = false;
+  }
+  void set_purge_keys() {
+    purge_keys = true;
+    key_op = true;
+  }
+
+  void set_bucket_quota(RGWQuotaInfo& quota) {
+    bucket_quota = quota;
+    bucket_quota_specified = true;
+  }
+
+  void set_user_quota(RGWQuotaInfo& quota) {
+    user_quota = quota;
+    user_quota_specified = true;
+  }
+
+  bool is_populated() { return populated; };
+  bool is_initialized() { return initialized; };
+  bool has_existing_user() { return existing_user; };
+  bool has_existing_key() { return existing_key; };
+  bool has_existing_subuser() { return existing_subuser; };
+  bool has_existing_email() { return existing_email; };
+  bool has_subuser() { return subuser_specified; };
+  bool has_key_op() { return key_op; };
+  bool has_caps_op() { return caps_specified; };
+  bool has_suspension_op() { return suspension_op; };
+  bool has_subuser_perm() { return perm_specified; };
+  bool has_op_mask() { return op_mask_specified; };
+  bool will_gen_access() { return gen_access; };
+  bool will_gen_secret() { return gen_secret; };
+  bool will_gen_subuser() { return gen_subuser; };
+  bool will_purge_keys() { return purge_keys; };
+  bool will_purge_data() { return purge_data; };
+  bool will_generate_subuser() { return gen_subuser; };
+  bool has_bucket_quota() { return bucket_quota_specified; }
+  bool has_user_quota() { return user_quota_specified; }
+  void set_populated() { populated = true; };
+  void clear_populated() { populated = false; };
+  void set_initialized() { initialized = true; };
+  void set_existing_user(bool flag) { existing_user = flag; };
+  void set_existing_key(bool flag) { existing_key = flag; };
+  void set_existing_subuser(bool flag) { existing_subuser = flag; };
+  void set_existing_email(bool flag) { existing_email = flag; };
+  void set_purge_data(bool flag) { purge_data = flag; };
+  void set_generate_subuser(bool flag) { gen_subuser = flag; };
+  __u8 get_suspension_status() { return suspended; };
+  int32_t get_key_type() {return key_type; };
+  uint32_t get_subuser_perm() { return perm_mask; };
+  uint32_t get_max_buckets() { return max_buckets; };
+  uint32_t get_op_mask() { return op_mask; };
+  RGWQuotaInfo& get_bucket_quota() { return bucket_quota; }
+  RGWQuotaInfo& get_user_quota() { return user_quota; }
+
+  std::string get_user_id() { return user_id; };
+  std::string get_subuser() { return subuser; };
+  std::string get_access_key() { return id; };
+  std::string get_secret_key() { return key; };
+  std::string get_caps() { return caps; };
+  std::string get_user_email() { return user_email; };
+  std::string get_display_name() { return display_name; };
+  map& get_temp_url_keys() { return temp_url_keys; };
+
+  RGWUserInfo&  get_user_info() { return info; };
+
+  map *get_swift_keys() { return &info.swift_keys; };
+  map *get_access_keys() { return &info.access_keys; };
+  map *get_subusers() { return &info.subusers; };
+
+  RGWUserCaps *get_caps_obj() { return &info.caps; };
+
+  std::string build_default_swift_kid() {
+    if (user_id.empty() || subuser.empty())
+      return "";
+
+    std::string kid = user_id;
+    kid.append(":");
+    kid.append(subuser);
+
+    return kid;
+  }
+
+  std::string generate_subuser() {
+    if (user_id.empty())
+      return "";
+
+    std::string generated_subuser = user_id;
+    std::string rand_suffix;
+
+    int sub_buf_size = RAND_SUBUSER_LEN + 1;
+    char sub_buf[RAND_SUBUSER_LEN + 1];
+
+    if (gen_rand_alphanumeric_upper(g_ceph_context, sub_buf, sub_buf_size) < 0)
+      return "";
+
+    rand_suffix = sub_buf;
+    if (rand_suffix.empty())
+      return "";
+
+    generated_subuser.append(rand_suffix);
+    subuser = generated_subuser;
+
+    return generated_subuser;
+  }
+
+  RGWUserAdminOpState() : user_id(RGW_USER_ANON_ID), user_email(""), display_name(""), id(""), key ("")
+  {
+    max_buckets = RGW_DEFAULT_MAX_BUCKETS;
+    key_type = -1;
+    perm_mask = 0;
+    suspended = 0;
+    system = 0;
+    op_mask = 0;
+
+    existing_user = false;
+    existing_key = false;
+    existing_subuser = false;
+    existing_email = false;
+    subuser_specified = false;
+    caps_specified = false;
+    purge_keys = false;
+    gen_secret = false;
+    gen_access = false;
+    gen_subuser = false;
+    id_specified = false;
+    key_specified = false;
+    type_specified = false;
+    purge_data = false;
+    display_name_specified = false;
+    user_email_specified = false;
+    max_buckets_specified = false;
+    perm_specified = false;
+    op_mask_specified = false;
+    suspension_op = false;
+    system_specified = false;
+    key_op = false;
+    populated = false;
+    initialized = false;
+    key_params_checked = false;
+    subuser_params_checked = false;
+    user_params_checked = false;
+    bucket_quota_specified = false;
+    temp_url_key_specified = false;
+    user_quota_specified = false;
+  }
+};
+
+class RGWUser;
+
+class RGWAccessKeyPool
+{
+  RGWUser *user;
+
+  std::map key_type_map;
+  std::string user_id;
+  RGWRados *store;
+
+  map *swift_keys;
+  map *access_keys;
+
+  // we don't want to allow keys for the anonymous user or a null user
+  bool keys_allowed;
+
+private:
+  int create_key(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+  int generate_key(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+  int modify_key(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  int check_key_owner(RGWUserAdminOpState& op_state);
+  bool check_existing_key(RGWUserAdminOpState& op_state);
+  int check_op(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  /* API Contract Fulfilment */
+  int execute_add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+  int execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+
+  int add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+  int remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+public:
+  RGWAccessKeyPool(RGWUser* usr);
+  ~RGWAccessKeyPool();
+
+  int init(RGWUserAdminOpState& op_state);
+
+  /* API Contracted Methods */
+  int add(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+  int remove(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  friend class RGWUser;
+  friend class RGWSubUserPool;
+};
+
+class RGWSubUserPool
+{
+  RGWUser *user;
+
+  string user_id;
+  RGWRados *store;
+  bool subusers_allowed;
+
+  map *subuser_map;
+
+private:
+  int check_op(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  /* API Contract Fulfillment */
+  int execute_add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+  int execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+  int execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+
+  int add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+  int remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+  int modify(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+public:
+  RGWSubUserPool(RGWUser *user);
+  ~RGWSubUserPool();
+
+  bool exists(std::string subuser);
+  int init(RGWUserAdminOpState& op_state);
+
+  /* API contracted methods */
+  int add(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+  int remove(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+  int modify(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  friend class RGWUser;
+};
+
+class RGWUserCapPool
+{
+  RGWUserCaps *caps;
+  bool caps_allowed;
+  RGWUser *user;
+
+private:
+  int add(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+  int remove(RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_save);
+
+public:
+  RGWUserCapPool(RGWUser *user);
+  ~RGWUserCapPool();
+
+  int init(RGWUserAdminOpState& op_state);
+
+  /* API contracted methods */
+  int add(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+  int remove(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  friend class RGWUser;
+};
+
+class RGWUser
+{
+
+private:
+  RGWUserInfo old_info;
+  RGWRados *store;
+
+  string user_id;
+  bool info_stored;
+
+  void set_populated() { info_stored = true; };
+  void clear_populated() { info_stored = false; };
+  bool is_populated() { return info_stored; };
+
+  int check_op(RGWUserAdminOpState&  req, std::string *err_msg);
+  int update(RGWUserAdminOpState& op_state, std::string *err_msg);
+
+  void clear_members();
+  void init_default();
+
+  /* API Contract Fulfillment */
+  int execute_add(RGWUserAdminOpState& op_state, std::string *err_msg);
+  int execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg);
+  int execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg);
+
+public:
+  RGWUser();
+  ~RGWUser();
+
+  int init(RGWRados *storage, RGWUserAdminOpState& op_state);
+
+  int init_storage(RGWRados *storage);
+  int init(RGWUserAdminOpState& op_state);
+  int init_members(RGWUserAdminOpState& op_state);
+
+  RGWRados *get_store() { return store; };
+
+  /* API Contracted Members */
+  RGWUserCapPool caps;
+  RGWAccessKeyPool keys;
+  RGWSubUserPool subusers;
+
+  /* API Contracted Methods */
+  int add(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+  int remove(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  /* remove an already populated RGWUser */
+  int remove(std::string *err_msg = NULL);
+
+  int modify(RGWUserAdminOpState& op_state, std::string *err_msg = NULL);
+
+  /* retrieve info from an existing user in the RGW system */
+  int info(RGWUserAdminOpState& op_state, RGWUserInfo& fetched_info, std::string *err_msg = NULL);
+
+  /* info from an already populated RGWUser */
+  int info (RGWUserInfo& fetched_info, std::string *err_msg = NULL);
+
+  friend class RGWAccessKeyPool;
+  friend class RGWSubUserPool;
+  friend class RGWUserCapPool;
+};
+
+/* Wrapers for admin API functionality */
+
+class RGWUserAdminOp_User
+{
+public:
+  static int info(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+
+  static int create(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+
+  static int modify(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+
+  static int remove(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+};
+
+class RGWUserAdminOp_Subuser
+{
+public:
+  static int create(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+
+  static int modify(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+
+  static int remove(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+};
+
+class RGWUserAdminOp_Key
+{
+public:
+  static int create(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+
+  static int remove(RGWRados *store,
+                  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+};
+
+class RGWUserAdminOp_Caps
+{
+public:
+  static int add(RGWRados *store,
+		  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+
+  static int remove(RGWRados *store,
+		  RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher);
+};
+
+class RGWMetadataManager;
+
+extern void rgw_user_init(RGWMetadataManager *mm);
+
+#endif
diff --git a/ceph/src/rgw/rgw_xml.cc b/ceph/src/rgw/rgw_xml.cc
new file mode 100644
index 00000000..9595a02c
--- /dev/null
+++ b/ceph/src/rgw/rgw_xml.cc
@@ -0,0 +1,237 @@
+#include 
+
+#include 
+#include 
+
+#include "include/types.h"
+
+#include "rgw_common.h"
+#include "rgw_xml.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+XMLObjIter::
+XMLObjIter()
+{
+}
+
+XMLObjIter::
+~XMLObjIter()
+{
+}
+
+void XMLObjIter::
+set(const XMLObjIter::map_iter_t &_cur, const XMLObjIter::map_iter_t &_end)
+{
+  cur = _cur;
+  end = _end;
+}
+
+XMLObj *XMLObjIter::
+get_next()
+{
+  XMLObj *obj = NULL;
+  if (cur != end) {
+    obj = cur->second;
+    ++cur;
+  }
+  return obj;
+};
+
+ostream& operator<<(ostream& out, XMLObj& obj) {
+   out << obj.obj_type << ": " << obj.data;
+   return out;
+}
+
+XMLObj::
+~XMLObj()
+{
+}
+
+bool XMLObj::
+xml_start(XMLObj *parent, const char *el, const char **attr)
+{
+  this->parent = parent;
+  obj_type = el;
+  for (int i = 0; attr[i]; i += 2) {
+    attr_map[attr[i]] = string(attr[i + 1]);
+  }
+  return true;
+}
+
+bool XMLObj::
+xml_end(const char *el)
+{
+  return true;
+}
+
+void XMLObj::
+xml_handle_data(const char *s, int len)
+{
+  data.append(s, len);
+}
+
+string& XMLObj::
+XMLObj::get_data()
+{
+  return data;
+}
+
+XMLObj *XMLObj::
+XMLObj::get_parent()
+{
+  return parent;
+}
+
+void XMLObj::
+add_child(string el, XMLObj *obj)
+{
+  children.insert(pair(el, obj));
+}
+
+bool XMLObj::
+get_attr(string name, string& attr)
+{
+  map::iterator iter = attr_map.find(name);
+  if (iter == attr_map.end())
+    return false;
+  attr = iter->second;
+  return true;
+}
+
+XMLObjIter XMLObj::
+find(string name)
+{
+  XMLObjIter iter;
+  map::iterator first;
+  map::iterator last;
+  first = children.find(name);
+  if (first != children.end()) {
+    last = children.upper_bound(name);
+  }else
+    last = children.end();
+  iter.set(first, last);
+  return iter;
+}
+
+XMLObj *XMLObj::
+find_first(string name)
+{
+  XMLObjIter iter;
+  map::iterator first;
+  first = children.find(name);
+  if (first != children.end())
+    return first->second;
+  return NULL;
+}
+static void xml_start(void *data, const char *el, const char **attr) {
+  RGWXMLParser *handler = static_cast(data);
+
+  if (!handler->xml_start(el, attr))
+    handler->set_failure();
+}
+
+RGWXMLParser::
+RGWXMLParser() : buf(NULL), buf_len(0), cur_obj(NULL), success(true)
+{
+  p = XML_ParserCreate(NULL);
+}
+
+RGWXMLParser::
+~RGWXMLParser()
+{
+  XML_ParserFree(p);
+
+  free(buf);
+  vector::iterator iter;
+  for (iter = objs.begin(); iter != objs.end(); ++iter) {
+    XMLObj *obj = *iter;
+    delete obj;
+  }
+}
+
+bool RGWXMLParser::xml_start(const char *el, const char **attr) {
+  XMLObj * obj = alloc_obj(el);
+  if (!obj) {
+    obj = new XMLObj();
+  }
+  if (!obj->xml_start(cur_obj, el, attr))
+    return false;
+  if (cur_obj) {
+    cur_obj->add_child(el, obj);
+  } else {
+    children.insert(pair(el, obj));
+  }
+  cur_obj = obj;
+
+  objs.push_back(obj);
+  return true;
+}
+
+static void xml_end(void *data, const char *el) {
+  RGWXMLParser *handler = static_cast(data);
+
+  if (!handler->xml_end(el))
+    handler->set_failure();
+}
+
+bool RGWXMLParser::xml_end(const char *el) {
+  XMLObj *parent_obj = cur_obj->get_parent();
+  if (!cur_obj->xml_end(el))
+    return false;
+  cur_obj = parent_obj;
+  return true;
+}
+
+static void handle_data(void *data, const char *s, int len)
+{
+  RGWXMLParser *handler = static_cast(data);
+
+  handler->handle_data(s, len);
+}
+
+void RGWXMLParser::handle_data(const char *s, int len)
+{
+  cur_obj->xml_handle_data(s, len);
+}
+
+
+bool RGWXMLParser::init()
+{
+  if (!p) {
+    return false;
+  }
+  XML_SetElementHandler(p, ::xml_start, ::xml_end);
+  XML_SetCharacterDataHandler(p, ::handle_data);
+  XML_SetUserData(p, (void *)this);
+  return true;
+}
+
+bool RGWXMLParser::parse(const char *_buf, int len, int done)
+{
+  int pos = buf_len;
+  char *tmp_buf;
+  tmp_buf = (char *)realloc(buf, buf_len + len);
+  if (tmp_buf == NULL){
+    free(buf);
+    buf = NULL;
+    return false;
+  } else {
+    buf = tmp_buf;
+  }
+
+  memcpy(&buf[buf_len], _buf, len);
+  buf_len += len;
+
+  success = true;
+  if (!XML_Parse(p, &buf[pos], len, done)) {
+    fprintf(stderr, "Parse error at line %d:\n%s\n",
+	      (int)XML_GetCurrentLineNumber(p),
+	      XML_ErrorString(XML_GetErrorCode(p)));
+    success = false;
+  }
+
+  return success;
+}
diff --git a/ceph/src/rgw/rgw_xml.h b/ceph/src/rgw/rgw_xml.h
new file mode 100644
index 00000000..5004e98b
--- /dev/null
+++ b/ceph/src/rgw/rgw_xml.h
@@ -0,0 +1,85 @@
+#ifndef CEPH_RGW_XML_H
+#define CEPH_RGW_XML_H
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+using namespace std;
+
+
+class XMLObj;
+
+class XMLObjIter {
+  typedef map::iterator map_iter_t;
+  map_iter_t cur;
+  map_iter_t end;
+public:
+  XMLObjIter();
+  ~XMLObjIter();
+  void set(const XMLObjIter::map_iter_t &_cur, const XMLObjIter::map_iter_t &_end);
+  XMLObj *get_next();
+};
+
+/**
+ * Represents a block of XML.
+ * Give the class an XML blob, and it will parse the blob into
+ * an attr_name->value map.
+ * This really ought to be an abstract class or something; it
+ * shouldn't be the startpoint for any parsing. Look at RGWXMLParser for that.
+ */
+class XMLObj
+{
+  XMLObj *parent;
+  string obj_type;
+protected:
+  string data;
+  multimap children;
+  map attr_map;
+public:
+
+  XMLObj() : parent(NULL) {}
+
+  virtual ~XMLObj();
+  bool xml_start(XMLObj *parent, const char *el, const char **attr);
+  virtual bool xml_end(const char *el);
+  virtual void xml_handle_data(const char *s, int len);
+  string& get_data();
+  XMLObj *get_parent();
+  void add_child(string el, XMLObj *obj);
+  bool get_attr(string name, string& attr);
+  XMLObjIter find(string name);
+  XMLObj *find_first(string name);
+
+  friend ostream& operator<<(ostream& out, XMLObj& obj);
+};
+
+class RGWXMLParser : public XMLObj
+{
+  XML_Parser p;
+  char *buf;
+  int buf_len;
+  XMLObj *cur_obj;
+  vector objs;
+protected:
+  virtual XMLObj *alloc_obj(const char *el) = 0;
+public:
+  RGWXMLParser();
+  virtual ~RGWXMLParser();
+  bool init();
+  bool xml_start(const char *el, const char **attr);
+  bool xml_end(const char *el);
+  void handle_data(const char *s, int len);
+
+  bool parse(const char *buf, int len, int done);
+  const char *get_xml() { return buf; }
+  void set_failure() { success = false; }
+
+private:
+  bool success;
+};
+
+#endif
diff --git a/ceph/src/sample.ceph.conf b/ceph/src/sample.ceph.conf
new file mode 100644
index 00000000..139a05cf
--- /dev/null
+++ b/ceph/src/sample.ceph.conf
@@ -0,0 +1,364 @@
+##
+# Sample ceph ceph.conf file.
+##
+# This file defines cluster membership, the various locations
+# that Ceph stores data, and any other runtime options.
+
+# If a 'host' is defined for a daemon, the init.d start/stop script will
+# verify that it matches the hostname (or else ignore it).  If it is
+# not defined, it is assumed that the daemon is intended to start on
+# the current host (e.g., in a setup with a startup.conf on each
+# node).
+
+## Metavariables
+# $cluster    ; Expands to the Ceph Storage Cluster name. Useful
+#             ; when running multiple Ceph Storage Clusters
+#             ; on the same hardware.
+#             ; Example: /etc/ceph/$cluster.keyring
+#             ; (Default: ceph)
+#
+# $type       ; Expands to one of mds, osd, or mon, depending on
+#             ; the type of the instant daemon.
+#             ; Example: /var/lib/ceph/$type
+#
+# $id         ; Expands to the daemon identifier. For osd.0, this
+#             ; would be 0; for mds.a, it would be a.
+#             ; Example: /var/lib/ceph/$type/$cluster-$id
+#
+# $host       ; Expands to the host name of the instant daemon.
+#
+# $name       ; Expands to $type.$id.
+#             ; Example: /var/run/ceph/$cluster-$name.asok
+
+[global]
+### http://ceph.com/docs/master/rados/configuration/general-config-ref/
+
+    ;fsid                       = {UUID}    # use `uuidgen` to generate your own UUID
+    ;public network             = 192.168.0.0/24
+    ;cluster network            = 192.168.0.0/24
+
+    # Each running Ceph daemon has a running process identifier (PID) file.
+    # The PID file is generated upon start-up.
+    # Type: String (optional)
+    # (Default: N/A). The default path is /var/run/$cluster/$name.pid.
+    pid file                   = /var/run/ceph/$name.pid
+
+    # If set, when the Ceph Storage Cluster starts, Ceph sets the max open fds
+    # at the OS level (i.e., the max # of file descriptors).
+    # It helps prevents Ceph OSD Daemons from running out of file descriptors.
+    # Type: 64-bit Integer (optional)
+    # (Default: 0)
+    ;max open files             = 131072
+
+
+### http://ceph.com/docs/master/rados/operations/authentication
+### http://ceph.com/docs/master/rados/configuration/auth-config-ref/
+
+    # If enabled, the Ceph Storage Cluster daemons (i.e., ceph-mon, ceph-osd,
+    # and ceph-mds) must authenticate with each other.
+    # Type: String (optional); Valid settings are "cephx" or "none".
+    # (Default: cephx)
+    auth cluster required      = cephx
+
+    # If enabled, the Ceph Storage Cluster daemons require Ceph Clients to
+    # authenticate with the Ceph Storage Cluster in order to access Ceph
+    # services.
+    # Type: String (optional); Valid settings are "cephx" or "none".
+    # (Default: cephx)
+    auth service required      = cephx
+
+    # If enabled, the Ceph Client requires the Ceph Storage Cluster to
+    # authenticate with the Ceph Client.
+    # Type: String (optional); Valid settings are "cephx" or "none".
+    # (Default: cephx)
+    auth client required       = cephx
+
+    # If set to true, Ceph requires signatures on all message traffic between
+    # the Ceph Client and the Ceph Storage Cluster, and between daemons
+    # comprising the Ceph Storage Cluster.
+    # Type: Boolean (optional)
+    # (Default: false)
+    cephx require signatures   = true    ; everywhere possible
+
+    # The path to the keyring file.
+    # Type: String (optional)
+    # Default: /etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin
+    ;keyring                  = /etc/ceph/$cluster.$name.keyring
+
+
+### http://ceph.com/docs/master/rados/configuration/pool-pg-config-ref/
+
+
+    ## Replication level, number of data copies.
+    # Type: 32-bit Integer
+    # (Default: 2)
+    ;osd pool default size      = 2
+
+    ## Replication level in degraded state, less than 'osd pool default size' value.
+    # Sets the minimum number of written replicas for objects in the
+    # pool in order to acknowledge a write operation to the client. If
+    # minimum is not met, Ceph will not acknowledge the write to the
+    # client. This setting ensures a minimum number of replicas when
+    # operating in degraded mode.
+    # Type: 32-bit Integer
+    # (Default: 0), which means no particular minimum. If 0, minimum is size - (size / 2).
+    ;osd pool default min size  = 1
+
+    ## Ensure you have a realistic number of placement groups. We recommend
+    ## approximately 100 per OSD. E.g., total number of OSDs multiplied by 100
+    ## divided by the number of replicas (i.e., osd pool default size). So for
+    ## 10 OSDs and osd pool default size = 3, we'd recommend approximately
+    ## (100 * 10) / 3 = 333
+
+    # Description: The default number of placement groups for a pool. The
+    #              default value is the same as pg_num with mkpool.
+    # Type: 32-bit Integer
+    # (Default: 8)
+    ;osd pool default pg num    = 100
+
+    # Description: The default number of placement groups for placement for a
+    #              pool. The default value is the same as pgp_num with mkpool.
+    #              PG and PGP should be equal (for now).
+    # Type: 32-bit Integer
+    # (Default: 8)
+    ;osd pool default pgp num   = 100
+
+    # The default CRUSH ruleset to use when creating a pool
+    # Type: 32-bit Integer
+    # (Default: 0)
+    ;osd pool default crush rule = 0
+
+    # The bucket type to use for chooseleaf in a CRUSH rule.
+    # Uses ordinal rank rather than name.
+    # Type: 32-bit Integer
+    # (Default: 1) Typically a host containing one or more Ceph OSD Daemons.
+    ;osd crush chooseleaf type = 1
+
+
+### http://ceph.com/docs/bobtail/rados/configuration/log-and-debug-ref/
+
+    # Default: /var/log/ceph/$cluster-$name.log
+    ;log file                   = /var/log/ceph/$cluster-$name.log
+
+    ;log to syslog              = true
+
+
+### http://ceph.com/docs/master/rados/configuration/ms-ref/
+
+    # Enable if you want your daemons to bind to IPv6 address instead of
+    # IPv4 ones. (Not required if you specify a daemon or cluster IP.)
+    # Type: Boolean
+    # (Default: false)
+    ;ms bind ipv6               = true
+
+
+### http://ceph.com/docs/master/rados/configuration/filestore-config-ref/
+
+    # The maximum interval in seconds for synchronizing the filestore.
+    # Type: Double (optional)
+    # (Default: 5)
+    ;filestore max sync interval = 5
+
+    # Use object map for XATTRS. Set to true for ext4 file systems only.
+    # Type: Boolean (optional)
+    # (Default: false)
+    ;filestore xattr use omap    = true
+
+### http://ceph.com/docs/master/rados/configuration/journal-ref/
+
+##################
+## Monitors
+## You need at least one. You need at least three if you want to
+## tolerate any node failures. Always create an odd number.
+[mon]
+### http://ceph.com/docs/argonaut/config-ref/mon-config/
+### http://ceph.com/docs/master/rados/configuration/mon-config-ref/
+### http://ceph.com/docs/dumpling/rados/configuration/mon-osd-interaction/
+
+    # The IDs of initial monitors in a cluster during startup.
+    # If specified, Ceph requires an odd number of monitors to form an
+    # initial quorum (e.g., 3).
+    # Type: String
+    # (Default: None)
+    ;mon initial members        = mycephhost
+
+    ;mon host                   = cephhost01,cephhost02
+    ;mon addr                   = 192.168.0.101,192.168.0.102
+
+    # The monitor’s data location
+    # Default: /var/lib/ceph/mon/$cluster-$id
+    ;mon data                   = /var/lib/ceph/mon/$name
+
+    # The clock drift in seconds allowed between monitors.
+    # Type: Float
+    # (Default: .050)
+    ;mon clock drift allowed    = .15
+
+    # Exponential backoff for clock drift warnings
+    # Type: Float
+    # (Default: 5)
+    ;mon clock drift warn backoff = 30    ; Tell the monitor to backoff from this warning for 30 seconds
+
+    # The percentage of disk space used before an OSD is considered full.
+    # Type: Float
+    # (Default: .95)
+    ;mon osd full ratio         = .95
+
+    # The percentage of disk space used before an OSD is considered nearfull.
+    # Type: Float
+    # (Default: .85)
+    ;mon osd nearfull ratio     = .85
+
+
+### http://ceph.com/docs/next/rados/troubleshooting/log-and-debug/
+
+    # logging, for debugging monitor crashes, in order of
+    # their likelihood of being helpful :)
+    ;debug ms                   = 1
+    ;debug mon                  = 20
+    ;debug paxos                = 20
+    ;debug auth                 = 20
+
+
+;[mon.alpha]
+;    host                       = alpha
+;    mon addr                   = 192.168.0.10:6789
+
+;[mon.beta]
+;    host                       = beta
+;    mon addr                   = 192.168.0.11:6789
+
+;[mon.gamma]
+;    host                       = gamma
+;    mon addr                   = 192.168.0.12:6789
+
+
+##################
+## Metadata servers
+# You must deploy at least one metadata server to use CephFS. There is
+# experimental support for running multiple metadata servers. Do not run
+# multiple metadata servers in production.
+[mds]
+### http://ceph.com/docs/argonaut/config-ref/mds-config/
+### http://ceph.com/docs/master/cephfs/mds-config-ref/
+
+    # where the mds keeps it's secret encryption keys
+    ;keyring                    = /var/lib/ceph/mds/$name/keyring
+
+    ; mds logging to debug issues.
+    ;debug ms                   = 1
+    ;debug mds                  = 20
+
+
+;[mds.alpha]
+;    host                       = alpha
+
+;[mds.beta]
+;    host                       = beta
+
+##################
+## osd
+# You need at least one.  Two or more if you want data to be replicated.
+# Define as many as you like.
+[osd]
+### http://ceph.com/docs/argonaut/config-ref/osd-config/
+### http://ceph.com/docs/bobtail/rados/configuration/osd-config-ref/
+
+    # The path to the OSDs data.
+    # You must create the directory when deploying Ceph.
+    # You should mount a drive for OSD data at this mount point.
+    # We do not recommend changing the default.
+    # Type: String
+    # Default: /var/lib/ceph/osd/$cluster-$id
+    ;osd data                     = /var/lib/ceph/osd/$name
+
+    ## You can change the number of recovery operations to speed up recovery
+    ## or slow it down if your machines can't handle it
+
+    # The number of active recovery requests per OSD at one time.
+    # More requests will accelerate recovery, but the requests
+    # places an increased load on the cluster.
+    # Type: 32-bit Integer
+    # (Default: 5)
+    ;osd recovery max active      = 3
+
+
+    # You may add settings for mkcephfs so that it will create and mount
+    # the file system for you. Remove the comment `#` character for
+    # the following settings and replace the values in parenthesis
+    # with appropriate values, or leave the following settings commented
+    # out to accept the default values. You must specify the --mkfs
+    # option with mkcephfs in order for the deployment script to
+    # utilize the following settings, and you must define the 'devs'
+    # option for each osd instance; see below.
+
+    #osd mkfs type = {fs-type}
+    #osd mkfs options {fs-type}   = {mkfs options}   # default for xfs is "-f"
+    #osd mount options {fs-type}  = {mount options} # default mount option is "rw, noatime"
+    ;osd mkfs type                = btrfs
+    ;osd mount options btrfs      = noatime,nodiratime
+
+    ## Ideally, make this a separate disk or partition.  A few
+    ## hundred MB should be enough; more if you have fast or many
+    ## disks.  You can use a file under the osd data dir if need be
+    ## (e.g. /data/$name/journal), but it will be slower than a
+    ## separate disk or partition.
+    # The path to the OSD’s journal. This may be a path to a file or a block
+    # device (such as a partition of an SSD). If it is a file, you must
+    # create the directory to contain it.
+    # We recommend using a drive separate from the osd data drive.
+    # Type: String
+    # Default: /var/lib/ceph/osd/$cluster-$id/journal
+    ;osd journal                  = /var/lib/ceph/osd/$name/journal
+
+### http://ceph.com/docs/master/rados/configuration/journal-ref/
+
+    # The size of the journal in megabytes. If this is 0,
+    # and the journal is a block device, the entire block device is used.
+    # Since v0.54, this is ignored if the journal is a block device,
+    # and the entire block device is used.
+    # Type: 32-bit Integer
+    # (Default: 5120)
+    # Recommended: Begin with 1GB. Should be at least twice the product
+    # of the expected speed multiplied by "filestore max sync interval".
+    ;osd journal size             = 1000     ; journal size, in megabytes
+
+    ## If you want to run the journal on a tmpfs, disable DirectIO
+    # Enables direct i/o to the journal.
+    # Requires journal block align set to true.
+    # Type: Boolean
+    # Required: Yes when using aio.
+    # (Default: true)
+    ;journal dio                  = false
+
+    # Check log files for corruption. Can be computationally expensive.
+    # Type: Boolean
+    # (Default: false)
+    ;osd check for log corruption = true
+
+    # osd logging to debug osd issues, in order of likelihood of being helpful
+    ;debug ms                     = 1
+    ;debug osd                    = 20
+    ;debug filestore              = 20
+    ;debug journal                = 20
+
+
+
+;[osd.0]
+;    host                         = delta
+
+    # If 'devs' is not specified, you're responsible for
+    # setting up the 'osd data' dir (e.g. `mkdir /var/lib/ceph/osd/ceph-0`)
+    ;devs                         = /dev/sdx
+
+;[osd.1]
+;    host                         = epsilon
+;    devs                         = /dev/sdy
+
+;[osd.2]
+;    host                         = zeta
+;    devs                         = /dev/sdx
+
+;[osd.3]
+;    host                         = eta
+;    devs                         = /dev/sdy
diff --git a/ceph/src/stop.sh b/ceph/src/stop.sh
new file mode 100755
index 00000000..d7ce234c
--- /dev/null
+++ b/ceph/src/stop.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 Inktank 
+# Copyright (C) 2013 Cloudwatt 
+#
+# Author: Loic Dachary 
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+
+test -d dev/osd0/. && test -e dev/sudo && SUDO="sudo"
+
+[ -z "$CEPH_BIN" ] && CEPH_BIN=.
+
+do_killall() {
+    pg=`pgrep -f ceph-run.*$1`
+    [ -n "$pg" ] && kill $pg
+    $SUDO killall $1
+}
+
+usage="usage: $0 [all] [mon] [mds] [osd]\n"
+
+stop_all=1
+stop_mon=0
+stop_mds=0
+stop_osd=0
+stop_rgw=0
+
+while [ $# -ge 1 ]; do
+    case $1 in
+        all )
+            stop_all=1
+            ;;
+        mon | ceph-mon )
+            stop_mon=1
+            stop_all=0
+            ;;
+        mds | ceph-mds )
+            stop_mds=1
+            stop_all=0
+            ;;
+        osd | ceph-osd )
+            stop_osd=1
+            stop_all=0
+            ;;
+        * )
+            printf "$usage"
+            exit
+    esac
+    shift
+done
+
+if [ $stop_all -eq 1 ]; then
+    while read DEV; do
+        # While it is currently possible to create an rbd image with
+        # whitespace chars in its name, krbd will refuse mapping such
+        # an image, so we can safely split on whitespace here.  (The
+        # same goes for whitespace chars in names of the pools that
+        # contain rbd images).
+        DEV="$(echo "${DEV}" | tr -s '[:space:]' | awk '{ print $5 }')"
+        sudo "${CEPH_BIN}"/rbd unmap "${DEV}"
+    done < <("${CEPH_BIN}"/rbd showmapped | tail -n +2)
+
+    if [ -n "$("${CEPH_BIN}"/rbd showmapped)" ]; then
+        echo "WARNING: Some rbd images are still mapped!" >&2
+    fi
+
+    for p in ceph-mon ceph-mds ceph-osd radosgw lt-radosgw apache2 ; do
+        for try in 0 1 1 1 1 ; do
+            if ! pkill $p ; then
+                break
+            fi
+            sleep $try
+        done
+    done
+
+    pkill -f valgrind.bin.\*ceph-mon
+    $SUDO pkill -f valgrind.bin.\*ceph-osd
+    pkill -f valgrind.bin.\*ceph-mds
+else
+    [ $stop_mon -eq 1 ] && do_killall ceph-mon
+    [ $stop_mds -eq 1 ] && do_killall ceph-mds
+    [ $stop_osd -eq 1 ] && do_killall ceph-osd
+    [ $stop_rgw -eq 1 ] && do_killall radosgw lt-radosgw apache2
+fi
diff --git a/ceph/src/test/Makefile.am b/ceph/src/test/Makefile.am
new file mode 100644
index 00000000..f527b96b
--- /dev/null
+++ b/ceph/src/test/Makefile.am
@@ -0,0 +1,907 @@
+include test/erasure-code/Makefile.am
+
+## Unknown/other tests
+
+ceph_test_timers_SOURCES = test/TestTimers.cc
+ceph_test_timers_LDADD = $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_timers
+
+ceph_test_signal_handlers_SOURCES = test/TestSignalHandlers.cc
+ceph_test_signal_handlers_LDADD = $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_signal_handlers
+
+ceph_test_rados_SOURCES = \
+	test/osd/TestRados.cc \
+	test/osd/TestOpStat.cc \
+	test/osd/Object.cc \
+	test/osd/RadosModel.cc
+ceph_test_rados_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_rados
+
+ceph_test_mutate_SOURCES = test/test_mutate.cc
+ceph_test_mutate_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_mutate
+
+ceph_test_rewrite_latency_SOURCES = test/test_rewrite_latency.cc
+ceph_test_rewrite_latency_LDADD = $(LIBCOMMON) $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS)
+bin_DEBUGPROGRAMS += ceph_test_rewrite_latency
+
+ceph_test_msgr_SOURCES = test/testmsgr.cc
+ceph_test_msgr_LDADD = $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_msgr
+
+ceph_streamtest_SOURCES = test/streamtest.cc
+ceph_streamtest_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_streamtest
+
+ceph_test_trans_SOURCES = test/test_trans.cc
+ceph_test_trans_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_trans
+
+ceph_test_crypto_SOURCES = test/testcrypto.cc
+ceph_test_crypto_LDADD = $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_crypto
+
+ceph_test_keys_SOURCES = test/testkeys.cc
+ceph_test_keys_LDADD = $(LIBMON) $(CEPH_GLOBAL) 
+bin_DEBUGPROGRAMS += ceph_test_keys
+
+
+## Dencoder test
+
+ceph_dencoder_SOURCES = \
+	test/encoding/ceph_dencoder.cc \
+	$(DENCODER_SOURCES)
+ceph_dencoder_LDADD = \
+	$(LIBOSD_TYPES) \
+	$(LIBOS_TYPES) \
+	$(LIBMDS) \
+	$(LIBMON_TYPES) \
+	$(DENCODER_DEPS) \
+	$(CEPH_GLOBAL)
+
+# These should always use explicit _CFLAGS/_CXXFLAGS so avoid basename conflicts
+ceph_dencoder_CFLAGS = ${AM_CFLAGS}
+ceph_dencoder_CXXFLAGS = ${AM_CXXFLAGS}
+
+if COMPILER_HAS_VTA
+ceph_dencoder_CFLAGS += -fno-var-tracking-assignments
+ceph_dencoder_CXXFLAGS += -fno-var-tracking-assignments
+endif
+
+bin_PROGRAMS += ceph-dencoder
+
+get_command_descriptions_SOURCES = test/common/get_command_descriptions.cc
+get_command_descriptions_LDADD = $(LIBMON) $(LIBCOMMON) $(CEPH_GLOBAL)
+noinst_PROGRAMS += get_command_descriptions
+
+
+## Build tests
+# These should all use explicit _CXXFLAGS so avoid basename conflicts
+
+if WITH_BUILD_TESTS
+test_build_libcommon_SOURCES = \
+	test/buildtest_skeleton.cc \
+	$(libcommon_la_SOURCES)
+test_build_libcommon_LDADD = \
+	$(LIBCOMMON_DEPS) \
+	$(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS)
+test_build_libcommon_LDFLAGS = -static-libtool-libs
+test_build_libcommon_CFLAGS = $(AM_CFLAGS)
+test_build_libcommon_CXXFLAGS = $(AM_CXXFLAGS)
+bin_DEBUGPROGRAMS += test_build_libcommon
+
+test_build_librados_SOURCES = \
+	test/buildtest_skeleton.cc \
+	$(librados_la_SOURCES)
+test_build_librados_LDADD = \
+	$(LIBRADOS_DEPS) \
+	$(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS)
+test_build_librados_LDFLAGS = -static-libtool-libs
+test_build_librados_CFLAGS = $(AM_CFLAGS)
+test_build_librados_CXXFLAGS = $(AM_CXXFLAGS)
+bin_DEBUGPROGRAMS += test_build_librados
+
+test_build_librgw_SOURCES = \
+	test/buildtest_skeleton.cc \
+	$(librgw_la_SOURCES)
+test_build_librgw_LDADD = \
+	$(LIBRGW_DEPS) \
+	$(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS) \
+	$(CEPH_GLOBAL)
+test_build_librgw_LDFLAGS = -static-libtool-libs
+test_build_librgw_CFLAGS = $(AM_CFLAGS)
+test_build_librgw_CXXFLAGS = $(AM_CXXFLAGS)
+bin_DEBUGPROGRAMS += test_build_librgw
+
+# I dont get this one... testing the osdc build but link in libcephfs?
+test_build_libcephfs_SOURCES = \
+	test/buildtest_skeleton.cc \
+	$(libosdc_la_SOURCES)
+test_build_libcephfs_LDADD = \
+	$(LIBCEPHFS) -lexpat \
+	$(PTHREAD_LIBS) $(CRYPTO_LIBS) $(EXTRALIBS)
+test_build_libcephfs_LDFLAGS = -static-libtool-libs
+test_build_libcephfs_CFLAGS = $(AM_CFLAGS)
+test_build_libcephfs_CXXFLAGS = $(AM_CXXFLAGS)
+bin_DEBUGPROGRAMS += test_build_libcephfs
+
+endif # WITH_BUILD_TESTS
+
+
+## Benchmarks
+
+ceph_smalliobench_SOURCES = \
+	test/bench/small_io_bench.cc \
+	test/bench/rados_backend.cc \
+	test/bench/detailed_stat_collector.cc \
+	test/bench/bencher.cc
+ceph_smalliobench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_smalliobench
+
+ceph_smalliobenchfs_SOURCES = \
+	test/bench/small_io_bench_fs.cc \
+	test/bench/testfilestore_backend.cc \
+	test/bench/detailed_stat_collector.cc \
+	test/bench/bencher.cc
+ceph_smalliobenchfs_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_smalliobenchfs
+
+ceph_smalliobenchdumb_SOURCES = \
+	test/bench/small_io_bench_dumb.cc \
+	test/bench/dumb_backend.cc \
+	test/bench/detailed_stat_collector.cc \
+	test/bench/bencher.cc
+ceph_smalliobenchdumb_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_smalliobenchdumb
+
+ceph_smalliobenchrbd_SOURCES = \
+	test/bench/small_io_bench_rbd.cc \
+	test/bench/rbd_backend.cc \
+	test/bench/detailed_stat_collector.cc \
+	test/bench/bencher.cc
+ceph_smalliobenchrbd_LDADD = $(LIBRBD) $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_smalliobenchrbd
+
+ceph_tpbench_SOURCES = \
+	test/bench/tp_bench.cc \
+	test/bench/detailed_stat_collector.cc
+ceph_tpbench_LDADD = $(LIBRADOS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_tpbench
+
+ceph_omapbench_SOURCES = test/omap_bench.cc
+ceph_omapbench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_omapbench
+
+if LINUX
+ceph_kvstorebench_SOURCES = \
+	test/kv_store_bench.cc \
+	key_value_store/kv_flat_btree_async.cc
+ceph_kvstorebench_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_kvstorebench
+endif
+
+## System tests
+
+if LINUX
+libsystest_la_SOURCES = \
+	test/system/cross_process_sem.cc \
+	test/system/systest_runnable.cc \
+	test/system/systest_settings.cc
+libsystest_la_LIBADD = $(CEPH_GLOBAL)
+noinst_LTLIBRARIES += libsystest.la
+
+ceph_test_rados_list_parallel_SOURCES = \
+	test/system/rados_list_parallel.cc \
+	test/system/st_rados_create_pool.cc \
+	test/system/st_rados_list_objects.cc
+ceph_test_rados_list_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+bin_DEBUGPROGRAMS += ceph_test_rados_list_parallel
+
+ceph_test_rados_open_pools_parallel_SOURCES = \
+	test/system/rados_open_pools_parallel.cc \
+	test/system/st_rados_create_pool.cc
+ceph_test_rados_open_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+bin_DEBUGPROGRAMS += ceph_test_rados_open_pools_parallel
+
+ceph_test_rados_delete_pools_parallel_SOURCES = \
+	test/system/rados_delete_pools_parallel.cc \
+	test/system/st_rados_create_pool.cc \
+	test/system/st_rados_delete_pool.cc \
+	test/system/st_rados_list_objects.cc
+ceph_test_rados_delete_pools_parallel_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+bin_DEBUGPROGRAMS += ceph_test_rados_delete_pools_parallel
+
+ceph_test_rados_watch_notify_SOURCES = \
+	test/system/rados_watch_notify.cc \
+	test/system/st_rados_create_pool.cc \
+	test/system/st_rados_delete_pool.cc \
+	test/system/st_rados_delete_objs.cc \
+	test/system/st_rados_watch.cc \
+	test/system/st_rados_notify.cc
+ceph_test_rados_watch_notify_LDADD = $(LIBRADOS) libsystest.la $(PTHREAD_LIBS)
+bin_DEBUGPROGRAMS += ceph_test_rados_watch_notify
+endif
+
+ceph_bench_log_SOURCES = test/bench_log.cc
+ceph_bench_log_LDADD = $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_bench_log
+
+
+
+## Unit tests
+
+check_SCRIPTS += \
+	unittest_bufferlist.sh \
+	test/encoding/check-generated.sh \
+	test/mon/osd-pool-create.sh \
+	test/mon/misc.sh \
+	test/mon/osd-crush.sh \
+	test/mon/osd-erasure-code-profile.sh \
+	test/mon/mkfs.sh \
+	test/ceph-disk.sh \
+	test/mon/mon-handle-forward.sh \
+	test/vstart_wrapped_tests.sh
+
+EXTRA_DIST += \
+	$(srcdir)/test/mon/mon-test-helpers.sh \
+	$(srcdir)/test/osd/osd-test-helpers.sh
+
+# target to build but not run the unit tests
+unittests:: $(check_PROGRAMS)
+
+UNITTEST_CXXFLAGS = \
+	$(AM_CXXFLAGS) \
+	-I$(top_srcdir)/src/gtest/include \
+	-I$(top_builddir)/src/gtest/include
+UNITTEST_LDADD = \
+	$(top_builddir)/src/gtest/lib/libgtest.a \
+	$(top_builddir)/src/gtest/lib/libgtest_main.a \
+	$(PTHREAD_LIBS)
+
+unittest_encoding_SOURCES = test/encoding.cc
+unittest_encoding_LDADD = $(LIBCEPHFS) $(LIBRADOS) -lm $(UNITTEST_LDADD)
+unittest_encoding_CXXFLAGS = $(UNITTEST_CXXFLAGS) -fno-strict-aliasing
+check_PROGRAMS += unittest_encoding
+
+unittest_addrs_SOURCES = test/test_addrs.cc
+unittest_addrs_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_addrs_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_addrs
+
+unittest_bloom_filter_SOURCES = test/common/test_bloom_filter.cc
+unittest_bloom_filter_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_bloom_filter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_bloom_filter
+
+unittest_histogram_SOURCES = test/common/histogram.cc
+unittest_histogram_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_histogram_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_histogram
+
+unittest_str_map_SOURCES = test/common/test_str_map.cc
+unittest_str_map_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_str_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_str_map
+
+unittest_sharedptr_registry_SOURCES = test/common/test_sharedptr_registry.cc
+unittest_sharedptr_registry_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_sharedptr_registry_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_sharedptr_registry
+
+unittest_sloppy_crc_map_SOURCES = test/common/test_sloppy_crc_map.cc
+unittest_sloppy_crc_map_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_sloppy_crc_map_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_sloppy_crc_map
+
+unittest_util_SOURCES = test/common/test_util.cc
+unittest_util_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_util_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CRYPTO_LIBS) $(EXTRALIBS)
+check_PROGRAMS += unittest_util
+
+unittest_crush_indep_SOURCES = test/crush/indep.cc
+unittest_crush_indep_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_crush_indep_LDADD = $(LIBCOMMON) -lm $(UNITTEST_LDADD) $(CEPH_CRUSH) $(EXTRALIBS) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_crush_indep
+
+unittest_osdmap_SOURCES = test/osd/TestOSDMap.cc
+unittest_osdmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_osdmap_LDADD = $(UNITTEST_LDADD) $(LIBCOMMON) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_osdmap
+
+unittest_workqueue_SOURCES = test/test_workqueue.cc
+unittest_workqueue_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_workqueue_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_workqueue
+
+unittest_striper_SOURCES = test/test_striper.cc 
+unittest_striper_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_striper_LDADD = $(LIBOSDC) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_striper
+
+unittest_prebufferedstreambuf_SOURCES = test/test_prebufferedstreambuf.cc 
+unittest_prebufferedstreambuf_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_prebufferedstreambuf_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(EXTRALIBS)
+check_PROGRAMS += unittest_prebufferedstreambuf
+
+unittest_str_list_SOURCES = test/test_str_list.cc
+unittest_str_list_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_str_list_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_str_list
+
+unittest_log_SOURCES = log/test.cc
+unittest_log_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD)
+unittest_log_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
+check_PROGRAMS += unittest_log
+
+unittest_throttle_SOURCES = test/common/Throttle.cc
+unittest_throttle_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_throttle_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
+check_PROGRAMS += unittest_throttle
+
+unittest_crush_wrapper_SOURCES = test/crush/TestCrushWrapper.cc
+unittest_crush_wrapper_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(LIBCRUSH)
+unittest_crush_wrapper_CXXFLAGS = $(UNITTEST_CXXFLAGS) -O2
+check_PROGRAMS += unittest_crush_wrapper
+
+unittest_base64_SOURCES = test/base64.cc
+unittest_base64_LDADD = $(LIBCEPHFS) -lm $(UNITTEST_LDADD)
+unittest_base64_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_base64
+
+unittest_ceph_argparse_SOURCES = test/ceph_argparse.cc
+unittest_ceph_argparse_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_ceph_argparse_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_ceph_argparse
+
+unittest_ceph_compatset_SOURCES = test/ceph_compatset.cc
+unittest_ceph_compatset_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_ceph_compatset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_ceph_compatset
+
+unittest_osd_types_SOURCES = test/osd/types.cc
+unittest_osd_types_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) 
+check_PROGRAMS += unittest_osd_types
+
+unittest_pglog_SOURCES = test/osd/TestPGLog.cc
+unittest_pglog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_pglog_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_pglog
+
+unittest_ecbackend_SOURCES = test/osd/TestECBackend.cc
+unittest_ecbackend_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_ecbackend_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_ecbackend
+
+unittest_hitset_SOURCES = test/osd/hitset.cc
+unittest_hitset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_hitset_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_hitset
+
+if LINUX
+unittest_pglog_LDADD += -ldl
+endif # LINUX
+
+unittest_gather_SOURCES = test/gather.cc
+unittest_gather_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_gather_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_gather
+
+unittest_run_cmd_SOURCES = test/run_cmd.cc
+unittest_run_cmd_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD)
+unittest_run_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_run_cmd
+
+unittest_signals_SOURCES = test/signals.cc
+unittest_signals_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_signals_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_signals
+
+unittest_simple_spin_SOURCES = test/simple_spin.cc
+unittest_simple_spin_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD)
+unittest_simple_spin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_simple_spin
+
+unittest_librados_SOURCES = test/librados/librados.cc
+unittest_librados_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+unittest_librados_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_librados
+
+unittest_bufferlist_SOURCES = test/bufferlist.cc
+unittest_bufferlist_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) 
+unittest_bufferlist_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_bufferlist
+
+unittest_crc32c_SOURCES = test/common/test_crc32c.cc
+unittest_crc32c_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_crc32c_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_crc32c
+
+unittest_arch_SOURCES = test/test_arch.cc
+unittest_arch_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_arch_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_arch
+
+unittest_crypto_SOURCES = test/crypto.cc
+unittest_crypto_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_crypto_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_crypto
+
+unittest_crypto_init_SOURCES = test/crypto_init.cc
+unittest_crypto_init_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_crypto_init_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_crypto_init
+
+unittest_perf_counters_SOURCES = test/perf_counters.cc
+unittest_perf_counters_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_perf_counters_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_perf_counters
+
+unittest_admin_socket_SOURCES = test/admin_socket.cc
+unittest_admin_socket_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_admin_socket_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_admin_socket
+
+unittest_ceph_crypto_SOURCES = test/ceph_crypto.cc
+unittest_ceph_crypto_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_ceph_crypto_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_ceph_crypto
+
+unittest_utf8_SOURCES = test/utf8.cc
+unittest_utf8_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_utf8_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_utf8
+
+unittest_mime_SOURCES = test/mime.cc
+unittest_mime_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_mime_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_mime
+
+unittest_escape_SOURCES = test/escape.cc
+unittest_escape_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_escape_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_escape
+
+unittest_chain_xattr_SOURCES = test/objectstore/chain_xattr.cc
+unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_chain_xattr
+
+unittest_flatindex_SOURCES = test/os/TestFlatIndex.cc
+unittest_flatindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_flatindex_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_flatindex
+
+unittest_strtol_SOURCES = test/strtol.cc
+unittest_strtol_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_strtol_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_strtol
+
+unittest_confutils_SOURCES = test/confutils.cc
+unittest_confutils_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_confutils_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_confutils
+
+unittest_config_SOURCES = test/common/test_config.cc
+unittest_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_config
+
+unittest_context_SOURCES = test/common/test_context.cc
+unittest_context_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_context_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_context
+
+unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc
+unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_heartbeatmap
+
+# why does this include rgw/rgw_formats.cc...?
+unittest_formatter_SOURCES = \
+	test/formatter.cc \
+	rgw/rgw_formats.cc
+unittest_formatter_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_formatter_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_formatter
+
+unittest_libcephfs_config_SOURCES = test/libcephfs_config.cc
+unittest_libcephfs_config_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD)
+unittest_libcephfs_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_libcephfs_config
+
+unittest_lfnindex_SOURCES = test/os/TestLFNIndex.cc
+unittest_lfnindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_lfnindex_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_lfnindex
+
+unittest_librados_config_SOURCES = test/librados/librados_config.cc
+unittest_librados_config_LDADD = $(LIBRADOS) $(UNITTEST_LDADD)
+unittest_librados_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_librados_config
+
+#unittest_librgw_link_SOURCES = test/librgw_link.cc
+#unittest_librgw_link_LDFLAGS = $(PTHREAD_CFLAGS) ${AM_LDFLAGS}
+#unittest_librgw_link_LDADD = $(LIBRGW) ${UNITTEST_LDADD}
+#unittest_librgw_link_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
+#check_PROGRAMS += unittest_librgw_link
+
+unittest_daemon_config_SOURCES = test/daemon_config.cc
+unittest_daemon_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_daemon_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_daemon_config
+
+unittest_osd_osdcap_SOURCES = test/osd/osdcap.cc 
+unittest_osd_osdcap_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_osd_osdcap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_osd_osdcap
+
+unittest_mon_moncap_SOURCES = test/mon/moncap.cc
+unittest_mon_moncap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_mon_moncap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_mon_moncap
+
+unittest_mon_pgmap_SOURCES = test/mon/PGMap.cc
+unittest_mon_pgmap_LDADD = $(LIBMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_mon_pgmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_mon_pgmap
+
+#if WITH_RADOSGW
+#unittest_librgw_SOURCES = test/librgw.cc
+#unittest_librgw_LDFLAGS = -lrt $(PTHREAD_CFLAGS) -lcurl ${AM_LDFLAGS}
+#unittest_librgw_LDADD =  librgw.la $(LIBRADOS) ${UNITTEST_LDADD} -lexpat $(CEPH_GLOBAL)
+#unittest_librgw_CXXFLAGS = ${CRYPTO_CFLAGS} ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
+#check_PROGRAMS += unittest_librgw
+#endif # WITH_RADOSGW
+
+unittest_ipaddr_SOURCES = test/test_ipaddr.cc
+unittest_ipaddr_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+unittest_ipaddr_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_ipaddr
+
+unittest_texttable_SOURCES = test/test_texttable.cc
+unittest_texttable_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD)
+unittest_texttable_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+check_PROGRAMS += unittest_texttable
+
+unittest_on_exit_SOURCES = test/on_exit.cc
+unittest_on_exit_LDADD = $(PTHREAD_LIBS)
+check_PROGRAMS += unittest_on_exit
+
+check_SCRIPTS += test/pybind/test_ceph_argparse.py
+
+if WITH_RADOSGW
+ceph_test_cors_SOURCES = test/test_cors.cc
+ceph_test_cors_LDADD = \
+	$(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \
+	$(UNITTEST_LDADD) \
+	-lcurl -luuid -lexpat
+ceph_test_cors_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cors
+
+ceph_test_rgw_manifest_SOURCES = test/rgw/test_rgw_manifest.cc
+ceph_test_rgw_manifest_LDADD = \
+	$(LIBRADOS) $(LIBRGW) $(LIBRGW_DEPS) $(CEPH_GLOBAL) \
+	$(UNITTEST_LDADD) $(CRYPTO_LIBS) \
+	-lcurl -luuid -lexpat
+
+ceph_test_rgw_manifest_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rgw_manifest
+
+ceph_test_cls_rgw_meta_SOURCES = test/test_rgw_admin_meta.cc
+ceph_test_cls_rgw_meta_LDADD = \
+	$(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \
+	$(UNITTEST_LDADD) $(CRYPTO_LIBS) \
+	-lcurl -luuid -lexpat \
+	libcls_version_client.a libcls_log_client.a \
+	libcls_statelog_client.a libcls_refcount_client.la \
+	libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
+ceph_test_cls_rgw_meta_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_rgw_meta
+
+ceph_test_cls_rgw_log_SOURCES = test/test_rgw_admin_log.cc
+ceph_test_cls_rgw_log_LDADD = \
+	$(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \
+	$(UNITTEST_LDADD) $(CRYPTO_LIBS) \
+	-lcurl -luuid -lexpat \
+	libcls_version_client.a libcls_log_client.a \
+	libcls_statelog_client.a libcls_refcount_client.la \
+	libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
+ceph_test_cls_rgw_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_rgw_log
+
+ceph_test_cls_rgw_opstate_SOURCES = test/test_rgw_admin_opstate.cc
+ceph_test_cls_rgw_opstate_LDADD = \
+	$(LIBRADOS) $(LIBRGW) $(CEPH_GLOBAL) \
+	$(UNITTEST_LDADD) $(CRYPTO_LIBS) \
+	-lcurl -luuid -lexpat \
+	libcls_version_client.a libcls_log_client.a \
+	libcls_statelog_client.a libcls_refcount_client.la \
+	libcls_rgw_client.la libcls_user_client.a libcls_lock_client.la
+ceph_test_cls_rgw_opstate_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_rgw_opstate
+endif # WITH_RADOSGW
+
+libradostest_la_SOURCES = \
+	test/librados/test.cc \
+	test/librados/TestCase.cc
+noinst_LTLIBRARIES += libradostest.la
+libradostest_la_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+RADOS_TEST_LDADD = libradostest.la
+
+ceph_multi_stress_watch_SOURCES = test/multi_stress_watch.cc
+ceph_multi_stress_watch_LDADD = $(LIBRADOS) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+bin_DEBUGPROGRAMS += ceph_multi_stress_watch
+
+ceph_test_librbd_SOURCES = test/librbd/test_librbd.cc
+ceph_test_librbd_LDADD = $(LIBRBD) $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_librbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_librbd
+
+if LINUX
+ceph_test_librbd_fsx_SOURCES = test/librbd/fsx.c
+ceph_test_librbd_fsx_LDADD = $(LIBRBD) $(LIBRADOS) -lm
+ceph_test_librbd_fsx_CFLAGS = ${AM_CFLAGS} -Wno-format
+bin_DEBUGPROGRAMS += ceph_test_librbd_fsx
+endif
+
+ceph_test_cls_rbd_SOURCES = test/cls_rbd/test_cls_rbd.cc
+ceph_test_cls_rbd_LDADD = $(LIBRADOS) libcls_rbd_client.la libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_cls_rbd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_rbd
+
+ceph_test_cls_refcount_SOURCES = test/cls_refcount/test_cls_refcount.cc
+ceph_test_cls_refcount_LDADD = $(LIBRADOS) libcls_refcount_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_cls_refcount_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_refcount
+
+ceph_test_cls_version_SOURCES = test/cls_version/test_cls_version.cc
+ceph_test_cls_version_LDADD = $(LIBRADOS) libcls_version_client.a $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_cls_version_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_version
+
+ceph_test_cls_log_SOURCES = test/cls_log/test_cls_log.cc
+ceph_test_cls_log_LDADD = $(LIBRADOS) libcls_log_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_cls_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_log
+
+ceph_test_cls_statelog_SOURCES = test/cls_statelog/test_cls_statelog.cc
+ceph_test_cls_statelog_LDADD = $(LIBRADOS) libcls_statelog_client.a $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_cls_statelog_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_statelog
+
+ceph_test_cls_replica_log_SOURCES = test/cls_replica_log/test_cls_replica_log.cc
+ceph_test_cls_replica_log_LDADD = \
+	$(LIBRADOS) libcls_replica_log_client.a \
+	$(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_cls_replica_log_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_replica_log
+
+ceph_test_cls_lock_SOURCES = test/cls_lock/test_cls_lock.cc
+ceph_test_cls_lock_LDADD = $(LIBRADOS) libcls_lock_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_cls_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_lock
+
+ceph_test_cls_hello_SOURCES = test/cls_hello/test_cls_hello.cc
+ceph_test_cls_hello_LDADD = \
+	$(LIBRADOS) $(CRYPTO_LIBS) \
+	$(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_cls_hello_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_hello
+
+if WITH_RADOSGW
+ceph_test_cls_rgw_SOURCES = test/cls_rgw/test_cls_rgw.cc
+ceph_test_cls_rgw_LDADD = $(LIBRADOS) libcls_rgw_client.la $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_cls_rgw_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_cls_rgw
+endif # WITH_RADOSGW
+
+ceph_test_mon_workloadgen_SOURCES = test/mon/test_mon_workloadgen.cc
+ceph_test_mon_workloadgen_LDADD = $(LIBOS) $(LIBOSDC) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_mon_workloadgen
+
+ceph_test_rados_api_cmd_SOURCES = test/librados/cmd.cc
+ceph_test_rados_api_cmd_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_cmd_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_cmd
+
+ceph_test_rados_api_io_SOURCES = test/librados/io.cc
+ceph_test_rados_api_io_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_io_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_io
+
+ceph_test_rados_api_c_write_operations_SOURCES = \
+	test/librados/c_write_operations.cc
+ceph_test_rados_api_c_write_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_c_write_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_c_write_operations
+
+ceph_test_rados_api_c_read_operations_SOURCES = \
+	test/librados/c_read_operations.cc
+ceph_test_rados_api_c_read_operations_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_c_read_operations_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_c_read_operations
+
+ceph_test_rados_api_aio_SOURCES = test/librados/aio.cc
+ceph_test_rados_api_aio_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_aio_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_aio
+
+ceph_test_rados_api_list_SOURCES = test/librados/list.cc
+ceph_test_rados_api_list_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_list_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_list
+
+ceph_test_rados_api_pool_SOURCES = test/librados/pool.cc
+ceph_test_rados_api_pool_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_pool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_pool
+
+ceph_test_rados_api_stat_SOURCES = test/librados/stat.cc
+ceph_test_rados_api_stat_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_stat_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_stat
+
+ceph_test_rados_api_watch_notify_SOURCES = test/librados/watch_notify.cc
+ceph_test_rados_api_watch_notify_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_watch_notify_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_watch_notify
+
+ceph_test_rados_api_snapshots_SOURCES = test/librados/snapshots.cc
+ceph_test_rados_api_snapshots_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_snapshots_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_snapshots
+
+ceph_test_rados_api_cls_SOURCES = test/librados/cls.cc
+ceph_test_rados_api_cls_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_cls_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_cls
+
+ceph_test_rados_api_misc_SOURCES = test/librados/misc.cc
+ceph_test_rados_api_misc_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_misc_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_misc
+
+ceph_test_rados_api_tier_SOURCES = \
+	test/librados/tier.cc \
+	osd/HitSet.cc
+ceph_test_rados_api_tier_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_tier_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_tier
+
+ceph_test_rados_api_lock_SOURCES = test/librados/lock.cc
+ceph_test_rados_api_lock_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_rados_api_lock_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_rados_api_lock
+
+ceph_test_libcephfs_SOURCES = \
+	test/libcephfs/test.cc \
+	test/libcephfs/readdir_r_cb.cc \
+	test/libcephfs/caps.cc \
+	test/libcephfs/multiclient.cc
+ceph_test_libcephfs_LDADD = $(LIBCEPHFS) $(UNITTEST_LDADD)
+ceph_test_libcephfs_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_libcephfs
+
+if LINUX
+ceph_test_objectstore_SOURCES = test/objectstore/store_test.cc
+ceph_test_objectstore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_objectstore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_objectstore
+endif
+
+ceph_test_objectstore_workloadgen_SOURCES = \
+	test/objectstore/workload_generator.cc \
+	test/objectstore/TestObjectStoreState.cc
+ceph_test_objectstore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_objectstore_workloadgen
+
+ceph_test_filestore_idempotent_SOURCES = \
+	test/objectstore/test_idempotent.cc \
+	test/objectstore/FileStoreTracker.cc \
+	test/common/ObjectContents.cc
+ceph_test_filestore_idempotent_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent
+
+ceph_test_filestore_idempotent_sequence_SOURCES = \
+	test/objectstore/test_idempotent_sequence.cc \
+	test/objectstore/DeterministicOpSequence.cc \
+	test/objectstore/TestObjectStoreState.cc \
+	test/objectstore/FileStoreDiff.cc
+ceph_test_filestore_idempotent_sequence_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent_sequence
+
+ceph_xattr_bench_SOURCES = test/xattr_bench.cc
+ceph_xattr_bench_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_xattr_bench_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_xattr_bench
+
+ceph_test_filejournal_SOURCES = test/test_filejournal.cc
+ceph_test_filejournal_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_filejournal_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_filejournal
+
+ceph_test_stress_watch_SOURCES = test/test_stress_watch.cc
+ceph_test_stress_watch_LDADD = $(LIBRADOS) $(UNITTEST_LDADD) $(RADOS_TEST_LDADD)
+ceph_test_stress_watch_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_stress_watch
+
+ceph_test_objectcacher_stress_SOURCES = \
+	test/osdc/object_cacher_stress.cc \
+	test/osdc/FakeWriteback.cc
+ceph_test_objectcacher_stress_LDADD = $(LIBOSDC) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_objectcacher_stress
+
+ceph_test_snap_mapper_SOURCES = test/test_snap_mapper.cc
+ceph_test_snap_mapper_LDADD = $(LIBOSD) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_snap_mapper_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_snap_mapper
+
+ceph_test_object_map_SOURCES = \
+	test/ObjectMap/test_object_map.cc \
+	test/ObjectMap/KeyValueDBMemory.cc
+ceph_test_object_map_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_object_map_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_object_map
+
+ceph_test_keyvaluedb_atomicity_SOURCES = test/ObjectMap/test_keyvaluedb_atomicity.cc
+ceph_test_keyvaluedb_atomicity_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_keyvaluedb_atomicity_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_atomicity
+
+ceph_test_keyvaluedb_iterators_SOURCES = \
+	test/ObjectMap/test_keyvaluedb_iterators.cc \
+	test/ObjectMap/KeyValueDBMemory.cc
+ceph_test_keyvaluedb_iterators_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_keyvaluedb_iterators_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_iterators
+
+ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc
+bin_DEBUGPROGRAMS += ceph_test_cfuse_cache_invalidate
+
+ceph_test_c_headers_SOURCES = test/test_c_headers.c
+ceph_test_c_headers_LDADD = $(LIBRADOS) $(LIBCEPHFS)
+bin_DEBUGPROGRAMS += ceph_test_c_headers
+
+ceph_test_get_blkdev_size_SOURCES = test/test_get_blkdev_size.cc
+ceph_test_get_blkdev_size_LDADD = $(LIBCOMMON)
+bin_DEBUGPROGRAMS += ceph_test_get_blkdev_size
+
+noinst_HEADERS += \
+	test/bench/backend.h \
+	test/bench/bencher.h \
+	test/bench/detailed_stat_collector.h \
+	test/bench/distribution.h \
+	test/bench/dumb_backend.h \
+	test/bench/rados_backend.h \
+	test/bench/rbd_backend.h \
+	test/bench/stat_collector.h \
+	test/bench/testfilestore_backend.h \
+	test/common/ObjectContents.h \
+	test/encoding/types.h \
+	test/objectstore/DeterministicOpSequence.h \
+	test/objectstore/FileStoreDiff.h \
+	test/objectstore/FileStoreTracker.h \
+	test/objectstore/TestObjectStoreState.h \
+	test/objectstore/workload_generator.h \
+	test/kv_store_bench.h \
+	test/librados/test.h \
+	test/librados/TestCase.h \
+	test/ObjectMap/KeyValueDBMemory.h \
+	test/omap_bench.h \
+	test/osdc/FakeWriteback.h \
+	test/osd/Object.h \
+	test/osd/RadosModel.h \
+	test/osd/TestOpStat.h \
+	test/system/cross_process_sem.h \
+	test/system/st_rados_create_pool.h \
+	test/system/st_rados_delete_objs.h \
+	test/system/st_rados_delete_pool.h \
+	test/system/st_rados_list_objects.h \
+	test/system/st_rados_notify.h \
+	test/system/st_rados_watch.h \
+	test/system/systest_runnable.h \
+	test/system/systest_settings.h \
+	test/unit.h
+
diff --git a/ceph/src/test/ObjectMap/KeyValueDBMemory.cc b/ceph/src/test/ObjectMap/KeyValueDBMemory.cc
new file mode 100644
index 00000000..c03e1696
--- /dev/null
+++ b/ceph/src/test/ObjectMap/KeyValueDBMemory.cc
@@ -0,0 +1,249 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "include/encoding.h"
+#include "KeyValueDBMemory.h"
+#include 
+#include 
+#include "include/memory.h"
+#include 
+
+using namespace std;
+
+/**
+ * Iterate over the whole key space of the in-memory store
+ *
+ * @note Removing keys from the store while iterating over the store key-space
+ *	 may result in unspecified behavior.
+ *	 If one wants to safely iterate over the store while updating the
+ *	 store, one should instead use a snapshot iterator, which provides
+ *	 strong read-consistency.
+ */
+class WholeSpaceMemIterator : public KeyValueDB::WholeSpaceIteratorImpl {
+protected:
+  KeyValueDBMemory *db;
+  bool ready;
+
+  map, bufferlist>::iterator it;
+
+public:
+  WholeSpaceMemIterator(KeyValueDBMemory *db) : db(db), ready(false) { }
+  virtual ~WholeSpaceMemIterator() { }
+
+  int seek_to_first() {
+    if (db->db.empty()) {
+      it = db->db.end();
+      ready = false;
+      return 0;
+    }
+    it = db->db.begin();
+    ready = true;
+    return 0;
+  }
+
+  int seek_to_first(const string &prefix) {
+    it = db->db.lower_bound(make_pair(prefix, ""));
+    if (db->db.empty() || (it == db->db.end())) {
+      it = db->db.end();
+      ready = false;
+      return 0;
+    }
+    ready = true;
+    return 0;
+  }
+
+  int seek_to_last() {
+    it = db->db.end();
+    if (db->db.empty()) {
+      ready = false;
+      return 0;
+    }
+    --it;
+    assert(it != db->db.end());
+    ready = true;
+    return 0;
+  }
+
+  int seek_to_last(const string &prefix) {
+    string tmp(prefix);
+    tmp.append(1, (char) 0);
+    it = db->db.upper_bound(make_pair(tmp,""));
+
+    if (db->db.empty() || (it == db->db.end())) {
+      seek_to_last();
+    }
+    else {
+      ready = true;
+      prev();
+    }
+    return 0;
+  }
+
+  int lower_bound(const string &prefix, const string &to) {
+    it = db->db.lower_bound(make_pair(prefix,to));
+    if ((db->db.size() == 0) || (it == db->db.end())) {
+      it = db->db.end();
+      ready = false;
+      return 0;
+    }
+
+    assert(it != db->db.end());
+
+    ready = true;
+    return 0;
+  }
+
+  int upper_bound(const string &prefix, const string &after) {
+    it = db->db.upper_bound(make_pair(prefix,after));
+    if ((db->db.size() == 0) || (it == db->db.end())) {
+      it = db->db.end();
+      ready = false;
+      return 0;
+    }
+    assert(it != db->db.end());
+    ready = true;
+    return 0;
+  }
+
+  bool valid() {
+    return ready && (it != db->db.end());
+  }
+
+  bool begin() {
+    return ready && (it == db->db.begin());
+  }
+
+  int prev() {
+    if (!begin() && ready)
+      --it;
+    else
+      it = db->db.end();
+    return 0;
+  }
+
+  int next() {
+    if (valid())
+      ++it;
+    return 0;
+  }
+
+  string key() {
+    if (valid())
+      return (*it).first.second;
+    else
+      return "";
+  }
+
+  pair raw_key() {
+    if (valid())
+      return (*it).first;
+    else
+      return make_pair("", "");
+  }
+
+  bufferlist value() {
+    if (valid())
+      return (*it).second;
+    else
+      return bufferlist();
+  }
+
+  int status() {
+    return 0;
+  }
+};
+
+int KeyValueDBMemory::get(const string &prefix,
+			  const std::set &key,
+			  map *out) {
+  if (!exists_prefix(prefix))
+    return 0;
+
+  for (std::set::const_iterator i = key.begin();
+       i != key.end();
+       ++i) {
+    pair k(prefix, *i);
+    if (db.count(k))
+      (*out)[*i] = db[k];
+  }
+  return 0;
+}
+
+int KeyValueDBMemory::get_keys(const string &prefix,
+			       const std::set &key,
+			       std::set *out) {
+  if (!exists_prefix(prefix))
+    return 0;
+
+  for (std::set::const_iterator i = key.begin();
+       i != key.end();
+       ++i) {
+    if (db.count(make_pair(prefix, *i)))
+      out->insert(*i);
+  }
+  return 0;
+}
+
+int KeyValueDBMemory::set(const string &prefix,
+			  const string &key,
+			  const bufferlist &bl) {
+  db[make_pair(prefix,key)] = bl;
+  return 0;
+}
+
+int KeyValueDBMemory::rmkey(const string &prefix,
+			    const string &key) {
+  db.erase(make_pair(prefix,key));
+  return 0;
+}
+
+int KeyValueDBMemory::rmkeys_by_prefix(const string &prefix) {
+  map,bufferlist>::iterator i;
+  i = db.lower_bound(make_pair(prefix, ""));
+  if (i == db.end())
+    return 0;
+
+  while (i != db.end()) {
+    std::pair key = (*i).first;
+    if (key.first != prefix)
+      break;
+
+    ++i;
+    rmkey(key.first, key.second);
+  }
+  return 0;
+}
+
+KeyValueDB::WholeSpaceIterator KeyValueDBMemory::_get_iterator() {
+  return ceph::shared_ptr(
+    new WholeSpaceMemIterator(this)
+  );
+}
+
+class WholeSpaceSnapshotMemIterator : public WholeSpaceMemIterator {
+public:
+
+  /**
+   * @note
+   * We perform a copy of the db map, which is populated by bufferlists.
+   *
+   * These are designed as shallow containers, thus there is a chance that
+   * changing the underlying memory pages will lead to the iterator seeing
+   * erroneous states.
+   *
+   * Although we haven't verified this yet, there is this chance, so we should
+   * keep it in mind.
+   */
+
+  WholeSpaceSnapshotMemIterator(KeyValueDBMemory *db) :
+    WholeSpaceMemIterator(db) { }
+  ~WholeSpaceSnapshotMemIterator() {
+    delete db;
+  }
+};
+
+KeyValueDB::WholeSpaceIterator KeyValueDBMemory::_get_snapshot_iterator() {
+  KeyValueDBMemory *snap_db = new KeyValueDBMemory(this);
+  return ceph::shared_ptr(
+    new WholeSpaceSnapshotMemIterator(snap_db)
+  );
+}
diff --git a/ceph/src/test/ObjectMap/KeyValueDBMemory.h b/ceph/src/test/ObjectMap/KeyValueDBMemory.h
new file mode 100644
index 00000000..8df39667
--- /dev/null
+++ b/ceph/src/test/ObjectMap/KeyValueDBMemory.h
@@ -0,0 +1,169 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include 
+#include 
+#include 
+#include "include/memory.h"
+
+#include "os/KeyValueDB.h"
+#include "include/buffer.h"
+#include "include/Context.h"
+
+using std::string;
+
+class KeyValueDBMemory : public KeyValueDB {
+public:
+  std::map,bufferlist> db;
+
+  KeyValueDBMemory() { }
+  KeyValueDBMemory(KeyValueDBMemory *db) : db(db->db) { }
+  virtual ~KeyValueDBMemory() { }
+
+  virtual int init() {
+    return 0;
+  }
+  virtual int open(ostream &out) {
+    return 0;
+  }
+  virtual int create_and_open(ostream &out) {
+    return 0;
+  }
+
+  int get(
+    const string &prefix,
+    const std::set &key,
+    std::map *out
+    );
+
+  int get_keys(
+    const string &prefix,
+    const std::set &key,
+    std::set *out
+    );
+
+  int set(
+    const string &prefix,
+    const string &key,
+    const bufferlist &bl
+    );
+
+  int rmkey(
+    const string &prefix,
+    const string &key
+    );
+
+  int rmkeys_by_prefix(
+    const string &prefix
+    );
+
+  class TransactionImpl_ : public TransactionImpl {
+  public:
+    list on_commit;
+    KeyValueDBMemory *db;
+
+    TransactionImpl_(KeyValueDBMemory *db) : db(db) {}
+
+
+    struct SetOp : public Context {
+      KeyValueDBMemory *db;
+      std::pair key;
+      bufferlist value;
+      SetOp(KeyValueDBMemory *db,
+	    const std::pair &key,
+	    const bufferlist &value)
+	: db(db), key(key), value(value) {}
+      void finish(int r) {
+	db->set(key.first, key.second, value);
+      }
+    };
+
+    void set(const string &prefix, const string &k, const bufferlist& bl) {
+      on_commit.push_back(new SetOp(db, std::make_pair(prefix, k), bl));
+    }
+
+    struct RmKeysOp : public Context {
+      KeyValueDBMemory *db;
+      std::pair key;
+      RmKeysOp(KeyValueDBMemory *db,
+	       const std::pair &key)
+	: db(db), key(key) {}
+      void finish(int r) {
+	db->rmkey(key.first, key.second);
+      }
+    };
+
+    void rmkey(const string &prefix, const string &key) {
+      on_commit.push_back(new RmKeysOp(db, std::make_pair(prefix, key)));
+    }
+
+    struct RmKeysByPrefixOp : public Context {
+      KeyValueDBMemory *db;
+      string prefix;
+      RmKeysByPrefixOp(KeyValueDBMemory *db,
+		       const string &prefix)
+	: db(db), prefix(prefix) {}
+      void finish(int r) {
+	db->rmkeys_by_prefix(prefix);
+      }
+    };
+    void rmkeys_by_prefix(const string &prefix) {
+      on_commit.push_back(new RmKeysByPrefixOp(db, prefix));
+    }
+
+    int complete() {
+      for (list::iterator i = on_commit.begin();
+	   i != on_commit.end();
+	   on_commit.erase(i++)) {
+	(*i)->complete(0);
+      }
+      return 0;
+    }
+
+    ~TransactionImpl_() {
+      for (list::iterator i = on_commit.begin();
+	   i != on_commit.end();
+	   on_commit.erase(i++)) {
+	delete *i;
+      }
+    }
+  };
+
+  Transaction get_transaction() {
+    return Transaction(new TransactionImpl_(this));
+  }
+
+  int submit_transaction(Transaction trans) {
+    return static_cast(trans.get())->complete();
+  }
+
+  uint64_t get_estimated_size(map &extras) {
+    uint64_t total_size = 0;
+
+    for (map,bufferlist>::iterator p = db.begin();
+         p != db.end(); ++p) {
+      string prefix = p->first.first;
+      bufferlist &bl = p->second;
+
+      uint64_t sz = bl.length();
+      total_size += sz;
+      if (extras.count(prefix) == 0)
+        extras[prefix] = 0;
+      extras[prefix] += sz;
+    }
+
+    return total_size;
+  }
+
+private:
+  bool exists_prefix(const string &prefix) {
+    std::map,bufferlist>::iterator it;
+    it = db.lower_bound(std::make_pair(prefix, ""));
+    return ((it != db.end()) && ((*it).first.first == prefix));
+  }
+
+  friend class WholeSpaceMemIterator;
+
+protected:
+  WholeSpaceIterator _get_iterator();
+  WholeSpaceIterator _get_snapshot_iterator();
+};
diff --git a/ceph/src/test/ObjectMap/test_keyvaluedb_atomicity.cc b/ceph/src/test/ObjectMap/test_keyvaluedb_atomicity.cc
new file mode 100644
index 00000000..04cef95d
--- /dev/null
+++ b/ceph/src/test/ObjectMap/test_keyvaluedb_atomicity.cc
@@ -0,0 +1,106 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include 
+#include "include/buffer.h"
+#include "os/LevelDBStore.h"
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+#include 
+#include 
+#include "stdlib.h"
+
+const string CONTROL_PREFIX = "CONTROL";
+const string PRIMARY_PREFIX = "PREFIX";
+const int NUM_COPIES = 100;
+const int NUM_THREADS = 30;
+
+string prefix_gen(int i) {
+  stringstream ss;
+  ss << PRIMARY_PREFIX << "_" << i << std::endl;
+  return ss.str();
+}
+
+int verify(KeyValueDB *db) {
+  // Verify
+  {
+    map iterators;
+    for (int i = 0; i < NUM_COPIES; ++i) {
+      iterators[i] = db->get_iterator(prefix_gen(i));
+      iterators[i]->seek_to_first();
+    }
+    while (iterators.rbegin()->second->valid()) {
+      for (map::iterator i = iterators.begin();
+	   i != iterators.end();
+	   ++i) {
+	assert(i->second->valid());
+	assert(i->second->key() == iterators.rbegin()->second->key());
+	bufferlist r = i->second->value();
+	bufferlist l = iterators.rbegin()->second->value();
+	i->second->next();
+      }
+    }
+    for (map::iterator i = iterators.begin();
+	 i != iterators.end();
+	 ++i) {
+      assert(!i->second->valid());
+    }
+  }
+  return 0;
+}
+
+void *write(void *_db) {
+  KeyValueDB *db = static_cast(_db);
+  std::cout << "Writing..." << std::endl;
+  for (int i = 0; i < 12000; ++i) {
+    if (!(i % 10)) {
+      std::cout << "Iteration: " << i << std::endl;
+    }
+    int key_num = rand();
+    stringstream key;
+    key << key_num << std::endl;
+    map to_set;
+    stringstream val;
+    val << i << std::endl;
+    bufferptr bp(val.str().c_str(), val.str().size() + 1);
+    to_set[key.str()].push_back(bp);
+    
+    KeyValueDB::Transaction t = db->get_transaction();
+    for (int j = 0; j < NUM_COPIES; ++j) {
+      t->set(prefix_gen(j), to_set);
+    }
+    assert(!db->submit_transaction(t));
+  }
+  return 0;
+}
+
+int main() {
+  char *path = getenv("OBJECT_MAP_PATH");
+  boost::scoped_ptr< KeyValueDB > db;
+  if (!path) {
+    std::cerr << "No path found, OBJECT_MAP_PATH undefined" << std::endl;
+    return 0;
+  }
+  string strpath(path);
+  std::cerr << "Using path: " << strpath << std::endl;
+  LevelDBStore *store = new LevelDBStore(NULL, strpath);
+  assert(!store->create_and_open(std::cerr));
+  db.reset(store);
+
+  verify(db.get());
+
+  vector threads(NUM_THREADS);
+  for (vector::iterator i = threads.begin();
+       i != threads.end();
+       ++i) {
+    pthread_create(&*i, 0, &write, static_cast(db.get()));
+  }
+  for (vector::iterator i = threads.begin();
+       i != threads.end();
+       ++i) {
+    void *tmp;
+    pthread_join(*i, &tmp);
+  }
+  verify(db.get());
+}
diff --git a/ceph/src/test/ObjectMap/test_keyvaluedb_iterators.cc b/ceph/src/test/ObjectMap/test_keyvaluedb_iterators.cc
new file mode 100644
index 00000000..cbe2ab1f
--- /dev/null
+++ b/ceph/src/test/ObjectMap/test_keyvaluedb_iterators.cc
@@ -0,0 +1,1787 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include "include/memory.h"
+#include 
+#include 
+#include 
+#include 
+
+#include "test/ObjectMap/KeyValueDBMemory.h"
+#include "os/KeyValueDB.h"
+#include "os/LevelDBStore.h"
+#include 
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+string store_path;
+
+class IteratorTest : public ::testing::Test
+{
+public:
+  boost::scoped_ptr db;
+  boost::scoped_ptr mock;
+
+  virtual void SetUp() {
+    assert(!store_path.empty());
+
+    LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path);
+    assert(!db_ptr->create_and_open(std::cerr));
+    db.reset(db_ptr);
+    mock.reset(new KeyValueDBMemory());
+  }
+
+  virtual void TearDown() { }
+
+  ::testing::AssertionResult validate_db_clear(KeyValueDB *store) {
+    KeyValueDB::WholeSpaceIterator it = store->get_iterator();
+    it->seek_to_first();
+    while (it->valid()) {
+      pair k = it->raw_key();
+      if (mock->db.count(k)) {
+	return ::testing::AssertionFailure()
+		<< __func__
+		<< " mock store count " << mock->db.count(k)
+		<< " key(" << k.first << "," << k.second << ")";
+      }
+      it->next();
+    }
+    return ::testing::AssertionSuccess();
+  }
+
+  ::testing::AssertionResult validate_db_match() {
+    KeyValueDB::WholeSpaceIterator it = db->get_iterator();
+    it->seek_to_first();
+    while (it->valid()) {
+      pair k = it->raw_key();
+      if (!mock->db.count(k)) {
+	return ::testing::AssertionFailure()
+		<< __func__
+		<< " mock db.count() " << mock->db.count(k)
+		<< " key(" << k.first << "," << k.second << ")";
+      }
+
+      bufferlist it_bl = it->value();
+      bufferlist mock_bl = mock->db[k];
+
+      string it_val = _bl_to_str(it_bl);
+      string mock_val = _bl_to_str(mock_bl);
+
+      if (it_val != mock_val) {
+	return ::testing::AssertionFailure()
+		<< __func__
+		<< " key(" << k.first << "," << k.second << ")"
+		<< " mismatch db value(" << it_val << ")"
+		<< " mock value(" << mock_val << ")";
+      }
+      it->next();
+    }
+    return ::testing::AssertionSuccess();
+  }
+
+  ::testing::AssertionResult validate_iterator(
+				KeyValueDB::WholeSpaceIterator it,
+				string expected_prefix,
+				string expected_key,
+				string expected_value) {
+    if (!it->valid()) {
+      return ::testing::AssertionFailure()
+	      << __func__
+	      << " iterator not valid";
+    }
+    pair key = it->raw_key();
+
+    if (expected_prefix != key.first) {
+      return ::testing::AssertionFailure()
+	      << __func__
+	      << " expected prefix '" << expected_prefix << "'"
+	      << " got prefix '" << key.first << "'";
+    }
+
+    if (expected_key != it->key()) {
+      return ::testing::AssertionFailure()
+	      << __func__
+	      << " expected key '" << expected_key << "'"
+	      << " got key '" << it->key() << "'";
+    }
+
+    if (it->key() != key.second) {
+      return ::testing::AssertionFailure()
+	      << __func__
+	      << " key '" << it->key() << "'"
+	      << " does not match"
+	      << " pair key '" << key.second << "'";
+    }
+
+    if (_bl_to_str(it->value()) != expected_value) {
+      return ::testing::AssertionFailure()
+	<< __func__
+	<< " key '(" << key.first << "," << key.second << ")''"
+	<< " expected value '" << expected_value << "'"
+	<< " got value '" << _bl_to_str(it->value()) << "'";
+    }
+
+    return ::testing::AssertionSuccess();
+  }
+
+  /**
+   * Checks if each key in the queue can be forward sequentially read from
+   * the iterator iter. All keys must be present and be prefixed with prefix,
+   * otherwise the validation will fail.
+   *
+   * Assumes that each key value must be based on the key name and generated
+   * by _gen_val().
+   */
+  void validate_prefix(KeyValueDB::WholeSpaceIterator iter,
+      string &prefix, deque &keys) {
+
+    while (!keys.empty()) {
+      ASSERT_TRUE(iter->valid());
+      string expected_key = keys.front();
+      keys.pop_front();
+      string expected_value = _gen_val_str(expected_key);
+
+      ASSERT_TRUE(validate_iterator(iter, prefix,
+		  expected_key, expected_value));
+
+      iter->next();
+    }
+  }
+  /**
+   * Checks if each key in the queue can be backward sequentially read from
+   * the iterator iter. All keys must be present and be prefixed with prefix,
+   * otherwise the validation will fail.
+   *
+   * Assumes that each key value must be based on the key name and generated
+   * by _gen_val().
+   */
+  void validate_prefix_backwards(KeyValueDB::WholeSpaceIterator iter,
+      string &prefix, deque &keys) {
+
+    while (!keys.empty()) {
+      ASSERT_TRUE(iter->valid());
+      string expected_key = keys.front();
+      keys.pop_front();
+      string expected_value = _gen_val_str(expected_key);
+
+      ASSERT_TRUE(validate_iterator(iter, prefix,
+		  expected_key, expected_value));
+
+      iter->prev();
+    }
+  }
+
+  void clear(KeyValueDB *store) {
+    KeyValueDB::WholeSpaceIterator it = store->get_snapshot_iterator();
+    it->seek_to_first();
+    KeyValueDB::Transaction t = store->get_transaction();
+    while (it->valid()) {
+      pair k = it->raw_key();
+      t->rmkey(k.first, k.second);
+      it->next();
+    }
+    store->submit_transaction_sync(t);
+  }
+
+  string _bl_to_str(bufferlist val) {
+    string str(val.c_str(), val.length());
+    return str;
+  }
+
+  string _gen_val_str(string key) {
+    ostringstream ss;
+    ss << "##value##" << key << "##";
+    return ss.str();
+ }
+
+  bufferlist _gen_val(string key) {
+    bufferlist bl;
+    bl.append(_gen_val_str(key));
+    return bl;
+  }
+
+  void print_iterator(KeyValueDB::WholeSpaceIterator iter) {
+    if (!iter->valid()) {
+      std::cerr << __func__ << " iterator is not valid; stop." << std::endl;
+      return;
+    }
+
+    int i = 0;
+    while (iter->valid()) {
+      pair k = iter->raw_key();
+      std::cerr << __func__
+		<< " pos " << (++i)
+		<< " key (" << k.first << "," << k.second << ")"
+		<< " value(" << _bl_to_str(iter->value()) << ")" << std::endl;
+      iter->next();
+    }
+  }
+
+  void print_db(KeyValueDB *store) {
+    KeyValueDB::WholeSpaceIterator it = store->get_iterator();
+    it->seek_to_first();
+    print_iterator(it);
+  }
+};
+
+// ------- Remove Keys / Remove Keys By Prefix -------
+class RmKeysTest : public IteratorTest
+{
+public:
+  string prefix1;
+  string prefix2;
+  string prefix3;
+
+  void init(KeyValueDB *db) {
+    KeyValueDB::Transaction tx = db->get_transaction();
+
+    tx->set(prefix1, "11", _gen_val("11"));
+    tx->set(prefix1, "12", _gen_val("12"));
+    tx->set(prefix1, "13", _gen_val("13"));
+    tx->set(prefix2, "21", _gen_val("21"));
+    tx->set(prefix2, "22", _gen_val("22"));
+    tx->set(prefix2, "23", _gen_val("23"));
+    tx->set(prefix3, "31", _gen_val("31"));
+    tx->set(prefix3, "32", _gen_val("32"));
+    tx->set(prefix3, "33", _gen_val("33"));
+
+    db->submit_transaction_sync(tx);
+  }
+
+  virtual void SetUp() {
+    IteratorTest::SetUp();
+
+    prefix1 = "_PREFIX_1_";
+    prefix2 = "_PREFIX_2_";
+    prefix3 = "_PREFIX_3_";
+
+    clear(db.get());
+    ASSERT_TRUE(validate_db_clear(db.get()));
+    clear(mock.get());
+    ASSERT_TRUE(validate_db_match());
+
+    init(db.get());
+    init(mock.get());
+
+    ASSERT_TRUE(validate_db_match());
+  }
+
+  virtual void TearDown() {
+    IteratorTest::TearDown();
+  }
+
+
+  /**
+   * Test the transaction's rmkeys behavior when we remove a given prefix
+   * from the beginning of the key space, or from the end of the key space,
+   * or even simply in the middle.
+   */
+  void RmKeysByPrefix(KeyValueDB *store) {
+    // remove prefix2 ; check if prefix1 remains, and then prefix3
+    KeyValueDB::Transaction tx = store->get_transaction();
+    // remove the prefix in the middle of the key space
+    tx->rmkeys_by_prefix(prefix2);
+    store->submit_transaction_sync(tx);
+
+    deque key_deque;
+    KeyValueDB::WholeSpaceIterator iter = store->get_iterator();
+    iter->seek_to_first();
+
+    // check for prefix1
+    key_deque.push_back("11");
+    key_deque.push_back("12");
+    key_deque.push_back("13");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    // check for prefix3
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("31");
+    key_deque.push_back("32");
+    key_deque.push_back("33");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    ASSERT_FALSE(iter->valid());
+
+    clear(store);
+    ASSERT_TRUE(validate_db_clear(store));
+    init(store);
+
+    // remove prefix1 ; check if prefix2 and then prefix3 remain
+    tx = store->get_transaction();
+    // remove the prefix at the beginning of the key space
+    tx->rmkeys_by_prefix(prefix1);
+    store->submit_transaction_sync(tx);
+
+    iter = store->get_iterator();
+    iter->seek_to_first();
+
+    // check for prefix2
+    key_deque.clear();
+    key_deque.push_back("21");
+    key_deque.push_back("22");
+    key_deque.push_back("23");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    // check for prefix3
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("31");
+    key_deque.push_back("32");
+    key_deque.push_back("33");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    ASSERT_FALSE(iter->valid());
+
+    clear(store);
+    ASSERT_TRUE(validate_db_clear(store));
+    init(store);
+
+    // remove prefix3 ; check if prefix1 and then prefix2 remain
+    tx = store->get_transaction();
+    // remove the prefix at the end of the key space
+    tx->rmkeys_by_prefix(prefix3);
+    store->submit_transaction_sync(tx);
+
+    iter = store->get_iterator();
+    iter->seek_to_first();
+
+    // check for prefix1
+    key_deque.clear();
+    key_deque.push_back("11");
+    key_deque.push_back("12");
+    key_deque.push_back("13");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    // check for prefix2
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("21");
+    key_deque.push_back("22");
+    key_deque.push_back("23");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    ASSERT_FALSE(iter->valid());
+  }
+
+  /**
+   * Test how the leveldb's whole-space iterator behaves when we remove
+   * keys from the store while iterating over them.
+   */
+  void RmKeysWhileIteratingSnapshot(KeyValueDB *store,
+				    KeyValueDB::WholeSpaceIterator iter) {
+
+    SCOPED_TRACE("RmKeysWhileIteratingSnapshot");
+
+    iter->seek_to_first();
+    ASSERT_TRUE(iter->valid());
+
+    KeyValueDB::Transaction t = store->get_transaction();
+    t->rmkey(prefix1, "11");
+    t->rmkey(prefix1, "12");
+    t->rmkey(prefix2, "23");
+    t->rmkey(prefix3, "33");
+    store->submit_transaction_sync(t);
+
+    deque key_deque;
+
+    // check for prefix1
+    key_deque.push_back("11");
+    key_deque.push_back("12");
+    key_deque.push_back("13");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    // check for prefix2
+    key_deque.clear();
+    key_deque.push_back("21");
+    key_deque.push_back("22");
+    key_deque.push_back("23");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    // check for prefix3
+    key_deque.clear();
+    key_deque.push_back("31");
+    key_deque.push_back("32");
+    key_deque.push_back("33");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    iter->next();
+    ASSERT_FALSE(iter->valid());
+
+    // make sure those keys were removed from the store
+    KeyValueDB::WholeSpaceIterator tmp_it = store->get_iterator();
+    tmp_it->seek_to_first();
+    ASSERT_TRUE(tmp_it->valid());
+
+    key_deque.clear();
+    key_deque.push_back("13");
+    validate_prefix(tmp_it, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    ASSERT_TRUE(tmp_it->valid());
+    key_deque.clear();
+    key_deque.push_back("21");
+    key_deque.push_back("22");
+    validate_prefix(tmp_it, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    ASSERT_TRUE(tmp_it->valid());
+    key_deque.clear();
+    key_deque.push_back("31");
+    key_deque.push_back("32");
+    validate_prefix(tmp_it, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+
+    ASSERT_FALSE(tmp_it->valid());
+  }
+};
+
+TEST_F(RmKeysTest, RmKeysByPrefixLevelDB)
+{
+  SCOPED_TRACE("LevelDB");
+  RmKeysByPrefix(db.get());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(RmKeysTest, RmKeysByPrefixMockDB)
+{
+  SCOPED_TRACE("Mock DB");
+  RmKeysByPrefix(mock.get());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+/**
+ * If you refer to function RmKeysTest::RmKeysWhileIteratingSnapshot(),
+ * you will notice that we seek the iterator to the first key, and then
+ * we go on to remove several keys from the underlying store, including
+ * the first couple keys.
+ *
+ * We would expect that during this test, as soon as we removed the keys
+ * from the store, the iterator would get invalid, or cause some sort of
+ * unexpected mess.
+ *
+ * Instead, the current version of leveldb handles it perfectly, by making
+ * the iterator to use a snapshot instead of the store's real state. This
+ * way, LevelDBStore's whole-space iterator will behave much like its own
+ * whole-space snapshot iterator.
+ *
+ * However, this particular behavior of the iterator hasn't been documented
+ * on leveldb, and we should assume that it can be changed at any point in
+ * time.
+ *
+ * Therefore, we keep this test, being exactly the same as the one for the
+ * whole-space snapshot iterator, as we currently assume they should behave
+ * identically. If this test fails, at some point, and the whole-space
+ * snapshot iterator passes, then it probably means that leveldb changed
+ * how its iterator behaves.
+ */
+TEST_F(RmKeysTest, RmKeysWhileIteratingLevelDB)
+{
+  SCOPED_TRACE("LevelDB -- WholeSpaceIterator");
+  RmKeysWhileIteratingSnapshot(db.get(), db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(RmKeysTest, RmKeysWhileIteratingMockDB)
+{
+  std::cout << "There is no safe way to test key removal while iterating\n"
+	    << "over the mock store without using snapshots" << std::endl;
+}
+
+TEST_F(RmKeysTest, RmKeysWhileIteratingSnapshotLevelDB)
+{
+  SCOPED_TRACE("LevelDB -- WholeSpaceSnapshotIterator");
+  RmKeysWhileIteratingSnapshot(db.get(), db->get_snapshot_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(RmKeysTest, RmKeysWhileIteratingSnapshotMockDB)
+{
+  SCOPED_TRACE("Mock DB -- WholeSpaceSnapshotIterator");
+  RmKeysWhileIteratingSnapshot(mock.get(), mock->get_snapshot_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+// ------- Set Keys / Update Values -------
+class SetKeysTest : public IteratorTest
+{
+public:
+  string prefix1;
+  string prefix2;
+
+  void init(KeyValueDB *db) {
+    KeyValueDB::Transaction tx = db->get_transaction();
+
+    tx->set(prefix1, "aaa", _gen_val("aaa"));
+    tx->set(prefix1, "ccc", _gen_val("ccc"));
+    tx->set(prefix1, "eee", _gen_val("eee"));
+    tx->set(prefix2, "vvv", _gen_val("vvv"));
+    tx->set(prefix2, "xxx", _gen_val("xxx"));
+    tx->set(prefix2, "zzz", _gen_val("zzz"));
+
+    db->submit_transaction_sync(tx);
+  }
+
+  virtual void SetUp() {
+    IteratorTest::SetUp();
+
+    prefix1 = "_PREFIX_1_";
+    prefix2 = "_PREFIX_2_";
+
+    clear(db.get());
+    ASSERT_TRUE(validate_db_clear(db.get()));
+    clear(mock.get());
+    ASSERT_TRUE(validate_db_match());
+
+    init(db.get());
+    init(mock.get());
+
+    ASSERT_TRUE(validate_db_match());
+  }
+
+  virtual void TearDown() {
+    IteratorTest::TearDown();
+  }
+
+  /**
+   * Make sure that the iterator picks on new keys added if it hasn't yet
+   * iterated away from that position.
+   * 
+   * This should only happen for the whole-space iterator when not using
+   * the snapshot version.
+   *
+   * We don't need to test the validity of all elements, but we do test
+   * inserting while moving from the first element to the last, using next()
+   * to move forward, and then we test the same behavior while iterating
+   * from the last element to the first, using prev() to move backwards.
+   */
+  void SetKeysWhileIterating(KeyValueDB *store,
+			     KeyValueDB::WholeSpaceIterator iter) {
+    iter->seek_to_first();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1, "aaa",
+	  	_gen_val_str("aaa")));
+    iter->next();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1, "ccc",
+		_bl_to_str(_gen_val("ccc"))));
+
+    // insert new key 'ddd' after 'ccc' and before 'eee'
+    KeyValueDB::Transaction tx = store->get_transaction();
+    tx->set(prefix1, "ddd", _gen_val("ddd"));
+    store->submit_transaction_sync(tx);
+
+    iter->next();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1, "ddd",
+		_gen_val_str("ddd")));
+
+    iter->seek_to_last();
+    ASSERT_TRUE(iter->valid());
+    tx = store->get_transaction();
+    tx->set(prefix2, "yyy", _gen_val("yyy"));
+    store->submit_transaction_sync(tx);
+
+    iter->prev();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix2,
+				  "yyy", _gen_val_str("yyy")));
+  }
+
+  /**
+   * Make sure that the whole-space snapshot iterator does not pick on new keys
+   * added to the store since we created the iterator, thus guaranteeing
+   * read-consistency.
+   *
+   * We don't need to test the validity of all elements, but we do test
+   * inserting while moving from the first element to the last, using next()
+   * to move forward, and then we test the same behavior while iterating
+   * from the last element to the first, using prev() to move backwards.
+   */
+  void SetKeysWhileIteratingSnapshot(KeyValueDB *store,
+				     KeyValueDB::WholeSpaceIterator iter) {
+    iter->seek_to_first();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1, "aaa",
+	  	_gen_val_str("aaa")));
+    iter->next();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1, "ccc",
+		_bl_to_str(_gen_val("ccc"))));
+
+    // insert new key 'ddd' after 'ccc' and before 'eee'
+    KeyValueDB::Transaction tx = store->get_transaction();
+    tx->set(prefix1, "ddd", _gen_val("ddd"));
+    store->submit_transaction_sync(tx);
+
+    iter->next();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1, "eee",
+		_gen_val_str("eee")));
+
+    iter->seek_to_last();
+    ASSERT_TRUE(iter->valid());
+    tx = store->get_transaction();
+    tx->set(prefix2, "yyy", _gen_val("yyy"));
+    store->submit_transaction_sync(tx);
+
+    iter->prev();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix2,
+				  "xxx", _gen_val_str("xxx")));
+  }
+
+  /**
+   * Make sure that the whole-space iterator is able to read values changed on
+   * the store, even after we moved to the updated position.
+   *
+   * This should only be possible when not using the whole-space snapshot
+   * version of the iterator.
+   */
+  void UpdateValuesWhileIterating(KeyValueDB *store,
+				  KeyValueDB::WholeSpaceIterator iter) {
+    iter->seek_to_first();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1,
+				  "aaa", _gen_val_str("aaa")));
+
+    KeyValueDB::Transaction tx = store->get_transaction();
+    tx->set(prefix1, "aaa", _gen_val("aaa_1"));
+    store->submit_transaction_sync(tx);
+
+    ASSERT_TRUE(validate_iterator(iter, prefix1,
+				  "aaa", _gen_val_str("aaa_1")));
+
+    iter->seek_to_last();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix2,
+				  "zzz", _gen_val_str("zzz")));
+
+    tx = store->get_transaction();
+    tx->set(prefix2, "zzz", _gen_val("zzz_1"));
+    store->submit_transaction_sync(tx);
+
+    ASSERT_TRUE(validate_iterator(iter, prefix2,
+				  "zzz", _gen_val_str("zzz_1")));
+  }
+
+  /**
+   * Make sure that the whole-space iterator is able to read values changed on
+   * the store, even after we moved to the updated position.
+   *
+   * This should only be possible when not using the whole-space snapshot
+   * version of the iterator.
+   */
+  void UpdateValuesWhileIteratingSnapshot(
+				  KeyValueDB *store,
+				  KeyValueDB::WholeSpaceIterator iter) {
+    iter->seek_to_first();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix1,
+				  "aaa", _gen_val_str("aaa")));
+
+    KeyValueDB::Transaction tx = store->get_transaction();
+    tx->set(prefix1, "aaa", _gen_val("aaa_1"));
+    store->submit_transaction_sync(tx);
+
+    ASSERT_TRUE(validate_iterator(iter, prefix1,
+				  "aaa", _gen_val_str("aaa")));
+
+    iter->seek_to_last();
+    ASSERT_TRUE(iter->valid());
+    ASSERT_TRUE(validate_iterator(iter, prefix2,
+				  "zzz", _gen_val_str("zzz")));
+
+    tx = store->get_transaction();
+    tx->set(prefix2, "zzz", _gen_val("zzz_1"));
+    store->submit_transaction_sync(tx);
+
+    ASSERT_TRUE(validate_iterator(iter, prefix2,
+				  "zzz", _gen_val_str("zzz")));
+
+    // check those values were really changed in the store
+    KeyValueDB::WholeSpaceIterator tmp_iter = store->get_iterator();
+    tmp_iter->seek_to_first();
+    ASSERT_TRUE(tmp_iter->valid());
+    ASSERT_TRUE(validate_iterator(tmp_iter, prefix1,
+				  "aaa", _gen_val_str("aaa_1")));
+    tmp_iter->seek_to_last();
+    ASSERT_TRUE(tmp_iter->valid());
+    ASSERT_TRUE(validate_iterator(tmp_iter, prefix2,
+				  "zzz", _gen_val_str("zzz_1")));
+  }
+
+
+};
+
+TEST_F(SetKeysTest, DISABLED_SetKeysWhileIteratingLevelDB)
+{
+  SCOPED_TRACE("LevelDB: SetKeysWhileIteratingLevelDB");
+  SetKeysWhileIterating(db.get(), db->get_iterator());
+  ASSERT_TRUE(HasFatalFailure());
+}
+
+TEST_F(SetKeysTest, SetKeysWhileIteratingMockDB)
+{
+  SCOPED_TRACE("Mock DB: SetKeysWhileIteratingMockDB");
+  SetKeysWhileIterating(mock.get(), mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SetKeysTest, SetKeysWhileIteratingSnapshotLevelDB)
+{
+  SCOPED_TRACE("LevelDB: SetKeysWhileIteratingSnapshotLevelDB");
+  SetKeysWhileIteratingSnapshot(db.get(), db->get_snapshot_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SetKeysTest, SetKeysWhileIteratingSnapshotMockDB)
+{
+  SCOPED_TRACE("MockDB: SetKeysWhileIteratingSnapshotMockDB");
+  SetKeysWhileIteratingSnapshot(mock.get(), mock->get_snapshot_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SetKeysTest, DISABLED_UpdateValuesWhileIteratingLevelDB)
+{
+  SCOPED_TRACE("LevelDB: UpdateValuesWhileIteratingLevelDB");
+  UpdateValuesWhileIterating(db.get(), db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SetKeysTest, UpdateValuesWhileIteratingMockDB)
+{
+  SCOPED_TRACE("MockDB: UpdateValuesWhileIteratingMockDB");
+  UpdateValuesWhileIterating(mock.get(), mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SetKeysTest, UpdateValuesWhileIteratingSnapshotLevelDB)
+{
+  SCOPED_TRACE("LevelDB: UpdateValuesWhileIteratingSnapshotLevelDB");
+  UpdateValuesWhileIteratingSnapshot(db.get(), db->get_snapshot_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SetKeysTest, UpdateValuesWhileIteratingSnapshotMockDB)
+{
+  SCOPED_TRACE("MockDB: UpdateValuesWhileIteratingSnapshotMockDB");
+  UpdateValuesWhileIteratingSnapshot(mock.get(), mock->get_snapshot_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+
+class BoundsTest : public IteratorTest
+{
+public:
+  string prefix1;
+  string prefix2;
+  string prefix3;
+
+  void init(KeyValueDB *store) {
+    KeyValueDB::Transaction tx = store->get_transaction();
+
+    tx->set(prefix1, "aaa", _gen_val("aaa"));
+    tx->set(prefix1, "ccc", _gen_val("ccc"));
+    tx->set(prefix1, "eee", _gen_val("eee"));
+    tx->set(prefix2, "vvv", _gen_val("vvv"));
+    tx->set(prefix2, "xxx", _gen_val("xxx"));
+    tx->set(prefix2, "zzz", _gen_val("zzz"));
+    tx->set(prefix3, "aaa", _gen_val("aaa"));
+    tx->set(prefix3, "mmm", _gen_val("mmm"));
+    tx->set(prefix3, "yyy", _gen_val("yyy"));
+
+    store->submit_transaction_sync(tx);
+  }
+
+  virtual void SetUp() {
+    IteratorTest::SetUp();
+
+    prefix1 = "_PREFIX_1_";
+    prefix2 = "_PREFIX_2_";
+    prefix3 = "_PREFIX_4_";
+
+    clear(db.get());
+    ASSERT_TRUE(validate_db_clear(db.get()));
+    clear(mock.get());
+    ASSERT_TRUE(validate_db_match());
+
+    init(db.get());
+    init(mock.get());
+
+    ASSERT_TRUE(validate_db_match());
+  }
+
+  virtual void TearDown() {
+    IteratorTest::TearDown();
+  }
+
+  void LowerBoundWithEmptyKeyOnWholeSpaceIterator(
+			    KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    // see what happens when we have an empty key and try to get to the
+    // first available prefix
+    iter->lower_bound(prefix1, "");
+    ASSERT_TRUE(iter->valid());
+
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+    // if we got here without problems, then it is safe to assume the
+    // remaining prefixes are intact.
+
+    // see what happens when we have an empty key and try to get to the
+    // middle of the key-space
+    iter->lower_bound(prefix2, "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+
+    key_deque.push_back("vvv");
+    key_deque.push_back("xxx");
+    key_deque.push_back("zzz");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+    // if we got here without problems, then it is safe to assume the
+    // remaining prefixes are intact.
+
+    // see what happens when we have an empty key and try to get to the
+    // last prefix on the key-space
+    iter->lower_bound(prefix3, "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+
+    key_deque.push_back("aaa");
+    key_deque.push_back("mmm");
+    key_deque.push_back("yyy");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+    // we reached the end of the key_space, so the iterator should no longer
+    // be valid
+    
+    // see what happens when we look for an inexistent prefix, that will
+    // compare higher than the existing prefixes, with an empty key
+    // expected: reach the store's end; iterator becomes invalid
+    iter->lower_bound("_PREFIX_9_", "");
+    ASSERT_FALSE(iter->valid());
+
+    // see what happens when we look for an inexistent prefix, that will
+    // compare lower than the existing prefixes, with an empty key
+    // expected: find the first prefix; iterator is valid
+    iter->lower_bound("_PREFIX_0_", "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // see what happens when we look for an empty prefix (that should compare
+    // lower than any existing prefixes)
+    // expected: find the first prefix; iterator is valid
+    iter->lower_bound("", "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+  }
+
+  void LowerBoundWithEmptyPrefixOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    // check for an empty prefix, with key 'aaa'. Since this key is shared
+    // among two different prefixes, it is relevant to check which will be
+    // found first.
+    // expected: find key (prefix1, aaa); iterator is valid
+    iter->lower_bound("", "aaa");
+    ASSERT_TRUE(iter->valid());
+
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+    // since we found prefix1, it is safe to assume that the remaining
+    // prefixes (prefix2 and prefix3) will follow
+
+    // any lower_bound operation with an empty prefix should always put the
+    // iterator in the first key in the key-space, despite what key is
+    // specified. This means that looking for ("","AAAAAAAAAA") should
+    // also position the iterator on (prefix1, aaa).
+    // expected: find key (prefix1, aaa); iterator is valid
+    iter->lower_bound("", "AAAAAAAAAA");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // note: this test is a duplicate of the one in the function above. Why?
+    // Well, because it also fits here (being its prefix empty), and one could
+    // very well run solely this test (instead of the whole battery) and would
+    // certainly expect this case to be tested.
+
+    // see what happens when we look for an empty prefix (that should compare
+    // lower than any existing prefixes)
+    // expected: find the first prefix; iterator is valid
+    iter->lower_bound("", "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+  }
+
+  void LowerBoundOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    // check that we find the first key in the store
+    // expected: find (prefix1, aaa); iterator is valid
+    iter->lower_bound(prefix1, "aaa");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // check that we find the last key in the store
+    // expected: find (prefix3, yyy); iterator is valid
+    iter->lower_bound(prefix3, "yyy");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("yyy");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+
+    // check that looking for non-existent prefix '_PREFIX_0_' will
+    // always result in the first value of prefix1 (prefix1,"aaa")
+    // expected: find (prefix1, aaa); iterator is valid
+    iter->lower_bound("_PREFIX_0_", "AAAAA");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // check that looking for non-existent prefix '_PREFIX_3_' will
+    // always result in the first value of prefix3 (prefix4,"aaa")
+    // expected: find (prefix3, aaa); iterator is valid
+    iter->lower_bound("_PREFIX_3_", "AAAAA");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // check that looking for non-existent prefix '_PREFIX_9_' will
+    // always result in an invalid iterator.
+    // expected: iterator is invalid
+    iter->lower_bound("_PREFIX_9_", "AAAAA");
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void UpperBoundWithEmptyKeyOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    // check that looking for (prefix1, "") will result in finding
+    // the first key in prefix1 (prefix1, "aaa")
+    // expected: find (prefix1, aaa); iterator is valid
+    iter->upper_bound(prefix1, "");
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // check that looking for (prefix2, "") will result in finding
+    // the first key in prefix2 (prefix2, vvv)
+    // expected: find (prefix2, aaa); iterator is valid
+    iter->upper_bound(prefix2, "");
+    key_deque.push_back("vvv");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+
+    // check that looking for (prefix3, "") will result in finding
+    // the first key in prefix3 (prefix3, aaa)
+    // expected: find (prefix3, aaa); iterator is valid
+    iter->upper_bound(prefix3, "");
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // see what happens when we look for an inexistent prefix, that will
+    // compare higher than the existing prefixes, with an empty key
+    // expected: reach the store's end; iterator becomes invalid
+    iter->upper_bound("_PREFIX_9_", "");
+    ASSERT_FALSE(iter->valid());
+
+    // see what happens when we look for an inexistent prefix, that will
+    // compare lower than the existing prefixes, with an empty key
+    // expected: find the first prefix; iterator is valid
+    iter->upper_bound("_PREFIX_0_", "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // see what happens when we look for an empty prefix (that should compare
+    // lower than any existing prefixes)
+    // expected: find the first prefix; iterator is valid
+    iter->upper_bound("", "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+  }
+
+  void UpperBoundWithEmptyPrefixOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    // check for an empty prefix, with key 'aaa'. Since this key is shared
+    // among two different prefixes, it is relevant to check which will be
+    // found first.
+    // expected: find key (prefix1, aaa); iterator is valid
+    iter->upper_bound("", "aaa");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // any upper_bound operation with an empty prefix should always put the
+    // iterator in the first key whose prefix compares greater, despite the
+    // key that is specified. This means that looking for ("","AAAAAAAAAA")
+    // should position the iterator on (prefix1, aaa).
+    // expected: find key (prefix1, aaa); iterator is valid
+    iter->upper_bound("", "AAAAAAAAAA");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // note: this test is a duplicate of the one in the function above. Why?
+    // Well, because it also fits here (being its prefix empty), and one could
+    // very well run solely this test (instead of the whole battery) and would
+    // certainly expect this case to be tested.
+
+    // see what happens when we look for an empty prefix (that should compare
+    // lower than any existing prefixes)
+    // expected: find the first prefix; iterator is valid
+    iter->upper_bound("", "");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+  }
+
+  void UpperBoundOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    // check that we find the second key in the store
+    // expected: find (prefix1, ccc); iterator is valid
+    iter->upper_bound(prefix1, "bbb");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("ccc");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // check that we find the last key in the store
+    // expected: find (prefix3, yyy); iterator is valid
+    iter->upper_bound(prefix3, "xxx");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("yyy");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+
+    // check that looking for non-existent prefix '_PREFIX_0_' will
+    // always result in the first value of prefix1 (prefix1,"aaa")
+    // expected: find (prefix1, aaa); iterator is valid
+    iter->upper_bound("_PREFIX_0_", "AAAAA");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // check that looking for non-existent prefix '_PREFIX_3_' will
+    // always result in the first value of prefix3 (prefix3,"aaa")
+    // expected: find (prefix3, aaa); iterator is valid
+    iter->upper_bound("_PREFIX_3_", "AAAAA");
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix3, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // check that looking for non-existent prefix '_PREFIX_9_' will
+    // always result in an invalid iterator.
+    // expected: iterator is invalid
+    iter->upper_bound("_PREFIX_9_", "AAAAA");
+    ASSERT_FALSE(iter->valid());
+  }
+};
+
+TEST_F(BoundsTest, LowerBoundWithEmptyKeyOnWholeSpaceIteratorLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Lower Bound, Empty Key, Whole-Space Iterator");
+  LowerBoundWithEmptyKeyOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, LowerBoundWithEmptyKeyOnWholeSpaceIteratorMockDB)
+{
+  SCOPED_TRACE("MockDB: Lower Bound, Empty Key, Whole-Space Iterator");
+  LowerBoundWithEmptyKeyOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, LowerBoundWithEmptyPrefixOnWholeSpaceIteratorLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Lower Bound, Empty Prefix, Whole-Space Iterator");
+  LowerBoundWithEmptyPrefixOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, LowerBoundWithEmptyPrefixOnWholeSpaceIteratorMockDB)
+{
+  SCOPED_TRACE("MockDB: Lower Bound, Empty Prefix, Whole-Space Iterator");
+  LowerBoundWithEmptyPrefixOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, LowerBoundOnWholeSpaceIteratorLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Lower Bound, Whole-Space Iterator");
+  LowerBoundOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, LowerBoundOnWholeSpaceIteratorMockDB)
+{
+  SCOPED_TRACE("MockDB: Lower Bound, Whole-Space Iterator");
+  LowerBoundOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, UpperBoundWithEmptyKeyOnWholeSpaceIteratorLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Upper Bound, Empty Key, Whole-Space Iterator");
+  UpperBoundWithEmptyKeyOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, UpperBoundWithEmptyKeyOnWholeSpaceIteratorMockDB)
+{
+  SCOPED_TRACE("MockDB: Upper Bound, Empty Key, Whole-Space Iterator");
+  UpperBoundWithEmptyKeyOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, UpperBoundWithEmptyPrefixOnWholeSpaceIteratorLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Upper Bound, Empty Prefix, Whole-Space Iterator");
+  UpperBoundWithEmptyPrefixOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, UpperBoundWithEmptyPrefixOnWholeSpaceIteratorMockDB)
+{
+  SCOPED_TRACE("MockDB: Upper Bound, Empty Prefix, Whole-Space Iterator");
+  UpperBoundWithEmptyPrefixOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, UpperBoundOnWholeSpaceIteratorLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Upper Bound, Whole-Space Iterator");
+  UpperBoundOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(BoundsTest, UpperBoundOnWholeSpaceIteratorMockDB)
+{
+  SCOPED_TRACE("MockDB: Upper Bound, Whole-Space Iterator");
+  UpperBoundOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+
+class SeeksTest : public IteratorTest
+{
+public:
+  string prefix0;
+  string prefix1;
+  string prefix2;
+  string prefix3;
+  string prefix4;
+  string prefix5;
+
+  void init(KeyValueDB *store) {
+    KeyValueDB::Transaction tx = store->get_transaction();
+
+    tx->set(prefix1, "aaa", _gen_val("aaa"));
+    tx->set(prefix1, "ccc", _gen_val("ccc"));
+    tx->set(prefix1, "eee", _gen_val("eee"));
+    tx->set(prefix2, "vvv", _gen_val("vvv"));
+    tx->set(prefix2, "xxx", _gen_val("xxx"));
+    tx->set(prefix2, "zzz", _gen_val("zzz"));
+    tx->set(prefix4, "aaa", _gen_val("aaa"));
+    tx->set(prefix4, "mmm", _gen_val("mmm"));
+    tx->set(prefix4, "yyy", _gen_val("yyy"));
+
+    store->submit_transaction_sync(tx);
+  }
+
+  virtual void SetUp() {
+    IteratorTest::SetUp();
+
+    prefix0 = "_PREFIX_0_";
+    prefix1 = "_PREFIX_1_";
+    prefix2 = "_PREFIX_2_";
+    prefix3 = "_PREFIX_3_";
+    prefix4 = "_PREFIX_4_";
+    prefix5 = "_PREFIX_5_";
+
+    clear(db.get());
+    ASSERT_TRUE(validate_db_clear(db.get()));
+    clear(mock.get());
+    ASSERT_TRUE(validate_db_match());
+
+    init(db.get());
+    init(mock.get());
+
+    ASSERT_TRUE(validate_db_match());
+  }
+
+  virtual void TearDown() {
+    IteratorTest::TearDown();
+  }
+
+
+  void SeekToFirstOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    iter->seek_to_first();
+    ASSERT_TRUE(iter->valid());
+    deque key_deque;
+    key_deque.push_back("aaa");
+    key_deque.push_back("ccc");
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+  }
+
+  void SeekToFirstWithPrefixOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+
+    // if the prefix is empty, we must end up seeking to the first key.
+    // expected: seek to (prefix1, aaa); iterator is valid
+    iter->seek_to_first("");
+    ASSERT_TRUE(iter->valid());
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to non-existent prefix that compares lower than the
+    // first available prefix
+    // expected: seek to (prefix1, aaa); iterator is valid
+    iter->seek_to_first(prefix0);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to non-existent prefix
+    // expected: seek to (prefix4, aaa); iterator is valid
+    iter->seek_to_first(prefix3);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix4, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to non-existent prefix that compares greater than the
+    // last available prefix
+    // expected: iterator is invalid
+    iter->seek_to_first(prefix5);
+    ASSERT_FALSE(iter->valid());
+
+    // try seeking to the first prefix and make sure we end up in its first
+    // position
+    // expected: seek to (prefix1,aaa); iterator is valid
+    iter->seek_to_first(prefix1);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to the second prefix and make sure we end up in its
+    // first position
+    // expected: seek to (prefix2,vvv); iterator is valid
+    iter->seek_to_first(prefix2);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("vvv");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to the last prefix and make sure we end up in its
+    // first position
+    // expected: seek to (prefix4,aaa); iterator is valid
+    iter->seek_to_first(prefix4);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("aaa");
+    validate_prefix(iter, prefix4, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+  }
+
+  void SeekToLastOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    iter->seek_to_last();
+    key_deque.push_back("yyy");
+    validate_prefix(iter, prefix4, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void SeekToLastWithPrefixOnWholeSpaceIterator(
+			  KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+
+    // if the prefix is empty, we must end up seeking to last position
+    // that has an empty prefix, or to the previous position to the first
+    // position whose prefix compares higher than empty.
+    // expected: iterator is invalid (because (prefix1,aaa) is the first
+    //		 position that compared higher than an empty prefix)
+    iter->seek_to_last("");
+    ASSERT_FALSE(iter->valid());
+
+    // try seeking to non-existent prefix that compares lower than the
+    // first available prefix
+    // expected: iterator is invalid (because (prefix1,aaa) is the first
+    //		 position that compared higher than prefix0)
+    iter->seek_to_last(prefix0);
+    ASSERT_FALSE(iter->valid());
+
+    // try seeking to non-existent prefix
+    // expected: seek to (prefix2, zzz); iterator is valid
+    iter->seek_to_last(prefix3);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("zzz");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to non-existent prefix that compares greater than the
+    // last available prefix
+    // expected: iterator is in the last position of the store;
+    //		 i.e., (prefix4,yyy)
+    iter->seek_to_last(prefix5);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("yyy");
+    validate_prefix(iter, prefix4, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+
+    // try seeking to the first prefix and make sure we end up in its last
+    // position
+    // expected: seek to (prefix1,eee); iterator is valid
+    iter->seek_to_last(prefix1);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("eee");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to the second prefix and make sure we end up in its
+    // last position
+    // expected: seek to (prefix2,vvv); iterator is valid
+    iter->seek_to_last(prefix2);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("zzz");
+    validate_prefix(iter, prefix2, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_TRUE(iter->valid());
+
+    // try seeking to the last prefix and make sure we end up in its
+    // last position
+    // expected: seek to (prefix4,aaa); iterator is valid
+    iter->seek_to_last(prefix4);
+    ASSERT_TRUE(iter->valid());
+    key_deque.clear();
+    key_deque.push_back("yyy");
+    validate_prefix(iter, prefix4, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+  }
+};
+
+TEST_F(SeeksTest, SeekToFirstOnWholeSpaceIteratorLevelDB) {
+  SCOPED_TRACE("LevelDB: Seek To First, Whole Space Iterator");
+  SeekToFirstOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SeeksTest, SeekToFirstOnWholeSpaceIteratorMockDB) {
+  SCOPED_TRACE("MockDB: Seek To First, Whole Space Iterator");
+  SeekToFirstOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SeeksTest, SeekToFirstWithPrefixOnWholeSpaceIteratorLevelDB) {
+  SCOPED_TRACE("LevelDB: Seek To First, With Prefix, Whole Space Iterator");
+  SeekToFirstWithPrefixOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SeeksTest, SeekToFirstWithPrefixOnWholeSpaceIteratorMockDB) {
+  SCOPED_TRACE("MockDB: Seek To First, With Prefix, Whole Space Iterator");
+  SeekToFirstWithPrefixOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SeeksTest, SeekToLastOnWholeSpaceIteratorLevelDB) {
+  SCOPED_TRACE("LevelDB: Seek To Last, Whole Space Iterator");
+  SeekToLastOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SeeksTest, SeekToLastOnWholeSpaceIteratorMockDB) {
+  SCOPED_TRACE("MockDB: Seek To Last, Whole Space Iterator");
+  SeekToLastOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SeeksTest, SeekToLastWithPrefixOnWholeSpaceIteratorLevelDB) {
+  SCOPED_TRACE("LevelDB: Seek To Last, With Prefix, Whole Space Iterator");
+  SeekToLastWithPrefixOnWholeSpaceIterator(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(SeeksTest, SeekToLastWithPrefixOnWholeSpaceIteratorMockDB) {
+  SCOPED_TRACE("MockDB: Seek To Last, With Prefix, Whole Space Iterator");
+  SeekToLastWithPrefixOnWholeSpaceIterator(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+class KeySpaceIteration : public IteratorTest
+{
+public:
+  string prefix1;
+
+  void init(KeyValueDB *store) {
+    KeyValueDB::Transaction tx = store->get_transaction();
+
+    tx->set(prefix1, "aaa", _gen_val("aaa"));
+    tx->set(prefix1, "vvv", _gen_val("vvv"));
+    tx->set(prefix1, "zzz", _gen_val("zzz"));
+
+    store->submit_transaction_sync(tx);
+  }
+
+  virtual void SetUp() {
+    IteratorTest::SetUp();
+
+    prefix1 = "_PREFIX_1_";
+
+    clear(db.get());
+    ASSERT_TRUE(validate_db_clear(db.get()));
+    clear(mock.get());
+    ASSERT_TRUE(validate_db_match());
+
+    init(db.get());
+    init(mock.get());
+
+    ASSERT_TRUE(validate_db_match());
+  }
+
+  virtual void TearDown() {
+    IteratorTest::TearDown();
+  }
+
+  void ForwardIteration(KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    iter->seek_to_first();
+    key_deque.push_back("aaa");
+    key_deque.push_back("vvv");
+    key_deque.push_back("zzz");
+    validate_prefix(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void BackwardIteration(KeyValueDB::WholeSpaceIterator iter) {
+    deque key_deque;
+    iter->seek_to_last();
+    key_deque.push_back("zzz");
+    key_deque.push_back("vvv");
+    key_deque.push_back("aaa");
+    validate_prefix_backwards(iter, prefix1, key_deque);
+    ASSERT_FALSE(HasFatalFailure());
+    ASSERT_FALSE(iter->valid());
+  }
+};
+
+TEST_F(KeySpaceIteration, ForwardIterationLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Forward Iteration, Whole Space Iterator");
+  ForwardIteration(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(KeySpaceIteration, ForwardIterationMockDB) {
+  SCOPED_TRACE("MockDB: Forward Iteration, Whole Space Iterator");
+  ForwardIteration(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(KeySpaceIteration, BackwardIterationLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Backward Iteration, Whole Space Iterator");
+  BackwardIteration(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(KeySpaceIteration, BackwardIterationMockDB) {
+  SCOPED_TRACE("MockDB: Backward Iteration, Whole Space Iterator");
+  BackwardIteration(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+class EmptyStore : public IteratorTest
+{
+public:
+  virtual void SetUp() {
+    IteratorTest::SetUp();
+
+    clear(db.get());
+    ASSERT_TRUE(validate_db_clear(db.get()));
+    clear(mock.get());
+    ASSERT_TRUE(validate_db_match());
+  }
+
+  void SeekToFirst(KeyValueDB::WholeSpaceIterator iter) {
+    // expected: iterator is invalid
+    iter->seek_to_first();
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void SeekToFirstWithPrefix(KeyValueDB::WholeSpaceIterator iter) {
+    // expected: iterator is invalid
+    iter->seek_to_first("prefix");
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void SeekToLast(KeyValueDB::WholeSpaceIterator iter) {
+    // expected: iterator is invalid
+    iter->seek_to_last();
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void SeekToLastWithPrefix(KeyValueDB::WholeSpaceIterator iter) {
+    // expected: iterator is invalid
+    iter->seek_to_last("prefix");
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void LowerBound(KeyValueDB::WholeSpaceIterator iter) {
+    // expected: iterator is invalid
+    iter->lower_bound("prefix", "");
+    ASSERT_FALSE(iter->valid());
+
+    // expected: iterator is invalid
+    iter->lower_bound("", "key");
+    ASSERT_FALSE(iter->valid());
+
+    // expected: iterator is invalid
+    iter->lower_bound("prefix", "key");
+    ASSERT_FALSE(iter->valid());
+  }
+
+  void UpperBound(KeyValueDB::WholeSpaceIterator iter) {
+    // expected: iterator is invalid
+    iter->upper_bound("prefix", "");
+    ASSERT_FALSE(iter->valid());
+
+    // expected: iterator is invalid
+    iter->upper_bound("", "key");
+    ASSERT_FALSE(iter->valid());
+
+    // expected: iterator is invalid
+    iter->upper_bound("prefix", "key");
+    ASSERT_FALSE(iter->valid());
+  }
+};
+
+TEST_F(EmptyStore, SeekToFirstLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Empty Store, Seek To First");
+  SeekToFirst(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, SeekToFirstMockDB)
+{
+  SCOPED_TRACE("MockDB: Empty Store, Seek To First");
+  SeekToFirst(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, SeekToFirstWithPrefixLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Empty Store, Seek To First With Prefix");
+  SeekToFirstWithPrefix(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, SeekToFirstWithPrefixMockDB)
+{
+  SCOPED_TRACE("MockDB: Empty Store, Seek To First With Prefix");
+  SeekToFirstWithPrefix(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, SeekToLastLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Empty Store, Seek To Last");
+  SeekToLast(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, SeekToLastMockDB)
+{
+  SCOPED_TRACE("MockDB: Empty Store, Seek To Last");
+  SeekToLast(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, SeekToLastWithPrefixLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Empty Store, Seek To Last With Prefix");
+  SeekToLastWithPrefix(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, SeekToLastWithPrefixMockDB)
+{
+  SCOPED_TRACE("MockDB: Empty Store, Seek To Last With Prefix");
+  SeekToLastWithPrefix(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, LowerBoundLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Empty Store, Lower Bound");
+  LowerBound(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, LowerBoundMockDB)
+{
+  SCOPED_TRACE("MockDB: Empty Store, Lower Bound");
+  LowerBound(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, UpperBoundLevelDB)
+{
+  SCOPED_TRACE("LevelDB: Empty Store, Upper Bound");
+  UpperBound(db->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+TEST_F(EmptyStore, UpperBoundMockDB)
+{
+  SCOPED_TRACE("MockDB: Empty Store, Upper Bound");
+  UpperBound(mock->get_iterator());
+  ASSERT_FALSE(HasFatalFailure());
+}
+
+
+int main(int argc, char *argv[])
+{
+  vector args;
+  argv_to_vec(argc, (const char **) argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+
+  if (argc < 2) {
+    std::cerr << "Usage: " << argv[0]
+	      << "[ceph_options] [gtest_options] " << std::endl;
+    return 1;
+  }
+  store_path = string(argv[1]);
+
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/ObjectMap/test_object_map.cc b/ceph/src/test/ObjectMap/test_object_map.cc
new file mode 100644
index 00000000..eecd9ea4
--- /dev/null
+++ b/ceph/src/test/ObjectMap/test_object_map.cc
@@ -0,0 +1,786 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "include/memory.h"
+#include 
+#include 
+#include 
+
+#include "include/buffer.h"
+#include "test/ObjectMap/KeyValueDBMemory.h"
+#include "os/KeyValueDB.h"
+#include "os/DBObjectMap.h"
+#include "os/HashIndex.h"
+#include "os/LevelDBStore.h"
+#include 
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include 
+
+#include "gtest/gtest.h"
+#include "stdlib.h"
+
+using namespace std;
+
+template 
+typename T::iterator rand_choose(T &cont) {
+  if (cont.size() == 0) {
+    return cont.end();
+  }
+  int index = rand() % cont.size();
+  typename T::iterator retval = cont.begin();
+
+  for (; index > 0; --index) ++retval;
+  return retval;
+}
+
+string num_str(unsigned i) {
+  char buf[100];
+  snprintf(buf, sizeof(buf), "%.10u", i);
+  return string(buf);
+}
+
+class ObjectMapTester {
+public:
+  ObjectMap *db;
+  set key_space;
+  set object_name_space;
+  map > omap;
+  map hmap;
+  map > xattrs;
+  unsigned seq;
+
+  ObjectMapTester() : db(0), seq(0) {}
+
+  string val_from_key(const string &object, const string &key) {
+    return object + "_" + key + "_" + num_str(seq++);
+  }
+
+  void set_key(const string &objname, const string &key, const string &value) {
+    set_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+	    key, value);
+  }
+
+  void set_xattr(const string &objname, const string &key, const string &value) {
+    set_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+	      key, value);
+  }
+
+  void set_key(ghobject_t hoid,
+	       string key, string value) {
+    map to_write;
+    bufferptr bp(value.c_str(), value.size());
+    bufferlist bl;
+    bl.append(bp);
+    to_write.insert(make_pair(key, bl));
+    db->set_keys(hoid, to_write);
+  }
+
+  void set_xattr(ghobject_t hoid,
+		 string key, string value) {
+    map to_write;
+    bufferptr bp(value.c_str(), value.size());
+    bufferlist bl;
+    bl.append(bp);
+    to_write.insert(make_pair(key, bl));
+    db->set_xattrs(hoid, to_write);
+  }
+
+  void set_header(const string &objname, const string &value) {
+    set_header(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+	       value);
+  }
+
+  void set_header(ghobject_t hoid,
+		  const string &value) {
+    bufferlist header;
+    header.append(bufferptr(value.c_str(), value.size() + 1));
+    db->set_header(hoid, header);
+  }
+
+  int get_header(const string &objname, string *value) {
+    return get_header(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+		      value);
+  }
+
+  int get_header(ghobject_t hoid,
+		 string *value) {
+    bufferlist header;
+    int r = db->get_header(hoid, &header);
+    if (r < 0)
+      return r;
+    if (header.length())
+      *value = string(header.c_str());
+    else
+      *value = string("");
+    return 0;
+  }
+
+  int get_xattr(const string &objname, const string &key, string *value) {
+    return get_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+		     key, value);
+  }
+
+  int get_xattr(ghobject_t hoid,
+		string key, string *value) {
+    set to_get;
+    to_get.insert(key);
+    map got;
+    db->get_xattrs(hoid, to_get, &got);
+    if (!got.empty()) {
+      *value = string(got.begin()->second.c_str(),
+		      got.begin()->second.length());
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  int get_key(const string &objname, const string &key, string *value) {
+    return get_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+		   key, value);
+  }
+
+  int get_key(ghobject_t hoid,
+	      string key, string *value) {
+    set to_get;
+    to_get.insert(key);
+    map got;
+    db->get_values(hoid, to_get, &got);
+    if (!got.empty()) {
+      *value = string(got.begin()->second.c_str(),
+		      got.begin()->second.length());
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  void remove_key(const string &objname, const string &key) {
+    remove_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+	       key);
+  }
+
+  void remove_key(ghobject_t hoid,
+		  string key) {
+    set to_remove;
+    to_remove.insert(key);
+    db->rm_keys(hoid, to_remove);
+  }
+
+  void remove_xattr(const string &objname, const string &key) {
+    remove_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+		 key);
+  }
+
+  void remove_xattr(ghobject_t hoid,
+		    string key) {
+    set to_remove;
+    to_remove.insert(key);
+    db->remove_xattrs(hoid, to_remove);
+  }
+
+  void clone(const string &objname, const string &target) {
+    clone(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
+	  ghobject_t(hobject_t(sobject_t(target, CEPH_NOSNAP))));
+  }
+
+  void clone(ghobject_t hoid,
+	     ghobject_t hoid2) {
+    db->clone(hoid, hoid2);
+  }
+
+  void clear(const string &objname) {
+    clear(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))));
+  }
+
+  void clear(ghobject_t hoid) {
+    db->clear(hoid);
+  }
+
+  void clear_omap(const string &objname) {
+    clear_omap(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))));
+  }
+
+  void clear_omap(const ghobject_t &objname) {
+    db->clear_keys_header(objname);
+  }
+
+  void def_init() {
+    for (unsigned i = 0; i < 1000; ++i) {
+      key_space.insert("key_" + num_str(i));
+    }
+    for (unsigned i = 0; i < 1000; ++i) {
+      object_name_space.insert("name_" + num_str(i));
+    }
+  }
+
+  void init_key_set(const set &keys) {
+    key_space = keys;
+  }
+
+  void init_object_name_space(const set &onamespace) {
+    object_name_space = onamespace;
+  }
+
+  void auto_set_xattr(ostream &out) {
+    set::iterator key = rand_choose(key_space);
+    set::iterator object = rand_choose(object_name_space);
+
+    string value = val_from_key(*object, *key);
+
+    xattrs[*object][*key] = value;
+    set_xattr(*object, *key, value);
+
+    out << "auto_set_xattr " << *object << ": " << *key << " -> "
+	<< value << std::endl;
+  }
+
+  void auto_set_key(ostream &out) {
+    set::iterator key = rand_choose(key_space);
+    set::iterator object = rand_choose(object_name_space);
+
+    string value = val_from_key(*object, *key);
+
+    omap[*object][*key] = value;
+    set_key(*object, *key, value);
+
+    out << "auto_set_key " << *object << ": " << *key << " -> "
+	<< value << std::endl;
+  }
+
+  void xattrs_on_object(const string &object, set *out) {
+    if (!xattrs.count(object))
+      return;
+    const map &xmap = xattrs.find(object)->second;
+    for (map::const_iterator i = xmap.begin();
+	 i != xmap.end();
+	 ++i) {
+      out->insert(i->first);
+    }
+  }
+
+  void keys_on_object(const string &object, set *out) {
+    if (!omap.count(object))
+      return;
+    const map &kmap = omap.find(object)->second;
+    for (map::const_iterator i = kmap.begin();
+	 i != kmap.end();
+	 ++i) {
+      out->insert(i->first);
+    }
+  }
+
+  void xattrs_off_object(const string &object, set *out) {
+    *out = key_space;
+    set xspace;
+    xattrs_on_object(object, &xspace);
+    for (set::iterator i = xspace.begin();
+	 i != xspace.end();
+	 ++i) {
+      out->erase(*i);
+    }
+  }
+
+  void keys_off_object(const string &object, set *out) {
+    *out = key_space;
+    set kspace;
+    keys_on_object(object, &kspace);
+    for (set::iterator i = kspace.begin();
+	 i != kspace.end();
+	 ++i) {
+      out->erase(*i);
+    }
+  }
+
+  int auto_check_present_xattr(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    set xspace;
+    xattrs_on_object(*object, &xspace);
+    set::iterator key = rand_choose(xspace);
+    if (key == xspace.end()) {
+      return 1;
+    }
+
+    string result;
+    int r = get_xattr(*object, *key, &result);
+    if (!r) {
+      out << "auto_check_present_key: failed to find key "
+	  << *key << " on object " << *object << std::endl;
+      return 0;
+    }
+
+    if (result != xattrs[*object][*key]) {
+      out << "auto_check_present_key: for key "
+	  << *key << " on object " << *object
+	  << " found value " << result << " where we should have found "
+	  << xattrs[*object][*key] << std::endl;
+      return 0;
+    }
+
+    out << "auto_check_present_key: for key "
+	<< *key << " on object " << *object
+	<< " found value " << result << " where we should have found "
+	<< xattrs[*object][*key] << std::endl;
+    return 1;
+  }
+
+
+  int auto_check_present_key(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    set kspace;
+    keys_on_object(*object, &kspace);
+    set::iterator key = rand_choose(kspace);
+    if (key == kspace.end()) {
+      return 1;
+    }
+
+    string result;
+    int r = get_key(*object, *key, &result);
+    if (!r) {
+      out << "auto_check_present_key: failed to find key "
+	  << *key << " on object " << *object << std::endl;
+      return 0;
+    }
+
+    if (result != omap[*object][*key]) {
+      out << "auto_check_present_key: for key "
+	  << *key << " on object " << *object
+	  << " found value " << result << " where we should have found "
+	  << omap[*object][*key] << std::endl;
+      return 0;
+    }
+
+    out << "auto_check_present_key: for key "
+	<< *key << " on object " << *object
+	<< " found value " << result << " where we should have found "
+	<< omap[*object][*key] << std::endl;
+    return 1;
+  }
+
+  int auto_check_absent_xattr(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    set xspace;
+    xattrs_off_object(*object, &xspace);
+    set::iterator key = rand_choose(xspace);
+    if (key == xspace.end()) {
+      return 1;
+    }
+
+    string result;
+    int r = get_xattr(*object, *key, &result);
+    if (!r) {
+      out << "auto_check_absent_key: did not find key "
+	  << *key << " on object " << *object << std::endl;
+      return 1;
+    }
+
+    out << "auto_check_basent_key: for key "
+	<< *key << " on object " << *object
+	<< " found value " << result << " where we should have found nothing"
+	<< std::endl;
+    return 0;
+  }
+
+  int auto_check_absent_key(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    set kspace;
+    keys_off_object(*object, &kspace);
+    set::iterator key = rand_choose(kspace);
+    if (key == kspace.end()) {
+      return 1;
+    }
+
+    string result;
+    int r = get_key(*object, *key, &result);
+    if (!r) {
+      out << "auto_check_absent_key: did not find key "
+	  << *key << " on object " << *object << std::endl;
+      return 1;
+    }
+
+    out << "auto_check_basent_key: for key "
+	<< *key << " on object " << *object
+	<< " found value " << result << " where we should have found nothing"
+	<< std::endl;
+    return 0;
+  }
+
+  void auto_clone_key(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    set::iterator target = rand_choose(object_name_space);
+    while (target == object) {
+      target = rand_choose(object_name_space);
+    }
+    out << "clone " << *object << " to " << *target;
+    clone(*object, *target);
+    if (!omap.count(*object)) {
+      out << " source missing.";
+      omap.erase(*target);
+    } else {
+      out << " source present.";
+      omap[*target] = omap[*object];
+    }
+    if (!hmap.count(*object)) {
+      out << " hmap source missing." << std::endl;
+      hmap.erase(*target);
+    } else {
+      out << " hmap source present." << std::endl;
+      hmap[*target] = hmap[*object];
+    }
+    if (!xattrs.count(*object)) {
+      out << " hmap source missing." << std::endl;
+      xattrs.erase(*target);
+    } else {
+      out << " hmap source present." << std::endl;
+      xattrs[*target] = xattrs[*object];
+    }
+  }
+
+  void auto_remove_key(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    set kspace;
+    keys_on_object(*object, &kspace);
+    set::iterator key = rand_choose(kspace);
+    if (key == kspace.end()) {
+      return;
+    }
+    out << "removing " << *key << " from " << *object << std::endl;
+    omap[*object].erase(*key);
+    remove_key(*object, *key);
+  }
+
+  void auto_remove_xattr(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    set kspace;
+    xattrs_on_object(*object, &kspace);
+    set::iterator key = rand_choose(kspace);
+    if (key == kspace.end()) {
+      return;
+    }
+    out << "removing xattr " << *key << " from " << *object << std::endl;
+    xattrs[*object].erase(*key);
+    remove_xattr(*object, *key);
+  }
+
+  void auto_delete_object(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    out << "auto_delete_object " << *object << std::endl;
+    clear(*object);
+    omap.erase(*object);
+    hmap.erase(*object);
+    xattrs.erase(*object);
+  }
+
+  void auto_clear_omap(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    out << "auto_clear_object " << *object << std::endl;
+    clear_omap(*object);
+    omap.erase(*object);
+    hmap.erase(*object);
+  }
+
+  void auto_write_header(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    string header = val_from_key(*object, "HEADER");
+    out << "auto_write_header: " << *object << " -> " << header << std::endl;
+    set_header(*object, header);
+    hmap[*object] = header;
+  }
+
+  int auto_verify_header(ostream &out) {
+    set::iterator object = rand_choose(object_name_space);
+    out << "verify_header: " << *object << " ";
+    string header;
+    int r = get_header(*object, &header);
+    if (r < 0) {
+      assert(0);
+    }
+    if (header.size() == 0) {
+      if (hmap.count(*object)) {
+	out << " failed to find header " << hmap[*object] << std::endl;
+	return 0;
+      } else {
+	out << " found no header" << std::endl;
+	return 1;
+      }
+    }
+
+    if (!hmap.count(*object)) {
+      out << " found header " << header << " should have been empty"
+	      << std::endl;
+      return 0;
+    } else if (header == hmap[*object]) {
+      out << " found correct header " << header << std::endl;
+      return 1;
+    } else {
+      out << " found incorrect header " << header
+	  << " where we should have found " << hmap[*object] << std::endl;
+      return 0;
+    }
+  }
+};
+
+class ObjectMapTest : public ::testing::Test {
+public:
+  boost::scoped_ptr< ObjectMap > db;
+  ObjectMapTester tester;
+  virtual void SetUp() {
+    char *path = getenv("OBJECT_MAP_PATH");
+    if (!path) {
+      db.reset(new DBObjectMap(new KeyValueDBMemory()));
+      tester.db = db.get();
+      return;
+    }
+
+    string strpath(path);
+
+    cerr << "using path " << strpath << std::endl;;
+    LevelDBStore *store = new LevelDBStore(g_ceph_context, strpath);
+    assert(!store->create_and_open(cerr));
+
+    db.reset(new DBObjectMap(store));
+    tester.db = db.get();
+  }
+
+  virtual void TearDown() {
+    std::cerr << "Checking..." << std::endl;
+    assert(db->check(std::cerr));
+  }
+};
+
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+TEST_F(ObjectMapTest, CreateOneObject) {
+  ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)), 100, 0);
+  map to_set;
+  string key("test");
+  string val("test_val");
+  bufferptr bp(val.c_str(), val.size());
+  bufferlist bl;
+  bl.append(bp);
+  to_set.insert(make_pair(key, bl));
+  ASSERT_FALSE(db->set_keys(hoid, to_set));
+
+  map got;
+  set to_get;
+  to_get.insert(key);
+  to_get.insert("not there");
+  db->get_values(hoid, to_get, &got);
+  ASSERT_EQ(got.size(), (unsigned)1);
+  ASSERT_EQ(string(got[key].c_str(), got[key].length()), val);
+
+  bufferlist header;
+  got.clear();
+  db->get(hoid, &header, &got);
+  ASSERT_EQ(got.size(), (unsigned)1);
+  ASSERT_EQ(string(got[key].c_str(), got[key].length()), val);
+  ASSERT_EQ(header.length(), (unsigned)0);
+
+  db->rm_keys(hoid, to_get);
+  got.clear();
+  db->get(hoid, &header, &got);
+  ASSERT_EQ(got.size(), (unsigned)0);
+
+  map attrs;
+  attrs["attr1"] = bl;
+  db->set_xattrs(hoid, attrs);
+
+  db->set_header(hoid, bl);
+
+  db->clear_keys_header(hoid);
+  set attrs_got;
+  db->get_all_xattrs(hoid, &attrs_got);
+  ASSERT_EQ(attrs_got.size(), 1U);
+  ASSERT_EQ(*(attrs_got.begin()), "attr1");
+  db->get(hoid, &header, &got);
+  ASSERT_EQ(got.size(), (unsigned)0);
+  ASSERT_EQ(header.length(), 0U);
+  got.clear();
+
+  db->clear(hoid);
+  db->get(hoid, &header, &got);
+  ASSERT_EQ(got.size(), (unsigned)0);
+  attrs_got.clear();
+  db->get_all_xattrs(hoid, &attrs_got);
+  ASSERT_EQ(attrs_got.size(), 0U);
+}
+
+TEST_F(ObjectMapTest, CloneOneObject) {
+  ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)), 200, 0);
+  ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP)), 201, 1);
+
+  tester.set_key(hoid, "foo", "bar");
+  tester.set_key(hoid, "foo2", "bar2");
+  string result;
+  int r = tester.get_key(hoid, "foo", &result);
+  ASSERT_EQ(r, 1);
+  ASSERT_EQ(result, "bar");
+
+  db->clone(hoid, hoid2);
+  r = tester.get_key(hoid, "foo", &result);
+  ASSERT_EQ(r, 1);
+  ASSERT_EQ(result, "bar");
+  r = tester.get_key(hoid2, "foo", &result);
+  ASSERT_EQ(r, 1);
+  ASSERT_EQ(result, "bar");
+
+  tester.remove_key(hoid, "foo");
+  r = tester.get_key(hoid2, "foo", &result);
+  ASSERT_EQ(r, 1);
+  ASSERT_EQ(result, "bar");
+  r = tester.get_key(hoid, "foo", &result);
+  ASSERT_EQ(r, 0);
+  r = tester.get_key(hoid, "foo2", &result);
+  ASSERT_EQ(r, 1);
+  ASSERT_EQ(result, "bar2");
+
+  tester.set_key(hoid, "foo", "baz");
+  tester.remove_key(hoid, "foo");
+  r = tester.get_key(hoid, "foo", &result);
+  ASSERT_EQ(r, 0);
+
+  tester.set_key(hoid, "foo2", "baz");
+  tester.remove_key(hoid, "foo2");
+  r = tester.get_key(hoid, "foo2", &result);
+  ASSERT_EQ(r, 0);
+
+  map got;
+  bufferlist header;
+
+  got.clear();
+  db->clear(hoid);
+  db->get(hoid, &header, &got);
+  ASSERT_EQ(got.size(), (unsigned)0);
+
+  got.clear();
+  r = db->clear(hoid2);
+  ASSERT_EQ(0, r);
+  db->get(hoid2, &header, &got);
+  ASSERT_EQ(got.size(), (unsigned)0);
+
+  tester.set_key(hoid, "baz", "bar");
+  got.clear();
+  db->get(hoid, &header, &got);
+  ASSERT_EQ(got.size(), (unsigned)1);
+  db->clear(hoid);
+  db->clear(hoid2);
+}
+
+TEST_F(ObjectMapTest, OddEvenClone) {
+  ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
+  ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP)));
+
+  for (unsigned i = 0; i < 1000; ++i) {
+    tester.set_key(hoid, "foo" + num_str(i), "bar" + num_str(i));
+  }
+
+  db->clone(hoid, hoid2);
+
+  int r = 0;
+  for (unsigned i = 0; i < 1000; ++i) {
+    string result;
+    r = tester.get_key(hoid, "foo" + num_str(i), &result);
+    ASSERT_EQ(1, r);
+    ASSERT_EQ("bar" + num_str(i), result);
+    r = tester.get_key(hoid2, "foo" + num_str(i), &result);
+    ASSERT_EQ(1, r);
+    ASSERT_EQ("bar" + num_str(i), result);
+
+    if (i % 2) {
+      tester.remove_key(hoid, "foo" + num_str(i));
+    } else {
+      tester.remove_key(hoid2, "foo" + num_str(i));
+    }
+  }
+
+  for (unsigned i = 0; i < 1000; ++i) {
+    string result;
+    string result2;
+    r = tester.get_key(hoid, "foo" + num_str(i), &result);
+    int r2 = tester.get_key(hoid2, "foo" + num_str(i), &result2);
+    if (i % 2) {
+      ASSERT_EQ(0, r);
+      ASSERT_EQ(1, r2);
+      ASSERT_EQ("bar" + num_str(i), result2);
+    } else {
+      ASSERT_EQ(0, r2);
+      ASSERT_EQ(1, r);
+      ASSERT_EQ("bar" + num_str(i), result);
+    }
+  }
+
+  {
+    ObjectMap::ObjectMapIterator iter = db->get_iterator(hoid);
+    iter->seek_to_first();
+    for (unsigned i = 0; i < 1000; ++i) {
+      if (!(i % 2)) {
+	ASSERT_TRUE(iter->valid());
+	ASSERT_EQ("foo" + num_str(i), iter->key());
+	iter->next();
+      }
+    }
+  }
+
+  {
+    ObjectMap::ObjectMapIterator iter2 = db->get_iterator(hoid2);
+    iter2->seek_to_first();
+    for (unsigned i = 0; i < 1000; ++i) {
+      if (i % 2) {
+	ASSERT_TRUE(iter2->valid());
+	ASSERT_EQ("foo" + num_str(i), iter2->key());
+	iter2->next();
+      }
+    }
+  }
+
+  db->clear(hoid);
+  db->clear(hoid2);
+}
+
+TEST_F(ObjectMapTest, RandomTest) {
+  tester.def_init();
+  for (unsigned i = 0; i < 5000; ++i) {
+    unsigned val = rand();
+    val <<= 8;
+    val %= 100;
+    if (!(i%100))
+      std::cout << "on op " << i
+		<< " val is " << val << std::endl;
+
+    if (val < 7) {
+      tester.auto_write_header(std::cerr);
+    } else if (val < 14) {
+      ASSERT_TRUE(tester.auto_verify_header(std::cerr));
+    } else if (val < 30) {
+      tester.auto_set_key(std::cerr);
+    } else if (val < 42) {
+      tester.auto_set_xattr(std::cerr);
+    } else if (val < 55) {
+      ASSERT_TRUE(tester.auto_check_present_key(std::cerr));
+    } else if (val < 62) {
+      ASSERT_TRUE(tester.auto_check_present_xattr(std::cerr));
+    } else if (val < 70) {
+      ASSERT_TRUE(tester.auto_check_absent_key(std::cerr));
+    } else if (val < 72) {
+      ASSERT_TRUE(tester.auto_check_absent_xattr(std::cerr));
+    } else if (val < 73) {
+      tester.auto_clear_omap(std::cerr);
+    } else if (val < 76) {
+      tester.auto_delete_object(std::cerr);
+    } else if (val < 85) {
+      tester.auto_clone_key(std::cerr);
+    } else if (val < 92) {
+      tester.auto_remove_xattr(std::cerr);
+    } else {
+      tester.auto_remove_key(std::cerr);
+    }
+  }
+}
diff --git a/ceph/src/test/TestSignalHandlers.cc b/ceph/src/test/TestSignalHandlers.cc
new file mode 100644
index 00000000..ae0bc2ae
--- /dev/null
+++ b/ceph/src/test/TestSignalHandlers.cc
@@ -0,0 +1,97 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+/*
+ * TestSignalHandlers
+ *
+ * Test the Ceph signal handlers
+ */
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/errno.h"
+#include "common/debug.h"
+#include "common/config.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using std::string;
+
+// avoid compiler warning about dereferencing NULL pointer
+static int* get_null()
+{
+  return 0;
+}
+
+static void simple_segv_test()
+{
+  generic_dout(-1) << "triggering SIGSEGV..." << dendl;
+  int i = *get_null();
+  std::cout << "i = " << i << std::endl;
+}
+
+static void infinite_recursion_test_impl()
+{
+  infinite_recursion_test_impl();
+}
+
+static void infinite_recursion_test()
+{
+  generic_dout(0) << "triggering SIGSEGV with infinite recursion..." << dendl;
+  infinite_recursion_test_impl();
+}
+
+static void usage()
+{
+  cerr << "usage: TestSignalHandlers [test]" << std::endl;
+  cerr << "--simple_segv: run simple_segv test" << std::endl;
+  cerr << "--infinite_recursion: run infinite_recursion test" << std::endl;
+  generic_client_usage(); // Will exit()
+}
+
+typedef void (*test_fn_t)(void);
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  test_fn_t fn = NULL;
+  for (std::vector::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+    } else if (ceph_argparse_flag(args, i, "--infinite_recursion", (char*)NULL)) {
+      fn = infinite_recursion_test;
+    } else if (ceph_argparse_flag(args, i, "-s", "--simple_segv", (char*)NULL)) {
+      fn = simple_segv_test;
+    } else {
+      cerr << "Garbage at end of command line." << std::endl;
+      usage();
+    }
+  }
+  if (!fn) {
+    std::cerr << "Please select a test to run. Type -h for help." << std::endl;
+    usage();
+  }
+  fn();
+  return 0;
+}
diff --git a/ceph/src/test/TestTimers.cc b/ceph/src/test/TestTimers.cc
new file mode 100644
index 00000000..6e6f4735
--- /dev/null
+++ b/ceph/src/test/TestTimers.cc
@@ -0,0 +1,284 @@
+#include "common/ceph_argparse.h"
+#include "common/Mutex.h"
+#include "common/Timer.h"
+#include "global/global_init.h"
+
+#include 
+
+/*
+ * TestTimers
+ *
+ * Tests the timer classes
+ */
+#define MAX_TEST_CONTEXTS 5
+
+class TestContext;
+
+namespace
+{
+  int array[MAX_TEST_CONTEXTS];
+  int array_idx;
+  TestContext* test_contexts[MAX_TEST_CONTEXTS];
+
+  Mutex array_lock("test_timers_mutex");
+}
+
+class TestContext : public Context
+{
+public:
+  TestContext(int num_)
+    : num(num_)
+  {
+  }
+
+  virtual void finish(int r)
+  {
+    array_lock.Lock();
+    cout << "TestContext " << num << std::endl;
+    array[array_idx++] = num;
+    array_lock.Unlock();
+  }
+
+  virtual ~TestContext()
+  {
+  }
+
+protected:
+  int num;
+};
+
+class StrictOrderTestContext : public TestContext
+{
+public:
+  StrictOrderTestContext (int num_)
+    : TestContext(num_)
+  {
+  }
+
+  virtual void finish(int r)
+  {
+    array_lock.Lock();
+    cout << "StrictOrderTestContext " << num << std::endl;
+    array[num] = num;
+    array_lock.Unlock();
+  }
+
+  virtual ~StrictOrderTestContext()
+  {
+  }
+};
+
+static void print_status(const char *str, int ret)
+{
+  cout << str << ": ";
+  cout << ((ret == 0) ? "SUCCESS" : "FAILURE");
+  cout << std::endl;
+}
+
+template 
+static int basic_timer_test(T &timer, Mutex *lock)
+{
+  int ret = 0;
+  memset(&array, 0, sizeof(array));
+  array_idx = 0;
+  memset(&test_contexts, 0, sizeof(test_contexts));
+
+  cout << __PRETTY_FUNCTION__ << std::endl;
+
+  for (int i = 0; i < MAX_TEST_CONTEXTS; ++i) {
+    test_contexts[i] = new TestContext(i);
+  }
+
+
+  for (int i = 0; i < MAX_TEST_CONTEXTS; ++i) {
+    if (lock)
+      lock->Lock();
+    utime_t inc(2 * i, 0);
+    utime_t t = ceph_clock_now(g_ceph_context) + inc;
+    timer.add_event_at(t, test_contexts[i]);
+    if (lock)
+      lock->Unlock();
+  }
+
+  bool done = false;
+  do {
+    sleep(1);
+    array_lock.Lock();
+    done = (array_idx == MAX_TEST_CONTEXTS);
+    array_lock.Unlock();
+  } while (!done);
+
+  for (int i = 0; i < MAX_TEST_CONTEXTS; ++i) {
+    if (array[i] != i) {
+      ret = 1;
+      cout << "error: expected array[" << i << "] = " << i
+	   << "; got " << array[i] << " instead." << std::endl;
+    }
+  }
+
+  return ret;
+}
+
+static int test_out_of_order_insertion(SafeTimer &timer, Mutex *lock)
+{
+  int ret = 0;
+  memset(&array, 0, sizeof(array));
+  array_idx = 0;
+  memset(&test_contexts, 0, sizeof(test_contexts));
+
+  cout << __PRETTY_FUNCTION__ << std::endl;
+
+  test_contexts[0] = new StrictOrderTestContext(0);
+  test_contexts[1] = new StrictOrderTestContext(1);
+
+  {
+    utime_t inc(100, 0);
+    utime_t t = ceph_clock_now(g_ceph_context) + inc;
+    lock->Lock();
+    timer.add_event_at(t, test_contexts[0]);
+    lock->Unlock();
+  }
+
+  {
+    utime_t inc(2, 0);
+    utime_t t = ceph_clock_now(g_ceph_context) + inc;
+    lock->Lock();
+    timer.add_event_at(t, test_contexts[1]);
+    lock->Unlock();
+  }
+
+  int secs = 0;
+  for (; secs < 100 ; ++secs) {
+    sleep(1);
+    array_lock.Lock();
+    int a = array[1];
+    array_lock.Unlock();
+    if (a == 1)
+      break;
+  }
+
+  if (secs == 100) {
+    ret = 1;
+    cout << "error: expected array[" << 1 << "] = " << 1
+	 << "; got " << array[1] << " instead." << std::endl;
+  }
+
+  return ret;
+}
+
+static int safe_timer_cancel_all_test(SafeTimer &safe_timer, Mutex& safe_timer_lock)
+{
+  cout << __PRETTY_FUNCTION__ << std::endl;
+
+  int ret = 0;
+  memset(&array, 0, sizeof(array));
+  array_idx = 0;
+  memset(&test_contexts, 0, sizeof(test_contexts));
+
+  for (int i = 0; i < MAX_TEST_CONTEXTS; ++i) {
+    test_contexts[i] = new TestContext(i);
+  }
+
+  safe_timer_lock.Lock();
+  for (int i = 0; i < MAX_TEST_CONTEXTS; ++i) {
+    utime_t inc(4 * i, 0);
+    utime_t t = ceph_clock_now(g_ceph_context) + inc;
+    safe_timer.add_event_at(t, test_contexts[i]);
+  }
+  safe_timer_lock.Unlock();
+
+  sleep(10);
+
+  safe_timer_lock.Lock();
+  safe_timer.cancel_all_events();
+  safe_timer_lock.Unlock();
+
+  for (int i = 0; i < array_idx; ++i) {
+    if (array[i] != i) {
+      ret = 1;
+      cout << "error: expected array[" << i << "] = " << i
+	   << "; got " << array[i] << " instead." << std::endl;
+    }
+  }
+
+  return ret;
+}
+
+static int safe_timer_cancellation_test(SafeTimer &safe_timer, Mutex& safe_timer_lock)
+{
+  cout << __PRETTY_FUNCTION__ << std::endl;
+
+  int ret = 0;
+  memset(&array, 0, sizeof(array));
+  array_idx = 0;
+  memset(&test_contexts, 0, sizeof(test_contexts));
+
+  for (int i = 0; i < MAX_TEST_CONTEXTS; ++i) {
+    test_contexts[i] = new StrictOrderTestContext(i);
+  }
+
+  safe_timer_lock.Lock();
+  for (int i = 0; i < MAX_TEST_CONTEXTS; ++i) {
+    utime_t inc(4 * i, 0);
+    utime_t t = ceph_clock_now(g_ceph_context) + inc;
+    safe_timer.add_event_at(t, test_contexts[i]);
+  }
+  safe_timer_lock.Unlock();
+
+  // cancel the even-numbered events
+  for (int i = 0; i < MAX_TEST_CONTEXTS; i += 2) {
+    safe_timer_lock.Lock();
+    safe_timer.cancel_event(test_contexts[i]);
+    safe_timer_lock.Unlock();
+  }
+
+  sleep(20);
+
+  safe_timer_lock.Lock();
+  safe_timer.cancel_all_events();
+  safe_timer_lock.Unlock();
+
+  for (int i = 1; i < array_idx; i += 2) {
+    if (array[i] != i) {
+      ret = 1;
+      cout << "error: expected array[" << i << "] = " << i
+	   << "; got " << array[i] << " instead." << std::endl;
+    }
+  }
+
+  return ret;
+}
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  int ret;
+  Mutex safe_timer_lock("safe_timer_lock");
+  SafeTimer safe_timer(g_ceph_context, safe_timer_lock);
+
+  ret = basic_timer_test (safe_timer, &safe_timer_lock);
+  if (ret)
+    goto done;
+
+  ret = safe_timer_cancel_all_test(safe_timer, safe_timer_lock);
+  if (ret)
+    goto done;
+
+  ret = safe_timer_cancellation_test(safe_timer, safe_timer_lock);
+  if (ret)
+    goto done;
+
+  ret = test_out_of_order_insertion(safe_timer, &safe_timer_lock);
+  if (ret)
+    goto done;
+
+done:
+  print_status(argv[0], ret);
+  return ret;
+}
diff --git a/ceph/src/test/admin_socket.cc b/ceph/src/test/admin_socket.cc
new file mode 100644
index 00000000..ab52cecb
--- /dev/null
+++ b/ceph/src/test/admin_socket.cc
@@ -0,0 +1,312 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/admin_socket.h"
+#include "common/admin_socket_client.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+#include 
+#include 
+#include 
+#include 
+
+class AdminSocketTest
+{
+public:
+  AdminSocketTest(AdminSocket *asokc)
+    : m_asokc(asokc)
+  {
+  }
+  bool init(const std::string &uri) {
+    return m_asokc->init(uri);
+  }
+  string bind_and_listen(const std::string &sock_path, int *fd) {
+    return m_asokc->bind_and_listen(sock_path, fd);
+  }
+  bool shutdown() {
+    m_asokc->shutdown();
+    return true;
+  }
+  AdminSocket *m_asokc;
+};
+
+TEST(AdminSocket, Teardown) {
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+  AdminSocketTest asoct(asokc.get());
+  ASSERT_EQ(true, asoct.shutdown());
+}
+
+TEST(AdminSocket, TeardownSetup) {
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+  AdminSocketTest asoct(asokc.get());
+  ASSERT_EQ(true, asoct.shutdown());
+  ASSERT_EQ(true, asoct.init(get_rand_socket_path()));
+  ASSERT_EQ(true, asoct.shutdown());
+}
+
+TEST(AdminSocket, SendHelp) {
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+  AdminSocketTest asoct(asokc.get());
+  ASSERT_EQ(true, asoct.shutdown());
+  ASSERT_EQ(true, asoct.init(get_rand_socket_path()));
+  AdminSocketClient client(get_rand_socket_path());
+
+  {
+    string help;
+    ASSERT_EQ("", client.do_request("{\"prefix\":\"help\"}", &help));
+    ASSERT_NE(string::npos, help.find("\"list available commands\""));
+  }
+  {
+    string help;
+    ASSERT_EQ("", client.do_request("{"
+				    " \"prefix\":\"help\","
+				    " \"format\":\"xml\","
+				    "}", &help));
+    ASSERT_NE(string::npos, help.find(">list available commands<"));
+  }
+  {
+    string help;
+    ASSERT_EQ("", client.do_request("{"
+				    " \"prefix\":\"help\","
+				    " \"format\":\"UNSUPPORTED\","
+				    "}", &help));
+    ASSERT_NE(string::npos, help.find("\"list available commands\""));
+  }
+  ASSERT_EQ(true, asoct.shutdown());
+}
+
+TEST(AdminSocket, SendNoOp) {
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+  AdminSocketTest asoct(asokc.get());
+  ASSERT_EQ(true, asoct.shutdown());
+  ASSERT_EQ(true, asoct.init(get_rand_socket_path()));
+  AdminSocketClient client(get_rand_socket_path());
+  string version;
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"0\"}", &version));
+  ASSERT_EQ(CEPH_ADMIN_SOCK_VERSION, version);
+  ASSERT_EQ(true, asoct.shutdown());
+}
+
+class MyTest : public AdminSocketHook {
+  bool call(std::string command, cmdmap_t& cmdmap, std::string format, bufferlist& result) {
+    std::vector args;
+    cmd_getval(g_ceph_context, cmdmap, "args", args);
+    result.append(command);
+    result.append("|");
+    string resultstr;
+    for (std::vector::iterator it = args.begin();
+	 it != args.end(); ++it) {
+      if (it != args.begin())
+	resultstr += ' ';
+      resultstr += *it;
+    }
+    result.append(resultstr);
+    return true;
+  }
+};
+
+TEST(AdminSocket, RegisterCommand) {
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+  AdminSocketTest asoct(asokc.get());
+  ASSERT_EQ(true, asoct.shutdown());
+  ASSERT_EQ(true, asoct.init(get_rand_socket_path()));
+  AdminSocketClient client(get_rand_socket_path());
+  ASSERT_EQ(0, asoct.m_asokc->register_command("test", "test", new MyTest(), ""));
+  string result;
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test\"}", &result));
+  ASSERT_EQ("test|", result);
+  ASSERT_EQ(true, asoct.shutdown());
+}
+
+class MyTest2 : public AdminSocketHook {
+  bool call(std::string command, cmdmap_t& cmdmap, std::string format, bufferlist& result) {
+    std::vector args;
+    cmd_getval(g_ceph_context, cmdmap, "args", args);
+    result.append(command);
+    result.append("|");
+    string resultstr;
+    for (std::vector::iterator it = args.begin();
+	 it != args.end(); ++it) {
+      if (it != args.begin())
+	resultstr += ' ';
+      resultstr += *it;
+    }
+    result.append(resultstr);
+    return true;
+  }
+};
+
+TEST(AdminSocket, RegisterCommandPrefixes) {
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+  AdminSocketTest asoct(asokc.get());
+  ASSERT_EQ(true, asoct.shutdown());
+  ASSERT_EQ(true, asoct.init(get_rand_socket_path()));
+  AdminSocketClient client(get_rand_socket_path());
+  ASSERT_EQ(0, asoct.m_asokc->register_command("test", "test name=args,type=CephString,n=N", new MyTest(), ""));
+  ASSERT_EQ(0, asoct.m_asokc->register_command("test command", "test command name=args,type=CephString,n=N", new MyTest2(), ""));
+  string result;
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test\"}", &result));
+  ASSERT_EQ("test|", result);
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test command\"}", &result));
+  ASSERT_EQ("test command|", result);
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test command\",\"args\":[\"post\"]}", &result));
+  ASSERT_EQ("test command|post", result);
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test command\",\"args\":[\" post\"]}", &result));
+  ASSERT_EQ("test command| post", result);
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test\",\"args\":[\"this thing\"]}", &result));
+  ASSERT_EQ("test|this thing", result);
+
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test\",\"args\":[\" command post\"]}", &result));
+  ASSERT_EQ("test| command post", result);
+  ASSERT_EQ("", client.do_request("{\"prefix\":\"test\",\"args\":[\" this thing\"]}", &result));
+  ASSERT_EQ("test| this thing", result);
+  ASSERT_EQ(true, asoct.shutdown());
+}
+
+class BlockingHook : public AdminSocketHook {
+public:
+  Mutex _lock;
+  Cond _cond;
+
+  BlockingHook() : _lock("BlockingHook::_lock") {}
+
+  bool call(std::string command, cmdmap_t& cmdmap, std::string format, bufferlist& result) {
+    Mutex::Locker l(_lock);
+    _cond.Wait(_lock);
+    return true;
+  }
+};
+
+TEST(AdminSocketClient, Ping) {
+  string path = get_rand_socket_path();
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+  AdminSocketClient client(path);
+  // no socket
+  {
+    bool ok;
+    std::string result = client.ping(&ok);
+    EXPECT_NE(std::string::npos, result.find("No such file or directory"));
+    ASSERT_FALSE(ok);
+  }
+  // file exists but does not allow connections (no process, wrong type...)
+  ASSERT_TRUE(::creat(path.c_str(), 0777));
+  {
+    bool ok;
+    std::string result = client.ping(&ok);
+    EXPECT_NE(std::string::npos, result.find("Connection refused"));
+    ASSERT_FALSE(ok);
+  }
+  // a daemon is connected to the socket
+  {
+    AdminSocketTest asoct(asokc.get());
+    ASSERT_TRUE(asoct.init(path));
+    bool ok;
+    std::string result = client.ping(&ok);
+    EXPECT_EQ("", result);
+    ASSERT_TRUE(ok);
+    ASSERT_TRUE(asoct.shutdown());
+  }
+  // hardcoded five seconds timeout prevents infinite blockage
+  {
+    AdminSocketTest asoct(asokc.get());
+    BlockingHook *blocking = new BlockingHook();
+    ASSERT_EQ(0, asoct.m_asokc->register_command("0", "0", blocking, ""));
+    ASSERT_TRUE(asoct.init(path));
+    bool ok;
+    std::string result = client.ping(&ok);
+    EXPECT_NE(std::string::npos, result.find("Resource temporarily unavailable"));
+    ASSERT_FALSE(ok);
+    {
+      Mutex::Locker l(blocking->_lock);
+      blocking->_cond.Signal();
+    }
+    ASSERT_TRUE(asoct.shutdown());
+    delete blocking;
+  }
+}
+
+TEST(AdminSocket, bind_and_listen) {
+  string path = get_rand_socket_path();
+  std::auto_ptr
+      asokc(new AdminSocket(g_ceph_context));
+
+  AdminSocketTest asoct(asokc.get());
+  // successfull bind
+  {
+    int fd = 0;
+    string message;
+    message = asoct.bind_and_listen(path, &fd);
+    ASSERT_NE(0, fd);
+    ASSERT_EQ("", message);
+    ASSERT_EQ(0, ::close(fd));
+    ASSERT_EQ(0, ::unlink(path.c_str()));
+  }
+  // silently discard an existing file
+  {
+    int fd = 0;
+    string message;
+    ASSERT_TRUE(::creat(path.c_str(), 0777));
+    message = asoct.bind_and_listen(path, &fd);
+    ASSERT_NE(0, fd);
+    ASSERT_EQ("", message);
+    ASSERT_EQ(0, ::close(fd));
+    ASSERT_EQ(0, ::unlink(path.c_str()));
+  }
+  // do not take over a live socket
+  {
+    ASSERT_TRUE(asoct.init(path));
+    int fd = 0;
+    string message;
+    message = asoct.bind_and_listen(path, &fd);
+    std::cout << "message: " << message << std::endl;
+    EXPECT_NE(std::string::npos, message.find("File exists"));
+    ASSERT_TRUE(asoct.shutdown());
+  }
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  vector def_args;
+  global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd .. ;
+ *   make unittest_admin_socket &&
+ *    valgrind \
+ *    --max-stackframe=20000000 --tool=memcheck \
+ *   ./unittest_admin_socket --debug-asok 20 # --gtest_filter=AdminSocket*.*
+ * "
+ * End:
+ */
+
diff --git a/ceph/src/test/base64.cc b/ceph/src/test/base64.cc
new file mode 100644
index 00000000..9fcdef41
--- /dev/null
+++ b/ceph/src/test/base64.cc
@@ -0,0 +1,100 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/armor.h"
+#include "common/config.h"
+#include "include/buffer.h"
+#include "include/encoding.h"
+
+#include "gtest/gtest.h"
+
+TEST(RoundTrip, SimpleRoundTrip) {
+  static const int OUT_LEN = 4096;
+  const char * const original = "abracadabra";
+  const char * const correctly_encoded = "YWJyYWNhZGFicmE=";
+  char out[OUT_LEN];
+  memset(out, 0, sizeof(out));
+  int alen = ceph_armor(out, out + OUT_LEN, original, original + strlen(original));
+  ASSERT_STREQ(correctly_encoded, out);
+
+  char out2[OUT_LEN];
+  memset(out2, 0, sizeof(out2));
+  ceph_unarmor(out2, out2 + OUT_LEN, out, out + alen);
+  ASSERT_STREQ(original, out2);
+}
+
+TEST(RoundTrip, RandomRoundTrips) {
+  static const int IN_MAX = 1024;
+  static const int OUT_MAX = 4096;
+  static const int ITERS = 1000;
+  for (int i = 0; i < ITERS; ++i) {
+    unsigned int seed = i;
+    int in_len = rand_r(&seed) % IN_MAX;
+
+    char in[IN_MAX];
+    memset(in, 0, sizeof(in));
+    for (int j = 0; j < in_len; ++j) {
+      in[j] = rand_r(&seed) % 0xff;
+    }
+    char out[OUT_MAX];
+    memset(out, 0, sizeof(out));
+    int alen = ceph_armor(out, out + OUT_MAX, in, in + in_len);
+    ASSERT_GE(alen, 0);
+
+    char decoded[IN_MAX];
+    memset(decoded, 0, sizeof(decoded));
+    int blen = ceph_unarmor(decoded, decoded + IN_MAX, out, out + alen);
+    ASSERT_GE(blen, 0);
+
+    ASSERT_EQ(memcmp(in, decoded, in_len), 0);
+  }
+}
+
+TEST(EdgeCase, EndsInNewline) {
+  static const int OUT_MAX = 4096;
+
+  char b64[] =
+    "aaaa\n";
+
+    char decoded[OUT_MAX];
+    memset(decoded, 0, sizeof(decoded));
+    int blen = ceph_unarmor(decoded, decoded + OUT_MAX, b64, b64 + sizeof(b64)-1);
+    ASSERT_GE(blen, 0);
+}
+
+TEST(FuzzEncoding, BadDecode1) {
+  static const int OUT_LEN = 4096;
+  const char * const bad_encoded = "FAKEBASE64 foo";
+  char out[OUT_LEN];
+  memset(out, 0, sizeof(out));
+  int alen = ceph_unarmor(out, out + OUT_LEN, bad_encoded, bad_encoded + strlen(bad_encoded));
+  ASSERT_LT(alen, 0);
+}
+
+TEST(FuzzEncoding, BadDecode2) {
+  string str("FAKEBASE64 foo");
+  bool failed = false;
+  try {
+    bufferlist bl;
+    bl.append(str);
+
+    bufferlist cl;
+    cl.decode_base64(bl);
+    cl.hexdump(std::cerr);
+  }
+  catch (const buffer::error &err) {
+    failed = true;
+  }
+  ASSERT_EQ(failed, true);
+}
diff --git a/ceph/src/test/bench/backend.h b/ceph/src/test/bench/backend.h
new file mode 100644
index 00000000..740e0981
--- /dev/null
+++ b/ceph/src/test/bench/backend.h
@@ -0,0 +1,26 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef BACKENDH
+#define BACKENDH
+
+#include "include/Context.h"
+
+class Backend {
+public:
+  virtual void write(
+    const string &oid,
+    uint64_t offset,
+    const bufferlist &bl,
+    Context *on_applied,
+    Context *on_commit) = 0;
+
+  virtual void read(
+    const string &oid,
+    uint64_t offset,
+    uint64_t length,
+    bufferlist *bl,
+    Context *on_complete) = 0;
+  virtual ~Backend() {}
+};
+
+#endif
diff --git a/ceph/src/test/bench/bencher.cc b/ceph/src/test/bench/bencher.cc
new file mode 100644
index 00000000..c1781466
--- /dev/null
+++ b/ceph/src/test/bench/bencher.cc
@@ -0,0 +1,201 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "bencher.h"
+#include "include/utime.h"
+#include 
+#include "include/memory.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+
+template
+struct C_Holder : public Context {
+  T obj;
+  C_Holder(
+    T obj)
+    : obj(obj) {}
+  void finish(int r) {
+    return;
+  }
+};
+
+struct OnDelete {
+  Context *c;
+  OnDelete(Context *c) : c(c) {}
+  ~OnDelete() { c->complete(0); }
+};
+
+struct Cleanup : public Context {
+  Bencher *bench;
+  Cleanup(Bencher *bench) : bench(bench) {}
+  void finish(int r) {
+    bench->complete_op();
+  }
+};
+
+struct OnWriteApplied : public Context {
+  Bencher *bench;
+  uint64_t seq;
+  ceph::shared_ptr on_delete;
+  OnWriteApplied(
+    Bencher *bench, uint64_t seq,
+    ceph::shared_ptr on_delete
+    ) : bench(bench), seq(seq), on_delete(on_delete) {}
+  void finish(int r) {
+    bench->stat_collector->write_applied(seq);
+  }
+};
+
+struct OnWriteCommit : public Context {
+  Bencher *bench;
+  uint64_t seq;
+  ceph::shared_ptr on_delete;
+  OnWriteCommit(
+    Bencher *bench, uint64_t seq,
+    ceph::shared_ptr on_delete
+    ) : bench(bench), seq(seq), on_delete(on_delete) {}
+  void finish(int r) {
+    bench->stat_collector->write_committed(seq);
+  }
+};
+
+struct OnReadComplete : public Context {
+  Bencher *bench;
+  uint64_t seq;
+  boost::scoped_ptr bl;
+  OnReadComplete(Bencher *bench, uint64_t seq, bufferlist *bl) :
+    bench(bench), seq(seq), bl(bl) {}
+  void finish(int r) {
+    bench->stat_collector->read_complete(seq);
+    bench->complete_op();
+  }
+};
+
+void Bencher::start_op() {
+  Mutex::Locker l(lock);
+  while (open_ops >= max_in_flight)
+    open_ops_cond.Wait(lock);
+  ++open_ops;
+}
+
+void Bencher::drain_ops() {
+  Mutex::Locker l(lock);
+  while (open_ops)
+    open_ops_cond.Wait(lock);
+}
+
+void Bencher::complete_op() {
+  Mutex::Locker l(lock);
+  assert(open_ops > 0);
+  --open_ops;
+  open_ops_cond.Signal();
+}
+
+struct OnFinish {
+  bool *done;
+  Mutex *lock;
+  Cond *cond;
+  OnFinish(
+    bool *done,
+    Mutex *lock,
+    Cond *cond) :
+    done(done), lock(lock), cond(cond) {}
+  ~OnFinish() {
+    Mutex::Locker l(*lock);
+    *done = true;
+    cond->Signal();
+  }
+};
+
+void Bencher::init(
+  const set &objects,
+  uint64_t size,
+  std::ostream *out
+  )
+{
+  bufferlist bl;
+  for (uint64_t i = 0; i < size; ++i) {
+    bl.append(0);
+  }
+  Mutex lock("init_lock");
+  Cond cond;
+  bool done = 0;
+  {
+    ceph::shared_ptr on_finish(
+      new OnFinish(&done, &lock, &cond));
+    uint64_t num = 0;
+    for (set::const_iterator i = objects.begin();
+	 i != objects.end();
+	 ++i, ++num) {
+      if (!(num % 20))
+	*out << "Creating " << num << "/" << objects.size() << std::endl;
+      backend->write(
+	*i,
+	0,
+	bl,
+	new C_Holder >(on_finish),
+	new C_Holder >(on_finish)
+	);
+    }
+  }
+  {
+    Mutex::Locker l(lock);
+    while (!done)
+      cond.Wait(lock);
+  }
+}
+
+void Bencher::run_bench()
+{
+  time_t end = time(0) + max_duration;
+  uint64_t ops = 0;
+
+  bufferlist bl;
+
+  while ((!max_duration || time(0) < end) && (!max_ops || ops < max_ops)) {
+    start_op();
+    uint64_t seq = stat_collector->next_seq();
+    boost::tuple next =
+      (*op_dist)();
+    string obj_name = next.get<0>();
+    uint64_t offset = next.get<1>();
+    uint64_t length = next.get<2>();
+    OpType op_type = next.get<3>();
+    switch (op_type) {
+      case WRITE: {
+	ceph::shared_ptr on_delete(
+	  new OnDelete(new Cleanup(this)));
+	stat_collector->start_write(seq, length);
+	while (bl.length() < length) {
+	  bl.append(rand());
+	}
+	backend->write(
+	  obj_name,
+	  offset,
+	  bl,
+	  new OnWriteApplied(
+	    this, seq, on_delete),
+	  new OnWriteCommit(
+	    this, seq, on_delete)
+	  );
+	break;
+      }
+      case READ: {
+	stat_collector->start_read(seq, length);
+	bufferlist *read_bl = new bufferlist;
+	backend->read(
+	  obj_name,
+	  offset,
+	  length,
+	  read_bl,
+	  new OnReadComplete(
+	    this, seq, read_bl)
+	  );
+	break;
+      }
+      default: {
+	assert(0);
+      }
+    }
+  }
+  drain_ops();
+}
diff --git a/ceph/src/test/bench/bencher.h b/ceph/src/test/bench/bencher.h
new file mode 100644
index 00000000..a4f6321b
--- /dev/null
+++ b/ceph/src/test/bench/bencher.h
@@ -0,0 +1,147 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef BENCHERH
+#define BENCHERH
+
+#include 
+#include "distribution.h"
+#include "stat_collector.h"
+#include "backend.h"
+#include 
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/Thread.h"
+
+struct OnWriteApplied;
+struct OnWriteCommit;
+struct OnReadComplete;
+struct Cleanup;
+
+class Bencher : public Thread {
+public:
+  enum OpType {
+    WRITE,
+    READ
+  };
+
+private:
+  boost::scoped_ptr<
+    Distribution > > op_dist;
+  ceph::shared_ptr stat_collector;
+  boost::scoped_ptr backend;
+  const uint64_t max_in_flight;
+  const uint64_t max_duration;
+  const uint64_t max_ops;
+
+  Mutex lock;
+  Cond open_ops_cond;
+  uint64_t open_ops;
+  void start_op();
+  void drain_ops();
+  void complete_op();
+public:
+  Bencher(
+    Distribution > *op_gen,
+    ceph::shared_ptr stat_collector,
+    Backend *backend,
+    uint64_t max_in_flight,
+    uint64_t max_duration,
+    uint64_t max_ops) :
+    op_dist(op_gen),
+    stat_collector(stat_collector),
+    backend(backend),
+    max_in_flight(max_in_flight),
+    max_duration(max_duration),
+    max_ops(max_ops),
+    lock("Bencher::lock"),
+    open_ops(0)
+  {}
+  Bencher(
+    Distribution > *op_gen,
+    StatCollector *stat_collector,
+    Backend *backend,
+    uint64_t max_in_flight,
+    uint64_t max_duration,
+    uint64_t max_ops) :
+    op_dist(op_gen),
+    stat_collector(stat_collector),
+    backend(backend),
+    max_in_flight(max_in_flight),
+    max_duration(max_duration),
+    max_ops(max_ops),
+    lock("Bencher::lock"),
+    open_ops(0)
+  {}
+  Bencher(
+    Distribution *object_gen,
+    Distribution *offset_gen,
+    Distribution *length_gen,
+    Distribution *op_type_gen,
+    StatCollector *stat_collector,
+    Backend *backend,
+    uint64_t max_in_flight,
+    uint64_t max_duration,
+    uint64_t max_ops) :
+    op_dist(
+      new FourTupleDist(
+	object_gen, offset_gen, length_gen, op_type_gen)),
+    stat_collector(stat_collector),
+    backend(backend),
+    max_in_flight(max_in_flight),
+    max_duration(max_duration),
+    max_ops(max_ops),
+    lock("Bencher::lock"),
+    open_ops(0)
+  {}
+
+  void init(
+    const set &objects,
+    uint64_t size,
+    std::ostream *out
+    );
+
+  void run_bench();
+  void *entry() {
+    run_bench();
+    return 0;
+  }
+  friend struct OnWriteApplied;
+  friend struct OnWriteCommit;
+  friend struct OnReadComplete;
+  friend struct Cleanup;
+};
+
+class SequentialLoad :
+  public Distribution<
+  boost::tuple > {
+  set objects;
+  uint64_t size;
+  uint64_t length;
+  set::iterator object_pos;
+  uint64_t cur_pos;
+  boost::scoped_ptr > op_dist;
+  SequentialLoad(const SequentialLoad &other);
+public:
+  SequentialLoad(
+    const set &_objects, uint64_t size,
+    uint64_t length,
+    Distribution *op_dist)
+    : objects(_objects), size(size), length(length),
+      object_pos(objects.begin()), cur_pos(0),
+      op_dist(op_dist) {}
+
+  boost::tuple
+  operator()() {
+    boost::tuple ret =
+      boost::make_tuple(*object_pos, cur_pos, length, (*op_dist)());
+    cur_pos += length;
+    if (cur_pos > size) {
+      cur_pos = 0;
+      ++object_pos;
+    }
+    if (object_pos == objects.end())
+      object_pos = objects.begin();
+    return ret;
+  }
+};
+#endif
diff --git a/ceph/src/test/bench/detailed_stat_collector.cc b/ceph/src/test/bench/detailed_stat_collector.cc
new file mode 100644
index 00000000..d3aceae9
--- /dev/null
+++ b/ceph/src/test/bench/detailed_stat_collector.cc
@@ -0,0 +1,163 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "detailed_stat_collector.h"
+#include 
+#include 
+#include 
+
+void DetailedStatCollector::Op::dump(
+  ostream *out,
+  Formatter *f)
+{
+  if (!out)
+    return;
+  f->open_object_section(type.c_str());
+  f->dump_string("type", type);
+  f->dump_float("start", start);
+  f->dump_float("latency", latency);
+  f->dump_int("size", size);
+  f->dump_int("seq", seq);
+  f->close_section();
+  f->flush(*out);
+  *out << std::endl;
+}
+
+static utime_t cur_time()
+{
+  struct timeval tv;
+  gettimeofday(&tv, 0);
+  return utime_t(&tv);
+}
+
+DetailedStatCollector::Aggregator::Aggregator()
+  : recent_size(0), total_size(0), recent_latency(0),
+    total_latency(0), recent_ops(0), total_ops(0), started(false)
+{}
+
+void DetailedStatCollector::Aggregator::add(const Op &op)
+{
+  if (!started) {
+    last = first = op.start;
+    started = true;
+  }
+  ++recent_ops;
+  ++total_ops;
+  recent_size += op.size;
+  total_size += op.size;
+  recent_latency += op.latency;
+  total_latency += op.latency;
+}
+
+void DetailedStatCollector::Aggregator::dump(Formatter *f)
+{
+  utime_t now = cur_time();
+  f->dump_stream("time") << now;
+  f->dump_float("avg_recent_latency", recent_latency / recent_ops);
+  f->dump_float("avg_total_latency", total_latency / total_ops);
+  f->dump_float("avg_recent_iops", recent_ops / (now - last));
+  f->dump_float("avg_total_iops", total_ops / (now - first));
+  f->dump_float("avg_recent_throughput", recent_size / (now - last));
+  f->dump_float("avg_total_throughput", total_size / (now - first));
+  f->dump_float("avg_recent_throughput_mb",
+		(recent_size / (now - last)) / (1024*1024));
+  f->dump_float("avg_total_throughput_mb",
+		(total_size / (now - first)) / (1024*1024));
+  f->dump_float("duration", now - last);
+  last = now;
+  recent_latency = 0;
+  recent_size = 0;
+  recent_ops = 0;
+}
+
+DetailedStatCollector::DetailedStatCollector(
+  double bin_size,
+  Formatter *formatter,
+  ostream *out,
+  ostream *summary_out,
+  AdditionalPrinting *details
+  ) : bin_size(bin_size), f(formatter), out(out),
+      summary_out(summary_out), details(details),
+      lock("Stat::lock"), cur_seq(0) {
+  last_dump = cur_time();
+}
+
+uint64_t DetailedStatCollector::next_seq()
+{
+  Mutex::Locker l(lock);
+  if (summary_out && ((cur_time() - last_dump) > bin_size)) {
+    f->open_object_section("stats");
+    for (map::iterator i = aggregators.begin();
+	 i != aggregators.end();
+	 ++i) {
+      f->open_object_section(i->first.c_str());
+      i->second.dump(f.get());
+      f->close_section();
+    }
+    f->close_section();
+    f->flush(*summary_out);
+    *summary_out << std::endl;
+    if (details) {
+      (*details)(summary_out);
+      *summary_out << std::endl;
+    }
+    last_dump = cur_time();
+  }
+  return cur_seq++;
+}
+
+void DetailedStatCollector::start_write(uint64_t seq, uint64_t length)
+{
+  Mutex::Locker l(lock);
+  utime_t now(cur_time());
+  not_committed.insert(make_pair(seq, make_pair(length, now)));
+  not_applied.insert(make_pair(seq, make_pair(length, now)));
+}
+
+void DetailedStatCollector::start_read(uint64_t seq, uint64_t length)
+{
+  Mutex::Locker l(lock);
+  utime_t now(cur_time());
+  not_read.insert(make_pair(seq, make_pair(length, now)));
+}
+
+void DetailedStatCollector::write_applied(uint64_t seq)
+{
+  Mutex::Locker l(lock);
+  Op op(
+    "write_applied",
+    not_applied[seq].second,
+    cur_time() - not_applied[seq].second,
+    not_applied[seq].first,
+    seq);
+  op.dump(out, f.get());
+  aggregators["write_applied"].add(op);
+  not_applied.erase(seq);
+}
+
+void DetailedStatCollector::write_committed(uint64_t seq)
+{
+  Mutex::Locker l(lock);
+  Op op(
+    "write_committed",
+    not_committed[seq].second,
+    cur_time() - not_committed[seq].second,
+    not_committed[seq].first,
+    seq);
+  op.dump(out, f.get());
+  aggregators["write_committed"].add(op);
+  not_committed.erase(seq);
+}
+
+void DetailedStatCollector::read_complete(uint64_t seq)
+{
+  Mutex::Locker l(lock);
+  Op op(
+    "read",
+    not_read[seq].second,
+    cur_time() - not_read[seq].second,
+    not_read[seq].first,
+    seq);
+  op.dump(out, f.get());
+  aggregators["read"].add(op);
+  not_read.erase(seq);
+}
diff --git a/ceph/src/test/bench/detailed_stat_collector.h b/ceph/src/test/bench/detailed_stat_collector.h
new file mode 100644
index 00000000..60b7cab3
--- /dev/null
+++ b/ceph/src/test/bench/detailed_stat_collector.h
@@ -0,0 +1,96 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef DETAILEDSTATCOLLECTERH
+#define DETAILEDSTATCOLLECTERH
+
+#include "stat_collector.h"
+#include "common/Formatter.h"
+#include 
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "include/utime.h"
+#include 
+#include 
+#include 
+#include 
+
+class DetailedStatCollector : public StatCollector {
+public:
+  class AdditionalPrinting {
+  public:
+    virtual void operator()(std::ostream *) = 0;
+    virtual ~AdditionalPrinting() {}
+  };
+private:
+  struct Op {
+    string type;
+    utime_t start;
+    double latency;
+    uint64_t size;
+    uint64_t seq;
+    Op(
+      string type,
+      utime_t start,
+      double latency,
+      uint64_t size,
+      uint64_t seq)
+      : type(type), start(start), latency(latency),
+	size(size), seq(seq) {}
+    void dump(ostream *out, Formatter *f);
+  };
+  class Aggregator {
+    uint64_t recent_size;
+    uint64_t total_size;
+    double recent_latency;
+    double total_latency;
+    utime_t last;
+    utime_t first;
+    uint64_t recent_ops;
+    uint64_t total_ops;
+    bool started;
+  public:
+    Aggregator();
+
+    void add(const Op &op);
+    void dump(Formatter *f);
+  };
+  const double bin_size;
+  boost::scoped_ptr f;
+  ostream *out;
+  ostream *summary_out;
+  boost::scoped_ptr details;
+  utime_t last_dump;
+
+  Mutex lock;
+  Cond cond;
+
+  map aggregators;
+
+  map > not_applied;
+  map > not_committed;
+  map > not_read;
+
+  uint64_t cur_seq;
+
+  void dump(
+    const string &type,
+    boost::tuple stuff);
+public:
+  DetailedStatCollector(
+    double bin_size,
+    Formatter *formatter,
+    ostream *out,
+    ostream *summary_out,
+    AdditionalPrinting *details = 0
+    );
+
+  uint64_t next_seq();
+  void start_write(uint64_t seq, uint64_t size);
+  void start_read(uint64_t seq, uint64_t size);
+  void write_applied(uint64_t seq);
+  void write_committed(uint64_t seq);
+  void read_complete(uint64_t seq);
+
+};
+
+#endif
diff --git a/ceph/src/test/bench/distribution.h b/ceph/src/test/bench/distribution.h
new file mode 100644
index 00000000..d3525b8b
--- /dev/null
+++ b/ceph/src/test/bench/distribution.h
@@ -0,0 +1,136 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef DISTIRITBIONHPP
+#define DISTIRITBIONHPP
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+typedef boost::mt11213b rngen_t;
+
+template 
+class Distribution {
+public:
+  virtual T operator()() = 0;
+  virtual ~Distribution() {}
+};
+
+template 
+class FourTupleDist : public Distribution > {
+  boost::scoped_ptr > t;
+  boost::scoped_ptr > u;
+  boost::scoped_ptr > v;
+  boost::scoped_ptr > w;
+public:
+  FourTupleDist(
+    Distribution *t,
+    Distribution *u,
+    Distribution *v,
+    Distribution *w)
+    : t(t), u(u), v(v), w(w) {}
+  boost::tuple operator()() {
+    return boost::make_tuple((*t)(), (*u)(), (*v)(), (*w)());
+  }
+};
+
+template 
+class RandomDist : public Distribution {
+  rngen_t rng;
+  std::map contents;
+public:
+  RandomDist(const rngen_t &rng, std::set &initial) : rng(rng) {
+    uint64_t count = 0;
+    for (typename std::set::iterator i = initial.begin();
+	 i != initial.end();
+	 ++i, ++count) {
+      contents.insert(std::make_pair(count, *i));
+    }
+  }
+  virtual T operator()() {
+    assert(contents.size());
+    boost::uniform_int<> value(0, contents.size() - 1);
+    return contents.find(value(rng))->second;
+  }
+};
+
+template 
+class WeightedDist : public Distribution {
+  rngen_t rng;
+  double total;
+  std::map contents;
+public:
+  WeightedDist(const rngen_t &rng, const std::set > &initial)
+    : rng(rng), total(0) {
+    for (typename std::set >::const_iterator i =
+	   initial.begin();
+	 i != initial.end();
+	 ++i) {
+      total += i->first;
+      contents.insert(std::make_pair(total, i->second));
+    }
+  }
+  virtual T operator()() {
+    return contents.lower_bound(
+      boost::uniform_real<>(0, total)(rng))->second;
+  }
+};
+
+template 
+class SequentialDist : public Distribution {
+  rngen_t rng;
+  std::vector contents;
+  typename std::vector::iterator cur;
+public:
+  SequentialDist(rngen_t rng, U &initial) : rng(rng) {
+    contents.insert(initial.begin(), initial.end());
+    cur = contents.begin();
+  }
+  virtual T operator()() {
+    assert(contents.size());
+    if (cur == contents.end())
+      cur = contents.begin();
+    return *(cur++);
+  }
+};
+
+class UniformRandom : public Distribution {
+  rngen_t rng;
+  uint64_t min;
+  uint64_t max;
+public:
+  UniformRandom(const rngen_t &rng, uint64_t min, uint64_t max) :
+    rng(rng), min(min), max(max) {}
+  virtual uint64_t operator()() {
+    return boost::uniform_int(min, max)(rng);
+  }
+};
+
+class Align : public Distribution {
+  boost::scoped_ptr > dist;
+  uint64_t align;
+public:
+  Align(Distribution *dist, uint64_t align) :
+    dist(dist), align(align) {}
+  virtual uint64_t operator()() {
+    uint64_t ret = (*dist)();
+    return ret - (ret % align);
+  }
+};
+
+class Uniform : public Distribution {
+  uint64_t val;
+public:
+  Uniform(uint64_t val) : val(val) {}
+  virtual uint64_t operator()() {
+    return val;
+  }
+};
+
+#endif
diff --git a/ceph/src/test/bench/dumb_backend.cc b/ceph/src/test/bench/dumb_backend.cc
new file mode 100644
index 00000000..bf78ab22
--- /dev/null
+++ b/ceph/src/test/bench/dumb_backend.cc
@@ -0,0 +1,117 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "acconfig.h"
+
+#include 
+#include "dumb_backend.h"
+
+string DumbBackend::get_full_path(const string &oid)
+{
+	return path + "/" + oid;
+}
+
+void DumbBackend::_write(
+  const string &oid,
+  uint64_t offset,
+  const bufferlist &bl,
+  Context *on_applied,
+  Context *on_commit)
+{
+  string full_path(get_full_path(oid));
+  int fd = ::open(
+    full_path.c_str(), O_CREAT|O_WRONLY, 0777);
+  if (fd < 0) {
+    std::cout << full_path << ": errno is " << errno << std::endl;
+    assert(0);
+  }
+
+  int r =  ::lseek(fd, offset, SEEK_SET);
+  if (r < 0) {
+    r = errno;
+    std::cout << "lseek failed, errno is: " << r << std::endl;
+    ::close(fd);
+    return;
+  }
+  bl.write_fd(fd);
+  on_applied->complete(0);
+  if (do_fsync)
+    ::fsync(fd);
+#ifdef HAVE_SYNC_FILE_RANGE
+  if (do_sync_file_range)
+    ::sync_file_range(fd, offset, bl.length(),
+		      SYNC_FILE_RANGE_WAIT_AFTER);
+#else
+# warning "sync_file_range not supported!"
+#endif
+#ifdef HAVE_POSIX_FADVISE
+  if (do_fadvise) {
+    int fa_r = ::posix_fadvise(fd, offset, bl.length(), POSIX_FADV_DONTNEED);
+    if (fa_r) {
+        std::cout << "posix_fadvise failed, errno is: " << fa_r << std::endl;
+    }
+  }
+#else
+# warning "posix_fadvise not supported!"
+#endif
+  ::close(fd);
+  {
+    Mutex::Locker l(pending_commit_mutex);
+    pending_commits.insert(on_commit);
+  }
+  sem.Put();
+}
+
+void DumbBackend::read(
+  const string &oid,
+  uint64_t offset,
+  uint64_t length,
+  bufferlist *bl,
+  Context *on_complete)
+{
+  string full_path(get_full_path(oid));
+  int fd = ::open(
+    full_path.c_str(), 0, O_RDONLY);
+  if (fd < 0) return;
+
+  int r = ::lseek(fd, offset, SEEK_SET);
+  if (r < 0) {
+    r = errno;
+    std::cout << "lseek failed, errno is: " << r << std::endl;
+    ::close(fd);
+    return;
+  }
+
+  bl->read_fd(fd, length);
+  ::close(fd);
+  on_complete->complete(0);
+}
+
+void DumbBackend::sync_loop()
+{
+  while (1) {
+    sleep(sync_interval);
+    {
+      Mutex::Locker l(sync_loop_mutex);
+      if (sync_loop_stop != 0) {
+	sync_loop_stop = 2;
+	sync_loop_cond.Signal();
+	break;
+      }
+    }
+    tp.pause();
+#ifdef HAVE_SYS_SYNCFS
+    ::syncfs(sync_fd);
+#else
+    ::sync();
+#endif
+    {
+      Mutex::Locker l(pending_commit_mutex);
+      for (set::iterator i = pending_commits.begin();
+	   i != pending_commits.end();
+	   pending_commits.erase(i++)) {
+	(*i)->complete(0);
+      }
+    }
+    tp.unpause();
+  }
+}
diff --git a/ceph/src/test/bench/dumb_backend.h b/ceph/src/test/bench/dumb_backend.h
new file mode 100644
index 00000000..3c70b1a0
--- /dev/null
+++ b/ceph/src/test/bench/dumb_backend.h
@@ -0,0 +1,168 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef DUMBBACKEND
+#define DUMBBACKEND
+
+#include "backend.h"
+#include "include/Context.h"
+#include "os/ObjectStore.h"
+#include "common/WorkQueue.h"
+#include "common/Semaphore.h"
+
+#include 
+
+class DumbBackend : public Backend {
+	const string path;
+
+  struct write_item {
+    const string oid;
+    bufferlist bl;
+    uint64_t offset;
+    Context *on_applied;
+    Context *on_commit;
+    write_item(
+      const string &oid,
+      const bufferlist &bl,
+      uint64_t offset,
+      Context *on_applied,
+      Context *on_commit) :
+      oid(oid), bl(bl), offset(offset), on_applied(on_applied),
+      on_commit(on_commit) {}
+  };
+
+  Semaphore sem;
+
+  bool do_fsync;
+  bool do_sync_file_range;
+  bool do_fadvise;
+  unsigned sync_interval;
+  int sync_fd;
+  ThreadPool tp;
+
+  class SyncThread : public Thread {
+    DumbBackend *backend;
+  public:
+    SyncThread(DumbBackend *backend) : backend(backend) {}
+    void *entry() {
+      backend->sync_loop();
+      return 0;
+    }
+  } thread;
+  friend class SyncThread;
+
+  Mutex sync_loop_mutex;
+  Cond sync_loop_cond;
+  int sync_loop_stop; // 0 for running, 1 for stopping, 2 for stopped
+  void sync_loop();
+
+  Mutex pending_commit_mutex;
+  set pending_commits;
+
+  class WriteQueue : public ThreadPool::WorkQueue {
+    deque item_queue;
+    DumbBackend *backend;
+
+  public:
+    WriteQueue(
+      DumbBackend *backend,
+      time_t ti,
+      ThreadPool *tp) :
+      ThreadPool::WorkQueue("DumbBackend::queue", ti, ti*10, tp),
+      backend(backend) {}
+    bool _enqueue(write_item *item) {
+      item_queue.push_back(item);
+      return true;
+    }
+    void _dequeue(write_item*) { assert(0); }
+    write_item *_dequeue() {
+      if (item_queue.empty())
+	return 0;
+      write_item *retval = item_queue.front();
+      item_queue.pop_front();
+      return retval;
+    }
+    bool _empty() {
+      return item_queue.empty();
+    }
+    void _process(write_item *item) {
+      return backend->_write(
+	item->oid,
+	item->offset,
+	item->bl,
+	item->on_applied,
+	item->on_commit);
+    }
+    void _clear() {
+      return item_queue.clear();
+    }
+  } queue;
+  friend class WriteQueue;
+
+  string get_full_path(const string &oid);
+
+  void _write(
+    const string &oid,
+    uint64_t offset,
+    const bufferlist &bl,
+    Context *on_applied,
+    Context *on_commit);
+
+public:
+  DumbBackend(
+    const string &path,
+    bool do_fsync,
+    bool do_sync_file_range,
+    bool do_fadvise,
+    unsigned sync_interval,
+    int sync_fd,
+    unsigned worker_threads,
+    CephContext *cct)
+    : path(path), do_fsync(do_fsync),
+      do_sync_file_range(do_sync_file_range),
+      do_fadvise(do_fadvise),
+      sync_interval(sync_interval),
+      sync_fd(sync_fd),
+      tp(cct, "DumbBackend::tp", worker_threads),
+      thread(this),
+      sync_loop_mutex("DumbBackend::sync_loop_mutex"),
+      sync_loop_stop(0),
+      pending_commit_mutex("DumbBackend::pending_commit_mutex"),
+      queue(this, 20, &tp) {
+    thread.create();
+    tp.start();
+    for (unsigned i = 0; i < 10*worker_threads; ++i) {
+      sem.Put();
+    }
+  }
+  ~DumbBackend() {
+    {
+      Mutex::Locker l(sync_loop_mutex);
+      if (sync_loop_stop == 0)
+	sync_loop_stop = 1;
+      while (sync_loop_stop < 2)
+	sync_loop_cond.Wait(sync_loop_mutex);
+    }
+    tp.stop();
+    thread.join();
+  }
+  void write(
+    const string &oid,
+    uint64_t offset,
+    const bufferlist &bl,
+    Context *on_applied,
+    Context *on_commit) {
+    sem.Get();
+    queue.queue(
+      new write_item(
+	oid, bl, offset, on_applied, on_commit));
+  }
+
+  void read(
+    const string &oid,
+    uint64_t offset,
+    uint64_t length,
+    bufferlist *bl,
+    Context *on_complete);
+};
+
+#endif
diff --git a/ceph/src/test/bench/rados_backend.cc b/ceph/src/test/bench/rados_backend.cc
new file mode 100644
index 00000000..89d808a3
--- /dev/null
+++ b/ceph/src/test/bench/rados_backend.cc
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "rados_backend.h"
+#include 
+
+typedef boost::tuple arg_type;
+
+void on_applied(void *completion, void *_arg) {
+  arg_type *arg = static_cast(_arg);
+  arg->get<1>()->complete(0);
+}
+
+void on_complete(void *completion, void *_arg) {
+  arg_type *arg = static_cast(_arg);
+  arg->get<0>()->complete(0);
+  arg->get<2>()->release();
+  delete arg;
+}
+
+void RadosBackend::write(
+  const string &oid,
+  uint64_t offset,
+  const bufferlist &bl,
+  Context *on_write_applied,
+  Context *on_commit)
+{
+  librados::AioCompletion *completion = librados::Rados::aio_create_completion();
+
+
+  void *arg = static_cast(new arg_type(on_commit, on_write_applied,
+					       completion));
+
+  completion->set_safe_callback(
+    arg,
+    on_complete);
+
+  completion->set_complete_callback(
+    arg,
+    on_applied);
+
+  ioctx->aio_write(oid, completion, bl, bl.length(), offset);
+}
+
+void RadosBackend::read(
+  const string &oid,
+  uint64_t offset,
+  uint64_t length,
+  bufferlist *bl,
+  Context *on_read_complete)
+{
+  librados::AioCompletion *completion = librados::Rados::aio_create_completion();
+
+
+  void *arg = static_cast(new arg_type(on_read_complete, 0,
+					       completion));
+
+  completion->set_complete_callback(
+    arg,
+    on_complete);
+
+  ioctx->aio_read(oid, completion, bl, length, offset);
+}
diff --git a/ceph/src/test/bench/rados_backend.h b/ceph/src/test/bench/rados_backend.h
new file mode 100644
index 00000000..911d6c7d
--- /dev/null
+++ b/ceph/src/test/bench/rados_backend.h
@@ -0,0 +1,31 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef RADOSBACKENDH
+#define RADOSBACKENDH
+
+#include "backend.h"
+#include "include/Context.h"
+#include "include/rados/librados.hpp"
+
+class RadosBackend : public Backend {
+  librados::IoCtx *ioctx;
+public:
+  RadosBackend(
+    librados::IoCtx *ioctx)
+    : ioctx(ioctx) {}
+  void write(
+    const string &oid,
+    uint64_t offset,
+    const bufferlist &bl,
+    Context *on_applied,
+    Context *on_commit);
+
+  void read(
+    const string &oid,
+    uint64_t offset,
+    uint64_t length,
+    bufferlist *bl,
+    Context *on_complete);
+};
+
+#endif
diff --git a/ceph/src/test/bench/rbd_backend.cc b/ceph/src/test/bench/rbd_backend.cc
new file mode 100644
index 00000000..79288968
--- /dev/null
+++ b/ceph/src/test/bench/rbd_backend.cc
@@ -0,0 +1,51 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "rbd_backend.h"
+#include 
+
+typedef boost::tuple arg_type;
+
+void on_complete(void *completion, void *_arg) {
+  arg_type *arg = static_cast(_arg);
+  librbd::RBD::AioCompletion *comp =
+    static_cast(completion);
+  ssize_t r = comp->get_return_value();
+  assert(r >= 0);
+  arg->get<0>()->complete(0);
+  if (arg->get<1>())
+    arg->get<1>()->complete(0);
+  comp->release();
+  delete arg;
+}
+
+void RBDBackend::write(
+  const string &oid,
+  uint64_t offset,
+  const bufferlist &bl,
+  Context *on_write_applied,
+  Context *on_commit)
+{
+  bufferlist &bl_non_const = const_cast(bl);
+  ceph::shared_ptr image = (*m_images)[oid];
+  void *arg = static_cast(new arg_type(on_commit, on_write_applied));
+  librbd::RBD::AioCompletion *completion =
+    new librbd::RBD::AioCompletion(arg, on_complete);
+  int r = image->aio_write(offset, (size_t) bl_non_const.length(),
+			   bl_non_const, completion);
+  assert(r >= 0);
+}
+
+void RBDBackend::read(
+  const string &oid,
+  uint64_t offset,
+  uint64_t length,
+  bufferlist *bl,
+  Context *on_read_complete)
+{
+  ceph::shared_ptr image = (*m_images)[oid];
+  void *arg = static_cast(new arg_type(on_read_complete, NULL));
+  librbd::RBD::AioCompletion *completion =
+    new librbd::RBD::AioCompletion(arg, on_complete);
+  int r = image->aio_read(offset, (size_t) length, *bl, completion);
+  assert(r >= 0);
+}
diff --git a/ceph/src/test/bench/rbd_backend.h b/ceph/src/test/bench/rbd_backend.h
new file mode 100644
index 00000000..9c3bc8bb
--- /dev/null
+++ b/ceph/src/test/bench/rbd_backend.h
@@ -0,0 +1,30 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef CEPH_TEST_SMALLIOBENCH_RBD_BACKEND_H
+#define CEPH_TEST_SMALLIOBENCH_RBD_BACKEND_H
+
+#include "backend.h"
+#include "include/Context.h"
+#include "include/rbd/librbd.hpp"
+
+class RBDBackend : public Backend {
+  map > *m_images;
+public:
+  RBDBackend(map > *images)
+    : m_images(images) {}
+  void write(
+    const string &oid,
+    uint64_t offset,
+    const bufferlist &bl,
+    Context *on_applied,
+    Context *on_commit);
+
+  void read(
+    const string &oid,
+    uint64_t offset,
+    uint64_t length,
+    bufferlist *bl,
+    Context *on_complete);
+};
+
+#endif
diff --git a/ceph/src/test/bench/small_io_bench.cc b/ceph/src/test/bench/small_io_bench.cc
new file mode 100644
index 00000000..2b200279
--- /dev/null
+++ b/ceph/src/test/bench/small_io_bench.cc
@@ -0,0 +1,206 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "bencher.h"
+#include "rados_backend.h"
+#include "detailed_stat_collector.h"
+#include "distribution.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+int main(int argc, char **argv)
+{
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("num-concurrent-ops", po::value()->default_value(10),
+     "set number of concurrent ops")
+    ("num-objects", po::value()->default_value(500),
+     "set number of objects to use")
+    ("object-size", po::value()->default_value(4<<20),
+     "set object size")
+    ("io-size", po::value()->default_value(4<<10),
+     "set io size")
+    ("write-ratio", po::value()->default_value(0.75),
+     "set ratio of read to write")
+    ("duration", po::value()->default_value(0),
+     "set max duration, 0 for unlimited")
+    ("max-ops", po::value()->default_value(0),
+     "set max ops, 0 for unlimited")
+    ("seed", po::value(),
+     "seed")
+    ("ceph-client-id", po::value()->default_value("admin"),
+     "set ceph client id")
+    ("pool-name", po::value()->default_value("data"),
+     "set pool")
+    ("op-dump-file", po::value()->default_value(""),
+     "set file for dumping op details, omit for stderr")
+    ("init-only", po::value()->default_value(false),
+     "populate object set")
+    ("do-not-init", po::value()->default_value(false),
+     "use existing object set")
+    ("use-prefix", po::value()->default_value(""),
+     "use previously populated prefix")
+    ("offset-align", po::value()->default_value(4096),
+     "align offset by")
+    ("sequential", po::value()->default_value(false),
+     "use sequential access pattern")
+    ("disable-detailed-ops", po::value()->default_value(false),
+     "don't dump per op stats")
+    ;
+
+  po::variables_map vm;
+  po::store(po::parse_command_line(argc, argv, desc), vm);
+  po::notify(vm);
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  if (vm["do-not-init"].as() && !vm["use-prefix"].as().size()) {
+    cout << "Must supply prefix if do-not-init is specified" << std::endl;
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  if (vm["init-only"].as() && !vm["use-prefix"].as().size()) {
+    cout << "Must supply prefix for init-only" << std::endl;
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  string prefix;
+  if (vm["use-prefix"].as().size()) {
+    prefix = vm["use-prefix"].as();
+  } else {
+    char hostname_cstr[100];
+    gethostname(hostname_cstr, 100);
+    stringstream hostpid;
+    hostpid << hostname_cstr << getpid() << "-";
+    prefix = hostpid.str();
+  }
+
+  set objects;
+  for (unsigned i = 0; i < vm["num-objects"].as();
+       ++i) {
+    stringstream name;
+    name << prefix << "-object_" << i;
+    objects.insert(name.str());
+  }
+
+  rngen_t rng;
+  if (vm.count("seed"))
+    rng = rngen_t(vm["seed"].as());
+
+  set > ops;
+  ops.insert(make_pair(vm["write-ratio"].as(), Bencher::WRITE));
+  ops.insert(make_pair(1-vm["write-ratio"].as(), Bencher::READ));
+
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  int r = rados.init(vm["ceph-client-id"].as().c_str());
+  if (r < 0) {
+    cerr << "error in init r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.conf_read_file(NULL);
+  if (r < 0) {
+    cerr << "error in conf_read_file r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.conf_parse_env(NULL);
+  if (r < 0) {
+    cerr << "error in conf_parse_env r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.connect();
+  if (r < 0) {
+    cerr << "error in connect r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.ioctx_create(vm["pool-name"].as().c_str(), ioctx);
+  if (r < 0) {
+    cerr << "error in ioctx_create r=" << r << std::endl;
+    return -r;
+  }
+
+  ostream *detailed_ops = 0;
+  ofstream myfile;
+  if (vm["disable-detailed-ops"].as()) {
+    detailed_ops = 0;
+  } else if (vm["op-dump-file"].as().size()) {
+    myfile.open(vm["op-dump-file"].as().c_str());
+    detailed_ops = &myfile;
+  } else {
+    detailed_ops = &cerr;
+  }
+
+  Distribution<
+    boost::tuple > *gen = 0;
+  if (vm["sequential"].as()) {
+    std::cout << "Using Sequential generator" << std::endl;
+    gen = new SequentialLoad(
+      objects,
+      vm["object-size"].as(),
+      vm["io-size"].as(),
+      new WeightedDist(rng, ops)
+      );
+  } else {
+    std::cout << "Using random generator" << std::endl;
+    gen = new FourTupleDist(
+      new RandomDist(rng, objects),
+      new Align(
+	new UniformRandom(
+	  rng,
+	  0,
+	  vm["object-size"].as() - vm["io-size"].as()),
+	vm["offset-align"].as()
+	),
+      new Uniform(vm["io-size"].as()),
+      new WeightedDist(rng, ops)
+      );
+  }
+
+  Bencher bencher(
+    gen,
+    new DetailedStatCollector(1, new JSONFormatter, detailed_ops, &cout),
+    new RadosBackend(&ioctx),
+    vm["num-concurrent-ops"].as(),
+    vm["duration"].as(),
+    vm["max-ops"].as());
+
+  if (!vm["do-not-init"].as()) {
+    bencher.init(objects, vm["object-size"].as(), &std::cout);
+    cout << "Created objects..." << std::endl;
+  } else {
+    cout << "Not initing objects..." << std::endl;
+  }
+
+  if (!vm["init-only"].as()) {
+    bencher.run_bench();
+  } else {
+    cout << "init-only" << std::endl;
+  }
+
+  rados.shutdown();
+  if (vm["op-dump-file"].as().size()) {
+    myfile.close();
+  }
+  return 0;
+}
diff --git a/ceph/src/test/bench/small_io_bench_dumb.cc b/ceph/src/test/bench/small_io_bench_dumb.cc
new file mode 100644
index 00000000..73841c3c
--- /dev/null
+++ b/ceph/src/test/bench/small_io_bench_dumb.cc
@@ -0,0 +1,231 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "acconfig.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "bencher.h"
+#include "rados_backend.h"
+#include "detailed_stat_collector.h"
+#include "distribution.h"
+#include "global/global_init.h"
+#include "os/FileStore.h"
+#include "dumb_backend.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+int main(int argc, char **argv)
+{
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("num-concurrent-ops", po::value()->default_value(10),
+     "set number of concurrent ops")
+    ("num-objects", po::value()->default_value(500),
+     "set number of objects to use")
+    ("object-size", po::value()->default_value(4<<20),
+     "set object size")
+    ("io-size", po::value()->default_value(4<<10),
+     "set io size")
+    ("write-ratio", po::value()->default_value(0.75),
+     "set ratio of read to write")
+    ("duration", po::value()->default_value(0),
+     "set max duration, 0 for unlimited")
+    ("max-ops", po::value()->default_value(0),
+     "set max ops, 0 for unlimited")
+    ("seed", po::value(),
+     "seed")
+    ("num-colls", po::value()->default_value(20),
+     "number of collections")
+    ("op-dump-file", po::value()->default_value(""),
+     "set file for dumping op details, omit for stderr")
+    ("filestore-path", po::value(),
+     "path to filestore directory, mandatory")
+    ("offset-align", po::value()->default_value(4096),
+     "align offset by")
+    ("fsync", po::value()->default_value(false),
+     "fsync after each write")
+    ("sync-file-range", po::value()->default_value(false),
+     "sync-file-range after each write")
+    ("fadvise", po::value()->default_value(false),
+     "fadvise after each write")
+    ("sync-interval", po::value()->default_value(30),
+     "frequency to sync")
+    ("sequential", po::value()->default_value(false),
+     "use sequential access pattern")
+    ("disable-detailed-ops", po::value()->default_value(false),
+     "don't dump per op stats")
+    ;
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+    po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+  po::store(
+    parsed,
+    vm);
+  po::notify(vm);
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+    CODE_ENVIRONMENT_UTILITY,
+    CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  if (!vm.count("filestore-path")) {
+    cout << "Must provide filestore-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  }
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  rngen_t rng;
+  if (vm.count("seed"))
+    rng = rngen_t(vm["seed"].as());
+
+  set > ops;
+  ops.insert(make_pair(vm["write-ratio"].as(), Bencher::WRITE));
+  ops.insert(make_pair(1-vm["write-ratio"].as(), Bencher::READ));
+
+  cout << "Creating objects.." << std::endl;
+  bufferlist bl;
+  for (uint64_t i = 0; i < vm["object-size"].as(); ++i) {
+    bl.append(0);
+  }
+  set objects;
+
+  for (uint64_t num = 0; num < vm["num-objects"].as(); ++num) {
+    unsigned col_num = num % vm["num-colls"].as();
+    stringstream coll, obj;
+    coll << "collection_" << col_num;
+    obj << "obj_" << num;
+    if (num == col_num) {
+      std::cout << "collection " << coll.str() << std::endl;
+      string coll_str(
+	vm["filestore-path"].as() + string("/") + coll.str());
+      int r = ::mkdir(
+	coll_str.c_str(),
+	0777);
+      if (r < 0) {
+	std::cerr << "Error " << errno << " creating collection" << std::endl;
+	return 1;
+      }
+    }
+    objects.insert(coll.str() + "/" + obj.str());
+  }
+  string meta_str(vm["filestore-path"].as() + string("/meta"));
+  int r = ::mkdir(
+    meta_str.c_str(),
+    0777);
+  if (r < 0) {
+    std::cerr << "Error " << errno << " creating collection" << std::endl;
+    return 1;
+  }
+  r = ::open(meta_str.c_str(), 0);
+  if (r < 0) {
+    std::cerr << "Error " << errno << " opening meta" << std::endl;
+    return 1;
+  }
+  int sync_fd = r;
+
+  ostream *detailed_ops = 0;
+  ofstream myfile;
+  if (vm["disable-detailed-ops"].as()) {
+    detailed_ops = 0;
+  } else if (vm["op-dump-file"].as().size()) {
+    myfile.open(vm["op-dump-file"].as().c_str());
+    detailed_ops = &myfile;
+  } else {
+    detailed_ops = &cerr;
+  }
+
+  Distribution<
+    boost::tuple > *gen = 0;
+  if (vm["sequential"].as()) {
+    std::cout << "Using Sequential generator" << std::endl;
+    gen = new SequentialLoad(
+      objects,
+      vm["object-size"].as(),
+      vm["io-size"].as(),
+      new WeightedDist(rng, ops)
+      );
+  } else {
+    std::cout << "Using random generator" << std::endl;
+    gen = new FourTupleDist(
+      new RandomDist(rng, objects),
+      new Align(
+	new UniformRandom(
+	  rng,
+	  0,
+	  vm["object-size"].as() - vm["io-size"].as()),
+	vm["offset-align"].as()
+	),
+      new Uniform(vm["io-size"].as()),
+      new WeightedDist(rng, ops)
+      );
+  }
+
+#ifndef HAVE_SYNC_FILE_RANGE
+  if (vm["sync-file-range"].as())
+    std::cerr << "Warning: sync_file_range(2) not supported!" << std::endl;
+#endif
+
+#ifndef HAVE_POSIX_FADVISE
+  if (vm["fadvise"].as())
+    std::cerr << "Warning: posix_fadvise(2) not supported!" << std::endl;
+#endif
+
+  Bencher bencher(
+    gen,
+    new DetailedStatCollector(1, new JSONFormatter, detailed_ops, &cout),
+    new DumbBackend(
+      vm["filestore-path"].as(),
+      vm["fsync"].as(),
+      vm["sync-file-range"].as(),
+      vm["fadvise"].as(),
+      vm["sync-interval"].as(),
+      sync_fd,
+      10,
+      g_ceph_context),
+    vm["num-concurrent-ops"].as(),
+    vm["duration"].as(),
+    vm["max-ops"].as());
+
+  bencher.init(objects, vm["object-size"].as(), &std::cout);
+  cout << "Created objects..." << std::endl;
+
+  bencher.run_bench();
+
+  if (vm["op-dump-file"].as().size()) {
+    myfile.close();
+  }
+  return 0;
+}
diff --git a/ceph/src/test/bench/small_io_bench_fs.cc b/ceph/src/test/bench/small_io_bench_fs.cc
new file mode 100644
index 00000000..7f03f5ad
--- /dev/null
+++ b/ceph/src/test/bench/small_io_bench_fs.cc
@@ -0,0 +1,246 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "bencher.h"
+#include "rados_backend.h"
+#include "detailed_stat_collector.h"
+#include "distribution.h"
+#include "global/global_init.h"
+#include "os/FileStore.h"
+#include "testfilestore_backend.h"
+#include "common/perf_counters.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+struct MorePrinting : public DetailedStatCollector::AdditionalPrinting {
+  CephContext *cct;
+  MorePrinting(CephContext *cct) : cct(cct) {}
+  void operator()(std::ostream *out) {
+    bufferlist bl;
+    Formatter *f = new_formatter("json-pretty");
+    cct->get_perfcounters_collection()->dump_formatted(f, 0);
+    f->flush(bl);
+    delete f;
+    bl.append('\0');
+    *out << bl.c_str() << std::endl;
+  }
+};
+
+int main(int argc, char **argv)
+{
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("num-concurrent-ops", po::value()->default_value(10),
+     "set number of concurrent ops")
+    ("num-objects", po::value()->default_value(500),
+     "set number of objects to use")
+    ("object-size", po::value()->default_value(4<<20),
+     "set object size")
+    ("io-size", po::value()->default_value(4<<10),
+     "set io size")
+    ("write-ratio", po::value()->default_value(0.75),
+     "set ratio of read to write")
+    ("duration", po::value()->default_value(0),
+     "set max duration, 0 for unlimited")
+    ("max-ops", po::value()->default_value(0),
+     "set max ops, 0 for unlimited")
+    ("seed", po::value(),
+     "seed")
+    ("num-colls", po::value()->default_value(20),
+     "number of collections")
+    ("op-dump-file", po::value()->default_value(""),
+     "set file for dumping op details, omit for stderr")
+    ("filestore-path", po::value(),
+     "path to filestore directory, mandatory")
+    ("journal-path", po::value(),
+     "path to journal, mandatory")
+    ("offset-align", po::value()->default_value(4096),
+     "align offset by")
+    ("write-infos", po::value()->default_value(false),
+      "write info objects with main writes")
+    ("sequential", po::value()->default_value(false),
+     "do sequential writes like rbd")
+    ("disable-detailed-ops", po::value()->default_value(false),
+     "don't dump per op stats")
+    ("num-writers", po::value()->default_value(1),
+     "num write threads")
+    ;
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+    po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+  po::store(
+    parsed,
+    vm);
+  po::notify(vm);
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+    CODE_ENVIRONMENT_UTILITY,
+    CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  if (!vm.count("filestore-path") || !vm.count("journal-path")) {
+    cout << "Must provide filestore-path and journal-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  }
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  rngen_t rng;
+  if (vm.count("seed"))
+    rng = rngen_t(vm["seed"].as());
+
+  set > ops;
+  ops.insert(make_pair(vm["write-ratio"].as(), Bencher::WRITE));
+  ops.insert(make_pair(1-vm["write-ratio"].as(), Bencher::READ));
+
+  FileStore fs(vm["filestore-path"].as(),
+	       vm["journal-path"].as());
+
+  if (fs.mkfs() < 0) {
+    cout << "mkfs failed" << std::endl;
+    return 1;
+  }
+
+  if (fs.mount() < 0) {
+    cout << "mount failed" << std::endl;
+    return 1;
+  }
+
+  ostream *detailed_ops = 0;
+  ofstream myfile;
+  if (vm["disable-detailed-ops"].as()) {
+    detailed_ops = 0;
+  } else if (vm["op-dump-file"].as().size()) {
+    myfile.open(vm["op-dump-file"].as().c_str());
+    detailed_ops = &myfile;
+  } else {
+    detailed_ops = &cerr;
+  }
+
+  ceph::shared_ptr col(
+    new DetailedStatCollector(
+      1, new JSONFormatter, detailed_ops, &cout,
+      new MorePrinting(g_ceph_context)));
+
+  cout << "Creating objects.." << std::endl;
+  bufferlist bl;
+  for (uint64_t i = 0; i < vm["object-size"].as(); ++i) {
+    bl.append(0);
+  }
+
+  for (uint64_t num = 0; num < vm["num-colls"].as(); ++num) {
+    stringstream coll;
+    coll << "collection_" << num;
+    std::cout << "collection " << coll.str() << std::endl;
+    ObjectStore::Transaction t;
+    t.create_collection(coll_t(coll.str()));
+    fs.apply_transaction(t);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(coll_t(string("meta")));
+    fs.apply_transaction(t);
+  }
+
+  vector > benchers(
+    vm["num-writers"].as());
+  for (vector >::iterator i = benchers.begin();
+       i != benchers.end();
+       ++i) {
+    set objects;
+    for (uint64_t num = 0; num < vm["num-objects"].as(); ++num) {
+      unsigned col_num = num % vm["num-colls"].as();
+      stringstream coll, obj;
+      coll << "collection_" << col_num;
+      obj << "obj_" << num << "_bencher_" << (i - benchers.begin());
+      objects.insert(coll.str() + string("/") + obj.str());
+    }
+    Distribution<
+      boost::tuple > *gen = 0;
+    if (vm["sequential"].as()) {
+      std::cout << "Using Sequential generator" << std::endl;
+      gen = new SequentialLoad(
+	objects,
+	vm["object-size"].as(),
+	vm["io-size"].as(),
+	new WeightedDist(rng, ops)
+	);
+    } else {
+      std::cout << "Using random generator" << std::endl;
+      gen = new FourTupleDist(
+	new RandomDist(rng, objects),
+	new Align(
+	  new UniformRandom(
+	    rng,
+	    0,
+	    vm["object-size"].as() - vm["io-size"].as()),
+	  vm["offset-align"].as()
+	  ),
+	new Uniform(vm["io-size"].as()),
+	new WeightedDist(rng, ops)
+	);
+    }
+
+    Bencher *bencher = new Bencher(
+      gen,
+      col,
+      new TestFileStoreBackend(&fs, vm["write-infos"].as()),
+      vm["num-concurrent-ops"].as(),
+      vm["duration"].as(),
+      vm["max-ops"].as());
+
+    bencher->init(objects, vm["object-size"].as(), &std::cout);
+    cout << "Created objects..." << std::endl;
+    (*i).reset(bencher);
+  }
+
+  for (vector >::iterator i = benchers.begin();
+       i != benchers.end();
+       ++i) {
+    (*i)->create();
+  }
+  for (vector >::iterator i = benchers.begin();
+       i != benchers.end();
+       ++i) {
+    (*i)->join();
+  }
+
+  fs.umount();
+  if (vm["op-dump-file"].as().size()) {
+    myfile.close();
+  }
+  return 0;
+}
diff --git a/ceph/src/test/bench/small_io_bench_rbd.cc b/ceph/src/test/bench/small_io_bench_rbd.cc
new file mode 100644
index 00000000..ba7071ed
--- /dev/null
+++ b/ceph/src/test/bench/small_io_bench_rbd.cc
@@ -0,0 +1,200 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "bencher.h"
+#include "rbd_backend.h"
+#include "detailed_stat_collector.h"
+#include "distribution.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+int main(int argc, char **argv)
+{
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("num-concurrent-ops", po::value()->default_value(10),
+     "set number of concurrent ops")
+    ("num-images", po::value()->default_value(2),
+     "set number of rbd images to use")
+    ("image-size", po::value()->default_value(4096),
+     "set image size in megabytes")
+    ("order", po::value()->default_value(22),
+     "set log_2(object size)")
+    ("io-size", po::value()->default_value(4<<10),
+     "set io size")
+    ("write-ratio", po::value()->default_value(0.25),
+     "set ratio of read to write")
+    ("duration", po::value()->default_value(0),
+     "set max duration, 0 for unlimited")
+    ("max-ops", po::value()->default_value(0),
+     "set max ops, 0 for unlimited")
+    ("seed", po::value(),
+     "seed")
+    ("ceph-client-id", po::value()->default_value("admin"),
+     "set ceph client id")
+    ("pool-name", po::value()->default_value("data"),
+     "set pool")
+    ("op-dump-file", po::value()->default_value(""),
+     "set file for dumping op details, omit for stderr")
+    ("offset-align", po::value()->default_value(4096),
+     "align offset by")
+    ("sequential", po::value()->default_value(false),
+     "use sequential access pattern")
+    ("disable-detailed-ops", po::value()->default_value(false),
+     "don't dump per op stats")
+    ;
+
+  po::variables_map vm;
+  po::store(po::parse_command_line(argc, argv, desc), vm);
+  po::notify(vm);
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  string prefix;
+  char hostname_cstr[100];
+  gethostname(hostname_cstr, 100);
+  stringstream hostpid;
+  hostpid << hostname_cstr << getpid() << "-";
+  prefix = hostpid.str();
+
+  set image_names;
+  for (unsigned i = 0; i < vm["num-images"].as();
+       ++i) {
+    stringstream name;
+    name << prefix << "-image_" << i;
+    image_names.insert(name.str());
+  }
+
+  rngen_t rng;
+  if (vm.count("seed"))
+    rng = rngen_t(vm["seed"].as());
+
+  set > ops;
+  ops.insert(make_pair(vm["write-ratio"].as(), Bencher::WRITE));
+  ops.insert(make_pair(1-vm["write-ratio"].as(), Bencher::READ));
+
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  int r = rados.init(vm["ceph-client-id"].as().c_str());
+  if (r < 0) {
+    cerr << "error in init r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.conf_read_file(NULL);
+  if (r < 0) {
+    cerr << "error in conf_read_file r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.conf_parse_env(NULL);
+  if (r < 0) {
+    cerr << "error in conf_parse_env r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.connect();
+  if (r < 0) {
+    cerr << "error in connect r=" << r << std::endl;
+    return -r;
+  }
+  r = rados.ioctx_create(vm["pool-name"].as().c_str(), ioctx);
+  if (r < 0) {
+    cerr << "error in ioctx_create r=" << r << std::endl;
+    return -r;
+  }
+
+  ostream *detailed_ops = 0;
+  ofstream myfile;
+  if (vm["disable-detailed-ops"].as()) {
+    detailed_ops = 0;
+  } else if (vm["op-dump-file"].as().size()) {
+    myfile.open(vm["op-dump-file"].as().c_str());
+    detailed_ops = &myfile;
+  } else {
+    detailed_ops = &cerr;
+  }
+
+  librbd::RBD rbd;
+  {
+    map > images;
+    int order = vm["order"].as();
+    uint64_t image_size = ((uint64_t)vm["image-size"].as()) << 20;
+    for (set::const_iterator i = image_names.begin();
+	 i != image_names.end(); ++i) {
+      r = rbd.create(ioctx, i->c_str(), image_size, &order);
+      if (r < 0) {
+	cerr << "error creating image " << *i << " r=" << r << std::endl;
+	return -r;
+      }
+      ceph::shared_ptr image(new librbd::Image());
+      r = rbd.open(ioctx, *image, i->c_str());
+      if (r < 0) {
+	cerr << "error opening image " << *i << " r=" << r << std::endl;
+	return -r;
+      }
+      images[*i] = image;
+    }
+
+    Distribution<
+      boost::tuple > *gen = 0;
+    if (vm["sequential"].as()) {
+      std::cout << "Using Sequential generator" << std::endl;
+      gen = new SequentialLoad(
+	image_names,
+	image_size,
+	vm["io-size"].as(),
+	new WeightedDist(rng, ops)
+	);
+    } else {
+      std::cout << "Using random generator" << std::endl;
+      gen = new FourTupleDist(
+	new RandomDist(rng, image_names),
+	new Align(
+	  new UniformRandom(
+	    rng,
+	    0,
+	    image_size - vm["io-size"].as()),
+	  vm["offset-align"].as()
+	  ),
+	new Uniform(vm["io-size"].as()),
+	new WeightedDist(rng, ops)
+        );
+    }
+
+    Bencher bencher(
+      gen,
+      new DetailedStatCollector(1, new JSONFormatter, detailed_ops, &cout),
+      new RBDBackend(&images),
+      vm["num-concurrent-ops"].as(),
+      vm["duration"].as(),
+      vm["max-ops"].as());
+
+    bencher.run_bench();
+  }
+
+  for (set::const_iterator i = image_names.begin();
+       i != image_names.end(); ++i) {
+    rbd.remove(ioctx, i->c_str());
+  }
+  rados.shutdown();
+  if (vm["op-dump-file"].as().size()) {
+    myfile.close();
+  }
+  return 0;
+}
diff --git a/ceph/src/test/bench/stat_collector.h b/ceph/src/test/bench/stat_collector.h
new file mode 100644
index 00000000..4aef2bd6
--- /dev/null
+++ b/ceph/src/test/bench/stat_collector.h
@@ -0,0 +1,19 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef STATCOLLECTORH
+#define STATCOLLECTORH
+
+#include 
+
+class StatCollector {
+public:
+  virtual uint64_t next_seq() = 0;
+  virtual void start_write(uint64_t seq, uint64_t size) = 0;
+  virtual void start_read(uint64_t seq, uint64_t size) = 0;
+  virtual void write_applied(uint64_t seq) = 0;
+  virtual void write_committed(uint64_t seq) = 0;
+  virtual void read_complete(uint64_t seq) = 0;
+  virtual ~StatCollector() {}
+};
+
+#endif
diff --git a/ceph/src/test/bench/testfilestore_backend.cc b/ceph/src/test/bench/testfilestore_backend.cc
new file mode 100644
index 00000000..4e2ca800
--- /dev/null
+++ b/ceph/src/test/bench/testfilestore_backend.cc
@@ -0,0 +1,77 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include "testfilestore_backend.h"
+#include "global/global_init.h"
+#include "os/ObjectStore.h"
+
+struct C_DeleteTransWrapper : public Context {
+  Context *c;
+  ObjectStore::Transaction *t;
+  C_DeleteTransWrapper(
+    ObjectStore::Transaction *t,
+    Context *c) : c(c), t(t) {}
+  void finish(int r) {
+    c->complete(r);
+    delete t;
+  }
+};
+
+TestFileStoreBackend::TestFileStoreBackend(
+  ObjectStore *os, bool write_infos)
+  : os(os), finisher(g_ceph_context), write_infos(write_infos)
+{
+  finisher.start();
+}
+
+void TestFileStoreBackend::write(
+  const string &oid,
+  uint64_t offset,
+  const bufferlist &bl,
+  Context *on_applied,
+  Context *on_commit)
+{
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  size_t sep = oid.find("/");
+  assert(sep != string::npos);
+  assert(sep + 1 < oid.size());
+  string coll_str(oid.substr(0, sep));
+
+  if (!osrs.count(coll_str))
+    osrs.insert(make_pair(coll_str, ObjectStore::Sequencer(coll_str)));
+  ObjectStore::Sequencer *osr = &(osrs.find(coll_str)->second);
+
+
+  coll_t c(coll_str);
+  hobject_t h(sobject_t(oid.substr(sep+1), 0));
+  t->write(c, h, offset, bl.length(), bl);
+
+  if (write_infos) {
+    bufferlist bl2;
+    for (uint64_t j = 0; j < 128; ++j) bl2.append(0);
+    coll_t meta("meta");
+    hobject_t info(sobject_t(string("info_")+coll_str, 0));
+    t->write(meta, info, 0, bl2.length(), bl2);
+  }
+
+  os->queue_transaction(
+    osr,
+    t,
+    new C_DeleteTransWrapper(t, on_applied),
+    on_commit);
+}
+
+void TestFileStoreBackend::read(
+  const string &oid,
+  uint64_t offset,
+  uint64_t length,
+  bufferlist *bl,
+  Context *on_complete)
+{
+  size_t sep = oid.find("/");
+  assert(sep != string::npos);
+  assert(sep + 1 < oid.size());
+  coll_t c(oid.substr(0, sep));
+  hobject_t h(sobject_t(oid.substr(sep+1), 0));
+  os->read(c, h, offset, length, *bl);
+  finisher.queue(on_complete);
+}
diff --git a/ceph/src/test/bench/testfilestore_backend.h b/ceph/src/test/bench/testfilestore_backend.h
new file mode 100644
index 00000000..2c3f8e6d
--- /dev/null
+++ b/ceph/src/test/bench/testfilestore_backend.h
@@ -0,0 +1,37 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef TESTFILESTOREBACKENDH
+#define TESTFILESTOREBACKENDH
+
+#include "common/Finisher.h"
+#include "backend.h"
+#include "include/Context.h"
+#include "os/ObjectStore.h"
+
+class TestFileStoreBackend : public Backend {
+  ObjectStore *os;
+  Finisher finisher;
+  map osrs;
+  const bool write_infos;
+
+public:
+  TestFileStoreBackend(ObjectStore *os, bool write_infos);
+  ~TestFileStoreBackend() {
+    finisher.stop();
+  }
+  void write(
+    const string &oid,
+    uint64_t offset,
+    const bufferlist &bl,
+    Context *on_applied,
+    Context *on_commit);
+
+  void read(
+    const string &oid,
+    uint64_t offset,
+    uint64_t length,
+    bufferlist *bl,
+    Context *on_complete);
+};
+
+#endif
diff --git a/ceph/src/test/bench/tp_bench.cc b/ceph/src/test/bench/tp_bench.cc
new file mode 100644
index 00000000..31a4db37
--- /dev/null
+++ b/ceph/src/test/bench/tp_bench.cc
@@ -0,0 +1,205 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "bencher.h"
+#include "rados_backend.h"
+#include "detailed_stat_collector.h"
+#include "distribution.h"
+#include "global/global_init.h"
+#include "common/WorkQueue.h"
+#include "common/Semaphore.h"
+#include "common/Finisher.h"
+
+namespace po = boost::program_options;
+using namespace std;
+class Queueable {
+public:
+  virtual void queue(unsigned *) = 0;
+  virtual void start() = 0;
+  virtual void stop() = 0;
+  virtual ~Queueable() {};
+};
+class Base : public Queueable {
+  DetailedStatCollector *col;
+  Semaphore *sem;
+public:
+  Base(DetailedStatCollector *col,
+       Semaphore *sem) : col(col), sem(sem) {}
+  void queue(unsigned *item) {
+    col->read_complete(*item);
+    sem->Put();
+    delete item;
+  }
+  void start() {}
+  void stop() {}
+};
+class WQWrapper : public Queueable {
+  boost::scoped_ptr > wq;
+  boost::scoped_ptr tp;
+public:
+  WQWrapper(ThreadPool::WorkQueue *wq, ThreadPool *tp):
+    wq(wq), tp(tp) {}
+  void queue(unsigned *item) { wq->queue(item); }
+  void start() { tp->start(); }
+  void stop() { tp->stop(); }
+};
+class FinisherWrapper : public Queueable {
+  class CB : public Context {
+    Queueable *next;
+    unsigned *item;
+  public:
+    CB(Queueable *next, unsigned *item) : next(next), item(item) {}
+    void finish(int) {
+      next->queue(item);
+    }
+  };
+  Finisher f;
+  Queueable *next;
+public:
+  FinisherWrapper(CephContext *cct, Queueable *next) :
+    f(cct), next(next) {}
+  void queue(unsigned *item) {
+    f.queue(new CB(next, item));
+  }
+  void start() { f.start(); }
+  void stop() { f.stop(); }
+};
+class PassAlong : public ThreadPool::WorkQueue {
+  Queueable *next;
+  list q;
+  bool _enqueue(unsigned *item) {
+    q.push_back(item);
+    return true;
+  }
+  void _dequeue(unsigned *item) { assert(0); }
+  unsigned *_dequeue() {
+    if (q.empty())
+      return 0;
+    unsigned *val = q.front();
+    q.pop_front();
+    return val;
+  }
+  void _process(unsigned *item) {
+    next->queue(item);
+  }
+  void _clear() { q.clear(); }
+  bool _empty() { return q.empty(); }
+public:
+  PassAlong(ThreadPool *tp, Queueable *next) :
+    ThreadPool::WorkQueue("TestQueue", 100, 100, tp), next(next) {}
+};
+
+int main(int argc, char **argv)
+{
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("num-threads", po::value()->default_value(10),
+     "set number of threads")
+    ("queue-size", po::value()->default_value(30),
+     "queue size")
+    ("num-items", po::value()->default_value(3000000),
+     "num items")
+    ("layers", po::value()->default_value(""),
+     "layer desc")
+    ;
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+    po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+  po::store(
+    parsed,
+    vm);
+  po::notify(vm);
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+    CODE_ENVIRONMENT_UTILITY,
+    CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  DetailedStatCollector col(1, new JSONFormatter, 0, &cout);
+  Semaphore sem;
+  for (unsigned i = 0; i < vm["queue-size"].as(); ++i)
+    sem.Put();
+
+  typedef list QQ;
+  QQ wqs;
+  wqs.push_back(
+    new Base(&col, &sem));
+  string layers(vm["layers"].as());
+  unsigned num = 0;
+  for (string::reverse_iterator i = layers.rbegin();
+       i != layers.rend(); ++i) {
+    stringstream ss;
+    ss << "Test " << num;
+    if (*i == 'q') {
+      ThreadPool *tp =
+	new ThreadPool(
+	  g_ceph_context, ss.str(), vm["num-threads"].as(), 0);
+      wqs.push_back(
+	new WQWrapper(
+	  new PassAlong(tp, wqs.back()),
+	  tp
+	  ));
+    } else if (*i == 'f') {
+      wqs.push_back(
+	new FinisherWrapper(
+	  g_ceph_context, wqs.back()));
+    }
+    ++num;
+  }
+
+  for (QQ::iterator i = wqs.begin();
+       i != wqs.end();
+       ++i) {
+    (*i)->start();
+  }
+
+  for (uint64_t i = 0; i < vm["num-items"].as(); ++i) {
+    sem.Get();
+    unsigned *item = new unsigned(col.next_seq());
+    col.start_read(*item, 1);
+    wqs.back()->queue(item);
+  }
+
+  for (QQ::iterator i = wqs.begin();
+       i != wqs.end();
+       ++i) {
+    (*i)->stop();
+  }
+  for (QQ::iterator i = wqs.begin(); i != wqs.end(); wqs.erase(i++)) {
+    delete *i;
+  }
+  return 0;
+}
diff --git a/ceph/src/test/bench_log.cc b/ceph/src/test/bench_log.cc
new file mode 100644
index 00000000..425f3986
--- /dev/null
+++ b/ceph/src/test/bench_log.cc
@@ -0,0 +1,71 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+#include "common/Thread.h"
+#include "common/debug.h"
+#include "common/Clock.h"
+#include "common/config.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+
+struct T : public Thread {
+  int num;
+  set myset;
+  map mymap;
+  T(int n) : num(n) {
+    myset.insert(123);
+    myset.insert(456);
+    mymap[1] = "foo";
+    mymap[10] = "bar";
+  }
+
+  void *entry() {
+    while (num-- > 0)
+      generic_dout(0) << "this is a typical log line.  set "
+		      << myset << " and map " << mymap << dendl;
+    return 0;
+  }
+};
+
+int main(int argc, const char **argv)
+{
+  int threads = atoi(argv[1]);
+  int num = atoi(argv[2]);
+
+  cout << threads << " threads, " << num << " lines per thread" << std::endl;
+
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_OSD, CODE_ENVIRONMENT_UTILITY, 0);
+
+  utime_t start = ceph_clock_now(NULL);
+
+  list ls;
+  for (int i=0; icreate();
+    ls.push_back(t);
+  }
+
+  for (int i=0; ijoin();
+    delete t;    
+  }
+
+  utime_t t = ceph_clock_now(NULL);
+  t -= start;
+  cout << " flushing.. " << t << " so far ..." << std::endl;
+
+  g_ceph_context->_log->flush();
+
+  utime_t end = ceph_clock_now(NULL);
+  utime_t dur = end - start;
+
+  cout << dur << std::endl;
+  return 0;
+}
diff --git a/ceph/src/test/bufferlist.cc b/ceph/src/test/bufferlist.cc
new file mode 100644
index 00000000..ff872389
--- /dev/null
+++ b/ceph/src/test/bufferlist.cc
@@ -0,0 +1,2195 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include "include/memory.h"
+#include 
+#include 
+#include 
+
+#include "include/buffer.h"
+#include "include/utime.h"
+#include "include/encoding.h"
+#include "common/environment.h"
+#include "common/Clock.h"
+#include "common/safe_io.h"
+
+#include "gtest/gtest.h"
+#include "stdlib.h"
+#include "fcntl.h"
+#include "sys/stat.h"
+
+#define MAX_TEST 1000000
+
+TEST(Buffer, constructors) {
+  bool ceph_buffer_track = get_env_bool("CEPH_BUFFER_TRACK");
+  unsigned len = 17;
+  //
+  // buffer::create
+  //
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    bufferptr ptr(buffer::create(len));
+    EXPECT_EQ(len, ptr.length());
+    if (ceph_buffer_track)
+      EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+  }
+  //
+  // buffer::claim_char
+  //
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    char* str = new char[len];
+    ::memset(str, 'X', len);
+    bufferptr ptr(buffer::claim_char(len, str));
+    if (ceph_buffer_track)
+      EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+    EXPECT_EQ(len, ptr.length());
+    EXPECT_EQ(str, ptr.c_str());
+    bufferptr clone = ptr.clone();
+    EXPECT_EQ(0, ::memcmp(clone.c_str(), ptr.c_str(), len));
+  }
+  //
+  // buffer::create_static
+  //
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    char* str = new char[len];
+    bufferptr ptr(buffer::create_static(len, str));
+    if (ceph_buffer_track)
+      EXPECT_EQ(0, buffer::get_total_alloc());
+    EXPECT_EQ(len, ptr.length());
+    EXPECT_EQ(str, ptr.c_str());
+    delete [] str;
+  }
+  //
+  // buffer::create_malloc
+  //
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    bufferptr ptr(buffer::create_malloc(len));
+    if (ceph_buffer_track)
+      EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+    EXPECT_EQ(len, ptr.length());
+    // this doesn't throw on my x86_64 wheezy box --sage
+    //EXPECT_THROW(buffer::create_malloc((unsigned)ULLONG_MAX), buffer::bad_alloc);
+  }
+  //
+  // buffer::claim_malloc
+  //
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    char* str = (char*)malloc(len);
+    ::memset(str, 'X', len);
+    bufferptr ptr(buffer::claim_malloc(len, str));
+    if (ceph_buffer_track)
+      EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+    EXPECT_EQ(len, ptr.length());
+    EXPECT_EQ(str, ptr.c_str());
+    bufferptr clone = ptr.clone();
+    EXPECT_EQ(0, ::memcmp(clone.c_str(), ptr.c_str(), len));
+  }
+  //
+  // buffer::copy
+  //
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    const std::string expected(len, 'X');
+    bufferptr ptr(buffer::copy(expected.c_str(), expected.size()));
+    if (ceph_buffer_track)
+      EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+    EXPECT_NE(expected.c_str(), ptr.c_str());
+    EXPECT_EQ(0, ::memcmp(expected.c_str(), ptr.c_str(), len));
+  }
+  //
+  // buffer::create_page_aligned
+  //
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    bufferptr ptr(buffer::create_page_aligned(len));
+    ::memset(ptr.c_str(), 'X', len);
+    if (ceph_buffer_track)
+      EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+    // doesn't throw on my x86_64 wheezy box --sage
+    //EXPECT_THROW(buffer::create_page_aligned((unsigned)ULLONG_MAX), buffer::bad_alloc);
+#ifndef DARWIN
+    ASSERT_TRUE(ptr.is_page_aligned());
+#endif // DARWIN 
+    bufferptr clone = ptr.clone();
+    EXPECT_EQ(0, ::memcmp(clone.c_str(), ptr.c_str(), len));
+  }
+#ifdef CEPH_HAVE_SPLICE
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+  {
+    // no fd
+    EXPECT_THROW(buffer::create_zero_copy(len, -1, NULL), buffer::error_code);
+
+    unsigned zc_len = 4;
+    ::unlink("testfile");
+    ::system("echo ABC > testfile");
+    int fd = ::open("testfile", O_RDONLY);
+    bufferptr ptr(buffer::create_zero_copy(zc_len, fd, NULL));
+    EXPECT_EQ(zc_len, ptr.length());
+    if (ceph_buffer_track)
+      EXPECT_EQ(zc_len, (unsigned)buffer::get_total_alloc());
+    ::close(fd);
+    ::unlink("testfile");
+  }
+#endif
+  if (ceph_buffer_track)
+    EXPECT_EQ(0, buffer::get_total_alloc());
+}
+
+TEST(BufferRaw, ostream) {
+  bufferptr ptr(1);
+  std::ostringstream stream;
+  stream << *ptr.get_raw();
+  EXPECT_GT(stream.str().size(), stream.str().find("buffer::raw("));
+  EXPECT_GT(stream.str().size(), stream.str().find("len 1 nref 1)"));
+}
+
+#ifdef CEPH_HAVE_SPLICE
+class TestRawPipe : public ::testing::Test {
+protected:
+  virtual void SetUp() {
+    len = 4;
+    ::unlink("testfile");
+    ::system("echo ABC > testfile");
+    fd = ::open("testfile", O_RDONLY);
+    assert(fd >= 0);
+  }
+  virtual void TearDown() {
+    ::close(fd);
+    ::unlink("testfile");
+  }
+  int fd;
+  unsigned len;
+};
+
+TEST_F(TestRawPipe, create_zero_copy) {
+  bufferptr ptr(buffer::create_zero_copy(len, fd, NULL));
+  EXPECT_EQ(len, ptr.length());
+  if (get_env_bool("CEPH_BUFFER_TRACK"))
+    EXPECT_EQ(len, (unsigned)buffer::get_total_alloc());
+}
+
+TEST_F(TestRawPipe, c_str_no_fd) {
+  EXPECT_THROW(bufferptr ptr(buffer::create_zero_copy(len, -1, NULL)),
+	       buffer::error_code);
+}
+
+TEST_F(TestRawPipe, c_str_basic) {
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, NULL));
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+  EXPECT_EQ(len, ptr.length());
+}
+
+TEST_F(TestRawPipe, c_str_twice) {
+  // make sure we're creating a copy of the data and not consuming it
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, NULL));
+  EXPECT_EQ(len, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+}
+
+TEST_F(TestRawPipe, c_str_basic_offset) {
+  loff_t offset = len - 1;
+  ::lseek(fd, offset, SEEK_SET);
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len - offset, fd, NULL));
+  EXPECT_EQ(len - offset, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "\n", len - offset));
+}
+
+TEST_F(TestRawPipe, c_str_dest_short) {
+  ::lseek(fd, 1, SEEK_SET);
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(2, fd, NULL));
+  EXPECT_EQ(2u, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "BC", 2));
+}
+
+TEST_F(TestRawPipe, c_str_source_short) {
+  ::lseek(fd, 1, SEEK_SET);
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, NULL));
+  EXPECT_EQ(len - 1, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "BC\n", len - 1));
+}
+
+TEST_F(TestRawPipe, c_str_explicit_zero_offset) {
+  int64_t offset = 0;
+  ::lseek(fd, 1, SEEK_SET);
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, &offset));
+  EXPECT_EQ(len, offset);
+  EXPECT_EQ(len, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "ABC\n", len));
+}
+
+TEST_F(TestRawPipe, c_str_explicit_positive_offset) {
+  int64_t offset = 1;
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len - offset, fd,
+						     &offset));
+  EXPECT_EQ(len, offset);
+  EXPECT_EQ(len - 1, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "BC\n", len - 1));
+}
+
+TEST_F(TestRawPipe, c_str_explicit_positive_empty_result) {
+  int64_t offset = len;
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len - offset, fd,
+						     &offset));
+  EXPECT_EQ(len, offset);
+  EXPECT_EQ(0u, ptr.length());
+}
+
+TEST_F(TestRawPipe, c_str_source_short_explicit_offset) {
+  int64_t offset = 1;
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(len, fd, &offset));
+  EXPECT_EQ(len, offset);
+  EXPECT_EQ(len - 1, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "BC\n", len - 1));
+}
+
+TEST_F(TestRawPipe, c_str_dest_short_explicit_offset) {
+  int64_t offset = 1;
+  bufferptr ptr = bufferptr(buffer::create_zero_copy(2, fd, &offset));
+  EXPECT_EQ(3, offset);
+  EXPECT_EQ(2u, ptr.length());
+  EXPECT_EQ(0, memcmp(ptr.c_str(), "BC", 2));
+}
+
+TEST_F(TestRawPipe, buffer_list_read_fd_zero_copy) {
+  bufferlist bl;
+  EXPECT_EQ(-EBADF, bl.read_fd_zero_copy(-1, len));
+  bl = bufferlist();
+  EXPECT_EQ(0, bl.read_fd_zero_copy(fd, len));
+  EXPECT_EQ(len, bl.length());
+  EXPECT_EQ(0u, bl.buffers().front().unused_tail_length());
+  EXPECT_EQ(1u, bl.buffers().size());
+  EXPECT_EQ(len, bl.buffers().front().raw_length());
+  EXPECT_EQ(0, memcmp(bl.c_str(), "ABC\n", len));
+  EXPECT_TRUE(bl.can_zero_copy());
+}
+
+TEST_F(TestRawPipe, buffer_list_write_fd_zero_copy) {
+  ::unlink("testfile_out");
+  bufferlist bl;
+  EXPECT_EQ(0, bl.read_fd_zero_copy(fd, len));
+  EXPECT_TRUE(bl.can_zero_copy());
+  int out_fd = ::open("testfile_out", O_RDWR|O_CREAT|O_TRUNC, 0600);
+  EXPECT_EQ(0, bl.write_fd_zero_copy(out_fd));
+  struct stat st;
+  memset(&st, 0, sizeof(st));
+  EXPECT_EQ(0, ::stat("testfile_out", &st));
+  EXPECT_EQ(len, st.st_size);
+  char buf[len + 1];
+  EXPECT_EQ((int)len, safe_read(out_fd, buf, len + 1));
+  EXPECT_EQ(0, memcmp(buf, "ABC\n", len));
+  ::close(out_fd);
+  ::unlink("testfile_out");
+}
+#endif // CEPH_HAVE_SPLICE
+
+//                                     
+// +-----------+                +-----+
+// |           |                |     |
+// |  offset   +----------------+     |
+// |           |                |     |
+// |  length   +----            |     |
+// |           |    \-------    |     |
+// +-----------+            \---+     |
+// |   ptr     |                +-----+
+// +-----------+                | raw |
+//                              +-----+
+//
+TEST(BufferPtr, constructors) {
+  unsigned len = 17;
+  //
+  // ptr::ptr()
+  //
+  {
+    buffer::ptr ptr;
+    EXPECT_FALSE(ptr.have_raw());
+    EXPECT_EQ((unsigned)0, ptr.offset());
+    EXPECT_EQ((unsigned)0, ptr.length());
+  }
+  //
+  // ptr::ptr(raw *r)
+  //
+  {
+    bufferptr ptr(buffer::create(len));
+    EXPECT_TRUE(ptr.have_raw());
+    EXPECT_EQ((unsigned)0, ptr.offset());
+    EXPECT_EQ(len, ptr.length());
+    EXPECT_EQ(ptr.raw_length(), ptr.length());
+    EXPECT_EQ(1, ptr.raw_nref());
+  }
+  //
+  // ptr::ptr(unsigned l)
+  //
+  {
+    bufferptr ptr(len);
+    EXPECT_TRUE(ptr.have_raw());
+    EXPECT_EQ((unsigned)0, ptr.offset());
+    EXPECT_EQ(len, ptr.length());
+    EXPECT_EQ(1, ptr.raw_nref());
+  }
+  //
+  // ptr(const char *d, unsigned l)
+  //
+  {
+    const std::string str(len, 'X');
+    bufferptr ptr(str.c_str(), len);
+    EXPECT_TRUE(ptr.have_raw());
+    EXPECT_EQ((unsigned)0, ptr.offset());
+    EXPECT_EQ(len, ptr.length());
+    EXPECT_EQ(1, ptr.raw_nref());
+    EXPECT_EQ(0, ::memcmp(str.c_str(), ptr.c_str(), len));
+  }
+  //
+  // ptr(const ptr& p)
+  //
+  {
+    const std::string str(len, 'X');
+    bufferptr original(str.c_str(), len);
+    bufferptr ptr(original);
+    EXPECT_TRUE(ptr.have_raw());
+    EXPECT_EQ(original.get_raw(), ptr.get_raw());
+    EXPECT_EQ(2, ptr.raw_nref());
+    EXPECT_EQ(0, ::memcmp(original.c_str(), ptr.c_str(), len));
+  }
+  //
+  // ptr(const ptr& p, unsigned o, unsigned l)
+  //
+  {
+    const std::string str(len, 'X');
+    bufferptr original(str.c_str(), len);
+    bufferptr ptr(original, 0, 0);
+    EXPECT_TRUE(ptr.have_raw());
+    EXPECT_EQ(original.get_raw(), ptr.get_raw());
+    EXPECT_EQ(2, ptr.raw_nref());
+    EXPECT_EQ(0, ::memcmp(original.c_str(), ptr.c_str(), len));
+    EXPECT_THROW(bufferptr(original, 0, original.length() + 1), FailedAssertion);
+    EXPECT_THROW(bufferptr(bufferptr(), 0, 0), FailedAssertion);
+  }
+}
+
+TEST(BufferPtr, assignment) {
+  unsigned len = 17;
+  //
+  // override a bufferptr set with the same raw
+  //
+  {
+    bufferptr original(len);
+    bufferptr same_raw(original.get_raw());
+    unsigned offset = 5;
+    unsigned length = len - offset;
+    original.set_offset(offset);
+    original.set_length(length);
+    same_raw = original;
+    ASSERT_EQ(2, original.raw_nref());
+    ASSERT_EQ(same_raw.get_raw(), original.get_raw());
+    ASSERT_EQ(same_raw.offset(), original.offset());
+    ASSERT_EQ(same_raw.length(), original.length());
+  }
+
+  //
+  // self assignment is a noop
+  //
+  {
+    bufferptr original(len);
+    original = original;
+    ASSERT_EQ(1, original.raw_nref());
+    ASSERT_EQ((unsigned)0, original.offset());
+    ASSERT_EQ(len, original.length());
+  }
+  
+  //
+  // a copy points to the same raw
+  //
+  {
+    bufferptr original(len);
+    unsigned offset = 5;
+    unsigned length = len - offset;
+    original.set_offset(offset);
+    original.set_length(length);
+    bufferptr ptr;
+    ptr = original;
+    ASSERT_EQ(2, original.raw_nref());
+    ASSERT_EQ(ptr.get_raw(), original.get_raw());
+    ASSERT_EQ(original.offset(), ptr.offset());
+    ASSERT_EQ(original.length(), ptr.length());
+  }
+}
+
+TEST(BufferPtr, clone) {
+  unsigned len = 17;
+  bufferptr ptr(len);
+  ::memset(ptr.c_str(), 'X', len);
+  bufferptr clone = ptr.clone();
+  EXPECT_EQ(0, ::memcmp(clone.c_str(), ptr.c_str(), len));
+}
+
+TEST(BufferPtr, swap) {
+  unsigned len = 17;
+
+  bufferptr ptr1(len);
+  ::memset(ptr1.c_str(), 'X', len);
+  unsigned ptr1_offset = 4;
+  ptr1.set_offset(ptr1_offset);
+  unsigned ptr1_length = 3;
+  ptr1.set_length(ptr1_length);
+
+  bufferptr ptr2(len);
+  ::memset(ptr2.c_str(), 'Y', len);
+  unsigned ptr2_offset = 5;
+  ptr2.set_offset(ptr2_offset);
+  unsigned ptr2_length = 7;
+  ptr2.set_length(ptr2_length);
+
+  ptr1.swap(ptr2);
+
+  EXPECT_EQ(ptr2_length, ptr1.length());
+  EXPECT_EQ(ptr2_offset, ptr1.offset());
+  EXPECT_EQ('Y', ptr1[0]);
+
+  EXPECT_EQ(ptr1_length, ptr2.length());
+  EXPECT_EQ(ptr1_offset, ptr2.offset());
+  EXPECT_EQ('X', ptr2[0]);
+}
+
+TEST(BufferPtr, release) {
+  unsigned len = 17;
+
+  bufferptr ptr1(len);
+  {
+    bufferptr ptr2(ptr1);
+    EXPECT_EQ(2, ptr1.raw_nref());
+  }
+  EXPECT_EQ(1, ptr1.raw_nref());
+}
+
+TEST(BufferPtr, have_raw) {
+  {
+    bufferptr ptr;
+    EXPECT_FALSE(ptr.have_raw());
+  }
+  {
+    bufferptr ptr(1);
+    EXPECT_TRUE(ptr.have_raw());
+  }
+}
+
+TEST(BufferPtr, at_buffer_head) {
+  bufferptr ptr(2);
+  EXPECT_TRUE(ptr.at_buffer_head());
+  ptr.set_offset(1);
+  EXPECT_FALSE(ptr.at_buffer_head());
+}
+
+TEST(BufferPtr, at_buffer_tail) {
+  bufferptr ptr(2);
+  EXPECT_TRUE(ptr.at_buffer_tail());
+  ptr.set_length(1);
+  EXPECT_FALSE(ptr.at_buffer_tail());
+}
+
+TEST(BufferPtr, is_n_page_sized) {
+  {
+    bufferptr ptr(CEPH_PAGE_SIZE);
+    EXPECT_TRUE(ptr.is_n_page_sized());
+  }
+  {
+    bufferptr ptr(1);
+    EXPECT_FALSE(ptr.is_n_page_sized());
+  }
+}
+
+TEST(BufferPtr, accessors) {
+  unsigned len = 17;
+  bufferptr ptr(len);
+  ptr.c_str()[0] = 'X';
+  ptr[1] = 'Y';
+  const bufferptr const_ptr(ptr);
+
+  EXPECT_NE((void*)NULL, (void*)ptr.get_raw());
+  EXPECT_EQ('X', ptr.c_str()[0]);
+  {
+    bufferptr ptr;
+    EXPECT_THROW(ptr.c_str(), FailedAssertion);
+    EXPECT_THROW(ptr[0], FailedAssertion);
+  }
+  EXPECT_EQ('X', const_ptr.c_str()[0]);
+  {
+    const bufferptr const_ptr;
+    EXPECT_THROW(const_ptr.c_str(), FailedAssertion);
+    EXPECT_THROW(const_ptr[0], FailedAssertion);
+  }
+  EXPECT_EQ(len, const_ptr.length());
+  EXPECT_EQ((unsigned)0, const_ptr.offset());
+  EXPECT_EQ((unsigned)0, const_ptr.start());
+  EXPECT_EQ(len, const_ptr.end());
+  EXPECT_EQ(len, const_ptr.end());
+  {
+    bufferptr ptr(len);
+    unsigned unused = 1;
+    ptr.set_length(ptr.length() - unused);
+    EXPECT_EQ(unused, ptr.unused_tail_length());
+  }
+  {
+    bufferptr ptr;
+    EXPECT_EQ((unsigned)0, ptr.unused_tail_length());
+  }
+  EXPECT_THROW(ptr[len], FailedAssertion);
+  EXPECT_THROW(const_ptr[len], FailedAssertion);
+  {
+    const bufferptr const_ptr;
+    EXPECT_THROW(const_ptr.raw_c_str(), FailedAssertion);
+    EXPECT_THROW(const_ptr.raw_length(), FailedAssertion);
+    EXPECT_THROW(const_ptr.raw_nref(), FailedAssertion);
+  }
+  EXPECT_NE((const char *)NULL, const_ptr.raw_c_str());
+  EXPECT_EQ(len, const_ptr.raw_length());
+  EXPECT_EQ(2, const_ptr.raw_nref());
+  {
+    bufferptr ptr(len);
+    unsigned wasted = 1;
+    ptr.set_length(ptr.length() - wasted * 2);
+    ptr.set_offset(wasted);
+    EXPECT_EQ(wasted * 2, ptr.wasted());
+  }
+}
+
+TEST(BufferPtr, cmp) {
+  bufferptr empty;
+  bufferptr a("A", 1);
+  bufferptr ab("AB", 2);
+  bufferptr af("AF", 2);
+  bufferptr acc("ACC", 3);
+  EXPECT_GE(-1, empty.cmp(a));
+  EXPECT_LE(1, a.cmp(empty));
+  EXPECT_GE(-1, a.cmp(ab));
+  EXPECT_LE(1, ab.cmp(a));
+  EXPECT_EQ(0, ab.cmp(ab));
+  EXPECT_GE(-1, ab.cmp(af));
+  EXPECT_LE(1, af.cmp(ab));
+  EXPECT_GE(-1, acc.cmp(af));
+  EXPECT_LE(1, af.cmp(acc));
+}
+
+TEST(BufferPtr, is_zero) {
+  char str[2] = { '\0', 'X' };
+  {
+    const bufferptr ptr(buffer::create_static(2, str));
+    EXPECT_FALSE(ptr.is_zero());
+  }
+  {
+    const bufferptr ptr(buffer::create_static(1, str));
+    EXPECT_TRUE(ptr.is_zero());
+  }
+}
+
+TEST(BufferPtr, copy_out) {
+  {
+    const bufferptr ptr;
+    EXPECT_THROW(ptr.copy_out((unsigned)0, (unsigned)0, NULL), FailedAssertion);
+  }
+  {
+    char in[] = "ABC";
+    const bufferptr ptr(buffer::create_static(strlen(in), in));
+    EXPECT_THROW(ptr.copy_out((unsigned)0, strlen(in) + 1, NULL), buffer::end_of_buffer);
+    EXPECT_THROW(ptr.copy_out(strlen(in) + 1, (unsigned)0, NULL), buffer::end_of_buffer);
+    char out[1] = { 'X' };
+    ptr.copy_out((unsigned)1, (unsigned)1, out);
+    EXPECT_EQ('B', out[0]);
+  }
+}
+
+TEST(BufferPtr, copy_in) {
+  {
+    bufferptr ptr;
+    EXPECT_THROW(ptr.copy_in((unsigned)0, (unsigned)0, NULL), FailedAssertion);
+  }
+  {
+    char in[] = "ABCD";
+    bufferptr ptr(2);
+    EXPECT_THROW(ptr.copy_in((unsigned)0, strlen(in) + 1, NULL), FailedAssertion);
+    EXPECT_THROW(ptr.copy_in(strlen(in) + 1, (unsigned)0, NULL), FailedAssertion);
+    ptr.copy_in((unsigned)0, (unsigned)2, in);
+    EXPECT_EQ(in[0], ptr[0]);
+    EXPECT_EQ(in[1], ptr[1]);
+  }
+}
+
+TEST(BufferPtr, append) {
+  {
+    bufferptr ptr;
+    EXPECT_THROW(ptr.append('A'), FailedAssertion);
+    EXPECT_THROW(ptr.append("B", (unsigned)1), FailedAssertion);
+  }
+  {
+    bufferptr ptr(2);
+    EXPECT_THROW(ptr.append('A'), FailedAssertion);
+    EXPECT_THROW(ptr.append("B", (unsigned)1), FailedAssertion);
+    ptr.set_length(0);
+    ptr.append('A');
+    EXPECT_EQ((unsigned)1, ptr.length());
+    EXPECT_EQ('A', ptr[0]);
+    ptr.append("B", (unsigned)1);
+    EXPECT_EQ((unsigned)2, ptr.length());
+    EXPECT_EQ('B', ptr[1]);
+  }
+}
+
+TEST(BufferPtr, zero) {
+  char str[] = "XXXX";
+  bufferptr ptr(buffer::create_static(strlen(str), str));
+  EXPECT_THROW(ptr.zero(ptr.length() + 1, 0), FailedAssertion);
+  ptr.zero(1, 1);
+  EXPECT_EQ('X', ptr[0]);
+  EXPECT_EQ('\0', ptr[1]);
+  EXPECT_EQ('X', ptr[2]);
+  ptr.zero();
+  EXPECT_EQ('\0', ptr[0]);
+}
+
+TEST(BufferPtr, ostream) {
+  {
+    bufferptr ptr;
+    std::ostringstream stream;
+    stream << ptr;
+    EXPECT_GT(stream.str().size(), stream.str().find("buffer:ptr(0~0 no raw"));
+  }
+  {
+    char str[] = "XXXX";
+    bufferptr ptr(buffer::create_static(strlen(str), str));
+    std::ostringstream stream;
+    stream << ptr;
+    EXPECT_GT(stream.str().size(), stream.str().find("len 4 nref 1)"));
+  }  
+}
+
+//
+//                                             +---------+
+//                                             | +-----+ |
+//    list              ptr                    | |     | |
+// +----------+       +-----+                  | |     | |
+// | append_  >------->     >-------------------->     | |
+// |  buffer  |       +-----+                  | |     | |
+// +----------+                        ptr     | |     | |
+// |   _len   |      list            +-----+   | |     | |
+// +----------+    +------+     ,--->+     >----->     | |
+// | _buffers >---->      >-----     +-----+   | +-----+ |
+// +----------+    +----^-+     \      ptr     |   raw   |
+// |  last_p  |        /         `-->+-----+   | +-----+ |
+// +--------+-+       /              +     >----->     | |
+//          |       ,-          ,--->+-----+   | |     | |
+//          |      /        ,---               | |     | |
+//          |     /     ,---                   | |     | |
+//        +-v--+-^--+--^+-------+              | |     | |
+//        | bl | ls | p | p_off >--------------->|     | |
+//        +----+----+-----+-----+              | +-----+ |
+//        |               | off >------------->|   raw   |
+//        +---------------+-----+              |         |
+//              iterator                       +---------+
+//
+TEST(BufferListIterator, constructors) {
+  //
+  // iterator()
+  //
+  {
+    buffer::list::iterator i;
+    EXPECT_EQ((unsigned)0, i.get_off());
+  }
+
+  //
+  // iterator(list *l, unsigned o=0)
+  //
+  {
+    bufferlist bl;
+    bl.append("ABC", 3);
+
+    {
+      bufferlist::iterator i(&bl);
+      EXPECT_EQ((unsigned)0, i.get_off());
+      EXPECT_EQ('A', *i);
+    }
+    {
+      bufferlist::iterator i(&bl, 1);
+      EXPECT_EQ('B', *i);
+      EXPECT_EQ((unsigned)2, i.get_remaining());
+    }
+  }
+
+  //
+  // iterator(list *l, unsigned o, std::list::iterator ip, unsigned po)
+  // not tested because of http://tracker.ceph.com/issues/4101
+
+  //
+  // iterator(const iterator& other)
+  //
+  {
+    bufferlist bl;
+    bl.append("ABC", 3);
+    bufferlist::iterator i(&bl, 1);
+    bufferlist::iterator j(i);
+    EXPECT_EQ(*i, *j);
+    ++j;
+    EXPECT_NE(*i, *j);
+    EXPECT_EQ('B', *i);
+    EXPECT_EQ('C', *j);
+    bl.c_str()[1] = 'X';
+    j.advance(-1);
+    EXPECT_EQ('X', *j);
+  }
+}
+
+TEST(BufferListIterator, operator_equal) {
+  bufferlist bl;
+  bl.append("ABC", 3);
+  bufferlist::iterator i(&bl, 1);
+
+  i = i;
+  EXPECT_EQ('B', *i);
+  bufferlist::iterator j;
+  j = i;
+  EXPECT_EQ('B', *j);
+}
+
+TEST(BufferListIterator, get_off) {
+  bufferlist bl;
+  bl.append("ABC", 3);
+  bufferlist::iterator i(&bl, 1);
+  EXPECT_EQ((unsigned)1, i.get_off());
+}
+
+TEST(BufferListIterator, get_remaining) {
+  bufferlist bl;
+  bl.append("ABC", 3);
+  bufferlist::iterator i(&bl, 1);
+  EXPECT_EQ((unsigned)2, i.get_remaining());
+}
+
+TEST(BufferListIterator, end) {
+  bufferlist bl;
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_TRUE(i.end());
+  }
+  bl.append("ABC", 3);
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_FALSE(i.end());
+  }
+}
+
+TEST(BufferListIterator, advance) {
+  bufferlist bl;
+  const std::string one("ABC");
+  bl.append(bufferptr(one.c_str(), one.size()));
+  const std::string two("DEF");
+  bl.append(bufferptr(two.c_str(), two.size()));
+
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer);
+  }
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_THROW(i.advance(-1), buffer::end_of_buffer);
+  }
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_EQ('A', *i);
+    i.advance(1);
+    EXPECT_EQ('B', *i);
+    i.advance(3);
+    EXPECT_EQ('E', *i);
+    i.advance(-3);
+    EXPECT_EQ('B', *i);
+    i.advance(-1);
+    EXPECT_EQ('A', *i);
+  }
+}
+
+TEST(BufferListIterator, seek) {
+  bufferlist bl;
+  bl.append("ABC", 3);
+  bufferlist::iterator i(&bl, 1);
+  EXPECT_EQ('B', *i);
+  i.seek(2);
+  EXPECT_EQ('C', *i);
+}
+
+TEST(BufferListIterator, operator_star) {
+  bufferlist bl;
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_THROW(*i, buffer::end_of_buffer);
+  }
+  bl.append("ABC", 3);
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_EQ('A', *i);
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer);
+    EXPECT_THROW(*i, buffer::end_of_buffer);
+  }
+}
+
+TEST(BufferListIterator, operator_plus_plus) {
+  bufferlist bl;
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_THROW(++i, buffer::end_of_buffer);
+  }
+  bl.append("ABC", 3);
+  {
+    bufferlist::iterator i(&bl);
+    ++i;
+    EXPECT_EQ('B', *i);
+  }  
+}
+
+TEST(BufferListIterator, get_current_ptr) {
+  bufferlist bl;
+  {
+    bufferlist::iterator i(&bl);
+    EXPECT_THROW(++i, buffer::end_of_buffer);
+  }
+  bl.append("ABC", 3);
+  {
+    bufferlist::iterator i(&bl, 1);
+    const buffer::ptr ptr = i.get_current_ptr();
+    EXPECT_EQ('B', ptr[0]);
+    EXPECT_EQ((unsigned)1, ptr.offset());
+    EXPECT_EQ((unsigned)2, ptr.length());
+  }  
+}
+
+TEST(BufferListIterator, copy) {
+  bufferlist bl;
+  const char *expected = "ABC";
+  bl.append(expected, 3);
+  //
+  // void copy(unsigned len, char *dest);
+  //
+  {
+    char* copy = (char*)malloc(3);
+    ::memset(copy, 'X', 3);
+    bufferlist::iterator i(&bl);
+    //
+    // demonstrates that it seeks back to offset if p == ls->end()
+    //
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer); 
+    i.copy(2, copy);
+    EXPECT_EQ(0, ::memcmp(copy, expected, 2));
+    EXPECT_EQ('X', copy[2]);
+    i.seek(0);
+    i.copy(3, copy);
+    EXPECT_EQ(0, ::memcmp(copy, expected, 3));
+  }
+  //
+  // void buffer::list::iterator::copy(unsigned len, ptr &dest)
+  //
+  {
+    bufferptr ptr;
+    bufferlist::iterator i(&bl);
+    i.copy(2, ptr);
+    EXPECT_EQ((unsigned)2, ptr.length());
+    EXPECT_EQ('A', ptr[0]);
+    EXPECT_EQ('B', ptr[1]);
+  }
+  //
+  // void buffer::list::iterator::copy(unsigned len, list &dest)
+  //
+  {
+    bufferlist copy;
+    bufferlist::iterator i(&bl);
+    //
+    // demonstrates that it seeks back to offset if p == ls->end()
+    //
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer); 
+    i.copy(2, copy);
+    EXPECT_EQ(0, ::memcmp(copy.c_str(), expected, 2));
+    i.seek(0);
+    i.copy(3, copy);
+    EXPECT_EQ('A', copy[0]);
+    EXPECT_EQ('B', copy[1]);
+    EXPECT_EQ('A', copy[2]);
+    EXPECT_EQ('B', copy[3]);
+    EXPECT_EQ('C', copy[4]);
+    EXPECT_EQ((unsigned)(2 + 3), copy.length());
+  }
+  //
+  // void buffer::list::iterator::copy_all(list &dest)
+  //
+  {
+    bufferlist copy;
+    bufferlist::iterator i(&bl);
+    //
+    // demonstrates that it seeks back to offset if p == ls->end()
+    //
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer); 
+    i.copy_all(copy);
+    EXPECT_EQ('A', copy[0]);
+    EXPECT_EQ('B', copy[1]);
+    EXPECT_EQ('C', copy[2]);
+    EXPECT_EQ((unsigned)3, copy.length());
+  }
+  //
+  // void copy(unsigned len, std::string &dest)
+  //
+  {
+    std::string copy;
+    bufferlist::iterator i(&bl);
+    //
+    // demonstrates that it seeks back to offset if p == ls->end()
+    //
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer); 
+    i.copy(2, copy);
+    EXPECT_EQ(0, ::memcmp(copy.c_str(), expected, 2));
+    i.seek(0);
+    i.copy(3, copy);
+    EXPECT_EQ('A', copy[0]);
+    EXPECT_EQ('B', copy[1]);
+    EXPECT_EQ('A', copy[2]);
+    EXPECT_EQ('B', copy[3]);
+    EXPECT_EQ('C', copy[4]);
+    EXPECT_EQ((unsigned)(2 + 3), copy.length());
+  }
+}
+
+TEST(BufferListIterator, copy_in) {
+  bufferlist bl;
+  const char *existing = "XXX";
+  bl.append(existing, 3);
+  //
+  // void buffer::list::iterator::copy_in(unsigned len, const char *src)
+  //
+  {
+    bufferlist::iterator i(&bl);
+    //
+    // demonstrates that it seeks back to offset if p == ls->end()
+    //
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer); 
+    const char *expected = "ABC";
+    i.copy_in(3, expected);
+    EXPECT_EQ(0, ::memcmp(bl.c_str(), expected, 3));
+    EXPECT_EQ('A', bl[0]);
+    EXPECT_EQ('B', bl[1]);
+    EXPECT_EQ('C', bl[2]);
+    EXPECT_EQ((unsigned)3, bl.length());
+  }
+  //
+  // void buffer::list::iterator::copy_in(unsigned len, const list& otherl)
+  //
+  {
+    bufferlist::iterator i(&bl);
+    //
+    // demonstrates that it seeks back to offset if p == ls->end()
+    //
+    EXPECT_THROW(i.advance(200), buffer::end_of_buffer); 
+    bufferlist expected;
+    expected.append("ABC", 3);
+    i.copy_in(3, expected);
+    EXPECT_EQ(0, ::memcmp(bl.c_str(), expected.c_str(), 3));
+    EXPECT_EQ('A', bl[0]);
+    EXPECT_EQ('B', bl[1]);
+    EXPECT_EQ('C', bl[2]);
+    EXPECT_EQ((unsigned)3, bl.length());
+  }
+}
+
+TEST(BufferList, constructors) {
+  //
+  // list()
+  //
+  {
+    bufferlist bl;
+    ASSERT_EQ((unsigned)0, bl.length());
+  }
+  //
+  // list(unsigned prealloc)
+  //
+  {
+    bufferlist bl(1);
+    ASSERT_EQ((unsigned)0, bl.length());
+    bl.append('A');
+    ASSERT_EQ('A', bl[0]);
+  }
+  //
+  // list(const list& other)
+  //
+  {
+    bufferlist bl(1);
+    bl.append('A');
+    ASSERT_EQ('A', bl[0]);
+    bufferlist copy(bl);
+    ASSERT_EQ('A', copy[0]);
+  }
+}
+
+TEST(BufferList, operator_equal) {
+  bufferlist bl;
+  bl.append("ABC", 3);
+  {
+    std::string dest;
+    bl.copy(1, 1, dest);
+    ASSERT_EQ('B', dest[0]);
+  }
+  bufferlist copy;
+  copy = bl;
+  {
+    std::string dest;
+    copy.copy(1, 1, dest);
+    ASSERT_EQ('B', dest[0]);
+  }
+}
+
+TEST(BufferList, buffers) {
+  bufferlist bl;
+  ASSERT_EQ((unsigned)0, bl.buffers().size());
+  bl.append('A');
+  ASSERT_EQ((unsigned)1, bl.buffers().size());
+}
+
+TEST(BufferList, swap) {
+  bufferlist b1;
+  b1.append('A');
+
+  bufferlist b2;
+  b2.append('B');
+
+  b1.swap(b2);
+
+  std::string s1;
+  b1.copy(0, 1, s1);
+  ASSERT_EQ('B', s1[0]);
+
+  std::string s2;
+  b2.copy(0, 1, s2);
+  ASSERT_EQ('A', s2[0]);
+}
+
+TEST(BufferList, length) {
+  bufferlist bl;
+  ASSERT_EQ((unsigned)0, bl.length());
+  bl.append('A');
+  ASSERT_EQ((unsigned)1, bl.length());
+}
+
+TEST(BufferList, contents_equal) {
+  //
+  // A BB
+  // AB B
+  //
+  bufferlist bl1;
+  bl1.append("A");
+  bl1.append("BB");
+  bufferlist bl2;
+  ASSERT_FALSE(bl1.contents_equal(bl2)); // different length
+  bl2.append("AB");
+  bl2.append("B");
+  ASSERT_TRUE(bl1.contents_equal(bl2)); // same length same content
+  //
+  // ABC
+  //
+  bufferlist bl3;
+  bl3.append("ABC");
+  ASSERT_FALSE(bl1.contents_equal(bl3)); // same length different content
+}
+
+TEST(BufferList, is_page_aligned) {
+  {
+    bufferlist bl;
+    EXPECT_TRUE(bl.is_page_aligned());
+  }
+  {
+    bufferlist bl;
+    bufferptr ptr(buffer::create_page_aligned(2));
+    ptr.set_offset(1);
+    ptr.set_length(1);
+    bl.append(ptr);
+    EXPECT_FALSE(bl.is_page_aligned());
+    bl.rebuild_page_aligned();
+    EXPECT_TRUE(bl.is_page_aligned());
+  }
+  {
+    bufferlist bl;
+    bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
+    ptr.set_offset(1);
+    ptr.set_length(CEPH_PAGE_SIZE);
+    bl.append(ptr);
+    EXPECT_FALSE(bl.is_page_aligned());
+    bl.rebuild_page_aligned();
+    EXPECT_TRUE(bl.is_page_aligned());
+  }
+}
+
+TEST(BufferList, is_n_page_sized) {
+  {
+    bufferlist bl;
+    EXPECT_TRUE(bl.is_n_page_sized());
+  }
+  {
+    bufferlist bl;
+    bl.append_zero(1);
+    EXPECT_FALSE(bl.is_n_page_sized());
+  }
+  {
+    bufferlist bl;
+    bl.append_zero(CEPH_PAGE_SIZE);
+    EXPECT_TRUE(bl.is_n_page_sized());
+  }
+}
+
+TEST(BufferList, is_zero) {
+  {
+    bufferlist bl;
+    EXPECT_TRUE(bl.is_zero());
+  }
+  {
+    bufferlist bl;
+    bl.append('A');
+    EXPECT_FALSE(bl.is_zero());
+  }
+  {
+    bufferlist bl;
+    bl.append_zero(1);
+    EXPECT_TRUE(bl.is_zero());
+  }
+}
+
+TEST(BufferList, clear) {
+  bufferlist bl;
+  unsigned len = 17;
+  bl.append_zero(len);
+  bl.clear();
+  EXPECT_EQ((unsigned)0, bl.length());
+  EXPECT_EQ((unsigned)0, bl.buffers().size());
+}
+
+TEST(BufferList, push_front) {
+  //
+  // void push_front(ptr& bp)
+  //
+  {
+    bufferlist bl;
+    bufferptr ptr;
+    bl.push_front(ptr);
+    EXPECT_EQ((unsigned)0, bl.length());
+    EXPECT_EQ((unsigned)0, bl.buffers().size());
+  }
+  unsigned len = 17;
+  {
+    bufferlist bl;
+    bl.append('A');
+    bufferptr ptr(len);
+    ptr.c_str()[0] = 'B';
+    bl.push_front(ptr);
+    EXPECT_EQ((unsigned)(1 + len), bl.length());
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_EQ('B', bl.buffers().front()[0]);
+    EXPECT_EQ(ptr.get_raw(), bl.buffers().front().get_raw());
+  }
+  //
+  // void push_front(raw *r)
+  //
+  {
+    bufferlist bl;
+    bl.append('A');
+    bufferptr ptr(len);
+    ptr.c_str()[0] = 'B';
+    bl.push_front(ptr.get_raw());
+    EXPECT_EQ((unsigned)(1 + len), bl.length());
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_EQ('B', bl.buffers().front()[0]);
+    EXPECT_EQ(ptr.get_raw(), bl.buffers().front().get_raw());
+  }
+}
+
+TEST(BufferList, push_back) {
+  //
+  // void push_back(ptr& bp)
+  //
+  {
+    bufferlist bl;
+    bufferptr ptr;
+    bl.push_back(ptr);
+    EXPECT_EQ((unsigned)0, bl.length());
+    EXPECT_EQ((unsigned)0, bl.buffers().size());
+  }
+  unsigned len = 17;
+  {
+    bufferlist bl;
+    bl.append('A');
+    bufferptr ptr(len);
+    ptr.c_str()[0] = 'B';
+    bl.push_back(ptr);
+    EXPECT_EQ((unsigned)(1 + len), bl.length());
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_EQ('B', bl.buffers().back()[0]);
+    EXPECT_EQ(ptr.get_raw(), bl.buffers().back().get_raw());
+  }
+  //
+  // void push_back(raw *r)
+  //
+  {
+    bufferlist bl;
+    bl.append('A');
+    bufferptr ptr(len);
+    ptr.c_str()[0] = 'B';
+    bl.push_back(ptr.get_raw());
+    EXPECT_EQ((unsigned)(1 + len), bl.length());
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_EQ('B', bl.buffers().back()[0]);
+    EXPECT_EQ(ptr.get_raw(), bl.buffers().back().get_raw());
+  }
+}
+
+TEST(BufferList, is_contiguous) {
+  bufferlist bl;
+  EXPECT_TRUE(bl.is_contiguous());
+  EXPECT_EQ((unsigned)0, bl.buffers().size());
+  bl.append('A');  
+  EXPECT_TRUE(bl.is_contiguous());
+  EXPECT_EQ((unsigned)1, bl.buffers().size());
+  bufferptr ptr(1);
+  bl.push_back(ptr);
+  EXPECT_FALSE(bl.is_contiguous());
+  EXPECT_EQ((unsigned)2, bl.buffers().size());
+}
+
+TEST(BufferList, rebuild) {
+  {
+    bufferlist bl;
+    bufferptr ptr(buffer::create_page_aligned(2));
+    ptr.set_offset(1);
+    ptr.set_length(1);
+    bl.append(ptr);
+    EXPECT_FALSE(bl.is_page_aligned());
+    bl.rebuild();
+    EXPECT_FALSE(bl.is_page_aligned());
+  }
+  {
+    bufferlist bl;
+    const std::string str(CEPH_PAGE_SIZE, 'X');
+    bl.append(str.c_str(), str.size());
+    bl.append(str.c_str(), str.size());
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_TRUE(bl.is_page_aligned());
+    bl.rebuild();
+    EXPECT_TRUE(bl.is_page_aligned());
+    EXPECT_EQ((unsigned)1, bl.buffers().size());
+  }
+}
+
+TEST(BufferList, rebuild_page_aligned) {
+  {
+    bufferlist bl;
+    {
+      bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
+      ptr.set_offset(1);
+      ptr.set_length(CEPH_PAGE_SIZE);
+      bl.append(ptr);
+    }
+    EXPECT_EQ((unsigned)1, bl.buffers().size());
+    EXPECT_FALSE(bl.is_page_aligned());
+    bl.rebuild_page_aligned();
+    EXPECT_TRUE(bl.is_page_aligned());
+    EXPECT_EQ((unsigned)1, bl.buffers().size());
+  }
+  {
+    bufferlist bl;
+    {
+      bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE));
+      EXPECT_TRUE(ptr.is_page_aligned());
+      EXPECT_TRUE(ptr.is_n_page_sized());
+      bl.append(ptr);
+    }
+    {
+      bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
+      EXPECT_TRUE(ptr.is_page_aligned());
+      EXPECT_FALSE(ptr.is_n_page_sized());
+      bl.append(ptr);
+    }
+    {
+      bufferptr ptr(buffer::create_page_aligned(2));
+      ptr.set_offset(1);
+      ptr.set_length(1);
+      EXPECT_FALSE(ptr.is_page_aligned());
+      EXPECT_FALSE(ptr.is_n_page_sized());
+      bl.append(ptr);
+    }
+    {
+      bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE - 2));
+      EXPECT_TRUE(ptr.is_page_aligned());
+      EXPECT_FALSE(ptr.is_n_page_sized());
+      bl.append(ptr);
+    }
+    {
+      bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE));
+      EXPECT_TRUE(ptr.is_page_aligned());
+      EXPECT_TRUE(ptr.is_n_page_sized());
+      bl.append(ptr);
+    }
+    {
+      bufferptr ptr(buffer::create_page_aligned(CEPH_PAGE_SIZE + 1));
+      ptr.set_offset(1);
+      ptr.set_length(CEPH_PAGE_SIZE);
+      EXPECT_FALSE(ptr.is_page_aligned());
+      EXPECT_TRUE(ptr.is_n_page_sized());
+      bl.append(ptr);
+    }
+    EXPECT_EQ((unsigned)6, bl.buffers().size());
+    EXPECT_TRUE((bl.length() & ~CEPH_PAGE_MASK) == 0);
+    EXPECT_FALSE(bl.is_page_aligned());
+    bl.rebuild_page_aligned();
+    EXPECT_TRUE(bl.is_page_aligned());
+    EXPECT_EQ((unsigned)4, bl.buffers().size());
+  }
+}
+
+TEST(BufferList, claim) {
+  bufferlist from;
+  {
+    bufferptr ptr(2);
+    from.append(ptr);
+  }
+  bufferlist to;
+  {
+    bufferptr ptr(4);
+    to.append(ptr);
+  }
+  EXPECT_EQ((unsigned)4, to.length());
+  EXPECT_EQ((unsigned)1, to.buffers().size());
+  to.claim(from);
+  EXPECT_EQ((unsigned)2, to.length());
+  EXPECT_EQ((unsigned)1, to.buffers().size());
+  EXPECT_EQ((unsigned)0, from.buffers().size());
+  EXPECT_EQ((unsigned)0, from.length());
+}
+
+TEST(BufferList, claim_append) {
+  bufferlist from;
+  {
+    bufferptr ptr(2);
+    from.append(ptr);
+  }
+  bufferlist to;
+  {
+    bufferptr ptr(4);
+    to.append(ptr);
+  }
+  EXPECT_EQ((unsigned)4, to.length());
+  EXPECT_EQ((unsigned)1, to.buffers().size());
+  to.claim_append(from);
+  EXPECT_EQ((unsigned)(4 + 2), to.length());
+  EXPECT_EQ((unsigned)4, to.buffers().front().length());
+  EXPECT_EQ((unsigned)2, to.buffers().back().length());
+  EXPECT_EQ((unsigned)2, to.buffers().size());
+  EXPECT_EQ((unsigned)0, from.buffers().size());
+  EXPECT_EQ((unsigned)0, from.length());
+}
+
+TEST(BufferList, claim_prepend) {
+  bufferlist from;
+  {
+    bufferptr ptr(2);
+    from.append(ptr);
+  }
+  bufferlist to;
+  {
+    bufferptr ptr(4);
+    to.append(ptr);
+  }
+  EXPECT_EQ((unsigned)4, to.length());
+  EXPECT_EQ((unsigned)1, to.buffers().size());
+  to.claim_prepend(from);
+  EXPECT_EQ((unsigned)(2 + 4), to.length());
+  EXPECT_EQ((unsigned)2, to.buffers().front().length());
+  EXPECT_EQ((unsigned)4, to.buffers().back().length());
+  EXPECT_EQ((unsigned)2, to.buffers().size());
+  EXPECT_EQ((unsigned)0, from.buffers().size());
+  EXPECT_EQ((unsigned)0, from.length());
+}
+
+TEST(BufferList, begin) {
+  bufferlist bl;
+  bl.append("ABC");
+  bufferlist::iterator i = bl.begin();
+  EXPECT_EQ('A', *i);
+}
+
+TEST(BufferList, end) {
+  bufferlist bl;
+  bl.append("ABC");
+  bufferlist::iterator i = bl.end();
+  i.advance(-1);
+  EXPECT_EQ('C', *i);
+}
+
+TEST(BufferList, copy) {
+  //
+  // void copy(unsigned off, unsigned len, char *dest) const;
+  //
+  {
+    bufferlist bl;
+    EXPECT_THROW(bl.copy((unsigned)100, (unsigned)100, (char*)0), buffer::end_of_buffer);
+    const char *expected = "ABC";
+    bl.append(expected);
+    char *dest = new char[2];
+    bl.copy(1, 2, dest);
+    EXPECT_EQ(0, ::memcmp(expected + 1, dest, 2));
+    delete [] dest;
+  }
+  //
+  // void copy(unsigned off, unsigned len, list &dest) const;
+  //
+  {
+    bufferlist bl;
+    bufferlist dest;
+    EXPECT_THROW(bl.copy((unsigned)100, (unsigned)100, dest), buffer::end_of_buffer);
+    const char *expected = "ABC";
+    bl.append(expected);
+    bl.copy(1, 2, dest);
+    EXPECT_EQ(0, ::memcmp(expected + 1, dest.c_str(), 2));
+  }
+  //
+  // void copy(unsigned off, unsigned len, std::string &dest) const;
+  //
+  {
+    bufferlist bl;
+    std::string dest;
+    EXPECT_THROW(bl.copy((unsigned)100, (unsigned)100, dest), buffer::end_of_buffer);
+    const char *expected = "ABC";
+    bl.append(expected);
+    bl.copy(1, 2, dest);
+    EXPECT_EQ(0, ::memcmp(expected + 1, dest.c_str(), 2));
+  }
+}
+
+TEST(BufferList, copy_in) {
+  //
+  // void copy_in(unsigned off, unsigned len, const char *src);
+  //
+  {
+    bufferlist bl;
+    bl.append("XXX");
+    EXPECT_THROW(bl.copy_in((unsigned)100, (unsigned)100, (char*)0), buffer::end_of_buffer);
+    bl.copy_in(1, 2, "AB");
+    EXPECT_EQ(0, ::memcmp("XAB", bl.c_str(), 3));
+  }
+  //
+  // void copy_in(unsigned off, unsigned len, const list& src);
+  //
+  {
+    bufferlist bl;
+    bl.append("XXX");
+    bufferlist src;
+    src.append("ABC");
+    EXPECT_THROW(bl.copy_in((unsigned)100, (unsigned)100, src), buffer::end_of_buffer);
+    bl.copy_in(1, 2, src);
+    EXPECT_EQ(0, ::memcmp("XAB", bl.c_str(), 3));    
+  }
+}
+
+TEST(BufferList, append) {
+  //
+  // void append(char c);
+  //
+  {
+    bufferlist bl;
+    EXPECT_EQ((unsigned)0, bl.buffers().size());
+    bl.append('A');
+    EXPECT_EQ((unsigned)1, bl.buffers().size());
+    EXPECT_TRUE(bl.is_page_aligned());
+  }
+  //
+  // void append(const char *data, unsigned len);
+  //
+  {
+    bufferlist bl(CEPH_PAGE_SIZE);
+    std::string str(CEPH_PAGE_SIZE * 2, 'X');
+    bl.append(str.c_str(), str.size());
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_EQ(CEPH_PAGE_SIZE, bl.buffers().front().length());
+    EXPECT_EQ(CEPH_PAGE_SIZE, bl.buffers().back().length());
+  }
+  //
+  // void append(const std::string& s);
+  //
+  {
+    bufferlist bl(CEPH_PAGE_SIZE);
+    std::string str(CEPH_PAGE_SIZE * 2, 'X');
+    bl.append(str);
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_EQ(CEPH_PAGE_SIZE, bl.buffers().front().length());
+    EXPECT_EQ(CEPH_PAGE_SIZE, bl.buffers().back().length());
+  }
+  //
+  // void append(const ptr& bp);
+  //
+  {
+    bufferlist bl;
+    EXPECT_EQ((unsigned)0, bl.buffers().size());
+    EXPECT_EQ((unsigned)0, bl.length());
+    {
+      bufferptr ptr;
+      bl.append(ptr);
+      EXPECT_EQ((unsigned)0, bl.buffers().size());
+      EXPECT_EQ((unsigned)0, bl.length());
+    }
+    {
+      bufferptr ptr(3);
+      bl.append(ptr);
+      EXPECT_EQ((unsigned)1, bl.buffers().size());
+      EXPECT_EQ((unsigned)3, bl.length());
+    }
+  }
+  //
+  // void append(const ptr& bp, unsigned off, unsigned len);
+  //
+  {
+    bufferlist bl;
+    bl.append('A');
+    bufferptr back(bl.buffers().back());
+    bufferptr in(back);
+    EXPECT_EQ((unsigned)1, bl.buffers().size());
+    EXPECT_EQ((unsigned)1, bl.length());
+    EXPECT_THROW(bl.append(in, (unsigned)100, (unsigned)100), FailedAssertion);
+    EXPECT_LT((unsigned)0, in.unused_tail_length());
+    in.append('B');
+    bl.append(in, back.end(), 1);
+    EXPECT_EQ((unsigned)1, bl.buffers().size());
+    EXPECT_EQ((unsigned)2, bl.length());
+    EXPECT_EQ('B', bl[1]);
+  }
+  {
+    bufferlist bl;
+    EXPECT_EQ((unsigned)0, bl.buffers().size());
+    EXPECT_EQ((unsigned)0, bl.length());
+    bufferptr ptr(2);
+    ptr.set_length(0);
+    ptr.append("AB", 2);
+    bl.append(ptr, 1, 1);
+    EXPECT_EQ((unsigned)1, bl.buffers().size());
+    EXPECT_EQ((unsigned)1, bl.length());
+  }
+  //
+  // void append(const list& bl);
+  //
+  {
+    bufferlist bl;
+    bl.append('A');
+    bufferlist other;
+    other.append('B');
+    bl.append(other);
+    EXPECT_EQ((unsigned)2, bl.buffers().size());
+    EXPECT_EQ('B', bl[1]);
+  }
+  //
+  // void append(std::istream& in);
+  //
+  {
+    bufferlist bl;
+    std::string expected("ABC\nDEF\n");
+    std::istringstream is("ABC\n\nDEF");
+    bl.append(is);
+    EXPECT_EQ(0, ::memcmp(expected.c_str(), bl.c_str(), expected.size()));
+    EXPECT_EQ(expected.size(), bl.length());
+  }
+}
+
+TEST(BufferList, append_zero) {
+  bufferlist bl;
+  bl.append('A');
+  EXPECT_EQ((unsigned)1, bl.buffers().size());
+  EXPECT_EQ((unsigned)1, bl.length());
+  bl.append_zero(1);
+  EXPECT_EQ((unsigned)2, bl.buffers().size());
+  EXPECT_EQ((unsigned)2, bl.length());
+  EXPECT_EQ('\0', bl[1]);
+}
+
+TEST(BufferList, operator_brackets) {
+  bufferlist bl;
+  EXPECT_THROW(bl[1], buffer::end_of_buffer);
+  bl.append('A');
+  bufferlist other;
+  other.append('B');
+  bl.append(other);
+  EXPECT_EQ((unsigned)2, bl.buffers().size());
+  EXPECT_EQ('B', bl[1]);
+}
+
+TEST(BufferList, c_str) {
+  bufferlist bl;
+  EXPECT_EQ((const char*)NULL, bl.c_str());
+  bl.append('A');
+  bufferlist other;
+  other.append('B');
+  bl.append(other);
+  EXPECT_EQ((unsigned)2, bl.buffers().size());
+  EXPECT_EQ(0, ::memcmp("AB", bl.c_str(), 2));
+}
+
+TEST(BufferList, substr_of) {
+  bufferlist bl;
+  EXPECT_THROW(bl.substr_of(bl, 1, 1), buffer::end_of_buffer);
+  const char *s[] = {
+    "ABC",
+    "DEF",
+    "GHI",
+    "JKL"
+  };
+  for (unsigned i = 0; i < 4; i++) {
+    bufferptr ptr(s[i], strlen(s[i]));
+    bl.push_back(ptr);
+  }
+  EXPECT_EQ((unsigned)4, bl.buffers().size());
+
+  bufferlist other;
+  other.append("TO BE CLEARED");
+  other.substr_of(bl, 4, 4);
+  EXPECT_EQ((unsigned)2, other.buffers().size());
+  EXPECT_EQ((unsigned)4, other.length());
+  EXPECT_EQ(0, ::memcmp("EFGH", other.c_str(), 4));
+}
+
+TEST(BufferList, splice) {
+  bufferlist bl;
+  EXPECT_THROW(bl.splice(1, 1), buffer::end_of_buffer);
+  const char *s[] = {
+    "ABC",
+    "DEF",
+    "GHI",
+    "JKL"
+  };
+  for (unsigned i = 0; i < 4; i++) {
+    bufferptr ptr(s[i], strlen(s[i]));
+    bl.push_back(ptr);
+  }
+  EXPECT_EQ((unsigned)4, bl.buffers().size());
+  bl.splice(0, 0);
+
+  bufferlist other;
+  other.append('X');
+  bl.splice(4, 4, &other);
+  EXPECT_EQ((unsigned)3, other.buffers().size());
+  EXPECT_EQ((unsigned)5, other.length());
+  EXPECT_EQ(0, ::memcmp("XEFGH", other.c_str(), other.length()));
+  EXPECT_EQ((unsigned)8, bl.length());
+  {
+    bufferlist tmp(bl);
+    EXPECT_EQ(0, ::memcmp("ABCDIJKL", tmp.c_str(), tmp.length()));
+  }
+
+  bl.splice(4, 4);
+  EXPECT_EQ((unsigned)4, bl.length());
+  EXPECT_EQ(0, ::memcmp("ABCD", bl.c_str(), bl.length()));
+}
+
+TEST(BufferList, write) {
+  std::ostringstream stream;
+  bufferlist bl;
+  bl.append("ABC");
+  bl.write(1, 2, stream);
+  EXPECT_EQ("BC", stream.str());
+}
+
+TEST(BufferList, encode_base64) {
+  bufferlist bl;
+  bl.append("ABCD");
+  bufferlist other;
+  bl.encode_base64(other);
+  const char *expected = "QUJDRA==";
+  EXPECT_EQ(0, ::memcmp(expected, other.c_str(), strlen(expected)));
+}
+
+TEST(BufferList, decode_base64) {
+  bufferlist bl;
+  bl.append("QUJDRA==");
+  bufferlist other;
+  other.decode_base64(bl);
+  const char *expected = "ABCD";
+  EXPECT_EQ(0, ::memcmp(expected, other.c_str(), strlen(expected)));
+  bufferlist malformed;
+  malformed.append("QUJDRA");
+  EXPECT_THROW(other.decode_base64(malformed), buffer::malformed_input);
+}
+
+TEST(BufferList, hexdump) {
+  bufferlist bl;
+  std::ostringstream stream;
+  bl.append("013245678901234\0006789012345678901234", 32);
+  bl.hexdump(stream);
+  EXPECT_EQ("0000 : 30 31 33 32 34 35 36 37 38 39 30 31 32 33 34 00 : 013245678901234.\n"
+	    "0010 : 36 37 38 39 30 31 32 33 34 35 36 37 38 39 30 31 : 6789012345678901\n",
+	    stream.str());
+}
+
+TEST(BufferList, read_file) {
+  std::string error;
+  bufferlist bl;
+  ::unlink("testfile");
+  EXPECT_EQ(-ENOENT, bl.read_file("UNLIKELY", &error));
+  ::system("echo ABC > testfile ; chmod 0 testfile");
+  EXPECT_EQ(-EACCES, bl.read_file("testfile", &error));
+  ::system("chmod +r testfile");
+  EXPECT_EQ(0, bl.read_file("testfile", &error));
+  ::unlink("testfile");
+  EXPECT_EQ((unsigned)4, bl.length());
+  std::string actual(bl.c_str(), bl.length());
+  EXPECT_EQ("ABC\n", actual);
+}
+
+TEST(BufferList, read_fd) {
+  unsigned len = 4;
+  ::unlink("testfile");
+  ::system("echo ABC > testfile");
+  int fd = -1;
+  bufferlist bl;
+  EXPECT_EQ(-EBADF, bl.read_fd(fd, len));
+  fd = ::open("testfile", O_RDONLY);
+  EXPECT_EQ(len, (unsigned)bl.read_fd(fd, len));
+  EXPECT_EQ(CEPH_PAGE_SIZE - len, bl.buffers().front().unused_tail_length());
+  EXPECT_EQ(len, bl.length());
+  ::close(fd);
+  ::unlink("testfile");
+}
+
+TEST(BufferList, write_file) {
+  ::unlink("testfile");
+  int mode = 0600;
+  bufferlist bl;
+  EXPECT_EQ(-ENOENT, bl.write_file("un/like/ly", mode));
+  bl.append("ABC");
+  EXPECT_EQ(0, bl.write_file("testfile", mode));
+  struct stat st;
+  memset(&st, 0, sizeof(st));
+  ::stat("testfile", &st);
+  EXPECT_EQ((unsigned)(mode | S_IFREG), st.st_mode);
+  ::unlink("testfile");
+}
+
+TEST(BufferList, write_fd) {
+  ::unlink("testfile");
+  int fd = ::open("testfile", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+  bufferlist bl;
+  for (unsigned i = 0; i < IOV_MAX * 2; i++) {
+    bufferptr ptr("A", 1);
+    bl.push_back(ptr);
+  }
+  EXPECT_EQ(0, bl.write_fd(fd));
+  ::close(fd);
+  struct stat st;
+  memset(&st, 0, sizeof(st));
+  ::stat("testfile", &st);
+  EXPECT_EQ(IOV_MAX * 2, st.st_size);
+  ::unlink("testfile");
+}
+
+TEST(BufferList, crc32c) {
+  bufferlist bl;
+  __u32 crc = 0;
+  bl.append("A");
+  crc = bl.crc32c(crc);
+  EXPECT_EQ((unsigned)0xB3109EBF, crc);
+  crc = bl.crc32c(crc);
+  EXPECT_EQ((unsigned)0x5FA5C0CC, crc);
+}
+
+TEST(BufferList, crc32c_append) {
+  bufferlist bl1;
+  bufferlist bl2;
+
+  for (int j = 0; j < 200; ++j) {
+    bufferlist bl;
+    for (int i = 0; i < 200; ++i) {
+      char x = rand();
+      bl.append(x);
+      bl1.append(x);
+    }
+    bl.crc32c(rand()); // mess with the cached bufferptr crc values
+    bl2.append(bl);
+  }
+  ASSERT_EQ(bl1.crc32c(0), bl2.crc32c(0));
+}
+
+TEST(BufferList, crc32c_append_perf) {
+  int len = 256 * 1024 * 1024;
+  bufferptr a(len);
+  bufferptr b(len);
+  bufferptr c(len);
+  bufferptr d(len);
+  std::cout << "populating large buffers (a, b=c=d)" << std::endl;
+  char *pa = a.c_str();
+  char *pb = b.c_str();
+  char *pc = c.c_str();
+  char *pd = c.c_str();
+  for (int i=0; i(bufferlist& l, bufferlist& r)
+  //
+  ASSERT_FALSE(a > ab);
+  ASSERT_TRUE(ab > a);
+  ASSERT_TRUE(ac > ab);
+  ASSERT_FALSE(ab > ac);
+  ASSERT_FALSE(ab > ab);
+  //
+  // bool operator>=(bufferlist& l, bufferlist& r)
+  //
+  ASSERT_FALSE(a >= ab);
+  ASSERT_TRUE(ab >= a);
+  ASSERT_TRUE(ac >= ab);
+  ASSERT_FALSE(ab >= ac);
+  ASSERT_TRUE(ab >= ab);
+  //
+  // bool operator<(bufferlist& l, bufferlist& r)
+  //
+  ASSERT_TRUE(a < ab);
+  ASSERT_FALSE(ab < a);
+  ASSERT_FALSE(ac < ab);
+  ASSERT_TRUE(ab < ac);
+  ASSERT_FALSE(ab < ab);
+  //
+  // bool operator<=(bufferlist& l, bufferlist& r)
+  //
+  ASSERT_TRUE(a <= ab);
+  ASSERT_FALSE(ab <= a);
+  ASSERT_FALSE(ac <= ab);
+  ASSERT_TRUE(ab <= ac);
+  ASSERT_TRUE(ab <= ab);
+  //
+  // bool operator==(bufferlist &l, bufferlist &r)
+  //
+  ASSERT_FALSE(a == ab);
+  ASSERT_FALSE(ac == ab);
+  ASSERT_TRUE(ab == ab);
+}
+
+TEST(BufferList, ostream) {
+  std::ostringstream stream;
+  bufferlist bl;
+  const char *s[] = {
+    "ABC",
+    "DEF"
+  };
+  for (unsigned i = 0; i < 2; i++) {
+    bufferptr ptr(s[i], strlen(s[i]));
+    bl.push_back(ptr);
+  }
+  stream << bl;
+  std::cerr << stream.str() << std::endl;
+  EXPECT_GT(stream.str().size(), stream.str().find("list(len=6,"));
+  EXPECT_GT(stream.str().size(), stream.str().find("len 3 nref 1),\n"));
+  EXPECT_GT(stream.str().size(), stream.str().find("len 3 nref 1)\n"));
+}
+
+TEST(BufferList, zero) {
+  //
+  // void zero()
+  //
+  {
+    bufferlist bl;
+    bl.append('A');
+    EXPECT_EQ('A', bl[0]);
+    bl.zero();
+    EXPECT_EQ('\0', bl[0]);
+  }
+  //
+  // void zero(unsigned o, unsigned l)
+  //
+  const char *s[] = {
+    "ABC",
+    "DEF",
+    "GHI",
+    "KLM"
+  };
+  {
+    bufferlist bl;
+    bufferptr ptr(s[0], strlen(s[0]));
+    bl.push_back(ptr);
+    bl.zero((unsigned)0, (unsigned)1);
+    EXPECT_EQ(0, ::memcmp("\0BC", bl.c_str(), 3));
+  }
+  {
+    bufferlist bl;
+    for (unsigned i = 0; i < 4; i++) {
+      bufferptr ptr(s[i], strlen(s[i]));
+      bl.push_back(ptr);
+    }
+    EXPECT_THROW(bl.zero((unsigned)0, (unsigned)2000), FailedAssertion);
+    bl.zero((unsigned)2, (unsigned)5);
+    EXPECT_EQ(0, ::memcmp("AB\0\0\0\0\0HIKLM", bl.c_str(), 9));
+  }
+  {
+    bufferlist bl;
+    for (unsigned i = 0; i < 4; i++) {
+      bufferptr ptr(s[i], strlen(s[i]));
+      bl.push_back(ptr);
+    }
+    bl.zero((unsigned)3, (unsigned)3);
+    EXPECT_EQ(0, ::memcmp("ABC\0\0\0GHIKLM", bl.c_str(), 9));
+  }
+}
+
+TEST(BufferList, EmptyAppend) {
+  bufferlist bl;
+  bufferptr ptr;
+  bl.push_back(ptr);
+  ASSERT_EQ(bl.begin().end(), 1);
+}
+
+TEST(BufferList, TestPtrAppend) {
+  bufferlist bl;
+  char correct[MAX_TEST];
+  int curpos = 0;
+  int length = random() % 5 > 0 ? random() % 1000 : 0;
+  while (curpos + length < MAX_TEST) {
+    if (!length) {
+      bufferptr ptr;
+      bl.push_back(ptr);
+    } else {
+      char *current = correct + curpos;
+      for (int i = 0; i < length; ++i) {
+        char next = random() % 255;
+        correct[curpos++] = next;
+      }
+      bufferptr ptr(current, length);
+      bl.append(ptr);
+    }
+    length = random() % 5 > 0 ? random() % 1000 : 0;
+  }
+  ASSERT_EQ(memcmp(bl.c_str(), correct, curpos), 0);
+}
+
+TEST(BufferList, TestDirectAppend) {
+  bufferlist bl;
+  char correct[MAX_TEST];
+  int curpos = 0;
+  int length = random() % 5 > 0 ? random() % 1000 : 0;
+  while (curpos + length < MAX_TEST) {
+    char *current = correct + curpos;
+    for (int i = 0; i < length; ++i) {
+      char next = random() % 255;
+      correct[curpos++] = next;
+    }
+    bl.append(current, length);
+    length = random() % 5 > 0 ? random() % 1000 : 0;
+  }
+  ASSERT_EQ(memcmp(bl.c_str(), correct, curpos), 0);
+}
+
+TEST(BufferList, TestCopyAll) {
+  const static size_t BIG_SZ = 10737414;
+  ceph::shared_ptr  big(
+      (unsigned char*)malloc(BIG_SZ), free);
+  unsigned char c = 0;
+  for (size_t i = 0; i < BIG_SZ; ++i) {
+    big.get()[i] = c++;
+  }
+  bufferlist bl;
+  bl.append((const char*)big.get(), BIG_SZ);
+  bufferlist::iterator i = bl.begin();
+  bufferlist bl2;
+  i.copy_all(bl2);
+  ASSERT_EQ(bl2.length(), BIG_SZ);
+  ceph::shared_ptr  big2(
+      (unsigned char*)malloc(BIG_SZ), free);
+  bl2.copy(0, BIG_SZ, (char*)big2.get());
+  ASSERT_EQ(memcmp(big.get(), big2.get(), BIG_SZ), 0);
+}
+
+TEST(BufferHash, all) {
+  {
+    bufferlist bl;
+    bl.append("A");
+    bufferhash hash;
+    EXPECT_EQ((unsigned)0, hash.digest());
+    hash.update(bl);
+    EXPECT_EQ((unsigned)0xB3109EBF, hash.digest());
+    hash.update(bl);
+    EXPECT_EQ((unsigned)0x5FA5C0CC, hash.digest());
+  }
+  {
+    bufferlist bl;
+    bl.append("A");
+    bufferhash hash;
+    EXPECT_EQ((unsigned)0, hash.digest());
+    bufferhash& returned_hash =  hash << bl;
+    EXPECT_EQ(&returned_hash, &hash);
+    EXPECT_EQ((unsigned)0xB3109EBF, hash.digest());
+  }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make unittest_bufferlist && 
+ *    ulimit -s unlimited ; CEPH_BUFFER_TRACK=true valgrind \
+ *    --max-stackframe=20000000 --tool=memcheck \
+ *    ./unittest_bufferlist # --gtest_filter=BufferList.constructors"
+ * End:
+ */
+
diff --git a/ceph/src/test/buildtest_skeleton.cc b/ceph/src/test/buildtest_skeleton.cc
new file mode 100644
index 00000000..8215a05d
--- /dev/null
+++ b/ceph/src/test/buildtest_skeleton.cc
@@ -0,0 +1,13 @@
+#include "common/common_init.h"
+
+/* This program exists to test that we can build libcommon without
+ * referencing g_ceph_context
+ *
+ * This program will go away as soon as we actually don't use g_ceph_context in
+ * more programs. Obviously, at that point, those programs will provide an
+ * equivalent test.
+ */
+int main(int argc, char **argv)
+{
+  return 0;
+}
diff --git a/ceph/src/test/ceph_argparse.cc b/ceph/src/test/ceph_argparse.cc
new file mode 100644
index 00000000..d1a790fd
--- /dev/null
+++ b/ceph/src/test/ceph_argparse.cc
@@ -0,0 +1,445 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/ceph_argparse.h"
+
+#include "gtest/gtest.h"
+#include 
+
+/* Holds a std::vector with C-strings.
+ * Will free() them properly in the destructor.
+ *
+ * Note: the ceph_argparse functions modify the vector, removing elements as
+ * they find them.  So we keep a parallel vector, orig, to make sure that we
+ * never forget to delete a string.
+ */
+class VectorContainer
+{
+public:
+  VectorContainer(const char** arr_) {
+    for (const char **a = arr_; *a; ++a) {
+      const char *str = (const char*)strdup(*a);
+      arr.push_back(str);
+      orig.push_back(str);
+    }
+  }
+  ~VectorContainer() {
+    for (std::vector::iterator i = orig.begin();
+	   i != orig.end(); ++i)
+    {
+	 free((void*)*i);
+    }
+  }
+  void refresh() {
+    arr.assign(orig.begin(), orig.end());
+  }
+  std::vector < const char* > arr;
+
+private:
+  std::vector < const char* > orig;
+};
+
+TEST(CephArgParse, SimpleArgParse) {
+  const char *BAR5[] = { "./myprog", "--bar", "5", NULL };
+  const char *FOO[] = { "./myprog", "--foo", "--baz", NULL };
+  const char *NONE[] = { "./myprog", NULL };
+
+  bool found_foo = false;
+  std::string found_bar;
+  VectorContainer bar5(BAR5);
+  for (std::vector::iterator i = bar5.arr.begin();
+       i != bar5.arr.end(); )
+  {
+      if (ceph_argparse_flag(bar5.arr, i, "--foo", (char*)NULL)) {
+	found_foo = true;
+      }
+      else if (ceph_argparse_witharg(bar5.arr, i, &found_bar, "--bar", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_foo, false);
+  ASSERT_EQ(found_bar, "5");
+
+  found_foo = false;
+  found_bar = "";
+  VectorContainer foo(FOO);
+  for (std::vector::iterator i = foo.arr.begin();
+       i != foo.arr.end(); )
+  {
+      if (ceph_argparse_flag(foo.arr, i, "--foo", (char*)NULL)) {
+	found_foo = true;
+      }
+      else if (ceph_argparse_witharg(foo.arr, i, &found_bar, "--bar", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_foo, true);
+  ASSERT_EQ(found_bar, "");
+
+  found_foo = false;
+  found_bar = "";
+  VectorContainer none(NONE);
+  for (std::vector::iterator i = none.arr.begin();
+       i != none.arr.end(); )
+  {
+      if (ceph_argparse_flag(none.arr, i, "--foo", (char*)NULL)) {
+	found_foo = true;
+      }
+      else if (ceph_argparse_witharg(none.arr, i, &found_bar, "--bar", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_foo, false);
+  ASSERT_EQ(found_bar, "");
+}
+
+TEST(CephArgParse, DoubleDash) {
+  const char *ARGS[] = { "./myprog", "--foo", "5", "--", "--bar", "6", NULL };
+
+  int foo = -1, bar = -1;
+  VectorContainer args(ARGS);
+  for (std::vector::iterator i = args.arr.begin();
+       i != args.arr.end(); )
+  {
+    std::string myarg;
+    if (ceph_argparse_double_dash(args.arr, i)) {
+      break;
+    }
+    else if (ceph_argparse_witharg(args.arr, i, &myarg, "--foo", (char*)NULL)) {
+      foo = atoi(myarg.c_str());
+    }
+    else if (ceph_argparse_witharg(args.arr, i, &myarg, "--bar", (char*)NULL)) {
+      bar = atoi(myarg.c_str());
+    }
+    else
+      ++i;
+  }
+  ASSERT_EQ(foo, 5);
+  ASSERT_EQ(bar, -1);
+}
+
+
+TEST(CephArgParse, WithDashesAndUnderscores) {
+  const char *BAZSTUFF1[] = { "./myprog", "--goo", "--baz-stuff", "50", "--end", NULL };
+  const char *BAZSTUFF2[] = { "./myprog", "--goo2", "--baz_stuff", "50", NULL };
+  const char *BAZSTUFF3[] = { "./myprog", "--goo2", "--baz-stuff=50", "50", NULL };
+  const char *BAZSTUFF4[] = { "./myprog", "--goo2", "--baz_stuff=50", "50", NULL };
+  const char *NONE1[] = { "./myprog", NULL };
+  const char *NONE2[] = { "./myprog", "--goo2", "--baz_stuff2", "50", NULL };
+  const char *NONE3[] = { "./myprog", "--goo2", "__baz_stuff", "50", NULL };
+
+  // as flag
+  std::string found_baz;
+  VectorContainer bazstuff1(BAZSTUFF1);
+  for (std::vector::iterator i = bazstuff1.arr.begin();
+       i != bazstuff1.arr.end(); )
+  {
+      if (ceph_argparse_flag(bazstuff1.arr, i, "--baz-stuff", (char*)NULL)) {
+	found_baz = "true";
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "true");
+
+  // as flag
+  found_baz = "";
+  VectorContainer bazstuff2(BAZSTUFF2);
+  for (std::vector::iterator i = bazstuff2.arr.begin();
+       i != bazstuff2.arr.end(); )
+  {
+      if (ceph_argparse_flag(bazstuff2.arr, i, "--baz-stuff", (char*)NULL)) {
+	found_baz = "true";
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "true");
+
+  // with argument
+  found_baz = "";
+  bazstuff1.refresh();
+  for (std::vector::iterator i = bazstuff1.arr.begin();
+       i != bazstuff1.arr.end(); )
+  {
+      if (ceph_argparse_witharg(bazstuff1.arr, i, &found_baz, "--baz-stuff", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "50");
+
+  // with argument
+  found_baz = "";
+  bazstuff2.refresh();
+  for (std::vector::iterator i = bazstuff2.arr.begin();
+       i != bazstuff2.arr.end(); )
+  {
+      if (ceph_argparse_witharg(bazstuff2.arr, i, &found_baz, "--baz-stuff", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "50");
+
+  // with argument
+  found_baz = "";
+  VectorContainer bazstuff3(BAZSTUFF3);
+  for (std::vector::iterator i = bazstuff3.arr.begin();
+       i != bazstuff3.arr.end(); )
+  {
+      if (ceph_argparse_witharg(bazstuff3.arr, i, &found_baz, "--baz-stuff", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "50");
+
+  // with argument
+  found_baz = "";
+  VectorContainer bazstuff4(BAZSTUFF4);
+  for (std::vector::iterator i = bazstuff4.arr.begin();
+       i != bazstuff4.arr.end(); )
+  {
+      if (ceph_argparse_witharg(bazstuff4.arr, i, &found_baz, "--baz-stuff", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "50");
+
+  // not found
+  found_baz = "";
+  VectorContainer none1(NONE1);
+  for (std::vector::iterator i = none1.arr.begin();
+       i != none1.arr.end(); )
+  {
+      if (ceph_argparse_flag(none1.arr, i, "--baz-stuff", (char*)NULL)) {
+	found_baz = "true";
+      }
+      else if (ceph_argparse_witharg(none1.arr, i, &found_baz, "--baz-stuff", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "");
+
+  // not found
+  found_baz = "";
+  VectorContainer none2(NONE2);
+  for (std::vector::iterator i = none2.arr.begin();
+       i != none2.arr.end(); )
+  {
+      if (ceph_argparse_flag(none2.arr, i, "--baz-stuff", (char*)NULL)) {
+	found_baz = "true";
+      }
+      else if (ceph_argparse_witharg(none2.arr, i, &found_baz, "--baz-stuff", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "");
+
+  // not found
+  found_baz = "";
+  VectorContainer none3(NONE3);
+  for (std::vector::iterator i = none3.arr.begin();
+       i != none3.arr.end(); )
+  {
+      if (ceph_argparse_flag(none3.arr, i, "--baz-stuff", (char*)NULL)) {
+	found_baz = "true";
+      }
+      else if (ceph_argparse_witharg(none3.arr, i, &found_baz, "--baz-stuff", (char*)NULL)) {
+      }
+      else
+	++i;
+  }
+  ASSERT_EQ(found_baz, "");
+}
+
+
+TEST(CephArgParse, WithInt) {
+  const char *BAZSTUFF1[] = { "./myprog", "--foo", "50", "--bar", "52", NULL };
+  const char *BAZSTUFF2[] = { "./myprog", "--foo", "--bar", "52", NULL };
+  const char *BAZSTUFF3[] = { "./myprog", "--foo", "40", "--", "--bar", "42", NULL };
+
+  // normal test
+  VectorContainer bazstuff1(BAZSTUFF1);
+  ostringstream err;
+  int foo = -1, bar = -1;
+  for (std::vector::iterator i = bazstuff1.arr.begin();
+       i != bazstuff1.arr.end(); )
+  {
+    if (ceph_argparse_double_dash(bazstuff1.arr, i)) {
+      break;
+    } else if (ceph_argparse_withint(bazstuff1.arr, i, &foo, &err, "--foo", (char*)NULL)) {
+      ASSERT_EQ(string(""), err.str());
+    } else if (ceph_argparse_withint(bazstuff1.arr, i, &bar, &err, "--bar", (char*)NULL)) {
+      ASSERT_EQ(string(""), err.str());
+    }
+    else {
+      ++i;
+    }
+  }
+  ASSERT_EQ(foo, 50);
+  ASSERT_EQ(bar, 52);
+
+  // parse error test
+  VectorContainer bazstuff2(BAZSTUFF2);
+  ostringstream err2;
+  for (std::vector::iterator i = bazstuff2.arr.begin();
+       i != bazstuff2.arr.end(); )
+  {
+    if (ceph_argparse_double_dash(bazstuff2.arr, i)) {
+      break;
+    } else if (ceph_argparse_withint(bazstuff2.arr, i, &foo, &err2, "--foo", (char*)NULL)) {
+      ASSERT_NE(string(""), err2.str());
+    }
+    else {
+      ++i;
+    }
+  }
+
+  // double dash test
+  VectorContainer bazstuff3(BAZSTUFF3);
+  foo = -1, bar = -1;
+  for (std::vector::iterator i = bazstuff3.arr.begin();
+       i != bazstuff3.arr.end(); )
+  {
+    if (ceph_argparse_double_dash(bazstuff3.arr, i)) {
+      break;
+    } else if (ceph_argparse_withint(bazstuff3.arr, i, &foo, &err, "--foo", (char*)NULL)) {
+      ASSERT_EQ(string(""), err.str());
+    } else if (ceph_argparse_withint(bazstuff3.arr, i, &bar, &err, "--bar", (char*)NULL)) {
+      ASSERT_EQ(string(""), err.str());
+    }
+    else {
+      ++i;
+    }
+  }
+  ASSERT_EQ(foo, 40);
+  ASSERT_EQ(bar, -1);
+}
+
+TEST(CephArgParse, env_to_vec) {
+  {
+    std::vector args;
+    unsetenv("CEPH_ARGS");
+    unsetenv("WHATEVER");
+    env_to_vec(args);
+    EXPECT_EQ(0u, args.size());
+    env_to_vec(args, "WHATEVER");
+    EXPECT_EQ(0u, args.size());
+    args.push_back("a");
+    setenv("CEPH_ARGS", "b c", 0);
+    env_to_vec(args);
+    EXPECT_EQ(3u, args.size());
+    EXPECT_EQ(string("b"), args[1]);
+    EXPECT_EQ(string("c"), args[2]);
+    setenv("WHATEVER", "d e", 0);
+    env_to_vec(args, "WHATEVER");
+    EXPECT_EQ(5u, args.size());
+    EXPECT_EQ(string("d"), args[3]);
+    EXPECT_EQ(string("e"), args[4]);
+  }
+  {
+    std::vector args;
+    unsetenv("CEPH_ARGS");
+    args.push_back("a");
+    args.push_back("--");
+    args.push_back("c");
+    setenv("CEPH_ARGS", "b -- d", 0);
+    env_to_vec(args);
+    EXPECT_EQ(5u, args.size());
+    EXPECT_EQ(string("a"), args[0]);
+    EXPECT_EQ(string("b"), args[1]);
+    EXPECT_EQ(string("--"), args[2]);
+    EXPECT_EQ(string("c"), args[3]);
+    EXPECT_EQ(string("d"), args[4]);
+  }
+  {
+    std::vector args;
+    unsetenv("CEPH_ARGS");
+    args.push_back("a");
+    args.push_back("--");
+    setenv("CEPH_ARGS", "b -- c", 0);
+    env_to_vec(args);
+    EXPECT_EQ(4u, args.size());
+    EXPECT_EQ(string("a"), args[0]);
+    EXPECT_EQ(string("b"), args[1]);
+    EXPECT_EQ(string("--"), args[2]);
+    EXPECT_EQ(string("c"), args[3]);
+  }
+  {
+    std::vector args;
+    unsetenv("CEPH_ARGS");
+    args.push_back("--");
+    args.push_back("c");
+    setenv("CEPH_ARGS", "b -- d", 0);
+    env_to_vec(args);
+    EXPECT_EQ(4u, args.size());
+    EXPECT_EQ(string("b"), args[0]);
+    EXPECT_EQ(string("--"), args[1]);
+    EXPECT_EQ(string("c"), args[2]);
+    EXPECT_EQ(string("d"), args[3]);
+  }
+  {
+    std::vector args;
+    unsetenv("CEPH_ARGS");
+    args.push_back("b");
+    setenv("CEPH_ARGS", "c -- d", 0);
+    env_to_vec(args);
+    EXPECT_EQ(4u, args.size());
+    EXPECT_EQ(string("b"), args[0]);
+    EXPECT_EQ(string("c"), args[1]);
+    EXPECT_EQ(string("--"), args[2]);
+    EXPECT_EQ(string("d"), args[3]);
+  }
+  {
+    std::vector args;
+    unsetenv("CEPH_ARGS");
+    args.push_back("a");
+    args.push_back("--");
+    args.push_back("c");
+    setenv("CEPH_ARGS", "-- d", 0);
+    env_to_vec(args);
+    EXPECT_EQ(4u, args.size());
+    EXPECT_EQ(string("a"), args[0]);
+    EXPECT_EQ(string("--"), args[1]);
+    EXPECT_EQ(string("c"), args[2]);
+    EXPECT_EQ(string("d"), args[3]);
+  }
+  {
+    std::vector args;
+    unsetenv("CEPH_ARGS");
+    args.push_back("a");
+    args.push_back("--");
+    args.push_back("c");
+    setenv("CEPH_ARGS", "d", 0);
+    env_to_vec(args);
+    EXPECT_EQ(4u, args.size());
+    EXPECT_EQ(string("a"), args[0]);
+    EXPECT_EQ(string("d"), args[1]);
+    EXPECT_EQ(string("--"), args[2]);
+    EXPECT_EQ(string("c"), args[3]);
+  }
+}
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make unittest_ceph_argparse && ./unittest_ceph_argparse"
+ * End:
+ */
diff --git a/ceph/src/test/ceph_compatset.cc b/ceph/src/test/ceph_compatset.cc
new file mode 100644
index 00000000..2b57db01
--- /dev/null
+++ b/ceph/src/test/ceph_compatset.cc
@@ -0,0 +1,164 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/types.h"
+#include "include/compat.h"
+
+//#undef assert
+//#define	assert(foo) if (!(foo)) abort();
+
+#include "include/CompatSet.h"
+
+#include "gtest/gtest.h"
+#include 
+
+TEST(CephCompatSet, AllSet) {
+  CompatSet::FeatureSet compat;
+  CompatSet::FeatureSet ro;
+  CompatSet::FeatureSet incompat;
+
+  EXPECT_THROW(compat.insert(CompatSet::Feature(0, "test")), FailedAssertion);
+  EXPECT_THROW(compat.insert(CompatSet::Feature(64, "test")), FailedAssertion);
+
+  for (int i = 1; i < 64; i++) {
+    stringstream cname;
+    cname << string("c") << i;
+    compat.insert(CompatSet::Feature(i,cname.str().c_str()));
+    stringstream roname;
+    roname << string("r") << i;
+    ro.insert(CompatSet::Feature(i,roname.str().c_str()));
+    stringstream iname;
+    iname << string("i") << i;
+    incompat.insert(CompatSet::Feature(i,iname.str().c_str()));
+  }
+  CompatSet tcs(compat, ro, incompat);
+
+  //cout << tcs << std::endl;
+
+  //Due to a workaround for a bug bit 0 is always set even though it is
+  //not a legal feature.
+  EXPECT_EQ(tcs.compat.mask, (uint64_t)0xffffffffffffffff);
+  EXPECT_EQ(tcs.ro_compat.mask, (uint64_t)0xffffffffffffffff);
+  EXPECT_EQ(tcs.incompat.mask, (uint64_t)0xffffffffffffffff);
+
+  for (int i = 1; i < 64; i++) {
+    EXPECT_TRUE(tcs.compat.contains(i));
+    stringstream cname;
+    cname << string("c") << i;
+    EXPECT_TRUE(tcs.compat.contains(CompatSet::Feature(i,cname.str().c_str())));
+    tcs.compat.remove(i);
+
+    EXPECT_TRUE(tcs.ro_compat.contains(i));
+    stringstream roname;
+    roname << string("r") << i;
+    EXPECT_TRUE(tcs.ro_compat.contains(CompatSet::Feature(i,roname.str().c_str())));
+    tcs.ro_compat.remove(i);
+
+    EXPECT_TRUE(tcs.incompat.contains(i));
+    stringstream iname;
+    iname << string("i") << i;
+    EXPECT_TRUE(tcs.incompat.contains(CompatSet::Feature(i,iname.str().c_str())));
+    tcs.incompat.remove(i);
+  }
+  //Due to a workaround for a bug bit 0 is always set even though it is
+  //not a legal feature.
+  EXPECT_EQ(tcs.compat.mask, (uint64_t)1);
+  EXPECT_TRUE(tcs.compat.names.empty());
+  EXPECT_EQ(tcs.ro_compat.mask, (uint64_t)1);
+  EXPECT_TRUE(tcs.ro_compat.names.empty());
+  EXPECT_EQ(tcs.incompat.mask, (uint64_t)1);
+  EXPECT_TRUE(tcs.incompat.names.empty());
+}
+
+TEST(CephCompatSet, other) {
+  CompatSet s1, s2, s1dup;
+
+  s1.compat.insert(CompatSet::Feature(1, "c1"));
+  s1.compat.insert(CompatSet::Feature(2, "c2"));
+  s1.compat.insert(CompatSet::Feature(32, "c32"));
+  s1.ro_compat.insert(CompatSet::Feature(63, "r63"));
+  s1.incompat.insert(CompatSet::Feature(1, "i1"));
+
+  s2.compat.insert(CompatSet::Feature(1, "c1"));
+  s2.compat.insert(CompatSet::Feature(32, "c32"));
+  s2.ro_compat.insert(CompatSet::Feature(63, "r63"));
+  s2.incompat.insert(CompatSet::Feature(1, "i1"));
+
+  s1dup = s1;
+
+  //Check exact match
+  EXPECT_EQ(s1.compare(s1dup), 0);
+
+  //Check superset
+  EXPECT_EQ(s1.compare(s2), 1);
+
+  //Check missing features
+  EXPECT_EQ(s2.compare(s1), -1);
+
+  CompatSet diff = s2.unsupported(s1);
+  EXPECT_EQ(diff.compat.mask, (uint64_t)1<<2 | 1);
+  EXPECT_EQ(diff.ro_compat.mask, (uint64_t)1);
+  EXPECT_EQ(diff.incompat.mask, (uint64_t)1);
+
+  CompatSet s3 = s1;
+  s3.incompat.insert(CompatSet::Feature(4, "i4"));
+
+  diff = s1.unsupported(s3);
+  EXPECT_EQ(diff.compat.mask, (uint64_t)1);
+  EXPECT_EQ(diff.ro_compat.mask, (uint64_t)1);
+  EXPECT_EQ(diff.incompat.mask, (uint64_t)1<<4 | 1);
+}
+
+TEST(CephCompatSet, merge) {
+  CompatSet s1, s2, s1dup, s2dup;
+
+  s1.compat.insert(CompatSet::Feature(1, "c1"));
+  s1.compat.insert(CompatSet::Feature(2, "c2"));
+  s1.compat.insert(CompatSet::Feature(32, "c32"));
+  s1.ro_compat.insert(CompatSet::Feature(63, "r63"));
+  s1.incompat.insert(CompatSet::Feature(1, "i1"));
+
+  s1dup = s1;
+
+  s2.compat.insert(CompatSet::Feature(1, "c1"));
+  s2.compat.insert(CompatSet::Feature(32, "c32"));
+  s2.ro_compat.insert(CompatSet::Feature(1, "r1"));
+  s2.ro_compat.insert(CompatSet::Feature(63, "r63"));
+  s2.incompat.insert(CompatSet::Feature(1, "i1"));
+
+  s2dup = s2;
+
+  //Nothing to merge if they are the same
+  EXPECT_FALSE(s1.merge(s1dup));
+  EXPECT_FALSE(s2.merge(s2dup));
+
+  EXPECT_TRUE(s1.merge(s2));
+  EXPECT_EQ(s1.compat.mask, (uint64_t)1<<1 | (uint64_t)1<<2 | (uint64_t)1<<32 | 1);
+  EXPECT_EQ(s1.ro_compat.mask, (uint64_t)1<<1 | (uint64_t)1<<63 | 1);
+  EXPECT_EQ(s1.incompat.mask, (uint64_t)1<<1 | 1);
+
+  EXPECT_TRUE(s2.merge(s1dup));
+  EXPECT_EQ(s2.compat.mask, (uint64_t)1<<1 | (uint64_t)1<<2 | (uint64_t)1<<32 | 1);
+  EXPECT_EQ(s2.ro_compat.mask, (uint64_t)1<<1 | (uint64_t)1<<63 | 1);
+  EXPECT_EQ(s2.incompat.mask, (uint64_t)1<<1 | 1);
+}
diff --git a/ceph/src/test/ceph_crypto.cc b/ceph/src/test/ceph_crypto.cc
new file mode 100644
index 00000000..11d41018
--- /dev/null
+++ b/ceph/src/test/ceph_crypto.cc
@@ -0,0 +1,142 @@
+#include "common/ceph_crypto.h"
+
+#include "test/unit.h"
+
+class CryptoEnvironment: public ::testing::Environment {
+public:
+  void SetUp() {
+    ceph::crypto::init(g_ceph_context);
+  }
+};
+
+::testing::Environment* const crypto_env = ::testing::AddGlobalTestEnvironment(new CryptoEnvironment);
+
+TEST(MD5, Simple) {
+  ceph::crypto::MD5 h;
+  h.Update((const byte*)"foo", 3);
+  unsigned char digest[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  h.Final(digest);
+  int err;
+  unsigned char want_digest[CEPH_CRYPTO_MD5_DIGESTSIZE] = {
+    0xac, 0xbd, 0x18, 0xdb, 0x4c, 0xc2, 0xf8, 0x5c,
+    0xed, 0xef, 0x65, 0x4f, 0xcc, 0xc4, 0xa4, 0xd8,
+  };
+  err = memcmp(digest, want_digest, CEPH_CRYPTO_MD5_DIGESTSIZE);
+  ASSERT_EQ(0, err);
+}
+
+TEST(MD5, MultiUpdate) {
+  ceph::crypto::MD5 h;
+  h.Update((const byte*)"", 0);
+  h.Update((const byte*)"fo", 2);
+  h.Update((const byte*)"", 0);
+  h.Update((const byte*)"o", 1);
+  h.Update((const byte*)"", 0);
+  unsigned char digest[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  h.Final(digest);
+  int err;
+  unsigned char want_digest[CEPH_CRYPTO_MD5_DIGESTSIZE] = {
+    0xac, 0xbd, 0x18, 0xdb, 0x4c, 0xc2, 0xf8, 0x5c,
+    0xed, 0xef, 0x65, 0x4f, 0xcc, 0xc4, 0xa4, 0xd8,
+  };
+  err = memcmp(digest, want_digest, CEPH_CRYPTO_MD5_DIGESTSIZE);
+  ASSERT_EQ(0, err);
+}
+
+TEST(MD5, Restart) {
+  ceph::crypto::MD5 h;
+  h.Update((const byte*)"bar", 3);
+  h.Restart();
+  h.Update((const byte*)"foo", 3);
+  unsigned char digest[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  h.Final(digest);
+  int err;
+  unsigned char want_digest[CEPH_CRYPTO_MD5_DIGESTSIZE] = {
+    0xac, 0xbd, 0x18, 0xdb, 0x4c, 0xc2, 0xf8, 0x5c,
+    0xed, 0xef, 0x65, 0x4f, 0xcc, 0xc4, 0xa4, 0xd8,
+  };
+  err = memcmp(digest, want_digest, CEPH_CRYPTO_MD5_DIGESTSIZE);
+  ASSERT_EQ(0, err);
+}
+
+TEST(HMACSHA1, Simple) {
+  ceph::crypto::HMACSHA1 h((const byte*)"sekrit", 6);
+  h.Update((const byte*)"foo", 3);
+  unsigned char digest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+  h.Final(digest);
+  int err;
+  unsigned char want_digest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE] = {
+    0x04, 0xbc, 0x52, 0x66, 0xb6, 0xff, 0xad, 0xad, 0x9d, 0x57,
+    0xce, 0x13, 0xea, 0x8c, 0xf5, 0x6b, 0xf9, 0x95, 0x2f, 0xd6,
+  };
+  err = memcmp(digest, want_digest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+  ASSERT_EQ(0, err);
+}
+
+TEST(HMACSHA1, MultiUpdate) {
+  ceph::crypto::HMACSHA1 h((const byte*)"sekrit", 6);
+  h.Update((const byte*)"", 0);
+  h.Update((const byte*)"fo", 2);
+  h.Update((const byte*)"", 0);
+  h.Update((const byte*)"o", 1);
+  h.Update((const byte*)"", 0);
+  unsigned char digest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+  h.Final(digest);
+  int err;
+  unsigned char want_digest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE] = {
+    0x04, 0xbc, 0x52, 0x66, 0xb6, 0xff, 0xad, 0xad, 0x9d, 0x57,
+    0xce, 0x13, 0xea, 0x8c, 0xf5, 0x6b, 0xf9, 0x95, 0x2f, 0xd6,
+  };
+  err = memcmp(digest, want_digest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+  ASSERT_EQ(0, err);
+}
+
+TEST(HMACSHA1, Restart) {
+  ceph::crypto::HMACSHA1 h((const byte*)"sekrit", 6);
+  h.Update((const byte*)"bar", 3);
+  h.Restart();
+  h.Update((const byte*)"foo", 3);
+  unsigned char digest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+  h.Final(digest);
+  int err;
+  unsigned char want_digest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE] = {
+    0x04, 0xbc, 0x52, 0x66, 0xb6, 0xff, 0xad, 0xad, 0x9d, 0x57,
+    0xce, 0x13, 0xea, 0x8c, 0xf5, 0x6b, 0xf9, 0x95, 0x2f, 0xd6,
+  };
+  err = memcmp(digest, want_digest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+  ASSERT_EQ(0, err);
+}
+
+class ForkDeathTest : public ::testing::Test {
+ protected:
+  virtual void SetUp() {
+    // shutdown NSS so it can be reinitialized after the fork
+    ceph::crypto::shutdown();
+  }
+
+  virtual void TearDown() {
+    // undo the NSS shutdown we did in the parent process, after the
+    // test is done
+    ceph::crypto::init(g_ceph_context);
+  }
+};
+
+void do_simple_crypto() {
+  // ensure that the shutdown/fork/init sequence results in a working
+  // NSS crypto library; this function is run in the child, after the
+  // fork, and if you comment out the ceph::crypto::init, or if the
+  // trick were to fail, you would see this ending in an assert and
+  // not exit status 0
+  ceph::crypto::init(g_ceph_context);
+  ceph::crypto::MD5 h;
+  h.Update((const byte*)"foo", 3);
+  unsigned char digest[CEPH_CRYPTO_MD5_DIGESTSIZE];
+  h.Final(digest);
+  exit(0);
+}
+
+#if GTEST_HAS_DEATH_TEST
+TEST_F(ForkDeathTest, MD5) {
+  ASSERT_EXIT(do_simple_crypto(), ::testing::ExitedWithCode(0), "^$");
+}
+#endif //GTEST_HAS_DEATH_TEST
diff --git a/ceph/src/test/cli/.gitignore b/ceph/src/test/cli/.gitignore
new file mode 100644
index 00000000..d2bfb7e0
--- /dev/null
+++ b/ceph/src/test/cli/.gitignore
@@ -0,0 +1 @@
+*.t.err
diff --git a/ceph/src/test/cli/ceph-authtool/add-key-segv.t b/ceph/src/test/cli/ceph-authtool/add-key-segv.t
new file mode 100644
index 00000000..724deb0b
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/add-key-segv.t
@@ -0,0 +1,6 @@
+  $ ceph-authtool kring --create-keyring
+  creating kring
+
+  $ ceph-authtool kring --add-key 'FAKEBASE64 foo'
+  can't decode key 'FAKEBASE64 foo'
+  [1]
diff --git a/ceph/src/test/cli/ceph-authtool/add-key.t b/ceph/src/test/cli/ceph-authtool/add-key.t
new file mode 100644
index 00000000..2bfcce0b
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/add-key.t
@@ -0,0 +1,14 @@
+  $ ceph-authtool kring --create-keyring
+  creating kring
+
+  $ ceph-authtool kring --add-key 'AQAK7yxNeF+nHBAA0SgSdbs8IkJrxroDeJ6SwQ== 18446744073709551615'
+  added entity client.admin auth auth(auid = 18446744073709551615 key=AQAK7yxNeF+nHBAA0SgSdbs8IkJrxroDeJ6SwQ== with 0 caps)
+
+# cram makes matching escape-containing lines with regexps a bit ugly
+  $ ceph-authtool kring --list
+  [client.admin]
+  \tkey = AQAK7yxNeF+nHBAA0SgSdbs8IkJrxroDeJ6SwQ== (esc)
+
+  $ cat kring
+  [client.admin]
+  \tkey = AQAK7yxNeF+nHBAA0SgSdbs8IkJrxroDeJ6SwQ== (esc)
diff --git a/ceph/src/test/cli/ceph-authtool/cap-bin.t b/ceph/src/test/cli/ceph-authtool/cap-bin.t
new file mode 100644
index 00000000..50efa5c9
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/cap-bin.t
@@ -0,0 +1,6 @@
+  $ ceph-authtool kring --create-keyring --gen-key
+  creating kring
+
+  $ ceph-authtool --cap osd 'allow rx pool=swimming' kring
+  $ ceph-authtool kring --list|grep -P '^\tcaps '
+  \tcaps osd = "allow rx pool=swimming" (esc)
diff --git a/ceph/src/test/cli/ceph-authtool/cap-invalid.t b/ceph/src/test/cli/ceph-authtool/cap-invalid.t
new file mode 100644
index 00000000..c857e26a
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/cap-invalid.t
@@ -0,0 +1,12 @@
+  $ ceph-authtool kring --create-keyring --gen-key
+  creating kring
+
+# TODO is this nice?
+  $ ceph-authtool --cap osd 'broken' kring
+  $ ceph-authtool kring --list|grep -P '^\tcaps '
+  \tcaps osd = "broken" (esc)
+
+# TODO is this nice?
+  $ ceph-authtool --cap xyzzy 'broken' kring
+  $ ceph-authtool kring --list|grep -P '^\tcaps '
+  \tcaps xyzzy = "broken" (esc)
diff --git a/ceph/src/test/cli/ceph-authtool/cap-overwrite.t b/ceph/src/test/cli/ceph-authtool/cap-overwrite.t
new file mode 100644
index 00000000..30788dfd
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/cap-overwrite.t
@@ -0,0 +1,11 @@
+  $ ceph-authtool kring --create-keyring --gen-key
+  creating kring
+
+  $ ceph-authtool --cap osd 'allow rx pool=swimming' kring
+  $ ceph-authtool kring --list|grep -P '^\tcaps '
+  \tcaps osd = "allow rx pool=swimming" (esc)
+
+# TODO it seems --cap overwrites all previous caps; is this wanted?
+  $ ceph-authtool --cap mds 'allow' kring
+  $ ceph-authtool kring --list|grep -P '^\tcaps '
+  \tcaps mds = "allow" (esc)
diff --git a/ceph/src/test/cli/ceph-authtool/cap.t b/ceph/src/test/cli/ceph-authtool/cap.t
new file mode 100644
index 00000000..9681d2ac
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/cap.t
@@ -0,0 +1,11 @@
+  $ ceph-authtool kring --create-keyring --gen-key
+  creating kring
+
+  $ ceph-authtool --cap osd 'allow rx pool=swimming' kring
+  $ ceph-authtool kring --list|grep -P '^\tcaps '
+  \tcaps osd = "allow rx pool=swimming" (esc)
+
+  $ cat kring
+  [client.admin]
+  \\tkey = [a-zA-Z0-9+/]+=* \(esc\) (re)
+  \tcaps osd = "allow rx pool=swimming" (esc)
diff --git a/ceph/src/test/cli/ceph-authtool/create-gen-list-bin.t b/ceph/src/test/cli/ceph-authtool/create-gen-list-bin.t
new file mode 100644
index 00000000..bfd2a432
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/create-gen-list-bin.t
@@ -0,0 +1,17 @@
+  $ ceph-authtool kring --create-keyring
+  creating kring
+
+  $ ceph-authtool kring --list
+
+  $ ceph-authtool kring --gen-key
+
+# cram makes matching escape-containing lines with regexps a bit ugly
+  $ ceph-authtool kring --list
+  [client.admin]
+  \\tkey = [a-zA-Z0-9+/]+=* \(esc\) (re)
+
+# synonym
+  $ ceph-authtool kring -l
+  [client.admin]
+  \\tkey = [a-zA-Z0-9+/]+=* \(esc\) (re)
+
diff --git a/ceph/src/test/cli/ceph-authtool/create-gen-list.t b/ceph/src/test/cli/ceph-authtool/create-gen-list.t
new file mode 100644
index 00000000..00f4ed5e
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/create-gen-list.t
@@ -0,0 +1,20 @@
+  $ ceph-authtool kring --create-keyring
+  creating kring
+
+  $ ceph-authtool kring --list
+
+  $ ceph-authtool kring --gen-key
+
+# cram makes matching escape-containing lines with regexps a bit ugly
+  $ ceph-authtool kring --list
+  [client.admin]
+  \\tkey = [a-zA-Z0-9+/]+=* \(esc\) (re)
+
+# synonym
+  $ ceph-authtool kring -l
+  [client.admin]
+  \\tkey = [a-zA-Z0-9+/]+=* \(esc\) (re)
+
+  $ cat kring
+  [client.admin]
+  \\tkey = [a-zA-Z0-9+/]+=* \(esc\) (re)
diff --git a/ceph/src/test/cli/ceph-authtool/help.t b/ceph/src/test/cli/ceph-authtool/help.t
new file mode 100644
index 00000000..062c967a
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/help.t
@@ -0,0 +1,25 @@
+# TODO synchronize with man page
+  $ ceph-authtool --help
+  no command specified
+  usage: ceph-authtool keyringfile [OPTIONS]...
+  where the options are:
+    -l, --list                    will list all keys and capabilities present in
+                                  the keyring
+    -p, --print-key               will print an encoded key for the specified
+                                  entityname. This is suitable for the
+                                  'mount -o secret=..' argument
+    -C, --create-keyring          will create a new keyring, overwriting any
+                                  existing keyringfile
+    -g, --gen-key                 will generate a new secret key for the
+                                  specified entityname
+    --gen-print-key               will generate a new secret key without set it
+                                  to the keyringfile, prints the secret to stdout
+    --import-keyring              will import the content of a given keyring
+                                  into the keyringfile
+    -u, --set-uid                 sets the auid (authenticated user id) for the
+                                  specified entityname
+    -a, --add-key                 will add an encoded key to the keyring
+    --cap subsystem capability    will set the capability for given subsystem
+    --caps capsfile               will set all of capabilities associated with a
+                                  given key, for all subsystems
+  [1]
diff --git a/ceph/src/test/cli/ceph-authtool/list-empty-bin.t b/ceph/src/test/cli/ceph-authtool/list-empty-bin.t
new file mode 100644
index 00000000..1b465fe6
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/list-empty-bin.t
@@ -0,0 +1,5 @@
+  $ touch empty
+
+  $ ceph-authtool --list empty
+
+  $ ceph-authtool -l empty
diff --git a/ceph/src/test/cli/ceph-authtool/list-empty.t b/ceph/src/test/cli/ceph-authtool/list-empty.t
new file mode 100644
index 00000000..1b465fe6
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/list-empty.t
@@ -0,0 +1,5 @@
+  $ touch empty
+
+  $ ceph-authtool --list empty
+
+  $ ceph-authtool -l empty
diff --git a/ceph/src/test/cli/ceph-authtool/list-nonexistent-bin.t b/ceph/src/test/cli/ceph-authtool/list-nonexistent-bin.t
new file mode 100644
index 00000000..4aecd3e2
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/list-nonexistent-bin.t
@@ -0,0 +1,7 @@
+  $ ceph-authtool --list nonexistent
+  can't open nonexistent: can't open nonexistent: (2) No such file or directory
+  [1]
+
+  $ ceph-authtool -l nonexistent
+  can't open nonexistent: can't open nonexistent: (2) No such file or directory
+  [1]
diff --git a/ceph/src/test/cli/ceph-authtool/list-nonexistent.t b/ceph/src/test/cli/ceph-authtool/list-nonexistent.t
new file mode 100644
index 00000000..4aecd3e2
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/list-nonexistent.t
@@ -0,0 +1,7 @@
+  $ ceph-authtool --list nonexistent
+  can't open nonexistent: can't open nonexistent: (2) No such file or directory
+  [1]
+
+  $ ceph-authtool -l nonexistent
+  can't open nonexistent: can't open nonexistent: (2) No such file or directory
+  [1]
diff --git a/ceph/src/test/cli/ceph-authtool/manpage.t b/ceph/src/test/cli/ceph-authtool/manpage.t
new file mode 100644
index 00000000..a9e1408d
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/manpage.t
@@ -0,0 +1,53 @@
+  $ ceph-authtool
+  ceph-authtool: must specify filename
+  usage: ceph-authtool keyringfile [OPTIONS]...
+  where the options are:
+    -l, --list                    will list all keys and capabilities present in
+                                  the keyring
+    -p, --print-key               will print an encoded key for the specified
+                                  entityname. This is suitable for the
+                                  'mount -o secret=..' argument
+    -C, --create-keyring          will create a new keyring, overwriting any
+                                  existing keyringfile
+    -g, --gen-key                 will generate a new secret key for the
+                                  specified entityname
+    --gen-print-key               will generate a new secret key without set it
+                                  to the keyringfile, prints the secret to stdout
+    --import-keyring              will import the content of a given keyring
+                                  into the keyringfile
+    -u, --set-uid                 sets the auid (authenticated user id) for the
+                                  specified entityname
+    -a, --add-key                 will add an encoded key to the keyring
+    --cap subsystem capability    will set the capability for given subsystem
+    --caps capsfile               will set all of capabilities associated with a
+                                  given key, for all subsystems
+  [1]
+
+# demonstrate that manpage examples fail without config
+# TODO fix the manpage
+  $ ceph-authtool --create-keyring --name client.foo --gen-key keyring
+  creating keyring
+
+# work around the above
+  $ touch ceph.conf
+
+To create a new keyring containing a key for client.foo:
+
+  $ ceph-authtool --create-keyring --id foo --gen-key keyring
+  creating keyring
+
+  $ ceph-authtool --create-keyring --name client.foo --gen-key keyring
+  creating keyring
+
+To associate some capabilities with the key (namely, the ability to mount a Ceph filesystem):
+
+  $ ceph-authtool -n client.foo --cap mds 'allow' --cap osd 'allow rw pool=data' --cap mon 'allow r' keyring
+
+To display the contents of the keyring:
+
+  $ ceph-authtool -l keyring
+  [client.foo]
+  \\tkey = [a-zA-Z0-9+/]+=* \(esc\) (re)
+  \tcaps mds = "allow" (esc)
+  \tcaps mon = "allow r" (esc)
+  \tcaps osd = "allow rw pool=data" (esc)
diff --git a/ceph/src/test/cli/ceph-authtool/simple.t b/ceph/src/test/cli/ceph-authtool/simple.t
new file mode 100644
index 00000000..b86476a5
--- /dev/null
+++ b/ceph/src/test/cli/ceph-authtool/simple.t
@@ -0,0 +1,24 @@
+  $ ceph-authtool
+  ceph-authtool: must specify filename
+  usage: ceph-authtool keyringfile [OPTIONS]...
+  where the options are:
+    -l, --list                    will list all keys and capabilities present in
+                                  the keyring
+    -p, --print-key               will print an encoded key for the specified
+                                  entityname. This is suitable for the
+                                  'mount -o secret=..' argument
+    -C, --create-keyring          will create a new keyring, overwriting any
+                                  existing keyringfile
+    -g, --gen-key                 will generate a new secret key for the
+                                  specified entityname
+    --gen-print-key               will generate a new secret key without set it
+                                  to the keyringfile, prints the secret to stdout
+    --import-keyring              will import the content of a given keyring
+                                  into the keyringfile
+    -u, --set-uid                 sets the auid (authenticated user id) for the
+                                  specified entityname
+    -a, --add-key                 will add an encoded key to the keyring
+    --cap subsystem capability    will set the capability for given subsystem
+    --caps capsfile               will set all of capabilities associated with a
+                                  given key, for all subsystems
+  [1]
diff --git a/ceph/src/test/cli/ceph-conf/env-vs-args.t b/ceph/src/test/cli/ceph-conf/env-vs-args.t
new file mode 100644
index 00000000..76b2dec9
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/env-vs-args.t
@@ -0,0 +1,10 @@
+# we can use CEPH_CONF to override the normal configuration file location.
+  $ env CEPH_CONF=from-env ceph-conf -s foo bar
+  .* \-1 did not load config file, using default settings. (re)
+  [1]
+
+# command-line arguments should override environment
+  $ env -u CEPH_CONF ceph-conf -c from-args
+  global_init: unable to open config file from search list from-args
+  [1]
+
diff --git a/ceph/src/test/cli/ceph-conf/help.t b/ceph/src/test/cli/ceph-conf/help.t
new file mode 100644
index 00000000..3fe8ca3f
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/help.t
@@ -0,0 +1,36 @@
+  $ ceph-conf --help
+  Ceph configuration query tool
+  
+  USAGE
+  ceph-conf  
+  
+  ACTIONS
+    -L|--list-all-sections          List all sections
+    -l|--list-sections      List sections with the given prefix
+    --filter-key               Filter section list to only include sections
+                                    with given key defined.
+    --filter-key-value =  Filter section list to only include sections
+                                    with given key/value pair.
+    --lookup                   Print a configuration setting to stdout.
+                                    Returns 0 (success) if the configuration setting is
+                                    found; 1 otherwise.
+    -r|--resolve-search             search for the first file that exists and
+                                    can be opened in the resulted comma
+                                    delimited search list.
+  
+  FLAGS
+    --name name                     Set type.id
+    [-s 
]                  Add to list of sections to search
+  
+  If there is no action given, the action will default to --lookup.
+  
+  EXAMPLES
+  [$] ceph-conf --name mon.0 -c /etc/ceph/ceph.conf 'mon addr' (re)
+  Find out what the value of 'mon add' is for monitor 0.
+  
+  [$] ceph-conf -l mon (re)
+  List sections beginning with 'mon'.
+  
+  RETURN CODE
+  Return code will be 0 on success; error code otherwise.
+  [1]
diff --git a/ceph/src/test/cli/ceph-conf/invalid-args.t b/ceph/src/test/cli/ceph-conf/invalid-args.t
new file mode 100644
index 00000000..d0fe553e
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/invalid-args.t
@@ -0,0 +1,17 @@
+  $ cat >test.conf < [bar]
+  > bar = green
+  > EOF
+
+# TODO output an error
+  $ ceph-conf -c test.conf broken
+  [1]
+
+  $ ceph-conf -c test.conf --name total.garbage
+  error parsing 'total.garbage': expected string of the form TYPE.ID, valid types are: auth, mon, osd, mds, client
+  [1]
+
+  $ ceph-conf -c test.conf -s bar
+  You must give an action, such as --lookup or --list-all-sections.
+  Pass --help for more help.
+  [1]
diff --git a/ceph/src/test/cli/ceph-conf/manpage.t b/ceph/src/test/cli/ceph-conf/manpage.t
new file mode 100644
index 00000000..425f2718
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/manpage.t
@@ -0,0 +1,33 @@
+# setup
+  $ cat >foo.conf <<'EOF'
+  > ; ---------------------
+  > [group cephnet]
+  > 	addr = 10.3.14.0/24
+  > 
+  > [global]
+  > 	pid file = /home/sage/ceph/src/out/$name.pid
+  > 
+  > [osd]
+  > 	osd data = /mnt/osd$id
+  > [osd.3]
+  > 	host = cosd3
+  > EOF
+
+To extract the value of the "osd data" option for the osd0 daemon,
+
+  $ ceph-conf -c foo.conf "osd data" --name osd.0
+  /mnt/osd0
+
+This is equivalent to doing specifying sections [osd0], [osd.0],
+[osd], or [global], in that order of preference:
+
+# TODO the "admin" here seems like an actual bug
+
+  $ ceph-conf -c foo.conf "osd data" -s osd0 -s osd.0 -s osd -s global
+  /mnt/osdadmin
+
+To list all sections that begin with osd:
+
+  $ ceph-conf -c foo.conf -l osd
+  osd
+  osd.3
diff --git a/ceph/src/test/cli/ceph-conf/option.t b/ceph/src/test/cli/ceph-conf/option.t
new file mode 100644
index 00000000..a8b5b6e3
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/option.t
@@ -0,0 +1,67 @@
+  $ cat >test.conf < [bar]
+  > bar = green
+  > [foo]
+  > bar = blue
+  > [baz]
+  > bar = yellow
+  > [thud]
+  > bar = red
+  > [nobar]
+  > other = 42
+  > EOF
+
+  $ ceph-conf -c test.conf bar -s foo
+  blue
+
+# test the funny "equals sign" argument passing convention
+  $ ceph-conf --conf=test.conf bar -s foo
+  blue
+
+  $ ceph-conf --conf=test.conf -L
+  bar
+  baz
+  foo
+  global
+  nobar
+  thud
+
+  $ ceph-conf --conf=test.conf --list-all-sections
+  bar
+  baz
+  foo
+  global
+  nobar
+  thud
+
+  $ ceph-conf --conf=test.conf --list_all_sections
+  bar
+  baz
+  foo
+  global
+  nobar
+  thud
+
+# TODO man page stops in the middle of a sentence
+
+  $ ceph-conf -c test.conf bar -s xyzzy
+  [1]
+
+  $ ceph-conf -c test.conf bar -s xyzzy
+  [1]
+
+  $ ceph-conf -c test.conf bar -s xyzzy -s thud
+  red
+
+  $ ceph-conf -c test.conf bar -s nobar -s thud
+  red
+
+  $ ceph-conf -c test.conf bar -s thud -s baz
+  red
+
+  $ ceph-conf -c test.conf bar -s baz -s thud
+  yellow
+
+  $ ceph-conf -c test.conf bar -s xyzzy -s nobar -s thud -s baz
+  red
+
diff --git a/ceph/src/test/cli/ceph-conf/sections.t b/ceph/src/test/cli/ceph-conf/sections.t
new file mode 100644
index 00000000..63063cc9
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/sections.t
@@ -0,0 +1,18 @@
+  $ cat >test.conf < [bar]
+  > bar = green
+  > [foo]
+  > bar = blue
+  > [baz]
+  > bar = yellow
+  > [thud]
+  > bar = yellow
+  > EOF
+
+  $ ceph-conf -c test.conf -l bar
+  bar
+
+  $ ceph-conf -c test.conf -l b
+  bar
+  baz
+
diff --git a/ceph/src/test/cli/ceph-conf/show-config-value.t b/ceph/src/test/cli/ceph-conf/show-config-value.t
new file mode 100644
index 00000000..f2f82045
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/show-config-value.t
@@ -0,0 +1,20 @@
+
+# should reflect daemon defaults
+
+  $ ceph-conf -n osd.0 --show-config-value log_file -c /dev/null
+  /var/log/ceph/ceph-osd.0.log
+  $ CEPH_ARGS="--fsid 96a3abe6-7552-4635-a79b-f3c096ff8b95" ceph-conf -n osd.0 --show-config-value fsid -c /dev/null
+  96a3abe6-7552-4635-a79b-f3c096ff8b95
+  $ ceph-conf -n osd.0 --show-config-value INVALID -c /dev/null
+  failed to get config option 'INVALID': option not found
+  [1]
+  $ echo '[global]' > $TESTDIR/ceph.conf
+  $ echo 'mon_host=$public_network' >> $TESTDIR/ceph.conf
+  $ echo 'public_network=$mon_host' >> $TESTDIR/ceph.conf
+  $ ceph-conf --show-config-value mon_host -c $TESTDIR/ceph.conf
+  variable expansion loop at public_network=$mon_host
+  expansion stack: 
+  mon_host=$public_network
+  public_network=$mon_host
+  $mon_host
+  $ rm $TESTDIR/ceph.conf
diff --git a/ceph/src/test/cli/ceph-conf/show-config.t b/ceph/src/test/cli/ceph-conf/show-config.t
new file mode 100644
index 00000000..cfd72399
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/show-config.t
@@ -0,0 +1,6 @@
+  $ ceph-conf -n osd.0 --show-config -c /dev/null | grep ceph-osd
+  admin_socket = /var/run/ceph/ceph-osd.0.asok
+  log_file = /var/log/ceph/ceph-osd.0.log
+  mon_debug_dump_location = /var/log/ceph/ceph-osd.0.tdump
+  $ CEPH_ARGS="--fsid 96a3abe6-7552-4635-a79b-f3c096ff8b95" ceph-conf -n osd.0 --show-config -c /dev/null | grep fsid
+  fsid = 96a3abe6-7552-4635-a79b-f3c096ff8b95
diff --git a/ceph/src/test/cli/ceph-conf/simple.t b/ceph/src/test/cli/ceph-conf/simple.t
new file mode 100644
index 00000000..043ca203
--- /dev/null
+++ b/ceph/src/test/cli/ceph-conf/simple.t
@@ -0,0 +1,4 @@
+  $ ceph-conf
+  You must give an action, such as --lookup or --list-all-sections.
+  Pass --help for more help.
+  [1]
diff --git a/ceph/src/test/cli/crushtool/add-item.t b/ceph/src/test/cli/crushtool/add-item.t
new file mode 100644
index 00000000..f058b639
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/add-item.t
@@ -0,0 +1,15 @@
+  $ crushtool -i "$TESTDIR/simple.template" --add-item 0 1.0 device0 --loc host host0 --loc cluster cluster0 -o one > /dev/null
+  $ crushtool -i one --add-item 1 1.0 device1 --loc host host0 --loc cluster cluster0 -o two > /dev/null
+  $ crushtool -d two -o final
+  $ cmp final "$TESTDIR/simple.template.two"
+  $ crushtool -i two --add-item 1 1.0 device1 --loc host host0 --loc cluster cluster0 -o three 2>/dev/null >/dev/null || echo FAIL
+  FAIL
+  $ crushtool -i two --remove-item device1 -o four > /dev/null
+  $ crushtool -d four -o final
+  $ cmp final "$TESTDIR/simple.template.four"
+  $ crushtool -i two --update-item 1 2.0 osd1 --loc host host1 --loc cluster cluster0 -o five > /dev/null
+  $ crushtool -d five -o final
+  $ cmp final "$TESTDIR/simple.template.five"
+  $ crushtool -i five --update-item 1 2.0 osd1 --loc host host1 --loc cluster cluster0 -o six > /dev/null
+  $ crushtool -d six -o final
+  $ cmp final "$TESTDIR/simple.template.five"
diff --git a/ceph/src/test/cli/crushtool/bad-mappings.crushmap.txt b/ceph/src/test/cli/crushtool/bad-mappings.crushmap.txt
new file mode 100644
index 00000000..a3d5dfee
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/bad-mappings.crushmap.txt
@@ -0,0 +1,39 @@
+device 0 device0
+device 1 device1
+device 2 device2
+device 3 device3
+device 4 device4
+
+type 0 osd
+type 1 domain
+
+domain root {
+	id -1
+	alg straw
+	hash 0
+	item device0 weight 1.000
+	item device1 weight 1.000
+	item device2 weight 1.000
+	item device3 weight 1.000
+	item device4 weight 1.000
+}
+
+rule rule-firstn {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take root
+	step choose firstn 0 type osd
+	step emit
+}
+
+rule rule-indep {
+	ruleset 1
+	type erasure
+	min_size 1
+	max_size 10
+	step take root
+	step choose indep 0 type osd
+	step emit
+}
diff --git a/ceph/src/test/cli/crushtool/bad-mappings.t b/ceph/src/test/cli/crushtool/bad-mappings.t
new file mode 100644
index 00000000..dbdcd576
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/bad-mappings.t
@@ -0,0 +1,6 @@
+  $ crushtool -c "$TESTDIR/bad-mappings.crushmap.txt" -o "$TESTDIR/bad-mappings.crushmap"
+  $ crushtool -i "$TESTDIR/bad-mappings.crushmap" --test --show-bad-mappings --rule 0 --x 1 --num-rep 10
+  bad mapping rule 0 x 1 num_rep 10 result [4,0,2,3,1]
+  $ crushtool -i "$TESTDIR/bad-mappings.crushmap" --test --show-bad-mappings --rule 1 --x 1 --num-rep 10
+  bad mapping rule 1 x 1 num_rep 10 result [4,0,2,1,3,2147483647,2147483647,2147483647,2147483647,2147483647]
+  $ rm -f "$TESTDIR/bad-mappings.crushmap"
diff --git a/ceph/src/test/cli/crushtool/build.t b/ceph/src/test/cli/crushtool/build.t
new file mode 100644
index 00000000..ca0804de
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/build.t
@@ -0,0 +1,104 @@
+  $ map="$TESTDIR/build.crushmap"
+
+#
+# display the crush tree by default
+#
+  $ crushtool --outfn "$map" --build --num_osds 5 node straw 2 rack straw 1 root straw 0
+  .* (re)
+  # id\tweight\ttype name\treweight (esc)
+  -7\t5\troot root (esc)
+  -4\t2\t\track rack0 (esc)
+  -1\t2\t\t\tnode node0 (esc)
+  0\t1\t\t\t\tosd.0\t1\t (esc)
+  1\t1\t\t\t\tosd.1\t1\t (esc)
+  -5\t2\t\track rack1 (esc)
+  -2\t2\t\t\tnode node1 (esc)
+  2\t1\t\t\t\tosd.2\t1\t (esc)
+  3\t1\t\t\t\tosd.3\t1\t (esc)
+  -6\t1\t\track rack2 (esc)
+  -3\t1\t\t\tnode node2 (esc)
+  4\t1\t\t\t\tosd.4\t1\t (esc)
+  
+
+#  
+# silence all messages with --debug-crush 0
+#
+  $ CEPH_ARGS="--debug-crush 0" crushtool --outfn "$map" --build --num_osds 5 node straw 2 rack straw 1 root straw 0
+
+#
+# display a warning if there is more than one root
+#
+  $ crushtool --outfn "$map" --build --num_osds 5 node straw 2 rack straw 1 
+  .* (re)
+  # id\tweight\ttype name\treweight (esc)
+  -6\t1\track rack2 (esc)
+  -3\t1\t\tnode node2 (esc)
+  4\t1\t\t\tosd.4\t1\t (esc)
+  -5\t2\track rack1 (esc)
+  -2\t2\t\tnode node1 (esc)
+  2\t1\t\t\tosd.2\t1\t (esc)
+  3\t1\t\t\tosd.3\t1\t (esc)
+  -4\t2\track rack0 (esc)
+  -1\t2\t\tnode node0 (esc)
+  0\t1\t\t\tosd.0\t1\t (esc)
+  1\t1\t\t\tosd.1\t1\t (esc)
+  
+  .* The crush rulesets will use the root rack0 (re)
+  and ignore the others.
+  There are 3 roots, they can be
+  grouped into a single root by appending something like:
+    root straw 0
+  
+#
+# crush rulesets are generated using the OSDMap helpers
+#
+  $ CEPH_ARGS="--debug-crush 0" crushtool --outfn "$map" --build --num_osds 1 root straw 0
+  $ crushtool -o "$map.txt" -d "$map"
+  $ cat "$map.txt"
+  # begin crush map
+  tunable choose_local_tries 0
+  tunable choose_local_fallback_tries 0
+  tunable choose_total_tries 50
+  tunable chooseleaf_descend_once 1
+  
+  # devices
+  device 0 device0
+  
+  # types
+  type 0 device
+  type 1 root
+  
+  # buckets
+  root root {
+  \tid -1\t\t# do not change unnecessarily (esc)
+  \t# weight 1.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem device0 weight 1.000 (esc)
+  }
+  
+  # rules
+  rule replicated_ruleset {
+  \truleset 0 (esc)
+  \ttype replicated (esc)
+  \tmin_size 1 (esc)
+  \tmax_size 10 (esc)
+  \tstep take root (esc)
+  \tstep chooseleaf firstn 0 type root (esc)
+  \tstep emit (esc)
+  }
+  
+  # end crush map
+  $ rm "$map" "$map.txt"
+
+#
+# Wrong number of arguments 
+#
+  $ crushtool --outfn "$map" --debug-crush 0 --build --num_osds 5 node straw 0
+  remaining args: [--debug-crush,0,node,straw,0]
+  layers must be specified with 3-tuples of (name, buckettype, size)
+  [1]
+
+# Local Variables:
+# compile-command: "cd ../../.. ; make crushtool && test/run-cli-tests"
+# End:
diff --git a/ceph/src/test/cli/crushtool/compile-decompile-recompile.t b/ceph/src/test/cli/crushtool/compile-decompile-recompile.t
new file mode 100644
index 00000000..67247563
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/compile-decompile-recompile.t
@@ -0,0 +1,11 @@
+  $ cp "$TESTDIR/need_tree_order.crush" .
+  $ crushtool -c need_tree_order.crush -o nto.compiled
+  $ crushtool -d nto.compiled -o nto.conf
+  $ crushtool -c nto.conf -o nto.recompiled
+
+# as the input file is actually exactly what decompilation will spit
+# back out, comments and all, and the compiled format is completely
+# deterministic, we can compare the files to make sure everything
+# worked
+  $ cmp need_tree_order.crush nto.conf
+  $ cmp nto.compiled nto.recompiled
diff --git a/ceph/src/test/cli/crushtool/five-devices.crushmap b/ceph/src/test/cli/crushtool/five-devices.crushmap
new file mode 100644
index 0000000000000000000000000000000000000000..ced76fc37a7d821c207b75a5a151fe9ce8b16065
GIT binary patch
literal 368
zcmb7=Q47K_41_P66WPD4)Ca*Cnf?EouO)AS8n`x>UV0&t=_F|mEP+Gd6=6MFdM!$!
zlBNJUG^0ZG8@l-8K_~yoY$eARiciMy8Z7npr+)fT-)9LcZhuznj)Cf1x!xIyVIJ 1 rounds
+        [--weight|-w devno weight]
+                           where weight is 0 to 1.0
+        [--simulate]       simulate placements using a random
+                           number generator in place of the CRUSH
+                           algorithm
+     -i mapfn --add-item id weight name [--loc type name ...]
+                           insert an item into the hierarchy at the
+                           given location
+     -i mapfn --update-item id weight name [--loc type name ...]
+                           insert or move an item into the hierarchy at the
+                           given location
+     -i mapfn --remove-item name
+                           remove the given item
+     -i mapfn --reweight-item name weight
+                           reweight a given item (and adjust ancestor
+                           weights as needed)
+     -i mapfn --reweight   recalculate all bucket weights
+     --show-utilization    show OSD usage
+     --show utilization-all
+                           include zero weight items
+     --show-statistics     show chi squared statistics
+     --show-bad-mappings   show bad mappings
+     --show-choose-tries   show choose tries histogram
+     --set-choose-local-tries N
+                           set choose local retries before re-descent
+     --set-choose-local-fallback-tries N
+                           set choose local retries using fallback
+                           permutation before re-descent
+     --set-choose-total-tries N
+                           set choose total descent attempts
+     --set-chooseleaf-descend-once <0|1>
+                           set chooseleaf to (not) retry the recursive descent
+     --set-chooseleaf-vary-r <0|1>
+                           set chooseleaf to (not) vary r based on parent
+     --output-name name
+                           prepend the data file(s) generated during the
+                           testing routine with name
+     --output-csv
+                           export select data generated during testing routine
+                           to CSV files for off-line post-processing
+                           use --help-output for more information
+  $ crushtool --help-output
+  data output from testing routine ...
+            absolute_weights
+                  the decimal weight of each OSD
+                  data layout: ROW MAJOR
+                               OSD id (int), weight (int)
+             batch_device_expected_utilization_all
+                   the expected number of objects each OSD should receive per placement batch
+                   which may be a decimal value
+                   data layout: COLUMN MAJOR
+                                round (int), objects expected on OSD 0...OSD n (float)
+             batch_device_utilization_all
+                   the number of objects stored on each OSD during each placement round
+                   data layout: COLUMN MAJOR
+                                round (int), objects stored on OSD 0...OSD n (int)
+             device_utilization_all
+                    the number of objects stored on each OSD at the end of placements
+                    data_layout: ROW MAJOR
+                                 OSD id (int), objects stored (int), objects expected (float)
+             device_utilization
+                    the number of objects stored on each OSD marked 'up' at the end of placements
+                    data_layout: ROW MAJOR
+                                 OSD id (int), objects stored (int), objects expected (float)
+             placement_information
+                    the map of input -> OSD
+                    data_layout: ROW MAJOR
+                                 input (int), OSD's mapped (int)
+             proportional_weights_all
+                    the proportional weight of each OSD specified in the CRUSH map
+                    data_layout: ROW MAJOR
+                                 OSD id (int), proportional weight (float)
+             proportional_weights
+                    the proportional weight of each 'up' OSD specified in the CRUSH map
+                    data_layout: ROW MAJOR
+                                 OSD id (int), proportional weight (float)
diff --git a/ceph/src/test/cli/crushtool/multitype.after b/ceph/src/test/cli/crushtool/multitype.after
new file mode 100644
index 00000000..9c7da0aa
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/multitype.after
@@ -0,0 +1,87 @@
+# begin crush map
+
+# devices
+device 0 osd0
+device 1 osd1
+device 2 osd2
+device 3 osd3
+device 4 osd4
+device 5 osd5
+device 6 osd6
+device 7 osd7
+device 8 osd8
+device 9 osd9
+
+# types
+type 0 osd
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item osd0 weight 2.000
+	item osd1 weight 1.000
+}
+host host1 {
+	id -3		# do not change unnecessarily
+	# weight 3.000
+	alg list	# add new items at the end; do not change order unnecessarily
+	hash 0	# rjenkins1
+	item osd3 weight 2.000
+	item osd4 weight 1.000
+}
+host host2 {
+	id -4		# do not change unnecessarily
+	# weight 5.500
+	alg tree	# do not change pos for existing items unnecessarily
+	hash 0	# rjenkins1
+	item osd5 weight 1.000 pos 0
+	item osd6 weight 2.000 pos 1
+	item osd7 weight 0.500 pos 2
+	item osd8 weight 1.000 pos 3
+	item osd9 weight 1.000 pos 4
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 11.500
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 3.000
+	item host1 weight 3.000
+	item host2 weight 5.500
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/multitype.before b/ceph/src/test/cli/crushtool/multitype.before
new file mode 100644
index 00000000..8adcda9c
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/multitype.before
@@ -0,0 +1,87 @@
+# begin crush map
+
+# devices
+device 0 osd0
+device 1 osd1
+device 2 osd2
+device 3 osd3
+device 4 osd4
+device 5 osd5
+device 6 osd6
+device 7 osd7
+device 8 osd8
+device 9 osd9
+
+# types
+type 0 osd
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item osd0 weight 1.000
+	item osd1 weight 1.000
+}
+host host1 {
+	id -3		# do not change unnecessarily
+	# weight 2.000
+	alg list
+	hash 0	# rjenkins1
+	item osd3 weight 1.000
+	item osd4 weight 1.000
+}
+host host2 {
+	id -4		# do not change unnecessarily
+	# weight 2.000
+	alg tree
+	hash 0	# rjenkins1
+	item osd5 weight 1.000
+	item osd6 weight 1.000
+	item osd7 weight 1.000
+	item osd8 weight 1.000
+	item osd9 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 2.000
+	item host1 weight 2.000
+	item host2 weight 5.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/need_tree_order.crush b/ceph/src/test/cli/crushtool/need_tree_order.crush
new file mode 100644
index 00000000..d7da94e9
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/need_tree_order.crush
@@ -0,0 +1,64 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 device1
+device 2 device2
+
+# types
+type 0 device
+type 1 host
+type 2 rack
+type 3 root
+
+# buckets
+host host0 {
+	id -1		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+host host1 {
+	id -2		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device1 weight 1.000
+}
+host host2 {
+	id -5		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device2 weight 1.000
+}
+rack rack0 {
+	id -3		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 1.000
+	item host2 weight 1.000
+}
+root root {
+	id -4		# do not change unnecessarily
+	# weight 4.000
+	alg straw
+	hash 0	# rjenkins1
+	item rack0 weight 4.000
+}
+
+# rules
+rule data {
+	ruleset 1
+	type replicated
+	min_size 2
+	max_size 2
+	step take root
+	step chooseleaf firstn 0 type rack
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/output-csv.t b/ceph/src/test/cli/crushtool/output-csv.t
new file mode 100644
index 00000000..ad995974
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/output-csv.t
@@ -0,0 +1,51 @@
+# first test that CSV files are created for each ruleset 
+$ crushtool -i five-devices.crushmap --test --num-rep 1 --min-x 0 --max-x 9 --output-csv
+$ if [ ! -f data-absolute_weights.csv ]; then echo FAIL; fi
+$ if [ ! -f data-batch_device_expected_utilization_all.csv ]; then echo FAIL; fi 
+$ if [ ! -f data-batch_device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f data-device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f data-device_utilization.csv ]; then echo FAIL; fi
+$ if [ ! -f data-placement_information.csv ]; then echo FAIL; fi
+$ if [ ! -f data-proportional_weights_all.csv ]; then echo FAIL; fi
+$ if [ ! -f data-proportional_weights.csv ]; then echo FAIL; fi
+$ if [ ! -f metadata-absolute_weights.csv ]; then echo FAIL; fi
+$ if [ ! -f metadata-batch_device_expected_utilization_all.csv ]; then echo FAIL; fi 
+$ if [ ! -f metadata-batch_device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f metadata-device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f metadata-device_utilization.csv ]; then echo FAIL; fi
+$ if [ ! -f metadata-placement_information.csv ]; then echo FAIL; fi
+$ if [ ! -f metadata-proportional_weights_all.csv ]; then echo FAIL; fi
+$ if [ ! -f metadata-proportional_weights.csv ]; then echo FAIL; fi
+$ if [ ! -f rbd-absolute_weights.csv ]; then echo FAIL; fi
+$ if [ ! -f rbd-batch_device_expected_utilization_all.csv ]; then echo FAIL; fi 
+$ if [ ! -f rbd-batch_device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f rbd-device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f rbd-device_utilization.csv ]; then echo FAIL; fi
+$ if [ ! -f rbd-placement_information.csv ]; then echo FAIL; fi
+$ if [ ! -f rbd-proportional_weights_all.csv ]; then echo FAIL; fi
+$ if [ ! -f rbd-proportional_weights.csv ]; then echo FAIL; fi
+$ rm data*csv
+$ rm metadata*csv
+$ rm rbd*csv
+# now check that the CSV files are made to the proper length
+$ crushtool -i five-devices.crushmap --test --rule 0 --num-rep 1 --min-x 0 --max-x 9 --output-csv
+$ if [ $(wc -l data-absolute_weights.csv | awk '{print $1}') != "5" ]; then echo FAIL; fi
+$ if [ $(wc -l data-batch_device_expected_utilization_all.csv | awk '{print $1}') != "5" ]; then echo FAIL; fi
+$ if [ $(wc -l data-batch_device_utilization_all.csv | awk '{print $1}') != "5" ]; then echo FAIL; fi
+$ if [ $(wc -l data-device_utilization_all.csv | awk '{print $1}') != "5" ]; then echo FAIL; fi
+$ if [ $(wc -l data-device_utilization.csv | awk '{print $1}') != "5" ]; then echo FAIL; fi
+$ if [ $(wc -l data-placement_information.csv | awk '{print $1}') != "10" ]; then echo FAIL; fi
+$ if [ $(wc -l data-proportional_weights_all.csv | awk '{print $1}') != "5" ]; then echo FAIL; fi
+$ if [ $(wc -l data-proportional_weights.csv | awk '{print $1}') != "5" ]; then echo FAIL; fi
+$ rm data*csv
+# finally check that user supplied tags are prepended correctly
+$ crushtool -i five-devices.crushmap --test --rule 0 --num-rep 1 --min-x 0 --max-x 9 --output-name "test-tag" --output-csv
+$ if [ ! -f test-tag-data-absolute_weights.csv ]; then echo FAIL; fi
+$ if [ ! -f test-tag-data-batch_device_expected_utilization_all.csv ]; then echo FAIL; fi 
+$ if [ ! -f test-tag-data-batch_device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f test-tag-data-device_utilization_all.csv ]; then echo FAIL; fi
+$ if [ ! -f test-tag-data-device_utilization.csv ]; then echo FAIL; fi
+$ if [ ! -f test-tag-data-placement_information.csv ]; then echo FAIL; fi
+$ if [ ! -f test-tag-data-proportional_weights_all.csv ]; then echo FAIL; fi
+$ if [ ! -f test-tag-data-proportional_weights.csv ]; then echo FAIL; fi
+$ rm test-tag*csv
diff --git a/ceph/src/test/cli/crushtool/reweight.t b/ceph/src/test/cli/crushtool/reweight.t
new file mode 100644
index 00000000..f4da5c9c
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/reweight.t
@@ -0,0 +1,8 @@
+  $ crushtool -c "$TESTDIR/multitype.before" -o mt > /dev/null
+  $ crushtool -i mt --reweight-item osd0 2.0 -o mt > /dev/null
+  $ crushtool -i mt --reweight-item osd3 2.0 -o mt > /dev/null
+  $ crushtool -i mt --reweight-item osd6 2.0 -o mt > /dev/null
+  $ crushtool -i mt --reweight-item osd7 .5 -o mt > /dev/null
+  $ crushtool -d mt -o final
+  $ diff final "$TESTDIR/multitype.after"
+  $ rm mt final
diff --git a/ceph/src/test/cli/crushtool/reweight_multiple.t b/ceph/src/test/cli/crushtool/reweight_multiple.t
new file mode 100644
index 00000000..7eff6733
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/reweight_multiple.t
@@ -0,0 +1,5 @@
+  $ crushtool -c "$TESTDIR/simple.template.multitree" -o mt
+  $ crushtool -i mt --reweight-item osd1 2.5 -o mt
+  crushtool reweighting item osd1 to 2.5
+  $ crushtool -d mt -o mt.txt
+  $ diff mt.txt "$TESTDIR/simple.template.multitree.reweighted"
diff --git a/ceph/src/test/cli/crushtool/set-choose.crushmap.txt b/ceph/src/test/cli/crushtool/set-choose.crushmap.txt
new file mode 100644
index 00000000..bc890e02
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/set-choose.crushmap.txt
@@ -0,0 +1,144 @@
+# begin crush map
+tunable choose_local_tries 1
+tunable choose_local_fallback_tries 2
+tunable choose_total_tries 35
+tunable chooseleaf_descend_once 1
+
+# devices
+device 0 device0
+device 1 device1
+device 2 device2
+device 3 device3
+device 4 device4
+device 5 device5
+device 6 device6
+device 7 device7
+device 8 device8
+
+# types
+type 0 device
+type 1 host
+type 2 rack
+type 3 root
+
+# buckets
+host host0 {
+	id -1		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+	item device1 weight 1.000
+	item device2 weight 1.000
+}
+host host1 {
+	id -2		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item device3 weight 1.000
+	item device4 weight 1.000
+	item device5 weight 1.000
+}
+host host2 {
+	id -3		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item device6 weight 1.000
+	item device7 weight 1.000
+	item device8 weight 1.000
+}
+rack rack0 {
+	id -4		# do not change unnecessarily
+	# weight 9.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 3.000
+	item host1 weight 3.000
+	item host2 weight 3.000
+}
+root root0 {
+	id -5		# do not change unnecessarily
+	# weight 9.000
+	alg straw
+	hash 0	# rjenkins1
+	item rack0 weight 9.000
+}
+
+# rules
+rule choose {
+	ruleset 1
+	type replicated
+	min_size 2
+	max_size 3
+	step take root0
+	step choose firstn 0 type host
+	step choose firstn 1 type device
+	step emit
+}
+
+rule choose-two {
+	ruleset 2
+	type replicated
+	min_size 2
+	max_size 3
+	step take root0
+	step choose firstn 0 type device
+	step emit
+}
+
+rule chooseleaf {
+	ruleset 3
+	type replicated
+	min_size 2
+	max_size 3
+	step take root0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+rule choose-set {
+	ruleset 4
+	type replicated
+	min_size 2
+	max_size 3
+	step set_choose_tries 3
+	step set_choose_local_tries 2
+	step set_choose_local_fallback_tries 2
+	step set_chooseleaf_tries 3
+	step take root0
+	step choose firstn 0 type host
+	step choose firstn 1 type device
+	step emit
+}
+
+rule choose-set-two {
+	ruleset 5
+	type replicated
+	min_size 2
+	max_size 3
+	step set_choose_tries 3
+	step set_choose_local_tries 2
+	step set_choose_local_fallback_tries 2
+	step set_chooseleaf_tries 3
+	step take root0
+	step choose firstn 0 type device
+	step emit
+}
+
+rule chooseleaf-set {
+	ruleset 6
+	type replicated
+	min_size 2
+	max_size 3
+	step set_choose_tries 3
+	step set_choose_local_tries 2
+	step set_choose_local_fallback_tries 2
+	step set_chooseleaf_tries 3
+	step take root0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/set-choose.t b/ceph/src/test/cli/crushtool/set-choose.t
new file mode 100644
index 00000000..e160ad78
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/set-choose.t
@@ -0,0 +1,36941 @@
+  $ crushtool -c "$TESTDIR/set-choose.crushmap.txt" -o set-choose.crushmap
+  $ crushtool -i set-choose.crushmap --test --show-statistics
+  rule 0 (choose), x = 0..1023, numrep = 2..3
+  CRUSH rule 0 x 0 [0,3]
+  CRUSH rule 0 x 1 [0,8]
+  CRUSH rule 0 x 2 [1,4]
+  CRUSH rule 0 x 3 [8,0]
+  CRUSH rule 0 x 4 [5,1]
+  CRUSH rule 0 x 5 [7,0]
+  CRUSH rule 0 x 6 [2,6]
+  CRUSH rule 0 x 7 [5,6]
+  CRUSH rule 0 x 8 [5,7]
+  CRUSH rule 0 x 9 [2,4]
+  CRUSH rule 0 x 10 [0,8]
+  CRUSH rule 0 x 11 [0,6]
+  CRUSH rule 0 x 12 [0,3]
+  CRUSH rule 0 x 13 [3,8]
+  CRUSH rule 0 x 14 [7,1]
+  CRUSH rule 0 x 15 [7,1]
+  CRUSH rule 0 x 16 [3,7]
+  CRUSH rule 0 x 17 [5,1]
+  CRUSH rule 0 x 18 [1,3]
+  CRUSH rule 0 x 19 [7,5]
+  CRUSH rule 0 x 20 [2,4]
+  CRUSH rule 0 x 21 [3,6]
+  CRUSH rule 0 x 22 [8,5]
+  CRUSH rule 0 x 23 [3,7]
+  CRUSH rule 0 x 24 [1,6]
+  CRUSH rule 0 x 25 [3,8]
+  CRUSH rule 0 x 26 [2,7]
+  CRUSH rule 0 x 27 [3,1]
+  CRUSH rule 0 x 28 [6,2]
+  CRUSH rule 0 x 29 [8,5]
+  CRUSH rule 0 x 30 [5,6]
+  CRUSH rule 0 x 31 [8,2]
+  CRUSH rule 0 x 32 [3,7]
+  CRUSH rule 0 x 33 [2,7]
+  CRUSH rule 0 x 34 [2,3]
+  CRUSH rule 0 x 35 [0,6]
+  CRUSH rule 0 x 36 [3,6]
+  CRUSH rule 0 x 37 [0,4]
+  CRUSH rule 0 x 38 [4,6]
+  CRUSH rule 0 x 39 [3,7]
+  CRUSH rule 0 x 40 [7,2]
+  CRUSH rule 0 x 41 [0,7]
+  CRUSH rule 0 x 42 [4,7]
+  CRUSH rule 0 x 43 [0,3]
+  CRUSH rule 0 x 44 [1,8]
+  CRUSH rule 0 x 45 [8,2]
+  CRUSH rule 0 x 46 [2,4]
+  CRUSH rule 0 x 47 [4,1]
+  CRUSH rule 0 x 48 [4,6]
+  CRUSH rule 0 x 49 [5,6]
+  CRUSH rule 0 x 50 [3,1]
+  CRUSH rule 0 x 51 [3,6]
+  CRUSH rule 0 x 52 [8,2]
+  CRUSH rule 0 x 53 [3,6]
+  CRUSH rule 0 x 54 [7,4]
+  CRUSH rule 0 x 55 [8,0]
+  CRUSH rule 0 x 56 [6,5]
+  CRUSH rule 0 x 57 [5,8]
+  CRUSH rule 0 x 58 [1,8]
+  CRUSH rule 0 x 59 [4,0]
+  CRUSH rule 0 x 60 [3,1]
+  CRUSH rule 0 x 61 [4,8]
+  CRUSH rule 0 x 62 [7,0]
+  CRUSH rule 0 x 63 [5,6]
+  CRUSH rule 0 x 64 [4,1]
+  CRUSH rule 0 x 65 [7,4]
+  CRUSH rule 0 x 66 [5,6]
+  CRUSH rule 0 x 67 [5,0]
+  CRUSH rule 0 x 68 [0,3]
+  CRUSH rule 0 x 69 [5,2]
+  CRUSH rule 0 x 70 [7,2]
+  CRUSH rule 0 x 71 [2,7]
+  CRUSH rule 0 x 72 [6,2]
+  CRUSH rule 0 x 73 [2,7]
+  CRUSH rule 0 x 74 [0,8]
+  CRUSH rule 0 x 75 [3,0]
+  CRUSH rule 0 x 76 [5,2]
+  CRUSH rule 0 x 77 [7,0]
+  CRUSH rule 0 x 78 [1,4]
+  CRUSH rule 0 x 79 [5,0]
+  CRUSH rule 0 x 80 [0,4]
+  CRUSH rule 0 x 81 [0,5]
+  CRUSH rule 0 x 82 [7,2]
+  CRUSH rule 0 x 83 [2,6]
+  CRUSH rule 0 x 84 [7,0]
+  CRUSH rule 0 x 85 [3,7]
+  CRUSH rule 0 x 86 [0,6]
+  CRUSH rule 0 x 87 [0,6]
+  CRUSH rule 0 x 88 [1,8]
+  CRUSH rule 0 x 89 [3,1]
+  CRUSH rule 0 x 90 [6,3]
+  CRUSH rule 0 x 91 [3,6]
+  CRUSH rule 0 x 92 [1,6]
+  CRUSH rule 0 x 93 [7,4]
+  CRUSH rule 0 x 94 [0,3]
+  CRUSH rule 0 x 95 [7,3]
+  CRUSH rule 0 x 96 [3,8]
+  CRUSH rule 0 x 97 [8,4]
+  CRUSH rule 0 x 98 [2,7]
+  CRUSH rule 0 x 99 [0,8]
+  CRUSH rule 0 x 100 [1,6]
+  CRUSH rule 0 x 101 [3,8]
+  CRUSH rule 0 x 102 [4,0]
+  CRUSH rule 0 x 103 [4,7]
+  CRUSH rule 0 x 104 [7,4]
+  CRUSH rule 0 x 105 [2,3]
+  CRUSH rule 0 x 106 [1,6]
+  CRUSH rule 0 x 107 [3,1]
+  CRUSH rule 0 x 108 [7,1]
+  CRUSH rule 0 x 109 [1,3]
+  CRUSH rule 0 x 110 [3,2]
+  CRUSH rule 0 x 111 [2,4]
+  CRUSH rule 0 x 112 [2,6]
+  CRUSH rule 0 x 113 [6,1]
+  CRUSH rule 0 x 114 [7,5]
+  CRUSH rule 0 x 115 [8,2]
+  CRUSH rule 0 x 116 [1,7]
+  CRUSH rule 0 x 117 [7,5]
+  CRUSH rule 0 x 118 [0,3]
+  CRUSH rule 0 x 119 [5,7]
+  CRUSH rule 0 x 120 [0,5]
+  CRUSH rule 0 x 121 [2,8]
+  CRUSH rule 0 x 122 [8,4]
+  CRUSH rule 0 x 123 [2,4]
+  CRUSH rule 0 x 124 [3,0]
+  CRUSH rule 0 x 125 [0,7]
+  CRUSH rule 0 x 126 [4,1]
+  CRUSH rule 0 x 127 [3,6]
+  CRUSH rule 0 x 128 [3,8]
+  CRUSH rule 0 x 129 [0,4]
+  CRUSH rule 0 x 130 [3,8]
+  CRUSH rule 0 x 131 [1,5]
+  CRUSH rule 0 x 132 [1,3]
+  CRUSH rule 0 x 133 [3,6]
+  CRUSH rule 0 x 134 [1,7]
+  CRUSH rule 0 x 135 [5,7]
+  CRUSH rule 0 x 136 [2,3]
+  CRUSH rule 0 x 137 [7,3]
+  CRUSH rule 0 x 138 [8,3]
+  CRUSH rule 0 x 139 [3,0]
+  CRUSH rule 0 x 140 [1,8]
+  CRUSH rule 0 x 141 [6,0]
+  CRUSH rule 0 x 142 [3,2]
+  CRUSH rule 0 x 143 [5,7]
+  CRUSH rule 0 x 144 [8,2]
+  CRUSH rule 0 x 145 [8,4]
+  CRUSH rule 0 x 146 [2,8]
+  CRUSH rule 0 x 147 [2,6]
+  CRUSH rule 0 x 148 [3,0]
+  CRUSH rule 0 x 149 [4,6]
+  CRUSH rule 0 x 150 [1,8]
+  CRUSH rule 0 x 151 [3,8]
+  CRUSH rule 0 x 152 [8,3]
+  CRUSH rule 0 x 153 [8,3]
+  CRUSH rule 0 x 154 [3,0]
+  CRUSH rule 0 x 155 [3,6]
+  CRUSH rule 0 x 156 [4,2]
+  CRUSH rule 0 x 157 [4,1]
+  CRUSH rule 0 x 158 [2,6]
+  CRUSH rule 0 x 159 [7,0]
+  CRUSH rule 0 x 160 [2,8]
+  CRUSH rule 0 x 161 [1,3]
+  CRUSH rule 0 x 162 [0,8]
+  CRUSH rule 0 x 163 [5,8]
+  CRUSH rule 0 x 164 [7,2]
+  CRUSH rule 0 x 165 [7,2]
+  CRUSH rule 0 x 166 [2,3]
+  CRUSH rule 0 x 167 [0,6]
+  CRUSH rule 0 x 168 [4,0]
+  CRUSH rule 0 x 169 [2,7]
+  CRUSH rule 0 x 170 [1,3]
+  CRUSH rule 0 x 171 [7,3]
+  CRUSH rule 0 x 172 [0,8]
+  CRUSH rule 0 x 173 [8,5]
+  CRUSH rule 0 x 174 [1,4]
+  CRUSH rule 0 x 175 [6,0]
+  CRUSH rule 0 x 176 [4,2]
+  CRUSH rule 0 x 177 [5,1]
+  CRUSH rule 0 x 178 [3,0]
+  CRUSH rule 0 x 179 [4,1]
+  CRUSH rule 0 x 180 [3,7]
+  CRUSH rule 0 x 181 [6,0]
+  CRUSH rule 0 x 182 [8,3]
+  CRUSH rule 0 x 183 [7,5]
+  CRUSH rule 0 x 184 [5,6]
+  CRUSH rule 0 x 185 [6,0]
+  CRUSH rule 0 x 186 [2,4]
+  CRUSH rule 0 x 187 [1,6]
+  CRUSH rule 0 x 188 [1,6]
+  CRUSH rule 0 x 189 [0,6]
+  CRUSH rule 0 x 190 [4,1]
+  CRUSH rule 0 x 191 [7,0]
+  CRUSH rule 0 x 192 [5,2]
+  CRUSH rule 0 x 193 [4,0]
+  CRUSH rule 0 x 194 [1,3]
+  CRUSH rule 0 x 195 [6,5]
+  CRUSH rule 0 x 196 [6,1]
+  CRUSH rule 0 x 197 [6,5]
+  CRUSH rule 0 x 198 [2,3]
+  CRUSH rule 0 x 199 [0,5]
+  CRUSH rule 0 x 200 [0,3]
+  CRUSH rule 0 x 201 [7,1]
+  CRUSH rule 0 x 202 [6,4]
+  CRUSH rule 0 x 203 [4,6]
+  CRUSH rule 0 x 204 [2,4]
+  CRUSH rule 0 x 205 [0,7]
+  CRUSH rule 0 x 206 [0,8]
+  CRUSH rule 0 x 207 [3,2]
+  CRUSH rule 0 x 208 [7,2]
+  CRUSH rule 0 x 209 [1,8]
+  CRUSH rule 0 x 210 [1,4]
+  CRUSH rule 0 x 211 [5,2]
+  CRUSH rule 0 x 212 [7,5]
+  CRUSH rule 0 x 213 [8,4]
+  CRUSH rule 0 x 214 [4,7]
+  CRUSH rule 0 x 215 [8,0]
+  CRUSH rule 0 x 216 [5,2]
+  CRUSH rule 0 x 217 [0,7]
+  CRUSH rule 0 x 218 [0,7]
+  CRUSH rule 0 x 219 [4,6]
+  CRUSH rule 0 x 220 [5,8]
+  CRUSH rule 0 x 221 [3,7]
+  CRUSH rule 0 x 222 [6,4]
+  CRUSH rule 0 x 223 [1,3]
+  CRUSH rule 0 x 224 [1,3]
+  CRUSH rule 0 x 225 [8,0]
+  CRUSH rule 0 x 226 [7,0]
+  CRUSH rule 0 x 227 [3,0]
+  CRUSH rule 0 x 228 [5,6]
+  CRUSH rule 0 x 229 [3,7]
+  CRUSH rule 0 x 230 [4,6]
+  CRUSH rule 0 x 231 [4,7]
+  CRUSH rule 0 x 232 [2,8]
+  CRUSH rule 0 x 233 [3,6]
+  CRUSH rule 0 x 234 [0,4]
+  CRUSH rule 0 x 235 [3,6]
+  CRUSH rule 0 x 236 [5,0]
+  CRUSH rule 0 x 237 [4,8]
+  CRUSH rule 0 x 238 [4,2]
+  CRUSH rule 0 x 239 [8,3]
+  CRUSH rule 0 x 240 [5,8]
+  CRUSH rule 0 x 241 [3,1]
+  CRUSH rule 0 x 242 [3,2]
+  CRUSH rule 0 x 243 [4,8]
+  CRUSH rule 0 x 244 [4,6]
+  CRUSH rule 0 x 245 [7,2]
+  CRUSH rule 0 x 246 [1,3]
+  CRUSH rule 0 x 247 [6,1]
+  CRUSH rule 0 x 248 [8,0]
+  CRUSH rule 0 x 249 [2,5]
+  CRUSH rule 0 x 250 [2,3]
+  CRUSH rule 0 x 251 [2,3]
+  CRUSH rule 0 x 252 [3,7]
+  CRUSH rule 0 x 253 [3,0]
+  CRUSH rule 0 x 254 [3,2]
+  CRUSH rule 0 x 255 [1,7]
+  CRUSH rule 0 x 256 [5,6]
+  CRUSH rule 0 x 257 [2,6]
+  CRUSH rule 0 x 258 [5,2]
+  CRUSH rule 0 x 259 [4,6]
+  CRUSH rule 0 x 260 [3,8]
+  CRUSH rule 0 x 261 [8,5]
+  CRUSH rule 0 x 262 [5,6]
+  CRUSH rule 0 x 263 [6,1]
+  CRUSH rule 0 x 264 [3,6]
+  CRUSH rule 0 x 265 [8,4]
+  CRUSH rule 0 x 266 [8,0]
+  CRUSH rule 0 x 267 [2,4]
+  CRUSH rule 0 x 268 [0,6]
+  CRUSH rule 0 x 269 [0,6]
+  CRUSH rule 0 x 270 [5,1]
+  CRUSH rule 0 x 271 [7,4]
+  CRUSH rule 0 x 272 [2,6]
+  CRUSH rule 0 x 273 [3,2]
+  CRUSH rule 0 x 274 [6,3]
+  CRUSH rule 0 x 275 [4,8]
+  CRUSH rule 0 x 276 [7,0]
+  CRUSH rule 0 x 277 [6,4]
+  CRUSH rule 0 x 278 [6,2]
+  CRUSH rule 0 x 279 [8,5]
+  CRUSH rule 0 x 280 [0,7]
+  CRUSH rule 0 x 281 [8,1]
+  CRUSH rule 0 x 282 [3,2]
+  CRUSH rule 0 x 283 [8,1]
+  CRUSH rule 0 x 284 [6,3]
+  CRUSH rule 0 x 285 [5,7]
+  CRUSH rule 0 x 286 [2,8]
+  CRUSH rule 0 x 287 [0,3]
+  CRUSH rule 0 x 288 [8,1]
+  CRUSH rule 0 x 289 [4,6]
+  CRUSH rule 0 x 290 [1,5]
+  CRUSH rule 0 x 291 [0,5]
+  CRUSH rule 0 x 292 [8,1]
+  CRUSH rule 0 x 293 [6,2]
+  CRUSH rule 0 x 294 [7,5]
+  CRUSH rule 0 x 295 [4,6]
+  CRUSH rule 0 x 296 [3,1]
+  CRUSH rule 0 x 297 [6,0]
+  CRUSH rule 0 x 298 [1,5]
+  CRUSH rule 0 x 299 [2,8]
+  CRUSH rule 0 x 300 [8,4]
+  CRUSH rule 0 x 301 [0,7]
+  CRUSH rule 0 x 302 [3,1]
+  CRUSH rule 0 x 303 [7,4]
+  CRUSH rule 0 x 304 [2,6]
+  CRUSH rule 0 x 305 [5,6]
+  CRUSH rule 0 x 306 [0,8]
+  CRUSH rule 0 x 307 [0,7]
+  CRUSH rule 0 x 308 [0,8]
+  CRUSH rule 0 x 309 [7,4]
+  CRUSH rule 0 x 310 [4,0]
+  CRUSH rule 0 x 311 [3,8]
+  CRUSH rule 0 x 312 [2,6]
+  CRUSH rule 0 x 313 [5,1]
+  CRUSH rule 0 x 314 [4,0]
+  CRUSH rule 0 x 315 [2,3]
+  CRUSH rule 0 x 316 [6,4]
+  CRUSH rule 0 x 317 [2,7]
+  CRUSH rule 0 x 318 [8,1]
+  CRUSH rule 0 x 319 [5,2]
+  CRUSH rule 0 x 320 [3,8]
+  CRUSH rule 0 x 321 [1,5]
+  CRUSH rule 0 x 322 [2,6]
+  CRUSH rule 0 x 323 [4,7]
+  CRUSH rule 0 x 324 [7,2]
+  CRUSH rule 0 x 325 [4,6]
+  CRUSH rule 0 x 326 [3,1]
+  CRUSH rule 0 x 327 [0,8]
+  CRUSH rule 0 x 328 [7,3]
+  CRUSH rule 0 x 329 [5,7]
+  CRUSH rule 0 x 330 [3,7]
+  CRUSH rule 0 x 331 [2,7]
+  CRUSH rule 0 x 332 [2,3]
+  CRUSH rule 0 x 333 [6,4]
+  CRUSH rule 0 x 334 [8,3]
+  CRUSH rule 0 x 335 [7,2]
+  CRUSH rule 0 x 336 [4,6]
+  CRUSH rule 0 x 337 [7,0]
+  CRUSH rule 0 x 338 [5,8]
+  CRUSH rule 0 x 339 [7,5]
+  CRUSH rule 0 x 340 [2,6]
+  CRUSH rule 0 x 341 [5,1]
+  CRUSH rule 0 x 342 [0,8]
+  CRUSH rule 0 x 343 [6,5]
+  CRUSH rule 0 x 344 [6,1]
+  CRUSH rule 0 x 345 [4,7]
+  CRUSH rule 0 x 346 [8,0]
+  CRUSH rule 0 x 347 [3,0]
+  CRUSH rule 0 x 348 [8,0]
+  CRUSH rule 0 x 349 [1,7]
+  CRUSH rule 0 x 350 [8,5]
+  CRUSH rule 0 x 351 [3,8]
+  CRUSH rule 0 x 352 [1,8]
+  CRUSH rule 0 x 353 [6,5]
+  CRUSH rule 0 x 354 [0,5]
+  CRUSH rule 0 x 355 [3,8]
+  CRUSH rule 0 x 356 [3,1]
+  CRUSH rule 0 x 357 [6,1]
+  CRUSH rule 0 x 358 [2,8]
+  CRUSH rule 0 x 359 [6,0]
+  CRUSH rule 0 x 360 [5,0]
+  CRUSH rule 0 x 361 [8,5]
+  CRUSH rule 0 x 362 [4,0]
+  CRUSH rule 0 x 363 [4,2]
+  CRUSH rule 0 x 364 [2,5]
+  CRUSH rule 0 x 365 [6,4]
+  CRUSH rule 0 x 366 [7,0]
+  CRUSH rule 0 x 367 [4,2]
+  CRUSH rule 0 x 368 [7,3]
+  CRUSH rule 0 x 369 [3,7]
+  CRUSH rule 0 x 370 [8,2]
+  CRUSH rule 0 x 371 [1,5]
+  CRUSH rule 0 x 372 [3,1]
+  CRUSH rule 0 x 373 [0,6]
+  CRUSH rule 0 x 374 [3,8]
+  CRUSH rule 0 x 375 [6,5]
+  CRUSH rule 0 x 376 [7,1]
+  CRUSH rule 0 x 377 [1,4]
+  CRUSH rule 0 x 378 [0,6]
+  CRUSH rule 0 x 379 [8,3]
+  CRUSH rule 0 x 380 [2,3]
+  CRUSH rule 0 x 381 [0,3]
+  CRUSH rule 0 x 382 [1,3]
+  CRUSH rule 0 x 383 [4,7]
+  CRUSH rule 0 x 384 [7,0]
+  CRUSH rule 0 x 385 [7,4]
+  CRUSH rule 0 x 386 [0,4]
+  CRUSH rule 0 x 387 [1,5]
+  CRUSH rule 0 x 388 [5,2]
+  CRUSH rule 0 x 389 [1,4]
+  CRUSH rule 0 x 390 [5,8]
+  CRUSH rule 0 x 391 [5,6]
+  CRUSH rule 0 x 392 [1,7]
+  CRUSH rule 0 x 393 [4,0]
+  CRUSH rule 0 x 394 [4,8]
+  CRUSH rule 0 x 395 [4,0]
+  CRUSH rule 0 x 396 [4,2]
+  CRUSH rule 0 x 397 [2,4]
+  CRUSH rule 0 x 398 [2,3]
+  CRUSH rule 0 x 399 [8,5]
+  CRUSH rule 0 x 400 [8,1]
+  CRUSH rule 0 x 401 [0,3]
+  CRUSH rule 0 x 402 [7,4]
+  CRUSH rule 0 x 403 [0,4]
+  CRUSH rule 0 x 404 [4,0]
+  CRUSH rule 0 x 405 [6,4]
+  CRUSH rule 0 x 406 [2,6]
+  CRUSH rule 0 x 407 [2,7]
+  CRUSH rule 0 x 408 [4,1]
+  CRUSH rule 0 x 409 [7,5]
+  CRUSH rule 0 x 410 [8,3]
+  CRUSH rule 0 x 411 [2,7]
+  CRUSH rule 0 x 412 [0,3]
+  CRUSH rule 0 x 413 [5,2]
+  CRUSH rule 0 x 414 [4,0]
+  CRUSH rule 0 x 415 [0,6]
+  CRUSH rule 0 x 416 [2,8]
+  CRUSH rule 0 x 417 [8,0]
+  CRUSH rule 0 x 418 [7,1]
+  CRUSH rule 0 x 419 [8,5]
+  CRUSH rule 0 x 420 [1,4]
+  CRUSH rule 0 x 421 [8,3]
+  CRUSH rule 0 x 422 [6,3]
+  CRUSH rule 0 x 423 [0,3]
+  CRUSH rule 0 x 424 [8,5]
+  CRUSH rule 0 x 425 [1,4]
+  CRUSH rule 0 x 426 [6,2]
+  CRUSH rule 0 x 427 [0,8]
+  CRUSH rule 0 x 428 [5,8]
+  CRUSH rule 0 x 429 [4,8]
+  CRUSH rule 0 x 430 [3,7]
+  CRUSH rule 0 x 431 [5,0]
+  CRUSH rule 0 x 432 [7,0]
+  CRUSH rule 0 x 433 [6,4]
+  CRUSH rule 0 x 434 [5,0]
+  CRUSH rule 0 x 435 [0,4]
+  CRUSH rule 0 x 436 [4,0]
+  CRUSH rule 0 x 437 [7,3]
+  CRUSH rule 0 x 438 [0,5]
+  CRUSH rule 0 x 439 [1,3]
+  CRUSH rule 0 x 440 [2,6]
+  CRUSH rule 0 x 441 [5,8]
+  CRUSH rule 0 x 442 [2,3]
+  CRUSH rule 0 x 443 [6,0]
+  CRUSH rule 0 x 444 [7,1]
+  CRUSH rule 0 x 445 [6,4]
+  CRUSH rule 0 x 446 [4,0]
+  CRUSH rule 0 x 447 [2,4]
+  CRUSH rule 0 x 448 [7,0]
+  CRUSH rule 0 x 449 [7,4]
+  CRUSH rule 0 x 450 [4,0]
+  CRUSH rule 0 x 451 [6,5]
+  CRUSH rule 0 x 452 [8,4]
+  CRUSH rule 0 x 453 [6,5]
+  CRUSH rule 0 x 454 [6,5]
+  CRUSH rule 0 x 455 [2,8]
+  CRUSH rule 0 x 456 [6,2]
+  CRUSH rule 0 x 457 [7,1]
+  CRUSH rule 0 x 458 [2,8]
+  CRUSH rule 0 x 459 [2,6]
+  CRUSH rule 0 x 460 [6,5]
+  CRUSH rule 0 x 461 [6,4]
+  CRUSH rule 0 x 462 [8,0]
+  CRUSH rule 0 x 463 [6,0]
+  CRUSH rule 0 x 464 [7,4]
+  CRUSH rule 0 x 465 [7,1]
+  CRUSH rule 0 x 466 [5,6]
+  CRUSH rule 0 x 467 [6,5]
+  CRUSH rule 0 x 468 [7,2]
+  CRUSH rule 0 x 469 [7,2]
+  CRUSH rule 0 x 470 [3,0]
+  CRUSH rule 0 x 471 [0,6]
+  CRUSH rule 0 x 472 [5,0]
+  CRUSH rule 0 x 473 [1,4]
+  CRUSH rule 0 x 474 [6,1]
+  CRUSH rule 0 x 475 [6,2]
+  CRUSH rule 0 x 476 [4,7]
+  CRUSH rule 0 x 477 [5,6]
+  CRUSH rule 0 x 478 [6,1]
+  CRUSH rule 0 x 479 [0,3]
+  CRUSH rule 0 x 480 [1,6]
+  CRUSH rule 0 x 481 [2,5]
+  CRUSH rule 0 x 482 [4,8]
+  CRUSH rule 0 x 483 [0,6]
+  CRUSH rule 0 x 484 [1,8]
+  CRUSH rule 0 x 485 [4,8]
+  CRUSH rule 0 x 486 [4,0]
+  CRUSH rule 0 x 487 [5,0]
+  CRUSH rule 0 x 488 [5,7]
+  CRUSH rule 0 x 489 [2,8]
+  CRUSH rule 0 x 490 [6,4]
+  CRUSH rule 0 x 491 [1,6]
+  CRUSH rule 0 x 492 [6,5]
+  CRUSH rule 0 x 493 [0,8]
+  CRUSH rule 0 x 494 [1,6]
+  CRUSH rule 0 x 495 [3,0]
+  CRUSH rule 0 x 496 [7,5]
+  CRUSH rule 0 x 497 [5,7]
+  CRUSH rule 0 x 498 [0,4]
+  CRUSH rule 0 x 499 [8,5]
+  CRUSH rule 0 x 500 [3,6]
+  CRUSH rule 0 x 501 [0,7]
+  CRUSH rule 0 x 502 [7,1]
+  CRUSH rule 0 x 503 [2,3]
+  CRUSH rule 0 x 504 [5,8]
+  CRUSH rule 0 x 505 [0,7]
+  CRUSH rule 0 x 506 [5,1]
+  CRUSH rule 0 x 507 [6,0]
+  CRUSH rule 0 x 508 [0,4]
+  CRUSH rule 0 x 509 [7,4]
+  CRUSH rule 0 x 510 [6,2]
+  CRUSH rule 0 x 511 [5,6]
+  CRUSH rule 0 x 512 [7,2]
+  CRUSH rule 0 x 513 [7,2]
+  CRUSH rule 0 x 514 [4,7]
+  CRUSH rule 0 x 515 [8,3]
+  CRUSH rule 0 x 516 [4,1]
+  CRUSH rule 0 x 517 [7,0]
+  CRUSH rule 0 x 518 [4,6]
+  CRUSH rule 0 x 519 [7,3]
+  CRUSH rule 0 x 520 [2,8]
+  CRUSH rule 0 x 521 [8,0]
+  CRUSH rule 0 x 522 [6,0]
+  CRUSH rule 0 x 523 [4,1]
+  CRUSH rule 0 x 524 [0,4]
+  CRUSH rule 0 x 525 [0,3]
+  CRUSH rule 0 x 526 [1,3]
+  CRUSH rule 0 x 527 [0,4]
+  CRUSH rule 0 x 528 [5,2]
+  CRUSH rule 0 x 529 [5,6]
+  CRUSH rule 0 x 530 [6,4]
+  CRUSH rule 0 x 531 [6,0]
+  CRUSH rule 0 x 532 [6,5]
+  CRUSH rule 0 x 533 [5,8]
+  CRUSH rule 0 x 534 [7,4]
+  CRUSH rule 0 x 535 [8,0]
+  CRUSH rule 0 x 536 [6,2]
+  CRUSH rule 0 x 537 [3,8]
+  CRUSH rule 0 x 538 [6,4]
+  CRUSH rule 0 x 539 [8,4]
+  CRUSH rule 0 x 540 [0,7]
+  CRUSH rule 0 x 541 [2,5]
+  CRUSH rule 0 x 542 [3,0]
+  CRUSH rule 0 x 543 [6,2]
+  CRUSH rule 0 x 544 [3,7]
+  CRUSH rule 0 x 545 [5,7]
+  CRUSH rule 0 x 546 [6,2]
+  CRUSH rule 0 x 547 [8,1]
+  CRUSH rule 0 x 548 [5,1]
+  CRUSH rule 0 x 549 [5,7]
+  CRUSH rule 0 x 550 [0,5]
+  CRUSH rule 0 x 551 [7,4]
+  CRUSH rule 0 x 552 [5,7]
+  CRUSH rule 0 x 553 [4,2]
+  CRUSH rule 0 x 554 [0,6]
+  CRUSH rule 0 x 555 [5,0]
+  CRUSH rule 0 x 556 [3,6]
+  CRUSH rule 0 x 557 [7,4]
+  CRUSH rule 0 x 558 [3,2]
+  CRUSH rule 0 x 559 [4,1]
+  CRUSH rule 0 x 560 [8,4]
+  CRUSH rule 0 x 561 [6,4]
+  CRUSH rule 0 x 562 [3,0]
+  CRUSH rule 0 x 563 [2,7]
+  CRUSH rule 0 x 564 [5,1]
+  CRUSH rule 0 x 565 [3,8]
+  CRUSH rule 0 x 566 [4,6]
+  CRUSH rule 0 x 567 [3,7]
+  CRUSH rule 0 x 568 [7,4]
+  CRUSH rule 0 x 569 [3,1]
+  CRUSH rule 0 x 570 [1,4]
+  CRUSH rule 0 x 571 [3,6]
+  CRUSH rule 0 x 572 [3,0]
+  CRUSH rule 0 x 573 [3,1]
+  CRUSH rule 0 x 574 [2,3]
+  CRUSH rule 0 x 575 [8,1]
+  CRUSH rule 0 x 576 [4,7]
+  CRUSH rule 0 x 577 [8,2]
+  CRUSH rule 0 x 578 [6,0]
+  CRUSH rule 0 x 579 [3,2]
+  CRUSH rule 0 x 580 [3,0]
+  CRUSH rule 0 x 581 [7,1]
+  CRUSH rule 0 x 582 [2,8]
+  CRUSH rule 0 x 583 [6,2]
+  CRUSH rule 0 x 584 [8,0]
+  CRUSH rule 0 x 585 [7,0]
+  CRUSH rule 0 x 586 [0,8]
+  CRUSH rule 0 x 587 [2,5]
+  CRUSH rule 0 x 588 [3,8]
+  CRUSH rule 0 x 589 [7,1]
+  CRUSH rule 0 x 590 [6,2]
+  CRUSH rule 0 x 591 [5,2]
+  CRUSH rule 0 x 592 [2,4]
+  CRUSH rule 0 x 593 [0,8]
+  CRUSH rule 0 x 594 [0,6]
+  CRUSH rule 0 x 595 [7,1]
+  CRUSH rule 0 x 596 [4,0]
+  CRUSH rule 0 x 597 [3,1]
+  CRUSH rule 0 x 598 [3,0]
+  CRUSH rule 0 x 599 [5,1]
+  CRUSH rule 0 x 600 [7,0]
+  CRUSH rule 0 x 601 [0,6]
+  CRUSH rule 0 x 602 [3,8]
+  CRUSH rule 0 x 603 [5,0]
+  CRUSH rule 0 x 604 [7,4]
+  CRUSH rule 0 x 605 [3,2]
+  CRUSH rule 0 x 606 [2,7]
+  CRUSH rule 0 x 607 [0,5]
+  CRUSH rule 0 x 608 [5,1]
+  CRUSH rule 0 x 609 [5,0]
+  CRUSH rule 0 x 610 [3,8]
+  CRUSH rule 0 x 611 [1,8]
+  CRUSH rule 0 x 612 [2,8]
+  CRUSH rule 0 x 613 [7,1]
+  CRUSH rule 0 x 614 [7,2]
+  CRUSH rule 0 x 615 [6,2]
+  CRUSH rule 0 x 616 [0,7]
+  CRUSH rule 0 x 617 [6,2]
+  CRUSH rule 0 x 618 [7,3]
+  CRUSH rule 0 x 619 [5,0]
+  CRUSH rule 0 x 620 [4,1]
+  CRUSH rule 0 x 621 [5,6]
+  CRUSH rule 0 x 622 [0,3]
+  CRUSH rule 0 x 623 [0,8]
+  CRUSH rule 0 x 624 [3,2]
+  CRUSH rule 0 x 625 [2,5]
+  CRUSH rule 0 x 626 [7,2]
+  CRUSH rule 0 x 627 [2,6]
+  CRUSH rule 0 x 628 [8,1]
+  CRUSH rule 0 x 629 [2,6]
+  CRUSH rule 0 x 630 [2,6]
+  CRUSH rule 0 x 631 [0,6]
+  CRUSH rule 0 x 632 [7,0]
+  CRUSH rule 0 x 633 [8,4]
+  CRUSH rule 0 x 634 [0,5]
+  CRUSH rule 0 x 635 [5,6]
+  CRUSH rule 0 x 636 [1,3]
+  CRUSH rule 0 x 637 [4,0]
+  CRUSH rule 0 x 638 [6,2]
+  CRUSH rule 0 x 639 [4,0]
+  CRUSH rule 0 x 640 [3,2]
+  CRUSH rule 0 x 641 [7,2]
+  CRUSH rule 0 x 642 [2,8]
+  CRUSH rule 0 x 643 [3,0]
+  CRUSH rule 0 x 644 [8,0]
+  CRUSH rule 0 x 645 [5,7]
+  CRUSH rule 0 x 646 [8,0]
+  CRUSH rule 0 x 647 [7,0]
+  CRUSH rule 0 x 648 [0,8]
+  CRUSH rule 0 x 649 [4,7]
+  CRUSH rule 0 x 650 [7,5]
+  CRUSH rule 0 x 651 [3,6]
+  CRUSH rule 0 x 652 [3,6]
+  CRUSH rule 0 x 653 [8,3]
+  CRUSH rule 0 x 654 [7,4]
+  CRUSH rule 0 x 655 [0,5]
+  CRUSH rule 0 x 656 [4,7]
+  CRUSH rule 0 x 657 [6,0]
+  CRUSH rule 0 x 658 [5,8]
+  CRUSH rule 0 x 659 [4,7]
+  CRUSH rule 0 x 660 [7,3]
+  CRUSH rule 0 x 661 [1,7]
+  CRUSH rule 0 x 662 [4,2]
+  CRUSH rule 0 x 663 [1,3]
+  CRUSH rule 0 x 664 [1,3]
+  CRUSH rule 0 x 665 [5,6]
+  CRUSH rule 0 x 666 [2,7]
+  CRUSH rule 0 x 667 [1,3]
+  CRUSH rule 0 x 668 [3,7]
+  CRUSH rule 0 x 669 [6,3]
+  CRUSH rule 0 x 670 [4,1]
+  CRUSH rule 0 x 671 [0,8]
+  CRUSH rule 0 x 672 [4,2]
+  CRUSH rule 0 x 673 [5,0]
+  CRUSH rule 0 x 674 [3,0]
+  CRUSH rule 0 x 675 [0,8]
+  CRUSH rule 0 x 676 [0,3]
+  CRUSH rule 0 x 677 [4,1]
+  CRUSH rule 0 x 678 [2,3]
+  CRUSH rule 0 x 679 [6,0]
+  CRUSH rule 0 x 680 [0,3]
+  CRUSH rule 0 x 681 [4,6]
+  CRUSH rule 0 x 682 [0,4]
+  CRUSH rule 0 x 683 [0,3]
+  CRUSH rule 0 x 684 [7,2]
+  CRUSH rule 0 x 685 [7,2]
+  CRUSH rule 0 x 686 [1,5]
+  CRUSH rule 0 x 687 [3,7]
+  CRUSH rule 0 x 688 [5,6]
+  CRUSH rule 0 x 689 [6,5]
+  CRUSH rule 0 x 690 [8,0]
+  CRUSH rule 0 x 691 [3,1]
+  CRUSH rule 0 x 692 [7,1]
+  CRUSH rule 0 x 693 [6,5]
+  CRUSH rule 0 x 694 [6,4]
+  CRUSH rule 0 x 695 [0,6]
+  CRUSH rule 0 x 696 [1,5]
+  CRUSH rule 0 x 697 [6,0]
+  CRUSH rule 0 x 698 [6,0]
+  CRUSH rule 0 x 699 [1,8]
+  CRUSH rule 0 x 700 [0,4]
+  CRUSH rule 0 x 701 [4,0]
+  CRUSH rule 0 x 702 [3,0]
+  CRUSH rule 0 x 703 [8,4]
+  CRUSH rule 0 x 704 [0,4]
+  CRUSH rule 0 x 705 [8,0]
+  CRUSH rule 0 x 706 [1,5]
+  CRUSH rule 0 x 707 [7,3]
+  CRUSH rule 0 x 708 [3,7]
+  CRUSH rule 0 x 709 [6,5]
+  CRUSH rule 0 x 710 [8,5]
+  CRUSH rule 0 x 711 [2,4]
+  CRUSH rule 0 x 712 [2,3]
+  CRUSH rule 0 x 713 [6,3]
+  CRUSH rule 0 x 714 [3,0]
+  CRUSH rule 0 x 715 [1,3]
+  CRUSH rule 0 x 716 [3,6]
+  CRUSH rule 0 x 717 [8,0]
+  CRUSH rule 0 x 718 [3,7]
+  CRUSH rule 0 x 719 [2,6]
+  CRUSH rule 0 x 720 [6,0]
+  CRUSH rule 0 x 721 [5,7]
+  CRUSH rule 0 x 722 [5,7]
+  CRUSH rule 0 x 723 [5,2]
+  CRUSH rule 0 x 724 [0,7]
+  CRUSH rule 0 x 725 [0,4]
+  CRUSH rule 0 x 726 [3,7]
+  CRUSH rule 0 x 727 [4,7]
+  CRUSH rule 0 x 728 [2,6]
+  CRUSH rule 0 x 729 [5,6]
+  CRUSH rule 0 x 730 [3,8]
+  CRUSH rule 0 x 731 [4,1]
+  CRUSH rule 0 x 732 [1,4]
+  CRUSH rule 0 x 733 [5,6]
+  CRUSH rule 0 x 734 [6,5]
+  CRUSH rule 0 x 735 [4,6]
+  CRUSH rule 0 x 736 [3,8]
+  CRUSH rule 0 x 737 [1,8]
+  CRUSH rule 0 x 738 [5,1]
+  CRUSH rule 0 x 739 [0,7]
+  CRUSH rule 0 x 740 [0,7]
+  CRUSH rule 0 x 741 [7,2]
+  CRUSH rule 0 x 742 [8,2]
+  CRUSH rule 0 x 743 [7,0]
+  CRUSH rule 0 x 744 [4,6]
+  CRUSH rule 0 x 745 [3,0]
+  CRUSH rule 0 x 746 [4,1]
+  CRUSH rule 0 x 747 [6,2]
+  CRUSH rule 0 x 748 [2,6]
+  CRUSH rule 0 x 749 [4,7]
+  CRUSH rule 0 x 750 [1,7]
+  CRUSH rule 0 x 751 [2,7]
+  CRUSH rule 0 x 752 [8,0]
+  CRUSH rule 0 x 753 [7,4]
+  CRUSH rule 0 x 754 [8,5]
+  CRUSH rule 0 x 755 [1,6]
+  CRUSH rule 0 x 756 [5,8]
+  CRUSH rule 0 x 757 [8,0]
+  CRUSH rule 0 x 758 [6,1]
+  CRUSH rule 0 x 759 [8,5]
+  CRUSH rule 0 x 760 [1,5]
+  CRUSH rule 0 x 761 [4,2]
+  CRUSH rule 0 x 762 [2,8]
+  CRUSH rule 0 x 763 [8,3]
+  CRUSH rule 0 x 764 [1,7]
+  CRUSH rule 0 x 765 [6,4]
+  CRUSH rule 0 x 766 [8,3]
+  CRUSH rule 0 x 767 [1,8]
+  CRUSH rule 0 x 768 [8,5]
+  CRUSH rule 0 x 769 [6,2]
+  CRUSH rule 0 x 770 [6,1]
+  CRUSH rule 0 x 771 [7,2]
+  CRUSH rule 0 x 772 [8,4]
+  CRUSH rule 0 x 773 [3,2]
+  CRUSH rule 0 x 774 [4,7]
+  CRUSH rule 0 x 775 [6,5]
+  CRUSH rule 0 x 776 [7,0]
+  CRUSH rule 0 x 777 [3,0]
+  CRUSH rule 0 x 778 [1,6]
+  CRUSH rule 0 x 779 [2,6]
+  CRUSH rule 0 x 780 [0,5]
+  CRUSH rule 0 x 781 [6,5]
+  CRUSH rule 0 x 782 [5,2]
+  CRUSH rule 0 x 783 [7,0]
+  CRUSH rule 0 x 784 [0,5]
+  CRUSH rule 0 x 785 [6,1]
+  CRUSH rule 0 x 786 [7,3]
+  CRUSH rule 0 x 787 [1,8]
+  CRUSH rule 0 x 788 [6,0]
+  CRUSH rule 0 x 789 [0,5]
+  CRUSH rule 0 x 790 [8,3]
+  CRUSH rule 0 x 791 [3,6]
+  CRUSH rule 0 x 792 [5,6]
+  CRUSH rule 0 x 793 [6,2]
+  CRUSH rule 0 x 794 [2,8]
+  CRUSH rule 0 x 795 [0,4]
+  CRUSH rule 0 x 796 [3,7]
+  CRUSH rule 0 x 797 [2,3]
+  CRUSH rule 0 x 798 [6,0]
+  CRUSH rule 0 x 799 [5,2]
+  CRUSH rule 0 x 800 [5,2]
+  CRUSH rule 0 x 801 [3,7]
+  CRUSH rule 0 x 802 [1,6]
+  CRUSH rule 0 x 803 [0,4]
+  CRUSH rule 0 x 804 [6,0]
+  CRUSH rule 0 x 805 [3,8]
+  CRUSH rule 0 x 806 [1,5]
+  CRUSH rule 0 x 807 [5,7]
+  CRUSH rule 0 x 808 [4,7]
+  CRUSH rule 0 x 809 [1,3]
+  CRUSH rule 0 x 810 [5,7]
+  CRUSH rule 0 x 811 [8,4]
+  CRUSH rule 0 x 812 [8,3]
+  CRUSH rule 0 x 813 [6,4]
+  CRUSH rule 0 x 814 [3,8]
+  CRUSH rule 0 x 815 [3,1]
+  CRUSH rule 0 x 816 [2,6]
+  CRUSH rule 0 x 817 [4,6]
+  CRUSH rule 0 x 818 [3,1]
+  CRUSH rule 0 x 819 [5,0]
+  CRUSH rule 0 x 820 [3,7]
+  CRUSH rule 0 x 821 [4,8]
+  CRUSH rule 0 x 822 [2,3]
+  CRUSH rule 0 x 823 [4,7]
+  CRUSH rule 0 x 824 [3,7]
+  CRUSH rule 0 x 825 [2,8]
+  CRUSH rule 0 x 826 [7,1]
+  CRUSH rule 0 x 827 [0,6]
+  CRUSH rule 0 x 828 [2,5]
+  CRUSH rule 0 x 829 [5,6]
+  CRUSH rule 0 x 830 [2,4]
+  CRUSH rule 0 x 831 [1,6]
+  CRUSH rule 0 x 832 [4,8]
+  CRUSH rule 0 x 833 [2,6]
+  CRUSH rule 0 x 834 [3,0]
+  CRUSH rule 0 x 835 [8,5]
+  CRUSH rule 0 x 836 [3,8]
+  CRUSH rule 0 x 837 [6,4]
+  CRUSH rule 0 x 838 [6,0]
+  CRUSH rule 0 x 839 [5,2]
+  CRUSH rule 0 x 840 [7,3]
+  CRUSH rule 0 x 841 [4,8]
+  CRUSH rule 0 x 842 [2,5]
+  CRUSH rule 0 x 843 [6,4]
+  CRUSH rule 0 x 844 [4,8]
+  CRUSH rule 0 x 845 [3,6]
+  CRUSH rule 0 x 846 [3,0]
+  CRUSH rule 0 x 847 [0,8]
+  CRUSH rule 0 x 848 [2,6]
+  CRUSH rule 0 x 849 [4,8]
+  CRUSH rule 0 x 850 [1,5]
+  CRUSH rule 0 x 851 [6,5]
+  CRUSH rule 0 x 852 [7,4]
+  CRUSH rule 0 x 853 [6,2]
+  CRUSH rule 0 x 854 [7,0]
+  CRUSH rule 0 x 855 [5,7]
+  CRUSH rule 0 x 856 [6,3]
+  CRUSH rule 0 x 857 [8,4]
+  CRUSH rule 0 x 858 [6,5]
+  CRUSH rule 0 x 859 [6,0]
+  CRUSH rule 0 x 860 [4,2]
+  CRUSH rule 0 x 861 [8,4]
+  CRUSH rule 0 x 862 [6,0]
+  CRUSH rule 0 x 863 [8,1]
+  CRUSH rule 0 x 864 [5,6]
+  CRUSH rule 0 x 865 [8,0]
+  CRUSH rule 0 x 866 [3,6]
+  CRUSH rule 0 x 867 [6,3]
+  CRUSH rule 0 x 868 [6,4]
+  CRUSH rule 0 x 869 [8,4]
+  CRUSH rule 0 x 870 [0,5]
+  CRUSH rule 0 x 871 [3,1]
+  CRUSH rule 0 x 872 [5,0]
+  CRUSH rule 0 x 873 [4,6]
+  CRUSH rule 0 x 874 [2,6]
+  CRUSH rule 0 x 875 [2,6]
+  CRUSH rule 0 x 876 [5,8]
+  CRUSH rule 0 x 877 [6,4]
+  CRUSH rule 0 x 878 [5,0]
+  CRUSH rule 0 x 879 [7,3]
+  CRUSH rule 0 x 880 [3,1]
+  CRUSH rule 0 x 881 [5,8]
+  CRUSH rule 0 x 882 [4,1]
+  CRUSH rule 0 x 883 [2,5]
+  CRUSH rule 0 x 884 [6,0]
+  CRUSH rule 0 x 885 [5,1]
+  CRUSH rule 0 x 886 [3,7]
+  CRUSH rule 0 x 887 [7,5]
+  CRUSH rule 0 x 888 [6,1]
+  CRUSH rule 0 x 889 [2,8]
+  CRUSH rule 0 x 890 [7,1]
+  CRUSH rule 0 x 891 [1,7]
+  CRUSH rule 0 x 892 [6,0]
+  CRUSH rule 0 x 893 [2,3]
+  CRUSH rule 0 x 894 [7,4]
+  CRUSH rule 0 x 895 [5,1]
+  CRUSH rule 0 x 896 [1,6]
+  CRUSH rule 0 x 897 [4,0]
+  CRUSH rule 0 x 898 [0,3]
+  CRUSH rule 0 x 899 [1,6]
+  CRUSH rule 0 x 900 [4,0]
+  CRUSH rule 0 x 901 [5,2]
+  CRUSH rule 0 x 902 [8,4]
+  CRUSH rule 0 x 903 [5,6]
+  CRUSH rule 0 x 904 [5,7]
+  CRUSH rule 0 x 905 [6,0]
+  CRUSH rule 0 x 906 [1,7]
+  CRUSH rule 0 x 907 [7,2]
+  CRUSH rule 0 x 908 [5,6]
+  CRUSH rule 0 x 909 [2,3]
+  CRUSH rule 0 x 910 [6,5]
+  CRUSH rule 0 x 911 [5,7]
+  CRUSH rule 0 x 912 [0,8]
+  CRUSH rule 0 x 913 [7,1]
+  CRUSH rule 0 x 914 [6,4]
+  CRUSH rule 0 x 915 [8,2]
+  CRUSH rule 0 x 916 [3,0]
+  CRUSH rule 0 x 917 [1,3]
+  CRUSH rule 0 x 918 [8,0]
+  CRUSH rule 0 x 919 [6,2]
+  CRUSH rule 0 x 920 [7,4]
+  CRUSH rule 0 x 921 [1,4]
+  CRUSH rule 0 x 922 [6,3]
+  CRUSH rule 0 x 923 [5,6]
+  CRUSH rule 0 x 924 [3,1]
+  CRUSH rule 0 x 925 [5,6]
+  CRUSH rule 0 x 926 [3,0]
+  CRUSH rule 0 x 927 [1,6]
+  CRUSH rule 0 x 928 [8,1]
+  CRUSH rule 0 x 929 [4,2]
+  CRUSH rule 0 x 930 [2,5]
+  CRUSH rule 0 x 931 [5,2]
+  CRUSH rule 0 x 932 [4,2]
+  CRUSH rule 0 x 933 [8,4]
+  CRUSH rule 0 x 934 [5,8]
+  CRUSH rule 0 x 935 [6,3]
+  CRUSH rule 0 x 936 [0,7]
+  CRUSH rule 0 x 937 [5,8]
+  CRUSH rule 0 x 938 [6,4]
+  CRUSH rule 0 x 939 [2,8]
+  CRUSH rule 0 x 940 [8,5]
+  CRUSH rule 0 x 941 [5,0]
+  CRUSH rule 0 x 942 [1,6]
+  CRUSH rule 0 x 943 [8,2]
+  CRUSH rule 0 x 944 [4,8]
+  CRUSH rule 0 x 945 [7,0]
+  CRUSH rule 0 x 946 [2,8]
+  CRUSH rule 0 x 947 [4,2]
+  CRUSH rule 0 x 948 [7,4]
+  CRUSH rule 0 x 949 [6,2]
+  CRUSH rule 0 x 950 [3,7]
+  CRUSH rule 0 x 951 [4,6]
+  CRUSH rule 0 x 952 [2,7]
+  CRUSH rule 0 x 953 [1,3]
+  CRUSH rule 0 x 954 [4,0]
+  CRUSH rule 0 x 955 [8,1]
+  CRUSH rule 0 x 956 [1,7]
+  CRUSH rule 0 x 957 [7,0]
+  CRUSH rule 0 x 958 [8,3]
+  CRUSH rule 0 x 959 [5,1]
+  CRUSH rule 0 x 960 [3,6]
+  CRUSH rule 0 x 961 [4,1]
+  CRUSH rule 0 x 962 [7,5]
+  CRUSH rule 0 x 963 [0,5]
+  CRUSH rule 0 x 964 [3,2]
+  CRUSH rule 0 x 965 [7,3]
+  CRUSH rule 0 x 966 [3,6]
+  CRUSH rule 0 x 967 [8,4]
+  CRUSH rule 0 x 968 [7,0]
+  CRUSH rule 0 x 969 [8,0]
+  CRUSH rule 0 x 970 [0,8]
+  CRUSH rule 0 x 971 [1,8]
+  CRUSH rule 0 x 972 [1,7]
+  CRUSH rule 0 x 973 [1,8]
+  CRUSH rule 0 x 974 [5,1]
+  CRUSH rule 0 x 975 [3,8]
+  CRUSH rule 0 x 976 [4,7]
+  CRUSH rule 0 x 977 [8,3]
+  CRUSH rule 0 x 978 [7,0]
+  CRUSH rule 0 x 979 [7,2]
+  CRUSH rule 0 x 980 [6,2]
+  CRUSH rule 0 x 981 [7,5]
+  CRUSH rule 0 x 982 [4,1]
+  CRUSH rule 0 x 983 [3,6]
+  CRUSH rule 0 x 984 [0,8]
+  CRUSH rule 0 x 985 [2,4]
+  CRUSH rule 0 x 986 [8,4]
+  CRUSH rule 0 x 987 [0,4]
+  CRUSH rule 0 x 988 [1,4]
+  CRUSH rule 0 x 989 [0,8]
+  CRUSH rule 0 x 990 [1,6]
+  CRUSH rule 0 x 991 [0,4]
+  CRUSH rule 0 x 992 [7,0]
+  CRUSH rule 0 x 993 [0,6]
+  CRUSH rule 0 x 994 [3,6]
+  CRUSH rule 0 x 995 [7,0]
+  CRUSH rule 0 x 996 [6,5]
+  CRUSH rule 0 x 997 [6,3]
+  CRUSH rule 0 x 998 [8,0]
+  CRUSH rule 0 x 999 [0,7]
+  CRUSH rule 0 x 1000 [8,4]
+  CRUSH rule 0 x 1001 [2,3]
+  CRUSH rule 0 x 1002 [1,3]
+  CRUSH rule 0 x 1003 [2,7]
+  CRUSH rule 0 x 1004 [6,0]
+  CRUSH rule 0 x 1005 [6,1]
+  CRUSH rule 0 x 1006 [1,8]
+  CRUSH rule 0 x 1007 [1,3]
+  CRUSH rule 0 x 1008 [1,7]
+  CRUSH rule 0 x 1009 [6,5]
+  CRUSH rule 0 x 1010 [3,1]
+  CRUSH rule 0 x 1011 [3,0]
+  CRUSH rule 0 x 1012 [3,1]
+  CRUSH rule 0 x 1013 [5,2]
+  CRUSH rule 0 x 1014 [2,8]
+  CRUSH rule 0 x 1015 [6,3]
+  CRUSH rule 0 x 1016 [2,5]
+  CRUSH rule 0 x 1017 [6,1]
+  CRUSH rule 0 x 1018 [5,1]
+  CRUSH rule 0 x 1019 [5,8]
+  CRUSH rule 0 x 1020 [5,0]
+  CRUSH rule 0 x 1021 [5,2]
+  CRUSH rule 0 x 1022 [1,7]
+  CRUSH rule 0 x 1023 [3,0]
+  rule 0 (choose) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [0,3,7]
+  CRUSH rule 0 x 1 [0,8,5]
+  CRUSH rule 0 x 2 [1,4,8]
+  CRUSH rule 0 x 3 [8,0,4]
+  CRUSH rule 0 x 4 [5,1,8]
+  CRUSH rule 0 x 5 [7,0,3]
+  CRUSH rule 0 x 6 [2,6,3]
+  CRUSH rule 0 x 7 [5,6,1]
+  CRUSH rule 0 x 8 [5,7,2]
+  CRUSH rule 0 x 9 [2,4,8]
+  CRUSH rule 0 x 10 [0,8,4]
+  CRUSH rule 0 x 11 [0,6,3]
+  CRUSH rule 0 x 12 [0,3,7]
+  CRUSH rule 0 x 13 [3,8,0]
+  CRUSH rule 0 x 14 [7,1,5]
+  CRUSH rule 0 x 15 [7,1,3]
+  CRUSH rule 0 x 16 [3,7,1]
+  CRUSH rule 0 x 17 [5,1,7]
+  CRUSH rule 0 x 18 [1,3,6]
+  CRUSH rule 0 x 19 [7,5,2]
+  CRUSH rule 0 x 20 [2,4,7]
+  CRUSH rule 0 x 21 [3,6,0]
+  CRUSH rule 0 x 22 [8,5,1]
+  CRUSH rule 0 x 23 [3,7,2]
+  CRUSH rule 0 x 24 [1,6,3]
+  CRUSH rule 0 x 25 [3,8,1]
+  CRUSH rule 0 x 26 [2,7,3]
+  CRUSH rule 0 x 27 [3,1,7]
+  CRUSH rule 0 x 28 [6,2,4]
+  CRUSH rule 0 x 29 [8,5,2]
+  CRUSH rule 0 x 30 [5,6,0]
+  CRUSH rule 0 x 31 [8,2,5]
+  CRUSH rule 0 x 32 [3,7,2]
+  CRUSH rule 0 x 33 [2,7,4]
+  CRUSH rule 0 x 34 [2,3,7]
+  CRUSH rule 0 x 35 [0,6,3]
+  CRUSH rule 0 x 36 [3,6,1]
+  CRUSH rule 0 x 37 [0,4,7]
+  CRUSH rule 0 x 38 [4,6,1]
+  CRUSH rule 0 x 39 [3,7,2]
+  CRUSH rule 0 x 40 [7,2,5]
+  CRUSH rule 0 x 41 [0,7,4]
+  CRUSH rule 0 x 42 [4,7,2]
+  CRUSH rule 0 x 43 [0,3,8]
+  CRUSH rule 0 x 44 [1,8,4]
+  CRUSH rule 0 x 45 [8,2,3]
+  CRUSH rule 0 x 46 [2,4,8]
+  CRUSH rule 0 x 47 [4,1,7]
+  CRUSH rule 0 x 48 [4,6,0]
+  CRUSH rule 0 x 49 [5,6,0]
+  CRUSH rule 0 x 50 [3,1,7]
+  CRUSH rule 0 x 51 [3,6,1]
+  CRUSH rule 0 x 52 [8,2,3]
+  CRUSH rule 0 x 53 [3,6,2]
+  CRUSH rule 0 x 54 [7,4,0]
+  CRUSH rule 0 x 55 [8,0,3]
+  CRUSH rule 0 x 56 [6,5,1]
+  CRUSH rule 0 x 57 [5,8,0]
+  CRUSH rule 0 x 58 [1,8,5]
+  CRUSH rule 0 x 59 [4,0,7]
+  CRUSH rule 0 x 60 [3,1,8]
+  CRUSH rule 0 x 61 [4,8,0]
+  CRUSH rule 0 x 62 [7,0,4]
+  CRUSH rule 0 x 63 [5,6,2]
+  CRUSH rule 0 x 64 [4,1,8]
+  CRUSH rule 0 x 65 [7,4,2]
+  CRUSH rule 0 x 66 [5,6,0]
+  CRUSH rule 0 x 67 [5,0,6]
+  CRUSH rule 0 x 68 [0,3,8]
+  CRUSH rule 0 x 69 [5,2,7]
+  CRUSH rule 0 x 70 [7,2,3]
+  CRUSH rule 0 x 71 [2,7,4]
+  CRUSH rule 0 x 72 [6,2,4]
+  CRUSH rule 0 x 73 [2,7,3]
+  CRUSH rule 0 x 74 [0,8,3]
+  CRUSH rule 0 x 75 [3,0,7]
+  CRUSH rule 0 x 76 [5,2,6]
+  CRUSH rule 0 x 77 [7,0,5]
+  CRUSH rule 0 x 78 [1,4,7]
+  CRUSH rule 0 x 79 [5,0,7]
+  CRUSH rule 0 x 80 [0,4,6]
+  CRUSH rule 0 x 81 [0,5,8]
+  CRUSH rule 0 x 82 [7,2,5]
+  CRUSH rule 0 x 83 [2,6,4]
+  CRUSH rule 0 x 84 [7,0,3]
+  CRUSH rule 0 x 85 [3,7,0]
+  CRUSH rule 0 x 86 [0,6,4]
+  CRUSH rule 0 x 87 [0,6,5]
+  CRUSH rule 0 x 88 [1,8,4]
+  CRUSH rule 0 x 89 [3,1,8]
+  CRUSH rule 0 x 90 [6,3,2]
+  CRUSH rule 0 x 91 [3,6,2]
+  CRUSH rule 0 x 92 [1,6,3]
+  CRUSH rule 0 x 93 [7,4,0]
+  CRUSH rule 0 x 94 [0,3,7]
+  CRUSH rule 0 x 95 [7,3,1]
+  CRUSH rule 0 x 96 [3,8,2]
+  CRUSH rule 0 x 97 [8,4,1]
+  CRUSH rule 0 x 98 [2,7,4]
+  CRUSH rule 0 x 99 [0,8,5]
+  CRUSH rule 0 x 100 [1,6,3]
+  CRUSH rule 0 x 101 [3,8,1]
+  CRUSH rule 0 x 102 [4,0,7]
+  CRUSH rule 0 x 103 [4,7,0]
+  CRUSH rule 0 x 104 [7,4,0]
+  CRUSH rule 0 x 105 [2,3,8]
+  CRUSH rule 0 x 106 [1,6,5]
+  CRUSH rule 0 x 107 [3,1,7]
+  CRUSH rule 0 x 108 [7,1,3]
+  CRUSH rule 0 x 109 [1,3,8]
+  CRUSH rule 0 x 110 [3,2,6]
+  CRUSH rule 0 x 111 [2,4,7]
+  CRUSH rule 0 x 112 [2,6,3]
+  CRUSH rule 0 x 113 [6,1,4]
+  CRUSH rule 0 x 114 [7,5,0]
+  CRUSH rule 0 x 115 [8,2,3]
+  CRUSH rule 0 x 116 [1,7,4]
+  CRUSH rule 0 x 117 [7,5,1]
+  CRUSH rule 0 x 118 [0,3,6]
+  CRUSH rule 0 x 119 [5,7,1]
+  CRUSH rule 0 x 120 [0,5,6]
+  CRUSH rule 0 x 121 [2,8,4]
+  CRUSH rule 0 x 122 [8,4,2]
+  CRUSH rule 0 x 123 [2,4,7]
+  CRUSH rule 0 x 124 [3,0,8]
+  CRUSH rule 0 x 125 [0,7,3]
+  CRUSH rule 0 x 126 [4,1,7]
+  CRUSH rule 0 x 127 [3,6,1]
+  CRUSH rule 0 x 128 [3,8,1]
+  CRUSH rule 0 x 129 [0,4,7]
+  CRUSH rule 0 x 130 [3,8,1]
+  CRUSH rule 0 x 131 [1,5,6]
+  CRUSH rule 0 x 132 [1,3,8]
+  CRUSH rule 0 x 133 [3,6,1]
+  CRUSH rule 0 x 134 [1,7,4]
+  CRUSH rule 0 x 135 [5,7,0]
+  CRUSH rule 0 x 136 [2,3,8]
+  CRUSH rule 0 x 137 [7,3,1]
+  CRUSH rule 0 x 138 [8,3,0]
+  CRUSH rule 0 x 139 [3,0,8]
+  CRUSH rule 0 x 140 [1,8,3]
+  CRUSH rule 0 x 141 [6,0,4]
+  CRUSH rule 0 x 142 [3,2,8]
+  CRUSH rule 0 x 143 [5,7,0]
+  CRUSH rule 0 x 144 [8,2,5]
+  CRUSH rule 0 x 145 [8,4,2]
+  CRUSH rule 0 x 146 [2,8,4]
+  CRUSH rule 0 x 147 [2,6,4]
+  CRUSH rule 0 x 148 [3,0,8]
+  CRUSH rule 0 x 149 [4,6,2]
+  CRUSH rule 0 x 150 [1,8,4]
+  CRUSH rule 0 x 151 [3,8,1]
+  CRUSH rule 0 x 152 [8,3,0]
+  CRUSH rule 0 x 153 [8,3,0]
+  CRUSH rule 0 x 154 [3,0,6]
+  CRUSH rule 0 x 155 [3,6,0]
+  CRUSH rule 0 x 156 [4,2,7]
+  CRUSH rule 0 x 157 [4,1,7]
+  CRUSH rule 0 x 158 [2,6,3]
+  CRUSH rule 0 x 159 [7,0,3]
+  CRUSH rule 0 x 160 [2,8,5]
+  CRUSH rule 0 x 161 [1,3,6]
+  CRUSH rule 0 x 162 [0,8,5]
+  CRUSH rule 0 x 163 [5,8,2]
+  CRUSH rule 0 x 164 [7,2,4]
+  CRUSH rule 0 x 165 [7,2,4]
+  CRUSH rule 0 x 166 [2,3,7]
+  CRUSH rule 0 x 167 [0,6,3]
+  CRUSH rule 0 x 168 [4,0,7]
+  CRUSH rule 0 x 169 [2,7,3]
+  CRUSH rule 0 x 170 [1,3,8]
+  CRUSH rule 0 x 171 [7,3,2]
+  CRUSH rule 0 x 172 [0,8,4]
+  CRUSH rule 0 x 173 [8,5,1]
+  CRUSH rule 0 x 174 [1,4,8]
+  CRUSH rule 0 x 175 [6,0,3]
+  CRUSH rule 0 x 176 [4,2,7]
+  CRUSH rule 0 x 177 [5,1,8]
+  CRUSH rule 0 x 178 [3,0,6]
+  CRUSH rule 0 x 179 [4,1,7]
+  CRUSH rule 0 x 180 [3,7,2]
+  CRUSH rule 0 x 181 [6,0,5]
+  CRUSH rule 0 x 182 [8,3,1]
+  CRUSH rule 0 x 183 [7,5,1]
+  CRUSH rule 0 x 184 [5,6,1]
+  CRUSH rule 0 x 185 [6,0,3]
+  CRUSH rule 0 x 186 [2,4,8]
+  CRUSH rule 0 x 187 [1,6,3]
+  CRUSH rule 0 x 188 [1,6,3]
+  CRUSH rule 0 x 189 [0,6,5]
+  CRUSH rule 0 x 190 [4,1,8]
+  CRUSH rule 0 x 191 [7,0,5]
+  CRUSH rule 0 x 192 [5,2,6]
+  CRUSH rule 0 x 193 [4,0,6]
+  CRUSH rule 0 x 194 [1,3,7]
+  CRUSH rule 0 x 195 [6,5,2]
+  CRUSH rule 0 x 196 [6,1,3]
+  CRUSH rule 0 x 197 [6,5,1]
+  CRUSH rule 0 x 198 [2,3,6]
+  CRUSH rule 0 x 199 [0,5,7]
+  CRUSH rule 0 x 200 [0,3,8]
+  CRUSH rule 0 x 201 [7,1,5]
+  CRUSH rule 0 x 202 [6,4,1]
+  CRUSH rule 0 x 203 [4,6,1]
+  CRUSH rule 0 x 204 [2,4,8]
+  CRUSH rule 0 x 205 [0,7,4]
+  CRUSH rule 0 x 206 [0,8,4]
+  CRUSH rule 0 x 207 [3,2,7]
+  CRUSH rule 0 x 208 [7,2,4]
+  CRUSH rule 0 x 209 [1,8,3]
+  CRUSH rule 0 x 210 [1,4,6]
+  CRUSH rule 0 x 211 [5,2,7]
+  CRUSH rule 0 x 212 [7,5,0]
+  CRUSH rule 0 x 213 [8,4,0]
+  CRUSH rule 0 x 214 [4,7,1]
+  CRUSH rule 0 x 215 [8,0,5]
+  CRUSH rule 0 x 216 [5,2,8]
+  CRUSH rule 0 x 217 [0,7,5]
+  CRUSH rule 0 x 218 [0,7,4]
+  CRUSH rule 0 x 219 [4,6,0]
+  CRUSH rule 0 x 220 [5,8,0]
+  CRUSH rule 0 x 221 [3,7,0]
+  CRUSH rule 0 x 222 [6,4,1]
+  CRUSH rule 0 x 223 [1,3,6]
+  CRUSH rule 0 x 224 [1,3,8]
+  CRUSH rule 0 x 225 [8,0,3]
+  CRUSH rule 0 x 226 [7,0,4]
+  CRUSH rule 0 x 227 [3,0,7]
+  CRUSH rule 0 x 228 [5,6,1]
+  CRUSH rule 0 x 229 [3,7,0]
+  CRUSH rule 0 x 230 [4,6,1]
+  CRUSH rule 0 x 231 [4,7,1]
+  CRUSH rule 0 x 232 [2,8,4]
+  CRUSH rule 0 x 233 [3,6,0]
+  CRUSH rule 0 x 234 [0,4,6]
+  CRUSH rule 0 x 235 [3,6,1]
+  CRUSH rule 0 x 236 [5,0,8]
+  CRUSH rule 0 x 237 [4,8,0]
+  CRUSH rule 0 x 238 [4,2,6]
+  CRUSH rule 0 x 239 [8,3,2]
+  CRUSH rule 0 x 240 [5,8,2]
+  CRUSH rule 0 x 241 [3,1,7]
+  CRUSH rule 0 x 242 [3,2,6]
+  CRUSH rule 0 x 243 [4,8,2]
+  CRUSH rule 0 x 244 [4,6,0]
+  CRUSH rule 0 x 245 [7,2,3]
+  CRUSH rule 0 x 246 [1,3,7]
+  CRUSH rule 0 x 247 [6,1,4]
+  CRUSH rule 0 x 248 [8,0,3]
+  CRUSH rule 0 x 249 [2,5,8]
+  CRUSH rule 0 x 250 [2,3,8]
+  CRUSH rule 0 x 251 [2,3,7]
+  CRUSH rule 0 x 252 [3,7,2]
+  CRUSH rule 0 x 253 [3,0,7]
+  CRUSH rule 0 x 254 [3,2,8]
+  CRUSH rule 0 x 255 [1,7,4]
+  CRUSH rule 0 x 256 [5,6,1]
+  CRUSH rule 0 x 257 [2,6,4]
+  CRUSH rule 0 x 258 [5,2,8]
+  CRUSH rule 0 x 259 [4,6,1]
+  CRUSH rule 0 x 260 [3,8,1]
+  CRUSH rule 0 x 261 [8,5,2]
+  CRUSH rule 0 x 262 [5,6,2]
+  CRUSH rule 0 x 263 [6,1,5]
+  CRUSH rule 0 x 264 [3,6,1]
+  CRUSH rule 0 x 265 [8,4,0]
+  CRUSH rule 0 x 266 [8,0,4]
+  CRUSH rule 0 x 267 [2,4,6]
+  CRUSH rule 0 x 268 [0,6,4]
+  CRUSH rule 0 x 269 [0,6,5]
+  CRUSH rule 0 x 270 [5,1,8]
+  CRUSH rule 0 x 271 [7,4,2]
+  CRUSH rule 0 x 272 [2,6,5]
+  CRUSH rule 0 x 273 [3,2,7]
+  CRUSH rule 0 x 274 [6,3,2]
+  CRUSH rule 0 x 275 [4,8,1]
+  CRUSH rule 0 x 276 [7,0,5]
+  CRUSH rule 0 x 277 [6,4,1]
+  CRUSH rule 0 x 278 [6,2,3]
+  CRUSH rule 0 x 279 [8,5,0]
+  CRUSH rule 0 x 280 [0,7,4]
+  CRUSH rule 0 x 281 [8,1,5]
+  CRUSH rule 0 x 282 [3,2,8]
+  CRUSH rule 0 x 283 [8,1,3]
+  CRUSH rule 0 x 284 [6,3,1]
+  CRUSH rule 0 x 285 [5,7,2]
+  CRUSH rule 0 x 286 [2,8,4]
+  CRUSH rule 0 x 287 [0,3,8]
+  CRUSH rule 0 x 288 [8,1,4]
+  CRUSH rule 0 x 289 [4,6,0]
+  CRUSH rule 0 x 290 [1,5,6]
+  CRUSH rule 0 x 291 [0,5,8]
+  CRUSH rule 0 x 292 [8,1,4]
+  CRUSH rule 0 x 293 [6,2,3]
+  CRUSH rule 0 x 294 [7,5,2]
+  CRUSH rule 0 x 295 [4,6,0]
+  CRUSH rule 0 x 296 [3,1,6]
+  CRUSH rule 0 x 297 [6,0,5]
+  CRUSH rule 0 x 298 [1,5,6]
+  CRUSH rule 0 x 299 [2,8,4]
+  CRUSH rule 0 x 300 [8,4,1]
+  CRUSH rule 0 x 301 [0,7,3]
+  CRUSH rule 0 x 302 [3,1,6]
+  CRUSH rule 0 x 303 [7,4,1]
+  CRUSH rule 0 x 304 [2,6,4]
+  CRUSH rule 0 x 305 [5,6,1]
+  CRUSH rule 0 x 306 [0,8,3]
+  CRUSH rule 0 x 307 [0,7,4]
+  CRUSH rule 0 x 308 [0,8,5]
+  CRUSH rule 0 x 309 [7,4,2]
+  CRUSH rule 0 x 310 [4,0,6]
+  CRUSH rule 0 x 311 [3,8,2]
+  CRUSH rule 0 x 312 [2,6,5]
+  CRUSH rule 0 x 313 [5,1,7]
+  CRUSH rule 0 x 314 [4,0,6]
+  CRUSH rule 0 x 315 [2,3,7]
+  CRUSH rule 0 x 316 [6,4,1]
+  CRUSH rule 0 x 317 [2,7,4]
+  CRUSH rule 0 x 318 [8,1,4]
+  CRUSH rule 0 x 319 [5,2,6]
+  CRUSH rule 0 x 320 [3,8,0]
+  CRUSH rule 0 x 321 [1,5,8]
+  CRUSH rule 0 x 322 [2,6,3]
+  CRUSH rule 0 x 323 [4,7,0]
+  CRUSH rule 0 x 324 [7,2,5]
+  CRUSH rule 0 x 325 [4,6,0]
+  CRUSH rule 0 x 326 [3,1,7]
+  CRUSH rule 0 x 327 [0,8,3]
+  CRUSH rule 0 x 328 [7,3,1]
+  CRUSH rule 0 x 329 [5,7,0]
+  CRUSH rule 0 x 330 [3,7,0]
+  CRUSH rule 0 x 331 [2,7,3]
+  CRUSH rule 0 x 332 [2,3,8]
+  CRUSH rule 0 x 333 [6,4,1]
+  CRUSH rule 0 x 334 [8,3,2]
+  CRUSH rule 0 x 335 [7,2,3]
+  CRUSH rule 0 x 336 [4,6,2]
+  CRUSH rule 0 x 337 [7,0,4]
+  CRUSH rule 0 x 338 [5,8,2]
+  CRUSH rule 0 x 339 [7,5,0]
+  CRUSH rule 0 x 340 [2,6,5]
+  CRUSH rule 0 x 341 [5,1,7]
+  CRUSH rule 0 x 342 [0,8,5]
+  CRUSH rule 0 x 343 [6,5,0]
+  CRUSH rule 0 x 344 [6,1,4]
+  CRUSH rule 0 x 345 [4,7,0]
+  CRUSH rule 0 x 346 [8,0,3]
+  CRUSH rule 0 x 347 [3,0,8]
+  CRUSH rule 0 x 348 [8,0,3]
+  CRUSH rule 0 x 349 [1,7,3]
+  CRUSH rule 0 x 350 [8,5,1]
+  CRUSH rule 0 x 351 [3,8,0]
+  CRUSH rule 0 x 352 [1,8,4]
+  CRUSH rule 0 x 353 [6,5,1]
+  CRUSH rule 0 x 354 [0,5,6]
+  CRUSH rule 0 x 355 [3,8,0]
+  CRUSH rule 0 x 356 [3,1,8]
+  CRUSH rule 0 x 357 [6,1,3]
+  CRUSH rule 0 x 358 [2,8,5]
+  CRUSH rule 0 x 359 [6,0,5]
+  CRUSH rule 0 x 360 [5,0,8]
+  CRUSH rule 0 x 361 [8,5,0]
+  CRUSH rule 0 x 362 [4,0,8]
+  CRUSH rule 0 x 363 [4,2,8]
+  CRUSH rule 0 x 364 [2,5,7]
+  CRUSH rule 0 x 365 [6,4,2]
+  CRUSH rule 0 x 366 [7,0,3]
+  CRUSH rule 0 x 367 [4,2,7]
+  CRUSH rule 0 x 368 [7,3,1]
+  CRUSH rule 0 x 369 [3,7,2]
+  CRUSH rule 0 x 370 [8,2,4]
+  CRUSH rule 0 x 371 [1,5,8]
+  CRUSH rule 0 x 372 [3,1,8]
+  CRUSH rule 0 x 373 [0,6,4]
+  CRUSH rule 0 x 374 [3,8,1]
+  CRUSH rule 0 x 375 [6,5,2]
+  CRUSH rule 0 x 376 [7,1,3]
+  CRUSH rule 0 x 377 [1,4,7]
+  CRUSH rule 0 x 378 [0,6,4]
+  CRUSH rule 0 x 379 [8,3,0]
+  CRUSH rule 0 x 380 [2,3,8]
+  CRUSH rule 0 x 381 [0,3,7]
+  CRUSH rule 0 x 382 [1,3,7]
+  CRUSH rule 0 x 383 [4,7,1]
+  CRUSH rule 0 x 384 [7,0,4]
+  CRUSH rule 0 x 385 [7,4,0]
+  CRUSH rule 0 x 386 [0,4,6]
+  CRUSH rule 0 x 387 [1,5,8]
+  CRUSH rule 0 x 388 [5,2,7]
+  CRUSH rule 0 x 389 [1,4,8]
+  CRUSH rule 0 x 390 [5,8,1]
+  CRUSH rule 0 x 391 [5,6,0]
+  CRUSH rule 0 x 392 [1,7,5]
+  CRUSH rule 0 x 393 [4,0,6]
+  CRUSH rule 0 x 394 [4,8,2]
+  CRUSH rule 0 x 395 [4,0,8]
+  CRUSH rule 0 x 396 [4,2,6]
+  CRUSH rule 0 x 397 [2,4,7]
+  CRUSH rule 0 x 398 [2,3,6]
+  CRUSH rule 0 x 399 [8,5,2]
+  CRUSH rule 0 x 400 [8,1,3]
+  CRUSH rule 0 x 401 [0,3,6]
+  CRUSH rule 0 x 402 [7,4,2]
+  CRUSH rule 0 x 403 [0,4,7]
+  CRUSH rule 0 x 404 [4,0,6]
+  CRUSH rule 0 x 405 [6,4,0]
+  CRUSH rule 0 x 406 [2,6,4]
+  CRUSH rule 0 x 407 [2,7,5]
+  CRUSH rule 0 x 408 [4,1,7]
+  CRUSH rule 0 x 409 [7,5,0]
+  CRUSH rule 0 x 410 [8,3,1]
+  CRUSH rule 0 x 411 [2,7,4]
+  CRUSH rule 0 x 412 [0,3,7]
+  CRUSH rule 0 x 413 [5,2,8]
+  CRUSH rule 0 x 414 [4,0,8]
+  CRUSH rule 0 x 415 [0,6,3]
+  CRUSH rule 0 x 416 [2,8,5]
+  CRUSH rule 0 x 417 [8,0,3]
+  CRUSH rule 0 x 418 [7,1,3]
+  CRUSH rule 0 x 419 [8,5,2]
+  CRUSH rule 0 x 420 [1,4,7]
+  CRUSH rule 0 x 421 [8,3,0]
+  CRUSH rule 0 x 422 [6,3,1]
+  CRUSH rule 0 x 423 [0,3,7]
+  CRUSH rule 0 x 424 [8,5,1]
+  CRUSH rule 0 x 425 [1,4,8]
+  CRUSH rule 0 x 426 [6,2,4]
+  CRUSH rule 0 x 427 [0,8,3]
+  CRUSH rule 0 x 428 [5,8,1]
+  CRUSH rule 0 x 429 [4,8,0]
+  CRUSH rule 0 x 430 [3,7,0]
+  CRUSH rule 0 x 431 [5,0,7]
+  CRUSH rule 0 x 432 [7,0,4]
+  CRUSH rule 0 x 433 [6,4,0]
+  CRUSH rule 0 x 434 [5,0,7]
+  CRUSH rule 0 x 435 [0,4,6]
+  CRUSH rule 0 x 436 [4,0,7]
+  CRUSH rule 0 x 437 [7,3,1]
+  CRUSH rule 0 x 438 [0,5,8]
+  CRUSH rule 0 x 439 [1,3,8]
+  CRUSH rule 0 x 440 [2,6,5]
+  CRUSH rule 0 x 441 [5,8,0]
+  CRUSH rule 0 x 442 [2,3,6]
+  CRUSH rule 0 x 443 [6,0,3]
+  CRUSH rule 0 x 444 [7,1,4]
+  CRUSH rule 0 x 445 [6,4,0]
+  CRUSH rule 0 x 446 [4,0,8]
+  CRUSH rule 0 x 447 [2,4,6]
+  CRUSH rule 0 x 448 [7,0,5]
+  CRUSH rule 0 x 449 [7,4,2]
+  CRUSH rule 0 x 450 [4,0,6]
+  CRUSH rule 0 x 451 [6,5,1]
+  CRUSH rule 0 x 452 [8,4,0]
+  CRUSH rule 0 x 453 [6,5,0]
+  CRUSH rule 0 x 454 [6,5,0]
+  CRUSH rule 0 x 455 [2,8,4]
+  CRUSH rule 0 x 456 [6,2,5]
+  CRUSH rule 0 x 457 [7,1,5]
+  CRUSH rule 0 x 458 [2,8,5]
+  CRUSH rule 0 x 459 [2,6,5]
+  CRUSH rule 0 x 460 [6,5,2]
+  CRUSH rule 0 x 461 [6,4,2]
+  CRUSH rule 0 x 462 [8,0,4]
+  CRUSH rule 0 x 463 [6,0,4]
+  CRUSH rule 0 x 464 [7,4,1]
+  CRUSH rule 0 x 465 [7,1,5]
+  CRUSH rule 0 x 466 [5,6,2]
+  CRUSH rule 0 x 467 [6,5,1]
+  CRUSH rule 0 x 468 [7,2,3]
+  CRUSH rule 0 x 469 [7,2,4]
+  CRUSH rule 0 x 470 [3,0,8]
+  CRUSH rule 0 x 471 [0,6,4]
+  CRUSH rule 0 x 472 [5,0,8]
+  CRUSH rule 0 x 473 [1,4,7]
+  CRUSH rule 0 x 474 [6,1,4]
+  CRUSH rule 0 x 475 [6,2,3]
+  CRUSH rule 0 x 476 [4,7,1]
+  CRUSH rule 0 x 477 [5,6,1]
+  CRUSH rule 0 x 478 [6,1,3]
+  CRUSH rule 0 x 479 [0,3,6]
+  CRUSH rule 0 x 480 [1,6,4]
+  CRUSH rule 0 x 481 [2,5,7]
+  CRUSH rule 0 x 482 [4,8,0]
+  CRUSH rule 0 x 483 [0,6,4]
+  CRUSH rule 0 x 484 [1,8,5]
+  CRUSH rule 0 x 485 [4,8,1]
+  CRUSH rule 0 x 486 [4,0,8]
+  CRUSH rule 0 x 487 [5,0,8]
+  CRUSH rule 0 x 488 [5,7,1]
+  CRUSH rule 0 x 489 [2,8,4]
+  CRUSH rule 0 x 490 [6,4,0]
+  CRUSH rule 0 x 491 [1,6,5]
+  CRUSH rule 0 x 492 [6,5,2]
+  CRUSH rule 0 x 493 [0,8,3]
+  CRUSH rule 0 x 494 [1,6,4]
+  CRUSH rule 0 x 495 [3,0,6]
+  CRUSH rule 0 x 496 [7,5,1]
+  CRUSH rule 0 x 497 [5,7,2]
+  CRUSH rule 0 x 498 [0,4,6]
+  CRUSH rule 0 x 499 [8,5,2]
+  CRUSH rule 0 x 500 [3,6,1]
+  CRUSH rule 0 x 501 [0,7,5]
+  CRUSH rule 0 x 502 [7,1,3]
+  CRUSH rule 0 x 503 [2,3,7]
+  CRUSH rule 0 x 504 [5,8,2]
+  CRUSH rule 0 x 505 [0,7,5]
+  CRUSH rule 0 x 506 [5,1,7]
+  CRUSH rule 0 x 507 [6,0,3]
+  CRUSH rule 0 x 508 [0,4,7]
+  CRUSH rule 0 x 509 [7,4,0]
+  CRUSH rule 0 x 510 [6,2,5]
+  CRUSH rule 0 x 511 [5,6,2]
+  CRUSH rule 0 x 512 [7,2,4]
+  CRUSH rule 0 x 513 [7,2,4]
+  CRUSH rule 0 x 514 [4,7,0]
+  CRUSH rule 0 x 515 [8,3,0]
+  CRUSH rule 0 x 516 [4,1,7]
+  CRUSH rule 0 x 517 [7,0,4]
+  CRUSH rule 0 x 518 [4,6,0]
+  CRUSH rule 0 x 519 [7,3,1]
+  CRUSH rule 0 x 520 [2,8,5]
+  CRUSH rule 0 x 521 [8,0,3]
+  CRUSH rule 0 x 522 [6,0,4]
+  CRUSH rule 0 x 523 [4,1,7]
+  CRUSH rule 0 x 524 [0,4,7]
+  CRUSH rule 0 x 525 [0,3,8]
+  CRUSH rule 0 x 526 [1,3,8]
+  CRUSH rule 0 x 527 [0,4,6]
+  CRUSH rule 0 x 528 [5,2,7]
+  CRUSH rule 0 x 529 [5,6,1]
+  CRUSH rule 0 x 530 [6,4,1]
+  CRUSH rule 0 x 531 [6,0,3]
+  CRUSH rule 0 x 532 [6,5,1]
+  CRUSH rule 0 x 533 [5,8,0]
+  CRUSH rule 0 x 534 [7,4,2]
+  CRUSH rule 0 x 535 [8,0,3]
+  CRUSH rule 0 x 536 [6,2,3]
+  CRUSH rule 0 x 537 [3,8,0]
+  CRUSH rule 0 x 538 [6,4,2]
+  CRUSH rule 0 x 539 [8,4,2]
+  CRUSH rule 0 x 540 [0,7,4]
+  CRUSH rule 0 x 541 [2,5,7]
+  CRUSH rule 0 x 542 [3,0,8]
+  CRUSH rule 0 x 543 [6,2,4]
+  CRUSH rule 0 x 544 [3,7,1]
+  CRUSH rule 0 x 545 [5,7,1]
+  CRUSH rule 0 x 546 [6,2,3]
+  CRUSH rule 0 x 547 [8,1,5]
+  CRUSH rule 0 x 548 [5,1,8]
+  CRUSH rule 0 x 549 [5,7,1]
+  CRUSH rule 0 x 550 [0,5,6]
+  CRUSH rule 0 x 551 [7,4,0]
+  CRUSH rule 0 x 552 [5,7,2]
+  CRUSH rule 0 x 553 [4,2,8]
+  CRUSH rule 0 x 554 [0,6,5]
+  CRUSH rule 0 x 555 [5,0,8]
+  CRUSH rule 0 x 556 [3,6,1]
+  CRUSH rule 0 x 557 [7,4,0]
+  CRUSH rule 0 x 558 [3,2,6]
+  CRUSH rule 0 x 559 [4,1,8]
+  CRUSH rule 0 x 560 [8,4,2]
+  CRUSH rule 0 x 561 [6,4,1]
+  CRUSH rule 0 x 562 [3,0,8]
+  CRUSH rule 0 x 563 [2,7,4]
+  CRUSH rule 0 x 564 [5,1,7]
+  CRUSH rule 0 x 565 [3,8,0]
+  CRUSH rule 0 x 566 [4,6,0]
+  CRUSH rule 0 x 567 [3,7,0]
+  CRUSH rule 0 x 568 [7,4,0]
+  CRUSH rule 0 x 569 [3,1,6]
+  CRUSH rule 0 x 570 [1,4,7]
+  CRUSH rule 0 x 571 [3,6,0]
+  CRUSH rule 0 x 572 [3,0,7]
+  CRUSH rule 0 x 573 [3,1,7]
+  CRUSH rule 0 x 574 [2,3,6]
+  CRUSH rule 0 x 575 [8,1,5]
+  CRUSH rule 0 x 576 [4,7,1]
+  CRUSH rule 0 x 577 [8,2,3]
+  CRUSH rule 0 x 578 [6,0,4]
+  CRUSH rule 0 x 579 [3,2,6]
+  CRUSH rule 0 x 580 [3,0,8]
+  CRUSH rule 0 x 581 [7,1,5]
+  CRUSH rule 0 x 582 [2,8,5]
+  CRUSH rule 0 x 583 [6,2,3]
+  CRUSH rule 0 x 584 [8,0,5]
+  CRUSH rule 0 x 585 [7,0,5]
+  CRUSH rule 0 x 586 [0,8,5]
+  CRUSH rule 0 x 587 [2,5,6]
+  CRUSH rule 0 x 588 [3,8,0]
+  CRUSH rule 0 x 589 [7,1,5]
+  CRUSH rule 0 x 590 [6,2,5]
+  CRUSH rule 0 x 591 [5,2,7]
+  CRUSH rule 0 x 592 [2,4,6]
+  CRUSH rule 0 x 593 [0,8,4]
+  CRUSH rule 0 x 594 [0,6,3]
+  CRUSH rule 0 x 595 [7,1,5]
+  CRUSH rule 0 x 596 [4,0,6]
+  CRUSH rule 0 x 597 [3,1,7]
+  CRUSH rule 0 x 598 [3,0,7]
+  CRUSH rule 0 x 599 [5,1,6]
+  CRUSH rule 0 x 600 [7,0,4]
+  CRUSH rule 0 x 601 [0,6,3]
+  CRUSH rule 0 x 602 [3,8,2]
+  CRUSH rule 0 x 603 [5,0,8]
+  CRUSH rule 0 x 604 [7,4,0]
+  CRUSH rule 0 x 605 [3,2,6]
+  CRUSH rule 0 x 606 [2,7,3]
+  CRUSH rule 0 x 607 [0,5,8]
+  CRUSH rule 0 x 608 [5,1,8]
+  CRUSH rule 0 x 609 [5,0,6]
+  CRUSH rule 0 x 610 [3,8,2]
+  CRUSH rule 0 x 611 [1,8,3]
+  CRUSH rule 0 x 612 [2,8,3]
+  CRUSH rule 0 x 613 [7,1,3]
+  CRUSH rule 0 x 614 [7,2,3]
+  CRUSH rule 0 x 615 [6,2,3]
+  CRUSH rule 0 x 616 [0,7,5]
+  CRUSH rule 0 x 617 [6,2,3]
+  CRUSH rule 0 x 618 [7,3,1]
+  CRUSH rule 0 x 619 [5,0,6]
+  CRUSH rule 0 x 620 [4,1,8]
+  CRUSH rule 0 x 621 [5,6,0]
+  CRUSH rule 0 x 622 [0,3,8]
+  CRUSH rule 0 x 623 [0,8,4]
+  CRUSH rule 0 x 624 [3,2,8]
+  CRUSH rule 0 x 625 [2,5,8]
+  CRUSH rule 0 x 626 [7,2,3]
+  CRUSH rule 0 x 627 [2,6,5]
+  CRUSH rule 0 x 628 [8,1,3]
+  CRUSH rule 0 x 629 [2,6,3]
+  CRUSH rule 0 x 630 [2,6,4]
+  CRUSH rule 0 x 631 [0,6,5]
+  CRUSH rule 0 x 632 [7,0,4]
+  CRUSH rule 0 x 633 [8,4,2]
+  CRUSH rule 0 x 634 [0,5,6]
+  CRUSH rule 0 x 635 [5,6,2]
+  CRUSH rule 0 x 636 [1,3,7]
+  CRUSH rule 0 x 637 [4,0,8]
+  CRUSH rule 0 x 638 [6,2,5]
+  CRUSH rule 0 x 639 [4,0,6]
+  CRUSH rule 0 x 640 [3,2,7]
+  CRUSH rule 0 x 641 [7,2,3]
+  CRUSH rule 0 x 642 [2,8,4]
+  CRUSH rule 0 x 643 [3,0,8]
+  CRUSH rule 0 x 644 [8,0,4]
+  CRUSH rule 0 x 645 [5,7,2]
+  CRUSH rule 0 x 646 [8,0,3]
+  CRUSH rule 0 x 647 [7,0,5]
+  CRUSH rule 0 x 648 [0,8,3]
+  CRUSH rule 0 x 649 [4,7,1]
+  CRUSH rule 0 x 650 [7,5,1]
+  CRUSH rule 0 x 651 [3,6,1]
+  CRUSH rule 0 x 652 [3,6,1]
+  CRUSH rule 0 x 653 [8,3,0]
+  CRUSH rule 0 x 654 [7,4,0]
+  CRUSH rule 0 x 655 [0,5,6]
+  CRUSH rule 0 x 656 [4,7,0]
+  CRUSH rule 0 x 657 [6,0,5]
+  CRUSH rule 0 x 658 [5,8,0]
+  CRUSH rule 0 x 659 [4,7,0]
+  CRUSH rule 0 x 660 [7,3,0]
+  CRUSH rule 0 x 661 [1,7,3]
+  CRUSH rule 0 x 662 [4,2,8]
+  CRUSH rule 0 x 663 [1,3,7]
+  CRUSH rule 0 x 664 [1,3,6]
+  CRUSH rule 0 x 665 [5,6,1]
+  CRUSH rule 0 x 666 [2,7,4]
+  CRUSH rule 0 x 667 [1,3,8]
+  CRUSH rule 0 x 668 [3,7,0]
+  CRUSH rule 0 x 669 [6,3,0]
+  CRUSH rule 0 x 670 [4,1,6]
+  CRUSH rule 0 x 671 [0,8,5]
+  CRUSH rule 0 x 672 [4,2,8]
+  CRUSH rule 0 x 673 [5,0,7]
+  CRUSH rule 0 x 674 [3,0,7]
+  CRUSH rule 0 x 675 [0,8,5]
+  CRUSH rule 0 x 676 [0,3,7]
+  CRUSH rule 0 x 677 [4,1,6]
+  CRUSH rule 0 x 678 [2,3,8]
+  CRUSH rule 0 x 679 [6,0,5]
+  CRUSH rule 0 x 680 [0,3,8]
+  CRUSH rule 0 x 681 [4,6,1]
+  CRUSH rule 0 x 682 [0,4,8]
+  CRUSH rule 0 x 683 [0,3,8]
+  CRUSH rule 0 x 684 [7,2,5]
+  CRUSH rule 0 x 685 [7,2,5]
+  CRUSH rule 0 x 686 [1,5,8]
+  CRUSH rule 0 x 687 [3,7,2]
+  CRUSH rule 0 x 688 [5,6,2]
+  CRUSH rule 0 x 689 [6,5,1]
+  CRUSH rule 0 x 690 [8,0,4]
+  CRUSH rule 0 x 691 [3,1,6]
+  CRUSH rule 0 x 692 [7,1,4]
+  CRUSH rule 0 x 693 [6,5,2]
+  CRUSH rule 0 x 694 [6,4,2]
+  CRUSH rule 0 x 695 [0,6,3]
+  CRUSH rule 0 x 696 [1,5,8]
+  CRUSH rule 0 x 697 [6,0,4]
+  CRUSH rule 0 x 698 [6,0,3]
+  CRUSH rule 0 x 699 [1,8,3]
+  CRUSH rule 0 x 700 [0,4,6]
+  CRUSH rule 0 x 701 [4,0,6]
+  CRUSH rule 0 x 702 [3,0,8]
+  CRUSH rule 0 x 703 [8,4,0]
+  CRUSH rule 0 x 704 [0,4,6]
+  CRUSH rule 0 x 705 [8,0,4]
+  CRUSH rule 0 x 706 [1,5,8]
+  CRUSH rule 0 x 707 [7,3,0]
+  CRUSH rule 0 x 708 [3,7,1]
+  CRUSH rule 0 x 709 [6,5,1]
+  CRUSH rule 0 x 710 [8,5,1]
+  CRUSH rule 0 x 711 [2,4,7]
+  CRUSH rule 0 x 712 [2,3,6]
+  CRUSH rule 0 x 713 [6,3,1]
+  CRUSH rule 0 x 714 [3,0,6]
+  CRUSH rule 0 x 715 [1,3,6]
+  CRUSH rule 0 x 716 [3,6,1]
+  CRUSH rule 0 x 717 [8,0,5]
+  CRUSH rule 0 x 718 [3,7,0]
+  CRUSH rule 0 x 719 [2,6,3]
+  CRUSH rule 0 x 720 [6,0,4]
+  CRUSH rule 0 x 721 [5,7,0]
+  CRUSH rule 0 x 722 [5,7,1]
+  CRUSH rule 0 x 723 [5,2,7]
+  CRUSH rule 0 x 724 [0,7,3]
+  CRUSH rule 0 x 725 [0,4,6]
+  CRUSH rule 0 x 726 [3,7,2]
+  CRUSH rule 0 x 727 [4,7,2]
+  CRUSH rule 0 x 728 [2,6,3]
+  CRUSH rule 0 x 729 [5,6,1]
+  CRUSH rule 0 x 730 [3,8,2]
+  CRUSH rule 0 x 731 [4,1,6]
+  CRUSH rule 0 x 732 [1,4,8]
+  CRUSH rule 0 x 733 [5,6,2]
+  CRUSH rule 0 x 734 [6,5,0]
+  CRUSH rule 0 x 735 [4,6,1]
+  CRUSH rule 0 x 736 [3,8,1]
+  CRUSH rule 0 x 737 [1,8,3]
+  CRUSH rule 0 x 738 [5,1,6]
+  CRUSH rule 0 x 739 [0,7,4]
+  CRUSH rule 0 x 740 [0,7,3]
+  CRUSH rule 0 x 741 [7,2,3]
+  CRUSH rule 0 x 742 [8,2,3]
+  CRUSH rule 0 x 743 [7,0,5]
+  CRUSH rule 0 x 744 [4,6,1]
+  CRUSH rule 0 x 745 [3,0,8]
+  CRUSH rule 0 x 746 [4,1,7]
+  CRUSH rule 0 x 747 [6,2,4]
+  CRUSH rule 0 x 748 [2,6,5]
+  CRUSH rule 0 x 749 [4,7,2]
+  CRUSH rule 0 x 750 [1,7,4]
+  CRUSH rule 0 x 751 [2,7,3]
+  CRUSH rule 0 x 752 [8,0,3]
+  CRUSH rule 0 x 753 [7,4,2]
+  CRUSH rule 0 x 754 [8,5,2]
+  CRUSH rule 0 x 755 [1,6,5]
+  CRUSH rule 0 x 756 [5,8,1]
+  CRUSH rule 0 x 757 [8,0,4]
+  CRUSH rule 0 x 758 [6,1,5]
+  CRUSH rule 0 x 759 [8,5,0]
+  CRUSH rule 0 x 760 [1,5,7]
+  CRUSH rule 0 x 761 [4,2,6]
+  CRUSH rule 0 x 762 [2,8,3]
+  CRUSH rule 0 x 763 [8,3,1]
+  CRUSH rule 0 x 764 [1,7,4]
+  CRUSH rule 0 x 765 [6,4,0]
+  CRUSH rule 0 x 766 [8,3,0]
+  CRUSH rule 0 x 767 [1,8,5]
+  CRUSH rule 0 x 768 [8,5,0]
+  CRUSH rule 0 x 769 [6,2,4]
+  CRUSH rule 0 x 770 [6,1,4]
+  CRUSH rule 0 x 771 [7,2,4]
+  CRUSH rule 0 x 772 [8,4,2]
+  CRUSH rule 0 x 773 [3,2,6]
+  CRUSH rule 0 x 774 [4,7,1]
+  CRUSH rule 0 x 775 [6,5,0]
+  CRUSH rule 0 x 776 [7,0,3]
+  CRUSH rule 0 x 777 [3,0,6]
+  CRUSH rule 0 x 778 [1,6,3]
+  CRUSH rule 0 x 779 [2,6,5]
+  CRUSH rule 0 x 780 [0,5,8]
+  CRUSH rule 0 x 781 [6,5,0]
+  CRUSH rule 0 x 782 [5,2,8]
+  CRUSH rule 0 x 783 [7,0,3]
+  CRUSH rule 0 x 784 [0,5,8]
+  CRUSH rule 0 x 785 [6,1,3]
+  CRUSH rule 0 x 786 [7,3,0]
+  CRUSH rule 0 x 787 [1,8,5]
+  CRUSH rule 0 x 788 [6,0,4]
+  CRUSH rule 0 x 789 [0,5,8]
+  CRUSH rule 0 x 790 [8,3,1]
+  CRUSH rule 0 x 791 [3,6,2]
+  CRUSH rule 0 x 792 [5,6,1]
+  CRUSH rule 0 x 793 [6,2,3]
+  CRUSH rule 0 x 794 [2,8,3]
+  CRUSH rule 0 x 795 [0,4,6]
+  CRUSH rule 0 x 796 [3,7,1]
+  CRUSH rule 0 x 797 [2,3,6]
+  CRUSH rule 0 x 798 [6,0,3]
+  CRUSH rule 0 x 799 [5,2,7]
+  CRUSH rule 0 x 800 [5,2,8]
+  CRUSH rule 0 x 801 [3,7,2]
+  CRUSH rule 0 x 802 [1,6,4]
+  CRUSH rule 0 x 803 [0,4,7]
+  CRUSH rule 0 x 804 [6,0,5]
+  CRUSH rule 0 x 805 [3,8,0]
+  CRUSH rule 0 x 806 [1,5,6]
+  CRUSH rule 0 x 807 [5,7,2]
+  CRUSH rule 0 x 808 [4,7,2]
+  CRUSH rule 0 x 809 [1,3,6]
+  CRUSH rule 0 x 810 [5,7,2]
+  CRUSH rule 0 x 811 [8,4,1]
+  CRUSH rule 0 x 812 [8,3,1]
+  CRUSH rule 0 x 813 [6,4,2]
+  CRUSH rule 0 x 814 [3,8,0]
+  CRUSH rule 0 x 815 [3,1,6]
+  CRUSH rule 0 x 816 [2,6,5]
+  CRUSH rule 0 x 817 [4,6,2]
+  CRUSH rule 0 x 818 [3,1,7]
+  CRUSH rule 0 x 819 [5,0,7]
+  CRUSH rule 0 x 820 [3,7,1]
+  CRUSH rule 0 x 821 [4,8,0]
+  CRUSH rule 0 x 822 [2,3,7]
+  CRUSH rule 0 x 823 [4,7,2]
+  CRUSH rule 0 x 824 [3,7,1]
+  CRUSH rule 0 x 825 [2,8,4]
+  CRUSH rule 0 x 826 [7,1,5]
+  CRUSH rule 0 x 827 [0,6,3]
+  CRUSH rule 0 x 828 [2,5,8]
+  CRUSH rule 0 x 829 [5,6,0]
+  CRUSH rule 0 x 830 [2,4,7]
+  CRUSH rule 0 x 831 [1,6,4]
+  CRUSH rule 0 x 832 [4,8,2]
+  CRUSH rule 0 x 833 [2,6,5]
+  CRUSH rule 0 x 834 [3,0,8]
+  CRUSH rule 0 x 835 [8,5,0]
+  CRUSH rule 0 x 836 [3,8,1]
+  CRUSH rule 0 x 837 [6,4,0]
+  CRUSH rule 0 x 838 [6,0,4]
+  CRUSH rule 0 x 839 [5,2,8]
+  CRUSH rule 0 x 840 [7,3,2]
+  CRUSH rule 0 x 841 [4,8,0]
+  CRUSH rule 0 x 842 [2,5,8]
+  CRUSH rule 0 x 843 [6,4,2]
+  CRUSH rule 0 x 844 [4,8,2]
+  CRUSH rule 0 x 845 [3,6,2]
+  CRUSH rule 0 x 846 [3,0,6]
+  CRUSH rule 0 x 847 [0,8,5]
+  CRUSH rule 0 x 848 [2,6,5]
+  CRUSH rule 0 x 849 [4,8,1]
+  CRUSH rule 0 x 850 [1,5,7]
+  CRUSH rule 0 x 851 [6,5,1]
+  CRUSH rule 0 x 852 [7,4,2]
+  CRUSH rule 0 x 853 [6,2,5]
+  CRUSH rule 0 x 854 [7,0,3]
+  CRUSH rule 0 x 855 [5,7,2]
+  CRUSH rule 0 x 856 [6,3,1]
+  CRUSH rule 0 x 857 [8,4,0]
+  CRUSH rule 0 x 858 [6,5,2]
+  CRUSH rule 0 x 859 [6,0,5]
+  CRUSH rule 0 x 860 [4,2,8]
+  CRUSH rule 0 x 861 [8,4,1]
+  CRUSH rule 0 x 862 [6,0,5]
+  CRUSH rule 0 x 863 [8,1,5]
+  CRUSH rule 0 x 864 [5,6,1]
+  CRUSH rule 0 x 865 [8,0,3]
+  CRUSH rule 0 x 866 [3,6,2]
+  CRUSH rule 0 x 867 [6,3,1]
+  CRUSH rule 0 x 868 [6,4,2]
+  CRUSH rule 0 x 869 [8,4,2]
+  CRUSH rule 0 x 870 [0,5,7]
+  CRUSH rule 0 x 871 [3,1,6]
+  CRUSH rule 0 x 872 [5,0,6]
+  CRUSH rule 0 x 873 [4,6,1]
+  CRUSH rule 0 x 874 [2,6,4]
+  CRUSH rule 0 x 875 [2,6,5]
+  CRUSH rule 0 x 876 [5,8,1]
+  CRUSH rule 0 x 877 [6,4,0]
+  CRUSH rule 0 x 878 [5,0,7]
+  CRUSH rule 0 x 879 [7,3,1]
+  CRUSH rule 0 x 880 [3,1,7]
+  CRUSH rule 0 x 881 [5,8,2]
+  CRUSH rule 0 x 882 [4,1,6]
+  CRUSH rule 0 x 883 [2,5,6]
+  CRUSH rule 0 x 884 [6,0,3]
+  CRUSH rule 0 x 885 [5,1,6]
+  CRUSH rule 0 x 886 [3,7,2]
+  CRUSH rule 0 x 887 [7,5,2]
+  CRUSH rule 0 x 888 [6,1,4]
+  CRUSH rule 0 x 889 [2,8,3]
+  CRUSH rule 0 x 890 [7,1,4]
+  CRUSH rule 0 x 891 [1,7,4]
+  CRUSH rule 0 x 892 [6,0,5]
+  CRUSH rule 0 x 893 [2,3,6]
+  CRUSH rule 0 x 894 [7,4,1]
+  CRUSH rule 0 x 895 [5,1,6]
+  CRUSH rule 0 x 896 [1,6,3]
+  CRUSH rule 0 x 897 [4,0,8]
+  CRUSH rule 0 x 898 [0,3,8]
+  CRUSH rule 0 x 899 [1,6,4]
+  CRUSH rule 0 x 900 [4,0,7]
+  CRUSH rule 0 x 901 [5,2,6]
+  CRUSH rule 0 x 902 [8,4,2]
+  CRUSH rule 0 x 903 [5,6,1]
+  CRUSH rule 0 x 904 [5,7,0]
+  CRUSH rule 0 x 905 [6,0,3]
+  CRUSH rule 0 x 906 [1,7,4]
+  CRUSH rule 0 x 907 [7,2,3]
+  CRUSH rule 0 x 908 [5,6,1]
+  CRUSH rule 0 x 909 [2,3,8]
+  CRUSH rule 0 x 910 [6,5,0]
+  CRUSH rule 0 x 911 [5,7,1]
+  CRUSH rule 0 x 912 [0,8,5]
+  CRUSH rule 0 x 913 [7,1,4]
+  CRUSH rule 0 x 914 [6,4,0]
+  CRUSH rule 0 x 915 [8,2,3]
+  CRUSH rule 0 x 916 [3,0,6]
+  CRUSH rule 0 x 917 [1,3,6]
+  CRUSH rule 0 x 918 [8,0,5]
+  CRUSH rule 0 x 919 [6,2,3]
+  CRUSH rule 0 x 920 [7,4,2]
+  CRUSH rule 0 x 921 [1,4,8]
+  CRUSH rule 0 x 922 [6,3,1]
+  CRUSH rule 0 x 923 [5,6,0]
+  CRUSH rule 0 x 924 [3,1,8]
+  CRUSH rule 0 x 925 [5,6,0]
+  CRUSH rule 0 x 926 [3,0,7]
+  CRUSH rule 0 x 927 [1,6,5]
+  CRUSH rule 0 x 928 [8,1,4]
+  CRUSH rule 0 x 929 [4,2,6]
+  CRUSH rule 0 x 930 [2,5,8]
+  CRUSH rule 0 x 931 [5,2,8]
+  CRUSH rule 0 x 932 [4,2,7]
+  CRUSH rule 0 x 933 [8,4,2]
+  CRUSH rule 0 x 934 [5,8,0]
+  CRUSH rule 0 x 935 [6,3,0]
+  CRUSH rule 0 x 936 [0,7,3]
+  CRUSH rule 0 x 937 [5,8,1]
+  CRUSH rule 0 x 938 [6,4,0]
+  CRUSH rule 0 x 939 [2,8,3]
+  CRUSH rule 0 x 940 [8,5,2]
+  CRUSH rule 0 x 941 [5,0,6]
+  CRUSH rule 0 x 942 [1,6,4]
+  CRUSH rule 0 x 943 [8,2,4]
+  CRUSH rule 0 x 944 [4,8,1]
+  CRUSH rule 0 x 945 [7,0,3]
+  CRUSH rule 0 x 946 [2,8,3]
+  CRUSH rule 0 x 947 [4,2,6]
+  CRUSH rule 0 x 948 [7,4,1]
+  CRUSH rule 0 x 949 [6,2,5]
+  CRUSH rule 0 x 950 [3,7,0]
+  CRUSH rule 0 x 951 [4,6,2]
+  CRUSH rule 0 x 952 [2,7,4]
+  CRUSH rule 0 x 953 [1,3,6]
+  CRUSH rule 0 x 954 [4,0,8]
+  CRUSH rule 0 x 955 [8,1,4]
+  CRUSH rule 0 x 956 [1,7,4]
+  CRUSH rule 0 x 957 [7,0,3]
+  CRUSH rule 0 x 958 [8,3,2]
+  CRUSH rule 0 x 959 [5,1,6]
+  CRUSH rule 0 x 960 [3,6,2]
+  CRUSH rule 0 x 961 [4,1,6]
+  CRUSH rule 0 x 962 [7,5,0]
+  CRUSH rule 0 x 963 [0,5,8]
+  CRUSH rule 0 x 964 [3,2,7]
+  CRUSH rule 0 x 965 [7,3,1]
+  CRUSH rule 0 x 966 [3,6,0]
+  CRUSH rule 0 x 967 [8,4,2]
+  CRUSH rule 0 x 968 [7,0,4]
+  CRUSH rule 0 x 969 [8,0,4]
+  CRUSH rule 0 x 970 [0,8,3]
+  CRUSH rule 0 x 971 [1,8,5]
+  CRUSH rule 0 x 972 [1,7,5]
+  CRUSH rule 0 x 973 [1,8,3]
+  CRUSH rule 0 x 974 [5,1,7]
+  CRUSH rule 0 x 975 [3,8,0]
+  CRUSH rule 0 x 976 [4,7,2]
+  CRUSH rule 0 x 977 [8,3,2]
+  CRUSH rule 0 x 978 [7,0,3]
+  CRUSH rule 0 x 979 [7,2,5]
+  CRUSH rule 0 x 980 [6,2,3]
+  CRUSH rule 0 x 981 [7,5,1]
+  CRUSH rule 0 x 982 [4,1,8]
+  CRUSH rule 0 x 983 [3,6,0]
+  CRUSH rule 0 x 984 [0,8,4]
+  CRUSH rule 0 x 985 [2,4,8]
+  CRUSH rule 0 x 986 [8,4,2]
+  CRUSH rule 0 x 987 [0,4,8]
+  CRUSH rule 0 x 988 [1,4,6]
+  CRUSH rule 0 x 989 [0,8,5]
+  CRUSH rule 0 x 990 [1,6,4]
+  CRUSH rule 0 x 991 [0,4,8]
+  CRUSH rule 0 x 992 [7,0,4]
+  CRUSH rule 0 x 993 [0,6,4]
+  CRUSH rule 0 x 994 [3,6,1]
+  CRUSH rule 0 x 995 [7,0,4]
+  CRUSH rule 0 x 996 [6,5,1]
+  CRUSH rule 0 x 997 [6,3,1]
+  CRUSH rule 0 x 998 [8,0,4]
+  CRUSH rule 0 x 999 [0,7,5]
+  CRUSH rule 0 x 1000 [8,4,1]
+  CRUSH rule 0 x 1001 [2,3,6]
+  CRUSH rule 0 x 1002 [1,3,8]
+  CRUSH rule 0 x 1003 [2,7,3]
+  CRUSH rule 0 x 1004 [6,0,3]
+  CRUSH rule 0 x 1005 [6,1,4]
+  CRUSH rule 0 x 1006 [1,8,4]
+  CRUSH rule 0 x 1007 [1,3,8]
+  CRUSH rule 0 x 1008 [1,7,4]
+  CRUSH rule 0 x 1009 [6,5,1]
+  CRUSH rule 0 x 1010 [3,1,6]
+  CRUSH rule 0 x 1011 [3,0,8]
+  CRUSH rule 0 x 1012 [3,1,8]
+  CRUSH rule 0 x 1013 [5,2,8]
+  CRUSH rule 0 x 1014 [2,8,3]
+  CRUSH rule 0 x 1015 [6,3,2]
+  CRUSH rule 0 x 1016 [2,5,6]
+  CRUSH rule 0 x 1017 [6,1,5]
+  CRUSH rule 0 x 1018 [5,1,6]
+  CRUSH rule 0 x 1019 [5,8,1]
+  CRUSH rule 0 x 1020 [5,0,7]
+  CRUSH rule 0 x 1021 [5,2,7]
+  CRUSH rule 0 x 1022 [1,7,4]
+  CRUSH rule 0 x 1023 [3,0,7]
+  rule 0 (choose) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 1 (choose-two), x = 0..1023, numrep = 2..3
+  CRUSH rule 1 x 0 [0,2]
+  CRUSH rule 1 x 1 [0,8]
+  CRUSH rule 1 x 2 [1,3]
+  CRUSH rule 1 x 3 [8,0]
+  CRUSH rule 1 x 4 [5,4]
+  CRUSH rule 1 x 5 [7,8]
+  CRUSH rule 1 x 6 [2,6]
+  CRUSH rule 1 x 7 [5,4]
+  CRUSH rule 1 x 8 [5,3]
+  CRUSH rule 1 x 9 [2,3]
+  CRUSH rule 1 x 10 [0,2]
+  CRUSH rule 1 x 11 [0,7]
+  CRUSH rule 1 x 12 [0,2]
+  CRUSH rule 1 x 13 [3,8]
+  CRUSH rule 1 x 14 [7,6]
+  CRUSH rule 1 x 15 [7,2]
+  CRUSH rule 1 x 16 [3,6]
+  CRUSH rule 1 x 17 [5,4]
+  CRUSH rule 1 x 18 [1,4]
+  CRUSH rule 1 x 19 [7,5]
+  CRUSH rule 1 x 20 [2,4]
+  CRUSH rule 1 x 21 [3,7]
+  CRUSH rule 1 x 22 [8,3]
+  CRUSH rule 1 x 23 [3,6]
+  CRUSH rule 1 x 24 [1,0]
+  CRUSH rule 1 x 25 [3,7]
+  CRUSH rule 1 x 26 [2,8]
+  CRUSH rule 1 x 27 [3,1]
+  CRUSH rule 1 x 28 [6,0]
+  CRUSH rule 1 x 29 [8,5]
+  CRUSH rule 1 x 30 [5,7]
+  CRUSH rule 1 x 31 [8,7]
+  CRUSH rule 1 x 32 [3,6]
+  CRUSH rule 1 x 33 [2,7]
+  CRUSH rule 1 x 34 [2,5]
+  CRUSH rule 1 x 35 [0,8]
+  CRUSH rule 1 x 36 [3,8]
+  CRUSH rule 1 x 37 [0,4]
+  CRUSH rule 1 x 38 [4,8]
+  CRUSH rule 1 x 39 [3,7]
+  CRUSH rule 1 x 40 [7,8]
+  CRUSH rule 1 x 41 [0,2]
+  CRUSH rule 1 x 42 [4,3]
+  CRUSH rule 1 x 43 [0,3]
+  CRUSH rule 1 x 44 [1,6]
+  CRUSH rule 1 x 45 [8,0]
+  CRUSH rule 1 x 46 [2,4]
+  CRUSH rule 1 x 47 [4,5]
+  CRUSH rule 1 x 48 [4,6]
+  CRUSH rule 1 x 49 [5,4]
+  CRUSH rule 1 x 50 [3,4]
+  CRUSH rule 1 x 51 [3,6]
+  CRUSH rule 1 x 52 [8,6]
+  CRUSH rule 1 x 53 [3,5]
+  CRUSH rule 1 x 54 [7,6]
+  CRUSH rule 1 x 55 [8,7]
+  CRUSH rule 1 x 56 [6,4]
+  CRUSH rule 1 x 57 [5,3]
+  CRUSH rule 1 x 58 [1,0]
+  CRUSH rule 1 x 59 [4,2]
+  CRUSH rule 1 x 60 [3,5]
+  CRUSH rule 1 x 61 [4,6]
+  CRUSH rule 1 x 62 [7,0]
+  CRUSH rule 1 x 63 [5,6]
+  CRUSH rule 1 x 64 [4,5]
+  CRUSH rule 1 x 65 [7,3]
+  CRUSH rule 1 x 66 [5,4]
+  CRUSH rule 1 x 67 [5,0]
+  CRUSH rule 1 x 68 [0,5]
+  CRUSH rule 1 x 69 [5,1]
+  CRUSH rule 1 x 70 [7,0]
+  CRUSH rule 1 x 71 [2,8]
+  CRUSH rule 1 x 72 [6,1]
+  CRUSH rule 1 x 73 [2,7]
+  CRUSH rule 1 x 74 [0,7]
+  CRUSH rule 1 x 75 [3,2]
+  CRUSH rule 1 x 76 [5,1]
+  CRUSH rule 1 x 77 [7,2]
+  CRUSH rule 1 x 78 [1,4]
+  CRUSH rule 1 x 79 [5,4]
+  CRUSH rule 1 x 80 [0,3]
+  CRUSH rule 1 x 81 [0,2]
+  CRUSH rule 1 x 82 [7,1]
+  CRUSH rule 1 x 83 [2,6]
+  CRUSH rule 1 x 84 [7,2]
+  CRUSH rule 1 x 85 [3,4]
+  CRUSH rule 1 x 86 [0,2]
+  CRUSH rule 1 x 87 [0,7]
+  CRUSH rule 1 x 88 [1,6]
+  CRUSH rule 1 x 89 [3,0]
+  CRUSH rule 1 x 90 [6,7]
+  CRUSH rule 1 x 91 [3,8]
+  CRUSH rule 1 x 92 [1,8]
+  CRUSH rule 1 x 93 [7,4]
+  CRUSH rule 1 x 94 [0,4]
+  CRUSH rule 1 x 95 [7,5]
+  CRUSH rule 1 x 96 [3,6]
+  CRUSH rule 1 x 97 [8,7]
+  CRUSH rule 1 x 98 [2,0]
+  CRUSH rule 1 x 99 [0,7]
+  CRUSH rule 1 x 100 [1,7]
+  CRUSH rule 1 x 101 [3,7]
+  CRUSH rule 1 x 102 [4,2]
+  CRUSH rule 1 x 103 [4,7]
+  CRUSH rule 1 x 104 [7,4]
+  CRUSH rule 1 x 105 [2,4]
+  CRUSH rule 1 x 106 [1,6]
+  CRUSH rule 1 x 107 [3,2]
+  CRUSH rule 1 x 108 [7,2]
+  CRUSH rule 1 x 109 [1,2]
+  CRUSH rule 1 x 110 [3,2]
+  CRUSH rule 1 x 111 [2,1]
+  CRUSH rule 1 x 112 [2,0]
+  CRUSH rule 1 x 113 [6,2]
+  CRUSH rule 1 x 114 [7,6]
+  CRUSH rule 1 x 115 [8,2]
+  CRUSH rule 1 x 116 [1,2]
+  CRUSH rule 1 x 117 [7,3]
+  CRUSH rule 1 x 118 [0,3]
+  CRUSH rule 1 x 119 [5,6]
+  CRUSH rule 1 x 120 [0,2]
+  CRUSH rule 1 x 121 [2,0]
+  CRUSH rule 1 x 122 [8,5]
+  CRUSH rule 1 x 123 [2,5]
+  CRUSH rule 1 x 124 [3,5]
+  CRUSH rule 1 x 125 [0,7]
+  CRUSH rule 1 x 126 [4,2]
+  CRUSH rule 1 x 127 [3,6]
+  CRUSH rule 1 x 128 [3,5]
+  CRUSH rule 1 x 129 [0,2]
+  CRUSH rule 1 x 130 [3,8]
+  CRUSH rule 1 x 131 [1,2]
+  CRUSH rule 1 x 132 [1,2]
+  CRUSH rule 1 x 133 [3,6]
+  CRUSH rule 1 x 134 [1,8]
+  CRUSH rule 1 x 135 [5,6]
+  CRUSH rule 1 x 136 [2,1]
+  CRUSH rule 1 x 137 [7,3]
+  CRUSH rule 1 x 138 [8,7]
+  CRUSH rule 1 x 139 [3,0]
+  CRUSH rule 1 x 140 [1,6]
+  CRUSH rule 1 x 141 [6,8]
+  CRUSH rule 1 x 142 [3,5]
+  CRUSH rule 1 x 143 [5,8]
+  CRUSH rule 1 x 144 [8,1]
+  CRUSH rule 1 x 145 [8,5]
+  CRUSH rule 1 x 146 [2,6]
+  CRUSH rule 1 x 147 [2,8]
+  CRUSH rule 1 x 148 [3,4]
+  CRUSH rule 1 x 149 [4,8]
+  CRUSH rule 1 x 150 [1,6]
+  CRUSH rule 1 x 151 [3,4]
+  CRUSH rule 1 x 152 [8,4]
+  CRUSH rule 1 x 153 [8,6]
+  CRUSH rule 1 x 154 [3,2]
+  CRUSH rule 1 x 155 [3,5]
+  CRUSH rule 1 x 156 [4,5]
+  CRUSH rule 1 x 157 [4,1]
+  CRUSH rule 1 x 158 [2,8]
+  CRUSH rule 1 x 159 [7,0]
+  CRUSH rule 1 x 160 [2,8]
+  CRUSH rule 1 x 161 [1,5]
+  CRUSH rule 1 x 162 [0,6]
+  CRUSH rule 1 x 163 [5,6]
+  CRUSH rule 1 x 164 [7,8]
+  CRUSH rule 1 x 165 [7,0]
+  CRUSH rule 1 x 166 [2,4]
+  CRUSH rule 1 x 167 [0,1]
+  CRUSH rule 1 x 168 [4,2]
+  CRUSH rule 1 x 169 [2,6]
+  CRUSH rule 1 x 170 [1,0]
+  CRUSH rule 1 x 171 [7,5]
+  CRUSH rule 1 x 172 [0,7]
+  CRUSH rule 1 x 173 [8,5]
+  CRUSH rule 1 x 174 [1,4]
+  CRUSH rule 1 x 175 [6,0]
+  CRUSH rule 1 x 176 [4,3]
+  CRUSH rule 1 x 177 [5,3]
+  CRUSH rule 1 x 178 [3,0]
+  CRUSH rule 1 x 179 [4,1]
+  CRUSH rule 1 x 180 [3,8]
+  CRUSH rule 1 x 181 [6,2]
+  CRUSH rule 1 x 182 [8,5]
+  CRUSH rule 1 x 183 [7,8]
+  CRUSH rule 1 x 184 [5,7]
+  CRUSH rule 1 x 185 [6,8]
+  CRUSH rule 1 x 186 [2,1]
+  CRUSH rule 1 x 187 [1,6]
+  CRUSH rule 1 x 188 [1,8]
+  CRUSH rule 1 x 189 [0,7]
+  CRUSH rule 1 x 190 [4,0]
+  CRUSH rule 1 x 191 [7,6]
+  CRUSH rule 1 x 192 [5,0]
+  CRUSH rule 1 x 193 [4,2]
+  CRUSH rule 1 x 194 [1,3]
+  CRUSH rule 1 x 195 [6,4]
+  CRUSH rule 1 x 196 [6,7]
+  CRUSH rule 1 x 197 [6,5]
+  CRUSH rule 1 x 198 [2,5]
+  CRUSH rule 1 x 199 [0,5]
+  CRUSH rule 1 x 200 [0,5]
+  CRUSH rule 1 x 201 [7,1]
+  CRUSH rule 1 x 202 [6,3]
+  CRUSH rule 1 x 203 [4,8]
+  CRUSH rule 1 x 204 [2,1]
+  CRUSH rule 1 x 205 [0,7]
+  CRUSH rule 1 x 206 [0,1]
+  CRUSH rule 1 x 207 [3,0]
+  CRUSH rule 1 x 208 [7,1]
+  CRUSH rule 1 x 209 [1,2]
+  CRUSH rule 1 x 210 [1,2]
+  CRUSH rule 1 x 211 [5,4]
+  CRUSH rule 1 x 212 [7,5]
+  CRUSH rule 1 x 213 [8,4]
+  CRUSH rule 1 x 214 [4,5]
+  CRUSH rule 1 x 215 [8,1]
+  CRUSH rule 1 x 216 [5,0]
+  CRUSH rule 1 x 217 [0,1]
+  CRUSH rule 1 x 218 [0,7]
+  CRUSH rule 1 x 219 [4,8]
+  CRUSH rule 1 x 220 [5,7]
+  CRUSH rule 1 x 221 [3,6]
+  CRUSH rule 1 x 222 [6,8]
+  CRUSH rule 1 x 223 [1,3]
+  CRUSH rule 1 x 224 [1,5]
+  CRUSH rule 1 x 225 [8,6]
+  CRUSH rule 1 x 226 [7,2]
+  CRUSH rule 1 x 227 [3,4]
+  CRUSH rule 1 x 228 [5,4]
+  CRUSH rule 1 x 229 [3,4]
+  CRUSH rule 1 x 230 [4,7]
+  CRUSH rule 1 x 231 [4,3]
+  CRUSH rule 1 x 232 [2,7]
+  CRUSH rule 1 x 233 [3,4]
+  CRUSH rule 1 x 234 [0,1]
+  CRUSH rule 1 x 235 [3,8]
+  CRUSH rule 1 x 236 [5,2]
+  CRUSH rule 1 x 237 [4,7]
+  CRUSH rule 1 x 238 [4,3]
+  CRUSH rule 1 x 239 [8,7]
+  CRUSH rule 1 x 240 [5,7]
+  CRUSH rule 1 x 241 [3,1]
+  CRUSH rule 1 x 242 [3,5]
+  CRUSH rule 1 x 243 [4,7]
+  CRUSH rule 1 x 244 [4,6]
+  CRUSH rule 1 x 245 [7,6]
+  CRUSH rule 1 x 246 [1,5]
+  CRUSH rule 1 x 247 [6,0]
+  CRUSH rule 1 x 248 [8,0]
+  CRUSH rule 1 x 249 [2,0]
+  CRUSH rule 1 x 250 [2,1]
+  CRUSH rule 1 x 251 [2,3]
+  CRUSH rule 1 x 252 [3,7]
+  CRUSH rule 1 x 253 [3,2]
+  CRUSH rule 1 x 254 [3,5]
+  CRUSH rule 1 x 255 [1,7]
+  CRUSH rule 1 x 256 [5,7]
+  CRUSH rule 1 x 257 [2,8]
+  CRUSH rule 1 x 258 [5,3]
+  CRUSH rule 1 x 259 [4,3]
+  CRUSH rule 1 x 260 [3,6]
+  CRUSH rule 1 x 261 [8,7]
+  CRUSH rule 1 x 262 [5,4]
+  CRUSH rule 1 x 263 [6,8]
+  CRUSH rule 1 x 264 [3,6]
+  CRUSH rule 1 x 265 [8,6]
+  CRUSH rule 1 x 266 [8,2]
+  CRUSH rule 1 x 267 [2,3]
+  CRUSH rule 1 x 268 [0,7]
+  CRUSH rule 1 x 269 [0,8]
+  CRUSH rule 1 x 270 [5,0]
+  CRUSH rule 1 x 271 [7,5]
+  CRUSH rule 1 x 272 [2,8]
+  CRUSH rule 1 x 273 [3,5]
+  CRUSH rule 1 x 274 [6,8]
+  CRUSH rule 1 x 275 [4,3]
+  CRUSH rule 1 x 276 [7,1]
+  CRUSH rule 1 x 277 [6,4]
+  CRUSH rule 1 x 278 [6,8]
+  CRUSH rule 1 x 279 [8,3]
+  CRUSH rule 1 x 280 [0,6]
+  CRUSH rule 1 x 281 [8,0]
+  CRUSH rule 1 x 282 [3,1]
+  CRUSH rule 1 x 283 [8,2]
+  CRUSH rule 1 x 284 [6,3]
+  CRUSH rule 1 x 285 [5,3]
+  CRUSH rule 1 x 286 [2,1]
+  CRUSH rule 1 x 287 [0,4]
+  CRUSH rule 1 x 288 [8,0]
+  CRUSH rule 1 x 289 [4,6]
+  CRUSH rule 1 x 290 [1,3]
+  CRUSH rule 1 x 291 [0,1]
+  CRUSH rule 1 x 292 [8,0]
+  CRUSH rule 1 x 293 [6,0]
+  CRUSH rule 1 x 294 [7,4]
+  CRUSH rule 1 x 295 [4,8]
+  CRUSH rule 1 x 296 [3,1]
+  CRUSH rule 1 x 297 [6,2]
+  CRUSH rule 1 x 298 [1,2]
+  CRUSH rule 1 x 299 [2,0]
+  CRUSH rule 1 x 300 [8,7]
+  CRUSH rule 1 x 301 [0,8]
+  CRUSH rule 1 x 302 [3,0]
+  CRUSH rule 1 x 303 [7,5]
+  CRUSH rule 1 x 304 [2,7]
+  CRUSH rule 1 x 305 [5,8]
+  CRUSH rule 1 x 306 [0,7]
+  CRUSH rule 1 x 307 [0,2]
+  CRUSH rule 1 x 308 [0,8]
+  CRUSH rule 1 x 309 [7,4]
+  CRUSH rule 1 x 310 [4,3]
+  CRUSH rule 1 x 311 [3,4]
+  CRUSH rule 1 x 312 [2,1]
+  CRUSH rule 1 x 313 [5,3]
+  CRUSH rule 1 x 314 [4,5]
+  CRUSH rule 1 x 315 [2,0]
+  CRUSH rule 1 x 316 [6,3]
+  CRUSH rule 1 x 317 [2,6]
+  CRUSH rule 1 x 318 [8,6]
+  CRUSH rule 1 x 319 [5,0]
+  CRUSH rule 1 x 320 [3,7]
+  CRUSH rule 1 x 321 [1,3]
+  CRUSH rule 1 x 322 [2,7]
+  CRUSH rule 1 x 323 [4,7]
+  CRUSH rule 1 x 324 [7,0]
+  CRUSH rule 1 x 325 [4,6]
+  CRUSH rule 1 x 326 [3,4]
+  CRUSH rule 1 x 327 [0,6]
+  CRUSH rule 1 x 328 [7,4]
+  CRUSH rule 1 x 329 [5,6]
+  CRUSH rule 1 x 330 [3,7]
+  CRUSH rule 1 x 331 [2,6]
+  CRUSH rule 1 x 332 [2,0]
+  CRUSH rule 1 x 333 [6,8]
+  CRUSH rule 1 x 334 [8,3]
+  CRUSH rule 1 x 335 [7,1]
+  CRUSH rule 1 x 336 [4,6]
+  CRUSH rule 1 x 337 [7,2]
+  CRUSH rule 1 x 338 [5,6]
+  CRUSH rule 1 x 339 [7,5]
+  CRUSH rule 1 x 340 [2,0]
+  CRUSH rule 1 x 341 [5,1]
+  CRUSH rule 1 x 342 [0,7]
+  CRUSH rule 1 x 343 [6,7]
+  CRUSH rule 1 x 344 [6,0]
+  CRUSH rule 1 x 345 [4,3]
+  CRUSH rule 1 x 346 [8,0]
+  CRUSH rule 1 x 347 [3,1]
+  CRUSH rule 1 x 348 [8,0]
+  CRUSH rule 1 x 349 [1,6]
+  CRUSH rule 1 x 350 [8,5]
+  CRUSH rule 1 x 351 [3,6]
+  CRUSH rule 1 x 352 [1,0]
+  CRUSH rule 1 x 353 [6,4]
+  CRUSH rule 1 x 354 [0,3]
+  CRUSH rule 1 x 355 [3,4]
+  CRUSH rule 1 x 356 [3,5]
+  CRUSH rule 1 x 357 [6,1]
+  CRUSH rule 1 x 358 [2,1]
+  CRUSH rule 1 x 359 [6,7]
+  CRUSH rule 1 x 360 [5,3]
+  CRUSH rule 1 x 361 [8,4]
+  CRUSH rule 1 x 362 [4,5]
+  CRUSH rule 1 x 363 [4,0]
+  CRUSH rule 1 x 364 [2,5]
+  CRUSH rule 1 x 365 [6,7]
+  CRUSH rule 1 x 366 [7,2]
+  CRUSH rule 1 x 367 [4,5]
+  CRUSH rule 1 x 368 [7,4]
+  CRUSH rule 1 x 369 [3,7]
+  CRUSH rule 1 x 370 [8,7]
+  CRUSH rule 1 x 371 [1,3]
+  CRUSH rule 1 x 372 [3,1]
+  CRUSH rule 1 x 373 [0,6]
+  CRUSH rule 1 x 374 [3,8]
+  CRUSH rule 1 x 375 [6,4]
+  CRUSH rule 1 x 376 [7,1]
+  CRUSH rule 1 x 377 [1,2]
+  CRUSH rule 1 x 378 [0,1]
+  CRUSH rule 1 x 379 [8,5]
+  CRUSH rule 1 x 380 [2,5]
+  CRUSH rule 1 x 381 [0,4]
+  CRUSH rule 1 x 382 [1,5]
+  CRUSH rule 1 x 383 [4,3]
+  CRUSH rule 1 x 384 [7,0]
+  CRUSH rule 1 x 385 [7,4]
+  CRUSH rule 1 x 386 [0,3]
+  CRUSH rule 1 x 387 [1,3]
+  CRUSH rule 1 x 388 [5,0]
+  CRUSH rule 1 x 389 [1,5]
+  CRUSH rule 1 x 390 [5,6]
+  CRUSH rule 1 x 391 [5,6]
+  CRUSH rule 1 x 392 [1,8]
+  CRUSH rule 1 x 393 [4,2]
+  CRUSH rule 1 x 394 [4,7]
+  CRUSH rule 1 x 395 [4,0]
+  CRUSH rule 1 x 396 [4,2]
+  CRUSH rule 1 x 397 [2,1]
+  CRUSH rule 1 x 398 [2,4]
+  CRUSH rule 1 x 399 [8,7]
+  CRUSH rule 1 x 400 [8,1]
+  CRUSH rule 1 x 401 [0,1]
+  CRUSH rule 1 x 402 [7,8]
+  CRUSH rule 1 x 403 [0,1]
+  CRUSH rule 1 x 404 [4,3]
+  CRUSH rule 1 x 405 [6,5]
+  CRUSH rule 1 x 406 [2,0]
+  CRUSH rule 1 x 407 [2,8]
+  CRUSH rule 1 x 408 [4,1]
+  CRUSH rule 1 x 409 [7,3]
+  CRUSH rule 1 x 410 [8,6]
+  CRUSH rule 1 x 411 [2,0]
+  CRUSH rule 1 x 412 [0,5]
+  CRUSH rule 1 x 413 [5,0]
+  CRUSH rule 1 x 414 [4,1]
+  CRUSH rule 1 x 415 [0,6]
+  CRUSH rule 1 x 416 [2,1]
+  CRUSH rule 1 x 417 [8,7]
+  CRUSH rule 1 x 418 [7,6]
+  CRUSH rule 1 x 419 [8,3]
+  CRUSH rule 1 x 420 [1,4]
+  CRUSH rule 1 x 421 [8,6]
+  CRUSH rule 1 x 422 [6,7]
+  CRUSH rule 1 x 423 [0,5]
+  CRUSH rule 1 x 424 [8,4]
+  CRUSH rule 1 x 425 [1,3]
+  CRUSH rule 1 x 426 [6,7]
+  CRUSH rule 1 x 427 [0,7]
+  CRUSH rule 1 x 428 [5,4]
+  CRUSH rule 1 x 429 [4,6]
+  CRUSH rule 1 x 430 [3,6]
+  CRUSH rule 1 x 431 [5,3]
+  CRUSH rule 1 x 432 [7,1]
+  CRUSH rule 1 x 433 [6,5]
+  CRUSH rule 1 x 434 [5,2]
+  CRUSH rule 1 x 435 [0,5]
+  CRUSH rule 1 x 436 [4,0]
+  CRUSH rule 1 x 437 [7,5]
+  CRUSH rule 1 x 438 [0,3]
+  CRUSH rule 1 x 439 [1,3]
+  CRUSH rule 1 x 440 [2,7]
+  CRUSH rule 1 x 441 [5,7]
+  CRUSH rule 1 x 442 [2,4]
+  CRUSH rule 1 x 443 [6,8]
+  CRUSH rule 1 x 444 [7,0]
+  CRUSH rule 1 x 445 [6,3]
+  CRUSH rule 1 x 446 [4,3]
+  CRUSH rule 1 x 447 [2,1]
+  CRUSH rule 1 x 448 [7,2]
+  CRUSH rule 1 x 449 [7,8]
+  CRUSH rule 1 x 450 [4,5]
+  CRUSH rule 1 x 451 [6,8]
+  CRUSH rule 1 x 452 [8,3]
+  CRUSH rule 1 x 453 [6,8]
+  CRUSH rule 1 x 454 [6,7]
+  CRUSH rule 1 x 455 [2,7]
+  CRUSH rule 1 x 456 [6,8]
+  CRUSH rule 1 x 457 [7,2]
+  CRUSH rule 1 x 458 [2,8]
+  CRUSH rule 1 x 459 [2,0]
+  CRUSH rule 1 x 460 [6,5]
+  CRUSH rule 1 x 461 [6,5]
+  CRUSH rule 1 x 462 [8,1]
+  CRUSH rule 1 x 463 [6,7]
+  CRUSH rule 1 x 464 [7,4]
+  CRUSH rule 1 x 465 [7,6]
+  CRUSH rule 1 x 466 [5,8]
+  CRUSH rule 1 x 467 [6,4]
+  CRUSH rule 1 x 468 [7,8]
+  CRUSH rule 1 x 469 [7,0]
+  CRUSH rule 1 x 470 [3,0]
+  CRUSH rule 1 x 471 [0,1]
+  CRUSH rule 1 x 472 [5,4]
+  CRUSH rule 1 x 473 [1,0]
+  CRUSH rule 1 x 474 [6,0]
+  CRUSH rule 1 x 475 [6,7]
+  CRUSH rule 1 x 476 [4,3]
+  CRUSH rule 1 x 477 [5,8]
+  CRUSH rule 1 x 478 [6,7]
+  CRUSH rule 1 x 479 [0,5]
+  CRUSH rule 1 x 480 [1,8]
+  CRUSH rule 1 x 481 [2,4]
+  CRUSH rule 1 x 482 [4,3]
+  CRUSH rule 1 x 483 [0,2]
+  CRUSH rule 1 x 484 [1,2]
+  CRUSH rule 1 x 485 [4,7]
+  CRUSH rule 1 x 486 [4,1]
+  CRUSH rule 1 x 487 [5,0]
+  CRUSH rule 1 x 488 [5,7]
+  CRUSH rule 1 x 489 [2,8]
+  CRUSH rule 1 x 490 [6,4]
+  CRUSH rule 1 x 491 [1,0]
+  CRUSH rule 1 x 492 [6,5]
+  CRUSH rule 1 x 493 [0,2]
+  CRUSH rule 1 x 494 [1,0]
+  CRUSH rule 1 x 495 [3,4]
+  CRUSH rule 1 x 496 [7,5]
+  CRUSH rule 1 x 497 [5,7]
+  CRUSH rule 1 x 498 [0,5]
+  CRUSH rule 1 x 499 [8,4]
+  CRUSH rule 1 x 500 [3,6]
+  CRUSH rule 1 x 501 [0,7]
+  CRUSH rule 1 x 502 [7,1]
+  CRUSH rule 1 x 503 [2,3]
+  CRUSH rule 1 x 504 [5,6]
+  CRUSH rule 1 x 505 [0,7]
+  CRUSH rule 1 x 506 [5,3]
+  CRUSH rule 1 x 507 [6,0]
+  CRUSH rule 1 x 508 [0,1]
+  CRUSH rule 1 x 509 [7,5]
+  CRUSH rule 1 x 510 [6,0]
+  CRUSH rule 1 x 511 [5,8]
+  CRUSH rule 1 x 512 [7,6]
+  CRUSH rule 1 x 513 [7,2]
+  CRUSH rule 1 x 514 [4,3]
+  CRUSH rule 1 x 515 [8,5]
+  CRUSH rule 1 x 516 [4,0]
+  CRUSH rule 1 x 517 [7,8]
+  CRUSH rule 1 x 518 [4,6]
+  CRUSH rule 1 x 519 [7,3]
+  CRUSH rule 1 x 520 [2,6]
+  CRUSH rule 1 x 521 [8,7]
+  CRUSH rule 1 x 522 [6,8]
+  CRUSH rule 1 x 523 [4,2]
+  CRUSH rule 1 x 524 [0,4]
+  CRUSH rule 1 x 525 [0,4]
+  CRUSH rule 1 x 526 [1,5]
+  CRUSH rule 1 x 527 [0,2]
+  CRUSH rule 1 x 528 [5,3]
+  CRUSH rule 1 x 529 [5,7]
+  CRUSH rule 1 x 530 [6,7]
+  CRUSH rule 1 x 531 [6,1]
+  CRUSH rule 1 x 532 [6,3]
+  CRUSH rule 1 x 533 [5,6]
+  CRUSH rule 1 x 534 [7,3]
+  CRUSH rule 1 x 535 [8,6]
+  CRUSH rule 1 x 536 [6,7]
+  CRUSH rule 1 x 537 [3,7]
+  CRUSH rule 1 x 538 [6,8]
+  CRUSH rule 1 x 539 [8,3]
+  CRUSH rule 1 x 540 [0,6]
+  CRUSH rule 1 x 541 [2,3]
+  CRUSH rule 1 x 542 [3,5]
+  CRUSH rule 1 x 543 [6,0]
+  CRUSH rule 1 x 544 [3,7]
+  CRUSH rule 1 x 545 [5,7]
+  CRUSH rule 1 x 546 [6,1]
+  CRUSH rule 1 x 547 [8,2]
+  CRUSH rule 1 x 548 [5,2]
+  CRUSH rule 1 x 549 [5,8]
+  CRUSH rule 1 x 550 [0,5]
+  CRUSH rule 1 x 551 [7,5]
+  CRUSH rule 1 x 552 [5,4]
+  CRUSH rule 1 x 553 [4,2]
+  CRUSH rule 1 x 554 [0,8]
+  CRUSH rule 1 x 555 [5,0]
+  CRUSH rule 1 x 556 [3,4]
+  CRUSH rule 1 x 557 [7,4]
+  CRUSH rule 1 x 558 [3,1]
+  CRUSH rule 1 x 559 [4,2]
+  CRUSH rule 1 x 560 [8,3]
+  CRUSH rule 1 x 561 [6,3]
+  CRUSH rule 1 x 562 [3,4]
+  CRUSH rule 1 x 563 [2,6]
+  CRUSH rule 1 x 564 [5,1]
+  CRUSH rule 1 x 565 [3,6]
+  CRUSH rule 1 x 566 [4,7]
+  CRUSH rule 1 x 567 [3,6]
+  CRUSH rule 1 x 568 [7,4]
+  CRUSH rule 1 x 569 [3,4]
+  CRUSH rule 1 x 570 [1,5]
+  CRUSH rule 1 x 571 [3,7]
+  CRUSH rule 1 x 572 [3,4]
+  CRUSH rule 1 x 573 [3,0]
+  CRUSH rule 1 x 574 [2,0]
+  CRUSH rule 1 x 575 [8,6]
+  CRUSH rule 1 x 576 [4,6]
+  CRUSH rule 1 x 577 [8,2]
+  CRUSH rule 1 x 578 [6,8]
+  CRUSH rule 1 x 579 [3,1]
+  CRUSH rule 1 x 580 [3,0]
+  CRUSH rule 1 x 581 [7,2]
+  CRUSH rule 1 x 582 [2,8]
+  CRUSH rule 1 x 583 [6,0]
+  CRUSH rule 1 x 584 [8,1]
+  CRUSH rule 1 x 585 [7,0]
+  CRUSH rule 1 x 586 [0,1]
+  CRUSH rule 1 x 587 [2,5]
+  CRUSH rule 1 x 588 [3,4]
+  CRUSH rule 1 x 589 [7,1]
+  CRUSH rule 1 x 590 [6,2]
+  CRUSH rule 1 x 591 [5,2]
+  CRUSH rule 1 x 592 [2,0]
+  CRUSH rule 1 x 593 [0,8]
+  CRUSH rule 1 x 594 [0,7]
+  CRUSH rule 1 x 595 [7,1]
+  CRUSH rule 1 x 596 [4,3]
+  CRUSH rule 1 x 597 [3,1]
+  CRUSH rule 1 x 598 [3,2]
+  CRUSH rule 1 x 599 [5,2]
+  CRUSH rule 1 x 600 [7,0]
+  CRUSH rule 1 x 601 [0,7]
+  CRUSH rule 1 x 602 [3,7]
+  CRUSH rule 1 x 603 [5,1]
+  CRUSH rule 1 x 604 [7,5]
+  CRUSH rule 1 x 605 [3,0]
+  CRUSH rule 1 x 606 [2,0]
+  CRUSH rule 1 x 607 [0,4]
+  CRUSH rule 1 x 608 [5,3]
+  CRUSH rule 1 x 609 [5,2]
+  CRUSH rule 1 x 610 [3,7]
+  CRUSH rule 1 x 611 [1,0]
+  CRUSH rule 1 x 612 [2,0]
+  CRUSH rule 1 x 613 [7,2]
+  CRUSH rule 1 x 614 [7,8]
+  CRUSH rule 1 x 615 [6,8]
+  CRUSH rule 1 x 616 [0,8]
+  CRUSH rule 1 x 617 [6,1]
+  CRUSH rule 1 x 618 [7,6]
+  CRUSH rule 1 x 619 [5,1]
+  CRUSH rule 1 x 620 [4,1]
+  CRUSH rule 1 x 621 [5,8]
+  CRUSH rule 1 x 622 [0,4]
+  CRUSH rule 1 x 623 [0,6]
+  CRUSH rule 1 x 624 [3,5]
+  CRUSH rule 1 x 625 [2,3]
+  CRUSH rule 1 x 626 [7,8]
+  CRUSH rule 1 x 627 [2,7]
+  CRUSH rule 1 x 628 [8,0]
+  CRUSH rule 1 x 629 [2,6]
+  CRUSH rule 1 x 630 [2,7]
+  CRUSH rule 1 x 631 [0,7]
+  CRUSH rule 1 x 632 [7,0]
+  CRUSH rule 1 x 633 [8,6]
+  CRUSH rule 1 x 634 [0,4]
+  CRUSH rule 1 x 635 [5,6]
+  CRUSH rule 1 x 636 [1,4]
+  CRUSH rule 1 x 637 [4,1]
+  CRUSH rule 1 x 638 [6,8]
+  CRUSH rule 1 x 639 [4,2]
+  CRUSH rule 1 x 640 [3,1]
+  CRUSH rule 1 x 641 [7,2]
+  CRUSH rule 1 x 642 [2,0]
+  CRUSH rule 1 x 643 [3,5]
+  CRUSH rule 1 x 644 [8,1]
+  CRUSH rule 1 x 645 [5,4]
+  CRUSH rule 1 x 646 [8,0]
+  CRUSH rule 1 x 647 [7,1]
+  CRUSH rule 1 x 648 [0,6]
+  CRUSH rule 1 x 649 [4,7]
+  CRUSH rule 1 x 650 [7,8]
+  CRUSH rule 1 x 651 [3,7]
+  CRUSH rule 1 x 652 [3,4]
+  CRUSH rule 1 x 653 [8,5]
+  CRUSH rule 1 x 654 [7,5]
+  CRUSH rule 1 x 655 [0,3]
+  CRUSH rule 1 x 656 [4,3]
+  CRUSH rule 1 x 657 [6,1]
+  CRUSH rule 1 x 658 [5,4]
+  CRUSH rule 1 x 659 [4,6]
+  CRUSH rule 1 x 660 [7,8]
+  CRUSH rule 1 x 661 [1,8]
+  CRUSH rule 1 x 662 [4,5]
+  CRUSH rule 1 x 663 [1,3]
+  CRUSH rule 1 x 664 [1,4]
+  CRUSH rule 1 x 665 [5,7]
+  CRUSH rule 1 x 666 [2,8]
+  CRUSH rule 1 x 667 [1,3]
+  CRUSH rule 1 x 668 [3,7]
+  CRUSH rule 1 x 669 [6,4]
+  CRUSH rule 1 x 670 [4,0]
+  CRUSH rule 1 x 671 [0,2]
+  CRUSH rule 1 x 672 [4,3]
+  CRUSH rule 1 x 673 [5,3]
+  CRUSH rule 1 x 674 [3,1]
+  CRUSH rule 1 x 675 [0,8]
+  CRUSH rule 1 x 676 [0,2]
+  CRUSH rule 1 x 677 [4,1]
+  CRUSH rule 1 x 678 [2,3]
+  CRUSH rule 1 x 679 [6,0]
+  CRUSH rule 1 x 680 [0,4]
+  CRUSH rule 1 x 681 [4,7]
+  CRUSH rule 1 x 682 [0,5]
+  CRUSH rule 1 x 683 [0,1]
+  CRUSH rule 1 x 684 [7,1]
+  CRUSH rule 1 x 685 [7,1]
+  CRUSH rule 1 x 686 [1,4]
+  CRUSH rule 1 x 687 [3,5]
+  CRUSH rule 1 x 688 [5,7]
+  CRUSH rule 1 x 689 [6,5]
+  CRUSH rule 1 x 690 [8,1]
+  CRUSH rule 1 x 691 [3,0]
+  CRUSH rule 1 x 692 [7,2]
+  CRUSH rule 1 x 693 [6,7]
+  CRUSH rule 1 x 694 [6,5]
+  CRUSH rule 1 x 695 [0,8]
+  CRUSH rule 1 x 696 [1,4]
+  CRUSH rule 1 x 697 [6,1]
+  CRUSH rule 1 x 698 [6,2]
+  CRUSH rule 1 x 699 [1,6]
+  CRUSH rule 1 x 700 [0,3]
+  CRUSH rule 1 x 701 [4,3]
+  CRUSH rule 1 x 702 [3,5]
+  CRUSH rule 1 x 703 [8,3]
+  CRUSH rule 1 x 704 [0,3]
+  CRUSH rule 1 x 705 [8,6]
+  CRUSH rule 1 x 706 [1,2]
+  CRUSH rule 1 x 707 [7,8]
+  CRUSH rule 1 x 708 [3,5]
+  CRUSH rule 1 x 709 [6,3]
+  CRUSH rule 1 x 710 [8,4]
+  CRUSH rule 1 x 711 [2,3]
+  CRUSH rule 1 x 712 [2,3]
+  CRUSH rule 1 x 713 [6,7]
+  CRUSH rule 1 x 714 [3,2]
+  CRUSH rule 1 x 715 [1,2]
+  CRUSH rule 1 x 716 [3,6]
+  CRUSH rule 1 x 717 [8,7]
+  CRUSH rule 1 x 718 [3,7]
+  CRUSH rule 1 x 719 [2,6]
+  CRUSH rule 1 x 720 [6,8]
+  CRUSH rule 1 x 721 [5,4]
+  CRUSH rule 1 x 722 [5,4]
+  CRUSH rule 1 x 723 [5,1]
+  CRUSH rule 1 x 724 [0,6]
+  CRUSH rule 1 x 725 [0,1]
+  CRUSH rule 1 x 726 [3,8]
+  CRUSH rule 1 x 727 [4,6]
+  CRUSH rule 1 x 728 [2,1]
+  CRUSH rule 1 x 729 [5,3]
+  CRUSH rule 1 x 730 [3,7]
+  CRUSH rule 1 x 731 [4,1]
+  CRUSH rule 1 x 732 [1,5]
+  CRUSH rule 1 x 733 [5,4]
+  CRUSH rule 1 x 734 [6,4]
+  CRUSH rule 1 x 735 [4,8]
+  CRUSH rule 1 x 736 [3,5]
+  CRUSH rule 1 x 737 [1,0]
+  CRUSH rule 1 x 738 [5,2]
+  CRUSH rule 1 x 739 [0,1]
+  CRUSH rule 1 x 740 [0,1]
+  CRUSH rule 1 x 741 [7,8]
+  CRUSH rule 1 x 742 [8,2]
+  CRUSH rule 1 x 743 [7,0]
+  CRUSH rule 1 x 744 [4,7]
+  CRUSH rule 1 x 745 [3,4]
+  CRUSH rule 1 x 746 [4,1]
+  CRUSH rule 1 x 747 [6,0]
+  CRUSH rule 1 x 748 [2,7]
+  CRUSH rule 1 x 749 [4,5]
+  CRUSH rule 1 x 750 [1,6]
+  CRUSH rule 1 x 751 [2,1]
+  CRUSH rule 1 x 752 [8,1]
+  CRUSH rule 1 x 753 [7,8]
+  CRUSH rule 1 x 754 [8,6]
+  CRUSH rule 1 x 755 [1,2]
+  CRUSH rule 1 x 756 [5,6]
+  CRUSH rule 1 x 757 [8,6]
+  CRUSH rule 1 x 758 [6,0]
+  CRUSH rule 1 x 759 [8,5]
+  CRUSH rule 1 x 760 [1,5]
+  CRUSH rule 1 x 761 [4,1]
+  CRUSH rule 1 x 762 [2,7]
+  CRUSH rule 1 x 763 [8,6]
+  CRUSH rule 1 x 764 [1,7]
+  CRUSH rule 1 x 765 [6,5]
+  CRUSH rule 1 x 766 [8,5]
+  CRUSH rule 1 x 767 [1,2]
+  CRUSH rule 1 x 768 [8,3]
+  CRUSH rule 1 x 769 [6,2]
+  CRUSH rule 1 x 770 [6,0]
+  CRUSH rule 1 x 771 [7,0]
+  CRUSH rule 1 x 772 [8,3]
+  CRUSH rule 1 x 773 [3,1]
+  CRUSH rule 1 x 774 [4,6]
+  CRUSH rule 1 x 775 [6,8]
+  CRUSH rule 1 x 776 [7,2]
+  CRUSH rule 1 x 777 [3,1]
+  CRUSH rule 1 x 778 [1,8]
+  CRUSH rule 1 x 779 [2,7]
+  CRUSH rule 1 x 780 [0,2]
+  CRUSH rule 1 x 781 [6,3]
+  CRUSH rule 1 x 782 [5,4]
+  CRUSH rule 1 x 783 [7,1]
+  CRUSH rule 1 x 784 [0,1]
+  CRUSH rule 1 x 785 [6,1]
+  CRUSH rule 1 x 786 [7,6]
+  CRUSH rule 1 x 787 [1,0]
+  CRUSH rule 1 x 788 [6,0]
+  CRUSH rule 1 x 789 [0,4]
+  CRUSH rule 1 x 790 [8,4]
+  CRUSH rule 1 x 791 [3,8]
+  CRUSH rule 1 x 792 [5,8]
+  CRUSH rule 1 x 793 [6,1]
+  CRUSH rule 1 x 794 [2,6]
+  CRUSH rule 1 x 795 [0,3]
+  CRUSH rule 1 x 796 [3,4]
+  CRUSH rule 1 x 797 [2,3]
+  CRUSH rule 1 x 798 [6,8]
+  CRUSH rule 1 x 799 [5,1]
+  CRUSH rule 1 x 800 [5,0]
+  CRUSH rule 1 x 801 [3,6]
+  CRUSH rule 1 x 802 [1,8]
+  CRUSH rule 1 x 803 [0,5]
+  CRUSH rule 1 x 804 [6,2]
+  CRUSH rule 1 x 805 [3,6]
+  CRUSH rule 1 x 806 [1,3]
+  CRUSH rule 1 x 807 [5,4]
+  CRUSH rule 1 x 808 [4,6]
+  CRUSH rule 1 x 809 [1,4]
+  CRUSH rule 1 x 810 [5,7]
+  CRUSH rule 1 x 811 [8,4]
+  CRUSH rule 1 x 812 [8,5]
+  CRUSH rule 1 x 813 [6,4]
+  CRUSH rule 1 x 814 [3,6]
+  CRUSH rule 1 x 815 [3,1]
+  CRUSH rule 1 x 816 [2,1]
+  CRUSH rule 1 x 817 [4,3]
+  CRUSH rule 1 x 818 [3,5]
+  CRUSH rule 1 x 819 [5,1]
+  CRUSH rule 1 x 820 [3,5]
+  CRUSH rule 1 x 821 [4,5]
+  CRUSH rule 1 x 822 [2,0]
+  CRUSH rule 1 x 823 [4,8]
+  CRUSH rule 1 x 824 [3,7]
+  CRUSH rule 1 x 825 [2,8]
+  CRUSH rule 1 x 826 [7,0]
+  CRUSH rule 1 x 827 [0,8]
+  CRUSH rule 1 x 828 [2,3]
+  CRUSH rule 1 x 829 [5,6]
+  CRUSH rule 1 x 830 [2,3]
+  CRUSH rule 1 x 831 [1,6]
+  CRUSH rule 1 x 832 [4,5]
+  CRUSH rule 1 x 833 [2,1]
+  CRUSH rule 1 x 834 [3,4]
+  CRUSH rule 1 x 835 [8,4]
+  CRUSH rule 1 x 836 [3,4]
+  CRUSH rule 1 x 837 [6,3]
+  CRUSH rule 1 x 838 [6,7]
+  CRUSH rule 1 x 839 [5,0]
+  CRUSH rule 1 x 840 [7,8]
+  CRUSH rule 1 x 841 [4,8]
+  CRUSH rule 1 x 842 [2,4]
+  CRUSH rule 1 x 843 [6,4]
+  CRUSH rule 1 x 844 [4,8]
+  CRUSH rule 1 x 845 [3,8]
+  CRUSH rule 1 x 846 [3,2]
+  CRUSH rule 1 x 847 [0,2]
+  CRUSH rule 1 x 848 [2,6]
+  CRUSH rule 1 x 849 [4,5]
+  CRUSH rule 1 x 850 [1,0]
+  CRUSH rule 1 x 851 [6,8]
+  CRUSH rule 1 x 852 [7,3]
+  CRUSH rule 1 x 853 [6,8]
+  CRUSH rule 1 x 854 [7,6]
+  CRUSH rule 1 x 855 [5,7]
+  CRUSH rule 1 x 856 [6,7]
+  CRUSH rule 1 x 857 [8,5]
+  CRUSH rule 1 x 858 [6,4]
+  CRUSH rule 1 x 859 [6,0]
+  CRUSH rule 1 x 860 [4,1]
+  CRUSH rule 1 x 861 [8,7]
+  CRUSH rule 1 x 862 [6,1]
+  CRUSH rule 1 x 863 [8,7]
+  CRUSH rule 1 x 864 [5,6]
+  CRUSH rule 1 x 865 [8,1]
+  CRUSH rule 1 x 866 [3,4]
+  CRUSH rule 1 x 867 [6,5]
+  CRUSH rule 1 x 868 [6,3]
+  CRUSH rule 1 x 869 [8,7]
+  CRUSH rule 1 x 870 [0,4]
+  CRUSH rule 1 x 871 [3,4]
+  CRUSH rule 1 x 872 [5,1]
+  CRUSH rule 1 x 873 [4,6]
+  CRUSH rule 1 x 874 [2,6]
+  CRUSH rule 1 x 875 [2,6]
+  CRUSH rule 1 x 876 [5,8]
+  CRUSH rule 1 x 877 [6,4]
+  CRUSH rule 1 x 878 [5,4]
+  CRUSH rule 1 x 879 [7,4]
+  CRUSH rule 1 x 880 [3,5]
+  CRUSH rule 1 x 881 [5,6]
+  CRUSH rule 1 x 882 [4,0]
+  CRUSH rule 1 x 883 [2,1]
+  CRUSH rule 1 x 884 [6,0]
+  CRUSH rule 1 x 885 [5,1]
+  CRUSH rule 1 x 886 [3,6]
+  CRUSH rule 1 x 887 [7,4]
+  CRUSH rule 1 x 888 [6,8]
+  CRUSH rule 1 x 889 [2,1]
+  CRUSH rule 1 x 890 [7,2]
+  CRUSH rule 1 x 891 [1,8]
+  CRUSH rule 1 x 892 [6,2]
+  CRUSH rule 1 x 893 [2,3]
+  CRUSH rule 1 x 894 [7,5]
+  CRUSH rule 1 x 895 [5,3]
+  CRUSH rule 1 x 896 [1,8]
+  CRUSH rule 1 x 897 [4,2]
+  CRUSH rule 1 x 898 [0,5]
+  CRUSH rule 1 x 899 [1,7]
+  CRUSH rule 1 x 900 [4,1]
+  CRUSH rule 1 x 901 [5,0]
+  CRUSH rule 1 x 902 [8,5]
+  CRUSH rule 1 x 903 [5,7]
+  CRUSH rule 1 x 904 [5,6]
+  CRUSH rule 1 x 905 [6,2]
+  CRUSH rule 1 x 906 [1,2]
+  CRUSH rule 1 x 907 [7,1]
+  CRUSH rule 1 x 908 [5,8]
+  CRUSH rule 1 x 909 [2,3]
+  CRUSH rule 1 x 910 [6,4]
+  CRUSH rule 1 x 911 [5,8]
+  CRUSH rule 1 x 912 [0,1]
+  CRUSH rule 1 x 913 [7,6]
+  CRUSH rule 1 x 914 [6,4]
+  CRUSH rule 1 x 915 [8,2]
+  CRUSH rule 1 x 916 [3,1]
+  CRUSH rule 1 x 917 [1,5]
+  CRUSH rule 1 x 918 [8,2]
+  CRUSH rule 1 x 919 [6,2]
+  CRUSH rule 1 x 920 [7,6]
+  CRUSH rule 1 x 921 [1,4]
+  CRUSH rule 1 x 922 [6,7]
+  CRUSH rule 1 x 923 [5,3]
+  CRUSH rule 1 x 924 [3,5]
+  CRUSH rule 1 x 925 [5,7]
+  CRUSH rule 1 x 926 [3,4]
+  CRUSH rule 1 x 927 [1,6]
+  CRUSH rule 1 x 928 [8,1]
+  CRUSH rule 1 x 929 [4,5]
+  CRUSH rule 1 x 930 [2,4]
+  CRUSH rule 1 x 931 [5,0]
+  CRUSH rule 1 x 932 [4,3]
+  CRUSH rule 1 x 933 [8,5]
+  CRUSH rule 1 x 934 [5,3]
+  CRUSH rule 1 x 935 [6,3]
+  CRUSH rule 1 x 936 [0,6]
+  CRUSH rule 1 x 937 [5,4]
+  CRUSH rule 1 x 938 [6,5]
+  CRUSH rule 1 x 939 [2,7]
+  CRUSH rule 1 x 940 [8,7]
+  CRUSH rule 1 x 941 [5,2]
+  CRUSH rule 1 x 942 [1,0]
+  CRUSH rule 1 x 943 [8,2]
+  CRUSH rule 1 x 944 [4,3]
+  CRUSH rule 1 x 945 [7,2]
+  CRUSH rule 1 x 946 [2,0]
+  CRUSH rule 1 x 947 [4,5]
+  CRUSH rule 1 x 948 [7,8]
+  CRUSH rule 1 x 949 [6,1]
+  CRUSH rule 1 x 950 [3,5]
+  CRUSH rule 1 x 951 [4,5]
+  CRUSH rule 1 x 952 [2,0]
+  CRUSH rule 1 x 953 [1,3]
+  CRUSH rule 1 x 954 [4,2]
+  CRUSH rule 1 x 955 [8,6]
+  CRUSH rule 1 x 956 [1,0]
+  CRUSH rule 1 x 957 [7,6]
+  CRUSH rule 1 x 958 [8,7]
+  CRUSH rule 1 x 959 [5,2]
+  CRUSH rule 1 x 960 [3,6]
+  CRUSH rule 1 x 961 [4,0]
+  CRUSH rule 1 x 962 [7,4]
+  CRUSH rule 1 x 963 [0,5]
+  CRUSH rule 1 x 964 [3,1]
+  CRUSH rule 1 x 965 [7,6]
+  CRUSH rule 1 x 966 [3,8]
+  CRUSH rule 1 x 967 [8,6]
+  CRUSH rule 1 x 968 [7,2]
+  CRUSH rule 1 x 969 [8,0]
+  CRUSH rule 1 x 970 [0,6]
+  CRUSH rule 1 x 971 [1,7]
+  CRUSH rule 1 x 972 [1,8]
+  CRUSH rule 1 x 973 [1,2]
+  CRUSH rule 1 x 974 [5,3]
+  CRUSH rule 1 x 975 [3,7]
+  CRUSH rule 1 x 976 [4,3]
+  CRUSH rule 1 x 977 [8,3]
+  CRUSH rule 1 x 978 [7,2]
+  CRUSH rule 1 x 979 [7,6]
+  CRUSH rule 1 x 980 [6,0]
+  CRUSH rule 1 x 981 [7,3]
+  CRUSH rule 1 x 982 [4,2]
+  CRUSH rule 1 x 983 [3,5]
+  CRUSH rule 1 x 984 [0,2]
+  CRUSH rule 1 x 985 [2,5]
+  CRUSH rule 1 x 986 [8,7]
+  CRUSH rule 1 x 987 [0,5]
+  CRUSH rule 1 x 988 [1,3]
+  CRUSH rule 1 x 989 [0,6]
+  CRUSH rule 1 x 990 [1,0]
+  CRUSH rule 1 x 991 [0,4]
+  CRUSH rule 1 x 992 [7,1]
+  CRUSH rule 1 x 993 [0,6]
+  CRUSH rule 1 x 994 [3,4]
+  CRUSH rule 1 x 995 [7,6]
+  CRUSH rule 1 x 996 [6,7]
+  CRUSH rule 1 x 997 [6,4]
+  CRUSH rule 1 x 998 [8,1]
+  CRUSH rule 1 x 999 [0,7]
+  CRUSH rule 1 x 1000 [8,5]
+  CRUSH rule 1 x 1001 [2,0]
+  CRUSH rule 1 x 1002 [1,3]
+  CRUSH rule 1 x 1003 [2,8]
+  CRUSH rule 1 x 1004 [6,1]
+  CRUSH rule 1 x 1005 [6,1]
+  CRUSH rule 1 x 1006 [1,0]
+  CRUSH rule 1 x 1007 [1,2]
+  CRUSH rule 1 x 1008 [1,7]
+  CRUSH rule 1 x 1009 [6,8]
+  CRUSH rule 1 x 1010 [3,4]
+  CRUSH rule 1 x 1011 [3,0]
+  CRUSH rule 1 x 1012 [3,0]
+  CRUSH rule 1 x 1013 [5,1]
+  CRUSH rule 1 x 1014 [2,8]
+  CRUSH rule 1 x 1015 [6,8]
+  CRUSH rule 1 x 1016 [2,0]
+  CRUSH rule 1 x 1017 [6,0]
+  CRUSH rule 1 x 1018 [5,4]
+  CRUSH rule 1 x 1019 [5,3]
+  CRUSH rule 1 x 1020 [5,1]
+  CRUSH rule 1 x 1021 [5,2]
+  CRUSH rule 1 x 1022 [1,6]
+  CRUSH rule 1 x 1023 [3,2]
+  rule 1 (choose-two) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [0,2,3]
+  CRUSH rule 1 x 1 [0,8,2]
+  CRUSH rule 1 x 2 [1,3,5]
+  CRUSH rule 1 x 3 [8,0,6]
+  CRUSH rule 1 x 4 [5,4,0]
+  CRUSH rule 1 x 5 [7,8,0]
+  CRUSH rule 1 x 6 [2,6,7]
+  CRUSH rule 1 x 7 [5,4,3]
+  CRUSH rule 1 x 8 [5,3,8]
+  CRUSH rule 1 x 9 [2,3,1]
+  CRUSH rule 1 x 10 [0,2,8]
+  CRUSH rule 1 x 11 [0,7,2]
+  CRUSH rule 1 x 12 [0,2,4]
+  CRUSH rule 1 x 13 [3,8,5]
+  CRUSH rule 1 x 14 [7,6,8]
+  CRUSH rule 1 x 15 [7,2,4]
+  CRUSH rule 1 x 16 [3,6,0]
+  CRUSH rule 1 x 17 [5,4,0]
+  CRUSH rule 1 x 18 [1,4,5]
+  CRUSH rule 1 x 19 [7,5,4]
+  CRUSH rule 1 x 20 [2,4,7]
+  CRUSH rule 1 x 21 [3,7,2]
+  CRUSH rule 1 x 22 [8,3,7]
+  CRUSH rule 1 x 23 [3,6,8]
+  CRUSH rule 1 x 24 [1,0,6]
+  CRUSH rule 1 x 25 [3,7,0]
+  CRUSH rule 1 x 26 [2,8,7]
+  CRUSH rule 1 x 27 [3,1,5]
+  CRUSH rule 1 x 28 [6,0,3]
+  CRUSH rule 1 x 29 [8,5,7]
+  CRUSH rule 1 x 30 [5,7,3]
+  CRUSH rule 1 x 31 [8,7,6]
+  CRUSH rule 1 x 32 [3,6,8]
+  CRUSH rule 1 x 33 [2,7,8]
+  CRUSH rule 1 x 34 [2,5,4]
+  CRUSH rule 1 x 35 [0,8,1]
+  CRUSH rule 1 x 36 [3,8,2]
+  CRUSH rule 1 x 37 [0,4,5]
+  CRUSH rule 1 x 38 [4,8,7]
+  CRUSH rule 1 x 39 [3,7,0]
+  CRUSH rule 1 x 40 [7,8,1]
+  CRUSH rule 1 x 41 [0,2,6]
+  CRUSH rule 1 x 42 [4,3,6]
+  CRUSH rule 1 x 43 [0,3,7]
+  CRUSH rule 1 x 44 [1,6,7]
+  CRUSH rule 1 x 45 [8,0,4]
+  CRUSH rule 1 x 46 [2,4,3]
+  CRUSH rule 1 x 47 [4,5,3]
+  CRUSH rule 1 x 48 [4,6,5]
+  CRUSH rule 1 x 49 [5,4,8]
+  CRUSH rule 1 x 50 [3,4,1]
+  CRUSH rule 1 x 51 [3,6,0]
+  CRUSH rule 1 x 52 [8,6,1]
+  CRUSH rule 1 x 53 [3,5,4]
+  CRUSH rule 1 x 54 [7,6,8]
+  CRUSH rule 1 x 55 [8,7,0]
+  CRUSH rule 1 x 56 [6,4,8]
+  CRUSH rule 1 x 57 [5,3,6]
+  CRUSH rule 1 x 58 [1,0,2]
+  CRUSH rule 1 x 59 [4,2,1]
+  CRUSH rule 1 x 60 [3,5,0]
+  CRUSH rule 1 x 61 [4,6,8]
+  CRUSH rule 1 x 62 [7,0,1]
+  CRUSH rule 1 x 63 [5,6,8]
+  CRUSH rule 1 x 64 [4,5,1]
+  CRUSH rule 1 x 65 [7,3,6]
+  CRUSH rule 1 x 66 [5,4,3]
+  CRUSH rule 1 x 67 [5,0,8]
+  CRUSH rule 1 x 68 [0,5,8]
+  CRUSH rule 1 x 69 [5,1,6]
+  CRUSH rule 1 x 70 [7,0,2]
+  CRUSH rule 1 x 71 [2,8,6]
+  CRUSH rule 1 x 72 [6,1,5]
+  CRUSH rule 1 x 73 [2,7,3]
+  CRUSH rule 1 x 74 [0,7,1]
+  CRUSH rule 1 x 75 [3,2,0]
+  CRUSH rule 1 x 76 [5,1,7]
+  CRUSH rule 1 x 77 [7,2,6]
+  CRUSH rule 1 x 78 [1,4,2]
+  CRUSH rule 1 x 79 [5,4,0]
+  CRUSH rule 1 x 80 [0,3,5]
+  CRUSH rule 1 x 81 [0,2,5]
+  CRUSH rule 1 x 82 [7,1,6]
+  CRUSH rule 1 x 83 [2,6,8]
+  CRUSH rule 1 x 84 [7,2,1]
+  CRUSH rule 1 x 85 [3,4,8]
+  CRUSH rule 1 x 86 [0,2,1]
+  CRUSH rule 1 x 87 [0,7,4]
+  CRUSH rule 1 x 88 [1,6,0]
+  CRUSH rule 1 x 89 [3,0,2]
+  CRUSH rule 1 x 90 [6,7,5]
+  CRUSH rule 1 x 91 [3,8,7]
+  CRUSH rule 1 x 92 [1,8,4]
+  CRUSH rule 1 x 93 [7,4,5]
+  CRUSH rule 1 x 94 [0,4,3]
+  CRUSH rule 1 x 95 [7,5,1]
+  CRUSH rule 1 x 96 [3,6,4]
+  CRUSH rule 1 x 97 [8,7,4]
+  CRUSH rule 1 x 98 [2,0,7]
+  CRUSH rule 1 x 99 [0,7,2]
+  CRUSH rule 1 x 100 [1,7,4]
+  CRUSH rule 1 x 101 [3,7,2]
+  CRUSH rule 1 x 102 [4,2,7]
+  CRUSH rule 1 x 103 [4,7,6]
+  CRUSH rule 1 x 104 [7,4,3]
+  CRUSH rule 1 x 105 [2,4,1]
+  CRUSH rule 1 x 106 [1,6,2]
+  CRUSH rule 1 x 107 [3,2,1]
+  CRUSH rule 1 x 108 [7,2,8]
+  CRUSH rule 1 x 109 [1,2,5]
+  CRUSH rule 1 x 110 [3,2,7]
+  CRUSH rule 1 x 111 [2,1,0]
+  CRUSH rule 1 x 112 [2,0,6]
+  CRUSH rule 1 x 113 [6,2,1]
+  CRUSH rule 1 x 114 [7,6,3]
+  CRUSH rule 1 x 115 [8,2,3]
+  CRUSH rule 1 x 116 [1,2,6]
+  CRUSH rule 1 x 117 [7,3,6]
+  CRUSH rule 1 x 118 [0,3,8]
+  CRUSH rule 1 x 119 [5,6,1]
+  CRUSH rule 1 x 120 [0,2,5]
+  CRUSH rule 1 x 121 [2,0,1]
+  CRUSH rule 1 x 122 [8,5,0]
+  CRUSH rule 1 x 123 [2,5,1]
+  CRUSH rule 1 x 124 [3,5,1]
+  CRUSH rule 1 x 125 [0,7,3]
+  CRUSH rule 1 x 126 [4,2,6]
+  CRUSH rule 1 x 127 [3,6,7]
+  CRUSH rule 1 x 128 [3,5,8]
+  CRUSH rule 1 x 129 [0,2,1]
+  CRUSH rule 1 x 130 [3,8,5]
+  CRUSH rule 1 x 131 [1,2,0]
+  CRUSH rule 1 x 132 [1,2,3]
+  CRUSH rule 1 x 133 [3,6,7]
+  CRUSH rule 1 x 134 [1,8,2]
+  CRUSH rule 1 x 135 [5,6,4]
+  CRUSH rule 1 x 136 [2,1,5]
+  CRUSH rule 1 x 137 [7,3,4]
+  CRUSH rule 1 x 138 [8,7,6]
+  CRUSH rule 1 x 139 [3,0,5]
+  CRUSH rule 1 x 140 [1,6,7]
+  CRUSH rule 1 x 141 [6,8,1]
+  CRUSH rule 1 x 142 [3,5,0]
+  CRUSH rule 1 x 143 [5,8,7]
+  CRUSH rule 1 x 144 [8,1,0]
+  CRUSH rule 1 x 145 [8,5,6]
+  CRUSH rule 1 x 146 [2,6,4]
+  CRUSH rule 1 x 147 [2,8,7]
+  CRUSH rule 1 x 148 [3,4,0]
+  CRUSH rule 1 x 149 [4,8,6]
+  CRUSH rule 1 x 150 [1,6,5]
+  CRUSH rule 1 x 151 [3,4,5]
+  CRUSH rule 1 x 152 [8,4,6]
+  CRUSH rule 1 x 153 [8,6,7]
+  CRUSH rule 1 x 154 [3,2,0]
+  CRUSH rule 1 x 155 [3,5,8]
+  CRUSH rule 1 x 156 [4,5,3]
+  CRUSH rule 1 x 157 [4,1,2]
+  CRUSH rule 1 x 158 [2,8,1]
+  CRUSH rule 1 x 159 [7,0,5]
+  CRUSH rule 1 x 160 [2,8,1]
+  CRUSH rule 1 x 161 [1,5,2]
+  CRUSH rule 1 x 162 [0,6,2]
+  CRUSH rule 1 x 163 [5,6,1]
+  CRUSH rule 1 x 164 [7,8,1]
+  CRUSH rule 1 x 165 [7,0,5]
+  CRUSH rule 1 x 166 [2,4,1]
+  CRUSH rule 1 x 167 [0,1,6]
+  CRUSH rule 1 x 168 [4,2,3]
+  CRUSH rule 1 x 169 [2,6,1]
+  CRUSH rule 1 x 170 [1,0,5]
+  CRUSH rule 1 x 171 [7,5,4]
+  CRUSH rule 1 x 172 [0,7,8]
+  CRUSH rule 1 x 173 [8,5,3]
+  CRUSH rule 1 x 174 [1,4,7]
+  CRUSH rule 1 x 175 [6,0,4]
+  CRUSH rule 1 x 176 [4,3,5]
+  CRUSH rule 1 x 177 [5,3,0]
+  CRUSH rule 1 x 178 [3,0,2]
+  CRUSH rule 1 x 179 [4,1,2]
+  CRUSH rule 1 x 180 [3,8,7]
+  CRUSH rule 1 x 181 [6,2,0]
+  CRUSH rule 1 x 182 [8,5,3]
+  CRUSH rule 1 x 183 [7,8,6]
+  CRUSH rule 1 x 184 [5,7,3]
+  CRUSH rule 1 x 185 [6,8,0]
+  CRUSH rule 1 x 186 [2,1,4]
+  CRUSH rule 1 x 187 [1,6,7]
+  CRUSH rule 1 x 188 [1,8,7]
+  CRUSH rule 1 x 189 [0,7,8]
+  CRUSH rule 1 x 190 [4,0,1]
+  CRUSH rule 1 x 191 [7,6,8]
+  CRUSH rule 1 x 192 [5,0,1]
+  CRUSH rule 1 x 193 [4,2,6]
+  CRUSH rule 1 x 194 [1,3,4]
+  CRUSH rule 1 x 195 [6,4,3]
+  CRUSH rule 1 x 196 [6,7,2]
+  CRUSH rule 1 x 197 [6,5,2]
+  CRUSH rule 1 x 198 [2,5,6]
+  CRUSH rule 1 x 199 [0,5,7]
+  CRUSH rule 1 x 200 [0,5,1]
+  CRUSH rule 1 x 201 [7,1,0]
+  CRUSH rule 1 x 202 [6,3,4]
+  CRUSH rule 1 x 203 [4,8,7]
+  CRUSH rule 1 x 204 [2,1,5]
+  CRUSH rule 1 x 205 [0,7,6]
+  CRUSH rule 1 x 206 [0,1,7]
+  CRUSH rule 1 x 207 [3,0,5]
+  CRUSH rule 1 x 208 [7,1,0]
+  CRUSH rule 1 x 209 [1,2,0]
+  CRUSH rule 1 x 210 [1,2,3]
+  CRUSH rule 1 x 211 [5,4,2]
+  CRUSH rule 1 x 212 [7,5,0]
+  CRUSH rule 1 x 213 [8,4,0]
+  CRUSH rule 1 x 214 [4,5,7]
+  CRUSH rule 1 x 215 [8,1,7]
+  CRUSH rule 1 x 216 [5,0,2]
+  CRUSH rule 1 x 217 [0,1,7]
+  CRUSH rule 1 x 218 [0,7,8]
+  CRUSH rule 1 x 219 [4,8,2]
+  CRUSH rule 1 x 220 [5,7,3]
+  CRUSH rule 1 x 221 [3,6,5]
+  CRUSH rule 1 x 222 [6,8,3]
+  CRUSH rule 1 x 223 [1,3,6]
+  CRUSH rule 1 x 224 [1,5,0]
+  CRUSH rule 1 x 225 [8,6,2]
+  CRUSH rule 1 x 226 [7,2,3]
+  CRUSH rule 1 x 227 [3,4,0]
+  CRUSH rule 1 x 228 [5,4,8]
+  CRUSH rule 1 x 229 [3,4,7]
+  CRUSH rule 1 x 230 [4,7,2]
+  CRUSH rule 1 x 231 [4,3,5]
+  CRUSH rule 1 x 232 [2,7,4]
+  CRUSH rule 1 x 233 [3,4,5]
+  CRUSH rule 1 x 234 [0,1,2]
+  CRUSH rule 1 x 235 [3,8,0]
+  CRUSH rule 1 x 236 [5,2,7]
+  CRUSH rule 1 x 237 [4,7,6]
+  CRUSH rule 1 x 238 [4,3,5]
+  CRUSH rule 1 x 239 [8,7,6]
+  CRUSH rule 1 x 240 [5,7,4]
+  CRUSH rule 1 x 241 [3,1,5]
+  CRUSH rule 1 x 242 [3,5,4]
+  CRUSH rule 1 x 243 [4,7,3]
+  CRUSH rule 1 x 244 [4,6,0]
+  CRUSH rule 1 x 245 [7,6,1]
+  CRUSH rule 1 x 246 [1,5,2]
+  CRUSH rule 1 x 247 [6,0,2]
+  CRUSH rule 1 x 248 [8,0,7]
+  CRUSH rule 1 x 249 [2,0,1]
+  CRUSH rule 1 x 250 [2,1,0]
+  CRUSH rule 1 x 251 [2,3,6]
+  CRUSH rule 1 x 252 [3,7,8]
+  CRUSH rule 1 x 253 [3,2,4]
+  CRUSH rule 1 x 254 [3,5,4]
+  CRUSH rule 1 x 255 [1,7,8]
+  CRUSH rule 1 x 256 [5,7,0]
+  CRUSH rule 1 x 257 [2,8,4]
+  CRUSH rule 1 x 258 [5,3,0]
+  CRUSH rule 1 x 259 [4,3,6]
+  CRUSH rule 1 x 260 [3,6,2]
+  CRUSH rule 1 x 261 [8,7,4]
+  CRUSH rule 1 x 262 [5,4,8]
+  CRUSH rule 1 x 263 [6,8,2]
+  CRUSH rule 1 x 264 [3,6,5]
+  CRUSH rule 1 x 265 [8,6,7]
+  CRUSH rule 1 x 266 [8,2,1]
+  CRUSH rule 1 x 267 [2,3,0]
+  CRUSH rule 1 x 268 [0,7,2]
+  CRUSH rule 1 x 269 [0,8,2]
+  CRUSH rule 1 x 270 [5,0,1]
+  CRUSH rule 1 x 271 [7,5,4]
+  CRUSH rule 1 x 272 [2,8,3]
+  CRUSH rule 1 x 273 [3,5,0]
+  CRUSH rule 1 x 274 [6,8,3]
+  CRUSH rule 1 x 275 [4,3,6]
+  CRUSH rule 1 x 276 [7,1,2]
+  CRUSH rule 1 x 277 [6,4,0]
+  CRUSH rule 1 x 278 [6,8,0]
+  CRUSH rule 1 x 279 [8,3,6]
+  CRUSH rule 1 x 280 [0,6,2]
+  CRUSH rule 1 x 281 [8,0,2]
+  CRUSH rule 1 x 282 [3,1,6]
+  CRUSH rule 1 x 283 [8,2,0]
+  CRUSH rule 1 x 284 [6,3,0]
+  CRUSH rule 1 x 285 [5,3,7]
+  CRUSH rule 1 x 286 [2,1,6]
+  CRUSH rule 1 x 287 [0,4,1]
+  CRUSH rule 1 x 288 [8,0,7]
+  CRUSH rule 1 x 289 [4,6,2]
+  CRUSH rule 1 x 290 [1,3,7]
+  CRUSH rule 1 x 291 [0,1,4]
+  CRUSH rule 1 x 292 [8,0,2]
+  CRUSH rule 1 x 293 [6,0,8]
+  CRUSH rule 1 x 294 [7,4,5]
+  CRUSH rule 1 x 295 [4,8,7]
+  CRUSH rule 1 x 296 [3,1,5]
+  CRUSH rule 1 x 297 [6,2,8]
+  CRUSH rule 1 x 298 [1,2,3]
+  CRUSH rule 1 x 299 [2,0,7]
+  CRUSH rule 1 x 300 [8,7,3]
+  CRUSH rule 1 x 301 [0,8,1]
+  CRUSH rule 1 x 302 [3,0,6]
+  CRUSH rule 1 x 303 [7,5,8]
+  CRUSH rule 1 x 304 [2,7,5]
+  CRUSH rule 1 x 305 [5,8,2]
+  CRUSH rule 1 x 306 [0,7,1]
+  CRUSH rule 1 x 307 [0,2,8]
+  CRUSH rule 1 x 308 [0,8,4]
+  CRUSH rule 1 x 309 [7,4,5]
+  CRUSH rule 1 x 310 [4,3,1]
+  CRUSH rule 1 x 311 [3,4,5]
+  CRUSH rule 1 x 312 [2,1,0]
+  CRUSH rule 1 x 313 [5,3,4]
+  CRUSH rule 1 x 314 [4,5,0]
+  CRUSH rule 1 x 315 [2,0,1]
+  CRUSH rule 1 x 316 [6,3,8]
+  CRUSH rule 1 x 317 [2,6,0]
+  CRUSH rule 1 x 318 [8,6,7]
+  CRUSH rule 1 x 319 [5,0,8]
+  CRUSH rule 1 x 320 [3,7,1]
+  CRUSH rule 1 x 321 [1,3,0]
+  CRUSH rule 1 x 322 [2,7,3]
+  CRUSH rule 1 x 323 [4,7,0]
+  CRUSH rule 1 x 324 [7,0,8]
+  CRUSH rule 1 x 325 [4,6,0]
+  CRUSH rule 1 x 326 [3,4,1]
+  CRUSH rule 1 x 327 [0,6,7]
+  CRUSH rule 1 x 328 [7,4,8]
+  CRUSH rule 1 x 329 [5,6,3]
+  CRUSH rule 1 x 330 [3,7,4]
+  CRUSH rule 1 x 331 [2,6,3]
+  CRUSH rule 1 x 332 [2,0,3]
+  CRUSH rule 1 x 333 [6,8,5]
+  CRUSH rule 1 x 334 [8,3,5]
+  CRUSH rule 1 x 335 [7,1,2]
+  CRUSH rule 1 x 336 [4,6,0]
+  CRUSH rule 1 x 337 [7,2,6]
+  CRUSH rule 1 x 338 [5,6,4]
+  CRUSH rule 1 x 339 [7,5,2]
+  CRUSH rule 1 x 340 [2,0,1]
+  CRUSH rule 1 x 341 [5,1,7]
+  CRUSH rule 1 x 342 [0,7,4]
+  CRUSH rule 1 x 343 [6,7,3]
+  CRUSH rule 1 x 344 [6,0,5]
+  CRUSH rule 1 x 345 [4,3,5]
+  CRUSH rule 1 x 346 [8,0,5]
+  CRUSH rule 1 x 347 [3,1,5]
+  CRUSH rule 1 x 348 [8,0,2]
+  CRUSH rule 1 x 349 [1,6,7]
+  CRUSH rule 1 x 350 [8,5,7]
+  CRUSH rule 1 x 351 [3,6,4]
+  CRUSH rule 1 x 352 [1,0,2]
+  CRUSH rule 1 x 353 [6,4,5]
+  CRUSH rule 1 x 354 [0,3,1]
+  CRUSH rule 1 x 355 [3,4,6]
+  CRUSH rule 1 x 356 [3,5,4]
+  CRUSH rule 1 x 357 [6,1,2]
+  CRUSH rule 1 x 358 [2,1,8]
+  CRUSH rule 1 x 359 [6,7,8]
+  CRUSH rule 1 x 360 [5,3,1]
+  CRUSH rule 1 x 361 [8,4,5]
+  CRUSH rule 1 x 362 [4,5,3]
+  CRUSH rule 1 x 363 [4,0,3]
+  CRUSH rule 1 x 364 [2,5,0]
+  CRUSH rule 1 x 365 [6,7,8]
+  CRUSH rule 1 x 366 [7,2,8]
+  CRUSH rule 1 x 367 [4,5,0]
+  CRUSH rule 1 x 368 [7,4,6]
+  CRUSH rule 1 x 369 [3,7,1]
+  CRUSH rule 1 x 370 [8,7,6]
+  CRUSH rule 1 x 371 [1,3,5]
+  CRUSH rule 1 x 372 [3,1,8]
+  CRUSH rule 1 x 373 [0,6,8]
+  CRUSH rule 1 x 374 [3,8,4]
+  CRUSH rule 1 x 375 [6,4,5]
+  CRUSH rule 1 x 376 [7,1,2]
+  CRUSH rule 1 x 377 [1,2,4]
+  CRUSH rule 1 x 378 [0,1,2]
+  CRUSH rule 1 x 379 [8,5,4]
+  CRUSH rule 1 x 380 [2,5,0]
+  CRUSH rule 1 x 381 [0,4,7]
+  CRUSH rule 1 x 382 [1,5,2]
+  CRUSH rule 1 x 383 [4,3,5]
+  CRUSH rule 1 x 384 [7,0,6]
+  CRUSH rule 1 x 385 [7,4,6]
+  CRUSH rule 1 x 386 [0,3,5]
+  CRUSH rule 1 x 387 [1,3,6]
+  CRUSH rule 1 x 388 [5,0,3]
+  CRUSH rule 1 x 389 [1,5,4]
+  CRUSH rule 1 x 390 [5,6,0]
+  CRUSH rule 1 x 391 [5,6,2]
+  CRUSH rule 1 x 392 [1,8,6]
+  CRUSH rule 1 x 393 [4,2,6]
+  CRUSH rule 1 x 394 [4,7,6]
+  CRUSH rule 1 x 395 [4,0,1]
+  CRUSH rule 1 x 396 [4,2,5]
+  CRUSH rule 1 x 397 [2,1,5]
+  CRUSH rule 1 x 398 [2,4,5]
+  CRUSH rule 1 x 399 [8,7,3]
+  CRUSH rule 1 x 400 [8,1,4]
+  CRUSH rule 1 x 401 [0,1,2]
+  CRUSH rule 1 x 402 [7,8,6]
+  CRUSH rule 1 x 403 [0,1,4]
+  CRUSH rule 1 x 404 [4,3,2]
+  CRUSH rule 1 x 405 [6,5,2]
+  CRUSH rule 1 x 406 [2,0,1]
+  CRUSH rule 1 x 407 [2,8,7]
+  CRUSH rule 1 x 408 [4,1,6]
+  CRUSH rule 1 x 409 [7,3,6]
+  CRUSH rule 1 x 410 [8,6,3]
+  CRUSH rule 1 x 411 [2,0,6]
+  CRUSH rule 1 x 412 [0,5,8]
+  CRUSH rule 1 x 413 [5,0,8]
+  CRUSH rule 1 x 414 [4,1,3]
+  CRUSH rule 1 x 415 [0,6,1]
+  CRUSH rule 1 x 416 [2,1,7]
+  CRUSH rule 1 x 417 [8,7,2]
+  CRUSH rule 1 x 418 [7,6,8]
+  CRUSH rule 1 x 419 [8,3,0]
+  CRUSH rule 1 x 420 [1,4,2]
+  CRUSH rule 1 x 421 [8,6,7]
+  CRUSH rule 1 x 422 [6,7,8]
+  CRUSH rule 1 x 423 [0,5,6]
+  CRUSH rule 1 x 424 [8,4,7]
+  CRUSH rule 1 x 425 [1,3,5]
+  CRUSH rule 1 x 426 [6,7,2]
+  CRUSH rule 1 x 427 [0,7,5]
+  CRUSH rule 1 x 428 [5,4,3]
+  CRUSH rule 1 x 429 [4,6,7]
+  CRUSH rule 1 x 430 [3,6,5]
+  CRUSH rule 1 x 431 [5,3,0]
+  CRUSH rule 1 x 432 [7,1,2]
+  CRUSH rule 1 x 433 [6,5,1]
+  CRUSH rule 1 x 434 [5,2,1]
+  CRUSH rule 1 x 435 [0,5,3]
+  CRUSH rule 1 x 436 [4,0,5]
+  CRUSH rule 1 x 437 [7,5,3]
+  CRUSH rule 1 x 438 [0,3,8]
+  CRUSH rule 1 x 439 [1,3,4]
+  CRUSH rule 1 x 440 [2,7,1]
+  CRUSH rule 1 x 441 [5,7,2]
+  CRUSH rule 1 x 442 [2,4,5]
+  CRUSH rule 1 x 443 [6,8,0]
+  CRUSH rule 1 x 444 [7,0,1]
+  CRUSH rule 1 x 445 [6,3,5]
+  CRUSH rule 1 x 446 [4,3,5]
+  CRUSH rule 1 x 447 [2,1,4]
+  CRUSH rule 1 x 448 [7,2,5]
+  CRUSH rule 1 x 449 [7,8,3]
+  CRUSH rule 1 x 450 [4,5,2]
+  CRUSH rule 1 x 451 [6,8,3]
+  CRUSH rule 1 x 452 [8,3,5]
+  CRUSH rule 1 x 453 [6,8,7]
+  CRUSH rule 1 x 454 [6,7,5]
+  CRUSH rule 1 x 455 [2,7,5]
+  CRUSH rule 1 x 456 [6,8,7]
+  CRUSH rule 1 x 457 [7,2,8]
+  CRUSH rule 1 x 458 [2,8,1]
+  CRUSH rule 1 x 459 [2,0,6]
+  CRUSH rule 1 x 460 [6,5,7]
+  CRUSH rule 1 x 461 [6,5,4]
+  CRUSH rule 1 x 462 [8,1,5]
+  CRUSH rule 1 x 463 [6,7,2]
+  CRUSH rule 1 x 464 [7,4,2]
+  CRUSH rule 1 x 465 [7,6,0]
+  CRUSH rule 1 x 466 [5,8,4]
+  CRUSH rule 1 x 467 [6,4,7]
+  CRUSH rule 1 x 468 [7,8,0]
+  CRUSH rule 1 x 469 [7,0,1]
+  CRUSH rule 1 x 470 [3,0,5]
+  CRUSH rule 1 x 471 [0,1,2]
+  CRUSH rule 1 x 472 [5,4,0]
+  CRUSH rule 1 x 473 [1,0,4]
+  CRUSH rule 1 x 474 [6,0,7]
+  CRUSH rule 1 x 475 [6,7,8]
+  CRUSH rule 1 x 476 [4,3,5]
+  CRUSH rule 1 x 477 [5,8,6]
+  CRUSH rule 1 x 478 [6,7,2]
+  CRUSH rule 1 x 479 [0,5,8]
+  CRUSH rule 1 x 480 [1,8,6]
+  CRUSH rule 1 x 481 [2,4,5]
+  CRUSH rule 1 x 482 [4,3,5]
+  CRUSH rule 1 x 483 [0,2,6]
+  CRUSH rule 1 x 484 [1,2,7]
+  CRUSH rule 1 x 485 [4,7,3]
+  CRUSH rule 1 x 486 [4,1,7]
+  CRUSH rule 1 x 487 [5,0,1]
+  CRUSH rule 1 x 488 [5,7,1]
+  CRUSH rule 1 x 489 [2,8,5]
+  CRUSH rule 1 x 490 [6,4,1]
+  CRUSH rule 1 x 491 [1,0,6]
+  CRUSH rule 1 x 492 [6,5,4]
+  CRUSH rule 1 x 493 [0,2,7]
+  CRUSH rule 1 x 494 [1,0,8]
+  CRUSH rule 1 x 495 [3,4,5]
+  CRUSH rule 1 x 496 [7,5,4]
+  CRUSH rule 1 x 497 [5,7,3]
+  CRUSH rule 1 x 498 [0,5,8]
+  CRUSH rule 1 x 499 [8,4,3]
+  CRUSH rule 1 x 500 [3,6,5]
+  CRUSH rule 1 x 501 [0,7,2]
+  CRUSH rule 1 x 502 [7,1,4]
+  CRUSH rule 1 x 503 [2,3,5]
+  CRUSH rule 1 x 504 [5,6,3]
+  CRUSH rule 1 x 505 [0,7,6]
+  CRUSH rule 1 x 506 [5,3,0]
+  CRUSH rule 1 x 507 [6,0,8]
+  CRUSH rule 1 x 508 [0,1,2]
+  CRUSH rule 1 x 509 [7,5,8]
+  CRUSH rule 1 x 510 [6,0,2]
+  CRUSH rule 1 x 511 [5,8,7]
+  CRUSH rule 1 x 512 [7,6,2]
+  CRUSH rule 1 x 513 [7,2,1]
+  CRUSH rule 1 x 514 [4,3,5]
+  CRUSH rule 1 x 515 [8,5,6]
+  CRUSH rule 1 x 516 [4,0,1]
+  CRUSH rule 1 x 517 [7,8,6]
+  CRUSH rule 1 x 518 [4,6,7]
+  CRUSH rule 1 x 519 [7,3,5]
+  CRUSH rule 1 x 520 [2,6,3]
+  CRUSH rule 1 x 521 [8,7,6]
+  CRUSH rule 1 x 522 [6,8,0]
+  CRUSH rule 1 x 523 [4,2,5]
+  CRUSH rule 1 x 524 [0,4,5]
+  CRUSH rule 1 x 525 [0,4,1]
+  CRUSH rule 1 x 526 [1,5,8]
+  CRUSH rule 1 x 527 [0,2,4]
+  CRUSH rule 1 x 528 [5,3,0]
+  CRUSH rule 1 x 529 [5,7,0]
+  CRUSH rule 1 x 530 [6,7,8]
+  CRUSH rule 1 x 531 [6,1,0]
+  CRUSH rule 1 x 532 [6,3,7]
+  CRUSH rule 1 x 533 [5,6,3]
+  CRUSH rule 1 x 534 [7,3,5]
+  CRUSH rule 1 x 535 [8,6,0]
+  CRUSH rule 1 x 536 [6,7,0]
+  CRUSH rule 1 x 537 [3,7,5]
+  CRUSH rule 1 x 538 [6,8,3]
+  CRUSH rule 1 x 539 [8,3,7]
+  CRUSH rule 1 x 540 [0,6,1]
+  CRUSH rule 1 x 541 [2,3,1]
+  CRUSH rule 1 x 542 [3,5,4]
+  CRUSH rule 1 x 543 [6,0,8]
+  CRUSH rule 1 x 544 [3,7,6]
+  CRUSH rule 1 x 545 [5,7,0]
+  CRUSH rule 1 x 546 [6,1,7]
+  CRUSH rule 1 x 547 [8,2,0]
+  CRUSH rule 1 x 548 [5,2,0]
+  CRUSH rule 1 x 549 [5,8,2]
+  CRUSH rule 1 x 550 [0,5,4]
+  CRUSH rule 1 x 551 [7,5,4]
+  CRUSH rule 1 x 552 [5,4,3]
+  CRUSH rule 1 x 553 [4,2,3]
+  CRUSH rule 1 x 554 [0,8,5]
+  CRUSH rule 1 x 555 [5,0,8]
+  CRUSH rule 1 x 556 [3,4,5]
+  CRUSH rule 1 x 557 [7,4,6]
+  CRUSH rule 1 x 558 [3,1,0]
+  CRUSH rule 1 x 559 [4,2,6]
+  CRUSH rule 1 x 560 [8,3,5]
+  CRUSH rule 1 x 561 [6,3,7]
+  CRUSH rule 1 x 562 [3,4,1]
+  CRUSH rule 1 x 563 [2,6,0]
+  CRUSH rule 1 x 564 [5,1,7]
+  CRUSH rule 1 x 565 [3,6,5]
+  CRUSH rule 1 x 566 [4,7,2]
+  CRUSH rule 1 x 567 [3,6,1]
+  CRUSH rule 1 x 568 [7,4,1]
+  CRUSH rule 1 x 569 [3,4,2]
+  CRUSH rule 1 x 570 [1,5,4]
+  CRUSH rule 1 x 571 [3,7,5]
+  CRUSH rule 1 x 572 [3,4,0]
+  CRUSH rule 1 x 573 [3,0,7]
+  CRUSH rule 1 x 574 [2,0,1]
+  CRUSH rule 1 x 575 [8,6,0]
+  CRUSH rule 1 x 576 [4,6,8]
+  CRUSH rule 1 x 577 [8,2,6]
+  CRUSH rule 1 x 578 [6,8,7]
+  CRUSH rule 1 x 579 [3,1,6]
+  CRUSH rule 1 x 580 [3,0,7]
+  CRUSH rule 1 x 581 [7,2,6]
+  CRUSH rule 1 x 582 [2,8,7]
+  CRUSH rule 1 x 583 [6,0,8]
+  CRUSH rule 1 x 584 [8,1,2]
+  CRUSH rule 1 x 585 [7,0,5]
+  CRUSH rule 1 x 586 [0,1,7]
+  CRUSH rule 1 x 587 [2,5,0]
+  CRUSH rule 1 x 588 [3,4,7]
+  CRUSH rule 1 x 589 [7,1,4]
+  CRUSH rule 1 x 590 [6,2,3]
+  CRUSH rule 1 x 591 [5,2,4]
+  CRUSH rule 1 x 592 [2,0,3]
+  CRUSH rule 1 x 593 [0,8,1]
+  CRUSH rule 1 x 594 [0,7,4]
+  CRUSH rule 1 x 595 [7,1,8]
+  CRUSH rule 1 x 596 [4,3,5]
+  CRUSH rule 1 x 597 [3,1,2]
+  CRUSH rule 1 x 598 [3,2,4]
+  CRUSH rule 1 x 599 [5,2,1]
+  CRUSH rule 1 x 600 [7,0,8]
+  CRUSH rule 1 x 601 [0,7,8]
+  CRUSH rule 1 x 602 [3,7,5]
+  CRUSH rule 1 x 603 [5,1,6]
+  CRUSH rule 1 x 604 [7,5,4]
+  CRUSH rule 1 x 605 [3,0,5]
+  CRUSH rule 1 x 606 [2,0,1]
+  CRUSH rule 1 x 607 [0,4,8]
+  CRUSH rule 1 x 608 [5,3,2]
+  CRUSH rule 1 x 609 [5,2,4]
+  CRUSH rule 1 x 610 [3,7,6]
+  CRUSH rule 1 x 611 [1,0,2]
+  CRUSH rule 1 x 612 [2,0,8]
+  CRUSH rule 1 x 613 [7,2,0]
+  CRUSH rule 1 x 614 [7,8,6]
+  CRUSH rule 1 x 615 [6,8,2]
+  CRUSH rule 1 x 616 [0,8,1]
+  CRUSH rule 1 x 617 [6,1,2]
+  CRUSH rule 1 x 618 [7,6,5]
+  CRUSH rule 1 x 619 [5,1,8]
+  CRUSH rule 1 x 620 [4,1,0]
+  CRUSH rule 1 x 621 [5,8,3]
+  CRUSH rule 1 x 622 [0,4,3]
+  CRUSH rule 1 x 623 [0,6,2]
+  CRUSH rule 1 x 624 [3,5,1]
+  CRUSH rule 1 x 625 [2,3,7]
+  CRUSH rule 1 x 626 [7,8,0]
+  CRUSH rule 1 x 627 [2,7,1]
+  CRUSH rule 1 x 628 [8,0,5]
+  CRUSH rule 1 x 629 [2,6,5]
+  CRUSH rule 1 x 630 [2,7,1]
+  CRUSH rule 1 x 631 [0,7,8]
+  CRUSH rule 1 x 632 [7,0,2]
+  CRUSH rule 1 x 633 [8,6,5]
+  CRUSH rule 1 x 634 [0,4,2]
+  CRUSH rule 1 x 635 [5,6,4]
+  CRUSH rule 1 x 636 [1,4,2]
+  CRUSH rule 1 x 637 [4,1,2]
+  CRUSH rule 1 x 638 [6,8,1]
+  CRUSH rule 1 x 639 [4,2,1]
+  CRUSH rule 1 x 640 [3,1,0]
+  CRUSH rule 1 x 641 [7,2,0]
+  CRUSH rule 1 x 642 [2,0,1]
+  CRUSH rule 1 x 643 [3,5,1]
+  CRUSH rule 1 x 644 [8,1,5]
+  CRUSH rule 1 x 645 [5,4,6]
+  CRUSH rule 1 x 646 [8,0,3]
+  CRUSH rule 1 x 647 [7,1,3]
+  CRUSH rule 1 x 648 [0,6,3]
+  CRUSH rule 1 x 649 [4,7,5]
+  CRUSH rule 1 x 650 [7,8,6]
+  CRUSH rule 1 x 651 [3,7,6]
+  CRUSH rule 1 x 652 [3,4,8]
+  CRUSH rule 1 x 653 [8,5,2]
+  CRUSH rule 1 x 654 [7,5,2]
+  CRUSH rule 1 x 655 [0,3,4]
+  CRUSH rule 1 x 656 [4,3,8]
+  CRUSH rule 1 x 657 [6,1,2]
+  CRUSH rule 1 x 658 [5,4,6]
+  CRUSH rule 1 x 659 [4,6,7]
+  CRUSH rule 1 x 660 [7,8,6]
+  CRUSH rule 1 x 661 [1,8,3]
+  CRUSH rule 1 x 662 [4,5,3]
+  CRUSH rule 1 x 663 [1,3,8]
+  CRUSH rule 1 x 664 [1,4,0]
+  CRUSH rule 1 x 665 [5,7,6]
+  CRUSH rule 1 x 666 [2,8,4]
+  CRUSH rule 1 x 667 [1,3,5]
+  CRUSH rule 1 x 668 [3,7,6]
+  CRUSH rule 1 x 669 [6,4,5]
+  CRUSH rule 1 x 670 [4,0,3]
+  CRUSH rule 1 x 671 [0,2,1]
+  CRUSH rule 1 x 672 [4,3,2]
+  CRUSH rule 1 x 673 [5,3,0]
+  CRUSH rule 1 x 674 [3,1,8]
+  CRUSH rule 1 x 675 [0,8,6]
+  CRUSH rule 1 x 676 [0,2,4]
+  CRUSH rule 1 x 677 [4,1,3]
+  CRUSH rule 1 x 678 [2,3,1]
+  CRUSH rule 1 x 679 [6,0,2]
+  CRUSH rule 1 x 680 [0,4,6]
+  CRUSH rule 1 x 681 [4,7,3]
+  CRUSH rule 1 x 682 [0,5,2]
+  CRUSH rule 1 x 683 [0,1,5]
+  CRUSH rule 1 x 684 [7,1,6]
+  CRUSH rule 1 x 685 [7,1,6]
+  CRUSH rule 1 x 686 [1,4,3]
+  CRUSH rule 1 x 687 [3,5,7]
+  CRUSH rule 1 x 688 [5,7,2]
+  CRUSH rule 1 x 689 [6,5,0]
+  CRUSH rule 1 x 690 [8,1,7]
+  CRUSH rule 1 x 691 [3,0,5]
+  CRUSH rule 1 x 692 [7,2,8]
+  CRUSH rule 1 x 693 [6,7,3]
+  CRUSH rule 1 x 694 [6,5,1]
+  CRUSH rule 1 x 695 [0,8,7]
+  CRUSH rule 1 x 696 [1,4,3]
+  CRUSH rule 1 x 697 [6,1,2]
+  CRUSH rule 1 x 698 [6,2,0]
+  CRUSH rule 1 x 699 [1,6,8]
+  CRUSH rule 1 x 700 [0,3,1]
+  CRUSH rule 1 x 701 [4,3,2]
+  CRUSH rule 1 x 702 [3,5,0]
+  CRUSH rule 1 x 703 [8,3,4]
+  CRUSH rule 1 x 704 [0,3,8]
+  CRUSH rule 1 x 705 [8,6,0]
+  CRUSH rule 1 x 706 [1,2,4]
+  CRUSH rule 1 x 707 [7,8,6]
+  CRUSH rule 1 x 708 [3,5,8]
+  CRUSH rule 1 x 709 [6,3,0]
+  CRUSH rule 1 x 710 [8,4,3]
+  CRUSH rule 1 x 711 [2,3,8]
+  CRUSH rule 1 x 712 [2,3,7]
+  CRUSH rule 1 x 713 [6,7,8]
+  CRUSH rule 1 x 714 [3,2,0]
+  CRUSH rule 1 x 715 [1,2,4]
+  CRUSH rule 1 x 716 [3,6,0]
+  CRUSH rule 1 x 717 [8,7,2]
+  CRUSH rule 1 x 718 [3,7,8]
+  CRUSH rule 1 x 719 [2,6,3]
+  CRUSH rule 1 x 720 [6,8,2]
+  CRUSH rule 1 x 721 [5,4,7]
+  CRUSH rule 1 x 722 [5,4,6]
+  CRUSH rule 1 x 723 [5,1,0]
+  CRUSH rule 1 x 724 [0,6,2]
+  CRUSH rule 1 x 725 [0,1,5]
+  CRUSH rule 1 x 726 [3,8,4]
+  CRUSH rule 1 x 727 [4,6,8]
+  CRUSH rule 1 x 728 [2,1,0]
+  CRUSH rule 1 x 729 [5,3,7]
+  CRUSH rule 1 x 730 [3,7,5]
+  CRUSH rule 1 x 731 [4,1,5]
+  CRUSH rule 1 x 732 [1,5,3]
+  CRUSH rule 1 x 733 [5,4,7]
+  CRUSH rule 1 x 734 [6,4,2]
+  CRUSH rule 1 x 735 [4,8,3]
+  CRUSH rule 1 x 736 [3,5,6]
+  CRUSH rule 1 x 737 [1,0,8]
+  CRUSH rule 1 x 738 [5,2,3]
+  CRUSH rule 1 x 739 [0,1,2]
+  CRUSH rule 1 x 740 [0,1,7]
+  CRUSH rule 1 x 741 [7,8,0]
+  CRUSH rule 1 x 742 [8,2,1]
+  CRUSH rule 1 x 743 [7,0,1]
+  CRUSH rule 1 x 744 [4,7,3]
+  CRUSH rule 1 x 745 [3,4,1]
+  CRUSH rule 1 x 746 [4,1,3]
+  CRUSH rule 1 x 747 [6,0,3]
+  CRUSH rule 1 x 748 [2,7,0]
+  CRUSH rule 1 x 749 [4,5,8]
+  CRUSH rule 1 x 750 [1,6,3]
+  CRUSH rule 1 x 751 [2,1,6]
+  CRUSH rule 1 x 752 [8,1,5]
+  CRUSH rule 1 x 753 [7,8,3]
+  CRUSH rule 1 x 754 [8,6,7]
+  CRUSH rule 1 x 755 [1,2,0]
+  CRUSH rule 1 x 756 [5,6,1]
+  CRUSH rule 1 x 757 [8,6,1]
+  CRUSH rule 1 x 758 [6,0,3]
+  CRUSH rule 1 x 759 [8,5,3]
+  CRUSH rule 1 x 760 [1,5,4]
+  CRUSH rule 1 x 761 [4,1,2]
+  CRUSH rule 1 x 762 [2,7,8]
+  CRUSH rule 1 x 763 [8,6,7]
+  CRUSH rule 1 x 764 [1,7,0]
+  CRUSH rule 1 x 765 [6,5,2]
+  CRUSH rule 1 x 766 [8,5,7]
+  CRUSH rule 1 x 767 [1,2,0]
+  CRUSH rule 1 x 768 [8,3,2]
+  CRUSH rule 1 x 769 [6,2,8]
+  CRUSH rule 1 x 770 [6,0,7]
+  CRUSH rule 1 x 771 [7,0,3]
+  CRUSH rule 1 x 772 [8,3,7]
+  CRUSH rule 1 x 773 [3,1,5]
+  CRUSH rule 1 x 774 [4,6,5]
+  CRUSH rule 1 x 775 [6,8,4]
+  CRUSH rule 1 x 776 [7,2,1]
+  CRUSH rule 1 x 777 [3,1,6]
+  CRUSH rule 1 x 778 [1,8,0]
+  CRUSH rule 1 x 779 [2,7,3]
+  CRUSH rule 1 x 780 [0,2,3]
+  CRUSH rule 1 x 781 [6,3,7]
+  CRUSH rule 1 x 782 [5,4,0]
+  CRUSH rule 1 x 783 [7,1,8]
+  CRUSH rule 1 x 784 [0,1,5]
+  CRUSH rule 1 x 785 [6,1,2]
+  CRUSH rule 1 x 786 [7,6,5]
+  CRUSH rule 1 x 787 [1,0,6]
+  CRUSH rule 1 x 788 [6,0,8]
+  CRUSH rule 1 x 789 [0,4,3]
+  CRUSH rule 1 x 790 [8,4,7]
+  CRUSH rule 1 x 791 [3,8,7]
+  CRUSH rule 1 x 792 [5,8,4]
+  CRUSH rule 1 x 793 [6,1,3]
+  CRUSH rule 1 x 794 [2,6,8]
+  CRUSH rule 1 x 795 [0,3,2]
+  CRUSH rule 1 x 796 [3,4,5]
+  CRUSH rule 1 x 797 [2,3,4]
+  CRUSH rule 1 x 798 [6,8,7]
+  CRUSH rule 1 x 799 [5,1,4]
+  CRUSH rule 1 x 800 [5,0,3]
+  CRUSH rule 1 x 801 [3,6,8]
+  CRUSH rule 1 x 802 [1,8,0]
+  CRUSH rule 1 x 803 [0,5,7]
+  CRUSH rule 1 x 804 [6,2,5]
+  CRUSH rule 1 x 805 [3,6,7]
+  CRUSH rule 1 x 806 [1,3,4]
+  CRUSH rule 1 x 807 [5,4,7]
+  CRUSH rule 1 x 808 [4,6,2]
+  CRUSH rule 1 x 809 [1,4,5]
+  CRUSH rule 1 x 810 [5,7,3]
+  CRUSH rule 1 x 811 [8,4,5]
+  CRUSH rule 1 x 812 [8,5,4]
+  CRUSH rule 1 x 813 [6,4,8]
+  CRUSH rule 1 x 814 [3,6,8]
+  CRUSH rule 1 x 815 [3,1,2]
+  CRUSH rule 1 x 816 [2,1,8]
+  CRUSH rule 1 x 817 [4,3,7]
+  CRUSH rule 1 x 818 [3,5,0]
+  CRUSH rule 1 x 819 [5,1,4]
+  CRUSH rule 1 x 820 [3,5,4]
+  CRUSH rule 1 x 821 [4,5,8]
+  CRUSH rule 1 x 822 [2,0,5]
+  CRUSH rule 1 x 823 [4,8,2]
+  CRUSH rule 1 x 824 [3,7,4]
+  CRUSH rule 1 x 825 [2,8,7]
+  CRUSH rule 1 x 826 [7,0,5]
+  CRUSH rule 1 x 827 [0,8,2]
+  CRUSH rule 1 x 828 [2,3,1]
+  CRUSH rule 1 x 829 [5,6,7]
+  CRUSH rule 1 x 830 [2,3,5]
+  CRUSH rule 1 x 831 [1,6,0]
+  CRUSH rule 1 x 832 [4,5,3]
+  CRUSH rule 1 x 833 [2,1,7]
+  CRUSH rule 1 x 834 [3,4,2]
+  CRUSH rule 1 x 835 [8,4,3]
+  CRUSH rule 1 x 836 [3,4,5]
+  CRUSH rule 1 x 837 [6,3,1]
+  CRUSH rule 1 x 838 [6,7,2]
+  CRUSH rule 1 x 839 [5,0,6]
+  CRUSH rule 1 x 840 [7,8,5]
+  CRUSH rule 1 x 841 [4,8,5]
+  CRUSH rule 1 x 842 [2,4,0]
+  CRUSH rule 1 x 843 [6,4,7]
+  CRUSH rule 1 x 844 [4,8,1]
+  CRUSH rule 1 x 845 [3,8,6]
+  CRUSH rule 1 x 846 [3,2,0]
+  CRUSH rule 1 x 847 [0,2,7]
+  CRUSH rule 1 x 848 [2,6,1]
+  CRUSH rule 1 x 849 [4,5,3]
+  CRUSH rule 1 x 850 [1,0,5]
+  CRUSH rule 1 x 851 [6,8,7]
+  CRUSH rule 1 x 852 [7,3,8]
+  CRUSH rule 1 x 853 [6,8,1]
+  CRUSH rule 1 x 854 [7,6,0]
+  CRUSH rule 1 x 855 [5,7,2]
+  CRUSH rule 1 x 856 [6,7,3]
+  CRUSH rule 1 x 857 [8,5,0]
+  CRUSH rule 1 x 858 [6,4,1]
+  CRUSH rule 1 x 859 [6,0,7]
+  CRUSH rule 1 x 860 [4,1,2]
+  CRUSH rule 1 x 861 [8,7,6]
+  CRUSH rule 1 x 862 [6,1,7]
+  CRUSH rule 1 x 863 [8,7,0]
+  CRUSH rule 1 x 864 [5,6,8]
+  CRUSH rule 1 x 865 [8,1,0]
+  CRUSH rule 1 x 866 [3,4,8]
+  CRUSH rule 1 x 867 [6,5,1]
+  CRUSH rule 1 x 868 [6,3,0]
+  CRUSH rule 1 x 869 [8,7,3]
+  CRUSH rule 1 x 870 [0,4,8]
+  CRUSH rule 1 x 871 [3,4,5]
+  CRUSH rule 1 x 872 [5,1,3]
+  CRUSH rule 1 x 873 [4,6,5]
+  CRUSH rule 1 x 874 [2,6,1]
+  CRUSH rule 1 x 875 [2,6,4]
+  CRUSH rule 1 x 876 [5,8,1]
+  CRUSH rule 1 x 877 [6,4,2]
+  CRUSH rule 1 x 878 [5,4,0]
+  CRUSH rule 1 x 879 [7,4,8]
+  CRUSH rule 1 x 880 [3,5,0]
+  CRUSH rule 1 x 881 [5,6,1]
+  CRUSH rule 1 x 882 [4,0,2]
+  CRUSH rule 1 x 883 [2,1,0]
+  CRUSH rule 1 x 884 [6,0,4]
+  CRUSH rule 1 x 885 [5,1,4]
+  CRUSH rule 1 x 886 [3,6,4]
+  CRUSH rule 1 x 887 [7,4,0]
+  CRUSH rule 1 x 888 [6,8,0]
+  CRUSH rule 1 x 889 [2,1,7]
+  CRUSH rule 1 x 890 [7,2,0]
+  CRUSH rule 1 x 891 [1,8,0]
+  CRUSH rule 1 x 892 [6,2,3]
+  CRUSH rule 1 x 893 [2,3,7]
+  CRUSH rule 1 x 894 [7,5,0]
+  CRUSH rule 1 x 895 [5,3,2]
+  CRUSH rule 1 x 896 [1,8,2]
+  CRUSH rule 1 x 897 [4,2,6]
+  CRUSH rule 1 x 898 [0,5,4]
+  CRUSH rule 1 x 899 [1,7,6]
+  CRUSH rule 1 x 900 [4,1,0]
+  CRUSH rule 1 x 901 [5,0,1]
+  CRUSH rule 1 x 902 [8,5,7]
+  CRUSH rule 1 x 903 [5,7,3]
+  CRUSH rule 1 x 904 [5,6,8]
+  CRUSH rule 1 x 905 [6,2,5]
+  CRUSH rule 1 x 906 [1,2,0]
+  CRUSH rule 1 x 907 [7,1,0]
+  CRUSH rule 1 x 908 [5,8,1]
+  CRUSH rule 1 x 909 [2,3,4]
+  CRUSH rule 1 x 910 [6,4,0]
+  CRUSH rule 1 x 911 [5,8,4]
+  CRUSH rule 1 x 912 [0,1,7]
+  CRUSH rule 1 x 913 [7,6,8]
+  CRUSH rule 1 x 914 [6,4,7]
+  CRUSH rule 1 x 915 [8,2,6]
+  CRUSH rule 1 x 916 [3,1,4]
+  CRUSH rule 1 x 917 [1,5,3]
+  CRUSH rule 1 x 918 [8,2,1]
+  CRUSH rule 1 x 919 [6,2,8]
+  CRUSH rule 1 x 920 [7,6,4]
+  CRUSH rule 1 x 921 [1,4,5]
+  CRUSH rule 1 x 922 [6,7,8]
+  CRUSH rule 1 x 923 [5,3,6]
+  CRUSH rule 1 x 924 [3,5,4]
+  CRUSH rule 1 x 925 [5,7,3]
+  CRUSH rule 1 x 926 [3,4,5]
+  CRUSH rule 1 x 927 [1,6,3]
+  CRUSH rule 1 x 928 [8,1,2]
+  CRUSH rule 1 x 929 [4,5,1]
+  CRUSH rule 1 x 930 [2,4,6]
+  CRUSH rule 1 x 931 [5,0,1]
+  CRUSH rule 1 x 932 [4,3,0]
+  CRUSH rule 1 x 933 [8,5,4]
+  CRUSH rule 1 x 934 [5,3,8]
+  CRUSH rule 1 x 935 [6,3,4]
+  CRUSH rule 1 x 936 [0,6,7]
+  CRUSH rule 1 x 937 [5,4,3]
+  CRUSH rule 1 x 938 [6,5,8]
+  CRUSH rule 1 x 939 [2,7,0]
+  CRUSH rule 1 x 940 [8,7,6]
+  CRUSH rule 1 x 941 [5,2,0]
+  CRUSH rule 1 x 942 [1,0,2]
+  CRUSH rule 1 x 943 [8,2,4]
+  CRUSH rule 1 x 944 [4,3,7]
+  CRUSH rule 1 x 945 [7,2,4]
+  CRUSH rule 1 x 946 [2,0,7]
+  CRUSH rule 1 x 947 [4,5,3]
+  CRUSH rule 1 x 948 [7,8,6]
+  CRUSH rule 1 x 949 [6,1,7]
+  CRUSH rule 1 x 950 [3,5,8]
+  CRUSH rule 1 x 951 [4,5,3]
+  CRUSH rule 1 x 952 [2,0,7]
+  CRUSH rule 1 x 953 [1,3,5]
+  CRUSH rule 1 x 954 [4,2,5]
+  CRUSH rule 1 x 955 [8,6,0]
+  CRUSH rule 1 x 956 [1,0,8]
+  CRUSH rule 1 x 957 [7,6,1]
+  CRUSH rule 1 x 958 [8,7,5]
+  CRUSH rule 1 x 959 [5,2,7]
+  CRUSH rule 1 x 960 [3,6,5]
+  CRUSH rule 1 x 961 [4,0,2]
+  CRUSH rule 1 x 962 [7,4,3]
+  CRUSH rule 1 x 963 [0,5,2]
+  CRUSH rule 1 x 964 [3,1,4]
+  CRUSH rule 1 x 965 [7,6,5]
+  CRUSH rule 1 x 966 [3,8,4]
+  CRUSH rule 1 x 967 [8,6,5]
+  CRUSH rule 1 x 968 [7,2,4]
+  CRUSH rule 1 x 969 [8,0,6]
+  CRUSH rule 1 x 970 [0,6,3]
+  CRUSH rule 1 x 971 [1,7,8]
+  CRUSH rule 1 x 972 [1,8,4]
+  CRUSH rule 1 x 973 [1,2,0]
+  CRUSH rule 1 x 974 [5,3,2]
+  CRUSH rule 1 x 975 [3,7,4]
+  CRUSH rule 1 x 976 [4,3,5]
+  CRUSH rule 1 x 977 [8,3,2]
+  CRUSH rule 1 x 978 [7,2,8]
+  CRUSH rule 1 x 979 [7,6,0]
+  CRUSH rule 1 x 980 [6,0,7]
+  CRUSH rule 1 x 981 [7,3,2]
+  CRUSH rule 1 x 982 [4,2,0]
+  CRUSH rule 1 x 983 [3,5,6]
+  CRUSH rule 1 x 984 [0,2,1]
+  CRUSH rule 1 x 985 [2,5,4]
+  CRUSH rule 1 x 986 [8,7,3]
+  CRUSH rule 1 x 987 [0,5,1]
+  CRUSH rule 1 x 988 [1,3,5]
+  CRUSH rule 1 x 989 [0,6,3]
+  CRUSH rule 1 x 990 [1,0,8]
+  CRUSH rule 1 x 991 [0,4,1]
+  CRUSH rule 1 x 992 [7,1,5]
+  CRUSH rule 1 x 993 [0,6,2]
+  CRUSH rule 1 x 994 [3,4,5]
+  CRUSH rule 1 x 995 [7,6,2]
+  CRUSH rule 1 x 996 [6,7,5]
+  CRUSH rule 1 x 997 [6,4,1]
+  CRUSH rule 1 x 998 [8,1,2]
+  CRUSH rule 1 x 999 [0,7,8]
+  CRUSH rule 1 x 1000 [8,5,0]
+  CRUSH rule 1 x 1001 [2,0,4]
+  CRUSH rule 1 x 1002 [1,3,2]
+  CRUSH rule 1 x 1003 [2,8,7]
+  CRUSH rule 1 x 1004 [6,1,2]
+  CRUSH rule 1 x 1005 [6,1,2]
+  CRUSH rule 1 x 1006 [1,0,2]
+  CRUSH rule 1 x 1007 [1,2,4]
+  CRUSH rule 1 x 1008 [1,7,0]
+  CRUSH rule 1 x 1009 [6,8,5]
+  CRUSH rule 1 x 1010 [3,4,0]
+  CRUSH rule 1 x 1011 [3,0,4]
+  CRUSH rule 1 x 1012 [3,0,7]
+  CRUSH rule 1 x 1013 [5,1,0]
+  CRUSH rule 1 x 1014 [2,8,4]
+  CRUSH rule 1 x 1015 [6,8,4]
+  CRUSH rule 1 x 1016 [2,0,1]
+  CRUSH rule 1 x 1017 [6,0,2]
+  CRUSH rule 1 x 1018 [5,4,3]
+  CRUSH rule 1 x 1019 [5,3,8]
+  CRUSH rule 1 x 1020 [5,1,3]
+  CRUSH rule 1 x 1021 [5,2,1]
+  CRUSH rule 1 x 1022 [1,6,7]
+  CRUSH rule 1 x 1023 [3,2,0]
+  rule 1 (choose-two) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 2 (chooseleaf), x = 0..1023, numrep = 2..3
+  CRUSH rule 2 x 0 [0,5]
+  CRUSH rule 2 x 1 [0,8]
+  CRUSH rule 2 x 2 [1,3]
+  CRUSH rule 2 x 3 [8,0]
+  CRUSH rule 2 x 4 [5,0]
+  CRUSH rule 2 x 5 [7,0]
+  CRUSH rule 2 x 6 [2,6]
+  CRUSH rule 2 x 7 [5,8]
+  CRUSH rule 2 x 8 [5,6]
+  CRUSH rule 2 x 9 [2,3]
+  CRUSH rule 2 x 10 [0,7]
+  CRUSH rule 2 x 11 [0,7]
+  CRUSH rule 2 x 12 [0,4]
+  CRUSH rule 2 x 13 [3,8]
+  CRUSH rule 2 x 14 [7,0]
+  CRUSH rule 2 x 15 [7,2]
+  CRUSH rule 2 x 16 [3,6]
+  CRUSH rule 2 x 17 [5,1]
+  CRUSH rule 2 x 18 [1,4]
+  CRUSH rule 2 x 19 [7,5]
+  CRUSH rule 2 x 20 [2,4]
+  CRUSH rule 2 x 21 [3,7]
+  CRUSH rule 2 x 22 [8,3]
+  CRUSH rule 2 x 23 [3,6]
+  CRUSH rule 2 x 24 [1,7]
+  CRUSH rule 2 x 25 [3,7]
+  CRUSH rule 2 x 26 [2,8]
+  CRUSH rule 2 x 27 [3,1]
+  CRUSH rule 2 x 28 [6,0]
+  CRUSH rule 2 x 29 [8,5]
+  CRUSH rule 2 x 30 [5,7]
+  CRUSH rule 2 x 31 [8,0]
+  CRUSH rule 2 x 32 [3,6]
+  CRUSH rule 2 x 33 [2,7]
+  CRUSH rule 2 x 34 [2,5]
+  CRUSH rule 2 x 35 [0,8]
+  CRUSH rule 2 x 36 [3,8]
+  CRUSH rule 2 x 37 [0,4]
+  CRUSH rule 2 x 38 [4,8]
+  CRUSH rule 2 x 39 [3,7]
+  CRUSH rule 2 x 40 [7,2]
+  CRUSH rule 2 x 41 [0,6]
+  CRUSH rule 2 x 42 [4,6]
+  CRUSH rule 2 x 43 [0,3]
+  CRUSH rule 2 x 44 [1,6]
+  CRUSH rule 2 x 45 [8,0]
+  CRUSH rule 2 x 46 [2,4]
+  CRUSH rule 2 x 47 [4,2]
+  CRUSH rule 2 x 48 [4,6]
+  CRUSH rule 2 x 49 [5,7]
+  CRUSH rule 2 x 50 [3,1]
+  CRUSH rule 2 x 51 [3,6]
+  CRUSH rule 2 x 52 [8,1]
+  CRUSH rule 2 x 53 [3,8]
+  CRUSH rule 2 x 54 [7,3]
+  CRUSH rule 2 x 55 [8,2]
+  CRUSH rule 2 x 56 [6,4]
+  CRUSH rule 2 x 57 [5,8]
+  CRUSH rule 2 x 58 [1,8]
+  CRUSH rule 2 x 59 [4,2]
+  CRUSH rule 2 x 60 [3,2]
+  CRUSH rule 2 x 61 [4,6]
+  CRUSH rule 2 x 62 [7,0]
+  CRUSH rule 2 x 63 [5,6]
+  CRUSH rule 2 x 64 [4,2]
+  CRUSH rule 2 x 65 [7,3]
+  CRUSH rule 2 x 66 [5,6]
+  CRUSH rule 2 x 67 [5,0]
+  CRUSH rule 2 x 68 [0,5]
+  CRUSH rule 2 x 69 [5,1]
+  CRUSH rule 2 x 70 [7,0]
+  CRUSH rule 2 x 71 [2,8]
+  CRUSH rule 2 x 72 [6,1]
+  CRUSH rule 2 x 73 [2,7]
+  CRUSH rule 2 x 74 [0,7]
+  CRUSH rule 2 x 75 [3,2]
+  CRUSH rule 2 x 76 [5,1]
+  CRUSH rule 2 x 77 [7,2]
+  CRUSH rule 2 x 78 [1,4]
+  CRUSH rule 2 x 79 [5,1]
+  CRUSH rule 2 x 80 [0,3]
+  CRUSH rule 2 x 81 [0,3]
+  CRUSH rule 2 x 82 [7,1]
+  CRUSH rule 2 x 83 [2,6]
+  CRUSH rule 2 x 84 [7,2]
+  CRUSH rule 2 x 85 [3,8]
+  CRUSH rule 2 x 86 [0,6]
+  CRUSH rule 2 x 87 [0,7]
+  CRUSH rule 2 x 88 [1,6]
+  CRUSH rule 2 x 89 [3,0]
+  CRUSH rule 2 x 90 [6,4]
+  CRUSH rule 2 x 91 [3,8]
+  CRUSH rule 2 x 92 [1,8]
+  CRUSH rule 2 x 93 [7,4]
+  CRUSH rule 2 x 94 [0,4]
+  CRUSH rule 2 x 95 [7,5]
+  CRUSH rule 2 x 96 [3,6]
+  CRUSH rule 2 x 97 [8,4]
+  CRUSH rule 2 x 98 [2,7]
+  CRUSH rule 2 x 99 [0,7]
+  CRUSH rule 2 x 100 [1,7]
+  CRUSH rule 2 x 101 [3,7]
+  CRUSH rule 2 x 102 [4,2]
+  CRUSH rule 2 x 103 [4,7]
+  CRUSH rule 2 x 104 [7,4]
+  CRUSH rule 2 x 105 [2,4]
+  CRUSH rule 2 x 106 [1,6]
+  CRUSH rule 2 x 107 [3,2]
+  CRUSH rule 2 x 108 [7,2]
+  CRUSH rule 2 x 109 [1,4]
+  CRUSH rule 2 x 110 [3,2]
+  CRUSH rule 2 x 111 [2,3]
+  CRUSH rule 2 x 112 [2,6]
+  CRUSH rule 2 x 113 [6,2]
+  CRUSH rule 2 x 114 [7,3]
+  CRUSH rule 2 x 115 [8,2]
+  CRUSH rule 2 x 116 [1,6]
+  CRUSH rule 2 x 117 [7,3]
+  CRUSH rule 2 x 118 [0,3]
+  CRUSH rule 2 x 119 [5,6]
+  CRUSH rule 2 x 120 [0,3]
+  CRUSH rule 2 x 121 [2,7]
+  CRUSH rule 2 x 122 [8,5]
+  CRUSH rule 2 x 123 [2,5]
+  CRUSH rule 2 x 124 [3,2]
+  CRUSH rule 2 x 125 [0,7]
+  CRUSH rule 2 x 126 [4,2]
+  CRUSH rule 2 x 127 [3,6]
+  CRUSH rule 2 x 128 [3,6]
+  CRUSH rule 2 x 129 [0,3]
+  CRUSH rule 2 x 130 [3,8]
+  CRUSH rule 2 x 131 [1,3]
+  CRUSH rule 2 x 132 [1,4]
+  CRUSH rule 2 x 133 [3,6]
+  CRUSH rule 2 x 134 [1,8]
+  CRUSH rule 2 x 135 [5,6]
+  CRUSH rule 2 x 136 [2,3]
+  CRUSH rule 2 x 137 [7,3]
+  CRUSH rule 2 x 138 [8,4]
+  CRUSH rule 2 x 139 [3,0]
+  CRUSH rule 2 x 140 [1,6]
+  CRUSH rule 2 x 141 [6,2]
+  CRUSH rule 2 x 142 [3,0]
+  CRUSH rule 2 x 143 [5,8]
+  CRUSH rule 2 x 144 [8,1]
+  CRUSH rule 2 x 145 [8,5]
+  CRUSH rule 2 x 146 [2,6]
+  CRUSH rule 2 x 147 [2,8]
+  CRUSH rule 2 x 148 [3,1]
+  CRUSH rule 2 x 149 [4,8]
+  CRUSH rule 2 x 150 [1,6]
+  CRUSH rule 2 x 151 [3,6]
+  CRUSH rule 2 x 152 [8,4]
+  CRUSH rule 2 x 153 [8,4]
+  CRUSH rule 2 x 154 [3,2]
+  CRUSH rule 2 x 155 [3,7]
+  CRUSH rule 2 x 156 [4,2]
+  CRUSH rule 2 x 157 [4,1]
+  CRUSH rule 2 x 158 [2,8]
+  CRUSH rule 2 x 159 [7,0]
+  CRUSH rule 2 x 160 [2,8]
+  CRUSH rule 2 x 161 [1,5]
+  CRUSH rule 2 x 162 [0,6]
+  CRUSH rule 2 x 163 [5,6]
+  CRUSH rule 2 x 164 [7,1]
+  CRUSH rule 2 x 165 [7,0]
+  CRUSH rule 2 x 166 [2,4]
+  CRUSH rule 2 x 167 [0,7]
+  CRUSH rule 2 x 168 [4,2]
+  CRUSH rule 2 x 169 [2,6]
+  CRUSH rule 2 x 170 [1,4]
+  CRUSH rule 2 x 171 [7,5]
+  CRUSH rule 2 x 172 [0,7]
+  CRUSH rule 2 x 173 [8,5]
+  CRUSH rule 2 x 174 [1,4]
+  CRUSH rule 2 x 175 [6,0]
+  CRUSH rule 2 x 176 [4,1]
+  CRUSH rule 2 x 177 [5,1]
+  CRUSH rule 2 x 178 [3,0]
+  CRUSH rule 2 x 179 [4,1]
+  CRUSH rule 2 x 180 [3,8]
+  CRUSH rule 2 x 181 [6,2]
+  CRUSH rule 2 x 182 [8,5]
+  CRUSH rule 2 x 183 [7,5]
+  CRUSH rule 2 x 184 [5,7]
+  CRUSH rule 2 x 185 [6,1]
+  CRUSH rule 2 x 186 [2,5]
+  CRUSH rule 2 x 187 [1,6]
+  CRUSH rule 2 x 188 [1,8]
+  CRUSH rule 2 x 189 [0,7]
+  CRUSH rule 2 x 190 [4,0]
+  CRUSH rule 2 x 191 [7,1]
+  CRUSH rule 2 x 192 [5,0]
+  CRUSH rule 2 x 193 [4,2]
+  CRUSH rule 2 x 194 [1,3]
+  CRUSH rule 2 x 195 [6,4]
+  CRUSH rule 2 x 196 [6,0]
+  CRUSH rule 2 x 197 [6,5]
+  CRUSH rule 2 x 198 [2,5]
+  CRUSH rule 2 x 199 [0,5]
+  CRUSH rule 2 x 200 [0,5]
+  CRUSH rule 2 x 201 [7,1]
+  CRUSH rule 2 x 202 [6,3]
+  CRUSH rule 2 x 203 [4,8]
+  CRUSH rule 2 x 204 [2,3]
+  CRUSH rule 2 x 205 [0,7]
+  CRUSH rule 2 x 206 [0,7]
+  CRUSH rule 2 x 207 [3,0]
+  CRUSH rule 2 x 208 [7,1]
+  CRUSH rule 2 x 209 [1,8]
+  CRUSH rule 2 x 210 [1,4]
+  CRUSH rule 2 x 211 [5,2]
+  CRUSH rule 2 x 212 [7,5]
+  CRUSH rule 2 x 213 [8,4]
+  CRUSH rule 2 x 214 [4,8]
+  CRUSH rule 2 x 215 [8,1]
+  CRUSH rule 2 x 216 [5,0]
+  CRUSH rule 2 x 217 [0,7]
+  CRUSH rule 2 x 218 [0,7]
+  CRUSH rule 2 x 219 [4,8]
+  CRUSH rule 2 x 220 [5,7]
+  CRUSH rule 2 x 221 [3,6]
+  CRUSH rule 2 x 222 [6,4]
+  CRUSH rule 2 x 223 [1,3]
+  CRUSH rule 2 x 224 [1,5]
+  CRUSH rule 2 x 225 [8,2]
+  CRUSH rule 2 x 226 [7,2]
+  CRUSH rule 2 x 227 [3,1]
+  CRUSH rule 2 x 228 [5,6]
+  CRUSH rule 2 x 229 [3,8]
+  CRUSH rule 2 x 230 [4,7]
+  CRUSH rule 2 x 231 [4,7]
+  CRUSH rule 2 x 232 [2,7]
+  CRUSH rule 2 x 233 [3,7]
+  CRUSH rule 2 x 234 [0,3]
+  CRUSH rule 2 x 235 [3,8]
+  CRUSH rule 2 x 236 [5,2]
+  CRUSH rule 2 x 237 [4,7]
+  CRUSH rule 2 x 238 [4,2]
+  CRUSH rule 2 x 239 [8,4]
+  CRUSH rule 2 x 240 [5,7]
+  CRUSH rule 2 x 241 [3,1]
+  CRUSH rule 2 x 242 [3,2]
+  CRUSH rule 2 x 243 [4,7]
+  CRUSH rule 2 x 244 [4,6]
+  CRUSH rule 2 x 245 [7,0]
+  CRUSH rule 2 x 246 [1,5]
+  CRUSH rule 2 x 247 [6,0]
+  CRUSH rule 2 x 248 [8,0]
+  CRUSH rule 2 x 249 [2,4]
+  CRUSH rule 2 x 250 [2,5]
+  CRUSH rule 2 x 251 [2,3]
+  CRUSH rule 2 x 252 [3,7]
+  CRUSH rule 2 x 253 [3,2]
+  CRUSH rule 2 x 254 [3,2]
+  CRUSH rule 2 x 255 [1,7]
+  CRUSH rule 2 x 256 [5,7]
+  CRUSH rule 2 x 257 [2,8]
+  CRUSH rule 2 x 258 [5,0]
+  CRUSH rule 2 x 259 [4,6]
+  CRUSH rule 2 x 260 [3,6]
+  CRUSH rule 2 x 261 [8,5]
+  CRUSH rule 2 x 262 [5,6]
+  CRUSH rule 2 x 263 [6,1]
+  CRUSH rule 2 x 264 [3,6]
+  CRUSH rule 2 x 265 [8,5]
+  CRUSH rule 2 x 266 [8,2]
+  CRUSH rule 2 x 267 [2,3]
+  CRUSH rule 2 x 268 [0,7]
+  CRUSH rule 2 x 269 [0,8]
+  CRUSH rule 2 x 270 [5,0]
+  CRUSH rule 2 x 271 [7,5]
+  CRUSH rule 2 x 272 [2,8]
+  CRUSH rule 2 x 273 [3,1]
+  CRUSH rule 2 x 274 [6,3]
+  CRUSH rule 2 x 275 [4,7]
+  CRUSH rule 2 x 276 [7,1]
+  CRUSH rule 2 x 277 [6,4]
+  CRUSH rule 2 x 278 [6,1]
+  CRUSH rule 2 x 279 [8,3]
+  CRUSH rule 2 x 280 [0,6]
+  CRUSH rule 2 x 281 [8,0]
+  CRUSH rule 2 x 282 [3,1]
+  CRUSH rule 2 x 283 [8,2]
+  CRUSH rule 2 x 284 [6,3]
+  CRUSH rule 2 x 285 [5,7]
+  CRUSH rule 2 x 286 [2,6]
+  CRUSH rule 2 x 287 [0,4]
+  CRUSH rule 2 x 288 [8,0]
+  CRUSH rule 2 x 289 [4,6]
+  CRUSH rule 2 x 290 [1,3]
+  CRUSH rule 2 x 291 [0,3]
+  CRUSH rule 2 x 292 [8,0]
+  CRUSH rule 2 x 293 [6,0]
+  CRUSH rule 2 x 294 [7,4]
+  CRUSH rule 2 x 295 [4,8]
+  CRUSH rule 2 x 296 [3,1]
+  CRUSH rule 2 x 297 [6,2]
+  CRUSH rule 2 x 298 [1,5]
+  CRUSH rule 2 x 299 [2,8]
+  CRUSH rule 2 x 300 [8,3]
+  CRUSH rule 2 x 301 [0,8]
+  CRUSH rule 2 x 302 [3,0]
+  CRUSH rule 2 x 303 [7,5]
+  CRUSH rule 2 x 304 [2,7]
+  CRUSH rule 2 x 305 [5,8]
+  CRUSH rule 2 x 306 [0,7]
+  CRUSH rule 2 x 307 [0,7]
+  CRUSH rule 2 x 308 [0,8]
+  CRUSH rule 2 x 309 [7,4]
+  CRUSH rule 2 x 310 [4,1]
+  CRUSH rule 2 x 311 [3,6]
+  CRUSH rule 2 x 312 [2,6]
+  CRUSH rule 2 x 313 [5,1]
+  CRUSH rule 2 x 314 [4,2]
+  CRUSH rule 2 x 315 [2,4]
+  CRUSH rule 2 x 316 [6,3]
+  CRUSH rule 2 x 317 [2,6]
+  CRUSH rule 2 x 318 [8,1]
+  CRUSH rule 2 x 319 [5,0]
+  CRUSH rule 2 x 320 [3,7]
+  CRUSH rule 2 x 321 [1,3]
+  CRUSH rule 2 x 322 [2,7]
+  CRUSH rule 2 x 323 [4,7]
+  CRUSH rule 2 x 324 [7,0]
+  CRUSH rule 2 x 325 [4,6]
+  CRUSH rule 2 x 326 [3,2]
+  CRUSH rule 2 x 327 [0,6]
+  CRUSH rule 2 x 328 [7,4]
+  CRUSH rule 2 x 329 [5,6]
+  CRUSH rule 2 x 330 [3,7]
+  CRUSH rule 2 x 331 [2,6]
+  CRUSH rule 2 x 332 [2,4]
+  CRUSH rule 2 x 333 [6,5]
+  CRUSH rule 2 x 334 [8,3]
+  CRUSH rule 2 x 335 [7,1]
+  CRUSH rule 2 x 336 [4,6]
+  CRUSH rule 2 x 337 [7,2]
+  CRUSH rule 2 x 338 [5,6]
+  CRUSH rule 2 x 339 [7,5]
+  CRUSH rule 2 x 340 [2,8]
+  CRUSH rule 2 x 341 [5,1]
+  CRUSH rule 2 x 342 [0,7]
+  CRUSH rule 2 x 343 [6,3]
+  CRUSH rule 2 x 344 [6,0]
+  CRUSH rule 2 x 345 [4,7]
+  CRUSH rule 2 x 346 [8,0]
+  CRUSH rule 2 x 347 [3,1]
+  CRUSH rule 2 x 348 [8,0]
+  CRUSH rule 2 x 349 [1,6]
+  CRUSH rule 2 x 350 [8,5]
+  CRUSH rule 2 x 351 [3,6]
+  CRUSH rule 2 x 352 [1,8]
+  CRUSH rule 2 x 353 [6,4]
+  CRUSH rule 2 x 354 [0,3]
+  CRUSH rule 2 x 355 [3,8]
+  CRUSH rule 2 x 356 [3,0]
+  CRUSH rule 2 x 357 [6,1]
+  CRUSH rule 2 x 358 [2,8]
+  CRUSH rule 2 x 359 [6,1]
+  CRUSH rule 2 x 360 [5,2]
+  CRUSH rule 2 x 361 [8,4]
+  CRUSH rule 2 x 362 [4,1]
+  CRUSH rule 2 x 363 [4,0]
+  CRUSH rule 2 x 364 [2,5]
+  CRUSH rule 2 x 365 [6,5]
+  CRUSH rule 2 x 366 [7,2]
+  CRUSH rule 2 x 367 [4,0]
+  CRUSH rule 2 x 368 [7,4]
+  CRUSH rule 2 x 369 [3,7]
+  CRUSH rule 2 x 370 [8,2]
+  CRUSH rule 2 x 371 [1,3]
+  CRUSH rule 2 x 372 [3,1]
+  CRUSH rule 2 x 373 [0,6]
+  CRUSH rule 2 x 374 [3,8]
+  CRUSH rule 2 x 375 [6,4]
+  CRUSH rule 2 x 376 [7,1]
+  CRUSH rule 2 x 377 [1,3]
+  CRUSH rule 2 x 378 [0,8]
+  CRUSH rule 2 x 379 [8,5]
+  CRUSH rule 2 x 380 [2,5]
+  CRUSH rule 2 x 381 [0,4]
+  CRUSH rule 2 x 382 [1,5]
+  CRUSH rule 2 x 383 [4,6]
+  CRUSH rule 2 x 384 [7,0]
+  CRUSH rule 2 x 385 [7,4]
+  CRUSH rule 2 x 386 [0,3]
+  CRUSH rule 2 x 387 [1,3]
+  CRUSH rule 2 x 388 [5,0]
+  CRUSH rule 2 x 389 [1,5]
+  CRUSH rule 2 x 390 [5,6]
+  CRUSH rule 2 x 391 [5,6]
+  CRUSH rule 2 x 392 [1,8]
+  CRUSH rule 2 x 393 [4,2]
+  CRUSH rule 2 x 394 [4,7]
+  CRUSH rule 2 x 395 [4,0]
+  CRUSH rule 2 x 396 [4,2]
+  CRUSH rule 2 x 397 [2,4]
+  CRUSH rule 2 x 398 [2,4]
+  CRUSH rule 2 x 399 [8,4]
+  CRUSH rule 2 x 400 [8,1]
+  CRUSH rule 2 x 401 [0,5]
+  CRUSH rule 2 x 402 [7,5]
+  CRUSH rule 2 x 403 [0,3]
+  CRUSH rule 2 x 404 [4,2]
+  CRUSH rule 2 x 405 [6,5]
+  CRUSH rule 2 x 406 [2,6]
+  CRUSH rule 2 x 407 [2,8]
+  CRUSH rule 2 x 408 [4,1]
+  CRUSH rule 2 x 409 [7,3]
+  CRUSH rule 2 x 410 [8,3]
+  CRUSH rule 2 x 411 [2,8]
+  CRUSH rule 2 x 412 [0,5]
+  CRUSH rule 2 x 413 [5,0]
+  CRUSH rule 2 x 414 [4,1]
+  CRUSH rule 2 x 415 [0,6]
+  CRUSH rule 2 x 416 [2,6]
+  CRUSH rule 2 x 417 [8,2]
+  CRUSH rule 2 x 418 [7,1]
+  CRUSH rule 2 x 419 [8,3]
+  CRUSH rule 2 x 420 [1,4]
+  CRUSH rule 2 x 421 [8,4]
+  CRUSH rule 2 x 422 [6,4]
+  CRUSH rule 2 x 423 [0,5]
+  CRUSH rule 2 x 424 [8,4]
+  CRUSH rule 2 x 425 [1,3]
+  CRUSH rule 2 x 426 [6,0]
+  CRUSH rule 2 x 427 [0,7]
+  CRUSH rule 2 x 428 [5,7]
+  CRUSH rule 2 x 429 [4,6]
+  CRUSH rule 2 x 430 [3,6]
+  CRUSH rule 2 x 431 [5,0]
+  CRUSH rule 2 x 432 [7,1]
+  CRUSH rule 2 x 433 [6,5]
+  CRUSH rule 2 x 434 [5,2]
+  CRUSH rule 2 x 435 [0,5]
+  CRUSH rule 2 x 436 [4,0]
+  CRUSH rule 2 x 437 [7,5]
+  CRUSH rule 2 x 438 [0,3]
+  CRUSH rule 2 x 439 [1,3]
+  CRUSH rule 2 x 440 [2,7]
+  CRUSH rule 2 x 441 [5,7]
+  CRUSH rule 2 x 442 [2,4]
+  CRUSH rule 2 x 443 [6,0]
+  CRUSH rule 2 x 444 [7,0]
+  CRUSH rule 2 x 445 [6,3]
+  CRUSH rule 2 x 446 [4,1]
+  CRUSH rule 2 x 447 [2,3]
+  CRUSH rule 2 x 448 [7,2]
+  CRUSH rule 2 x 449 [7,5]
+  CRUSH rule 2 x 450 [4,1]
+  CRUSH rule 2 x 451 [6,5]
+  CRUSH rule 2 x 452 [8,3]
+  CRUSH rule 2 x 453 [6,5]
+  CRUSH rule 2 x 454 [6,4]
+  CRUSH rule 2 x 455 [2,7]
+  CRUSH rule 2 x 456 [6,2]
+  CRUSH rule 2 x 457 [7,2]
+  CRUSH rule 2 x 458 [2,8]
+  CRUSH rule 2 x 459 [2,7]
+  CRUSH rule 2 x 460 [6,5]
+  CRUSH rule 2 x 461 [6,5]
+  CRUSH rule 2 x 462 [8,1]
+  CRUSH rule 2 x 463 [6,0]
+  CRUSH rule 2 x 464 [7,4]
+  CRUSH rule 2 x 465 [7,2]
+  CRUSH rule 2 x 466 [5,8]
+  CRUSH rule 2 x 467 [6,4]
+  CRUSH rule 2 x 468 [7,0]
+  CRUSH rule 2 x 469 [7,0]
+  CRUSH rule 2 x 470 [3,0]
+  CRUSH rule 2 x 471 [0,7]
+  CRUSH rule 2 x 472 [5,1]
+  CRUSH rule 2 x 473 [1,4]
+  CRUSH rule 2 x 474 [6,0]
+  CRUSH rule 2 x 475 [6,2]
+  CRUSH rule 2 x 476 [4,6]
+  CRUSH rule 2 x 477 [5,8]
+  CRUSH rule 2 x 478 [6,2]
+  CRUSH rule 2 x 479 [0,5]
+  CRUSH rule 2 x 480 [1,8]
+  CRUSH rule 2 x 481 [2,4]
+  CRUSH rule 2 x 482 [4,7]
+  CRUSH rule 2 x 483 [0,6]
+  CRUSH rule 2 x 484 [1,7]
+  CRUSH rule 2 x 485 [4,7]
+  CRUSH rule 2 x 486 [4,1]
+  CRUSH rule 2 x 487 [5,0]
+  CRUSH rule 2 x 488 [5,7]
+  CRUSH rule 2 x 489 [2,8]
+  CRUSH rule 2 x 490 [6,4]
+  CRUSH rule 2 x 491 [1,7]
+  CRUSH rule 2 x 492 [6,5]
+  CRUSH rule 2 x 493 [0,7]
+  CRUSH rule 2 x 494 [1,7]
+  CRUSH rule 2 x 495 [3,1]
+  CRUSH rule 2 x 496 [7,5]
+  CRUSH rule 2 x 497 [5,7]
+  CRUSH rule 2 x 498 [0,5]
+  CRUSH rule 2 x 499 [8,4]
+  CRUSH rule 2 x 500 [3,6]
+  CRUSH rule 2 x 501 [0,7]
+  CRUSH rule 2 x 502 [7,1]
+  CRUSH rule 2 x 503 [2,3]
+  CRUSH rule 2 x 504 [5,6]
+  CRUSH rule 2 x 505 [0,7]
+  CRUSH rule 2 x 506 [5,2]
+  CRUSH rule 2 x 507 [6,0]
+  CRUSH rule 2 x 508 [0,3]
+  CRUSH rule 2 x 509 [7,5]
+  CRUSH rule 2 x 510 [6,0]
+  CRUSH rule 2 x 511 [5,8]
+  CRUSH rule 2 x 512 [7,0]
+  CRUSH rule 2 x 513 [7,2]
+  CRUSH rule 2 x 514 [4,6]
+  CRUSH rule 2 x 515 [8,5]
+  CRUSH rule 2 x 516 [4,0]
+  CRUSH rule 2 x 517 [7,2]
+  CRUSH rule 2 x 518 [4,6]
+  CRUSH rule 2 x 519 [7,3]
+  CRUSH rule 2 x 520 [2,6]
+  CRUSH rule 2 x 521 [8,0]
+  CRUSH rule 2 x 522 [6,0]
+  CRUSH rule 2 x 523 [4,2]
+  CRUSH rule 2 x 524 [0,4]
+  CRUSH rule 2 x 525 [0,4]
+  CRUSH rule 2 x 526 [1,5]
+  CRUSH rule 2 x 527 [0,5]
+  CRUSH rule 2 x 528 [5,0]
+  CRUSH rule 2 x 529 [5,7]
+  CRUSH rule 2 x 530 [6,5]
+  CRUSH rule 2 x 531 [6,1]
+  CRUSH rule 2 x 532 [6,3]
+  CRUSH rule 2 x 533 [5,6]
+  CRUSH rule 2 x 534 [7,3]
+  CRUSH rule 2 x 535 [8,1]
+  CRUSH rule 2 x 536 [6,2]
+  CRUSH rule 2 x 537 [3,7]
+  CRUSH rule 2 x 538 [6,3]
+  CRUSH rule 2 x 539 [8,3]
+  CRUSH rule 2 x 540 [0,6]
+  CRUSH rule 2 x 541 [2,3]
+  CRUSH rule 2 x 542 [3,2]
+  CRUSH rule 2 x 543 [6,0]
+  CRUSH rule 2 x 544 [3,7]
+  CRUSH rule 2 x 545 [5,7]
+  CRUSH rule 2 x 546 [6,1]
+  CRUSH rule 2 x 547 [8,2]
+  CRUSH rule 2 x 548 [5,2]
+  CRUSH rule 2 x 549 [5,8]
+  CRUSH rule 2 x 550 [0,5]
+  CRUSH rule 2 x 551 [7,5]
+  CRUSH rule 2 x 552 [5,8]
+  CRUSH rule 2 x 553 [4,2]
+  CRUSH rule 2 x 554 [0,8]
+  CRUSH rule 2 x 555 [5,0]
+  CRUSH rule 2 x 556 [3,6]
+  CRUSH rule 2 x 557 [7,4]
+  CRUSH rule 2 x 558 [3,1]
+  CRUSH rule 2 x 559 [4,2]
+  CRUSH rule 2 x 560 [8,3]
+  CRUSH rule 2 x 561 [6,3]
+  CRUSH rule 2 x 562 [3,0]
+  CRUSH rule 2 x 563 [2,6]
+  CRUSH rule 2 x 564 [5,1]
+  CRUSH rule 2 x 565 [3,6]
+  CRUSH rule 2 x 566 [4,7]
+  CRUSH rule 2 x 567 [3,6]
+  CRUSH rule 2 x 568 [7,4]
+  CRUSH rule 2 x 569 [3,1]
+  CRUSH rule 2 x 570 [1,5]
+  CRUSH rule 2 x 571 [3,7]
+  CRUSH rule 2 x 572 [3,2]
+  CRUSH rule 2 x 573 [3,0]
+  CRUSH rule 2 x 574 [2,5]
+  CRUSH rule 2 x 575 [8,2]
+  CRUSH rule 2 x 576 [4,6]
+  CRUSH rule 2 x 577 [8,2]
+  CRUSH rule 2 x 578 [6,1]
+  CRUSH rule 2 x 579 [3,1]
+  CRUSH rule 2 x 580 [3,0]
+  CRUSH rule 2 x 581 [7,2]
+  CRUSH rule 2 x 582 [2,8]
+  CRUSH rule 2 x 583 [6,0]
+  CRUSH rule 2 x 584 [8,1]
+  CRUSH rule 2 x 585 [7,0]
+  CRUSH rule 2 x 586 [0,7]
+  CRUSH rule 2 x 587 [2,5]
+  CRUSH rule 2 x 588 [3,7]
+  CRUSH rule 2 x 589 [7,1]
+  CRUSH rule 2 x 590 [6,2]
+  CRUSH rule 2 x 591 [5,2]
+  CRUSH rule 2 x 592 [2,4]
+  CRUSH rule 2 x 593 [0,8]
+  CRUSH rule 2 x 594 [0,7]
+  CRUSH rule 2 x 595 [7,1]
+  CRUSH rule 2 x 596 [4,0]
+  CRUSH rule 2 x 597 [3,1]
+  CRUSH rule 2 x 598 [3,2]
+  CRUSH rule 2 x 599 [5,2]
+  CRUSH rule 2 x 600 [7,0]
+  CRUSH rule 2 x 601 [0,7]
+  CRUSH rule 2 x 602 [3,7]
+  CRUSH rule 2 x 603 [5,1]
+  CRUSH rule 2 x 604 [7,5]
+  CRUSH rule 2 x 605 [3,0]
+  CRUSH rule 2 x 606 [2,7]
+  CRUSH rule 2 x 607 [0,4]
+  CRUSH rule 2 x 608 [5,2]
+  CRUSH rule 2 x 609 [5,2]
+  CRUSH rule 2 x 610 [3,7]
+  CRUSH rule 2 x 611 [1,8]
+  CRUSH rule 2 x 612 [2,6]
+  CRUSH rule 2 x 613 [7,2]
+  CRUSH rule 2 x 614 [7,2]
+  CRUSH rule 2 x 615 [6,0]
+  CRUSH rule 2 x 616 [0,8]
+  CRUSH rule 2 x 617 [6,1]
+  CRUSH rule 2 x 618 [7,4]
+  CRUSH rule 2 x 619 [5,1]
+  CRUSH rule 2 x 620 [4,1]
+  CRUSH rule 2 x 621 [5,8]
+  CRUSH rule 2 x 622 [0,4]
+  CRUSH rule 2 x 623 [0,6]
+  CRUSH rule 2 x 624 [3,2]
+  CRUSH rule 2 x 625 [2,3]
+  CRUSH rule 2 x 626 [7,0]
+  CRUSH rule 2 x 627 [2,7]
+  CRUSH rule 2 x 628 [8,0]
+  CRUSH rule 2 x 629 [2,6]
+  CRUSH rule 2 x 630 [2,7]
+  CRUSH rule 2 x 631 [0,7]
+  CRUSH rule 2 x 632 [7,0]
+  CRUSH rule 2 x 633 [8,3]
+  CRUSH rule 2 x 634 [0,4]
+  CRUSH rule 2 x 635 [5,6]
+  CRUSH rule 2 x 636 [1,4]
+  CRUSH rule 2 x 637 [4,1]
+  CRUSH rule 2 x 638 [6,0]
+  CRUSH rule 2 x 639 [4,2]
+  CRUSH rule 2 x 640 [3,1]
+  CRUSH rule 2 x 641 [7,2]
+  CRUSH rule 2 x 642 [2,7]
+  CRUSH rule 2 x 643 [3,0]
+  CRUSH rule 2 x 644 [8,1]
+  CRUSH rule 2 x 645 [5,7]
+  CRUSH rule 2 x 646 [8,0]
+  CRUSH rule 2 x 647 [7,1]
+  CRUSH rule 2 x 648 [0,6]
+  CRUSH rule 2 x 649 [4,7]
+  CRUSH rule 2 x 650 [7,3]
+  CRUSH rule 2 x 651 [3,7]
+  CRUSH rule 2 x 652 [3,7]
+  CRUSH rule 2 x 653 [8,5]
+  CRUSH rule 2 x 654 [7,5]
+  CRUSH rule 2 x 655 [0,3]
+  CRUSH rule 2 x 656 [4,7]
+  CRUSH rule 2 x 657 [6,1]
+  CRUSH rule 2 x 658 [5,6]
+  CRUSH rule 2 x 659 [4,6]
+  CRUSH rule 2 x 660 [7,4]
+  CRUSH rule 2 x 661 [1,8]
+  CRUSH rule 2 x 662 [4,2]
+  CRUSH rule 2 x 663 [1,3]
+  CRUSH rule 2 x 664 [1,4]
+  CRUSH rule 2 x 665 [5,7]
+  CRUSH rule 2 x 666 [2,8]
+  CRUSH rule 2 x 667 [1,3]
+  CRUSH rule 2 x 668 [3,7]
+  CRUSH rule 2 x 669 [6,4]
+  CRUSH rule 2 x 670 [4,0]
+  CRUSH rule 2 x 671 [0,7]
+  CRUSH rule 2 x 672 [4,2]
+  CRUSH rule 2 x 673 [5,2]
+  CRUSH rule 2 x 674 [3,1]
+  CRUSH rule 2 x 675 [0,8]
+  CRUSH rule 2 x 676 [0,4]
+  CRUSH rule 2 x 677 [4,1]
+  CRUSH rule 2 x 678 [2,3]
+  CRUSH rule 2 x 679 [6,0]
+  CRUSH rule 2 x 680 [0,4]
+  CRUSH rule 2 x 681 [4,7]
+  CRUSH rule 2 x 682 [0,5]
+  CRUSH rule 2 x 683 [0,5]
+  CRUSH rule 2 x 684 [7,1]
+  CRUSH rule 2 x 685 [7,1]
+  CRUSH rule 2 x 686 [1,4]
+  CRUSH rule 2 x 687 [3,6]
+  CRUSH rule 2 x 688 [5,7]
+  CRUSH rule 2 x 689 [6,5]
+  CRUSH rule 2 x 690 [8,1]
+  CRUSH rule 2 x 691 [3,0]
+  CRUSH rule 2 x 692 [7,2]
+  CRUSH rule 2 x 693 [6,3]
+  CRUSH rule 2 x 694 [6,5]
+  CRUSH rule 2 x 695 [0,8]
+  CRUSH rule 2 x 696 [1,4]
+  CRUSH rule 2 x 697 [6,1]
+  CRUSH rule 2 x 698 [6,2]
+  CRUSH rule 2 x 699 [1,6]
+  CRUSH rule 2 x 700 [0,3]
+  CRUSH rule 2 x 701 [4,1]
+  CRUSH rule 2 x 702 [3,2]
+  CRUSH rule 2 x 703 [8,3]
+  CRUSH rule 2 x 704 [0,3]
+  CRUSH rule 2 x 705 [8,0]
+  CRUSH rule 2 x 706 [1,5]
+  CRUSH rule 2 x 707 [7,3]
+  CRUSH rule 2 x 708 [3,7]
+  CRUSH rule 2 x 709 [6,3]
+  CRUSH rule 2 x 710 [8,4]
+  CRUSH rule 2 x 711 [2,3]
+  CRUSH rule 2 x 712 [2,3]
+  CRUSH rule 2 x 713 [6,3]
+  CRUSH rule 2 x 714 [3,2]
+  CRUSH rule 2 x 715 [1,3]
+  CRUSH rule 2 x 716 [3,6]
+  CRUSH rule 2 x 717 [8,2]
+  CRUSH rule 2 x 718 [3,7]
+  CRUSH rule 2 x 719 [2,6]
+  CRUSH rule 2 x 720 [6,1]
+  CRUSH rule 2 x 721 [5,7]
+  CRUSH rule 2 x 722 [5,7]
+  CRUSH rule 2 x 723 [5,1]
+  CRUSH rule 2 x 724 [0,6]
+  CRUSH rule 2 x 725 [0,3]
+  CRUSH rule 2 x 726 [3,8]
+  CRUSH rule 2 x 727 [4,6]
+  CRUSH rule 2 x 728 [2,7]
+  CRUSH rule 2 x 729 [5,6]
+  CRUSH rule 2 x 730 [3,7]
+  CRUSH rule 2 x 731 [4,1]
+  CRUSH rule 2 x 732 [1,5]
+  CRUSH rule 2 x 733 [5,7]
+  CRUSH rule 2 x 734 [6,4]
+  CRUSH rule 2 x 735 [4,8]
+  CRUSH rule 2 x 736 [3,8]
+  CRUSH rule 2 x 737 [1,6]
+  CRUSH rule 2 x 738 [5,2]
+  CRUSH rule 2 x 739 [0,7]
+  CRUSH rule 2 x 740 [0,8]
+  CRUSH rule 2 x 741 [7,1]
+  CRUSH rule 2 x 742 [8,2]
+  CRUSH rule 2 x 743 [7,0]
+  CRUSH rule 2 x 744 [4,7]
+  CRUSH rule 2 x 745 [3,1]
+  CRUSH rule 2 x 746 [4,1]
+  CRUSH rule 2 x 747 [6,0]
+  CRUSH rule 2 x 748 [2,7]
+  CRUSH rule 2 x 749 [4,8]
+  CRUSH rule 2 x 750 [1,6]
+  CRUSH rule 2 x 751 [2,8]
+  CRUSH rule 2 x 752 [8,1]
+  CRUSH rule 2 x 753 [7,3]
+  CRUSH rule 2 x 754 [8,5]
+  CRUSH rule 2 x 755 [1,6]
+  CRUSH rule 2 x 756 [5,6]
+  CRUSH rule 2 x 757 [8,0]
+  CRUSH rule 2 x 758 [6,0]
+  CRUSH rule 2 x 759 [8,5]
+  CRUSH rule 2 x 760 [1,5]
+  CRUSH rule 2 x 761 [4,1]
+  CRUSH rule 2 x 762 [2,7]
+  CRUSH rule 2 x 763 [8,5]
+  CRUSH rule 2 x 764 [1,7]
+  CRUSH rule 2 x 765 [6,5]
+  CRUSH rule 2 x 766 [8,5]
+  CRUSH rule 2 x 767 [1,8]
+  CRUSH rule 2 x 768 [8,3]
+  CRUSH rule 2 x 769 [6,2]
+  CRUSH rule 2 x 770 [6,0]
+  CRUSH rule 2 x 771 [7,0]
+  CRUSH rule 2 x 772 [8,3]
+  CRUSH rule 2 x 773 [3,1]
+  CRUSH rule 2 x 774 [4,6]
+  CRUSH rule 2 x 775 [6,4]
+  CRUSH rule 2 x 776 [7,2]
+  CRUSH rule 2 x 777 [3,1]
+  CRUSH rule 2 x 778 [1,8]
+  CRUSH rule 2 x 779 [2,7]
+  CRUSH rule 2 x 780 [0,5]
+  CRUSH rule 2 x 781 [6,3]
+  CRUSH rule 2 x 782 [5,0]
+  CRUSH rule 2 x 783 [7,1]
+  CRUSH rule 2 x 784 [0,4]
+  CRUSH rule 2 x 785 [6,1]
+  CRUSH rule 2 x 786 [7,3]
+  CRUSH rule 2 x 787 [1,6]
+  CRUSH rule 2 x 788 [6,0]
+  CRUSH rule 2 x 789 [0,4]
+  CRUSH rule 2 x 790 [8,4]
+  CRUSH rule 2 x 791 [3,8]
+  CRUSH rule 2 x 792 [5,8]
+  CRUSH rule 2 x 793 [6,1]
+  CRUSH rule 2 x 794 [2,6]
+  CRUSH rule 2 x 795 [0,3]
+  CRUSH rule 2 x 796 [3,7]
+  CRUSH rule 2 x 797 [2,3]
+  CRUSH rule 2 x 798 [6,1]
+  CRUSH rule 2 x 799 [5,1]
+  CRUSH rule 2 x 800 [5,0]
+  CRUSH rule 2 x 801 [3,6]
+  CRUSH rule 2 x 802 [1,8]
+  CRUSH rule 2 x 803 [0,5]
+  CRUSH rule 2 x 804 [6,2]
+  CRUSH rule 2 x 805 [3,6]
+  CRUSH rule 2 x 806 [1,3]
+  CRUSH rule 2 x 807 [5,7]
+  CRUSH rule 2 x 808 [4,6]
+  CRUSH rule 2 x 809 [1,4]
+  CRUSH rule 2 x 810 [5,7]
+  CRUSH rule 2 x 811 [8,4]
+  CRUSH rule 2 x 812 [8,5]
+  CRUSH rule 2 x 813 [6,4]
+  CRUSH rule 2 x 814 [3,6]
+  CRUSH rule 2 x 815 [3,1]
+  CRUSH rule 2 x 816 [2,7]
+  CRUSH rule 2 x 817 [4,8]
+  CRUSH rule 2 x 818 [3,0]
+  CRUSH rule 2 x 819 [5,1]
+  CRUSH rule 2 x 820 [3,6]
+  CRUSH rule 2 x 821 [4,6]
+  CRUSH rule 2 x 822 [2,5]
+  CRUSH rule 2 x 823 [4,8]
+  CRUSH rule 2 x 824 [3,7]
+  CRUSH rule 2 x 825 [2,8]
+  CRUSH rule 2 x 826 [7,0]
+  CRUSH rule 2 x 827 [0,8]
+  CRUSH rule 2 x 828 [2,3]
+  CRUSH rule 2 x 829 [5,6]
+  CRUSH rule 2 x 830 [2,3]
+  CRUSH rule 2 x 831 [1,6]
+  CRUSH rule 2 x 832 [4,7]
+  CRUSH rule 2 x 833 [2,7]
+  CRUSH rule 2 x 834 [3,1]
+  CRUSH rule 2 x 835 [8,4]
+  CRUSH rule 2 x 836 [3,7]
+  CRUSH rule 2 x 837 [6,3]
+  CRUSH rule 2 x 838 [6,2]
+  CRUSH rule 2 x 839 [5,0]
+  CRUSH rule 2 x 840 [7,3]
+  CRUSH rule 2 x 841 [4,8]
+  CRUSH rule 2 x 842 [2,4]
+  CRUSH rule 2 x 843 [6,4]
+  CRUSH rule 2 x 844 [4,8]
+  CRUSH rule 2 x 845 [3,8]
+  CRUSH rule 2 x 846 [3,2]
+  CRUSH rule 2 x 847 [0,8]
+  CRUSH rule 2 x 848 [2,6]
+  CRUSH rule 2 x 849 [4,6]
+  CRUSH rule 2 x 850 [1,3]
+  CRUSH rule 2 x 851 [6,4]
+  CRUSH rule 2 x 852 [7,3]
+  CRUSH rule 2 x 853 [6,0]
+  CRUSH rule 2 x 854 [7,0]
+  CRUSH rule 2 x 855 [5,7]
+  CRUSH rule 2 x 856 [6,3]
+  CRUSH rule 2 x 857 [8,5]
+  CRUSH rule 2 x 858 [6,4]
+  CRUSH rule 2 x 859 [6,0]
+  CRUSH rule 2 x 860 [4,1]
+  CRUSH rule 2 x 861 [8,3]
+  CRUSH rule 2 x 862 [6,1]
+  CRUSH rule 2 x 863 [8,2]
+  CRUSH rule 2 x 864 [5,6]
+  CRUSH rule 2 x 865 [8,1]
+  CRUSH rule 2 x 866 [3,6]
+  CRUSH rule 2 x 867 [6,5]
+  CRUSH rule 2 x 868 [6,3]
+  CRUSH rule 2 x 869 [8,5]
+  CRUSH rule 2 x 870 [0,4]
+  CRUSH rule 2 x 871 [3,2]
+  CRUSH rule 2 x 872 [5,1]
+  CRUSH rule 2 x 873 [4,6]
+  CRUSH rule 2 x 874 [2,6]
+  CRUSH rule 2 x 875 [2,6]
+  CRUSH rule 2 x 876 [5,8]
+  CRUSH rule 2 x 877 [6,4]
+  CRUSH rule 2 x 878 [5,2]
+  CRUSH rule 2 x 879 [7,4]
+  CRUSH rule 2 x 880 [3,2]
+  CRUSH rule 2 x 881 [5,6]
+  CRUSH rule 2 x 882 [4,0]
+  CRUSH rule 2 x 883 [2,3]
+  CRUSH rule 2 x 884 [6,0]
+  CRUSH rule 2 x 885 [5,1]
+  CRUSH rule 2 x 886 [3,6]
+  CRUSH rule 2 x 887 [7,4]
+  CRUSH rule 2 x 888 [6,2]
+  CRUSH rule 2 x 889 [2,6]
+  CRUSH rule 2 x 890 [7,2]
+  CRUSH rule 2 x 891 [1,8]
+  CRUSH rule 2 x 892 [6,2]
+  CRUSH rule 2 x 893 [2,3]
+  CRUSH rule 2 x 894 [7,5]
+  CRUSH rule 2 x 895 [5,1]
+  CRUSH rule 2 x 896 [1,8]
+  CRUSH rule 2 x 897 [4,2]
+  CRUSH rule 2 x 898 [0,5]
+  CRUSH rule 2 x 899 [1,7]
+  CRUSH rule 2 x 900 [4,1]
+  CRUSH rule 2 x 901 [5,0]
+  CRUSH rule 2 x 902 [8,5]
+  CRUSH rule 2 x 903 [5,7]
+  CRUSH rule 2 x 904 [5,6]
+  CRUSH rule 2 x 905 [6,2]
+  CRUSH rule 2 x 906 [1,6]
+  CRUSH rule 2 x 907 [7,1]
+  CRUSH rule 2 x 908 [5,8]
+  CRUSH rule 2 x 909 [2,3]
+  CRUSH rule 2 x 910 [6,4]
+  CRUSH rule 2 x 911 [5,8]
+  CRUSH rule 2 x 912 [0,7]
+  CRUSH rule 2 x 913 [7,2]
+  CRUSH rule 2 x 914 [6,4]
+  CRUSH rule 2 x 915 [8,2]
+  CRUSH rule 2 x 916 [3,1]
+  CRUSH rule 2 x 917 [1,5]
+  CRUSH rule 2 x 918 [8,2]
+  CRUSH rule 2 x 919 [6,2]
+  CRUSH rule 2 x 920 [7,4]
+  CRUSH rule 2 x 921 [1,4]
+  CRUSH rule 2 x 922 [6,4]
+  CRUSH rule 2 x 923 [5,8]
+  CRUSH rule 2 x 924 [3,1]
+  CRUSH rule 2 x 925 [5,7]
+  CRUSH rule 2 x 926 [3,0]
+  CRUSH rule 2 x 927 [1,6]
+  CRUSH rule 2 x 928 [8,1]
+  CRUSH rule 2 x 929 [4,1]
+  CRUSH rule 2 x 930 [2,4]
+  CRUSH rule 2 x 931 [5,0]
+  CRUSH rule 2 x 932 [4,1]
+  CRUSH rule 2 x 933 [8,5]
+  CRUSH rule 2 x 934 [5,6]
+  CRUSH rule 2 x 935 [6,3]
+  CRUSH rule 2 x 936 [0,6]
+  CRUSH rule 2 x 937 [5,8]
+  CRUSH rule 2 x 938 [6,5]
+  CRUSH rule 2 x 939 [2,7]
+  CRUSH rule 2 x 940 [8,5]
+  CRUSH rule 2 x 941 [5,2]
+  CRUSH rule 2 x 942 [1,8]
+  CRUSH rule 2 x 943 [8,2]
+  CRUSH rule 2 x 944 [4,8]
+  CRUSH rule 2 x 945 [7,2]
+  CRUSH rule 2 x 946 [2,8]
+  CRUSH rule 2 x 947 [4,2]
+  CRUSH rule 2 x 948 [7,5]
+  CRUSH rule 2 x 949 [6,1]
+  CRUSH rule 2 x 950 [3,6]
+  CRUSH rule 2 x 951 [4,8]
+  CRUSH rule 2 x 952 [2,7]
+  CRUSH rule 2 x 953 [1,3]
+  CRUSH rule 2 x 954 [4,2]
+  CRUSH rule 2 x 955 [8,0]
+  CRUSH rule 2 x 956 [1,6]
+  CRUSH rule 2 x 957 [7,1]
+  CRUSH rule 2 x 958 [8,4]
+  CRUSH rule 2 x 959 [5,2]
+  CRUSH rule 2 x 960 [3,6]
+  CRUSH rule 2 x 961 [4,0]
+  CRUSH rule 2 x 962 [7,4]
+  CRUSH rule 2 x 963 [0,5]
+  CRUSH rule 2 x 964 [3,1]
+  CRUSH rule 2 x 965 [7,4]
+  CRUSH rule 2 x 966 [3,8]
+  CRUSH rule 2 x 967 [8,5]
+  CRUSH rule 2 x 968 [7,2]
+  CRUSH rule 2 x 969 [8,0]
+  CRUSH rule 2 x 970 [0,6]
+  CRUSH rule 2 x 971 [1,7]
+  CRUSH rule 2 x 972 [1,8]
+  CRUSH rule 2 x 973 [1,6]
+  CRUSH rule 2 x 974 [5,1]
+  CRUSH rule 2 x 975 [3,7]
+  CRUSH rule 2 x 976 [4,8]
+  CRUSH rule 2 x 977 [8,3]
+  CRUSH rule 2 x 978 [7,2]
+  CRUSH rule 2 x 979 [7,1]
+  CRUSH rule 2 x 980 [6,0]
+  CRUSH rule 2 x 981 [7,3]
+  CRUSH rule 2 x 982 [4,2]
+  CRUSH rule 2 x 983 [3,7]
+  CRUSH rule 2 x 984 [0,7]
+  CRUSH rule 2 x 985 [2,5]
+  CRUSH rule 2 x 986 [8,3]
+  CRUSH rule 2 x 987 [0,5]
+  CRUSH rule 2 x 988 [1,3]
+  CRUSH rule 2 x 989 [0,6]
+  CRUSH rule 2 x 990 [1,6]
+  CRUSH rule 2 x 991 [0,4]
+  CRUSH rule 2 x 992 [7,1]
+  CRUSH rule 2 x 993 [0,6]
+  CRUSH rule 2 x 994 [3,7]
+  CRUSH rule 2 x 995 [7,1]
+  CRUSH rule 2 x 996 [6,5]
+  CRUSH rule 2 x 997 [6,4]
+  CRUSH rule 2 x 998 [8,1]
+  CRUSH rule 2 x 999 [0,7]
+  CRUSH rule 2 x 1000 [8,5]
+  CRUSH rule 2 x 1001 [2,5]
+  CRUSH rule 2 x 1002 [1,3]
+  CRUSH rule 2 x 1003 [2,8]
+  CRUSH rule 2 x 1004 [6,1]
+  CRUSH rule 2 x 1005 [6,1]
+  CRUSH rule 2 x 1006 [1,6]
+  CRUSH rule 2 x 1007 [1,5]
+  CRUSH rule 2 x 1008 [1,7]
+  CRUSH rule 2 x 1009 [6,4]
+  CRUSH rule 2 x 1010 [3,1]
+  CRUSH rule 2 x 1011 [3,0]
+  CRUSH rule 2 x 1012 [3,0]
+  CRUSH rule 2 x 1013 [5,1]
+  CRUSH rule 2 x 1014 [2,8]
+  CRUSH rule 2 x 1015 [6,5]
+  CRUSH rule 2 x 1016 [2,4]
+  CRUSH rule 2 x 1017 [6,0]
+  CRUSH rule 2 x 1018 [5,0]
+  CRUSH rule 2 x 1019 [5,8]
+  CRUSH rule 2 x 1020 [5,1]
+  CRUSH rule 2 x 1021 [5,2]
+  CRUSH rule 2 x 1022 [1,6]
+  CRUSH rule 2 x 1023 [3,2]
+  rule 2 (chooseleaf) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 2 x 0 [0,5,7]
+  CRUSH rule 2 x 1 [0,8,5]
+  CRUSH rule 2 x 2 [1,3,7]
+  CRUSH rule 2 x 3 [8,0,4]
+  CRUSH rule 2 x 4 [5,0,7]
+  CRUSH rule 2 x 5 [7,0,4]
+  CRUSH rule 2 x 6 [2,6,3]
+  CRUSH rule 2 x 7 [5,8,2]
+  CRUSH rule 2 x 8 [5,6,0]
+  CRUSH rule 2 x 9 [2,3,8]
+  CRUSH rule 2 x 10 [0,7,4]
+  CRUSH rule 2 x 11 [0,7,4]
+  CRUSH rule 2 x 12 [0,4,6]
+  CRUSH rule 2 x 13 [3,8,1]
+  CRUSH rule 2 x 14 [7,0,5]
+  CRUSH rule 2 x 15 [7,2,4]
+  CRUSH rule 2 x 16 [3,6,0]
+  CRUSH rule 2 x 17 [5,1,6]
+  CRUSH rule 2 x 18 [1,4,6]
+  CRUSH rule 2 x 19 [7,5,0]
+  CRUSH rule 2 x 20 [2,4,7]
+  CRUSH rule 2 x 21 [3,7,2]
+  CRUSH rule 2 x 22 [8,3,1]
+  CRUSH rule 2 x 23 [3,6,2]
+  CRUSH rule 2 x 24 [1,7,3]
+  CRUSH rule 2 x 25 [3,7,0]
+  CRUSH rule 2 x 26 [2,8,4]
+  CRUSH rule 2 x 27 [3,1,8]
+  CRUSH rule 2 x 28 [6,0,3]
+  CRUSH rule 2 x 29 [8,5,2]
+  CRUSH rule 2 x 30 [5,7,0]
+  CRUSH rule 2 x 31 [8,0,4]
+  CRUSH rule 2 x 32 [3,6,2]
+  CRUSH rule 2 x 33 [2,7,5]
+  CRUSH rule 2 x 34 [2,5,7]
+  CRUSH rule 2 x 35 [0,8,5]
+  CRUSH rule 2 x 36 [3,8,2]
+  CRUSH rule 2 x 37 [0,4,6]
+  CRUSH rule 2 x 38 [4,8,2]
+  CRUSH rule 2 x 39 [3,7,0]
+  CRUSH rule 2 x 40 [7,2,3]
+  CRUSH rule 2 x 41 [0,6,4]
+  CRUSH rule 2 x 42 [4,6,2]
+  CRUSH rule 2 x 43 [0,3,7]
+  CRUSH rule 2 x 44 [1,6,5]
+  CRUSH rule 2 x 45 [8,0,4]
+  CRUSH rule 2 x 46 [2,4,8]
+  CRUSH rule 2 x 47 [4,2,8]
+  CRUSH rule 2 x 48 [4,6,2]
+  CRUSH rule 2 x 49 [5,7,2]
+  CRUSH rule 2 x 50 [3,1,7]
+  CRUSH rule 2 x 51 [3,6,0]
+  CRUSH rule 2 x 52 [8,1,4]
+  CRUSH rule 2 x 53 [3,8,0]
+  CRUSH rule 2 x 54 [7,3,1]
+  CRUSH rule 2 x 55 [8,2,4]
+  CRUSH rule 2 x 56 [6,4,2]
+  CRUSH rule 2 x 57 [5,8,1]
+  CRUSH rule 2 x 58 [1,8,5]
+  CRUSH rule 2 x 59 [4,2,7]
+  CRUSH rule 2 x 60 [3,2,6]
+  CRUSH rule 2 x 61 [4,6,0]
+  CRUSH rule 2 x 62 [7,0,5]
+  CRUSH rule 2 x 63 [5,6,0]
+  CRUSH rule 2 x 64 [4,2,8]
+  CRUSH rule 2 x 65 [7,3,0]
+  CRUSH rule 2 x 66 [5,6,2]
+  CRUSH rule 2 x 67 [5,0,8]
+  CRUSH rule 2 x 68 [0,5,8]
+  CRUSH rule 2 x 69 [5,1,6]
+  CRUSH rule 2 x 70 [7,0,5]
+  CRUSH rule 2 x 71 [2,8,5]
+  CRUSH rule 2 x 72 [6,1,5]
+  CRUSH rule 2 x 73 [2,7,3]
+  CRUSH rule 2 x 74 [0,7,3]
+  CRUSH rule 2 x 75 [3,2,6]
+  CRUSH rule 2 x 76 [5,1,7]
+  CRUSH rule 2 x 77 [7,2,5]
+  CRUSH rule 2 x 78 [1,4,8]
+  CRUSH rule 2 x 79 [5,1,7]
+  CRUSH rule 2 x 80 [0,3,6]
+  CRUSH rule 2 x 81 [0,3,6]
+  CRUSH rule 2 x 82 [7,1,5]
+  CRUSH rule 2 x 83 [2,6,3]
+  CRUSH rule 2 x 84 [7,2,4]
+  CRUSH rule 2 x 85 [3,8,0]
+  CRUSH rule 2 x 86 [0,6,3]
+  CRUSH rule 2 x 87 [0,7,4]
+  CRUSH rule 2 x 88 [1,6,5]
+  CRUSH rule 2 x 89 [3,0,7]
+  CRUSH rule 2 x 90 [6,4,1]
+  CRUSH rule 2 x 91 [3,8,1]
+  CRUSH rule 2 x 92 [1,8,4]
+  CRUSH rule 2 x 93 [7,4,2]
+  CRUSH rule 2 x 94 [0,4,8]
+  CRUSH rule 2 x 95 [7,5,1]
+  CRUSH rule 2 x 96 [3,6,1]
+  CRUSH rule 2 x 97 [8,4,0]
+  CRUSH rule 2 x 98 [2,7,5]
+  CRUSH rule 2 x 99 [0,7,3]
+  CRUSH rule 2 x 100 [1,7,4]
+  CRUSH rule 2 x 101 [3,7,2]
+  CRUSH rule 2 x 102 [4,2,7]
+  CRUSH rule 2 x 103 [4,7,0]
+  CRUSH rule 2 x 104 [7,4,1]
+  CRUSH rule 2 x 105 [2,4,6]
+  CRUSH rule 2 x 106 [1,6,5]
+  CRUSH rule 2 x 107 [3,2,6]
+  CRUSH rule 2 x 108 [7,2,5]
+  CRUSH rule 2 x 109 [1,4,6]
+  CRUSH rule 2 x 110 [3,2,7]
+  CRUSH rule 2 x 111 [2,3,6]
+  CRUSH rule 2 x 112 [2,6,4]
+  CRUSH rule 2 x 113 [6,2,4]
+  CRUSH rule 2 x 114 [7,3,1]
+  CRUSH rule 2 x 115 [8,2,3]
+  CRUSH rule 2 x 116 [1,6,3]
+  CRUSH rule 2 x 117 [7,3,2]
+  CRUSH rule 2 x 118 [0,3,8]
+  CRUSH rule 2 x 119 [5,6,1]
+  CRUSH rule 2 x 120 [0,3,7]
+  CRUSH rule 2 x 121 [2,7,5]
+  CRUSH rule 2 x 122 [8,5,0]
+  CRUSH rule 2 x 123 [2,5,8]
+  CRUSH rule 2 x 124 [3,2,8]
+  CRUSH rule 2 x 125 [0,7,3]
+  CRUSH rule 2 x 126 [4,2,6]
+  CRUSH rule 2 x 127 [3,6,2]
+  CRUSH rule 2 x 128 [3,6,1]
+  CRUSH rule 2 x 129 [0,3,7]
+  CRUSH rule 2 x 130 [3,8,2]
+  CRUSH rule 2 x 131 [1,3,8]
+  CRUSH rule 2 x 132 [1,4,6]
+  CRUSH rule 2 x 133 [3,6,2]
+  CRUSH rule 2 x 134 [1,8,5]
+  CRUSH rule 2 x 135 [5,6,2]
+  CRUSH rule 2 x 136 [2,3,6]
+  CRUSH rule 2 x 137 [7,3,2]
+  CRUSH rule 2 x 138 [8,4,2]
+  CRUSH rule 2 x 139 [3,0,7]
+  CRUSH rule 2 x 140 [1,6,4]
+  CRUSH rule 2 x 141 [6,2,3]
+  CRUSH rule 2 x 142 [3,0,8]
+  CRUSH rule 2 x 143 [5,8,1]
+  CRUSH rule 2 x 144 [8,1,3]
+  CRUSH rule 2 x 145 [8,5,0]
+  CRUSH rule 2 x 146 [2,6,4]
+  CRUSH rule 2 x 147 [2,8,4]
+  CRUSH rule 2 x 148 [3,1,7]
+  CRUSH rule 2 x 149 [4,8,1]
+  CRUSH rule 2 x 150 [1,6,5]
+  CRUSH rule 2 x 151 [3,6,0]
+  CRUSH rule 2 x 152 [8,4,2]
+  CRUSH rule 2 x 153 [8,4,0]
+  CRUSH rule 2 x 154 [3,2,7]
+  CRUSH rule 2 x 155 [3,7,2]
+  CRUSH rule 2 x 156 [4,2,6]
+  CRUSH rule 2 x 157 [4,1,6]
+  CRUSH rule 2 x 158 [2,8,5]
+  CRUSH rule 2 x 159 [7,0,5]
+  CRUSH rule 2 x 160 [2,8,5]
+  CRUSH rule 2 x 161 [1,5,6]
+  CRUSH rule 2 x 162 [0,6,4]
+  CRUSH rule 2 x 163 [5,6,1]
+  CRUSH rule 2 x 164 [7,1,5]
+  CRUSH rule 2 x 165 [7,0,5]
+  CRUSH rule 2 x 166 [2,4,6]
+  CRUSH rule 2 x 167 [0,7,4]
+  CRUSH rule 2 x 168 [4,2,7]
+  CRUSH rule 2 x 169 [2,6,4]
+  CRUSH rule 2 x 170 [1,4,8]
+  CRUSH rule 2 x 171 [7,5,0]
+  CRUSH rule 2 x 172 [0,7,5]
+  CRUSH rule 2 x 173 [8,5,1]
+  CRUSH rule 2 x 174 [1,4,7]
+  CRUSH rule 2 x 175 [6,0,4]
+  CRUSH rule 2 x 176 [4,1,7]
+  CRUSH rule 2 x 177 [5,1,6]
+  CRUSH rule 2 x 178 [3,0,8]
+  CRUSH rule 2 x 179 [4,1,7]
+  CRUSH rule 2 x 180 [3,8,1]
+  CRUSH rule 2 x 181 [6,2,4]
+  CRUSH rule 2 x 182 [8,5,1]
+  CRUSH rule 2 x 183 [7,5,1]
+  CRUSH rule 2 x 184 [5,7,2]
+  CRUSH rule 2 x 185 [6,1,4]
+  CRUSH rule 2 x 186 [2,5,7]
+  CRUSH rule 2 x 187 [1,6,4]
+  CRUSH rule 2 x 188 [1,8,5]
+  CRUSH rule 2 x 189 [0,7,4]
+  CRUSH rule 2 x 190 [4,0,8]
+  CRUSH rule 2 x 191 [7,1,3]
+  CRUSH rule 2 x 192 [5,0,8]
+  CRUSH rule 2 x 193 [4,2,6]
+  CRUSH rule 2 x 194 [1,3,8]
+  CRUSH rule 2 x 195 [6,4,1]
+  CRUSH rule 2 x 196 [6,0,3]
+  CRUSH rule 2 x 197 [6,5,2]
+  CRUSH rule 2 x 198 [2,5,6]
+  CRUSH rule 2 x 199 [0,5,7]
+  CRUSH rule 2 x 200 [0,5,6]
+  CRUSH rule 2 x 201 [7,1,5]
+  CRUSH rule 2 x 202 [6,3,1]
+  CRUSH rule 2 x 203 [4,8,1]
+  CRUSH rule 2 x 204 [2,3,7]
+  CRUSH rule 2 x 205 [0,7,5]
+  CRUSH rule 2 x 206 [0,7,5]
+  CRUSH rule 2 x 207 [3,0,7]
+  CRUSH rule 2 x 208 [7,1,4]
+  CRUSH rule 2 x 209 [1,8,4]
+  CRUSH rule 2 x 210 [1,4,8]
+  CRUSH rule 2 x 211 [5,2,8]
+  CRUSH rule 2 x 212 [7,5,0]
+  CRUSH rule 2 x 213 [8,4,0]
+  CRUSH rule 2 x 214 [4,8,2]
+  CRUSH rule 2 x 215 [8,1,4]
+  CRUSH rule 2 x 216 [5,0,6]
+  CRUSH rule 2 x 217 [0,7,3]
+  CRUSH rule 2 x 218 [0,7,5]
+  CRUSH rule 2 x 219 [4,8,2]
+  CRUSH rule 2 x 220 [5,7,1]
+  CRUSH rule 2 x 221 [3,6,0]
+  CRUSH rule 2 x 222 [6,4,1]
+  CRUSH rule 2 x 223 [1,3,6]
+  CRUSH rule 2 x 224 [1,5,8]
+  CRUSH rule 2 x 225 [8,2,3]
+  CRUSH rule 2 x 226 [7,2,3]
+  CRUSH rule 2 x 227 [3,1,6]
+  CRUSH rule 2 x 228 [5,6,0]
+  CRUSH rule 2 x 229 [3,8,2]
+  CRUSH rule 2 x 230 [4,7,2]
+  CRUSH rule 2 x 231 [4,7,2]
+  CRUSH rule 2 x 232 [2,7,4]
+  CRUSH rule 2 x 233 [3,7,2]
+  CRUSH rule 2 x 234 [0,3,6]
+  CRUSH rule 2 x 235 [3,8,0]
+  CRUSH rule 2 x 236 [5,2,7]
+  CRUSH rule 2 x 237 [4,7,1]
+  CRUSH rule 2 x 238 [4,2,6]
+  CRUSH rule 2 x 239 [8,4,1]
+  CRUSH rule 2 x 240 [5,7,0]
+  CRUSH rule 2 x 241 [3,1,8]
+  CRUSH rule 2 x 242 [3,2,6]
+  CRUSH rule 2 x 243 [4,7,0]
+  CRUSH rule 2 x 244 [4,6,0]
+  CRUSH rule 2 x 245 [7,0,5]
+  CRUSH rule 2 x 246 [1,5,8]
+  CRUSH rule 2 x 247 [6,0,4]
+  CRUSH rule 2 x 248 [8,0,3]
+  CRUSH rule 2 x 249 [2,4,7]
+  CRUSH rule 2 x 250 [2,5,6]
+  CRUSH rule 2 x 251 [2,3,6]
+  CRUSH rule 2 x 252 [3,7,0]
+  CRUSH rule 2 x 253 [3,2,6]
+  CRUSH rule 2 x 254 [3,2,7]
+  CRUSH rule 2 x 255 [1,7,5]
+  CRUSH rule 2 x 256 [5,7,0]
+  CRUSH rule 2 x 257 [2,8,4]
+  CRUSH rule 2 x 258 [5,0,6]
+  CRUSH rule 2 x 259 [4,6,2]
+  CRUSH rule 2 x 260 [3,6,2]
+  CRUSH rule 2 x 261 [8,5,2]
+  CRUSH rule 2 x 262 [5,6,0]
+  CRUSH rule 2 x 263 [6,1,5]
+  CRUSH rule 2 x 264 [3,6,2]
+  CRUSH rule 2 x 265 [8,5,2]
+  CRUSH rule 2 x 266 [8,2,5]
+  CRUSH rule 2 x 267 [2,3,8]
+  CRUSH rule 2 x 268 [0,7,4]
+  CRUSH rule 2 x 269 [0,8,4]
+  CRUSH rule 2 x 270 [5,0,7]
+  CRUSH rule 2 x 271 [7,5,1]
+  CRUSH rule 2 x 272 [2,8,3]
+  CRUSH rule 2 x 273 [3,1,7]
+  CRUSH rule 2 x 274 [6,3,1]
+  CRUSH rule 2 x 275 [4,7,0]
+  CRUSH rule 2 x 276 [7,1,4]
+  CRUSH rule 2 x 277 [6,4,0]
+  CRUSH rule 2 x 278 [6,1,4]
+  CRUSH rule 2 x 279 [8,3,2]
+  CRUSH rule 2 x 280 [0,6,4]
+  CRUSH rule 2 x 281 [8,0,3]
+  CRUSH rule 2 x 282 [3,1,6]
+  CRUSH rule 2 x 283 [8,2,5]
+  CRUSH rule 2 x 284 [6,3,0]
+  CRUSH rule 2 x 285 [5,7,0]
+  CRUSH rule 2 x 286 [2,6,3]
+  CRUSH rule 2 x 287 [0,4,6]
+  CRUSH rule 2 x 288 [8,0,4]
+  CRUSH rule 2 x 289 [4,6,2]
+  CRUSH rule 2 x 290 [1,3,7]
+  CRUSH rule 2 x 291 [0,3,8]
+  CRUSH rule 2 x 292 [8,0,5]
+  CRUSH rule 2 x 293 [6,0,4]
+  CRUSH rule 2 x 294 [7,4,1]
+  CRUSH rule 2 x 295 [4,8,0]
+  CRUSH rule 2 x 296 [3,1,6]
+  CRUSH rule 2 x 297 [6,2,4]
+  CRUSH rule 2 x 298 [1,5,7]
+  CRUSH rule 2 x 299 [2,8,3]
+  CRUSH rule 2 x 300 [8,3,0]
+  CRUSH rule 2 x 301 [0,8,4]
+  CRUSH rule 2 x 302 [3,0,6]
+  CRUSH rule 2 x 303 [7,5,1]
+  CRUSH rule 2 x 304 [2,7,5]
+  CRUSH rule 2 x 305 [5,8,2]
+  CRUSH rule 2 x 306 [0,7,3]
+  CRUSH rule 2 x 307 [0,7,5]
+  CRUSH rule 2 x 308 [0,8,4]
+  CRUSH rule 2 x 309 [7,4,0]
+  CRUSH rule 2 x 310 [4,1,7]
+  CRUSH rule 2 x 311 [3,6,0]
+  CRUSH rule 2 x 312 [2,6,3]
+  CRUSH rule 2 x 313 [5,1,6]
+  CRUSH rule 2 x 314 [4,2,6]
+  CRUSH rule 2 x 315 [2,4,8]
+  CRUSH rule 2 x 316 [6,3,1]
+  CRUSH rule 2 x 317 [2,6,5]
+  CRUSH rule 2 x 318 [8,1,4]
+  CRUSH rule 2 x 319 [5,0,8]
+  CRUSH rule 2 x 320 [3,7,1]
+  CRUSH rule 2 x 321 [1,3,8]
+  CRUSH rule 2 x 322 [2,7,3]
+  CRUSH rule 2 x 323 [4,7,0]
+  CRUSH rule 2 x 324 [7,0,3]
+  CRUSH rule 2 x 325 [4,6,0]
+  CRUSH rule 2 x 326 [3,2,6]
+  CRUSH rule 2 x 327 [0,6,3]
+  CRUSH rule 2 x 328 [7,4,1]
+  CRUSH rule 2 x 329 [5,6,2]
+  CRUSH rule 2 x 330 [3,7,2]
+  CRUSH rule 2 x 331 [2,6,3]
+  CRUSH rule 2 x 332 [2,4,6]
+  CRUSH rule 2 x 333 [6,5,1]
+  CRUSH rule 2 x 334 [8,3,2]
+  CRUSH rule 2 x 335 [7,1,4]
+  CRUSH rule 2 x 336 [4,6,0]
+  CRUSH rule 2 x 337 [7,2,3]
+  CRUSH rule 2 x 338 [5,6,1]
+  CRUSH rule 2 x 339 [7,5,2]
+  CRUSH rule 2 x 340 [2,8,4]
+  CRUSH rule 2 x 341 [5,1,7]
+  CRUSH rule 2 x 342 [0,7,4]
+  CRUSH rule 2 x 343 [6,3,0]
+  CRUSH rule 2 x 344 [6,0,5]
+  CRUSH rule 2 x 345 [4,7,1]
+  CRUSH rule 2 x 346 [8,0,5]
+  CRUSH rule 2 x 347 [3,1,7]
+  CRUSH rule 2 x 348 [8,0,5]
+  CRUSH rule 2 x 349 [1,6,4]
+  CRUSH rule 2 x 350 [8,5,1]
+  CRUSH rule 2 x 351 [3,6,0]
+  CRUSH rule 2 x 352 [1,8,3]
+  CRUSH rule 2 x 353 [6,4,0]
+  CRUSH rule 2 x 354 [0,3,6]
+  CRUSH rule 2 x 355 [3,8,2]
+  CRUSH rule 2 x 356 [3,0,6]
+  CRUSH rule 2 x 357 [6,1,4]
+  CRUSH rule 2 x 358 [2,8,4]
+  CRUSH rule 2 x 359 [6,1,3]
+  CRUSH rule 2 x 360 [5,2,7]
+  CRUSH rule 2 x 361 [8,4,1]
+  CRUSH rule 2 x 362 [4,1,6]
+  CRUSH rule 2 x 363 [4,0,6]
+  CRUSH rule 2 x 364 [2,5,6]
+  CRUSH rule 2 x 365 [6,5,2]
+  CRUSH rule 2 x 366 [7,2,3]
+  CRUSH rule 2 x 367 [4,0,6]
+  CRUSH rule 2 x 368 [7,4,0]
+  CRUSH rule 2 x 369 [3,7,1]
+  CRUSH rule 2 x 370 [8,2,5]
+  CRUSH rule 2 x 371 [1,3,6]
+  CRUSH rule 2 x 372 [3,1,8]
+  CRUSH rule 2 x 373 [0,6,4]
+  CRUSH rule 2 x 374 [3,8,1]
+  CRUSH rule 2 x 375 [6,4,1]
+  CRUSH rule 2 x 376 [7,1,5]
+  CRUSH rule 2 x 377 [1,3,6]
+  CRUSH rule 2 x 378 [0,8,3]
+  CRUSH rule 2 x 379 [8,5,2]
+  CRUSH rule 2 x 380 [2,5,8]
+  CRUSH rule 2 x 381 [0,4,7]
+  CRUSH rule 2 x 382 [1,5,8]
+  CRUSH rule 2 x 383 [4,6,0]
+  CRUSH rule 2 x 384 [7,0,4]
+  CRUSH rule 2 x 385 [7,4,0]
+  CRUSH rule 2 x 386 [0,3,6]
+  CRUSH rule 2 x 387 [1,3,6]
+  CRUSH rule 2 x 388 [5,0,8]
+  CRUSH rule 2 x 389 [1,5,7]
+  CRUSH rule 2 x 390 [5,6,0]
+  CRUSH rule 2 x 391 [5,6,2]
+  CRUSH rule 2 x 392 [1,8,5]
+  CRUSH rule 2 x 393 [4,2,6]
+  CRUSH rule 2 x 394 [4,7,0]
+  CRUSH rule 2 x 395 [4,0,8]
+  CRUSH rule 2 x 396 [4,2,7]
+  CRUSH rule 2 x 397 [2,4,8]
+  CRUSH rule 2 x 398 [2,4,8]
+  CRUSH rule 2 x 399 [8,4,2]
+  CRUSH rule 2 x 400 [8,1,4]
+  CRUSH rule 2 x 401 [0,5,8]
+  CRUSH rule 2 x 402 [7,5,2]
+  CRUSH rule 2 x 403 [0,3,8]
+  CRUSH rule 2 x 404 [4,2,8]
+  CRUSH rule 2 x 405 [6,5,2]
+  CRUSH rule 2 x 406 [2,6,5]
+  CRUSH rule 2 x 407 [2,8,5]
+  CRUSH rule 2 x 408 [4,1,6]
+  CRUSH rule 2 x 409 [7,3,0]
+  CRUSH rule 2 x 410 [8,3,2]
+  CRUSH rule 2 x 411 [2,8,3]
+  CRUSH rule 2 x 412 [0,5,8]
+  CRUSH rule 2 x 413 [5,0,8]
+  CRUSH rule 2 x 414 [4,1,6]
+  CRUSH rule 2 x 415 [0,6,4]
+  CRUSH rule 2 x 416 [2,6,3]
+  CRUSH rule 2 x 417 [8,2,4]
+  CRUSH rule 2 x 418 [7,1,4]
+  CRUSH rule 2 x 419 [8,3,0]
+  CRUSH rule 2 x 420 [1,4,6]
+  CRUSH rule 2 x 421 [8,4,0]
+  CRUSH rule 2 x 422 [6,4,1]
+  CRUSH rule 2 x 423 [0,5,6]
+  CRUSH rule 2 x 424 [8,4,1]
+  CRUSH rule 2 x 425 [1,3,7]
+  CRUSH rule 2 x 426 [6,0,5]
+  CRUSH rule 2 x 427 [0,7,5]
+  CRUSH rule 2 x 428 [5,7,1]
+  CRUSH rule 2 x 429 [4,6,0]
+  CRUSH rule 2 x 430 [3,6,2]
+  CRUSH rule 2 x 431 [5,0,7]
+  CRUSH rule 2 x 432 [7,1,3]
+  CRUSH rule 2 x 433 [6,5,1]
+  CRUSH rule 2 x 434 [5,2,8]
+  CRUSH rule 2 x 435 [0,5,6]
+  CRUSH rule 2 x 436 [4,0,7]
+  CRUSH rule 2 x 437 [7,5,2]
+  CRUSH rule 2 x 438 [0,3,8]
+  CRUSH rule 2 x 439 [1,3,7]
+  CRUSH rule 2 x 440 [2,7,5]
+  CRUSH rule 2 x 441 [5,7,2]
+  CRUSH rule 2 x 442 [2,4,7]
+  CRUSH rule 2 x 443 [6,0,4]
+  CRUSH rule 2 x 444 [7,0,4]
+  CRUSH rule 2 x 445 [6,3,1]
+  CRUSH rule 2 x 446 [4,1,8]
+  CRUSH rule 2 x 447 [2,3,7]
+  CRUSH rule 2 x 448 [7,2,5]
+  CRUSH rule 2 x 449 [7,5,1]
+  CRUSH rule 2 x 450 [4,1,8]
+  CRUSH rule 2 x 451 [6,5,0]
+  CRUSH rule 2 x 452 [8,3,0]
+  CRUSH rule 2 x 453 [6,5,1]
+  CRUSH rule 2 x 454 [6,4,2]
+  CRUSH rule 2 x 455 [2,7,5]
+  CRUSH rule 2 x 456 [6,2,5]
+  CRUSH rule 2 x 457 [7,2,5]
+  CRUSH rule 2 x 458 [2,8,5]
+  CRUSH rule 2 x 459 [2,7,5]
+  CRUSH rule 2 x 460 [6,5,0]
+  CRUSH rule 2 x 461 [6,5,2]
+  CRUSH rule 2 x 462 [8,1,5]
+  CRUSH rule 2 x 463 [6,0,4]
+  CRUSH rule 2 x 464 [7,4,2]
+  CRUSH rule 2 x 465 [7,2,5]
+  CRUSH rule 2 x 466 [5,8,2]
+  CRUSH rule 2 x 467 [6,4,0]
+  CRUSH rule 2 x 468 [7,0,3]
+  CRUSH rule 2 x 469 [7,0,5]
+  CRUSH rule 2 x 470 [3,0,6]
+  CRUSH rule 2 x 471 [0,7,5]
+  CRUSH rule 2 x 472 [5,1,8]
+  CRUSH rule 2 x 473 [1,4,6]
+  CRUSH rule 2 x 474 [6,0,3]
+  CRUSH rule 2 x 475 [6,2,3]
+  CRUSH rule 2 x 476 [4,6,1]
+  CRUSH rule 2 x 477 [5,8,2]
+  CRUSH rule 2 x 478 [6,2,3]
+  CRUSH rule 2 x 479 [0,5,8]
+  CRUSH rule 2 x 480 [1,8,3]
+  CRUSH rule 2 x 481 [2,4,7]
+  CRUSH rule 2 x 482 [4,7,0]
+  CRUSH rule 2 x 483 [0,6,4]
+  CRUSH rule 2 x 484 [1,7,4]
+  CRUSH rule 2 x 485 [4,7,0]
+  CRUSH rule 2 x 486 [4,1,7]
+  CRUSH rule 2 x 487 [5,0,8]
+  CRUSH rule 2 x 488 [5,7,1]
+  CRUSH rule 2 x 489 [2,8,5]
+  CRUSH rule 2 x 490 [6,4,1]
+  CRUSH rule 2 x 491 [1,7,5]
+  CRUSH rule 2 x 492 [6,5,0]
+  CRUSH rule 2 x 493 [0,7,3]
+  CRUSH rule 2 x 494 [1,7,4]
+  CRUSH rule 2 x 495 [3,1,8]
+  CRUSH rule 2 x 496 [7,5,0]
+  CRUSH rule 2 x 497 [5,7,0]
+  CRUSH rule 2 x 498 [0,5,8]
+  CRUSH rule 2 x 499 [8,4,2]
+  CRUSH rule 2 x 500 [3,6,0]
+  CRUSH rule 2 x 501 [0,7,3]
+  CRUSH rule 2 x 502 [7,1,4]
+  CRUSH rule 2 x 503 [2,3,7]
+  CRUSH rule 2 x 504 [5,6,2]
+  CRUSH rule 2 x 505 [0,7,3]
+  CRUSH rule 2 x 506 [5,2,8]
+  CRUSH rule 2 x 507 [6,0,3]
+  CRUSH rule 2 x 508 [0,3,8]
+  CRUSH rule 2 x 509 [7,5,2]
+  CRUSH rule 2 x 510 [6,0,4]
+  CRUSH rule 2 x 511 [5,8,2]
+  CRUSH rule 2 x 512 [7,0,4]
+  CRUSH rule 2 x 513 [7,2,4]
+  CRUSH rule 2 x 514 [4,6,1]
+  CRUSH rule 2 x 515 [8,5,0]
+  CRUSH rule 2 x 516 [4,0,8]
+  CRUSH rule 2 x 517 [7,2,4]
+  CRUSH rule 2 x 518 [4,6,1]
+  CRUSH rule 2 x 519 [7,3,1]
+  CRUSH rule 2 x 520 [2,6,3]
+  CRUSH rule 2 x 521 [8,0,3]
+  CRUSH rule 2 x 522 [6,0,4]
+  CRUSH rule 2 x 523 [4,2,7]
+  CRUSH rule 2 x 524 [0,4,8]
+  CRUSH rule 2 x 525 [0,4,6]
+  CRUSH rule 2 x 526 [1,5,8]
+  CRUSH rule 2 x 527 [0,5,6]
+  CRUSH rule 2 x 528 [5,0,6]
+  CRUSH rule 2 x 529 [5,7,0]
+  CRUSH rule 2 x 530 [6,5,2]
+  CRUSH rule 2 x 531 [6,1,3]
+  CRUSH rule 2 x 532 [6,3,0]
+  CRUSH rule 2 x 533 [5,6,2]
+  CRUSH rule 2 x 534 [7,3,1]
+  CRUSH rule 2 x 535 [8,1,5]
+  CRUSH rule 2 x 536 [6,2,4]
+  CRUSH rule 2 x 537 [3,7,2]
+  CRUSH rule 2 x 538 [6,3,0]
+  CRUSH rule 2 x 539 [8,3,1]
+  CRUSH rule 2 x 540 [0,6,3]
+  CRUSH rule 2 x 541 [2,3,8]
+  CRUSH rule 2 x 542 [3,2,8]
+  CRUSH rule 2 x 543 [6,0,4]
+  CRUSH rule 2 x 544 [3,7,0]
+  CRUSH rule 2 x 545 [5,7,0]
+  CRUSH rule 2 x 546 [6,1,5]
+  CRUSH rule 2 x 547 [8,2,4]
+  CRUSH rule 2 x 548 [5,2,8]
+  CRUSH rule 2 x 549 [5,8,2]
+  CRUSH rule 2 x 550 [0,5,7]
+  CRUSH rule 2 x 551 [7,5,0]
+  CRUSH rule 2 x 552 [5,8,1]
+  CRUSH rule 2 x 553 [4,2,7]
+  CRUSH rule 2 x 554 [0,8,5]
+  CRUSH rule 2 x 555 [5,0,8]
+  CRUSH rule 2 x 556 [3,6,0]
+  CRUSH rule 2 x 557 [7,4,0]
+  CRUSH rule 2 x 558 [3,1,6]
+  CRUSH rule 2 x 559 [4,2,6]
+  CRUSH rule 2 x 560 [8,3,2]
+  CRUSH rule 2 x 561 [6,3,1]
+  CRUSH rule 2 x 562 [3,0,6]
+  CRUSH rule 2 x 563 [2,6,4]
+  CRUSH rule 2 x 564 [5,1,7]
+  CRUSH rule 2 x 565 [3,6,2]
+  CRUSH rule 2 x 566 [4,7,2]
+  CRUSH rule 2 x 567 [3,6,1]
+  CRUSH rule 2 x 568 [7,4,1]
+  CRUSH rule 2 x 569 [3,1,7]
+  CRUSH rule 2 x 570 [1,5,8]
+  CRUSH rule 2 x 571 [3,7,1]
+  CRUSH rule 2 x 572 [3,2,8]
+  CRUSH rule 2 x 573 [3,0,7]
+  CRUSH rule 2 x 574 [2,5,8]
+  CRUSH rule 2 x 575 [8,2,5]
+  CRUSH rule 2 x 576 [4,6,0]
+  CRUSH rule 2 x 577 [8,2,5]
+  CRUSH rule 2 x 578 [6,1,4]
+  CRUSH rule 2 x 579 [3,1,6]
+  CRUSH rule 2 x 580 [3,0,7]
+  CRUSH rule 2 x 581 [7,2,4]
+  CRUSH rule 2 x 582 [2,8,5]
+  CRUSH rule 2 x 583 [6,0,3]
+  CRUSH rule 2 x 584 [8,1,3]
+  CRUSH rule 2 x 585 [7,0,5]
+  CRUSH rule 2 x 586 [0,7,5]
+  CRUSH rule 2 x 587 [2,5,7]
+  CRUSH rule 2 x 588 [3,7,1]
+  CRUSH rule 2 x 589 [7,1,4]
+  CRUSH rule 2 x 590 [6,2,3]
+  CRUSH rule 2 x 591 [5,2,8]
+  CRUSH rule 2 x 592 [2,4,7]
+  CRUSH rule 2 x 593 [0,8,3]
+  CRUSH rule 2 x 594 [0,7,4]
+  CRUSH rule 2 x 595 [7,1,3]
+  CRUSH rule 2 x 596 [4,0,6]
+  CRUSH rule 2 x 597 [3,1,7]
+  CRUSH rule 2 x 598 [3,2,6]
+  CRUSH rule 2 x 599 [5,2,8]
+  CRUSH rule 2 x 600 [7,0,3]
+  CRUSH rule 2 x 601 [0,7,3]
+  CRUSH rule 2 x 602 [3,7,1]
+  CRUSH rule 2 x 603 [5,1,6]
+  CRUSH rule 2 x 604 [7,5,0]
+  CRUSH rule 2 x 605 [3,0,7]
+  CRUSH rule 2 x 606 [2,7,4]
+  CRUSH rule 2 x 607 [0,4,8]
+  CRUSH rule 2 x 608 [5,2,7]
+  CRUSH rule 2 x 609 [5,2,8]
+  CRUSH rule 2 x 610 [3,7,0]
+  CRUSH rule 2 x 611 [1,8,5]
+  CRUSH rule 2 x 612 [2,6,5]
+  CRUSH rule 2 x 613 [7,2,3]
+  CRUSH rule 2 x 614 [7,2,3]
+  CRUSH rule 2 x 615 [6,0,5]
+  CRUSH rule 2 x 616 [0,8,5]
+  CRUSH rule 2 x 617 [6,1,4]
+  CRUSH rule 2 x 618 [7,4,0]
+  CRUSH rule 2 x 619 [5,1,8]
+  CRUSH rule 2 x 620 [4,1,6]
+  CRUSH rule 2 x 621 [5,8,2]
+  CRUSH rule 2 x 622 [0,4,8]
+  CRUSH rule 2 x 623 [0,6,3]
+  CRUSH rule 2 x 624 [3,2,8]
+  CRUSH rule 2 x 625 [2,3,7]
+  CRUSH rule 2 x 626 [7,0,3]
+  CRUSH rule 2 x 627 [2,7,3]
+  CRUSH rule 2 x 628 [8,0,5]
+  CRUSH rule 2 x 629 [2,6,5]
+  CRUSH rule 2 x 630 [2,7,5]
+  CRUSH rule 2 x 631 [0,7,3]
+  CRUSH rule 2 x 632 [7,0,3]
+  CRUSH rule 2 x 633 [8,3,1]
+  CRUSH rule 2 x 634 [0,4,7]
+  CRUSH rule 2 x 635 [5,6,2]
+  CRUSH rule 2 x 636 [1,4,8]
+  CRUSH rule 2 x 637 [4,1,7]
+  CRUSH rule 2 x 638 [6,0,4]
+  CRUSH rule 2 x 639 [4,2,8]
+  CRUSH rule 2 x 640 [3,1,8]
+  CRUSH rule 2 x 641 [7,2,3]
+  CRUSH rule 2 x 642 [2,7,3]
+  CRUSH rule 2 x 643 [3,0,8]
+  CRUSH rule 2 x 644 [8,1,5]
+  CRUSH rule 2 x 645 [5,7,1]
+  CRUSH rule 2 x 646 [8,0,3]
+  CRUSH rule 2 x 647 [7,1,3]
+  CRUSH rule 2 x 648 [0,6,3]
+  CRUSH rule 2 x 649 [4,7,0]
+  CRUSH rule 2 x 650 [7,3,1]
+  CRUSH rule 2 x 651 [3,7,0]
+  CRUSH rule 2 x 652 [3,7,0]
+  CRUSH rule 2 x 653 [8,5,2]
+  CRUSH rule 2 x 654 [7,5,2]
+  CRUSH rule 2 x 655 [0,3,7]
+  CRUSH rule 2 x 656 [4,7,1]
+  CRUSH rule 2 x 657 [6,1,4]
+  CRUSH rule 2 x 658 [5,6,0]
+  CRUSH rule 2 x 659 [4,6,2]
+  CRUSH rule 2 x 660 [7,4,1]
+  CRUSH rule 2 x 661 [1,8,3]
+  CRUSH rule 2 x 662 [4,2,7]
+  CRUSH rule 2 x 663 [1,3,8]
+  CRUSH rule 2 x 664 [1,4,7]
+  CRUSH rule 2 x 665 [5,7,0]
+  CRUSH rule 2 x 666 [2,8,4]
+  CRUSH rule 2 x 667 [1,3,7]
+  CRUSH rule 2 x 668 [3,7,1]
+  CRUSH rule 2 x 669 [6,4,0]
+  CRUSH rule 2 x 670 [4,0,6]
+  CRUSH rule 2 x 671 [0,7,3]
+  CRUSH rule 2 x 672 [4,2,7]
+  CRUSH rule 2 x 673 [5,2,7]
+  CRUSH rule 2 x 674 [3,1,8]
+  CRUSH rule 2 x 675 [0,8,3]
+  CRUSH rule 2 x 676 [0,4,8]
+  CRUSH rule 2 x 677 [4,1,7]
+  CRUSH rule 2 x 678 [2,3,8]
+  CRUSH rule 2 x 679 [6,0,5]
+  CRUSH rule 2 x 680 [0,4,6]
+  CRUSH rule 2 x 681 [4,7,1]
+  CRUSH rule 2 x 682 [0,5,7]
+  CRUSH rule 2 x 683 [0,5,6]
+  CRUSH rule 2 x 684 [7,1,5]
+  CRUSH rule 2 x 685 [7,1,5]
+  CRUSH rule 2 x 686 [1,4,8]
+  CRUSH rule 2 x 687 [3,6,1]
+  CRUSH rule 2 x 688 [5,7,2]
+  CRUSH rule 2 x 689 [6,5,0]
+  CRUSH rule 2 x 690 [8,1,3]
+  CRUSH rule 2 x 691 [3,0,6]
+  CRUSH rule 2 x 692 [7,2,3]
+  CRUSH rule 2 x 693 [6,3,1]
+  CRUSH rule 2 x 694 [6,5,1]
+  CRUSH rule 2 x 695 [0,8,4]
+  CRUSH rule 2 x 696 [1,4,8]
+  CRUSH rule 2 x 697 [6,1,3]
+  CRUSH rule 2 x 698 [6,2,4]
+  CRUSH rule 2 x 699 [1,6,3]
+  CRUSH rule 2 x 700 [0,3,7]
+  CRUSH rule 2 x 701 [4,1,7]
+  CRUSH rule 2 x 702 [3,2,8]
+  CRUSH rule 2 x 703 [8,3,1]
+  CRUSH rule 2 x 704 [0,3,8]
+  CRUSH rule 2 x 705 [8,0,4]
+  CRUSH rule 2 x 706 [1,5,6]
+  CRUSH rule 2 x 707 [7,3,1]
+  CRUSH rule 2 x 708 [3,7,1]
+  CRUSH rule 2 x 709 [6,3,0]
+  CRUSH rule 2 x 710 [8,4,0]
+  CRUSH rule 2 x 711 [2,3,8]
+  CRUSH rule 2 x 712 [2,3,7]
+  CRUSH rule 2 x 713 [6,3,0]
+  CRUSH rule 2 x 714 [3,2,7]
+  CRUSH rule 2 x 715 [1,3,6]
+  CRUSH rule 2 x 716 [3,6,0]
+  CRUSH rule 2 x 717 [8,2,5]
+  CRUSH rule 2 x 718 [3,7,2]
+  CRUSH rule 2 x 719 [2,6,3]
+  CRUSH rule 2 x 720 [6,1,4]
+  CRUSH rule 2 x 721 [5,7,2]
+  CRUSH rule 2 x 722 [5,7,1]
+  CRUSH rule 2 x 723 [5,1,7]
+  CRUSH rule 2 x 724 [0,6,3]
+  CRUSH rule 2 x 725 [0,3,7]
+  CRUSH rule 2 x 726 [3,8,1]
+  CRUSH rule 2 x 727 [4,6,1]
+  CRUSH rule 2 x 728 [2,7,4]
+  CRUSH rule 2 x 729 [5,6,2]
+  CRUSH rule 2 x 730 [3,7,2]
+  CRUSH rule 2 x 731 [4,1,8]
+  CRUSH rule 2 x 732 [1,5,6]
+  CRUSH rule 2 x 733 [5,7,0]
+  CRUSH rule 2 x 734 [6,4,2]
+  CRUSH rule 2 x 735 [4,8,1]
+  CRUSH rule 2 x 736 [3,8,1]
+  CRUSH rule 2 x 737 [1,6,4]
+  CRUSH rule 2 x 738 [5,2,7]
+  CRUSH rule 2 x 739 [0,7,4]
+  CRUSH rule 2 x 740 [0,8,4]
+  CRUSH rule 2 x 741 [7,1,4]
+  CRUSH rule 2 x 742 [8,2,3]
+  CRUSH rule 2 x 743 [7,0,5]
+  CRUSH rule 2 x 744 [4,7,1]
+  CRUSH rule 2 x 745 [3,1,8]
+  CRUSH rule 2 x 746 [4,1,7]
+  CRUSH rule 2 x 747 [6,0,3]
+  CRUSH rule 2 x 748 [2,7,5]
+  CRUSH rule 2 x 749 [4,8,0]
+  CRUSH rule 2 x 750 [1,6,3]
+  CRUSH rule 2 x 751 [2,8,3]
+  CRUSH rule 2 x 752 [8,1,5]
+  CRUSH rule 2 x 753 [7,3,1]
+  CRUSH rule 2 x 754 [8,5,2]
+  CRUSH rule 2 x 755 [1,6,3]
+  CRUSH rule 2 x 756 [5,6,1]
+  CRUSH rule 2 x 757 [8,0,5]
+  CRUSH rule 2 x 758 [6,0,3]
+  CRUSH rule 2 x 759 [8,5,2]
+  CRUSH rule 2 x 760 [1,5,7]
+  CRUSH rule 2 x 761 [4,1,8]
+  CRUSH rule 2 x 762 [2,7,5]
+  CRUSH rule 2 x 763 [8,5,1]
+  CRUSH rule 2 x 764 [1,7,5]
+  CRUSH rule 2 x 765 [6,5,2]
+  CRUSH rule 2 x 766 [8,5,1]
+  CRUSH rule 2 x 767 [1,8,3]
+  CRUSH rule 2 x 768 [8,3,2]
+  CRUSH rule 2 x 769 [6,2,5]
+  CRUSH rule 2 x 770 [6,0,4]
+  CRUSH rule 2 x 771 [7,0,3]
+  CRUSH rule 2 x 772 [8,3,1]
+  CRUSH rule 2 x 773 [3,1,7]
+  CRUSH rule 2 x 774 [4,6,2]
+  CRUSH rule 2 x 775 [6,4,2]
+  CRUSH rule 2 x 776 [7,2,5]
+  CRUSH rule 2 x 777 [3,1,6]
+  CRUSH rule 2 x 778 [1,8,4]
+  CRUSH rule 2 x 779 [2,7,3]
+  CRUSH rule 2 x 780 [0,5,7]
+  CRUSH rule 2 x 781 [6,3,2]
+  CRUSH rule 2 x 782 [5,0,8]
+  CRUSH rule 2 x 783 [7,1,3]
+  CRUSH rule 2 x 784 [0,4,6]
+  CRUSH rule 2 x 785 [6,1,3]
+  CRUSH rule 2 x 786 [7,3,1]
+  CRUSH rule 2 x 787 [1,6,4]
+  CRUSH rule 2 x 788 [6,0,3]
+  CRUSH rule 2 x 789 [0,4,8]
+  CRUSH rule 2 x 790 [8,4,0]
+  CRUSH rule 2 x 791 [3,8,0]
+  CRUSH rule 2 x 792 [5,8,0]
+  CRUSH rule 2 x 793 [6,1,3]
+  CRUSH rule 2 x 794 [2,6,4]
+  CRUSH rule 2 x 795 [0,3,8]
+  CRUSH rule 2 x 796 [3,7,2]
+  CRUSH rule 2 x 797 [2,3,8]
+  CRUSH rule 2 x 798 [6,1,5]
+  CRUSH rule 2 x 799 [5,1,8]
+  CRUSH rule 2 x 800 [5,0,7]
+  CRUSH rule 2 x 801 [3,6,1]
+  CRUSH rule 2 x 802 [1,8,5]
+  CRUSH rule 2 x 803 [0,5,7]
+  CRUSH rule 2 x 804 [6,2,5]
+  CRUSH rule 2 x 805 [3,6,1]
+  CRUSH rule 2 x 806 [1,3,7]
+  CRUSH rule 2 x 807 [5,7,2]
+  CRUSH rule 2 x 808 [4,6,2]
+  CRUSH rule 2 x 809 [1,4,8]
+  CRUSH rule 2 x 810 [5,7,2]
+  CRUSH rule 2 x 811 [8,4,0]
+  CRUSH rule 2 x 812 [8,5,2]
+  CRUSH rule 2 x 813 [6,4,2]
+  CRUSH rule 2 x 814 [3,6,1]
+  CRUSH rule 2 x 815 [3,1,8]
+  CRUSH rule 2 x 816 [2,7,3]
+  CRUSH rule 2 x 817 [4,8,2]
+  CRUSH rule 2 x 818 [3,0,7]
+  CRUSH rule 2 x 819 [5,1,8]
+  CRUSH rule 2 x 820 [3,6,0]
+  CRUSH rule 2 x 821 [4,6,2]
+  CRUSH rule 2 x 822 [2,5,8]
+  CRUSH rule 2 x 823 [4,8,2]
+  CRUSH rule 2 x 824 [3,7,2]
+  CRUSH rule 2 x 825 [2,8,5]
+  CRUSH rule 2 x 826 [7,0,5]
+  CRUSH rule 2 x 827 [0,8,3]
+  CRUSH rule 2 x 828 [2,3,8]
+  CRUSH rule 2 x 829 [5,6,1]
+  CRUSH rule 2 x 830 [2,3,8]
+  CRUSH rule 2 x 831 [1,6,3]
+  CRUSH rule 2 x 832 [4,7,0]
+  CRUSH rule 2 x 833 [2,7,3]
+  CRUSH rule 2 x 834 [3,1,7]
+  CRUSH rule 2 x 835 [8,4,1]
+  CRUSH rule 2 x 836 [3,7,1]
+  CRUSH rule 2 x 837 [6,3,1]
+  CRUSH rule 2 x 838 [6,2,4]
+  CRUSH rule 2 x 839 [5,0,6]
+  CRUSH rule 2 x 840 [7,3,2]
+  CRUSH rule 2 x 841 [4,8,2]
+  CRUSH rule 2 x 842 [2,4,6]
+  CRUSH rule 2 x 843 [6,4,1]
+  CRUSH rule 2 x 844 [4,8,1]
+  CRUSH rule 2 x 845 [3,8,2]
+  CRUSH rule 2 x 846 [3,2,7]
+  CRUSH rule 2 x 847 [0,8,4]
+  CRUSH rule 2 x 848 [2,6,5]
+  CRUSH rule 2 x 849 [4,6,2]
+  CRUSH rule 2 x 850 [1,3,6]
+  CRUSH rule 2 x 851 [6,4,0]
+  CRUSH rule 2 x 852 [7,3,0]
+  CRUSH rule 2 x 853 [6,0,4]
+  CRUSH rule 2 x 854 [7,0,4]
+  CRUSH rule 2 x 855 [5,7,2]
+  CRUSH rule 2 x 856 [6,3,2]
+  CRUSH rule 2 x 857 [8,5,0]
+  CRUSH rule 2 x 858 [6,4,1]
+  CRUSH rule 2 x 859 [6,0,5]
+  CRUSH rule 2 x 860 [4,1,7]
+  CRUSH rule 2 x 861 [8,3,1]
+  CRUSH rule 2 x 862 [6,1,4]
+  CRUSH rule 2 x 863 [8,2,3]
+  CRUSH rule 2 x 864 [5,6,2]
+  CRUSH rule 2 x 865 [8,1,3]
+  CRUSH rule 2 x 866 [3,6,0]
+  CRUSH rule 2 x 867 [6,5,1]
+  CRUSH rule 2 x 868 [6,3,0]
+  CRUSH rule 2 x 869 [8,5,2]
+  CRUSH rule 2 x 870 [0,4,8]
+  CRUSH rule 2 x 871 [3,2,8]
+  CRUSH rule 2 x 872 [5,1,8]
+  CRUSH rule 2 x 873 [4,6,2]
+  CRUSH rule 2 x 874 [2,6,4]
+  CRUSH rule 2 x 875 [2,6,4]
+  CRUSH rule 2 x 876 [5,8,1]
+  CRUSH rule 2 x 877 [6,4,2]
+  CRUSH rule 2 x 878 [5,2,7]
+  CRUSH rule 2 x 879 [7,4,2]
+  CRUSH rule 2 x 880 [3,2,8]
+  CRUSH rule 2 x 881 [5,6,1]
+  CRUSH rule 2 x 882 [4,0,7]
+  CRUSH rule 2 x 883 [2,3,7]
+  CRUSH rule 2 x 884 [6,0,4]
+  CRUSH rule 2 x 885 [5,1,8]
+  CRUSH rule 2 x 886 [3,6,0]
+  CRUSH rule 2 x 887 [7,4,0]
+  CRUSH rule 2 x 888 [6,2,5]
+  CRUSH rule 2 x 889 [2,6,4]
+  CRUSH rule 2 x 890 [7,2,4]
+  CRUSH rule 2 x 891 [1,8,5]
+  CRUSH rule 2 x 892 [6,2,3]
+  CRUSH rule 2 x 893 [2,3,7]
+  CRUSH rule 2 x 894 [7,5,0]
+  CRUSH rule 2 x 895 [5,1,8]
+  CRUSH rule 2 x 896 [1,8,5]
+  CRUSH rule 2 x 897 [4,2,6]
+  CRUSH rule 2 x 898 [0,5,7]
+  CRUSH rule 2 x 899 [1,7,5]
+  CRUSH rule 2 x 900 [4,1,6]
+  CRUSH rule 2 x 901 [5,0,8]
+  CRUSH rule 2 x 902 [8,5,0]
+  CRUSH rule 2 x 903 [5,7,1]
+  CRUSH rule 2 x 904 [5,6,2]
+  CRUSH rule 2 x 905 [6,2,5]
+  CRUSH rule 2 x 906 [1,6,3]
+  CRUSH rule 2 x 907 [7,1,5]
+  CRUSH rule 2 x 908 [5,8,1]
+  CRUSH rule 2 x 909 [2,3,7]
+  CRUSH rule 2 x 910 [6,4,0]
+  CRUSH rule 2 x 911 [5,8,1]
+  CRUSH rule 2 x 912 [0,7,3]
+  CRUSH rule 2 x 913 [7,2,4]
+  CRUSH rule 2 x 914 [6,4,0]
+  CRUSH rule 2 x 915 [8,2,3]
+  CRUSH rule 2 x 916 [3,1,8]
+  CRUSH rule 2 x 917 [1,5,8]
+  CRUSH rule 2 x 918 [8,2,4]
+  CRUSH rule 2 x 919 [6,2,3]
+  CRUSH rule 2 x 920 [7,4,0]
+  CRUSH rule 2 x 921 [1,4,6]
+  CRUSH rule 2 x 922 [6,4,0]
+  CRUSH rule 2 x 923 [5,8,2]
+  CRUSH rule 2 x 924 [3,1,7]
+  CRUSH rule 2 x 925 [5,7,2]
+  CRUSH rule 2 x 926 [3,0,8]
+  CRUSH rule 2 x 927 [1,6,3]
+  CRUSH rule 2 x 928 [8,1,3]
+  CRUSH rule 2 x 929 [4,1,7]
+  CRUSH rule 2 x 930 [2,4,6]
+  CRUSH rule 2 x 931 [5,0,7]
+  CRUSH rule 2 x 932 [4,1,8]
+  CRUSH rule 2 x 933 [8,5,0]
+  CRUSH rule 2 x 934 [5,6,0]
+  CRUSH rule 2 x 935 [6,3,1]
+  CRUSH rule 2 x 936 [0,6,5]
+  CRUSH rule 2 x 937 [5,8,2]
+  CRUSH rule 2 x 938 [6,5,2]
+  CRUSH rule 2 x 939 [2,7,5]
+  CRUSH rule 2 x 940 [8,5,0]
+  CRUSH rule 2 x 941 [5,2,8]
+  CRUSH rule 2 x 942 [1,8,4]
+  CRUSH rule 2 x 943 [8,2,4]
+  CRUSH rule 2 x 944 [4,8,2]
+  CRUSH rule 2 x 945 [7,2,4]
+  CRUSH rule 2 x 946 [2,8,5]
+  CRUSH rule 2 x 947 [4,2,8]
+  CRUSH rule 2 x 948 [7,5,0]
+  CRUSH rule 2 x 949 [6,1,3]
+  CRUSH rule 2 x 950 [3,6,0]
+  CRUSH rule 2 x 951 [4,8,1]
+  CRUSH rule 2 x 952 [2,7,3]
+  CRUSH rule 2 x 953 [1,3,6]
+  CRUSH rule 2 x 954 [4,2,7]
+  CRUSH rule 2 x 955 [8,0,4]
+  CRUSH rule 2 x 956 [1,6,4]
+  CRUSH rule 2 x 957 [7,1,3]
+  CRUSH rule 2 x 958 [8,4,1]
+  CRUSH rule 2 x 959 [5,2,7]
+  CRUSH rule 2 x 960 [3,6,0]
+  CRUSH rule 2 x 961 [4,0,8]
+  CRUSH rule 2 x 962 [7,4,0]
+  CRUSH rule 2 x 963 [0,5,6]
+  CRUSH rule 2 x 964 [3,1,8]
+  CRUSH rule 2 x 965 [7,4,0]
+  CRUSH rule 2 x 966 [3,8,0]
+  CRUSH rule 2 x 967 [8,5,0]
+  CRUSH rule 2 x 968 [7,2,4]
+  CRUSH rule 2 x 969 [8,0,5]
+  CRUSH rule 2 x 970 [0,6,3]
+  CRUSH rule 2 x 971 [1,7,3]
+  CRUSH rule 2 x 972 [1,8,4]
+  CRUSH rule 2 x 973 [1,6,3]
+  CRUSH rule 2 x 974 [5,1,8]
+  CRUSH rule 2 x 975 [3,7,0]
+  CRUSH rule 2 x 976 [4,8,2]
+  CRUSH rule 2 x 977 [8,3,2]
+  CRUSH rule 2 x 978 [7,2,4]
+  CRUSH rule 2 x 979 [7,1,5]
+  CRUSH rule 2 x 980 [6,0,5]
+  CRUSH rule 2 x 981 [7,3,2]
+  CRUSH rule 2 x 982 [4,2,8]
+  CRUSH rule 2 x 983 [3,7,0]
+  CRUSH rule 2 x 984 [0,7,3]
+  CRUSH rule 2 x 985 [2,5,7]
+  CRUSH rule 2 x 986 [8,3,0]
+  CRUSH rule 2 x 987 [0,5,8]
+  CRUSH rule 2 x 988 [1,3,7]
+  CRUSH rule 2 x 989 [0,6,3]
+  CRUSH rule 2 x 990 [1,6,5]
+  CRUSH rule 2 x 991 [0,4,8]
+  CRUSH rule 2 x 992 [7,1,5]
+  CRUSH rule 2 x 993 [0,6,3]
+  CRUSH rule 2 x 994 [3,7,2]
+  CRUSH rule 2 x 995 [7,1,5]
+  CRUSH rule 2 x 996 [6,5,0]
+  CRUSH rule 2 x 997 [6,4,1]
+  CRUSH rule 2 x 998 [8,1,5]
+  CRUSH rule 2 x 999 [0,7,4]
+  CRUSH rule 2 x 1000 [8,5,0]
+  CRUSH rule 2 x 1001 [2,5,6]
+  CRUSH rule 2 x 1002 [1,3,7]
+  CRUSH rule 2 x 1003 [2,8,3]
+  CRUSH rule 2 x 1004 [6,1,3]
+  CRUSH rule 2 x 1005 [6,1,5]
+  CRUSH rule 2 x 1006 [1,6,5]
+  CRUSH rule 2 x 1007 [1,5,7]
+  CRUSH rule 2 x 1008 [1,7,3]
+  CRUSH rule 2 x 1009 [6,4,1]
+  CRUSH rule 2 x 1010 [3,1,7]
+  CRUSH rule 2 x 1011 [3,0,8]
+  CRUSH rule 2 x 1012 [3,0,7]
+  CRUSH rule 2 x 1013 [5,1,7]
+  CRUSH rule 2 x 1014 [2,8,4]
+  CRUSH rule 2 x 1015 [6,5,0]
+  CRUSH rule 2 x 1016 [2,4,7]
+  CRUSH rule 2 x 1017 [6,0,3]
+  CRUSH rule 2 x 1018 [5,0,6]
+  CRUSH rule 2 x 1019 [5,8,2]
+  CRUSH rule 2 x 1020 [5,1,7]
+  CRUSH rule 2 x 1021 [5,2,6]
+  CRUSH rule 2 x 1022 [1,6,4]
+  CRUSH rule 2 x 1023 [3,2,8]
+  rule 2 (chooseleaf) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 3 (choose-set), x = 0..1023, numrep = 2..3
+  CRUSH rule 3 x 0 [0,3]
+  CRUSH rule 3 x 1 [0,8]
+  CRUSH rule 3 x 2 [1,4]
+  CRUSH rule 3 x 3 [8,0]
+  CRUSH rule 3 x 4 [5,1]
+  CRUSH rule 3 x 5 [7,0]
+  CRUSH rule 3 x 6 [2,6]
+  CRUSH rule 3 x 7 [5,6]
+  CRUSH rule 3 x 8 [5,7]
+  CRUSH rule 3 x 9 [2,4]
+  CRUSH rule 3 x 10 [0,8]
+  CRUSH rule 3 x 11 [0,6]
+  CRUSH rule 3 x 12 [0,3]
+  CRUSH rule 3 x 13 [3,8]
+  CRUSH rule 3 x 14 [7,1]
+  CRUSH rule 3 x 15 [7,1]
+  CRUSH rule 3 x 16 [3,7]
+  CRUSH rule 3 x 17 [5,1]
+  CRUSH rule 3 x 18 [1,3]
+  CRUSH rule 3 x 19 [7,5]
+  CRUSH rule 3 x 20 [2,4]
+  CRUSH rule 3 x 21 [3,6]
+  CRUSH rule 3 x 22 [8,5]
+  CRUSH rule 3 x 23 [3,7]
+  CRUSH rule 3 x 24 [1,6]
+  CRUSH rule 3 x 25 [3,8]
+  CRUSH rule 3 x 26 [2,7]
+  CRUSH rule 3 x 27 [3,1]
+  CRUSH rule 3 x 28 [6,2]
+  CRUSH rule 3 x 29 [8,5]
+  CRUSH rule 3 x 30 [5,6]
+  CRUSH rule 3 x 31 [8,2]
+  CRUSH rule 3 x 32 [3,7]
+  CRUSH rule 3 x 33 [2,7]
+  CRUSH rule 3 x 34 [2,3]
+  CRUSH rule 3 x 35 [0,6]
+  CRUSH rule 3 x 36 [3,6]
+  CRUSH rule 3 x 37 [0,4]
+  CRUSH rule 3 x 38 [4,6]
+  CRUSH rule 3 x 39 [3,7]
+  CRUSH rule 3 x 40 [7,2]
+  CRUSH rule 3 x 41 [0,7]
+  CRUSH rule 3 x 42 [4,7]
+  CRUSH rule 3 x 43 [0,3]
+  CRUSH rule 3 x 44 [1,8]
+  CRUSH rule 3 x 45 [8,2]
+  CRUSH rule 3 x 46 [2,4]
+  CRUSH rule 3 x 47 [4,1]
+  CRUSH rule 3 x 48 [4,6]
+  CRUSH rule 3 x 49 [5,6]
+  CRUSH rule 3 x 50 [3,1]
+  CRUSH rule 3 x 51 [3,6]
+  CRUSH rule 3 x 52 [8,2]
+  CRUSH rule 3 x 53 [3,6]
+  CRUSH rule 3 x 54 [7,4]
+  CRUSH rule 3 x 55 [8,0]
+  CRUSH rule 3 x 56 [6,5]
+  CRUSH rule 3 x 57 [5,8]
+  CRUSH rule 3 x 58 [1,8]
+  CRUSH rule 3 x 59 [4,0]
+  CRUSH rule 3 x 60 [3,1]
+  CRUSH rule 3 x 61 [4,8]
+  CRUSH rule 3 x 62 [7,0]
+  CRUSH rule 3 x 63 [5,6]
+  CRUSH rule 3 x 64 [4,1]
+  CRUSH rule 3 x 65 [7,4]
+  CRUSH rule 3 x 66 [5,6]
+  CRUSH rule 3 x 67 [5,0]
+  CRUSH rule 3 x 68 [0,3]
+  CRUSH rule 3 x 69 [5,2]
+  CRUSH rule 3 x 70 [7,2]
+  CRUSH rule 3 x 71 [2,7]
+  CRUSH rule 3 x 72 [6,2]
+  CRUSH rule 3 x 73 [2,7]
+  CRUSH rule 3 x 74 [0,8]
+  CRUSH rule 3 x 75 [3,0]
+  CRUSH rule 3 x 76 [5,2]
+  CRUSH rule 3 x 77 [7,0]
+  CRUSH rule 3 x 78 [1,4]
+  CRUSH rule 3 x 79 [5,0]
+  CRUSH rule 3 x 80 [0,4]
+  CRUSH rule 3 x 81 [0,5]
+  CRUSH rule 3 x 82 [7,2]
+  CRUSH rule 3 x 83 [2,6]
+  CRUSH rule 3 x 84 [7,0]
+  CRUSH rule 3 x 85 [3,7]
+  CRUSH rule 3 x 86 [0,6]
+  CRUSH rule 3 x 87 [0,6]
+  CRUSH rule 3 x 88 [1,8]
+  CRUSH rule 3 x 89 [3,1]
+  CRUSH rule 3 x 90 [6,3]
+  CRUSH rule 3 x 91 [3,6]
+  CRUSH rule 3 x 92 [1,6]
+  CRUSH rule 3 x 93 [7,4]
+  CRUSH rule 3 x 94 [0,3]
+  CRUSH rule 3 x 95 [7,3]
+  CRUSH rule 3 x 96 [3,8]
+  CRUSH rule 3 x 97 [8,4]
+  CRUSH rule 3 x 98 [2,7]
+  CRUSH rule 3 x 99 [0,8]
+  CRUSH rule 3 x 100 [1,6]
+  CRUSH rule 3 x 101 [3,8]
+  CRUSH rule 3 x 102 [4,0]
+  CRUSH rule 3 x 103 [4,7]
+  CRUSH rule 3 x 104 [7,4]
+  CRUSH rule 3 x 105 [2,3]
+  CRUSH rule 3 x 106 [1,6]
+  CRUSH rule 3 x 107 [3,1]
+  CRUSH rule 3 x 108 [7,1]
+  CRUSH rule 3 x 109 [1,3]
+  CRUSH rule 3 x 110 [3,2]
+  CRUSH rule 3 x 111 [2,4]
+  CRUSH rule 3 x 112 [2,6]
+  CRUSH rule 3 x 113 [6,1]
+  CRUSH rule 3 x 114 [7,5]
+  CRUSH rule 3 x 115 [8,2]
+  CRUSH rule 3 x 116 [1,7]
+  CRUSH rule 3 x 117 [7,5]
+  CRUSH rule 3 x 118 [0,3]
+  CRUSH rule 3 x 119 [5,7]
+  CRUSH rule 3 x 120 [0,5]
+  CRUSH rule 3 x 121 [2,8]
+  CRUSH rule 3 x 122 [8,4]
+  CRUSH rule 3 x 123 [2,4]
+  CRUSH rule 3 x 124 [3,0]
+  CRUSH rule 3 x 125 [0,7]
+  CRUSH rule 3 x 126 [4,1]
+  CRUSH rule 3 x 127 [3,6]
+  CRUSH rule 3 x 128 [3,8]
+  CRUSH rule 3 x 129 [0,4]
+  CRUSH rule 3 x 130 [3,8]
+  CRUSH rule 3 x 131 [1,5]
+  CRUSH rule 3 x 132 [1,3]
+  CRUSH rule 3 x 133 [3,6]
+  CRUSH rule 3 x 134 [1,7]
+  CRUSH rule 3 x 135 [5,7]
+  CRUSH rule 3 x 136 [2,3]
+  CRUSH rule 3 x 137 [7,3]
+  CRUSH rule 3 x 138 [8,3]
+  CRUSH rule 3 x 139 [3,0]
+  CRUSH rule 3 x 140 [1,8]
+  CRUSH rule 3 x 141 [6,0]
+  CRUSH rule 3 x 142 [3,2]
+  CRUSH rule 3 x 143 [5,7]
+  CRUSH rule 3 x 144 [8,2]
+  CRUSH rule 3 x 145 [8,4]
+  CRUSH rule 3 x 146 [2,8]
+  CRUSH rule 3 x 147 [2,6]
+  CRUSH rule 3 x 148 [3,0]
+  CRUSH rule 3 x 149 [4,6]
+  CRUSH rule 3 x 150 [1,8]
+  CRUSH rule 3 x 151 [3,8]
+  CRUSH rule 3 x 152 [8,3]
+  CRUSH rule 3 x 153 [8,3]
+  CRUSH rule 3 x 154 [3,0]
+  CRUSH rule 3 x 155 [3,6]
+  CRUSH rule 3 x 156 [4,2]
+  CRUSH rule 3 x 157 [4,1]
+  CRUSH rule 3 x 158 [2,6]
+  CRUSH rule 3 x 159 [7,0]
+  CRUSH rule 3 x 160 [2,8]
+  CRUSH rule 3 x 161 [1,3]
+  CRUSH rule 3 x 162 [0,8]
+  CRUSH rule 3 x 163 [5,8]
+  CRUSH rule 3 x 164 [7,2]
+  CRUSH rule 3 x 165 [7,2]
+  CRUSH rule 3 x 166 [2,3]
+  CRUSH rule 3 x 167 [0,6]
+  CRUSH rule 3 x 168 [4,0]
+  CRUSH rule 3 x 169 [2,7]
+  CRUSH rule 3 x 170 [1,3]
+  CRUSH rule 3 x 171 [7,3]
+  CRUSH rule 3 x 172 [0,8]
+  CRUSH rule 3 x 173 [8,5]
+  CRUSH rule 3 x 174 [1,4]
+  CRUSH rule 3 x 175 [6,0]
+  CRUSH rule 3 x 176 [4,2]
+  CRUSH rule 3 x 177 [5,1]
+  CRUSH rule 3 x 178 [3,0]
+  CRUSH rule 3 x 179 [4,1]
+  CRUSH rule 3 x 180 [3,7]
+  CRUSH rule 3 x 181 [6,0]
+  CRUSH rule 3 x 182 [8,3]
+  CRUSH rule 3 x 183 [7,5]
+  CRUSH rule 3 x 184 [5,6]
+  CRUSH rule 3 x 185 [6,0]
+  CRUSH rule 3 x 186 [2,4]
+  CRUSH rule 3 x 187 [1,6]
+  CRUSH rule 3 x 188 [1,6]
+  CRUSH rule 3 x 189 [0,6]
+  CRUSH rule 3 x 190 [4,1]
+  CRUSH rule 3 x 191 [7,0]
+  CRUSH rule 3 x 192 [5,2]
+  CRUSH rule 3 x 193 [4,0]
+  CRUSH rule 3 x 194 [1,3]
+  CRUSH rule 3 x 195 [6,5]
+  CRUSH rule 3 x 196 [6,1]
+  CRUSH rule 3 x 197 [6,5]
+  CRUSH rule 3 x 198 [2,3]
+  CRUSH rule 3 x 199 [0,5]
+  CRUSH rule 3 x 200 [0,3]
+  CRUSH rule 3 x 201 [7,1]
+  CRUSH rule 3 x 202 [6,4]
+  CRUSH rule 3 x 203 [4,6]
+  CRUSH rule 3 x 204 [2,4]
+  CRUSH rule 3 x 205 [0,7]
+  CRUSH rule 3 x 206 [0,8]
+  CRUSH rule 3 x 207 [3,2]
+  CRUSH rule 3 x 208 [7,2]
+  CRUSH rule 3 x 209 [1,8]
+  CRUSH rule 3 x 210 [1,4]
+  CRUSH rule 3 x 211 [5,2]
+  CRUSH rule 3 x 212 [7,5]
+  CRUSH rule 3 x 213 [8,4]
+  CRUSH rule 3 x 214 [4,7]
+  CRUSH rule 3 x 215 [8,0]
+  CRUSH rule 3 x 216 [5,2]
+  CRUSH rule 3 x 217 [0,7]
+  CRUSH rule 3 x 218 [0,7]
+  CRUSH rule 3 x 219 [4,6]
+  CRUSH rule 3 x 220 [5,8]
+  CRUSH rule 3 x 221 [3,7]
+  CRUSH rule 3 x 222 [6,4]
+  CRUSH rule 3 x 223 [1,3]
+  CRUSH rule 3 x 224 [1,3]
+  CRUSH rule 3 x 225 [8,0]
+  CRUSH rule 3 x 226 [7,0]
+  CRUSH rule 3 x 227 [3,0]
+  CRUSH rule 3 x 228 [5,6]
+  CRUSH rule 3 x 229 [3,7]
+  CRUSH rule 3 x 230 [4,6]
+  CRUSH rule 3 x 231 [4,7]
+  CRUSH rule 3 x 232 [2,8]
+  CRUSH rule 3 x 233 [3,6]
+  CRUSH rule 3 x 234 [0,4]
+  CRUSH rule 3 x 235 [3,6]
+  CRUSH rule 3 x 236 [5,0]
+  CRUSH rule 3 x 237 [4,8]
+  CRUSH rule 3 x 238 [4,2]
+  CRUSH rule 3 x 239 [8,3]
+  CRUSH rule 3 x 240 [5,8]
+  CRUSH rule 3 x 241 [3,1]
+  CRUSH rule 3 x 242 [3,2]
+  CRUSH rule 3 x 243 [4,8]
+  CRUSH rule 3 x 244 [4,6]
+  CRUSH rule 3 x 245 [7,2]
+  CRUSH rule 3 x 246 [1,3]
+  CRUSH rule 3 x 247 [6,1]
+  CRUSH rule 3 x 248 [8,0]
+  CRUSH rule 3 x 249 [2,5]
+  CRUSH rule 3 x 250 [2,3]
+  CRUSH rule 3 x 251 [2,3]
+  CRUSH rule 3 x 252 [3,7]
+  CRUSH rule 3 x 253 [3,0]
+  CRUSH rule 3 x 254 [3,2]
+  CRUSH rule 3 x 255 [1,7]
+  CRUSH rule 3 x 256 [5,6]
+  CRUSH rule 3 x 257 [2,6]
+  CRUSH rule 3 x 258 [5,2]
+  CRUSH rule 3 x 259 [4,6]
+  CRUSH rule 3 x 260 [3,8]
+  CRUSH rule 3 x 261 [8,5]
+  CRUSH rule 3 x 262 [5,6]
+  CRUSH rule 3 x 263 [6,1]
+  CRUSH rule 3 x 264 [3,6]
+  CRUSH rule 3 x 265 [8,4]
+  CRUSH rule 3 x 266 [8,0]
+  CRUSH rule 3 x 267 [2,4]
+  CRUSH rule 3 x 268 [0,6]
+  CRUSH rule 3 x 269 [0,6]
+  CRUSH rule 3 x 270 [5,1]
+  CRUSH rule 3 x 271 [7,4]
+  CRUSH rule 3 x 272 [2,6]
+  CRUSH rule 3 x 273 [3,2]
+  CRUSH rule 3 x 274 [6,3]
+  CRUSH rule 3 x 275 [4,8]
+  CRUSH rule 3 x 276 [7,0]
+  CRUSH rule 3 x 277 [6,4]
+  CRUSH rule 3 x 278 [6,2]
+  CRUSH rule 3 x 279 [8,5]
+  CRUSH rule 3 x 280 [0,7]
+  CRUSH rule 3 x 281 [8,1]
+  CRUSH rule 3 x 282 [3,2]
+  CRUSH rule 3 x 283 [8,1]
+  CRUSH rule 3 x 284 [6,3]
+  CRUSH rule 3 x 285 [5,7]
+  CRUSH rule 3 x 286 [2,8]
+  CRUSH rule 3 x 287 [0,3]
+  CRUSH rule 3 x 288 [8,1]
+  CRUSH rule 3 x 289 [4,6]
+  CRUSH rule 3 x 290 [1,5]
+  CRUSH rule 3 x 291 [0,5]
+  CRUSH rule 3 x 292 [8,1]
+  CRUSH rule 3 x 293 [6,2]
+  CRUSH rule 3 x 294 [7,5]
+  CRUSH rule 3 x 295 [4,6]
+  CRUSH rule 3 x 296 [3,1]
+  CRUSH rule 3 x 297 [6,0]
+  CRUSH rule 3 x 298 [1,5]
+  CRUSH rule 3 x 299 [2,8]
+  CRUSH rule 3 x 300 [8,4]
+  CRUSH rule 3 x 301 [0,7]
+  CRUSH rule 3 x 302 [3,1]
+  CRUSH rule 3 x 303 [7,4]
+  CRUSH rule 3 x 304 [2,6]
+  CRUSH rule 3 x 305 [5,6]
+  CRUSH rule 3 x 306 [0,8]
+  CRUSH rule 3 x 307 [0,7]
+  CRUSH rule 3 x 308 [0,8]
+  CRUSH rule 3 x 309 [7,4]
+  CRUSH rule 3 x 310 [4,0]
+  CRUSH rule 3 x 311 [3,8]
+  CRUSH rule 3 x 312 [2,6]
+  CRUSH rule 3 x 313 [5,1]
+  CRUSH rule 3 x 314 [4,0]
+  CRUSH rule 3 x 315 [2,3]
+  CRUSH rule 3 x 316 [6,4]
+  CRUSH rule 3 x 317 [2,7]
+  CRUSH rule 3 x 318 [8,1]
+  CRUSH rule 3 x 319 [5,2]
+  CRUSH rule 3 x 320 [3,8]
+  CRUSH rule 3 x 321 [1,5]
+  CRUSH rule 3 x 322 [2,6]
+  CRUSH rule 3 x 323 [4,7]
+  CRUSH rule 3 x 324 [7,2]
+  CRUSH rule 3 x 325 [4,6]
+  CRUSH rule 3 x 326 [3,1]
+  CRUSH rule 3 x 327 [0,8]
+  CRUSH rule 3 x 328 [7,3]
+  CRUSH rule 3 x 329 [5,7]
+  CRUSH rule 3 x 330 [3,7]
+  CRUSH rule 3 x 331 [2,7]
+  CRUSH rule 3 x 332 [2,3]
+  CRUSH rule 3 x 333 [6,4]
+  CRUSH rule 3 x 334 [8,3]
+  CRUSH rule 3 x 335 [7,2]
+  CRUSH rule 3 x 336 [4,6]
+  CRUSH rule 3 x 337 [7,0]
+  CRUSH rule 3 x 338 [5,8]
+  CRUSH rule 3 x 339 [7,5]
+  CRUSH rule 3 x 340 [2,6]
+  CRUSH rule 3 x 341 [5,1]
+  CRUSH rule 3 x 342 [0,8]
+  CRUSH rule 3 x 343 [6,5]
+  CRUSH rule 3 x 344 [6,1]
+  CRUSH rule 3 x 345 [4,7]
+  CRUSH rule 3 x 346 [8,0]
+  CRUSH rule 3 x 347 [3,0]
+  CRUSH rule 3 x 348 [8,0]
+  CRUSH rule 3 x 349 [1,7]
+  CRUSH rule 3 x 350 [8,5]
+  CRUSH rule 3 x 351 [3,8]
+  CRUSH rule 3 x 352 [1,8]
+  CRUSH rule 3 x 353 [6,5]
+  CRUSH rule 3 x 354 [0,5]
+  CRUSH rule 3 x 355 [3,8]
+  CRUSH rule 3 x 356 [3,1]
+  CRUSH rule 3 x 357 [6,1]
+  CRUSH rule 3 x 358 [2,8]
+  CRUSH rule 3 x 359 [6,0]
+  CRUSH rule 3 x 360 [5,0]
+  CRUSH rule 3 x 361 [8,5]
+  CRUSH rule 3 x 362 [4,0]
+  CRUSH rule 3 x 363 [4,2]
+  CRUSH rule 3 x 364 [2,5]
+  CRUSH rule 3 x 365 [6,4]
+  CRUSH rule 3 x 366 [7,0]
+  CRUSH rule 3 x 367 [4,2]
+  CRUSH rule 3 x 368 [7,3]
+  CRUSH rule 3 x 369 [3,7]
+  CRUSH rule 3 x 370 [8,2]
+  CRUSH rule 3 x 371 [1,5]
+  CRUSH rule 3 x 372 [3,1]
+  CRUSH rule 3 x 373 [0,6]
+  CRUSH rule 3 x 374 [3,8]
+  CRUSH rule 3 x 375 [6,5]
+  CRUSH rule 3 x 376 [7,1]
+  CRUSH rule 3 x 377 [1,4]
+  CRUSH rule 3 x 378 [0,6]
+  CRUSH rule 3 x 379 [8,3]
+  CRUSH rule 3 x 380 [2,3]
+  CRUSH rule 3 x 381 [0,3]
+  CRUSH rule 3 x 382 [1,3]
+  CRUSH rule 3 x 383 [4,7]
+  CRUSH rule 3 x 384 [7,0]
+  CRUSH rule 3 x 385 [7,4]
+  CRUSH rule 3 x 386 [0,4]
+  CRUSH rule 3 x 387 [1,5]
+  CRUSH rule 3 x 388 [5,2]
+  CRUSH rule 3 x 389 [1,4]
+  CRUSH rule 3 x 390 [5,8]
+  CRUSH rule 3 x 391 [5,6]
+  CRUSH rule 3 x 392 [1,7]
+  CRUSH rule 3 x 393 [4,0]
+  CRUSH rule 3 x 394 [4,8]
+  CRUSH rule 3 x 395 [4,0]
+  CRUSH rule 3 x 396 [4,2]
+  CRUSH rule 3 x 397 [2,4]
+  CRUSH rule 3 x 398 [2,3]
+  CRUSH rule 3 x 399 [8,5]
+  CRUSH rule 3 x 400 [8,1]
+  CRUSH rule 3 x 401 [0,3]
+  CRUSH rule 3 x 402 [7,4]
+  CRUSH rule 3 x 403 [0,4]
+  CRUSH rule 3 x 404 [4,0]
+  CRUSH rule 3 x 405 [6,4]
+  CRUSH rule 3 x 406 [2,6]
+  CRUSH rule 3 x 407 [2,7]
+  CRUSH rule 3 x 408 [4,1]
+  CRUSH rule 3 x 409 [7,5]
+  CRUSH rule 3 x 410 [8,3]
+  CRUSH rule 3 x 411 [2,7]
+  CRUSH rule 3 x 412 [0,3]
+  CRUSH rule 3 x 413 [5,2]
+  CRUSH rule 3 x 414 [4,0]
+  CRUSH rule 3 x 415 [0,6]
+  CRUSH rule 3 x 416 [2,8]
+  CRUSH rule 3 x 417 [8,0]
+  CRUSH rule 3 x 418 [7,1]
+  CRUSH rule 3 x 419 [8,5]
+  CRUSH rule 3 x 420 [1,4]
+  CRUSH rule 3 x 421 [8,3]
+  CRUSH rule 3 x 422 [6,3]
+  CRUSH rule 3 x 423 [0,3]
+  CRUSH rule 3 x 424 [8,5]
+  CRUSH rule 3 x 425 [1,4]
+  CRUSH rule 3 x 426 [6,2]
+  CRUSH rule 3 x 427 [0,8]
+  CRUSH rule 3 x 428 [5,8]
+  CRUSH rule 3 x 429 [4,8]
+  CRUSH rule 3 x 430 [3,7]
+  CRUSH rule 3 x 431 [5,0]
+  CRUSH rule 3 x 432 [7,0]
+  CRUSH rule 3 x 433 [6,4]
+  CRUSH rule 3 x 434 [5,0]
+  CRUSH rule 3 x 435 [0,4]
+  CRUSH rule 3 x 436 [4,0]
+  CRUSH rule 3 x 437 [7,3]
+  CRUSH rule 3 x 438 [0,5]
+  CRUSH rule 3 x 439 [1,3]
+  CRUSH rule 3 x 440 [2,6]
+  CRUSH rule 3 x 441 [5,8]
+  CRUSH rule 3 x 442 [2,3]
+  CRUSH rule 3 x 443 [6,0]
+  CRUSH rule 3 x 444 [7,1]
+  CRUSH rule 3 x 445 [6,4]
+  CRUSH rule 3 x 446 [4,0]
+  CRUSH rule 3 x 447 [2,4]
+  CRUSH rule 3 x 448 [7,0]
+  CRUSH rule 3 x 449 [7,4]
+  CRUSH rule 3 x 450 [4,0]
+  CRUSH rule 3 x 451 [6,5]
+  CRUSH rule 3 x 452 [8,4]
+  CRUSH rule 3 x 453 [6,5]
+  CRUSH rule 3 x 454 [6,5]
+  CRUSH rule 3 x 455 [2,8]
+  CRUSH rule 3 x 456 [6,2]
+  CRUSH rule 3 x 457 [7,1]
+  CRUSH rule 3 x 458 [2,8]
+  CRUSH rule 3 x 459 [2,6]
+  CRUSH rule 3 x 460 [6,5]
+  CRUSH rule 3 x 461 [6,4]
+  CRUSH rule 3 x 462 [8,0]
+  CRUSH rule 3 x 463 [6,0]
+  CRUSH rule 3 x 464 [7,4]
+  CRUSH rule 3 x 465 [7,1]
+  CRUSH rule 3 x 466 [5,6]
+  CRUSH rule 3 x 467 [6,5]
+  CRUSH rule 3 x 468 [7,2]
+  CRUSH rule 3 x 469 [7,2]
+  CRUSH rule 3 x 470 [3,0]
+  CRUSH rule 3 x 471 [0,6]
+  CRUSH rule 3 x 472 [5,0]
+  CRUSH rule 3 x 473 [1,4]
+  CRUSH rule 3 x 474 [6,1]
+  CRUSH rule 3 x 475 [6,2]
+  CRUSH rule 3 x 476 [4,7]
+  CRUSH rule 3 x 477 [5,6]
+  CRUSH rule 3 x 478 [6,1]
+  CRUSH rule 3 x 479 [0,3]
+  CRUSH rule 3 x 480 [1,6]
+  CRUSH rule 3 x 481 [2,5]
+  CRUSH rule 3 x 482 [4,8]
+  CRUSH rule 3 x 483 [0,6]
+  CRUSH rule 3 x 484 [1,8]
+  CRUSH rule 3 x 485 [4,8]
+  CRUSH rule 3 x 486 [4,0]
+  CRUSH rule 3 x 487 [5,0]
+  CRUSH rule 3 x 488 [5,7]
+  CRUSH rule 3 x 489 [2,8]
+  CRUSH rule 3 x 490 [6,4]
+  CRUSH rule 3 x 491 [1,6]
+  CRUSH rule 3 x 492 [6,5]
+  CRUSH rule 3 x 493 [0,8]
+  CRUSH rule 3 x 494 [1,6]
+  CRUSH rule 3 x 495 [3,0]
+  CRUSH rule 3 x 496 [7,5]
+  CRUSH rule 3 x 497 [5,7]
+  CRUSH rule 3 x 498 [0,4]
+  CRUSH rule 3 x 499 [8,5]
+  CRUSH rule 3 x 500 [3,6]
+  CRUSH rule 3 x 501 [0,7]
+  CRUSH rule 3 x 502 [7,1]
+  CRUSH rule 3 x 503 [2,3]
+  CRUSH rule 3 x 504 [5,8]
+  CRUSH rule 3 x 505 [0,7]
+  CRUSH rule 3 x 506 [5,1]
+  CRUSH rule 3 x 507 [6,0]
+  CRUSH rule 3 x 508 [0,4]
+  CRUSH rule 3 x 509 [7,4]
+  CRUSH rule 3 x 510 [6,2]
+  CRUSH rule 3 x 511 [5,6]
+  CRUSH rule 3 x 512 [7,2]
+  CRUSH rule 3 x 513 [7,2]
+  CRUSH rule 3 x 514 [4,7]
+  CRUSH rule 3 x 515 [8,3]
+  CRUSH rule 3 x 516 [4,1]
+  CRUSH rule 3 x 517 [7,0]
+  CRUSH rule 3 x 518 [4,6]
+  CRUSH rule 3 x 519 [7,3]
+  CRUSH rule 3 x 520 [2,8]
+  CRUSH rule 3 x 521 [8,0]
+  CRUSH rule 3 x 522 [6,0]
+  CRUSH rule 3 x 523 [4,1]
+  CRUSH rule 3 x 524 [0,4]
+  CRUSH rule 3 x 525 [0,3]
+  CRUSH rule 3 x 526 [1,3]
+  CRUSH rule 3 x 527 [0,4]
+  CRUSH rule 3 x 528 [5,2]
+  CRUSH rule 3 x 529 [5,6]
+  CRUSH rule 3 x 530 [6,4]
+  CRUSH rule 3 x 531 [6,0]
+  CRUSH rule 3 x 532 [6,5]
+  CRUSH rule 3 x 533 [5,8]
+  CRUSH rule 3 x 534 [7,4]
+  CRUSH rule 3 x 535 [8,0]
+  CRUSH rule 3 x 536 [6,2]
+  CRUSH rule 3 x 537 [3,8]
+  CRUSH rule 3 x 538 [6,4]
+  CRUSH rule 3 x 539 [8,4]
+  CRUSH rule 3 x 540 [0,7]
+  CRUSH rule 3 x 541 [2,5]
+  CRUSH rule 3 x 542 [3,0]
+  CRUSH rule 3 x 543 [6,2]
+  CRUSH rule 3 x 544 [3,7]
+  CRUSH rule 3 x 545 [5,7]
+  CRUSH rule 3 x 546 [6,2]
+  CRUSH rule 3 x 547 [8,1]
+  CRUSH rule 3 x 548 [5,1]
+  CRUSH rule 3 x 549 [5,7]
+  CRUSH rule 3 x 550 [0,5]
+  CRUSH rule 3 x 551 [7,4]
+  CRUSH rule 3 x 552 [5,7]
+  CRUSH rule 3 x 553 [4,2]
+  CRUSH rule 3 x 554 [0,6]
+  CRUSH rule 3 x 555 [5,0]
+  CRUSH rule 3 x 556 [3,6]
+  CRUSH rule 3 x 557 [7,4]
+  CRUSH rule 3 x 558 [3,2]
+  CRUSH rule 3 x 559 [4,1]
+  CRUSH rule 3 x 560 [8,4]
+  CRUSH rule 3 x 561 [6,4]
+  CRUSH rule 3 x 562 [3,0]
+  CRUSH rule 3 x 563 [2,7]
+  CRUSH rule 3 x 564 [5,1]
+  CRUSH rule 3 x 565 [3,8]
+  CRUSH rule 3 x 566 [4,6]
+  CRUSH rule 3 x 567 [3,7]
+  CRUSH rule 3 x 568 [7,4]
+  CRUSH rule 3 x 569 [3,1]
+  CRUSH rule 3 x 570 [1,4]
+  CRUSH rule 3 x 571 [3,6]
+  CRUSH rule 3 x 572 [3,0]
+  CRUSH rule 3 x 573 [3,1]
+  CRUSH rule 3 x 574 [2,3]
+  CRUSH rule 3 x 575 [8,1]
+  CRUSH rule 3 x 576 [4,7]
+  CRUSH rule 3 x 577 [8,2]
+  CRUSH rule 3 x 578 [6,0]
+  CRUSH rule 3 x 579 [3,2]
+  CRUSH rule 3 x 580 [3,0]
+  CRUSH rule 3 x 581 [7,1]
+  CRUSH rule 3 x 582 [2,8]
+  CRUSH rule 3 x 583 [6,2]
+  CRUSH rule 3 x 584 [8,0]
+  CRUSH rule 3 x 585 [7,0]
+  CRUSH rule 3 x 586 [0,8]
+  CRUSH rule 3 x 587 [2,5]
+  CRUSH rule 3 x 588 [3,8]
+  CRUSH rule 3 x 589 [7,1]
+  CRUSH rule 3 x 590 [6,2]
+  CRUSH rule 3 x 591 [5,2]
+  CRUSH rule 3 x 592 [2,4]
+  CRUSH rule 3 x 593 [0,8]
+  CRUSH rule 3 x 594 [0,6]
+  CRUSH rule 3 x 595 [7,1]
+  CRUSH rule 3 x 596 [4,0]
+  CRUSH rule 3 x 597 [3,1]
+  CRUSH rule 3 x 598 [3,0]
+  CRUSH rule 3 x 599 [5,1]
+  CRUSH rule 3 x 600 [7,0]
+  CRUSH rule 3 x 601 [0,6]
+  CRUSH rule 3 x 602 [3,8]
+  CRUSH rule 3 x 603 [5,0]
+  CRUSH rule 3 x 604 [7,4]
+  CRUSH rule 3 x 605 [3,2]
+  CRUSH rule 3 x 606 [2,7]
+  CRUSH rule 3 x 607 [0,5]
+  CRUSH rule 3 x 608 [5,1]
+  CRUSH rule 3 x 609 [5,0]
+  CRUSH rule 3 x 610 [3,8]
+  CRUSH rule 3 x 611 [1,8]
+  CRUSH rule 3 x 612 [2,8]
+  CRUSH rule 3 x 613 [7,1]
+  CRUSH rule 3 x 614 [7,2]
+  CRUSH rule 3 x 615 [6,2]
+  CRUSH rule 3 x 616 [0,7]
+  CRUSH rule 3 x 617 [6,2]
+  CRUSH rule 3 x 618 [7,3]
+  CRUSH rule 3 x 619 [5,0]
+  CRUSH rule 3 x 620 [4,1]
+  CRUSH rule 3 x 621 [5,6]
+  CRUSH rule 3 x 622 [0,3]
+  CRUSH rule 3 x 623 [0,8]
+  CRUSH rule 3 x 624 [3,2]
+  CRUSH rule 3 x 625 [2,5]
+  CRUSH rule 3 x 626 [7,2]
+  CRUSH rule 3 x 627 [2,6]
+  CRUSH rule 3 x 628 [8,1]
+  CRUSH rule 3 x 629 [2,6]
+  CRUSH rule 3 x 630 [2,6]
+  CRUSH rule 3 x 631 [0,6]
+  CRUSH rule 3 x 632 [7,0]
+  CRUSH rule 3 x 633 [8,4]
+  CRUSH rule 3 x 634 [0,5]
+  CRUSH rule 3 x 635 [5,6]
+  CRUSH rule 3 x 636 [1,3]
+  CRUSH rule 3 x 637 [4,0]
+  CRUSH rule 3 x 638 [6,2]
+  CRUSH rule 3 x 639 [4,0]
+  CRUSH rule 3 x 640 [3,2]
+  CRUSH rule 3 x 641 [7,2]
+  CRUSH rule 3 x 642 [2,8]
+  CRUSH rule 3 x 643 [3,0]
+  CRUSH rule 3 x 644 [8,0]
+  CRUSH rule 3 x 645 [5,7]
+  CRUSH rule 3 x 646 [8,0]
+  CRUSH rule 3 x 647 [7,0]
+  CRUSH rule 3 x 648 [0,8]
+  CRUSH rule 3 x 649 [4,7]
+  CRUSH rule 3 x 650 [7,5]
+  CRUSH rule 3 x 651 [3,6]
+  CRUSH rule 3 x 652 [3,6]
+  CRUSH rule 3 x 653 [8,3]
+  CRUSH rule 3 x 654 [7,4]
+  CRUSH rule 3 x 655 [0,5]
+  CRUSH rule 3 x 656 [4,7]
+  CRUSH rule 3 x 657 [6,0]
+  CRUSH rule 3 x 658 [5,8]
+  CRUSH rule 3 x 659 [4,7]
+  CRUSH rule 3 x 660 [7,3]
+  CRUSH rule 3 x 661 [1,7]
+  CRUSH rule 3 x 662 [4,2]
+  CRUSH rule 3 x 663 [1,3]
+  CRUSH rule 3 x 664 [1,3]
+  CRUSH rule 3 x 665 [5,6]
+  CRUSH rule 3 x 666 [2,7]
+  CRUSH rule 3 x 667 [1,3]
+  CRUSH rule 3 x 668 [3,7]
+  CRUSH rule 3 x 669 [6,3]
+  CRUSH rule 3 x 670 [4,1]
+  CRUSH rule 3 x 671 [0,8]
+  CRUSH rule 3 x 672 [4,2]
+  CRUSH rule 3 x 673 [5,0]
+  CRUSH rule 3 x 674 [3,0]
+  CRUSH rule 3 x 675 [0,8]
+  CRUSH rule 3 x 676 [0,3]
+  CRUSH rule 3 x 677 [4,1]
+  CRUSH rule 3 x 678 [2,3]
+  CRUSH rule 3 x 679 [6,0]
+  CRUSH rule 3 x 680 [0,3]
+  CRUSH rule 3 x 681 [4,6]
+  CRUSH rule 3 x 682 [0,4]
+  CRUSH rule 3 x 683 [0,3]
+  CRUSH rule 3 x 684 [7,2]
+  CRUSH rule 3 x 685 [7,2]
+  CRUSH rule 3 x 686 [1,5]
+  CRUSH rule 3 x 687 [3,7]
+  CRUSH rule 3 x 688 [5,6]
+  CRUSH rule 3 x 689 [6,5]
+  CRUSH rule 3 x 690 [8,0]
+  CRUSH rule 3 x 691 [3,1]
+  CRUSH rule 3 x 692 [7,1]
+  CRUSH rule 3 x 693 [6,5]
+  CRUSH rule 3 x 694 [6,4]
+  CRUSH rule 3 x 695 [0,6]
+  CRUSH rule 3 x 696 [1,5]
+  CRUSH rule 3 x 697 [6,0]
+  CRUSH rule 3 x 698 [6,0]
+  CRUSH rule 3 x 699 [1,8]
+  CRUSH rule 3 x 700 [0,4]
+  CRUSH rule 3 x 701 [4,0]
+  CRUSH rule 3 x 702 [3,0]
+  CRUSH rule 3 x 703 [8,4]
+  CRUSH rule 3 x 704 [0,4]
+  CRUSH rule 3 x 705 [8,0]
+  CRUSH rule 3 x 706 [1,5]
+  CRUSH rule 3 x 707 [7,3]
+  CRUSH rule 3 x 708 [3,7]
+  CRUSH rule 3 x 709 [6,5]
+  CRUSH rule 3 x 710 [8,5]
+  CRUSH rule 3 x 711 [2,4]
+  CRUSH rule 3 x 712 [2,3]
+  CRUSH rule 3 x 713 [6,3]
+  CRUSH rule 3 x 714 [3,0]
+  CRUSH rule 3 x 715 [1,3]
+  CRUSH rule 3 x 716 [3,6]
+  CRUSH rule 3 x 717 [8,0]
+  CRUSH rule 3 x 718 [3,7]
+  CRUSH rule 3 x 719 [2,6]
+  CRUSH rule 3 x 720 [6,0]
+  CRUSH rule 3 x 721 [5,7]
+  CRUSH rule 3 x 722 [5,7]
+  CRUSH rule 3 x 723 [5,2]
+  CRUSH rule 3 x 724 [0,7]
+  CRUSH rule 3 x 725 [0,4]
+  CRUSH rule 3 x 726 [3,7]
+  CRUSH rule 3 x 727 [4,7]
+  CRUSH rule 3 x 728 [2,6]
+  CRUSH rule 3 x 729 [5,6]
+  CRUSH rule 3 x 730 [3,8]
+  CRUSH rule 3 x 731 [4,1]
+  CRUSH rule 3 x 732 [1,4]
+  CRUSH rule 3 x 733 [5,6]
+  CRUSH rule 3 x 734 [6,5]
+  CRUSH rule 3 x 735 [4,6]
+  CRUSH rule 3 x 736 [3,8]
+  CRUSH rule 3 x 737 [1,8]
+  CRUSH rule 3 x 738 [5,1]
+  CRUSH rule 3 x 739 [0,7]
+  CRUSH rule 3 x 740 [0,7]
+  CRUSH rule 3 x 741 [7,2]
+  CRUSH rule 3 x 742 [8,2]
+  CRUSH rule 3 x 743 [7,0]
+  CRUSH rule 3 x 744 [4,6]
+  CRUSH rule 3 x 745 [3,0]
+  CRUSH rule 3 x 746 [4,1]
+  CRUSH rule 3 x 747 [6,2]
+  CRUSH rule 3 x 748 [2,6]
+  CRUSH rule 3 x 749 [4,7]
+  CRUSH rule 3 x 750 [1,7]
+  CRUSH rule 3 x 751 [2,7]
+  CRUSH rule 3 x 752 [8,0]
+  CRUSH rule 3 x 753 [7,4]
+  CRUSH rule 3 x 754 [8,5]
+  CRUSH rule 3 x 755 [1,6]
+  CRUSH rule 3 x 756 [5,8]
+  CRUSH rule 3 x 757 [8,0]
+  CRUSH rule 3 x 758 [6,1]
+  CRUSH rule 3 x 759 [8,5]
+  CRUSH rule 3 x 760 [1,5]
+  CRUSH rule 3 x 761 [4,2]
+  CRUSH rule 3 x 762 [2,8]
+  CRUSH rule 3 x 763 [8,3]
+  CRUSH rule 3 x 764 [1,7]
+  CRUSH rule 3 x 765 [6,4]
+  CRUSH rule 3 x 766 [8,3]
+  CRUSH rule 3 x 767 [1,8]
+  CRUSH rule 3 x 768 [8,5]
+  CRUSH rule 3 x 769 [6,2]
+  CRUSH rule 3 x 770 [6,1]
+  CRUSH rule 3 x 771 [7,2]
+  CRUSH rule 3 x 772 [8,4]
+  CRUSH rule 3 x 773 [3,2]
+  CRUSH rule 3 x 774 [4,7]
+  CRUSH rule 3 x 775 [6,5]
+  CRUSH rule 3 x 776 [7,0]
+  CRUSH rule 3 x 777 [3,0]
+  CRUSH rule 3 x 778 [1,6]
+  CRUSH rule 3 x 779 [2,6]
+  CRUSH rule 3 x 780 [0,5]
+  CRUSH rule 3 x 781 [6,5]
+  CRUSH rule 3 x 782 [5,2]
+  CRUSH rule 3 x 783 [7,0]
+  CRUSH rule 3 x 784 [0,5]
+  CRUSH rule 3 x 785 [6,1]
+  CRUSH rule 3 x 786 [7,3]
+  CRUSH rule 3 x 787 [1,8]
+  CRUSH rule 3 x 788 [6,0]
+  CRUSH rule 3 x 789 [0,5]
+  CRUSH rule 3 x 790 [8,3]
+  CRUSH rule 3 x 791 [3,6]
+  CRUSH rule 3 x 792 [5,6]
+  CRUSH rule 3 x 793 [6,2]
+  CRUSH rule 3 x 794 [2,8]
+  CRUSH rule 3 x 795 [0,4]
+  CRUSH rule 3 x 796 [3,7]
+  CRUSH rule 3 x 797 [2,3]
+  CRUSH rule 3 x 798 [6,0]
+  CRUSH rule 3 x 799 [5,2]
+  CRUSH rule 3 x 800 [5,2]
+  CRUSH rule 3 x 801 [3,7]
+  CRUSH rule 3 x 802 [1,6]
+  CRUSH rule 3 x 803 [0,4]
+  CRUSH rule 3 x 804 [6,0]
+  CRUSH rule 3 x 805 [3,8]
+  CRUSH rule 3 x 806 [1,5]
+  CRUSH rule 3 x 807 [5,7]
+  CRUSH rule 3 x 808 [4,7]
+  CRUSH rule 3 x 809 [1,3]
+  CRUSH rule 3 x 810 [5,7]
+  CRUSH rule 3 x 811 [8,4]
+  CRUSH rule 3 x 812 [8,3]
+  CRUSH rule 3 x 813 [6,4]
+  CRUSH rule 3 x 814 [3,8]
+  CRUSH rule 3 x 815 [3,1]
+  CRUSH rule 3 x 816 [2,6]
+  CRUSH rule 3 x 817 [4,6]
+  CRUSH rule 3 x 818 [3,1]
+  CRUSH rule 3 x 819 [5,0]
+  CRUSH rule 3 x 820 [3,7]
+  CRUSH rule 3 x 821 [4,8]
+  CRUSH rule 3 x 822 [2,3]
+  CRUSH rule 3 x 823 [4,7]
+  CRUSH rule 3 x 824 [3,7]
+  CRUSH rule 3 x 825 [2,8]
+  CRUSH rule 3 x 826 [7,1]
+  CRUSH rule 3 x 827 [0,6]
+  CRUSH rule 3 x 828 [2,5]
+  CRUSH rule 3 x 829 [5,6]
+  CRUSH rule 3 x 830 [2,4]
+  CRUSH rule 3 x 831 [1,6]
+  CRUSH rule 3 x 832 [4,8]
+  CRUSH rule 3 x 833 [2,6]
+  CRUSH rule 3 x 834 [3,0]
+  CRUSH rule 3 x 835 [8,5]
+  CRUSH rule 3 x 836 [3,8]
+  CRUSH rule 3 x 837 [6,4]
+  CRUSH rule 3 x 838 [6,0]
+  CRUSH rule 3 x 839 [5,2]
+  CRUSH rule 3 x 840 [7,3]
+  CRUSH rule 3 x 841 [4,8]
+  CRUSH rule 3 x 842 [2,5]
+  CRUSH rule 3 x 843 [6,4]
+  CRUSH rule 3 x 844 [4,8]
+  CRUSH rule 3 x 845 [3,6]
+  CRUSH rule 3 x 846 [3,0]
+  CRUSH rule 3 x 847 [0,8]
+  CRUSH rule 3 x 848 [2,6]
+  CRUSH rule 3 x 849 [4,8]
+  CRUSH rule 3 x 850 [1,5]
+  CRUSH rule 3 x 851 [6,5]
+  CRUSH rule 3 x 852 [7,4]
+  CRUSH rule 3 x 853 [6,2]
+  CRUSH rule 3 x 854 [7,0]
+  CRUSH rule 3 x 855 [5,7]
+  CRUSH rule 3 x 856 [6,3]
+  CRUSH rule 3 x 857 [8,4]
+  CRUSH rule 3 x 858 [6,5]
+  CRUSH rule 3 x 859 [6,0]
+  CRUSH rule 3 x 860 [4,2]
+  CRUSH rule 3 x 861 [8,4]
+  CRUSH rule 3 x 862 [6,0]
+  CRUSH rule 3 x 863 [8,1]
+  CRUSH rule 3 x 864 [5,6]
+  CRUSH rule 3 x 865 [8,0]
+  CRUSH rule 3 x 866 [3,6]
+  CRUSH rule 3 x 867 [6,3]
+  CRUSH rule 3 x 868 [6,4]
+  CRUSH rule 3 x 869 [8,4]
+  CRUSH rule 3 x 870 [0,5]
+  CRUSH rule 3 x 871 [3,1]
+  CRUSH rule 3 x 872 [5,0]
+  CRUSH rule 3 x 873 [4,6]
+  CRUSH rule 3 x 874 [2,6]
+  CRUSH rule 3 x 875 [2,6]
+  CRUSH rule 3 x 876 [5,8]
+  CRUSH rule 3 x 877 [6,4]
+  CRUSH rule 3 x 878 [5,0]
+  CRUSH rule 3 x 879 [7,3]
+  CRUSH rule 3 x 880 [3,1]
+  CRUSH rule 3 x 881 [5,8]
+  CRUSH rule 3 x 882 [4,1]
+  CRUSH rule 3 x 883 [2,5]
+  CRUSH rule 3 x 884 [6,0]
+  CRUSH rule 3 x 885 [5,1]
+  CRUSH rule 3 x 886 [3,7]
+  CRUSH rule 3 x 887 [7,5]
+  CRUSH rule 3 x 888 [6,1]
+  CRUSH rule 3 x 889 [2,8]
+  CRUSH rule 3 x 890 [7,1]
+  CRUSH rule 3 x 891 [1,7]
+  CRUSH rule 3 x 892 [6,0]
+  CRUSH rule 3 x 893 [2,3]
+  CRUSH rule 3 x 894 [7,4]
+  CRUSH rule 3 x 895 [5,1]
+  CRUSH rule 3 x 896 [1,6]
+  CRUSH rule 3 x 897 [4,0]
+  CRUSH rule 3 x 898 [0,3]
+  CRUSH rule 3 x 899 [1,6]
+  CRUSH rule 3 x 900 [4,0]
+  CRUSH rule 3 x 901 [5,2]
+  CRUSH rule 3 x 902 [8,4]
+  CRUSH rule 3 x 903 [5,6]
+  CRUSH rule 3 x 904 [5,7]
+  CRUSH rule 3 x 905 [6,0]
+  CRUSH rule 3 x 906 [1,7]
+  CRUSH rule 3 x 907 [7,2]
+  CRUSH rule 3 x 908 [5,6]
+  CRUSH rule 3 x 909 [2,3]
+  CRUSH rule 3 x 910 [6,5]
+  CRUSH rule 3 x 911 [5,7]
+  CRUSH rule 3 x 912 [0,8]
+  CRUSH rule 3 x 913 [7,1]
+  CRUSH rule 3 x 914 [6,4]
+  CRUSH rule 3 x 915 [8,2]
+  CRUSH rule 3 x 916 [3,0]
+  CRUSH rule 3 x 917 [1,3]
+  CRUSH rule 3 x 918 [8,0]
+  CRUSH rule 3 x 919 [6,2]
+  CRUSH rule 3 x 920 [7,4]
+  CRUSH rule 3 x 921 [1,4]
+  CRUSH rule 3 x 922 [6,3]
+  CRUSH rule 3 x 923 [5,6]
+  CRUSH rule 3 x 924 [3,1]
+  CRUSH rule 3 x 925 [5,6]
+  CRUSH rule 3 x 926 [3,0]
+  CRUSH rule 3 x 927 [1,6]
+  CRUSH rule 3 x 928 [8,1]
+  CRUSH rule 3 x 929 [4,2]
+  CRUSH rule 3 x 930 [2,5]
+  CRUSH rule 3 x 931 [5,2]
+  CRUSH rule 3 x 932 [4,2]
+  CRUSH rule 3 x 933 [8,4]
+  CRUSH rule 3 x 934 [5,8]
+  CRUSH rule 3 x 935 [6,3]
+  CRUSH rule 3 x 936 [0,7]
+  CRUSH rule 3 x 937 [5,8]
+  CRUSH rule 3 x 938 [6,4]
+  CRUSH rule 3 x 939 [2,8]
+  CRUSH rule 3 x 940 [8,5]
+  CRUSH rule 3 x 941 [5,0]
+  CRUSH rule 3 x 942 [1,6]
+  CRUSH rule 3 x 943 [8,2]
+  CRUSH rule 3 x 944 [4,8]
+  CRUSH rule 3 x 945 [7,0]
+  CRUSH rule 3 x 946 [2,8]
+  CRUSH rule 3 x 947 [4,2]
+  CRUSH rule 3 x 948 [7,4]
+  CRUSH rule 3 x 949 [6,2]
+  CRUSH rule 3 x 950 [3,7]
+  CRUSH rule 3 x 951 [4,6]
+  CRUSH rule 3 x 952 [2,7]
+  CRUSH rule 3 x 953 [1,3]
+  CRUSH rule 3 x 954 [4,0]
+  CRUSH rule 3 x 955 [8,1]
+  CRUSH rule 3 x 956 [1,7]
+  CRUSH rule 3 x 957 [7,0]
+  CRUSH rule 3 x 958 [8,3]
+  CRUSH rule 3 x 959 [5,1]
+  CRUSH rule 3 x 960 [3,6]
+  CRUSH rule 3 x 961 [4,1]
+  CRUSH rule 3 x 962 [7,5]
+  CRUSH rule 3 x 963 [0,5]
+  CRUSH rule 3 x 964 [3,2]
+  CRUSH rule 3 x 965 [7,3]
+  CRUSH rule 3 x 966 [3,6]
+  CRUSH rule 3 x 967 [8,4]
+  CRUSH rule 3 x 968 [7,0]
+  CRUSH rule 3 x 969 [8,0]
+  CRUSH rule 3 x 970 [0,8]
+  CRUSH rule 3 x 971 [1,8]
+  CRUSH rule 3 x 972 [1,7]
+  CRUSH rule 3 x 973 [1,8]
+  CRUSH rule 3 x 974 [5,1]
+  CRUSH rule 3 x 975 [3,8]
+  CRUSH rule 3 x 976 [4,7]
+  CRUSH rule 3 x 977 [8,3]
+  CRUSH rule 3 x 978 [7,0]
+  CRUSH rule 3 x 979 [7,2]
+  CRUSH rule 3 x 980 [6,2]
+  CRUSH rule 3 x 981 [7,5]
+  CRUSH rule 3 x 982 [4,1]
+  CRUSH rule 3 x 983 [3,6]
+  CRUSH rule 3 x 984 [0,8]
+  CRUSH rule 3 x 985 [2,4]
+  CRUSH rule 3 x 986 [8,4]
+  CRUSH rule 3 x 987 [0,4]
+  CRUSH rule 3 x 988 [1,4]
+  CRUSH rule 3 x 989 [0,8]
+  CRUSH rule 3 x 990 [1,6]
+  CRUSH rule 3 x 991 [0,4]
+  CRUSH rule 3 x 992 [7,0]
+  CRUSH rule 3 x 993 [0,6]
+  CRUSH rule 3 x 994 [3,6]
+  CRUSH rule 3 x 995 [7,0]
+  CRUSH rule 3 x 996 [6,5]
+  CRUSH rule 3 x 997 [6,3]
+  CRUSH rule 3 x 998 [8,0]
+  CRUSH rule 3 x 999 [0,7]
+  CRUSH rule 3 x 1000 [8,4]
+  CRUSH rule 3 x 1001 [2,3]
+  CRUSH rule 3 x 1002 [1,3]
+  CRUSH rule 3 x 1003 [2,7]
+  CRUSH rule 3 x 1004 [6,0]
+  CRUSH rule 3 x 1005 [6,1]
+  CRUSH rule 3 x 1006 [1,8]
+  CRUSH rule 3 x 1007 [1,3]
+  CRUSH rule 3 x 1008 [1,7]
+  CRUSH rule 3 x 1009 [6,5]
+  CRUSH rule 3 x 1010 [3,1]
+  CRUSH rule 3 x 1011 [3,0]
+  CRUSH rule 3 x 1012 [3,1]
+  CRUSH rule 3 x 1013 [5,2]
+  CRUSH rule 3 x 1014 [2,8]
+  CRUSH rule 3 x 1015 [6,3]
+  CRUSH rule 3 x 1016 [2,5]
+  CRUSH rule 3 x 1017 [6,1]
+  CRUSH rule 3 x 1018 [5,1]
+  CRUSH rule 3 x 1019 [5,8]
+  CRUSH rule 3 x 1020 [5,0]
+  CRUSH rule 3 x 1021 [5,2]
+  CRUSH rule 3 x 1022 [1,7]
+  CRUSH rule 3 x 1023 [3,0]
+  rule 3 (choose-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [0,3,7]
+  CRUSH rule 3 x 1 [0,8,5]
+  CRUSH rule 3 x 2 [1,4,8]
+  CRUSH rule 3 x 3 [8,0,4]
+  CRUSH rule 3 x 4 [5,1,8]
+  CRUSH rule 3 x 5 [7,0,3]
+  CRUSH rule 3 x 6 [2,6,3]
+  CRUSH rule 3 x 7 [5,6,1]
+  CRUSH rule 3 x 8 [5,7,2]
+  CRUSH rule 3 x 9 [2,4,8]
+  CRUSH rule 3 x 10 [0,8,4]
+  CRUSH rule 3 x 11 [0,6,3]
+  CRUSH rule 3 x 12 [0,3,7]
+  CRUSH rule 3 x 13 [3,8,0]
+  CRUSH rule 3 x 14 [7,1,5]
+  CRUSH rule 3 x 15 [7,1,3]
+  CRUSH rule 3 x 16 [3,7,1]
+  CRUSH rule 3 x 17 [5,1,7]
+  CRUSH rule 3 x 18 [1,3,6]
+  CRUSH rule 3 x 19 [7,5,2]
+  CRUSH rule 3 x 20 [2,4,7]
+  CRUSH rule 3 x 21 [3,6,0]
+  CRUSH rule 3 x 22 [8,5,1]
+  CRUSH rule 3 x 23 [3,7,2]
+  CRUSH rule 3 x 24 [1,6,3]
+  CRUSH rule 3 x 25 [3,8,1]
+  CRUSH rule 3 x 26 [2,7,3]
+  CRUSH rule 3 x 27 [3,1,7]
+  CRUSH rule 3 x 28 [6,2,4]
+  CRUSH rule 3 x 29 [8,5,2]
+  CRUSH rule 3 x 30 [5,6,0]
+  CRUSH rule 3 x 31 [8,2,5]
+  CRUSH rule 3 x 32 [3,7,2]
+  CRUSH rule 3 x 33 [2,7,4]
+  CRUSH rule 3 x 34 [2,3,7]
+  CRUSH rule 3 x 35 [0,6,3]
+  CRUSH rule 3 x 36 [3,6,1]
+  CRUSH rule 3 x 37 [0,4,7]
+  CRUSH rule 3 x 38 [4,6,1]
+  CRUSH rule 3 x 39 [3,7,2]
+  CRUSH rule 3 x 40 [7,2,5]
+  CRUSH rule 3 x 41 [0,7,4]
+  CRUSH rule 3 x 42 [4,7,2]
+  CRUSH rule 3 x 43 [0,3,8]
+  CRUSH rule 3 x 44 [1,8,4]
+  CRUSH rule 3 x 45 [8,2,3]
+  CRUSH rule 3 x 46 [2,4,8]
+  CRUSH rule 3 x 47 [4,1,7]
+  CRUSH rule 3 x 48 [4,6,0]
+  CRUSH rule 3 x 49 [5,6,0]
+  CRUSH rule 3 x 50 [3,1,7]
+  CRUSH rule 3 x 51 [3,6,1]
+  CRUSH rule 3 x 52 [8,2,3]
+  CRUSH rule 3 x 53 [3,6,2]
+  CRUSH rule 3 x 54 [7,4,0]
+  CRUSH rule 3 x 55 [8,0,3]
+  CRUSH rule 3 x 56 [6,5,1]
+  CRUSH rule 3 x 57 [5,8,0]
+  CRUSH rule 3 x 58 [1,8,5]
+  CRUSH rule 3 x 59 [4,0,7]
+  CRUSH rule 3 x 60 [3,1,8]
+  CRUSH rule 3 x 61 [4,8,0]
+  CRUSH rule 3 x 62 [7,0,4]
+  CRUSH rule 3 x 63 [5,6,2]
+  CRUSH rule 3 x 64 [4,1,8]
+  CRUSH rule 3 x 65 [7,4,2]
+  CRUSH rule 3 x 66 [5,6,0]
+  CRUSH rule 3 x 67 [5,0,6]
+  CRUSH rule 3 x 68 [0,3,8]
+  CRUSH rule 3 x 69 [5,2,7]
+  CRUSH rule 3 x 70 [7,2,3]
+  CRUSH rule 3 x 71 [2,7,4]
+  CRUSH rule 3 x 72 [6,2,4]
+  CRUSH rule 3 x 73 [2,7,3]
+  CRUSH rule 3 x 74 [0,8,3]
+  CRUSH rule 3 x 75 [3,0,7]
+  CRUSH rule 3 x 76 [5,2,6]
+  CRUSH rule 3 x 77 [7,0,5]
+  CRUSH rule 3 x 78 [1,4,7]
+  CRUSH rule 3 x 79 [5,0,7]
+  CRUSH rule 3 x 80 [0,4,6]
+  CRUSH rule 3 x 81 [0,5,8]
+  CRUSH rule 3 x 82 [7,2,5]
+  CRUSH rule 3 x 83 [2,6,4]
+  CRUSH rule 3 x 84 [7,0,3]
+  CRUSH rule 3 x 85 [3,7,0]
+  CRUSH rule 3 x 86 [0,6,4]
+  CRUSH rule 3 x 87 [0,6,5]
+  CRUSH rule 3 x 88 [1,8,4]
+  CRUSH rule 3 x 89 [3,1,8]
+  CRUSH rule 3 x 90 [6,3,2]
+  CRUSH rule 3 x 91 [3,6,2]
+  CRUSH rule 3 x 92 [1,6,3]
+  CRUSH rule 3 x 93 [7,4,0]
+  CRUSH rule 3 x 94 [0,3,7]
+  CRUSH rule 3 x 95 [7,3,1]
+  CRUSH rule 3 x 96 [3,8,2]
+  CRUSH rule 3 x 97 [8,4,1]
+  CRUSH rule 3 x 98 [2,7,4]
+  CRUSH rule 3 x 99 [0,8,5]
+  CRUSH rule 3 x 100 [1,6,3]
+  CRUSH rule 3 x 101 [3,8,1]
+  CRUSH rule 3 x 102 [4,0,7]
+  CRUSH rule 3 x 103 [4,7,0]
+  CRUSH rule 3 x 104 [7,4,0]
+  CRUSH rule 3 x 105 [2,3,8]
+  CRUSH rule 3 x 106 [1,6,5]
+  CRUSH rule 3 x 107 [3,1,7]
+  CRUSH rule 3 x 108 [7,1,3]
+  CRUSH rule 3 x 109 [1,3,8]
+  CRUSH rule 3 x 110 [3,2,6]
+  CRUSH rule 3 x 111 [2,4,7]
+  CRUSH rule 3 x 112 [2,6,3]
+  CRUSH rule 3 x 113 [6,1,4]
+  CRUSH rule 3 x 114 [7,5,0]
+  CRUSH rule 3 x 115 [8,2,3]
+  CRUSH rule 3 x 116 [1,7,4]
+  CRUSH rule 3 x 117 [7,5,1]
+  CRUSH rule 3 x 118 [0,3,6]
+  CRUSH rule 3 x 119 [5,7,1]
+  CRUSH rule 3 x 120 [0,5,6]
+  CRUSH rule 3 x 121 [2,8,4]
+  CRUSH rule 3 x 122 [8,4,2]
+  CRUSH rule 3 x 123 [2,4,7]
+  CRUSH rule 3 x 124 [3,0,8]
+  CRUSH rule 3 x 125 [0,7,3]
+  CRUSH rule 3 x 126 [4,1,7]
+  CRUSH rule 3 x 127 [3,6,1]
+  CRUSH rule 3 x 128 [3,8,1]
+  CRUSH rule 3 x 129 [0,4,7]
+  CRUSH rule 3 x 130 [3,8,1]
+  CRUSH rule 3 x 131 [1,5,6]
+  CRUSH rule 3 x 132 [1,3,8]
+  CRUSH rule 3 x 133 [3,6,1]
+  CRUSH rule 3 x 134 [1,7,4]
+  CRUSH rule 3 x 135 [5,7,0]
+  CRUSH rule 3 x 136 [2,3,8]
+  CRUSH rule 3 x 137 [7,3,1]
+  CRUSH rule 3 x 138 [8,3,0]
+  CRUSH rule 3 x 139 [3,0,8]
+  CRUSH rule 3 x 140 [1,8,3]
+  CRUSH rule 3 x 141 [6,0,4]
+  CRUSH rule 3 x 142 [3,2,8]
+  CRUSH rule 3 x 143 [5,7,0]
+  CRUSH rule 3 x 144 [8,2,5]
+  CRUSH rule 3 x 145 [8,4,2]
+  CRUSH rule 3 x 146 [2,8,4]
+  CRUSH rule 3 x 147 [2,6,4]
+  CRUSH rule 3 x 148 [3,0,8]
+  CRUSH rule 3 x 149 [4,6,2]
+  CRUSH rule 3 x 150 [1,8,4]
+  CRUSH rule 3 x 151 [3,8,1]
+  CRUSH rule 3 x 152 [8,3,0]
+  CRUSH rule 3 x 153 [8,3,0]
+  CRUSH rule 3 x 154 [3,0,6]
+  CRUSH rule 3 x 155 [3,6,0]
+  CRUSH rule 3 x 156 [4,2,7]
+  CRUSH rule 3 x 157 [4,1,7]
+  CRUSH rule 3 x 158 [2,6,3]
+  CRUSH rule 3 x 159 [7,0,3]
+  CRUSH rule 3 x 160 [2,8,5]
+  CRUSH rule 3 x 161 [1,3,6]
+  CRUSH rule 3 x 162 [0,8,5]
+  CRUSH rule 3 x 163 [5,8,2]
+  CRUSH rule 3 x 164 [7,2,4]
+  CRUSH rule 3 x 165 [7,2,4]
+  CRUSH rule 3 x 166 [2,3,7]
+  CRUSH rule 3 x 167 [0,6,3]
+  CRUSH rule 3 x 168 [4,0,7]
+  CRUSH rule 3 x 169 [2,7,3]
+  CRUSH rule 3 x 170 [1,3,8]
+  CRUSH rule 3 x 171 [7,3,2]
+  CRUSH rule 3 x 172 [0,8,4]
+  CRUSH rule 3 x 173 [8,5,1]
+  CRUSH rule 3 x 174 [1,4,8]
+  CRUSH rule 3 x 175 [6,0,3]
+  CRUSH rule 3 x 176 [4,2,7]
+  CRUSH rule 3 x 177 [5,1,8]
+  CRUSH rule 3 x 178 [3,0,6]
+  CRUSH rule 3 x 179 [4,1,7]
+  CRUSH rule 3 x 180 [3,7,2]
+  CRUSH rule 3 x 181 [6,0,5]
+  CRUSH rule 3 x 182 [8,3,1]
+  CRUSH rule 3 x 183 [7,5,1]
+  CRUSH rule 3 x 184 [5,6,1]
+  CRUSH rule 3 x 185 [6,0,3]
+  CRUSH rule 3 x 186 [2,4,8]
+  CRUSH rule 3 x 187 [1,6,3]
+  CRUSH rule 3 x 188 [1,6,3]
+  CRUSH rule 3 x 189 [0,6,5]
+  CRUSH rule 3 x 190 [4,1,8]
+  CRUSH rule 3 x 191 [7,0,5]
+  CRUSH rule 3 x 192 [5,2,6]
+  CRUSH rule 3 x 193 [4,0,6]
+  CRUSH rule 3 x 194 [1,3,7]
+  CRUSH rule 3 x 195 [6,5,2]
+  CRUSH rule 3 x 196 [6,1,3]
+  CRUSH rule 3 x 197 [6,5,1]
+  CRUSH rule 3 x 198 [2,3,6]
+  CRUSH rule 3 x 199 [0,5,7]
+  CRUSH rule 3 x 200 [0,3,8]
+  CRUSH rule 3 x 201 [7,1,5]
+  CRUSH rule 3 x 202 [6,4,1]
+  CRUSH rule 3 x 203 [4,6,1]
+  CRUSH rule 3 x 204 [2,4,8]
+  CRUSH rule 3 x 205 [0,7,4]
+  CRUSH rule 3 x 206 [0,8,4]
+  CRUSH rule 3 x 207 [3,2,7]
+  CRUSH rule 3 x 208 [7,2,4]
+  CRUSH rule 3 x 209 [1,8,3]
+  CRUSH rule 3 x 210 [1,4,6]
+  CRUSH rule 3 x 211 [5,2,7]
+  CRUSH rule 3 x 212 [7,5,0]
+  CRUSH rule 3 x 213 [8,4,0]
+  CRUSH rule 3 x 214 [4,7,1]
+  CRUSH rule 3 x 215 [8,0,5]
+  CRUSH rule 3 x 216 [5,2,8]
+  CRUSH rule 3 x 217 [0,7,5]
+  CRUSH rule 3 x 218 [0,7,4]
+  CRUSH rule 3 x 219 [4,6,0]
+  CRUSH rule 3 x 220 [5,8,0]
+  CRUSH rule 3 x 221 [3,7,0]
+  CRUSH rule 3 x 222 [6,4,1]
+  CRUSH rule 3 x 223 [1,3,6]
+  CRUSH rule 3 x 224 [1,3,8]
+  CRUSH rule 3 x 225 [8,0,3]
+  CRUSH rule 3 x 226 [7,0,4]
+  CRUSH rule 3 x 227 [3,0,7]
+  CRUSH rule 3 x 228 [5,6,1]
+  CRUSH rule 3 x 229 [3,7,0]
+  CRUSH rule 3 x 230 [4,6,1]
+  CRUSH rule 3 x 231 [4,7,1]
+  CRUSH rule 3 x 232 [2,8,4]
+  CRUSH rule 3 x 233 [3,6,0]
+  CRUSH rule 3 x 234 [0,4,6]
+  CRUSH rule 3 x 235 [3,6,1]
+  CRUSH rule 3 x 236 [5,0,8]
+  CRUSH rule 3 x 237 [4,8,0]
+  CRUSH rule 3 x 238 [4,2,6]
+  CRUSH rule 3 x 239 [8,3,2]
+  CRUSH rule 3 x 240 [5,8,2]
+  CRUSH rule 3 x 241 [3,1,7]
+  CRUSH rule 3 x 242 [3,2,6]
+  CRUSH rule 3 x 243 [4,8,2]
+  CRUSH rule 3 x 244 [4,6,0]
+  CRUSH rule 3 x 245 [7,2,3]
+  CRUSH rule 3 x 246 [1,3,7]
+  CRUSH rule 3 x 247 [6,1,4]
+  CRUSH rule 3 x 248 [8,0,3]
+  CRUSH rule 3 x 249 [2,5,8]
+  CRUSH rule 3 x 250 [2,3,8]
+  CRUSH rule 3 x 251 [2,3,7]
+  CRUSH rule 3 x 252 [3,7,2]
+  CRUSH rule 3 x 253 [3,0,7]
+  CRUSH rule 3 x 254 [3,2,8]
+  CRUSH rule 3 x 255 [1,7,4]
+  CRUSH rule 3 x 256 [5,6,1]
+  CRUSH rule 3 x 257 [2,6,4]
+  CRUSH rule 3 x 258 [5,2,8]
+  CRUSH rule 3 x 259 [4,6,1]
+  CRUSH rule 3 x 260 [3,8,1]
+  CRUSH rule 3 x 261 [8,5,2]
+  CRUSH rule 3 x 262 [5,6,2]
+  CRUSH rule 3 x 263 [6,1,5]
+  CRUSH rule 3 x 264 [3,6,1]
+  CRUSH rule 3 x 265 [8,4,0]
+  CRUSH rule 3 x 266 [8,0,4]
+  CRUSH rule 3 x 267 [2,4,6]
+  CRUSH rule 3 x 268 [0,6,4]
+  CRUSH rule 3 x 269 [0,6,5]
+  CRUSH rule 3 x 270 [5,1,8]
+  CRUSH rule 3 x 271 [7,4,2]
+  CRUSH rule 3 x 272 [2,6,5]
+  CRUSH rule 3 x 273 [3,2,7]
+  CRUSH rule 3 x 274 [6,3,2]
+  CRUSH rule 3 x 275 [4,8,1]
+  CRUSH rule 3 x 276 [7,0,5]
+  CRUSH rule 3 x 277 [6,4,1]
+  CRUSH rule 3 x 278 [6,2,3]
+  CRUSH rule 3 x 279 [8,5,0]
+  CRUSH rule 3 x 280 [0,7,4]
+  CRUSH rule 3 x 281 [8,1,5]
+  CRUSH rule 3 x 282 [3,2,8]
+  CRUSH rule 3 x 283 [8,1,3]
+  CRUSH rule 3 x 284 [6,3,1]
+  CRUSH rule 3 x 285 [5,7,2]
+  CRUSH rule 3 x 286 [2,8,4]
+  CRUSH rule 3 x 287 [0,3,8]
+  CRUSH rule 3 x 288 [8,1,4]
+  CRUSH rule 3 x 289 [4,6,0]
+  CRUSH rule 3 x 290 [1,5,6]
+  CRUSH rule 3 x 291 [0,5,8]
+  CRUSH rule 3 x 292 [8,1,4]
+  CRUSH rule 3 x 293 [6,2,3]
+  CRUSH rule 3 x 294 [7,5,2]
+  CRUSH rule 3 x 295 [4,6,0]
+  CRUSH rule 3 x 296 [3,1,6]
+  CRUSH rule 3 x 297 [6,0,5]
+  CRUSH rule 3 x 298 [1,5,6]
+  CRUSH rule 3 x 299 [2,8,4]
+  CRUSH rule 3 x 300 [8,4,1]
+  CRUSH rule 3 x 301 [0,7,3]
+  CRUSH rule 3 x 302 [3,1,6]
+  CRUSH rule 3 x 303 [7,4,1]
+  CRUSH rule 3 x 304 [2,6,4]
+  CRUSH rule 3 x 305 [5,6,1]
+  CRUSH rule 3 x 306 [0,8,3]
+  CRUSH rule 3 x 307 [0,7,4]
+  CRUSH rule 3 x 308 [0,8,5]
+  CRUSH rule 3 x 309 [7,4,2]
+  CRUSH rule 3 x 310 [4,0,6]
+  CRUSH rule 3 x 311 [3,8,2]
+  CRUSH rule 3 x 312 [2,6,5]
+  CRUSH rule 3 x 313 [5,1,7]
+  CRUSH rule 3 x 314 [4,0,6]
+  CRUSH rule 3 x 315 [2,3,7]
+  CRUSH rule 3 x 316 [6,4,1]
+  CRUSH rule 3 x 317 [2,7,4]
+  CRUSH rule 3 x 318 [8,1,4]
+  CRUSH rule 3 x 319 [5,2,6]
+  CRUSH rule 3 x 320 [3,8,0]
+  CRUSH rule 3 x 321 [1,5,8]
+  CRUSH rule 3 x 322 [2,6,3]
+  CRUSH rule 3 x 323 [4,7,0]
+  CRUSH rule 3 x 324 [7,2,5]
+  CRUSH rule 3 x 325 [4,6,0]
+  CRUSH rule 3 x 326 [3,1,7]
+  CRUSH rule 3 x 327 [0,8,3]
+  CRUSH rule 3 x 328 [7,3,1]
+  CRUSH rule 3 x 329 [5,7,0]
+  CRUSH rule 3 x 330 [3,7,0]
+  CRUSH rule 3 x 331 [2,7,3]
+  CRUSH rule 3 x 332 [2,3,8]
+  CRUSH rule 3 x 333 [6,4,1]
+  CRUSH rule 3 x 334 [8,3,2]
+  CRUSH rule 3 x 335 [7,2,3]
+  CRUSH rule 3 x 336 [4,6,2]
+  CRUSH rule 3 x 337 [7,0,4]
+  CRUSH rule 3 x 338 [5,8,2]
+  CRUSH rule 3 x 339 [7,5,0]
+  CRUSH rule 3 x 340 [2,6,5]
+  CRUSH rule 3 x 341 [5,1,7]
+  CRUSH rule 3 x 342 [0,8,5]
+  CRUSH rule 3 x 343 [6,5,0]
+  CRUSH rule 3 x 344 [6,1,4]
+  CRUSH rule 3 x 345 [4,7,0]
+  CRUSH rule 3 x 346 [8,0,3]
+  CRUSH rule 3 x 347 [3,0,8]
+  CRUSH rule 3 x 348 [8,0,3]
+  CRUSH rule 3 x 349 [1,7,3]
+  CRUSH rule 3 x 350 [8,5,1]
+  CRUSH rule 3 x 351 [3,8,0]
+  CRUSH rule 3 x 352 [1,8,4]
+  CRUSH rule 3 x 353 [6,5,1]
+  CRUSH rule 3 x 354 [0,5,6]
+  CRUSH rule 3 x 355 [3,8,0]
+  CRUSH rule 3 x 356 [3,1,8]
+  CRUSH rule 3 x 357 [6,1,3]
+  CRUSH rule 3 x 358 [2,8,5]
+  CRUSH rule 3 x 359 [6,0,5]
+  CRUSH rule 3 x 360 [5,0,8]
+  CRUSH rule 3 x 361 [8,5,0]
+  CRUSH rule 3 x 362 [4,0,8]
+  CRUSH rule 3 x 363 [4,2,8]
+  CRUSH rule 3 x 364 [2,5,7]
+  CRUSH rule 3 x 365 [6,4,2]
+  CRUSH rule 3 x 366 [7,0,3]
+  CRUSH rule 3 x 367 [4,2,7]
+  CRUSH rule 3 x 368 [7,3,1]
+  CRUSH rule 3 x 369 [3,7,2]
+  CRUSH rule 3 x 370 [8,2,4]
+  CRUSH rule 3 x 371 [1,5,8]
+  CRUSH rule 3 x 372 [3,1,8]
+  CRUSH rule 3 x 373 [0,6,4]
+  CRUSH rule 3 x 374 [3,8,1]
+  CRUSH rule 3 x 375 [6,5,2]
+  CRUSH rule 3 x 376 [7,1,3]
+  CRUSH rule 3 x 377 [1,4,7]
+  CRUSH rule 3 x 378 [0,6,4]
+  CRUSH rule 3 x 379 [8,3,0]
+  CRUSH rule 3 x 380 [2,3,8]
+  CRUSH rule 3 x 381 [0,3,7]
+  CRUSH rule 3 x 382 [1,3,7]
+  CRUSH rule 3 x 383 [4,7,1]
+  CRUSH rule 3 x 384 [7,0,4]
+  CRUSH rule 3 x 385 [7,4,0]
+  CRUSH rule 3 x 386 [0,4,6]
+  CRUSH rule 3 x 387 [1,5,8]
+  CRUSH rule 3 x 388 [5,2,7]
+  CRUSH rule 3 x 389 [1,4,8]
+  CRUSH rule 3 x 390 [5,8,1]
+  CRUSH rule 3 x 391 [5,6,0]
+  CRUSH rule 3 x 392 [1,7,5]
+  CRUSH rule 3 x 393 [4,0,6]
+  CRUSH rule 3 x 394 [4,8,2]
+  CRUSH rule 3 x 395 [4,0,8]
+  CRUSH rule 3 x 396 [4,2,6]
+  CRUSH rule 3 x 397 [2,4,7]
+  CRUSH rule 3 x 398 [2,3,6]
+  CRUSH rule 3 x 399 [8,5,2]
+  CRUSH rule 3 x 400 [8,1,3]
+  CRUSH rule 3 x 401 [0,3,6]
+  CRUSH rule 3 x 402 [7,4,2]
+  CRUSH rule 3 x 403 [0,4,7]
+  CRUSH rule 3 x 404 [4,0,6]
+  CRUSH rule 3 x 405 [6,4,0]
+  CRUSH rule 3 x 406 [2,6,4]
+  CRUSH rule 3 x 407 [2,7,5]
+  CRUSH rule 3 x 408 [4,1,7]
+  CRUSH rule 3 x 409 [7,5,0]
+  CRUSH rule 3 x 410 [8,3,1]
+  CRUSH rule 3 x 411 [2,7,4]
+  CRUSH rule 3 x 412 [0,3,7]
+  CRUSH rule 3 x 413 [5,2,8]
+  CRUSH rule 3 x 414 [4,0,8]
+  CRUSH rule 3 x 415 [0,6,3]
+  CRUSH rule 3 x 416 [2,8,5]
+  CRUSH rule 3 x 417 [8,0,3]
+  CRUSH rule 3 x 418 [7,1,3]
+  CRUSH rule 3 x 419 [8,5,2]
+  CRUSH rule 3 x 420 [1,4,7]
+  CRUSH rule 3 x 421 [8,3,0]
+  CRUSH rule 3 x 422 [6,3,1]
+  CRUSH rule 3 x 423 [0,3,7]
+  CRUSH rule 3 x 424 [8,5,1]
+  CRUSH rule 3 x 425 [1,4,8]
+  CRUSH rule 3 x 426 [6,2,4]
+  CRUSH rule 3 x 427 [0,8,3]
+  CRUSH rule 3 x 428 [5,8,1]
+  CRUSH rule 3 x 429 [4,8,0]
+  CRUSH rule 3 x 430 [3,7,0]
+  CRUSH rule 3 x 431 [5,0,7]
+  CRUSH rule 3 x 432 [7,0,4]
+  CRUSH rule 3 x 433 [6,4,0]
+  CRUSH rule 3 x 434 [5,0,7]
+  CRUSH rule 3 x 435 [0,4,6]
+  CRUSH rule 3 x 436 [4,0,7]
+  CRUSH rule 3 x 437 [7,3,1]
+  CRUSH rule 3 x 438 [0,5,8]
+  CRUSH rule 3 x 439 [1,3,8]
+  CRUSH rule 3 x 440 [2,6,5]
+  CRUSH rule 3 x 441 [5,8,0]
+  CRUSH rule 3 x 442 [2,3,6]
+  CRUSH rule 3 x 443 [6,0,3]
+  CRUSH rule 3 x 444 [7,1,4]
+  CRUSH rule 3 x 445 [6,4,0]
+  CRUSH rule 3 x 446 [4,0,8]
+  CRUSH rule 3 x 447 [2,4,6]
+  CRUSH rule 3 x 448 [7,0,5]
+  CRUSH rule 3 x 449 [7,4,2]
+  CRUSH rule 3 x 450 [4,0,6]
+  CRUSH rule 3 x 451 [6,5,1]
+  CRUSH rule 3 x 452 [8,4,0]
+  CRUSH rule 3 x 453 [6,5,0]
+  CRUSH rule 3 x 454 [6,5,0]
+  CRUSH rule 3 x 455 [2,8,4]
+  CRUSH rule 3 x 456 [6,2,5]
+  CRUSH rule 3 x 457 [7,1,5]
+  CRUSH rule 3 x 458 [2,8,5]
+  CRUSH rule 3 x 459 [2,6,5]
+  CRUSH rule 3 x 460 [6,5,2]
+  CRUSH rule 3 x 461 [6,4,2]
+  CRUSH rule 3 x 462 [8,0,4]
+  CRUSH rule 3 x 463 [6,0,4]
+  CRUSH rule 3 x 464 [7,4,1]
+  CRUSH rule 3 x 465 [7,1,5]
+  CRUSH rule 3 x 466 [5,6,2]
+  CRUSH rule 3 x 467 [6,5,1]
+  CRUSH rule 3 x 468 [7,2,3]
+  CRUSH rule 3 x 469 [7,2,4]
+  CRUSH rule 3 x 470 [3,0,8]
+  CRUSH rule 3 x 471 [0,6,4]
+  CRUSH rule 3 x 472 [5,0,8]
+  CRUSH rule 3 x 473 [1,4,7]
+  CRUSH rule 3 x 474 [6,1,4]
+  CRUSH rule 3 x 475 [6,2,3]
+  CRUSH rule 3 x 476 [4,7,1]
+  CRUSH rule 3 x 477 [5,6,1]
+  CRUSH rule 3 x 478 [6,1,3]
+  CRUSH rule 3 x 479 [0,3,6]
+  CRUSH rule 3 x 480 [1,6,4]
+  CRUSH rule 3 x 481 [2,5,7]
+  CRUSH rule 3 x 482 [4,8,0]
+  CRUSH rule 3 x 483 [0,6,4]
+  CRUSH rule 3 x 484 [1,8,5]
+  CRUSH rule 3 x 485 [4,8,1]
+  CRUSH rule 3 x 486 [4,0,8]
+  CRUSH rule 3 x 487 [5,0,8]
+  CRUSH rule 3 x 488 [5,7,1]
+  CRUSH rule 3 x 489 [2,8,4]
+  CRUSH rule 3 x 490 [6,4,0]
+  CRUSH rule 3 x 491 [1,6,5]
+  CRUSH rule 3 x 492 [6,5,2]
+  CRUSH rule 3 x 493 [0,8,3]
+  CRUSH rule 3 x 494 [1,6,4]
+  CRUSH rule 3 x 495 [3,0,6]
+  CRUSH rule 3 x 496 [7,5,1]
+  CRUSH rule 3 x 497 [5,7,2]
+  CRUSH rule 3 x 498 [0,4,6]
+  CRUSH rule 3 x 499 [8,5,2]
+  CRUSH rule 3 x 500 [3,6,1]
+  CRUSH rule 3 x 501 [0,7,5]
+  CRUSH rule 3 x 502 [7,1,3]
+  CRUSH rule 3 x 503 [2,3,7]
+  CRUSH rule 3 x 504 [5,8,2]
+  CRUSH rule 3 x 505 [0,7,5]
+  CRUSH rule 3 x 506 [5,1,7]
+  CRUSH rule 3 x 507 [6,0,3]
+  CRUSH rule 3 x 508 [0,4,7]
+  CRUSH rule 3 x 509 [7,4,0]
+  CRUSH rule 3 x 510 [6,2,5]
+  CRUSH rule 3 x 511 [5,6,2]
+  CRUSH rule 3 x 512 [7,2,4]
+  CRUSH rule 3 x 513 [7,2,4]
+  CRUSH rule 3 x 514 [4,7,0]
+  CRUSH rule 3 x 515 [8,3,0]
+  CRUSH rule 3 x 516 [4,1,7]
+  CRUSH rule 3 x 517 [7,0,4]
+  CRUSH rule 3 x 518 [4,6,0]
+  CRUSH rule 3 x 519 [7,3,1]
+  CRUSH rule 3 x 520 [2,8,5]
+  CRUSH rule 3 x 521 [8,0,3]
+  CRUSH rule 3 x 522 [6,0,4]
+  CRUSH rule 3 x 523 [4,1,7]
+  CRUSH rule 3 x 524 [0,4,7]
+  CRUSH rule 3 x 525 [0,3,8]
+  CRUSH rule 3 x 526 [1,3,8]
+  CRUSH rule 3 x 527 [0,4,6]
+  CRUSH rule 3 x 528 [5,2,7]
+  CRUSH rule 3 x 529 [5,6,1]
+  CRUSH rule 3 x 530 [6,4,1]
+  CRUSH rule 3 x 531 [6,0,3]
+  CRUSH rule 3 x 532 [6,5,1]
+  CRUSH rule 3 x 533 [5,8,0]
+  CRUSH rule 3 x 534 [7,4,2]
+  CRUSH rule 3 x 535 [8,0,3]
+  CRUSH rule 3 x 536 [6,2,3]
+  CRUSH rule 3 x 537 [3,8,0]
+  CRUSH rule 3 x 538 [6,4,2]
+  CRUSH rule 3 x 539 [8,4,2]
+  CRUSH rule 3 x 540 [0,7,4]
+  CRUSH rule 3 x 541 [2,5,7]
+  CRUSH rule 3 x 542 [3,0,8]
+  CRUSH rule 3 x 543 [6,2,4]
+  CRUSH rule 3 x 544 [3,7,1]
+  CRUSH rule 3 x 545 [5,7,1]
+  CRUSH rule 3 x 546 [6,2,3]
+  CRUSH rule 3 x 547 [8,1,5]
+  CRUSH rule 3 x 548 [5,1,8]
+  CRUSH rule 3 x 549 [5,7,1]
+  CRUSH rule 3 x 550 [0,5,6]
+  CRUSH rule 3 x 551 [7,4,0]
+  CRUSH rule 3 x 552 [5,7,2]
+  CRUSH rule 3 x 553 [4,2,8]
+  CRUSH rule 3 x 554 [0,6,5]
+  CRUSH rule 3 x 555 [5,0,8]
+  CRUSH rule 3 x 556 [3,6,1]
+  CRUSH rule 3 x 557 [7,4,0]
+  CRUSH rule 3 x 558 [3,2,6]
+  CRUSH rule 3 x 559 [4,1,8]
+  CRUSH rule 3 x 560 [8,4,2]
+  CRUSH rule 3 x 561 [6,4,1]
+  CRUSH rule 3 x 562 [3,0,8]
+  CRUSH rule 3 x 563 [2,7,4]
+  CRUSH rule 3 x 564 [5,1,7]
+  CRUSH rule 3 x 565 [3,8,0]
+  CRUSH rule 3 x 566 [4,6,0]
+  CRUSH rule 3 x 567 [3,7,0]
+  CRUSH rule 3 x 568 [7,4,0]
+  CRUSH rule 3 x 569 [3,1,6]
+  CRUSH rule 3 x 570 [1,4,7]
+  CRUSH rule 3 x 571 [3,6,0]
+  CRUSH rule 3 x 572 [3,0,7]
+  CRUSH rule 3 x 573 [3,1,7]
+  CRUSH rule 3 x 574 [2,3,6]
+  CRUSH rule 3 x 575 [8,1,5]
+  CRUSH rule 3 x 576 [4,7,1]
+  CRUSH rule 3 x 577 [8,2,3]
+  CRUSH rule 3 x 578 [6,0,4]
+  CRUSH rule 3 x 579 [3,2,6]
+  CRUSH rule 3 x 580 [3,0,8]
+  CRUSH rule 3 x 581 [7,1,5]
+  CRUSH rule 3 x 582 [2,8,5]
+  CRUSH rule 3 x 583 [6,2,3]
+  CRUSH rule 3 x 584 [8,0,5]
+  CRUSH rule 3 x 585 [7,0,5]
+  CRUSH rule 3 x 586 [0,8,5]
+  CRUSH rule 3 x 587 [2,5,6]
+  CRUSH rule 3 x 588 [3,8,0]
+  CRUSH rule 3 x 589 [7,1,5]
+  CRUSH rule 3 x 590 [6,2,5]
+  CRUSH rule 3 x 591 [5,2,7]
+  CRUSH rule 3 x 592 [2,4,6]
+  CRUSH rule 3 x 593 [0,8,4]
+  CRUSH rule 3 x 594 [0,6,3]
+  CRUSH rule 3 x 595 [7,1,5]
+  CRUSH rule 3 x 596 [4,0,6]
+  CRUSH rule 3 x 597 [3,1,7]
+  CRUSH rule 3 x 598 [3,0,7]
+  CRUSH rule 3 x 599 [5,1,6]
+  CRUSH rule 3 x 600 [7,0,4]
+  CRUSH rule 3 x 601 [0,6,3]
+  CRUSH rule 3 x 602 [3,8,2]
+  CRUSH rule 3 x 603 [5,0,8]
+  CRUSH rule 3 x 604 [7,4,0]
+  CRUSH rule 3 x 605 [3,2,6]
+  CRUSH rule 3 x 606 [2,7,3]
+  CRUSH rule 3 x 607 [0,5,8]
+  CRUSH rule 3 x 608 [5,1,8]
+  CRUSH rule 3 x 609 [5,0,6]
+  CRUSH rule 3 x 610 [3,8,2]
+  CRUSH rule 3 x 611 [1,8,3]
+  CRUSH rule 3 x 612 [2,8,3]
+  CRUSH rule 3 x 613 [7,1,3]
+  CRUSH rule 3 x 614 [7,2,3]
+  CRUSH rule 3 x 615 [6,2,3]
+  CRUSH rule 3 x 616 [0,7,5]
+  CRUSH rule 3 x 617 [6,2,3]
+  CRUSH rule 3 x 618 [7,3,1]
+  CRUSH rule 3 x 619 [5,0,6]
+  CRUSH rule 3 x 620 [4,1,8]
+  CRUSH rule 3 x 621 [5,6,0]
+  CRUSH rule 3 x 622 [0,3,8]
+  CRUSH rule 3 x 623 [0,8,4]
+  CRUSH rule 3 x 624 [3,2,8]
+  CRUSH rule 3 x 625 [2,5,8]
+  CRUSH rule 3 x 626 [7,2,3]
+  CRUSH rule 3 x 627 [2,6,5]
+  CRUSH rule 3 x 628 [8,1,3]
+  CRUSH rule 3 x 629 [2,6,3]
+  CRUSH rule 3 x 630 [2,6,4]
+  CRUSH rule 3 x 631 [0,6,5]
+  CRUSH rule 3 x 632 [7,0,4]
+  CRUSH rule 3 x 633 [8,4,2]
+  CRUSH rule 3 x 634 [0,5,6]
+  CRUSH rule 3 x 635 [5,6,2]
+  CRUSH rule 3 x 636 [1,3,7]
+  CRUSH rule 3 x 637 [4,0,8]
+  CRUSH rule 3 x 638 [6,2,5]
+  CRUSH rule 3 x 639 [4,0,6]
+  CRUSH rule 3 x 640 [3,2,7]
+  CRUSH rule 3 x 641 [7,2,3]
+  CRUSH rule 3 x 642 [2,8,4]
+  CRUSH rule 3 x 643 [3,0,8]
+  CRUSH rule 3 x 644 [8,0,4]
+  CRUSH rule 3 x 645 [5,7,2]
+  CRUSH rule 3 x 646 [8,0,3]
+  CRUSH rule 3 x 647 [7,0,5]
+  CRUSH rule 3 x 648 [0,8,3]
+  CRUSH rule 3 x 649 [4,7,1]
+  CRUSH rule 3 x 650 [7,5,1]
+  CRUSH rule 3 x 651 [3,6,1]
+  CRUSH rule 3 x 652 [3,6,1]
+  CRUSH rule 3 x 653 [8,3,0]
+  CRUSH rule 3 x 654 [7,4,0]
+  CRUSH rule 3 x 655 [0,5,6]
+  CRUSH rule 3 x 656 [4,7,0]
+  CRUSH rule 3 x 657 [6,0,5]
+  CRUSH rule 3 x 658 [5,8,0]
+  CRUSH rule 3 x 659 [4,7,0]
+  CRUSH rule 3 x 660 [7,3,0]
+  CRUSH rule 3 x 661 [1,7,3]
+  CRUSH rule 3 x 662 [4,2,8]
+  CRUSH rule 3 x 663 [1,3,7]
+  CRUSH rule 3 x 664 [1,3,6]
+  CRUSH rule 3 x 665 [5,6,1]
+  CRUSH rule 3 x 666 [2,7,4]
+  CRUSH rule 3 x 667 [1,3,8]
+  CRUSH rule 3 x 668 [3,7,0]
+  CRUSH rule 3 x 669 [6,3,0]
+  CRUSH rule 3 x 670 [4,1,6]
+  CRUSH rule 3 x 671 [0,8,5]
+  CRUSH rule 3 x 672 [4,2,8]
+  CRUSH rule 3 x 673 [5,0,7]
+  CRUSH rule 3 x 674 [3,0,7]
+  CRUSH rule 3 x 675 [0,8,5]
+  CRUSH rule 3 x 676 [0,3,7]
+  CRUSH rule 3 x 677 [4,1,6]
+  CRUSH rule 3 x 678 [2,3,8]
+  CRUSH rule 3 x 679 [6,0,5]
+  CRUSH rule 3 x 680 [0,3,8]
+  CRUSH rule 3 x 681 [4,6,1]
+  CRUSH rule 3 x 682 [0,4,8]
+  CRUSH rule 3 x 683 [0,3,8]
+  CRUSH rule 3 x 684 [7,2,5]
+  CRUSH rule 3 x 685 [7,2,5]
+  CRUSH rule 3 x 686 [1,5,8]
+  CRUSH rule 3 x 687 [3,7,2]
+  CRUSH rule 3 x 688 [5,6,2]
+  CRUSH rule 3 x 689 [6,5,1]
+  CRUSH rule 3 x 690 [8,0,4]
+  CRUSH rule 3 x 691 [3,1,6]
+  CRUSH rule 3 x 692 [7,1,4]
+  CRUSH rule 3 x 693 [6,5,2]
+  CRUSH rule 3 x 694 [6,4,2]
+  CRUSH rule 3 x 695 [0,6,3]
+  CRUSH rule 3 x 696 [1,5,8]
+  CRUSH rule 3 x 697 [6,0,4]
+  CRUSH rule 3 x 698 [6,0,3]
+  CRUSH rule 3 x 699 [1,8,3]
+  CRUSH rule 3 x 700 [0,4,6]
+  CRUSH rule 3 x 701 [4,0,6]
+  CRUSH rule 3 x 702 [3,0,8]
+  CRUSH rule 3 x 703 [8,4,0]
+  CRUSH rule 3 x 704 [0,4,6]
+  CRUSH rule 3 x 705 [8,0,4]
+  CRUSH rule 3 x 706 [1,5,8]
+  CRUSH rule 3 x 707 [7,3,0]
+  CRUSH rule 3 x 708 [3,7,1]
+  CRUSH rule 3 x 709 [6,5,1]
+  CRUSH rule 3 x 710 [8,5,1]
+  CRUSH rule 3 x 711 [2,4,7]
+  CRUSH rule 3 x 712 [2,3,6]
+  CRUSH rule 3 x 713 [6,3,1]
+  CRUSH rule 3 x 714 [3,0,6]
+  CRUSH rule 3 x 715 [1,3,6]
+  CRUSH rule 3 x 716 [3,6,1]
+  CRUSH rule 3 x 717 [8,0,5]
+  CRUSH rule 3 x 718 [3,7,0]
+  CRUSH rule 3 x 719 [2,6,3]
+  CRUSH rule 3 x 720 [6,0,4]
+  CRUSH rule 3 x 721 [5,7,0]
+  CRUSH rule 3 x 722 [5,7,1]
+  CRUSH rule 3 x 723 [5,2,7]
+  CRUSH rule 3 x 724 [0,7,3]
+  CRUSH rule 3 x 725 [0,4,6]
+  CRUSH rule 3 x 726 [3,7,2]
+  CRUSH rule 3 x 727 [4,7,2]
+  CRUSH rule 3 x 728 [2,6,3]
+  CRUSH rule 3 x 729 [5,6,1]
+  CRUSH rule 3 x 730 [3,8,2]
+  CRUSH rule 3 x 731 [4,1,6]
+  CRUSH rule 3 x 732 [1,4,8]
+  CRUSH rule 3 x 733 [5,6,2]
+  CRUSH rule 3 x 734 [6,5,0]
+  CRUSH rule 3 x 735 [4,6,1]
+  CRUSH rule 3 x 736 [3,8,1]
+  CRUSH rule 3 x 737 [1,8,3]
+  CRUSH rule 3 x 738 [5,1,6]
+  CRUSH rule 3 x 739 [0,7,4]
+  CRUSH rule 3 x 740 [0,7,3]
+  CRUSH rule 3 x 741 [7,2,3]
+  CRUSH rule 3 x 742 [8,2,3]
+  CRUSH rule 3 x 743 [7,0,5]
+  CRUSH rule 3 x 744 [4,6,1]
+  CRUSH rule 3 x 745 [3,0,8]
+  CRUSH rule 3 x 746 [4,1,7]
+  CRUSH rule 3 x 747 [6,2,4]
+  CRUSH rule 3 x 748 [2,6,5]
+  CRUSH rule 3 x 749 [4,7,2]
+  CRUSH rule 3 x 750 [1,7,4]
+  CRUSH rule 3 x 751 [2,7,3]
+  CRUSH rule 3 x 752 [8,0,3]
+  CRUSH rule 3 x 753 [7,4,2]
+  CRUSH rule 3 x 754 [8,5,2]
+  CRUSH rule 3 x 755 [1,6,5]
+  CRUSH rule 3 x 756 [5,8,1]
+  CRUSH rule 3 x 757 [8,0,4]
+  CRUSH rule 3 x 758 [6,1,5]
+  CRUSH rule 3 x 759 [8,5,0]
+  CRUSH rule 3 x 760 [1,5,7]
+  CRUSH rule 3 x 761 [4,2,6]
+  CRUSH rule 3 x 762 [2,8,3]
+  CRUSH rule 3 x 763 [8,3,1]
+  CRUSH rule 3 x 764 [1,7,4]
+  CRUSH rule 3 x 765 [6,4,0]
+  CRUSH rule 3 x 766 [8,3,0]
+  CRUSH rule 3 x 767 [1,8,5]
+  CRUSH rule 3 x 768 [8,5,0]
+  CRUSH rule 3 x 769 [6,2,4]
+  CRUSH rule 3 x 770 [6,1,4]
+  CRUSH rule 3 x 771 [7,2,4]
+  CRUSH rule 3 x 772 [8,4,2]
+  CRUSH rule 3 x 773 [3,2,6]
+  CRUSH rule 3 x 774 [4,7,1]
+  CRUSH rule 3 x 775 [6,5,0]
+  CRUSH rule 3 x 776 [7,0,3]
+  CRUSH rule 3 x 777 [3,0,6]
+  CRUSH rule 3 x 778 [1,6,3]
+  CRUSH rule 3 x 779 [2,6,5]
+  CRUSH rule 3 x 780 [0,5,8]
+  CRUSH rule 3 x 781 [6,5,0]
+  CRUSH rule 3 x 782 [5,2,8]
+  CRUSH rule 3 x 783 [7,0,3]
+  CRUSH rule 3 x 784 [0,5,8]
+  CRUSH rule 3 x 785 [6,1,3]
+  CRUSH rule 3 x 786 [7,3,0]
+  CRUSH rule 3 x 787 [1,8,5]
+  CRUSH rule 3 x 788 [6,0,4]
+  CRUSH rule 3 x 789 [0,5,8]
+  CRUSH rule 3 x 790 [8,3,1]
+  CRUSH rule 3 x 791 [3,6,2]
+  CRUSH rule 3 x 792 [5,6,1]
+  CRUSH rule 3 x 793 [6,2,3]
+  CRUSH rule 3 x 794 [2,8,3]
+  CRUSH rule 3 x 795 [0,4,6]
+  CRUSH rule 3 x 796 [3,7,1]
+  CRUSH rule 3 x 797 [2,3,6]
+  CRUSH rule 3 x 798 [6,0,3]
+  CRUSH rule 3 x 799 [5,2,7]
+  CRUSH rule 3 x 800 [5,2,8]
+  CRUSH rule 3 x 801 [3,7,2]
+  CRUSH rule 3 x 802 [1,6,4]
+  CRUSH rule 3 x 803 [0,4,7]
+  CRUSH rule 3 x 804 [6,0,5]
+  CRUSH rule 3 x 805 [3,8,0]
+  CRUSH rule 3 x 806 [1,5,6]
+  CRUSH rule 3 x 807 [5,7,2]
+  CRUSH rule 3 x 808 [4,7,2]
+  CRUSH rule 3 x 809 [1,3,6]
+  CRUSH rule 3 x 810 [5,7,2]
+  CRUSH rule 3 x 811 [8,4,1]
+  CRUSH rule 3 x 812 [8,3,1]
+  CRUSH rule 3 x 813 [6,4,2]
+  CRUSH rule 3 x 814 [3,8,0]
+  CRUSH rule 3 x 815 [3,1,6]
+  CRUSH rule 3 x 816 [2,6,5]
+  CRUSH rule 3 x 817 [4,6,2]
+  CRUSH rule 3 x 818 [3,1,7]
+  CRUSH rule 3 x 819 [5,0,7]
+  CRUSH rule 3 x 820 [3,7,1]
+  CRUSH rule 3 x 821 [4,8,0]
+  CRUSH rule 3 x 822 [2,3,7]
+  CRUSH rule 3 x 823 [4,7,2]
+  CRUSH rule 3 x 824 [3,7,1]
+  CRUSH rule 3 x 825 [2,8,4]
+  CRUSH rule 3 x 826 [7,1,5]
+  CRUSH rule 3 x 827 [0,6,3]
+  CRUSH rule 3 x 828 [2,5,8]
+  CRUSH rule 3 x 829 [5,6,0]
+  CRUSH rule 3 x 830 [2,4,7]
+  CRUSH rule 3 x 831 [1,6,4]
+  CRUSH rule 3 x 832 [4,8,2]
+  CRUSH rule 3 x 833 [2,6,5]
+  CRUSH rule 3 x 834 [3,0,8]
+  CRUSH rule 3 x 835 [8,5,0]
+  CRUSH rule 3 x 836 [3,8,1]
+  CRUSH rule 3 x 837 [6,4,0]
+  CRUSH rule 3 x 838 [6,0,4]
+  CRUSH rule 3 x 839 [5,2,8]
+  CRUSH rule 3 x 840 [7,3,2]
+  CRUSH rule 3 x 841 [4,8,0]
+  CRUSH rule 3 x 842 [2,5,8]
+  CRUSH rule 3 x 843 [6,4,2]
+  CRUSH rule 3 x 844 [4,8,2]
+  CRUSH rule 3 x 845 [3,6,2]
+  CRUSH rule 3 x 846 [3,0,6]
+  CRUSH rule 3 x 847 [0,8,5]
+  CRUSH rule 3 x 848 [2,6,5]
+  CRUSH rule 3 x 849 [4,8,1]
+  CRUSH rule 3 x 850 [1,5,7]
+  CRUSH rule 3 x 851 [6,5,1]
+  CRUSH rule 3 x 852 [7,4,2]
+  CRUSH rule 3 x 853 [6,2,5]
+  CRUSH rule 3 x 854 [7,0,3]
+  CRUSH rule 3 x 855 [5,7,2]
+  CRUSH rule 3 x 856 [6,3,1]
+  CRUSH rule 3 x 857 [8,4,0]
+  CRUSH rule 3 x 858 [6,5,2]
+  CRUSH rule 3 x 859 [6,0,5]
+  CRUSH rule 3 x 860 [4,2,8]
+  CRUSH rule 3 x 861 [8,4,1]
+  CRUSH rule 3 x 862 [6,0,5]
+  CRUSH rule 3 x 863 [8,1,5]
+  CRUSH rule 3 x 864 [5,6,1]
+  CRUSH rule 3 x 865 [8,0,3]
+  CRUSH rule 3 x 866 [3,6,2]
+  CRUSH rule 3 x 867 [6,3,1]
+  CRUSH rule 3 x 868 [6,4,2]
+  CRUSH rule 3 x 869 [8,4,2]
+  CRUSH rule 3 x 870 [0,5,7]
+  CRUSH rule 3 x 871 [3,1,6]
+  CRUSH rule 3 x 872 [5,0,6]
+  CRUSH rule 3 x 873 [4,6,1]
+  CRUSH rule 3 x 874 [2,6,4]
+  CRUSH rule 3 x 875 [2,6,5]
+  CRUSH rule 3 x 876 [5,8,1]
+  CRUSH rule 3 x 877 [6,4,0]
+  CRUSH rule 3 x 878 [5,0,7]
+  CRUSH rule 3 x 879 [7,3,1]
+  CRUSH rule 3 x 880 [3,1,7]
+  CRUSH rule 3 x 881 [5,8,2]
+  CRUSH rule 3 x 882 [4,1,6]
+  CRUSH rule 3 x 883 [2,5,6]
+  CRUSH rule 3 x 884 [6,0,3]
+  CRUSH rule 3 x 885 [5,1,6]
+  CRUSH rule 3 x 886 [3,7,2]
+  CRUSH rule 3 x 887 [7,5,2]
+  CRUSH rule 3 x 888 [6,1,4]
+  CRUSH rule 3 x 889 [2,8,3]
+  CRUSH rule 3 x 890 [7,1,4]
+  CRUSH rule 3 x 891 [1,7,4]
+  CRUSH rule 3 x 892 [6,0,5]
+  CRUSH rule 3 x 893 [2,3,6]
+  CRUSH rule 3 x 894 [7,4,1]
+  CRUSH rule 3 x 895 [5,1,6]
+  CRUSH rule 3 x 896 [1,6,3]
+  CRUSH rule 3 x 897 [4,0,8]
+  CRUSH rule 3 x 898 [0,3,8]
+  CRUSH rule 3 x 899 [1,6,4]
+  CRUSH rule 3 x 900 [4,0,7]
+  CRUSH rule 3 x 901 [5,2,6]
+  CRUSH rule 3 x 902 [8,4,2]
+  CRUSH rule 3 x 903 [5,6,1]
+  CRUSH rule 3 x 904 [5,7,0]
+  CRUSH rule 3 x 905 [6,0,3]
+  CRUSH rule 3 x 906 [1,7,4]
+  CRUSH rule 3 x 907 [7,2,3]
+  CRUSH rule 3 x 908 [5,6,1]
+  CRUSH rule 3 x 909 [2,3,8]
+  CRUSH rule 3 x 910 [6,5,0]
+  CRUSH rule 3 x 911 [5,7,1]
+  CRUSH rule 3 x 912 [0,8,5]
+  CRUSH rule 3 x 913 [7,1,4]
+  CRUSH rule 3 x 914 [6,4,0]
+  CRUSH rule 3 x 915 [8,2,3]
+  CRUSH rule 3 x 916 [3,0,6]
+  CRUSH rule 3 x 917 [1,3,6]
+  CRUSH rule 3 x 918 [8,0,5]
+  CRUSH rule 3 x 919 [6,2,3]
+  CRUSH rule 3 x 920 [7,4,2]
+  CRUSH rule 3 x 921 [1,4,8]
+  CRUSH rule 3 x 922 [6,3,1]
+  CRUSH rule 3 x 923 [5,6,0]
+  CRUSH rule 3 x 924 [3,1,8]
+  CRUSH rule 3 x 925 [5,6,0]
+  CRUSH rule 3 x 926 [3,0,7]
+  CRUSH rule 3 x 927 [1,6,5]
+  CRUSH rule 3 x 928 [8,1,4]
+  CRUSH rule 3 x 929 [4,2,6]
+  CRUSH rule 3 x 930 [2,5,8]
+  CRUSH rule 3 x 931 [5,2,8]
+  CRUSH rule 3 x 932 [4,2,7]
+  CRUSH rule 3 x 933 [8,4,2]
+  CRUSH rule 3 x 934 [5,8,0]
+  CRUSH rule 3 x 935 [6,3,0]
+  CRUSH rule 3 x 936 [0,7,3]
+  CRUSH rule 3 x 937 [5,8,1]
+  CRUSH rule 3 x 938 [6,4,0]
+  CRUSH rule 3 x 939 [2,8,3]
+  CRUSH rule 3 x 940 [8,5,2]
+  CRUSH rule 3 x 941 [5,0,6]
+  CRUSH rule 3 x 942 [1,6,4]
+  CRUSH rule 3 x 943 [8,2,4]
+  CRUSH rule 3 x 944 [4,8,1]
+  CRUSH rule 3 x 945 [7,0,3]
+  CRUSH rule 3 x 946 [2,8,3]
+  CRUSH rule 3 x 947 [4,2,6]
+  CRUSH rule 3 x 948 [7,4,1]
+  CRUSH rule 3 x 949 [6,2,5]
+  CRUSH rule 3 x 950 [3,7,0]
+  CRUSH rule 3 x 951 [4,6,2]
+  CRUSH rule 3 x 952 [2,7,4]
+  CRUSH rule 3 x 953 [1,3,6]
+  CRUSH rule 3 x 954 [4,0,8]
+  CRUSH rule 3 x 955 [8,1,4]
+  CRUSH rule 3 x 956 [1,7,4]
+  CRUSH rule 3 x 957 [7,0,3]
+  CRUSH rule 3 x 958 [8,3,2]
+  CRUSH rule 3 x 959 [5,1,6]
+  CRUSH rule 3 x 960 [3,6,2]
+  CRUSH rule 3 x 961 [4,1,6]
+  CRUSH rule 3 x 962 [7,5,0]
+  CRUSH rule 3 x 963 [0,5,8]
+  CRUSH rule 3 x 964 [3,2,7]
+  CRUSH rule 3 x 965 [7,3,1]
+  CRUSH rule 3 x 966 [3,6,0]
+  CRUSH rule 3 x 967 [8,4,2]
+  CRUSH rule 3 x 968 [7,0,4]
+  CRUSH rule 3 x 969 [8,0,4]
+  CRUSH rule 3 x 970 [0,8,3]
+  CRUSH rule 3 x 971 [1,8,5]
+  CRUSH rule 3 x 972 [1,7,5]
+  CRUSH rule 3 x 973 [1,8,3]
+  CRUSH rule 3 x 974 [5,1,7]
+  CRUSH rule 3 x 975 [3,8,0]
+  CRUSH rule 3 x 976 [4,7,2]
+  CRUSH rule 3 x 977 [8,3,2]
+  CRUSH rule 3 x 978 [7,0,3]
+  CRUSH rule 3 x 979 [7,2,5]
+  CRUSH rule 3 x 980 [6,2,3]
+  CRUSH rule 3 x 981 [7,5,1]
+  CRUSH rule 3 x 982 [4,1,8]
+  CRUSH rule 3 x 983 [3,6,0]
+  CRUSH rule 3 x 984 [0,8,4]
+  CRUSH rule 3 x 985 [2,4,8]
+  CRUSH rule 3 x 986 [8,4,2]
+  CRUSH rule 3 x 987 [0,4,8]
+  CRUSH rule 3 x 988 [1,4,6]
+  CRUSH rule 3 x 989 [0,8,5]
+  CRUSH rule 3 x 990 [1,6,4]
+  CRUSH rule 3 x 991 [0,4,8]
+  CRUSH rule 3 x 992 [7,0,4]
+  CRUSH rule 3 x 993 [0,6,4]
+  CRUSH rule 3 x 994 [3,6,1]
+  CRUSH rule 3 x 995 [7,0,4]
+  CRUSH rule 3 x 996 [6,5,1]
+  CRUSH rule 3 x 997 [6,3,1]
+  CRUSH rule 3 x 998 [8,0,4]
+  CRUSH rule 3 x 999 [0,7,5]
+  CRUSH rule 3 x 1000 [8,4,1]
+  CRUSH rule 3 x 1001 [2,3,6]
+  CRUSH rule 3 x 1002 [1,3,8]
+  CRUSH rule 3 x 1003 [2,7,3]
+  CRUSH rule 3 x 1004 [6,0,3]
+  CRUSH rule 3 x 1005 [6,1,4]
+  CRUSH rule 3 x 1006 [1,8,4]
+  CRUSH rule 3 x 1007 [1,3,8]
+  CRUSH rule 3 x 1008 [1,7,4]
+  CRUSH rule 3 x 1009 [6,5,1]
+  CRUSH rule 3 x 1010 [3,1,6]
+  CRUSH rule 3 x 1011 [3,0,8]
+  CRUSH rule 3 x 1012 [3,1,8]
+  CRUSH rule 3 x 1013 [5,2,8]
+  CRUSH rule 3 x 1014 [2,8,3]
+  CRUSH rule 3 x 1015 [6,3,2]
+  CRUSH rule 3 x 1016 [2,5,6]
+  CRUSH rule 3 x 1017 [6,1,5]
+  CRUSH rule 3 x 1018 [5,1,6]
+  CRUSH rule 3 x 1019 [5,8,1]
+  CRUSH rule 3 x 1020 [5,0,7]
+  CRUSH rule 3 x 1021 [5,2,7]
+  CRUSH rule 3 x 1022 [1,7,4]
+  CRUSH rule 3 x 1023 [3,0,7]
+  rule 3 (choose-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 4 (choose-set-two), x = 0..1023, numrep = 2..3
+  CRUSH rule 4 x 0 [0,2]
+  CRUSH rule 4 x 1 [0,8]
+  CRUSH rule 4 x 2 [1,3]
+  CRUSH rule 4 x 3 [8,0]
+  CRUSH rule 4 x 4 [5,4]
+  CRUSH rule 4 x 5 [7,8]
+  CRUSH rule 4 x 6 [2,6]
+  CRUSH rule 4 x 7 [5,4]
+  CRUSH rule 4 x 8 [5,3]
+  CRUSH rule 4 x 9 [2,3]
+  CRUSH rule 4 x 10 [0,2]
+  CRUSH rule 4 x 11 [0,7]
+  CRUSH rule 4 x 12 [0,2]
+  CRUSH rule 4 x 13 [3,8]
+  CRUSH rule 4 x 14 [7,6]
+  CRUSH rule 4 x 15 [7,2]
+  CRUSH rule 4 x 16 [3,6]
+  CRUSH rule 4 x 17 [5,4]
+  CRUSH rule 4 x 18 [1,4]
+  CRUSH rule 4 x 19 [7,5]
+  CRUSH rule 4 x 20 [2,4]
+  CRUSH rule 4 x 21 [3,7]
+  CRUSH rule 4 x 22 [8,3]
+  CRUSH rule 4 x 23 [3,6]
+  CRUSH rule 4 x 24 [1,0]
+  CRUSH rule 4 x 25 [3,7]
+  CRUSH rule 4 x 26 [2,8]
+  CRUSH rule 4 x 27 [3,1]
+  CRUSH rule 4 x 28 [6,0]
+  CRUSH rule 4 x 29 [8,5]
+  CRUSH rule 4 x 30 [5,7]
+  CRUSH rule 4 x 31 [8,7]
+  CRUSH rule 4 x 32 [3,6]
+  CRUSH rule 4 x 33 [2,7]
+  CRUSH rule 4 x 34 [2,5]
+  CRUSH rule 4 x 35 [0,8]
+  CRUSH rule 4 x 36 [3,8]
+  CRUSH rule 4 x 37 [0,4]
+  CRUSH rule 4 x 38 [4,8]
+  CRUSH rule 4 x 39 [3,7]
+  CRUSH rule 4 x 40 [7,8]
+  CRUSH rule 4 x 41 [0,2]
+  CRUSH rule 4 x 42 [4,3]
+  CRUSH rule 4 x 43 [0,3]
+  CRUSH rule 4 x 44 [1,6]
+  CRUSH rule 4 x 45 [8,0]
+  CRUSH rule 4 x 46 [2,4]
+  CRUSH rule 4 x 47 [4,5]
+  CRUSH rule 4 x 48 [4,6]
+  CRUSH rule 4 x 49 [5,4]
+  CRUSH rule 4 x 50 [3,4]
+  CRUSH rule 4 x 51 [3,6]
+  CRUSH rule 4 x 52 [8,6]
+  CRUSH rule 4 x 53 [3,5]
+  CRUSH rule 4 x 54 [7,6]
+  CRUSH rule 4 x 55 [8,7]
+  CRUSH rule 4 x 56 [6,4]
+  CRUSH rule 4 x 57 [5,3]
+  CRUSH rule 4 x 58 [1,0]
+  CRUSH rule 4 x 59 [4,2]
+  CRUSH rule 4 x 60 [3,5]
+  CRUSH rule 4 x 61 [4,6]
+  CRUSH rule 4 x 62 [7,0]
+  CRUSH rule 4 x 63 [5,6]
+  CRUSH rule 4 x 64 [4,5]
+  CRUSH rule 4 x 65 [7,3]
+  CRUSH rule 4 x 66 [5,4]
+  CRUSH rule 4 x 67 [5,0]
+  CRUSH rule 4 x 68 [0,5]
+  CRUSH rule 4 x 69 [5,1]
+  CRUSH rule 4 x 70 [7,0]
+  CRUSH rule 4 x 71 [2,8]
+  CRUSH rule 4 x 72 [6,1]
+  CRUSH rule 4 x 73 [2,7]
+  CRUSH rule 4 x 74 [0,7]
+  CRUSH rule 4 x 75 [3,2]
+  CRUSH rule 4 x 76 [5,1]
+  CRUSH rule 4 x 77 [7,2]
+  CRUSH rule 4 x 78 [1,4]
+  CRUSH rule 4 x 79 [5,4]
+  CRUSH rule 4 x 80 [0,3]
+  CRUSH rule 4 x 81 [0,2]
+  CRUSH rule 4 x 82 [7,1]
+  CRUSH rule 4 x 83 [2,6]
+  CRUSH rule 4 x 84 [7,2]
+  CRUSH rule 4 x 85 [3,4]
+  CRUSH rule 4 x 86 [0,2]
+  CRUSH rule 4 x 87 [0,7]
+  CRUSH rule 4 x 88 [1,6]
+  CRUSH rule 4 x 89 [3,0]
+  CRUSH rule 4 x 90 [6,7]
+  CRUSH rule 4 x 91 [3,8]
+  CRUSH rule 4 x 92 [1,8]
+  CRUSH rule 4 x 93 [7,4]
+  CRUSH rule 4 x 94 [0,4]
+  CRUSH rule 4 x 95 [7,5]
+  CRUSH rule 4 x 96 [3,6]
+  CRUSH rule 4 x 97 [8,7]
+  CRUSH rule 4 x 98 [2,0]
+  CRUSH rule 4 x 99 [0,7]
+  CRUSH rule 4 x 100 [1,7]
+  CRUSH rule 4 x 101 [3,7]
+  CRUSH rule 4 x 102 [4,2]
+  CRUSH rule 4 x 103 [4,7]
+  CRUSH rule 4 x 104 [7,4]
+  CRUSH rule 4 x 105 [2,4]
+  CRUSH rule 4 x 106 [1,6]
+  CRUSH rule 4 x 107 [3,2]
+  CRUSH rule 4 x 108 [7,2]
+  CRUSH rule 4 x 109 [1,2]
+  CRUSH rule 4 x 110 [3,2]
+  CRUSH rule 4 x 111 [2,1]
+  CRUSH rule 4 x 112 [2,0]
+  CRUSH rule 4 x 113 [6,2]
+  CRUSH rule 4 x 114 [7,6]
+  CRUSH rule 4 x 115 [8,2]
+  CRUSH rule 4 x 116 [1,2]
+  CRUSH rule 4 x 117 [7,3]
+  CRUSH rule 4 x 118 [0,3]
+  CRUSH rule 4 x 119 [5,6]
+  CRUSH rule 4 x 120 [0,2]
+  CRUSH rule 4 x 121 [2,0]
+  CRUSH rule 4 x 122 [8,5]
+  CRUSH rule 4 x 123 [2,5]
+  CRUSH rule 4 x 124 [3,5]
+  CRUSH rule 4 x 125 [0,7]
+  CRUSH rule 4 x 126 [4,2]
+  CRUSH rule 4 x 127 [3,6]
+  CRUSH rule 4 x 128 [3,5]
+  CRUSH rule 4 x 129 [0,2]
+  CRUSH rule 4 x 130 [3,8]
+  CRUSH rule 4 x 131 [1,2]
+  CRUSH rule 4 x 132 [1,2]
+  CRUSH rule 4 x 133 [3,6]
+  CRUSH rule 4 x 134 [1,8]
+  CRUSH rule 4 x 135 [5,6]
+  CRUSH rule 4 x 136 [2,1]
+  CRUSH rule 4 x 137 [7,3]
+  CRUSH rule 4 x 138 [8,7]
+  CRUSH rule 4 x 139 [3,0]
+  CRUSH rule 4 x 140 [1,6]
+  CRUSH rule 4 x 141 [6,8]
+  CRUSH rule 4 x 142 [3,5]
+  CRUSH rule 4 x 143 [5,8]
+  CRUSH rule 4 x 144 [8,1]
+  CRUSH rule 4 x 145 [8,5]
+  CRUSH rule 4 x 146 [2,6]
+  CRUSH rule 4 x 147 [2,8]
+  CRUSH rule 4 x 148 [3,4]
+  CRUSH rule 4 x 149 [4,8]
+  CRUSH rule 4 x 150 [1,6]
+  CRUSH rule 4 x 151 [3,4]
+  CRUSH rule 4 x 152 [8,4]
+  CRUSH rule 4 x 153 [8,6]
+  CRUSH rule 4 x 154 [3,2]
+  CRUSH rule 4 x 155 [3,5]
+  CRUSH rule 4 x 156 [4,5]
+  CRUSH rule 4 x 157 [4,1]
+  CRUSH rule 4 x 158 [2,8]
+  CRUSH rule 4 x 159 [7,0]
+  CRUSH rule 4 x 160 [2,8]
+  CRUSH rule 4 x 161 [1,5]
+  CRUSH rule 4 x 162 [0,6]
+  CRUSH rule 4 x 163 [5,6]
+  CRUSH rule 4 x 164 [7,8]
+  CRUSH rule 4 x 165 [7,0]
+  CRUSH rule 4 x 166 [2,4]
+  CRUSH rule 4 x 167 [0,1]
+  CRUSH rule 4 x 168 [4,2]
+  CRUSH rule 4 x 169 [2,6]
+  CRUSH rule 4 x 170 [1,0]
+  CRUSH rule 4 x 171 [7,5]
+  CRUSH rule 4 x 172 [0,7]
+  CRUSH rule 4 x 173 [8,5]
+  CRUSH rule 4 x 174 [1,4]
+  CRUSH rule 4 x 175 [6,0]
+  CRUSH rule 4 x 176 [4,3]
+  CRUSH rule 4 x 177 [5,3]
+  CRUSH rule 4 x 178 [3,0]
+  CRUSH rule 4 x 179 [4,1]
+  CRUSH rule 4 x 180 [3,8]
+  CRUSH rule 4 x 181 [6,2]
+  CRUSH rule 4 x 182 [8,5]
+  CRUSH rule 4 x 183 [7,8]
+  CRUSH rule 4 x 184 [5,7]
+  CRUSH rule 4 x 185 [6,8]
+  CRUSH rule 4 x 186 [2,1]
+  CRUSH rule 4 x 187 [1,6]
+  CRUSH rule 4 x 188 [1,8]
+  CRUSH rule 4 x 189 [0,7]
+  CRUSH rule 4 x 190 [4,0]
+  CRUSH rule 4 x 191 [7,6]
+  CRUSH rule 4 x 192 [5,0]
+  CRUSH rule 4 x 193 [4,2]
+  CRUSH rule 4 x 194 [1,3]
+  CRUSH rule 4 x 195 [6,4]
+  CRUSH rule 4 x 196 [6,7]
+  CRUSH rule 4 x 197 [6,5]
+  CRUSH rule 4 x 198 [2,5]
+  CRUSH rule 4 x 199 [0,5]
+  CRUSH rule 4 x 200 [0,5]
+  CRUSH rule 4 x 201 [7,1]
+  CRUSH rule 4 x 202 [6,3]
+  CRUSH rule 4 x 203 [4,8]
+  CRUSH rule 4 x 204 [2,1]
+  CRUSH rule 4 x 205 [0,7]
+  CRUSH rule 4 x 206 [0,1]
+  CRUSH rule 4 x 207 [3,0]
+  CRUSH rule 4 x 208 [7,1]
+  CRUSH rule 4 x 209 [1,2]
+  CRUSH rule 4 x 210 [1,2]
+  CRUSH rule 4 x 211 [5,4]
+  CRUSH rule 4 x 212 [7,5]
+  CRUSH rule 4 x 213 [8,4]
+  CRUSH rule 4 x 214 [4,5]
+  CRUSH rule 4 x 215 [8,1]
+  CRUSH rule 4 x 216 [5,0]
+  CRUSH rule 4 x 217 [0,1]
+  CRUSH rule 4 x 218 [0,7]
+  CRUSH rule 4 x 219 [4,8]
+  CRUSH rule 4 x 220 [5,7]
+  CRUSH rule 4 x 221 [3,6]
+  CRUSH rule 4 x 222 [6,8]
+  CRUSH rule 4 x 223 [1,3]
+  CRUSH rule 4 x 224 [1,5]
+  CRUSH rule 4 x 225 [8,6]
+  CRUSH rule 4 x 226 [7,2]
+  CRUSH rule 4 x 227 [3,4]
+  CRUSH rule 4 x 228 [5,4]
+  CRUSH rule 4 x 229 [3,4]
+  CRUSH rule 4 x 230 [4,7]
+  CRUSH rule 4 x 231 [4,3]
+  CRUSH rule 4 x 232 [2,7]
+  CRUSH rule 4 x 233 [3,4]
+  CRUSH rule 4 x 234 [0,1]
+  CRUSH rule 4 x 235 [3,8]
+  CRUSH rule 4 x 236 [5,2]
+  CRUSH rule 4 x 237 [4,7]
+  CRUSH rule 4 x 238 [4,3]
+  CRUSH rule 4 x 239 [8,7]
+  CRUSH rule 4 x 240 [5,7]
+  CRUSH rule 4 x 241 [3,1]
+  CRUSH rule 4 x 242 [3,5]
+  CRUSH rule 4 x 243 [4,7]
+  CRUSH rule 4 x 244 [4,6]
+  CRUSH rule 4 x 245 [7,6]
+  CRUSH rule 4 x 246 [1,5]
+  CRUSH rule 4 x 247 [6,0]
+  CRUSH rule 4 x 248 [8,0]
+  CRUSH rule 4 x 249 [2,0]
+  CRUSH rule 4 x 250 [2,1]
+  CRUSH rule 4 x 251 [2,3]
+  CRUSH rule 4 x 252 [3,7]
+  CRUSH rule 4 x 253 [3,2]
+  CRUSH rule 4 x 254 [3,5]
+  CRUSH rule 4 x 255 [1,7]
+  CRUSH rule 4 x 256 [5,7]
+  CRUSH rule 4 x 257 [2,8]
+  CRUSH rule 4 x 258 [5,3]
+  CRUSH rule 4 x 259 [4,3]
+  CRUSH rule 4 x 260 [3,6]
+  CRUSH rule 4 x 261 [8,7]
+  CRUSH rule 4 x 262 [5,4]
+  CRUSH rule 4 x 263 [6,8]
+  CRUSH rule 4 x 264 [3,6]
+  CRUSH rule 4 x 265 [8,6]
+  CRUSH rule 4 x 266 [8,2]
+  CRUSH rule 4 x 267 [2,3]
+  CRUSH rule 4 x 268 [0,7]
+  CRUSH rule 4 x 269 [0,8]
+  CRUSH rule 4 x 270 [5,0]
+  CRUSH rule 4 x 271 [7,5]
+  CRUSH rule 4 x 272 [2,8]
+  CRUSH rule 4 x 273 [3,5]
+  CRUSH rule 4 x 274 [6,8]
+  CRUSH rule 4 x 275 [4,3]
+  CRUSH rule 4 x 276 [7,1]
+  CRUSH rule 4 x 277 [6,4]
+  CRUSH rule 4 x 278 [6,8]
+  CRUSH rule 4 x 279 [8,3]
+  CRUSH rule 4 x 280 [0,6]
+  CRUSH rule 4 x 281 [8,0]
+  CRUSH rule 4 x 282 [3,1]
+  CRUSH rule 4 x 283 [8,2]
+  CRUSH rule 4 x 284 [6,3]
+  CRUSH rule 4 x 285 [5,3]
+  CRUSH rule 4 x 286 [2,1]
+  CRUSH rule 4 x 287 [0,4]
+  CRUSH rule 4 x 288 [8,0]
+  CRUSH rule 4 x 289 [4,6]
+  CRUSH rule 4 x 290 [1,3]
+  CRUSH rule 4 x 291 [0,1]
+  CRUSH rule 4 x 292 [8,0]
+  CRUSH rule 4 x 293 [6,0]
+  CRUSH rule 4 x 294 [7,4]
+  CRUSH rule 4 x 295 [4,8]
+  CRUSH rule 4 x 296 [3,1]
+  CRUSH rule 4 x 297 [6,2]
+  CRUSH rule 4 x 298 [1,2]
+  CRUSH rule 4 x 299 [2,0]
+  CRUSH rule 4 x 300 [8,7]
+  CRUSH rule 4 x 301 [0,8]
+  CRUSH rule 4 x 302 [3,0]
+  CRUSH rule 4 x 303 [7,5]
+  CRUSH rule 4 x 304 [2,7]
+  CRUSH rule 4 x 305 [5,8]
+  CRUSH rule 4 x 306 [0,7]
+  CRUSH rule 4 x 307 [0,2]
+  CRUSH rule 4 x 308 [0,8]
+  CRUSH rule 4 x 309 [7,4]
+  CRUSH rule 4 x 310 [4,3]
+  CRUSH rule 4 x 311 [3,4]
+  CRUSH rule 4 x 312 [2,1]
+  CRUSH rule 4 x 313 [5,3]
+  CRUSH rule 4 x 314 [4,5]
+  CRUSH rule 4 x 315 [2,0]
+  CRUSH rule 4 x 316 [6,3]
+  CRUSH rule 4 x 317 [2,6]
+  CRUSH rule 4 x 318 [8,6]
+  CRUSH rule 4 x 319 [5,0]
+  CRUSH rule 4 x 320 [3,7]
+  CRUSH rule 4 x 321 [1,3]
+  CRUSH rule 4 x 322 [2,7]
+  CRUSH rule 4 x 323 [4,7]
+  CRUSH rule 4 x 324 [7,0]
+  CRUSH rule 4 x 325 [4,6]
+  CRUSH rule 4 x 326 [3,4]
+  CRUSH rule 4 x 327 [0,6]
+  CRUSH rule 4 x 328 [7,4]
+  CRUSH rule 4 x 329 [5,6]
+  CRUSH rule 4 x 330 [3,7]
+  CRUSH rule 4 x 331 [2,6]
+  CRUSH rule 4 x 332 [2,0]
+  CRUSH rule 4 x 333 [6,8]
+  CRUSH rule 4 x 334 [8,3]
+  CRUSH rule 4 x 335 [7,1]
+  CRUSH rule 4 x 336 [4,6]
+  CRUSH rule 4 x 337 [7,2]
+  CRUSH rule 4 x 338 [5,6]
+  CRUSH rule 4 x 339 [7,5]
+  CRUSH rule 4 x 340 [2,0]
+  CRUSH rule 4 x 341 [5,1]
+  CRUSH rule 4 x 342 [0,7]
+  CRUSH rule 4 x 343 [6,7]
+  CRUSH rule 4 x 344 [6,0]
+  CRUSH rule 4 x 345 [4,3]
+  CRUSH rule 4 x 346 [8,0]
+  CRUSH rule 4 x 347 [3,1]
+  CRUSH rule 4 x 348 [8,0]
+  CRUSH rule 4 x 349 [1,6]
+  CRUSH rule 4 x 350 [8,5]
+  CRUSH rule 4 x 351 [3,6]
+  CRUSH rule 4 x 352 [1,0]
+  CRUSH rule 4 x 353 [6,4]
+  CRUSH rule 4 x 354 [0,3]
+  CRUSH rule 4 x 355 [3,4]
+  CRUSH rule 4 x 356 [3,5]
+  CRUSH rule 4 x 357 [6,1]
+  CRUSH rule 4 x 358 [2,1]
+  CRUSH rule 4 x 359 [6,7]
+  CRUSH rule 4 x 360 [5,3]
+  CRUSH rule 4 x 361 [8,4]
+  CRUSH rule 4 x 362 [4,5]
+  CRUSH rule 4 x 363 [4,0]
+  CRUSH rule 4 x 364 [2,5]
+  CRUSH rule 4 x 365 [6,7]
+  CRUSH rule 4 x 366 [7,2]
+  CRUSH rule 4 x 367 [4,5]
+  CRUSH rule 4 x 368 [7,4]
+  CRUSH rule 4 x 369 [3,7]
+  CRUSH rule 4 x 370 [8,7]
+  CRUSH rule 4 x 371 [1,3]
+  CRUSH rule 4 x 372 [3,1]
+  CRUSH rule 4 x 373 [0,6]
+  CRUSH rule 4 x 374 [3,8]
+  CRUSH rule 4 x 375 [6,4]
+  CRUSH rule 4 x 376 [7,1]
+  CRUSH rule 4 x 377 [1,2]
+  CRUSH rule 4 x 378 [0,1]
+  CRUSH rule 4 x 379 [8,5]
+  CRUSH rule 4 x 380 [2,5]
+  CRUSH rule 4 x 381 [0,4]
+  CRUSH rule 4 x 382 [1,5]
+  CRUSH rule 4 x 383 [4,3]
+  CRUSH rule 4 x 384 [7,0]
+  CRUSH rule 4 x 385 [7,4]
+  CRUSH rule 4 x 386 [0,3]
+  CRUSH rule 4 x 387 [1,3]
+  CRUSH rule 4 x 388 [5,0]
+  CRUSH rule 4 x 389 [1,5]
+  CRUSH rule 4 x 390 [5,6]
+  CRUSH rule 4 x 391 [5,6]
+  CRUSH rule 4 x 392 [1,8]
+  CRUSH rule 4 x 393 [4,2]
+  CRUSH rule 4 x 394 [4,7]
+  CRUSH rule 4 x 395 [4,0]
+  CRUSH rule 4 x 396 [4,2]
+  CRUSH rule 4 x 397 [2,1]
+  CRUSH rule 4 x 398 [2,4]
+  CRUSH rule 4 x 399 [8,7]
+  CRUSH rule 4 x 400 [8,1]
+  CRUSH rule 4 x 401 [0,1]
+  CRUSH rule 4 x 402 [7,8]
+  CRUSH rule 4 x 403 [0,1]
+  CRUSH rule 4 x 404 [4,3]
+  CRUSH rule 4 x 405 [6,5]
+  CRUSH rule 4 x 406 [2,0]
+  CRUSH rule 4 x 407 [2,8]
+  CRUSH rule 4 x 408 [4,1]
+  CRUSH rule 4 x 409 [7,3]
+  CRUSH rule 4 x 410 [8,6]
+  CRUSH rule 4 x 411 [2,0]
+  CRUSH rule 4 x 412 [0,5]
+  CRUSH rule 4 x 413 [5,0]
+  CRUSH rule 4 x 414 [4,1]
+  CRUSH rule 4 x 415 [0,6]
+  CRUSH rule 4 x 416 [2,1]
+  CRUSH rule 4 x 417 [8,7]
+  CRUSH rule 4 x 418 [7,6]
+  CRUSH rule 4 x 419 [8,3]
+  CRUSH rule 4 x 420 [1,4]
+  CRUSH rule 4 x 421 [8,6]
+  CRUSH rule 4 x 422 [6,7]
+  CRUSH rule 4 x 423 [0,5]
+  CRUSH rule 4 x 424 [8,4]
+  CRUSH rule 4 x 425 [1,3]
+  CRUSH rule 4 x 426 [6,7]
+  CRUSH rule 4 x 427 [0,7]
+  CRUSH rule 4 x 428 [5,4]
+  CRUSH rule 4 x 429 [4,6]
+  CRUSH rule 4 x 430 [3,6]
+  CRUSH rule 4 x 431 [5,3]
+  CRUSH rule 4 x 432 [7,1]
+  CRUSH rule 4 x 433 [6,5]
+  CRUSH rule 4 x 434 [5,2]
+  CRUSH rule 4 x 435 [0,5]
+  CRUSH rule 4 x 436 [4,0]
+  CRUSH rule 4 x 437 [7,5]
+  CRUSH rule 4 x 438 [0,3]
+  CRUSH rule 4 x 439 [1,3]
+  CRUSH rule 4 x 440 [2,7]
+  CRUSH rule 4 x 441 [5,7]
+  CRUSH rule 4 x 442 [2,4]
+  CRUSH rule 4 x 443 [6,8]
+  CRUSH rule 4 x 444 [7,0]
+  CRUSH rule 4 x 445 [6,3]
+  CRUSH rule 4 x 446 [4,3]
+  CRUSH rule 4 x 447 [2,1]
+  CRUSH rule 4 x 448 [7,2]
+  CRUSH rule 4 x 449 [7,8]
+  CRUSH rule 4 x 450 [4,5]
+  CRUSH rule 4 x 451 [6,8]
+  CRUSH rule 4 x 452 [8,3]
+  CRUSH rule 4 x 453 [6,8]
+  CRUSH rule 4 x 454 [6,7]
+  CRUSH rule 4 x 455 [2,7]
+  CRUSH rule 4 x 456 [6,8]
+  CRUSH rule 4 x 457 [7,2]
+  CRUSH rule 4 x 458 [2,8]
+  CRUSH rule 4 x 459 [2,0]
+  CRUSH rule 4 x 460 [6,5]
+  CRUSH rule 4 x 461 [6,5]
+  CRUSH rule 4 x 462 [8,1]
+  CRUSH rule 4 x 463 [6,7]
+  CRUSH rule 4 x 464 [7,4]
+  CRUSH rule 4 x 465 [7,6]
+  CRUSH rule 4 x 466 [5,8]
+  CRUSH rule 4 x 467 [6,4]
+  CRUSH rule 4 x 468 [7,8]
+  CRUSH rule 4 x 469 [7,0]
+  CRUSH rule 4 x 470 [3,0]
+  CRUSH rule 4 x 471 [0,1]
+  CRUSH rule 4 x 472 [5,4]
+  CRUSH rule 4 x 473 [1,0]
+  CRUSH rule 4 x 474 [6,0]
+  CRUSH rule 4 x 475 [6,7]
+  CRUSH rule 4 x 476 [4,3]
+  CRUSH rule 4 x 477 [5,8]
+  CRUSH rule 4 x 478 [6,7]
+  CRUSH rule 4 x 479 [0,5]
+  CRUSH rule 4 x 480 [1,8]
+  CRUSH rule 4 x 481 [2,4]
+  CRUSH rule 4 x 482 [4,3]
+  CRUSH rule 4 x 483 [0,2]
+  CRUSH rule 4 x 484 [1,2]
+  CRUSH rule 4 x 485 [4,7]
+  CRUSH rule 4 x 486 [4,1]
+  CRUSH rule 4 x 487 [5,0]
+  CRUSH rule 4 x 488 [5,7]
+  CRUSH rule 4 x 489 [2,8]
+  CRUSH rule 4 x 490 [6,4]
+  CRUSH rule 4 x 491 [1,0]
+  CRUSH rule 4 x 492 [6,5]
+  CRUSH rule 4 x 493 [0,2]
+  CRUSH rule 4 x 494 [1,0]
+  CRUSH rule 4 x 495 [3,4]
+  CRUSH rule 4 x 496 [7,5]
+  CRUSH rule 4 x 497 [5,7]
+  CRUSH rule 4 x 498 [0,5]
+  CRUSH rule 4 x 499 [8,4]
+  CRUSH rule 4 x 500 [3,6]
+  CRUSH rule 4 x 501 [0,7]
+  CRUSH rule 4 x 502 [7,1]
+  CRUSH rule 4 x 503 [2,3]
+  CRUSH rule 4 x 504 [5,6]
+  CRUSH rule 4 x 505 [0,7]
+  CRUSH rule 4 x 506 [5,3]
+  CRUSH rule 4 x 507 [6,0]
+  CRUSH rule 4 x 508 [0,1]
+  CRUSH rule 4 x 509 [7,5]
+  CRUSH rule 4 x 510 [6,0]
+  CRUSH rule 4 x 511 [5,8]
+  CRUSH rule 4 x 512 [7,6]
+  CRUSH rule 4 x 513 [7,2]
+  CRUSH rule 4 x 514 [4,3]
+  CRUSH rule 4 x 515 [8,5]
+  CRUSH rule 4 x 516 [4,0]
+  CRUSH rule 4 x 517 [7,8]
+  CRUSH rule 4 x 518 [4,6]
+  CRUSH rule 4 x 519 [7,3]
+  CRUSH rule 4 x 520 [2,6]
+  CRUSH rule 4 x 521 [8,7]
+  CRUSH rule 4 x 522 [6,8]
+  CRUSH rule 4 x 523 [4,2]
+  CRUSH rule 4 x 524 [0,4]
+  CRUSH rule 4 x 525 [0,4]
+  CRUSH rule 4 x 526 [1,5]
+  CRUSH rule 4 x 527 [0,2]
+  CRUSH rule 4 x 528 [5,3]
+  CRUSH rule 4 x 529 [5,7]
+  CRUSH rule 4 x 530 [6,7]
+  CRUSH rule 4 x 531 [6,1]
+  CRUSH rule 4 x 532 [6,3]
+  CRUSH rule 4 x 533 [5,6]
+  CRUSH rule 4 x 534 [7,3]
+  CRUSH rule 4 x 535 [8,6]
+  CRUSH rule 4 x 536 [6,7]
+  CRUSH rule 4 x 537 [3,7]
+  CRUSH rule 4 x 538 [6,8]
+  CRUSH rule 4 x 539 [8,3]
+  CRUSH rule 4 x 540 [0,6]
+  CRUSH rule 4 x 541 [2,3]
+  CRUSH rule 4 x 542 [3,5]
+  CRUSH rule 4 x 543 [6,0]
+  CRUSH rule 4 x 544 [3,7]
+  CRUSH rule 4 x 545 [5,7]
+  CRUSH rule 4 x 546 [6,1]
+  CRUSH rule 4 x 547 [8,2]
+  CRUSH rule 4 x 548 [5,2]
+  CRUSH rule 4 x 549 [5,8]
+  CRUSH rule 4 x 550 [0,5]
+  CRUSH rule 4 x 551 [7,5]
+  CRUSH rule 4 x 552 [5,4]
+  CRUSH rule 4 x 553 [4,2]
+  CRUSH rule 4 x 554 [0,8]
+  CRUSH rule 4 x 555 [5,0]
+  CRUSH rule 4 x 556 [3,4]
+  CRUSH rule 4 x 557 [7,4]
+  CRUSH rule 4 x 558 [3,1]
+  CRUSH rule 4 x 559 [4,2]
+  CRUSH rule 4 x 560 [8,3]
+  CRUSH rule 4 x 561 [6,3]
+  CRUSH rule 4 x 562 [3,4]
+  CRUSH rule 4 x 563 [2,6]
+  CRUSH rule 4 x 564 [5,1]
+  CRUSH rule 4 x 565 [3,6]
+  CRUSH rule 4 x 566 [4,7]
+  CRUSH rule 4 x 567 [3,6]
+  CRUSH rule 4 x 568 [7,4]
+  CRUSH rule 4 x 569 [3,4]
+  CRUSH rule 4 x 570 [1,5]
+  CRUSH rule 4 x 571 [3,7]
+  CRUSH rule 4 x 572 [3,4]
+  CRUSH rule 4 x 573 [3,0]
+  CRUSH rule 4 x 574 [2,0]
+  CRUSH rule 4 x 575 [8,6]
+  CRUSH rule 4 x 576 [4,6]
+  CRUSH rule 4 x 577 [8,2]
+  CRUSH rule 4 x 578 [6,8]
+  CRUSH rule 4 x 579 [3,1]
+  CRUSH rule 4 x 580 [3,0]
+  CRUSH rule 4 x 581 [7,2]
+  CRUSH rule 4 x 582 [2,8]
+  CRUSH rule 4 x 583 [6,0]
+  CRUSH rule 4 x 584 [8,1]
+  CRUSH rule 4 x 585 [7,0]
+  CRUSH rule 4 x 586 [0,1]
+  CRUSH rule 4 x 587 [2,5]
+  CRUSH rule 4 x 588 [3,4]
+  CRUSH rule 4 x 589 [7,1]
+  CRUSH rule 4 x 590 [6,2]
+  CRUSH rule 4 x 591 [5,2]
+  CRUSH rule 4 x 592 [2,0]
+  CRUSH rule 4 x 593 [0,8]
+  CRUSH rule 4 x 594 [0,7]
+  CRUSH rule 4 x 595 [7,1]
+  CRUSH rule 4 x 596 [4,3]
+  CRUSH rule 4 x 597 [3,1]
+  CRUSH rule 4 x 598 [3,2]
+  CRUSH rule 4 x 599 [5,2]
+  CRUSH rule 4 x 600 [7,0]
+  CRUSH rule 4 x 601 [0,7]
+  CRUSH rule 4 x 602 [3,7]
+  CRUSH rule 4 x 603 [5,1]
+  CRUSH rule 4 x 604 [7,5]
+  CRUSH rule 4 x 605 [3,0]
+  CRUSH rule 4 x 606 [2,0]
+  CRUSH rule 4 x 607 [0,4]
+  CRUSH rule 4 x 608 [5,3]
+  CRUSH rule 4 x 609 [5,2]
+  CRUSH rule 4 x 610 [3,7]
+  CRUSH rule 4 x 611 [1,0]
+  CRUSH rule 4 x 612 [2,0]
+  CRUSH rule 4 x 613 [7,2]
+  CRUSH rule 4 x 614 [7,8]
+  CRUSH rule 4 x 615 [6,8]
+  CRUSH rule 4 x 616 [0,8]
+  CRUSH rule 4 x 617 [6,1]
+  CRUSH rule 4 x 618 [7,6]
+  CRUSH rule 4 x 619 [5,1]
+  CRUSH rule 4 x 620 [4,1]
+  CRUSH rule 4 x 621 [5,8]
+  CRUSH rule 4 x 622 [0,4]
+  CRUSH rule 4 x 623 [0,6]
+  CRUSH rule 4 x 624 [3,5]
+  CRUSH rule 4 x 625 [2,3]
+  CRUSH rule 4 x 626 [7,8]
+  CRUSH rule 4 x 627 [2,7]
+  CRUSH rule 4 x 628 [8,0]
+  CRUSH rule 4 x 629 [2,6]
+  CRUSH rule 4 x 630 [2,7]
+  CRUSH rule 4 x 631 [0,7]
+  CRUSH rule 4 x 632 [7,0]
+  CRUSH rule 4 x 633 [8,6]
+  CRUSH rule 4 x 634 [0,4]
+  CRUSH rule 4 x 635 [5,6]
+  CRUSH rule 4 x 636 [1,4]
+  CRUSH rule 4 x 637 [4,1]
+  CRUSH rule 4 x 638 [6,8]
+  CRUSH rule 4 x 639 [4,2]
+  CRUSH rule 4 x 640 [3,1]
+  CRUSH rule 4 x 641 [7,2]
+  CRUSH rule 4 x 642 [2,0]
+  CRUSH rule 4 x 643 [3,5]
+  CRUSH rule 4 x 644 [8,1]
+  CRUSH rule 4 x 645 [5,4]
+  CRUSH rule 4 x 646 [8,0]
+  CRUSH rule 4 x 647 [7,1]
+  CRUSH rule 4 x 648 [0,6]
+  CRUSH rule 4 x 649 [4,7]
+  CRUSH rule 4 x 650 [7,8]
+  CRUSH rule 4 x 651 [3,7]
+  CRUSH rule 4 x 652 [3,4]
+  CRUSH rule 4 x 653 [8,5]
+  CRUSH rule 4 x 654 [7,5]
+  CRUSH rule 4 x 655 [0,3]
+  CRUSH rule 4 x 656 [4,3]
+  CRUSH rule 4 x 657 [6,1]
+  CRUSH rule 4 x 658 [5,4]
+  CRUSH rule 4 x 659 [4,6]
+  CRUSH rule 4 x 660 [7,8]
+  CRUSH rule 4 x 661 [1,8]
+  CRUSH rule 4 x 662 [4,5]
+  CRUSH rule 4 x 663 [1,3]
+  CRUSH rule 4 x 664 [1,4]
+  CRUSH rule 4 x 665 [5,7]
+  CRUSH rule 4 x 666 [2,8]
+  CRUSH rule 4 x 667 [1,3]
+  CRUSH rule 4 x 668 [3,7]
+  CRUSH rule 4 x 669 [6,4]
+  CRUSH rule 4 x 670 [4,0]
+  CRUSH rule 4 x 671 [0,2]
+  CRUSH rule 4 x 672 [4,3]
+  CRUSH rule 4 x 673 [5,3]
+  CRUSH rule 4 x 674 [3,1]
+  CRUSH rule 4 x 675 [0,8]
+  CRUSH rule 4 x 676 [0,2]
+  CRUSH rule 4 x 677 [4,1]
+  CRUSH rule 4 x 678 [2,3]
+  CRUSH rule 4 x 679 [6,0]
+  CRUSH rule 4 x 680 [0,4]
+  CRUSH rule 4 x 681 [4,7]
+  CRUSH rule 4 x 682 [0,5]
+  CRUSH rule 4 x 683 [0,1]
+  CRUSH rule 4 x 684 [7,1]
+  CRUSH rule 4 x 685 [7,1]
+  CRUSH rule 4 x 686 [1,4]
+  CRUSH rule 4 x 687 [3,5]
+  CRUSH rule 4 x 688 [5,7]
+  CRUSH rule 4 x 689 [6,5]
+  CRUSH rule 4 x 690 [8,1]
+  CRUSH rule 4 x 691 [3,0]
+  CRUSH rule 4 x 692 [7,2]
+  CRUSH rule 4 x 693 [6,7]
+  CRUSH rule 4 x 694 [6,5]
+  CRUSH rule 4 x 695 [0,8]
+  CRUSH rule 4 x 696 [1,4]
+  CRUSH rule 4 x 697 [6,1]
+  CRUSH rule 4 x 698 [6,2]
+  CRUSH rule 4 x 699 [1,6]
+  CRUSH rule 4 x 700 [0,3]
+  CRUSH rule 4 x 701 [4,3]
+  CRUSH rule 4 x 702 [3,5]
+  CRUSH rule 4 x 703 [8,3]
+  CRUSH rule 4 x 704 [0,3]
+  CRUSH rule 4 x 705 [8,6]
+  CRUSH rule 4 x 706 [1,2]
+  CRUSH rule 4 x 707 [7,8]
+  CRUSH rule 4 x 708 [3,5]
+  CRUSH rule 4 x 709 [6,3]
+  CRUSH rule 4 x 710 [8,4]
+  CRUSH rule 4 x 711 [2,3]
+  CRUSH rule 4 x 712 [2,3]
+  CRUSH rule 4 x 713 [6,7]
+  CRUSH rule 4 x 714 [3,2]
+  CRUSH rule 4 x 715 [1,2]
+  CRUSH rule 4 x 716 [3,6]
+  CRUSH rule 4 x 717 [8,7]
+  CRUSH rule 4 x 718 [3,7]
+  CRUSH rule 4 x 719 [2,6]
+  CRUSH rule 4 x 720 [6,8]
+  CRUSH rule 4 x 721 [5,4]
+  CRUSH rule 4 x 722 [5,4]
+  CRUSH rule 4 x 723 [5,1]
+  CRUSH rule 4 x 724 [0,6]
+  CRUSH rule 4 x 725 [0,1]
+  CRUSH rule 4 x 726 [3,8]
+  CRUSH rule 4 x 727 [4,6]
+  CRUSH rule 4 x 728 [2,1]
+  CRUSH rule 4 x 729 [5,3]
+  CRUSH rule 4 x 730 [3,7]
+  CRUSH rule 4 x 731 [4,1]
+  CRUSH rule 4 x 732 [1,5]
+  CRUSH rule 4 x 733 [5,4]
+  CRUSH rule 4 x 734 [6,4]
+  CRUSH rule 4 x 735 [4,8]
+  CRUSH rule 4 x 736 [3,5]
+  CRUSH rule 4 x 737 [1,0]
+  CRUSH rule 4 x 738 [5,2]
+  CRUSH rule 4 x 739 [0,1]
+  CRUSH rule 4 x 740 [0,1]
+  CRUSH rule 4 x 741 [7,8]
+  CRUSH rule 4 x 742 [8,2]
+  CRUSH rule 4 x 743 [7,0]
+  CRUSH rule 4 x 744 [4,7]
+  CRUSH rule 4 x 745 [3,4]
+  CRUSH rule 4 x 746 [4,1]
+  CRUSH rule 4 x 747 [6,0]
+  CRUSH rule 4 x 748 [2,7]
+  CRUSH rule 4 x 749 [4,5]
+  CRUSH rule 4 x 750 [1,6]
+  CRUSH rule 4 x 751 [2,1]
+  CRUSH rule 4 x 752 [8,1]
+  CRUSH rule 4 x 753 [7,8]
+  CRUSH rule 4 x 754 [8,6]
+  CRUSH rule 4 x 755 [1,2]
+  CRUSH rule 4 x 756 [5,6]
+  CRUSH rule 4 x 757 [8,6]
+  CRUSH rule 4 x 758 [6,0]
+  CRUSH rule 4 x 759 [8,5]
+  CRUSH rule 4 x 760 [1,5]
+  CRUSH rule 4 x 761 [4,1]
+  CRUSH rule 4 x 762 [2,7]
+  CRUSH rule 4 x 763 [8,6]
+  CRUSH rule 4 x 764 [1,7]
+  CRUSH rule 4 x 765 [6,5]
+  CRUSH rule 4 x 766 [8,5]
+  CRUSH rule 4 x 767 [1,2]
+  CRUSH rule 4 x 768 [8,3]
+  CRUSH rule 4 x 769 [6,2]
+  CRUSH rule 4 x 770 [6,0]
+  CRUSH rule 4 x 771 [7,0]
+  CRUSH rule 4 x 772 [8,3]
+  CRUSH rule 4 x 773 [3,1]
+  CRUSH rule 4 x 774 [4,6]
+  CRUSH rule 4 x 775 [6,8]
+  CRUSH rule 4 x 776 [7,2]
+  CRUSH rule 4 x 777 [3,1]
+  CRUSH rule 4 x 778 [1,8]
+  CRUSH rule 4 x 779 [2,7]
+  CRUSH rule 4 x 780 [0,2]
+  CRUSH rule 4 x 781 [6,3]
+  CRUSH rule 4 x 782 [5,4]
+  CRUSH rule 4 x 783 [7,1]
+  CRUSH rule 4 x 784 [0,1]
+  CRUSH rule 4 x 785 [6,1]
+  CRUSH rule 4 x 786 [7,6]
+  CRUSH rule 4 x 787 [1,0]
+  CRUSH rule 4 x 788 [6,0]
+  CRUSH rule 4 x 789 [0,4]
+  CRUSH rule 4 x 790 [8,4]
+  CRUSH rule 4 x 791 [3,8]
+  CRUSH rule 4 x 792 [5,8]
+  CRUSH rule 4 x 793 [6,1]
+  CRUSH rule 4 x 794 [2,6]
+  CRUSH rule 4 x 795 [0,3]
+  CRUSH rule 4 x 796 [3,4]
+  CRUSH rule 4 x 797 [2,3]
+  CRUSH rule 4 x 798 [6,8]
+  CRUSH rule 4 x 799 [5,1]
+  CRUSH rule 4 x 800 [5,0]
+  CRUSH rule 4 x 801 [3,6]
+  CRUSH rule 4 x 802 [1,8]
+  CRUSH rule 4 x 803 [0,5]
+  CRUSH rule 4 x 804 [6,2]
+  CRUSH rule 4 x 805 [3,6]
+  CRUSH rule 4 x 806 [1,3]
+  CRUSH rule 4 x 807 [5,4]
+  CRUSH rule 4 x 808 [4,6]
+  CRUSH rule 4 x 809 [1,4]
+  CRUSH rule 4 x 810 [5,7]
+  CRUSH rule 4 x 811 [8,4]
+  CRUSH rule 4 x 812 [8,5]
+  CRUSH rule 4 x 813 [6,4]
+  CRUSH rule 4 x 814 [3,6]
+  CRUSH rule 4 x 815 [3,1]
+  CRUSH rule 4 x 816 [2,1]
+  CRUSH rule 4 x 817 [4,3]
+  CRUSH rule 4 x 818 [3,5]
+  CRUSH rule 4 x 819 [5,1]
+  CRUSH rule 4 x 820 [3,5]
+  CRUSH rule 4 x 821 [4,5]
+  CRUSH rule 4 x 822 [2,0]
+  CRUSH rule 4 x 823 [4,8]
+  CRUSH rule 4 x 824 [3,7]
+  CRUSH rule 4 x 825 [2,8]
+  CRUSH rule 4 x 826 [7,0]
+  CRUSH rule 4 x 827 [0,8]
+  CRUSH rule 4 x 828 [2,3]
+  CRUSH rule 4 x 829 [5,6]
+  CRUSH rule 4 x 830 [2,3]
+  CRUSH rule 4 x 831 [1,6]
+  CRUSH rule 4 x 832 [4,5]
+  CRUSH rule 4 x 833 [2,1]
+  CRUSH rule 4 x 834 [3,4]
+  CRUSH rule 4 x 835 [8,4]
+  CRUSH rule 4 x 836 [3,4]
+  CRUSH rule 4 x 837 [6,3]
+  CRUSH rule 4 x 838 [6,7]
+  CRUSH rule 4 x 839 [5,0]
+  CRUSH rule 4 x 840 [7,8]
+  CRUSH rule 4 x 841 [4,8]
+  CRUSH rule 4 x 842 [2,4]
+  CRUSH rule 4 x 843 [6,4]
+  CRUSH rule 4 x 844 [4,8]
+  CRUSH rule 4 x 845 [3,8]
+  CRUSH rule 4 x 846 [3,2]
+  CRUSH rule 4 x 847 [0,2]
+  CRUSH rule 4 x 848 [2,6]
+  CRUSH rule 4 x 849 [4,5]
+  CRUSH rule 4 x 850 [1,0]
+  CRUSH rule 4 x 851 [6,8]
+  CRUSH rule 4 x 852 [7,3]
+  CRUSH rule 4 x 853 [6,8]
+  CRUSH rule 4 x 854 [7,6]
+  CRUSH rule 4 x 855 [5,7]
+  CRUSH rule 4 x 856 [6,7]
+  CRUSH rule 4 x 857 [8,5]
+  CRUSH rule 4 x 858 [6,4]
+  CRUSH rule 4 x 859 [6,0]
+  CRUSH rule 4 x 860 [4,1]
+  CRUSH rule 4 x 861 [8,7]
+  CRUSH rule 4 x 862 [6,1]
+  CRUSH rule 4 x 863 [8,7]
+  CRUSH rule 4 x 864 [5,6]
+  CRUSH rule 4 x 865 [8,1]
+  CRUSH rule 4 x 866 [3,4]
+  CRUSH rule 4 x 867 [6,5]
+  CRUSH rule 4 x 868 [6,3]
+  CRUSH rule 4 x 869 [8,7]
+  CRUSH rule 4 x 870 [0,4]
+  CRUSH rule 4 x 871 [3,4]
+  CRUSH rule 4 x 872 [5,1]
+  CRUSH rule 4 x 873 [4,6]
+  CRUSH rule 4 x 874 [2,6]
+  CRUSH rule 4 x 875 [2,6]
+  CRUSH rule 4 x 876 [5,8]
+  CRUSH rule 4 x 877 [6,4]
+  CRUSH rule 4 x 878 [5,4]
+  CRUSH rule 4 x 879 [7,4]
+  CRUSH rule 4 x 880 [3,5]
+  CRUSH rule 4 x 881 [5,6]
+  CRUSH rule 4 x 882 [4,0]
+  CRUSH rule 4 x 883 [2,1]
+  CRUSH rule 4 x 884 [6,0]
+  CRUSH rule 4 x 885 [5,1]
+  CRUSH rule 4 x 886 [3,6]
+  CRUSH rule 4 x 887 [7,4]
+  CRUSH rule 4 x 888 [6,8]
+  CRUSH rule 4 x 889 [2,1]
+  CRUSH rule 4 x 890 [7,2]
+  CRUSH rule 4 x 891 [1,8]
+  CRUSH rule 4 x 892 [6,2]
+  CRUSH rule 4 x 893 [2,3]
+  CRUSH rule 4 x 894 [7,5]
+  CRUSH rule 4 x 895 [5,3]
+  CRUSH rule 4 x 896 [1,8]
+  CRUSH rule 4 x 897 [4,2]
+  CRUSH rule 4 x 898 [0,5]
+  CRUSH rule 4 x 899 [1,7]
+  CRUSH rule 4 x 900 [4,1]
+  CRUSH rule 4 x 901 [5,0]
+  CRUSH rule 4 x 902 [8,5]
+  CRUSH rule 4 x 903 [5,7]
+  CRUSH rule 4 x 904 [5,6]
+  CRUSH rule 4 x 905 [6,2]
+  CRUSH rule 4 x 906 [1,2]
+  CRUSH rule 4 x 907 [7,1]
+  CRUSH rule 4 x 908 [5,8]
+  CRUSH rule 4 x 909 [2,3]
+  CRUSH rule 4 x 910 [6,4]
+  CRUSH rule 4 x 911 [5,8]
+  CRUSH rule 4 x 912 [0,1]
+  CRUSH rule 4 x 913 [7,6]
+  CRUSH rule 4 x 914 [6,4]
+  CRUSH rule 4 x 915 [8,2]
+  CRUSH rule 4 x 916 [3,1]
+  CRUSH rule 4 x 917 [1,5]
+  CRUSH rule 4 x 918 [8,2]
+  CRUSH rule 4 x 919 [6,2]
+  CRUSH rule 4 x 920 [7,6]
+  CRUSH rule 4 x 921 [1,4]
+  CRUSH rule 4 x 922 [6,7]
+  CRUSH rule 4 x 923 [5,3]
+  CRUSH rule 4 x 924 [3,5]
+  CRUSH rule 4 x 925 [5,7]
+  CRUSH rule 4 x 926 [3,4]
+  CRUSH rule 4 x 927 [1,6]
+  CRUSH rule 4 x 928 [8,1]
+  CRUSH rule 4 x 929 [4,5]
+  CRUSH rule 4 x 930 [2,4]
+  CRUSH rule 4 x 931 [5,0]
+  CRUSH rule 4 x 932 [4,3]
+  CRUSH rule 4 x 933 [8,5]
+  CRUSH rule 4 x 934 [5,3]
+  CRUSH rule 4 x 935 [6,3]
+  CRUSH rule 4 x 936 [0,6]
+  CRUSH rule 4 x 937 [5,4]
+  CRUSH rule 4 x 938 [6,5]
+  CRUSH rule 4 x 939 [2,7]
+  CRUSH rule 4 x 940 [8,7]
+  CRUSH rule 4 x 941 [5,2]
+  CRUSH rule 4 x 942 [1,0]
+  CRUSH rule 4 x 943 [8,2]
+  CRUSH rule 4 x 944 [4,3]
+  CRUSH rule 4 x 945 [7,2]
+  CRUSH rule 4 x 946 [2,0]
+  CRUSH rule 4 x 947 [4,5]
+  CRUSH rule 4 x 948 [7,8]
+  CRUSH rule 4 x 949 [6,1]
+  CRUSH rule 4 x 950 [3,5]
+  CRUSH rule 4 x 951 [4,5]
+  CRUSH rule 4 x 952 [2,0]
+  CRUSH rule 4 x 953 [1,3]
+  CRUSH rule 4 x 954 [4,2]
+  CRUSH rule 4 x 955 [8,6]
+  CRUSH rule 4 x 956 [1,0]
+  CRUSH rule 4 x 957 [7,6]
+  CRUSH rule 4 x 958 [8,7]
+  CRUSH rule 4 x 959 [5,2]
+  CRUSH rule 4 x 960 [3,6]
+  CRUSH rule 4 x 961 [4,0]
+  CRUSH rule 4 x 962 [7,4]
+  CRUSH rule 4 x 963 [0,5]
+  CRUSH rule 4 x 964 [3,1]
+  CRUSH rule 4 x 965 [7,6]
+  CRUSH rule 4 x 966 [3,8]
+  CRUSH rule 4 x 967 [8,6]
+  CRUSH rule 4 x 968 [7,2]
+  CRUSH rule 4 x 969 [8,0]
+  CRUSH rule 4 x 970 [0,6]
+  CRUSH rule 4 x 971 [1,7]
+  CRUSH rule 4 x 972 [1,8]
+  CRUSH rule 4 x 973 [1,2]
+  CRUSH rule 4 x 974 [5,3]
+  CRUSH rule 4 x 975 [3,7]
+  CRUSH rule 4 x 976 [4,3]
+  CRUSH rule 4 x 977 [8,3]
+  CRUSH rule 4 x 978 [7,2]
+  CRUSH rule 4 x 979 [7,6]
+  CRUSH rule 4 x 980 [6,0]
+  CRUSH rule 4 x 981 [7,3]
+  CRUSH rule 4 x 982 [4,2]
+  CRUSH rule 4 x 983 [3,5]
+  CRUSH rule 4 x 984 [0,2]
+  CRUSH rule 4 x 985 [2,5]
+  CRUSH rule 4 x 986 [8,7]
+  CRUSH rule 4 x 987 [0,5]
+  CRUSH rule 4 x 988 [1,3]
+  CRUSH rule 4 x 989 [0,6]
+  CRUSH rule 4 x 990 [1,0]
+  CRUSH rule 4 x 991 [0,4]
+  CRUSH rule 4 x 992 [7,1]
+  CRUSH rule 4 x 993 [0,6]
+  CRUSH rule 4 x 994 [3,4]
+  CRUSH rule 4 x 995 [7,6]
+  CRUSH rule 4 x 996 [6,7]
+  CRUSH rule 4 x 997 [6,4]
+  CRUSH rule 4 x 998 [8,1]
+  CRUSH rule 4 x 999 [0,7]
+  CRUSH rule 4 x 1000 [8,5]
+  CRUSH rule 4 x 1001 [2,0]
+  CRUSH rule 4 x 1002 [1,3]
+  CRUSH rule 4 x 1003 [2,8]
+  CRUSH rule 4 x 1004 [6,1]
+  CRUSH rule 4 x 1005 [6,1]
+  CRUSH rule 4 x 1006 [1,0]
+  CRUSH rule 4 x 1007 [1,2]
+  CRUSH rule 4 x 1008 [1,7]
+  CRUSH rule 4 x 1009 [6,8]
+  CRUSH rule 4 x 1010 [3,4]
+  CRUSH rule 4 x 1011 [3,0]
+  CRUSH rule 4 x 1012 [3,0]
+  CRUSH rule 4 x 1013 [5,1]
+  CRUSH rule 4 x 1014 [2,8]
+  CRUSH rule 4 x 1015 [6,8]
+  CRUSH rule 4 x 1016 [2,0]
+  CRUSH rule 4 x 1017 [6,0]
+  CRUSH rule 4 x 1018 [5,4]
+  CRUSH rule 4 x 1019 [5,3]
+  CRUSH rule 4 x 1020 [5,1]
+  CRUSH rule 4 x 1021 [5,2]
+  CRUSH rule 4 x 1022 [1,6]
+  CRUSH rule 4 x 1023 [3,2]
+  rule 4 (choose-set-two) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 4 x 0 [0,2,3]
+  CRUSH rule 4 x 1 [0,8,2]
+  CRUSH rule 4 x 2 [1,3,5]
+  CRUSH rule 4 x 3 [8,0,6]
+  CRUSH rule 4 x 4 [5,4,0]
+  CRUSH rule 4 x 5 [7,8,0]
+  CRUSH rule 4 x 6 [2,6,7]
+  CRUSH rule 4 x 7 [5,4,3]
+  CRUSH rule 4 x 8 [5,3,8]
+  CRUSH rule 4 x 9 [2,3,1]
+  CRUSH rule 4 x 10 [0,2,8]
+  CRUSH rule 4 x 11 [0,7,2]
+  CRUSH rule 4 x 12 [0,2,4]
+  CRUSH rule 4 x 13 [3,8,5]
+  CRUSH rule 4 x 14 [7,6,8]
+  CRUSH rule 4 x 15 [7,2,4]
+  CRUSH rule 4 x 16 [3,6,0]
+  CRUSH rule 4 x 17 [5,4,0]
+  CRUSH rule 4 x 18 [1,4,5]
+  CRUSH rule 4 x 19 [7,5,4]
+  CRUSH rule 4 x 20 [2,4,7]
+  CRUSH rule 4 x 21 [3,7,2]
+  CRUSH rule 4 x 22 [8,3,7]
+  CRUSH rule 4 x 23 [3,6,8]
+  CRUSH rule 4 x 24 [1,0,6]
+  CRUSH rule 4 x 25 [3,7,0]
+  CRUSH rule 4 x 26 [2,8,7]
+  CRUSH rule 4 x 27 [3,1,5]
+  CRUSH rule 4 x 28 [6,0,3]
+  CRUSH rule 4 x 29 [8,5,7]
+  CRUSH rule 4 x 30 [5,7,3]
+  CRUSH rule 4 x 31 [8,7,6]
+  CRUSH rule 4 x 32 [3,6,8]
+  CRUSH rule 4 x 33 [2,7,8]
+  CRUSH rule 4 x 34 [2,5,4]
+  CRUSH rule 4 x 35 [0,8,1]
+  CRUSH rule 4 x 36 [3,8,2]
+  CRUSH rule 4 x 37 [0,4,5]
+  CRUSH rule 4 x 38 [4,8,7]
+  CRUSH rule 4 x 39 [3,7,0]
+  CRUSH rule 4 x 40 [7,8,1]
+  CRUSH rule 4 x 41 [0,2,6]
+  CRUSH rule 4 x 42 [4,3,6]
+  CRUSH rule 4 x 43 [0,3,7]
+  CRUSH rule 4 x 44 [1,6,7]
+  CRUSH rule 4 x 45 [8,0,4]
+  CRUSH rule 4 x 46 [2,4,3]
+  CRUSH rule 4 x 47 [4,5,3]
+  CRUSH rule 4 x 48 [4,6,5]
+  CRUSH rule 4 x 49 [5,4,8]
+  CRUSH rule 4 x 50 [3,4,1]
+  CRUSH rule 4 x 51 [3,6,0]
+  CRUSH rule 4 x 52 [8,6,1]
+  CRUSH rule 4 x 53 [3,5,4]
+  CRUSH rule 4 x 54 [7,6,8]
+  CRUSH rule 4 x 55 [8,7,0]
+  CRUSH rule 4 x 56 [6,4,8]
+  CRUSH rule 4 x 57 [5,3,6]
+  CRUSH rule 4 x 58 [1,0,2]
+  CRUSH rule 4 x 59 [4,2,1]
+  CRUSH rule 4 x 60 [3,5,0]
+  CRUSH rule 4 x 61 [4,6,8]
+  CRUSH rule 4 x 62 [7,0,1]
+  CRUSH rule 4 x 63 [5,6,8]
+  CRUSH rule 4 x 64 [4,5,1]
+  CRUSH rule 4 x 65 [7,3,6]
+  CRUSH rule 4 x 66 [5,4,3]
+  CRUSH rule 4 x 67 [5,0,8]
+  CRUSH rule 4 x 68 [0,5,8]
+  CRUSH rule 4 x 69 [5,1,6]
+  CRUSH rule 4 x 70 [7,0,2]
+  CRUSH rule 4 x 71 [2,8,6]
+  CRUSH rule 4 x 72 [6,1,5]
+  CRUSH rule 4 x 73 [2,7,3]
+  CRUSH rule 4 x 74 [0,7,1]
+  CRUSH rule 4 x 75 [3,2,0]
+  CRUSH rule 4 x 76 [5,1,7]
+  CRUSH rule 4 x 77 [7,2,6]
+  CRUSH rule 4 x 78 [1,4,2]
+  CRUSH rule 4 x 79 [5,4,0]
+  CRUSH rule 4 x 80 [0,3,5]
+  CRUSH rule 4 x 81 [0,2,5]
+  CRUSH rule 4 x 82 [7,1,6]
+  CRUSH rule 4 x 83 [2,6,8]
+  CRUSH rule 4 x 84 [7,2,1]
+  CRUSH rule 4 x 85 [3,4,8]
+  CRUSH rule 4 x 86 [0,2,1]
+  CRUSH rule 4 x 87 [0,7,4]
+  CRUSH rule 4 x 88 [1,6,0]
+  CRUSH rule 4 x 89 [3,0,2]
+  CRUSH rule 4 x 90 [6,7,5]
+  CRUSH rule 4 x 91 [3,8,7]
+  CRUSH rule 4 x 92 [1,8,4]
+  CRUSH rule 4 x 93 [7,4,5]
+  CRUSH rule 4 x 94 [0,4,3]
+  CRUSH rule 4 x 95 [7,5,1]
+  CRUSH rule 4 x 96 [3,6,4]
+  CRUSH rule 4 x 97 [8,7,4]
+  CRUSH rule 4 x 98 [2,0,7]
+  CRUSH rule 4 x 99 [0,7,2]
+  CRUSH rule 4 x 100 [1,7,4]
+  CRUSH rule 4 x 101 [3,7,2]
+  CRUSH rule 4 x 102 [4,2,7]
+  CRUSH rule 4 x 103 [4,7,6]
+  CRUSH rule 4 x 104 [7,4,3]
+  CRUSH rule 4 x 105 [2,4,1]
+  CRUSH rule 4 x 106 [1,6,2]
+  CRUSH rule 4 x 107 [3,2,1]
+  CRUSH rule 4 x 108 [7,2,8]
+  CRUSH rule 4 x 109 [1,2,5]
+  CRUSH rule 4 x 110 [3,2,7]
+  CRUSH rule 4 x 111 [2,1,0]
+  CRUSH rule 4 x 112 [2,0,6]
+  CRUSH rule 4 x 113 [6,2,1]
+  CRUSH rule 4 x 114 [7,6,3]
+  CRUSH rule 4 x 115 [8,2,3]
+  CRUSH rule 4 x 116 [1,2,6]
+  CRUSH rule 4 x 117 [7,3,6]
+  CRUSH rule 4 x 118 [0,3,8]
+  CRUSH rule 4 x 119 [5,6,1]
+  CRUSH rule 4 x 120 [0,2,5]
+  CRUSH rule 4 x 121 [2,0,1]
+  CRUSH rule 4 x 122 [8,5,0]
+  CRUSH rule 4 x 123 [2,5,1]
+  CRUSH rule 4 x 124 [3,5,1]
+  CRUSH rule 4 x 125 [0,7,3]
+  CRUSH rule 4 x 126 [4,2,6]
+  CRUSH rule 4 x 127 [3,6,7]
+  CRUSH rule 4 x 128 [3,5,8]
+  CRUSH rule 4 x 129 [0,2,1]
+  CRUSH rule 4 x 130 [3,8,5]
+  CRUSH rule 4 x 131 [1,2,0]
+  CRUSH rule 4 x 132 [1,2,3]
+  CRUSH rule 4 x 133 [3,6,7]
+  CRUSH rule 4 x 134 [1,8,2]
+  CRUSH rule 4 x 135 [5,6,4]
+  CRUSH rule 4 x 136 [2,1,5]
+  CRUSH rule 4 x 137 [7,3,4]
+  CRUSH rule 4 x 138 [8,7,6]
+  CRUSH rule 4 x 139 [3,0,5]
+  CRUSH rule 4 x 140 [1,6,7]
+  CRUSH rule 4 x 141 [6,8,1]
+  CRUSH rule 4 x 142 [3,5,0]
+  CRUSH rule 4 x 143 [5,8,7]
+  CRUSH rule 4 x 144 [8,1,0]
+  CRUSH rule 4 x 145 [8,5,6]
+  CRUSH rule 4 x 146 [2,6,4]
+  CRUSH rule 4 x 147 [2,8,7]
+  CRUSH rule 4 x 148 [3,4,0]
+  CRUSH rule 4 x 149 [4,8,6]
+  CRUSH rule 4 x 150 [1,6,5]
+  CRUSH rule 4 x 151 [3,4,5]
+  CRUSH rule 4 x 152 [8,4,6]
+  CRUSH rule 4 x 153 [8,6,7]
+  CRUSH rule 4 x 154 [3,2,0]
+  CRUSH rule 4 x 155 [3,5,8]
+  CRUSH rule 4 x 156 [4,5,3]
+  CRUSH rule 4 x 157 [4,1,2]
+  CRUSH rule 4 x 158 [2,8,1]
+  CRUSH rule 4 x 159 [7,0,5]
+  CRUSH rule 4 x 160 [2,8,1]
+  CRUSH rule 4 x 161 [1,5,2]
+  CRUSH rule 4 x 162 [0,6,2]
+  CRUSH rule 4 x 163 [5,6,1]
+  CRUSH rule 4 x 164 [7,8,1]
+  CRUSH rule 4 x 165 [7,0,5]
+  CRUSH rule 4 x 166 [2,4,1]
+  CRUSH rule 4 x 167 [0,1,6]
+  CRUSH rule 4 x 168 [4,2,3]
+  CRUSH rule 4 x 169 [2,6,1]
+  CRUSH rule 4 x 170 [1,0,5]
+  CRUSH rule 4 x 171 [7,5,4]
+  CRUSH rule 4 x 172 [0,7,8]
+  CRUSH rule 4 x 173 [8,5,3]
+  CRUSH rule 4 x 174 [1,4,7]
+  CRUSH rule 4 x 175 [6,0,4]
+  CRUSH rule 4 x 176 [4,3,5]
+  CRUSH rule 4 x 177 [5,3,0]
+  CRUSH rule 4 x 178 [3,0,2]
+  CRUSH rule 4 x 179 [4,1,2]
+  CRUSH rule 4 x 180 [3,8,7]
+  CRUSH rule 4 x 181 [6,2,0]
+  CRUSH rule 4 x 182 [8,5,3]
+  CRUSH rule 4 x 183 [7,8,6]
+  CRUSH rule 4 x 184 [5,7,3]
+  CRUSH rule 4 x 185 [6,8,0]
+  CRUSH rule 4 x 186 [2,1,4]
+  CRUSH rule 4 x 187 [1,6,7]
+  CRUSH rule 4 x 188 [1,8,7]
+  CRUSH rule 4 x 189 [0,7,8]
+  CRUSH rule 4 x 190 [4,0,1]
+  CRUSH rule 4 x 191 [7,6,8]
+  CRUSH rule 4 x 192 [5,0,1]
+  CRUSH rule 4 x 193 [4,2,6]
+  CRUSH rule 4 x 194 [1,3,4]
+  CRUSH rule 4 x 195 [6,4,3]
+  CRUSH rule 4 x 196 [6,7,2]
+  CRUSH rule 4 x 197 [6,5,2]
+  CRUSH rule 4 x 198 [2,5,6]
+  CRUSH rule 4 x 199 [0,5,7]
+  CRUSH rule 4 x 200 [0,5,1]
+  CRUSH rule 4 x 201 [7,1,0]
+  CRUSH rule 4 x 202 [6,3,4]
+  CRUSH rule 4 x 203 [4,8,7]
+  CRUSH rule 4 x 204 [2,1,5]
+  CRUSH rule 4 x 205 [0,7,6]
+  CRUSH rule 4 x 206 [0,1,7]
+  CRUSH rule 4 x 207 [3,0,5]
+  CRUSH rule 4 x 208 [7,1,0]
+  CRUSH rule 4 x 209 [1,2,0]
+  CRUSH rule 4 x 210 [1,2,3]
+  CRUSH rule 4 x 211 [5,4,2]
+  CRUSH rule 4 x 212 [7,5,0]
+  CRUSH rule 4 x 213 [8,4,0]
+  CRUSH rule 4 x 214 [4,5,7]
+  CRUSH rule 4 x 215 [8,1,7]
+  CRUSH rule 4 x 216 [5,0,2]
+  CRUSH rule 4 x 217 [0,1,7]
+  CRUSH rule 4 x 218 [0,7,8]
+  CRUSH rule 4 x 219 [4,8,2]
+  CRUSH rule 4 x 220 [5,7,3]
+  CRUSH rule 4 x 221 [3,6,5]
+  CRUSH rule 4 x 222 [6,8,3]
+  CRUSH rule 4 x 223 [1,3,6]
+  CRUSH rule 4 x 224 [1,5,0]
+  CRUSH rule 4 x 225 [8,6,2]
+  CRUSH rule 4 x 226 [7,2,3]
+  CRUSH rule 4 x 227 [3,4,0]
+  CRUSH rule 4 x 228 [5,4,8]
+  CRUSH rule 4 x 229 [3,4,7]
+  CRUSH rule 4 x 230 [4,7,2]
+  CRUSH rule 4 x 231 [4,3,5]
+  CRUSH rule 4 x 232 [2,7,4]
+  CRUSH rule 4 x 233 [3,4,5]
+  CRUSH rule 4 x 234 [0,1,2]
+  CRUSH rule 4 x 235 [3,8,0]
+  CRUSH rule 4 x 236 [5,2,7]
+  CRUSH rule 4 x 237 [4,7,6]
+  CRUSH rule 4 x 238 [4,3,5]
+  CRUSH rule 4 x 239 [8,7,6]
+  CRUSH rule 4 x 240 [5,7,4]
+  CRUSH rule 4 x 241 [3,1,5]
+  CRUSH rule 4 x 242 [3,5,4]
+  CRUSH rule 4 x 243 [4,7,3]
+  CRUSH rule 4 x 244 [4,6,0]
+  CRUSH rule 4 x 245 [7,6,1]
+  CRUSH rule 4 x 246 [1,5,2]
+  CRUSH rule 4 x 247 [6,0,2]
+  CRUSH rule 4 x 248 [8,0,7]
+  CRUSH rule 4 x 249 [2,0,1]
+  CRUSH rule 4 x 250 [2,1,0]
+  CRUSH rule 4 x 251 [2,3,6]
+  CRUSH rule 4 x 252 [3,7,8]
+  CRUSH rule 4 x 253 [3,2,4]
+  CRUSH rule 4 x 254 [3,5,4]
+  CRUSH rule 4 x 255 [1,7,8]
+  CRUSH rule 4 x 256 [5,7,0]
+  CRUSH rule 4 x 257 [2,8,4]
+  CRUSH rule 4 x 258 [5,3,0]
+  CRUSH rule 4 x 259 [4,3,6]
+  CRUSH rule 4 x 260 [3,6,2]
+  CRUSH rule 4 x 261 [8,7,4]
+  CRUSH rule 4 x 262 [5,4,8]
+  CRUSH rule 4 x 263 [6,8,2]
+  CRUSH rule 4 x 264 [3,6,5]
+  CRUSH rule 4 x 265 [8,6,7]
+  CRUSH rule 4 x 266 [8,2,1]
+  CRUSH rule 4 x 267 [2,3,0]
+  CRUSH rule 4 x 268 [0,7,2]
+  CRUSH rule 4 x 269 [0,8,2]
+  CRUSH rule 4 x 270 [5,0,1]
+  CRUSH rule 4 x 271 [7,5,4]
+  CRUSH rule 4 x 272 [2,8,3]
+  CRUSH rule 4 x 273 [3,5,0]
+  CRUSH rule 4 x 274 [6,8,3]
+  CRUSH rule 4 x 275 [4,3,6]
+  CRUSH rule 4 x 276 [7,1,2]
+  CRUSH rule 4 x 277 [6,4,0]
+  CRUSH rule 4 x 278 [6,8,0]
+  CRUSH rule 4 x 279 [8,3,6]
+  CRUSH rule 4 x 280 [0,6,2]
+  CRUSH rule 4 x 281 [8,0,2]
+  CRUSH rule 4 x 282 [3,1,6]
+  CRUSH rule 4 x 283 [8,2,0]
+  CRUSH rule 4 x 284 [6,3,0]
+  CRUSH rule 4 x 285 [5,3,7]
+  CRUSH rule 4 x 286 [2,1,6]
+  CRUSH rule 4 x 287 [0,4,1]
+  CRUSH rule 4 x 288 [8,0,7]
+  CRUSH rule 4 x 289 [4,6,2]
+  CRUSH rule 4 x 290 [1,3,7]
+  CRUSH rule 4 x 291 [0,1,4]
+  CRUSH rule 4 x 292 [8,0,2]
+  CRUSH rule 4 x 293 [6,0,8]
+  CRUSH rule 4 x 294 [7,4,5]
+  CRUSH rule 4 x 295 [4,8,7]
+  CRUSH rule 4 x 296 [3,1,5]
+  CRUSH rule 4 x 297 [6,2,8]
+  CRUSH rule 4 x 298 [1,2,3]
+  CRUSH rule 4 x 299 [2,0,7]
+  CRUSH rule 4 x 300 [8,7,3]
+  CRUSH rule 4 x 301 [0,8,1]
+  CRUSH rule 4 x 302 [3,0,6]
+  CRUSH rule 4 x 303 [7,5,8]
+  CRUSH rule 4 x 304 [2,7,5]
+  CRUSH rule 4 x 305 [5,8,2]
+  CRUSH rule 4 x 306 [0,7,1]
+  CRUSH rule 4 x 307 [0,2,8]
+  CRUSH rule 4 x 308 [0,8,4]
+  CRUSH rule 4 x 309 [7,4,5]
+  CRUSH rule 4 x 310 [4,3,1]
+  CRUSH rule 4 x 311 [3,4,5]
+  CRUSH rule 4 x 312 [2,1,0]
+  CRUSH rule 4 x 313 [5,3,4]
+  CRUSH rule 4 x 314 [4,5,0]
+  CRUSH rule 4 x 315 [2,0,1]
+  CRUSH rule 4 x 316 [6,3,8]
+  CRUSH rule 4 x 317 [2,6,0]
+  CRUSH rule 4 x 318 [8,6,7]
+  CRUSH rule 4 x 319 [5,0,8]
+  CRUSH rule 4 x 320 [3,7,1]
+  CRUSH rule 4 x 321 [1,3,0]
+  CRUSH rule 4 x 322 [2,7,3]
+  CRUSH rule 4 x 323 [4,7,0]
+  CRUSH rule 4 x 324 [7,0,8]
+  CRUSH rule 4 x 325 [4,6,0]
+  CRUSH rule 4 x 326 [3,4,1]
+  CRUSH rule 4 x 327 [0,6,7]
+  CRUSH rule 4 x 328 [7,4,8]
+  CRUSH rule 4 x 329 [5,6,3]
+  CRUSH rule 4 x 330 [3,7,4]
+  CRUSH rule 4 x 331 [2,6,3]
+  CRUSH rule 4 x 332 [2,0,3]
+  CRUSH rule 4 x 333 [6,8,5]
+  CRUSH rule 4 x 334 [8,3,5]
+  CRUSH rule 4 x 335 [7,1,2]
+  CRUSH rule 4 x 336 [4,6,0]
+  CRUSH rule 4 x 337 [7,2,6]
+  CRUSH rule 4 x 338 [5,6,4]
+  CRUSH rule 4 x 339 [7,5,2]
+  CRUSH rule 4 x 340 [2,0,1]
+  CRUSH rule 4 x 341 [5,1,7]
+  CRUSH rule 4 x 342 [0,7,4]
+  CRUSH rule 4 x 343 [6,7,3]
+  CRUSH rule 4 x 344 [6,0,5]
+  CRUSH rule 4 x 345 [4,3,5]
+  CRUSH rule 4 x 346 [8,0,5]
+  CRUSH rule 4 x 347 [3,1,5]
+  CRUSH rule 4 x 348 [8,0,2]
+  CRUSH rule 4 x 349 [1,6,7]
+  CRUSH rule 4 x 350 [8,5,7]
+  CRUSH rule 4 x 351 [3,6,4]
+  CRUSH rule 4 x 352 [1,0,2]
+  CRUSH rule 4 x 353 [6,4,5]
+  CRUSH rule 4 x 354 [0,3,1]
+  CRUSH rule 4 x 355 [3,4,6]
+  CRUSH rule 4 x 356 [3,5,4]
+  CRUSH rule 4 x 357 [6,1,2]
+  CRUSH rule 4 x 358 [2,1,8]
+  CRUSH rule 4 x 359 [6,7,8]
+  CRUSH rule 4 x 360 [5,3,1]
+  CRUSH rule 4 x 361 [8,4,5]
+  CRUSH rule 4 x 362 [4,5,3]
+  CRUSH rule 4 x 363 [4,0,3]
+  CRUSH rule 4 x 364 [2,5,0]
+  CRUSH rule 4 x 365 [6,7,8]
+  CRUSH rule 4 x 366 [7,2,8]
+  CRUSH rule 4 x 367 [4,5,0]
+  CRUSH rule 4 x 368 [7,4,6]
+  CRUSH rule 4 x 369 [3,7,1]
+  CRUSH rule 4 x 370 [8,7,6]
+  CRUSH rule 4 x 371 [1,3,5]
+  CRUSH rule 4 x 372 [3,1,8]
+  CRUSH rule 4 x 373 [0,6,8]
+  CRUSH rule 4 x 374 [3,8,4]
+  CRUSH rule 4 x 375 [6,4,5]
+  CRUSH rule 4 x 376 [7,1,2]
+  CRUSH rule 4 x 377 [1,2,4]
+  CRUSH rule 4 x 378 [0,1,2]
+  CRUSH rule 4 x 379 [8,5,4]
+  CRUSH rule 4 x 380 [2,5,0]
+  CRUSH rule 4 x 381 [0,4,7]
+  CRUSH rule 4 x 382 [1,5,2]
+  CRUSH rule 4 x 383 [4,3,5]
+  CRUSH rule 4 x 384 [7,0,6]
+  CRUSH rule 4 x 385 [7,4,6]
+  CRUSH rule 4 x 386 [0,3,5]
+  CRUSH rule 4 x 387 [1,3,6]
+  CRUSH rule 4 x 388 [5,0,3]
+  CRUSH rule 4 x 389 [1,5,4]
+  CRUSH rule 4 x 390 [5,6,0]
+  CRUSH rule 4 x 391 [5,6,2]
+  CRUSH rule 4 x 392 [1,8,6]
+  CRUSH rule 4 x 393 [4,2,6]
+  CRUSH rule 4 x 394 [4,7,6]
+  CRUSH rule 4 x 395 [4,0,1]
+  CRUSH rule 4 x 396 [4,2,5]
+  CRUSH rule 4 x 397 [2,1,5]
+  CRUSH rule 4 x 398 [2,4,5]
+  CRUSH rule 4 x 399 [8,7,3]
+  CRUSH rule 4 x 400 [8,1,4]
+  CRUSH rule 4 x 401 [0,1,2]
+  CRUSH rule 4 x 402 [7,8,6]
+  CRUSH rule 4 x 403 [0,1,4]
+  CRUSH rule 4 x 404 [4,3,2]
+  CRUSH rule 4 x 405 [6,5,2]
+  CRUSH rule 4 x 406 [2,0,1]
+  CRUSH rule 4 x 407 [2,8,7]
+  CRUSH rule 4 x 408 [4,1,6]
+  CRUSH rule 4 x 409 [7,3,6]
+  CRUSH rule 4 x 410 [8,6,3]
+  CRUSH rule 4 x 411 [2,0,6]
+  CRUSH rule 4 x 412 [0,5,8]
+  CRUSH rule 4 x 413 [5,0,8]
+  CRUSH rule 4 x 414 [4,1,3]
+  CRUSH rule 4 x 415 [0,6,1]
+  CRUSH rule 4 x 416 [2,1,7]
+  CRUSH rule 4 x 417 [8,7,2]
+  CRUSH rule 4 x 418 [7,6,8]
+  CRUSH rule 4 x 419 [8,3,0]
+  CRUSH rule 4 x 420 [1,4,2]
+  CRUSH rule 4 x 421 [8,6,7]
+  CRUSH rule 4 x 422 [6,7,8]
+  CRUSH rule 4 x 423 [0,5,6]
+  CRUSH rule 4 x 424 [8,4,7]
+  CRUSH rule 4 x 425 [1,3,5]
+  CRUSH rule 4 x 426 [6,7,2]
+  CRUSH rule 4 x 427 [0,7,5]
+  CRUSH rule 4 x 428 [5,4,3]
+  CRUSH rule 4 x 429 [4,6,7]
+  CRUSH rule 4 x 430 [3,6,5]
+  CRUSH rule 4 x 431 [5,3,0]
+  CRUSH rule 4 x 432 [7,1,2]
+  CRUSH rule 4 x 433 [6,5,1]
+  CRUSH rule 4 x 434 [5,2,1]
+  CRUSH rule 4 x 435 [0,5,3]
+  CRUSH rule 4 x 436 [4,0,5]
+  CRUSH rule 4 x 437 [7,5,3]
+  CRUSH rule 4 x 438 [0,3,8]
+  CRUSH rule 4 x 439 [1,3,4]
+  CRUSH rule 4 x 440 [2,7,1]
+  CRUSH rule 4 x 441 [5,7,2]
+  CRUSH rule 4 x 442 [2,4,5]
+  CRUSH rule 4 x 443 [6,8,0]
+  CRUSH rule 4 x 444 [7,0,1]
+  CRUSH rule 4 x 445 [6,3,5]
+  CRUSH rule 4 x 446 [4,3,5]
+  CRUSH rule 4 x 447 [2,1,4]
+  CRUSH rule 4 x 448 [7,2,5]
+  CRUSH rule 4 x 449 [7,8,3]
+  CRUSH rule 4 x 450 [4,5,2]
+  CRUSH rule 4 x 451 [6,8,3]
+  CRUSH rule 4 x 452 [8,3,5]
+  CRUSH rule 4 x 453 [6,8,7]
+  CRUSH rule 4 x 454 [6,7,5]
+  CRUSH rule 4 x 455 [2,7,5]
+  CRUSH rule 4 x 456 [6,8,7]
+  CRUSH rule 4 x 457 [7,2,8]
+  CRUSH rule 4 x 458 [2,8,1]
+  CRUSH rule 4 x 459 [2,0,6]
+  CRUSH rule 4 x 460 [6,5,7]
+  CRUSH rule 4 x 461 [6,5,4]
+  CRUSH rule 4 x 462 [8,1,5]
+  CRUSH rule 4 x 463 [6,7,2]
+  CRUSH rule 4 x 464 [7,4,2]
+  CRUSH rule 4 x 465 [7,6,0]
+  CRUSH rule 4 x 466 [5,8,4]
+  CRUSH rule 4 x 467 [6,4,7]
+  CRUSH rule 4 x 468 [7,8,0]
+  CRUSH rule 4 x 469 [7,0,1]
+  CRUSH rule 4 x 470 [3,0,5]
+  CRUSH rule 4 x 471 [0,1,2]
+  CRUSH rule 4 x 472 [5,4,0]
+  CRUSH rule 4 x 473 [1,0,4]
+  CRUSH rule 4 x 474 [6,0,7]
+  CRUSH rule 4 x 475 [6,7,8]
+  CRUSH rule 4 x 476 [4,3,5]
+  CRUSH rule 4 x 477 [5,8,6]
+  CRUSH rule 4 x 478 [6,7,2]
+  CRUSH rule 4 x 479 [0,5,8]
+  CRUSH rule 4 x 480 [1,8,6]
+  CRUSH rule 4 x 481 [2,4,5]
+  CRUSH rule 4 x 482 [4,3,5]
+  CRUSH rule 4 x 483 [0,2,6]
+  CRUSH rule 4 x 484 [1,2,7]
+  CRUSH rule 4 x 485 [4,7,3]
+  CRUSH rule 4 x 486 [4,1,7]
+  CRUSH rule 4 x 487 [5,0,1]
+  CRUSH rule 4 x 488 [5,7,1]
+  CRUSH rule 4 x 489 [2,8,5]
+  CRUSH rule 4 x 490 [6,4,1]
+  CRUSH rule 4 x 491 [1,0,6]
+  CRUSH rule 4 x 492 [6,5,4]
+  CRUSH rule 4 x 493 [0,2,7]
+  CRUSH rule 4 x 494 [1,0,8]
+  CRUSH rule 4 x 495 [3,4,5]
+  CRUSH rule 4 x 496 [7,5,4]
+  CRUSH rule 4 x 497 [5,7,3]
+  CRUSH rule 4 x 498 [0,5,8]
+  CRUSH rule 4 x 499 [8,4,3]
+  CRUSH rule 4 x 500 [3,6,5]
+  CRUSH rule 4 x 501 [0,7,2]
+  CRUSH rule 4 x 502 [7,1,4]
+  CRUSH rule 4 x 503 [2,3,5]
+  CRUSH rule 4 x 504 [5,6,3]
+  CRUSH rule 4 x 505 [0,7,6]
+  CRUSH rule 4 x 506 [5,3,0]
+  CRUSH rule 4 x 507 [6,0,8]
+  CRUSH rule 4 x 508 [0,1,2]
+  CRUSH rule 4 x 509 [7,5,8]
+  CRUSH rule 4 x 510 [6,0,2]
+  CRUSH rule 4 x 511 [5,8,7]
+  CRUSH rule 4 x 512 [7,6,2]
+  CRUSH rule 4 x 513 [7,2,1]
+  CRUSH rule 4 x 514 [4,3,5]
+  CRUSH rule 4 x 515 [8,5,6]
+  CRUSH rule 4 x 516 [4,0,1]
+  CRUSH rule 4 x 517 [7,8,6]
+  CRUSH rule 4 x 518 [4,6,7]
+  CRUSH rule 4 x 519 [7,3,5]
+  CRUSH rule 4 x 520 [2,6,3]
+  CRUSH rule 4 x 521 [8,7,6]
+  CRUSH rule 4 x 522 [6,8,0]
+  CRUSH rule 4 x 523 [4,2,5]
+  CRUSH rule 4 x 524 [0,4,5]
+  CRUSH rule 4 x 525 [0,4,1]
+  CRUSH rule 4 x 526 [1,5,8]
+  CRUSH rule 4 x 527 [0,2,4]
+  CRUSH rule 4 x 528 [5,3,0]
+  CRUSH rule 4 x 529 [5,7,0]
+  CRUSH rule 4 x 530 [6,7,8]
+  CRUSH rule 4 x 531 [6,1,0]
+  CRUSH rule 4 x 532 [6,3,7]
+  CRUSH rule 4 x 533 [5,6,3]
+  CRUSH rule 4 x 534 [7,3,5]
+  CRUSH rule 4 x 535 [8,6,0]
+  CRUSH rule 4 x 536 [6,7,0]
+  CRUSH rule 4 x 537 [3,7,5]
+  CRUSH rule 4 x 538 [6,8,3]
+  CRUSH rule 4 x 539 [8,3,7]
+  CRUSH rule 4 x 540 [0,6,1]
+  CRUSH rule 4 x 541 [2,3,1]
+  CRUSH rule 4 x 542 [3,5,4]
+  CRUSH rule 4 x 543 [6,0,8]
+  CRUSH rule 4 x 544 [3,7,6]
+  CRUSH rule 4 x 545 [5,7,0]
+  CRUSH rule 4 x 546 [6,1,7]
+  CRUSH rule 4 x 547 [8,2,0]
+  CRUSH rule 4 x 548 [5,2,0]
+  CRUSH rule 4 x 549 [5,8,2]
+  CRUSH rule 4 x 550 [0,5,4]
+  CRUSH rule 4 x 551 [7,5,4]
+  CRUSH rule 4 x 552 [5,4,3]
+  CRUSH rule 4 x 553 [4,2,3]
+  CRUSH rule 4 x 554 [0,8,5]
+  CRUSH rule 4 x 555 [5,0,8]
+  CRUSH rule 4 x 556 [3,4,5]
+  CRUSH rule 4 x 557 [7,4,6]
+  CRUSH rule 4 x 558 [3,1,0]
+  CRUSH rule 4 x 559 [4,2,6]
+  CRUSH rule 4 x 560 [8,3,5]
+  CRUSH rule 4 x 561 [6,3,7]
+  CRUSH rule 4 x 562 [3,4,1]
+  CRUSH rule 4 x 563 [2,6,0]
+  CRUSH rule 4 x 564 [5,1,7]
+  CRUSH rule 4 x 565 [3,6,5]
+  CRUSH rule 4 x 566 [4,7,2]
+  CRUSH rule 4 x 567 [3,6,1]
+  CRUSH rule 4 x 568 [7,4,1]
+  CRUSH rule 4 x 569 [3,4,2]
+  CRUSH rule 4 x 570 [1,5,4]
+  CRUSH rule 4 x 571 [3,7,5]
+  CRUSH rule 4 x 572 [3,4,0]
+  CRUSH rule 4 x 573 [3,0,7]
+  CRUSH rule 4 x 574 [2,0,1]
+  CRUSH rule 4 x 575 [8,6,0]
+  CRUSH rule 4 x 576 [4,6,8]
+  CRUSH rule 4 x 577 [8,2,6]
+  CRUSH rule 4 x 578 [6,8,7]
+  CRUSH rule 4 x 579 [3,1,6]
+  CRUSH rule 4 x 580 [3,0,7]
+  CRUSH rule 4 x 581 [7,2,6]
+  CRUSH rule 4 x 582 [2,8,7]
+  CRUSH rule 4 x 583 [6,0,8]
+  CRUSH rule 4 x 584 [8,1,2]
+  CRUSH rule 4 x 585 [7,0,5]
+  CRUSH rule 4 x 586 [0,1,7]
+  CRUSH rule 4 x 587 [2,5,0]
+  CRUSH rule 4 x 588 [3,4,7]
+  CRUSH rule 4 x 589 [7,1,4]
+  CRUSH rule 4 x 590 [6,2,3]
+  CRUSH rule 4 x 591 [5,2,4]
+  CRUSH rule 4 x 592 [2,0,3]
+  CRUSH rule 4 x 593 [0,8,1]
+  CRUSH rule 4 x 594 [0,7,4]
+  CRUSH rule 4 x 595 [7,1,8]
+  CRUSH rule 4 x 596 [4,3,5]
+  CRUSH rule 4 x 597 [3,1,2]
+  CRUSH rule 4 x 598 [3,2,4]
+  CRUSH rule 4 x 599 [5,2,1]
+  CRUSH rule 4 x 600 [7,0,8]
+  CRUSH rule 4 x 601 [0,7,8]
+  CRUSH rule 4 x 602 [3,7,5]
+  CRUSH rule 4 x 603 [5,1,6]
+  CRUSH rule 4 x 604 [7,5,4]
+  CRUSH rule 4 x 605 [3,0,5]
+  CRUSH rule 4 x 606 [2,0,1]
+  CRUSH rule 4 x 607 [0,4,8]
+  CRUSH rule 4 x 608 [5,3,2]
+  CRUSH rule 4 x 609 [5,2,4]
+  CRUSH rule 4 x 610 [3,7,6]
+  CRUSH rule 4 x 611 [1,0,2]
+  CRUSH rule 4 x 612 [2,0,8]
+  CRUSH rule 4 x 613 [7,2,0]
+  CRUSH rule 4 x 614 [7,8,6]
+  CRUSH rule 4 x 615 [6,8,2]
+  CRUSH rule 4 x 616 [0,8,1]
+  CRUSH rule 4 x 617 [6,1,2]
+  CRUSH rule 4 x 618 [7,6,5]
+  CRUSH rule 4 x 619 [5,1,8]
+  CRUSH rule 4 x 620 [4,1,0]
+  CRUSH rule 4 x 621 [5,8,3]
+  CRUSH rule 4 x 622 [0,4,3]
+  CRUSH rule 4 x 623 [0,6,2]
+  CRUSH rule 4 x 624 [3,5,1]
+  CRUSH rule 4 x 625 [2,3,7]
+  CRUSH rule 4 x 626 [7,8,0]
+  CRUSH rule 4 x 627 [2,7,1]
+  CRUSH rule 4 x 628 [8,0,5]
+  CRUSH rule 4 x 629 [2,6,5]
+  CRUSH rule 4 x 630 [2,7,1]
+  CRUSH rule 4 x 631 [0,7,8]
+  CRUSH rule 4 x 632 [7,0,2]
+  CRUSH rule 4 x 633 [8,6,5]
+  CRUSH rule 4 x 634 [0,4,2]
+  CRUSH rule 4 x 635 [5,6,4]
+  CRUSH rule 4 x 636 [1,4,2]
+  CRUSH rule 4 x 637 [4,1,2]
+  CRUSH rule 4 x 638 [6,8,1]
+  CRUSH rule 4 x 639 [4,2,1]
+  CRUSH rule 4 x 640 [3,1,0]
+  CRUSH rule 4 x 641 [7,2,0]
+  CRUSH rule 4 x 642 [2,0,1]
+  CRUSH rule 4 x 643 [3,5,1]
+  CRUSH rule 4 x 644 [8,1,5]
+  CRUSH rule 4 x 645 [5,4,6]
+  CRUSH rule 4 x 646 [8,0,3]
+  CRUSH rule 4 x 647 [7,1,3]
+  CRUSH rule 4 x 648 [0,6,3]
+  CRUSH rule 4 x 649 [4,7,5]
+  CRUSH rule 4 x 650 [7,8,6]
+  CRUSH rule 4 x 651 [3,7,6]
+  CRUSH rule 4 x 652 [3,4,8]
+  CRUSH rule 4 x 653 [8,5,2]
+  CRUSH rule 4 x 654 [7,5,2]
+  CRUSH rule 4 x 655 [0,3,4]
+  CRUSH rule 4 x 656 [4,3,8]
+  CRUSH rule 4 x 657 [6,1,2]
+  CRUSH rule 4 x 658 [5,4,6]
+  CRUSH rule 4 x 659 [4,6,7]
+  CRUSH rule 4 x 660 [7,8,6]
+  CRUSH rule 4 x 661 [1,8,3]
+  CRUSH rule 4 x 662 [4,5,3]
+  CRUSH rule 4 x 663 [1,3,8]
+  CRUSH rule 4 x 664 [1,4,0]
+  CRUSH rule 4 x 665 [5,7,6]
+  CRUSH rule 4 x 666 [2,8,4]
+  CRUSH rule 4 x 667 [1,3,5]
+  CRUSH rule 4 x 668 [3,7,6]
+  CRUSH rule 4 x 669 [6,4,5]
+  CRUSH rule 4 x 670 [4,0,3]
+  CRUSH rule 4 x 671 [0,2,1]
+  CRUSH rule 4 x 672 [4,3,2]
+  CRUSH rule 4 x 673 [5,3,0]
+  CRUSH rule 4 x 674 [3,1,8]
+  CRUSH rule 4 x 675 [0,8,6]
+  CRUSH rule 4 x 676 [0,2,4]
+  CRUSH rule 4 x 677 [4,1,3]
+  CRUSH rule 4 x 678 [2,3,1]
+  CRUSH rule 4 x 679 [6,0,2]
+  CRUSH rule 4 x 680 [0,4,6]
+  CRUSH rule 4 x 681 [4,7,3]
+  CRUSH rule 4 x 682 [0,5,2]
+  CRUSH rule 4 x 683 [0,1,5]
+  CRUSH rule 4 x 684 [7,1,6]
+  CRUSH rule 4 x 685 [7,1,6]
+  CRUSH rule 4 x 686 [1,4,3]
+  CRUSH rule 4 x 687 [3,5,7]
+  CRUSH rule 4 x 688 [5,7,2]
+  CRUSH rule 4 x 689 [6,5,0]
+  CRUSH rule 4 x 690 [8,1,7]
+  CRUSH rule 4 x 691 [3,0,5]
+  CRUSH rule 4 x 692 [7,2,8]
+  CRUSH rule 4 x 693 [6,7,3]
+  CRUSH rule 4 x 694 [6,5,1]
+  CRUSH rule 4 x 695 [0,8,7]
+  CRUSH rule 4 x 696 [1,4,3]
+  CRUSH rule 4 x 697 [6,1,2]
+  CRUSH rule 4 x 698 [6,2,0]
+  CRUSH rule 4 x 699 [1,6,8]
+  CRUSH rule 4 x 700 [0,3,1]
+  CRUSH rule 4 x 701 [4,3,2]
+  CRUSH rule 4 x 702 [3,5,0]
+  CRUSH rule 4 x 703 [8,3,4]
+  CRUSH rule 4 x 704 [0,3,8]
+  CRUSH rule 4 x 705 [8,6,0]
+  CRUSH rule 4 x 706 [1,2,4]
+  CRUSH rule 4 x 707 [7,8,6]
+  CRUSH rule 4 x 708 [3,5,8]
+  CRUSH rule 4 x 709 [6,3,0]
+  CRUSH rule 4 x 710 [8,4,3]
+  CRUSH rule 4 x 711 [2,3,8]
+  CRUSH rule 4 x 712 [2,3,7]
+  CRUSH rule 4 x 713 [6,7,8]
+  CRUSH rule 4 x 714 [3,2,0]
+  CRUSH rule 4 x 715 [1,2,4]
+  CRUSH rule 4 x 716 [3,6,0]
+  CRUSH rule 4 x 717 [8,7,2]
+  CRUSH rule 4 x 718 [3,7,8]
+  CRUSH rule 4 x 719 [2,6,3]
+  CRUSH rule 4 x 720 [6,8,2]
+  CRUSH rule 4 x 721 [5,4,7]
+  CRUSH rule 4 x 722 [5,4,6]
+  CRUSH rule 4 x 723 [5,1,0]
+  CRUSH rule 4 x 724 [0,6,2]
+  CRUSH rule 4 x 725 [0,1,5]
+  CRUSH rule 4 x 726 [3,8,4]
+  CRUSH rule 4 x 727 [4,6,8]
+  CRUSH rule 4 x 728 [2,1,0]
+  CRUSH rule 4 x 729 [5,3,7]
+  CRUSH rule 4 x 730 [3,7,5]
+  CRUSH rule 4 x 731 [4,1,5]
+  CRUSH rule 4 x 732 [1,5,3]
+  CRUSH rule 4 x 733 [5,4,7]
+  CRUSH rule 4 x 734 [6,4,2]
+  CRUSH rule 4 x 735 [4,8,3]
+  CRUSH rule 4 x 736 [3,5,6]
+  CRUSH rule 4 x 737 [1,0,8]
+  CRUSH rule 4 x 738 [5,2,3]
+  CRUSH rule 4 x 739 [0,1,2]
+  CRUSH rule 4 x 740 [0,1,7]
+  CRUSH rule 4 x 741 [7,8,0]
+  CRUSH rule 4 x 742 [8,2,1]
+  CRUSH rule 4 x 743 [7,0,1]
+  CRUSH rule 4 x 744 [4,7,3]
+  CRUSH rule 4 x 745 [3,4,1]
+  CRUSH rule 4 x 746 [4,1,3]
+  CRUSH rule 4 x 747 [6,0,3]
+  CRUSH rule 4 x 748 [2,7,0]
+  CRUSH rule 4 x 749 [4,5,8]
+  CRUSH rule 4 x 750 [1,6,3]
+  CRUSH rule 4 x 751 [2,1,6]
+  CRUSH rule 4 x 752 [8,1,5]
+  CRUSH rule 4 x 753 [7,8,3]
+  CRUSH rule 4 x 754 [8,6,7]
+  CRUSH rule 4 x 755 [1,2,0]
+  CRUSH rule 4 x 756 [5,6,1]
+  CRUSH rule 4 x 757 [8,6,1]
+  CRUSH rule 4 x 758 [6,0,3]
+  CRUSH rule 4 x 759 [8,5,3]
+  CRUSH rule 4 x 760 [1,5,4]
+  CRUSH rule 4 x 761 [4,1,2]
+  CRUSH rule 4 x 762 [2,7,8]
+  CRUSH rule 4 x 763 [8,6,7]
+  CRUSH rule 4 x 764 [1,7,0]
+  CRUSH rule 4 x 765 [6,5,2]
+  CRUSH rule 4 x 766 [8,5,7]
+  CRUSH rule 4 x 767 [1,2,0]
+  CRUSH rule 4 x 768 [8,3,2]
+  CRUSH rule 4 x 769 [6,2,8]
+  CRUSH rule 4 x 770 [6,0,7]
+  CRUSH rule 4 x 771 [7,0,3]
+  CRUSH rule 4 x 772 [8,3,7]
+  CRUSH rule 4 x 773 [3,1,5]
+  CRUSH rule 4 x 774 [4,6,5]
+  CRUSH rule 4 x 775 [6,8,4]
+  CRUSH rule 4 x 776 [7,2,1]
+  CRUSH rule 4 x 777 [3,1,6]
+  CRUSH rule 4 x 778 [1,8,0]
+  CRUSH rule 4 x 779 [2,7,3]
+  CRUSH rule 4 x 780 [0,2,3]
+  CRUSH rule 4 x 781 [6,3,7]
+  CRUSH rule 4 x 782 [5,4,0]
+  CRUSH rule 4 x 783 [7,1,8]
+  CRUSH rule 4 x 784 [0,1,5]
+  CRUSH rule 4 x 785 [6,1,2]
+  CRUSH rule 4 x 786 [7,6,5]
+  CRUSH rule 4 x 787 [1,0,6]
+  CRUSH rule 4 x 788 [6,0,8]
+  CRUSH rule 4 x 789 [0,4,3]
+  CRUSH rule 4 x 790 [8,4,7]
+  CRUSH rule 4 x 791 [3,8,7]
+  CRUSH rule 4 x 792 [5,8,4]
+  CRUSH rule 4 x 793 [6,1,3]
+  CRUSH rule 4 x 794 [2,6,8]
+  CRUSH rule 4 x 795 [0,3,2]
+  CRUSH rule 4 x 796 [3,4,5]
+  CRUSH rule 4 x 797 [2,3,4]
+  CRUSH rule 4 x 798 [6,8,7]
+  CRUSH rule 4 x 799 [5,1,4]
+  CRUSH rule 4 x 800 [5,0,3]
+  CRUSH rule 4 x 801 [3,6,8]
+  CRUSH rule 4 x 802 [1,8,0]
+  CRUSH rule 4 x 803 [0,5,7]
+  CRUSH rule 4 x 804 [6,2,5]
+  CRUSH rule 4 x 805 [3,6,7]
+  CRUSH rule 4 x 806 [1,3,4]
+  CRUSH rule 4 x 807 [5,4,7]
+  CRUSH rule 4 x 808 [4,6,2]
+  CRUSH rule 4 x 809 [1,4,5]
+  CRUSH rule 4 x 810 [5,7,3]
+  CRUSH rule 4 x 811 [8,4,5]
+  CRUSH rule 4 x 812 [8,5,4]
+  CRUSH rule 4 x 813 [6,4,8]
+  CRUSH rule 4 x 814 [3,6,8]
+  CRUSH rule 4 x 815 [3,1,2]
+  CRUSH rule 4 x 816 [2,1,8]
+  CRUSH rule 4 x 817 [4,3,7]
+  CRUSH rule 4 x 818 [3,5,0]
+  CRUSH rule 4 x 819 [5,1,4]
+  CRUSH rule 4 x 820 [3,5,4]
+  CRUSH rule 4 x 821 [4,5,8]
+  CRUSH rule 4 x 822 [2,0,5]
+  CRUSH rule 4 x 823 [4,8,2]
+  CRUSH rule 4 x 824 [3,7,4]
+  CRUSH rule 4 x 825 [2,8,7]
+  CRUSH rule 4 x 826 [7,0,5]
+  CRUSH rule 4 x 827 [0,8,2]
+  CRUSH rule 4 x 828 [2,3,1]
+  CRUSH rule 4 x 829 [5,6,7]
+  CRUSH rule 4 x 830 [2,3,5]
+  CRUSH rule 4 x 831 [1,6,0]
+  CRUSH rule 4 x 832 [4,5,3]
+  CRUSH rule 4 x 833 [2,1,7]
+  CRUSH rule 4 x 834 [3,4,2]
+  CRUSH rule 4 x 835 [8,4,3]
+  CRUSH rule 4 x 836 [3,4,5]
+  CRUSH rule 4 x 837 [6,3,1]
+  CRUSH rule 4 x 838 [6,7,2]
+  CRUSH rule 4 x 839 [5,0,6]
+  CRUSH rule 4 x 840 [7,8,5]
+  CRUSH rule 4 x 841 [4,8,5]
+  CRUSH rule 4 x 842 [2,4,0]
+  CRUSH rule 4 x 843 [6,4,7]
+  CRUSH rule 4 x 844 [4,8,1]
+  CRUSH rule 4 x 845 [3,8,6]
+  CRUSH rule 4 x 846 [3,2,0]
+  CRUSH rule 4 x 847 [0,2,7]
+  CRUSH rule 4 x 848 [2,6,1]
+  CRUSH rule 4 x 849 [4,5,3]
+  CRUSH rule 4 x 850 [1,0,5]
+  CRUSH rule 4 x 851 [6,8,7]
+  CRUSH rule 4 x 852 [7,3,8]
+  CRUSH rule 4 x 853 [6,8,1]
+  CRUSH rule 4 x 854 [7,6,0]
+  CRUSH rule 4 x 855 [5,7,2]
+  CRUSH rule 4 x 856 [6,7,3]
+  CRUSH rule 4 x 857 [8,5,0]
+  CRUSH rule 4 x 858 [6,4,1]
+  CRUSH rule 4 x 859 [6,0,7]
+  CRUSH rule 4 x 860 [4,1,2]
+  CRUSH rule 4 x 861 [8,7,6]
+  CRUSH rule 4 x 862 [6,1,7]
+  CRUSH rule 4 x 863 [8,7,0]
+  CRUSH rule 4 x 864 [5,6,8]
+  CRUSH rule 4 x 865 [8,1,0]
+  CRUSH rule 4 x 866 [3,4,8]
+  CRUSH rule 4 x 867 [6,5,1]
+  CRUSH rule 4 x 868 [6,3,0]
+  CRUSH rule 4 x 869 [8,7,3]
+  CRUSH rule 4 x 870 [0,4,8]
+  CRUSH rule 4 x 871 [3,4,5]
+  CRUSH rule 4 x 872 [5,1,3]
+  CRUSH rule 4 x 873 [4,6,5]
+  CRUSH rule 4 x 874 [2,6,1]
+  CRUSH rule 4 x 875 [2,6,4]
+  CRUSH rule 4 x 876 [5,8,1]
+  CRUSH rule 4 x 877 [6,4,2]
+  CRUSH rule 4 x 878 [5,4,0]
+  CRUSH rule 4 x 879 [7,4,8]
+  CRUSH rule 4 x 880 [3,5,0]
+  CRUSH rule 4 x 881 [5,6,1]
+  CRUSH rule 4 x 882 [4,0,2]
+  CRUSH rule 4 x 883 [2,1,0]
+  CRUSH rule 4 x 884 [6,0,4]
+  CRUSH rule 4 x 885 [5,1,4]
+  CRUSH rule 4 x 886 [3,6,4]
+  CRUSH rule 4 x 887 [7,4,0]
+  CRUSH rule 4 x 888 [6,8,0]
+  CRUSH rule 4 x 889 [2,1,7]
+  CRUSH rule 4 x 890 [7,2,0]
+  CRUSH rule 4 x 891 [1,8,0]
+  CRUSH rule 4 x 892 [6,2,3]
+  CRUSH rule 4 x 893 [2,3,7]
+  CRUSH rule 4 x 894 [7,5,0]
+  CRUSH rule 4 x 895 [5,3,2]
+  CRUSH rule 4 x 896 [1,8,2]
+  CRUSH rule 4 x 897 [4,2,6]
+  CRUSH rule 4 x 898 [0,5,4]
+  CRUSH rule 4 x 899 [1,7,6]
+  CRUSH rule 4 x 900 [4,1,0]
+  CRUSH rule 4 x 901 [5,0,1]
+  CRUSH rule 4 x 902 [8,5,7]
+  CRUSH rule 4 x 903 [5,7,3]
+  CRUSH rule 4 x 904 [5,6,8]
+  CRUSH rule 4 x 905 [6,2,5]
+  CRUSH rule 4 x 906 [1,2,0]
+  CRUSH rule 4 x 907 [7,1,0]
+  CRUSH rule 4 x 908 [5,8,1]
+  CRUSH rule 4 x 909 [2,3,4]
+  CRUSH rule 4 x 910 [6,4,0]
+  CRUSH rule 4 x 911 [5,8,4]
+  CRUSH rule 4 x 912 [0,1,7]
+  CRUSH rule 4 x 913 [7,6,8]
+  CRUSH rule 4 x 914 [6,4,7]
+  CRUSH rule 4 x 915 [8,2,6]
+  CRUSH rule 4 x 916 [3,1,4]
+  CRUSH rule 4 x 917 [1,5,3]
+  CRUSH rule 4 x 918 [8,2,1]
+  CRUSH rule 4 x 919 [6,2,8]
+  CRUSH rule 4 x 920 [7,6,4]
+  CRUSH rule 4 x 921 [1,4,5]
+  CRUSH rule 4 x 922 [6,7,8]
+  CRUSH rule 4 x 923 [5,3,6]
+  CRUSH rule 4 x 924 [3,5,4]
+  CRUSH rule 4 x 925 [5,7,3]
+  CRUSH rule 4 x 926 [3,4,5]
+  CRUSH rule 4 x 927 [1,6,3]
+  CRUSH rule 4 x 928 [8,1,2]
+  CRUSH rule 4 x 929 [4,5,1]
+  CRUSH rule 4 x 930 [2,4,6]
+  CRUSH rule 4 x 931 [5,0,1]
+  CRUSH rule 4 x 932 [4,3,0]
+  CRUSH rule 4 x 933 [8,5,4]
+  CRUSH rule 4 x 934 [5,3,8]
+  CRUSH rule 4 x 935 [6,3,4]
+  CRUSH rule 4 x 936 [0,6,7]
+  CRUSH rule 4 x 937 [5,4,3]
+  CRUSH rule 4 x 938 [6,5,8]
+  CRUSH rule 4 x 939 [2,7,0]
+  CRUSH rule 4 x 940 [8,7,6]
+  CRUSH rule 4 x 941 [5,2,0]
+  CRUSH rule 4 x 942 [1,0,2]
+  CRUSH rule 4 x 943 [8,2,4]
+  CRUSH rule 4 x 944 [4,3,7]
+  CRUSH rule 4 x 945 [7,2,4]
+  CRUSH rule 4 x 946 [2,0,7]
+  CRUSH rule 4 x 947 [4,5,3]
+  CRUSH rule 4 x 948 [7,8,6]
+  CRUSH rule 4 x 949 [6,1,7]
+  CRUSH rule 4 x 950 [3,5,8]
+  CRUSH rule 4 x 951 [4,5,3]
+  CRUSH rule 4 x 952 [2,0,7]
+  CRUSH rule 4 x 953 [1,3,5]
+  CRUSH rule 4 x 954 [4,2,5]
+  CRUSH rule 4 x 955 [8,6,0]
+  CRUSH rule 4 x 956 [1,0,8]
+  CRUSH rule 4 x 957 [7,6,1]
+  CRUSH rule 4 x 958 [8,7,5]
+  CRUSH rule 4 x 959 [5,2,7]
+  CRUSH rule 4 x 960 [3,6,5]
+  CRUSH rule 4 x 961 [4,0,2]
+  CRUSH rule 4 x 962 [7,4,3]
+  CRUSH rule 4 x 963 [0,5,2]
+  CRUSH rule 4 x 964 [3,1,4]
+  CRUSH rule 4 x 965 [7,6,5]
+  CRUSH rule 4 x 966 [3,8,4]
+  CRUSH rule 4 x 967 [8,6,5]
+  CRUSH rule 4 x 968 [7,2,4]
+  CRUSH rule 4 x 969 [8,0,6]
+  CRUSH rule 4 x 970 [0,6,3]
+  CRUSH rule 4 x 971 [1,7,8]
+  CRUSH rule 4 x 972 [1,8,4]
+  CRUSH rule 4 x 973 [1,2,0]
+  CRUSH rule 4 x 974 [5,3,2]
+  CRUSH rule 4 x 975 [3,7,4]
+  CRUSH rule 4 x 976 [4,3,5]
+  CRUSH rule 4 x 977 [8,3,2]
+  CRUSH rule 4 x 978 [7,2,8]
+  CRUSH rule 4 x 979 [7,6,0]
+  CRUSH rule 4 x 980 [6,0,7]
+  CRUSH rule 4 x 981 [7,3,2]
+  CRUSH rule 4 x 982 [4,2,0]
+  CRUSH rule 4 x 983 [3,5,6]
+  CRUSH rule 4 x 984 [0,2,1]
+  CRUSH rule 4 x 985 [2,5,4]
+  CRUSH rule 4 x 986 [8,7,3]
+  CRUSH rule 4 x 987 [0,5,1]
+  CRUSH rule 4 x 988 [1,3,5]
+  CRUSH rule 4 x 989 [0,6,3]
+  CRUSH rule 4 x 990 [1,0,8]
+  CRUSH rule 4 x 991 [0,4,1]
+  CRUSH rule 4 x 992 [7,1,5]
+  CRUSH rule 4 x 993 [0,6,2]
+  CRUSH rule 4 x 994 [3,4,5]
+  CRUSH rule 4 x 995 [7,6,2]
+  CRUSH rule 4 x 996 [6,7,5]
+  CRUSH rule 4 x 997 [6,4,1]
+  CRUSH rule 4 x 998 [8,1,2]
+  CRUSH rule 4 x 999 [0,7,8]
+  CRUSH rule 4 x 1000 [8,5,0]
+  CRUSH rule 4 x 1001 [2,0,4]
+  CRUSH rule 4 x 1002 [1,3,2]
+  CRUSH rule 4 x 1003 [2,8,7]
+  CRUSH rule 4 x 1004 [6,1,2]
+  CRUSH rule 4 x 1005 [6,1,2]
+  CRUSH rule 4 x 1006 [1,0,2]
+  CRUSH rule 4 x 1007 [1,2,4]
+  CRUSH rule 4 x 1008 [1,7,0]
+  CRUSH rule 4 x 1009 [6,8,5]
+  CRUSH rule 4 x 1010 [3,4,0]
+  CRUSH rule 4 x 1011 [3,0,4]
+  CRUSH rule 4 x 1012 [3,0,7]
+  CRUSH rule 4 x 1013 [5,1,0]
+  CRUSH rule 4 x 1014 [2,8,4]
+  CRUSH rule 4 x 1015 [6,8,4]
+  CRUSH rule 4 x 1016 [2,0,1]
+  CRUSH rule 4 x 1017 [6,0,2]
+  CRUSH rule 4 x 1018 [5,4,3]
+  CRUSH rule 4 x 1019 [5,3,8]
+  CRUSH rule 4 x 1020 [5,1,3]
+  CRUSH rule 4 x 1021 [5,2,1]
+  CRUSH rule 4 x 1022 [1,6,7]
+  CRUSH rule 4 x 1023 [3,2,0]
+  rule 4 (choose-set-two) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 5 (chooseleaf-set), x = 0..1023, numrep = 2..3
+  CRUSH rule 5 x 0 [0,5]
+  CRUSH rule 5 x 1 [0,8]
+  CRUSH rule 5 x 2 [1,3]
+  CRUSH rule 5 x 3 [8,0]
+  CRUSH rule 5 x 4 [5,0]
+  CRUSH rule 5 x 5 [7,0]
+  CRUSH rule 5 x 6 [2,6]
+  CRUSH rule 5 x 7 [5,8]
+  CRUSH rule 5 x 8 [5,6]
+  CRUSH rule 5 x 9 [2,3]
+  CRUSH rule 5 x 10 [0,7]
+  CRUSH rule 5 x 11 [0,7]
+  CRUSH rule 5 x 12 [0,4]
+  CRUSH rule 5 x 13 [3,8]
+  CRUSH rule 5 x 14 [7,0]
+  CRUSH rule 5 x 15 [7,2]
+  CRUSH rule 5 x 16 [3,6]
+  CRUSH rule 5 x 17 [5,1]
+  CRUSH rule 5 x 18 [1,4]
+  CRUSH rule 5 x 19 [7,5]
+  CRUSH rule 5 x 20 [2,4]
+  CRUSH rule 5 x 21 [3,7]
+  CRUSH rule 5 x 22 [8,3]
+  CRUSH rule 5 x 23 [3,6]
+  CRUSH rule 5 x 24 [1,7]
+  CRUSH rule 5 x 25 [3,7]
+  CRUSH rule 5 x 26 [2,8]
+  CRUSH rule 5 x 27 [3,1]
+  CRUSH rule 5 x 28 [6,0]
+  CRUSH rule 5 x 29 [8,5]
+  CRUSH rule 5 x 30 [5,7]
+  CRUSH rule 5 x 31 [8,0]
+  CRUSH rule 5 x 32 [3,6]
+  CRUSH rule 5 x 33 [2,7]
+  CRUSH rule 5 x 34 [2,5]
+  CRUSH rule 5 x 35 [0,8]
+  CRUSH rule 5 x 36 [3,8]
+  CRUSH rule 5 x 37 [0,4]
+  CRUSH rule 5 x 38 [4,8]
+  CRUSH rule 5 x 39 [3,7]
+  CRUSH rule 5 x 40 [7,2]
+  CRUSH rule 5 x 41 [0,6]
+  CRUSH rule 5 x 42 [4,6]
+  CRUSH rule 5 x 43 [0,3]
+  CRUSH rule 5 x 44 [1,6]
+  CRUSH rule 5 x 45 [8,0]
+  CRUSH rule 5 x 46 [2,4]
+  CRUSH rule 5 x 47 [4,2]
+  CRUSH rule 5 x 48 [4,6]
+  CRUSH rule 5 x 49 [5,7]
+  CRUSH rule 5 x 50 [3,1]
+  CRUSH rule 5 x 51 [3,6]
+  CRUSH rule 5 x 52 [8,1]
+  CRUSH rule 5 x 53 [3,8]
+  CRUSH rule 5 x 54 [7,3]
+  CRUSH rule 5 x 55 [8,2]
+  CRUSH rule 5 x 56 [6,4]
+  CRUSH rule 5 x 57 [5,8]
+  CRUSH rule 5 x 58 [1,8]
+  CRUSH rule 5 x 59 [4,2]
+  CRUSH rule 5 x 60 [3,2]
+  CRUSH rule 5 x 61 [4,6]
+  CRUSH rule 5 x 62 [7,0]
+  CRUSH rule 5 x 63 [5,6]
+  CRUSH rule 5 x 64 [4,2]
+  CRUSH rule 5 x 65 [7,3]
+  CRUSH rule 5 x 66 [5,6]
+  CRUSH rule 5 x 67 [5,0]
+  CRUSH rule 5 x 68 [0,5]
+  CRUSH rule 5 x 69 [5,1]
+  CRUSH rule 5 x 70 [7,0]
+  CRUSH rule 5 x 71 [2,8]
+  CRUSH rule 5 x 72 [6,1]
+  CRUSH rule 5 x 73 [2,7]
+  CRUSH rule 5 x 74 [0,7]
+  CRUSH rule 5 x 75 [3,2]
+  CRUSH rule 5 x 76 [5,1]
+  CRUSH rule 5 x 77 [7,2]
+  CRUSH rule 5 x 78 [1,4]
+  CRUSH rule 5 x 79 [5,1]
+  CRUSH rule 5 x 80 [0,3]
+  CRUSH rule 5 x 81 [0,3]
+  CRUSH rule 5 x 82 [7,1]
+  CRUSH rule 5 x 83 [2,6]
+  CRUSH rule 5 x 84 [7,2]
+  CRUSH rule 5 x 85 [3,8]
+  CRUSH rule 5 x 86 [0,6]
+  CRUSH rule 5 x 87 [0,7]
+  CRUSH rule 5 x 88 [1,6]
+  CRUSH rule 5 x 89 [3,0]
+  CRUSH rule 5 x 90 [6,4]
+  CRUSH rule 5 x 91 [3,8]
+  CRUSH rule 5 x 92 [1,8]
+  CRUSH rule 5 x 93 [7,4]
+  CRUSH rule 5 x 94 [0,4]
+  CRUSH rule 5 x 95 [7,5]
+  CRUSH rule 5 x 96 [3,6]
+  CRUSH rule 5 x 97 [8,4]
+  CRUSH rule 5 x 98 [2,7]
+  CRUSH rule 5 x 99 [0,7]
+  CRUSH rule 5 x 100 [1,7]
+  CRUSH rule 5 x 101 [3,7]
+  CRUSH rule 5 x 102 [4,2]
+  CRUSH rule 5 x 103 [4,7]
+  CRUSH rule 5 x 104 [7,4]
+  CRUSH rule 5 x 105 [2,4]
+  CRUSH rule 5 x 106 [1,6]
+  CRUSH rule 5 x 107 [3,2]
+  CRUSH rule 5 x 108 [7,2]
+  CRUSH rule 5 x 109 [1,4]
+  CRUSH rule 5 x 110 [3,2]
+  CRUSH rule 5 x 111 [2,3]
+  CRUSH rule 5 x 112 [2,6]
+  CRUSH rule 5 x 113 [6,2]
+  CRUSH rule 5 x 114 [7,3]
+  CRUSH rule 5 x 115 [8,2]
+  CRUSH rule 5 x 116 [1,6]
+  CRUSH rule 5 x 117 [7,3]
+  CRUSH rule 5 x 118 [0,3]
+  CRUSH rule 5 x 119 [5,6]
+  CRUSH rule 5 x 120 [0,3]
+  CRUSH rule 5 x 121 [2,7]
+  CRUSH rule 5 x 122 [8,5]
+  CRUSH rule 5 x 123 [2,5]
+  CRUSH rule 5 x 124 [3,2]
+  CRUSH rule 5 x 125 [0,7]
+  CRUSH rule 5 x 126 [4,2]
+  CRUSH rule 5 x 127 [3,6]
+  CRUSH rule 5 x 128 [3,6]
+  CRUSH rule 5 x 129 [0,3]
+  CRUSH rule 5 x 130 [3,8]
+  CRUSH rule 5 x 131 [1,3]
+  CRUSH rule 5 x 132 [1,4]
+  CRUSH rule 5 x 133 [3,6]
+  CRUSH rule 5 x 134 [1,8]
+  CRUSH rule 5 x 135 [5,6]
+  CRUSH rule 5 x 136 [2,3]
+  CRUSH rule 5 x 137 [7,3]
+  CRUSH rule 5 x 138 [8,4]
+  CRUSH rule 5 x 139 [3,0]
+  CRUSH rule 5 x 140 [1,6]
+  CRUSH rule 5 x 141 [6,2]
+  CRUSH rule 5 x 142 [3,0]
+  CRUSH rule 5 x 143 [5,8]
+  CRUSH rule 5 x 144 [8,1]
+  CRUSH rule 5 x 145 [8,5]
+  CRUSH rule 5 x 146 [2,6]
+  CRUSH rule 5 x 147 [2,8]
+  CRUSH rule 5 x 148 [3,1]
+  CRUSH rule 5 x 149 [4,8]
+  CRUSH rule 5 x 150 [1,6]
+  CRUSH rule 5 x 151 [3,6]
+  CRUSH rule 5 x 152 [8,4]
+  CRUSH rule 5 x 153 [8,4]
+  CRUSH rule 5 x 154 [3,2]
+  CRUSH rule 5 x 155 [3,7]
+  CRUSH rule 5 x 156 [4,2]
+  CRUSH rule 5 x 157 [4,1]
+  CRUSH rule 5 x 158 [2,8]
+  CRUSH rule 5 x 159 [7,0]
+  CRUSH rule 5 x 160 [2,8]
+  CRUSH rule 5 x 161 [1,5]
+  CRUSH rule 5 x 162 [0,6]
+  CRUSH rule 5 x 163 [5,6]
+  CRUSH rule 5 x 164 [7,1]
+  CRUSH rule 5 x 165 [7,0]
+  CRUSH rule 5 x 166 [2,4]
+  CRUSH rule 5 x 167 [0,7]
+  CRUSH rule 5 x 168 [4,2]
+  CRUSH rule 5 x 169 [2,6]
+  CRUSH rule 5 x 170 [1,4]
+  CRUSH rule 5 x 171 [7,5]
+  CRUSH rule 5 x 172 [0,7]
+  CRUSH rule 5 x 173 [8,5]
+  CRUSH rule 5 x 174 [1,4]
+  CRUSH rule 5 x 175 [6,0]
+  CRUSH rule 5 x 176 [4,1]
+  CRUSH rule 5 x 177 [5,1]
+  CRUSH rule 5 x 178 [3,0]
+  CRUSH rule 5 x 179 [4,1]
+  CRUSH rule 5 x 180 [3,8]
+  CRUSH rule 5 x 181 [6,2]
+  CRUSH rule 5 x 182 [8,5]
+  CRUSH rule 5 x 183 [7,5]
+  CRUSH rule 5 x 184 [5,7]
+  CRUSH rule 5 x 185 [6,1]
+  CRUSH rule 5 x 186 [2,5]
+  CRUSH rule 5 x 187 [1,6]
+  CRUSH rule 5 x 188 [1,8]
+  CRUSH rule 5 x 189 [0,7]
+  CRUSH rule 5 x 190 [4,0]
+  CRUSH rule 5 x 191 [7,1]
+  CRUSH rule 5 x 192 [5,0]
+  CRUSH rule 5 x 193 [4,2]
+  CRUSH rule 5 x 194 [1,3]
+  CRUSH rule 5 x 195 [6,4]
+  CRUSH rule 5 x 196 [6,0]
+  CRUSH rule 5 x 197 [6,5]
+  CRUSH rule 5 x 198 [2,5]
+  CRUSH rule 5 x 199 [0,5]
+  CRUSH rule 5 x 200 [0,5]
+  CRUSH rule 5 x 201 [7,1]
+  CRUSH rule 5 x 202 [6,3]
+  CRUSH rule 5 x 203 [4,8]
+  CRUSH rule 5 x 204 [2,3]
+  CRUSH rule 5 x 205 [0,7]
+  CRUSH rule 5 x 206 [0,7]
+  CRUSH rule 5 x 207 [3,0]
+  CRUSH rule 5 x 208 [7,1]
+  CRUSH rule 5 x 209 [1,8]
+  CRUSH rule 5 x 210 [1,4]
+  CRUSH rule 5 x 211 [5,2]
+  CRUSH rule 5 x 212 [7,5]
+  CRUSH rule 5 x 213 [8,4]
+  CRUSH rule 5 x 214 [4,8]
+  CRUSH rule 5 x 215 [8,1]
+  CRUSH rule 5 x 216 [5,0]
+  CRUSH rule 5 x 217 [0,7]
+  CRUSH rule 5 x 218 [0,7]
+  CRUSH rule 5 x 219 [4,8]
+  CRUSH rule 5 x 220 [5,7]
+  CRUSH rule 5 x 221 [3,6]
+  CRUSH rule 5 x 222 [6,4]
+  CRUSH rule 5 x 223 [1,3]
+  CRUSH rule 5 x 224 [1,5]
+  CRUSH rule 5 x 225 [8,2]
+  CRUSH rule 5 x 226 [7,2]
+  CRUSH rule 5 x 227 [3,1]
+  CRUSH rule 5 x 228 [5,6]
+  CRUSH rule 5 x 229 [3,8]
+  CRUSH rule 5 x 230 [4,7]
+  CRUSH rule 5 x 231 [4,7]
+  CRUSH rule 5 x 232 [2,7]
+  CRUSH rule 5 x 233 [3,7]
+  CRUSH rule 5 x 234 [0,3]
+  CRUSH rule 5 x 235 [3,8]
+  CRUSH rule 5 x 236 [5,2]
+  CRUSH rule 5 x 237 [4,7]
+  CRUSH rule 5 x 238 [4,2]
+  CRUSH rule 5 x 239 [8,4]
+  CRUSH rule 5 x 240 [5,7]
+  CRUSH rule 5 x 241 [3,1]
+  CRUSH rule 5 x 242 [3,2]
+  CRUSH rule 5 x 243 [4,7]
+  CRUSH rule 5 x 244 [4,6]
+  CRUSH rule 5 x 245 [7,0]
+  CRUSH rule 5 x 246 [1,5]
+  CRUSH rule 5 x 247 [6,0]
+  CRUSH rule 5 x 248 [8,0]
+  CRUSH rule 5 x 249 [2,4]
+  CRUSH rule 5 x 250 [2,5]
+  CRUSH rule 5 x 251 [2,3]
+  CRUSH rule 5 x 252 [3,7]
+  CRUSH rule 5 x 253 [3,2]
+  CRUSH rule 5 x 254 [3,2]
+  CRUSH rule 5 x 255 [1,7]
+  CRUSH rule 5 x 256 [5,7]
+  CRUSH rule 5 x 257 [2,8]
+  CRUSH rule 5 x 258 [5,0]
+  CRUSH rule 5 x 259 [4,6]
+  CRUSH rule 5 x 260 [3,6]
+  CRUSH rule 5 x 261 [8,5]
+  CRUSH rule 5 x 262 [5,6]
+  CRUSH rule 5 x 263 [6,1]
+  CRUSH rule 5 x 264 [3,6]
+  CRUSH rule 5 x 265 [8,5]
+  CRUSH rule 5 x 266 [8,2]
+  CRUSH rule 5 x 267 [2,3]
+  CRUSH rule 5 x 268 [0,7]
+  CRUSH rule 5 x 269 [0,8]
+  CRUSH rule 5 x 270 [5,0]
+  CRUSH rule 5 x 271 [7,5]
+  CRUSH rule 5 x 272 [2,8]
+  CRUSH rule 5 x 273 [3,1]
+  CRUSH rule 5 x 274 [6,3]
+  CRUSH rule 5 x 275 [4,7]
+  CRUSH rule 5 x 276 [7,1]
+  CRUSH rule 5 x 277 [6,4]
+  CRUSH rule 5 x 278 [6,1]
+  CRUSH rule 5 x 279 [8,3]
+  CRUSH rule 5 x 280 [0,6]
+  CRUSH rule 5 x 281 [8,0]
+  CRUSH rule 5 x 282 [3,1]
+  CRUSH rule 5 x 283 [8,2]
+  CRUSH rule 5 x 284 [6,3]
+  CRUSH rule 5 x 285 [5,7]
+  CRUSH rule 5 x 286 [2,6]
+  CRUSH rule 5 x 287 [0,4]
+  CRUSH rule 5 x 288 [8,0]
+  CRUSH rule 5 x 289 [4,6]
+  CRUSH rule 5 x 290 [1,3]
+  CRUSH rule 5 x 291 [0,3]
+  CRUSH rule 5 x 292 [8,0]
+  CRUSH rule 5 x 293 [6,0]
+  CRUSH rule 5 x 294 [7,4]
+  CRUSH rule 5 x 295 [4,8]
+  CRUSH rule 5 x 296 [3,1]
+  CRUSH rule 5 x 297 [6,2]
+  CRUSH rule 5 x 298 [1,5]
+  CRUSH rule 5 x 299 [2,8]
+  CRUSH rule 5 x 300 [8,3]
+  CRUSH rule 5 x 301 [0,8]
+  CRUSH rule 5 x 302 [3,0]
+  CRUSH rule 5 x 303 [7,5]
+  CRUSH rule 5 x 304 [2,7]
+  CRUSH rule 5 x 305 [5,8]
+  CRUSH rule 5 x 306 [0,7]
+  CRUSH rule 5 x 307 [0,7]
+  CRUSH rule 5 x 308 [0,8]
+  CRUSH rule 5 x 309 [7,4]
+  CRUSH rule 5 x 310 [4,1]
+  CRUSH rule 5 x 311 [3,6]
+  CRUSH rule 5 x 312 [2,6]
+  CRUSH rule 5 x 313 [5,1]
+  CRUSH rule 5 x 314 [4,2]
+  CRUSH rule 5 x 315 [2,4]
+  CRUSH rule 5 x 316 [6,3]
+  CRUSH rule 5 x 317 [2,6]
+  CRUSH rule 5 x 318 [8,1]
+  CRUSH rule 5 x 319 [5,0]
+  CRUSH rule 5 x 320 [3,7]
+  CRUSH rule 5 x 321 [1,3]
+  CRUSH rule 5 x 322 [2,7]
+  CRUSH rule 5 x 323 [4,7]
+  CRUSH rule 5 x 324 [7,0]
+  CRUSH rule 5 x 325 [4,6]
+  CRUSH rule 5 x 326 [3,2]
+  CRUSH rule 5 x 327 [0,6]
+  CRUSH rule 5 x 328 [7,4]
+  CRUSH rule 5 x 329 [5,6]
+  CRUSH rule 5 x 330 [3,7]
+  CRUSH rule 5 x 331 [2,6]
+  CRUSH rule 5 x 332 [2,4]
+  CRUSH rule 5 x 333 [6,5]
+  CRUSH rule 5 x 334 [8,3]
+  CRUSH rule 5 x 335 [7,1]
+  CRUSH rule 5 x 336 [4,6]
+  CRUSH rule 5 x 337 [7,2]
+  CRUSH rule 5 x 338 [5,6]
+  CRUSH rule 5 x 339 [7,5]
+  CRUSH rule 5 x 340 [2,8]
+  CRUSH rule 5 x 341 [5,1]
+  CRUSH rule 5 x 342 [0,7]
+  CRUSH rule 5 x 343 [6,3]
+  CRUSH rule 5 x 344 [6,0]
+  CRUSH rule 5 x 345 [4,7]
+  CRUSH rule 5 x 346 [8,0]
+  CRUSH rule 5 x 347 [3,1]
+  CRUSH rule 5 x 348 [8,0]
+  CRUSH rule 5 x 349 [1,6]
+  CRUSH rule 5 x 350 [8,5]
+  CRUSH rule 5 x 351 [3,6]
+  CRUSH rule 5 x 352 [1,8]
+  CRUSH rule 5 x 353 [6,4]
+  CRUSH rule 5 x 354 [0,3]
+  CRUSH rule 5 x 355 [3,8]
+  CRUSH rule 5 x 356 [3,0]
+  CRUSH rule 5 x 357 [6,1]
+  CRUSH rule 5 x 358 [2,8]
+  CRUSH rule 5 x 359 [6,1]
+  CRUSH rule 5 x 360 [5,2]
+  CRUSH rule 5 x 361 [8,4]
+  CRUSH rule 5 x 362 [4,1]
+  CRUSH rule 5 x 363 [4,0]
+  CRUSH rule 5 x 364 [2,5]
+  CRUSH rule 5 x 365 [6,5]
+  CRUSH rule 5 x 366 [7,2]
+  CRUSH rule 5 x 367 [4,0]
+  CRUSH rule 5 x 368 [7,4]
+  CRUSH rule 5 x 369 [3,7]
+  CRUSH rule 5 x 370 [8,2]
+  CRUSH rule 5 x 371 [1,3]
+  CRUSH rule 5 x 372 [3,1]
+  CRUSH rule 5 x 373 [0,6]
+  CRUSH rule 5 x 374 [3,8]
+  CRUSH rule 5 x 375 [6,4]
+  CRUSH rule 5 x 376 [7,1]
+  CRUSH rule 5 x 377 [1,3]
+  CRUSH rule 5 x 378 [0,8]
+  CRUSH rule 5 x 379 [8,5]
+  CRUSH rule 5 x 380 [2,5]
+  CRUSH rule 5 x 381 [0,4]
+  CRUSH rule 5 x 382 [1,5]
+  CRUSH rule 5 x 383 [4,6]
+  CRUSH rule 5 x 384 [7,0]
+  CRUSH rule 5 x 385 [7,4]
+  CRUSH rule 5 x 386 [0,3]
+  CRUSH rule 5 x 387 [1,3]
+  CRUSH rule 5 x 388 [5,0]
+  CRUSH rule 5 x 389 [1,5]
+  CRUSH rule 5 x 390 [5,6]
+  CRUSH rule 5 x 391 [5,6]
+  CRUSH rule 5 x 392 [1,8]
+  CRUSH rule 5 x 393 [4,2]
+  CRUSH rule 5 x 394 [4,7]
+  CRUSH rule 5 x 395 [4,0]
+  CRUSH rule 5 x 396 [4,2]
+  CRUSH rule 5 x 397 [2,4]
+  CRUSH rule 5 x 398 [2,4]
+  CRUSH rule 5 x 399 [8,4]
+  CRUSH rule 5 x 400 [8,1]
+  CRUSH rule 5 x 401 [0,5]
+  CRUSH rule 5 x 402 [7,5]
+  CRUSH rule 5 x 403 [0,3]
+  CRUSH rule 5 x 404 [4,2]
+  CRUSH rule 5 x 405 [6,5]
+  CRUSH rule 5 x 406 [2,6]
+  CRUSH rule 5 x 407 [2,8]
+  CRUSH rule 5 x 408 [4,1]
+  CRUSH rule 5 x 409 [7,3]
+  CRUSH rule 5 x 410 [8,3]
+  CRUSH rule 5 x 411 [2,8]
+  CRUSH rule 5 x 412 [0,5]
+  CRUSH rule 5 x 413 [5,0]
+  CRUSH rule 5 x 414 [4,1]
+  CRUSH rule 5 x 415 [0,6]
+  CRUSH rule 5 x 416 [2,6]
+  CRUSH rule 5 x 417 [8,2]
+  CRUSH rule 5 x 418 [7,1]
+  CRUSH rule 5 x 419 [8,3]
+  CRUSH rule 5 x 420 [1,4]
+  CRUSH rule 5 x 421 [8,4]
+  CRUSH rule 5 x 422 [6,4]
+  CRUSH rule 5 x 423 [0,5]
+  CRUSH rule 5 x 424 [8,4]
+  CRUSH rule 5 x 425 [1,3]
+  CRUSH rule 5 x 426 [6,0]
+  CRUSH rule 5 x 427 [0,7]
+  CRUSH rule 5 x 428 [5,7]
+  CRUSH rule 5 x 429 [4,6]
+  CRUSH rule 5 x 430 [3,6]
+  CRUSH rule 5 x 431 [5,0]
+  CRUSH rule 5 x 432 [7,1]
+  CRUSH rule 5 x 433 [6,5]
+  CRUSH rule 5 x 434 [5,2]
+  CRUSH rule 5 x 435 [0,5]
+  CRUSH rule 5 x 436 [4,0]
+  CRUSH rule 5 x 437 [7,5]
+  CRUSH rule 5 x 438 [0,3]
+  CRUSH rule 5 x 439 [1,3]
+  CRUSH rule 5 x 440 [2,7]
+  CRUSH rule 5 x 441 [5,7]
+  CRUSH rule 5 x 442 [2,4]
+  CRUSH rule 5 x 443 [6,0]
+  CRUSH rule 5 x 444 [7,0]
+  CRUSH rule 5 x 445 [6,3]
+  CRUSH rule 5 x 446 [4,1]
+  CRUSH rule 5 x 447 [2,3]
+  CRUSH rule 5 x 448 [7,2]
+  CRUSH rule 5 x 449 [7,5]
+  CRUSH rule 5 x 450 [4,1]
+  CRUSH rule 5 x 451 [6,5]
+  CRUSH rule 5 x 452 [8,3]
+  CRUSH rule 5 x 453 [6,5]
+  CRUSH rule 5 x 454 [6,4]
+  CRUSH rule 5 x 455 [2,7]
+  CRUSH rule 5 x 456 [6,2]
+  CRUSH rule 5 x 457 [7,2]
+  CRUSH rule 5 x 458 [2,8]
+  CRUSH rule 5 x 459 [2,7]
+  CRUSH rule 5 x 460 [6,5]
+  CRUSH rule 5 x 461 [6,5]
+  CRUSH rule 5 x 462 [8,1]
+  CRUSH rule 5 x 463 [6,0]
+  CRUSH rule 5 x 464 [7,4]
+  CRUSH rule 5 x 465 [7,2]
+  CRUSH rule 5 x 466 [5,8]
+  CRUSH rule 5 x 467 [6,4]
+  CRUSH rule 5 x 468 [7,0]
+  CRUSH rule 5 x 469 [7,0]
+  CRUSH rule 5 x 470 [3,0]
+  CRUSH rule 5 x 471 [0,7]
+  CRUSH rule 5 x 472 [5,1]
+  CRUSH rule 5 x 473 [1,4]
+  CRUSH rule 5 x 474 [6,0]
+  CRUSH rule 5 x 475 [6,2]
+  CRUSH rule 5 x 476 [4,6]
+  CRUSH rule 5 x 477 [5,8]
+  CRUSH rule 5 x 478 [6,2]
+  CRUSH rule 5 x 479 [0,5]
+  CRUSH rule 5 x 480 [1,8]
+  CRUSH rule 5 x 481 [2,4]
+  CRUSH rule 5 x 482 [4,7]
+  CRUSH rule 5 x 483 [0,6]
+  CRUSH rule 5 x 484 [1,7]
+  CRUSH rule 5 x 485 [4,7]
+  CRUSH rule 5 x 486 [4,1]
+  CRUSH rule 5 x 487 [5,0]
+  CRUSH rule 5 x 488 [5,7]
+  CRUSH rule 5 x 489 [2,8]
+  CRUSH rule 5 x 490 [6,4]
+  CRUSH rule 5 x 491 [1,7]
+  CRUSH rule 5 x 492 [6,5]
+  CRUSH rule 5 x 493 [0,7]
+  CRUSH rule 5 x 494 [1,7]
+  CRUSH rule 5 x 495 [3,1]
+  CRUSH rule 5 x 496 [7,5]
+  CRUSH rule 5 x 497 [5,7]
+  CRUSH rule 5 x 498 [0,5]
+  CRUSH rule 5 x 499 [8,4]
+  CRUSH rule 5 x 500 [3,6]
+  CRUSH rule 5 x 501 [0,7]
+  CRUSH rule 5 x 502 [7,1]
+  CRUSH rule 5 x 503 [2,3]
+  CRUSH rule 5 x 504 [5,6]
+  CRUSH rule 5 x 505 [0,7]
+  CRUSH rule 5 x 506 [5,2]
+  CRUSH rule 5 x 507 [6,0]
+  CRUSH rule 5 x 508 [0,3]
+  CRUSH rule 5 x 509 [7,5]
+  CRUSH rule 5 x 510 [6,0]
+  CRUSH rule 5 x 511 [5,8]
+  CRUSH rule 5 x 512 [7,0]
+  CRUSH rule 5 x 513 [7,2]
+  CRUSH rule 5 x 514 [4,6]
+  CRUSH rule 5 x 515 [8,5]
+  CRUSH rule 5 x 516 [4,0]
+  CRUSH rule 5 x 517 [7,2]
+  CRUSH rule 5 x 518 [4,6]
+  CRUSH rule 5 x 519 [7,3]
+  CRUSH rule 5 x 520 [2,6]
+  CRUSH rule 5 x 521 [8,0]
+  CRUSH rule 5 x 522 [6,0]
+  CRUSH rule 5 x 523 [4,2]
+  CRUSH rule 5 x 524 [0,4]
+  CRUSH rule 5 x 525 [0,4]
+  CRUSH rule 5 x 526 [1,5]
+  CRUSH rule 5 x 527 [0,5]
+  CRUSH rule 5 x 528 [5,0]
+  CRUSH rule 5 x 529 [5,7]
+  CRUSH rule 5 x 530 [6,5]
+  CRUSH rule 5 x 531 [6,1]
+  CRUSH rule 5 x 532 [6,3]
+  CRUSH rule 5 x 533 [5,6]
+  CRUSH rule 5 x 534 [7,3]
+  CRUSH rule 5 x 535 [8,1]
+  CRUSH rule 5 x 536 [6,2]
+  CRUSH rule 5 x 537 [3,7]
+  CRUSH rule 5 x 538 [6,3]
+  CRUSH rule 5 x 539 [8,3]
+  CRUSH rule 5 x 540 [0,6]
+  CRUSH rule 5 x 541 [2,3]
+  CRUSH rule 5 x 542 [3,2]
+  CRUSH rule 5 x 543 [6,0]
+  CRUSH rule 5 x 544 [3,7]
+  CRUSH rule 5 x 545 [5,7]
+  CRUSH rule 5 x 546 [6,1]
+  CRUSH rule 5 x 547 [8,2]
+  CRUSH rule 5 x 548 [5,2]
+  CRUSH rule 5 x 549 [5,8]
+  CRUSH rule 5 x 550 [0,5]
+  CRUSH rule 5 x 551 [7,5]
+  CRUSH rule 5 x 552 [5,8]
+  CRUSH rule 5 x 553 [4,2]
+  CRUSH rule 5 x 554 [0,8]
+  CRUSH rule 5 x 555 [5,0]
+  CRUSH rule 5 x 556 [3,6]
+  CRUSH rule 5 x 557 [7,4]
+  CRUSH rule 5 x 558 [3,1]
+  CRUSH rule 5 x 559 [4,2]
+  CRUSH rule 5 x 560 [8,3]
+  CRUSH rule 5 x 561 [6,3]
+  CRUSH rule 5 x 562 [3,0]
+  CRUSH rule 5 x 563 [2,6]
+  CRUSH rule 5 x 564 [5,1]
+  CRUSH rule 5 x 565 [3,6]
+  CRUSH rule 5 x 566 [4,7]
+  CRUSH rule 5 x 567 [3,6]
+  CRUSH rule 5 x 568 [7,4]
+  CRUSH rule 5 x 569 [3,1]
+  CRUSH rule 5 x 570 [1,5]
+  CRUSH rule 5 x 571 [3,7]
+  CRUSH rule 5 x 572 [3,2]
+  CRUSH rule 5 x 573 [3,0]
+  CRUSH rule 5 x 574 [2,5]
+  CRUSH rule 5 x 575 [8,2]
+  CRUSH rule 5 x 576 [4,6]
+  CRUSH rule 5 x 577 [8,2]
+  CRUSH rule 5 x 578 [6,1]
+  CRUSH rule 5 x 579 [3,1]
+  CRUSH rule 5 x 580 [3,0]
+  CRUSH rule 5 x 581 [7,2]
+  CRUSH rule 5 x 582 [2,8]
+  CRUSH rule 5 x 583 [6,0]
+  CRUSH rule 5 x 584 [8,1]
+  CRUSH rule 5 x 585 [7,0]
+  CRUSH rule 5 x 586 [0,7]
+  CRUSH rule 5 x 587 [2,5]
+  CRUSH rule 5 x 588 [3,7]
+  CRUSH rule 5 x 589 [7,1]
+  CRUSH rule 5 x 590 [6,2]
+  CRUSH rule 5 x 591 [5,2]
+  CRUSH rule 5 x 592 [2,4]
+  CRUSH rule 5 x 593 [0,8]
+  CRUSH rule 5 x 594 [0,7]
+  CRUSH rule 5 x 595 [7,1]
+  CRUSH rule 5 x 596 [4,0]
+  CRUSH rule 5 x 597 [3,1]
+  CRUSH rule 5 x 598 [3,2]
+  CRUSH rule 5 x 599 [5,2]
+  CRUSH rule 5 x 600 [7,0]
+  CRUSH rule 5 x 601 [0,7]
+  CRUSH rule 5 x 602 [3,7]
+  CRUSH rule 5 x 603 [5,1]
+  CRUSH rule 5 x 604 [7,5]
+  CRUSH rule 5 x 605 [3,0]
+  CRUSH rule 5 x 606 [2,7]
+  CRUSH rule 5 x 607 [0,4]
+  CRUSH rule 5 x 608 [5,2]
+  CRUSH rule 5 x 609 [5,2]
+  CRUSH rule 5 x 610 [3,7]
+  CRUSH rule 5 x 611 [1,8]
+  CRUSH rule 5 x 612 [2,6]
+  CRUSH rule 5 x 613 [7,2]
+  CRUSH rule 5 x 614 [7,2]
+  CRUSH rule 5 x 615 [6,0]
+  CRUSH rule 5 x 616 [0,8]
+  CRUSH rule 5 x 617 [6,1]
+  CRUSH rule 5 x 618 [7,4]
+  CRUSH rule 5 x 619 [5,1]
+  CRUSH rule 5 x 620 [4,1]
+  CRUSH rule 5 x 621 [5,8]
+  CRUSH rule 5 x 622 [0,4]
+  CRUSH rule 5 x 623 [0,6]
+  CRUSH rule 5 x 624 [3,2]
+  CRUSH rule 5 x 625 [2,3]
+  CRUSH rule 5 x 626 [7,0]
+  CRUSH rule 5 x 627 [2,7]
+  CRUSH rule 5 x 628 [8,0]
+  CRUSH rule 5 x 629 [2,6]
+  CRUSH rule 5 x 630 [2,7]
+  CRUSH rule 5 x 631 [0,7]
+  CRUSH rule 5 x 632 [7,0]
+  CRUSH rule 5 x 633 [8,3]
+  CRUSH rule 5 x 634 [0,4]
+  CRUSH rule 5 x 635 [5,6]
+  CRUSH rule 5 x 636 [1,4]
+  CRUSH rule 5 x 637 [4,1]
+  CRUSH rule 5 x 638 [6,0]
+  CRUSH rule 5 x 639 [4,2]
+  CRUSH rule 5 x 640 [3,1]
+  CRUSH rule 5 x 641 [7,2]
+  CRUSH rule 5 x 642 [2,7]
+  CRUSH rule 5 x 643 [3,0]
+  CRUSH rule 5 x 644 [8,1]
+  CRUSH rule 5 x 645 [5,7]
+  CRUSH rule 5 x 646 [8,0]
+  CRUSH rule 5 x 647 [7,1]
+  CRUSH rule 5 x 648 [0,6]
+  CRUSH rule 5 x 649 [4,7]
+  CRUSH rule 5 x 650 [7,3]
+  CRUSH rule 5 x 651 [3,7]
+  CRUSH rule 5 x 652 [3,7]
+  CRUSH rule 5 x 653 [8,5]
+  CRUSH rule 5 x 654 [7,5]
+  CRUSH rule 5 x 655 [0,3]
+  CRUSH rule 5 x 656 [4,7]
+  CRUSH rule 5 x 657 [6,1]
+  CRUSH rule 5 x 658 [5,6]
+  CRUSH rule 5 x 659 [4,6]
+  CRUSH rule 5 x 660 [7,4]
+  CRUSH rule 5 x 661 [1,8]
+  CRUSH rule 5 x 662 [4,2]
+  CRUSH rule 5 x 663 [1,3]
+  CRUSH rule 5 x 664 [1,4]
+  CRUSH rule 5 x 665 [5,7]
+  CRUSH rule 5 x 666 [2,8]
+  CRUSH rule 5 x 667 [1,3]
+  CRUSH rule 5 x 668 [3,7]
+  CRUSH rule 5 x 669 [6,4]
+  CRUSH rule 5 x 670 [4,0]
+  CRUSH rule 5 x 671 [0,7]
+  CRUSH rule 5 x 672 [4,2]
+  CRUSH rule 5 x 673 [5,2]
+  CRUSH rule 5 x 674 [3,1]
+  CRUSH rule 5 x 675 [0,8]
+  CRUSH rule 5 x 676 [0,4]
+  CRUSH rule 5 x 677 [4,1]
+  CRUSH rule 5 x 678 [2,3]
+  CRUSH rule 5 x 679 [6,0]
+  CRUSH rule 5 x 680 [0,4]
+  CRUSH rule 5 x 681 [4,7]
+  CRUSH rule 5 x 682 [0,5]
+  CRUSH rule 5 x 683 [0,5]
+  CRUSH rule 5 x 684 [7,1]
+  CRUSH rule 5 x 685 [7,1]
+  CRUSH rule 5 x 686 [1,4]
+  CRUSH rule 5 x 687 [3,6]
+  CRUSH rule 5 x 688 [5,7]
+  CRUSH rule 5 x 689 [6,5]
+  CRUSH rule 5 x 690 [8,1]
+  CRUSH rule 5 x 691 [3,0]
+  CRUSH rule 5 x 692 [7,2]
+  CRUSH rule 5 x 693 [6,3]
+  CRUSH rule 5 x 694 [6,5]
+  CRUSH rule 5 x 695 [0,8]
+  CRUSH rule 5 x 696 [1,4]
+  CRUSH rule 5 x 697 [6,1]
+  CRUSH rule 5 x 698 [6,2]
+  CRUSH rule 5 x 699 [1,6]
+  CRUSH rule 5 x 700 [0,3]
+  CRUSH rule 5 x 701 [4,1]
+  CRUSH rule 5 x 702 [3,2]
+  CRUSH rule 5 x 703 [8,3]
+  CRUSH rule 5 x 704 [0,3]
+  CRUSH rule 5 x 705 [8,0]
+  CRUSH rule 5 x 706 [1,5]
+  CRUSH rule 5 x 707 [7,3]
+  CRUSH rule 5 x 708 [3,7]
+  CRUSH rule 5 x 709 [6,3]
+  CRUSH rule 5 x 710 [8,4]
+  CRUSH rule 5 x 711 [2,3]
+  CRUSH rule 5 x 712 [2,3]
+  CRUSH rule 5 x 713 [6,3]
+  CRUSH rule 5 x 714 [3,2]
+  CRUSH rule 5 x 715 [1,3]
+  CRUSH rule 5 x 716 [3,6]
+  CRUSH rule 5 x 717 [8,2]
+  CRUSH rule 5 x 718 [3,7]
+  CRUSH rule 5 x 719 [2,6]
+  CRUSH rule 5 x 720 [6,1]
+  CRUSH rule 5 x 721 [5,7]
+  CRUSH rule 5 x 722 [5,7]
+  CRUSH rule 5 x 723 [5,1]
+  CRUSH rule 5 x 724 [0,6]
+  CRUSH rule 5 x 725 [0,3]
+  CRUSH rule 5 x 726 [3,8]
+  CRUSH rule 5 x 727 [4,6]
+  CRUSH rule 5 x 728 [2,7]
+  CRUSH rule 5 x 729 [5,6]
+  CRUSH rule 5 x 730 [3,7]
+  CRUSH rule 5 x 731 [4,1]
+  CRUSH rule 5 x 732 [1,5]
+  CRUSH rule 5 x 733 [5,7]
+  CRUSH rule 5 x 734 [6,4]
+  CRUSH rule 5 x 735 [4,8]
+  CRUSH rule 5 x 736 [3,8]
+  CRUSH rule 5 x 737 [1,6]
+  CRUSH rule 5 x 738 [5,2]
+  CRUSH rule 5 x 739 [0,7]
+  CRUSH rule 5 x 740 [0,8]
+  CRUSH rule 5 x 741 [7,1]
+  CRUSH rule 5 x 742 [8,2]
+  CRUSH rule 5 x 743 [7,0]
+  CRUSH rule 5 x 744 [4,7]
+  CRUSH rule 5 x 745 [3,1]
+  CRUSH rule 5 x 746 [4,1]
+  CRUSH rule 5 x 747 [6,0]
+  CRUSH rule 5 x 748 [2,7]
+  CRUSH rule 5 x 749 [4,8]
+  CRUSH rule 5 x 750 [1,6]
+  CRUSH rule 5 x 751 [2,8]
+  CRUSH rule 5 x 752 [8,1]
+  CRUSH rule 5 x 753 [7,3]
+  CRUSH rule 5 x 754 [8,5]
+  CRUSH rule 5 x 755 [1,6]
+  CRUSH rule 5 x 756 [5,6]
+  CRUSH rule 5 x 757 [8,0]
+  CRUSH rule 5 x 758 [6,0]
+  CRUSH rule 5 x 759 [8,5]
+  CRUSH rule 5 x 760 [1,5]
+  CRUSH rule 5 x 761 [4,1]
+  CRUSH rule 5 x 762 [2,7]
+  CRUSH rule 5 x 763 [8,5]
+  CRUSH rule 5 x 764 [1,7]
+  CRUSH rule 5 x 765 [6,5]
+  CRUSH rule 5 x 766 [8,5]
+  CRUSH rule 5 x 767 [1,8]
+  CRUSH rule 5 x 768 [8,3]
+  CRUSH rule 5 x 769 [6,2]
+  CRUSH rule 5 x 770 [6,0]
+  CRUSH rule 5 x 771 [7,0]
+  CRUSH rule 5 x 772 [8,3]
+  CRUSH rule 5 x 773 [3,1]
+  CRUSH rule 5 x 774 [4,6]
+  CRUSH rule 5 x 775 [6,4]
+  CRUSH rule 5 x 776 [7,2]
+  CRUSH rule 5 x 777 [3,1]
+  CRUSH rule 5 x 778 [1,8]
+  CRUSH rule 5 x 779 [2,7]
+  CRUSH rule 5 x 780 [0,5]
+  CRUSH rule 5 x 781 [6,3]
+  CRUSH rule 5 x 782 [5,0]
+  CRUSH rule 5 x 783 [7,1]
+  CRUSH rule 5 x 784 [0,4]
+  CRUSH rule 5 x 785 [6,1]
+  CRUSH rule 5 x 786 [7,3]
+  CRUSH rule 5 x 787 [1,6]
+  CRUSH rule 5 x 788 [6,0]
+  CRUSH rule 5 x 789 [0,4]
+  CRUSH rule 5 x 790 [8,4]
+  CRUSH rule 5 x 791 [3,8]
+  CRUSH rule 5 x 792 [5,8]
+  CRUSH rule 5 x 793 [6,1]
+  CRUSH rule 5 x 794 [2,6]
+  CRUSH rule 5 x 795 [0,3]
+  CRUSH rule 5 x 796 [3,7]
+  CRUSH rule 5 x 797 [2,3]
+  CRUSH rule 5 x 798 [6,1]
+  CRUSH rule 5 x 799 [5,1]
+  CRUSH rule 5 x 800 [5,0]
+  CRUSH rule 5 x 801 [3,6]
+  CRUSH rule 5 x 802 [1,8]
+  CRUSH rule 5 x 803 [0,5]
+  CRUSH rule 5 x 804 [6,2]
+  CRUSH rule 5 x 805 [3,6]
+  CRUSH rule 5 x 806 [1,3]
+  CRUSH rule 5 x 807 [5,7]
+  CRUSH rule 5 x 808 [4,6]
+  CRUSH rule 5 x 809 [1,4]
+  CRUSH rule 5 x 810 [5,7]
+  CRUSH rule 5 x 811 [8,4]
+  CRUSH rule 5 x 812 [8,5]
+  CRUSH rule 5 x 813 [6,4]
+  CRUSH rule 5 x 814 [3,6]
+  CRUSH rule 5 x 815 [3,1]
+  CRUSH rule 5 x 816 [2,7]
+  CRUSH rule 5 x 817 [4,8]
+  CRUSH rule 5 x 818 [3,0]
+  CRUSH rule 5 x 819 [5,1]
+  CRUSH rule 5 x 820 [3,6]
+  CRUSH rule 5 x 821 [4,6]
+  CRUSH rule 5 x 822 [2,5]
+  CRUSH rule 5 x 823 [4,8]
+  CRUSH rule 5 x 824 [3,7]
+  CRUSH rule 5 x 825 [2,8]
+  CRUSH rule 5 x 826 [7,0]
+  CRUSH rule 5 x 827 [0,8]
+  CRUSH rule 5 x 828 [2,3]
+  CRUSH rule 5 x 829 [5,6]
+  CRUSH rule 5 x 830 [2,3]
+  CRUSH rule 5 x 831 [1,6]
+  CRUSH rule 5 x 832 [4,7]
+  CRUSH rule 5 x 833 [2,7]
+  CRUSH rule 5 x 834 [3,1]
+  CRUSH rule 5 x 835 [8,4]
+  CRUSH rule 5 x 836 [3,7]
+  CRUSH rule 5 x 837 [6,3]
+  CRUSH rule 5 x 838 [6,2]
+  CRUSH rule 5 x 839 [5,0]
+  CRUSH rule 5 x 840 [7,3]
+  CRUSH rule 5 x 841 [4,8]
+  CRUSH rule 5 x 842 [2,4]
+  CRUSH rule 5 x 843 [6,4]
+  CRUSH rule 5 x 844 [4,8]
+  CRUSH rule 5 x 845 [3,8]
+  CRUSH rule 5 x 846 [3,2]
+  CRUSH rule 5 x 847 [0,8]
+  CRUSH rule 5 x 848 [2,6]
+  CRUSH rule 5 x 849 [4,6]
+  CRUSH rule 5 x 850 [1,3]
+  CRUSH rule 5 x 851 [6,4]
+  CRUSH rule 5 x 852 [7,3]
+  CRUSH rule 5 x 853 [6,0]
+  CRUSH rule 5 x 854 [7,0]
+  CRUSH rule 5 x 855 [5,7]
+  CRUSH rule 5 x 856 [6,3]
+  CRUSH rule 5 x 857 [8,5]
+  CRUSH rule 5 x 858 [6,4]
+  CRUSH rule 5 x 859 [6,0]
+  CRUSH rule 5 x 860 [4,1]
+  CRUSH rule 5 x 861 [8,3]
+  CRUSH rule 5 x 862 [6,1]
+  CRUSH rule 5 x 863 [8,2]
+  CRUSH rule 5 x 864 [5,6]
+  CRUSH rule 5 x 865 [8,1]
+  CRUSH rule 5 x 866 [3,6]
+  CRUSH rule 5 x 867 [6,5]
+  CRUSH rule 5 x 868 [6,3]
+  CRUSH rule 5 x 869 [8,5]
+  CRUSH rule 5 x 870 [0,4]
+  CRUSH rule 5 x 871 [3,2]
+  CRUSH rule 5 x 872 [5,1]
+  CRUSH rule 5 x 873 [4,6]
+  CRUSH rule 5 x 874 [2,6]
+  CRUSH rule 5 x 875 [2,6]
+  CRUSH rule 5 x 876 [5,8]
+  CRUSH rule 5 x 877 [6,4]
+  CRUSH rule 5 x 878 [5,2]
+  CRUSH rule 5 x 879 [7,4]
+  CRUSH rule 5 x 880 [3,2]
+  CRUSH rule 5 x 881 [5,6]
+  CRUSH rule 5 x 882 [4,0]
+  CRUSH rule 5 x 883 [2,3]
+  CRUSH rule 5 x 884 [6,0]
+  CRUSH rule 5 x 885 [5,1]
+  CRUSH rule 5 x 886 [3,6]
+  CRUSH rule 5 x 887 [7,4]
+  CRUSH rule 5 x 888 [6,2]
+  CRUSH rule 5 x 889 [2,6]
+  CRUSH rule 5 x 890 [7,2]
+  CRUSH rule 5 x 891 [1,8]
+  CRUSH rule 5 x 892 [6,2]
+  CRUSH rule 5 x 893 [2,3]
+  CRUSH rule 5 x 894 [7,5]
+  CRUSH rule 5 x 895 [5,1]
+  CRUSH rule 5 x 896 [1,8]
+  CRUSH rule 5 x 897 [4,2]
+  CRUSH rule 5 x 898 [0,5]
+  CRUSH rule 5 x 899 [1,7]
+  CRUSH rule 5 x 900 [4,1]
+  CRUSH rule 5 x 901 [5,0]
+  CRUSH rule 5 x 902 [8,5]
+  CRUSH rule 5 x 903 [5,7]
+  CRUSH rule 5 x 904 [5,6]
+  CRUSH rule 5 x 905 [6,2]
+  CRUSH rule 5 x 906 [1,6]
+  CRUSH rule 5 x 907 [7,1]
+  CRUSH rule 5 x 908 [5,8]
+  CRUSH rule 5 x 909 [2,3]
+  CRUSH rule 5 x 910 [6,4]
+  CRUSH rule 5 x 911 [5,8]
+  CRUSH rule 5 x 912 [0,7]
+  CRUSH rule 5 x 913 [7,2]
+  CRUSH rule 5 x 914 [6,4]
+  CRUSH rule 5 x 915 [8,2]
+  CRUSH rule 5 x 916 [3,1]
+  CRUSH rule 5 x 917 [1,5]
+  CRUSH rule 5 x 918 [8,2]
+  CRUSH rule 5 x 919 [6,2]
+  CRUSH rule 5 x 920 [7,4]
+  CRUSH rule 5 x 921 [1,4]
+  CRUSH rule 5 x 922 [6,4]
+  CRUSH rule 5 x 923 [5,8]
+  CRUSH rule 5 x 924 [3,1]
+  CRUSH rule 5 x 925 [5,7]
+  CRUSH rule 5 x 926 [3,0]
+  CRUSH rule 5 x 927 [1,6]
+  CRUSH rule 5 x 928 [8,1]
+  CRUSH rule 5 x 929 [4,1]
+  CRUSH rule 5 x 930 [2,4]
+  CRUSH rule 5 x 931 [5,0]
+  CRUSH rule 5 x 932 [4,1]
+  CRUSH rule 5 x 933 [8,5]
+  CRUSH rule 5 x 934 [5,6]
+  CRUSH rule 5 x 935 [6,3]
+  CRUSH rule 5 x 936 [0,6]
+  CRUSH rule 5 x 937 [5,8]
+  CRUSH rule 5 x 938 [6,5]
+  CRUSH rule 5 x 939 [2,7]
+  CRUSH rule 5 x 940 [8,5]
+  CRUSH rule 5 x 941 [5,2]
+  CRUSH rule 5 x 942 [1,8]
+  CRUSH rule 5 x 943 [8,2]
+  CRUSH rule 5 x 944 [4,8]
+  CRUSH rule 5 x 945 [7,2]
+  CRUSH rule 5 x 946 [2,8]
+  CRUSH rule 5 x 947 [4,2]
+  CRUSH rule 5 x 948 [7,5]
+  CRUSH rule 5 x 949 [6,1]
+  CRUSH rule 5 x 950 [3,6]
+  CRUSH rule 5 x 951 [4,8]
+  CRUSH rule 5 x 952 [2,7]
+  CRUSH rule 5 x 953 [1,3]
+  CRUSH rule 5 x 954 [4,2]
+  CRUSH rule 5 x 955 [8,0]
+  CRUSH rule 5 x 956 [1,6]
+  CRUSH rule 5 x 957 [7,1]
+  CRUSH rule 5 x 958 [8,4]
+  CRUSH rule 5 x 959 [5,2]
+  CRUSH rule 5 x 960 [3,6]
+  CRUSH rule 5 x 961 [4,0]
+  CRUSH rule 5 x 962 [7,4]
+  CRUSH rule 5 x 963 [0,5]
+  CRUSH rule 5 x 964 [3,1]
+  CRUSH rule 5 x 965 [7,4]
+  CRUSH rule 5 x 966 [3,8]
+  CRUSH rule 5 x 967 [8,5]
+  CRUSH rule 5 x 968 [7,2]
+  CRUSH rule 5 x 969 [8,0]
+  CRUSH rule 5 x 970 [0,6]
+  CRUSH rule 5 x 971 [1,7]
+  CRUSH rule 5 x 972 [1,8]
+  CRUSH rule 5 x 973 [1,6]
+  CRUSH rule 5 x 974 [5,1]
+  CRUSH rule 5 x 975 [3,7]
+  CRUSH rule 5 x 976 [4,8]
+  CRUSH rule 5 x 977 [8,3]
+  CRUSH rule 5 x 978 [7,2]
+  CRUSH rule 5 x 979 [7,1]
+  CRUSH rule 5 x 980 [6,0]
+  CRUSH rule 5 x 981 [7,3]
+  CRUSH rule 5 x 982 [4,2]
+  CRUSH rule 5 x 983 [3,7]
+  CRUSH rule 5 x 984 [0,7]
+  CRUSH rule 5 x 985 [2,5]
+  CRUSH rule 5 x 986 [8,3]
+  CRUSH rule 5 x 987 [0,5]
+  CRUSH rule 5 x 988 [1,3]
+  CRUSH rule 5 x 989 [0,6]
+  CRUSH rule 5 x 990 [1,6]
+  CRUSH rule 5 x 991 [0,4]
+  CRUSH rule 5 x 992 [7,1]
+  CRUSH rule 5 x 993 [0,6]
+  CRUSH rule 5 x 994 [3,7]
+  CRUSH rule 5 x 995 [7,1]
+  CRUSH rule 5 x 996 [6,5]
+  CRUSH rule 5 x 997 [6,4]
+  CRUSH rule 5 x 998 [8,1]
+  CRUSH rule 5 x 999 [0,7]
+  CRUSH rule 5 x 1000 [8,5]
+  CRUSH rule 5 x 1001 [2,5]
+  CRUSH rule 5 x 1002 [1,3]
+  CRUSH rule 5 x 1003 [2,8]
+  CRUSH rule 5 x 1004 [6,1]
+  CRUSH rule 5 x 1005 [6,1]
+  CRUSH rule 5 x 1006 [1,6]
+  CRUSH rule 5 x 1007 [1,5]
+  CRUSH rule 5 x 1008 [1,7]
+  CRUSH rule 5 x 1009 [6,4]
+  CRUSH rule 5 x 1010 [3,1]
+  CRUSH rule 5 x 1011 [3,0]
+  CRUSH rule 5 x 1012 [3,0]
+  CRUSH rule 5 x 1013 [5,1]
+  CRUSH rule 5 x 1014 [2,8]
+  CRUSH rule 5 x 1015 [6,5]
+  CRUSH rule 5 x 1016 [2,4]
+  CRUSH rule 5 x 1017 [6,0]
+  CRUSH rule 5 x 1018 [5,0]
+  CRUSH rule 5 x 1019 [5,8]
+  CRUSH rule 5 x 1020 [5,1]
+  CRUSH rule 5 x 1021 [5,2]
+  CRUSH rule 5 x 1022 [1,6]
+  CRUSH rule 5 x 1023 [3,2]
+  rule 5 (chooseleaf-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 5 x 0 [0,5,7]
+  CRUSH rule 5 x 1 [0,8,5]
+  CRUSH rule 5 x 2 [1,3,7]
+  CRUSH rule 5 x 3 [8,0,4]
+  CRUSH rule 5 x 4 [5,0,7]
+  CRUSH rule 5 x 5 [7,0,4]
+  CRUSH rule 5 x 6 [2,6,3]
+  CRUSH rule 5 x 7 [5,8,2]
+  CRUSH rule 5 x 8 [5,6,0]
+  CRUSH rule 5 x 9 [2,3,8]
+  CRUSH rule 5 x 10 [0,7,4]
+  CRUSH rule 5 x 11 [0,7,4]
+  CRUSH rule 5 x 12 [0,4,6]
+  CRUSH rule 5 x 13 [3,8,1]
+  CRUSH rule 5 x 14 [7,0,5]
+  CRUSH rule 5 x 15 [7,2,4]
+  CRUSH rule 5 x 16 [3,6,0]
+  CRUSH rule 5 x 17 [5,1,6]
+  CRUSH rule 5 x 18 [1,4,6]
+  CRUSH rule 5 x 19 [7,5,0]
+  CRUSH rule 5 x 20 [2,4,7]
+  CRUSH rule 5 x 21 [3,7,2]
+  CRUSH rule 5 x 22 [8,3,1]
+  CRUSH rule 5 x 23 [3,6,2]
+  CRUSH rule 5 x 24 [1,7,3]
+  CRUSH rule 5 x 25 [3,7,0]
+  CRUSH rule 5 x 26 [2,8,4]
+  CRUSH rule 5 x 27 [3,1,8]
+  CRUSH rule 5 x 28 [6,0,3]
+  CRUSH rule 5 x 29 [8,5,2]
+  CRUSH rule 5 x 30 [5,7,0]
+  CRUSH rule 5 x 31 [8,0,4]
+  CRUSH rule 5 x 32 [3,6,2]
+  CRUSH rule 5 x 33 [2,7,5]
+  CRUSH rule 5 x 34 [2,5,7]
+  CRUSH rule 5 x 35 [0,8,5]
+  CRUSH rule 5 x 36 [3,8,2]
+  CRUSH rule 5 x 37 [0,4,6]
+  CRUSH rule 5 x 38 [4,8,2]
+  CRUSH rule 5 x 39 [3,7,0]
+  CRUSH rule 5 x 40 [7,2,3]
+  CRUSH rule 5 x 41 [0,6,4]
+  CRUSH rule 5 x 42 [4,6,2]
+  CRUSH rule 5 x 43 [0,3,7]
+  CRUSH rule 5 x 44 [1,6,5]
+  CRUSH rule 5 x 45 [8,0,4]
+  CRUSH rule 5 x 46 [2,4,8]
+  CRUSH rule 5 x 47 [4,2,8]
+  CRUSH rule 5 x 48 [4,6,2]
+  CRUSH rule 5 x 49 [5,7,2]
+  CRUSH rule 5 x 50 [3,1,7]
+  CRUSH rule 5 x 51 [3,6,0]
+  CRUSH rule 5 x 52 [8,1,4]
+  CRUSH rule 5 x 53 [3,8,0]
+  CRUSH rule 5 x 54 [7,3,1]
+  CRUSH rule 5 x 55 [8,2,4]
+  CRUSH rule 5 x 56 [6,4,2]
+  CRUSH rule 5 x 57 [5,8,1]
+  CRUSH rule 5 x 58 [1,8,5]
+  CRUSH rule 5 x 59 [4,2,7]
+  CRUSH rule 5 x 60 [3,2,6]
+  CRUSH rule 5 x 61 [4,6,0]
+  CRUSH rule 5 x 62 [7,0,5]
+  CRUSH rule 5 x 63 [5,6,0]
+  CRUSH rule 5 x 64 [4,2,8]
+  CRUSH rule 5 x 65 [7,3,0]
+  CRUSH rule 5 x 66 [5,6,2]
+  CRUSH rule 5 x 67 [5,0,8]
+  CRUSH rule 5 x 68 [0,5,8]
+  CRUSH rule 5 x 69 [5,1,6]
+  CRUSH rule 5 x 70 [7,0,5]
+  CRUSH rule 5 x 71 [2,8,5]
+  CRUSH rule 5 x 72 [6,1,5]
+  CRUSH rule 5 x 73 [2,7,3]
+  CRUSH rule 5 x 74 [0,7,3]
+  CRUSH rule 5 x 75 [3,2,6]
+  CRUSH rule 5 x 76 [5,1,7]
+  CRUSH rule 5 x 77 [7,2,5]
+  CRUSH rule 5 x 78 [1,4,8]
+  CRUSH rule 5 x 79 [5,1,7]
+  CRUSH rule 5 x 80 [0,3,6]
+  CRUSH rule 5 x 81 [0,3,6]
+  CRUSH rule 5 x 82 [7,1,5]
+  CRUSH rule 5 x 83 [2,6,3]
+  CRUSH rule 5 x 84 [7,2,4]
+  CRUSH rule 5 x 85 [3,8,0]
+  CRUSH rule 5 x 86 [0,6,3]
+  CRUSH rule 5 x 87 [0,7,4]
+  CRUSH rule 5 x 88 [1,6,5]
+  CRUSH rule 5 x 89 [3,0,7]
+  CRUSH rule 5 x 90 [6,4,1]
+  CRUSH rule 5 x 91 [3,8,1]
+  CRUSH rule 5 x 92 [1,8,4]
+  CRUSH rule 5 x 93 [7,4,2]
+  CRUSH rule 5 x 94 [0,4,8]
+  CRUSH rule 5 x 95 [7,5,1]
+  CRUSH rule 5 x 96 [3,6,1]
+  CRUSH rule 5 x 97 [8,4,0]
+  CRUSH rule 5 x 98 [2,7,5]
+  CRUSH rule 5 x 99 [0,7,3]
+  CRUSH rule 5 x 100 [1,7,4]
+  CRUSH rule 5 x 101 [3,7,2]
+  CRUSH rule 5 x 102 [4,2,7]
+  CRUSH rule 5 x 103 [4,7,0]
+  CRUSH rule 5 x 104 [7,4,1]
+  CRUSH rule 5 x 105 [2,4,6]
+  CRUSH rule 5 x 106 [1,6,5]
+  CRUSH rule 5 x 107 [3,2,6]
+  CRUSH rule 5 x 108 [7,2,5]
+  CRUSH rule 5 x 109 [1,4,6]
+  CRUSH rule 5 x 110 [3,2,7]
+  CRUSH rule 5 x 111 [2,3,6]
+  CRUSH rule 5 x 112 [2,6,4]
+  CRUSH rule 5 x 113 [6,2,4]
+  CRUSH rule 5 x 114 [7,3,1]
+  CRUSH rule 5 x 115 [8,2,3]
+  CRUSH rule 5 x 116 [1,6,3]
+  CRUSH rule 5 x 117 [7,3,2]
+  CRUSH rule 5 x 118 [0,3,8]
+  CRUSH rule 5 x 119 [5,6,1]
+  CRUSH rule 5 x 120 [0,3,7]
+  CRUSH rule 5 x 121 [2,7,5]
+  CRUSH rule 5 x 122 [8,5,0]
+  CRUSH rule 5 x 123 [2,5,8]
+  CRUSH rule 5 x 124 [3,2,8]
+  CRUSH rule 5 x 125 [0,7,3]
+  CRUSH rule 5 x 126 [4,2,6]
+  CRUSH rule 5 x 127 [3,6,2]
+  CRUSH rule 5 x 128 [3,6,1]
+  CRUSH rule 5 x 129 [0,3,7]
+  CRUSH rule 5 x 130 [3,8,2]
+  CRUSH rule 5 x 131 [1,3,8]
+  CRUSH rule 5 x 132 [1,4,6]
+  CRUSH rule 5 x 133 [3,6,2]
+  CRUSH rule 5 x 134 [1,8,5]
+  CRUSH rule 5 x 135 [5,6,2]
+  CRUSH rule 5 x 136 [2,3,6]
+  CRUSH rule 5 x 137 [7,3,2]
+  CRUSH rule 5 x 138 [8,4,2]
+  CRUSH rule 5 x 139 [3,0,7]
+  CRUSH rule 5 x 140 [1,6,4]
+  CRUSH rule 5 x 141 [6,2,3]
+  CRUSH rule 5 x 142 [3,0,8]
+  CRUSH rule 5 x 143 [5,8,1]
+  CRUSH rule 5 x 144 [8,1,3]
+  CRUSH rule 5 x 145 [8,5,0]
+  CRUSH rule 5 x 146 [2,6,4]
+  CRUSH rule 5 x 147 [2,8,4]
+  CRUSH rule 5 x 148 [3,1,7]
+  CRUSH rule 5 x 149 [4,8,1]
+  CRUSH rule 5 x 150 [1,6,5]
+  CRUSH rule 5 x 151 [3,6,0]
+  CRUSH rule 5 x 152 [8,4,2]
+  CRUSH rule 5 x 153 [8,4,0]
+  CRUSH rule 5 x 154 [3,2,7]
+  CRUSH rule 5 x 155 [3,7,2]
+  CRUSH rule 5 x 156 [4,2,6]
+  CRUSH rule 5 x 157 [4,1,6]
+  CRUSH rule 5 x 158 [2,8,5]
+  CRUSH rule 5 x 159 [7,0,5]
+  CRUSH rule 5 x 160 [2,8,5]
+  CRUSH rule 5 x 161 [1,5,6]
+  CRUSH rule 5 x 162 [0,6,4]
+  CRUSH rule 5 x 163 [5,6,1]
+  CRUSH rule 5 x 164 [7,1,5]
+  CRUSH rule 5 x 165 [7,0,5]
+  CRUSH rule 5 x 166 [2,4,6]
+  CRUSH rule 5 x 167 [0,7,4]
+  CRUSH rule 5 x 168 [4,2,7]
+  CRUSH rule 5 x 169 [2,6,4]
+  CRUSH rule 5 x 170 [1,4,8]
+  CRUSH rule 5 x 171 [7,5,0]
+  CRUSH rule 5 x 172 [0,7,5]
+  CRUSH rule 5 x 173 [8,5,1]
+  CRUSH rule 5 x 174 [1,4,7]
+  CRUSH rule 5 x 175 [6,0,4]
+  CRUSH rule 5 x 176 [4,1,7]
+  CRUSH rule 5 x 177 [5,1,6]
+  CRUSH rule 5 x 178 [3,0,8]
+  CRUSH rule 5 x 179 [4,1,7]
+  CRUSH rule 5 x 180 [3,8,1]
+  CRUSH rule 5 x 181 [6,2,4]
+  CRUSH rule 5 x 182 [8,5,1]
+  CRUSH rule 5 x 183 [7,5,1]
+  CRUSH rule 5 x 184 [5,7,2]
+  CRUSH rule 5 x 185 [6,1,4]
+  CRUSH rule 5 x 186 [2,5,7]
+  CRUSH rule 5 x 187 [1,6,4]
+  CRUSH rule 5 x 188 [1,8,5]
+  CRUSH rule 5 x 189 [0,7,4]
+  CRUSH rule 5 x 190 [4,0,8]
+  CRUSH rule 5 x 191 [7,1,3]
+  CRUSH rule 5 x 192 [5,0,8]
+  CRUSH rule 5 x 193 [4,2,6]
+  CRUSH rule 5 x 194 [1,3,8]
+  CRUSH rule 5 x 195 [6,4,1]
+  CRUSH rule 5 x 196 [6,0,3]
+  CRUSH rule 5 x 197 [6,5,2]
+  CRUSH rule 5 x 198 [2,5,6]
+  CRUSH rule 5 x 199 [0,5,7]
+  CRUSH rule 5 x 200 [0,5,6]
+  CRUSH rule 5 x 201 [7,1,5]
+  CRUSH rule 5 x 202 [6,3,1]
+  CRUSH rule 5 x 203 [4,8,1]
+  CRUSH rule 5 x 204 [2,3,7]
+  CRUSH rule 5 x 205 [0,7,5]
+  CRUSH rule 5 x 206 [0,7,5]
+  CRUSH rule 5 x 207 [3,0,7]
+  CRUSH rule 5 x 208 [7,1,4]
+  CRUSH rule 5 x 209 [1,8,4]
+  CRUSH rule 5 x 210 [1,4,8]
+  CRUSH rule 5 x 211 [5,2,8]
+  CRUSH rule 5 x 212 [7,5,0]
+  CRUSH rule 5 x 213 [8,4,0]
+  CRUSH rule 5 x 214 [4,8,2]
+  CRUSH rule 5 x 215 [8,1,4]
+  CRUSH rule 5 x 216 [5,0,6]
+  CRUSH rule 5 x 217 [0,7,3]
+  CRUSH rule 5 x 218 [0,7,5]
+  CRUSH rule 5 x 219 [4,8,2]
+  CRUSH rule 5 x 220 [5,7,1]
+  CRUSH rule 5 x 221 [3,6,0]
+  CRUSH rule 5 x 222 [6,4,1]
+  CRUSH rule 5 x 223 [1,3,6]
+  CRUSH rule 5 x 224 [1,5,8]
+  CRUSH rule 5 x 225 [8,2,3]
+  CRUSH rule 5 x 226 [7,2,3]
+  CRUSH rule 5 x 227 [3,1,6]
+  CRUSH rule 5 x 228 [5,6,0]
+  CRUSH rule 5 x 229 [3,8,2]
+  CRUSH rule 5 x 230 [4,7,2]
+  CRUSH rule 5 x 231 [4,7,2]
+  CRUSH rule 5 x 232 [2,7,4]
+  CRUSH rule 5 x 233 [3,7,2]
+  CRUSH rule 5 x 234 [0,3,6]
+  CRUSH rule 5 x 235 [3,8,0]
+  CRUSH rule 5 x 236 [5,2,7]
+  CRUSH rule 5 x 237 [4,7,1]
+  CRUSH rule 5 x 238 [4,2,6]
+  CRUSH rule 5 x 239 [8,4,1]
+  CRUSH rule 5 x 240 [5,7,0]
+  CRUSH rule 5 x 241 [3,1,8]
+  CRUSH rule 5 x 242 [3,2,6]
+  CRUSH rule 5 x 243 [4,7,0]
+  CRUSH rule 5 x 244 [4,6,0]
+  CRUSH rule 5 x 245 [7,0,5]
+  CRUSH rule 5 x 246 [1,5,8]
+  CRUSH rule 5 x 247 [6,0,4]
+  CRUSH rule 5 x 248 [8,0,3]
+  CRUSH rule 5 x 249 [2,4,7]
+  CRUSH rule 5 x 250 [2,5,6]
+  CRUSH rule 5 x 251 [2,3,6]
+  CRUSH rule 5 x 252 [3,7,0]
+  CRUSH rule 5 x 253 [3,2,6]
+  CRUSH rule 5 x 254 [3,2,7]
+  CRUSH rule 5 x 255 [1,7,5]
+  CRUSH rule 5 x 256 [5,7,0]
+  CRUSH rule 5 x 257 [2,8,4]
+  CRUSH rule 5 x 258 [5,0,6]
+  CRUSH rule 5 x 259 [4,6,2]
+  CRUSH rule 5 x 260 [3,6,2]
+  CRUSH rule 5 x 261 [8,5,2]
+  CRUSH rule 5 x 262 [5,6,0]
+  CRUSH rule 5 x 263 [6,1,5]
+  CRUSH rule 5 x 264 [3,6,2]
+  CRUSH rule 5 x 265 [8,5,2]
+  CRUSH rule 5 x 266 [8,2,5]
+  CRUSH rule 5 x 267 [2,3,8]
+  CRUSH rule 5 x 268 [0,7,4]
+  CRUSH rule 5 x 269 [0,8,4]
+  CRUSH rule 5 x 270 [5,0,7]
+  CRUSH rule 5 x 271 [7,5,1]
+  CRUSH rule 5 x 272 [2,8,3]
+  CRUSH rule 5 x 273 [3,1,7]
+  CRUSH rule 5 x 274 [6,3,1]
+  CRUSH rule 5 x 275 [4,7,0]
+  CRUSH rule 5 x 276 [7,1,4]
+  CRUSH rule 5 x 277 [6,4,0]
+  CRUSH rule 5 x 278 [6,1,4]
+  CRUSH rule 5 x 279 [8,3,2]
+  CRUSH rule 5 x 280 [0,6,4]
+  CRUSH rule 5 x 281 [8,0,3]
+  CRUSH rule 5 x 282 [3,1,6]
+  CRUSH rule 5 x 283 [8,2,5]
+  CRUSH rule 5 x 284 [6,3,0]
+  CRUSH rule 5 x 285 [5,7,0]
+  CRUSH rule 5 x 286 [2,6,3]
+  CRUSH rule 5 x 287 [0,4,6]
+  CRUSH rule 5 x 288 [8,0,4]
+  CRUSH rule 5 x 289 [4,6,2]
+  CRUSH rule 5 x 290 [1,3,7]
+  CRUSH rule 5 x 291 [0,3,8]
+  CRUSH rule 5 x 292 [8,0,5]
+  CRUSH rule 5 x 293 [6,0,4]
+  CRUSH rule 5 x 294 [7,4,1]
+  CRUSH rule 5 x 295 [4,8,0]
+  CRUSH rule 5 x 296 [3,1,6]
+  CRUSH rule 5 x 297 [6,2,4]
+  CRUSH rule 5 x 298 [1,5,7]
+  CRUSH rule 5 x 299 [2,8,3]
+  CRUSH rule 5 x 300 [8,3,0]
+  CRUSH rule 5 x 301 [0,8,4]
+  CRUSH rule 5 x 302 [3,0,6]
+  CRUSH rule 5 x 303 [7,5,1]
+  CRUSH rule 5 x 304 [2,7,5]
+  CRUSH rule 5 x 305 [5,8,2]
+  CRUSH rule 5 x 306 [0,7,3]
+  CRUSH rule 5 x 307 [0,7,5]
+  CRUSH rule 5 x 308 [0,8,4]
+  CRUSH rule 5 x 309 [7,4,0]
+  CRUSH rule 5 x 310 [4,1,7]
+  CRUSH rule 5 x 311 [3,6,0]
+  CRUSH rule 5 x 312 [2,6,3]
+  CRUSH rule 5 x 313 [5,1,6]
+  CRUSH rule 5 x 314 [4,2,6]
+  CRUSH rule 5 x 315 [2,4,8]
+  CRUSH rule 5 x 316 [6,3,1]
+  CRUSH rule 5 x 317 [2,6,5]
+  CRUSH rule 5 x 318 [8,1,4]
+  CRUSH rule 5 x 319 [5,0,8]
+  CRUSH rule 5 x 320 [3,7,1]
+  CRUSH rule 5 x 321 [1,3,8]
+  CRUSH rule 5 x 322 [2,7,3]
+  CRUSH rule 5 x 323 [4,7,0]
+  CRUSH rule 5 x 324 [7,0,3]
+  CRUSH rule 5 x 325 [4,6,0]
+  CRUSH rule 5 x 326 [3,2,6]
+  CRUSH rule 5 x 327 [0,6,3]
+  CRUSH rule 5 x 328 [7,4,1]
+  CRUSH rule 5 x 329 [5,6,2]
+  CRUSH rule 5 x 330 [3,7,2]
+  CRUSH rule 5 x 331 [2,6,3]
+  CRUSH rule 5 x 332 [2,4,6]
+  CRUSH rule 5 x 333 [6,5,1]
+  CRUSH rule 5 x 334 [8,3,2]
+  CRUSH rule 5 x 335 [7,1,4]
+  CRUSH rule 5 x 336 [4,6,0]
+  CRUSH rule 5 x 337 [7,2,3]
+  CRUSH rule 5 x 338 [5,6,1]
+  CRUSH rule 5 x 339 [7,5,2]
+  CRUSH rule 5 x 340 [2,8,4]
+  CRUSH rule 5 x 341 [5,1,7]
+  CRUSH rule 5 x 342 [0,7,4]
+  CRUSH rule 5 x 343 [6,3,0]
+  CRUSH rule 5 x 344 [6,0,5]
+  CRUSH rule 5 x 345 [4,7,1]
+  CRUSH rule 5 x 346 [8,0,5]
+  CRUSH rule 5 x 347 [3,1,7]
+  CRUSH rule 5 x 348 [8,0,5]
+  CRUSH rule 5 x 349 [1,6,4]
+  CRUSH rule 5 x 350 [8,5,1]
+  CRUSH rule 5 x 351 [3,6,0]
+  CRUSH rule 5 x 352 [1,8,3]
+  CRUSH rule 5 x 353 [6,4,0]
+  CRUSH rule 5 x 354 [0,3,6]
+  CRUSH rule 5 x 355 [3,8,2]
+  CRUSH rule 5 x 356 [3,0,6]
+  CRUSH rule 5 x 357 [6,1,4]
+  CRUSH rule 5 x 358 [2,8,4]
+  CRUSH rule 5 x 359 [6,1,3]
+  CRUSH rule 5 x 360 [5,2,7]
+  CRUSH rule 5 x 361 [8,4,1]
+  CRUSH rule 5 x 362 [4,1,6]
+  CRUSH rule 5 x 363 [4,0,6]
+  CRUSH rule 5 x 364 [2,5,6]
+  CRUSH rule 5 x 365 [6,5,2]
+  CRUSH rule 5 x 366 [7,2,3]
+  CRUSH rule 5 x 367 [4,0,6]
+  CRUSH rule 5 x 368 [7,4,0]
+  CRUSH rule 5 x 369 [3,7,1]
+  CRUSH rule 5 x 370 [8,2,5]
+  CRUSH rule 5 x 371 [1,3,6]
+  CRUSH rule 5 x 372 [3,1,8]
+  CRUSH rule 5 x 373 [0,6,4]
+  CRUSH rule 5 x 374 [3,8,1]
+  CRUSH rule 5 x 375 [6,4,1]
+  CRUSH rule 5 x 376 [7,1,5]
+  CRUSH rule 5 x 377 [1,3,6]
+  CRUSH rule 5 x 378 [0,8,3]
+  CRUSH rule 5 x 379 [8,5,2]
+  CRUSH rule 5 x 380 [2,5,8]
+  CRUSH rule 5 x 381 [0,4,7]
+  CRUSH rule 5 x 382 [1,5,8]
+  CRUSH rule 5 x 383 [4,6,0]
+  CRUSH rule 5 x 384 [7,0,4]
+  CRUSH rule 5 x 385 [7,4,0]
+  CRUSH rule 5 x 386 [0,3,6]
+  CRUSH rule 5 x 387 [1,3,6]
+  CRUSH rule 5 x 388 [5,0,8]
+  CRUSH rule 5 x 389 [1,5,7]
+  CRUSH rule 5 x 390 [5,6,0]
+  CRUSH rule 5 x 391 [5,6,2]
+  CRUSH rule 5 x 392 [1,8,5]
+  CRUSH rule 5 x 393 [4,2,6]
+  CRUSH rule 5 x 394 [4,7,0]
+  CRUSH rule 5 x 395 [4,0,8]
+  CRUSH rule 5 x 396 [4,2,7]
+  CRUSH rule 5 x 397 [2,4,8]
+  CRUSH rule 5 x 398 [2,4,8]
+  CRUSH rule 5 x 399 [8,4,2]
+  CRUSH rule 5 x 400 [8,1,4]
+  CRUSH rule 5 x 401 [0,5,8]
+  CRUSH rule 5 x 402 [7,5,2]
+  CRUSH rule 5 x 403 [0,3,8]
+  CRUSH rule 5 x 404 [4,2,8]
+  CRUSH rule 5 x 405 [6,5,2]
+  CRUSH rule 5 x 406 [2,6,5]
+  CRUSH rule 5 x 407 [2,8,5]
+  CRUSH rule 5 x 408 [4,1,6]
+  CRUSH rule 5 x 409 [7,3,0]
+  CRUSH rule 5 x 410 [8,3,2]
+  CRUSH rule 5 x 411 [2,8,3]
+  CRUSH rule 5 x 412 [0,5,8]
+  CRUSH rule 5 x 413 [5,0,8]
+  CRUSH rule 5 x 414 [4,1,6]
+  CRUSH rule 5 x 415 [0,6,4]
+  CRUSH rule 5 x 416 [2,6,3]
+  CRUSH rule 5 x 417 [8,2,4]
+  CRUSH rule 5 x 418 [7,1,4]
+  CRUSH rule 5 x 419 [8,3,0]
+  CRUSH rule 5 x 420 [1,4,6]
+  CRUSH rule 5 x 421 [8,4,0]
+  CRUSH rule 5 x 422 [6,4,1]
+  CRUSH rule 5 x 423 [0,5,6]
+  CRUSH rule 5 x 424 [8,4,1]
+  CRUSH rule 5 x 425 [1,3,7]
+  CRUSH rule 5 x 426 [6,0,5]
+  CRUSH rule 5 x 427 [0,7,5]
+  CRUSH rule 5 x 428 [5,7,1]
+  CRUSH rule 5 x 429 [4,6,0]
+  CRUSH rule 5 x 430 [3,6,2]
+  CRUSH rule 5 x 431 [5,0,7]
+  CRUSH rule 5 x 432 [7,1,3]
+  CRUSH rule 5 x 433 [6,5,1]
+  CRUSH rule 5 x 434 [5,2,8]
+  CRUSH rule 5 x 435 [0,5,6]
+  CRUSH rule 5 x 436 [4,0,7]
+  CRUSH rule 5 x 437 [7,5,2]
+  CRUSH rule 5 x 438 [0,3,8]
+  CRUSH rule 5 x 439 [1,3,7]
+  CRUSH rule 5 x 440 [2,7,5]
+  CRUSH rule 5 x 441 [5,7,2]
+  CRUSH rule 5 x 442 [2,4,7]
+  CRUSH rule 5 x 443 [6,0,4]
+  CRUSH rule 5 x 444 [7,0,4]
+  CRUSH rule 5 x 445 [6,3,1]
+  CRUSH rule 5 x 446 [4,1,8]
+  CRUSH rule 5 x 447 [2,3,7]
+  CRUSH rule 5 x 448 [7,2,5]
+  CRUSH rule 5 x 449 [7,5,1]
+  CRUSH rule 5 x 450 [4,1,8]
+  CRUSH rule 5 x 451 [6,5,0]
+  CRUSH rule 5 x 452 [8,3,0]
+  CRUSH rule 5 x 453 [6,5,1]
+  CRUSH rule 5 x 454 [6,4,2]
+  CRUSH rule 5 x 455 [2,7,5]
+  CRUSH rule 5 x 456 [6,2,5]
+  CRUSH rule 5 x 457 [7,2,5]
+  CRUSH rule 5 x 458 [2,8,5]
+  CRUSH rule 5 x 459 [2,7,5]
+  CRUSH rule 5 x 460 [6,5,0]
+  CRUSH rule 5 x 461 [6,5,2]
+  CRUSH rule 5 x 462 [8,1,5]
+  CRUSH rule 5 x 463 [6,0,4]
+  CRUSH rule 5 x 464 [7,4,2]
+  CRUSH rule 5 x 465 [7,2,5]
+  CRUSH rule 5 x 466 [5,8,2]
+  CRUSH rule 5 x 467 [6,4,0]
+  CRUSH rule 5 x 468 [7,0,3]
+  CRUSH rule 5 x 469 [7,0,5]
+  CRUSH rule 5 x 470 [3,0,6]
+  CRUSH rule 5 x 471 [0,7,5]
+  CRUSH rule 5 x 472 [5,1,8]
+  CRUSH rule 5 x 473 [1,4,6]
+  CRUSH rule 5 x 474 [6,0,3]
+  CRUSH rule 5 x 475 [6,2,3]
+  CRUSH rule 5 x 476 [4,6,1]
+  CRUSH rule 5 x 477 [5,8,2]
+  CRUSH rule 5 x 478 [6,2,3]
+  CRUSH rule 5 x 479 [0,5,8]
+  CRUSH rule 5 x 480 [1,8,3]
+  CRUSH rule 5 x 481 [2,4,7]
+  CRUSH rule 5 x 482 [4,7,0]
+  CRUSH rule 5 x 483 [0,6,4]
+  CRUSH rule 5 x 484 [1,7,4]
+  CRUSH rule 5 x 485 [4,7,0]
+  CRUSH rule 5 x 486 [4,1,7]
+  CRUSH rule 5 x 487 [5,0,8]
+  CRUSH rule 5 x 488 [5,7,1]
+  CRUSH rule 5 x 489 [2,8,5]
+  CRUSH rule 5 x 490 [6,4,1]
+  CRUSH rule 5 x 491 [1,7,5]
+  CRUSH rule 5 x 492 [6,5,0]
+  CRUSH rule 5 x 493 [0,7,3]
+  CRUSH rule 5 x 494 [1,7,4]
+  CRUSH rule 5 x 495 [3,1,8]
+  CRUSH rule 5 x 496 [7,5,0]
+  CRUSH rule 5 x 497 [5,7,0]
+  CRUSH rule 5 x 498 [0,5,8]
+  CRUSH rule 5 x 499 [8,4,2]
+  CRUSH rule 5 x 500 [3,6,0]
+  CRUSH rule 5 x 501 [0,7,3]
+  CRUSH rule 5 x 502 [7,1,4]
+  CRUSH rule 5 x 503 [2,3,7]
+  CRUSH rule 5 x 504 [5,6,2]
+  CRUSH rule 5 x 505 [0,7,3]
+  CRUSH rule 5 x 506 [5,2,8]
+  CRUSH rule 5 x 507 [6,0,3]
+  CRUSH rule 5 x 508 [0,3,8]
+  CRUSH rule 5 x 509 [7,5,2]
+  CRUSH rule 5 x 510 [6,0,4]
+  CRUSH rule 5 x 511 [5,8,2]
+  CRUSH rule 5 x 512 [7,0,4]
+  CRUSH rule 5 x 513 [7,2,4]
+  CRUSH rule 5 x 514 [4,6,1]
+  CRUSH rule 5 x 515 [8,5,0]
+  CRUSH rule 5 x 516 [4,0,8]
+  CRUSH rule 5 x 517 [7,2,4]
+  CRUSH rule 5 x 518 [4,6,1]
+  CRUSH rule 5 x 519 [7,3,1]
+  CRUSH rule 5 x 520 [2,6,3]
+  CRUSH rule 5 x 521 [8,0,3]
+  CRUSH rule 5 x 522 [6,0,4]
+  CRUSH rule 5 x 523 [4,2,7]
+  CRUSH rule 5 x 524 [0,4,8]
+  CRUSH rule 5 x 525 [0,4,6]
+  CRUSH rule 5 x 526 [1,5,8]
+  CRUSH rule 5 x 527 [0,5,6]
+  CRUSH rule 5 x 528 [5,0,6]
+  CRUSH rule 5 x 529 [5,7,0]
+  CRUSH rule 5 x 530 [6,5,2]
+  CRUSH rule 5 x 531 [6,1,3]
+  CRUSH rule 5 x 532 [6,3,0]
+  CRUSH rule 5 x 533 [5,6,2]
+  CRUSH rule 5 x 534 [7,3,1]
+  CRUSH rule 5 x 535 [8,1,5]
+  CRUSH rule 5 x 536 [6,2,4]
+  CRUSH rule 5 x 537 [3,7,2]
+  CRUSH rule 5 x 538 [6,3,0]
+  CRUSH rule 5 x 539 [8,3,1]
+  CRUSH rule 5 x 540 [0,6,3]
+  CRUSH rule 5 x 541 [2,3,8]
+  CRUSH rule 5 x 542 [3,2,8]
+  CRUSH rule 5 x 543 [6,0,4]
+  CRUSH rule 5 x 544 [3,7,0]
+  CRUSH rule 5 x 545 [5,7,0]
+  CRUSH rule 5 x 546 [6,1,5]
+  CRUSH rule 5 x 547 [8,2,4]
+  CRUSH rule 5 x 548 [5,2,8]
+  CRUSH rule 5 x 549 [5,8,2]
+  CRUSH rule 5 x 550 [0,5,7]
+  CRUSH rule 5 x 551 [7,5,0]
+  CRUSH rule 5 x 552 [5,8,1]
+  CRUSH rule 5 x 553 [4,2,7]
+  CRUSH rule 5 x 554 [0,8,5]
+  CRUSH rule 5 x 555 [5,0,8]
+  CRUSH rule 5 x 556 [3,6,0]
+  CRUSH rule 5 x 557 [7,4,0]
+  CRUSH rule 5 x 558 [3,1,6]
+  CRUSH rule 5 x 559 [4,2,6]
+  CRUSH rule 5 x 560 [8,3,2]
+  CRUSH rule 5 x 561 [6,3,1]
+  CRUSH rule 5 x 562 [3,0,6]
+  CRUSH rule 5 x 563 [2,6,4]
+  CRUSH rule 5 x 564 [5,1,7]
+  CRUSH rule 5 x 565 [3,6,2]
+  CRUSH rule 5 x 566 [4,7,2]
+  CRUSH rule 5 x 567 [3,6,1]
+  CRUSH rule 5 x 568 [7,4,1]
+  CRUSH rule 5 x 569 [3,1,7]
+  CRUSH rule 5 x 570 [1,5,8]
+  CRUSH rule 5 x 571 [3,7,1]
+  CRUSH rule 5 x 572 [3,2,8]
+  CRUSH rule 5 x 573 [3,0,7]
+  CRUSH rule 5 x 574 [2,5,8]
+  CRUSH rule 5 x 575 [8,2,5]
+  CRUSH rule 5 x 576 [4,6,0]
+  CRUSH rule 5 x 577 [8,2,5]
+  CRUSH rule 5 x 578 [6,1,4]
+  CRUSH rule 5 x 579 [3,1,6]
+  CRUSH rule 5 x 580 [3,0,7]
+  CRUSH rule 5 x 581 [7,2,4]
+  CRUSH rule 5 x 582 [2,8,5]
+  CRUSH rule 5 x 583 [6,0,3]
+  CRUSH rule 5 x 584 [8,1,3]
+  CRUSH rule 5 x 585 [7,0,5]
+  CRUSH rule 5 x 586 [0,7,5]
+  CRUSH rule 5 x 587 [2,5,7]
+  CRUSH rule 5 x 588 [3,7,1]
+  CRUSH rule 5 x 589 [7,1,4]
+  CRUSH rule 5 x 590 [6,2,3]
+  CRUSH rule 5 x 591 [5,2,8]
+  CRUSH rule 5 x 592 [2,4,7]
+  CRUSH rule 5 x 593 [0,8,3]
+  CRUSH rule 5 x 594 [0,7,4]
+  CRUSH rule 5 x 595 [7,1,3]
+  CRUSH rule 5 x 596 [4,0,6]
+  CRUSH rule 5 x 597 [3,1,7]
+  CRUSH rule 5 x 598 [3,2,6]
+  CRUSH rule 5 x 599 [5,2,8]
+  CRUSH rule 5 x 600 [7,0,3]
+  CRUSH rule 5 x 601 [0,7,3]
+  CRUSH rule 5 x 602 [3,7,1]
+  CRUSH rule 5 x 603 [5,1,6]
+  CRUSH rule 5 x 604 [7,5,0]
+  CRUSH rule 5 x 605 [3,0,7]
+  CRUSH rule 5 x 606 [2,7,4]
+  CRUSH rule 5 x 607 [0,4,8]
+  CRUSH rule 5 x 608 [5,2,7]
+  CRUSH rule 5 x 609 [5,2,8]
+  CRUSH rule 5 x 610 [3,7,0]
+  CRUSH rule 5 x 611 [1,8,5]
+  CRUSH rule 5 x 612 [2,6,5]
+  CRUSH rule 5 x 613 [7,2,3]
+  CRUSH rule 5 x 614 [7,2,3]
+  CRUSH rule 5 x 615 [6,0,5]
+  CRUSH rule 5 x 616 [0,8,5]
+  CRUSH rule 5 x 617 [6,1,4]
+  CRUSH rule 5 x 618 [7,4,0]
+  CRUSH rule 5 x 619 [5,1,8]
+  CRUSH rule 5 x 620 [4,1,6]
+  CRUSH rule 5 x 621 [5,8,2]
+  CRUSH rule 5 x 622 [0,4,8]
+  CRUSH rule 5 x 623 [0,6,3]
+  CRUSH rule 5 x 624 [3,2,8]
+  CRUSH rule 5 x 625 [2,3,7]
+  CRUSH rule 5 x 626 [7,0,3]
+  CRUSH rule 5 x 627 [2,7,3]
+  CRUSH rule 5 x 628 [8,0,5]
+  CRUSH rule 5 x 629 [2,6,5]
+  CRUSH rule 5 x 630 [2,7,5]
+  CRUSH rule 5 x 631 [0,7,3]
+  CRUSH rule 5 x 632 [7,0,3]
+  CRUSH rule 5 x 633 [8,3,1]
+  CRUSH rule 5 x 634 [0,4,7]
+  CRUSH rule 5 x 635 [5,6,2]
+  CRUSH rule 5 x 636 [1,4,8]
+  CRUSH rule 5 x 637 [4,1,7]
+  CRUSH rule 5 x 638 [6,0,4]
+  CRUSH rule 5 x 639 [4,2,8]
+  CRUSH rule 5 x 640 [3,1,8]
+  CRUSH rule 5 x 641 [7,2,3]
+  CRUSH rule 5 x 642 [2,7,3]
+  CRUSH rule 5 x 643 [3,0,8]
+  CRUSH rule 5 x 644 [8,1,5]
+  CRUSH rule 5 x 645 [5,7,1]
+  CRUSH rule 5 x 646 [8,0,3]
+  CRUSH rule 5 x 647 [7,1,3]
+  CRUSH rule 5 x 648 [0,6,3]
+  CRUSH rule 5 x 649 [4,7,0]
+  CRUSH rule 5 x 650 [7,3,1]
+  CRUSH rule 5 x 651 [3,7,0]
+  CRUSH rule 5 x 652 [3,7,0]
+  CRUSH rule 5 x 653 [8,5,2]
+  CRUSH rule 5 x 654 [7,5,2]
+  CRUSH rule 5 x 655 [0,3,7]
+  CRUSH rule 5 x 656 [4,7,1]
+  CRUSH rule 5 x 657 [6,1,4]
+  CRUSH rule 5 x 658 [5,6,0]
+  CRUSH rule 5 x 659 [4,6,2]
+  CRUSH rule 5 x 660 [7,4,1]
+  CRUSH rule 5 x 661 [1,8,3]
+  CRUSH rule 5 x 662 [4,2,7]
+  CRUSH rule 5 x 663 [1,3,8]
+  CRUSH rule 5 x 664 [1,4,7]
+  CRUSH rule 5 x 665 [5,7,0]
+  CRUSH rule 5 x 666 [2,8,4]
+  CRUSH rule 5 x 667 [1,3,7]
+  CRUSH rule 5 x 668 [3,7,1]
+  CRUSH rule 5 x 669 [6,4,0]
+  CRUSH rule 5 x 670 [4,0,6]
+  CRUSH rule 5 x 671 [0,7,3]
+  CRUSH rule 5 x 672 [4,2,7]
+  CRUSH rule 5 x 673 [5,2,7]
+  CRUSH rule 5 x 674 [3,1,8]
+  CRUSH rule 5 x 675 [0,8,3]
+  CRUSH rule 5 x 676 [0,4,8]
+  CRUSH rule 5 x 677 [4,1,7]
+  CRUSH rule 5 x 678 [2,3,8]
+  CRUSH rule 5 x 679 [6,0,5]
+  CRUSH rule 5 x 680 [0,4,6]
+  CRUSH rule 5 x 681 [4,7,1]
+  CRUSH rule 5 x 682 [0,5,7]
+  CRUSH rule 5 x 683 [0,5,6]
+  CRUSH rule 5 x 684 [7,1,5]
+  CRUSH rule 5 x 685 [7,1,5]
+  CRUSH rule 5 x 686 [1,4,8]
+  CRUSH rule 5 x 687 [3,6,1]
+  CRUSH rule 5 x 688 [5,7,2]
+  CRUSH rule 5 x 689 [6,5,0]
+  CRUSH rule 5 x 690 [8,1,3]
+  CRUSH rule 5 x 691 [3,0,6]
+  CRUSH rule 5 x 692 [7,2,3]
+  CRUSH rule 5 x 693 [6,3,1]
+  CRUSH rule 5 x 694 [6,5,1]
+  CRUSH rule 5 x 695 [0,8,4]
+  CRUSH rule 5 x 696 [1,4,8]
+  CRUSH rule 5 x 697 [6,1,3]
+  CRUSH rule 5 x 698 [6,2,4]
+  CRUSH rule 5 x 699 [1,6,3]
+  CRUSH rule 5 x 700 [0,3,7]
+  CRUSH rule 5 x 701 [4,1,7]
+  CRUSH rule 5 x 702 [3,2,8]
+  CRUSH rule 5 x 703 [8,3,1]
+  CRUSH rule 5 x 704 [0,3,8]
+  CRUSH rule 5 x 705 [8,0,4]
+  CRUSH rule 5 x 706 [1,5,6]
+  CRUSH rule 5 x 707 [7,3,1]
+  CRUSH rule 5 x 708 [3,7,1]
+  CRUSH rule 5 x 709 [6,3,0]
+  CRUSH rule 5 x 710 [8,4,0]
+  CRUSH rule 5 x 711 [2,3,8]
+  CRUSH rule 5 x 712 [2,3,7]
+  CRUSH rule 5 x 713 [6,3,0]
+  CRUSH rule 5 x 714 [3,2,7]
+  CRUSH rule 5 x 715 [1,3,6]
+  CRUSH rule 5 x 716 [3,6,0]
+  CRUSH rule 5 x 717 [8,2,5]
+  CRUSH rule 5 x 718 [3,7,2]
+  CRUSH rule 5 x 719 [2,6,3]
+  CRUSH rule 5 x 720 [6,1,4]
+  CRUSH rule 5 x 721 [5,7,2]
+  CRUSH rule 5 x 722 [5,7,1]
+  CRUSH rule 5 x 723 [5,1,7]
+  CRUSH rule 5 x 724 [0,6,3]
+  CRUSH rule 5 x 725 [0,3,7]
+  CRUSH rule 5 x 726 [3,8,1]
+  CRUSH rule 5 x 727 [4,6,1]
+  CRUSH rule 5 x 728 [2,7,4]
+  CRUSH rule 5 x 729 [5,6,2]
+  CRUSH rule 5 x 730 [3,7,2]
+  CRUSH rule 5 x 731 [4,1,8]
+  CRUSH rule 5 x 732 [1,5,6]
+  CRUSH rule 5 x 733 [5,7,0]
+  CRUSH rule 5 x 734 [6,4,2]
+  CRUSH rule 5 x 735 [4,8,1]
+  CRUSH rule 5 x 736 [3,8,1]
+  CRUSH rule 5 x 737 [1,6,4]
+  CRUSH rule 5 x 738 [5,2,7]
+  CRUSH rule 5 x 739 [0,7,4]
+  CRUSH rule 5 x 740 [0,8,4]
+  CRUSH rule 5 x 741 [7,1,4]
+  CRUSH rule 5 x 742 [8,2,3]
+  CRUSH rule 5 x 743 [7,0,5]
+  CRUSH rule 5 x 744 [4,7,1]
+  CRUSH rule 5 x 745 [3,1,8]
+  CRUSH rule 5 x 746 [4,1,7]
+  CRUSH rule 5 x 747 [6,0,3]
+  CRUSH rule 5 x 748 [2,7,5]
+  CRUSH rule 5 x 749 [4,8,0]
+  CRUSH rule 5 x 750 [1,6,3]
+  CRUSH rule 5 x 751 [2,8,3]
+  CRUSH rule 5 x 752 [8,1,5]
+  CRUSH rule 5 x 753 [7,3,1]
+  CRUSH rule 5 x 754 [8,5,2]
+  CRUSH rule 5 x 755 [1,6,3]
+  CRUSH rule 5 x 756 [5,6,1]
+  CRUSH rule 5 x 757 [8,0,5]
+  CRUSH rule 5 x 758 [6,0,3]
+  CRUSH rule 5 x 759 [8,5,2]
+  CRUSH rule 5 x 760 [1,5,7]
+  CRUSH rule 5 x 761 [4,1,8]
+  CRUSH rule 5 x 762 [2,7,5]
+  CRUSH rule 5 x 763 [8,5,1]
+  CRUSH rule 5 x 764 [1,7,5]
+  CRUSH rule 5 x 765 [6,5,2]
+  CRUSH rule 5 x 766 [8,5,1]
+  CRUSH rule 5 x 767 [1,8,3]
+  CRUSH rule 5 x 768 [8,3,2]
+  CRUSH rule 5 x 769 [6,2,5]
+  CRUSH rule 5 x 770 [6,0,4]
+  CRUSH rule 5 x 771 [7,0,3]
+  CRUSH rule 5 x 772 [8,3,1]
+  CRUSH rule 5 x 773 [3,1,7]
+  CRUSH rule 5 x 774 [4,6,2]
+  CRUSH rule 5 x 775 [6,4,2]
+  CRUSH rule 5 x 776 [7,2,5]
+  CRUSH rule 5 x 777 [3,1,6]
+  CRUSH rule 5 x 778 [1,8,4]
+  CRUSH rule 5 x 779 [2,7,3]
+  CRUSH rule 5 x 780 [0,5,7]
+  CRUSH rule 5 x 781 [6,3,2]
+  CRUSH rule 5 x 782 [5,0,8]
+  CRUSH rule 5 x 783 [7,1,3]
+  CRUSH rule 5 x 784 [0,4,6]
+  CRUSH rule 5 x 785 [6,1,3]
+  CRUSH rule 5 x 786 [7,3,1]
+  CRUSH rule 5 x 787 [1,6,4]
+  CRUSH rule 5 x 788 [6,0,3]
+  CRUSH rule 5 x 789 [0,4,8]
+  CRUSH rule 5 x 790 [8,4,0]
+  CRUSH rule 5 x 791 [3,8,0]
+  CRUSH rule 5 x 792 [5,8,0]
+  CRUSH rule 5 x 793 [6,1,3]
+  CRUSH rule 5 x 794 [2,6,4]
+  CRUSH rule 5 x 795 [0,3,8]
+  CRUSH rule 5 x 796 [3,7,2]
+  CRUSH rule 5 x 797 [2,3,8]
+  CRUSH rule 5 x 798 [6,1,5]
+  CRUSH rule 5 x 799 [5,1,8]
+  CRUSH rule 5 x 800 [5,0,7]
+  CRUSH rule 5 x 801 [3,6,1]
+  CRUSH rule 5 x 802 [1,8,5]
+  CRUSH rule 5 x 803 [0,5,7]
+  CRUSH rule 5 x 804 [6,2,5]
+  CRUSH rule 5 x 805 [3,6,1]
+  CRUSH rule 5 x 806 [1,3,7]
+  CRUSH rule 5 x 807 [5,7,2]
+  CRUSH rule 5 x 808 [4,6,2]
+  CRUSH rule 5 x 809 [1,4,8]
+  CRUSH rule 5 x 810 [5,7,2]
+  CRUSH rule 5 x 811 [8,4,0]
+  CRUSH rule 5 x 812 [8,5,2]
+  CRUSH rule 5 x 813 [6,4,2]
+  CRUSH rule 5 x 814 [3,6,1]
+  CRUSH rule 5 x 815 [3,1,8]
+  CRUSH rule 5 x 816 [2,7,3]
+  CRUSH rule 5 x 817 [4,8,2]
+  CRUSH rule 5 x 818 [3,0,7]
+  CRUSH rule 5 x 819 [5,1,8]
+  CRUSH rule 5 x 820 [3,6,0]
+  CRUSH rule 5 x 821 [4,6,2]
+  CRUSH rule 5 x 822 [2,5,8]
+  CRUSH rule 5 x 823 [4,8,2]
+  CRUSH rule 5 x 824 [3,7,2]
+  CRUSH rule 5 x 825 [2,8,5]
+  CRUSH rule 5 x 826 [7,0,5]
+  CRUSH rule 5 x 827 [0,8,3]
+  CRUSH rule 5 x 828 [2,3,8]
+  CRUSH rule 5 x 829 [5,6,1]
+  CRUSH rule 5 x 830 [2,3,8]
+  CRUSH rule 5 x 831 [1,6,3]
+  CRUSH rule 5 x 832 [4,7,0]
+  CRUSH rule 5 x 833 [2,7,3]
+  CRUSH rule 5 x 834 [3,1,7]
+  CRUSH rule 5 x 835 [8,4,1]
+  CRUSH rule 5 x 836 [3,7,1]
+  CRUSH rule 5 x 837 [6,3,1]
+  CRUSH rule 5 x 838 [6,2,4]
+  CRUSH rule 5 x 839 [5,0,6]
+  CRUSH rule 5 x 840 [7,3,2]
+  CRUSH rule 5 x 841 [4,8,2]
+  CRUSH rule 5 x 842 [2,4,6]
+  CRUSH rule 5 x 843 [6,4,1]
+  CRUSH rule 5 x 844 [4,8,1]
+  CRUSH rule 5 x 845 [3,8,2]
+  CRUSH rule 5 x 846 [3,2,7]
+  CRUSH rule 5 x 847 [0,8,4]
+  CRUSH rule 5 x 848 [2,6,5]
+  CRUSH rule 5 x 849 [4,6,2]
+  CRUSH rule 5 x 850 [1,3,6]
+  CRUSH rule 5 x 851 [6,4,0]
+  CRUSH rule 5 x 852 [7,3,0]
+  CRUSH rule 5 x 853 [6,0,4]
+  CRUSH rule 5 x 854 [7,0,4]
+  CRUSH rule 5 x 855 [5,7,2]
+  CRUSH rule 5 x 856 [6,3,2]
+  CRUSH rule 5 x 857 [8,5,0]
+  CRUSH rule 5 x 858 [6,4,1]
+  CRUSH rule 5 x 859 [6,0,5]
+  CRUSH rule 5 x 860 [4,1,7]
+  CRUSH rule 5 x 861 [8,3,1]
+  CRUSH rule 5 x 862 [6,1,4]
+  CRUSH rule 5 x 863 [8,2,3]
+  CRUSH rule 5 x 864 [5,6,2]
+  CRUSH rule 5 x 865 [8,1,3]
+  CRUSH rule 5 x 866 [3,6,0]
+  CRUSH rule 5 x 867 [6,5,1]
+  CRUSH rule 5 x 868 [6,3,0]
+  CRUSH rule 5 x 869 [8,5,2]
+  CRUSH rule 5 x 870 [0,4,8]
+  CRUSH rule 5 x 871 [3,2,8]
+  CRUSH rule 5 x 872 [5,1,8]
+  CRUSH rule 5 x 873 [4,6,2]
+  CRUSH rule 5 x 874 [2,6,4]
+  CRUSH rule 5 x 875 [2,6,4]
+  CRUSH rule 5 x 876 [5,8,1]
+  CRUSH rule 5 x 877 [6,4,2]
+  CRUSH rule 5 x 878 [5,2,7]
+  CRUSH rule 5 x 879 [7,4,2]
+  CRUSH rule 5 x 880 [3,2,8]
+  CRUSH rule 5 x 881 [5,6,1]
+  CRUSH rule 5 x 882 [4,0,7]
+  CRUSH rule 5 x 883 [2,3,7]
+  CRUSH rule 5 x 884 [6,0,4]
+  CRUSH rule 5 x 885 [5,1,8]
+  CRUSH rule 5 x 886 [3,6,0]
+  CRUSH rule 5 x 887 [7,4,0]
+  CRUSH rule 5 x 888 [6,2,5]
+  CRUSH rule 5 x 889 [2,6,4]
+  CRUSH rule 5 x 890 [7,2,4]
+  CRUSH rule 5 x 891 [1,8,5]
+  CRUSH rule 5 x 892 [6,2,3]
+  CRUSH rule 5 x 893 [2,3,7]
+  CRUSH rule 5 x 894 [7,5,0]
+  CRUSH rule 5 x 895 [5,1,8]
+  CRUSH rule 5 x 896 [1,8,5]
+  CRUSH rule 5 x 897 [4,2,6]
+  CRUSH rule 5 x 898 [0,5,7]
+  CRUSH rule 5 x 899 [1,7,5]
+  CRUSH rule 5 x 900 [4,1,6]
+  CRUSH rule 5 x 901 [5,0,8]
+  CRUSH rule 5 x 902 [8,5,0]
+  CRUSH rule 5 x 903 [5,7,1]
+  CRUSH rule 5 x 904 [5,6,2]
+  CRUSH rule 5 x 905 [6,2,5]
+  CRUSH rule 5 x 906 [1,6,3]
+  CRUSH rule 5 x 907 [7,1,5]
+  CRUSH rule 5 x 908 [5,8,1]
+  CRUSH rule 5 x 909 [2,3,7]
+  CRUSH rule 5 x 910 [6,4,0]
+  CRUSH rule 5 x 911 [5,8,1]
+  CRUSH rule 5 x 912 [0,7,3]
+  CRUSH rule 5 x 913 [7,2,4]
+  CRUSH rule 5 x 914 [6,4,0]
+  CRUSH rule 5 x 915 [8,2,3]
+  CRUSH rule 5 x 916 [3,1,8]
+  CRUSH rule 5 x 917 [1,5,8]
+  CRUSH rule 5 x 918 [8,2,4]
+  CRUSH rule 5 x 919 [6,2,3]
+  CRUSH rule 5 x 920 [7,4,0]
+  CRUSH rule 5 x 921 [1,4,6]
+  CRUSH rule 5 x 922 [6,4,0]
+  CRUSH rule 5 x 923 [5,8,2]
+  CRUSH rule 5 x 924 [3,1,7]
+  CRUSH rule 5 x 925 [5,7,2]
+  CRUSH rule 5 x 926 [3,0,8]
+  CRUSH rule 5 x 927 [1,6,3]
+  CRUSH rule 5 x 928 [8,1,3]
+  CRUSH rule 5 x 929 [4,1,7]
+  CRUSH rule 5 x 930 [2,4,6]
+  CRUSH rule 5 x 931 [5,0,7]
+  CRUSH rule 5 x 932 [4,1,8]
+  CRUSH rule 5 x 933 [8,5,0]
+  CRUSH rule 5 x 934 [5,6,0]
+  CRUSH rule 5 x 935 [6,3,1]
+  CRUSH rule 5 x 936 [0,6,5]
+  CRUSH rule 5 x 937 [5,8,2]
+  CRUSH rule 5 x 938 [6,5,2]
+  CRUSH rule 5 x 939 [2,7,5]
+  CRUSH rule 5 x 940 [8,5,0]
+  CRUSH rule 5 x 941 [5,2,8]
+  CRUSH rule 5 x 942 [1,8,4]
+  CRUSH rule 5 x 943 [8,2,4]
+  CRUSH rule 5 x 944 [4,8,2]
+  CRUSH rule 5 x 945 [7,2,4]
+  CRUSH rule 5 x 946 [2,8,5]
+  CRUSH rule 5 x 947 [4,2,8]
+  CRUSH rule 5 x 948 [7,5,0]
+  CRUSH rule 5 x 949 [6,1,3]
+  CRUSH rule 5 x 950 [3,6,0]
+  CRUSH rule 5 x 951 [4,8,1]
+  CRUSH rule 5 x 952 [2,7,3]
+  CRUSH rule 5 x 953 [1,3,6]
+  CRUSH rule 5 x 954 [4,2,7]
+  CRUSH rule 5 x 955 [8,0,4]
+  CRUSH rule 5 x 956 [1,6,4]
+  CRUSH rule 5 x 957 [7,1,3]
+  CRUSH rule 5 x 958 [8,4,1]
+  CRUSH rule 5 x 959 [5,2,7]
+  CRUSH rule 5 x 960 [3,6,0]
+  CRUSH rule 5 x 961 [4,0,8]
+  CRUSH rule 5 x 962 [7,4,0]
+  CRUSH rule 5 x 963 [0,5,6]
+  CRUSH rule 5 x 964 [3,1,8]
+  CRUSH rule 5 x 965 [7,4,0]
+  CRUSH rule 5 x 966 [3,8,0]
+  CRUSH rule 5 x 967 [8,5,0]
+  CRUSH rule 5 x 968 [7,2,4]
+  CRUSH rule 5 x 969 [8,0,5]
+  CRUSH rule 5 x 970 [0,6,3]
+  CRUSH rule 5 x 971 [1,7,3]
+  CRUSH rule 5 x 972 [1,8,4]
+  CRUSH rule 5 x 973 [1,6,3]
+  CRUSH rule 5 x 974 [5,1,8]
+  CRUSH rule 5 x 975 [3,7,0]
+  CRUSH rule 5 x 976 [4,8,2]
+  CRUSH rule 5 x 977 [8,3,2]
+  CRUSH rule 5 x 978 [7,2,4]
+  CRUSH rule 5 x 979 [7,1,5]
+  CRUSH rule 5 x 980 [6,0,5]
+  CRUSH rule 5 x 981 [7,3,2]
+  CRUSH rule 5 x 982 [4,2,8]
+  CRUSH rule 5 x 983 [3,7,0]
+  CRUSH rule 5 x 984 [0,7,3]
+  CRUSH rule 5 x 985 [2,5,7]
+  CRUSH rule 5 x 986 [8,3,0]
+  CRUSH rule 5 x 987 [0,5,8]
+  CRUSH rule 5 x 988 [1,3,7]
+  CRUSH rule 5 x 989 [0,6,3]
+  CRUSH rule 5 x 990 [1,6,5]
+  CRUSH rule 5 x 991 [0,4,8]
+  CRUSH rule 5 x 992 [7,1,5]
+  CRUSH rule 5 x 993 [0,6,3]
+  CRUSH rule 5 x 994 [3,7,2]
+  CRUSH rule 5 x 995 [7,1,5]
+  CRUSH rule 5 x 996 [6,5,0]
+  CRUSH rule 5 x 997 [6,4,1]
+  CRUSH rule 5 x 998 [8,1,5]
+  CRUSH rule 5 x 999 [0,7,4]
+  CRUSH rule 5 x 1000 [8,5,0]
+  CRUSH rule 5 x 1001 [2,5,6]
+  CRUSH rule 5 x 1002 [1,3,7]
+  CRUSH rule 5 x 1003 [2,8,3]
+  CRUSH rule 5 x 1004 [6,1,3]
+  CRUSH rule 5 x 1005 [6,1,5]
+  CRUSH rule 5 x 1006 [1,6,5]
+  CRUSH rule 5 x 1007 [1,5,7]
+  CRUSH rule 5 x 1008 [1,7,3]
+  CRUSH rule 5 x 1009 [6,4,1]
+  CRUSH rule 5 x 1010 [3,1,7]
+  CRUSH rule 5 x 1011 [3,0,8]
+  CRUSH rule 5 x 1012 [3,0,7]
+  CRUSH rule 5 x 1013 [5,1,7]
+  CRUSH rule 5 x 1014 [2,8,4]
+  CRUSH rule 5 x 1015 [6,5,0]
+  CRUSH rule 5 x 1016 [2,4,7]
+  CRUSH rule 5 x 1017 [6,0,3]
+  CRUSH rule 5 x 1018 [5,0,6]
+  CRUSH rule 5 x 1019 [5,8,2]
+  CRUSH rule 5 x 1020 [5,1,7]
+  CRUSH rule 5 x 1021 [5,2,6]
+  CRUSH rule 5 x 1022 [1,6,4]
+  CRUSH rule 5 x 1023 [3,2,8]
+  rule 5 (chooseleaf-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+  $ crushtool -i set-choose.crushmap --test --show-statistics --weight 0 0 --weight 1 0 --weight 3 0 --weight 4 0
+  rule 0 (choose), x = 0..1023, numrep = 2..3
+  CRUSH rule 0 x 0 [2,5]
+  CRUSH rule 0 x 1 [2,8]
+  CRUSH rule 0 x 2 [2,5]
+  CRUSH rule 0 x 3 [8,2]
+  CRUSH rule 0 x 4 [5,2]
+  CRUSH rule 0 x 5 [7,2]
+  CRUSH rule 0 x 6 [2,6]
+  CRUSH rule 0 x 7 [5,6]
+  CRUSH rule 0 x 8 [5,7]
+  CRUSH rule 0 x 9 [2,5]
+  CRUSH rule 0 x 10 [2,8]
+  CRUSH rule 0 x 11 [2,6]
+  CRUSH rule 0 x 12 [2,5]
+  CRUSH rule 0 x 13 [5,8]
+  CRUSH rule 0 x 14 [7,2]
+  CRUSH rule 0 x 15 [7,2]
+  CRUSH rule 0 x 16 [5,7]
+  CRUSH rule 0 x 17 [5,2]
+  CRUSH rule 0 x 18 [2,5]
+  CRUSH rule 0 x 19 [7,5]
+  CRUSH rule 0 x 20 [2,5]
+  CRUSH rule 0 x 21 [5,6]
+  CRUSH rule 0 x 22 [8,5]
+  CRUSH rule 0 x 23 [5,7]
+  CRUSH rule 0 x 24 [2,6]
+  CRUSH rule 0 x 25 [5,8]
+  CRUSH rule 0 x 26 [2,7]
+  CRUSH rule 0 x 27 [5,2]
+  CRUSH rule 0 x 28 [6,2]
+  CRUSH rule 0 x 29 [8,5]
+  CRUSH rule 0 x 30 [5,6]
+  CRUSH rule 0 x 31 [8,2]
+  CRUSH rule 0 x 32 [5,7]
+  CRUSH rule 0 x 33 [2,7]
+  CRUSH rule 0 x 34 [2,5]
+  CRUSH rule 0 x 35 [2,6]
+  CRUSH rule 0 x 36 [5,6]
+  CRUSH rule 0 x 37 [2,5]
+  CRUSH rule 0 x 38 [5,6]
+  CRUSH rule 0 x 39 [5,7]
+  CRUSH rule 0 x 40 [7,2]
+  CRUSH rule 0 x 41 [2,7]
+  CRUSH rule 0 x 42 [5,7]
+  CRUSH rule 0 x 43 [2,5]
+  CRUSH rule 0 x 44 [2,8]
+  CRUSH rule 0 x 45 [8,2]
+  CRUSH rule 0 x 46 [2,5]
+  CRUSH rule 0 x 47 [5,2]
+  CRUSH rule 0 x 48 [5,6]
+  CRUSH rule 0 x 49 [5,6]
+  CRUSH rule 0 x 50 [5,2]
+  CRUSH rule 0 x 51 [5,6]
+  CRUSH rule 0 x 52 [8,2]
+  CRUSH rule 0 x 53 [5,6]
+  CRUSH rule 0 x 54 [7,5]
+  CRUSH rule 0 x 55 [8,2]
+  CRUSH rule 0 x 56 [6,5]
+  CRUSH rule 0 x 57 [5,8]
+  CRUSH rule 0 x 58 [2,8]
+  CRUSH rule 0 x 59 [5,2]
+  CRUSH rule 0 x 60 [5,2]
+  CRUSH rule 0 x 61 [5,8]
+  CRUSH rule 0 x 62 [7,2]
+  CRUSH rule 0 x 63 [5,6]
+  CRUSH rule 0 x 64 [5,2]
+  CRUSH rule 0 x 65 [7,5]
+  CRUSH rule 0 x 66 [5,6]
+  CRUSH rule 0 x 67 [5,2]
+  CRUSH rule 0 x 68 [2,5]
+  CRUSH rule 0 x 69 [5,2]
+  CRUSH rule 0 x 70 [7,2]
+  CRUSH rule 0 x 71 [2,7]
+  CRUSH rule 0 x 72 [6,2]
+  CRUSH rule 0 x 73 [2,7]
+  CRUSH rule 0 x 74 [2,8]
+  CRUSH rule 0 x 75 [5,2]
+  CRUSH rule 0 x 76 [5,2]
+  CRUSH rule 0 x 77 [7,2]
+  CRUSH rule 0 x 78 [2,5]
+  CRUSH rule 0 x 79 [5,2]
+  CRUSH rule 0 x 80 [2,5]
+  CRUSH rule 0 x 81 [2,5]
+  CRUSH rule 0 x 82 [7,2]
+  CRUSH rule 0 x 83 [2,6]
+  CRUSH rule 0 x 84 [7,2]
+  CRUSH rule 0 x 85 [5,7]
+  CRUSH rule 0 x 86 [2,6]
+  CRUSH rule 0 x 87 [2,6]
+  CRUSH rule 0 x 88 [2,8]
+  CRUSH rule 0 x 89 [5,2]
+  CRUSH rule 0 x 90 [6,5]
+  CRUSH rule 0 x 91 [5,6]
+  CRUSH rule 0 x 92 [2,6]
+  CRUSH rule 0 x 93 [7,5]
+  CRUSH rule 0 x 94 [2,5]
+  CRUSH rule 0 x 95 [7,5]
+  CRUSH rule 0 x 96 [5,8]
+  CRUSH rule 0 x 97 [8,5]
+  CRUSH rule 0 x 98 [2,7]
+  CRUSH rule 0 x 99 [2,8]
+  CRUSH rule 0 x 100 [2,6]
+  CRUSH rule 0 x 101 [5,8]
+  CRUSH rule 0 x 102 [5,2]
+  CRUSH rule 0 x 103 [5,7]
+  CRUSH rule 0 x 104 [7,5]
+  CRUSH rule 0 x 105 [2,5]
+  CRUSH rule 0 x 106 [2,6]
+  CRUSH rule 0 x 107 [5,2]
+  CRUSH rule 0 x 108 [7,2]
+  CRUSH rule 0 x 109 [2,5]
+  CRUSH rule 0 x 110 [5,2]
+  CRUSH rule 0 x 111 [2,5]
+  CRUSH rule 0 x 112 [2,6]
+  CRUSH rule 0 x 113 [6,2]
+  CRUSH rule 0 x 114 [7,5]
+  CRUSH rule 0 x 115 [8,2]
+  CRUSH rule 0 x 116 [2,7]
+  CRUSH rule 0 x 117 [7,5]
+  CRUSH rule 0 x 118 [2,5]
+  CRUSH rule 0 x 119 [5,7]
+  CRUSH rule 0 x 120 [2,5]
+  CRUSH rule 0 x 121 [2,8]
+  CRUSH rule 0 x 122 [8,5]
+  CRUSH rule 0 x 123 [2,5]
+  CRUSH rule 0 x 124 [5,2]
+  CRUSH rule 0 x 125 [2,7]
+  CRUSH rule 0 x 126 [5,2]
+  CRUSH rule 0 x 127 [5,6]
+  CRUSH rule 0 x 128 [5,8]
+  CRUSH rule 0 x 129 [2,5]
+  CRUSH rule 0 x 130 [5,8]
+  CRUSH rule 0 x 131 [2,5]
+  CRUSH rule 0 x 132 [2,5]
+  CRUSH rule 0 x 133 [5,6]
+  CRUSH rule 0 x 134 [2,7]
+  CRUSH rule 0 x 135 [5,7]
+  CRUSH rule 0 x 136 [2,5]
+  CRUSH rule 0 x 137 [7,5]
+  CRUSH rule 0 x 138 [8,5]
+  CRUSH rule 0 x 139 [5,2]
+  CRUSH rule 0 x 140 [2,8]
+  CRUSH rule 0 x 141 [6,2]
+  CRUSH rule 0 x 142 [5,2]
+  CRUSH rule 0 x 143 [5,7]
+  CRUSH rule 0 x 144 [8,2]
+  CRUSH rule 0 x 145 [8,5]
+  CRUSH rule 0 x 146 [2,8]
+  CRUSH rule 0 x 147 [2,6]
+  CRUSH rule 0 x 148 [5,2]
+  CRUSH rule 0 x 149 [5,6]
+  CRUSH rule 0 x 150 [2,8]
+  CRUSH rule 0 x 151 [5,8]
+  CRUSH rule 0 x 152 [8,5]
+  CRUSH rule 0 x 153 [8,5]
+  CRUSH rule 0 x 154 [5,2]
+  CRUSH rule 0 x 155 [5,6]
+  CRUSH rule 0 x 156 [5,2]
+  CRUSH rule 0 x 157 [5,2]
+  CRUSH rule 0 x 158 [2,6]
+  CRUSH rule 0 x 159 [7,2]
+  CRUSH rule 0 x 160 [2,8]
+  CRUSH rule 0 x 161 [2,5]
+  CRUSH rule 0 x 162 [2,8]
+  CRUSH rule 0 x 163 [5,8]
+  CRUSH rule 0 x 164 [7,2]
+  CRUSH rule 0 x 165 [7,2]
+  CRUSH rule 0 x 166 [2,5]
+  CRUSH rule 0 x 167 [2,6]
+  CRUSH rule 0 x 168 [5,2]
+  CRUSH rule 0 x 169 [2,7]
+  CRUSH rule 0 x 170 [2,5]
+  CRUSH rule 0 x 171 [7,5]
+  CRUSH rule 0 x 172 [2,8]
+  CRUSH rule 0 x 173 [8,5]
+  CRUSH rule 0 x 174 [2,5]
+  CRUSH rule 0 x 175 [6,2]
+  CRUSH rule 0 x 176 [5,2]
+  CRUSH rule 0 x 177 [5,2]
+  CRUSH rule 0 x 178 [5,2]
+  CRUSH rule 0 x 179 [5,2]
+  CRUSH rule 0 x 180 [5,7]
+  CRUSH rule 0 x 181 [6,2]
+  CRUSH rule 0 x 182 [8,5]
+  CRUSH rule 0 x 183 [7,5]
+  CRUSH rule 0 x 184 [5,6]
+  CRUSH rule 0 x 185 [6,2]
+  CRUSH rule 0 x 186 [2,5]
+  CRUSH rule 0 x 187 [2,6]
+  CRUSH rule 0 x 188 [2,6]
+  CRUSH rule 0 x 189 [2,6]
+  CRUSH rule 0 x 190 [5,2]
+  CRUSH rule 0 x 191 [7,2]
+  CRUSH rule 0 x 192 [5,2]
+  CRUSH rule 0 x 193 [5,2]
+  CRUSH rule 0 x 194 [2,5]
+  CRUSH rule 0 x 195 [6,5]
+  CRUSH rule 0 x 196 [6,2]
+  CRUSH rule 0 x 197 [6,5]
+  CRUSH rule 0 x 198 [2,5]
+  CRUSH rule 0 x 199 [2,5]
+  CRUSH rule 0 x 200 [2,5]
+  CRUSH rule 0 x 201 [7,2]
+  CRUSH rule 0 x 202 [6,5]
+  CRUSH rule 0 x 203 [5,6]
+  CRUSH rule 0 x 204 [2,5]
+  CRUSH rule 0 x 205 [2,7]
+  CRUSH rule 0 x 206 [2,8]
+  CRUSH rule 0 x 207 [5,2]
+  CRUSH rule 0 x 208 [7,2]
+  CRUSH rule 0 x 209 [2,8]
+  CRUSH rule 0 x 210 [2,5]
+  CRUSH rule 0 x 211 [5,2]
+  CRUSH rule 0 x 212 [7,5]
+  CRUSH rule 0 x 213 [8,5]
+  CRUSH rule 0 x 214 [5,7]
+  CRUSH rule 0 x 215 [8,2]
+  CRUSH rule 0 x 216 [5,2]
+  CRUSH rule 0 x 217 [2,7]
+  CRUSH rule 0 x 218 [2,7]
+  CRUSH rule 0 x 219 [5,6]
+  CRUSH rule 0 x 220 [5,8]
+  CRUSH rule 0 x 221 [5,7]
+  CRUSH rule 0 x 222 [6,5]
+  CRUSH rule 0 x 223 [2,5]
+  CRUSH rule 0 x 224 [2,5]
+  CRUSH rule 0 x 225 [8,2]
+  CRUSH rule 0 x 226 [7,2]
+  CRUSH rule 0 x 227 [5,2]
+  CRUSH rule 0 x 228 [5,6]
+  CRUSH rule 0 x 229 [5,7]
+  CRUSH rule 0 x 230 [5,6]
+  CRUSH rule 0 x 231 [5,7]
+  CRUSH rule 0 x 232 [2,8]
+  CRUSH rule 0 x 233 [5,6]
+  CRUSH rule 0 x 234 [2,5]
+  CRUSH rule 0 x 235 [5,6]
+  CRUSH rule 0 x 236 [5,2]
+  CRUSH rule 0 x 237 [5,8]
+  CRUSH rule 0 x 238 [5,2]
+  CRUSH rule 0 x 239 [8,5]
+  CRUSH rule 0 x 240 [5,8]
+  CRUSH rule 0 x 241 [5,2]
+  CRUSH rule 0 x 242 [5,2]
+  CRUSH rule 0 x 243 [5,8]
+  CRUSH rule 0 x 244 [5,6]
+  CRUSH rule 0 x 245 [7,2]
+  CRUSH rule 0 x 246 [2,5]
+  CRUSH rule 0 x 247 [6,2]
+  CRUSH rule 0 x 248 [8,2]
+  CRUSH rule 0 x 249 [2,5]
+  CRUSH rule 0 x 250 [2,5]
+  CRUSH rule 0 x 251 [2,5]
+  CRUSH rule 0 x 252 [5,7]
+  CRUSH rule 0 x 253 [5,2]
+  CRUSH rule 0 x 254 [5,2]
+  CRUSH rule 0 x 255 [2,7]
+  CRUSH rule 0 x 256 [5,6]
+  CRUSH rule 0 x 257 [2,6]
+  CRUSH rule 0 x 258 [5,2]
+  CRUSH rule 0 x 259 [5,6]
+  CRUSH rule 0 x 260 [5,8]
+  CRUSH rule 0 x 261 [8,5]
+  CRUSH rule 0 x 262 [5,6]
+  CRUSH rule 0 x 263 [6,2]
+  CRUSH rule 0 x 264 [5,6]
+  CRUSH rule 0 x 265 [8,5]
+  CRUSH rule 0 x 266 [8,2]
+  CRUSH rule 0 x 267 [2,5]
+  CRUSH rule 0 x 268 [2,6]
+  CRUSH rule 0 x 269 [2,6]
+  CRUSH rule 0 x 270 [5,2]
+  CRUSH rule 0 x 271 [7,5]
+  CRUSH rule 0 x 272 [2,6]
+  CRUSH rule 0 x 273 [5,2]
+  CRUSH rule 0 x 274 [6,5]
+  CRUSH rule 0 x 275 [5,8]
+  CRUSH rule 0 x 276 [7,2]
+  CRUSH rule 0 x 277 [6,5]
+  CRUSH rule 0 x 278 [6,2]
+  CRUSH rule 0 x 279 [8,5]
+  CRUSH rule 0 x 280 [2,7]
+  CRUSH rule 0 x 281 [8,2]
+  CRUSH rule 0 x 282 [5,2]
+  CRUSH rule 0 x 283 [8,2]
+  CRUSH rule 0 x 284 [6,5]
+  CRUSH rule 0 x 285 [5,7]
+  CRUSH rule 0 x 286 [2,8]
+  CRUSH rule 0 x 287 [2,5]
+  CRUSH rule 0 x 288 [8,2]
+  CRUSH rule 0 x 289 [5,6]
+  CRUSH rule 0 x 290 [2,5]
+  CRUSH rule 0 x 291 [2,5]
+  CRUSH rule 0 x 292 [8,2]
+  CRUSH rule 0 x 293 [6,2]
+  CRUSH rule 0 x 294 [7,5]
+  CRUSH rule 0 x 295 [5,6]
+  CRUSH rule 0 x 296 [5,2]
+  CRUSH rule 0 x 297 [6,2]
+  CRUSH rule 0 x 298 [2,5]
+  CRUSH rule 0 x 299 [2,8]
+  CRUSH rule 0 x 300 [8,5]
+  CRUSH rule 0 x 301 [2,7]
+  CRUSH rule 0 x 302 [5,2]
+  CRUSH rule 0 x 303 [7,5]
+  CRUSH rule 0 x 304 [2,6]
+  CRUSH rule 0 x 305 [5,6]
+  CRUSH rule 0 x 306 [2,8]
+  CRUSH rule 0 x 307 [2,7]
+  CRUSH rule 0 x 308 [2,8]
+  CRUSH rule 0 x 309 [7,5]
+  CRUSH rule 0 x 310 [5,2]
+  CRUSH rule 0 x 311 [5,8]
+  CRUSH rule 0 x 312 [2,6]
+  CRUSH rule 0 x 313 [5,2]
+  CRUSH rule 0 x 314 [5,2]
+  CRUSH rule 0 x 315 [2,5]
+  CRUSH rule 0 x 316 [6,5]
+  CRUSH rule 0 x 317 [2,7]
+  CRUSH rule 0 x 318 [8,2]
+  CRUSH rule 0 x 319 [5,2]
+  CRUSH rule 0 x 320 [5,8]
+  CRUSH rule 0 x 321 [2,5]
+  CRUSH rule 0 x 322 [2,6]
+  CRUSH rule 0 x 323 [5,7]
+  CRUSH rule 0 x 324 [7,2]
+  CRUSH rule 0 x 325 [5,6]
+  CRUSH rule 0 x 326 [5,2]
+  CRUSH rule 0 x 327 [2,8]
+  CRUSH rule 0 x 328 [7,5]
+  CRUSH rule 0 x 329 [5,7]
+  CRUSH rule 0 x 330 [5,7]
+  CRUSH rule 0 x 331 [2,7]
+  CRUSH rule 0 x 332 [2,5]
+  CRUSH rule 0 x 333 [6,5]
+  CRUSH rule 0 x 334 [8,5]
+  CRUSH rule 0 x 335 [7,2]
+  CRUSH rule 0 x 336 [5,6]
+  CRUSH rule 0 x 337 [7,2]
+  CRUSH rule 0 x 338 [5,8]
+  CRUSH rule 0 x 339 [7,5]
+  CRUSH rule 0 x 340 [2,6]
+  CRUSH rule 0 x 341 [5,2]
+  CRUSH rule 0 x 342 [2,8]
+  CRUSH rule 0 x 343 [6,5]
+  CRUSH rule 0 x 344 [6,2]
+  CRUSH rule 0 x 345 [5,7]
+  CRUSH rule 0 x 346 [8,2]
+  CRUSH rule 0 x 347 [5,2]
+  CRUSH rule 0 x 348 [8,2]
+  CRUSH rule 0 x 349 [2,7]
+  CRUSH rule 0 x 350 [8,5]
+  CRUSH rule 0 x 351 [5,8]
+  CRUSH rule 0 x 352 [2,8]
+  CRUSH rule 0 x 353 [6,5]
+  CRUSH rule 0 x 354 [2,5]
+  CRUSH rule 0 x 355 [5,8]
+  CRUSH rule 0 x 356 [5,2]
+  CRUSH rule 0 x 357 [6,2]
+  CRUSH rule 0 x 358 [2,8]
+  CRUSH rule 0 x 359 [6,2]
+  CRUSH rule 0 x 360 [5,2]
+  CRUSH rule 0 x 361 [8,5]
+  CRUSH rule 0 x 362 [5,2]
+  CRUSH rule 0 x 363 [5,2]
+  CRUSH rule 0 x 364 [2,5]
+  CRUSH rule 0 x 365 [6,5]
+  CRUSH rule 0 x 366 [7,2]
+  CRUSH rule 0 x 367 [5,2]
+  CRUSH rule 0 x 368 [7,5]
+  CRUSH rule 0 x 369 [5,7]
+  CRUSH rule 0 x 370 [8,2]
+  CRUSH rule 0 x 371 [2,5]
+  CRUSH rule 0 x 372 [5,2]
+  CRUSH rule 0 x 373 [2,6]
+  CRUSH rule 0 x 374 [5,8]
+  CRUSH rule 0 x 375 [6,5]
+  CRUSH rule 0 x 376 [7,2]
+  CRUSH rule 0 x 377 [2,5]
+  CRUSH rule 0 x 378 [2,6]
+  CRUSH rule 0 x 379 [8,5]
+  CRUSH rule 0 x 380 [2,5]
+  CRUSH rule 0 x 381 [2,5]
+  CRUSH rule 0 x 382 [2,5]
+  CRUSH rule 0 x 383 [5,7]
+  CRUSH rule 0 x 384 [7,2]
+  CRUSH rule 0 x 385 [7,5]
+  CRUSH rule 0 x 386 [2,5]
+  CRUSH rule 0 x 387 [2,5]
+  CRUSH rule 0 x 388 [5,2]
+  CRUSH rule 0 x 389 [2,5]
+  CRUSH rule 0 x 390 [5,8]
+  CRUSH rule 0 x 391 [5,6]
+  CRUSH rule 0 x 392 [2,7]
+  CRUSH rule 0 x 393 [5,2]
+  CRUSH rule 0 x 394 [5,8]
+  CRUSH rule 0 x 395 [5,2]
+  CRUSH rule 0 x 396 [5,2]
+  CRUSH rule 0 x 397 [2,5]
+  CRUSH rule 0 x 398 [2,5]
+  CRUSH rule 0 x 399 [8,5]
+  CRUSH rule 0 x 400 [8,2]
+  CRUSH rule 0 x 401 [2,5]
+  CRUSH rule 0 x 402 [7,5]
+  CRUSH rule 0 x 403 [2,5]
+  CRUSH rule 0 x 404 [5,2]
+  CRUSH rule 0 x 405 [6,5]
+  CRUSH rule 0 x 406 [2,6]
+  CRUSH rule 0 x 407 [2,7]
+  CRUSH rule 0 x 408 [5,2]
+  CRUSH rule 0 x 409 [7,5]
+  CRUSH rule 0 x 410 [8,5]
+  CRUSH rule 0 x 411 [2,7]
+  CRUSH rule 0 x 412 [2,5]
+  CRUSH rule 0 x 413 [5,2]
+  CRUSH rule 0 x 414 [5,2]
+  CRUSH rule 0 x 415 [2,6]
+  CRUSH rule 0 x 416 [2,8]
+  CRUSH rule 0 x 417 [8,2]
+  CRUSH rule 0 x 418 [7,2]
+  CRUSH rule 0 x 419 [8,5]
+  CRUSH rule 0 x 420 [2,5]
+  CRUSH rule 0 x 421 [8,5]
+  CRUSH rule 0 x 422 [6,5]
+  CRUSH rule 0 x 423 [2,5]
+  CRUSH rule 0 x 424 [8,5]
+  CRUSH rule 0 x 425 [2,5]
+  CRUSH rule 0 x 426 [6,2]
+  CRUSH rule 0 x 427 [2,8]
+  CRUSH rule 0 x 428 [5,8]
+  CRUSH rule 0 x 429 [5,8]
+  CRUSH rule 0 x 430 [5,7]
+  CRUSH rule 0 x 431 [5,2]
+  CRUSH rule 0 x 432 [7,2]
+  CRUSH rule 0 x 433 [6,5]
+  CRUSH rule 0 x 434 [5,2]
+  CRUSH rule 0 x 435 [2,5]
+  CRUSH rule 0 x 436 [5,2]
+  CRUSH rule 0 x 437 [7,5]
+  CRUSH rule 0 x 438 [2,5]
+  CRUSH rule 0 x 439 [2,5]
+  CRUSH rule 0 x 440 [2,6]
+  CRUSH rule 0 x 441 [5,8]
+  CRUSH rule 0 x 442 [2,5]
+  CRUSH rule 0 x 443 [6,2]
+  CRUSH rule 0 x 444 [7,2]
+  CRUSH rule 0 x 445 [6,5]
+  CRUSH rule 0 x 446 [5,2]
+  CRUSH rule 0 x 447 [2,5]
+  CRUSH rule 0 x 448 [7,2]
+  CRUSH rule 0 x 449 [7,5]
+  CRUSH rule 0 x 450 [5,2]
+  CRUSH rule 0 x 451 [6,5]
+  CRUSH rule 0 x 452 [8,5]
+  CRUSH rule 0 x 453 [6,5]
+  CRUSH rule 0 x 454 [6,5]
+  CRUSH rule 0 x 455 [2,8]
+  CRUSH rule 0 x 456 [6,2]
+  CRUSH rule 0 x 457 [7,2]
+  CRUSH rule 0 x 458 [2,8]
+  CRUSH rule 0 x 459 [2,6]
+  CRUSH rule 0 x 460 [6,5]
+  CRUSH rule 0 x 461 [6,5]
+  CRUSH rule 0 x 462 [8,2]
+  CRUSH rule 0 x 463 [6,2]
+  CRUSH rule 0 x 464 [7,5]
+  CRUSH rule 0 x 465 [7,2]
+  CRUSH rule 0 x 466 [5,6]
+  CRUSH rule 0 x 467 [6,5]
+  CRUSH rule 0 x 468 [7,2]
+  CRUSH rule 0 x 469 [7,2]
+  CRUSH rule 0 x 470 [5,2]
+  CRUSH rule 0 x 471 [2,6]
+  CRUSH rule 0 x 472 [5,2]
+  CRUSH rule 0 x 473 [2,5]
+  CRUSH rule 0 x 474 [6,2]
+  CRUSH rule 0 x 475 [6,2]
+  CRUSH rule 0 x 476 [5,7]
+  CRUSH rule 0 x 477 [5,6]
+  CRUSH rule 0 x 478 [6,2]
+  CRUSH rule 0 x 479 [2,5]
+  CRUSH rule 0 x 480 [2,6]
+  CRUSH rule 0 x 481 [2,5]
+  CRUSH rule 0 x 482 [5,8]
+  CRUSH rule 0 x 483 [2,6]
+  CRUSH rule 0 x 484 [2,8]
+  CRUSH rule 0 x 485 [5,8]
+  CRUSH rule 0 x 486 [5,2]
+  CRUSH rule 0 x 487 [5,2]
+  CRUSH rule 0 x 488 [5,7]
+  CRUSH rule 0 x 489 [2,8]
+  CRUSH rule 0 x 490 [6,5]
+  CRUSH rule 0 x 491 [2,6]
+  CRUSH rule 0 x 492 [6,5]
+  CRUSH rule 0 x 493 [2,8]
+  CRUSH rule 0 x 494 [2,6]
+  CRUSH rule 0 x 495 [5,2]
+  CRUSH rule 0 x 496 [7,5]
+  CRUSH rule 0 x 497 [5,7]
+  CRUSH rule 0 x 498 [2,5]
+  CRUSH rule 0 x 499 [8,5]
+  CRUSH rule 0 x 500 [5,6]
+  CRUSH rule 0 x 501 [2,7]
+  CRUSH rule 0 x 502 [7,2]
+  CRUSH rule 0 x 503 [2,5]
+  CRUSH rule 0 x 504 [5,8]
+  CRUSH rule 0 x 505 [2,7]
+  CRUSH rule 0 x 506 [5,2]
+  CRUSH rule 0 x 507 [6,2]
+  CRUSH rule 0 x 508 [2,5]
+  CRUSH rule 0 x 509 [7,5]
+  CRUSH rule 0 x 510 [6,2]
+  CRUSH rule 0 x 511 [5,6]
+  CRUSH rule 0 x 512 [7,2]
+  CRUSH rule 0 x 513 [7,2]
+  CRUSH rule 0 x 514 [5,7]
+  CRUSH rule 0 x 515 [8,5]
+  CRUSH rule 0 x 516 [5,2]
+  CRUSH rule 0 x 517 [7,2]
+  CRUSH rule 0 x 518 [5,6]
+  CRUSH rule 0 x 519 [7,5]
+  CRUSH rule 0 x 520 [2,8]
+  CRUSH rule 0 x 521 [8,2]
+  CRUSH rule 0 x 522 [6,2]
+  CRUSH rule 0 x 523 [5,2]
+  CRUSH rule 0 x 524 [2,5]
+  CRUSH rule 0 x 525 [2,5]
+  CRUSH rule 0 x 526 [2,5]
+  CRUSH rule 0 x 527 [2,5]
+  CRUSH rule 0 x 528 [5,2]
+  CRUSH rule 0 x 529 [5,6]
+  CRUSH rule 0 x 530 [6,5]
+  CRUSH rule 0 x 531 [6,2]
+  CRUSH rule 0 x 532 [6,5]
+  CRUSH rule 0 x 533 [5,8]
+  CRUSH rule 0 x 534 [7,5]
+  CRUSH rule 0 x 535 [8,2]
+  CRUSH rule 0 x 536 [6,2]
+  CRUSH rule 0 x 537 [5,8]
+  CRUSH rule 0 x 538 [6,5]
+  CRUSH rule 0 x 539 [8,5]
+  CRUSH rule 0 x 540 [2,7]
+  CRUSH rule 0 x 541 [2,5]
+  CRUSH rule 0 x 542 [5,2]
+  CRUSH rule 0 x 543 [6,2]
+  CRUSH rule 0 x 544 [5,7]
+  CRUSH rule 0 x 545 [5,7]
+  CRUSH rule 0 x 546 [6,2]
+  CRUSH rule 0 x 547 [8,2]
+  CRUSH rule 0 x 548 [5,2]
+  CRUSH rule 0 x 549 [5,7]
+  CRUSH rule 0 x 550 [2,5]
+  CRUSH rule 0 x 551 [7,5]
+  CRUSH rule 0 x 552 [5,7]
+  CRUSH rule 0 x 553 [5,2]
+  CRUSH rule 0 x 554 [2,6]
+  CRUSH rule 0 x 555 [5,2]
+  CRUSH rule 0 x 556 [5,6]
+  CRUSH rule 0 x 557 [7,5]
+  CRUSH rule 0 x 558 [5,2]
+  CRUSH rule 0 x 559 [5,2]
+  CRUSH rule 0 x 560 [8,5]
+  CRUSH rule 0 x 561 [6,5]
+  CRUSH rule 0 x 562 [5,2]
+  CRUSH rule 0 x 563 [2,7]
+  CRUSH rule 0 x 564 [5,2]
+  CRUSH rule 0 x 565 [5,8]
+  CRUSH rule 0 x 566 [5,6]
+  CRUSH rule 0 x 567 [5,7]
+  CRUSH rule 0 x 568 [7,5]
+  CRUSH rule 0 x 569 [5,2]
+  CRUSH rule 0 x 570 [2,5]
+  CRUSH rule 0 x 571 [5,6]
+  CRUSH rule 0 x 572 [5,2]
+  CRUSH rule 0 x 573 [5,2]
+  CRUSH rule 0 x 574 [2,5]
+  CRUSH rule 0 x 575 [8,2]
+  CRUSH rule 0 x 576 [5,7]
+  CRUSH rule 0 x 577 [8,2]
+  CRUSH rule 0 x 578 [6,2]
+  CRUSH rule 0 x 579 [5,2]
+  CRUSH rule 0 x 580 [5,2]
+  CRUSH rule 0 x 581 [7,2]
+  CRUSH rule 0 x 582 [2,8]
+  CRUSH rule 0 x 583 [6,2]
+  CRUSH rule 0 x 584 [8,2]
+  CRUSH rule 0 x 585 [7,2]
+  CRUSH rule 0 x 586 [2,8]
+  CRUSH rule 0 x 587 [2,5]
+  CRUSH rule 0 x 588 [5,8]
+  CRUSH rule 0 x 589 [7,2]
+  CRUSH rule 0 x 590 [6,2]
+  CRUSH rule 0 x 591 [5,2]
+  CRUSH rule 0 x 592 [2,5]
+  CRUSH rule 0 x 593 [2,8]
+  CRUSH rule 0 x 594 [2,6]
+  CRUSH rule 0 x 595 [7,2]
+  CRUSH rule 0 x 596 [5,2]
+  CRUSH rule 0 x 597 [5,2]
+  CRUSH rule 0 x 598 [5,2]
+  CRUSH rule 0 x 599 [5,2]
+  CRUSH rule 0 x 600 [7,2]
+  CRUSH rule 0 x 601 [2,6]
+  CRUSH rule 0 x 602 [5,8]
+  CRUSH rule 0 x 603 [5,2]
+  CRUSH rule 0 x 604 [7,5]
+  CRUSH rule 0 x 605 [5,2]
+  CRUSH rule 0 x 606 [2,7]
+  CRUSH rule 0 x 607 [2,5]
+  CRUSH rule 0 x 608 [5,2]
+  CRUSH rule 0 x 609 [5,2]
+  CRUSH rule 0 x 610 [5,8]
+  CRUSH rule 0 x 611 [2,8]
+  CRUSH rule 0 x 612 [2,8]
+  CRUSH rule 0 x 613 [7,2]
+  CRUSH rule 0 x 614 [7,2]
+  CRUSH rule 0 x 615 [6,2]
+  CRUSH rule 0 x 616 [2,7]
+  CRUSH rule 0 x 617 [6,2]
+  CRUSH rule 0 x 618 [7,5]
+  CRUSH rule 0 x 619 [5,2]
+  CRUSH rule 0 x 620 [5,2]
+  CRUSH rule 0 x 621 [5,6]
+  CRUSH rule 0 x 622 [2,5]
+  CRUSH rule 0 x 623 [2,8]
+  CRUSH rule 0 x 624 [5,2]
+  CRUSH rule 0 x 625 [2,5]
+  CRUSH rule 0 x 626 [7,2]
+  CRUSH rule 0 x 627 [2,6]
+  CRUSH rule 0 x 628 [8,2]
+  CRUSH rule 0 x 629 [2,6]
+  CRUSH rule 0 x 630 [2,6]
+  CRUSH rule 0 x 631 [2,6]
+  CRUSH rule 0 x 632 [7,2]
+  CRUSH rule 0 x 633 [8,5]
+  CRUSH rule 0 x 634 [2,5]
+  CRUSH rule 0 x 635 [5,6]
+  CRUSH rule 0 x 636 [2,5]
+  CRUSH rule 0 x 637 [5,2]
+  CRUSH rule 0 x 638 [6,2]
+  CRUSH rule 0 x 639 [5,2]
+  CRUSH rule 0 x 640 [5,2]
+  CRUSH rule 0 x 641 [7,2]
+  CRUSH rule 0 x 642 [2,8]
+  CRUSH rule 0 x 643 [5,2]
+  CRUSH rule 0 x 644 [8,2]
+  CRUSH rule 0 x 645 [5,7]
+  CRUSH rule 0 x 646 [8,2]
+  CRUSH rule 0 x 647 [7,2]
+  CRUSH rule 0 x 648 [2,8]
+  CRUSH rule 0 x 649 [5,7]
+  CRUSH rule 0 x 650 [7,5]
+  CRUSH rule 0 x 651 [5,6]
+  CRUSH rule 0 x 652 [5,6]
+  CRUSH rule 0 x 653 [8,5]
+  CRUSH rule 0 x 654 [7,5]
+  CRUSH rule 0 x 655 [2,5]
+  CRUSH rule 0 x 656 [5,7]
+  CRUSH rule 0 x 657 [6,2]
+  CRUSH rule 0 x 658 [5,8]
+  CRUSH rule 0 x 659 [5,7]
+  CRUSH rule 0 x 660 [7,5]
+  CRUSH rule 0 x 661 [2,7]
+  CRUSH rule 0 x 662 [5,2]
+  CRUSH rule 0 x 663 [2,5]
+  CRUSH rule 0 x 664 [2,5]
+  CRUSH rule 0 x 665 [5,6]
+  CRUSH rule 0 x 666 [2,7]
+  CRUSH rule 0 x 667 [2,5]
+  CRUSH rule 0 x 668 [5,7]
+  CRUSH rule 0 x 669 [6,5]
+  CRUSH rule 0 x 670 [5,2]
+  CRUSH rule 0 x 671 [2,8]
+  CRUSH rule 0 x 672 [5,2]
+  CRUSH rule 0 x 673 [5,2]
+  CRUSH rule 0 x 674 [5,2]
+  CRUSH rule 0 x 675 [2,8]
+  CRUSH rule 0 x 676 [2,5]
+  CRUSH rule 0 x 677 [5,2]
+  CRUSH rule 0 x 678 [2,5]
+  CRUSH rule 0 x 679 [6,2]
+  CRUSH rule 0 x 680 [2,5]
+  CRUSH rule 0 x 681 [5,6]
+  CRUSH rule 0 x 682 [2,5]
+  CRUSH rule 0 x 683 [2,5]
+  CRUSH rule 0 x 684 [7,2]
+  CRUSH rule 0 x 685 [7,2]
+  CRUSH rule 0 x 686 [2,5]
+  CRUSH rule 0 x 687 [5,7]
+  CRUSH rule 0 x 688 [5,6]
+  CRUSH rule 0 x 689 [6,5]
+  CRUSH rule 0 x 690 [8,2]
+  CRUSH rule 0 x 691 [5,2]
+  CRUSH rule 0 x 692 [7,2]
+  CRUSH rule 0 x 693 [6,5]
+  CRUSH rule 0 x 694 [6,5]
+  CRUSH rule 0 x 695 [2,6]
+  CRUSH rule 0 x 696 [2,5]
+  CRUSH rule 0 x 697 [6,2]
+  CRUSH rule 0 x 698 [6,2]
+  CRUSH rule 0 x 699 [2,8]
+  CRUSH rule 0 x 700 [2,5]
+  CRUSH rule 0 x 701 [5,2]
+  CRUSH rule 0 x 702 [5,2]
+  CRUSH rule 0 x 703 [8,5]
+  CRUSH rule 0 x 704 [2,5]
+  CRUSH rule 0 x 705 [8,2]
+  CRUSH rule 0 x 706 [2,5]
+  CRUSH rule 0 x 707 [7,5]
+  CRUSH rule 0 x 708 [5,7]
+  CRUSH rule 0 x 709 [6,5]
+  CRUSH rule 0 x 710 [8,5]
+  CRUSH rule 0 x 711 [2,5]
+  CRUSH rule 0 x 712 [2,5]
+  CRUSH rule 0 x 713 [6,5]
+  CRUSH rule 0 x 714 [5,2]
+  CRUSH rule 0 x 715 [2,5]
+  CRUSH rule 0 x 716 [5,6]
+  CRUSH rule 0 x 717 [8,2]
+  CRUSH rule 0 x 718 [5,7]
+  CRUSH rule 0 x 719 [2,6]
+  CRUSH rule 0 x 720 [6,2]
+  CRUSH rule 0 x 721 [5,7]
+  CRUSH rule 0 x 722 [5,7]
+  CRUSH rule 0 x 723 [5,2]
+  CRUSH rule 0 x 724 [2,7]
+  CRUSH rule 0 x 725 [2,5]
+  CRUSH rule 0 x 726 [5,7]
+  CRUSH rule 0 x 727 [5,7]
+  CRUSH rule 0 x 728 [2,6]
+  CRUSH rule 0 x 729 [5,6]
+  CRUSH rule 0 x 730 [5,8]
+  CRUSH rule 0 x 731 [5,2]
+  CRUSH rule 0 x 732 [2,5]
+  CRUSH rule 0 x 733 [5,6]
+  CRUSH rule 0 x 734 [6,5]
+  CRUSH rule 0 x 735 [5,6]
+  CRUSH rule 0 x 736 [5,8]
+  CRUSH rule 0 x 737 [2,8]
+  CRUSH rule 0 x 738 [5,2]
+  CRUSH rule 0 x 739 [2,7]
+  CRUSH rule 0 x 740 [2,7]
+  CRUSH rule 0 x 741 [7,2]
+  CRUSH rule 0 x 742 [8,2]
+  CRUSH rule 0 x 743 [7,2]
+  CRUSH rule 0 x 744 [5,6]
+  CRUSH rule 0 x 745 [5,2]
+  CRUSH rule 0 x 746 [5,2]
+  CRUSH rule 0 x 747 [6,2]
+  CRUSH rule 0 x 748 [2,6]
+  CRUSH rule 0 x 749 [5,7]
+  CRUSH rule 0 x 750 [2,7]
+  CRUSH rule 0 x 751 [2,7]
+  CRUSH rule 0 x 752 [8,2]
+  CRUSH rule 0 x 753 [7,5]
+  CRUSH rule 0 x 754 [8,5]
+  CRUSH rule 0 x 755 [2,6]
+  CRUSH rule 0 x 756 [5,8]
+  CRUSH rule 0 x 757 [8,2]
+  CRUSH rule 0 x 758 [6,2]
+  CRUSH rule 0 x 759 [8,5]
+  CRUSH rule 0 x 760 [2,5]
+  CRUSH rule 0 x 761 [5,2]
+  CRUSH rule 0 x 762 [2,8]
+  CRUSH rule 0 x 763 [8,5]
+  CRUSH rule 0 x 764 [2,7]
+  CRUSH rule 0 x 765 [6,5]
+  CRUSH rule 0 x 766 [8,5]
+  CRUSH rule 0 x 767 [2,8]
+  CRUSH rule 0 x 768 [8,5]
+  CRUSH rule 0 x 769 [6,2]
+  CRUSH rule 0 x 770 [6,2]
+  CRUSH rule 0 x 771 [7,2]
+  CRUSH rule 0 x 772 [8,5]
+  CRUSH rule 0 x 773 [5,2]
+  CRUSH rule 0 x 774 [5,7]
+  CRUSH rule 0 x 775 [6,5]
+  CRUSH rule 0 x 776 [7,2]
+  CRUSH rule 0 x 777 [5,2]
+  CRUSH rule 0 x 778 [2,6]
+  CRUSH rule 0 x 779 [2,6]
+  CRUSH rule 0 x 780 [2,5]
+  CRUSH rule 0 x 781 [6,5]
+  CRUSH rule 0 x 782 [5,2]
+  CRUSH rule 0 x 783 [7,2]
+  CRUSH rule 0 x 784 [2,5]
+  CRUSH rule 0 x 785 [6,2]
+  CRUSH rule 0 x 786 [7,5]
+  CRUSH rule 0 x 787 [2,8]
+  CRUSH rule 0 x 788 [6,2]
+  CRUSH rule 0 x 789 [2,5]
+  CRUSH rule 0 x 790 [8,5]
+  CRUSH rule 0 x 791 [5,6]
+  CRUSH rule 0 x 792 [5,6]
+  CRUSH rule 0 x 793 [6,2]
+  CRUSH rule 0 x 794 [2,8]
+  CRUSH rule 0 x 795 [2,5]
+  CRUSH rule 0 x 796 [5,7]
+  CRUSH rule 0 x 797 [2,5]
+  CRUSH rule 0 x 798 [6,2]
+  CRUSH rule 0 x 799 [5,2]
+  CRUSH rule 0 x 800 [5,2]
+  CRUSH rule 0 x 801 [5,7]
+  CRUSH rule 0 x 802 [2,6]
+  CRUSH rule 0 x 803 [2,5]
+  CRUSH rule 0 x 804 [6,2]
+  CRUSH rule 0 x 805 [5,8]
+  CRUSH rule 0 x 806 [2,5]
+  CRUSH rule 0 x 807 [5,7]
+  CRUSH rule 0 x 808 [5,7]
+  CRUSH rule 0 x 809 [2,5]
+  CRUSH rule 0 x 810 [5,7]
+  CRUSH rule 0 x 811 [8,5]
+  CRUSH rule 0 x 812 [8,5]
+  CRUSH rule 0 x 813 [6,5]
+  CRUSH rule 0 x 814 [5,8]
+  CRUSH rule 0 x 815 [5,2]
+  CRUSH rule 0 x 816 [2,6]
+  CRUSH rule 0 x 817 [5,6]
+  CRUSH rule 0 x 818 [5,2]
+  CRUSH rule 0 x 819 [5,2]
+  CRUSH rule 0 x 820 [5,7]
+  CRUSH rule 0 x 821 [5,8]
+  CRUSH rule 0 x 822 [2,5]
+  CRUSH rule 0 x 823 [5,7]
+  CRUSH rule 0 x 824 [5,7]
+  CRUSH rule 0 x 825 [2,8]
+  CRUSH rule 0 x 826 [7,2]
+  CRUSH rule 0 x 827 [2,6]
+  CRUSH rule 0 x 828 [2,5]
+  CRUSH rule 0 x 829 [5,6]
+  CRUSH rule 0 x 830 [2,5]
+  CRUSH rule 0 x 831 [2,6]
+  CRUSH rule 0 x 832 [5,8]
+  CRUSH rule 0 x 833 [2,6]
+  CRUSH rule 0 x 834 [5,2]
+  CRUSH rule 0 x 835 [8,5]
+  CRUSH rule 0 x 836 [5,8]
+  CRUSH rule 0 x 837 [6,5]
+  CRUSH rule 0 x 838 [6,2]
+  CRUSH rule 0 x 839 [5,2]
+  CRUSH rule 0 x 840 [7,5]
+  CRUSH rule 0 x 841 [5,8]
+  CRUSH rule 0 x 842 [2,5]
+  CRUSH rule 0 x 843 [6,5]
+  CRUSH rule 0 x 844 [5,8]
+  CRUSH rule 0 x 845 [5,6]
+  CRUSH rule 0 x 846 [5,2]
+  CRUSH rule 0 x 847 [2,8]
+  CRUSH rule 0 x 848 [2,6]
+  CRUSH rule 0 x 849 [5,8]
+  CRUSH rule 0 x 850 [2,5]
+  CRUSH rule 0 x 851 [6,5]
+  CRUSH rule 0 x 852 [7,5]
+  CRUSH rule 0 x 853 [6,2]
+  CRUSH rule 0 x 854 [7,2]
+  CRUSH rule 0 x 855 [5,7]
+  CRUSH rule 0 x 856 [6,5]
+  CRUSH rule 0 x 857 [8,5]
+  CRUSH rule 0 x 858 [6,5]
+  CRUSH rule 0 x 859 [6,2]
+  CRUSH rule 0 x 860 [5,2]
+  CRUSH rule 0 x 861 [8,5]
+  CRUSH rule 0 x 862 [6,2]
+  CRUSH rule 0 x 863 [8,2]
+  CRUSH rule 0 x 864 [5,6]
+  CRUSH rule 0 x 865 [8,2]
+  CRUSH rule 0 x 866 [5,6]
+  CRUSH rule 0 x 867 [6,5]
+  CRUSH rule 0 x 868 [6,5]
+  CRUSH rule 0 x 869 [8,5]
+  CRUSH rule 0 x 870 [2,5]
+  CRUSH rule 0 x 871 [5,2]
+  CRUSH rule 0 x 872 [5,2]
+  CRUSH rule 0 x 873 [5,6]
+  CRUSH rule 0 x 874 [2,6]
+  CRUSH rule 0 x 875 [2,6]
+  CRUSH rule 0 x 876 [5,8]
+  CRUSH rule 0 x 877 [6,5]
+  CRUSH rule 0 x 878 [5,2]
+  CRUSH rule 0 x 879 [7,5]
+  CRUSH rule 0 x 880 [5,2]
+  CRUSH rule 0 x 881 [5,8]
+  CRUSH rule 0 x 882 [5,2]
+  CRUSH rule 0 x 883 [2,5]
+  CRUSH rule 0 x 884 [6,2]
+  CRUSH rule 0 x 885 [5,2]
+  CRUSH rule 0 x 886 [5,7]
+  CRUSH rule 0 x 887 [7,5]
+  CRUSH rule 0 x 888 [6,2]
+  CRUSH rule 0 x 889 [2,8]
+  CRUSH rule 0 x 890 [7,2]
+  CRUSH rule 0 x 891 [2,7]
+  CRUSH rule 0 x 892 [6,2]
+  CRUSH rule 0 x 893 [2,5]
+  CRUSH rule 0 x 894 [7,5]
+  CRUSH rule 0 x 895 [5,2]
+  CRUSH rule 0 x 896 [2,6]
+  CRUSH rule 0 x 897 [5,2]
+  CRUSH rule 0 x 898 [2,5]
+  CRUSH rule 0 x 899 [2,6]
+  CRUSH rule 0 x 900 [5,2]
+  CRUSH rule 0 x 901 [5,2]
+  CRUSH rule 0 x 902 [8,5]
+  CRUSH rule 0 x 903 [5,6]
+  CRUSH rule 0 x 904 [5,7]
+  CRUSH rule 0 x 905 [6,2]
+  CRUSH rule 0 x 906 [2,7]
+  CRUSH rule 0 x 907 [7,2]
+  CRUSH rule 0 x 908 [5,6]
+  CRUSH rule 0 x 909 [2,5]
+  CRUSH rule 0 x 910 [6,5]
+  CRUSH rule 0 x 911 [5,7]
+  CRUSH rule 0 x 912 [2,8]
+  CRUSH rule 0 x 913 [7,2]
+  CRUSH rule 0 x 914 [6,5]
+  CRUSH rule 0 x 915 [8,2]
+  CRUSH rule 0 x 916 [5,2]
+  CRUSH rule 0 x 917 [2,5]
+  CRUSH rule 0 x 918 [8,2]
+  CRUSH rule 0 x 919 [6,2]
+  CRUSH rule 0 x 920 [7,5]
+  CRUSH rule 0 x 921 [2,5]
+  CRUSH rule 0 x 922 [6,5]
+  CRUSH rule 0 x 923 [5,6]
+  CRUSH rule 0 x 924 [5,2]
+  CRUSH rule 0 x 925 [5,6]
+  CRUSH rule 0 x 926 [5,2]
+  CRUSH rule 0 x 927 [2,6]
+  CRUSH rule 0 x 928 [8,2]
+  CRUSH rule 0 x 929 [5,2]
+  CRUSH rule 0 x 930 [2,5]
+  CRUSH rule 0 x 931 [5,2]
+  CRUSH rule 0 x 932 [5,2]
+  CRUSH rule 0 x 933 [8,5]
+  CRUSH rule 0 x 934 [5,8]
+  CRUSH rule 0 x 935 [6,5]
+  CRUSH rule 0 x 936 [2,7]
+  CRUSH rule 0 x 937 [5,8]
+  CRUSH rule 0 x 938 [6,5]
+  CRUSH rule 0 x 939 [2,8]
+  CRUSH rule 0 x 940 [8,5]
+  CRUSH rule 0 x 941 [5,2]
+  CRUSH rule 0 x 942 [2,6]
+  CRUSH rule 0 x 943 [8,2]
+  CRUSH rule 0 x 944 [5,8]
+  CRUSH rule 0 x 945 [7,2]
+  CRUSH rule 0 x 946 [2,8]
+  CRUSH rule 0 x 947 [5,2]
+  CRUSH rule 0 x 948 [7,5]
+  CRUSH rule 0 x 949 [6,2]
+  CRUSH rule 0 x 950 [5,7]
+  CRUSH rule 0 x 951 [5,6]
+  CRUSH rule 0 x 952 [2,7]
+  CRUSH rule 0 x 953 [2,5]
+  CRUSH rule 0 x 954 [5,2]
+  CRUSH rule 0 x 955 [8,2]
+  CRUSH rule 0 x 956 [2,7]
+  CRUSH rule 0 x 957 [7,2]
+  CRUSH rule 0 x 958 [8,5]
+  CRUSH rule 0 x 959 [5,2]
+  CRUSH rule 0 x 960 [5,6]
+  CRUSH rule 0 x 961 [5,2]
+  CRUSH rule 0 x 962 [7,5]
+  CRUSH rule 0 x 963 [2,5]
+  CRUSH rule 0 x 964 [5,2]
+  CRUSH rule 0 x 965 [7,5]
+  CRUSH rule 0 x 966 [5,6]
+  CRUSH rule 0 x 967 [8,5]
+  CRUSH rule 0 x 968 [7,2]
+  CRUSH rule 0 x 969 [8,2]
+  CRUSH rule 0 x 970 [2,8]
+  CRUSH rule 0 x 971 [2,8]
+  CRUSH rule 0 x 972 [2,7]
+  CRUSH rule 0 x 973 [2,8]
+  CRUSH rule 0 x 974 [5,2]
+  CRUSH rule 0 x 975 [5,8]
+  CRUSH rule 0 x 976 [5,7]
+  CRUSH rule 0 x 977 [8,5]
+  CRUSH rule 0 x 978 [7,2]
+  CRUSH rule 0 x 979 [7,2]
+  CRUSH rule 0 x 980 [6,2]
+  CRUSH rule 0 x 981 [7,5]
+  CRUSH rule 0 x 982 [5,2]
+  CRUSH rule 0 x 983 [5,6]
+  CRUSH rule 0 x 984 [2,8]
+  CRUSH rule 0 x 985 [2,5]
+  CRUSH rule 0 x 986 [8,5]
+  CRUSH rule 0 x 987 [2,5]
+  CRUSH rule 0 x 988 [2,5]
+  CRUSH rule 0 x 989 [2,8]
+  CRUSH rule 0 x 990 [2,6]
+  CRUSH rule 0 x 991 [2,5]
+  CRUSH rule 0 x 992 [7,2]
+  CRUSH rule 0 x 993 [2,6]
+  CRUSH rule 0 x 994 [5,6]
+  CRUSH rule 0 x 995 [7,2]
+  CRUSH rule 0 x 996 [6,5]
+  CRUSH rule 0 x 997 [6,5]
+  CRUSH rule 0 x 998 [8,2]
+  CRUSH rule 0 x 999 [2,7]
+  CRUSH rule 0 x 1000 [8,5]
+  CRUSH rule 0 x 1001 [2,5]
+  CRUSH rule 0 x 1002 [2,5]
+  CRUSH rule 0 x 1003 [2,7]
+  CRUSH rule 0 x 1004 [6,2]
+  CRUSH rule 0 x 1005 [6,2]
+  CRUSH rule 0 x 1006 [2,8]
+  CRUSH rule 0 x 1007 [2,5]
+  CRUSH rule 0 x 1008 [2,7]
+  CRUSH rule 0 x 1009 [6,5]
+  CRUSH rule 0 x 1010 [5,2]
+  CRUSH rule 0 x 1011 [5,2]
+  CRUSH rule 0 x 1012 [5,2]
+  CRUSH rule 0 x 1013 [5,2]
+  CRUSH rule 0 x 1014 [2,8]
+  CRUSH rule 0 x 1015 [6,5]
+  CRUSH rule 0 x 1016 [2,5]
+  CRUSH rule 0 x 1017 [6,2]
+  CRUSH rule 0 x 1018 [5,2]
+  CRUSH rule 0 x 1019 [5,8]
+  CRUSH rule 0 x 1020 [5,2]
+  CRUSH rule 0 x 1021 [5,2]
+  CRUSH rule 0 x 1022 [2,7]
+  CRUSH rule 0 x 1023 [5,2]
+  rule 0 (choose) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [2,5,7]
+  CRUSH rule 0 x 1 [2,8,5]
+  CRUSH rule 0 x 2 [2,5,8]
+  CRUSH rule 0 x 3 [8,2,5]
+  CRUSH rule 0 x 4 [5,2,8]
+  CRUSH rule 0 x 5 [7,2,5]
+  CRUSH rule 0 x 6 [2,6,5]
+  CRUSH rule 0 x 7 [5,6,2]
+  CRUSH rule 0 x 8 [5,7,2]
+  CRUSH rule 0 x 9 [2,5,8]
+  CRUSH rule 0 x 10 [2,8,5]
+  CRUSH rule 0 x 11 [2,6,5]
+  CRUSH rule 0 x 12 [2,5,7]
+  CRUSH rule 0 x 13 [5,8,2]
+  CRUSH rule 0 x 14 [7,2,5]
+  CRUSH rule 0 x 15 [7,2,5]
+  CRUSH rule 0 x 16 [5,7,2]
+  CRUSH rule 0 x 17 [5,2,7]
+  CRUSH rule 0 x 18 [2,5,6]
+  CRUSH rule 0 x 19 [7,5,2]
+  CRUSH rule 0 x 20 [2,5,7]
+  CRUSH rule 0 x 21 [5,6,2]
+  CRUSH rule 0 x 22 [8,5,2]
+  CRUSH rule 0 x 23 [5,7,2]
+  CRUSH rule 0 x 24 [2,6,5]
+  CRUSH rule 0 x 25 [5,8,2]
+  CRUSH rule 0 x 26 [2,7,5]
+  CRUSH rule 0 x 27 [5,2,7]
+  CRUSH rule 0 x 28 [6,2,5]
+  CRUSH rule 0 x 29 [8,5,2]
+  CRUSH rule 0 x 30 [5,6,2]
+  CRUSH rule 0 x 31 [8,2,5]
+  CRUSH rule 0 x 32 [5,7,2]
+  CRUSH rule 0 x 33 [2,7,5]
+  CRUSH rule 0 x 34 [2,5,7]
+  CRUSH rule 0 x 35 [2,6,5]
+  CRUSH rule 0 x 36 [5,6,2]
+  CRUSH rule 0 x 37 [2,5,7]
+  CRUSH rule 0 x 38 [5,6,2]
+  CRUSH rule 0 x 39 [5,7,2]
+  CRUSH rule 0 x 40 [7,2,5]
+  CRUSH rule 0 x 41 [2,7,5]
+  CRUSH rule 0 x 42 [5,7,2]
+  CRUSH rule 0 x 43 [2,5,8]
+  CRUSH rule 0 x 44 [2,8,5]
+  CRUSH rule 0 x 45 [8,2,5]
+  CRUSH rule 0 x 46 [2,5,8]
+  CRUSH rule 0 x 47 [5,2,7]
+  CRUSH rule 0 x 48 [5,6,2]
+  CRUSH rule 0 x 49 [5,6,2]
+  CRUSH rule 0 x 50 [5,2,7]
+  CRUSH rule 0 x 51 [5,6,2]
+  CRUSH rule 0 x 52 [8,2,5]
+  CRUSH rule 0 x 53 [5,6,2]
+  CRUSH rule 0 x 54 [7,5,2]
+  CRUSH rule 0 x 55 [8,2,5]
+  CRUSH rule 0 x 56 [6,5,2]
+  CRUSH rule 0 x 57 [5,8,2]
+  CRUSH rule 0 x 58 [2,8,5]
+  CRUSH rule 0 x 59 [5,2,7]
+  CRUSH rule 0 x 60 [5,2,8]
+  CRUSH rule 0 x 61 [5,8,2]
+  CRUSH rule 0 x 62 [7,2,5]
+  CRUSH rule 0 x 63 [5,6,2]
+  CRUSH rule 0 x 64 [5,2,8]
+  CRUSH rule 0 x 65 [7,5,2]
+  CRUSH rule 0 x 66 [5,6,2]
+  CRUSH rule 0 x 67 [5,2,6]
+  CRUSH rule 0 x 68 [2,5,8]
+  CRUSH rule 0 x 69 [5,2,7]
+  CRUSH rule 0 x 70 [7,2,5]
+  CRUSH rule 0 x 71 [2,7,5]
+  CRUSH rule 0 x 72 [6,2,5]
+  CRUSH rule 0 x 73 [2,7,5]
+  CRUSH rule 0 x 74 [2,8,5]
+  CRUSH rule 0 x 75 [5,2,7]
+  CRUSH rule 0 x 76 [5,2,6]
+  CRUSH rule 0 x 77 [7,2,5]
+  CRUSH rule 0 x 78 [2,5,7]
+  CRUSH rule 0 x 79 [5,2,7]
+  CRUSH rule 0 x 80 [2,5,6]
+  CRUSH rule 0 x 81 [2,5,8]
+  CRUSH rule 0 x 82 [7,2,5]
+  CRUSH rule 0 x 83 [2,6,5]
+  CRUSH rule 0 x 84 [7,2,5]
+  CRUSH rule 0 x 85 [5,7,2]
+  CRUSH rule 0 x 86 [2,6,5]
+  CRUSH rule 0 x 87 [2,6,5]
+  CRUSH rule 0 x 88 [2,8,5]
+  CRUSH rule 0 x 89 [5,2,8]
+  CRUSH rule 0 x 90 [6,5,2]
+  CRUSH rule 0 x 91 [5,6,2]
+  CRUSH rule 0 x 92 [2,6,5]
+  CRUSH rule 0 x 93 [7,5,2]
+  CRUSH rule 0 x 94 [2,5,7]
+  CRUSH rule 0 x 95 [7,5,2]
+  CRUSH rule 0 x 96 [5,8,2]
+  CRUSH rule 0 x 97 [8,5,2]
+  CRUSH rule 0 x 98 [2,7,5]
+  CRUSH rule 0 x 99 [2,8,5]
+  CRUSH rule 0 x 100 [2,6,5]
+  CRUSH rule 0 x 101 [5,8,2]
+  CRUSH rule 0 x 102 [5,2,7]
+  CRUSH rule 0 x 103 [5,7,2]
+  CRUSH rule 0 x 104 [7,5,2]
+  CRUSH rule 0 x 105 [2,5,8]
+  CRUSH rule 0 x 106 [2,6,5]
+  CRUSH rule 0 x 107 [5,2,7]
+  CRUSH rule 0 x 108 [7,2,5]
+  CRUSH rule 0 x 109 [2,5,8]
+  CRUSH rule 0 x 110 [5,2,6]
+  CRUSH rule 0 x 111 [2,5,7]
+  CRUSH rule 0 x 112 [2,6,5]
+  CRUSH rule 0 x 113 [6,2,5]
+  CRUSH rule 0 x 114 [7,5,2]
+  CRUSH rule 0 x 115 [8,2,5]
+  CRUSH rule 0 x 116 [2,7,5]
+  CRUSH rule 0 x 117 [7,5,2]
+  CRUSH rule 0 x 118 [2,5,6]
+  CRUSH rule 0 x 119 [5,7,2]
+  CRUSH rule 0 x 120 [2,5,6]
+  CRUSH rule 0 x 121 [2,8,5]
+  CRUSH rule 0 x 122 [8,5,2]
+  CRUSH rule 0 x 123 [2,5,7]
+  CRUSH rule 0 x 124 [5,2,8]
+  CRUSH rule 0 x 125 [2,7,5]
+  CRUSH rule 0 x 126 [5,2,7]
+  CRUSH rule 0 x 127 [5,6,2]
+  CRUSH rule 0 x 128 [5,8,2]
+  CRUSH rule 0 x 129 [2,5,7]
+  CRUSH rule 0 x 130 [5,8,2]
+  CRUSH rule 0 x 131 [2,5,6]
+  CRUSH rule 0 x 132 [2,5,8]
+  CRUSH rule 0 x 133 [5,6,2]
+  CRUSH rule 0 x 134 [2,7,5]
+  CRUSH rule 0 x 135 [5,7,2]
+  CRUSH rule 0 x 136 [2,5,8]
+  CRUSH rule 0 x 137 [7,5,2]
+  CRUSH rule 0 x 138 [8,5,2]
+  CRUSH rule 0 x 139 [5,2,8]
+  CRUSH rule 0 x 140 [2,8,5]
+  CRUSH rule 0 x 141 [6,2,5]
+  CRUSH rule 0 x 142 [5,2,8]
+  CRUSH rule 0 x 143 [5,7,2]
+  CRUSH rule 0 x 144 [8,2,5]
+  CRUSH rule 0 x 145 [8,5,2]
+  CRUSH rule 0 x 146 [2,8,5]
+  CRUSH rule 0 x 147 [2,6,5]
+  CRUSH rule 0 x 148 [5,2,8]
+  CRUSH rule 0 x 149 [5,6,2]
+  CRUSH rule 0 x 150 [2,8,5]
+  CRUSH rule 0 x 151 [5,8,2]
+  CRUSH rule 0 x 152 [8,5,2]
+  CRUSH rule 0 x 153 [8,5,2]
+  CRUSH rule 0 x 154 [5,2,6]
+  CRUSH rule 0 x 155 [5,6,2]
+  CRUSH rule 0 x 156 [5,2,7]
+  CRUSH rule 0 x 157 [5,2,7]
+  CRUSH rule 0 x 158 [2,6,5]
+  CRUSH rule 0 x 159 [7,2,5]
+  CRUSH rule 0 x 160 [2,8,5]
+  CRUSH rule 0 x 161 [2,5,6]
+  CRUSH rule 0 x 162 [2,8,5]
+  CRUSH rule 0 x 163 [5,8,2]
+  CRUSH rule 0 x 164 [7,2,5]
+  CRUSH rule 0 x 165 [7,2,5]
+  CRUSH rule 0 x 166 [2,5,7]
+  CRUSH rule 0 x 167 [2,6,5]
+  CRUSH rule 0 x 168 [5,2,7]
+  CRUSH rule 0 x 169 [2,7,5]
+  CRUSH rule 0 x 170 [2,5,8]
+  CRUSH rule 0 x 171 [7,5,2]
+  CRUSH rule 0 x 172 [2,8,5]
+  CRUSH rule 0 x 173 [8,5,2]
+  CRUSH rule 0 x 174 [2,5,8]
+  CRUSH rule 0 x 175 [6,2,5]
+  CRUSH rule 0 x 176 [5,2,7]
+  CRUSH rule 0 x 177 [5,2,8]
+  CRUSH rule 0 x 178 [5,2,6]
+  CRUSH rule 0 x 179 [5,2,7]
+  CRUSH rule 0 x 180 [5,7,2]
+  CRUSH rule 0 x 181 [6,2,5]
+  CRUSH rule 0 x 182 [8,5,2]
+  CRUSH rule 0 x 183 [7,5,2]
+  CRUSH rule 0 x 184 [5,6,2]
+  CRUSH rule 0 x 185 [6,2,5]
+  CRUSH rule 0 x 186 [2,5,8]
+  CRUSH rule 0 x 187 [2,6,5]
+  CRUSH rule 0 x 188 [2,6,5]
+  CRUSH rule 0 x 189 [2,6,5]
+  CRUSH rule 0 x 190 [5,2,8]
+  CRUSH rule 0 x 191 [7,2,5]
+  CRUSH rule 0 x 192 [5,2,6]
+  CRUSH rule 0 x 193 [5,2,6]
+  CRUSH rule 0 x 194 [2,5,7]
+  CRUSH rule 0 x 195 [6,5,2]
+  CRUSH rule 0 x 196 [6,2,5]
+  CRUSH rule 0 x 197 [6,5,2]
+  CRUSH rule 0 x 198 [2,5,6]
+  CRUSH rule 0 x 199 [2,5,7]
+  CRUSH rule 0 x 200 [2,5,8]
+  CRUSH rule 0 x 201 [7,2,5]
+  CRUSH rule 0 x 202 [6,5,2]
+  CRUSH rule 0 x 203 [5,6,2]
+  CRUSH rule 0 x 204 [2,5,8]
+  CRUSH rule 0 x 205 [2,7,5]
+  CRUSH rule 0 x 206 [2,8,5]
+  CRUSH rule 0 x 207 [5,2,7]
+  CRUSH rule 0 x 208 [7,2,5]
+  CRUSH rule 0 x 209 [2,8,5]
+  CRUSH rule 0 x 210 [2,5,6]
+  CRUSH rule 0 x 211 [5,2,7]
+  CRUSH rule 0 x 212 [7,5,2]
+  CRUSH rule 0 x 213 [8,5,2]
+  CRUSH rule 0 x 214 [5,7,2]
+  CRUSH rule 0 x 215 [8,2,5]
+  CRUSH rule 0 x 216 [5,2,8]
+  CRUSH rule 0 x 217 [2,7,5]
+  CRUSH rule 0 x 218 [2,7,5]
+  CRUSH rule 0 x 219 [5,6,2]
+  CRUSH rule 0 x 220 [5,8,2]
+  CRUSH rule 0 x 221 [5,7,2]
+  CRUSH rule 0 x 222 [6,5,2]
+  CRUSH rule 0 x 223 [2,5,6]
+  CRUSH rule 0 x 224 [2,5,8]
+  CRUSH rule 0 x 225 [8,2,5]
+  CRUSH rule 0 x 226 [7,2,5]
+  CRUSH rule 0 x 227 [5,2,7]
+  CRUSH rule 0 x 228 [5,6,2]
+  CRUSH rule 0 x 229 [5,7,2]
+  CRUSH rule 0 x 230 [5,6,2]
+  CRUSH rule 0 x 231 [5,7,2]
+  CRUSH rule 0 x 232 [2,8,5]
+  CRUSH rule 0 x 233 [5,6,2]
+  CRUSH rule 0 x 234 [2,5,6]
+  CRUSH rule 0 x 235 [5,6,2]
+  CRUSH rule 0 x 236 [5,2,8]
+  CRUSH rule 0 x 237 [5,8,2]
+  CRUSH rule 0 x 238 [5,2,6]
+  CRUSH rule 0 x 239 [8,5,2]
+  CRUSH rule 0 x 240 [5,8,2]
+  CRUSH rule 0 x 241 [5,2,7]
+  CRUSH rule 0 x 242 [5,2,6]
+  CRUSH rule 0 x 243 [5,8,2]
+  CRUSH rule 0 x 244 [5,6,2]
+  CRUSH rule 0 x 245 [7,2,5]
+  CRUSH rule 0 x 246 [2,5,7]
+  CRUSH rule 0 x 247 [6,2,5]
+  CRUSH rule 0 x 248 [8,2,5]
+  CRUSH rule 0 x 249 [2,5,8]
+  CRUSH rule 0 x 250 [2,5,8]
+  CRUSH rule 0 x 251 [2,5,7]
+  CRUSH rule 0 x 252 [5,7,2]
+  CRUSH rule 0 x 253 [5,2,7]
+  CRUSH rule 0 x 254 [5,2,8]
+  CRUSH rule 0 x 255 [2,7,5]
+  CRUSH rule 0 x 256 [5,6,2]
+  CRUSH rule 0 x 257 [2,6,5]
+  CRUSH rule 0 x 258 [5,2,8]
+  CRUSH rule 0 x 259 [5,6,2]
+  CRUSH rule 0 x 260 [5,8,2]
+  CRUSH rule 0 x 261 [8,5,2]
+  CRUSH rule 0 x 262 [5,6,2]
+  CRUSH rule 0 x 263 [6,2,5]
+  CRUSH rule 0 x 264 [5,6,2]
+  CRUSH rule 0 x 265 [8,5,2]
+  CRUSH rule 0 x 266 [8,2,5]
+  CRUSH rule 0 x 267 [2,5,6]
+  CRUSH rule 0 x 268 [2,6,5]
+  CRUSH rule 0 x 269 [2,6,5]
+  CRUSH rule 0 x 270 [5,2,8]
+  CRUSH rule 0 x 271 [7,5,2]
+  CRUSH rule 0 x 272 [2,6,5]
+  CRUSH rule 0 x 273 [5,2,7]
+  CRUSH rule 0 x 274 [6,5,2]
+  CRUSH rule 0 x 275 [5,8,2]
+  CRUSH rule 0 x 276 [7,2,5]
+  CRUSH rule 0 x 277 [6,5,2]
+  CRUSH rule 0 x 278 [6,2,5]
+  CRUSH rule 0 x 279 [8,5,2]
+  CRUSH rule 0 x 280 [2,7,5]
+  CRUSH rule 0 x 281 [8,2,5]
+  CRUSH rule 0 x 282 [5,2,8]
+  CRUSH rule 0 x 283 [8,2,5]
+  CRUSH rule 0 x 284 [6,5,2]
+  CRUSH rule 0 x 285 [5,7,2]
+  CRUSH rule 0 x 286 [2,8,5]
+  CRUSH rule 0 x 287 [2,5,8]
+  CRUSH rule 0 x 288 [8,2,5]
+  CRUSH rule 0 x 289 [5,6,2]
+  CRUSH rule 0 x 290 [2,5,6]
+  CRUSH rule 0 x 291 [2,5,8]
+  CRUSH rule 0 x 292 [8,2,5]
+  CRUSH rule 0 x 293 [6,2,5]
+  CRUSH rule 0 x 294 [7,5,2]
+  CRUSH rule 0 x 295 [5,6,2]
+  CRUSH rule 0 x 296 [5,2,6]
+  CRUSH rule 0 x 297 [6,2,5]
+  CRUSH rule 0 x 298 [2,5,6]
+  CRUSH rule 0 x 299 [2,8,5]
+  CRUSH rule 0 x 300 [8,5,2]
+  CRUSH rule 0 x 301 [2,7,5]
+  CRUSH rule 0 x 302 [5,2,6]
+  CRUSH rule 0 x 303 [7,5,2]
+  CRUSH rule 0 x 304 [2,6,5]
+  CRUSH rule 0 x 305 [5,6,2]
+  CRUSH rule 0 x 306 [2,8,5]
+  CRUSH rule 0 x 307 [2,7,5]
+  CRUSH rule 0 x 308 [2,8,5]
+  CRUSH rule 0 x 309 [7,5,2]
+  CRUSH rule 0 x 310 [5,2,6]
+  CRUSH rule 0 x 311 [5,8,2]
+  CRUSH rule 0 x 312 [2,6,5]
+  CRUSH rule 0 x 313 [5,2,7]
+  CRUSH rule 0 x 314 [5,2,6]
+  CRUSH rule 0 x 315 [2,5,7]
+  CRUSH rule 0 x 316 [6,5,2]
+  CRUSH rule 0 x 317 [2,7,5]
+  CRUSH rule 0 x 318 [8,2,5]
+  CRUSH rule 0 x 319 [5,2,6]
+  CRUSH rule 0 x 320 [5,8,2]
+  CRUSH rule 0 x 321 [2,5,8]
+  CRUSH rule 0 x 322 [2,6,5]
+  CRUSH rule 0 x 323 [5,7,2]
+  CRUSH rule 0 x 324 [7,2,5]
+  CRUSH rule 0 x 325 [5,6,2]
+  CRUSH rule 0 x 326 [5,2,7]
+  CRUSH rule 0 x 327 [2,8,5]
+  CRUSH rule 0 x 328 [7,5,2]
+  CRUSH rule 0 x 329 [5,7,2]
+  CRUSH rule 0 x 330 [5,7,2]
+  CRUSH rule 0 x 331 [2,7,5]
+  CRUSH rule 0 x 332 [2,5,8]
+  CRUSH rule 0 x 333 [6,5,2]
+  CRUSH rule 0 x 334 [8,5,2]
+  CRUSH rule 0 x 335 [7,2,5]
+  CRUSH rule 0 x 336 [5,6,2]
+  CRUSH rule 0 x 337 [7,2,5]
+  CRUSH rule 0 x 338 [5,8,2]
+  CRUSH rule 0 x 339 [7,5,2]
+  CRUSH rule 0 x 340 [2,6,5]
+  CRUSH rule 0 x 341 [5,2,7]
+  CRUSH rule 0 x 342 [2,8,5]
+  CRUSH rule 0 x 343 [6,5,2]
+  CRUSH rule 0 x 344 [6,2,5]
+  CRUSH rule 0 x 345 [5,7,2]
+  CRUSH rule 0 x 346 [8,2,5]
+  CRUSH rule 0 x 347 [5,2,8]
+  CRUSH rule 0 x 348 [8,2,5]
+  CRUSH rule 0 x 349 [2,7,5]
+  CRUSH rule 0 x 350 [8,5,2]
+  CRUSH rule 0 x 351 [5,8,2]
+  CRUSH rule 0 x 352 [2,8,5]
+  CRUSH rule 0 x 353 [6,5,2]
+  CRUSH rule 0 x 354 [2,5,6]
+  CRUSH rule 0 x 355 [5,8,2]
+  CRUSH rule 0 x 356 [5,2,8]
+  CRUSH rule 0 x 357 [6,2,5]
+  CRUSH rule 0 x 358 [2,8,5]
+  CRUSH rule 0 x 359 [6,2,5]
+  CRUSH rule 0 x 360 [5,2,8]
+  CRUSH rule 0 x 361 [8,5,2]
+  CRUSH rule 0 x 362 [5,2,8]
+  CRUSH rule 0 x 363 [5,2,8]
+  CRUSH rule 0 x 364 [2,5,7]
+  CRUSH rule 0 x 365 [6,5,2]
+  CRUSH rule 0 x 366 [7,2,5]
+  CRUSH rule 0 x 367 [5,2,7]
+  CRUSH rule 0 x 368 [7,5,2]
+  CRUSH rule 0 x 369 [5,7,2]
+  CRUSH rule 0 x 370 [8,2,5]
+  CRUSH rule 0 x 371 [2,5,8]
+  CRUSH rule 0 x 372 [5,2,8]
+  CRUSH rule 0 x 373 [2,6,5]
+  CRUSH rule 0 x 374 [5,8,2]
+  CRUSH rule 0 x 375 [6,5,2]
+  CRUSH rule 0 x 376 [7,2,5]
+  CRUSH rule 0 x 377 [2,5,7]
+  CRUSH rule 0 x 378 [2,6,5]
+  CRUSH rule 0 x 379 [8,5,2]
+  CRUSH rule 0 x 380 [2,5,8]
+  CRUSH rule 0 x 381 [2,5,7]
+  CRUSH rule 0 x 382 [2,5,7]
+  CRUSH rule 0 x 383 [5,7,2]
+  CRUSH rule 0 x 384 [7,2,5]
+  CRUSH rule 0 x 385 [7,5,2]
+  CRUSH rule 0 x 386 [2,5,6]
+  CRUSH rule 0 x 387 [2,5,8]
+  CRUSH rule 0 x 388 [5,2,7]
+  CRUSH rule 0 x 389 [2,5,8]
+  CRUSH rule 0 x 390 [5,8,2]
+  CRUSH rule 0 x 391 [5,6,2]
+  CRUSH rule 0 x 392 [2,7,5]
+  CRUSH rule 0 x 393 [5,2,6]
+  CRUSH rule 0 x 394 [5,8,2]
+  CRUSH rule 0 x 395 [5,2,8]
+  CRUSH rule 0 x 396 [5,2,6]
+  CRUSH rule 0 x 397 [2,5,7]
+  CRUSH rule 0 x 398 [2,5,6]
+  CRUSH rule 0 x 399 [8,5,2]
+  CRUSH rule 0 x 400 [8,2,5]
+  CRUSH rule 0 x 401 [2,5,6]
+  CRUSH rule 0 x 402 [7,5,2]
+  CRUSH rule 0 x 403 [2,5,7]
+  CRUSH rule 0 x 404 [5,2,6]
+  CRUSH rule 0 x 405 [6,5,2]
+  CRUSH rule 0 x 406 [2,6,5]
+  CRUSH rule 0 x 407 [2,7,5]
+  CRUSH rule 0 x 408 [5,2,7]
+  CRUSH rule 0 x 409 [7,5,2]
+  CRUSH rule 0 x 410 [8,5,2]
+  CRUSH rule 0 x 411 [2,7,5]
+  CRUSH rule 0 x 412 [2,5,7]
+  CRUSH rule 0 x 413 [5,2,8]
+  CRUSH rule 0 x 414 [5,2,8]
+  CRUSH rule 0 x 415 [2,6,5]
+  CRUSH rule 0 x 416 [2,8,5]
+  CRUSH rule 0 x 417 [8,2,5]
+  CRUSH rule 0 x 418 [7,2,5]
+  CRUSH rule 0 x 419 [8,5,2]
+  CRUSH rule 0 x 420 [2,5,7]
+  CRUSH rule 0 x 421 [8,5,2]
+  CRUSH rule 0 x 422 [6,5,2]
+  CRUSH rule 0 x 423 [2,5,7]
+  CRUSH rule 0 x 424 [8,5,2]
+  CRUSH rule 0 x 425 [2,5,8]
+  CRUSH rule 0 x 426 [6,2,5]
+  CRUSH rule 0 x 427 [2,8,5]
+  CRUSH rule 0 x 428 [5,8,2]
+  CRUSH rule 0 x 429 [5,8,2]
+  CRUSH rule 0 x 430 [5,7,2]
+  CRUSH rule 0 x 431 [5,2,7]
+  CRUSH rule 0 x 432 [7,2,5]
+  CRUSH rule 0 x 433 [6,5,2]
+  CRUSH rule 0 x 434 [5,2,7]
+  CRUSH rule 0 x 435 [2,5,6]
+  CRUSH rule 0 x 436 [5,2,7]
+  CRUSH rule 0 x 437 [7,5,2]
+  CRUSH rule 0 x 438 [2,5,8]
+  CRUSH rule 0 x 439 [2,5,8]
+  CRUSH rule 0 x 440 [2,6,5]
+  CRUSH rule 0 x 441 [5,8,2]
+  CRUSH rule 0 x 442 [2,5,6]
+  CRUSH rule 0 x 443 [6,2,5]
+  CRUSH rule 0 x 444 [7,2,5]
+  CRUSH rule 0 x 445 [6,5,2]
+  CRUSH rule 0 x 446 [5,2,8]
+  CRUSH rule 0 x 447 [2,5,6]
+  CRUSH rule 0 x 448 [7,2,5]
+  CRUSH rule 0 x 449 [7,5,2]
+  CRUSH rule 0 x 450 [5,2,6]
+  CRUSH rule 0 x 451 [6,5,2]
+  CRUSH rule 0 x 452 [8,5,2]
+  CRUSH rule 0 x 453 [6,5,2]
+  CRUSH rule 0 x 454 [6,5,2]
+  CRUSH rule 0 x 455 [2,8,5]
+  CRUSH rule 0 x 456 [6,2,5]
+  CRUSH rule 0 x 457 [7,2,5]
+  CRUSH rule 0 x 458 [2,8,5]
+  CRUSH rule 0 x 459 [2,6,5]
+  CRUSH rule 0 x 460 [6,5,2]
+  CRUSH rule 0 x 461 [6,5,2]
+  CRUSH rule 0 x 462 [8,2,5]
+  CRUSH rule 0 x 463 [6,2,5]
+  CRUSH rule 0 x 464 [7,5,2]
+  CRUSH rule 0 x 465 [7,2,5]
+  CRUSH rule 0 x 466 [5,6,2]
+  CRUSH rule 0 x 467 [6,5,2]
+  CRUSH rule 0 x 468 [7,2,5]
+  CRUSH rule 0 x 469 [7,2,5]
+  CRUSH rule 0 x 470 [5,2,8]
+  CRUSH rule 0 x 471 [2,6,5]
+  CRUSH rule 0 x 472 [5,2,8]
+  CRUSH rule 0 x 473 [2,5,7]
+  CRUSH rule 0 x 474 [6,2,5]
+  CRUSH rule 0 x 475 [6,2,5]
+  CRUSH rule 0 x 476 [5,7,2]
+  CRUSH rule 0 x 477 [5,6,2]
+  CRUSH rule 0 x 478 [6,2,5]
+  CRUSH rule 0 x 479 [2,5,6]
+  CRUSH rule 0 x 480 [2,6,5]
+  CRUSH rule 0 x 481 [2,5,7]
+  CRUSH rule 0 x 482 [5,8,2]
+  CRUSH rule 0 x 483 [2,6,5]
+  CRUSH rule 0 x 484 [2,8,5]
+  CRUSH rule 0 x 485 [5,8,2]
+  CRUSH rule 0 x 486 [5,2,8]
+  CRUSH rule 0 x 487 [5,2,8]
+  CRUSH rule 0 x 488 [5,7,2]
+  CRUSH rule 0 x 489 [2,8,5]
+  CRUSH rule 0 x 490 [6,5,2]
+  CRUSH rule 0 x 491 [2,6,5]
+  CRUSH rule 0 x 492 [6,5,2]
+  CRUSH rule 0 x 493 [2,8,5]
+  CRUSH rule 0 x 494 [2,6,5]
+  CRUSH rule 0 x 495 [5,2,6]
+  CRUSH rule 0 x 496 [7,5,2]
+  CRUSH rule 0 x 497 [5,7,2]
+  CRUSH rule 0 x 498 [2,5,6]
+  CRUSH rule 0 x 499 [8,5,2]
+  CRUSH rule 0 x 500 [5,6,2]
+  CRUSH rule 0 x 501 [2,7,5]
+  CRUSH rule 0 x 502 [7,2,5]
+  CRUSH rule 0 x 503 [2,5,7]
+  CRUSH rule 0 x 504 [5,8,2]
+  CRUSH rule 0 x 505 [2,7,5]
+  CRUSH rule 0 x 506 [5,2,7]
+  CRUSH rule 0 x 507 [6,2,5]
+  CRUSH rule 0 x 508 [2,5,7]
+  CRUSH rule 0 x 509 [7,5,2]
+  CRUSH rule 0 x 510 [6,2,5]
+  CRUSH rule 0 x 511 [5,6,2]
+  CRUSH rule 0 x 512 [7,2,5]
+  CRUSH rule 0 x 513 [7,2,5]
+  CRUSH rule 0 x 514 [5,7,2]
+  CRUSH rule 0 x 515 [8,5,2]
+  CRUSH rule 0 x 516 [5,2,7]
+  CRUSH rule 0 x 517 [7,2,5]
+  CRUSH rule 0 x 518 [5,6,2]
+  CRUSH rule 0 x 519 [7,5,2]
+  CRUSH rule 0 x 520 [2,8,5]
+  CRUSH rule 0 x 521 [8,2,5]
+  CRUSH rule 0 x 522 [6,2,5]
+  CRUSH rule 0 x 523 [5,2,7]
+  CRUSH rule 0 x 524 [2,5,7]
+  CRUSH rule 0 x 525 [2,5,8]
+  CRUSH rule 0 x 526 [2,5,8]
+  CRUSH rule 0 x 527 [2,5,6]
+  CRUSH rule 0 x 528 [5,2,7]
+  CRUSH rule 0 x 529 [5,6,2]
+  CRUSH rule 0 x 530 [6,5,2]
+  CRUSH rule 0 x 531 [6,2,5]
+  CRUSH rule 0 x 532 [6,5,2]
+  CRUSH rule 0 x 533 [5,8,2]
+  CRUSH rule 0 x 534 [7,5,2]
+  CRUSH rule 0 x 535 [8,2,5]
+  CRUSH rule 0 x 536 [6,2,5]
+  CRUSH rule 0 x 537 [5,8,2]
+  CRUSH rule 0 x 538 [6,5,2]
+  CRUSH rule 0 x 539 [8,5,2]
+  CRUSH rule 0 x 540 [2,7,5]
+  CRUSH rule 0 x 541 [2,5,7]
+  CRUSH rule 0 x 542 [5,2,8]
+  CRUSH rule 0 x 543 [6,2,5]
+  CRUSH rule 0 x 544 [5,7,2]
+  CRUSH rule 0 x 545 [5,7,2]
+  CRUSH rule 0 x 546 [6,2,5]
+  CRUSH rule 0 x 547 [8,2,5]
+  CRUSH rule 0 x 548 [5,2,8]
+  CRUSH rule 0 x 549 [5,7,2]
+  CRUSH rule 0 x 550 [2,5,6]
+  CRUSH rule 0 x 551 [7,5,2]
+  CRUSH rule 0 x 552 [5,7,2]
+  CRUSH rule 0 x 553 [5,2,8]
+  CRUSH rule 0 x 554 [2,6,5]
+  CRUSH rule 0 x 555 [5,2,8]
+  CRUSH rule 0 x 556 [5,6,2]
+  CRUSH rule 0 x 557 [7,5,2]
+  CRUSH rule 0 x 558 [5,2,6]
+  CRUSH rule 0 x 559 [5,2,8]
+  CRUSH rule 0 x 560 [8,5,2]
+  CRUSH rule 0 x 561 [6,5,2]
+  CRUSH rule 0 x 562 [5,2,8]
+  CRUSH rule 0 x 563 [2,7,5]
+  CRUSH rule 0 x 564 [5,2,7]
+  CRUSH rule 0 x 565 [5,8,2]
+  CRUSH rule 0 x 566 [5,6,2]
+  CRUSH rule 0 x 567 [5,7,2]
+  CRUSH rule 0 x 568 [7,5,2]
+  CRUSH rule 0 x 569 [5,2,6]
+  CRUSH rule 0 x 570 [2,5,7]
+  CRUSH rule 0 x 571 [5,6,2]
+  CRUSH rule 0 x 572 [5,2,7]
+  CRUSH rule 0 x 573 [5,2,7]
+  CRUSH rule 0 x 574 [2,5,6]
+  CRUSH rule 0 x 575 [8,2,5]
+  CRUSH rule 0 x 576 [5,7,2]
+  CRUSH rule 0 x 577 [8,2,5]
+  CRUSH rule 0 x 578 [6,2,5]
+  CRUSH rule 0 x 579 [5,2,6]
+  CRUSH rule 0 x 580 [5,2,8]
+  CRUSH rule 0 x 581 [7,2,5]
+  CRUSH rule 0 x 582 [2,8,5]
+  CRUSH rule 0 x 583 [6,2,5]
+  CRUSH rule 0 x 584 [8,2,5]
+  CRUSH rule 0 x 585 [7,2,5]
+  CRUSH rule 0 x 586 [2,8,5]
+  CRUSH rule 0 x 587 [2,5,6]
+  CRUSH rule 0 x 588 [5,8,2]
+  CRUSH rule 0 x 589 [7,2,5]
+  CRUSH rule 0 x 590 [6,2,5]
+  CRUSH rule 0 x 591 [5,2,7]
+  CRUSH rule 0 x 592 [2,5,6]
+  CRUSH rule 0 x 593 [2,8,5]
+  CRUSH rule 0 x 594 [2,6,5]
+  CRUSH rule 0 x 595 [7,2,5]
+  CRUSH rule 0 x 596 [5,2,6]
+  CRUSH rule 0 x 597 [5,2,7]
+  CRUSH rule 0 x 598 [5,2,7]
+  CRUSH rule 0 x 599 [5,2,6]
+  CRUSH rule 0 x 600 [7,2,5]
+  CRUSH rule 0 x 601 [2,6,5]
+  CRUSH rule 0 x 602 [5,8,2]
+  CRUSH rule 0 x 603 [5,2,8]
+  CRUSH rule 0 x 604 [7,5,2]
+  CRUSH rule 0 x 605 [5,2,6]
+  CRUSH rule 0 x 606 [2,7,5]
+  CRUSH rule 0 x 607 [2,5,8]
+  CRUSH rule 0 x 608 [5,2,8]
+  CRUSH rule 0 x 609 [5,2,6]
+  CRUSH rule 0 x 610 [5,8,2]
+  CRUSH rule 0 x 611 [2,8,5]
+  CRUSH rule 0 x 612 [2,8,5]
+  CRUSH rule 0 x 613 [7,2,5]
+  CRUSH rule 0 x 614 [7,2,5]
+  CRUSH rule 0 x 615 [6,2,5]
+  CRUSH rule 0 x 616 [2,7,5]
+  CRUSH rule 0 x 617 [6,2,5]
+  CRUSH rule 0 x 618 [7,5,2]
+  CRUSH rule 0 x 619 [5,2,6]
+  CRUSH rule 0 x 620 [5,2,8]
+  CRUSH rule 0 x 621 [5,6,2]
+  CRUSH rule 0 x 622 [2,5,8]
+  CRUSH rule 0 x 623 [2,8,5]
+  CRUSH rule 0 x 624 [5,2,8]
+  CRUSH rule 0 x 625 [2,5,8]
+  CRUSH rule 0 x 626 [7,2,5]
+  CRUSH rule 0 x 627 [2,6,5]
+  CRUSH rule 0 x 628 [8,2,5]
+  CRUSH rule 0 x 629 [2,6,5]
+  CRUSH rule 0 x 630 [2,6,5]
+  CRUSH rule 0 x 631 [2,6,5]
+  CRUSH rule 0 x 632 [7,2,5]
+  CRUSH rule 0 x 633 [8,5,2]
+  CRUSH rule 0 x 634 [2,5,6]
+  CRUSH rule 0 x 635 [5,6,2]
+  CRUSH rule 0 x 636 [2,5,7]
+  CRUSH rule 0 x 637 [5,2,8]
+  CRUSH rule 0 x 638 [6,2,5]
+  CRUSH rule 0 x 639 [5,2,6]
+  CRUSH rule 0 x 640 [5,2,7]
+  CRUSH rule 0 x 641 [7,2,5]
+  CRUSH rule 0 x 642 [2,8,5]
+  CRUSH rule 0 x 643 [5,2,8]
+  CRUSH rule 0 x 644 [8,2,5]
+  CRUSH rule 0 x 645 [5,7,2]
+  CRUSH rule 0 x 646 [8,2,5]
+  CRUSH rule 0 x 647 [7,2,5]
+  CRUSH rule 0 x 648 [2,8,5]
+  CRUSH rule 0 x 649 [5,7,2]
+  CRUSH rule 0 x 650 [7,5,2]
+  CRUSH rule 0 x 651 [5,6,2]
+  CRUSH rule 0 x 652 [5,6,2]
+  CRUSH rule 0 x 653 [8,5,2]
+  CRUSH rule 0 x 654 [7,5,2]
+  CRUSH rule 0 x 655 [2,5,6]
+  CRUSH rule 0 x 656 [5,7,2]
+  CRUSH rule 0 x 657 [6,2,5]
+  CRUSH rule 0 x 658 [5,8,2]
+  CRUSH rule 0 x 659 [5,7,2]
+  CRUSH rule 0 x 660 [7,5,2]
+  CRUSH rule 0 x 661 [2,7,5]
+  CRUSH rule 0 x 662 [5,2,8]
+  CRUSH rule 0 x 663 [2,5,7]
+  CRUSH rule 0 x 664 [2,5,6]
+  CRUSH rule 0 x 665 [5,6,2]
+  CRUSH rule 0 x 666 [2,7,5]
+  CRUSH rule 0 x 667 [2,5,8]
+  CRUSH rule 0 x 668 [5,7,2]
+  CRUSH rule 0 x 669 [6,5,2]
+  CRUSH rule 0 x 670 [5,2,6]
+  CRUSH rule 0 x 671 [2,8,5]
+  CRUSH rule 0 x 672 [5,2,8]
+  CRUSH rule 0 x 673 [5,2,7]
+  CRUSH rule 0 x 674 [5,2,7]
+  CRUSH rule 0 x 675 [2,8,5]
+  CRUSH rule 0 x 676 [2,5,7]
+  CRUSH rule 0 x 677 [5,2,6]
+  CRUSH rule 0 x 678 [2,5,8]
+  CRUSH rule 0 x 679 [6,2,5]
+  CRUSH rule 0 x 680 [2,5,8]
+  CRUSH rule 0 x 681 [5,6,2]
+  CRUSH rule 0 x 682 [2,5,8]
+  CRUSH rule 0 x 683 [2,5,8]
+  CRUSH rule 0 x 684 [7,2,5]
+  CRUSH rule 0 x 685 [7,2,5]
+  CRUSH rule 0 x 686 [2,5,8]
+  CRUSH rule 0 x 687 [5,7,2]
+  CRUSH rule 0 x 688 [5,6,2]
+  CRUSH rule 0 x 689 [6,5,2]
+  CRUSH rule 0 x 690 [8,2,5]
+  CRUSH rule 0 x 691 [5,2,6]
+  CRUSH rule 0 x 692 [7,2,5]
+  CRUSH rule 0 x 693 [6,5,2]
+  CRUSH rule 0 x 694 [6,5,2]
+  CRUSH rule 0 x 695 [2,6,5]
+  CRUSH rule 0 x 696 [2,5,8]
+  CRUSH rule 0 x 697 [6,2,5]
+  CRUSH rule 0 x 698 [6,2,5]
+  CRUSH rule 0 x 699 [2,8,5]
+  CRUSH rule 0 x 700 [2,5,6]
+  CRUSH rule 0 x 701 [5,2,6]
+  CRUSH rule 0 x 702 [5,2,8]
+  CRUSH rule 0 x 703 [8,5,2]
+  CRUSH rule 0 x 704 [2,5,6]
+  CRUSH rule 0 x 705 [8,2,5]
+  CRUSH rule 0 x 706 [2,5,8]
+  CRUSH rule 0 x 707 [7,5,2]
+  CRUSH rule 0 x 708 [5,7,2]
+  CRUSH rule 0 x 709 [6,5,2]
+  CRUSH rule 0 x 710 [8,5,2]
+  CRUSH rule 0 x 711 [2,5,7]
+  CRUSH rule 0 x 712 [2,5,6]
+  CRUSH rule 0 x 713 [6,5,2]
+  CRUSH rule 0 x 714 [5,2,6]
+  CRUSH rule 0 x 715 [2,5,6]
+  CRUSH rule 0 x 716 [5,6,2]
+  CRUSH rule 0 x 717 [8,2,5]
+  CRUSH rule 0 x 718 [5,7,2]
+  CRUSH rule 0 x 719 [2,6,5]
+  CRUSH rule 0 x 720 [6,2,5]
+  CRUSH rule 0 x 721 [5,7,2]
+  CRUSH rule 0 x 722 [5,7,2]
+  CRUSH rule 0 x 723 [5,2,7]
+  CRUSH rule 0 x 724 [2,7,5]
+  CRUSH rule 0 x 725 [2,5,6]
+  CRUSH rule 0 x 726 [5,7,2]
+  CRUSH rule 0 x 727 [5,7,2]
+  CRUSH rule 0 x 728 [2,6,5]
+  CRUSH rule 0 x 729 [5,6,2]
+  CRUSH rule 0 x 730 [5,8,2]
+  CRUSH rule 0 x 731 [5,2,6]
+  CRUSH rule 0 x 732 [2,5,8]
+  CRUSH rule 0 x 733 [5,6,2]
+  CRUSH rule 0 x 734 [6,5,2]
+  CRUSH rule 0 x 735 [5,6,2]
+  CRUSH rule 0 x 736 [5,8,2]
+  CRUSH rule 0 x 737 [2,8,5]
+  CRUSH rule 0 x 738 [5,2,6]
+  CRUSH rule 0 x 739 [2,7,5]
+  CRUSH rule 0 x 740 [2,7,5]
+  CRUSH rule 0 x 741 [7,2,5]
+  CRUSH rule 0 x 742 [8,2,5]
+  CRUSH rule 0 x 743 [7,2,5]
+  CRUSH rule 0 x 744 [5,6,2]
+  CRUSH rule 0 x 745 [5,2,8]
+  CRUSH rule 0 x 746 [5,2,7]
+  CRUSH rule 0 x 747 [6,2,5]
+  CRUSH rule 0 x 748 [2,6,5]
+  CRUSH rule 0 x 749 [5,7,2]
+  CRUSH rule 0 x 750 [2,7,5]
+  CRUSH rule 0 x 751 [2,7,5]
+  CRUSH rule 0 x 752 [8,2,5]
+  CRUSH rule 0 x 753 [7,5,2]
+  CRUSH rule 0 x 754 [8,5,2]
+  CRUSH rule 0 x 755 [2,6,5]
+  CRUSH rule 0 x 756 [5,8,2]
+  CRUSH rule 0 x 757 [8,2,5]
+  CRUSH rule 0 x 758 [6,2,5]
+  CRUSH rule 0 x 759 [8,5,2]
+  CRUSH rule 0 x 760 [2,5,7]
+  CRUSH rule 0 x 761 [5,2,6]
+  CRUSH rule 0 x 762 [2,8,5]
+  CRUSH rule 0 x 763 [8,5,2]
+  CRUSH rule 0 x 764 [2,7,5]
+  CRUSH rule 0 x 765 [6,5,2]
+  CRUSH rule 0 x 766 [8,5,2]
+  CRUSH rule 0 x 767 [2,8,5]
+  CRUSH rule 0 x 768 [8,5,2]
+  CRUSH rule 0 x 769 [6,2,5]
+  CRUSH rule 0 x 770 [6,2,5]
+  CRUSH rule 0 x 771 [7,2,5]
+  CRUSH rule 0 x 772 [8,5,2]
+  CRUSH rule 0 x 773 [5,2,6]
+  CRUSH rule 0 x 774 [5,7,2]
+  CRUSH rule 0 x 775 [6,5,2]
+  CRUSH rule 0 x 776 [7,2,5]
+  CRUSH rule 0 x 777 [5,2,6]
+  CRUSH rule 0 x 778 [2,6,5]
+  CRUSH rule 0 x 779 [2,6,5]
+  CRUSH rule 0 x 780 [2,5,8]
+  CRUSH rule 0 x 781 [6,5,2]
+  CRUSH rule 0 x 782 [5,2,8]
+  CRUSH rule 0 x 783 [7,2,5]
+  CRUSH rule 0 x 784 [2,5,8]
+  CRUSH rule 0 x 785 [6,2,5]
+  CRUSH rule 0 x 786 [7,5,2]
+  CRUSH rule 0 x 787 [2,8,5]
+  CRUSH rule 0 x 788 [6,2,5]
+  CRUSH rule 0 x 789 [2,5,8]
+  CRUSH rule 0 x 790 [8,5,2]
+  CRUSH rule 0 x 791 [5,6,2]
+  CRUSH rule 0 x 792 [5,6,2]
+  CRUSH rule 0 x 793 [6,2,5]
+  CRUSH rule 0 x 794 [2,8,5]
+  CRUSH rule 0 x 795 [2,5,6]
+  CRUSH rule 0 x 796 [5,7,2]
+  CRUSH rule 0 x 797 [2,5,6]
+  CRUSH rule 0 x 798 [6,2,5]
+  CRUSH rule 0 x 799 [5,2,7]
+  CRUSH rule 0 x 800 [5,2,8]
+  CRUSH rule 0 x 801 [5,7,2]
+  CRUSH rule 0 x 802 [2,6,5]
+  CRUSH rule 0 x 803 [2,5,7]
+  CRUSH rule 0 x 804 [6,2,5]
+  CRUSH rule 0 x 805 [5,8,2]
+  CRUSH rule 0 x 806 [2,5,6]
+  CRUSH rule 0 x 807 [5,7,2]
+  CRUSH rule 0 x 808 [5,7,2]
+  CRUSH rule 0 x 809 [2,5,6]
+  CRUSH rule 0 x 810 [5,7,2]
+  CRUSH rule 0 x 811 [8,5,2]
+  CRUSH rule 0 x 812 [8,5,2]
+  CRUSH rule 0 x 813 [6,5,2]
+  CRUSH rule 0 x 814 [5,8,2]
+  CRUSH rule 0 x 815 [5,2,6]
+  CRUSH rule 0 x 816 [2,6,5]
+  CRUSH rule 0 x 817 [5,6,2]
+  CRUSH rule 0 x 818 [5,2,7]
+  CRUSH rule 0 x 819 [5,2,7]
+  CRUSH rule 0 x 820 [5,7,2]
+  CRUSH rule 0 x 821 [5,8,2]
+  CRUSH rule 0 x 822 [2,5,7]
+  CRUSH rule 0 x 823 [5,7,2]
+  CRUSH rule 0 x 824 [5,7,2]
+  CRUSH rule 0 x 825 [2,8,5]
+  CRUSH rule 0 x 826 [7,2,5]
+  CRUSH rule 0 x 827 [2,6,5]
+  CRUSH rule 0 x 828 [2,5,8]
+  CRUSH rule 0 x 829 [5,6,2]
+  CRUSH rule 0 x 830 [2,5,7]
+  CRUSH rule 0 x 831 [2,6,5]
+  CRUSH rule 0 x 832 [5,8,2]
+  CRUSH rule 0 x 833 [2,6,5]
+  CRUSH rule 0 x 834 [5,2,8]
+  CRUSH rule 0 x 835 [8,5,2]
+  CRUSH rule 0 x 836 [5,8,2]
+  CRUSH rule 0 x 837 [6,5,2]
+  CRUSH rule 0 x 838 [6,2,5]
+  CRUSH rule 0 x 839 [5,2,8]
+  CRUSH rule 0 x 840 [7,5,2]
+  CRUSH rule 0 x 841 [5,8,2]
+  CRUSH rule 0 x 842 [2,5,8]
+  CRUSH rule 0 x 843 [6,5,2]
+  CRUSH rule 0 x 844 [5,8,2]
+  CRUSH rule 0 x 845 [5,6,2]
+  CRUSH rule 0 x 846 [5,2,6]
+  CRUSH rule 0 x 847 [2,8,5]
+  CRUSH rule 0 x 848 [2,6,5]
+  CRUSH rule 0 x 849 [5,8,2]
+  CRUSH rule 0 x 850 [2,5,7]
+  CRUSH rule 0 x 851 [6,5,2]
+  CRUSH rule 0 x 852 [7,5,2]
+  CRUSH rule 0 x 853 [6,2,5]
+  CRUSH rule 0 x 854 [7,2,5]
+  CRUSH rule 0 x 855 [5,7,2]
+  CRUSH rule 0 x 856 [6,5,2]
+  CRUSH rule 0 x 857 [8,5,2]
+  CRUSH rule 0 x 858 [6,5,2]
+  CRUSH rule 0 x 859 [6,2,5]
+  CRUSH rule 0 x 860 [5,2,8]
+  CRUSH rule 0 x 861 [8,5,2]
+  CRUSH rule 0 x 862 [6,2,5]
+  CRUSH rule 0 x 863 [8,2,5]
+  CRUSH rule 0 x 864 [5,6,2]
+  CRUSH rule 0 x 865 [8,2,5]
+  CRUSH rule 0 x 866 [5,6,2]
+  CRUSH rule 0 x 867 [6,5,2]
+  CRUSH rule 0 x 868 [6,5,2]
+  CRUSH rule 0 x 869 [8,5,2]
+  CRUSH rule 0 x 870 [2,5,7]
+  CRUSH rule 0 x 871 [5,2,6]
+  CRUSH rule 0 x 872 [5,2,6]
+  CRUSH rule 0 x 873 [5,6,2]
+  CRUSH rule 0 x 874 [2,6,5]
+  CRUSH rule 0 x 875 [2,6,5]
+  CRUSH rule 0 x 876 [5,8,2]
+  CRUSH rule 0 x 877 [6,5,2]
+  CRUSH rule 0 x 878 [5,2,7]
+  CRUSH rule 0 x 879 [7,5,2]
+  CRUSH rule 0 x 880 [5,2,7]
+  CRUSH rule 0 x 881 [5,8,2]
+  CRUSH rule 0 x 882 [5,2,6]
+  CRUSH rule 0 x 883 [2,5,6]
+  CRUSH rule 0 x 884 [6,2,5]
+  CRUSH rule 0 x 885 [5,2,6]
+  CRUSH rule 0 x 886 [5,7,2]
+  CRUSH rule 0 x 887 [7,5,2]
+  CRUSH rule 0 x 888 [6,2,5]
+  CRUSH rule 0 x 889 [2,8,5]
+  CRUSH rule 0 x 890 [7,2,5]
+  CRUSH rule 0 x 891 [2,7,5]
+  CRUSH rule 0 x 892 [6,2,5]
+  CRUSH rule 0 x 893 [2,5,6]
+  CRUSH rule 0 x 894 [7,5,2]
+  CRUSH rule 0 x 895 [5,2,6]
+  CRUSH rule 0 x 896 [2,6,5]
+  CRUSH rule 0 x 897 [5,2,8]
+  CRUSH rule 0 x 898 [2,5,8]
+  CRUSH rule 0 x 899 [2,6,5]
+  CRUSH rule 0 x 900 [5,2,7]
+  CRUSH rule 0 x 901 [5,2,6]
+  CRUSH rule 0 x 902 [8,5,2]
+  CRUSH rule 0 x 903 [5,6,2]
+  CRUSH rule 0 x 904 [5,7,2]
+  CRUSH rule 0 x 905 [6,2,5]
+  CRUSH rule 0 x 906 [2,7,5]
+  CRUSH rule 0 x 907 [7,2,5]
+  CRUSH rule 0 x 908 [5,6,2]
+  CRUSH rule 0 x 909 [2,5,8]
+  CRUSH rule 0 x 910 [6,5,2]
+  CRUSH rule 0 x 911 [5,7,2]
+  CRUSH rule 0 x 912 [2,8,5]
+  CRUSH rule 0 x 913 [7,2,5]
+  CRUSH rule 0 x 914 [6,5,2]
+  CRUSH rule 0 x 915 [8,2,5]
+  CRUSH rule 0 x 916 [5,2,6]
+  CRUSH rule 0 x 917 [2,5,6]
+  CRUSH rule 0 x 918 [8,2,5]
+  CRUSH rule 0 x 919 [6,2,5]
+  CRUSH rule 0 x 920 [7,5,2]
+  CRUSH rule 0 x 921 [2,5,8]
+  CRUSH rule 0 x 922 [6,5,2]
+  CRUSH rule 0 x 923 [5,6,2]
+  CRUSH rule 0 x 924 [5,2,8]
+  CRUSH rule 0 x 925 [5,6,2]
+  CRUSH rule 0 x 926 [5,2,7]
+  CRUSH rule 0 x 927 [2,6,5]
+  CRUSH rule 0 x 928 [8,2,5]
+  CRUSH rule 0 x 929 [5,2,6]
+  CRUSH rule 0 x 930 [2,5,8]
+  CRUSH rule 0 x 931 [5,2,8]
+  CRUSH rule 0 x 932 [5,2,7]
+  CRUSH rule 0 x 933 [8,5,2]
+  CRUSH rule 0 x 934 [5,8,2]
+  CRUSH rule 0 x 935 [6,5,2]
+  CRUSH rule 0 x 936 [2,7,5]
+  CRUSH rule 0 x 937 [5,8,2]
+  CRUSH rule 0 x 938 [6,5,2]
+  CRUSH rule 0 x 939 [2,8,5]
+  CRUSH rule 0 x 940 [8,5,2]
+  CRUSH rule 0 x 941 [5,2,6]
+  CRUSH rule 0 x 942 [2,6,5]
+  CRUSH rule 0 x 943 [8,2,5]
+  CRUSH rule 0 x 944 [5,8,2]
+  CRUSH rule 0 x 945 [7,2,5]
+  CRUSH rule 0 x 946 [2,8,5]
+  CRUSH rule 0 x 947 [5,2,6]
+  CRUSH rule 0 x 948 [7,5,2]
+  CRUSH rule 0 x 949 [6,2,5]
+  CRUSH rule 0 x 950 [5,7,2]
+  CRUSH rule 0 x 951 [5,6,2]
+  CRUSH rule 0 x 952 [2,7,5]
+  CRUSH rule 0 x 953 [2,5,6]
+  CRUSH rule 0 x 954 [5,2,8]
+  CRUSH rule 0 x 955 [8,2,5]
+  CRUSH rule 0 x 956 [2,7,5]
+  CRUSH rule 0 x 957 [7,2,5]
+  CRUSH rule 0 x 958 [8,5,2]
+  CRUSH rule 0 x 959 [5,2,6]
+  CRUSH rule 0 x 960 [5,6,2]
+  CRUSH rule 0 x 961 [5,2,6]
+  CRUSH rule 0 x 962 [7,5,2]
+  CRUSH rule 0 x 963 [2,5,8]
+  CRUSH rule 0 x 964 [5,2,7]
+  CRUSH rule 0 x 965 [7,5,2]
+  CRUSH rule 0 x 966 [5,6,2]
+  CRUSH rule 0 x 967 [8,5,2]
+  CRUSH rule 0 x 968 [7,2,5]
+  CRUSH rule 0 x 969 [8,2,5]
+  CRUSH rule 0 x 970 [2,8,5]
+  CRUSH rule 0 x 971 [2,8,5]
+  CRUSH rule 0 x 972 [2,7,5]
+  CRUSH rule 0 x 973 [2,8,5]
+  CRUSH rule 0 x 974 [5,2,7]
+  CRUSH rule 0 x 975 [5,8,2]
+  CRUSH rule 0 x 976 [5,7,2]
+  CRUSH rule 0 x 977 [8,5,2]
+  CRUSH rule 0 x 978 [7,2,5]
+  CRUSH rule 0 x 979 [7,2,5]
+  CRUSH rule 0 x 980 [6,2,5]
+  CRUSH rule 0 x 981 [7,5,2]
+  CRUSH rule 0 x 982 [5,2,8]
+  CRUSH rule 0 x 983 [5,6,2]
+  CRUSH rule 0 x 984 [2,8,5]
+  CRUSH rule 0 x 985 [2,5,8]
+  CRUSH rule 0 x 986 [8,5,2]
+  CRUSH rule 0 x 987 [2,5,8]
+  CRUSH rule 0 x 988 [2,5,6]
+  CRUSH rule 0 x 989 [2,8,5]
+  CRUSH rule 0 x 990 [2,6,5]
+  CRUSH rule 0 x 991 [2,5,8]
+  CRUSH rule 0 x 992 [7,2,5]
+  CRUSH rule 0 x 993 [2,6,5]
+  CRUSH rule 0 x 994 [5,6,2]
+  CRUSH rule 0 x 995 [7,2,5]
+  CRUSH rule 0 x 996 [6,5,2]
+  CRUSH rule 0 x 997 [6,5,2]
+  CRUSH rule 0 x 998 [8,2,5]
+  CRUSH rule 0 x 999 [2,7,5]
+  CRUSH rule 0 x 1000 [8,5,2]
+  CRUSH rule 0 x 1001 [2,5,6]
+  CRUSH rule 0 x 1002 [2,5,8]
+  CRUSH rule 0 x 1003 [2,7,5]
+  CRUSH rule 0 x 1004 [6,2,5]
+  CRUSH rule 0 x 1005 [6,2,5]
+  CRUSH rule 0 x 1006 [2,8,5]
+  CRUSH rule 0 x 1007 [2,5,8]
+  CRUSH rule 0 x 1008 [2,7,5]
+  CRUSH rule 0 x 1009 [6,5,2]
+  CRUSH rule 0 x 1010 [5,2,6]
+  CRUSH rule 0 x 1011 [5,2,8]
+  CRUSH rule 0 x 1012 [5,2,8]
+  CRUSH rule 0 x 1013 [5,2,8]
+  CRUSH rule 0 x 1014 [2,8,5]
+  CRUSH rule 0 x 1015 [6,5,2]
+  CRUSH rule 0 x 1016 [2,5,6]
+  CRUSH rule 0 x 1017 [6,2,5]
+  CRUSH rule 0 x 1018 [5,2,6]
+  CRUSH rule 0 x 1019 [5,8,2]
+  CRUSH rule 0 x 1020 [5,2,7]
+  CRUSH rule 0 x 1021 [5,2,7]
+  CRUSH rule 0 x 1022 [2,7,5]
+  CRUSH rule 0 x 1023 [5,2,7]
+  rule 0 (choose) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 1 (choose-two), x = 0..1023, numrep = 2..3
+  CRUSH rule 1 x 0 [2,8]
+  CRUSH rule 1 x 1 [2,8]
+  CRUSH rule 1 x 2 [2,5]
+  CRUSH rule 1 x 3 [8,2]
+  CRUSH rule 1 x 4 [5,7]
+  CRUSH rule 1 x 5 [7,8]
+  CRUSH rule 1 x 6 [2,6]
+  CRUSH rule 1 x 7 [5,7]
+  CRUSH rule 1 x 8 [5,2]
+  CRUSH rule 1 x 9 [2,5]
+  CRUSH rule 1 x 10 [2,8]
+  CRUSH rule 1 x 11 [2,7]
+  CRUSH rule 1 x 12 [2,7]
+  CRUSH rule 1 x 13 [5,8]
+  CRUSH rule 1 x 14 [7,6]
+  CRUSH rule 1 x 15 [7,2]
+  CRUSH rule 1 x 16 [5,6]
+  CRUSH rule 1 x 17 [5,7]
+  CRUSH rule 1 x 18 [2,5]
+  CRUSH rule 1 x 19 [7,5]
+  CRUSH rule 1 x 20 [2,5]
+  CRUSH rule 1 x 21 [5,7]
+  CRUSH rule 1 x 22 [8,5]
+  CRUSH rule 1 x 23 [5,6]
+  CRUSH rule 1 x 24 [2,8]
+  CRUSH rule 1 x 25 [5,7]
+  CRUSH rule 1 x 26 [2,8]
+  CRUSH rule 1 x 27 [5,2]
+  CRUSH rule 1 x 28 [6,2]
+  CRUSH rule 1 x 29 [8,5]
+  CRUSH rule 1 x 30 [5,7]
+  CRUSH rule 1 x 31 [8,7]
+  CRUSH rule 1 x 32 [5,6]
+  CRUSH rule 1 x 33 [2,7]
+  CRUSH rule 1 x 34 [2,5]
+  CRUSH rule 1 x 35 [2,8]
+  CRUSH rule 1 x 36 [5,8]
+  CRUSH rule 1 x 37 [2,5]
+  CRUSH rule 1 x 38 [5,8]
+  CRUSH rule 1 x 39 [5,7]
+  CRUSH rule 1 x 40 [7,8]
+  CRUSH rule 1 x 41 [2,6]
+  CRUSH rule 1 x 42 [5,2]
+  CRUSH rule 1 x 43 [2,5]
+  CRUSH rule 1 x 44 [2,6]
+  CRUSH rule 1 x 45 [8,2]
+  CRUSH rule 1 x 46 [2,5]
+  CRUSH rule 1 x 47 [5,2]
+  CRUSH rule 1 x 48 [5,6]
+  CRUSH rule 1 x 49 [5,2]
+  CRUSH rule 1 x 50 [5,6]
+  CRUSH rule 1 x 51 [5,6]
+  CRUSH rule 1 x 52 [8,6]
+  CRUSH rule 1 x 53 [5,2]
+  CRUSH rule 1 x 54 [7,6]
+  CRUSH rule 1 x 55 [8,7]
+  CRUSH rule 1 x 56 [6,5]
+  CRUSH rule 1 x 57 [5,2]
+  CRUSH rule 1 x 58 [2,5]
+  CRUSH rule 1 x 59 [5,2]
+  CRUSH rule 1 x 60 [5,7]
+  CRUSH rule 1 x 61 [5,6]
+  CRUSH rule 1 x 62 [7,2]
+  CRUSH rule 1 x 63 [5,6]
+  CRUSH rule 1 x 64 [5,2]
+  CRUSH rule 1 x 65 [7,5]
+  CRUSH rule 1 x 66 [5,2]
+  CRUSH rule 1 x 67 [5,2]
+  CRUSH rule 1 x 68 [2,5]
+  CRUSH rule 1 x 69 [5,2]
+  CRUSH rule 1 x 70 [7,2]
+  CRUSH rule 1 x 71 [2,8]
+  CRUSH rule 1 x 72 [6,2]
+  CRUSH rule 1 x 73 [2,7]
+  CRUSH rule 1 x 74 [2,7]
+  CRUSH rule 1 x 75 [5,2]
+  CRUSH rule 1 x 76 [5,2]
+  CRUSH rule 1 x 77 [7,2]
+  CRUSH rule 1 x 78 [2,5]
+  CRUSH rule 1 x 79 [5,2]
+  CRUSH rule 1 x 80 [2,5]
+  CRUSH rule 1 x 81 [2]
+  CRUSH rule 1 x 82 [7,2]
+  CRUSH rule 1 x 83 [2,6]
+  CRUSH rule 1 x 84 [7,2]
+  CRUSH rule 1 x 85 [5,8]
+  CRUSH rule 1 x 86 [2,7]
+  CRUSH rule 1 x 87 [2,7]
+  CRUSH rule 1 x 88 [2,6]
+  CRUSH rule 1 x 89 [5,2]
+  CRUSH rule 1 x 90 [6,7]
+  CRUSH rule 1 x 91 [5,8]
+  CRUSH rule 1 x 92 [2,8]
+  CRUSH rule 1 x 93 [7,5]
+  CRUSH rule 1 x 94 [2,5]
+  CRUSH rule 1 x 95 [7,5]
+  CRUSH rule 1 x 96 [5,6]
+  CRUSH rule 1 x 97 [8,7]
+  CRUSH rule 1 x 98 [2,6]
+  CRUSH rule 1 x 99 [2,7]
+  CRUSH rule 1 x 100 [2,7]
+  CRUSH rule 1 x 101 [5,7]
+  CRUSH rule 1 x 102 [5,2]
+  CRUSH rule 1 x 103 [5,7]
+  CRUSH rule 1 x 104 [7,5]
+  CRUSH rule 1 x 105 [2,5]
+  CRUSH rule 1 x 106 [2,6]
+  CRUSH rule 1 x 107 [5,2]
+  CRUSH rule 1 x 108 [7,2]
+  CRUSH rule 1 x 109 [2,5]
+  CRUSH rule 1 x 110 [5,2]
+  CRUSH rule 1 x 111 [2,5]
+  CRUSH rule 1 x 112 [2,5]
+  CRUSH rule 1 x 113 [6,2]
+  CRUSH rule 1 x 114 [7,6]
+  CRUSH rule 1 x 115 [8,2]
+  CRUSH rule 1 x 116 [2,6]
+  CRUSH rule 1 x 117 [7,5]
+  CRUSH rule 1 x 118 [2,5]
+  CRUSH rule 1 x 119 [5,6]
+  CRUSH rule 1 x 120 [2,5]
+  CRUSH rule 1 x 121 [2,5]
+  CRUSH rule 1 x 122 [8,5]
+  CRUSH rule 1 x 123 [2,5]
+  CRUSH rule 1 x 124 [5,8]
+  CRUSH rule 1 x 125 [2,7]
+  CRUSH rule 1 x 126 [5,2]
+  CRUSH rule 1 x 127 [5,6]
+  CRUSH rule 1 x 128 [5,8]
+  CRUSH rule 1 x 129 [2,7]
+  CRUSH rule 1 x 130 [5,8]
+  CRUSH rule 1 x 131 [2,8]
+  CRUSH rule 1 x 132 [2,7]
+  CRUSH rule 1 x 133 [5,6]
+  CRUSH rule 1 x 134 [2,8]
+  CRUSH rule 1 x 135 [5,6]
+  CRUSH rule 1 x 136 [2,5]
+  CRUSH rule 1 x 137 [7,5]
+  CRUSH rule 1 x 138 [8,7]
+  CRUSH rule 1 x 139 [5,2]
+  CRUSH rule 1 x 140 [2,6]
+  CRUSH rule 1 x 141 [6,8]
+  CRUSH rule 1 x 142 [5,6]
+  CRUSH rule 1 x 143 [5,8]
+  CRUSH rule 1 x 144 [8,2]
+  CRUSH rule 1 x 145 [8,5]
+  CRUSH rule 1 x 146 [2,6]
+  CRUSH rule 1 x 147 [2,8]
+  CRUSH rule 1 x 148 [5,2]
+  CRUSH rule 1 x 149 [5,8]
+  CRUSH rule 1 x 150 [2,6]
+  CRUSH rule 1 x 151 [5,2]
+  CRUSH rule 1 x 152 [8,5]
+  CRUSH rule 1 x 153 [8,6]
+  CRUSH rule 1 x 154 [5,2]
+  CRUSH rule 1 x 155 [5,7]
+  CRUSH rule 1 x 156 [5,8]
+  CRUSH rule 1 x 157 [5,2]
+  CRUSH rule 1 x 158 [2,8]
+  CRUSH rule 1 x 159 [7,2]
+  CRUSH rule 1 x 160 [2,8]
+  CRUSH rule 1 x 161 [2,5]
+  CRUSH rule 1 x 162 [2,6]
+  CRUSH rule 1 x 163 [5,6]
+  CRUSH rule 1 x 164 [7,8]
+  CRUSH rule 1 x 165 [7,2]
+  CRUSH rule 1 x 166 [2,5]
+  CRUSH rule 1 x 167 [2,6]
+  CRUSH rule 1 x 168 [5,2]
+  CRUSH rule 1 x 169 [2,6]
+  CRUSH rule 1 x 170 [2,8]
+  CRUSH rule 1 x 171 [7,5]
+  CRUSH rule 1 x 172 [2,7]
+  CRUSH rule 1 x 173 [8,5]
+  CRUSH rule 1 x 174 [2,5]
+  CRUSH rule 1 x 175 [6,2]
+  CRUSH rule 1 x 176 [5,2]
+  CRUSH rule 1 x 177 [5,8]
+  CRUSH rule 1 x 178 [5,2]
+  CRUSH rule 1 x 179 [5,2]
+  CRUSH rule 1 x 180 [5,8]
+  CRUSH rule 1 x 181 [6,2]
+  CRUSH rule 1 x 182 [8,5]
+  CRUSH rule 1 x 183 [7,8]
+  CRUSH rule 1 x 184 [5,7]
+  CRUSH rule 1 x 185 [6,8]
+  CRUSH rule 1 x 186 [2,8]
+  CRUSH rule 1 x 187 [2,6]
+  CRUSH rule 1 x 188 [2,8]
+  CRUSH rule 1 x 189 [2,7]
+  CRUSH rule 1 x 190 [5,2]
+  CRUSH rule 1 x 191 [7,6]
+  CRUSH rule 1 x 192 [5,2]
+  CRUSH rule 1 x 193 [5,2]
+  CRUSH rule 1 x 194 [2,5]
+  CRUSH rule 1 x 195 [6,5]
+  CRUSH rule 1 x 196 [6,7]
+  CRUSH rule 1 x 197 [6,5]
+  CRUSH rule 1 x 198 [2,5]
+  CRUSH rule 1 x 199 [2,5]
+  CRUSH rule 1 x 200 [2,5]
+  CRUSH rule 1 x 201 [7,2]
+  CRUSH rule 1 x 202 [6,5]
+  CRUSH rule 1 x 203 [5,8]
+  CRUSH rule 1 x 204 [2,5]
+  CRUSH rule 1 x 205 [2,7]
+  CRUSH rule 1 x 206 [2,5]
+  CRUSH rule 1 x 207 [5,2]
+  CRUSH rule 1 x 208 [7,2]
+  CRUSH rule 1 x 209 [2,5]
+  CRUSH rule 1 x 210 [2,8]
+  CRUSH rule 1 x 211 [5,6]
+  CRUSH rule 1 x 212 [7,5]
+  CRUSH rule 1 x 213 [8,5]
+  CRUSH rule 1 x 214 [5,7]
+  CRUSH rule 1 x 215 [8,2]
+  CRUSH rule 1 x 216 [5,2]
+  CRUSH rule 1 x 217 [2,5]
+  CRUSH rule 1 x 218 [2,7]
+  CRUSH rule 1 x 219 [5,8]
+  CRUSH rule 1 x 220 [5,7]
+  CRUSH rule 1 x 221 [5,6]
+  CRUSH rule 1 x 222 [6,8]
+  CRUSH rule 1 x 223 [2,5]
+  CRUSH rule 1 x 224 [2,5]
+  CRUSH rule 1 x 225 [8,6]
+  CRUSH rule 1 x 226 [7,2]
+  CRUSH rule 1 x 227 [5,6]
+  CRUSH rule 1 x 228 [5,6]
+  CRUSH rule 1 x 229 [5,2]
+  CRUSH rule 1 x 230 [5,7]
+  CRUSH rule 1 x 231 [5,6]
+  CRUSH rule 1 x 232 [2,7]
+  CRUSH rule 1 x 233 [5,8]
+  CRUSH rule 1 x 234 [2,7]
+  CRUSH rule 1 x 235 [5,8]
+  CRUSH rule 1 x 236 [5,2]
+  CRUSH rule 1 x 237 [5,7]
+  CRUSH rule 1 x 238 [5,6]
+  CRUSH rule 1 x 239 [8,7]
+  CRUSH rule 1 x 240 [5,7]
+  CRUSH rule 1 x 241 [5,2]
+  CRUSH rule 1 x 242 [5,8]
+  CRUSH rule 1 x 243 [5,7]
+  CRUSH rule 1 x 244 [5,6]
+  CRUSH rule 1 x 245 [7,6]
+  CRUSH rule 1 x 246 [2,5]
+  CRUSH rule 1 x 247 [6,2]
+  CRUSH rule 1 x 248 [8,2]
+  CRUSH rule 1 x 249 [2,5]
+  CRUSH rule 1 x 250 [2,7]
+  CRUSH rule 1 x 251 [2,5]
+  CRUSH rule 1 x 252 [5,7]
+  CRUSH rule 1 x 253 [5,2]
+  CRUSH rule 1 x 254 [5,6]
+  CRUSH rule 1 x 255 [2,7]
+  CRUSH rule 1 x 256 [5,7]
+  CRUSH rule 1 x 257 [2,8]
+  CRUSH rule 1 x 258 [5]
+  CRUSH rule 1 x 259 [5,7]
+  CRUSH rule 1 x 260 [5,6]
+  CRUSH rule 1 x 261 [8,7]
+  CRUSH rule 1 x 262 [5,2]
+  CRUSH rule 1 x 263 [6,8]
+  CRUSH rule 1 x 264 [5,6]
+  CRUSH rule 1 x 265 [8,6]
+  CRUSH rule 1 x 266 [8,2]
+  CRUSH rule 1 x 267 [2,5]
+  CRUSH rule 1 x 268 [2,7]
+  CRUSH rule 1 x 269 [2,8]
+  CRUSH rule 1 x 270 [5,2]
+  CRUSH rule 1 x 271 [7,5]
+  CRUSH rule 1 x 272 [2,8]
+  CRUSH rule 1 x 273 [5,2]
+  CRUSH rule 1 x 274 [6,8]
+  CRUSH rule 1 x 275 [5,8]
+  CRUSH rule 1 x 276 [7,2]
+  CRUSH rule 1 x 277 [6,5]
+  CRUSH rule 1 x 278 [6,8]
+  CRUSH rule 1 x 279 [8,5]
+  CRUSH rule 1 x 280 [2,6]
+  CRUSH rule 1 x 281 [8,2]
+  CRUSH rule 1 x 282 [5,2]
+  CRUSH rule 1 x 283 [8,2]
+  CRUSH rule 1 x 284 [6,5]
+  CRUSH rule 1 x 285 [5,6]
+  CRUSH rule 1 x 286 [2,5]
+  CRUSH rule 1 x 287 [2,5]
+  CRUSH rule 1 x 288 [8,2]
+  CRUSH rule 1 x 289 [5,6]
+  CRUSH rule 1 x 290 [2,5]
+  CRUSH rule 1 x 291 [2,5]
+  CRUSH rule 1 x 292 [8,2]
+  CRUSH rule 1 x 293 [6,2]
+  CRUSH rule 1 x 294 [7,5]
+  CRUSH rule 1 x 295 [5,8]
+  CRUSH rule 1 x 296 [5,2]
+  CRUSH rule 1 x 297 [6,2]
+  CRUSH rule 1 x 298 [2,8]
+  CRUSH rule 1 x 299 [2,5]
+  CRUSH rule 1 x 300 [8,7]
+  CRUSH rule 1 x 301 [2,8]
+  CRUSH rule 1 x 302 [5,2]
+  CRUSH rule 1 x 303 [7,5]
+  CRUSH rule 1 x 304 [2,7]
+  CRUSH rule 1 x 305 [5,8]
+  CRUSH rule 1 x 306 [2,7]
+  CRUSH rule 1 x 307 [2,7]
+  CRUSH rule 1 x 308 [2,8]
+  CRUSH rule 1 x 309 [7,5]
+  CRUSH rule 1 x 310 [5,6]
+  CRUSH rule 1 x 311 [5,2]
+  CRUSH rule 1 x 312 [2,6]
+  CRUSH rule 1 x 313 [5,8]
+  CRUSH rule 1 x 314 [5,2]
+  CRUSH rule 1 x 315 [2,5]
+  CRUSH rule 1 x 316 [6,5]
+  CRUSH rule 1 x 317 [2,6]
+  CRUSH rule 1 x 318 [8,6]
+  CRUSH rule 1 x 319 [5,2]
+  CRUSH rule 1 x 320 [5,7]
+  CRUSH rule 1 x 321 [2,5]
+  CRUSH rule 1 x 322 [2,7]
+  CRUSH rule 1 x 323 [5,7]
+  CRUSH rule 1 x 324 [7,2]
+  CRUSH rule 1 x 325 [5,6]
+  CRUSH rule 1 x 326 [5,6]
+  CRUSH rule 1 x 327 [2,6]
+  CRUSH rule 1 x 328 [7,5]
+  CRUSH rule 1 x 329 [5,6]
+  CRUSH rule 1 x 330 [5,7]
+  CRUSH rule 1 x 331 [2,6]
+  CRUSH rule 1 x 332 [2,5]
+  CRUSH rule 1 x 333 [6,8]
+  CRUSH rule 1 x 334 [8,5]
+  CRUSH rule 1 x 335 [7,2]
+  CRUSH rule 1 x 336 [5,6]
+  CRUSH rule 1 x 337 [7,2]
+  CRUSH rule 1 x 338 [5,6]
+  CRUSH rule 1 x 339 [7,5]
+  CRUSH rule 1 x 340 [2,5]
+  CRUSH rule 1 x 341 [5,2]
+  CRUSH rule 1 x 342 [2,7]
+  CRUSH rule 1 x 343 [6,7]
+  CRUSH rule 1 x 344 [6,2]
+  CRUSH rule 1 x 345 [5,2]
+  CRUSH rule 1 x 346 [8,2]
+  CRUSH rule 1 x 347 [5,2]
+  CRUSH rule 1 x 348 [8,2]
+  CRUSH rule 1 x 349 [2,6]
+  CRUSH rule 1 x 350 [8,5]
+  CRUSH rule 1 x 351 [5,6]
+  CRUSH rule 1 x 352 [2,7]
+  CRUSH rule 1 x 353 [6,5]
+  CRUSH rule 1 x 354 [2,5]
+  CRUSH rule 1 x 355 [5,2]
+  CRUSH rule 1 x 356 [5,8]
+  CRUSH rule 1 x 357 [6,2]
+  CRUSH rule 1 x 358 [2,5]
+  CRUSH rule 1 x 359 [6,7]
+  CRUSH rule 1 x 360 [5,2]
+  CRUSH rule 1 x 361 [8,5]
+  CRUSH rule 1 x 362 [5,7]
+  CRUSH rule 1 x 363 [5,2]
+  CRUSH rule 1 x 364 [2,5]
+  CRUSH rule 1 x 365 [6,7]
+  CRUSH rule 1 x 366 [7,2]
+  CRUSH rule 1 x 367 [5,7]
+  CRUSH rule 1 x 368 [7,5]
+  CRUSH rule 1 x 369 [5,7]
+  CRUSH rule 1 x 370 [8,7]
+  CRUSH rule 1 x 371 [2,5]
+  CRUSH rule 1 x 372 [5,2]
+  CRUSH rule 1 x 373 [2,6]
+  CRUSH rule 1 x 374 [5,8]
+  CRUSH rule 1 x 375 [6,5]
+  CRUSH rule 1 x 376 [7,2]
+  CRUSH rule 1 x 377 [2,5]
+  CRUSH rule 1 x 378 [2,6]
+  CRUSH rule 1 x 379 [8,5]
+  CRUSH rule 1 x 380 [2,5]
+  CRUSH rule 1 x 381 [2,5]
+  CRUSH rule 1 x 382 [2,5]
+  CRUSH rule 1 x 383 [5,2]
+  CRUSH rule 1 x 384 [7,2]
+  CRUSH rule 1 x 385 [7,5]
+  CRUSH rule 1 x 386 [2,5]
+  CRUSH rule 1 x 387 [2,5]
+  CRUSH rule 1 x 388 [5,2]
+  CRUSH rule 1 x 389 [2,5]
+  CRUSH rule 1 x 390 [5,6]
+  CRUSH rule 1 x 391 [5,6]
+  CRUSH rule 1 x 392 [2,8]
+  CRUSH rule 1 x 393 [5,2]
+  CRUSH rule 1 x 394 [5,7]
+  CRUSH rule 1 x 395 [5,2]
+  CRUSH rule 1 x 396 [5,2]
+  CRUSH rule 1 x 397 [2,7]
+  CRUSH rule 1 x 398 [2,5]
+  CRUSH rule 1 x 399 [8,7]
+  CRUSH rule 1 x 400 [8,2]
+  CRUSH rule 1 x 401 [2,5]
+  CRUSH rule 1 x 402 [7,8]
+  CRUSH rule 1 x 403 [2,7]
+  CRUSH rule 1 x 404 [5,2]
+  CRUSH rule 1 x 405 [6,5]
+  CRUSH rule 1 x 406 [2,6]
+  CRUSH rule 1 x 407 [2,8]
+  CRUSH rule 1 x 408 [5,2]
+  CRUSH rule 1 x 409 [7,5]
+  CRUSH rule 1 x 410 [8,6]
+  CRUSH rule 1 x 411 [2,6]
+  CRUSH rule 1 x 412 [2,5]
+  CRUSH rule 1 x 413 [5,2]
+  CRUSH rule 1 x 414 [5,2]
+  CRUSH rule 1 x 415 [2,6]
+  CRUSH rule 1 x 416 [2,5]
+  CRUSH rule 1 x 417 [8,7]
+  CRUSH rule 1 x 418 [7,6]
+  CRUSH rule 1 x 419 [8,5]
+  CRUSH rule 1 x 420 [2,5]
+  CRUSH rule 1 x 421 [8,6]
+  CRUSH rule 1 x 422 [6,7]
+  CRUSH rule 1 x 423 [2,5]
+  CRUSH rule 1 x 424 [8,5]
+  CRUSH rule 1 x 425 [2,5]
+  CRUSH rule 1 x 426 [6,7]
+  CRUSH rule 1 x 427 [2,7]
+  CRUSH rule 1 x 428 [5,6]
+  CRUSH rule 1 x 429 [5,6]
+  CRUSH rule 1 x 430 [5,6]
+  CRUSH rule 1 x 431 [5,2]
+  CRUSH rule 1 x 432 [7,2]
+  CRUSH rule 1 x 433 [6,5]
+  CRUSH rule 1 x 434 [5,2]
+  CRUSH rule 1 x 435 [2,5]
+  CRUSH rule 1 x 436 [5,2]
+  CRUSH rule 1 x 437 [7,5]
+  CRUSH rule 1 x 438 [2,5]
+  CRUSH rule 1 x 439 [2,5]
+  CRUSH rule 1 x 440 [2,7]
+  CRUSH rule 1 x 441 [5,7]
+  CRUSH rule 1 x 442 [2,5]
+  CRUSH rule 1 x 443 [6,8]
+  CRUSH rule 1 x 444 [7,2]
+  CRUSH rule 1 x 445 [6,5]
+  CRUSH rule 1 x 446 [5,7]
+  CRUSH rule 1 x 447 [2,5]
+  CRUSH rule 1 x 448 [7,2]
+  CRUSH rule 1 x 449 [7,8]
+  CRUSH rule 1 x 450 [5,8]
+  CRUSH rule 1 x 451 [6,8]
+  CRUSH rule 1 x 452 [8,5]
+  CRUSH rule 1 x 453 [6,8]
+  CRUSH rule 1 x 454 [6,7]
+  CRUSH rule 1 x 455 [2,7]
+  CRUSH rule 1 x 456 [6,8]
+  CRUSH rule 1 x 457 [7,2]
+  CRUSH rule 1 x 458 [2,8]
+  CRUSH rule 1 x 459 [2,5]
+  CRUSH rule 1 x 460 [6,5]
+  CRUSH rule 1 x 461 [6,5]
+  CRUSH rule 1 x 462 [8,2]
+  CRUSH rule 1 x 463 [6,7]
+  CRUSH rule 1 x 464 [7,5]
+  CRUSH rule 1 x 465 [7,6]
+  CRUSH rule 1 x 466 [5,8]
+  CRUSH rule 1 x 467 [6,5]
+  CRUSH rule 1 x 468 [7,8]
+  CRUSH rule 1 x 469 [7,2]
+  CRUSH rule 1 x 470 [5,2]
+  CRUSH rule 1 x 471 [2,6]
+  CRUSH rule 1 x 472 [5,6]
+  CRUSH rule 1 x 473 [2,5]
+  CRUSH rule 1 x 474 [6,2]
+  CRUSH rule 1 x 475 [6,7]
+  CRUSH rule 1 x 476 [5,2]
+  CRUSH rule 1 x 477 [5,8]
+  CRUSH rule 1 x 478 [6,7]
+  CRUSH rule 1 x 479 [2,5]
+  CRUSH rule 1 x 480 [2,8]
+  CRUSH rule 1 x 481 [2,5]
+  CRUSH rule 1 x 482 [5,7]
+  CRUSH rule 1 x 483 [2,7]
+  CRUSH rule 1 x 484 [2,7]
+  CRUSH rule 1 x 485 [5,7]
+  CRUSH rule 1 x 486 [5,2]
+  CRUSH rule 1 x 487 [5,2]
+  CRUSH rule 1 x 488 [5,7]
+  CRUSH rule 1 x 489 [2,8]
+  CRUSH rule 1 x 490 [6,5]
+  CRUSH rule 1 x 491 [2,6]
+  CRUSH rule 1 x 492 [6,5]
+  CRUSH rule 1 x 493 [2,8]
+  CRUSH rule 1 x 494 [2,7]
+  CRUSH rule 1 x 495 [5,6]
+  CRUSH rule 1 x 496 [7,5]
+  CRUSH rule 1 x 497 [5,7]
+  CRUSH rule 1 x 498 [2,5]
+  CRUSH rule 1 x 499 [8,5]
+  CRUSH rule 1 x 500 [5,6]
+  CRUSH rule 1 x 501 [2,7]
+  CRUSH rule 1 x 502 [7,2]
+  CRUSH rule 1 x 503 [2,5]
+  CRUSH rule 1 x 504 [5,6]
+  CRUSH rule 1 x 505 [2,7]
+  CRUSH rule 1 x 506 [5,2]
+  CRUSH rule 1 x 507 [6,2]
+  CRUSH rule 1 x 508 [2,7]
+  CRUSH rule 1 x 509 [7,5]
+  CRUSH rule 1 x 510 [6,2]
+  CRUSH rule 1 x 511 [5,8]
+  CRUSH rule 1 x 512 [7,6]
+  CRUSH rule 1 x 513 [7,2]
+  CRUSH rule 1 x 514 [5,7]
+  CRUSH rule 1 x 515 [8,5]
+  CRUSH rule 1 x 516 [5,2]
+  CRUSH rule 1 x 517 [7,8]
+  CRUSH rule 1 x 518 [5,6]
+  CRUSH rule 1 x 519 [7,5]
+  CRUSH rule 1 x 520 [2,6]
+  CRUSH rule 1 x 521 [8,7]
+  CRUSH rule 1 x 522 [6,8]
+  CRUSH rule 1 x 523 [5,2]
+  CRUSH rule 1 x 524 [2,5]
+  CRUSH rule 1 x 525 [2,5]
+  CRUSH rule 1 x 526 [2,5]
+  CRUSH rule 1 x 527 [2,5]
+  CRUSH rule 1 x 528 [5,2]
+  CRUSH rule 1 x 529 [5,7]
+  CRUSH rule 1 x 530 [6,7]
+  CRUSH rule 1 x 531 [6,2]
+  CRUSH rule 1 x 532 [6,5]
+  CRUSH rule 1 x 533 [5,6]
+  CRUSH rule 1 x 534 [7,5]
+  CRUSH rule 1 x 535 [8,6]
+  CRUSH rule 1 x 536 [6,7]
+  CRUSH rule 1 x 537 [5,7]
+  CRUSH rule 1 x 538 [6,8]
+  CRUSH rule 1 x 539 [8,5]
+  CRUSH rule 1 x 540 [2,6]
+  CRUSH rule 1 x 541 [2,5]
+  CRUSH rule 1 x 542 [5,2]
+  CRUSH rule 1 x 543 [6,2]
+  CRUSH rule 1 x 544 [5,7]
+  CRUSH rule 1 x 545 [5,7]
+  CRUSH rule 1 x 546 [6,2]
+  CRUSH rule 1 x 547 [8,2]
+  CRUSH rule 1 x 548 [5,2]
+  CRUSH rule 1 x 549 [5,8]
+  CRUSH rule 1 x 550 [2,5]
+  CRUSH rule 1 x 551 [7,5]
+  CRUSH rule 1 x 552 [5,2]
+  CRUSH rule 1 x 553 [5,2]
+  CRUSH rule 1 x 554 [2,8]
+  CRUSH rule 1 x 555 [5,2]
+  CRUSH rule 1 x 556 [5,6]
+  CRUSH rule 1 x 557 [7,5]
+  CRUSH rule 1 x 558 [5,2]
+  CRUSH rule 1 x 559 [5,2]
+  CRUSH rule 1 x 560 [8,5]
+  CRUSH rule 1 x 561 [6,5]
+  CRUSH rule 1 x 562 [5,7]
+  CRUSH rule 1 x 563 [2,6]
+  CRUSH rule 1 x 564 [5,2]
+  CRUSH rule 1 x 565 [5,6]
+  CRUSH rule 1 x 566 [5,7]
+  CRUSH rule 1 x 567 [5,6]
+  CRUSH rule 1 x 568 [7,5]
+  CRUSH rule 1 x 569 [5,2]
+  CRUSH rule 1 x 570 [2,5]
+  CRUSH rule 1 x 571 [5,7]
+  CRUSH rule 1 x 572 [5]
+  CRUSH rule 1 x 573 [5,2]
+  CRUSH rule 1 x 574 [2,8]
+  CRUSH rule 1 x 575 [8,6]
+  CRUSH rule 1 x 576 [5,6]
+  CRUSH rule 1 x 577 [8,2]
+  CRUSH rule 1 x 578 [6,8]
+  CRUSH rule 1 x 579 [5,2]
+  CRUSH rule 1 x 580 [5,2]
+  CRUSH rule 1 x 581 [7,2]
+  CRUSH rule 1 x 582 [2,8]
+  CRUSH rule 1 x 583 [6,2]
+  CRUSH rule 1 x 584 [8,2]
+  CRUSH rule 1 x 585 [7,2]
+  CRUSH rule 1 x 586 [2,6]
+  CRUSH rule 1 x 587 [2,5]
+  CRUSH rule 1 x 588 [5,2]
+  CRUSH rule 1 x 589 [7,2]
+  CRUSH rule 1 x 590 [6,2]
+  CRUSH rule 1 x 591 [5,2]
+  CRUSH rule 1 x 592 [2,5]
+  CRUSH rule 1 x 593 [2,8]
+  CRUSH rule 1 x 594 [2,7]
+  CRUSH rule 1 x 595 [7,2]
+  CRUSH rule 1 x 596 [5,7]
+  CRUSH rule 1 x 597 [5,2]
+  CRUSH rule 1 x 598 [5,2]
+  CRUSH rule 1 x 599 [5,2]
+  CRUSH rule 1 x 600 [7,2]
+  CRUSH rule 1 x 601 [2,7]
+  CRUSH rule 1 x 602 [5,7]
+  CRUSH rule 1 x 603 [5,2]
+  CRUSH rule 1 x 604 [7,5]
+  CRUSH rule 1 x 605 [5,2]
+  CRUSH rule 1 x 606 [2,6]
+  CRUSH rule 1 x 607 [2,5]
+  CRUSH rule 1 x 608 [5,2]
+  CRUSH rule 1 x 609 [5,2]
+  CRUSH rule 1 x 610 [5,7]
+  CRUSH rule 1 x 611 [2,5]
+  CRUSH rule 1 x 612 [2,8]
+  CRUSH rule 1 x 613 [7,2]
+  CRUSH rule 1 x 614 [7,8]
+  CRUSH rule 1 x 615 [6,8]
+  CRUSH rule 1 x 616 [2,8]
+  CRUSH rule 1 x 617 [6,2]
+  CRUSH rule 1 x 618 [7,6]
+  CRUSH rule 1 x 619 [5,2]
+  CRUSH rule 1 x 620 [5,2]
+  CRUSH rule 1 x 621 [5,8]
+  CRUSH rule 1 x 622 [2,5]
+  CRUSH rule 1 x 623 [2,6]
+  CRUSH rule 1 x 624 [5,7]
+  CRUSH rule 1 x 625 [2,5]
+  CRUSH rule 1 x 626 [7,8]
+  CRUSH rule 1 x 627 [2,7]
+  CRUSH rule 1 x 628 [8,2]
+  CRUSH rule 1 x 629 [2,6]
+  CRUSH rule 1 x 630 [2,7]
+  CRUSH rule 1 x 631 [2,7]
+  CRUSH rule 1 x 632 [7,2]
+  CRUSH rule 1 x 633 [8,6]
+  CRUSH rule 1 x 634 [2,5]
+  CRUSH rule 1 x 635 [5,6]
+  CRUSH rule 1 x 636 [2,5]
+  CRUSH rule 1 x 637 [5,2]
+  CRUSH rule 1 x 638 [6,8]
+  CRUSH rule 1 x 639 [5,2]
+  CRUSH rule 1 x 640 [5,2]
+  CRUSH rule 1 x 641 [7,2]
+  CRUSH rule 1 x 642 [2,8]
+  CRUSH rule 1 x 643 [5,7]
+  CRUSH rule 1 x 644 [8,2]
+  CRUSH rule 1 x 645 [5,2]
+  CRUSH rule 1 x 646 [8,2]
+  CRUSH rule 1 x 647 [7,2]
+  CRUSH rule 1 x 648 [2,6]
+  CRUSH rule 1 x 649 [5,7]
+  CRUSH rule 1 x 650 [7,8]
+  CRUSH rule 1 x 651 [5,7]
+  CRUSH rule 1 x 652 [5,6]
+  CRUSH rule 1 x 653 [8,5]
+  CRUSH rule 1 x 654 [7,5]
+  CRUSH rule 1 x 655 [2,5]
+  CRUSH rule 1 x 656 [5,2]
+  CRUSH rule 1 x 657 [6,2]
+  CRUSH rule 1 x 658 [5,2]
+  CRUSH rule 1 x 659 [5,6]
+  CRUSH rule 1 x 660 [7,8]
+  CRUSH rule 1 x 661 [2,8]
+  CRUSH rule 1 x 662 [5,2]
+  CRUSH rule 1 x 663 [2,5]
+  CRUSH rule 1 x 664 [2,5]
+  CRUSH rule 1 x 665 [5,7]
+  CRUSH rule 1 x 666 [2,8]
+  CRUSH rule 1 x 667 [2,5]
+  CRUSH rule 1 x 668 [5,7]
+  CRUSH rule 1 x 669 [6,5]
+  CRUSH rule 1 x 670 [5,2]
+  CRUSH rule 1 x 671 [2,5]
+  CRUSH rule 1 x 672 [5,6]
+  CRUSH rule 1 x 673 [5,2]
+  CRUSH rule 1 x 674 [5,2]
+  CRUSH rule 1 x 675 [2,8]
+  CRUSH rule 1 x 676 [2,5]
+  CRUSH rule 1 x 677 [5,2]
+  CRUSH rule 1 x 678 [2,5]
+  CRUSH rule 1 x 679 [6,2]
+  CRUSH rule 1 x 680 [2,5]
+  CRUSH rule 1 x 681 [5,7]
+  CRUSH rule 1 x 682 [2,5]
+  CRUSH rule 1 x 683 [2,8]
+  CRUSH rule 1 x 684 [7,2]
+  CRUSH rule 1 x 685 [7,2]
+  CRUSH rule 1 x 686 [2,5]
+  CRUSH rule 1 x 687 [5,7]
+  CRUSH rule 1 x 688 [5,7]
+  CRUSH rule 1 x 689 [6,5]
+  CRUSH rule 1 x 690 [8,2]
+  CRUSH rule 1 x 691 [5,2]
+  CRUSH rule 1 x 692 [7,2]
+  CRUSH rule 1 x 693 [6,7]
+  CRUSH rule 1 x 694 [6,5]
+  CRUSH rule 1 x 695 [2,8]
+  CRUSH rule 1 x 696 [2,5]
+  CRUSH rule 1 x 697 [6,2]
+  CRUSH rule 1 x 698 [6,2]
+  CRUSH rule 1 x 699 [2,6]
+  CRUSH rule 1 x 700 [2,5]
+  CRUSH rule 1 x 701 [5,2]
+  CRUSH rule 1 x 702 [5,6]
+  CRUSH rule 1 x 703 [8,5]
+  CRUSH rule 1 x 704 [2,5]
+  CRUSH rule 1 x 705 [8,6]
+  CRUSH rule 1 x 706 [2,7]
+  CRUSH rule 1 x 707 [7,8]
+  CRUSH rule 1 x 708 [5,7]
+  CRUSH rule 1 x 709 [6,5]
+  CRUSH rule 1 x 710 [8,5]
+  CRUSH rule 1 x 711 [2,5]
+  CRUSH rule 1 x 712 [2,5]
+  CRUSH rule 1 x 713 [6,7]
+  CRUSH rule 1 x 714 [5,2]
+  CRUSH rule 1 x 715 [2,5]
+  CRUSH rule 1 x 716 [5,6]
+  CRUSH rule 1 x 717 [8,7]
+  CRUSH rule 1 x 718 [5,7]
+  CRUSH rule 1 x 719 [2,6]
+  CRUSH rule 1 x 720 [6,8]
+  CRUSH rule 1 x 721 [5,6]
+  CRUSH rule 1 x 722 [5,2]
+  CRUSH rule 1 x 723 [5,2]
+  CRUSH rule 1 x 724 [2,6]
+  CRUSH rule 1 x 725 [2,8]
+  CRUSH rule 1 x 726 [5,8]
+  CRUSH rule 1 x 727 [5,6]
+  CRUSH rule 1 x 728 [2,6]
+  CRUSH rule 1 x 729 [5,6]
+  CRUSH rule 1 x 730 [5,7]
+  CRUSH rule 1 x 731 [5,2]
+  CRUSH rule 1 x 732 [2,5]
+  CRUSH rule 1 x 733 [5,2]
+  CRUSH rule 1 x 734 [6,5]
+  CRUSH rule 1 x 735 [5,8]
+  CRUSH rule 1 x 736 [5,8]
+  CRUSH rule 1 x 737 [2,5]
+  CRUSH rule 1 x 738 [5,2]
+  CRUSH rule 1 x 739 [2,8]
+  CRUSH rule 1 x 740 [2,6]
+  CRUSH rule 1 x 741 [7,8]
+  CRUSH rule 1 x 742 [8,2]
+  CRUSH rule 1 x 743 [7,2]
+  CRUSH rule 1 x 744 [5,7]
+  CRUSH rule 1 x 745 [5,6]
+  CRUSH rule 1 x 746 [5,2]
+  CRUSH rule 1 x 747 [6,2]
+  CRUSH rule 1 x 748 [2,7]
+  CRUSH rule 1 x 749 [5,8]
+  CRUSH rule 1 x 750 [2,6]
+  CRUSH rule 1 x 751 [2,8]
+  CRUSH rule 1 x 752 [8,2]
+  CRUSH rule 1 x 753 [7,8]
+  CRUSH rule 1 x 754 [8,6]
+  CRUSH rule 1 x 755 [2,5]
+  CRUSH rule 1 x 756 [5,6]
+  CRUSH rule 1 x 757 [8,6]
+  CRUSH rule 1 x 758 [6,2]
+  CRUSH rule 1 x 759 [8,5]
+  CRUSH rule 1 x 760 [2,5]
+  CRUSH rule 1 x 761 [5,2]
+  CRUSH rule 1 x 762 [2,7]
+  CRUSH rule 1 x 763 [8,6]
+  CRUSH rule 1 x 764 [2,7]
+  CRUSH rule 1 x 765 [6,5]
+  CRUSH rule 1 x 766 [8,5]
+  CRUSH rule 1 x 767 [2,6]
+  CRUSH rule 1 x 768 [8,5]
+  CRUSH rule 1 x 769 [6,2]
+  CRUSH rule 1 x 770 [6,2]
+  CRUSH rule 1 x 771 [7,2]
+  CRUSH rule 1 x 772 [8,5]
+  CRUSH rule 1 x 773 [5,2]
+  CRUSH rule 1 x 774 [5,6]
+  CRUSH rule 1 x 775 [6,8]
+  CRUSH rule 1 x 776 [7,2]
+  CRUSH rule 1 x 777 [5,2]
+  CRUSH rule 1 x 778 [2,8]
+  CRUSH rule 1 x 779 [2,7]
+  CRUSH rule 1 x 780 [2,5]
+  CRUSH rule 1 x 781 [6,5]
+  CRUSH rule 1 x 782 [5,2]
+  CRUSH rule 1 x 783 [7,2]
+  CRUSH rule 1 x 784 [2,7]
+  CRUSH rule 1 x 785 [6,2]
+  CRUSH rule 1 x 786 [7,6]
+  CRUSH rule 1 x 787 [2,7]
+  CRUSH rule 1 x 788 [6,2]
+  CRUSH rule 1 x 789 [2,5]
+  CRUSH rule 1 x 790 [8,5]
+  CRUSH rule 1 x 791 [5,8]
+  CRUSH rule 1 x 792 [5,8]
+  CRUSH rule 1 x 793 [6,2]
+  CRUSH rule 1 x 794 [2,6]
+  CRUSH rule 1 x 795 [2,5]
+  CRUSH rule 1 x 796 [5,6]
+  CRUSH rule 1 x 797 [2,5]
+  CRUSH rule 1 x 798 [6,8]
+  CRUSH rule 1 x 799 [5,2]
+  CRUSH rule 1 x 800 [5,2]
+  CRUSH rule 1 x 801 [5,6]
+  CRUSH rule 1 x 802 [2,8]
+  CRUSH rule 1 x 803 [2,5]
+  CRUSH rule 1 x 804 [6,2]
+  CRUSH rule 1 x 805 [5,6]
+  CRUSH rule 1 x 806 [2,5]
+  CRUSH rule 1 x 807 [5,7]
+  CRUSH rule 1 x 808 [5,6]
+  CRUSH rule 1 x 809 [2,5]
+  CRUSH rule 1 x 810 [5,7]
+  CRUSH rule 1 x 811 [8,5]
+  CRUSH rule 1 x 812 [8,5]
+  CRUSH rule 1 x 813 [6,5]
+  CRUSH rule 1 x 814 [5,6]
+  CRUSH rule 1 x 815 [5,2]
+  CRUSH rule 1 x 816 [2,5]
+  CRUSH rule 1 x 817 [5,7]
+  CRUSH rule 1 x 818 [5,2]
+  CRUSH rule 1 x 819 [5,2]
+  CRUSH rule 1 x 820 [5,2]
+  CRUSH rule 1 x 821 [5,2]
+  CRUSH rule 1 x 822 [2,7]
+  CRUSH rule 1 x 823 [5,8]
+  CRUSH rule 1 x 824 [5,7]
+  CRUSH rule 1 x 825 [2,8]
+  CRUSH rule 1 x 826 [7,2]
+  CRUSH rule 1 x 827 [2,8]
+  CRUSH rule 1 x 828 [2,5]
+  CRUSH rule 1 x 829 [5,6]
+  CRUSH rule 1 x 830 [2,5]
+  CRUSH rule 1 x 831 [2,6]
+  CRUSH rule 1 x 832 [5,7]
+  CRUSH rule 1 x 833 [2,7]
+  CRUSH rule 1 x 834 [5]
+  CRUSH rule 1 x 835 [8,5]
+  CRUSH rule 1 x 836 [5,2]
+  CRUSH rule 1 x 837 [6,5]
+  CRUSH rule 1 x 838 [6,7]
+  CRUSH rule 1 x 839 [5,2]
+  CRUSH rule 1 x 840 [7,8]
+  CRUSH rule 1 x 841 [5,8]
+  CRUSH rule 1 x 842 [2,5]
+  CRUSH rule 1 x 843 [6,5]
+  CRUSH rule 1 x 844 [5,8]
+  CRUSH rule 1 x 845 [5,8]
+  CRUSH rule 1 x 846 [5,2]
+  CRUSH rule 1 x 847 [2,8]
+  CRUSH rule 1 x 848 [2,6]
+  CRUSH rule 1 x 849 [5,8]
+  CRUSH rule 1 x 850 [2,5]
+  CRUSH rule 1 x 851 [6,8]
+  CRUSH rule 1 x 852 [7,5]
+  CRUSH rule 1 x 853 [6,8]
+  CRUSH rule 1 x 854 [7,6]
+  CRUSH rule 1 x 855 [5,7]
+  CRUSH rule 1 x 856 [6,7]
+  CRUSH rule 1 x 857 [8,5]
+  CRUSH rule 1 x 858 [6,5]
+  CRUSH rule 1 x 859 [6,2]
+  CRUSH rule 1 x 860 [5,2]
+  CRUSH rule 1 x 861 [8,7]
+  CRUSH rule 1 x 862 [6,2]
+  CRUSH rule 1 x 863 [8,7]
+  CRUSH rule 1 x 864 [5,6]
+  CRUSH rule 1 x 865 [8,2]
+  CRUSH rule 1 x 866 [5,8]
+  CRUSH rule 1 x 867 [6,5]
+  CRUSH rule 1 x 868 [6,5]
+  CRUSH rule 1 x 869 [8,7]
+  CRUSH rule 1 x 870 [2,5]
+  CRUSH rule 1 x 871 [5,7]
+  CRUSH rule 1 x 872 [5,2]
+  CRUSH rule 1 x 873 [5,6]
+  CRUSH rule 1 x 874 [2,6]
+  CRUSH rule 1 x 875 [2,6]
+  CRUSH rule 1 x 876 [5,8]
+  CRUSH rule 1 x 877 [6,5]
+  CRUSH rule 1 x 878 [5,2]
+  CRUSH rule 1 x 879 [7,5]
+  CRUSH rule 1 x 880 [5,2]
+  CRUSH rule 1 x 881 [5,6]
+  CRUSH rule 1 x 882 [5,2]
+  CRUSH rule 1 x 883 [2,7]
+  CRUSH rule 1 x 884 [6,2]
+  CRUSH rule 1 x 885 [5,2]
+  CRUSH rule 1 x 886 [5,6]
+  CRUSH rule 1 x 887 [7,5]
+  CRUSH rule 1 x 888 [6,8]
+  CRUSH rule 1 x 889 [2,5]
+  CRUSH rule 1 x 890 [7,2]
+  CRUSH rule 1 x 891 [2,8]
+  CRUSH rule 1 x 892 [6,2]
+  CRUSH rule 1 x 893 [2,5]
+  CRUSH rule 1 x 894 [7,5]
+  CRUSH rule 1 x 895 [5,7]
+  CRUSH rule 1 x 896 [2,8]
+  CRUSH rule 1 x 897 [5,2]
+  CRUSH rule 1 x 898 [2,5]
+  CRUSH rule 1 x 899 [2,7]
+  CRUSH rule 1 x 900 [5,2]
+  CRUSH rule 1 x 901 [5,2]
+  CRUSH rule 1 x 902 [8,5]
+  CRUSH rule 1 x 903 [5,7]
+  CRUSH rule 1 x 904 [5,6]
+  CRUSH rule 1 x 905 [6,2]
+  CRUSH rule 1 x 906 [2,5]
+  CRUSH rule 1 x 907 [7,2]
+  CRUSH rule 1 x 908 [5,8]
+  CRUSH rule 1 x 909 [2,5]
+  CRUSH rule 1 x 910 [6,5]
+  CRUSH rule 1 x 911 [5,8]
+  CRUSH rule 1 x 912 [2,6]
+  CRUSH rule 1 x 913 [7,6]
+  CRUSH rule 1 x 914 [6,5]
+  CRUSH rule 1 x 915 [8,2]
+  CRUSH rule 1 x 916 [5,2]
+  CRUSH rule 1 x 917 [2,5]
+  CRUSH rule 1 x 918 [8,2]
+  CRUSH rule 1 x 919 [6,2]
+  CRUSH rule 1 x 920 [7,6]
+  CRUSH rule 1 x 921 [2,5]
+  CRUSH rule 1 x 922 [6,7]
+  CRUSH rule 1 x 923 [5,8]
+  CRUSH rule 1 x 924 [5,6]
+  CRUSH rule 1 x 925 [5,7]
+  CRUSH rule 1 x 926 [5,8]
+  CRUSH rule 1 x 927 [2,6]
+  CRUSH rule 1 x 928 [8,2]
+  CRUSH rule 1 x 929 [5,2]
+  CRUSH rule 1 x 930 [2,5]
+  CRUSH rule 1 x 931 [5,2]
+  CRUSH rule 1 x 932 [5,8]
+  CRUSH rule 1 x 933 [8,5]
+  CRUSH rule 1 x 934 [5,2]
+  CRUSH rule 1 x 935 [6,5]
+  CRUSH rule 1 x 936 [2,6]
+  CRUSH rule 1 x 937 [5,6]
+  CRUSH rule 1 x 938 [6,5]
+  CRUSH rule 1 x 939 [2,7]
+  CRUSH rule 1 x 940 [8,7]
+  CRUSH rule 1 x 941 [5,2]
+  CRUSH rule 1 x 942 [2,8]
+  CRUSH rule 1 x 943 [8,2]
+  CRUSH rule 1 x 944 [5,2]
+  CRUSH rule 1 x 945 [7,2]
+  CRUSH rule 1 x 946 [2,5]
+  CRUSH rule 1 x 947 [5,6]
+  CRUSH rule 1 x 948 [7,8]
+  CRUSH rule 1 x 949 [6,2]
+  CRUSH rule 1 x 950 [5,7]
+  CRUSH rule 1 x 951 [5,8]
+  CRUSH rule 1 x 952 [2,5]
+  CRUSH rule 1 x 953 [2,5]
+  CRUSH rule 1 x 954 [5,2]
+  CRUSH rule 1 x 955 [8,6]
+  CRUSH rule 1 x 956 [2,8]
+  CRUSH rule 1 x 957 [7,6]
+  CRUSH rule 1 x 958 [8,7]
+  CRUSH rule 1 x 959 [5,2]
+  CRUSH rule 1 x 960 [5,6]
+  CRUSH rule 1 x 961 [5,2]
+  CRUSH rule 1 x 962 [7,5]
+  CRUSH rule 1 x 963 [2,5]
+  CRUSH rule 1 x 964 [5,2]
+  CRUSH rule 1 x 965 [7,6]
+  CRUSH rule 1 x 966 [5,8]
+  CRUSH rule 1 x 967 [8,6]
+  CRUSH rule 1 x 968 [7,2]
+  CRUSH rule 1 x 969 [8,2]
+  CRUSH rule 1 x 970 [2,6]
+  CRUSH rule 1 x 971 [2,7]
+  CRUSH rule 1 x 972 [2,8]
+  CRUSH rule 1 x 973 [2,6]
+  CRUSH rule 1 x 974 [5,2]
+  CRUSH rule 1 x 975 [5,7]
+  CRUSH rule 1 x 976 [5,6]
+  CRUSH rule 1 x 977 [8,5]
+  CRUSH rule 1 x 978 [7,2]
+  CRUSH rule 1 x 979 [7,6]
+  CRUSH rule 1 x 980 [6,2]
+  CRUSH rule 1 x 981 [7,5]
+  CRUSH rule 1 x 982 [5,2]
+  CRUSH rule 1 x 983 [5,6]
+  CRUSH rule 1 x 984 [2,8]
+  CRUSH rule 1 x 985 [2,5]
+  CRUSH rule 1 x 986 [8,7]
+  CRUSH rule 1 x 987 [2,5]
+  CRUSH rule 1 x 988 [2,5]
+  CRUSH rule 1 x 989 [2,6]
+  CRUSH rule 1 x 990 [2,5]
+  CRUSH rule 1 x 991 [2,5]
+  CRUSH rule 1 x 992 [7,2]
+  CRUSH rule 1 x 993 [2,6]
+  CRUSH rule 1 x 994 [5,2]
+  CRUSH rule 1 x 995 [7,6]
+  CRUSH rule 1 x 996 [6,7]
+  CRUSH rule 1 x 997 [6,5]
+  CRUSH rule 1 x 998 [8,2]
+  CRUSH rule 1 x 999 [2,7]
+  CRUSH rule 1 x 1000 [8,5]
+  CRUSH rule 1 x 1001 [2,5]
+  CRUSH rule 1 x 1002 [2,5]
+  CRUSH rule 1 x 1003 [2,8]
+  CRUSH rule 1 x 1004 [6,2]
+  CRUSH rule 1 x 1005 [6,2]
+  CRUSH rule 1 x 1006 [2,5]
+  CRUSH rule 1 x 1007 [2,8]
+  CRUSH rule 1 x 1008 [2,7]
+  CRUSH rule 1 x 1009 [6,8]
+  CRUSH rule 1 x 1010 [5,6]
+  CRUSH rule 1 x 1011 [5,2]
+  CRUSH rule 1 x 1012 [5,2]
+  CRUSH rule 1 x 1013 [5,2]
+  CRUSH rule 1 x 1014 [2,8]
+  CRUSH rule 1 x 1015 [6,8]
+  CRUSH rule 1 x 1016 [2,6]
+  CRUSH rule 1 x 1017 [6,2]
+  CRUSH rule 1 x 1018 [5,2]
+  CRUSH rule 1 x 1019 [5,7]
+  CRUSH rule 1 x 1020 [5,2]
+  CRUSH rule 1 x 1021 [5,2]
+  CRUSH rule 1 x 1022 [2,6]
+  CRUSH rule 1 x 1023 [5,2]
+  rule 1 (choose-two) num_rep 2 result size == 1:\t4/1024 (esc)
+  rule 1 (choose-two) num_rep 2 result size == 2:\t1020/1024 (esc)
+  CRUSH rule 1 x 0 [2,8,5]
+  CRUSH rule 1 x 1 [2,8,6]
+  CRUSH rule 1 x 2 [2,5,8]
+  CRUSH rule 1 x 3 [8,2,6]
+  CRUSH rule 1 x 4 [5,7,2]
+  CRUSH rule 1 x 5 [7,8,2]
+  CRUSH rule 1 x 6 [2,6,7]
+  CRUSH rule 1 x 7 [5,7,8]
+  CRUSH rule 1 x 8 [5,2,8]
+  CRUSH rule 1 x 9 [2,5,6]
+  CRUSH rule 1 x 10 [2,8,6]
+  CRUSH rule 1 x 11 [2,7,5]
+  CRUSH rule 1 x 12 [2,7,5]
+  CRUSH rule 1 x 13 [5,8,2]
+  CRUSH rule 1 x 14 [7,6,8]
+  CRUSH rule 1 x 15 [7,2,5]
+  CRUSH rule 1 x 16 [5,6,2]
+  CRUSH rule 1 x 17 [5,7,2]
+  CRUSH rule 1 x 18 [2,5,8]
+  CRUSH rule 1 x 19 [7,5,8]
+  CRUSH rule 1 x 20 [2,5,7]
+  CRUSH rule 1 x 21 [5,7,2]
+  CRUSH rule 1 x 22 [8,5,7]
+  CRUSH rule 1 x 23 [5,6,8]
+  CRUSH rule 1 x 24 [2,8,6]
+  CRUSH rule 1 x 25 [5,7,2]
+  CRUSH rule 1 x 26 [2,8,7]
+  CRUSH rule 1 x 27 [5,2]
+  CRUSH rule 1 x 28 [6,2,5]
+  CRUSH rule 1 x 29 [8,5,7]
+  CRUSH rule 1 x 30 [5,7,2]
+  CRUSH rule 1 x 31 [8,7,6]
+  CRUSH rule 1 x 32 [5,6,8]
+  CRUSH rule 1 x 33 [2,7,8]
+  CRUSH rule 1 x 34 [2,5,7]
+  CRUSH rule 1 x 35 [2,8,6]
+  CRUSH rule 1 x 36 [5,8,2]
+  CRUSH rule 1 x 37 [2,5,8]
+  CRUSH rule 1 x 38 [5,8,7]
+  CRUSH rule 1 x 39 [5,7,2]
+  CRUSH rule 1 x 40 [7,8,2]
+  CRUSH rule 1 x 41 [2,6,8]
+  CRUSH rule 1 x 42 [5,2,6]
+  CRUSH rule 1 x 43 [2,5,7]
+  CRUSH rule 1 x 44 [2,6,7]
+  CRUSH rule 1 x 45 [8,2,5]
+  CRUSH rule 1 x 46 [2,5,7]
+  CRUSH rule 1 x 47 [5,2,8]
+  CRUSH rule 1 x 48 [5,6,7]
+  CRUSH rule 1 x 49 [5,2,8]
+  CRUSH rule 1 x 50 [5,6,2]
+  CRUSH rule 1 x 51 [5,6,2]
+  CRUSH rule 1 x 52 [8,6,2]
+  CRUSH rule 1 x 53 [5,2,8]
+  CRUSH rule 1 x 54 [7,6,8]
+  CRUSH rule 1 x 55 [8,7,2]
+  CRUSH rule 1 x 56 [6,5,8]
+  CRUSH rule 1 x 57 [5,2,6]
+  CRUSH rule 1 x 58 [2,5,6]
+  CRUSH rule 1 x 59 [5,2,7]
+  CRUSH rule 1 x 60 [5,7,2]
+  CRUSH rule 1 x 61 [5,6,8]
+  CRUSH rule 1 x 62 [7,2,6]
+  CRUSH rule 1 x 63 [5,6,8]
+  CRUSH rule 1 x 64 [5,2,8]
+  CRUSH rule 1 x 65 [7,5,6]
+  CRUSH rule 1 x 66 [5,2,6]
+  CRUSH rule 1 x 67 [5,2,8]
+  CRUSH rule 1 x 68 [2,5,8]
+  CRUSH rule 1 x 69 [5,2,6]
+  CRUSH rule 1 x 70 [7,2,5]
+  CRUSH rule 1 x 71 [2,8,6]
+  CRUSH rule 1 x 72 [6,2,5]
+  CRUSH rule 1 x 73 [2,7,5]
+  CRUSH rule 1 x 74 [2,7,5]
+  CRUSH rule 1 x 75 [5,2,7]
+  CRUSH rule 1 x 76 [5,2,7]
+  CRUSH rule 1 x 77 [7,2,6]
+  CRUSH rule 1 x 78 [2,5,8]
+  CRUSH rule 1 x 79 [5,2,7]
+  CRUSH rule 1 x 80 [2,5,7]
+  CRUSH rule 1 x 81 [2,5]
+  CRUSH rule 1 x 82 [7,2,6]
+  CRUSH rule 1 x 83 [2,6,8]
+  CRUSH rule 1 x 84 [7,2,6]
+  CRUSH rule 1 x 85 [5,8,7]
+  CRUSH rule 1 x 86 [2,7,8]
+  CRUSH rule 1 x 87 [2,7,5]
+  CRUSH rule 1 x 88 [2,6,5]
+  CRUSH rule 1 x 89 [5,2,6]
+  CRUSH rule 1 x 90 [6,7,5]
+  CRUSH rule 1 x 91 [5,8,7]
+  CRUSH rule 1 x 92 [2,8,5]
+  CRUSH rule 1 x 93 [7,5,8]
+  CRUSH rule 1 x 94 [2,5,7]
+  CRUSH rule 1 x 95 [7,5,2]
+  CRUSH rule 1 x 96 [5,6,7]
+  CRUSH rule 1 x 97 [8,7,5]
+  CRUSH rule 1 x 98 [2,6,7]
+  CRUSH rule 1 x 99 [2,7,8]
+  CRUSH rule 1 x 100 [2,7,5]
+  CRUSH rule 1 x 101 [5,7,2]
+  CRUSH rule 1 x 102 [5,2,7]
+  CRUSH rule 1 x 103 [5,7,6]
+  CRUSH rule 1 x 104 [7,5,2]
+  CRUSH rule 1 x 105 [2,5,6]
+  CRUSH rule 1 x 106 [2,6,5]
+  CRUSH rule 1 x 107 [5,2,7]
+  CRUSH rule 1 x 108 [7,2,8]
+  CRUSH rule 1 x 109 [2,5,8]
+  CRUSH rule 1 x 110 [5,2,7]
+  CRUSH rule 1 x 111 [2,5,8]
+  CRUSH rule 1 x 112 [2,5,6]
+  CRUSH rule 1 x 113 [6,2,8]
+  CRUSH rule 1 x 114 [7,6,5]
+  CRUSH rule 1 x 115 [8,2,5]
+  CRUSH rule 1 x 116 [2,6,7]
+  CRUSH rule 1 x 117 [7,5,6]
+  CRUSH rule 1 x 118 [2,5,8]
+  CRUSH rule 1 x 119 [5,6,2]
+  CRUSH rule 1 x 120 [2,5,8]
+  CRUSH rule 1 x 121 [2,5,7]
+  CRUSH rule 1 x 122 [8,5,2]
+  CRUSH rule 1 x 123 [2,5]
+  CRUSH rule 1 x 124 [5,8,2]
+  CRUSH rule 1 x 125 [2,7,5]
+  CRUSH rule 1 x 126 [5,2,6]
+  CRUSH rule 1 x 127 [5,6,7]
+  CRUSH rule 1 x 128 [5,8,6]
+  CRUSH rule 1 x 129 [2,7,6]
+  CRUSH rule 1 x 130 [5,8,2]
+  CRUSH rule 1 x 131 [2,8,7]
+  CRUSH rule 1 x 132 [2,7,5]
+  CRUSH rule 1 x 133 [5,6,7]
+  CRUSH rule 1 x 134 [2,8,7]
+  CRUSH rule 1 x 135 [5,6,2]
+  CRUSH rule 1 x 136 [2,5,6]
+  CRUSH rule 1 x 137 [7,5,6]
+  CRUSH rule 1 x 138 [8,7,6]
+  CRUSH rule 1 x 139 [5,2,7]
+  CRUSH rule 1 x 140 [2,6,7]
+  CRUSH rule 1 x 141 [6,8,2]
+  CRUSH rule 1 x 142 [5,6,2]
+  CRUSH rule 1 x 143 [5,8,7]
+  CRUSH rule 1 x 144 [8,2,5]
+  CRUSH rule 1 x 145 [8,5,6]
+  CRUSH rule 1 x 146 [2,6,5]
+  CRUSH rule 1 x 147 [2,8,7]
+  CRUSH rule 1 x 148 [5,2,6]
+  CRUSH rule 1 x 149 [5,8,6]
+  CRUSH rule 1 x 150 [2,6,5]
+  CRUSH rule 1 x 151 [5,2]
+  CRUSH rule 1 x 152 [8,5,6]
+  CRUSH rule 1 x 153 [8,6,7]
+  CRUSH rule 1 x 154 [5,2,6]
+  CRUSH rule 1 x 155 [5,7,8]
+  CRUSH rule 1 x 156 [5,8,7]
+  CRUSH rule 1 x 157 [5,2,7]
+  CRUSH rule 1 x 158 [2,8,7]
+  CRUSH rule 1 x 159 [7,2,5]
+  CRUSH rule 1 x 160 [2,8,7]
+  CRUSH rule 1 x 161 [2,5,6]
+  CRUSH rule 1 x 162 [2,6,7]
+  CRUSH rule 1 x 163 [5,6,2]
+  CRUSH rule 1 x 164 [7,8,2]
+  CRUSH rule 1 x 165 [7,2,5]
+  CRUSH rule 1 x 166 [2,5,8]
+  CRUSH rule 1 x 167 [2,6,8]
+  CRUSH rule 1 x 168 [5,2,6]
+  CRUSH rule 1 x 169 [2,6,8]
+  CRUSH rule 1 x 170 [2,8,5]
+  CRUSH rule 1 x 171 [7,5,8]
+  CRUSH rule 1 x 172 [2,7,8]
+  CRUSH rule 1 x 173 [8,5,6]
+  CRUSH rule 1 x 174 [2,5,7]
+  CRUSH rule 1 x 175 [6,2,5]
+  CRUSH rule 1 x 176 [5,2,6]
+  CRUSH rule 1 x 177 [5,8,2]
+  CRUSH rule 1 x 178 [5,2,7]
+  CRUSH rule 1 x 179 [5,2,8]
+  CRUSH rule 1 x 180 [5,8,7]
+  CRUSH rule 1 x 181 [6,2,8]
+  CRUSH rule 1 x 182 [8,5,2]
+  CRUSH rule 1 x 183 [7,8,6]
+  CRUSH rule 1 x 184 [5,7,6]
+  CRUSH rule 1 x 185 [6,8,2]
+  CRUSH rule 1 x 186 [2,8,5]
+  CRUSH rule 1 x 187 [2,6,7]
+  CRUSH rule 1 x 188 [2,8,7]
+  CRUSH rule 1 x 189 [2,7,8]
+  CRUSH rule 1 x 190 [5,2,8]
+  CRUSH rule 1 x 191 [7,6,8]
+  CRUSH rule 1 x 192 [5,2,7]
+  CRUSH rule 1 x 193 [5,2,6]
+  CRUSH rule 1 x 194 [2,5,6]
+  CRUSH rule 1 x 195 [6,5,8]
+  CRUSH rule 1 x 196 [6,7,2]
+  CRUSH rule 1 x 197 [6,5,2]
+  CRUSH rule 1 x 198 [2,5,6]
+  CRUSH rule 1 x 199 [2,5,7]
+  CRUSH rule 1 x 200 [2,5,7]
+  CRUSH rule 1 x 201 [7,2,8]
+  CRUSH rule 1 x 202 [6,5,2]
+  CRUSH rule 1 x 203 [5,8,7]
+  CRUSH rule 1 x 204 [2,5,8]
+  CRUSH rule 1 x 205 [2,7,6]
+  CRUSH rule 1 x 206 [2,5,7]
+  CRUSH rule 1 x 207 [5,2,8]
+  CRUSH rule 1 x 208 [7,2,8]
+  CRUSH rule 1 x 209 [2,5,7]
+  CRUSH rule 1 x 210 [2,8,5]
+  CRUSH rule 1 x 211 [5,6,2]
+  CRUSH rule 1 x 212 [7,5,2]
+  CRUSH rule 1 x 213 [8,5,2]
+  CRUSH rule 1 x 214 [5,7,8]
+  CRUSH rule 1 x 215 [8,2,7]
+  CRUSH rule 1 x 216 [5,2,6]
+  CRUSH rule 1 x 217 [2,5,7]
+  CRUSH rule 1 x 218 [2,7,8]
+  CRUSH rule 1 x 219 [5,8,2]
+  CRUSH rule 1 x 220 [5,7,8]
+  CRUSH rule 1 x 221 [5,6,8]
+  CRUSH rule 1 x 222 [6,8,5]
+  CRUSH rule 1 x 223 [2,5,6]
+  CRUSH rule 1 x 224 [2,5,8]
+  CRUSH rule 1 x 225 [8,6,2]
+  CRUSH rule 1 x 226 [7,2,5]
+  CRUSH rule 1 x 227 [5,6,2]
+  CRUSH rule 1 x 228 [5,6,8]
+  CRUSH rule 1 x 229 [5,2,7]
+  CRUSH rule 1 x 230 [5,7,2]
+  CRUSH rule 1 x 231 [5,6,2]
+  CRUSH rule 1 x 232 [2,7,5]
+  CRUSH rule 1 x 233 [5,8,7]
+  CRUSH rule 1 x 234 [2,7,5]
+  CRUSH rule 1 x 235 [5,8,2]
+  CRUSH rule 1 x 236 [5,2,7]
+  CRUSH rule 1 x 237 [5,7,6]
+  CRUSH rule 1 x 238 [5,6,2]
+  CRUSH rule 1 x 239 [8,7,6]
+  CRUSH rule 1 x 240 [5,7,2]
+  CRUSH rule 1 x 241 [5,2,8]
+  CRUSH rule 1 x 242 [5,8,2]
+  CRUSH rule 1 x 243 [5,7,8]
+  CRUSH rule 1 x 244 [5,6,2]
+  CRUSH rule 1 x 245 [7,6,2]
+  CRUSH rule 1 x 246 [2,5,7]
+  CRUSH rule 1 x 247 [6,2,7]
+  CRUSH rule 1 x 248 [8,2,7]
+  CRUSH rule 1 x 249 [2,5,8]
+  CRUSH rule 1 x 250 [2,7,5]
+  CRUSH rule 1 x 251 [2,5,6]
+  CRUSH rule 1 x 252 [5,7,8]
+  CRUSH rule 1 x 253 [5,2,8]
+  CRUSH rule 1 x 254 [5,6,7]
+  CRUSH rule 1 x 255 [2,7,8]
+  CRUSH rule 1 x 256 [5,7,2]
+  CRUSH rule 1 x 257 [2,8,5]
+  CRUSH rule 1 x 258 [5,2]
+  CRUSH rule 1 x 259 [5,7,6]
+  CRUSH rule 1 x 260 [5,6,2]
+  CRUSH rule 1 x 261 [8,7,5]
+  CRUSH rule 1 x 262 [5,2,8]
+  CRUSH rule 1 x 263 [6,8,2]
+  CRUSH rule 1 x 264 [5,6,2]
+  CRUSH rule 1 x 265 [8,6,7]
+  CRUSH rule 1 x 266 [8,2,5]
+  CRUSH rule 1 x 267 [2,5,8]
+  CRUSH rule 1 x 268 [2,7,5]
+  CRUSH rule 1 x 269 [2,8,7]
+  CRUSH rule 1 x 270 [5,2,7]
+  CRUSH rule 1 x 271 [7,5,6]
+  CRUSH rule 1 x 272 [2,8,5]
+  CRUSH rule 1 x 273 [5,2]
+  CRUSH rule 1 x 274 [6,8,5]
+  CRUSH rule 1 x 275 [5,8,6]
+  CRUSH rule 1 x 276 [7,2,6]
+  CRUSH rule 1 x 277 [6,5,2]
+  CRUSH rule 1 x 278 [6,8,2]
+  CRUSH rule 1 x 279 [8,5,6]
+  CRUSH rule 1 x 280 [2,6,7]
+  CRUSH rule 1 x 281 [8,2,6]
+  CRUSH rule 1 x 282 [5,2,6]
+  CRUSH rule 1 x 283 [8,2,7]
+  CRUSH rule 1 x 284 [6,5,2]
+  CRUSH rule 1 x 285 [5,6,7]
+  CRUSH rule 1 x 286 [2,5,6]
+  CRUSH rule 1 x 287 [2,5,6]
+  CRUSH rule 1 x 288 [8,2,7]
+  CRUSH rule 1 x 289 [5,6,2]
+  CRUSH rule 1 x 290 [2,5,7]
+  CRUSH rule 1 x 291 [2,5,7]
+  CRUSH rule 1 x 292 [8,2,7]
+  CRUSH rule 1 x 293 [6,2,8]
+  CRUSH rule 1 x 294 [7,5,2]
+  CRUSH rule 1 x 295 [5,8,7]
+  CRUSH rule 1 x 296 [5,2,6]
+  CRUSH rule 1 x 297 [6,2,8]
+  CRUSH rule 1 x 298 [2,8,5]
+  CRUSH rule 1 x 299 [2,5,7]
+  CRUSH rule 1 x 300 [8,7,5]
+  CRUSH rule 1 x 301 [2,8,7]
+  CRUSH rule 1 x 302 [5,2,6]
+  CRUSH rule 1 x 303 [7,5,8]
+  CRUSH rule 1 x 304 [2,7,5]
+  CRUSH rule 1 x 305 [5,8,2]
+  CRUSH rule 1 x 306 [2,7,5]
+  CRUSH rule 1 x 307 [2,7,8]
+  CRUSH rule 1 x 308 [2,8,5]
+  CRUSH rule 1 x 309 [7,5,6]
+  CRUSH rule 1 x 310 [5,6,2]
+  CRUSH rule 1 x 311 [5,2,7]
+  CRUSH rule 1 x 312 [2,6,5]
+  CRUSH rule 1 x 313 [5,8,6]
+  CRUSH rule 1 x 314 [5,2,6]
+  CRUSH rule 1 x 315 [2,5,8]
+  CRUSH rule 1 x 316 [6,5,8]
+  CRUSH rule 1 x 317 [2,6,7]
+  CRUSH rule 1 x 318 [8,6,7]
+  CRUSH rule 1 x 319 [5,2,8]
+  CRUSH rule 1 x 320 [5,7,2]
+  CRUSH rule 1 x 321 [2,5]
+  CRUSH rule 1 x 322 [2,7,5]
+  CRUSH rule 1 x 323 [5,7,2]
+  CRUSH rule 1 x 324 [7,2,8]
+  CRUSH rule 1 x 325 [5,6,2]
+  CRUSH rule 1 x 326 [5,6,2]
+  CRUSH rule 1 x 327 [2,6,7]
+  CRUSH rule 1 x 328 [7,5,8]
+  CRUSH rule 1 x 329 [5,6,2]
+  CRUSH rule 1 x 330 [5,7,2]
+  CRUSH rule 1 x 331 [2,6,5]
+  CRUSH rule 1 x 332 [2,5,8]
+  CRUSH rule 1 x 333 [6,8,5]
+  CRUSH rule 1 x 334 [8,5,6]
+  CRUSH rule 1 x 335 [7,2,5]
+  CRUSH rule 1 x 336 [5,6,2]
+  CRUSH rule 1 x 337 [7,2,6]
+  CRUSH rule 1 x 338 [5,6,8]
+  CRUSH rule 1 x 339 [7,5,2]
+  CRUSH rule 1 x 340 [2,5,6]
+  CRUSH rule 1 x 341 [5,2,7]
+  CRUSH rule 1 x 342 [2,7,5]
+  CRUSH rule 1 x 343 [6,7,5]
+  CRUSH rule 1 x 344 [6,2,5]
+  CRUSH rule 1 x 345 [5,2,7]
+  CRUSH rule 1 x 346 [8,2,5]
+  CRUSH rule 1 x 347 [5,2,6]
+  CRUSH rule 1 x 348 [8,2,7]
+  CRUSH rule 1 x 349 [2,6,7]
+  CRUSH rule 1 x 350 [8,5,7]
+  CRUSH rule 1 x 351 [5,6,2]
+  CRUSH rule 1 x 352 [2,7,5]
+  CRUSH rule 1 x 353 [6,5,8]
+  CRUSH rule 1 x 354 [2,5,8]
+  CRUSH rule 1 x 355 [5,2,6]
+  CRUSH rule 1 x 356 [5,8,2]
+  CRUSH rule 1 x 357 [6,2,7]
+  CRUSH rule 1 x 358 [2,5,8]
+  CRUSH rule 1 x 359 [6,7,8]
+  CRUSH rule 1 x 360 [5,2,6]
+  CRUSH rule 1 x 361 [8,5,2]
+  CRUSH rule 1 x 362 [5,7,6]
+  CRUSH rule 1 x 363 [5,2,6]
+  CRUSH rule 1 x 364 [2,5,7]
+  CRUSH rule 1 x 365 [6,7,8]
+  CRUSH rule 1 x 366 [7,2,8]
+  CRUSH rule 1 x 367 [5,7,2]
+  CRUSH rule 1 x 368 [7,5,6]
+  CRUSH rule 1 x 369 [5,7,2]
+  CRUSH rule 1 x 370 [8,7,6]
+  CRUSH rule 1 x 371 [2,5,8]
+  CRUSH rule 1 x 372 [5,2,8]
+  CRUSH rule 1 x 373 [2,6,8]
+  CRUSH rule 1 x 374 [5,8,7]
+  CRUSH rule 1 x 375 [6,5,7]
+  CRUSH rule 1 x 376 [7,2,5]
+  CRUSH rule 1 x 377 [2,5,6]
+  CRUSH rule 1 x 378 [2,6,7]
+  CRUSH rule 1 x 379 [8,5,7]
+  CRUSH rule 1 x 380 [2,5]
+  CRUSH rule 1 x 381 [2,5,7]
+  CRUSH rule 1 x 382 [2,5,6]
+  CRUSH rule 1 x 383 [5,2]
+  CRUSH rule 1 x 384 [7,2,6]
+  CRUSH rule 1 x 385 [7,5,6]
+  CRUSH rule 1 x 386 [2,5,8]
+  CRUSH rule 1 x 387 [2,5,6]
+  CRUSH rule 1 x 388 [5,2,8]
+  CRUSH rule 1 x 389 [2,5,8]
+  CRUSH rule 1 x 390 [5,6,2]
+  CRUSH rule 1 x 391 [5,6,2]
+  CRUSH rule 1 x 392 [2,8,6]
+  CRUSH rule 1 x 393 [5,2,6]
+  CRUSH rule 1 x 394 [5,7,6]
+  CRUSH rule 1 x 395 [5,2,8]
+  CRUSH rule 1 x 396 [5,2,8]
+  CRUSH rule 1 x 397 [2,7,5]
+  CRUSH rule 1 x 398 [2,5,6]
+  CRUSH rule 1 x 399 [8,7,5]
+  CRUSH rule 1 x 400 [8,2,5]
+  CRUSH rule 1 x 401 [2,5,6]
+  CRUSH rule 1 x 402 [7,8,6]
+  CRUSH rule 1 x 403 [2,7,5]
+  CRUSH rule 1 x 404 [5,2,7]
+  CRUSH rule 1 x 405 [6,5,2]
+  CRUSH rule 1 x 406 [2,6,5]
+  CRUSH rule 1 x 407 [2,8,7]
+  CRUSH rule 1 x 408 [5,2,6]
+  CRUSH rule 1 x 409 [7,5,6]
+  CRUSH rule 1 x 410 [8,6,5]
+  CRUSH rule 1 x 411 [2,6,8]
+  CRUSH rule 1 x 412 [2,5,8]
+  CRUSH rule 1 x 413 [5,2,8]
+  CRUSH rule 1 x 414 [5,2,8]
+  CRUSH rule 1 x 415 [2,6,5]
+  CRUSH rule 1 x 416 [2,5,7]
+  CRUSH rule 1 x 417 [8,7,2]
+  CRUSH rule 1 x 418 [7,6,8]
+  CRUSH rule 1 x 419 [8,5,2]
+  CRUSH rule 1 x 420 [2,5,8]
+  CRUSH rule 1 x 421 [8,6,7]
+  CRUSH rule 1 x 422 [6,7,8]
+  CRUSH rule 1 x 423 [2,5,6]
+  CRUSH rule 1 x 424 [8,5,7]
+  CRUSH rule 1 x 425 [2,5,7]
+  CRUSH rule 1 x 426 [6,7,2]
+  CRUSH rule 1 x 427 [2,7,5]
+  CRUSH rule 1 x 428 [5,6,7]
+  CRUSH rule 1 x 429 [5,6,7]
+  CRUSH rule 1 x 430 [5,6,2]
+  CRUSH rule 1 x 431 [5,2]
+  CRUSH rule 1 x 432 [7,2,8]
+  CRUSH rule 1 x 433 [6,5,2]
+  CRUSH rule 1 x 434 [5,2,6]
+  CRUSH rule 1 x 435 [2,5,8]
+  CRUSH rule 1 x 436 [5,2,6]
+  CRUSH rule 1 x 437 [7,5,2]
+  CRUSH rule 1 x 438 [2,5,8]
+  CRUSH rule 1 x 439 [2,5,7]
+  CRUSH rule 1 x 440 [2,7,6]
+  CRUSH rule 1 x 441 [5,7,2]
+  CRUSH rule 1 x 442 [2,5,7]
+  CRUSH rule 1 x 443 [6,8,2]
+  CRUSH rule 1 x 444 [7,2,8]
+  CRUSH rule 1 x 445 [6,5,7]
+  CRUSH rule 1 x 446 [5,7,6]
+  CRUSH rule 1 x 447 [2,5,8]
+  CRUSH rule 1 x 448 [7,2,5]
+  CRUSH rule 1 x 449 [7,8,5]
+  CRUSH rule 1 x 450 [5,8,2]
+  CRUSH rule 1 x 451 [6,8,5]
+  CRUSH rule 1 x 452 [8,5,6]
+  CRUSH rule 1 x 453 [6,8,7]
+  CRUSH rule 1 x 454 [6,7,5]
+  CRUSH rule 1 x 455 [2,7,5]
+  CRUSH rule 1 x 456 [6,8,7]
+  CRUSH rule 1 x 457 [7,2,8]
+  CRUSH rule 1 x 458 [2,8,7]
+  CRUSH rule 1 x 459 [2,5,6]
+  CRUSH rule 1 x 460 [6,5,7]
+  CRUSH rule 1 x 461 [6,5,8]
+  CRUSH rule 1 x 462 [8,2,5]
+  CRUSH rule 1 x 463 [6,7,2]
+  CRUSH rule 1 x 464 [7,5,2]
+  CRUSH rule 1 x 465 [7,6,2]
+  CRUSH rule 1 x 466 [5,8,7]
+  CRUSH rule 1 x 467 [6,5,7]
+  CRUSH rule 1 x 468 [7,8,2]
+  CRUSH rule 1 x 469 [7,2,6]
+  CRUSH rule 1 x 470 [5,2,6]
+  CRUSH rule 1 x 471 [2,6,8]
+  CRUSH rule 1 x 472 [5,6,2]
+  CRUSH rule 1 x 473 [2,5,6]
+  CRUSH rule 1 x 474 [6,2,7]
+  CRUSH rule 1 x 475 [6,7,8]
+  CRUSH rule 1 x 476 [5,2]
+  CRUSH rule 1 x 477 [5,8,6]
+  CRUSH rule 1 x 478 [6,7,2]
+  CRUSH rule 1 x 479 [2,5,8]
+  CRUSH rule 1 x 480 [2,8,6]
+  CRUSH rule 1 x 481 [2,5,7]
+  CRUSH rule 1 x 482 [5,7,2]
+  CRUSH rule 1 x 483 [2,7,6]
+  CRUSH rule 1 x 484 [2,7,6]
+  CRUSH rule 1 x 485 [5,7,2]
+  CRUSH rule 1 x 486 [5,2,7]
+  CRUSH rule 1 x 487 [5,2]
+  CRUSH rule 1 x 488 [5,7,2]
+  CRUSH rule 1 x 489 [2,8,5]
+  CRUSH rule 1 x 490 [6,5,2]
+  CRUSH rule 1 x 491 [2,6,7]
+  CRUSH rule 1 x 492 [6,5,2]
+  CRUSH rule 1 x 493 [2,8,7]
+  CRUSH rule 1 x 494 [2,7,8]
+  CRUSH rule 1 x 495 [5,6,2]
+  CRUSH rule 1 x 496 [7,5,6]
+  CRUSH rule 1 x 497 [5,7,6]
+  CRUSH rule 1 x 498 [2,5,8]
+  CRUSH rule 1 x 499 [8,5,2]
+  CRUSH rule 1 x 500 [5,6,2]
+  CRUSH rule 1 x 501 [2,7,5]
+  CRUSH rule 1 x 502 [7,2,5]
+  CRUSH rule 1 x 503 [2,5,6]
+  CRUSH rule 1 x 504 [5,6,2]
+  CRUSH rule 1 x 505 [2,7,6]
+  CRUSH rule 1 x 506 [5,2]
+  CRUSH rule 1 x 507 [6,2,8]
+  CRUSH rule 1 x 508 [2,7,8]
+  CRUSH rule 1 x 509 [7,5,8]
+  CRUSH rule 1 x 510 [6,2,5]
+  CRUSH rule 1 x 511 [5,8,7]
+  CRUSH rule 1 x 512 [7,6,2]
+  CRUSH rule 1 x 513 [7,2,5]
+  CRUSH rule 1 x 514 [5,7,8]
+  CRUSH rule 1 x 515 [8,5,6]
+  CRUSH rule 1 x 516 [5,2,6]
+  CRUSH rule 1 x 517 [7,8,6]
+  CRUSH rule 1 x 518 [5,6,7]
+  CRUSH rule 1 x 519 [7,5,8]
+  CRUSH rule 1 x 520 [2,6,5]
+  CRUSH rule 1 x 521 [8,7,6]
+  CRUSH rule 1 x 522 [6,8,2]
+  CRUSH rule 1 x 523 [5,2,6]
+  CRUSH rule 1 x 524 [2,5,8]
+  CRUSH rule 1 x 525 [2,5,7]
+  CRUSH rule 1 x 526 [2,5,8]
+  CRUSH rule 1 x 527 [2,5,8]
+  CRUSH rule 1 x 528 [5,2,7]
+  CRUSH rule 1 x 529 [5,7,2]
+  CRUSH rule 1 x 530 [6,7,8]
+  CRUSH rule 1 x 531 [6,2,5]
+  CRUSH rule 1 x 532 [6,5,7]
+  CRUSH rule 1 x 533 [5,6,2]
+  CRUSH rule 1 x 534 [7,5,2]
+  CRUSH rule 1 x 535 [8,6,2]
+  CRUSH rule 1 x 536 [6,7,2]
+  CRUSH rule 1 x 537 [5,7,6]
+  CRUSH rule 1 x 538 [6,8,5]
+  CRUSH rule 1 x 539 [8,5,7]
+  CRUSH rule 1 x 540 [2,6,5]
+  CRUSH rule 1 x 541 [2,5,7]
+  CRUSH rule 1 x 542 [5,2,8]
+  CRUSH rule 1 x 543 [6,2,8]
+  CRUSH rule 1 x 544 [5,7,6]
+  CRUSH rule 1 x 545 [5,7,2]
+  CRUSH rule 1 x 546 [6,2,7]
+  CRUSH rule 1 x 547 [8,2,5]
+  CRUSH rule 1 x 548 [5,2,7]
+  CRUSH rule 1 x 549 [5,8,2]
+  CRUSH rule 1 x 550 [2,5,8]
+  CRUSH rule 1 x 551 [7,5,6]
+  CRUSH rule 1 x 552 [5,2]
+  CRUSH rule 1 x 553 [5,2,6]
+  CRUSH rule 1 x 554 [2,8,5]
+  CRUSH rule 1 x 555 [5,2,8]
+  CRUSH rule 1 x 556 [5,6,7]
+  CRUSH rule 1 x 557 [7,5,6]
+  CRUSH rule 1 x 558 [5,2,6]
+  CRUSH rule 1 x 559 [5,2,6]
+  CRUSH rule 1 x 560 [8,5,7]
+  CRUSH rule 1 x 561 [6,5,7]
+  CRUSH rule 1 x 562 [5,7,2]
+  CRUSH rule 1 x 563 [2,6,8]
+  CRUSH rule 1 x 564 [5,2,7]
+  CRUSH rule 1 x 565 [5,6,7]
+  CRUSH rule 1 x 566 [5,7,2]
+  CRUSH rule 1 x 567 [5,6,2]
+  CRUSH rule 1 x 568 [7,5,2]
+  CRUSH rule 1 x 569 [5,2,8]
+  CRUSH rule 1 x 570 [2,5]
+  CRUSH rule 1 x 571 [5,7,8]
+  CRUSH rule 1 x 572 [5,2]
+  CRUSH rule 1 x 573 [5,2,7]
+  CRUSH rule 1 x 574 [2,8,6]
+  CRUSH rule 1 x 575 [8,6,2]
+  CRUSH rule 1 x 576 [5,6,8]
+  CRUSH rule 1 x 577 [8,2,6]
+  CRUSH rule 1 x 578 [6,8,7]
+  CRUSH rule 1 x 579 [5,2,6]
+  CRUSH rule 1 x 580 [5,2,7]
+  CRUSH rule 1 x 581 [7,2,6]
+  CRUSH rule 1 x 582 [2,8,7]
+  CRUSH rule 1 x 583 [6,2,8]
+  CRUSH rule 1 x 584 [8,2,6]
+  CRUSH rule 1 x 585 [7,2,5]
+  CRUSH rule 1 x 586 [2,6,7]
+  CRUSH rule 1 x 587 [2,5,6]
+  CRUSH rule 1 x 588 [5,2,7]
+  CRUSH rule 1 x 589 [7,2,5]
+  CRUSH rule 1 x 590 [6,2,5]
+  CRUSH rule 1 x 591 [5,2,8]
+  CRUSH rule 1 x 592 [2,5,7]
+  CRUSH rule 1 x 593 [2,8,6]
+  CRUSH rule 1 x 594 [2,7,5]
+  CRUSH rule 1 x 595 [7,2,8]
+  CRUSH rule 1 x 596 [5,7,2]
+  CRUSH rule 1 x 597 [5,2,7]
+  CRUSH rule 1 x 598 [5,2,8]
+  CRUSH rule 1 x 599 [5,2,6]
+  CRUSH rule 1 x 600 [7,2,8]
+  CRUSH rule 1 x 601 [2,7,8]
+  CRUSH rule 1 x 602 [5,7,8]
+  CRUSH rule 1 x 603 [5,2,6]
+  CRUSH rule 1 x 604 [7,5,2]
+  CRUSH rule 1 x 605 [5,2,8]
+  CRUSH rule 1 x 606 [2,6,8]
+  CRUSH rule 1 x 607 [2,5,8]
+  CRUSH rule 1 x 608 [5,2,6]
+  CRUSH rule 1 x 609 [5,2]
+  CRUSH rule 1 x 610 [5,7,6]
+  CRUSH rule 1 x 611 [2,5,7]
+  CRUSH rule 1 x 612 [2,8,7]
+  CRUSH rule 1 x 613 [7,2,5]
+  CRUSH rule 1 x 614 [7,8,6]
+  CRUSH rule 1 x 615 [6,8,2]
+  CRUSH rule 1 x 616 [2,8,7]
+  CRUSH rule 1 x 617 [6,2,7]
+  CRUSH rule 1 x 618 [7,6,5]
+  CRUSH rule 1 x 619 [5,2,8]
+  CRUSH rule 1 x 620 [5,2,6]
+  CRUSH rule 1 x 621 [5,8,2]
+  CRUSH rule 1 x 622 [2,5,8]
+  CRUSH rule 1 x 623 [2,6,5]
+  CRUSH rule 1 x 624 [5,7,2]
+  CRUSH rule 1 x 625 [2,5,7]
+  CRUSH rule 1 x 626 [7,8,2]
+  CRUSH rule 1 x 627 [2,7,8]
+  CRUSH rule 1 x 628 [8,2,5]
+  CRUSH rule 1 x 629 [2,6,5]
+  CRUSH rule 1 x 630 [2,7,5]
+  CRUSH rule 1 x 631 [2,7,8]
+  CRUSH rule 1 x 632 [7,2,5]
+  CRUSH rule 1 x 633 [8,6,5]
+  CRUSH rule 1 x 634 [2,5,8]
+  CRUSH rule 1 x 635 [5,6,2]
+  CRUSH rule 1 x 636 [2,5,6]
+  CRUSH rule 1 x 637 [5,2,7]
+  CRUSH rule 1 x 638 [6,8,2]
+  CRUSH rule 1 x 639 [5,2,7]
+  CRUSH rule 1 x 640 [5,2,6]
+  CRUSH rule 1 x 641 [7,2,6]
+  CRUSH rule 1 x 642 [2,8,5]
+  CRUSH rule 1 x 643 [5,7,2]
+  CRUSH rule 1 x 644 [8,2,5]
+  CRUSH rule 1 x 645 [5,2,6]
+  CRUSH rule 1 x 646 [8,2,5]
+  CRUSH rule 1 x 647 [7,2,5]
+  CRUSH rule 1 x 648 [2,6,5]
+  CRUSH rule 1 x 649 [5,7,6]
+  CRUSH rule 1 x 650 [7,8,6]
+  CRUSH rule 1 x 651 [5,7,6]
+  CRUSH rule 1 x 652 [5,6,8]
+  CRUSH rule 1 x 653 [8,5,2]
+  CRUSH rule 1 x 654 [7,5,2]
+  CRUSH rule 1 x 655 [2,5,6]
+  CRUSH rule 1 x 656 [5,2,8]
+  CRUSH rule 1 x 657 [6,2,8]
+  CRUSH rule 1 x 658 [5,2,6]
+  CRUSH rule 1 x 659 [5,6,7]
+  CRUSH rule 1 x 660 [7,8,6]
+  CRUSH rule 1 x 661 [2,8,5]
+  CRUSH rule 1 x 662 [5,2]
+  CRUSH rule 1 x 663 [2,5,8]
+  CRUSH rule 1 x 664 [2,5]
+  CRUSH rule 1 x 665 [5,7,6]
+  CRUSH rule 1 x 666 [2,8,5]
+  CRUSH rule 1 x 667 [2,5,6]
+  CRUSH rule 1 x 668 [5,7,6]
+  CRUSH rule 1 x 669 [6,5,2]
+  CRUSH rule 1 x 670 [5,2]
+  CRUSH rule 1 x 671 [2,5,8]
+  CRUSH rule 1 x 672 [5,6,2]
+  CRUSH rule 1 x 673 [5,2,7]
+  CRUSH rule 1 x 674 [5,2,8]
+  CRUSH rule 1 x 675 [2,8,6]
+  CRUSH rule 1 x 676 [2,5]
+  CRUSH rule 1 x 677 [5,2,7]
+  CRUSH rule 1 x 678 [2,5,6]
+  CRUSH rule 1 x 679 [6,2,7]
+  CRUSH rule 1 x 680 [2,5,6]
+  CRUSH rule 1 x 681 [5,7,2]
+  CRUSH rule 1 x 682 [2,5,7]
+  CRUSH rule 1 x 683 [2,8,5]
+  CRUSH rule 1 x 684 [7,2,6]
+  CRUSH rule 1 x 685 [7,2,6]
+  CRUSH rule 1 x 686 [2,5,8]
+  CRUSH rule 1 x 687 [5,7,6]
+  CRUSH rule 1 x 688 [5,7,2]
+  CRUSH rule 1 x 689 [6,5,2]
+  CRUSH rule 1 x 690 [8,2,7]
+  CRUSH rule 1 x 691 [5,2]
+  CRUSH rule 1 x 692 [7,2,8]
+  CRUSH rule 1 x 693 [6,7,5]
+  CRUSH rule 1 x 694 [6,5,2]
+  CRUSH rule 1 x 695 [2,8,7]
+  CRUSH rule 1 x 696 [2,5,8]
+  CRUSH rule 1 x 697 [6,2,7]
+  CRUSH rule 1 x 698 [6,2,5]
+  CRUSH rule 1 x 699 [2,6,8]
+  CRUSH rule 1 x 700 [2,5,7]
+  CRUSH rule 1 x 701 [5,2,7]
+  CRUSH rule 1 x 702 [5,6,2]
+  CRUSH rule 1 x 703 [8,5,2]
+  CRUSH rule 1 x 704 [2,5,8]
+  CRUSH rule 1 x 705 [8,6,2]
+  CRUSH rule 1 x 706 [2,7,5]
+  CRUSH rule 1 x 707 [7,8,6]
+  CRUSH rule 1 x 708 [5,7,8]
+  CRUSH rule 1 x 709 [6,5,2]
+  CRUSH rule 1 x 710 [8,5,2]
+  CRUSH rule 1 x 711 [2,5,8]
+  CRUSH rule 1 x 712 [2,5,7]
+  CRUSH rule 1 x 713 [6,7,8]
+  CRUSH rule 1 x 714 [5,2,6]
+  CRUSH rule 1 x 715 [2,5,6]
+  CRUSH rule 1 x 716 [5,6,2]
+  CRUSH rule 1 x 717 [8,7,2]
+  CRUSH rule 1 x 718 [5,7,8]
+  CRUSH rule 1 x 719 [2,6,5]
+  CRUSH rule 1 x 720 [6,8,2]
+  CRUSH rule 1 x 721 [5,6,7]
+  CRUSH rule 1 x 722 [5,2,6]
+  CRUSH rule 1 x 723 [5,2,6]
+  CRUSH rule 1 x 724 [2,6,5]
+  CRUSH rule 1 x 725 [2,8,5]
+  CRUSH rule 1 x 726 [5,8,2]
+  CRUSH rule 1 x 727 [5,6,8]
+  CRUSH rule 1 x 728 [2,6,8]
+  CRUSH rule 1 x 729 [5,6,7]
+  CRUSH rule 1 x 730 [5,7,6]
+  CRUSH rule 1 x 731 [5,2,6]
+  CRUSH rule 1 x 732 [2,5,6]
+  CRUSH rule 1 x 733 [5,2,7]
+  CRUSH rule 1 x 734 [6,5,2]
+  CRUSH rule 1 x 735 [5,8,2]
+  CRUSH rule 1 x 736 [5,8,6]
+  CRUSH rule 1 x 737 [2,5,8]
+  CRUSH rule 1 x 738 [5,2,7]
+  CRUSH rule 1 x 739 [2,8,5]
+  CRUSH rule 1 x 740 [2,6,7]
+  CRUSH rule 1 x 741 [7,8,2]
+  CRUSH rule 1 x 742 [8,2,5]
+  CRUSH rule 1 x 743 [7,2,8]
+  CRUSH rule 1 x 744 [5,7,2]
+  CRUSH rule 1 x 745 [5,6,2]
+  CRUSH rule 1 x 746 [5,2,8]
+  CRUSH rule 1 x 747 [6,2,5]
+  CRUSH rule 1 x 748 [2,7,8]
+  CRUSH rule 1 x 749 [5,8,7]
+  CRUSH rule 1 x 750 [2,6,5]
+  CRUSH rule 1 x 751 [2,8,6]
+  CRUSH rule 1 x 752 [8,2,5]
+  CRUSH rule 1 x 753 [7,8,5]
+  CRUSH rule 1 x 754 [8,6,7]
+  CRUSH rule 1 x 755 [2,5]
+  CRUSH rule 1 x 756 [5,6,2]
+  CRUSH rule 1 x 757 [8,6,2]
+  CRUSH rule 1 x 758 [6,2,5]
+  CRUSH rule 1 x 759 [8,5,6]
+  CRUSH rule 1 x 760 [2,5,8]
+  CRUSH rule 1 x 761 [5,2,7]
+  CRUSH rule 1 x 762 [2,7,8]
+  CRUSH rule 1 x 763 [8,6,7]
+  CRUSH rule 1 x 764 [2,7,5]
+  CRUSH rule 1 x 765 [6,5,2]
+  CRUSH rule 1 x 766 [8,5,7]
+  CRUSH rule 1 x 767 [2,6,5]
+  CRUSH rule 1 x 768 [8,5,2]
+  CRUSH rule 1 x 769 [6,2,8]
+  CRUSH rule 1 x 770 [6,2,7]
+  CRUSH rule 1 x 771 [7,2,5]
+  CRUSH rule 1 x 772 [8,5,7]
+  CRUSH rule 1 x 773 [5,2,7]
+  CRUSH rule 1 x 774 [5,6,2]
+  CRUSH rule 1 x 775 [6,8,5]
+  CRUSH rule 1 x 776 [7,2,5]
+  CRUSH rule 1 x 777 [5,2,6]
+  CRUSH rule 1 x 778 [2,8,5]
+  CRUSH rule 1 x 779 [2,7,5]
+  CRUSH rule 1 x 780 [2,5,6]
+  CRUSH rule 1 x 781 [6,5,7]
+  CRUSH rule 1 x 782 [5,2,8]
+  CRUSH rule 1 x 783 [7,2,8]
+  CRUSH rule 1 x 784 [2,7,5]
+  CRUSH rule 1 x 785 [6,2,5]
+  CRUSH rule 1 x 786 [7,6,5]
+  CRUSH rule 1 x 787 [2,7,6]
+  CRUSH rule 1 x 788 [6,2,8]
+  CRUSH rule 1 x 789 [2,5,8]
+  CRUSH rule 1 x 790 [8,5,7]
+  CRUSH rule 1 x 791 [5,8,7]
+  CRUSH rule 1 x 792 [5,8,7]
+  CRUSH rule 1 x 793 [6,2,5]
+  CRUSH rule 1 x 794 [2,6,8]
+  CRUSH rule 1 x 795 [2,5,6]
+  CRUSH rule 1 x 796 [5,6,7]
+  CRUSH rule 1 x 797 [2,5,7]
+  CRUSH rule 1 x 798 [6,8,7]
+  CRUSH rule 1 x 799 [5,2,8]
+  CRUSH rule 1 x 800 [5,2,7]
+  CRUSH rule 1 x 801 [5,6,8]
+  CRUSH rule 1 x 802 [2,8,7]
+  CRUSH rule 1 x 803 [2,5,7]
+  CRUSH rule 1 x 804 [6,2,5]
+  CRUSH rule 1 x 805 [5,6,7]
+  CRUSH rule 1 x 806 [2,5,8]
+  CRUSH rule 1 x 807 [5,7,8]
+  CRUSH rule 1 x 808 [5,6,2]
+  CRUSH rule 1 x 809 [2,5,7]
+  CRUSH rule 1 x 810 [5,7,2]
+  CRUSH rule 1 x 811 [8,5,2]
+  CRUSH rule 1 x 812 [8,5,2]
+  CRUSH rule 1 x 813 [6,5,8]
+  CRUSH rule 1 x 814 [5,6,8]
+  CRUSH rule 1 x 815 [5,2,8]
+  CRUSH rule 1 x 816 [2,5,8]
+  CRUSH rule 1 x 817 [5,7,6]
+  CRUSH rule 1 x 818 [5,2,8]
+  CRUSH rule 1 x 819 [5,2,6]
+  CRUSH rule 1 x 820 [5,2,8]
+  CRUSH rule 1 x 821 [5,2,8]
+  CRUSH rule 1 x 822 [2,7,5]
+  CRUSH rule 1 x 823 [5,8,2]
+  CRUSH rule 1 x 824 [5,7,2]
+  CRUSH rule 1 x 825 [2,8,7]
+  CRUSH rule 1 x 826 [7,2,5]
+  CRUSH rule 1 x 827 [2,8,7]
+  CRUSH rule 1 x 828 [2,5,7]
+  CRUSH rule 1 x 829 [5,6,7]
+  CRUSH rule 1 x 830 [2,5,8]
+  CRUSH rule 1 x 831 [2,6,8]
+  CRUSH rule 1 x 832 [5,7,2]
+  CRUSH rule 1 x 833 [2,7,8]
+  CRUSH rule 1 x 834 [5,2]
+  CRUSH rule 1 x 835 [8,5,6]
+  CRUSH rule 1 x 836 [5,2]
+  CRUSH rule 1 x 837 [6,5,2]
+  CRUSH rule 1 x 838 [6,7,2]
+  CRUSH rule 1 x 839 [5,2,6]
+  CRUSH rule 1 x 840 [7,8,5]
+  CRUSH rule 1 x 841 [5,8,7]
+  CRUSH rule 1 x 842 [2,5,7]
+  CRUSH rule 1 x 843 [6,5,7]
+  CRUSH rule 1 x 844 [5,8,2]
+  CRUSH rule 1 x 845 [5,8,6]
+  CRUSH rule 1 x 846 [5,2,7]
+  CRUSH rule 1 x 847 [2,8,7]
+  CRUSH rule 1 x 848 [2,6,8]
+  CRUSH rule 1 x 849 [5,8,2]
+  CRUSH rule 1 x 850 [2,5,6]
+  CRUSH rule 1 x 851 [6,8,7]
+  CRUSH rule 1 x 852 [7,5,8]
+  CRUSH rule 1 x 853 [6,8,2]
+  CRUSH rule 1 x 854 [7,6,2]
+  CRUSH rule 1 x 855 [5,7,2]
+  CRUSH rule 1 x 856 [6,7,5]
+  CRUSH rule 1 x 857 [8,5,2]
+  CRUSH rule 1 x 858 [6,5,2]
+  CRUSH rule 1 x 859 [6,2,7]
+  CRUSH rule 1 x 860 [5,2,6]
+  CRUSH rule 1 x 861 [8,7,6]
+  CRUSH rule 1 x 862 [6,2,7]
+  CRUSH rule 1 x 863 [8,7,2]
+  CRUSH rule 1 x 864 [5,6,8]
+  CRUSH rule 1 x 865 [8,2,6]
+  CRUSH rule 1 x 866 [5,8,7]
+  CRUSH rule 1 x 867 [6,5,2]
+  CRUSH rule 1 x 868 [6,5,2]
+  CRUSH rule 1 x 869 [8,7,5]
+  CRUSH rule 1 x 870 [2,5,8]
+  CRUSH rule 1 x 871 [5,7,8]
+  CRUSH rule 1 x 872 [5,2,7]
+  CRUSH rule 1 x 873 [5,6,7]
+  CRUSH rule 1 x 874 [2,6,7]
+  CRUSH rule 1 x 875 [2,6,5]
+  CRUSH rule 1 x 876 [5,8,2]
+  CRUSH rule 1 x 877 [6,5,2]
+  CRUSH rule 1 x 878 [5,2]
+  CRUSH rule 1 x 879 [7,5,8]
+  CRUSH rule 1 x 880 [5,2,6]
+  CRUSH rule 1 x 881 [5,6,2]
+  CRUSH rule 1 x 882 [5,2]
+  CRUSH rule 1 x 883 [2,7,5]
+  CRUSH rule 1 x 884 [6,2,5]
+  CRUSH rule 1 x 885 [5,2,6]
+  CRUSH rule 1 x 886 [5,6,8]
+  CRUSH rule 1 x 887 [7,5,2]
+  CRUSH rule 1 x 888 [6,8,2]
+  CRUSH rule 1 x 889 [2,5,7]
+  CRUSH rule 1 x 890 [7,2,6]
+  CRUSH rule 1 x 891 [2,8,5]
+  CRUSH rule 1 x 892 [6,2,5]
+  CRUSH rule 1 x 893 [2,5,7]
+  CRUSH rule 1 x 894 [7,5,2]
+  CRUSH rule 1 x 895 [5,7,2]
+  CRUSH rule 1 x 896 [2,8,6]
+  CRUSH rule 1 x 897 [5,2,6]
+  CRUSH rule 1 x 898 [2,5,7]
+  CRUSH rule 1 x 899 [2,7,6]
+  CRUSH rule 1 x 900 [5,2,6]
+  CRUSH rule 1 x 901 [5,2,7]
+  CRUSH rule 1 x 902 [8,5,7]
+  CRUSH rule 1 x 903 [5,7,2]
+  CRUSH rule 1 x 904 [5,6,8]
+  CRUSH rule 1 x 905 [6,2,5]
+  CRUSH rule 1 x 906 [2,5,7]
+  CRUSH rule 1 x 907 [7,2]
+  CRUSH rule 1 x 908 [5,8,2]
+  CRUSH rule 1 x 909 [2,5,8]
+  CRUSH rule 1 x 910 [6,5,2]
+  CRUSH rule 1 x 911 [5,8,2]
+  CRUSH rule 1 x 912 [2,6,7]
+  CRUSH rule 1 x 913 [7,6,8]
+  CRUSH rule 1 x 914 [6,5,7]
+  CRUSH rule 1 x 915 [8,2,6]
+  CRUSH rule 1 x 916 [5,2,7]
+  CRUSH rule 1 x 917 [2,5,6]
+  CRUSH rule 1 x 918 [8,2,7]
+  CRUSH rule 1 x 919 [6,2,8]
+  CRUSH rule 1 x 920 [7,6,5]
+  CRUSH rule 1 x 921 [2,5,7]
+  CRUSH rule 1 x 922 [6,7,8]
+  CRUSH rule 1 x 923 [5,8,6]
+  CRUSH rule 1 x 924 [5,6,2]
+  CRUSH rule 1 x 925 [5,7,2]
+  CRUSH rule 1 x 926 [5,8,2]
+  CRUSH rule 1 x 927 [2,6,5]
+  CRUSH rule 1 x 928 [8,2,5]
+  CRUSH rule 1 x 929 [5,2]
+  CRUSH rule 1 x 930 [2,5,6]
+  CRUSH rule 1 x 931 [5,2]
+  CRUSH rule 1 x 932 [5,8,2]
+  CRUSH rule 1 x 933 [8,5,2]
+  CRUSH rule 1 x 934 [5,2,8]
+  CRUSH rule 1 x 935 [6,5,2]
+  CRUSH rule 1 x 936 [2,6,7]
+  CRUSH rule 1 x 937 [5,6,7]
+  CRUSH rule 1 x 938 [6,5,8]
+  CRUSH rule 1 x 939 [2,7,6]
+  CRUSH rule 1 x 940 [8,7,6]
+  CRUSH rule 1 x 941 [5,2,8]
+  CRUSH rule 1 x 942 [2,8,7]
+  CRUSH rule 1 x 943 [8,2,5]
+  CRUSH rule 1 x 944 [5,2,7]
+  CRUSH rule 1 x 945 [7,2,5]
+  CRUSH rule 1 x 946 [2,5,7]
+  CRUSH rule 1 x 947 [5,6,2]
+  CRUSH rule 1 x 948 [7,8,6]
+  CRUSH rule 1 x 949 [6,2,7]
+  CRUSH rule 1 x 950 [5,7,8]
+  CRUSH rule 1 x 951 [5,8,7]
+  CRUSH rule 1 x 952 [2,5,7]
+  CRUSH rule 1 x 953 [2,5,6]
+  CRUSH rule 1 x 954 [5,2,7]
+  CRUSH rule 1 x 955 [8,6,2]
+  CRUSH rule 1 x 956 [2,8,6]
+  CRUSH rule 1 x 957 [7,6,2]
+  CRUSH rule 1 x 958 [8,7,5]
+  CRUSH rule 1 x 959 [5,2,7]
+  CRUSH rule 1 x 960 [5,6,7]
+  CRUSH rule 1 x 961 [5,2,6]
+  CRUSH rule 1 x 962 [7,5,2]
+  CRUSH rule 1 x 963 [2,5,8]
+  CRUSH rule 1 x 964 [5,2,6]
+  CRUSH rule 1 x 965 [7,6,5]
+  CRUSH rule 1 x 966 [5,8,6]
+  CRUSH rule 1 x 967 [8,6,5]
+  CRUSH rule 1 x 968 [7,2,5]
+  CRUSH rule 1 x 969 [8,2,6]
+  CRUSH rule 1 x 970 [2,6,5]
+  CRUSH rule 1 x 971 [2,7,8]
+  CRUSH rule 1 x 972 [2,8,5]
+  CRUSH rule 1 x 973 [2,6,5]
+  CRUSH rule 1 x 974 [5,2,7]
+  CRUSH rule 1 x 975 [5,7,8]
+  CRUSH rule 1 x 976 [5,6,7]
+  CRUSH rule 1 x 977 [8,5,2]
+  CRUSH rule 1 x 978 [7,2,8]
+  CRUSH rule 1 x 979 [7,6,2]
+  CRUSH rule 1 x 980 [6,2,7]
+  CRUSH rule 1 x 981 [7,5,2]
+  CRUSH rule 1 x 982 [5,2,6]
+  CRUSH rule 1 x 983 [5,6,8]
+  CRUSH rule 1 x 984 [2,8,5]
+  CRUSH rule 1 x 985 [2,5,6]
+  CRUSH rule 1 x 986 [8,7,5]
+  CRUSH rule 1 x 987 [2,5,8]
+  CRUSH rule 1 x 988 [2,5,6]
+  CRUSH rule 1 x 989 [2,6,5]
+  CRUSH rule 1 x 990 [2,5,8]
+  CRUSH rule 1 x 991 [2,5,8]
+  CRUSH rule 1 x 992 [7,2,5]
+  CRUSH rule 1 x 993 [2,6,5]
+  CRUSH rule 1 x 994 [5,2,8]
+  CRUSH rule 1 x 995 [7,6,2]
+  CRUSH rule 1 x 996 [6,7,5]
+  CRUSH rule 1 x 997 [6,5,2]
+  CRUSH rule 1 x 998 [8,2,5]
+  CRUSH rule 1 x 999 [2,7,8]
+  CRUSH rule 1 x 1000 [8,5,2]
+  CRUSH rule 1 x 1001 [2,5]
+  CRUSH rule 1 x 1002 [2,5,7]
+  CRUSH rule 1 x 1003 [2,8,7]
+  CRUSH rule 1 x 1004 [6,2,8]
+  CRUSH rule 1 x 1005 [6,2,8]
+  CRUSH rule 1 x 1006 [2,5]
+  CRUSH rule 1 x 1007 [2,8,5]
+  CRUSH rule 1 x 1008 [2,7,5]
+  CRUSH rule 1 x 1009 [6,8,5]
+  CRUSH rule 1 x 1010 [5,6,2]
+  CRUSH rule 1 x 1011 [5,2,7]
+  CRUSH rule 1 x 1012 [5,2,7]
+  CRUSH rule 1 x 1013 [5,2,7]
+  CRUSH rule 1 x 1014 [2,8,5]
+  CRUSH rule 1 x 1015 [6,8,5]
+  CRUSH rule 1 x 1016 [2,6,5]
+  CRUSH rule 1 x 1017 [6,2,5]
+  CRUSH rule 1 x 1018 [5,2,7]
+  CRUSH rule 1 x 1019 [5,7,8]
+  CRUSH rule 1 x 1020 [5,2,7]
+  CRUSH rule 1 x 1021 [5,2,8]
+  CRUSH rule 1 x 1022 [2,6,7]
+  CRUSH rule 1 x 1023 [5,2,8]
+  rule 1 (choose-two) num_rep 3 result size == 2:\t32/1024 (esc)
+  rule 1 (choose-two) num_rep 3 result size == 3:\t992/1024 (esc)
+  rule 2 (chooseleaf), x = 0..1023, numrep = 2..3
+  CRUSH rule 2 x 0 [2,5]
+  CRUSH rule 2 x 1 [2,8]
+  CRUSH rule 2 x 2 [2,5]
+  CRUSH rule 2 x 3 [8,2]
+  CRUSH rule 2 x 4 [5,2]
+  CRUSH rule 2 x 5 [7,2]
+  CRUSH rule 2 x 6 [2,6]
+  CRUSH rule 2 x 7 [5,8]
+  CRUSH rule 2 x 8 [5,6]
+  CRUSH rule 2 x 9 [2,5]
+  CRUSH rule 2 x 10 [2,7]
+  CRUSH rule 2 x 11 [2,7]
+  CRUSH rule 2 x 12 [2,5]
+  CRUSH rule 2 x 13 [5,8]
+  CRUSH rule 2 x 14 [7,2]
+  CRUSH rule 2 x 15 [7,2]
+  CRUSH rule 2 x 16 [5,6]
+  CRUSH rule 2 x 17 [5,2]
+  CRUSH rule 2 x 18 [2,5]
+  CRUSH rule 2 x 19 [7,5]
+  CRUSH rule 2 x 20 [2,5]
+  CRUSH rule 2 x 21 [5,7]
+  CRUSH rule 2 x 22 [8,5]
+  CRUSH rule 2 x 23 [5,6]
+  CRUSH rule 2 x 24 [2,7]
+  CRUSH rule 2 x 25 [5,7]
+  CRUSH rule 2 x 26 [2,8]
+  CRUSH rule 2 x 27 [5,2]
+  CRUSH rule 2 x 28 [6,2]
+  CRUSH rule 2 x 29 [8,5]
+  CRUSH rule 2 x 30 [5,7]
+  CRUSH rule 2 x 31 [8,2]
+  CRUSH rule 2 x 32 [5,6]
+  CRUSH rule 2 x 33 [2,7]
+  CRUSH rule 2 x 34 [2,5]
+  CRUSH rule 2 x 35 [2,8]
+  CRUSH rule 2 x 36 [5,8]
+  CRUSH rule 2 x 37 [2,5]
+  CRUSH rule 2 x 38 [5,8]
+  CRUSH rule 2 x 39 [5,7]
+  CRUSH rule 2 x 40 [7,2]
+  CRUSH rule 2 x 41 [2,6]
+  CRUSH rule 2 x 42 [5,6]
+  CRUSH rule 2 x 43 [2,5]
+  CRUSH rule 2 x 44 [2,6]
+  CRUSH rule 2 x 45 [8,2]
+  CRUSH rule 2 x 46 [2,5]
+  CRUSH rule 2 x 47 [5,2]
+  CRUSH rule 2 x 48 [5,6]
+  CRUSH rule 2 x 49 [5,7]
+  CRUSH rule 2 x 50 [5,2]
+  CRUSH rule 2 x 51 [5,6]
+  CRUSH rule 2 x 52 [8,2]
+  CRUSH rule 2 x 53 [5,8]
+  CRUSH rule 2 x 54 [7,5]
+  CRUSH rule 2 x 55 [8,2]
+  CRUSH rule 2 x 56 [6,5]
+  CRUSH rule 2 x 57 [5,8]
+  CRUSH rule 2 x 58 [2,8]
+  CRUSH rule 2 x 59 [5,2]
+  CRUSH rule 2 x 60 [5,2]
+  CRUSH rule 2 x 61 [5,6]
+  CRUSH rule 2 x 62 [7,2]
+  CRUSH rule 2 x 63 [5,6]
+  CRUSH rule 2 x 64 [5,2]
+  CRUSH rule 2 x 65 [7,5]
+  CRUSH rule 2 x 66 [5,6]
+  CRUSH rule 2 x 67 [5,2]
+  CRUSH rule 2 x 68 [2,5]
+  CRUSH rule 2 x 69 [5,2]
+  CRUSH rule 2 x 70 [7,2]
+  CRUSH rule 2 x 71 [2,8]
+  CRUSH rule 2 x 72 [6,2]
+  CRUSH rule 2 x 73 [2,7]
+  CRUSH rule 2 x 74 [2,7]
+  CRUSH rule 2 x 75 [5,2]
+  CRUSH rule 2 x 76 [5,2]
+  CRUSH rule 2 x 77 [7,2]
+  CRUSH rule 2 x 78 [2,5]
+  CRUSH rule 2 x 79 [5,2]
+  CRUSH rule 2 x 80 [2,5]
+  CRUSH rule 2 x 81 [2,5]
+  CRUSH rule 2 x 82 [7,2]
+  CRUSH rule 2 x 83 [2,6]
+  CRUSH rule 2 x 84 [7,2]
+  CRUSH rule 2 x 85 [5,8]
+  CRUSH rule 2 x 86 [2,6]
+  CRUSH rule 2 x 87 [2,7]
+  CRUSH rule 2 x 88 [2,6]
+  CRUSH rule 2 x 89 [5,2]
+  CRUSH rule 2 x 90 [6,5]
+  CRUSH rule 2 x 91 [5,8]
+  CRUSH rule 2 x 92 [2,8]
+  CRUSH rule 2 x 93 [7,5]
+  CRUSH rule 2 x 94 [2,5]
+  CRUSH rule 2 x 95 [7,5]
+  CRUSH rule 2 x 96 [5,6]
+  CRUSH rule 2 x 97 [8,5]
+  CRUSH rule 2 x 98 [2,7]
+  CRUSH rule 2 x 99 [2,7]
+  CRUSH rule 2 x 100 [2,7]
+  CRUSH rule 2 x 101 [5,7]
+  CRUSH rule 2 x 102 [5,2]
+  CRUSH rule 2 x 103 [5,7]
+  CRUSH rule 2 x 104 [7,5]
+  CRUSH rule 2 x 105 [2,5]
+  CRUSH rule 2 x 106 [2,6]
+  CRUSH rule 2 x 107 [5,2]
+  CRUSH rule 2 x 108 [7,2]
+  CRUSH rule 2 x 109 [2,5]
+  CRUSH rule 2 x 110 [5,2]
+  CRUSH rule 2 x 111 [2,5]
+  CRUSH rule 2 x 112 [2,6]
+  CRUSH rule 2 x 113 [6,2]
+  CRUSH rule 2 x 114 [7,5]
+  CRUSH rule 2 x 115 [8,2]
+  CRUSH rule 2 x 116 [2,6]
+  CRUSH rule 2 x 117 [7,5]
+  CRUSH rule 2 x 118 [2,5]
+  CRUSH rule 2 x 119 [5,6]
+  CRUSH rule 2 x 120 [2,5]
+  CRUSH rule 2 x 121 [2,7]
+  CRUSH rule 2 x 122 [8,5]
+  CRUSH rule 2 x 123 [2,5]
+  CRUSH rule 2 x 124 [5,2]
+  CRUSH rule 2 x 125 [2,7]
+  CRUSH rule 2 x 126 [5,2]
+  CRUSH rule 2 x 127 [5,6]
+  CRUSH rule 2 x 128 [5,6]
+  CRUSH rule 2 x 129 [2,5]
+  CRUSH rule 2 x 130 [5,8]
+  CRUSH rule 2 x 131 [2,5]
+  CRUSH rule 2 x 132 [2,5]
+  CRUSH rule 2 x 133 [5,6]
+  CRUSH rule 2 x 134 [2,8]
+  CRUSH rule 2 x 135 [5,6]
+  CRUSH rule 2 x 136 [2,5]
+  CRUSH rule 2 x 137 [7,5]
+  CRUSH rule 2 x 138 [8,5]
+  CRUSH rule 2 x 139 [5,2]
+  CRUSH rule 2 x 140 [2,6]
+  CRUSH rule 2 x 141 [6,2]
+  CRUSH rule 2 x 142 [5,2]
+  CRUSH rule 2 x 143 [5,8]
+  CRUSH rule 2 x 144 [8,2]
+  CRUSH rule 2 x 145 [8,5]
+  CRUSH rule 2 x 146 [2,6]
+  CRUSH rule 2 x 147 [2,8]
+  CRUSH rule 2 x 148 [5,2]
+  CRUSH rule 2 x 149 [5,8]
+  CRUSH rule 2 x 150 [2,6]
+  CRUSH rule 2 x 151 [5,6]
+  CRUSH rule 2 x 152 [8,5]
+  CRUSH rule 2 x 153 [8,5]
+  CRUSH rule 2 x 154 [5,2]
+  CRUSH rule 2 x 155 [5,7]
+  CRUSH rule 2 x 156 [5,2]
+  CRUSH rule 2 x 157 [5,2]
+  CRUSH rule 2 x 158 [2,8]
+  CRUSH rule 2 x 159 [7,2]
+  CRUSH rule 2 x 160 [2,8]
+  CRUSH rule 2 x 161 [2,5]
+  CRUSH rule 2 x 162 [2,6]
+  CRUSH rule 2 x 163 [5,6]
+  CRUSH rule 2 x 164 [7,2]
+  CRUSH rule 2 x 165 [7,2]
+  CRUSH rule 2 x 166 [2,5]
+  CRUSH rule 2 x 167 [2,7]
+  CRUSH rule 2 x 168 [5,2]
+  CRUSH rule 2 x 169 [2,6]
+  CRUSH rule 2 x 170 [2,5]
+  CRUSH rule 2 x 171 [7,5]
+  CRUSH rule 2 x 172 [2,7]
+  CRUSH rule 2 x 173 [8,5]
+  CRUSH rule 2 x 174 [2,5]
+  CRUSH rule 2 x 175 [6,2]
+  CRUSH rule 2 x 176 [5,2]
+  CRUSH rule 2 x 177 [5,2]
+  CRUSH rule 2 x 178 [5,2]
+  CRUSH rule 2 x 179 [5,2]
+  CRUSH rule 2 x 180 [5,8]
+  CRUSH rule 2 x 181 [6,2]
+  CRUSH rule 2 x 182 [8,5]
+  CRUSH rule 2 x 183 [7,5]
+  CRUSH rule 2 x 184 [5,7]
+  CRUSH rule 2 x 185 [6,2]
+  CRUSH rule 2 x 186 [2,5]
+  CRUSH rule 2 x 187 [2,6]
+  CRUSH rule 2 x 188 [2,8]
+  CRUSH rule 2 x 189 [2,7]
+  CRUSH rule 2 x 190 [5,2]
+  CRUSH rule 2 x 191 [7,2]
+  CRUSH rule 2 x 192 [5,2]
+  CRUSH rule 2 x 193 [5,2]
+  CRUSH rule 2 x 194 [2,5]
+  CRUSH rule 2 x 195 [6,5]
+  CRUSH rule 2 x 196 [6,2]
+  CRUSH rule 2 x 197 [6,5]
+  CRUSH rule 2 x 198 [2,5]
+  CRUSH rule 2 x 199 [2,5]
+  CRUSH rule 2 x 200 [2,5]
+  CRUSH rule 2 x 201 [7,2]
+  CRUSH rule 2 x 202 [6,5]
+  CRUSH rule 2 x 203 [5,8]
+  CRUSH rule 2 x 204 [2,5]
+  CRUSH rule 2 x 205 [2,7]
+  CRUSH rule 2 x 206 [2,7]
+  CRUSH rule 2 x 207 [5,2]
+  CRUSH rule 2 x 208 [7,2]
+  CRUSH rule 2 x 209 [2,8]
+  CRUSH rule 2 x 210 [2,5]
+  CRUSH rule 2 x 211 [5,2]
+  CRUSH rule 2 x 212 [7,5]
+  CRUSH rule 2 x 213 [8,5]
+  CRUSH rule 2 x 214 [5,8]
+  CRUSH rule 2 x 215 [8,2]
+  CRUSH rule 2 x 216 [5,2]
+  CRUSH rule 2 x 217 [2,7]
+  CRUSH rule 2 x 218 [2,7]
+  CRUSH rule 2 x 219 [5,8]
+  CRUSH rule 2 x 220 [5,7]
+  CRUSH rule 2 x 221 [5,6]
+  CRUSH rule 2 x 222 [6,5]
+  CRUSH rule 2 x 223 [2,5]
+  CRUSH rule 2 x 224 [2,5]
+  CRUSH rule 2 x 225 [8,2]
+  CRUSH rule 2 x 226 [7,2]
+  CRUSH rule 2 x 227 [5,2]
+  CRUSH rule 2 x 228 [5,6]
+  CRUSH rule 2 x 229 [5,8]
+  CRUSH rule 2 x 230 [5,7]
+  CRUSH rule 2 x 231 [5,7]
+  CRUSH rule 2 x 232 [2,7]
+  CRUSH rule 2 x 233 [5,7]
+  CRUSH rule 2 x 234 [2,5]
+  CRUSH rule 2 x 235 [5,8]
+  CRUSH rule 2 x 236 [5,2]
+  CRUSH rule 2 x 237 [5,7]
+  CRUSH rule 2 x 238 [5,2]
+  CRUSH rule 2 x 239 [8,5]
+  CRUSH rule 2 x 240 [5,7]
+  CRUSH rule 2 x 241 [5,2]
+  CRUSH rule 2 x 242 [5,2]
+  CRUSH rule 2 x 243 [5,7]
+  CRUSH rule 2 x 244 [5,6]
+  CRUSH rule 2 x 245 [7,2]
+  CRUSH rule 2 x 246 [2,5]
+  CRUSH rule 2 x 247 [6,2]
+  CRUSH rule 2 x 248 [8,2]
+  CRUSH rule 2 x 249 [2,5]
+  CRUSH rule 2 x 250 [2,5]
+  CRUSH rule 2 x 251 [2,5]
+  CRUSH rule 2 x 252 [5,7]
+  CRUSH rule 2 x 253 [5,2]
+  CRUSH rule 2 x 254 [5,2]
+  CRUSH rule 2 x 255 [2,7]
+  CRUSH rule 2 x 256 [5,7]
+  CRUSH rule 2 x 257 [2,8]
+  CRUSH rule 2 x 258 [5,2]
+  CRUSH rule 2 x 259 [5,6]
+  CRUSH rule 2 x 260 [5,6]
+  CRUSH rule 2 x 261 [8,5]
+  CRUSH rule 2 x 262 [5,6]
+  CRUSH rule 2 x 263 [6,2]
+  CRUSH rule 2 x 264 [5,6]
+  CRUSH rule 2 x 265 [8,5]
+  CRUSH rule 2 x 266 [8,2]
+  CRUSH rule 2 x 267 [2,5]
+  CRUSH rule 2 x 268 [2,7]
+  CRUSH rule 2 x 269 [2,8]
+  CRUSH rule 2 x 270 [5,2]
+  CRUSH rule 2 x 271 [7,5]
+  CRUSH rule 2 x 272 [2,8]
+  CRUSH rule 2 x 273 [5,2]
+  CRUSH rule 2 x 274 [6,5]
+  CRUSH rule 2 x 275 [5,7]
+  CRUSH rule 2 x 276 [7,2]
+  CRUSH rule 2 x 277 [6,5]
+  CRUSH rule 2 x 278 [6,2]
+  CRUSH rule 2 x 279 [8,5]
+  CRUSH rule 2 x 280 [2,6]
+  CRUSH rule 2 x 281 [8,2]
+  CRUSH rule 2 x 282 [5,2]
+  CRUSH rule 2 x 283 [8,2]
+  CRUSH rule 2 x 284 [6,5]
+  CRUSH rule 2 x 285 [5,7]
+  CRUSH rule 2 x 286 [2,6]
+  CRUSH rule 2 x 287 [2,5]
+  CRUSH rule 2 x 288 [8,2]
+  CRUSH rule 2 x 289 [5,6]
+  CRUSH rule 2 x 290 [2,5]
+  CRUSH rule 2 x 291 [2,5]
+  CRUSH rule 2 x 292 [8,2]
+  CRUSH rule 2 x 293 [6,2]
+  CRUSH rule 2 x 294 [7,5]
+  CRUSH rule 2 x 295 [5,8]
+  CRUSH rule 2 x 296 [5,2]
+  CRUSH rule 2 x 297 [6,2]
+  CRUSH rule 2 x 298 [2,5]
+  CRUSH rule 2 x 299 [2,8]
+  CRUSH rule 2 x 300 [8,5]
+  CRUSH rule 2 x 301 [2,8]
+  CRUSH rule 2 x 302 [5,2]
+  CRUSH rule 2 x 303 [7,5]
+  CRUSH rule 2 x 304 [2,7]
+  CRUSH rule 2 x 305 [5,8]
+  CRUSH rule 2 x 306 [2,7]
+  CRUSH rule 2 x 307 [2,7]
+  CRUSH rule 2 x 308 [2,8]
+  CRUSH rule 2 x 309 [7,5]
+  CRUSH rule 2 x 310 [5,2]
+  CRUSH rule 2 x 311 [5,6]
+  CRUSH rule 2 x 312 [2,6]
+  CRUSH rule 2 x 313 [5,2]
+  CRUSH rule 2 x 314 [5,2]
+  CRUSH rule 2 x 315 [2,5]
+  CRUSH rule 2 x 316 [6,5]
+  CRUSH rule 2 x 317 [2,6]
+  CRUSH rule 2 x 318 [8,2]
+  CRUSH rule 2 x 319 [5,2]
+  CRUSH rule 2 x 320 [5,7]
+  CRUSH rule 2 x 321 [2,5]
+  CRUSH rule 2 x 322 [2,7]
+  CRUSH rule 2 x 323 [5,7]
+  CRUSH rule 2 x 324 [7,2]
+  CRUSH rule 2 x 325 [5,6]
+  CRUSH rule 2 x 326 [5,2]
+  CRUSH rule 2 x 327 [2,6]
+  CRUSH rule 2 x 328 [7,5]
+  CRUSH rule 2 x 329 [5,6]
+  CRUSH rule 2 x 330 [5,7]
+  CRUSH rule 2 x 331 [2,6]
+  CRUSH rule 2 x 332 [2,5]
+  CRUSH rule 2 x 333 [6,5]
+  CRUSH rule 2 x 334 [8,5]
+  CRUSH rule 2 x 335 [7,2]
+  CRUSH rule 2 x 336 [5,6]
+  CRUSH rule 2 x 337 [7,2]
+  CRUSH rule 2 x 338 [5,6]
+  CRUSH rule 2 x 339 [7,5]
+  CRUSH rule 2 x 340 [2,8]
+  CRUSH rule 2 x 341 [5,2]
+  CRUSH rule 2 x 342 [2,7]
+  CRUSH rule 2 x 343 [6,5]
+  CRUSH rule 2 x 344 [6,2]
+  CRUSH rule 2 x 345 [5,7]
+  CRUSH rule 2 x 346 [8,2]
+  CRUSH rule 2 x 347 [5,2]
+  CRUSH rule 2 x 348 [8,2]
+  CRUSH rule 2 x 349 [2,6]
+  CRUSH rule 2 x 350 [8,5]
+  CRUSH rule 2 x 351 [5,6]
+  CRUSH rule 2 x 352 [2,8]
+  CRUSH rule 2 x 353 [6,5]
+  CRUSH rule 2 x 354 [2,5]
+  CRUSH rule 2 x 355 [5,8]
+  CRUSH rule 2 x 356 [5,2]
+  CRUSH rule 2 x 357 [6,2]
+  CRUSH rule 2 x 358 [2,8]
+  CRUSH rule 2 x 359 [6,2]
+  CRUSH rule 2 x 360 [5,2]
+  CRUSH rule 2 x 361 [8,5]
+  CRUSH rule 2 x 362 [5,2]
+  CRUSH rule 2 x 363 [5,2]
+  CRUSH rule 2 x 364 [2,5]
+  CRUSH rule 2 x 365 [6,5]
+  CRUSH rule 2 x 366 [7,2]
+  CRUSH rule 2 x 367 [5,2]
+  CRUSH rule 2 x 368 [7,5]
+  CRUSH rule 2 x 369 [5,7]
+  CRUSH rule 2 x 370 [8,2]
+  CRUSH rule 2 x 371 [2,5]
+  CRUSH rule 2 x 372 [5,2]
+  CRUSH rule 2 x 373 [2,6]
+  CRUSH rule 2 x 374 [5,8]
+  CRUSH rule 2 x 375 [6,5]
+  CRUSH rule 2 x 376 [7,2]
+  CRUSH rule 2 x 377 [2,5]
+  CRUSH rule 2 x 378 [2,8]
+  CRUSH rule 2 x 379 [8,5]
+  CRUSH rule 2 x 380 [2,5]
+  CRUSH rule 2 x 381 [2,5]
+  CRUSH rule 2 x 382 [2,5]
+  CRUSH rule 2 x 383 [5,6]
+  CRUSH rule 2 x 384 [7,2]
+  CRUSH rule 2 x 385 [7,5]
+  CRUSH rule 2 x 386 [2,5]
+  CRUSH rule 2 x 387 [2,5]
+  CRUSH rule 2 x 388 [5,2]
+  CRUSH rule 2 x 389 [2,5]
+  CRUSH rule 2 x 390 [5,6]
+  CRUSH rule 2 x 391 [5,6]
+  CRUSH rule 2 x 392 [2,8]
+  CRUSH rule 2 x 393 [5,2]
+  CRUSH rule 2 x 394 [5,7]
+  CRUSH rule 2 x 395 [5,2]
+  CRUSH rule 2 x 396 [5,2]
+  CRUSH rule 2 x 397 [2,5]
+  CRUSH rule 2 x 398 [2,5]
+  CRUSH rule 2 x 399 [8,5]
+  CRUSH rule 2 x 400 [8,2]
+  CRUSH rule 2 x 401 [2,5]
+  CRUSH rule 2 x 402 [7,5]
+  CRUSH rule 2 x 403 [2,5]
+  CRUSH rule 2 x 404 [5,2]
+  CRUSH rule 2 x 405 [6,5]
+  CRUSH rule 2 x 406 [2,6]
+  CRUSH rule 2 x 407 [2,8]
+  CRUSH rule 2 x 408 [5,2]
+  CRUSH rule 2 x 409 [7,5]
+  CRUSH rule 2 x 410 [8,5]
+  CRUSH rule 2 x 411 [2,8]
+  CRUSH rule 2 x 412 [2,5]
+  CRUSH rule 2 x 413 [5,2]
+  CRUSH rule 2 x 414 [5,2]
+  CRUSH rule 2 x 415 [2,6]
+  CRUSH rule 2 x 416 [2,6]
+  CRUSH rule 2 x 417 [8,2]
+  CRUSH rule 2 x 418 [7,2]
+  CRUSH rule 2 x 419 [8,5]
+  CRUSH rule 2 x 420 [2,5]
+  CRUSH rule 2 x 421 [8,5]
+  CRUSH rule 2 x 422 [6,5]
+  CRUSH rule 2 x 423 [2,5]
+  CRUSH rule 2 x 424 [8,5]
+  CRUSH rule 2 x 425 [2,5]
+  CRUSH rule 2 x 426 [6,2]
+  CRUSH rule 2 x 427 [2,7]
+  CRUSH rule 2 x 428 [5,7]
+  CRUSH rule 2 x 429 [5,6]
+  CRUSH rule 2 x 430 [5,6]
+  CRUSH rule 2 x 431 [5,2]
+  CRUSH rule 2 x 432 [7,2]
+  CRUSH rule 2 x 433 [6,5]
+  CRUSH rule 2 x 434 [5,2]
+  CRUSH rule 2 x 435 [2,5]
+  CRUSH rule 2 x 436 [5,2]
+  CRUSH rule 2 x 437 [7,5]
+  CRUSH rule 2 x 438 [2,5]
+  CRUSH rule 2 x 439 [2,5]
+  CRUSH rule 2 x 440 [2,7]
+  CRUSH rule 2 x 441 [5,7]
+  CRUSH rule 2 x 442 [2,5]
+  CRUSH rule 2 x 443 [6,2]
+  CRUSH rule 2 x 444 [7,2]
+  CRUSH rule 2 x 445 [6,5]
+  CRUSH rule 2 x 446 [5,2]
+  CRUSH rule 2 x 447 [2,5]
+  CRUSH rule 2 x 448 [7,2]
+  CRUSH rule 2 x 449 [7,5]
+  CRUSH rule 2 x 450 [5,2]
+  CRUSH rule 2 x 451 [6,5]
+  CRUSH rule 2 x 452 [8,5]
+  CRUSH rule 2 x 453 [6,5]
+  CRUSH rule 2 x 454 [6,5]
+  CRUSH rule 2 x 455 [2,7]
+  CRUSH rule 2 x 456 [6,2]
+  CRUSH rule 2 x 457 [7,2]
+  CRUSH rule 2 x 458 [2,8]
+  CRUSH rule 2 x 459 [2,7]
+  CRUSH rule 2 x 460 [6,5]
+  CRUSH rule 2 x 461 [6,5]
+  CRUSH rule 2 x 462 [8,2]
+  CRUSH rule 2 x 463 [6,2]
+  CRUSH rule 2 x 464 [7,5]
+  CRUSH rule 2 x 465 [7,2]
+  CRUSH rule 2 x 466 [5,8]
+  CRUSH rule 2 x 467 [6,5]
+  CRUSH rule 2 x 468 [7,2]
+  CRUSH rule 2 x 469 [7,2]
+  CRUSH rule 2 x 470 [5,2]
+  CRUSH rule 2 x 471 [2,7]
+  CRUSH rule 2 x 472 [5,2]
+  CRUSH rule 2 x 473 [2,5]
+  CRUSH rule 2 x 474 [6,2]
+  CRUSH rule 2 x 475 [6,2]
+  CRUSH rule 2 x 476 [5,6]
+  CRUSH rule 2 x 477 [5,8]
+  CRUSH rule 2 x 478 [6,2]
+  CRUSH rule 2 x 479 [2,5]
+  CRUSH rule 2 x 480 [2,8]
+  CRUSH rule 2 x 481 [2,5]
+  CRUSH rule 2 x 482 [5,7]
+  CRUSH rule 2 x 483 [2,6]
+  CRUSH rule 2 x 484 [2,7]
+  CRUSH rule 2 x 485 [5,7]
+  CRUSH rule 2 x 486 [5,2]
+  CRUSH rule 2 x 487 [5,2]
+  CRUSH rule 2 x 488 [5,7]
+  CRUSH rule 2 x 489 [2,8]
+  CRUSH rule 2 x 490 [6,5]
+  CRUSH rule 2 x 491 [2,7]
+  CRUSH rule 2 x 492 [6,5]
+  CRUSH rule 2 x 493 [2,7]
+  CRUSH rule 2 x 494 [2,7]
+  CRUSH rule 2 x 495 [5,2]
+  CRUSH rule 2 x 496 [7,5]
+  CRUSH rule 2 x 497 [5,7]
+  CRUSH rule 2 x 498 [2,5]
+  CRUSH rule 2 x 499 [8,5]
+  CRUSH rule 2 x 500 [5,6]
+  CRUSH rule 2 x 501 [2,7]
+  CRUSH rule 2 x 502 [7,2]
+  CRUSH rule 2 x 503 [2,5]
+  CRUSH rule 2 x 504 [5,6]
+  CRUSH rule 2 x 505 [2,7]
+  CRUSH rule 2 x 506 [5,2]
+  CRUSH rule 2 x 507 [6,2]
+  CRUSH rule 2 x 508 [2,5]
+  CRUSH rule 2 x 509 [7,5]
+  CRUSH rule 2 x 510 [6,2]
+  CRUSH rule 2 x 511 [5,8]
+  CRUSH rule 2 x 512 [7,2]
+  CRUSH rule 2 x 513 [7,2]
+  CRUSH rule 2 x 514 [5,6]
+  CRUSH rule 2 x 515 [8,5]
+  CRUSH rule 2 x 516 [5,2]
+  CRUSH rule 2 x 517 [7,2]
+  CRUSH rule 2 x 518 [5,6]
+  CRUSH rule 2 x 519 [7,5]
+  CRUSH rule 2 x 520 [2,6]
+  CRUSH rule 2 x 521 [8,2]
+  CRUSH rule 2 x 522 [6,2]
+  CRUSH rule 2 x 523 [5,2]
+  CRUSH rule 2 x 524 [2,5]
+  CRUSH rule 2 x 525 [2,5]
+  CRUSH rule 2 x 526 [2,5]
+  CRUSH rule 2 x 527 [2,5]
+  CRUSH rule 2 x 528 [5,2]
+  CRUSH rule 2 x 529 [5,7]
+  CRUSH rule 2 x 530 [6,5]
+  CRUSH rule 2 x 531 [6,2]
+  CRUSH rule 2 x 532 [6,5]
+  CRUSH rule 2 x 533 [5,6]
+  CRUSH rule 2 x 534 [7,5]
+  CRUSH rule 2 x 535 [8,2]
+  CRUSH rule 2 x 536 [6,2]
+  CRUSH rule 2 x 537 [5,7]
+  CRUSH rule 2 x 538 [6,5]
+  CRUSH rule 2 x 539 [8,5]
+  CRUSH rule 2 x 540 [2,6]
+  CRUSH rule 2 x 541 [2,5]
+  CRUSH rule 2 x 542 [5,2]
+  CRUSH rule 2 x 543 [6,2]
+  CRUSH rule 2 x 544 [5,7]
+  CRUSH rule 2 x 545 [5,7]
+  CRUSH rule 2 x 546 [6,2]
+  CRUSH rule 2 x 547 [8,2]
+  CRUSH rule 2 x 548 [5,2]
+  CRUSH rule 2 x 549 [5,8]
+  CRUSH rule 2 x 550 [2,5]
+  CRUSH rule 2 x 551 [7,5]
+  CRUSH rule 2 x 552 [5,8]
+  CRUSH rule 2 x 553 [5,2]
+  CRUSH rule 2 x 554 [2,8]
+  CRUSH rule 2 x 555 [5,2]
+  CRUSH rule 2 x 556 [5,6]
+  CRUSH rule 2 x 557 [7,5]
+  CRUSH rule 2 x 558 [5,2]
+  CRUSH rule 2 x 559 [5,2]
+  CRUSH rule 2 x 560 [8,5]
+  CRUSH rule 2 x 561 [6,5]
+  CRUSH rule 2 x 562 [5,2]
+  CRUSH rule 2 x 563 [2,6]
+  CRUSH rule 2 x 564 [5,2]
+  CRUSH rule 2 x 565 [5,6]
+  CRUSH rule 2 x 566 [5,7]
+  CRUSH rule 2 x 567 [5,6]
+  CRUSH rule 2 x 568 [7,5]
+  CRUSH rule 2 x 569 [5,2]
+  CRUSH rule 2 x 570 [2,5]
+  CRUSH rule 2 x 571 [5,7]
+  CRUSH rule 2 x 572 [5,2]
+  CRUSH rule 2 x 573 [5,2]
+  CRUSH rule 2 x 574 [2,5]
+  CRUSH rule 2 x 575 [8,2]
+  CRUSH rule 2 x 576 [5,6]
+  CRUSH rule 2 x 577 [8,2]
+  CRUSH rule 2 x 578 [6,2]
+  CRUSH rule 2 x 579 [5,2]
+  CRUSH rule 2 x 580 [5,2]
+  CRUSH rule 2 x 581 [7,2]
+  CRUSH rule 2 x 582 [2,8]
+  CRUSH rule 2 x 583 [6,2]
+  CRUSH rule 2 x 584 [8,2]
+  CRUSH rule 2 x 585 [7,2]
+  CRUSH rule 2 x 586 [2,7]
+  CRUSH rule 2 x 587 [2,5]
+  CRUSH rule 2 x 588 [5,7]
+  CRUSH rule 2 x 589 [7,2]
+  CRUSH rule 2 x 590 [6,2]
+  CRUSH rule 2 x 591 [5,2]
+  CRUSH rule 2 x 592 [2,5]
+  CRUSH rule 2 x 593 [2,8]
+  CRUSH rule 2 x 594 [2,7]
+  CRUSH rule 2 x 595 [7,2]
+  CRUSH rule 2 x 596 [5,2]
+  CRUSH rule 2 x 597 [5,2]
+  CRUSH rule 2 x 598 [5,2]
+  CRUSH rule 2 x 599 [5,2]
+  CRUSH rule 2 x 600 [7,2]
+  CRUSH rule 2 x 601 [2,7]
+  CRUSH rule 2 x 602 [5,7]
+  CRUSH rule 2 x 603 [5,2]
+  CRUSH rule 2 x 604 [7,5]
+  CRUSH rule 2 x 605 [5,2]
+  CRUSH rule 2 x 606 [2,7]
+  CRUSH rule 2 x 607 [2,5]
+  CRUSH rule 2 x 608 [5,2]
+  CRUSH rule 2 x 609 [5,2]
+  CRUSH rule 2 x 610 [5,7]
+  CRUSH rule 2 x 611 [2,8]
+  CRUSH rule 2 x 612 [2,6]
+  CRUSH rule 2 x 613 [7,2]
+  CRUSH rule 2 x 614 [7,2]
+  CRUSH rule 2 x 615 [6,2]
+  CRUSH rule 2 x 616 [2,8]
+  CRUSH rule 2 x 617 [6,2]
+  CRUSH rule 2 x 618 [7,5]
+  CRUSH rule 2 x 619 [5,2]
+  CRUSH rule 2 x 620 [5,2]
+  CRUSH rule 2 x 621 [5,8]
+  CRUSH rule 2 x 622 [2,5]
+  CRUSH rule 2 x 623 [2,6]
+  CRUSH rule 2 x 624 [5,2]
+  CRUSH rule 2 x 625 [2,5]
+  CRUSH rule 2 x 626 [7,2]
+  CRUSH rule 2 x 627 [2,7]
+  CRUSH rule 2 x 628 [8,2]
+  CRUSH rule 2 x 629 [2,6]
+  CRUSH rule 2 x 630 [2,7]
+  CRUSH rule 2 x 631 [2,7]
+  CRUSH rule 2 x 632 [7,2]
+  CRUSH rule 2 x 633 [8,5]
+  CRUSH rule 2 x 634 [2,5]
+  CRUSH rule 2 x 635 [5,6]
+  CRUSH rule 2 x 636 [2,5]
+  CRUSH rule 2 x 637 [5,2]
+  CRUSH rule 2 x 638 [6,2]
+  CRUSH rule 2 x 639 [5,2]
+  CRUSH rule 2 x 640 [5,2]
+  CRUSH rule 2 x 641 [7,2]
+  CRUSH rule 2 x 642 [2,7]
+  CRUSH rule 2 x 643 [5,2]
+  CRUSH rule 2 x 644 [8,2]
+  CRUSH rule 2 x 645 [5,7]
+  CRUSH rule 2 x 646 [8,2]
+  CRUSH rule 2 x 647 [7,2]
+  CRUSH rule 2 x 648 [2,6]
+  CRUSH rule 2 x 649 [5,7]
+  CRUSH rule 2 x 650 [7,5]
+  CRUSH rule 2 x 651 [5,7]
+  CRUSH rule 2 x 652 [5,7]
+  CRUSH rule 2 x 653 [8,5]
+  CRUSH rule 2 x 654 [7,5]
+  CRUSH rule 2 x 655 [2,5]
+  CRUSH rule 2 x 656 [5,7]
+  CRUSH rule 2 x 657 [6,2]
+  CRUSH rule 2 x 658 [5,6]
+  CRUSH rule 2 x 659 [5,6]
+  CRUSH rule 2 x 660 [7,5]
+  CRUSH rule 2 x 661 [2,8]
+  CRUSH rule 2 x 662 [5,2]
+  CRUSH rule 2 x 663 [2,5]
+  CRUSH rule 2 x 664 [2,5]
+  CRUSH rule 2 x 665 [5,7]
+  CRUSH rule 2 x 666 [2,8]
+  CRUSH rule 2 x 667 [2,5]
+  CRUSH rule 2 x 668 [5,7]
+  CRUSH rule 2 x 669 [6,5]
+  CRUSH rule 2 x 670 [5,2]
+  CRUSH rule 2 x 671 [2,7]
+  CRUSH rule 2 x 672 [5,2]
+  CRUSH rule 2 x 673 [5,2]
+  CRUSH rule 2 x 674 [5,2]
+  CRUSH rule 2 x 675 [2,8]
+  CRUSH rule 2 x 676 [2,5]
+  CRUSH rule 2 x 677 [5,2]
+  CRUSH rule 2 x 678 [2,5]
+  CRUSH rule 2 x 679 [6,2]
+  CRUSH rule 2 x 680 [2,5]
+  CRUSH rule 2 x 681 [5,7]
+  CRUSH rule 2 x 682 [2,5]
+  CRUSH rule 2 x 683 [2,5]
+  CRUSH rule 2 x 684 [7,2]
+  CRUSH rule 2 x 685 [7,2]
+  CRUSH rule 2 x 686 [2,5]
+  CRUSH rule 2 x 687 [5,6]
+  CRUSH rule 2 x 688 [5,7]
+  CRUSH rule 2 x 689 [6,5]
+  CRUSH rule 2 x 690 [8,2]
+  CRUSH rule 2 x 691 [5,2]
+  CRUSH rule 2 x 692 [7,2]
+  CRUSH rule 2 x 693 [6,5]
+  CRUSH rule 2 x 694 [6,5]
+  CRUSH rule 2 x 695 [2,8]
+  CRUSH rule 2 x 696 [2,5]
+  CRUSH rule 2 x 697 [6,2]
+  CRUSH rule 2 x 698 [6,2]
+  CRUSH rule 2 x 699 [2,6]
+  CRUSH rule 2 x 700 [2,5]
+  CRUSH rule 2 x 701 [5,2]
+  CRUSH rule 2 x 702 [5,2]
+  CRUSH rule 2 x 703 [8,5]
+  CRUSH rule 2 x 704 [2,5]
+  CRUSH rule 2 x 705 [8,2]
+  CRUSH rule 2 x 706 [2,5]
+  CRUSH rule 2 x 707 [7,5]
+  CRUSH rule 2 x 708 [5,7]
+  CRUSH rule 2 x 709 [6,5]
+  CRUSH rule 2 x 710 [8,5]
+  CRUSH rule 2 x 711 [2,5]
+  CRUSH rule 2 x 712 [2,5]
+  CRUSH rule 2 x 713 [6,5]
+  CRUSH rule 2 x 714 [5,2]
+  CRUSH rule 2 x 715 [2,5]
+  CRUSH rule 2 x 716 [5,6]
+  CRUSH rule 2 x 717 [8,2]
+  CRUSH rule 2 x 718 [5,7]
+  CRUSH rule 2 x 719 [2,6]
+  CRUSH rule 2 x 720 [6,2]
+  CRUSH rule 2 x 721 [5,7]
+  CRUSH rule 2 x 722 [5,7]
+  CRUSH rule 2 x 723 [5,2]
+  CRUSH rule 2 x 724 [2,6]
+  CRUSH rule 2 x 725 [2,5]
+  CRUSH rule 2 x 726 [5,8]
+  CRUSH rule 2 x 727 [5,6]
+  CRUSH rule 2 x 728 [2,7]
+  CRUSH rule 2 x 729 [5,6]
+  CRUSH rule 2 x 730 [5,7]
+  CRUSH rule 2 x 731 [5,2]
+  CRUSH rule 2 x 732 [2,5]
+  CRUSH rule 2 x 733 [5,7]
+  CRUSH rule 2 x 734 [6,5]
+  CRUSH rule 2 x 735 [5,8]
+  CRUSH rule 2 x 736 [5,8]
+  CRUSH rule 2 x 737 [2,6]
+  CRUSH rule 2 x 738 [5,2]
+  CRUSH rule 2 x 739 [2,7]
+  CRUSH rule 2 x 740 [2,8]
+  CRUSH rule 2 x 741 [7,2]
+  CRUSH rule 2 x 742 [8,2]
+  CRUSH rule 2 x 743 [7,2]
+  CRUSH rule 2 x 744 [5,7]
+  CRUSH rule 2 x 745 [5,2]
+  CRUSH rule 2 x 746 [5,2]
+  CRUSH rule 2 x 747 [6,2]
+  CRUSH rule 2 x 748 [2,7]
+  CRUSH rule 2 x 749 [5,8]
+  CRUSH rule 2 x 750 [2,6]
+  CRUSH rule 2 x 751 [2,8]
+  CRUSH rule 2 x 752 [8,2]
+  CRUSH rule 2 x 753 [7,5]
+  CRUSH rule 2 x 754 [8,5]
+  CRUSH rule 2 x 755 [2,6]
+  CRUSH rule 2 x 756 [5,6]
+  CRUSH rule 2 x 757 [8,2]
+  CRUSH rule 2 x 758 [6,2]
+  CRUSH rule 2 x 759 [8,5]
+  CRUSH rule 2 x 760 [2,5]
+  CRUSH rule 2 x 761 [5,2]
+  CRUSH rule 2 x 762 [2,7]
+  CRUSH rule 2 x 763 [8,5]
+  CRUSH rule 2 x 764 [2,7]
+  CRUSH rule 2 x 765 [6,5]
+  CRUSH rule 2 x 766 [8,5]
+  CRUSH rule 2 x 767 [2,8]
+  CRUSH rule 2 x 768 [8,5]
+  CRUSH rule 2 x 769 [6,2]
+  CRUSH rule 2 x 770 [6,2]
+  CRUSH rule 2 x 771 [7,2]
+  CRUSH rule 2 x 772 [8,5]
+  CRUSH rule 2 x 773 [5,2]
+  CRUSH rule 2 x 774 [5,6]
+  CRUSH rule 2 x 775 [6,5]
+  CRUSH rule 2 x 776 [7,2]
+  CRUSH rule 2 x 777 [5,2]
+  CRUSH rule 2 x 778 [2,8]
+  CRUSH rule 2 x 779 [2,7]
+  CRUSH rule 2 x 780 [2,5]
+  CRUSH rule 2 x 781 [6,5]
+  CRUSH rule 2 x 782 [5,2]
+  CRUSH rule 2 x 783 [7,2]
+  CRUSH rule 2 x 784 [2,5]
+  CRUSH rule 2 x 785 [6,2]
+  CRUSH rule 2 x 786 [7,5]
+  CRUSH rule 2 x 787 [2,6]
+  CRUSH rule 2 x 788 [6,2]
+  CRUSH rule 2 x 789 [2,5]
+  CRUSH rule 2 x 790 [8,5]
+  CRUSH rule 2 x 791 [5,8]
+  CRUSH rule 2 x 792 [5,8]
+  CRUSH rule 2 x 793 [6,2]
+  CRUSH rule 2 x 794 [2,6]
+  CRUSH rule 2 x 795 [2,5]
+  CRUSH rule 2 x 796 [5,7]
+  CRUSH rule 2 x 797 [2,5]
+  CRUSH rule 2 x 798 [6,2]
+  CRUSH rule 2 x 799 [5,2]
+  CRUSH rule 2 x 800 [5,2]
+  CRUSH rule 2 x 801 [5,6]
+  CRUSH rule 2 x 802 [2,8]
+  CRUSH rule 2 x 803 [2,5]
+  CRUSH rule 2 x 804 [6,2]
+  CRUSH rule 2 x 805 [5,6]
+  CRUSH rule 2 x 806 [2,5]
+  CRUSH rule 2 x 807 [5,7]
+  CRUSH rule 2 x 808 [5,6]
+  CRUSH rule 2 x 809 [2,5]
+  CRUSH rule 2 x 810 [5,7]
+  CRUSH rule 2 x 811 [8,5]
+  CRUSH rule 2 x 812 [8,5]
+  CRUSH rule 2 x 813 [6,5]
+  CRUSH rule 2 x 814 [5,6]
+  CRUSH rule 2 x 815 [5,2]
+  CRUSH rule 2 x 816 [2,7]
+  CRUSH rule 2 x 817 [5,8]
+  CRUSH rule 2 x 818 [5,2]
+  CRUSH rule 2 x 819 [5,2]
+  CRUSH rule 2 x 820 [5,6]
+  CRUSH rule 2 x 821 [5,6]
+  CRUSH rule 2 x 822 [2,5]
+  CRUSH rule 2 x 823 [5,8]
+  CRUSH rule 2 x 824 [5,7]
+  CRUSH rule 2 x 825 [2,8]
+  CRUSH rule 2 x 826 [7,2]
+  CRUSH rule 2 x 827 [2,8]
+  CRUSH rule 2 x 828 [2,5]
+  CRUSH rule 2 x 829 [5,6]
+  CRUSH rule 2 x 830 [2,5]
+  CRUSH rule 2 x 831 [2,6]
+  CRUSH rule 2 x 832 [5,7]
+  CRUSH rule 2 x 833 [2,7]
+  CRUSH rule 2 x 834 [5,2]
+  CRUSH rule 2 x 835 [8,5]
+  CRUSH rule 2 x 836 [5,7]
+  CRUSH rule 2 x 837 [6,5]
+  CRUSH rule 2 x 838 [6,2]
+  CRUSH rule 2 x 839 [5,2]
+  CRUSH rule 2 x 840 [7,5]
+  CRUSH rule 2 x 841 [5,8]
+  CRUSH rule 2 x 842 [2,5]
+  CRUSH rule 2 x 843 [6,5]
+  CRUSH rule 2 x 844 [5,8]
+  CRUSH rule 2 x 845 [5,8]
+  CRUSH rule 2 x 846 [5,2]
+  CRUSH rule 2 x 847 [2,8]
+  CRUSH rule 2 x 848 [2,6]
+  CRUSH rule 2 x 849 [5,6]
+  CRUSH rule 2 x 850 [2,5]
+  CRUSH rule 2 x 851 [6,5]
+  CRUSH rule 2 x 852 [7,5]
+  CRUSH rule 2 x 853 [6,2]
+  CRUSH rule 2 x 854 [7,2]
+  CRUSH rule 2 x 855 [5,7]
+  CRUSH rule 2 x 856 [6,5]
+  CRUSH rule 2 x 857 [8,5]
+  CRUSH rule 2 x 858 [6,5]
+  CRUSH rule 2 x 859 [6,2]
+  CRUSH rule 2 x 860 [5,2]
+  CRUSH rule 2 x 861 [8,5]
+  CRUSH rule 2 x 862 [6,2]
+  CRUSH rule 2 x 863 [8,2]
+  CRUSH rule 2 x 864 [5,6]
+  CRUSH rule 2 x 865 [8,2]
+  CRUSH rule 2 x 866 [5,6]
+  CRUSH rule 2 x 867 [6,5]
+  CRUSH rule 2 x 868 [6,5]
+  CRUSH rule 2 x 869 [8,5]
+  CRUSH rule 2 x 870 [2,5]
+  CRUSH rule 2 x 871 [5,2]
+  CRUSH rule 2 x 872 [5,2]
+  CRUSH rule 2 x 873 [5,6]
+  CRUSH rule 2 x 874 [2,6]
+  CRUSH rule 2 x 875 [2,6]
+  CRUSH rule 2 x 876 [5,8]
+  CRUSH rule 2 x 877 [6,5]
+  CRUSH rule 2 x 878 [5,2]
+  CRUSH rule 2 x 879 [7,5]
+  CRUSH rule 2 x 880 [5,2]
+  CRUSH rule 2 x 881 [5,6]
+  CRUSH rule 2 x 882 [5,2]
+  CRUSH rule 2 x 883 [2,5]
+  CRUSH rule 2 x 884 [6,2]
+  CRUSH rule 2 x 885 [5,2]
+  CRUSH rule 2 x 886 [5,6]
+  CRUSH rule 2 x 887 [7,5]
+  CRUSH rule 2 x 888 [6,2]
+  CRUSH rule 2 x 889 [2,6]
+  CRUSH rule 2 x 890 [7,2]
+  CRUSH rule 2 x 891 [2,8]
+  CRUSH rule 2 x 892 [6,2]
+  CRUSH rule 2 x 893 [2,5]
+  CRUSH rule 2 x 894 [7,5]
+  CRUSH rule 2 x 895 [5,2]
+  CRUSH rule 2 x 896 [2,8]
+  CRUSH rule 2 x 897 [5,2]
+  CRUSH rule 2 x 898 [2,5]
+  CRUSH rule 2 x 899 [2,7]
+  CRUSH rule 2 x 900 [5,2]
+  CRUSH rule 2 x 901 [5,2]
+  CRUSH rule 2 x 902 [8,5]
+  CRUSH rule 2 x 903 [5,7]
+  CRUSH rule 2 x 904 [5,6]
+  CRUSH rule 2 x 905 [6,2]
+  CRUSH rule 2 x 906 [2,6]
+  CRUSH rule 2 x 907 [7,2]
+  CRUSH rule 2 x 908 [5,8]
+  CRUSH rule 2 x 909 [2,5]
+  CRUSH rule 2 x 910 [6,5]
+  CRUSH rule 2 x 911 [5,8]
+  CRUSH rule 2 x 912 [2,7]
+  CRUSH rule 2 x 913 [7,2]
+  CRUSH rule 2 x 914 [6,5]
+  CRUSH rule 2 x 915 [8,2]
+  CRUSH rule 2 x 916 [5,2]
+  CRUSH rule 2 x 917 [2,5]
+  CRUSH rule 2 x 918 [8,2]
+  CRUSH rule 2 x 919 [6,2]
+  CRUSH rule 2 x 920 [7,5]
+  CRUSH rule 2 x 921 [2,5]
+  CRUSH rule 2 x 922 [6,5]
+  CRUSH rule 2 x 923 [5,8]
+  CRUSH rule 2 x 924 [5,2]
+  CRUSH rule 2 x 925 [5,7]
+  CRUSH rule 2 x 926 [5,2]
+  CRUSH rule 2 x 927 [2,6]
+  CRUSH rule 2 x 928 [8,2]
+  CRUSH rule 2 x 929 [5,2]
+  CRUSH rule 2 x 930 [2,5]
+  CRUSH rule 2 x 931 [5,2]
+  CRUSH rule 2 x 932 [5,2]
+  CRUSH rule 2 x 933 [8,5]
+  CRUSH rule 2 x 934 [5,6]
+  CRUSH rule 2 x 935 [6,5]
+  CRUSH rule 2 x 936 [2,6]
+  CRUSH rule 2 x 937 [5,8]
+  CRUSH rule 2 x 938 [6,5]
+  CRUSH rule 2 x 939 [2,7]
+  CRUSH rule 2 x 940 [8,5]
+  CRUSH rule 2 x 941 [5,2]
+  CRUSH rule 2 x 942 [2,8]
+  CRUSH rule 2 x 943 [8,2]
+  CRUSH rule 2 x 944 [5,8]
+  CRUSH rule 2 x 945 [7,2]
+  CRUSH rule 2 x 946 [2,8]
+  CRUSH rule 2 x 947 [5,2]
+  CRUSH rule 2 x 948 [7,5]
+  CRUSH rule 2 x 949 [6,2]
+  CRUSH rule 2 x 950 [5,6]
+  CRUSH rule 2 x 951 [5,8]
+  CRUSH rule 2 x 952 [2,7]
+  CRUSH rule 2 x 953 [2,5]
+  CRUSH rule 2 x 954 [5,2]
+  CRUSH rule 2 x 955 [8,2]
+  CRUSH rule 2 x 956 [2,6]
+  CRUSH rule 2 x 957 [7,2]
+  CRUSH rule 2 x 958 [8,5]
+  CRUSH rule 2 x 959 [5,2]
+  CRUSH rule 2 x 960 [5,6]
+  CRUSH rule 2 x 961 [5,2]
+  CRUSH rule 2 x 962 [7,5]
+  CRUSH rule 2 x 963 [2,5]
+  CRUSH rule 2 x 964 [5,2]
+  CRUSH rule 2 x 965 [7,5]
+  CRUSH rule 2 x 966 [5,8]
+  CRUSH rule 2 x 967 [8,5]
+  CRUSH rule 2 x 968 [7,2]
+  CRUSH rule 2 x 969 [8,2]
+  CRUSH rule 2 x 970 [2,6]
+  CRUSH rule 2 x 971 [2,7]
+  CRUSH rule 2 x 972 [2,8]
+  CRUSH rule 2 x 973 [2,6]
+  CRUSH rule 2 x 974 [5,2]
+  CRUSH rule 2 x 975 [5,7]
+  CRUSH rule 2 x 976 [5,8]
+  CRUSH rule 2 x 977 [8,5]
+  CRUSH rule 2 x 978 [7,2]
+  CRUSH rule 2 x 979 [7,2]
+  CRUSH rule 2 x 980 [6,2]
+  CRUSH rule 2 x 981 [7,5]
+  CRUSH rule 2 x 982 [5,2]
+  CRUSH rule 2 x 983 [5,7]
+  CRUSH rule 2 x 984 [2,7]
+  CRUSH rule 2 x 985 [2,5]
+  CRUSH rule 2 x 986 [8,5]
+  CRUSH rule 2 x 987 [2,5]
+  CRUSH rule 2 x 988 [2,5]
+  CRUSH rule 2 x 989 [2,6]
+  CRUSH rule 2 x 990 [2,6]
+  CRUSH rule 2 x 991 [2,5]
+  CRUSH rule 2 x 992 [7,2]
+  CRUSH rule 2 x 993 [2,6]
+  CRUSH rule 2 x 994 [5,7]
+  CRUSH rule 2 x 995 [7,2]
+  CRUSH rule 2 x 996 [6,5]
+  CRUSH rule 2 x 997 [6,5]
+  CRUSH rule 2 x 998 [8,2]
+  CRUSH rule 2 x 999 [2,7]
+  CRUSH rule 2 x 1000 [8,5]
+  CRUSH rule 2 x 1001 [2,5]
+  CRUSH rule 2 x 1002 [2,5]
+  CRUSH rule 2 x 1003 [2,8]
+  CRUSH rule 2 x 1004 [6,2]
+  CRUSH rule 2 x 1005 [6,2]
+  CRUSH rule 2 x 1006 [2,6]
+  CRUSH rule 2 x 1007 [2,5]
+  CRUSH rule 2 x 1008 [2,7]
+  CRUSH rule 2 x 1009 [6,5]
+  CRUSH rule 2 x 1010 [5,2]
+  CRUSH rule 2 x 1011 [5,2]
+  CRUSH rule 2 x 1012 [5,2]
+  CRUSH rule 2 x 1013 [5,2]
+  CRUSH rule 2 x 1014 [2,8]
+  CRUSH rule 2 x 1015 [6,5]
+  CRUSH rule 2 x 1016 [2,5]
+  CRUSH rule 2 x 1017 [6,2]
+  CRUSH rule 2 x 1018 [5,2]
+  CRUSH rule 2 x 1019 [5,8]
+  CRUSH rule 2 x 1020 [5,2]
+  CRUSH rule 2 x 1021 [5,2]
+  CRUSH rule 2 x 1022 [2,6]
+  CRUSH rule 2 x 1023 [5,2]
+  rule 2 (chooseleaf) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 2 x 0 [2,5,7]
+  CRUSH rule 2 x 1 [2,8,5]
+  CRUSH rule 2 x 2 [2,5,7]
+  CRUSH rule 2 x 3 [8,2,5]
+  CRUSH rule 2 x 4 [5,2,7]
+  CRUSH rule 2 x 5 [7,2,5]
+  CRUSH rule 2 x 6 [2,6,5]
+  CRUSH rule 2 x 7 [5,8,2]
+  CRUSH rule 2 x 8 [5,6,2]
+  CRUSH rule 2 x 9 [2,5,8]
+  CRUSH rule 2 x 10 [2,7,5]
+  CRUSH rule 2 x 11 [2,7,5]
+  CRUSH rule 2 x 12 [2,5,6]
+  CRUSH rule 2 x 13 [5,8,2]
+  CRUSH rule 2 x 14 [7,2,5]
+  CRUSH rule 2 x 15 [7,2,5]
+  CRUSH rule 2 x 16 [5,6,2]
+  CRUSH rule 2 x 17 [5,2,6]
+  CRUSH rule 2 x 18 [2,5,6]
+  CRUSH rule 2 x 19 [7,5,2]
+  CRUSH rule 2 x 20 [2,5,7]
+  CRUSH rule 2 x 21 [5,7,2]
+  CRUSH rule 2 x 22 [8,5,2]
+  CRUSH rule 2 x 23 [5,6,2]
+  CRUSH rule 2 x 24 [2,7,5]
+  CRUSH rule 2 x 25 [5,7,2]
+  CRUSH rule 2 x 26 [2,8,5]
+  CRUSH rule 2 x 27 [5,2,8]
+  CRUSH rule 2 x 28 [6,2,5]
+  CRUSH rule 2 x 29 [8,5,2]
+  CRUSH rule 2 x 30 [5,7,2]
+  CRUSH rule 2 x 31 [8,2,5]
+  CRUSH rule 2 x 32 [5,6,2]
+  CRUSH rule 2 x 33 [2,7,5]
+  CRUSH rule 2 x 34 [2,5,7]
+  CRUSH rule 2 x 35 [2,8,5]
+  CRUSH rule 2 x 36 [5,8,2]
+  CRUSH rule 2 x 37 [2,5,6]
+  CRUSH rule 2 x 38 [5,8,2]
+  CRUSH rule 2 x 39 [5,7,2]
+  CRUSH rule 2 x 40 [7,2,5]
+  CRUSH rule 2 x 41 [2,6,5]
+  CRUSH rule 2 x 42 [5,6,2]
+  CRUSH rule 2 x 43 [2,5,7]
+  CRUSH rule 2 x 44 [2,6,5]
+  CRUSH rule 2 x 45 [8,2,5]
+  CRUSH rule 2 x 46 [2,5,8]
+  CRUSH rule 2 x 47 [5,2,8]
+  CRUSH rule 2 x 48 [5,6,2]
+  CRUSH rule 2 x 49 [5,7,2]
+  CRUSH rule 2 x 50 [5,2,7]
+  CRUSH rule 2 x 51 [5,6,2]
+  CRUSH rule 2 x 52 [8,2,5]
+  CRUSH rule 2 x 53 [5,8,2]
+  CRUSH rule 2 x 54 [7,5,2]
+  CRUSH rule 2 x 55 [8,2,5]
+  CRUSH rule 2 x 56 [6,5,2]
+  CRUSH rule 2 x 57 [5,8,2]
+  CRUSH rule 2 x 58 [2,8,5]
+  CRUSH rule 2 x 59 [5,2,7]
+  CRUSH rule 2 x 60 [5,2,6]
+  CRUSH rule 2 x 61 [5,6,2]
+  CRUSH rule 2 x 62 [7,2,5]
+  CRUSH rule 2 x 63 [5,6,2]
+  CRUSH rule 2 x 64 [5,2,8]
+  CRUSH rule 2 x 65 [7,5,2]
+  CRUSH rule 2 x 66 [5,6,2]
+  CRUSH rule 2 x 67 [5,2,8]
+  CRUSH rule 2 x 68 [2,5,8]
+  CRUSH rule 2 x 69 [5,2,6]
+  CRUSH rule 2 x 70 [7,2,5]
+  CRUSH rule 2 x 71 [2,8,5]
+  CRUSH rule 2 x 72 [6,2,5]
+  CRUSH rule 2 x 73 [2,7,5]
+  CRUSH rule 2 x 74 [2,7,5]
+  CRUSH rule 2 x 75 [5,2,6]
+  CRUSH rule 2 x 76 [5,2,7]
+  CRUSH rule 2 x 77 [7,2,5]
+  CRUSH rule 2 x 78 [2,5,8]
+  CRUSH rule 2 x 79 [5,2,7]
+  CRUSH rule 2 x 80 [2,5,6]
+  CRUSH rule 2 x 81 [2,5,6]
+  CRUSH rule 2 x 82 [7,2,5]
+  CRUSH rule 2 x 83 [2,6,5]
+  CRUSH rule 2 x 84 [7,2,5]
+  CRUSH rule 2 x 85 [5,8,2]
+  CRUSH rule 2 x 86 [2,6,5]
+  CRUSH rule 2 x 87 [2,7,5]
+  CRUSH rule 2 x 88 [2,6,5]
+  CRUSH rule 2 x 89 [5,2,7]
+  CRUSH rule 2 x 90 [6,5,2]
+  CRUSH rule 2 x 91 [5,8,2]
+  CRUSH rule 2 x 92 [2,8,5]
+  CRUSH rule 2 x 93 [7,5,2]
+  CRUSH rule 2 x 94 [2,5,8]
+  CRUSH rule 2 x 95 [7,5,2]
+  CRUSH rule 2 x 96 [5,6,2]
+  CRUSH rule 2 x 97 [8,5,2]
+  CRUSH rule 2 x 98 [2,7,5]
+  CRUSH rule 2 x 99 [2,7,5]
+  CRUSH rule 2 x 100 [2,7,5]
+  CRUSH rule 2 x 101 [5,7,2]
+  CRUSH rule 2 x 102 [5,2,7]
+  CRUSH rule 2 x 103 [5,7,2]
+  CRUSH rule 2 x 104 [7,5,2]
+  CRUSH rule 2 x 105 [2,5,6]
+  CRUSH rule 2 x 106 [2,6,5]
+  CRUSH rule 2 x 107 [5,2,6]
+  CRUSH rule 2 x 108 [7,2,5]
+  CRUSH rule 2 x 109 [2,5,6]
+  CRUSH rule 2 x 110 [5,2,7]
+  CRUSH rule 2 x 111 [2,5,6]
+  CRUSH rule 2 x 112 [2,6,5]
+  CRUSH rule 2 x 113 [6,2,5]
+  CRUSH rule 2 x 114 [7,5,2]
+  CRUSH rule 2 x 115 [8,2,5]
+  CRUSH rule 2 x 116 [2,6,5]
+  CRUSH rule 2 x 117 [7,5,2]
+  CRUSH rule 2 x 118 [2,5,8]
+  CRUSH rule 2 x 119 [5,6,2]
+  CRUSH rule 2 x 120 [2,5,7]
+  CRUSH rule 2 x 121 [2,7,5]
+  CRUSH rule 2 x 122 [8,5,2]
+  CRUSH rule 2 x 123 [2,5,8]
+  CRUSH rule 2 x 124 [5,2,8]
+  CRUSH rule 2 x 125 [2,7,5]
+  CRUSH rule 2 x 126 [5,2,6]
+  CRUSH rule 2 x 127 [5,6,2]
+  CRUSH rule 2 x 128 [5,6,2]
+  CRUSH rule 2 x 129 [2,5,7]
+  CRUSH rule 2 x 130 [5,8,2]
+  CRUSH rule 2 x 131 [2,5,8]
+  CRUSH rule 2 x 132 [2,5,6]
+  CRUSH rule 2 x 133 [5,6,2]
+  CRUSH rule 2 x 134 [2,8,5]
+  CRUSH rule 2 x 135 [5,6,2]
+  CRUSH rule 2 x 136 [2,5,6]
+  CRUSH rule 2 x 137 [7,5,2]
+  CRUSH rule 2 x 138 [8,5,2]
+  CRUSH rule 2 x 139 [5,2,7]
+  CRUSH rule 2 x 140 [2,6,5]
+  CRUSH rule 2 x 141 [6,2,5]
+  CRUSH rule 2 x 142 [5,2,8]
+  CRUSH rule 2 x 143 [5,8,2]
+  CRUSH rule 2 x 144 [8,2,5]
+  CRUSH rule 2 x 145 [8,5,2]
+  CRUSH rule 2 x 146 [2,6,5]
+  CRUSH rule 2 x 147 [2,8,5]
+  CRUSH rule 2 x 148 [5,2,7]
+  CRUSH rule 2 x 149 [5,8,2]
+  CRUSH rule 2 x 150 [2,6,5]
+  CRUSH rule 2 x 151 [5,6,2]
+  CRUSH rule 2 x 152 [8,5,2]
+  CRUSH rule 2 x 153 [8,5,2]
+  CRUSH rule 2 x 154 [5,2,7]
+  CRUSH rule 2 x 155 [5,7,2]
+  CRUSH rule 2 x 156 [5,2,6]
+  CRUSH rule 2 x 157 [5,2,6]
+  CRUSH rule 2 x 158 [2,8,5]
+  CRUSH rule 2 x 159 [7,2,5]
+  CRUSH rule 2 x 160 [2,8,5]
+  CRUSH rule 2 x 161 [2,5,6]
+  CRUSH rule 2 x 162 [2,6,5]
+  CRUSH rule 2 x 163 [5,6,2]
+  CRUSH rule 2 x 164 [7,2,5]
+  CRUSH rule 2 x 165 [7,2,5]
+  CRUSH rule 2 x 166 [2,5,6]
+  CRUSH rule 2 x 167 [2,7,5]
+  CRUSH rule 2 x 168 [5,2,7]
+  CRUSH rule 2 x 169 [2,6,5]
+  CRUSH rule 2 x 170 [2,5,8]
+  CRUSH rule 2 x 171 [7,5,2]
+  CRUSH rule 2 x 172 [2,7,5]
+  CRUSH rule 2 x 173 [8,5,2]
+  CRUSH rule 2 x 174 [2,5,7]
+  CRUSH rule 2 x 175 [6,2,5]
+  CRUSH rule 2 x 176 [5,2,7]
+  CRUSH rule 2 x 177 [5,2,6]
+  CRUSH rule 2 x 178 [5,2,8]
+  CRUSH rule 2 x 179 [5,2,7]
+  CRUSH rule 2 x 180 [5,8,2]
+  CRUSH rule 2 x 181 [6,2,5]
+  CRUSH rule 2 x 182 [8,5,2]
+  CRUSH rule 2 x 183 [7,5,2]
+  CRUSH rule 2 x 184 [5,7,2]
+  CRUSH rule 2 x 185 [6,2,5]
+  CRUSH rule 2 x 186 [2,5,7]
+  CRUSH rule 2 x 187 [2,6,5]
+  CRUSH rule 2 x 188 [2,8,5]
+  CRUSH rule 2 x 189 [2,7,5]
+  CRUSH rule 2 x 190 [5,2,8]
+  CRUSH rule 2 x 191 [7,2,5]
+  CRUSH rule 2 x 192 [5,2,8]
+  CRUSH rule 2 x 193 [5,2,6]
+  CRUSH rule 2 x 194 [2,5,8]
+  CRUSH rule 2 x 195 [6,5,2]
+  CRUSH rule 2 x 196 [6,2,5]
+  CRUSH rule 2 x 197 [6,5,2]
+  CRUSH rule 2 x 198 [2,5,6]
+  CRUSH rule 2 x 199 [2,5,7]
+  CRUSH rule 2 x 200 [2,5,6]
+  CRUSH rule 2 x 201 [7,2,5]
+  CRUSH rule 2 x 202 [6,5,2]
+  CRUSH rule 2 x 203 [5,8,2]
+  CRUSH rule 2 x 204 [2,5,7]
+  CRUSH rule 2 x 205 [2,7,5]
+  CRUSH rule 2 x 206 [2,7,5]
+  CRUSH rule 2 x 207 [5,2,7]
+  CRUSH rule 2 x 208 [7,2,5]
+  CRUSH rule 2 x 209 [2,8,5]
+  CRUSH rule 2 x 210 [2,5,8]
+  CRUSH rule 2 x 211 [5,2,8]
+  CRUSH rule 2 x 212 [7,5,2]
+  CRUSH rule 2 x 213 [8,5,2]
+  CRUSH rule 2 x 214 [5,8,2]
+  CRUSH rule 2 x 215 [8,2,5]
+  CRUSH rule 2 x 216 [5,2,6]
+  CRUSH rule 2 x 217 [2,7,5]
+  CRUSH rule 2 x 218 [2,7,5]
+  CRUSH rule 2 x 219 [5,8,2]
+  CRUSH rule 2 x 220 [5,7,2]
+  CRUSH rule 2 x 221 [5,6,2]
+  CRUSH rule 2 x 222 [6,5,2]
+  CRUSH rule 2 x 223 [2,5,6]
+  CRUSH rule 2 x 224 [2,5,8]
+  CRUSH rule 2 x 225 [8,2,5]
+  CRUSH rule 2 x 226 [7,2,5]
+  CRUSH rule 2 x 227 [5,2,6]
+  CRUSH rule 2 x 228 [5,6,2]
+  CRUSH rule 2 x 229 [5,8,2]
+  CRUSH rule 2 x 230 [5,7,2]
+  CRUSH rule 2 x 231 [5,7,2]
+  CRUSH rule 2 x 232 [2,7,5]
+  CRUSH rule 2 x 233 [5,7,2]
+  CRUSH rule 2 x 234 [2,5,6]
+  CRUSH rule 2 x 235 [5,8,2]
+  CRUSH rule 2 x 236 [5,2,7]
+  CRUSH rule 2 x 237 [5,7,2]
+  CRUSH rule 2 x 238 [5,2,6]
+  CRUSH rule 2 x 239 [8,5,2]
+  CRUSH rule 2 x 240 [5,7,2]
+  CRUSH rule 2 x 241 [5,2,8]
+  CRUSH rule 2 x 242 [5,2,6]
+  CRUSH rule 2 x 243 [5,7,2]
+  CRUSH rule 2 x 244 [5,6,2]
+  CRUSH rule 2 x 245 [7,2,5]
+  CRUSH rule 2 x 246 [2,5,8]
+  CRUSH rule 2 x 247 [6,2,5]
+  CRUSH rule 2 x 248 [8,2,5]
+  CRUSH rule 2 x 249 [2,5,7]
+  CRUSH rule 2 x 250 [2,5,6]
+  CRUSH rule 2 x 251 [2,5,6]
+  CRUSH rule 2 x 252 [5,7,2]
+  CRUSH rule 2 x 253 [5,2,6]
+  CRUSH rule 2 x 254 [5,2,7]
+  CRUSH rule 2 x 255 [2,7,5]
+  CRUSH rule 2 x 256 [5,7,2]
+  CRUSH rule 2 x 257 [2,8,5]
+  CRUSH rule 2 x 258 [5,2,6]
+  CRUSH rule 2 x 259 [5,6,2]
+  CRUSH rule 2 x 260 [5,6,2]
+  CRUSH rule 2 x 261 [8,5,2]
+  CRUSH rule 2 x 262 [5,6,2]
+  CRUSH rule 2 x 263 [6,2,5]
+  CRUSH rule 2 x 264 [5,6,2]
+  CRUSH rule 2 x 265 [8,5,2]
+  CRUSH rule 2 x 266 [8,2,5]
+  CRUSH rule 2 x 267 [2,5,8]
+  CRUSH rule 2 x 268 [2,7,5]
+  CRUSH rule 2 x 269 [2,8,5]
+  CRUSH rule 2 x 270 [5,2,7]
+  CRUSH rule 2 x 271 [7,5,2]
+  CRUSH rule 2 x 272 [2,8,5]
+  CRUSH rule 2 x 273 [5,2,7]
+  CRUSH rule 2 x 274 [6,5,2]
+  CRUSH rule 2 x 275 [5,7,2]
+  CRUSH rule 2 x 276 [7,2,5]
+  CRUSH rule 2 x 277 [6,5,2]
+  CRUSH rule 2 x 278 [6,2,5]
+  CRUSH rule 2 x 279 [8,5,2]
+  CRUSH rule 2 x 280 [2,6,5]
+  CRUSH rule 2 x 281 [8,2,5]
+  CRUSH rule 2 x 282 [5,2,6]
+  CRUSH rule 2 x 283 [8,2,5]
+  CRUSH rule 2 x 284 [6,5,2]
+  CRUSH rule 2 x 285 [5,7,2]
+  CRUSH rule 2 x 286 [2,6,5]
+  CRUSH rule 2 x 287 [2,5,6]
+  CRUSH rule 2 x 288 [8,2,5]
+  CRUSH rule 2 x 289 [5,6,2]
+  CRUSH rule 2 x 290 [2,5,7]
+  CRUSH rule 2 x 291 [2,5,8]
+  CRUSH rule 2 x 292 [8,2,5]
+  CRUSH rule 2 x 293 [6,2,5]
+  CRUSH rule 2 x 294 [7,5,2]
+  CRUSH rule 2 x 295 [5,8,2]
+  CRUSH rule 2 x 296 [5,2,6]
+  CRUSH rule 2 x 297 [6,2,5]
+  CRUSH rule 2 x 298 [2,5,7]
+  CRUSH rule 2 x 299 [2,8,5]
+  CRUSH rule 2 x 300 [8,5,2]
+  CRUSH rule 2 x 301 [2,8,5]
+  CRUSH rule 2 x 302 [5,2,6]
+  CRUSH rule 2 x 303 [7,5,2]
+  CRUSH rule 2 x 304 [2,7,5]
+  CRUSH rule 2 x 305 [5,8,2]
+  CRUSH rule 2 x 306 [2,7,5]
+  CRUSH rule 2 x 307 [2,7,5]
+  CRUSH rule 2 x 308 [2,8,5]
+  CRUSH rule 2 x 309 [7,5,2]
+  CRUSH rule 2 x 310 [5,2,7]
+  CRUSH rule 2 x 311 [5,6,2]
+  CRUSH rule 2 x 312 [2,6,5]
+  CRUSH rule 2 x 313 [5,2,6]
+  CRUSH rule 2 x 314 [5,2,6]
+  CRUSH rule 2 x 315 [2,5,8]
+  CRUSH rule 2 x 316 [6,5,2]
+  CRUSH rule 2 x 317 [2,6,5]
+  CRUSH rule 2 x 318 [8,2,5]
+  CRUSH rule 2 x 319 [5,2,8]
+  CRUSH rule 2 x 320 [5,7,2]
+  CRUSH rule 2 x 321 [2,5,8]
+  CRUSH rule 2 x 322 [2,7,5]
+  CRUSH rule 2 x 323 [5,7,2]
+  CRUSH rule 2 x 324 [7,2,5]
+  CRUSH rule 2 x 325 [5,6,2]
+  CRUSH rule 2 x 326 [5,2,6]
+  CRUSH rule 2 x 327 [2,6,5]
+  CRUSH rule 2 x 328 [7,5,2]
+  CRUSH rule 2 x 329 [5,6,2]
+  CRUSH rule 2 x 330 [5,7,2]
+  CRUSH rule 2 x 331 [2,6,5]
+  CRUSH rule 2 x 332 [2,5,6]
+  CRUSH rule 2 x 333 [6,5,2]
+  CRUSH rule 2 x 334 [8,5,2]
+  CRUSH rule 2 x 335 [7,2,5]
+  CRUSH rule 2 x 336 [5,6,2]
+  CRUSH rule 2 x 337 [7,2,5]
+  CRUSH rule 2 x 338 [5,6,2]
+  CRUSH rule 2 x 339 [7,5,2]
+  CRUSH rule 2 x 340 [2,8,5]
+  CRUSH rule 2 x 341 [5,2,7]
+  CRUSH rule 2 x 342 [2,7,5]
+  CRUSH rule 2 x 343 [6,5,2]
+  CRUSH rule 2 x 344 [6,2,5]
+  CRUSH rule 2 x 345 [5,7,2]
+  CRUSH rule 2 x 346 [8,2,5]
+  CRUSH rule 2 x 347 [5,2,7]
+  CRUSH rule 2 x 348 [8,2,5]
+  CRUSH rule 2 x 349 [2,6,5]
+  CRUSH rule 2 x 350 [8,5,2]
+  CRUSH rule 2 x 351 [5,6,2]
+  CRUSH rule 2 x 352 [2,8,5]
+  CRUSH rule 2 x 353 [6,5,2]
+  CRUSH rule 2 x 354 [2,5,6]
+  CRUSH rule 2 x 355 [5,8,2]
+  CRUSH rule 2 x 356 [5,2,6]
+  CRUSH rule 2 x 357 [6,2,5]
+  CRUSH rule 2 x 358 [2,8,5]
+  CRUSH rule 2 x 359 [6,2,5]
+  CRUSH rule 2 x 360 [5,2,7]
+  CRUSH rule 2 x 361 [8,5,2]
+  CRUSH rule 2 x 362 [5,2,6]
+  CRUSH rule 2 x 363 [5,2,6]
+  CRUSH rule 2 x 364 [2,5,6]
+  CRUSH rule 2 x 365 [6,5,2]
+  CRUSH rule 2 x 366 [7,2,5]
+  CRUSH rule 2 x 367 [5,2,6]
+  CRUSH rule 2 x 368 [7,5,2]
+  CRUSH rule 2 x 369 [5,7,2]
+  CRUSH rule 2 x 370 [8,2,5]
+  CRUSH rule 2 x 371 [2,5,6]
+  CRUSH rule 2 x 372 [5,2,8]
+  CRUSH rule 2 x 373 [2,6,5]
+  CRUSH rule 2 x 374 [5,8,2]
+  CRUSH rule 2 x 375 [6,5,2]
+  CRUSH rule 2 x 376 [7,2,5]
+  CRUSH rule 2 x 377 [2,5,6]
+  CRUSH rule 2 x 378 [2,8,5]
+  CRUSH rule 2 x 379 [8,5,2]
+  CRUSH rule 2 x 380 [2,5,8]
+  CRUSH rule 2 x 381 [2,5,7]
+  CRUSH rule 2 x 382 [2,5,8]
+  CRUSH rule 2 x 383 [5,6,2]
+  CRUSH rule 2 x 384 [7,2,5]
+  CRUSH rule 2 x 385 [7,5,2]
+  CRUSH rule 2 x 386 [2,5,6]
+  CRUSH rule 2 x 387 [2,5,6]
+  CRUSH rule 2 x 388 [5,2,8]
+  CRUSH rule 2 x 389 [2,5,7]
+  CRUSH rule 2 x 390 [5,6,2]
+  CRUSH rule 2 x 391 [5,6,2]
+  CRUSH rule 2 x 392 [2,8,5]
+  CRUSH rule 2 x 393 [5,2,6]
+  CRUSH rule 2 x 394 [5,7,2]
+  CRUSH rule 2 x 395 [5,2,8]
+  CRUSH rule 2 x 396 [5,2,7]
+  CRUSH rule 2 x 397 [2,5,8]
+  CRUSH rule 2 x 398 [2,5,8]
+  CRUSH rule 2 x 399 [8,5,2]
+  CRUSH rule 2 x 400 [8,2,5]
+  CRUSH rule 2 x 401 [2,5,8]
+  CRUSH rule 2 x 402 [7,5,2]
+  CRUSH rule 2 x 403 [2,5,8]
+  CRUSH rule 2 x 404 [5,2,8]
+  CRUSH rule 2 x 405 [6,5,2]
+  CRUSH rule 2 x 406 [2,6,5]
+  CRUSH rule 2 x 407 [2,8,5]
+  CRUSH rule 2 x 408 [5,2,6]
+  CRUSH rule 2 x 409 [7,5,2]
+  CRUSH rule 2 x 410 [8,5,2]
+  CRUSH rule 2 x 411 [2,8,5]
+  CRUSH rule 2 x 412 [2,5,8]
+  CRUSH rule 2 x 413 [5,2,8]
+  CRUSH rule 2 x 414 [5,2,6]
+  CRUSH rule 2 x 415 [2,6,5]
+  CRUSH rule 2 x 416 [2,6,5]
+  CRUSH rule 2 x 417 [8,2,5]
+  CRUSH rule 2 x 418 [7,2,5]
+  CRUSH rule 2 x 419 [8,5,2]
+  CRUSH rule 2 x 420 [2,5,6]
+  CRUSH rule 2 x 421 [8,5,2]
+  CRUSH rule 2 x 422 [6,5,2]
+  CRUSH rule 2 x 423 [2,5,6]
+  CRUSH rule 2 x 424 [8,5,2]
+  CRUSH rule 2 x 425 [2,5,7]
+  CRUSH rule 2 x 426 [6,2,5]
+  CRUSH rule 2 x 427 [2,7,5]
+  CRUSH rule 2 x 428 [5,7,2]
+  CRUSH rule 2 x 429 [5,6,2]
+  CRUSH rule 2 x 430 [5,6,2]
+  CRUSH rule 2 x 431 [5,2,7]
+  CRUSH rule 2 x 432 [7,2,5]
+  CRUSH rule 2 x 433 [6,5,2]
+  CRUSH rule 2 x 434 [5,2,8]
+  CRUSH rule 2 x 435 [2,5,6]
+  CRUSH rule 2 x 436 [5,2,7]
+  CRUSH rule 2 x 437 [7,5,2]
+  CRUSH rule 2 x 438 [2,5,8]
+  CRUSH rule 2 x 439 [2,5,7]
+  CRUSH rule 2 x 440 [2,7,5]
+  CRUSH rule 2 x 441 [5,7,2]
+  CRUSH rule 2 x 442 [2,5,7]
+  CRUSH rule 2 x 443 [6,2,5]
+  CRUSH rule 2 x 444 [7,2,5]
+  CRUSH rule 2 x 445 [6,5,2]
+  CRUSH rule 2 x 446 [5,2,8]
+  CRUSH rule 2 x 447 [2,5,7]
+  CRUSH rule 2 x 448 [7,2,5]
+  CRUSH rule 2 x 449 [7,5,2]
+  CRUSH rule 2 x 450 [5,2,8]
+  CRUSH rule 2 x 451 [6,5,2]
+  CRUSH rule 2 x 452 [8,5,2]
+  CRUSH rule 2 x 453 [6,5,2]
+  CRUSH rule 2 x 454 [6,5,2]
+  CRUSH rule 2 x 455 [2,7,5]
+  CRUSH rule 2 x 456 [6,2,5]
+  CRUSH rule 2 x 457 [7,2,5]
+  CRUSH rule 2 x 458 [2,8,5]
+  CRUSH rule 2 x 459 [2,7,5]
+  CRUSH rule 2 x 460 [6,5,2]
+  CRUSH rule 2 x 461 [6,5,2]
+  CRUSH rule 2 x 462 [8,2,5]
+  CRUSH rule 2 x 463 [6,2,5]
+  CRUSH rule 2 x 464 [7,5,2]
+  CRUSH rule 2 x 465 [7,2,5]
+  CRUSH rule 2 x 466 [5,8,2]
+  CRUSH rule 2 x 467 [6,5,2]
+  CRUSH rule 2 x 468 [7,2,5]
+  CRUSH rule 2 x 469 [7,2,5]
+  CRUSH rule 2 x 470 [5,2,6]
+  CRUSH rule 2 x 471 [2,7,5]
+  CRUSH rule 2 x 472 [5,2,8]
+  CRUSH rule 2 x 473 [2,5,6]
+  CRUSH rule 2 x 474 [6,2,5]
+  CRUSH rule 2 x 475 [6,2,5]
+  CRUSH rule 2 x 476 [5,6,2]
+  CRUSH rule 2 x 477 [5,8,2]
+  CRUSH rule 2 x 478 [6,2,5]
+  CRUSH rule 2 x 479 [2,5,8]
+  CRUSH rule 2 x 480 [2,8,5]
+  CRUSH rule 2 x 481 [2,5,7]
+  CRUSH rule 2 x 482 [5,7,2]
+  CRUSH rule 2 x 483 [2,6,5]
+  CRUSH rule 2 x 484 [2,7,5]
+  CRUSH rule 2 x 485 [5,7,2]
+  CRUSH rule 2 x 486 [5,2,7]
+  CRUSH rule 2 x 487 [5,2,8]
+  CRUSH rule 2 x 488 [5,7,2]
+  CRUSH rule 2 x 489 [2,8,5]
+  CRUSH rule 2 x 490 [6,5,2]
+  CRUSH rule 2 x 491 [2,7,5]
+  CRUSH rule 2 x 492 [6,5,2]
+  CRUSH rule 2 x 493 [2,7,5]
+  CRUSH rule 2 x 494 [2,7,5]
+  CRUSH rule 2 x 495 [5,2,8]
+  CRUSH rule 2 x 496 [7,5,2]
+  CRUSH rule 2 x 497 [5,7,2]
+  CRUSH rule 2 x 498 [2,5,8]
+  CRUSH rule 2 x 499 [8,5,2]
+  CRUSH rule 2 x 500 [5,6,2]
+  CRUSH rule 2 x 501 [2,7,5]
+  CRUSH rule 2 x 502 [7,2,5]
+  CRUSH rule 2 x 503 [2,5,7]
+  CRUSH rule 2 x 504 [5,6,2]
+  CRUSH rule 2 x 505 [2,7,5]
+  CRUSH rule 2 x 506 [5,2,8]
+  CRUSH rule 2 x 507 [6,2,5]
+  CRUSH rule 2 x 508 [2,5,8]
+  CRUSH rule 2 x 509 [7,5,2]
+  CRUSH rule 2 x 510 [6,2,5]
+  CRUSH rule 2 x 511 [5,8,2]
+  CRUSH rule 2 x 512 [7,2,5]
+  CRUSH rule 2 x 513 [7,2,5]
+  CRUSH rule 2 x 514 [5,6,2]
+  CRUSH rule 2 x 515 [8,5,2]
+  CRUSH rule 2 x 516 [5,2,8]
+  CRUSH rule 2 x 517 [7,2,5]
+  CRUSH rule 2 x 518 [5,6,2]
+  CRUSH rule 2 x 519 [7,5,2]
+  CRUSH rule 2 x 520 [2,6,5]
+  CRUSH rule 2 x 521 [8,2,5]
+  CRUSH rule 2 x 522 [6,2,5]
+  CRUSH rule 2 x 523 [5,2,7]
+  CRUSH rule 2 x 524 [2,5,8]
+  CRUSH rule 2 x 525 [2,5,6]
+  CRUSH rule 2 x 526 [2,5,8]
+  CRUSH rule 2 x 527 [2,5,6]
+  CRUSH rule 2 x 528 [5,2,6]
+  CRUSH rule 2 x 529 [5,7,2]
+  CRUSH rule 2 x 530 [6,5,2]
+  CRUSH rule 2 x 531 [6,2,5]
+  CRUSH rule 2 x 532 [6,5,2]
+  CRUSH rule 2 x 533 [5,6,2]
+  CRUSH rule 2 x 534 [7,5,2]
+  CRUSH rule 2 x 535 [8,2,5]
+  CRUSH rule 2 x 536 [6,2,5]
+  CRUSH rule 2 x 537 [5,7,2]
+  CRUSH rule 2 x 538 [6,5,2]
+  CRUSH rule 2 x 539 [8,5,2]
+  CRUSH rule 2 x 540 [2,6,5]
+  CRUSH rule 2 x 541 [2,5,8]
+  CRUSH rule 2 x 542 [5,2,8]
+  CRUSH rule 2 x 543 [6,2,5]
+  CRUSH rule 2 x 544 [5,7,2]
+  CRUSH rule 2 x 545 [5,7,2]
+  CRUSH rule 2 x 546 [6,2,5]
+  CRUSH rule 2 x 547 [8,2,5]
+  CRUSH rule 2 x 548 [5,2,8]
+  CRUSH rule 2 x 549 [5,8,2]
+  CRUSH rule 2 x 550 [2,5,7]
+  CRUSH rule 2 x 551 [7,5,2]
+  CRUSH rule 2 x 552 [5,8,2]
+  CRUSH rule 2 x 553 [5,2,7]
+  CRUSH rule 2 x 554 [2,8,5]
+  CRUSH rule 2 x 555 [5,2,8]
+  CRUSH rule 2 x 556 [5,6,2]
+  CRUSH rule 2 x 557 [7,5,2]
+  CRUSH rule 2 x 558 [5,2,6]
+  CRUSH rule 2 x 559 [5,2,6]
+  CRUSH rule 2 x 560 [8,5,2]
+  CRUSH rule 2 x 561 [6,5,2]
+  CRUSH rule 2 x 562 [5,2,6]
+  CRUSH rule 2 x 563 [2,6,5]
+  CRUSH rule 2 x 564 [5,2,7]
+  CRUSH rule 2 x 565 [5,6,2]
+  CRUSH rule 2 x 566 [5,7,2]
+  CRUSH rule 2 x 567 [5,6,2]
+  CRUSH rule 2 x 568 [7,5,2]
+  CRUSH rule 2 x 569 [5,2,7]
+  CRUSH rule 2 x 570 [2,5,8]
+  CRUSH rule 2 x 571 [5,7,2]
+  CRUSH rule 2 x 572 [5,2,8]
+  CRUSH rule 2 x 573 [5,2,7]
+  CRUSH rule 2 x 574 [2,5,8]
+  CRUSH rule 2 x 575 [8,2,5]
+  CRUSH rule 2 x 576 [5,6,2]
+  CRUSH rule 2 x 577 [8,2,5]
+  CRUSH rule 2 x 578 [6,2,5]
+  CRUSH rule 2 x 579 [5,2,6]
+  CRUSH rule 2 x 580 [5,2,7]
+  CRUSH rule 2 x 581 [7,2,5]
+  CRUSH rule 2 x 582 [2,8,5]
+  CRUSH rule 2 x 583 [6,2,5]
+  CRUSH rule 2 x 584 [8,2,5]
+  CRUSH rule 2 x 585 [7,2,5]
+  CRUSH rule 2 x 586 [2,7,5]
+  CRUSH rule 2 x 587 [2,5,7]
+  CRUSH rule 2 x 588 [5,7,2]
+  CRUSH rule 2 x 589 [7,2,5]
+  CRUSH rule 2 x 590 [6,2,5]
+  CRUSH rule 2 x 591 [5,2,8]
+  CRUSH rule 2 x 592 [2,5,7]
+  CRUSH rule 2 x 593 [2,8,5]
+  CRUSH rule 2 x 594 [2,7,5]
+  CRUSH rule 2 x 595 [7,2,5]
+  CRUSH rule 2 x 596 [5,2,6]
+  CRUSH rule 2 x 597 [5,2,7]
+  CRUSH rule 2 x 598 [5,2,6]
+  CRUSH rule 2 x 599 [5,2,8]
+  CRUSH rule 2 x 600 [7,2,5]
+  CRUSH rule 2 x 601 [2,7,5]
+  CRUSH rule 2 x 602 [5,7,2]
+  CRUSH rule 2 x 603 [5,2,6]
+  CRUSH rule 2 x 604 [7,5,2]
+  CRUSH rule 2 x 605 [5,2,7]
+  CRUSH rule 2 x 606 [2,7,5]
+  CRUSH rule 2 x 607 [2,5,8]
+  CRUSH rule 2 x 608 [5,2,7]
+  CRUSH rule 2 x 609 [5,2,8]
+  CRUSH rule 2 x 610 [5,7,2]
+  CRUSH rule 2 x 611 [2,8,5]
+  CRUSH rule 2 x 612 [2,6,5]
+  CRUSH rule 2 x 613 [7,2,5]
+  CRUSH rule 2 x 614 [7,2,5]
+  CRUSH rule 2 x 615 [6,2,5]
+  CRUSH rule 2 x 616 [2,8,5]
+  CRUSH rule 2 x 617 [6,2,5]
+  CRUSH rule 2 x 618 [7,5,2]
+  CRUSH rule 2 x 619 [5,2,8]
+  CRUSH rule 2 x 620 [5,2,6]
+  CRUSH rule 2 x 621 [5,8,2]
+  CRUSH rule 2 x 622 [2,5,8]
+  CRUSH rule 2 x 623 [2,6,5]
+  CRUSH rule 2 x 624 [5,2,8]
+  CRUSH rule 2 x 625 [2,5,7]
+  CRUSH rule 2 x 626 [7,2,5]
+  CRUSH rule 2 x 627 [2,7,5]
+  CRUSH rule 2 x 628 [8,2,5]
+  CRUSH rule 2 x 629 [2,6,5]
+  CRUSH rule 2 x 630 [2,7,5]
+  CRUSH rule 2 x 631 [2,7,5]
+  CRUSH rule 2 x 632 [7,2,5]
+  CRUSH rule 2 x 633 [8,5,2]
+  CRUSH rule 2 x 634 [2,5,7]
+  CRUSH rule 2 x 635 [5,6,2]
+  CRUSH rule 2 x 636 [2,5,8]
+  CRUSH rule 2 x 637 [5,2,7]
+  CRUSH rule 2 x 638 [6,2,5]
+  CRUSH rule 2 x 639 [5,2,8]
+  CRUSH rule 2 x 640 [5,2,8]
+  CRUSH rule 2 x 641 [7,2,5]
+  CRUSH rule 2 x 642 [2,7,5]
+  CRUSH rule 2 x 643 [5,2,8]
+  CRUSH rule 2 x 644 [8,2,5]
+  CRUSH rule 2 x 645 [5,7,2]
+  CRUSH rule 2 x 646 [8,2,5]
+  CRUSH rule 2 x 647 [7,2,5]
+  CRUSH rule 2 x 648 [2,6,5]
+  CRUSH rule 2 x 649 [5,7,2]
+  CRUSH rule 2 x 650 [7,5,2]
+  CRUSH rule 2 x 651 [5,7,2]
+  CRUSH rule 2 x 652 [5,7,2]
+  CRUSH rule 2 x 653 [8,5,2]
+  CRUSH rule 2 x 654 [7,5,2]
+  CRUSH rule 2 x 655 [2,5,7]
+  CRUSH rule 2 x 656 [5,7,2]
+  CRUSH rule 2 x 657 [6,2,5]
+  CRUSH rule 2 x 658 [5,6,2]
+  CRUSH rule 2 x 659 [5,6,2]
+  CRUSH rule 2 x 660 [7,5,2]
+  CRUSH rule 2 x 661 [2,8,5]
+  CRUSH rule 2 x 662 [5,2,7]
+  CRUSH rule 2 x 663 [2,5,8]
+  CRUSH rule 2 x 664 [2,5,7]
+  CRUSH rule 2 x 665 [5,7,2]
+  CRUSH rule 2 x 666 [2,8,5]
+  CRUSH rule 2 x 667 [2,5,7]
+  CRUSH rule 2 x 668 [5,7,2]
+  CRUSH rule 2 x 669 [6,5,2]
+  CRUSH rule 2 x 670 [5,2,6]
+  CRUSH rule 2 x 671 [2,7,5]
+  CRUSH rule 2 x 672 [5,2,7]
+  CRUSH rule 2 x 673 [5,2,7]
+  CRUSH rule 2 x 674 [5,2,8]
+  CRUSH rule 2 x 675 [2,8,5]
+  CRUSH rule 2 x 676 [2,5,8]
+  CRUSH rule 2 x 677 [5,2,7]
+  CRUSH rule 2 x 678 [2,5,8]
+  CRUSH rule 2 x 679 [6,2,5]
+  CRUSH rule 2 x 680 [2,5,6]
+  CRUSH rule 2 x 681 [5,7,2]
+  CRUSH rule 2 x 682 [2,5,7]
+  CRUSH rule 2 x 683 [2,5,6]
+  CRUSH rule 2 x 684 [7,2,5]
+  CRUSH rule 2 x 685 [7,2,5]
+  CRUSH rule 2 x 686 [2,5,8]
+  CRUSH rule 2 x 687 [5,6,2]
+  CRUSH rule 2 x 688 [5,7,2]
+  CRUSH rule 2 x 689 [6,5,2]
+  CRUSH rule 2 x 690 [8,2,5]
+  CRUSH rule 2 x 691 [5,2,6]
+  CRUSH rule 2 x 692 [7,2,5]
+  CRUSH rule 2 x 693 [6,5,2]
+  CRUSH rule 2 x 694 [6,5,2]
+  CRUSH rule 2 x 695 [2,8,5]
+  CRUSH rule 2 x 696 [2,5,8]
+  CRUSH rule 2 x 697 [6,2,5]
+  CRUSH rule 2 x 698 [6,2,5]
+  CRUSH rule 2 x 699 [2,6,5]
+  CRUSH rule 2 x 700 [2,5,7]
+  CRUSH rule 2 x 701 [5,2,7]
+  CRUSH rule 2 x 702 [5,2,8]
+  CRUSH rule 2 x 703 [8,5,2]
+  CRUSH rule 2 x 704 [2,5,8]
+  CRUSH rule 2 x 705 [8,2,5]
+  CRUSH rule 2 x 706 [2,5,6]
+  CRUSH rule 2 x 707 [7,5,2]
+  CRUSH rule 2 x 708 [5,7,2]
+  CRUSH rule 2 x 709 [6,5,2]
+  CRUSH rule 2 x 710 [8,5,2]
+  CRUSH rule 2 x 711 [2,5,8]
+  CRUSH rule 2 x 712 [2,5,7]
+  CRUSH rule 2 x 713 [6,5,2]
+  CRUSH rule 2 x 714 [5,2,7]
+  CRUSH rule 2 x 715 [2,5,6]
+  CRUSH rule 2 x 716 [5,6,2]
+  CRUSH rule 2 x 717 [8,2,5]
+  CRUSH rule 2 x 718 [5,7,2]
+  CRUSH rule 2 x 719 [2,6,5]
+  CRUSH rule 2 x 720 [6,2,5]
+  CRUSH rule 2 x 721 [5,7,2]
+  CRUSH rule 2 x 722 [5,7,2]
+  CRUSH rule 2 x 723 [5,2,7]
+  CRUSH rule 2 x 724 [2,6,5]
+  CRUSH rule 2 x 725 [2,5,7]
+  CRUSH rule 2 x 726 [5,8,2]
+  CRUSH rule 2 x 727 [5,6,2]
+  CRUSH rule 2 x 728 [2,7,5]
+  CRUSH rule 2 x 729 [5,6,2]
+  CRUSH rule 2 x 730 [5,7,2]
+  CRUSH rule 2 x 731 [5,2,8]
+  CRUSH rule 2 x 732 [2,5,6]
+  CRUSH rule 2 x 733 [5,7,2]
+  CRUSH rule 2 x 734 [6,5,2]
+  CRUSH rule 2 x 735 [5,8,2]
+  CRUSH rule 2 x 736 [5,8,2]
+  CRUSH rule 2 x 737 [2,6,5]
+  CRUSH rule 2 x 738 [5,2,7]
+  CRUSH rule 2 x 739 [2,7,5]
+  CRUSH rule 2 x 740 [2,8,5]
+  CRUSH rule 2 x 741 [7,2,5]
+  CRUSH rule 2 x 742 [8,2,5]
+  CRUSH rule 2 x 743 [7,2,5]
+  CRUSH rule 2 x 744 [5,7,2]
+  CRUSH rule 2 x 745 [5,2,8]
+  CRUSH rule 2 x 746 [5,2,7]
+  CRUSH rule 2 x 747 [6,2,5]
+  CRUSH rule 2 x 748 [2,7,5]
+  CRUSH rule 2 x 749 [5,8,2]
+  CRUSH rule 2 x 750 [2,6,5]
+  CRUSH rule 2 x 751 [2,8,5]
+  CRUSH rule 2 x 752 [8,2,5]
+  CRUSH rule 2 x 753 [7,5,2]
+  CRUSH rule 2 x 754 [8,5,2]
+  CRUSH rule 2 x 755 [2,6,5]
+  CRUSH rule 2 x 756 [5,6,2]
+  CRUSH rule 2 x 757 [8,2,5]
+  CRUSH rule 2 x 758 [6,2,5]
+  CRUSH rule 2 x 759 [8,5,2]
+  CRUSH rule 2 x 760 [2,5,7]
+  CRUSH rule 2 x 761 [5,2,8]
+  CRUSH rule 2 x 762 [2,7,5]
+  CRUSH rule 2 x 763 [8,5,2]
+  CRUSH rule 2 x 764 [2,7,5]
+  CRUSH rule 2 x 765 [6,5,2]
+  CRUSH rule 2 x 766 [8,5,2]
+  CRUSH rule 2 x 767 [2,8,5]
+  CRUSH rule 2 x 768 [8,5,2]
+  CRUSH rule 2 x 769 [6,2,5]
+  CRUSH rule 2 x 770 [6,2,5]
+  CRUSH rule 2 x 771 [7,2,5]
+  CRUSH rule 2 x 772 [8,5,2]
+  CRUSH rule 2 x 773 [5,2,7]
+  CRUSH rule 2 x 774 [5,6,2]
+  CRUSH rule 2 x 775 [6,5,2]
+  CRUSH rule 2 x 776 [7,2,5]
+  CRUSH rule 2 x 777 [5,2,6]
+  CRUSH rule 2 x 778 [2,8,5]
+  CRUSH rule 2 x 779 [2,7,5]
+  CRUSH rule 2 x 780 [2,5,7]
+  CRUSH rule 2 x 781 [6,5,2]
+  CRUSH rule 2 x 782 [5,2,8]
+  CRUSH rule 2 x 783 [7,2,5]
+  CRUSH rule 2 x 784 [2,5,6]
+  CRUSH rule 2 x 785 [6,2,5]
+  CRUSH rule 2 x 786 [7,5,2]
+  CRUSH rule 2 x 787 [2,6,5]
+  CRUSH rule 2 x 788 [6,2,5]
+  CRUSH rule 2 x 789 [2,5,8]
+  CRUSH rule 2 x 790 [8,5,2]
+  CRUSH rule 2 x 791 [5,8,2]
+  CRUSH rule 2 x 792 [5,8,2]
+  CRUSH rule 2 x 793 [6,2,5]
+  CRUSH rule 2 x 794 [2,6,5]
+  CRUSH rule 2 x 795 [2,5,8]
+  CRUSH rule 2 x 796 [5,7,2]
+  CRUSH rule 2 x 797 [2,5,8]
+  CRUSH rule 2 x 798 [6,2,5]
+  CRUSH rule 2 x 799 [5,2,8]
+  CRUSH rule 2 x 800 [5,2,7]
+  CRUSH rule 2 x 801 [5,6,2]
+  CRUSH rule 2 x 802 [2,8,5]
+  CRUSH rule 2 x 803 [2,5,7]
+  CRUSH rule 2 x 804 [6,2,5]
+  CRUSH rule 2 x 805 [5,6,2]
+  CRUSH rule 2 x 806 [2,5,7]
+  CRUSH rule 2 x 807 [5,7,2]
+  CRUSH rule 2 x 808 [5,6,2]
+  CRUSH rule 2 x 809 [2,5,8]
+  CRUSH rule 2 x 810 [5,7,2]
+  CRUSH rule 2 x 811 [8,5,2]
+  CRUSH rule 2 x 812 [8,5,2]
+  CRUSH rule 2 x 813 [6,5,2]
+  CRUSH rule 2 x 814 [5,6,2]
+  CRUSH rule 2 x 815 [5,2,8]
+  CRUSH rule 2 x 816 [2,7,5]
+  CRUSH rule 2 x 817 [5,8,2]
+  CRUSH rule 2 x 818 [5,2,7]
+  CRUSH rule 2 x 819 [5,2,8]
+  CRUSH rule 2 x 820 [5,6,2]
+  CRUSH rule 2 x 821 [5,6,2]
+  CRUSH rule 2 x 822 [2,5,8]
+  CRUSH rule 2 x 823 [5,8,2]
+  CRUSH rule 2 x 824 [5,7,2]
+  CRUSH rule 2 x 825 [2,8,5]
+  CRUSH rule 2 x 826 [7,2,5]
+  CRUSH rule 2 x 827 [2,8,5]
+  CRUSH rule 2 x 828 [2,5,8]
+  CRUSH rule 2 x 829 [5,6,2]
+  CRUSH rule 2 x 830 [2,5,8]
+  CRUSH rule 2 x 831 [2,6,5]
+  CRUSH rule 2 x 832 [5,7,2]
+  CRUSH rule 2 x 833 [2,7,5]
+  CRUSH rule 2 x 834 [5,2,7]
+  CRUSH rule 2 x 835 [8,5,2]
+  CRUSH rule 2 x 836 [5,7,2]
+  CRUSH rule 2 x 837 [6,5,2]
+  CRUSH rule 2 x 838 [6,2,5]
+  CRUSH rule 2 x 839 [5,2,6]
+  CRUSH rule 2 x 840 [7,5,2]
+  CRUSH rule 2 x 841 [5,8,2]
+  CRUSH rule 2 x 842 [2,5,6]
+  CRUSH rule 2 x 843 [6,5,2]
+  CRUSH rule 2 x 844 [5,8,2]
+  CRUSH rule 2 x 845 [5,8,2]
+  CRUSH rule 2 x 846 [5,2,7]
+  CRUSH rule 2 x 847 [2,8,5]
+  CRUSH rule 2 x 848 [2,6,5]
+  CRUSH rule 2 x 849 [5,6,2]
+  CRUSH rule 2 x 850 [2,5,6]
+  CRUSH rule 2 x 851 [6,5,2]
+  CRUSH rule 2 x 852 [7,5,2]
+  CRUSH rule 2 x 853 [6,2,5]
+  CRUSH rule 2 x 854 [7,2,5]
+  CRUSH rule 2 x 855 [5,7,2]
+  CRUSH rule 2 x 856 [6,5,2]
+  CRUSH rule 2 x 857 [8,5,2]
+  CRUSH rule 2 x 858 [6,5,2]
+  CRUSH rule 2 x 859 [6,2,5]
+  CRUSH rule 2 x 860 [5,2,7]
+  CRUSH rule 2 x 861 [8,5,2]
+  CRUSH rule 2 x 862 [6,2,5]
+  CRUSH rule 2 x 863 [8,2,5]
+  CRUSH rule 2 x 864 [5,6,2]
+  CRUSH rule 2 x 865 [8,2,5]
+  CRUSH rule 2 x 866 [5,6,2]
+  CRUSH rule 2 x 867 [6,5,2]
+  CRUSH rule 2 x 868 [6,5,2]
+  CRUSH rule 2 x 869 [8,5,2]
+  CRUSH rule 2 x 870 [2,5,8]
+  CRUSH rule 2 x 871 [5,2,8]
+  CRUSH rule 2 x 872 [5,2,8]
+  CRUSH rule 2 x 873 [5,6,2]
+  CRUSH rule 2 x 874 [2,6,5]
+  CRUSH rule 2 x 875 [2,6,5]
+  CRUSH rule 2 x 876 [5,8,2]
+  CRUSH rule 2 x 877 [6,5,2]
+  CRUSH rule 2 x 878 [5,2,7]
+  CRUSH rule 2 x 879 [7,5,2]
+  CRUSH rule 2 x 880 [5,2,8]
+  CRUSH rule 2 x 881 [5,6,2]
+  CRUSH rule 2 x 882 [5,2,7]
+  CRUSH rule 2 x 883 [2,5,7]
+  CRUSH rule 2 x 884 [6,2,5]
+  CRUSH rule 2 x 885 [5,2,8]
+  CRUSH rule 2 x 886 [5,6,2]
+  CRUSH rule 2 x 887 [7,5,2]
+  CRUSH rule 2 x 888 [6,2,5]
+  CRUSH rule 2 x 889 [2,6,5]
+  CRUSH rule 2 x 890 [7,2,5]
+  CRUSH rule 2 x 891 [2,8,5]
+  CRUSH rule 2 x 892 [6,2,5]
+  CRUSH rule 2 x 893 [2,5,7]
+  CRUSH rule 2 x 894 [7,5,2]
+  CRUSH rule 2 x 895 [5,2,8]
+  CRUSH rule 2 x 896 [2,8,5]
+  CRUSH rule 2 x 897 [5,2,6]
+  CRUSH rule 2 x 898 [2,5,7]
+  CRUSH rule 2 x 899 [2,7,5]
+  CRUSH rule 2 x 900 [5,2,6]
+  CRUSH rule 2 x 901 [5,2,8]
+  CRUSH rule 2 x 902 [8,5,2]
+  CRUSH rule 2 x 903 [5,7,2]
+  CRUSH rule 2 x 904 [5,6,2]
+  CRUSH rule 2 x 905 [6,2,5]
+  CRUSH rule 2 x 906 [2,6,5]
+  CRUSH rule 2 x 907 [7,2,5]
+  CRUSH rule 2 x 908 [5,8,2]
+  CRUSH rule 2 x 909 [2,5,7]
+  CRUSH rule 2 x 910 [6,5,2]
+  CRUSH rule 2 x 911 [5,8,2]
+  CRUSH rule 2 x 912 [2,7,5]
+  CRUSH rule 2 x 913 [7,2,5]
+  CRUSH rule 2 x 914 [6,5,2]
+  CRUSH rule 2 x 915 [8,2,5]
+  CRUSH rule 2 x 916 [5,2,8]
+  CRUSH rule 2 x 917 [2,5,8]
+  CRUSH rule 2 x 918 [8,2,5]
+  CRUSH rule 2 x 919 [6,2,5]
+  CRUSH rule 2 x 920 [7,5,2]
+  CRUSH rule 2 x 921 [2,5,6]
+  CRUSH rule 2 x 922 [6,5,2]
+  CRUSH rule 2 x 923 [5,8,2]
+  CRUSH rule 2 x 924 [5,2,7]
+  CRUSH rule 2 x 925 [5,7,2]
+  CRUSH rule 2 x 926 [5,2,8]
+  CRUSH rule 2 x 927 [2,6,5]
+  CRUSH rule 2 x 928 [8,2,5]
+  CRUSH rule 2 x 929 [5,2,7]
+  CRUSH rule 2 x 930 [2,5,6]
+  CRUSH rule 2 x 931 [5,2,7]
+  CRUSH rule 2 x 932 [5,2,8]
+  CRUSH rule 2 x 933 [8,5,2]
+  CRUSH rule 2 x 934 [5,6,2]
+  CRUSH rule 2 x 935 [6,5,2]
+  CRUSH rule 2 x 936 [2,6,5]
+  CRUSH rule 2 x 937 [5,8,2]
+  CRUSH rule 2 x 938 [6,5,2]
+  CRUSH rule 2 x 939 [2,7,5]
+  CRUSH rule 2 x 940 [8,5,2]
+  CRUSH rule 2 x 941 [5,2,8]
+  CRUSH rule 2 x 942 [2,8,5]
+  CRUSH rule 2 x 943 [8,2,5]
+  CRUSH rule 2 x 944 [5,8,2]
+  CRUSH rule 2 x 945 [7,2,5]
+  CRUSH rule 2 x 946 [2,8,5]
+  CRUSH rule 2 x 947 [5,2,8]
+  CRUSH rule 2 x 948 [7,5,2]
+  CRUSH rule 2 x 949 [6,2,5]
+  CRUSH rule 2 x 950 [5,6,2]
+  CRUSH rule 2 x 951 [5,8,2]
+  CRUSH rule 2 x 952 [2,7,5]
+  CRUSH rule 2 x 953 [2,5,6]
+  CRUSH rule 2 x 954 [5,2,7]
+  CRUSH rule 2 x 955 [8,2,5]
+  CRUSH rule 2 x 956 [2,6,5]
+  CRUSH rule 2 x 957 [7,2,5]
+  CRUSH rule 2 x 958 [8,5,2]
+  CRUSH rule 2 x 959 [5,2,7]
+  CRUSH rule 2 x 960 [5,6,2]
+  CRUSH rule 2 x 961 [5,2,8]
+  CRUSH rule 2 x 962 [7,5,2]
+  CRUSH rule 2 x 963 [2,5,6]
+  CRUSH rule 2 x 964 [5,2,8]
+  CRUSH rule 2 x 965 [7,5,2]
+  CRUSH rule 2 x 966 [5,8,2]
+  CRUSH rule 2 x 967 [8,5,2]
+  CRUSH rule 2 x 968 [7,2,5]
+  CRUSH rule 2 x 969 [8,2,5]
+  CRUSH rule 2 x 970 [2,6,5]
+  CRUSH rule 2 x 971 [2,7,5]
+  CRUSH rule 2 x 972 [2,8,5]
+  CRUSH rule 2 x 973 [2,6,5]
+  CRUSH rule 2 x 974 [5,2,8]
+  CRUSH rule 2 x 975 [5,7,2]
+  CRUSH rule 2 x 976 [5,8,2]
+  CRUSH rule 2 x 977 [8,5,2]
+  CRUSH rule 2 x 978 [7,2,5]
+  CRUSH rule 2 x 979 [7,2,5]
+  CRUSH rule 2 x 980 [6,2,5]
+  CRUSH rule 2 x 981 [7,5,2]
+  CRUSH rule 2 x 982 [5,2,8]
+  CRUSH rule 2 x 983 [5,7,2]
+  CRUSH rule 2 x 984 [2,7,5]
+  CRUSH rule 2 x 985 [2,5,7]
+  CRUSH rule 2 x 986 [8,5,2]
+  CRUSH rule 2 x 987 [2,5,8]
+  CRUSH rule 2 x 988 [2,5,7]
+  CRUSH rule 2 x 989 [2,6,5]
+  CRUSH rule 2 x 990 [2,6,5]
+  CRUSH rule 2 x 991 [2,5,8]
+  CRUSH rule 2 x 992 [7,2,5]
+  CRUSH rule 2 x 993 [2,6,5]
+  CRUSH rule 2 x 994 [5,7,2]
+  CRUSH rule 2 x 995 [7,2,5]
+  CRUSH rule 2 x 996 [6,5,2]
+  CRUSH rule 2 x 997 [6,5,2]
+  CRUSH rule 2 x 998 [8,2,5]
+  CRUSH rule 2 x 999 [2,7,5]
+  CRUSH rule 2 x 1000 [8,5,2]
+  CRUSH rule 2 x 1001 [2,5,6]
+  CRUSH rule 2 x 1002 [2,5,7]
+  CRUSH rule 2 x 1003 [2,8,5]
+  CRUSH rule 2 x 1004 [6,2,5]
+  CRUSH rule 2 x 1005 [6,2,5]
+  CRUSH rule 2 x 1006 [2,6,5]
+  CRUSH rule 2 x 1007 [2,5,7]
+  CRUSH rule 2 x 1008 [2,7,5]
+  CRUSH rule 2 x 1009 [6,5,2]
+  CRUSH rule 2 x 1010 [5,2,7]
+  CRUSH rule 2 x 1011 [5,2,8]
+  CRUSH rule 2 x 1012 [5,2,7]
+  CRUSH rule 2 x 1013 [5,2,7]
+  CRUSH rule 2 x 1014 [2,8,5]
+  CRUSH rule 2 x 1015 [6,5,2]
+  CRUSH rule 2 x 1016 [2,5,7]
+  CRUSH rule 2 x 1017 [6,2,5]
+  CRUSH rule 2 x 1018 [5,2,6]
+  CRUSH rule 2 x 1019 [5,8,2]
+  CRUSH rule 2 x 1020 [5,2,7]
+  CRUSH rule 2 x 1021 [5,2,6]
+  CRUSH rule 2 x 1022 [2,6,5]
+  CRUSH rule 2 x 1023 [5,2,8]
+  rule 2 (chooseleaf) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 3 (choose-set), x = 0..1023, numrep = 2..3
+  CRUSH rule 3 x 0 [2,5]
+  CRUSH rule 3 x 1 [2,8]
+  CRUSH rule 3 x 2 [2,5]
+  CRUSH rule 3 x 3 [8,2]
+  CRUSH rule 3 x 4 [5,2]
+  CRUSH rule 3 x 5 [7,2]
+  CRUSH rule 3 x 6 [2,6]
+  CRUSH rule 3 x 7 [5,6]
+  CRUSH rule 3 x 8 [5,7]
+  CRUSH rule 3 x 9 [2,5]
+  CRUSH rule 3 x 10 [2,8]
+  CRUSH rule 3 x 11 [2,6]
+  CRUSH rule 3 x 12 [2,5]
+  CRUSH rule 3 x 13 [5,8]
+  CRUSH rule 3 x 14 [7,2]
+  CRUSH rule 3 x 15 [7,2]
+  CRUSH rule 3 x 16 [5,7]
+  CRUSH rule 3 x 17 [5,2]
+  CRUSH rule 3 x 18 [2,5]
+  CRUSH rule 3 x 19 [7,5]
+  CRUSH rule 3 x 20 [2,5]
+  CRUSH rule 3 x 21 [5,6]
+  CRUSH rule 3 x 22 [8,5]
+  CRUSH rule 3 x 23 [5,7]
+  CRUSH rule 3 x 24 [2,6]
+  CRUSH rule 3 x 25 [5,8]
+  CRUSH rule 3 x 26 [2,7]
+  CRUSH rule 3 x 27 [5,2]
+  CRUSH rule 3 x 28 [6,2]
+  CRUSH rule 3 x 29 [8,5]
+  CRUSH rule 3 x 30 [5,6]
+  CRUSH rule 3 x 31 [8,2]
+  CRUSH rule 3 x 32 [5,7]
+  CRUSH rule 3 x 33 [2,7]
+  CRUSH rule 3 x 34 [2,5]
+  CRUSH rule 3 x 35 [2,6]
+  CRUSH rule 3 x 36 [5,6]
+  CRUSH rule 3 x 37 [2,5]
+  CRUSH rule 3 x 38 [5,6]
+  CRUSH rule 3 x 39 [5,7]
+  CRUSH rule 3 x 40 [7,2]
+  CRUSH rule 3 x 41 [2,7]
+  CRUSH rule 3 x 42 [5,7]
+  CRUSH rule 3 x 43 [2,5]
+  CRUSH rule 3 x 44 [2,8]
+  CRUSH rule 3 x 45 [8,2]
+  CRUSH rule 3 x 46 [2,5]
+  CRUSH rule 3 x 47 [5,2]
+  CRUSH rule 3 x 48 [5,6]
+  CRUSH rule 3 x 49 [5,6]
+  CRUSH rule 3 x 50 [5,2]
+  CRUSH rule 3 x 51 [5,6]
+  CRUSH rule 3 x 52 [8,2]
+  CRUSH rule 3 x 53 [5,6]
+  CRUSH rule 3 x 54 [7,5]
+  CRUSH rule 3 x 55 [8,2]
+  CRUSH rule 3 x 56 [6,5]
+  CRUSH rule 3 x 57 [5,8]
+  CRUSH rule 3 x 58 [2,8]
+  CRUSH rule 3 x 59 [5,2]
+  CRUSH rule 3 x 60 [5,2]
+  CRUSH rule 3 x 61 [5,8]
+  CRUSH rule 3 x 62 [7,2]
+  CRUSH rule 3 x 63 [5,6]
+  CRUSH rule 3 x 64 [5,2]
+  CRUSH rule 3 x 65 [7,5]
+  CRUSH rule 3 x 66 [5,6]
+  CRUSH rule 3 x 67 [5,2]
+  CRUSH rule 3 x 68 [2,5]
+  CRUSH rule 3 x 69 [5,2]
+  CRUSH rule 3 x 70 [7,2]
+  CRUSH rule 3 x 71 [2,7]
+  CRUSH rule 3 x 72 [6,2]
+  CRUSH rule 3 x 73 [2,7]
+  CRUSH rule 3 x 74 [2,8]
+  CRUSH rule 3 x 75 [5,2]
+  CRUSH rule 3 x 76 [5,2]
+  CRUSH rule 3 x 77 [7,2]
+  CRUSH rule 3 x 78 [2,5]
+  CRUSH rule 3 x 79 [5,2]
+  CRUSH rule 3 x 80 [2,5]
+  CRUSH rule 3 x 81 [2,5]
+  CRUSH rule 3 x 82 [7,2]
+  CRUSH rule 3 x 83 [2,6]
+  CRUSH rule 3 x 84 [7,2]
+  CRUSH rule 3 x 85 [5,7]
+  CRUSH rule 3 x 86 [2,6]
+  CRUSH rule 3 x 87 [2,6]
+  CRUSH rule 3 x 88 [2,8]
+  CRUSH rule 3 x 89 [5,2]
+  CRUSH rule 3 x 90 [6,5]
+  CRUSH rule 3 x 91 [5,6]
+  CRUSH rule 3 x 92 [2,6]
+  CRUSH rule 3 x 93 [7,5]
+  CRUSH rule 3 x 94 [2,5]
+  CRUSH rule 3 x 95 [7,5]
+  CRUSH rule 3 x 96 [5,8]
+  CRUSH rule 3 x 97 [8,5]
+  CRUSH rule 3 x 98 [2,7]
+  CRUSH rule 3 x 99 [2,8]
+  CRUSH rule 3 x 100 [2,6]
+  CRUSH rule 3 x 101 [5,8]
+  CRUSH rule 3 x 102 [5,2]
+  CRUSH rule 3 x 103 [5,7]
+  CRUSH rule 3 x 104 [7,5]
+  CRUSH rule 3 x 105 [2,5]
+  CRUSH rule 3 x 106 [2,6]
+  CRUSH rule 3 x 107 [5,2]
+  CRUSH rule 3 x 108 [7,2]
+  CRUSH rule 3 x 109 [2,5]
+  CRUSH rule 3 x 110 [5,2]
+  CRUSH rule 3 x 111 [2,5]
+  CRUSH rule 3 x 112 [2,6]
+  CRUSH rule 3 x 113 [6,2]
+  CRUSH rule 3 x 114 [7,5]
+  CRUSH rule 3 x 115 [8,2]
+  CRUSH rule 3 x 116 [2,7]
+  CRUSH rule 3 x 117 [7,5]
+  CRUSH rule 3 x 118 [2,5]
+  CRUSH rule 3 x 119 [5,7]
+  CRUSH rule 3 x 120 [2,5]
+  CRUSH rule 3 x 121 [2,8]
+  CRUSH rule 3 x 122 [8,5]
+  CRUSH rule 3 x 123 [2,5]
+  CRUSH rule 3 x 124 [5,2]
+  CRUSH rule 3 x 125 [2,7]
+  CRUSH rule 3 x 126 [5,2]
+  CRUSH rule 3 x 127 [5,6]
+  CRUSH rule 3 x 128 [5,8]
+  CRUSH rule 3 x 129 [2,5]
+  CRUSH rule 3 x 130 [5,8]
+  CRUSH rule 3 x 131 [2,5]
+  CRUSH rule 3 x 132 [2,5]
+  CRUSH rule 3 x 133 [5,6]
+  CRUSH rule 3 x 134 [2,7]
+  CRUSH rule 3 x 135 [5,7]
+  CRUSH rule 3 x 136 [2,5]
+  CRUSH rule 3 x 137 [7,5]
+  CRUSH rule 3 x 138 [8,5]
+  CRUSH rule 3 x 139 [5,2]
+  CRUSH rule 3 x 140 [2,8]
+  CRUSH rule 3 x 141 [6,2]
+  CRUSH rule 3 x 142 [5,2]
+  CRUSH rule 3 x 143 [5,7]
+  CRUSH rule 3 x 144 [8,2]
+  CRUSH rule 3 x 145 [8,5]
+  CRUSH rule 3 x 146 [2,8]
+  CRUSH rule 3 x 147 [2,6]
+  CRUSH rule 3 x 148 [5,2]
+  CRUSH rule 3 x 149 [5,6]
+  CRUSH rule 3 x 150 [2,8]
+  CRUSH rule 3 x 151 [5,8]
+  CRUSH rule 3 x 152 [8,5]
+  CRUSH rule 3 x 153 [8,5]
+  CRUSH rule 3 x 154 [5,2]
+  CRUSH rule 3 x 155 [5,6]
+  CRUSH rule 3 x 156 [5,2]
+  CRUSH rule 3 x 157 [5,2]
+  CRUSH rule 3 x 158 [2,6]
+  CRUSH rule 3 x 159 [7,2]
+  CRUSH rule 3 x 160 [2,8]
+  CRUSH rule 3 x 161 [2,5]
+  CRUSH rule 3 x 162 [2,8]
+  CRUSH rule 3 x 163 [5,8]
+  CRUSH rule 3 x 164 [7,2]
+  CRUSH rule 3 x 165 [7,2]
+  CRUSH rule 3 x 166 [2,5]
+  CRUSH rule 3 x 167 [2,6]
+  CRUSH rule 3 x 168 [5,2]
+  CRUSH rule 3 x 169 [2,7]
+  CRUSH rule 3 x 170 [2,5]
+  CRUSH rule 3 x 171 [7,5]
+  CRUSH rule 3 x 172 [2,8]
+  CRUSH rule 3 x 173 [8,5]
+  CRUSH rule 3 x 174 [2,5]
+  CRUSH rule 3 x 175 [6,2]
+  CRUSH rule 3 x 176 [5,2]
+  CRUSH rule 3 x 177 [5,2]
+  CRUSH rule 3 x 178 [5,2]
+  CRUSH rule 3 x 179 [5,2]
+  CRUSH rule 3 x 180 [5,7]
+  CRUSH rule 3 x 181 [6,2]
+  CRUSH rule 3 x 182 [8,5]
+  CRUSH rule 3 x 183 [7,5]
+  CRUSH rule 3 x 184 [5,6]
+  CRUSH rule 3 x 185 [6,2]
+  CRUSH rule 3 x 186 [2,5]
+  CRUSH rule 3 x 187 [2,6]
+  CRUSH rule 3 x 188 [2,6]
+  CRUSH rule 3 x 189 [2,6]
+  CRUSH rule 3 x 190 [5,2]
+  CRUSH rule 3 x 191 [7,2]
+  CRUSH rule 3 x 192 [5,2]
+  CRUSH rule 3 x 193 [5,2]
+  CRUSH rule 3 x 194 [2,5]
+  CRUSH rule 3 x 195 [6,5]
+  CRUSH rule 3 x 196 [6,2]
+  CRUSH rule 3 x 197 [6,5]
+  CRUSH rule 3 x 198 [2,5]
+  CRUSH rule 3 x 199 [2,5]
+  CRUSH rule 3 x 200 [2,5]
+  CRUSH rule 3 x 201 [7,2]
+  CRUSH rule 3 x 202 [6,5]
+  CRUSH rule 3 x 203 [5,6]
+  CRUSH rule 3 x 204 [2,5]
+  CRUSH rule 3 x 205 [2,7]
+  CRUSH rule 3 x 206 [2,8]
+  CRUSH rule 3 x 207 [5,2]
+  CRUSH rule 3 x 208 [7,2]
+  CRUSH rule 3 x 209 [2,8]
+  CRUSH rule 3 x 210 [2,5]
+  CRUSH rule 3 x 211 [5,2]
+  CRUSH rule 3 x 212 [7,5]
+  CRUSH rule 3 x 213 [8,5]
+  CRUSH rule 3 x 214 [5,7]
+  CRUSH rule 3 x 215 [8,2]
+  CRUSH rule 3 x 216 [5,2]
+  CRUSH rule 3 x 217 [2,7]
+  CRUSH rule 3 x 218 [2,7]
+  CRUSH rule 3 x 219 [5,6]
+  CRUSH rule 3 x 220 [5,8]
+  CRUSH rule 3 x 221 [5,7]
+  CRUSH rule 3 x 222 [6,5]
+  CRUSH rule 3 x 223 [2,5]
+  CRUSH rule 3 x 224 [2,5]
+  CRUSH rule 3 x 225 [8,2]
+  CRUSH rule 3 x 226 [7,2]
+  CRUSH rule 3 x 227 [5,2]
+  CRUSH rule 3 x 228 [5,6]
+  CRUSH rule 3 x 229 [5,7]
+  CRUSH rule 3 x 230 [5,6]
+  CRUSH rule 3 x 231 [5,7]
+  CRUSH rule 3 x 232 [2,8]
+  CRUSH rule 3 x 233 [5,6]
+  CRUSH rule 3 x 234 [2,5]
+  CRUSH rule 3 x 235 [5,6]
+  CRUSH rule 3 x 236 [5,2]
+  CRUSH rule 3 x 237 [5,8]
+  CRUSH rule 3 x 238 [5,2]
+  CRUSH rule 3 x 239 [8,5]
+  CRUSH rule 3 x 240 [5,8]
+  CRUSH rule 3 x 241 [5,2]
+  CRUSH rule 3 x 242 [5,2]
+  CRUSH rule 3 x 243 [5,8]
+  CRUSH rule 3 x 244 [5,6]
+  CRUSH rule 3 x 245 [7,2]
+  CRUSH rule 3 x 246 [2,5]
+  CRUSH rule 3 x 247 [6,2]
+  CRUSH rule 3 x 248 [8,2]
+  CRUSH rule 3 x 249 [2,5]
+  CRUSH rule 3 x 250 [2,5]
+  CRUSH rule 3 x 251 [2,5]
+  CRUSH rule 3 x 252 [5,7]
+  CRUSH rule 3 x 253 [5,2]
+  CRUSH rule 3 x 254 [5,2]
+  CRUSH rule 3 x 255 [2,7]
+  CRUSH rule 3 x 256 [5,6]
+  CRUSH rule 3 x 257 [2,6]
+  CRUSH rule 3 x 258 [5,2]
+  CRUSH rule 3 x 259 [5,6]
+  CRUSH rule 3 x 260 [5,8]
+  CRUSH rule 3 x 261 [8,5]
+  CRUSH rule 3 x 262 [5,6]
+  CRUSH rule 3 x 263 [6,2]
+  CRUSH rule 3 x 264 [5,6]
+  CRUSH rule 3 x 265 [8,5]
+  CRUSH rule 3 x 266 [8,2]
+  CRUSH rule 3 x 267 [2,5]
+  CRUSH rule 3 x 268 [2,6]
+  CRUSH rule 3 x 269 [2,6]
+  CRUSH rule 3 x 270 [5,2]
+  CRUSH rule 3 x 271 [7,5]
+  CRUSH rule 3 x 272 [2,6]
+  CRUSH rule 3 x 273 [5,2]
+  CRUSH rule 3 x 274 [6,5]
+  CRUSH rule 3 x 275 [5,8]
+  CRUSH rule 3 x 276 [7,2]
+  CRUSH rule 3 x 277 [6,5]
+  CRUSH rule 3 x 278 [6,2]
+  CRUSH rule 3 x 279 [8,5]
+  CRUSH rule 3 x 280 [2,7]
+  CRUSH rule 3 x 281 [8,2]
+  CRUSH rule 3 x 282 [5,2]
+  CRUSH rule 3 x 283 [8,2]
+  CRUSH rule 3 x 284 [6,5]
+  CRUSH rule 3 x 285 [5,7]
+  CRUSH rule 3 x 286 [2,8]
+  CRUSH rule 3 x 287 [2,5]
+  CRUSH rule 3 x 288 [8,2]
+  CRUSH rule 3 x 289 [5,6]
+  CRUSH rule 3 x 290 [2,5]
+  CRUSH rule 3 x 291 [2,5]
+  CRUSH rule 3 x 292 [8,2]
+  CRUSH rule 3 x 293 [6,2]
+  CRUSH rule 3 x 294 [7,5]
+  CRUSH rule 3 x 295 [5,6]
+  CRUSH rule 3 x 296 [5,2]
+  CRUSH rule 3 x 297 [6,2]
+  CRUSH rule 3 x 298 [2,5]
+  CRUSH rule 3 x 299 [2,8]
+  CRUSH rule 3 x 300 [8,5]
+  CRUSH rule 3 x 301 [2,7]
+  CRUSH rule 3 x 302 [5,2]
+  CRUSH rule 3 x 303 [7,5]
+  CRUSH rule 3 x 304 [2,6]
+  CRUSH rule 3 x 305 [5,6]
+  CRUSH rule 3 x 306 [2,8]
+  CRUSH rule 3 x 307 [2,7]
+  CRUSH rule 3 x 308 [2,8]
+  CRUSH rule 3 x 309 [7,5]
+  CRUSH rule 3 x 310 [5,2]
+  CRUSH rule 3 x 311 [5,8]
+  CRUSH rule 3 x 312 [2,6]
+  CRUSH rule 3 x 313 [5,2]
+  CRUSH rule 3 x 314 [5,2]
+  CRUSH rule 3 x 315 [2,5]
+  CRUSH rule 3 x 316 [6,5]
+  CRUSH rule 3 x 317 [2,7]
+  CRUSH rule 3 x 318 [8,2]
+  CRUSH rule 3 x 319 [5,2]
+  CRUSH rule 3 x 320 [5,8]
+  CRUSH rule 3 x 321 [2,5]
+  CRUSH rule 3 x 322 [2,6]
+  CRUSH rule 3 x 323 [5,7]
+  CRUSH rule 3 x 324 [7,2]
+  CRUSH rule 3 x 325 [5,6]
+  CRUSH rule 3 x 326 [5,2]
+  CRUSH rule 3 x 327 [2,8]
+  CRUSH rule 3 x 328 [7,5]
+  CRUSH rule 3 x 329 [5,7]
+  CRUSH rule 3 x 330 [5,7]
+  CRUSH rule 3 x 331 [2,7]
+  CRUSH rule 3 x 332 [2,5]
+  CRUSH rule 3 x 333 [6,5]
+  CRUSH rule 3 x 334 [8,5]
+  CRUSH rule 3 x 335 [7,2]
+  CRUSH rule 3 x 336 [5,6]
+  CRUSH rule 3 x 337 [7,2]
+  CRUSH rule 3 x 338 [5,8]
+  CRUSH rule 3 x 339 [7,5]
+  CRUSH rule 3 x 340 [2,6]
+  CRUSH rule 3 x 341 [5,2]
+  CRUSH rule 3 x 342 [2,8]
+  CRUSH rule 3 x 343 [6,5]
+  CRUSH rule 3 x 344 [6,2]
+  CRUSH rule 3 x 345 [5,7]
+  CRUSH rule 3 x 346 [8,2]
+  CRUSH rule 3 x 347 [5,2]
+  CRUSH rule 3 x 348 [8,2]
+  CRUSH rule 3 x 349 [2,7]
+  CRUSH rule 3 x 350 [8,5]
+  CRUSH rule 3 x 351 [5,8]
+  CRUSH rule 3 x 352 [2,8]
+  CRUSH rule 3 x 353 [6,5]
+  CRUSH rule 3 x 354 [2,5]
+  CRUSH rule 3 x 355 [5,8]
+  CRUSH rule 3 x 356 [5,2]
+  CRUSH rule 3 x 357 [6,2]
+  CRUSH rule 3 x 358 [2,8]
+  CRUSH rule 3 x 359 [6,2]
+  CRUSH rule 3 x 360 [5,2]
+  CRUSH rule 3 x 361 [8,5]
+  CRUSH rule 3 x 362 [5,2]
+  CRUSH rule 3 x 363 [5,2]
+  CRUSH rule 3 x 364 [2,5]
+  CRUSH rule 3 x 365 [6,5]
+  CRUSH rule 3 x 366 [7,2]
+  CRUSH rule 3 x 367 [5,2]
+  CRUSH rule 3 x 368 [7,5]
+  CRUSH rule 3 x 369 [5,7]
+  CRUSH rule 3 x 370 [8,2]
+  CRUSH rule 3 x 371 [2,5]
+  CRUSH rule 3 x 372 [5,2]
+  CRUSH rule 3 x 373 [2,6]
+  CRUSH rule 3 x 374 [5,8]
+  CRUSH rule 3 x 375 [6,5]
+  CRUSH rule 3 x 376 [7,2]
+  CRUSH rule 3 x 377 [2,5]
+  CRUSH rule 3 x 378 [2,6]
+  CRUSH rule 3 x 379 [8,5]
+  CRUSH rule 3 x 380 [2,5]
+  CRUSH rule 3 x 381 [2,5]
+  CRUSH rule 3 x 382 [2,5]
+  CRUSH rule 3 x 383 [5,7]
+  CRUSH rule 3 x 384 [7,2]
+  CRUSH rule 3 x 385 [7,5]
+  CRUSH rule 3 x 386 [2,5]
+  CRUSH rule 3 x 387 [2,5]
+  CRUSH rule 3 x 388 [5,2]
+  CRUSH rule 3 x 389 [2,5]
+  CRUSH rule 3 x 390 [5,8]
+  CRUSH rule 3 x 391 [5,6]
+  CRUSH rule 3 x 392 [2,7]
+  CRUSH rule 3 x 393 [5,2]
+  CRUSH rule 3 x 394 [5,8]
+  CRUSH rule 3 x 395 [5,2]
+  CRUSH rule 3 x 396 [5,2]
+  CRUSH rule 3 x 397 [2,5]
+  CRUSH rule 3 x 398 [2,5]
+  CRUSH rule 3 x 399 [8,5]
+  CRUSH rule 3 x 400 [8,2]
+  CRUSH rule 3 x 401 [2,5]
+  CRUSH rule 3 x 402 [7,5]
+  CRUSH rule 3 x 403 [2,5]
+  CRUSH rule 3 x 404 [5,2]
+  CRUSH rule 3 x 405 [6,5]
+  CRUSH rule 3 x 406 [2,6]
+  CRUSH rule 3 x 407 [2,7]
+  CRUSH rule 3 x 408 [5,2]
+  CRUSH rule 3 x 409 [7,5]
+  CRUSH rule 3 x 410 [8,5]
+  CRUSH rule 3 x 411 [2,7]
+  CRUSH rule 3 x 412 [2,5]
+  CRUSH rule 3 x 413 [5,2]
+  CRUSH rule 3 x 414 [5,2]
+  CRUSH rule 3 x 415 [2,6]
+  CRUSH rule 3 x 416 [2,8]
+  CRUSH rule 3 x 417 [8,2]
+  CRUSH rule 3 x 418 [7,2]
+  CRUSH rule 3 x 419 [8,5]
+  CRUSH rule 3 x 420 [2,5]
+  CRUSH rule 3 x 421 [8,5]
+  CRUSH rule 3 x 422 [6,5]
+  CRUSH rule 3 x 423 [2,5]
+  CRUSH rule 3 x 424 [8,5]
+  CRUSH rule 3 x 425 [2,5]
+  CRUSH rule 3 x 426 [6,2]
+  CRUSH rule 3 x 427 [2,8]
+  CRUSH rule 3 x 428 [5,8]
+  CRUSH rule 3 x 429 [5,8]
+  CRUSH rule 3 x 430 [5,7]
+  CRUSH rule 3 x 431 [5,2]
+  CRUSH rule 3 x 432 [7,2]
+  CRUSH rule 3 x 433 [6,5]
+  CRUSH rule 3 x 434 [5,2]
+  CRUSH rule 3 x 435 [2,5]
+  CRUSH rule 3 x 436 [5,2]
+  CRUSH rule 3 x 437 [7,5]
+  CRUSH rule 3 x 438 [2,5]
+  CRUSH rule 3 x 439 [2,5]
+  CRUSH rule 3 x 440 [2,6]
+  CRUSH rule 3 x 441 [5,8]
+  CRUSH rule 3 x 442 [2,5]
+  CRUSH rule 3 x 443 [6,2]
+  CRUSH rule 3 x 444 [7,2]
+  CRUSH rule 3 x 445 [6,5]
+  CRUSH rule 3 x 446 [5,2]
+  CRUSH rule 3 x 447 [2,5]
+  CRUSH rule 3 x 448 [7,2]
+  CRUSH rule 3 x 449 [7,5]
+  CRUSH rule 3 x 450 [5,2]
+  CRUSH rule 3 x 451 [6,5]
+  CRUSH rule 3 x 452 [8,5]
+  CRUSH rule 3 x 453 [6,5]
+  CRUSH rule 3 x 454 [6,5]
+  CRUSH rule 3 x 455 [2,8]
+  CRUSH rule 3 x 456 [6,2]
+  CRUSH rule 3 x 457 [7,2]
+  CRUSH rule 3 x 458 [2,8]
+  CRUSH rule 3 x 459 [2,6]
+  CRUSH rule 3 x 460 [6,5]
+  CRUSH rule 3 x 461 [6,5]
+  CRUSH rule 3 x 462 [8,2]
+  CRUSH rule 3 x 463 [6,2]
+  CRUSH rule 3 x 464 [7,5]
+  CRUSH rule 3 x 465 [7,2]
+  CRUSH rule 3 x 466 [5,6]
+  CRUSH rule 3 x 467 [6,5]
+  CRUSH rule 3 x 468 [7,2]
+  CRUSH rule 3 x 469 [7,2]
+  CRUSH rule 3 x 470 [5,2]
+  CRUSH rule 3 x 471 [2,6]
+  CRUSH rule 3 x 472 [5,2]
+  CRUSH rule 3 x 473 [2,5]
+  CRUSH rule 3 x 474 [6,2]
+  CRUSH rule 3 x 475 [6,2]
+  CRUSH rule 3 x 476 [5,7]
+  CRUSH rule 3 x 477 [5,6]
+  CRUSH rule 3 x 478 [6,2]
+  CRUSH rule 3 x 479 [2,5]
+  CRUSH rule 3 x 480 [2,6]
+  CRUSH rule 3 x 481 [2,5]
+  CRUSH rule 3 x 482 [5,8]
+  CRUSH rule 3 x 483 [2,6]
+  CRUSH rule 3 x 484 [2,8]
+  CRUSH rule 3 x 485 [5,8]
+  CRUSH rule 3 x 486 [5,2]
+  CRUSH rule 3 x 487 [5,2]
+  CRUSH rule 3 x 488 [5,7]
+  CRUSH rule 3 x 489 [2,8]
+  CRUSH rule 3 x 490 [6,5]
+  CRUSH rule 3 x 491 [2,6]
+  CRUSH rule 3 x 492 [6,5]
+  CRUSH rule 3 x 493 [2,8]
+  CRUSH rule 3 x 494 [2,6]
+  CRUSH rule 3 x 495 [5,2]
+  CRUSH rule 3 x 496 [7,5]
+  CRUSH rule 3 x 497 [5,7]
+  CRUSH rule 3 x 498 [2,5]
+  CRUSH rule 3 x 499 [8,5]
+  CRUSH rule 3 x 500 [5,6]
+  CRUSH rule 3 x 501 [2,7]
+  CRUSH rule 3 x 502 [7,2]
+  CRUSH rule 3 x 503 [2,5]
+  CRUSH rule 3 x 504 [5,8]
+  CRUSH rule 3 x 505 [2,7]
+  CRUSH rule 3 x 506 [5,2]
+  CRUSH rule 3 x 507 [6,2]
+  CRUSH rule 3 x 508 [2,5]
+  CRUSH rule 3 x 509 [7,5]
+  CRUSH rule 3 x 510 [6,2]
+  CRUSH rule 3 x 511 [5,6]
+  CRUSH rule 3 x 512 [7,2]
+  CRUSH rule 3 x 513 [7,2]
+  CRUSH rule 3 x 514 [5,7]
+  CRUSH rule 3 x 515 [8,5]
+  CRUSH rule 3 x 516 [5,2]
+  CRUSH rule 3 x 517 [7,2]
+  CRUSH rule 3 x 518 [5,6]
+  CRUSH rule 3 x 519 [7,5]
+  CRUSH rule 3 x 520 [2,8]
+  CRUSH rule 3 x 521 [8,2]
+  CRUSH rule 3 x 522 [6,2]
+  CRUSH rule 3 x 523 [5,2]
+  CRUSH rule 3 x 524 [2,5]
+  CRUSH rule 3 x 525 [2,5]
+  CRUSH rule 3 x 526 [2,5]
+  CRUSH rule 3 x 527 [2,5]
+  CRUSH rule 3 x 528 [5,2]
+  CRUSH rule 3 x 529 [5,6]
+  CRUSH rule 3 x 530 [6,5]
+  CRUSH rule 3 x 531 [6,2]
+  CRUSH rule 3 x 532 [6,5]
+  CRUSH rule 3 x 533 [5,8]
+  CRUSH rule 3 x 534 [7,5]
+  CRUSH rule 3 x 535 [8,2]
+  CRUSH rule 3 x 536 [6,2]
+  CRUSH rule 3 x 537 [5,8]
+  CRUSH rule 3 x 538 [6,5]
+  CRUSH rule 3 x 539 [8,5]
+  CRUSH rule 3 x 540 [2,7]
+  CRUSH rule 3 x 541 [2,5]
+  CRUSH rule 3 x 542 [5,2]
+  CRUSH rule 3 x 543 [6,2]
+  CRUSH rule 3 x 544 [5,7]
+  CRUSH rule 3 x 545 [5,7]
+  CRUSH rule 3 x 546 [6,2]
+  CRUSH rule 3 x 547 [8,2]
+  CRUSH rule 3 x 548 [5,2]
+  CRUSH rule 3 x 549 [5,7]
+  CRUSH rule 3 x 550 [2,5]
+  CRUSH rule 3 x 551 [7,5]
+  CRUSH rule 3 x 552 [5,7]
+  CRUSH rule 3 x 553 [5,2]
+  CRUSH rule 3 x 554 [2,6]
+  CRUSH rule 3 x 555 [5,2]
+  CRUSH rule 3 x 556 [5,6]
+  CRUSH rule 3 x 557 [7,5]
+  CRUSH rule 3 x 558 [5,2]
+  CRUSH rule 3 x 559 [5,2]
+  CRUSH rule 3 x 560 [8,5]
+  CRUSH rule 3 x 561 [6,5]
+  CRUSH rule 3 x 562 [5,2]
+  CRUSH rule 3 x 563 [2,7]
+  CRUSH rule 3 x 564 [5,2]
+  CRUSH rule 3 x 565 [5,8]
+  CRUSH rule 3 x 566 [5,6]
+  CRUSH rule 3 x 567 [5,7]
+  CRUSH rule 3 x 568 [7,5]
+  CRUSH rule 3 x 569 [5,2]
+  CRUSH rule 3 x 570 [2,5]
+  CRUSH rule 3 x 571 [5,6]
+  CRUSH rule 3 x 572 [5,2]
+  CRUSH rule 3 x 573 [5,2]
+  CRUSH rule 3 x 574 [2,5]
+  CRUSH rule 3 x 575 [8,2]
+  CRUSH rule 3 x 576 [5,7]
+  CRUSH rule 3 x 577 [8,2]
+  CRUSH rule 3 x 578 [6,2]
+  CRUSH rule 3 x 579 [5,2]
+  CRUSH rule 3 x 580 [5,2]
+  CRUSH rule 3 x 581 [7,2]
+  CRUSH rule 3 x 582 [2,8]
+  CRUSH rule 3 x 583 [6,2]
+  CRUSH rule 3 x 584 [8,2]
+  CRUSH rule 3 x 585 [7,2]
+  CRUSH rule 3 x 586 [2,8]
+  CRUSH rule 3 x 587 [2,5]
+  CRUSH rule 3 x 588 [5,8]
+  CRUSH rule 3 x 589 [7,2]
+  CRUSH rule 3 x 590 [6,2]
+  CRUSH rule 3 x 591 [5,2]
+  CRUSH rule 3 x 592 [2,5]
+  CRUSH rule 3 x 593 [2,8]
+  CRUSH rule 3 x 594 [2,6]
+  CRUSH rule 3 x 595 [7,2]
+  CRUSH rule 3 x 596 [5,2]
+  CRUSH rule 3 x 597 [5,2]
+  CRUSH rule 3 x 598 [5,2]
+  CRUSH rule 3 x 599 [5,2]
+  CRUSH rule 3 x 600 [7,2]
+  CRUSH rule 3 x 601 [2,6]
+  CRUSH rule 3 x 602 [5,8]
+  CRUSH rule 3 x 603 [5,2]
+  CRUSH rule 3 x 604 [7,5]
+  CRUSH rule 3 x 605 [5,2]
+  CRUSH rule 3 x 606 [2,7]
+  CRUSH rule 3 x 607 [2,5]
+  CRUSH rule 3 x 608 [5,2]
+  CRUSH rule 3 x 609 [5,2]
+  CRUSH rule 3 x 610 [5,8]
+  CRUSH rule 3 x 611 [2,8]
+  CRUSH rule 3 x 612 [2,8]
+  CRUSH rule 3 x 613 [7,2]
+  CRUSH rule 3 x 614 [7,2]
+  CRUSH rule 3 x 615 [6,2]
+  CRUSH rule 3 x 616 [2,7]
+  CRUSH rule 3 x 617 [6,2]
+  CRUSH rule 3 x 618 [7,5]
+  CRUSH rule 3 x 619 [5,2]
+  CRUSH rule 3 x 620 [5,2]
+  CRUSH rule 3 x 621 [5,6]
+  CRUSH rule 3 x 622 [2,5]
+  CRUSH rule 3 x 623 [2,8]
+  CRUSH rule 3 x 624 [5,2]
+  CRUSH rule 3 x 625 [2,5]
+  CRUSH rule 3 x 626 [7,2]
+  CRUSH rule 3 x 627 [2,6]
+  CRUSH rule 3 x 628 [8,2]
+  CRUSH rule 3 x 629 [2,6]
+  CRUSH rule 3 x 630 [2,6]
+  CRUSH rule 3 x 631 [2,6]
+  CRUSH rule 3 x 632 [7,2]
+  CRUSH rule 3 x 633 [8,5]
+  CRUSH rule 3 x 634 [2,5]
+  CRUSH rule 3 x 635 [5,6]
+  CRUSH rule 3 x 636 [2,5]
+  CRUSH rule 3 x 637 [5,2]
+  CRUSH rule 3 x 638 [6,2]
+  CRUSH rule 3 x 639 [5,2]
+  CRUSH rule 3 x 640 [5,2]
+  CRUSH rule 3 x 641 [7,2]
+  CRUSH rule 3 x 642 [2,8]
+  CRUSH rule 3 x 643 [5,2]
+  CRUSH rule 3 x 644 [8,2]
+  CRUSH rule 3 x 645 [5,7]
+  CRUSH rule 3 x 646 [8,2]
+  CRUSH rule 3 x 647 [7,2]
+  CRUSH rule 3 x 648 [2,8]
+  CRUSH rule 3 x 649 [5,7]
+  CRUSH rule 3 x 650 [7,5]
+  CRUSH rule 3 x 651 [5,6]
+  CRUSH rule 3 x 652 [5,6]
+  CRUSH rule 3 x 653 [8,5]
+  CRUSH rule 3 x 654 [7,5]
+  CRUSH rule 3 x 655 [2,5]
+  CRUSH rule 3 x 656 [5,7]
+  CRUSH rule 3 x 657 [6,2]
+  CRUSH rule 3 x 658 [5,8]
+  CRUSH rule 3 x 659 [5,7]
+  CRUSH rule 3 x 660 [7,5]
+  CRUSH rule 3 x 661 [2,7]
+  CRUSH rule 3 x 662 [5,2]
+  CRUSH rule 3 x 663 [2,5]
+  CRUSH rule 3 x 664 [2,5]
+  CRUSH rule 3 x 665 [5,6]
+  CRUSH rule 3 x 666 [2,7]
+  CRUSH rule 3 x 667 [2,5]
+  CRUSH rule 3 x 668 [5,7]
+  CRUSH rule 3 x 669 [6,5]
+  CRUSH rule 3 x 670 [5,2]
+  CRUSH rule 3 x 671 [2,8]
+  CRUSH rule 3 x 672 [5,2]
+  CRUSH rule 3 x 673 [5,2]
+  CRUSH rule 3 x 674 [5,2]
+  CRUSH rule 3 x 675 [2,8]
+  CRUSH rule 3 x 676 [2,5]
+  CRUSH rule 3 x 677 [5,2]
+  CRUSH rule 3 x 678 [2,5]
+  CRUSH rule 3 x 679 [6,2]
+  CRUSH rule 3 x 680 [2,5]
+  CRUSH rule 3 x 681 [5,6]
+  CRUSH rule 3 x 682 [2,5]
+  CRUSH rule 3 x 683 [2,5]
+  CRUSH rule 3 x 684 [7,2]
+  CRUSH rule 3 x 685 [7,2]
+  CRUSH rule 3 x 686 [2,5]
+  CRUSH rule 3 x 687 [5,7]
+  CRUSH rule 3 x 688 [5,6]
+  CRUSH rule 3 x 689 [6,5]
+  CRUSH rule 3 x 690 [8,2]
+  CRUSH rule 3 x 691 [5,2]
+  CRUSH rule 3 x 692 [7,2]
+  CRUSH rule 3 x 693 [6,5]
+  CRUSH rule 3 x 694 [6,5]
+  CRUSH rule 3 x 695 [2,6]
+  CRUSH rule 3 x 696 [2,5]
+  CRUSH rule 3 x 697 [6,2]
+  CRUSH rule 3 x 698 [6,2]
+  CRUSH rule 3 x 699 [2,8]
+  CRUSH rule 3 x 700 [2,5]
+  CRUSH rule 3 x 701 [5,2]
+  CRUSH rule 3 x 702 [5,2]
+  CRUSH rule 3 x 703 [8,5]
+  CRUSH rule 3 x 704 [2,5]
+  CRUSH rule 3 x 705 [8,2]
+  CRUSH rule 3 x 706 [2,5]
+  CRUSH rule 3 x 707 [7,5]
+  CRUSH rule 3 x 708 [5,7]
+  CRUSH rule 3 x 709 [6,5]
+  CRUSH rule 3 x 710 [8,5]
+  CRUSH rule 3 x 711 [2,5]
+  CRUSH rule 3 x 712 [2,5]
+  CRUSH rule 3 x 713 [6,5]
+  CRUSH rule 3 x 714 [5,2]
+  CRUSH rule 3 x 715 [2,5]
+  CRUSH rule 3 x 716 [5,6]
+  CRUSH rule 3 x 717 [8,2]
+  CRUSH rule 3 x 718 [5,7]
+  CRUSH rule 3 x 719 [2,6]
+  CRUSH rule 3 x 720 [6,2]
+  CRUSH rule 3 x 721 [5,7]
+  CRUSH rule 3 x 722 [5,7]
+  CRUSH rule 3 x 723 [5,2]
+  CRUSH rule 3 x 724 [2,7]
+  CRUSH rule 3 x 725 [2,5]
+  CRUSH rule 3 x 726 [5,7]
+  CRUSH rule 3 x 727 [5,7]
+  CRUSH rule 3 x 728 [2,6]
+  CRUSH rule 3 x 729 [5,6]
+  CRUSH rule 3 x 730 [5,8]
+  CRUSH rule 3 x 731 [5,2]
+  CRUSH rule 3 x 732 [2,5]
+  CRUSH rule 3 x 733 [5,6]
+  CRUSH rule 3 x 734 [6,5]
+  CRUSH rule 3 x 735 [5,6]
+  CRUSH rule 3 x 736 [5,8]
+  CRUSH rule 3 x 737 [2,8]
+  CRUSH rule 3 x 738 [5,2]
+  CRUSH rule 3 x 739 [2,7]
+  CRUSH rule 3 x 740 [2,7]
+  CRUSH rule 3 x 741 [7,2]
+  CRUSH rule 3 x 742 [8,2]
+  CRUSH rule 3 x 743 [7,2]
+  CRUSH rule 3 x 744 [5,6]
+  CRUSH rule 3 x 745 [5,2]
+  CRUSH rule 3 x 746 [5,2]
+  CRUSH rule 3 x 747 [6,2]
+  CRUSH rule 3 x 748 [2,6]
+  CRUSH rule 3 x 749 [5,7]
+  CRUSH rule 3 x 750 [2,7]
+  CRUSH rule 3 x 751 [2,7]
+  CRUSH rule 3 x 752 [8,2]
+  CRUSH rule 3 x 753 [7,5]
+  CRUSH rule 3 x 754 [8,5]
+  CRUSH rule 3 x 755 [2,6]
+  CRUSH rule 3 x 756 [5,8]
+  CRUSH rule 3 x 757 [8,2]
+  CRUSH rule 3 x 758 [6,2]
+  CRUSH rule 3 x 759 [8,5]
+  CRUSH rule 3 x 760 [2,5]
+  CRUSH rule 3 x 761 [5,2]
+  CRUSH rule 3 x 762 [2,8]
+  CRUSH rule 3 x 763 [8,5]
+  CRUSH rule 3 x 764 [2,7]
+  CRUSH rule 3 x 765 [6,5]
+  CRUSH rule 3 x 766 [8,5]
+  CRUSH rule 3 x 767 [2,8]
+  CRUSH rule 3 x 768 [8,5]
+  CRUSH rule 3 x 769 [6,2]
+  CRUSH rule 3 x 770 [6,2]
+  CRUSH rule 3 x 771 [7,2]
+  CRUSH rule 3 x 772 [8,5]
+  CRUSH rule 3 x 773 [5,2]
+  CRUSH rule 3 x 774 [5,7]
+  CRUSH rule 3 x 775 [6,5]
+  CRUSH rule 3 x 776 [7,2]
+  CRUSH rule 3 x 777 [5,2]
+  CRUSH rule 3 x 778 [2,6]
+  CRUSH rule 3 x 779 [2,6]
+  CRUSH rule 3 x 780 [2,5]
+  CRUSH rule 3 x 781 [6,5]
+  CRUSH rule 3 x 782 [5,2]
+  CRUSH rule 3 x 783 [7,2]
+  CRUSH rule 3 x 784 [2,5]
+  CRUSH rule 3 x 785 [6,2]
+  CRUSH rule 3 x 786 [7,5]
+  CRUSH rule 3 x 787 [2,8]
+  CRUSH rule 3 x 788 [6,2]
+  CRUSH rule 3 x 789 [2,5]
+  CRUSH rule 3 x 790 [8,5]
+  CRUSH rule 3 x 791 [5,6]
+  CRUSH rule 3 x 792 [5,6]
+  CRUSH rule 3 x 793 [6,2]
+  CRUSH rule 3 x 794 [2,8]
+  CRUSH rule 3 x 795 [2,5]
+  CRUSH rule 3 x 796 [5,7]
+  CRUSH rule 3 x 797 [2,5]
+  CRUSH rule 3 x 798 [6,2]
+  CRUSH rule 3 x 799 [5,2]
+  CRUSH rule 3 x 800 [5,2]
+  CRUSH rule 3 x 801 [5,7]
+  CRUSH rule 3 x 802 [2,6]
+  CRUSH rule 3 x 803 [2,5]
+  CRUSH rule 3 x 804 [6,2]
+  CRUSH rule 3 x 805 [5,8]
+  CRUSH rule 3 x 806 [2,5]
+  CRUSH rule 3 x 807 [5,7]
+  CRUSH rule 3 x 808 [5,7]
+  CRUSH rule 3 x 809 [2,5]
+  CRUSH rule 3 x 810 [5,7]
+  CRUSH rule 3 x 811 [8,5]
+  CRUSH rule 3 x 812 [8,5]
+  CRUSH rule 3 x 813 [6,5]
+  CRUSH rule 3 x 814 [5,8]
+  CRUSH rule 3 x 815 [5,2]
+  CRUSH rule 3 x 816 [2,6]
+  CRUSH rule 3 x 817 [5,6]
+  CRUSH rule 3 x 818 [5,2]
+  CRUSH rule 3 x 819 [5,2]
+  CRUSH rule 3 x 820 [5,7]
+  CRUSH rule 3 x 821 [5,8]
+  CRUSH rule 3 x 822 [2,5]
+  CRUSH rule 3 x 823 [5,7]
+  CRUSH rule 3 x 824 [5,7]
+  CRUSH rule 3 x 825 [2,8]
+  CRUSH rule 3 x 826 [7,2]
+  CRUSH rule 3 x 827 [2,6]
+  CRUSH rule 3 x 828 [2,5]
+  CRUSH rule 3 x 829 [5,6]
+  CRUSH rule 3 x 830 [2,5]
+  CRUSH rule 3 x 831 [2,6]
+  CRUSH rule 3 x 832 [5,8]
+  CRUSH rule 3 x 833 [2,6]
+  CRUSH rule 3 x 834 [5,2]
+  CRUSH rule 3 x 835 [8,5]
+  CRUSH rule 3 x 836 [5,8]
+  CRUSH rule 3 x 837 [6,5]
+  CRUSH rule 3 x 838 [6,2]
+  CRUSH rule 3 x 839 [5,2]
+  CRUSH rule 3 x 840 [7,5]
+  CRUSH rule 3 x 841 [5,8]
+  CRUSH rule 3 x 842 [2,5]
+  CRUSH rule 3 x 843 [6,5]
+  CRUSH rule 3 x 844 [5,8]
+  CRUSH rule 3 x 845 [5,6]
+  CRUSH rule 3 x 846 [5,2]
+  CRUSH rule 3 x 847 [2,8]
+  CRUSH rule 3 x 848 [2,6]
+  CRUSH rule 3 x 849 [5,8]
+  CRUSH rule 3 x 850 [2,5]
+  CRUSH rule 3 x 851 [6,5]
+  CRUSH rule 3 x 852 [7,5]
+  CRUSH rule 3 x 853 [6,2]
+  CRUSH rule 3 x 854 [7,2]
+  CRUSH rule 3 x 855 [5,7]
+  CRUSH rule 3 x 856 [6,5]
+  CRUSH rule 3 x 857 [8,5]
+  CRUSH rule 3 x 858 [6,5]
+  CRUSH rule 3 x 859 [6,2]
+  CRUSH rule 3 x 860 [5,2]
+  CRUSH rule 3 x 861 [8,5]
+  CRUSH rule 3 x 862 [6,2]
+  CRUSH rule 3 x 863 [8,2]
+  CRUSH rule 3 x 864 [5,6]
+  CRUSH rule 3 x 865 [8,2]
+  CRUSH rule 3 x 866 [5,6]
+  CRUSH rule 3 x 867 [6,5]
+  CRUSH rule 3 x 868 [6,5]
+  CRUSH rule 3 x 869 [8,5]
+  CRUSH rule 3 x 870 [2,5]
+  CRUSH rule 3 x 871 [5,2]
+  CRUSH rule 3 x 872 [5,2]
+  CRUSH rule 3 x 873 [5,6]
+  CRUSH rule 3 x 874 [2,6]
+  CRUSH rule 3 x 875 [2,6]
+  CRUSH rule 3 x 876 [5,8]
+  CRUSH rule 3 x 877 [6,5]
+  CRUSH rule 3 x 878 [5,2]
+  CRUSH rule 3 x 879 [7,5]
+  CRUSH rule 3 x 880 [5,2]
+  CRUSH rule 3 x 881 [5,8]
+  CRUSH rule 3 x 882 [5,2]
+  CRUSH rule 3 x 883 [2,5]
+  CRUSH rule 3 x 884 [6,2]
+  CRUSH rule 3 x 885 [5,2]
+  CRUSH rule 3 x 886 [5,7]
+  CRUSH rule 3 x 887 [7,5]
+  CRUSH rule 3 x 888 [6,2]
+  CRUSH rule 3 x 889 [2,8]
+  CRUSH rule 3 x 890 [7,2]
+  CRUSH rule 3 x 891 [2,7]
+  CRUSH rule 3 x 892 [6,2]
+  CRUSH rule 3 x 893 [2,5]
+  CRUSH rule 3 x 894 [7,5]
+  CRUSH rule 3 x 895 [5,2]
+  CRUSH rule 3 x 896 [2,6]
+  CRUSH rule 3 x 897 [5,2]
+  CRUSH rule 3 x 898 [2,5]
+  CRUSH rule 3 x 899 [2,6]
+  CRUSH rule 3 x 900 [5,2]
+  CRUSH rule 3 x 901 [5,2]
+  CRUSH rule 3 x 902 [8,5]
+  CRUSH rule 3 x 903 [5,6]
+  CRUSH rule 3 x 904 [5,7]
+  CRUSH rule 3 x 905 [6,2]
+  CRUSH rule 3 x 906 [2,7]
+  CRUSH rule 3 x 907 [7,2]
+  CRUSH rule 3 x 908 [5,6]
+  CRUSH rule 3 x 909 [2,5]
+  CRUSH rule 3 x 910 [6,5]
+  CRUSH rule 3 x 911 [5,7]
+  CRUSH rule 3 x 912 [2,8]
+  CRUSH rule 3 x 913 [7,2]
+  CRUSH rule 3 x 914 [6,5]
+  CRUSH rule 3 x 915 [8,2]
+  CRUSH rule 3 x 916 [5,2]
+  CRUSH rule 3 x 917 [2,5]
+  CRUSH rule 3 x 918 [8,2]
+  CRUSH rule 3 x 919 [6,2]
+  CRUSH rule 3 x 920 [7,5]
+  CRUSH rule 3 x 921 [2,5]
+  CRUSH rule 3 x 922 [6,5]
+  CRUSH rule 3 x 923 [5,6]
+  CRUSH rule 3 x 924 [5,2]
+  CRUSH rule 3 x 925 [5,6]
+  CRUSH rule 3 x 926 [5,2]
+  CRUSH rule 3 x 927 [2,6]
+  CRUSH rule 3 x 928 [8,2]
+  CRUSH rule 3 x 929 [5,2]
+  CRUSH rule 3 x 930 [2,5]
+  CRUSH rule 3 x 931 [5,2]
+  CRUSH rule 3 x 932 [5,2]
+  CRUSH rule 3 x 933 [8,5]
+  CRUSH rule 3 x 934 [5,8]
+  CRUSH rule 3 x 935 [6,5]
+  CRUSH rule 3 x 936 [2,7]
+  CRUSH rule 3 x 937 [5,8]
+  CRUSH rule 3 x 938 [6,5]
+  CRUSH rule 3 x 939 [2,8]
+  CRUSH rule 3 x 940 [8,5]
+  CRUSH rule 3 x 941 [5,2]
+  CRUSH rule 3 x 942 [2,6]
+  CRUSH rule 3 x 943 [8,2]
+  CRUSH rule 3 x 944 [5,8]
+  CRUSH rule 3 x 945 [7,2]
+  CRUSH rule 3 x 946 [2,8]
+  CRUSH rule 3 x 947 [5,2]
+  CRUSH rule 3 x 948 [7,5]
+  CRUSH rule 3 x 949 [6,2]
+  CRUSH rule 3 x 950 [5,7]
+  CRUSH rule 3 x 951 [5,6]
+  CRUSH rule 3 x 952 [2,7]
+  CRUSH rule 3 x 953 [2,5]
+  CRUSH rule 3 x 954 [5,2]
+  CRUSH rule 3 x 955 [8,2]
+  CRUSH rule 3 x 956 [2,7]
+  CRUSH rule 3 x 957 [7,2]
+  CRUSH rule 3 x 958 [8,5]
+  CRUSH rule 3 x 959 [5,2]
+  CRUSH rule 3 x 960 [5,6]
+  CRUSH rule 3 x 961 [5,2]
+  CRUSH rule 3 x 962 [7,5]
+  CRUSH rule 3 x 963 [2,5]
+  CRUSH rule 3 x 964 [5,2]
+  CRUSH rule 3 x 965 [7,5]
+  CRUSH rule 3 x 966 [5,6]
+  CRUSH rule 3 x 967 [8,5]
+  CRUSH rule 3 x 968 [7,2]
+  CRUSH rule 3 x 969 [8,2]
+  CRUSH rule 3 x 970 [2,8]
+  CRUSH rule 3 x 971 [2,8]
+  CRUSH rule 3 x 972 [2,7]
+  CRUSH rule 3 x 973 [2,8]
+  CRUSH rule 3 x 974 [5,2]
+  CRUSH rule 3 x 975 [5,8]
+  CRUSH rule 3 x 976 [5,7]
+  CRUSH rule 3 x 977 [8,5]
+  CRUSH rule 3 x 978 [7,2]
+  CRUSH rule 3 x 979 [7,2]
+  CRUSH rule 3 x 980 [6,2]
+  CRUSH rule 3 x 981 [7,5]
+  CRUSH rule 3 x 982 [5,2]
+  CRUSH rule 3 x 983 [5,6]
+  CRUSH rule 3 x 984 [2,8]
+  CRUSH rule 3 x 985 [2,5]
+  CRUSH rule 3 x 986 [8,5]
+  CRUSH rule 3 x 987 [2,5]
+  CRUSH rule 3 x 988 [2,5]
+  CRUSH rule 3 x 989 [2,8]
+  CRUSH rule 3 x 990 [2,6]
+  CRUSH rule 3 x 991 [2,5]
+  CRUSH rule 3 x 992 [7,2]
+  CRUSH rule 3 x 993 [2,6]
+  CRUSH rule 3 x 994 [5,6]
+  CRUSH rule 3 x 995 [7,2]
+  CRUSH rule 3 x 996 [6,5]
+  CRUSH rule 3 x 997 [6,5]
+  CRUSH rule 3 x 998 [8,2]
+  CRUSH rule 3 x 999 [2,7]
+  CRUSH rule 3 x 1000 [8,5]
+  CRUSH rule 3 x 1001 [2,5]
+  CRUSH rule 3 x 1002 [2,5]
+  CRUSH rule 3 x 1003 [2,7]
+  CRUSH rule 3 x 1004 [6,2]
+  CRUSH rule 3 x 1005 [6,2]
+  CRUSH rule 3 x 1006 [2,8]
+  CRUSH rule 3 x 1007 [2,5]
+  CRUSH rule 3 x 1008 [2,7]
+  CRUSH rule 3 x 1009 [6,5]
+  CRUSH rule 3 x 1010 [5,2]
+  CRUSH rule 3 x 1011 [5,2]
+  CRUSH rule 3 x 1012 [5,2]
+  CRUSH rule 3 x 1013 [5,2]
+  CRUSH rule 3 x 1014 [2,8]
+  CRUSH rule 3 x 1015 [6,5]
+  CRUSH rule 3 x 1016 [2,5]
+  CRUSH rule 3 x 1017 [6,2]
+  CRUSH rule 3 x 1018 [5,2]
+  CRUSH rule 3 x 1019 [5,8]
+  CRUSH rule 3 x 1020 [5,2]
+  CRUSH rule 3 x 1021 [5,2]
+  CRUSH rule 3 x 1022 [2,7]
+  CRUSH rule 3 x 1023 [5,2]
+  rule 3 (choose-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [2,5,7]
+  CRUSH rule 3 x 1 [2,8,5]
+  CRUSH rule 3 x 2 [2,5,8]
+  CRUSH rule 3 x 3 [8,2,5]
+  CRUSH rule 3 x 4 [5,2,8]
+  CRUSH rule 3 x 5 [7,2,5]
+  CRUSH rule 3 x 6 [2,6,5]
+  CRUSH rule 3 x 7 [5,6,2]
+  CRUSH rule 3 x 8 [5,7,2]
+  CRUSH rule 3 x 9 [2,5,8]
+  CRUSH rule 3 x 10 [2,8,5]
+  CRUSH rule 3 x 11 [2,6,5]
+  CRUSH rule 3 x 12 [2,5,7]
+  CRUSH rule 3 x 13 [5,8,2]
+  CRUSH rule 3 x 14 [7,2,5]
+  CRUSH rule 3 x 15 [7,2,5]
+  CRUSH rule 3 x 16 [5,7,2]
+  CRUSH rule 3 x 17 [5,2,7]
+  CRUSH rule 3 x 18 [2,5,6]
+  CRUSH rule 3 x 19 [7,5,2]
+  CRUSH rule 3 x 20 [2,5,7]
+  CRUSH rule 3 x 21 [5,6,2]
+  CRUSH rule 3 x 22 [8,5,2]
+  CRUSH rule 3 x 23 [5,7,2]
+  CRUSH rule 3 x 24 [2,6,5]
+  CRUSH rule 3 x 25 [5,8,2]
+  CRUSH rule 3 x 26 [2,7,5]
+  CRUSH rule 3 x 27 [5,2,7]
+  CRUSH rule 3 x 28 [6,2,5]
+  CRUSH rule 3 x 29 [8,5,2]
+  CRUSH rule 3 x 30 [5,6,2]
+  CRUSH rule 3 x 31 [8,2,5]
+  CRUSH rule 3 x 32 [5,7,2]
+  CRUSH rule 3 x 33 [2,7,5]
+  CRUSH rule 3 x 34 [2,5,7]
+  CRUSH rule 3 x 35 [2,6,5]
+  CRUSH rule 3 x 36 [5,6,2]
+  CRUSH rule 3 x 37 [2,5,7]
+  CRUSH rule 3 x 38 [5,6,2]
+  CRUSH rule 3 x 39 [5,7,2]
+  CRUSH rule 3 x 40 [7,2,5]
+  CRUSH rule 3 x 41 [2,7,5]
+  CRUSH rule 3 x 42 [5,7,2]
+  CRUSH rule 3 x 43 [2,5,8]
+  CRUSH rule 3 x 44 [2,8,5]
+  CRUSH rule 3 x 45 [8,2,5]
+  CRUSH rule 3 x 46 [2,5,8]
+  CRUSH rule 3 x 47 [5,2,7]
+  CRUSH rule 3 x 48 [5,6,2]
+  CRUSH rule 3 x 49 [5,6,2]
+  CRUSH rule 3 x 50 [5,2,7]
+  CRUSH rule 3 x 51 [5,6,2]
+  CRUSH rule 3 x 52 [8,2,5]
+  CRUSH rule 3 x 53 [5,6,2]
+  CRUSH rule 3 x 54 [7,5,2]
+  CRUSH rule 3 x 55 [8,2,5]
+  CRUSH rule 3 x 56 [6,5,2]
+  CRUSH rule 3 x 57 [5,8,2]
+  CRUSH rule 3 x 58 [2,8,5]
+  CRUSH rule 3 x 59 [5,2,7]
+  CRUSH rule 3 x 60 [5,2,8]
+  CRUSH rule 3 x 61 [5,8,2]
+  CRUSH rule 3 x 62 [7,2,5]
+  CRUSH rule 3 x 63 [5,6,2]
+  CRUSH rule 3 x 64 [5,2,8]
+  CRUSH rule 3 x 65 [7,5,2]
+  CRUSH rule 3 x 66 [5,6,2]
+  CRUSH rule 3 x 67 [5,2,6]
+  CRUSH rule 3 x 68 [2,5,8]
+  CRUSH rule 3 x 69 [5,2,7]
+  CRUSH rule 3 x 70 [7,2,5]
+  CRUSH rule 3 x 71 [2,7,5]
+  CRUSH rule 3 x 72 [6,2,5]
+  CRUSH rule 3 x 73 [2,7,5]
+  CRUSH rule 3 x 74 [2,8,5]
+  CRUSH rule 3 x 75 [5,2,7]
+  CRUSH rule 3 x 76 [5,2,6]
+  CRUSH rule 3 x 77 [7,2,5]
+  CRUSH rule 3 x 78 [2,5,7]
+  CRUSH rule 3 x 79 [5,2,7]
+  CRUSH rule 3 x 80 [2,5,6]
+  CRUSH rule 3 x 81 [2,5,8]
+  CRUSH rule 3 x 82 [7,2,5]
+  CRUSH rule 3 x 83 [2,6,5]
+  CRUSH rule 3 x 84 [7,2,5]
+  CRUSH rule 3 x 85 [5,7,2]
+  CRUSH rule 3 x 86 [2,6,5]
+  CRUSH rule 3 x 87 [2,6,5]
+  CRUSH rule 3 x 88 [2,8,5]
+  CRUSH rule 3 x 89 [5,2,8]
+  CRUSH rule 3 x 90 [6,5,2]
+  CRUSH rule 3 x 91 [5,6,2]
+  CRUSH rule 3 x 92 [2,6,5]
+  CRUSH rule 3 x 93 [7,5,2]
+  CRUSH rule 3 x 94 [2,5,7]
+  CRUSH rule 3 x 95 [7,5,2]
+  CRUSH rule 3 x 96 [5,8,2]
+  CRUSH rule 3 x 97 [8,5,2]
+  CRUSH rule 3 x 98 [2,7,5]
+  CRUSH rule 3 x 99 [2,8,5]
+  CRUSH rule 3 x 100 [2,6,5]
+  CRUSH rule 3 x 101 [5,8,2]
+  CRUSH rule 3 x 102 [5,2,7]
+  CRUSH rule 3 x 103 [5,7,2]
+  CRUSH rule 3 x 104 [7,5,2]
+  CRUSH rule 3 x 105 [2,5,8]
+  CRUSH rule 3 x 106 [2,6,5]
+  CRUSH rule 3 x 107 [5,2,7]
+  CRUSH rule 3 x 108 [7,2,5]
+  CRUSH rule 3 x 109 [2,5,8]
+  CRUSH rule 3 x 110 [5,2,6]
+  CRUSH rule 3 x 111 [2,5,7]
+  CRUSH rule 3 x 112 [2,6,5]
+  CRUSH rule 3 x 113 [6,2,5]
+  CRUSH rule 3 x 114 [7,5,2]
+  CRUSH rule 3 x 115 [8,2,5]
+  CRUSH rule 3 x 116 [2,7,5]
+  CRUSH rule 3 x 117 [7,5,2]
+  CRUSH rule 3 x 118 [2,5,6]
+  CRUSH rule 3 x 119 [5,7,2]
+  CRUSH rule 3 x 120 [2,5,6]
+  CRUSH rule 3 x 121 [2,8,5]
+  CRUSH rule 3 x 122 [8,5,2]
+  CRUSH rule 3 x 123 [2,5,7]
+  CRUSH rule 3 x 124 [5,2,8]
+  CRUSH rule 3 x 125 [2,7,5]
+  CRUSH rule 3 x 126 [5,2,7]
+  CRUSH rule 3 x 127 [5,6,2]
+  CRUSH rule 3 x 128 [5,8,2]
+  CRUSH rule 3 x 129 [2,5,7]
+  CRUSH rule 3 x 130 [5,8,2]
+  CRUSH rule 3 x 131 [2,5,6]
+  CRUSH rule 3 x 132 [2,5,8]
+  CRUSH rule 3 x 133 [5,6,2]
+  CRUSH rule 3 x 134 [2,7,5]
+  CRUSH rule 3 x 135 [5,7,2]
+  CRUSH rule 3 x 136 [2,5,8]
+  CRUSH rule 3 x 137 [7,5,2]
+  CRUSH rule 3 x 138 [8,5,2]
+  CRUSH rule 3 x 139 [5,2,8]
+  CRUSH rule 3 x 140 [2,8,5]
+  CRUSH rule 3 x 141 [6,2,5]
+  CRUSH rule 3 x 142 [5,2,8]
+  CRUSH rule 3 x 143 [5,7,2]
+  CRUSH rule 3 x 144 [8,2,5]
+  CRUSH rule 3 x 145 [8,5,2]
+  CRUSH rule 3 x 146 [2,8,5]
+  CRUSH rule 3 x 147 [2,6,5]
+  CRUSH rule 3 x 148 [5,2,8]
+  CRUSH rule 3 x 149 [5,6,2]
+  CRUSH rule 3 x 150 [2,8,5]
+  CRUSH rule 3 x 151 [5,8,2]
+  CRUSH rule 3 x 152 [8,5,2]
+  CRUSH rule 3 x 153 [8,5,2]
+  CRUSH rule 3 x 154 [5,2,6]
+  CRUSH rule 3 x 155 [5,6,2]
+  CRUSH rule 3 x 156 [5,2,7]
+  CRUSH rule 3 x 157 [5,2,7]
+  CRUSH rule 3 x 158 [2,6,5]
+  CRUSH rule 3 x 159 [7,2,5]
+  CRUSH rule 3 x 160 [2,8,5]
+  CRUSH rule 3 x 161 [2,5,6]
+  CRUSH rule 3 x 162 [2,8,5]
+  CRUSH rule 3 x 163 [5,8,2]
+  CRUSH rule 3 x 164 [7,2,5]
+  CRUSH rule 3 x 165 [7,2,5]
+  CRUSH rule 3 x 166 [2,5,7]
+  CRUSH rule 3 x 167 [2,6,5]
+  CRUSH rule 3 x 168 [5,2,7]
+  CRUSH rule 3 x 169 [2,7,5]
+  CRUSH rule 3 x 170 [2,5,8]
+  CRUSH rule 3 x 171 [7,5,2]
+  CRUSH rule 3 x 172 [2,8,5]
+  CRUSH rule 3 x 173 [8,5,2]
+  CRUSH rule 3 x 174 [2,5,8]
+  CRUSH rule 3 x 175 [6,2,5]
+  CRUSH rule 3 x 176 [5,2,7]
+  CRUSH rule 3 x 177 [5,2,8]
+  CRUSH rule 3 x 178 [5,2,6]
+  CRUSH rule 3 x 179 [5,2,7]
+  CRUSH rule 3 x 180 [5,7,2]
+  CRUSH rule 3 x 181 [6,2,5]
+  CRUSH rule 3 x 182 [8,5,2]
+  CRUSH rule 3 x 183 [7,5,2]
+  CRUSH rule 3 x 184 [5,6,2]
+  CRUSH rule 3 x 185 [6,2,5]
+  CRUSH rule 3 x 186 [2,5,8]
+  CRUSH rule 3 x 187 [2,6,5]
+  CRUSH rule 3 x 188 [2,6,5]
+  CRUSH rule 3 x 189 [2,6,5]
+  CRUSH rule 3 x 190 [5,2,8]
+  CRUSH rule 3 x 191 [7,2,5]
+  CRUSH rule 3 x 192 [5,2,6]
+  CRUSH rule 3 x 193 [5,2,6]
+  CRUSH rule 3 x 194 [2,5,7]
+  CRUSH rule 3 x 195 [6,5,2]
+  CRUSH rule 3 x 196 [6,2,5]
+  CRUSH rule 3 x 197 [6,5,2]
+  CRUSH rule 3 x 198 [2,5,6]
+  CRUSH rule 3 x 199 [2,5,7]
+  CRUSH rule 3 x 200 [2,5,8]
+  CRUSH rule 3 x 201 [7,2,5]
+  CRUSH rule 3 x 202 [6,5,2]
+  CRUSH rule 3 x 203 [5,6,2]
+  CRUSH rule 3 x 204 [2,5,8]
+  CRUSH rule 3 x 205 [2,7,5]
+  CRUSH rule 3 x 206 [2,8,5]
+  CRUSH rule 3 x 207 [5,2,7]
+  CRUSH rule 3 x 208 [7,2,5]
+  CRUSH rule 3 x 209 [2,8,5]
+  CRUSH rule 3 x 210 [2,5,6]
+  CRUSH rule 3 x 211 [5,2,7]
+  CRUSH rule 3 x 212 [7,5,2]
+  CRUSH rule 3 x 213 [8,5,2]
+  CRUSH rule 3 x 214 [5,7,2]
+  CRUSH rule 3 x 215 [8,2,5]
+  CRUSH rule 3 x 216 [5,2,8]
+  CRUSH rule 3 x 217 [2,7,5]
+  CRUSH rule 3 x 218 [2,7,5]
+  CRUSH rule 3 x 219 [5,6,2]
+  CRUSH rule 3 x 220 [5,8,2]
+  CRUSH rule 3 x 221 [5,7,2]
+  CRUSH rule 3 x 222 [6,5,2]
+  CRUSH rule 3 x 223 [2,5,6]
+  CRUSH rule 3 x 224 [2,5,8]
+  CRUSH rule 3 x 225 [8,2,5]
+  CRUSH rule 3 x 226 [7,2,5]
+  CRUSH rule 3 x 227 [5,2,7]
+  CRUSH rule 3 x 228 [5,6,2]
+  CRUSH rule 3 x 229 [5,7,2]
+  CRUSH rule 3 x 230 [5,6,2]
+  CRUSH rule 3 x 231 [5,7,2]
+  CRUSH rule 3 x 232 [2,8,5]
+  CRUSH rule 3 x 233 [5,6,2]
+  CRUSH rule 3 x 234 [2,5,6]
+  CRUSH rule 3 x 235 [5,6,2]
+  CRUSH rule 3 x 236 [5,2,8]
+  CRUSH rule 3 x 237 [5,8,2]
+  CRUSH rule 3 x 238 [5,2,6]
+  CRUSH rule 3 x 239 [8,5,2]
+  CRUSH rule 3 x 240 [5,8,2]
+  CRUSH rule 3 x 241 [5,2,7]
+  CRUSH rule 3 x 242 [5,2,6]
+  CRUSH rule 3 x 243 [5,8,2]
+  CRUSH rule 3 x 244 [5,6,2]
+  CRUSH rule 3 x 245 [7,2,5]
+  CRUSH rule 3 x 246 [2,5,7]
+  CRUSH rule 3 x 247 [6,2,5]
+  CRUSH rule 3 x 248 [8,2,5]
+  CRUSH rule 3 x 249 [2,5,8]
+  CRUSH rule 3 x 250 [2,5,8]
+  CRUSH rule 3 x 251 [2,5,7]
+  CRUSH rule 3 x 252 [5,7,2]
+  CRUSH rule 3 x 253 [5,2,7]
+  CRUSH rule 3 x 254 [5,2,8]
+  CRUSH rule 3 x 255 [2,7,5]
+  CRUSH rule 3 x 256 [5,6,2]
+  CRUSH rule 3 x 257 [2,6,5]
+  CRUSH rule 3 x 258 [5,2,8]
+  CRUSH rule 3 x 259 [5,6,2]
+  CRUSH rule 3 x 260 [5,8,2]
+  CRUSH rule 3 x 261 [8,5,2]
+  CRUSH rule 3 x 262 [5,6,2]
+  CRUSH rule 3 x 263 [6,2,5]
+  CRUSH rule 3 x 264 [5,6,2]
+  CRUSH rule 3 x 265 [8,5,2]
+  CRUSH rule 3 x 266 [8,2,5]
+  CRUSH rule 3 x 267 [2,5,6]
+  CRUSH rule 3 x 268 [2,6,5]
+  CRUSH rule 3 x 269 [2,6,5]
+  CRUSH rule 3 x 270 [5,2,8]
+  CRUSH rule 3 x 271 [7,5,2]
+  CRUSH rule 3 x 272 [2,6,5]
+  CRUSH rule 3 x 273 [5,2,7]
+  CRUSH rule 3 x 274 [6,5,2]
+  CRUSH rule 3 x 275 [5,8,2]
+  CRUSH rule 3 x 276 [7,2,5]
+  CRUSH rule 3 x 277 [6,5,2]
+  CRUSH rule 3 x 278 [6,2,5]
+  CRUSH rule 3 x 279 [8,5,2]
+  CRUSH rule 3 x 280 [2,7,5]
+  CRUSH rule 3 x 281 [8,2,5]
+  CRUSH rule 3 x 282 [5,2,8]
+  CRUSH rule 3 x 283 [8,2,5]
+  CRUSH rule 3 x 284 [6,5,2]
+  CRUSH rule 3 x 285 [5,7,2]
+  CRUSH rule 3 x 286 [2,8,5]
+  CRUSH rule 3 x 287 [2,5,8]
+  CRUSH rule 3 x 288 [8,2,5]
+  CRUSH rule 3 x 289 [5,6,2]
+  CRUSH rule 3 x 290 [2,5,6]
+  CRUSH rule 3 x 291 [2,5,8]
+  CRUSH rule 3 x 292 [8,2,5]
+  CRUSH rule 3 x 293 [6,2,5]
+  CRUSH rule 3 x 294 [7,5,2]
+  CRUSH rule 3 x 295 [5,6,2]
+  CRUSH rule 3 x 296 [5,2,6]
+  CRUSH rule 3 x 297 [6,2,5]
+  CRUSH rule 3 x 298 [2,5,6]
+  CRUSH rule 3 x 299 [2,8,5]
+  CRUSH rule 3 x 300 [8,5,2]
+  CRUSH rule 3 x 301 [2,7,5]
+  CRUSH rule 3 x 302 [5,2,6]
+  CRUSH rule 3 x 303 [7,5,2]
+  CRUSH rule 3 x 304 [2,6,5]
+  CRUSH rule 3 x 305 [5,6,2]
+  CRUSH rule 3 x 306 [2,8,5]
+  CRUSH rule 3 x 307 [2,7,5]
+  CRUSH rule 3 x 308 [2,8,5]
+  CRUSH rule 3 x 309 [7,5,2]
+  CRUSH rule 3 x 310 [5,2,6]
+  CRUSH rule 3 x 311 [5,8,2]
+  CRUSH rule 3 x 312 [2,6,5]
+  CRUSH rule 3 x 313 [5,2,7]
+  CRUSH rule 3 x 314 [5,2,6]
+  CRUSH rule 3 x 315 [2,5,7]
+  CRUSH rule 3 x 316 [6,5,2]
+  CRUSH rule 3 x 317 [2,7,5]
+  CRUSH rule 3 x 318 [8,2,5]
+  CRUSH rule 3 x 319 [5,2,6]
+  CRUSH rule 3 x 320 [5,8,2]
+  CRUSH rule 3 x 321 [2,5,8]
+  CRUSH rule 3 x 322 [2,6,5]
+  CRUSH rule 3 x 323 [5,7,2]
+  CRUSH rule 3 x 324 [7,2,5]
+  CRUSH rule 3 x 325 [5,6,2]
+  CRUSH rule 3 x 326 [5,2,7]
+  CRUSH rule 3 x 327 [2,8,5]
+  CRUSH rule 3 x 328 [7,5,2]
+  CRUSH rule 3 x 329 [5,7,2]
+  CRUSH rule 3 x 330 [5,7,2]
+  CRUSH rule 3 x 331 [2,7,5]
+  CRUSH rule 3 x 332 [2,5,8]
+  CRUSH rule 3 x 333 [6,5,2]
+  CRUSH rule 3 x 334 [8,5,2]
+  CRUSH rule 3 x 335 [7,2,5]
+  CRUSH rule 3 x 336 [5,6,2]
+  CRUSH rule 3 x 337 [7,2,5]
+  CRUSH rule 3 x 338 [5,8,2]
+  CRUSH rule 3 x 339 [7,5,2]
+  CRUSH rule 3 x 340 [2,6,5]
+  CRUSH rule 3 x 341 [5,2,7]
+  CRUSH rule 3 x 342 [2,8,5]
+  CRUSH rule 3 x 343 [6,5,2]
+  CRUSH rule 3 x 344 [6,2,5]
+  CRUSH rule 3 x 345 [5,7,2]
+  CRUSH rule 3 x 346 [8,2,5]
+  CRUSH rule 3 x 347 [5,2,8]
+  CRUSH rule 3 x 348 [8,2,5]
+  CRUSH rule 3 x 349 [2,7,5]
+  CRUSH rule 3 x 350 [8,5,2]
+  CRUSH rule 3 x 351 [5,8,2]
+  CRUSH rule 3 x 352 [2,8,5]
+  CRUSH rule 3 x 353 [6,5,2]
+  CRUSH rule 3 x 354 [2,5,6]
+  CRUSH rule 3 x 355 [5,8,2]
+  CRUSH rule 3 x 356 [5,2,8]
+  CRUSH rule 3 x 357 [6,2,5]
+  CRUSH rule 3 x 358 [2,8,5]
+  CRUSH rule 3 x 359 [6,2,5]
+  CRUSH rule 3 x 360 [5,2,8]
+  CRUSH rule 3 x 361 [8,5,2]
+  CRUSH rule 3 x 362 [5,2,8]
+  CRUSH rule 3 x 363 [5,2,8]
+  CRUSH rule 3 x 364 [2,5,7]
+  CRUSH rule 3 x 365 [6,5,2]
+  CRUSH rule 3 x 366 [7,2,5]
+  CRUSH rule 3 x 367 [5,2,7]
+  CRUSH rule 3 x 368 [7,5,2]
+  CRUSH rule 3 x 369 [5,7,2]
+  CRUSH rule 3 x 370 [8,2,5]
+  CRUSH rule 3 x 371 [2,5,8]
+  CRUSH rule 3 x 372 [5,2,8]
+  CRUSH rule 3 x 373 [2,6,5]
+  CRUSH rule 3 x 374 [5,8,2]
+  CRUSH rule 3 x 375 [6,5,2]
+  CRUSH rule 3 x 376 [7,2,5]
+  CRUSH rule 3 x 377 [2,5,7]
+  CRUSH rule 3 x 378 [2,6,5]
+  CRUSH rule 3 x 379 [8,5,2]
+  CRUSH rule 3 x 380 [2,5,8]
+  CRUSH rule 3 x 381 [2,5,7]
+  CRUSH rule 3 x 382 [2,5,7]
+  CRUSH rule 3 x 383 [5,7,2]
+  CRUSH rule 3 x 384 [7,2,5]
+  CRUSH rule 3 x 385 [7,5,2]
+  CRUSH rule 3 x 386 [2,5,6]
+  CRUSH rule 3 x 387 [2,5,8]
+  CRUSH rule 3 x 388 [5,2,7]
+  CRUSH rule 3 x 389 [2,5,8]
+  CRUSH rule 3 x 390 [5,8,2]
+  CRUSH rule 3 x 391 [5,6,2]
+  CRUSH rule 3 x 392 [2,7,5]
+  CRUSH rule 3 x 393 [5,2,6]
+  CRUSH rule 3 x 394 [5,8,2]
+  CRUSH rule 3 x 395 [5,2,8]
+  CRUSH rule 3 x 396 [5,2,6]
+  CRUSH rule 3 x 397 [2,5,7]
+  CRUSH rule 3 x 398 [2,5,6]
+  CRUSH rule 3 x 399 [8,5,2]
+  CRUSH rule 3 x 400 [8,2,5]
+  CRUSH rule 3 x 401 [2,5,6]
+  CRUSH rule 3 x 402 [7,5,2]
+  CRUSH rule 3 x 403 [2,5,7]
+  CRUSH rule 3 x 404 [5,2,6]
+  CRUSH rule 3 x 405 [6,5,2]
+  CRUSH rule 3 x 406 [2,6,5]
+  CRUSH rule 3 x 407 [2,7,5]
+  CRUSH rule 3 x 408 [5,2,7]
+  CRUSH rule 3 x 409 [7,5,2]
+  CRUSH rule 3 x 410 [8,5,2]
+  CRUSH rule 3 x 411 [2,7,5]
+  CRUSH rule 3 x 412 [2,5,7]
+  CRUSH rule 3 x 413 [5,2,8]
+  CRUSH rule 3 x 414 [5,2,8]
+  CRUSH rule 3 x 415 [2,6,5]
+  CRUSH rule 3 x 416 [2,8,5]
+  CRUSH rule 3 x 417 [8,2,5]
+  CRUSH rule 3 x 418 [7,2,5]
+  CRUSH rule 3 x 419 [8,5,2]
+  CRUSH rule 3 x 420 [2,5,7]
+  CRUSH rule 3 x 421 [8,5,2]
+  CRUSH rule 3 x 422 [6,5,2]
+  CRUSH rule 3 x 423 [2,5,7]
+  CRUSH rule 3 x 424 [8,5,2]
+  CRUSH rule 3 x 425 [2,5,8]
+  CRUSH rule 3 x 426 [6,2,5]
+  CRUSH rule 3 x 427 [2,8,5]
+  CRUSH rule 3 x 428 [5,8,2]
+  CRUSH rule 3 x 429 [5,8,2]
+  CRUSH rule 3 x 430 [5,7,2]
+  CRUSH rule 3 x 431 [5,2,7]
+  CRUSH rule 3 x 432 [7,2,5]
+  CRUSH rule 3 x 433 [6,5,2]
+  CRUSH rule 3 x 434 [5,2,7]
+  CRUSH rule 3 x 435 [2,5,6]
+  CRUSH rule 3 x 436 [5,2,7]
+  CRUSH rule 3 x 437 [7,5,2]
+  CRUSH rule 3 x 438 [2,5,8]
+  CRUSH rule 3 x 439 [2,5,8]
+  CRUSH rule 3 x 440 [2,6,5]
+  CRUSH rule 3 x 441 [5,8,2]
+  CRUSH rule 3 x 442 [2,5,6]
+  CRUSH rule 3 x 443 [6,2,5]
+  CRUSH rule 3 x 444 [7,2,5]
+  CRUSH rule 3 x 445 [6,5,2]
+  CRUSH rule 3 x 446 [5,2,8]
+  CRUSH rule 3 x 447 [2,5,6]
+  CRUSH rule 3 x 448 [7,2,5]
+  CRUSH rule 3 x 449 [7,5,2]
+  CRUSH rule 3 x 450 [5,2,6]
+  CRUSH rule 3 x 451 [6,5,2]
+  CRUSH rule 3 x 452 [8,5,2]
+  CRUSH rule 3 x 453 [6,5,2]
+  CRUSH rule 3 x 454 [6,5,2]
+  CRUSH rule 3 x 455 [2,8,5]
+  CRUSH rule 3 x 456 [6,2,5]
+  CRUSH rule 3 x 457 [7,2,5]
+  CRUSH rule 3 x 458 [2,8,5]
+  CRUSH rule 3 x 459 [2,6,5]
+  CRUSH rule 3 x 460 [6,5,2]
+  CRUSH rule 3 x 461 [6,5,2]
+  CRUSH rule 3 x 462 [8,2,5]
+  CRUSH rule 3 x 463 [6,2,5]
+  CRUSH rule 3 x 464 [7,5,2]
+  CRUSH rule 3 x 465 [7,2,5]
+  CRUSH rule 3 x 466 [5,6,2]
+  CRUSH rule 3 x 467 [6,5,2]
+  CRUSH rule 3 x 468 [7,2,5]
+  CRUSH rule 3 x 469 [7,2,5]
+  CRUSH rule 3 x 470 [5,2,8]
+  CRUSH rule 3 x 471 [2,6,5]
+  CRUSH rule 3 x 472 [5,2,8]
+  CRUSH rule 3 x 473 [2,5,7]
+  CRUSH rule 3 x 474 [6,2,5]
+  CRUSH rule 3 x 475 [6,2,5]
+  CRUSH rule 3 x 476 [5,7,2]
+  CRUSH rule 3 x 477 [5,6,2]
+  CRUSH rule 3 x 478 [6,2,5]
+  CRUSH rule 3 x 479 [2,5,6]
+  CRUSH rule 3 x 480 [2,6,5]
+  CRUSH rule 3 x 481 [2,5,7]
+  CRUSH rule 3 x 482 [5,8,2]
+  CRUSH rule 3 x 483 [2,6,5]
+  CRUSH rule 3 x 484 [2,8,5]
+  CRUSH rule 3 x 485 [5,8,2]
+  CRUSH rule 3 x 486 [5,2,8]
+  CRUSH rule 3 x 487 [5,2,8]
+  CRUSH rule 3 x 488 [5,7,2]
+  CRUSH rule 3 x 489 [2,8,5]
+  CRUSH rule 3 x 490 [6,5,2]
+  CRUSH rule 3 x 491 [2,6,5]
+  CRUSH rule 3 x 492 [6,5,2]
+  CRUSH rule 3 x 493 [2,8,5]
+  CRUSH rule 3 x 494 [2,6,5]
+  CRUSH rule 3 x 495 [5,2,6]
+  CRUSH rule 3 x 496 [7,5,2]
+  CRUSH rule 3 x 497 [5,7,2]
+  CRUSH rule 3 x 498 [2,5,6]
+  CRUSH rule 3 x 499 [8,5,2]
+  CRUSH rule 3 x 500 [5,6,2]
+  CRUSH rule 3 x 501 [2,7,5]
+  CRUSH rule 3 x 502 [7,2,5]
+  CRUSH rule 3 x 503 [2,5,7]
+  CRUSH rule 3 x 504 [5,8,2]
+  CRUSH rule 3 x 505 [2,7,5]
+  CRUSH rule 3 x 506 [5,2,7]
+  CRUSH rule 3 x 507 [6,2,5]
+  CRUSH rule 3 x 508 [2,5,7]
+  CRUSH rule 3 x 509 [7,5,2]
+  CRUSH rule 3 x 510 [6,2,5]
+  CRUSH rule 3 x 511 [5,6,2]
+  CRUSH rule 3 x 512 [7,2,5]
+  CRUSH rule 3 x 513 [7,2,5]
+  CRUSH rule 3 x 514 [5,7,2]
+  CRUSH rule 3 x 515 [8,5,2]
+  CRUSH rule 3 x 516 [5,2,7]
+  CRUSH rule 3 x 517 [7,2,5]
+  CRUSH rule 3 x 518 [5,6,2]
+  CRUSH rule 3 x 519 [7,5,2]
+  CRUSH rule 3 x 520 [2,8,5]
+  CRUSH rule 3 x 521 [8,2,5]
+  CRUSH rule 3 x 522 [6,2,5]
+  CRUSH rule 3 x 523 [5,2,7]
+  CRUSH rule 3 x 524 [2,5,7]
+  CRUSH rule 3 x 525 [2,5,8]
+  CRUSH rule 3 x 526 [2,5,8]
+  CRUSH rule 3 x 527 [2,5,6]
+  CRUSH rule 3 x 528 [5,2,7]
+  CRUSH rule 3 x 529 [5,6,2]
+  CRUSH rule 3 x 530 [6,5,2]
+  CRUSH rule 3 x 531 [6,2,5]
+  CRUSH rule 3 x 532 [6,5,2]
+  CRUSH rule 3 x 533 [5,8,2]
+  CRUSH rule 3 x 534 [7,5,2]
+  CRUSH rule 3 x 535 [8,2,5]
+  CRUSH rule 3 x 536 [6,2,5]
+  CRUSH rule 3 x 537 [5,8,2]
+  CRUSH rule 3 x 538 [6,5,2]
+  CRUSH rule 3 x 539 [8,5,2]
+  CRUSH rule 3 x 540 [2,7,5]
+  CRUSH rule 3 x 541 [2,5,7]
+  CRUSH rule 3 x 542 [5,2,8]
+  CRUSH rule 3 x 543 [6,2,5]
+  CRUSH rule 3 x 544 [5,7,2]
+  CRUSH rule 3 x 545 [5,7,2]
+  CRUSH rule 3 x 546 [6,2,5]
+  CRUSH rule 3 x 547 [8,2,5]
+  CRUSH rule 3 x 548 [5,2,8]
+  CRUSH rule 3 x 549 [5,7,2]
+  CRUSH rule 3 x 550 [2,5,6]
+  CRUSH rule 3 x 551 [7,5,2]
+  CRUSH rule 3 x 552 [5,7,2]
+  CRUSH rule 3 x 553 [5,2,8]
+  CRUSH rule 3 x 554 [2,6,5]
+  CRUSH rule 3 x 555 [5,2,8]
+  CRUSH rule 3 x 556 [5,6,2]
+  CRUSH rule 3 x 557 [7,5,2]
+  CRUSH rule 3 x 558 [5,2,6]
+  CRUSH rule 3 x 559 [5,2,8]
+  CRUSH rule 3 x 560 [8,5,2]
+  CRUSH rule 3 x 561 [6,5,2]
+  CRUSH rule 3 x 562 [5,2,8]
+  CRUSH rule 3 x 563 [2,7,5]
+  CRUSH rule 3 x 564 [5,2,7]
+  CRUSH rule 3 x 565 [5,8,2]
+  CRUSH rule 3 x 566 [5,6,2]
+  CRUSH rule 3 x 567 [5,7,2]
+  CRUSH rule 3 x 568 [7,5,2]
+  CRUSH rule 3 x 569 [5,2,6]
+  CRUSH rule 3 x 570 [2,5,7]
+  CRUSH rule 3 x 571 [5,6,2]
+  CRUSH rule 3 x 572 [5,2,7]
+  CRUSH rule 3 x 573 [5,2,7]
+  CRUSH rule 3 x 574 [2,5,6]
+  CRUSH rule 3 x 575 [8,2,5]
+  CRUSH rule 3 x 576 [5,7,2]
+  CRUSH rule 3 x 577 [8,2,5]
+  CRUSH rule 3 x 578 [6,2,5]
+  CRUSH rule 3 x 579 [5,2,6]
+  CRUSH rule 3 x 580 [5,2,8]
+  CRUSH rule 3 x 581 [7,2,5]
+  CRUSH rule 3 x 582 [2,8,5]
+  CRUSH rule 3 x 583 [6,2,5]
+  CRUSH rule 3 x 584 [8,2,5]
+  CRUSH rule 3 x 585 [7,2,5]
+  CRUSH rule 3 x 586 [2,8,5]
+  CRUSH rule 3 x 587 [2,5,6]
+  CRUSH rule 3 x 588 [5,8,2]
+  CRUSH rule 3 x 589 [7,2,5]
+  CRUSH rule 3 x 590 [6,2,5]
+  CRUSH rule 3 x 591 [5,2,7]
+  CRUSH rule 3 x 592 [2,5,6]
+  CRUSH rule 3 x 593 [2,8,5]
+  CRUSH rule 3 x 594 [2,6,5]
+  CRUSH rule 3 x 595 [7,2,5]
+  CRUSH rule 3 x 596 [5,2,6]
+  CRUSH rule 3 x 597 [5,2,7]
+  CRUSH rule 3 x 598 [5,2,7]
+  CRUSH rule 3 x 599 [5,2,6]
+  CRUSH rule 3 x 600 [7,2,5]
+  CRUSH rule 3 x 601 [2,6,5]
+  CRUSH rule 3 x 602 [5,8,2]
+  CRUSH rule 3 x 603 [5,2,8]
+  CRUSH rule 3 x 604 [7,5,2]
+  CRUSH rule 3 x 605 [5,2,6]
+  CRUSH rule 3 x 606 [2,7,5]
+  CRUSH rule 3 x 607 [2,5,8]
+  CRUSH rule 3 x 608 [5,2,8]
+  CRUSH rule 3 x 609 [5,2,6]
+  CRUSH rule 3 x 610 [5,8,2]
+  CRUSH rule 3 x 611 [2,8,5]
+  CRUSH rule 3 x 612 [2,8,5]
+  CRUSH rule 3 x 613 [7,2,5]
+  CRUSH rule 3 x 614 [7,2,5]
+  CRUSH rule 3 x 615 [6,2,5]
+  CRUSH rule 3 x 616 [2,7,5]
+  CRUSH rule 3 x 617 [6,2,5]
+  CRUSH rule 3 x 618 [7,5,2]
+  CRUSH rule 3 x 619 [5,2,6]
+  CRUSH rule 3 x 620 [5,2,8]
+  CRUSH rule 3 x 621 [5,6,2]
+  CRUSH rule 3 x 622 [2,5,8]
+  CRUSH rule 3 x 623 [2,8,5]
+  CRUSH rule 3 x 624 [5,2,8]
+  CRUSH rule 3 x 625 [2,5,8]
+  CRUSH rule 3 x 626 [7,2,5]
+  CRUSH rule 3 x 627 [2,6,5]
+  CRUSH rule 3 x 628 [8,2,5]
+  CRUSH rule 3 x 629 [2,6,5]
+  CRUSH rule 3 x 630 [2,6,5]
+  CRUSH rule 3 x 631 [2,6,5]
+  CRUSH rule 3 x 632 [7,2,5]
+  CRUSH rule 3 x 633 [8,5,2]
+  CRUSH rule 3 x 634 [2,5,6]
+  CRUSH rule 3 x 635 [5,6,2]
+  CRUSH rule 3 x 636 [2,5,7]
+  CRUSH rule 3 x 637 [5,2,8]
+  CRUSH rule 3 x 638 [6,2,5]
+  CRUSH rule 3 x 639 [5,2,6]
+  CRUSH rule 3 x 640 [5,2,7]
+  CRUSH rule 3 x 641 [7,2,5]
+  CRUSH rule 3 x 642 [2,8,5]
+  CRUSH rule 3 x 643 [5,2,8]
+  CRUSH rule 3 x 644 [8,2,5]
+  CRUSH rule 3 x 645 [5,7,2]
+  CRUSH rule 3 x 646 [8,2,5]
+  CRUSH rule 3 x 647 [7,2,5]
+  CRUSH rule 3 x 648 [2,8,5]
+  CRUSH rule 3 x 649 [5,7,2]
+  CRUSH rule 3 x 650 [7,5,2]
+  CRUSH rule 3 x 651 [5,6,2]
+  CRUSH rule 3 x 652 [5,6,2]
+  CRUSH rule 3 x 653 [8,5,2]
+  CRUSH rule 3 x 654 [7,5,2]
+  CRUSH rule 3 x 655 [2,5,6]
+  CRUSH rule 3 x 656 [5,7,2]
+  CRUSH rule 3 x 657 [6,2,5]
+  CRUSH rule 3 x 658 [5,8,2]
+  CRUSH rule 3 x 659 [5,7,2]
+  CRUSH rule 3 x 660 [7,5,2]
+  CRUSH rule 3 x 661 [2,7,5]
+  CRUSH rule 3 x 662 [5,2,8]
+  CRUSH rule 3 x 663 [2,5,7]
+  CRUSH rule 3 x 664 [2,5,6]
+  CRUSH rule 3 x 665 [5,6,2]
+  CRUSH rule 3 x 666 [2,7,5]
+  CRUSH rule 3 x 667 [2,5,8]
+  CRUSH rule 3 x 668 [5,7,2]
+  CRUSH rule 3 x 669 [6,5,2]
+  CRUSH rule 3 x 670 [5,2,6]
+  CRUSH rule 3 x 671 [2,8,5]
+  CRUSH rule 3 x 672 [5,2,8]
+  CRUSH rule 3 x 673 [5,2,7]
+  CRUSH rule 3 x 674 [5,2,7]
+  CRUSH rule 3 x 675 [2,8,5]
+  CRUSH rule 3 x 676 [2,5,7]
+  CRUSH rule 3 x 677 [5,2,6]
+  CRUSH rule 3 x 678 [2,5,8]
+  CRUSH rule 3 x 679 [6,2,5]
+  CRUSH rule 3 x 680 [2,5,8]
+  CRUSH rule 3 x 681 [5,6,2]
+  CRUSH rule 3 x 682 [2,5,8]
+  CRUSH rule 3 x 683 [2,5,8]
+  CRUSH rule 3 x 684 [7,2,5]
+  CRUSH rule 3 x 685 [7,2,5]
+  CRUSH rule 3 x 686 [2,5,8]
+  CRUSH rule 3 x 687 [5,7,2]
+  CRUSH rule 3 x 688 [5,6,2]
+  CRUSH rule 3 x 689 [6,5,2]
+  CRUSH rule 3 x 690 [8,2,5]
+  CRUSH rule 3 x 691 [5,2,6]
+  CRUSH rule 3 x 692 [7,2,5]
+  CRUSH rule 3 x 693 [6,5,2]
+  CRUSH rule 3 x 694 [6,5,2]
+  CRUSH rule 3 x 695 [2,6,5]
+  CRUSH rule 3 x 696 [2,5,8]
+  CRUSH rule 3 x 697 [6,2,5]
+  CRUSH rule 3 x 698 [6,2,5]
+  CRUSH rule 3 x 699 [2,8,5]
+  CRUSH rule 3 x 700 [2,5,6]
+  CRUSH rule 3 x 701 [5,2,6]
+  CRUSH rule 3 x 702 [5,2,8]
+  CRUSH rule 3 x 703 [8,5,2]
+  CRUSH rule 3 x 704 [2,5,6]
+  CRUSH rule 3 x 705 [8,2,5]
+  CRUSH rule 3 x 706 [2,5,8]
+  CRUSH rule 3 x 707 [7,5,2]
+  CRUSH rule 3 x 708 [5,7,2]
+  CRUSH rule 3 x 709 [6,5,2]
+  CRUSH rule 3 x 710 [8,5,2]
+  CRUSH rule 3 x 711 [2,5,7]
+  CRUSH rule 3 x 712 [2,5,6]
+  CRUSH rule 3 x 713 [6,5,2]
+  CRUSH rule 3 x 714 [5,2,6]
+  CRUSH rule 3 x 715 [2,5,6]
+  CRUSH rule 3 x 716 [5,6,2]
+  CRUSH rule 3 x 717 [8,2,5]
+  CRUSH rule 3 x 718 [5,7,2]
+  CRUSH rule 3 x 719 [2,6,5]
+  CRUSH rule 3 x 720 [6,2,5]
+  CRUSH rule 3 x 721 [5,7,2]
+  CRUSH rule 3 x 722 [5,7,2]
+  CRUSH rule 3 x 723 [5,2,7]
+  CRUSH rule 3 x 724 [2,7,5]
+  CRUSH rule 3 x 725 [2,5,6]
+  CRUSH rule 3 x 726 [5,7,2]
+  CRUSH rule 3 x 727 [5,7,2]
+  CRUSH rule 3 x 728 [2,6,5]
+  CRUSH rule 3 x 729 [5,6,2]
+  CRUSH rule 3 x 730 [5,8,2]
+  CRUSH rule 3 x 731 [5,2,6]
+  CRUSH rule 3 x 732 [2,5,8]
+  CRUSH rule 3 x 733 [5,6,2]
+  CRUSH rule 3 x 734 [6,5,2]
+  CRUSH rule 3 x 735 [5,6,2]
+  CRUSH rule 3 x 736 [5,8,2]
+  CRUSH rule 3 x 737 [2,8,5]
+  CRUSH rule 3 x 738 [5,2,6]
+  CRUSH rule 3 x 739 [2,7,5]
+  CRUSH rule 3 x 740 [2,7,5]
+  CRUSH rule 3 x 741 [7,2,5]
+  CRUSH rule 3 x 742 [8,2,5]
+  CRUSH rule 3 x 743 [7,2,5]
+  CRUSH rule 3 x 744 [5,6,2]
+  CRUSH rule 3 x 745 [5,2,8]
+  CRUSH rule 3 x 746 [5,2,7]
+  CRUSH rule 3 x 747 [6,2,5]
+  CRUSH rule 3 x 748 [2,6,5]
+  CRUSH rule 3 x 749 [5,7,2]
+  CRUSH rule 3 x 750 [2,7,5]
+  CRUSH rule 3 x 751 [2,7,5]
+  CRUSH rule 3 x 752 [8,2,5]
+  CRUSH rule 3 x 753 [7,5,2]
+  CRUSH rule 3 x 754 [8,5,2]
+  CRUSH rule 3 x 755 [2,6,5]
+  CRUSH rule 3 x 756 [5,8,2]
+  CRUSH rule 3 x 757 [8,2,5]
+  CRUSH rule 3 x 758 [6,2,5]
+  CRUSH rule 3 x 759 [8,5,2]
+  CRUSH rule 3 x 760 [2,5,7]
+  CRUSH rule 3 x 761 [5,2,6]
+  CRUSH rule 3 x 762 [2,8,5]
+  CRUSH rule 3 x 763 [8,5,2]
+  CRUSH rule 3 x 764 [2,7,5]
+  CRUSH rule 3 x 765 [6,5,2]
+  CRUSH rule 3 x 766 [8,5,2]
+  CRUSH rule 3 x 767 [2,8,5]
+  CRUSH rule 3 x 768 [8,5,2]
+  CRUSH rule 3 x 769 [6,2,5]
+  CRUSH rule 3 x 770 [6,2,5]
+  CRUSH rule 3 x 771 [7,2,5]
+  CRUSH rule 3 x 772 [8,5,2]
+  CRUSH rule 3 x 773 [5,2,6]
+  CRUSH rule 3 x 774 [5,7,2]
+  CRUSH rule 3 x 775 [6,5,2]
+  CRUSH rule 3 x 776 [7,2,5]
+  CRUSH rule 3 x 777 [5,2,6]
+  CRUSH rule 3 x 778 [2,6,5]
+  CRUSH rule 3 x 779 [2,6,5]
+  CRUSH rule 3 x 780 [2,5,8]
+  CRUSH rule 3 x 781 [6,5,2]
+  CRUSH rule 3 x 782 [5,2,8]
+  CRUSH rule 3 x 783 [7,2,5]
+  CRUSH rule 3 x 784 [2,5,8]
+  CRUSH rule 3 x 785 [6,2,5]
+  CRUSH rule 3 x 786 [7,5,2]
+  CRUSH rule 3 x 787 [2,8,5]
+  CRUSH rule 3 x 788 [6,2,5]
+  CRUSH rule 3 x 789 [2,5,8]
+  CRUSH rule 3 x 790 [8,5,2]
+  CRUSH rule 3 x 791 [5,6,2]
+  CRUSH rule 3 x 792 [5,6,2]
+  CRUSH rule 3 x 793 [6,2,5]
+  CRUSH rule 3 x 794 [2,8,5]
+  CRUSH rule 3 x 795 [2,5,6]
+  CRUSH rule 3 x 796 [5,7,2]
+  CRUSH rule 3 x 797 [2,5,6]
+  CRUSH rule 3 x 798 [6,2,5]
+  CRUSH rule 3 x 799 [5,2,7]
+  CRUSH rule 3 x 800 [5,2,8]
+  CRUSH rule 3 x 801 [5,7,2]
+  CRUSH rule 3 x 802 [2,6,5]
+  CRUSH rule 3 x 803 [2,5,7]
+  CRUSH rule 3 x 804 [6,2,5]
+  CRUSH rule 3 x 805 [5,8,2]
+  CRUSH rule 3 x 806 [2,5,6]
+  CRUSH rule 3 x 807 [5,7,2]
+  CRUSH rule 3 x 808 [5,7,2]
+  CRUSH rule 3 x 809 [2,5,6]
+  CRUSH rule 3 x 810 [5,7,2]
+  CRUSH rule 3 x 811 [8,5,2]
+  CRUSH rule 3 x 812 [8,5,2]
+  CRUSH rule 3 x 813 [6,5,2]
+  CRUSH rule 3 x 814 [5,8,2]
+  CRUSH rule 3 x 815 [5,2,6]
+  CRUSH rule 3 x 816 [2,6,5]
+  CRUSH rule 3 x 817 [5,6,2]
+  CRUSH rule 3 x 818 [5,2,7]
+  CRUSH rule 3 x 819 [5,2,7]
+  CRUSH rule 3 x 820 [5,7,2]
+  CRUSH rule 3 x 821 [5,8,2]
+  CRUSH rule 3 x 822 [2,5,7]
+  CRUSH rule 3 x 823 [5,7,2]
+  CRUSH rule 3 x 824 [5,7,2]
+  CRUSH rule 3 x 825 [2,8,5]
+  CRUSH rule 3 x 826 [7,2,5]
+  CRUSH rule 3 x 827 [2,6,5]
+  CRUSH rule 3 x 828 [2,5,8]
+  CRUSH rule 3 x 829 [5,6,2]
+  CRUSH rule 3 x 830 [2,5,7]
+  CRUSH rule 3 x 831 [2,6,5]
+  CRUSH rule 3 x 832 [5,8,2]
+  CRUSH rule 3 x 833 [2,6,5]
+  CRUSH rule 3 x 834 [5,2,8]
+  CRUSH rule 3 x 835 [8,5,2]
+  CRUSH rule 3 x 836 [5,8,2]
+  CRUSH rule 3 x 837 [6,5,2]
+  CRUSH rule 3 x 838 [6,2,5]
+  CRUSH rule 3 x 839 [5,2,8]
+  CRUSH rule 3 x 840 [7,5,2]
+  CRUSH rule 3 x 841 [5,8,2]
+  CRUSH rule 3 x 842 [2,5,8]
+  CRUSH rule 3 x 843 [6,5,2]
+  CRUSH rule 3 x 844 [5,8,2]
+  CRUSH rule 3 x 845 [5,6,2]
+  CRUSH rule 3 x 846 [5,2,6]
+  CRUSH rule 3 x 847 [2,8,5]
+  CRUSH rule 3 x 848 [2,6,5]
+  CRUSH rule 3 x 849 [5,8,2]
+  CRUSH rule 3 x 850 [2,5,7]
+  CRUSH rule 3 x 851 [6,5,2]
+  CRUSH rule 3 x 852 [7,5,2]
+  CRUSH rule 3 x 853 [6,2,5]
+  CRUSH rule 3 x 854 [7,2,5]
+  CRUSH rule 3 x 855 [5,7,2]
+  CRUSH rule 3 x 856 [6,5,2]
+  CRUSH rule 3 x 857 [8,5,2]
+  CRUSH rule 3 x 858 [6,5,2]
+  CRUSH rule 3 x 859 [6,2,5]
+  CRUSH rule 3 x 860 [5,2,8]
+  CRUSH rule 3 x 861 [8,5,2]
+  CRUSH rule 3 x 862 [6,2,5]
+  CRUSH rule 3 x 863 [8,2,5]
+  CRUSH rule 3 x 864 [5,6,2]
+  CRUSH rule 3 x 865 [8,2,5]
+  CRUSH rule 3 x 866 [5,6,2]
+  CRUSH rule 3 x 867 [6,5,2]
+  CRUSH rule 3 x 868 [6,5,2]
+  CRUSH rule 3 x 869 [8,5,2]
+  CRUSH rule 3 x 870 [2,5,7]
+  CRUSH rule 3 x 871 [5,2,6]
+  CRUSH rule 3 x 872 [5,2,6]
+  CRUSH rule 3 x 873 [5,6,2]
+  CRUSH rule 3 x 874 [2,6,5]
+  CRUSH rule 3 x 875 [2,6,5]
+  CRUSH rule 3 x 876 [5,8,2]
+  CRUSH rule 3 x 877 [6,5,2]
+  CRUSH rule 3 x 878 [5,2,7]
+  CRUSH rule 3 x 879 [7,5,2]
+  CRUSH rule 3 x 880 [5,2,7]
+  CRUSH rule 3 x 881 [5,8,2]
+  CRUSH rule 3 x 882 [5,2,6]
+  CRUSH rule 3 x 883 [2,5,6]
+  CRUSH rule 3 x 884 [6,2,5]
+  CRUSH rule 3 x 885 [5,2,6]
+  CRUSH rule 3 x 886 [5,7,2]
+  CRUSH rule 3 x 887 [7,5,2]
+  CRUSH rule 3 x 888 [6,2,5]
+  CRUSH rule 3 x 889 [2,8,5]
+  CRUSH rule 3 x 890 [7,2,5]
+  CRUSH rule 3 x 891 [2,7,5]
+  CRUSH rule 3 x 892 [6,2,5]
+  CRUSH rule 3 x 893 [2,5,6]
+  CRUSH rule 3 x 894 [7,5,2]
+  CRUSH rule 3 x 895 [5,2,6]
+  CRUSH rule 3 x 896 [2,6,5]
+  CRUSH rule 3 x 897 [5,2,8]
+  CRUSH rule 3 x 898 [2,5,8]
+  CRUSH rule 3 x 899 [2,6,5]
+  CRUSH rule 3 x 900 [5,2,7]
+  CRUSH rule 3 x 901 [5,2,6]
+  CRUSH rule 3 x 902 [8,5,2]
+  CRUSH rule 3 x 903 [5,6,2]
+  CRUSH rule 3 x 904 [5,7,2]
+  CRUSH rule 3 x 905 [6,2,5]
+  CRUSH rule 3 x 906 [2,7,5]
+  CRUSH rule 3 x 907 [7,2,5]
+  CRUSH rule 3 x 908 [5,6,2]
+  CRUSH rule 3 x 909 [2,5,8]
+  CRUSH rule 3 x 910 [6,5,2]
+  CRUSH rule 3 x 911 [5,7,2]
+  CRUSH rule 3 x 912 [2,8,5]
+  CRUSH rule 3 x 913 [7,2,5]
+  CRUSH rule 3 x 914 [6,5,2]
+  CRUSH rule 3 x 915 [8,2,5]
+  CRUSH rule 3 x 916 [5,2,6]
+  CRUSH rule 3 x 917 [2,5,6]
+  CRUSH rule 3 x 918 [8,2,5]
+  CRUSH rule 3 x 919 [6,2,5]
+  CRUSH rule 3 x 920 [7,5,2]
+  CRUSH rule 3 x 921 [2,5,8]
+  CRUSH rule 3 x 922 [6,5,2]
+  CRUSH rule 3 x 923 [5,6,2]
+  CRUSH rule 3 x 924 [5,2,8]
+  CRUSH rule 3 x 925 [5,6,2]
+  CRUSH rule 3 x 926 [5,2,7]
+  CRUSH rule 3 x 927 [2,6,5]
+  CRUSH rule 3 x 928 [8,2,5]
+  CRUSH rule 3 x 929 [5,2,6]
+  CRUSH rule 3 x 930 [2,5,8]
+  CRUSH rule 3 x 931 [5,2,8]
+  CRUSH rule 3 x 932 [5,2,7]
+  CRUSH rule 3 x 933 [8,5,2]
+  CRUSH rule 3 x 934 [5,8,2]
+  CRUSH rule 3 x 935 [6,5,2]
+  CRUSH rule 3 x 936 [2,7,5]
+  CRUSH rule 3 x 937 [5,8,2]
+  CRUSH rule 3 x 938 [6,5,2]
+  CRUSH rule 3 x 939 [2,8,5]
+  CRUSH rule 3 x 940 [8,5,2]
+  CRUSH rule 3 x 941 [5,2,6]
+  CRUSH rule 3 x 942 [2,6,5]
+  CRUSH rule 3 x 943 [8,2,5]
+  CRUSH rule 3 x 944 [5,8,2]
+  CRUSH rule 3 x 945 [7,2,5]
+  CRUSH rule 3 x 946 [2,8,5]
+  CRUSH rule 3 x 947 [5,2,6]
+  CRUSH rule 3 x 948 [7,5,2]
+  CRUSH rule 3 x 949 [6,2,5]
+  CRUSH rule 3 x 950 [5,7,2]
+  CRUSH rule 3 x 951 [5,6,2]
+  CRUSH rule 3 x 952 [2,7,5]
+  CRUSH rule 3 x 953 [2,5,6]
+  CRUSH rule 3 x 954 [5,2,8]
+  CRUSH rule 3 x 955 [8,2,5]
+  CRUSH rule 3 x 956 [2,7,5]
+  CRUSH rule 3 x 957 [7,2,5]
+  CRUSH rule 3 x 958 [8,5,2]
+  CRUSH rule 3 x 959 [5,2,6]
+  CRUSH rule 3 x 960 [5,6,2]
+  CRUSH rule 3 x 961 [5,2,6]
+  CRUSH rule 3 x 962 [7,5,2]
+  CRUSH rule 3 x 963 [2,5,8]
+  CRUSH rule 3 x 964 [5,2,7]
+  CRUSH rule 3 x 965 [7,5,2]
+  CRUSH rule 3 x 966 [5,6,2]
+  CRUSH rule 3 x 967 [8,5,2]
+  CRUSH rule 3 x 968 [7,2,5]
+  CRUSH rule 3 x 969 [8,2,5]
+  CRUSH rule 3 x 970 [2,8,5]
+  CRUSH rule 3 x 971 [2,8,5]
+  CRUSH rule 3 x 972 [2,7,5]
+  CRUSH rule 3 x 973 [2,8,5]
+  CRUSH rule 3 x 974 [5,2,7]
+  CRUSH rule 3 x 975 [5,8,2]
+  CRUSH rule 3 x 976 [5,7,2]
+  CRUSH rule 3 x 977 [8,5,2]
+  CRUSH rule 3 x 978 [7,2,5]
+  CRUSH rule 3 x 979 [7,2,5]
+  CRUSH rule 3 x 980 [6,2,5]
+  CRUSH rule 3 x 981 [7,5,2]
+  CRUSH rule 3 x 982 [5,2,8]
+  CRUSH rule 3 x 983 [5,6,2]
+  CRUSH rule 3 x 984 [2,8,5]
+  CRUSH rule 3 x 985 [2,5,8]
+  CRUSH rule 3 x 986 [8,5,2]
+  CRUSH rule 3 x 987 [2,5,8]
+  CRUSH rule 3 x 988 [2,5,6]
+  CRUSH rule 3 x 989 [2,8,5]
+  CRUSH rule 3 x 990 [2,6,5]
+  CRUSH rule 3 x 991 [2,5,8]
+  CRUSH rule 3 x 992 [7,2,5]
+  CRUSH rule 3 x 993 [2,6,5]
+  CRUSH rule 3 x 994 [5,6,2]
+  CRUSH rule 3 x 995 [7,2,5]
+  CRUSH rule 3 x 996 [6,5,2]
+  CRUSH rule 3 x 997 [6,5,2]
+  CRUSH rule 3 x 998 [8,2,5]
+  CRUSH rule 3 x 999 [2,7,5]
+  CRUSH rule 3 x 1000 [8,5,2]
+  CRUSH rule 3 x 1001 [2,5,6]
+  CRUSH rule 3 x 1002 [2,5,8]
+  CRUSH rule 3 x 1003 [2,7,5]
+  CRUSH rule 3 x 1004 [6,2,5]
+  CRUSH rule 3 x 1005 [6,2,5]
+  CRUSH rule 3 x 1006 [2,8,5]
+  CRUSH rule 3 x 1007 [2,5,8]
+  CRUSH rule 3 x 1008 [2,7,5]
+  CRUSH rule 3 x 1009 [6,5,2]
+  CRUSH rule 3 x 1010 [5,2,6]
+  CRUSH rule 3 x 1011 [5,2,8]
+  CRUSH rule 3 x 1012 [5,2,8]
+  CRUSH rule 3 x 1013 [5,2,8]
+  CRUSH rule 3 x 1014 [2,8,5]
+  CRUSH rule 3 x 1015 [6,5,2]
+  CRUSH rule 3 x 1016 [2,5,6]
+  CRUSH rule 3 x 1017 [6,2,5]
+  CRUSH rule 3 x 1018 [5,2,6]
+  CRUSH rule 3 x 1019 [5,8,2]
+  CRUSH rule 3 x 1020 [5,2,7]
+  CRUSH rule 3 x 1021 [5,2,7]
+  CRUSH rule 3 x 1022 [2,7,5]
+  CRUSH rule 3 x 1023 [5,2,7]
+  rule 3 (choose-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+  rule 4 (choose-set-two), x = 0..1023, numrep = 2..3
+  CRUSH rule 4 x 0 [2]
+  CRUSH rule 4 x 1 [2,8]
+  CRUSH rule 4 x 2 [2,5]
+  CRUSH rule 4 x 3 [8,2]
+  CRUSH rule 4 x 4 [5]
+  CRUSH rule 4 x 5 [7,8]
+  CRUSH rule 4 x 6 [2,6]
+  CRUSH rule 4 x 7 [5]
+  CRUSH rule 4 x 8 [5]
+  CRUSH rule 4 x 9 [2,5]
+  CRUSH rule 4 x 10 [2]
+  CRUSH rule 4 x 11 [2,7]
+  CRUSH rule 4 x 12 [2]
+  CRUSH rule 4 x 13 [5,8]
+  CRUSH rule 4 x 14 [7,6]
+  CRUSH rule 4 x 15 [7,2]
+  CRUSH rule 4 x 16 [5,6]
+  CRUSH rule 4 x 17 [5]
+  CRUSH rule 4 x 18 [2,5]
+  CRUSH rule 4 x 19 [7,5]
+  CRUSH rule 4 x 20 [2,5]
+  CRUSH rule 4 x 21 [5,7]
+  CRUSH rule 4 x 22 [8,5]
+  CRUSH rule 4 x 23 [5,6]
+  CRUSH rule 4 x 24 [2]
+  CRUSH rule 4 x 25 [5,7]
+  CRUSH rule 4 x 26 [2,8]
+  CRUSH rule 4 x 27 [5,2]
+  CRUSH rule 4 x 28 [6,2]
+  CRUSH rule 4 x 29 [8,5]
+  CRUSH rule 4 x 30 [5,7]
+  CRUSH rule 4 x 31 [8,7]
+  CRUSH rule 4 x 32 [5,6]
+  CRUSH rule 4 x 33 [2,7]
+  CRUSH rule 4 x 34 [2,5]
+  CRUSH rule 4 x 35 [2,8]
+  CRUSH rule 4 x 36 [5,8]
+  CRUSH rule 4 x 37 [2,5]
+  CRUSH rule 4 x 38 [5,8]
+  CRUSH rule 4 x 39 [5,7]
+  CRUSH rule 4 x 40 [7,8]
+  CRUSH rule 4 x 41 [2]
+  CRUSH rule 4 x 42 [5]
+  CRUSH rule 4 x 43 [2,5]
+  CRUSH rule 4 x 44 [2,6]
+  CRUSH rule 4 x 45 [8,2]
+  CRUSH rule 4 x 46 [2,5]
+  CRUSH rule 4 x 47 [5]
+  CRUSH rule 4 x 48 [5,6]
+  CRUSH rule 4 x 49 [5]
+  CRUSH rule 4 x 50 [5]
+  CRUSH rule 4 x 51 [5,6]
+  CRUSH rule 4 x 52 [8,6]
+  CRUSH rule 4 x 53 [5]
+  CRUSH rule 4 x 54 [7,6]
+  CRUSH rule 4 x 55 [8,7]
+  CRUSH rule 4 x 56 [6,5]
+  CRUSH rule 4 x 57 [5]
+  CRUSH rule 4 x 58 [2]
+  CRUSH rule 4 x 59 [5,2]
+  CRUSH rule 4 x 60 [5]
+  CRUSH rule 4 x 61 [5,6]
+  CRUSH rule 4 x 62 [7,2]
+  CRUSH rule 4 x 63 [5,6]
+  CRUSH rule 4 x 64 [5]
+  CRUSH rule 4 x 65 [7,5]
+  CRUSH rule 4 x 66 [5]
+  CRUSH rule 4 x 67 [5,2]
+  CRUSH rule 4 x 68 [2,5]
+  CRUSH rule 4 x 69 [5,2]
+  CRUSH rule 4 x 70 [7,2]
+  CRUSH rule 4 x 71 [2,8]
+  CRUSH rule 4 x 72 [6,2]
+  CRUSH rule 4 x 73 [2,7]
+  CRUSH rule 4 x 74 [2,7]
+  CRUSH rule 4 x 75 [5,2]
+  CRUSH rule 4 x 76 [5,2]
+  CRUSH rule 4 x 77 [7,2]
+  CRUSH rule 4 x 78 [2,5]
+  CRUSH rule 4 x 79 [5]
+  CRUSH rule 4 x 80 [2,5]
+  CRUSH rule 4 x 81 [2]
+  CRUSH rule 4 x 82 [7,2]
+  CRUSH rule 4 x 83 [2,6]
+  CRUSH rule 4 x 84 [7,2]
+  CRUSH rule 4 x 85 [5]
+  CRUSH rule 4 x 86 [2]
+  CRUSH rule 4 x 87 [2,7]
+  CRUSH rule 4 x 88 [2,6]
+  CRUSH rule 4 x 89 [5,2]
+  CRUSH rule 4 x 90 [6,7]
+  CRUSH rule 4 x 91 [5,8]
+  CRUSH rule 4 x 92 [2,8]
+  CRUSH rule 4 x 93 [7,5]
+  CRUSH rule 4 x 94 [2,5]
+  CRUSH rule 4 x 95 [7,5]
+  CRUSH rule 4 x 96 [5,6]
+  CRUSH rule 4 x 97 [8,7]
+  CRUSH rule 4 x 98 [2]
+  CRUSH rule 4 x 99 [2,7]
+  CRUSH rule 4 x 100 [2,7]
+  CRUSH rule 4 x 101 [5,7]
+  CRUSH rule 4 x 102 [5,2]
+  CRUSH rule 4 x 103 [5,7]
+  CRUSH rule 4 x 104 [7,5]
+  CRUSH rule 4 x 105 [2,5]
+  CRUSH rule 4 x 106 [2,6]
+  CRUSH rule 4 x 107 [5,2]
+  CRUSH rule 4 x 108 [7,2]
+  CRUSH rule 4 x 109 [2]
+  CRUSH rule 4 x 110 [5,2]
+  CRUSH rule 4 x 111 [2]
+  CRUSH rule 4 x 112 [2]
+  CRUSH rule 4 x 113 [6,2]
+  CRUSH rule 4 x 114 [7,6]
+  CRUSH rule 4 x 115 [8,2]
+  CRUSH rule 4 x 116 [2]
+  CRUSH rule 4 x 117 [7,5]
+  CRUSH rule 4 x 118 [2,5]
+  CRUSH rule 4 x 119 [5,6]
+  CRUSH rule 4 x 120 [2]
+  CRUSH rule 4 x 121 [2]
+  CRUSH rule 4 x 122 [8,5]
+  CRUSH rule 4 x 123 [2,5]
+  CRUSH rule 4 x 124 [5]
+  CRUSH rule 4 x 125 [2,7]
+  CRUSH rule 4 x 126 [5,2]
+  CRUSH rule 4 x 127 [5,6]
+  CRUSH rule 4 x 128 [5]
+  CRUSH rule 4 x 129 [2]
+  CRUSH rule 4 x 130 [5,8]
+  CRUSH rule 4 x 131 [2]
+  CRUSH rule 4 x 132 [2]
+  CRUSH rule 4 x 133 [5,6]
+  CRUSH rule 4 x 134 [2,8]
+  CRUSH rule 4 x 135 [5,6]
+  CRUSH rule 4 x 136 [2]
+  CRUSH rule 4 x 137 [7,5]
+  CRUSH rule 4 x 138 [8,7]
+  CRUSH rule 4 x 139 [5,2]
+  CRUSH rule 4 x 140 [2,6]
+  CRUSH rule 4 x 141 [6,8]
+  CRUSH rule 4 x 142 [5]
+  CRUSH rule 4 x 143 [5,8]
+  CRUSH rule 4 x 144 [8,2]
+  CRUSH rule 4 x 145 [8,5]
+  CRUSH rule 4 x 146 [2,6]
+  CRUSH rule 4 x 147 [2,8]
+  CRUSH rule 4 x 148 [5]
+  CRUSH rule 4 x 149 [5,8]
+  CRUSH rule 4 x 150 [2,6]
+  CRUSH rule 4 x 151 [5]
+  CRUSH rule 4 x 152 [8,5]
+  CRUSH rule 4 x 153 [8,6]
+  CRUSH rule 4 x 154 [5,2]
+  CRUSH rule 4 x 155 [5]
+  CRUSH rule 4 x 156 [5]
+  CRUSH rule 4 x 157 [5,2]
+  CRUSH rule 4 x 158 [2,8]
+  CRUSH rule 4 x 159 [7,2]
+  CRUSH rule 4 x 160 [2,8]
+  CRUSH rule 4 x 161 [2,5]
+  CRUSH rule 4 x 162 [2,6]
+  CRUSH rule 4 x 163 [5,6]
+  CRUSH rule 4 x 164 [7,8]
+  CRUSH rule 4 x 165 [7,2]
+  CRUSH rule 4 x 166 [2,5]
+  CRUSH rule 4 x 167 [2]
+  CRUSH rule 4 x 168 [5,2]
+  CRUSH rule 4 x 169 [2,6]
+  CRUSH rule 4 x 170 [2]
+  CRUSH rule 4 x 171 [7,5]
+  CRUSH rule 4 x 172 [2,7]
+  CRUSH rule 4 x 173 [8,5]
+  CRUSH rule 4 x 174 [2,5]
+  CRUSH rule 4 x 175 [6,2]
+  CRUSH rule 4 x 176 [5]
+  CRUSH rule 4 x 177 [5]
+  CRUSH rule 4 x 178 [5,2]
+  CRUSH rule 4 x 179 [5,2]
+  CRUSH rule 4 x 180 [5,8]
+  CRUSH rule 4 x 181 [6,2]
+  CRUSH rule 4 x 182 [8,5]
+  CRUSH rule 4 x 183 [7,8]
+  CRUSH rule 4 x 184 [5,7]
+  CRUSH rule 4 x 185 [6,8]
+  CRUSH rule 4 x 186 [2]
+  CRUSH rule 4 x 187 [2,6]
+  CRUSH rule 4 x 188 [2,8]
+  CRUSH rule 4 x 189 [2,7]
+  CRUSH rule 4 x 190 [5,2]
+  CRUSH rule 4 x 191 [7,6]
+  CRUSH rule 4 x 192 [5,2]
+  CRUSH rule 4 x 193 [5,2]
+  CRUSH rule 4 x 194 [2,5]
+  CRUSH rule 4 x 195 [6,5]
+  CRUSH rule 4 x 196 [6,7]
+  CRUSH rule 4 x 197 [6,5]
+  CRUSH rule 4 x 198 [2,5]
+  CRUSH rule 4 x 199 [2,5]
+  CRUSH rule 4 x 200 [2,5]
+  CRUSH rule 4 x 201 [7,2]
+  CRUSH rule 4 x 202 [6,5]
+  CRUSH rule 4 x 203 [5,8]
+  CRUSH rule 4 x 204 [2]
+  CRUSH rule 4 x 205 [2,7]
+  CRUSH rule 4 x 206 [2]
+  CRUSH rule 4 x 207 [5,2]
+  CRUSH rule 4 x 208 [7,2]
+  CRUSH rule 4 x 209 [2]
+  CRUSH rule 4 x 210 [2]
+  CRUSH rule 4 x 211 [5]
+  CRUSH rule 4 x 212 [7,5]
+  CRUSH rule 4 x 213 [8,5]
+  CRUSH rule 4 x 214 [5]
+  CRUSH rule 4 x 215 [8,2]
+  CRUSH rule 4 x 216 [5,2]
+  CRUSH rule 4 x 217 [2]
+  CRUSH rule 4 x 218 [2,7]
+  CRUSH rule 4 x 219 [5,8]
+  CRUSH rule 4 x 220 [5,7]
+  CRUSH rule 4 x 221 [5,6]
+  CRUSH rule 4 x 222 [6,8]
+  CRUSH rule 4 x 223 [2,5]
+  CRUSH rule 4 x 224 [2,5]
+  CRUSH rule 4 x 225 [8,6]
+  CRUSH rule 4 x 226 [7,2]
+  CRUSH rule 4 x 227 [5]
+  CRUSH rule 4 x 228 [5]
+  CRUSH rule 4 x 229 [5]
+  CRUSH rule 4 x 230 [5,7]
+  CRUSH rule 4 x 231 [5]
+  CRUSH rule 4 x 232 [2,7]
+  CRUSH rule 4 x 233 [5]
+  CRUSH rule 4 x 234 [2]
+  CRUSH rule 4 x 235 [5,8]
+  CRUSH rule 4 x 236 [5,2]
+  CRUSH rule 4 x 237 [5,7]
+  CRUSH rule 4 x 238 [5]
+  CRUSH rule 4 x 239 [8,7]
+  CRUSH rule 4 x 240 [5,7]
+  CRUSH rule 4 x 241 [5,2]
+  CRUSH rule 4 x 242 [5]
+  CRUSH rule 4 x 243 [5,7]
+  CRUSH rule 4 x 244 [5,6]
+  CRUSH rule 4 x 245 [7,6]
+  CRUSH rule 4 x 246 [2,5]
+  CRUSH rule 4 x 247 [6,2]
+  CRUSH rule 4 x 248 [8,2]
+  CRUSH rule 4 x 249 [2]
+  CRUSH rule 4 x 250 [2]
+  CRUSH rule 4 x 251 [2,5]
+  CRUSH rule 4 x 252 [5,7]
+  CRUSH rule 4 x 253 [5,2]
+  CRUSH rule 4 x 254 [5]
+  CRUSH rule 4 x 255 [2,7]
+  CRUSH rule 4 x 256 [5,7]
+  CRUSH rule 4 x 257 [2,8]
+  CRUSH rule 4 x 258 [5]
+  CRUSH rule 4 x 259 [5]
+  CRUSH rule 4 x 260 [5,6]
+  CRUSH rule 4 x 261 [8,7]
+  CRUSH rule 4 x 262 [5]
+  CRUSH rule 4 x 263 [6,8]
+  CRUSH rule 4 x 264 [5,6]
+  CRUSH rule 4 x 265 [8,6]
+  CRUSH rule 4 x 266 [8,2]
+  CRUSH rule 4 x 267 [2,5]
+  CRUSH rule 4 x 268 [2,7]
+  CRUSH rule 4 x 269 [2,8]
+  CRUSH rule 4 x 270 [5,2]
+  CRUSH rule 4 x 271 [7,5]
+  CRUSH rule 4 x 272 [2,8]
+  CRUSH rule 4 x 273 [5]
+  CRUSH rule 4 x 274 [6,8]
+  CRUSH rule 4 x 275 [5]
+  CRUSH rule 4 x 276 [7,2]
+  CRUSH rule 4 x 277 [6,5]
+  CRUSH rule 4 x 278 [6,8]
+  CRUSH rule 4 x 279 [8,5]
+  CRUSH rule 4 x 280 [2,6]
+  CRUSH rule 4 x 281 [8,2]
+  CRUSH rule 4 x 282 [5,2]
+  CRUSH rule 4 x 283 [8,2]
+  CRUSH rule 4 x 284 [6,5]
+  CRUSH rule 4 x 285 [5]
+  CRUSH rule 4 x 286 [2]
+  CRUSH rule 4 x 287 [2,5]
+  CRUSH rule 4 x 288 [8,2]
+  CRUSH rule 4 x 289 [5,6]
+  CRUSH rule 4 x 290 [2,5]
+  CRUSH rule 4 x 291 [2]
+  CRUSH rule 4 x 292 [8,2]
+  CRUSH rule 4 x 293 [6,2]
+  CRUSH rule 4 x 294 [7,5]
+  CRUSH rule 4 x 295 [5,8]
+  CRUSH rule 4 x 296 [5,2]
+  CRUSH rule 4 x 297 [6,2]
+  CRUSH rule 4 x 298 [2]
+  CRUSH rule 4 x 299 [2]
+  CRUSH rule 4 x 300 [8,7]
+  CRUSH rule 4 x 301 [2,8]
+  CRUSH rule 4 x 302 [5,2]
+  CRUSH rule 4 x 303 [7,5]
+  CRUSH rule 4 x 304 [2,7]
+  CRUSH rule 4 x 305 [5,8]
+  CRUSH rule 4 x 306 [2,7]
+  CRUSH rule 4 x 307 [2]
+  CRUSH rule 4 x 308 [2,8]
+  CRUSH rule 4 x 309 [7,5]
+  CRUSH rule 4 x 310 [5]
+  CRUSH rule 4 x 311 [5]
+  CRUSH rule 4 x 312 [2]
+  CRUSH rule 4 x 313 [5]
+  CRUSH rule 4 x 314 [5]
+  CRUSH rule 4 x 315 [2]
+  CRUSH rule 4 x 316 [6,5]
+  CRUSH rule 4 x 317 [2,6]
+  CRUSH rule 4 x 318 [8,6]
+  CRUSH rule 4 x 319 [5,2]
+  CRUSH rule 4 x 320 [5,7]
+  CRUSH rule 4 x 321 [2,5]
+  CRUSH rule 4 x 322 [2,7]
+  CRUSH rule 4 x 323 [5,7]
+  CRUSH rule 4 x 324 [7,2]
+  CRUSH rule 4 x 325 [5,6]
+  CRUSH rule 4 x 326 [5]
+  CRUSH rule 4 x 327 [2,6]
+  CRUSH rule 4 x 328 [7,5]
+  CRUSH rule 4 x 329 [5,6]
+  CRUSH rule 4 x 330 [5,7]
+  CRUSH rule 4 x 331 [2,6]
+  CRUSH rule 4 x 332 [2]
+  CRUSH rule 4 x 333 [6,8]
+  CRUSH rule 4 x 334 [8,5]
+  CRUSH rule 4 x 335 [7,2]
+  CRUSH rule 4 x 336 [5,6]
+  CRUSH rule 4 x 337 [7,2]
+  CRUSH rule 4 x 338 [5,6]
+  CRUSH rule 4 x 339 [7,5]
+  CRUSH rule 4 x 340 [2]
+  CRUSH rule 4 x 341 [5,2]
+  CRUSH rule 4 x 342 [2,7]
+  CRUSH rule 4 x 343 [6,7]
+  CRUSH rule 4 x 344 [6,2]
+  CRUSH rule 4 x 345 [5]
+  CRUSH rule 4 x 346 [8,2]
+  CRUSH rule 4 x 347 [5,2]
+  CRUSH rule 4 x 348 [8,2]
+  CRUSH rule 4 x 349 [2,6]
+  CRUSH rule 4 x 350 [8,5]
+  CRUSH rule 4 x 351 [5,6]
+  CRUSH rule 4 x 352 [2]
+  CRUSH rule 4 x 353 [6,5]
+  CRUSH rule 4 x 354 [2,5]
+  CRUSH rule 4 x 355 [5]
+  CRUSH rule 4 x 356 [5]
+  CRUSH rule 4 x 357 [6,2]
+  CRUSH rule 4 x 358 [2]
+  CRUSH rule 4 x 359 [6,7]
+  CRUSH rule 4 x 360 [5]
+  CRUSH rule 4 x 361 [8,5]
+  CRUSH rule 4 x 362 [5]
+  CRUSH rule 4 x 363 [5,2]
+  CRUSH rule 4 x 364 [2,5]
+  CRUSH rule 4 x 365 [6,7]
+  CRUSH rule 4 x 366 [7,2]
+  CRUSH rule 4 x 367 [5]
+  CRUSH rule 4 x 368 [7,5]
+  CRUSH rule 4 x 369 [5,7]
+  CRUSH rule 4 x 370 [8,7]
+  CRUSH rule 4 x 371 [2,5]
+  CRUSH rule 4 x 372 [5,2]
+  CRUSH rule 4 x 373 [2,6]
+  CRUSH rule 4 x 374 [5,8]
+  CRUSH rule 4 x 375 [6,5]
+  CRUSH rule 4 x 376 [7,2]
+  CRUSH rule 4 x 377 [2]
+  CRUSH rule 4 x 378 [2]
+  CRUSH rule 4 x 379 [8,5]
+  CRUSH rule 4 x 380 [2,5]
+  CRUSH rule 4 x 381 [2,5]
+  CRUSH rule 4 x 382 [2,5]
+  CRUSH rule 4 x 383 [5]
+  CRUSH rule 4 x 384 [7,2]
+  CRUSH rule 4 x 385 [7,5]
+  CRUSH rule 4 x 386 [2,5]
+  CRUSH rule 4 x 387 [2,5]
+  CRUSH rule 4 x 388 [5,2]
+  CRUSH rule 4 x 389 [2,5]
+  CRUSH rule 4 x 390 [5,6]
+  CRUSH rule 4 x 391 [5,6]
+  CRUSH rule 4 x 392 [2,8]
+  CRUSH rule 4 x 393 [5,2]
+  CRUSH rule 4 x 394 [5,7]
+  CRUSH rule 4 x 395 [5,2]
+  CRUSH rule 4 x 396 [5,2]
+  CRUSH rule 4 x 397 [2]
+  CRUSH rule 4 x 398 [2,5]
+  CRUSH rule 4 x 399 [8,7]
+  CRUSH rule 4 x 400 [8,2]
+  CRUSH rule 4 x 401 [2]
+  CRUSH rule 4 x 402 [7,8]
+  CRUSH rule 4 x 403 [2]
+  CRUSH rule 4 x 404 [5]
+  CRUSH rule 4 x 405 [6,5]
+  CRUSH rule 4 x 406 [2]
+  CRUSH rule 4 x 407 [2,8]
+  CRUSH rule 4 x 408 [5,2]
+  CRUSH rule 4 x 409 [7,5]
+  CRUSH rule 4 x 410 [8,6]
+  CRUSH rule 4 x 411 [2]
+  CRUSH rule 4 x 412 [2,5]
+  CRUSH rule 4 x 413 [5,2]
+  CRUSH rule 4 x 414 [5,2]
+  CRUSH rule 4 x 415 [2,6]
+  CRUSH rule 4 x 416 [2]
+  CRUSH rule 4 x 417 [8,7]
+  CRUSH rule 4 x 418 [7,6]
+  CRUSH rule 4 x 419 [8,5]
+  CRUSH rule 4 x 420 [2,5]
+  CRUSH rule 4 x 421 [8,6]
+  CRUSH rule 4 x 422 [6,7]
+  CRUSH rule 4 x 423 [2,5]
+  CRUSH rule 4 x 424 [8,5]
+  CRUSH rule 4 x 425 [2,5]
+  CRUSH rule 4 x 426 [6,7]
+  CRUSH rule 4 x 427 [2,7]
+  CRUSH rule 4 x 428 [5]
+  CRUSH rule 4 x 429 [5,6]
+  CRUSH rule 4 x 430 [5,6]
+  CRUSH rule 4 x 431 [5]
+  CRUSH rule 4 x 432 [7,2]
+  CRUSH rule 4 x 433 [6,5]
+  CRUSH rule 4 x 434 [5,2]
+  CRUSH rule 4 x 435 [2,5]
+  CRUSH rule 4 x 436 [5,2]
+  CRUSH rule 4 x 437 [7,5]
+  CRUSH rule 4 x 438 [2,5]
+  CRUSH rule 4 x 439 [2,5]
+  CRUSH rule 4 x 440 [2,7]
+  CRUSH rule 4 x 441 [5,7]
+  CRUSH rule 4 x 442 [2,5]
+  CRUSH rule 4 x 443 [6,8]
+  CRUSH rule 4 x 444 [7,2]
+  CRUSH rule 4 x 445 [6,5]
+  CRUSH rule 4 x 446 [5]
+  CRUSH rule 4 x 447 [2]
+  CRUSH rule 4 x 448 [7,2]
+  CRUSH rule 4 x 449 [7,8]
+  CRUSH rule 4 x 450 [5]
+  CRUSH rule 4 x 451 [6,8]
+  CRUSH rule 4 x 452 [8,5]
+  CRUSH rule 4 x 453 [6,8]
+  CRUSH rule 4 x 454 [6,7]
+  CRUSH rule 4 x 455 [2,7]
+  CRUSH rule 4 x 456 [6,8]
+  CRUSH rule 4 x 457 [7,2]
+  CRUSH rule 4 x 458 [2,8]
+  CRUSH rule 4 x 459 [2]
+  CRUSH rule 4 x 460 [6,5]
+  CRUSH rule 4 x 461 [6,5]
+  CRUSH rule 4 x 462 [8,2]
+  CRUSH rule 4 x 463 [6,7]
+  CRUSH rule 4 x 464 [7,5]
+  CRUSH rule 4 x 465 [7,6]
+  CRUSH rule 4 x 466 [5,8]
+  CRUSH rule 4 x 467 [6,5]
+  CRUSH rule 4 x 468 [7,8]
+  CRUSH rule 4 x 469 [7,2]
+  CRUSH rule 4 x 470 [5,2]
+  CRUSH rule 4 x 471 [2]
+  CRUSH rule 4 x 472 [5]
+  CRUSH rule 4 x 473 [2]
+  CRUSH rule 4 x 474 [6,2]
+  CRUSH rule 4 x 475 [6,7]
+  CRUSH rule 4 x 476 [5]
+  CRUSH rule 4 x 477 [5,8]
+  CRUSH rule 4 x 478 [6,7]
+  CRUSH rule 4 x 479 [2,5]
+  CRUSH rule 4 x 480 [2,8]
+  CRUSH rule 4 x 481 [2,5]
+  CRUSH rule 4 x 482 [5]
+  CRUSH rule 4 x 483 [2]
+  CRUSH rule 4 x 484 [2]
+  CRUSH rule 4 x 485 [5,7]
+  CRUSH rule 4 x 486 [5,2]
+  CRUSH rule 4 x 487 [5,2]
+  CRUSH rule 4 x 488 [5,7]
+  CRUSH rule 4 x 489 [2,8]
+  CRUSH rule 4 x 490 [6,5]
+  CRUSH rule 4 x 491 [2]
+  CRUSH rule 4 x 492 [6,5]
+  CRUSH rule 4 x 493 [2]
+  CRUSH rule 4 x 494 [2]
+  CRUSH rule 4 x 495 [5]
+  CRUSH rule 4 x 496 [7,5]
+  CRUSH rule 4 x 497 [5,7]
+  CRUSH rule 4 x 498 [2,5]
+  CRUSH rule 4 x 499 [8,5]
+  CRUSH rule 4 x 500 [5,6]
+  CRUSH rule 4 x 501 [2,7]
+  CRUSH rule 4 x 502 [7,2]
+  CRUSH rule 4 x 503 [2,5]
+  CRUSH rule 4 x 504 [5,6]
+  CRUSH rule 4 x 505 [2,7]
+  CRUSH rule 4 x 506 [5]
+  CRUSH rule 4 x 507 [6,2]
+  CRUSH rule 4 x 508 [2]
+  CRUSH rule 4 x 509 [7,5]
+  CRUSH rule 4 x 510 [6,2]
+  CRUSH rule 4 x 511 [5,8]
+  CRUSH rule 4 x 512 [7,6]
+  CRUSH rule 4 x 513 [7,2]
+  CRUSH rule 4 x 514 [5]
+  CRUSH rule 4 x 515 [8,5]
+  CRUSH rule 4 x 516 [5,2]
+  CRUSH rule 4 x 517 [7,8]
+  CRUSH rule 4 x 518 [5,6]
+  CRUSH rule 4 x 519 [7,5]
+  CRUSH rule 4 x 520 [2,6]
+  CRUSH rule 4 x 521 [8,7]
+  CRUSH rule 4 x 522 [6,8]
+  CRUSH rule 4 x 523 [5,2]
+  CRUSH rule 4 x 524 [2,5]
+  CRUSH rule 4 x 525 [2,5]
+  CRUSH rule 4 x 526 [2,5]
+  CRUSH rule 4 x 527 [2]
+  CRUSH rule 4 x 528 [5]
+  CRUSH rule 4 x 529 [5,7]
+  CRUSH rule 4 x 530 [6,7]
+  CRUSH rule 4 x 531 [6,2]
+  CRUSH rule 4 x 532 [6,5]
+  CRUSH rule 4 x 533 [5,6]
+  CRUSH rule 4 x 534 [7,5]
+  CRUSH rule 4 x 535 [8,6]
+  CRUSH rule 4 x 536 [6,7]
+  CRUSH rule 4 x 537 [5,7]
+  CRUSH rule 4 x 538 [6,8]
+  CRUSH rule 4 x 539 [8,5]
+  CRUSH rule 4 x 540 [2,6]
+  CRUSH rule 4 x 541 [2,5]
+  CRUSH rule 4 x 542 [5]
+  CRUSH rule 4 x 543 [6,2]
+  CRUSH rule 4 x 544 [5,7]
+  CRUSH rule 4 x 545 [5,7]
+  CRUSH rule 4 x 546 [6,2]
+  CRUSH rule 4 x 547 [8,2]
+  CRUSH rule 4 x 548 [5,2]
+  CRUSH rule 4 x 549 [5,8]
+  CRUSH rule 4 x 550 [2,5]
+  CRUSH rule 4 x 551 [7,5]
+  CRUSH rule 4 x 552 [5]
+  CRUSH rule 4 x 553 [5,2]
+  CRUSH rule 4 x 554 [2,8]
+  CRUSH rule 4 x 555 [5,2]
+  CRUSH rule 4 x 556 [5]
+  CRUSH rule 4 x 557 [7,5]
+  CRUSH rule 4 x 558 [5,2]
+  CRUSH rule 4 x 559 [5,2]
+  CRUSH rule 4 x 560 [8,5]
+  CRUSH rule 4 x 561 [6,5]
+  CRUSH rule 4 x 562 [5]
+  CRUSH rule 4 x 563 [2,6]
+  CRUSH rule 4 x 564 [5,2]
+  CRUSH rule 4 x 565 [5,6]
+  CRUSH rule 4 x 566 [5,7]
+  CRUSH rule 4 x 567 [5,6]
+  CRUSH rule 4 x 568 [7,5]
+  CRUSH rule 4 x 569 [5]
+  CRUSH rule 4 x 570 [2,5]
+  CRUSH rule 4 x 571 [5,7]
+  CRUSH rule 4 x 572 [5]
+  CRUSH rule 4 x 573 [5,2]
+  CRUSH rule 4 x 574 [2]
+  CRUSH rule 4 x 575 [8,6]
+  CRUSH rule 4 x 576 [5,6]
+  CRUSH rule 4 x 577 [8,2]
+  CRUSH rule 4 x 578 [6,8]
+  CRUSH rule 4 x 579 [5,2]
+  CRUSH rule 4 x 580 [5,2]
+  CRUSH rule 4 x 581 [7,2]
+  CRUSH rule 4 x 582 [2,8]
+  CRUSH rule 4 x 583 [6,2]
+  CRUSH rule 4 x 584 [8,2]
+  CRUSH rule 4 x 585 [7,2]
+  CRUSH rule 4 x 586 [2]
+  CRUSH rule 4 x 587 [2,5]
+  CRUSH rule 4 x 588 [5]
+  CRUSH rule 4 x 589 [7,2]
+  CRUSH rule 4 x 590 [6,2]
+  CRUSH rule 4 x 591 [5,2]
+  CRUSH rule 4 x 592 [2]
+  CRUSH rule 4 x 593 [2,8]
+  CRUSH rule 4 x 594 [2,7]
+  CRUSH rule 4 x 595 [7,2]
+  CRUSH rule 4 x 596 [5]
+  CRUSH rule 4 x 597 [5,2]
+  CRUSH rule 4 x 598 [5,2]
+  CRUSH rule 4 x 599 [5,2]
+  CRUSH rule 4 x 600 [7,2]
+  CRUSH rule 4 x 601 [2,7]
+  CRUSH rule 4 x 602 [5,7]
+  CRUSH rule 4 x 603 [5,2]
+  CRUSH rule 4 x 604 [7,5]
+  CRUSH rule 4 x 605 [5,2]
+  CRUSH rule 4 x 606 [2]
+  CRUSH rule 4 x 607 [2,5]
+  CRUSH rule 4 x 608 [5]
+  CRUSH rule 4 x 609 [5,2]
+  CRUSH rule 4 x 610 [5,7]
+  CRUSH rule 4 x 611 [2]
+  CRUSH rule 4 x 612 [2]
+  CRUSH rule 4 x 613 [7,2]
+  CRUSH rule 4 x 614 [7,8]
+  CRUSH rule 4 x 615 [6,8]
+  CRUSH rule 4 x 616 [2,8]
+  CRUSH rule 4 x 617 [6,2]
+  CRUSH rule 4 x 618 [7,6]
+  CRUSH rule 4 x 619 [5,2]
+  CRUSH rule 4 x 620 [5,2]
+  CRUSH rule 4 x 621 [5,8]
+  CRUSH rule 4 x 622 [2,5]
+  CRUSH rule 4 x 623 [2,6]
+  CRUSH rule 4 x 624 [5]
+  CRUSH rule 4 x 625 [2,5]
+  CRUSH rule 4 x 626 [7,8]
+  CRUSH rule 4 x 627 [2,7]
+  CRUSH rule 4 x 628 [8,2]
+  CRUSH rule 4 x 629 [2,6]
+  CRUSH rule 4 x 630 [2,7]
+  CRUSH rule 4 x 631 [2,7]
+  CRUSH rule 4 x 632 [7,2]
+  CRUSH rule 4 x 633 [8,6]
+  CRUSH rule 4 x 634 [2,5]
+  CRUSH rule 4 x 635 [5,6]
+  CRUSH rule 4 x 636 [2,5]
+  CRUSH rule 4 x 637 [5,2]
+  CRUSH rule 4 x 638 [6,8]
+  CRUSH rule 4 x 639 [5,2]
+  CRUSH rule 4 x 640 [5,2]
+  CRUSH rule 4 x 641 [7,2]
+  CRUSH rule 4 x 642 [2]
+  CRUSH rule 4 x 643 [5]
+  CRUSH rule 4 x 644 [8,2]
+  CRUSH rule 4 x 645 [5]
+  CRUSH rule 4 x 646 [8,2]
+  CRUSH rule 4 x 647 [7,2]
+  CRUSH rule 4 x 648 [2,6]
+  CRUSH rule 4 x 649 [5,7]
+  CRUSH rule 4 x 650 [7,8]
+  CRUSH rule 4 x 651 [5,7]
+  CRUSH rule 4 x 652 [5]
+  CRUSH rule 4 x 653 [8,5]
+  CRUSH rule 4 x 654 [7,5]
+  CRUSH rule 4 x 655 [2,5]
+  CRUSH rule 4 x 656 [5]
+  CRUSH rule 4 x 657 [6,2]
+  CRUSH rule 4 x 658 [5]
+  CRUSH rule 4 x 659 [5,6]
+  CRUSH rule 4 x 660 [7,8]
+  CRUSH rule 4 x 661 [2,8]
+  CRUSH rule 4 x 662 [5]
+  CRUSH rule 4 x 663 [2,5]
+  CRUSH rule 4 x 664 [2,5]
+  CRUSH rule 4 x 665 [5,7]
+  CRUSH rule 4 x 666 [2,8]
+  CRUSH rule 4 x 667 [2,5]
+  CRUSH rule 4 x 668 [5,7]
+  CRUSH rule 4 x 669 [6,5]
+  CRUSH rule 4 x 670 [5,2]
+  CRUSH rule 4 x 671 [2]
+  CRUSH rule 4 x 672 [5]
+  CRUSH rule 4 x 673 [5]
+  CRUSH rule 4 x 674 [5,2]
+  CRUSH rule 4 x 675 [2,8]
+  CRUSH rule 4 x 676 [2]
+  CRUSH rule 4 x 677 [5,2]
+  CRUSH rule 4 x 678 [2,5]
+  CRUSH rule 4 x 679 [6,2]
+  CRUSH rule 4 x 680 [2,5]
+  CRUSH rule 4 x 681 [5,7]
+  CRUSH rule 4 x 682 [2,5]
+  CRUSH rule 4 x 683 [2]
+  CRUSH rule 4 x 684 [7,2]
+  CRUSH rule 4 x 685 [7,2]
+  CRUSH rule 4 x 686 [2,5]
+  CRUSH rule 4 x 687 [5]
+  CRUSH rule 4 x 688 [5,7]
+  CRUSH rule 4 x 689 [6,5]
+  CRUSH rule 4 x 690 [8,2]
+  CRUSH rule 4 x 691 [5,2]
+  CRUSH rule 4 x 692 [7,2]
+  CRUSH rule 4 x 693 [6,7]
+  CRUSH rule 4 x 694 [6,5]
+  CRUSH rule 4 x 695 [2,8]
+  CRUSH rule 4 x 696 [2,5]
+  CRUSH rule 4 x 697 [6,2]
+  CRUSH rule 4 x 698 [6,2]
+  CRUSH rule 4 x 699 [2,6]
+  CRUSH rule 4 x 700 [2,5]
+  CRUSH rule 4 x 701 [5]
+  CRUSH rule 4 x 702 [5]
+  CRUSH rule 4 x 703 [8,5]
+  CRUSH rule 4 x 704 [2,5]
+  CRUSH rule 4 x 705 [8,6]
+  CRUSH rule 4 x 706 [2]
+  CRUSH rule 4 x 707 [7,8]
+  CRUSH rule 4 x 708 [5]
+  CRUSH rule 4 x 709 [6,5]
+  CRUSH rule 4 x 710 [8,5]
+  CRUSH rule 4 x 711 [2,5]
+  CRUSH rule 4 x 712 [2,5]
+  CRUSH rule 4 x 713 [6,7]
+  CRUSH rule 4 x 714 [5,2]
+  CRUSH rule 4 x 715 [2]
+  CRUSH rule 4 x 716 [5,6]
+  CRUSH rule 4 x 717 [8,7]
+  CRUSH rule 4 x 718 [5,7]
+  CRUSH rule 4 x 719 [2,6]
+  CRUSH rule 4 x 720 [6,8]
+  CRUSH rule 4 x 721 [5]
+  CRUSH rule 4 x 722 [5]
+  CRUSH rule 4 x 723 [5,2]
+  CRUSH rule 4 x 724 [2,6]
+  CRUSH rule 4 x 725 [2]
+  CRUSH rule 4 x 726 [5,8]
+  CRUSH rule 4 x 727 [5,6]
+  CRUSH rule 4 x 728 [2]
+  CRUSH rule 4 x 729 [5]
+  CRUSH rule 4 x 730 [5,7]
+  CRUSH rule 4 x 731 [5,2]
+  CRUSH rule 4 x 732 [2,5]
+  CRUSH rule 4 x 733 [5]
+  CRUSH rule 4 x 734 [6,5]
+  CRUSH rule 4 x 735 [5,8]
+  CRUSH rule 4 x 736 [5]
+  CRUSH rule 4 x 737 [2]
+  CRUSH rule 4 x 738 [5,2]
+  CRUSH rule 4 x 739 [2]
+  CRUSH rule 4 x 740 [2]
+  CRUSH rule 4 x 741 [7,8]
+  CRUSH rule 4 x 742 [8,2]
+  CRUSH rule 4 x 743 [7,2]
+  CRUSH rule 4 x 744 [5,7]
+  CRUSH rule 4 x 745 [5]
+  CRUSH rule 4 x 746 [5,2]
+  CRUSH rule 4 x 747 [6,2]
+  CRUSH rule 4 x 748 [2,7]
+  CRUSH rule 4 x 749 [5]
+  CRUSH rule 4 x 750 [2,6]
+  CRUSH rule 4 x 751 [2]
+  CRUSH rule 4 x 752 [8,2]
+  CRUSH rule 4 x 753 [7,8]
+  CRUSH rule 4 x 754 [8,6]
+  CRUSH rule 4 x 755 [2]
+  CRUSH rule 4 x 756 [5,6]
+  CRUSH rule 4 x 757 [8,6]
+  CRUSH rule 4 x 758 [6,2]
+  CRUSH rule 4 x 759 [8,5]
+  CRUSH rule 4 x 760 [2,5]
+  CRUSH rule 4 x 761 [5,2]
+  CRUSH rule 4 x 762 [2,7]
+  CRUSH rule 4 x 763 [8,6]
+  CRUSH rule 4 x 764 [2,7]
+  CRUSH rule 4 x 765 [6,5]
+  CRUSH rule 4 x 766 [8,5]
+  CRUSH rule 4 x 767 [2]
+  CRUSH rule 4 x 768 [8,5]
+  CRUSH rule 4 x 769 [6,2]
+  CRUSH rule 4 x 770 [6,2]
+  CRUSH rule 4 x 771 [7,2]
+  CRUSH rule 4 x 772 [8,5]
+  CRUSH rule 4 x 773 [5,2]
+  CRUSH rule 4 x 774 [5,6]
+  CRUSH rule 4 x 775 [6,8]
+  CRUSH rule 4 x 776 [7,2]
+  CRUSH rule 4 x 777 [5,2]
+  CRUSH rule 4 x 778 [2,8]
+  CRUSH rule 4 x 779 [2,7]
+  CRUSH rule 4 x 780 [2]
+  CRUSH rule 4 x 781 [6,5]
+  CRUSH rule 4 x 782 [5]
+  CRUSH rule 4 x 783 [7,2]
+  CRUSH rule 4 x 784 [2]
+  CRUSH rule 4 x 785 [6,2]
+  CRUSH rule 4 x 786 [7,6]
+  CRUSH rule 4 x 787 [2]
+  CRUSH rule 4 x 788 [6,2]
+  CRUSH rule 4 x 789 [2,5]
+  CRUSH rule 4 x 790 [8,5]
+  CRUSH rule 4 x 791 [5,8]
+  CRUSH rule 4 x 792 [5,8]
+  CRUSH rule 4 x 793 [6,2]
+  CRUSH rule 4 x 794 [2,6]
+  CRUSH rule 4 x 795 [2,5]
+  CRUSH rule 4 x 796 [5]
+  CRUSH rule 4 x 797 [2,5]
+  CRUSH rule 4 x 798 [6,8]
+  CRUSH rule 4 x 799 [5,2]
+  CRUSH rule 4 x 800 [5,2]
+  CRUSH rule 4 x 801 [5,6]
+  CRUSH rule 4 x 802 [2,8]
+  CRUSH rule 4 x 803 [2,5]
+  CRUSH rule 4 x 804 [6,2]
+  CRUSH rule 4 x 805 [5,6]
+  CRUSH rule 4 x 806 [2,5]
+  CRUSH rule 4 x 807 [5]
+  CRUSH rule 4 x 808 [5,6]
+  CRUSH rule 4 x 809 [2,5]
+  CRUSH rule 4 x 810 [5,7]
+  CRUSH rule 4 x 811 [8,5]
+  CRUSH rule 4 x 812 [8,5]
+  CRUSH rule 4 x 813 [6,5]
+  CRUSH rule 4 x 814 [5,6]
+  CRUSH rule 4 x 815 [5,2]
+  CRUSH rule 4 x 816 [2]
+  CRUSH rule 4 x 817 [5]
+  CRUSH rule 4 x 818 [5]
+  CRUSH rule 4 x 819 [5,2]
+  CRUSH rule 4 x 820 [5]
+  CRUSH rule 4 x 821 [5]
+  CRUSH rule 4 x 822 [2]
+  CRUSH rule 4 x 823 [5,8]
+  CRUSH rule 4 x 824 [5,7]
+  CRUSH rule 4 x 825 [2,8]
+  CRUSH rule 4 x 826 [7,2]
+  CRUSH rule 4 x 827 [2,8]
+  CRUSH rule 4 x 828 [2,5]
+  CRUSH rule 4 x 829 [5,6]
+  CRUSH rule 4 x 830 [2,5]
+  CRUSH rule 4 x 831 [2,6]
+  CRUSH rule 4 x 832 [5]
+  CRUSH rule 4 x 833 [2]
+  CRUSH rule 4 x 834 [5]
+  CRUSH rule 4 x 835 [8,5]
+  CRUSH rule 4 x 836 [5]
+  CRUSH rule 4 x 837 [6,5]
+  CRUSH rule 4 x 838 [6,7]
+  CRUSH rule 4 x 839 [5,2]
+  CRUSH rule 4 x 840 [7,8]
+  CRUSH rule 4 x 841 [5,8]
+  CRUSH rule 4 x 842 [2,5]
+  CRUSH rule 4 x 843 [6,5]
+  CRUSH rule 4 x 844 [5,8]
+  CRUSH rule 4 x 845 [5,8]
+  CRUSH rule 4 x 846 [5,2]
+  CRUSH rule 4 x 847 [2]
+  CRUSH rule 4 x 848 [2,6]
+  CRUSH rule 4 x 849 [5]
+  CRUSH rule 4 x 850 [2]
+  CRUSH rule 4 x 851 [6,8]
+  CRUSH rule 4 x 852 [7,5]
+  CRUSH rule 4 x 853 [6,8]
+  CRUSH rule 4 x 854 [7,6]
+  CRUSH rule 4 x 855 [5,7]
+  CRUSH rule 4 x 856 [6,7]
+  CRUSH rule 4 x 857 [8,5]
+  CRUSH rule 4 x 858 [6,5]
+  CRUSH rule 4 x 859 [6,2]
+  CRUSH rule 4 x 860 [5,2]
+  CRUSH rule 4 x 861 [8,7]
+  CRUSH rule 4 x 862 [6,2]
+  CRUSH rule 4 x 863 [8,7]
+  CRUSH rule 4 x 864 [5,6]
+  CRUSH rule 4 x 865 [8,2]
+  CRUSH rule 4 x 866 [5]
+  CRUSH rule 4 x 867 [6,5]
+  CRUSH rule 4 x 868 [6,5]
+  CRUSH rule 4 x 869 [8,7]
+  CRUSH rule 4 x 870 [2,5]
+  CRUSH rule 4 x 871 [5]
+  CRUSH rule 4 x 872 [5,2]
+  CRUSH rule 4 x 873 [5,6]
+  CRUSH rule 4 x 874 [2,6]
+  CRUSH rule 4 x 875 [2,6]
+  CRUSH rule 4 x 876 [5,8]
+  CRUSH rule 4 x 877 [6,5]
+  CRUSH rule 4 x 878 [5]
+  CRUSH rule 4 x 879 [7,5]
+  CRUSH rule 4 x 880 [5]
+  CRUSH rule 4 x 881 [5,6]
+  CRUSH rule 4 x 882 [5,2]
+  CRUSH rule 4 x 883 [2]
+  CRUSH rule 4 x 884 [6,2]
+  CRUSH rule 4 x 885 [5,2]
+  CRUSH rule 4 x 886 [5,6]
+  CRUSH rule 4 x 887 [7,5]
+  CRUSH rule 4 x 888 [6,8]
+  CRUSH rule 4 x 889 [2]
+  CRUSH rule 4 x 890 [7,2]
+  CRUSH rule 4 x 891 [2,8]
+  CRUSH rule 4 x 892 [6,2]
+  CRUSH rule 4 x 893 [2,5]
+  CRUSH rule 4 x 894 [7,5]
+  CRUSH rule 4 x 895 [5]
+  CRUSH rule 4 x 896 [2,8]
+  CRUSH rule 4 x 897 [5,2]
+  CRUSH rule 4 x 898 [2,5]
+  CRUSH rule 4 x 899 [2,7]
+  CRUSH rule 4 x 900 [5,2]
+  CRUSH rule 4 x 901 [5,2]
+  CRUSH rule 4 x 902 [8,5]
+  CRUSH rule 4 x 903 [5,7]
+  CRUSH rule 4 x 904 [5,6]
+  CRUSH rule 4 x 905 [6,2]
+  CRUSH rule 4 x 906 [2]
+  CRUSH rule 4 x 907 [7,2]
+  CRUSH rule 4 x 908 [5,8]
+  CRUSH rule 4 x 909 [2,5]
+  CRUSH rule 4 x 910 [6,5]
+  CRUSH rule 4 x 911 [5,8]
+  CRUSH rule 4 x 912 [2]
+  CRUSH rule 4 x 913 [7,6]
+  CRUSH rule 4 x 914 [6,5]
+  CRUSH rule 4 x 915 [8,2]
+  CRUSH rule 4 x 916 [5,2]
+  CRUSH rule 4 x 917 [2,5]
+  CRUSH rule 4 x 918 [8,2]
+  CRUSH rule 4 x 919 [6,2]
+  CRUSH rule 4 x 920 [7,6]
+  CRUSH rule 4 x 921 [2,5]
+  CRUSH rule 4 x 922 [6,7]
+  CRUSH rule 4 x 923 [5]
+  CRUSH rule 4 x 924 [5]
+  CRUSH rule 4 x 925 [5,7]
+  CRUSH rule 4 x 926 [5]
+  CRUSH rule 4 x 927 [2,6]
+  CRUSH rule 4 x 928 [8,2]
+  CRUSH rule 4 x 929 [5]
+  CRUSH rule 4 x 930 [2,5]
+  CRUSH rule 4 x 931 [5,2]
+  CRUSH rule 4 x 932 [5]
+  CRUSH rule 4 x 933 [8,5]
+  CRUSH rule 4 x 934 [5]
+  CRUSH rule 4 x 935 [6,5]
+  CRUSH rule 4 x 936 [2,6]
+  CRUSH rule 4 x 937 [5]
+  CRUSH rule 4 x 938 [6,5]
+  CRUSH rule 4 x 939 [2,7]
+  CRUSH rule 4 x 940 [8,7]
+  CRUSH rule 4 x 941 [5,2]
+  CRUSH rule 4 x 942 [2]
+  CRUSH rule 4 x 943 [8,2]
+  CRUSH rule 4 x 944 [5]
+  CRUSH rule 4 x 945 [7,2]
+  CRUSH rule 4 x 946 [2]
+  CRUSH rule 4 x 947 [5]
+  CRUSH rule 4 x 948 [7,8]
+  CRUSH rule 4 x 949 [6,2]
+  CRUSH rule 4 x 950 [5]
+  CRUSH rule 4 x 951 [5]
+  CRUSH rule 4 x 952 [2]
+  CRUSH rule 4 x 953 [2,5]
+  CRUSH rule 4 x 954 [5,2]
+  CRUSH rule 4 x 955 [8,6]
+  CRUSH rule 4 x 956 [2]
+  CRUSH rule 4 x 957 [7,6]
+  CRUSH rule 4 x 958 [8,7]
+  CRUSH rule 4 x 959 [5,2]
+  CRUSH rule 4 x 960 [5,6]
+  CRUSH rule 4 x 961 [5,2]
+  CRUSH rule 4 x 962 [7,5]
+  CRUSH rule 4 x 963 [2,5]
+  CRUSH rule 4 x 964 [5,2]
+  CRUSH rule 4 x 965 [7,6]
+  CRUSH rule 4 x 966 [5,8]
+  CRUSH rule 4 x 967 [8,6]
+  CRUSH rule 4 x 968 [7,2]
+  CRUSH rule 4 x 969 [8,2]
+  CRUSH rule 4 x 970 [2,6]
+  CRUSH rule 4 x 971 [2,7]
+  CRUSH rule 4 x 972 [2,8]
+  CRUSH rule 4 x 973 [2]
+  CRUSH rule 4 x 974 [5]
+  CRUSH rule 4 x 975 [5,7]
+  CRUSH rule 4 x 976 [5]
+  CRUSH rule 4 x 977 [8,5]
+  CRUSH rule 4 x 978 [7,2]
+  CRUSH rule 4 x 979 [7,6]
+  CRUSH rule 4 x 980 [6,2]
+  CRUSH rule 4 x 981 [7,5]
+  CRUSH rule 4 x 982 [5,2]
+  CRUSH rule 4 x 983 [5]
+  CRUSH rule 4 x 984 [2]
+  CRUSH rule 4 x 985 [2,5]
+  CRUSH rule 4 x 986 [8,7]
+  CRUSH rule 4 x 987 [2,5]
+  CRUSH rule 4 x 988 [2,5]
+  CRUSH rule 4 x 989 [2,6]
+  CRUSH rule 4 x 990 [2]
+  CRUSH rule 4 x 991 [2,5]
+  CRUSH rule 4 x 992 [7,2]
+  CRUSH rule 4 x 993 [2,6]
+  CRUSH rule 4 x 994 [5]
+  CRUSH rule 4 x 995 [7,6]
+  CRUSH rule 4 x 996 [6,7]
+  CRUSH rule 4 x 997 [6,5]
+  CRUSH rule 4 x 998 [8,2]
+  CRUSH rule 4 x 999 [2,7]
+  CRUSH rule 4 x 1000 [8,5]
+  CRUSH rule 4 x 1001 [2]
+  CRUSH rule 4 x 1002 [2,5]
+  CRUSH rule 4 x 1003 [2,8]
+  CRUSH rule 4 x 1004 [6,2]
+  CRUSH rule 4 x 1005 [6,2]
+  CRUSH rule 4 x 1006 [2]
+  CRUSH rule 4 x 1007 [2]
+  CRUSH rule 4 x 1008 [2,7]
+  CRUSH rule 4 x 1009 [6,8]
+  CRUSH rule 4 x 1010 [5]
+  CRUSH rule 4 x 1011 [5,2]
+  CRUSH rule 4 x 1012 [5,2]
+  CRUSH rule 4 x 1013 [5,2]
+  CRUSH rule 4 x 1014 [2,8]
+  CRUSH rule 4 x 1015 [6,8]
+  CRUSH rule 4 x 1016 [2]
+  CRUSH rule 4 x 1017 [6,2]
+  CRUSH rule 4 x 1018 [5]
+  CRUSH rule 4 x 1019 [5]
+  CRUSH rule 4 x 1020 [5,2]
+  CRUSH rule 4 x 1021 [5,2]
+  CRUSH rule 4 x 1022 [2,6]
+  CRUSH rule 4 x 1023 [5,2]
+  rule 4 (choose-set-two) num_rep 2 result size == 1:\t230/1024 (esc)
+  rule 4 (choose-set-two) num_rep 2 result size == 2:\t794/1024 (esc)
+  CRUSH rule 4 x 0 [2,5]
+  CRUSH rule 4 x 1 [2,8]
+  CRUSH rule 4 x 2 [2,5]
+  CRUSH rule 4 x 3 [8,2,6]
+  CRUSH rule 4 x 4 [5,2]
+  CRUSH rule 4 x 5 [7,8,2]
+  CRUSH rule 4 x 6 [2,6,7]
+  CRUSH rule 4 x 7 [5]
+  CRUSH rule 4 x 8 [5,8]
+  CRUSH rule 4 x 9 [2,5]
+  CRUSH rule 4 x 10 [2,8]
+  CRUSH rule 4 x 11 [2,7]
+  CRUSH rule 4 x 12 [2,5]
+  CRUSH rule 4 x 13 [5,8]
+  CRUSH rule 4 x 14 [7,6,8]
+  CRUSH rule 4 x 15 [7,2,5]
+  CRUSH rule 4 x 16 [5,6,2]
+  CRUSH rule 4 x 17 [5,2]
+  CRUSH rule 4 x 18 [2,5]
+  CRUSH rule 4 x 19 [7,5]
+  CRUSH rule 4 x 20 [2,5,7]
+  CRUSH rule 4 x 21 [5,7,2]
+  CRUSH rule 4 x 22 [8,5,7]
+  CRUSH rule 4 x 23 [5,6,8]
+  CRUSH rule 4 x 24 [2,6]
+  CRUSH rule 4 x 25 [5,7,2]
+  CRUSH rule 4 x 26 [2,8,7]
+  CRUSH rule 4 x 27 [5,2]
+  CRUSH rule 4 x 28 [6,2,5]
+  CRUSH rule 4 x 29 [8,5,7]
+  CRUSH rule 4 x 30 [5,7]
+  CRUSH rule 4 x 31 [8,7,6]
+  CRUSH rule 4 x 32 [5,6,8]
+  CRUSH rule 4 x 33 [2,7,8]
+  CRUSH rule 4 x 34 [2,5]
+  CRUSH rule 4 x 35 [2,8]
+  CRUSH rule 4 x 36 [5,8,2]
+  CRUSH rule 4 x 37 [2,5]
+  CRUSH rule 4 x 38 [5,8,7]
+  CRUSH rule 4 x 39 [5,7,2]
+  CRUSH rule 4 x 40 [7,8,2]
+  CRUSH rule 4 x 41 [2,6]
+  CRUSH rule 4 x 42 [5,6]
+  CRUSH rule 4 x 43 [2,5,7]
+  CRUSH rule 4 x 44 [2,6,7]
+  CRUSH rule 4 x 45 [8,2,5]
+  CRUSH rule 4 x 46 [2,5]
+  CRUSH rule 4 x 47 [5]
+  CRUSH rule 4 x 48 [5,6]
+  CRUSH rule 4 x 49 [5,8]
+  CRUSH rule 4 x 50 [5,2]
+  CRUSH rule 4 x 51 [5,6,2]
+  CRUSH rule 4 x 52 [8,6,2]
+  CRUSH rule 4 x 53 [5]
+  CRUSH rule 4 x 54 [7,6,8]
+  CRUSH rule 4 x 55 [8,7,2]
+  CRUSH rule 4 x 56 [6,5,8]
+  CRUSH rule 4 x 57 [5,6]
+  CRUSH rule 4 x 58 [2]
+  CRUSH rule 4 x 59 [5,2]
+  CRUSH rule 4 x 60 [5,2]
+  CRUSH rule 4 x 61 [5,6,8]
+  CRUSH rule 4 x 62 [7,2]
+  CRUSH rule 4 x 63 [5,6,8]
+  CRUSH rule 4 x 64 [5,2]
+  CRUSH rule 4 x 65 [7,5,6]
+  CRUSH rule 4 x 66 [5]
+  CRUSH rule 4 x 67 [5,2,8]
+  CRUSH rule 4 x 68 [2,5,8]
+  CRUSH rule 4 x 69 [5,2,6]
+  CRUSH rule 4 x 70 [7,2]
+  CRUSH rule 4 x 71 [2,8,6]
+  CRUSH rule 4 x 72 [6,2,5]
+  CRUSH rule 4 x 73 [2,7,5]
+  CRUSH rule 4 x 74 [2,7]
+  CRUSH rule 4 x 75 [5,2]
+  CRUSH rule 4 x 76 [5,2,7]
+  CRUSH rule 4 x 77 [7,2,6]
+  CRUSH rule 4 x 78 [2,5]
+  CRUSH rule 4 x 79 [5,2]
+  CRUSH rule 4 x 80 [2,5]
+  CRUSH rule 4 x 81 [2,5]
+  CRUSH rule 4 x 82 [7,2,6]
+  CRUSH rule 4 x 83 [2,6,8]
+  CRUSH rule 4 x 84 [7,2]
+  CRUSH rule 4 x 85 [5,8]
+  CRUSH rule 4 x 86 [2]
+  CRUSH rule 4 x 87 [2,7,5]
+  CRUSH rule 4 x 88 [2,6]
+  CRUSH rule 4 x 89 [5,2]
+  CRUSH rule 4 x 90 [6,7,5]
+  CRUSH rule 4 x 91 [5,8,7]
+  CRUSH rule 4 x 92 [2,8,5]
+  CRUSH rule 4 x 93 [7,5]
+  CRUSH rule 4 x 94 [2,5]
+  CRUSH rule 4 x 95 [7,5,2]
+  CRUSH rule 4 x 96 [5,6]
+  CRUSH rule 4 x 97 [8,7,5]
+  CRUSH rule 4 x 98 [2,7]
+  CRUSH rule 4 x 99 [2,7]
+  CRUSH rule 4 x 100 [2,7,5]
+  CRUSH rule 4 x 101 [5,7,2]
+  CRUSH rule 4 x 102 [5,2,7]
+  CRUSH rule 4 x 103 [5,7,6]
+  CRUSH rule 4 x 104 [7,5]
+  CRUSH rule 4 x 105 [2,5]
+  CRUSH rule 4 x 106 [2,6]
+  CRUSH rule 4 x 107 [5,2]
+  CRUSH rule 4 x 108 [7,2,8]
+  CRUSH rule 4 x 109 [2,5]
+  CRUSH rule 4 x 110 [5,2,7]
+  CRUSH rule 4 x 111 [2]
+  CRUSH rule 4 x 112 [2,6]
+  CRUSH rule 4 x 113 [6,2]
+  CRUSH rule 4 x 114 [7,6,5]
+  CRUSH rule 4 x 115 [8,2,5]
+  CRUSH rule 4 x 116 [2,6]
+  CRUSH rule 4 x 117 [7,5,6]
+  CRUSH rule 4 x 118 [2,5,8]
+  CRUSH rule 4 x 119 [5,6,2]
+  CRUSH rule 4 x 120 [2,5]
+  CRUSH rule 4 x 121 [2]
+  CRUSH rule 4 x 122 [8,5,2]
+  CRUSH rule 4 x 123 [2,5]
+  CRUSH rule 4 x 124 [5,2]
+  CRUSH rule 4 x 125 [2,7,5]
+  CRUSH rule 4 x 126 [5,2,6]
+  CRUSH rule 4 x 127 [5,6,7]
+  CRUSH rule 4 x 128 [5,8]
+  CRUSH rule 4 x 129 [2]
+  CRUSH rule 4 x 130 [5,8]
+  CRUSH rule 4 x 131 [2]
+  CRUSH rule 4 x 132 [2,5]
+  CRUSH rule 4 x 133 [5,6,7]
+  CRUSH rule 4 x 134 [2,8]
+  CRUSH rule 4 x 135 [5,6]
+  CRUSH rule 4 x 136 [2,5]
+  CRUSH rule 4 x 137 [7,5]
+  CRUSH rule 4 x 138 [8,7,6]
+  CRUSH rule 4 x 139 [5,2]
+  CRUSH rule 4 x 140 [2,6,7]
+  CRUSH rule 4 x 141 [6,8,2]
+  CRUSH rule 4 x 142 [5,2]
+  CRUSH rule 4 x 143 [5,8,7]
+  CRUSH rule 4 x 144 [8,2]
+  CRUSH rule 4 x 145 [8,5,6]
+  CRUSH rule 4 x 146 [2,6,5]
+  CRUSH rule 4 x 147 [2,8,7]
+  CRUSH rule 4 x 148 [5,2]
+  CRUSH rule 4 x 149 [5,8,6]
+  CRUSH rule 4 x 150 [2,6,5]
+  CRUSH rule 4 x 151 [5]
+  CRUSH rule 4 x 152 [8,5,6]
+  CRUSH rule 4 x 153 [8,6,7]
+  CRUSH rule 4 x 154 [5,2]
+  CRUSH rule 4 x 155 [5,8]
+  CRUSH rule 4 x 156 [5]
+  CRUSH rule 4 x 157 [5,2]
+  CRUSH rule 4 x 158 [2,8]
+  CRUSH rule 4 x 159 [7,2,5]
+  CRUSH rule 4 x 160 [2,8]
+  CRUSH rule 4 x 161 [2,5]
+  CRUSH rule 4 x 162 [2,6]
+  CRUSH rule 4 x 163 [5,6,2]
+  CRUSH rule 4 x 164 [7,8,2]
+  CRUSH rule 4 x 165 [7,2,5]
+  CRUSH rule 4 x 166 [2,5]
+  CRUSH rule 4 x 167 [2,6]
+  CRUSH rule 4 x 168 [5,2]
+  CRUSH rule 4 x 169 [2,6]
+  CRUSH rule 4 x 170 [2,5]
+  CRUSH rule 4 x 171 [7,5]
+  CRUSH rule 4 x 172 [2,7,8]
+  CRUSH rule 4 x 173 [8,5]
+  CRUSH rule 4 x 174 [2,5,7]
+  CRUSH rule 4 x 175 [6,2,5]
+  CRUSH rule 4 x 176 [5]
+  CRUSH rule 4 x 177 [5,2]
+  CRUSH rule 4 x 178 [5,2]
+  CRUSH rule 4 x 179 [5,2]
+  CRUSH rule 4 x 180 [5,8,7]
+  CRUSH rule 4 x 181 [6,2]
+  CRUSH rule 4 x 182 [8,5]
+  CRUSH rule 4 x 183 [7,8,6]
+  CRUSH rule 4 x 184 [5,7]
+  CRUSH rule 4 x 185 [6,8,2]
+  CRUSH rule 4 x 186 [2,5]
+  CRUSH rule 4 x 187 [2,6,7]
+  CRUSH rule 4 x 188 [2,8,7]
+  CRUSH rule 4 x 189 [2,7,8]
+  CRUSH rule 4 x 190 [5,2]
+  CRUSH rule 4 x 191 [7,6,8]
+  CRUSH rule 4 x 192 [5,2]
+  CRUSH rule 4 x 193 [5,2,6]
+  CRUSH rule 4 x 194 [2,5]
+  CRUSH rule 4 x 195 [6,5]
+  CRUSH rule 4 x 196 [6,7,2]
+  CRUSH rule 4 x 197 [6,5,2]
+  CRUSH rule 4 x 198 [2,5,6]
+  CRUSH rule 4 x 199 [2,5,7]
+  CRUSH rule 4 x 200 [2,5]
+  CRUSH rule 4 x 201 [7,2]
+  CRUSH rule 4 x 202 [6,5]
+  CRUSH rule 4 x 203 [5,8,7]
+  CRUSH rule 4 x 204 [2,5]
+  CRUSH rule 4 x 205 [2,7,6]
+  CRUSH rule 4 x 206 [2,7]
+  CRUSH rule 4 x 207 [5,2]
+  CRUSH rule 4 x 208 [7,2]
+  CRUSH rule 4 x 209 [2]
+  CRUSH rule 4 x 210 [2,5]
+  CRUSH rule 4 x 211 [5,2]
+  CRUSH rule 4 x 212 [7,5,2]
+  CRUSH rule 4 x 213 [8,5,2]
+  CRUSH rule 4 x 214 [5,7]
+  CRUSH rule 4 x 215 [8,2,7]
+  CRUSH rule 4 x 216 [5,2]
+  CRUSH rule 4 x 217 [2,7]
+  CRUSH rule 4 x 218 [2,7,8]
+  CRUSH rule 4 x 219 [5,8,2]
+  CRUSH rule 4 x 220 [5,7]
+  CRUSH rule 4 x 221 [5,6]
+  CRUSH rule 4 x 222 [6,8,5]
+  CRUSH rule 4 x 223 [2,5,6]
+  CRUSH rule 4 x 224 [2,5]
+  CRUSH rule 4 x 225 [8,6,2]
+  CRUSH rule 4 x 226 [7,2,5]
+  CRUSH rule 4 x 227 [5,2]
+  CRUSH rule 4 x 228 [5,8]
+  CRUSH rule 4 x 229 [5,7]
+  CRUSH rule 4 x 230 [5,7,2]
+  CRUSH rule 4 x 231 [5]
+  CRUSH rule 4 x 232 [2,7,5]
+  CRUSH rule 4 x 233 [5]
+  CRUSH rule 4 x 234 [2]
+  CRUSH rule 4 x 235 [5,8,2]
+  CRUSH rule 4 x 236 [5,2,7]
+  CRUSH rule 4 x 237 [5,7,6]
+  CRUSH rule 4 x 238 [5]
+  CRUSH rule 4 x 239 [8,7,6]
+  CRUSH rule 4 x 240 [5,7]
+  CRUSH rule 4 x 241 [5,2]
+  CRUSH rule 4 x 242 [5]
+  CRUSH rule 4 x 243 [5,7]
+  CRUSH rule 4 x 244 [5,6,2]
+  CRUSH rule 4 x 245 [7,6,2]
+  CRUSH rule 4 x 246 [2,5]
+  CRUSH rule 4 x 247 [6,2]
+  CRUSH rule 4 x 248 [8,2,7]
+  CRUSH rule 4 x 249 [2]
+  CRUSH rule 4 x 250 [2]
+  CRUSH rule 4 x 251 [2,5,6]
+  CRUSH rule 4 x 252 [5,7,8]
+  CRUSH rule 4 x 253 [5,2]
+  CRUSH rule 4 x 254 [5]
+  CRUSH rule 4 x 255 [2,7,8]
+  CRUSH rule 4 x 256 [5,7,2]
+  CRUSH rule 4 x 257 [2,8,5]
+  CRUSH rule 4 x 258 [5,2]
+  CRUSH rule 4 x 259 [5,6]
+  CRUSH rule 4 x 260 [5,6,2]
+  CRUSH rule 4 x 261 [8,7,5]
+  CRUSH rule 4 x 262 [5,8]
+  CRUSH rule 4 x 263 [6,8,2]
+  CRUSH rule 4 x 264 [5,6]
+  CRUSH rule 4 x 265 [8,6,7]
+  CRUSH rule 4 x 266 [8,2]
+  CRUSH rule 4 x 267 [2,5]
+  CRUSH rule 4 x 268 [2,7]
+  CRUSH rule 4 x 269 [2,8]
+  CRUSH rule 4 x 270 [5,2]
+  CRUSH rule 4 x 271 [7,5]
+  CRUSH rule 4 x 272 [2,8,5]
+  CRUSH rule 4 x 273 [5,2]
+  CRUSH rule 4 x 274 [6,8,5]
+  CRUSH rule 4 x 275 [5,6]
+  CRUSH rule 4 x 276 [7,2]
+  CRUSH rule 4 x 277 [6,5,2]
+  CRUSH rule 4 x 278 [6,8,2]
+  CRUSH rule 4 x 279 [8,5,6]
+  CRUSH rule 4 x 280 [2,6]
+  CRUSH rule 4 x 281 [8,2]
+  CRUSH rule 4 x 282 [5,2,6]
+  CRUSH rule 4 x 283 [8,2]
+  CRUSH rule 4 x 284 [6,5,2]
+  CRUSH rule 4 x 285 [5,7]
+  CRUSH rule 4 x 286 [2,6]
+  CRUSH rule 4 x 287 [2,5]
+  CRUSH rule 4 x 288 [8,2,7]
+  CRUSH rule 4 x 289 [5,6,2]
+  CRUSH rule 4 x 290 [2,5,7]
+  CRUSH rule 4 x 291 [2,5]
+  CRUSH rule 4 x 292 [8,2]
+  CRUSH rule 4 x 293 [6,2,8]
+  CRUSH rule 4 x 294 [7,5]
+  CRUSH rule 4 x 295 [5,8,7]
+  CRUSH rule 4 x 296 [5,2]
+  CRUSH rule 4 x 297 [6,2,8]
+  CRUSH rule 4 x 298 [2,5]
+  CRUSH rule 4 x 299 [2,7]
+  CRUSH rule 4 x 300 [8,7,5]
+  CRUSH rule 4 x 301 [2,8]
+  CRUSH rule 4 x 302 [5,2,6]
+  CRUSH rule 4 x 303 [7,5,8]
+  CRUSH rule 4 x 304 [2,7,5]
+  CRUSH rule 4 x 305 [5,8,2]
+  CRUSH rule 4 x 306 [2,7]
+  CRUSH rule 4 x 307 [2,8]
+  CRUSH rule 4 x 308 [2,8,5]
+  CRUSH rule 4 x 309 [7,5]
+  CRUSH rule 4 x 310 [5,2]
+  CRUSH rule 4 x 311 [5]
+  CRUSH rule 4 x 312 [2]
+  CRUSH rule 4 x 313 [5]
+  CRUSH rule 4 x 314 [5,2]
+  CRUSH rule 4 x 315 [2]
+  CRUSH rule 4 x 316 [6,5,8]
+  CRUSH rule 4 x 317 [2,6]
+  CRUSH rule 4 x 318 [8,6,7]
+  CRUSH rule 4 x 319 [5,2,8]
+  CRUSH rule 4 x 320 [5,7,2]
+  CRUSH rule 4 x 321 [2,5]
+  CRUSH rule 4 x 322 [2,7,5]
+  CRUSH rule 4 x 323 [5,7,2]
+  CRUSH rule 4 x 324 [7,2,8]
+  CRUSH rule 4 x 325 [5,6,2]
+  CRUSH rule 4 x 326 [5,2]
+  CRUSH rule 4 x 327 [2,6,7]
+  CRUSH rule 4 x 328 [7,5,8]
+  CRUSH rule 4 x 329 [5,6]
+  CRUSH rule 4 x 330 [5,7]
+  CRUSH rule 4 x 331 [2,6,5]
+  CRUSH rule 4 x 332 [2,5]
+  CRUSH rule 4 x 333 [6,8,5]
+  CRUSH rule 4 x 334 [8,5]
+  CRUSH rule 4 x 335 [7,2]
+  CRUSH rule 4 x 336 [5,6,2]
+  CRUSH rule 4 x 337 [7,2,6]
+  CRUSH rule 4 x 338 [5,6]
+  CRUSH rule 4 x 339 [7,5,2]
+  CRUSH rule 4 x 340 [2]
+  CRUSH rule 4 x 341 [5,2,7]
+  CRUSH rule 4 x 342 [2,7,5]
+  CRUSH rule 4 x 343 [6,7,5]
+  CRUSH rule 4 x 344 [6,2,5]
+  CRUSH rule 4 x 345 [5]
+  CRUSH rule 4 x 346 [8,2,5]
+  CRUSH rule 4 x 347 [5,2]
+  CRUSH rule 4 x 348 [8,2]
+  CRUSH rule 4 x 349 [2,6,7]
+  CRUSH rule 4 x 350 [8,5,7]
+  CRUSH rule 4 x 351 [5,6]
+  CRUSH rule 4 x 352 [2]
+  CRUSH rule 4 x 353 [6,5]
+  CRUSH rule 4 x 354 [2,5]
+  CRUSH rule 4 x 355 [5,6]
+  CRUSH rule 4 x 356 [5]
+  CRUSH rule 4 x 357 [6,2]
+  CRUSH rule 4 x 358 [2,8]
+  CRUSH rule 4 x 359 [6,7,8]
+  CRUSH rule 4 x 360 [5,2]
+  CRUSH rule 4 x 361 [8,5]
+  CRUSH rule 4 x 362 [5]
+  CRUSH rule 4 x 363 [5,2]
+  CRUSH rule 4 x 364 [2,5]
+  CRUSH rule 4 x 365 [6,7,8]
+  CRUSH rule 4 x 366 [7,2,8]
+  CRUSH rule 4 x 367 [5,2]
+  CRUSH rule 4 x 368 [7,5,6]
+  CRUSH rule 4 x 369 [5,7,2]
+  CRUSH rule 4 x 370 [8,7,6]
+  CRUSH rule 4 x 371 [2,5]
+  CRUSH rule 4 x 372 [5,2,8]
+  CRUSH rule 4 x 373 [2,6,8]
+  CRUSH rule 4 x 374 [5,8]
+  CRUSH rule 4 x 375 [6,5]
+  CRUSH rule 4 x 376 [7,2]
+  CRUSH rule 4 x 377 [2,5]
+  CRUSH rule 4 x 378 [2]
+  CRUSH rule 4 x 379 [8,5]
+  CRUSH rule 4 x 380 [2,5]
+  CRUSH rule 4 x 381 [2,5,7]
+  CRUSH rule 4 x 382 [2,5]
+  CRUSH rule 4 x 383 [5]
+  CRUSH rule 4 x 384 [7,2,6]
+  CRUSH rule 4 x 385 [7,5,6]
+  CRUSH rule 4 x 386 [2,5]
+  CRUSH rule 4 x 387 [2,5,6]
+  CRUSH rule 4 x 388 [5,2]
+  CRUSH rule 4 x 389 [2,5]
+  CRUSH rule 4 x 390 [5,6,2]
+  CRUSH rule 4 x 391 [5,6,2]
+  CRUSH rule 4 x 392 [2,8,6]
+  CRUSH rule 4 x 393 [5,2,6]
+  CRUSH rule 4 x 394 [5,7,6]
+  CRUSH rule 4 x 395 [5,2]
+  CRUSH rule 4 x 396 [5,2]
+  CRUSH rule 4 x 397 [2,5]
+  CRUSH rule 4 x 398 [2,5]
+  CRUSH rule 4 x 399 [8,7,5]
+  CRUSH rule 4 x 400 [8,2,5]
+  CRUSH rule 4 x 401 [2]
+  CRUSH rule 4 x 402 [7,8,6]
+  CRUSH rule 4 x 403 [2,5]
+  CRUSH rule 4 x 404 [5,2]
+  CRUSH rule 4 x 405 [6,5,2]
+  CRUSH rule 4 x 406 [2]
+  CRUSH rule 4 x 407 [2,8,7]
+  CRUSH rule 4 x 408 [5,2,6]
+  CRUSH rule 4 x 409 [7,5,6]
+  CRUSH rule 4 x 410 [8,6,5]
+  CRUSH rule 4 x 411 [2,6]
+  CRUSH rule 4 x 412 [2,5,8]
+  CRUSH rule 4 x 413 [5,2,8]
+  CRUSH rule 4 x 414 [5,2]
+  CRUSH rule 4 x 415 [2,6]
+  CRUSH rule 4 x 416 [2,7]
+  CRUSH rule 4 x 417 [8,7,2]
+  CRUSH rule 4 x 418 [7,6,8]
+  CRUSH rule 4 x 419 [8,5,2]
+  CRUSH rule 4 x 420 [2,5]
+  CRUSH rule 4 x 421 [8,6,7]
+  CRUSH rule 4 x 422 [6,7,8]
+  CRUSH rule 4 x 423 [2,5,6]
+  CRUSH rule 4 x 424 [8,5,7]
+  CRUSH rule 4 x 425 [2,5]
+  CRUSH rule 4 x 426 [6,7,2]
+  CRUSH rule 4 x 427 [2,7,5]
+  CRUSH rule 4 x 428 [5]
+  CRUSH rule 4 x 429 [5,6,7]
+  CRUSH rule 4 x 430 [5,6]
+  CRUSH rule 4 x 431 [5,2]
+  CRUSH rule 4 x 432 [7,2]
+  CRUSH rule 4 x 433 [6,5,2]
+  CRUSH rule 4 x 434 [5,2]
+  CRUSH rule 4 x 435 [2,5]
+  CRUSH rule 4 x 436 [5,2]
+  CRUSH rule 4 x 437 [7,5]
+  CRUSH rule 4 x 438 [2,5,8]
+  CRUSH rule 4 x 439 [2,5]
+  CRUSH rule 4 x 440 [2,7]
+  CRUSH rule 4 x 441 [5,7,2]
+  CRUSH rule 4 x 442 [2,5]
+  CRUSH rule 4 x 443 [6,8,2]
+  CRUSH rule 4 x 444 [7,2]
+  CRUSH rule 4 x 445 [6,5]
+  CRUSH rule 4 x 446 [5]
+  CRUSH rule 4 x 447 [2,5]
+  CRUSH rule 4 x 448 [7,2,5]
+  CRUSH rule 4 x 449 [7,8,5]
+  CRUSH rule 4 x 450 [5,2]
+  CRUSH rule 4 x 451 [6,8,5]
+  CRUSH rule 4 x 452 [8,5]
+  CRUSH rule 4 x 453 [6,8,7]
+  CRUSH rule 4 x 454 [6,7,5]
+  CRUSH rule 4 x 455 [2,7,5]
+  CRUSH rule 4 x 456 [6,8,7]
+  CRUSH rule 4 x 457 [7,2,8]
+  CRUSH rule 4 x 458 [2,8]
+  CRUSH rule 4 x 459 [2,6]
+  CRUSH rule 4 x 460 [6,5,7]
+  CRUSH rule 4 x 461 [6,5]
+  CRUSH rule 4 x 462 [8,2,5]
+  CRUSH rule 4 x 463 [6,7,2]
+  CRUSH rule 4 x 464 [7,5,2]
+  CRUSH rule 4 x 465 [7,6,2]
+  CRUSH rule 4 x 466 [5,8]
+  CRUSH rule 4 x 467 [6,5,7]
+  CRUSH rule 4 x 468 [7,8,2]
+  CRUSH rule 4 x 469 [7,2]
+  CRUSH rule 4 x 470 [5,2]
+  CRUSH rule 4 x 471 [2]
+  CRUSH rule 4 x 472 [5,2]
+  CRUSH rule 4 x 473 [2,5]
+  CRUSH rule 4 x 474 [6,2,7]
+  CRUSH rule 4 x 475 [6,7,8]
+  CRUSH rule 4 x 476 [5]
+  CRUSH rule 4 x 477 [5,8,6]
+  CRUSH rule 4 x 478 [6,7,2]
+  CRUSH rule 4 x 479 [2,5,8]
+  CRUSH rule 4 x 480 [2,8,6]
+  CRUSH rule 4 x 481 [2,5]
+  CRUSH rule 4 x 482 [5]
+  CRUSH rule 4 x 483 [2,6]
+  CRUSH rule 4 x 484 [2,7]
+  CRUSH rule 4 x 485 [5,7]
+  CRUSH rule 4 x 486 [5,2,7]
+  CRUSH rule 4 x 487 [5,2]
+  CRUSH rule 4 x 488 [5,7,2]
+  CRUSH rule 4 x 489 [2,8,5]
+  CRUSH rule 4 x 490 [6,5,2]
+  CRUSH rule 4 x 491 [2,6]
+  CRUSH rule 4 x 492 [6,5]
+  CRUSH rule 4 x 493 [2,7]
+  CRUSH rule 4 x 494 [2,8]
+  CRUSH rule 4 x 495 [5]
+  CRUSH rule 4 x 496 [7,5]
+  CRUSH rule 4 x 497 [5,7]
+  CRUSH rule 4 x 498 [2,5,8]
+  CRUSH rule 4 x 499 [8,5]
+  CRUSH rule 4 x 500 [5,6]
+  CRUSH rule 4 x 501 [2,7]
+  CRUSH rule 4 x 502 [7,2,5]
+  CRUSH rule 4 x 503 [2,5]
+  CRUSH rule 4 x 504 [5,6]
+  CRUSH rule 4 x 505 [2,7,6]
+  CRUSH rule 4 x 506 [5,2]
+  CRUSH rule 4 x 507 [6,2,8]
+  CRUSH rule 4 x 508 [2]
+  CRUSH rule 4 x 509 [7,5,8]
+  CRUSH rule 4 x 510 [6,2]
+  CRUSH rule 4 x 511 [5,8,7]
+  CRUSH rule 4 x 512 [7,6,2]
+  CRUSH rule 4 x 513 [7,2]
+  CRUSH rule 4 x 514 [5]
+  CRUSH rule 4 x 515 [8,5,6]
+  CRUSH rule 4 x 516 [5,2]
+  CRUSH rule 4 x 517 [7,8,6]
+  CRUSH rule 4 x 518 [5,6,7]
+  CRUSH rule 4 x 519 [7,5]
+  CRUSH rule 4 x 520 [2,6,5]
+  CRUSH rule 4 x 521 [8,7,6]
+  CRUSH rule 4 x 522 [6,8,2]
+  CRUSH rule 4 x 523 [5,2]
+  CRUSH rule 4 x 524 [2,5]
+  CRUSH rule 4 x 525 [2,5]
+  CRUSH rule 4 x 526 [2,5,8]
+  CRUSH rule 4 x 527 [2,5]
+  CRUSH rule 4 x 528 [5,2]
+  CRUSH rule 4 x 529 [5,7,2]
+  CRUSH rule 4 x 530 [6,7,8]
+  CRUSH rule 4 x 531 [6,2]
+  CRUSH rule 4 x 532 [6,5,7]
+  CRUSH rule 4 x 533 [5,6]
+  CRUSH rule 4 x 534 [7,5]
+  CRUSH rule 4 x 535 [8,6,2]
+  CRUSH rule 4 x 536 [6,7,2]
+  CRUSH rule 4 x 537 [5,7]
+  CRUSH rule 4 x 538 [6,8,5]
+  CRUSH rule 4 x 539 [8,5,7]
+  CRUSH rule 4 x 540 [2,6]
+  CRUSH rule 4 x 541 [2,5]
+  CRUSH rule 4 x 542 [5]
+  CRUSH rule 4 x 543 [6,2,8]
+  CRUSH rule 4 x 544 [5,7,6]
+  CRUSH rule 4 x 545 [5,7,2]
+  CRUSH rule 4 x 546 [6,2,7]
+  CRUSH rule 4 x 547 [8,2]
+  CRUSH rule 4 x 548 [5,2]
+  CRUSH rule 4 x 549 [5,8,2]
+  CRUSH rule 4 x 550 [2,5]
+  CRUSH rule 4 x 551 [7,5]
+  CRUSH rule 4 x 552 [5]
+  CRUSH rule 4 x 553 [5,2]
+  CRUSH rule 4 x 554 [2,8,5]
+  CRUSH rule 4 x 555 [5,2,8]
+  CRUSH rule 4 x 556 [5]
+  CRUSH rule 4 x 557 [7,5,6]
+  CRUSH rule 4 x 558 [5,2]
+  CRUSH rule 4 x 559 [5,2,6]
+  CRUSH rule 4 x 560 [8,5]
+  CRUSH rule 4 x 561 [6,5,7]
+  CRUSH rule 4 x 562 [5,2]
+  CRUSH rule 4 x 563 [2,6]
+  CRUSH rule 4 x 564 [5,2,7]
+  CRUSH rule 4 x 565 [5,6]
+  CRUSH rule 4 x 566 [5,7,2]
+  CRUSH rule 4 x 567 [5,6,2]
+  CRUSH rule 4 x 568 [7,5,2]
+  CRUSH rule 4 x 569 [5,2]
+  CRUSH rule 4 x 570 [2,5]
+  CRUSH rule 4 x 571 [5,7]
+  CRUSH rule 4 x 572 [5,2]
+  CRUSH rule 4 x 573 [5,2,7]
+  CRUSH rule 4 x 574 [2]
+  CRUSH rule 4 x 575 [8,6,2]
+  CRUSH rule 4 x 576 [5,6,8]
+  CRUSH rule 4 x 577 [8,2,6]
+  CRUSH rule 4 x 578 [6,8,7]
+  CRUSH rule 4 x 579 [5,2,6]
+  CRUSH rule 4 x 580 [5,2,7]
+  CRUSH rule 4 x 581 [7,2,6]
+  CRUSH rule 4 x 582 [2,8,7]
+  CRUSH rule 4 x 583 [6,2,8]
+  CRUSH rule 4 x 584 [8,2]
+  CRUSH rule 4 x 585 [7,2,5]
+  CRUSH rule 4 x 586 [2,7]
+  CRUSH rule 4 x 587 [2,5]
+  CRUSH rule 4 x 588 [5,7]
+  CRUSH rule 4 x 589 [7,2,5]
+  CRUSH rule 4 x 590 [6,2,5]
+  CRUSH rule 4 x 591 [5,2]
+  CRUSH rule 4 x 592 [2,5]
+  CRUSH rule 4 x 593 [2,8]
+  CRUSH rule 4 x 594 [2,7,5]
+  CRUSH rule 4 x 595 [7,2,8]
+  CRUSH rule 4 x 596 [5]
+  CRUSH rule 4 x 597 [5,2]
+  CRUSH rule 4 x 598 [5,2]
+  CRUSH rule 4 x 599 [5,2]
+  CRUSH rule 4 x 600 [7,2,8]
+  CRUSH rule 4 x 601 [2,7,8]
+  CRUSH rule 4 x 602 [5,7]
+  CRUSH rule 4 x 603 [5,2,6]
+  CRUSH rule 4 x 604 [7,5]
+  CRUSH rule 4 x 605 [5,2]
+  CRUSH rule 4 x 606 [2]
+  CRUSH rule 4 x 607 [2,5,8]
+  CRUSH rule 4 x 608 [5,2]
+  CRUSH rule 4 x 609 [5,2]
+  CRUSH rule 4 x 610 [5,7,6]
+  CRUSH rule 4 x 611 [2]
+  CRUSH rule 4 x 612 [2,8]
+  CRUSH rule 4 x 613 [7,2]
+  CRUSH rule 4 x 614 [7,8,6]
+  CRUSH rule 4 x 615 [6,8,2]
+  CRUSH rule 4 x 616 [2,8]
+  CRUSH rule 4 x 617 [6,2]
+  CRUSH rule 4 x 618 [7,6,5]
+  CRUSH rule 4 x 619 [5,2,8]
+  CRUSH rule 4 x 620 [5,2]
+  CRUSH rule 4 x 621 [5,8]
+  CRUSH rule 4 x 622 [2,5]
+  CRUSH rule 4 x 623 [2,6]
+  CRUSH rule 4 x 624 [5,2]
+  CRUSH rule 4 x 625 [2,5,7]
+  CRUSH rule 4 x 626 [7,8,2]
+  CRUSH rule 4 x 627 [2,7]
+  CRUSH rule 4 x 628 [8,2,5]
+  CRUSH rule 4 x 629 [2,6,5]
+  CRUSH rule 4 x 630 [2,7]
+  CRUSH rule 4 x 631 [2,7,8]
+  CRUSH rule 4 x 632 [7,2]
+  CRUSH rule 4 x 633 [8,6,5]
+  CRUSH rule 4 x 634 [2,5]
+  CRUSH rule 4 x 635 [5,6]
+  CRUSH rule 4 x 636 [2,5]
+  CRUSH rule 4 x 637 [5,2]
+  CRUSH rule 4 x 638 [6,8,2]
+  CRUSH rule 4 x 639 [5,2]
+  CRUSH rule 4 x 640 [5,2]
+  CRUSH rule 4 x 641 [7,2]
+  CRUSH rule 4 x 642 [2]
+  CRUSH rule 4 x 643 [5,2]
+  CRUSH rule 4 x 644 [8,2,5]
+  CRUSH rule 4 x 645 [5,6]
+  CRUSH rule 4 x 646 [8,2,5]
+  CRUSH rule 4 x 647 [7,2,5]
+  CRUSH rule 4 x 648 [2,6,5]
+  CRUSH rule 4 x 649 [5,7]
+  CRUSH rule 4 x 650 [7,8,6]
+  CRUSH rule 4 x 651 [5,7,6]
+  CRUSH rule 4 x 652 [5,8]
+  CRUSH rule 4 x 653 [8,5,2]
+  CRUSH rule 4 x 654 [7,5,2]
+  CRUSH rule 4 x 655 [2,5]
+  CRUSH rule 4 x 656 [5,8]
+  CRUSH rule 4 x 657 [6,2]
+  CRUSH rule 4 x 658 [5,6]
+  CRUSH rule 4 x 659 [5,6,7]
+  CRUSH rule 4 x 660 [7,8,6]
+  CRUSH rule 4 x 661 [2,8,5]
+  CRUSH rule 4 x 662 [5]
+  CRUSH rule 4 x 663 [2,5,8]
+  CRUSH rule 4 x 664 [2,5]
+  CRUSH rule 4 x 665 [5,7,6]
+  CRUSH rule 4 x 666 [2,8,5]
+  CRUSH rule 4 x 667 [2,5]
+  CRUSH rule 4 x 668 [5,7,6]
+  CRUSH rule 4 x 669 [6,5]
+  CRUSH rule 4 x 670 [5,2]
+  CRUSH rule 4 x 671 [2]
+  CRUSH rule 4 x 672 [5,2]
+  CRUSH rule 4 x 673 [5,2]
+  CRUSH rule 4 x 674 [5,2,8]
+  CRUSH rule 4 x 675 [2,8,6]
+  CRUSH rule 4 x 676 [2,5]
+  CRUSH rule 4 x 677 [5,2]
+  CRUSH rule 4 x 678 [2,5]
+  CRUSH rule 4 x 679 [6,2]
+  CRUSH rule 4 x 680 [2,5,6]
+  CRUSH rule 4 x 681 [5,7]
+  CRUSH rule 4 x 682 [2,5]
+  CRUSH rule 4 x 683 [2,5]
+  CRUSH rule 4 x 684 [7,2,6]
+  CRUSH rule 4 x 685 [7,2,6]
+  CRUSH rule 4 x 686 [2,5]
+  CRUSH rule 4 x 687 [5,7]
+  CRUSH rule 4 x 688 [5,7,2]
+  CRUSH rule 4 x 689 [6,5,2]
+  CRUSH rule 4 x 690 [8,2,7]
+  CRUSH rule 4 x 691 [5,2]
+  CRUSH rule 4 x 692 [7,2,8]
+  CRUSH rule 4 x 693 [6,7,5]
+  CRUSH rule 4 x 694 [6,5,2]
+  CRUSH rule 4 x 695 [2,8,7]
+  CRUSH rule 4 x 696 [2,5]
+  CRUSH rule 4 x 697 [6,2]
+  CRUSH rule 4 x 698 [6,2]
+  CRUSH rule 4 x 699 [2,6,8]
+  CRUSH rule 4 x 700 [2,5]
+  CRUSH rule 4 x 701 [5,2]
+  CRUSH rule 4 x 702 [5,2]
+  CRUSH rule 4 x 703 [8,5]
+  CRUSH rule 4 x 704 [2,5,8]
+  CRUSH rule 4 x 705 [8,6,2]
+  CRUSH rule 4 x 706 [2,5]
+  CRUSH rule 4 x 707 [7,8,6]
+  CRUSH rule 4 x 708 [5,8]
+  CRUSH rule 4 x 709 [6,5,2]
+  CRUSH rule 4 x 710 [8,5]
+  CRUSH rule 4 x 711 [2,5,8]
+  CRUSH rule 4 x 712 [2,5,7]
+  CRUSH rule 4 x 713 [6,7,8]
+  CRUSH rule 4 x 714 [5,2]
+  CRUSH rule 4 x 715 [2,5]
+  CRUSH rule 4 x 716 [5,6,2]
+  CRUSH rule 4 x 717 [8,7,2]
+  CRUSH rule 4 x 718 [5,7,8]
+  CRUSH rule 4 x 719 [2,6,5]
+  CRUSH rule 4 x 720 [6,8,2]
+  CRUSH rule 4 x 721 [5,7]
+  CRUSH rule 4 x 722 [5,6]
+  CRUSH rule 4 x 723 [5,2]
+  CRUSH rule 4 x 724 [2,6]
+  CRUSH rule 4 x 725 [2,5]
+  CRUSH rule 4 x 726 [5,8]
+  CRUSH rule 4 x 727 [5,6,8]
+  CRUSH rule 4 x 728 [2]
+  CRUSH rule 4 x 729 [5,7]
+  CRUSH rule 4 x 730 [5,7]
+  CRUSH rule 4 x 731 [5,2]
+  CRUSH rule 4 x 732 [2,5]
+  CRUSH rule 4 x 733 [5,7]
+  CRUSH rule 4 x 734 [6,5,2]
+  CRUSH rule 4 x 735 [5,8]
+  CRUSH rule 4 x 736 [5,6]
+  CRUSH rule 4 x 737 [2,8]
+  CRUSH rule 4 x 738 [5,2]
+  CRUSH rule 4 x 739 [2]
+  CRUSH rule 4 x 740 [2,7]
+  CRUSH rule 4 x 741 [7,8,2]
+  CRUSH rule 4 x 742 [8,2]
+  CRUSH rule 4 x 743 [7,2]
+  CRUSH rule 4 x 744 [5,7]
+  CRUSH rule 4 x 745 [5,2]
+  CRUSH rule 4 x 746 [5,2]
+  CRUSH rule 4 x 747 [6,2,5]
+  CRUSH rule 4 x 748 [2,7]
+  CRUSH rule 4 x 749 [5,8]
+  CRUSH rule 4 x 750 [2,6,5]
+  CRUSH rule 4 x 751 [2,6]
+  CRUSH rule 4 x 752 [8,2,5]
+  CRUSH rule 4 x 753 [7,8,5]
+  CRUSH rule 4 x 754 [8,6,7]
+  CRUSH rule 4 x 755 [2]
+  CRUSH rule 4 x 756 [5,6,2]
+  CRUSH rule 4 x 757 [8,6,2]
+  CRUSH rule 4 x 758 [6,2,5]
+  CRUSH rule 4 x 759 [8,5]
+  CRUSH rule 4 x 760 [2,5]
+  CRUSH rule 4 x 761 [5,2]
+  CRUSH rule 4 x 762 [2,7,8]
+  CRUSH rule 4 x 763 [8,6,7]
+  CRUSH rule 4 x 764 [2,7]
+  CRUSH rule 4 x 765 [6,5,2]
+  CRUSH rule 4 x 766 [8,5,7]
+  CRUSH rule 4 x 767 [2]
+  CRUSH rule 4 x 768 [8,5,2]
+  CRUSH rule 4 x 769 [6,2,8]
+  CRUSH rule 4 x 770 [6,2,7]
+  CRUSH rule 4 x 771 [7,2,5]
+  CRUSH rule 4 x 772 [8,5,7]
+  CRUSH rule 4 x 773 [5,2]
+  CRUSH rule 4 x 774 [5,6]
+  CRUSH rule 4 x 775 [6,8,5]
+  CRUSH rule 4 x 776 [7,2]
+  CRUSH rule 4 x 777 [5,2,6]
+  CRUSH rule 4 x 778 [2,8]
+  CRUSH rule 4 x 779 [2,7,5]
+  CRUSH rule 4 x 780 [2,5]
+  CRUSH rule 4 x 781 [6,5,7]
+  CRUSH rule 4 x 782 [5,2]
+  CRUSH rule 4 x 783 [7,2,8]
+  CRUSH rule 4 x 784 [2,5]
+  CRUSH rule 4 x 785 [6,2]
+  CRUSH rule 4 x 786 [7,6,5]
+  CRUSH rule 4 x 787 [2,6]
+  CRUSH rule 4 x 788 [6,2,8]
+  CRUSH rule 4 x 789 [2,5]
+  CRUSH rule 4 x 790 [8,5,7]
+  CRUSH rule 4 x 791 [5,8,7]
+  CRUSH rule 4 x 792 [5,8]
+  CRUSH rule 4 x 793 [6,2,5]
+  CRUSH rule 4 x 794 [2,6,8]
+  CRUSH rule 4 x 795 [2,5]
+  CRUSH rule 4 x 796 [5]
+  CRUSH rule 4 x 797 [2,5]
+  CRUSH rule 4 x 798 [6,8,7]
+  CRUSH rule 4 x 799 [5,2]
+  CRUSH rule 4 x 800 [5,2]
+  CRUSH rule 4 x 801 [5,6,8]
+  CRUSH rule 4 x 802 [2,8]
+  CRUSH rule 4 x 803 [2,5,7]
+  CRUSH rule 4 x 804 [6,2,5]
+  CRUSH rule 4 x 805 [5,6,7]
+  CRUSH rule 4 x 806 [2,5]
+  CRUSH rule 4 x 807 [5,7]
+  CRUSH rule 4 x 808 [5,6,2]
+  CRUSH rule 4 x 809 [2,5]
+  CRUSH rule 4 x 810 [5,7]
+  CRUSH rule 4 x 811 [8,5]
+  CRUSH rule 4 x 812 [8,5]
+  CRUSH rule 4 x 813 [6,5,8]
+  CRUSH rule 4 x 814 [5,6,8]
+  CRUSH rule 4 x 815 [5,2]
+  CRUSH rule 4 x 816 [2,8]
+  CRUSH rule 4 x 817 [5,7]
+  CRUSH rule 4 x 818 [5,2]
+  CRUSH rule 4 x 819 [5,2]
+  CRUSH rule 4 x 820 [5]
+  CRUSH rule 4 x 821 [5,8]
+  CRUSH rule 4 x 822 [2,5]
+  CRUSH rule 4 x 823 [5,8,2]
+  CRUSH rule 4 x 824 [5,7]
+  CRUSH rule 4 x 825 [2,8,7]
+  CRUSH rule 4 x 826 [7,2,5]
+  CRUSH rule 4 x 827 [2,8]
+  CRUSH rule 4 x 828 [2,5]
+  CRUSH rule 4 x 829 [5,6,7]
+  CRUSH rule 4 x 830 [2,5]
+  CRUSH rule 4 x 831 [2,6]
+  CRUSH rule 4 x 832 [5]
+  CRUSH rule 4 x 833 [2,7]
+  CRUSH rule 4 x 834 [5,2]
+  CRUSH rule 4 x 835 [8,5]
+  CRUSH rule 4 x 836 [5]
+  CRUSH rule 4 x 837 [6,5,2]
+  CRUSH rule 4 x 838 [6,7,2]
+  CRUSH rule 4 x 839 [5,2,6]
+  CRUSH rule 4 x 840 [7,8,5]
+  CRUSH rule 4 x 841 [5,8]
+  CRUSH rule 4 x 842 [2,5]
+  CRUSH rule 4 x 843 [6,5,7]
+  CRUSH rule 4 x 844 [5,8,2]
+  CRUSH rule 4 x 845 [5,8,6]
+  CRUSH rule 4 x 846 [5,2]
+  CRUSH rule 4 x 847 [2,7]
+  CRUSH rule 4 x 848 [2,6]
+  CRUSH rule 4 x 849 [5]
+  CRUSH rule 4 x 850 [2,5]
+  CRUSH rule 4 x 851 [6,8,7]
+  CRUSH rule 4 x 852 [7,5,8]
+  CRUSH rule 4 x 853 [6,8,2]
+  CRUSH rule 4 x 854 [7,6,2]
+  CRUSH rule 4 x 855 [5,7,2]
+  CRUSH rule 4 x 856 [6,7,5]
+  CRUSH rule 4 x 857 [8,5,2]
+  CRUSH rule 4 x 858 [6,5,2]
+  CRUSH rule 4 x 859 [6,2,7]
+  CRUSH rule 4 x 860 [5,2]
+  CRUSH rule 4 x 861 [8,7,6]
+  CRUSH rule 4 x 862 [6,2,7]
+  CRUSH rule 4 x 863 [8,7,2]
+  CRUSH rule 4 x 864 [5,6,8]
+  CRUSH rule 4 x 865 [8,2]
+  CRUSH rule 4 x 866 [5,8]
+  CRUSH rule 4 x 867 [6,5,2]
+  CRUSH rule 4 x 868 [6,5,2]
+  CRUSH rule 4 x 869 [8,7,5]
+  CRUSH rule 4 x 870 [2,5,8]
+  CRUSH rule 4 x 871 [5]
+  CRUSH rule 4 x 872 [5,2]
+  CRUSH rule 4 x 873 [5,6]
+  CRUSH rule 4 x 874 [2,6]
+  CRUSH rule 4 x 875 [2,6,5]
+  CRUSH rule 4 x 876 [5,8,2]
+  CRUSH rule 4 x 877 [6,5,2]
+  CRUSH rule 4 x 878 [5,2]
+  CRUSH rule 4 x 879 [7,5,8]
+  CRUSH rule 4 x 880 [5,2]
+  CRUSH rule 4 x 881 [5,6,2]
+  CRUSH rule 4 x 882 [5,2]
+  CRUSH rule 4 x 883 [2]
+  CRUSH rule 4 x 884 [6,2,5]
+  CRUSH rule 4 x 885 [5,2]
+  CRUSH rule 4 x 886 [5,6]
+  CRUSH rule 4 x 887 [7,5,2]
+  CRUSH rule 4 x 888 [6,8,2]
+  CRUSH rule 4 x 889 [2,7]
+  CRUSH rule 4 x 890 [7,2]
+  CRUSH rule 4 x 891 [2,8]
+  CRUSH rule 4 x 892 [6,2,5]
+  CRUSH rule 4 x 893 [2,5,7]
+  CRUSH rule 4 x 894 [7,5,2]
+  CRUSH rule 4 x 895 [5,2]
+  CRUSH rule 4 x 896 [2,8]
+  CRUSH rule 4 x 897 [5,2,6]
+  CRUSH rule 4 x 898 [2,5]
+  CRUSH rule 4 x 899 [2,7,6]
+  CRUSH rule 4 x 900 [5,2]
+  CRUSH rule 4 x 901 [5,2]
+  CRUSH rule 4 x 902 [8,5,7]
+  CRUSH rule 4 x 903 [5,7]
+  CRUSH rule 4 x 904 [5,6,8]
+  CRUSH rule 4 x 905 [6,2,5]
+  CRUSH rule 4 x 906 [2]
+  CRUSH rule 4 x 907 [7,2]
+  CRUSH rule 4 x 908 [5,8,2]
+  CRUSH rule 4 x 909 [2,5]
+  CRUSH rule 4 x 910 [6,5,2]
+  CRUSH rule 4 x 911 [5,8]
+  CRUSH rule 4 x 912 [2,7]
+  CRUSH rule 4 x 913 [7,6,8]
+  CRUSH rule 4 x 914 [6,5,7]
+  CRUSH rule 4 x 915 [8,2,6]
+  CRUSH rule 4 x 916 [5,2]
+  CRUSH rule 4 x 917 [2,5]
+  CRUSH rule 4 x 918 [8,2]
+  CRUSH rule 4 x 919 [6,2,8]
+  CRUSH rule 4 x 920 [7,6,5]
+  CRUSH rule 4 x 921 [2,5]
+  CRUSH rule 4 x 922 [6,7,8]
+  CRUSH rule 4 x 923 [5,6]
+  CRUSH rule 4 x 924 [5]
+  CRUSH rule 4 x 925 [5,7]
+  CRUSH rule 4 x 926 [5]
+  CRUSH rule 4 x 927 [2,6,5]
+  CRUSH rule 4 x 928 [8,2]
+  CRUSH rule 4 x 929 [5,2]
+  CRUSH rule 4 x 930 [2,5,6]
+  CRUSH rule 4 x 931 [5,2]
+  CRUSH rule 4 x 932 [5,2]
+  CRUSH rule 4 x 933 [8,5]
+  CRUSH rule 4 x 934 [5,8]
+  CRUSH rule 4 x 935 [6,5]
+  CRUSH rule 4 x 936 [2,6,7]
+  CRUSH rule 4 x 937 [5]
+  CRUSH rule 4 x 938 [6,5,8]
+  CRUSH rule 4 x 939 [2,7]
+  CRUSH rule 4 x 940 [8,7,6]
+  CRUSH rule 4 x 941 [5,2]
+  CRUSH rule 4 x 942 [2]
+  CRUSH rule 4 x 943 [8,2,5]
+  CRUSH rule 4 x 944 [5,7]
+  CRUSH rule 4 x 945 [7,2,5]
+  CRUSH rule 4 x 946 [2,7]
+  CRUSH rule 4 x 947 [5]
+  CRUSH rule 4 x 948 [7,8,6]
+  CRUSH rule 4 x 949 [6,2,7]
+  CRUSH rule 4 x 950 [5,8]
+  CRUSH rule 4 x 951 [5]
+  CRUSH rule 4 x 952 [2,7]
+  CRUSH rule 4 x 953 [2,5]
+  CRUSH rule 4 x 954 [5,2]
+  CRUSH rule 4 x 955 [8,6,2]
+  CRUSH rule 4 x 956 [2,8]
+  CRUSH rule 4 x 957 [7,6,2]
+  CRUSH rule 4 x 958 [8,7,5]
+  CRUSH rule 4 x 959 [5,2,7]
+  CRUSH rule 4 x 960 [5,6]
+  CRUSH rule 4 x 961 [5,2]
+  CRUSH rule 4 x 962 [7,5]
+  CRUSH rule 4 x 963 [2,5]
+  CRUSH rule 4 x 964 [5,2]
+  CRUSH rule 4 x 965 [7,6,5]
+  CRUSH rule 4 x 966 [5,8]
+  CRUSH rule 4 x 967 [8,6,5]
+  CRUSH rule 4 x 968 [7,2,5]
+  CRUSH rule 4 x 969 [8,2,6]
+  CRUSH rule 4 x 970 [2,6,5]
+  CRUSH rule 4 x 971 [2,7,8]
+  CRUSH rule 4 x 972 [2,8,5]
+  CRUSH rule 4 x 973 [2]
+  CRUSH rule 4 x 974 [5,2]
+  CRUSH rule 4 x 975 [5,7]
+  CRUSH rule 4 x 976 [5]
+  CRUSH rule 4 x 977 [8,5,2]
+  CRUSH rule 4 x 978 [7,2,8]
+  CRUSH rule 4 x 979 [7,6,2]
+  CRUSH rule 4 x 980 [6,2,7]
+  CRUSH rule 4 x 981 [7,5,2]
+  CRUSH rule 4 x 982 [5,2]
+  CRUSH rule 4 x 983 [5,6]
+  CRUSH rule 4 x 984 [2]
+  CRUSH rule 4 x 985 [2,5]
+  CRUSH rule 4 x 986 [8,7,5]
+  CRUSH rule 4 x 987 [2,5]
+  CRUSH rule 4 x 988 [2,5]
+  CRUSH rule 4 x 989 [2,6,5]
+  CRUSH rule 4 x 990 [2,8]
+  CRUSH rule 4 x 991 [2,5]
+  CRUSH rule 4 x 992 [7,2,5]
+  CRUSH rule 4 x 993 [2,6]
+  CRUSH rule 4 x 994 [5]
+  CRUSH rule 4 x 995 [7,6,2]
+  CRUSH rule 4 x 996 [6,7,5]
+  CRUSH rule 4 x 997 [6,5,2]
+  CRUSH rule 4 x 998 [8,2]
+  CRUSH rule 4 x 999 [2,7,8]
+  CRUSH rule 4 x 1000 [8,5,2]
+  CRUSH rule 4 x 1001 [2,5]
+  CRUSH rule 4 x 1002 [2,5]
+  CRUSH rule 4 x 1003 [2,8,7]
+  CRUSH rule 4 x 1004 [6,2]
+  CRUSH rule 4 x 1005 [6,2]
+  CRUSH rule 4 x 1006 [2]
+  CRUSH rule 4 x 1007 [2,5]
+  CRUSH rule 4 x 1008 [2,7]
+  CRUSH rule 4 x 1009 [6,8,5]
+  CRUSH rule 4 x 1010 [5,2]
+  CRUSH rule 4 x 1011 [5,2]
+  CRUSH rule 4 x 1012 [5,2,7]
+  CRUSH rule 4 x 1013 [5,2]
+  CRUSH rule 4 x 1014 [2,8,5]
+  CRUSH rule 4 x 1015 [6,8,5]
+  CRUSH rule 4 x 1016 [2]
+  CRUSH rule 4 x 1017 [6,2]
+  CRUSH rule 4 x 1018 [5]
+  CRUSH rule 4 x 1019 [5,8]
+  CRUSH rule 4 x 1020 [5,2]
+  CRUSH rule 4 x 1021 [5,2]
+  CRUSH rule 4 x 1022 [2,6,7]
+  CRUSH rule 4 x 1023 [5,2]
+  rule 4 (choose-set-two) num_rep 3 result size == 1:\t78/1024 (esc)
+  rule 4 (choose-set-two) num_rep 3 result size == 2:\t480/1024 (esc)
+  rule 4 (choose-set-two) num_rep 3 result size == 3:\t466/1024 (esc)
+  rule 5 (chooseleaf-set), x = 0..1023, numrep = 2..3
+  CRUSH rule 5 x 0 [2,5]
+  CRUSH rule 5 x 1 [2,8]
+  CRUSH rule 5 x 2 [2,5]
+  CRUSH rule 5 x 3 [8,2]
+  CRUSH rule 5 x 4 [5,2]
+  CRUSH rule 5 x 5 [7,2]
+  CRUSH rule 5 x 6 [2,6]
+  CRUSH rule 5 x 7 [5,8]
+  CRUSH rule 5 x 8 [5,6]
+  CRUSH rule 5 x 9 [2,5]
+  CRUSH rule 5 x 10 [2,7]
+  CRUSH rule 5 x 11 [2,7]
+  CRUSH rule 5 x 12 [2,5]
+  CRUSH rule 5 x 13 [5,8]
+  CRUSH rule 5 x 14 [7,2]
+  CRUSH rule 5 x 15 [7,2]
+  CRUSH rule 5 x 16 [5,6]
+  CRUSH rule 5 x 17 [5,2]
+  CRUSH rule 5 x 18 [2,5]
+  CRUSH rule 5 x 19 [7,5]
+  CRUSH rule 5 x 20 [2,5]
+  CRUSH rule 5 x 21 [5,7]
+  CRUSH rule 5 x 22 [8,5]
+  CRUSH rule 5 x 23 [5,6]
+  CRUSH rule 5 x 24 [2,7]
+  CRUSH rule 5 x 25 [5,7]
+  CRUSH rule 5 x 26 [2,8]
+  CRUSH rule 5 x 27 [5,2]
+  CRUSH rule 5 x 28 [6,2]
+  CRUSH rule 5 x 29 [8,5]
+  CRUSH rule 5 x 30 [5,7]
+  CRUSH rule 5 x 31 [8,2]
+  CRUSH rule 5 x 32 [5,6]
+  CRUSH rule 5 x 33 [2,7]
+  CRUSH rule 5 x 34 [2,5]
+  CRUSH rule 5 x 35 [2,8]
+  CRUSH rule 5 x 36 [5,8]
+  CRUSH rule 5 x 37 [2,5]
+  CRUSH rule 5 x 38 [5,8]
+  CRUSH rule 5 x 39 [5,7]
+  CRUSH rule 5 x 40 [7,2]
+  CRUSH rule 5 x 41 [2,6]
+  CRUSH rule 5 x 42 [5,6]
+  CRUSH rule 5 x 43 [2,5]
+  CRUSH rule 5 x 44 [2,6]
+  CRUSH rule 5 x 45 [8,2]
+  CRUSH rule 5 x 46 [2,5]
+  CRUSH rule 5 x 47 [5,2]
+  CRUSH rule 5 x 48 [5,6]
+  CRUSH rule 5 x 49 [5,7]
+  CRUSH rule 5 x 50 [5,2]
+  CRUSH rule 5 x 51 [5,6]
+  CRUSH rule 5 x 52 [8,2]
+  CRUSH rule 5 x 53 [5,8]
+  CRUSH rule 5 x 54 [7,5]
+  CRUSH rule 5 x 55 [8,2]
+  CRUSH rule 5 x 56 [6,5]
+  CRUSH rule 5 x 57 [5,8]
+  CRUSH rule 5 x 58 [2,8]
+  CRUSH rule 5 x 59 [5,2]
+  CRUSH rule 5 x 60 [5,2]
+  CRUSH rule 5 x 61 [5,6]
+  CRUSH rule 5 x 62 [7,2]
+  CRUSH rule 5 x 63 [5,6]
+  CRUSH rule 5 x 64 [5,2]
+  CRUSH rule 5 x 65 [7,5]
+  CRUSH rule 5 x 66 [5,6]
+  CRUSH rule 5 x 67 [5,2]
+  CRUSH rule 5 x 68 [2,5]
+  CRUSH rule 5 x 69 [5,2]
+  CRUSH rule 5 x 70 [7,2]
+  CRUSH rule 5 x 71 [2,8]
+  CRUSH rule 5 x 72 [6,2]
+  CRUSH rule 5 x 73 [2,7]
+  CRUSH rule 5 x 74 [2,7]
+  CRUSH rule 5 x 75 [5,2]
+  CRUSH rule 5 x 76 [5,2]
+  CRUSH rule 5 x 77 [7,2]
+  CRUSH rule 5 x 78 [2,5]
+  CRUSH rule 5 x 79 [5,2]
+  CRUSH rule 5 x 80 [2,5]
+  CRUSH rule 5 x 81 [2,5]
+  CRUSH rule 5 x 82 [7,2]
+  CRUSH rule 5 x 83 [2,6]
+  CRUSH rule 5 x 84 [7,2]
+  CRUSH rule 5 x 85 [5,8]
+  CRUSH rule 5 x 86 [2,6]
+  CRUSH rule 5 x 87 [2,7]
+  CRUSH rule 5 x 88 [2,6]
+  CRUSH rule 5 x 89 [5,2]
+  CRUSH rule 5 x 90 [6,5]
+  CRUSH rule 5 x 91 [5,8]
+  CRUSH rule 5 x 92 [2,8]
+  CRUSH rule 5 x 93 [7,5]
+  CRUSH rule 5 x 94 [2,5]
+  CRUSH rule 5 x 95 [7,5]
+  CRUSH rule 5 x 96 [5,6]
+  CRUSH rule 5 x 97 [8,5]
+  CRUSH rule 5 x 98 [2,7]
+  CRUSH rule 5 x 99 [2,7]
+  CRUSH rule 5 x 100 [2,7]
+  CRUSH rule 5 x 101 [5,7]
+  CRUSH rule 5 x 102 [5,2]
+  CRUSH rule 5 x 103 [5,7]
+  CRUSH rule 5 x 104 [7,5]
+  CRUSH rule 5 x 105 [2,5]
+  CRUSH rule 5 x 106 [2,6]
+  CRUSH rule 5 x 107 [5,2]
+  CRUSH rule 5 x 108 [7,2]
+  CRUSH rule 5 x 109 [2,5]
+  CRUSH rule 5 x 110 [5,2]
+  CRUSH rule 5 x 111 [2,5]
+  CRUSH rule 5 x 112 [2,6]
+  CRUSH rule 5 x 113 [6,2]
+  CRUSH rule 5 x 114 [7,5]
+  CRUSH rule 5 x 115 [8,2]
+  CRUSH rule 5 x 116 [2,6]
+  CRUSH rule 5 x 117 [7,5]
+  CRUSH rule 5 x 118 [2,5]
+  CRUSH rule 5 x 119 [5,6]
+  CRUSH rule 5 x 120 [2,5]
+  CRUSH rule 5 x 121 [2,7]
+  CRUSH rule 5 x 122 [8,5]
+  CRUSH rule 5 x 123 [2,5]
+  CRUSH rule 5 x 124 [5,2]
+  CRUSH rule 5 x 125 [2,7]
+  CRUSH rule 5 x 126 [5,2]
+  CRUSH rule 5 x 127 [5,6]
+  CRUSH rule 5 x 128 [5,6]
+  CRUSH rule 5 x 129 [2,5]
+  CRUSH rule 5 x 130 [5,8]
+  CRUSH rule 5 x 131 [2,5]
+  CRUSH rule 5 x 132 [2,5]
+  CRUSH rule 5 x 133 [5,6]
+  CRUSH rule 5 x 134 [2,8]
+  CRUSH rule 5 x 135 [5,6]
+  CRUSH rule 5 x 136 [2,5]
+  CRUSH rule 5 x 137 [7,5]
+  CRUSH rule 5 x 138 [8,5]
+  CRUSH rule 5 x 139 [5,2]
+  CRUSH rule 5 x 140 [2,6]
+  CRUSH rule 5 x 141 [6,2]
+  CRUSH rule 5 x 142 [5,2]
+  CRUSH rule 5 x 143 [5,8]
+  CRUSH rule 5 x 144 [8,2]
+  CRUSH rule 5 x 145 [8,5]
+  CRUSH rule 5 x 146 [2,6]
+  CRUSH rule 5 x 147 [2,8]
+  CRUSH rule 5 x 148 [5,2]
+  CRUSH rule 5 x 149 [5,8]
+  CRUSH rule 5 x 150 [2,6]
+  CRUSH rule 5 x 151 [5,6]
+  CRUSH rule 5 x 152 [8,5]
+  CRUSH rule 5 x 153 [8,5]
+  CRUSH rule 5 x 154 [5,2]
+  CRUSH rule 5 x 155 [5,7]
+  CRUSH rule 5 x 156 [5,2]
+  CRUSH rule 5 x 157 [5,2]
+  CRUSH rule 5 x 158 [2,8]
+  CRUSH rule 5 x 159 [7,2]
+  CRUSH rule 5 x 160 [2,8]
+  CRUSH rule 5 x 161 [2,5]
+  CRUSH rule 5 x 162 [2,6]
+  CRUSH rule 5 x 163 [5,6]
+  CRUSH rule 5 x 164 [7,2]
+  CRUSH rule 5 x 165 [7,2]
+  CRUSH rule 5 x 166 [2,5]
+  CRUSH rule 5 x 167 [2,7]
+  CRUSH rule 5 x 168 [5,2]
+  CRUSH rule 5 x 169 [2,6]
+  CRUSH rule 5 x 170 [2,5]
+  CRUSH rule 5 x 171 [7,5]
+  CRUSH rule 5 x 172 [2,7]
+  CRUSH rule 5 x 173 [8,5]
+  CRUSH rule 5 x 174 [2,5]
+  CRUSH rule 5 x 175 [6,2]
+  CRUSH rule 5 x 176 [5,2]
+  CRUSH rule 5 x 177 [5,2]
+  CRUSH rule 5 x 178 [5,2]
+  CRUSH rule 5 x 179 [5,2]
+  CRUSH rule 5 x 180 [5,8]
+  CRUSH rule 5 x 181 [6,2]
+  CRUSH rule 5 x 182 [8,5]
+  CRUSH rule 5 x 183 [7,5]
+  CRUSH rule 5 x 184 [5,7]
+  CRUSH rule 5 x 185 [6,2]
+  CRUSH rule 5 x 186 [2,5]
+  CRUSH rule 5 x 187 [2,6]
+  CRUSH rule 5 x 188 [2,8]
+  CRUSH rule 5 x 189 [2,7]
+  CRUSH rule 5 x 190 [5,2]
+  CRUSH rule 5 x 191 [7,2]
+  CRUSH rule 5 x 192 [5,2]
+  CRUSH rule 5 x 193 [5,2]
+  CRUSH rule 5 x 194 [2,5]
+  CRUSH rule 5 x 195 [6,5]
+  CRUSH rule 5 x 196 [6,2]
+  CRUSH rule 5 x 197 [6,5]
+  CRUSH rule 5 x 198 [2,5]
+  CRUSH rule 5 x 199 [2,5]
+  CRUSH rule 5 x 200 [2,5]
+  CRUSH rule 5 x 201 [7,2]
+  CRUSH rule 5 x 202 [6,5]
+  CRUSH rule 5 x 203 [5,8]
+  CRUSH rule 5 x 204 [2,5]
+  CRUSH rule 5 x 205 [2,7]
+  CRUSH rule 5 x 206 [2,7]
+  CRUSH rule 5 x 207 [5,2]
+  CRUSH rule 5 x 208 [7,2]
+  CRUSH rule 5 x 209 [2,8]
+  CRUSH rule 5 x 210 [2,5]
+  CRUSH rule 5 x 211 [5,2]
+  CRUSH rule 5 x 212 [7,5]
+  CRUSH rule 5 x 213 [8,5]
+  CRUSH rule 5 x 214 [5,8]
+  CRUSH rule 5 x 215 [8,2]
+  CRUSH rule 5 x 216 [5,2]
+  CRUSH rule 5 x 217 [2,7]
+  CRUSH rule 5 x 218 [2,7]
+  CRUSH rule 5 x 219 [5,8]
+  CRUSH rule 5 x 220 [5,7]
+  CRUSH rule 5 x 221 [5,6]
+  CRUSH rule 5 x 222 [6,5]
+  CRUSH rule 5 x 223 [2,5]
+  CRUSH rule 5 x 224 [2,5]
+  CRUSH rule 5 x 225 [8,2]
+  CRUSH rule 5 x 226 [7,2]
+  CRUSH rule 5 x 227 [5,2]
+  CRUSH rule 5 x 228 [5,6]
+  CRUSH rule 5 x 229 [5,8]
+  CRUSH rule 5 x 230 [5,7]
+  CRUSH rule 5 x 231 [5,7]
+  CRUSH rule 5 x 232 [2,7]
+  CRUSH rule 5 x 233 [5,7]
+  CRUSH rule 5 x 234 [2,5]
+  CRUSH rule 5 x 235 [5,8]
+  CRUSH rule 5 x 236 [5,2]
+  CRUSH rule 5 x 237 [5,7]
+  CRUSH rule 5 x 238 [5,2]
+  CRUSH rule 5 x 239 [8,5]
+  CRUSH rule 5 x 240 [5,7]
+  CRUSH rule 5 x 241 [5,2]
+  CRUSH rule 5 x 242 [5,2]
+  CRUSH rule 5 x 243 [5,7]
+  CRUSH rule 5 x 244 [5,6]
+  CRUSH rule 5 x 245 [7,2]
+  CRUSH rule 5 x 246 [2,5]
+  CRUSH rule 5 x 247 [6,2]
+  CRUSH rule 5 x 248 [8,2]
+  CRUSH rule 5 x 249 [2,5]
+  CRUSH rule 5 x 250 [2,5]
+  CRUSH rule 5 x 251 [2,5]
+  CRUSH rule 5 x 252 [5,7]
+  CRUSH rule 5 x 253 [5,2]
+  CRUSH rule 5 x 254 [5,2]
+  CRUSH rule 5 x 255 [2,7]
+  CRUSH rule 5 x 256 [5,7]
+  CRUSH rule 5 x 257 [2,8]
+  CRUSH rule 5 x 258 [5,2]
+  CRUSH rule 5 x 259 [5,6]
+  CRUSH rule 5 x 260 [5,6]
+  CRUSH rule 5 x 261 [8,5]
+  CRUSH rule 5 x 262 [5,6]
+  CRUSH rule 5 x 263 [6,2]
+  CRUSH rule 5 x 264 [5,6]
+  CRUSH rule 5 x 265 [8,5]
+  CRUSH rule 5 x 266 [8,2]
+  CRUSH rule 5 x 267 [2,5]
+  CRUSH rule 5 x 268 [2,7]
+  CRUSH rule 5 x 269 [2,8]
+  CRUSH rule 5 x 270 [5,2]
+  CRUSH rule 5 x 271 [7,5]
+  CRUSH rule 5 x 272 [2,8]
+  CRUSH rule 5 x 273 [5,2]
+  CRUSH rule 5 x 274 [6,5]
+  CRUSH rule 5 x 275 [5,7]
+  CRUSH rule 5 x 276 [7,2]
+  CRUSH rule 5 x 277 [6,5]
+  CRUSH rule 5 x 278 [6,2]
+  CRUSH rule 5 x 279 [8,5]
+  CRUSH rule 5 x 280 [2,6]
+  CRUSH rule 5 x 281 [8,2]
+  CRUSH rule 5 x 282 [5,2]
+  CRUSH rule 5 x 283 [8,2]
+  CRUSH rule 5 x 284 [6,5]
+  CRUSH rule 5 x 285 [5,7]
+  CRUSH rule 5 x 286 [2,6]
+  CRUSH rule 5 x 287 [2,5]
+  CRUSH rule 5 x 288 [8,2]
+  CRUSH rule 5 x 289 [5,6]
+  CRUSH rule 5 x 290 [2,5]
+  CRUSH rule 5 x 291 [2,5]
+  CRUSH rule 5 x 292 [8,2]
+  CRUSH rule 5 x 293 [6,2]
+  CRUSH rule 5 x 294 [7,5]
+  CRUSH rule 5 x 295 [5,8]
+  CRUSH rule 5 x 296 [5,2]
+  CRUSH rule 5 x 297 [6,2]
+  CRUSH rule 5 x 298 [2,5]
+  CRUSH rule 5 x 299 [2,8]
+  CRUSH rule 5 x 300 [8,5]
+  CRUSH rule 5 x 301 [2,8]
+  CRUSH rule 5 x 302 [5,2]
+  CRUSH rule 5 x 303 [7,5]
+  CRUSH rule 5 x 304 [2,7]
+  CRUSH rule 5 x 305 [5,8]
+  CRUSH rule 5 x 306 [2,7]
+  CRUSH rule 5 x 307 [2,7]
+  CRUSH rule 5 x 308 [2,8]
+  CRUSH rule 5 x 309 [7,5]
+  CRUSH rule 5 x 310 [5,2]
+  CRUSH rule 5 x 311 [5,6]
+  CRUSH rule 5 x 312 [2,6]
+  CRUSH rule 5 x 313 [5,2]
+  CRUSH rule 5 x 314 [5,2]
+  CRUSH rule 5 x 315 [2,5]
+  CRUSH rule 5 x 316 [6,5]
+  CRUSH rule 5 x 317 [2,6]
+  CRUSH rule 5 x 318 [8,2]
+  CRUSH rule 5 x 319 [5,2]
+  CRUSH rule 5 x 320 [5,7]
+  CRUSH rule 5 x 321 [2,5]
+  CRUSH rule 5 x 322 [2,7]
+  CRUSH rule 5 x 323 [5,7]
+  CRUSH rule 5 x 324 [7,2]
+  CRUSH rule 5 x 325 [5,6]
+  CRUSH rule 5 x 326 [5,2]
+  CRUSH rule 5 x 327 [2,6]
+  CRUSH rule 5 x 328 [7,5]
+  CRUSH rule 5 x 329 [5,6]
+  CRUSH rule 5 x 330 [5,7]
+  CRUSH rule 5 x 331 [2,6]
+  CRUSH rule 5 x 332 [2,5]
+  CRUSH rule 5 x 333 [6,5]
+  CRUSH rule 5 x 334 [8,5]
+  CRUSH rule 5 x 335 [7,2]
+  CRUSH rule 5 x 336 [5,6]
+  CRUSH rule 5 x 337 [7,2]
+  CRUSH rule 5 x 338 [5,6]
+  CRUSH rule 5 x 339 [7,5]
+  CRUSH rule 5 x 340 [2,8]
+  CRUSH rule 5 x 341 [5,2]
+  CRUSH rule 5 x 342 [2,7]
+  CRUSH rule 5 x 343 [6,5]
+  CRUSH rule 5 x 344 [6,2]
+  CRUSH rule 5 x 345 [5,7]
+  CRUSH rule 5 x 346 [8,2]
+  CRUSH rule 5 x 347 [5,2]
+  CRUSH rule 5 x 348 [8,2]
+  CRUSH rule 5 x 349 [2,6]
+  CRUSH rule 5 x 350 [8,5]
+  CRUSH rule 5 x 351 [5,6]
+  CRUSH rule 5 x 352 [2,8]
+  CRUSH rule 5 x 353 [6,5]
+  CRUSH rule 5 x 354 [2,5]
+  CRUSH rule 5 x 355 [5,8]
+  CRUSH rule 5 x 356 [5,2]
+  CRUSH rule 5 x 357 [6,2]
+  CRUSH rule 5 x 358 [2,8]
+  CRUSH rule 5 x 359 [6,2]
+  CRUSH rule 5 x 360 [5,2]
+  CRUSH rule 5 x 361 [8,5]
+  CRUSH rule 5 x 362 [5,2]
+  CRUSH rule 5 x 363 [5,2]
+  CRUSH rule 5 x 364 [2,5]
+  CRUSH rule 5 x 365 [6,5]
+  CRUSH rule 5 x 366 [7,2]
+  CRUSH rule 5 x 367 [5,2]
+  CRUSH rule 5 x 368 [7,5]
+  CRUSH rule 5 x 369 [5,7]
+  CRUSH rule 5 x 370 [8,2]
+  CRUSH rule 5 x 371 [2,5]
+  CRUSH rule 5 x 372 [5,2]
+  CRUSH rule 5 x 373 [2,6]
+  CRUSH rule 5 x 374 [5,8]
+  CRUSH rule 5 x 375 [6,5]
+  CRUSH rule 5 x 376 [7,2]
+  CRUSH rule 5 x 377 [2,5]
+  CRUSH rule 5 x 378 [2,8]
+  CRUSH rule 5 x 379 [8,5]
+  CRUSH rule 5 x 380 [2,5]
+  CRUSH rule 5 x 381 [2,5]
+  CRUSH rule 5 x 382 [2,5]
+  CRUSH rule 5 x 383 [5,6]
+  CRUSH rule 5 x 384 [7,2]
+  CRUSH rule 5 x 385 [7,5]
+  CRUSH rule 5 x 386 [2,5]
+  CRUSH rule 5 x 387 [2,5]
+  CRUSH rule 5 x 388 [5,2]
+  CRUSH rule 5 x 389 [2,5]
+  CRUSH rule 5 x 390 [5,6]
+  CRUSH rule 5 x 391 [5,6]
+  CRUSH rule 5 x 392 [2,8]
+  CRUSH rule 5 x 393 [5,2]
+  CRUSH rule 5 x 394 [5,7]
+  CRUSH rule 5 x 395 [5,2]
+  CRUSH rule 5 x 396 [5,2]
+  CRUSH rule 5 x 397 [2,5]
+  CRUSH rule 5 x 398 [2,5]
+  CRUSH rule 5 x 399 [8,5]
+  CRUSH rule 5 x 400 [8,2]
+  CRUSH rule 5 x 401 [2,5]
+  CRUSH rule 5 x 402 [7,5]
+  CRUSH rule 5 x 403 [2,5]
+  CRUSH rule 5 x 404 [5,2]
+  CRUSH rule 5 x 405 [6,5]
+  CRUSH rule 5 x 406 [2,6]
+  CRUSH rule 5 x 407 [2,8]
+  CRUSH rule 5 x 408 [5,2]
+  CRUSH rule 5 x 409 [7,5]
+  CRUSH rule 5 x 410 [8,5]
+  CRUSH rule 5 x 411 [2,8]
+  CRUSH rule 5 x 412 [2,5]
+  CRUSH rule 5 x 413 [5,2]
+  CRUSH rule 5 x 414 [5,2]
+  CRUSH rule 5 x 415 [2,6]
+  CRUSH rule 5 x 416 [2,6]
+  CRUSH rule 5 x 417 [8,2]
+  CRUSH rule 5 x 418 [7,2]
+  CRUSH rule 5 x 419 [8,5]
+  CRUSH rule 5 x 420 [2,5]
+  CRUSH rule 5 x 421 [8,5]
+  CRUSH rule 5 x 422 [6,5]
+  CRUSH rule 5 x 423 [2,5]
+  CRUSH rule 5 x 424 [8,5]
+  CRUSH rule 5 x 425 [2,5]
+  CRUSH rule 5 x 426 [6,2]
+  CRUSH rule 5 x 427 [2,7]
+  CRUSH rule 5 x 428 [5,7]
+  CRUSH rule 5 x 429 [5,6]
+  CRUSH rule 5 x 430 [5,6]
+  CRUSH rule 5 x 431 [5,2]
+  CRUSH rule 5 x 432 [7,2]
+  CRUSH rule 5 x 433 [6,5]
+  CRUSH rule 5 x 434 [5,2]
+  CRUSH rule 5 x 435 [2,5]
+  CRUSH rule 5 x 436 [5,2]
+  CRUSH rule 5 x 437 [7,5]
+  CRUSH rule 5 x 438 [2,5]
+  CRUSH rule 5 x 439 [2,5]
+  CRUSH rule 5 x 440 [2,7]
+  CRUSH rule 5 x 441 [5,7]
+  CRUSH rule 5 x 442 [2,5]
+  CRUSH rule 5 x 443 [6,2]
+  CRUSH rule 5 x 444 [7,2]
+  CRUSH rule 5 x 445 [6,5]
+  CRUSH rule 5 x 446 [5,2]
+  CRUSH rule 5 x 447 [2,5]
+  CRUSH rule 5 x 448 [7,2]
+  CRUSH rule 5 x 449 [7,5]
+  CRUSH rule 5 x 450 [5,2]
+  CRUSH rule 5 x 451 [6,5]
+  CRUSH rule 5 x 452 [8,5]
+  CRUSH rule 5 x 453 [6,5]
+  CRUSH rule 5 x 454 [6,5]
+  CRUSH rule 5 x 455 [2,7]
+  CRUSH rule 5 x 456 [6,2]
+  CRUSH rule 5 x 457 [7,2]
+  CRUSH rule 5 x 458 [2,8]
+  CRUSH rule 5 x 459 [2,7]
+  CRUSH rule 5 x 460 [6,5]
+  CRUSH rule 5 x 461 [6,5]
+  CRUSH rule 5 x 462 [8,2]
+  CRUSH rule 5 x 463 [6,2]
+  CRUSH rule 5 x 464 [7,5]
+  CRUSH rule 5 x 465 [7,2]
+  CRUSH rule 5 x 466 [5,8]
+  CRUSH rule 5 x 467 [6,5]
+  CRUSH rule 5 x 468 [7,2]
+  CRUSH rule 5 x 469 [7,2]
+  CRUSH rule 5 x 470 [5,2]
+  CRUSH rule 5 x 471 [2,7]
+  CRUSH rule 5 x 472 [5,2]
+  CRUSH rule 5 x 473 [2,5]
+  CRUSH rule 5 x 474 [6,2]
+  CRUSH rule 5 x 475 [6,2]
+  CRUSH rule 5 x 476 [5,6]
+  CRUSH rule 5 x 477 [5,8]
+  CRUSH rule 5 x 478 [6,2]
+  CRUSH rule 5 x 479 [2,5]
+  CRUSH rule 5 x 480 [2,8]
+  CRUSH rule 5 x 481 [2,5]
+  CRUSH rule 5 x 482 [5,7]
+  CRUSH rule 5 x 483 [2,6]
+  CRUSH rule 5 x 484 [2,7]
+  CRUSH rule 5 x 485 [5,7]
+  CRUSH rule 5 x 486 [5,2]
+  CRUSH rule 5 x 487 [5,2]
+  CRUSH rule 5 x 488 [5,7]
+  CRUSH rule 5 x 489 [2,8]
+  CRUSH rule 5 x 490 [6,5]
+  CRUSH rule 5 x 491 [2,7]
+  CRUSH rule 5 x 492 [6,5]
+  CRUSH rule 5 x 493 [2,7]
+  CRUSH rule 5 x 494 [2,7]
+  CRUSH rule 5 x 495 [5,2]
+  CRUSH rule 5 x 496 [7,5]
+  CRUSH rule 5 x 497 [5,7]
+  CRUSH rule 5 x 498 [2,5]
+  CRUSH rule 5 x 499 [8,5]
+  CRUSH rule 5 x 500 [5,6]
+  CRUSH rule 5 x 501 [2,7]
+  CRUSH rule 5 x 502 [7,2]
+  CRUSH rule 5 x 503 [2,5]
+  CRUSH rule 5 x 504 [5,6]
+  CRUSH rule 5 x 505 [2,7]
+  CRUSH rule 5 x 506 [5,2]
+  CRUSH rule 5 x 507 [6,2]
+  CRUSH rule 5 x 508 [2,5]
+  CRUSH rule 5 x 509 [7,5]
+  CRUSH rule 5 x 510 [6,2]
+  CRUSH rule 5 x 511 [5,8]
+  CRUSH rule 5 x 512 [7,2]
+  CRUSH rule 5 x 513 [7,2]
+  CRUSH rule 5 x 514 [5,6]
+  CRUSH rule 5 x 515 [8,5]
+  CRUSH rule 5 x 516 [5,2]
+  CRUSH rule 5 x 517 [7,2]
+  CRUSH rule 5 x 518 [5,6]
+  CRUSH rule 5 x 519 [7,5]
+  CRUSH rule 5 x 520 [2,6]
+  CRUSH rule 5 x 521 [8,2]
+  CRUSH rule 5 x 522 [6,2]
+  CRUSH rule 5 x 523 [5,2]
+  CRUSH rule 5 x 524 [2,5]
+  CRUSH rule 5 x 525 [2,5]
+  CRUSH rule 5 x 526 [2,5]
+  CRUSH rule 5 x 527 [2,5]
+  CRUSH rule 5 x 528 [5,2]
+  CRUSH rule 5 x 529 [5,7]
+  CRUSH rule 5 x 530 [6,5]
+  CRUSH rule 5 x 531 [6,2]
+  CRUSH rule 5 x 532 [6,5]
+  CRUSH rule 5 x 533 [5,6]
+  CRUSH rule 5 x 534 [7,5]
+  CRUSH rule 5 x 535 [8,2]
+  CRUSH rule 5 x 536 [6,2]
+  CRUSH rule 5 x 537 [5,7]
+  CRUSH rule 5 x 538 [6,5]
+  CRUSH rule 5 x 539 [8,5]
+  CRUSH rule 5 x 540 [2,6]
+  CRUSH rule 5 x 541 [2,5]
+  CRUSH rule 5 x 542 [5,2]
+  CRUSH rule 5 x 543 [6,2]
+  CRUSH rule 5 x 544 [5,7]
+  CRUSH rule 5 x 545 [5,7]
+  CRUSH rule 5 x 546 [6,2]
+  CRUSH rule 5 x 547 [8,2]
+  CRUSH rule 5 x 548 [5,2]
+  CRUSH rule 5 x 549 [5,8]
+  CRUSH rule 5 x 550 [2,5]
+  CRUSH rule 5 x 551 [7,5]
+  CRUSH rule 5 x 552 [5,8]
+  CRUSH rule 5 x 553 [5,2]
+  CRUSH rule 5 x 554 [2,8]
+  CRUSH rule 5 x 555 [5,2]
+  CRUSH rule 5 x 556 [5,6]
+  CRUSH rule 5 x 557 [7,5]
+  CRUSH rule 5 x 558 [5,2]
+  CRUSH rule 5 x 559 [5,2]
+  CRUSH rule 5 x 560 [8,5]
+  CRUSH rule 5 x 561 [6,5]
+  CRUSH rule 5 x 562 [5,2]
+  CRUSH rule 5 x 563 [2,6]
+  CRUSH rule 5 x 564 [5,2]
+  CRUSH rule 5 x 565 [5,6]
+  CRUSH rule 5 x 566 [5,7]
+  CRUSH rule 5 x 567 [5,6]
+  CRUSH rule 5 x 568 [7,5]
+  CRUSH rule 5 x 569 [5,2]
+  CRUSH rule 5 x 570 [2,5]
+  CRUSH rule 5 x 571 [5,7]
+  CRUSH rule 5 x 572 [5,2]
+  CRUSH rule 5 x 573 [5,2]
+  CRUSH rule 5 x 574 [2,5]
+  CRUSH rule 5 x 575 [8,2]
+  CRUSH rule 5 x 576 [5,6]
+  CRUSH rule 5 x 577 [8,2]
+  CRUSH rule 5 x 578 [6,2]
+  CRUSH rule 5 x 579 [5,2]
+  CRUSH rule 5 x 580 [5,2]
+  CRUSH rule 5 x 581 [7,2]
+  CRUSH rule 5 x 582 [2,8]
+  CRUSH rule 5 x 583 [6,2]
+  CRUSH rule 5 x 584 [8,2]
+  CRUSH rule 5 x 585 [7,2]
+  CRUSH rule 5 x 586 [2,7]
+  CRUSH rule 5 x 587 [2,5]
+  CRUSH rule 5 x 588 [5,7]
+  CRUSH rule 5 x 589 [7,2]
+  CRUSH rule 5 x 590 [6,2]
+  CRUSH rule 5 x 591 [5,2]
+  CRUSH rule 5 x 592 [2,5]
+  CRUSH rule 5 x 593 [2,8]
+  CRUSH rule 5 x 594 [2,7]
+  CRUSH rule 5 x 595 [7,2]
+  CRUSH rule 5 x 596 [5,2]
+  CRUSH rule 5 x 597 [5,2]
+  CRUSH rule 5 x 598 [5,2]
+  CRUSH rule 5 x 599 [5,2]
+  CRUSH rule 5 x 600 [7,2]
+  CRUSH rule 5 x 601 [2,7]
+  CRUSH rule 5 x 602 [5,7]
+  CRUSH rule 5 x 603 [5,2]
+  CRUSH rule 5 x 604 [7,5]
+  CRUSH rule 5 x 605 [5,2]
+  CRUSH rule 5 x 606 [2,7]
+  CRUSH rule 5 x 607 [2,5]
+  CRUSH rule 5 x 608 [5,2]
+  CRUSH rule 5 x 609 [5,2]
+  CRUSH rule 5 x 610 [5,7]
+  CRUSH rule 5 x 611 [2,8]
+  CRUSH rule 5 x 612 [2,6]
+  CRUSH rule 5 x 613 [7,2]
+  CRUSH rule 5 x 614 [7,2]
+  CRUSH rule 5 x 615 [6,2]
+  CRUSH rule 5 x 616 [2,8]
+  CRUSH rule 5 x 617 [6,2]
+  CRUSH rule 5 x 618 [7,5]
+  CRUSH rule 5 x 619 [5,2]
+  CRUSH rule 5 x 620 [5,2]
+  CRUSH rule 5 x 621 [5,8]
+  CRUSH rule 5 x 622 [2,5]
+  CRUSH rule 5 x 623 [2,6]
+  CRUSH rule 5 x 624 [5,2]
+  CRUSH rule 5 x 625 [2,5]
+  CRUSH rule 5 x 626 [7,2]
+  CRUSH rule 5 x 627 [2,7]
+  CRUSH rule 5 x 628 [8,2]
+  CRUSH rule 5 x 629 [2,6]
+  CRUSH rule 5 x 630 [2,7]
+  CRUSH rule 5 x 631 [2,7]
+  CRUSH rule 5 x 632 [7,2]
+  CRUSH rule 5 x 633 [8,5]
+  CRUSH rule 5 x 634 [2,5]
+  CRUSH rule 5 x 635 [5,6]
+  CRUSH rule 5 x 636 [2,5]
+  CRUSH rule 5 x 637 [5,2]
+  CRUSH rule 5 x 638 [6,2]
+  CRUSH rule 5 x 639 [5,2]
+  CRUSH rule 5 x 640 [5,2]
+  CRUSH rule 5 x 641 [7,2]
+  CRUSH rule 5 x 642 [2,7]
+  CRUSH rule 5 x 643 [5,2]
+  CRUSH rule 5 x 644 [8,2]
+  CRUSH rule 5 x 645 [5,7]
+  CRUSH rule 5 x 646 [8,2]
+  CRUSH rule 5 x 647 [7,2]
+  CRUSH rule 5 x 648 [2,6]
+  CRUSH rule 5 x 649 [5,7]
+  CRUSH rule 5 x 650 [7,5]
+  CRUSH rule 5 x 651 [5,7]
+  CRUSH rule 5 x 652 [5,7]
+  CRUSH rule 5 x 653 [8,5]
+  CRUSH rule 5 x 654 [7,5]
+  CRUSH rule 5 x 655 [2,5]
+  CRUSH rule 5 x 656 [5,7]
+  CRUSH rule 5 x 657 [6,2]
+  CRUSH rule 5 x 658 [5,6]
+  CRUSH rule 5 x 659 [5,6]
+  CRUSH rule 5 x 660 [7,5]
+  CRUSH rule 5 x 661 [2,8]
+  CRUSH rule 5 x 662 [5,2]
+  CRUSH rule 5 x 663 [2,5]
+  CRUSH rule 5 x 664 [2,5]
+  CRUSH rule 5 x 665 [5,7]
+  CRUSH rule 5 x 666 [2,8]
+  CRUSH rule 5 x 667 [2,5]
+  CRUSH rule 5 x 668 [5,7]
+  CRUSH rule 5 x 669 [6,5]
+  CRUSH rule 5 x 670 [5,2]
+  CRUSH rule 5 x 671 [2,7]
+  CRUSH rule 5 x 672 [5,2]
+  CRUSH rule 5 x 673 [5,2]
+  CRUSH rule 5 x 674 [5,2]
+  CRUSH rule 5 x 675 [2,8]
+  CRUSH rule 5 x 676 [2,5]
+  CRUSH rule 5 x 677 [5,2]
+  CRUSH rule 5 x 678 [2,5]
+  CRUSH rule 5 x 679 [6,2]
+  CRUSH rule 5 x 680 [2,5]
+  CRUSH rule 5 x 681 [5,7]
+  CRUSH rule 5 x 682 [2,5]
+  CRUSH rule 5 x 683 [2,5]
+  CRUSH rule 5 x 684 [7,2]
+  CRUSH rule 5 x 685 [7,2]
+  CRUSH rule 5 x 686 [2,5]
+  CRUSH rule 5 x 687 [5,6]
+  CRUSH rule 5 x 688 [5,7]
+  CRUSH rule 5 x 689 [6,5]
+  CRUSH rule 5 x 690 [8,2]
+  CRUSH rule 5 x 691 [5,2]
+  CRUSH rule 5 x 692 [7,2]
+  CRUSH rule 5 x 693 [6,5]
+  CRUSH rule 5 x 694 [6,5]
+  CRUSH rule 5 x 695 [2,8]
+  CRUSH rule 5 x 696 [2,5]
+  CRUSH rule 5 x 697 [6,2]
+  CRUSH rule 5 x 698 [6,2]
+  CRUSH rule 5 x 699 [2,6]
+  CRUSH rule 5 x 700 [2,5]
+  CRUSH rule 5 x 701 [5,2]
+  CRUSH rule 5 x 702 [5,2]
+  CRUSH rule 5 x 703 [8,5]
+  CRUSH rule 5 x 704 [2,5]
+  CRUSH rule 5 x 705 [8,2]
+  CRUSH rule 5 x 706 [2,5]
+  CRUSH rule 5 x 707 [7,5]
+  CRUSH rule 5 x 708 [5,7]
+  CRUSH rule 5 x 709 [6,5]
+  CRUSH rule 5 x 710 [8,5]
+  CRUSH rule 5 x 711 [2,5]
+  CRUSH rule 5 x 712 [2,5]
+  CRUSH rule 5 x 713 [6,5]
+  CRUSH rule 5 x 714 [5,2]
+  CRUSH rule 5 x 715 [2,5]
+  CRUSH rule 5 x 716 [5,6]
+  CRUSH rule 5 x 717 [8,2]
+  CRUSH rule 5 x 718 [5,7]
+  CRUSH rule 5 x 719 [2,6]
+  CRUSH rule 5 x 720 [6,2]
+  CRUSH rule 5 x 721 [5,7]
+  CRUSH rule 5 x 722 [5,7]
+  CRUSH rule 5 x 723 [5,2]
+  CRUSH rule 5 x 724 [2,6]
+  CRUSH rule 5 x 725 [2,5]
+  CRUSH rule 5 x 726 [5,8]
+  CRUSH rule 5 x 727 [5,6]
+  CRUSH rule 5 x 728 [2,7]
+  CRUSH rule 5 x 729 [5,6]
+  CRUSH rule 5 x 730 [5,7]
+  CRUSH rule 5 x 731 [5,2]
+  CRUSH rule 5 x 732 [2,5]
+  CRUSH rule 5 x 733 [5,7]
+  CRUSH rule 5 x 734 [6,5]
+  CRUSH rule 5 x 735 [5,8]
+  CRUSH rule 5 x 736 [5,8]
+  CRUSH rule 5 x 737 [2,6]
+  CRUSH rule 5 x 738 [5,2]
+  CRUSH rule 5 x 739 [2,7]
+  CRUSH rule 5 x 740 [2,8]
+  CRUSH rule 5 x 741 [7,2]
+  CRUSH rule 5 x 742 [8,2]
+  CRUSH rule 5 x 743 [7,2]
+  CRUSH rule 5 x 744 [5,7]
+  CRUSH rule 5 x 745 [5,2]
+  CRUSH rule 5 x 746 [5,2]
+  CRUSH rule 5 x 747 [6,2]
+  CRUSH rule 5 x 748 [2,7]
+  CRUSH rule 5 x 749 [5,8]
+  CRUSH rule 5 x 750 [2,6]
+  CRUSH rule 5 x 751 [2,8]
+  CRUSH rule 5 x 752 [8,2]
+  CRUSH rule 5 x 753 [7,5]
+  CRUSH rule 5 x 754 [8,5]
+  CRUSH rule 5 x 755 [2,6]
+  CRUSH rule 5 x 756 [5,6]
+  CRUSH rule 5 x 757 [8,2]
+  CRUSH rule 5 x 758 [6,2]
+  CRUSH rule 5 x 759 [8,5]
+  CRUSH rule 5 x 760 [2,5]
+  CRUSH rule 5 x 761 [5,2]
+  CRUSH rule 5 x 762 [2,7]
+  CRUSH rule 5 x 763 [8,5]
+  CRUSH rule 5 x 764 [2,7]
+  CRUSH rule 5 x 765 [6,5]
+  CRUSH rule 5 x 766 [8,5]
+  CRUSH rule 5 x 767 [2,8]
+  CRUSH rule 5 x 768 [8,5]
+  CRUSH rule 5 x 769 [6,2]
+  CRUSH rule 5 x 770 [6,2]
+  CRUSH rule 5 x 771 [7,2]
+  CRUSH rule 5 x 772 [8,5]
+  CRUSH rule 5 x 773 [5,2]
+  CRUSH rule 5 x 774 [5,6]
+  CRUSH rule 5 x 775 [6,5]
+  CRUSH rule 5 x 776 [7,2]
+  CRUSH rule 5 x 777 [5,2]
+  CRUSH rule 5 x 778 [2,8]
+  CRUSH rule 5 x 779 [2,7]
+  CRUSH rule 5 x 780 [2,5]
+  CRUSH rule 5 x 781 [6,5]
+  CRUSH rule 5 x 782 [5,2]
+  CRUSH rule 5 x 783 [7,2]
+  CRUSH rule 5 x 784 [2,5]
+  CRUSH rule 5 x 785 [6,2]
+  CRUSH rule 5 x 786 [7,5]
+  CRUSH rule 5 x 787 [2,6]
+  CRUSH rule 5 x 788 [6,2]
+  CRUSH rule 5 x 789 [2,5]
+  CRUSH rule 5 x 790 [8,5]
+  CRUSH rule 5 x 791 [5,8]
+  CRUSH rule 5 x 792 [5,8]
+  CRUSH rule 5 x 793 [6,2]
+  CRUSH rule 5 x 794 [2,6]
+  CRUSH rule 5 x 795 [2,5]
+  CRUSH rule 5 x 796 [5,7]
+  CRUSH rule 5 x 797 [2,5]
+  CRUSH rule 5 x 798 [6,2]
+  CRUSH rule 5 x 799 [5,2]
+  CRUSH rule 5 x 800 [5,2]
+  CRUSH rule 5 x 801 [5,6]
+  CRUSH rule 5 x 802 [2,8]
+  CRUSH rule 5 x 803 [2,5]
+  CRUSH rule 5 x 804 [6,2]
+  CRUSH rule 5 x 805 [5,6]
+  CRUSH rule 5 x 806 [2,5]
+  CRUSH rule 5 x 807 [5,7]
+  CRUSH rule 5 x 808 [5,6]
+  CRUSH rule 5 x 809 [2,5]
+  CRUSH rule 5 x 810 [5,7]
+  CRUSH rule 5 x 811 [8,5]
+  CRUSH rule 5 x 812 [8,5]
+  CRUSH rule 5 x 813 [6,5]
+  CRUSH rule 5 x 814 [5,6]
+  CRUSH rule 5 x 815 [5,2]
+  CRUSH rule 5 x 816 [2,7]
+  CRUSH rule 5 x 817 [5,8]
+  CRUSH rule 5 x 818 [5,2]
+  CRUSH rule 5 x 819 [5,2]
+  CRUSH rule 5 x 820 [5,6]
+  CRUSH rule 5 x 821 [5,6]
+  CRUSH rule 5 x 822 [2,5]
+  CRUSH rule 5 x 823 [5,8]
+  CRUSH rule 5 x 824 [5,7]
+  CRUSH rule 5 x 825 [2,8]
+  CRUSH rule 5 x 826 [7,2]
+  CRUSH rule 5 x 827 [2,8]
+  CRUSH rule 5 x 828 [2,5]
+  CRUSH rule 5 x 829 [5,6]
+  CRUSH rule 5 x 830 [2,5]
+  CRUSH rule 5 x 831 [2,6]
+  CRUSH rule 5 x 832 [5,7]
+  CRUSH rule 5 x 833 [2,7]
+  CRUSH rule 5 x 834 [5,2]
+  CRUSH rule 5 x 835 [8,5]
+  CRUSH rule 5 x 836 [5,7]
+  CRUSH rule 5 x 837 [6,5]
+  CRUSH rule 5 x 838 [6,2]
+  CRUSH rule 5 x 839 [5,2]
+  CRUSH rule 5 x 840 [7,5]
+  CRUSH rule 5 x 841 [5,8]
+  CRUSH rule 5 x 842 [2,5]
+  CRUSH rule 5 x 843 [6,5]
+  CRUSH rule 5 x 844 [5,8]
+  CRUSH rule 5 x 845 [5,8]
+  CRUSH rule 5 x 846 [5,2]
+  CRUSH rule 5 x 847 [2,8]
+  CRUSH rule 5 x 848 [2,6]
+  CRUSH rule 5 x 849 [5,6]
+  CRUSH rule 5 x 850 [2,5]
+  CRUSH rule 5 x 851 [6,5]
+  CRUSH rule 5 x 852 [7,5]
+  CRUSH rule 5 x 853 [6,2]
+  CRUSH rule 5 x 854 [7,2]
+  CRUSH rule 5 x 855 [5,7]
+  CRUSH rule 5 x 856 [6,5]
+  CRUSH rule 5 x 857 [8,5]
+  CRUSH rule 5 x 858 [6,5]
+  CRUSH rule 5 x 859 [6,2]
+  CRUSH rule 5 x 860 [5,2]
+  CRUSH rule 5 x 861 [8,5]
+  CRUSH rule 5 x 862 [6,2]
+  CRUSH rule 5 x 863 [8,2]
+  CRUSH rule 5 x 864 [5,6]
+  CRUSH rule 5 x 865 [8,2]
+  CRUSH rule 5 x 866 [5,6]
+  CRUSH rule 5 x 867 [6,5]
+  CRUSH rule 5 x 868 [6,5]
+  CRUSH rule 5 x 869 [8,5]
+  CRUSH rule 5 x 870 [2,5]
+  CRUSH rule 5 x 871 [5,2]
+  CRUSH rule 5 x 872 [5,2]
+  CRUSH rule 5 x 873 [5,6]
+  CRUSH rule 5 x 874 [2,6]
+  CRUSH rule 5 x 875 [2,6]
+  CRUSH rule 5 x 876 [5,8]
+  CRUSH rule 5 x 877 [6,5]
+  CRUSH rule 5 x 878 [5,2]
+  CRUSH rule 5 x 879 [7,5]
+  CRUSH rule 5 x 880 [5,2]
+  CRUSH rule 5 x 881 [5,6]
+  CRUSH rule 5 x 882 [5,2]
+  CRUSH rule 5 x 883 [2,5]
+  CRUSH rule 5 x 884 [6,2]
+  CRUSH rule 5 x 885 [5,2]
+  CRUSH rule 5 x 886 [5,6]
+  CRUSH rule 5 x 887 [7,5]
+  CRUSH rule 5 x 888 [6,2]
+  CRUSH rule 5 x 889 [2,6]
+  CRUSH rule 5 x 890 [7,2]
+  CRUSH rule 5 x 891 [2,8]
+  CRUSH rule 5 x 892 [6,2]
+  CRUSH rule 5 x 893 [2,5]
+  CRUSH rule 5 x 894 [7,5]
+  CRUSH rule 5 x 895 [5,2]
+  CRUSH rule 5 x 896 [2,8]
+  CRUSH rule 5 x 897 [5,2]
+  CRUSH rule 5 x 898 [2,5]
+  CRUSH rule 5 x 899 [2,7]
+  CRUSH rule 5 x 900 [5,2]
+  CRUSH rule 5 x 901 [5,2]
+  CRUSH rule 5 x 902 [8,5]
+  CRUSH rule 5 x 903 [5,7]
+  CRUSH rule 5 x 904 [5,6]
+  CRUSH rule 5 x 905 [6,2]
+  CRUSH rule 5 x 906 [2,6]
+  CRUSH rule 5 x 907 [7,2]
+  CRUSH rule 5 x 908 [5,8]
+  CRUSH rule 5 x 909 [2,5]
+  CRUSH rule 5 x 910 [6,5]
+  CRUSH rule 5 x 911 [5,8]
+  CRUSH rule 5 x 912 [2,7]
+  CRUSH rule 5 x 913 [7,2]
+  CRUSH rule 5 x 914 [6,5]
+  CRUSH rule 5 x 915 [8,2]
+  CRUSH rule 5 x 916 [5,2]
+  CRUSH rule 5 x 917 [2,5]
+  CRUSH rule 5 x 918 [8,2]
+  CRUSH rule 5 x 919 [6,2]
+  CRUSH rule 5 x 920 [7,5]
+  CRUSH rule 5 x 921 [2,5]
+  CRUSH rule 5 x 922 [6,5]
+  CRUSH rule 5 x 923 [5,8]
+  CRUSH rule 5 x 924 [5,2]
+  CRUSH rule 5 x 925 [5,7]
+  CRUSH rule 5 x 926 [5,2]
+  CRUSH rule 5 x 927 [2,6]
+  CRUSH rule 5 x 928 [8,2]
+  CRUSH rule 5 x 929 [5,2]
+  CRUSH rule 5 x 930 [2,5]
+  CRUSH rule 5 x 931 [5,2]
+  CRUSH rule 5 x 932 [5,2]
+  CRUSH rule 5 x 933 [8,5]
+  CRUSH rule 5 x 934 [5,6]
+  CRUSH rule 5 x 935 [6,5]
+  CRUSH rule 5 x 936 [2,6]
+  CRUSH rule 5 x 937 [5,8]
+  CRUSH rule 5 x 938 [6,5]
+  CRUSH rule 5 x 939 [2,7]
+  CRUSH rule 5 x 940 [8,5]
+  CRUSH rule 5 x 941 [5,2]
+  CRUSH rule 5 x 942 [2,8]
+  CRUSH rule 5 x 943 [8,2]
+  CRUSH rule 5 x 944 [5,8]
+  CRUSH rule 5 x 945 [7,2]
+  CRUSH rule 5 x 946 [2,8]
+  CRUSH rule 5 x 947 [5,2]
+  CRUSH rule 5 x 948 [7,5]
+  CRUSH rule 5 x 949 [6,2]
+  CRUSH rule 5 x 950 [5,6]
+  CRUSH rule 5 x 951 [5,8]
+  CRUSH rule 5 x 952 [2,7]
+  CRUSH rule 5 x 953 [2,5]
+  CRUSH rule 5 x 954 [5,2]
+  CRUSH rule 5 x 955 [8,2]
+  CRUSH rule 5 x 956 [2,6]
+  CRUSH rule 5 x 957 [7,2]
+  CRUSH rule 5 x 958 [8,5]
+  CRUSH rule 5 x 959 [5,2]
+  CRUSH rule 5 x 960 [5,6]
+  CRUSH rule 5 x 961 [5,2]
+  CRUSH rule 5 x 962 [7,5]
+  CRUSH rule 5 x 963 [2,5]
+  CRUSH rule 5 x 964 [5,2]
+  CRUSH rule 5 x 965 [7,5]
+  CRUSH rule 5 x 966 [5,8]
+  CRUSH rule 5 x 967 [8,5]
+  CRUSH rule 5 x 968 [7,2]
+  CRUSH rule 5 x 969 [8,2]
+  CRUSH rule 5 x 970 [2,6]
+  CRUSH rule 5 x 971 [2,7]
+  CRUSH rule 5 x 972 [2,8]
+  CRUSH rule 5 x 973 [2,6]
+  CRUSH rule 5 x 974 [5,2]
+  CRUSH rule 5 x 975 [5,7]
+  CRUSH rule 5 x 976 [5,8]
+  CRUSH rule 5 x 977 [8,5]
+  CRUSH rule 5 x 978 [7,2]
+  CRUSH rule 5 x 979 [7,2]
+  CRUSH rule 5 x 980 [6,2]
+  CRUSH rule 5 x 981 [7,5]
+  CRUSH rule 5 x 982 [5,2]
+  CRUSH rule 5 x 983 [5,7]
+  CRUSH rule 5 x 984 [2,7]
+  CRUSH rule 5 x 985 [2,5]
+  CRUSH rule 5 x 986 [8,5]
+  CRUSH rule 5 x 987 [2,5]
+  CRUSH rule 5 x 988 [2,5]
+  CRUSH rule 5 x 989 [2,6]
+  CRUSH rule 5 x 990 [2,6]
+  CRUSH rule 5 x 991 [2,5]
+  CRUSH rule 5 x 992 [7,2]
+  CRUSH rule 5 x 993 [2,6]
+  CRUSH rule 5 x 994 [5,7]
+  CRUSH rule 5 x 995 [7,2]
+  CRUSH rule 5 x 996 [6,5]
+  CRUSH rule 5 x 997 [6,5]
+  CRUSH rule 5 x 998 [8,2]
+  CRUSH rule 5 x 999 [2,7]
+  CRUSH rule 5 x 1000 [8,5]
+  CRUSH rule 5 x 1001 [2,5]
+  CRUSH rule 5 x 1002 [2,5]
+  CRUSH rule 5 x 1003 [2,8]
+  CRUSH rule 5 x 1004 [6,2]
+  CRUSH rule 5 x 1005 [6,2]
+  CRUSH rule 5 x 1006 [2,6]
+  CRUSH rule 5 x 1007 [2,5]
+  CRUSH rule 5 x 1008 [2,7]
+  CRUSH rule 5 x 1009 [6,5]
+  CRUSH rule 5 x 1010 [5,2]
+  CRUSH rule 5 x 1011 [5,2]
+  CRUSH rule 5 x 1012 [5,2]
+  CRUSH rule 5 x 1013 [5,2]
+  CRUSH rule 5 x 1014 [2,8]
+  CRUSH rule 5 x 1015 [6,5]
+  CRUSH rule 5 x 1016 [2,5]
+  CRUSH rule 5 x 1017 [6,2]
+  CRUSH rule 5 x 1018 [5,2]
+  CRUSH rule 5 x 1019 [5,8]
+  CRUSH rule 5 x 1020 [5,2]
+  CRUSH rule 5 x 1021 [5,2]
+  CRUSH rule 5 x 1022 [2,6]
+  CRUSH rule 5 x 1023 [5,2]
+  rule 5 (chooseleaf-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 5 x 0 [2,5,7]
+  CRUSH rule 5 x 1 [2,8,5]
+  CRUSH rule 5 x 2 [2,5,7]
+  CRUSH rule 5 x 3 [8,2,5]
+  CRUSH rule 5 x 4 [5,2,7]
+  CRUSH rule 5 x 5 [7,2,5]
+  CRUSH rule 5 x 6 [2,6,5]
+  CRUSH rule 5 x 7 [5,8,2]
+  CRUSH rule 5 x 8 [5,6,2]
+  CRUSH rule 5 x 9 [2,5,8]
+  CRUSH rule 5 x 10 [2,7,5]
+  CRUSH rule 5 x 11 [2,7,5]
+  CRUSH rule 5 x 12 [2,5,6]
+  CRUSH rule 5 x 13 [5,8,2]
+  CRUSH rule 5 x 14 [7,2,5]
+  CRUSH rule 5 x 15 [7,2,5]
+  CRUSH rule 5 x 16 [5,6,2]
+  CRUSH rule 5 x 17 [5,2,6]
+  CRUSH rule 5 x 18 [2,5,6]
+  CRUSH rule 5 x 19 [7,5,2]
+  CRUSH rule 5 x 20 [2,5,7]
+  CRUSH rule 5 x 21 [5,7,2]
+  CRUSH rule 5 x 22 [8,5,2]
+  CRUSH rule 5 x 23 [5,6,2]
+  CRUSH rule 5 x 24 [2,7,5]
+  CRUSH rule 5 x 25 [5,7,2]
+  CRUSH rule 5 x 26 [2,8,5]
+  CRUSH rule 5 x 27 [5,2,8]
+  CRUSH rule 5 x 28 [6,2,5]
+  CRUSH rule 5 x 29 [8,5,2]
+  CRUSH rule 5 x 30 [5,7,2]
+  CRUSH rule 5 x 31 [8,2,5]
+  CRUSH rule 5 x 32 [5,6,2]
+  CRUSH rule 5 x 33 [2,7,5]
+  CRUSH rule 5 x 34 [2,5,7]
+  CRUSH rule 5 x 35 [2,8,5]
+  CRUSH rule 5 x 36 [5,8,2]
+  CRUSH rule 5 x 37 [2,5,6]
+  CRUSH rule 5 x 38 [5,8,2]
+  CRUSH rule 5 x 39 [5,7,2]
+  CRUSH rule 5 x 40 [7,2,5]
+  CRUSH rule 5 x 41 [2,6,5]
+  CRUSH rule 5 x 42 [5,6,2]
+  CRUSH rule 5 x 43 [2,5,7]
+  CRUSH rule 5 x 44 [2,6,5]
+  CRUSH rule 5 x 45 [8,2,5]
+  CRUSH rule 5 x 46 [2,5,8]
+  CRUSH rule 5 x 47 [5,2,8]
+  CRUSH rule 5 x 48 [5,6,2]
+  CRUSH rule 5 x 49 [5,7,2]
+  CRUSH rule 5 x 50 [5,2,7]
+  CRUSH rule 5 x 51 [5,6,2]
+  CRUSH rule 5 x 52 [8,2,5]
+  CRUSH rule 5 x 53 [5,8,2]
+  CRUSH rule 5 x 54 [7,5,2]
+  CRUSH rule 5 x 55 [8,2,5]
+  CRUSH rule 5 x 56 [6,5,2]
+  CRUSH rule 5 x 57 [5,8,2]
+  CRUSH rule 5 x 58 [2,8,5]
+  CRUSH rule 5 x 59 [5,2,7]
+  CRUSH rule 5 x 60 [5,2,6]
+  CRUSH rule 5 x 61 [5,6,2]
+  CRUSH rule 5 x 62 [7,2,5]
+  CRUSH rule 5 x 63 [5,6,2]
+  CRUSH rule 5 x 64 [5,2,8]
+  CRUSH rule 5 x 65 [7,5,2]
+  CRUSH rule 5 x 66 [5,6,2]
+  CRUSH rule 5 x 67 [5,2,8]
+  CRUSH rule 5 x 68 [2,5,8]
+  CRUSH rule 5 x 69 [5,2,6]
+  CRUSH rule 5 x 70 [7,2,5]
+  CRUSH rule 5 x 71 [2,8,5]
+  CRUSH rule 5 x 72 [6,2,5]
+  CRUSH rule 5 x 73 [2,7,5]
+  CRUSH rule 5 x 74 [2,7,5]
+  CRUSH rule 5 x 75 [5,2,6]
+  CRUSH rule 5 x 76 [5,2,7]
+  CRUSH rule 5 x 77 [7,2,5]
+  CRUSH rule 5 x 78 [2,5,8]
+  CRUSH rule 5 x 79 [5,2,7]
+  CRUSH rule 5 x 80 [2,5,6]
+  CRUSH rule 5 x 81 [2,5,6]
+  CRUSH rule 5 x 82 [7,2,5]
+  CRUSH rule 5 x 83 [2,6,5]
+  CRUSH rule 5 x 84 [7,2,5]
+  CRUSH rule 5 x 85 [5,8,2]
+  CRUSH rule 5 x 86 [2,6,5]
+  CRUSH rule 5 x 87 [2,7,5]
+  CRUSH rule 5 x 88 [2,6,5]
+  CRUSH rule 5 x 89 [5,2,7]
+  CRUSH rule 5 x 90 [6,5,2]
+  CRUSH rule 5 x 91 [5,8,2]
+  CRUSH rule 5 x 92 [2,8,5]
+  CRUSH rule 5 x 93 [7,5,2]
+  CRUSH rule 5 x 94 [2,5,8]
+  CRUSH rule 5 x 95 [7,5,2]
+  CRUSH rule 5 x 96 [5,6,2]
+  CRUSH rule 5 x 97 [8,5,2]
+  CRUSH rule 5 x 98 [2,7,5]
+  CRUSH rule 5 x 99 [2,7,5]
+  CRUSH rule 5 x 100 [2,7,5]
+  CRUSH rule 5 x 101 [5,7,2]
+  CRUSH rule 5 x 102 [5,2,7]
+  CRUSH rule 5 x 103 [5,7,2]
+  CRUSH rule 5 x 104 [7,5,2]
+  CRUSH rule 5 x 105 [2,5,6]
+  CRUSH rule 5 x 106 [2,6,5]
+  CRUSH rule 5 x 107 [5,2,6]
+  CRUSH rule 5 x 108 [7,2,5]
+  CRUSH rule 5 x 109 [2,5,6]
+  CRUSH rule 5 x 110 [5,2,7]
+  CRUSH rule 5 x 111 [2,5,6]
+  CRUSH rule 5 x 112 [2,6,5]
+  CRUSH rule 5 x 113 [6,2,5]
+  CRUSH rule 5 x 114 [7,5,2]
+  CRUSH rule 5 x 115 [8,2,5]
+  CRUSH rule 5 x 116 [2,6,5]
+  CRUSH rule 5 x 117 [7,5,2]
+  CRUSH rule 5 x 118 [2,5,8]
+  CRUSH rule 5 x 119 [5,6,2]
+  CRUSH rule 5 x 120 [2,5,7]
+  CRUSH rule 5 x 121 [2,7,5]
+  CRUSH rule 5 x 122 [8,5,2]
+  CRUSH rule 5 x 123 [2,5,8]
+  CRUSH rule 5 x 124 [5,2,8]
+  CRUSH rule 5 x 125 [2,7,5]
+  CRUSH rule 5 x 126 [5,2,6]
+  CRUSH rule 5 x 127 [5,6,2]
+  CRUSH rule 5 x 128 [5,6,2]
+  CRUSH rule 5 x 129 [2,5,7]
+  CRUSH rule 5 x 130 [5,8,2]
+  CRUSH rule 5 x 131 [2,5,8]
+  CRUSH rule 5 x 132 [2,5,6]
+  CRUSH rule 5 x 133 [5,6,2]
+  CRUSH rule 5 x 134 [2,8,5]
+  CRUSH rule 5 x 135 [5,6,2]
+  CRUSH rule 5 x 136 [2,5,6]
+  CRUSH rule 5 x 137 [7,5,2]
+  CRUSH rule 5 x 138 [8,5,2]
+  CRUSH rule 5 x 139 [5,2,7]
+  CRUSH rule 5 x 140 [2,6,5]
+  CRUSH rule 5 x 141 [6,2,5]
+  CRUSH rule 5 x 142 [5,2,8]
+  CRUSH rule 5 x 143 [5,8,2]
+  CRUSH rule 5 x 144 [8,2,5]
+  CRUSH rule 5 x 145 [8,5,2]
+  CRUSH rule 5 x 146 [2,6,5]
+  CRUSH rule 5 x 147 [2,8,5]
+  CRUSH rule 5 x 148 [5,2,7]
+  CRUSH rule 5 x 149 [5,8,2]
+  CRUSH rule 5 x 150 [2,6,5]
+  CRUSH rule 5 x 151 [5,6,2]
+  CRUSH rule 5 x 152 [8,5,2]
+  CRUSH rule 5 x 153 [8,5,2]
+  CRUSH rule 5 x 154 [5,2,7]
+  CRUSH rule 5 x 155 [5,7,2]
+  CRUSH rule 5 x 156 [5,2,6]
+  CRUSH rule 5 x 157 [5,2,6]
+  CRUSH rule 5 x 158 [2,8,5]
+  CRUSH rule 5 x 159 [7,2,5]
+  CRUSH rule 5 x 160 [2,8,5]
+  CRUSH rule 5 x 161 [2,5,6]
+  CRUSH rule 5 x 162 [2,6,5]
+  CRUSH rule 5 x 163 [5,6,2]
+  CRUSH rule 5 x 164 [7,2,5]
+  CRUSH rule 5 x 165 [7,2,5]
+  CRUSH rule 5 x 166 [2,5,6]
+  CRUSH rule 5 x 167 [2,7,5]
+  CRUSH rule 5 x 168 [5,2,7]
+  CRUSH rule 5 x 169 [2,6,5]
+  CRUSH rule 5 x 170 [2,5,8]
+  CRUSH rule 5 x 171 [7,5,2]
+  CRUSH rule 5 x 172 [2,7,5]
+  CRUSH rule 5 x 173 [8,5,2]
+  CRUSH rule 5 x 174 [2,5,7]
+  CRUSH rule 5 x 175 [6,2,5]
+  CRUSH rule 5 x 176 [5,2,7]
+  CRUSH rule 5 x 177 [5,2,6]
+  CRUSH rule 5 x 178 [5,2,8]
+  CRUSH rule 5 x 179 [5,2,7]
+  CRUSH rule 5 x 180 [5,8,2]
+  CRUSH rule 5 x 181 [6,2,5]
+  CRUSH rule 5 x 182 [8,5,2]
+  CRUSH rule 5 x 183 [7,5,2]
+  CRUSH rule 5 x 184 [5,7,2]
+  CRUSH rule 5 x 185 [6,2,5]
+  CRUSH rule 5 x 186 [2,5,7]
+  CRUSH rule 5 x 187 [2,6,5]
+  CRUSH rule 5 x 188 [2,8,5]
+  CRUSH rule 5 x 189 [2,7,5]
+  CRUSH rule 5 x 190 [5,2,8]
+  CRUSH rule 5 x 191 [7,2,5]
+  CRUSH rule 5 x 192 [5,2,8]
+  CRUSH rule 5 x 193 [5,2,6]
+  CRUSH rule 5 x 194 [2,5,8]
+  CRUSH rule 5 x 195 [6,5,2]
+  CRUSH rule 5 x 196 [6,2,5]
+  CRUSH rule 5 x 197 [6,5,2]
+  CRUSH rule 5 x 198 [2,5,6]
+  CRUSH rule 5 x 199 [2,5,7]
+  CRUSH rule 5 x 200 [2,5,6]
+  CRUSH rule 5 x 201 [7,2,5]
+  CRUSH rule 5 x 202 [6,5,2]
+  CRUSH rule 5 x 203 [5,8,2]
+  CRUSH rule 5 x 204 [2,5,7]
+  CRUSH rule 5 x 205 [2,7,5]
+  CRUSH rule 5 x 206 [2,7,5]
+  CRUSH rule 5 x 207 [5,2,7]
+  CRUSH rule 5 x 208 [7,2,5]
+  CRUSH rule 5 x 209 [2,8,5]
+  CRUSH rule 5 x 210 [2,5,8]
+  CRUSH rule 5 x 211 [5,2,8]
+  CRUSH rule 5 x 212 [7,5,2]
+  CRUSH rule 5 x 213 [8,5,2]
+  CRUSH rule 5 x 214 [5,8,2]
+  CRUSH rule 5 x 215 [8,2,5]
+  CRUSH rule 5 x 216 [5,2,6]
+  CRUSH rule 5 x 217 [2,7,5]
+  CRUSH rule 5 x 218 [2,7,5]
+  CRUSH rule 5 x 219 [5,8,2]
+  CRUSH rule 5 x 220 [5,7,2]
+  CRUSH rule 5 x 221 [5,6,2]
+  CRUSH rule 5 x 222 [6,5,2]
+  CRUSH rule 5 x 223 [2,5,6]
+  CRUSH rule 5 x 224 [2,5,8]
+  CRUSH rule 5 x 225 [8,2,5]
+  CRUSH rule 5 x 226 [7,2,5]
+  CRUSH rule 5 x 227 [5,2,6]
+  CRUSH rule 5 x 228 [5,6,2]
+  CRUSH rule 5 x 229 [5,8,2]
+  CRUSH rule 5 x 230 [5,7,2]
+  CRUSH rule 5 x 231 [5,7,2]
+  CRUSH rule 5 x 232 [2,7,5]
+  CRUSH rule 5 x 233 [5,7,2]
+  CRUSH rule 5 x 234 [2,5,6]
+  CRUSH rule 5 x 235 [5,8,2]
+  CRUSH rule 5 x 236 [5,2,7]
+  CRUSH rule 5 x 237 [5,7,2]
+  CRUSH rule 5 x 238 [5,2,6]
+  CRUSH rule 5 x 239 [8,5,2]
+  CRUSH rule 5 x 240 [5,7,2]
+  CRUSH rule 5 x 241 [5,2,8]
+  CRUSH rule 5 x 242 [5,2,6]
+  CRUSH rule 5 x 243 [5,7,2]
+  CRUSH rule 5 x 244 [5,6,2]
+  CRUSH rule 5 x 245 [7,2,5]
+  CRUSH rule 5 x 246 [2,5,8]
+  CRUSH rule 5 x 247 [6,2,5]
+  CRUSH rule 5 x 248 [8,2,5]
+  CRUSH rule 5 x 249 [2,5,7]
+  CRUSH rule 5 x 250 [2,5,6]
+  CRUSH rule 5 x 251 [2,5,6]
+  CRUSH rule 5 x 252 [5,7,2]
+  CRUSH rule 5 x 253 [5,2,6]
+  CRUSH rule 5 x 254 [5,2,7]
+  CRUSH rule 5 x 255 [2,7,5]
+  CRUSH rule 5 x 256 [5,7,2]
+  CRUSH rule 5 x 257 [2,8,5]
+  CRUSH rule 5 x 258 [5,2,6]
+  CRUSH rule 5 x 259 [5,6,2]
+  CRUSH rule 5 x 260 [5,6,2]
+  CRUSH rule 5 x 261 [8,5,2]
+  CRUSH rule 5 x 262 [5,6,2]
+  CRUSH rule 5 x 263 [6,2,5]
+  CRUSH rule 5 x 264 [5,6,2]
+  CRUSH rule 5 x 265 [8,5,2]
+  CRUSH rule 5 x 266 [8,2,5]
+  CRUSH rule 5 x 267 [2,5,8]
+  CRUSH rule 5 x 268 [2,7,5]
+  CRUSH rule 5 x 269 [2,8,5]
+  CRUSH rule 5 x 270 [5,2,7]
+  CRUSH rule 5 x 271 [7,5,2]
+  CRUSH rule 5 x 272 [2,8,5]
+  CRUSH rule 5 x 273 [5,2,7]
+  CRUSH rule 5 x 274 [6,5,2]
+  CRUSH rule 5 x 275 [5,7,2]
+  CRUSH rule 5 x 276 [7,2,5]
+  CRUSH rule 5 x 277 [6,5,2]
+  CRUSH rule 5 x 278 [6,2,5]
+  CRUSH rule 5 x 279 [8,5,2]
+  CRUSH rule 5 x 280 [2,6,5]
+  CRUSH rule 5 x 281 [8,2,5]
+  CRUSH rule 5 x 282 [5,2,6]
+  CRUSH rule 5 x 283 [8,2,5]
+  CRUSH rule 5 x 284 [6,5,2]
+  CRUSH rule 5 x 285 [5,7,2]
+  CRUSH rule 5 x 286 [2,6,5]
+  CRUSH rule 5 x 287 [2,5,6]
+  CRUSH rule 5 x 288 [8,2,5]
+  CRUSH rule 5 x 289 [5,6,2]
+  CRUSH rule 5 x 290 [2,5,7]
+  CRUSH rule 5 x 291 [2,5,8]
+  CRUSH rule 5 x 292 [8,2,5]
+  CRUSH rule 5 x 293 [6,2,5]
+  CRUSH rule 5 x 294 [7,5,2]
+  CRUSH rule 5 x 295 [5,8,2]
+  CRUSH rule 5 x 296 [5,2,6]
+  CRUSH rule 5 x 297 [6,2,5]
+  CRUSH rule 5 x 298 [2,5,7]
+  CRUSH rule 5 x 299 [2,8,5]
+  CRUSH rule 5 x 300 [8,5,2]
+  CRUSH rule 5 x 301 [2,8,5]
+  CRUSH rule 5 x 302 [5,2,6]
+  CRUSH rule 5 x 303 [7,5,2]
+  CRUSH rule 5 x 304 [2,7,5]
+  CRUSH rule 5 x 305 [5,8,2]
+  CRUSH rule 5 x 306 [2,7,5]
+  CRUSH rule 5 x 307 [2,7,5]
+  CRUSH rule 5 x 308 [2,8,5]
+  CRUSH rule 5 x 309 [7,5,2]
+  CRUSH rule 5 x 310 [5,2,7]
+  CRUSH rule 5 x 311 [5,6,2]
+  CRUSH rule 5 x 312 [2,6,5]
+  CRUSH rule 5 x 313 [5,2,6]
+  CRUSH rule 5 x 314 [5,2,6]
+  CRUSH rule 5 x 315 [2,5,8]
+  CRUSH rule 5 x 316 [6,5,2]
+  CRUSH rule 5 x 317 [2,6,5]
+  CRUSH rule 5 x 318 [8,2,5]
+  CRUSH rule 5 x 319 [5,2,8]
+  CRUSH rule 5 x 320 [5,7,2]
+  CRUSH rule 5 x 321 [2,5,8]
+  CRUSH rule 5 x 322 [2,7,5]
+  CRUSH rule 5 x 323 [5,7,2]
+  CRUSH rule 5 x 324 [7,2,5]
+  CRUSH rule 5 x 325 [5,6,2]
+  CRUSH rule 5 x 326 [5,2,6]
+  CRUSH rule 5 x 327 [2,6,5]
+  CRUSH rule 5 x 328 [7,5,2]
+  CRUSH rule 5 x 329 [5,6,2]
+  CRUSH rule 5 x 330 [5,7,2]
+  CRUSH rule 5 x 331 [2,6,5]
+  CRUSH rule 5 x 332 [2,5,6]
+  CRUSH rule 5 x 333 [6,5,2]
+  CRUSH rule 5 x 334 [8,5,2]
+  CRUSH rule 5 x 335 [7,2,5]
+  CRUSH rule 5 x 336 [5,6,2]
+  CRUSH rule 5 x 337 [7,2,5]
+  CRUSH rule 5 x 338 [5,6,2]
+  CRUSH rule 5 x 339 [7,5,2]
+  CRUSH rule 5 x 340 [2,8,5]
+  CRUSH rule 5 x 341 [5,2,7]
+  CRUSH rule 5 x 342 [2,7,5]
+  CRUSH rule 5 x 343 [6,5,2]
+  CRUSH rule 5 x 344 [6,2,5]
+  CRUSH rule 5 x 345 [5,7,2]
+  CRUSH rule 5 x 346 [8,2,5]
+  CRUSH rule 5 x 347 [5,2,7]
+  CRUSH rule 5 x 348 [8,2,5]
+  CRUSH rule 5 x 349 [2,6,5]
+  CRUSH rule 5 x 350 [8,5,2]
+  CRUSH rule 5 x 351 [5,6,2]
+  CRUSH rule 5 x 352 [2,8,5]
+  CRUSH rule 5 x 353 [6,5,2]
+  CRUSH rule 5 x 354 [2,5,6]
+  CRUSH rule 5 x 355 [5,8,2]
+  CRUSH rule 5 x 356 [5,2,6]
+  CRUSH rule 5 x 357 [6,2,5]
+  CRUSH rule 5 x 358 [2,8,5]
+  CRUSH rule 5 x 359 [6,2,5]
+  CRUSH rule 5 x 360 [5,2,7]
+  CRUSH rule 5 x 361 [8,5,2]
+  CRUSH rule 5 x 362 [5,2,6]
+  CRUSH rule 5 x 363 [5,2,6]
+  CRUSH rule 5 x 364 [2,5,6]
+  CRUSH rule 5 x 365 [6,5,2]
+  CRUSH rule 5 x 366 [7,2,5]
+  CRUSH rule 5 x 367 [5,2,6]
+  CRUSH rule 5 x 368 [7,5,2]
+  CRUSH rule 5 x 369 [5,7,2]
+  CRUSH rule 5 x 370 [8,2,5]
+  CRUSH rule 5 x 371 [2,5,6]
+  CRUSH rule 5 x 372 [5,2,8]
+  CRUSH rule 5 x 373 [2,6,5]
+  CRUSH rule 5 x 374 [5,8,2]
+  CRUSH rule 5 x 375 [6,5,2]
+  CRUSH rule 5 x 376 [7,2,5]
+  CRUSH rule 5 x 377 [2,5,6]
+  CRUSH rule 5 x 378 [2,8,5]
+  CRUSH rule 5 x 379 [8,5,2]
+  CRUSH rule 5 x 380 [2,5,8]
+  CRUSH rule 5 x 381 [2,5,7]
+  CRUSH rule 5 x 382 [2,5,8]
+  CRUSH rule 5 x 383 [5,6,2]
+  CRUSH rule 5 x 384 [7,2,5]
+  CRUSH rule 5 x 385 [7,5,2]
+  CRUSH rule 5 x 386 [2,5,6]
+  CRUSH rule 5 x 387 [2,5,6]
+  CRUSH rule 5 x 388 [5,2,8]
+  CRUSH rule 5 x 389 [2,5,7]
+  CRUSH rule 5 x 390 [5,6,2]
+  CRUSH rule 5 x 391 [5,6,2]
+  CRUSH rule 5 x 392 [2,8,5]
+  CRUSH rule 5 x 393 [5,2,6]
+  CRUSH rule 5 x 394 [5,7,2]
+  CRUSH rule 5 x 395 [5,2,8]
+  CRUSH rule 5 x 396 [5,2,7]
+  CRUSH rule 5 x 397 [2,5,8]
+  CRUSH rule 5 x 398 [2,5,8]
+  CRUSH rule 5 x 399 [8,5,2]
+  CRUSH rule 5 x 400 [8,2,5]
+  CRUSH rule 5 x 401 [2,5,8]
+  CRUSH rule 5 x 402 [7,5,2]
+  CRUSH rule 5 x 403 [2,5,8]
+  CRUSH rule 5 x 404 [5,2,8]
+  CRUSH rule 5 x 405 [6,5,2]
+  CRUSH rule 5 x 406 [2,6,5]
+  CRUSH rule 5 x 407 [2,8,5]
+  CRUSH rule 5 x 408 [5,2,6]
+  CRUSH rule 5 x 409 [7,5,2]
+  CRUSH rule 5 x 410 [8,5,2]
+  CRUSH rule 5 x 411 [2,8,5]
+  CRUSH rule 5 x 412 [2,5,8]
+  CRUSH rule 5 x 413 [5,2,8]
+  CRUSH rule 5 x 414 [5,2,6]
+  CRUSH rule 5 x 415 [2,6,5]
+  CRUSH rule 5 x 416 [2,6,5]
+  CRUSH rule 5 x 417 [8,2,5]
+  CRUSH rule 5 x 418 [7,2,5]
+  CRUSH rule 5 x 419 [8,5,2]
+  CRUSH rule 5 x 420 [2,5,6]
+  CRUSH rule 5 x 421 [8,5,2]
+  CRUSH rule 5 x 422 [6,5,2]
+  CRUSH rule 5 x 423 [2,5,6]
+  CRUSH rule 5 x 424 [8,5,2]
+  CRUSH rule 5 x 425 [2,5,7]
+  CRUSH rule 5 x 426 [6,2,5]
+  CRUSH rule 5 x 427 [2,7,5]
+  CRUSH rule 5 x 428 [5,7,2]
+  CRUSH rule 5 x 429 [5,6,2]
+  CRUSH rule 5 x 430 [5,6,2]
+  CRUSH rule 5 x 431 [5,2,7]
+  CRUSH rule 5 x 432 [7,2,5]
+  CRUSH rule 5 x 433 [6,5,2]
+  CRUSH rule 5 x 434 [5,2,8]
+  CRUSH rule 5 x 435 [2,5,6]
+  CRUSH rule 5 x 436 [5,2,7]
+  CRUSH rule 5 x 437 [7,5,2]
+  CRUSH rule 5 x 438 [2,5,8]
+  CRUSH rule 5 x 439 [2,5,7]
+  CRUSH rule 5 x 440 [2,7,5]
+  CRUSH rule 5 x 441 [5,7,2]
+  CRUSH rule 5 x 442 [2,5,7]
+  CRUSH rule 5 x 443 [6,2,5]
+  CRUSH rule 5 x 444 [7,2,5]
+  CRUSH rule 5 x 445 [6,5,2]
+  CRUSH rule 5 x 446 [5,2,8]
+  CRUSH rule 5 x 447 [2,5,7]
+  CRUSH rule 5 x 448 [7,2,5]
+  CRUSH rule 5 x 449 [7,5,2]
+  CRUSH rule 5 x 450 [5,2,8]
+  CRUSH rule 5 x 451 [6,5,2]
+  CRUSH rule 5 x 452 [8,5,2]
+  CRUSH rule 5 x 453 [6,5,2]
+  CRUSH rule 5 x 454 [6,5,2]
+  CRUSH rule 5 x 455 [2,7,5]
+  CRUSH rule 5 x 456 [6,2,5]
+  CRUSH rule 5 x 457 [7,2,5]
+  CRUSH rule 5 x 458 [2,8,5]
+  CRUSH rule 5 x 459 [2,7,5]
+  CRUSH rule 5 x 460 [6,5,2]
+  CRUSH rule 5 x 461 [6,5,2]
+  CRUSH rule 5 x 462 [8,2,5]
+  CRUSH rule 5 x 463 [6,2,5]
+  CRUSH rule 5 x 464 [7,5,2]
+  CRUSH rule 5 x 465 [7,2,5]
+  CRUSH rule 5 x 466 [5,8,2]
+  CRUSH rule 5 x 467 [6,5,2]
+  CRUSH rule 5 x 468 [7,2,5]
+  CRUSH rule 5 x 469 [7,2,5]
+  CRUSH rule 5 x 470 [5,2,6]
+  CRUSH rule 5 x 471 [2,7,5]
+  CRUSH rule 5 x 472 [5,2,8]
+  CRUSH rule 5 x 473 [2,5,6]
+  CRUSH rule 5 x 474 [6,2,5]
+  CRUSH rule 5 x 475 [6,2,5]
+  CRUSH rule 5 x 476 [5,6,2]
+  CRUSH rule 5 x 477 [5,8,2]
+  CRUSH rule 5 x 478 [6,2,5]
+  CRUSH rule 5 x 479 [2,5,8]
+  CRUSH rule 5 x 480 [2,8,5]
+  CRUSH rule 5 x 481 [2,5,7]
+  CRUSH rule 5 x 482 [5,7,2]
+  CRUSH rule 5 x 483 [2,6,5]
+  CRUSH rule 5 x 484 [2,7,5]
+  CRUSH rule 5 x 485 [5,7,2]
+  CRUSH rule 5 x 486 [5,2,7]
+  CRUSH rule 5 x 487 [5,2,8]
+  CRUSH rule 5 x 488 [5,7,2]
+  CRUSH rule 5 x 489 [2,8,5]
+  CRUSH rule 5 x 490 [6,5,2]
+  CRUSH rule 5 x 491 [2,7,5]
+  CRUSH rule 5 x 492 [6,5,2]
+  CRUSH rule 5 x 493 [2,7,5]
+  CRUSH rule 5 x 494 [2,7,5]
+  CRUSH rule 5 x 495 [5,2,8]
+  CRUSH rule 5 x 496 [7,5,2]
+  CRUSH rule 5 x 497 [5,7,2]
+  CRUSH rule 5 x 498 [2,5,8]
+  CRUSH rule 5 x 499 [8,5,2]
+  CRUSH rule 5 x 500 [5,6,2]
+  CRUSH rule 5 x 501 [2,7,5]
+  CRUSH rule 5 x 502 [7,2,5]
+  CRUSH rule 5 x 503 [2,5,7]
+  CRUSH rule 5 x 504 [5,6,2]
+  CRUSH rule 5 x 505 [2,7,5]
+  CRUSH rule 5 x 506 [5,2,8]
+  CRUSH rule 5 x 507 [6,2,5]
+  CRUSH rule 5 x 508 [2,5,8]
+  CRUSH rule 5 x 509 [7,5,2]
+  CRUSH rule 5 x 510 [6,2,5]
+  CRUSH rule 5 x 511 [5,8,2]
+  CRUSH rule 5 x 512 [7,2,5]
+  CRUSH rule 5 x 513 [7,2,5]
+  CRUSH rule 5 x 514 [5,6,2]
+  CRUSH rule 5 x 515 [8,5,2]
+  CRUSH rule 5 x 516 [5,2,8]
+  CRUSH rule 5 x 517 [7,2,5]
+  CRUSH rule 5 x 518 [5,6,2]
+  CRUSH rule 5 x 519 [7,5,2]
+  CRUSH rule 5 x 520 [2,6,5]
+  CRUSH rule 5 x 521 [8,2,5]
+  CRUSH rule 5 x 522 [6,2,5]
+  CRUSH rule 5 x 523 [5,2,7]
+  CRUSH rule 5 x 524 [2,5,8]
+  CRUSH rule 5 x 525 [2,5,6]
+  CRUSH rule 5 x 526 [2,5,8]
+  CRUSH rule 5 x 527 [2,5,6]
+  CRUSH rule 5 x 528 [5,2,6]
+  CRUSH rule 5 x 529 [5,7,2]
+  CRUSH rule 5 x 530 [6,5,2]
+  CRUSH rule 5 x 531 [6,2,5]
+  CRUSH rule 5 x 532 [6,5,2]
+  CRUSH rule 5 x 533 [5,6,2]
+  CRUSH rule 5 x 534 [7,5,2]
+  CRUSH rule 5 x 535 [8,2,5]
+  CRUSH rule 5 x 536 [6,2,5]
+  CRUSH rule 5 x 537 [5,7,2]
+  CRUSH rule 5 x 538 [6,5,2]
+  CRUSH rule 5 x 539 [8,5,2]
+  CRUSH rule 5 x 540 [2,6,5]
+  CRUSH rule 5 x 541 [2,5,8]
+  CRUSH rule 5 x 542 [5,2,8]
+  CRUSH rule 5 x 543 [6,2,5]
+  CRUSH rule 5 x 544 [5,7,2]
+  CRUSH rule 5 x 545 [5,7,2]
+  CRUSH rule 5 x 546 [6,2,5]
+  CRUSH rule 5 x 547 [8,2,5]
+  CRUSH rule 5 x 548 [5,2,8]
+  CRUSH rule 5 x 549 [5,8,2]
+  CRUSH rule 5 x 550 [2,5,7]
+  CRUSH rule 5 x 551 [7,5,2]
+  CRUSH rule 5 x 552 [5,8,2]
+  CRUSH rule 5 x 553 [5,2,7]
+  CRUSH rule 5 x 554 [2,8,5]
+  CRUSH rule 5 x 555 [5,2,8]
+  CRUSH rule 5 x 556 [5,6,2]
+  CRUSH rule 5 x 557 [7,5,2]
+  CRUSH rule 5 x 558 [5,2,6]
+  CRUSH rule 5 x 559 [5,2,6]
+  CRUSH rule 5 x 560 [8,5,2]
+  CRUSH rule 5 x 561 [6,5,2]
+  CRUSH rule 5 x 562 [5,2,6]
+  CRUSH rule 5 x 563 [2,6,5]
+  CRUSH rule 5 x 564 [5,2,7]
+  CRUSH rule 5 x 565 [5,6,2]
+  CRUSH rule 5 x 566 [5,7,2]
+  CRUSH rule 5 x 567 [5,6,2]
+  CRUSH rule 5 x 568 [7,5,2]
+  CRUSH rule 5 x 569 [5,2,7]
+  CRUSH rule 5 x 570 [2,5,8]
+  CRUSH rule 5 x 571 [5,7,2]
+  CRUSH rule 5 x 572 [5,2,8]
+  CRUSH rule 5 x 573 [5,2,7]
+  CRUSH rule 5 x 574 [2,5,8]
+  CRUSH rule 5 x 575 [8,2,5]
+  CRUSH rule 5 x 576 [5,6,2]
+  CRUSH rule 5 x 577 [8,2,5]
+  CRUSH rule 5 x 578 [6,2,5]
+  CRUSH rule 5 x 579 [5,2,6]
+  CRUSH rule 5 x 580 [5,2,7]
+  CRUSH rule 5 x 581 [7,2,5]
+  CRUSH rule 5 x 582 [2,8,5]
+  CRUSH rule 5 x 583 [6,2,5]
+  CRUSH rule 5 x 584 [8,2,5]
+  CRUSH rule 5 x 585 [7,2,5]
+  CRUSH rule 5 x 586 [2,7,5]
+  CRUSH rule 5 x 587 [2,5,7]
+  CRUSH rule 5 x 588 [5,7,2]
+  CRUSH rule 5 x 589 [7,2,5]
+  CRUSH rule 5 x 590 [6,2,5]
+  CRUSH rule 5 x 591 [5,2,8]
+  CRUSH rule 5 x 592 [2,5,7]
+  CRUSH rule 5 x 593 [2,8,5]
+  CRUSH rule 5 x 594 [2,7,5]
+  CRUSH rule 5 x 595 [7,2,5]
+  CRUSH rule 5 x 596 [5,2,6]
+  CRUSH rule 5 x 597 [5,2,7]
+  CRUSH rule 5 x 598 [5,2,6]
+  CRUSH rule 5 x 599 [5,2,8]
+  CRUSH rule 5 x 600 [7,2,5]
+  CRUSH rule 5 x 601 [2,7,5]
+  CRUSH rule 5 x 602 [5,7,2]
+  CRUSH rule 5 x 603 [5,2,6]
+  CRUSH rule 5 x 604 [7,5,2]
+  CRUSH rule 5 x 605 [5,2,7]
+  CRUSH rule 5 x 606 [2,7,5]
+  CRUSH rule 5 x 607 [2,5,8]
+  CRUSH rule 5 x 608 [5,2,7]
+  CRUSH rule 5 x 609 [5,2,8]
+  CRUSH rule 5 x 610 [5,7,2]
+  CRUSH rule 5 x 611 [2,8,5]
+  CRUSH rule 5 x 612 [2,6,5]
+  CRUSH rule 5 x 613 [7,2,5]
+  CRUSH rule 5 x 614 [7,2,5]
+  CRUSH rule 5 x 615 [6,2,5]
+  CRUSH rule 5 x 616 [2,8,5]
+  CRUSH rule 5 x 617 [6,2,5]
+  CRUSH rule 5 x 618 [7,5,2]
+  CRUSH rule 5 x 619 [5,2,8]
+  CRUSH rule 5 x 620 [5,2,6]
+  CRUSH rule 5 x 621 [5,8,2]
+  CRUSH rule 5 x 622 [2,5,8]
+  CRUSH rule 5 x 623 [2,6,5]
+  CRUSH rule 5 x 624 [5,2,8]
+  CRUSH rule 5 x 625 [2,5,7]
+  CRUSH rule 5 x 626 [7,2,5]
+  CRUSH rule 5 x 627 [2,7,5]
+  CRUSH rule 5 x 628 [8,2,5]
+  CRUSH rule 5 x 629 [2,6,5]
+  CRUSH rule 5 x 630 [2,7,5]
+  CRUSH rule 5 x 631 [2,7,5]
+  CRUSH rule 5 x 632 [7,2,5]
+  CRUSH rule 5 x 633 [8,5,2]
+  CRUSH rule 5 x 634 [2,5,7]
+  CRUSH rule 5 x 635 [5,6,2]
+  CRUSH rule 5 x 636 [2,5,8]
+  CRUSH rule 5 x 637 [5,2,7]
+  CRUSH rule 5 x 638 [6,2,5]
+  CRUSH rule 5 x 639 [5,2,8]
+  CRUSH rule 5 x 640 [5,2,8]
+  CRUSH rule 5 x 641 [7,2,5]
+  CRUSH rule 5 x 642 [2,7,5]
+  CRUSH rule 5 x 643 [5,2,8]
+  CRUSH rule 5 x 644 [8,2,5]
+  CRUSH rule 5 x 645 [5,7,2]
+  CRUSH rule 5 x 646 [8,2,5]
+  CRUSH rule 5 x 647 [7,2,5]
+  CRUSH rule 5 x 648 [2,6,5]
+  CRUSH rule 5 x 649 [5,7,2]
+  CRUSH rule 5 x 650 [7,5,2]
+  CRUSH rule 5 x 651 [5,7,2]
+  CRUSH rule 5 x 652 [5,7,2]
+  CRUSH rule 5 x 653 [8,5,2]
+  CRUSH rule 5 x 654 [7,5,2]
+  CRUSH rule 5 x 655 [2,5,7]
+  CRUSH rule 5 x 656 [5,7,2]
+  CRUSH rule 5 x 657 [6,2,5]
+  CRUSH rule 5 x 658 [5,6,2]
+  CRUSH rule 5 x 659 [5,6,2]
+  CRUSH rule 5 x 660 [7,5,2]
+  CRUSH rule 5 x 661 [2,8,5]
+  CRUSH rule 5 x 662 [5,2,7]
+  CRUSH rule 5 x 663 [2,5,8]
+  CRUSH rule 5 x 664 [2,5,7]
+  CRUSH rule 5 x 665 [5,7,2]
+  CRUSH rule 5 x 666 [2,8,5]
+  CRUSH rule 5 x 667 [2,5,7]
+  CRUSH rule 5 x 668 [5,7,2]
+  CRUSH rule 5 x 669 [6,5,2]
+  CRUSH rule 5 x 670 [5,2,6]
+  CRUSH rule 5 x 671 [2,7,5]
+  CRUSH rule 5 x 672 [5,2,7]
+  CRUSH rule 5 x 673 [5,2,7]
+  CRUSH rule 5 x 674 [5,2,8]
+  CRUSH rule 5 x 675 [2,8,5]
+  CRUSH rule 5 x 676 [2,5,8]
+  CRUSH rule 5 x 677 [5,2,7]
+  CRUSH rule 5 x 678 [2,5,8]
+  CRUSH rule 5 x 679 [6,2,5]
+  CRUSH rule 5 x 680 [2,5,6]
+  CRUSH rule 5 x 681 [5,7,2]
+  CRUSH rule 5 x 682 [2,5,7]
+  CRUSH rule 5 x 683 [2,5,6]
+  CRUSH rule 5 x 684 [7,2,5]
+  CRUSH rule 5 x 685 [7,2,5]
+  CRUSH rule 5 x 686 [2,5,8]
+  CRUSH rule 5 x 687 [5,6,2]
+  CRUSH rule 5 x 688 [5,7,2]
+  CRUSH rule 5 x 689 [6,5,2]
+  CRUSH rule 5 x 690 [8,2,5]
+  CRUSH rule 5 x 691 [5,2,6]
+  CRUSH rule 5 x 692 [7,2,5]
+  CRUSH rule 5 x 693 [6,5,2]
+  CRUSH rule 5 x 694 [6,5,2]
+  CRUSH rule 5 x 695 [2,8,5]
+  CRUSH rule 5 x 696 [2,5,8]
+  CRUSH rule 5 x 697 [6,2,5]
+  CRUSH rule 5 x 698 [6,2,5]
+  CRUSH rule 5 x 699 [2,6,5]
+  CRUSH rule 5 x 700 [2,5,7]
+  CRUSH rule 5 x 701 [5,2,7]
+  CRUSH rule 5 x 702 [5,2,8]
+  CRUSH rule 5 x 703 [8,5,2]
+  CRUSH rule 5 x 704 [2,5,8]
+  CRUSH rule 5 x 705 [8,2,5]
+  CRUSH rule 5 x 706 [2,5,6]
+  CRUSH rule 5 x 707 [7,5,2]
+  CRUSH rule 5 x 708 [5,7,2]
+  CRUSH rule 5 x 709 [6,5,2]
+  CRUSH rule 5 x 710 [8,5,2]
+  CRUSH rule 5 x 711 [2,5,8]
+  CRUSH rule 5 x 712 [2,5,7]
+  CRUSH rule 5 x 713 [6,5,2]
+  CRUSH rule 5 x 714 [5,2,7]
+  CRUSH rule 5 x 715 [2,5,6]
+  CRUSH rule 5 x 716 [5,6,2]
+  CRUSH rule 5 x 717 [8,2,5]
+  CRUSH rule 5 x 718 [5,7,2]
+  CRUSH rule 5 x 719 [2,6,5]
+  CRUSH rule 5 x 720 [6,2,5]
+  CRUSH rule 5 x 721 [5,7,2]
+  CRUSH rule 5 x 722 [5,7,2]
+  CRUSH rule 5 x 723 [5,2,7]
+  CRUSH rule 5 x 724 [2,6,5]
+  CRUSH rule 5 x 725 [2,5,7]
+  CRUSH rule 5 x 726 [5,8,2]
+  CRUSH rule 5 x 727 [5,6,2]
+  CRUSH rule 5 x 728 [2,7,5]
+  CRUSH rule 5 x 729 [5,6,2]
+  CRUSH rule 5 x 730 [5,7,2]
+  CRUSH rule 5 x 731 [5,2,8]
+  CRUSH rule 5 x 732 [2,5,6]
+  CRUSH rule 5 x 733 [5,7,2]
+  CRUSH rule 5 x 734 [6,5,2]
+  CRUSH rule 5 x 735 [5,8,2]
+  CRUSH rule 5 x 736 [5,8,2]
+  CRUSH rule 5 x 737 [2,6,5]
+  CRUSH rule 5 x 738 [5,2,7]
+  CRUSH rule 5 x 739 [2,7,5]
+  CRUSH rule 5 x 740 [2,8,5]
+  CRUSH rule 5 x 741 [7,2,5]
+  CRUSH rule 5 x 742 [8,2,5]
+  CRUSH rule 5 x 743 [7,2,5]
+  CRUSH rule 5 x 744 [5,7,2]
+  CRUSH rule 5 x 745 [5,2,8]
+  CRUSH rule 5 x 746 [5,2,7]
+  CRUSH rule 5 x 747 [6,2,5]
+  CRUSH rule 5 x 748 [2,7,5]
+  CRUSH rule 5 x 749 [5,8,2]
+  CRUSH rule 5 x 750 [2,6,5]
+  CRUSH rule 5 x 751 [2,8,5]
+  CRUSH rule 5 x 752 [8,2,5]
+  CRUSH rule 5 x 753 [7,5,2]
+  CRUSH rule 5 x 754 [8,5,2]
+  CRUSH rule 5 x 755 [2,6,5]
+  CRUSH rule 5 x 756 [5,6,2]
+  CRUSH rule 5 x 757 [8,2,5]
+  CRUSH rule 5 x 758 [6,2,5]
+  CRUSH rule 5 x 759 [8,5,2]
+  CRUSH rule 5 x 760 [2,5,7]
+  CRUSH rule 5 x 761 [5,2,8]
+  CRUSH rule 5 x 762 [2,7,5]
+  CRUSH rule 5 x 763 [8,5,2]
+  CRUSH rule 5 x 764 [2,7,5]
+  CRUSH rule 5 x 765 [6,5,2]
+  CRUSH rule 5 x 766 [8,5,2]
+  CRUSH rule 5 x 767 [2,8,5]
+  CRUSH rule 5 x 768 [8,5,2]
+  CRUSH rule 5 x 769 [6,2,5]
+  CRUSH rule 5 x 770 [6,2,5]
+  CRUSH rule 5 x 771 [7,2,5]
+  CRUSH rule 5 x 772 [8,5,2]
+  CRUSH rule 5 x 773 [5,2,7]
+  CRUSH rule 5 x 774 [5,6,2]
+  CRUSH rule 5 x 775 [6,5,2]
+  CRUSH rule 5 x 776 [7,2,5]
+  CRUSH rule 5 x 777 [5,2,6]
+  CRUSH rule 5 x 778 [2,8,5]
+  CRUSH rule 5 x 779 [2,7,5]
+  CRUSH rule 5 x 780 [2,5,7]
+  CRUSH rule 5 x 781 [6,5,2]
+  CRUSH rule 5 x 782 [5,2,8]
+  CRUSH rule 5 x 783 [7,2,5]
+  CRUSH rule 5 x 784 [2,5,6]
+  CRUSH rule 5 x 785 [6,2,5]
+  CRUSH rule 5 x 786 [7,5,2]
+  CRUSH rule 5 x 787 [2,6,5]
+  CRUSH rule 5 x 788 [6,2,5]
+  CRUSH rule 5 x 789 [2,5,8]
+  CRUSH rule 5 x 790 [8,5,2]
+  CRUSH rule 5 x 791 [5,8,2]
+  CRUSH rule 5 x 792 [5,8,2]
+  CRUSH rule 5 x 793 [6,2,5]
+  CRUSH rule 5 x 794 [2,6,5]
+  CRUSH rule 5 x 795 [2,5,8]
+  CRUSH rule 5 x 796 [5,7,2]
+  CRUSH rule 5 x 797 [2,5,8]
+  CRUSH rule 5 x 798 [6,2,5]
+  CRUSH rule 5 x 799 [5,2,8]
+  CRUSH rule 5 x 800 [5,2,7]
+  CRUSH rule 5 x 801 [5,6,2]
+  CRUSH rule 5 x 802 [2,8,5]
+  CRUSH rule 5 x 803 [2,5,7]
+  CRUSH rule 5 x 804 [6,2,5]
+  CRUSH rule 5 x 805 [5,6,2]
+  CRUSH rule 5 x 806 [2,5,7]
+  CRUSH rule 5 x 807 [5,7,2]
+  CRUSH rule 5 x 808 [5,6,2]
+  CRUSH rule 5 x 809 [2,5,8]
+  CRUSH rule 5 x 810 [5,7,2]
+  CRUSH rule 5 x 811 [8,5,2]
+  CRUSH rule 5 x 812 [8,5,2]
+  CRUSH rule 5 x 813 [6,5,2]
+  CRUSH rule 5 x 814 [5,6,2]
+  CRUSH rule 5 x 815 [5,2,8]
+  CRUSH rule 5 x 816 [2,7,5]
+  CRUSH rule 5 x 817 [5,8,2]
+  CRUSH rule 5 x 818 [5,2,7]
+  CRUSH rule 5 x 819 [5,2,8]
+  CRUSH rule 5 x 820 [5,6,2]
+  CRUSH rule 5 x 821 [5,6,2]
+  CRUSH rule 5 x 822 [2,5,8]
+  CRUSH rule 5 x 823 [5,8,2]
+  CRUSH rule 5 x 824 [5,7,2]
+  CRUSH rule 5 x 825 [2,8,5]
+  CRUSH rule 5 x 826 [7,2,5]
+  CRUSH rule 5 x 827 [2,8,5]
+  CRUSH rule 5 x 828 [2,5,8]
+  CRUSH rule 5 x 829 [5,6,2]
+  CRUSH rule 5 x 830 [2,5,8]
+  CRUSH rule 5 x 831 [2,6,5]
+  CRUSH rule 5 x 832 [5,7,2]
+  CRUSH rule 5 x 833 [2,7,5]
+  CRUSH rule 5 x 834 [5,2,7]
+  CRUSH rule 5 x 835 [8,5,2]
+  CRUSH rule 5 x 836 [5,7,2]
+  CRUSH rule 5 x 837 [6,5,2]
+  CRUSH rule 5 x 838 [6,2,5]
+  CRUSH rule 5 x 839 [5,2,6]
+  CRUSH rule 5 x 840 [7,5,2]
+  CRUSH rule 5 x 841 [5,8,2]
+  CRUSH rule 5 x 842 [2,5,6]
+  CRUSH rule 5 x 843 [6,5,2]
+  CRUSH rule 5 x 844 [5,8,2]
+  CRUSH rule 5 x 845 [5,8,2]
+  CRUSH rule 5 x 846 [5,2,7]
+  CRUSH rule 5 x 847 [2,8,5]
+  CRUSH rule 5 x 848 [2,6,5]
+  CRUSH rule 5 x 849 [5,6,2]
+  CRUSH rule 5 x 850 [2,5,6]
+  CRUSH rule 5 x 851 [6,5,2]
+  CRUSH rule 5 x 852 [7,5,2]
+  CRUSH rule 5 x 853 [6,2,5]
+  CRUSH rule 5 x 854 [7,2,5]
+  CRUSH rule 5 x 855 [5,7,2]
+  CRUSH rule 5 x 856 [6,5,2]
+  CRUSH rule 5 x 857 [8,5,2]
+  CRUSH rule 5 x 858 [6,5,2]
+  CRUSH rule 5 x 859 [6,2,5]
+  CRUSH rule 5 x 860 [5,2,7]
+  CRUSH rule 5 x 861 [8,5,2]
+  CRUSH rule 5 x 862 [6,2,5]
+  CRUSH rule 5 x 863 [8,2,5]
+  CRUSH rule 5 x 864 [5,6,2]
+  CRUSH rule 5 x 865 [8,2,5]
+  CRUSH rule 5 x 866 [5,6,2]
+  CRUSH rule 5 x 867 [6,5,2]
+  CRUSH rule 5 x 868 [6,5,2]
+  CRUSH rule 5 x 869 [8,5,2]
+  CRUSH rule 5 x 870 [2,5,8]
+  CRUSH rule 5 x 871 [5,2,8]
+  CRUSH rule 5 x 872 [5,2,8]
+  CRUSH rule 5 x 873 [5,6,2]
+  CRUSH rule 5 x 874 [2,6,5]
+  CRUSH rule 5 x 875 [2,6,5]
+  CRUSH rule 5 x 876 [5,8,2]
+  CRUSH rule 5 x 877 [6,5,2]
+  CRUSH rule 5 x 878 [5,2,7]
+  CRUSH rule 5 x 879 [7,5,2]
+  CRUSH rule 5 x 880 [5,2,8]
+  CRUSH rule 5 x 881 [5,6,2]
+  CRUSH rule 5 x 882 [5,2,7]
+  CRUSH rule 5 x 883 [2,5,7]
+  CRUSH rule 5 x 884 [6,2,5]
+  CRUSH rule 5 x 885 [5,2,8]
+  CRUSH rule 5 x 886 [5,6,2]
+  CRUSH rule 5 x 887 [7,5,2]
+  CRUSH rule 5 x 888 [6,2,5]
+  CRUSH rule 5 x 889 [2,6,5]
+  CRUSH rule 5 x 890 [7,2,5]
+  CRUSH rule 5 x 891 [2,8,5]
+  CRUSH rule 5 x 892 [6,2,5]
+  CRUSH rule 5 x 893 [2,5,7]
+  CRUSH rule 5 x 894 [7,5,2]
+  CRUSH rule 5 x 895 [5,2,8]
+  CRUSH rule 5 x 896 [2,8,5]
+  CRUSH rule 5 x 897 [5,2,6]
+  CRUSH rule 5 x 898 [2,5,7]
+  CRUSH rule 5 x 899 [2,7,5]
+  CRUSH rule 5 x 900 [5,2,6]
+  CRUSH rule 5 x 901 [5,2,8]
+  CRUSH rule 5 x 902 [8,5,2]
+  CRUSH rule 5 x 903 [5,7,2]
+  CRUSH rule 5 x 904 [5,6,2]
+  CRUSH rule 5 x 905 [6,2,5]
+  CRUSH rule 5 x 906 [2,6,5]
+  CRUSH rule 5 x 907 [7,2,5]
+  CRUSH rule 5 x 908 [5,8,2]
+  CRUSH rule 5 x 909 [2,5,7]
+  CRUSH rule 5 x 910 [6,5,2]
+  CRUSH rule 5 x 911 [5,8,2]
+  CRUSH rule 5 x 912 [2,7,5]
+  CRUSH rule 5 x 913 [7,2,5]
+  CRUSH rule 5 x 914 [6,5,2]
+  CRUSH rule 5 x 915 [8,2,5]
+  CRUSH rule 5 x 916 [5,2,8]
+  CRUSH rule 5 x 917 [2,5,8]
+  CRUSH rule 5 x 918 [8,2,5]
+  CRUSH rule 5 x 919 [6,2,5]
+  CRUSH rule 5 x 920 [7,5,2]
+  CRUSH rule 5 x 921 [2,5,6]
+  CRUSH rule 5 x 922 [6,5,2]
+  CRUSH rule 5 x 923 [5,8,2]
+  CRUSH rule 5 x 924 [5,2,7]
+  CRUSH rule 5 x 925 [5,7,2]
+  CRUSH rule 5 x 926 [5,2,8]
+  CRUSH rule 5 x 927 [2,6,5]
+  CRUSH rule 5 x 928 [8,2,5]
+  CRUSH rule 5 x 929 [5,2,7]
+  CRUSH rule 5 x 930 [2,5,6]
+  CRUSH rule 5 x 931 [5,2,7]
+  CRUSH rule 5 x 932 [5,2,8]
+  CRUSH rule 5 x 933 [8,5,2]
+  CRUSH rule 5 x 934 [5,6,2]
+  CRUSH rule 5 x 935 [6,5,2]
+  CRUSH rule 5 x 936 [2,6,5]
+  CRUSH rule 5 x 937 [5,8,2]
+  CRUSH rule 5 x 938 [6,5,2]
+  CRUSH rule 5 x 939 [2,7,5]
+  CRUSH rule 5 x 940 [8,5,2]
+  CRUSH rule 5 x 941 [5,2,8]
+  CRUSH rule 5 x 942 [2,8,5]
+  CRUSH rule 5 x 943 [8,2,5]
+  CRUSH rule 5 x 944 [5,8,2]
+  CRUSH rule 5 x 945 [7,2,5]
+  CRUSH rule 5 x 946 [2,8,5]
+  CRUSH rule 5 x 947 [5,2,8]
+  CRUSH rule 5 x 948 [7,5,2]
+  CRUSH rule 5 x 949 [6,2,5]
+  CRUSH rule 5 x 950 [5,6,2]
+  CRUSH rule 5 x 951 [5,8,2]
+  CRUSH rule 5 x 952 [2,7,5]
+  CRUSH rule 5 x 953 [2,5,6]
+  CRUSH rule 5 x 954 [5,2,7]
+  CRUSH rule 5 x 955 [8,2,5]
+  CRUSH rule 5 x 956 [2,6,5]
+  CRUSH rule 5 x 957 [7,2,5]
+  CRUSH rule 5 x 958 [8,5,2]
+  CRUSH rule 5 x 959 [5,2,7]
+  CRUSH rule 5 x 960 [5,6,2]
+  CRUSH rule 5 x 961 [5,2,8]
+  CRUSH rule 5 x 962 [7,5,2]
+  CRUSH rule 5 x 963 [2,5,6]
+  CRUSH rule 5 x 964 [5,2,8]
+  CRUSH rule 5 x 965 [7,5,2]
+  CRUSH rule 5 x 966 [5,8,2]
+  CRUSH rule 5 x 967 [8,5,2]
+  CRUSH rule 5 x 968 [7,2,5]
+  CRUSH rule 5 x 969 [8,2,5]
+  CRUSH rule 5 x 970 [2,6,5]
+  CRUSH rule 5 x 971 [2,7,5]
+  CRUSH rule 5 x 972 [2,8,5]
+  CRUSH rule 5 x 973 [2,6,5]
+  CRUSH rule 5 x 974 [5,2,8]
+  CRUSH rule 5 x 975 [5,7,2]
+  CRUSH rule 5 x 976 [5,8,2]
+  CRUSH rule 5 x 977 [8,5,2]
+  CRUSH rule 5 x 978 [7,2,5]
+  CRUSH rule 5 x 979 [7,2,5]
+  CRUSH rule 5 x 980 [6,2,5]
+  CRUSH rule 5 x 981 [7,5,2]
+  CRUSH rule 5 x 982 [5,2,8]
+  CRUSH rule 5 x 983 [5,7,2]
+  CRUSH rule 5 x 984 [2,7,5]
+  CRUSH rule 5 x 985 [2,5,7]
+  CRUSH rule 5 x 986 [8,5,2]
+  CRUSH rule 5 x 987 [2,5,8]
+  CRUSH rule 5 x 988 [2,5,7]
+  CRUSH rule 5 x 989 [2,6,5]
+  CRUSH rule 5 x 990 [2,6,5]
+  CRUSH rule 5 x 991 [2,5,8]
+  CRUSH rule 5 x 992 [7,2,5]
+  CRUSH rule 5 x 993 [2,6,5]
+  CRUSH rule 5 x 994 [5,7,2]
+  CRUSH rule 5 x 995 [7,2,5]
+  CRUSH rule 5 x 996 [6,5,2]
+  CRUSH rule 5 x 997 [6,5,2]
+  CRUSH rule 5 x 998 [8,2,5]
+  CRUSH rule 5 x 999 [2,7,5]
+  CRUSH rule 5 x 1000 [8,5,2]
+  CRUSH rule 5 x 1001 [2,5,6]
+  CRUSH rule 5 x 1002 [2,5,7]
+  CRUSH rule 5 x 1003 [2,8,5]
+  CRUSH rule 5 x 1004 [6,2,5]
+  CRUSH rule 5 x 1005 [6,2,5]
+  CRUSH rule 5 x 1006 [2,6,5]
+  CRUSH rule 5 x 1007 [2,5,7]
+  CRUSH rule 5 x 1008 [2,7,5]
+  CRUSH rule 5 x 1009 [6,5,2]
+  CRUSH rule 5 x 1010 [5,2,7]
+  CRUSH rule 5 x 1011 [5,2,8]
+  CRUSH rule 5 x 1012 [5,2,7]
+  CRUSH rule 5 x 1013 [5,2,7]
+  CRUSH rule 5 x 1014 [2,8,5]
+  CRUSH rule 5 x 1015 [6,5,2]
+  CRUSH rule 5 x 1016 [2,5,7]
+  CRUSH rule 5 x 1017 [6,2,5]
+  CRUSH rule 5 x 1018 [5,2,6]
+  CRUSH rule 5 x 1019 [5,8,2]
+  CRUSH rule 5 x 1020 [5,2,7]
+  CRUSH rule 5 x 1021 [5,2,6]
+  CRUSH rule 5 x 1022 [2,6,5]
+  CRUSH rule 5 x 1023 [5,2,8]
+  rule 5 (chooseleaf-set) num_rep 3 result size == 3:\t1024/1024 (esc)
+  $ crushtool -i set-choose.crushmap --test --show-statistics --weight 0 0 --weight 3 0 --weight 4 .5 --weight 5 0 --weight 6 .1 --weight 7 0
+  rule 0 (choose), x = 0..1023, numrep = 2..3
+  CRUSH rule 0 x 0 [2,4]
+  CRUSH rule 0 x 1 [2,8]
+  CRUSH rule 0 x 2 [1]
+  CRUSH rule 0 x 3 [8,1]
+  CRUSH rule 0 x 4 [4,1]
+  CRUSH rule 0 x 5 [8,1]
+  CRUSH rule 0 x 6 [2,8]
+  CRUSH rule 0 x 7 [4,8]
+  CRUSH rule 0 x 8 [4,8]
+  CRUSH rule 0 x 9 [2,4]
+  CRUSH rule 0 x 10 [2,8]
+  CRUSH rule 0 x 11 [2,8]
+  CRUSH rule 0 x 12 [2]
+  CRUSH rule 0 x 13 [4,8]
+  CRUSH rule 0 x 14 [8,1]
+  CRUSH rule 0 x 15 [8,1]
+  CRUSH rule 0 x 16 [8]
+  CRUSH rule 0 x 17 [4,1]
+  CRUSH rule 0 x 18 [1]
+  CRUSH rule 0 x 19 [8,4]
+  CRUSH rule 0 x 20 [2]
+  CRUSH rule 0 x 21 [8]
+  CRUSH rule 0 x 22 [8]
+  CRUSH rule 0 x 23 [4,8]
+  CRUSH rule 0 x 24 [1,8]
+  CRUSH rule 0 x 25 [4,8]
+  CRUSH rule 0 x 26 [2,8]
+  CRUSH rule 0 x 27 [4,1]
+  CRUSH rule 0 x 28 [8,2]
+  CRUSH rule 0 x 29 [8,4]
+  CRUSH rule 0 x 30 [4,8]
+  CRUSH rule 0 x 31 [8,2]
+  CRUSH rule 0 x 32 [6]
+  CRUSH rule 0 x 33 [2,8]
+  CRUSH rule 0 x 34 [2]
+  CRUSH rule 0 x 35 [1,8]
+  CRUSH rule 0 x 36 [8]
+  CRUSH rule 0 x 37 [1]
+  CRUSH rule 0 x 38 [4,8]
+  CRUSH rule 0 x 39 [8]
+  CRUSH rule 0 x 40 [8,2]
+  CRUSH rule 0 x 41 [2,8]
+  CRUSH rule 0 x 42 [8]
+  CRUSH rule 0 x 43 [1]
+  CRUSH rule 0 x 44 [1,8]
+  CRUSH rule 0 x 45 [8,2]
+  CRUSH rule 0 x 46 [2]
+  CRUSH rule 0 x 47 [4,1]
+  CRUSH rule 0 x 48 [8]
+  CRUSH rule 0 x 49 [8]
+  CRUSH rule 0 x 50 [4,1]
+  CRUSH rule 0 x 51 [8]
+  CRUSH rule 0 x 52 [8,2]
+  CRUSH rule 0 x 53 [4,8]
+  CRUSH rule 0 x 54 [8,4]
+  CRUSH rule 0 x 55 [8,2]
+  CRUSH rule 0 x 56 [8,4]
+  CRUSH rule 0 x 57 [8]
+  CRUSH rule 0 x 58 [1,8]
+  CRUSH rule 0 x 59 [2]
+  CRUSH rule 0 x 60 [4,1]
+  CRUSH rule 0 x 61 [4,8]
+  CRUSH rule 0 x 62 [8,1]
+  CRUSH rule 0 x 63 [8]
+  CRUSH rule 0 x 64 [4,1]
+  CRUSH rule 0 x 65 [8,4]
+  CRUSH rule 0 x 66 [4,8]
+  CRUSH rule 0 x 67 [4,2]
+  CRUSH rule 0 x 68 [1]
+  CRUSH rule 0 x 69 [2]
+  CRUSH rule 0 x 70 [8,2]
+  CRUSH rule 0 x 71 [2,8]
+  CRUSH rule 0 x 72 [8,2]
+  CRUSH rule 0 x 73 [2,8]
+  CRUSH rule 0 x 74 [1,8]
+  CRUSH rule 0 x 75 [4,2]
+  CRUSH rule 0 x 76 [4,2]
+  CRUSH rule 0 x 77 [8,2]
+  CRUSH rule 0 x 78 [1]
+  CRUSH rule 0 x 79 [4,1]
+  CRUSH rule 0 x 80 [2,4]
+  CRUSH rule 0 x 81 [2]
+  CRUSH rule 0 x 82 [6,2]
+  CRUSH rule 0 x 83 [2,8]
+  CRUSH rule 0 x 84 [8,2]
+  CRUSH rule 0 x 85 [4,8]
+  CRUSH rule 0 x 86 [2,8]
+  CRUSH rule 0 x 87 [2,8]
+  CRUSH rule 0 x 88 [1,8]
+  CRUSH rule 0 x 89 [1]
+  CRUSH rule 0 x 90 [8,4]
+  CRUSH rule 0 x 91 [4,8]
+  CRUSH rule 0 x 92 [1,8]
+  CRUSH rule 0 x 93 [8,4]
+  CRUSH rule 0 x 94 [1]
+  CRUSH rule 0 x 95 [8]
+  CRUSH rule 0 x 96 [8]
+  CRUSH rule 0 x 97 [8]
+  CRUSH rule 0 x 98 [2,8]
+  CRUSH rule 0 x 99 [2,8]
+  CRUSH rule 0 x 100 [1,8]
+  CRUSH rule 0 x 101 [8]
+  CRUSH rule 0 x 102 [2]
+  CRUSH rule 0 x 103 [8]
+  CRUSH rule 0 x 104 [8,4]
+  CRUSH rule 0 x 105 [2,4]
+  CRUSH rule 0 x 106 [1,8]
+  CRUSH rule 0 x 107 [1]
+  CRUSH rule 0 x 108 [8,1]
+  CRUSH rule 0 x 109 [1,4]
+  CRUSH rule 0 x 110 [4,2]
+  CRUSH rule 0 x 111 [2,4]
+  CRUSH rule 0 x 112 [2,8]
+  CRUSH rule 0 x 113 [8,1]
+  CRUSH rule 0 x 114 [8,4]
+  CRUSH rule 0 x 115 [8,2]
+  CRUSH rule 0 x 116 [1,8]
+  CRUSH rule 0 x 117 [6]
+  CRUSH rule 0 x 118 [2]
+  CRUSH rule 0 x 119 [8]
+  CRUSH rule 0 x 120 [2,4]
+  CRUSH rule 0 x 121 [2,8]
+  CRUSH rule 0 x 122 [8]
+  CRUSH rule 0 x 123 [2]
+  CRUSH rule 0 x 124 [2]
+  CRUSH rule 0 x 125 [1,8]
+  CRUSH rule 0 x 126 [1]
+  CRUSH rule 0 x 127 [4,8]
+  CRUSH rule 0 x 128 [8]
+  CRUSH rule 0 x 129 [2,4]
+  CRUSH rule 0 x 130 [4,8]
+  CRUSH rule 0 x 131 [1,4]
+  CRUSH rule 0 x 132 [1]
+  CRUSH rule 0 x 133 [8]
+  CRUSH rule 0 x 134 [1,8]
+  CRUSH rule 0 x 135 [4,8]
+  CRUSH rule 0 x 136 [2,4]
+  CRUSH rule 0 x 137 [8,4]
+  CRUSH rule 0 x 138 [8,4]
+  CRUSH rule 0 x 139 [4,2]
+  CRUSH rule 0 x 140 [1,8]
+  CRUSH rule 0 x 141 [8,2]
+  CRUSH rule 0 x 142 [4,2]
+  CRUSH rule 0 x 143 [4,8]
+  CRUSH rule 0 x 144 [8,2]
+  CRUSH rule 0 x 145 [8]
+  CRUSH rule 0 x 146 [2,8]
+  CRUSH rule 0 x 147 [2,8]
+  CRUSH rule 0 x 148 [4,1]
+  CRUSH rule 0 x 149 [4,8]
+  CRUSH rule 0 x 150 [1,8]
+  CRUSH rule 0 x 151 [8]
+  CRUSH rule 0 x 152 [8]
+  CRUSH rule 0 x 153 [8,4]
+  CRUSH rule 0 x 154 [4,2]
+  CRUSH rule 0 x 155 [4,8]
+  CRUSH rule 0 x 156 [4,2]
+  CRUSH rule 0 x 157 [1]
+  CRUSH rule 0 x 158 [2,8]
+  CRUSH rule 0 x 159 [8,2]
+  CRUSH rule 0 x 160 [2,8]
+  CRUSH rule 0 x 161 [1,4]
+  CRUSH rule 0 x 162 [1,8]
+  CRUSH rule 0 x 163 [4,8]
+  CRUSH rule 0 x 164 [8,2]
+  CRUSH rule 0 x 165 [8,2]
+  CRUSH rule 0 x 166 [2]
+  CRUSH rule 0 x 167 [1,8]
+  CRUSH rule 0 x 168 [4,2]
+  CRUSH rule 0 x 169 [2,8]
+  CRUSH rule 0 x 170 [1]
+  CRUSH rule 0 x 171 [8,4]
+  CRUSH rule 0 x 172 [1,8]
+  CRUSH rule 0 x 173 [8,4]
+  CRUSH rule 0 x 174 [1]
+  CRUSH rule 0 x 175 [8,1]
+  CRUSH rule 0 x 176 [2]
+  CRUSH rule 0 x 177 [1]
+  CRUSH rule 0 x 178 [4,2]
+  CRUSH rule 0 x 179 [1]
+  CRUSH rule 0 x 180 [8]
+  CRUSH rule 0 x 181 [8,2]
+  CRUSH rule 0 x 182 [8]
+  CRUSH rule 0 x 183 [8,4]
+  CRUSH rule 0 x 184 [4,8]
+  CRUSH rule 0 x 185 [8,1]
+  CRUSH rule 0 x 186 [2,4]
+  CRUSH rule 0 x 187 [1,8]
+  CRUSH rule 0 x 188 [1,8]
+  CRUSH rule 0 x 189 [1,8]
+  CRUSH rule 0 x 190 [1]
+  CRUSH rule 0 x 191 [8,1]
+  CRUSH rule 0 x 192 [4,2]
+  CRUSH rule 0 x 193 [4,2]
+  CRUSH rule 0 x 194 [1]
+  CRUSH rule 0 x 195 [8,4]
+  CRUSH rule 0 x 196 [8,1]
+  CRUSH rule 0 x 197 [8,4]
+  CRUSH rule 0 x 198 [2]
+  CRUSH rule 0 x 199 [1,4]
+  CRUSH rule 0 x 200 [1]
+  CRUSH rule 0 x 201 [8,1]
+  CRUSH rule 0 x 202 [8]
+  CRUSH rule 0 x 203 [8]
+  CRUSH rule 0 x 204 [2,4]
+  CRUSH rule 0 x 205 [1,8]
+  CRUSH rule 0 x 206 [1,8]
+  CRUSH rule 0 x 207 [2]
+  CRUSH rule 0 x 208 [8,2]
+  CRUSH rule 0 x 209 [1,8]
+  CRUSH rule 0 x 210 [1,4]
+  CRUSH rule 0 x 211 [4,2]
+  CRUSH rule 0 x 212 [8]
+  CRUSH rule 0 x 213 [8,4]
+  CRUSH rule 0 x 214 [8]
+  CRUSH rule 0 x 215 [8,1]
+  CRUSH rule 0 x 216 [2]
+  CRUSH rule 0 x 217 [1,8]
+  CRUSH rule 0 x 218 [2,8]
+  CRUSH rule 0 x 219 [8]
+  CRUSH rule 0 x 220 [4,8]
+  CRUSH rule 0 x 221 [8]
+  CRUSH rule 0 x 222 [8]
+  CRUSH rule 0 x 223 [1]
+  CRUSH rule 0 x 224 [1,4]
+  CRUSH rule 0 x 225 [8,2]
+  CRUSH rule 0 x 226 [8,2]
+  CRUSH rule 0 x 227 [4,1]
+  CRUSH rule 0 x 228 [8]
+  CRUSH rule 0 x 229 [4,8]
+  CRUSH rule 0 x 230 [4,8]
+  CRUSH rule 0 x 231 [4,8]
+  CRUSH rule 0 x 232 [2,8]
+  CRUSH rule 0 x 233 [8]
+  CRUSH rule 0 x 234 [1]
+  CRUSH rule 0 x 235 [4,8]
+  CRUSH rule 0 x 236 [2]
+  CRUSH rule 0 x 237 [4,8]
+  CRUSH rule 0 x 238 [2]
+  CRUSH rule 0 x 239 [8]
+  CRUSH rule 0 x 240 [4,8]
+  CRUSH rule 0 x 241 [1]
+  CRUSH rule 0 x 242 [2]
+  CRUSH rule 0 x 243 [8]
+  CRUSH rule 0 x 244 [4,8]
+  CRUSH rule 0 x 245 [8,2]
+  CRUSH rule 0 x 246 [1]
+  CRUSH rule 0 x 247 [8,1]
+  CRUSH rule 0 x 248 [8,1]
+  CRUSH rule 0 x 249 [2]
+  CRUSH rule 0 x 250 [2,4]
+  CRUSH rule 0 x 251 [2]
+  CRUSH rule 0 x 252 [4,8]
+  CRUSH rule 0 x 253 [2]
+  CRUSH rule 0 x 254 [4,2]
+  CRUSH rule 0 x 255 [1,8]
+  CRUSH rule 0 x 256 [4,8]
+  CRUSH rule 0 x 257 [2,6]
+  CRUSH rule 0 x 258 [4,2]
+  CRUSH rule 0 x 259 [6]
+  CRUSH rule 0 x 260 [8]
+  CRUSH rule 0 x 261 [8]
+  CRUSH rule 0 x 262 [8]
+  CRUSH rule 0 x 263 [8,1]
+  CRUSH rule 0 x 264 [8]
+  CRUSH rule 0 x 265 [8]
+  CRUSH rule 0 x 266 [8,2]
+  CRUSH rule 0 x 267 [2]
+  CRUSH rule 0 x 268 [1,8]
+  CRUSH rule 0 x 269 [1,8]
+  CRUSH rule 0 x 270 [4,1]
+  CRUSH rule 0 x 271 [8,4]
+  CRUSH rule 0 x 272 [2,8]
+  CRUSH rule 0 x 273 [4,2]
+  CRUSH rule 0 x 274 [8,4]
+  CRUSH rule 0 x 275 [4,8]
+  CRUSH rule 0 x 276 [8,1]
+  CRUSH rule 0 x 277 [8]
+  CRUSH rule 0 x 278 [8,2]
+  CRUSH rule 0 x 279 [8,4]
+  CRUSH rule 0 x 280 [2,8]
+  CRUSH rule 0 x 281 [8,1]
+  CRUSH rule 0 x 282 [2]
+  CRUSH rule 0 x 283 [8,1]
+  CRUSH rule 0 x 284 [8]
+  CRUSH rule 0 x 285 [4,8]
+  CRUSH rule 0 x 286 [2,8]
+  CRUSH rule 0 x 287 [1]
+  CRUSH rule 0 x 288 [8,1]
+  CRUSH rule 0 x 289 [4,8]
+  CRUSH rule 0 x 290 [1,4]
+  CRUSH rule 0 x 291 [1,4]
+  CRUSH rule 0 x 292 [8,1]
+  CRUSH rule 0 x 293 [8,2]
+  CRUSH rule 0 x 294 [8,4]
+  CRUSH rule 0 x 295 [4,8]
+  CRUSH rule 0 x 296 [4,1]
+  CRUSH rule 0 x 297 [8,2]
+  CRUSH rule 0 x 298 [1]
+  CRUSH rule 0 x 299 [2,8]
+  CRUSH rule 0 x 300 [8]
+  CRUSH rule 0 x 301 [1,8]
+  CRUSH rule 0 x 302 [1]
+  CRUSH rule 0 x 303 [8,4]
+  CRUSH rule 0 x 304 [2,8]
+  CRUSH rule 0 x 305 [8]
+  CRUSH rule 0 x 306 [1,8]
+  CRUSH rule 0 x 307 [2,8]
+  CRUSH rule 0 x 308 [2,8]
+  CRUSH rule 0 x 309 [8]
+  CRUSH rule 0 x 310 [4,1]
+  CRUSH rule 0 x 311 [4,8]
+  CRUSH rule 0 x 312 [2,8]
+  CRUSH rule 0 x 313 [4,1]
+  CRUSH rule 0 x 314 [2]
+  CRUSH rule 0 x 315 [2]
+  CRUSH rule 0 x 316 [8]
+  CRUSH rule 0 x 317 [2,8]
+  CRUSH rule 0 x 318 [8,1]
+  CRUSH rule 0 x 319 [2]
+  CRUSH rule 0 x 320 [8]
+  CRUSH rule 0 x 321 [1]
+  CRUSH rule 0 x 322 [2,6]
+  CRUSH rule 0 x 323 [4,8]
+  CRUSH rule 0 x 324 [8,2]
+  CRUSH rule 0 x 325 [4,8]
+  CRUSH rule 0 x 326 [1]
+  CRUSH rule 0 x 327 [1,8]
+  CRUSH rule 0 x 328 [8,4]
+  CRUSH rule 0 x 329 [4,8]
+  CRUSH rule 0 x 330 [4,8]
+  CRUSH rule 0 x 331 [2,8]
+  CRUSH rule 0 x 332 [2]
+  CRUSH rule 0 x 333 [8]
+  CRUSH rule 0 x 334 [8]
+  CRUSH rule 0 x 335 [8,2]
+  CRUSH rule 0 x 336 [4,8]
+  CRUSH rule 0 x 337 [8,2]
+  CRUSH rule 0 x 338 [8]
+  CRUSH rule 0 x 339 [8]
+  CRUSH rule 0 x 340 [2,8]
+  CRUSH rule 0 x 341 [4,1]
+  CRUSH rule 0 x 342 [2,8]
+  CRUSH rule 0 x 343 [8]
+  CRUSH rule 0 x 344 [6,1]
+  CRUSH rule 0 x 345 [8]
+  CRUSH rule 0 x 346 [8,2]
+  CRUSH rule 0 x 347 [4,1]
+  CRUSH rule 0 x 348 [8,2]
+  CRUSH rule 0 x 349 [1,8]
+  CRUSH rule 0 x 350 [8]
+  CRUSH rule 0 x 351 [8]
+  CRUSH rule 0 x 352 [1,8]
+  CRUSH rule 0 x 353 [8]
+  CRUSH rule 0 x 354 [1]
+  CRUSH rule 0 x 355 [8]
+  CRUSH rule 0 x 356 [4,1]
+  CRUSH rule 0 x 357 [8,1]
+  CRUSH rule 0 x 358 [2,8]
+  CRUSH rule 0 x 359 [6,1]
+  CRUSH rule 0 x 360 [2]
+  CRUSH rule 0 x 361 [8,4]
+  CRUSH rule 0 x 362 [4,1]
+  CRUSH rule 0 x 363 [4,2]
+  CRUSH rule 0 x 364 [2]
+  CRUSH rule 0 x 365 [8]
+  CRUSH rule 0 x 366 [8,2]
+  CRUSH rule 0 x 367 [4,2]
+  CRUSH rule 0 x 368 [8,4]
+  CRUSH rule 0 x 369 [8]
+  CRUSH rule 0 x 370 [8,2]
+  CRUSH rule 0 x 371 [1,4]
+  CRUSH rule 0 x 372 [1]
+  CRUSH rule 0 x 373 [1,8]
+  CRUSH rule 0 x 374 [8]
+  CRUSH rule 0 x 375 [8,4]
+  CRUSH rule 0 x 376 [8,1]
+  CRUSH rule 0 x 377 [1,4]
+  CRUSH rule 0 x 378 [1,8]
+  CRUSH rule 0 x 379 [8]
+  CRUSH rule 0 x 380 [2]
+  CRUSH rule 0 x 381 [1,4]
+  CRUSH rule 0 x 382 [1,4]
+  CRUSH rule 0 x 383 [4,8]
+  CRUSH rule 0 x 384 [8,2]
+  CRUSH rule 0 x 385 [8]
+  CRUSH rule 0 x 386 [1]
+  CRUSH rule 0 x 387 [1,4]
+  CRUSH rule 0 x 388 [2]
+  CRUSH rule 0 x 389 [1,4]
+  CRUSH rule 0 x 390 [4,8]
+  CRUSH rule 0 x 391 [4,8]
+  CRUSH rule 0 x 392 [1,8]
+  CRUSH rule 0 x 393 [2]
+  CRUSH rule 0 x 394 [8]
+  CRUSH rule 0 x 395 [1]
+  CRUSH rule 0 x 396 [4,2]
+  CRUSH rule 0 x 397 [2,4]
+  CRUSH rule 0 x 398 [2,4]
+  CRUSH rule 0 x 399 [8,4]
+  CRUSH rule 0 x 400 [8,1]
+  CRUSH rule 0 x 401 [1,4]
+  CRUSH rule 0 x 402 [8,4]
+  CRUSH rule 0 x 403 [1,4]
+  CRUSH rule 0 x 404 [4,2]
+  CRUSH rule 0 x 405 [8,4]
+  CRUSH rule 0 x 406 [2,8]
+  CRUSH rule 0 x 407 [2,8]
+  CRUSH rule 0 x 408 [4,1]
+  CRUSH rule 0 x 409 [8,4]
+  CRUSH rule 0 x 410 [8,4]
+  CRUSH rule 0 x 411 [2,8]
+  CRUSH rule 0 x 412 [2]
+  CRUSH rule 0 x 413 [2]
+  CRUSH rule 0 x 414 [4,1]
+  CRUSH rule 0 x 415 [2,8]
+  CRUSH rule 0 x 416 [2,8]
+  CRUSH rule 0 x 417 [8,2]
+  CRUSH rule 0 x 418 [8,1]
+  CRUSH rule 0 x 419 [8,4]
+  CRUSH rule 0 x 420 [1,4]
+  CRUSH rule 0 x 421 [8,4]
+  CRUSH rule 0 x 422 [6]
+  CRUSH rule 0 x 423 [2,4]
+  CRUSH rule 0 x 424 [8]
+  CRUSH rule 0 x 425 [1]
+  CRUSH rule 0 x 426 [8,2]
+  CRUSH rule 0 x 427 [1,8]
+  CRUSH rule 0 x 428 [4,8]
+  CRUSH rule 0 x 429 [4,8]
+  CRUSH rule 0 x 430 [4,8]
+  CRUSH rule 0 x 431 [4,2]
+  CRUSH rule 0 x 432 [8,1]
+  CRUSH rule 0 x 433 [8]
+  CRUSH rule 0 x 434 [2]
+  CRUSH rule 0 x 435 [2]
+  CRUSH rule 0 x 436 [4,1]
+  CRUSH rule 0 x 437 [8]
+  CRUSH rule 0 x 438 [2,4]
+  CRUSH rule 0 x 439 [1]
+  CRUSH rule 0 x 440 [2,8]
+  CRUSH rule 0 x 441 [4,8]
+  CRUSH rule 0 x 442 [2]
+  CRUSH rule 0 x 443 [8,1]
+  CRUSH rule 0 x 444 [8,1]
+  CRUSH rule 0 x 445 [8]
+  CRUSH rule 0 x 446 [1]
+  CRUSH rule 0 x 447 [2,4]
+  CRUSH rule 0 x 448 [8,2]
+  CRUSH rule 0 x 449 [8]
+  CRUSH rule 0 x 450 [1]
+  CRUSH rule 0 x 451 [8,4]
+  CRUSH rule 0 x 452 [8]
+  CRUSH rule 0 x 453 [6]
+  CRUSH rule 0 x 454 [8]
+  CRUSH rule 0 x 455 [2,8]
+  CRUSH rule 0 x 456 [8,2]
+  CRUSH rule 0 x 457 [8,1]
+  CRUSH rule 0 x 458 [2,8]
+  CRUSH rule 0 x 459 [2,8]
+  CRUSH rule 0 x 460 [8]
+  CRUSH rule 0 x 461 [8]
+  CRUSH rule 0 x 462 [8,1]
+  CRUSH rule 0 x 463 [8,2]
+  CRUSH rule 0 x 464 [8,4]
+  CRUSH rule 0 x 465 [6,1]
+  CRUSH rule 0 x 466 [8]
+  CRUSH rule 0 x 467 [8]
+  CRUSH rule 0 x 468 [8,2]
+  CRUSH rule 0 x 469 [8,2]
+  CRUSH rule 0 x 470 [4,1]
+  CRUSH rule 0 x 471 [1,8]
+  CRUSH rule 0 x 472 [1]
+  CRUSH rule 0 x 473 [1,4]
+  CRUSH rule 0 x 474 [8,1]
+  CRUSH rule 0 x 475 [8,2]
+  CRUSH rule 0 x 476 [4,8]
+  CRUSH rule 0 x 477 [4,8]
+  CRUSH rule 0 x 478 [8,1]
+  CRUSH rule 0 x 479 [2]
+  CRUSH rule 0 x 480 [1,8]
+  CRUSH rule 0 x 481 [2,4]
+  CRUSH rule 0 x 482 [8]
+  CRUSH rule 0 x 483 [2,8]
+  CRUSH rule 0 x 484 [1,8]
+  CRUSH rule 0 x 485 [8]
+  CRUSH rule 0 x 486 [4,1]
+  CRUSH rule 0 x 487 [1]
+  CRUSH rule 0 x 488 [8]
+  CRUSH rule 0 x 489 [2,8]
+  CRUSH rule 0 x 490 [6]
+  CRUSH rule 0 x 491 [1,8]
+  CRUSH rule 0 x 492 [8]
+  CRUSH rule 0 x 493 [2,8]
+  CRUSH rule 0 x 494 [1,8]
+  CRUSH rule 0 x 495 [4,1]
+  CRUSH rule 0 x 496 [8,4]
+  CRUSH rule 0 x 497 [4,8]
+  CRUSH rule 0 x 498 [2,4]
+  CRUSH rule 0 x 499 [8,4]
+  CRUSH rule 0 x 500 [4,8]
+  CRUSH rule 0 x 501 [2,8]
+  CRUSH rule 0 x 502 [6,1]
+  CRUSH rule 0 x 503 [2]
+  CRUSH rule 0 x 504 [8]
+  CRUSH rule 0 x 505 [1,8]
+  CRUSH rule 0 x 506 [4,1]
+  CRUSH rule 0 x 507 [8,2]
+  CRUSH rule 0 x 508 [1]
+  CRUSH rule 0 x 509 [8]
+  CRUSH rule 0 x 510 [8,2]
+  CRUSH rule 0 x 511 [4,8]
+  CRUSH rule 0 x 512 [8,2]
+  CRUSH rule 0 x 513 [8,2]
+  CRUSH rule 0 x 514 [8]
+  CRUSH rule 0 x 515 [8,4]
+  CRUSH rule 0 x 516 [4,1]
+  CRUSH rule 0 x 517 [8,2]
+  CRUSH rule 0 x 518 [4,8]
+  CRUSH rule 0 x 519 [8,4]
+  CRUSH rule 0 x 520 [2,8]
+  CRUSH rule 0 x 521 [8,2]
+  CRUSH rule 0 x 522 [8,1]
+  CRUSH rule 0 x 523 [4,1]
+  CRUSH rule 0 x 524 [2]
+  CRUSH rule 0 x 525 [2]
+  CRUSH rule 0 x 526 [1]
+  CRUSH rule 0 x 527 [1,4]
+  CRUSH rule 0 x 528 [2]
+  CRUSH rule 0 x 529 [4,8]
+  CRUSH rule 0 x 530 [8]
+  CRUSH rule 0 x 531 [8,1]
+  CRUSH rule 0 x 532 [6,4]
+  CRUSH rule 0 x 533 [4,8]
+  CRUSH rule 0 x 534 [8]
+  CRUSH rule 0 x 535 [8,1]
+  CRUSH rule 0 x 536 [8,2]
+  CRUSH rule 0 x 537 [4,8]
+  CRUSH rule 0 x 538 [8,4]
+  CRUSH rule 0 x 539 [8]
+  CRUSH rule 0 x 540 [1,8]
+  CRUSH rule 0 x 541 [2,4]
+  CRUSH rule 0 x 542 [2]
+  CRUSH rule 0 x 543 [8,2]
+  CRUSH rule 0 x 544 [4,8]
+  CRUSH rule 0 x 545 [8]
+  CRUSH rule 0 x 546 [8,2]
+  CRUSH rule 0 x 547 [8,1]
+  CRUSH rule 0 x 548 [4,1]
+  CRUSH rule 0 x 549 [8]
+  CRUSH rule 0 x 550 [2,4]
+  CRUSH rule 0 x 551 [8]
+  CRUSH rule 0 x 552 [4,8]
+  CRUSH rule 0 x 553 [2]
+  CRUSH rule 0 x 554 [1,8]
+  CRUSH rule 0 x 555 [4,2]
+  CRUSH rule 0 x 556 [8]
+  CRUSH rule 0 x 557 [8]
+  CRUSH rule 0 x 558 [4,2]
+  CRUSH rule 0 x 559 [1]
+  CRUSH rule 0 x 560 [8]
+  CRUSH rule 0 x 561 [8,4]
+  CRUSH rule 0 x 562 [4,1]
+  CRUSH rule 0 x 563 [2,8]
+  CRUSH rule 0 x 564 [1]
+  CRUSH rule 0 x 565 [4,8]
+  CRUSH rule 0 x 566 [4,8]
+  CRUSH rule 0 x 567 [4,8]
+  CRUSH rule 0 x 568 [8]
+  CRUSH rule 0 x 569 [4,1]
+  CRUSH rule 0 x 570 [1]
+  CRUSH rule 0 x 571 [6]
+  CRUSH rule 0 x 572 [4,2]
+  CRUSH rule 0 x 573 [1]
+  CRUSH rule 0 x 574 [2]
+  CRUSH rule 0 x 575 [8,1]
+  CRUSH rule 0 x 576 [4,8]
+  CRUSH rule 0 x 577 [8,2]
+  CRUSH rule 0 x 578 [8,1]
+  CRUSH rule 0 x 579 [4,2]
+  CRUSH rule 0 x 580 [1]
+  CRUSH rule 0 x 581 [8,1]
+  CRUSH rule 0 x 582 [2,8]
+  CRUSH rule 0 x 583 [8,2]
+  CRUSH rule 0 x 584 [8,1]
+  CRUSH rule 0 x 585 [8,1]
+  CRUSH rule 0 x 586 [1,8]
+  CRUSH rule 0 x 587 [2,4]
+  CRUSH rule 0 x 588 [4,8]
+  CRUSH rule 0 x 589 [8,1]
+  CRUSH rule 0 x 590 [8,2]
+  CRUSH rule 0 x 591 [4,2]
+  CRUSH rule 0 x 592 [2,4]
+  CRUSH rule 0 x 593 [1,8]
+  CRUSH rule 0 x 594 [2,8]
+  CRUSH rule 0 x 595 [8,1]
+  CRUSH rule 0 x 596 [2]
+  CRUSH rule 0 x 597 [1]
+  CRUSH rule 0 x 598 [2]
+  CRUSH rule 0 x 599 [4,1]
+  CRUSH rule 0 x 600 [8,1]
+  CRUSH rule 0 x 601 [1,8]
+  CRUSH rule 0 x 602 [8]
+  CRUSH rule 0 x 603 [1]
+  CRUSH rule 0 x 604 [8]
+  CRUSH rule 0 x 605 [2]
+  CRUSH rule 0 x 606 [2,6]
+  CRUSH rule 0 x 607 [2,4]
+  CRUSH rule 0 x 608 [4,1]
+  CRUSH rule 0 x 609 [4,2]
+  CRUSH rule 0 x 610 [8]
+  CRUSH rule 0 x 611 [1,8]
+  CRUSH rule 0 x 612 [2,8]
+  CRUSH rule 0 x 613 [8,1]
+  CRUSH rule 0 x 614 [8,2]
+  CRUSH rule 0 x 615 [8,2]
+  CRUSH rule 0 x 616 [1,8]
+  CRUSH rule 0 x 617 [8,2]
+  CRUSH rule 0 x 618 [8,4]
+  CRUSH rule 0 x 619 [4,1]
+  CRUSH rule 0 x 620 [1]
+  CRUSH rule 0 x 621 [8]
+  CRUSH rule 0 x 622 [2,4]
+  CRUSH rule 0 x 623 [2,8]
+  CRUSH rule 0 x 624 [4,2]
+  CRUSH rule 0 x 625 [2]
+  CRUSH rule 0 x 626 [8,2]
+  CRUSH rule 0 x 627 [2,8]
+  CRUSH rule 0 x 628 [8,1]
+  CRUSH rule 0 x 629 [2,8]
+  CRUSH rule 0 x 630 [2,8]
+  CRUSH rule 0 x 631 [1,8]
+  CRUSH rule 0 x 632 [8,2]
+  CRUSH rule 0 x 633 [8]
+  CRUSH rule 0 x 634 [1]
+  CRUSH rule 0 x 635 [4,8]
+  CRUSH rule 0 x 636 [1,4]
+  CRUSH rule 0 x 637 [1]
+  CRUSH rule 0 x 638 [8,2]
+  CRUSH rule 0 x 639 [2]
+  CRUSH rule 0 x 640 [2]
+  CRUSH rule 0 x 641 [8,2]
+  CRUSH rule 0 x 642 [2,8]
+  CRUSH rule 0 x 643 [1]
+  CRUSH rule 0 x 644 [8,1]
+  CRUSH rule 0 x 645 [8]
+  CRUSH rule 0 x 646 [8,1]
+  CRUSH rule 0 x 647 [8,1]
+  CRUSH rule 0 x 648 [1,8]
+  CRUSH rule 0 x 649 [4,8]
+  CRUSH rule 0 x 650 [8,4]
+  CRUSH rule 0 x 651 [4,6]
+  CRUSH rule 0 x 652 [4,8]
+  CRUSH rule 0 x 653 [8]
+  CRUSH rule 0 x 654 [6]
+  CRUSH rule 0 x 655 [1,4]
+  CRUSH rule 0 x 656 [8]
+  CRUSH rule 0 x 657 [6,1]
+  CRUSH rule 0 x 658 [8]
+  CRUSH rule 0 x 659 [4,8]
+  CRUSH rule 0 x 660 [8]
+  CRUSH rule 0 x 661 [1,8]
+  CRUSH rule 0 x 662 [2]
+  CRUSH rule 0 x 663 [1,4]
+  CRUSH rule 0 x 664 [1,4]
+  CRUSH rule 0 x 665 [4,6]
+  CRUSH rule 0 x 666 [2,8]
+  CRUSH rule 0 x 667 [1,4]
+  CRUSH rule 0 x 668 [4,8]
+  CRUSH rule 0 x 669 [6,4]
+  CRUSH rule 0 x 670 [4,1]
+  CRUSH rule 0 x 671 [2,8]
+  CRUSH rule 0 x 672 [4,2]
+  CRUSH rule 0 x 673 [4,2]
+  CRUSH rule 0 x 674 [1]
+  CRUSH rule 0 x 675 [1,8]
+  CRUSH rule 0 x 676 [2,4]
+  CRUSH rule 0 x 677 [4,1]
+  CRUSH rule 0 x 678 [2,4]
+  CRUSH rule 0 x 679 [8,2]
+  CRUSH rule 0 x 680 [2]
+  CRUSH rule 0 x 681 [8]
+  CRUSH rule 0 x 682 [1,4]
+  CRUSH rule 0 x 683 [1,4]
+  CRUSH rule 0 x 684 [8,2]
+  CRUSH rule 0 x 685 [8,2]
+  CRUSH rule 0 x 686 [1,4]
+  CRUSH rule 0 x 687 [6]
+  CRUSH rule 0 x 688 [4,8]
+  CRUSH rule 0 x 689 [8,4]
+  CRUSH rule 0 x 690 [8,1]
+  CRUSH rule 0 x 691 [1]
+  CRUSH rule 0 x 692 [8,1]
+  CRUSH rule 0 x 693 [8,4]
+  CRUSH rule 0 x 694 [8,4]
+  CRUSH rule 0 x 695 [2,8]
+  CRUSH rule 0 x 696 [1]
+  CRUSH rule 0 x 697 [8,1]
+  CRUSH rule 0 x 698 [8,2]
+  CRUSH rule 0 x 699 [1,8]
+  CRUSH rule 0 x 700 [1]
+  CRUSH rule 0 x 701 [1]
+  CRUSH rule 0 x 702 [2]
+  CRUSH rule 0 x 703 [8]
+  CRUSH rule 0 x 704 [1,4]
+  CRUSH rule 0 x 705 [8,1]
+  CRUSH rule 0 x 706 [1,4]
+  CRUSH rule 0 x 707 [8,4]
+  CRUSH rule 0 x 708 [4,8]
+  CRUSH rule 0 x 709 [8]
+  CRUSH rule 0 x 710 [8]
+  CRUSH rule 0 x 711 [2,4]
+  CRUSH rule 0 x 712 [2]
+  CRUSH rule 0 x 713 [8,4]
+  CRUSH rule 0 x 714 [2]
+  CRUSH rule 0 x 715 [1]
+  CRUSH rule 0 x 716 [4,8]
+  CRUSH rule 0 x 717 [8,2]
+  CRUSH rule 0 x 718 [8]
+  CRUSH rule 0 x 719 [2,6]
+  CRUSH rule 0 x 720 [8,1]
+  CRUSH rule 0 x 721 [4,6]
+  CRUSH rule 0 x 722 [8]
+  CRUSH rule 0 x 723 [4,2]
+  CRUSH rule 0 x 724 [2,6]
+  CRUSH rule 0 x 725 [1]
+  CRUSH rule 0 x 726 [4,8]
+  CRUSH rule 0 x 727 [4,8]
+  CRUSH rule 0 x 728 [2,8]
+  CRUSH rule 0 x 729 [8]
+  CRUSH rule 0 x 730 [4,8]
+  CRUSH rule 0 x 731 [4,1]
+  CRUSH rule 0 x 732 [1]
+  CRUSH rule 0 x 733 [4,8]
+  CRUSH rule 0 x 734 [8,4]
+  CRUSH rule 0 x 735 [4,8]
+  CRUSH rule 0 x 736 [4,8]
+  CRUSH rule 0 x 737 [1,8]
+  CRUSH rule 0 x 738 [4,1]
+  CRUSH rule 0 x 739 [2,8]
+  CRUSH rule 0 x 740 [1,8]
+  CRUSH rule 0 x 741 [8,2]
+  CRUSH rule 0 x 742 [8,2]
+  CRUSH rule 0 x 743 [8,1]
+  CRUSH rule 0 x 744 [4,8]
+  CRUSH rule 0 x 745 [1]
+  CRUSH rule 0 x 746 [1]
+  CRUSH rule 0 x 747 [8,2]
+  CRUSH rule 0 x 748 [2,8]
+  CRUSH rule 0 x 749 [4,8]
+  CRUSH rule 0 x 750 [1,8]
+  CRUSH rule 0 x 751 [2,8]
+  CRUSH rule 0 x 752 [8,1]
+  CRUSH rule 0 x 753 [8,4]
+  CRUSH rule 0 x 754 [8,4]
+  CRUSH rule 0 x 755 [1,8]
+  CRUSH rule 0 x 756 [8]
+  CRUSH rule 0 x 757 [8,1]
+  CRUSH rule 0 x 758 [8,1]
+  CRUSH rule 0 x 759 [8,4]
+  CRUSH rule 0 x 760 [1,4]
+  CRUSH rule 0 x 761 [2]
+  CRUSH rule 0 x 762 [2,8]
+  CRUSH rule 0 x 763 [8,4]
+  CRUSH rule 0 x 764 [1,8]
+  CRUSH rule 0 x 765 [8]
+  CRUSH rule 0 x 766 [8]
+  CRUSH rule 0 x 767 [1,8]
+  CRUSH rule 0 x 768 [8,4]
+  CRUSH rule 0 x 769 [8,2]
+  CRUSH rule 0 x 770 [8,1]
+  CRUSH rule 0 x 771 [8,2]
+  CRUSH rule 0 x 772 [8,4]
+  CRUSH rule 0 x 773 [4,2]
+  CRUSH rule 0 x 774 [8]
+  CRUSH rule 0 x 775 [8,4]
+  CRUSH rule 0 x 776 [6,2]
+  CRUSH rule 0 x 777 [4,1]
+  CRUSH rule 0 x 778 [1,6]
+  CRUSH rule 0 x 779 [2,8]
+  CRUSH rule 0 x 780 [2,4]
+  CRUSH rule 0 x 781 [8]
+  CRUSH rule 0 x 782 [4,2]
+  CRUSH rule 0 x 783 [8,1]
+  CRUSH rule 0 x 784 [1,4]
+  CRUSH rule 0 x 785 [8,1]
+  CRUSH rule 0 x 786 [8]
+  CRUSH rule 0 x 787 [1,8]
+  CRUSH rule 0 x 788 [8,2]
+  CRUSH rule 0 x 789 [1]
+  CRUSH rule 0 x 790 [8]
+  CRUSH rule 0 x 791 [4,8]
+  CRUSH rule 0 x 792 [4,8]
+  CRUSH rule 0 x 793 [8,2]
+  CRUSH rule 0 x 794 [2,8]
+  CRUSH rule 0 x 795 [1]
+  CRUSH rule 0 x 796 [8]
+  CRUSH rule 0 x 797 [2,4]
+  CRUSH rule 0 x 798 [6,1]
+  CRUSH rule 0 x 799 [4,2]
+  CRUSH rule 0 x 800 [2]
+  CRUSH rule 0 x 801 [4,8]
+  CRUSH rule 0 x 802 [1,8]
+  CRUSH rule 0 x 803 [2]
+  CRUSH rule 0 x 804 [8,2]
+  CRUSH rule 0 x 805 [8]
+  CRUSH rule 0 x 806 [1,4]
+  CRUSH rule 0 x 807 [4,8]
+  CRUSH rule 0 x 808 [8]
+  CRUSH rule 0 x 809 [1]
+  CRUSH rule 0 x 810 [8]
+  CRUSH rule 0 x 811 [8]
+  CRUSH rule 0 x 812 [8,4]
+  CRUSH rule 0 x 813 [8,4]
+  CRUSH rule 0 x 814 [8]
+  CRUSH rule 0 x 815 [4,1]
+  CRUSH rule 0 x 816 [2,8]
+  CRUSH rule 0 x 817 [8]
+  CRUSH rule 0 x 818 [1]
+  CRUSH rule 0 x 819 [1]
+  CRUSH rule 0 x 820 [4,8]
+  CRUSH rule 0 x 821 [4,8]
+  CRUSH rule 0 x 822 [2,4]
+  CRUSH rule 0 x 823 [4,8]
+  CRUSH rule 0 x 824 [8]
+  CRUSH rule 0 x 825 [2,8]
+  CRUSH rule 0 x 826 [8,1]
+  CRUSH rule 0 x 827 [2,6]
+  CRUSH rule 0 x 828 [2]
+  CRUSH rule 0 x 829 [8]
+  CRUSH rule 0 x 830 [2,4]
+  CRUSH rule 0 x 831 [1,8]
+  CRUSH rule 0 x 832 [4,8]
+  CRUSH rule 0 x 833 [2,8]
+  CRUSH rule 0 x 834 [1]
+  CRUSH rule 0 x 835 [8,4]
+  CRUSH rule 0 x 836 [4,8]
+  CRUSH rule 0 x 837 [8,4]
+  CRUSH rule 0 x 838 [6,2]
+  CRUSH rule 0 x 839 [2]
+  CRUSH rule 0 x 840 [8]
+  CRUSH rule 0 x 841 [4,8]
+  CRUSH rule 0 x 842 [2]
+  CRUSH rule 0 x 843 [8,4]
+  CRUSH rule 0 x 844 [8]
+  CRUSH rule 0 x 845 [4,8]
+  CRUSH rule 0 x 846 [4,2]
+  CRUSH rule 0 x 847 [2,8]
+  CRUSH rule 0 x 848 [2,8]
+  CRUSH rule 0 x 849 [4,8]
+  CRUSH rule 0 x 850 [1]
+  CRUSH rule 0 x 851 [6]
+  CRUSH rule 0 x 852 [8,4]
+  CRUSH rule 0 x 853 [6,2]
+  CRUSH rule 0 x 854 [8,2]
+  CRUSH rule 0 x 855 [8]
+  CRUSH rule 0 x 856 [8,4]
+  CRUSH rule 0 x 857 [8]
+  CRUSH rule 0 x 858 [6]
+  CRUSH rule 0 x 859 [8,2]
+  CRUSH rule 0 x 860 [2]
+  CRUSH rule 0 x 861 [8]
+  CRUSH rule 0 x 862 [8,1]
+  CRUSH rule 0 x 863 [8,1]
+  CRUSH rule 0 x 864 [8]
+  CRUSH rule 0 x 865 [8,1]
+  CRUSH rule 0 x 866 [8]
+  CRUSH rule 0 x 867 [8]
+  CRUSH rule 0 x 868 [8]
+  CRUSH rule 0 x 869 [8,4]
+  CRUSH rule 0 x 870 [2]
+  CRUSH rule 0 x 871 [1]
+  CRUSH rule 0 x 872 [1]
+  CRUSH rule 0 x 873 [4,8]
+  CRUSH rule 0 x 874 [2,6]
+  CRUSH rule 0 x 875 [2,8]
+  CRUSH rule 0 x 876 [4,8]
+  CRUSH rule 0 x 877 [8,4]
+  CRUSH rule 0 x 878 [2]
+  CRUSH rule 0 x 879 [8]
+  CRUSH rule 0 x 880 [1]
+  CRUSH rule 0 x 881 [4,8]
+  CRUSH rule 0 x 882 [1]
+  CRUSH rule 0 x 883 [2,4]
+  CRUSH rule 0 x 884 [8,1]
+  CRUSH rule 0 x 885 [4,1]
+  CRUSH rule 0 x 886 [8]
+  CRUSH rule 0 x 887 [8,4]
+  CRUSH rule 0 x 888 [8,1]
+  CRUSH rule 0 x 889 [2,8]
+  CRUSH rule 0 x 890 [8,1]
+  CRUSH rule 0 x 891 [1,8]
+  CRUSH rule 0 x 892 [8,2]
+  CRUSH rule 0 x 893 [2]
+  CRUSH rule 0 x 894 [8,4]
+  CRUSH rule 0 x 895 [4,1]
+  CRUSH rule 0 x 896 [1,8]
+  CRUSH rule 0 x 897 [2]
+  CRUSH rule 0 x 898 [1,4]
+  CRUSH rule 0 x 899 [1,8]
+  CRUSH rule 0 x 900 [4,1]
+  CRUSH rule 0 x 901 [2]
+  CRUSH rule 0 x 902 [8,4]
+  CRUSH rule 0 x 903 [8]
+  CRUSH rule 0 x 904 [8]
+  CRUSH rule 0 x 905 [8,2]
+  CRUSH rule 0 x 906 [1,8]
+  CRUSH rule 0 x 907 [8,2]
+  CRUSH rule 0 x 908 [8]
+  CRUSH rule 0 x 909 [2]
+  CRUSH rule 0 x 910 [8]
+  CRUSH rule 0 x 911 [8]
+  CRUSH rule 0 x 912 [1,8]
+  CRUSH rule 0 x 913 [8,1]
+  CRUSH rule 0 x 914 [6,4]
+  CRUSH rule 0 x 915 [8,2]
+  CRUSH rule 0 x 916 [4,1]
+  CRUSH rule 0 x 917 [1,4]
+  CRUSH rule 0 x 918 [8,2]
+  CRUSH rule 0 x 919 [8,2]
+  CRUSH rule 0 x 920 [8]
+  CRUSH rule 0 x 921 [1]
+  CRUSH rule 0 x 922 [8,4]
+  CRUSH rule 0 x 923 [4,8]
+  CRUSH rule 0 x 924 [1]
+  CRUSH rule 0 x 925 [4,8]
+  CRUSH rule 0 x 926 [2]
+  CRUSH rule 0 x 927 [1,8]
+  CRUSH rule 0 x 928 [8,1]
+  CRUSH rule 0 x 929 [4,2]
+  CRUSH rule 0 x 930 [2]
+  CRUSH rule 0 x 931 [2]
+  CRUSH rule 0 x 932 [4,2]
+  CRUSH rule 0 x 933 [8,4]
+  CRUSH rule 0 x 934 [8]
+  CRUSH rule 0 x 935 [8]
+  CRUSH rule 0 x 936 [1,8]
+  CRUSH rule 0 x 937 [4,8]
+  CRUSH rule 0 x 938 [8,4]
+  CRUSH rule 0 x 939 [2,8]
+  CRUSH rule 0 x 940 [8]
+  CRUSH rule 0 x 941 [2]
+  CRUSH rule 0 x 942 [1,8]
+  CRUSH rule 0 x 943 [8,2]
+  CRUSH rule 0 x 944 [8]
+  CRUSH rule 0 x 945 [8,2]
+  CRUSH rule 0 x 946 [2,8]
+  CRUSH rule 0 x 947 [2]
+  CRUSH rule 0 x 948 [8]
+  CRUSH rule 0 x 949 [6,2]
+  CRUSH rule 0 x 950 [8]
+  CRUSH rule 0 x 951 [8]
+  CRUSH rule 0 x 952 [2,8]
+  CRUSH rule 0 x 953 [1,4]
+  CRUSH rule 0 x 954 [2]
+  CRUSH rule 0 x 955 [8,1]
+  CRUSH rule 0 x 956 [1,8]
+  CRUSH rule 0 x 957 [8,1]
+  CRUSH rule 0 x 958 [8,4]
+  CRUSH rule 0 x 959 [4,1]
+  CRUSH rule 0 x 960 [6]
+  CRUSH rule 0 x 961 [1]
+  CRUSH rule 0 x 962 [8,4]
+  CRUSH rule 0 x 963 [2,4]
+  CRUSH rule 0 x 964 [2]
+  CRUSH rule 0 x 965 [8]
+  CRUSH rule 0 x 966 [4,8]
+  CRUSH rule 0 x 967 [8,4]
+  CRUSH rule 0 x 968 [8,2]
+  CRUSH rule 0 x 969 [8,2]
+  CRUSH rule 0 x 970 [2,8]
+  CRUSH rule 0 x 971 [1,8]
+  CRUSH rule 0 x 972 [1,8]
+  CRUSH rule 0 x 973 [1,8]
+  CRUSH rule 0 x 974 [4,1]
+  CRUSH rule 0 x 975 [4,8]
+  CRUSH rule 0 x 976 [4,8]
+  CRUSH rule 0 x 977 [8,4]
+  CRUSH rule 0 x 978 [8,2]
+  CRUSH rule 0 x 979 [8,2]
+  CRUSH rule 0 x 980 [8,2]
+  CRUSH rule 0 x 981 [8]
+  CRUSH rule 0 x 982 [1]
+  CRUSH rule 0 x 983 [4,8]
+  CRUSH rule 0 x 984 [2,8]
+  CRUSH rule 0 x 985 [2,4]
+  CRUSH rule 0 x 986 [8,4]
+  CRUSH rule 0 x 987 [2]
+  CRUSH rule 0 x 988 [1,4]
+  CRUSH rule 0 x 989 [1,8]
+  CRUSH rule 0 x 990 [1,8]
+  CRUSH rule 0 x 991 [1,4]
+  CRUSH rule 0 x 992 [8,1]
+  CRUSH rule 0 x 993 [2,8]
+  CRUSH rule 0 x 994 [4,8]
+  CRUSH rule 0 x 995 [8,1]
+  CRUSH rule 0 x 996 [8,4]
+  CRUSH rule 0 x 997 [8,4]
+  CRUSH rule 0 x 998 [8,1]
+  CRUSH rule 0 x 999 [1,8]
+  CRUSH rule 0 x 1000 [8,4]
+  CRUSH rule 0 x 1001 [2]
+  CRUSH rule 0 x 1002 [1]
+  CRUSH rule 0 x 1003 [2,8]
+  CRUSH rule 0 x 1004 [8,1]
+  CRUSH rule 0 x 1005 [8,1]
+  CRUSH rule 0 x 1006 [1,8]
+  CRUSH rule 0 x 1007 [1,4]
+  CRUSH rule 0 x 1008 [1,8]
+  CRUSH rule 0 x 1009 [6,4]
+  CRUSH rule 0 x 1010 [1]
+  CRUSH rule 0 x 1011 [4,2]
+  CRUSH rule 0 x 1012 [1]
+  CRUSH rule 0 x 1013 [2]
+  CRUSH rule 0 x 1014 [2,8]
+  CRUSH rule 0 x 1015 [8]
+  CRUSH rule 0 x 1016 [2,4]
+  CRUSH rule 0 x 1017 [6,1]
+  CRUSH rule 0 x 1018 [4,1]
+  CRUSH rule 0 x 1019 [4,8]
+  CRUSH rule 0 x 1020 [1]
+  CRUSH rule 0 x 1021 [2]
+  CRUSH rule 0 x 1022 [1,8]
+  CRUSH rule 0 x 1023 [4,2]
+  rule 0 (choose) num_rep 2 result size == 1:\t325/1024 (esc)
+  rule 0 (choose) num_rep 2 result size == 2:\t699/1024 (esc)
+  CRUSH rule 0 x 0 [2,4,8]
+  CRUSH rule 0 x 1 [2,8,4]
+  CRUSH rule 0 x 2 [1,8]
+  CRUSH rule 0 x 3 [8,1]
+  CRUSH rule 0 x 4 [4,1,8]
+  CRUSH rule 0 x 5 [8,1]
+  CRUSH rule 0 x 6 [2,8,4]
+  CRUSH rule 0 x 7 [4,8,1]
+  CRUSH rule 0 x 8 [4,8,2]
+  CRUSH rule 0 x 9 [2,4,8]
+  CRUSH rule 0 x 10 [2,8]
+  CRUSH rule 0 x 11 [2,8]
+  CRUSH rule 0 x 12 [2,8]
+  CRUSH rule 0 x 13 [4,8,1]
+  CRUSH rule 0 x 14 [8,1]
+  CRUSH rule 0 x 15 [8,1]
+  CRUSH rule 0 x 16 [8,1]
+  CRUSH rule 0 x 17 [4,1,8]
+  CRUSH rule 0 x 18 [1,6]
+  CRUSH rule 0 x 19 [8,4,2]
+  CRUSH rule 0 x 20 [2,8]
+  CRUSH rule 0 x 21 [8,2]
+  CRUSH rule 0 x 22 [8,1]
+  CRUSH rule 0 x 23 [4,8,2]
+  CRUSH rule 0 x 24 [1,8,4]
+  CRUSH rule 0 x 25 [4,8,1]
+  CRUSH rule 0 x 26 [2,8,4]
+  CRUSH rule 0 x 27 [4,1,8]
+  CRUSH rule 0 x 28 [8,2]
+  CRUSH rule 0 x 29 [8,4,2]
+  CRUSH rule 0 x 30 [4,8,1]
+  CRUSH rule 0 x 31 [8,2]
+  CRUSH rule 0 x 32 [6,2]
+  CRUSH rule 0 x 33 [2,8]
+  CRUSH rule 0 x 34 [2,8]
+  CRUSH rule 0 x 35 [1,8,4]
+  CRUSH rule 0 x 36 [8,1]
+  CRUSH rule 0 x 37 [1,8]
+  CRUSH rule 0 x 38 [4,8,1]
+  CRUSH rule 0 x 39 [8,2]
+  CRUSH rule 0 x 40 [8,2,4]
+  CRUSH rule 0 x 41 [2,8,4]
+  CRUSH rule 0 x 42 [8,2]
+  CRUSH rule 0 x 43 [1,8]
+  CRUSH rule 0 x 44 [1,8,4]
+  CRUSH rule 0 x 45 [8,2,4]
+  CRUSH rule 0 x 46 [2,8]
+  CRUSH rule 0 x 47 [4,1,8]
+  CRUSH rule 0 x 48 [8,1]
+  CRUSH rule 0 x 49 [8,2]
+  CRUSH rule 0 x 50 [4,1,8]
+  CRUSH rule 0 x 51 [8,1]
+  CRUSH rule 0 x 52 [8,2,4]
+  CRUSH rule 0 x 53 [4,8,2]
+  CRUSH rule 0 x 54 [8,4,2]
+  CRUSH rule 0 x 55 [8,2]
+  CRUSH rule 0 x 56 [8,4,1]
+  CRUSH rule 0 x 57 [8,1]
+  CRUSH rule 0 x 58 [1,8]
+  CRUSH rule 0 x 59 [2,8]
+  CRUSH rule 0 x 60 [4,1,8]
+  CRUSH rule 0 x 61 [4,8,2]
+  CRUSH rule 0 x 62 [8,1]
+  CRUSH rule 0 x 63 [8,2]
+  CRUSH rule 0 x 64 [4,1,8]
+  CRUSH rule 0 x 65 [8,4,2]
+  CRUSH rule 0 x 66 [4,8,1]
+  CRUSH rule 0 x 67 [4,2,8]
+  CRUSH rule 0 x 68 [1,8]
+  CRUSH rule 0 x 69 [2,8]
+  CRUSH rule 0 x 70 [8,2]
+  CRUSH rule 0 x 71 [2,8,4]
+  CRUSH rule 0 x 72 [8,2,4]
+  CRUSH rule 0 x 73 [2,8]
+  CRUSH rule 0 x 74 [1,8]
+  CRUSH rule 0 x 75 [4,2,8]
+  CRUSH rule 0 x 76 [4,2,6]
+  CRUSH rule 0 x 77 [8,2,4]
+  CRUSH rule 0 x 78 [1,6]
+  CRUSH rule 0 x 79 [4,1,8]
+  CRUSH rule 0 x 80 [2,4,8]
+  CRUSH rule 0 x 81 [2,8]
+  CRUSH rule 0 x 82 [6,2]
+  CRUSH rule 0 x 83 [2,8]
+  CRUSH rule 0 x 84 [8,2]
+  CRUSH rule 0 x 85 [4,8,2]
+  CRUSH rule 0 x 86 [2,8]
+  CRUSH rule 0 x 87 [2,8,4]
+  CRUSH rule 0 x 88 [1,8]
+  CRUSH rule 0 x 89 [1,8]
+  CRUSH rule 0 x 90 [8,4,2]
+  CRUSH rule 0 x 91 [4,8,2]
+  CRUSH rule 0 x 92 [1,8]
+  CRUSH rule 0 x 93 [8,4,1]
+  CRUSH rule 0 x 94 [1,8]
+  CRUSH rule 0 x 95 [8,1]
+  CRUSH rule 0 x 96 [8,2]
+  CRUSH rule 0 x 97 [8,1]
+  CRUSH rule 0 x 98 [2,8]
+  CRUSH rule 0 x 99 [2,8]
+  CRUSH rule 0 x 100 [1,8,4]
+  CRUSH rule 0 x 101 [8,1]
+  CRUSH rule 0 x 102 [2,8]
+  CRUSH rule 0 x 103 [8,1]
+  CRUSH rule 0 x 104 [8,4,1]
+  CRUSH rule 0 x 105 [2,4,8]
+  CRUSH rule 0 x 106 [1,8,4]
+  CRUSH rule 0 x 107 [1,8]
+  CRUSH rule 0 x 108 [8,1]
+  CRUSH rule 0 x 109 [1,4,8]
+  CRUSH rule 0 x 110 [4,2,8]
+  CRUSH rule 0 x 111 [2,4,8]
+  CRUSH rule 0 x 112 [2,8]
+  CRUSH rule 0 x 113 [8,1]
+  CRUSH rule 0 x 114 [8,4,2]
+  CRUSH rule 0 x 115 [8,2,4]
+  CRUSH rule 0 x 116 [1,8]
+  CRUSH rule 0 x 117 [6,1]
+  CRUSH rule 0 x 118 [2,8]
+  CRUSH rule 0 x 119 [8,1]
+  CRUSH rule 0 x 120 [2,4,8]
+  CRUSH rule 0 x 121 [2,8,4]
+  CRUSH rule 0 x 122 [8,2]
+  CRUSH rule 0 x 123 [2,8]
+  CRUSH rule 0 x 124 [2,8]
+  CRUSH rule 0 x 125 [1,8,4]
+  CRUSH rule 0 x 126 [1,8]
+  CRUSH rule 0 x 127 [4,8,1]
+  CRUSH rule 0 x 128 [8,1]
+  CRUSH rule 0 x 129 [2,4,8]
+  CRUSH rule 0 x 130 [4,8,1]
+  CRUSH rule 0 x 131 [1,4,8]
+  CRUSH rule 0 x 132 [1,8]
+  CRUSH rule 0 x 133 [8,1]
+  CRUSH rule 0 x 134 [1,8,4]
+  CRUSH rule 0 x 135 [4,8,2]
+  CRUSH rule 0 x 136 [2,4,8]
+  CRUSH rule 0 x 137 [8,4,1]
+  CRUSH rule 0 x 138 [8,4,2]
+  CRUSH rule 0 x 139 [4,2,8]
+  CRUSH rule 0 x 140 [1,8,4]
+  CRUSH rule 0 x 141 [8,2]
+  CRUSH rule 0 x 142 [4,2,8]
+  CRUSH rule 0 x 143 [4,8,1]
+  CRUSH rule 0 x 144 [8,2]
+  CRUSH rule 0 x 145 [8,2]
+  CRUSH rule 0 x 146 [2,8]
+  CRUSH rule 0 x 147 [2,8,4]
+  CRUSH rule 0 x 148 [4,1,8]
+  CRUSH rule 0 x 149 [4,8,2]
+  CRUSH rule 0 x 150 [1,8]
+  CRUSH rule 0 x 151 [8,1]
+  CRUSH rule 0 x 152 [8,2]
+  CRUSH rule 0 x 153 [8,4,1]
+  CRUSH rule 0 x 154 [4,2,8]
+  CRUSH rule 0 x 155 [4,8,1]
+  CRUSH rule 0 x 156 [4,2,8]
+  CRUSH rule 0 x 157 [1,8]
+  CRUSH rule 0 x 158 [2,8,4]
+  CRUSH rule 0 x 159 [8,2,4]
+  CRUSH rule 0 x 160 [2,8,4]
+  CRUSH rule 0 x 161 [1,4,8]
+  CRUSH rule 0 x 162 [1,8]
+  CRUSH rule 0 x 163 [4,8,2]
+  CRUSH rule 0 x 164 [8,2]
+  CRUSH rule 0 x 165 [8,2,4]
+  CRUSH rule 0 x 166 [2,8]
+  CRUSH rule 0 x 167 [1,8,4]
+  CRUSH rule 0 x 168 [4,2,8]
+  CRUSH rule 0 x 169 [2,8,4]
+  CRUSH rule 0 x 170 [1,8]
+  CRUSH rule 0 x 171 [8,4,2]
+  CRUSH rule 0 x 172 [1,8]
+  CRUSH rule 0 x 173 [8,4,1]
+  CRUSH rule 0 x 174 [1,8]
+  CRUSH rule 0 x 175 [8,1]
+  CRUSH rule 0 x 176 [2,8]
+  CRUSH rule 0 x 177 [1,8]
+  CRUSH rule 0 x 178 [4,2,8]
+  CRUSH rule 0 x 179 [1,8]
+  CRUSH rule 0 x 180 [8,2]
+  CRUSH rule 0 x 181 [8,2,4]
+  CRUSH rule 0 x 182 [8,1]
+  CRUSH rule 0 x 183 [8,4,1]
+  CRUSH rule 0 x 184 [4,8,1]
+  CRUSH rule 0 x 185 [8,1,4]
+  CRUSH rule 0 x 186 [2,4,8]
+  CRUSH rule 0 x 187 [1,8]
+  CRUSH rule 0 x 188 [1,8,4]
+  CRUSH rule 0 x 189 [1,8,4]
+  CRUSH rule 0 x 190 [1,8]
+  CRUSH rule 0 x 191 [8,1,4]
+  CRUSH rule 0 x 192 [4,2,8]
+  CRUSH rule 0 x 193 [4,2,8]
+  CRUSH rule 0 x 194 [1,8]
+  CRUSH rule 0 x 195 [8,4,2]
+  CRUSH rule 0 x 196 [8,1]
+  CRUSH rule 0 x 197 [8,4,1]
+  CRUSH rule 0 x 198 [2,8]
+  CRUSH rule 0 x 199 [1,4,8]
+  CRUSH rule 0 x 200 [1,8]
+  CRUSH rule 0 x 201 [8,1,4]
+  CRUSH rule 0 x 202 [8,1]
+  CRUSH rule 0 x 203 [8,1]
+  CRUSH rule 0 x 204 [2,4,8]
+  CRUSH rule 0 x 205 [1,8]
+  CRUSH rule 0 x 206 [1,8,4]
+  CRUSH rule 0 x 207 [2,8]
+  CRUSH rule 0 x 208 [8,2]
+  CRUSH rule 0 x 209 [1,8]
+  CRUSH rule 0 x 210 [1,4,8]
+  CRUSH rule 0 x 211 [4,2,8]
+  CRUSH rule 0 x 212 [8,1]
+  CRUSH rule 0 x 213 [8,4,1]
+  CRUSH rule 0 x 214 [8,1]
+  CRUSH rule 0 x 215 [8,1]
+  CRUSH rule 0 x 216 [2,8]
+  CRUSH rule 0 x 217 [1,8,4]
+  CRUSH rule 0 x 218 [2,8]
+  CRUSH rule 0 x 219 [8,2]
+  CRUSH rule 0 x 220 [4,8,1]
+  CRUSH rule 0 x 221 [8,1]
+  CRUSH rule 0 x 222 [8,1]
+  CRUSH rule 0 x 223 [1,8]
+  CRUSH rule 0 x 224 [1,4,8]
+  CRUSH rule 0 x 225 [8,2]
+  CRUSH rule 0 x 226 [8,2,4]
+  CRUSH rule 0 x 227 [4,1,8]
+  CRUSH rule 0 x 228 [8,1]
+  CRUSH rule 0 x 229 [4,8,1]
+  CRUSH rule 0 x 230 [4,8,1]
+  CRUSH rule 0 x 231 [4,8,1]
+  CRUSH rule 0 x 232 [2,8,4]
+  CRUSH rule 0 x 233 [8,2]
+  CRUSH rule 0 x 234 [1,8]
+  CRUSH rule 0 x 235 [4,8,1]
+  CRUSH rule 0 x 236 [2,8]
+  CRUSH rule 0 x 237 [4,8,2]
+  CRUSH rule 0 x 238 [2,8]
+  CRUSH rule 0 x 239 [8,2]
+  CRUSH rule 0 x 240 [4,8,2]
+  CRUSH rule 0 x 241 [1,8]
+  CRUSH rule 0 x 242 [2,8]
+  CRUSH rule 0 x 243 [8,2]
+  CRUSH rule 0 x 244 [4,8,2]
+  CRUSH rule 0 x 245 [8,2]
+  CRUSH rule 0 x 246 [1,8]
+  CRUSH rule 0 x 247 [8,1]
+  CRUSH rule 0 x 248 [8,1,4]
+  CRUSH rule 0 x 249 [2,8]
+  CRUSH rule 0 x 250 [2,4,8]
+  CRUSH rule 0 x 251 [2,8]
+  CRUSH rule 0 x 252 [4,8,2]
+  CRUSH rule 0 x 253 [2,8]
+  CRUSH rule 0 x 254 [4,2,8]
+  CRUSH rule 0 x 255 [1,8]
+  CRUSH rule 0 x 256 [4,8,1]
+  CRUSH rule 0 x 257 [2,6,4]
+  CRUSH rule 0 x 258 [4,2,8]
+  CRUSH rule 0 x 259 [6,1]
+  CRUSH rule 0 x 260 [8,1]
+  CRUSH rule 0 x 261 [8,2]
+  CRUSH rule 0 x 262 [8,2]
+  CRUSH rule 0 x 263 [8,1,4]
+  CRUSH rule 0 x 264 [8,1]
+  CRUSH rule 0 x 265 [8,2]
+  CRUSH rule 0 x 266 [8,2,4]
+  CRUSH rule 0 x 267 [2,8]
+  CRUSH rule 0 x 268 [1,8]
+  CRUSH rule 0 x 269 [1,8,4]
+  CRUSH rule 0 x 270 [4,1,8]
+  CRUSH rule 0 x 271 [8,4,2]
+  CRUSH rule 0 x 272 [2,8,4]
+  CRUSH rule 0 x 273 [4,2,8]
+  CRUSH rule 0 x 274 [8,4,2]
+  CRUSH rule 0 x 275 [4,8,1]
+  CRUSH rule 0 x 276 [8,1,4]
+  CRUSH rule 0 x 277 [8,1]
+  CRUSH rule 0 x 278 [8,2,4]
+  CRUSH rule 0 x 279 [8,4,1]
+  CRUSH rule 0 x 280 [2,8,4]
+  CRUSH rule 0 x 281 [8,1]
+  CRUSH rule 0 x 282 [2,8]
+  CRUSH rule 0 x 283 [8,1]
+  CRUSH rule 0 x 284 [8,1]
+  CRUSH rule 0 x 285 [4,8,2]
+  CRUSH rule 0 x 286 [2,8,4]
+  CRUSH rule 0 x 287 [1,8]
+  CRUSH rule 0 x 288 [8,1,4]
+  CRUSH rule 0 x 289 [4,8,2]
+  CRUSH rule 0 x 290 [1,4,8]
+  CRUSH rule 0 x 291 [1,4,8]
+  CRUSH rule 0 x 292 [8,1,4]
+  CRUSH rule 0 x 293 [8,2]
+  CRUSH rule 0 x 294 [8,4,2]
+  CRUSH rule 0 x 295 [4,8,1]
+  CRUSH rule 0 x 296 [4,1,8]
+  CRUSH rule 0 x 297 [8,2,4]
+  CRUSH rule 0 x 298 [1,8]
+  CRUSH rule 0 x 299 [2,8]
+  CRUSH rule 0 x 300 [8,1]
+  CRUSH rule 0 x 301 [1,8]
+  CRUSH rule 0 x 302 [1,8]
+  CRUSH rule 0 x 303 [8,4,1]
+  CRUSH rule 0 x 304 [2,8]
+  CRUSH rule 0 x 305 [8,1]
+  CRUSH rule 0 x 306 [1,8]
+  CRUSH rule 0 x 307 [2,8]
+  CRUSH rule 0 x 308 [2,8,4]
+  CRUSH rule 0 x 309 [8,2]
+  CRUSH rule 0 x 310 [4,1,6]
+  CRUSH rule 0 x 311 [4,8,2]
+  CRUSH rule 0 x 312 [2,8]
+  CRUSH rule 0 x 313 [4,1,8]
+  CRUSH rule 0 x 314 [2,6]
+  CRUSH rule 0 x 315 [2,8]
+  CRUSH rule 0 x 316 [8,1]
+  CRUSH rule 0 x 317 [2,8]
+  CRUSH rule 0 x 318 [8,1]
+  CRUSH rule 0 x 319 [2,8]
+  CRUSH rule 0 x 320 [8,1]
+  CRUSH rule 0 x 321 [1,8]
+  CRUSH rule 0 x 322 [2,6,4]
+  CRUSH rule 0 x 323 [4,8,2]
+  CRUSH rule 0 x 324 [8,2,4]
+  CRUSH rule 0 x 325 [4,8,1]
+  CRUSH rule 0 x 326 [1,6]
+  CRUSH rule 0 x 327 [1,8]
+  CRUSH rule 0 x 328 [8,4,1]
+  CRUSH rule 0 x 329 [4,8,2]
+  CRUSH rule 0 x 330 [4,8,1]
+  CRUSH rule 0 x 331 [2,8]
+  CRUSH rule 0 x 332 [2,8]
+  CRUSH rule 0 x 333 [8,1]
+  CRUSH rule 0 x 334 [8,2]
+  CRUSH rule 0 x 335 [8,2]
+  CRUSH rule 0 x 336 [4,8,2]
+  CRUSH rule 0 x 337 [8,2,4]
+  CRUSH rule 0 x 338 [8,2]
+  CRUSH rule 0 x 339 [8,2]
+  CRUSH rule 0 x 340 [2,8,4]
+  CRUSH rule 0 x 341 [4,1,6]
+  CRUSH rule 0 x 342 [2,8,4]
+  CRUSH rule 0 x 343 [8,1]
+  CRUSH rule 0 x 344 [6,1,4]
+  CRUSH rule 0 x 345 [8,2]
+  CRUSH rule 0 x 346 [8,2,4]
+  CRUSH rule 0 x 347 [4,1,8]
+  CRUSH rule 0 x 348 [8,2,4]
+  CRUSH rule 0 x 349 [1,8]
+  CRUSH rule 0 x 350 [8,1]
+  CRUSH rule 0 x 351 [8,2]
+  CRUSH rule 0 x 352 [1,8,4]
+  CRUSH rule 0 x 353 [8,1]
+  CRUSH rule 0 x 354 [1,8]
+  CRUSH rule 0 x 355 [8,2]
+  CRUSH rule 0 x 356 [4,1,8]
+  CRUSH rule 0 x 357 [8,1,4]
+  CRUSH rule 0 x 358 [2,8,4]
+  CRUSH rule 0 x 359 [6,1,4]
+  CRUSH rule 0 x 360 [2,8]
+  CRUSH rule 0 x 361 [8,4,1]
+  CRUSH rule 0 x 362 [4,1,8]
+  CRUSH rule 0 x 363 [4,2,8]
+  CRUSH rule 0 x 364 [2,8]
+  CRUSH rule 0 x 365 [8,2]
+  CRUSH rule 0 x 366 [8,2]
+  CRUSH rule 0 x 367 [4,2,8]
+  CRUSH rule 0 x 368 [8,4,1]
+  CRUSH rule 0 x 369 [8,2]
+  CRUSH rule 0 x 370 [8,2]
+  CRUSH rule 0 x 371 [1,4,8]
+  CRUSH rule 0 x 372 [1,8]
+  CRUSH rule 0 x 373 [1,8]
+  CRUSH rule 0 x 374 [8,1]
+  CRUSH rule 0 x 375 [8,4,2]
+  CRUSH rule 0 x 376 [8,1,4]
+  CRUSH rule 0 x 377 [1,4,8]
+  CRUSH rule 0 x 378 [1,8]
+  CRUSH rule 0 x 379 [8,2]
+  CRUSH rule 0 x 380 [2,8]
+  CRUSH rule 0 x 381 [1,4,8]
+  CRUSH rule 0 x 382 [1,4,8]
+  CRUSH rule 0 x 383 [4,8,1]
+  CRUSH rule 0 x 384 [8,2,4]
+  CRUSH rule 0 x 385 [8,1]
+  CRUSH rule 0 x 386 [1,8]
+  CRUSH rule 0 x 387 [1,4,8]
+  CRUSH rule 0 x 388 [2,6]
+  CRUSH rule 0 x 389 [1,4,8]
+  CRUSH rule 0 x 390 [4,8,1]
+  CRUSH rule 0 x 391 [4,8,2]
+  CRUSH rule 0 x 392 [1,8,4]
+  CRUSH rule 0 x 393 [2,8]
+  CRUSH rule 0 x 394 [8,2]
+  CRUSH rule 0 x 395 [1,8]
+  CRUSH rule 0 x 396 [4,2,8]
+  CRUSH rule 0 x 397 [2,4,6]
+  CRUSH rule 0 x 398 [2,4,8]
+  CRUSH rule 0 x 399 [8,4,2]
+  CRUSH rule 0 x 400 [8,1,4]
+  CRUSH rule 0 x 401 [1,4,8]
+  CRUSH rule 0 x 402 [8,4,2]
+  CRUSH rule 0 x 403 [1,4,8]
+  CRUSH rule 0 x 404 [4,2,8]
+  CRUSH rule 0 x 405 [8,4,2]
+  CRUSH rule 0 x 406 [2,8]
+  CRUSH rule 0 x 407 [2,8,4]
+  CRUSH rule 0 x 408 [4,1,8]
+  CRUSH rule 0 x 409 [8,4,2]
+  CRUSH rule 0 x 410 [8,4,1]
+  CRUSH rule 0 x 411 [2,8,4]
+  CRUSH rule 0 x 412 [2,6]
+  CRUSH rule 0 x 413 [2,8]
+  CRUSH rule 0 x 414 [4,1,8]
+  CRUSH rule 0 x 415 [2,8]
+  CRUSH rule 0 x 416 [2,8]
+  CRUSH rule 0 x 417 [8,2]
+  CRUSH rule 0 x 418 [8,1,4]
+  CRUSH rule 0 x 419 [8,4,2]
+  CRUSH rule 0 x 420 [1,4,8]
+  CRUSH rule 0 x 421 [8,4,1]
+  CRUSH rule 0 x 422 [6,1]
+  CRUSH rule 0 x 423 [2,4,8]
+  CRUSH rule 0 x 424 [8,1]
+  CRUSH rule 0 x 425 [1,8]
+  CRUSH rule 0 x 426 [8,2]
+  CRUSH rule 0 x 427 [1,8]
+  CRUSH rule 0 x 428 [4,8,1]
+  CRUSH rule 0 x 429 [4,8,1]
+  CRUSH rule 0 x 430 [4,8,2]
+  CRUSH rule 0 x 431 [4,2,8]
+  CRUSH rule 0 x 432 [8,1]
+  CRUSH rule 0 x 433 [8,1]
+  CRUSH rule 0 x 434 [2,8]
+  CRUSH rule 0 x 435 [2,6]
+  CRUSH rule 0 x 436 [4,1,8]
+  CRUSH rule 0 x 437 [8,1]
+  CRUSH rule 0 x 438 [2,4,8]
+  CRUSH rule 0 x 439 [1,8]
+  CRUSH rule 0 x 440 [2,8]
+  CRUSH rule 0 x 441 [4,8,2]
+  CRUSH rule 0 x 442 [2,8]
+  CRUSH rule 0 x 443 [8,1,4]
+  CRUSH rule 0 x 444 [8,1]
+  CRUSH rule 0 x 445 [8,2]
+  CRUSH rule 0 x 446 [1,8]
+  CRUSH rule 0 x 447 [2,4,8]
+  CRUSH rule 0 x 448 [8,2,4]
+  CRUSH rule 0 x 449 [8,2]
+  CRUSH rule 0 x 450 [1,8]
+  CRUSH rule 0 x 451 [8,4,1]
+  CRUSH rule 0 x 452 [8,2]
+  CRUSH rule 0 x 453 [6,2]
+  CRUSH rule 0 x 454 [8,2]
+  CRUSH rule 0 x 455 [2,8,4]
+  CRUSH rule 0 x 456 [8,2]
+  CRUSH rule 0 x 457 [8,1]
+  CRUSH rule 0 x 458 [2,8]
+  CRUSH rule 0 x 459 [2,8,4]
+  CRUSH rule 0 x 460 [8,2]
+  CRUSH rule 0 x 461 [8,2]
+  CRUSH rule 0 x 462 [8,1]
+  CRUSH rule 0 x 463 [8,2]
+  CRUSH rule 0 x 464 [8,4,1]
+  CRUSH rule 0 x 465 [6,1,4]
+  CRUSH rule 0 x 466 [8,2]
+  CRUSH rule 0 x 467 [8,1]
+  CRUSH rule 0 x 468 [8,2,4]
+  CRUSH rule 0 x 469 [8,2]
+  CRUSH rule 0 x 470 [4,1,8]
+  CRUSH rule 0 x 471 [1,8]
+  CRUSH rule 0 x 472 [1,8]
+  CRUSH rule 0 x 473 [1,4,8]
+  CRUSH rule 0 x 474 [8,1]
+  CRUSH rule 0 x 475 [8,2,4]
+  CRUSH rule 0 x 476 [4,8,1]
+  CRUSH rule 0 x 477 [4,8,1]
+  CRUSH rule 0 x 478 [8,1,4]
+  CRUSH rule 0 x 479 [2,8]
+  CRUSH rule 0 x 480 [1,8]
+  CRUSH rule 0 x 481 [2,4,6]
+  CRUSH rule 0 x 482 [8,1]
+  CRUSH rule 0 x 483 [2,8,4]
+  CRUSH rule 0 x 484 [1,8]
+  CRUSH rule 0 x 485 [8,1]
+  CRUSH rule 0 x 486 [4,1,8]
+  CRUSH rule 0 x 487 [1,8]
+  CRUSH rule 0 x 488 [8,1]
+  CRUSH rule 0 x 489 [2,8]
+  CRUSH rule 0 x 490 [6,2]
+  CRUSH rule 0 x 491 [1,8]
+  CRUSH rule 0 x 492 [8,2]
+  CRUSH rule 0 x 493 [2,8]
+  CRUSH rule 0 x 494 [1,8]
+  CRUSH rule 0 x 495 [4,1,6]
+  CRUSH rule 0 x 496 [8,4,1]
+  CRUSH rule 0 x 497 [4,8,2]
+  CRUSH rule 0 x 498 [2,4,8]
+  CRUSH rule 0 x 499 [8,4,2]
+  CRUSH rule 0 x 500 [4,8,1]
+  CRUSH rule 0 x 501 [2,8]
+  CRUSH rule 0 x 502 [6,1]
+  CRUSH rule 0 x 503 [2,8]
+  CRUSH rule 0 x 504 [8,2]
+  CRUSH rule 0 x 505 [1,8]
+  CRUSH rule 0 x 506 [4,1,8]
+  CRUSH rule 0 x 507 [8,2,4]
+  CRUSH rule 0 x 508 [1,8]
+  CRUSH rule 0 x 509 [8,1]
+  CRUSH rule 0 x 510 [8,2]
+  CRUSH rule 0 x 511 [4,8,2]
+  CRUSH rule 0 x 512 [8,2]
+  CRUSH rule 0 x 513 [8,2]
+  CRUSH rule 0 x 514 [8,2]
+  CRUSH rule 0 x 515 [8,4,2]
+  CRUSH rule 0 x 516 [4,1,8]
+  CRUSH rule 0 x 517 [8,2]
+  CRUSH rule 0 x 518 [4,8,1]
+  CRUSH rule 0 x 519 [8,4,1]
+  CRUSH rule 0 x 520 [2,8,4]
+  CRUSH rule 0 x 521 [8,2,4]
+  CRUSH rule 0 x 522 [8,1,4]
+  CRUSH rule 0 x 523 [4,1,8]
+  CRUSH rule 0 x 524 [2,6]
+  CRUSH rule 0 x 525 [2,8]
+  CRUSH rule 0 x 526 [1,8]
+  CRUSH rule 0 x 527 [1,4,6]
+  CRUSH rule 0 x 528 [2,8]
+  CRUSH rule 0 x 529 [4,8,1]
+  CRUSH rule 0 x 530 [8,1]
+  CRUSH rule 0 x 531 [8,1,4]
+  CRUSH rule 0 x 532 [6,4,1]
+  CRUSH rule 0 x 533 [4,8,1]
+  CRUSH rule 0 x 534 [8,2]
+  CRUSH rule 0 x 535 [8,1]
+  CRUSH rule 0 x 536 [8,2]
+  CRUSH rule 0 x 537 [4,8,1]
+  CRUSH rule 0 x 538 [8,4,2]
+  CRUSH rule 0 x 539 [8,2]
+  CRUSH rule 0 x 540 [1,8,4]
+  CRUSH rule 0 x 541 [2,4,8]
+  CRUSH rule 0 x 542 [2,8]
+  CRUSH rule 0 x 543 [8,2]
+  CRUSH rule 0 x 544 [4,8,1]
+  CRUSH rule 0 x 545 [8,1]
+  CRUSH rule 0 x 546 [8,2,4]
+  CRUSH rule 0 x 547 [8,1,4]
+  CRUSH rule 0 x 548 [4,1,8]
+  CRUSH rule 0 x 549 [8,1]
+  CRUSH rule 0 x 550 [2,4,8]
+  CRUSH rule 0 x 551 [8,1]
+  CRUSH rule 0 x 552 [4,8,2]
+  CRUSH rule 0 x 553 [2,8]
+  CRUSH rule 0 x 554 [1,8]
+  CRUSH rule 0 x 555 [4,2,8]
+  CRUSH rule 0 x 556 [8,1]
+  CRUSH rule 0 x 557 [8,2]
+  CRUSH rule 0 x 558 [4,2,8]
+  CRUSH rule 0 x 559 [1,8]
+  CRUSH rule 0 x 560 [8,2]
+  CRUSH rule 0 x 561 [8,4,1]
+  CRUSH rule 0 x 562 [4,1,8]
+  CRUSH rule 0 x 563 [2,8]
+  CRUSH rule 0 x 564 [1,8]
+  CRUSH rule 0 x 565 [4,8,1]
+  CRUSH rule 0 x 566 [4,8,2]
+  CRUSH rule 0 x 567 [4,8,1]
+  CRUSH rule 0 x 568 [8,1]
+  CRUSH rule 0 x 569 [4,1,8]
+  CRUSH rule 0 x 570 [1,8]
+  CRUSH rule 0 x 571 [6,1]
+  CRUSH rule 0 x 572 [4,2,8]
+  CRUSH rule 0 x 573 [1,8]
+  CRUSH rule 0 x 574 [2,8]
+  CRUSH rule 0 x 575 [8,1,4]
+  CRUSH rule 0 x 576 [4,8,1]
+  CRUSH rule 0 x 577 [8,2]
+  CRUSH rule 0 x 578 [8,1]
+  CRUSH rule 0 x 579 [4,2,8]
+  CRUSH rule 0 x 580 [1,8]
+  CRUSH rule 0 x 581 [8,1,4]
+  CRUSH rule 0 x 582 [2,8,4]
+  CRUSH rule 0 x 583 [8,2]
+  CRUSH rule 0 x 584 [8,1,4]
+  CRUSH rule 0 x 585 [8,1,4]
+  CRUSH rule 0 x 586 [1,8,4]
+  CRUSH rule 0 x 587 [2,4,8]
+  CRUSH rule 0 x 588 [4,8,1]
+  CRUSH rule 0 x 589 [8,1]
+  CRUSH rule 0 x 590 [8,2]
+  CRUSH rule 0 x 591 [4,2,8]
+  CRUSH rule 0 x 592 [2,4,8]
+  CRUSH rule 0 x 593 [1,8,4]
+  CRUSH rule 0 x 594 [2,8]
+  CRUSH rule 0 x 595 [8,1]
+  CRUSH rule 0 x 596 [2,8]
+  CRUSH rule 0 x 597 [1,8]
+  CRUSH rule 0 x 598 [2,8]
+  CRUSH rule 0 x 599 [4,1,8]
+  CRUSH rule 0 x 600 [8,1,4]
+  CRUSH rule 0 x 601 [1,8,4]
+  CRUSH rule 0 x 602 [8,2]
+  CRUSH rule 0 x 603 [1,8]
+  CRUSH rule 0 x 604 [8,2]
+  CRUSH rule 0 x 605 [2,8]
+  CRUSH rule 0 x 606 [2,6,4]
+  CRUSH rule 0 x 607 [2,4,8]
+  CRUSH rule 0 x 608 [4,1,8]
+  CRUSH rule 0 x 609 [4,2,8]
+  CRUSH rule 0 x 610 [8,2]
+  CRUSH rule 0 x 611 [1,8]
+  CRUSH rule 0 x 612 [2,8]
+  CRUSH rule 0 x 613 [8,1,4]
+  CRUSH rule 0 x 614 [8,2,4]
+  CRUSH rule 0 x 615 [8,2,4]
+  CRUSH rule 0 x 616 [1,8]
+  CRUSH rule 0 x 617 [8,2,4]
+  CRUSH rule 0 x 618 [8,4,1]
+  CRUSH rule 0 x 619 [4,1,8]
+  CRUSH rule 0 x 620 [1,8]
+  CRUSH rule 0 x 621 [8,1]
+  CRUSH rule 0 x 622 [2,4,8]
+  CRUSH rule 0 x 623 [2,8]
+  CRUSH rule 0 x 624 [4,2,8]
+  CRUSH rule 0 x 625 [2,8]
+  CRUSH rule 0 x 626 [8,2,4]
+  CRUSH rule 0 x 627 [2,8,4]
+  CRUSH rule 0 x 628 [8,1]
+  CRUSH rule 0 x 629 [2,8,4]
+  CRUSH rule 0 x 630 [2,8]
+  CRUSH rule 0 x 631 [1,8,4]
+  CRUSH rule 0 x 632 [8,2]
+  CRUSH rule 0 x 633 [8,2]
+  CRUSH rule 0 x 634 [1,8]
+  CRUSH rule 0 x 635 [4,8,2]
+  CRUSH rule 0 x 636 [1,4,8]
+  CRUSH rule 0 x 637 [1,8]
+  CRUSH rule 0 x 638 [8,2,4]
+  CRUSH rule 0 x 639 [2,8]
+  CRUSH rule 0 x 640 [2,8]
+  CRUSH rule 0 x 641 [8,2]
+  CRUSH rule 0 x 642 [2,8]
+  CRUSH rule 0 x 643 [1,8]
+  CRUSH rule 0 x 644 [8,1]
+  CRUSH rule 0 x 645 [8,2]
+  CRUSH rule 0 x 646 [8,1,4]
+  CRUSH rule 0 x 647 [8,1]
+  CRUSH rule 0 x 648 [1,8]
+  CRUSH rule 0 x 649 [4,8,1]
+  CRUSH rule 0 x 650 [8,4,1]
+  CRUSH rule 0 x 651 [4,6,1]
+  CRUSH rule 0 x 652 [4,8,1]
+  CRUSH rule 0 x 653 [8,2]
+  CRUSH rule 0 x 654 [6,2]
+  CRUSH rule 0 x 655 [1,4,8]
+  CRUSH rule 0 x 656 [8,1]
+  CRUSH rule 0 x 657 [6,1]
+  CRUSH rule 0 x 658 [8,2]
+  CRUSH rule 0 x 659 [4,8,1]
+  CRUSH rule 0 x 660 [8,2]
+  CRUSH rule 0 x 661 [1,8]
+  CRUSH rule 0 x 662 [2,8]
+  CRUSH rule 0 x 663 [1,4,8]
+  CRUSH rule 0 x 664 [1,4,8]
+  CRUSH rule 0 x 665 [4,6,1]
+  CRUSH rule 0 x 666 [2,8]
+  CRUSH rule 0 x 667 [1,4,8]
+  CRUSH rule 0 x 668 [4,8,2]
+  CRUSH rule 0 x 669 [6,4,1]
+  CRUSH rule 0 x 670 [4,1,8]
+  CRUSH rule 0 x 671 [2,8]
+  CRUSH rule 0 x 672 [4,2,8]
+  CRUSH rule 0 x 673 [4,2,8]
+  CRUSH rule 0 x 674 [1,8]
+  CRUSH rule 0 x 675 [1,8,4]
+  CRUSH rule 0 x 676 [2,4,8]
+  CRUSH rule 0 x 677 [4,1,8]
+  CRUSH rule 0 x 678 [2,4,8]
+  CRUSH rule 0 x 679 [8,2]
+  CRUSH rule 0 x 680 [2,8]
+  CRUSH rule 0 x 681 [8,1]
+  CRUSH rule 0 x 682 [1,4,8]
+  CRUSH rule 0 x 683 [1,4,8]
+  CRUSH rule 0 x 684 [8,2,4]
+  CRUSH rule 0 x 685 [8,2,4]
+  CRUSH rule 0 x 686 [1,4,8]
+  CRUSH rule 0 x 687 [6,2]
+  CRUSH rule 0 x 688 [4,8,2]
+  CRUSH rule 0 x 689 [8,4,1]
+  CRUSH rule 0 x 690 [8,1,4]
+  CRUSH rule 0 x 691 [1,8]
+  CRUSH rule 0 x 692 [8,1]
+  CRUSH rule 0 x 693 [8,4,2]
+  CRUSH rule 0 x 694 [8,4,2]
+  CRUSH rule 0 x 695 [2,8,4]
+  CRUSH rule 0 x 696 [1,8]
+  CRUSH rule 0 x 697 [8,1,4]
+  CRUSH rule 0 x 698 [8,2,4]
+  CRUSH rule 0 x 699 [1,8,4]
+  CRUSH rule 0 x 700 [1,6]
+  CRUSH rule 0 x 701 [1,8]
+  CRUSH rule 0 x 702 [2,8]
+  CRUSH rule 0 x 703 [8,1]
+  CRUSH rule 0 x 704 [1,4,8]
+  CRUSH rule 0 x 705 [8,1,4]
+  CRUSH rule 0 x 706 [1,4,8]
+  CRUSH rule 0 x 707 [8,4,1]
+  CRUSH rule 0 x 708 [4,8,1]
+  CRUSH rule 0 x 709 [8,1]
+  CRUSH rule 0 x 710 [8,1]
+  CRUSH rule 0 x 711 [2,4,8]
+  CRUSH rule 0 x 712 [2,8]
+  CRUSH rule 0 x 713 [8,4,1]
+  CRUSH rule 0 x 714 [2,8]
+  CRUSH rule 0 x 715 [1,8]
+  CRUSH rule 0 x 716 [4,8,1]
+  CRUSH rule 0 x 717 [8,2,4]
+  CRUSH rule 0 x 718 [8,1]
+  CRUSH rule 0 x 719 [2,6,4]
+  CRUSH rule 0 x 720 [8,1,4]
+  CRUSH rule 0 x 721 [4,6,1]
+  CRUSH rule 0 x 722 [8,1]
+  CRUSH rule 0 x 723 [4,2,8]
+  CRUSH rule 0 x 724 [2,6]
+  CRUSH rule 0 x 725 [1,8]
+  CRUSH rule 0 x 726 [4,8,2]
+  CRUSH rule 0 x 727 [4,8,2]
+  CRUSH rule 0 x 728 [2,8,4]
+  CRUSH rule 0 x 729 [8,1]
+  CRUSH rule 0 x 730 [4,8,2]
+  CRUSH rule 0 x 731 [4,1,8]
+  CRUSH rule 0 x 732 [1,8]
+  CRUSH rule 0 x 733 [4,8,2]
+  CRUSH rule 0 x 734 [8,4,1]
+  CRUSH rule 0 x 735 [4,8,1]
+  CRUSH rule 0 x 736 [4,8,1]
+  CRUSH rule 0 x 737 [1,8,4]
+  CRUSH rule 0 x 738 [4,1,8]
+  CRUSH rule 0 x 739 [2,8]
+  CRUSH rule 0 x 740 [1,8,4]
+  CRUSH rule 0 x 741 [8,2]
+  CRUSH rule 0 x 742 [8,2]
+  CRUSH rule 0 x 743 [8,1,4]
+  CRUSH rule 0 x 744 [4,8,1]
+  CRUSH rule 0 x 745 [1,8]
+  CRUSH rule 0 x 746 [1,8]
+  CRUSH rule 0 x 747 [8,2]
+  CRUSH rule 0 x 748 [2,8,4]
+  CRUSH rule 0 x 749 [4,8,2]
+  CRUSH rule 0 x 750 [1,8,4]
+  CRUSH rule 0 x 751 [2,8]
+  CRUSH rule 0 x 752 [8,1]
+  CRUSH rule 0 x 753 [8,4,2]
+  CRUSH rule 0 x 754 [8,4,2]
+  CRUSH rule 0 x 755 [1,8,4]
+  CRUSH rule 0 x 756 [8,1]
+  CRUSH rule 0 x 757 [8,1,4]
+  CRUSH rule 0 x 758 [8,1]
+  CRUSH rule 0 x 759 [8,4,2]
+  CRUSH rule 0 x 760 [1,4,8]
+  CRUSH rule 0 x 761 [2,6]
+  CRUSH rule 0 x 762 [2,8]
+  CRUSH rule 0 x 763 [8,4,1]
+  CRUSH rule 0 x 764 [1,8]
+  CRUSH rule 0 x 765 [8,2]
+  CRUSH rule 0 x 766 [8,1]
+  CRUSH rule 0 x 767 [1,8,4]
+  CRUSH rule 0 x 768 [8,4,2]
+  CRUSH rule 0 x 769 [8,2,4]
+  CRUSH rule 0 x 770 [8,1,4]
+  CRUSH rule 0 x 771 [8,2,4]
+  CRUSH rule 0 x 772 [8,4,2]
+  CRUSH rule 0 x 773 [4,2,8]
+  CRUSH rule 0 x 774 [8,1]
+  CRUSH rule 0 x 775 [8,4,2]
+  CRUSH rule 0 x 776 [6,2]
+  CRUSH rule 0 x 777 [4,1,8]
+  CRUSH rule 0 x 778 [1,6,4]
+  CRUSH rule 0 x 779 [2,8]
+  CRUSH rule 0 x 780 [2,4,8]
+  CRUSH rule 0 x 781 [8,2]
+  CRUSH rule 0 x 782 [4,2,8]
+  CRUSH rule 0 x 783 [8,1,4]
+  CRUSH rule 0 x 784 [1,4,8]
+  CRUSH rule 0 x 785 [8,1,4]
+  CRUSH rule 0 x 786 [8,1]
+  CRUSH rule 0 x 787 [1,8,4]
+  CRUSH rule 0 x 788 [8,2,4]
+  CRUSH rule 0 x 789 [1,8]
+  CRUSH rule 0 x 790 [8,1]
+  CRUSH rule 0 x 791 [4,8,2]
+  CRUSH rule 0 x 792 [4,8,1]
+  CRUSH rule 0 x 793 [8,2,4]
+  CRUSH rule 0 x 794 [2,8,4]
+  CRUSH rule 0 x 795 [1,8]
+  CRUSH rule 0 x 796 [8,1]
+  CRUSH rule 0 x 797 [2,4,8]
+  CRUSH rule 0 x 798 [6,1]
+  CRUSH rule 0 x 799 [4,2,8]
+  CRUSH rule 0 x 800 [2,8]
+  CRUSH rule 0 x 801 [4,8,2]
+  CRUSH rule 0 x 802 [1,8,4]
+  CRUSH rule 0 x 803 [2,8]
+  CRUSH rule 0 x 804 [8,2]
+  CRUSH rule 0 x 805 [8,2]
+  CRUSH rule 0 x 806 [1,4,8]
+  CRUSH rule 0 x 807 [4,8,2]
+  CRUSH rule 0 x 808 [8,2]
+  CRUSH rule 0 x 809 [1,8]
+  CRUSH rule 0 x 810 [8,2]
+  CRUSH rule 0 x 811 [8,1]
+  CRUSH rule 0 x 812 [8,4,1]
+  CRUSH rule 0 x 813 [8,4,2]
+  CRUSH rule 0 x 814 [8,2]
+  CRUSH rule 0 x 815 [4,1,8]
+  CRUSH rule 0 x 816 [2,8]
+  CRUSH rule 0 x 817 [8,2]
+  CRUSH rule 0 x 818 [1,8]
+  CRUSH rule 0 x 819 [1,8]
+  CRUSH rule 0 x 820 [4,8,1]
+  CRUSH rule 0 x 821 [4,8,2]
+  CRUSH rule 0 x 822 [2,4,8]
+  CRUSH rule 0 x 823 [4,8,2]
+  CRUSH rule 0 x 824 [8,1]
+  CRUSH rule 0 x 825 [2,8,4]
+  CRUSH rule 0 x 826 [8,1,4]
+  CRUSH rule 0 x 827 [2,6,4]
+  CRUSH rule 0 x 828 [2,8]
+  CRUSH rule 0 x 829 [8,1]
+  CRUSH rule 0 x 830 [2,4,8]
+  CRUSH rule 0 x 831 [1,8]
+  CRUSH rule 0 x 832 [4,8,2]
+  CRUSH rule 0 x 833 [2,8]
+  CRUSH rule 0 x 834 [1,8]
+  CRUSH rule 0 x 835 [8,4,1]
+  CRUSH rule 0 x 836 [4,8,1]
+  CRUSH rule 0 x 837 [8,4,1]
+  CRUSH rule 0 x 838 [6,2,4]
+  CRUSH rule 0 x 839 [2,8]
+  CRUSH rule 0 x 840 [8,2]
+  CRUSH rule 0 x 841 [4,8,1]
+  CRUSH rule 0 x 842 [2,8]
+  CRUSH rule 0 x 843 [8,4,2]
+  CRUSH rule 0 x 844 [8,2]
+  CRUSH rule 0 x 845 [4,8,2]
+  CRUSH rule 0 x 846 [4,2,8]
+  CRUSH rule 0 x 847 [2,8]
+  CRUSH rule 0 x 848 [2,8,4]
+  CRUSH rule 0 x 849 [4,8,1]
+  CRUSH rule 0 x 850 [1,6]
+  CRUSH rule 0 x 851 [6,1]
+  CRUSH rule 0 x 852 [8,4,2]
+  CRUSH rule 0 x 853 [6,2]
+  CRUSH rule 0 x 854 [8,2]
+  CRUSH rule 0 x 855 [8,2]
+  CRUSH rule 0 x 856 [8,4,1]
+  CRUSH rule 0 x 857 [8,1]
+  CRUSH rule 0 x 858 [6,2]
+  CRUSH rule 0 x 859 [8,2,4]
+  CRUSH rule 0 x 860 [2,8]
+  CRUSH rule 0 x 861 [8,1]
+  CRUSH rule 0 x 862 [8,1]
+  CRUSH rule 0 x 863 [8,1]
+  CRUSH rule 0 x 864 [8,1]
+  CRUSH rule 0 x 865 [8,1]
+  CRUSH rule 0 x 866 [8,2]
+  CRUSH rule 0 x 867 [8,1]
+  CRUSH rule 0 x 868 [8,2]
+  CRUSH rule 0 x 869 [8,4,2]
+  CRUSH rule 0 x 870 [2,8]
+  CRUSH rule 0 x 871 [1,8]
+  CRUSH rule 0 x 872 [1,8]
+  CRUSH rule 0 x 873 [4,8,1]
+  CRUSH rule 0 x 874 [2,6]
+  CRUSH rule 0 x 875 [2,8,4]
+  CRUSH rule 0 x 876 [4,8,1]
+  CRUSH rule 0 x 877 [8,4,2]
+  CRUSH rule 0 x 878 [2,8]
+  CRUSH rule 0 x 879 [8,1]
+  CRUSH rule 0 x 880 [1,8]
+  CRUSH rule 0 x 881 [4,8,2]
+  CRUSH rule 0 x 882 [1,8]
+  CRUSH rule 0 x 883 [2,4,8]
+  CRUSH rule 0 x 884 [8,1,4]
+  CRUSH rule 0 x 885 [4,1,8]
+  CRUSH rule 0 x 886 [8,2]
+  CRUSH rule 0 x 887 [8,4,2]
+  CRUSH rule 0 x 888 [8,1]
+  CRUSH rule 0 x 889 [2,8]
+  CRUSH rule 0 x 890 [8,1,4]
+  CRUSH rule 0 x 891 [1,8]
+  CRUSH rule 0 x 892 [8,2,4]
+  CRUSH rule 0 x 893 [2,6]
+  CRUSH rule 0 x 894 [8,4,1]
+  CRUSH rule 0 x 895 [4,1,8]
+  CRUSH rule 0 x 896 [1,8]
+  CRUSH rule 0 x 897 [2,8]
+  CRUSH rule 0 x 898 [1,4,8]
+  CRUSH rule 0 x 899 [1,8]
+  CRUSH rule 0 x 900 [4,1,8]
+  CRUSH rule 0 x 901 [2,8]
+  CRUSH rule 0 x 902 [8,4,2]
+  CRUSH rule 0 x 903 [8,1]
+  CRUSH rule 0 x 904 [8,2]
+  CRUSH rule 0 x 905 [8,2]
+  CRUSH rule 0 x 906 [1,8]
+  CRUSH rule 0 x 907 [8,2]
+  CRUSH rule 0 x 908 [8,1]
+  CRUSH rule 0 x 909 [2,8]
+  CRUSH rule 0 x 910 [8,1]
+  CRUSH rule 0 x 911 [8,1]
+  CRUSH rule 0 x 912 [1,8]
+  CRUSH rule 0 x 913 [8,1,4]
+  CRUSH rule 0 x 914 [6,4,1]
+  CRUSH rule 0 x 915 [8,2]
+  CRUSH rule 0 x 916 [4,1,8]
+  CRUSH rule 0 x 917 [1,4,8]
+  CRUSH rule 0 x 918 [8,2]
+  CRUSH rule 0 x 919 [8,2]
+  CRUSH rule 0 x 920 [8,2]
+  CRUSH rule 0 x 921 [1,8]
+  CRUSH rule 0 x 922 [8,4,1]
+  CRUSH rule 0 x 923 [4,8,2]
+  CRUSH rule 0 x 924 [1,8]
+  CRUSH rule 0 x 925 [4,8,2]
+  CRUSH rule 0 x 926 [2,8]
+  CRUSH rule 0 x 927 [1,8,4]
+  CRUSH rule 0 x 928 [8,1]
+  CRUSH rule 0 x 929 [4,2,8]
+  CRUSH rule 0 x 930 [2,8]
+  CRUSH rule 0 x 931 [2,8]
+  CRUSH rule 0 x 932 [4,2,8]
+  CRUSH rule 0 x 933 [8,4,2]
+  CRUSH rule 0 x 934 [8,2]
+  CRUSH rule 0 x 935 [8,2]
+  CRUSH rule 0 x 936 [1,8]
+  CRUSH rule 0 x 937 [4,8,1]
+  CRUSH rule 0 x 938 [8,4,2]
+  CRUSH rule 0 x 939 [2,8,4]
+  CRUSH rule 0 x 940 [8,2]
+  CRUSH rule 0 x 941 [2,6]
+  CRUSH rule 0 x 942 [1,8]
+  CRUSH rule 0 x 943 [8,2]
+  CRUSH rule 0 x 944 [8,1]
+  CRUSH rule 0 x 945 [8,2,4]
+  CRUSH rule 0 x 946 [2,8,4]
+  CRUSH rule 0 x 947 [2,8]
+  CRUSH rule 0 x 948 [8,1]
+  CRUSH rule 0 x 949 [6,2]
+  CRUSH rule 0 x 950 [8,1]
+  CRUSH rule 0 x 951 [8,2]
+  CRUSH rule 0 x 952 [2,8,4]
+  CRUSH rule 0 x 953 [1,4,8]
+  CRUSH rule 0 x 954 [2,8]
+  CRUSH rule 0 x 955 [8,1,4]
+  CRUSH rule 0 x 956 [1,8,4]
+  CRUSH rule 0 x 957 [8,1,4]
+  CRUSH rule 0 x 958 [8,4,2]
+  CRUSH rule 0 x 959 [4,1,8]
+  CRUSH rule 0 x 960 [6,2]
+  CRUSH rule 0 x 961 [1,8]
+  CRUSH rule 0 x 962 [8,4,2]
+  CRUSH rule 0 x 963 [2,4,8]
+  CRUSH rule 0 x 964 [2,8]
+  CRUSH rule 0 x 965 [8,1]
+  CRUSH rule 0 x 966 [4,8,1]
+  CRUSH rule 0 x 967 [8,4,2]
+  CRUSH rule 0 x 968 [8,2]
+  CRUSH rule 0 x 969 [8,2,4]
+  CRUSH rule 0 x 970 [2,8,4]
+  CRUSH rule 0 x 971 [1,8]
+  CRUSH rule 0 x 972 [1,8]
+  CRUSH rule 0 x 973 [1,8]
+  CRUSH rule 0 x 974 [4,1,8]
+  CRUSH rule 0 x 975 [4,8,1]
+  CRUSH rule 0 x 976 [4,8,2]
+  CRUSH rule 0 x 977 [8,4,2]
+  CRUSH rule 0 x 978 [8,2,4]
+  CRUSH rule 0 x 979 [8,2,4]
+  CRUSH rule 0 x 980 [8,2,4]
+  CRUSH rule 0 x 981 [8,1]
+  CRUSH rule 0 x 982 [1,8]
+  CRUSH rule 0 x 983 [4,8,1]
+  CRUSH rule 0 x 984 [2,8]
+  CRUSH rule 0 x 985 [2,4,8]
+  CRUSH rule 0 x 986 [8,4,2]
+  CRUSH rule 0 x 987 [2,8]
+  CRUSH rule 0 x 988 [1,4,6]
+  CRUSH rule 0 x 989 [1,8]
+  CRUSH rule 0 x 990 [1,8,4]
+  CRUSH rule 0 x 991 [1,4,8]
+  CRUSH rule 0 x 992 [8,1,4]
+  CRUSH rule 0 x 993 [2,8,4]
+  CRUSH rule 0 x 994 [4,8,1]
+  CRUSH rule 0 x 995 [8,1,4]
+  CRUSH rule 0 x 996 [8,4,1]
+  CRUSH rule 0 x 997 [8,4,1]
+  CRUSH rule 0 x 998 [8,1,4]
+  CRUSH rule 0 x 999 [1,8,4]
+  CRUSH rule 0 x 1000 [8,4,1]
+  CRUSH rule 0 x 1001 [2,8]
+  CRUSH rule 0 x 1002 [1,8]
+  CRUSH rule 0 x 1003 [2,8]
+  CRUSH rule 0 x 1004 [8,1,4]
+  CRUSH rule 0 x 1005 [8,1]
+  CRUSH rule 0 x 1006 [1,8,4]
+  CRUSH rule 0 x 1007 [1,4,8]
+  CRUSH rule 0 x 1008 [1,8]
+  CRUSH rule 0 x 1009 [6,4,1]
+  CRUSH rule 0 x 1010 [1,8]
+  CRUSH rule 0 x 1011 [4,2,8]
+  CRUSH rule 0 x 1012 [1,8]
+  CRUSH rule 0 x 1013 [2,8]
+  CRUSH rule 0 x 1014 [2,8,4]
+  CRUSH rule 0 x 1015 [8,2]
+  CRUSH rule 0 x 1016 [2,4,8]
+  CRUSH rule 0 x 1017 [6,1,4]
+  CRUSH rule 0 x 1018 [4,1,8]
+  CRUSH rule 0 x 1019 [4,8,1]
+  CRUSH rule 0 x 1020 [1,8]
+  CRUSH rule 0 x 1021 [2,8]
+  CRUSH rule 0 x 1022 [1,8,4]
+  CRUSH rule 0 x 1023 [4,2,8]
+  rule 0 (choose) num_rep 3 result size == 2:\t501/1024 (esc)
+  rule 0 (choose) num_rep 3 result size == 3:\t523/1024 (esc)
+  rule 1 (choose-two), x = 0..1023, numrep = 2..3
+  CRUSH rule 1 x 0 [2,1]
+  CRUSH rule 1 x 1 [2,8]
+  CRUSH rule 1 x 2 [1,8]
+  CRUSH rule 1 x 3 [8,1]
+  CRUSH rule 1 x 4 [4,8]
+  CRUSH rule 1 x 5 [8,2]
+  CRUSH rule 1 x 6 [2,8]
+  CRUSH rule 1 x 7 [4,8]
+  CRUSH rule 1 x 8 [4,2]
+  CRUSH rule 1 x 9 [2,4]
+  CRUSH rule 1 x 10 [2,1]
+  CRUSH rule 1 x 11 [2,8]
+  CRUSH rule 1 x 12 [2,1]
+  CRUSH rule 1 x 13 [4,8]
+  CRUSH rule 1 x 14 [8,1]
+  CRUSH rule 1 x 15 [8,2]
+  CRUSH rule 1 x 16 [8,1]
+  CRUSH rule 1 x 17 [4,8]
+  CRUSH rule 1 x 18 [1,6]
+  CRUSH rule 1 x 19 [8,4]
+  CRUSH rule 1 x 20 [2,1]
+  CRUSH rule 1 x 21 [8,1]
+  CRUSH rule 1 x 22 [8]
+  CRUSH rule 1 x 23 [4,8]
+  CRUSH rule 1 x 24 [1,2]
+  CRUSH rule 1 x 25 [4,8]
+  CRUSH rule 1 x 26 [2,8]
+  CRUSH rule 1 x 27 [4,1]
+  CRUSH rule 1 x 28 [8,2]
+  CRUSH rule 1 x 29 [8,4]
+  CRUSH rule 1 x 30 [4,8]
+  CRUSH rule 1 x 31 [8,2]
+  CRUSH rule 1 x 32 [2,6]
+  CRUSH rule 1 x 33 [2,8]
+  CRUSH rule 1 x 34 [2,1]
+  CRUSH rule 1 x 35 [1,8]
+  CRUSH rule 1 x 36 [8,1]
+  CRUSH rule 1 x 37 [1,8]
+  CRUSH rule 1 x 38 [4,8]
+  CRUSH rule 1 x 39 [8,1]
+  CRUSH rule 1 x 40 [8,2]
+  CRUSH rule 1 x 41 [2,1]
+  CRUSH rule 1 x 42 [1,2]
+  CRUSH rule 1 x 43 [1,2]
+  CRUSH rule 1 x 44 [1,8]
+  CRUSH rule 1 x 45 [8,2]
+  CRUSH rule 1 x 46 [2,1]
+  CRUSH rule 1 x 47 [4,2]
+  CRUSH rule 1 x 48 [8,1]
+  CRUSH rule 1 x 49 [8,2]
+  CRUSH rule 1 x 50 [4,8]
+  CRUSH rule 1 x 51 [2,8]
+  CRUSH rule 1 x 52 [8,2]
+  CRUSH rule 1 x 53 [4,2]
+  CRUSH rule 1 x 54 [8,4]
+  CRUSH rule 1 x 55 [8,1]
+  CRUSH rule 1 x 56 [8,4]
+  CRUSH rule 1 x 57 [2,1]
+  CRUSH rule 1 x 58 [1,2]
+  CRUSH rule 1 x 59 [8,2]
+  CRUSH rule 1 x 60 [4,8]
+  CRUSH rule 1 x 61 [4,8]
+  CRUSH rule 1 x 62 [8,1]
+  CRUSH rule 1 x 63 [8,1]
+  CRUSH rule 1 x 64 [4,2]
+  CRUSH rule 1 x 65 [8,4]
+  CRUSH rule 1 x 66 [4,2]
+  CRUSH rule 1 x 67 [4,2]
+  CRUSH rule 1 x 68 [1,2]
+  CRUSH rule 1 x 69 [1,2]
+  CRUSH rule 1 x 70 [8,2]
+  CRUSH rule 1 x 71 [2,8]
+  CRUSH rule 1 x 72 [8,1]
+  CRUSH rule 1 x 73 [2,8]
+  CRUSH rule 1 x 74 [1,8]
+  CRUSH rule 1 x 75 [4,2]
+  CRUSH rule 1 x 76 [4,1]
+  CRUSH rule 1 x 77 [8,2]
+  CRUSH rule 1 x 78 [1]
+  CRUSH rule 1 x 79 [4,1]
+  CRUSH rule 1 x 80 [2,4]
+  CRUSH rule 1 x 81 [2,1]
+  CRUSH rule 1 x 82 [6,1]
+  CRUSH rule 1 x 83 [2,8]
+  CRUSH rule 1 x 84 [8,2]
+  CRUSH rule 1 x 85 [4,8]
+  CRUSH rule 1 x 86 [2,1]
+  CRUSH rule 1 x 87 [2,8]
+  CRUSH rule 1 x 88 [1,6]
+  CRUSH rule 1 x 89 [2,1]
+  CRUSH rule 1 x 90 [8,2]
+  CRUSH rule 1 x 91 [4,8]
+  CRUSH rule 1 x 92 [1,8]
+  CRUSH rule 1 x 93 [8,4]
+  CRUSH rule 1 x 94 [1,2]
+  CRUSH rule 1 x 95 [8,1]
+  CRUSH rule 1 x 96 [1,8]
+  CRUSH rule 1 x 97 [8,1]
+  CRUSH rule 1 x 98 [2,1]
+  CRUSH rule 1 x 99 [2,8]
+  CRUSH rule 1 x 100 [1,8]
+  CRUSH rule 1 x 101 [8,2]
+  CRUSH rule 1 x 102 [2,1]
+  CRUSH rule 1 x 103 [2,8]
+  CRUSH rule 1 x 104 [8,4]
+  CRUSH rule 1 x 105 [2,4]
+  CRUSH rule 1 x 106 [1,8]
+  CRUSH rule 1 x 107 [2,1]
+  CRUSH rule 1 x 108 [8,2]
+  CRUSH rule 1 x 109 [1,2]
+  CRUSH rule 1 x 110 [4,2]
+  CRUSH rule 1 x 111 [2,1]
+  CRUSH rule 1 x 112 [2,1]
+  CRUSH rule 1 x 113 [8,2]
+  CRUSH rule 1 x 114 [8,4]
+  CRUSH rule 1 x 115 [8,2]
+  CRUSH rule 1 x 116 [1,2]
+  CRUSH rule 1 x 117 [6,8]
+  CRUSH rule 1 x 118 [2,8]
+  CRUSH rule 1 x 119 [2,8]
+  CRUSH rule 1 x 120 [2,1]
+  CRUSH rule 1 x 121 [2,1]
+  CRUSH rule 1 x 122 [8,1]
+  CRUSH rule 1 x 123 [2,8]
+  CRUSH rule 1 x 124 [8]
+  CRUSH rule 1 x 125 [1,8]
+  CRUSH rule 1 x 126 [8,2]
+  CRUSH rule 1 x 127 [4,8]
+  CRUSH rule 1 x 128 [2,8]
+  CRUSH rule 1 x 129 [2,1]
+  CRUSH rule 1 x 130 [4,8]
+  CRUSH rule 1 x 131 [1,2]
+  CRUSH rule 1 x 132 [1,2]
+  CRUSH rule 1 x 133 [8,2]
+  CRUSH rule 1 x 134 [1,8]
+  CRUSH rule 1 x 135 [4,8]
+  CRUSH rule 1 x 136 [2,1]
+  CRUSH rule 1 x 137 [8,4]
+  CRUSH rule 1 x 138 [8,4]
+  CRUSH rule 1 x 139 [4,2]
+  CRUSH rule 1 x 140 [1,8]
+  CRUSH rule 1 x 141 [8,1]
+  CRUSH rule 1 x 142 [4,6]
+  CRUSH rule 1 x 143 [4,8]
+  CRUSH rule 1 x 144 [8,1]
+  CRUSH rule 1 x 145 [8,2]
+  CRUSH rule 1 x 146 [2,8]
+  CRUSH rule 1 x 147 [2,8]
+  CRUSH rule 1 x 148 [4,1]
+  CRUSH rule 1 x 149 [4,8]
+  CRUSH rule 1 x 150 [1,8]
+  CRUSH rule 1 x 151 [1,2]
+  CRUSH rule 1 x 152 [8,1]
+  CRUSH rule 1 x 153 [8,4]
+  CRUSH rule 1 x 154 [4,2]
+  CRUSH rule 1 x 155 [4,8]
+  CRUSH rule 1 x 156 [4,8]
+  CRUSH rule 1 x 157 [2,1]
+  CRUSH rule 1 x 158 [2,8]
+  CRUSH rule 1 x 159 [8,2]
+  CRUSH rule 1 x 160 [2,8]
+  CRUSH rule 1 x 161 [1,4]
+  CRUSH rule 1 x 162 [1,8]
+  CRUSH rule 1 x 163 [4,8]
+  CRUSH rule 1 x 164 [8,2]
+  CRUSH rule 1 x 165 [8,2]
+  CRUSH rule 1 x 166 [2,1]
+  CRUSH rule 1 x 167 [1,2]
+  CRUSH rule 1 x 168 [4,2]
+  CRUSH rule 1 x 169 [2,8]
+  CRUSH rule 1 x 170 [1,2]
+  CRUSH rule 1 x 171 [8,4]
+  CRUSH rule 1 x 172 [1,8]
+  CRUSH rule 1 x 173 [8,4]
+  CRUSH rule 1 x 174 [1,2]
+  CRUSH rule 1 x 175 [8,1]
+  CRUSH rule 1 x 176 [2,1]
+  CRUSH rule 1 x 177 [8,2]
+  CRUSH rule 1 x 178 [4,2]
+  CRUSH rule 1 x 179 [8,1]
+  CRUSH rule 1 x 180 [1,8]
+  CRUSH rule 1 x 181 [8,2]
+  CRUSH rule 1 x 182 [8,1]
+  CRUSH rule 1 x 183 [8,1]
+  CRUSH rule 1 x 184 [4,8]
+  CRUSH rule 1 x 185 [8,4]
+  CRUSH rule 1 x 186 [2,1]
+  CRUSH rule 1 x 187 [1,8]
+  CRUSH rule 1 x 188 [1,8]
+  CRUSH rule 1 x 189 [1,8]
+  CRUSH rule 1 x 190 [1,2]
+  CRUSH rule 1 x 191 [8,4]
+  CRUSH rule 1 x 192 [4,1]
+  CRUSH rule 1 x 193 [4,2]
+  CRUSH rule 1 x 194 [1,8]
+  CRUSH rule 1 x 195 [8,4]
+  CRUSH rule 1 x 196 [8,1]
+  CRUSH rule 1 x 197 [8,4]
+  CRUSH rule 1 x 198 [2,1]
+  CRUSH rule 1 x 199 [1,4]
+  CRUSH rule 1 x 200 [1,8]
+  CRUSH rule 1 x 201 [8,1]
+  CRUSH rule 1 x 202 [8,2]
+  CRUSH rule 1 x 203 [1,8]
+  CRUSH rule 1 x 204 [2,1]
+  CRUSH rule 1 x 205 [1,8]
+  CRUSH rule 1 x 206 [1,2]
+  CRUSH rule 1 x 207 [2,1]
+  CRUSH rule 1 x 208 [8,1]
+  CRUSH rule 1 x 209 [1,2]
+  CRUSH rule 1 x 210 [1,2]
+  CRUSH rule 1 x 211 [4,8]
+  CRUSH rule 1 x 212 [8,1]
+  CRUSH rule 1 x 213 [8,4]
+  CRUSH rule 1 x 214 [8]
+  CRUSH rule 1 x 215 [8,1]
+  CRUSH rule 1 x 216 [8,1]
+  CRUSH rule 1 x 217 [1,2]
+  CRUSH rule 1 x 218 [2,8]
+  CRUSH rule 1 x 219 [2,8]
+  CRUSH rule 1 x 220 [4,8]
+  CRUSH rule 1 x 221 [1,8]
+  CRUSH rule 1 x 222 [8,1]
+  CRUSH rule 1 x 223 [1,2]
+  CRUSH rule 1 x 224 [1,4]
+  CRUSH rule 1 x 225 [8,2]
+  CRUSH rule 1 x 226 [8,2]
+  CRUSH rule 1 x 227 [4,8]
+  CRUSH rule 1 x 228 [1,8]
+  CRUSH rule 1 x 229 [4,1]
+  CRUSH rule 1 x 230 [4,8]
+  CRUSH rule 1 x 231 [4,8]
+  CRUSH rule 1 x 232 [2,8]
+  CRUSH rule 1 x 233 [8,2]
+  CRUSH rule 1 x 234 [1,2]
+  CRUSH rule 1 x 235 [4,8]
+  CRUSH rule 1 x 236 [1,2]
+  CRUSH rule 1 x 237 [4,8]
+  CRUSH rule 1 x 238 [8]
+  CRUSH rule 1 x 239 [8,6]
+  CRUSH rule 1 x 240 [4,8]
+  CRUSH rule 1 x 241 [2,1]
+  CRUSH rule 1 x 242 [2,8]
+  CRUSH rule 1 x 243 [8,2]
+  CRUSH rule 1 x 244 [4,8]
+  CRUSH rule 1 x 245 [8,2]
+  CRUSH rule 1 x 246 [1,2]
+  CRUSH rule 1 x 247 [8,2]
+  CRUSH rule 1 x 248 [8,2]
+  CRUSH rule 1 x 249 [2,1]
+  CRUSH rule 1 x 250 [2,1]
+  CRUSH rule 1 x 251 [2,8]
+  CRUSH rule 1 x 252 [4,8]
+  CRUSH rule 1 x 253 [1,2]
+  CRUSH rule 1 x 254 [4,8]
+  CRUSH rule 1 x 255 [1,8]
+  CRUSH rule 1 x 256 [4,8]
+  CRUSH rule 1 x 257 [2,8]
+  CRUSH rule 1 x 258 [4]
+  CRUSH rule 1 x 259 [8,6]
+  CRUSH rule 1 x 260 [8,2]
+  CRUSH rule 1 x 261 [8,1]
+  CRUSH rule 1 x 262 [8,2]
+  CRUSH rule 1 x 263 [8,4]
+  CRUSH rule 1 x 264 [1,8]
+  CRUSH rule 1 x 265 [8]
+  CRUSH rule 1 x 266 [8,2]
+  CRUSH rule 1 x 267 [2,8]
+  CRUSH rule 1 x 268 [1,8]
+  CRUSH rule 1 x 269 [1,8]
+  CRUSH rule 1 x 270 [4,1]
+  CRUSH rule 1 x 271 [8,4]
+  CRUSH rule 1 x 272 [2,8]
+  CRUSH rule 1 x 273 [4,1]
+  CRUSH rule 1 x 274 [8,2]
+  CRUSH rule 1 x 275 [4,8]
+  CRUSH rule 1 x 276 [8,1]
+  CRUSH rule 1 x 277 [8,1]
+  CRUSH rule 1 x 278 [8,2]
+  CRUSH rule 1 x 279 [8,4]
+  CRUSH rule 1 x 280 [2,8]
+  CRUSH rule 1 x 281 [8,2]
+  CRUSH rule 1 x 282 [1,2]
+  CRUSH rule 1 x 283 [8,2]
+  CRUSH rule 1 x 284 [8]
+  CRUSH rule 1 x 285 [4,8]
+  CRUSH rule 1 x 286 [2,1]
+  CRUSH rule 1 x 287 [1,2]
+  CRUSH rule 1 x 288 [8,1]
+  CRUSH rule 1 x 289 [4,8]
+  CRUSH rule 1 x 290 [1,4]
+  CRUSH rule 1 x 291 [1,2]
+  CRUSH rule 1 x 292 [8,2]
+  CRUSH rule 1 x 293 [8,1]
+  CRUSH rule 1 x 294 [8,4]
+  CRUSH rule 1 x 295 [4,8]
+  CRUSH rule 1 x 296 [4,1]
+  CRUSH rule 1 x 297 [8,2]
+  CRUSH rule 1 x 298 [1,2]
+  CRUSH rule 1 x 299 [2,1]
+  CRUSH rule 1 x 300 [8,2]
+  CRUSH rule 1 x 301 [1,8]
+  CRUSH rule 1 x 302 [8,1]
+  CRUSH rule 1 x 303 [8,4]
+  CRUSH rule 1 x 304 [2,8]
+  CRUSH rule 1 x 305 [2,8]
+  CRUSH rule 1 x 306 [1,8]
+  CRUSH rule 1 x 307 [2,1]
+  CRUSH rule 1 x 308 [2,8]
+  CRUSH rule 1 x 309 [8,2]
+  CRUSH rule 1 x 310 [4,6]
+  CRUSH rule 1 x 311 [4,2]
+  CRUSH rule 1 x 312 [2,1]
+  CRUSH rule 1 x 313 [4,8]
+  CRUSH rule 1 x 314 [6,1]
+  CRUSH rule 1 x 315 [2,1]
+  CRUSH rule 1 x 316 [8,2]
+  CRUSH rule 1 x 317 [2,8]
+  CRUSH rule 1 x 318 [8,2]
+  CRUSH rule 1 x 319 [8,2]
+  CRUSH rule 1 x 320 [8,2]
+  CRUSH rule 1 x 321 [1,2]
+  CRUSH rule 1 x 322 [2,8]
+  CRUSH rule 1 x 323 [4,8]
+  CRUSH rule 1 x 324 [8,1]
+  CRUSH rule 1 x 325 [4,8]
+  CRUSH rule 1 x 326 [8,6]
+  CRUSH rule 1 x 327 [1,8]
+  CRUSH rule 1 x 328 [8,4]
+  CRUSH rule 1 x 329 [4,8]
+  CRUSH rule 1 x 330 [4,8]
+  CRUSH rule 1 x 331 [2,8]
+  CRUSH rule 1 x 332 [2,1]
+  CRUSH rule 1 x 333 [8,1]
+  CRUSH rule 1 x 334 [8,1]
+  CRUSH rule 1 x 335 [8,1]
+  CRUSH rule 1 x 336 [4,8]
+  CRUSH rule 1 x 337 [8,2]
+  CRUSH rule 1 x 338 [1,8]
+  CRUSH rule 1 x 339 [8]
+  CRUSH rule 1 x 340 [2,1]
+  CRUSH rule 1 x 341 [4,1]
+  CRUSH rule 1 x 342 [2,8]
+  CRUSH rule 1 x 343 [8]
+  CRUSH rule 1 x 344 [6,2]
+  CRUSH rule 1 x 345 [2,1]
+  CRUSH rule 1 x 346 [8,2]
+  CRUSH rule 1 x 347 [4,1]
+  CRUSH rule 1 x 348 [8,2]
+  CRUSH rule 1 x 349 [1,8]
+  CRUSH rule 1 x 350 [8,1]
+  CRUSH rule 1 x 351 [1,8]
+  CRUSH rule 1 x 352 [1,2]
+  CRUSH rule 1 x 353 [8,1]
+  CRUSH rule 1 x 354 [1,8]
+  CRUSH rule 1 x 355 [1,2]
+  CRUSH rule 1 x 356 [4,8]
+  CRUSH rule 1 x 357 [8,1]
+  CRUSH rule 1 x 358 [2,1]
+  CRUSH rule 1 x 359 [6,8]
+  CRUSH rule 1 x 360 [1,2]
+  CRUSH rule 1 x 361 [8,4]
+  CRUSH rule 1 x 362 [4,6]
+  CRUSH rule 1 x 363 [4,1]
+  CRUSH rule 1 x 364 [2,8]
+  CRUSH rule 1 x 365 [8,1]
+  CRUSH rule 1 x 366 [8,2]
+  CRUSH rule 1 x 367 [4,8]
+  CRUSH rule 1 x 368 [8,4]
+  CRUSH rule 1 x 369 [8]
+  CRUSH rule 1 x 370 [8,2]
+  CRUSH rule 1 x 371 [1,4]
+  CRUSH rule 1 x 372 [8,1]
+  CRUSH rule 1 x 373 [1,8]
+  CRUSH rule 1 x 374 [2,8]
+  CRUSH rule 1 x 375 [8,4]
+  CRUSH rule 1 x 376 [8,1]
+  CRUSH rule 1 x 377 [1,2]
+  CRUSH rule 1 x 378 [1,2]
+  CRUSH rule 1 x 379 [8,1]
+  CRUSH rule 1 x 380 [2,1]
+  CRUSH rule 1 x 381 [1,4]
+  CRUSH rule 1 x 382 [1,4]
+  CRUSH rule 1 x 383 [4,1]
+  CRUSH rule 1 x 384 [8,2]
+  CRUSH rule 1 x 385 [8,1]
+  CRUSH rule 1 x 386 [1,8]
+  CRUSH rule 1 x 387 [1,4]
+  CRUSH rule 1 x 388 [8,1]
+  CRUSH rule 1 x 389 [1,4]
+  CRUSH rule 1 x 390 [4,8]
+  CRUSH rule 1 x 391 [4,8]
+  CRUSH rule 1 x 392 [1,8]
+  CRUSH rule 1 x 393 [8,2]
+  CRUSH rule 1 x 394 [8,1]
+  CRUSH rule 1 x 395 [8,1]
+  CRUSH rule 1 x 396 [4,2]
+  CRUSH rule 1 x 397 [2,1]
+  CRUSH rule 1 x 398 [2,4]
+  CRUSH rule 1 x 399 [8,2]
+  CRUSH rule 1 x 400 [8,1]
+  CRUSH rule 1 x 401 [1,2]
+  CRUSH rule 1 x 402 [8,1]
+  CRUSH rule 1 x 403 [1,2]
+  CRUSH rule 1 x 404 [4,2]
+  CRUSH rule 1 x 405 [8,4]
+  CRUSH rule 1 x 406 [2,1]
+  CRUSH rule 1 x 407 [2,8]
+  CRUSH rule 1 x 408 [4,1]
+  CRUSH rule 1 x 409 [8,4]
+  CRUSH rule 1 x 410 [8,1]
+  CRUSH rule 1 x 411 [2,1]
+  CRUSH rule 1 x 412 [2,6]
+  CRUSH rule 1 x 413 [8,2]
+  CRUSH rule 1 x 414 [4,1]
+  CRUSH rule 1 x 415 [2,8]
+  CRUSH rule 1 x 416 [2,1]
+  CRUSH rule 1 x 417 [8,6]
+  CRUSH rule 1 x 418 [8,2]
+  CRUSH rule 1 x 419 [8,4]
+  CRUSH rule 1 x 420 [1,4]
+  CRUSH rule 1 x 421 [8,4]
+  CRUSH rule 1 x 422 [6,8]
+  CRUSH rule 1 x 423 [2,4]
+  CRUSH rule 1 x 424 [8,2]
+  CRUSH rule 1 x 425 [1,2]
+  CRUSH rule 1 x 426 [8]
+  CRUSH rule 1 x 427 [1,8]
+  CRUSH rule 1 x 428 [4,8]
+  CRUSH rule 1 x 429 [4,8]
+  CRUSH rule 1 x 430 [4,8]
+  CRUSH rule 1 x 431 [4,1]
+  CRUSH rule 1 x 432 [8,1]
+  CRUSH rule 1 x 433 [8,2]
+  CRUSH rule 1 x 434 [8,2]
+  CRUSH rule 1 x 435 [2,6]
+  CRUSH rule 1 x 436 [4,1]
+  CRUSH rule 1 x 437 [8,2]
+  CRUSH rule 1 x 438 [2,4]
+  CRUSH rule 1 x 439 [1,6]
+  CRUSH rule 1 x 440 [2,8]
+  CRUSH rule 1 x 441 [4,6]
+  CRUSH rule 1 x 442 [2,1]
+  CRUSH rule 1 x 443 [8,4]
+  CRUSH rule 1 x 444 [8,1]
+  CRUSH rule 1 x 445 [8,1]
+  CRUSH rule 1 x 446 [2,8]
+  CRUSH rule 1 x 447 [2,1]
+  CRUSH rule 1 x 448 [8,2]
+  CRUSH rule 1 x 449 [8,6]
+  CRUSH rule 1 x 450 [1,8]
+  CRUSH rule 1 x 451 [8,4]
+  CRUSH rule 1 x 452 [8]
+  CRUSH rule 1 x 453 [6,8]
+  CRUSH rule 1 x 454 [8,2]
+  CRUSH rule 1 x 455 [2,8]
+  CRUSH rule 1 x 456 [8,2]
+  CRUSH rule 1 x 457 [8,2]
+  CRUSH rule 1 x 458 [2,8]
+  CRUSH rule 1 x 459 [2,1]
+  CRUSH rule 1 x 460 [8,2]
+  CRUSH rule 1 x 461 [8,2]
+  CRUSH rule 1 x 462 [8,1]
+  CRUSH rule 1 x 463 [8,2]
+  CRUSH rule 1 x 464 [8,4]
+  CRUSH rule 1 x 465 [6,8]
+  CRUSH rule 1 x 466 [8,2]
+  CRUSH rule 1 x 467 [8,2]
+  CRUSH rule 1 x 468 [8,4]
+  CRUSH rule 1 x 469 [8,1]
+  CRUSH rule 1 x 470 [4,2]
+  CRUSH rule 1 x 471 [1,2]
+  CRUSH rule 1 x 472 [1,8]
+  CRUSH rule 1 x 473 [1,2]
+  CRUSH rule 1 x 474 [8,1]
+  CRUSH rule 1 x 475 [8,4]
+  CRUSH rule 1 x 476 [4,1]
+  CRUSH rule 1 x 477 [4,8]
+  CRUSH rule 1 x 478 [8,1]
+  CRUSH rule 1 x 479 [2,1]
+  CRUSH rule 1 x 480 [1,8]
+  CRUSH rule 1 x 481 [2,4]
+  CRUSH rule 1 x 482 [2,8]
+  CRUSH rule 1 x 483 [2,1]
+  CRUSH rule 1 x 484 [1,2]
+  CRUSH rule 1 x 485 [1,8]
+  CRUSH rule 1 x 486 [4,1]
+  CRUSH rule 1 x 487 [8,1]
+  CRUSH rule 1 x 488 [2,8]
+  CRUSH rule 1 x 489 [2,8]
+  CRUSH rule 1 x 490 [6,2]
+  CRUSH rule 1 x 491 [1,2]
+  CRUSH rule 1 x 492 [8,2]
+  CRUSH rule 1 x 493 [2,1]
+  CRUSH rule 1 x 494 [1,2]
+  CRUSH rule 1 x 495 [4,6]
+  CRUSH rule 1 x 496 [8,4]
+  CRUSH rule 1 x 497 [4,8]
+  CRUSH rule 1 x 498 [2,4]
+  CRUSH rule 1 x 499 [8,4]
+  CRUSH rule 1 x 500 [4,8]
+  CRUSH rule 1 x 501 [2,8]
+  CRUSH rule 1 x 502 [6,1]
+  CRUSH rule 1 x 503 [2,1]
+  CRUSH rule 1 x 504 [8,2]
+  CRUSH rule 1 x 505 [1,8]
+  CRUSH rule 1 x 506 [4,1]
+  CRUSH rule 1 x 507 [8,1]
+  CRUSH rule 1 x 508 [1,2]
+  CRUSH rule 1 x 509 [8]
+  CRUSH rule 1 x 510 [8,2]
+  CRUSH rule 1 x 511 [4,8]
+  CRUSH rule 1 x 512 [8,2]
+  CRUSH rule 1 x 513 [8,2]
+  CRUSH rule 1 x 514 [1,8]
+  CRUSH rule 1 x 515 [8,4]
+  CRUSH rule 1 x 516 [4,1]
+  CRUSH rule 1 x 517 [8,2]
+  CRUSH rule 1 x 518 [4,8]
+  CRUSH rule 1 x 519 [8,4]
+  CRUSH rule 1 x 520 [2,8]
+  CRUSH rule 1 x 521 [8,2]
+  CRUSH rule 1 x 522 [8,4]
+  CRUSH rule 1 x 523 [4,2]
+  CRUSH rule 1 x 524 [2,1]
+  CRUSH rule 1 x 525 [2,8]
+  CRUSH rule 1 x 526 [1,2]
+  CRUSH rule 1 x 527 [1,2]
+  CRUSH rule 1 x 528 [8,2]
+  CRUSH rule 1 x 529 [4,8]
+  CRUSH rule 1 x 530 [8]
+  CRUSH rule 1 x 531 [8,1]
+  CRUSH rule 1 x 532 [6,4]
+  CRUSH rule 1 x 533 [4,8]
+  CRUSH rule 1 x 534 [8,2]
+  CRUSH rule 1 x 535 [8,6]
+  CRUSH rule 1 x 536 [8,2]
+  CRUSH rule 1 x 537 [4,8]
+  CRUSH rule 1 x 538 [8,4]
+  CRUSH rule 1 x 539 [8,2]
+  CRUSH rule 1 x 540 [1,8]
+  CRUSH rule 1 x 541 [2,4]
+  CRUSH rule 1 x 542 [2,1]
+  CRUSH rule 1 x 543 [8,2]
+  CRUSH rule 1 x 544 [4,8]
+  CRUSH rule 1 x 545 [1,8]
+  CRUSH rule 1 x 546 [8,1]
+  CRUSH rule 1 x 547 [8,2]
+  CRUSH rule 1 x 548 [4,2]
+  CRUSH rule 1 x 549 [1,8]
+  CRUSH rule 1 x 550 [2,4]
+  CRUSH rule 1 x 551 [8,2]
+  CRUSH rule 1 x 552 [4,1]
+  CRUSH rule 1 x 553 [8,2]
+  CRUSH rule 1 x 554 [1,8]
+  CRUSH rule 1 x 555 [4,1]
+  CRUSH rule 1 x 556 [2,8]
+  CRUSH rule 1 x 557 [8,1]
+  CRUSH rule 1 x 558 [4,1]
+  CRUSH rule 1 x 559 [1,2]
+  CRUSH rule 1 x 560 [8,1]
+  CRUSH rule 1 x 561 [8,4]
+  CRUSH rule 1 x 562 [4,8]
+  CRUSH rule 1 x 563 [2,8]
+  CRUSH rule 1 x 564 [8,1]
+  CRUSH rule 1 x 565 [4,8]
+  CRUSH rule 1 x 566 [4,8]
+  CRUSH rule 1 x 567 [4,8]
+  CRUSH rule 1 x 568 [8,2]
+  CRUSH rule 1 x 569 [4,2]
+  CRUSH rule 1 x 570 [1,8]
+  CRUSH rule 1 x 571 [6,8]
+  CRUSH rule 1 x 572 [4]
+  CRUSH rule 1 x 573 [1,2]
+  CRUSH rule 1 x 574 [2,1]
+  CRUSH rule 1 x 575 [8,2]
+  CRUSH rule 1 x 576 [4,8]
+  CRUSH rule 1 x 577 [8,2]
+  CRUSH rule 1 x 578 [8,1]
+  CRUSH rule 1 x 579 [4,1]
+  CRUSH rule 1 x 580 [8,2]
+  CRUSH rule 1 x 581 [8,2]
+  CRUSH rule 1 x 582 [2,8]
+  CRUSH rule 1 x 583 [8,1]
+  CRUSH rule 1 x 584 [8,1]
+  CRUSH rule 1 x 585 [8,1]
+  CRUSH rule 1 x 586 [1,2]
+  CRUSH rule 1 x 587 [2,4]
+  CRUSH rule 1 x 588 [4,1]
+  CRUSH rule 1 x 589 [8,1]
+  CRUSH rule 1 x 590 [8,2]
+  CRUSH rule 1 x 591 [4,2]
+  CRUSH rule 1 x 592 [2,1]
+  CRUSH rule 1 x 593 [1,8]
+  CRUSH rule 1 x 594 [2,8]
+  CRUSH rule 1 x 595 [8,1]
+  CRUSH rule 1 x 596 [2,8]
+  CRUSH rule 1 x 597 [1,2]
+  CRUSH rule 1 x 598 [8,2]
+  CRUSH rule 1 x 599 [4,2]
+  CRUSH rule 1 x 600 [8,1]
+  CRUSH rule 1 x 601 [1,8]
+  CRUSH rule 1 x 602 [2,8]
+  CRUSH rule 1 x 603 [1,2]
+  CRUSH rule 1 x 604 [8,2]
+  CRUSH rule 1 x 605 [8,2]
+  CRUSH rule 1 x 606 [2,1]
+  CRUSH rule 1 x 607 [2,4]
+  CRUSH rule 1 x 608 [4,1]
+  CRUSH rule 1 x 609 [4,2]
+  CRUSH rule 1 x 610 [1,8]
+  CRUSH rule 1 x 611 [1,2]
+  CRUSH rule 1 x 612 [2,1]
+  CRUSH rule 1 x 613 [8,2]
+  CRUSH rule 1 x 614 [8,4]
+  CRUSH rule 1 x 615 [8,1]
+  CRUSH rule 1 x 616 [1,8]
+  CRUSH rule 1 x 617 [8,1]
+  CRUSH rule 1 x 618 [8,1]
+  CRUSH rule 1 x 619 [4,1]
+  CRUSH rule 1 x 620 [8,1]
+  CRUSH rule 1 x 621 [2,8]
+  CRUSH rule 1 x 622 [2,4]
+  CRUSH rule 1 x 623 [2,8]
+  CRUSH rule 1 x 624 [4,8]
+  CRUSH rule 1 x 625 [2,1]
+  CRUSH rule 1 x 626 [8,1]
+  CRUSH rule 1 x 627 [2,8]
+  CRUSH rule 1 x 628 [8,2]
+  CRUSH rule 1 x 629 [2,8]
+  CRUSH rule 1 x 630 [2,8]
+  CRUSH rule 1 x 631 [1,8]
+  CRUSH rule 1 x 632 [8,2]
+  CRUSH rule 1 x 633 [8,2]
+  CRUSH rule 1 x 634 [1,8]
+  CRUSH rule 1 x 635 [4,8]
+  CRUSH rule 1 x 636 [1,4]
+  CRUSH rule 1 x 637 [1,2]
+  CRUSH rule 1 x 638 [8,1]
+  CRUSH rule 1 x 639 [2,1]
+  CRUSH rule 1 x 640 [1,2]
+  CRUSH rule 1 x 641 [8,2]
+  CRUSH rule 1 x 642 [2,1]
+  CRUSH rule 1 x 643 [8,2]
+  CRUSH rule 1 x 644 [8,1]
+  CRUSH rule 1 x 645 [2,1]
+  CRUSH rule 1 x 646 [8,1]
+  CRUSH rule 1 x 647 [8,1]
+  CRUSH rule 1 x 648 [1,8]
+  CRUSH rule 1 x 649 [4,8]
+  CRUSH rule 1 x 650 [8,4]
+  CRUSH rule 1 x 651 [4,6]
+  CRUSH rule 1 x 652 [4,8]
+  CRUSH rule 1 x 653 [8,1]
+  CRUSH rule 1 x 654 [6,1]
+  CRUSH rule 1 x 655 [1,4]
+  CRUSH rule 1 x 656 [8,2]
+  CRUSH rule 1 x 657 [6,1]
+  CRUSH rule 1 x 658 [8,1]
+  CRUSH rule 1 x 659 [4,8]
+  CRUSH rule 1 x 660 [8,1]
+  CRUSH rule 1 x 661 [1,8]
+  CRUSH rule 1 x 662 [8,2]
+  CRUSH rule 1 x 663 [1,4]
+  CRUSH rule 1 x 664 [1,4]
+  CRUSH rule 1 x 665 [4,6]
+  CRUSH rule 1 x 666 [2,8]
+  CRUSH rule 1 x 667 [1,4]
+  CRUSH rule 1 x 668 [4,8]
+  CRUSH rule 1 x 669 [6,4]
+  CRUSH rule 1 x 670 [4,2]
+  CRUSH rule 1 x 671 [2,1]
+  CRUSH rule 1 x 672 [4,8]
+  CRUSH rule 1 x 673 [4,2]
+  CRUSH rule 1 x 674 [8,1]
+  CRUSH rule 1 x 675 [1,8]
+  CRUSH rule 1 x 676 [2,1]
+  CRUSH rule 1 x 677 [4,1]
+  CRUSH rule 1 x 678 [2,4]
+  CRUSH rule 1 x 679 [8,2]
+  CRUSH rule 1 x 680 [2]
+  CRUSH rule 1 x 681 [2,8]
+  CRUSH rule 1 x 682 [1,4]
+  CRUSH rule 1 x 683 [1,2]
+  CRUSH rule 1 x 684 [8,1]
+  CRUSH rule 1 x 685 [8,1]
+  CRUSH rule 1 x 686 [1,4]
+  CRUSH rule 1 x 687 [1,6]
+  CRUSH rule 1 x 688 [4,8]
+  CRUSH rule 1 x 689 [8,4]
+  CRUSH rule 1 x 690 [8,1]
+  CRUSH rule 1 x 691 [8,1]
+  CRUSH rule 1 x 692 [8,2]
+  CRUSH rule 1 x 693 [8,4]
+  CRUSH rule 1 x 694 [8,4]
+  CRUSH rule 1 x 695 [2,8]
+  CRUSH rule 1 x 696 [1,2]
+  CRUSH rule 1 x 697 [8,1]
+  CRUSH rule 1 x 698 [8,2]
+  CRUSH rule 1 x 699 [1,8]
+  CRUSH rule 1 x 700 [1,2]
+  CRUSH rule 1 x 701 [2,1]
+  CRUSH rule 1 x 702 [8]
+  CRUSH rule 1 x 703 [8,1]
+  CRUSH rule 1 x 704 [1,4]
+  CRUSH rule 1 x 705 [8,4]
+  CRUSH rule 1 x 706 [1,2]
+  CRUSH rule 1 x 707 [8,2]
+  CRUSH rule 1 x 708 [4,8]
+  CRUSH rule 1 x 709 [8,2]
+  CRUSH rule 1 x 710 [8,1]
+  CRUSH rule 1 x 711 [2,4]
+  CRUSH rule 1 x 712 [2,8]
+  CRUSH rule 1 x 713 [8,2]
+  CRUSH rule 1 x 714 [1,2]
+  CRUSH rule 1 x 715 [1,2]
+  CRUSH rule 1 x 716 [4,8]
+  CRUSH rule 1 x 717 [8,4]
+  CRUSH rule 1 x 718 [2,8]
+  CRUSH rule 1 x 719 [2,6]
+  CRUSH rule 1 x 720 [8,1]
+  CRUSH rule 1 x 721 [4,6]
+  CRUSH rule 1 x 722 [8,1]
+  CRUSH rule 1 x 723 [4,1]
+  CRUSH rule 1 x 724 [2,6]
+  CRUSH rule 1 x 725 [1,2]
+  CRUSH rule 1 x 726 [4,8]
+  CRUSH rule 1 x 727 [4,8]
+  CRUSH rule 1 x 728 [2,1]
+  CRUSH rule 1 x 729 [2,8]
+  CRUSH rule 1 x 730 [4,8]
+  CRUSH rule 1 x 731 [4,1]
+  CRUSH rule 1 x 732 [1,2]
+  CRUSH rule 1 x 733 [4,1]
+  CRUSH rule 1 x 734 [8,4]
+  CRUSH rule 1 x 735 [4,8]
+  CRUSH rule 1 x 736 [4,8]
+  CRUSH rule 1 x 737 [1,2]
+  CRUSH rule 1 x 738 [4,2]
+  CRUSH rule 1 x 739 [2,1]
+  CRUSH rule 1 x 740 [1,2]
+  CRUSH rule 1 x 741 [8,2]
+  CRUSH rule 1 x 742 [8,2]
+  CRUSH rule 1 x 743 [8,1]
+  CRUSH rule 1 x 744 [4,8]
+  CRUSH rule 1 x 745 [8,2]
+  CRUSH rule 1 x 746 [8,1]
+  CRUSH rule 1 x 747 [8,1]
+  CRUSH rule 1 x 748 [2,8]
+  CRUSH rule 1 x 749 [4,8]
+  CRUSH rule 1 x 750 [1,8]
+  CRUSH rule 1 x 751 [2,1]
+  CRUSH rule 1 x 752 [8,1]
+  CRUSH rule 1 x 753 [8,1]
+  CRUSH rule 1 x 754 [8,4]
+  CRUSH rule 1 x 755 [1,2]
+  CRUSH rule 1 x 756 [8,2]
+  CRUSH rule 1 x 757 [8,4]
+  CRUSH rule 1 x 758 [8,2]
+  CRUSH rule 1 x 759 [8,4]
+  CRUSH rule 1 x 760 [1,4]
+  CRUSH rule 1 x 761 [1,2]
+  CRUSH rule 1 x 762 [2,8]
+  CRUSH rule 1 x 763 [8,2]
+  CRUSH rule 1 x 764 [1,8]
+  CRUSH rule 1 x 765 [8,2]
+  CRUSH rule 1 x 766 [8]
+  CRUSH rule 1 x 767 [1,2]
+  CRUSH rule 1 x 768 [8,4]
+  CRUSH rule 1 x 769 [8,2]
+  CRUSH rule 1 x 770 [8,2]
+  CRUSH rule 1 x 771 [8,1]
+  CRUSH rule 1 x 772 [8,4]
+  CRUSH rule 1 x 773 [4,1]
+  CRUSH rule 1 x 774 [8,1]
+  CRUSH rule 1 x 775 [8,2]
+  CRUSH rule 1 x 776 [6,2]
+  CRUSH rule 1 x 777 [4,1]
+  CRUSH rule 1 x 778 [1,8]
+  CRUSH rule 1 x 779 [2,8]
+  CRUSH rule 1 x 780 [2,1]
+  CRUSH rule 1 x 781 [8,2]
+  CRUSH rule 1 x 782 [4,1]
+  CRUSH rule 1 x 783 [8,1]
+  CRUSH rule 1 x 784 [1,2]
+  CRUSH rule 1 x 785 [8,1]
+  CRUSH rule 1 x 786 [8,2]
+  CRUSH rule 1 x 787 [1,2]
+  CRUSH rule 1 x 788 [8,2]
+  CRUSH rule 1 x 789 [1,6]
+  CRUSH rule 1 x 790 [8,2]
+  CRUSH rule 1 x 791 [4,8]
+  CRUSH rule 1 x 792 [4,8]
+  CRUSH rule 1 x 793 [8,1]
+  CRUSH rule 1 x 794 [2,8]
+  CRUSH rule 1 x 795 [1,8]
+  CRUSH rule 1 x 796 [2,8]
+  CRUSH rule 1 x 797 [2,4]
+  CRUSH rule 1 x 798 [6,8]
+  CRUSH rule 1 x 799 [4,1]
+  CRUSH rule 1 x 800 [8,2]
+  CRUSH rule 1 x 801 [4,8]
+  CRUSH rule 1 x 802 [1,8]
+  CRUSH rule 1 x 803 [2,8]
+  CRUSH rule 1 x 804 [8,2]
+  CRUSH rule 1 x 805 [2,8]
+  CRUSH rule 1 x 806 [1,4]
+  CRUSH rule 1 x 807 [4,8]
+  CRUSH rule 1 x 808 [2,8]
+  CRUSH rule 1 x 809 [1,8]
+  CRUSH rule 1 x 810 [2,8]
+  CRUSH rule 1 x 811 [8,2]
+  CRUSH rule 1 x 812 [8,4]
+  CRUSH rule 1 x 813 [8,4]
+  CRUSH rule 1 x 814 [8,1]
+  CRUSH rule 1 x 815 [4,1]
+  CRUSH rule 1 x 816 [2,1]
+  CRUSH rule 1 x 817 [8,1]
+  CRUSH rule 1 x 818 [8,2]
+  CRUSH rule 1 x 819 [8,1]
+  CRUSH rule 1 x 820 [4,1]
+  CRUSH rule 1 x 821 [4,1]
+  CRUSH rule 1 x 822 [2,1]
+  CRUSH rule 1 x 823 [4,8]
+  CRUSH rule 1 x 824 [8,2]
+  CRUSH rule 1 x 825 [2,8]
+  CRUSH rule 1 x 826 [8,2]
+  CRUSH rule 1 x 827 [2,8]
+  CRUSH rule 1 x 828 [2,1]
+  CRUSH rule 1 x 829 [8,2]
+  CRUSH rule 1 x 830 [2,4]
+  CRUSH rule 1 x 831 [1,8]
+  CRUSH rule 1 x 832 [4,8]
+  CRUSH rule 1 x 833 [2,1]
+  CRUSH rule 1 x 834 [2]
+  CRUSH rule 1 x 835 [8,4]
+  CRUSH rule 1 x 836 [4,1]
+  CRUSH rule 1 x 837 [8,4]
+  CRUSH rule 1 x 838 [6,8]
+  CRUSH rule 1 x 839 [8,2]
+  CRUSH rule 1 x 840 [8,2]
+  CRUSH rule 1 x 841 [4,8]
+  CRUSH rule 1 x 842 [2]
+  CRUSH rule 1 x 843 [8,4]
+  CRUSH rule 1 x 844 [1,8]
+  CRUSH rule 1 x 845 [4,8]
+  CRUSH rule 1 x 846 [4,2]
+  CRUSH rule 1 x 847 [2,1]
+  CRUSH rule 1 x 848 [2,8]
+  CRUSH rule 1 x 849 [4,8]
+  CRUSH rule 1 x 850 [1,2]
+  CRUSH rule 1 x 851 [6,8]
+  CRUSH rule 1 x 852 [8,4]
+  CRUSH rule 1 x 853 [6,8]
+  CRUSH rule 1 x 854 [8,1]
+  CRUSH rule 1 x 855 [8,1]
+  CRUSH rule 1 x 856 [8,4]
+  CRUSH rule 1 x 857 [8,1]
+  CRUSH rule 1 x 858 [6,2]
+  CRUSH rule 1 x 859 [8,2]
+  CRUSH rule 1 x 860 [8,1]
+  CRUSH rule 1 x 861 [8,1]
+  CRUSH rule 1 x 862 [8,1]
+  CRUSH rule 1 x 863 [8,1]
+  CRUSH rule 1 x 864 [8,2]
+  CRUSH rule 1 x 865 [8,1]
+  CRUSH rule 1 x 866 [2,8]
+  CRUSH rule 1 x 867 [8]
+  CRUSH rule 1 x 868 [8,2]
+  CRUSH rule 1 x 869 [8,6]
+  CRUSH rule 1 x 870 [2,1]
+  CRUSH rule 1 x 871 [2,8]
+  CRUSH rule 1 x 872 [8,1]
+  CRUSH rule 1 x 873 [4,8]
+  CRUSH rule 1 x 874 [2,6]
+  CRUSH rule 1 x 875 [2,8]
+  CRUSH rule 1 x 876 [4,8]
+  CRUSH rule 1 x 877 [8,4]
+  CRUSH rule 1 x 878 [8,1]
+  CRUSH rule 1 x 879 [8,2]
+  CRUSH rule 1 x 880 [6,1]
+  CRUSH rule 1 x 881 [4,8]
+  CRUSH rule 1 x 882 [8,2]
+  CRUSH rule 1 x 883 [2,1]
+  CRUSH rule 1 x 884 [8,2]
+  CRUSH rule 1 x 885 [4,1]
+  CRUSH rule 1 x 886 [2,8]
+  CRUSH rule 1 x 887 [8,4]
+  CRUSH rule 1 x 888 [8,2]
+  CRUSH rule 1 x 889 [2,1]
+  CRUSH rule 1 x 890 [8,2]
+  CRUSH rule 1 x 891 [1,8]
+  CRUSH rule 1 x 892 [8,2]
+  CRUSH rule 1 x 893 [2,8]
+  CRUSH rule 1 x 894 [8,4]
+  CRUSH rule 1 x 895 [4,8]
+  CRUSH rule 1 x 896 [1,8]
+  CRUSH rule 1 x 897 [8,2]
+  CRUSH rule 1 x 898 [1,4]
+  CRUSH rule 1 x 899 [1,8]
+  CRUSH rule 1 x 900 [4,1]
+  CRUSH rule 1 x 901 [8,1]
+  CRUSH rule 1 x 902 [8,4]
+  CRUSH rule 1 x 903 [1,8]
+  CRUSH rule 1 x 904 [1,8]
+  CRUSH rule 1 x 905 [8,2]
+  CRUSH rule 1 x 906 [1,2]
+  CRUSH rule 1 x 907 [8,1]
+  CRUSH rule 1 x 908 [8,1]
+  CRUSH rule 1 x 909 [2,1]
+  CRUSH rule 1 x 910 [8,2]
+  CRUSH rule 1 x 911 [8,1]
+  CRUSH rule 1 x 912 [1,2]
+  CRUSH rule 1 x 913 [8,4]
+  CRUSH rule 1 x 914 [6,4]
+  CRUSH rule 1 x 915 [8,2]
+  CRUSH rule 1 x 916 [4,1]
+  CRUSH rule 1 x 917 [1,4]
+  CRUSH rule 1 x 918 [8,2]
+  CRUSH rule 1 x 919 [8,2]
+  CRUSH rule 1 x 920 [8,1]
+  CRUSH rule 1 x 921 [1,2]
+  CRUSH rule 1 x 922 [8,4]
+  CRUSH rule 1 x 923 [4,8]
+  CRUSH rule 1 x 924 [1,8]
+  CRUSH rule 1 x 925 [4,8]
+  CRUSH rule 1 x 926 [1,8]
+  CRUSH rule 1 x 927 [1,8]
+  CRUSH rule 1 x 928 [8,1]
+  CRUSH rule 1 x 929 [4,1]
+  CRUSH rule 1 x 930 [2,1]
+  CRUSH rule 1 x 931 [8,1]
+  CRUSH rule 1 x 932 [4,8]
+  CRUSH rule 1 x 933 [8,4]
+  CRUSH rule 1 x 934 [8,1]
+  CRUSH rule 1 x 935 [8,1]
+  CRUSH rule 1 x 936 [1,8]
+  CRUSH rule 1 x 937 [4,8]
+  CRUSH rule 1 x 938 [8,4]
+  CRUSH rule 1 x 939 [2,8]
+  CRUSH rule 1 x 940 [8,2]
+  CRUSH rule 1 x 941 [8,2]
+  CRUSH rule 1 x 942 [1,2]
+  CRUSH rule 1 x 943 [8,2]
+  CRUSH rule 1 x 944 [2,1]
+  CRUSH rule 1 x 945 [8,2]
+  CRUSH rule 1 x 946 [2,1]
+  CRUSH rule 1 x 947 [8,1]
+  CRUSH rule 1 x 948 [8,1]
+  CRUSH rule 1 x 949 [6,1]
+  CRUSH rule 1 x 950 [8,1]
+  CRUSH rule 1 x 951 [2,8]
+  CRUSH rule 1 x 952 [2,1]
+  CRUSH rule 1 x 953 [1,4]
+  CRUSH rule 1 x 954 [8,2]
+  CRUSH rule 1 x 955 [8,1]
+  CRUSH rule 1 x 956 [1,2]
+  CRUSH rule 1 x 957 [8,1]
+  CRUSH rule 1 x 958 [8,2]
+  CRUSH rule 1 x 959 [4,2]
+  CRUSH rule 1 x 960 [2,6]
+  CRUSH rule 1 x 961 [1,2]
+  CRUSH rule 1 x 962 [8,4]
+  CRUSH rule 1 x 963 [2,4]
+  CRUSH rule 1 x 964 [8,1]
+  CRUSH rule 1 x 965 [8,2]
+  CRUSH rule 1 x 966 [4,8]
+  CRUSH rule 1 x 967 [8,1]
+  CRUSH rule 1 x 968 [8,2]
+  CRUSH rule 1 x 969 [8,2]
+  CRUSH rule 1 x 970 [2,8]
+  CRUSH rule 1 x 971 [1,8]
+  CRUSH rule 1 x 972 [1,8]
+  CRUSH rule 1 x 973 [1,2]
+  CRUSH rule 1 x 974 [4,1]
+  CRUSH rule 1 x 975 [4,8]
+  CRUSH rule 1 x 976 [4,8]
+  CRUSH rule 1 x 977 [8,4]
+  CRUSH rule 1 x 978 [8,2]
+  CRUSH rule 1 x 979 [8,1]
+  CRUSH rule 1 x 980 [8,2]
+  CRUSH rule 1 x 981 [8]
+  CRUSH rule 1 x 982 [1,2]
+  CRUSH rule 1 x 983 [4,8]
+  CRUSH rule 1 x 984 [2,1]
+  CRUSH rule 1 x 985 [2,4]
+  CRUSH rule 1 x 986 [8,4]
+  CRUSH rule 1 x 987 [2,1]
+  CRUSH rule 1 x 988 [1,4]
+  CRUSH rule 1 x 989 [1,8]
+  CRUSH rule 1 x 990 [1,2]
+  CRUSH rule 1 x 991 [1,4]
+  CRUSH rule 1 x 992 [8,1]
+  CRUSH rule 1 x 993 [2,8]
+  CRUSH rule 1 x 994 [4,2]
+  CRUSH rule 1 x 995 [8,1]
+  CRUSH rule 1 x 996 [8,2]
+  CRUSH rule 1 x 997 [8,4]
+  CRUSH rule 1 x 998 [8,1]
+  CRUSH rule 1 x 999 [1,8]
+  CRUSH rule 1 x 1000 [8,4]
+  CRUSH rule 1 x 1001 [2,1]
+  CRUSH rule 1 x 1002 [1,2]
+  CRUSH rule 1 x 1003 [2,8]
+  CRUSH rule 1 x 1004 [8,1]
+  CRUSH rule 1 x 1005 [8,1]
+  CRUSH rule 1 x 1006 [1,2]
+  CRUSH rule 1 x 1007 [1,2]
+  CRUSH rule 1 x 1008 [1,8]
+  CRUSH rule 1 x 1009 [6,8]
+  CRUSH rule 1 x 1010 [2,8]
+  CRUSH rule 1 x 1011 [4,2]
+  CRUSH rule 1 x 1012 [1,2]
+  CRUSH rule 1 x 1013 [1,2]
+  CRUSH rule 1 x 1014 [2,8]
+  CRUSH rule 1 x 1015 [8,1]
+  CRUSH rule 1 x 1016 [2,1]
+  CRUSH rule 1 x 1017 [6,2]
+  CRUSH rule 1 x 1018 [4,2]
+  CRUSH rule 1 x 1019 [4,8]
+  CRUSH rule 1 x 1020 [1,2]
+  CRUSH rule 1 x 1021 [8,2]
+  CRUSH rule 1 x 1022 [1,8]
+  CRUSH rule 1 x 1023 [4,2]
+  rule 1 (choose-two) num_rep 2 result size == 1:\t23/1024 (esc)
+  rule 1 (choose-two) num_rep 2 result size == 2:\t1001/1024 (esc)
+  CRUSH rule 1 x 0 [2,1,4]
+  CRUSH rule 1 x 1 [2,8,1]
+  CRUSH rule 1 x 2 [1,8,2]
+  CRUSH rule 1 x 3 [8,1]
+  CRUSH rule 1 x 4 [4,8,2]
+  CRUSH rule 1 x 5 [8,2,1]
+  CRUSH rule 1 x 6 [2,8,1]
+  CRUSH rule 1 x 7 [4,8,2]
+  CRUSH rule 1 x 8 [4,2,8]
+  CRUSH rule 1 x 9 [2,4,1]
+  CRUSH rule 1 x 10 [2,1,8]
+  CRUSH rule 1 x 11 [2,8,1]
+  CRUSH rule 1 x 12 [2,1,8]
+  CRUSH rule 1 x 13 [4,8,2]
+  CRUSH rule 1 x 14 [8,1,2]
+  CRUSH rule 1 x 15 [8,2,1]
+  CRUSH rule 1 x 16 [8,1,2]
+  CRUSH rule 1 x 17 [4,8,2]
+  CRUSH rule 1 x 18 [1,6,8]
+  CRUSH rule 1 x 19 [8,4]
+  CRUSH rule 1 x 20 [2,1,8]
+  CRUSH rule 1 x 21 [8,1,2]
+  CRUSH rule 1 x 22 [8,2]
+  CRUSH rule 1 x 23 [4,8,1]
+  CRUSH rule 1 x 24 [1,2,8]
+  CRUSH rule 1 x 25 [4,8,1]
+  CRUSH rule 1 x 26 [2,8,1]
+  CRUSH rule 1 x 27 [4,1]
+  CRUSH rule 1 x 28 [8,2,1]
+  CRUSH rule 1 x 29 [8,4,1]
+  CRUSH rule 1 x 30 [4,8,1]
+  CRUSH rule 1 x 31 [8,2,1]
+  CRUSH rule 1 x 32 [2,6,8]
+  CRUSH rule 1 x 33 [2,8,1]
+  CRUSH rule 1 x 34 [2,1,8]
+  CRUSH rule 1 x 35 [1,8,2]
+  CRUSH rule 1 x 36 [8,1,2]
+  CRUSH rule 1 x 37 [1,8,2]
+  CRUSH rule 1 x 38 [4,8,1]
+  CRUSH rule 1 x 39 [8,1,2]
+  CRUSH rule 1 x 40 [8,2,1]
+  CRUSH rule 1 x 41 [2,1,8]
+  CRUSH rule 1 x 42 [1,2,8]
+  CRUSH rule 1 x 43 [1,2,8]
+  CRUSH rule 1 x 44 [1,8,2]
+  CRUSH rule 1 x 45 [8,2,4]
+  CRUSH rule 1 x 46 [2,1,8]
+  CRUSH rule 1 x 47 [4,2,8]
+  CRUSH rule 1 x 48 [8,1]
+  CRUSH rule 1 x 49 [8,2,1]
+  CRUSH rule 1 x 50 [4,8,1]
+  CRUSH rule 1 x 51 [2,8,1]
+  CRUSH rule 1 x 52 [8,2,1]
+  CRUSH rule 1 x 53 [4,2,8]
+  CRUSH rule 1 x 54 [8,4]
+  CRUSH rule 1 x 55 [8,1,2]
+  CRUSH rule 1 x 56 [8,4,2]
+  CRUSH rule 1 x 57 [2,1,8]
+  CRUSH rule 1 x 58 [1,2,8]
+  CRUSH rule 1 x 59 [8,2,1]
+  CRUSH rule 1 x 60 [4,8,1]
+  CRUSH rule 1 x 61 [4,8,2]
+  CRUSH rule 1 x 62 [8,1,2]
+  CRUSH rule 1 x 63 [8,1,2]
+  CRUSH rule 1 x 64 [4,2,1]
+  CRUSH rule 1 x 65 [8,4]
+  CRUSH rule 1 x 66 [4,2,1]
+  CRUSH rule 1 x 67 [4,2,8]
+  CRUSH rule 1 x 68 [1,2,8]
+  CRUSH rule 1 x 69 [1,2,8]
+  CRUSH rule 1 x 70 [8,2,1]
+  CRUSH rule 1 x 71 [2,8,1]
+  CRUSH rule 1 x 72 [8,1,4]
+  CRUSH rule 1 x 73 [2,8,1]
+  CRUSH rule 1 x 74 [1,8,2]
+  CRUSH rule 1 x 75 [4,2,1]
+  CRUSH rule 1 x 76 [4,1,6]
+  CRUSH rule 1 x 77 [8,2,4]
+  CRUSH rule 1 x 78 [1,2]
+  CRUSH rule 1 x 79 [4,1,2]
+  CRUSH rule 1 x 80 [2,4,1]
+  CRUSH rule 1 x 81 [2,1]
+  CRUSH rule 1 x 82 [6,1,8]
+  CRUSH rule 1 x 83 [2,8]
+  CRUSH rule 1 x 84 [8,2,1]
+  CRUSH rule 1 x 85 [4,8,1]
+  CRUSH rule 1 x 86 [2,1,8]
+  CRUSH rule 1 x 87 [2,8,4]
+  CRUSH rule 1 x 88 [1,6,2]
+  CRUSH rule 1 x 89 [2,1,8]
+  CRUSH rule 1 x 90 [8,2,4]
+  CRUSH rule 1 x 91 [4,8,1]
+  CRUSH rule 1 x 92 [1,8,2]
+  CRUSH rule 1 x 93 [8,4,1]
+  CRUSH rule 1 x 94 [1,2,8]
+  CRUSH rule 1 x 95 [8,1,2]
+  CRUSH rule 1 x 96 [1,8,2]
+  CRUSH rule 1 x 97 [8,1,2]
+  CRUSH rule 1 x 98 [2,1,8]
+  CRUSH rule 1 x 99 [2,8,1]
+  CRUSH rule 1 x 100 [1,8,4]
+  CRUSH rule 1 x 101 [8,2,1]
+  CRUSH rule 1 x 102 [2,1,8]
+  CRUSH rule 1 x 103 [2,8,1]
+  CRUSH rule 1 x 104 [8,4,1]
+  CRUSH rule 1 x 105 [2,4,1]
+  CRUSH rule 1 x 106 [1,8,2]
+  CRUSH rule 1 x 107 [2,1,8]
+  CRUSH rule 1 x 108 [8,2]
+  CRUSH rule 1 x 109 [1,2,4]
+  CRUSH rule 1 x 110 [4,2,8]
+  CRUSH rule 1 x 111 [2,1,4]
+  CRUSH rule 1 x 112 [2,1,8]
+  CRUSH rule 1 x 113 [8,2,1]
+  CRUSH rule 1 x 114 [8,4,1]
+  CRUSH rule 1 x 115 [8,2,4]
+  CRUSH rule 1 x 116 [1,2,8]
+  CRUSH rule 1 x 117 [6,8,1]
+  CRUSH rule 1 x 118 [2,8,1]
+  CRUSH rule 1 x 119 [2,8,1]
+  CRUSH rule 1 x 120 [2,1,4]
+  CRUSH rule 1 x 121 [2,1,8]
+  CRUSH rule 1 x 122 [8,1,2]
+  CRUSH rule 1 x 123 [2,8,1]
+  CRUSH rule 1 x 124 [8,1]
+  CRUSH rule 1 x 125 [1,8,4]
+  CRUSH rule 1 x 126 [8,2,1]
+  CRUSH rule 1 x 127 [4,8,2]
+  CRUSH rule 1 x 128 [2,8,6]
+  CRUSH rule 1 x 129 [2,1,8]
+  CRUSH rule 1 x 130 [4,8,1]
+  CRUSH rule 1 x 131 [1,2,8]
+  CRUSH rule 1 x 132 [1,2,8]
+  CRUSH rule 1 x 133 [8,2,1]
+  CRUSH rule 1 x 134 [1,8,2]
+  CRUSH rule 1 x 135 [4,8,2]
+  CRUSH rule 1 x 136 [2,1,4]
+  CRUSH rule 1 x 137 [8,4]
+  CRUSH rule 1 x 138 [8,4,2]
+  CRUSH rule 1 x 139 [4,2,6]
+  CRUSH rule 1 x 140 [1,8,2]
+  CRUSH rule 1 x 141 [8,1,2]
+  CRUSH rule 1 x 142 [4,6,1]
+  CRUSH rule 1 x 143 [4,8,1]
+  CRUSH rule 1 x 144 [8,1,2]
+  CRUSH rule 1 x 145 [8,2,1]
+  CRUSH rule 1 x 146 [2,8,1]
+  CRUSH rule 1 x 147 [2,8,4]
+  CRUSH rule 1 x 148 [4,1,2]
+  CRUSH rule 1 x 149 [4,8,2]
+  CRUSH rule 1 x 150 [1,8,2]
+  CRUSH rule 1 x 151 [1,2]
+  CRUSH rule 1 x 152 [8,1,2]
+  CRUSH rule 1 x 153 [8,4,1]
+  CRUSH rule 1 x 154 [4,2,1]
+  CRUSH rule 1 x 155 [4,8,2]
+  CRUSH rule 1 x 156 [4,8,2]
+  CRUSH rule 1 x 157 [2,1,8]
+  CRUSH rule 1 x 158 [2,8,1]
+  CRUSH rule 1 x 159 [8,2,4]
+  CRUSH rule 1 x 160 [2,8,1]
+  CRUSH rule 1 x 161 [1,4,2]
+  CRUSH rule 1 x 162 [1,8,2]
+  CRUSH rule 1 x 163 [4,8,1]
+  CRUSH rule 1 x 164 [8,2,1]
+  CRUSH rule 1 x 165 [8,2,4]
+  CRUSH rule 1 x 166 [2,1,8]
+  CRUSH rule 1 x 167 [1,2,8]
+  CRUSH rule 1 x 168 [4,2,8]
+  CRUSH rule 1 x 169 [2,8,1]
+  CRUSH rule 1 x 170 [1,2,8]
+  CRUSH rule 1 x 171 [8,4,1]
+  CRUSH rule 1 x 172 [1,8]
+  CRUSH rule 1 x 173 [8,4]
+  CRUSH rule 1 x 174 [1,2,6]
+  CRUSH rule 1 x 175 [8,1,2]
+  CRUSH rule 1 x 176 [2,1,8]
+  CRUSH rule 1 x 177 [8,2,1]
+  CRUSH rule 1 x 178 [4,2,1]
+  CRUSH rule 1 x 179 [8,1,2]
+  CRUSH rule 1 x 180 [1,8,2]
+  CRUSH rule 1 x 181 [8,2,1]
+  CRUSH rule 1 x 182 [8,1,2]
+  CRUSH rule 1 x 183 [8,1,2]
+  CRUSH rule 1 x 184 [4,8]
+  CRUSH rule 1 x 185 [8,4,1]
+  CRUSH rule 1 x 186 [2,1,4]
+  CRUSH rule 1 x 187 [1,8,2]
+  CRUSH rule 1 x 188 [1,8,2]
+  CRUSH rule 1 x 189 [1,8,2]
+  CRUSH rule 1 x 190 [1,2,8]
+  CRUSH rule 1 x 191 [8,4,1]
+  CRUSH rule 1 x 192 [4,1,2]
+  CRUSH rule 1 x 193 [4,2,8]
+  CRUSH rule 1 x 194 [1,8,2]
+  CRUSH rule 1 x 195 [8,4,1]
+  CRUSH rule 1 x 196 [8,1,2]
+  CRUSH rule 1 x 197 [8,4,2]
+  CRUSH rule 1 x 198 [2,1,8]
+  CRUSH rule 1 x 199 [1,4,8]
+  CRUSH rule 1 x 200 [1,8,2]
+  CRUSH rule 1 x 201 [8,1,2]
+  CRUSH rule 1 x 202 [8,2,1]
+  CRUSH rule 1 x 203 [1,8]
+  CRUSH rule 1 x 204 [2,1,4]
+  CRUSH rule 1 x 205 [1,8,2]
+  CRUSH rule 1 x 206 [1,2,8]
+  CRUSH rule 1 x 207 [2,1,8]
+  CRUSH rule 1 x 208 [8,1,2]
+  CRUSH rule 1 x 209 [1,2,8]
+  CRUSH rule 1 x 210 [1,2,4]
+  CRUSH rule 1 x 211 [4,8,2]
+  CRUSH rule 1 x 212 [8,1,2]
+  CRUSH rule 1 x 213 [8,4,2]
+  CRUSH rule 1 x 214 [8,1]
+  CRUSH rule 1 x 215 [8,1,2]
+  CRUSH rule 1 x 216 [8,1,2]
+  CRUSH rule 1 x 217 [1,2,8]
+  CRUSH rule 1 x 218 [2,8,1]
+  CRUSH rule 1 x 219 [2,8,1]
+  CRUSH rule 1 x 220 [4,8,2]
+  CRUSH rule 1 x 221 [1,8,2]
+  CRUSH rule 1 x 222 [8,1,2]
+  CRUSH rule 1 x 223 [1,2,8]
+  CRUSH rule 1 x 224 [1,4,2]
+  CRUSH rule 1 x 225 [8,2,1]
+  CRUSH rule 1 x 226 [8,2,4]
+  CRUSH rule 1 x 227 [4,8,1]
+  CRUSH rule 1 x 228 [1,8,2]
+  CRUSH rule 1 x 229 [4,1,6]
+  CRUSH rule 1 x 230 [4,8,2]
+  CRUSH rule 1 x 231 [4,8,2]
+  CRUSH rule 1 x 232 [2,8,4]
+  CRUSH rule 1 x 233 [8,2,1]
+  CRUSH rule 1 x 234 [1,2,8]
+  CRUSH rule 1 x 235 [4,8,1]
+  CRUSH rule 1 x 236 [1,2,6]
+  CRUSH rule 1 x 237 [4,8,2]
+  CRUSH rule 1 x 238 [8,1]
+  CRUSH rule 1 x 239 [8,6,1]
+  CRUSH rule 1 x 240 [4,8,1]
+  CRUSH rule 1 x 241 [2,1,8]
+  CRUSH rule 1 x 242 [2,8,1]
+  CRUSH rule 1 x 243 [8,2,1]
+  CRUSH rule 1 x 244 [4,8,2]
+  CRUSH rule 1 x 245 [8,2,1]
+  CRUSH rule 1 x 246 [1,2,8]
+  CRUSH rule 1 x 247 [8,2,1]
+  CRUSH rule 1 x 248 [8,2,4]
+  CRUSH rule 1 x 249 [2,1,8]
+  CRUSH rule 1 x 250 [2,1,4]
+  CRUSH rule 1 x 251 [2,8,1]
+  CRUSH rule 1 x 252 [4,8,2]
+  CRUSH rule 1 x 253 [1,2,8]
+  CRUSH rule 1 x 254 [4,8,1]
+  CRUSH rule 1 x 255 [1,8,2]
+  CRUSH rule 1 x 256 [4,8,1]
+  CRUSH rule 1 x 257 [2,8,4]
+  CRUSH rule 1 x 258 [4,1]
+  CRUSH rule 1 x 259 [8,6,2]
+  CRUSH rule 1 x 260 [8,2,1]
+  CRUSH rule 1 x 261 [8,1,2]
+  CRUSH rule 1 x 262 [8,2,1]
+  CRUSH rule 1 x 263 [8,4,2]
+  CRUSH rule 1 x 264 [1,8,2]
+  CRUSH rule 1 x 265 [8,1]
+  CRUSH rule 1 x 266 [8,2,1]
+  CRUSH rule 1 x 267 [2,8,1]
+  CRUSH rule 1 x 268 [1,8,2]
+  CRUSH rule 1 x 269 [1,8,2]
+  CRUSH rule 1 x 270 [4,1,2]
+  CRUSH rule 1 x 271 [8,4,2]
+  CRUSH rule 1 x 272 [2,8,4]
+  CRUSH rule 1 x 273 [4,1,2]
+  CRUSH rule 1 x 274 [8,2,4]
+  CRUSH rule 1 x 275 [4,8,1]
+  CRUSH rule 1 x 276 [8,1,2]
+  CRUSH rule 1 x 277 [8,1,2]
+  CRUSH rule 1 x 278 [8,2,1]
+  CRUSH rule 1 x 279 [8,4,2]
+  CRUSH rule 1 x 280 [2,8,1]
+  CRUSH rule 1 x 281 [8,2,1]
+  CRUSH rule 1 x 282 [1,2,8]
+  CRUSH rule 1 x 283 [8,2,1]
+  CRUSH rule 1 x 284 [8,2]
+  CRUSH rule 1 x 285 [4,8,1]
+  CRUSH rule 1 x 286 [2,1,8]
+  CRUSH rule 1 x 287 [1,2,8]
+  CRUSH rule 1 x 288 [8,1,4]
+  CRUSH rule 1 x 289 [4,8,2]
+  CRUSH rule 1 x 290 [1,4,8]
+  CRUSH rule 1 x 291 [1,2,4]
+  CRUSH rule 1 x 292 [8,2,1]
+  CRUSH rule 1 x 293 [8,1,2]
+  CRUSH rule 1 x 294 [8,4,2]
+  CRUSH rule 1 x 295 [4,8,1]
+  CRUSH rule 1 x 296 [4,1,8]
+  CRUSH rule 1 x 297 [8,2,1]
+  CRUSH rule 1 x 298 [1,2,8]
+  CRUSH rule 1 x 299 [2,1,8]
+  CRUSH rule 1 x 300 [8,2]
+  CRUSH rule 1 x 301 [1,8,2]
+  CRUSH rule 1 x 302 [8,1,2]
+  CRUSH rule 1 x 303 [8,4,1]
+  CRUSH rule 1 x 304 [2,8,1]
+  CRUSH rule 1 x 305 [2,8,1]
+  CRUSH rule 1 x 306 [1,8,2]
+  CRUSH rule 1 x 307 [2,1,8]
+  CRUSH rule 1 x 308 [2,8,4]
+  CRUSH rule 1 x 309 [8,2]
+  CRUSH rule 1 x 310 [4,6,1]
+  CRUSH rule 1 x 311 [4,2,1]
+  CRUSH rule 1 x 312 [2,1,8]
+  CRUSH rule 1 x 313 [4,8]
+  CRUSH rule 1 x 314 [6,1,2]
+  CRUSH rule 1 x 315 [2,1,8]
+  CRUSH rule 1 x 316 [8,2,1]
+  CRUSH rule 1 x 317 [2,8,1]
+  CRUSH rule 1 x 318 [8,2,1]
+  CRUSH rule 1 x 319 [8,2,1]
+  CRUSH rule 1 x 320 [8,2,1]
+  CRUSH rule 1 x 321 [1,2]
+  CRUSH rule 1 x 322 [2,8,4]
+  CRUSH rule 1 x 323 [4,8,1]
+  CRUSH rule 1 x 324 [8,1,4]
+  CRUSH rule 1 x 325 [4,8,2]
+  CRUSH rule 1 x 326 [8,6,1]
+  CRUSH rule 1 x 327 [1,8,2]
+  CRUSH rule 1 x 328 [8,4,1]
+  CRUSH rule 1 x 329 [4,8,2]
+  CRUSH rule 1 x 330 [4,8,1]
+  CRUSH rule 1 x 331 [2,8,1]
+  CRUSH rule 1 x 332 [2,1,8]
+  CRUSH rule 1 x 333 [8,1,2]
+  CRUSH rule 1 x 334 [8,1]
+  CRUSH rule 1 x 335 [8,1,2]
+  CRUSH rule 1 x 336 [4,8,2]
+  CRUSH rule 1 x 337 [8,2,4]
+  CRUSH rule 1 x 338 [1,8]
+  CRUSH rule 1 x 339 [8,2]
+  CRUSH rule 1 x 340 [2,1,8]
+  CRUSH rule 1 x 341 [4,1,8]
+  CRUSH rule 1 x 342 [2,8,4]
+  CRUSH rule 1 x 343 [8,1]
+  CRUSH rule 1 x 344 [6,2,4]
+  CRUSH rule 1 x 345 [2,1,8]
+  CRUSH rule 1 x 346 [8,2,4]
+  CRUSH rule 1 x 347 [4,1,8]
+  CRUSH rule 1 x 348 [8,2,1]
+  CRUSH rule 1 x 349 [1,8,2]
+  CRUSH rule 1 x 350 [8,1,2]
+  CRUSH rule 1 x 351 [1,8,2]
+  CRUSH rule 1 x 352 [1,2,4]
+  CRUSH rule 1 x 353 [8,1,2]
+  CRUSH rule 1 x 354 [1,8,2]
+  CRUSH rule 1 x 355 [1,2,8]
+  CRUSH rule 1 x 356 [4,8,1]
+  CRUSH rule 1 x 357 [8,1,2]
+  CRUSH rule 1 x 358 [2,1,8]
+  CRUSH rule 1 x 359 [6,8,1]
+  CRUSH rule 1 x 360 [1,2,8]
+  CRUSH rule 1 x 361 [8,4,2]
+  CRUSH rule 1 x 362 [4,6,8]
+  CRUSH rule 1 x 363 [4,1,2]
+  CRUSH rule 1 x 364 [2,8,1]
+  CRUSH rule 1 x 365 [8,1,2]
+  CRUSH rule 1 x 366 [8,2,1]
+  CRUSH rule 1 x 367 [4,8,2]
+  CRUSH rule 1 x 368 [8,4,2]
+  CRUSH rule 1 x 369 [8,1]
+  CRUSH rule 1 x 370 [8,2]
+  CRUSH rule 1 x 371 [1,4,2]
+  CRUSH rule 1 x 372 [8,1,2]
+  CRUSH rule 1 x 373 [1,8]
+  CRUSH rule 1 x 374 [2,8]
+  CRUSH rule 1 x 375 [8,4]
+  CRUSH rule 1 x 376 [8,1,2]
+  CRUSH rule 1 x 377 [1,2,4]
+  CRUSH rule 1 x 378 [1,2,8]
+  CRUSH rule 1 x 379 [8,1,2]
+  CRUSH rule 1 x 380 [2,1]
+  CRUSH rule 1 x 381 [1,4,8]
+  CRUSH rule 1 x 382 [1,4,2]
+  CRUSH rule 1 x 383 [4,1,2]
+  CRUSH rule 1 x 384 [8,2,1]
+  CRUSH rule 1 x 385 [8,1,6]
+  CRUSH rule 1 x 386 [1,8,2]
+  CRUSH rule 1 x 387 [1,4,8]
+  CRUSH rule 1 x 388 [8,1,6]
+  CRUSH rule 1 x 389 [1,4,8]
+  CRUSH rule 1 x 390 [4,8,1]
+  CRUSH rule 1 x 391 [4,8,2]
+  CRUSH rule 1 x 392 [1,8,2]
+  CRUSH rule 1 x 393 [8,2]
+  CRUSH rule 1 x 394 [8,1,2]
+  CRUSH rule 1 x 395 [8,1,2]
+  CRUSH rule 1 x 396 [4,2,8]
+  CRUSH rule 1 x 397 [2,1,4]
+  CRUSH rule 1 x 398 [2,4,1]
+  CRUSH rule 1 x 399 [8,2,4]
+  CRUSH rule 1 x 400 [8,1,4]
+  CRUSH rule 1 x 401 [1,2,4]
+  CRUSH rule 1 x 402 [8,1,2]
+  CRUSH rule 1 x 403 [1,2,4]
+  CRUSH rule 1 x 404 [4,2,1]
+  CRUSH rule 1 x 405 [8,4,2]
+  CRUSH rule 1 x 406 [2,1]
+  CRUSH rule 1 x 407 [2,8,4]
+  CRUSH rule 1 x 408 [4,1,8]
+  CRUSH rule 1 x 409 [8,4,2]
+  CRUSH rule 1 x 410 [8,1,4]
+  CRUSH rule 1 x 411 [2,1,8]
+  CRUSH rule 1 x 412 [2,6,8]
+  CRUSH rule 1 x 413 [8,2]
+  CRUSH rule 1 x 414 [4,1,8]
+  CRUSH rule 1 x 415 [2,8,1]
+  CRUSH rule 1 x 416 [2,1,8]
+  CRUSH rule 1 x 417 [8,6,2]
+  CRUSH rule 1 x 418 [8,2,4]
+  CRUSH rule 1 x 419 [8,4,1]
+  CRUSH rule 1 x 420 [1,4,2]
+  CRUSH rule 1 x 421 [8,4]
+  CRUSH rule 1 x 422 [6,8,2]
+  CRUSH rule 1 x 423 [2,4,8]
+  CRUSH rule 1 x 424 [8,2,1]
+  CRUSH rule 1 x 425 [1,2,8]
+  CRUSH rule 1 x 426 [8,2]
+  CRUSH rule 1 x 427 [1,8,2]
+  CRUSH rule 1 x 428 [4,8]
+  CRUSH rule 1 x 429 [4,8,2]
+  CRUSH rule 1 x 430 [4,8,1]
+  CRUSH rule 1 x 431 [4,1,2]
+  CRUSH rule 1 x 432 [8,1,2]
+  CRUSH rule 1 x 433 [8,2,1]
+  CRUSH rule 1 x 434 [8,2,1]
+  CRUSH rule 1 x 435 [2,6,1]
+  CRUSH rule 1 x 436 [4,1,8]
+  CRUSH rule 1 x 437 [8,2,1]
+  CRUSH rule 1 x 438 [2,4,8]
+  CRUSH rule 1 x 439 [1,6,8]
+  CRUSH rule 1 x 440 [2,8,1]
+  CRUSH rule 1 x 441 [4,6,2]
+  CRUSH rule 1 x 442 [2,1,8]
+  CRUSH rule 1 x 443 [8,4,2]
+  CRUSH rule 1 x 444 [8,1,2]
+  CRUSH rule 1 x 445 [8,1,2]
+  CRUSH rule 1 x 446 [2,8,1]
+  CRUSH rule 1 x 447 [2,1,4]
+  CRUSH rule 1 x 448 [8,2,4]
+  CRUSH rule 1 x 449 [8,6,2]
+  CRUSH rule 1 x 450 [1,8,2]
+  CRUSH rule 1 x 451 [8,4]
+  CRUSH rule 1 x 452 [8]
+  CRUSH rule 1 x 453 [6,8,2]
+  CRUSH rule 1 x 454 [8,2,1]
+  CRUSH rule 1 x 455 [2,8,4]
+  CRUSH rule 1 x 456 [8,2]
+  CRUSH rule 1 x 457 [8,2,1]
+  CRUSH rule 1 x 458 [2,8,1]
+  CRUSH rule 1 x 459 [2,1,8]
+  CRUSH rule 1 x 460 [8,2,1]
+  CRUSH rule 1 x 461 [8,2,1]
+  CRUSH rule 1 x 462 [8,1,2]
+  CRUSH rule 1 x 463 [8,2,1]
+  CRUSH rule 1 x 464 [8,4,2]
+  CRUSH rule 1 x 465 [6,8,1]
+  CRUSH rule 1 x 466 [8,2,1]
+  CRUSH rule 1 x 467 [8,2,1]
+  CRUSH rule 1 x 468 [8,4,1]
+  CRUSH rule 1 x 469 [8,1,2]
+  CRUSH rule 1 x 470 [4,2,6]
+  CRUSH rule 1 x 471 [1,2,8]
+  CRUSH rule 1 x 472 [1,8,2]
+  CRUSH rule 1 x 473 [1,2,4]
+  CRUSH rule 1 x 474 [8,1]
+  CRUSH rule 1 x 475 [8,4,1]
+  CRUSH rule 1 x 476 [4,1,2]
+  CRUSH rule 1 x 477 [4,8,1]
+  CRUSH rule 1 x 478 [8,1,2]
+  CRUSH rule 1 x 479 [2,1,8]
+  CRUSH rule 1 x 480 [1,8,2]
+  CRUSH rule 1 x 481 [2,4,1]
+  CRUSH rule 1 x 482 [2,8,1]
+  CRUSH rule 1 x 483 [2,1,8]
+  CRUSH rule 1 x 484 [1,2,8]
+  CRUSH rule 1 x 485 [1,8,2]
+  CRUSH rule 1 x 486 [4,1,8]
+  CRUSH rule 1 x 487 [8,1,2]
+  CRUSH rule 1 x 488 [2,8,1]
+  CRUSH rule 1 x 489 [2,8,1]
+  CRUSH rule 1 x 490 [6,2,1]
+  CRUSH rule 1 x 491 [1,2,8]
+  CRUSH rule 1 x 492 [8,2,1]
+  CRUSH rule 1 x 493 [2,1,8]
+  CRUSH rule 1 x 494 [1,2,8]
+  CRUSH rule 1 x 495 [4,6,1]
+  CRUSH rule 1 x 496 [8,4]
+  CRUSH rule 1 x 497 [4,8,2]
+  CRUSH rule 1 x 498 [2,4,8]
+  CRUSH rule 1 x 499 [8,4,1]
+  CRUSH rule 1 x 500 [4,8,1]
+  CRUSH rule 1 x 501 [2,8,1]
+  CRUSH rule 1 x 502 [6,1,8]
+  CRUSH rule 1 x 503 [2,1,8]
+  CRUSH rule 1 x 504 [8,2,1]
+  CRUSH rule 1 x 505 [1,8,2]
+  CRUSH rule 1 x 506 [4,1,2]
+  CRUSH rule 1 x 507 [8,1,2]
+  CRUSH rule 1 x 508 [1,2,8]
+  CRUSH rule 1 x 509 [8,2]
+  CRUSH rule 1 x 510 [8,2,1]
+  CRUSH rule 1 x 511 [4,8]
+  CRUSH rule 1 x 512 [8,2,1]
+  CRUSH rule 1 x 513 [8,2,1]
+  CRUSH rule 1 x 514 [1,8,2]
+  CRUSH rule 1 x 515 [8,4,2]
+  CRUSH rule 1 x 516 [4,1,2]
+  CRUSH rule 1 x 517 [8,2,1]
+  CRUSH rule 1 x 518 [4,8,1]
+  CRUSH rule 1 x 519 [8,4,1]
+  CRUSH rule 1 x 520 [2,8,4]
+  CRUSH rule 1 x 521 [8,2,1]
+  CRUSH rule 1 x 522 [8,4,1]
+  CRUSH rule 1 x 523 [4,2,1]
+  CRUSH rule 1 x 524 [2,1,8]
+  CRUSH rule 1 x 525 [2,8,1]
+  CRUSH rule 1 x 526 [1,2,8]
+  CRUSH rule 1 x 527 [1,2,4]
+  CRUSH rule 1 x 528 [8,2,1]
+  CRUSH rule 1 x 529 [4,8,2]
+  CRUSH rule 1 x 530 [8,2]
+  CRUSH rule 1 x 531 [8,1,2]
+  CRUSH rule 1 x 532 [6,4,8]
+  CRUSH rule 1 x 533 [4,8,2]
+  CRUSH rule 1 x 534 [8,2,1]
+  CRUSH rule 1 x 535 [8,6,1]
+  CRUSH rule 1 x 536 [8,2,1]
+  CRUSH rule 1 x 537 [4,8]
+  CRUSH rule 1 x 538 [8,4,1]
+  CRUSH rule 1 x 539 [8,2,1]
+  CRUSH rule 1 x 540 [1,8,2]
+  CRUSH rule 1 x 541 [2,4,1]
+  CRUSH rule 1 x 542 [2,1,8]
+  CRUSH rule 1 x 543 [8,2,1]
+  CRUSH rule 1 x 544 [4,8,1]
+  CRUSH rule 1 x 545 [1,8,2]
+  CRUSH rule 1 x 546 [8,1,2]
+  CRUSH rule 1 x 547 [8,2,1]
+  CRUSH rule 1 x 548 [4,2,1]
+  CRUSH rule 1 x 549 [1,8,2]
+  CRUSH rule 1 x 550 [2,4,1]
+  CRUSH rule 1 x 551 [8,2]
+  CRUSH rule 1 x 552 [4,1,2]
+  CRUSH rule 1 x 553 [8,2]
+  CRUSH rule 1 x 554 [1,8,2]
+  CRUSH rule 1 x 555 [4,1,8]
+  CRUSH rule 1 x 556 [2,8,1]
+  CRUSH rule 1 x 557 [8,1]
+  CRUSH rule 1 x 558 [4,1,2]
+  CRUSH rule 1 x 559 [1,2,8]
+  CRUSH rule 1 x 560 [8,1]
+  CRUSH rule 1 x 561 [8,4,1]
+  CRUSH rule 1 x 562 [4,8,1]
+  CRUSH rule 1 x 563 [2,8,1]
+  CRUSH rule 1 x 564 [8,1,2]
+  CRUSH rule 1 x 565 [4,8,2]
+  CRUSH rule 1 x 566 [4,8,2]
+  CRUSH rule 1 x 567 [4,8,1]
+  CRUSH rule 1 x 568 [8,2,1]
+  CRUSH rule 1 x 569 [4,2,1]
+  CRUSH rule 1 x 570 [1,8,2]
+  CRUSH rule 1 x 571 [6,8,1]
+  CRUSH rule 1 x 572 [4,1]
+  CRUSH rule 1 x 573 [1,2,8]
+  CRUSH rule 1 x 574 [2,1,8]
+  CRUSH rule 1 x 575 [8,2,1]
+  CRUSH rule 1 x 576 [4,8,2]
+  CRUSH rule 1 x 577 [8,2,1]
+  CRUSH rule 1 x 578 [8,1,2]
+  CRUSH rule 1 x 579 [4,1,8]
+  CRUSH rule 1 x 580 [8,2,1]
+  CRUSH rule 1 x 581 [8,2,1]
+  CRUSH rule 1 x 582 [2,8,4]
+  CRUSH rule 1 x 583 [8,1,2]
+  CRUSH rule 1 x 584 [8,1,2]
+  CRUSH rule 1 x 585 [8,1,4]
+  CRUSH rule 1 x 586 [1,2,8]
+  CRUSH rule 1 x 587 [2,4,1]
+  CRUSH rule 1 x 588 [4,1,8]
+  CRUSH rule 1 x 589 [8,1]
+  CRUSH rule 1 x 590 [8,2,1]
+  CRUSH rule 1 x 591 [4,2,1]
+  CRUSH rule 1 x 592 [2,1,4]
+  CRUSH rule 1 x 593 [1,8,2]
+  CRUSH rule 1 x 594 [2,8,1]
+  CRUSH rule 1 x 595 [8,1,2]
+  CRUSH rule 1 x 596 [2,8,1]
+  CRUSH rule 1 x 597 [1,2,8]
+  CRUSH rule 1 x 598 [8,2,1]
+  CRUSH rule 1 x 599 [4,2,1]
+  CRUSH rule 1 x 600 [8,1,2]
+  CRUSH rule 1 x 601 [1,8,4]
+  CRUSH rule 1 x 602 [2,8,1]
+  CRUSH rule 1 x 603 [1,2,8]
+  CRUSH rule 1 x 604 [8,2,1]
+  CRUSH rule 1 x 605 [8,2,1]
+  CRUSH rule 1 x 606 [2,1,8]
+  CRUSH rule 1 x 607 [2,4,8]
+  CRUSH rule 1 x 608 [4,1,2]
+  CRUSH rule 1 x 609 [4,2,1]
+  CRUSH rule 1 x 610 [1,8,2]
+  CRUSH rule 1 x 611 [1,2,8]
+  CRUSH rule 1 x 612 [2,1,8]
+  CRUSH rule 1 x 613 [8,2,1]
+  CRUSH rule 1 x 614 [8,4,1]
+  CRUSH rule 1 x 615 [8,1,2]
+  CRUSH rule 1 x 616 [1,8,2]
+  CRUSH rule 1 x 617 [8,1,2]
+  CRUSH rule 1 x 618 [8,1,4]
+  CRUSH rule 1 x 619 [4,1,8]
+  CRUSH rule 1 x 620 [8,1,2]
+  CRUSH rule 1 x 621 [2,8,1]
+  CRUSH rule 1 x 622 [2,4,1]
+  CRUSH rule 1 x 623 [2,8,1]
+  CRUSH rule 1 x 624 [4,8,1]
+  CRUSH rule 1 x 625 [2,1,8]
+  CRUSH rule 1 x 626 [8,1,2]
+  CRUSH rule 1 x 627 [2,8,1]
+  CRUSH rule 1 x 628 [8,2,1]
+  CRUSH rule 1 x 629 [2,8,4]
+  CRUSH rule 1 x 630 [2,8,1]
+  CRUSH rule 1 x 631 [1,8,4]
+  CRUSH rule 1 x 632 [8,2,1]
+  CRUSH rule 1 x 633 [8,2,1]
+  CRUSH rule 1 x 634 [1,8,2]
+  CRUSH rule 1 x 635 [4,8,1]
+  CRUSH rule 1 x 636 [1,4,2]
+  CRUSH rule 1 x 637 [1,2,8]
+  CRUSH rule 1 x 638 [8,1,2]
+  CRUSH rule 1 x 639 [2,1,8]
+  CRUSH rule 1 x 640 [1,2,8]
+  CRUSH rule 1 x 641 [8,2,1]
+  CRUSH rule 1 x 642 [2,1,8]
+  CRUSH rule 1 x 643 [8,2,1]
+  CRUSH rule 1 x 644 [8,1,2]
+  CRUSH rule 1 x 645 [2,1,8]
+  CRUSH rule 1 x 646 [8,1,4]
+  CRUSH rule 1 x 647 [8,1,2]
+  CRUSH rule 1 x 648 [1,8,2]
+  CRUSH rule 1 x 649 [4,8,1]
+  CRUSH rule 1 x 650 [8,4,1]
+  CRUSH rule 1 x 651 [4,6,8]
+  CRUSH rule 1 x 652 [4,8,1]
+  CRUSH rule 1 x 653 [8,1,2]
+  CRUSH rule 1 x 654 [6,1,2]
+  CRUSH rule 1 x 655 [1,4,8]
+  CRUSH rule 1 x 656 [8,2,1]
+  CRUSH rule 1 x 657 [6,1,2]
+  CRUSH rule 1 x 658 [8,1,2]
+  CRUSH rule 1 x 659 [4,8,1]
+  CRUSH rule 1 x 660 [8,1,2]
+  CRUSH rule 1 x 661 [1,8,2]
+  CRUSH rule 1 x 662 [8,2]
+  CRUSH rule 1 x 663 [1,4,8]
+  CRUSH rule 1 x 664 [1,4,2]
+  CRUSH rule 1 x 665 [4,6,8]
+  CRUSH rule 1 x 666 [2,8,1]
+  CRUSH rule 1 x 667 [1,4,2]
+  CRUSH rule 1 x 668 [4,8,2]
+  CRUSH rule 1 x 669 [6,4,2]
+  CRUSH rule 1 x 670 [4,2,1]
+  CRUSH rule 1 x 671 [2,1,8]
+  CRUSH rule 1 x 672 [4,8,2]
+  CRUSH rule 1 x 673 [4,2,1]
+  CRUSH rule 1 x 674 [8,1,2]
+  CRUSH rule 1 x 675 [1,8,6]
+  CRUSH rule 1 x 676 [2,1,4]
+  CRUSH rule 1 x 677 [4,1,8]
+  CRUSH rule 1 x 678 [2,4,1]
+  CRUSH rule 1 x 679 [8,2,1]
+  CRUSH rule 1 x 680 [2,8]
+  CRUSH rule 1 x 681 [2,8,1]
+  CRUSH rule 1 x 682 [1,4,2]
+  CRUSH rule 1 x 683 [1,2,4]
+  CRUSH rule 1 x 684 [8,1,4]
+  CRUSH rule 1 x 685 [8,1,2]
+  CRUSH rule 1 x 686 [1,4,2]
+  CRUSH rule 1 x 687 [1,6,8]
+  CRUSH rule 1 x 688 [4,8,2]
+  CRUSH rule 1 x 689 [8,4,2]
+  CRUSH rule 1 x 690 [8,1,4]
+  CRUSH rule 1 x 691 [8,1,2]
+  CRUSH rule 1 x 692 [8,2,1]
+  CRUSH rule 1 x 693 [8,4,1]
+  CRUSH rule 1 x 694 [8,4,1]
+  CRUSH rule 1 x 695 [2,8,4]
+  CRUSH rule 1 x 696 [1,2,8]
+  CRUSH rule 1 x 697 [8,1,2]
+  CRUSH rule 1 x 698 [8,2,1]
+  CRUSH rule 1 x 699 [1,8,2]
+  CRUSH rule 1 x 700 [1,2,8]
+  CRUSH rule 1 x 701 [2,1,8]
+  CRUSH rule 1 x 702 [8,1]
+  CRUSH rule 1 x 703 [8,1,2]
+  CRUSH rule 1 x 704 [1,4,8]
+  CRUSH rule 1 x 705 [8,4,2]
+  CRUSH rule 1 x 706 [1,2,4]
+  CRUSH rule 1 x 707 [8,2,1]
+  CRUSH rule 1 x 708 [4,8,1]
+  CRUSH rule 1 x 709 [8,2,1]
+  CRUSH rule 1 x 710 [8,1,2]
+  CRUSH rule 1 x 711 [2,4,8]
+  CRUSH rule 1 x 712 [2,8,1]
+  CRUSH rule 1 x 713 [8,2,4]
+  CRUSH rule 1 x 714 [1,2,8]
+  CRUSH rule 1 x 715 [1,2,8]
+  CRUSH rule 1 x 716 [4,8,2]
+  CRUSH rule 1 x 717 [8,4,2]
+  CRUSH rule 1 x 718 [2,8,6]
+  CRUSH rule 1 x 719 [2,6,4]
+  CRUSH rule 1 x 720 [8,1,2]
+  CRUSH rule 1 x 721 [4,6,8]
+  CRUSH rule 1 x 722 [8,1,2]
+  CRUSH rule 1 x 723 [4,1,2]
+  CRUSH rule 1 x 724 [2,6,1]
+  CRUSH rule 1 x 725 [1,2,8]
+  CRUSH rule 1 x 726 [4,8,1]
+  CRUSH rule 1 x 727 [4,8,1]
+  CRUSH rule 1 x 728 [2,1,8]
+  CRUSH rule 1 x 729 [2,8]
+  CRUSH rule 1 x 730 [4,8,2]
+  CRUSH rule 1 x 731 [4,1,8]
+  CRUSH rule 1 x 732 [1,2,8]
+  CRUSH rule 1 x 733 [4,1,8]
+  CRUSH rule 1 x 734 [8,4,2]
+  CRUSH rule 1 x 735 [4,8,2]
+  CRUSH rule 1 x 736 [4,8]
+  CRUSH rule 1 x 737 [1,2,8]
+  CRUSH rule 1 x 738 [4,2,8]
+  CRUSH rule 1 x 739 [2,1,8]
+  CRUSH rule 1 x 740 [1,2,8]
+  CRUSH rule 1 x 741 [8,2,1]
+  CRUSH rule 1 x 742 [8,2,1]
+  CRUSH rule 1 x 743 [8,1,2]
+  CRUSH rule 1 x 744 [4,8,2]
+  CRUSH rule 1 x 745 [8,2,1]
+  CRUSH rule 1 x 746 [8,1,2]
+  CRUSH rule 1 x 747 [8,1,2]
+  CRUSH rule 1 x 748 [2,8,1]
+  CRUSH rule 1 x 749 [4,8,1]
+  CRUSH rule 1 x 750 [1,8,4]
+  CRUSH rule 1 x 751 [2,1,8]
+  CRUSH rule 1 x 752 [8,1]
+  CRUSH rule 1 x 753 [8,1,4]
+  CRUSH rule 1 x 754 [8,4,2]
+  CRUSH rule 1 x 755 [1,2,4]
+  CRUSH rule 1 x 756 [8,2,1]
+  CRUSH rule 1 x 757 [8,4,1]
+  CRUSH rule 1 x 758 [8,2]
+  CRUSH rule 1 x 759 [8,4,1]
+  CRUSH rule 1 x 760 [1,4,2]
+  CRUSH rule 1 x 761 [1,2,6]
+  CRUSH rule 1 x 762 [2,8,1]
+  CRUSH rule 1 x 763 [8,2,4]
+  CRUSH rule 1 x 764 [1,8,2]
+  CRUSH rule 1 x 765 [8,2,1]
+  CRUSH rule 1 x 766 [8]
+  CRUSH rule 1 x 767 [1,2,4]
+  CRUSH rule 1 x 768 [8,4,2]
+  CRUSH rule 1 x 769 [8,2,4]
+  CRUSH rule 1 x 770 [8,2,4]
+  CRUSH rule 1 x 771 [8,1,4]
+  CRUSH rule 1 x 772 [8,4,2]
+  CRUSH rule 1 x 773 [4,1,8]
+  CRUSH rule 1 x 774 [8,1,2]
+  CRUSH rule 1 x 775 [8,2,4]
+  CRUSH rule 1 x 776 [6,2,1]
+  CRUSH rule 1 x 777 [4,1,8]
+  CRUSH rule 1 x 778 [1,8,2]
+  CRUSH rule 1 x 779 [2,8,1]
+  CRUSH rule 1 x 780 [2,1,4]
+  CRUSH rule 1 x 781 [8,2,1]
+  CRUSH rule 1 x 782 [4,1,2]
+  CRUSH rule 1 x 783 [8,1,4]
+  CRUSH rule 1 x 784 [1,2,4]
+  CRUSH rule 1 x 785 [8,1,2]
+  CRUSH rule 1 x 786 [8,2,1]
+  CRUSH rule 1 x 787 [1,2,6]
+  CRUSH rule 1 x 788 [8,2,1]
+  CRUSH rule 1 x 789 [1,6,8]
+  CRUSH rule 1 x 790 [8,2,1]
+  CRUSH rule 1 x 791 [4,8,1]
+  CRUSH rule 1 x 792 [4,8]
+  CRUSH rule 1 x 793 [8,1,4]
+  CRUSH rule 1 x 794 [2,8,4]
+  CRUSH rule 1 x 795 [1,8,2]
+  CRUSH rule 1 x 796 [2,8,1]
+  CRUSH rule 1 x 797 [2,4,8]
+  CRUSH rule 1 x 798 [6,8,1]
+  CRUSH rule 1 x 799 [4,1,8]
+  CRUSH rule 1 x 800 [8,2,1]
+  CRUSH rule 1 x 801 [4,8,2]
+  CRUSH rule 1 x 802 [1,8,2]
+  CRUSH rule 1 x 803 [2,8,1]
+  CRUSH rule 1 x 804 [8,2,1]
+  CRUSH rule 1 x 805 [2,8,1]
+  CRUSH rule 1 x 806 [1,4,2]
+  CRUSH rule 1 x 807 [4,8]
+  CRUSH rule 1 x 808 [2,8,1]
+  CRUSH rule 1 x 809 [1,8,2]
+  CRUSH rule 1 x 810 [2,8,1]
+  CRUSH rule 1 x 811 [8,2,1]
+  CRUSH rule 1 x 812 [8,4,1]
+  CRUSH rule 1 x 813 [8,4,2]
+  CRUSH rule 1 x 814 [8,1]
+  CRUSH rule 1 x 815 [4,1,2]
+  CRUSH rule 1 x 816 [2,1,8]
+  CRUSH rule 1 x 817 [8,1,2]
+  CRUSH rule 1 x 818 [8,2,1]
+  CRUSH rule 1 x 819 [8,1,2]
+  CRUSH rule 1 x 820 [4,1,2]
+  CRUSH rule 1 x 821 [4,1,8]
+  CRUSH rule 1 x 822 [2,1,4]
+  CRUSH rule 1 x 823 [4,8,2]
+  CRUSH rule 1 x 824 [8,2,1]
+  CRUSH rule 1 x 825 [2,8,4]
+  CRUSH rule 1 x 826 [8,2,4]
+  CRUSH rule 1 x 827 [2,8,1]
+  CRUSH rule 1 x 828 [2,1,8]
+  CRUSH rule 1 x 829 [8,2]
+  CRUSH rule 1 x 830 [2,4,1]
+  CRUSH rule 1 x 831 [1,8,2]
+  CRUSH rule 1 x 832 [4,8,1]
+  CRUSH rule 1 x 833 [2,1,8]
+  CRUSH rule 1 x 834 [2,1]
+  CRUSH rule 1 x 835 [8,4,2]
+  CRUSH rule 1 x 836 [4,1,2]
+  CRUSH rule 1 x 837 [8,4,1]
+  CRUSH rule 1 x 838 [6,8,2]
+  CRUSH rule 1 x 839 [8,2,6]
+  CRUSH rule 1 x 840 [8,2,1]
+  CRUSH rule 1 x 841 [4,8,2]
+  CRUSH rule 1 x 842 [2,1]
+  CRUSH rule 1 x 843 [8,4,1]
+  CRUSH rule 1 x 844 [1,8,2]
+  CRUSH rule 1 x 845 [4,8,2]
+  CRUSH rule 1 x 846 [4,2,1]
+  CRUSH rule 1 x 847 [2,1,8]
+  CRUSH rule 1 x 848 [2,8,1]
+  CRUSH rule 1 x 849 [4,8,1]
+  CRUSH rule 1 x 850 [1,2,6]
+  CRUSH rule 1 x 851 [6,8]
+  CRUSH rule 1 x 852 [8,4,2]
+  CRUSH rule 1 x 853 [6,8,1]
+  CRUSH rule 1 x 854 [8,1,2]
+  CRUSH rule 1 x 855 [8,1,2]
+  CRUSH rule 1 x 856 [8,4,1]
+  CRUSH rule 1 x 857 [8,1,2]
+  CRUSH rule 1 x 858 [6,2,1]
+  CRUSH rule 1 x 859 [8,2,1]
+  CRUSH rule 1 x 860 [8,1,2]
+  CRUSH rule 1 x 861 [8,1]
+  CRUSH rule 1 x 862 [8,1]
+  CRUSH rule 1 x 863 [8,1,2]
+  CRUSH rule 1 x 864 [8,2,1]
+  CRUSH rule 1 x 865 [8,1,2]
+  CRUSH rule 1 x 866 [2,8]
+  CRUSH rule 1 x 867 [8,1]
+  CRUSH rule 1 x 868 [8,2,1]
+  CRUSH rule 1 x 869 [8,6,4]
+  CRUSH rule 1 x 870 [2,1,8]
+  CRUSH rule 1 x 871 [2,8,1]
+  CRUSH rule 1 x 872 [8,1,2]
+  CRUSH rule 1 x 873 [4,8,2]
+  CRUSH rule 1 x 874 [2,6,1]
+  CRUSH rule 1 x 875 [2,8,4]
+  CRUSH rule 1 x 876 [4,8,1]
+  CRUSH rule 1 x 877 [8,4,2]
+  CRUSH rule 1 x 878 [8,1,2]
+  CRUSH rule 1 x 879 [8,2,1]
+  CRUSH rule 1 x 880 [6,1,2]
+  CRUSH rule 1 x 881 [4,8,1]
+  CRUSH rule 1 x 882 [8,2,1]
+  CRUSH rule 1 x 883 [2,1,4]
+  CRUSH rule 1 x 884 [8,2,4]
+  CRUSH rule 1 x 885 [4,1,8]
+  CRUSH rule 1 x 886 [2,8,1]
+  CRUSH rule 1 x 887 [8,4,1]
+  CRUSH rule 1 x 888 [8,2,1]
+  CRUSH rule 1 x 889 [2,1,8]
+  CRUSH rule 1 x 890 [8,2,1]
+  CRUSH rule 1 x 891 [1,8,2]
+  CRUSH rule 1 x 892 [8,2,4]
+  CRUSH rule 1 x 893 [2,8,6]
+  CRUSH rule 1 x 894 [8,4,2]
+  CRUSH rule 1 x 895 [4,8,2]
+  CRUSH rule 1 x 896 [1,8,2]
+  CRUSH rule 1 x 897 [8,2,1]
+  CRUSH rule 1 x 898 [1,4,8]
+  CRUSH rule 1 x 899 [1,8,2]
+  CRUSH rule 1 x 900 [4,1,2]
+  CRUSH rule 1 x 901 [8,1,2]
+  CRUSH rule 1 x 902 [8,4,2]
+  CRUSH rule 1 x 903 [1,8,2]
+  CRUSH rule 1 x 904 [1,8,2]
+  CRUSH rule 1 x 905 [8,2,1]
+  CRUSH rule 1 x 906 [1,2,8]
+  CRUSH rule 1 x 907 [8,1,2]
+  CRUSH rule 1 x 908 [8,1,2]
+  CRUSH rule 1 x 909 [2,1,8]
+  CRUSH rule 1 x 910 [8,2,1]
+  CRUSH rule 1 x 911 [8,1,2]
+  CRUSH rule 1 x 912 [1,2,8]
+  CRUSH rule 1 x 913 [8,4,1]
+  CRUSH rule 1 x 914 [6,4,8]
+  CRUSH rule 1 x 915 [8,2]
+  CRUSH rule 1 x 916 [4,1,2]
+  CRUSH rule 1 x 917 [1,4,8]
+  CRUSH rule 1 x 918 [8,2,1]
+  CRUSH rule 1 x 919 [8,2,1]
+  CRUSH rule 1 x 920 [8,1,2]
+  CRUSH rule 1 x 921 [1,2,8]
+  CRUSH rule 1 x 922 [8,4,2]
+  CRUSH rule 1 x 923 [4,8,1]
+  CRUSH rule 1 x 924 [1,8,2]
+  CRUSH rule 1 x 925 [4,8,2]
+  CRUSH rule 1 x 926 [1,8,2]
+  CRUSH rule 1 x 927 [1,8,4]
+  CRUSH rule 1 x 928 [8,1,2]
+  CRUSH rule 1 x 929 [4,1,2]
+  CRUSH rule 1 x 930 [2,1,8]
+  CRUSH rule 1 x 931 [8,1,2]
+  CRUSH rule 1 x 932 [4,8,1]
+  CRUSH rule 1 x 933 [8,4,2]
+  CRUSH rule 1 x 934 [8,1,2]
+  CRUSH rule 1 x 935 [8,1,2]
+  CRUSH rule 1 x 936 [1,8,2]
+  CRUSH rule 1 x 937 [4,8]
+  CRUSH rule 1 x 938 [8,4,2]
+  CRUSH rule 1 x 939 [2,8,1]
+  CRUSH rule 1 x 940 [8,2,1]
+  CRUSH rule 1 x 941 [8,2,1]
+  CRUSH rule 1 x 942 [1,2,8]
+  CRUSH rule 1 x 943 [8,2,1]
+  CRUSH rule 1 x 944 [2,1,8]
+  CRUSH rule 1 x 945 [8,2,4]
+  CRUSH rule 1 x 946 [2,1,8]
+  CRUSH rule 1 x 947 [8,1,2]
+  CRUSH rule 1 x 948 [8,1,2]
+  CRUSH rule 1 x 949 [6,1,8]
+  CRUSH rule 1 x 950 [8,1,2]
+  CRUSH rule 1 x 951 [2,8,1]
+  CRUSH rule 1 x 952 [2,1,8]
+  CRUSH rule 1 x 953 [1,4,2]
+  CRUSH rule 1 x 954 [8,2,1]
+  CRUSH rule 1 x 955 [8,1,2]
+  CRUSH rule 1 x 956 [1,2,8]
+  CRUSH rule 1 x 957 [8,1,2]
+  CRUSH rule 1 x 958 [8,2,4]
+  CRUSH rule 1 x 959 [4,2,8]
+  CRUSH rule 1 x 960 [2,6,8]
+  CRUSH rule 1 x 961 [1,2,8]
+  CRUSH rule 1 x 962 [8,4,1]
+  CRUSH rule 1 x 963 [2,4,1]
+  CRUSH rule 1 x 964 [8,1,2]
+  CRUSH rule 1 x 965 [8,2,1]
+  CRUSH rule 1 x 966 [4,8,1]
+  CRUSH rule 1 x 967 [8,1,4]
+  CRUSH rule 1 x 968 [8,2,1]
+  CRUSH rule 1 x 969 [8,2,4]
+  CRUSH rule 1 x 970 [2,8,4]
+  CRUSH rule 1 x 971 [1,8,2]
+  CRUSH rule 1 x 972 [1,8,2]
+  CRUSH rule 1 x 973 [1,2]
+  CRUSH rule 1 x 974 [4,1,2]
+  CRUSH rule 1 x 975 [4,8]
+  CRUSH rule 1 x 976 [4,8,2]
+  CRUSH rule 1 x 977 [8,4,2]
+  CRUSH rule 1 x 978 [8,2,1]
+  CRUSH rule 1 x 979 [8,1,2]
+  CRUSH rule 1 x 980 [8,2,1]
+  CRUSH rule 1 x 981 [8,2]
+  CRUSH rule 1 x 982 [1,2,8]
+  CRUSH rule 1 x 983 [4,8,1]
+  CRUSH rule 1 x 984 [2,1,8]
+  CRUSH rule 1 x 985 [2,4,8]
+  CRUSH rule 1 x 986 [8,4,1]
+  CRUSH rule 1 x 987 [2,1,8]
+  CRUSH rule 1 x 988 [1,4,6]
+  CRUSH rule 1 x 989 [1,8,2]
+  CRUSH rule 1 x 990 [1,2,8]
+  CRUSH rule 1 x 991 [1,4,2]
+  CRUSH rule 1 x 992 [8,1,4]
+  CRUSH rule 1 x 993 [2,8,1]
+  CRUSH rule 1 x 994 [4,2,1]
+  CRUSH rule 1 x 995 [8,1,2]
+  CRUSH rule 1 x 996 [8,2,4]
+  CRUSH rule 1 x 997 [8,4,1]
+  CRUSH rule 1 x 998 [8,1,2]
+  CRUSH rule 1 x 999 [1,8,4]
+  CRUSH rule 1 x 1000 [8,4,2]
+  CRUSH rule 1 x 1001 [2,1]
+  CRUSH rule 1 x 1002 [1,2,8]
+  CRUSH rule 1 x 1003 [2,8]
+  CRUSH rule 1 x 1004 [8,1,2]
+  CRUSH rule 1 x 1005 [8,1,2]
+  CRUSH rule 1 x 1006 [1,2,4]
+  CRUSH rule 1 x 1007 [1,2,4]
+  CRUSH rule 1 x 1008 [1,8,2]
+  CRUSH rule 1 x 1009 [6,8,4]
+  CRUSH rule 1 x 1010 [2,8,1]
+  CRUSH rule 1 x 1011 [4,2,8]
+  CRUSH rule 1 x 1012 [1,2,8]
+  CRUSH rule 1 x 1013 [1,2,8]
+  CRUSH rule 1 x 1014 [2,8,4]
+  CRUSH rule 1 x 1015 [8,1,2]
+  CRUSH rule 1 x 1016 [2,1,4]
+  CRUSH rule 1 x 1017 [6,2,1]
+  CRUSH rule 1 x 1018 [4,2,1]
+  CRUSH rule 1 x 1019 [4,8,2]
+  CRUSH rule 1 x 1020 [1,2,8]
+  CRUSH rule 1 x 1021 [8,2,1]
+  CRUSH rule 1 x 1022 [1,8,4]
+  CRUSH rule 1 x 1023 [4,2,1]
+  rule 1 (choose-two) num_rep 3 result size == 1:\t2/1024 (esc)
+  rule 1 (choose-two) num_rep 3 result size == 2:\t82/1024 (esc)
+  rule 1 (choose-two) num_rep 3 result size == 3:\t940/1024 (esc)
+  rule 2 (chooseleaf), x = 0..1023, numrep = 2..3
+  CRUSH rule 2 x 0 [2,4]
+  CRUSH rule 2 x 1 [2,8]
+  CRUSH rule 2 x 2 [1,8]
+  CRUSH rule 2 x 3 [8,1]
+  CRUSH rule 2 x 4 [4,2]
+  CRUSH rule 2 x 5 [8,2]
+  CRUSH rule 2 x 6 [2,8]
+  CRUSH rule 2 x 7 [4,8]
+  CRUSH rule 2 x 8 [4,8]
+  CRUSH rule 2 x 9 [2,4]
+  CRUSH rule 2 x 10 [2,8]
+  CRUSH rule 2 x 11 [2,8]
+  CRUSH rule 2 x 12 [2,8]
+  CRUSH rule 2 x 13 [4,8]
+  CRUSH rule 2 x 14 [8,2]
+  CRUSH rule 2 x 15 [8,2]
+  CRUSH rule 2 x 16 [8,2]
+  CRUSH rule 2 x 17 [4,1]
+  CRUSH rule 2 x 18 [1,8]
+  CRUSH rule 2 x 19 [8,4]
+  CRUSH rule 2 x 20 [2,8]
+  CRUSH rule 2 x 21 [8,2]
+  CRUSH rule 2 x 22 [8,1]
+  CRUSH rule 2 x 23 [4,8]
+  CRUSH rule 2 x 24 [1,8]
+  CRUSH rule 2 x 25 [4,8]
+  CRUSH rule 2 x 26 [2,8]
+  CRUSH rule 2 x 27 [4,1]
+  CRUSH rule 2 x 28 [8,2]
+  CRUSH rule 2 x 29 [8,4]
+  CRUSH rule 2 x 30 [4,8]
+  CRUSH rule 2 x 31 [8,1]
+  CRUSH rule 2 x 32 [6,1]
+  CRUSH rule 2 x 33 [2,8]
+  CRUSH rule 2 x 34 [2,8]
+  CRUSH rule 2 x 35 [1,8]
+  CRUSH rule 2 x 36 [8,2]
+  CRUSH rule 2 x 37 [1,8]
+  CRUSH rule 2 x 38 [4,8]
+  CRUSH rule 2 x 39 [8,2]
+  CRUSH rule 2 x 40 [8,2]
+  CRUSH rule 2 x 41 [2,8]
+  CRUSH rule 2 x 42 [8,2]
+  CRUSH rule 2 x 43 [1,8]
+  CRUSH rule 2 x 44 [1,8]
+  CRUSH rule 2 x 45 [8,2]
+  CRUSH rule 2 x 46 [2,8]
+  CRUSH rule 2 x 47 [4,2]
+  CRUSH rule 2 x 48 [8,1]
+  CRUSH rule 2 x 49 [8,2]
+  CRUSH rule 2 x 50 [4,1]
+  CRUSH rule 2 x 51 [8,2]
+  CRUSH rule 2 x 52 [8,1]
+  CRUSH rule 2 x 53 [4,8]
+  CRUSH rule 2 x 54 [8,4]
+  CRUSH rule 2 x 55 [8,2]
+  CRUSH rule 2 x 56 [8,4]
+  CRUSH rule 2 x 57 [8,1]
+  CRUSH rule 2 x 58 [1,8]
+  CRUSH rule 2 x 59 [2,8]
+  CRUSH rule 2 x 60 [4,2]
+  CRUSH rule 2 x 61 [4,8]
+  CRUSH rule 2 x 62 [8,1]
+  CRUSH rule 2 x 63 [8,2]
+  CRUSH rule 2 x 64 [4,2]
+  CRUSH rule 2 x 65 [8,4]
+  CRUSH rule 2 x 66 [4,8]
+  CRUSH rule 2 x 67 [4,2]
+  CRUSH rule 2 x 68 [1,8]
+  CRUSH rule 2 x 69 [2,8]
+  CRUSH rule 2 x 70 [8,2]
+  CRUSH rule 2 x 71 [2,8]
+  CRUSH rule 2 x 72 [8,1]
+  CRUSH rule 2 x 73 [2,8]
+  CRUSH rule 2 x 74 [1,8]
+  CRUSH rule 2 x 75 [4,2]
+  CRUSH rule 2 x 76 [4,1]
+  CRUSH rule 2 x 77 [8,2]
+  CRUSH rule 2 x 78 [1,6]
+  CRUSH rule 2 x 79 [4,1]
+  CRUSH rule 2 x 80 [2,4]
+  CRUSH rule 2 x 81 [2,8]
+  CRUSH rule 2 x 82 [6,1]
+  CRUSH rule 2 x 83 [2,8]
+  CRUSH rule 2 x 84 [8,2]
+  CRUSH rule 2 x 85 [4,8]
+  CRUSH rule 2 x 86 [2,8]
+  CRUSH rule 2 x 87 [2,8]
+  CRUSH rule 2 x 88 [1,6]
+  CRUSH rule 2 x 89 [1,8]
+  CRUSH rule 2 x 90 [8,4]
+  CRUSH rule 2 x 91 [4,8]
+  CRUSH rule 2 x 92 [1,8]
+  CRUSH rule 2 x 93 [8,4]
+  CRUSH rule 2 x 94 [1,8]
+  CRUSH rule 2 x 95 [8,1]
+  CRUSH rule 2 x 96 [8,2]
+  CRUSH rule 2 x 97 [8,1]
+  CRUSH rule 2 x 98 [2,8]
+  CRUSH rule 2 x 99 [2,8]
+  CRUSH rule 2 x 100 [1,8]
+  CRUSH rule 2 x 101 [8,1]
+  CRUSH rule 2 x 102 [2,8]
+  CRUSH rule 2 x 103 [8,2]
+  CRUSH rule 2 x 104 [8,4]
+  CRUSH rule 2 x 105 [2,4]
+  CRUSH rule 2 x 106 [1,8]
+  CRUSH rule 2 x 107 [1,8]
+  CRUSH rule 2 x 108 [8,2]
+  CRUSH rule 2 x 109 [1,4]
+  CRUSH rule 2 x 110 [4,2]
+  CRUSH rule 2 x 111 [2,4]
+  CRUSH rule 2 x 112 [2,8]
+  CRUSH rule 2 x 113 [8,2]
+  CRUSH rule 2 x 114 [8,4]
+  CRUSH rule 2 x 115 [8,2]
+  CRUSH rule 2 x 116 [1,8]
+  CRUSH rule 2 x 117 [6,1]
+  CRUSH rule 2 x 118 [2,8]
+  CRUSH rule 2 x 119 [8,1]
+  CRUSH rule 2 x 120 [2,4]
+  CRUSH rule 2 x 121 [2,8]
+  CRUSH rule 2 x 122 [8,1]
+  CRUSH rule 2 x 123 [2,8]
+  CRUSH rule 2 x 124 [2,8]
+  CRUSH rule 2 x 125 [1,8]
+  CRUSH rule 2 x 126 [1,8]
+  CRUSH rule 2 x 127 [4,8]
+  CRUSH rule 2 x 128 [8,2]
+  CRUSH rule 2 x 129 [2,4]
+  CRUSH rule 2 x 130 [4,8]
+  CRUSH rule 2 x 131 [1,4]
+  CRUSH rule 2 x 132 [1,8]
+  CRUSH rule 2 x 133 [8,1]
+  CRUSH rule 2 x 134 [1,8]
+  CRUSH rule 2 x 135 [4,8]
+  CRUSH rule 2 x 136 [2,4]
+  CRUSH rule 2 x 137 [8,4]
+  CRUSH rule 2 x 138 [8,4]
+  CRUSH rule 2 x 139 [4,2]
+  CRUSH rule 2 x 140 [1,8]
+  CRUSH rule 2 x 141 [8,2]
+  CRUSH rule 2 x 142 [4,1]
+  CRUSH rule 2 x 143 [4,8]
+  CRUSH rule 2 x 144 [8,1]
+  CRUSH rule 2 x 145 [8,1]
+  CRUSH rule 2 x 146 [2,8]
+  CRUSH rule 2 x 147 [2,8]
+  CRUSH rule 2 x 148 [4,1]
+  CRUSH rule 2 x 149 [4,8]
+  CRUSH rule 2 x 150 [1,8]
+  CRUSH rule 2 x 151 [1,8]
+  CRUSH rule 2 x 152 [8,2]
+  CRUSH rule 2 x 153 [8,4]
+  CRUSH rule 2 x 154 [4,2]
+  CRUSH rule 2 x 155 [4,8]
+  CRUSH rule 2 x 156 [4,2]
+  CRUSH rule 2 x 157 [1,8]
+  CRUSH rule 2 x 158 [2,8]
+  CRUSH rule 2 x 159 [8,2]
+  CRUSH rule 2 x 160 [2,8]
+  CRUSH rule 2 x 161 [1,4]
+  CRUSH rule 2 x 162 [1,8]
+  CRUSH rule 2 x 163 [4,8]
+  CRUSH rule 2 x 164 [8,1]
+  CRUSH rule 2 x 165 [8,2]
+  CRUSH rule 2 x 166 [2,8]
+  CRUSH rule 2 x 167 [1,8]
+  CRUSH rule 2 x 168 [4,2]
+  CRUSH rule 2 x 169 [2,8]
+  CRUSH rule 2 x 170 [1,8]
+  CRUSH rule 2 x 171 [8,4]
+  CRUSH rule 2 x 172 [1,8]
+  CRUSH rule 2 x 173 [8,4]
+  CRUSH rule 2 x 174 [1,6]
+  CRUSH rule 2 x 175 [8,1]
+  CRUSH rule 2 x 176 [2,8]
+  CRUSH rule 2 x 177 [1,8]
+  CRUSH rule 2 x 178 [4,2]
+  CRUSH rule 2 x 179 [1,8]
+  CRUSH rule 2 x 180 [8,1]
+  CRUSH rule 2 x 181 [8,2]
+  CRUSH rule 2 x 182 [8,1]
+  CRUSH rule 2 x 183 [8,4]
+  CRUSH rule 2 x 184 [4,8]
+  CRUSH rule 2 x 185 [8,1]
+  CRUSH rule 2 x 186 [2,4]
+  CRUSH rule 2 x 187 [1,8]
+  CRUSH rule 2 x 188 [1,8]
+  CRUSH rule 2 x 189 [1,8]
+  CRUSH rule 2 x 190 [1,8]
+  CRUSH rule 2 x 191 [8,1]
+  CRUSH rule 2 x 192 [4,1]
+  CRUSH rule 2 x 193 [4,2]
+  CRUSH rule 2 x 194 [1,8]
+  CRUSH rule 2 x 195 [8,4]
+  CRUSH rule 2 x 196 [8,2]
+  CRUSH rule 2 x 197 [8,4]
+  CRUSH rule 2 x 198 [2,8]
+  CRUSH rule 2 x 199 [1,4]
+  CRUSH rule 2 x 200 [1,8]
+  CRUSH rule 2 x 201 [8,1]
+  CRUSH rule 2 x 202 [8,1]
+  CRUSH rule 2 x 203 [8,1]
+  CRUSH rule 2 x 204 [2,4]
+  CRUSH rule 2 x 205 [1,8]
+  CRUSH rule 2 x 206 [1,8]
+  CRUSH rule 2 x 207 [2,8]
+  CRUSH rule 2 x 208 [8,1]
+  CRUSH rule 2 x 209 [1,8]
+  CRUSH rule 2 x 210 [1,4]
+  CRUSH rule 2 x 211 [4,2]
+  CRUSH rule 2 x 212 [8,1]
+  CRUSH rule 2 x 213 [8,4]
+  CRUSH rule 2 x 214 [8,2]
+  CRUSH rule 2 x 215 [8,1]
+  CRUSH rule 2 x 216 [2,8]
+  CRUSH rule 2 x 217 [1,8]
+  CRUSH rule 2 x 218 [2,8]
+  CRUSH rule 2 x 219 [8,2]
+  CRUSH rule 2 x 220 [4,8]
+  CRUSH rule 2 x 221 [8,1]
+  CRUSH rule 2 x 222 [8,1]
+  CRUSH rule 2 x 223 [1,8]
+  CRUSH rule 2 x 224 [1,4]
+  CRUSH rule 2 x 225 [8,2]
+  CRUSH rule 2 x 226 [8,2]
+  CRUSH rule 2 x 227 [4,1]
+  CRUSH rule 2 x 228 [8,2]
+  CRUSH rule 2 x 229 [4,8]
+  CRUSH rule 2 x 230 [4,8]
+  CRUSH rule 2 x 231 [4,8]
+  CRUSH rule 2 x 232 [2,8]
+  CRUSH rule 2 x 233 [2,8]
+  CRUSH rule 2 x 234 [1,8]
+  CRUSH rule 2 x 235 [4,8]
+  CRUSH rule 2 x 236 [2,6]
+  CRUSH rule 2 x 237 [4,8]
+  CRUSH rule 2 x 238 [2,8]
+  CRUSH rule 2 x 239 [8,1]
+  CRUSH rule 2 x 240 [4,8]
+  CRUSH rule 2 x 241 [1,8]
+  CRUSH rule 2 x 242 [8,2]
+  CRUSH rule 2 x 243 [8,2]
+  CRUSH rule 2 x 244 [4,8]
+  CRUSH rule 2 x 245 [8,1]
+  CRUSH rule 2 x 246 [1,8]
+  CRUSH rule 2 x 247 [8,2]
+  CRUSH rule 2 x 248 [8,2]
+  CRUSH rule 2 x 249 [2,8]
+  CRUSH rule 2 x 250 [2,4]
+  CRUSH rule 2 x 251 [2,8]
+  CRUSH rule 2 x 252 [4,8]
+  CRUSH rule 2 x 253 [2,8]
+  CRUSH rule 2 x 254 [4,2]
+  CRUSH rule 2 x 255 [1,8]
+  CRUSH rule 2 x 256 [4,8]
+  CRUSH rule 2 x 257 [2,8]
+  CRUSH rule 2 x 258 [4,2]
+  CRUSH rule 2 x 259 [6,2]
+  CRUSH rule 2 x 260 [8,2]
+  CRUSH rule 2 x 261 [8,1]
+  CRUSH rule 2 x 262 [8,1]
+  CRUSH rule 2 x 263 [8,1]
+  CRUSH rule 2 x 264 [8,2]
+  CRUSH rule 2 x 265 [8,2]
+  CRUSH rule 2 x 266 [8,2]
+  CRUSH rule 2 x 267 [2,8]
+  CRUSH rule 2 x 268 [1,8]
+  CRUSH rule 2 x 269 [1,8]
+  CRUSH rule 2 x 270 [4,1]
+  CRUSH rule 2 x 271 [8,4]
+  CRUSH rule 2 x 272 [2,8]
+  CRUSH rule 2 x 273 [4,1]
+  CRUSH rule 2 x 274 [8,4]
+  CRUSH rule 2 x 275 [4,8]
+  CRUSH rule 2 x 276 [8,1]
+  CRUSH rule 2 x 277 [8,1]
+  CRUSH rule 2 x 278 [8,1]
+  CRUSH rule 2 x 279 [8,4]
+  CRUSH rule 2 x 280 [2,8]
+  CRUSH rule 2 x 281 [8,2]
+  CRUSH rule 2 x 282 [2,8]
+  CRUSH rule 2 x 283 [8,2]
+  CRUSH rule 2 x 284 [8,2]
+  CRUSH rule 2 x 285 [4,8]
+  CRUSH rule 2 x 286 [2,8]
+  CRUSH rule 2 x 287 [1,8]
+  CRUSH rule 2 x 288 [8,1]
+  CRUSH rule 2 x 289 [4,8]
+  CRUSH rule 2 x 290 [1,4]
+  CRUSH rule 2 x 291 [1,4]
+  CRUSH rule 2 x 292 [8,2]
+  CRUSH rule 2 x 293 [8,1]
+  CRUSH rule 2 x 294 [8,4]
+  CRUSH rule 2 x 295 [4,8]
+  CRUSH rule 2 x 296 [4,1]
+  CRUSH rule 2 x 297 [8,2]
+  CRUSH rule 2 x 298 [1,8]
+  CRUSH rule 2 x 299 [2,8]
+  CRUSH rule 2 x 300 [8,2]
+  CRUSH rule 2 x 301 [1,8]
+  CRUSH rule 2 x 302 [1,8]
+  CRUSH rule 2 x 303 [8,4]
+  CRUSH rule 2 x 304 [2,8]
+  CRUSH rule 2 x 305 [8,2]
+  CRUSH rule 2 x 306 [1,8]
+  CRUSH rule 2 x 307 [2,8]
+  CRUSH rule 2 x 308 [2,8]
+  CRUSH rule 2 x 309 [8,1]
+  CRUSH rule 2 x 310 [4,1]
+  CRUSH rule 2 x 311 [4,8]
+  CRUSH rule 2 x 312 [2,8]
+  CRUSH rule 2 x 313 [4,1]
+  CRUSH rule 2 x 314 [2,8]
+  CRUSH rule 2 x 315 [2,8]
+  CRUSH rule 2 x 316 [8,1]
+  CRUSH rule 2 x 317 [2,8]
+  CRUSH rule 2 x 318 [8,1]
+  CRUSH rule 2 x 319 [2,8]
+  CRUSH rule 2 x 320 [8,1]
+  CRUSH rule 2 x 321 [1,8]
+  CRUSH rule 2 x 322 [2,8]
+  CRUSH rule 2 x 323 [4,8]
+  CRUSH rule 2 x 324 [8,1]
+  CRUSH rule 2 x 325 [4,8]
+  CRUSH rule 2 x 326 [1,6]
+  CRUSH rule 2 x 327 [1,8]
+  CRUSH rule 2 x 328 [8,4]
+  CRUSH rule 2 x 329 [4,8]
+  CRUSH rule 2 x 330 [4,8]
+  CRUSH rule 2 x 331 [2,8]
+  CRUSH rule 2 x 332 [2,8]
+  CRUSH rule 2 x 333 [8,1]
+  CRUSH rule 2 x 334 [8,2]
+  CRUSH rule 2 x 335 [8,1]
+  CRUSH rule 2 x 336 [4,8]
+  CRUSH rule 2 x 337 [8,2]
+  CRUSH rule 2 x 338 [8,1]
+  CRUSH rule 2 x 339 [8,2]
+  CRUSH rule 2 x 340 [2,8]
+  CRUSH rule 2 x 341 [4,1]
+  CRUSH rule 2 x 342 [2,8]
+  CRUSH rule 2 x 343 [8,1]
+  CRUSH rule 2 x 344 [6,2]
+  CRUSH rule 2 x 345 [2,8]
+  CRUSH rule 2 x 346 [8,2]
+  CRUSH rule 2 x 347 [4,1]
+  CRUSH rule 2 x 348 [8,2]
+  CRUSH rule 2 x 349 [1,8]
+  CRUSH rule 2 x 350 [8,1]
+  CRUSH rule 2 x 351 [8,2]
+  CRUSH rule 2 x 352 [1,8]
+  CRUSH rule 2 x 353 [8,1]
+  CRUSH rule 2 x 354 [1,8]
+  CRUSH rule 2 x 355 [8,2]
+  CRUSH rule 2 x 356 [4,1]
+  CRUSH rule 2 x 357 [8,1]
+  CRUSH rule 2 x 358 [2,8]
+  CRUSH rule 2 x 359 [6,1]
+  CRUSH rule 2 x 360 [2,8]
+  CRUSH rule 2 x 361 [8,4]
+  CRUSH rule 2 x 362 [4,1]
+  CRUSH rule 2 x 363 [4,1]
+  CRUSH rule 2 x 364 [2,8]
+  CRUSH rule 2 x 365 [8,1]
+  CRUSH rule 2 x 366 [8,2]
+  CRUSH rule 2 x 367 [4,2]
+  CRUSH rule 2 x 368 [8,4]
+  CRUSH rule 2 x 369 [8,1]
+  CRUSH rule 2 x 370 [8,2]
+  CRUSH rule 2 x 371 [1,4]
+  CRUSH rule 2 x 372 [1,8]
+  CRUSH rule 2 x 373 [1,8]
+  CRUSH rule 2 x 374 [8,1]
+  CRUSH rule 2 x 375 [8,4]
+  CRUSH rule 2 x 376 [8,1]
+  CRUSH rule 2 x 377 [1,4]
+  CRUSH rule 2 x 378 [1,8]
+  CRUSH rule 2 x 379 [8,2]
+  CRUSH rule 2 x 380 [2,8]
+  CRUSH rule 2 x 381 [1,4]
+  CRUSH rule 2 x 382 [1,4]
+  CRUSH rule 2 x 383 [4,8]
+  CRUSH rule 2 x 384 [8,2]
+  CRUSH rule 2 x 385 [8,1]
+  CRUSH rule 2 x 386 [1,8]
+  CRUSH rule 2 x 387 [1,4]
+  CRUSH rule 2 x 388 [2,6]
+  CRUSH rule 2 x 389 [1,4]
+  CRUSH rule 2 x 390 [4,8]
+  CRUSH rule 2 x 391 [4,8]
+  CRUSH rule 2 x 392 [1,8]
+  CRUSH rule 2 x 393 [2,8]
+  CRUSH rule 2 x 394 [8,2]
+  CRUSH rule 2 x 395 [1,8]
+  CRUSH rule 2 x 396 [4,2]
+  CRUSH rule 2 x 397 [2,4]
+  CRUSH rule 2 x 398 [2,4]
+  CRUSH rule 2 x 399 [8,4]
+  CRUSH rule 2 x 400 [8,1]
+  CRUSH rule 2 x 401 [1,4]
+  CRUSH rule 2 x 402 [8,4]
+  CRUSH rule 2 x 403 [1,4]
+  CRUSH rule 2 x 404 [4,2]
+  CRUSH rule 2 x 405 [8,4]
+  CRUSH rule 2 x 406 [2,8]
+  CRUSH rule 2 x 407 [2,8]
+  CRUSH rule 2 x 408 [4,1]
+  CRUSH rule 2 x 409 [8,4]
+  CRUSH rule 2 x 410 [8,4]
+  CRUSH rule 2 x 411 [2,8]
+  CRUSH rule 2 x 412 [2,6]
+  CRUSH rule 2 x 413 [2,8]
+  CRUSH rule 2 x 414 [4,1]
+  CRUSH rule 2 x 415 [2,8]
+  CRUSH rule 2 x 416 [2,8]
+  CRUSH rule 2 x 417 [8,2]
+  CRUSH rule 2 x 418 [8,1]
+  CRUSH rule 2 x 419 [8,4]
+  CRUSH rule 2 x 420 [1,4]
+  CRUSH rule 2 x 421 [8,4]
+  CRUSH rule 2 x 422 [6,2]
+  CRUSH rule 2 x 423 [2,4]
+  CRUSH rule 2 x 424 [8,1]
+  CRUSH rule 2 x 425 [1,8]
+  CRUSH rule 2 x 426 [8,2]
+  CRUSH rule 2 x 427 [1,8]
+  CRUSH rule 2 x 428 [4,8]
+  CRUSH rule 2 x 429 [4,8]
+  CRUSH rule 2 x 430 [4,8]
+  CRUSH rule 2 x 431 [4,1]
+  CRUSH rule 2 x 432 [8,1]
+  CRUSH rule 2 x 433 [8,1]
+  CRUSH rule 2 x 434 [2,8]
+  CRUSH rule 2 x 435 [2,8]
+  CRUSH rule 2 x 436 [4,1]
+  CRUSH rule 2 x 437 [8,2]
+  CRUSH rule 2 x 438 [2,4]
+  CRUSH rule 2 x 439 [1,6]
+  CRUSH rule 2 x 440 [2,8]
+  CRUSH rule 2 x 441 [4,6]
+  CRUSH rule 2 x 442 [2,8]
+  CRUSH rule 2 x 443 [8,2]
+  CRUSH rule 2 x 444 [8,1]
+  CRUSH rule 2 x 445 [8,2]
+  CRUSH rule 2 x 446 [8,1]
+  CRUSH rule 2 x 447 [2,4]
+  CRUSH rule 2 x 448 [8,2]
+  CRUSH rule 2 x 449 [8,1]
+  CRUSH rule 2 x 450 [1,8]
+  CRUSH rule 2 x 451 [8,4]
+  CRUSH rule 2 x 452 [8,2]
+  CRUSH rule 2 x 453 [6,2]
+  CRUSH rule 2 x 454 [8,2]
+  CRUSH rule 2 x 455 [2,8]
+  CRUSH rule 2 x 456 [8,2]
+  CRUSH rule 2 x 457 [8,2]
+  CRUSH rule 2 x 458 [2,8]
+  CRUSH rule 2 x 459 [2,8]
+  CRUSH rule 2 x 460 [8,2]
+  CRUSH rule 2 x 461 [8,1]
+  CRUSH rule 2 x 462 [8,1]
+  CRUSH rule 2 x 463 [8,2]
+  CRUSH rule 2 x 464 [8,4]
+  CRUSH rule 2 x 465 [6,2]
+  CRUSH rule 2 x 466 [8,1]
+  CRUSH rule 2 x 467 [8,2]
+  CRUSH rule 2 x 468 [8,1]
+  CRUSH rule 2 x 469 [8,1]
+  CRUSH rule 2 x 470 [4,2]
+  CRUSH rule 2 x 471 [1,8]
+  CRUSH rule 2 x 472 [1,8]
+  CRUSH rule 2 x 473 [1,4]
+  CRUSH rule 2 x 474 [8,1]
+  CRUSH rule 2 x 475 [8,2]
+  CRUSH rule 2 x 476 [4,8]
+  CRUSH rule 2 x 477 [4,8]
+  CRUSH rule 2 x 478 [8,2]
+  CRUSH rule 2 x 479 [2,8]
+  CRUSH rule 2 x 480 [1,8]
+  CRUSH rule 2 x 481 [2,4]
+  CRUSH rule 2 x 482 [1,8]
+  CRUSH rule 2 x 483 [2,8]
+  CRUSH rule 2 x 484 [1,8]
+  CRUSH rule 2 x 485 [8,1]
+  CRUSH rule 2 x 486 [4,1]
+  CRUSH rule 2 x 487 [1,8]
+  CRUSH rule 2 x 488 [8,1]
+  CRUSH rule 2 x 489 [2,8]
+  CRUSH rule 2 x 490 [6,2]
+  CRUSH rule 2 x 491 [1,8]
+  CRUSH rule 2 x 492 [8,1]
+  CRUSH rule 2 x 493 [2,8]
+  CRUSH rule 2 x 494 [1,8]
+  CRUSH rule 2 x 495 [4,1]
+  CRUSH rule 2 x 496 [8,4]
+  CRUSH rule 2 x 497 [4,8]
+  CRUSH rule 2 x 498 [2,4]
+  CRUSH rule 2 x 499 [8,4]
+  CRUSH rule 2 x 500 [4,8]
+  CRUSH rule 2 x 501 [2,8]
+  CRUSH rule 2 x 502 [6,1]
+  CRUSH rule 2 x 503 [2,8]
+  CRUSH rule 2 x 504 [8,1]
+  CRUSH rule 2 x 505 [1,8]
+  CRUSH rule 2 x 506 [4,2]
+  CRUSH rule 2 x 507 [8,1]
+  CRUSH rule 2 x 508 [1,8]
+  CRUSH rule 2 x 509 [8,1]
+  CRUSH rule 2 x 510 [8,2]
+  CRUSH rule 2 x 511 [4,8]
+  CRUSH rule 2 x 512 [8,2]
+  CRUSH rule 2 x 513 [8,2]
+  CRUSH rule 2 x 514 [2,8]
+  CRUSH rule 2 x 515 [8,4]
+  CRUSH rule 2 x 516 [4,1]
+  CRUSH rule 2 x 517 [8,2]
+  CRUSH rule 2 x 518 [4,8]
+  CRUSH rule 2 x 519 [8,4]
+  CRUSH rule 2 x 520 [2,8]
+  CRUSH rule 2 x 521 [8,2]
+  CRUSH rule 2 x 522 [8,1]
+  CRUSH rule 2 x 523 [4,2]
+  CRUSH rule 2 x 524 [2,6]
+  CRUSH rule 2 x 525 [2,8]
+  CRUSH rule 2 x 526 [1,8]
+  CRUSH rule 2 x 527 [1,4]
+  CRUSH rule 2 x 528 [2,8]
+  CRUSH rule 2 x 529 [4,8]
+  CRUSH rule 2 x 530 [8,1]
+  CRUSH rule 2 x 531 [8,1]
+  CRUSH rule 2 x 532 [6,4]
+  CRUSH rule 2 x 533 [4,8]
+  CRUSH rule 2 x 534 [8,1]
+  CRUSH rule 2 x 535 [8,1]
+  CRUSH rule 2 x 536 [8,2]
+  CRUSH rule 2 x 537 [4,8]
+  CRUSH rule 2 x 538 [8,4]
+  CRUSH rule 2 x 539 [8,1]
+  CRUSH rule 2 x 540 [1,8]
+  CRUSH rule 2 x 541 [2,4]
+  CRUSH rule 2 x 542 [2,8]
+  CRUSH rule 2 x 543 [8,2]
+  CRUSH rule 2 x 544 [4,8]
+  CRUSH rule 2 x 545 [8,1]
+  CRUSH rule 2 x 546 [8,1]
+  CRUSH rule 2 x 547 [8,2]
+  CRUSH rule 2 x 548 [4,2]
+  CRUSH rule 2 x 549 [8,2]
+  CRUSH rule 2 x 550 [2,4]
+  CRUSH rule 2 x 551 [8,1]
+  CRUSH rule 2 x 552 [4,8]
+  CRUSH rule 2 x 553 [2,8]
+  CRUSH rule 2 x 554 [1,8]
+  CRUSH rule 2 x 555 [4,1]
+  CRUSH rule 2 x 556 [8,1]
+  CRUSH rule 2 x 557 [8,2]
+  CRUSH rule 2 x 558 [4,1]
+  CRUSH rule 2 x 559 [1,8]
+  CRUSH rule 2 x 560 [8,1]
+  CRUSH rule 2 x 561 [8,4]
+  CRUSH rule 2 x 562 [4,1]
+  CRUSH rule 2 x 563 [2,8]
+  CRUSH rule 2 x 564 [1,8]
+  CRUSH rule 2 x 565 [4,8]
+  CRUSH rule 2 x 566 [4,8]
+  CRUSH rule 2 x 567 [4,8]
+  CRUSH rule 2 x 568 [8,1]
+  CRUSH rule 2 x 569 [4,1]
+  CRUSH rule 2 x 570 [1,8]
+  CRUSH rule 2 x 571 [6,1]
+  CRUSH rule 2 x 572 [4,2]
+  CRUSH rule 2 x 573 [1,8]
+  CRUSH rule 2 x 574 [2,8]
+  CRUSH rule 2 x 575 [8,2]
+  CRUSH rule 2 x 576 [4,8]
+  CRUSH rule 2 x 577 [8,2]
+  CRUSH rule 2 x 578 [8,1]
+  CRUSH rule 2 x 579 [4,1]
+  CRUSH rule 2 x 580 [1,8]
+  CRUSH rule 2 x 581 [8,2]
+  CRUSH rule 2 x 582 [2,8]
+  CRUSH rule 2 x 583 [8,1]
+  CRUSH rule 2 x 584 [8,1]
+  CRUSH rule 2 x 585 [8,1]
+  CRUSH rule 2 x 586 [1,8]
+  CRUSH rule 2 x 587 [2,4]
+  CRUSH rule 2 x 588 [4,8]
+  CRUSH rule 2 x 589 [8,1]
+  CRUSH rule 2 x 590 [8,2]
+  CRUSH rule 2 x 591 [4,2]
+  CRUSH rule 2 x 592 [2,4]
+  CRUSH rule 2 x 593 [1,8]
+  CRUSH rule 2 x 594 [2,8]
+  CRUSH rule 2 x 595 [8,1]
+  CRUSH rule 2 x 596 [8,2]
+  CRUSH rule 2 x 597 [1,8]
+  CRUSH rule 2 x 598 [2,8]
+  CRUSH rule 2 x 599 [4,2]
+  CRUSH rule 2 x 600 [8,1]
+  CRUSH rule 2 x 601 [1,8]
+  CRUSH rule 2 x 602 [8,2]
+  CRUSH rule 2 x 603 [1,8]
+  CRUSH rule 2 x 604 [8,2]
+  CRUSH rule 2 x 605 [2,8]
+  CRUSH rule 2 x 606 [2,6]
+  CRUSH rule 2 x 607 [2,4]
+  CRUSH rule 2 x 608 [4,2]
+  CRUSH rule 2 x 609 [4,2]
+  CRUSH rule 2 x 610 [8,1]
+  CRUSH rule 2 x 611 [1,8]
+  CRUSH rule 2 x 612 [2,8]
+  CRUSH rule 2 x 613 [8,2]
+  CRUSH rule 2 x 614 [8,2]
+  CRUSH rule 2 x 615 [8,2]
+  CRUSH rule 2 x 616 [1,8]
+  CRUSH rule 2 x 617 [8,1]
+  CRUSH rule 2 x 618 [8,4]
+  CRUSH rule 2 x 619 [4,1]
+  CRUSH rule 2 x 620 [1,8]
+  CRUSH rule 2 x 621 [8,1]
+  CRUSH rule 2 x 622 [2,4]
+  CRUSH rule 2 x 623 [2,8]
+  CRUSH rule 2 x 624 [4,2]
+  CRUSH rule 2 x 625 [2,8]
+  CRUSH rule 2 x 626 [8,2]
+  CRUSH rule 2 x 627 [2,8]
+  CRUSH rule 2 x 628 [8,2]
+  CRUSH rule 2 x 629 [2,8]
+  CRUSH rule 2 x 630 [2,8]
+  CRUSH rule 2 x 631 [1,8]
+  CRUSH rule 2 x 632 [8,2]
+  CRUSH rule 2 x 633 [8,2]
+  CRUSH rule 2 x 634 [1,8]
+  CRUSH rule 2 x 635 [4,8]
+  CRUSH rule 2 x 636 [1,4]
+  CRUSH rule 2 x 637 [1,8]
+  CRUSH rule 2 x 638 [8,1]
+  CRUSH rule 2 x 639 [2,8]
+  CRUSH rule 2 x 640 [2,8]
+  CRUSH rule 2 x 641 [8,2]
+  CRUSH rule 2 x 642 [2,8]
+  CRUSH rule 2 x 643 [1,8]
+  CRUSH rule 2 x 644 [8,1]
+  CRUSH rule 2 x 645 [8,1]
+  CRUSH rule 2 x 646 [8,1]
+  CRUSH rule 2 x 647 [8,1]
+  CRUSH rule 2 x 648 [1,8]
+  CRUSH rule 2 x 649 [4,8]
+  CRUSH rule 2 x 650 [8,4]
+  CRUSH rule 2 x 651 [4,6]
+  CRUSH rule 2 x 652 [4,8]
+  CRUSH rule 2 x 653 [8,2]
+  CRUSH rule 2 x 654 [6,2]
+  CRUSH rule 2 x 655 [1,4]
+  CRUSH rule 2 x 656 [8,1]
+  CRUSH rule 2 x 657 [6,1]
+  CRUSH rule 2 x 658 [8,2]
+  CRUSH rule 2 x 659 [4,8]
+  CRUSH rule 2 x 660 [8,2]
+  CRUSH rule 2 x 661 [1,8]
+  CRUSH rule 2 x 662 [8,2]
+  CRUSH rule 2 x 663 [1,4]
+  CRUSH rule 2 x 664 [1,4]
+  CRUSH rule 2 x 665 [4,6]
+  CRUSH rule 2 x 666 [2,8]
+  CRUSH rule 2 x 667 [1,4]
+  CRUSH rule 2 x 668 [4,8]
+  CRUSH rule 2 x 669 [6,4]
+  CRUSH rule 2 x 670 [4,2]
+  CRUSH rule 2 x 671 [2,8]
+  CRUSH rule 2 x 672 [4,2]
+  CRUSH rule 2 x 673 [4,2]
+  CRUSH rule 2 x 674 [1,8]
+  CRUSH rule 2 x 675 [1,8]
+  CRUSH rule 2 x 676 [2,4]
+  CRUSH rule 2 x 677 [4,1]
+  CRUSH rule 2 x 678 [2,4]
+  CRUSH rule 2 x 679 [8,2]
+  CRUSH rule 2 x 680 [2,8]
+  CRUSH rule 2 x 681 [8,1]
+  CRUSH rule 2 x 682 [1,4]
+  CRUSH rule 2 x 683 [1,4]
+  CRUSH rule 2 x 684 [8,1]
+  CRUSH rule 2 x 685 [8,1]
+  CRUSH rule 2 x 686 [1,4]
+  CRUSH rule 2 x 687 [6,1]
+  CRUSH rule 2 x 688 [4,8]
+  CRUSH rule 2 x 689 [8,4]
+  CRUSH rule 2 x 690 [8,1]
+  CRUSH rule 2 x 691 [1,8]
+  CRUSH rule 2 x 692 [8,2]
+  CRUSH rule 2 x 693 [8,4]
+  CRUSH rule 2 x 694 [8,4]
+  CRUSH rule 2 x 695 [2,8]
+  CRUSH rule 2 x 696 [1,8]
+  CRUSH rule 2 x 697 [8,1]
+  CRUSH rule 2 x 698 [8,2]
+  CRUSH rule 2 x 699 [1,8]
+  CRUSH rule 2 x 700 [1,8]
+  CRUSH rule 2 x 701 [1,8]
+  CRUSH rule 2 x 702 [2,8]
+  CRUSH rule 2 x 703 [8,1]
+  CRUSH rule 2 x 704 [1,4]
+  CRUSH rule 2 x 705 [8,1]
+  CRUSH rule 2 x 706 [1,4]
+  CRUSH rule 2 x 707 [8,4]
+  CRUSH rule 2 x 708 [4,8]
+  CRUSH rule 2 x 709 [8,2]
+  CRUSH rule 2 x 710 [8,2]
+  CRUSH rule 2 x 711 [2,4]
+  CRUSH rule 2 x 712 [2,8]
+  CRUSH rule 2 x 713 [8,4]
+  CRUSH rule 2 x 714 [2,8]
+  CRUSH rule 2 x 715 [1,8]
+  CRUSH rule 2 x 716 [4,8]
+  CRUSH rule 2 x 717 [8,2]
+  CRUSH rule 2 x 718 [8,1]
+  CRUSH rule 2 x 719 [2,6]
+  CRUSH rule 2 x 720 [8,1]
+  CRUSH rule 2 x 721 [4,6]
+  CRUSH rule 2 x 722 [8,2]
+  CRUSH rule 2 x 723 [4,1]
+  CRUSH rule 2 x 724 [2,6]
+  CRUSH rule 2 x 725 [1,8]
+  CRUSH rule 2 x 726 [4,8]
+  CRUSH rule 2 x 727 [4,8]
+  CRUSH rule 2 x 728 [2,8]
+  CRUSH rule 2 x 729 [8,2]
+  CRUSH rule 2 x 730 [4,8]
+  CRUSH rule 2 x 731 [4,1]
+  CRUSH rule 2 x 732 [1,8]
+  CRUSH rule 2 x 733 [4,8]
+  CRUSH rule 2 x 734 [8,4]
+  CRUSH rule 2 x 735 [4,8]
+  CRUSH rule 2 x 736 [4,8]
+  CRUSH rule 2 x 737 [1,8]
+  CRUSH rule 2 x 738 [4,2]
+  CRUSH rule 2 x 739 [2,8]
+  CRUSH rule 2 x 740 [1,8]
+  CRUSH rule 2 x 741 [8,1]
+  CRUSH rule 2 x 742 [8,2]
+  CRUSH rule 2 x 743 [8,1]
+  CRUSH rule 2 x 744 [4,8]
+  CRUSH rule 2 x 745 [1,8]
+  CRUSH rule 2 x 746 [1,8]
+  CRUSH rule 2 x 747 [8,1]
+  CRUSH rule 2 x 748 [2,8]
+  CRUSH rule 2 x 749 [4,8]
+  CRUSH rule 2 x 750 [1,8]
+  CRUSH rule 2 x 751 [2,8]
+  CRUSH rule 2 x 752 [8,1]
+  CRUSH rule 2 x 753 [8,4]
+  CRUSH rule 2 x 754 [8,4]
+  CRUSH rule 2 x 755 [1,8]
+  CRUSH rule 2 x 756 [8,1]
+  CRUSH rule 2 x 757 [8,1]
+  CRUSH rule 2 x 758 [8,2]
+  CRUSH rule 2 x 759 [8,4]
+  CRUSH rule 2 x 760 [1,4]
+  CRUSH rule 2 x 761 [2,8]
+  CRUSH rule 2 x 762 [2,8]
+  CRUSH rule 2 x 763 [8,4]
+  CRUSH rule 2 x 764 [1,8]
+  CRUSH rule 2 x 765 [8,2]
+  CRUSH rule 2 x 766 [8,1]
+  CRUSH rule 2 x 767 [1,8]
+  CRUSH rule 2 x 768 [8,4]
+  CRUSH rule 2 x 769 [8,2]
+  CRUSH rule 2 x 770 [8,2]
+  CRUSH rule 2 x 771 [8,1]
+  CRUSH rule 2 x 772 [8,4]
+  CRUSH rule 2 x 773 [4,1]
+  CRUSH rule 2 x 774 [8,1]
+  CRUSH rule 2 x 775 [8,4]
+  CRUSH rule 2 x 776 [6,2]
+  CRUSH rule 2 x 777 [4,1]
+  CRUSH rule 2 x 778 [1,8]
+  CRUSH rule 2 x 779 [2,8]
+  CRUSH rule 2 x 780 [2,4]
+  CRUSH rule 2 x 781 [8,2]
+  CRUSH rule 2 x 782 [4,1]
+  CRUSH rule 2 x 783 [8,1]
+  CRUSH rule 2 x 784 [1,4]
+  CRUSH rule 2 x 785 [8,1]
+  CRUSH rule 2 x 786 [8,1]
+  CRUSH rule 2 x 787 [1,6]
+  CRUSH rule 2 x 788 [8,2]
+  CRUSH rule 2 x 789 [1,8]
+  CRUSH rule 2 x 790 [8,1]
+  CRUSH rule 2 x 791 [4,8]
+  CRUSH rule 2 x 792 [4,8]
+  CRUSH rule 2 x 793 [8,1]
+  CRUSH rule 2 x 794 [2,8]
+  CRUSH rule 2 x 795 [1,8]
+  CRUSH rule 2 x 796 [8,2]
+  CRUSH rule 2 x 797 [2,4]
+  CRUSH rule 2 x 798 [6,1]
+  CRUSH rule 2 x 799 [4,1]
+  CRUSH rule 2 x 800 [2,8]
+  CRUSH rule 2 x 801 [4,8]
+  CRUSH rule 2 x 802 [1,8]
+  CRUSH rule 2 x 803 [2,8]
+  CRUSH rule 2 x 804 [8,2]
+  CRUSH rule 2 x 805 [8,2]
+  CRUSH rule 2 x 806 [1,4]
+  CRUSH rule 2 x 807 [4,8]
+  CRUSH rule 2 x 808 [8,2]
+  CRUSH rule 2 x 809 [1,8]
+  CRUSH rule 2 x 810 [8,2]
+  CRUSH rule 2 x 811 [8,1]
+  CRUSH rule 2 x 812 [8,4]
+  CRUSH rule 2 x 813 [8,4]
+  CRUSH rule 2 x 814 [8,2]
+  CRUSH rule 2 x 815 [4,1]
+  CRUSH rule 2 x 816 [2,8]
+  CRUSH rule 2 x 817 [8,1]
+  CRUSH rule 2 x 818 [1,8]
+  CRUSH rule 2 x 819 [1,8]
+  CRUSH rule 2 x 820 [4,8]
+  CRUSH rule 2 x 821 [4,8]
+  CRUSH rule 2 x 822 [2,4]
+  CRUSH rule 2 x 823 [4,8]
+  CRUSH rule 2 x 824 [8,2]
+  CRUSH rule 2 x 825 [2,8]
+  CRUSH rule 2 x 826 [8,2]
+  CRUSH rule 2 x 827 [2,8]
+  CRUSH rule 2 x 828 [2,8]
+  CRUSH rule 2 x 829 [8,1]
+  CRUSH rule 2 x 830 [2,4]
+  CRUSH rule 2 x 831 [1,8]
+  CRUSH rule 2 x 832 [4,8]
+  CRUSH rule 2 x 833 [2,8]
+  CRUSH rule 2 x 834 [1,8]
+  CRUSH rule 2 x 835 [8,4]
+  CRUSH rule 2 x 836 [4,8]
+  CRUSH rule 2 x 837 [8,4]
+  CRUSH rule 2 x 838 [6,2]
+  CRUSH rule 2 x 839 [2,8]
+  CRUSH rule 2 x 840 [8,1]
+  CRUSH rule 2 x 841 [4,8]
+  CRUSH rule 2 x 842 [2,8]
+  CRUSH rule 2 x 843 [8,4]
+  CRUSH rule 2 x 844 [8,2]
+  CRUSH rule 2 x 845 [4,8]
+  CRUSH rule 2 x 846 [4,2]
+  CRUSH rule 2 x 847 [2,8]
+  CRUSH rule 2 x 848 [2,8]
+  CRUSH rule 2 x 849 [4,8]
+  CRUSH rule 2 x 850 [1,6]
+  CRUSH rule 2 x 851 [6,1]
+  CRUSH rule 2 x 852 [8,4]
+  CRUSH rule 2 x 853 [6,1]
+  CRUSH rule 2 x 854 [8,1]
+  CRUSH rule 2 x 855 [8,1]
+  CRUSH rule 2 x 856 [8,4]
+  CRUSH rule 2 x 857 [8,2]
+  CRUSH rule 2 x 858 [6,1]
+  CRUSH rule 2 x 859 [8,2]
+  CRUSH rule 2 x 860 [2,8]
+  CRUSH rule 2 x 861 [8,2]
+  CRUSH rule 2 x 862 [8,1]
+  CRUSH rule 2 x 863 [8,2]
+  CRUSH rule 2 x 864 [8,2]
+  CRUSH rule 2 x 865 [8,1]
+  CRUSH rule 2 x 866 [8,2]
+  CRUSH rule 2 x 867 [8,2]
+  CRUSH rule 2 x 868 [8,1]
+  CRUSH rule 2 x 869 [8,4]
+  CRUSH rule 2 x 870 [2,8]
+  CRUSH rule 2 x 871 [1,8]
+  CRUSH rule 2 x 872 [1,8]
+  CRUSH rule 2 x 873 [4,8]
+  CRUSH rule 2 x 874 [2,6]
+  CRUSH rule 2 x 875 [2,8]
+  CRUSH rule 2 x 876 [4,8]
+  CRUSH rule 2 x 877 [8,4]
+  CRUSH rule 2 x 878 [2,8]
+  CRUSH rule 2 x 879 [8,1]
+  CRUSH rule 2 x 880 [1,8]
+  CRUSH rule 2 x 881 [4,8]
+  CRUSH rule 2 x 882 [1,8]
+  CRUSH rule 2 x 883 [2,4]
+  CRUSH rule 2 x 884 [8,2]
+  CRUSH rule 2 x 885 [4,1]
+  CRUSH rule 2 x 886 [8,2]
+  CRUSH rule 2 x 887 [8,4]
+  CRUSH rule 2 x 888 [8,2]
+  CRUSH rule 2 x 889 [2,6]
+  CRUSH rule 2 x 890 [8,2]
+  CRUSH rule 2 x 891 [1,8]
+  CRUSH rule 2 x 892 [8,2]
+  CRUSH rule 2 x 893 [2,6]
+  CRUSH rule 2 x 894 [8,4]
+  CRUSH rule 2 x 895 [4,1]
+  CRUSH rule 2 x 896 [1,8]
+  CRUSH rule 2 x 897 [2,8]
+  CRUSH rule 2 x 898 [1,4]
+  CRUSH rule 2 x 899 [1,8]
+  CRUSH rule 2 x 900 [4,1]
+  CRUSH rule 2 x 901 [2,8]
+  CRUSH rule 2 x 902 [8,4]
+  CRUSH rule 2 x 903 [8,2]
+  CRUSH rule 2 x 904 [8,2]
+  CRUSH rule 2 x 905 [8,2]
+  CRUSH rule 2 x 906 [1,8]
+  CRUSH rule 2 x 907 [8,1]
+  CRUSH rule 2 x 908 [8,1]
+  CRUSH rule 2 x 909 [2,8]
+  CRUSH rule 2 x 910 [8,2]
+  CRUSH rule 2 x 911 [8,1]
+  CRUSH rule 2 x 912 [1,8]
+  CRUSH rule 2 x 913 [8,2]
+  CRUSH rule 2 x 914 [6,4]
+  CRUSH rule 2 x 915 [8,2]
+  CRUSH rule 2 x 916 [4,1]
+  CRUSH rule 2 x 917 [1,4]
+  CRUSH rule 2 x 918 [8,2]
+  CRUSH rule 2 x 919 [8,2]
+  CRUSH rule 2 x 920 [8,1]
+  CRUSH rule 2 x 921 [1,8]
+  CRUSH rule 2 x 922 [8,4]
+  CRUSH rule 2 x 923 [4,8]
+  CRUSH rule 2 x 924 [8,1]
+  CRUSH rule 2 x 925 [4,8]
+  CRUSH rule 2 x 926 [2,8]
+  CRUSH rule 2 x 927 [1,8]
+  CRUSH rule 2 x 928 [8,1]
+  CRUSH rule 2 x 929 [4,1]
+  CRUSH rule 2 x 930 [2,8]
+  CRUSH rule 2 x 931 [2,8]
+  CRUSH rule 2 x 932 [4,1]
+  CRUSH rule 2 x 933 [8,4]
+  CRUSH rule 2 x 934 [8,2]
+  CRUSH rule 2 x 935 [8,2]
+  CRUSH rule 2 x 936 [1,8]
+  CRUSH rule 2 x 937 [4,8]
+  CRUSH rule 2 x 938 [8,4]
+  CRUSH rule 2 x 939 [2,8]
+  CRUSH rule 2 x 940 [8,1]
+  CRUSH rule 2 x 941 [2,8]
+  CRUSH rule 2 x 942 [1,8]
+  CRUSH rule 2 x 943 [8,2]
+  CRUSH rule 2 x 944 [8,2]
+  CRUSH rule 2 x 945 [8,2]
+  CRUSH rule 2 x 946 [2,8]
+  CRUSH rule 2 x 947 [8,2]
+  CRUSH rule 2 x 948 [8,1]
+  CRUSH rule 2 x 949 [6,1]
+  CRUSH rule 2 x 950 [8,1]
+  CRUSH rule 2 x 951 [8,1]
+  CRUSH rule 2 x 952 [2,8]
+  CRUSH rule 2 x 953 [1,4]
+  CRUSH rule 2 x 954 [2,8]
+  CRUSH rule 2 x 955 [8,1]
+  CRUSH rule 2 x 956 [1,8]
+  CRUSH rule 2 x 957 [8,1]
+  CRUSH rule 2 x 958 [8,4]
+  CRUSH rule 2 x 959 [4,2]
+  CRUSH rule 2 x 960 [6,1]
+  CRUSH rule 2 x 961 [1,8]
+  CRUSH rule 2 x 962 [8,4]
+  CRUSH rule 2 x 963 [2,4]
+  CRUSH rule 2 x 964 [2,8]
+  CRUSH rule 2 x 965 [8,2]
+  CRUSH rule 2 x 966 [4,8]
+  CRUSH rule 2 x 967 [8,4]
+  CRUSH rule 2 x 968 [8,2]
+  CRUSH rule 2 x 969 [8,2]
+  CRUSH rule 2 x 970 [2,8]
+  CRUSH rule 2 x 971 [1,8]
+  CRUSH rule 2 x 972 [1,8]
+  CRUSH rule 2 x 973 [1,8]
+  CRUSH rule 2 x 974 [4,1]
+  CRUSH rule 2 x 975 [4,8]
+  CRUSH rule 2 x 976 [4,8]
+  CRUSH rule 2 x 977 [8,4]
+  CRUSH rule 2 x 978 [8,2]
+  CRUSH rule 2 x 979 [8,1]
+  CRUSH rule 2 x 980 [8,2]
+  CRUSH rule 2 x 981 [8,2]
+  CRUSH rule 2 x 982 [1,8]
+  CRUSH rule 2 x 983 [4,8]
+  CRUSH rule 2 x 984 [2,8]
+  CRUSH rule 2 x 985 [2,4]
+  CRUSH rule 2 x 986 [8,4]
+  CRUSH rule 2 x 987 [2,8]
+  CRUSH rule 2 x 988 [1,4]
+  CRUSH rule 2 x 989 [1,8]
+  CRUSH rule 2 x 990 [1,8]
+  CRUSH rule 2 x 991 [1,4]
+  CRUSH rule 2 x 992 [8,1]
+  CRUSH rule 2 x 993 [2,8]
+  CRUSH rule 2 x 994 [4,8]
+  CRUSH rule 2 x 995 [8,1]
+  CRUSH rule 2 x 996 [8,4]
+  CRUSH rule 2 x 997 [8,4]
+  CRUSH rule 2 x 998 [8,1]
+  CRUSH rule 2 x 999 [1,8]
+  CRUSH rule 2 x 1000 [8,4]
+  CRUSH rule 2 x 1001 [2,8]
+  CRUSH rule 2 x 1002 [1,8]
+  CRUSH rule 2 x 1003 [2,8]
+  CRUSH rule 2 x 1004 [8,1]
+  CRUSH rule 2 x 1005 [8,1]
+  CRUSH rule 2 x 1006 [1,8]
+  CRUSH rule 2 x 1007 [1,4]
+  CRUSH rule 2 x 1008 [1,8]
+  CRUSH rule 2 x 1009 [6,4]
+  CRUSH rule 2 x 1010 [1,8]
+  CRUSH rule 2 x 1011 [4,2]
+  CRUSH rule 2 x 1012 [1,8]
+  CRUSH rule 2 x 1013 [2,8]
+  CRUSH rule 2 x 1014 [2,8]
+  CRUSH rule 2 x 1015 [8,1]
+  CRUSH rule 2 x 1016 [2,4]
+  CRUSH rule 2 x 1017 [6,2]
+  CRUSH rule 2 x 1018 [4,1]
+  CRUSH rule 2 x 1019 [4,8]
+  CRUSH rule 2 x 1020 [1,8]
+  CRUSH rule 2 x 1021 [2,8]
+  CRUSH rule 2 x 1022 [1,8]
+  CRUSH rule 2 x 1023 [4,2]
+  rule 2 (chooseleaf) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 2 x 0 [2,4,8]
+  CRUSH rule 2 x 1 [2,8,4]
+  CRUSH rule 2 x 2 [1,8]
+  CRUSH rule 2 x 3 [8,1]
+  CRUSH rule 2 x 4 [4,2,6]
+  CRUSH rule 2 x 5 [8,2]
+  CRUSH rule 2 x 6 [2,8,4]
+  CRUSH rule 2 x 7 [4,8,2]
+  CRUSH rule 2 x 8 [4,8,1]
+  CRUSH rule 2 x 9 [2,4,8]
+  CRUSH rule 2 x 10 [2,8]
+  CRUSH rule 2 x 11 [2,8]
+  CRUSH rule 2 x 12 [2,8]
+  CRUSH rule 2 x 13 [4,8,1]
+  CRUSH rule 2 x 14 [8,2]
+  CRUSH rule 2 x 15 [8,2]
+  CRUSH rule 2 x 16 [8,2]
+  CRUSH rule 2 x 17 [4,1,8]
+  CRUSH rule 2 x 18 [1,8]
+  CRUSH rule 2 x 19 [8,4,2]
+  CRUSH rule 2 x 20 [2,8]
+  CRUSH rule 2 x 21 [8,2]
+  CRUSH rule 2 x 22 [8,1]
+  CRUSH rule 2 x 23 [4,8,2]
+  CRUSH rule 2 x 24 [1,8,4]
+  CRUSH rule 2 x 25 [4,8,1]
+  CRUSH rule 2 x 26 [2,8,4]
+  CRUSH rule 2 x 27 [4,1,8]
+  CRUSH rule 2 x 28 [8,2]
+  CRUSH rule 2 x 29 [8,4,2]
+  CRUSH rule 2 x 30 [4,8,2]
+  CRUSH rule 2 x 31 [8,1]
+  CRUSH rule 2 x 32 [6,1]
+  CRUSH rule 2 x 33 [2,8]
+  CRUSH rule 2 x 34 [2,8]
+  CRUSH rule 2 x 35 [1,8,4]
+  CRUSH rule 2 x 36 [8,2]
+  CRUSH rule 2 x 37 [1,8]
+  CRUSH rule 2 x 38 [4,8,2]
+  CRUSH rule 2 x 39 [8,2]
+  CRUSH rule 2 x 40 [8,2,4]
+  CRUSH rule 2 x 41 [2,8,4]
+  CRUSH rule 2 x 42 [8,2]
+  CRUSH rule 2 x 43 [1,8]
+  CRUSH rule 2 x 44 [1,8,4]
+  CRUSH rule 2 x 45 [8,2,4]
+  CRUSH rule 2 x 46 [2,8]
+  CRUSH rule 2 x 47 [4,2,8]
+  CRUSH rule 2 x 48 [8,1]
+  CRUSH rule 2 x 49 [8,2]
+  CRUSH rule 2 x 50 [4,1,8]
+  CRUSH rule 2 x 51 [8,2]
+  CRUSH rule 2 x 52 [8,1,4]
+  CRUSH rule 2 x 53 [4,8,2]
+  CRUSH rule 2 x 54 [8,4,1]
+  CRUSH rule 2 x 55 [8,2]
+  CRUSH rule 2 x 56 [8,4,2]
+  CRUSH rule 2 x 57 [8,1]
+  CRUSH rule 2 x 58 [1,8]
+  CRUSH rule 2 x 59 [2,8]
+  CRUSH rule 2 x 60 [4,2,8]
+  CRUSH rule 2 x 61 [4,8,2]
+  CRUSH rule 2 x 62 [8,1]
+  CRUSH rule 2 x 63 [8,2]
+  CRUSH rule 2 x 64 [4,2,8]
+  CRUSH rule 2 x 65 [8,4,2]
+  CRUSH rule 2 x 66 [4,8,2]
+  CRUSH rule 2 x 67 [4,2,8]
+  CRUSH rule 2 x 68 [1,8]
+  CRUSH rule 2 x 69 [2,8]
+  CRUSH rule 2 x 70 [8,2]
+  CRUSH rule 2 x 71 [2,8,4]
+  CRUSH rule 2 x 72 [8,1,4]
+  CRUSH rule 2 x 73 [2,8]
+  CRUSH rule 2 x 74 [1,8]
+  CRUSH rule 2 x 75 [4,2,8]
+  CRUSH rule 2 x 76 [4,1,6]
+  CRUSH rule 2 x 77 [8,2,4]
+  CRUSH rule 2 x 78 [1,6]
+  CRUSH rule 2 x 79 [4,1,8]
+  CRUSH rule 2 x 80 [2,4,8]
+  CRUSH rule 2 x 81 [2,8]
+  CRUSH rule 2 x 82 [6,1]
+  CRUSH rule 2 x 83 [2,8]
+  CRUSH rule 2 x 84 [8,2]
+  CRUSH rule 2 x 85 [4,8,1]
+  CRUSH rule 2 x 86 [2,8]
+  CRUSH rule 2 x 87 [2,8,4]
+  CRUSH rule 2 x 88 [1,6]
+  CRUSH rule 2 x 89 [1,8]
+  CRUSH rule 2 x 90 [8,4,1]
+  CRUSH rule 2 x 91 [4,8,1]
+  CRUSH rule 2 x 92 [1,8]
+  CRUSH rule 2 x 93 [8,4,2]
+  CRUSH rule 2 x 94 [1,8]
+  CRUSH rule 2 x 95 [8,1]
+  CRUSH rule 2 x 96 [8,2]
+  CRUSH rule 2 x 97 [8,1]
+  CRUSH rule 2 x 98 [2,8]
+  CRUSH rule 2 x 99 [2,8]
+  CRUSH rule 2 x 100 [1,8,4]
+  CRUSH rule 2 x 101 [8,1]
+  CRUSH rule 2 x 102 [2,8]
+  CRUSH rule 2 x 103 [8,2]
+  CRUSH rule 2 x 104 [8,4,1]
+  CRUSH rule 2 x 105 [2,4,8]
+  CRUSH rule 2 x 106 [1,8,4]
+  CRUSH rule 2 x 107 [1,8]
+  CRUSH rule 2 x 108 [8,2]
+  CRUSH rule 2 x 109 [1,4,8]
+  CRUSH rule 2 x 110 [4,2,8]
+  CRUSH rule 2 x 111 [2,4,8]
+  CRUSH rule 2 x 112 [2,8]
+  CRUSH rule 2 x 113 [8,2]
+  CRUSH rule 2 x 114 [8,4,1]
+  CRUSH rule 2 x 115 [8,2,4]
+  CRUSH rule 2 x 116 [1,8]
+  CRUSH rule 2 x 117 [6,1]
+  CRUSH rule 2 x 118 [2,8]
+  CRUSH rule 2 x 119 [8,1]
+  CRUSH rule 2 x 120 [2,4,8]
+  CRUSH rule 2 x 121 [2,8,4]
+  CRUSH rule 2 x 122 [8,1]
+  CRUSH rule 2 x 123 [2,8]
+  CRUSH rule 2 x 124 [2,8]
+  CRUSH rule 2 x 125 [1,8,4]
+  CRUSH rule 2 x 126 [1,8]
+  CRUSH rule 2 x 127 [4,8,2]
+  CRUSH rule 2 x 128 [8,2]
+  CRUSH rule 2 x 129 [2,4,8]
+  CRUSH rule 2 x 130 [4,8,2]
+  CRUSH rule 2 x 131 [1,4,8]
+  CRUSH rule 2 x 132 [1,8]
+  CRUSH rule 2 x 133 [8,1]
+  CRUSH rule 2 x 134 [1,8,4]
+  CRUSH rule 2 x 135 [4,8,2]
+  CRUSH rule 2 x 136 [2,4,8]
+  CRUSH rule 2 x 137 [8,4,2]
+  CRUSH rule 2 x 138 [8,4,2]
+  CRUSH rule 2 x 139 [4,2,8]
+  CRUSH rule 2 x 140 [1,8,4]
+  CRUSH rule 2 x 141 [8,2]
+  CRUSH rule 2 x 142 [4,1,8]
+  CRUSH rule 2 x 143 [4,8,1]
+  CRUSH rule 2 x 144 [8,1]
+  CRUSH rule 2 x 145 [8,1]
+  CRUSH rule 2 x 146 [2,8]
+  CRUSH rule 2 x 147 [2,8,4]
+  CRUSH rule 2 x 148 [4,1,8]
+  CRUSH rule 2 x 149 [4,8,1]
+  CRUSH rule 2 x 150 [1,8]
+  CRUSH rule 2 x 151 [1,8]
+  CRUSH rule 2 x 152 [8,2]
+  CRUSH rule 2 x 153 [8,4,2]
+  CRUSH rule 2 x 154 [4,2,8]
+  CRUSH rule 2 x 155 [4,8,2]
+  CRUSH rule 2 x 156 [4,2,8]
+  CRUSH rule 2 x 157 [1,8]
+  CRUSH rule 2 x 158 [2,8,4]
+  CRUSH rule 2 x 159 [8,2,4]
+  CRUSH rule 2 x 160 [2,8,4]
+  CRUSH rule 2 x 161 [1,4,8]
+  CRUSH rule 2 x 162 [1,8]
+  CRUSH rule 2 x 163 [4,8,1]
+  CRUSH rule 2 x 164 [8,1]
+  CRUSH rule 2 x 165 [8,2,4]
+  CRUSH rule 2 x 166 [2,8]
+  CRUSH rule 2 x 167 [1,8,4]
+  CRUSH rule 2 x 168 [4,2,8]
+  CRUSH rule 2 x 169 [2,8,4]
+  CRUSH rule 2 x 170 [1,8]
+  CRUSH rule 2 x 171 [8,4,2]
+  CRUSH rule 2 x 172 [1,8]
+  CRUSH rule 2 x 173 [8,4,1]
+  CRUSH rule 2 x 174 [1,6]
+  CRUSH rule 2 x 175 [8,1]
+  CRUSH rule 2 x 176 [2,8]
+  CRUSH rule 2 x 177 [1,8]
+  CRUSH rule 2 x 178 [4,2,8]
+  CRUSH rule 2 x 179 [1,8]
+  CRUSH rule 2 x 180 [8,1]
+  CRUSH rule 2 x 181 [8,2,4]
+  CRUSH rule 2 x 182 [8,1]
+  CRUSH rule 2 x 183 [8,4,1]
+  CRUSH rule 2 x 184 [4,8,2]
+  CRUSH rule 2 x 185 [8,1,4]
+  CRUSH rule 2 x 186 [2,4,8]
+  CRUSH rule 2 x 187 [1,8]
+  CRUSH rule 2 x 188 [1,8,4]
+  CRUSH rule 2 x 189 [1,8,4]
+  CRUSH rule 2 x 190 [1,8]
+  CRUSH rule 2 x 191 [8,1,4]
+  CRUSH rule 2 x 192 [4,1,8]
+  CRUSH rule 2 x 193 [4,2,8]
+  CRUSH rule 2 x 194 [1,8]
+  CRUSH rule 2 x 195 [8,4,1]
+  CRUSH rule 2 x 196 [8,2]
+  CRUSH rule 2 x 197 [8,4,2]
+  CRUSH rule 2 x 198 [2,8]
+  CRUSH rule 2 x 199 [1,4,8]
+  CRUSH rule 2 x 200 [1,8]
+  CRUSH rule 2 x 201 [8,1,4]
+  CRUSH rule 2 x 202 [8,1]
+  CRUSH rule 2 x 203 [8,1]
+  CRUSH rule 2 x 204 [2,4,8]
+  CRUSH rule 2 x 205 [1,8]
+  CRUSH rule 2 x 206 [1,8,4]
+  CRUSH rule 2 x 207 [2,8]
+  CRUSH rule 2 x 208 [8,1]
+  CRUSH rule 2 x 209 [1,8]
+  CRUSH rule 2 x 210 [1,4,8]
+  CRUSH rule 2 x 211 [4,2,8]
+  CRUSH rule 2 x 212 [8,1]
+  CRUSH rule 2 x 213 [8,4,2]
+  CRUSH rule 2 x 214 [8,2]
+  CRUSH rule 2 x 215 [8,1]
+  CRUSH rule 2 x 216 [2,8]
+  CRUSH rule 2 x 217 [1,8,4]
+  CRUSH rule 2 x 218 [2,8]
+  CRUSH rule 2 x 219 [8,2]
+  CRUSH rule 2 x 220 [4,8,1]
+  CRUSH rule 2 x 221 [8,1]
+  CRUSH rule 2 x 222 [8,1]
+  CRUSH rule 2 x 223 [1,8]
+  CRUSH rule 2 x 224 [1,4,8]
+  CRUSH rule 2 x 225 [8,2]
+  CRUSH rule 2 x 226 [8,2,4]
+  CRUSH rule 2 x 227 [4,1,8]
+  CRUSH rule 2 x 228 [8,2]
+  CRUSH rule 2 x 229 [4,8,2]
+  CRUSH rule 2 x 230 [4,8,2]
+  CRUSH rule 2 x 231 [4,8,2]
+  CRUSH rule 2 x 232 [2,8,4]
+  CRUSH rule 2 x 233 [2,8]
+  CRUSH rule 2 x 234 [1,8]
+  CRUSH rule 2 x 235 [4,8,1]
+  CRUSH rule 2 x 236 [2,6]
+  CRUSH rule 2 x 237 [4,8,1]
+  CRUSH rule 2 x 238 [2,8]
+  CRUSH rule 2 x 239 [8,1]
+  CRUSH rule 2 x 240 [4,8,2]
+  CRUSH rule 2 x 241 [1,8]
+  CRUSH rule 2 x 242 [8,2]
+  CRUSH rule 2 x 243 [8,2]
+  CRUSH rule 2 x 244 [4,8,2]
+  CRUSH rule 2 x 245 [8,1]
+  CRUSH rule 2 x 246 [1,8]
+  CRUSH rule 2 x 247 [8,2]
+  CRUSH rule 2 x 248 [8,2,4]
+  CRUSH rule 2 x 249 [2,8]
+  CRUSH rule 2 x 250 [2,4,6]
+  CRUSH rule 2 x 251 [2,8]
+  CRUSH rule 2 x 252 [4,8,1]
+  CRUSH rule 2 x 253 [2,8]
+  CRUSH rule 2 x 254 [4,2,8]
+  CRUSH rule 2 x 255 [1,8]
+  CRUSH rule 2 x 256 [4,8,1]
+  CRUSH rule 2 x 257 [2,8,4]
+  CRUSH rule 2 x 258 [4,2,8]
+  CRUSH rule 2 x 259 [6,2]
+  CRUSH rule 2 x 260 [8,2]
+  CRUSH rule 2 x 261 [8,1]
+  CRUSH rule 2 x 262 [8,1]
+  CRUSH rule 2 x 263 [8,1,4]
+  CRUSH rule 2 x 264 [8,2]
+  CRUSH rule 2 x 265 [8,2]
+  CRUSH rule 2 x 266 [8,2,4]
+  CRUSH rule 2 x 267 [2,8]
+  CRUSH rule 2 x 268 [1,8]
+  CRUSH rule 2 x 269 [1,8,4]
+  CRUSH rule 2 x 270 [4,1,8]
+  CRUSH rule 2 x 271 [8,4,1]
+  CRUSH rule 2 x 272 [2,8,4]
+  CRUSH rule 2 x 273 [4,1,8]
+  CRUSH rule 2 x 274 [8,4,1]
+  CRUSH rule 2 x 275 [4,8,1]
+  CRUSH rule 2 x 276 [8,1,4]
+  CRUSH rule 2 x 277 [8,1]
+  CRUSH rule 2 x 278 [8,1,4]
+  CRUSH rule 2 x 279 [8,4,2]
+  CRUSH rule 2 x 280 [2,8,4]
+  CRUSH rule 2 x 281 [8,2]
+  CRUSH rule 2 x 282 [2,8]
+  CRUSH rule 2 x 283 [8,2]
+  CRUSH rule 2 x 284 [8,2]
+  CRUSH rule 2 x 285 [4,8,2]
+  CRUSH rule 2 x 286 [2,8,4]
+  CRUSH rule 2 x 287 [1,8]
+  CRUSH rule 2 x 288 [8,1,4]
+  CRUSH rule 2 x 289 [4,8,2]
+  CRUSH rule 2 x 290 [1,4,8]
+  CRUSH rule 2 x 291 [1,4,8]
+  CRUSH rule 2 x 292 [8,2,4]
+  CRUSH rule 2 x 293 [8,1]
+  CRUSH rule 2 x 294 [8,4,1]
+  CRUSH rule 2 x 295 [4,8,2]
+  CRUSH rule 2 x 296 [4,1,8]
+  CRUSH rule 2 x 297 [8,2,4]
+  CRUSH rule 2 x 298 [1,8]
+  CRUSH rule 2 x 299 [2,8]
+  CRUSH rule 2 x 300 [8,2]
+  CRUSH rule 2 x 301 [1,8]
+  CRUSH rule 2 x 302 [1,8]
+  CRUSH rule 2 x 303 [8,4,1]
+  CRUSH rule 2 x 304 [2,8]
+  CRUSH rule 2 x 305 [8,2]
+  CRUSH rule 2 x 306 [1,8]
+  CRUSH rule 2 x 307 [2,8]
+  CRUSH rule 2 x 308 [2,8,4]
+  CRUSH rule 2 x 309 [8,1]
+  CRUSH rule 2 x 310 [4,1,6]
+  CRUSH rule 2 x 311 [4,8,1]
+  CRUSH rule 2 x 312 [2,8]
+  CRUSH rule 2 x 313 [4,1,8]
+  CRUSH rule 2 x 314 [2,8]
+  CRUSH rule 2 x 315 [2,8]
+  CRUSH rule 2 x 316 [8,1]
+  CRUSH rule 2 x 317 [2,8]
+  CRUSH rule 2 x 318 [8,1]
+  CRUSH rule 2 x 319 [2,8]
+  CRUSH rule 2 x 320 [8,1]
+  CRUSH rule 2 x 321 [1,8]
+  CRUSH rule 2 x 322 [2,8,4]
+  CRUSH rule 2 x 323 [4,8,1]
+  CRUSH rule 2 x 324 [8,1,4]
+  CRUSH rule 2 x 325 [4,8,2]
+  CRUSH rule 2 x 326 [1,6]
+  CRUSH rule 2 x 327 [1,8]
+  CRUSH rule 2 x 328 [8,4,1]
+  CRUSH rule 2 x 329 [4,8,2]
+  CRUSH rule 2 x 330 [4,8,2]
+  CRUSH rule 2 x 331 [2,8]
+  CRUSH rule 2 x 332 [2,8]
+  CRUSH rule 2 x 333 [8,1]
+  CRUSH rule 2 x 334 [8,2]
+  CRUSH rule 2 x 335 [8,1]
+  CRUSH rule 2 x 336 [4,8,2]
+  CRUSH rule 2 x 337 [8,2,4]
+  CRUSH rule 2 x 338 [8,1]
+  CRUSH rule 2 x 339 [8,2]
+  CRUSH rule 2 x 340 [2,8,4]
+  CRUSH rule 2 x 341 [4,1,8]
+  CRUSH rule 2 x 342 [2,8,4]
+  CRUSH rule 2 x 343 [8,1]
+  CRUSH rule 2 x 344 [6,2,4]
+  CRUSH rule 2 x 345 [2,8]
+  CRUSH rule 2 x 346 [8,2,4]
+  CRUSH rule 2 x 347 [4,1,8]
+  CRUSH rule 2 x 348 [8,2,4]
+  CRUSH rule 2 x 349 [1,8]
+  CRUSH rule 2 x 350 [8,1]
+  CRUSH rule 2 x 351 [8,2]
+  CRUSH rule 2 x 352 [1,8,4]
+  CRUSH rule 2 x 353 [8,1]
+  CRUSH rule 2 x 354 [1,8]
+  CRUSH rule 2 x 355 [8,2]
+  CRUSH rule 2 x 356 [4,1,8]
+  CRUSH rule 2 x 357 [8,1,4]
+  CRUSH rule 2 x 358 [2,8,4]
+  CRUSH rule 2 x 359 [6,1,4]
+  CRUSH rule 2 x 360 [2,8]
+  CRUSH rule 2 x 361 [8,4,1]
+  CRUSH rule 2 x 362 [4,1,6]
+  CRUSH rule 2 x 363 [4,1,8]
+  CRUSH rule 2 x 364 [2,8]
+  CRUSH rule 2 x 365 [8,1]
+  CRUSH rule 2 x 366 [8,2]
+  CRUSH rule 2 x 367 [4,2,8]
+  CRUSH rule 2 x 368 [8,4,1]
+  CRUSH rule 2 x 369 [8,1]
+  CRUSH rule 2 x 370 [8,2]
+  CRUSH rule 2 x 371 [1,4,8]
+  CRUSH rule 2 x 372 [1,8]
+  CRUSH rule 2 x 373 [1,8]
+  CRUSH rule 2 x 374 [8,1]
+  CRUSH rule 2 x 375 [8,4,1]
+  CRUSH rule 2 x 376 [8,1,4]
+  CRUSH rule 2 x 377 [1,4,8]
+  CRUSH rule 2 x 378 [1,8]
+  CRUSH rule 2 x 379 [8,2]
+  CRUSH rule 2 x 380 [2,8]
+  CRUSH rule 2 x 381 [1,4,8]
+  CRUSH rule 2 x 382 [1,4,8]
+  CRUSH rule 2 x 383 [4,8,2]
+  CRUSH rule 2 x 384 [8,2,4]
+  CRUSH rule 2 x 385 [8,1]
+  CRUSH rule 2 x 386 [1,8]
+  CRUSH rule 2 x 387 [1,4,8]
+  CRUSH rule 2 x 388 [2,6]
+  CRUSH rule 2 x 389 [1,4,8]
+  CRUSH rule 2 x 390 [4,8,1]
+  CRUSH rule 2 x 391 [4,8,2]
+  CRUSH rule 2 x 392 [1,8,4]
+  CRUSH rule 2 x 393 [2,8]
+  CRUSH rule 2 x 394 [8,2]
+  CRUSH rule 2 x 395 [1,8]
+  CRUSH rule 2 x 396 [4,2,8]
+  CRUSH rule 2 x 397 [2,4,8]
+  CRUSH rule 2 x 398 [2,4,8]
+  CRUSH rule 2 x 399 [8,4,2]
+  CRUSH rule 2 x 400 [8,1,4]
+  CRUSH rule 2 x 401 [1,4,8]
+  CRUSH rule 2 x 402 [8,4,2]
+  CRUSH rule 2 x 403 [1,4,8]
+  CRUSH rule 2 x 404 [4,2,8]
+  CRUSH rule 2 x 405 [8,4,2]
+  CRUSH rule 2 x 406 [2,8]
+  CRUSH rule 2 x 407 [2,8,4]
+  CRUSH rule 2 x 408 [4,1,8]
+  CRUSH rule 2 x 409 [8,4,1]
+  CRUSH rule 2 x 410 [8,4,2]
+  CRUSH rule 2 x 411 [2,8,4]
+  CRUSH rule 2 x 412 [2,6]
+  CRUSH rule 2 x 413 [2,8]
+  CRUSH rule 2 x 414 [4,1,6]
+  CRUSH rule 2 x 415 [2,8]
+  CRUSH rule 2 x 416 [2,8]
+  CRUSH rule 2 x 417 [8,2]
+  CRUSH rule 2 x 418 [8,1,4]
+  CRUSH rule 2 x 419 [8,4,1]
+  CRUSH rule 2 x 420 [1,4,8]
+  CRUSH rule 2 x 421 [8,4,1]
+  CRUSH rule 2 x 422 [6,2]
+  CRUSH rule 2 x 423 [2,4,8]
+  CRUSH rule 2 x 424 [8,1]
+  CRUSH rule 2 x 425 [1,8]
+  CRUSH rule 2 x 426 [8,2]
+  CRUSH rule 2 x 427 [1,8]
+  CRUSH rule 2 x 428 [4,8,1]
+  CRUSH rule 2 x 429 [4,8,2]
+  CRUSH rule 2 x 430 [4,8,2]
+  CRUSH rule 2 x 431 [4,1,8]
+  CRUSH rule 2 x 432 [8,1]
+  CRUSH rule 2 x 433 [8,1]
+  CRUSH rule 2 x 434 [2,8]
+  CRUSH rule 2 x 435 [2,8]
+  CRUSH rule 2 x 436 [4,1,8]
+  CRUSH rule 2 x 437 [8,2]
+  CRUSH rule 2 x 438 [2,4,8]
+  CRUSH rule 2 x 439 [1,6]
+  CRUSH rule 2 x 440 [2,8]
+  CRUSH rule 2 x 441 [4,6,2]
+  CRUSH rule 2 x 442 [2,8]
+  CRUSH rule 2 x 443 [8,2,4]
+  CRUSH rule 2 x 444 [8,1]
+  CRUSH rule 2 x 445 [8,2]
+  CRUSH rule 2 x 446 [8,1]
+  CRUSH rule 2 x 447 [2,4,8]
+  CRUSH rule 2 x 448 [8,2,4]
+  CRUSH rule 2 x 449 [8,1]
+  CRUSH rule 2 x 450 [1,8]
+  CRUSH rule 2 x 451 [8,4,2]
+  CRUSH rule 2 x 452 [8,2]
+  CRUSH rule 2 x 453 [6,2]
+  CRUSH rule 2 x 454 [8,2]
+  CRUSH rule 2 x 455 [2,8,4]
+  CRUSH rule 2 x 456 [8,2]
+  CRUSH rule 2 x 457 [8,2]
+  CRUSH rule 2 x 458 [2,8]
+  CRUSH rule 2 x 459 [2,8,4]
+  CRUSH rule 2 x 460 [8,2]
+  CRUSH rule 2 x 461 [8,1]
+  CRUSH rule 2 x 462 [8,1]
+  CRUSH rule 2 x 463 [8,2]
+  CRUSH rule 2 x 464 [8,4,2]
+  CRUSH rule 2 x 465 [6,2,4]
+  CRUSH rule 2 x 466 [8,1]
+  CRUSH rule 2 x 467 [8,2]
+  CRUSH rule 2 x 468 [8,1,4]
+  CRUSH rule 2 x 469 [8,1]
+  CRUSH rule 2 x 470 [4,2,6]
+  CRUSH rule 2 x 471 [1,8]
+  CRUSH rule 2 x 472 [1,8]
+  CRUSH rule 2 x 473 [1,4,8]
+  CRUSH rule 2 x 474 [8,1]
+  CRUSH rule 2 x 475 [8,2,4]
+  CRUSH rule 2 x 476 [4,8,1]
+  CRUSH rule 2 x 477 [4,8,2]
+  CRUSH rule 2 x 478 [8,2,4]
+  CRUSH rule 2 x 479 [2,8]
+  CRUSH rule 2 x 480 [1,8]
+  CRUSH rule 2 x 481 [2,4,6]
+  CRUSH rule 2 x 482 [1,8]
+  CRUSH rule 2 x 483 [2,8,4]
+  CRUSH rule 2 x 484 [1,8]
+  CRUSH rule 2 x 485 [8,1]
+  CRUSH rule 2 x 486 [4,1,8]
+  CRUSH rule 2 x 487 [1,8]
+  CRUSH rule 2 x 488 [8,1]
+  CRUSH rule 2 x 489 [2,8]
+  CRUSH rule 2 x 490 [6,2]
+  CRUSH rule 2 x 491 [1,8]
+  CRUSH rule 2 x 492 [8,1]
+  CRUSH rule 2 x 493 [2,8]
+  CRUSH rule 2 x 494 [1,8]
+  CRUSH rule 2 x 495 [4,1,8]
+  CRUSH rule 2 x 496 [8,4,1]
+  CRUSH rule 2 x 497 [4,8,1]
+  CRUSH rule 2 x 498 [2,4,8]
+  CRUSH rule 2 x 499 [8,4,2]
+  CRUSH rule 2 x 500 [4,8,2]
+  CRUSH rule 2 x 501 [2,8]
+  CRUSH rule 2 x 502 [6,1]
+  CRUSH rule 2 x 503 [2,8]
+  CRUSH rule 2 x 504 [8,1]
+  CRUSH rule 2 x 505 [1,8]
+  CRUSH rule 2 x 506 [4,2,8]
+  CRUSH rule 2 x 507 [8,1,4]
+  CRUSH rule 2 x 508 [1,8]
+  CRUSH rule 2 x 509 [8,1]
+  CRUSH rule 2 x 510 [8,2]
+  CRUSH rule 2 x 511 [4,8,2]
+  CRUSH rule 2 x 512 [8,2]
+  CRUSH rule 2 x 513 [8,2]
+  CRUSH rule 2 x 514 [2,8]
+  CRUSH rule 2 x 515 [8,4,1]
+  CRUSH rule 2 x 516 [4,1,8]
+  CRUSH rule 2 x 517 [8,2]
+  CRUSH rule 2 x 518 [4,8,1]
+  CRUSH rule 2 x 519 [8,4,1]
+  CRUSH rule 2 x 520 [2,8,4]
+  CRUSH rule 2 x 521 [8,2,4]
+  CRUSH rule 2 x 522 [8,1,4]
+  CRUSH rule 2 x 523 [4,2,8]
+  CRUSH rule 2 x 524 [2,6]
+  CRUSH rule 2 x 525 [2,8]
+  CRUSH rule 2 x 526 [1,8]
+  CRUSH rule 2 x 527 [1,4,6]
+  CRUSH rule 2 x 528 [2,8]
+  CRUSH rule 2 x 529 [4,8,2]
+  CRUSH rule 2 x 530 [8,1]
+  CRUSH rule 2 x 531 [8,1,4]
+  CRUSH rule 2 x 532 [6,4,1]
+  CRUSH rule 2 x 533 [4,8,2]
+  CRUSH rule 2 x 534 [8,1]
+  CRUSH rule 2 x 535 [8,1]
+  CRUSH rule 2 x 536 [8,2]
+  CRUSH rule 2 x 537 [4,8,2]
+  CRUSH rule 2 x 538 [8,4,1]
+  CRUSH rule 2 x 539 [8,1]
+  CRUSH rule 2 x 540 [1,8,4]
+  CRUSH rule 2 x 541 [2,4,8]
+  CRUSH rule 2 x 542 [2,8]
+  CRUSH rule 2 x 543 [8,2]
+  CRUSH rule 2 x 544 [4,8,2]
+  CRUSH rule 2 x 545 [8,1]
+  CRUSH rule 2 x 546 [8,1,4]
+  CRUSH rule 2 x 547 [8,2,4]
+  CRUSH rule 2 x 548 [4,2,8]
+  CRUSH rule 2 x 549 [8,2]
+  CRUSH rule 2 x 550 [2,4,8]
+  CRUSH rule 2 x 551 [8,1]
+  CRUSH rule 2 x 552 [4,8,1]
+  CRUSH rule 2 x 553 [2,8]
+  CRUSH rule 2 x 554 [1,8]
+  CRUSH rule 2 x 555 [4,1,8]
+  CRUSH rule 2 x 556 [8,1]
+  CRUSH rule 2 x 557 [8,2]
+  CRUSH rule 2 x 558 [4,1,8]
+  CRUSH rule 2 x 559 [1,8]
+  CRUSH rule 2 x 560 [8,1]
+  CRUSH rule 2 x 561 [8,4,1]
+  CRUSH rule 2 x 562 [4,1,8]
+  CRUSH rule 2 x 563 [2,8]
+  CRUSH rule 2 x 564 [1,8]
+  CRUSH rule 2 x 565 [4,8,2]
+  CRUSH rule 2 x 566 [4,8,2]
+  CRUSH rule 2 x 567 [4,8,1]
+  CRUSH rule 2 x 568 [8,1]
+  CRUSH rule 2 x 569 [4,1,8]
+  CRUSH rule 2 x 570 [1,8]
+  CRUSH rule 2 x 571 [6,1]
+  CRUSH rule 2 x 572 [4,2,8]
+  CRUSH rule 2 x 573 [1,8]
+  CRUSH rule 2 x 574 [2,8]
+  CRUSH rule 2 x 575 [8,2,4]
+  CRUSH rule 2 x 576 [4,8,2]
+  CRUSH rule 2 x 577 [8,2]
+  CRUSH rule 2 x 578 [8,1]
+  CRUSH rule 2 x 579 [4,1,8]
+  CRUSH rule 2 x 580 [1,8]
+  CRUSH rule 2 x 581 [8,2,4]
+  CRUSH rule 2 x 582 [2,8,4]
+  CRUSH rule 2 x 583 [8,1]
+  CRUSH rule 2 x 584 [8,1,4]
+  CRUSH rule 2 x 585 [8,1,4]
+  CRUSH rule 2 x 586 [1,8,4]
+  CRUSH rule 2 x 587 [2,4,8]
+  CRUSH rule 2 x 588 [4,8,1]
+  CRUSH rule 2 x 589 [8,1]
+  CRUSH rule 2 x 590 [8,2]
+  CRUSH rule 2 x 591 [4,2,8]
+  CRUSH rule 2 x 592 [2,4,8]
+  CRUSH rule 2 x 593 [1,8,4]
+  CRUSH rule 2 x 594 [2,8]
+  CRUSH rule 2 x 595 [8,1]
+  CRUSH rule 2 x 596 [8,2]
+  CRUSH rule 2 x 597 [1,8]
+  CRUSH rule 2 x 598 [2,8]
+  CRUSH rule 2 x 599 [4,2,8]
+  CRUSH rule 2 x 600 [8,1,4]
+  CRUSH rule 2 x 601 [1,8,4]
+  CRUSH rule 2 x 602 [8,2]
+  CRUSH rule 2 x 603 [1,8]
+  CRUSH rule 2 x 604 [8,2]
+  CRUSH rule 2 x 605 [2,8]
+  CRUSH rule 2 x 606 [2,6,4]
+  CRUSH rule 2 x 607 [2,4,8]
+  CRUSH rule 2 x 608 [4,2,8]
+  CRUSH rule 2 x 609 [4,2,8]
+  CRUSH rule 2 x 610 [8,1]
+  CRUSH rule 2 x 611 [1,8]
+  CRUSH rule 2 x 612 [2,8]
+  CRUSH rule 2 x 613 [8,2,4]
+  CRUSH rule 2 x 614 [8,2,4]
+  CRUSH rule 2 x 615 [8,2,4]
+  CRUSH rule 2 x 616 [1,8]
+  CRUSH rule 2 x 617 [8,1,4]
+  CRUSH rule 2 x 618 [8,4,2]
+  CRUSH rule 2 x 619 [4,1,8]
+  CRUSH rule 2 x 620 [1,8]
+  CRUSH rule 2 x 621 [8,1]
+  CRUSH rule 2 x 622 [2,4,8]
+  CRUSH rule 2 x 623 [2,8]
+  CRUSH rule 2 x 624 [4,2,8]
+  CRUSH rule 2 x 625 [2,8]
+  CRUSH rule 2 x 626 [8,2,4]
+  CRUSH rule 2 x 627 [2,8,4]
+  CRUSH rule 2 x 628 [8,2]
+  CRUSH rule 2 x 629 [2,8,4]
+  CRUSH rule 2 x 630 [2,8]
+  CRUSH rule 2 x 631 [1,8,4]
+  CRUSH rule 2 x 632 [8,2]
+  CRUSH rule 2 x 633 [8,2]
+  CRUSH rule 2 x 634 [1,8]
+  CRUSH rule 2 x 635 [4,8,2]
+  CRUSH rule 2 x 636 [1,4,8]
+  CRUSH rule 2 x 637 [1,8]
+  CRUSH rule 2 x 638 [8,1,4]
+  CRUSH rule 2 x 639 [2,8]
+  CRUSH rule 2 x 640 [2,8]
+  CRUSH rule 2 x 641 [8,2]
+  CRUSH rule 2 x 642 [2,8]
+  CRUSH rule 2 x 643 [1,8]
+  CRUSH rule 2 x 644 [8,1]
+  CRUSH rule 2 x 645 [8,1]
+  CRUSH rule 2 x 646 [8,1,4]
+  CRUSH rule 2 x 647 [8,1]
+  CRUSH rule 2 x 648 [1,8]
+  CRUSH rule 2 x 649 [4,8,2]
+  CRUSH rule 2 x 650 [8,4,1]
+  CRUSH rule 2 x 651 [4,6,1]
+  CRUSH rule 2 x 652 [4,8,1]
+  CRUSH rule 2 x 653 [8,2]
+  CRUSH rule 2 x 654 [6,2]
+  CRUSH rule 2 x 655 [1,4,8]
+  CRUSH rule 2 x 656 [8,1]
+  CRUSH rule 2 x 657 [6,1]
+  CRUSH rule 2 x 658 [8,2]
+  CRUSH rule 2 x 659 [4,8,2]
+  CRUSH rule 2 x 660 [8,2]
+  CRUSH rule 2 x 661 [1,8]
+  CRUSH rule 2 x 662 [8,2]
+  CRUSH rule 2 x 663 [1,4,8]
+  CRUSH rule 2 x 664 [1,4,8]
+  CRUSH rule 2 x 665 [4,6,1]
+  CRUSH rule 2 x 666 [2,8]
+  CRUSH rule 2 x 667 [1,4,8]
+  CRUSH rule 2 x 668 [4,8,1]
+  CRUSH rule 2 x 669 [6,4,2]
+  CRUSH rule 2 x 670 [4,2,8]
+  CRUSH rule 2 x 671 [2,8]
+  CRUSH rule 2 x 672 [4,2,8]
+  CRUSH rule 2 x 673 [4,2,8]
+  CRUSH rule 2 x 674 [1,8]
+  CRUSH rule 2 x 675 [1,8,4]
+  CRUSH rule 2 x 676 [2,4,8]
+  CRUSH rule 2 x 677 [4,1,8]
+  CRUSH rule 2 x 678 [2,4,8]
+  CRUSH rule 2 x 679 [8,2]
+  CRUSH rule 2 x 680 [2,8]
+  CRUSH rule 2 x 681 [8,1]
+  CRUSH rule 2 x 682 [1,4,8]
+  CRUSH rule 2 x 683 [1,4,8]
+  CRUSH rule 2 x 684 [8,1,4]
+  CRUSH rule 2 x 685 [8,1,4]
+  CRUSH rule 2 x 686 [1,4,8]
+  CRUSH rule 2 x 687 [6,1]
+  CRUSH rule 2 x 688 [4,8,2]
+  CRUSH rule 2 x 689 [8,4,2]
+  CRUSH rule 2 x 690 [8,1,4]
+  CRUSH rule 2 x 691 [1,8]
+  CRUSH rule 2 x 692 [8,2]
+  CRUSH rule 2 x 693 [8,4,1]
+  CRUSH rule 2 x 694 [8,4,1]
+  CRUSH rule 2 x 695 [2,8,4]
+  CRUSH rule 2 x 696 [1,8]
+  CRUSH rule 2 x 697 [8,1,4]
+  CRUSH rule 2 x 698 [8,2,4]
+  CRUSH rule 2 x 699 [1,8,4]
+  CRUSH rule 2 x 700 [1,8]
+  CRUSH rule 2 x 701 [1,8]
+  CRUSH rule 2 x 702 [2,8]
+  CRUSH rule 2 x 703 [8,1]
+  CRUSH rule 2 x 704 [1,4,8]
+  CRUSH rule 2 x 705 [8,1,4]
+  CRUSH rule 2 x 706 [1,4,8]
+  CRUSH rule 2 x 707 [8,4,1]
+  CRUSH rule 2 x 708 [4,8,1]
+  CRUSH rule 2 x 709 [8,2]
+  CRUSH rule 2 x 710 [8,2]
+  CRUSH rule 2 x 711 [2,4,8]
+  CRUSH rule 2 x 712 [2,8]
+  CRUSH rule 2 x 713 [8,4,1]
+  CRUSH rule 2 x 714 [2,8]
+  CRUSH rule 2 x 715 [1,8]
+  CRUSH rule 2 x 716 [4,8,2]
+  CRUSH rule 2 x 717 [8,2,4]
+  CRUSH rule 2 x 718 [8,1]
+  CRUSH rule 2 x 719 [2,6,4]
+  CRUSH rule 2 x 720 [8,1,4]
+  CRUSH rule 2 x 721 [4,6,2]
+  CRUSH rule 2 x 722 [8,2]
+  CRUSH rule 2 x 723 [4,1,8]
+  CRUSH rule 2 x 724 [2,6]
+  CRUSH rule 2 x 725 [1,8]
+  CRUSH rule 2 x 726 [4,8,1]
+  CRUSH rule 2 x 727 [4,8,1]
+  CRUSH rule 2 x 728 [2,8,4]
+  CRUSH rule 2 x 729 [8,2]
+  CRUSH rule 2 x 730 [4,8,2]
+  CRUSH rule 2 x 731 [4,1,8]
+  CRUSH rule 2 x 732 [1,8]
+  CRUSH rule 2 x 733 [4,8,1]
+  CRUSH rule 2 x 734 [8,4,2]
+  CRUSH rule 2 x 735 [4,8,1]
+  CRUSH rule 2 x 736 [4,8,1]
+  CRUSH rule 2 x 737 [1,8,4]
+  CRUSH rule 2 x 738 [4,2,8]
+  CRUSH rule 2 x 739 [2,8]
+  CRUSH rule 2 x 740 [1,8,4]
+  CRUSH rule 2 x 741 [8,1]
+  CRUSH rule 2 x 742 [8,2]
+  CRUSH rule 2 x 743 [8,1,4]
+  CRUSH rule 2 x 744 [4,8,1]
+  CRUSH rule 2 x 745 [1,8]
+  CRUSH rule 2 x 746 [1,8]
+  CRUSH rule 2 x 747 [8,1]
+  CRUSH rule 2 x 748 [2,8,4]
+  CRUSH rule 2 x 749 [4,8,2]
+  CRUSH rule 2 x 750 [1,8,4]
+  CRUSH rule 2 x 751 [2,8]
+  CRUSH rule 2 x 752 [8,1]
+  CRUSH rule 2 x 753 [8,4,1]
+  CRUSH rule 2 x 754 [8,4,2]
+  CRUSH rule 2 x 755 [1,8,4]
+  CRUSH rule 2 x 756 [8,1]
+  CRUSH rule 2 x 757 [8,1,4]
+  CRUSH rule 2 x 758 [8,2]
+  CRUSH rule 2 x 759 [8,4,2]
+  CRUSH rule 2 x 760 [1,4,8]
+  CRUSH rule 2 x 761 [2,8]
+  CRUSH rule 2 x 762 [2,8]
+  CRUSH rule 2 x 763 [8,4,1]
+  CRUSH rule 2 x 764 [1,8]
+  CRUSH rule 2 x 765 [8,2]
+  CRUSH rule 2 x 766 [8,1]
+  CRUSH rule 2 x 767 [1,8,4]
+  CRUSH rule 2 x 768 [8,4,2]
+  CRUSH rule 2 x 769 [8,2,4]
+  CRUSH rule 2 x 770 [8,2,4]
+  CRUSH rule 2 x 771 [8,1,4]
+  CRUSH rule 2 x 772 [8,4,1]
+  CRUSH rule 2 x 773 [4,1,8]
+  CRUSH rule 2 x 774 [8,1]
+  CRUSH rule 2 x 775 [8,4,2]
+  CRUSH rule 2 x 776 [6,2]
+  CRUSH rule 2 x 777 [4,1,8]
+  CRUSH rule 2 x 778 [1,8,4]
+  CRUSH rule 2 x 779 [2,8]
+  CRUSH rule 2 x 780 [2,4,8]
+  CRUSH rule 2 x 781 [8,2]
+  CRUSH rule 2 x 782 [4,1,8]
+  CRUSH rule 2 x 783 [8,1,4]
+  CRUSH rule 2 x 784 [1,4,8]
+  CRUSH rule 2 x 785 [8,1,4]
+  CRUSH rule 2 x 786 [8,1]
+  CRUSH rule 2 x 787 [1,6,4]
+  CRUSH rule 2 x 788 [8,2,4]
+  CRUSH rule 2 x 789 [1,8]
+  CRUSH rule 2 x 790 [8,1]
+  CRUSH rule 2 x 791 [4,8,2]
+  CRUSH rule 2 x 792 [4,8,2]
+  CRUSH rule 2 x 793 [8,1,4]
+  CRUSH rule 2 x 794 [2,8,4]
+  CRUSH rule 2 x 795 [1,8]
+  CRUSH rule 2 x 796 [8,2]
+  CRUSH rule 2 x 797 [2,4,8]
+  CRUSH rule 2 x 798 [6,1]
+  CRUSH rule 2 x 799 [4,1,8]
+  CRUSH rule 2 x 800 [2,8]
+  CRUSH rule 2 x 801 [4,8,1]
+  CRUSH rule 2 x 802 [1,8,4]
+  CRUSH rule 2 x 803 [2,8]
+  CRUSH rule 2 x 804 [8,2]
+  CRUSH rule 2 x 805 [8,2]
+  CRUSH rule 2 x 806 [1,4,8]
+  CRUSH rule 2 x 807 [4,8,2]
+  CRUSH rule 2 x 808 [8,2]
+  CRUSH rule 2 x 809 [1,8]
+  CRUSH rule 2 x 810 [8,2]
+  CRUSH rule 2 x 811 [8,1]
+  CRUSH rule 2 x 812 [8,4,2]
+  CRUSH rule 2 x 813 [8,4,2]
+  CRUSH rule 2 x 814 [8,2]
+  CRUSH rule 2 x 815 [4,1,8]
+  CRUSH rule 2 x 816 [2,8]
+  CRUSH rule 2 x 817 [8,1]
+  CRUSH rule 2 x 818 [1,8]
+  CRUSH rule 2 x 819 [1,8]
+  CRUSH rule 2 x 820 [4,8,2]
+  CRUSH rule 2 x 821 [4,8,2]
+  CRUSH rule 2 x 822 [2,4,8]
+  CRUSH rule 2 x 823 [4,8,2]
+  CRUSH rule 2 x 824 [8,2]
+  CRUSH rule 2 x 825 [2,8,4]
+  CRUSH rule 2 x 826 [8,2,4]
+  CRUSH rule 2 x 827 [2,8,4]
+  CRUSH rule 2 x 828 [2,8]
+  CRUSH rule 2 x 829 [8,1]
+  CRUSH rule 2 x 830 [2,4,8]
+  CRUSH rule 2 x 831 [1,8]
+  CRUSH rule 2 x 832 [4,8,2]
+  CRUSH rule 2 x 833 [2,8]
+  CRUSH rule 2 x 834 [1,8]
+  CRUSH rule 2 x 835 [8,4,1]
+  CRUSH rule 2 x 836 [4,8,1]
+  CRUSH rule 2 x 837 [8,4,1]
+  CRUSH rule 2 x 838 [6,2,4]
+  CRUSH rule 2 x 839 [2,8]
+  CRUSH rule 2 x 840 [8,1]
+  CRUSH rule 2 x 841 [4,8,2]
+  CRUSH rule 2 x 842 [2,8]
+  CRUSH rule 2 x 843 [8,4,1]
+  CRUSH rule 2 x 844 [8,2]
+  CRUSH rule 2 x 845 [4,8,2]
+  CRUSH rule 2 x 846 [4,2,8]
+  CRUSH rule 2 x 847 [2,8]
+  CRUSH rule 2 x 848 [2,8,4]
+  CRUSH rule 2 x 849 [4,8,2]
+  CRUSH rule 2 x 850 [1,6]
+  CRUSH rule 2 x 851 [6,1]
+  CRUSH rule 2 x 852 [8,4,2]
+  CRUSH rule 2 x 853 [6,1]
+  CRUSH rule 2 x 854 [8,1]
+  CRUSH rule 2 x 855 [8,1]
+  CRUSH rule 2 x 856 [8,4,2]
+  CRUSH rule 2 x 857 [8,2]
+  CRUSH rule 2 x 858 [6,1]
+  CRUSH rule 2 x 859 [8,2,4]
+  CRUSH rule 2 x 860 [2,8]
+  CRUSH rule 2 x 861 [8,2]
+  CRUSH rule 2 x 862 [8,1]
+  CRUSH rule 2 x 863 [8,2]
+  CRUSH rule 2 x 864 [8,2]
+  CRUSH rule 2 x 865 [8,1]
+  CRUSH rule 2 x 866 [8,2]
+  CRUSH rule 2 x 867 [8,2]
+  CRUSH rule 2 x 868 [8,1]
+  CRUSH rule 2 x 869 [8,4,2]
+  CRUSH rule 2 x 870 [2,8]
+  CRUSH rule 2 x 871 [1,8]
+  CRUSH rule 2 x 872 [1,8]
+  CRUSH rule 2 x 873 [4,8,2]
+  CRUSH rule 2 x 874 [2,6]
+  CRUSH rule 2 x 875 [2,8,4]
+  CRUSH rule 2 x 876 [4,8,1]
+  CRUSH rule 2 x 877 [8,4,2]
+  CRUSH rule 2 x 878 [2,8]
+  CRUSH rule 2 x 879 [8,1]
+  CRUSH rule 2 x 880 [1,8]
+  CRUSH rule 2 x 881 [4,8,1]
+  CRUSH rule 2 x 882 [1,8]
+  CRUSH rule 2 x 883 [2,4,8]
+  CRUSH rule 2 x 884 [8,2,4]
+  CRUSH rule 2 x 885 [4,1,8]
+  CRUSH rule 2 x 886 [8,2]
+  CRUSH rule 2 x 887 [8,4,1]
+  CRUSH rule 2 x 888 [8,2]
+  CRUSH rule 2 x 889 [2,6]
+  CRUSH rule 2 x 890 [8,2,4]
+  CRUSH rule 2 x 891 [1,8]
+  CRUSH rule 2 x 892 [8,2,4]
+  CRUSH rule 2 x 893 [2,6]
+  CRUSH rule 2 x 894 [8,4,2]
+  CRUSH rule 2 x 895 [4,1,8]
+  CRUSH rule 2 x 896 [1,8]
+  CRUSH rule 2 x 897 [2,8]
+  CRUSH rule 2 x 898 [1,4,8]
+  CRUSH rule 2 x 899 [1,8]
+  CRUSH rule 2 x 900 [4,1,8]
+  CRUSH rule 2 x 901 [2,8]
+  CRUSH rule 2 x 902 [8,4,1]
+  CRUSH rule 2 x 903 [8,2]
+  CRUSH rule 2 x 904 [8,2]
+  CRUSH rule 2 x 905 [8,2]
+  CRUSH rule 2 x 906 [1,8]
+  CRUSH rule 2 x 907 [8,1]
+  CRUSH rule 2 x 908 [8,1]
+  CRUSH rule 2 x 909 [2,8]
+  CRUSH rule 2 x 910 [8,2]
+  CRUSH rule 2 x 911 [8,1]
+  CRUSH rule 2 x 912 [1,8]
+  CRUSH rule 2 x 913 [8,2,4]
+  CRUSH rule 2 x 914 [6,4,2]
+  CRUSH rule 2 x 915 [8,2]
+  CRUSH rule 2 x 916 [4,1,8]
+  CRUSH rule 2 x 917 [1,4,8]
+  CRUSH rule 2 x 918 [8,2]
+  CRUSH rule 2 x 919 [8,2]
+  CRUSH rule 2 x 920 [8,1]
+  CRUSH rule 2 x 921 [1,8]
+  CRUSH rule 2 x 922 [8,4,2]
+  CRUSH rule 2 x 923 [4,8,2]
+  CRUSH rule 2 x 924 [8,1]
+  CRUSH rule 2 x 925 [4,8,2]
+  CRUSH rule 2 x 926 [2,8]
+  CRUSH rule 2 x 927 [1,8,4]
+  CRUSH rule 2 x 928 [8,1]
+  CRUSH rule 2 x 929 [4,1,8]
+  CRUSH rule 2 x 930 [2,8]
+  CRUSH rule 2 x 931 [2,8]
+  CRUSH rule 2 x 932 [4,1,8]
+  CRUSH rule 2 x 933 [8,4,1]
+  CRUSH rule 2 x 934 [8,2]
+  CRUSH rule 2 x 935 [8,2]
+  CRUSH rule 2 x 936 [1,8]
+  CRUSH rule 2 x 937 [4,8,2]
+  CRUSH rule 2 x 938 [8,4,2]
+  CRUSH rule 2 x 939 [2,8,4]
+  CRUSH rule 2 x 940 [8,1]
+  CRUSH rule 2 x 941 [2,8]
+  CRUSH rule 2 x 942 [1,8]
+  CRUSH rule 2 x 943 [8,2]
+  CRUSH rule 2 x 944 [8,2]
+  CRUSH rule 2 x 945 [8,2,4]
+  CRUSH rule 2 x 946 [2,8,4]
+  CRUSH rule 2 x 947 [8,2]
+  CRUSH rule 2 x 948 [8,1]
+  CRUSH rule 2 x 949 [6,1]
+  CRUSH rule 2 x 950 [8,1]
+  CRUSH rule 2 x 951 [8,1]
+  CRUSH rule 2 x 952 [2,8,4]
+  CRUSH rule 2 x 953 [1,4,8]
+  CRUSH rule 2 x 954 [2,8]
+  CRUSH rule 2 x 955 [8,1,4]
+  CRUSH rule 2 x 956 [1,8,4]
+  CRUSH rule 2 x 957 [8,1,4]
+  CRUSH rule 2 x 958 [8,4,1]
+  CRUSH rule 2 x 959 [4,2,8]
+  CRUSH rule 2 x 960 [6,1]
+  CRUSH rule 2 x 961 [1,8]
+  CRUSH rule 2 x 962 [8,4,2]
+  CRUSH rule 2 x 963 [2,4,6]
+  CRUSH rule 2 x 964 [2,8]
+  CRUSH rule 2 x 965 [8,2]
+  CRUSH rule 2 x 966 [4,8,1]
+  CRUSH rule 2 x 967 [8,4,2]
+  CRUSH rule 2 x 968 [8,2]
+  CRUSH rule 2 x 969 [8,2,4]
+  CRUSH rule 2 x 970 [2,8,4]
+  CRUSH rule 2 x 971 [1,8]
+  CRUSH rule 2 x 972 [1,8]
+  CRUSH rule 2 x 973 [1,8]
+  CRUSH rule 2 x 974 [4,1,8]
+  CRUSH rule 2 x 975 [4,8,1]
+  CRUSH rule 2 x 976 [4,8,2]
+  CRUSH rule 2 x 977 [8,4,2]
+  CRUSH rule 2 x 978 [8,2,4]
+  CRUSH rule 2 x 979 [8,1,4]
+  CRUSH rule 2 x 980 [8,2,4]
+  CRUSH rule 2 x 981 [8,2]
+  CRUSH rule 2 x 982 [1,8]
+  CRUSH rule 2 x 983 [4,8,2]
+  CRUSH rule 2 x 984 [2,8]
+  CRUSH rule 2 x 985 [2,4,8]
+  CRUSH rule 2 x 986 [8,4,1]
+  CRUSH rule 2 x 987 [2,8]
+  CRUSH rule 2 x 988 [1,4,6]
+  CRUSH rule 2 x 989 [1,8]
+  CRUSH rule 2 x 990 [1,8,4]
+  CRUSH rule 2 x 991 [1,4,8]
+  CRUSH rule 2 x 992 [8,1,4]
+  CRUSH rule 2 x 993 [2,8,4]
+  CRUSH rule 2 x 994 [4,8,2]
+  CRUSH rule 2 x 995 [8,1,4]
+  CRUSH rule 2 x 996 [8,4,1]
+  CRUSH rule 2 x 997 [8,4,1]
+  CRUSH rule 2 x 998 [8,1,4]
+  CRUSH rule 2 x 999 [1,8,4]
+  CRUSH rule 2 x 1000 [8,4,2]
+  CRUSH rule 2 x 1001 [2,8]
+  CRUSH rule 2 x 1002 [1,8]
+  CRUSH rule 2 x 1003 [2,8]
+  CRUSH rule 2 x 1004 [8,1,4]
+  CRUSH rule 2 x 1005 [8,1]
+  CRUSH rule 2 x 1006 [1,8,4]
+  CRUSH rule 2 x 1007 [1,4,8]
+  CRUSH rule 2 x 1008 [1,8]
+  CRUSH rule 2 x 1009 [6,4,1]
+  CRUSH rule 2 x 1010 [1,8]
+  CRUSH rule 2 x 1011 [4,2,8]
+  CRUSH rule 2 x 1012 [1,8]
+  CRUSH rule 2 x 1013 [2,8]
+  CRUSH rule 2 x 1014 [2,8,4]
+  CRUSH rule 2 x 1015 [8,1]
+  CRUSH rule 2 x 1016 [2,4,8]
+  CRUSH rule 2 x 1017 [6,2,4]
+  CRUSH rule 2 x 1018 [4,1,8]
+  CRUSH rule 2 x 1019 [4,8,2]
+  CRUSH rule 2 x 1020 [1,8]
+  CRUSH rule 2 x 1021 [2,8]
+  CRUSH rule 2 x 1022 [1,8,4]
+  CRUSH rule 2 x 1023 [4,2,8]
+  rule 2 (chooseleaf) num_rep 3 result size == 2:\t501/1024 (esc)
+  rule 2 (chooseleaf) num_rep 3 result size == 3:\t523/1024 (esc)
+  rule 3 (choose-set), x = 0..1023, numrep = 2..3
+  CRUSH rule 3 x 0 [2,4]
+  CRUSH rule 3 x 1 [2,8]
+  CRUSH rule 3 x 2 [1]
+  CRUSH rule 3 x 3 [8,1]
+  CRUSH rule 3 x 4 [4,1]
+  CRUSH rule 3 x 5 [8,1]
+  CRUSH rule 3 x 6 [2,8]
+  CRUSH rule 3 x 7 [4,8]
+  CRUSH rule 3 x 8 [4,8]
+  CRUSH rule 3 x 9 [2,4]
+  CRUSH rule 3 x 10 [2,8]
+  CRUSH rule 3 x 11 [2,8]
+  CRUSH rule 3 x 12 [2]
+  CRUSH rule 3 x 13 [4,8]
+  CRUSH rule 3 x 14 [8,1]
+  CRUSH rule 3 x 15 [8,1]
+  CRUSH rule 3 x 16 [8]
+  CRUSH rule 3 x 17 [4,1]
+  CRUSH rule 3 x 18 [1]
+  CRUSH rule 3 x 19 [8,4]
+  CRUSH rule 3 x 20 [2]
+  CRUSH rule 3 x 21 [8]
+  CRUSH rule 3 x 22 [8]
+  CRUSH rule 3 x 23 [4,8]
+  CRUSH rule 3 x 24 [1,8]
+  CRUSH rule 3 x 25 [4,8]
+  CRUSH rule 3 x 26 [2,8]
+  CRUSH rule 3 x 27 [4,1]
+  CRUSH rule 3 x 28 [8,2]
+  CRUSH rule 3 x 29 [8,4]
+  CRUSH rule 3 x 30 [4,8]
+  CRUSH rule 3 x 31 [8,2]
+  CRUSH rule 3 x 32 [6]
+  CRUSH rule 3 x 33 [2,8]
+  CRUSH rule 3 x 34 [2]
+  CRUSH rule 3 x 35 [1,8]
+  CRUSH rule 3 x 36 [8]
+  CRUSH rule 3 x 37 [1]
+  CRUSH rule 3 x 38 [4,8]
+  CRUSH rule 3 x 39 [8]
+  CRUSH rule 3 x 40 [8,2]
+  CRUSH rule 3 x 41 [2,8]
+  CRUSH rule 3 x 42 [8]
+  CRUSH rule 3 x 43 [1]
+  CRUSH rule 3 x 44 [1,8]
+  CRUSH rule 3 x 45 [8,2]
+  CRUSH rule 3 x 46 [2]
+  CRUSH rule 3 x 47 [4,1]
+  CRUSH rule 3 x 48 [8]
+  CRUSH rule 3 x 49 [8]
+  CRUSH rule 3 x 50 [4,1]
+  CRUSH rule 3 x 51 [8]
+  CRUSH rule 3 x 52 [8,2]
+  CRUSH rule 3 x 53 [4,8]
+  CRUSH rule 3 x 54 [8,4]
+  CRUSH rule 3 x 55 [8,2]
+  CRUSH rule 3 x 56 [8,4]
+  CRUSH rule 3 x 57 [8]
+  CRUSH rule 3 x 58 [1,8]
+  CRUSH rule 3 x 59 [2]
+  CRUSH rule 3 x 60 [4,1]
+  CRUSH rule 3 x 61 [4,8]
+  CRUSH rule 3 x 62 [8,1]
+  CRUSH rule 3 x 63 [8]
+  CRUSH rule 3 x 64 [4,1]
+  CRUSH rule 3 x 65 [8,4]
+  CRUSH rule 3 x 66 [4,8]
+  CRUSH rule 3 x 67 [4,2]
+  CRUSH rule 3 x 68 [1]
+  CRUSH rule 3 x 69 [2]
+  CRUSH rule 3 x 70 [8,2]
+  CRUSH rule 3 x 71 [2,8]
+  CRUSH rule 3 x 72 [8,2]
+  CRUSH rule 3 x 73 [2,8]
+  CRUSH rule 3 x 74 [1,8]
+  CRUSH rule 3 x 75 [4,2]
+  CRUSH rule 3 x 76 [4,2]
+  CRUSH rule 3 x 77 [8,2]
+  CRUSH rule 3 x 78 [1]
+  CRUSH rule 3 x 79 [4,1]
+  CRUSH rule 3 x 80 [2,4]
+  CRUSH rule 3 x 81 [2]
+  CRUSH rule 3 x 82 [6,2]
+  CRUSH rule 3 x 83 [2,8]
+  CRUSH rule 3 x 84 [8,2]
+  CRUSH rule 3 x 85 [4,8]
+  CRUSH rule 3 x 86 [2,8]
+  CRUSH rule 3 x 87 [2,8]
+  CRUSH rule 3 x 88 [1,8]
+  CRUSH rule 3 x 89 [1]
+  CRUSH rule 3 x 90 [8,4]
+  CRUSH rule 3 x 91 [4,8]
+  CRUSH rule 3 x 92 [1,8]
+  CRUSH rule 3 x 93 [8,4]
+  CRUSH rule 3 x 94 [1]
+  CRUSH rule 3 x 95 [8]
+  CRUSH rule 3 x 96 [8]
+  CRUSH rule 3 x 97 [8]
+  CRUSH rule 3 x 98 [2,8]
+  CRUSH rule 3 x 99 [2,8]
+  CRUSH rule 3 x 100 [1,8]
+  CRUSH rule 3 x 101 [8]
+  CRUSH rule 3 x 102 [2]
+  CRUSH rule 3 x 103 [8]
+  CRUSH rule 3 x 104 [8,4]
+  CRUSH rule 3 x 105 [2,4]
+  CRUSH rule 3 x 106 [1,8]
+  CRUSH rule 3 x 107 [1]
+  CRUSH rule 3 x 108 [8,1]
+  CRUSH rule 3 x 109 [1,4]
+  CRUSH rule 3 x 110 [4,2]
+  CRUSH rule 3 x 111 [2,4]
+  CRUSH rule 3 x 112 [2,8]
+  CRUSH rule 3 x 113 [8,1]
+  CRUSH rule 3 x 114 [8,4]
+  CRUSH rule 3 x 115 [8,2]
+  CRUSH rule 3 x 116 [1,8]
+  CRUSH rule 3 x 117 [6]
+  CRUSH rule 3 x 118 [2]
+  CRUSH rule 3 x 119 [8]
+  CRUSH rule 3 x 120 [2,4]
+  CRUSH rule 3 x 121 [2,8]
+  CRUSH rule 3 x 122 [8]
+  CRUSH rule 3 x 123 [2]
+  CRUSH rule 3 x 124 [2]
+  CRUSH rule 3 x 125 [1,8]
+  CRUSH rule 3 x 126 [1]
+  CRUSH rule 3 x 127 [4,8]
+  CRUSH rule 3 x 128 [8]
+  CRUSH rule 3 x 129 [2,4]
+  CRUSH rule 3 x 130 [4,8]
+  CRUSH rule 3 x 131 [1,4]
+  CRUSH rule 3 x 132 [1]
+  CRUSH rule 3 x 133 [8]
+  CRUSH rule 3 x 134 [1,8]
+  CRUSH rule 3 x 135 [4,8]
+  CRUSH rule 3 x 136 [2,4]
+  CRUSH rule 3 x 137 [8,4]
+  CRUSH rule 3 x 138 [8,4]
+  CRUSH rule 3 x 139 [4,2]
+  CRUSH rule 3 x 140 [1,8]
+  CRUSH rule 3 x 141 [8,2]
+  CRUSH rule 3 x 142 [4,2]
+  CRUSH rule 3 x 143 [4,8]
+  CRUSH rule 3 x 144 [8,2]
+  CRUSH rule 3 x 145 [8]
+  CRUSH rule 3 x 146 [2,8]
+  CRUSH rule 3 x 147 [2,8]
+  CRUSH rule 3 x 148 [4,1]
+  CRUSH rule 3 x 149 [4,8]
+  CRUSH rule 3 x 150 [1,8]
+  CRUSH rule 3 x 151 [8]
+  CRUSH rule 3 x 152 [8]
+  CRUSH rule 3 x 153 [8,4]
+  CRUSH rule 3 x 154 [4,2]
+  CRUSH rule 3 x 155 [4,8]
+  CRUSH rule 3 x 156 [4,2]
+  CRUSH rule 3 x 157 [1]
+  CRUSH rule 3 x 158 [2,8]
+  CRUSH rule 3 x 159 [8,2]
+  CRUSH rule 3 x 160 [2,8]
+  CRUSH rule 3 x 161 [1,4]
+  CRUSH rule 3 x 162 [1,8]
+  CRUSH rule 3 x 163 [4,8]
+  CRUSH rule 3 x 164 [8,2]
+  CRUSH rule 3 x 165 [8,2]
+  CRUSH rule 3 x 166 [2]
+  CRUSH rule 3 x 167 [1,8]
+  CRUSH rule 3 x 168 [4,2]
+  CRUSH rule 3 x 169 [2,8]
+  CRUSH rule 3 x 170 [1]
+  CRUSH rule 3 x 171 [8,4]
+  CRUSH rule 3 x 172 [1,8]
+  CRUSH rule 3 x 173 [8,4]
+  CRUSH rule 3 x 174 [1]
+  CRUSH rule 3 x 175 [8,1]
+  CRUSH rule 3 x 176 [2]
+  CRUSH rule 3 x 177 [1]
+  CRUSH rule 3 x 178 [4,2]
+  CRUSH rule 3 x 179 [1]
+  CRUSH rule 3 x 180 [8]
+  CRUSH rule 3 x 181 [8,2]
+  CRUSH rule 3 x 182 [8]
+  CRUSH rule 3 x 183 [8,4]
+  CRUSH rule 3 x 184 [4,8]
+  CRUSH rule 3 x 185 [8,1]
+  CRUSH rule 3 x 186 [2,4]
+  CRUSH rule 3 x 187 [1,8]
+  CRUSH rule 3 x 188 [1,8]
+  CRUSH rule 3 x 189 [1,8]
+  CRUSH rule 3 x 190 [1]
+  CRUSH rule 3 x 191 [8,1]
+  CRUSH rule 3 x 192 [4,2]
+  CRUSH rule 3 x 193 [4,2]
+  CRUSH rule 3 x 194 [1]
+  CRUSH rule 3 x 195 [8,4]
+  CRUSH rule 3 x 196 [8,1]
+  CRUSH rule 3 x 197 [8,4]
+  CRUSH rule 3 x 198 [2]
+  CRUSH rule 3 x 199 [1,4]
+  CRUSH rule 3 x 200 [1]
+  CRUSH rule 3 x 201 [8,1]
+  CRUSH rule 3 x 202 [8]
+  CRUSH rule 3 x 203 [8]
+  CRUSH rule 3 x 204 [2,4]
+  CRUSH rule 3 x 205 [1,8]
+  CRUSH rule 3 x 206 [1,8]
+  CRUSH rule 3 x 207 [2]
+  CRUSH rule 3 x 208 [8,2]
+  CRUSH rule 3 x 209 [1,8]
+  CRUSH rule 3 x 210 [1,4]
+  CRUSH rule 3 x 211 [4,2]
+  CRUSH rule 3 x 212 [8]
+  CRUSH rule 3 x 213 [8,4]
+  CRUSH rule 3 x 214 [8]
+  CRUSH rule 3 x 215 [8,1]
+  CRUSH rule 3 x 216 [2]
+  CRUSH rule 3 x 217 [1,8]
+  CRUSH rule 3 x 218 [2,8]
+  CRUSH rule 3 x 219 [8]
+  CRUSH rule 3 x 220 [4,8]
+  CRUSH rule 3 x 221 [8]
+  CRUSH rule 3 x 222 [8]
+  CRUSH rule 3 x 223 [1]
+  CRUSH rule 3 x 224 [1,4]
+  CRUSH rule 3 x 225 [8,2]
+  CRUSH rule 3 x 226 [8,2]
+  CRUSH rule 3 x 227 [4,1]
+  CRUSH rule 3 x 228 [8]
+  CRUSH rule 3 x 229 [4,8]
+  CRUSH rule 3 x 230 [4,8]
+  CRUSH rule 3 x 231 [4,8]
+  CRUSH rule 3 x 232 [2,8]
+  CRUSH rule 3 x 233 [8]
+  CRUSH rule 3 x 234 [1]
+  CRUSH rule 3 x 235 [4,8]
+  CRUSH rule 3 x 236 [2]
+  CRUSH rule 3 x 237 [4,8]
+  CRUSH rule 3 x 238 [2]
+  CRUSH rule 3 x 239 [8]
+  CRUSH rule 3 x 240 [4,8]
+  CRUSH rule 3 x 241 [1]
+  CRUSH rule 3 x 242 [2]
+  CRUSH rule 3 x 243 [8]
+  CRUSH rule 3 x 244 [4,8]
+  CRUSH rule 3 x 245 [8,2]
+  CRUSH rule 3 x 246 [1]
+  CRUSH rule 3 x 247 [8,1]
+  CRUSH rule 3 x 248 [8,1]
+  CRUSH rule 3 x 249 [2]
+  CRUSH rule 3 x 250 [2,4]
+  CRUSH rule 3 x 251 [2]
+  CRUSH rule 3 x 252 [4,8]
+  CRUSH rule 3 x 253 [2]
+  CRUSH rule 3 x 254 [4,2]
+  CRUSH rule 3 x 255 [1,8]
+  CRUSH rule 3 x 256 [4,8]
+  CRUSH rule 3 x 257 [2,6]
+  CRUSH rule 3 x 258 [4,2]
+  CRUSH rule 3 x 259 [6]
+  CRUSH rule 3 x 260 [8]
+  CRUSH rule 3 x 261 [8]
+  CRUSH rule 3 x 262 [8]
+  CRUSH rule 3 x 263 [8,1]
+  CRUSH rule 3 x 264 [8]
+  CRUSH rule 3 x 265 [8]
+  CRUSH rule 3 x 266 [8,2]
+  CRUSH rule 3 x 267 [2]
+  CRUSH rule 3 x 268 [1,8]
+  CRUSH rule 3 x 269 [1,8]
+  CRUSH rule 3 x 270 [4,1]
+  CRUSH rule 3 x 271 [8,4]
+  CRUSH rule 3 x 272 [2,8]
+  CRUSH rule 3 x 273 [4,2]
+  CRUSH rule 3 x 274 [8,4]
+  CRUSH rule 3 x 275 [4,8]
+  CRUSH rule 3 x 276 [8,1]
+  CRUSH rule 3 x 277 [8]
+  CRUSH rule 3 x 278 [8,2]
+  CRUSH rule 3 x 279 [8,4]
+  CRUSH rule 3 x 280 [2,8]
+  CRUSH rule 3 x 281 [8,1]
+  CRUSH rule 3 x 282 [2]
+  CRUSH rule 3 x 283 [8,1]
+  CRUSH rule 3 x 284 [8]
+  CRUSH rule 3 x 285 [4,8]
+  CRUSH rule 3 x 286 [2,8]
+  CRUSH rule 3 x 287 [1]
+  CRUSH rule 3 x 288 [8,1]
+  CRUSH rule 3 x 289 [4,8]
+  CRUSH rule 3 x 290 [1,4]
+  CRUSH rule 3 x 291 [1,4]
+  CRUSH rule 3 x 292 [8,1]
+  CRUSH rule 3 x 293 [8,2]
+  CRUSH rule 3 x 294 [8,4]
+  CRUSH rule 3 x 295 [4,8]
+  CRUSH rule 3 x 296 [4,1]
+  CRUSH rule 3 x 297 [8,2]
+  CRUSH rule 3 x 298 [1]
+  CRUSH rule 3 x 299 [2,8]
+  CRUSH rule 3 x 300 [8]
+  CRUSH rule 3 x 301 [1,8]
+  CRUSH rule 3 x 302 [1]
+  CRUSH rule 3 x 303 [8,4]
+  CRUSH rule 3 x 304 [2,8]
+  CRUSH rule 3 x 305 [8]
+  CRUSH rule 3 x 306 [1,8]
+  CRUSH rule 3 x 307 [2,8]
+  CRUSH rule 3 x 308 [2,8]
+  CRUSH rule 3 x 309 [8]
+  CRUSH rule 3 x 310 [4,1]
+  CRUSH rule 3 x 311 [4,8]
+  CRUSH rule 3 x 312 [2,8]
+  CRUSH rule 3 x 313 [4,1]
+  CRUSH rule 3 x 314 [2]
+  CRUSH rule 3 x 315 [2]
+  CRUSH rule 3 x 316 [8]
+  CRUSH rule 3 x 317 [2,8]
+  CRUSH rule 3 x 318 [8,1]
+  CRUSH rule 3 x 319 [2]
+  CRUSH rule 3 x 320 [8]
+  CRUSH rule 3 x 321 [1]
+  CRUSH rule 3 x 322 [2,6]
+  CRUSH rule 3 x 323 [4,8]
+  CRUSH rule 3 x 324 [8,2]
+  CRUSH rule 3 x 325 [4,8]
+  CRUSH rule 3 x 326 [1]
+  CRUSH rule 3 x 327 [1,8]
+  CRUSH rule 3 x 328 [8,4]
+  CRUSH rule 3 x 329 [4,8]
+  CRUSH rule 3 x 330 [4,8]
+  CRUSH rule 3 x 331 [2,8]
+  CRUSH rule 3 x 332 [2]
+  CRUSH rule 3 x 333 [8]
+  CRUSH rule 3 x 334 [8]
+  CRUSH rule 3 x 335 [8,2]
+  CRUSH rule 3 x 336 [4,8]
+  CRUSH rule 3 x 337 [8,2]
+  CRUSH rule 3 x 338 [8]
+  CRUSH rule 3 x 339 [8]
+  CRUSH rule 3 x 340 [2,8]
+  CRUSH rule 3 x 341 [4,1]
+  CRUSH rule 3 x 342 [2,8]
+  CRUSH rule 3 x 343 [8]
+  CRUSH rule 3 x 344 [6,1]
+  CRUSH rule 3 x 345 [8]
+  CRUSH rule 3 x 346 [8,2]
+  CRUSH rule 3 x 347 [4,1]
+  CRUSH rule 3 x 348 [8,2]
+  CRUSH rule 3 x 349 [1,8]
+  CRUSH rule 3 x 350 [8]
+  CRUSH rule 3 x 351 [8]
+  CRUSH rule 3 x 352 [1,8]
+  CRUSH rule 3 x 353 [8]
+  CRUSH rule 3 x 354 [1]
+  CRUSH rule 3 x 355 [8]
+  CRUSH rule 3 x 356 [4,1]
+  CRUSH rule 3 x 357 [8,1]
+  CRUSH rule 3 x 358 [2,8]
+  CRUSH rule 3 x 359 [6,1]
+  CRUSH rule 3 x 360 [2]
+  CRUSH rule 3 x 361 [8,4]
+  CRUSH rule 3 x 362 [4,1]
+  CRUSH rule 3 x 363 [4,2]
+  CRUSH rule 3 x 364 [2]
+  CRUSH rule 3 x 365 [8]
+  CRUSH rule 3 x 366 [8,2]
+  CRUSH rule 3 x 367 [4,2]
+  CRUSH rule 3 x 368 [8,4]
+  CRUSH rule 3 x 369 [8]
+  CRUSH rule 3 x 370 [8,2]
+  CRUSH rule 3 x 371 [1,4]
+  CRUSH rule 3 x 372 [1]
+  CRUSH rule 3 x 373 [1,8]
+  CRUSH rule 3 x 374 [8]
+  CRUSH rule 3 x 375 [8,4]
+  CRUSH rule 3 x 376 [8,1]
+  CRUSH rule 3 x 377 [1,4]
+  CRUSH rule 3 x 378 [1,8]
+  CRUSH rule 3 x 379 [8]
+  CRUSH rule 3 x 380 [2]
+  CRUSH rule 3 x 381 [1,4]
+  CRUSH rule 3 x 382 [1,4]
+  CRUSH rule 3 x 383 [4,8]
+  CRUSH rule 3 x 384 [8,2]
+  CRUSH rule 3 x 385 [8]
+  CRUSH rule 3 x 386 [1]
+  CRUSH rule 3 x 387 [1,4]
+  CRUSH rule 3 x 388 [2]
+  CRUSH rule 3 x 389 [1,4]
+  CRUSH rule 3 x 390 [4,8]
+  CRUSH rule 3 x 391 [4,8]
+  CRUSH rule 3 x 392 [1,8]
+  CRUSH rule 3 x 393 [2]
+  CRUSH rule 3 x 394 [8]
+  CRUSH rule 3 x 395 [1]
+  CRUSH rule 3 x 396 [4,2]
+  CRUSH rule 3 x 397 [2,4]
+  CRUSH rule 3 x 398 [2,4]
+  CRUSH rule 3 x 399 [8,4]
+  CRUSH rule 3 x 400 [8,1]
+  CRUSH rule 3 x 401 [1,4]
+  CRUSH rule 3 x 402 [8,4]
+  CRUSH rule 3 x 403 [1,4]
+  CRUSH rule 3 x 404 [4,2]
+  CRUSH rule 3 x 405 [8,4]
+  CRUSH rule 3 x 406 [2,8]
+  CRUSH rule 3 x 407 [2,8]
+  CRUSH rule 3 x 408 [4,1]
+  CRUSH rule 3 x 409 [8,4]
+  CRUSH rule 3 x 410 [8,4]
+  CRUSH rule 3 x 411 [2,8]
+  CRUSH rule 3 x 412 [2]
+  CRUSH rule 3 x 413 [2]
+  CRUSH rule 3 x 414 [4,1]
+  CRUSH rule 3 x 415 [2,8]
+  CRUSH rule 3 x 416 [2,8]
+  CRUSH rule 3 x 417 [8,2]
+  CRUSH rule 3 x 418 [8,1]
+  CRUSH rule 3 x 419 [8,4]
+  CRUSH rule 3 x 420 [1,4]
+  CRUSH rule 3 x 421 [8,4]
+  CRUSH rule 3 x 422 [6]
+  CRUSH rule 3 x 423 [2,4]
+  CRUSH rule 3 x 424 [8]
+  CRUSH rule 3 x 425 [1]
+  CRUSH rule 3 x 426 [8,2]
+  CRUSH rule 3 x 427 [1,8]
+  CRUSH rule 3 x 428 [4,8]
+  CRUSH rule 3 x 429 [4,8]
+  CRUSH rule 3 x 430 [4,8]
+  CRUSH rule 3 x 431 [4,2]
+  CRUSH rule 3 x 432 [8,1]
+  CRUSH rule 3 x 433 [8]
+  CRUSH rule 3 x 434 [2]
+  CRUSH rule 3 x 435 [2]
+  CRUSH rule 3 x 436 [4,1]
+  CRUSH rule 3 x 437 [8]
+  CRUSH rule 3 x 438 [2,4]
+  CRUSH rule 3 x 439 [1]
+  CRUSH rule 3 x 440 [2,8]
+  CRUSH rule 3 x 441 [4,8]
+  CRUSH rule 3 x 442 [2]
+  CRUSH rule 3 x 443 [8,1]
+  CRUSH rule 3 x 444 [8,1]
+  CRUSH rule 3 x 445 [8]
+  CRUSH rule 3 x 446 [1]
+  CRUSH rule 3 x 447 [2,4]
+  CRUSH rule 3 x 448 [8,2]
+  CRUSH rule 3 x 449 [8]
+  CRUSH rule 3 x 450 [1]
+  CRUSH rule 3 x 451 [8,4]
+  CRUSH rule 3 x 452 [8]
+  CRUSH rule 3 x 453 [6]
+  CRUSH rule 3 x 454 [8]
+  CRUSH rule 3 x 455 [2,8]
+  CRUSH rule 3 x 456 [8,2]
+  CRUSH rule 3 x 457 [8,1]
+  CRUSH rule 3 x 458 [2,8]
+  CRUSH rule 3 x 459 [2,8]
+  CRUSH rule 3 x 460 [8]
+  CRUSH rule 3 x 461 [8]
+  CRUSH rule 3 x 462 [8,1]
+  CRUSH rule 3 x 463 [8,2]
+  CRUSH rule 3 x 464 [8,4]
+  CRUSH rule 3 x 465 [6,1]
+  CRUSH rule 3 x 466 [8]
+  CRUSH rule 3 x 467 [8]
+  CRUSH rule 3 x 468 [8,2]
+  CRUSH rule 3 x 469 [8,2]
+  CRUSH rule 3 x 470 [4,1]
+  CRUSH rule 3 x 471 [1,8]
+  CRUSH rule 3 x 472 [1]
+  CRUSH rule 3 x 473 [1,4]
+  CRUSH rule 3 x 474 [8,1]
+  CRUSH rule 3 x 475 [8,2]
+  CRUSH rule 3 x 476 [4,8]
+  CRUSH rule 3 x 477 [4,8]
+  CRUSH rule 3 x 478 [8,1]
+  CRUSH rule 3 x 479 [2]
+  CRUSH rule 3 x 480 [1,8]
+  CRUSH rule 3 x 481 [2,4]
+  CRUSH rule 3 x 482 [8]
+  CRUSH rule 3 x 483 [2,8]
+  CRUSH rule 3 x 484 [1,8]
+  CRUSH rule 3 x 485 [8]
+  CRUSH rule 3 x 486 [4,1]
+  CRUSH rule 3 x 487 [1]
+  CRUSH rule 3 x 488 [8]
+  CRUSH rule 3 x 489 [2,8]
+  CRUSH rule 3 x 490 [6]
+  CRUSH rule 3 x 491 [1,8]
+  CRUSH rule 3 x 492 [8]
+  CRUSH rule 3 x 493 [2,8]
+  CRUSH rule 3 x 494 [1,8]
+  CRUSH rule 3 x 495 [4,1]
+  CRUSH rule 3 x 496 [8,4]
+  CRUSH rule 3 x 497 [4,8]
+  CRUSH rule 3 x 498 [2,4]
+  CRUSH rule 3 x 499 [8,4]
+  CRUSH rule 3 x 500 [4,8]
+  CRUSH rule 3 x 501 [2,8]
+  CRUSH rule 3 x 502 [6,1]
+  CRUSH rule 3 x 503 [2]
+  CRUSH rule 3 x 504 [8]
+  CRUSH rule 3 x 505 [1,8]
+  CRUSH rule 3 x 506 [4,1]
+  CRUSH rule 3 x 507 [8,2]
+  CRUSH rule 3 x 508 [1]
+  CRUSH rule 3 x 509 [8]
+  CRUSH rule 3 x 510 [8,2]
+  CRUSH rule 3 x 511 [4,8]
+  CRUSH rule 3 x 512 [8,2]
+  CRUSH rule 3 x 513 [8,2]
+  CRUSH rule 3 x 514 [8]
+  CRUSH rule 3 x 515 [8,4]
+  CRUSH rule 3 x 516 [4,1]
+  CRUSH rule 3 x 517 [8,2]
+  CRUSH rule 3 x 518 [4,8]
+  CRUSH rule 3 x 519 [8,4]
+  CRUSH rule 3 x 520 [2,8]
+  CRUSH rule 3 x 521 [8,2]
+  CRUSH rule 3 x 522 [8,1]
+  CRUSH rule 3 x 523 [4,1]
+  CRUSH rule 3 x 524 [2]
+  CRUSH rule 3 x 525 [2]
+  CRUSH rule 3 x 526 [1]
+  CRUSH rule 3 x 527 [1,4]
+  CRUSH rule 3 x 528 [2]
+  CRUSH rule 3 x 529 [4,8]
+  CRUSH rule 3 x 530 [8]
+  CRUSH rule 3 x 531 [8,1]
+  CRUSH rule 3 x 532 [6,4]
+  CRUSH rule 3 x 533 [4,8]
+  CRUSH rule 3 x 534 [8]
+  CRUSH rule 3 x 535 [8,1]
+  CRUSH rule 3 x 536 [8,2]
+  CRUSH rule 3 x 537 [4,8]
+  CRUSH rule 3 x 538 [8,4]
+  CRUSH rule 3 x 539 [8]
+  CRUSH rule 3 x 540 [1,8]
+  CRUSH rule 3 x 541 [2,4]
+  CRUSH rule 3 x 542 [2]
+  CRUSH rule 3 x 543 [8,2]
+  CRUSH rule 3 x 544 [4,8]
+  CRUSH rule 3 x 545 [8]
+  CRUSH rule 3 x 546 [8,2]
+  CRUSH rule 3 x 547 [8,1]
+  CRUSH rule 3 x 548 [4,1]
+  CRUSH rule 3 x 549 [8]
+  CRUSH rule 3 x 550 [2,4]
+  CRUSH rule 3 x 551 [8]
+  CRUSH rule 3 x 552 [4,8]
+  CRUSH rule 3 x 553 [2]
+  CRUSH rule 3 x 554 [1,8]
+  CRUSH rule 3 x 555 [4,2]
+  CRUSH rule 3 x 556 [8]
+  CRUSH rule 3 x 557 [8]
+  CRUSH rule 3 x 558 [4,2]
+  CRUSH rule 3 x 559 [1]
+  CRUSH rule 3 x 560 [8]
+  CRUSH rule 3 x 561 [8,4]
+  CRUSH rule 3 x 562 [4,1]
+  CRUSH rule 3 x 563 [2,8]
+  CRUSH rule 3 x 564 [1]
+  CRUSH rule 3 x 565 [4,8]
+  CRUSH rule 3 x 566 [4,8]
+  CRUSH rule 3 x 567 [4,8]
+  CRUSH rule 3 x 568 [8]
+  CRUSH rule 3 x 569 [4,1]
+  CRUSH rule 3 x 570 [1]
+  CRUSH rule 3 x 571 [6]
+  CRUSH rule 3 x 572 [4,2]
+  CRUSH rule 3 x 573 [1]
+  CRUSH rule 3 x 574 [2]
+  CRUSH rule 3 x 575 [8,1]
+  CRUSH rule 3 x 576 [4,8]
+  CRUSH rule 3 x 577 [8,2]
+  CRUSH rule 3 x 578 [8,1]
+  CRUSH rule 3 x 579 [4,2]
+  CRUSH rule 3 x 580 [1]
+  CRUSH rule 3 x 581 [8,1]
+  CRUSH rule 3 x 582 [2,8]
+  CRUSH rule 3 x 583 [8,2]
+  CRUSH rule 3 x 584 [8,1]
+  CRUSH rule 3 x 585 [8,1]
+  CRUSH rule 3 x 586 [1,8]
+  CRUSH rule 3 x 587 [2,4]
+  CRUSH rule 3 x 588 [4,8]
+  CRUSH rule 3 x 589 [8,1]
+  CRUSH rule 3 x 590 [8,2]
+  CRUSH rule 3 x 591 [4,2]
+  CRUSH rule 3 x 592 [2,4]
+  CRUSH rule 3 x 593 [1,8]
+  CRUSH rule 3 x 594 [2,8]
+  CRUSH rule 3 x 595 [8,1]
+  CRUSH rule 3 x 596 [2]
+  CRUSH rule 3 x 597 [1]
+  CRUSH rule 3 x 598 [2]
+  CRUSH rule 3 x 599 [4,1]
+  CRUSH rule 3 x 600 [8,1]
+  CRUSH rule 3 x 601 [1,8]
+  CRUSH rule 3 x 602 [8]
+  CRUSH rule 3 x 603 [1]
+  CRUSH rule 3 x 604 [8]
+  CRUSH rule 3 x 605 [2]
+  CRUSH rule 3 x 606 [2,6]
+  CRUSH rule 3 x 607 [2,4]
+  CRUSH rule 3 x 608 [4,1]
+  CRUSH rule 3 x 609 [4,2]
+  CRUSH rule 3 x 610 [8]
+  CRUSH rule 3 x 611 [1,8]
+  CRUSH rule 3 x 612 [2,8]
+  CRUSH rule 3 x 613 [8,1]
+  CRUSH rule 3 x 614 [8,2]
+  CRUSH rule 3 x 615 [8,2]
+  CRUSH rule 3 x 616 [1,8]
+  CRUSH rule 3 x 617 [8,2]
+  CRUSH rule 3 x 618 [8,4]
+  CRUSH rule 3 x 619 [4,1]
+  CRUSH rule 3 x 620 [1]
+  CRUSH rule 3 x 621 [8]
+  CRUSH rule 3 x 622 [2,4]
+  CRUSH rule 3 x 623 [2,8]
+  CRUSH rule 3 x 624 [4,2]
+  CRUSH rule 3 x 625 [2]
+  CRUSH rule 3 x 626 [8,2]
+  CRUSH rule 3 x 627 [2,8]
+  CRUSH rule 3 x 628 [8,1]
+  CRUSH rule 3 x 629 [2,8]
+  CRUSH rule 3 x 630 [2,8]
+  CRUSH rule 3 x 631 [1,8]
+  CRUSH rule 3 x 632 [8,2]
+  CRUSH rule 3 x 633 [8]
+  CRUSH rule 3 x 634 [1]
+  CRUSH rule 3 x 635 [4,8]
+  CRUSH rule 3 x 636 [1,4]
+  CRUSH rule 3 x 637 [1]
+  CRUSH rule 3 x 638 [8,2]
+  CRUSH rule 3 x 639 [2]
+  CRUSH rule 3 x 640 [2]
+  CRUSH rule 3 x 641 [8,2]
+  CRUSH rule 3 x 642 [2,8]
+  CRUSH rule 3 x 643 [1]
+  CRUSH rule 3 x 644 [8,1]
+  CRUSH rule 3 x 645 [8]
+  CRUSH rule 3 x 646 [8,1]
+  CRUSH rule 3 x 647 [8,1]
+  CRUSH rule 3 x 648 [1,8]
+  CRUSH rule 3 x 649 [4,8]
+  CRUSH rule 3 x 650 [8,4]
+  CRUSH rule 3 x 651 [4,6]
+  CRUSH rule 3 x 652 [4,8]
+  CRUSH rule 3 x 653 [8]
+  CRUSH rule 3 x 654 [6]
+  CRUSH rule 3 x 655 [1,4]
+  CRUSH rule 3 x 656 [8]
+  CRUSH rule 3 x 657 [6,1]
+  CRUSH rule 3 x 658 [8]
+  CRUSH rule 3 x 659 [4,8]
+  CRUSH rule 3 x 660 [8]
+  CRUSH rule 3 x 661 [1,8]
+  CRUSH rule 3 x 662 [2]
+  CRUSH rule 3 x 663 [1,4]
+  CRUSH rule 3 x 664 [1,4]
+  CRUSH rule 3 x 665 [4,6]
+  CRUSH rule 3 x 666 [2,8]
+  CRUSH rule 3 x 667 [1,4]
+  CRUSH rule 3 x 668 [4,8]
+  CRUSH rule 3 x 669 [6,4]
+  CRUSH rule 3 x 670 [4,1]
+  CRUSH rule 3 x 671 [2,8]
+  CRUSH rule 3 x 672 [4,2]
+  CRUSH rule 3 x 673 [4,2]
+  CRUSH rule 3 x 674 [1]
+  CRUSH rule 3 x 675 [1,8]
+  CRUSH rule 3 x 676 [2,4]
+  CRUSH rule 3 x 677 [4,1]
+  CRUSH rule 3 x 678 [2,4]
+  CRUSH rule 3 x 679 [8,2]
+  CRUSH rule 3 x 680 [2]
+  CRUSH rule 3 x 681 [8]
+  CRUSH rule 3 x 682 [1,4]
+  CRUSH rule 3 x 683 [1,4]
+  CRUSH rule 3 x 684 [8,2]
+  CRUSH rule 3 x 685 [8,2]
+  CRUSH rule 3 x 686 [1,4]
+  CRUSH rule 3 x 687 [6]
+  CRUSH rule 3 x 688 [4,8]
+  CRUSH rule 3 x 689 [8,4]
+  CRUSH rule 3 x 690 [8,1]
+  CRUSH rule 3 x 691 [1]
+  CRUSH rule 3 x 692 [8,1]
+  CRUSH rule 3 x 693 [8,4]
+  CRUSH rule 3 x 694 [8,4]
+  CRUSH rule 3 x 695 [2,8]
+  CRUSH rule 3 x 696 [1]
+  CRUSH rule 3 x 697 [8,1]
+  CRUSH rule 3 x 698 [8,2]
+  CRUSH rule 3 x 699 [1,8]
+  CRUSH rule 3 x 700 [1]
+  CRUSH rule 3 x 701 [1]
+  CRUSH rule 3 x 702 [2]
+  CRUSH rule 3 x 703 [8]
+  CRUSH rule 3 x 704 [1,4]
+  CRUSH rule 3 x 705 [8,1]
+  CRUSH rule 3 x 706 [1,4]
+  CRUSH rule 3 x 707 [8,4]
+  CRUSH rule 3 x 708 [4,8]
+  CRUSH rule 3 x 709 [8]
+  CRUSH rule 3 x 710 [8]
+  CRUSH rule 3 x 711 [2,4]
+  CRUSH rule 3 x 712 [2]
+  CRUSH rule 3 x 713 [8,4]
+  CRUSH rule 3 x 714 [2]
+  CRUSH rule 3 x 715 [1]
+  CRUSH rule 3 x 716 [4,8]
+  CRUSH rule 3 x 717 [8,2]
+  CRUSH rule 3 x 718 [8]
+  CRUSH rule 3 x 719 [2,6]
+  CRUSH rule 3 x 720 [8,1]
+  CRUSH rule 3 x 721 [4,6]
+  CRUSH rule 3 x 722 [8]
+  CRUSH rule 3 x 723 [4,2]
+  CRUSH rule 3 x 724 [2,6]
+  CRUSH rule 3 x 725 [1]
+  CRUSH rule 3 x 726 [4,8]
+  CRUSH rule 3 x 727 [4,8]
+  CRUSH rule 3 x 728 [2,8]
+  CRUSH rule 3 x 729 [8]
+  CRUSH rule 3 x 730 [4,8]
+  CRUSH rule 3 x 731 [4,1]
+  CRUSH rule 3 x 732 [1]
+  CRUSH rule 3 x 733 [4,8]
+  CRUSH rule 3 x 734 [8,4]
+  CRUSH rule 3 x 735 [4,8]
+  CRUSH rule 3 x 736 [4,8]
+  CRUSH rule 3 x 737 [1,8]
+  CRUSH rule 3 x 738 [4,1]
+  CRUSH rule 3 x 739 [2,8]
+  CRUSH rule 3 x 740 [1,8]
+  CRUSH rule 3 x 741 [8,2]
+  CRUSH rule 3 x 742 [8,2]
+  CRUSH rule 3 x 743 [8,1]
+  CRUSH rule 3 x 744 [4,8]
+  CRUSH rule 3 x 745 [1]
+  CRUSH rule 3 x 746 [1]
+  CRUSH rule 3 x 747 [8,2]
+  CRUSH rule 3 x 748 [2,8]
+  CRUSH rule 3 x 749 [4,8]
+  CRUSH rule 3 x 750 [1,8]
+  CRUSH rule 3 x 751 [2,8]
+  CRUSH rule 3 x 752 [8,1]
+  CRUSH rule 3 x 753 [8,4]
+  CRUSH rule 3 x 754 [8,4]
+  CRUSH rule 3 x 755 [1,8]
+  CRUSH rule 3 x 756 [8]
+  CRUSH rule 3 x 757 [8,1]
+  CRUSH rule 3 x 758 [8,1]
+  CRUSH rule 3 x 759 [8,4]
+  CRUSH rule 3 x 760 [1,4]
+  CRUSH rule 3 x 761 [2]
+  CRUSH rule 3 x 762 [2,8]
+  CRUSH rule 3 x 763 [8,4]
+  CRUSH rule 3 x 764 [1,8]
+  CRUSH rule 3 x 765 [8]
+  CRUSH rule 3 x 766 [8]
+  CRUSH rule 3 x 767 [1,8]
+  CRUSH rule 3 x 768 [8,4]
+  CRUSH rule 3 x 769 [8,2]
+  CRUSH rule 3 x 770 [8,1]
+  CRUSH rule 3 x 771 [8,2]
+  CRUSH rule 3 x 772 [8,4]
+  CRUSH rule 3 x 773 [4,2]
+  CRUSH rule 3 x 774 [8]
+  CRUSH rule 3 x 775 [8,4]
+  CRUSH rule 3 x 776 [6,2]
+  CRUSH rule 3 x 777 [4,1]
+  CRUSH rule 3 x 778 [1,6]
+  CRUSH rule 3 x 779 [2,8]
+  CRUSH rule 3 x 780 [2,4]
+  CRUSH rule 3 x 781 [8]
+  CRUSH rule 3 x 782 [4,2]
+  CRUSH rule 3 x 783 [8,1]
+  CRUSH rule 3 x 784 [1,4]
+  CRUSH rule 3 x 785 [8,1]
+  CRUSH rule 3 x 786 [8]
+  CRUSH rule 3 x 787 [1,8]
+  CRUSH rule 3 x 788 [8,2]
+  CRUSH rule 3 x 789 [1]
+  CRUSH rule 3 x 790 [8]
+  CRUSH rule 3 x 791 [4,8]
+  CRUSH rule 3 x 792 [4,8]
+  CRUSH rule 3 x 793 [8,2]
+  CRUSH rule 3 x 794 [2,8]
+  CRUSH rule 3 x 795 [1]
+  CRUSH rule 3 x 796 [8]
+  CRUSH rule 3 x 797 [2,4]
+  CRUSH rule 3 x 798 [6,1]
+  CRUSH rule 3 x 799 [4,2]
+  CRUSH rule 3 x 800 [2]
+  CRUSH rule 3 x 801 [4,8]
+  CRUSH rule 3 x 802 [1,8]
+  CRUSH rule 3 x 803 [2]
+  CRUSH rule 3 x 804 [8,2]
+  CRUSH rule 3 x 805 [8]
+  CRUSH rule 3 x 806 [1,4]
+  CRUSH rule 3 x 807 [4,8]
+  CRUSH rule 3 x 808 [8]
+  CRUSH rule 3 x 809 [1]
+  CRUSH rule 3 x 810 [8]
+  CRUSH rule 3 x 811 [8]
+  CRUSH rule 3 x 812 [8,4]
+  CRUSH rule 3 x 813 [8,4]
+  CRUSH rule 3 x 814 [8]
+  CRUSH rule 3 x 815 [4,1]
+  CRUSH rule 3 x 816 [2,8]
+  CRUSH rule 3 x 817 [8]
+  CRUSH rule 3 x 818 [1]
+  CRUSH rule 3 x 819 [1]
+  CRUSH rule 3 x 820 [4,8]
+  CRUSH rule 3 x 821 [4,8]
+  CRUSH rule 3 x 822 [2,4]
+  CRUSH rule 3 x 823 [4,8]
+  CRUSH rule 3 x 824 [8]
+  CRUSH rule 3 x 825 [2,8]
+  CRUSH rule 3 x 826 [8,1]
+  CRUSH rule 3 x 827 [2,6]
+  CRUSH rule 3 x 828 [2]
+  CRUSH rule 3 x 829 [8]
+  CRUSH rule 3 x 830 [2,4]
+  CRUSH rule 3 x 831 [1,8]
+  CRUSH rule 3 x 832 [4,8]
+  CRUSH rule 3 x 833 [2,8]
+  CRUSH rule 3 x 834 [1]
+  CRUSH rule 3 x 835 [8,4]
+  CRUSH rule 3 x 836 [4,8]
+  CRUSH rule 3 x 837 [8,4]
+  CRUSH rule 3 x 838 [6,2]
+  CRUSH rule 3 x 839 [2]
+  CRUSH rule 3 x 840 [8]
+  CRUSH rule 3 x 841 [4,8]
+  CRUSH rule 3 x 842 [2]
+  CRUSH rule 3 x 843 [8,4]
+  CRUSH rule 3 x 844 [8]
+  CRUSH rule 3 x 845 [4,8]
+  CRUSH rule 3 x 846 [4,2]
+  CRUSH rule 3 x 847 [2,8]
+  CRUSH rule 3 x 848 [2,8]
+  CRUSH rule 3 x 849 [4,8]
+  CRUSH rule 3 x 850 [1]
+  CRUSH rule 3 x 851 [6]
+  CRUSH rule 3 x 852 [8,4]
+  CRUSH rule 3 x 853 [6,2]
+  CRUSH rule 3 x 854 [8,2]
+  CRUSH rule 3 x 855 [8]
+  CRUSH rule 3 x 856 [8,4]
+  CRUSH rule 3 x 857 [8]
+  CRUSH rule 3 x 858 [6]
+  CRUSH rule 3 x 859 [8,2]
+  CRUSH rule 3 x 860 [2]
+  CRUSH rule 3 x 861 [8]
+  CRUSH rule 3 x 862 [8,1]
+  CRUSH rule 3 x 863 [8,1]
+  CRUSH rule 3 x 864 [8]
+  CRUSH rule 3 x 865 [8,1]
+  CRUSH rule 3 x 866 [8]
+  CRUSH rule 3 x 867 [8]
+  CRUSH rule 3 x 868 [8]
+  CRUSH rule 3 x 869 [8,4]
+  CRUSH rule 3 x 870 [2]
+  CRUSH rule 3 x 871 [1]
+  CRUSH rule 3 x 872 [1]
+  CRUSH rule 3 x 873 [4,8]
+  CRUSH rule 3 x 874 [2,6]
+  CRUSH rule 3 x 875 [2,8]
+  CRUSH rule 3 x 876 [4,8]
+  CRUSH rule 3 x 877 [8,4]
+  CRUSH rule 3 x 878 [2]
+  CRUSH rule 3 x 879 [8]
+  CRUSH rule 3 x 880 [1]
+  CRUSH rule 3 x 881 [4,8]
+  CRUSH rule 3 x 882 [1]
+  CRUSH rule 3 x 883 [2,4]
+  CRUSH rule 3 x 884 [8,1]
+  CRUSH rule 3 x 885 [4,1]
+  CRUSH rule 3 x 886 [8]
+  CRUSH rule 3 x 887 [8,4]
+  CRUSH rule 3 x 888 [8,1]
+  CRUSH rule 3 x 889 [2,8]
+  CRUSH rule 3 x 890 [8,1]
+  CRUSH rule 3 x 891 [1,8]
+  CRUSH rule 3 x 892 [8,2]
+  CRUSH rule 3 x 893 [2]
+  CRUSH rule 3 x 894 [8,4]
+  CRUSH rule 3 x 895 [4,1]
+  CRUSH rule 3 x 896 [1,8]
+  CRUSH rule 3 x 897 [2]
+  CRUSH rule 3 x 898 [1,4]
+  CRUSH rule 3 x 899 [1,8]
+  CRUSH rule 3 x 900 [4,1]
+  CRUSH rule 3 x 901 [2]
+  CRUSH rule 3 x 902 [8,4]
+  CRUSH rule 3 x 903 [8]
+  CRUSH rule 3 x 904 [8]
+  CRUSH rule 3 x 905 [8,2]
+  CRUSH rule 3 x 906 [1,8]
+  CRUSH rule 3 x 907 [8,2]
+  CRUSH rule 3 x 908 [8]
+  CRUSH rule 3 x 909 [2]
+  CRUSH rule 3 x 910 [8]
+  CRUSH rule 3 x 911 [8]
+  CRUSH rule 3 x 912 [1,8]
+  CRUSH rule 3 x 913 [8,1]
+  CRUSH rule 3 x 914 [6,4]
+  CRUSH rule 3 x 915 [8,2]
+  CRUSH rule 3 x 916 [4,1]
+  CRUSH rule 3 x 917 [1,4]
+  CRUSH rule 3 x 918 [8,2]
+  CRUSH rule 3 x 919 [8,2]
+  CRUSH rule 3 x 920 [8]
+  CRUSH rule 3 x 921 [1]
+  CRUSH rule 3 x 922 [8,4]
+  CRUSH rule 3 x 923 [4,8]
+  CRUSH rule 3 x 924 [1]
+  CRUSH rule 3 x 925 [4,8]
+  CRUSH rule 3 x 926 [2]
+  CRUSH rule 3 x 927 [1,8]
+  CRUSH rule 3 x 928 [8,1]
+  CRUSH rule 3 x 929 [4,2]
+  CRUSH rule 3 x 930 [2]
+  CRUSH rule 3 x 931 [2]
+  CRUSH rule 3 x 932 [4,2]
+  CRUSH rule 3 x 933 [8,4]
+  CRUSH rule 3 x 934 [8]
+  CRUSH rule 3 x 935 [8]
+  CRUSH rule 3 x 936 [1,8]
+  CRUSH rule 3 x 937 [4,8]
+  CRUSH rule 3 x 938 [8,4]
+  CRUSH rule 3 x 939 [2,8]
+  CRUSH rule 3 x 940 [8]
+  CRUSH rule 3 x 941 [2]
+  CRUSH rule 3 x 942 [1,8]
+  CRUSH rule 3 x 943 [8,2]
+  CRUSH rule 3 x 944 [8]
+  CRUSH rule 3 x 945 [8,2]
+  CRUSH rule 3 x 946 [2,8]
+  CRUSH rule 3 x 947 [2]
+  CRUSH rule 3 x 948 [8]
+  CRUSH rule 3 x 949 [6,2]
+  CRUSH rule 3 x 950 [8]
+  CRUSH rule 3 x 951 [8]
+  CRUSH rule 3 x 952 [2,8]
+  CRUSH rule 3 x 953 [1,4]
+  CRUSH rule 3 x 954 [2]
+  CRUSH rule 3 x 955 [8,1]
+  CRUSH rule 3 x 956 [1,8]
+  CRUSH rule 3 x 957 [8,1]
+  CRUSH rule 3 x 958 [8,4]
+  CRUSH rule 3 x 959 [4,1]
+  CRUSH rule 3 x 960 [6]
+  CRUSH rule 3 x 961 [1]
+  CRUSH rule 3 x 962 [8,4]
+  CRUSH rule 3 x 963 [2,4]
+  CRUSH rule 3 x 964 [2]
+  CRUSH rule 3 x 965 [8]
+  CRUSH rule 3 x 966 [4,8]
+  CRUSH rule 3 x 967 [8,4]
+  CRUSH rule 3 x 968 [8,2]
+  CRUSH rule 3 x 969 [8,2]
+  CRUSH rule 3 x 970 [2,8]
+  CRUSH rule 3 x 971 [1,8]
+  CRUSH rule 3 x 972 [1,8]
+  CRUSH rule 3 x 973 [1,8]
+  CRUSH rule 3 x 974 [4,1]
+  CRUSH rule 3 x 975 [4,8]
+  CRUSH rule 3 x 976 [4,8]
+  CRUSH rule 3 x 977 [8,4]
+  CRUSH rule 3 x 978 [8,2]
+  CRUSH rule 3 x 979 [8,2]
+  CRUSH rule 3 x 980 [8,2]
+  CRUSH rule 3 x 981 [8]
+  CRUSH rule 3 x 982 [1]
+  CRUSH rule 3 x 983 [4,8]
+  CRUSH rule 3 x 984 [2,8]
+  CRUSH rule 3 x 985 [2,4]
+  CRUSH rule 3 x 986 [8,4]
+  CRUSH rule 3 x 987 [2]
+  CRUSH rule 3 x 988 [1,4]
+  CRUSH rule 3 x 989 [1,8]
+  CRUSH rule 3 x 990 [1,8]
+  CRUSH rule 3 x 991 [1,4]
+  CRUSH rule 3 x 992 [8,1]
+  CRUSH rule 3 x 993 [2,8]
+  CRUSH rule 3 x 994 [4,8]
+  CRUSH rule 3 x 995 [8,1]
+  CRUSH rule 3 x 996 [8,4]
+  CRUSH rule 3 x 997 [8,4]
+  CRUSH rule 3 x 998 [8,1]
+  CRUSH rule 3 x 999 [1,8]
+  CRUSH rule 3 x 1000 [8,4]
+  CRUSH rule 3 x 1001 [2]
+  CRUSH rule 3 x 1002 [1]
+  CRUSH rule 3 x 1003 [2,8]
+  CRUSH rule 3 x 1004 [8,1]
+  CRUSH rule 3 x 1005 [8,1]
+  CRUSH rule 3 x 1006 [1,8]
+  CRUSH rule 3 x 1007 [1,4]
+  CRUSH rule 3 x 1008 [1,8]
+  CRUSH rule 3 x 1009 [6,4]
+  CRUSH rule 3 x 1010 [1]
+  CRUSH rule 3 x 1011 [4,2]
+  CRUSH rule 3 x 1012 [1]
+  CRUSH rule 3 x 1013 [2]
+  CRUSH rule 3 x 1014 [2,8]
+  CRUSH rule 3 x 1015 [8]
+  CRUSH rule 3 x 1016 [2,4]
+  CRUSH rule 3 x 1017 [6,1]
+  CRUSH rule 3 x 1018 [4,1]
+  CRUSH rule 3 x 1019 [4,8]
+  CRUSH rule 3 x 1020 [1]
+  CRUSH rule 3 x 1021 [2]
+  CRUSH rule 3 x 1022 [1,8]
+  CRUSH rule 3 x 1023 [4,2]
+  rule 3 (choose-set) num_rep 2 result size == 1:\t325/1024 (esc)
+  rule 3 (choose-set) num_rep 2 result size == 2:\t699/1024 (esc)
+  CRUSH rule 3 x 0 [2,4,8]
+  CRUSH rule 3 x 1 [2,8,4]
+  CRUSH rule 3 x 2 [1,8]
+  CRUSH rule 3 x 3 [8,1]
+  CRUSH rule 3 x 4 [4,1,8]
+  CRUSH rule 3 x 5 [8,1]
+  CRUSH rule 3 x 6 [2,8,4]
+  CRUSH rule 3 x 7 [4,8,1]
+  CRUSH rule 3 x 8 [4,8,2]
+  CRUSH rule 3 x 9 [2,4,8]
+  CRUSH rule 3 x 10 [2,8]
+  CRUSH rule 3 x 11 [2,8]
+  CRUSH rule 3 x 12 [2,8]
+  CRUSH rule 3 x 13 [4,8,1]
+  CRUSH rule 3 x 14 [8,1]
+  CRUSH rule 3 x 15 [8,1]
+  CRUSH rule 3 x 16 [8,1]
+  CRUSH rule 3 x 17 [4,1,8]
+  CRUSH rule 3 x 18 [1,6]
+  CRUSH rule 3 x 19 [8,4,2]
+  CRUSH rule 3 x 20 [2,8]
+  CRUSH rule 3 x 21 [8,2]
+  CRUSH rule 3 x 22 [8,1]
+  CRUSH rule 3 x 23 [4,8,2]
+  CRUSH rule 3 x 24 [1,8,4]
+  CRUSH rule 3 x 25 [4,8,1]
+  CRUSH rule 3 x 26 [2,8,4]
+  CRUSH rule 3 x 27 [4,1,8]
+  CRUSH rule 3 x 28 [8,2]
+  CRUSH rule 3 x 29 [8,4,2]
+  CRUSH rule 3 x 30 [4,8,1]
+  CRUSH rule 3 x 31 [8,2]
+  CRUSH rule 3 x 32 [6,2]
+  CRUSH rule 3 x 33 [2,8]
+  CRUSH rule 3 x 34 [2,8]
+  CRUSH rule 3 x 35 [1,8,4]
+  CRUSH rule 3 x 36 [8,1]
+  CRUSH rule 3 x 37 [1,8]
+  CRUSH rule 3 x 38 [4,8,1]
+  CRUSH rule 3 x 39 [8,2]
+  CRUSH rule 3 x 40 [8,2,4]
+  CRUSH rule 3 x 41 [2,8,4]
+  CRUSH rule 3 x 42 [8,2]
+  CRUSH rule 3 x 43 [1,8]
+  CRUSH rule 3 x 44 [1,8,4]
+  CRUSH rule 3 x 45 [8,2,4]
+  CRUSH rule 3 x 46 [2,8]
+  CRUSH rule 3 x 47 [4,1,8]
+  CRUSH rule 3 x 48 [8,1]
+  CRUSH rule 3 x 49 [8,2]
+  CRUSH rule 3 x 50 [4,1,8]
+  CRUSH rule 3 x 51 [8,1]
+  CRUSH rule 3 x 52 [8,2,4]
+  CRUSH rule 3 x 53 [4,8,2]
+  CRUSH rule 3 x 54 [8,4,2]
+  CRUSH rule 3 x 55 [8,2]
+  CRUSH rule 3 x 56 [8,4,1]
+  CRUSH rule 3 x 57 [8,1]
+  CRUSH rule 3 x 58 [1,8]
+  CRUSH rule 3 x 59 [2,8]
+  CRUSH rule 3 x 60 [4,1,8]
+  CRUSH rule 3 x 61 [4,8,2]
+  CRUSH rule 3 x 62 [8,1]
+  CRUSH rule 3 x 63 [8,2]
+  CRUSH rule 3 x 64 [4,1,8]
+  CRUSH rule 3 x 65 [8,4,2]
+  CRUSH rule 3 x 66 [4,8,1]
+  CRUSH rule 3 x 67 [4,2,8]
+  CRUSH rule 3 x 68 [1,8]
+  CRUSH rule 3 x 69 [2,8]
+  CRUSH rule 3 x 70 [8,2]
+  CRUSH rule 3 x 71 [2,8,4]
+  CRUSH rule 3 x 72 [8,2,4]
+  CRUSH rule 3 x 73 [2,8]
+  CRUSH rule 3 x 74 [1,8]
+  CRUSH rule 3 x 75 [4,2,8]
+  CRUSH rule 3 x 76 [4,2,6]
+  CRUSH rule 3 x 77 [8,2,4]
+  CRUSH rule 3 x 78 [1,6]
+  CRUSH rule 3 x 79 [4,1,8]
+  CRUSH rule 3 x 80 [2,4,8]
+  CRUSH rule 3 x 81 [2,8]
+  CRUSH rule 3 x 82 [6,2]
+  CRUSH rule 3 x 83 [2,8]
+  CRUSH rule 3 x 84 [8,2]
+  CRUSH rule 3 x 85 [4,8,2]
+  CRUSH rule 3 x 86 [2,8]
+  CRUSH rule 3 x 87 [2,8,4]
+  CRUSH rule 3 x 88 [1,8]
+  CRUSH rule 3 x 89 [1,8]
+  CRUSH rule 3 x 90 [8,4,2]
+  CRUSH rule 3 x 91 [4,8,2]
+  CRUSH rule 3 x 92 [1,8]
+  CRUSH rule 3 x 93 [8,4,1]
+  CRUSH rule 3 x 94 [1,8]
+  CRUSH rule 3 x 95 [8,1]
+  CRUSH rule 3 x 96 [8,2]
+  CRUSH rule 3 x 97 [8,1]
+  CRUSH rule 3 x 98 [2,8]
+  CRUSH rule 3 x 99 [2,8]
+  CRUSH rule 3 x 100 [1,8,4]
+  CRUSH rule 3 x 101 [8,1]
+  CRUSH rule 3 x 102 [2,8]
+  CRUSH rule 3 x 103 [8,1]
+  CRUSH rule 3 x 104 [8,4,1]
+  CRUSH rule 3 x 105 [2,4,8]
+  CRUSH rule 3 x 106 [1,8,4]
+  CRUSH rule 3 x 107 [1,8]
+  CRUSH rule 3 x 108 [8,1]
+  CRUSH rule 3 x 109 [1,4,8]
+  CRUSH rule 3 x 110 [4,2,8]
+  CRUSH rule 3 x 111 [2,4,8]
+  CRUSH rule 3 x 112 [2,8]
+  CRUSH rule 3 x 113 [8,1]
+  CRUSH rule 3 x 114 [8,4,2]
+  CRUSH rule 3 x 115 [8,2,4]
+  CRUSH rule 3 x 116 [1,8]
+  CRUSH rule 3 x 117 [6,1]
+  CRUSH rule 3 x 118 [2,8]
+  CRUSH rule 3 x 119 [8,1]
+  CRUSH rule 3 x 120 [2,4,8]
+  CRUSH rule 3 x 121 [2,8,4]
+  CRUSH rule 3 x 122 [8,2]
+  CRUSH rule 3 x 123 [2,8]
+  CRUSH rule 3 x 124 [2,8]
+  CRUSH rule 3 x 125 [1,8,4]
+  CRUSH rule 3 x 126 [1,8]
+  CRUSH rule 3 x 127 [4,8,1]
+  CRUSH rule 3 x 128 [8,1]
+  CRUSH rule 3 x 129 [2,4,8]
+  CRUSH rule 3 x 130 [4,8,1]
+  CRUSH rule 3 x 131 [1,4,8]
+  CRUSH rule 3 x 132 [1,8]
+  CRUSH rule 3 x 133 [8,1]
+  CRUSH rule 3 x 134 [1,8,4]
+  CRUSH rule 3 x 135 [4,8,2]
+  CRUSH rule 3 x 136 [2,4,8]
+  CRUSH rule 3 x 137 [8,4,1]
+  CRUSH rule 3 x 138 [8,4,2]
+  CRUSH rule 3 x 139 [4,2,8]
+  CRUSH rule 3 x 140 [1,8,4]
+  CRUSH rule 3 x 141 [8,2]
+  CRUSH rule 3 x 142 [4,2,8]
+  CRUSH rule 3 x 143 [4,8,1]
+  CRUSH rule 3 x 144 [8,2]
+  CRUSH rule 3 x 145 [8,2]
+  CRUSH rule 3 x 146 [2,8]
+  CRUSH rule 3 x 147 [2,8,4]
+  CRUSH rule 3 x 148 [4,1,8]
+  CRUSH rule 3 x 149 [4,8,2]
+  CRUSH rule 3 x 150 [1,8]
+  CRUSH rule 3 x 151 [8,1]
+  CRUSH rule 3 x 152 [8,2]
+  CRUSH rule 3 x 153 [8,4,1]
+  CRUSH rule 3 x 154 [4,2,8]
+  CRUSH rule 3 x 155 [4,8,1]
+  CRUSH rule 3 x 156 [4,2,8]
+  CRUSH rule 3 x 157 [1,8]
+  CRUSH rule 3 x 158 [2,8,4]
+  CRUSH rule 3 x 159 [8,2,4]
+  CRUSH rule 3 x 160 [2,8,4]
+  CRUSH rule 3 x 161 [1,4,8]
+  CRUSH rule 3 x 162 [1,8]
+  CRUSH rule 3 x 163 [4,8,2]
+  CRUSH rule 3 x 164 [8,2]
+  CRUSH rule 3 x 165 [8,2,4]
+  CRUSH rule 3 x 166 [2,8]
+  CRUSH rule 3 x 167 [1,8,4]
+  CRUSH rule 3 x 168 [4,2,8]
+  CRUSH rule 3 x 169 [2,8,4]
+  CRUSH rule 3 x 170 [1,8]
+  CRUSH rule 3 x 171 [8,4,2]
+  CRUSH rule 3 x 172 [1,8]
+  CRUSH rule 3 x 173 [8,4,1]
+  CRUSH rule 3 x 174 [1,8]
+  CRUSH rule 3 x 175 [8,1]
+  CRUSH rule 3 x 176 [2,8]
+  CRUSH rule 3 x 177 [1,8]
+  CRUSH rule 3 x 178 [4,2,8]
+  CRUSH rule 3 x 179 [1,8]
+  CRUSH rule 3 x 180 [8,2]
+  CRUSH rule 3 x 181 [8,2,4]
+  CRUSH rule 3 x 182 [8,1]
+  CRUSH rule 3 x 183 [8,4,1]
+  CRUSH rule 3 x 184 [4,8,1]
+  CRUSH rule 3 x 185 [8,1,4]
+  CRUSH rule 3 x 186 [2,4,8]
+  CRUSH rule 3 x 187 [1,8]
+  CRUSH rule 3 x 188 [1,8,4]
+  CRUSH rule 3 x 189 [1,8,4]
+  CRUSH rule 3 x 190 [1,8]
+  CRUSH rule 3 x 191 [8,1,4]
+  CRUSH rule 3 x 192 [4,2,8]
+  CRUSH rule 3 x 193 [4,2,8]
+  CRUSH rule 3 x 194 [1,8]
+  CRUSH rule 3 x 195 [8,4,2]
+  CRUSH rule 3 x 196 [8,1]
+  CRUSH rule 3 x 197 [8,4,1]
+  CRUSH rule 3 x 198 [2,8]
+  CRUSH rule 3 x 199 [1,4,8]
+  CRUSH rule 3 x 200 [1,8]
+  CRUSH rule 3 x 201 [8,1,4]
+  CRUSH rule 3 x 202 [8,1]
+  CRUSH rule 3 x 203 [8,1]
+  CRUSH rule 3 x 204 [2,4,8]
+  CRUSH rule 3 x 205 [1,8]
+  CRUSH rule 3 x 206 [1,8,4]
+  CRUSH rule 3 x 207 [2,8]
+  CRUSH rule 3 x 208 [8,2]
+  CRUSH rule 3 x 209 [1,8]
+  CRUSH rule 3 x 210 [1,4,8]
+  CRUSH rule 3 x 211 [4,2,8]
+  CRUSH rule 3 x 212 [8,1]
+  CRUSH rule 3 x 213 [8,4,1]
+  CRUSH rule 3 x 214 [8,1]
+  CRUSH rule 3 x 215 [8,1]
+  CRUSH rule 3 x 216 [2,8]
+  CRUSH rule 3 x 217 [1,8,4]
+  CRUSH rule 3 x 218 [2,8]
+  CRUSH rule 3 x 219 [8,2]
+  CRUSH rule 3 x 220 [4,8,1]
+  CRUSH rule 3 x 221 [8,1]
+  CRUSH rule 3 x 222 [8,1]
+  CRUSH rule 3 x 223 [1,8]
+  CRUSH rule 3 x 224 [1,4,8]
+  CRUSH rule 3 x 225 [8,2]
+  CRUSH rule 3 x 226 [8,2,4]
+  CRUSH rule 3 x 227 [4,1,8]
+  CRUSH rule 3 x 228 [8,1]
+  CRUSH rule 3 x 229 [4,8,1]
+  CRUSH rule 3 x 230 [4,8,1]
+  CRUSH rule 3 x 231 [4,8,1]
+  CRUSH rule 3 x 232 [2,8,4]
+  CRUSH rule 3 x 233 [8,2]
+  CRUSH rule 3 x 234 [1,8]
+  CRUSH rule 3 x 235 [4,8,1]
+  CRUSH rule 3 x 236 [2,8]
+  CRUSH rule 3 x 237 [4,8,2]
+  CRUSH rule 3 x 238 [2,8]
+  CRUSH rule 3 x 239 [8,2]
+  CRUSH rule 3 x 240 [4,8,2]
+  CRUSH rule 3 x 241 [1,8]
+  CRUSH rule 3 x 242 [2,8]
+  CRUSH rule 3 x 243 [8,2]
+  CRUSH rule 3 x 244 [4,8,2]
+  CRUSH rule 3 x 245 [8,2]
+  CRUSH rule 3 x 246 [1,8]
+  CRUSH rule 3 x 247 [8,1]
+  CRUSH rule 3 x 248 [8,1,4]
+  CRUSH rule 3 x 249 [2,8]
+  CRUSH rule 3 x 250 [2,4,8]
+  CRUSH rule 3 x 251 [2,8]
+  CRUSH rule 3 x 252 [4,8,2]
+  CRUSH rule 3 x 253 [2,8]
+  CRUSH rule 3 x 254 [4,2,8]
+  CRUSH rule 3 x 255 [1,8]
+  CRUSH rule 3 x 256 [4,8,1]
+  CRUSH rule 3 x 257 [2,6,4]
+  CRUSH rule 3 x 258 [4,2,8]
+  CRUSH rule 3 x 259 [6,1]
+  CRUSH rule 3 x 260 [8,1]
+  CRUSH rule 3 x 261 [8,2]
+  CRUSH rule 3 x 262 [8,2]
+  CRUSH rule 3 x 263 [8,1,4]
+  CRUSH rule 3 x 264 [8,1]
+  CRUSH rule 3 x 265 [8,2]
+  CRUSH rule 3 x 266 [8,2,4]
+  CRUSH rule 3 x 267 [2,8]
+  CRUSH rule 3 x 268 [1,8]
+  CRUSH rule 3 x 269 [1,8,4]
+  CRUSH rule 3 x 270 [4,1,8]
+  CRUSH rule 3 x 271 [8,4,2]
+  CRUSH rule 3 x 272 [2,8,4]
+  CRUSH rule 3 x 273 [4,2,8]
+  CRUSH rule 3 x 274 [8,4,2]
+  CRUSH rule 3 x 275 [4,8,1]
+  CRUSH rule 3 x 276 [8,1,4]
+  CRUSH rule 3 x 277 [8,1]
+  CRUSH rule 3 x 278 [8,2,4]
+  CRUSH rule 3 x 279 [8,4,1]
+  CRUSH rule 3 x 280 [2,8,4]
+  CRUSH rule 3 x 281 [8,1]
+  CRUSH rule 3 x 282 [2,8]
+  CRUSH rule 3 x 283 [8,1]
+  CRUSH rule 3 x 284 [8,1]
+  CRUSH rule 3 x 285 [4,8,2]
+  CRUSH rule 3 x 286 [2,8,4]
+  CRUSH rule 3 x 287 [1,8]
+  CRUSH rule 3 x 288 [8,1,4]
+  CRUSH rule 3 x 289 [4,8,2]
+  CRUSH rule 3 x 290 [1,4,8]
+  CRUSH rule 3 x 291 [1,4,8]
+  CRUSH rule 3 x 292 [8,1,4]
+  CRUSH rule 3 x 293 [8,2]
+  CRUSH rule 3 x 294 [8,4,2]
+  CRUSH rule 3 x 295 [4,8,1]
+  CRUSH rule 3 x 296 [4,1,8]
+  CRUSH rule 3 x 297 [8,2,4]
+  CRUSH rule 3 x 298 [1,8]
+  CRUSH rule 3 x 299 [2,8]
+  CRUSH rule 3 x 300 [8,1]
+  CRUSH rule 3 x 301 [1,8]
+  CRUSH rule 3 x 302 [1,8]
+  CRUSH rule 3 x 303 [8,4,1]
+  CRUSH rule 3 x 304 [2,8]
+  CRUSH rule 3 x 305 [8,1]
+  CRUSH rule 3 x 306 [1,8]
+  CRUSH rule 3 x 307 [2,8]
+  CRUSH rule 3 x 308 [2,8,4]
+  CRUSH rule 3 x 309 [8,2]
+  CRUSH rule 3 x 310 [4,1,6]
+  CRUSH rule 3 x 311 [4,8,2]
+  CRUSH rule 3 x 312 [2,8]
+  CRUSH rule 3 x 313 [4,1,8]
+  CRUSH rule 3 x 314 [2,6]
+  CRUSH rule 3 x 315 [2,8]
+  CRUSH rule 3 x 316 [8,1]
+  CRUSH rule 3 x 317 [2,8]
+  CRUSH rule 3 x 318 [8,1]
+  CRUSH rule 3 x 319 [2,8]
+  CRUSH rule 3 x 320 [8,1]
+  CRUSH rule 3 x 321 [1,8]
+  CRUSH rule 3 x 322 [2,6,4]
+  CRUSH rule 3 x 323 [4,8,2]
+  CRUSH rule 3 x 324 [8,2,4]
+  CRUSH rule 3 x 325 [4,8,1]
+  CRUSH rule 3 x 326 [1,6]
+  CRUSH rule 3 x 327 [1,8]
+  CRUSH rule 3 x 328 [8,4,1]
+  CRUSH rule 3 x 329 [4,8,2]
+  CRUSH rule 3 x 330 [4,8,1]
+  CRUSH rule 3 x 331 [2,8]
+  CRUSH rule 3 x 332 [2,8]
+  CRUSH rule 3 x 333 [8,1]
+  CRUSH rule 3 x 334 [8,2]
+  CRUSH rule 3 x 335 [8,2]
+  CRUSH rule 3 x 336 [4,8,2]
+  CRUSH rule 3 x 337 [8,2,4]
+  CRUSH rule 3 x 338 [8,2]
+  CRUSH rule 3 x 339 [8,2]
+  CRUSH rule 3 x 340 [2,8,4]
+  CRUSH rule 3 x 341 [4,1,6]
+  CRUSH rule 3 x 342 [2,8,4]
+  CRUSH rule 3 x 343 [8,1]
+  CRUSH rule 3 x 344 [6,1,4]
+  CRUSH rule 3 x 345 [8,2]
+  CRUSH rule 3 x 346 [8,2,4]
+  CRUSH rule 3 x 347 [4,1,8]
+  CRUSH rule 3 x 348 [8,2,4]
+  CRUSH rule 3 x 349 [1,8]
+  CRUSH rule 3 x 350 [8,1]
+  CRUSH rule 3 x 351 [8,2]
+  CRUSH rule 3 x 352 [1,8,4]
+  CRUSH rule 3 x 353 [8,1]
+  CRUSH rule 3 x 354 [1,8]
+  CRUSH rule 3 x 355 [8,2]
+  CRUSH rule 3 x 356 [4,1,8]
+  CRUSH rule 3 x 357 [8,1,4]
+  CRUSH rule 3 x 358 [2,8,4]
+  CRUSH rule 3 x 359 [6,1,4]
+  CRUSH rule 3 x 360 [2,8]
+  CRUSH rule 3 x 361 [8,4,1]
+  CRUSH rule 3 x 362 [4,1,8]
+  CRUSH rule 3 x 363 [4,2,8]
+  CRUSH rule 3 x 364 [2,8]
+  CRUSH rule 3 x 365 [8,2]
+  CRUSH rule 3 x 366 [8,2]
+  CRUSH rule 3 x 367 [4,2,8]
+  CRUSH rule 3 x 368 [8,4,1]
+  CRUSH rule 3 x 369 [8,2]
+  CRUSH rule 3 x 370 [8,2]
+  CRUSH rule 3 x 371 [1,4,8]
+  CRUSH rule 3 x 372 [1,8]
+  CRUSH rule 3 x 373 [1,8]
+  CRUSH rule 3 x 374 [8,1]
+  CRUSH rule 3 x 375 [8,4,2]
+  CRUSH rule 3 x 376 [8,1,4]
+  CRUSH rule 3 x 377 [1,4,8]
+  CRUSH rule 3 x 378 [1,8]
+  CRUSH rule 3 x 379 [8,2]
+  CRUSH rule 3 x 380 [2,8]
+  CRUSH rule 3 x 381 [1,4,8]
+  CRUSH rule 3 x 382 [1,4,8]
+  CRUSH rule 3 x 383 [4,8,1]
+  CRUSH rule 3 x 384 [8,2,4]
+  CRUSH rule 3 x 385 [8,1]
+  CRUSH rule 3 x 386 [1,8]
+  CRUSH rule 3 x 387 [1,4,8]
+  CRUSH rule 3 x 388 [2,6]
+  CRUSH rule 3 x 389 [1,4,8]
+  CRUSH rule 3 x 390 [4,8,1]
+  CRUSH rule 3 x 391 [4,8,2]
+  CRUSH rule 3 x 392 [1,8,4]
+  CRUSH rule 3 x 393 [2,8]
+  CRUSH rule 3 x 394 [8,2]
+  CRUSH rule 3 x 395 [1,8]
+  CRUSH rule 3 x 396 [4,2,8]
+  CRUSH rule 3 x 397 [2,4,6]
+  CRUSH rule 3 x 398 [2,4,8]
+  CRUSH rule 3 x 399 [8,4,2]
+  CRUSH rule 3 x 400 [8,1,4]
+  CRUSH rule 3 x 401 [1,4,8]
+  CRUSH rule 3 x 402 [8,4,2]
+  CRUSH rule 3 x 403 [1,4,8]
+  CRUSH rule 3 x 404 [4,2,8]
+  CRUSH rule 3 x 405 [8,4,2]
+  CRUSH rule 3 x 406 [2,8]
+  CRUSH rule 3 x 407 [2,8,4]
+  CRUSH rule 3 x 408 [4,1,8]
+  CRUSH rule 3 x 409 [8,4,2]
+  CRUSH rule 3 x 410 [8,4,1]
+  CRUSH rule 3 x 411 [2,8,4]
+  CRUSH rule 3 x 412 [2,6]
+  CRUSH rule 3 x 413 [2,8]
+  CRUSH rule 3 x 414 [4,1,8]
+  CRUSH rule 3 x 415 [2,8]
+  CRUSH rule 3 x 416 [2,8]
+  CRUSH rule 3 x 417 [8,2]
+  CRUSH rule 3 x 418 [8,1,4]
+  CRUSH rule 3 x 419 [8,4,2]
+  CRUSH rule 3 x 420 [1,4,8]
+  CRUSH rule 3 x 421 [8,4,1]
+  CRUSH rule 3 x 422 [6,1]
+  CRUSH rule 3 x 423 [2,4,8]
+  CRUSH rule 3 x 424 [8,1]
+  CRUSH rule 3 x 425 [1,8]
+  CRUSH rule 3 x 426 [8,2]
+  CRUSH rule 3 x 427 [1,8]
+  CRUSH rule 3 x 428 [4,8,1]
+  CRUSH rule 3 x 429 [4,8,1]
+  CRUSH rule 3 x 430 [4,8,2]
+  CRUSH rule 3 x 431 [4,2,8]
+  CRUSH rule 3 x 432 [8,1]
+  CRUSH rule 3 x 433 [8,1]
+  CRUSH rule 3 x 434 [2,8]
+  CRUSH rule 3 x 435 [2,6]
+  CRUSH rule 3 x 436 [4,1,8]
+  CRUSH rule 3 x 437 [8,1]
+  CRUSH rule 3 x 438 [2,4,8]
+  CRUSH rule 3 x 439 [1,8]
+  CRUSH rule 3 x 440 [2,8]
+  CRUSH rule 3 x 441 [4,8,2]
+  CRUSH rule 3 x 442 [2,8]
+  CRUSH rule 3 x 443 [8,1,4]
+  CRUSH rule 3 x 444 [8,1]
+  CRUSH rule 3 x 445 [8,2]
+  CRUSH rule 3 x 446 [1,8]
+  CRUSH rule 3 x 447 [2,4,8]
+  CRUSH rule 3 x 448 [8,2,4]
+  CRUSH rule 3 x 449 [8,2]
+  CRUSH rule 3 x 450 [1,8]
+  CRUSH rule 3 x 451 [8,4,1]
+  CRUSH rule 3 x 452 [8,2]
+  CRUSH rule 3 x 453 [6,2]
+  CRUSH rule 3 x 454 [8,2]
+  CRUSH rule 3 x 455 [2,8,4]
+  CRUSH rule 3 x 456 [8,2]
+  CRUSH rule 3 x 457 [8,1]
+  CRUSH rule 3 x 458 [2,8]
+  CRUSH rule 3 x 459 [2,8,4]
+  CRUSH rule 3 x 460 [8,2]
+  CRUSH rule 3 x 461 [8,2]
+  CRUSH rule 3 x 462 [8,1]
+  CRUSH rule 3 x 463 [8,2]
+  CRUSH rule 3 x 464 [8,4,1]
+  CRUSH rule 3 x 465 [6,1,4]
+  CRUSH rule 3 x 466 [8,2]
+  CRUSH rule 3 x 467 [8,1]
+  CRUSH rule 3 x 468 [8,2,4]
+  CRUSH rule 3 x 469 [8,2]
+  CRUSH rule 3 x 470 [4,1,8]
+  CRUSH rule 3 x 471 [1,8]
+  CRUSH rule 3 x 472 [1,8]
+  CRUSH rule 3 x 473 [1,4,8]
+  CRUSH rule 3 x 474 [8,1]
+  CRUSH rule 3 x 475 [8,2,4]
+  CRUSH rule 3 x 476 [4,8,1]
+  CRUSH rule 3 x 477 [4,8,1]
+  CRUSH rule 3 x 478 [8,1,4]
+  CRUSH rule 3 x 479 [2,8]
+  CRUSH rule 3 x 480 [1,8]
+  CRUSH rule 3 x 481 [2,4,6]
+  CRUSH rule 3 x 482 [8,1]
+  CRUSH rule 3 x 483 [2,8,4]
+  CRUSH rule 3 x 484 [1,8]
+  CRUSH rule 3 x 485 [8,1]
+  CRUSH rule 3 x 486 [4,1,8]
+  CRUSH rule 3 x 487 [1,8]
+  CRUSH rule 3 x 488 [8,1]
+  CRUSH rule 3 x 489 [2,8]
+  CRUSH rule 3 x 490 [6,2]
+  CRUSH rule 3 x 491 [1,8]
+  CRUSH rule 3 x 492 [8,2]
+  CRUSH rule 3 x 493 [2,8]
+  CRUSH rule 3 x 494 [1,8]
+  CRUSH rule 3 x 495 [4,1,6]
+  CRUSH rule 3 x 496 [8,4,1]
+  CRUSH rule 3 x 497 [4,8,2]
+  CRUSH rule 3 x 498 [2,4,8]
+  CRUSH rule 3 x 499 [8,4,2]
+  CRUSH rule 3 x 500 [4,8,1]
+  CRUSH rule 3 x 501 [2,8]
+  CRUSH rule 3 x 502 [6,1]
+  CRUSH rule 3 x 503 [2,8]
+  CRUSH rule 3 x 504 [8,2]
+  CRUSH rule 3 x 505 [1,8]
+  CRUSH rule 3 x 506 [4,1,8]
+  CRUSH rule 3 x 507 [8,2,4]
+  CRUSH rule 3 x 508 [1,8]
+  CRUSH rule 3 x 509 [8,1]
+  CRUSH rule 3 x 510 [8,2]
+  CRUSH rule 3 x 511 [4,8,2]
+  CRUSH rule 3 x 512 [8,2]
+  CRUSH rule 3 x 513 [8,2]
+  CRUSH rule 3 x 514 [8,2]
+  CRUSH rule 3 x 515 [8,4,2]
+  CRUSH rule 3 x 516 [4,1,8]
+  CRUSH rule 3 x 517 [8,2]
+  CRUSH rule 3 x 518 [4,8,1]
+  CRUSH rule 3 x 519 [8,4,1]
+  CRUSH rule 3 x 520 [2,8,4]
+  CRUSH rule 3 x 521 [8,2,4]
+  CRUSH rule 3 x 522 [8,1,4]
+  CRUSH rule 3 x 523 [4,1,8]
+  CRUSH rule 3 x 524 [2,6]
+  CRUSH rule 3 x 525 [2,8]
+  CRUSH rule 3 x 526 [1,8]
+  CRUSH rule 3 x 527 [1,4,6]
+  CRUSH rule 3 x 528 [2,8]
+  CRUSH rule 3 x 529 [4,8,1]
+  CRUSH rule 3 x 530 [8,1]
+  CRUSH rule 3 x 531 [8,1,4]
+  CRUSH rule 3 x 532 [6,4,1]
+  CRUSH rule 3 x 533 [4,8,1]
+  CRUSH rule 3 x 534 [8,2]
+  CRUSH rule 3 x 535 [8,1]
+  CRUSH rule 3 x 536 [8,2]
+  CRUSH rule 3 x 537 [4,8,1]
+  CRUSH rule 3 x 538 [8,4,2]
+  CRUSH rule 3 x 539 [8,2]
+  CRUSH rule 3 x 540 [1,8,4]
+  CRUSH rule 3 x 541 [2,4,8]
+  CRUSH rule 3 x 542 [2,8]
+  CRUSH rule 3 x 543 [8,2]
+  CRUSH rule 3 x 544 [4,8,1]
+  CRUSH rule 3 x 545 [8,1]
+  CRUSH rule 3 x 546 [8,2,4]
+  CRUSH rule 3 x 547 [8,1,4]
+  CRUSH rule 3 x 548 [4,1,8]
+  CRUSH rule 3 x 549 [8,1]
+  CRUSH rule 3 x 550 [2,4,8]
+  CRUSH rule 3 x 551 [8,1]
+  CRUSH rule 3 x 552 [4,8,2]
+  CRUSH rule 3 x 553 [2,8]
+  CRUSH rule 3 x 554 [1,8]
+  CRUSH rule 3 x 555 [4,2,8]
+  CRUSH rule 3 x 556 [8,1]
+  CRUSH rule 3 x 557 [8,2]
+  CRUSH rule 3 x 558 [4,2,8]
+  CRUSH rule 3 x 559 [1,8]
+  CRUSH rule 3 x 560 [8,2]
+  CRUSH rule 3 x 561 [8,4,1]
+  CRUSH rule 3 x 562 [4,1,8]
+  CRUSH rule 3 x 563 [2,8]
+  CRUSH rule 3 x 564 [1,8]
+  CRUSH rule 3 x 565 [4,8,1]
+  CRUSH rule 3 x 566 [4,8,2]
+  CRUSH rule 3 x 567 [4,8,1]
+  CRUSH rule 3 x 568 [8,1]
+  CRUSH rule 3 x 569 [4,1,8]
+  CRUSH rule 3 x 570 [1,8]
+  CRUSH rule 3 x 571 [6,1]
+  CRUSH rule 3 x 572 [4,2,8]
+  CRUSH rule 3 x 573 [1,8]
+  CRUSH rule 3 x 574 [2,8]
+  CRUSH rule 3 x 575 [8,1,4]
+  CRUSH rule 3 x 576 [4,8,1]
+  CRUSH rule 3 x 577 [8,2]
+  CRUSH rule 3 x 578 [8,1]
+  CRUSH rule 3 x 579 [4,2,8]
+  CRUSH rule 3 x 580 [1,8]
+  CRUSH rule 3 x 581 [8,1,4]
+  CRUSH rule 3 x 582 [2,8,4]
+  CRUSH rule 3 x 583 [8,2]
+  CRUSH rule 3 x 584 [8,1,4]
+  CRUSH rule 3 x 585 [8,1,4]
+  CRUSH rule 3 x 586 [1,8,4]
+  CRUSH rule 3 x 587 [2,4,8]
+  CRUSH rule 3 x 588 [4,8,1]
+  CRUSH rule 3 x 589 [8,1]
+  CRUSH rule 3 x 590 [8,2]
+  CRUSH rule 3 x 591 [4,2,8]
+  CRUSH rule 3 x 592 [2,4,8]
+  CRUSH rule 3 x 593 [1,8,4]
+  CRUSH rule 3 x 594 [2,8]
+  CRUSH rule 3 x 595 [8,1]
+  CRUSH rule 3 x 596 [2,8]
+  CRUSH rule 3 x 597 [1,8]
+  CRUSH rule 3 x 598 [2,8]
+  CRUSH rule 3 x 599 [4,1,8]
+  CRUSH rule 3 x 600 [8,1,4]
+  CRUSH rule 3 x 601 [1,8,4]
+  CRUSH rule 3 x 602 [8,2]
+  CRUSH rule 3 x 603 [1,8]
+  CRUSH rule 3 x 604 [8,2]
+  CRUSH rule 3 x 605 [2,8]
+  CRUSH rule 3 x 606 [2,6,4]
+  CRUSH rule 3 x 607 [2,4,8]
+  CRUSH rule 3 x 608 [4,1,8]
+  CRUSH rule 3 x 609 [4,2,8]
+  CRUSH rule 3 x 610 [8,2]
+  CRUSH rule 3 x 611 [1,8]
+  CRUSH rule 3 x 612 [2,8]
+  CRUSH rule 3 x 613 [8,1,4]
+  CRUSH rule 3 x 614 [8,2,4]
+  CRUSH rule 3 x 615 [8,2,4]
+  CRUSH rule 3 x 616 [1,8]
+  CRUSH rule 3 x 617 [8,2,4]
+  CRUSH rule 3 x 618 [8,4,1]
+  CRUSH rule 3 x 619 [4,1,8]
+  CRUSH rule 3 x 620 [1,8]
+  CRUSH rule 3 x 621 [8,1]
+  CRUSH rule 3 x 622 [2,4,8]
+  CRUSH rule 3 x 623 [2,8]
+  CRUSH rule 3 x 624 [4,2,8]
+  CRUSH rule 3 x 625 [2,8]
+  CRUSH rule 3 x 626 [8,2,4]
+  CRUSH rule 3 x 627 [2,8,4]
+  CRUSH rule 3 x 628 [8,1]
+  CRUSH rule 3 x 629 [2,8,4]
+  CRUSH rule 3 x 630 [2,8]
+  CRUSH rule 3 x 631 [1,8,4]
+  CRUSH rule 3 x 632 [8,2]
+  CRUSH rule 3 x 633 [8,2]
+  CRUSH rule 3 x 634 [1,8]
+  CRUSH rule 3 x 635 [4,8,2]
+  CRUSH rule 3 x 636 [1,4,8]
+  CRUSH rule 3 x 637 [1,8]
+  CRUSH rule 3 x 638 [8,2,4]
+  CRUSH rule 3 x 639 [2,8]
+  CRUSH rule 3 x 640 [2,8]
+  CRUSH rule 3 x 641 [8,2]
+  CRUSH rule 3 x 642 [2,8]
+  CRUSH rule 3 x 643 [1,8]
+  CRUSH rule 3 x 644 [8,1]
+  CRUSH rule 3 x 645 [8,2]
+  CRUSH rule 3 x 646 [8,1,4]
+  CRUSH rule 3 x 647 [8,1]
+  CRUSH rule 3 x 648 [1,8]
+  CRUSH rule 3 x 649 [4,8,1]
+  CRUSH rule 3 x 650 [8,4,1]
+  CRUSH rule 3 x 651 [4,6,1]
+  CRUSH rule 3 x 652 [4,8,1]
+  CRUSH rule 3 x 653 [8,2]
+  CRUSH rule 3 x 654 [6,2]
+  CRUSH rule 3 x 655 [1,4,8]
+  CRUSH rule 3 x 656 [8,1]
+  CRUSH rule 3 x 657 [6,1]
+  CRUSH rule 3 x 658 [8,2]
+  CRUSH rule 3 x 659 [4,8,1]
+  CRUSH rule 3 x 660 [8,2]
+  CRUSH rule 3 x 661 [1,8]
+  CRUSH rule 3 x 662 [2,8]
+  CRUSH rule 3 x 663 [1,4,8]
+  CRUSH rule 3 x 664 [1,4,8]
+  CRUSH rule 3 x 665 [4,6,1]
+  CRUSH rule 3 x 666 [2,8]
+  CRUSH rule 3 x 667 [1,4,8]
+  CRUSH rule 3 x 668 [4,8,2]
+  CRUSH rule 3 x 669 [6,4,1]
+  CRUSH rule 3 x 670 [4,1,8]
+  CRUSH rule 3 x 671 [2,8]
+  CRUSH rule 3 x 672 [4,2,8]
+  CRUSH rule 3 x 673 [4,2,8]
+  CRUSH rule 3 x 674 [1,8]
+  CRUSH rule 3 x 675 [1,8,4]
+  CRUSH rule 3 x 676 [2,4,8]
+  CRUSH rule 3 x 677 [4,1,8]
+  CRUSH rule 3 x 678 [2,4,8]
+  CRUSH rule 3 x 679 [8,2]
+  CRUSH rule 3 x 680 [2,8]
+  CRUSH rule 3 x 681 [8,1]
+  CRUSH rule 3 x 682 [1,4,8]
+  CRUSH rule 3 x 683 [1,4,8]
+  CRUSH rule 3 x 684 [8,2,4]
+  CRUSH rule 3 x 685 [8,2,4]
+  CRUSH rule 3 x 686 [1,4,8]
+  CRUSH rule 3 x 687 [6,2]
+  CRUSH rule 3 x 688 [4,8,2]
+  CRUSH rule 3 x 689 [8,4,1]
+  CRUSH rule 3 x 690 [8,1,4]
+  CRUSH rule 3 x 691 [1,8]
+  CRUSH rule 3 x 692 [8,1]
+  CRUSH rule 3 x 693 [8,4,2]
+  CRUSH rule 3 x 694 [8,4,2]
+  CRUSH rule 3 x 695 [2,8,4]
+  CRUSH rule 3 x 696 [1,8]
+  CRUSH rule 3 x 697 [8,1,4]
+  CRUSH rule 3 x 698 [8,2,4]
+  CRUSH rule 3 x 699 [1,8,4]
+  CRUSH rule 3 x 700 [1,6]
+  CRUSH rule 3 x 701 [1,8]
+  CRUSH rule 3 x 702 [2,8]
+  CRUSH rule 3 x 703 [8,1]
+  CRUSH rule 3 x 704 [1,4,8]
+  CRUSH rule 3 x 705 [8,1,4]
+  CRUSH rule 3 x 706 [1,4,8]
+  CRUSH rule 3 x 707 [8,4,1]
+  CRUSH rule 3 x 708 [4,8,1]
+  CRUSH rule 3 x 709 [8,1]
+  CRUSH rule 3 x 710 [8,1]
+  CRUSH rule 3 x 711 [2,4,8]
+  CRUSH rule 3 x 712 [2,8]
+  CRUSH rule 3 x 713 [8,4,1]
+  CRUSH rule 3 x 714 [2,8]
+  CRUSH rule 3 x 715 [1,8]
+  CRUSH rule 3 x 716 [4,8,1]
+  CRUSH rule 3 x 717 [8,2,4]
+  CRUSH rule 3 x 718 [8,1]
+  CRUSH rule 3 x 719 [2,6,4]
+  CRUSH rule 3 x 720 [8,1,4]
+  CRUSH rule 3 x 721 [4,6,1]
+  CRUSH rule 3 x 722 [8,1]
+  CRUSH rule 3 x 723 [4,2,8]
+  CRUSH rule 3 x 724 [2,6]
+  CRUSH rule 3 x 725 [1,8]
+  CRUSH rule 3 x 726 [4,8,2]
+  CRUSH rule 3 x 727 [4,8,2]
+  CRUSH rule 3 x 728 [2,8,4]
+  CRUSH rule 3 x 729 [8,1]
+  CRUSH rule 3 x 730 [4,8,2]
+  CRUSH rule 3 x 731 [4,1,8]
+  CRUSH rule 3 x 732 [1,8]
+  CRUSH rule 3 x 733 [4,8,2]
+  CRUSH rule 3 x 734 [8,4,1]
+  CRUSH rule 3 x 735 [4,8,1]
+  CRUSH rule 3 x 736 [4,8,1]
+  CRUSH rule 3 x 737 [1,8,4]
+  CRUSH rule 3 x 738 [4,1,8]
+  CRUSH rule 3 x 739 [2,8]
+  CRUSH rule 3 x 740 [1,8,4]
+  CRUSH rule 3 x 741 [8,2]
+  CRUSH rule 3 x 742 [8,2]
+  CRUSH rule 3 x 743 [8,1,4]
+  CRUSH rule 3 x 744 [4,8,1]
+  CRUSH rule 3 x 745 [1,8]
+  CRUSH rule 3 x 746 [1,8]
+  CRUSH rule 3 x 747 [8,2]
+  CRUSH rule 3 x 748 [2,8,4]
+  CRUSH rule 3 x 749 [4,8,2]
+  CRUSH rule 3 x 750 [1,8,4]
+  CRUSH rule 3 x 751 [2,8]
+  CRUSH rule 3 x 752 [8,1]
+  CRUSH rule 3 x 753 [8,4,2]
+  CRUSH rule 3 x 754 [8,4,2]
+  CRUSH rule 3 x 755 [1,8,4]
+  CRUSH rule 3 x 756 [8,1]
+  CRUSH rule 3 x 757 [8,1,4]
+  CRUSH rule 3 x 758 [8,1]
+  CRUSH rule 3 x 759 [8,4,2]
+  CRUSH rule 3 x 760 [1,4,8]
+  CRUSH rule 3 x 761 [2,6]
+  CRUSH rule 3 x 762 [2,8]
+  CRUSH rule 3 x 763 [8,4,1]
+  CRUSH rule 3 x 764 [1,8]
+  CRUSH rule 3 x 765 [8,2]
+  CRUSH rule 3 x 766 [8,1]
+  CRUSH rule 3 x 767 [1,8,4]
+  CRUSH rule 3 x 768 [8,4,2]
+  CRUSH rule 3 x 769 [8,2,4]
+  CRUSH rule 3 x 770 [8,1,4]
+  CRUSH rule 3 x 771 [8,2,4]
+  CRUSH rule 3 x 772 [8,4,2]
+  CRUSH rule 3 x 773 [4,2,8]
+  CRUSH rule 3 x 774 [8,1]
+  CRUSH rule 3 x 775 [8,4,2]
+  CRUSH rule 3 x 776 [6,2]
+  CRUSH rule 3 x 777 [4,1,8]
+  CRUSH rule 3 x 778 [1,6,4]
+  CRUSH rule 3 x 779 [2,8]
+  CRUSH rule 3 x 780 [2,4,8]
+  CRUSH rule 3 x 781 [8,2]
+  CRUSH rule 3 x 782 [4,2,8]
+  CRUSH rule 3 x 783 [8,1,4]
+  CRUSH rule 3 x 784 [1,4,8]
+  CRUSH rule 3 x 785 [8,1,4]
+  CRUSH rule 3 x 786 [8,1]
+  CRUSH rule 3 x 787 [1,8,4]
+  CRUSH rule 3 x 788 [8,2,4]
+  CRUSH rule 3 x 789 [1,8]
+  CRUSH rule 3 x 790 [8,1]
+  CRUSH rule 3 x 791 [4,8,2]
+  CRUSH rule 3 x 792 [4,8,1]
+  CRUSH rule 3 x 793 [8,2,4]
+  CRUSH rule 3 x 794 [2,8,4]
+  CRUSH rule 3 x 795 [1,8]
+  CRUSH rule 3 x 796 [8,1]
+  CRUSH rule 3 x 797 [2,4,8]
+  CRUSH rule 3 x 798 [6,1]
+  CRUSH rule 3 x 799 [4,2,8]
+  CRUSH rule 3 x 800 [2,8]
+  CRUSH rule 3 x 801 [4,8,2]
+  CRUSH rule 3 x 802 [1,8,4]
+  CRUSH rule 3 x 803 [2,8]
+  CRUSH rule 3 x 804 [8,2]
+  CRUSH rule 3 x 805 [8,2]
+  CRUSH rule 3 x 806 [1,4,8]
+  CRUSH rule 3 x 807 [4,8,2]
+  CRUSH rule 3 x 808 [8,2]
+  CRUSH rule 3 x 809 [1,8]
+  CRUSH rule 3 x 810 [8,2]
+  CRUSH rule 3 x 811 [8,1]
+  CRUSH rule 3 x 812 [8,4,1]
+  CRUSH rule 3 x 813 [8,4,2]
+  CRUSH rule 3 x 814 [8,2]
+  CRUSH rule 3 x 815 [4,1,8]
+  CRUSH rule 3 x 816 [2,8]
+  CRUSH rule 3 x 817 [8,2]
+  CRUSH rule 3 x 818 [1,8]
+  CRUSH rule 3 x 819 [1,8]
+  CRUSH rule 3 x 820 [4,8,1]
+  CRUSH rule 3 x 821 [4,8,2]
+  CRUSH rule 3 x 822 [2,4,8]
+  CRUSH rule 3 x 823 [4,8,2]
+  CRUSH rule 3 x 824 [8,1]
+  CRUSH rule 3 x 825 [2,8,4]
+  CRUSH rule 3 x 826 [8,1,4]
+  CRUSH rule 3 x 827 [2,6,4]
+  CRUSH rule 3 x 828 [2,8]
+  CRUSH rule 3 x 829 [8,1]
+  CRUSH rule 3 x 830 [2,4,8]
+  CRUSH rule 3 x 831 [1,8]
+  CRUSH rule 3 x 832 [4,8,2]
+  CRUSH rule 3 x 833 [2,8]
+  CRUSH rule 3 x 834 [1,8]
+  CRUSH rule 3 x 835 [8,4,1]
+  CRUSH rule 3 x 836 [4,8,1]
+  CRUSH rule 3 x 837 [8,4,1]
+  CRUSH rule 3 x 838 [6,2,4]
+  CRUSH rule 3 x 839 [2,8]
+  CRUSH rule 3 x 840 [8,2]
+  CRUSH rule 3 x 841 [4,8,1]
+  CRUSH rule 3 x 842 [2,8]
+  CRUSH rule 3 x 843 [8,4,2]
+  CRUSH rule 3 x 844 [8,2]
+  CRUSH rule 3 x 845 [4,8,2]
+  CRUSH rule 3 x 846 [4,2,8]
+  CRUSH rule 3 x 847 [2,8]
+  CRUSH rule 3 x 848 [2,8,4]
+  CRUSH rule 3 x 849 [4,8,1]
+  CRUSH rule 3 x 850 [1,6]
+  CRUSH rule 3 x 851 [6,1]
+  CRUSH rule 3 x 852 [8,4,2]
+  CRUSH rule 3 x 853 [6,2]
+  CRUSH rule 3 x 854 [8,2]
+  CRUSH rule 3 x 855 [8,2]
+  CRUSH rule 3 x 856 [8,4,1]
+  CRUSH rule 3 x 857 [8,1]
+  CRUSH rule 3 x 858 [6,2]
+  CRUSH rule 3 x 859 [8,2,4]
+  CRUSH rule 3 x 860 [2,8]
+  CRUSH rule 3 x 861 [8,1]
+  CRUSH rule 3 x 862 [8,1]
+  CRUSH rule 3 x 863 [8,1]
+  CRUSH rule 3 x 864 [8,1]
+  CRUSH rule 3 x 865 [8,1]
+  CRUSH rule 3 x 866 [8,2]
+  CRUSH rule 3 x 867 [8,1]
+  CRUSH rule 3 x 868 [8,2]
+  CRUSH rule 3 x 869 [8,4,2]
+  CRUSH rule 3 x 870 [2,8]
+  CRUSH rule 3 x 871 [1,8]
+  CRUSH rule 3 x 872 [1,8]
+  CRUSH rule 3 x 873 [4,8,1]
+  CRUSH rule 3 x 874 [2,6]
+  CRUSH rule 3 x 875 [2,8,4]
+  CRUSH rule 3 x 876 [4,8,1]
+  CRUSH rule 3 x 877 [8,4,2]
+  CRUSH rule 3 x 878 [2,8]
+  CRUSH rule 3 x 879 [8,1]
+  CRUSH rule 3 x 880 [1,8]
+  CRUSH rule 3 x 881 [4,8,2]
+  CRUSH rule 3 x 882 [1,8]
+  CRUSH rule 3 x 883 [2,4,8]
+  CRUSH rule 3 x 884 [8,1,4]
+  CRUSH rule 3 x 885 [4,1,8]
+  CRUSH rule 3 x 886 [8,2]
+  CRUSH rule 3 x 887 [8,4,2]
+  CRUSH rule 3 x 888 [8,1]
+  CRUSH rule 3 x 889 [2,8]
+  CRUSH rule 3 x 890 [8,1,4]
+  CRUSH rule 3 x 891 [1,8]
+  CRUSH rule 3 x 892 [8,2,4]
+  CRUSH rule 3 x 893 [2,6]
+  CRUSH rule 3 x 894 [8,4,1]
+  CRUSH rule 3 x 895 [4,1,8]
+  CRUSH rule 3 x 896 [1,8]
+  CRUSH rule 3 x 897 [2,8]
+  CRUSH rule 3 x 898 [1,4,8]
+  CRUSH rule 3 x 899 [1,8]
+  CRUSH rule 3 x 900 [4,1,8]
+  CRUSH rule 3 x 901 [2,8]
+  CRUSH rule 3 x 902 [8,4,2]
+  CRUSH rule 3 x 903 [8,1]
+  CRUSH rule 3 x 904 [8,2]
+  CRUSH rule 3 x 905 [8,2]
+  CRUSH rule 3 x 906 [1,8]
+  CRUSH rule 3 x 907 [8,2]
+  CRUSH rule 3 x 908 [8,1]
+  CRUSH rule 3 x 909 [2,8]
+  CRUSH rule 3 x 910 [8,1]
+  CRUSH rule 3 x 911 [8,1]
+  CRUSH rule 3 x 912 [1,8]
+  CRUSH rule 3 x 913 [8,1,4]
+  CRUSH rule 3 x 914 [6,4,1]
+  CRUSH rule 3 x 915 [8,2]
+  CRUSH rule 3 x 916 [4,1,8]
+  CRUSH rule 3 x 917 [1,4,8]
+  CRUSH rule 3 x 918 [8,2]
+  CRUSH rule 3 x 919 [8,2]
+  CRUSH rule 3 x 920 [8,2]
+  CRUSH rule 3 x 921 [1,8]
+  CRUSH rule 3 x 922 [8,4,1]
+  CRUSH rule 3 x 923 [4,8,2]
+  CRUSH rule 3 x 924 [1,8]
+  CRUSH rule 3 x 925 [4,8,2]
+  CRUSH rule 3 x 926 [2,8]
+  CRUSH rule 3 x 927 [1,8,4]
+  CRUSH rule 3 x 928 [8,1]
+  CRUSH rule 3 x 929 [4,2,8]
+  CRUSH rule 3 x 930 [2,8]
+  CRUSH rule 3 x 931 [2,8]
+  CRUSH rule 3 x 932 [4,2,8]
+  CRUSH rule 3 x 933 [8,4,2]
+  CRUSH rule 3 x 934 [8,2]
+  CRUSH rule 3 x 935 [8,2]
+  CRUSH rule 3 x 936 [1,8]
+  CRUSH rule 3 x 937 [4,8,1]
+  CRUSH rule 3 x 938 [8,4,2]
+  CRUSH rule 3 x 939 [2,8,4]
+  CRUSH rule 3 x 940 [8,2]
+  CRUSH rule 3 x 941 [2,6]
+  CRUSH rule 3 x 942 [1,8]
+  CRUSH rule 3 x 943 [8,2]
+  CRUSH rule 3 x 944 [8,1]
+  CRUSH rule 3 x 945 [8,2,4]
+  CRUSH rule 3 x 946 [2,8,4]
+  CRUSH rule 3 x 947 [2,8]
+  CRUSH rule 3 x 948 [8,1]
+  CRUSH rule 3 x 949 [6,2]
+  CRUSH rule 3 x 950 [8,1]
+  CRUSH rule 3 x 951 [8,2]
+  CRUSH rule 3 x 952 [2,8,4]
+  CRUSH rule 3 x 953 [1,4,8]
+  CRUSH rule 3 x 954 [2,8]
+  CRUSH rule 3 x 955 [8,1,4]
+  CRUSH rule 3 x 956 [1,8,4]
+  CRUSH rule 3 x 957 [8,1,4]
+  CRUSH rule 3 x 958 [8,4,2]
+  CRUSH rule 3 x 959 [4,1,8]
+  CRUSH rule 3 x 960 [6,2]
+  CRUSH rule 3 x 961 [1,8]
+  CRUSH rule 3 x 962 [8,4,2]
+  CRUSH rule 3 x 963 [2,4,8]
+  CRUSH rule 3 x 964 [2,8]
+  CRUSH rule 3 x 965 [8,1]
+  CRUSH rule 3 x 966 [4,8,1]
+  CRUSH rule 3 x 967 [8,4,2]
+  CRUSH rule 3 x 968 [8,2]
+  CRUSH rule 3 x 969 [8,2,4]
+  CRUSH rule 3 x 970 [2,8,4]
+  CRUSH rule 3 x 971 [1,8]
+  CRUSH rule 3 x 972 [1,8]
+  CRUSH rule 3 x 973 [1,8]
+  CRUSH rule 3 x 974 [4,1,8]
+  CRUSH rule 3 x 975 [4,8,1]
+  CRUSH rule 3 x 976 [4,8,2]
+  CRUSH rule 3 x 977 [8,4,2]
+  CRUSH rule 3 x 978 [8,2,4]
+  CRUSH rule 3 x 979 [8,2,4]
+  CRUSH rule 3 x 980 [8,2,4]
+  CRUSH rule 3 x 981 [8,1]
+  CRUSH rule 3 x 982 [1,8]
+  CRUSH rule 3 x 983 [4,8,1]
+  CRUSH rule 3 x 984 [2,8]
+  CRUSH rule 3 x 985 [2,4,8]
+  CRUSH rule 3 x 986 [8,4,2]
+  CRUSH rule 3 x 987 [2,8]
+  CRUSH rule 3 x 988 [1,4,6]
+  CRUSH rule 3 x 989 [1,8]
+  CRUSH rule 3 x 990 [1,8,4]
+  CRUSH rule 3 x 991 [1,4,8]
+  CRUSH rule 3 x 992 [8,1,4]
+  CRUSH rule 3 x 993 [2,8,4]
+  CRUSH rule 3 x 994 [4,8,1]
+  CRUSH rule 3 x 995 [8,1,4]
+  CRUSH rule 3 x 996 [8,4,1]
+  CRUSH rule 3 x 997 [8,4,1]
+  CRUSH rule 3 x 998 [8,1,4]
+  CRUSH rule 3 x 999 [1,8,4]
+  CRUSH rule 3 x 1000 [8,4,1]
+  CRUSH rule 3 x 1001 [2,8]
+  CRUSH rule 3 x 1002 [1,8]
+  CRUSH rule 3 x 1003 [2,8]
+  CRUSH rule 3 x 1004 [8,1,4]
+  CRUSH rule 3 x 1005 [8,1]
+  CRUSH rule 3 x 1006 [1,8,4]
+  CRUSH rule 3 x 1007 [1,4,8]
+  CRUSH rule 3 x 1008 [1,8]
+  CRUSH rule 3 x 1009 [6,4,1]
+  CRUSH rule 3 x 1010 [1,8]
+  CRUSH rule 3 x 1011 [4,2,8]
+  CRUSH rule 3 x 1012 [1,8]
+  CRUSH rule 3 x 1013 [2,8]
+  CRUSH rule 3 x 1014 [2,8,4]
+  CRUSH rule 3 x 1015 [8,2]
+  CRUSH rule 3 x 1016 [2,4,8]
+  CRUSH rule 3 x 1017 [6,1,4]
+  CRUSH rule 3 x 1018 [4,1,8]
+  CRUSH rule 3 x 1019 [4,8,1]
+  CRUSH rule 3 x 1020 [1,8]
+  CRUSH rule 3 x 1021 [2,8]
+  CRUSH rule 3 x 1022 [1,8,4]
+  CRUSH rule 3 x 1023 [4,2,8]
+  rule 3 (choose-set) num_rep 3 result size == 2:\t501/1024 (esc)
+  rule 3 (choose-set) num_rep 3 result size == 3:\t523/1024 (esc)
+  rule 4 (choose-set-two), x = 0..1023, numrep = 2..3
+  CRUSH rule 4 x 0 [2,1]
+  CRUSH rule 4 x 1 [2,8]
+  CRUSH rule 4 x 2 [1]
+  CRUSH rule 4 x 3 [8,1]
+  CRUSH rule 4 x 4 [4]
+  CRUSH rule 4 x 5 [8]
+  CRUSH rule 4 x 6 [2,8]
+  CRUSH rule 4 x 7 [4]
+  CRUSH rule 4 x 8 [4]
+  CRUSH rule 4 x 9 [2,4]
+  CRUSH rule 4 x 10 [2,1]
+  CRUSH rule 4 x 11 [2,8]
+  CRUSH rule 4 x 12 [2,1]
+  CRUSH rule 4 x 13 [4,8]
+  CRUSH rule 4 x 14 [8]
+  CRUSH rule 4 x 15 [8,2]
+  CRUSH rule 4 x 16 [8]
+  CRUSH rule 4 x 17 [4]
+  CRUSH rule 4 x 18 [1]
+  CRUSH rule 4 x 19 [8,4]
+  CRUSH rule 4 x 20 [2]
+  CRUSH rule 4 x 21 [8]
+  CRUSH rule 4 x 22 [8]
+  CRUSH rule 4 x 23 [4,8]
+  CRUSH rule 4 x 24 [1,2]
+  CRUSH rule 4 x 25 [4,8]
+  CRUSH rule 4 x 26 [2,8]
+  CRUSH rule 4 x 27 [4,1]
+  CRUSH rule 4 x 28 [8,2]
+  CRUSH rule 4 x 29 [8,4]
+  CRUSH rule 4 x 30 [4,8]
+  CRUSH rule 4 x 31 [8]
+  CRUSH rule 4 x 32 [6]
+  CRUSH rule 4 x 33 [2,8]
+  CRUSH rule 4 x 34 [2]
+  CRUSH rule 4 x 35 [1,8]
+  CRUSH rule 4 x 36 [8]
+  CRUSH rule 4 x 37 [1]
+  CRUSH rule 4 x 38 [4,8]
+  CRUSH rule 4 x 39 [8]
+  CRUSH rule 4 x 40 [8]
+  CRUSH rule 4 x 41 [2,1]
+  CRUSH rule 4 x 42 []
+  CRUSH rule 4 x 43 [1]
+  CRUSH rule 4 x 44 [1,8]
+  CRUSH rule 4 x 45 [8,2]
+  CRUSH rule 4 x 46 [2]
+  CRUSH rule 4 x 47 [4]
+  CRUSH rule 4 x 48 [8]
+  CRUSH rule 4 x 49 []
+  CRUSH rule 4 x 50 [4]
+  CRUSH rule 4 x 51 [8]
+  CRUSH rule 4 x 52 [8]
+  CRUSH rule 4 x 53 [4]
+  CRUSH rule 4 x 54 [8]
+  CRUSH rule 4 x 55 [8]
+  CRUSH rule 4 x 56 [8,4]
+  CRUSH rule 4 x 57 []
+  CRUSH rule 4 x 58 [1,2]
+  CRUSH rule 4 x 59 [2]
+  CRUSH rule 4 x 60 [4]
+  CRUSH rule 4 x 61 [4,8]
+  CRUSH rule 4 x 62 [8,1]
+  CRUSH rule 4 x 63 [8]
+  CRUSH rule 4 x 64 [4]
+  CRUSH rule 4 x 65 [8,4]
+  CRUSH rule 4 x 66 [4]
+  CRUSH rule 4 x 67 [4,2]
+  CRUSH rule 4 x 68 [1]
+  CRUSH rule 4 x 69 [1]
+  CRUSH rule 4 x 70 [8,2]
+  CRUSH rule 4 x 71 [2,8]
+  CRUSH rule 4 x 72 [8,1]
+  CRUSH rule 4 x 73 [2,8]
+  CRUSH rule 4 x 74 [1,8]
+  CRUSH rule 4 x 75 [4,2]
+  CRUSH rule 4 x 76 [4,1]
+  CRUSH rule 4 x 77 [8,2]
+  CRUSH rule 4 x 78 [1]
+  CRUSH rule 4 x 79 [4]
+  CRUSH rule 4 x 80 [2,4]
+  CRUSH rule 4 x 81 [2,1]
+  CRUSH rule 4 x 82 [6,1]
+  CRUSH rule 4 x 83 [2,8]
+  CRUSH rule 4 x 84 [8,2]
+  CRUSH rule 4 x 85 [4]
+  CRUSH rule 4 x 86 [2,1]
+  CRUSH rule 4 x 87 [2,8]
+  CRUSH rule 4 x 88 [1,6]
+  CRUSH rule 4 x 89 [2]
+  CRUSH rule 4 x 90 [8]
+  CRUSH rule 4 x 91 [4,8]
+  CRUSH rule 4 x 92 [1,8]
+  CRUSH rule 4 x 93 [8,4]
+  CRUSH rule 4 x 94 [1]
+  CRUSH rule 4 x 95 [8]
+  CRUSH rule 4 x 96 [8]
+  CRUSH rule 4 x 97 [8]
+  CRUSH rule 4 x 98 [2,1]
+  CRUSH rule 4 x 99 [2,8]
+  CRUSH rule 4 x 100 [1,8]
+  CRUSH rule 4 x 101 [8]
+  CRUSH rule 4 x 102 [2]
+  CRUSH rule 4 x 103 [8]
+  CRUSH rule 4 x 104 [8,4]
+  CRUSH rule 4 x 105 [2,4]
+  CRUSH rule 4 x 106 [1,8]
+  CRUSH rule 4 x 107 [2]
+  CRUSH rule 4 x 108 [8,2]
+  CRUSH rule 4 x 109 [1,2]
+  CRUSH rule 4 x 110 [4,2]
+  CRUSH rule 4 x 111 [2,1]
+  CRUSH rule 4 x 112 [2,1]
+  CRUSH rule 4 x 113 [8,2]
+  CRUSH rule 4 x 114 [8]
+  CRUSH rule 4 x 115 [8,2]
+  CRUSH rule 4 x 116 [1,2]
+  CRUSH rule 4 x 117 [6]
+  CRUSH rule 4 x 118 [2]
+  CRUSH rule 4 x 119 [8]
+  CRUSH rule 4 x 120 [2,1]
+  CRUSH rule 4 x 121 [2,1]
+  CRUSH rule 4 x 122 [8]
+  CRUSH rule 4 x 123 [2]
+  CRUSH rule 4 x 124 []
+  CRUSH rule 4 x 125 [1,8]
+  CRUSH rule 4 x 126 [2]
+  CRUSH rule 4 x 127 [4,8]
+  CRUSH rule 4 x 128 []
+  CRUSH rule 4 x 129 [2,1]
+  CRUSH rule 4 x 130 [4,8]
+  CRUSH rule 4 x 131 [1,2]
+  CRUSH rule 4 x 132 [1,2]
+  CRUSH rule 4 x 133 [8]
+  CRUSH rule 4 x 134 [1,8]
+  CRUSH rule 4 x 135 [4,8]
+  CRUSH rule 4 x 136 [2,1]
+  CRUSH rule 4 x 137 [8,4]
+  CRUSH rule 4 x 138 [8]
+  CRUSH rule 4 x 139 [4,2]
+  CRUSH rule 4 x 140 [1,8]
+  CRUSH rule 4 x 141 [8]
+  CRUSH rule 4 x 142 [4]
+  CRUSH rule 4 x 143 [4,8]
+  CRUSH rule 4 x 144 [8,1]
+  CRUSH rule 4 x 145 [8]
+  CRUSH rule 4 x 146 [2,8]
+  CRUSH rule 4 x 147 [2,8]
+  CRUSH rule 4 x 148 [4]
+  CRUSH rule 4 x 149 [4,8]
+  CRUSH rule 4 x 150 [1,8]
+  CRUSH rule 4 x 151 []
+  CRUSH rule 4 x 152 [8]
+  CRUSH rule 4 x 153 [8]
+  CRUSH rule 4 x 154 [4,2]
+  CRUSH rule 4 x 155 [4]
+  CRUSH rule 4 x 156 [4]
+  CRUSH rule 4 x 157 [1]
+  CRUSH rule 4 x 158 [2,8]
+  CRUSH rule 4 x 159 [8,2]
+  CRUSH rule 4 x 160 [2,8]
+  CRUSH rule 4 x 161 [1,4]
+  CRUSH rule 4 x 162 [1,8]
+  CRUSH rule 4 x 163 [4,8]
+  CRUSH rule 4 x 164 [8]
+  CRUSH rule 4 x 165 [8,2]
+  CRUSH rule 4 x 166 [2]
+  CRUSH rule 4 x 167 [1,2]
+  CRUSH rule 4 x 168 [4,2]
+  CRUSH rule 4 x 169 [2,8]
+  CRUSH rule 4 x 170 [1,2]
+  CRUSH rule 4 x 171 [8,4]
+  CRUSH rule 4 x 172 [1,8]
+  CRUSH rule 4 x 173 [8,4]
+  CRUSH rule 4 x 174 [1]
+  CRUSH rule 4 x 175 [8,1]
+  CRUSH rule 4 x 176 []
+  CRUSH rule 4 x 177 []
+  CRUSH rule 4 x 178 [4,2]
+  CRUSH rule 4 x 179 [1]
+  CRUSH rule 4 x 180 [8]
+  CRUSH rule 4 x 181 [8,2]
+  CRUSH rule 4 x 182 [8]
+  CRUSH rule 4 x 183 [8]
+  CRUSH rule 4 x 184 [4,8]
+  CRUSH rule 4 x 185 [8]
+  CRUSH rule 4 x 186 [2,1]
+  CRUSH rule 4 x 187 [1,8]
+  CRUSH rule 4 x 188 [1,8]
+  CRUSH rule 4 x 189 [1,8]
+  CRUSH rule 4 x 190 [1]
+  CRUSH rule 4 x 191 [8]
+  CRUSH rule 4 x 192 [4,1]
+  CRUSH rule 4 x 193 [4,2]
+  CRUSH rule 4 x 194 [1]
+  CRUSH rule 4 x 195 [8,4]
+  CRUSH rule 4 x 196 [8]
+  CRUSH rule 4 x 197 [8,4]
+  CRUSH rule 4 x 198 [2]
+  CRUSH rule 4 x 199 [1,4]
+  CRUSH rule 4 x 200 [1]
+  CRUSH rule 4 x 201 [8,1]
+  CRUSH rule 4 x 202 [8]
+  CRUSH rule 4 x 203 [8]
+  CRUSH rule 4 x 204 [2,1]
+  CRUSH rule 4 x 205 [1,8]
+  CRUSH rule 4 x 206 [1,2]
+  CRUSH rule 4 x 207 [2]
+  CRUSH rule 4 x 208 [8,1]
+  CRUSH rule 4 x 209 [1,2]
+  CRUSH rule 4 x 210 [1,2]
+  CRUSH rule 4 x 211 [4]
+  CRUSH rule 4 x 212 [8]
+  CRUSH rule 4 x 213 [8,4]
+  CRUSH rule 4 x 214 []
+  CRUSH rule 4 x 215 [8,1]
+  CRUSH rule 4 x 216 [1]
+  CRUSH rule 4 x 217 [1,2]
+  CRUSH rule 4 x 218 [2,8]
+  CRUSH rule 4 x 219 [8]
+  CRUSH rule 4 x 220 [4,8]
+  CRUSH rule 4 x 221 [8]
+  CRUSH rule 4 x 222 [8]
+  CRUSH rule 4 x 223 [1]
+  CRUSH rule 4 x 224 [1,4]
+  CRUSH rule 4 x 225 [8]
+  CRUSH rule 4 x 226 [8,2]
+  CRUSH rule 4 x 227 [4]
+  CRUSH rule 4 x 228 []
+  CRUSH rule 4 x 229 [4]
+  CRUSH rule 4 x 230 [4,8]
+  CRUSH rule 4 x 231 [4]
+  CRUSH rule 4 x 232 [2,8]
+  CRUSH rule 4 x 233 []
+  CRUSH rule 4 x 234 [1,2]
+  CRUSH rule 4 x 235 [4,8]
+  CRUSH rule 4 x 236 [2]
+  CRUSH rule 4 x 237 [4,8]
+  CRUSH rule 4 x 238 []
+  CRUSH rule 4 x 239 [8,6]
+  CRUSH rule 4 x 240 [4,8]
+  CRUSH rule 4 x 241 [1]
+  CRUSH rule 4 x 242 []
+  CRUSH rule 4 x 243 [8]
+  CRUSH rule 4 x 244 [4,8]
+  CRUSH rule 4 x 245 [8]
+  CRUSH rule 4 x 246 [1]
+  CRUSH rule 4 x 247 [8,2]
+  CRUSH rule 4 x 248 [8,2]
+  CRUSH rule 4 x 249 [2,1]
+  CRUSH rule 4 x 250 [2,1]
+  CRUSH rule 4 x 251 [2]
+  CRUSH rule 4 x 252 [4,8]
+  CRUSH rule 4 x 253 [2]
+  CRUSH rule 4 x 254 [4]
+  CRUSH rule 4 x 255 [1,8]
+  CRUSH rule 4 x 256 [4,8]
+  CRUSH rule 4 x 257 [2,8]
+  CRUSH rule 4 x 258 [4]
+  CRUSH rule 4 x 259 []
+  CRUSH rule 4 x 260 [8]
+  CRUSH rule 4 x 261 [8]
+  CRUSH rule 4 x 262 []
+  CRUSH rule 4 x 263 [8]
+  CRUSH rule 4 x 264 [8]
+  CRUSH rule 4 x 265 [8]
+  CRUSH rule 4 x 266 [8,2]
+  CRUSH rule 4 x 267 [2]
+  CRUSH rule 4 x 268 [1,8]
+  CRUSH rule 4 x 269 [1,8]
+  CRUSH rule 4 x 270 [4,1]
+  CRUSH rule 4 x 271 [8,4]
+  CRUSH rule 4 x 272 [2,8]
+  CRUSH rule 4 x 273 [4]
+  CRUSH rule 4 x 274 [8]
+  CRUSH rule 4 x 275 [4]
+  CRUSH rule 4 x 276 [8,1]
+  CRUSH rule 4 x 277 [8]
+  CRUSH rule 4 x 278 [8]
+  CRUSH rule 4 x 279 [8,4]
+  CRUSH rule 4 x 280 [2,8]
+  CRUSH rule 4 x 281 [8,2]
+  CRUSH rule 4 x 282 [1]
+  CRUSH rule 4 x 283 [8,2]
+  CRUSH rule 4 x 284 [8]
+  CRUSH rule 4 x 285 [4]
+  CRUSH rule 4 x 286 [2,1]
+  CRUSH rule 4 x 287 [1]
+  CRUSH rule 4 x 288 [8,1]
+  CRUSH rule 4 x 289 [4,8]
+  CRUSH rule 4 x 290 [1,4]
+  CRUSH rule 4 x 291 [1,2]
+  CRUSH rule 4 x 292 [8,2]
+  CRUSH rule 4 x 293 [8,1]
+  CRUSH rule 4 x 294 [8,4]
+  CRUSH rule 4 x 295 [4,8]
+  CRUSH rule 4 x 296 [4,1]
+  CRUSH rule 4 x 297 [8,2]
+  CRUSH rule 4 x 298 [1,2]
+  CRUSH rule 4 x 299 [2,1]
+  CRUSH rule 4 x 300 [8]
+  CRUSH rule 4 x 301 [1,8]
+  CRUSH rule 4 x 302 [1]
+  CRUSH rule 4 x 303 [8,4]
+  CRUSH rule 4 x 304 [2,8]
+  CRUSH rule 4 x 305 [8]
+  CRUSH rule 4 x 306 [1,8]
+  CRUSH rule 4 x 307 [2,1]
+  CRUSH rule 4 x 308 [2,8]
+  CRUSH rule 4 x 309 [8]
+  CRUSH rule 4 x 310 [4]
+  CRUSH rule 4 x 311 [4]
+  CRUSH rule 4 x 312 [2,1]
+  CRUSH rule 4 x 313 [4]
+  CRUSH rule 4 x 314 []
+  CRUSH rule 4 x 315 [2,1]
+  CRUSH rule 4 x 316 [8]
+  CRUSH rule 4 x 317 [2,8]
+  CRUSH rule 4 x 318 [8]
+  CRUSH rule 4 x 319 [2]
+  CRUSH rule 4 x 320 [8]
+  CRUSH rule 4 x 321 [1]
+  CRUSH rule 4 x 322 [2,8]
+  CRUSH rule 4 x 323 [4,8]
+  CRUSH rule 4 x 324 [8,1]
+  CRUSH rule 4 x 325 [4,8]
+  CRUSH rule 4 x 326 []
+  CRUSH rule 4 x 327 [1,8]
+  CRUSH rule 4 x 328 [8,4]
+  CRUSH rule 4 x 329 [4,8]
+  CRUSH rule 4 x 330 [4,8]
+  CRUSH rule 4 x 331 [2,8]
+  CRUSH rule 4 x 332 [2,1]
+  CRUSH rule 4 x 333 [8]
+  CRUSH rule 4 x 334 [8]
+  CRUSH rule 4 x 335 [8,1]
+  CRUSH rule 4 x 336 [4,8]
+  CRUSH rule 4 x 337 [8,2]
+  CRUSH rule 4 x 338 [8]
+  CRUSH rule 4 x 339 [8]
+  CRUSH rule 4 x 340 [2,1]
+  CRUSH rule 4 x 341 [4,1]
+  CRUSH rule 4 x 342 [2,8]
+  CRUSH rule 4 x 343 [8]
+  CRUSH rule 4 x 344 [6,2]
+  CRUSH rule 4 x 345 []
+  CRUSH rule 4 x 346 [8,2]
+  CRUSH rule 4 x 347 [4,1]
+  CRUSH rule 4 x 348 [8,2]
+  CRUSH rule 4 x 349 [1,8]
+  CRUSH rule 4 x 350 [8]
+  CRUSH rule 4 x 351 [8]
+  CRUSH rule 4 x 352 [1,2]
+  CRUSH rule 4 x 353 [8]
+  CRUSH rule 4 x 354 [1]
+  CRUSH rule 4 x 355 []
+  CRUSH rule 4 x 356 [4]
+  CRUSH rule 4 x 357 [8,1]
+  CRUSH rule 4 x 358 [2,1]
+  CRUSH rule 4 x 359 [6,8]
+  CRUSH rule 4 x 360 []
+  CRUSH rule 4 x 361 [8,4]
+  CRUSH rule 4 x 362 [4]
+  CRUSH rule 4 x 363 [4,1]
+  CRUSH rule 4 x 364 [2]
+  CRUSH rule 4 x 365 [8]
+  CRUSH rule 4 x 366 [8,2]
+  CRUSH rule 4 x 367 [4]
+  CRUSH rule 4 x 368 [8,4]
+  CRUSH rule 4 x 369 [8]
+  CRUSH rule 4 x 370 [8]
+  CRUSH rule 4 x 371 [1,4]
+  CRUSH rule 4 x 372 [1]
+  CRUSH rule 4 x 373 [1,8]
+  CRUSH rule 4 x 374 [8]
+  CRUSH rule 4 x 375 [8,4]
+  CRUSH rule 4 x 376 [8,1]
+  CRUSH rule 4 x 377 [1,2]
+  CRUSH rule 4 x 378 [1,2]
+  CRUSH rule 4 x 379 [8]
+  CRUSH rule 4 x 380 [2]
+  CRUSH rule 4 x 381 [1,4]
+  CRUSH rule 4 x 382 [1,4]
+  CRUSH rule 4 x 383 [4]
+  CRUSH rule 4 x 384 [8,2]
+  CRUSH rule 4 x 385 [8]
+  CRUSH rule 4 x 386 [1]
+  CRUSH rule 4 x 387 [1,4]
+  CRUSH rule 4 x 388 [1]
+  CRUSH rule 4 x 389 [1,4]
+  CRUSH rule 4 x 390 [4,8]
+  CRUSH rule 4 x 391 [4,8]
+  CRUSH rule 4 x 392 [1,8]
+  CRUSH rule 4 x 393 [2]
+  CRUSH rule 4 x 394 [8]
+  CRUSH rule 4 x 395 [1]
+  CRUSH rule 4 x 396 [4,2]
+  CRUSH rule 4 x 397 [2,1]
+  CRUSH rule 4 x 398 [2,4]
+  CRUSH rule 4 x 399 [8]
+  CRUSH rule 4 x 400 [8,1]
+  CRUSH rule 4 x 401 [1,2]
+  CRUSH rule 4 x 402 [8]
+  CRUSH rule 4 x 403 [1,2]
+  CRUSH rule 4 x 404 [4]
+  CRUSH rule 4 x 405 [8,4]
+  CRUSH rule 4 x 406 [2,1]
+  CRUSH rule 4 x 407 [2,8]
+  CRUSH rule 4 x 408 [4,1]
+  CRUSH rule 4 x 409 [8,4]
+  CRUSH rule 4 x 410 [8]
+  CRUSH rule 4 x 411 [2,1]
+  CRUSH rule 4 x 412 [2]
+  CRUSH rule 4 x 413 [2]
+  CRUSH rule 4 x 414 [4,1]
+  CRUSH rule 4 x 415 [2,8]
+  CRUSH rule 4 x 416 [2,1]
+  CRUSH rule 4 x 417 [8,6]
+  CRUSH rule 4 x 418 [8]
+  CRUSH rule 4 x 419 [8,4]
+  CRUSH rule 4 x 420 [1,4]
+  CRUSH rule 4 x 421 [8]
+  CRUSH rule 4 x 422 [6,8]
+  CRUSH rule 4 x 423 [2,4]
+  CRUSH rule 4 x 424 [8]
+  CRUSH rule 4 x 425 [1]
+  CRUSH rule 4 x 426 [8]
+  CRUSH rule 4 x 427 [1,8]
+  CRUSH rule 4 x 428 [4]
+  CRUSH rule 4 x 429 [4,8]
+  CRUSH rule 4 x 430 [4,8]
+  CRUSH rule 4 x 431 [4]
+  CRUSH rule 4 x 432 [8,1]
+  CRUSH rule 4 x 433 [8]
+  CRUSH rule 4 x 434 [2]
+  CRUSH rule 4 x 435 [2]
+  CRUSH rule 4 x 436 [4,1]
+  CRUSH rule 4 x 437 [8]
+  CRUSH rule 4 x 438 [2,4]
+  CRUSH rule 4 x 439 [1]
+  CRUSH rule 4 x 440 [2,8]
+  CRUSH rule 4 x 441 [4,6]
+  CRUSH rule 4 x 442 [2]
+  CRUSH rule 4 x 443 [8]
+  CRUSH rule 4 x 444 [8,1]
+  CRUSH rule 4 x 445 [8]
+  CRUSH rule 4 x 446 []
+  CRUSH rule 4 x 447 [2,1]
+  CRUSH rule 4 x 448 [8,2]
+  CRUSH rule 4 x 449 [8,6]
+  CRUSH rule 4 x 450 []
+  CRUSH rule 4 x 451 [8]
+  CRUSH rule 4 x 452 [8]
+  CRUSH rule 4 x 453 [6,8]
+  CRUSH rule 4 x 454 [8]
+  CRUSH rule 4 x 455 [2,8]
+  CRUSH rule 4 x 456 [8]
+  CRUSH rule 4 x 457 [8,2]
+  CRUSH rule 4 x 458 [2,8]
+  CRUSH rule 4 x 459 [2,1]
+  CRUSH rule 4 x 460 [8]
+  CRUSH rule 4 x 461 [8]
+  CRUSH rule 4 x 462 [8,1]
+  CRUSH rule 4 x 463 [8]
+  CRUSH rule 4 x 464 [8,4]
+  CRUSH rule 4 x 465 [6,8]
+  CRUSH rule 4 x 466 [8]
+  CRUSH rule 4 x 467 [8]
+  CRUSH rule 4 x 468 [8]
+  CRUSH rule 4 x 469 [8,1]
+  CRUSH rule 4 x 470 [4,2]
+  CRUSH rule 4 x 471 [1,2]
+  CRUSH rule 4 x 472 []
+  CRUSH rule 4 x 473 [1,2]
+  CRUSH rule 4 x 474 [8,1]
+  CRUSH rule 4 x 475 [8]
+  CRUSH rule 4 x 476 [4]
+  CRUSH rule 4 x 477 [4,8]
+  CRUSH rule 4 x 478 [8]
+  CRUSH rule 4 x 479 [2]
+  CRUSH rule 4 x 480 [1,8]
+  CRUSH rule 4 x 481 [2,4]
+  CRUSH rule 4 x 482 []
+  CRUSH rule 4 x 483 [2,1]
+  CRUSH rule 4 x 484 [1,2]
+  CRUSH rule 4 x 485 [8]
+  CRUSH rule 4 x 486 [4,1]
+  CRUSH rule 4 x 487 [1]
+  CRUSH rule 4 x 488 [8]
+  CRUSH rule 4 x 489 [2,8]
+  CRUSH rule 4 x 490 [6]
+  CRUSH rule 4 x 491 [1,2]
+  CRUSH rule 4 x 492 [8]
+  CRUSH rule 4 x 493 [2,1]
+  CRUSH rule 4 x 494 [1,2]
+  CRUSH rule 4 x 495 [4]
+  CRUSH rule 4 x 496 [8,4]
+  CRUSH rule 4 x 497 [4,8]
+  CRUSH rule 4 x 498 [2,4]
+  CRUSH rule 4 x 499 [8,4]
+  CRUSH rule 4 x 500 [4,8]
+  CRUSH rule 4 x 501 [2,8]
+  CRUSH rule 4 x 502 [6,1]
+  CRUSH rule 4 x 503 [2]
+  CRUSH rule 4 x 504 [8]
+  CRUSH rule 4 x 505 [1,8]
+  CRUSH rule 4 x 506 [4]
+  CRUSH rule 4 x 507 [8,1]
+  CRUSH rule 4 x 508 [1,2]
+  CRUSH rule 4 x 509 [8]
+  CRUSH rule 4 x 510 [8,2]
+  CRUSH rule 4 x 511 [4,8]
+  CRUSH rule 4 x 512 [8]
+  CRUSH rule 4 x 513 [8,2]
+  CRUSH rule 4 x 514 []
+  CRUSH rule 4 x 515 [8,4]
+  CRUSH rule 4 x 516 [4,1]
+  CRUSH rule 4 x 517 [8]
+  CRUSH rule 4 x 518 [4,8]
+  CRUSH rule 4 x 519 [8,4]
+  CRUSH rule 4 x 520 [2,8]
+  CRUSH rule 4 x 521 [8]
+  CRUSH rule 4 x 522 [8]
+  CRUSH rule 4 x 523 [4,2]
+  CRUSH rule 4 x 524 [2]
+  CRUSH rule 4 x 525 [2]
+  CRUSH rule 4 x 526 [1]
+  CRUSH rule 4 x 527 [1,2]
+  CRUSH rule 4 x 528 []
+  CRUSH rule 4 x 529 [4,8]
+  CRUSH rule 4 x 530 [8]
+  CRUSH rule 4 x 531 [8,1]
+  CRUSH rule 4 x 532 [6,4]
+  CRUSH rule 4 x 533 [4,8]
+  CRUSH rule 4 x 534 [8]
+  CRUSH rule 4 x 535 [8,6]
+  CRUSH rule 4 x 536 [8]
+  CRUSH rule 4 x 537 [4,8]
+  CRUSH rule 4 x 538 [8]
+  CRUSH rule 4 x 539 [8]
+  CRUSH rule 4 x 540 [1,8]
+  CRUSH rule 4 x 541 [2,4]
+  CRUSH rule 4 x 542 []
+  CRUSH rule 4 x 543 [8,2]
+  CRUSH rule 4 x 544 [4,8]
+  CRUSH rule 4 x 545 [8]
+  CRUSH rule 4 x 546 [8,1]
+  CRUSH rule 4 x 547 [8,2]
+  CRUSH rule 4 x 548 [4,2]
+  CRUSH rule 4 x 549 [8]
+  CRUSH rule 4 x 550 [2,4]
+  CRUSH rule 4 x 551 [8]
+  CRUSH rule 4 x 552 [4]
+  CRUSH rule 4 x 553 [2]
+  CRUSH rule 4 x 554 [1,8]
+  CRUSH rule 4 x 555 [4,1]
+  CRUSH rule 4 x 556 []
+  CRUSH rule 4 x 557 [8]
+  CRUSH rule 4 x 558 [4,1]
+  CRUSH rule 4 x 559 [2]
+  CRUSH rule 4 x 560 [8]
+  CRUSH rule 4 x 561 [8,4]
+  CRUSH rule 4 x 562 [4]
+  CRUSH rule 4 x 563 [2,8]
+  CRUSH rule 4 x 564 [1]
+  CRUSH rule 4 x 565 [4,8]
+  CRUSH rule 4 x 566 [4,8]
+  CRUSH rule 4 x 567 [4,8]
+  CRUSH rule 4 x 568 [8]
+  CRUSH rule 4 x 569 [4]
+  CRUSH rule 4 x 570 [1]
+  CRUSH rule 4 x 571 [8]
+  CRUSH rule 4 x 572 [4]
+  CRUSH rule 4 x 573 [1]
+  CRUSH rule 4 x 574 [2,1]
+  CRUSH rule 4 x 575 [8]
+  CRUSH rule 4 x 576 [4,8]
+  CRUSH rule 4 x 577 [8,2]
+  CRUSH rule 4 x 578 [8]
+  CRUSH rule 4 x 579 [4,1]
+  CRUSH rule 4 x 580 [2]
+  CRUSH rule 4 x 581 [8,2]
+  CRUSH rule 4 x 582 [2,8]
+  CRUSH rule 4 x 583 [8,1]
+  CRUSH rule 4 x 584 [8,1]
+  CRUSH rule 4 x 585 [8,1]
+  CRUSH rule 4 x 586 [1,2]
+  CRUSH rule 4 x 587 [2,4]
+  CRUSH rule 4 x 588 [4]
+  CRUSH rule 4 x 589 [8,1]
+  CRUSH rule 4 x 590 [8,2]
+  CRUSH rule 4 x 591 [4,2]
+  CRUSH rule 4 x 592 [2,1]
+  CRUSH rule 4 x 593 [1,8]
+  CRUSH rule 4 x 594 [2,8]
+  CRUSH rule 4 x 595 [8,1]
+  CRUSH rule 4 x 596 []
+  CRUSH rule 4 x 597 [1]
+  CRUSH rule 4 x 598 [2]
+  CRUSH rule 4 x 599 [4,2]
+  CRUSH rule 4 x 600 [8,1]
+  CRUSH rule 4 x 601 [1,8]
+  CRUSH rule 4 x 602 [8]
+  CRUSH rule 4 x 603 [1]
+  CRUSH rule 4 x 604 [8]
+  CRUSH rule 4 x 605 [2]
+  CRUSH rule 4 x 606 [2,1]
+  CRUSH rule 4 x 607 [2,4]
+  CRUSH rule 4 x 608 [4]
+  CRUSH rule 4 x 609 [4,2]
+  CRUSH rule 4 x 610 [8]
+  CRUSH rule 4 x 611 [1,2]
+  CRUSH rule 4 x 612 [2,1]
+  CRUSH rule 4 x 613 [8,2]
+  CRUSH rule 4 x 614 [8]
+  CRUSH rule 4 x 615 [8]
+  CRUSH rule 4 x 616 [1,8]
+  CRUSH rule 4 x 617 [8,1]
+  CRUSH rule 4 x 618 [8]
+  CRUSH rule 4 x 619 [4,1]
+  CRUSH rule 4 x 620 [1]
+  CRUSH rule 4 x 621 [8]
+  CRUSH rule 4 x 622 [2,4]
+  CRUSH rule 4 x 623 [2,8]
+  CRUSH rule 4 x 624 [4]
+  CRUSH rule 4 x 625 [2]
+  CRUSH rule 4 x 626 [8]
+  CRUSH rule 4 x 627 [2,8]
+  CRUSH rule 4 x 628 [8,2]
+  CRUSH rule 4 x 629 [2,8]
+  CRUSH rule 4 x 630 [2,8]
+  CRUSH rule 4 x 631 [1,8]
+  CRUSH rule 4 x 632 [8,2]
+  CRUSH rule 4 x 633 [8]
+  CRUSH rule 4 x 634 [1]
+  CRUSH rule 4 x 635 [4,8]
+  CRUSH rule 4 x 636 [1,4]
+  CRUSH rule 4 x 637 [1]
+  CRUSH rule 4 x 638 [8]
+  CRUSH rule 4 x 639 [2]
+  CRUSH rule 4 x 640 [1]
+  CRUSH rule 4 x 641 [8,2]
+  CRUSH rule 4 x 642 [2,1]
+  CRUSH rule 4 x 643 []
+  CRUSH rule 4 x 644 [8,1]
+  CRUSH rule 4 x 645 []
+  CRUSH rule 4 x 646 [8,1]
+  CRUSH rule 4 x 647 [8,1]
+  CRUSH rule 4 x 648 [1,8]
+  CRUSH rule 4 x 649 [4,8]
+  CRUSH rule 4 x 650 [8]
+  CRUSH rule 4 x 651 [4,6]
+  CRUSH rule 4 x 652 [4]
+  CRUSH rule 4 x 653 [8]
+  CRUSH rule 4 x 654 [6]
+  CRUSH rule 4 x 655 [1,4]
+  CRUSH rule 4 x 656 []
+  CRUSH rule 4 x 657 [6,1]
+  CRUSH rule 4 x 658 []
+  CRUSH rule 4 x 659 [4,8]
+  CRUSH rule 4 x 660 [8]
+  CRUSH rule 4 x 661 [1,8]
+  CRUSH rule 4 x 662 []
+  CRUSH rule 4 x 663 [1,4]
+  CRUSH rule 4 x 664 [1,4]
+  CRUSH rule 4 x 665 [4,6]
+  CRUSH rule 4 x 666 [2,8]
+  CRUSH rule 4 x 667 [1,4]
+  CRUSH rule 4 x 668 [4,8]
+  CRUSH rule 4 x 669 [6,4]
+  CRUSH rule 4 x 670 [4,2]
+  CRUSH rule 4 x 671 [2,1]
+  CRUSH rule 4 x 672 [4]
+  CRUSH rule 4 x 673 [4]
+  CRUSH rule 4 x 674 [1]
+  CRUSH rule 4 x 675 [1,8]
+  CRUSH rule 4 x 676 [2,1]
+  CRUSH rule 4 x 677 [4,1]
+  CRUSH rule 4 x 678 [2,4]
+  CRUSH rule 4 x 679 [8,2]
+  CRUSH rule 4 x 680 [2]
+  CRUSH rule 4 x 681 [8]
+  CRUSH rule 4 x 682 [1,4]
+  CRUSH rule 4 x 683 [1,2]
+  CRUSH rule 4 x 684 [8,1]
+  CRUSH rule 4 x 685 [8,1]
+  CRUSH rule 4 x 686 [1,4]
+  CRUSH rule 4 x 687 []
+  CRUSH rule 4 x 688 [4,8]
+  CRUSH rule 4 x 689 [8,4]
+  CRUSH rule 4 x 690 [8,1]
+  CRUSH rule 4 x 691 [1]
+  CRUSH rule 4 x 692 [8,2]
+  CRUSH rule 4 x 693 [8]
+  CRUSH rule 4 x 694 [8,4]
+  CRUSH rule 4 x 695 [2,8]
+  CRUSH rule 4 x 696 [1]
+  CRUSH rule 4 x 697 [8,1]
+  CRUSH rule 4 x 698 [8,2]
+  CRUSH rule 4 x 699 [1,8]
+  CRUSH rule 4 x 700 [1]
+  CRUSH rule 4 x 701 []
+  CRUSH rule 4 x 702 []
+  CRUSH rule 4 x 703 [8]
+  CRUSH rule 4 x 704 [1,4]
+  CRUSH rule 4 x 705 [8]
+  CRUSH rule 4 x 706 [1,2]
+  CRUSH rule 4 x 707 [8]
+  CRUSH rule 4 x 708 [4]
+  CRUSH rule 4 x 709 [8]
+  CRUSH rule 4 x 710 [8]
+  CRUSH rule 4 x 711 [2,4]
+  CRUSH rule 4 x 712 [2]
+  CRUSH rule 4 x 713 [8]
+  CRUSH rule 4 x 714 [2]
+  CRUSH rule 4 x 715 [1,2]
+  CRUSH rule 4 x 716 [4,8]
+  CRUSH rule 4 x 717 [8]
+  CRUSH rule 4 x 718 [8]
+  CRUSH rule 4 x 719 [2,6]
+  CRUSH rule 4 x 720 [8]
+  CRUSH rule 4 x 721 [4]
+  CRUSH rule 4 x 722 []
+  CRUSH rule 4 x 723 [4,1]
+  CRUSH rule 4 x 724 [2,6]
+  CRUSH rule 4 x 725 [1,2]
+  CRUSH rule 4 x 726 [4,8]
+  CRUSH rule 4 x 727 [4,8]
+  CRUSH rule 4 x 728 [2,1]
+  CRUSH rule 4 x 729 []
+  CRUSH rule 4 x 730 [4,8]
+  CRUSH rule 4 x 731 [4,1]
+  CRUSH rule 4 x 732 [1]
+  CRUSH rule 4 x 733 [4]
+  CRUSH rule 4 x 734 [8,4]
+  CRUSH rule 4 x 735 [4,8]
+  CRUSH rule 4 x 736 [4]
+  CRUSH rule 4 x 737 [1,2]
+  CRUSH rule 4 x 738 [4,2]
+  CRUSH rule 4 x 739 [2,1]
+  CRUSH rule 4 x 740 [1,2]
+  CRUSH rule 4 x 741 [8]
+  CRUSH rule 4 x 742 [8,2]
+  CRUSH rule 4 x 743 [8,1]
+  CRUSH rule 4 x 744 [4,8]
+  CRUSH rule 4 x 745 []
+  CRUSH rule 4 x 746 [1]
+  CRUSH rule 4 x 747 [8,1]
+  CRUSH rule 4 x 748 [2,8]
+  CRUSH rule 4 x 749 [4]
+  CRUSH rule 4 x 750 [1,8]
+  CRUSH rule 4 x 751 [2,1]
+  CRUSH rule 4 x 752 [8,1]
+  CRUSH rule 4 x 753 [8]
+  CRUSH rule 4 x 754 [8]
+  CRUSH rule 4 x 755 [1,2]
+  CRUSH rule 4 x 756 [8]
+  CRUSH rule 4 x 757 [8]
+  CRUSH rule 4 x 758 [8,2]
+  CRUSH rule 4 x 759 [8,4]
+  CRUSH rule 4 x 760 [1,4]
+  CRUSH rule 4 x 761 [1]
+  CRUSH rule 4 x 762 [2,8]
+  CRUSH rule 4 x 763 [8]
+  CRUSH rule 4 x 764 [1,8]
+  CRUSH rule 4 x 765 [8]
+  CRUSH rule 4 x 766 [8]
+  CRUSH rule 4 x 767 [1,2]
+  CRUSH rule 4 x 768 [8,4]
+  CRUSH rule 4 x 769 [8,2]
+  CRUSH rule 4 x 770 [8,2]
+  CRUSH rule 4 x 771 [8,1]
+  CRUSH rule 4 x 772 [8,4]
+  CRUSH rule 4 x 773 [4,1]
+  CRUSH rule 4 x 774 [8]
+  CRUSH rule 4 x 775 [8]
+  CRUSH rule 4 x 776 [6,2]
+  CRUSH rule 4 x 777 [4,1]
+  CRUSH rule 4 x 778 [1,8]
+  CRUSH rule 4 x 779 [2,8]
+  CRUSH rule 4 x 780 [2,1]
+  CRUSH rule 4 x 781 [8]
+  CRUSH rule 4 x 782 [4]
+  CRUSH rule 4 x 783 [8,1]
+  CRUSH rule 4 x 784 [1,2]
+  CRUSH rule 4 x 785 [8,1]
+  CRUSH rule 4 x 786 [8]
+  CRUSH rule 4 x 787 [1,2]
+  CRUSH rule 4 x 788 [8,2]
+  CRUSH rule 4 x 789 [1]
+  CRUSH rule 4 x 790 [8]
+  CRUSH rule 4 x 791 [4,8]
+  CRUSH rule 4 x 792 [4,8]
+  CRUSH rule 4 x 793 [8,1]
+  CRUSH rule 4 x 794 [2,8]
+  CRUSH rule 4 x 795 [1]
+  CRUSH rule 4 x 796 []
+  CRUSH rule 4 x 797 [2,4]
+  CRUSH rule 4 x 798 [6,8]
+  CRUSH rule 4 x 799 [4,1]
+  CRUSH rule 4 x 800 [2]
+  CRUSH rule 4 x 801 [4,8]
+  CRUSH rule 4 x 802 [1,8]
+  CRUSH rule 4 x 803 [2]
+  CRUSH rule 4 x 804 [8,2]
+  CRUSH rule 4 x 805 [8]
+  CRUSH rule 4 x 806 [1,4]
+  CRUSH rule 4 x 807 [4]
+  CRUSH rule 4 x 808 [8]
+  CRUSH rule 4 x 809 [1]
+  CRUSH rule 4 x 810 [8]
+  CRUSH rule 4 x 811 [8]
+  CRUSH rule 4 x 812 [8,4]
+  CRUSH rule 4 x 813 [8,4]
+  CRUSH rule 4 x 814 [8]
+  CRUSH rule 4 x 815 [4,1]
+  CRUSH rule 4 x 816 [2,1]
+  CRUSH rule 4 x 817 []
+  CRUSH rule 4 x 818 []
+  CRUSH rule 4 x 819 [1]
+  CRUSH rule 4 x 820 [4]
+  CRUSH rule 4 x 821 [4]
+  CRUSH rule 4 x 822 [2,1]
+  CRUSH rule 4 x 823 [4,8]
+  CRUSH rule 4 x 824 [8]
+  CRUSH rule 4 x 825 [2,8]
+  CRUSH rule 4 x 826 [8,2]
+  CRUSH rule 4 x 827 [2,8]
+  CRUSH rule 4 x 828 [2]
+  CRUSH rule 4 x 829 [8]
+  CRUSH rule 4 x 830 [2,4]
+  CRUSH rule 4 x 831 [1,8]
+  CRUSH rule 4 x 832 [4]
+  CRUSH rule 4 x 833 [2,1]
+  CRUSH rule 4 x 834 []
+  CRUSH rule 4 x 835 [8,4]
+  CRUSH rule 4 x 836 [4]
+  CRUSH rule 4 x 837 [8,4]
+  CRUSH rule 4 x 838 [6,8]
+  CRUSH rule 4 x 839 [2]
+  CRUSH rule 4 x 840 [8]
+  CRUSH rule 4 x 841 [4,8]
+  CRUSH rule 4 x 842 [2]
+  CRUSH rule 4 x 843 [8,4]
+  CRUSH rule 4 x 844 [8]
+  CRUSH rule 4 x 845 [4,8]
+  CRUSH rule 4 x 846 [4,2]
+  CRUSH rule 4 x 847 [2,1]
+  CRUSH rule 4 x 848 [2,8]
+  CRUSH rule 4 x 849 [4]
+  CRUSH rule 4 x 850 [1,2]
+  CRUSH rule 4 x 851 [6,8]
+  CRUSH rule 4 x 852 [8,4]
+  CRUSH rule 4 x 853 [6,8]
+  CRUSH rule 4 x 854 [8]
+  CRUSH rule 4 x 855 [8]
+  CRUSH rule 4 x 856 [8]
+  CRUSH rule 4 x 857 [8]
+  CRUSH rule 4 x 858 [6]
+  CRUSH rule 4 x 859 [8,2]
+  CRUSH rule 4 x 860 [1]
+  CRUSH rule 4 x 861 [8]
+  CRUSH rule 4 x 862 [8,1]
+  CRUSH rule 4 x 863 [8]
+  CRUSH rule 4 x 864 [8]
+  CRUSH rule 4 x 865 [8,1]
+  CRUSH rule 4 x 866 []
+  CRUSH rule 4 x 867 [8]
+  CRUSH rule 4 x 868 [8]
+  CRUSH rule 4 x 869 [8,6]
+  CRUSH rule 4 x 870 [2]
+  CRUSH rule 4 x 871 []
+  CRUSH rule 4 x 872 [1]
+  CRUSH rule 4 x 873 [4,8]
+  CRUSH rule 4 x 874 [2,6]
+  CRUSH rule 4 x 875 [2,8]
+  CRUSH rule 4 x 876 [4,8]
+  CRUSH rule 4 x 877 [8,4]
+  CRUSH rule 4 x 878 []
+  CRUSH rule 4 x 879 [8]
+  CRUSH rule 4 x 880 []
+  CRUSH rule 4 x 881 [4,8]
+  CRUSH rule 4 x 882 [2]
+  CRUSH rule 4 x 883 [2,1]
+  CRUSH rule 4 x 884 [8,2]
+  CRUSH rule 4 x 885 [4,1]
+  CRUSH rule 4 x 886 [8]
+  CRUSH rule 4 x 887 [8,4]
+  CRUSH rule 4 x 888 [8]
+  CRUSH rule 4 x 889 [2,1]
+  CRUSH rule 4 x 890 [8,2]
+  CRUSH rule 4 x 891 [1,8]
+  CRUSH rule 4 x 892 [8,2]
+  CRUSH rule 4 x 893 [2]
+  CRUSH rule 4 x 894 [8,4]
+  CRUSH rule 4 x 895 [4]
+  CRUSH rule 4 x 896 [1,8]
+  CRUSH rule 4 x 897 [2]
+  CRUSH rule 4 x 898 [1,4]
+  CRUSH rule 4 x 899 [1,8]
+  CRUSH rule 4 x 900 [4,1]
+  CRUSH rule 4 x 901 [1]
+  CRUSH rule 4 x 902 [8,4]
+  CRUSH rule 4 x 903 [8]
+  CRUSH rule 4 x 904 [8]
+  CRUSH rule 4 x 905 [8,2]
+  CRUSH rule 4 x 906 [1,2]
+  CRUSH rule 4 x 907 [8,1]
+  CRUSH rule 4 x 908 [8]
+  CRUSH rule 4 x 909 [2]
+  CRUSH rule 4 x 910 [8]
+  CRUSH rule 4 x 911 [8]
+  CRUSH rule 4 x 912 [1,2]
+  CRUSH rule 4 x 913 [8]
+  CRUSH rule 4 x 914 [6,4]
+  CRUSH rule 4 x 915 [8,2]
+  CRUSH rule 4 x 916 [4,1]
+  CRUSH rule 4 x 917 [1,4]
+  CRUSH rule 4 x 918 [8,2]
+  CRUSH rule 4 x 919 [8,2]
+  CRUSH rule 4 x 920 [8]
+  CRUSH rule 4 x 921 [1]
+  CRUSH rule 4 x 922 [8]
+  CRUSH rule 4 x 923 [4]
+  CRUSH rule 4 x 924 []
+  CRUSH rule 4 x 925 [4,8]
+  CRUSH rule 4 x 926 []
+  CRUSH rule 4 x 927 [1,8]
+  CRUSH rule 4 x 928 [8,1]
+  CRUSH rule 4 x 929 [4]
+  CRUSH rule 4 x 930 [2]
+  CRUSH rule 4 x 931 [1]
+  CRUSH rule 4 x 932 [4]
+  CRUSH rule 4 x 933 [8,4]
+  CRUSH rule 4 x 934 []
+  CRUSH rule 4 x 935 [8]
+  CRUSH rule 4 x 936 [1,8]
+  CRUSH rule 4 x 937 [4]
+  CRUSH rule 4 x 938 [8,4]
+  CRUSH rule 4 x 939 [2,8]
+  CRUSH rule 4 x 940 [8]
+  CRUSH rule 4 x 941 [2]
+  CRUSH rule 4 x 942 [1,2]
+  CRUSH rule 4 x 943 [8,2]
+  CRUSH rule 4 x 944 []
+  CRUSH rule 4 x 945 [8,2]
+  CRUSH rule 4 x 946 [2,1]
+  CRUSH rule 4 x 947 []
+  CRUSH rule 4 x 948 [8]
+  CRUSH rule 4 x 949 [6,1]
+  CRUSH rule 4 x 950 []
+  CRUSH rule 4 x 951 []
+  CRUSH rule 4 x 952 [2,1]
+  CRUSH rule 4 x 953 [1,4]
+  CRUSH rule 4 x 954 [2]
+  CRUSH rule 4 x 955 [8]
+  CRUSH rule 4 x 956 [1,2]
+  CRUSH rule 4 x 957 [8]
+  CRUSH rule 4 x 958 [8]
+  CRUSH rule 4 x 959 [4,2]
+  CRUSH rule 4 x 960 [6]
+  CRUSH rule 4 x 961 [1]
+  CRUSH rule 4 x 962 [8,4]
+  CRUSH rule 4 x 963 [2,4]
+  CRUSH rule 4 x 964 [1]
+  CRUSH rule 4 x 965 [8]
+  CRUSH rule 4 x 966 [4,8]
+  CRUSH rule 4 x 967 [8]
+  CRUSH rule 4 x 968 [8,2]
+  CRUSH rule 4 x 969 [8,2]
+  CRUSH rule 4 x 970 [2,8]
+  CRUSH rule 4 x 971 [1,8]
+  CRUSH rule 4 x 972 [1,8]
+  CRUSH rule 4 x 973 [1,2]
+  CRUSH rule 4 x 974 [4]
+  CRUSH rule 4 x 975 [4,8]
+  CRUSH rule 4 x 976 [4]
+  CRUSH rule 4 x 977 [8,4]
+  CRUSH rule 4 x 978 [8,2]
+  CRUSH rule 4 x 979 [8]
+  CRUSH rule 4 x 980 [8,2]
+  CRUSH rule 4 x 981 [8]
+  CRUSH rule 4 x 982 [2]
+  CRUSH rule 4 x 983 [4]
+  CRUSH rule 4 x 984 [2,1]
+  CRUSH rule 4 x 985 [2,4]
+  CRUSH rule 4 x 986 [8]
+  CRUSH rule 4 x 987 [2]
+  CRUSH rule 4 x 988 [1,4]
+  CRUSH rule 4 x 989 [1,8]
+  CRUSH rule 4 x 990 [1,2]
+  CRUSH rule 4 x 991 [1,4]
+  CRUSH rule 4 x 992 [8,1]
+  CRUSH rule 4 x 993 [2,8]
+  CRUSH rule 4 x 994 [4]
+  CRUSH rule 4 x 995 [8]
+  CRUSH rule 4 x 996 [8]
+  CRUSH rule 4 x 997 [8,4]
+  CRUSH rule 4 x 998 [8,1]
+  CRUSH rule 4 x 999 [1,8]
+  CRUSH rule 4 x 1000 [8,4]
+  CRUSH rule 4 x 1001 [2,1]
+  CRUSH rule 4 x 1002 [1]
+  CRUSH rule 4 x 1003 [2,8]
+  CRUSH rule 4 x 1004 [8,1]
+  CRUSH rule 4 x 1005 [8,1]
+  CRUSH rule 4 x 1006 [1,2]
+  CRUSH rule 4 x 1007 [1,2]
+  CRUSH rule 4 x 1008 [1,8]
+  CRUSH rule 4 x 1009 [6,8]
+  CRUSH rule 4 x 1010 []
+  CRUSH rule 4 x 1011 [4,2]
+  CRUSH rule 4 x 1012 [2]
+  CRUSH rule 4 x 1013 [1]
+  CRUSH rule 4 x 1014 [2,8]
+  CRUSH rule 4 x 1015 [8]
+  CRUSH rule 4 x 1016 [2,1]
+  CRUSH rule 4 x 1017 [6,2]
+  CRUSH rule 4 x 1018 [4]
+  CRUSH rule 4 x 1019 [4]
+  CRUSH rule 4 x 1020 [1]
+  CRUSH rule 4 x 1021 [2]
+  CRUSH rule 4 x 1022 [1,8]
+  CRUSH rule 4 x 1023 [4,2]
+  rule 4 (choose-set-two) num_rep 2 result size == 0:\t56/1024 (esc)
+  rule 4 (choose-set-two) num_rep 2 result size == 1:\t397/1024 (esc)
+  rule 4 (choose-set-two) num_rep 2 result size == 2:\t571/1024 (esc)
+  CRUSH rule 4 x 0 [2,1,4]
+  CRUSH rule 4 x 1 [2,8,1]
+  CRUSH rule 4 x 2 [1]
+  CRUSH rule 4 x 3 [8,1]
+  CRUSH rule 4 x 4 [4,2]
+  CRUSH rule 4 x 5 [8,1]
+  CRUSH rule 4 x 6 [2,8]
+  CRUSH rule 4 x 7 [4]
+  CRUSH rule 4 x 8 [4,8]
+  CRUSH rule 4 x 9 [2,4,1]
+  CRUSH rule 4 x 10 [2,1,8]
+  CRUSH rule 4 x 11 [2,8,1]
+  CRUSH rule 4 x 12 [2,1]
+  CRUSH rule 4 x 13 [4,8]
+  CRUSH rule 4 x 14 [8]
+  CRUSH rule 4 x 15 [8,2]
+  CRUSH rule 4 x 16 [8,2]
+  CRUSH rule 4 x 17 [4,2]
+  CRUSH rule 4 x 18 [1]
+  CRUSH rule 4 x 19 [8,4]
+  CRUSH rule 4 x 20 [2,8]
+  CRUSH rule 4 x 21 [8,2]
+  CRUSH rule 4 x 22 [8]
+  CRUSH rule 4 x 23 [4,8]
+  CRUSH rule 4 x 24 [1,2,8]
+  CRUSH rule 4 x 25 [4,8,1]
+  CRUSH rule 4 x 26 [2,8]
+  CRUSH rule 4 x 27 [4,1]
+  CRUSH rule 4 x 28 [8,2]
+  CRUSH rule 4 x 29 [8,4]
+  CRUSH rule 4 x 30 [4,8]
+  CRUSH rule 4 x 31 [8]
+  CRUSH rule 4 x 32 [6,8]
+  CRUSH rule 4 x 33 [2,8]
+  CRUSH rule 4 x 34 [2]
+  CRUSH rule 4 x 35 [1,8,2]
+  CRUSH rule 4 x 36 [8,2]
+  CRUSH rule 4 x 37 [1]
+  CRUSH rule 4 x 38 [4,8]
+  CRUSH rule 4 x 39 [8,1]
+  CRUSH rule 4 x 40 [8,1]
+  CRUSH rule 4 x 41 [2,1,8]
+  CRUSH rule 4 x 42 [8]
+  CRUSH rule 4 x 43 [1,8]
+  CRUSH rule 4 x 44 [1,8]
+  CRUSH rule 4 x 45 [8,2,4]
+  CRUSH rule 4 x 46 [2]
+  CRUSH rule 4 x 47 [4]
+  CRUSH rule 4 x 48 [8]
+  CRUSH rule 4 x 49 [8]
+  CRUSH rule 4 x 50 [4,1]
+  CRUSH rule 4 x 51 [8,2]
+  CRUSH rule 4 x 52 [8,1]
+  CRUSH rule 4 x 53 [4]
+  CRUSH rule 4 x 54 [8]
+  CRUSH rule 4 x 55 [8,2]
+  CRUSH rule 4 x 56 [8,4]
+  CRUSH rule 4 x 57 [8]
+  CRUSH rule 4 x 58 [1,2]
+  CRUSH rule 4 x 59 [2,1]
+  CRUSH rule 4 x 60 [4,1]
+  CRUSH rule 4 x 61 [4,8]
+  CRUSH rule 4 x 62 [8,1,2]
+  CRUSH rule 4 x 63 [8]
+  CRUSH rule 4 x 64 [4,1]
+  CRUSH rule 4 x 65 [8,4]
+  CRUSH rule 4 x 66 [4]
+  CRUSH rule 4 x 67 [4,2,8]
+  CRUSH rule 4 x 68 [1,8]
+  CRUSH rule 4 x 69 [1,8]
+  CRUSH rule 4 x 70 [8,2,1]
+  CRUSH rule 4 x 71 [2,8]
+  CRUSH rule 4 x 72 [8,1,4]
+  CRUSH rule 4 x 73 [2,8]
+  CRUSH rule 4 x 74 [1,8,2]
+  CRUSH rule 4 x 75 [4,2,1]
+  CRUSH rule 4 x 76 [4,1,6]
+  CRUSH rule 4 x 77 [8,2]
+  CRUSH rule 4 x 78 [1,2]
+  CRUSH rule 4 x 79 [4,1]
+  CRUSH rule 4 x 80 [2,4]
+  CRUSH rule 4 x 81 [2,1]
+  CRUSH rule 4 x 82 [6,1,8]
+  CRUSH rule 4 x 83 [2,8]
+  CRUSH rule 4 x 84 [8,2,1]
+  CRUSH rule 4 x 85 [4,8]
+  CRUSH rule 4 x 86 [2,1]
+  CRUSH rule 4 x 87 [2,8,4]
+  CRUSH rule 4 x 88 [1,6,2]
+  CRUSH rule 4 x 89 [2,1]
+  CRUSH rule 4 x 90 [8,4]
+  CRUSH rule 4 x 91 [4,8]
+  CRUSH rule 4 x 92 [1,8]
+  CRUSH rule 4 x 93 [8,4]
+  CRUSH rule 4 x 94 [1]
+  CRUSH rule 4 x 95 [8,1]
+  CRUSH rule 4 x 96 [8]
+  CRUSH rule 4 x 97 [8]
+  CRUSH rule 4 x 98 [2,1,8]
+  CRUSH rule 4 x 99 [2,8,1]
+  CRUSH rule 4 x 100 [1,8,4]
+  CRUSH rule 4 x 101 [8,2]
+  CRUSH rule 4 x 102 [2,8]
+  CRUSH rule 4 x 103 [8]
+  CRUSH rule 4 x 104 [8,4]
+  CRUSH rule 4 x 105 [2,4,1]
+  CRUSH rule 4 x 106 [1,8,2]
+  CRUSH rule 4 x 107 [2,1]
+  CRUSH rule 4 x 108 [8,2]
+  CRUSH rule 4 x 109 [1,2,4]
+  CRUSH rule 4 x 110 [4,2,8]
+  CRUSH rule 4 x 111 [2,1]
+  CRUSH rule 4 x 112 [2,1,8]
+  CRUSH rule 4 x 113 [8,2,1]
+  CRUSH rule 4 x 114 [8,4]
+  CRUSH rule 4 x 115 [8,2,4]
+  CRUSH rule 4 x 116 [1,2,8]
+  CRUSH rule 4 x 117 [6,8]
+  CRUSH rule 4 x 118 [2,8]
+  CRUSH rule 4 x 119 [8,1]
+  CRUSH rule 4 x 120 [2,1,4]
+  CRUSH rule 4 x 121 [2,1]
+  CRUSH rule 4 x 122 [8,2]
+  CRUSH rule 4 x 123 [2,1]
+  CRUSH rule 4 x 124 [1]
+  CRUSH rule 4 x 125 [1,8,4]
+  CRUSH rule 4 x 126 [2,8]
+  CRUSH rule 4 x 127 [4,8]
+  CRUSH rule 4 x 128 [8]
+  CRUSH rule 4 x 129 [2,1]
+  CRUSH rule 4 x 130 [4,8]
+  CRUSH rule 4 x 131 [1,2]
+  CRUSH rule 4 x 132 [1,2]
+  CRUSH rule 4 x 133 [8]
+  CRUSH rule 4 x 134 [1,8,2]
+  CRUSH rule 4 x 135 [4,8]
+  CRUSH rule 4 x 136 [2,1,4]
+  CRUSH rule 4 x 137 [8,4]
+  CRUSH rule 4 x 138 [8]
+  CRUSH rule 4 x 139 [4,2]
+  CRUSH rule 4 x 140 [1,8]
+  CRUSH rule 4 x 141 [8,1]
+  CRUSH rule 4 x 142 [4,1]
+  CRUSH rule 4 x 143 [4,8]
+  CRUSH rule 4 x 144 [8,1,2]
+  CRUSH rule 4 x 145 [8]
+  CRUSH rule 4 x 146 [2,8]
+  CRUSH rule 4 x 147 [2,8]
+  CRUSH rule 4 x 148 [4,1]
+  CRUSH rule 4 x 149 [4,8]
+  CRUSH rule 4 x 150 [1,8]
+  CRUSH rule 4 x 151 []
+  CRUSH rule 4 x 152 [8]
+  CRUSH rule 4 x 153 [8]
+  CRUSH rule 4 x 154 [4,2,1]
+  CRUSH rule 4 x 155 [4,8]
+  CRUSH rule 4 x 156 [4]
+  CRUSH rule 4 x 157 [1,2]
+  CRUSH rule 4 x 158 [2,8,1]
+  CRUSH rule 4 x 159 [8,2,4]
+  CRUSH rule 4 x 160 [2,8,1]
+  CRUSH rule 4 x 161 [1,4,2]
+  CRUSH rule 4 x 162 [1,8,2]
+  CRUSH rule 4 x 163 [4,8,1]
+  CRUSH rule 4 x 164 [8,1]
+  CRUSH rule 4 x 165 [8,2,4]
+  CRUSH rule 4 x 166 [2,1]
+  CRUSH rule 4 x 167 [1,2,8]
+  CRUSH rule 4 x 168 [4,2]
+  CRUSH rule 4 x 169 [2,8,1]
+  CRUSH rule 4 x 170 [1,2]
+  CRUSH rule 4 x 171 [8,4]
+  CRUSH rule 4 x 172 [1,8]
+  CRUSH rule 4 x 173 [8,4]
+  CRUSH rule 4 x 174 [1,6]
+  CRUSH rule 4 x 175 [8,1]
+  CRUSH rule 4 x 176 []
+  CRUSH rule 4 x 177 [2]
+  CRUSH rule 4 x 178 [4,2,1]
+  CRUSH rule 4 x 179 [1,2]
+  CRUSH rule 4 x 180 [8]
+  CRUSH rule 4 x 181 [8,2,1]
+  CRUSH rule 4 x 182 [8]
+  CRUSH rule 4 x 183 [8]
+  CRUSH rule 4 x 184 [4,8]
+  CRUSH rule 4 x 185 [8,1]
+  CRUSH rule 4 x 186 [2,1,4]
+  CRUSH rule 4 x 187 [1,8]
+  CRUSH rule 4 x 188 [1,8]
+  CRUSH rule 4 x 189 [1,8]
+  CRUSH rule 4 x 190 [1,2]
+  CRUSH rule 4 x 191 [8]
+  CRUSH rule 4 x 192 [4,1,2]
+  CRUSH rule 4 x 193 [4,2,8]
+  CRUSH rule 4 x 194 [1]
+  CRUSH rule 4 x 195 [8,4]
+  CRUSH rule 4 x 196 [8,2]
+  CRUSH rule 4 x 197 [8,4,2]
+  CRUSH rule 4 x 198 [2,8]
+  CRUSH rule 4 x 199 [1,4,8]
+  CRUSH rule 4 x 200 [1,2]
+  CRUSH rule 4 x 201 [8,1,2]
+  CRUSH rule 4 x 202 [8]
+  CRUSH rule 4 x 203 [8]
+  CRUSH rule 4 x 204 [2,1,4]
+  CRUSH rule 4 x 205 [1,8]
+  CRUSH rule 4 x 206 [1,2,8]
+  CRUSH rule 4 x 207 [2]
+  CRUSH rule 4 x 208 [8,1,2]
+  CRUSH rule 4 x 209 [1,2]
+  CRUSH rule 4 x 210 [1,2,4]
+  CRUSH rule 4 x 211 [4,2]
+  CRUSH rule 4 x 212 [8,1]
+  CRUSH rule 4 x 213 [8,4,2]
+  CRUSH rule 4 x 214 [8]
+  CRUSH rule 4 x 215 [8,1]
+  CRUSH rule 4 x 216 [1,2]
+  CRUSH rule 4 x 217 [1,2,8]
+  CRUSH rule 4 x 218 [2,8]
+  CRUSH rule 4 x 219 [8,2]
+  CRUSH rule 4 x 220 [4,8]
+  CRUSH rule 4 x 221 [8]
+  CRUSH rule 4 x 222 [8]
+  CRUSH rule 4 x 223 [1,8]
+  CRUSH rule 4 x 224 [1,4,2]
+  CRUSH rule 4 x 225 [8,2]
+  CRUSH rule 4 x 226 [8,2,4]
+  CRUSH rule 4 x 227 [4,1]
+  CRUSH rule 4 x 228 [8]
+  CRUSH rule 4 x 229 [4,6]
+  CRUSH rule 4 x 230 [4,8,2]
+  CRUSH rule 4 x 231 [4]
+  CRUSH rule 4 x 232 [2,8,4]
+  CRUSH rule 4 x 233 []
+  CRUSH rule 4 x 234 [1,2]
+  CRUSH rule 4 x 235 [4,8,1]
+  CRUSH rule 4 x 236 [2,6]
+  CRUSH rule 4 x 237 [4,8]
+  CRUSH rule 4 x 238 []
+  CRUSH rule 4 x 239 [8,6]
+  CRUSH rule 4 x 240 [4,8]
+  CRUSH rule 4 x 241 [1]
+  CRUSH rule 4 x 242 []
+  CRUSH rule 4 x 243 [8]
+  CRUSH rule 4 x 244 [4,8,2]
+  CRUSH rule 4 x 245 [8,1]
+  CRUSH rule 4 x 246 [1,2]
+  CRUSH rule 4 x 247 [8,2,1]
+  CRUSH rule 4 x 248 [8,2]
+  CRUSH rule 4 x 249 [2,1]
+  CRUSH rule 4 x 250 [2,1]
+  CRUSH rule 4 x 251 [2,8]
+  CRUSH rule 4 x 252 [4,8]
+  CRUSH rule 4 x 253 [2]
+  CRUSH rule 4 x 254 [4]
+  CRUSH rule 4 x 255 [1,8]
+  CRUSH rule 4 x 256 [4,8,1]
+  CRUSH rule 4 x 257 [2,8,4]
+  CRUSH rule 4 x 258 [4,1]
+  CRUSH rule 4 x 259 [6]
+  CRUSH rule 4 x 260 [8,2]
+  CRUSH rule 4 x 261 [8]
+  CRUSH rule 4 x 262 [8]
+  CRUSH rule 4 x 263 [8,2]
+  CRUSH rule 4 x 264 [8]
+  CRUSH rule 4 x 265 [8]
+  CRUSH rule 4 x 266 [8,2,1]
+  CRUSH rule 4 x 267 [2,1]
+  CRUSH rule 4 x 268 [1,8,2]
+  CRUSH rule 4 x 269 [1,8,2]
+  CRUSH rule 4 x 270 [4,1,2]
+  CRUSH rule 4 x 271 [8,4]
+  CRUSH rule 4 x 272 [2,8,4]
+  CRUSH rule 4 x 273 [4,1]
+  CRUSH rule 4 x 274 [8,4]
+  CRUSH rule 4 x 275 [4,8]
+  CRUSH rule 4 x 276 [8,1,2]
+  CRUSH rule 4 x 277 [8,1]
+  CRUSH rule 4 x 278 [8,1]
+  CRUSH rule 4 x 279 [8,4]
+  CRUSH rule 4 x 280 [2,8,1]
+  CRUSH rule 4 x 281 [8,2,1]
+  CRUSH rule 4 x 282 [1,8]
+  CRUSH rule 4 x 283 [8,2,1]
+  CRUSH rule 4 x 284 [8,2]
+  CRUSH rule 4 x 285 [4,8]
+  CRUSH rule 4 x 286 [2,1,8]
+  CRUSH rule 4 x 287 [1,2]
+  CRUSH rule 4 x 288 [8,1]
+  CRUSH rule 4 x 289 [4,8,2]
+  CRUSH rule 4 x 290 [1,4,8]
+  CRUSH rule 4 x 291 [1,2,4]
+  CRUSH rule 4 x 292 [8,2,1]
+  CRUSH rule 4 x 293 [8,1]
+  CRUSH rule 4 x 294 [8,4]
+  CRUSH rule 4 x 295 [4,8]
+  CRUSH rule 4 x 296 [4,1]
+  CRUSH rule 4 x 297 [8,2]
+  CRUSH rule 4 x 298 [1,2]
+  CRUSH rule 4 x 299 [2,1,8]
+  CRUSH rule 4 x 300 [8]
+  CRUSH rule 4 x 301 [1,8,2]
+  CRUSH rule 4 x 302 [1,8]
+  CRUSH rule 4 x 303 [8,4]
+  CRUSH rule 4 x 304 [2,8]
+  CRUSH rule 4 x 305 [8,2]
+  CRUSH rule 4 x 306 [1,8,2]
+  CRUSH rule 4 x 307 [2,1,8]
+  CRUSH rule 4 x 308 [2,8,4]
+  CRUSH rule 4 x 309 [8]
+  CRUSH rule 4 x 310 [4,1]
+  CRUSH rule 4 x 311 [4]
+  CRUSH rule 4 x 312 [2,1]
+  CRUSH rule 4 x 313 [4]
+  CRUSH rule 4 x 314 [1]
+  CRUSH rule 4 x 315 [2,1]
+  CRUSH rule 4 x 316 [8]
+  CRUSH rule 4 x 317 [2,8,1]
+  CRUSH rule 4 x 318 [8]
+  CRUSH rule 4 x 319 [2,8]
+  CRUSH rule 4 x 320 [8,1]
+  CRUSH rule 4 x 321 [1,2]
+  CRUSH rule 4 x 322 [2,8,4]
+  CRUSH rule 4 x 323 [4,8,1]
+  CRUSH rule 4 x 324 [8,1]
+  CRUSH rule 4 x 325 [4,8,2]
+  CRUSH rule 4 x 326 [1]
+  CRUSH rule 4 x 327 [1,8]
+  CRUSH rule 4 x 328 [8,4]
+  CRUSH rule 4 x 329 [4,8]
+  CRUSH rule 4 x 330 [4,8]
+  CRUSH rule 4 x 331 [2,8]
+  CRUSH rule 4 x 332 [2,1]
+  CRUSH rule 4 x 333 [8]
+  CRUSH rule 4 x 334 [8]
+  CRUSH rule 4 x 335 [8,1,2]
+  CRUSH rule 4 x 336 [4,8,2]
+  CRUSH rule 4 x 337 [8,2]
+  CRUSH rule 4 x 338 [8]
+  CRUSH rule 4 x 339 [8,2]
+  CRUSH rule 4 x 340 [2,1]
+  CRUSH rule 4 x 341 [4,1,8]
+  CRUSH rule 4 x 342 [2,8,4]
+  CRUSH rule 4 x 343 [8]
+  CRUSH rule 4 x 344 [6,2,4]
+  CRUSH rule 4 x 345 []
+  CRUSH rule 4 x 346 [8,2,4]
+  CRUSH rule 4 x 347 [4,1]
+  CRUSH rule 4 x 348 [8,2,1]
+  CRUSH rule 4 x 349 [1,8]
+  CRUSH rule 4 x 350 [8]
+  CRUSH rule 4 x 351 [8]
+  CRUSH rule 4 x 352 [1,2]
+  CRUSH rule 4 x 353 [8]
+  CRUSH rule 4 x 354 [1,2]
+  CRUSH rule 4 x 355 [8]
+  CRUSH rule 4 x 356 [4]
+  CRUSH rule 4 x 357 [8,1,2]
+  CRUSH rule 4 x 358 [2,1,8]
+  CRUSH rule 4 x 359 [6,8]
+  CRUSH rule 4 x 360 [1]
+  CRUSH rule 4 x 361 [8,4]
+  CRUSH rule 4 x 362 [4]
+  CRUSH rule 4 x 363 [4,1]
+  CRUSH rule 4 x 364 [2,1]
+  CRUSH rule 4 x 365 [8]
+  CRUSH rule 4 x 366 [8,2]
+  CRUSH rule 4 x 367 [4,2]
+  CRUSH rule 4 x 368 [8,4]
+  CRUSH rule 4 x 369 [8,1]
+  CRUSH rule 4 x 370 [8]
+  CRUSH rule 4 x 371 [1,4]
+  CRUSH rule 4 x 372 [1,8]
+  CRUSH rule 4 x 373 [1,8]
+  CRUSH rule 4 x 374 [8]
+  CRUSH rule 4 x 375 [8,4]
+  CRUSH rule 4 x 376 [8,1,2]
+  CRUSH rule 4 x 377 [1,2,4]
+  CRUSH rule 4 x 378 [1,2]
+  CRUSH rule 4 x 379 [8]
+  CRUSH rule 4 x 380 [2,1]
+  CRUSH rule 4 x 381 [1,4,8]
+  CRUSH rule 4 x 382 [1,4,2]
+  CRUSH rule 4 x 383 [4]
+  CRUSH rule 4 x 384 [8,2]
+  CRUSH rule 4 x 385 [8,6]
+  CRUSH rule 4 x 386 [1]
+  CRUSH rule 4 x 387 [1,4,8]
+  CRUSH rule 4 x 388 [1]
+  CRUSH rule 4 x 389 [1,4]
+  CRUSH rule 4 x 390 [4,8,1]
+  CRUSH rule 4 x 391 [4,8,2]
+  CRUSH rule 4 x 392 [1,8]
+  CRUSH rule 4 x 393 [2,8]
+  CRUSH rule 4 x 394 [8]
+  CRUSH rule 4 x 395 [1,2]
+  CRUSH rule 4 x 396 [4,2]
+  CRUSH rule 4 x 397 [2,1,4]
+  CRUSH rule 4 x 398 [2,4]
+  CRUSH rule 4 x 399 [8,4]
+  CRUSH rule 4 x 400 [8,1,4]
+  CRUSH rule 4 x 401 [1,2]
+  CRUSH rule 4 x 402 [8]
+  CRUSH rule 4 x 403 [1,2,4]
+  CRUSH rule 4 x 404 [4,2]
+  CRUSH rule 4 x 405 [8,4,2]
+  CRUSH rule 4 x 406 [2,1]
+  CRUSH rule 4 x 407 [2,8]
+  CRUSH rule 4 x 408 [4,1,8]
+  CRUSH rule 4 x 409 [8,4]
+  CRUSH rule 4 x 410 [8,4]
+  CRUSH rule 4 x 411 [2,1,8]
+  CRUSH rule 4 x 412 [2,8]
+  CRUSH rule 4 x 413 [2,8]
+  CRUSH rule 4 x 414 [4,1]
+  CRUSH rule 4 x 415 [2,8,1]
+  CRUSH rule 4 x 416 [2,1,8]
+  CRUSH rule 4 x 417 [8,6,2]
+  CRUSH rule 4 x 418 [8]
+  CRUSH rule 4 x 419 [8,4,1]
+  CRUSH rule 4 x 420 [1,4,2]
+  CRUSH rule 4 x 421 [8]
+  CRUSH rule 4 x 422 [6,8]
+  CRUSH rule 4 x 423 [2,4,8]
+  CRUSH rule 4 x 424 [8]
+  CRUSH rule 4 x 425 [1]
+  CRUSH rule 4 x 426 [8,2]
+  CRUSH rule 4 x 427 [1,8]
+  CRUSH rule 4 x 428 [4]
+  CRUSH rule 4 x 429 [4,8]
+  CRUSH rule 4 x 430 [4,8]
+  CRUSH rule 4 x 431 [4,1]
+  CRUSH rule 4 x 432 [8,1,2]
+  CRUSH rule 4 x 433 [8,1]
+  CRUSH rule 4 x 434 [2,1]
+  CRUSH rule 4 x 435 [2]
+  CRUSH rule 4 x 436 [4,1]
+  CRUSH rule 4 x 437 [8]
+  CRUSH rule 4 x 438 [2,4,8]
+  CRUSH rule 4 x 439 [1]
+  CRUSH rule 4 x 440 [2,8,1]
+  CRUSH rule 4 x 441 [4,6,2]
+  CRUSH rule 4 x 442 [2]
+  CRUSH rule 4 x 443 [8,2]
+  CRUSH rule 4 x 444 [8,1,2]
+  CRUSH rule 4 x 445 [8]
+  CRUSH rule 4 x 446 []
+  CRUSH rule 4 x 447 [2,1,4]
+  CRUSH rule 4 x 448 [8,2,4]
+  CRUSH rule 4 x 449 [8,6]
+  CRUSH rule 4 x 450 [2]
+  CRUSH rule 4 x 451 [8,4]
+  CRUSH rule 4 x 452 [8]
+  CRUSH rule 4 x 453 [6,8]
+  CRUSH rule 4 x 454 [8]
+  CRUSH rule 4 x 455 [2,8,4]
+  CRUSH rule 4 x 456 [8]
+  CRUSH rule 4 x 457 [8,2]
+  CRUSH rule 4 x 458 [2,8,1]
+  CRUSH rule 4 x 459 [2,1,8]
+  CRUSH rule 4 x 460 [8]
+  CRUSH rule 4 x 461 [8]
+  CRUSH rule 4 x 462 [8,1]
+  CRUSH rule 4 x 463 [8,2]
+  CRUSH rule 4 x 464 [8,4,2]
+  CRUSH rule 4 x 465 [6,8,1]
+  CRUSH rule 4 x 466 [8]
+  CRUSH rule 4 x 467 [8]
+  CRUSH rule 4 x 468 [8,1]
+  CRUSH rule 4 x 469 [8,1,2]
+  CRUSH rule 4 x 470 [4,2]
+  CRUSH rule 4 x 471 [1,2]
+  CRUSH rule 4 x 472 [1]
+  CRUSH rule 4 x 473 [1,2,4]
+  CRUSH rule 4 x 474 [8,1]
+  CRUSH rule 4 x 475 [8]
+  CRUSH rule 4 x 476 [4]
+  CRUSH rule 4 x 477 [4,8]
+  CRUSH rule 4 x 478 [8,2]
+  CRUSH rule 4 x 479 [2,8]
+  CRUSH rule 4 x 480 [1,8]
+  CRUSH rule 4 x 481 [2,4]
+  CRUSH rule 4 x 482 []
+  CRUSH rule 4 x 483 [2,1,8]
+  CRUSH rule 4 x 484 [1,2,8]
+  CRUSH rule 4 x 485 [8]
+  CRUSH rule 4 x 486 [4,1,8]
+  CRUSH rule 4 x 487 [1,2]
+  CRUSH rule 4 x 488 [8,1]
+  CRUSH rule 4 x 489 [2,8]
+  CRUSH rule 4 x 490 [6,1]
+  CRUSH rule 4 x 491 [1,2,8]
+  CRUSH rule 4 x 492 [8]
+  CRUSH rule 4 x 493 [2,1,8]
+  CRUSH rule 4 x 494 [1,2,8]
+  CRUSH rule 4 x 495 [4]
+  CRUSH rule 4 x 496 [8,4]
+  CRUSH rule 4 x 497 [4,8]
+  CRUSH rule 4 x 498 [2,4,8]
+  CRUSH rule 4 x 499 [8,4]
+  CRUSH rule 4 x 500 [4,8]
+  CRUSH rule 4 x 501 [2,8,1]
+  CRUSH rule 4 x 502 [6,1]
+  CRUSH rule 4 x 503 [2]
+  CRUSH rule 4 x 504 [8]
+  CRUSH rule 4 x 505 [1,8]
+  CRUSH rule 4 x 506 [4,2]
+  CRUSH rule 4 x 507 [8,1]
+  CRUSH rule 4 x 508 [1,2]
+  CRUSH rule 4 x 509 [8]
+  CRUSH rule 4 x 510 [8,2,1]
+  CRUSH rule 4 x 511 [4,8]
+  CRUSH rule 4 x 512 [8,2]
+  CRUSH rule 4 x 513 [8,2,1]
+  CRUSH rule 4 x 514 []
+  CRUSH rule 4 x 515 [8,4]
+  CRUSH rule 4 x 516 [4,1,2]
+  CRUSH rule 4 x 517 [8]
+  CRUSH rule 4 x 518 [4,8]
+  CRUSH rule 4 x 519 [8,4]
+  CRUSH rule 4 x 520 [2,8,4]
+  CRUSH rule 4 x 521 [8]
+  CRUSH rule 4 x 522 [8,1]
+  CRUSH rule 4 x 523 [4,2]
+  CRUSH rule 4 x 524 [2]
+  CRUSH rule 4 x 525 [2,1]
+  CRUSH rule 4 x 526 [1,8]
+  CRUSH rule 4 x 527 [1,2,4]
+  CRUSH rule 4 x 528 [1]
+  CRUSH rule 4 x 529 [4,8,2]
+  CRUSH rule 4 x 530 [8]
+  CRUSH rule 4 x 531 [8,1,2]
+  CRUSH rule 4 x 532 [6,4,8]
+  CRUSH rule 4 x 533 [4,8]
+  CRUSH rule 4 x 534 [8]
+  CRUSH rule 4 x 535 [8,6,1]
+  CRUSH rule 4 x 536 [8,2]
+  CRUSH rule 4 x 537 [4,8]
+  CRUSH rule 4 x 538 [8,4]
+  CRUSH rule 4 x 539 [8]
+  CRUSH rule 4 x 540 [1,8,2]
+  CRUSH rule 4 x 541 [2,4,1]
+  CRUSH rule 4 x 542 []
+  CRUSH rule 4 x 543 [8,2]
+  CRUSH rule 4 x 544 [4,8]
+  CRUSH rule 4 x 545 [8,2]
+  CRUSH rule 4 x 546 [8,1]
+  CRUSH rule 4 x 547 [8,2,1]
+  CRUSH rule 4 x 548 [4,2,1]
+  CRUSH rule 4 x 549 [8,2]
+  CRUSH rule 4 x 550 [2,4]
+  CRUSH rule 4 x 551 [8]
+  CRUSH rule 4 x 552 [4]
+  CRUSH rule 4 x 553 [2]
+  CRUSH rule 4 x 554 [1,8]
+  CRUSH rule 4 x 555 [4,1,8]
+  CRUSH rule 4 x 556 []
+  CRUSH rule 4 x 557 [8]
+  CRUSH rule 4 x 558 [4,1,2]
+  CRUSH rule 4 x 559 [2,8]
+  CRUSH rule 4 x 560 [8]
+  CRUSH rule 4 x 561 [8,4]
+  CRUSH rule 4 x 562 [4,1]
+  CRUSH rule 4 x 563 [2,8,1]
+  CRUSH rule 4 x 564 [1,8]
+  CRUSH rule 4 x 565 [4,8]
+  CRUSH rule 4 x 566 [4,8,2]
+  CRUSH rule 4 x 567 [4,8,1]
+  CRUSH rule 4 x 568 [8,1]
+  CRUSH rule 4 x 569 [4,2]
+  CRUSH rule 4 x 570 [1]
+  CRUSH rule 4 x 571 [8]
+  CRUSH rule 4 x 572 [4,1]
+  CRUSH rule 4 x 573 [1,8]
+  CRUSH rule 4 x 574 [2,1]
+  CRUSH rule 4 x 575 [8,1]
+  CRUSH rule 4 x 576 [4,8]
+  CRUSH rule 4 x 577 [8,2]
+  CRUSH rule 4 x 578 [8]
+  CRUSH rule 4 x 579 [4,1,8]
+  CRUSH rule 4 x 580 [2,8]
+  CRUSH rule 4 x 581 [8,2]
+  CRUSH rule 4 x 582 [2,8]
+  CRUSH rule 4 x 583 [8,1]
+  CRUSH rule 4 x 584 [8,1,2]
+  CRUSH rule 4 x 585 [8,1,4]
+  CRUSH rule 4 x 586 [1,2,8]
+  CRUSH rule 4 x 587 [2,4,1]
+  CRUSH rule 4 x 588 [4,8]
+  CRUSH rule 4 x 589 [8,1]
+  CRUSH rule 4 x 590 [8,2]
+  CRUSH rule 4 x 591 [4,2]
+  CRUSH rule 4 x 592 [2,1,4]
+  CRUSH rule 4 x 593 [1,8,2]
+  CRUSH rule 4 x 594 [2,8]
+  CRUSH rule 4 x 595 [8,1]
+  CRUSH rule 4 x 596 []
+  CRUSH rule 4 x 597 [1,2]
+  CRUSH rule 4 x 598 [2]
+  CRUSH rule 4 x 599 [4,2,1]
+  CRUSH rule 4 x 600 [8,1]
+  CRUSH rule 4 x 601 [1,8]
+  CRUSH rule 4 x 602 [8]
+  CRUSH rule 4 x 603 [1,8]
+  CRUSH rule 4 x 604 [8]
+  CRUSH rule 4 x 605 [2]
+  CRUSH rule 4 x 606 [2,1]
+  CRUSH rule 4 x 607 [2,4,8]
+  CRUSH rule 4 x 608 [4,2]
+  CRUSH rule 4 x 609 [4,2]
+  CRUSH rule 4 x 610 [8]
+  CRUSH rule 4 x 611 [1,2]
+  CRUSH rule 4 x 612 [2,1,8]
+  CRUSH rule 4 x 613 [8,2,1]
+  CRUSH rule 4 x 614 [8]
+  CRUSH rule 4 x 615 [8,2]
+  CRUSH rule 4 x 616 [1,8,2]
+  CRUSH rule 4 x 617 [8,1,2]
+  CRUSH rule 4 x 618 [8,4]
+  CRUSH rule 4 x 619 [4,1,8]
+  CRUSH rule 4 x 620 [1,2]
+  CRUSH rule 4 x 621 [8]
+  CRUSH rule 4 x 622 [2,4]
+  CRUSH rule 4 x 623 [2,8,1]
+  CRUSH rule 4 x 624 [4,1]
+  CRUSH rule 4 x 625 [2,8]
+  CRUSH rule 4 x 626 [8,2]
+  CRUSH rule 4 x 627 [2,8,1]
+  CRUSH rule 4 x 628 [8,2]
+  CRUSH rule 4 x 629 [2,8,4]
+  CRUSH rule 4 x 630 [2,8,1]
+  CRUSH rule 4 x 631 [1,8]
+  CRUSH rule 4 x 632 [8,2,1]
+  CRUSH rule 4 x 633 [8]
+  CRUSH rule 4 x 634 [1,2]
+  CRUSH rule 4 x 635 [4,8]
+  CRUSH rule 4 x 636 [1,4,2]
+  CRUSH rule 4 x 637 [1,2]
+  CRUSH rule 4 x 638 [8,1]
+  CRUSH rule 4 x 639 [2,1]
+  CRUSH rule 4 x 640 [1,2]
+  CRUSH rule 4 x 641 [8,2,1]
+  CRUSH rule 4 x 642 [2,1]
+  CRUSH rule 4 x 643 [1]
+  CRUSH rule 4 x 644 [8,1]
+  CRUSH rule 4 x 645 [8]
+  CRUSH rule 4 x 646 [8,1,4]
+  CRUSH rule 4 x 647 [8,1]
+  CRUSH rule 4 x 648 [1,8]
+  CRUSH rule 4 x 649 [4,8]
+  CRUSH rule 4 x 650 [8]
+  CRUSH rule 4 x 651 [4,6,8]
+  CRUSH rule 4 x 652 [4,8]
+  CRUSH rule 4 x 653 [8,2]
+  CRUSH rule 4 x 654 [6,2]
+  CRUSH rule 4 x 655 [1,4]
+  CRUSH rule 4 x 656 [8]
+  CRUSH rule 4 x 657 [6,1,2]
+  CRUSH rule 4 x 658 [8]
+  CRUSH rule 4 x 659 [4,8]
+  CRUSH rule 4 x 660 [8]
+  CRUSH rule 4 x 661 [1,8]
+  CRUSH rule 4 x 662 []
+  CRUSH rule 4 x 663 [1,4,8]
+  CRUSH rule 4 x 664 [1,4,2]
+  CRUSH rule 4 x 665 [4,6,8]
+  CRUSH rule 4 x 666 [2,8]
+  CRUSH rule 4 x 667 [1,4]
+  CRUSH rule 4 x 668 [4,8]
+  CRUSH rule 4 x 669 [6,4]
+  CRUSH rule 4 x 670 [4,2]
+  CRUSH rule 4 x 671 [2,1]
+  CRUSH rule 4 x 672 [4,2]
+  CRUSH rule 4 x 673 [4,2]
+  CRUSH rule 4 x 674 [1,8]
+  CRUSH rule 4 x 675 [1,8,6]
+  CRUSH rule 4 x 676 [2,1,4]
+  CRUSH rule 4 x 677 [4,1]
+  CRUSH rule 4 x 678 [2,4,1]
+  CRUSH rule 4 x 679 [8,2,1]
+  CRUSH rule 4 x 680 [2,8]
+  CRUSH rule 4 x 681 [8]
+  CRUSH rule 4 x 682 [1,4,2]
+  CRUSH rule 4 x 683 [1,2,4]
+  CRUSH rule 4 x 684 [8,1]
+  CRUSH rule 4 x 685 [8,1]
+  CRUSH rule 4 x 686 [1,4]
+  CRUSH rule 4 x 687 [6]
+  CRUSH rule 4 x 688 [4,8,2]
+  CRUSH rule 4 x 689 [8,4,2]
+  CRUSH rule 4 x 690 [8,1]
+  CRUSH rule 4 x 691 [1]
+  CRUSH rule 4 x 692 [8,2]
+  CRUSH rule 4 x 693 [8,4]
+  CRUSH rule 4 x 694 [8,4,1]
+  CRUSH rule 4 x 695 [2,8]
+  CRUSH rule 4 x 696 [1]
+  CRUSH rule 4 x 697 [8,1,2]
+  CRUSH rule 4 x 698 [8,2,1]
+  CRUSH rule 4 x 699 [1,8]
+  CRUSH rule 4 x 700 [1,2]
+  CRUSH rule 4 x 701 [2]
+  CRUSH rule 4 x 702 [1]
+  CRUSH rule 4 x 703 [8]
+  CRUSH rule 4 x 704 [1,4,8]
+  CRUSH rule 4 x 705 [8,2]
+  CRUSH rule 4 x 706 [1,2,4]
+  CRUSH rule 4 x 707 [8]
+  CRUSH rule 4 x 708 [4,8]
+  CRUSH rule 4 x 709 [8,2]
+  CRUSH rule 4 x 710 [8]
+  CRUSH rule 4 x 711 [2,4,8]
+  CRUSH rule 4 x 712 [2,8]
+  CRUSH rule 4 x 713 [8]
+  CRUSH rule 4 x 714 [2,1]
+  CRUSH rule 4 x 715 [1,2]
+  CRUSH rule 4 x 716 [4,8,2]
+  CRUSH rule 4 x 717 [8,2]
+  CRUSH rule 4 x 718 [8,6]
+  CRUSH rule 4 x 719 [2,6,4]
+  CRUSH rule 4 x 720 [8,2]
+  CRUSH rule 4 x 721 [4,6]
+  CRUSH rule 4 x 722 [8]
+  CRUSH rule 4 x 723 [4,1,2]
+  CRUSH rule 4 x 724 [2,6,1]
+  CRUSH rule 4 x 725 [1,2]
+  CRUSH rule 4 x 726 [4,8]
+  CRUSH rule 4 x 727 [4,8]
+  CRUSH rule 4 x 728 [2,1]
+  CRUSH rule 4 x 729 [8]
+  CRUSH rule 4 x 730 [4,8]
+  CRUSH rule 4 x 731 [4,1]
+  CRUSH rule 4 x 732 [1]
+  CRUSH rule 4 x 733 [4,8]
+  CRUSH rule 4 x 734 [8,4,2]
+  CRUSH rule 4 x 735 [4,8]
+  CRUSH rule 4 x 736 [4,8]
+  CRUSH rule 4 x 737 [1,2,8]
+  CRUSH rule 4 x 738 [4,2]
+  CRUSH rule 4 x 739 [2,1]
+  CRUSH rule 4 x 740 [1,2,8]
+  CRUSH rule 4 x 741 [8,1]
+  CRUSH rule 4 x 742 [8,2,1]
+  CRUSH rule 4 x 743 [8,1,2]
+  CRUSH rule 4 x 744 [4,8]
+  CRUSH rule 4 x 745 [1]
+  CRUSH rule 4 x 746 [1]
+  CRUSH rule 4 x 747 [8,1]
+  CRUSH rule 4 x 748 [2,8,1]
+  CRUSH rule 4 x 749 [4,8]
+  CRUSH rule 4 x 750 [1,8,4]
+  CRUSH rule 4 x 751 [2,1,8]
+  CRUSH rule 4 x 752 [8,1]
+  CRUSH rule 4 x 753 [8,4]
+  CRUSH rule 4 x 754 [8]
+  CRUSH rule 4 x 755 [1,2]
+  CRUSH rule 4 x 756 [8,1]
+  CRUSH rule 4 x 757 [8,1]
+  CRUSH rule 4 x 758 [8,2]
+  CRUSH rule 4 x 759 [8,4]
+  CRUSH rule 4 x 760 [1,4]
+  CRUSH rule 4 x 761 [1,2]
+  CRUSH rule 4 x 762 [2,8]
+  CRUSH rule 4 x 763 [8]
+  CRUSH rule 4 x 764 [1,8,2]
+  CRUSH rule 4 x 765 [8,2]
+  CRUSH rule 4 x 766 [8]
+  CRUSH rule 4 x 767 [1,2]
+  CRUSH rule 4 x 768 [8,4,2]
+  CRUSH rule 4 x 769 [8,2]
+  CRUSH rule 4 x 770 [8,2]
+  CRUSH rule 4 x 771 [8,1,4]
+  CRUSH rule 4 x 772 [8,4]
+  CRUSH rule 4 x 773 [4,1]
+  CRUSH rule 4 x 774 [8]
+  CRUSH rule 4 x 775 [8,4]
+  CRUSH rule 4 x 776 [6,2,1]
+  CRUSH rule 4 x 777 [4,1,8]
+  CRUSH rule 4 x 778 [1,8,2]
+  CRUSH rule 4 x 779 [2,8]
+  CRUSH rule 4 x 780 [2,1,4]
+  CRUSH rule 4 x 781 [8]
+  CRUSH rule 4 x 782 [4,1]
+  CRUSH rule 4 x 783 [8,1]
+  CRUSH rule 4 x 784 [1,2,4]
+  CRUSH rule 4 x 785 [8,1,2]
+  CRUSH rule 4 x 786 [8]
+  CRUSH rule 4 x 787 [1,2,6]
+  CRUSH rule 4 x 788 [8,2]
+  CRUSH rule 4 x 789 [1]
+  CRUSH rule 4 x 790 [8]
+  CRUSH rule 4 x 791 [4,8]
+  CRUSH rule 4 x 792 [4,8]
+  CRUSH rule 4 x 793 [8,1,4]
+  CRUSH rule 4 x 794 [2,8]
+  CRUSH rule 4 x 795 [1,2]
+  CRUSH rule 4 x 796 []
+  CRUSH rule 4 x 797 [2,4]
+  CRUSH rule 4 x 798 [6,8]
+  CRUSH rule 4 x 799 [4,1]
+  CRUSH rule 4 x 800 [2]
+  CRUSH rule 4 x 801 [4,8]
+  CRUSH rule 4 x 802 [1,8,2]
+  CRUSH rule 4 x 803 [2,8]
+  CRUSH rule 4 x 804 [8,2]
+  CRUSH rule 4 x 805 [8]
+  CRUSH rule 4 x 806 [1,4]
+  CRUSH rule 4 x 807 [4,8]
+  CRUSH rule 4 x 808 [8,2]
+  CRUSH rule 4 x 809 [1]
+  CRUSH rule 4 x 810 [8]
+  CRUSH rule 4 x 811 [8]
+  CRUSH rule 4 x 812 [8,4]
+  CRUSH rule 4 x 813 [8,4]
+  CRUSH rule 4 x 814 [8]
+  CRUSH rule 4 x 815 [4,1,2]
+  CRUSH rule 4 x 816 [2,1,8]
+  CRUSH rule 4 x 817 [8]
+  CRUSH rule 4 x 818 [1]
+  CRUSH rule 4 x 819 [1]
+  CRUSH rule 4 x 820 [4]
+  CRUSH rule 4 x 821 [4,8]
+  CRUSH rule 4 x 822 [2,1,4]
+  CRUSH rule 4 x 823 [4,8,2]
+  CRUSH rule 4 x 824 [8]
+  CRUSH rule 4 x 825 [2,8]
+  CRUSH rule 4 x 826 [8,2,4]
+  CRUSH rule 4 x 827 [2,8,1]
+  CRUSH rule 4 x 828 [2,1]
+  CRUSH rule 4 x 829 [8]
+  CRUSH rule 4 x 830 [2,4]
+  CRUSH rule 4 x 831 [1,8,2]
+  CRUSH rule 4 x 832 [4]
+  CRUSH rule 4 x 833 [2,1,8]
+  CRUSH rule 4 x 834 [2]
+  CRUSH rule 4 x 835 [8,4]
+  CRUSH rule 4 x 836 [4]
+  CRUSH rule 4 x 837 [8,4,1]
+  CRUSH rule 4 x 838 [6,8,2]
+  CRUSH rule 4 x 839 [2,6]
+  CRUSH rule 4 x 840 [8]
+  CRUSH rule 4 x 841 [4,8]
+  CRUSH rule 4 x 842 [2,1]
+  CRUSH rule 4 x 843 [8,4]
+  CRUSH rule 4 x 844 [8,1]
+  CRUSH rule 4 x 845 [4,8]
+  CRUSH rule 4 x 846 [4,2,1]
+  CRUSH rule 4 x 847 [2,1,8]
+  CRUSH rule 4 x 848 [2,8,1]
+  CRUSH rule 4 x 849 [4]
+  CRUSH rule 4 x 850 [1,2]
+  CRUSH rule 4 x 851 [6,8]
+  CRUSH rule 4 x 852 [8,4]
+  CRUSH rule 4 x 853 [6,8,1]
+  CRUSH rule 4 x 854 [8,1]
+  CRUSH rule 4 x 855 [8,2]
+  CRUSH rule 4 x 856 [8,4]
+  CRUSH rule 4 x 857 [8,2]
+  CRUSH rule 4 x 858 [6,1]
+  CRUSH rule 4 x 859 [8,2]
+  CRUSH rule 4 x 860 [1,2]
+  CRUSH rule 4 x 861 [8]
+  CRUSH rule 4 x 862 [8,1]
+  CRUSH rule 4 x 863 [8,2]
+  CRUSH rule 4 x 864 [8]
+  CRUSH rule 4 x 865 [8,1,2]
+  CRUSH rule 4 x 866 [8]
+  CRUSH rule 4 x 867 [8,1]
+  CRUSH rule 4 x 868 [8,1]
+  CRUSH rule 4 x 869 [8,6,4]
+  CRUSH rule 4 x 870 [2,8]
+  CRUSH rule 4 x 871 []
+  CRUSH rule 4 x 872 [1]
+  CRUSH rule 4 x 873 [4,8]
+  CRUSH rule 4 x 874 [2,6,1]
+  CRUSH rule 4 x 875 [2,8,4]
+  CRUSH rule 4 x 876 [4,8,1]
+  CRUSH rule 4 x 877 [8,4,2]
+  CRUSH rule 4 x 878 [2]
+  CRUSH rule 4 x 879 [8]
+  CRUSH rule 4 x 880 [1]
+  CRUSH rule 4 x 881 [4,8,1]
+  CRUSH rule 4 x 882 [2,1]
+  CRUSH rule 4 x 883 [2,1]
+  CRUSH rule 4 x 884 [8,2,4]
+  CRUSH rule 4 x 885 [4,1]
+  CRUSH rule 4 x 886 [8]
+  CRUSH rule 4 x 887 [8,4,1]
+  CRUSH rule 4 x 888 [8,1]
+  CRUSH rule 4 x 889 [2,1,8]
+  CRUSH rule 4 x 890 [8,2,1]
+  CRUSH rule 4 x 891 [1,8,2]
+  CRUSH rule 4 x 892 [8,2,4]
+  CRUSH rule 4 x 893 [2,6]
+  CRUSH rule 4 x 894 [8,4,2]
+  CRUSH rule 4 x 895 [4,2]
+  CRUSH rule 4 x 896 [1,8,2]
+  CRUSH rule 4 x 897 [2,8]
+  CRUSH rule 4 x 898 [1,4]
+  CRUSH rule 4 x 899 [1,8]
+  CRUSH rule 4 x 900 [4,1,2]
+  CRUSH rule 4 x 901 [1,2]
+  CRUSH rule 4 x 902 [8,4]
+  CRUSH rule 4 x 903 [8]
+  CRUSH rule 4 x 904 [8]
+  CRUSH rule 4 x 905 [8,2]
+  CRUSH rule 4 x 906 [1,2]
+  CRUSH rule 4 x 907 [8,1,2]
+  CRUSH rule 4 x 908 [8,1]
+  CRUSH rule 4 x 909 [2]
+  CRUSH rule 4 x 910 [8,2]
+  CRUSH rule 4 x 911 [8]
+  CRUSH rule 4 x 912 [1,2,8]
+  CRUSH rule 4 x 913 [8]
+  CRUSH rule 4 x 914 [6,4,8]
+  CRUSH rule 4 x 915 [8,2]
+  CRUSH rule 4 x 916 [4,1]
+  CRUSH rule 4 x 917 [1,4]
+  CRUSH rule 4 x 918 [8,2,1]
+  CRUSH rule 4 x 919 [8,2]
+  CRUSH rule 4 x 920 [8]
+  CRUSH rule 4 x 921 [1]
+  CRUSH rule 4 x 922 [8]
+  CRUSH rule 4 x 923 [4,8]
+  CRUSH rule 4 x 924 []
+  CRUSH rule 4 x 925 [4,8]
+  CRUSH rule 4 x 926 []
+  CRUSH rule 4 x 927 [1,8,4]
+  CRUSH rule 4 x 928 [8,1,2]
+  CRUSH rule 4 x 929 [4,1]
+  CRUSH rule 4 x 930 [2,8]
+  CRUSH rule 4 x 931 [1,2]
+  CRUSH rule 4 x 932 [4,1]
+  CRUSH rule 4 x 933 [8,4]
+  CRUSH rule 4 x 934 [8]
+  CRUSH rule 4 x 935 [8]
+  CRUSH rule 4 x 936 [1,8]
+  CRUSH rule 4 x 937 [4]
+  CRUSH rule 4 x 938 [8,4]
+  CRUSH rule 4 x 939 [2,8,1]
+  CRUSH rule 4 x 940 [8]
+  CRUSH rule 4 x 941 [2,1]
+  CRUSH rule 4 x 942 [1,2]
+  CRUSH rule 4 x 943 [8,2]
+  CRUSH rule 4 x 944 [8]
+  CRUSH rule 4 x 945 [8,2,4]
+  CRUSH rule 4 x 946 [2,1,8]
+  CRUSH rule 4 x 947 []
+  CRUSH rule 4 x 948 [8]
+  CRUSH rule 4 x 949 [6,1,8]
+  CRUSH rule 4 x 950 [8]
+  CRUSH rule 4 x 951 []
+  CRUSH rule 4 x 952 [2,1,8]
+  CRUSH rule 4 x 953 [1,4]
+  CRUSH rule 4 x 954 [2]
+  CRUSH rule 4 x 955 [8,1]
+  CRUSH rule 4 x 956 [1,2,8]
+  CRUSH rule 4 x 957 [8,1]
+  CRUSH rule 4 x 958 [8,4]
+  CRUSH rule 4 x 959 [4,2,8]
+  CRUSH rule 4 x 960 [6]
+  CRUSH rule 4 x 961 [1,2]
+  CRUSH rule 4 x 962 [8,4]
+  CRUSH rule 4 x 963 [2,4,1]
+  CRUSH rule 4 x 964 [1]
+  CRUSH rule 4 x 965 [8]
+  CRUSH rule 4 x 966 [4,8]
+  CRUSH rule 4 x 967 [8,4]
+  CRUSH rule 4 x 968 [8,2]
+  CRUSH rule 4 x 969 [8,2]
+  CRUSH rule 4 x 970 [2,8,4]
+  CRUSH rule 4 x 971 [1,8]
+  CRUSH rule 4 x 972 [1,8]
+  CRUSH rule 4 x 973 [1,2]
+  CRUSH rule 4 x 974 [4,2]
+  CRUSH rule 4 x 975 [4,8]
+  CRUSH rule 4 x 976 [4]
+  CRUSH rule 4 x 977 [8,4,2]
+  CRUSH rule 4 x 978 [8,2]
+  CRUSH rule 4 x 979 [8,2]
+  CRUSH rule 4 x 980 [8,2]
+  CRUSH rule 4 x 981 [8,2]
+  CRUSH rule 4 x 982 [2,1]
+  CRUSH rule 4 x 983 [4,8]
+  CRUSH rule 4 x 984 [2,1]
+  CRUSH rule 4 x 985 [2,4]
+  CRUSH rule 4 x 986 [8,4]
+  CRUSH rule 4 x 987 [2,1]
+  CRUSH rule 4 x 988 [1,4]
+  CRUSH rule 4 x 989 [1,8]
+  CRUSH rule 4 x 990 [1,2,8]
+  CRUSH rule 4 x 991 [1,4,2]
+  CRUSH rule 4 x 992 [8,1,4]
+  CRUSH rule 4 x 993 [2,8,1]
+  CRUSH rule 4 x 994 [4]
+  CRUSH rule 4 x 995 [8,2]
+  CRUSH rule 4 x 996 [8,4]
+  CRUSH rule 4 x 997 [8,4,1]
+  CRUSH rule 4 x 998 [8,1,2]
+  CRUSH rule 4 x 999 [1,8]
+  CRUSH rule 4 x 1000 [8,4,2]
+  CRUSH rule 4 x 1001 [2,1]
+  CRUSH rule 4 x 1002 [1,2]
+  CRUSH rule 4 x 1003 [2,8]
+  CRUSH rule 4 x 1004 [8,1,2]
+  CRUSH rule 4 x 1005 [8,1,2]
+  CRUSH rule 4 x 1006 [1,2]
+  CRUSH rule 4 x 1007 [1,2,4]
+  CRUSH rule 4 x 1008 [1,8,2]
+  CRUSH rule 4 x 1009 [6,8,4]
+  CRUSH rule 4 x 1010 [1]
+  CRUSH rule 4 x 1011 [4,2]
+  CRUSH rule 4 x 1012 [2,8]
+  CRUSH rule 4 x 1013 [1,2]
+  CRUSH rule 4 x 1014 [2,8,4]
+  CRUSH rule 4 x 1015 [8]
+  CRUSH rule 4 x 1016 [2,1]
+  CRUSH rule 4 x 1017 [6,2,1]
+  CRUSH rule 4 x 1018 [4]
+  CRUSH rule 4 x 1019 [4,8]
+  CRUSH rule 4 x 1020 [1]
+  CRUSH rule 4 x 1021 [2,1]
+  CRUSH rule 4 x 1022 [1,8]
+  CRUSH rule 4 x 1023 [4,2,1]
+  rule 4 (choose-set-two) num_rep 3 result size == 0:\t19/1024 (esc)
+  rule 4 (choose-set-two) num_rep 3 result size == 1:\t210/1024 (esc)
+  rule 4 (choose-set-two) num_rep 3 result size == 2:\t508/1024 (esc)
+  rule 4 (choose-set-two) num_rep 3 result size == 3:\t287/1024 (esc)
+  rule 5 (chooseleaf-set), x = 0..1023, numrep = 2..3
+  CRUSH rule 5 x 0 [2,4]
+  CRUSH rule 5 x 1 [2,8]
+  CRUSH rule 5 x 2 [1,8]
+  CRUSH rule 5 x 3 [8,1]
+  CRUSH rule 5 x 4 [4,2]
+  CRUSH rule 5 x 5 [8,2]
+  CRUSH rule 5 x 6 [2,8]
+  CRUSH rule 5 x 7 [4,8]
+  CRUSH rule 5 x 8 [4,8]
+  CRUSH rule 5 x 9 [2,4]
+  CRUSH rule 5 x 10 [2,8]
+  CRUSH rule 5 x 11 [2,8]
+  CRUSH rule 5 x 12 [2,8]
+  CRUSH rule 5 x 13 [4,8]
+  CRUSH rule 5 x 14 [8,2]
+  CRUSH rule 5 x 15 [8,2]
+  CRUSH rule 5 x 16 [8,2]
+  CRUSH rule 5 x 17 [4,1]
+  CRUSH rule 5 x 18 [1,8]
+  CRUSH rule 5 x 19 [8,4]
+  CRUSH rule 5 x 20 [2,8]
+  CRUSH rule 5 x 21 [8,2]
+  CRUSH rule 5 x 22 [8,1]
+  CRUSH rule 5 x 23 [4,8]
+  CRUSH rule 5 x 24 [1,8]
+  CRUSH rule 5 x 25 [4,8]
+  CRUSH rule 5 x 26 [2,8]
+  CRUSH rule 5 x 27 [4,1]
+  CRUSH rule 5 x 28 [8,2]
+  CRUSH rule 5 x 29 [8,4]
+  CRUSH rule 5 x 30 [4,8]
+  CRUSH rule 5 x 31 [8,1]
+  CRUSH rule 5 x 32 [6,1]
+  CRUSH rule 5 x 33 [2,8]
+  CRUSH rule 5 x 34 [2,8]
+  CRUSH rule 5 x 35 [1,8]
+  CRUSH rule 5 x 36 [8,2]
+  CRUSH rule 5 x 37 [1,8]
+  CRUSH rule 5 x 38 [4,8]
+  CRUSH rule 5 x 39 [8,2]
+  CRUSH rule 5 x 40 [8,2]
+  CRUSH rule 5 x 41 [2,8]
+  CRUSH rule 5 x 42 [8,2]
+  CRUSH rule 5 x 43 [1,8]
+  CRUSH rule 5 x 44 [1,8]
+  CRUSH rule 5 x 45 [8,2]
+  CRUSH rule 5 x 46 [2,8]
+  CRUSH rule 5 x 47 [4,2]
+  CRUSH rule 5 x 48 [8,1]
+  CRUSH rule 5 x 49 [8,2]
+  CRUSH rule 5 x 50 [4,1]
+  CRUSH rule 5 x 51 [8,2]
+  CRUSH rule 5 x 52 [8,1]
+  CRUSH rule 5 x 53 [4,8]
+  CRUSH rule 5 x 54 [8,4]
+  CRUSH rule 5 x 55 [8,2]
+  CRUSH rule 5 x 56 [8,4]
+  CRUSH rule 5 x 57 [8,1]
+  CRUSH rule 5 x 58 [1,8]
+  CRUSH rule 5 x 59 [2,8]
+  CRUSH rule 5 x 60 [4,2]
+  CRUSH rule 5 x 61 [4,8]
+  CRUSH rule 5 x 62 [8,1]
+  CRUSH rule 5 x 63 [8,2]
+  CRUSH rule 5 x 64 [4,2]
+  CRUSH rule 5 x 65 [8,4]
+  CRUSH rule 5 x 66 [4,8]
+  CRUSH rule 5 x 67 [4,2]
+  CRUSH rule 5 x 68 [1,8]
+  CRUSH rule 5 x 69 [2,8]
+  CRUSH rule 5 x 70 [8,2]
+  CRUSH rule 5 x 71 [2,8]
+  CRUSH rule 5 x 72 [8,1]
+  CRUSH rule 5 x 73 [2,8]
+  CRUSH rule 5 x 74 [1,8]
+  CRUSH rule 5 x 75 [4,2]
+  CRUSH rule 5 x 76 [4,1]
+  CRUSH rule 5 x 77 [8,2]
+  CRUSH rule 5 x 78 [1,6]
+  CRUSH rule 5 x 79 [4,1]
+  CRUSH rule 5 x 80 [2,4]
+  CRUSH rule 5 x 81 [2,8]
+  CRUSH rule 5 x 82 [6,1]
+  CRUSH rule 5 x 83 [2,8]
+  CRUSH rule 5 x 84 [8,2]
+  CRUSH rule 5 x 85 [4,8]
+  CRUSH rule 5 x 86 [2,8]
+  CRUSH rule 5 x 87 [2,8]
+  CRUSH rule 5 x 88 [1,6]
+  CRUSH rule 5 x 89 [1,8]
+  CRUSH rule 5 x 90 [8,4]
+  CRUSH rule 5 x 91 [4,8]
+  CRUSH rule 5 x 92 [1,8]
+  CRUSH rule 5 x 93 [8,4]
+  CRUSH rule 5 x 94 [1,8]
+  CRUSH rule 5 x 95 [8,1]
+  CRUSH rule 5 x 96 [8,2]
+  CRUSH rule 5 x 97 [8,1]
+  CRUSH rule 5 x 98 [2,8]
+  CRUSH rule 5 x 99 [2,8]
+  CRUSH rule 5 x 100 [1,8]
+  CRUSH rule 5 x 101 [8,1]
+  CRUSH rule 5 x 102 [2,8]
+  CRUSH rule 5 x 103 [8,2]
+  CRUSH rule 5 x 104 [8,4]
+  CRUSH rule 5 x 105 [2,4]
+  CRUSH rule 5 x 106 [1,8]
+  CRUSH rule 5 x 107 [1,8]
+  CRUSH rule 5 x 108 [8,2]
+  CRUSH rule 5 x 109 [1,4]
+  CRUSH rule 5 x 110 [4,2]
+  CRUSH rule 5 x 111 [2,4]
+  CRUSH rule 5 x 112 [2,8]
+  CRUSH rule 5 x 113 [8,2]
+  CRUSH rule 5 x 114 [8,4]
+  CRUSH rule 5 x 115 [8,2]
+  CRUSH rule 5 x 116 [1,8]
+  CRUSH rule 5 x 117 [6,1]
+  CRUSH rule 5 x 118 [2,8]
+  CRUSH rule 5 x 119 [8,1]
+  CRUSH rule 5 x 120 [2,4]
+  CRUSH rule 5 x 121 [2,8]
+  CRUSH rule 5 x 122 [8,1]
+  CRUSH rule 5 x 123 [2,8]
+  CRUSH rule 5 x 124 [2,8]
+  CRUSH rule 5 x 125 [1,8]
+  CRUSH rule 5 x 126 [1,8]
+  CRUSH rule 5 x 127 [4,8]
+  CRUSH rule 5 x 128 [8,2]
+  CRUSH rule 5 x 129 [2,4]
+  CRUSH rule 5 x 130 [4,8]
+  CRUSH rule 5 x 131 [1,4]
+  CRUSH rule 5 x 132 [1,8]
+  CRUSH rule 5 x 133 [8,1]
+  CRUSH rule 5 x 134 [1,8]
+  CRUSH rule 5 x 135 [4,8]
+  CRUSH rule 5 x 136 [2,4]
+  CRUSH rule 5 x 137 [8,4]
+  CRUSH rule 5 x 138 [8,4]
+  CRUSH rule 5 x 139 [4,2]
+  CRUSH rule 5 x 140 [1,8]
+  CRUSH rule 5 x 141 [8,2]
+  CRUSH rule 5 x 142 [4,1]
+  CRUSH rule 5 x 143 [4,8]
+  CRUSH rule 5 x 144 [8,1]
+  CRUSH rule 5 x 145 [8,1]
+  CRUSH rule 5 x 146 [2,8]
+  CRUSH rule 5 x 147 [2,8]
+  CRUSH rule 5 x 148 [4,1]
+  CRUSH rule 5 x 149 [4,8]
+  CRUSH rule 5 x 150 [1,8]
+  CRUSH rule 5 x 151 [1,8]
+  CRUSH rule 5 x 152 [8,2]
+  CRUSH rule 5 x 153 [8,4]
+  CRUSH rule 5 x 154 [4,2]
+  CRUSH rule 5 x 155 [4,8]
+  CRUSH rule 5 x 156 [4,2]
+  CRUSH rule 5 x 157 [1,8]
+  CRUSH rule 5 x 158 [2,8]
+  CRUSH rule 5 x 159 [8,2]
+  CRUSH rule 5 x 160 [2,8]
+  CRUSH rule 5 x 161 [1,4]
+  CRUSH rule 5 x 162 [1,8]
+  CRUSH rule 5 x 163 [4,8]
+  CRUSH rule 5 x 164 [8,1]
+  CRUSH rule 5 x 165 [8,2]
+  CRUSH rule 5 x 166 [2,8]
+  CRUSH rule 5 x 167 [1,8]
+  CRUSH rule 5 x 168 [4,2]
+  CRUSH rule 5 x 169 [2,8]
+  CRUSH rule 5 x 170 [1,8]
+  CRUSH rule 5 x 171 [8,4]
+  CRUSH rule 5 x 172 [1,8]
+  CRUSH rule 5 x 173 [8,4]
+  CRUSH rule 5 x 174 [1,6]
+  CRUSH rule 5 x 175 [8,1]
+  CRUSH rule 5 x 176 [2,8]
+  CRUSH rule 5 x 177 [1,8]
+  CRUSH rule 5 x 178 [4,2]
+  CRUSH rule 5 x 179 [1,8]
+  CRUSH rule 5 x 180 [8,1]
+  CRUSH rule 5 x 181 [8,2]
+  CRUSH rule 5 x 182 [8,1]
+  CRUSH rule 5 x 183 [8,4]
+  CRUSH rule 5 x 184 [4,8]
+  CRUSH rule 5 x 185 [8,1]
+  CRUSH rule 5 x 186 [2,4]
+  CRUSH rule 5 x 187 [1,8]
+  CRUSH rule 5 x 188 [1,8]
+  CRUSH rule 5 x 189 [1,8]
+  CRUSH rule 5 x 190 [1,8]
+  CRUSH rule 5 x 191 [8,1]
+  CRUSH rule 5 x 192 [4,1]
+  CRUSH rule 5 x 193 [4,2]
+  CRUSH rule 5 x 194 [1,8]
+  CRUSH rule 5 x 195 [8,4]
+  CRUSH rule 5 x 196 [8,2]
+  CRUSH rule 5 x 197 [8,4]
+  CRUSH rule 5 x 198 [2,8]
+  CRUSH rule 5 x 199 [1,4]
+  CRUSH rule 5 x 200 [1,8]
+  CRUSH rule 5 x 201 [8,1]
+  CRUSH rule 5 x 202 [8,1]
+  CRUSH rule 5 x 203 [8,1]
+  CRUSH rule 5 x 204 [2,4]
+  CRUSH rule 5 x 205 [1,8]
+  CRUSH rule 5 x 206 [1,8]
+  CRUSH rule 5 x 207 [2,8]
+  CRUSH rule 5 x 208 [8,1]
+  CRUSH rule 5 x 209 [1,8]
+  CRUSH rule 5 x 210 [1,4]
+  CRUSH rule 5 x 211 [4,2]
+  CRUSH rule 5 x 212 [8,1]
+  CRUSH rule 5 x 213 [8,4]
+  CRUSH rule 5 x 214 [8,2]
+  CRUSH rule 5 x 215 [8,1]
+  CRUSH rule 5 x 216 [2,8]
+  CRUSH rule 5 x 217 [1,8]
+  CRUSH rule 5 x 218 [2,8]
+  CRUSH rule 5 x 219 [8,2]
+  CRUSH rule 5 x 220 [4,8]
+  CRUSH rule 5 x 221 [8,1]
+  CRUSH rule 5 x 222 [8,1]
+  CRUSH rule 5 x 223 [1,8]
+  CRUSH rule 5 x 224 [1,4]
+  CRUSH rule 5 x 225 [8,2]
+  CRUSH rule 5 x 226 [8,2]
+  CRUSH rule 5 x 227 [4,1]
+  CRUSH rule 5 x 228 [8,2]
+  CRUSH rule 5 x 229 [4,8]
+  CRUSH rule 5 x 230 [4,8]
+  CRUSH rule 5 x 231 [4,8]
+  CRUSH rule 5 x 232 [2,8]
+  CRUSH rule 5 x 233 [2,8]
+  CRUSH rule 5 x 234 [1,8]
+  CRUSH rule 5 x 235 [4,8]
+  CRUSH rule 5 x 236 [2,6]
+  CRUSH rule 5 x 237 [4,8]
+  CRUSH rule 5 x 238 [2,8]
+  CRUSH rule 5 x 239 [8,1]
+  CRUSH rule 5 x 240 [4,8]
+  CRUSH rule 5 x 241 [1,8]
+  CRUSH rule 5 x 242 [8,2]
+  CRUSH rule 5 x 243 [8,2]
+  CRUSH rule 5 x 244 [4,8]
+  CRUSH rule 5 x 245 [8,1]
+  CRUSH rule 5 x 246 [1,8]
+  CRUSH rule 5 x 247 [8,2]
+  CRUSH rule 5 x 248 [8,2]
+  CRUSH rule 5 x 249 [2,8]
+  CRUSH rule 5 x 250 [2,4]
+  CRUSH rule 5 x 251 [2,8]
+  CRUSH rule 5 x 252 [4,8]
+  CRUSH rule 5 x 253 [2,8]
+  CRUSH rule 5 x 254 [4,2]
+  CRUSH rule 5 x 255 [1,8]
+  CRUSH rule 5 x 256 [4,8]
+  CRUSH rule 5 x 257 [2,8]
+  CRUSH rule 5 x 258 [4,2]
+  CRUSH rule 5 x 259 [6,2]
+  CRUSH rule 5 x 260 [8,2]
+  CRUSH rule 5 x 261 [8,1]
+  CRUSH rule 5 x 262 [8,1]
+  CRUSH rule 5 x 263 [8,1]
+  CRUSH rule 5 x 264 [8,2]
+  CRUSH rule 5 x 265 [8,2]
+  CRUSH rule 5 x 266 [8,2]
+  CRUSH rule 5 x 267 [2,8]
+  CRUSH rule 5 x 268 [1,8]
+  CRUSH rule 5 x 269 [1,8]
+  CRUSH rule 5 x 270 [4,1]
+  CRUSH rule 5 x 271 [8,4]
+  CRUSH rule 5 x 272 [2,8]
+  CRUSH rule 5 x 273 [4,1]
+  CRUSH rule 5 x 274 [8,4]
+  CRUSH rule 5 x 275 [4,8]
+  CRUSH rule 5 x 276 [8,1]
+  CRUSH rule 5 x 277 [8,1]
+  CRUSH rule 5 x 278 [8,1]
+  CRUSH rule 5 x 279 [8,4]
+  CRUSH rule 5 x 280 [2,8]
+  CRUSH rule 5 x 281 [8,2]
+  CRUSH rule 5 x 282 [2,8]
+  CRUSH rule 5 x 283 [8,2]
+  CRUSH rule 5 x 284 [8,2]
+  CRUSH rule 5 x 285 [4,8]
+  CRUSH rule 5 x 286 [2,8]
+  CRUSH rule 5 x 287 [1,8]
+  CRUSH rule 5 x 288 [8,1]
+  CRUSH rule 5 x 289 [4,8]
+  CRUSH rule 5 x 290 [1,4]
+  CRUSH rule 5 x 291 [1,4]
+  CRUSH rule 5 x 292 [8,2]
+  CRUSH rule 5 x 293 [8,1]
+  CRUSH rule 5 x 294 [8,4]
+  CRUSH rule 5 x 295 [4,8]
+  CRUSH rule 5 x 296 [4,1]
+  CRUSH rule 5 x 297 [8,2]
+  CRUSH rule 5 x 298 [1,8]
+  CRUSH rule 5 x 299 [2,8]
+  CRUSH rule 5 x 300 [8,2]
+  CRUSH rule 5 x 301 [1,8]
+  CRUSH rule 5 x 302 [1,8]
+  CRUSH rule 5 x 303 [8,4]
+  CRUSH rule 5 x 304 [2,8]
+  CRUSH rule 5 x 305 [8,2]
+  CRUSH rule 5 x 306 [1,8]
+  CRUSH rule 5 x 307 [2,8]
+  CRUSH rule 5 x 308 [2,8]
+  CRUSH rule 5 x 309 [8,1]
+  CRUSH rule 5 x 310 [4,1]
+  CRUSH rule 5 x 311 [4,8]
+  CRUSH rule 5 x 312 [2,8]
+  CRUSH rule 5 x 313 [4,1]
+  CRUSH rule 5 x 314 [2,8]
+  CRUSH rule 5 x 315 [2,8]
+  CRUSH rule 5 x 316 [8,1]
+  CRUSH rule 5 x 317 [2,8]
+  CRUSH rule 5 x 318 [8,1]
+  CRUSH rule 5 x 319 [2,8]
+  CRUSH rule 5 x 320 [8,1]
+  CRUSH rule 5 x 321 [1,8]
+  CRUSH rule 5 x 322 [2,8]
+  CRUSH rule 5 x 323 [4,8]
+  CRUSH rule 5 x 324 [8,1]
+  CRUSH rule 5 x 325 [4,8]
+  CRUSH rule 5 x 326 [1,6]
+  CRUSH rule 5 x 327 [1,8]
+  CRUSH rule 5 x 328 [8,4]
+  CRUSH rule 5 x 329 [4,8]
+  CRUSH rule 5 x 330 [4,8]
+  CRUSH rule 5 x 331 [2,8]
+  CRUSH rule 5 x 332 [2,8]
+  CRUSH rule 5 x 333 [8,1]
+  CRUSH rule 5 x 334 [8,2]
+  CRUSH rule 5 x 335 [8,1]
+  CRUSH rule 5 x 336 [4,8]
+  CRUSH rule 5 x 337 [8,2]
+  CRUSH rule 5 x 338 [8,1]
+  CRUSH rule 5 x 339 [8,2]
+  CRUSH rule 5 x 340 [2,8]
+  CRUSH rule 5 x 341 [4,1]
+  CRUSH rule 5 x 342 [2,8]
+  CRUSH rule 5 x 343 [8,1]
+  CRUSH rule 5 x 344 [6,2]
+  CRUSH rule 5 x 345 [2,8]
+  CRUSH rule 5 x 346 [8,2]
+  CRUSH rule 5 x 347 [4,1]
+  CRUSH rule 5 x 348 [8,2]
+  CRUSH rule 5 x 349 [1,8]
+  CRUSH rule 5 x 350 [8,1]
+  CRUSH rule 5 x 351 [8,2]
+  CRUSH rule 5 x 352 [1,8]
+  CRUSH rule 5 x 353 [8,1]
+  CRUSH rule 5 x 354 [1,8]
+  CRUSH rule 5 x 355 [8,2]
+  CRUSH rule 5 x 356 [4,1]
+  CRUSH rule 5 x 357 [8,1]
+  CRUSH rule 5 x 358 [2,8]
+  CRUSH rule 5 x 359 [6,1]
+  CRUSH rule 5 x 360 [2,8]
+  CRUSH rule 5 x 361 [8,4]
+  CRUSH rule 5 x 362 [4,1]
+  CRUSH rule 5 x 363 [4,1]
+  CRUSH rule 5 x 364 [2,8]
+  CRUSH rule 5 x 365 [8,1]
+  CRUSH rule 5 x 366 [8,2]
+  CRUSH rule 5 x 367 [4,2]
+  CRUSH rule 5 x 368 [8,4]
+  CRUSH rule 5 x 369 [8,1]
+  CRUSH rule 5 x 370 [8,2]
+  CRUSH rule 5 x 371 [1,4]
+  CRUSH rule 5 x 372 [1,8]
+  CRUSH rule 5 x 373 [1,8]
+  CRUSH rule 5 x 374 [8,1]
+  CRUSH rule 5 x 375 [8,4]
+  CRUSH rule 5 x 376 [8,1]
+  CRUSH rule 5 x 377 [1,4]
+  CRUSH rule 5 x 378 [1,8]
+  CRUSH rule 5 x 379 [8,2]
+  CRUSH rule 5 x 380 [2,8]
+  CRUSH rule 5 x 381 [1,4]
+  CRUSH rule 5 x 382 [1,4]
+  CRUSH rule 5 x 383 [4,8]
+  CRUSH rule 5 x 384 [8,2]
+  CRUSH rule 5 x 385 [8,1]
+  CRUSH rule 5 x 386 [1,8]
+  CRUSH rule 5 x 387 [1,4]
+  CRUSH rule 5 x 388 [2,6]
+  CRUSH rule 5 x 389 [1,4]
+  CRUSH rule 5 x 390 [4,8]
+  CRUSH rule 5 x 391 [4,8]
+  CRUSH rule 5 x 392 [1,8]
+  CRUSH rule 5 x 393 [2,8]
+  CRUSH rule 5 x 394 [8,2]
+  CRUSH rule 5 x 395 [1,8]
+  CRUSH rule 5 x 396 [4,2]
+  CRUSH rule 5 x 397 [2,4]
+  CRUSH rule 5 x 398 [2,4]
+  CRUSH rule 5 x 399 [8,4]
+  CRUSH rule 5 x 400 [8,1]
+  CRUSH rule 5 x 401 [1,4]
+  CRUSH rule 5 x 402 [8,4]
+  CRUSH rule 5 x 403 [1,4]
+  CRUSH rule 5 x 404 [4,2]
+  CRUSH rule 5 x 405 [8,4]
+  CRUSH rule 5 x 406 [2,8]
+  CRUSH rule 5 x 407 [2,8]
+  CRUSH rule 5 x 408 [4,1]
+  CRUSH rule 5 x 409 [8,4]
+  CRUSH rule 5 x 410 [8,4]
+  CRUSH rule 5 x 411 [2,8]
+  CRUSH rule 5 x 412 [2,6]
+  CRUSH rule 5 x 413 [2,8]
+  CRUSH rule 5 x 414 [4,1]
+  CRUSH rule 5 x 415 [2,8]
+  CRUSH rule 5 x 416 [2,8]
+  CRUSH rule 5 x 417 [8,2]
+  CRUSH rule 5 x 418 [8,1]
+  CRUSH rule 5 x 419 [8,4]
+  CRUSH rule 5 x 420 [1,4]
+  CRUSH rule 5 x 421 [8,4]
+  CRUSH rule 5 x 422 [6,2]
+  CRUSH rule 5 x 423 [2,4]
+  CRUSH rule 5 x 424 [8,1]
+  CRUSH rule 5 x 425 [1,8]
+  CRUSH rule 5 x 426 [8,2]
+  CRUSH rule 5 x 427 [1,8]
+  CRUSH rule 5 x 428 [4,8]
+  CRUSH rule 5 x 429 [4,8]
+  CRUSH rule 5 x 430 [4,8]
+  CRUSH rule 5 x 431 [4,1]
+  CRUSH rule 5 x 432 [8,1]
+  CRUSH rule 5 x 433 [8,1]
+  CRUSH rule 5 x 434 [2,8]
+  CRUSH rule 5 x 435 [2,8]
+  CRUSH rule 5 x 436 [4,1]
+  CRUSH rule 5 x 437 [8,2]
+  CRUSH rule 5 x 438 [2,4]
+  CRUSH rule 5 x 439 [1,6]
+  CRUSH rule 5 x 440 [2,8]
+  CRUSH rule 5 x 441 [4,6]
+  CRUSH rule 5 x 442 [2,8]
+  CRUSH rule 5 x 443 [8,2]
+  CRUSH rule 5 x 444 [8,1]
+  CRUSH rule 5 x 445 [8,2]
+  CRUSH rule 5 x 446 [8,1]
+  CRUSH rule 5 x 447 [2,4]
+  CRUSH rule 5 x 448 [8,2]
+  CRUSH rule 5 x 449 [8,1]
+  CRUSH rule 5 x 450 [1,8]
+  CRUSH rule 5 x 451 [8,4]
+  CRUSH rule 5 x 452 [8,2]
+  CRUSH rule 5 x 453 [6,2]
+  CRUSH rule 5 x 454 [8,2]
+  CRUSH rule 5 x 455 [2,8]
+  CRUSH rule 5 x 456 [8,2]
+  CRUSH rule 5 x 457 [8,2]
+  CRUSH rule 5 x 458 [2,8]
+  CRUSH rule 5 x 459 [2,8]
+  CRUSH rule 5 x 460 [8,2]
+  CRUSH rule 5 x 461 [8,1]
+  CRUSH rule 5 x 462 [8,1]
+  CRUSH rule 5 x 463 [8,2]
+  CRUSH rule 5 x 464 [8,4]
+  CRUSH rule 5 x 465 [6,2]
+  CRUSH rule 5 x 466 [8,1]
+  CRUSH rule 5 x 467 [8,2]
+  CRUSH rule 5 x 468 [8,1]
+  CRUSH rule 5 x 469 [8,1]
+  CRUSH rule 5 x 470 [4,2]
+  CRUSH rule 5 x 471 [1,8]
+  CRUSH rule 5 x 472 [1,8]
+  CRUSH rule 5 x 473 [1,4]
+  CRUSH rule 5 x 474 [8,1]
+  CRUSH rule 5 x 475 [8,2]
+  CRUSH rule 5 x 476 [4,8]
+  CRUSH rule 5 x 477 [4,8]
+  CRUSH rule 5 x 478 [8,2]
+  CRUSH rule 5 x 479 [2,8]
+  CRUSH rule 5 x 480 [1,8]
+  CRUSH rule 5 x 481 [2,4]
+  CRUSH rule 5 x 482 [1,8]
+  CRUSH rule 5 x 483 [2,8]
+  CRUSH rule 5 x 484 [1,8]
+  CRUSH rule 5 x 485 [8,1]
+  CRUSH rule 5 x 486 [4,1]
+  CRUSH rule 5 x 487 [1,8]
+  CRUSH rule 5 x 488 [8,1]
+  CRUSH rule 5 x 489 [2,8]
+  CRUSH rule 5 x 490 [6,2]
+  CRUSH rule 5 x 491 [1,8]
+  CRUSH rule 5 x 492 [8,1]
+  CRUSH rule 5 x 493 [2,8]
+  CRUSH rule 5 x 494 [1,8]
+  CRUSH rule 5 x 495 [4,1]
+  CRUSH rule 5 x 496 [8,4]
+  CRUSH rule 5 x 497 [4,8]
+  CRUSH rule 5 x 498 [2,4]
+  CRUSH rule 5 x 499 [8,4]
+  CRUSH rule 5 x 500 [4,8]
+  CRUSH rule 5 x 501 [2,8]
+  CRUSH rule 5 x 502 [6,1]
+  CRUSH rule 5 x 503 [2,8]
+  CRUSH rule 5 x 504 [8,1]
+  CRUSH rule 5 x 505 [1,8]
+  CRUSH rule 5 x 506 [4,2]
+  CRUSH rule 5 x 507 [8,1]
+  CRUSH rule 5 x 508 [1,8]
+  CRUSH rule 5 x 509 [8,1]
+  CRUSH rule 5 x 510 [8,2]
+  CRUSH rule 5 x 511 [4,8]
+  CRUSH rule 5 x 512 [8,2]
+  CRUSH rule 5 x 513 [8,2]
+  CRUSH rule 5 x 514 [2,8]
+  CRUSH rule 5 x 515 [8,4]
+  CRUSH rule 5 x 516 [4,1]
+  CRUSH rule 5 x 517 [8,2]
+  CRUSH rule 5 x 518 [4,8]
+  CRUSH rule 5 x 519 [8,4]
+  CRUSH rule 5 x 520 [2,8]
+  CRUSH rule 5 x 521 [8,2]
+  CRUSH rule 5 x 522 [8,1]
+  CRUSH rule 5 x 523 [4,2]
+  CRUSH rule 5 x 524 [2,6]
+  CRUSH rule 5 x 525 [2,8]
+  CRUSH rule 5 x 526 [1,8]
+  CRUSH rule 5 x 527 [1,4]
+  CRUSH rule 5 x 528 [2,8]
+  CRUSH rule 5 x 529 [4,8]
+  CRUSH rule 5 x 530 [8,1]
+  CRUSH rule 5 x 531 [8,1]
+  CRUSH rule 5 x 532 [6,4]
+  CRUSH rule 5 x 533 [4,8]
+  CRUSH rule 5 x 534 [8,1]
+  CRUSH rule 5 x 535 [8,1]
+  CRUSH rule 5 x 536 [8,2]
+  CRUSH rule 5 x 537 [4,8]
+  CRUSH rule 5 x 538 [8,4]
+  CRUSH rule 5 x 539 [8,1]
+  CRUSH rule 5 x 540 [1,8]
+  CRUSH rule 5 x 541 [2,4]
+  CRUSH rule 5 x 542 [2,8]
+  CRUSH rule 5 x 543 [8,2]
+  CRUSH rule 5 x 544 [4,8]
+  CRUSH rule 5 x 545 [8,1]
+  CRUSH rule 5 x 546 [8,1]
+  CRUSH rule 5 x 547 [8,2]
+  CRUSH rule 5 x 548 [4,2]
+  CRUSH rule 5 x 549 [8,2]
+  CRUSH rule 5 x 550 [2,4]
+  CRUSH rule 5 x 551 [8,1]
+  CRUSH rule 5 x 552 [4,8]
+  CRUSH rule 5 x 553 [2,8]
+  CRUSH rule 5 x 554 [1,8]
+  CRUSH rule 5 x 555 [4,1]
+  CRUSH rule 5 x 556 [8,1]
+  CRUSH rule 5 x 557 [8,2]
+  CRUSH rule 5 x 558 [4,1]
+  CRUSH rule 5 x 559 [1,8]
+  CRUSH rule 5 x 560 [8,1]
+  CRUSH rule 5 x 561 [8,4]
+  CRUSH rule 5 x 562 [4,1]
+  CRUSH rule 5 x 563 [2,8]
+  CRUSH rule 5 x 564 [1,8]
+  CRUSH rule 5 x 565 [4,8]
+  CRUSH rule 5 x 566 [4,8]
+  CRUSH rule 5 x 567 [4,8]
+  CRUSH rule 5 x 568 [8,1]
+  CRUSH rule 5 x 569 [4,1]
+  CRUSH rule 5 x 570 [1,8]
+  CRUSH rule 5 x 571 [6,1]
+  CRUSH rule 5 x 572 [4,2]
+  CRUSH rule 5 x 573 [1,8]
+  CRUSH rule 5 x 574 [2,8]
+  CRUSH rule 5 x 575 [8,2]
+  CRUSH rule 5 x 576 [4,8]
+  CRUSH rule 5 x 577 [8,2]
+  CRUSH rule 5 x 578 [8,1]
+  CRUSH rule 5 x 579 [4,1]
+  CRUSH rule 5 x 580 [1,8]
+  CRUSH rule 5 x 581 [8,2]
+  CRUSH rule 5 x 582 [2,8]
+  CRUSH rule 5 x 583 [8,1]
+  CRUSH rule 5 x 584 [8,1]
+  CRUSH rule 5 x 585 [8,1]
+  CRUSH rule 5 x 586 [1,8]
+  CRUSH rule 5 x 587 [2,4]
+  CRUSH rule 5 x 588 [4,8]
+  CRUSH rule 5 x 589 [8,1]
+  CRUSH rule 5 x 590 [8,2]
+  CRUSH rule 5 x 591 [4,2]
+  CRUSH rule 5 x 592 [2,4]
+  CRUSH rule 5 x 593 [1,8]
+  CRUSH rule 5 x 594 [2,8]
+  CRUSH rule 5 x 595 [8,1]
+  CRUSH rule 5 x 596 [8,2]
+  CRUSH rule 5 x 597 [1,8]
+  CRUSH rule 5 x 598 [2,8]
+  CRUSH rule 5 x 599 [4,2]
+  CRUSH rule 5 x 600 [8,1]
+  CRUSH rule 5 x 601 [1,8]
+  CRUSH rule 5 x 602 [8,2]
+  CRUSH rule 5 x 603 [1,8]
+  CRUSH rule 5 x 604 [8,2]
+  CRUSH rule 5 x 605 [2,8]
+  CRUSH rule 5 x 606 [2,6]
+  CRUSH rule 5 x 607 [2,4]
+  CRUSH rule 5 x 608 [4,2]
+  CRUSH rule 5 x 609 [4,2]
+  CRUSH rule 5 x 610 [8,1]
+  CRUSH rule 5 x 611 [1,8]
+  CRUSH rule 5 x 612 [2,8]
+  CRUSH rule 5 x 613 [8,2]
+  CRUSH rule 5 x 614 [8,2]
+  CRUSH rule 5 x 615 [8,2]
+  CRUSH rule 5 x 616 [1,8]
+  CRUSH rule 5 x 617 [8,1]
+  CRUSH rule 5 x 618 [8,4]
+  CRUSH rule 5 x 619 [4,1]
+  CRUSH rule 5 x 620 [1,8]
+  CRUSH rule 5 x 621 [8,1]
+  CRUSH rule 5 x 622 [2,4]
+  CRUSH rule 5 x 623 [2,8]
+  CRUSH rule 5 x 624 [4,2]
+  CRUSH rule 5 x 625 [2,8]
+  CRUSH rule 5 x 626 [8,2]
+  CRUSH rule 5 x 627 [2,8]
+  CRUSH rule 5 x 628 [8,2]
+  CRUSH rule 5 x 629 [2,8]
+  CRUSH rule 5 x 630 [2,8]
+  CRUSH rule 5 x 631 [1,8]
+  CRUSH rule 5 x 632 [8,2]
+  CRUSH rule 5 x 633 [8,2]
+  CRUSH rule 5 x 634 [1,8]
+  CRUSH rule 5 x 635 [4,8]
+  CRUSH rule 5 x 636 [1,4]
+  CRUSH rule 5 x 637 [1,8]
+  CRUSH rule 5 x 638 [8,1]
+  CRUSH rule 5 x 639 [2,8]
+  CRUSH rule 5 x 640 [2,8]
+  CRUSH rule 5 x 641 [8,2]
+  CRUSH rule 5 x 642 [2,8]
+  CRUSH rule 5 x 643 [1,8]
+  CRUSH rule 5 x 644 [8,1]
+  CRUSH rule 5 x 645 [8,1]
+  CRUSH rule 5 x 646 [8,1]
+  CRUSH rule 5 x 647 [8,1]
+  CRUSH rule 5 x 648 [1,8]
+  CRUSH rule 5 x 649 [4,8]
+  CRUSH rule 5 x 650 [8,4]
+  CRUSH rule 5 x 651 [4,6]
+  CRUSH rule 5 x 652 [4,8]
+  CRUSH rule 5 x 653 [8,2]
+  CRUSH rule 5 x 654 [6,2]
+  CRUSH rule 5 x 655 [1,4]
+  CRUSH rule 5 x 656 [8,1]
+  CRUSH rule 5 x 657 [6,1]
+  CRUSH rule 5 x 658 [8,2]
+  CRUSH rule 5 x 659 [4,8]
+  CRUSH rule 5 x 660 [8,2]
+  CRUSH rule 5 x 661 [1,8]
+  CRUSH rule 5 x 662 [8,2]
+  CRUSH rule 5 x 663 [1,4]
+  CRUSH rule 5 x 664 [1,4]
+  CRUSH rule 5 x 665 [4,6]
+  CRUSH rule 5 x 666 [2,8]
+  CRUSH rule 5 x 667 [1,4]
+  CRUSH rule 5 x 668 [4,8]
+  CRUSH rule 5 x 669 [6,4]
+  CRUSH rule 5 x 670 [4,2]
+  CRUSH rule 5 x 671 [2,8]
+  CRUSH rule 5 x 672 [4,2]
+  CRUSH rule 5 x 673 [4,2]
+  CRUSH rule 5 x 674 [1,8]
+  CRUSH rule 5 x 675 [1,8]
+  CRUSH rule 5 x 676 [2,4]
+  CRUSH rule 5 x 677 [4,1]
+  CRUSH rule 5 x 678 [2,4]
+  CRUSH rule 5 x 679 [8,2]
+  CRUSH rule 5 x 680 [2,8]
+  CRUSH rule 5 x 681 [8,1]
+  CRUSH rule 5 x 682 [1,4]
+  CRUSH rule 5 x 683 [1,4]
+  CRUSH rule 5 x 684 [8,1]
+  CRUSH rule 5 x 685 [8,1]
+  CRUSH rule 5 x 686 [1,4]
+  CRUSH rule 5 x 687 [6,1]
+  CRUSH rule 5 x 688 [4,8]
+  CRUSH rule 5 x 689 [8,4]
+  CRUSH rule 5 x 690 [8,1]
+  CRUSH rule 5 x 691 [1,8]
+  CRUSH rule 5 x 692 [8,2]
+  CRUSH rule 5 x 693 [8,4]
+  CRUSH rule 5 x 694 [8,4]
+  CRUSH rule 5 x 695 [2,8]
+  CRUSH rule 5 x 696 [1,8]
+  CRUSH rule 5 x 697 [8,1]
+  CRUSH rule 5 x 698 [8,2]
+  CRUSH rule 5 x 699 [1,8]
+  CRUSH rule 5 x 700 [1,8]
+  CRUSH rule 5 x 701 [1,8]
+  CRUSH rule 5 x 702 [2,8]
+  CRUSH rule 5 x 703 [8,1]
+  CRUSH rule 5 x 704 [1,4]
+  CRUSH rule 5 x 705 [8,1]
+  CRUSH rule 5 x 706 [1,4]
+  CRUSH rule 5 x 707 [8,4]
+  CRUSH rule 5 x 708 [4,8]
+  CRUSH rule 5 x 709 [8,2]
+  CRUSH rule 5 x 710 [8,2]
+  CRUSH rule 5 x 711 [2,4]
+  CRUSH rule 5 x 712 [2,8]
+  CRUSH rule 5 x 713 [8,4]
+  CRUSH rule 5 x 714 [2,8]
+  CRUSH rule 5 x 715 [1,8]
+  CRUSH rule 5 x 716 [4,8]
+  CRUSH rule 5 x 717 [8,2]
+  CRUSH rule 5 x 718 [8,1]
+  CRUSH rule 5 x 719 [2,6]
+  CRUSH rule 5 x 720 [8,1]
+  CRUSH rule 5 x 721 [4,6]
+  CRUSH rule 5 x 722 [8,2]
+  CRUSH rule 5 x 723 [4,1]
+  CRUSH rule 5 x 724 [2,6]
+  CRUSH rule 5 x 725 [1,8]
+  CRUSH rule 5 x 726 [4,8]
+  CRUSH rule 5 x 727 [4,8]
+  CRUSH rule 5 x 728 [2,8]
+  CRUSH rule 5 x 729 [8,2]
+  CRUSH rule 5 x 730 [4,8]
+  CRUSH rule 5 x 731 [4,1]
+  CRUSH rule 5 x 732 [1,8]
+  CRUSH rule 5 x 733 [4,8]
+  CRUSH rule 5 x 734 [8,4]
+  CRUSH rule 5 x 735 [4,8]
+  CRUSH rule 5 x 736 [4,8]
+  CRUSH rule 5 x 737 [1,8]
+  CRUSH rule 5 x 738 [4,2]
+  CRUSH rule 5 x 739 [2,8]
+  CRUSH rule 5 x 740 [1,8]
+  CRUSH rule 5 x 741 [8,1]
+  CRUSH rule 5 x 742 [8,2]
+  CRUSH rule 5 x 743 [8,1]
+  CRUSH rule 5 x 744 [4,8]
+  CRUSH rule 5 x 745 [1,8]
+  CRUSH rule 5 x 746 [1,8]
+  CRUSH rule 5 x 747 [8,1]
+  CRUSH rule 5 x 748 [2,8]
+  CRUSH rule 5 x 749 [4,8]
+  CRUSH rule 5 x 750 [1,8]
+  CRUSH rule 5 x 751 [2,8]
+  CRUSH rule 5 x 752 [8,1]
+  CRUSH rule 5 x 753 [8,4]
+  CRUSH rule 5 x 754 [8,4]
+  CRUSH rule 5 x 755 [1,8]
+  CRUSH rule 5 x 756 [8,1]
+  CRUSH rule 5 x 757 [8,1]
+  CRUSH rule 5 x 758 [8,2]
+  CRUSH rule 5 x 759 [8,4]
+  CRUSH rule 5 x 760 [1,4]
+  CRUSH rule 5 x 761 [2,8]
+  CRUSH rule 5 x 762 [2,8]
+  CRUSH rule 5 x 763 [8,4]
+  CRUSH rule 5 x 764 [1,8]
+  CRUSH rule 5 x 765 [8,2]
+  CRUSH rule 5 x 766 [8,1]
+  CRUSH rule 5 x 767 [1,8]
+  CRUSH rule 5 x 768 [8,4]
+  CRUSH rule 5 x 769 [8,2]
+  CRUSH rule 5 x 770 [8,2]
+  CRUSH rule 5 x 771 [8,1]
+  CRUSH rule 5 x 772 [8,4]
+  CRUSH rule 5 x 773 [4,1]
+  CRUSH rule 5 x 774 [8,1]
+  CRUSH rule 5 x 775 [8,4]
+  CRUSH rule 5 x 776 [6,2]
+  CRUSH rule 5 x 777 [4,1]
+  CRUSH rule 5 x 778 [1,8]
+  CRUSH rule 5 x 779 [2,8]
+  CRUSH rule 5 x 780 [2,4]
+  CRUSH rule 5 x 781 [8,2]
+  CRUSH rule 5 x 782 [4,1]
+  CRUSH rule 5 x 783 [8,1]
+  CRUSH rule 5 x 784 [1,4]
+  CRUSH rule 5 x 785 [8,1]
+  CRUSH rule 5 x 786 [8,1]
+  CRUSH rule 5 x 787 [1,6]
+  CRUSH rule 5 x 788 [8,2]
+  CRUSH rule 5 x 789 [1,8]
+  CRUSH rule 5 x 790 [8,1]
+  CRUSH rule 5 x 791 [4,8]
+  CRUSH rule 5 x 792 [4,8]
+  CRUSH rule 5 x 793 [8,1]
+  CRUSH rule 5 x 794 [2,8]
+  CRUSH rule 5 x 795 [1,8]
+  CRUSH rule 5 x 796 [8,2]
+  CRUSH rule 5 x 797 [2,4]
+  CRUSH rule 5 x 798 [6,1]
+  CRUSH rule 5 x 799 [4,1]
+  CRUSH rule 5 x 800 [2,8]
+  CRUSH rule 5 x 801 [4,8]
+  CRUSH rule 5 x 802 [1,8]
+  CRUSH rule 5 x 803 [2,8]
+  CRUSH rule 5 x 804 [8,2]
+  CRUSH rule 5 x 805 [8,2]
+  CRUSH rule 5 x 806 [1,4]
+  CRUSH rule 5 x 807 [4,8]
+  CRUSH rule 5 x 808 [8,2]
+  CRUSH rule 5 x 809 [1,8]
+  CRUSH rule 5 x 810 [8,2]
+  CRUSH rule 5 x 811 [8,1]
+  CRUSH rule 5 x 812 [8,4]
+  CRUSH rule 5 x 813 [8,4]
+  CRUSH rule 5 x 814 [8,2]
+  CRUSH rule 5 x 815 [4,1]
+  CRUSH rule 5 x 816 [2,8]
+  CRUSH rule 5 x 817 [8,1]
+  CRUSH rule 5 x 818 [1,8]
+  CRUSH rule 5 x 819 [1,8]
+  CRUSH rule 5 x 820 [4,8]
+  CRUSH rule 5 x 821 [4,8]
+  CRUSH rule 5 x 822 [2,4]
+  CRUSH rule 5 x 823 [4,8]
+  CRUSH rule 5 x 824 [8,2]
+  CRUSH rule 5 x 825 [2,8]
+  CRUSH rule 5 x 826 [8,2]
+  CRUSH rule 5 x 827 [2,8]
+  CRUSH rule 5 x 828 [2,8]
+  CRUSH rule 5 x 829 [8,1]
+  CRUSH rule 5 x 830 [2,4]
+  CRUSH rule 5 x 831 [1,8]
+  CRUSH rule 5 x 832 [4,8]
+  CRUSH rule 5 x 833 [2,8]
+  CRUSH rule 5 x 834 [1,8]
+  CRUSH rule 5 x 835 [8,4]
+  CRUSH rule 5 x 836 [4,8]
+  CRUSH rule 5 x 837 [8,4]
+  CRUSH rule 5 x 838 [6,2]
+  CRUSH rule 5 x 839 [2,8]
+  CRUSH rule 5 x 840 [8,1]
+  CRUSH rule 5 x 841 [4,8]
+  CRUSH rule 5 x 842 [2,8]
+  CRUSH rule 5 x 843 [8,4]
+  CRUSH rule 5 x 844 [8,2]
+  CRUSH rule 5 x 845 [4,8]
+  CRUSH rule 5 x 846 [4,2]
+  CRUSH rule 5 x 847 [2,8]
+  CRUSH rule 5 x 848 [2,8]
+  CRUSH rule 5 x 849 [4,8]
+  CRUSH rule 5 x 850 [1,6]
+  CRUSH rule 5 x 851 [6,1]
+  CRUSH rule 5 x 852 [8,4]
+  CRUSH rule 5 x 853 [6,1]
+  CRUSH rule 5 x 854 [8,1]
+  CRUSH rule 5 x 855 [8,1]
+  CRUSH rule 5 x 856 [8,4]
+  CRUSH rule 5 x 857 [8,2]
+  CRUSH rule 5 x 858 [6,1]
+  CRUSH rule 5 x 859 [8,2]
+  CRUSH rule 5 x 860 [2,8]
+  CRUSH rule 5 x 861 [8,2]
+  CRUSH rule 5 x 862 [8,1]
+  CRUSH rule 5 x 863 [8,2]
+  CRUSH rule 5 x 864 [8,2]
+  CRUSH rule 5 x 865 [8,1]
+  CRUSH rule 5 x 866 [8,2]
+  CRUSH rule 5 x 867 [8,2]
+  CRUSH rule 5 x 868 [8,1]
+  CRUSH rule 5 x 869 [8,4]
+  CRUSH rule 5 x 870 [2,8]
+  CRUSH rule 5 x 871 [1,8]
+  CRUSH rule 5 x 872 [1,8]
+  CRUSH rule 5 x 873 [4,8]
+  CRUSH rule 5 x 874 [2,6]
+  CRUSH rule 5 x 875 [2,8]
+  CRUSH rule 5 x 876 [4,8]
+  CRUSH rule 5 x 877 [8,4]
+  CRUSH rule 5 x 878 [2,8]
+  CRUSH rule 5 x 879 [8,1]
+  CRUSH rule 5 x 880 [1,8]
+  CRUSH rule 5 x 881 [4,8]
+  CRUSH rule 5 x 882 [1,8]
+  CRUSH rule 5 x 883 [2,4]
+  CRUSH rule 5 x 884 [8,2]
+  CRUSH rule 5 x 885 [4,1]
+  CRUSH rule 5 x 886 [8,2]
+  CRUSH rule 5 x 887 [8,4]
+  CRUSH rule 5 x 888 [8,2]
+  CRUSH rule 5 x 889 [2,6]
+  CRUSH rule 5 x 890 [8,2]
+  CRUSH rule 5 x 891 [1,8]
+  CRUSH rule 5 x 892 [8,2]
+  CRUSH rule 5 x 893 [2,6]
+  CRUSH rule 5 x 894 [8,4]
+  CRUSH rule 5 x 895 [4,1]
+  CRUSH rule 5 x 896 [1,8]
+  CRUSH rule 5 x 897 [2,8]
+  CRUSH rule 5 x 898 [1,4]
+  CRUSH rule 5 x 899 [1,8]
+  CRUSH rule 5 x 900 [4,1]
+  CRUSH rule 5 x 901 [2,8]
+  CRUSH rule 5 x 902 [8,4]
+  CRUSH rule 5 x 903 [8,2]
+  CRUSH rule 5 x 904 [8,2]
+  CRUSH rule 5 x 905 [8,2]
+  CRUSH rule 5 x 906 [1,8]
+  CRUSH rule 5 x 907 [8,1]
+  CRUSH rule 5 x 908 [8,1]
+  CRUSH rule 5 x 909 [2,8]
+  CRUSH rule 5 x 910 [8,2]
+  CRUSH rule 5 x 911 [8,1]
+  CRUSH rule 5 x 912 [1,8]
+  CRUSH rule 5 x 913 [8,2]
+  CRUSH rule 5 x 914 [6,4]
+  CRUSH rule 5 x 915 [8,2]
+  CRUSH rule 5 x 916 [4,1]
+  CRUSH rule 5 x 917 [1,4]
+  CRUSH rule 5 x 918 [8,2]
+  CRUSH rule 5 x 919 [8,2]
+  CRUSH rule 5 x 920 [8,1]
+  CRUSH rule 5 x 921 [1,8]
+  CRUSH rule 5 x 922 [8,4]
+  CRUSH rule 5 x 923 [4,8]
+  CRUSH rule 5 x 924 [8,1]
+  CRUSH rule 5 x 925 [4,8]
+  CRUSH rule 5 x 926 [2,8]
+  CRUSH rule 5 x 927 [1,8]
+  CRUSH rule 5 x 928 [8,1]
+  CRUSH rule 5 x 929 [4,1]
+  CRUSH rule 5 x 930 [2,8]
+  CRUSH rule 5 x 931 [2,8]
+  CRUSH rule 5 x 932 [4,1]
+  CRUSH rule 5 x 933 [8,4]
+  CRUSH rule 5 x 934 [8,2]
+  CRUSH rule 5 x 935 [8,2]
+  CRUSH rule 5 x 936 [1,8]
+  CRUSH rule 5 x 937 [4,8]
+  CRUSH rule 5 x 938 [8,4]
+  CRUSH rule 5 x 939 [2,8]
+  CRUSH rule 5 x 940 [8,1]
+  CRUSH rule 5 x 941 [2,8]
+  CRUSH rule 5 x 942 [1,8]
+  CRUSH rule 5 x 943 [8,2]
+  CRUSH rule 5 x 944 [8,2]
+  CRUSH rule 5 x 945 [8,2]
+  CRUSH rule 5 x 946 [2,8]
+  CRUSH rule 5 x 947 [8,2]
+  CRUSH rule 5 x 948 [8,1]
+  CRUSH rule 5 x 949 [6,1]
+  CRUSH rule 5 x 950 [8,1]
+  CRUSH rule 5 x 951 [8,1]
+  CRUSH rule 5 x 952 [2,8]
+  CRUSH rule 5 x 953 [1,4]
+  CRUSH rule 5 x 954 [2,8]
+  CRUSH rule 5 x 955 [8,1]
+  CRUSH rule 5 x 956 [1,8]
+  CRUSH rule 5 x 957 [8,1]
+  CRUSH rule 5 x 958 [8,4]
+  CRUSH rule 5 x 959 [4,2]
+  CRUSH rule 5 x 960 [6,1]
+  CRUSH rule 5 x 961 [1,8]
+  CRUSH rule 5 x 962 [8,4]
+  CRUSH rule 5 x 963 [2,4]
+  CRUSH rule 5 x 964 [2,8]
+  CRUSH rule 5 x 965 [8,2]
+  CRUSH rule 5 x 966 [4,8]
+  CRUSH rule 5 x 967 [8,4]
+  CRUSH rule 5 x 968 [8,2]
+  CRUSH rule 5 x 969 [8,2]
+  CRUSH rule 5 x 970 [2,8]
+  CRUSH rule 5 x 971 [1,8]
+  CRUSH rule 5 x 972 [1,8]
+  CRUSH rule 5 x 973 [1,8]
+  CRUSH rule 5 x 974 [4,1]
+  CRUSH rule 5 x 975 [4,8]
+  CRUSH rule 5 x 976 [4,8]
+  CRUSH rule 5 x 977 [8,4]
+  CRUSH rule 5 x 978 [8,2]
+  CRUSH rule 5 x 979 [8,1]
+  CRUSH rule 5 x 980 [8,2]
+  CRUSH rule 5 x 981 [8,2]
+  CRUSH rule 5 x 982 [1,8]
+  CRUSH rule 5 x 983 [4,8]
+  CRUSH rule 5 x 984 [2,8]
+  CRUSH rule 5 x 985 [2,4]
+  CRUSH rule 5 x 986 [8,4]
+  CRUSH rule 5 x 987 [2,8]
+  CRUSH rule 5 x 988 [1,4]
+  CRUSH rule 5 x 989 [1,8]
+  CRUSH rule 5 x 990 [1,8]
+  CRUSH rule 5 x 991 [1,4]
+  CRUSH rule 5 x 992 [8,1]
+  CRUSH rule 5 x 993 [2,8]
+  CRUSH rule 5 x 994 [4,8]
+  CRUSH rule 5 x 995 [8,1]
+  CRUSH rule 5 x 996 [8,4]
+  CRUSH rule 5 x 997 [8,4]
+  CRUSH rule 5 x 998 [8,1]
+  CRUSH rule 5 x 999 [1,8]
+  CRUSH rule 5 x 1000 [8,4]
+  CRUSH rule 5 x 1001 [2,8]
+  CRUSH rule 5 x 1002 [1,8]
+  CRUSH rule 5 x 1003 [2,8]
+  CRUSH rule 5 x 1004 [8,1]
+  CRUSH rule 5 x 1005 [8,1]
+  CRUSH rule 5 x 1006 [1,8]
+  CRUSH rule 5 x 1007 [1,4]
+  CRUSH rule 5 x 1008 [1,8]
+  CRUSH rule 5 x 1009 [6,4]
+  CRUSH rule 5 x 1010 [1,8]
+  CRUSH rule 5 x 1011 [4,2]
+  CRUSH rule 5 x 1012 [1,8]
+  CRUSH rule 5 x 1013 [2,8]
+  CRUSH rule 5 x 1014 [2,8]
+  CRUSH rule 5 x 1015 [8,1]
+  CRUSH rule 5 x 1016 [2,4]
+  CRUSH rule 5 x 1017 [6,2]
+  CRUSH rule 5 x 1018 [4,1]
+  CRUSH rule 5 x 1019 [4,8]
+  CRUSH rule 5 x 1020 [1,8]
+  CRUSH rule 5 x 1021 [2,8]
+  CRUSH rule 5 x 1022 [1,8]
+  CRUSH rule 5 x 1023 [4,2]
+  rule 5 (chooseleaf-set) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 5 x 0 [2,4,8]
+  CRUSH rule 5 x 1 [2,8,4]
+  CRUSH rule 5 x 2 [1,8]
+  CRUSH rule 5 x 3 [8,1]
+  CRUSH rule 5 x 4 [4,2,6]
+  CRUSH rule 5 x 5 [8,2]
+  CRUSH rule 5 x 6 [2,8,4]
+  CRUSH rule 5 x 7 [4,8,2]
+  CRUSH rule 5 x 8 [4,8,1]
+  CRUSH rule 5 x 9 [2,4,8]
+  CRUSH rule 5 x 10 [2,8]
+  CRUSH rule 5 x 11 [2,8]
+  CRUSH rule 5 x 12 [2,8]
+  CRUSH rule 5 x 13 [4,8,1]
+  CRUSH rule 5 x 14 [8,2]
+  CRUSH rule 5 x 15 [8,2]
+  CRUSH rule 5 x 16 [8,2]
+  CRUSH rule 5 x 17 [4,1,8]
+  CRUSH rule 5 x 18 [1,8]
+  CRUSH rule 5 x 19 [8,4,2]
+  CRUSH rule 5 x 20 [2,8]
+  CRUSH rule 5 x 21 [8,2]
+  CRUSH rule 5 x 22 [8,1]
+  CRUSH rule 5 x 23 [4,8,2]
+  CRUSH rule 5 x 24 [1,8,4]
+  CRUSH rule 5 x 25 [4,8,1]
+  CRUSH rule 5 x 26 [2,8,4]
+  CRUSH rule 5 x 27 [4,1,8]
+  CRUSH rule 5 x 28 [8,2]
+  CRUSH rule 5 x 29 [8,4,2]
+  CRUSH rule 5 x 30 [4,8,2]
+  CRUSH rule 5 x 31 [8,1]
+  CRUSH rule 5 x 32 [6,1]
+  CRUSH rule 5 x 33 [2,8]
+  CRUSH rule 5 x 34 [2,8]
+  CRUSH rule 5 x 35 [1,8,4]
+  CRUSH rule 5 x 36 [8,2]
+  CRUSH rule 5 x 37 [1,8]
+  CRUSH rule 5 x 38 [4,8,2]
+  CRUSH rule 5 x 39 [8,2]
+  CRUSH rule 5 x 40 [8,2,4]
+  CRUSH rule 5 x 41 [2,8,4]
+  CRUSH rule 5 x 42 [8,2]
+  CRUSH rule 5 x 43 [1,8]
+  CRUSH rule 5 x 44 [1,8,4]
+  CRUSH rule 5 x 45 [8,2,4]
+  CRUSH rule 5 x 46 [2,8]
+  CRUSH rule 5 x 47 [4,2,8]
+  CRUSH rule 5 x 48 [8,1]
+  CRUSH rule 5 x 49 [8,2]
+  CRUSH rule 5 x 50 [4,1,8]
+  CRUSH rule 5 x 51 [8,2]
+  CRUSH rule 5 x 52 [8,1,4]
+  CRUSH rule 5 x 53 [4,8,2]
+  CRUSH rule 5 x 54 [8,4,1]
+  CRUSH rule 5 x 55 [8,2]
+  CRUSH rule 5 x 56 [8,4,2]
+  CRUSH rule 5 x 57 [8,1]
+  CRUSH rule 5 x 58 [1,8]
+  CRUSH rule 5 x 59 [2,8]
+  CRUSH rule 5 x 60 [4,2,8]
+  CRUSH rule 5 x 61 [4,8,2]
+  CRUSH rule 5 x 62 [8,1]
+  CRUSH rule 5 x 63 [8,2]
+  CRUSH rule 5 x 64 [4,2,8]
+  CRUSH rule 5 x 65 [8,4,2]
+  CRUSH rule 5 x 66 [4,8,2]
+  CRUSH rule 5 x 67 [4,2,8]
+  CRUSH rule 5 x 68 [1,8]
+  CRUSH rule 5 x 69 [2,8]
+  CRUSH rule 5 x 70 [8,2]
+  CRUSH rule 5 x 71 [2,8,4]
+  CRUSH rule 5 x 72 [8,1,4]
+  CRUSH rule 5 x 73 [2,8]
+  CRUSH rule 5 x 74 [1,8]
+  CRUSH rule 5 x 75 [4,2,8]
+  CRUSH rule 5 x 76 [4,1,6]
+  CRUSH rule 5 x 77 [8,2,4]
+  CRUSH rule 5 x 78 [1,6]
+  CRUSH rule 5 x 79 [4,1,8]
+  CRUSH rule 5 x 80 [2,4,8]
+  CRUSH rule 5 x 81 [2,8]
+  CRUSH rule 5 x 82 [6,1]
+  CRUSH rule 5 x 83 [2,8]
+  CRUSH rule 5 x 84 [8,2]
+  CRUSH rule 5 x 85 [4,8,1]
+  CRUSH rule 5 x 86 [2,8]
+  CRUSH rule 5 x 87 [2,8,4]
+  CRUSH rule 5 x 88 [1,6]
+  CRUSH rule 5 x 89 [1,8]
+  CRUSH rule 5 x 90 [8,4,1]
+  CRUSH rule 5 x 91 [4,8,1]
+  CRUSH rule 5 x 92 [1,8]
+  CRUSH rule 5 x 93 [8,4,2]
+  CRUSH rule 5 x 94 [1,8]
+  CRUSH rule 5 x 95 [8,1]
+  CRUSH rule 5 x 96 [8,2]
+  CRUSH rule 5 x 97 [8,1]
+  CRUSH rule 5 x 98 [2,8]
+  CRUSH rule 5 x 99 [2,8]
+  CRUSH rule 5 x 100 [1,8,4]
+  CRUSH rule 5 x 101 [8,1]
+  CRUSH rule 5 x 102 [2,8]
+  CRUSH rule 5 x 103 [8,2]
+  CRUSH rule 5 x 104 [8,4,1]
+  CRUSH rule 5 x 105 [2,4,8]
+  CRUSH rule 5 x 106 [1,8,4]
+  CRUSH rule 5 x 107 [1,8]
+  CRUSH rule 5 x 108 [8,2]
+  CRUSH rule 5 x 109 [1,4,8]
+  CRUSH rule 5 x 110 [4,2,8]
+  CRUSH rule 5 x 111 [2,4,8]
+  CRUSH rule 5 x 112 [2,8]
+  CRUSH rule 5 x 113 [8,2]
+  CRUSH rule 5 x 114 [8,4,1]
+  CRUSH rule 5 x 115 [8,2,4]
+  CRUSH rule 5 x 116 [1,8]
+  CRUSH rule 5 x 117 [6,1]
+  CRUSH rule 5 x 118 [2,8]
+  CRUSH rule 5 x 119 [8,1]
+  CRUSH rule 5 x 120 [2,4,8]
+  CRUSH rule 5 x 121 [2,8,4]
+  CRUSH rule 5 x 122 [8,1]
+  CRUSH rule 5 x 123 [2,8]
+  CRUSH rule 5 x 124 [2,8]
+  CRUSH rule 5 x 125 [1,8,4]
+  CRUSH rule 5 x 126 [1,8]
+  CRUSH rule 5 x 127 [4,8,2]
+  CRUSH rule 5 x 128 [8,2]
+  CRUSH rule 5 x 129 [2,4,8]
+  CRUSH rule 5 x 130 [4,8,2]
+  CRUSH rule 5 x 131 [1,4,8]
+  CRUSH rule 5 x 132 [1,8]
+  CRUSH rule 5 x 133 [8,1]
+  CRUSH rule 5 x 134 [1,8,4]
+  CRUSH rule 5 x 135 [4,8,2]
+  CRUSH rule 5 x 136 [2,4,8]
+  CRUSH rule 5 x 137 [8,4,2]
+  CRUSH rule 5 x 138 [8,4,2]
+  CRUSH rule 5 x 139 [4,2,8]
+  CRUSH rule 5 x 140 [1,8,4]
+  CRUSH rule 5 x 141 [8,2]
+  CRUSH rule 5 x 142 [4,1,8]
+  CRUSH rule 5 x 143 [4,8,1]
+  CRUSH rule 5 x 144 [8,1]
+  CRUSH rule 5 x 145 [8,1]
+  CRUSH rule 5 x 146 [2,8]
+  CRUSH rule 5 x 147 [2,8,4]
+  CRUSH rule 5 x 148 [4,1,8]
+  CRUSH rule 5 x 149 [4,8,1]
+  CRUSH rule 5 x 150 [1,8]
+  CRUSH rule 5 x 151 [1,8]
+  CRUSH rule 5 x 152 [8,2]
+  CRUSH rule 5 x 153 [8,4,2]
+  CRUSH rule 5 x 154 [4,2,8]
+  CRUSH rule 5 x 155 [4,8,2]
+  CRUSH rule 5 x 156 [4,2,8]
+  CRUSH rule 5 x 157 [1,8]
+  CRUSH rule 5 x 158 [2,8,4]
+  CRUSH rule 5 x 159 [8,2,4]
+  CRUSH rule 5 x 160 [2,8,4]
+  CRUSH rule 5 x 161 [1,4,8]
+  CRUSH rule 5 x 162 [1,8]
+  CRUSH rule 5 x 163 [4,8,1]
+  CRUSH rule 5 x 164 [8,1]
+  CRUSH rule 5 x 165 [8,2,4]
+  CRUSH rule 5 x 166 [2,8]
+  CRUSH rule 5 x 167 [1,8,4]
+  CRUSH rule 5 x 168 [4,2,8]
+  CRUSH rule 5 x 169 [2,8,4]
+  CRUSH rule 5 x 170 [1,8]
+  CRUSH rule 5 x 171 [8,4,2]
+  CRUSH rule 5 x 172 [1,8]
+  CRUSH rule 5 x 173 [8,4,1]
+  CRUSH rule 5 x 174 [1,6]
+  CRUSH rule 5 x 175 [8,1]
+  CRUSH rule 5 x 176 [2,8]
+  CRUSH rule 5 x 177 [1,8]
+  CRUSH rule 5 x 178 [4,2,8]
+  CRUSH rule 5 x 179 [1,8]
+  CRUSH rule 5 x 180 [8,1]
+  CRUSH rule 5 x 181 [8,2,4]
+  CRUSH rule 5 x 182 [8,1]
+  CRUSH rule 5 x 183 [8,4,1]
+  CRUSH rule 5 x 184 [4,8,2]
+  CRUSH rule 5 x 185 [8,1,4]
+  CRUSH rule 5 x 186 [2,4,8]
+  CRUSH rule 5 x 187 [1,8]
+  CRUSH rule 5 x 188 [1,8,4]
+  CRUSH rule 5 x 189 [1,8,4]
+  CRUSH rule 5 x 190 [1,8]
+  CRUSH rule 5 x 191 [8,1,4]
+  CRUSH rule 5 x 192 [4,1,8]
+  CRUSH rule 5 x 193 [4,2,8]
+  CRUSH rule 5 x 194 [1,8]
+  CRUSH rule 5 x 195 [8,4,1]
+  CRUSH rule 5 x 196 [8,2]
+  CRUSH rule 5 x 197 [8,4,2]
+  CRUSH rule 5 x 198 [2,8]
+  CRUSH rule 5 x 199 [1,4,8]
+  CRUSH rule 5 x 200 [1,8]
+  CRUSH rule 5 x 201 [8,1,4]
+  CRUSH rule 5 x 202 [8,1]
+  CRUSH rule 5 x 203 [8,1]
+  CRUSH rule 5 x 204 [2,4,8]
+  CRUSH rule 5 x 205 [1,8]
+  CRUSH rule 5 x 206 [1,8,4]
+  CRUSH rule 5 x 207 [2,8]
+  CRUSH rule 5 x 208 [8,1]
+  CRUSH rule 5 x 209 [1,8]
+  CRUSH rule 5 x 210 [1,4,8]
+  CRUSH rule 5 x 211 [4,2,8]
+  CRUSH rule 5 x 212 [8,1]
+  CRUSH rule 5 x 213 [8,4,2]
+  CRUSH rule 5 x 214 [8,2]
+  CRUSH rule 5 x 215 [8,1]
+  CRUSH rule 5 x 216 [2,8]
+  CRUSH rule 5 x 217 [1,8,4]
+  CRUSH rule 5 x 218 [2,8]
+  CRUSH rule 5 x 219 [8,2]
+  CRUSH rule 5 x 220 [4,8,1]
+  CRUSH rule 5 x 221 [8,1]
+  CRUSH rule 5 x 222 [8,1]
+  CRUSH rule 5 x 223 [1,8]
+  CRUSH rule 5 x 224 [1,4,8]
+  CRUSH rule 5 x 225 [8,2]
+  CRUSH rule 5 x 226 [8,2,4]
+  CRUSH rule 5 x 227 [4,1,8]
+  CRUSH rule 5 x 228 [8,2]
+  CRUSH rule 5 x 229 [4,8,2]
+  CRUSH rule 5 x 230 [4,8,2]
+  CRUSH rule 5 x 231 [4,8,2]
+  CRUSH rule 5 x 232 [2,8,4]
+  CRUSH rule 5 x 233 [2,8]
+  CRUSH rule 5 x 234 [1,8]
+  CRUSH rule 5 x 235 [4,8,1]
+  CRUSH rule 5 x 236 [2,6]
+  CRUSH rule 5 x 237 [4,8,1]
+  CRUSH rule 5 x 238 [2,8]
+  CRUSH rule 5 x 239 [8,1]
+  CRUSH rule 5 x 240 [4,8,2]
+  CRUSH rule 5 x 241 [1,8]
+  CRUSH rule 5 x 242 [8,2]
+  CRUSH rule 5 x 243 [8,2]
+  CRUSH rule 5 x 244 [4,8,2]
+  CRUSH rule 5 x 245 [8,1]
+  CRUSH rule 5 x 246 [1,8]
+  CRUSH rule 5 x 247 [8,2]
+  CRUSH rule 5 x 248 [8,2,4]
+  CRUSH rule 5 x 249 [2,8]
+  CRUSH rule 5 x 250 [2,4,6]
+  CRUSH rule 5 x 251 [2,8]
+  CRUSH rule 5 x 252 [4,8,1]
+  CRUSH rule 5 x 253 [2,8]
+  CRUSH rule 5 x 254 [4,2,8]
+  CRUSH rule 5 x 255 [1,8]
+  CRUSH rule 5 x 256 [4,8,1]
+  CRUSH rule 5 x 257 [2,8,4]
+  CRUSH rule 5 x 258 [4,2,8]
+  CRUSH rule 5 x 259 [6,2]
+  CRUSH rule 5 x 260 [8,2]
+  CRUSH rule 5 x 261 [8,1]
+  CRUSH rule 5 x 262 [8,1]
+  CRUSH rule 5 x 263 [8,1,4]
+  CRUSH rule 5 x 264 [8,2]
+  CRUSH rule 5 x 265 [8,2]
+  CRUSH rule 5 x 266 [8,2,4]
+  CRUSH rule 5 x 267 [2,8]
+  CRUSH rule 5 x 268 [1,8]
+  CRUSH rule 5 x 269 [1,8,4]
+  CRUSH rule 5 x 270 [4,1,8]
+  CRUSH rule 5 x 271 [8,4,1]
+  CRUSH rule 5 x 272 [2,8,4]
+  CRUSH rule 5 x 273 [4,1,8]
+  CRUSH rule 5 x 274 [8,4,1]
+  CRUSH rule 5 x 275 [4,8,1]
+  CRUSH rule 5 x 276 [8,1,4]
+  CRUSH rule 5 x 277 [8,1]
+  CRUSH rule 5 x 278 [8,1,4]
+  CRUSH rule 5 x 279 [8,4,2]
+  CRUSH rule 5 x 280 [2,8,4]
+  CRUSH rule 5 x 281 [8,2]
+  CRUSH rule 5 x 282 [2,8]
+  CRUSH rule 5 x 283 [8,2]
+  CRUSH rule 5 x 284 [8,2]
+  CRUSH rule 5 x 285 [4,8,2]
+  CRUSH rule 5 x 286 [2,8,4]
+  CRUSH rule 5 x 287 [1,8]
+  CRUSH rule 5 x 288 [8,1,4]
+  CRUSH rule 5 x 289 [4,8,2]
+  CRUSH rule 5 x 290 [1,4,8]
+  CRUSH rule 5 x 291 [1,4,8]
+  CRUSH rule 5 x 292 [8,2,4]
+  CRUSH rule 5 x 293 [8,1]
+  CRUSH rule 5 x 294 [8,4,1]
+  CRUSH rule 5 x 295 [4,8,2]
+  CRUSH rule 5 x 296 [4,1,8]
+  CRUSH rule 5 x 297 [8,2,4]
+  CRUSH rule 5 x 298 [1,8]
+  CRUSH rule 5 x 299 [2,8]
+  CRUSH rule 5 x 300 [8,2]
+  CRUSH rule 5 x 301 [1,8]
+  CRUSH rule 5 x 302 [1,8]
+  CRUSH rule 5 x 303 [8,4,1]
+  CRUSH rule 5 x 304 [2,8]
+  CRUSH rule 5 x 305 [8,2]
+  CRUSH rule 5 x 306 [1,8]
+  CRUSH rule 5 x 307 [2,8]
+  CRUSH rule 5 x 308 [2,8,4]
+  CRUSH rule 5 x 309 [8,1]
+  CRUSH rule 5 x 310 [4,1,6]
+  CRUSH rule 5 x 311 [4,8,1]
+  CRUSH rule 5 x 312 [2,8]
+  CRUSH rule 5 x 313 [4,1,8]
+  CRUSH rule 5 x 314 [2,8]
+  CRUSH rule 5 x 315 [2,8]
+  CRUSH rule 5 x 316 [8,1]
+  CRUSH rule 5 x 317 [2,8]
+  CRUSH rule 5 x 318 [8,1]
+  CRUSH rule 5 x 319 [2,8]
+  CRUSH rule 5 x 320 [8,1]
+  CRUSH rule 5 x 321 [1,8]
+  CRUSH rule 5 x 322 [2,8,4]
+  CRUSH rule 5 x 323 [4,8,1]
+  CRUSH rule 5 x 324 [8,1,4]
+  CRUSH rule 5 x 325 [4,8,2]
+  CRUSH rule 5 x 326 [1,6]
+  CRUSH rule 5 x 327 [1,8]
+  CRUSH rule 5 x 328 [8,4,1]
+  CRUSH rule 5 x 329 [4,8,2]
+  CRUSH rule 5 x 330 [4,8,2]
+  CRUSH rule 5 x 331 [2,8]
+  CRUSH rule 5 x 332 [2,8]
+  CRUSH rule 5 x 333 [8,1]
+  CRUSH rule 5 x 334 [8,2]
+  CRUSH rule 5 x 335 [8,1]
+  CRUSH rule 5 x 336 [4,8,2]
+  CRUSH rule 5 x 337 [8,2,4]
+  CRUSH rule 5 x 338 [8,1]
+  CRUSH rule 5 x 339 [8,2]
+  CRUSH rule 5 x 340 [2,8,4]
+  CRUSH rule 5 x 341 [4,1,8]
+  CRUSH rule 5 x 342 [2,8,4]
+  CRUSH rule 5 x 343 [8,1]
+  CRUSH rule 5 x 344 [6,2,4]
+  CRUSH rule 5 x 345 [2,8]
+  CRUSH rule 5 x 346 [8,2,4]
+  CRUSH rule 5 x 347 [4,1,8]
+  CRUSH rule 5 x 348 [8,2,4]
+  CRUSH rule 5 x 349 [1,8]
+  CRUSH rule 5 x 350 [8,1]
+  CRUSH rule 5 x 351 [8,2]
+  CRUSH rule 5 x 352 [1,8,4]
+  CRUSH rule 5 x 353 [8,1]
+  CRUSH rule 5 x 354 [1,8]
+  CRUSH rule 5 x 355 [8,2]
+  CRUSH rule 5 x 356 [4,1,8]
+  CRUSH rule 5 x 357 [8,1,4]
+  CRUSH rule 5 x 358 [2,8,4]
+  CRUSH rule 5 x 359 [6,1,4]
+  CRUSH rule 5 x 360 [2,8]
+  CRUSH rule 5 x 361 [8,4,1]
+  CRUSH rule 5 x 362 [4,1,6]
+  CRUSH rule 5 x 363 [4,1,8]
+  CRUSH rule 5 x 364 [2,8]
+  CRUSH rule 5 x 365 [8,1]
+  CRUSH rule 5 x 366 [8,2]
+  CRUSH rule 5 x 367 [4,2,8]
+  CRUSH rule 5 x 368 [8,4,1]
+  CRUSH rule 5 x 369 [8,1]
+  CRUSH rule 5 x 370 [8,2]
+  CRUSH rule 5 x 371 [1,4,8]
+  CRUSH rule 5 x 372 [1,8]
+  CRUSH rule 5 x 373 [1,8]
+  CRUSH rule 5 x 374 [8,1]
+  CRUSH rule 5 x 375 [8,4,1]
+  CRUSH rule 5 x 376 [8,1,4]
+  CRUSH rule 5 x 377 [1,4,8]
+  CRUSH rule 5 x 378 [1,8]
+  CRUSH rule 5 x 379 [8,2]
+  CRUSH rule 5 x 380 [2,8]
+  CRUSH rule 5 x 381 [1,4,8]
+  CRUSH rule 5 x 382 [1,4,8]
+  CRUSH rule 5 x 383 [4,8,2]
+  CRUSH rule 5 x 384 [8,2,4]
+  CRUSH rule 5 x 385 [8,1]
+  CRUSH rule 5 x 386 [1,8]
+  CRUSH rule 5 x 387 [1,4,8]
+  CRUSH rule 5 x 388 [2,6]
+  CRUSH rule 5 x 389 [1,4,8]
+  CRUSH rule 5 x 390 [4,8,1]
+  CRUSH rule 5 x 391 [4,8,2]
+  CRUSH rule 5 x 392 [1,8,4]
+  CRUSH rule 5 x 393 [2,8]
+  CRUSH rule 5 x 394 [8,2]
+  CRUSH rule 5 x 395 [1,8]
+  CRUSH rule 5 x 396 [4,2,8]
+  CRUSH rule 5 x 397 [2,4,8]
+  CRUSH rule 5 x 398 [2,4,8]
+  CRUSH rule 5 x 399 [8,4,2]
+  CRUSH rule 5 x 400 [8,1,4]
+  CRUSH rule 5 x 401 [1,4,8]
+  CRUSH rule 5 x 402 [8,4,2]
+  CRUSH rule 5 x 403 [1,4,8]
+  CRUSH rule 5 x 404 [4,2,8]
+  CRUSH rule 5 x 405 [8,4,2]
+  CRUSH rule 5 x 406 [2,8]
+  CRUSH rule 5 x 407 [2,8,4]
+  CRUSH rule 5 x 408 [4,1,8]
+  CRUSH rule 5 x 409 [8,4,1]
+  CRUSH rule 5 x 410 [8,4,2]
+  CRUSH rule 5 x 411 [2,8,4]
+  CRUSH rule 5 x 412 [2,6]
+  CRUSH rule 5 x 413 [2,8]
+  CRUSH rule 5 x 414 [4,1,6]
+  CRUSH rule 5 x 415 [2,8]
+  CRUSH rule 5 x 416 [2,8]
+  CRUSH rule 5 x 417 [8,2]
+  CRUSH rule 5 x 418 [8,1,4]
+  CRUSH rule 5 x 419 [8,4,1]
+  CRUSH rule 5 x 420 [1,4,8]
+  CRUSH rule 5 x 421 [8,4,1]
+  CRUSH rule 5 x 422 [6,2]
+  CRUSH rule 5 x 423 [2,4,8]
+  CRUSH rule 5 x 424 [8,1]
+  CRUSH rule 5 x 425 [1,8]
+  CRUSH rule 5 x 426 [8,2]
+  CRUSH rule 5 x 427 [1,8]
+  CRUSH rule 5 x 428 [4,8,1]
+  CRUSH rule 5 x 429 [4,8,2]
+  CRUSH rule 5 x 430 [4,8,2]
+  CRUSH rule 5 x 431 [4,1,8]
+  CRUSH rule 5 x 432 [8,1]
+  CRUSH rule 5 x 433 [8,1]
+  CRUSH rule 5 x 434 [2,8]
+  CRUSH rule 5 x 435 [2,8]
+  CRUSH rule 5 x 436 [4,1,8]
+  CRUSH rule 5 x 437 [8,2]
+  CRUSH rule 5 x 438 [2,4,8]
+  CRUSH rule 5 x 439 [1,6]
+  CRUSH rule 5 x 440 [2,8]
+  CRUSH rule 5 x 441 [4,6,2]
+  CRUSH rule 5 x 442 [2,8]
+  CRUSH rule 5 x 443 [8,2,4]
+  CRUSH rule 5 x 444 [8,1]
+  CRUSH rule 5 x 445 [8,2]
+  CRUSH rule 5 x 446 [8,1]
+  CRUSH rule 5 x 447 [2,4,8]
+  CRUSH rule 5 x 448 [8,2,4]
+  CRUSH rule 5 x 449 [8,1]
+  CRUSH rule 5 x 450 [1,8]
+  CRUSH rule 5 x 451 [8,4,2]
+  CRUSH rule 5 x 452 [8,2]
+  CRUSH rule 5 x 453 [6,2]
+  CRUSH rule 5 x 454 [8,2]
+  CRUSH rule 5 x 455 [2,8,4]
+  CRUSH rule 5 x 456 [8,2]
+  CRUSH rule 5 x 457 [8,2]
+  CRUSH rule 5 x 458 [2,8]
+  CRUSH rule 5 x 459 [2,8,4]
+  CRUSH rule 5 x 460 [8,2]
+  CRUSH rule 5 x 461 [8,1]
+  CRUSH rule 5 x 462 [8,1]
+  CRUSH rule 5 x 463 [8,2]
+  CRUSH rule 5 x 464 [8,4,2]
+  CRUSH rule 5 x 465 [6,2,4]
+  CRUSH rule 5 x 466 [8,1]
+  CRUSH rule 5 x 467 [8,2]
+  CRUSH rule 5 x 468 [8,1,4]
+  CRUSH rule 5 x 469 [8,1]
+  CRUSH rule 5 x 470 [4,2,6]
+  CRUSH rule 5 x 471 [1,8]
+  CRUSH rule 5 x 472 [1,8]
+  CRUSH rule 5 x 473 [1,4,8]
+  CRUSH rule 5 x 474 [8,1]
+  CRUSH rule 5 x 475 [8,2,4]
+  CRUSH rule 5 x 476 [4,8,1]
+  CRUSH rule 5 x 477 [4,8,2]
+  CRUSH rule 5 x 478 [8,2,4]
+  CRUSH rule 5 x 479 [2,8]
+  CRUSH rule 5 x 480 [1,8]
+  CRUSH rule 5 x 481 [2,4,6]
+  CRUSH rule 5 x 482 [1,8]
+  CRUSH rule 5 x 483 [2,8,4]
+  CRUSH rule 5 x 484 [1,8]
+  CRUSH rule 5 x 485 [8,1]
+  CRUSH rule 5 x 486 [4,1,8]
+  CRUSH rule 5 x 487 [1,8]
+  CRUSH rule 5 x 488 [8,1]
+  CRUSH rule 5 x 489 [2,8]
+  CRUSH rule 5 x 490 [6,2]
+  CRUSH rule 5 x 491 [1,8]
+  CRUSH rule 5 x 492 [8,1]
+  CRUSH rule 5 x 493 [2,8]
+  CRUSH rule 5 x 494 [1,8]
+  CRUSH rule 5 x 495 [4,1,8]
+  CRUSH rule 5 x 496 [8,4,1]
+  CRUSH rule 5 x 497 [4,8,1]
+  CRUSH rule 5 x 498 [2,4,8]
+  CRUSH rule 5 x 499 [8,4,2]
+  CRUSH rule 5 x 500 [4,8,2]
+  CRUSH rule 5 x 501 [2,8]
+  CRUSH rule 5 x 502 [6,1]
+  CRUSH rule 5 x 503 [2,8]
+  CRUSH rule 5 x 504 [8,1]
+  CRUSH rule 5 x 505 [1,8]
+  CRUSH rule 5 x 506 [4,2,8]
+  CRUSH rule 5 x 507 [8,1,4]
+  CRUSH rule 5 x 508 [1,8]
+  CRUSH rule 5 x 509 [8,1]
+  CRUSH rule 5 x 510 [8,2]
+  CRUSH rule 5 x 511 [4,8,2]
+  CRUSH rule 5 x 512 [8,2]
+  CRUSH rule 5 x 513 [8,2]
+  CRUSH rule 5 x 514 [2,8]
+  CRUSH rule 5 x 515 [8,4,1]
+  CRUSH rule 5 x 516 [4,1,8]
+  CRUSH rule 5 x 517 [8,2]
+  CRUSH rule 5 x 518 [4,8,1]
+  CRUSH rule 5 x 519 [8,4,1]
+  CRUSH rule 5 x 520 [2,8,4]
+  CRUSH rule 5 x 521 [8,2,4]
+  CRUSH rule 5 x 522 [8,1,4]
+  CRUSH rule 5 x 523 [4,2,8]
+  CRUSH rule 5 x 524 [2,6]
+  CRUSH rule 5 x 525 [2,8]
+  CRUSH rule 5 x 526 [1,8]
+  CRUSH rule 5 x 527 [1,4,6]
+  CRUSH rule 5 x 528 [2,8]
+  CRUSH rule 5 x 529 [4,8,2]
+  CRUSH rule 5 x 530 [8,1]
+  CRUSH rule 5 x 531 [8,1,4]
+  CRUSH rule 5 x 532 [6,4,1]
+  CRUSH rule 5 x 533 [4,8,2]
+  CRUSH rule 5 x 534 [8,1]
+  CRUSH rule 5 x 535 [8,1]
+  CRUSH rule 5 x 536 [8,2]
+  CRUSH rule 5 x 537 [4,8,2]
+  CRUSH rule 5 x 538 [8,4,1]
+  CRUSH rule 5 x 539 [8,1]
+  CRUSH rule 5 x 540 [1,8,4]
+  CRUSH rule 5 x 541 [2,4,8]
+  CRUSH rule 5 x 542 [2,8]
+  CRUSH rule 5 x 543 [8,2]
+  CRUSH rule 5 x 544 [4,8,2]
+  CRUSH rule 5 x 545 [8,1]
+  CRUSH rule 5 x 546 [8,1,4]
+  CRUSH rule 5 x 547 [8,2,4]
+  CRUSH rule 5 x 548 [4,2,8]
+  CRUSH rule 5 x 549 [8,2]
+  CRUSH rule 5 x 550 [2,4,8]
+  CRUSH rule 5 x 551 [8,1]
+  CRUSH rule 5 x 552 [4,8,1]
+  CRUSH rule 5 x 553 [2,8]
+  CRUSH rule 5 x 554 [1,8]
+  CRUSH rule 5 x 555 [4,1,8]
+  CRUSH rule 5 x 556 [8,1]
+  CRUSH rule 5 x 557 [8,2]
+  CRUSH rule 5 x 558 [4,1,8]
+  CRUSH rule 5 x 559 [1,8]
+  CRUSH rule 5 x 560 [8,1]
+  CRUSH rule 5 x 561 [8,4,1]
+  CRUSH rule 5 x 562 [4,1,8]
+  CRUSH rule 5 x 563 [2,8]
+  CRUSH rule 5 x 564 [1,8]
+  CRUSH rule 5 x 565 [4,8,2]
+  CRUSH rule 5 x 566 [4,8,2]
+  CRUSH rule 5 x 567 [4,8,1]
+  CRUSH rule 5 x 568 [8,1]
+  CRUSH rule 5 x 569 [4,1,8]
+  CRUSH rule 5 x 570 [1,8]
+  CRUSH rule 5 x 571 [6,1]
+  CRUSH rule 5 x 572 [4,2,8]
+  CRUSH rule 5 x 573 [1,8]
+  CRUSH rule 5 x 574 [2,8]
+  CRUSH rule 5 x 575 [8,2,4]
+  CRUSH rule 5 x 576 [4,8,2]
+  CRUSH rule 5 x 577 [8,2]
+  CRUSH rule 5 x 578 [8,1]
+  CRUSH rule 5 x 579 [4,1,8]
+  CRUSH rule 5 x 580 [1,8]
+  CRUSH rule 5 x 581 [8,2,4]
+  CRUSH rule 5 x 582 [2,8,4]
+  CRUSH rule 5 x 583 [8,1]
+  CRUSH rule 5 x 584 [8,1,4]
+  CRUSH rule 5 x 585 [8,1,4]
+  CRUSH rule 5 x 586 [1,8,4]
+  CRUSH rule 5 x 587 [2,4,8]
+  CRUSH rule 5 x 588 [4,8,1]
+  CRUSH rule 5 x 589 [8,1]
+  CRUSH rule 5 x 590 [8,2]
+  CRUSH rule 5 x 591 [4,2,8]
+  CRUSH rule 5 x 592 [2,4,8]
+  CRUSH rule 5 x 593 [1,8,4]
+  CRUSH rule 5 x 594 [2,8]
+  CRUSH rule 5 x 595 [8,1]
+  CRUSH rule 5 x 596 [8,2]
+  CRUSH rule 5 x 597 [1,8]
+  CRUSH rule 5 x 598 [2,8]
+  CRUSH rule 5 x 599 [4,2,8]
+  CRUSH rule 5 x 600 [8,1,4]
+  CRUSH rule 5 x 601 [1,8,4]
+  CRUSH rule 5 x 602 [8,2]
+  CRUSH rule 5 x 603 [1,8]
+  CRUSH rule 5 x 604 [8,2]
+  CRUSH rule 5 x 605 [2,8]
+  CRUSH rule 5 x 606 [2,6,4]
+  CRUSH rule 5 x 607 [2,4,8]
+  CRUSH rule 5 x 608 [4,2,8]
+  CRUSH rule 5 x 609 [4,2,8]
+  CRUSH rule 5 x 610 [8,1]
+  CRUSH rule 5 x 611 [1,8]
+  CRUSH rule 5 x 612 [2,8]
+  CRUSH rule 5 x 613 [8,2,4]
+  CRUSH rule 5 x 614 [8,2,4]
+  CRUSH rule 5 x 615 [8,2,4]
+  CRUSH rule 5 x 616 [1,8]
+  CRUSH rule 5 x 617 [8,1,4]
+  CRUSH rule 5 x 618 [8,4,2]
+  CRUSH rule 5 x 619 [4,1,8]
+  CRUSH rule 5 x 620 [1,8]
+  CRUSH rule 5 x 621 [8,1]
+  CRUSH rule 5 x 622 [2,4,8]
+  CRUSH rule 5 x 623 [2,8]
+  CRUSH rule 5 x 624 [4,2,8]
+  CRUSH rule 5 x 625 [2,8]
+  CRUSH rule 5 x 626 [8,2,4]
+  CRUSH rule 5 x 627 [2,8,4]
+  CRUSH rule 5 x 628 [8,2]
+  CRUSH rule 5 x 629 [2,8,4]
+  CRUSH rule 5 x 630 [2,8]
+  CRUSH rule 5 x 631 [1,8,4]
+  CRUSH rule 5 x 632 [8,2]
+  CRUSH rule 5 x 633 [8,2]
+  CRUSH rule 5 x 634 [1,8]
+  CRUSH rule 5 x 635 [4,8,2]
+  CRUSH rule 5 x 636 [1,4,8]
+  CRUSH rule 5 x 637 [1,8]
+  CRUSH rule 5 x 638 [8,1,4]
+  CRUSH rule 5 x 639 [2,8]
+  CRUSH rule 5 x 640 [2,8]
+  CRUSH rule 5 x 641 [8,2]
+  CRUSH rule 5 x 642 [2,8]
+  CRUSH rule 5 x 643 [1,8]
+  CRUSH rule 5 x 644 [8,1]
+  CRUSH rule 5 x 645 [8,1]
+  CRUSH rule 5 x 646 [8,1,4]
+  CRUSH rule 5 x 647 [8,1]
+  CRUSH rule 5 x 648 [1,8]
+  CRUSH rule 5 x 649 [4,8,2]
+  CRUSH rule 5 x 650 [8,4,1]
+  CRUSH rule 5 x 651 [4,6,1]
+  CRUSH rule 5 x 652 [4,8,1]
+  CRUSH rule 5 x 653 [8,2]
+  CRUSH rule 5 x 654 [6,2]
+  CRUSH rule 5 x 655 [1,4,8]
+  CRUSH rule 5 x 656 [8,1]
+  CRUSH rule 5 x 657 [6,1]
+  CRUSH rule 5 x 658 [8,2]
+  CRUSH rule 5 x 659 [4,8,2]
+  CRUSH rule 5 x 660 [8,2]
+  CRUSH rule 5 x 661 [1,8]
+  CRUSH rule 5 x 662 [8,2]
+  CRUSH rule 5 x 663 [1,4,8]
+  CRUSH rule 5 x 664 [1,4,8]
+  CRUSH rule 5 x 665 [4,6,1]
+  CRUSH rule 5 x 666 [2,8]
+  CRUSH rule 5 x 667 [1,4,8]
+  CRUSH rule 5 x 668 [4,8,1]
+  CRUSH rule 5 x 669 [6,4,2]
+  CRUSH rule 5 x 670 [4,2,8]
+  CRUSH rule 5 x 671 [2,8]
+  CRUSH rule 5 x 672 [4,2,8]
+  CRUSH rule 5 x 673 [4,2,8]
+  CRUSH rule 5 x 674 [1,8]
+  CRUSH rule 5 x 675 [1,8,4]
+  CRUSH rule 5 x 676 [2,4,8]
+  CRUSH rule 5 x 677 [4,1,8]
+  CRUSH rule 5 x 678 [2,4,8]
+  CRUSH rule 5 x 679 [8,2]
+  CRUSH rule 5 x 680 [2,8]
+  CRUSH rule 5 x 681 [8,1]
+  CRUSH rule 5 x 682 [1,4,8]
+  CRUSH rule 5 x 683 [1,4,8]
+  CRUSH rule 5 x 684 [8,1,4]
+  CRUSH rule 5 x 685 [8,1,4]
+  CRUSH rule 5 x 686 [1,4,8]
+  CRUSH rule 5 x 687 [6,1]
+  CRUSH rule 5 x 688 [4,8,2]
+  CRUSH rule 5 x 689 [8,4,2]
+  CRUSH rule 5 x 690 [8,1,4]
+  CRUSH rule 5 x 691 [1,8]
+  CRUSH rule 5 x 692 [8,2]
+  CRUSH rule 5 x 693 [8,4,1]
+  CRUSH rule 5 x 694 [8,4,1]
+  CRUSH rule 5 x 695 [2,8,4]
+  CRUSH rule 5 x 696 [1,8]
+  CRUSH rule 5 x 697 [8,1,4]
+  CRUSH rule 5 x 698 [8,2,4]
+  CRUSH rule 5 x 699 [1,8,4]
+  CRUSH rule 5 x 700 [1,8]
+  CRUSH rule 5 x 701 [1,8]
+  CRUSH rule 5 x 702 [2,8]
+  CRUSH rule 5 x 703 [8,1]
+  CRUSH rule 5 x 704 [1,4,8]
+  CRUSH rule 5 x 705 [8,1,4]
+  CRUSH rule 5 x 706 [1,4,8]
+  CRUSH rule 5 x 707 [8,4,1]
+  CRUSH rule 5 x 708 [4,8,1]
+  CRUSH rule 5 x 709 [8,2]
+  CRUSH rule 5 x 710 [8,2]
+  CRUSH rule 5 x 711 [2,4,8]
+  CRUSH rule 5 x 712 [2,8]
+  CRUSH rule 5 x 713 [8,4,1]
+  CRUSH rule 5 x 714 [2,8]
+  CRUSH rule 5 x 715 [1,8]
+  CRUSH rule 5 x 716 [4,8,2]
+  CRUSH rule 5 x 717 [8,2,4]
+  CRUSH rule 5 x 718 [8,1]
+  CRUSH rule 5 x 719 [2,6,4]
+  CRUSH rule 5 x 720 [8,1,4]
+  CRUSH rule 5 x 721 [4,6,2]
+  CRUSH rule 5 x 722 [8,2]
+  CRUSH rule 5 x 723 [4,1,8]
+  CRUSH rule 5 x 724 [2,6]
+  CRUSH rule 5 x 725 [1,8]
+  CRUSH rule 5 x 726 [4,8,1]
+  CRUSH rule 5 x 727 [4,8,1]
+  CRUSH rule 5 x 728 [2,8,4]
+  CRUSH rule 5 x 729 [8,2]
+  CRUSH rule 5 x 730 [4,8,2]
+  CRUSH rule 5 x 731 [4,1,8]
+  CRUSH rule 5 x 732 [1,8]
+  CRUSH rule 5 x 733 [4,8,1]
+  CRUSH rule 5 x 734 [8,4,2]
+  CRUSH rule 5 x 735 [4,8,1]
+  CRUSH rule 5 x 736 [4,8,1]
+  CRUSH rule 5 x 737 [1,8,4]
+  CRUSH rule 5 x 738 [4,2,8]
+  CRUSH rule 5 x 739 [2,8]
+  CRUSH rule 5 x 740 [1,8,4]
+  CRUSH rule 5 x 741 [8,1]
+  CRUSH rule 5 x 742 [8,2]
+  CRUSH rule 5 x 743 [8,1,4]
+  CRUSH rule 5 x 744 [4,8,1]
+  CRUSH rule 5 x 745 [1,8]
+  CRUSH rule 5 x 746 [1,8]
+  CRUSH rule 5 x 747 [8,1]
+  CRUSH rule 5 x 748 [2,8,4]
+  CRUSH rule 5 x 749 [4,8,2]
+  CRUSH rule 5 x 750 [1,8,4]
+  CRUSH rule 5 x 751 [2,8]
+  CRUSH rule 5 x 752 [8,1]
+  CRUSH rule 5 x 753 [8,4,1]
+  CRUSH rule 5 x 754 [8,4,2]
+  CRUSH rule 5 x 755 [1,8,4]
+  CRUSH rule 5 x 756 [8,1]
+  CRUSH rule 5 x 757 [8,1,4]
+  CRUSH rule 5 x 758 [8,2]
+  CRUSH rule 5 x 759 [8,4,2]
+  CRUSH rule 5 x 760 [1,4,8]
+  CRUSH rule 5 x 761 [2,8]
+  CRUSH rule 5 x 762 [2,8]
+  CRUSH rule 5 x 763 [8,4,1]
+  CRUSH rule 5 x 764 [1,8]
+  CRUSH rule 5 x 765 [8,2]
+  CRUSH rule 5 x 766 [8,1]
+  CRUSH rule 5 x 767 [1,8,4]
+  CRUSH rule 5 x 768 [8,4,2]
+  CRUSH rule 5 x 769 [8,2,4]
+  CRUSH rule 5 x 770 [8,2,4]
+  CRUSH rule 5 x 771 [8,1,4]
+  CRUSH rule 5 x 772 [8,4,1]
+  CRUSH rule 5 x 773 [4,1,8]
+  CRUSH rule 5 x 774 [8,1]
+  CRUSH rule 5 x 775 [8,4,2]
+  CRUSH rule 5 x 776 [6,2]
+  CRUSH rule 5 x 777 [4,1,8]
+  CRUSH rule 5 x 778 [1,8,4]
+  CRUSH rule 5 x 779 [2,8]
+  CRUSH rule 5 x 780 [2,4,8]
+  CRUSH rule 5 x 781 [8,2]
+  CRUSH rule 5 x 782 [4,1,8]
+  CRUSH rule 5 x 783 [8,1,4]
+  CRUSH rule 5 x 784 [1,4,8]
+  CRUSH rule 5 x 785 [8,1,4]
+  CRUSH rule 5 x 786 [8,1]
+  CRUSH rule 5 x 787 [1,6,4]
+  CRUSH rule 5 x 788 [8,2,4]
+  CRUSH rule 5 x 789 [1,8]
+  CRUSH rule 5 x 790 [8,1]
+  CRUSH rule 5 x 791 [4,8,2]
+  CRUSH rule 5 x 792 [4,8,2]
+  CRUSH rule 5 x 793 [8,1,4]
+  CRUSH rule 5 x 794 [2,8,4]
+  CRUSH rule 5 x 795 [1,8]
+  CRUSH rule 5 x 796 [8,2]
+  CRUSH rule 5 x 797 [2,4,8]
+  CRUSH rule 5 x 798 [6,1]
+  CRUSH rule 5 x 799 [4,1,8]
+  CRUSH rule 5 x 800 [2,8]
+  CRUSH rule 5 x 801 [4,8,1]
+  CRUSH rule 5 x 802 [1,8,4]
+  CRUSH rule 5 x 803 [2,8]
+  CRUSH rule 5 x 804 [8,2]
+  CRUSH rule 5 x 805 [8,2]
+  CRUSH rule 5 x 806 [1,4,8]
+  CRUSH rule 5 x 807 [4,8,2]
+  CRUSH rule 5 x 808 [8,2]
+  CRUSH rule 5 x 809 [1,8]
+  CRUSH rule 5 x 810 [8,2]
+  CRUSH rule 5 x 811 [8,1]
+  CRUSH rule 5 x 812 [8,4,2]
+  CRUSH rule 5 x 813 [8,4,2]
+  CRUSH rule 5 x 814 [8,2]
+  CRUSH rule 5 x 815 [4,1,8]
+  CRUSH rule 5 x 816 [2,8]
+  CRUSH rule 5 x 817 [8,1]
+  CRUSH rule 5 x 818 [1,8]
+  CRUSH rule 5 x 819 [1,8]
+  CRUSH rule 5 x 820 [4,8,2]
+  CRUSH rule 5 x 821 [4,8,2]
+  CRUSH rule 5 x 822 [2,4,8]
+  CRUSH rule 5 x 823 [4,8,2]
+  CRUSH rule 5 x 824 [8,2]
+  CRUSH rule 5 x 825 [2,8,4]
+  CRUSH rule 5 x 826 [8,2,4]
+  CRUSH rule 5 x 827 [2,8,4]
+  CRUSH rule 5 x 828 [2,8]
+  CRUSH rule 5 x 829 [8,1]
+  CRUSH rule 5 x 830 [2,4,8]
+  CRUSH rule 5 x 831 [1,8]
+  CRUSH rule 5 x 832 [4,8,2]
+  CRUSH rule 5 x 833 [2,8]
+  CRUSH rule 5 x 834 [1,8]
+  CRUSH rule 5 x 835 [8,4,1]
+  CRUSH rule 5 x 836 [4,8,1]
+  CRUSH rule 5 x 837 [8,4,1]
+  CRUSH rule 5 x 838 [6,2,4]
+  CRUSH rule 5 x 839 [2,8]
+  CRUSH rule 5 x 840 [8,1]
+  CRUSH rule 5 x 841 [4,8,2]
+  CRUSH rule 5 x 842 [2,8]
+  CRUSH rule 5 x 843 [8,4,1]
+  CRUSH rule 5 x 844 [8,2]
+  CRUSH rule 5 x 845 [4,8,2]
+  CRUSH rule 5 x 846 [4,2,8]
+  CRUSH rule 5 x 847 [2,8]
+  CRUSH rule 5 x 848 [2,8,4]
+  CRUSH rule 5 x 849 [4,8,2]
+  CRUSH rule 5 x 850 [1,6]
+  CRUSH rule 5 x 851 [6,1]
+  CRUSH rule 5 x 852 [8,4,2]
+  CRUSH rule 5 x 853 [6,1]
+  CRUSH rule 5 x 854 [8,1]
+  CRUSH rule 5 x 855 [8,1]
+  CRUSH rule 5 x 856 [8,4,2]
+  CRUSH rule 5 x 857 [8,2]
+  CRUSH rule 5 x 858 [6,1]
+  CRUSH rule 5 x 859 [8,2,4]
+  CRUSH rule 5 x 860 [2,8]
+  CRUSH rule 5 x 861 [8,2]
+  CRUSH rule 5 x 862 [8,1]
+  CRUSH rule 5 x 863 [8,2]
+  CRUSH rule 5 x 864 [8,2]
+  CRUSH rule 5 x 865 [8,1]
+  CRUSH rule 5 x 866 [8,2]
+  CRUSH rule 5 x 867 [8,2]
+  CRUSH rule 5 x 868 [8,1]
+  CRUSH rule 5 x 869 [8,4,2]
+  CRUSH rule 5 x 870 [2,8]
+  CRUSH rule 5 x 871 [1,8]
+  CRUSH rule 5 x 872 [1,8]
+  CRUSH rule 5 x 873 [4,8,2]
+  CRUSH rule 5 x 874 [2,6]
+  CRUSH rule 5 x 875 [2,8,4]
+  CRUSH rule 5 x 876 [4,8,1]
+  CRUSH rule 5 x 877 [8,4,2]
+  CRUSH rule 5 x 878 [2,8]
+  CRUSH rule 5 x 879 [8,1]
+  CRUSH rule 5 x 880 [1,8]
+  CRUSH rule 5 x 881 [4,8,1]
+  CRUSH rule 5 x 882 [1,8]
+  CRUSH rule 5 x 883 [2,4,8]
+  CRUSH rule 5 x 884 [8,2,4]
+  CRUSH rule 5 x 885 [4,1,8]
+  CRUSH rule 5 x 886 [8,2]
+  CRUSH rule 5 x 887 [8,4,1]
+  CRUSH rule 5 x 888 [8,2]
+  CRUSH rule 5 x 889 [2,6]
+  CRUSH rule 5 x 890 [8,2,4]
+  CRUSH rule 5 x 891 [1,8]
+  CRUSH rule 5 x 892 [8,2,4]
+  CRUSH rule 5 x 893 [2,6]
+  CRUSH rule 5 x 894 [8,4,2]
+  CRUSH rule 5 x 895 [4,1,8]
+  CRUSH rule 5 x 896 [1,8]
+  CRUSH rule 5 x 897 [2,8]
+  CRUSH rule 5 x 898 [1,4,8]
+  CRUSH rule 5 x 899 [1,8]
+  CRUSH rule 5 x 900 [4,1,8]
+  CRUSH rule 5 x 901 [2,8]
+  CRUSH rule 5 x 902 [8,4,1]
+  CRUSH rule 5 x 903 [8,2]
+  CRUSH rule 5 x 904 [8,2]
+  CRUSH rule 5 x 905 [8,2]
+  CRUSH rule 5 x 906 [1,8]
+  CRUSH rule 5 x 907 [8,1]
+  CRUSH rule 5 x 908 [8,1]
+  CRUSH rule 5 x 909 [2,8]
+  CRUSH rule 5 x 910 [8,2]
+  CRUSH rule 5 x 911 [8,1]
+  CRUSH rule 5 x 912 [1,8]
+  CRUSH rule 5 x 913 [8,2,4]
+  CRUSH rule 5 x 914 [6,4,2]
+  CRUSH rule 5 x 915 [8,2]
+  CRUSH rule 5 x 916 [4,1,8]
+  CRUSH rule 5 x 917 [1,4,8]
+  CRUSH rule 5 x 918 [8,2]
+  CRUSH rule 5 x 919 [8,2]
+  CRUSH rule 5 x 920 [8,1]
+  CRUSH rule 5 x 921 [1,8]
+  CRUSH rule 5 x 922 [8,4,2]
+  CRUSH rule 5 x 923 [4,8,2]
+  CRUSH rule 5 x 924 [8,1]
+  CRUSH rule 5 x 925 [4,8,2]
+  CRUSH rule 5 x 926 [2,8]
+  CRUSH rule 5 x 927 [1,8,4]
+  CRUSH rule 5 x 928 [8,1]
+  CRUSH rule 5 x 929 [4,1,8]
+  CRUSH rule 5 x 930 [2,8]
+  CRUSH rule 5 x 931 [2,8]
+  CRUSH rule 5 x 932 [4,1,8]
+  CRUSH rule 5 x 933 [8,4,1]
+  CRUSH rule 5 x 934 [8,2]
+  CRUSH rule 5 x 935 [8,2]
+  CRUSH rule 5 x 936 [1,8]
+  CRUSH rule 5 x 937 [4,8,2]
+  CRUSH rule 5 x 938 [8,4,2]
+  CRUSH rule 5 x 939 [2,8,4]
+  CRUSH rule 5 x 940 [8,1]
+  CRUSH rule 5 x 941 [2,8]
+  CRUSH rule 5 x 942 [1,8]
+  CRUSH rule 5 x 943 [8,2]
+  CRUSH rule 5 x 944 [8,2]
+  CRUSH rule 5 x 945 [8,2,4]
+  CRUSH rule 5 x 946 [2,8,4]
+  CRUSH rule 5 x 947 [8,2]
+  CRUSH rule 5 x 948 [8,1]
+  CRUSH rule 5 x 949 [6,1]
+  CRUSH rule 5 x 950 [8,1]
+  CRUSH rule 5 x 951 [8,1]
+  CRUSH rule 5 x 952 [2,8,4]
+  CRUSH rule 5 x 953 [1,4,8]
+  CRUSH rule 5 x 954 [2,8]
+  CRUSH rule 5 x 955 [8,1,4]
+  CRUSH rule 5 x 956 [1,8,4]
+  CRUSH rule 5 x 957 [8,1,4]
+  CRUSH rule 5 x 958 [8,4,1]
+  CRUSH rule 5 x 959 [4,2,8]
+  CRUSH rule 5 x 960 [6,1]
+  CRUSH rule 5 x 961 [1,8]
+  CRUSH rule 5 x 962 [8,4,2]
+  CRUSH rule 5 x 963 [2,4,6]
+  CRUSH rule 5 x 964 [2,8]
+  CRUSH rule 5 x 965 [8,2]
+  CRUSH rule 5 x 966 [4,8,1]
+  CRUSH rule 5 x 967 [8,4,2]
+  CRUSH rule 5 x 968 [8,2]
+  CRUSH rule 5 x 969 [8,2,4]
+  CRUSH rule 5 x 970 [2,8,4]
+  CRUSH rule 5 x 971 [1,8]
+  CRUSH rule 5 x 972 [1,8]
+  CRUSH rule 5 x 973 [1,8]
+  CRUSH rule 5 x 974 [4,1,8]
+  CRUSH rule 5 x 975 [4,8,1]
+  CRUSH rule 5 x 976 [4,8,2]
+  CRUSH rule 5 x 977 [8,4,2]
+  CRUSH rule 5 x 978 [8,2,4]
+  CRUSH rule 5 x 979 [8,1,4]
+  CRUSH rule 5 x 980 [8,2,4]
+  CRUSH rule 5 x 981 [8,2]
+  CRUSH rule 5 x 982 [1,8]
+  CRUSH rule 5 x 983 [4,8,2]
+  CRUSH rule 5 x 984 [2,8]
+  CRUSH rule 5 x 985 [2,4,8]
+  CRUSH rule 5 x 986 [8,4,1]
+  CRUSH rule 5 x 987 [2,8]
+  CRUSH rule 5 x 988 [1,4,6]
+  CRUSH rule 5 x 989 [1,8]
+  CRUSH rule 5 x 990 [1,8,4]
+  CRUSH rule 5 x 991 [1,4,8]
+  CRUSH rule 5 x 992 [8,1,4]
+  CRUSH rule 5 x 993 [2,8,4]
+  CRUSH rule 5 x 994 [4,8,2]
+  CRUSH rule 5 x 995 [8,1,4]
+  CRUSH rule 5 x 996 [8,4,1]
+  CRUSH rule 5 x 997 [8,4,1]
+  CRUSH rule 5 x 998 [8,1,4]
+  CRUSH rule 5 x 999 [1,8,4]
+  CRUSH rule 5 x 1000 [8,4,2]
+  CRUSH rule 5 x 1001 [2,8]
+  CRUSH rule 5 x 1002 [1,8]
+  CRUSH rule 5 x 1003 [2,8]
+  CRUSH rule 5 x 1004 [8,1,4]
+  CRUSH rule 5 x 1005 [8,1]
+  CRUSH rule 5 x 1006 [1,8,4]
+  CRUSH rule 5 x 1007 [1,4,8]
+  CRUSH rule 5 x 1008 [1,8]
+  CRUSH rule 5 x 1009 [6,4,1]
+  CRUSH rule 5 x 1010 [1,8]
+  CRUSH rule 5 x 1011 [4,2,8]
+  CRUSH rule 5 x 1012 [1,8]
+  CRUSH rule 5 x 1013 [2,8]
+  CRUSH rule 5 x 1014 [2,8,4]
+  CRUSH rule 5 x 1015 [8,1]
+  CRUSH rule 5 x 1016 [2,4,8]
+  CRUSH rule 5 x 1017 [6,2,4]
+  CRUSH rule 5 x 1018 [4,1,8]
+  CRUSH rule 5 x 1019 [4,8,2]
+  CRUSH rule 5 x 1020 [1,8]
+  CRUSH rule 5 x 1021 [2,8]
+  CRUSH rule 5 x 1022 [1,8,4]
+  CRUSH rule 5 x 1023 [4,2,8]
+  rule 5 (chooseleaf-set) num_rep 3 result size == 2:\t501/1024 (esc)
+  rule 5 (chooseleaf-set) num_rep 3 result size == 3:\t523/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/simple.template b/ceph/src/test/cli/crushtool/simple.template
new file mode 100644
index 0000000000000000000000000000000000000000..8f0076f548e9743e9769dd9b7c4de913a4143be3
GIT binary patch
literal 316
zcmb7A(F%Yd4BRX#`kH>pa!^nNYW;rgq`gKRY|Pu;oDwm0L{w=2wZ^4zr7F~GV(k_$
zbB`Kc7#bu6ILtJdo`34yN4>oR_XjzbNo<38PuVHkYE0{qvP4@LAMj9v4ZzaDXAcef
Nn`DnOMnDtCa0AVuB?JHf

literal 0
HcmV?d00001

diff --git a/ceph/src/test/cli/crushtool/simple.template.five b/ceph/src/test/cli/crushtool/simple.template.five
new file mode 100644
index 00000000..240e81de
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/simple.template.five
@@ -0,0 +1,65 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 osd1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+host host1 {
+	id -3		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item osd1 weight 2.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/simple.template.four b/ceph/src/test/cli/crushtool/simple.template.four
new file mode 100644
index 00000000..aa16bbde
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/simple.template.four
@@ -0,0 +1,56 @@
+# begin crush map
+
+# devices
+device 0 device0
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/simple.template.multitree b/ceph/src/test/cli/crushtool/simple.template.multitree
new file mode 100644
index 00000000..e2967a1c
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/simple.template.multitree
@@ -0,0 +1,70 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 osd1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+host host1 {
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item osd1 weight 2.000
+}
+cluster cluster0 {
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 2.000
+}
+
+cluster cluster1 {
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster1
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/simple.template.multitree.reweighted b/ceph/src/test/cli/crushtool/simple.template.multitree.reweighted
new file mode 100644
index 00000000..caad7ab7
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/simple.template.multitree.reweighted
@@ -0,0 +1,73 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 osd1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -1		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+host host1 {
+	id -2		# do not change unnecessarily
+	# weight 2.500
+	alg straw
+	hash 0	# rjenkins1
+	item osd1 weight 2.500
+}
+cluster cluster0 {
+	id -3		# do not change unnecessarily
+	# weight 3.500
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 2.500
+}
+cluster cluster1 {
+	id -4		# do not change unnecessarily
+	# weight 3.500
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 2.500
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster1
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/simple.template.one b/ceph/src/test/cli/crushtool/simple.template.one
new file mode 100644
index 00000000..9a3aee73
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/simple.template.one
@@ -0,0 +1,58 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 device1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+	item device1 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/simple.template.three b/ceph/src/test/cli/crushtool/simple.template.three
new file mode 100644
index 00000000..9a3aee73
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/simple.template.three
@@ -0,0 +1,58 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 device1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+	item device1 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/simple.template.two b/ceph/src/test/cli/crushtool/simple.template.two
new file mode 100644
index 00000000..9a3aee73
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/simple.template.two
@@ -0,0 +1,58 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 device1
+
+# types
+type 0 device
+type 1 host
+type 2 cluster
+
+# buckets
+host host0 {
+	id -2		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+	item device1 weight 1.000
+}
+cluster cluster0 {
+	id -1		# do not change unnecessarily
+	# weight 2.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 2.000
+}
+
+# rules
+rule data {
+	ruleset 0
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule metadata {
+	ruleset 1
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+rule rbd {
+	ruleset 2
+	type replicated
+	min_size 1
+	max_size 10
+	step take cluster0
+	step chooseleaf firstn 0 type host
+	step emit
+}
+
+# end crush map
diff --git a/ceph/src/test/cli/crushtool/test-map-a.crushmap b/ceph/src/test/cli/crushtool/test-map-a.crushmap
new file mode 100644
index 0000000000000000000000000000000000000000..00f7579ac174a85fd4789971d04c9b99d45fe41a
GIT binary patch
literal 31995
zcmeI*Wwac{y2kMcNpK79?u72DE@?ct26v4TAV!4X1PJa9ad&rjcXxN(-R|>-({t$M4H8`Mz>U){!+Tpo*vx+7<1F{{1fb&SH{%{>sfF^X!54M0=sV
z(LQKjv>)0ZRYpb4kgX%L{F$5O06caeItU$%4nc>a!_eXA2y`UM)+p*Iv;2{pls5NSX+M;%-Jt}I3Y#o{9
z>)b3I@K{IG33WzYP*>Cqbw@o=Pn4}u)KO;nDmP0nJk}faL48p_)E^B%1JNKf7!@@`
zwvNp5Wp0)scx)&dhK8dNXe1hiMx!xkEXvj>>L{~(k(*^49vhD)powS_nvABPsc0IS
zj*6NgTSsR3JU7b>JT?=}LbK5vG#AZ7^U(sd5M^r=b(C2?%gwR~k1a+^&{DJvEk`TR
zO0){CMn%n#ts}GS$j!0_kF7=P(0a51ZA6>UX0!!uMcEof9c7m7xmmW!V`vBZ41JEi
zKwqM-(AVf2RMZUFIx@?)+$`VXvG36L=m+#8`U(AvenG#Y-%z$jQAe3&Yi^d`@z@{e
zPxKf18~uaiC0SWi4()u=SI}#;?4Lzu*^--O7wlC5RYaB0u4p&3JK6*7iS|O-8buvt
zmd&|Y_QpQ@pncJPXn#~09e@r*2cd&eQ8Q%g$Sj+3vmAnb4n>Ed!_g7wNOTlB8Xbd<
zMcEof9c7k{xmk|GKF6aI(23|IbTT>xor+FFr=y}~$kvfrHsoeG1N)qbs-Uw_gkq$S
zMhP+~TcfC>%(6Z=i^V<;d6c5F(K+Z`bRIe%U4V+3AzMdgS(lsTLhMr&U4$-1m!M10
zW$1Er1-cStYZP^qS=Q!exeEJSjjlo0qH5?mbUnHO-H2{NMa__{BeSf@&2lsLxdq*d
zZbP@DJJ6lzE_64#2W4v%b(C3F=VrMV``m}_N7YdcR1?)gwNV}P04i#RY#o_pRc@BL
z*ylm?5PBHZL-kPu)DSg7kDzRgqK-1l%G@lCu}>4!6g5MSqUPu^^f-C~J&B5%AzMdg
zS&^INDeUt!dImjqHK+#jxx*Q+${aDPk%H34Mc;`
zU^D~`MZ?f=RMZUFIx@?m+$1jYeb8STqieM-$LQl&w+JQD#|~n`ILAnT)2O
zsc0ISj%J{lXcn4{ikcx?M`l@&n`I96nTzJ3`Dg)Jh!&y6XbD=1vNei2$}ID9vn<0t
z%h3w760Jh3(HgWCtwZZkQ8Q%g$Sm`6vuwaV8__1T8ErvZ(KfUl?LeQQY>lFhGRxfD
zET3baFVL6hEA%z`27QaZL*JtxP*F2v>&Pr~aFWT~Gy75miD(&5*4lv&_oPvMcu44egHhKzpLS(B5bt
zv@hBZWos05lv!rxX4xP6R7MA&1JOa~U~~vN6di^RM@7w$ts}F{$jx#D_Bj$Ag^otY
zpkvW-=y-GjIuT`S6m^tYrsrlk3HzLkPC=)l)6nVY40I-{g3dxk&5*4lvrNm)5@DYh
zDWp+?46?`}k5ZJaQPfdpnVOsBZ0vImIv1UX&PNxZ3sF^c5xN)^HAA+J%rYf6%O%+7
zQgj)*99@B~L|37!(KYB=l&w+JQD&K(o244|xei^AZa_Dpo6ybZ7IZ7R4HY#*wvNm)
zDL2dQ*yj#(C%OyWjqXABqWjSOs5;8lDC#J)Ow7$v1N+oOwNPzT2R(r5q6g7K=wVdU
z4B0v|%Y@u4^{`KU)BrU^jnE^gF=~REqGl*tqo||IGCnuUqu8f8dJH{|os&GH)dc^$oh-b8Pq
zx6wQ3UGyG$9~Cu2wvNm)CO69m*yls^5&9T?f<8s9P;1l%wME$)MIB|9(YaaLVW0M>
z1L}x6q0Xoa>WaFd?x?64vUOyZQMp-qV4t3-7wV1rpuVUd>W>DXfhb#}sH4m>GB?X0
z>@ye*K||3nG#rgUBhe@{8WlA|wvNm)A~(wz>@yaPL*vl|G!acglhG736=iD_b(C3#
z=VqCPeWs%sXeOG4W}`W1E}DntqoQWW){$9;wL&sMYzZAUxMXXtbE1^NDCH))nd*&0P1WtM@tS;}Fb@@N-S0aZkm(5`4Vv^&}Z6*WV)j?6M3H_M*bXD_rj
z+6V26_Cx!l%IE-eAj;M#>L|1H&&_fW_Bj|Gf(}K8p~KM;=ty)FIvN!}1la>%0;osG^x=c4mawnkA$nWcAbmh-XC1?WOl6jMpDWQ-=xTHgx)xPK*P-jt4JccqsH4o%GdIhP*ykp6Gr9%cif%)(7otBbU&(&YM`2^7OIW1HHtdQEZuXn)WJRvpt|Tm^bmR&
z)kF1B1Jn=|HAA+J%+f74OC#*_2x^R)pr)u9dK5KBkDVmqWZm2uzfqJ4|s5dHVhHM>~r9*C(KG>%(>WBKH0cao^ga)G_Xei3o
zDC#J)w9m~l4Eqd6BhW}R3XMi%&{#AMjYmbzkgX%Lw9Cyh0sBlulh9-|1x-cM&~!8d
z%|zK6MIB|9wz*kmVV~J(4w{SRq4{V5T8I{*#i*zmvUOyZHn~}rV4tOE8Cs53pp|G9
zT8-AAwJ2MosH4o%IycKY?6V$iKpW8}v>9zdThTVO9Thc0wvNmqpIFK{cP*eIwbc^$^A-lpOD<2BlpF~
z{U~xDh}?f6_dUq{2J$|?yuU8*Ys>q|@;orP`{n2C^7C!^xw8B`cRU)0#v=K7o&20kem*5X7m}aH$j?FK=MVC_UtVv^>s)z#
zDX;6~^_0Ahk=GCMyj`9z%kxxuekjk&P@THtlgk#?Z)!YB){9JVZ(aW@P%9>WTtuz
zYgVr(E6IsTzDagSQ#nl=Oi3zF7fSN-1fnD-Pbo^W@+6}qBTqX@O5}-1Nm`zo
zl%(XzN=dOieJLrDCp4LKDo=4r^75poBqvXUO0ug%nKhx3wW0L7P-=asctfaYV<44o
z3gtJ4a$7>#t)a}eP|5aCdPgYrS*ZB)P|+8G;`GZ<{;N>#>rnQaQ0CiE$#VPwSl}OI@GMylnlYIsyXZs9V&gK~s`67dnKU5j4?2^Il8o+0lB-tmGtA+f1
zJ2#E?2o>)cO6?U&?;R@HCzRPYl-)0s+dq`A94b8^kctir6(1Bz9UMv@5-K?~lsPPv
zJv@{hpHk3XtRC0VMb3!P4Vkmb~D1UOO^prqJbZV&hv{35w
zQ2LBe$(f-{l~DGqP%aAP<4~yzWFj3ZPC_XYO50G03uSyLn}%{{hw|ryO3w{sqw_+=
z=Z8`kgwhv=N~(r37lpDHhjN#M@|T86FALh`KQ`X+kt|c`5z##NJt+ICC0;uYB*7k
zASNS;W)#tmCb}_1KbDw|3zh{65T4IUrkKcgjCU5
zV!V#1))VyxVzQBFHWBS+qT53BTZ!qmkUH8m0I#)lEr;Y57|F*%ZGjw0HliS8JpKbDvt
z7cz~GC&nib)rmxX5-~ZMXig#8Q;F_0qCcIOo)J(-(V4`!3Q?U!)bdl6>`#u4l9*@|
z(Q2Ygh~5xW8#0a@G4@2267|`{RO_%Mog|Fn(K-72BN!>=x-vXH-}83TZr+kM0Fcc
z-%d>KAeuXg_Aa8ko9OQ$ruT+4(S5}Dexj;Q)HR4nO`?%^lS=Ja?{#I$Zm
z8$C#jA0n!UiMk#!sZTTwh_)foH6r>)h-u@HE^0!In-WzsqJET^G$)$Ji1u-!dxGen
zB&JV=^wHDA_!**lmZ)10ljn%$d7^!R=vor}i^TM$kZJTXF@A-pUM1?+h{@|j^9Iqr
zNpx=!{oBO!oq$Ocy-SSWBdYg_`U7I}A<=w9v>y}QCq(}#F>Mtxj#?ArHbm8ysM`^f
z_C(WxXgd;JC!+66OuK|sQCDKzji|a4bq`|FlW2MoZEvFML-c)#X}^#<>Q9UZ5Y<4U
z9z;wA6U`8!9ZGb=h<-RR9T75#MiS#uL^YbI#}JdTL^F*Oms_#ekn0s7Sct_
ziSY`eT1nKah{+G2Wf1_8{s#
ziOF6>vp3Q1Lv;HR{eHxB|ByPWOpFg8ssoAoAYyVb(HugwhZ5akM1MFjJtAZh9Z8Ij
zBC4Z_`WRwzEYTcCw8sT;sKf|y)MG*=Pr)kJp<(O*kUtA$LX>xl97M0EpE-$+buBAT0t
z_7(|$vs4KFVWsdboUc|bz)j0WE|Bb#II^1NlacOnwN<7Wukk9=wBtKuZ1+x
z>%{mCqI#34-y$Y&6U{qB`!3PFNA&L#(+@)0=tE-s5m9|i)SnQOPl=`#(Y7YKHbmc+
zn6?Y)qV~kN15tG(>Q2O@GtqP*+O9;`jp(}*(;gvx)RP$ZBC6g*-G`X;C7OOj+n?wL
z5dA=6Iw)is4JO7zh-xTN4~g
zJ5l{X)PE9_zli2HIWE<6VfV0#R2aCY6Y0SEAjG=yoUi
zJ&5U^A$_zLG2WY~_95zhiOGIMvp>;RCb|QN{y<`SP{=eom>3^IREHAvVZ`KcqB(+S
zk0iRIi2i6|dQ3ncMaL53irOrowr
zOwJ;jh-hP?lRN9PKkYvDnwTabRb+^S>~WhL}7{G%bksIih=>=wBeFEd!=e^dd2SiKt#C
z>Q{)#t3>k}(Y{V}ZxH>P#PqF@ar8DZeut>uCF=Kx$@@g}0nvU)bRQA@$HerLkSh9=
z7`Gy-)efh<1R$im8iQBlkP;*gJ^paT`!{VO-%cQ
zOrpNTxF1pVC+Y#jWFXNDBHF=3H-zYi64PNJO*EVsk07d%L_La_j3$~fL_3z~#u5E^
zVmcwDjV2P~Nklc7sHYH5`B>T1t$U5!G^{UO`M&63r^2T}^arh<+_GT^BNq))V6mM75EqHxZM~M6-ox
yw-VhpqTfzTp`E{W!C&Rm{dm`M=l|b${n|~di}5P1jC@by8nxuh3aB!YU;hJ)5|eQN

literal 0
HcmV?d00001

diff --git a/ceph/src/test/cli/crushtool/test-map-bobtail-tunables.t b/ceph/src/test/cli/crushtool/test-map-bobtail-tunables.t
new file mode 100644
index 00000000..7c38260f
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-bobtail-tunables.t
@@ -0,0 +1,10253 @@
+  $ crushtool -i "$TESTDIR/test-map-a.crushmap" --test --show-statistics --rule 0 --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 --set-chooseleaf-descend-once 1
+  crushtool successfully built or modified map.  Use '-o ' to write it out.
+  rule 0 (data), x = 0..1023, numrep = 1..10
+  CRUSH rule 0 x 0 [36]
+  CRUSH rule 0 x 1 [876]
+  CRUSH rule 0 x 2 [292]
+  CRUSH rule 0 x 3 [623]
+  CRUSH rule 0 x 4 [61]
+  CRUSH rule 0 x 5 [946]
+  CRUSH rule 0 x 6 [576]
+  CRUSH rule 0 x 7 [645]
+  CRUSH rule 0 x 8 [243]
+  CRUSH rule 0 x 9 [22]
+  CRUSH rule 0 x 10 [758]
+  CRUSH rule 0 x 11 [769]
+  CRUSH rule 0 x 12 [780]
+  CRUSH rule 0 x 13 [557]
+  CRUSH rule 0 x 14 [59]
+  CRUSH rule 0 x 15 [718]
+  CRUSH rule 0 x 16 [673]
+  CRUSH rule 0 x 17 [648]
+  CRUSH rule 0 x 18 [654]
+  CRUSH rule 0 x 19 [850]
+  CRUSH rule 0 x 20 [717]
+  CRUSH rule 0 x 21 [420]
+  CRUSH rule 0 x 22 [503]
+  CRUSH rule 0 x 23 [411]
+  CRUSH rule 0 x 24 [266]
+  CRUSH rule 0 x 25 [760]
+  CRUSH rule 0 x 26 [903]
+  CRUSH rule 0 x 27 [946]
+  CRUSH rule 0 x 28 [69]
+  CRUSH rule 0 x 29 [844]
+  CRUSH rule 0 x 30 [621]
+  CRUSH rule 0 x 31 [784]
+  CRUSH rule 0 x 32 [173]
+  CRUSH rule 0 x 33 [698]
+  CRUSH rule 0 x 34 [168]
+  CRUSH rule 0 x 35 [274]
+  CRUSH rule 0 x 36 [318]
+  CRUSH rule 0 x 37 [173]
+  CRUSH rule 0 x 38 [708]
+  CRUSH rule 0 x 39 [662]
+  CRUSH rule 0 x 40 [620]
+  CRUSH rule 0 x 41 [811]
+  CRUSH rule 0 x 42 [863]
+  CRUSH rule 0 x 43 [686]
+  CRUSH rule 0 x 44 [396]
+  CRUSH rule 0 x 45 [991]
+  CRUSH rule 0 x 46 [420]
+  CRUSH rule 0 x 47 [467]
+  CRUSH rule 0 x 48 [955]
+  CRUSH rule 0 x 49 [974]
+  CRUSH rule 0 x 50 [870]
+  CRUSH rule 0 x 51 [182]
+  CRUSH rule 0 x 52 [704]
+  CRUSH rule 0 x 53 [185]
+  CRUSH rule 0 x 54 [270]
+  CRUSH rule 0 x 55 [895]
+  CRUSH rule 0 x 56 [564]
+  CRUSH rule 0 x 57 [738]
+  CRUSH rule 0 x 58 [524]
+  CRUSH rule 0 x 59 [408]
+  CRUSH rule 0 x 60 [228]
+  CRUSH rule 0 x 61 [154]
+  CRUSH rule 0 x 62 [594]
+  CRUSH rule 0 x 63 [646]
+  CRUSH rule 0 x 64 [175]
+  CRUSH rule 0 x 65 [745]
+  CRUSH rule 0 x 66 [275]
+  CRUSH rule 0 x 67 [246]
+  CRUSH rule 0 x 68 [711]
+  CRUSH rule 0 x 69 [493]
+  CRUSH rule 0 x 70 [30]
+  CRUSH rule 0 x 71 [984]
+  CRUSH rule 0 x 72 [71]
+  CRUSH rule 0 x 73 [922]
+  CRUSH rule 0 x 74 [629]
+  CRUSH rule 0 x 75 [222]
+  CRUSH rule 0 x 76 [262]
+  CRUSH rule 0 x 77 [638]
+  CRUSH rule 0 x 78 [324]
+  CRUSH rule 0 x 79 [577]
+  CRUSH rule 0 x 80 [501]
+  CRUSH rule 0 x 81 [506]
+  CRUSH rule 0 x 82 [222]
+  CRUSH rule 0 x 83 [71]
+  CRUSH rule 0 x 84 [49]
+  CRUSH rule 0 x 85 [985]
+  CRUSH rule 0 x 86 [537]
+  CRUSH rule 0 x 87 [997]
+  CRUSH rule 0 x 88 [957]
+  CRUSH rule 0 x 89 [399]
+  CRUSH rule 0 x 90 [943]
+  CRUSH rule 0 x 91 [22]
+  CRUSH rule 0 x 92 [532]
+  CRUSH rule 0 x 93 [218]
+  CRUSH rule 0 x 94 [181]
+  CRUSH rule 0 x 95 [343]
+  CRUSH rule 0 x 96 [861]
+  CRUSH rule 0 x 97 [459]
+  CRUSH rule 0 x 98 [327]
+  CRUSH rule 0 x 99 [974]
+  CRUSH rule 0 x 100 [32]
+  CRUSH rule 0 x 101 [142]
+  CRUSH rule 0 x 102 [172]
+  CRUSH rule 0 x 103 [630]
+  CRUSH rule 0 x 104 [758]
+  CRUSH rule 0 x 105 [843]
+  CRUSH rule 0 x 106 [28]
+  CRUSH rule 0 x 107 [74]
+  CRUSH rule 0 x 108 [875]
+  CRUSH rule 0 x 109 [411]
+  CRUSH rule 0 x 110 [440]
+  CRUSH rule 0 x 111 [405]
+  CRUSH rule 0 x 112 [143]
+  CRUSH rule 0 x 113 [153]
+  CRUSH rule 0 x 114 [804]
+  CRUSH rule 0 x 115 [588]
+  CRUSH rule 0 x 116 [327]
+  CRUSH rule 0 x 117 [95]
+  CRUSH rule 0 x 118 [80]
+  CRUSH rule 0 x 119 [386]
+  CRUSH rule 0 x 120 [366]
+  CRUSH rule 0 x 121 [129]
+  CRUSH rule 0 x 122 [873]
+  CRUSH rule 0 x 123 [533]
+  CRUSH rule 0 x 124 [461]
+  CRUSH rule 0 x 125 [342]
+  CRUSH rule 0 x 126 [819]
+  CRUSH rule 0 x 127 [437]
+  CRUSH rule 0 x 128 [679]
+  CRUSH rule 0 x 129 [380]
+  CRUSH rule 0 x 130 [992]
+  CRUSH rule 0 x 131 [469]
+  CRUSH rule 0 x 132 [571]
+  CRUSH rule 0 x 133 [964]
+  CRUSH rule 0 x 134 [999]
+  CRUSH rule 0 x 135 [634]
+  CRUSH rule 0 x 136 [114]
+  CRUSH rule 0 x 137 [839]
+  CRUSH rule 0 x 138 [967]
+  CRUSH rule 0 x 139 [308]
+  CRUSH rule 0 x 140 [764]
+  CRUSH rule 0 x 141 [423]
+  CRUSH rule 0 x 142 [252]
+  CRUSH rule 0 x 143 [33]
+  CRUSH rule 0 x 144 [472]
+  CRUSH rule 0 x 145 [242]
+  CRUSH rule 0 x 146 [290]
+  CRUSH rule 0 x 147 [447]
+  CRUSH rule 0 x 148 [212]
+  CRUSH rule 0 x 149 [9]
+  CRUSH rule 0 x 150 [166]
+  CRUSH rule 0 x 151 [811]
+  CRUSH rule 0 x 152 [449]
+  CRUSH rule 0 x 153 [523]
+  CRUSH rule 0 x 154 [208]
+  CRUSH rule 0 x 155 [569]
+  CRUSH rule 0 x 156 [488]
+  CRUSH rule 0 x 157 [140]
+  CRUSH rule 0 x 158 [786]
+  CRUSH rule 0 x 159 [134]
+  CRUSH rule 0 x 160 [690]
+  CRUSH rule 0 x 161 [324]
+  CRUSH rule 0 x 162 [748]
+  CRUSH rule 0 x 163 [575]
+  CRUSH rule 0 x 164 [314]
+  CRUSH rule 0 x 165 [116]
+  CRUSH rule 0 x 166 [352]
+  CRUSH rule 0 x 167 [27]
+  CRUSH rule 0 x 168 [953]
+  CRUSH rule 0 x 169 [912]
+  CRUSH rule 0 x 170 [421]
+  CRUSH rule 0 x 171 [488]
+  CRUSH rule 0 x 172 [366]
+  CRUSH rule 0 x 173 [863]
+  CRUSH rule 0 x 174 [263]
+  CRUSH rule 0 x 175 [875]
+  CRUSH rule 0 x 176 [745]
+  CRUSH rule 0 x 177 [128]
+  CRUSH rule 0 x 178 [155]
+  CRUSH rule 0 x 179 [593]
+  CRUSH rule 0 x 180 [154]
+  CRUSH rule 0 x 181 [289]
+  CRUSH rule 0 x 182 [730]
+  CRUSH rule 0 x 183 [639]
+  CRUSH rule 0 x 184 [704]
+  CRUSH rule 0 x 185 [97]
+  CRUSH rule 0 x 186 [26]
+  CRUSH rule 0 x 187 [649]
+  CRUSH rule 0 x 188 [682]
+  CRUSH rule 0 x 189 [325]
+  CRUSH rule 0 x 190 [399]
+  CRUSH rule 0 x 191 [629]
+  CRUSH rule 0 x 192 [503]
+  CRUSH rule 0 x 193 [546]
+  CRUSH rule 0 x 194 [242]
+  CRUSH rule 0 x 195 [625]
+  CRUSH rule 0 x 196 [357]
+  CRUSH rule 0 x 197 [306]
+  CRUSH rule 0 x 198 [863]
+  CRUSH rule 0 x 199 [935]
+  CRUSH rule 0 x 200 [373]
+  CRUSH rule 0 x 201 [659]
+  CRUSH rule 0 x 202 [260]
+  CRUSH rule 0 x 203 [36]
+  CRUSH rule 0 x 204 [92]
+  CRUSH rule 0 x 205 [68]
+  CRUSH rule 0 x 206 [570]
+  CRUSH rule 0 x 207 [834]
+  CRUSH rule 0 x 208 [927]
+  CRUSH rule 0 x 209 [878]
+  CRUSH rule 0 x 210 [572]
+  CRUSH rule 0 x 211 [107]
+  CRUSH rule 0 x 212 [389]
+  CRUSH rule 0 x 213 [497]
+  CRUSH rule 0 x 214 [798]
+  CRUSH rule 0 x 215 [233]
+  CRUSH rule 0 x 216 [494]
+  CRUSH rule 0 x 217 [352]
+  CRUSH rule 0 x 218 [895]
+  CRUSH rule 0 x 219 [222]
+  CRUSH rule 0 x 220 [281]
+  CRUSH rule 0 x 221 [64]
+  CRUSH rule 0 x 222 [40]
+  CRUSH rule 0 x 223 [645]
+  CRUSH rule 0 x 224 [647]
+  CRUSH rule 0 x 225 [219]
+  CRUSH rule 0 x 226 [372]
+  CRUSH rule 0 x 227 [925]
+  CRUSH rule 0 x 228 [682]
+  CRUSH rule 0 x 229 [880]
+  CRUSH rule 0 x 230 [328]
+  CRUSH rule 0 x 231 [320]
+  CRUSH rule 0 x 232 [924]
+  CRUSH rule 0 x 233 [948]
+  CRUSH rule 0 x 234 [484]
+  CRUSH rule 0 x 235 [750]
+  CRUSH rule 0 x 236 [551]
+  CRUSH rule 0 x 237 [390]
+  CRUSH rule 0 x 238 [570]
+  CRUSH rule 0 x 239 [729]
+  CRUSH rule 0 x 240 [981]
+  CRUSH rule 0 x 241 [310]
+  CRUSH rule 0 x 242 [161]
+  CRUSH rule 0 x 243 [180]
+  CRUSH rule 0 x 244 [52]
+  CRUSH rule 0 x 245 [523]
+  CRUSH rule 0 x 246 [362]
+  CRUSH rule 0 x 247 [382]
+  CRUSH rule 0 x 248 [129]
+  CRUSH rule 0 x 249 [159]
+  CRUSH rule 0 x 250 [404]
+  CRUSH rule 0 x 251 [661]
+  CRUSH rule 0 x 252 [961]
+  CRUSH rule 0 x 253 [651]
+  CRUSH rule 0 x 254 [123]
+  CRUSH rule 0 x 255 [314]
+  CRUSH rule 0 x 256 [315]
+  CRUSH rule 0 x 257 [825]
+  CRUSH rule 0 x 258 [624]
+  CRUSH rule 0 x 259 [602]
+  CRUSH rule 0 x 260 [717]
+  CRUSH rule 0 x 261 [145]
+  CRUSH rule 0 x 262 [223]
+  CRUSH rule 0 x 263 [462]
+  CRUSH rule 0 x 264 [654]
+  CRUSH rule 0 x 265 [302]
+  CRUSH rule 0 x 266 [202]
+  CRUSH rule 0 x 267 [282]
+  CRUSH rule 0 x 268 [338]
+  CRUSH rule 0 x 269 [738]
+  CRUSH rule 0 x 270 [707]
+  CRUSH rule 0 x 271 [705]
+  CRUSH rule 0 x 272 [756]
+  CRUSH rule 0 x 273 [197]
+  CRUSH rule 0 x 274 [992]
+  CRUSH rule 0 x 275 [544]
+  CRUSH rule 0 x 276 [658]
+  CRUSH rule 0 x 277 [143]
+  CRUSH rule 0 x 278 [492]
+  CRUSH rule 0 x 279 [517]
+  CRUSH rule 0 x 280 [825]
+  CRUSH rule 0 x 281 [224]
+  CRUSH rule 0 x 282 [298]
+  CRUSH rule 0 x 283 [311]
+  CRUSH rule 0 x 284 [771]
+  CRUSH rule 0 x 285 [693]
+  CRUSH rule 0 x 286 [364]
+  CRUSH rule 0 x 287 [591]
+  CRUSH rule 0 x 288 [965]
+  CRUSH rule 0 x 289 [225]
+  CRUSH rule 0 x 290 [577]
+  CRUSH rule 0 x 291 [160]
+  CRUSH rule 0 x 292 [873]
+  CRUSH rule 0 x 293 [100]
+  CRUSH rule 0 x 294 [285]
+  CRUSH rule 0 x 295 [938]
+  CRUSH rule 0 x 296 [850]
+  CRUSH rule 0 x 297 [951]
+  CRUSH rule 0 x 298 [173]
+  CRUSH rule 0 x 299 [598]
+  CRUSH rule 0 x 300 [531]
+  CRUSH rule 0 x 301 [823]
+  CRUSH rule 0 x 302 [184]
+  CRUSH rule 0 x 303 [521]
+  CRUSH rule 0 x 304 [980]
+  CRUSH rule 0 x 305 [153]
+  CRUSH rule 0 x 306 [423]
+  CRUSH rule 0 x 307 [997]
+  CRUSH rule 0 x 308 [991]
+  CRUSH rule 0 x 309 [860]
+  CRUSH rule 0 x 310 [589]
+  CRUSH rule 0 x 311 [477]
+  CRUSH rule 0 x 312 [887]
+  CRUSH rule 0 x 313 [802]
+  CRUSH rule 0 x 314 [654]
+  CRUSH rule 0 x 315 [767]
+  CRUSH rule 0 x 316 [778]
+  CRUSH rule 0 x 317 [184]
+  CRUSH rule 0 x 318 [525]
+  CRUSH rule 0 x 319 [476]
+  CRUSH rule 0 x 320 [149]
+  CRUSH rule 0 x 321 [710]
+  CRUSH rule 0 x 322 [175]
+  CRUSH rule 0 x 323 [819]
+  CRUSH rule 0 x 324 [16]
+  CRUSH rule 0 x 325 [486]
+  CRUSH rule 0 x 326 [613]
+  CRUSH rule 0 x 327 [125]
+  CRUSH rule 0 x 328 [807]
+  CRUSH rule 0 x 329 [588]
+  CRUSH rule 0 x 330 [932]
+  CRUSH rule 0 x 331 [341]
+  CRUSH rule 0 x 332 [153]
+  CRUSH rule 0 x 333 [745]
+  CRUSH rule 0 x 334 [614]
+  CRUSH rule 0 x 335 [518]
+  CRUSH rule 0 x 336 [389]
+  CRUSH rule 0 x 337 [753]
+  CRUSH rule 0 x 338 [128]
+  CRUSH rule 0 x 339 [430]
+  CRUSH rule 0 x 340 [541]
+  CRUSH rule 0 x 341 [402]
+  CRUSH rule 0 x 342 [982]
+  CRUSH rule 0 x 343 [833]
+  CRUSH rule 0 x 344 [784]
+  CRUSH rule 0 x 345 [546]
+  CRUSH rule 0 x 346 [302]
+  CRUSH rule 0 x 347 [488]
+  CRUSH rule 0 x 348 [903]
+  CRUSH rule 0 x 349 [471]
+  CRUSH rule 0 x 350 [348]
+  CRUSH rule 0 x 351 [961]
+  CRUSH rule 0 x 352 [728]
+  CRUSH rule 0 x 353 [904]
+  CRUSH rule 0 x 354 [345]
+  CRUSH rule 0 x 355 [50]
+  CRUSH rule 0 x 356 [87]
+  CRUSH rule 0 x 357 [762]
+  CRUSH rule 0 x 358 [908]
+  CRUSH rule 0 x 359 [484]
+  CRUSH rule 0 x 360 [173]
+  CRUSH rule 0 x 361 [404]
+  CRUSH rule 0 x 362 [403]
+  CRUSH rule 0 x 363 [639]
+  CRUSH rule 0 x 364 [752]
+  CRUSH rule 0 x 365 [956]
+  CRUSH rule 0 x 366 [860]
+  CRUSH rule 0 x 367 [205]
+  CRUSH rule 0 x 368 [301]
+  CRUSH rule 0 x 369 [452]
+  CRUSH rule 0 x 370 [11]
+  CRUSH rule 0 x 371 [124]
+  CRUSH rule 0 x 372 [253]
+  CRUSH rule 0 x 373 [715]
+  CRUSH rule 0 x 374 [191]
+  CRUSH rule 0 x 375 [711]
+  CRUSH rule 0 x 376 [597]
+  CRUSH rule 0 x 377 [294]
+  CRUSH rule 0 x 378 [34]
+  CRUSH rule 0 x 379 [869]
+  CRUSH rule 0 x 380 [294]
+  CRUSH rule 0 x 381 [119]
+  CRUSH rule 0 x 382 [69]
+  CRUSH rule 0 x 383 [922]
+  CRUSH rule 0 x 384 [221]
+  CRUSH rule 0 x 385 [561]
+  CRUSH rule 0 x 386 [335]
+  CRUSH rule 0 x 387 [514]
+  CRUSH rule 0 x 388 [587]
+  CRUSH rule 0 x 389 [109]
+  CRUSH rule 0 x 390 [925]
+  CRUSH rule 0 x 391 [267]
+  CRUSH rule 0 x 392 [382]
+  CRUSH rule 0 x 393 [425]
+  CRUSH rule 0 x 394 [898]
+  CRUSH rule 0 x 395 [806]
+  CRUSH rule 0 x 396 [790]
+  CRUSH rule 0 x 397 [136]
+  CRUSH rule 0 x 398 [914]
+  CRUSH rule 0 x 399 [261]
+  CRUSH rule 0 x 400 [661]
+  CRUSH rule 0 x 401 [953]
+  CRUSH rule 0 x 402 [738]
+  CRUSH rule 0 x 403 [573]
+  CRUSH rule 0 x 404 [526]
+  CRUSH rule 0 x 405 [582]
+  CRUSH rule 0 x 406 [768]
+  CRUSH rule 0 x 407 [260]
+  CRUSH rule 0 x 408 [657]
+  CRUSH rule 0 x 409 [498]
+  CRUSH rule 0 x 410 [28]
+  CRUSH rule 0 x 411 [684]
+  CRUSH rule 0 x 412 [261]
+  CRUSH rule 0 x 413 [891]
+  CRUSH rule 0 x 414 [127]
+  CRUSH rule 0 x 415 [272]
+  CRUSH rule 0 x 416 [739]
+  CRUSH rule 0 x 417 [106]
+  CRUSH rule 0 x 418 [525]
+  CRUSH rule 0 x 419 [603]
+  CRUSH rule 0 x 420 [988]
+  CRUSH rule 0 x 421 [761]
+  CRUSH rule 0 x 422 [317]
+  CRUSH rule 0 x 423 [137]
+  CRUSH rule 0 x 424 [920]
+  CRUSH rule 0 x 425 [277]
+  CRUSH rule 0 x 426 [485]
+  CRUSH rule 0 x 427 [242]
+  CRUSH rule 0 x 428 [632]
+  CRUSH rule 0 x 429 [641]
+  CRUSH rule 0 x 430 [626]
+  CRUSH rule 0 x 431 [697]
+  CRUSH rule 0 x 432 [590]
+  CRUSH rule 0 x 433 [284]
+  CRUSH rule 0 x 434 [538]
+  CRUSH rule 0 x 435 [30]
+  CRUSH rule 0 x 436 [164]
+  CRUSH rule 0 x 437 [322]
+  CRUSH rule 0 x 438 [142]
+  CRUSH rule 0 x 439 [119]
+  CRUSH rule 0 x 440 [333]
+  CRUSH rule 0 x 441 [477]
+  CRUSH rule 0 x 442 [274]
+  CRUSH rule 0 x 443 [983]
+  CRUSH rule 0 x 444 [536]
+  CRUSH rule 0 x 445 [485]
+  CRUSH rule 0 x 446 [345]
+  CRUSH rule 0 x 447 [61]
+  CRUSH rule 0 x 448 [333]
+  CRUSH rule 0 x 449 [680]
+  CRUSH rule 0 x 450 [235]
+  CRUSH rule 0 x 451 [961]
+  CRUSH rule 0 x 452 [525]
+  CRUSH rule 0 x 453 [138]
+  CRUSH rule 0 x 454 [137]
+  CRUSH rule 0 x 455 [173]
+  CRUSH rule 0 x 456 [235]
+  CRUSH rule 0 x 457 [450]
+  CRUSH rule 0 x 458 [195]
+  CRUSH rule 0 x 459 [381]
+  CRUSH rule 0 x 460 [972]
+  CRUSH rule 0 x 461 [506]
+  CRUSH rule 0 x 462 [692]
+  CRUSH rule 0 x 463 [788]
+  CRUSH rule 0 x 464 [133]
+  CRUSH rule 0 x 465 [971]
+  CRUSH rule 0 x 466 [394]
+  CRUSH rule 0 x 467 [517]
+  CRUSH rule 0 x 468 [829]
+  CRUSH rule 0 x 469 [987]
+  CRUSH rule 0 x 470 [107]
+  CRUSH rule 0 x 471 [181]
+  CRUSH rule 0 x 472 [547]
+  CRUSH rule 0 x 473 [760]
+  CRUSH rule 0 x 474 [787]
+  CRUSH rule 0 x 475 [662]
+  CRUSH rule 0 x 476 [110]
+  CRUSH rule 0 x 477 [393]
+  CRUSH rule 0 x 478 [246]
+  CRUSH rule 0 x 479 [70]
+  CRUSH rule 0 x 480 [753]
+  CRUSH rule 0 x 481 [470]
+  CRUSH rule 0 x 482 [451]
+  CRUSH rule 0 x 483 [816]
+  CRUSH rule 0 x 484 [540]
+  CRUSH rule 0 x 485 [74]
+  CRUSH rule 0 x 486 [958]
+  CRUSH rule 0 x 487 [228]
+  CRUSH rule 0 x 488 [180]
+  CRUSH rule 0 x 489 [47]
+  CRUSH rule 0 x 490 [905]
+  CRUSH rule 0 x 491 [892]
+  CRUSH rule 0 x 492 [588]
+  CRUSH rule 0 x 493 [353]
+  CRUSH rule 0 x 494 [378]
+  CRUSH rule 0 x 495 [845]
+  CRUSH rule 0 x 496 [13]
+  CRUSH rule 0 x 497 [796]
+  CRUSH rule 0 x 498 [412]
+  CRUSH rule 0 x 499 [330]
+  CRUSH rule 0 x 500 [820]
+  CRUSH rule 0 x 501 [110]
+  CRUSH rule 0 x 502 [336]
+  CRUSH rule 0 x 503 [922]
+  CRUSH rule 0 x 504 [483]
+  CRUSH rule 0 x 505 [482]
+  CRUSH rule 0 x 506 [493]
+  CRUSH rule 0 x 507 [12]
+  CRUSH rule 0 x 508 [227]
+  CRUSH rule 0 x 509 [807]
+  CRUSH rule 0 x 510 [134]
+  CRUSH rule 0 x 511 [212]
+  CRUSH rule 0 x 512 [236]
+  CRUSH rule 0 x 513 [994]
+  CRUSH rule 0 x 514 [45]
+  CRUSH rule 0 x 515 [504]
+  CRUSH rule 0 x 516 [285]
+  CRUSH rule 0 x 517 [300]
+  CRUSH rule 0 x 518 [397]
+  CRUSH rule 0 x 519 [86]
+  CRUSH rule 0 x 520 [900]
+  CRUSH rule 0 x 521 [31]
+  CRUSH rule 0 x 522 [390]
+  CRUSH rule 0 x 523 [618]
+  CRUSH rule 0 x 524 [635]
+  CRUSH rule 0 x 525 [311]
+  CRUSH rule 0 x 526 [48]
+  CRUSH rule 0 x 527 [202]
+  CRUSH rule 0 x 528 [565]
+  CRUSH rule 0 x 529 [934]
+  CRUSH rule 0 x 530 [502]
+  CRUSH rule 0 x 531 [681]
+  CRUSH rule 0 x 532 [422]
+  CRUSH rule 0 x 533 [863]
+  CRUSH rule 0 x 534 [962]
+  CRUSH rule 0 x 535 [89]
+  CRUSH rule 0 x 536 [499]
+  CRUSH rule 0 x 537 [676]
+  CRUSH rule 0 x 538 [58]
+  CRUSH rule 0 x 539 [837]
+  CRUSH rule 0 x 540 [831]
+  CRUSH rule 0 x 541 [582]
+  CRUSH rule 0 x 542 [472]
+  CRUSH rule 0 x 543 [382]
+  CRUSH rule 0 x 544 [947]
+  CRUSH rule 0 x 545 [425]
+  CRUSH rule 0 x 546 [18]
+  CRUSH rule 0 x 547 [445]
+  CRUSH rule 0 x 548 [367]
+  CRUSH rule 0 x 549 [125]
+  CRUSH rule 0 x 550 [425]
+  CRUSH rule 0 x 551 [44]
+  CRUSH rule 0 x 552 [246]
+  CRUSH rule 0 x 553 [71]
+  CRUSH rule 0 x 554 [207]
+  CRUSH rule 0 x 555 [570]
+  CRUSH rule 0 x 556 [674]
+  CRUSH rule 0 x 557 [347]
+  CRUSH rule 0 x 558 [627]
+  CRUSH rule 0 x 559 [940]
+  CRUSH rule 0 x 560 [295]
+  CRUSH rule 0 x 561 [506]
+  CRUSH rule 0 x 562 [718]
+  CRUSH rule 0 x 563 [552]
+  CRUSH rule 0 x 564 [835]
+  CRUSH rule 0 x 565 [8]
+  CRUSH rule 0 x 566 [600]
+  CRUSH rule 0 x 567 [999]
+  CRUSH rule 0 x 568 [252]
+  CRUSH rule 0 x 569 [643]
+  CRUSH rule 0 x 570 [617]
+  CRUSH rule 0 x 571 [757]
+  CRUSH rule 0 x 572 [299]
+  CRUSH rule 0 x 573 [25]
+  CRUSH rule 0 x 574 [215]
+  CRUSH rule 0 x 575 [225]
+  CRUSH rule 0 x 576 [627]
+  CRUSH rule 0 x 577 [237]
+  CRUSH rule 0 x 578 [885]
+  CRUSH rule 0 x 579 [924]
+  CRUSH rule 0 x 580 [718]
+  CRUSH rule 0 x 581 [219]
+  CRUSH rule 0 x 582 [893]
+  CRUSH rule 0 x 583 [246]
+  CRUSH rule 0 x 584 [336]
+  CRUSH rule 0 x 585 [324]
+  CRUSH rule 0 x 586 [558]
+  CRUSH rule 0 x 587 [985]
+  CRUSH rule 0 x 588 [211]
+  CRUSH rule 0 x 589 [129]
+  CRUSH rule 0 x 590 [467]
+  CRUSH rule 0 x 591 [758]
+  CRUSH rule 0 x 592 [525]
+  CRUSH rule 0 x 593 [601]
+  CRUSH rule 0 x 594 [227]
+  CRUSH rule 0 x 595 [720]
+  CRUSH rule 0 x 596 [751]
+  CRUSH rule 0 x 597 [129]
+  CRUSH rule 0 x 598 [679]
+  CRUSH rule 0 x 599 [668]
+  CRUSH rule 0 x 600 [143]
+  CRUSH rule 0 x 601 [326]
+  CRUSH rule 0 x 602 [860]
+  CRUSH rule 0 x 603 [709]
+  CRUSH rule 0 x 604 [571]
+  CRUSH rule 0 x 605 [252]
+  CRUSH rule 0 x 606 [339]
+  CRUSH rule 0 x 607 [590]
+  CRUSH rule 0 x 608 [145]
+  CRUSH rule 0 x 609 [973]
+  CRUSH rule 0 x 610 [435]
+  CRUSH rule 0 x 611 [559]
+  CRUSH rule 0 x 612 [273]
+  CRUSH rule 0 x 613 [828]
+  CRUSH rule 0 x 614 [478]
+  CRUSH rule 0 x 615 [392]
+  CRUSH rule 0 x 616 [778]
+  CRUSH rule 0 x 617 [622]
+  CRUSH rule 0 x 618 [149]
+  CRUSH rule 0 x 619 [604]
+  CRUSH rule 0 x 620 [181]
+  CRUSH rule 0 x 621 [735]
+  CRUSH rule 0 x 622 [661]
+  CRUSH rule 0 x 623 [142]
+  CRUSH rule 0 x 624 [360]
+  CRUSH rule 0 x 625 [541]
+  CRUSH rule 0 x 626 [364]
+  CRUSH rule 0 x 627 [458]
+  CRUSH rule 0 x 628 [250]
+  CRUSH rule 0 x 629 [928]
+  CRUSH rule 0 x 630 [243]
+  CRUSH rule 0 x 631 [438]
+  CRUSH rule 0 x 632 [797]
+  CRUSH rule 0 x 633 [993]
+  CRUSH rule 0 x 634 [239]
+  CRUSH rule 0 x 635 [640]
+  CRUSH rule 0 x 636 [173]
+  CRUSH rule 0 x 637 [0]
+  CRUSH rule 0 x 638 [702]
+  CRUSH rule 0 x 639 [475]
+  CRUSH rule 0 x 640 [31]
+  CRUSH rule 0 x 641 [296]
+  CRUSH rule 0 x 642 [894]
+  CRUSH rule 0 x 643 [117]
+  CRUSH rule 0 x 644 [438]
+  CRUSH rule 0 x 645 [982]
+  CRUSH rule 0 x 646 [334]
+  CRUSH rule 0 x 647 [933]
+  CRUSH rule 0 x 648 [22]
+  CRUSH rule 0 x 649 [503]
+  CRUSH rule 0 x 650 [328]
+  CRUSH rule 0 x 651 [3]
+  CRUSH rule 0 x 652 [495]
+  CRUSH rule 0 x 653 [185]
+  CRUSH rule 0 x 654 [130]
+  CRUSH rule 0 x 655 [560]
+  CRUSH rule 0 x 656 [219]
+  CRUSH rule 0 x 657 [233]
+  CRUSH rule 0 x 658 [778]
+  CRUSH rule 0 x 659 [240]
+  CRUSH rule 0 x 660 [244]
+  CRUSH rule 0 x 661 [184]
+  CRUSH rule 0 x 662 [65]
+  CRUSH rule 0 x 663 [323]
+  CRUSH rule 0 x 664 [865]
+  CRUSH rule 0 x 665 [420]
+  CRUSH rule 0 x 666 [319]
+  CRUSH rule 0 x 667 [875]
+  CRUSH rule 0 x 668 [331]
+  CRUSH rule 0 x 669 [915]
+  CRUSH rule 0 x 670 [845]
+  CRUSH rule 0 x 671 [108]
+  CRUSH rule 0 x 672 [578]
+  CRUSH rule 0 x 673 [442]
+  CRUSH rule 0 x 674 [588]
+  CRUSH rule 0 x 675 [489]
+  CRUSH rule 0 x 676 [928]
+  CRUSH rule 0 x 677 [399]
+  CRUSH rule 0 x 678 [546]
+  CRUSH rule 0 x 679 [988]
+  CRUSH rule 0 x 680 [335]
+  CRUSH rule 0 x 681 [690]
+  CRUSH rule 0 x 682 [196]
+  CRUSH rule 0 x 683 [627]
+  CRUSH rule 0 x 684 [38]
+  CRUSH rule 0 x 685 [841]
+  CRUSH rule 0 x 686 [336]
+  CRUSH rule 0 x 687 [20]
+  CRUSH rule 0 x 688 [463]
+  CRUSH rule 0 x 689 [569]
+  CRUSH rule 0 x 690 [551]
+  CRUSH rule 0 x 691 [766]
+  CRUSH rule 0 x 692 [739]
+  CRUSH rule 0 x 693 [339]
+  CRUSH rule 0 x 694 [405]
+  CRUSH rule 0 x 695 [622]
+  CRUSH rule 0 x 696 [558]
+  CRUSH rule 0 x 697 [818]
+  CRUSH rule 0 x 698 [178]
+  CRUSH rule 0 x 699 [450]
+  CRUSH rule 0 x 700 [502]
+  CRUSH rule 0 x 701 [4]
+  CRUSH rule 0 x 702 [177]
+  CRUSH rule 0 x 703 [354]
+  CRUSH rule 0 x 704 [646]
+  CRUSH rule 0 x 705 [921]
+  CRUSH rule 0 x 706 [652]
+  CRUSH rule 0 x 707 [345]
+  CRUSH rule 0 x 708 [333]
+  CRUSH rule 0 x 709 [45]
+  CRUSH rule 0 x 710 [94]
+  CRUSH rule 0 x 711 [227]
+  CRUSH rule 0 x 712 [398]
+  CRUSH rule 0 x 713 [116]
+  CRUSH rule 0 x 714 [111]
+  CRUSH rule 0 x 715 [531]
+  CRUSH rule 0 x 716 [169]
+  CRUSH rule 0 x 717 [417]
+  CRUSH rule 0 x 718 [992]
+  CRUSH rule 0 x 719 [936]
+  CRUSH rule 0 x 720 [370]
+  CRUSH rule 0 x 721 [320]
+  CRUSH rule 0 x 722 [7]
+  CRUSH rule 0 x 723 [270]
+  CRUSH rule 0 x 724 [666]
+  CRUSH rule 0 x 725 [794]
+  CRUSH rule 0 x 726 [420]
+  CRUSH rule 0 x 727 [561]
+  CRUSH rule 0 x 728 [951]
+  CRUSH rule 0 x 729 [656]
+  CRUSH rule 0 x 730 [3]
+  CRUSH rule 0 x 731 [852]
+  CRUSH rule 0 x 732 [983]
+  CRUSH rule 0 x 733 [285]
+  CRUSH rule 0 x 734 [125]
+  CRUSH rule 0 x 735 [417]
+  CRUSH rule 0 x 736 [749]
+  CRUSH rule 0 x 737 [644]
+  CRUSH rule 0 x 738 [449]
+  CRUSH rule 0 x 739 [341]
+  CRUSH rule 0 x 740 [874]
+  CRUSH rule 0 x 741 [189]
+  CRUSH rule 0 x 742 [912]
+  CRUSH rule 0 x 743 [654]
+  CRUSH rule 0 x 744 [725]
+  CRUSH rule 0 x 745 [787]
+  CRUSH rule 0 x 746 [757]
+  CRUSH rule 0 x 747 [700]
+  CRUSH rule 0 x 748 [557]
+  CRUSH rule 0 x 749 [772]
+  CRUSH rule 0 x 750 [946]
+  CRUSH rule 0 x 751 [996]
+  CRUSH rule 0 x 752 [746]
+  CRUSH rule 0 x 753 [741]
+  CRUSH rule 0 x 754 [648]
+  CRUSH rule 0 x 755 [157]
+  CRUSH rule 0 x 756 [416]
+  CRUSH rule 0 x 757 [599]
+  CRUSH rule 0 x 758 [994]
+  CRUSH rule 0 x 759 [959]
+  CRUSH rule 0 x 760 [518]
+  CRUSH rule 0 x 761 [285]
+  CRUSH rule 0 x 762 [591]
+  CRUSH rule 0 x 763 [908]
+  CRUSH rule 0 x 764 [787]
+  CRUSH rule 0 x 765 [327]
+  CRUSH rule 0 x 766 [84]
+  CRUSH rule 0 x 767 [370]
+  CRUSH rule 0 x 768 [826]
+  CRUSH rule 0 x 769 [67]
+  CRUSH rule 0 x 770 [593]
+  CRUSH rule 0 x 771 [309]
+  CRUSH rule 0 x 772 [12]
+  CRUSH rule 0 x 773 [253]
+  CRUSH rule 0 x 774 [164]
+  CRUSH rule 0 x 775 [703]
+  CRUSH rule 0 x 776 [728]
+  CRUSH rule 0 x 777 [981]
+  CRUSH rule 0 x 778 [411]
+  CRUSH rule 0 x 779 [346]
+  CRUSH rule 0 x 780 [476]
+  CRUSH rule 0 x 781 [10]
+  CRUSH rule 0 x 782 [462]
+  CRUSH rule 0 x 783 [580]
+  CRUSH rule 0 x 784 [413]
+  CRUSH rule 0 x 785 [341]
+  CRUSH rule 0 x 786 [411]
+  CRUSH rule 0 x 787 [605]
+  CRUSH rule 0 x 788 [226]
+  CRUSH rule 0 x 789 [545]
+  CRUSH rule 0 x 790 [414]
+  CRUSH rule 0 x 791 [660]
+  CRUSH rule 0 x 792 [287]
+  CRUSH rule 0 x 793 [631]
+  CRUSH rule 0 x 794 [931]
+  CRUSH rule 0 x 795 [551]
+  CRUSH rule 0 x 796 [814]
+  CRUSH rule 0 x 797 [64]
+  CRUSH rule 0 x 798 [422]
+  CRUSH rule 0 x 799 [824]
+  CRUSH rule 0 x 800 [862]
+  CRUSH rule 0 x 801 [145]
+  CRUSH rule 0 x 802 [570]
+  CRUSH rule 0 x 803 [151]
+  CRUSH rule 0 x 804 [467]
+  CRUSH rule 0 x 805 [621]
+  CRUSH rule 0 x 806 [898]
+  CRUSH rule 0 x 807 [354]
+  CRUSH rule 0 x 808 [7]
+  CRUSH rule 0 x 809 [70]
+  CRUSH rule 0 x 810 [701]
+  CRUSH rule 0 x 811 [248]
+  CRUSH rule 0 x 812 [230]
+  CRUSH rule 0 x 813 [805]
+  CRUSH rule 0 x 814 [54]
+  CRUSH rule 0 x 815 [679]
+  CRUSH rule 0 x 816 [919]
+  CRUSH rule 0 x 817 [765]
+  CRUSH rule 0 x 818 [415]
+  CRUSH rule 0 x 819 [721]
+  CRUSH rule 0 x 820 [218]
+  CRUSH rule 0 x 821 [185]
+  CRUSH rule 0 x 822 [356]
+  CRUSH rule 0 x 823 [220]
+  CRUSH rule 0 x 824 [292]
+  CRUSH rule 0 x 825 [949]
+  CRUSH rule 0 x 826 [767]
+  CRUSH rule 0 x 827 [631]
+  CRUSH rule 0 x 828 [288]
+  CRUSH rule 0 x 829 [990]
+  CRUSH rule 0 x 830 [152]
+  CRUSH rule 0 x 831 [814]
+  CRUSH rule 0 x 832 [235]
+  CRUSH rule 0 x 833 [657]
+  CRUSH rule 0 x 834 [907]
+  CRUSH rule 0 x 835 [784]
+  CRUSH rule 0 x 836 [951]
+  CRUSH rule 0 x 837 [556]
+  CRUSH rule 0 x 838 [329]
+  CRUSH rule 0 x 839 [568]
+  CRUSH rule 0 x 840 [45]
+  CRUSH rule 0 x 841 [652]
+  CRUSH rule 0 x 842 [629]
+  CRUSH rule 0 x 843 [799]
+  CRUSH rule 0 x 844 [694]
+  CRUSH rule 0 x 845 [332]
+  CRUSH rule 0 x 846 [452]
+  CRUSH rule 0 x 847 [399]
+  CRUSH rule 0 x 848 [303]
+  CRUSH rule 0 x 849 [666]
+  CRUSH rule 0 x 850 [644]
+  CRUSH rule 0 x 851 [527]
+  CRUSH rule 0 x 852 [31]
+  CRUSH rule 0 x 853 [483]
+  CRUSH rule 0 x 854 [697]
+  CRUSH rule 0 x 855 [837]
+  CRUSH rule 0 x 856 [712]
+  CRUSH rule 0 x 857 [77]
+  CRUSH rule 0 x 858 [412]
+  CRUSH rule 0 x 859 [173]
+  CRUSH rule 0 x 860 [776]
+  CRUSH rule 0 x 861 [705]
+  CRUSH rule 0 x 862 [809]
+  CRUSH rule 0 x 863 [349]
+  CRUSH rule 0 x 864 [717]
+  CRUSH rule 0 x 865 [857]
+  CRUSH rule 0 x 866 [394]
+  CRUSH rule 0 x 867 [640]
+  CRUSH rule 0 x 868 [613]
+  CRUSH rule 0 x 869 [973]
+  CRUSH rule 0 x 870 [505]
+  CRUSH rule 0 x 871 [239]
+  CRUSH rule 0 x 872 [21]
+  CRUSH rule 0 x 873 [954]
+  CRUSH rule 0 x 874 [54]
+  CRUSH rule 0 x 875 [809]
+  CRUSH rule 0 x 876 [483]
+  CRUSH rule 0 x 877 [542]
+  CRUSH rule 0 x 878 [217]
+  CRUSH rule 0 x 879 [999]
+  CRUSH rule 0 x 880 [678]
+  CRUSH rule 0 x 881 [394]
+  CRUSH rule 0 x 882 [467]
+  CRUSH rule 0 x 883 [802]
+  CRUSH rule 0 x 884 [653]
+  CRUSH rule 0 x 885 [898]
+  CRUSH rule 0 x 886 [434]
+  CRUSH rule 0 x 887 [297]
+  CRUSH rule 0 x 888 [863]
+  CRUSH rule 0 x 889 [105]
+  CRUSH rule 0 x 890 [550]
+  CRUSH rule 0 x 891 [575]
+  CRUSH rule 0 x 892 [259]
+  CRUSH rule 0 x 893 [902]
+  CRUSH rule 0 x 894 [180]
+  CRUSH rule 0 x 895 [725]
+  CRUSH rule 0 x 896 [951]
+  CRUSH rule 0 x 897 [810]
+  CRUSH rule 0 x 898 [979]
+  CRUSH rule 0 x 899 [685]
+  CRUSH rule 0 x 900 [530]
+  CRUSH rule 0 x 901 [740]
+  CRUSH rule 0 x 902 [800]
+  CRUSH rule 0 x 903 [230]
+  CRUSH rule 0 x 904 [346]
+  CRUSH rule 0 x 905 [530]
+  CRUSH rule 0 x 906 [80]
+  CRUSH rule 0 x 907 [365]
+  CRUSH rule 0 x 908 [204]
+  CRUSH rule 0 x 909 [883]
+  CRUSH rule 0 x 910 [549]
+  CRUSH rule 0 x 911 [325]
+  CRUSH rule 0 x 912 [874]
+  CRUSH rule 0 x 913 [331]
+  CRUSH rule 0 x 914 [836]
+  CRUSH rule 0 x 915 [245]
+  CRUSH rule 0 x 916 [77]
+  CRUSH rule 0 x 917 [239]
+  CRUSH rule 0 x 918 [988]
+  CRUSH rule 0 x 919 [783]
+  CRUSH rule 0 x 920 [623]
+  CRUSH rule 0 x 921 [105]
+  CRUSH rule 0 x 922 [887]
+  CRUSH rule 0 x 923 [223]
+  CRUSH rule 0 x 924 [25]
+  CRUSH rule 0 x 925 [912]
+  CRUSH rule 0 x 926 [968]
+  CRUSH rule 0 x 927 [277]
+  CRUSH rule 0 x 928 [554]
+  CRUSH rule 0 x 929 [761]
+  CRUSH rule 0 x 930 [814]
+  CRUSH rule 0 x 931 [29]
+  CRUSH rule 0 x 932 [446]
+  CRUSH rule 0 x 933 [352]
+  CRUSH rule 0 x 934 [730]
+  CRUSH rule 0 x 935 [731]
+  CRUSH rule 0 x 936 [322]
+  CRUSH rule 0 x 937 [822]
+  CRUSH rule 0 x 938 [557]
+  CRUSH rule 0 x 939 [150]
+  CRUSH rule 0 x 940 [638]
+  CRUSH rule 0 x 941 [730]
+  CRUSH rule 0 x 942 [62]
+  CRUSH rule 0 x 943 [165]
+  CRUSH rule 0 x 944 [199]
+  CRUSH rule 0 x 945 [946]
+  CRUSH rule 0 x 946 [595]
+  CRUSH rule 0 x 947 [800]
+  CRUSH rule 0 x 948 [132]
+  CRUSH rule 0 x 949 [792]
+  CRUSH rule 0 x 950 [111]
+  CRUSH rule 0 x 951 [414]
+  CRUSH rule 0 x 952 [775]
+  CRUSH rule 0 x 953 [349]
+  CRUSH rule 0 x 954 [570]
+  CRUSH rule 0 x 955 [729]
+  CRUSH rule 0 x 956 [519]
+  CRUSH rule 0 x 957 [242]
+  CRUSH rule 0 x 958 [84]
+  CRUSH rule 0 x 959 [270]
+  CRUSH rule 0 x 960 [458]
+  CRUSH rule 0 x 961 [981]
+  CRUSH rule 0 x 962 [623]
+  CRUSH rule 0 x 963 [291]
+  CRUSH rule 0 x 964 [28]
+  CRUSH rule 0 x 965 [675]
+  CRUSH rule 0 x 966 [836]
+  CRUSH rule 0 x 967 [966]
+  CRUSH rule 0 x 968 [864]
+  CRUSH rule 0 x 969 [729]
+  CRUSH rule 0 x 970 [800]
+  CRUSH rule 0 x 971 [737]
+  CRUSH rule 0 x 972 [952]
+  CRUSH rule 0 x 973 [356]
+  CRUSH rule 0 x 974 [545]
+  CRUSH rule 0 x 975 [336]
+  CRUSH rule 0 x 976 [446]
+  CRUSH rule 0 x 977 [202]
+  CRUSH rule 0 x 978 [612]
+  CRUSH rule 0 x 979 [843]
+  CRUSH rule 0 x 980 [60]
+  CRUSH rule 0 x 981 [702]
+  CRUSH rule 0 x 982 [298]
+  CRUSH rule 0 x 983 [723]
+  CRUSH rule 0 x 984 [723]
+  CRUSH rule 0 x 985 [945]
+  CRUSH rule 0 x 986 [772]
+  CRUSH rule 0 x 987 [88]
+  CRUSH rule 0 x 988 [522]
+  CRUSH rule 0 x 989 [578]
+  CRUSH rule 0 x 990 [638]
+  CRUSH rule 0 x 991 [530]
+  CRUSH rule 0 x 992 [925]
+  CRUSH rule 0 x 993 [991]
+  CRUSH rule 0 x 994 [276]
+  CRUSH rule 0 x 995 [288]
+  CRUSH rule 0 x 996 [887]
+  CRUSH rule 0 x 997 [110]
+  CRUSH rule 0 x 998 [435]
+  CRUSH rule 0 x 999 [876]
+  CRUSH rule 0 x 1000 [178]
+  CRUSH rule 0 x 1001 [99]
+  CRUSH rule 0 x 1002 [515]
+  CRUSH rule 0 x 1003 [104]
+  CRUSH rule 0 x 1004 [269]
+  CRUSH rule 0 x 1005 [369]
+  CRUSH rule 0 x 1006 [40]
+  CRUSH rule 0 x 1007 [978]
+  CRUSH rule 0 x 1008 [965]
+  CRUSH rule 0 x 1009 [598]
+  CRUSH rule 0 x 1010 [767]
+  CRUSH rule 0 x 1011 [289]
+  CRUSH rule 0 x 1012 [128]
+  CRUSH rule 0 x 1013 [979]
+  CRUSH rule 0 x 1014 [979]
+  CRUSH rule 0 x 1015 [277]
+  CRUSH rule 0 x 1016 [262]
+  CRUSH rule 0 x 1017 [150]
+  CRUSH rule 0 x 1018 [555]
+  CRUSH rule 0 x 1019 [513]
+  CRUSH rule 0 x 1020 [158]
+  CRUSH rule 0 x 1021 [915]
+  CRUSH rule 0 x 1022 [967]
+  CRUSH rule 0 x 1023 [488]
+  rule 0 (data) num_rep 1 result size == 1:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705]
+  CRUSH rule 0 x 1 [876,250]
+  CRUSH rule 0 x 2 [292,832]
+  CRUSH rule 0 x 3 [623,387]
+  CRUSH rule 0 x 4 [61,334]
+  CRUSH rule 0 x 5 [946,557]
+  CRUSH rule 0 x 6 [576,668]
+  CRUSH rule 0 x 7 [645,753]
+  CRUSH rule 0 x 8 [243,6]
+  CRUSH rule 0 x 9 [22,578]
+  CRUSH rule 0 x 10 [758,828]
+  CRUSH rule 0 x 11 [769,120]
+  CRUSH rule 0 x 12 [780,364]
+  CRUSH rule 0 x 13 [557,18]
+  CRUSH rule 0 x 14 [59,561]
+  CRUSH rule 0 x 15 [718,928]
+  CRUSH rule 0 x 16 [673,632]
+  CRUSH rule 0 x 17 [648,43]
+  CRUSH rule 0 x 18 [654,219]
+  CRUSH rule 0 x 19 [850,545]
+  CRUSH rule 0 x 20 [717,785]
+  CRUSH rule 0 x 21 [420,57]
+  CRUSH rule 0 x 22 [503,998]
+  CRUSH rule 0 x 23 [411,663]
+  CRUSH rule 0 x 24 [266,861]
+  CRUSH rule 0 x 25 [760,483]
+  CRUSH rule 0 x 26 [903,24]
+  CRUSH rule 0 x 27 [946,188]
+  CRUSH rule 0 x 28 [69,312]
+  CRUSH rule 0 x 29 [844,883]
+  CRUSH rule 0 x 30 [621,18]
+  CRUSH rule 0 x 31 [784,943]
+  CRUSH rule 0 x 32 [173,374]
+  CRUSH rule 0 x 33 [698,336]
+  CRUSH rule 0 x 34 [168,836]
+  CRUSH rule 0 x 35 [274,509]
+  CRUSH rule 0 x 36 [318,215]
+  CRUSH rule 0 x 37 [173,604]
+  CRUSH rule 0 x 38 [708,444]
+  CRUSH rule 0 x 39 [662,198]
+  CRUSH rule 0 x 40 [620,801]
+  CRUSH rule 0 x 41 [811,264]
+  CRUSH rule 0 x 42 [863,179]
+  CRUSH rule 0 x 43 [686,822]
+  CRUSH rule 0 x 44 [396,222]
+  CRUSH rule 0 x 45 [991,694]
+  CRUSH rule 0 x 46 [420,909]
+  CRUSH rule 0 x 47 [467,211]
+  CRUSH rule 0 x 48 [955,329]
+  CRUSH rule 0 x 49 [974,891]
+  CRUSH rule 0 x 50 [870,441]
+  CRUSH rule 0 x 51 [182,930]
+  CRUSH rule 0 x 52 [704,812]
+  CRUSH rule 0 x 53 [185,713]
+  CRUSH rule 0 x 54 [270,441]
+  CRUSH rule 0 x 55 [895,734]
+  CRUSH rule 0 x 56 [564,963]
+  CRUSH rule 0 x 57 [738,130]
+  CRUSH rule 0 x 58 [524,113]
+  CRUSH rule 0 x 59 [408,337]
+  CRUSH rule 0 x 60 [228,790]
+  CRUSH rule 0 x 61 [154,843]
+  CRUSH rule 0 x 62 [594,811]
+  CRUSH rule 0 x 63 [646,67]
+  CRUSH rule 0 x 64 [175,542]
+  CRUSH rule 0 x 65 [745,619]
+  CRUSH rule 0 x 66 [275,468]
+  CRUSH rule 0 x 67 [246,958]
+  CRUSH rule 0 x 68 [711,473]
+  CRUSH rule 0 x 69 [493,924]
+  CRUSH rule 0 x 70 [30,499]
+  CRUSH rule 0 x 71 [984,883]
+  CRUSH rule 0 x 72 [71,286]
+  CRUSH rule 0 x 73 [922,618]
+  CRUSH rule 0 x 74 [629,414]
+  CRUSH rule 0 x 75 [222,20]
+  CRUSH rule 0 x 76 [262,366]
+  CRUSH rule 0 x 77 [638,469]
+  CRUSH rule 0 x 78 [324,511]
+  CRUSH rule 0 x 79 [577,990]
+  CRUSH rule 0 x 80 [501,95]
+  CRUSH rule 0 x 81 [506,812]
+  CRUSH rule 0 x 82 [222,145]
+  CRUSH rule 0 x 83 [71,634]
+  CRUSH rule 0 x 84 [49,761]
+  CRUSH rule 0 x 85 [985,896]
+  CRUSH rule 0 x 86 [537,745]
+  CRUSH rule 0 x 87 [997,317]
+  CRUSH rule 0 x 88 [957,350]
+  CRUSH rule 0 x 89 [399,730]
+  CRUSH rule 0 x 90 [943,706]
+  CRUSH rule 0 x 91 [22,368]
+  CRUSH rule 0 x 92 [532,424]
+  CRUSH rule 0 x 93 [218,489]
+  CRUSH rule 0 x 94 [181,96]
+  CRUSH rule 0 x 95 [343,957]
+  CRUSH rule 0 x 96 [861,270]
+  CRUSH rule 0 x 97 [459,706]
+  CRUSH rule 0 x 98 [327,867]
+  CRUSH rule 0 x 99 [974,133]
+  CRUSH rule 0 x 100 [32,445]
+  CRUSH rule 0 x 101 [142,90]
+  CRUSH rule 0 x 102 [172,129]
+  CRUSH rule 0 x 103 [630,47]
+  CRUSH rule 0 x 104 [758,133]
+  CRUSH rule 0 x 105 [843,604]
+  CRUSH rule 0 x 106 [28,681]
+  CRUSH rule 0 x 107 [74,320]
+  CRUSH rule 0 x 108 [875,593]
+  CRUSH rule 0 x 109 [411,985]
+  CRUSH rule 0 x 110 [440,774]
+  CRUSH rule 0 x 111 [405,742]
+  CRUSH rule 0 x 112 [143,181]
+  CRUSH rule 0 x 113 [153,846]
+  CRUSH rule 0 x 114 [804,892]
+  CRUSH rule 0 x 115 [588,508]
+  CRUSH rule 0 x 116 [327,148]
+  CRUSH rule 0 x 117 [95,594]
+  CRUSH rule 0 x 118 [80,957]
+  CRUSH rule 0 x 119 [386,932]
+  CRUSH rule 0 x 120 [366,312]
+  CRUSH rule 0 x 121 [129,154]
+  CRUSH rule 0 x 122 [873,1]
+  CRUSH rule 0 x 123 [533,415]
+  CRUSH rule 0 x 124 [461,691]
+  CRUSH rule 0 x 125 [342,599]
+  CRUSH rule 0 x 126 [819,781]
+  CRUSH rule 0 x 127 [437,893]
+  CRUSH rule 0 x 128 [679,994]
+  CRUSH rule 0 x 129 [380,685]
+  CRUSH rule 0 x 130 [992,52]
+  CRUSH rule 0 x 131 [469,90]
+  CRUSH rule 0 x 132 [571,250]
+  CRUSH rule 0 x 133 [964,728]
+  CRUSH rule 0 x 134 [999,19]
+  CRUSH rule 0 x 135 [634,101]
+  CRUSH rule 0 x 136 [114,889]
+  CRUSH rule 0 x 137 [839,8]
+  CRUSH rule 0 x 138 [967,949]
+  CRUSH rule 0 x 139 [308,711]
+  CRUSH rule 0 x 140 [764,936]
+  CRUSH rule 0 x 141 [423,302]
+  CRUSH rule 0 x 142 [252,821]
+  CRUSH rule 0 x 143 [33,808]
+  CRUSH rule 0 x 144 [472,88]
+  CRUSH rule 0 x 145 [242,208]
+  CRUSH rule 0 x 146 [290,70]
+  CRUSH rule 0 x 147 [447,352]
+  CRUSH rule 0 x 148 [212,644]
+  CRUSH rule 0 x 149 [9,775]
+  CRUSH rule 0 x 150 [166,456]
+  CRUSH rule 0 x 151 [811,875]
+  CRUSH rule 0 x 152 [449,617]
+  CRUSH rule 0 x 153 [523,537]
+  CRUSH rule 0 x 154 [208,559]
+  CRUSH rule 0 x 155 [569,325]
+  CRUSH rule 0 x 156 [488,121]
+  CRUSH rule 0 x 157 [140,723]
+  CRUSH rule 0 x 158 [786,451]
+  CRUSH rule 0 x 159 [134,664]
+  CRUSH rule 0 x 160 [690,112]
+  CRUSH rule 0 x 161 [324,912]
+  CRUSH rule 0 x 162 [748,567]
+  CRUSH rule 0 x 163 [575,499]
+  CRUSH rule 0 x 164 [314,489]
+  CRUSH rule 0 x 165 [116,209]
+  CRUSH rule 0 x 166 [352,706]
+  CRUSH rule 0 x 167 [27,743]
+  CRUSH rule 0 x 168 [953,898]
+  CRUSH rule 0 x 169 [912,147]
+  CRUSH rule 0 x 170 [421,515]
+  CRUSH rule 0 x 171 [488,584]
+  CRUSH rule 0 x 172 [366,443]
+  CRUSH rule 0 x 173 [863,291]
+  CRUSH rule 0 x 174 [263,555]
+  CRUSH rule 0 x 175 [875,961]
+  CRUSH rule 0 x 176 [745,83]
+  CRUSH rule 0 x 177 [128,244]
+  CRUSH rule 0 x 178 [155,41]
+  CRUSH rule 0 x 179 [593,833]
+  CRUSH rule 0 x 180 [154,734]
+  CRUSH rule 0 x 181 [289,675]
+  CRUSH rule 0 x 182 [730,931]
+  CRUSH rule 0 x 183 [639,237]
+  CRUSH rule 0 x 184 [704,312]
+  CRUSH rule 0 x 185 [97,100]
+  CRUSH rule 0 x 186 [26,665]
+  CRUSH rule 0 x 187 [649,14]
+  CRUSH rule 0 x 188 [682,695]
+  CRUSH rule 0 x 189 [325,693]
+  CRUSH rule 0 x 190 [399,933]
+  CRUSH rule 0 x 191 [629,533]
+  CRUSH rule 0 x 192 [503,578]
+  CRUSH rule 0 x 193 [546,333]
+  CRUSH rule 0 x 194 [242,473]
+  CRUSH rule 0 x 195 [625,719]
+  CRUSH rule 0 x 196 [357,114]
+  CRUSH rule 0 x 197 [306,954]
+  CRUSH rule 0 x 198 [863,791]
+  CRUSH rule 0 x 199 [935,906]
+  CRUSH rule 0 x 200 [373,774]
+  CRUSH rule 0 x 201 [659,320]
+  CRUSH rule 0 x 202 [260,433]
+  CRUSH rule 0 x 203 [36,239]
+  CRUSH rule 0 x 204 [92,516]
+  CRUSH rule 0 x 205 [68,395]
+  CRUSH rule 0 x 206 [570,530]
+  CRUSH rule 0 x 207 [834,457]
+  CRUSH rule 0 x 208 [927,484]
+  CRUSH rule 0 x 209 [878,66]
+  CRUSH rule 0 x 210 [572,981]
+  CRUSH rule 0 x 211 [107,597]
+  CRUSH rule 0 x 212 [389,107]
+  CRUSH rule 0 x 213 [497,717]
+  CRUSH rule 0 x 214 [798,65]
+  CRUSH rule 0 x 215 [233,419]
+  CRUSH rule 0 x 216 [494,464]
+  CRUSH rule 0 x 217 [352,396]
+  CRUSH rule 0 x 218 [895,864]
+  CRUSH rule 0 x 219 [222,534]
+  CRUSH rule 0 x 220 [281,19]
+  CRUSH rule 0 x 221 [64,928]
+  CRUSH rule 0 x 222 [40,544]
+  CRUSH rule 0 x 223 [645,556]
+  CRUSH rule 0 x 224 [647,165]
+  CRUSH rule 0 x 225 [219,714]
+  CRUSH rule 0 x 226 [372,511]
+  CRUSH rule 0 x 227 [925,156]
+  CRUSH rule 0 x 228 [682,404]
+  CRUSH rule 0 x 229 [880,838]
+  CRUSH rule 0 x 230 [328,659]
+  CRUSH rule 0 x 231 [320,383]
+  CRUSH rule 0 x 232 [924,846]
+  CRUSH rule 0 x 233 [948,652]
+  CRUSH rule 0 x 234 [484,943]
+  CRUSH rule 0 x 235 [750,65]
+  CRUSH rule 0 x 236 [551,787]
+  CRUSH rule 0 x 237 [390,157]
+  CRUSH rule 0 x 238 [570,6]
+  CRUSH rule 0 x 239 [729,959]
+  CRUSH rule 0 x 240 [981,241]
+  CRUSH rule 0 x 241 [310,816]
+  CRUSH rule 0 x 242 [161,63]
+  CRUSH rule 0 x 243 [180,394]
+  CRUSH rule 0 x 244 [52,174]
+  CRUSH rule 0 x 245 [523,121]
+  CRUSH rule 0 x 246 [362,893]
+  CRUSH rule 0 x 247 [382,184]
+  CRUSH rule 0 x 248 [129,114]
+  CRUSH rule 0 x 249 [159,683]
+  CRUSH rule 0 x 250 [404,945]
+  CRUSH rule 0 x 251 [661,225]
+  CRUSH rule 0 x 252 [961,226]
+  CRUSH rule 0 x 253 [651,97]
+  CRUSH rule 0 x 254 [123,33]
+  CRUSH rule 0 x 255 [314,649]
+  CRUSH rule 0 x 256 [315,215]
+  CRUSH rule 0 x 257 [825,264]
+  CRUSH rule 0 x 258 [624,789]
+  CRUSH rule 0 x 259 [602,542]
+  CRUSH rule 0 x 260 [717,878]
+  CRUSH rule 0 x 261 [145,517]
+  CRUSH rule 0 x 262 [223,1]
+  CRUSH rule 0 x 263 [462,211]
+  CRUSH rule 0 x 264 [654,471]
+  CRUSH rule 0 x 265 [302,794]
+  CRUSH rule 0 x 266 [202,132]
+  CRUSH rule 0 x 267 [282,938]
+  CRUSH rule 0 x 268 [338,309]
+  CRUSH rule 0 x 269 [738,122]
+  CRUSH rule 0 x 270 [707,982]
+  CRUSH rule 0 x 271 [705,432]
+  CRUSH rule 0 x 272 [756,545]
+  CRUSH rule 0 x 273 [197,502]
+  CRUSH rule 0 x 274 [992,44]
+  CRUSH rule 0 x 275 [544,789]
+  CRUSH rule 0 x 276 [658,467]
+  CRUSH rule 0 x 277 [143,490]
+  CRUSH rule 0 x 278 [492,647]
+  CRUSH rule 0 x 279 [517,792]
+  CRUSH rule 0 x 280 [825,740]
+  CRUSH rule 0 x 281 [224,629]
+  CRUSH rule 0 x 282 [298,661]
+  CRUSH rule 0 x 283 [311,606]
+  CRUSH rule 0 x 284 [771,466]
+  CRUSH rule 0 x 285 [693,362]
+  CRUSH rule 0 x 286 [364,477]
+  CRUSH rule 0 x 287 [591,611]
+  CRUSH rule 0 x 288 [965,541]
+  CRUSH rule 0 x 289 [225,551]
+  CRUSH rule 0 x 290 [577,762]
+  CRUSH rule 0 x 291 [160,903]
+  CRUSH rule 0 x 292 [873,598]
+  CRUSH rule 0 x 293 [100,234]
+  CRUSH rule 0 x 294 [285,943]
+  CRUSH rule 0 x 295 [938,262]
+  CRUSH rule 0 x 296 [850,327]
+  CRUSH rule 0 x 297 [951,53]
+  CRUSH rule 0 x 298 [173,336]
+  CRUSH rule 0 x 299 [598,591]
+  CRUSH rule 0 x 300 [531,957]
+  CRUSH rule 0 x 301 [823,628]
+  CRUSH rule 0 x 302 [184,80]
+  CRUSH rule 0 x 303 [521,766]
+  CRUSH rule 0 x 304 [980,127]
+  CRUSH rule 0 x 305 [153,816]
+  CRUSH rule 0 x 306 [423,739]
+  CRUSH rule 0 x 307 [997,557]
+  CRUSH rule 0 x 308 [991,874]
+  CRUSH rule 0 x 309 [860,394]
+  CRUSH rule 0 x 310 [589,818]
+  CRUSH rule 0 x 311 [477,774]
+  CRUSH rule 0 x 312 [887,853]
+  CRUSH rule 0 x 313 [802,646]
+  CRUSH rule 0 x 314 [654,974]
+  CRUSH rule 0 x 315 [767,227]
+  CRUSH rule 0 x 316 [778,83]
+  CRUSH rule 0 x 317 [184,418]
+  CRUSH rule 0 x 318 [525,410]
+  CRUSH rule 0 x 319 [476,724]
+  CRUSH rule 0 x 320 [149,610]
+  CRUSH rule 0 x 321 [710,79]
+  CRUSH rule 0 x 322 [175,275]
+  CRUSH rule 0 x 323 [819,604]
+  CRUSH rule 0 x 324 [16,745]
+  CRUSH rule 0 x 325 [486,400]
+  CRUSH rule 0 x 326 [613,765]
+  CRUSH rule 0 x 327 [125,289]
+  CRUSH rule 0 x 328 [807,383]
+  CRUSH rule 0 x 329 [588,938]
+  CRUSH rule 0 x 330 [932,644]
+  CRUSH rule 0 x 331 [341,953]
+  CRUSH rule 0 x 332 [153,726]
+  CRUSH rule 0 x 333 [745,845]
+  CRUSH rule 0 x 334 [614,751]
+  CRUSH rule 0 x 335 [518,721]
+  CRUSH rule 0 x 336 [389,424]
+  CRUSH rule 0 x 337 [753,508]
+  CRUSH rule 0 x 338 [128,810]
+  CRUSH rule 0 x 339 [430,308]
+  CRUSH rule 0 x 340 [541,44]
+  CRUSH rule 0 x 341 [402,26]
+  CRUSH rule 0 x 342 [982,57]
+  CRUSH rule 0 x 343 [833,412]
+  CRUSH rule 0 x 344 [784,533]
+  CRUSH rule 0 x 345 [546,300]
+  CRUSH rule 0 x 346 [302,420]
+  CRUSH rule 0 x 347 [488,778]
+  CRUSH rule 0 x 348 [903,744]
+  CRUSH rule 0 x 349 [471,547]
+  CRUSH rule 0 x 350 [348,221]
+  CRUSH rule 0 x 351 [961,582]
+  CRUSH rule 0 x 352 [728,137]
+  CRUSH rule 0 x 353 [904,202]
+  CRUSH rule 0 x 354 [345,226]
+  CRUSH rule 0 x 355 [50,430]
+  CRUSH rule 0 x 356 [87,185]
+  CRUSH rule 0 x 357 [762,459]
+  CRUSH rule 0 x 358 [908,25]
+  CRUSH rule 0 x 359 [484,15]
+  CRUSH rule 0 x 360 [173,378]
+  CRUSH rule 0 x 361 [404,577]
+  CRUSH rule 0 x 362 [403,1]
+  CRUSH rule 0 x 363 [639,911]
+  CRUSH rule 0 x 364 [752,689]
+  CRUSH rule 0 x 365 [956,999]
+  CRUSH rule 0 x 366 [860,925]
+  CRUSH rule 0 x 367 [205,609]
+  CRUSH rule 0 x 368 [301,284]
+  CRUSH rule 0 x 369 [452,658]
+  CRUSH rule 0 x 370 [11,467]
+  CRUSH rule 0 x 371 [124,487]
+  CRUSH rule 0 x 372 [253,48]
+  CRUSH rule 0 x 373 [715,605]
+  CRUSH rule 0 x 374 [191,887]
+  CRUSH rule 0 x 375 [711,385]
+  CRUSH rule 0 x 376 [597,818]
+  CRUSH rule 0 x 377 [294,256]
+  CRUSH rule 0 x 378 [34,151]
+  CRUSH rule 0 x 379 [869,136]
+  CRUSH rule 0 x 380 [294,97]
+  CRUSH rule 0 x 381 [119,710]
+  CRUSH rule 0 x 382 [69,631]
+  CRUSH rule 0 x 383 [922,588]
+  CRUSH rule 0 x 384 [221,945]
+  CRUSH rule 0 x 385 [561,737]
+  CRUSH rule 0 x 386 [335,442]
+  CRUSH rule 0 x 387 [514,43]
+  CRUSH rule 0 x 388 [587,89]
+  CRUSH rule 0 x 389 [109,641]
+  CRUSH rule 0 x 390 [925,149]
+  CRUSH rule 0 x 391 [267,87]
+  CRUSH rule 0 x 392 [382,485]
+  CRUSH rule 0 x 393 [425,721]
+  CRUSH rule 0 x 394 [898,18]
+  CRUSH rule 0 x 395 [806,876]
+  CRUSH rule 0 x 396 [790,970]
+  CRUSH rule 0 x 397 [136,363]
+  CRUSH rule 0 x 398 [914,116]
+  CRUSH rule 0 x 399 [261,94]
+  CRUSH rule 0 x 400 [661,197]
+  CRUSH rule 0 x 401 [953,979]
+  CRUSH rule 0 x 402 [738,819]
+  CRUSH rule 0 x 403 [573,238]
+  CRUSH rule 0 x 404 [526,848]
+  CRUSH rule 0 x 405 [582,505]
+  CRUSH rule 0 x 406 [768,324]
+  CRUSH rule 0 x 407 [260,951]
+  CRUSH rule 0 x 408 [657,81]
+  CRUSH rule 0 x 409 [498,89]
+  CRUSH rule 0 x 410 [28,793]
+  CRUSH rule 0 x 411 [684,992]
+  CRUSH rule 0 x 412 [261,958]
+  CRUSH rule 0 x 413 [891,835]
+  CRUSH rule 0 x 414 [127,459]
+  CRUSH rule 0 x 415 [272,540]
+  CRUSH rule 0 x 416 [739,617]
+  CRUSH rule 0 x 417 [106,209]
+  CRUSH rule 0 x 418 [525,441]
+  CRUSH rule 0 x 419 [603,673]
+  CRUSH rule 0 x 420 [988,213]
+  CRUSH rule 0 x 421 [761,521]
+  CRUSH rule 0 x 422 [317,160]
+  CRUSH rule 0 x 423 [137,807]
+  CRUSH rule 0 x 424 [920,37]
+  CRUSH rule 0 x 425 [277,693]
+  CRUSH rule 0 x 426 [485,936]
+  CRUSH rule 0 x 427 [242,515]
+  CRUSH rule 0 x 428 [632,635]
+  CRUSH rule 0 x 429 [641,73]
+  CRUSH rule 0 x 430 [626,585]
+  CRUSH rule 0 x 431 [697,76]
+  CRUSH rule 0 x 432 [590,526]
+  CRUSH rule 0 x 433 [284,387]
+  CRUSH rule 0 x 434 [538,985]
+  CRUSH rule 0 x 435 [30,318]
+  CRUSH rule 0 x 436 [164,919]
+  CRUSH rule 0 x 437 [322,212]
+  CRUSH rule 0 x 438 [142,392]
+  CRUSH rule 0 x 439 [119,370]
+  CRUSH rule 0 x 440 [333,403]
+  CRUSH rule 0 x 441 [477,727]
+  CRUSH rule 0 x 442 [274,590]
+  CRUSH rule 0 x 443 [983,748]
+  CRUSH rule 0 x 444 [536,509]
+  CRUSH rule 0 x 445 [485,528]
+  CRUSH rule 0 x 446 [345,634]
+  CRUSH rule 0 x 447 [61,845]
+  CRUSH rule 0 x 448 [333,232]
+  CRUSH rule 0 x 449 [680,16]
+  CRUSH rule 0 x 450 [235,214]
+  CRUSH rule 0 x 451 [961,468]
+  CRUSH rule 0 x 452 [525,479]
+  CRUSH rule 0 x 453 [138,466]
+  CRUSH rule 0 x 454 [137,625]
+  CRUSH rule 0 x 455 [173,150]
+  CRUSH rule 0 x 456 [235,226]
+  CRUSH rule 0 x 457 [450,577]
+  CRUSH rule 0 x 458 [195,537]
+  CRUSH rule 0 x 459 [381,555]
+  CRUSH rule 0 x 460 [972,730]
+  CRUSH rule 0 x 461 [506,279]
+  CRUSH rule 0 x 462 [692,959]
+  CRUSH rule 0 x 463 [788,667]
+  CRUSH rule 0 x 464 [133,122]
+  CRUSH rule 0 x 465 [971,190]
+  CRUSH rule 0 x 466 [394,576]
+  CRUSH rule 0 x 467 [517,28]
+  CRUSH rule 0 x 468 [829,143]
+  CRUSH rule 0 x 469 [987,936]
+  CRUSH rule 0 x 470 [107,982]
+  CRUSH rule 0 x 471 [181,897]
+  CRUSH rule 0 x 472 [547,512]
+  CRUSH rule 0 x 473 [760,997]
+  CRUSH rule 0 x 474 [787,418]
+  CRUSH rule 0 x 475 [662,312]
+  CRUSH rule 0 x 476 [110,495]
+  CRUSH rule 0 x 477 [393,954]
+  CRUSH rule 0 x 478 [246,483]
+  CRUSH rule 0 x 479 [70,929]
+  CRUSH rule 0 x 480 [753,119]
+  CRUSH rule 0 x 481 [470,429]
+  CRUSH rule 0 x 482 [451,566]
+  CRUSH rule 0 x 483 [816,72]
+  CRUSH rule 0 x 484 [540,454]
+  CRUSH rule 0 x 485 [74,582]
+  CRUSH rule 0 x 486 [958,595]
+  CRUSH rule 0 x 487 [228,302]
+  CRUSH rule 0 x 488 [180,529]
+  CRUSH rule 0 x 489 [47,617]
+  CRUSH rule 0 x 490 [905,822]
+  CRUSH rule 0 x 491 [892,370]
+  CRUSH rule 0 x 492 [588,959]
+  CRUSH rule 0 x 493 [353,461]
+  CRUSH rule 0 x 494 [378,848]
+  CRUSH rule 0 x 495 [845,653]
+  CRUSH rule 0 x 496 [13,988]
+  CRUSH rule 0 x 497 [796,877]
+  CRUSH rule 0 x 498 [412,337]
+  CRUSH rule 0 x 499 [330,695]
+  CRUSH rule 0 x 500 [820,272]
+  CRUSH rule 0 x 501 [110,44]
+  CRUSH rule 0 x 502 [336,595]
+  CRUSH rule 0 x 503 [922,211]
+  CRUSH rule 0 x 504 [483,52]
+  CRUSH rule 0 x 505 [482,598]
+  CRUSH rule 0 x 506 [493,123]
+  CRUSH rule 0 x 507 [12,598]
+  CRUSH rule 0 x 508 [227,157]
+  CRUSH rule 0 x 509 [807,242]
+  CRUSH rule 0 x 510 [134,437]
+  CRUSH rule 0 x 511 [212,54]
+  CRUSH rule 0 x 512 [236,630]
+  CRUSH rule 0 x 513 [994,693]
+  CRUSH rule 0 x 514 [45,508]
+  CRUSH rule 0 x 515 [504,138]
+  CRUSH rule 0 x 516 [285,409]
+  CRUSH rule 0 x 517 [300,232]
+  CRUSH rule 0 x 518 [397,674]
+  CRUSH rule 0 x 519 [86,750]
+  CRUSH rule 0 x 520 [900,833]
+  CRUSH rule 0 x 521 [31,47]
+  CRUSH rule 0 x 522 [390,16]
+  CRUSH rule 0 x 523 [618,308]
+  CRUSH rule 0 x 524 [635,189]
+  CRUSH rule 0 x 525 [311,916]
+  CRUSH rule 0 x 526 [48,738]
+  CRUSH rule 0 x 527 [202,851]
+  CRUSH rule 0 x 528 [565,827]
+  CRUSH rule 0 x 529 [934,864]
+  CRUSH rule 0 x 530 [502,934]
+  CRUSH rule 0 x 531 [681,627]
+  CRUSH rule 0 x 532 [422,6]
+  CRUSH rule 0 x 533 [863,68]
+  CRUSH rule 0 x 534 [962,931]
+  CRUSH rule 0 x 535 [89,565]
+  CRUSH rule 0 x 536 [499,351]
+  CRUSH rule 0 x 537 [676,547]
+  CRUSH rule 0 x 538 [58,644]
+  CRUSH rule 0 x 539 [837,953]
+  CRUSH rule 0 x 540 [831,50]
+  CRUSH rule 0 x 541 [582,757]
+  CRUSH rule 0 x 542 [472,132]
+  CRUSH rule 0 x 543 [382,272]
+  CRUSH rule 0 x 544 [947,930]
+  CRUSH rule 0 x 545 [425,570]
+  CRUSH rule 0 x 546 [18,65]
+  CRUSH rule 0 x 547 [445,715]
+  CRUSH rule 0 x 548 [367,569]
+  CRUSH rule 0 x 549 [125,715]
+  CRUSH rule 0 x 550 [425,599]
+  CRUSH rule 0 x 551 [44,1]
+  CRUSH rule 0 x 552 [246,104]
+  CRUSH rule 0 x 553 [71,703]
+  CRUSH rule 0 x 554 [207,124]
+  CRUSH rule 0 x 555 [570,28]
+  CRUSH rule 0 x 556 [674,152]
+  CRUSH rule 0 x 557 [347,817]
+  CRUSH rule 0 x 558 [627,426]
+  CRUSH rule 0 x 559 [940,630]
+  CRUSH rule 0 x 560 [295,903]
+  CRUSH rule 0 x 561 [506,682]
+  CRUSH rule 0 x 562 [718,529]
+  CRUSH rule 0 x 563 [552,332]
+  CRUSH rule 0 x 564 [835,769]
+  CRUSH rule 0 x 565 [8,167]
+  CRUSH rule 0 x 566 [600,481]
+  CRUSH rule 0 x 567 [999,994]
+  CRUSH rule 0 x 568 [252,431]
+  CRUSH rule 0 x 569 [643,218]
+  CRUSH rule 0 x 570 [617,635]
+  CRUSH rule 0 x 571 [757,80]
+  CRUSH rule 0 x 572 [299,348]
+  CRUSH rule 0 x 573 [25,505]
+  CRUSH rule 0 x 574 [215,431]
+  CRUSH rule 0 x 575 [225,252]
+  CRUSH rule 0 x 576 [627,94]
+  CRUSH rule 0 x 577 [237,809]
+  CRUSH rule 0 x 578 [885,313]
+  CRUSH rule 0 x 579 [924,575]
+  CRUSH rule 0 x 580 [718,51]
+  CRUSH rule 0 x 581 [219,807]
+  CRUSH rule 0 x 582 [893,701]
+  CRUSH rule 0 x 583 [246,930]
+  CRUSH rule 0 x 584 [336,432]
+  CRUSH rule 0 x 585 [324,999]
+  CRUSH rule 0 x 586 [558,230]
+  CRUSH rule 0 x 587 [985,830]
+  CRUSH rule 0 x 588 [211,544]
+  CRUSH rule 0 x 589 [129,21]
+  CRUSH rule 0 x 590 [467,969]
+  CRUSH rule 0 x 591 [758,514]
+  CRUSH rule 0 x 592 [525,253]
+  CRUSH rule 0 x 593 [601,885]
+  CRUSH rule 0 x 594 [227,60]
+  CRUSH rule 0 x 595 [720,854]
+  CRUSH rule 0 x 596 [751,195]
+  CRUSH rule 0 x 597 [129,574]
+  CRUSH rule 0 x 598 [679,207]
+  CRUSH rule 0 x 599 [668,315]
+  CRUSH rule 0 x 600 [143,396]
+  CRUSH rule 0 x 601 [326,573]
+  CRUSH rule 0 x 602 [860,281]
+  CRUSH rule 0 x 603 [709,328]
+  CRUSH rule 0 x 604 [571,62]
+  CRUSH rule 0 x 605 [252,739]
+  CRUSH rule 0 x 606 [339,236]
+  CRUSH rule 0 x 607 [590,248]
+  CRUSH rule 0 x 608 [145,635]
+  CRUSH rule 0 x 609 [973,547]
+  CRUSH rule 0 x 610 [435,816]
+  CRUSH rule 0 x 611 [559,283]
+  CRUSH rule 0 x 612 [273,149]
+  CRUSH rule 0 x 613 [828,614]
+  CRUSH rule 0 x 614 [478,748]
+  CRUSH rule 0 x 615 [392,155]
+  CRUSH rule 0 x 616 [778,637]
+  CRUSH rule 0 x 617 [622,713]
+  CRUSH rule 0 x 618 [149,877]
+  CRUSH rule 0 x 619 [604,163]
+  CRUSH rule 0 x 620 [181,23]
+  CRUSH rule 0 x 621 [735,902]
+  CRUSH rule 0 x 622 [661,824]
+  CRUSH rule 0 x 623 [142,121]
+  CRUSH rule 0 x 624 [360,716]
+  CRUSH rule 0 x 625 [541,167]
+  CRUSH rule 0 x 626 [364,431]
+  CRUSH rule 0 x 627 [458,137]
+  CRUSH rule 0 x 628 [250,350]
+  CRUSH rule 0 x 629 [928,160]
+  CRUSH rule 0 x 630 [243,19]
+  CRUSH rule 0 x 631 [438,221]
+  CRUSH rule 0 x 632 [797,368]
+  CRUSH rule 0 x 633 [993,749]
+  CRUSH rule 0 x 634 [239,351]
+  CRUSH rule 0 x 635 [640,965]
+  CRUSH rule 0 x 636 [173,290]
+  CRUSH rule 0 x 637 [0,918]
+  CRUSH rule 0 x 638 [702,235]
+  CRUSH rule 0 x 639 [475,687]
+  CRUSH rule 0 x 640 [31,664]
+  CRUSH rule 0 x 641 [296,473]
+  CRUSH rule 0 x 642 [894,273]
+  CRUSH rule 0 x 643 [117,111]
+  CRUSH rule 0 x 644 [438,336]
+  CRUSH rule 0 x 645 [982,702]
+  CRUSH rule 0 x 646 [334,804]
+  CRUSH rule 0 x 647 [933,787]
+  CRUSH rule 0 x 648 [22,444]
+  CRUSH rule 0 x 649 [503,229]
+  CRUSH rule 0 x 650 [328,659]
+  CRUSH rule 0 x 651 [3,880]
+  CRUSH rule 0 x 652 [495,977]
+  CRUSH rule 0 x 653 [185,718]
+  CRUSH rule 0 x 654 [130,528]
+  CRUSH rule 0 x 655 [560,872]
+  CRUSH rule 0 x 656 [219,885]
+  CRUSH rule 0 x 657 [233,684]
+  CRUSH rule 0 x 658 [778,6]
+  CRUSH rule 0 x 659 [240,663]
+  CRUSH rule 0 x 660 [244,855]
+  CRUSH rule 0 x 661 [184,270]
+  CRUSH rule 0 x 662 [65,883]
+  CRUSH rule 0 x 663 [323,721]
+  CRUSH rule 0 x 664 [865,113]
+  CRUSH rule 0 x 665 [420,850]
+  CRUSH rule 0 x 666 [319,767]
+  CRUSH rule 0 x 667 [875,39]
+  CRUSH rule 0 x 668 [331,122]
+  CRUSH rule 0 x 669 [915,521]
+  CRUSH rule 0 x 670 [845,659]
+  CRUSH rule 0 x 671 [108,634]
+  CRUSH rule 0 x 672 [578,216]
+  CRUSH rule 0 x 673 [442,74]
+  CRUSH rule 0 x 674 [588,364]
+  CRUSH rule 0 x 675 [489,698]
+  CRUSH rule 0 x 676 [928,911]
+  CRUSH rule 0 x 677 [399,269]
+  CRUSH rule 0 x 678 [546,752]
+  CRUSH rule 0 x 679 [988,25]
+  CRUSH rule 0 x 680 [335,963]
+  CRUSH rule 0 x 681 [690,462]
+  CRUSH rule 0 x 682 [196,588]
+  CRUSH rule 0 x 683 [627,25]
+  CRUSH rule 0 x 684 [38,804]
+  CRUSH rule 0 x 685 [841,368]
+  CRUSH rule 0 x 686 [336,287]
+  CRUSH rule 0 x 687 [20,682]
+  CRUSH rule 0 x 688 [463,371]
+  CRUSH rule 0 x 689 [569,250]
+  CRUSH rule 0 x 690 [551,144]
+  CRUSH rule 0 x 691 [766,464]
+  CRUSH rule 0 x 692 [739,634]
+  CRUSH rule 0 x 693 [339,297]
+  CRUSH rule 0 x 694 [405,26]
+  CRUSH rule 0 x 695 [622,576]
+  CRUSH rule 0 x 696 [558,902]
+  CRUSH rule 0 x 697 [818,222]
+  CRUSH rule 0 x 698 [178,48]
+  CRUSH rule 0 x 699 [450,244]
+  CRUSH rule 0 x 700 [502,771]
+  CRUSH rule 0 x 701 [4,612]
+  CRUSH rule 0 x 702 [177,630]
+  CRUSH rule 0 x 703 [354,178]
+  CRUSH rule 0 x 704 [646,601]
+  CRUSH rule 0 x 705 [921,401]
+  CRUSH rule 0 x 706 [652,877]
+  CRUSH rule 0 x 707 [345,745]
+  CRUSH rule 0 x 708 [333,607]
+  CRUSH rule 0 x 709 [45,187]
+  CRUSH rule 0 x 710 [94,855]
+  CRUSH rule 0 x 711 [227,653]
+  CRUSH rule 0 x 712 [398,953]
+  CRUSH rule 0 x 713 [116,800]
+  CRUSH rule 0 x 714 [111,629]
+  CRUSH rule 0 x 715 [531,291]
+  CRUSH rule 0 x 716 [169,541]
+  CRUSH rule 0 x 717 [417,446]
+  CRUSH rule 0 x 718 [992,383]
+  CRUSH rule 0 x 719 [936,674]
+  CRUSH rule 0 x 720 [370,188]
+  CRUSH rule 0 x 721 [320,859]
+  CRUSH rule 0 x 722 [7,2]
+  CRUSH rule 0 x 723 [270,553]
+  CRUSH rule 0 x 724 [666,822]
+  CRUSH rule 0 x 725 [794,406]
+  CRUSH rule 0 x 726 [420,556]
+  CRUSH rule 0 x 727 [561,461]
+  CRUSH rule 0 x 728 [951,330]
+  CRUSH rule 0 x 729 [656,644]
+  CRUSH rule 0 x 730 [3,558]
+  CRUSH rule 0 x 731 [852,89]
+  CRUSH rule 0 x 732 [983,840]
+  CRUSH rule 0 x 733 [285,396]
+  CRUSH rule 0 x 734 [125,510]
+  CRUSH rule 0 x 735 [417,773]
+  CRUSH rule 0 x 736 [749,396]
+  CRUSH rule 0 x 737 [644,991]
+  CRUSH rule 0 x 738 [449,683]
+  CRUSH rule 0 x 739 [341,220]
+  CRUSH rule 0 x 740 [874,524]
+  CRUSH rule 0 x 741 [189,472]
+  CRUSH rule 0 x 742 [912,581]
+  CRUSH rule 0 x 743 [654,914]
+  CRUSH rule 0 x 744 [725,295]
+  CRUSH rule 0 x 745 [787,858]
+  CRUSH rule 0 x 746 [757,848]
+  CRUSH rule 0 x 747 [700,81]
+  CRUSH rule 0 x 748 [557,436]
+  CRUSH rule 0 x 749 [772,622]
+  CRUSH rule 0 x 750 [946,97]
+  CRUSH rule 0 x 751 [996,618]
+  CRUSH rule 0 x 752 [746,887]
+  CRUSH rule 0 x 753 [741,14]
+  CRUSH rule 0 x 754 [648,349]
+  CRUSH rule 0 x 755 [157,460]
+  CRUSH rule 0 x 756 [416,97]
+  CRUSH rule 0 x 757 [599,839]
+  CRUSH rule 0 x 758 [994,218]
+  CRUSH rule 0 x 759 [959,682]
+  CRUSH rule 0 x 760 [518,943]
+  CRUSH rule 0 x 761 [285,849]
+  CRUSH rule 0 x 762 [591,313]
+  CRUSH rule 0 x 763 [908,411]
+  CRUSH rule 0 x 764 [787,234]
+  CRUSH rule 0 x 765 [327,921]
+  CRUSH rule 0 x 766 [84,161]
+  CRUSH rule 0 x 767 [370,895]
+  CRUSH rule 0 x 768 [826,760]
+  CRUSH rule 0 x 769 [67,768]
+  CRUSH rule 0 x 770 [593,909]
+  CRUSH rule 0 x 771 [309,935]
+  CRUSH rule 0 x 772 [12,125]
+  CRUSH rule 0 x 773 [253,466]
+  CRUSH rule 0 x 774 [164,390]
+  CRUSH rule 0 x 775 [703,47]
+  CRUSH rule 0 x 776 [728,231]
+  CRUSH rule 0 x 777 [981,621]
+  CRUSH rule 0 x 778 [411,456]
+  CRUSH rule 0 x 779 [346,121]
+  CRUSH rule 0 x 780 [476,39]
+  CRUSH rule 0 x 781 [10,130]
+  CRUSH rule 0 x 782 [462,246]
+  CRUSH rule 0 x 783 [580,373]
+  CRUSH rule 0 x 784 [413,113]
+  CRUSH rule 0 x 785 [341,856]
+  CRUSH rule 0 x 786 [411,140]
+  CRUSH rule 0 x 787 [605,522]
+  CRUSH rule 0 x 788 [226,545]
+  CRUSH rule 0 x 789 [545,320]
+  CRUSH rule 0 x 790 [414,748]
+  CRUSH rule 0 x 791 [660,906]
+  CRUSH rule 0 x 792 [287,392]
+  CRUSH rule 0 x 793 [631,133]
+  CRUSH rule 0 x 794 [931,517]
+  CRUSH rule 0 x 795 [551,962]
+  CRUSH rule 0 x 796 [814,4]
+  CRUSH rule 0 x 797 [64,201]
+  CRUSH rule 0 x 798 [422,530]
+  CRUSH rule 0 x 799 [824,32]
+  CRUSH rule 0 x 800 [862,623]
+  CRUSH rule 0 x 801 [145,550]
+  CRUSH rule 0 x 802 [570,19]
+  CRUSH rule 0 x 803 [151,812]
+  CRUSH rule 0 x 804 [467,93]
+  CRUSH rule 0 x 805 [621,223]
+  CRUSH rule 0 x 806 [898,957]
+  CRUSH rule 0 x 807 [354,531]
+  CRUSH rule 0 x 808 [7,96]
+  CRUSH rule 0 x 809 [70,734]
+  CRUSH rule 0 x 810 [701,18]
+  CRUSH rule 0 x 811 [248,547]
+  CRUSH rule 0 x 812 [230,576]
+  CRUSH rule 0 x 813 [805,114]
+  CRUSH rule 0 x 814 [54,619]
+  CRUSH rule 0 x 815 [679,412]
+  CRUSH rule 0 x 816 [919,448]
+  CRUSH rule 0 x 817 [765,830]
+  CRUSH rule 0 x 818 [415,566]
+  CRUSH rule 0 x 819 [721,319]
+  CRUSH rule 0 x 820 [218,301]
+  CRUSH rule 0 x 821 [185,795]
+  CRUSH rule 0 x 822 [356,261]
+  CRUSH rule 0 x 823 [220,281]
+  CRUSH rule 0 x 824 [292,809]
+  CRUSH rule 0 x 825 [949,778]
+  CRUSH rule 0 x 826 [767,818]
+  CRUSH rule 0 x 827 [631,83]
+  CRUSH rule 0 x 828 [288,986]
+  CRUSH rule 0 x 829 [990,667]
+  CRUSH rule 0 x 830 [152,571]
+  CRUSH rule 0 x 831 [814,563]
+  CRUSH rule 0 x 832 [235,641]
+  CRUSH rule 0 x 833 [657,565]
+  CRUSH rule 0 x 834 [907,231]
+  CRUSH rule 0 x 835 [784,262]
+  CRUSH rule 0 x 836 [951,158]
+  CRUSH rule 0 x 837 [556,498]
+  CRUSH rule 0 x 838 [329,274]
+  CRUSH rule 0 x 839 [568,209]
+  CRUSH rule 0 x 840 [45,579]
+  CRUSH rule 0 x 841 [652,702]
+  CRUSH rule 0 x 842 [629,984]
+  CRUSH rule 0 x 843 [799,690]
+  CRUSH rule 0 x 844 [694,600]
+  CRUSH rule 0 x 845 [332,30]
+  CRUSH rule 0 x 846 [452,251]
+  CRUSH rule 0 x 847 [399,681]
+  CRUSH rule 0 x 848 [303,138]
+  CRUSH rule 0 x 849 [666,346]
+  CRUSH rule 0 x 850 [644,511]
+  CRUSH rule 0 x 851 [527,546]
+  CRUSH rule 0 x 852 [31,809]
+  CRUSH rule 0 x 853 [483,330]
+  CRUSH rule 0 x 854 [697,953]
+  CRUSH rule 0 x 855 [837,996]
+  CRUSH rule 0 x 856 [712,40]
+  CRUSH rule 0 x 857 [77,984]
+  CRUSH rule 0 x 858 [412,384]
+  CRUSH rule 0 x 859 [173,760]
+  CRUSH rule 0 x 860 [776,429]
+  CRUSH rule 0 x 861 [705,405]
+  CRUSH rule 0 x 862 [809,44]
+  CRUSH rule 0 x 863 [349,496]
+  CRUSH rule 0 x 864 [717,858]
+  CRUSH rule 0 x 865 [857,603]
+  CRUSH rule 0 x 866 [394,304]
+  CRUSH rule 0 x 867 [640,773]
+  CRUSH rule 0 x 868 [613,950]
+  CRUSH rule 0 x 869 [973,889]
+  CRUSH rule 0 x 870 [505,35]
+  CRUSH rule 0 x 871 [239,264]
+  CRUSH rule 0 x 872 [21,767]
+  CRUSH rule 0 x 873 [954,666]
+  CRUSH rule 0 x 874 [54,510]
+  CRUSH rule 0 x 875 [809,418]
+  CRUSH rule 0 x 876 [483,457]
+  CRUSH rule 0 x 877 [542,531]
+  CRUSH rule 0 x 878 [217,674]
+  CRUSH rule 0 x 879 [999,475]
+  CRUSH rule 0 x 880 [678,573]
+  CRUSH rule 0 x 881 [394,835]
+  CRUSH rule 0 x 882 [467,382]
+  CRUSH rule 0 x 883 [802,744]
+  CRUSH rule 0 x 884 [653,660]
+  CRUSH rule 0 x 885 [898,704]
+  CRUSH rule 0 x 886 [434,357]
+  CRUSH rule 0 x 887 [297,226]
+  CRUSH rule 0 x 888 [863,324]
+  CRUSH rule 0 x 889 [105,102]
+  CRUSH rule 0 x 890 [550,248]
+  CRUSH rule 0 x 891 [575,928]
+  CRUSH rule 0 x 892 [259,862]
+  CRUSH rule 0 x 893 [902,880]
+  CRUSH rule 0 x 894 [180,169]
+  CRUSH rule 0 x 895 [725,849]
+  CRUSH rule 0 x 896 [951,34]
+  CRUSH rule 0 x 897 [810,352]
+  CRUSH rule 0 x 898 [979,433]
+  CRUSH rule 0 x 899 [685,668]
+  CRUSH rule 0 x 900 [530,978]
+  CRUSH rule 0 x 901 [740,107]
+  CRUSH rule 0 x 902 [800,743]
+  CRUSH rule 0 x 903 [230,267]
+  CRUSH rule 0 x 904 [346,949]
+  CRUSH rule 0 x 905 [530,397]
+  CRUSH rule 0 x 906 [80,426]
+  CRUSH rule 0 x 907 [365,968]
+  CRUSH rule 0 x 908 [204,832]
+  CRUSH rule 0 x 909 [883,989]
+  CRUSH rule 0 x 910 [549,593]
+  CRUSH rule 0 x 911 [325,847]
+  CRUSH rule 0 x 912 [874,888]
+  CRUSH rule 0 x 913 [331,463]
+  CRUSH rule 0 x 914 [836,468]
+  CRUSH rule 0 x 915 [245,228]
+  CRUSH rule 0 x 916 [77,967]
+  CRUSH rule 0 x 917 [239,60]
+  CRUSH rule 0 x 918 [988,115]
+  CRUSH rule 0 x 919 [783,139]
+  CRUSH rule 0 x 920 [623,408]
+  CRUSH rule 0 x 921 [105,799]
+  CRUSH rule 0 x 922 [887,505]
+  CRUSH rule 0 x 923 [223,318]
+  CRUSH rule 0 x 924 [25,778]
+  CRUSH rule 0 x 925 [912,601]
+  CRUSH rule 0 x 926 [968,133]
+  CRUSH rule 0 x 927 [277,724]
+  CRUSH rule 0 x 928 [554,203]
+  CRUSH rule 0 x 929 [761,802]
+  CRUSH rule 0 x 930 [814,61]
+  CRUSH rule 0 x 931 [29,193]
+  CRUSH rule 0 x 932 [446,198]
+  CRUSH rule 0 x 933 [352,742]
+  CRUSH rule 0 x 934 [730,2]
+  CRUSH rule 0 x 935 [731,23]
+  CRUSH rule 0 x 936 [322,975]
+  CRUSH rule 0 x 937 [822,221]
+  CRUSH rule 0 x 938 [557,850]
+  CRUSH rule 0 x 939 [150,11]
+  CRUSH rule 0 x 940 [638,398]
+  CRUSH rule 0 x 941 [730,342]
+  CRUSH rule 0 x 942 [62,292]
+  CRUSH rule 0 x 943 [165,314]
+  CRUSH rule 0 x 944 [199,625]
+  CRUSH rule 0 x 945 [946,999]
+  CRUSH rule 0 x 946 [595,93]
+  CRUSH rule 0 x 947 [800,582]
+  CRUSH rule 0 x 948 [132,551]
+  CRUSH rule 0 x 949 [792,920]
+  CRUSH rule 0 x 950 [111,345]
+  CRUSH rule 0 x 951 [414,619]
+  CRUSH rule 0 x 952 [775,469]
+  CRUSH rule 0 x 953 [349,1]
+  CRUSH rule 0 x 954 [570,940]
+  CRUSH rule 0 x 955 [729,774]
+  CRUSH rule 0 x 956 [519,141]
+  CRUSH rule 0 x 957 [242,709]
+  CRUSH rule 0 x 958 [84,217]
+  CRUSH rule 0 x 959 [270,413]
+  CRUSH rule 0 x 960 [458,192]
+  CRUSH rule 0 x 961 [981,388]
+  CRUSH rule 0 x 962 [623,834]
+  CRUSH rule 0 x 963 [291,167]
+  CRUSH rule 0 x 964 [28,156]
+  CRUSH rule 0 x 965 [675,557]
+  CRUSH rule 0 x 966 [836,306]
+  CRUSH rule 0 x 967 [966,386]
+  CRUSH rule 0 x 968 [864,756]
+  CRUSH rule 0 x 969 [729,625]
+  CRUSH rule 0 x 970 [800,362]
+  CRUSH rule 0 x 971 [737,381]
+  CRUSH rule 0 x 972 [952,245]
+  CRUSH rule 0 x 973 [356,455]
+  CRUSH rule 0 x 974 [545,758]
+  CRUSH rule 0 x 975 [336,191]
+  CRUSH rule 0 x 976 [446,208]
+  CRUSH rule 0 x 977 [202,896]
+  CRUSH rule 0 x 978 [612,324]
+  CRUSH rule 0 x 979 [843,457]
+  CRUSH rule 0 x 980 [60,914]
+  CRUSH rule 0 x 981 [702,749]
+  CRUSH rule 0 x 982 [298,928]
+  CRUSH rule 0 x 983 [723,572]
+  CRUSH rule 0 x 984 [723,864]
+  CRUSH rule 0 x 985 [945,459]
+  CRUSH rule 0 x 986 [772,664]
+  CRUSH rule 0 x 987 [88,324]
+  CRUSH rule 0 x 988 [522,927]
+  CRUSH rule 0 x 989 [578,332]
+  CRUSH rule 0 x 990 [638,228]
+  CRUSH rule 0 x 991 [530,221]
+  CRUSH rule 0 x 992 [925,705]
+  CRUSH rule 0 x 993 [991,301]
+  CRUSH rule 0 x 994 [276,51]
+  CRUSH rule 0 x 995 [288,836]
+  CRUSH rule 0 x 996 [887,983]
+  CRUSH rule 0 x 997 [110,924]
+  CRUSH rule 0 x 998 [435,830]
+  CRUSH rule 0 x 999 [876,738]
+  CRUSH rule 0 x 1000 [178,963]
+  CRUSH rule 0 x 1001 [99,519]
+  CRUSH rule 0 x 1002 [515,534]
+  CRUSH rule 0 x 1003 [104,611]
+  CRUSH rule 0 x 1004 [269,638]
+  CRUSH rule 0 x 1005 [369,223]
+  CRUSH rule 0 x 1006 [40,107]
+  CRUSH rule 0 x 1007 [978,111]
+  CRUSH rule 0 x 1008 [965,956]
+  CRUSH rule 0 x 1009 [598,476]
+  CRUSH rule 0 x 1010 [767,523]
+  CRUSH rule 0 x 1011 [289,871]
+  CRUSH rule 0 x 1012 [128,28]
+  CRUSH rule 0 x 1013 [979,765]
+  CRUSH rule 0 x 1014 [979,948]
+  CRUSH rule 0 x 1015 [277,790]
+  CRUSH rule 0 x 1016 [262,73]
+  CRUSH rule 0 x 1017 [150,269]
+  CRUSH rule 0 x 1018 [555,829]
+  CRUSH rule 0 x 1019 [513,356]
+  CRUSH rule 0 x 1020 [158,161]
+  CRUSH rule 0 x 1021 [915,998]
+  CRUSH rule 0 x 1022 [967,829]
+  CRUSH rule 0 x 1023 [488,257]
+  rule 0 (data) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536]
+  CRUSH rule 0 x 1 [876,250,334]
+  CRUSH rule 0 x 2 [292,832,53]
+  CRUSH rule 0 x 3 [623,387,124]
+  CRUSH rule 0 x 4 [61,334,710]
+  CRUSH rule 0 x 5 [946,557,713]
+  CRUSH rule 0 x 6 [576,668,212]
+  CRUSH rule 0 x 7 [645,753,906]
+  CRUSH rule 0 x 8 [243,6,863]
+  CRUSH rule 0 x 9 [22,578,251]
+  CRUSH rule 0 x 10 [758,828,360]
+  CRUSH rule 0 x 11 [769,120,124]
+  CRUSH rule 0 x 12 [780,364,689]
+  CRUSH rule 0 x 13 [557,18,351]
+  CRUSH rule 0 x 14 [59,561,249]
+  CRUSH rule 0 x 15 [718,928,993]
+  CRUSH rule 0 x 16 [673,632,841]
+  CRUSH rule 0 x 17 [648,43,560]
+  CRUSH rule 0 x 18 [654,219,181]
+  CRUSH rule 0 x 19 [850,545,377]
+  CRUSH rule 0 x 20 [717,785,974]
+  CRUSH rule 0 x 21 [420,57,519]
+  CRUSH rule 0 x 22 [503,998,193]
+  CRUSH rule 0 x 23 [411,663,168]
+  CRUSH rule 0 x 24 [266,861,353]
+  CRUSH rule 0 x 25 [760,483,818]
+  CRUSH rule 0 x 26 [903,24,573]
+  CRUSH rule 0 x 27 [946,188,289]
+  CRUSH rule 0 x 28 [69,312,73]
+  CRUSH rule 0 x 29 [844,883,337]
+  CRUSH rule 0 x 30 [621,18,613]
+  CRUSH rule 0 x 31 [784,943,814]
+  CRUSH rule 0 x 32 [173,374,369]
+  CRUSH rule 0 x 33 [698,336,357]
+  CRUSH rule 0 x 34 [168,836,210]
+  CRUSH rule 0 x 35 [274,509,534]
+  CRUSH rule 0 x 36 [318,215,153]
+  CRUSH rule 0 x 37 [173,604,109]
+  CRUSH rule 0 x 38 [708,444,683]
+  CRUSH rule 0 x 39 [662,198,417]
+  CRUSH rule 0 x 40 [620,801,414]
+  CRUSH rule 0 x 41 [811,264,177]
+  CRUSH rule 0 x 42 [863,179,527]
+  CRUSH rule 0 x 43 [686,822,988]
+  CRUSH rule 0 x 44 [396,222,46]
+  CRUSH rule 0 x 45 [991,694,253]
+  CRUSH rule 0 x 46 [420,909,184]
+  CRUSH rule 0 x 47 [467,211,605]
+  CRUSH rule 0 x 48 [955,329,368]
+  CRUSH rule 0 x 49 [974,891,931]
+  CRUSH rule 0 x 50 [870,441,691]
+  CRUSH rule 0 x 51 [182,930,25]
+  CRUSH rule 0 x 52 [704,812,894]
+  CRUSH rule 0 x 53 [185,713,631]
+  CRUSH rule 0 x 54 [270,441,100]
+  CRUSH rule 0 x 55 [895,734,958]
+  CRUSH rule 0 x 56 [564,963,683]
+  CRUSH rule 0 x 57 [738,130,208]
+  CRUSH rule 0 x 58 [524,113,806]
+  CRUSH rule 0 x 59 [408,337,668]
+  CRUSH rule 0 x 60 [228,790,857]
+  CRUSH rule 0 x 61 [154,843,717]
+  CRUSH rule 0 x 62 [594,811,549]
+  CRUSH rule 0 x 63 [646,67,884]
+  CRUSH rule 0 x 64 [175,542,155]
+  CRUSH rule 0 x 65 [745,619,131]
+  CRUSH rule 0 x 66 [275,468,23]
+  CRUSH rule 0 x 67 [246,958,524]
+  CRUSH rule 0 x 68 [711,473,403]
+  CRUSH rule 0 x 69 [493,924,850]
+  CRUSH rule 0 x 70 [30,499,644]
+  CRUSH rule 0 x 71 [984,883,574]
+  CRUSH rule 0 x 72 [71,286,942]
+  CRUSH rule 0 x 73 [922,618,3]
+  CRUSH rule 0 x 74 [629,414,185]
+  CRUSH rule 0 x 75 [222,20,174]
+  CRUSH rule 0 x 76 [262,366,339]
+  CRUSH rule 0 x 77 [638,469,992]
+  CRUSH rule 0 x 78 [324,511,788]
+  CRUSH rule 0 x 79 [577,990,64]
+  CRUSH rule 0 x 80 [501,95,278]
+  CRUSH rule 0 x 81 [506,812,9]
+  CRUSH rule 0 x 82 [222,145,80]
+  CRUSH rule 0 x 83 [71,634,61]
+  CRUSH rule 0 x 84 [49,761,773]
+  CRUSH rule 0 x 85 [985,896,708]
+  CRUSH rule 0 x 86 [537,745,93]
+  CRUSH rule 0 x 87 [997,317,463]
+  CRUSH rule 0 x 88 [957,350,890]
+  CRUSH rule 0 x 89 [399,730,148]
+  CRUSH rule 0 x 90 [943,706,683]
+  CRUSH rule 0 x 91 [22,368,149]
+  CRUSH rule 0 x 92 [532,424,426]
+  CRUSH rule 0 x 93 [218,489,405]
+  CRUSH rule 0 x 94 [181,96,102]
+  CRUSH rule 0 x 95 [343,957,820]
+  CRUSH rule 0 x 96 [861,270,87]
+  CRUSH rule 0 x 97 [459,706,45]
+  CRUSH rule 0 x 98 [327,867,353]
+  CRUSH rule 0 x 99 [974,133,468]
+  CRUSH rule 0 x 100 [32,445,547]
+  CRUSH rule 0 x 101 [142,90,337]
+  CRUSH rule 0 x 102 [172,129,139]
+  CRUSH rule 0 x 103 [630,47,161]
+  CRUSH rule 0 x 104 [758,133,278]
+  CRUSH rule 0 x 105 [843,604,47]
+  CRUSH rule 0 x 106 [28,681,193]
+  CRUSH rule 0 x 107 [74,320,85]
+  CRUSH rule 0 x 108 [875,593,575]
+  CRUSH rule 0 x 109 [411,985,811]
+  CRUSH rule 0 x 110 [440,774,799]
+  CRUSH rule 0 x 111 [405,742,276]
+  CRUSH rule 0 x 112 [143,181,922]
+  CRUSH rule 0 x 113 [153,846,160]
+  CRUSH rule 0 x 114 [804,892,939]
+  CRUSH rule 0 x 115 [588,508,958]
+  CRUSH rule 0 x 116 [327,148,637]
+  CRUSH rule 0 x 117 [95,594,989]
+  CRUSH rule 0 x 118 [80,957,897]
+  CRUSH rule 0 x 119 [386,932,951]
+  CRUSH rule 0 x 120 [366,312,653]
+  CRUSH rule 0 x 121 [129,154,847]
+  CRUSH rule 0 x 122 [873,1,110]
+  CRUSH rule 0 x 123 [533,415,789]
+  CRUSH rule 0 x 124 [461,691,898]
+  CRUSH rule 0 x 125 [342,599,830]
+  CRUSH rule 0 x 126 [819,781,822]
+  CRUSH rule 0 x 127 [437,893,585]
+  CRUSH rule 0 x 128 [679,994,982]
+  CRUSH rule 0 x 129 [380,685,947]
+  CRUSH rule 0 x 130 [992,52,466]
+  CRUSH rule 0 x 131 [469,90,208]
+  CRUSH rule 0 x 132 [571,250,316]
+  CRUSH rule 0 x 133 [964,728,329]
+  CRUSH rule 0 x 134 [999,19,716]
+  CRUSH rule 0 x 135 [634,101,52]
+  CRUSH rule 0 x 136 [114,889,692]
+  CRUSH rule 0 x 137 [839,8,959]
+  CRUSH rule 0 x 138 [967,949,138]
+  CRUSH rule 0 x 139 [308,711,736]
+  CRUSH rule 0 x 140 [764,936,926]
+  CRUSH rule 0 x 141 [423,302,112]
+  CRUSH rule 0 x 142 [252,821,715]
+  CRUSH rule 0 x 143 [33,808,518]
+  CRUSH rule 0 x 144 [472,88,969]
+  CRUSH rule 0 x 145 [242,208,252]
+  CRUSH rule 0 x 146 [290,70,570]
+  CRUSH rule 0 x 147 [447,352,657]
+  CRUSH rule 0 x 148 [212,644,432]
+  CRUSH rule 0 x 149 [9,775,87]
+  CRUSH rule 0 x 150 [166,456,582]
+  CRUSH rule 0 x 151 [811,875,307]
+  CRUSH rule 0 x 152 [449,617,223]
+  CRUSH rule 0 x 153 [523,537,695]
+  CRUSH rule 0 x 154 [208,559,874]
+  CRUSH rule 0 x 155 [569,325,192]
+  CRUSH rule 0 x 156 [488,121,521]
+  CRUSH rule 0 x 157 [140,723,633]
+  CRUSH rule 0 x 158 [786,451,320]
+  CRUSH rule 0 x 159 [134,664,517]
+  CRUSH rule 0 x 160 [690,112,414]
+  CRUSH rule 0 x 161 [324,912,397]
+  CRUSH rule 0 x 162 [748,567,284]
+  CRUSH rule 0 x 163 [575,499,31]
+  CRUSH rule 0 x 164 [314,489,308]
+  CRUSH rule 0 x 165 [116,209,750]
+  CRUSH rule 0 x 166 [352,706,701]
+  CRUSH rule 0 x 167 [27,743,174]
+  CRUSH rule 0 x 168 [953,898,880]
+  CRUSH rule 0 x 169 [912,147,266]
+  CRUSH rule 0 x 170 [421,515,828]
+  CRUSH rule 0 x 171 [488,584,880]
+  CRUSH rule 0 x 172 [366,443,957]
+  CRUSH rule 0 x 173 [863,291,625]
+  CRUSH rule 0 x 174 [263,555,650]
+  CRUSH rule 0 x 175 [875,961,361]
+  CRUSH rule 0 x 176 [745,83,701]
+  CRUSH rule 0 x 177 [128,244,41]
+  CRUSH rule 0 x 178 [155,41,264]
+  CRUSH rule 0 x 179 [593,833,202]
+  CRUSH rule 0 x 180 [154,734,17]
+  CRUSH rule 0 x 181 [289,675,723]
+  CRUSH rule 0 x 182 [730,931,560]
+  CRUSH rule 0 x 183 [639,237,794]
+  CRUSH rule 0 x 184 [704,312,685]
+  CRUSH rule 0 x 185 [97,100,762]
+  CRUSH rule 0 x 186 [26,665,554]
+  CRUSH rule 0 x 187 [649,14,740]
+  CRUSH rule 0 x 188 [682,695,590]
+  CRUSH rule 0 x 189 [325,693,726]
+  CRUSH rule 0 x 190 [399,933,136]
+  CRUSH rule 0 x 191 [629,533,17]
+  CRUSH rule 0 x 192 [503,578,38]
+  CRUSH rule 0 x 193 [546,333,651]
+  CRUSH rule 0 x 194 [242,473,58]
+  CRUSH rule 0 x 195 [625,719,135]
+  CRUSH rule 0 x 196 [357,114,125]
+  CRUSH rule 0 x 197 [306,954,453]
+  CRUSH rule 0 x 198 [863,791,311]
+  CRUSH rule 0 x 199 [935,906,929]
+  CRUSH rule 0 x 200 [373,774,229]
+  CRUSH rule 0 x 201 [659,320,477]
+  CRUSH rule 0 x 202 [260,433,524]
+  CRUSH rule 0 x 203 [36,239,675]
+  CRUSH rule 0 x 204 [92,516,993]
+  CRUSH rule 0 x 205 [68,395,473]
+  CRUSH rule 0 x 206 [570,530,642]
+  CRUSH rule 0 x 207 [834,457,850]
+  CRUSH rule 0 x 208 [927,484,640]
+  CRUSH rule 0 x 209 [878,66,58]
+  CRUSH rule 0 x 210 [572,981,484]
+  CRUSH rule 0 x 211 [107,597,780]
+  CRUSH rule 0 x 212 [389,107,838]
+  CRUSH rule 0 x 213 [497,717,567]
+  CRUSH rule 0 x 214 [798,65,254]
+  CRUSH rule 0 x 215 [233,419,283]
+  CRUSH rule 0 x 216 [494,464,742]
+  CRUSH rule 0 x 217 [352,396,309]
+  CRUSH rule 0 x 218 [895,864,988]
+  CRUSH rule 0 x 219 [222,534,277]
+  CRUSH rule 0 x 220 [281,19,584]
+  CRUSH rule 0 x 221 [64,928,963]
+  CRUSH rule 0 x 222 [40,544,161]
+  CRUSH rule 0 x 223 [645,556,159]
+  CRUSH rule 0 x 224 [647,165,957]
+  CRUSH rule 0 x 225 [219,714,858]
+  CRUSH rule 0 x 226 [372,511,181]
+  CRUSH rule 0 x 227 [925,156,714]
+  CRUSH rule 0 x 228 [682,404,839]
+  CRUSH rule 0 x 229 [880,838,770]
+  CRUSH rule 0 x 230 [328,659,916]
+  CRUSH rule 0 x 231 [320,383,669]
+  CRUSH rule 0 x 232 [924,846,394]
+  CRUSH rule 0 x 233 [948,652,575]
+  CRUSH rule 0 x 234 [484,943,42]
+  CRUSH rule 0 x 235 [750,65,590]
+  CRUSH rule 0 x 236 [551,787,490]
+  CRUSH rule 0 x 237 [390,157,166]
+  CRUSH rule 0 x 238 [570,6,989]
+  CRUSH rule 0 x 239 [729,959,376]
+  CRUSH rule 0 x 240 [981,241,156]
+  CRUSH rule 0 x 241 [310,816,641]
+  CRUSH rule 0 x 242 [161,63,642]
+  CRUSH rule 0 x 243 [180,394,33]
+  CRUSH rule 0 x 244 [52,174,685]
+  CRUSH rule 0 x 245 [523,121,915]
+  CRUSH rule 0 x 246 [362,893,390]
+  CRUSH rule 0 x 247 [382,184,116]
+  CRUSH rule 0 x 248 [129,114,852]
+  CRUSH rule 0 x 249 [159,683,91]
+  CRUSH rule 0 x 250 [404,945,569]
+  CRUSH rule 0 x 251 [661,225,738]
+  CRUSH rule 0 x 252 [961,226,542]
+  CRUSH rule 0 x 253 [651,97,225]
+  CRUSH rule 0 x 254 [123,33,741]
+  CRUSH rule 0 x 255 [314,649,891]
+  CRUSH rule 0 x 256 [315,215,651]
+  CRUSH rule 0 x 257 [825,264,867]
+  CRUSH rule 0 x 258 [624,789,370]
+  CRUSH rule 0 x 259 [602,542,70]
+  CRUSH rule 0 x 260 [717,878,43]
+  CRUSH rule 0 x 261 [145,517,20]
+  CRUSH rule 0 x 262 [223,1,561]
+  CRUSH rule 0 x 263 [462,211,405]
+  CRUSH rule 0 x 264 [654,471,266]
+  CRUSH rule 0 x 265 [302,794,704]
+  CRUSH rule 0 x 266 [202,132,884]
+  CRUSH rule 0 x 267 [282,938,657]
+  CRUSH rule 0 x 268 [338,309,356]
+  CRUSH rule 0 x 269 [738,122,266]
+  CRUSH rule 0 x 270 [707,982,946]
+  CRUSH rule 0 x 271 [705,432,364]
+  CRUSH rule 0 x 272 [756,545,942]
+  CRUSH rule 0 x 273 [197,502,527]
+  CRUSH rule 0 x 274 [992,44,653]
+  CRUSH rule 0 x 275 [544,789,170]
+  CRUSH rule 0 x 276 [658,467,577]
+  CRUSH rule 0 x 277 [143,490,880]
+  CRUSH rule 0 x 278 [492,647,355]
+  CRUSH rule 0 x 279 [517,792,604]
+  CRUSH rule 0 x 280 [825,740,27]
+  CRUSH rule 0 x 281 [224,629,120]
+  CRUSH rule 0 x 282 [298,661,380]
+  CRUSH rule 0 x 283 [311,606,208]
+  CRUSH rule 0 x 284 [771,466,371]
+  CRUSH rule 0 x 285 [693,362,404]
+  CRUSH rule 0 x 286 [364,477,285]
+  CRUSH rule 0 x 287 [591,611,828]
+  CRUSH rule 0 x 288 [965,541,848]
+  CRUSH rule 0 x 289 [225,551,948]
+  CRUSH rule 0 x 290 [577,762,777]
+  CRUSH rule 0 x 291 [160,903,477]
+  CRUSH rule 0 x 292 [873,598,216]
+  CRUSH rule 0 x 293 [100,234,874]
+  CRUSH rule 0 x 294 [285,943,379]
+  CRUSH rule 0 x 295 [938,262,880]
+  CRUSH rule 0 x 296 [850,327,86]
+  CRUSH rule 0 x 297 [951,53,99]
+  CRUSH rule 0 x 298 [173,336,85]
+  CRUSH rule 0 x 299 [598,591,315]
+  CRUSH rule 0 x 300 [531,957,62]
+  CRUSH rule 0 x 301 [823,628,23]
+  CRUSH rule 0 x 302 [184,80,780]
+  CRUSH rule 0 x 303 [521,766,222]
+  CRUSH rule 0 x 304 [980,127,807]
+  CRUSH rule 0 x 305 [153,816,22]
+  CRUSH rule 0 x 306 [423,739,664]
+  CRUSH rule 0 x 307 [997,557,682]
+  CRUSH rule 0 x 308 [991,874,534]
+  CRUSH rule 0 x 309 [860,394,724]
+  CRUSH rule 0 x 310 [589,818,546]
+  CRUSH rule 0 x 311 [477,774,225]
+  CRUSH rule 0 x 312 [887,853,950]
+  CRUSH rule 0 x 313 [802,646,447]
+  CRUSH rule 0 x 314 [654,974,229]
+  CRUSH rule 0 x 315 [767,227,28]
+  CRUSH rule 0 x 316 [778,83,733]
+  CRUSH rule 0 x 317 [184,418,642]
+  CRUSH rule 0 x 318 [525,410,500]
+  CRUSH rule 0 x 319 [476,724,569]
+  CRUSH rule 0 x 320 [149,610,697]
+  CRUSH rule 0 x 321 [710,79,667]
+  CRUSH rule 0 x 322 [175,275,323]
+  CRUSH rule 0 x 323 [819,604,638]
+  CRUSH rule 0 x 324 [16,745,511]
+  CRUSH rule 0 x 325 [486,400,872]
+  CRUSH rule 0 x 326 [613,765,207]
+  CRUSH rule 0 x 327 [125,289,738]
+  CRUSH rule 0 x 328 [807,383,476]
+  CRUSH rule 0 x 329 [588,938,599]
+  CRUSH rule 0 x 330 [932,644,41]
+  CRUSH rule 0 x 331 [341,953,950]
+  CRUSH rule 0 x 332 [153,726,459]
+  CRUSH rule 0 x 333 [745,845,853]
+  CRUSH rule 0 x 334 [614,751,807]
+  CRUSH rule 0 x 335 [518,721,221]
+  CRUSH rule 0 x 336 [389,424,77]
+  CRUSH rule 0 x 337 [753,508,765]
+  CRUSH rule 0 x 338 [128,810,490]
+  CRUSH rule 0 x 339 [430,308,58]
+  CRUSH rule 0 x 340 [541,44,630]
+  CRUSH rule 0 x 341 [402,26,631]
+  CRUSH rule 0 x 342 [982,57,992]
+  CRUSH rule 0 x 343 [833,412,572]
+  CRUSH rule 0 x 344 [784,533,792]
+  CRUSH rule 0 x 345 [546,300,304]
+  CRUSH rule 0 x 346 [302,420,428]
+  CRUSH rule 0 x 347 [488,778,101]
+  CRUSH rule 0 x 348 [903,744,937]
+  CRUSH rule 0 x 349 [471,547,582]
+  CRUSH rule 0 x 350 [348,221,823]
+  CRUSH rule 0 x 351 [961,582,705]
+  CRUSH rule 0 x 352 [728,137,461]
+  CRUSH rule 0 x 353 [904,202,184]
+  CRUSH rule 0 x 354 [345,226,319]
+  CRUSH rule 0 x 355 [50,430,175]
+  CRUSH rule 0 x 356 [87,185,55]
+  CRUSH rule 0 x 357 [762,459,921]
+  CRUSH rule 0 x 358 [908,25,280]
+  CRUSH rule 0 x 359 [484,15,132]
+  CRUSH rule 0 x 360 [173,378,337]
+  CRUSH rule 0 x 361 [404,577,115]
+  CRUSH rule 0 x 362 [403,1,422]
+  CRUSH rule 0 x 363 [639,911,510]
+  CRUSH rule 0 x 364 [752,689,610]
+  CRUSH rule 0 x 365 [956,999,212]
+  CRUSH rule 0 x 366 [860,925,924]
+  CRUSH rule 0 x 367 [205,609,647]
+  CRUSH rule 0 x 368 [301,284,810]
+  CRUSH rule 0 x 369 [452,658,339]
+  CRUSH rule 0 x 370 [11,467,695]
+  CRUSH rule 0 x 371 [124,487,55]
+  CRUSH rule 0 x 372 [253,48,979]
+  CRUSH rule 0 x 373 [715,605,775]
+  CRUSH rule 0 x 374 [191,887,920]
+  CRUSH rule 0 x 375 [711,385,651]
+  CRUSH rule 0 x 376 [597,818,49]
+  CRUSH rule 0 x 377 [294,256,933]
+  CRUSH rule 0 x 378 [34,151,681]
+  CRUSH rule 0 x 379 [869,136,315]
+  CRUSH rule 0 x 380 [294,97,575]
+  CRUSH rule 0 x 381 [119,710,219]
+  CRUSH rule 0 x 382 [69,631,508]
+  CRUSH rule 0 x 383 [922,588,589]
+  CRUSH rule 0 x 384 [221,945,671]
+  CRUSH rule 0 x 385 [561,737,953]
+  CRUSH rule 0 x 386 [335,442,788]
+  CRUSH rule 0 x 387 [514,43,353]
+  CRUSH rule 0 x 388 [587,89,157]
+  CRUSH rule 0 x 389 [109,641,255]
+  CRUSH rule 0 x 390 [925,149,421]
+  CRUSH rule 0 x 391 [267,87,387]
+  CRUSH rule 0 x 392 [382,485,370]
+  CRUSH rule 0 x 393 [425,721,221]
+  CRUSH rule 0 x 394 [898,18,38]
+  CRUSH rule 0 x 395 [806,876,269]
+  CRUSH rule 0 x 396 [790,970,437]
+  CRUSH rule 0 x 397 [136,363,507]
+  CRUSH rule 0 x 398 [914,116,558]
+  CRUSH rule 0 x 399 [261,94,299]
+  CRUSH rule 0 x 400 [661,197,338]
+  CRUSH rule 0 x 401 [953,979,287]
+  CRUSH rule 0 x 402 [738,819,618]
+  CRUSH rule 0 x 403 [573,238,425]
+  CRUSH rule 0 x 404 [526,848,790]
+  CRUSH rule 0 x 405 [582,505,330]
+  CRUSH rule 0 x 406 [768,324,493]
+  CRUSH rule 0 x 407 [260,951,437]
+  CRUSH rule 0 x 408 [657,81,770]
+  CRUSH rule 0 x 409 [498,89,182]
+  CRUSH rule 0 x 410 [28,793,737]
+  CRUSH rule 0 x 411 [684,992,60]
+  CRUSH rule 0 x 412 [261,958,699]
+  CRUSH rule 0 x 413 [891,835,297]
+  CRUSH rule 0 x 414 [127,459,119]
+  CRUSH rule 0 x 415 [272,540,631]
+  CRUSH rule 0 x 416 [739,617,115]
+  CRUSH rule 0 x 417 [106,209,157]
+  CRUSH rule 0 x 418 [525,441,147]
+  CRUSH rule 0 x 419 [603,673,615]
+  CRUSH rule 0 x 420 [988,213,251]
+  CRUSH rule 0 x 421 [761,521,748]
+  CRUSH rule 0 x 422 [317,160,924]
+  CRUSH rule 0 x 423 [137,807,168]
+  CRUSH rule 0 x 424 [920,37,146]
+  CRUSH rule 0 x 425 [277,693,285]
+  CRUSH rule 0 x 426 [485,936,407]
+  CRUSH rule 0 x 427 [242,515,9]
+  CRUSH rule 0 x 428 [632,635,26]
+  CRUSH rule 0 x 429 [641,73,465]
+  CRUSH rule 0 x 430 [626,585,6]
+  CRUSH rule 0 x 431 [697,76,753]
+  CRUSH rule 0 x 432 [590,526,306]
+  CRUSH rule 0 x 433 [284,387,149]
+  CRUSH rule 0 x 434 [538,985,79]
+  CRUSH rule 0 x 435 [30,318,593]
+  CRUSH rule 0 x 436 [164,919,851]
+  CRUSH rule 0 x 437 [322,212,163]
+  CRUSH rule 0 x 438 [142,392,85]
+  CRUSH rule 0 x 439 [119,370,68]
+  CRUSH rule 0 x 440 [333,403,187]
+  CRUSH rule 0 x 441 [477,727,906]
+  CRUSH rule 0 x 442 [274,590,933]
+  CRUSH rule 0 x 443 [983,748,574]
+  CRUSH rule 0 x 444 [536,509,431]
+  CRUSH rule 0 x 445 [485,528,209]
+  CRUSH rule 0 x 446 [345,634,42]
+  CRUSH rule 0 x 447 [61,845,767]
+  CRUSH rule 0 x 448 [333,232,292]
+  CRUSH rule 0 x 449 [680,16,484]
+  CRUSH rule 0 x 450 [235,214,79]
+  CRUSH rule 0 x 451 [961,468,333]
+  CRUSH rule 0 x 452 [525,479,153]
+  CRUSH rule 0 x 453 [138,466,302]
+  CRUSH rule 0 x 454 [137,625,215]
+  CRUSH rule 0 x 455 [173,150,997]
+  CRUSH rule 0 x 456 [235,226,238]
+  CRUSH rule 0 x 457 [450,577,253]
+  CRUSH rule 0 x 458 [195,537,91]
+  CRUSH rule 0 x 459 [381,555,312]
+  CRUSH rule 0 x 460 [972,730,534]
+  CRUSH rule 0 x 461 [506,279,142]
+  CRUSH rule 0 x 462 [692,959,578]
+  CRUSH rule 0 x 463 [788,667,949]
+  CRUSH rule 0 x 464 [133,122,588]
+  CRUSH rule 0 x 465 [971,190,230]
+  CRUSH rule 0 x 466 [394,576,148]
+  CRUSH rule 0 x 467 [517,28,366]
+  CRUSH rule 0 x 468 [829,143,874]
+  CRUSH rule 0 x 469 [987,936,106]
+  CRUSH rule 0 x 470 [107,982,56]
+  CRUSH rule 0 x 471 [181,897,629]
+  CRUSH rule 0 x 472 [547,512,172]
+  CRUSH rule 0 x 473 [760,997,824]
+  CRUSH rule 0 x 474 [787,418,743]
+  CRUSH rule 0 x 475 [662,312,253]
+  CRUSH rule 0 x 476 [110,495,185]
+  CRUSH rule 0 x 477 [393,954,834]
+  CRUSH rule 0 x 478 [246,483,480]
+  CRUSH rule 0 x 479 [70,929,697]
+  CRUSH rule 0 x 480 [753,119,961]
+  CRUSH rule 0 x 481 [470,429,677]
+  CRUSH rule 0 x 482 [451,566,961]
+  CRUSH rule 0 x 483 [816,72,371]
+  CRUSH rule 0 x 484 [540,454,389]
+  CRUSH rule 0 x 485 [74,582,624]
+  CRUSH rule 0 x 486 [958,595,199]
+  CRUSH rule 0 x 487 [228,302,804]
+  CRUSH rule 0 x 488 [180,529,722]
+  CRUSH rule 0 x 489 [47,617,812]
+  CRUSH rule 0 x 490 [905,822,479]
+  CRUSH rule 0 x 491 [892,370,609]
+  CRUSH rule 0 x 492 [588,959,127]
+  CRUSH rule 0 x 493 [353,461,593]
+  CRUSH rule 0 x 494 [378,848,443]
+  CRUSH rule 0 x 495 [845,653,768]
+  CRUSH rule 0 x 496 [13,988,0]
+  CRUSH rule 0 x 497 [796,877,788]
+  CRUSH rule 0 x 498 [412,337,270]
+  CRUSH rule 0 x 499 [330,695,8]
+  CRUSH rule 0 x 500 [820,272,547]
+  CRUSH rule 0 x 501 [110,44,132]
+  CRUSH rule 0 x 502 [336,595,650]
+  CRUSH rule 0 x 503 [922,211,157]
+  CRUSH rule 0 x 504 [483,52,122]
+  CRUSH rule 0 x 505 [482,598,224]
+  CRUSH rule 0 x 506 [493,123,43]
+  CRUSH rule 0 x 507 [12,598,264]
+  CRUSH rule 0 x 508 [227,157,611]
+  CRUSH rule 0 x 509 [807,242,363]
+  CRUSH rule 0 x 510 [134,437,227]
+  CRUSH rule 0 x 511 [212,54,83]
+  CRUSH rule 0 x 512 [236,630,758]
+  CRUSH rule 0 x 513 [994,693,644]
+  CRUSH rule 0 x 514 [45,508,831]
+  CRUSH rule 0 x 515 [504,138,480]
+  CRUSH rule 0 x 516 [285,409,136]
+  CRUSH rule 0 x 517 [300,232,23]
+  CRUSH rule 0 x 518 [397,674,98]
+  CRUSH rule 0 x 519 [86,750,772]
+  CRUSH rule 0 x 520 [900,833,614]
+  CRUSH rule 0 x 521 [31,47,236]
+  CRUSH rule 0 x 522 [390,16,280]
+  CRUSH rule 0 x 523 [618,308,424]
+  CRUSH rule 0 x 524 [635,189,687]
+  CRUSH rule 0 x 525 [311,916,699]
+  CRUSH rule 0 x 526 [48,738,227]
+  CRUSH rule 0 x 527 [202,851,889]
+  CRUSH rule 0 x 528 [565,827,590]
+  CRUSH rule 0 x 529 [934,864,241]
+  CRUSH rule 0 x 530 [502,934,298]
+  CRUSH rule 0 x 531 [681,627,942]
+  CRUSH rule 0 x 532 [422,6,147]
+  CRUSH rule 0 x 533 [863,68,364]
+  CRUSH rule 0 x 534 [962,931,775]
+  CRUSH rule 0 x 535 [89,565,397]
+  CRUSH rule 0 x 536 [499,351,760]
+  CRUSH rule 0 x 537 [676,547,787]
+  CRUSH rule 0 x 538 [58,644,571]
+  CRUSH rule 0 x 539 [837,953,457]
+  CRUSH rule 0 x 540 [831,50,132]
+  CRUSH rule 0 x 541 [582,757,121]
+  CRUSH rule 0 x 542 [472,132,790]
+  CRUSH rule 0 x 543 [382,272,797]
+  CRUSH rule 0 x 544 [947,930,496]
+  CRUSH rule 0 x 545 [425,570,305]
+  CRUSH rule 0 x 546 [18,65,529]
+  CRUSH rule 0 x 547 [445,715,600]
+  CRUSH rule 0 x 548 [367,569,980]
+  CRUSH rule 0 x 549 [125,715,671]
+  CRUSH rule 0 x 550 [425,599,744]
+  CRUSH rule 0 x 551 [44,1,528]
+  CRUSH rule 0 x 552 [246,104,68]
+  CRUSH rule 0 x 553 [71,703,615]
+  CRUSH rule 0 x 554 [207,124,217]
+  CRUSH rule 0 x 555 [570,28,317]
+  CRUSH rule 0 x 556 [674,152,421]
+  CRUSH rule 0 x 557 [347,817,191]
+  CRUSH rule 0 x 558 [627,426,369]
+  CRUSH rule 0 x 559 [940,630,924]
+  CRUSH rule 0 x 560 [295,903,541]
+  CRUSH rule 0 x 561 [506,682,384]
+  CRUSH rule 0 x 562 [718,529,87]
+  CRUSH rule 0 x 563 [552,332,747]
+  CRUSH rule 0 x 564 [835,769,736]
+  CRUSH rule 0 x 565 [8,167,539]
+  CRUSH rule 0 x 566 [600,481,301]
+  CRUSH rule 0 x 567 [999,994,509]
+  CRUSH rule 0 x 568 [252,431,157]
+  CRUSH rule 0 x 569 [643,218,943]
+  CRUSH rule 0 x 570 [617,635,765]
+  CRUSH rule 0 x 571 [757,80,59]
+  CRUSH rule 0 x 572 [299,348,575]
+  CRUSH rule 0 x 573 [25,505,270]
+  CRUSH rule 0 x 574 [215,431,624]
+  CRUSH rule 0 x 575 [225,252,611]
+  CRUSH rule 0 x 576 [627,94,159]
+  CRUSH rule 0 x 577 [237,809,778]
+  CRUSH rule 0 x 578 [885,313,120]
+  CRUSH rule 0 x 579 [924,575,787]
+  CRUSH rule 0 x 580 [718,51,766]
+  CRUSH rule 0 x 581 [219,807,129]
+  CRUSH rule 0 x 582 [893,701,598]
+  CRUSH rule 0 x 583 [246,930,964]
+  CRUSH rule 0 x 584 [336,432,680]
+  CRUSH rule 0 x 585 [324,999,397]
+  CRUSH rule 0 x 586 [558,230,976]
+  CRUSH rule 0 x 587 [985,830,597]
+  CRUSH rule 0 x 588 [211,544,57]
+  CRUSH rule 0 x 589 [129,21,112]
+  CRUSH rule 0 x 590 [467,969,652]
+  CRUSH rule 0 x 591 [758,514,316]
+  CRUSH rule 0 x 592 [525,253,190]
+  CRUSH rule 0 x 593 [601,885,339]
+  CRUSH rule 0 x 594 [227,60,450]
+  CRUSH rule 0 x 595 [720,854,496]
+  CRUSH rule 0 x 596 [751,195,997]
+  CRUSH rule 0 x 597 [129,574,714]
+  CRUSH rule 0 x 598 [679,207,604]
+  CRUSH rule 0 x 599 [668,315,683]
+  CRUSH rule 0 x 600 [143,396,464]
+  CRUSH rule 0 x 601 [326,573,873]
+  CRUSH rule 0 x 602 [860,281,875]
+  CRUSH rule 0 x 603 [709,328,445]
+  CRUSH rule 0 x 604 [571,62,814]
+  CRUSH rule 0 x 605 [252,739,860]
+  CRUSH rule 0 x 606 [339,236,759]
+  CRUSH rule 0 x 607 [590,248,759]
+  CRUSH rule 0 x 608 [145,635,309]
+  CRUSH rule 0 x 609 [973,547,223]
+  CRUSH rule 0 x 610 [435,816,961]
+  CRUSH rule 0 x 611 [559,283,422]
+  CRUSH rule 0 x 612 [273,149,123]
+  CRUSH rule 0 x 613 [828,614,642]
+  CRUSH rule 0 x 614 [478,748,393]
+  CRUSH rule 0 x 615 [392,155,144]
+  CRUSH rule 0 x 616 [778,637,452]
+  CRUSH rule 0 x 617 [622,713,996]
+  CRUSH rule 0 x 618 [149,877,270]
+  CRUSH rule 0 x 619 [604,163,656]
+  CRUSH rule 0 x 620 [181,23,409]
+  CRUSH rule 0 x 621 [735,902,386]
+  CRUSH rule 0 x 622 [661,824,717]
+  CRUSH rule 0 x 623 [142,121,643]
+  CRUSH rule 0 x 624 [360,716,420]
+  CRUSH rule 0 x 625 [541,167,385]
+  CRUSH rule 0 x 626 [364,431,610]
+  CRUSH rule 0 x 627 [458,137,557]
+  CRUSH rule 0 x 628 [250,350,556]
+  CRUSH rule 0 x 629 [928,160,710]
+  CRUSH rule 0 x 630 [243,19,918]
+  CRUSH rule 0 x 631 [438,221,574]
+  CRUSH rule 0 x 632 [797,368,247]
+  CRUSH rule 0 x 633 [993,749,525]
+  CRUSH rule 0 x 634 [239,351,633]
+  CRUSH rule 0 x 635 [640,965,25]
+  CRUSH rule 0 x 636 [173,290,297]
+  CRUSH rule 0 x 637 [0,918,98]
+  CRUSH rule 0 x 638 [702,235,424]
+  CRUSH rule 0 x 639 [475,687,31]
+  CRUSH rule 0 x 640 [31,664,399]
+  CRUSH rule 0 x 641 [296,473,108]
+  CRUSH rule 0 x 642 [894,273,427]
+  CRUSH rule 0 x 643 [117,111,732]
+  CRUSH rule 0 x 644 [438,336,327]
+  CRUSH rule 0 x 645 [982,702,351]
+  CRUSH rule 0 x 646 [334,804,146]
+  CRUSH rule 0 x 647 [933,787,185]
+  CRUSH rule 0 x 648 [22,444,400]
+  CRUSH rule 0 x 649 [503,229,213]
+  CRUSH rule 0 x 650 [328,659,420]
+  CRUSH rule 0 x 651 [3,880,823]
+  CRUSH rule 0 x 652 [495,977,563]
+  CRUSH rule 0 x 653 [185,718,804]
+  CRUSH rule 0 x 654 [130,528,380]
+  CRUSH rule 0 x 655 [560,872,454]
+  CRUSH rule 0 x 656 [219,885,178]
+  CRUSH rule 0 x 657 [233,684,813]
+  CRUSH rule 0 x 658 [778,6,756]
+  CRUSH rule 0 x 659 [240,663,306]
+  CRUSH rule 0 x 660 [244,855,196]
+  CRUSH rule 0 x 661 [184,270,128]
+  CRUSH rule 0 x 662 [65,883,921]
+  CRUSH rule 0 x 663 [323,721,594]
+  CRUSH rule 0 x 664 [865,113,512]
+  CRUSH rule 0 x 665 [420,850,591]
+  CRUSH rule 0 x 666 [319,767,246]
+  CRUSH rule 0 x 667 [875,39,343]
+  CRUSH rule 0 x 668 [331,122,263]
+  CRUSH rule 0 x 669 [915,521,402]
+  CRUSH rule 0 x 670 [845,659,943]
+  CRUSH rule 0 x 671 [108,634,527]
+  CRUSH rule 0 x 672 [578,216,110]
+  CRUSH rule 0 x 673 [442,74,579]
+  CRUSH rule 0 x 674 [588,364,281]
+  CRUSH rule 0 x 675 [489,698,744]
+  CRUSH rule 0 x 676 [928,911,40]
+  CRUSH rule 0 x 677 [399,269,692]
+  CRUSH rule 0 x 678 [546,752,544]
+  CRUSH rule 0 x 679 [988,25,275]
+  CRUSH rule 0 x 680 [335,963,382]
+  CRUSH rule 0 x 681 [690,462,623]
+  CRUSH rule 0 x 682 [196,588,154]
+  CRUSH rule 0 x 683 [627,25,421]
+  CRUSH rule 0 x 684 [38,804,592]
+  CRUSH rule 0 x 685 [841,368,548]
+  CRUSH rule 0 x 686 [336,287,525]
+  CRUSH rule 0 x 687 [20,682,924]
+  CRUSH rule 0 x 688 [463,371,780]
+  CRUSH rule 0 x 689 [569,250,78]
+  CRUSH rule 0 x 690 [551,144,587]
+  CRUSH rule 0 x 691 [766,464,446]
+  CRUSH rule 0 x 692 [739,634,18]
+  CRUSH rule 0 x 693 [339,297,118]
+  CRUSH rule 0 x 694 [405,26,830]
+  CRUSH rule 0 x 695 [622,576,597]
+  CRUSH rule 0 x 696 [558,902,689]
+  CRUSH rule 0 x 697 [818,222,406]
+  CRUSH rule 0 x 698 [178,48,402]
+  CRUSH rule 0 x 699 [450,244,180]
+  CRUSH rule 0 x 700 [502,771,987]
+  CRUSH rule 0 x 701 [4,612,782]
+  CRUSH rule 0 x 702 [177,630,232]
+  CRUSH rule 0 x 703 [354,178,389]
+  CRUSH rule 0 x 704 [646,601,156]
+  CRUSH rule 0 x 705 [921,401,890]
+  CRUSH rule 0 x 706 [652,877,562]
+  CRUSH rule 0 x 707 [345,745,67]
+  CRUSH rule 0 x 708 [333,607,180]
+  CRUSH rule 0 x 709 [45,187,302]
+  CRUSH rule 0 x 710 [94,855,43]
+  CRUSH rule 0 x 711 [227,653,731]
+  CRUSH rule 0 x 712 [398,953,136]
+  CRUSH rule 0 x 713 [116,800,503]
+  CRUSH rule 0 x 714 [111,629,866]
+  CRUSH rule 0 x 715 [531,291,486]
+  CRUSH rule 0 x 716 [169,541,291]
+  CRUSH rule 0 x 717 [417,446,994]
+  CRUSH rule 0 x 718 [992,383,298]
+  CRUSH rule 0 x 719 [936,674,324]
+  CRUSH rule 0 x 720 [370,188,174]
+  CRUSH rule 0 x 721 [320,859,278]
+  CRUSH rule 0 x 722 [7,2,673]
+  CRUSH rule 0 x 723 [270,553,831]
+  CRUSH rule 0 x 724 [666,822,708]
+  CRUSH rule 0 x 725 [794,406,875]
+  CRUSH rule 0 x 726 [420,556,341]
+  CRUSH rule 0 x 727 [561,461,129]
+  CRUSH rule 0 x 728 [951,330,196]
+  CRUSH rule 0 x 729 [656,644,436]
+  CRUSH rule 0 x 730 [3,558,629]
+  CRUSH rule 0 x 731 [852,89,75]
+  CRUSH rule 0 x 732 [983,840,869]
+  CRUSH rule 0 x 733 [285,396,388]
+  CRUSH rule 0 x 734 [125,510,402]
+  CRUSH rule 0 x 735 [417,773,686]
+  CRUSH rule 0 x 736 [749,396,632]
+  CRUSH rule 0 x 737 [644,991,946]
+  CRUSH rule 0 x 738 [449,683,290]
+  CRUSH rule 0 x 739 [341,220,641]
+  CRUSH rule 0 x 740 [874,524,674]
+  CRUSH rule 0 x 741 [189,472,712]
+  CRUSH rule 0 x 742 [912,581,114]
+  CRUSH rule 0 x 743 [654,914,425]
+  CRUSH rule 0 x 744 [725,295,579]
+  CRUSH rule 0 x 745 [787,858,850]
+  CRUSH rule 0 x 746 [757,848,704]
+  CRUSH rule 0 x 747 [700,81,867]
+  CRUSH rule 0 x 748 [557,436,238]
+  CRUSH rule 0 x 749 [772,622,337]
+  CRUSH rule 0 x 750 [946,97,376]
+  CRUSH rule 0 x 751 [996,618,343]
+  CRUSH rule 0 x 752 [746,887,695]
+  CRUSH rule 0 x 753 [741,14,463]
+  CRUSH rule 0 x 754 [648,349,333]
+  CRUSH rule 0 x 755 [157,460,466]
+  CRUSH rule 0 x 756 [416,97,197]
+  CRUSH rule 0 x 757 [599,839,776]
+  CRUSH rule 0 x 758 [994,218,620]
+  CRUSH rule 0 x 759 [959,682,514]
+  CRUSH rule 0 x 760 [518,943,215]
+  CRUSH rule 0 x 761 [285,849,420]
+  CRUSH rule 0 x 762 [591,313,41]
+  CRUSH rule 0 x 763 [908,411,200]
+  CRUSH rule 0 x 764 [787,234,894]
+  CRUSH rule 0 x 765 [327,921,882]
+  CRUSH rule 0 x 766 [84,161,878]
+  CRUSH rule 0 x 767 [370,895,702]
+  CRUSH rule 0 x 768 [826,760,879]
+  CRUSH rule 0 x 769 [67,768,663]
+  CRUSH rule 0 x 770 [593,909,482]
+  CRUSH rule 0 x 771 [309,935,121]
+  CRUSH rule 0 x 772 [12,125,797]
+  CRUSH rule 0 x 773 [253,466,820]
+  CRUSH rule 0 x 774 [164,390,705]
+  CRUSH rule 0 x 775 [703,47,43]
+  CRUSH rule 0 x 776 [728,231,80]
+  CRUSH rule 0 x 777 [981,621,568]
+  CRUSH rule 0 x 778 [411,456,544]
+  CRUSH rule 0 x 779 [346,121,519]
+  CRUSH rule 0 x 780 [476,39,288]
+  CRUSH rule 0 x 781 [10,130,585]
+  CRUSH rule 0 x 782 [462,246,581]
+  CRUSH rule 0 x 783 [580,373,153]
+  CRUSH rule 0 x 784 [413,113,978]
+  CRUSH rule 0 x 785 [341,856,332]
+  CRUSH rule 0 x 786 [411,140,313]
+  CRUSH rule 0 x 787 [605,522,211]
+  CRUSH rule 0 x 788 [226,545,35]
+  CRUSH rule 0 x 789 [545,320,414]
+  CRUSH rule 0 x 790 [414,748,816]
+  CRUSH rule 0 x 791 [660,906,406]
+  CRUSH rule 0 x 792 [287,392,514]
+  CRUSH rule 0 x 793 [631,133,850]
+  CRUSH rule 0 x 794 [931,517,543]
+  CRUSH rule 0 x 795 [551,962,477]
+  CRUSH rule 0 x 796 [814,4,95]
+  CRUSH rule 0 x 797 [64,201,299]
+  CRUSH rule 0 x 798 [422,530,114]
+  CRUSH rule 0 x 799 [824,32,679]
+  CRUSH rule 0 x 800 [862,623,489]
+  CRUSH rule 0 x 801 [145,550,329]
+  CRUSH rule 0 x 802 [570,19,847]
+  CRUSH rule 0 x 803 [151,812,662]
+  CRUSH rule 0 x 804 [467,93,264]
+  CRUSH rule 0 x 805 [621,223,938]
+  CRUSH rule 0 x 806 [898,957,805]
+  CRUSH rule 0 x 807 [354,531,422]
+  CRUSH rule 0 x 808 [7,96,76]
+  CRUSH rule 0 x 809 [70,734,719]
+  CRUSH rule 0 x 810 [701,18,972]
+  CRUSH rule 0 x 811 [248,547,103]
+  CRUSH rule 0 x 812 [230,576,821]
+  CRUSH rule 0 x 813 [805,114,683]
+  CRUSH rule 0 x 814 [54,619,973]
+  CRUSH rule 0 x 815 [679,412,613]
+  CRUSH rule 0 x 816 [919,448,826]
+  CRUSH rule 0 x 817 [765,830,436]
+  CRUSH rule 0 x 818 [415,566,644]
+  CRUSH rule 0 x 819 [721,319,865]
+  CRUSH rule 0 x 820 [218,301,333]
+  CRUSH rule 0 x 821 [185,795,680]
+  CRUSH rule 0 x 822 [356,261,54]
+  CRUSH rule 0 x 823 [220,281,549]
+  CRUSH rule 0 x 824 [292,809,887]
+  CRUSH rule 0 x 825 [949,778,101]
+  CRUSH rule 0 x 826 [767,818,833]
+  CRUSH rule 0 x 827 [631,83,406]
+  CRUSH rule 0 x 828 [288,986,445]
+  CRUSH rule 0 x 829 [990,667,915]
+  CRUSH rule 0 x 830 [152,571,778]
+  CRUSH rule 0 x 831 [814,563,630]
+  CRUSH rule 0 x 832 [235,641,616]
+  CRUSH rule 0 x 833 [657,565,922]
+  CRUSH rule 0 x 834 [907,231,644]
+  CRUSH rule 0 x 835 [784,262,771]
+  CRUSH rule 0 x 836 [951,158,366]
+  CRUSH rule 0 x 837 [556,498,334]
+  CRUSH rule 0 x 838 [329,274,964]
+  CRUSH rule 0 x 839 [568,209,939]
+  CRUSH rule 0 x 840 [45,579,842]
+  CRUSH rule 0 x 841 [652,702,24]
+  CRUSH rule 0 x 842 [629,984,314]
+  CRUSH rule 0 x 843 [799,690,688]
+  CRUSH rule 0 x 844 [694,600,534]
+  CRUSH rule 0 x 845 [332,30,179]
+  CRUSH rule 0 x 846 [452,251,712]
+  CRUSH rule 0 x 847 [399,681,847]
+  CRUSH rule 0 x 848 [303,138,440]
+  CRUSH rule 0 x 849 [666,346,708]
+  CRUSH rule 0 x 850 [644,511,345]
+  CRUSH rule 0 x 851 [527,546,737]
+  CRUSH rule 0 x 852 [31,809,94]
+  CRUSH rule 0 x 853 [483,330,869]
+  CRUSH rule 0 x 854 [697,953,968]
+  CRUSH rule 0 x 855 [837,996,239]
+  CRUSH rule 0 x 856 [712,40,547]
+  CRUSH rule 0 x 857 [77,984,576]
+  CRUSH rule 0 x 858 [412,384,841]
+  CRUSH rule 0 x 859 [173,760,26]
+  CRUSH rule 0 x 860 [776,429,328]
+  CRUSH rule 0 x 861 [705,405,477]
+  CRUSH rule 0 x 862 [809,44,788]
+  CRUSH rule 0 x 863 [349,496,963]
+  CRUSH rule 0 x 864 [717,858,101]
+  CRUSH rule 0 x 865 [857,603,586]
+  CRUSH rule 0 x 866 [394,304,71]
+  CRUSH rule 0 x 867 [640,773,663]
+  CRUSH rule 0 x 868 [613,950,712]
+  CRUSH rule 0 x 869 [973,889,524]
+  CRUSH rule 0 x 870 [505,35,386]
+  CRUSH rule 0 x 871 [239,264,262]
+  CRUSH rule 0 x 872 [21,767,456]
+  CRUSH rule 0 x 873 [954,666,980]
+  CRUSH rule 0 x 874 [54,510,947]
+  CRUSH rule 0 x 875 [809,418,452]
+  CRUSH rule 0 x 876 [483,457,61]
+  CRUSH rule 0 x 877 [542,531,952]
+  CRUSH rule 0 x 878 [217,674,857]
+  CRUSH rule 0 x 879 [999,475,134]
+  CRUSH rule 0 x 880 [678,573,935]
+  CRUSH rule 0 x 881 [394,835,789]
+  CRUSH rule 0 x 882 [467,382,353]
+  CRUSH rule 0 x 883 [802,744,237]
+  CRUSH rule 0 x 884 [653,660,638]
+  CRUSH rule 0 x 885 [898,704,307]
+  CRUSH rule 0 x 886 [434,357,938]
+  CRUSH rule 0 x 887 [297,226,711]
+  CRUSH rule 0 x 888 [863,324,443]
+  CRUSH rule 0 x 889 [105,102,308]
+  CRUSH rule 0 x 890 [550,248,606]
+  CRUSH rule 0 x 891 [575,928,880]
+  CRUSH rule 0 x 892 [259,862,133]
+  CRUSH rule 0 x 893 [902,880,543]
+  CRUSH rule 0 x 894 [180,169,916]
+  CRUSH rule 0 x 895 [725,849,182]
+  CRUSH rule 0 x 896 [951,34,874]
+  CRUSH rule 0 x 897 [810,352,73]
+  CRUSH rule 0 x 898 [979,433,719]
+  CRUSH rule 0 x 899 [685,668,534]
+  CRUSH rule 0 x 900 [530,978,41]
+  CRUSH rule 0 x 901 [740,107,336]
+  CRUSH rule 0 x 902 [800,743,693]
+  CRUSH rule 0 x 903 [230,267,842]
+  CRUSH rule 0 x 904 [346,949,460]
+  CRUSH rule 0 x 905 [530,397,619]
+  CRUSH rule 0 x 906 [80,426,138]
+  CRUSH rule 0 x 907 [365,968,475]
+  CRUSH rule 0 x 908 [204,832,742]
+  CRUSH rule 0 x 909 [883,989,146]
+  CRUSH rule 0 x 910 [549,593,249]
+  CRUSH rule 0 x 911 [325,847,352]
+  CRUSH rule 0 x 912 [874,888,582]
+  CRUSH rule 0 x 913 [331,463,342]
+  CRUSH rule 0 x 914 [836,468,601]
+  CRUSH rule 0 x 915 [245,228,100]
+  CRUSH rule 0 x 916 [77,967,364]
+  CRUSH rule 0 x 917 [239,60,866]
+  CRUSH rule 0 x 918 [988,115,922]
+  CRUSH rule 0 x 919 [783,139,696]
+  CRUSH rule 0 x 920 [623,408,685]
+  CRUSH rule 0 x 921 [105,799,144]
+  CRUSH rule 0 x 922 [887,505,652]
+  CRUSH rule 0 x 923 [223,318,552]
+  CRUSH rule 0 x 924 [25,778,366]
+  CRUSH rule 0 x 925 [912,601,297]
+  CRUSH rule 0 x 926 [968,133,144]
+  CRUSH rule 0 x 927 [277,724,214]
+  CRUSH rule 0 x 928 [554,203,658]
+  CRUSH rule 0 x 929 [761,802,367]
+  CRUSH rule 0 x 930 [814,61,788]
+  CRUSH rule 0 x 931 [29,193,61]
+  CRUSH rule 0 x 932 [446,198,862]
+  CRUSH rule 0 x 933 [352,742,216]
+  CRUSH rule 0 x 934 [730,2,332]
+  CRUSH rule 0 x 935 [731,23,736]
+  CRUSH rule 0 x 936 [322,975,20]
+  CRUSH rule 0 x 937 [822,221,841]
+  CRUSH rule 0 x 938 [557,850,66]
+  CRUSH rule 0 x 939 [150,11,971]
+  CRUSH rule 0 x 940 [638,398,169]
+  CRUSH rule 0 x 941 [730,342,929]
+  CRUSH rule 0 x 942 [62,292,166]
+  CRUSH rule 0 x 943 [165,314,519]
+  CRUSH rule 0 x 944 [199,625,766]
+  CRUSH rule 0 x 945 [946,999,699]
+  CRUSH rule 0 x 946 [595,93,852]
+  CRUSH rule 0 x 947 [800,582,356]
+  CRUSH rule 0 x 948 [132,551,139]
+  CRUSH rule 0 x 949 [792,920,466]
+  CRUSH rule 0 x 950 [111,345,176]
+  CRUSH rule 0 x 951 [414,619,648]
+  CRUSH rule 0 x 952 [775,469,500]
+  CRUSH rule 0 x 953 [349,1,5]
+  CRUSH rule 0 x 954 [570,940,410]
+  CRUSH rule 0 x 955 [729,774,823]
+  CRUSH rule 0 x 956 [519,141,575]
+  CRUSH rule 0 x 957 [242,709,611]
+  CRUSH rule 0 x 958 [84,217,227]
+  CRUSH rule 0 x 959 [270,413,918]
+  CRUSH rule 0 x 960 [458,192,307]
+  CRUSH rule 0 x 961 [981,388,777]
+  CRUSH rule 0 x 962 [623,834,277]
+  CRUSH rule 0 x 963 [291,167,714]
+  CRUSH rule 0 x 964 [28,156,788]
+  CRUSH rule 0 x 965 [675,557,290]
+  CRUSH rule 0 x 966 [836,306,946]
+  CRUSH rule 0 x 967 [966,386,735]
+  CRUSH rule 0 x 968 [864,756,690]
+  CRUSH rule 0 x 969 [729,625,480]
+  CRUSH rule 0 x 970 [800,362,646]
+  CRUSH rule 0 x 971 [737,381,153]
+  CRUSH rule 0 x 972 [952,245,720]
+  CRUSH rule 0 x 973 [356,455,579]
+  CRUSH rule 0 x 974 [545,758,586]
+  CRUSH rule 0 x 975 [336,191,202]
+  CRUSH rule 0 x 976 [446,208,757]
+  CRUSH rule 0 x 977 [202,896,196]
+  CRUSH rule 0 x 978 [612,324,996]
+  CRUSH rule 0 x 979 [843,457,675]
+  CRUSH rule 0 x 980 [60,914,881]
+  CRUSH rule 0 x 981 [702,749,937]
+  CRUSH rule 0 x 982 [298,928,738]
+  CRUSH rule 0 x 983 [723,572,395]
+  CRUSH rule 0 x 984 [723,864,804]
+  CRUSH rule 0 x 985 [945,459,868]
+  CRUSH rule 0 x 986 [772,664,535]
+  CRUSH rule 0 x 987 [88,324,312]
+  CRUSH rule 0 x 988 [522,927,131]
+  CRUSH rule 0 x 989 [578,332,208]
+  CRUSH rule 0 x 990 [638,228,414]
+  CRUSH rule 0 x 991 [530,221,451]
+  CRUSH rule 0 x 992 [925,705,275]
+  CRUSH rule 0 x 993 [991,301,43]
+  CRUSH rule 0 x 994 [276,51,868]
+  CRUSH rule 0 x 995 [288,836,753]
+  CRUSH rule 0 x 996 [887,983,252]
+  CRUSH rule 0 x 997 [110,924,386]
+  CRUSH rule 0 x 998 [435,830,485]
+  CRUSH rule 0 x 999 [876,738,357]
+  CRUSH rule 0 x 1000 [178,963,638]
+  CRUSH rule 0 x 1001 [99,519,66]
+  CRUSH rule 0 x 1002 [515,534,468]
+  CRUSH rule 0 x 1003 [104,611,937]
+  CRUSH rule 0 x 1004 [269,638,724]
+  CRUSH rule 0 x 1005 [369,223,309]
+  CRUSH rule 0 x 1006 [40,107,69]
+  CRUSH rule 0 x 1007 [978,111,416]
+  CRUSH rule 0 x 1008 [965,956,624]
+  CRUSH rule 0 x 1009 [598,476,356]
+  CRUSH rule 0 x 1010 [767,523,239]
+  CRUSH rule 0 x 1011 [289,871,207]
+  CRUSH rule 0 x 1012 [128,28,370]
+  CRUSH rule 0 x 1013 [979,765,660]
+  CRUSH rule 0 x 1014 [979,948,513]
+  CRUSH rule 0 x 1015 [277,790,396]
+  CRUSH rule 0 x 1016 [262,73,128]
+  CRUSH rule 0 x 1017 [150,269,61]
+  CRUSH rule 0 x 1018 [555,829,554]
+  CRUSH rule 0 x 1019 [513,356,265]
+  CRUSH rule 0 x 1020 [158,161,877]
+  CRUSH rule 0 x 1021 [915,998,957]
+  CRUSH rule 0 x 1022 [967,829,973]
+  CRUSH rule 0 x 1023 [488,257,614]
+  rule 0 (data) num_rep 3 result size == 3:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450]
+  CRUSH rule 0 x 1 [876,250,334,633]
+  CRUSH rule 0 x 2 [292,832,53,392]
+  CRUSH rule 0 x 3 [623,387,124,998]
+  CRUSH rule 0 x 4 [61,334,710,4]
+  CRUSH rule 0 x 5 [946,557,713,664]
+  CRUSH rule 0 x 6 [576,668,212,163]
+  CRUSH rule 0 x 7 [645,753,906,393]
+  CRUSH rule 0 x 8 [243,6,863,781]
+  CRUSH rule 0 x 9 [22,578,251,410]
+  CRUSH rule 0 x 10 [758,828,360,477]
+  CRUSH rule 0 x 11 [769,120,124,527]
+  CRUSH rule 0 x 12 [780,364,689,755]
+  CRUSH rule 0 x 13 [557,18,351,719]
+  CRUSH rule 0 x 14 [59,561,249,461]
+  CRUSH rule 0 x 15 [718,928,993,21]
+  CRUSH rule 0 x 16 [673,632,841,954]
+  CRUSH rule 0 x 17 [648,43,560,514]
+  CRUSH rule 0 x 18 [654,219,181,568]
+  CRUSH rule 0 x 19 [850,545,377,848]
+  CRUSH rule 0 x 20 [717,785,974,5]
+  CRUSH rule 0 x 21 [420,57,519,306]
+  CRUSH rule 0 x 22 [503,998,193,821]
+  CRUSH rule 0 x 23 [411,663,168,110]
+  CRUSH rule 0 x 24 [266,861,353,1]
+  CRUSH rule 0 x 25 [760,483,818,600]
+  CRUSH rule 0 x 26 [903,24,573,718]
+  CRUSH rule 0 x 27 [946,188,289,510]
+  CRUSH rule 0 x 28 [69,312,73,198]
+  CRUSH rule 0 x 29 [844,883,337,628]
+  CRUSH rule 0 x 30 [621,18,613,794]
+  CRUSH rule 0 x 31 [784,943,814,539]
+  CRUSH rule 0 x 32 [173,374,369,972]
+  CRUSH rule 0 x 33 [698,336,357,966]
+  CRUSH rule 0 x 34 [168,836,210,798]
+  CRUSH rule 0 x 35 [274,509,534,818]
+  CRUSH rule 0 x 36 [318,215,153,628]
+  CRUSH rule 0 x 37 [173,604,109,935]
+  CRUSH rule 0 x 38 [708,444,683,604]
+  CRUSH rule 0 x 39 [662,198,417,680]
+  CRUSH rule 0 x 40 [620,801,414,78]
+  CRUSH rule 0 x 41 [811,264,177,127]
+  CRUSH rule 0 x 42 [863,179,527,660]
+  CRUSH rule 0 x 43 [686,822,988,228]
+  CRUSH rule 0 x 44 [396,222,46,841]
+  CRUSH rule 0 x 45 [991,694,253,142]
+  CRUSH rule 0 x 46 [420,909,184,285]
+  CRUSH rule 0 x 47 [467,211,605,207]
+  CRUSH rule 0 x 48 [955,329,368,168]
+  CRUSH rule 0 x 49 [974,891,931,29]
+  CRUSH rule 0 x 50 [870,441,691,823]
+  CRUSH rule 0 x 51 [182,930,25,936]
+  CRUSH rule 0 x 52 [704,812,894,794]
+  CRUSH rule 0 x 53 [185,713,631,280]
+  CRUSH rule 0 x 54 [270,441,100,82]
+  CRUSH rule 0 x 55 [895,734,958,793]
+  CRUSH rule 0 x 56 [564,963,683,324]
+  CRUSH rule 0 x 57 [738,130,208,973]
+  CRUSH rule 0 x 58 [524,113,806,903]
+  CRUSH rule 0 x 59 [408,337,668,529]
+  CRUSH rule 0 x 60 [228,790,857,309]
+  CRUSH rule 0 x 61 [154,843,717,467]
+  CRUSH rule 0 x 62 [594,811,549,276]
+  CRUSH rule 0 x 63 [646,67,884,925]
+  CRUSH rule 0 x 64 [175,542,155,837]
+  CRUSH rule 0 x 65 [745,619,131,867]
+  CRUSH rule 0 x 66 [275,468,23,35]
+  CRUSH rule 0 x 67 [246,958,524,493]
+  CRUSH rule 0 x 68 [711,473,403,228]
+  CRUSH rule 0 x 69 [493,924,850,939]
+  CRUSH rule 0 x 70 [30,499,644,33]
+  CRUSH rule 0 x 71 [984,883,574,716]
+  CRUSH rule 0 x 72 [71,286,942,363]
+  CRUSH rule 0 x 73 [922,618,3,371]
+  CRUSH rule 0 x 74 [629,414,185,573]
+  CRUSH rule 0 x 75 [222,20,174,820]
+  CRUSH rule 0 x 76 [262,366,339,290]
+  CRUSH rule 0 x 77 [638,469,992,280]
+  CRUSH rule 0 x 78 [324,511,788,7]
+  CRUSH rule 0 x 79 [577,990,64,94]
+  CRUSH rule 0 x 80 [501,95,278,903]
+  CRUSH rule 0 x 81 [506,812,9,698]
+  CRUSH rule 0 x 82 [222,145,80,785]
+  CRUSH rule 0 x 83 [71,634,61,91]
+  CRUSH rule 0 x 84 [49,761,773,368]
+  CRUSH rule 0 x 85 [985,896,708,861]
+  CRUSH rule 0 x 86 [537,745,93,524]
+  CRUSH rule 0 x 87 [997,317,463,626]
+  CRUSH rule 0 x 88 [957,350,890,857]
+  CRUSH rule 0 x 89 [399,730,148,314]
+  CRUSH rule 0 x 90 [943,706,683,267]
+  CRUSH rule 0 x 91 [22,368,149,928]
+  CRUSH rule 0 x 92 [532,424,426,773]
+  CRUSH rule 0 x 93 [218,489,405,681]
+  CRUSH rule 0 x 94 [181,96,102,515]
+  CRUSH rule 0 x 95 [343,957,820,139]
+  CRUSH rule 0 x 96 [861,270,87,797]
+  CRUSH rule 0 x 97 [459,706,45,328]
+  CRUSH rule 0 x 98 [327,867,353,948]
+  CRUSH rule 0 x 99 [974,133,468,906]
+  CRUSH rule 0 x 100 [32,445,547,371]
+  CRUSH rule 0 x 101 [142,90,337,950]
+  CRUSH rule 0 x 102 [172,129,139,22]
+  CRUSH rule 0 x 103 [630,47,161,356]
+  CRUSH rule 0 x 104 [758,133,278,11]
+  CRUSH rule 0 x 105 [843,604,47,33]
+  CRUSH rule 0 x 106 [28,681,193,679]
+  CRUSH rule 0 x 107 [74,320,85,819]
+  CRUSH rule 0 x 108 [875,593,575,517]
+  CRUSH rule 0 x 109 [411,985,811,720]
+  CRUSH rule 0 x 110 [440,774,799,660]
+  CRUSH rule 0 x 111 [405,742,276,359]
+  CRUSH rule 0 x 112 [143,181,922,545]
+  CRUSH rule 0 x 113 [153,846,160,903]
+  CRUSH rule 0 x 114 [804,892,939,20]
+  CRUSH rule 0 x 115 [588,508,958,580]
+  CRUSH rule 0 x 116 [327,148,637,486]
+  CRUSH rule 0 x 117 [95,594,989,131]
+  CRUSH rule 0 x 118 [80,957,897,239]
+  CRUSH rule 0 x 119 [386,932,951,768]
+  CRUSH rule 0 x 120 [366,312,653,936]
+  CRUSH rule 0 x 121 [129,154,847,16]
+  CRUSH rule 0 x 122 [873,1,110,939]
+  CRUSH rule 0 x 123 [533,415,789,600]
+  CRUSH rule 0 x 124 [461,691,898,723]
+  CRUSH rule 0 x 125 [342,599,830,402]
+  CRUSH rule 0 x 126 [819,781,822,548]
+  CRUSH rule 0 x 127 [437,893,585,707]
+  CRUSH rule 0 x 128 [679,994,982,550]
+  CRUSH rule 0 x 129 [380,685,947,302]
+  CRUSH rule 0 x 130 [992,52,466,867]
+  CRUSH rule 0 x 131 [469,90,208,599]
+  CRUSH rule 0 x 132 [571,250,316,535]
+  CRUSH rule 0 x 133 [964,728,329,902]
+  CRUSH rule 0 x 134 [999,19,716,963]
+  CRUSH rule 0 x 135 [634,101,52,938]
+  CRUSH rule 0 x 136 [114,889,692,768]
+  CRUSH rule 0 x 137 [839,8,959,280]
+  CRUSH rule 0 x 138 [967,949,138,451]
+  CRUSH rule 0 x 139 [308,711,736,247]
+  CRUSH rule 0 x 140 [764,936,926,55]
+  CRUSH rule 0 x 141 [423,302,112,216]
+  CRUSH rule 0 x 142 [252,821,715,340]
+  CRUSH rule 0 x 143 [33,808,518,477]
+  CRUSH rule 0 x 144 [472,88,969,162]
+  CRUSH rule 0 x 145 [242,208,252,604]
+  CRUSH rule 0 x 146 [290,70,570,384]
+  CRUSH rule 0 x 147 [447,352,657,493]
+  CRUSH rule 0 x 148 [212,644,432,658]
+  CRUSH rule 0 x 149 [9,775,87,35]
+  CRUSH rule 0 x 150 [166,456,582,144]
+  CRUSH rule 0 x 151 [811,875,307,20]
+  CRUSH rule 0 x 152 [449,617,223,9]
+  CRUSH rule 0 x 153 [523,537,695,627]
+  CRUSH rule 0 x 154 [208,559,874,597]
+  CRUSH rule 0 x 155 [569,325,192,296]
+  CRUSH rule 0 x 156 [488,121,521,213]
+  CRUSH rule 0 x 157 [140,723,633,260]
+  CRUSH rule 0 x 158 [786,451,320,239]
+  CRUSH rule 0 x 159 [134,664,517,821]
+  CRUSH rule 0 x 160 [690,112,414,990]
+  CRUSH rule 0 x 161 [324,912,397,423]
+  CRUSH rule 0 x 162 [748,567,284,183]
+  CRUSH rule 0 x 163 [575,499,31,816]
+  CRUSH rule 0 x 164 [314,489,308,326]
+  CRUSH rule 0 x 165 [116,209,750,53]
+  CRUSH rule 0 x 166 [352,706,701,810]
+  CRUSH rule 0 x 167 [27,743,174,142]
+  CRUSH rule 0 x 168 [953,898,880,660]
+  CRUSH rule 0 x 169 [912,147,266,547]
+  CRUSH rule 0 x 170 [421,515,828,844]
+  CRUSH rule 0 x 171 [488,584,880,964]
+  CRUSH rule 0 x 172 [366,443,957,66]
+  CRUSH rule 0 x 173 [863,291,625,287]
+  CRUSH rule 0 x 174 [263,555,650,410]
+  CRUSH rule 0 x 175 [875,961,361,575]
+  CRUSH rule 0 x 176 [745,83,701,680]
+  CRUSH rule 0 x 177 [128,244,41,123]
+  CRUSH rule 0 x 178 [155,41,264,777]
+  CRUSH rule 0 x 179 [593,833,202,183]
+  CRUSH rule 0 x 180 [154,734,17,831]
+  CRUSH rule 0 x 181 [289,675,723,800]
+  CRUSH rule 0 x 182 [730,931,560,209]
+  CRUSH rule 0 x 183 [639,237,794,815]
+  CRUSH rule 0 x 184 [704,312,685,645]
+  CRUSH rule 0 x 185 [97,100,762,82]
+  CRUSH rule 0 x 186 [26,665,554,215]
+  CRUSH rule 0 x 187 [649,14,740,494]
+  CRUSH rule 0 x 188 [682,695,590,743]
+  CRUSH rule 0 x 189 [325,693,726,51]
+  CRUSH rule 0 x 190 [399,933,136,955]
+  CRUSH rule 0 x 191 [629,533,17,126]
+  CRUSH rule 0 x 192 [503,578,38,492]
+  CRUSH rule 0 x 193 [546,333,651,678]
+  CRUSH rule 0 x 194 [242,473,58,655]
+  CRUSH rule 0 x 195 [625,719,135,81]
+  CRUSH rule 0 x 196 [357,114,125,867]
+  CRUSH rule 0 x 197 [306,954,453,873]
+  CRUSH rule 0 x 198 [863,791,311,911]
+  CRUSH rule 0 x 199 [935,906,929,252]
+  CRUSH rule 0 x 200 [373,774,229,454]
+  CRUSH rule 0 x 201 [659,320,477,313]
+  CRUSH rule 0 x 202 [260,433,524,880]
+  CRUSH rule 0 x 203 [36,239,675,971]
+  CRUSH rule 0 x 204 [92,516,993,728]
+  CRUSH rule 0 x 205 [68,395,473,45]
+  CRUSH rule 0 x 206 [570,530,642,380]
+  CRUSH rule 0 x 207 [834,457,850,917]
+  CRUSH rule 0 x 208 [927,484,640,976]
+  CRUSH rule 0 x 209 [878,66,58,940]
+  CRUSH rule 0 x 210 [572,981,484,29]
+  CRUSH rule 0 x 211 [107,597,780,857]
+  CRUSH rule 0 x 212 [389,107,838,624]
+  CRUSH rule 0 x 213 [497,717,567,728]
+  CRUSH rule 0 x 214 [798,65,254,572]
+  CRUSH rule 0 x 215 [233,419,283,638]
+  CRUSH rule 0 x 216 [494,464,742,523]
+  CRUSH rule 0 x 217 [352,396,309,938]
+  CRUSH rule 0 x 218 [895,864,988,650]
+  CRUSH rule 0 x 219 [222,534,277,242]
+  CRUSH rule 0 x 220 [281,19,584,563]
+  CRUSH rule 0 x 221 [64,928,963,130]
+  CRUSH rule 0 x 222 [40,544,161,199]
+  CRUSH rule 0 x 223 [645,556,159,417]
+  CRUSH rule 0 x 224 [647,165,957,263]
+  CRUSH rule 0 x 225 [219,714,858,747]
+  CRUSH rule 0 x 226 [372,511,181,277]
+  CRUSH rule 0 x 227 [925,156,714,863]
+  CRUSH rule 0 x 228 [682,404,839,263]
+  CRUSH rule 0 x 229 [880,838,770,891]
+  CRUSH rule 0 x 230 [328,659,916,468]
+  CRUSH rule 0 x 231 [320,383,669,109]
+  CRUSH rule 0 x 232 [924,846,394,319]
+  CRUSH rule 0 x 233 [948,652,575,838]
+  CRUSH rule 0 x 234 [484,943,42,575]
+  CRUSH rule 0 x 235 [750,65,590,168]
+  CRUSH rule 0 x 236 [551,787,490,136]
+  CRUSH rule 0 x 237 [390,157,166,251]
+  CRUSH rule 0 x 238 [570,6,989,707]
+  CRUSH rule 0 x 239 [729,959,376,975]
+  CRUSH rule 0 x 240 [981,241,156,767]
+  CRUSH rule 0 x 241 [310,816,641,177]
+  CRUSH rule 0 x 242 [161,63,642,837]
+  CRUSH rule 0 x 243 [180,394,33,683]
+  CRUSH rule 0 x 244 [52,174,685,189]
+  CRUSH rule 0 x 245 [523,121,915,84]
+  CRUSH rule 0 x 246 [362,893,390,487]
+  CRUSH rule 0 x 247 [382,184,116,34]
+  CRUSH rule 0 x 248 [129,114,852,469]
+  CRUSH rule 0 x 249 [159,683,91,856]
+  CRUSH rule 0 x 250 [404,945,569,955]
+  CRUSH rule 0 x 251 [661,225,738,757]
+  CRUSH rule 0 x 252 [961,226,542,103]
+  CRUSH rule 0 x 253 [651,97,225,364]
+  CRUSH rule 0 x 254 [123,33,741,692]
+  CRUSH rule 0 x 255 [314,649,891,855]
+  CRUSH rule 0 x 256 [315,215,651,126]
+  CRUSH rule 0 x 257 [825,264,867,529]
+  CRUSH rule 0 x 258 [624,789,370,723]
+  CRUSH rule 0 x 259 [602,542,70,563]
+  CRUSH rule 0 x 260 [717,878,43,56]
+  CRUSH rule 0 x 261 [145,517,20,903]
+  CRUSH rule 0 x 262 [223,1,561,420]
+  CRUSH rule 0 x 263 [462,211,405,508]
+  CRUSH rule 0 x 264 [654,471,266,662]
+  CRUSH rule 0 x 265 [302,794,704,798]
+  CRUSH rule 0 x 266 [202,132,884,209]
+  CRUSH rule 0 x 267 [282,938,657,113]
+  CRUSH rule 0 x 268 [338,309,356,278]
+  CRUSH rule 0 x 269 [738,122,266,200]
+  CRUSH rule 0 x 270 [707,982,946,196]
+  CRUSH rule 0 x 271 [705,432,364,735]
+  CRUSH rule 0 x 272 [756,545,942,56]
+  CRUSH rule 0 x 273 [197,502,527,721]
+  CRUSH rule 0 x 274 [992,44,653,573]
+  CRUSH rule 0 x 275 [544,789,170,434]
+  CRUSH rule 0 x 276 [658,467,577,268]
+  CRUSH rule 0 x 277 [143,490,880,483]
+  CRUSH rule 0 x 278 [492,647,355,282]
+  CRUSH rule 0 x 279 [517,792,604,987]
+  CRUSH rule 0 x 280 [825,740,27,848]
+  CRUSH rule 0 x 281 [224,629,120,562]
+  CRUSH rule 0 x 282 [298,661,380,416]
+  CRUSH rule 0 x 283 [311,606,208,50]
+  CRUSH rule 0 x 284 [771,466,371,743]
+  CRUSH rule 0 x 285 [693,362,404,676]
+  CRUSH rule 0 x 286 [364,477,285,167]
+  CRUSH rule 0 x 287 [591,611,828,995]
+  CRUSH rule 0 x 288 [965,541,848,796]
+  CRUSH rule 0 x 289 [225,551,948,877]
+  CRUSH rule 0 x 290 [577,762,777,751]
+  CRUSH rule 0 x 291 [160,903,477,381]
+  CRUSH rule 0 x 292 [873,598,216,666]
+  CRUSH rule 0 x 293 [100,234,874,47]
+  CRUSH rule 0 x 294 [285,943,379,520]
+  CRUSH rule 0 x 295 [938,262,880,327]
+  CRUSH rule 0 x 296 [850,327,86,472]
+  CRUSH rule 0 x 297 [951,53,99,558]
+  CRUSH rule 0 x 298 [173,336,85,766]
+  CRUSH rule 0 x 299 [598,591,315,386]
+  CRUSH rule 0 x 300 [531,957,62,459]
+  CRUSH rule 0 x 301 [823,628,23,858]
+  CRUSH rule 0 x 302 [184,80,780,871]
+  CRUSH rule 0 x 303 [521,766,222,830]
+  CRUSH rule 0 x 304 [980,127,807,507]
+  CRUSH rule 0 x 305 [153,816,22,927]
+  CRUSH rule 0 x 306 [423,739,664,753]
+  CRUSH rule 0 x 307 [997,557,682,456]
+  CRUSH rule 0 x 308 [991,874,534,465]
+  CRUSH rule 0 x 309 [860,394,724,858]
+  CRUSH rule 0 x 310 [589,818,546,201]
+  CRUSH rule 0 x 311 [477,774,225,590]
+  CRUSH rule 0 x 312 [887,853,950,354]
+  CRUSH rule 0 x 313 [802,646,447,416]
+  CRUSH rule 0 x 314 [654,974,229,511]
+  CRUSH rule 0 x 315 [767,227,28,740]
+  CRUSH rule 0 x 316 [778,83,733,359]
+  CRUSH rule 0 x 317 [184,418,642,986]
+  CRUSH rule 0 x 318 [525,410,500,543]
+  CRUSH rule 0 x 319 [476,724,569,382]
+  CRUSH rule 0 x 320 [149,610,697,296]
+  CRUSH rule 0 x 321 [710,79,667,671]
+  CRUSH rule 0 x 322 [175,275,323,333]
+  CRUSH rule 0 x 323 [819,604,638,792]
+  CRUSH rule 0 x 324 [16,745,511,439]
+  CRUSH rule 0 x 325 [486,400,872,873]
+  CRUSH rule 0 x 326 [613,765,207,19]
+  CRUSH rule 0 x 327 [125,289,738,408]
+  CRUSH rule 0 x 328 [807,383,476,583]
+  CRUSH rule 0 x 329 [588,938,599,432]
+  CRUSH rule 0 x 330 [932,644,41,611]
+  CRUSH rule 0 x 331 [341,953,950,537]
+  CRUSH rule 0 x 332 [153,726,459,950]
+  CRUSH rule 0 x 333 [745,845,853,860]
+  CRUSH rule 0 x 334 [614,751,807,58]
+  CRUSH rule 0 x 335 [518,721,221,283]
+  CRUSH rule 0 x 336 [389,424,77,309]
+  CRUSH rule 0 x 337 [753,508,765,720]
+  CRUSH rule 0 x 338 [128,810,490,753]
+  CRUSH rule 0 x 339 [430,308,58,751]
+  CRUSH rule 0 x 340 [541,44,630,231]
+  CRUSH rule 0 x 341 [402,26,631,439]
+  CRUSH rule 0 x 342 [982,57,992,461]
+  CRUSH rule 0 x 343 [833,412,572,732]
+  CRUSH rule 0 x 344 [784,533,792,41]
+  CRUSH rule 0 x 345 [546,300,304,691]
+  CRUSH rule 0 x 346 [302,420,428,891]
+  CRUSH rule 0 x 347 [488,778,101,217]
+  CRUSH rule 0 x 348 [903,744,937,718]
+  CRUSH rule 0 x 349 [471,547,582,306]
+  CRUSH rule 0 x 350 [348,221,823,335]
+  CRUSH rule 0 x 351 [961,582,705,346]
+  CRUSH rule 0 x 352 [728,137,461,298]
+  CRUSH rule 0 x 353 [904,202,184,447]
+  CRUSH rule 0 x 354 [345,226,319,256]
+  CRUSH rule 0 x 355 [50,430,175,43]
+  CRUSH rule 0 x 356 [87,185,55,423]
+  CRUSH rule 0 x 357 [762,459,921,473]
+  CRUSH rule 0 x 358 [908,25,280,6]
+  CRUSH rule 0 x 359 [484,15,132,121]
+  CRUSH rule 0 x 360 [173,378,337,702]
+  CRUSH rule 0 x 361 [404,577,115,25]
+  CRUSH rule 0 x 362 [403,1,422,945]
+  CRUSH rule 0 x 363 [639,911,510,162]
+  CRUSH rule 0 x 364 [752,689,610,990]
+  CRUSH rule 0 x 365 [956,999,212,230]
+  CRUSH rule 0 x 366 [860,925,924,763]
+  CRUSH rule 0 x 367 [205,609,647,665]
+  CRUSH rule 0 x 368 [301,284,810,169]
+  CRUSH rule 0 x 369 [452,658,339,217]
+  CRUSH rule 0 x 370 [11,467,695,989]
+  CRUSH rule 0 x 371 [124,487,55,514]
+  CRUSH rule 0 x 372 [253,48,979,846]
+  CRUSH rule 0 x 373 [715,605,775,748]
+  CRUSH rule 0 x 374 [191,887,920,223]
+  CRUSH rule 0 x 375 [711,385,651,665]
+  CRUSH rule 0 x 376 [597,818,49,458]
+  CRUSH rule 0 x 377 [294,256,933,771]
+  CRUSH rule 0 x 378 [34,151,681,707]
+  CRUSH rule 0 x 379 [869,136,315,378]
+  CRUSH rule 0 x 380 [294,97,575,791]
+  CRUSH rule 0 x 381 [119,710,219,827]
+  CRUSH rule 0 x 382 [69,631,508,706]
+  CRUSH rule 0 x 383 [922,588,589,925]
+  CRUSH rule 0 x 384 [221,945,671,117]
+  CRUSH rule 0 x 385 [561,737,953,723]
+  CRUSH rule 0 x 386 [335,442,788,696]
+  CRUSH rule 0 x 387 [514,43,353,88]
+  CRUSH rule 0 x 388 [587,89,157,996]
+  CRUSH rule 0 x 389 [109,641,255,466]
+  CRUSH rule 0 x 390 [925,149,421,489]
+  CRUSH rule 0 x 391 [267,87,387,527]
+  CRUSH rule 0 x 392 [382,485,370,849]
+  CRUSH rule 0 x 393 [425,721,221,753]
+  CRUSH rule 0 x 394 [898,18,38,793]
+  CRUSH rule 0 x 395 [806,876,269,679]
+  CRUSH rule 0 x 396 [790,970,437,449]
+  CRUSH rule 0 x 397 [136,363,507,613]
+  CRUSH rule 0 x 398 [914,116,558,258]
+  CRUSH rule 0 x 399 [261,94,299,202]
+  CRUSH rule 0 x 400 [661,197,338,461]
+  CRUSH rule 0 x 401 [953,979,287,803]
+  CRUSH rule 0 x 402 [738,819,618,522]
+  CRUSH rule 0 x 403 [573,238,425,546]
+  CRUSH rule 0 x 404 [526,848,790,253]
+  CRUSH rule 0 x 405 [582,505,330,334]
+  CRUSH rule 0 x 406 [768,324,493,60]
+  CRUSH rule 0 x 407 [260,951,437,587]
+  CRUSH rule 0 x 408 [657,81,770,734]
+  CRUSH rule 0 x 409 [498,89,182,423]
+  CRUSH rule 0 x 410 [28,793,737,352]
+  CRUSH rule 0 x 411 [684,992,60,659]
+  CRUSH rule 0 x 412 [261,958,699,950]
+  CRUSH rule 0 x 413 [891,835,297,441]
+  CRUSH rule 0 x 414 [127,459,119,965]
+  CRUSH rule 0 x 415 [272,540,631,328]
+  CRUSH rule 0 x 416 [739,617,115,530]
+  CRUSH rule 0 x 417 [106,209,157,878]
+  CRUSH rule 0 x 418 [525,441,147,390]
+  CRUSH rule 0 x 419 [603,673,615,465]
+  CRUSH rule 0 x 420 [988,213,251,226]
+  CRUSH rule 0 x 421 [761,521,748,368]
+  CRUSH rule 0 x 422 [317,160,924,548]
+  CRUSH rule 0 x 423 [137,807,168,472]
+  CRUSH rule 0 x 424 [920,37,146,263]
+  CRUSH rule 0 x 425 [277,693,285,221]
+  CRUSH rule 0 x 426 [485,936,407,854]
+  CRUSH rule 0 x 427 [242,515,9,564]
+  CRUSH rule 0 x 428 [632,635,26,473]
+  CRUSH rule 0 x 429 [641,73,465,127]
+  CRUSH rule 0 x 430 [626,585,6,387]
+  CRUSH rule 0 x 431 [697,76,753,570]
+  CRUSH rule 0 x 432 [590,526,306,283]
+  CRUSH rule 0 x 433 [284,387,149,817]
+  CRUSH rule 0 x 434 [538,985,79,953]
+  CRUSH rule 0 x 435 [30,318,593,635]
+  CRUSH rule 0 x 436 [164,919,851,693]
+  CRUSH rule 0 x 437 [322,212,163,606]
+  CRUSH rule 0 x 438 [142,392,85,594]
+  CRUSH rule 0 x 439 [119,370,68,443]
+  CRUSH rule 0 x 440 [333,403,187,863]
+  CRUSH rule 0 x 441 [477,727,906,145]
+  CRUSH rule 0 x 442 [274,590,933,244]
+  CRUSH rule 0 x 443 [983,748,574,718]
+  CRUSH rule 0 x 444 [536,509,431,146]
+  CRUSH rule 0 x 445 [485,528,209,964]
+  CRUSH rule 0 x 446 [345,634,42,294]
+  CRUSH rule 0 x 447 [61,845,767,600]
+  CRUSH rule 0 x 448 [333,232,292,846]
+  CRUSH rule 0 x 449 [680,16,484,670]
+  CRUSH rule 0 x 450 [235,214,79,423]
+  CRUSH rule 0 x 451 [961,468,333,640]
+  CRUSH rule 0 x 452 [525,479,153,528]
+  CRUSH rule 0 x 453 [138,466,302,86]
+  CRUSH rule 0 x 454 [137,625,215,402]
+  CRUSH rule 0 x 455 [173,150,997,16]
+  CRUSH rule 0 x 456 [235,226,238,258]
+  CRUSH rule 0 x 457 [450,577,253,413]
+  CRUSH rule 0 x 458 [195,537,91,814]
+  CRUSH rule 0 x 459 [381,555,312,573]
+  CRUSH rule 0 x 460 [972,730,534,678]
+  CRUSH rule 0 x 461 [506,279,142,830]
+  CRUSH rule 0 x 462 [692,959,578,57]
+  CRUSH rule 0 x 463 [788,667,949,550]
+  CRUSH rule 0 x 464 [133,122,588,999]
+  CRUSH rule 0 x 465 [971,190,230,777]
+  CRUSH rule 0 x 466 [394,576,148,157]
+  CRUSH rule 0 x 467 [517,28,366,362]
+  CRUSH rule 0 x 468 [829,143,874,225]
+  CRUSH rule 0 x 469 [987,936,106,725]
+  CRUSH rule 0 x 470 [107,982,56,889]
+  CRUSH rule 0 x 471 [181,897,629,860]
+  CRUSH rule 0 x 472 [547,512,172,24]
+  CRUSH rule 0 x 473 [760,997,824,905]
+  CRUSH rule 0 x 474 [787,418,743,628]
+  CRUSH rule 0 x 475 [662,312,253,617]
+  CRUSH rule 0 x 476 [110,495,185,508]
+  CRUSH rule 0 x 477 [393,954,834,132]
+  CRUSH rule 0 x 478 [246,483,480,644]
+  CRUSH rule 0 x 479 [70,929,697,931]
+  CRUSH rule 0 x 480 [753,119,961,607]
+  CRUSH rule 0 x 481 [470,429,677,242]
+  CRUSH rule 0 x 482 [451,566,961,675]
+  CRUSH rule 0 x 483 [816,72,371,278]
+  CRUSH rule 0 x 484 [540,454,389,31]
+  CRUSH rule 0 x 485 [74,582,624,684]
+  CRUSH rule 0 x 486 [958,595,199,763]
+  CRUSH rule 0 x 487 [228,302,804,833]
+  CRUSH rule 0 x 488 [180,529,722,956]
+  CRUSH rule 0 x 489 [47,617,812,187]
+  CRUSH rule 0 x 490 [905,822,479,124]
+  CRUSH rule 0 x 491 [892,370,609,998]
+  CRUSH rule 0 x 492 [588,959,127,948]
+  CRUSH rule 0 x 493 [353,461,593,291]
+  CRUSH rule 0 x 494 [378,848,443,368]
+  CRUSH rule 0 x 495 [845,653,768,234]
+  CRUSH rule 0 x 496 [13,988,0,691]
+  CRUSH rule 0 x 497 [796,877,788,394]
+  CRUSH rule 0 x 498 [412,337,270,705]
+  CRUSH rule 0 x 499 [330,695,8,74]
+  CRUSH rule 0 x 500 [820,272,547,765]
+  CRUSH rule 0 x 501 [110,44,132,442]
+  CRUSH rule 0 x 502 [336,595,650,274]
+  CRUSH rule 0 x 503 [922,211,157,722]
+  CRUSH rule 0 x 504 [483,52,122,432]
+  CRUSH rule 0 x 505 [482,598,224,279]
+  CRUSH rule 0 x 506 [493,123,43,856]
+  CRUSH rule 0 x 507 [12,598,264,422]
+  CRUSH rule 0 x 508 [227,157,611,301]
+  CRUSH rule 0 x 509 [807,242,363,122]
+  CRUSH rule 0 x 510 [134,437,227,75]
+  CRUSH rule 0 x 511 [212,54,83,799]
+  CRUSH rule 0 x 512 [236,630,758,752]
+  CRUSH rule 0 x 513 [994,693,644,938]
+  CRUSH rule 0 x 514 [45,508,831,19]
+  CRUSH rule 0 x 515 [504,138,480,272]
+  CRUSH rule 0 x 516 [285,409,136,570]
+  CRUSH rule 0 x 517 [300,232,23,906]
+  CRUSH rule 0 x 518 [397,674,98,898]
+  CRUSH rule 0 x 519 [86,750,772,913]
+  CRUSH rule 0 x 520 [900,833,614,130]
+  CRUSH rule 0 x 521 [31,47,236,751]
+  CRUSH rule 0 x 522 [390,16,280,144]
+  CRUSH rule 0 x 523 [618,308,424,590]
+  CRUSH rule 0 x 524 [635,189,687,963]
+  CRUSH rule 0 x 525 [311,916,699,262]
+  CRUSH rule 0 x 526 [48,738,227,718]
+  CRUSH rule 0 x 527 [202,851,889,216]
+  CRUSH rule 0 x 528 [565,827,590,273]
+  CRUSH rule 0 x 529 [934,864,241,43]
+  CRUSH rule 0 x 530 [502,934,298,670]
+  CRUSH rule 0 x 531 [681,627,942,487]
+  CRUSH rule 0 x 532 [422,6,147,205]
+  CRUSH rule 0 x 533 [863,68,364,983]
+  CRUSH rule 0 x 534 [962,931,775,172]
+  CRUSH rule 0 x 535 [89,565,397,693]
+  CRUSH rule 0 x 536 [499,351,760,458]
+  CRUSH rule 0 x 537 [676,547,787,311]
+  CRUSH rule 0 x 538 [58,644,571,649]
+  CRUSH rule 0 x 539 [837,953,457,711]
+  CRUSH rule 0 x 540 [831,50,132,213]
+  CRUSH rule 0 x 541 [582,757,121,525]
+  CRUSH rule 0 x 542 [472,132,790,997]
+  CRUSH rule 0 x 543 [382,272,797,330]
+  CRUSH rule 0 x 544 [947,930,496,883]
+  CRUSH rule 0 x 545 [425,570,305,77]
+  CRUSH rule 0 x 546 [18,65,529,437]
+  CRUSH rule 0 x 547 [445,715,600,472]
+  CRUSH rule 0 x 548 [367,569,980,167]
+  CRUSH rule 0 x 549 [125,715,671,817]
+  CRUSH rule 0 x 550 [425,599,744,199]
+  CRUSH rule 0 x 551 [44,1,528,922]
+  CRUSH rule 0 x 552 [246,104,68,239]
+  CRUSH rule 0 x 553 [71,703,615,28]
+  CRUSH rule 0 x 554 [207,124,217,166]
+  CRUSH rule 0 x 555 [570,28,317,420]
+  CRUSH rule 0 x 556 [674,152,421,79]
+  CRUSH rule 0 x 557 [347,817,191,391]
+  CRUSH rule 0 x 558 [627,426,369,692]
+  CRUSH rule 0 x 559 [940,630,924,242]
+  CRUSH rule 0 x 560 [295,903,541,29]
+  CRUSH rule 0 x 561 [506,682,384,637]
+  CRUSH rule 0 x 562 [718,529,87,729]
+  CRUSH rule 0 x 563 [552,332,747,206]
+  CRUSH rule 0 x 564 [835,769,736,486]
+  CRUSH rule 0 x 565 [8,167,539,182]
+  CRUSH rule 0 x 566 [600,481,301,263]
+  CRUSH rule 0 x 567 [999,994,509,899]
+  CRUSH rule 0 x 568 [252,431,157,62]
+  CRUSH rule 0 x 569 [643,218,943,455]
+  CRUSH rule 0 x 570 [617,635,765,422]
+  CRUSH rule 0 x 571 [757,80,59,98]
+  CRUSH rule 0 x 572 [299,348,575,889]
+  CRUSH rule 0 x 573 [25,505,270,167]
+  CRUSH rule 0 x 574 [215,431,624,177]
+  CRUSH rule 0 x 575 [225,252,611,546]
+  CRUSH rule 0 x 576 [627,94,159,857]
+  CRUSH rule 0 x 577 [237,809,778,636]
+  CRUSH rule 0 x 578 [885,313,120,344]
+  CRUSH rule 0 x 579 [924,575,787,831]
+  CRUSH rule 0 x 580 [718,51,766,121]
+  CRUSH rule 0 x 581 [219,807,129,571]
+  CRUSH rule 0 x 582 [893,701,598,863]
+  CRUSH rule 0 x 583 [246,930,964,170]
+  CRUSH rule 0 x 584 [336,432,680,175]
+  CRUSH rule 0 x 585 [324,999,397,485]
+  CRUSH rule 0 x 586 [558,230,976,541]
+  CRUSH rule 0 x 587 [985,830,597,21]
+  CRUSH rule 0 x 588 [211,544,57,134]
+  CRUSH rule 0 x 589 [129,21,112,190]
+  CRUSH rule 0 x 590 [467,969,652,593]
+  CRUSH rule 0 x 591 [758,514,316,164]
+  CRUSH rule 0 x 592 [525,253,190,443]
+  CRUSH rule 0 x 593 [601,885,339,152]
+  CRUSH rule 0 x 594 [227,60,450,30]
+  CRUSH rule 0 x 595 [720,854,496,912]
+  CRUSH rule 0 x 596 [751,195,997,77]
+  CRUSH rule 0 x 597 [129,574,714,8]
+  CRUSH rule 0 x 598 [679,207,604,396]
+  CRUSH rule 0 x 599 [668,315,683,349]
+  CRUSH rule 0 x 600 [143,396,464,444]
+  CRUSH rule 0 x 601 [326,573,873,902]
+  CRUSH rule 0 x 602 [860,281,875,535]
+  CRUSH rule 0 x 603 [709,328,445,349]
+  CRUSH rule 0 x 604 [571,62,814,95]
+  CRUSH rule 0 x 605 [252,739,860,27]
+  CRUSH rule 0 x 606 [339,236,759,842]
+  CRUSH rule 0 x 607 [590,248,759,868]
+  CRUSH rule 0 x 608 [145,635,309,467]
+  CRUSH rule 0 x 609 [973,547,223,79]
+  CRUSH rule 0 x 610 [435,816,961,983]
+  CRUSH rule 0 x 611 [559,283,422,584]
+  CRUSH rule 0 x 612 [273,149,123,576]
+  CRUSH rule 0 x 613 [828,614,642,674]
+  CRUSH rule 0 x 614 [478,748,393,34]
+  CRUSH rule 0 x 615 [392,155,144,326]
+  CRUSH rule 0 x 616 [778,637,452,248]
+  CRUSH rule 0 x 617 [622,713,996,833]
+  CRUSH rule 0 x 618 [149,877,270,329]
+  CRUSH rule 0 x 619 [604,163,656,409]
+  CRUSH rule 0 x 620 [181,23,409,198]
+  CRUSH rule 0 x 621 [735,902,386,237]
+  CRUSH rule 0 x 622 [661,824,717,568]
+  CRUSH rule 0 x 623 [142,121,643,61]
+  CRUSH rule 0 x 624 [360,716,420,398]
+  CRUSH rule 0 x 625 [541,167,385,1]
+  CRUSH rule 0 x 626 [364,431,610,363]
+  CRUSH rule 0 x 627 [458,137,557,410]
+  CRUSH rule 0 x 628 [250,350,556,497]
+  CRUSH rule 0 x 629 [928,160,710,572]
+  CRUSH rule 0 x 630 [243,19,918,556]
+  CRUSH rule 0 x 631 [438,221,574,676]
+  CRUSH rule 0 x 632 [797,368,247,5]
+  CRUSH rule 0 x 633 [993,749,525,485]
+  CRUSH rule 0 x 634 [239,351,633,299]
+  CRUSH rule 0 x 635 [640,965,25,961]
+  CRUSH rule 0 x 636 [173,290,297,991]
+  CRUSH rule 0 x 637 [0,918,98,108]
+  CRUSH rule 0 x 638 [702,235,424,900]
+  CRUSH rule 0 x 639 [475,687,31,785]
+  CRUSH rule 0 x 640 [31,664,399,677]
+  CRUSH rule 0 x 641 [296,473,108,963]
+  CRUSH rule 0 x 642 [894,273,427,606]
+  CRUSH rule 0 x 643 [117,111,732,191]
+  CRUSH rule 0 x 644 [438,336,327,512]
+  CRUSH rule 0 x 645 [982,702,351,573]
+  CRUSH rule 0 x 646 [334,804,146,842]
+  CRUSH rule 0 x 647 [933,787,185,334]
+  CRUSH rule 0 x 648 [22,444,400,862]
+  CRUSH rule 0 x 649 [503,229,213,460]
+  CRUSH rule 0 x 650 [328,659,420,443]
+  CRUSH rule 0 x 651 [3,880,823,123]
+  CRUSH rule 0 x 652 [495,977,563,733]
+  CRUSH rule 0 x 653 [185,718,804,280]
+  CRUSH rule 0 x 654 [130,528,380,81]
+  CRUSH rule 0 x 655 [560,872,454,504]
+  CRUSH rule 0 x 656 [219,885,178,981]
+  CRUSH rule 0 x 657 [233,684,813,490]
+  CRUSH rule 0 x 658 [778,6,756,380]
+  CRUSH rule 0 x 659 [240,663,306,540]
+  CRUSH rule 0 x 660 [244,855,196,147]
+  CRUSH rule 0 x 661 [184,270,128,398]
+  CRUSH rule 0 x 662 [65,883,921,438]
+  CRUSH rule 0 x 663 [323,721,594,812]
+  CRUSH rule 0 x 664 [865,113,512,51]
+  CRUSH rule 0 x 665 [420,850,591,475]
+  CRUSH rule 0 x 666 [319,767,246,3]
+  CRUSH rule 0 x 667 [875,39,343,100]
+  CRUSH rule 0 x 668 [331,122,263,599]
+  CRUSH rule 0 x 669 [915,521,402,747]
+  CRUSH rule 0 x 670 [845,659,943,447]
+  CRUSH rule 0 x 671 [108,634,527,363]
+  CRUSH rule 0 x 672 [578,216,110,589]
+  CRUSH rule 0 x 673 [442,74,579,797]
+  CRUSH rule 0 x 674 [588,364,281,308]
+  CRUSH rule 0 x 675 [489,698,744,671]
+  CRUSH rule 0 x 676 [928,911,40,180]
+  CRUSH rule 0 x 677 [399,269,692,131]
+  CRUSH rule 0 x 678 [546,752,544,155]
+  CRUSH rule 0 x 679 [988,25,275,433]
+  CRUSH rule 0 x 680 [335,963,382,486]
+  CRUSH rule 0 x 681 [690,462,623,466]
+  CRUSH rule 0 x 682 [196,588,154,257]
+  CRUSH rule 0 x 683 [627,25,421,160]
+  CRUSH rule 0 x 684 [38,804,592,158]
+  CRUSH rule 0 x 685 [841,368,548,362]
+  CRUSH rule 0 x 686 [336,287,525,440]
+  CRUSH rule 0 x 687 [20,682,924,653]
+  CRUSH rule 0 x 688 [463,371,780,556]
+  CRUSH rule 0 x 689 [569,250,78,816]
+  CRUSH rule 0 x 690 [551,144,587,263]
+  CRUSH rule 0 x 691 [766,464,446,533]
+  CRUSH rule 0 x 692 [739,634,18,245]
+  CRUSH rule 0 x 693 [339,297,118,330]
+  CRUSH rule 0 x 694 [405,26,830,181]
+  CRUSH rule 0 x 695 [622,576,597,535]
+  CRUSH rule 0 x 696 [558,902,689,13]
+  CRUSH rule 0 x 697 [818,222,406,691]
+  CRUSH rule 0 x 698 [178,48,402,233]
+  CRUSH rule 0 x 699 [450,244,180,919]
+  CRUSH rule 0 x 700 [502,771,987,706]
+  CRUSH rule 0 x 701 [4,612,782,216]
+  CRUSH rule 0 x 702 [177,630,232,923]
+  CRUSH rule 0 x 703 [354,178,389,393]
+  CRUSH rule 0 x 704 [646,601,156,171]
+  CRUSH rule 0 x 705 [921,401,890,265]
+  CRUSH rule 0 x 706 [652,877,562,452]
+  CRUSH rule 0 x 707 [345,745,67,716]
+  CRUSH rule 0 x 708 [333,607,180,469]
+  CRUSH rule 0 x 709 [45,187,302,115]
+  CRUSH rule 0 x 710 [94,855,43,199]
+  CRUSH rule 0 x 711 [227,653,731,150]
+  CRUSH rule 0 x 712 [398,953,136,870]
+  CRUSH rule 0 x 713 [116,800,503,662]
+  CRUSH rule 0 x 714 [111,629,866,709]
+  CRUSH rule 0 x 715 [531,291,486,382]
+  CRUSH rule 0 x 716 [169,541,291,42]
+  CRUSH rule 0 x 717 [417,446,994,894]
+  CRUSH rule 0 x 718 [992,383,298,844]
+  CRUSH rule 0 x 719 [936,674,324,759]
+  CRUSH rule 0 x 720 [370,188,174,464]
+  CRUSH rule 0 x 721 [320,859,278,259]
+  CRUSH rule 0 x 722 [7,2,673,129]
+  CRUSH rule 0 x 723 [270,553,831,662]
+  CRUSH rule 0 x 724 [666,822,708,895]
+  CRUSH rule 0 x 725 [794,406,875,459]
+  CRUSH rule 0 x 726 [420,556,341,292]
+  CRUSH rule 0 x 727 [561,461,129,635]
+  CRUSH rule 0 x 728 [951,330,196,756]
+  CRUSH rule 0 x 729 [656,644,436,591]
+  CRUSH rule 0 x 730 [3,558,629,184]
+  CRUSH rule 0 x 731 [852,89,75,735]
+  CRUSH rule 0 x 732 [983,840,869,976]
+  CRUSH rule 0 x 733 [285,396,388,122]
+  CRUSH rule 0 x 734 [125,510,402,640]
+  CRUSH rule 0 x 735 [417,773,686,504]
+  CRUSH rule 0 x 736 [749,396,632,550]
+  CRUSH rule 0 x 737 [644,991,946,135]
+  CRUSH rule 0 x 738 [449,683,290,220]
+  CRUSH rule 0 x 739 [341,220,641,454]
+  CRUSH rule 0 x 740 [874,524,674,650]
+  CRUSH rule 0 x 741 [189,472,712,798]
+  CRUSH rule 0 x 742 [912,581,114,730]
+  CRUSH rule 0 x 743 [654,914,425,441]
+  CRUSH rule 0 x 744 [725,295,579,377]
+  CRUSH rule 0 x 745 [787,858,850,506]
+  CRUSH rule 0 x 746 [757,848,704,30]
+  CRUSH rule 0 x 747 [700,81,867,681]
+  CRUSH rule 0 x 748 [557,436,238,664]
+  CRUSH rule 0 x 749 [772,622,337,42]
+  CRUSH rule 0 x 750 [946,97,376,677]
+  CRUSH rule 0 x 751 [996,618,343,911]
+  CRUSH rule 0 x 752 [746,887,695,868]
+  CRUSH rule 0 x 753 [741,14,463,479]
+  CRUSH rule 0 x 754 [648,349,333,355]
+  CRUSH rule 0 x 755 [157,460,466,187]
+  CRUSH rule 0 x 756 [416,97,197,497]
+  CRUSH rule 0 x 757 [599,839,776,410]
+  CRUSH rule 0 x 758 [994,218,620,256]
+  CRUSH rule 0 x 759 [959,682,514,745]
+  CRUSH rule 0 x 760 [518,943,215,83]
+  CRUSH rule 0 x 761 [285,849,420,324]
+  CRUSH rule 0 x 762 [591,313,41,335]
+  CRUSH rule 0 x 763 [908,411,200,740]
+  CRUSH rule 0 x 764 [787,234,894,485]
+  CRUSH rule 0 x 765 [327,921,882,393]
+  CRUSH rule 0 x 766 [84,161,878,704]
+  CRUSH rule 0 x 767 [370,895,702,701]
+  CRUSH rule 0 x 768 [826,760,879,864]
+  CRUSH rule 0 x 769 [67,768,663,735]
+  CRUSH rule 0 x 770 [593,909,482,259]
+  CRUSH rule 0 x 771 [309,935,121,578]
+  CRUSH rule 0 x 772 [12,125,797,301]
+  CRUSH rule 0 x 773 [253,466,820,549]
+  CRUSH rule 0 x 774 [164,390,705,109]
+  CRUSH rule 0 x 775 [703,47,43,973]
+  CRUSH rule 0 x 776 [728,231,80,916]
+  CRUSH rule 0 x 777 [981,621,568,729]
+  CRUSH rule 0 x 778 [411,456,544,597]
+  CRUSH rule 0 x 779 [346,121,519,921]
+  CRUSH rule 0 x 780 [476,39,288,381]
+  CRUSH rule 0 x 781 [10,130,585,844]
+  CRUSH rule 0 x 782 [462,246,581,902]
+  CRUSH rule 0 x 783 [580,373,153,775]
+  CRUSH rule 0 x 784 [413,113,978,990]
+  CRUSH rule 0 x 785 [341,856,332,354]
+  CRUSH rule 0 x 786 [411,140,313,393]
+  CRUSH rule 0 x 787 [605,522,211,813]
+  CRUSH rule 0 x 788 [226,545,35,142]
+  CRUSH rule 0 x 789 [545,320,414,702]
+  CRUSH rule 0 x 790 [414,748,816,327]
+  CRUSH rule 0 x 791 [660,906,406,697]
+  CRUSH rule 0 x 792 [287,392,514,204]
+  CRUSH rule 0 x 793 [631,133,850,713]
+  CRUSH rule 0 x 794 [931,517,543,210]
+  CRUSH rule 0 x 795 [551,962,477,948]
+  CRUSH rule 0 x 796 [814,4,95,27]
+  CRUSH rule 0 x 797 [64,201,299,734]
+  CRUSH rule 0 x 798 [422,530,114,431]
+  CRUSH rule 0 x 799 [824,32,679,562]
+  CRUSH rule 0 x 800 [862,623,489,637]
+  CRUSH rule 0 x 801 [145,550,329,324]
+  CRUSH rule 0 x 802 [570,19,847,308]
+  CRUSH rule 0 x 803 [151,812,662,358]
+  CRUSH rule 0 x 804 [467,93,264,863]
+  CRUSH rule 0 x 805 [621,223,938,809]
+  CRUSH rule 0 x 806 [898,957,805,430]
+  CRUSH rule 0 x 807 [354,531,422,159]
+  CRUSH rule 0 x 808 [7,96,76,897]
+  CRUSH rule 0 x 809 [70,734,719,56]
+  CRUSH rule 0 x 810 [701,18,972,327]
+  CRUSH rule 0 x 811 [248,547,103,728]
+  CRUSH rule 0 x 812 [230,576,821,566]
+  CRUSH rule 0 x 813 [805,114,683,629]
+  CRUSH rule 0 x 814 [54,619,973,741]
+  CRUSH rule 0 x 815 [679,412,613,132]
+  CRUSH rule 0 x 816 [919,448,826,414]
+  CRUSH rule 0 x 817 [765,830,436,521]
+  CRUSH rule 0 x 818 [415,566,644,687]
+  CRUSH rule 0 x 819 [721,319,865,750]
+  CRUSH rule 0 x 820 [218,301,333,190]
+  CRUSH rule 0 x 821 [185,795,680,953]
+  CRUSH rule 0 x 822 [356,261,54,522]
+  CRUSH rule 0 x 823 [220,281,549,456]
+  CRUSH rule 0 x 824 [292,809,887,74]
+  CRUSH rule 0 x 825 [949,778,101,311]
+  CRUSH rule 0 x 826 [767,818,833,927]
+  CRUSH rule 0 x 827 [631,83,406,635]
+  CRUSH rule 0 x 828 [288,986,445,26]
+  CRUSH rule 0 x 829 [990,667,915,694]
+  CRUSH rule 0 x 830 [152,571,778,505]
+  CRUSH rule 0 x 831 [814,563,630,97]
+  CRUSH rule 0 x 832 [235,641,616,110]
+  CRUSH rule 0 x 833 [657,565,922,140]
+  CRUSH rule 0 x 834 [907,231,644,13]
+  CRUSH rule 0 x 835 [784,262,771,264]
+  CRUSH rule 0 x 836 [951,158,366,710]
+  CRUSH rule 0 x 837 [556,498,334,633]
+  CRUSH rule 0 x 838 [329,274,964,547]
+  CRUSH rule 0 x 839 [568,209,939,364]
+  CRUSH rule 0 x 840 [45,579,842,70]
+  CRUSH rule 0 x 841 [652,702,24,605]
+  CRUSH rule 0 x 842 [629,984,314,895]
+  CRUSH rule 0 x 843 [799,690,688,648]
+  CRUSH rule 0 x 844 [694,600,534,700]
+  CRUSH rule 0 x 845 [332,30,179,93]
+  CRUSH rule 0 x 846 [452,251,712,719]
+  CRUSH rule 0 x 847 [399,681,847,739]
+  CRUSH rule 0 x 848 [303,138,440,346]
+  CRUSH rule 0 x 849 [666,346,708,873]
+  CRUSH rule 0 x 850 [644,511,345,844]
+  CRUSH rule 0 x 851 [527,546,737,425]
+  CRUSH rule 0 x 852 [31,809,94,618]
+  CRUSH rule 0 x 853 [483,330,869,184]
+  CRUSH rule 0 x 854 [697,953,968,143]
+  CRUSH rule 0 x 855 [837,996,239,621]
+  CRUSH rule 0 x 856 [712,40,547,430]
+  CRUSH rule 0 x 857 [77,984,576,551]
+  CRUSH rule 0 x 858 [412,384,841,465]
+  CRUSH rule 0 x 859 [173,760,26,300]
+  CRUSH rule 0 x 860 [776,429,328,917]
+  CRUSH rule 0 x 861 [705,405,477,50]
+  CRUSH rule 0 x 862 [809,44,788,938]
+  CRUSH rule 0 x 863 [349,496,963,178]
+  CRUSH rule 0 x 864 [717,858,101,239]
+  CRUSH rule 0 x 865 [857,603,586,262]
+  CRUSH rule 0 x 866 [394,304,71,96]
+  CRUSH rule 0 x 867 [640,773,663,974]
+  CRUSH rule 0 x 868 [613,950,712,663]
+  CRUSH rule 0 x 869 [973,889,524,22]
+  CRUSH rule 0 x 870 [505,35,386,498]
+  CRUSH rule 0 x 871 [239,264,262,773]
+  CRUSH rule 0 x 872 [21,767,456,748]
+  CRUSH rule 0 x 873 [954,666,980,264]
+  CRUSH rule 0 x 874 [54,510,947,1]
+  CRUSH rule 0 x 875 [809,418,452,462]
+  CRUSH rule 0 x 876 [483,457,61,248]
+  CRUSH rule 0 x 877 [542,531,952,939]
+  CRUSH rule 0 x 878 [217,674,857,644]
+  CRUSH rule 0 x 879 [999,475,134,250]
+  CRUSH rule 0 x 880 [678,573,935,385]
+  CRUSH rule 0 x 881 [394,835,789,802]
+  CRUSH rule 0 x 882 [467,382,353,56]
+  CRUSH rule 0 x 883 [802,744,237,337]
+  CRUSH rule 0 x 884 [653,660,638,700]
+  CRUSH rule 0 x 885 [898,704,307,445]
+  CRUSH rule 0 x 886 [434,357,938,641]
+  CRUSH rule 0 x 887 [297,226,711,428]
+  CRUSH rule 0 x 888 [863,324,443,213]
+  CRUSH rule 0 x 889 [105,102,308,163]
+  CRUSH rule 0 x 890 [550,248,606,704]
+  CRUSH rule 0 x 891 [575,928,880,891]
+  CRUSH rule 0 x 892 [259,862,133,271]
+  CRUSH rule 0 x 893 [902,880,543,542]
+  CRUSH rule 0 x 894 [180,169,916,43]
+  CRUSH rule 0 x 895 [725,849,182,129]
+  CRUSH rule 0 x 896 [951,34,874,537]
+  CRUSH rule 0 x 897 [810,352,73,939]
+  CRUSH rule 0 x 898 [979,433,719,411]
+  CRUSH rule 0 x 899 [685,668,534,932]
+  CRUSH rule 0 x 900 [530,978,41,894]
+  CRUSH rule 0 x 901 [740,107,336,175]
+  CRUSH rule 0 x 902 [800,743,693,310]
+  CRUSH rule 0 x 903 [230,267,842,266]
+  CRUSH rule 0 x 904 [346,949,460,973]
+  CRUSH rule 0 x 905 [530,397,619,958]
+  CRUSH rule 0 x 906 [80,426,138,672]
+  CRUSH rule 0 x 907 [365,968,475,297]
+  CRUSH rule 0 x 908 [204,832,742,809]
+  CRUSH rule 0 x 909 [883,989,146,959]
+  CRUSH rule 0 x 910 [549,593,249,853]
+  CRUSH rule 0 x 911 [325,847,352,214]
+  CRUSH rule 0 x 912 [874,888,582,796]
+  CRUSH rule 0 x 913 [331,463,342,574]
+  CRUSH rule 0 x 914 [836,468,601,732]
+  CRUSH rule 0 x 915 [245,228,100,661]
+  CRUSH rule 0 x 916 [77,967,364,435]
+  CRUSH rule 0 x 917 [239,60,866,221]
+  CRUSH rule 0 x 918 [988,115,922,80]
+  CRUSH rule 0 x 919 [783,139,696,1]
+  CRUSH rule 0 x 920 [623,408,685,953]
+  CRUSH rule 0 x 921 [105,799,144,90]
+  CRUSH rule 0 x 922 [887,505,652,348]
+  CRUSH rule 0 x 923 [223,318,552,458]
+  CRUSH rule 0 x 924 [25,778,366,333]
+  CRUSH rule 0 x 925 [912,601,297,682]
+  CRUSH rule 0 x 926 [968,133,144,814]
+  CRUSH rule 0 x 927 [277,724,214,988]
+  CRUSH rule 0 x 928 [554,203,658,789]
+  CRUSH rule 0 x 929 [761,802,367,528]
+  CRUSH rule 0 x 930 [814,61,788,736]
+  CRUSH rule 0 x 931 [29,193,61,41]
+  CRUSH rule 0 x 932 [446,198,862,534]
+  CRUSH rule 0 x 933 [352,742,216,321]
+  CRUSH rule 0 x 934 [730,2,332,631]
+  CRUSH rule 0 x 935 [731,23,736,79]
+  CRUSH rule 0 x 936 [322,975,20,904]
+  CRUSH rule 0 x 937 [822,221,841,161]
+  CRUSH rule 0 x 938 [557,850,66,630]
+  CRUSH rule 0 x 939 [150,11,971,371]
+  CRUSH rule 0 x 940 [638,398,169,616]
+  CRUSH rule 0 x 941 [730,342,929,577]
+  CRUSH rule 0 x 942 [62,292,166,814]
+  CRUSH rule 0 x 943 [165,314,519,548]
+  CRUSH rule 0 x 944 [199,625,766,176]
+  CRUSH rule 0 x 945 [946,999,699,303]
+  CRUSH rule 0 x 946 [595,93,852,142]
+  CRUSH rule 0 x 947 [800,582,356,93]
+  CRUSH rule 0 x 948 [132,551,139,920]
+  CRUSH rule 0 x 949 [792,920,466,380]
+  CRUSH rule 0 x 950 [111,345,176,543]
+  CRUSH rule 0 x 951 [414,619,648,655]
+  CRUSH rule 0 x 952 [775,469,500,356]
+  CRUSH rule 0 x 953 [349,1,5,251]
+  CRUSH rule 0 x 954 [570,940,410,249]
+  CRUSH rule 0 x 955 [729,774,823,800]
+  CRUSH rule 0 x 956 [519,141,575,625]
+  CRUSH rule 0 x 957 [242,709,611,97]
+  CRUSH rule 0 x 958 [84,217,227,253]
+  CRUSH rule 0 x 959 [270,413,918,789]
+  CRUSH rule 0 x 960 [458,192,307,279]
+  CRUSH rule 0 x 961 [981,388,777,546]
+  CRUSH rule 0 x 962 [623,834,277,134]
+  CRUSH rule 0 x 963 [291,167,714,468]
+  CRUSH rule 0 x 964 [28,156,788,127]
+  CRUSH rule 0 x 965 [675,557,290,517]
+  CRUSH rule 0 x 966 [836,306,946,283]
+  CRUSH rule 0 x 967 [966,386,735,837]
+  CRUSH rule 0 x 968 [864,756,690,121]
+  CRUSH rule 0 x 969 [729,625,480,769]
+  CRUSH rule 0 x 970 [800,362,646,582]
+  CRUSH rule 0 x 971 [737,381,153,684]
+  CRUSH rule 0 x 972 [952,245,720,884]
+  CRUSH rule 0 x 973 [356,455,579,857]
+  CRUSH rule 0 x 974 [545,758,586,596]
+  CRUSH rule 0 x 975 [336,191,202,146]
+  CRUSH rule 0 x 976 [446,208,757,620]
+  CRUSH rule 0 x 977 [202,896,196,956]
+  CRUSH rule 0 x 978 [612,324,996,225]
+  CRUSH rule 0 x 979 [843,457,675,650]
+  CRUSH rule 0 x 980 [60,914,881,626]
+  CRUSH rule 0 x 981 [702,749,937,153]
+  CRUSH rule 0 x 982 [298,928,738,167]
+  CRUSH rule 0 x 983 [723,572,395,358]
+  CRUSH rule 0 x 984 [723,864,804,935]
+  CRUSH rule 0 x 985 [945,459,868,211]
+  CRUSH rule 0 x 986 [772,664,535,169]
+  CRUSH rule 0 x 987 [88,324,312,843]
+  CRUSH rule 0 x 988 [522,927,131,996]
+  CRUSH rule 0 x 989 [578,332,208,605]
+  CRUSH rule 0 x 990 [638,228,414,311]
+  CRUSH rule 0 x 991 [530,221,451,422]
+  CRUSH rule 0 x 992 [925,705,275,81]
+  CRUSH rule 0 x 993 [991,301,43,469]
+  CRUSH rule 0 x 994 [276,51,868,683]
+  CRUSH rule 0 x 995 [288,836,753,790]
+  CRUSH rule 0 x 996 [887,983,252,686]
+  CRUSH rule 0 x 997 [110,924,386,79]
+  CRUSH rule 0 x 998 [435,830,485,853]
+  CRUSH rule 0 x 999 [876,738,357,913]
+  CRUSH rule 0 x 1000 [178,963,638,430]
+  CRUSH rule 0 x 1001 [99,519,66,759]
+  CRUSH rule 0 x 1002 [515,534,468,866]
+  CRUSH rule 0 x 1003 [104,611,937,698]
+  CRUSH rule 0 x 1004 [269,638,724,375]
+  CRUSH rule 0 x 1005 [369,223,309,409]
+  CRUSH rule 0 x 1006 [40,107,69,275]
+  CRUSH rule 0 x 1007 [978,111,416,758]
+  CRUSH rule 0 x 1008 [965,956,624,832]
+  CRUSH rule 0 x 1009 [598,476,356,695]
+  CRUSH rule 0 x 1010 [767,523,239,517]
+  CRUSH rule 0 x 1011 [289,871,207,576]
+  CRUSH rule 0 x 1012 [128,28,370,31]
+  CRUSH rule 0 x 1013 [979,765,660,812]
+  CRUSH rule 0 x 1014 [979,948,513,88]
+  CRUSH rule 0 x 1015 [277,790,396,672]
+  CRUSH rule 0 x 1016 [262,73,128,886]
+  CRUSH rule 0 x 1017 [150,269,61,499]
+  CRUSH rule 0 x 1018 [555,829,554,944]
+  CRUSH rule 0 x 1019 [513,356,265,446]
+  CRUSH rule 0 x 1020 [158,161,877,704]
+  CRUSH rule 0 x 1021 [915,998,957,285]
+  CRUSH rule 0 x 1022 [967,829,973,640]
+  CRUSH rule 0 x 1023 [488,257,614,859]
+  rule 0 (data) num_rep 4 result size == 4:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604]
+  CRUSH rule 0 x 1 [876,250,334,633,744]
+  CRUSH rule 0 x 2 [292,832,53,392,386]
+  CRUSH rule 0 x 3 [623,387,124,998,749]
+  CRUSH rule 0 x 4 [61,334,710,4,994]
+  CRUSH rule 0 x 5 [946,557,713,664,141]
+  CRUSH rule 0 x 6 [576,668,212,163,732]
+  CRUSH rule 0 x 7 [645,753,906,393,341]
+  CRUSH rule 0 x 8 [243,6,863,781,211]
+  CRUSH rule 0 x 9 [22,578,251,410,297]
+  CRUSH rule 0 x 10 [758,828,360,477,821]
+  CRUSH rule 0 x 11 [769,120,124,527,119]
+  CRUSH rule 0 x 12 [780,364,689,755,675]
+  CRUSH rule 0 x 13 [557,18,351,719,742]
+  CRUSH rule 0 x 14 [59,561,249,461,971]
+  CRUSH rule 0 x 15 [718,928,993,21,76]
+  CRUSH rule 0 x 16 [673,632,841,954,788]
+  CRUSH rule 0 x 17 [648,43,560,514,142]
+  CRUSH rule 0 x 18 [654,219,181,568,381]
+  CRUSH rule 0 x 19 [850,545,377,848,863]
+  CRUSH rule 0 x 20 [717,785,974,5,225]
+  CRUSH rule 0 x 21 [420,57,519,306,312]
+  CRUSH rule 0 x 22 [503,998,193,821,634]
+  CRUSH rule 0 x 23 [411,663,168,110,899]
+  CRUSH rule 0 x 24 [266,861,353,1,456]
+  CRUSH rule 0 x 25 [760,483,818,600,509]
+  CRUSH rule 0 x 26 [903,24,573,718,112]
+  CRUSH rule 0 x 27 [946,188,289,510,687]
+  CRUSH rule 0 x 28 [69,312,73,198,256]
+  CRUSH rule 0 x 29 [844,883,337,628,496]
+  CRUSH rule 0 x 30 [621,18,613,794,910]
+  CRUSH rule 0 x 31 [784,943,814,539,962]
+  CRUSH rule 0 x 32 [173,374,369,972,315]
+  CRUSH rule 0 x 33 [698,336,357,966,582]
+  CRUSH rule 0 x 34 [168,836,210,798,904]
+  CRUSH rule 0 x 35 [274,509,534,818,912]
+  CRUSH rule 0 x 36 [318,215,153,628,87]
+  CRUSH rule 0 x 37 [173,604,109,935,203]
+  CRUSH rule 0 x 38 [708,444,683,604,722]
+  CRUSH rule 0 x 39 [662,198,417,680,226]
+  CRUSH rule 0 x 40 [620,801,414,78,560]
+  CRUSH rule 0 x 41 [811,264,177,127,148]
+  CRUSH rule 0 x 42 [863,179,527,660,133]
+  CRUSH rule 0 x 43 [686,822,988,228,791]
+  CRUSH rule 0 x 44 [396,222,46,841,536]
+  CRUSH rule 0 x 45 [991,694,253,142,54]
+  CRUSH rule 0 x 46 [420,909,184,285,508]
+  CRUSH rule 0 x 47 [467,211,605,207,241]
+  CRUSH rule 0 x 48 [955,329,368,168,698]
+  CRUSH rule 0 x 49 [974,891,931,29,813]
+  CRUSH rule 0 x 50 [870,441,691,823,761]
+  CRUSH rule 0 x 51 [182,930,25,936,97]
+  CRUSH rule 0 x 52 [704,812,894,794,481]
+  CRUSH rule 0 x 53 [185,713,631,280,345]
+  CRUSH rule 0 x 54 [270,441,100,82,983]
+  CRUSH rule 0 x 55 [895,734,958,793,651]
+  CRUSH rule 0 x 56 [564,963,683,324,40]
+  CRUSH rule 0 x 57 [738,130,208,973,498]
+  CRUSH rule 0 x 58 [524,113,806,903,531]
+  CRUSH rule 0 x 59 [408,337,668,529,34]
+  CRUSH rule 0 x 60 [228,790,857,309,616]
+  CRUSH rule 0 x 61 [154,843,717,467,883]
+  CRUSH rule 0 x 62 [594,811,549,276,693]
+  CRUSH rule 0 x 63 [646,67,884,925,941]
+  CRUSH rule 0 x 64 [175,542,155,837,594]
+  CRUSH rule 0 x 65 [745,619,131,867,269]
+  CRUSH rule 0 x 66 [275,468,23,35,328]
+  CRUSH rule 0 x 67 [246,958,524,493,636]
+  CRUSH rule 0 x 68 [711,473,403,228,835]
+  CRUSH rule 0 x 69 [493,924,850,939,950]
+  CRUSH rule 0 x 70 [30,499,644,33,804]
+  CRUSH rule 0 x 71 [984,883,574,716,575]
+  CRUSH rule 0 x 72 [71,286,942,363,628]
+  CRUSH rule 0 x 73 [922,618,3,371,464]
+  CRUSH rule 0 x 74 [629,414,185,573,678]
+  CRUSH rule 0 x 75 [222,20,174,820,312]
+  CRUSH rule 0 x 76 [262,366,339,290,718]
+  CRUSH rule 0 x 77 [638,469,992,280,773]
+  CRUSH rule 0 x 78 [324,511,788,7,308]
+  CRUSH rule 0 x 79 [577,990,64,94,447]
+  CRUSH rule 0 x 80 [501,95,278,903,631]
+  CRUSH rule 0 x 81 [506,812,9,698,173]
+  CRUSH rule 0 x 82 [222,145,80,785,835]
+  CRUSH rule 0 x 83 [71,634,61,91,856]
+  CRUSH rule 0 x 84 [49,761,773,368,318]
+  CRUSH rule 0 x 85 [985,896,708,861,325]
+  CRUSH rule 0 x 86 [537,745,93,524,466]
+  CRUSH rule 0 x 87 [997,317,463,626,685]
+  CRUSH rule 0 x 88 [957,350,890,857,375]
+  CRUSH rule 0 x 89 [399,730,148,314,159]
+  CRUSH rule 0 x 90 [943,706,683,267,579]
+  CRUSH rule 0 x 91 [22,368,149,928,140]
+  CRUSH rule 0 x 92 [532,424,426,773,623]
+  CRUSH rule 0 x 93 [218,489,405,681,549]
+  CRUSH rule 0 x 94 [181,96,102,515,776]
+  CRUSH rule 0 x 95 [343,957,820,139,334]
+  CRUSH rule 0 x 96 [861,270,87,797,0]
+  CRUSH rule 0 x 97 [459,706,45,328,274]
+  CRUSH rule 0 x 98 [327,867,353,948,728]
+  CRUSH rule 0 x 99 [974,133,468,906,235]
+  CRUSH rule 0 x 100 [32,445,547,371,960]
+  CRUSH rule 0 x 101 [142,90,337,950,970]
+  CRUSH rule 0 x 102 [172,129,139,22,403]
+  CRUSH rule 0 x 103 [630,47,161,356,911]
+  CRUSH rule 0 x 104 [758,133,278,11,947]
+  CRUSH rule 0 x 105 [843,604,47,33,401]
+  CRUSH rule 0 x 106 [28,681,193,679,990]
+  CRUSH rule 0 x 107 [74,320,85,819,315]
+  CRUSH rule 0 x 108 [875,593,575,517,107]
+  CRUSH rule 0 x 109 [411,985,811,720,198]
+  CRUSH rule 0 x 110 [440,774,799,660,715]
+  CRUSH rule 0 x 111 [405,742,276,359,936]
+  CRUSH rule 0 x 112 [143,181,922,545,185]
+  CRUSH rule 0 x 113 [153,846,160,903,789]
+  CRUSH rule 0 x 114 [804,892,939,20,312]
+  CRUSH rule 0 x 115 [588,508,958,580,232]
+  CRUSH rule 0 x 116 [327,148,637,486,712]
+  CRUSH rule 0 x 117 [95,594,989,131,714]
+  CRUSH rule 0 x 118 [80,957,897,239,359]
+  CRUSH rule 0 x 119 [386,932,951,768,679]
+  CRUSH rule 0 x 120 [366,312,653,936,71]
+  CRUSH rule 0 x 121 [129,154,847,16,471]
+  CRUSH rule 0 x 122 [873,1,110,939,90]
+  CRUSH rule 0 x 123 [533,415,789,600,713]
+  CRUSH rule 0 x 124 [461,691,898,723,957]
+  CRUSH rule 0 x 125 [342,599,830,402,615]
+  CRUSH rule 0 x 126 [819,781,822,548,279]
+  CRUSH rule 0 x 127 [437,893,585,707,353]
+  CRUSH rule 0 x 128 [679,994,982,550,991]
+  CRUSH rule 0 x 129 [380,685,947,302,698]
+  CRUSH rule 0 x 130 [992,52,466,867,998]
+  CRUSH rule 0 x 131 [469,90,208,599,829]
+  CRUSH rule 0 x 132 [571,250,316,535,54]
+  CRUSH rule 0 x 133 [964,728,329,902,108]
+  CRUSH rule 0 x 134 [999,19,716,963,323]
+  CRUSH rule 0 x 135 [634,101,52,938,413]
+  CRUSH rule 0 x 136 [114,889,692,768,694]
+  CRUSH rule 0 x 137 [839,8,959,280,922]
+  CRUSH rule 0 x 138 [967,949,138,451,292]
+  CRUSH rule 0 x 139 [308,711,736,247,632]
+  CRUSH rule 0 x 140 [764,936,926,55,331]
+  CRUSH rule 0 x 141 [423,302,112,216,603]
+  CRUSH rule 0 x 142 [252,821,715,340,635]
+  CRUSH rule 0 x 143 [33,808,518,477,325]
+  CRUSH rule 0 x 144 [472,88,969,162,401]
+  CRUSH rule 0 x 145 [242,208,252,604,266]
+  CRUSH rule 0 x 146 [290,70,570,384,934]
+  CRUSH rule 0 x 147 [447,352,657,493,467]
+  CRUSH rule 0 x 148 [212,644,432,658,109]
+  CRUSH rule 0 x 149 [9,775,87,35,260]
+  CRUSH rule 0 x 150 [166,456,582,144,324]
+  CRUSH rule 0 x 151 [811,875,307,20,782]
+  CRUSH rule 0 x 152 [449,617,223,9,182]
+  CRUSH rule 0 x 153 [523,537,695,627,959]
+  CRUSH rule 0 x 154 [208,559,874,597,243]
+  CRUSH rule 0 x 155 [569,325,192,296,367]
+  CRUSH rule 0 x 156 [488,121,521,213,595]
+  CRUSH rule 0 x 157 [140,723,633,260,487]
+  CRUSH rule 0 x 158 [786,451,320,239,667]
+  CRUSH rule 0 x 159 [134,664,517,821,667]
+  CRUSH rule 0 x 160 [690,112,414,990,183]
+  CRUSH rule 0 x 161 [324,912,397,423,991]
+  CRUSH rule 0 x 162 [748,567,284,183,463]
+  CRUSH rule 0 x 163 [575,499,31,816,749]
+  CRUSH rule 0 x 164 [314,489,308,326,51]
+  CRUSH rule 0 x 165 [116,209,750,53,813]
+  CRUSH rule 0 x 166 [352,706,701,810,718]
+  CRUSH rule 0 x 167 [27,743,174,142,551]
+  CRUSH rule 0 x 168 [953,898,880,660,500]
+  CRUSH rule 0 x 169 [912,147,266,547,331]
+  CRUSH rule 0 x 170 [421,515,828,844,151]
+  CRUSH rule 0 x 171 [488,584,880,964,936]
+  CRUSH rule 0 x 172 [366,443,957,66,162]
+  CRUSH rule 0 x 173 [863,291,625,287,158]
+  CRUSH rule 0 x 174 [263,555,650,410,339]
+  CRUSH rule 0 x 175 [875,961,361,575,33]
+  CRUSH rule 0 x 176 [745,83,701,680,250]
+  CRUSH rule 0 x 177 [128,244,41,123,422]
+  CRUSH rule 0 x 178 [155,41,264,777,314]
+  CRUSH rule 0 x 179 [593,833,202,183,971]
+  CRUSH rule 0 x 180 [154,734,17,831,824]
+  CRUSH rule 0 x 181 [289,675,723,800,166]
+  CRUSH rule 0 x 182 [730,931,560,209,943]
+  CRUSH rule 0 x 183 [639,237,794,815,827]
+  CRUSH rule 0 x 184 [704,312,685,645,691]
+  CRUSH rule 0 x 185 [97,100,762,82,999]
+  CRUSH rule 0 x 186 [26,665,554,215,280]
+  CRUSH rule 0 x 187 [649,14,740,494,402]
+  CRUSH rule 0 x 188 [682,695,590,743,927]
+  CRUSH rule 0 x 189 [325,693,726,51,448]
+  CRUSH rule 0 x 190 [399,933,136,955,57]
+  CRUSH rule 0 x 191 [629,533,17,126,60]
+  CRUSH rule 0 x 192 [503,578,38,492,222]
+  CRUSH rule 0 x 193 [546,333,651,678,823]
+  CRUSH rule 0 x 194 [242,473,58,655,277]
+  CRUSH rule 0 x 195 [625,719,135,81,636]
+  CRUSH rule 0 x 196 [357,114,125,867,250]
+  CRUSH rule 0 x 197 [306,954,453,873,211]
+  CRUSH rule 0 x 198 [863,791,311,911,206]
+  CRUSH rule 0 x 199 [935,906,929,252,893]
+  CRUSH rule 0 x 200 [373,774,229,454,909]
+  CRUSH rule 0 x 201 [659,320,477,313,779]
+  CRUSH rule 0 x 202 [260,433,524,880,223]
+  CRUSH rule 0 x 203 [36,239,675,971,703]
+  CRUSH rule 0 x 204 [92,516,993,728,279]
+  CRUSH rule 0 x 205 [68,395,473,45,683]
+  CRUSH rule 0 x 206 [570,530,642,380,311]
+  CRUSH rule 0 x 207 [834,457,850,917,456]
+  CRUSH rule 0 x 208 [927,484,640,976,803]
+  CRUSH rule 0 x 209 [878,66,58,940,48]
+  CRUSH rule 0 x 210 [572,981,484,29,0]
+  CRUSH rule 0 x 211 [107,597,780,857,895]
+  CRUSH rule 0 x 212 [389,107,838,624,698]
+  CRUSH rule 0 x 213 [497,717,567,728,905]
+  CRUSH rule 0 x 214 [798,65,254,572,32]
+  CRUSH rule 0 x 215 [233,419,283,638,520]
+  CRUSH rule 0 x 216 [494,464,742,523,459]
+  CRUSH rule 0 x 217 [352,396,309,938,66]
+  CRUSH rule 0 x 218 [895,864,988,650,593]
+  CRUSH rule 0 x 219 [222,534,277,242,658]
+  CRUSH rule 0 x 220 [281,19,584,563,858]
+  CRUSH rule 0 x 221 [64,928,963,130,312]
+  CRUSH rule 0 x 222 [40,544,161,199,861]
+  CRUSH rule 0 x 223 [645,556,159,417,46]
+  CRUSH rule 0 x 224 [647,165,957,263,961]
+  CRUSH rule 0 x 225 [219,714,858,747,461]
+  CRUSH rule 0 x 226 [372,511,181,277,695]
+  CRUSH rule 0 x 227 [925,156,714,863,257]
+  CRUSH rule 0 x 228 [682,404,839,263,521]
+  CRUSH rule 0 x 229 [880,838,770,891,236]
+  CRUSH rule 0 x 230 [328,659,916,468,646]
+  CRUSH rule 0 x 231 [320,383,669,109,627]
+  CRUSH rule 0 x 232 [924,846,394,319,43]
+  CRUSH rule 0 x 233 [948,652,575,838,498]
+  CRUSH rule 0 x 234 [484,943,42,575,936]
+  CRUSH rule 0 x 235 [750,65,590,168,870]
+  CRUSH rule 0 x 236 [551,787,490,136,370]
+  CRUSH rule 0 x 237 [390,157,166,251,752]
+  CRUSH rule 0 x 238 [570,6,989,707,514]
+  CRUSH rule 0 x 239 [729,959,376,975,496]
+  CRUSH rule 0 x 240 [981,241,156,767,631]
+  CRUSH rule 0 x 241 [310,816,641,177,996]
+  CRUSH rule 0 x 242 [161,63,642,837,763]
+  CRUSH rule 0 x 243 [180,394,33,683,189]
+  CRUSH rule 0 x 244 [52,174,685,189,78]
+  CRUSH rule 0 x 245 [523,121,915,84,386]
+  CRUSH rule 0 x 246 [362,893,390,487,817]
+  CRUSH rule 0 x 247 [382,184,116,34,143]
+  CRUSH rule 0 x 248 [129,114,852,469,359]
+  CRUSH rule 0 x 249 [159,683,91,856,475]
+  CRUSH rule 0 x 250 [404,945,569,955,228]
+  CRUSH rule 0 x 251 [661,225,738,757,37]
+  CRUSH rule 0 x 252 [961,226,542,103,945]
+  CRUSH rule 0 x 253 [651,97,225,364,189]
+  CRUSH rule 0 x 254 [123,33,741,692,599]
+  CRUSH rule 0 x 255 [314,649,891,855,517]
+  CRUSH rule 0 x 256 [315,215,651,126,470]
+  CRUSH rule 0 x 257 [825,264,867,529,409]
+  CRUSH rule 0 x 258 [624,789,370,723,131]
+  CRUSH rule 0 x 259 [602,542,70,563,947]
+  CRUSH rule 0 x 260 [717,878,43,56,377]
+  CRUSH rule 0 x 261 [145,517,20,903,786]
+  CRUSH rule 0 x 262 [223,1,561,420,16]
+  CRUSH rule 0 x 263 [462,211,405,508,787]
+  CRUSH rule 0 x 264 [654,471,266,662,135]
+  CRUSH rule 0 x 265 [302,794,704,798,659]
+  CRUSH rule 0 x 266 [202,132,884,209,551]
+  CRUSH rule 0 x 267 [282,938,657,113,672]
+  CRUSH rule 0 x 268 [338,309,356,278,928]
+  CRUSH rule 0 x 269 [738,122,266,200,894]
+  CRUSH rule 0 x 270 [707,982,946,196,407]
+  CRUSH rule 0 x 271 [705,432,364,735,512]
+  CRUSH rule 0 x 272 [756,545,942,56,542]
+  CRUSH rule 0 x 273 [197,502,527,721,239]
+  CRUSH rule 0 x 274 [992,44,653,573,527]
+  CRUSH rule 0 x 275 [544,789,170,434,23]
+  CRUSH rule 0 x 276 [658,467,577,268,336]
+  CRUSH rule 0 x 277 [143,490,880,483,928]
+  CRUSH rule 0 x 278 [492,647,355,282,834]
+  CRUSH rule 0 x 279 [517,792,604,987,527]
+  CRUSH rule 0 x 280 [825,740,27,848,514]
+  CRUSH rule 0 x 281 [224,629,120,562,616]
+  CRUSH rule 0 x 282 [298,661,380,416,35]
+  CRUSH rule 0 x 283 [311,606,208,50,913]
+  CRUSH rule 0 x 284 [771,466,371,743,672]
+  CRUSH rule 0 x 285 [693,362,404,676,797]
+  CRUSH rule 0 x 286 [364,477,285,167,270]
+  CRUSH rule 0 x 287 [591,611,828,995,170]
+  CRUSH rule 0 x 288 [965,541,848,796,251]
+  CRUSH rule 0 x 289 [225,551,948,877,219]
+  CRUSH rule 0 x 290 [577,762,777,751,291]
+  CRUSH rule 0 x 291 [160,903,477,381,490]
+  CRUSH rule 0 x 292 [873,598,216,666,222]
+  CRUSH rule 0 x 293 [100,234,874,47,28]
+  CRUSH rule 0 x 294 [285,943,379,520,725]
+  CRUSH rule 0 x 295 [938,262,880,327,687]
+  CRUSH rule 0 x 296 [850,327,86,472,1]
+  CRUSH rule 0 x 297 [951,53,99,558,753]
+  CRUSH rule 0 x 298 [173,336,85,766,910]
+  CRUSH rule 0 x 299 [598,591,315,386,895]
+  CRUSH rule 0 x 300 [531,957,62,459,156]
+  CRUSH rule 0 x 301 [823,628,23,858,629]
+  CRUSH rule 0 x 302 [184,80,780,871,531]
+  CRUSH rule 0 x 303 [521,766,222,830,988]
+  CRUSH rule 0 x 304 [980,127,807,507,555]
+  CRUSH rule 0 x 305 [153,816,22,927,696]
+  CRUSH rule 0 x 306 [423,739,664,753,178]
+  CRUSH rule 0 x 307 [997,557,682,456,479]
+  CRUSH rule 0 x 308 [991,874,534,465,330]
+  CRUSH rule 0 x 309 [860,394,724,858,246]
+  CRUSH rule 0 x 310 [589,818,546,201,94]
+  CRUSH rule 0 x 311 [477,774,225,590,830]
+  CRUSH rule 0 x 312 [887,853,950,354,58]
+  CRUSH rule 0 x 313 [802,646,447,416,557]
+  CRUSH rule 0 x 314 [654,974,229,511,562]
+  CRUSH rule 0 x 315 [767,227,28,740,828]
+  CRUSH rule 0 x 316 [778,83,733,359,858]
+  CRUSH rule 0 x 317 [184,418,642,986,939]
+  CRUSH rule 0 x 318 [525,410,500,543,212]
+  CRUSH rule 0 x 319 [476,724,569,382,409]
+  CRUSH rule 0 x 320 [149,610,697,296,818]
+  CRUSH rule 0 x 321 [710,79,667,671,234]
+  CRUSH rule 0 x 322 [175,275,323,333,744]
+  CRUSH rule 0 x 323 [819,604,638,792,316]
+  CRUSH rule 0 x 324 [16,745,511,439,272]
+  CRUSH rule 0 x 325 [486,400,872,873,251]
+  CRUSH rule 0 x 326 [613,765,207,19,359]
+  CRUSH rule 0 x 327 [125,289,738,408,456]
+  CRUSH rule 0 x 328 [807,383,476,583,645]
+  CRUSH rule 0 x 329 [588,938,599,432,446]
+  CRUSH rule 0 x 330 [932,644,41,611,209]
+  CRUSH rule 0 x 331 [341,953,950,537,578]
+  CRUSH rule 0 x 332 [153,726,459,950,466]
+  CRUSH rule 0 x 333 [745,845,853,860,52]
+  CRUSH rule 0 x 334 [614,751,807,58,396]
+  CRUSH rule 0 x 335 [518,721,221,283,454]
+  CRUSH rule 0 x 336 [389,424,77,309,5]
+  CRUSH rule 0 x 337 [753,508,765,720,221]
+  CRUSH rule 0 x 338 [128,810,490,753,406]
+  CRUSH rule 0 x 339 [430,308,58,751,856]
+  CRUSH rule 0 x 340 [541,44,630,231,289]
+  CRUSH rule 0 x 341 [402,26,631,439,165]
+  CRUSH rule 0 x 342 [982,57,992,461,131]
+  CRUSH rule 0 x 343 [833,412,572,732,107]
+  CRUSH rule 0 x 344 [784,533,792,41,642]
+  CRUSH rule 0 x 345 [546,300,304,691,763]
+  CRUSH rule 0 x 346 [302,420,428,891,357]
+  CRUSH rule 0 x 347 [488,778,101,217,366]
+  CRUSH rule 0 x 348 [903,744,937,718,85]
+  CRUSH rule 0 x 349 [471,547,582,306,600]
+  CRUSH rule 0 x 350 [348,221,823,335,383]
+  CRUSH rule 0 x 351 [961,582,705,346,361]
+  CRUSH rule 0 x 352 [728,137,461,298,36]
+  CRUSH rule 0 x 353 [904,202,184,447,58]
+  CRUSH rule 0 x 354 [345,226,319,256,544]
+  CRUSH rule 0 x 355 [50,430,175,43,187]
+  CRUSH rule 0 x 356 [87,185,55,423,829]
+  CRUSH rule 0 x 357 [762,459,921,473,182]
+  CRUSH rule 0 x 358 [908,25,280,6,808]
+  CRUSH rule 0 x 359 [484,15,132,121,394]
+  CRUSH rule 0 x 360 [173,378,337,702,145]
+  CRUSH rule 0 x 361 [404,577,115,25,56]
+  CRUSH rule 0 x 362 [403,1,422,945,132]
+  CRUSH rule 0 x 363 [639,911,510,162,418]
+  CRUSH rule 0 x 364 [752,689,610,990,665]
+  CRUSH rule 0 x 365 [956,999,212,230,624]
+  CRUSH rule 0 x 366 [860,925,924,763,687]
+  CRUSH rule 0 x 367 [205,609,647,665,969]
+  CRUSH rule 0 x 368 [301,284,810,169,78]
+  CRUSH rule 0 x 369 [452,658,339,217,674]
+  CRUSH rule 0 x 370 [11,467,695,989,394]
+  CRUSH rule 0 x 371 [124,487,55,514,313]
+  CRUSH rule 0 x 372 [253,48,979,846,207]
+  CRUSH rule 0 x 373 [715,605,775,748,227]
+  CRUSH rule 0 x 374 [191,887,920,223,714]
+  CRUSH rule 0 x 375 [711,385,651,665,15]
+  CRUSH rule 0 x 376 [597,818,49,458,415]
+  CRUSH rule 0 x 377 [294,256,933,771,184]
+  CRUSH rule 0 x 378 [34,151,681,707,552]
+  CRUSH rule 0 x 379 [869,136,315,378,813]
+  CRUSH rule 0 x 380 [294,97,575,791,690]
+  CRUSH rule 0 x 381 [119,710,219,827,328]
+  CRUSH rule 0 x 382 [69,631,508,706,697]
+  CRUSH rule 0 x 383 [922,588,589,925,471]
+  CRUSH rule 0 x 384 [221,945,671,117,857]
+  CRUSH rule 0 x 385 [561,737,953,723,658]
+  CRUSH rule 0 x 386 [335,442,788,696,507]
+  CRUSH rule 0 x 387 [514,43,353,88,100]
+  CRUSH rule 0 x 388 [587,89,157,996,915]
+  CRUSH rule 0 x 389 [109,641,255,466,372]
+  CRUSH rule 0 x 390 [925,149,421,489,599]
+  CRUSH rule 0 x 391 [267,87,387,527,768]
+  CRUSH rule 0 x 392 [382,485,370,849,936]
+  CRUSH rule 0 x 393 [425,721,221,753,268]
+  CRUSH rule 0 x 394 [898,18,38,793,173]
+  CRUSH rule 0 x 395 [806,876,269,679,32]
+  CRUSH rule 0 x 396 [790,970,437,449,875]
+  CRUSH rule 0 x 397 [136,363,507,613,11]
+  CRUSH rule 0 x 398 [914,116,558,258,722]
+  CRUSH rule 0 x 399 [261,94,299,202,174]
+  CRUSH rule 0 x 400 [661,197,338,461,977]
+  CRUSH rule 0 x 401 [953,979,287,803,41]
+  CRUSH rule 0 x 402 [738,819,618,522,667]
+  CRUSH rule 0 x 403 [573,238,425,546,130]
+  CRUSH rule 0 x 404 [526,848,790,253,922]
+  CRUSH rule 0 x 405 [582,505,330,334,201]
+  CRUSH rule 0 x 406 [768,324,493,60,186]
+  CRUSH rule 0 x 407 [260,951,437,587,692]
+  CRUSH rule 0 x 408 [657,81,770,734,830]
+  CRUSH rule 0 x 409 [498,89,182,423,672]
+  CRUSH rule 0 x 410 [28,793,737,352,166]
+  CRUSH rule 0 x 411 [684,992,60,659,769]
+  CRUSH rule 0 x 412 [261,958,699,950,165]
+  CRUSH rule 0 x 413 [891,835,297,441,384]
+  CRUSH rule 0 x 414 [127,459,119,965,662]
+  CRUSH rule 0 x 415 [272,540,631,328,609]
+  CRUSH rule 0 x 416 [739,617,115,530,339]
+  CRUSH rule 0 x 417 [106,209,157,878,117]
+  CRUSH rule 0 x 418 [525,441,147,390,320]
+  CRUSH rule 0 x 419 [603,673,615,465,266]
+  CRUSH rule 0 x 420 [988,213,251,226,209]
+  CRUSH rule 0 x 421 [761,521,748,368,923]
+  CRUSH rule 0 x 422 [317,160,924,548,198]
+  CRUSH rule 0 x 423 [137,807,168,472,619]
+  CRUSH rule 0 x 424 [920,37,146,263,598]
+  CRUSH rule 0 x 425 [277,693,285,221,478]
+  CRUSH rule 0 x 426 [485,936,407,854,726]
+  CRUSH rule 0 x 427 [242,515,9,564,174]
+  CRUSH rule 0 x 428 [632,635,26,473,494]
+  CRUSH rule 0 x 429 [641,73,465,127,171]
+  CRUSH rule 0 x 430 [626,585,6,387,881]
+  CRUSH rule 0 x 431 [697,76,753,570,964]
+  CRUSH rule 0 x 432 [590,526,306,283,656]
+  CRUSH rule 0 x 433 [284,387,149,817,886]
+  CRUSH rule 0 x 434 [538,985,79,953,770]
+  CRUSH rule 0 x 435 [30,318,593,635,975]
+  CRUSH rule 0 x 436 [164,919,851,693,0]
+  CRUSH rule 0 x 437 [322,212,163,606,302]
+  CRUSH rule 0 x 438 [142,392,85,594,376]
+  CRUSH rule 0 x 439 [119,370,68,443,997]
+  CRUSH rule 0 x 440 [333,403,187,863,475]
+  CRUSH rule 0 x 441 [477,727,906,145,429]
+  CRUSH rule 0 x 442 [274,590,933,244,434]
+  CRUSH rule 0 x 443 [983,748,574,718,700]
+  CRUSH rule 0 x 444 [536,509,431,146,170]
+  CRUSH rule 0 x 445 [485,528,209,964,753]
+  CRUSH rule 0 x 446 [345,634,42,294,711]
+  CRUSH rule 0 x 447 [61,845,767,600,321]
+  CRUSH rule 0 x 448 [333,232,292,846,364]
+  CRUSH rule 0 x 449 [680,16,484,670,851]
+  CRUSH rule 0 x 450 [235,214,79,423,96]
+  CRUSH rule 0 x 451 [961,468,333,640,823]
+  CRUSH rule 0 x 452 [525,479,153,528,570]
+  CRUSH rule 0 x 453 [138,466,302,86,249]
+  CRUSH rule 0 x 454 [137,625,215,402,389]
+  CRUSH rule 0 x 455 [173,150,997,16,846]
+  CRUSH rule 0 x 456 [235,226,238,258,347]
+  CRUSH rule 0 x 457 [450,577,253,413,717]
+  CRUSH rule 0 x 458 [195,537,91,814,351]
+  CRUSH rule 0 x 459 [381,555,312,573,915]
+  CRUSH rule 0 x 460 [972,730,534,678,756]
+  CRUSH rule 0 x 461 [506,279,142,830,784]
+  CRUSH rule 0 x 462 [692,959,578,57,983]
+  CRUSH rule 0 x 463 [788,667,949,550,685]
+  CRUSH rule 0 x 464 [133,122,588,999,270]
+  CRUSH rule 0 x 465 [971,190,230,777,452]
+  CRUSH rule 0 x 466 [394,576,148,157,103]
+  CRUSH rule 0 x 467 [517,28,366,362,984]
+  CRUSH rule 0 x 468 [829,143,874,225,162]
+  CRUSH rule 0 x 469 [987,936,106,725,633]
+  CRUSH rule 0 x 470 [107,982,56,889,67]
+  CRUSH rule 0 x 471 [181,897,629,860,307]
+  CRUSH rule 0 x 472 [547,512,172,24,705]
+  CRUSH rule 0 x 473 [760,997,824,905,888]
+  CRUSH rule 0 x 474 [787,418,743,628,272]
+  CRUSH rule 0 x 475 [662,312,253,617,105]
+  CRUSH rule 0 x 476 [110,495,185,508,961]
+  CRUSH rule 0 x 477 [393,954,834,132,841]
+  CRUSH rule 0 x 478 [246,483,480,644,985]
+  CRUSH rule 0 x 479 [70,929,697,931,744]
+  CRUSH rule 0 x 480 [753,119,961,607,317]
+  CRUSH rule 0 x 481 [470,429,677,242,574]
+  CRUSH rule 0 x 482 [451,566,961,675,354]
+  CRUSH rule 0 x 483 [816,72,371,278,635]
+  CRUSH rule 0 x 484 [540,454,389,31,654]
+  CRUSH rule 0 x 485 [74,582,624,684,566]
+  CRUSH rule 0 x 486 [958,595,199,763,715]
+  CRUSH rule 0 x 487 [228,302,804,833,876]
+  CRUSH rule 0 x 488 [180,529,722,956,353]
+  CRUSH rule 0 x 489 [47,617,812,187,291]
+  CRUSH rule 0 x 490 [905,822,479,124,750]
+  CRUSH rule 0 x 491 [892,370,609,998,433]
+  CRUSH rule 0 x 492 [588,959,127,948,505]
+  CRUSH rule 0 x 493 [353,461,593,291,301]
+  CRUSH rule 0 x 494 [378,848,443,368,507]
+  CRUSH rule 0 x 495 [845,653,768,234,405]
+  CRUSH rule 0 x 496 [13,988,0,691,389]
+  CRUSH rule 0 x 497 [796,877,788,394,648]
+  CRUSH rule 0 x 498 [412,337,270,705,511]
+  CRUSH rule 0 x 499 [330,695,8,74,618]
+  CRUSH rule 0 x 500 [820,272,547,765,755]
+  CRUSH rule 0 x 501 [110,44,132,442,294]
+  CRUSH rule 0 x 502 [336,595,650,274,993]
+  CRUSH rule 0 x 503 [922,211,157,722,502]
+  CRUSH rule 0 x 504 [483,52,122,432,778]
+  CRUSH rule 0 x 505 [482,598,224,279,480]
+  CRUSH rule 0 x 506 [493,123,43,856,936]
+  CRUSH rule 0 x 507 [12,598,264,422,416]
+  CRUSH rule 0 x 508 [227,157,611,301,223]
+  CRUSH rule 0 x 509 [807,242,363,122,582]
+  CRUSH rule 0 x 510 [134,437,227,75,313]
+  CRUSH rule 0 x 511 [212,54,83,799,457]
+  CRUSH rule 0 x 512 [236,630,758,752,361]
+  CRUSH rule 0 x 513 [994,693,644,938,846]
+  CRUSH rule 0 x 514 [45,508,831,19,817]
+  CRUSH rule 0 x 515 [504,138,480,272,530]
+  CRUSH rule 0 x 516 [285,409,136,570,841]
+  CRUSH rule 0 x 517 [300,232,23,906,438]
+  CRUSH rule 0 x 518 [397,674,98,898,967]
+  CRUSH rule 0 x 519 [86,750,772,913,101]
+  CRUSH rule 0 x 520 [900,833,614,130,261]
+  CRUSH rule 0 x 521 [31,47,236,751,911]
+  CRUSH rule 0 x 522 [390,16,280,144,291]
+  CRUSH rule 0 x 523 [618,308,424,590,300]
+  CRUSH rule 0 x 524 [635,189,687,963,601]
+  CRUSH rule 0 x 525 [311,916,699,262,775]
+  CRUSH rule 0 x 526 [48,738,227,718,244]
+  CRUSH rule 0 x 527 [202,851,889,216,763]
+  CRUSH rule 0 x 528 [565,827,590,273,918]
+  CRUSH rule 0 x 529 [934,864,241,43,466]
+  CRUSH rule 0 x 530 [502,934,298,670,986]
+  CRUSH rule 0 x 531 [681,627,942,487,288]
+  CRUSH rule 0 x 532 [422,6,147,205,861]
+  CRUSH rule 0 x 533 [863,68,364,983,247]
+  CRUSH rule 0 x 534 [962,931,775,172,663]
+  CRUSH rule 0 x 535 [89,565,397,693,839]
+  CRUSH rule 0 x 536 [499,351,760,458,918]
+  CRUSH rule 0 x 537 [676,547,787,311,867]
+  CRUSH rule 0 x 538 [58,644,571,649,941]
+  CRUSH rule 0 x 539 [837,953,457,711,458]
+  CRUSH rule 0 x 540 [831,50,132,213,197]
+  CRUSH rule 0 x 541 [582,757,121,525,532]
+  CRUSH rule 0 x 542 [472,132,790,997,948]
+  CRUSH rule 0 x 543 [382,272,797,330,315]
+  CRUSH rule 0 x 544 [947,930,496,883,509]
+  CRUSH rule 0 x 545 [425,570,305,77,821]
+  CRUSH rule 0 x 546 [18,65,529,437,343]
+  CRUSH rule 0 x 547 [445,715,600,472,213]
+  CRUSH rule 0 x 548 [367,569,980,167,627]
+  CRUSH rule 0 x 549 [125,715,671,817,285]
+  CRUSH rule 0 x 550 [425,599,744,199,923]
+  CRUSH rule 0 x 551 [44,1,528,922,944]
+  CRUSH rule 0 x 552 [246,104,68,239,123]
+  CRUSH rule 0 x 553 [71,703,615,28,593]
+  CRUSH rule 0 x 554 [207,124,217,166,525]
+  CRUSH rule 0 x 555 [570,28,317,420,931]
+  CRUSH rule 0 x 556 [674,152,421,79,215]
+  CRUSH rule 0 x 557 [347,817,191,391,741]
+  CRUSH rule 0 x 558 [627,426,369,692,815]
+  CRUSH rule 0 x 559 [940,630,924,242,224]
+  CRUSH rule 0 x 560 [295,903,541,29,245]
+  CRUSH rule 0 x 561 [506,682,384,637,878]
+  CRUSH rule 0 x 562 [718,529,87,729,842]
+  CRUSH rule 0 x 563 [552,332,747,206,274]
+  CRUSH rule 0 x 564 [835,769,736,486,630]
+  CRUSH rule 0 x 565 [8,167,539,182,607]
+  CRUSH rule 0 x 566 [600,481,301,263,90]
+  CRUSH rule 0 x 567 [999,994,509,899,947]
+  CRUSH rule 0 x 568 [252,431,157,62,601]
+  CRUSH rule 0 x 569 [643,218,943,455,83]
+  CRUSH rule 0 x 570 [617,635,765,422,250]
+  CRUSH rule 0 x 571 [757,80,59,98,328]
+  CRUSH rule 0 x 572 [299,348,575,889,943]
+  CRUSH rule 0 x 573 [25,505,270,167,58]
+  CRUSH rule 0 x 574 [215,431,624,177,628]
+  CRUSH rule 0 x 575 [225,252,611,546,32]
+  CRUSH rule 0 x 576 [627,94,159,857,430]
+  CRUSH rule 0 x 577 [237,809,778,636,61]
+  CRUSH rule 0 x 578 [885,313,120,344,771]
+  CRUSH rule 0 x 579 [924,575,787,831,47]
+  CRUSH rule 0 x 580 [718,51,766,121,118]
+  CRUSH rule 0 x 581 [219,807,129,571,856]
+  CRUSH rule 0 x 582 [893,701,598,863,285]
+  CRUSH rule 0 x 583 [246,930,964,170,993]
+  CRUSH rule 0 x 584 [336,432,680,175,495]
+  CRUSH rule 0 x 585 [324,999,397,485,457]
+  CRUSH rule 0 x 586 [558,230,976,541,816]
+  CRUSH rule 0 x 587 [985,830,597,21,308]
+  CRUSH rule 0 x 588 [211,544,57,134,162]
+  CRUSH rule 0 x 589 [129,21,112,190,885]
+  CRUSH rule 0 x 590 [467,969,652,593,287]
+  CRUSH rule 0 x 591 [758,514,316,164,35]
+  CRUSH rule 0 x 592 [525,253,190,443,315]
+  CRUSH rule 0 x 593 [601,885,339,152,297]
+  CRUSH rule 0 x 594 [227,60,450,30,717]
+  CRUSH rule 0 x 595 [720,854,496,912,80]
+  CRUSH rule 0 x 596 [751,195,997,77,261]
+  CRUSH rule 0 x 597 [129,574,714,8,789]
+  CRUSH rule 0 x 598 [679,207,604,396,841]
+  CRUSH rule 0 x 599 [668,315,683,349,681]
+  CRUSH rule 0 x 600 [143,396,464,444,59]
+  CRUSH rule 0 x 601 [326,573,873,902,136]
+  CRUSH rule 0 x 602 [860,281,875,535,672]
+  CRUSH rule 0 x 603 [709,328,445,349,190]
+  CRUSH rule 0 x 604 [571,62,814,95,866]
+  CRUSH rule 0 x 605 [252,739,860,27,313]
+  CRUSH rule 0 x 606 [339,236,759,842,67]
+  CRUSH rule 0 x 607 [590,248,759,868,433]
+  CRUSH rule 0 x 608 [145,635,309,467,875]
+  CRUSH rule 0 x 609 [973,547,223,79,762]
+  CRUSH rule 0 x 610 [435,816,961,983,255]
+  CRUSH rule 0 x 611 [559,283,422,584,176]
+  CRUSH rule 0 x 612 [273,149,123,576,911]
+  CRUSH rule 0 x 613 [828,614,642,674,33]
+  CRUSH rule 0 x 614 [478,748,393,34,171]
+  CRUSH rule 0 x 615 [392,155,144,326,626]
+  CRUSH rule 0 x 616 [778,637,452,248,15]
+  CRUSH rule 0 x 617 [622,713,996,833,611]
+  CRUSH rule 0 x 618 [149,877,270,329,180]
+  CRUSH rule 0 x 619 [604,163,656,409,322]
+  CRUSH rule 0 x 620 [181,23,409,198,64]
+  CRUSH rule 0 x 621 [735,902,386,237,939]
+  CRUSH rule 0 x 622 [661,824,717,568,858]
+  CRUSH rule 0 x 623 [142,121,643,61,695]
+  CRUSH rule 0 x 624 [360,716,420,398,49]
+  CRUSH rule 0 x 625 [541,167,385,1,601]
+  CRUSH rule 0 x 626 [364,431,610,363,535]
+  CRUSH rule 0 x 627 [458,137,557,410,287]
+  CRUSH rule 0 x 628 [250,350,556,497,821]
+  CRUSH rule 0 x 629 [928,160,710,572,365]
+  CRUSH rule 0 x 630 [243,19,918,556,601]
+  CRUSH rule 0 x 631 [438,221,574,676,797]
+  CRUSH rule 0 x 632 [797,368,247,5,32]
+  CRUSH rule 0 x 633 [993,749,525,485,27]
+  CRUSH rule 0 x 634 [239,351,633,299,651]
+  CRUSH rule 0 x 635 [640,965,25,961,306]
+  CRUSH rule 0 x 636 [173,290,297,991,937]
+  CRUSH rule 0 x 637 [0,918,98,108,111]
+  CRUSH rule 0 x 638 [702,235,424,900,983]
+  CRUSH rule 0 x 639 [475,687,31,785,918]
+  CRUSH rule 0 x 640 [31,664,399,677,123]
+  CRUSH rule 0 x 641 [296,473,108,963,341]
+  CRUSH rule 0 x 642 [894,273,427,606,677]
+  CRUSH rule 0 x 643 [117,111,732,191,114]
+  CRUSH rule 0 x 644 [438,336,327,512,599]
+  CRUSH rule 0 x 645 [982,702,351,573,907]
+  CRUSH rule 0 x 646 [334,804,146,842,697]
+  CRUSH rule 0 x 647 [933,787,185,334,752]
+  CRUSH rule 0 x 648 [22,444,400,862,207]
+  CRUSH rule 0 x 649 [503,229,213,460,639]
+  CRUSH rule 0 x 650 [328,659,420,443,739]
+  CRUSH rule 0 x 651 [3,880,823,123,378]
+  CRUSH rule 0 x 652 [495,977,563,733,92]
+  CRUSH rule 0 x 653 [185,718,804,280,975]
+  CRUSH rule 0 x 654 [130,528,380,81,906]
+  CRUSH rule 0 x 655 [560,872,454,504,319]
+  CRUSH rule 0 x 656 [219,885,178,981,863]
+  CRUSH rule 0 x 657 [233,684,813,490,208]
+  CRUSH rule 0 x 658 [778,6,756,380,750]
+  CRUSH rule 0 x 659 [240,663,306,540,789]
+  CRUSH rule 0 x 660 [244,855,196,147,678]
+  CRUSH rule 0 x 661 [184,270,128,398,910]
+  CRUSH rule 0 x 662 [65,883,921,438,79]
+  CRUSH rule 0 x 663 [323,721,594,812,43]
+  CRUSH rule 0 x 664 [865,113,512,51,427]
+  CRUSH rule 0 x 665 [420,850,591,475,202]
+  CRUSH rule 0 x 666 [319,767,246,3,369]
+  CRUSH rule 0 x 667 [875,39,343,100,829]
+  CRUSH rule 0 x 668 [331,122,263,599,355]
+  CRUSH rule 0 x 669 [915,521,402,747,673]
+  CRUSH rule 0 x 670 [845,659,943,447,401]
+  CRUSH rule 0 x 671 [108,634,527,363,856]
+  CRUSH rule 0 x 672 [578,216,110,589,302]
+  CRUSH rule 0 x 673 [442,74,579,797,622]
+  CRUSH rule 0 x 674 [588,364,281,308,645]
+  CRUSH rule 0 x 675 [489,698,744,671,870]
+  CRUSH rule 0 x 676 [928,911,40,180,722]
+  CRUSH rule 0 x 677 [399,269,692,131,615]
+  CRUSH rule 0 x 678 [546,752,544,155,5]
+  CRUSH rule 0 x 679 [988,25,275,433,628]
+  CRUSH rule 0 x 680 [335,963,382,486,749]
+  CRUSH rule 0 x 681 [690,462,623,466,49]
+  CRUSH rule 0 x 682 [196,588,154,257,807]
+  CRUSH rule 0 x 683 [627,25,421,160,873]
+  CRUSH rule 0 x 684 [38,804,592,158,991]
+  CRUSH rule 0 x 685 [841,368,548,362,166]
+  CRUSH rule 0 x 686 [336,287,525,440,166]
+  CRUSH rule 0 x 687 [20,682,924,653,356]
+  CRUSH rule 0 x 688 [463,371,780,556,385]
+  CRUSH rule 0 x 689 [569,250,78,816,847]
+  CRUSH rule 0 x 690 [551,144,587,263,378]
+  CRUSH rule 0 x 691 [766,464,446,533,449]
+  CRUSH rule 0 x 692 [739,634,18,245,624]
+  CRUSH rule 0 x 693 [339,297,118,330,817]
+  CRUSH rule 0 x 694 [405,26,830,181,533]
+  CRUSH rule 0 x 695 [622,576,597,535,600]
+  CRUSH rule 0 x 696 [558,902,689,13,715]
+  CRUSH rule 0 x 697 [818,222,406,691,427]
+  CRUSH rule 0 x 698 [178,48,402,233,841]
+  CRUSH rule 0 x 699 [450,244,180,919,332]
+  CRUSH rule 0 x 700 [502,771,987,706,416]
+  CRUSH rule 0 x 701 [4,612,782,216,853]
+  CRUSH rule 0 x 702 [177,630,232,923,281]
+  CRUSH rule 0 x 703 [354,178,389,393,778]
+  CRUSH rule 0 x 704 [646,601,156,171,603]
+  CRUSH rule 0 x 705 [921,401,890,265,244]
+  CRUSH rule 0 x 706 [652,877,562,452,26]
+  CRUSH rule 0 x 707 [345,745,67,716,789]
+  CRUSH rule 0 x 708 [333,607,180,469,170]
+  CRUSH rule 0 x 709 [45,187,302,115,896]
+  CRUSH rule 0 x 710 [94,855,43,199,18]
+  CRUSH rule 0 x 711 [227,653,731,150,842]
+  CRUSH rule 0 x 712 [398,953,136,870,181]
+  CRUSH rule 0 x 713 [116,800,503,662,635]
+  CRUSH rule 0 x 714 [111,629,866,709,902]
+  CRUSH rule 0 x 715 [531,291,486,382,192]
+  CRUSH rule 0 x 716 [169,541,291,42,343]
+  CRUSH rule 0 x 717 [417,446,994,894,239]
+  CRUSH rule 0 x 718 [992,383,298,844,377]
+  CRUSH rule 0 x 719 [936,674,324,759,194]
+  CRUSH rule 0 x 720 [370,188,174,464,644]
+  CRUSH rule 0 x 721 [320,859,278,259,170]
+  CRUSH rule 0 x 722 [7,2,673,129,96]
+  CRUSH rule 0 x 723 [270,553,831,662,38]
+  CRUSH rule 0 x 724 [666,822,708,895,633]
+  CRUSH rule 0 x 725 [794,406,875,459,981]
+  CRUSH rule 0 x 726 [420,556,341,292,240]
+  CRUSH rule 0 x 727 [561,461,129,635,965]
+  CRUSH rule 0 x 728 [951,330,196,756,589]
+  CRUSH rule 0 x 729 [656,644,436,591,27]
+  CRUSH rule 0 x 730 [3,558,629,184,50]
+  CRUSH rule 0 x 731 [852,89,75,735,713]
+  CRUSH rule 0 x 732 [983,840,869,976,697]
+  CRUSH rule 0 x 733 [285,396,388,122,387]
+  CRUSH rule 0 x 734 [125,510,402,640,676]
+  CRUSH rule 0 x 735 [417,773,686,504,459]
+  CRUSH rule 0 x 736 [749,396,632,550,779]
+  CRUSH rule 0 x 737 [644,991,946,135,448]
+  CRUSH rule 0 x 738 [449,683,290,220,245]
+  CRUSH rule 0 x 739 [341,220,641,454,740]
+  CRUSH rule 0 x 740 [874,524,674,650,472]
+  CRUSH rule 0 x 741 [189,472,712,798,715]
+  CRUSH rule 0 x 742 [912,581,114,730,21]
+  CRUSH rule 0 x 743 [654,914,425,441,763]
+  CRUSH rule 0 x 744 [725,295,579,377,162]
+  CRUSH rule 0 x 745 [787,858,850,506,612]
+  CRUSH rule 0 x 746 [757,848,704,30,47]
+  CRUSH rule 0 x 747 [700,81,867,681,801]
+  CRUSH rule 0 x 748 [557,436,238,664,293]
+  CRUSH rule 0 x 749 [772,622,337,42,156]
+  CRUSH rule 0 x 750 [946,97,376,677,316]
+  CRUSH rule 0 x 751 [996,618,343,911,83]
+  CRUSH rule 0 x 752 [746,887,695,868,610]
+  CRUSH rule 0 x 753 [741,14,463,479,172]
+  CRUSH rule 0 x 754 [648,349,333,355,65]
+  CRUSH rule 0 x 755 [157,460,466,187,959]
+  CRUSH rule 0 x 756 [416,97,197,497,227]
+  CRUSH rule 0 x 757 [599,839,776,410,256]
+  CRUSH rule 0 x 758 [994,218,620,256,361]
+  CRUSH rule 0 x 759 [959,682,514,745,100]
+  CRUSH rule 0 x 760 [518,943,215,83,706]
+  CRUSH rule 0 x 761 [285,849,420,324,987]
+  CRUSH rule 0 x 762 [591,313,41,335,110]
+  CRUSH rule 0 x 763 [908,411,200,740,292]
+  CRUSH rule 0 x 764 [787,234,894,485,883]
+  CRUSH rule 0 x 765 [327,921,882,393,444]
+  CRUSH rule 0 x 766 [84,161,878,704,416]
+  CRUSH rule 0 x 767 [370,895,702,701,890]
+  CRUSH rule 0 x 768 [826,760,879,864,460]
+  CRUSH rule 0 x 769 [67,768,663,735,814]
+  CRUSH rule 0 x 770 [593,909,482,259,5]
+  CRUSH rule 0 x 771 [309,935,121,578,937]
+  CRUSH rule 0 x 772 [12,125,797,301,348]
+  CRUSH rule 0 x 773 [253,466,820,549,591]
+  CRUSH rule 0 x 774 [164,390,705,109,881]
+  CRUSH rule 0 x 775 [703,47,43,973,643]
+  CRUSH rule 0 x 776 [728,231,80,916,2]
+  CRUSH rule 0 x 777 [981,621,568,729,869]
+  CRUSH rule 0 x 778 [411,456,544,597,789]
+  CRUSH rule 0 x 779 [346,121,519,921,587]
+  CRUSH rule 0 x 780 [476,39,288,381,303]
+  CRUSH rule 0 x 781 [10,130,585,844,729]
+  CRUSH rule 0 x 782 [462,246,581,902,623]
+  CRUSH rule 0 x 783 [580,373,153,775,668]
+  CRUSH rule 0 x 784 [413,113,978,990,994]
+  CRUSH rule 0 x 785 [341,856,332,354,59]
+  CRUSH rule 0 x 786 [411,140,313,393,215]
+  CRUSH rule 0 x 787 [605,522,211,813,636]
+  CRUSH rule 0 x 788 [226,545,35,142,726]
+  CRUSH rule 0 x 789 [545,320,414,702,731]
+  CRUSH rule 0 x 790 [414,748,816,327,130]
+  CRUSH rule 0 x 791 [660,906,406,697,916]
+  CRUSH rule 0 x 792 [287,392,514,204,75]
+  CRUSH rule 0 x 793 [631,133,850,713,720]
+  CRUSH rule 0 x 794 [931,517,543,210,963]
+  CRUSH rule 0 x 795 [551,962,477,948,425]
+  CRUSH rule 0 x 796 [814,4,95,27,368]
+  CRUSH rule 0 x 797 [64,201,299,734,605]
+  CRUSH rule 0 x 798 [422,530,114,431,565]
+  CRUSH rule 0 x 799 [824,32,679,562,266]
+  CRUSH rule 0 x 800 [862,623,489,637,861]
+  CRUSH rule 0 x 801 [145,550,329,324,734]
+  CRUSH rule 0 x 802 [570,19,847,308,387]
+  CRUSH rule 0 x 803 [151,812,662,358,880]
+  CRUSH rule 0 x 804 [467,93,264,863,176]
+  CRUSH rule 0 x 805 [621,223,938,809,591]
+  CRUSH rule 0 x 806 [898,957,805,430,499]
+  CRUSH rule 0 x 807 [354,531,422,159,921]
+  CRUSH rule 0 x 808 [7,96,76,897,446]
+  CRUSH rule 0 x 809 [70,734,719,56,687]
+  CRUSH rule 0 x 810 [701,18,972,327,771]
+  CRUSH rule 0 x 811 [248,547,103,728,901]
+  CRUSH rule 0 x 812 [230,576,821,566,993]
+  CRUSH rule 0 x 813 [805,114,683,629,462]
+  CRUSH rule 0 x 814 [54,619,973,741,497]
+  CRUSH rule 0 x 815 [679,412,613,132,969]
+  CRUSH rule 0 x 816 [919,448,826,414,36]
+  CRUSH rule 0 x 817 [765,830,436,521,332]
+  CRUSH rule 0 x 818 [415,566,644,687,692]
+  CRUSH rule 0 x 819 [721,319,865,750,546]
+  CRUSH rule 0 x 820 [218,301,333,190,686]
+  CRUSH rule 0 x 821 [185,795,680,953,329]
+  CRUSH rule 0 x 822 [356,261,54,522,900]
+  CRUSH rule 0 x 823 [220,281,549,456,64]
+  CRUSH rule 0 x 824 [292,809,887,74,776]
+  CRUSH rule 0 x 825 [949,778,101,311,110]
+  CRUSH rule 0 x 826 [767,818,833,927,356]
+  CRUSH rule 0 x 827 [631,83,406,635,657]
+  CRUSH rule 0 x 828 [288,986,445,26,414]
+  CRUSH rule 0 x 829 [990,667,915,694,974]
+  CRUSH rule 0 x 830 [152,571,778,505,685]
+  CRUSH rule 0 x 831 [814,563,630,97,582]
+  CRUSH rule 0 x 832 [235,641,616,110,979]
+  CRUSH rule 0 x 833 [657,565,922,140,825]
+  CRUSH rule 0 x 834 [907,231,644,13,617]
+  CRUSH rule 0 x 835 [784,262,771,264,612]
+  CRUSH rule 0 x 836 [951,158,366,710,43]
+  CRUSH rule 0 x 837 [556,498,334,633,895]
+  CRUSH rule 0 x 838 [329,274,964,547,119]
+  CRUSH rule 0 x 839 [568,209,939,364,658]
+  CRUSH rule 0 x 840 [45,579,842,70,655]
+  CRUSH rule 0 x 841 [652,702,24,605,152]
+  CRUSH rule 0 x 842 [629,984,314,895,408]
+  CRUSH rule 0 x 843 [799,690,688,648,151]
+  CRUSH rule 0 x 844 [694,600,534,700,569]
+  CRUSH rule 0 x 845 [332,30,179,93,951]
+  CRUSH rule 0 x 846 [452,251,712,719,404]
+  CRUSH rule 0 x 847 [399,681,847,739,13]
+  CRUSH rule 0 x 848 [303,138,440,346,547]
+  CRUSH rule 0 x 849 [666,346,708,873,64]
+  CRUSH rule 0 x 850 [644,511,345,844,545]
+  CRUSH rule 0 x 851 [527,546,737,425,100]
+  CRUSH rule 0 x 852 [31,809,94,618,156]
+  CRUSH rule 0 x 853 [483,330,869,184,46]
+  CRUSH rule 0 x 854 [697,953,968,143,502]
+  CRUSH rule 0 x 855 [837,996,239,621,32]
+  CRUSH rule 0 x 856 [712,40,547,430,195]
+  CRUSH rule 0 x 857 [77,984,576,551,568]
+  CRUSH rule 0 x 858 [412,384,841,465,572]
+  CRUSH rule 0 x 859 [173,760,26,300,87]
+  CRUSH rule 0 x 860 [776,429,328,917,658]
+  CRUSH rule 0 x 861 [705,405,477,50,73]
+  CRUSH rule 0 x 862 [809,44,788,938,964]
+  CRUSH rule 0 x 863 [349,496,963,178,675]
+  CRUSH rule 0 x 864 [717,858,101,239,992]
+  CRUSH rule 0 x 865 [857,603,586,262,550]
+  CRUSH rule 0 x 866 [394,304,71,96,642]
+  CRUSH rule 0 x 867 [640,773,663,974,261]
+  CRUSH rule 0 x 868 [613,950,712,663,460]
+  CRUSH rule 0 x 869 [973,889,524,22,671]
+  CRUSH rule 0 x 870 [505,35,386,498,348]
+  CRUSH rule 0 x 871 [239,264,262,773,781]
+  CRUSH rule 0 x 872 [21,767,456,748,783]
+  CRUSH rule 0 x 873 [954,666,980,264,435]
+  CRUSH rule 0 x 874 [54,510,947,1,500]
+  CRUSH rule 0 x 875 [809,418,452,462,88]
+  CRUSH rule 0 x 876 [483,457,61,248,523]
+  CRUSH rule 0 x 877 [542,531,952,939,710]
+  CRUSH rule 0 x 878 [217,674,857,644,678]
+  CRUSH rule 0 x 879 [999,475,134,250,319]
+  CRUSH rule 0 x 880 [678,573,935,385,570]
+  CRUSH rule 0 x 881 [394,835,789,802,587]
+  CRUSH rule 0 x 882 [467,382,353,56,979]
+  CRUSH rule 0 x 883 [802,744,237,337,50]
+  CRUSH rule 0 x 884 [653,660,638,700,31]
+  CRUSH rule 0 x 885 [898,704,307,445,879]
+  CRUSH rule 0 x 886 [434,357,938,641,737]
+  CRUSH rule 0 x 887 [297,226,711,428,370]
+  CRUSH rule 0 x 888 [863,324,443,213,902]
+  CRUSH rule 0 x 889 [105,102,308,163,947]
+  CRUSH rule 0 x 890 [550,248,606,704,615]
+  CRUSH rule 0 x 891 [575,928,880,891,826]
+  CRUSH rule 0 x 892 [259,862,133,271,292]
+  CRUSH rule 0 x 893 [902,880,543,542,37]
+  CRUSH rule 0 x 894 [180,169,916,43,945]
+  CRUSH rule 0 x 895 [725,849,182,129,177]
+  CRUSH rule 0 x 896 [951,34,874,537,969]
+  CRUSH rule 0 x 897 [810,352,73,939,943]
+  CRUSH rule 0 x 898 [979,433,719,411,787]
+  CRUSH rule 0 x 899 [685,668,534,932,399]
+  CRUSH rule 0 x 900 [530,978,41,894,941]
+  CRUSH rule 0 x 901 [740,107,336,175,574]
+  CRUSH rule 0 x 902 [800,743,693,310,67]
+  CRUSH rule 0 x 903 [230,267,842,266,550]
+  CRUSH rule 0 x 904 [346,949,460,973,696]
+  CRUSH rule 0 x 905 [530,397,619,958,576]
+  CRUSH rule 0 x 906 [80,426,138,672,73]
+  CRUSH rule 0 x 907 [365,968,475,297,296]
+  CRUSH rule 0 x 908 [204,832,742,809,862]
+  CRUSH rule 0 x 909 [883,989,146,959,366]
+  CRUSH rule 0 x 910 [549,593,249,853,792]
+  CRUSH rule 0 x 911 [325,847,352,214,851]
+  CRUSH rule 0 x 912 [874,888,582,796,557]
+  CRUSH rule 0 x 913 [331,463,342,574,989]
+  CRUSH rule 0 x 914 [836,468,601,732,607]
+  CRUSH rule 0 x 915 [245,228,100,661,799]
+  CRUSH rule 0 x 916 [77,967,364,435,27]
+  CRUSH rule 0 x 917 [239,60,866,221,772]
+  CRUSH rule 0 x 918 [988,115,922,80,201]
+  CRUSH rule 0 x 919 [783,139,696,1,848]
+  CRUSH rule 0 x 920 [623,408,685,953,974]
+  CRUSH rule 0 x 921 [105,799,144,90,399]
+  CRUSH rule 0 x 922 [887,505,652,348,514]
+  CRUSH rule 0 x 923 [223,318,552,458,743]
+  CRUSH rule 0 x 924 [25,778,366,333,163]
+  CRUSH rule 0 x 925 [912,601,297,682,770]
+  CRUSH rule 0 x 926 [968,133,144,814,155]
+  CRUSH rule 0 x 927 [277,724,214,988,690]
+  CRUSH rule 0 x 928 [554,203,658,789,298]
+  CRUSH rule 0 x 929 [761,802,367,528,758]
+  CRUSH rule 0 x 930 [814,61,788,736,660]
+  CRUSH rule 0 x 931 [29,193,61,41,343]
+  CRUSH rule 0 x 932 [446,198,862,534,168]
+  CRUSH rule 0 x 933 [352,742,216,321,525]
+  CRUSH rule 0 x 934 [730,2,332,631,613]
+  CRUSH rule 0 x 935 [731,23,736,79,361]
+  CRUSH rule 0 x 936 [322,975,20,904,827]
+  CRUSH rule 0 x 937 [822,221,841,161,723]
+  CRUSH rule 0 x 938 [557,850,66,630,499]
+  CRUSH rule 0 x 939 [150,11,971,371,124]
+  CRUSH rule 0 x 940 [638,398,169,616,333]
+  CRUSH rule 0 x 941 [730,342,929,577,451]
+  CRUSH rule 0 x 942 [62,292,166,814,587]
+  CRUSH rule 0 x 943 [165,314,519,548,41]
+  CRUSH rule 0 x 944 [199,625,766,176,194]
+  CRUSH rule 0 x 945 [946,999,699,303,38]
+  CRUSH rule 0 x 946 [595,93,852,142,503]
+  CRUSH rule 0 x 947 [800,582,356,93,716]
+  CRUSH rule 0 x 948 [132,551,139,920,87]
+  CRUSH rule 0 x 949 [792,920,466,380,97]
+  CRUSH rule 0 x 950 [111,345,176,543,879]
+  CRUSH rule 0 x 951 [414,619,648,655,364]
+  CRUSH rule 0 x 952 [775,469,500,356,287]
+  CRUSH rule 0 x 953 [349,1,5,251,168]
+  CRUSH rule 0 x 954 [570,940,410,249,929]
+  CRUSH rule 0 x 955 [729,774,823,800,7]
+  CRUSH rule 0 x 956 [519,141,575,625,738]
+  CRUSH rule 0 x 957 [242,709,611,97,760]
+  CRUSH rule 0 x 958 [84,217,227,253,246]
+  CRUSH rule 0 x 959 [270,413,918,789,703]
+  CRUSH rule 0 x 960 [458,192,307,279,920]
+  CRUSH rule 0 x 961 [981,388,777,546,359]
+  CRUSH rule 0 x 962 [623,834,277,134,729]
+  CRUSH rule 0 x 963 [291,167,714,468,109]
+  CRUSH rule 0 x 964 [28,156,788,127,598]
+  CRUSH rule 0 x 965 [675,557,290,517,840]
+  CRUSH rule 0 x 966 [836,306,946,283,642]
+  CRUSH rule 0 x 967 [966,386,735,837,392]
+  CRUSH rule 0 x 968 [864,756,690,121,328]
+  CRUSH rule 0 x 969 [729,625,480,769,512]
+  CRUSH rule 0 x 970 [800,362,646,582,309]
+  CRUSH rule 0 x 971 [737,381,153,684,298]
+  CRUSH rule 0 x 972 [952,245,720,884,334]
+  CRUSH rule 0 x 973 [356,455,579,857,832]
+  CRUSH rule 0 x 974 [545,758,586,596,790]
+  CRUSH rule 0 x 975 [336,191,202,146,720]
+  CRUSH rule 0 x 976 [446,208,757,620,252]
+  CRUSH rule 0 x 977 [202,896,196,956,763]
+  CRUSH rule 0 x 978 [612,324,996,225,418]
+  CRUSH rule 0 x 979 [843,457,675,650,958]
+  CRUSH rule 0 x 980 [60,914,881,626,850]
+  CRUSH rule 0 x 981 [702,749,937,153,724]
+  CRUSH rule 0 x 982 [298,928,738,167,99]
+  CRUSH rule 0 x 983 [723,572,395,358,900]
+  CRUSH rule 0 x 984 [723,864,804,935,846]
+  CRUSH rule 0 x 985 [945,459,868,211,524]
+  CRUSH rule 0 x 986 [772,664,535,169,297]
+  CRUSH rule 0 x 987 [88,324,312,843,661]
+  CRUSH rule 0 x 988 [522,927,131,996,351]
+  CRUSH rule 0 x 989 [578,332,208,605,975]
+  CRUSH rule 0 x 990 [638,228,414,311,738]
+  CRUSH rule 0 x 991 [530,221,451,422,879]
+  CRUSH rule 0 x 992 [925,705,275,81,234]
+  CRUSH rule 0 x 993 [991,301,43,469,830]
+  CRUSH rule 0 x 994 [276,51,868,683,843]
+  CRUSH rule 0 x 995 [288,836,753,790,758]
+  CRUSH rule 0 x 996 [887,983,252,686,470]
+  CRUSH rule 0 x 997 [110,924,386,79,705]
+  CRUSH rule 0 x 998 [435,830,485,853,926]
+  CRUSH rule 0 x 999 [876,738,357,913,723]
+  CRUSH rule 0 x 1000 [178,963,638,430,845]
+  CRUSH rule 0 x 1001 [99,519,66,759,583]
+  CRUSH rule 0 x 1002 [515,534,468,866,878]
+  CRUSH rule 0 x 1003 [104,611,937,698,94]
+  CRUSH rule 0 x 1004 [269,638,724,375,491]
+  CRUSH rule 0 x 1005 [369,223,309,409,822]
+  CRUSH rule 0 x 1006 [40,107,69,275,79]
+  CRUSH rule 0 x 1007 [978,111,416,758,454]
+  CRUSH rule 0 x 1008 [965,956,624,832,421]
+  CRUSH rule 0 x 1009 [598,476,356,695,919]
+  CRUSH rule 0 x 1010 [767,523,239,517,29]
+  CRUSH rule 0 x 1011 [289,871,207,576,347]
+  CRUSH rule 0 x 1012 [128,28,370,31,341]
+  CRUSH rule 0 x 1013 [979,765,660,812,666]
+  CRUSH rule 0 x 1014 [979,948,513,88,47]
+  CRUSH rule 0 x 1015 [277,790,396,672,542]
+  CRUSH rule 0 x 1016 [262,73,128,886,839]
+  CRUSH rule 0 x 1017 [150,269,61,499,832]
+  CRUSH rule 0 x 1018 [555,829,554,944,406]
+  CRUSH rule 0 x 1019 [513,356,265,446,65]
+  CRUSH rule 0 x 1020 [158,161,877,704,948]
+  CRUSH rule 0 x 1021 [915,998,957,285,546]
+  CRUSH rule 0 x 1022 [967,829,973,640,703]
+  CRUSH rule 0 x 1023 [488,257,614,859,325]
+  rule 0 (data) num_rep 5 result size == 5:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356]
+  CRUSH rule 0 x 87 [997,317,463,626,685,909]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652]
+  CRUSH rule 0 x 194 [242,473,58,655,277,792]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849]
+  CRUSH rule 0 x 257 [825,264,867,529,409,291]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939]
+  CRUSH rule 0 x 262 [223,1,561,420,16,88]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544]
+  CRUSH rule 0 x 324 [16,745,511,439,272,668]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493]
+  CRUSH rule 0 x 374 [191,887,920,223,714,961]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149]
+  CRUSH rule 0 x 445 [485,528,209,964,753,554]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604]
+  CRUSH rule 0 x 699 [450,244,180,919,332,747]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948]
+  CRUSH rule 0 x 711 [227,653,731,150,842,534]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757]
+  CRUSH rule 0 x 742 [912,581,114,730,21,687]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762]
+  CRUSH rule 0 x 813 [805,114,683,629,462,285]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296]
+  CRUSH rule 0 x 868 [613,950,712,663,460,643]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173]
+  CRUSH rule 0 x 926 [968,133,144,814,155,709]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596]
+  CRUSH rule 0 x 974 [545,758,586,596,790,116]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419]
+  rule 0 (data) num_rep 6 result size == 6:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38]
+  CRUSH rule 0 x 87 [997,317,463,626,685,909,49]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359]
+  CRUSH rule 0 x 194 [242,473,58,655,277,792,887]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189]
+  CRUSH rule 0 x 257 [825,264,867,529,409,291,732]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516]
+  CRUSH rule 0 x 262 [223,1,561,420,16,88,534]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236]
+  CRUSH rule 0 x 324 [16,745,511,439,272,668,959]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128]
+  CRUSH rule 0 x 374 [191,887,920,223,714,961,760]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,136]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182]
+  CRUSH rule 0 x 445 [485,528,209,964,753,554,931]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,506]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468]
+  CRUSH rule 0 x 699 [450,244,180,919,332,747,453]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449]
+  CRUSH rule 0 x 711 [227,653,731,150,842,534,110]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863]
+  CRUSH rule 0 x 742 [912,581,114,730,21,687,81]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675]
+  CRUSH rule 0 x 813 [805,114,683,629,462,285,450]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988]
+  CRUSH rule 0 x 868 [613,950,712,663,460,643,547]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969]
+  CRUSH rule 0 x 926 [968,133,144,814,155,709,158]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,16]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549]
+  CRUSH rule 0 x 974 [545,758,586,596,790,116,993]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50]
+  rule 0 (data) num_rep 7 result size == 7:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966,750]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672,820]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527,901]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481,169]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847,220]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964,872]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884,726]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578,14]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462,207]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3,569]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811,484]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380,821]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117,393]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78,170]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855,76]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437,680]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786,969]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935,605]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883,394]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51,834]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975,636]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263,267]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557,633]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477,468]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800,309]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248,908]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501,909]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676,560]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770,569]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719,581]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426,522]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813,217]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428,63]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618,288]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663,877]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75,580]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676,524]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311,758]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929,910]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856,248]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980,503]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930,74]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456,713]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514,40]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160,527]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658,876]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45,390]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959,800]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738,47]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822,628]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83,344]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406,281]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304,899]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882,503]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339,902]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508,763]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77,500]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670,67]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8,762]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643,511]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194,277]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812,14]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45,723]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705,268]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451,891]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862,221]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334,656]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783,593]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705,114]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871,361]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684,411]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587,264]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642,529]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835,705]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633,560]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366,258]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735,953]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197,690]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183,917]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339,24]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51,766]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247,963]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580,51]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66,197]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681,618]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567,908]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38,326]
+  CRUSH rule 0 x 87 [997,317,463,626,685,909,49,28]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99,737]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320,921]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412,184]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495,299]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167,634]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343,949]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82,422]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648,661]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204,750]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83,542]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270,511]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37,138]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9,168]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12,369]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923,106]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933,231]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401,85]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434,121]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878,493]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589,614]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631,996]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856,296]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510,472]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18,949]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725,413]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738,253]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598,418]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421,39]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9,448]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725,142]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766,210]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570,278]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49,126]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424,868]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551,43]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877,248]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482,254]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736,737]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689,209]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909,809]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666,691]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149,904]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270,425]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203,667]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922,597]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14,444]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893,281]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712,649]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846,890]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363,323]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400,885]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384,58]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178,532]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193,258]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424,751]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266,70]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697,610]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577,348]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929,196]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514,546]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352,820]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406,556]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484,553]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671,883]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807,50]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942,864]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443,98]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58,641]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271,229]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384,446]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899,902]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209,641]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242,999]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909,642]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148,88]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587,854]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110,329]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524,389]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548,676]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935,266]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667,463]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601,909]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835,840]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100,910]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36,356]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471,529]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780,932]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51,211]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240,316]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756,647]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856,992]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724,923]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736,846]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168,224]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485,571]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109,903]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74,45]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485,511]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369,270]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566,378]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833,650]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37,1]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527,237]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999,754]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123,759]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359,721]
+  CRUSH rule 0 x 194 [242,473,58,655,277,792,887,561]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755,471]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413,834]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666,316]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355,574]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960,369]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132,271]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495,76]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153,272]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669,676]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697,881]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776,463]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230,367]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76,708]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96,841]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522,185]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14,921]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922,372]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857,894]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687,903]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579,79]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982,826]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973,898]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264,6]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34,497]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697,805]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686,982]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61,559]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597,904]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465,429]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329,320]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606,465]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876,984]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966,217]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261,389]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262,884]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93,880]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50,182]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106,877]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796,835]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103,95]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471,753]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573,128]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327,509]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894,884]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426,427]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450,677]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413,136]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234,756]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799,21]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785,107]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605,837]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989,999]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590,840]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713,237]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886,650]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270,619]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58,354]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838,131]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797,675]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605,453]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607,95]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189,627]
+  CRUSH rule 0 x 257 [825,264,867,529,409,291,732,224]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863,427]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77,191]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533,646]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516,136]
+  CRUSH rule 0 x 262 [223,1,561,420,16,88,534,289]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773,979]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715,916]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833,987]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7,557]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972,645]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715,536]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146,14]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476,571]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263,138]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710,779]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982,735]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370,990]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992,823]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634,98]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783,648]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350,600]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952,250]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895,914]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443,604]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939,879]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369,544]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60,546]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582,975]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699,627]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137,890]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195,538]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795,377]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473,209]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557,86]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806,911]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775,636]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459,833]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440,73]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266,82]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232,343]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213,286]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924,106]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904,838]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220,432]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400,365]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561,905]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214,944]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685,838]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761,648]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459,250]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976,551]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857,153]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90,855]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256,798]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497,929]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24,81]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952,599]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749,841]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761,725]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892,86]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290,97]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800,868]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523,366]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868,841]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187,380]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236,969]
+  CRUSH rule 0 x 324 [16,745,511,439,272,668,959,845]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462,268]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461,509]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750,669]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33,806]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516,713]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420,520]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624,649]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644,821]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243,633]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408,175]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635,367]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698,533]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956,907]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69,11]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607,953]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707,328]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720,503]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516,661]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660,655]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142,114]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127,732]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419,962]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783,661]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862,513]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795,143]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841,164]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766,775]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899,665]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279,616]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612,33]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985,412]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629,228]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891,656]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874,643]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397,52]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29,529]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643,286]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265,35]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444,613]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203,17]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113,373]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59,914]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685,641]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616,93]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284,184]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850,419]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797,547]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212,241]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128,207]
+  CRUSH rule 0 x 374 [191,887,920,223,714,961,760,571]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934,619]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446,897]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654,487]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728,860]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115,557]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255,806]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773,496]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276,56]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29,197]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488,435]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910,329]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232,692]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164,934]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474,267]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340,222]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852,196]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,136,818]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901,82]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652,543]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15,591]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126,179]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726,935]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996,558]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349,672]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749,410]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536,592]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79,32]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658,449]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202,650]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299,577]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776,296]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718,578]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72,345]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246,695]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213,806]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949,507]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313,351]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560,155]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618,907]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97,124]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694,332]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889,344]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138,374]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848,972]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823,884]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506,670]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764,274]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839,547]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905,588]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785,395]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80,236]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791,565]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334,588]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225,94]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857,562]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859,699]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194,366]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513,591]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52,897]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644,646]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371,731]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10,976]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443,23]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755,841]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414,152]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800,174]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205,236]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864,799]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774,350]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182,145]
+  CRUSH rule 0 x 445 [485,528,209,964,753,554,931,638]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314,714]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58,531]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807,688]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258,548]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721,31]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878,33]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604,49]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514,5]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106,103]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295,967]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504,96]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762,975]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399,558]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147,483]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841,512]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385,797]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240,911]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538,111]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789,0]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137,466]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659,35]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187,640]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201,249]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681,551]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558,71]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256,978]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809,56]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756,663]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446,333]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237,764]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984,226]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753,794]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941,843]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158,489]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371,807]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135,375]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731,233]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448,437]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283,170]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866,661]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621,955]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857,782]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924,965]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154,478]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566,779]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188,563]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591,423]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231,474]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389,819]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823,789]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129,763]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542,745]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949,173]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440,509]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930,573]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880,279]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490,852]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262,926]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758,104]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764,558]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898,161]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591,702]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313,282]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798,808]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786,152]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600,968]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899,451]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52,185]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374,985]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481,820]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453,660]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519,737]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625,434]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375,328]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558,956]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495,354]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753,624]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834,212]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8,550]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45,478]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853,643]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270,35]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651,368]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278,926]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577,509]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925,474]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949,374]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54,931]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206,682]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859,30]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148,668]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152,797]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37,485]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528,722]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95,789]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738,277]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137,934]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324,134]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250,362]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117,172]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699,610]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428,267]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517,684]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37,639]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915,570]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161,901]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57,217]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218,916]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693,953]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623,659]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830,762]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593,267]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124,107]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185,356]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887,376]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700,339]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62,817]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903,900]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641,751]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738,873]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184,127]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267,639]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398,521]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494,624]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533,674]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329,848]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33,312]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878,978]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333,841]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389,486]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177,545]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700,521]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487,976]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557,630]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608,755]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874,902]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984,622]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469,193]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642,226]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73,628]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794,682]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952,421]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195,581]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753,180]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811,413]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54,796]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667,318]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269,455]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994,16]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917,525]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180,482]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725,991]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286,280]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599,364]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243,264]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633,596]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697,763]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924,667]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983,281]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857,899]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954,94]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578,386]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148,33]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249,41]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160,888]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570,43]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296,735]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958,580]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92,12]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149,401]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74,307]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364,8]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222,749]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519,967]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35,620]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725,118]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446,798]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485,478]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137,140]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308,111]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225,841]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467,432]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205,580]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538,46]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920,830]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219,211]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416,45]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275,599]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296,337]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849,357]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236,318]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887,57]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701,887]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27,214]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858,138]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897,449]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610,665]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500,631]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660,857]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279,317]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720,135]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372,890]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453,732]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722,748]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869,150]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715,221]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119,818]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198,291]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,506,546]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605,214]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708,6]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858,16]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547,850]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170,954]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63,859]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402,205]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464,902]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170,65]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585,260]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798,658]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796,56]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795,783]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943,554]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938,600]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168,302]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755,330]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954,315]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371,402]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229,506]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528,875]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673,569]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103,763]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666,352]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247,620]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795,347]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774,192]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367,718]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345,599]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652,821]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154,121]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911,638]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917,622]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115,248]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333,161]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970,639]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451,290]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268,525]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828,276]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488,804]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300,989]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664,489]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153,922]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468,180]
+  CRUSH rule 0 x 699 [450,244,180,919,332,747,453,519]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68,641]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585,513]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466,687]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796,607]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655,595]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372,253]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923,770]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2,133]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939,331]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733,607]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449,28]
+  CRUSH rule 0 x 711 [227,653,731,150,842,534,110,639]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895,459]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53,839]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875,649]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322,417]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138,197]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237,62]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544,891]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828,975]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214,76]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177,264]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823,833]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985,846]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616,879]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359,720]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966,535]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105,31]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753,760]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572,933]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760,800]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528,890]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368,271]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880,343]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535,627]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690,59]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845,278]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482,564]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429,397]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146,17]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214,494]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863,571]
+  CRUSH rule 0 x 742 [912,581,114,730,21,687,81,145]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451,631]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843,699]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926,314]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450,651]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879,857]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304,999]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383,506]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169,171]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388,17]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88,315]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481,702]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336,724]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192,279]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850,191]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121,690]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165,686]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15,347]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345,69]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373,361]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664,350]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387,775]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70,202]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402,123]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357,310]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251,951]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645,975]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213,527]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961,324]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933,571]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891,959]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783,951]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890,425]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885,976]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396,76]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563,860]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65,954]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772,645]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17,336]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714,954]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812,516]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626,961]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481,198]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632,151]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490,481]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600,528]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194,216]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237,916]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788,164]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124,742]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406,858]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376,812]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811,459]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268,94]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646,451]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596,196]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473,250]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859,994]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941,643]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219,662]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846,53]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834,881]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663,949]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121,157]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640,607]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802,136]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166,929]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23,145]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620,648]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948,202]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675,28]
+  CRUSH rule 0 x 813 [805,114,683,629,462,285,450,948]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401,266]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314,670]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44,822]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260,172]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769,826]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523,770]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535,787]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621,815]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883,112]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282,641]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559,886]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161,998]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910,63]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212,916]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937,595]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669,330]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448,55]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142,157]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656,135]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764,766]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83,483]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537,937]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351,961]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903,29]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983,998]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47,859]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815,109]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226,46]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575,1]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486,199]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899,382]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611,512]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606,237]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363,893]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700,249]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847,463]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358,35]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95,337]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469,511]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774,679]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441,302]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686,702]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224,810]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12,763]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688,61]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463,903]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699,907]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901,487]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490,409]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172,980]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43,15]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850,40]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255,481]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988,730]
+  CRUSH rule 0 x 868 [613,950,712,663,460,643,547,734]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718,431]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54,992]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387,515]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180,800]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199,358]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93,915]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634,435]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322,141]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181,460]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329,591]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145,750]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319,630]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570,109]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974,483]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202,148]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389,381]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174,972]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56,582]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472,947]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806,53]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399,382]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996,561]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706,701]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53,333]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672,320]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648,685]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599,829]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210,529]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12,481]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342,37]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124,653]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380,419]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157,292]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178,624]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66,738]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957,801]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685,6]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30,169]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664,331]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484,391]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686,965]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824,552]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789,255]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226,889]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925,746]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70,280]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126,79]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255,133]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725,707]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583,923]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888,980]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532,124]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633,290]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952,474]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964,384]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584,31]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969,168]
+  CRUSH rule 0 x 926 [968,133,144,814,155,709,158,96]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465,775]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847,752]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744,171]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832,654]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487,839]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530,462]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568,61]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533,116]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772,49]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138,802]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630,308]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286,395]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408,49]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25,883]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964,28]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,16,440]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759,851]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678,915]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952,885]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933,267]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922,868]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81,220]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799,961]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355,220]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829,408]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16,746]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141,619]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129,696]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536,766]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169,751]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393,281]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346,377]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543,519]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855,49]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455,708]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856,477]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485,701]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361,255]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59,229]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929,773]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19,674]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433,520]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518,956]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576,411]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344,520]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754,540]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549,524]
+  CRUSH rule 0 x 974 [545,758,586,596,790,116,993,644]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330,308]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397,58]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783,828]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514,169]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677,173]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398,943]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536,212]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395,198]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927,597]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950,840]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911,208]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864,555]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76,894]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865,47]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155,380]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340,526]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754,928]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117,546]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382,428]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557,378]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158,265]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459,764]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210,698]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786,762]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15,585]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317,102]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739,922]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729,370]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614,783]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891,113]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597,969]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234,945]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5,444]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975,723]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234,383]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23,241]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48,570]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268,647]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808,351]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969,81]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145,11]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456,560]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637,731]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463,926]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768,245]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495,865]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676,322]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871,828]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50,560]
+  rule 0 (data) num_rep 8 result size == 8:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966,750,695]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672,820,782]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527,901,106]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481,169,816]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847,220,87]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964,872,66]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884,726,456]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578,14,543]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462,207,759]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3,569,603]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811,484,296]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380,821,470]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117,393,435]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78,170,333]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855,76,269]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437,680,761]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786,969,378]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935,605,228]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883,394,188]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51,834,690]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975,636,387]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263,267,128]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557,633,812]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477,468,303]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800,309,622]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248,908,624]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501,909,219]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676,560,753]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770,569,359]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719,581,816]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426,522,208]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813,217,750]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428,63,801]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618,288,846]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663,877,177]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75,580,568]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676,524,510]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311,758,201]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929,910,698]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856,248,279]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980,503,287]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930,74,844]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456,713,348]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514,40,261]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160,527,250]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658,876,201]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45,390,546]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959,800,743]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738,47,812]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822,628,696]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83,344,713]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406,281,991]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304,899,629]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882,503,327]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339,902,81]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508,763,108]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77,500,553]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670,67,114]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8,762,842]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643,511,370]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194,277,985]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812,14,55]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45,723,926]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705,268,140]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451,891,654]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862,221,66]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334,656,719]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783,593,814]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705,114,981]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871,361,533]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684,411,114]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587,264,446]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642,529,966]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835,705,745]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633,560,565]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366,258,711]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735,953,188]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197,690,426]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183,917,464]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339,24,581]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51,766,822]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247,963,0]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580,51,939]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66,197,698]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681,618,723]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567,908,514]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38,326,385]
+  CRUSH rule 0 x 87 [997,317,463,626,685,909,49,28,698]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99,737,942]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320,921,812]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412,184,529]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495,299,812]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167,634,781]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343,949,51]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82,422,738]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648,661,46]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204,750,322]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83,542,131]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270,511,586]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37,138,326]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9,168,590]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12,369,9]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923,106,653]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933,231,520]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401,85,139]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434,121,488]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878,493,550]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589,614,814]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631,996,630]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856,296,122]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510,472,270]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18,949,341]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725,413,187]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738,253,213]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598,418,641]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421,39,241]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9,448,816]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725,142,304]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766,210,528]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570,278,867]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49,126,410]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424,868,469]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551,43,590]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877,248,753]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482,254,158]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736,737,508]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689,209,99]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909,809,553]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666,691,899]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149,904,70]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270,425,864]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203,667,528]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922,597,680]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14,444,709]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893,281,226]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712,649,27]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846,890,151]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363,323,153]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400,885,907]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384,58,373]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178,532,883]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193,258,445]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424,751,746]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266,70,210]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697,610,203]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577,348,1]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929,196,880]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514,546,861]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352,820,857]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406,556,532]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484,553,315]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671,883,204]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807,50,206]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942,864,388]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443,98,27]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58,641,186]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271,229,961]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384,446,836]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899,902,956]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209,641,228]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242,999,974]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909,642,188]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148,88,764]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587,854,482]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110,329,361]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524,389,185]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548,676,448]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935,266,883]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667,463,818]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601,909,60]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835,840,548]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100,910,446]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36,356,274]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471,529,359]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780,932,573]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51,211,409]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240,316,337]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756,647,45]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856,992,696]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724,923,450]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736,846,926]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168,224,705]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485,571,796]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109,903,96]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74,45,438]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485,511,14]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369,270,16]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566,378,816]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833,650,761]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37,1,939]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527,237,295]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999,754,339]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123,759,147]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359,721,996]
+  CRUSH rule 0 x 194 [242,473,58,655,277,792,887,561,449]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755,471,744]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413,834,832]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666,316,243]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355,574,781]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960,369,584]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132,271,128]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495,76,598]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153,272,944]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669,676,762]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697,881,64]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776,463,327]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230,367,890]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76,708,101]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96,841,811]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522,185,949]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14,921,544]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922,372,581]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857,894,60]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687,903,620]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579,79,258]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982,826,488]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973,898,556]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264,6,603]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34,497,108]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697,805,976]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686,982,0]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61,559,846]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597,904,897]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465,429,614]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329,320,645]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606,465,354]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876,984,491]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966,217,501]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261,389,281]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262,884,215]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93,880,959]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50,182,541]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106,877,130]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796,835,714]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103,95,634]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471,753,350]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573,128,154]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327,509,325]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894,884,824]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426,427,736]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450,677,659]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413,136,411]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234,756,496]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799,21,13]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785,107,816]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605,837,1]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989,999,138]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590,840,586]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713,237,468]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886,650,827]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270,619,450]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58,354,16]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838,131,387]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797,675,452]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605,453,987]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607,95,121]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189,627,592]
+  CRUSH rule 0 x 257 [825,264,867,529,409,291,732,224,841]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863,427,873]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77,191,669]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533,646,475]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516,136,87]
+  CRUSH rule 0 x 262 [223,1,561,420,16,88,534,289,498]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773,979,719]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715,916,633]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833,987,445]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7,557,76]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972,645,882]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715,536,983]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146,14,414]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476,571,314]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263,138,526]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710,779,161]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982,735,58]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370,990,320]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992,823,321]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634,98,457]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783,648,927]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350,600,283]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952,250,206]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895,914,892]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443,604,638]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939,879,338]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369,544,721]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60,546,39]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582,975,810]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699,627,725]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137,890,487]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195,538,356]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795,377,825]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473,209,59]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557,86,89]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806,911,738]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775,636,232]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459,833,503]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440,73,29]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266,82,671]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232,343,831]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213,286,61]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924,106,63]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904,838,458]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220,432,393]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400,365,697]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561,905,522]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214,944,845]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685,838,3]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761,648,867]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459,250,415]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976,551,126]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857,153,970]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90,855,441]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256,798,743]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497,929,92]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24,81,215]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952,599,763]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749,841,969]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761,725,923]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892,86,214]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290,97,529]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800,868,364]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523,366,891]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868,841,563]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187,380,947]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236,969,232]
+  CRUSH rule 0 x 324 [16,745,511,439,272,668,959,845,759]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462,268,124]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461,509,75]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750,669,296]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33,806,181]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516,713,223]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420,520,395]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624,649,626]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644,821,238]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243,633,309]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408,175,189]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635,367,997]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698,533,683]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956,907,464]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69,11,624]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607,953,125]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707,328,325]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720,503,209]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516,661,985]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660,655,149]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142,114,108]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127,732,290]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419,962,304]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783,661,622]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862,513,112]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795,143,529]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841,164,765]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766,775,518]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899,665,802]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279,616,892]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612,33,122]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985,412,599]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629,228,150]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891,656,196]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874,643,550]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397,52,702]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29,529,156]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643,286,552]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265,35,662]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444,613,499]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203,17,743]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113,373,426]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59,914,521]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685,641,894]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616,93,283]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284,184,718]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850,419,307]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797,547,778]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212,241,346]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128,207,88]
+  CRUSH rule 0 x 374 [191,887,920,223,714,961,760,571,549]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934,619,527]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446,897,460]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654,487,891]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728,860,968]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115,557,165]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255,806,429]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773,496,433]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276,56,278]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29,197,822]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488,435,223]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910,329,396]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232,692,742]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164,934,297]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474,267,640]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340,222,74]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852,196,469]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,136,818,516]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901,82,695]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652,543,10]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15,591,420]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126,179,607]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726,935,278]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996,558,602]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349,672,826]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749,410,815]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536,592,886]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79,32,343]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658,449,886]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202,650,501]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299,577,563]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776,296,19]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718,578,580]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72,345,709]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246,695,76]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213,806,168]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949,507,361]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313,351,497]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560,155,661]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618,907,9]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97,124,229]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694,332,572]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889,344,838]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138,374,470]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848,972,781]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823,884,832]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506,670,285]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764,274,623]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839,547,599]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905,588,312]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785,395,884]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80,236,988]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791,565,352]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334,588,571]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225,94,303]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857,562,976]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859,699,91]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194,366,279]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513,591,599]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52,897,705]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644,646,747]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371,731,906]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10,976,284]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443,23,696]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755,841,94]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414,152,331]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800,174,117]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205,236,86]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864,799,762]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774,350,37]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182,145,347]
+  CRUSH rule 0 x 445 [485,528,209,964,753,554,931,638,892]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314,714,212]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58,531,827]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807,688,21]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258,548,905]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721,31,312]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878,33,3]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604,49,922]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514,5,494]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106,103,511]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295,967,132]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504,96,890]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762,975,485]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399,558,15]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147,483,517]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841,512,70]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385,797,917]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240,911,375]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538,111,232]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789,0,653]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137,466,531]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659,35,797]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187,640,601]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201,249,555]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681,551,768]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558,71,676]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256,978,409]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809,56,476]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756,663,167]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446,333,245]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237,764,682]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984,226,333]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753,794,237]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941,843,751]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158,489,515]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371,807,687]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135,375,613]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731,233,640]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448,437,219]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283,170,278]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866,661,581]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621,955,400]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857,782,24]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924,965,25]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154,478,512]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566,779,507]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188,563,490]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591,423,668]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231,474,946]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389,819,956]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823,789,217]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129,763,39]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542,745,131]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949,173,398]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440,509,295]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930,573,357]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880,279,616]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490,852,962]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262,926,316]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758,104,831]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764,558,891]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898,161,78]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591,702,346]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313,282,207]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798,808,139]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786,152,921]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600,968,355]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899,451,415]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52,185,197]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374,985,944]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481,820,517]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453,660,93]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519,737,20]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625,434,527]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375,328,3]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558,956,664]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495,354,665]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753,624,769]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834,212,906]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8,550,769]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45,478,911]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853,643,625]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270,35,809]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651,368,118]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278,926,280]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577,509,195]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925,474,669]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949,374,988]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54,931,4]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206,682,627]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859,30,61]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148,668,436]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152,797,492]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37,485,88]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528,722,59]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95,789,348]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738,277,225]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137,934,547]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324,134,839]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250,362,614]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117,172,764]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699,610,785]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428,267,229]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517,684,154]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37,639,934]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915,570,546]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161,901,342]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57,217,21]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218,916,561]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693,953,606]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623,659,403]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830,762,691]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593,267,17]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124,107,766]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185,356,87]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887,376,658]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700,339,687]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62,817,766]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903,900,812]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641,751,930]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738,873,47]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184,127,448]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267,639,646]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398,521,59]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494,624,352]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533,674,23]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329,848,235]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33,312,202]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878,978,1]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333,841,193]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389,486,10]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177,545,839]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700,521,825]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487,976,977]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557,630,468]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608,755,326]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874,902,958]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984,622,175]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469,193,737]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642,226,122]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73,628,884]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794,682,127]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952,421,875]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195,581,649]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753,180,160]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811,413,436]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54,796,369]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667,318,496]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269,455,168]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994,16,777]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917,525,945]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180,482,449]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725,991,955]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286,280,507]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599,364,546]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243,264,31]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633,596,988]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697,763,442]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924,667,356]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983,281,292]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857,899,349]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954,94,88]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578,386,226]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148,33,420]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249,41,778]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160,888,767]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570,43,362]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296,735,245]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958,580,197]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92,12,62]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149,401,14]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74,307,976]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364,8,342]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222,749,697]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519,967,737]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35,620,268]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725,118,875]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446,798,869]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485,478,185]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137,140,488]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308,111,207]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225,841,868]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467,432,944]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205,580,972]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538,46,300]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920,830,171]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219,211,157]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416,45,624]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275,599,219]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296,337,676]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849,357,317]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236,318,228]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887,57,16]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701,887,355]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27,214,226]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858,138,726]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897,449,42]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610,665,299]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500,631,833]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660,857,123]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279,317,414]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720,135,369]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372,890,30]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453,732,262]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722,748,599]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869,150,743]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715,221,31]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119,818,459]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198,291,71]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,506,546,266]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605,214,833]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708,6,746]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858,16,128]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547,850,499]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170,954,22]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63,859,215]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402,205,609]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464,902,276]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170,65,906]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585,260,254]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798,658,28]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796,56,736]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795,783,386]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943,554,395]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938,600,517]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168,302,681]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755,330,584]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954,315,735]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371,402,725]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229,506,565]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528,875,982]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673,569,701]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103,763,527]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666,352,576]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247,620,437]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795,347,831]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774,192,454]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367,718,345]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345,599,30]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652,821,641]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154,121,804]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911,638,690]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917,622,156]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115,248,566]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333,161,74]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970,639,835]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451,290,789]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268,525,425]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828,276,264]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488,804,501]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300,989,804]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664,489,598]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153,922,986]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468,180,783]
+  CRUSH rule 0 x 699 [450,244,180,919,332,747,453,519,100]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68,641,109]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585,513,907]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466,687,742]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796,607,894]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655,595,888]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372,253,807]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923,770,516]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2,133,256]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939,331,41]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733,607,763]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449,28,731]
+  CRUSH rule 0 x 711 [227,653,731,150,842,534,110,639,452]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895,459,341]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53,839,56]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875,649,23]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322,417,973]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138,197,32]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237,62,327]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544,891,210]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828,975,119]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214,76,870]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177,264,867]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823,833,1]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985,846,77]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616,879,480]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359,720,128]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966,535,669]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105,31,506]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753,760,254]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572,933,434]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760,800,945]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528,890,625]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368,271,778]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880,343,590]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535,627,224]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690,59,294]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845,278,559]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482,564,259]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429,397,872]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146,17,314]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214,494,593]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863,571,876]
+  CRUSH rule 0 x 742 [912,581,114,730,21,687,81,145,695]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451,631,911]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843,699,24]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926,314,771]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450,651,105]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879,857,727]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304,999,685]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383,506,570]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169,171,9]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388,17,892]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88,315,728]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481,702,431]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336,724,262]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192,279,371]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850,191,991]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121,690,544]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165,686,449]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15,347,311]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345,69,39]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373,361,684]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664,350,339]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387,775,797]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70,202,557]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402,123,902]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357,310,890]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251,951,675]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645,975,947]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213,527,546]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961,324,309]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933,571,822]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891,959,487]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783,951,982]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890,425,599]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885,976,936]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396,76,680]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563,860,388]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65,954,125]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772,645,254]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17,336,147]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714,954,271]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812,516,774]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626,961,576]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481,198,171]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632,151,586]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490,481,627]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600,528,966]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194,216,486]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237,916,374]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788,164,691]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124,742,990]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406,858,694]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376,812,886]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811,459,344]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268,94,648]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646,451,67]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596,196,93]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473,250,839]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859,994,831]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941,643,398]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219,662,142]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846,53,783]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834,881,23]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663,949,380]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121,157,934]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640,607,790]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802,136,305]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166,929,234]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23,145,184]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620,648,433]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948,202,521]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675,28,263]
+  CRUSH rule 0 x 813 [805,114,683,629,462,285,450,948,742]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401,266,905]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314,670,928]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44,822,332]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260,172,193]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769,826,519]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523,770,56]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535,787,267]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621,815,313]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883,112,601]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282,641,216]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559,886,753]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161,998,370]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910,63,288]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212,916,692]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937,595,935]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669,330,822]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448,55,965]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142,157,957]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656,135,341]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764,766,853]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83,483,811]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537,937,101]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351,961,52]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903,29,454]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983,998,320]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47,859,402]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815,109,762]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226,46,918]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575,1,312]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486,199,966]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899,382,851]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611,512,855]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606,237,414]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363,893,592]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700,249,214]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847,463,995]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358,35,913]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95,337,677]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469,511,999]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774,679,616]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441,302,437]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686,702,919]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224,810,404]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12,763,594]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688,61,545]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463,903,272]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699,907,532]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901,487,725]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490,409,15]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172,980,772]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43,15,29]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850,40,170]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255,481,435]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988,730,753]
+  CRUSH rule 0 x 868 [613,950,712,663,460,643,547,734,16]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718,431,968]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54,992,726]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387,515,98]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180,800,521]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199,358,805]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93,915,801]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634,435,778]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322,141,82]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181,460,459]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329,591,59]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145,750,54]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319,630,888]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570,109,896]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974,483,412]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202,148,129]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389,381,347]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174,972,544]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56,582,915]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472,947,35]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806,53,385]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399,382,761]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996,561,485]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706,701,501]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53,333,458]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672,320,394]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648,685,895]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599,829,809]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210,529,491]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12,481,539]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342,37,303]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124,653,574]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380,419,667]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157,292,724]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178,624,733]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66,738,419]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957,801,74]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685,6,689]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30,169,506]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664,331,184]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484,391,841]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686,965,515]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824,552,717]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789,255,896]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226,889,69]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925,746,664]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70,280,837]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126,79,652]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255,133,892]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725,707,47]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583,923,863]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888,980,33]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532,124,911]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633,290,155]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952,474,67]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964,384,454]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584,31,151]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969,168,500]
+  CRUSH rule 0 x 926 [968,133,144,814,155,709,158,96,739]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465,775,725]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847,752,780]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744,171,144]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832,654,567]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487,839,776]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530,462,202]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568,61,945]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533,116,254]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772,49,567]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138,802,885]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630,308,973]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286,395,927]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408,49,977]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25,883,867]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964,28,633]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,16,440,31]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759,851,617]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678,915,619]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952,885,987]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933,267,846]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922,868,413]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81,220,725]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799,961,564]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355,220,528]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829,408,568]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16,746,835]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141,619,234]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129,696,115]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536,766,579]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169,751,667]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393,281,227]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346,377,425]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543,519,496]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855,49,548]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455,708,649]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856,477,895]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485,701,76]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361,255,507]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59,229,819]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929,773,928]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19,674,395]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433,520,916]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518,956,398]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576,411,416]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344,520,546]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754,540,79]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549,524,109]
+  CRUSH rule 0 x 974 [545,758,586,596,790,116,993,644,405]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330,308,744]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397,58,57]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783,828,409]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514,169,99]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677,173,903]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398,943,764]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536,212,247]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395,198,100]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927,597,103]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950,840,427]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911,208,91]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864,555,687]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76,894,480]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865,47,116]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155,380,797]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340,526,728]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754,928,288]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117,546,798]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382,428,451]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557,378,936]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158,265,110]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459,764,859]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210,698,273]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786,762,444]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15,585,898]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317,102,200]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739,922,343]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729,370,326]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614,783,865]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891,113,424]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597,969,911]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234,945,598]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5,444,795]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975,723,909]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234,383,604]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23,241,838]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48,570,639]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268,647,669]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808,351,572]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969,81,586]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145,11,965]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456,560,935]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637,731,738]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463,926,475]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768,245,337]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495,865,698]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676,322,671]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871,828,440]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50,560,595]
+  rule 0 (data) num_rep 9 result size == 9:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966,750,695,503]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672,820,782,802]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527,901,106,273]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481,169,816,732]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847,220,87,254]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964,872,66,161]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884,726,456,796]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578,14,543,287]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462,207,759,701]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3,569,603,47]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811,484,296,320]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380,821,470,230]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117,393,435,514]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78,170,333,295]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855,76,269,673]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437,680,761,483]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786,969,378,246]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935,605,228,737]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883,394,188,459]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51,834,690,375]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975,636,387,600]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263,267,128,828]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557,633,812,521]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477,468,303,367]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800,309,622,673]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248,908,624,643]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501,909,219,686]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676,560,753,77]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770,569,359,733]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719,581,816,349]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426,522,208,699]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813,217,750,155]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428,63,801,735]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618,288,846,659]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663,877,177,567]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75,580,568,523]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676,524,510,480]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311,758,201,999]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929,910,698,386]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856,248,279,416]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980,503,287,564]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930,74,844,943]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456,713,348,311]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514,40,261,223]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160,527,250,247]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658,876,201,45]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45,390,546,908]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959,800,743,161]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738,47,812,166]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822,628,696,407]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83,344,713,857]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406,281,991,336]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304,899,629,701]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882,503,327,402]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339,902,81,239]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508,763,108,185]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77,500,553,417]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670,67,114,685]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8,762,842,884]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643,511,370,336]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194,277,985,554]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812,14,55,752]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45,723,926,180]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705,268,140,942]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451,891,654,294]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862,221,66,354]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334,656,719,810]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783,593,814,970]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705,114,981,267]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871,361,533,433]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684,411,114,42]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587,264,446,572]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642,529,966,919]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835,705,745,986]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633,560,565,410]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366,258,711,657]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735,953,188,516]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197,690,426,681]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183,917,464,518]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339,24,581,969]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51,766,822,687]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247,963,0,732]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580,51,939,278]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66,197,698,318]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681,618,723,516]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567,908,514,355]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38,326,385,899]
+  CRUSH rule 0 x 87 [997,317,463,626,685,909,49,28,698,706]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99,737,942,647]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320,921,812,908]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412,184,529,127]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495,299,812,743]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167,634,781,242]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343,949,51,732]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82,422,738,933]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648,661,46,112]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204,750,322,75]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83,542,131,240]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270,511,586,230]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37,138,326,603]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9,168,590,873]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12,369,9,872]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923,106,653,999]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933,231,520,303]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401,85,139,855]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434,121,488,465]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878,493,550,484]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589,614,814,970]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631,996,630,597]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856,296,122,477]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510,472,270,753]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18,949,341,837]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725,413,187,840]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738,253,213,541]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598,418,641,891]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421,39,241,881]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9,448,816,609]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725,142,304,591]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766,210,528,252]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570,278,867,489]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49,126,410,33]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424,868,469,183]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551,43,590,51]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877,248,753,395]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482,254,158,641]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736,737,508,150]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689,209,99,479]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909,809,553,785]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666,691,899,665]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149,904,70,566]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270,425,864,38]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203,667,528,387]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922,597,680,25]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14,444,709,592]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893,281,226,739]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712,649,27,274]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846,890,151,872]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363,323,153,238]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400,885,907,214]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384,58,373,121]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178,532,883,380]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193,258,445,451]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424,751,746,854]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266,70,210,61]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697,610,203,382]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577,348,1,323]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929,196,880,458]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514,546,861,796]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352,820,857,282]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406,556,532,750]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484,553,315,504]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671,883,204,12]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807,50,206,368]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942,864,388,639]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443,98,27,120]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58,641,186,553]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271,229,961,163]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384,446,836,917]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899,902,956,424]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209,641,228,213]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242,999,974,652]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909,642,188,143]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148,88,764,40]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587,854,482,522]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110,329,361,743]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524,389,185,893]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548,676,448,991]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935,266,883,77]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667,463,818,819]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601,909,60,255]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835,840,548,588]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100,910,446,541]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36,356,274,976]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471,529,359,571]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780,932,573,814]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51,211,409,865]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240,316,337,361]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756,647,45,752]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856,992,696,784]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724,923,450,340]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736,846,926,129]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168,224,705,185]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485,571,796,587]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109,903,96,526]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74,45,438,26]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485,511,14,329]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369,270,16,920]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566,378,816,553]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833,650,761,468]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37,1,939,463]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527,237,295,716]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999,754,339,271]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123,759,147,99]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359,721,996,318]
+  CRUSH rule 0 x 194 [242,473,58,655,277,792,887,561,449,911]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755,471,744,910]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413,834,832,368]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666,316,243,320]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355,574,781,550]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960,369,584,612]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132,271,128,632]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495,76,598,301]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153,272,944,741]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669,676,762,200]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697,881,64,107]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776,463,327,721]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230,367,890,953]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76,708,101,928]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96,841,811,979]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522,185,949,590]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14,921,544,106]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922,372,581,629]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857,894,60,426]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687,903,620,572]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579,79,258,42]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982,826,488,314]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973,898,556,293]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264,6,603,317]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34,497,108,180]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697,805,976,758]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686,982,0,32]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61,559,846,994]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597,904,897,376]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465,429,614,711]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329,320,645,829]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606,465,354,404]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876,984,491,784]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966,217,501,536]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261,389,281,467]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262,884,215,687]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93,880,959,111]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50,182,541,483]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106,877,130,387]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796,835,714,751]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103,95,634,844]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471,753,350,224]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573,128,154,326]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327,509,325,245]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894,884,824,343]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426,427,736,836]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450,677,659,183]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413,136,411,549]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234,756,496,779]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799,21,13,874]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785,107,816,89]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605,837,1,141]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989,999,138,674]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590,840,586,594]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713,237,468,340]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886,650,827,663]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270,619,450,707]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58,354,16,905]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838,131,387,664]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797,675,452,129]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605,453,987,316]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607,95,121,784]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189,627,592,241]
+  CRUSH rule 0 x 257 [825,264,867,529,409,291,732,224,841,681]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863,427,873,223]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77,191,669,61]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533,646,475,686]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516,136,87,410]
+  CRUSH rule 0 x 262 [223,1,561,420,16,88,534,289,498,357]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773,979,719,421]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715,916,633,121]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833,987,445,23]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7,557,76,987]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972,645,882,451]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715,536,983,688]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146,14,414,236]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476,571,314,538]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263,138,526,607]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710,779,161,222]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982,735,58,1]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370,990,320,52]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992,823,321,784]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634,98,457,487]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783,648,927,285]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350,600,283,422]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952,250,206,714]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895,914,892,149]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443,604,638,571]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939,879,338,786]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369,544,721,267]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60,546,39,71]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582,975,810,703]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699,627,725,389]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137,890,487,621]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195,538,356,523]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795,377,825,874]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473,209,59,346]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557,86,89,417]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806,911,738,969]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775,636,232,518]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459,833,503,207]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440,73,29,38]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266,82,671,320]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232,343,831,540]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213,286,61,961]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924,106,63,457]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904,838,458,828]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220,432,393,433]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400,365,697,497]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561,905,522,342]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214,944,845,895]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685,838,3,983]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761,648,867,488]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459,250,415,194]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976,551,126,307]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857,153,970,99]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90,855,441,736]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256,798,743,645]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497,929,92,639]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24,81,215,850]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952,599,763,183]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749,841,969,314]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761,725,923,461]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892,86,214,189]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290,97,529,220]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800,868,364,427]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523,366,891,998]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868,841,563,961]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187,380,947,952]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236,969,232,741]
+  CRUSH rule 0 x 324 [16,745,511,439,272,668,959,845,759,659]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462,268,124,431]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461,509,75,767]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750,669,296,314]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33,806,181,597]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516,713,223,395]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420,520,395,665]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624,649,626,928]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644,821,238,85]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243,633,309,616]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408,175,189,500]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635,367,997,819]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698,533,683,851]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956,907,464,39]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69,11,624,272]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607,953,125,899]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707,328,325,81]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720,503,209,748]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516,661,985,860]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660,655,149,994]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142,114,108,961]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127,732,290,494]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419,962,304,12]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783,661,622,426]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862,513,112,334]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795,143,529,765]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841,164,765,563]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766,775,518,155]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899,665,802,620]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279,616,892,262]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612,33,122,190]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985,412,599,375]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629,228,150,889]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891,656,196,232]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874,643,550,633]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397,52,702,981]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29,529,156,595]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643,286,552,985]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265,35,662,708]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444,613,499,389]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203,17,743,570]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113,373,426,941]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59,914,521,629]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685,641,894,813]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616,93,283,353]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284,184,718,684]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850,419,307,965]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797,547,778,958]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212,241,346,153]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128,207,88,641]
+  CRUSH rule 0 x 374 [191,887,920,223,714,961,760,571,549,723]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934,619,527,735]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446,897,460,869]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654,487,891,733]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728,860,968,475]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115,557,165,292]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255,806,429,306]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773,496,433,750]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276,56,278,772]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29,197,822,218]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488,435,223,783]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910,329,396,482]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232,692,742,939]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164,934,297,902]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474,267,640,53]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340,222,74,503]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852,196,469,672]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,136,818,516,19]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901,82,695,640]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652,543,10,287]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15,591,420,525]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126,179,607,623]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726,935,278,138]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996,558,602,528]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349,672,826,569]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749,410,815,214]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536,592,886,981]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79,32,343,468]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658,449,886,260]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202,650,501,161]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299,577,563,37]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776,296,19,972]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718,578,580,249]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72,345,709,89]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246,695,76,647]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213,806,168,907]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949,507,361,615]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313,351,497,571]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560,155,661,678]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618,907,9,291]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97,124,229,641]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694,332,572,681]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889,344,838,541]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138,374,470,59]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848,972,781,361]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823,884,832,361]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506,670,285,2]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764,274,623,613]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839,547,599,779]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905,588,312,114]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785,395,884,360]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80,236,988,682]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791,565,352,949]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334,588,571,428]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225,94,303,757]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857,562,976,977]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859,699,91,148]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194,366,279,30]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513,591,599,474]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52,897,705,680]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644,646,747,123]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371,731,906,721]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10,976,284,126]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443,23,696,245]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755,841,94,52]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414,152,331,985]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800,174,117,518]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205,236,86,929]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864,799,762,611]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774,350,37,929]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182,145,347,172]
+  CRUSH rule 0 x 445 [485,528,209,964,753,554,931,638,892,46]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314,714,212,646]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58,531,827,968]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807,688,21,841]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258,548,905,357]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721,31,312,491]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878,33,3,917]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604,49,922,414]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514,5,494,960]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106,103,511,624]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295,967,132,319]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504,96,890,230]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762,975,485,228]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399,558,15,441]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147,483,517,733]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841,512,70,914]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385,797,917,561]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240,911,375,412]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538,111,232,539]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789,0,653,566]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137,466,531,493]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659,35,797,235]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187,640,601,622]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201,249,555,646]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681,551,768,522]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558,71,676,655]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256,978,409,691]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809,56,476,137]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756,663,167,196]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446,333,245,689]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237,764,682,318]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984,226,333,916]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753,794,237,996]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941,843,751,451]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158,489,515,496]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371,807,687,932]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135,375,613,657]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731,233,640,492]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448,437,219,982]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283,170,278,77]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866,661,581,943]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621,955,400,261]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857,782,24,970]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924,965,25,925]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154,478,512,528]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566,779,507,133]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188,563,490,369]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591,423,668,365]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231,474,946,897]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389,819,956,597]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823,789,217,720]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129,763,39,651]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542,745,131,753]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949,173,398,586]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440,509,295,921]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930,573,357,491]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880,279,616,919]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490,852,962,387]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262,926,316,527]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758,104,831,710]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764,558,891,406]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898,161,78,414]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591,702,346,619]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313,282,207,626]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798,808,139,377]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786,152,921,884]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600,968,355,109]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899,451,415,920]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52,185,197,986]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374,985,944,101]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481,820,517,850]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453,660,93,134]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519,737,20,892]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625,434,527,630]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375,328,3,688]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558,956,664,468]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495,354,665,945]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753,624,769,853]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834,212,906,305]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8,550,769,975]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45,478,911,233]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853,643,625,43]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270,35,809,509]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651,368,118,1]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278,926,280,321]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577,509,195,722]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925,474,669,212]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949,374,988,367]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54,931,4,279]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206,682,627,827]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859,30,61,75]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148,668,436,192]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152,797,492,432]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37,485,88,273]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528,722,59,237]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95,789,348,342]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738,277,225,142]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137,934,547,351]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324,134,839,685]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250,362,614,123]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117,172,764,372]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699,610,785,811]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428,267,229,379]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517,684,154,108]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37,639,934,330]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915,570,546,724]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161,901,342,941]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57,217,21,70]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218,916,561,416]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693,953,606,894]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623,659,403,573]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830,762,691,951]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593,267,17,386]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124,107,766,260]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185,356,87,113]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887,376,658,366]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700,339,687,507]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62,817,766,376]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903,900,812,290]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641,751,930,856]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738,873,47,84]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184,127,448,327]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267,639,646,85]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398,521,59,250]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494,624,352,562]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533,674,23,683]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329,848,235,502]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33,312,202,355]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878,978,1,291]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333,841,193,146]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389,486,10,402]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177,545,839,722]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700,521,825,444]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487,976,977,58]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557,630,468,348]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608,755,326,604]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874,902,958,415]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984,622,175,804]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469,193,737,681]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642,226,122,703]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73,628,884,255]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794,682,127,372]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952,421,875,65]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195,581,649,488]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753,180,160,465]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811,413,436,162]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54,796,369,958]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667,318,496,74]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269,455,168,635]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994,16,777,901]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917,525,945,715]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180,482,449,647]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725,991,955,316]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286,280,507,912]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599,364,546,849]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243,264,31,897]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633,596,988,727]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697,763,442,542]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924,667,356,316]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983,281,292,953]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857,899,349,926]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954,94,88,617]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578,386,226,269]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148,33,420,669]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249,41,778,929]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160,888,767,727]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570,43,362,401]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296,735,245,714]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958,580,197,897]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92,12,62,719]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149,401,14,59]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74,307,976,613]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364,8,342,512]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222,749,697,853]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519,967,737,892]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35,620,268,902]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725,118,875,359]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446,798,869,586]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485,478,185,854]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137,140,488,725]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308,111,207,48]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225,841,868,249]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467,432,944,781]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205,580,972,427]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538,46,300,112]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920,830,171,759]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219,211,157,614]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416,45,624,253]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275,599,219,357]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296,337,676,416]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849,357,317,599]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236,318,228,575]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887,57,16,319]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701,887,355,632]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27,214,226,515]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858,138,726,1]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897,449,42,193]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610,665,299,852]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500,631,833,439]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660,857,123,910]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279,317,414,917]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720,135,369,711]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372,890,30,747]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453,732,262,803]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722,748,599,556]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869,150,743,438]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715,221,31,92]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119,818,459,782]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198,291,71,792]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,506,546,266,489]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605,214,833,862]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708,6,746,734]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858,16,128,144]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547,850,499,125]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170,954,22,394]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63,859,215,171]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402,205,609,831]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464,902,276,289]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170,65,906,943]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585,260,254,209]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798,658,28,334]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796,56,736,0]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795,783,386,956]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943,554,395,713]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938,600,517,49]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168,302,681,978]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755,330,584,525]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954,315,735,751]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371,402,725,870]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229,506,565,362]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528,875,982,782]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673,569,701,403]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103,763,527,83]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666,352,576,959]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247,620,437,29]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795,347,831,761]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774,192,454,380]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367,718,345,677]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345,599,30,892]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652,821,641,757]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154,121,804,85]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911,638,690,393]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917,622,156,826]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115,248,566,11]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333,161,74,907]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970,639,835,238]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451,290,789,853]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268,525,425,499]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828,276,264,237]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488,804,501,885]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300,989,804,72]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664,489,598,261]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153,922,986,480]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468,180,783,915]
+  CRUSH rule 0 x 699 [450,244,180,919,332,747,453,519,100,401]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68,641,109,182]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585,513,907,414]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466,687,742,170]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796,607,894,1]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655,595,888,354]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372,253,807,28]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923,770,516,982]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2,133,256,374]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939,331,41,175]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733,607,763,845]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449,28,731,573]
+  CRUSH rule 0 x 711 [227,653,731,150,842,534,110,639,452,502]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895,459,341,833]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53,839,56,829]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875,649,23,79]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322,417,973,582]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138,197,32,415]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237,62,327,958]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544,891,210,370]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828,975,119,87]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214,76,870,779]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177,264,867,327]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823,833,1,774]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985,846,77,467]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616,879,480,309]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359,720,128,627]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966,535,669,74]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105,31,506,430]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753,760,254,379]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572,933,434,816]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760,800,945,743]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528,890,625,535]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368,271,778,172]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880,343,590,539]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535,627,224,790]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690,59,294,569]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845,278,559,613]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482,564,259,896]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429,397,872,716]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146,17,314,156]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214,494,593,155]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863,571,876,528]
+  CRUSH rule 0 x 742 [912,581,114,730,21,687,81,145,695,245]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451,631,911,829]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843,699,24,714]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926,314,771,910]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450,651,105,921]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879,857,727,565]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304,999,685,843]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383,506,570,828]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169,171,9,58]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388,17,892,537]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88,315,728,669]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481,702,431,675]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336,724,262,61]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192,279,371,970]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850,191,991,63]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121,690,544,28]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165,686,449,831]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15,347,311,552]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345,69,39,199]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373,361,684,654]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664,350,339,980]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387,775,797,990]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70,202,557,471]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402,123,902,592]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357,310,890,850]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251,951,675,322]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645,975,947,199]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213,527,546,42]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961,324,309,772]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933,571,822,256]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891,959,487,355]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783,951,982,160]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890,425,599,485]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885,976,936,221]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396,76,680,108]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563,860,388,456]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65,954,125,358]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772,645,254,759]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17,336,147,829]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714,954,271,58]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812,516,774,985]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626,961,576,119]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481,198,171,944]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632,151,586,360]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490,481,627,740]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600,528,966,556]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194,216,486,782]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237,916,374,670]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788,164,691,329]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124,742,990,317]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406,858,694,351]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376,812,886,264]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811,459,344,719]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268,94,648,402]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646,451,67,738]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596,196,93,636]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473,250,839,895]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859,994,831,60]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941,643,398,325]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219,662,142,28]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846,53,783,511]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834,881,23,229]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663,949,380,39]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121,157,934,660]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640,607,790,832]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802,136,305,983]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166,929,234,460]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23,145,184,465]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620,648,433,997]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948,202,521,278]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675,28,263,410]
+  CRUSH rule 0 x 813 [805,114,683,629,462,285,450,948,742,605]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401,266,905,320]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314,670,928,727]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44,822,332,959]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260,172,193,516]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769,826,519,277]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523,770,56,437]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535,787,267,46]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621,815,313,916]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883,112,601,15]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282,641,216,929]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559,886,753,749]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161,998,370,10]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910,63,288,836]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212,916,692,653]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937,595,935,672]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669,330,822,36]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448,55,965,851]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142,157,957,920]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656,135,341,922]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764,766,853,890]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83,483,811,98]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537,937,101,507]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351,961,52,44]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903,29,454,647]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983,998,320,935]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47,859,402,947]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815,109,762,642]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226,46,918,220]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575,1,312,542]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486,199,966,501]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899,382,851,472]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611,512,855,760]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606,237,414,844]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363,893,592,634]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700,249,214,100]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847,463,995,314]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358,35,913,310]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95,337,677,275]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469,511,999,340]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774,679,616,492]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441,302,437,53]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686,702,919,971]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224,810,404,126]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12,763,594,668]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688,61,545,491]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463,903,272,8]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699,907,532,627]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901,487,725,204]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490,409,15,842]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172,980,772,115]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43,15,29,974]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850,40,170,31]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255,481,435,119]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988,730,753,888]
+  CRUSH rule 0 x 868 [613,950,712,663,460,643,547,734,16,649]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718,431,968,472]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54,992,726,783]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387,515,98,232]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180,800,521,270]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199,358,805,255]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93,915,801,43]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634,435,778,884]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322,141,82,412]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181,460,459,527]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329,591,59,4]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145,750,54,997]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319,630,888,970]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570,109,896,826]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974,483,412,547]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202,148,129,72]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389,381,347,314]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174,972,544,894]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56,582,915,541]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472,947,35,528]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806,53,385,387]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399,382,761,907]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996,561,485,482]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706,701,501,680]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53,333,458,77]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672,320,394,373]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648,685,895,735]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599,829,809,713]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210,529,491,289]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12,481,539,562]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342,37,303,70]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124,653,574,384]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380,419,667,56]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157,292,724,805]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178,624,733,498]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66,738,419,199]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957,801,74,934]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685,6,689,387]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30,169,506,497]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664,331,184,461]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484,391,841,967]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686,965,515,421]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824,552,717,159]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789,255,896,868]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226,889,69,237]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925,746,664,533]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70,280,837,367]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126,79,652,793]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255,133,892,524]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725,707,47,216]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583,923,863,232]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888,980,33,261]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532,124,911,206]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633,290,155,137]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952,474,67,938]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964,384,454,448]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584,31,151,178]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969,168,500,68]
+  CRUSH rule 0 x 926 [968,133,144,814,155,709,158,96,739,175]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465,775,725,414]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847,752,780,738]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744,171,144,704]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832,654,567,160]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487,839,776,117]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530,462,202,11]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568,61,945,154]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533,116,254,569]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772,49,567,47]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138,802,885,447]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630,308,973,934]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286,395,927,611]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408,49,977,243]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25,883,867,270]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964,28,633,960]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,16,440,31,906]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759,851,617,420]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678,915,619,69]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952,885,987,775]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933,267,846,866]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922,868,413,545]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81,220,725,211]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799,961,564,71]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355,220,528,747]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829,408,568,734]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16,746,835,529]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141,619,234,517]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129,696,115,984]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536,766,579,398]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169,751,667,381]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393,281,227,412]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346,377,425,533]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543,519,496,956]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855,49,548,861]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455,708,649,93]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856,477,895,89]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485,701,76,55]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361,255,507,540]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59,229,819,610]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929,773,928,579]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19,674,395,483]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433,520,916,41]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518,956,398,269]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576,411,416,523]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344,520,546,612]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754,540,79,174]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549,524,109,364]
+  CRUSH rule 0 x 974 [545,758,586,596,790,116,993,644,405,869]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330,308,744,843]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397,58,57,603]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783,828,409,529]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514,169,99,878]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677,173,903,781]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398,943,764,867]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536,212,247,523]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395,198,100,580]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927,597,103,461]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950,840,427,469]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911,208,91,999]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864,555,687,212]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76,894,480,323]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865,47,116,230]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155,380,797,177]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340,526,728,595]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754,928,288,668]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117,546,798,777]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382,428,451,216]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557,378,936,192]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158,265,110,171]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459,764,859,776]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210,698,273,955]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786,762,444,561]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15,585,898,902]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317,102,200,662]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739,922,343,574]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729,370,326,640]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614,783,865,245]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891,113,424,320]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597,969,911,241]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234,945,598,498]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5,444,795,150]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975,723,909,93]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234,383,604,903]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23,241,838,865]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48,570,639,230]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268,647,669,90]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808,351,572,403]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969,81,586,62]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145,11,965,669]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456,560,935,733]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637,731,738,154]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463,926,475,316]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768,245,337,197]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495,865,698,164]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676,322,671,622]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871,828,440,449]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50,560,595,554]
+  rule 0 (data) num_rep 10 result size == 10:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-firefly-tunables.t b/ceph/src/test/cli/crushtool/test-map-firefly-tunables.t
new file mode 100644
index 00000000..481b6fd5
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-firefly-tunables.t
@@ -0,0 +1,10259 @@
+  $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 0 --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 --set-chooseleaf-descend-once 1 --set-chooseleaf-vary-r 1 --weight 12 0 --weight 20 0 --weight 30 0
+  crushtool successfully built or modified map.  Use '-o ' to write it out.
+  rule 0 (data), x = 0..1023, numrep = 1..10
+  CRUSH rule 0 x 0 [101]
+  CRUSH rule 0 x 1 [80]
+  CRUSH rule 0 x 2 [91]
+  CRUSH rule 0 x 3 [51]
+  CRUSH rule 0 x 4 [50]
+  CRUSH rule 0 x 5 [89]
+  CRUSH rule 0 x 6 [91]
+  CRUSH rule 0 x 7 [104]
+  CRUSH rule 0 x 8 [78]
+  CRUSH rule 0 x 9 [101]
+  CRUSH rule 0 x 10 [61]
+  CRUSH rule 0 x 11 [13]
+  CRUSH rule 0 x 12 [83]
+  CRUSH rule 0 x 13 [108]
+  CRUSH rule 0 x 14 [105]
+  CRUSH rule 0 x 15 [18]
+  CRUSH rule 0 x 16 [103]
+  CRUSH rule 0 x 17 [85]
+  CRUSH rule 0 x 18 [11]
+  CRUSH rule 0 x 19 [75]
+  CRUSH rule 0 x 20 [79]
+  CRUSH rule 0 x 21 [84]
+  CRUSH rule 0 x 22 [23]
+  CRUSH rule 0 x 23 [118]
+  CRUSH rule 0 x 24 [83]
+  CRUSH rule 0 x 25 [81]
+  CRUSH rule 0 x 26 [38]
+  CRUSH rule 0 x 27 [76]
+  CRUSH rule 0 x 28 [76]
+  CRUSH rule 0 x 29 [24]
+  CRUSH rule 0 x 30 [94]
+  CRUSH rule 0 x 31 [76]
+  CRUSH rule 0 x 32 [72]
+  CRUSH rule 0 x 33 [77]
+  CRUSH rule 0 x 34 [7]
+  CRUSH rule 0 x 35 [22]
+  CRUSH rule 0 x 36 [104]
+  CRUSH rule 0 x 37 [61]
+  CRUSH rule 0 x 38 [72]
+  CRUSH rule 0 x 39 [68]
+  CRUSH rule 0 x 40 [103]
+  CRUSH rule 0 x 41 [85]
+  CRUSH rule 0 x 42 [106]
+  CRUSH rule 0 x 43 [10]
+  CRUSH rule 0 x 44 [101]
+  CRUSH rule 0 x 45 [83]
+  CRUSH rule 0 x 46 [65]
+  CRUSH rule 0 x 47 [106]
+  CRUSH rule 0 x 48 [34]
+  CRUSH rule 0 x 49 [0]
+  CRUSH rule 0 x 50 [42]
+  CRUSH rule 0 x 51 [104]
+  CRUSH rule 0 x 52 [83]
+  CRUSH rule 0 x 53 [32]
+  CRUSH rule 0 x 54 [9]
+  CRUSH rule 0 x 55 [14]
+  CRUSH rule 0 x 56 [21]
+  CRUSH rule 0 x 57 [93]
+  CRUSH rule 0 x 58 [45]
+  CRUSH rule 0 x 59 [80]
+  CRUSH rule 0 x 60 [90]
+  CRUSH rule 0 x 61 [88]
+  CRUSH rule 0 x 62 [81]
+  CRUSH rule 0 x 63 [79]
+  CRUSH rule 0 x 64 [1]
+  CRUSH rule 0 x 65 [32]
+  CRUSH rule 0 x 66 [48]
+  CRUSH rule 0 x 67 [94]
+  CRUSH rule 0 x 68 [102]
+  CRUSH rule 0 x 69 [62]
+  CRUSH rule 0 x 70 [84]
+  CRUSH rule 0 x 71 [9]
+  CRUSH rule 0 x 72 [97]
+  CRUSH rule 0 x 73 [64]
+  CRUSH rule 0 x 74 [29]
+  CRUSH rule 0 x 75 [29]
+  CRUSH rule 0 x 76 [55]
+  CRUSH rule 0 x 77 [107]
+  CRUSH rule 0 x 78 [11]
+  CRUSH rule 0 x 79 [64]
+  CRUSH rule 0 x 80 [0]
+  CRUSH rule 0 x 81 [71]
+  CRUSH rule 0 x 82 [37]
+  CRUSH rule 0 x 83 [92]
+  CRUSH rule 0 x 84 [49]
+  CRUSH rule 0 x 85 [54]
+  CRUSH rule 0 x 86 [37]
+  CRUSH rule 0 x 87 [116]
+  CRUSH rule 0 x 88 [38]
+  CRUSH rule 0 x 89 [76]
+  CRUSH rule 0 x 90 [14]
+  CRUSH rule 0 x 91 [68]
+  CRUSH rule 0 x 92 [86]
+  CRUSH rule 0 x 93 [44]
+  CRUSH rule 0 x 94 [61]
+  CRUSH rule 0 x 95 [93]
+  CRUSH rule 0 x 96 [66]
+  CRUSH rule 0 x 97 [111]
+  CRUSH rule 0 x 98 [93]
+  CRUSH rule 0 x 99 [78]
+  CRUSH rule 0 x 100 [6]
+  CRUSH rule 0 x 101 [84]
+  CRUSH rule 0 x 102 [82]
+  CRUSH rule 0 x 103 [66]
+  CRUSH rule 0 x 104 [14]
+  CRUSH rule 0 x 105 [87]
+  CRUSH rule 0 x 106 [69]
+  CRUSH rule 0 x 107 [1]
+  CRUSH rule 0 x 108 [94]
+  CRUSH rule 0 x 109 [112]
+  CRUSH rule 0 x 110 [54]
+  CRUSH rule 0 x 111 [10]
+  CRUSH rule 0 x 112 [89]
+  CRUSH rule 0 x 113 [69]
+  CRUSH rule 0 x 114 [79]
+  CRUSH rule 0 x 115 [50]
+  CRUSH rule 0 x 116 [96]
+  CRUSH rule 0 x 117 [87]
+  CRUSH rule 0 x 118 [23]
+  CRUSH rule 0 x 119 [104]
+  CRUSH rule 0 x 120 [57]
+  CRUSH rule 0 x 121 [105]
+  CRUSH rule 0 x 122 [45]
+  CRUSH rule 0 x 123 [112]
+  CRUSH rule 0 x 124 [110]
+  CRUSH rule 0 x 125 [66]
+  CRUSH rule 0 x 126 [51]
+  CRUSH rule 0 x 127 [70]
+  CRUSH rule 0 x 128 [90]
+  CRUSH rule 0 x 129 [103]
+  CRUSH rule 0 x 130 [50]
+  CRUSH rule 0 x 131 [23]
+  CRUSH rule 0 x 132 [69]
+  CRUSH rule 0 x 133 [52]
+  CRUSH rule 0 x 134 [78]
+  CRUSH rule 0 x 135 [78]
+  CRUSH rule 0 x 136 [32]
+  CRUSH rule 0 x 137 [11]
+  CRUSH rule 0 x 138 [17]
+  CRUSH rule 0 x 139 [89]
+  CRUSH rule 0 x 140 [39]
+  CRUSH rule 0 x 141 [89]
+  CRUSH rule 0 x 142 [70]
+  CRUSH rule 0 x 143 [51]
+  CRUSH rule 0 x 144 [13]
+  CRUSH rule 0 x 145 [77]
+  CRUSH rule 0 x 146 [8]
+  CRUSH rule 0 x 147 [22]
+  CRUSH rule 0 x 148 [74]
+  CRUSH rule 0 x 149 [76]
+  CRUSH rule 0 x 150 [14]
+  CRUSH rule 0 x 151 [90]
+  CRUSH rule 0 x 152 [49]
+  CRUSH rule 0 x 153 [71]
+  CRUSH rule 0 x 154 [94]
+  CRUSH rule 0 x 155 [75]
+  CRUSH rule 0 x 156 [94]
+  CRUSH rule 0 x 157 [112]
+  CRUSH rule 0 x 158 [26]
+  CRUSH rule 0 x 159 [52]
+  CRUSH rule 0 x 160 [41]
+  CRUSH rule 0 x 161 [19]
+  CRUSH rule 0 x 162 [55]
+  CRUSH rule 0 x 163 [54]
+  CRUSH rule 0 x 164 [45]
+  CRUSH rule 0 x 165 [25]
+  CRUSH rule 0 x 166 [73]
+  CRUSH rule 0 x 167 [89]
+  CRUSH rule 0 x 168 [47]
+  CRUSH rule 0 x 169 [51]
+  CRUSH rule 0 x 170 [68]
+  CRUSH rule 0 x 171 [73]
+  CRUSH rule 0 x 172 [33]
+  CRUSH rule 0 x 173 [102]
+  CRUSH rule 0 x 174 [116]
+  CRUSH rule 0 x 175 [3]
+  CRUSH rule 0 x 176 [94]
+  CRUSH rule 0 x 177 [52]
+  CRUSH rule 0 x 178 [39]
+  CRUSH rule 0 x 179 [72]
+  CRUSH rule 0 x 180 [60]
+  CRUSH rule 0 x 181 [18]
+  CRUSH rule 0 x 182 [22]
+  CRUSH rule 0 x 183 [11]
+  CRUSH rule 0 x 184 [92]
+  CRUSH rule 0 x 185 [97]
+  CRUSH rule 0 x 186 [67]
+  CRUSH rule 0 x 187 [116]
+  CRUSH rule 0 x 188 [69]
+  CRUSH rule 0 x 189 [47]
+  CRUSH rule 0 x 190 [90]
+  CRUSH rule 0 x 191 [49]
+  CRUSH rule 0 x 192 [68]
+  CRUSH rule 0 x 193 [0]
+  CRUSH rule 0 x 194 [17]
+  CRUSH rule 0 x 195 [119]
+  CRUSH rule 0 x 196 [72]
+  CRUSH rule 0 x 197 [106]
+  CRUSH rule 0 x 198 [114]
+  CRUSH rule 0 x 199 [0]
+  CRUSH rule 0 x 200 [35]
+  CRUSH rule 0 x 201 [14]
+  CRUSH rule 0 x 202 [98]
+  CRUSH rule 0 x 203 [36]
+  CRUSH rule 0 x 204 [10]
+  CRUSH rule 0 x 205 [22]
+  CRUSH rule 0 x 206 [49]
+  CRUSH rule 0 x 207 [80]
+  CRUSH rule 0 x 208 [63]
+  CRUSH rule 0 x 209 [85]
+  CRUSH rule 0 x 210 [79]
+  CRUSH rule 0 x 211 [26]
+  CRUSH rule 0 x 212 [28]
+  CRUSH rule 0 x 213 [91]
+  CRUSH rule 0 x 214 [78]
+  CRUSH rule 0 x 215 [61]
+  CRUSH rule 0 x 216 [99]
+  CRUSH rule 0 x 217 [86]
+  CRUSH rule 0 x 218 [93]
+  CRUSH rule 0 x 219 [28]
+  CRUSH rule 0 x 220 [56]
+  CRUSH rule 0 x 221 [0]
+  CRUSH rule 0 x 222 [50]
+  CRUSH rule 0 x 223 [29]
+  CRUSH rule 0 x 224 [52]
+  CRUSH rule 0 x 225 [61]
+  CRUSH rule 0 x 226 [44]
+  CRUSH rule 0 x 227 [42]
+  CRUSH rule 0 x 228 [117]
+  CRUSH rule 0 x 229 [100]
+  CRUSH rule 0 x 230 [41]
+  CRUSH rule 0 x 231 [56]
+  CRUSH rule 0 x 232 [23]
+  CRUSH rule 0 x 233 [88]
+  CRUSH rule 0 x 234 [4]
+  CRUSH rule 0 x 235 [26]
+  CRUSH rule 0 x 236 [32]
+  CRUSH rule 0 x 237 [92]
+  CRUSH rule 0 x 238 [10]
+  CRUSH rule 0 x 239 [15]
+  CRUSH rule 0 x 240 [109]
+  CRUSH rule 0 x 241 [47]
+  CRUSH rule 0 x 242 [24]
+  CRUSH rule 0 x 243 [76]
+  CRUSH rule 0 x 244 [96]
+  CRUSH rule 0 x 245 [27]
+  CRUSH rule 0 x 246 [35]
+  CRUSH rule 0 x 247 [99]
+  CRUSH rule 0 x 248 [8]
+  CRUSH rule 0 x 249 [85]
+  CRUSH rule 0 x 250 [79]
+  CRUSH rule 0 x 251 [28]
+  CRUSH rule 0 x 252 [95]
+  CRUSH rule 0 x 253 [109]
+  CRUSH rule 0 x 254 [80]
+  CRUSH rule 0 x 255 [112]
+  CRUSH rule 0 x 256 [37]
+  CRUSH rule 0 x 257 [69]
+  CRUSH rule 0 x 258 [34]
+  CRUSH rule 0 x 259 [70]
+  CRUSH rule 0 x 260 [98]
+  CRUSH rule 0 x 261 [94]
+  CRUSH rule 0 x 262 [42]
+  CRUSH rule 0 x 263 [65]
+  CRUSH rule 0 x 264 [36]
+  CRUSH rule 0 x 265 [66]
+  CRUSH rule 0 x 266 [75]
+  CRUSH rule 0 x 267 [58]
+  CRUSH rule 0 x 268 [38]
+  CRUSH rule 0 x 269 [43]
+  CRUSH rule 0 x 270 [58]
+  CRUSH rule 0 x 271 [19]
+  CRUSH rule 0 x 272 [73]
+  CRUSH rule 0 x 273 [108]
+  CRUSH rule 0 x 274 [47]
+  CRUSH rule 0 x 275 [92]
+  CRUSH rule 0 x 276 [7]
+  CRUSH rule 0 x 277 [19]
+  CRUSH rule 0 x 278 [116]
+  CRUSH rule 0 x 279 [101]
+  CRUSH rule 0 x 280 [113]
+  CRUSH rule 0 x 281 [14]
+  CRUSH rule 0 x 282 [106]
+  CRUSH rule 0 x 283 [8]
+  CRUSH rule 0 x 284 [10]
+  CRUSH rule 0 x 285 [88]
+  CRUSH rule 0 x 286 [27]
+  CRUSH rule 0 x 287 [84]
+  CRUSH rule 0 x 288 [103]
+  CRUSH rule 0 x 289 [9]
+  CRUSH rule 0 x 290 [115]
+  CRUSH rule 0 x 291 [48]
+  CRUSH rule 0 x 292 [52]
+  CRUSH rule 0 x 293 [27]
+  CRUSH rule 0 x 294 [79]
+  CRUSH rule 0 x 295 [37]
+  CRUSH rule 0 x 296 [56]
+  CRUSH rule 0 x 297 [35]
+  CRUSH rule 0 x 298 [71]
+  CRUSH rule 0 x 299 [79]
+  CRUSH rule 0 x 300 [67]
+  CRUSH rule 0 x 301 [51]
+  CRUSH rule 0 x 302 [78]
+  CRUSH rule 0 x 303 [19]
+  CRUSH rule 0 x 304 [101]
+  CRUSH rule 0 x 305 [81]
+  CRUSH rule 0 x 306 [0]
+  CRUSH rule 0 x 307 [44]
+  CRUSH rule 0 x 308 [91]
+  CRUSH rule 0 x 309 [15]
+  CRUSH rule 0 x 310 [26]
+  CRUSH rule 0 x 311 [36]
+  CRUSH rule 0 x 312 [33]
+  CRUSH rule 0 x 313 [104]
+  CRUSH rule 0 x 314 [28]
+  CRUSH rule 0 x 315 [16]
+  CRUSH rule 0 x 316 [4]
+  CRUSH rule 0 x 317 [118]
+  CRUSH rule 0 x 318 [32]
+  CRUSH rule 0 x 319 [24]
+  CRUSH rule 0 x 320 [36]
+  CRUSH rule 0 x 321 [26]
+  CRUSH rule 0 x 322 [87]
+  CRUSH rule 0 x 323 [73]
+  CRUSH rule 0 x 324 [64]
+  CRUSH rule 0 x 325 [52]
+  CRUSH rule 0 x 326 [111]
+  CRUSH rule 0 x 327 [62]
+  CRUSH rule 0 x 328 [7]
+  CRUSH rule 0 x 329 [93]
+  CRUSH rule 0 x 330 [24]
+  CRUSH rule 0 x 331 [41]
+  CRUSH rule 0 x 332 [61]
+  CRUSH rule 0 x 333 [16]
+  CRUSH rule 0 x 334 [94]
+  CRUSH rule 0 x 335 [71]
+  CRUSH rule 0 x 336 [16]
+  CRUSH rule 0 x 337 [37]
+  CRUSH rule 0 x 338 [109]
+  CRUSH rule 0 x 339 [13]
+  CRUSH rule 0 x 340 [119]
+  CRUSH rule 0 x 341 [63]
+  CRUSH rule 0 x 342 [92]
+  CRUSH rule 0 x 343 [49]
+  CRUSH rule 0 x 344 [103]
+  CRUSH rule 0 x 345 [56]
+  CRUSH rule 0 x 346 [3]
+  CRUSH rule 0 x 347 [106]
+  CRUSH rule 0 x 348 [10]
+  CRUSH rule 0 x 349 [96]
+  CRUSH rule 0 x 350 [63]
+  CRUSH rule 0 x 351 [60]
+  CRUSH rule 0 x 352 [103]
+  CRUSH rule 0 x 353 [10]
+  CRUSH rule 0 x 354 [55]
+  CRUSH rule 0 x 355 [73]
+  CRUSH rule 0 x 356 [114]
+  CRUSH rule 0 x 357 [70]
+  CRUSH rule 0 x 358 [97]
+  CRUSH rule 0 x 359 [4]
+  CRUSH rule 0 x 360 [106]
+  CRUSH rule 0 x 361 [27]
+  CRUSH rule 0 x 362 [28]
+  CRUSH rule 0 x 363 [45]
+  CRUSH rule 0 x 364 [23]
+  CRUSH rule 0 x 365 [57]
+  CRUSH rule 0 x 366 [14]
+  CRUSH rule 0 x 367 [108]
+  CRUSH rule 0 x 368 [103]
+  CRUSH rule 0 x 369 [11]
+  CRUSH rule 0 x 370 [11]
+  CRUSH rule 0 x 371 [34]
+  CRUSH rule 0 x 372 [58]
+  CRUSH rule 0 x 373 [6]
+  CRUSH rule 0 x 374 [110]
+  CRUSH rule 0 x 375 [19]
+  CRUSH rule 0 x 376 [22]
+  CRUSH rule 0 x 377 [93]
+  CRUSH rule 0 x 378 [67]
+  CRUSH rule 0 x 379 [77]
+  CRUSH rule 0 x 380 [3]
+  CRUSH rule 0 x 381 [55]
+  CRUSH rule 0 x 382 [26]
+  CRUSH rule 0 x 383 [48]
+  CRUSH rule 0 x 384 [15]
+  CRUSH rule 0 x 385 [82]
+  CRUSH rule 0 x 386 [108]
+  CRUSH rule 0 x 387 [70]
+  CRUSH rule 0 x 388 [5]
+  CRUSH rule 0 x 389 [14]
+  CRUSH rule 0 x 390 [68]
+  CRUSH rule 0 x 391 [113]
+  CRUSH rule 0 x 392 [72]
+  CRUSH rule 0 x 393 [115]
+  CRUSH rule 0 x 394 [38]
+  CRUSH rule 0 x 395 [0]
+  CRUSH rule 0 x 396 [59]
+  CRUSH rule 0 x 397 [87]
+  CRUSH rule 0 x 398 [44]
+  CRUSH rule 0 x 399 [9]
+  CRUSH rule 0 x 400 [19]
+  CRUSH rule 0 x 401 [79]
+  CRUSH rule 0 x 402 [107]
+  CRUSH rule 0 x 403 [23]
+  CRUSH rule 0 x 404 [76]
+  CRUSH rule 0 x 405 [10]
+  CRUSH rule 0 x 406 [38]
+  CRUSH rule 0 x 407 [70]
+  CRUSH rule 0 x 408 [55]
+  CRUSH rule 0 x 409 [102]
+  CRUSH rule 0 x 410 [59]
+  CRUSH rule 0 x 411 [34]
+  CRUSH rule 0 x 412 [108]
+  CRUSH rule 0 x 413 [54]
+  CRUSH rule 0 x 414 [70]
+  CRUSH rule 0 x 415 [107]
+  CRUSH rule 0 x 416 [21]
+  CRUSH rule 0 x 417 [8]
+  CRUSH rule 0 x 418 [51]
+  CRUSH rule 0 x 419 [8]
+  CRUSH rule 0 x 420 [109]
+  CRUSH rule 0 x 421 [114]
+  CRUSH rule 0 x 422 [109]
+  CRUSH rule 0 x 423 [59]
+  CRUSH rule 0 x 424 [71]
+  CRUSH rule 0 x 425 [101]
+  CRUSH rule 0 x 426 [47]
+  CRUSH rule 0 x 427 [8]
+  CRUSH rule 0 x 428 [68]
+  CRUSH rule 0 x 429 [76]
+  CRUSH rule 0 x 430 [69]
+  CRUSH rule 0 x 431 [70]
+  CRUSH rule 0 x 432 [46]
+  CRUSH rule 0 x 433 [6]
+  CRUSH rule 0 x 434 [64]
+  CRUSH rule 0 x 435 [16]
+  CRUSH rule 0 x 436 [89]
+  CRUSH rule 0 x 437 [29]
+  CRUSH rule 0 x 438 [105]
+  CRUSH rule 0 x 439 [29]
+  CRUSH rule 0 x 440 [38]
+  CRUSH rule 0 x 441 [112]
+  CRUSH rule 0 x 442 [55]
+  CRUSH rule 0 x 443 [44]
+  CRUSH rule 0 x 444 [72]
+  CRUSH rule 0 x 445 [19]
+  CRUSH rule 0 x 446 [40]
+  CRUSH rule 0 x 447 [13]
+  CRUSH rule 0 x 448 [7]
+  CRUSH rule 0 x 449 [67]
+  CRUSH rule 0 x 450 [117]
+  CRUSH rule 0 x 451 [93]
+  CRUSH rule 0 x 452 [70]
+  CRUSH rule 0 x 453 [82]
+  CRUSH rule 0 x 454 [53]
+  CRUSH rule 0 x 455 [91]
+  CRUSH rule 0 x 456 [101]
+  CRUSH rule 0 x 457 [113]
+  CRUSH rule 0 x 458 [53]
+  CRUSH rule 0 x 459 [25]
+  CRUSH rule 0 x 460 [105]
+  CRUSH rule 0 x 461 [102]
+  CRUSH rule 0 x 462 [98]
+  CRUSH rule 0 x 463 [108]
+  CRUSH rule 0 x 464 [19]
+  CRUSH rule 0 x 465 [29]
+  CRUSH rule 0 x 466 [66]
+  CRUSH rule 0 x 467 [6]
+  CRUSH rule 0 x 468 [97]
+  CRUSH rule 0 x 469 [98]
+  CRUSH rule 0 x 470 [50]
+  CRUSH rule 0 x 471 [40]
+  CRUSH rule 0 x 472 [74]
+  CRUSH rule 0 x 473 [95]
+  CRUSH rule 0 x 474 [51]
+  CRUSH rule 0 x 475 [49]
+  CRUSH rule 0 x 476 [110]
+  CRUSH rule 0 x 477 [25]
+  CRUSH rule 0 x 478 [47]
+  CRUSH rule 0 x 479 [70]
+  CRUSH rule 0 x 480 [62]
+  CRUSH rule 0 x 481 [26]
+  CRUSH rule 0 x 482 [84]
+  CRUSH rule 0 x 483 [15]
+  CRUSH rule 0 x 484 [37]
+  CRUSH rule 0 x 485 [47]
+  CRUSH rule 0 x 486 [92]
+  CRUSH rule 0 x 487 [106]
+  CRUSH rule 0 x 488 [42]
+  CRUSH rule 0 x 489 [76]
+  CRUSH rule 0 x 490 [68]
+  CRUSH rule 0 x 491 [80]
+  CRUSH rule 0 x 492 [21]
+  CRUSH rule 0 x 493 [99]
+  CRUSH rule 0 x 494 [4]
+  CRUSH rule 0 x 495 [40]
+  CRUSH rule 0 x 496 [93]
+  CRUSH rule 0 x 497 [102]
+  CRUSH rule 0 x 498 [68]
+  CRUSH rule 0 x 499 [10]
+  CRUSH rule 0 x 500 [50]
+  CRUSH rule 0 x 501 [60]
+  CRUSH rule 0 x 502 [11]
+  CRUSH rule 0 x 503 [117]
+  CRUSH rule 0 x 504 [90]
+  CRUSH rule 0 x 505 [91]
+  CRUSH rule 0 x 506 [82]
+  CRUSH rule 0 x 507 [81]
+  CRUSH rule 0 x 508 [34]
+  CRUSH rule 0 x 509 [88]
+  CRUSH rule 0 x 510 [11]
+  CRUSH rule 0 x 511 [72]
+  CRUSH rule 0 x 512 [118]
+  CRUSH rule 0 x 513 [22]
+  CRUSH rule 0 x 514 [82]
+  CRUSH rule 0 x 515 [27]
+  CRUSH rule 0 x 516 [66]
+  CRUSH rule 0 x 517 [83]
+  CRUSH rule 0 x 518 [18]
+  CRUSH rule 0 x 519 [67]
+  CRUSH rule 0 x 520 [15]
+  CRUSH rule 0 x 521 [63]
+  CRUSH rule 0 x 522 [56]
+  CRUSH rule 0 x 523 [36]
+  CRUSH rule 0 x 524 [33]
+  CRUSH rule 0 x 525 [3]
+  CRUSH rule 0 x 526 [83]
+  CRUSH rule 0 x 527 [37]
+  CRUSH rule 0 x 528 [108]
+  CRUSH rule 0 x 529 [107]
+  CRUSH rule 0 x 530 [49]
+  CRUSH rule 0 x 531 [27]
+  CRUSH rule 0 x 532 [68]
+  CRUSH rule 0 x 533 [5]
+  CRUSH rule 0 x 534 [97]
+  CRUSH rule 0 x 535 [8]
+  CRUSH rule 0 x 536 [3]
+  CRUSH rule 0 x 537 [116]
+  CRUSH rule 0 x 538 [85]
+  CRUSH rule 0 x 539 [10]
+  CRUSH rule 0 x 540 [100]
+  CRUSH rule 0 x 541 [111]
+  CRUSH rule 0 x 542 [50]
+  CRUSH rule 0 x 543 [45]
+  CRUSH rule 0 x 544 [106]
+  CRUSH rule 0 x 545 [43]
+  CRUSH rule 0 x 546 [108]
+  CRUSH rule 0 x 547 [67]
+  CRUSH rule 0 x 548 [58]
+  CRUSH rule 0 x 549 [60]
+  CRUSH rule 0 x 550 [47]
+  CRUSH rule 0 x 551 [14]
+  CRUSH rule 0 x 552 [70]
+  CRUSH rule 0 x 553 [96]
+  CRUSH rule 0 x 554 [61]
+  CRUSH rule 0 x 555 [76]
+  CRUSH rule 0 x 556 [106]
+  CRUSH rule 0 x 557 [39]
+  CRUSH rule 0 x 558 [70]
+  CRUSH rule 0 x 559 [106]
+  CRUSH rule 0 x 560 [94]
+  CRUSH rule 0 x 561 [27]
+  CRUSH rule 0 x 562 [97]
+  CRUSH rule 0 x 563 [64]
+  CRUSH rule 0 x 564 [96]
+  CRUSH rule 0 x 565 [66]
+  CRUSH rule 0 x 566 [27]
+  CRUSH rule 0 x 567 [88]
+  CRUSH rule 0 x 568 [106]
+  CRUSH rule 0 x 569 [102]
+  CRUSH rule 0 x 570 [98]
+  CRUSH rule 0 x 571 [95]
+  CRUSH rule 0 x 572 [62]
+  CRUSH rule 0 x 573 [51]
+  CRUSH rule 0 x 574 [89]
+  CRUSH rule 0 x 575 [87]
+  CRUSH rule 0 x 576 [112]
+  CRUSH rule 0 x 577 [8]
+  CRUSH rule 0 x 578 [64]
+  CRUSH rule 0 x 579 [78]
+  CRUSH rule 0 x 580 [68]
+  CRUSH rule 0 x 581 [55]
+  CRUSH rule 0 x 582 [15]
+  CRUSH rule 0 x 583 [74]
+  CRUSH rule 0 x 584 [22]
+  CRUSH rule 0 x 585 [35]
+  CRUSH rule 0 x 586 [33]
+  CRUSH rule 0 x 587 [106]
+  CRUSH rule 0 x 588 [0]
+  CRUSH rule 0 x 589 [7]
+  CRUSH rule 0 x 590 [40]
+  CRUSH rule 0 x 591 [42]
+  CRUSH rule 0 x 592 [45]
+  CRUSH rule 0 x 593 [89]
+  CRUSH rule 0 x 594 [27]
+  CRUSH rule 0 x 595 [7]
+  CRUSH rule 0 x 596 [82]
+  CRUSH rule 0 x 597 [72]
+  CRUSH rule 0 x 598 [34]
+  CRUSH rule 0 x 599 [119]
+  CRUSH rule 0 x 600 [24]
+  CRUSH rule 0 x 601 [104]
+  CRUSH rule 0 x 602 [48]
+  CRUSH rule 0 x 603 [24]
+  CRUSH rule 0 x 604 [89]
+  CRUSH rule 0 x 605 [104]
+  CRUSH rule 0 x 606 [49]
+  CRUSH rule 0 x 607 [95]
+  CRUSH rule 0 x 608 [112]
+  CRUSH rule 0 x 609 [61]
+  CRUSH rule 0 x 610 [106]
+  CRUSH rule 0 x 611 [66]
+  CRUSH rule 0 x 612 [103]
+  CRUSH rule 0 x 613 [13]
+  CRUSH rule 0 x 614 [81]
+  CRUSH rule 0 x 615 [61]
+  CRUSH rule 0 x 616 [41]
+  CRUSH rule 0 x 617 [111]
+  CRUSH rule 0 x 618 [26]
+  CRUSH rule 0 x 619 [92]
+  CRUSH rule 0 x 620 [108]
+  CRUSH rule 0 x 621 [106]
+  CRUSH rule 0 x 622 [67]
+  CRUSH rule 0 x 623 [94]
+  CRUSH rule 0 x 624 [115]
+  CRUSH rule 0 x 625 [111]
+  CRUSH rule 0 x 626 [3]
+  CRUSH rule 0 x 627 [19]
+  CRUSH rule 0 x 628 [65]
+  CRUSH rule 0 x 629 [6]
+  CRUSH rule 0 x 630 [22]
+  CRUSH rule 0 x 631 [35]
+  CRUSH rule 0 x 632 [81]
+  CRUSH rule 0 x 633 [65]
+  CRUSH rule 0 x 634 [87]
+  CRUSH rule 0 x 635 [40]
+  CRUSH rule 0 x 636 [23]
+  CRUSH rule 0 x 637 [102]
+  CRUSH rule 0 x 638 [43]
+  CRUSH rule 0 x 639 [31]
+  CRUSH rule 0 x 640 [113]
+  CRUSH rule 0 x 641 [45]
+  CRUSH rule 0 x 642 [47]
+  CRUSH rule 0 x 643 [64]
+  CRUSH rule 0 x 644 [31]
+  CRUSH rule 0 x 645 [76]
+  CRUSH rule 0 x 646 [37]
+  CRUSH rule 0 x 647 [58]
+  CRUSH rule 0 x 648 [31]
+  CRUSH rule 0 x 649 [88]
+  CRUSH rule 0 x 650 [116]
+  CRUSH rule 0 x 651 [97]
+  CRUSH rule 0 x 652 [57]
+  CRUSH rule 0 x 653 [8]
+  CRUSH rule 0 x 654 [49]
+  CRUSH rule 0 x 655 [89]
+  CRUSH rule 0 x 656 [0]
+  CRUSH rule 0 x 657 [47]
+  CRUSH rule 0 x 658 [75]
+  CRUSH rule 0 x 659 [26]
+  CRUSH rule 0 x 660 [65]
+  CRUSH rule 0 x 661 [91]
+  CRUSH rule 0 x 662 [111]
+  CRUSH rule 0 x 663 [88]
+  CRUSH rule 0 x 664 [59]
+  CRUSH rule 0 x 665 [78]
+  CRUSH rule 0 x 666 [112]
+  CRUSH rule 0 x 667 [97]
+  CRUSH rule 0 x 668 [97]
+  CRUSH rule 0 x 669 [85]
+  CRUSH rule 0 x 670 [41]
+  CRUSH rule 0 x 671 [116]
+  CRUSH rule 0 x 672 [44]
+  CRUSH rule 0 x 673 [83]
+  CRUSH rule 0 x 674 [36]
+  CRUSH rule 0 x 675 [88]
+  CRUSH rule 0 x 676 [62]
+  CRUSH rule 0 x 677 [88]
+  CRUSH rule 0 x 678 [98]
+  CRUSH rule 0 x 679 [33]
+  CRUSH rule 0 x 680 [55]
+  CRUSH rule 0 x 681 [115]
+  CRUSH rule 0 x 682 [27]
+  CRUSH rule 0 x 683 [57]
+  CRUSH rule 0 x 684 [22]
+  CRUSH rule 0 x 685 [106]
+  CRUSH rule 0 x 686 [86]
+  CRUSH rule 0 x 687 [32]
+  CRUSH rule 0 x 688 [80]
+  CRUSH rule 0 x 689 [6]
+  CRUSH rule 0 x 690 [43]
+  CRUSH rule 0 x 691 [34]
+  CRUSH rule 0 x 692 [40]
+  CRUSH rule 0 x 693 [29]
+  CRUSH rule 0 x 694 [6]
+  CRUSH rule 0 x 695 [19]
+  CRUSH rule 0 x 696 [36]
+  CRUSH rule 0 x 697 [96]
+  CRUSH rule 0 x 698 [61]
+  CRUSH rule 0 x 699 [47]
+  CRUSH rule 0 x 700 [99]
+  CRUSH rule 0 x 701 [42]
+  CRUSH rule 0 x 702 [0]
+  CRUSH rule 0 x 703 [92]
+  CRUSH rule 0 x 704 [10]
+  CRUSH rule 0 x 705 [105]
+  CRUSH rule 0 x 706 [74]
+  CRUSH rule 0 x 707 [0]
+  CRUSH rule 0 x 708 [84]
+  CRUSH rule 0 x 709 [114]
+  CRUSH rule 0 x 710 [94]
+  CRUSH rule 0 x 711 [68]
+  CRUSH rule 0 x 712 [34]
+  CRUSH rule 0 x 713 [29]
+  CRUSH rule 0 x 714 [81]
+  CRUSH rule 0 x 715 [71]
+  CRUSH rule 0 x 716 [40]
+  CRUSH rule 0 x 717 [61]
+  CRUSH rule 0 x 718 [40]
+  CRUSH rule 0 x 719 [59]
+  CRUSH rule 0 x 720 [69]
+  CRUSH rule 0 x 721 [62]
+  CRUSH rule 0 x 722 [115]
+  CRUSH rule 0 x 723 [117]
+  CRUSH rule 0 x 724 [45]
+  CRUSH rule 0 x 725 [53]
+  CRUSH rule 0 x 726 [84]
+  CRUSH rule 0 x 727 [109]
+  CRUSH rule 0 x 728 [76]
+  CRUSH rule 0 x 729 [108]
+  CRUSH rule 0 x 730 [28]
+  CRUSH rule 0 x 731 [78]
+  CRUSH rule 0 x 732 [55]
+  CRUSH rule 0 x 733 [84]
+  CRUSH rule 0 x 734 [27]
+  CRUSH rule 0 x 735 [83]
+  CRUSH rule 0 x 736 [70]
+  CRUSH rule 0 x 737 [117]
+  CRUSH rule 0 x 738 [118]
+  CRUSH rule 0 x 739 [87]
+  CRUSH rule 0 x 740 [29]
+  CRUSH rule 0 x 741 [96]
+  CRUSH rule 0 x 742 [106]
+  CRUSH rule 0 x 743 [105]
+  CRUSH rule 0 x 744 [23]
+  CRUSH rule 0 x 745 [28]
+  CRUSH rule 0 x 746 [56]
+  CRUSH rule 0 x 747 [65]
+  CRUSH rule 0 x 748 [48]
+  CRUSH rule 0 x 749 [102]
+  CRUSH rule 0 x 750 [50]
+  CRUSH rule 0 x 751 [36]
+  CRUSH rule 0 x 752 [69]
+  CRUSH rule 0 x 753 [116]
+  CRUSH rule 0 x 754 [9]
+  CRUSH rule 0 x 755 [98]
+  CRUSH rule 0 x 756 [113]
+  CRUSH rule 0 x 757 [47]
+  CRUSH rule 0 x 758 [57]
+  CRUSH rule 0 x 759 [74]
+  CRUSH rule 0 x 760 [53]
+  CRUSH rule 0 x 761 [78]
+  CRUSH rule 0 x 762 [87]
+  CRUSH rule 0 x 763 [13]
+  CRUSH rule 0 x 764 [106]
+  CRUSH rule 0 x 765 [109]
+  CRUSH rule 0 x 766 [76]
+  CRUSH rule 0 x 767 [41]
+  CRUSH rule 0 x 768 [13]
+  CRUSH rule 0 x 769 [91]
+  CRUSH rule 0 x 770 [105]
+  CRUSH rule 0 x 771 [10]
+  CRUSH rule 0 x 772 [118]
+  CRUSH rule 0 x 773 [116]
+  CRUSH rule 0 x 774 [100]
+  CRUSH rule 0 x 775 [102]
+  CRUSH rule 0 x 776 [69]
+  CRUSH rule 0 x 777 [76]
+  CRUSH rule 0 x 778 [38]
+  CRUSH rule 0 x 779 [46]
+  CRUSH rule 0 x 780 [63]
+  CRUSH rule 0 x 781 [105]
+  CRUSH rule 0 x 782 [117]
+  CRUSH rule 0 x 783 [60]
+  CRUSH rule 0 x 784 [82]
+  CRUSH rule 0 x 785 [27]
+  CRUSH rule 0 x 786 [41]
+  CRUSH rule 0 x 787 [13]
+  CRUSH rule 0 x 788 [4]
+  CRUSH rule 0 x 789 [50]
+  CRUSH rule 0 x 790 [58]
+  CRUSH rule 0 x 791 [96]
+  CRUSH rule 0 x 792 [80]
+  CRUSH rule 0 x 793 [6]
+  CRUSH rule 0 x 794 [14]
+  CRUSH rule 0 x 795 [51]
+  CRUSH rule 0 x 796 [114]
+  CRUSH rule 0 x 797 [79]
+  CRUSH rule 0 x 798 [42]
+  CRUSH rule 0 x 799 [48]
+  CRUSH rule 0 x 800 [91]
+  CRUSH rule 0 x 801 [2]
+  CRUSH rule 0 x 802 [116]
+  CRUSH rule 0 x 803 [37]
+  CRUSH rule 0 x 804 [33]
+  CRUSH rule 0 x 805 [96]
+  CRUSH rule 0 x 806 [67]
+  CRUSH rule 0 x 807 [47]
+  CRUSH rule 0 x 808 [76]
+  CRUSH rule 0 x 809 [27]
+  CRUSH rule 0 x 810 [119]
+  CRUSH rule 0 x 811 [75]
+  CRUSH rule 0 x 812 [25]
+  CRUSH rule 0 x 813 [64]
+  CRUSH rule 0 x 814 [110]
+  CRUSH rule 0 x 815 [84]
+  CRUSH rule 0 x 816 [25]
+  CRUSH rule 0 x 817 [40]
+  CRUSH rule 0 x 818 [34]
+  CRUSH rule 0 x 819 [88]
+  CRUSH rule 0 x 820 [104]
+  CRUSH rule 0 x 821 [58]
+  CRUSH rule 0 x 822 [29]
+  CRUSH rule 0 x 823 [100]
+  CRUSH rule 0 x 824 [102]
+  CRUSH rule 0 x 825 [47]
+  CRUSH rule 0 x 826 [45]
+  CRUSH rule 0 x 827 [101]
+  CRUSH rule 0 x 828 [60]
+  CRUSH rule 0 x 829 [45]
+  CRUSH rule 0 x 830 [51]
+  CRUSH rule 0 x 831 [6]
+  CRUSH rule 0 x 832 [57]
+  CRUSH rule 0 x 833 [34]
+  CRUSH rule 0 x 834 [90]
+  CRUSH rule 0 x 835 [14]
+  CRUSH rule 0 x 836 [38]
+  CRUSH rule 0 x 837 [51]
+  CRUSH rule 0 x 838 [6]
+  CRUSH rule 0 x 839 [106]
+  CRUSH rule 0 x 840 [33]
+  CRUSH rule 0 x 841 [110]
+  CRUSH rule 0 x 842 [66]
+  CRUSH rule 0 x 843 [11]
+  CRUSH rule 0 x 844 [74]
+  CRUSH rule 0 x 845 [74]
+  CRUSH rule 0 x 846 [98]
+  CRUSH rule 0 x 847 [10]
+  CRUSH rule 0 x 848 [89]
+  CRUSH rule 0 x 849 [42]
+  CRUSH rule 0 x 850 [40]
+  CRUSH rule 0 x 851 [65]
+  CRUSH rule 0 x 852 [31]
+  CRUSH rule 0 x 853 [49]
+  CRUSH rule 0 x 854 [90]
+  CRUSH rule 0 x 855 [2]
+  CRUSH rule 0 x 856 [40]
+  CRUSH rule 0 x 857 [15]
+  CRUSH rule 0 x 858 [10]
+  CRUSH rule 0 x 859 [29]
+  CRUSH rule 0 x 860 [114]
+  CRUSH rule 0 x 861 [22]
+  CRUSH rule 0 x 862 [22]
+  CRUSH rule 0 x 863 [79]
+  CRUSH rule 0 x 864 [68]
+  CRUSH rule 0 x 865 [25]
+  CRUSH rule 0 x 866 [18]
+  CRUSH rule 0 x 867 [3]
+  CRUSH rule 0 x 868 [81]
+  CRUSH rule 0 x 869 [22]
+  CRUSH rule 0 x 870 [73]
+  CRUSH rule 0 x 871 [25]
+  CRUSH rule 0 x 872 [39]
+  CRUSH rule 0 x 873 [92]
+  CRUSH rule 0 x 874 [21]
+  CRUSH rule 0 x 875 [27]
+  CRUSH rule 0 x 876 [98]
+  CRUSH rule 0 x 877 [73]
+  CRUSH rule 0 x 878 [64]
+  CRUSH rule 0 x 879 [29]
+  CRUSH rule 0 x 880 [56]
+  CRUSH rule 0 x 881 [109]
+  CRUSH rule 0 x 882 [60]
+  CRUSH rule 0 x 883 [93]
+  CRUSH rule 0 x 884 [67]
+  CRUSH rule 0 x 885 [31]
+  CRUSH rule 0 x 886 [2]
+  CRUSH rule 0 x 887 [5]
+  CRUSH rule 0 x 888 [16]
+  CRUSH rule 0 x 889 [3]
+  CRUSH rule 0 x 890 [48]
+  CRUSH rule 0 x 891 [86]
+  CRUSH rule 0 x 892 [64]
+  CRUSH rule 0 x 893 [118]
+  CRUSH rule 0 x 894 [16]
+  CRUSH rule 0 x 895 [40]
+  CRUSH rule 0 x 896 [97]
+  CRUSH rule 0 x 897 [60]
+  CRUSH rule 0 x 898 [10]
+  CRUSH rule 0 x 899 [75]
+  CRUSH rule 0 x 900 [102]
+  CRUSH rule 0 x 901 [66]
+  CRUSH rule 0 x 902 [102]
+  CRUSH rule 0 x 903 [5]
+  CRUSH rule 0 x 904 [50]
+  CRUSH rule 0 x 905 [19]
+  CRUSH rule 0 x 906 [75]
+  CRUSH rule 0 x 907 [47]
+  CRUSH rule 0 x 908 [96]
+  CRUSH rule 0 x 909 [94]
+  CRUSH rule 0 x 910 [88]
+  CRUSH rule 0 x 911 [102]
+  CRUSH rule 0 x 912 [91]
+  CRUSH rule 0 x 913 [29]
+  CRUSH rule 0 x 914 [84]
+  CRUSH rule 0 x 915 [70]
+  CRUSH rule 0 x 916 [32]
+  CRUSH rule 0 x 917 [43]
+  CRUSH rule 0 x 918 [91]
+  CRUSH rule 0 x 919 [13]
+  CRUSH rule 0 x 920 [18]
+  CRUSH rule 0 x 921 [104]
+  CRUSH rule 0 x 922 [33]
+  CRUSH rule 0 x 923 [28]
+  CRUSH rule 0 x 924 [69]
+  CRUSH rule 0 x 925 [71]
+  CRUSH rule 0 x 926 [64]
+  CRUSH rule 0 x 927 [99]
+  CRUSH rule 0 x 928 [13]
+  CRUSH rule 0 x 929 [117]
+  CRUSH rule 0 x 930 [31]
+  CRUSH rule 0 x 931 [46]
+  CRUSH rule 0 x 932 [60]
+  CRUSH rule 0 x 933 [88]
+  CRUSH rule 0 x 934 [68]
+  CRUSH rule 0 x 935 [31]
+  CRUSH rule 0 x 936 [104]
+  CRUSH rule 0 x 937 [110]
+  CRUSH rule 0 x 938 [29]
+  CRUSH rule 0 x 939 [77]
+  CRUSH rule 0 x 940 [76]
+  CRUSH rule 0 x 941 [66]
+  CRUSH rule 0 x 942 [83]
+  CRUSH rule 0 x 943 [4]
+  CRUSH rule 0 x 944 [113]
+  CRUSH rule 0 x 945 [17]
+  CRUSH rule 0 x 946 [37]
+  CRUSH rule 0 x 947 [107]
+  CRUSH rule 0 x 948 [55]
+  CRUSH rule 0 x 949 [45]
+  CRUSH rule 0 x 950 [96]
+  CRUSH rule 0 x 951 [40]
+  CRUSH rule 0 x 952 [93]
+  CRUSH rule 0 x 953 [55]
+  CRUSH rule 0 x 954 [84]
+  CRUSH rule 0 x 955 [31]
+  CRUSH rule 0 x 956 [72]
+  CRUSH rule 0 x 957 [3]
+  CRUSH rule 0 x 958 [8]
+  CRUSH rule 0 x 959 [42]
+  CRUSH rule 0 x 960 [113]
+  CRUSH rule 0 x 961 [116]
+  CRUSH rule 0 x 962 [13]
+  CRUSH rule 0 x 963 [0]
+  CRUSH rule 0 x 964 [59]
+  CRUSH rule 0 x 965 [47]
+  CRUSH rule 0 x 966 [88]
+  CRUSH rule 0 x 967 [71]
+  CRUSH rule 0 x 968 [73]
+  CRUSH rule 0 x 969 [53]
+  CRUSH rule 0 x 970 [3]
+  CRUSH rule 0 x 971 [87]
+  CRUSH rule 0 x 972 [3]
+  CRUSH rule 0 x 973 [113]
+  CRUSH rule 0 x 974 [114]
+  CRUSH rule 0 x 975 [40]
+  CRUSH rule 0 x 976 [81]
+  CRUSH rule 0 x 977 [95]
+  CRUSH rule 0 x 978 [35]
+  CRUSH rule 0 x 979 [98]
+  CRUSH rule 0 x 980 [52]
+  CRUSH rule 0 x 981 [89]
+  CRUSH rule 0 x 982 [1]
+  CRUSH rule 0 x 983 [34]
+  CRUSH rule 0 x 984 [78]
+  CRUSH rule 0 x 985 [99]
+  CRUSH rule 0 x 986 [4]
+  CRUSH rule 0 x 987 [78]
+  CRUSH rule 0 x 988 [79]
+  CRUSH rule 0 x 989 [87]
+  CRUSH rule 0 x 990 [47]
+  CRUSH rule 0 x 991 [61]
+  CRUSH rule 0 x 992 [83]
+  CRUSH rule 0 x 993 [75]
+  CRUSH rule 0 x 994 [74]
+  CRUSH rule 0 x 995 [100]
+  CRUSH rule 0 x 996 [41]
+  CRUSH rule 0 x 997 [89]
+  CRUSH rule 0 x 998 [92]
+  CRUSH rule 0 x 999 [101]
+  CRUSH rule 0 x 1000 [9]
+  CRUSH rule 0 x 1001 [49]
+  CRUSH rule 0 x 1002 [99]
+  CRUSH rule 0 x 1003 [43]
+  CRUSH rule 0 x 1004 [89]
+  CRUSH rule 0 x 1005 [105]
+  CRUSH rule 0 x 1006 [45]
+  CRUSH rule 0 x 1007 [19]
+  CRUSH rule 0 x 1008 [31]
+  CRUSH rule 0 x 1009 [19]
+  CRUSH rule 0 x 1010 [42]
+  CRUSH rule 0 x 1011 [25]
+  CRUSH rule 0 x 1012 [68]
+  CRUSH rule 0 x 1013 [5]
+  CRUSH rule 0 x 1014 [33]
+  CRUSH rule 0 x 1015 [106]
+  CRUSH rule 0 x 1016 [88]
+  CRUSH rule 0 x 1017 [0]
+  CRUSH rule 0 x 1018 [63]
+  CRUSH rule 0 x 1019 [104]
+  CRUSH rule 0 x 1020 [96]
+  CRUSH rule 0 x 1021 [117]
+  CRUSH rule 0 x 1022 [73]
+  CRUSH rule 0 x 1023 [0]
+  rule 0 (data) num_rep 1 result size == 1:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [101,114]
+  CRUSH rule 0 x 1 [80,79]
+  CRUSH rule 0 x 2 [91,96]
+  CRUSH rule 0 x 3 [51,4]
+  CRUSH rule 0 x 4 [50,89]
+  CRUSH rule 0 x 5 [89,94]
+  CRUSH rule 0 x 6 [91,76]
+  CRUSH rule 0 x 7 [104,25]
+  CRUSH rule 0 x 8 [78,57]
+  CRUSH rule 0 x 9 [101,102]
+  CRUSH rule 0 x 10 [61,58]
+  CRUSH rule 0 x 11 [13,31]
+  CRUSH rule 0 x 12 [83,46]
+  CRUSH rule 0 x 13 [108,85]
+  CRUSH rule 0 x 14 [105,72]
+  CRUSH rule 0 x 15 [18,7]
+  CRUSH rule 0 x 16 [103,3]
+  CRUSH rule 0 x 17 [85,110]
+  CRUSH rule 0 x 18 [11,65]
+  CRUSH rule 0 x 19 [75,50]
+  CRUSH rule 0 x 20 [79,70]
+  CRUSH rule 0 x 21 [84,49]
+  CRUSH rule 0 x 22 [23,104]
+  CRUSH rule 0 x 23 [118,63]
+  CRUSH rule 0 x 24 [83,38]
+  CRUSH rule 0 x 25 [81,64]
+  CRUSH rule 0 x 26 [38,99]
+  CRUSH rule 0 x 27 [76,107]
+  CRUSH rule 0 x 28 [76,71]
+  CRUSH rule 0 x 29 [24,71]
+  CRUSH rule 0 x 30 [94,87]
+  CRUSH rule 0 x 31 [76,95]
+  CRUSH rule 0 x 32 [72,95]
+  CRUSH rule 0 x 33 [77,86]
+  CRUSH rule 0 x 34 [7,108]
+  CRUSH rule 0 x 35 [22,88]
+  CRUSH rule 0 x 36 [104,65]
+  CRUSH rule 0 x 37 [61,109]
+  CRUSH rule 0 x 38 [72,85]
+  CRUSH rule 0 x 39 [68,103]
+  CRUSH rule 0 x 40 [103,78]
+  CRUSH rule 0 x 41 [85,11]
+  CRUSH rule 0 x 42 [106,33]
+  CRUSH rule 0 x 43 [10,68]
+  CRUSH rule 0 x 44 [101,4]
+  CRUSH rule 0 x 45 [83,15]
+  CRUSH rule 0 x 46 [65,1]
+  CRUSH rule 0 x 47 [106,53]
+  CRUSH rule 0 x 48 [34,33]
+  CRUSH rule 0 x 49 [0,81]
+  CRUSH rule 0 x 50 [42,6]
+  CRUSH rule 0 x 51 [104,75]
+  CRUSH rule 0 x 52 [83,19]
+  CRUSH rule 0 x 53 [32,69]
+  CRUSH rule 0 x 54 [9,79]
+  CRUSH rule 0 x 55 [14,5]
+  CRUSH rule 0 x 56 [21,72]
+  CRUSH rule 0 x 57 [93,84]
+  CRUSH rule 0 x 58 [45,106]
+  CRUSH rule 0 x 59 [80,41]
+  CRUSH rule 0 x 60 [90,57]
+  CRUSH rule 0 x 61 [88,37]
+  CRUSH rule 0 x 62 [81,1]
+  CRUSH rule 0 x 63 [79,113]
+  CRUSH rule 0 x 64 [1,35]
+  CRUSH rule 0 x 65 [32,103]
+  CRUSH rule 0 x 66 [48,99]
+  CRUSH rule 0 x 67 [94,103]
+  CRUSH rule 0 x 68 [102,91]
+  CRUSH rule 0 x 69 [62,77]
+  CRUSH rule 0 x 70 [84,105]
+  CRUSH rule 0 x 71 [9,33]
+  CRUSH rule 0 x 72 [97,42]
+  CRUSH rule 0 x 73 [64,83]
+  CRUSH rule 0 x 74 [29,50]
+  CRUSH rule 0 x 75 [29,28]
+  CRUSH rule 0 x 76 [55,0]
+  CRUSH rule 0 x 77 [107,21]
+  CRUSH rule 0 x 78 [11,89]
+  CRUSH rule 0 x 79 [64,51]
+  CRUSH rule 0 x 80 [0,31]
+  CRUSH rule 0 x 81 [71,109]
+  CRUSH rule 0 x 82 [37,21]
+  CRUSH rule 0 x 83 [92,103]
+  CRUSH rule 0 x 84 [49,115]
+  CRUSH rule 0 x 85 [54,101]
+  CRUSH rule 0 x 86 [37,7]
+  CRUSH rule 0 x 87 [116,4]
+  CRUSH rule 0 x 88 [38,27]
+  CRUSH rule 0 x 89 [76,77]
+  CRUSH rule 0 x 90 [14,50]
+  CRUSH rule 0 x 91 [68,19]
+  CRUSH rule 0 x 92 [86,9]
+  CRUSH rule 0 x 93 [44,65]
+  CRUSH rule 0 x 94 [61,102]
+  CRUSH rule 0 x 95 [93,86]
+  CRUSH rule 0 x 96 [66,87]
+  CRUSH rule 0 x 97 [111,9]
+  CRUSH rule 0 x 98 [93,102]
+  CRUSH rule 0 x 99 [78,3]
+  CRUSH rule 0 x 100 [6,63]
+  CRUSH rule 0 x 101 [84,16]
+  CRUSH rule 0 x 102 [82,105]
+  CRUSH rule 0 x 103 [66,6]
+  CRUSH rule 0 x 104 [14,95]
+  CRUSH rule 0 x 105 [87,1]
+  CRUSH rule 0 x 106 [69,116]
+  CRUSH rule 0 x 107 [1,55]
+  CRUSH rule 0 x 108 [94,53]
+  CRUSH rule 0 x 109 [112,13]
+  CRUSH rule 0 x 110 [54,61]
+  CRUSH rule 0 x 111 [10,78]
+  CRUSH rule 0 x 112 [89,9]
+  CRUSH rule 0 x 113 [69,2]
+  CRUSH rule 0 x 114 [79,110]
+  CRUSH rule 0 x 115 [50,85]
+  CRUSH rule 0 x 116 [96,16]
+  CRUSH rule 0 x 117 [87,42]
+  CRUSH rule 0 x 118 [23,56]
+  CRUSH rule 0 x 119 [104,11]
+  CRUSH rule 0 x 120 [57,5]
+  CRUSH rule 0 x 121 [105,9]
+  CRUSH rule 0 x 122 [45,110]
+  CRUSH rule 0 x 123 [112,35]
+  CRUSH rule 0 x 124 [110,49]
+  CRUSH rule 0 x 125 [66,105]
+  CRUSH rule 0 x 126 [51,28]
+  CRUSH rule 0 x 127 [70,6]
+  CRUSH rule 0 x 128 [90,16]
+  CRUSH rule 0 x 129 [103,110]
+  CRUSH rule 0 x 130 [50,11]
+  CRUSH rule 0 x 131 [23,60]
+  CRUSH rule 0 x 132 [69,70]
+  CRUSH rule 0 x 133 [52,25]
+  CRUSH rule 0 x 134 [78,29]
+  CRUSH rule 0 x 135 [78,3]
+  CRUSH rule 0 x 136 [32,29]
+  CRUSH rule 0 x 137 [11,78]
+  CRUSH rule 0 x 138 [17,94]
+  CRUSH rule 0 x 139 [89,60]
+  CRUSH rule 0 x 140 [39,62]
+  CRUSH rule 0 x 141 [89,98]
+  CRUSH rule 0 x 142 [70,61]
+  CRUSH rule 0 x 143 [51,28]
+  CRUSH rule 0 x 144 [13,81]
+  CRUSH rule 0 x 145 [77,119]
+  CRUSH rule 0 x 146 [8,64]
+  CRUSH rule 0 x 147 [22,37]
+  CRUSH rule 0 x 148 [74,69]
+  CRUSH rule 0 x 149 [76,13]
+  CRUSH rule 0 x 150 [14,47]
+  CRUSH rule 0 x 151 [90,4]
+  CRUSH rule 0 x 152 [49,18]
+  CRUSH rule 0 x 153 [71,44]
+  CRUSH rule 0 x 154 [94,81]
+  CRUSH rule 0 x 155 [75,6]
+  CRUSH rule 0 x 156 [94,85]
+  CRUSH rule 0 x 157 [112,43]
+  CRUSH rule 0 x 158 [26,17]
+  CRUSH rule 0 x 159 [52,29]
+  CRUSH rule 0 x 160 [41,0]
+  CRUSH rule 0 x 161 [19,78]
+  CRUSH rule 0 x 162 [55,2]
+  CRUSH rule 0 x 163 [54,31]
+  CRUSH rule 0 x 164 [45,5]
+  CRUSH rule 0 x 165 [25,72]
+  CRUSH rule 0 x 166 [73,36]
+  CRUSH rule 0 x 167 [89,58]
+  CRUSH rule 0 x 168 [47,40]
+  CRUSH rule 0 x 169 [51,21]
+  CRUSH rule 0 x 170 [68,91]
+  CRUSH rule 0 x 171 [73,90]
+  CRUSH rule 0 x 172 [33,15]
+  CRUSH rule 0 x 173 [102,59]
+  CRUSH rule 0 x 174 [116,25]
+  CRUSH rule 0 x 175 [3,41]
+  CRUSH rule 0 x 176 [94,91]
+  CRUSH rule 0 x 177 [52,85]
+  CRUSH rule 0 x 178 [39,2]
+  CRUSH rule 0 x 179 [72,97]
+  CRUSH rule 0 x 180 [60,7]
+  CRUSH rule 0 x 181 [18,59]
+  CRUSH rule 0 x 182 [22,90]
+  CRUSH rule 0 x 183 [11,74]
+  CRUSH rule 0 x 184 [92,101]
+  CRUSH rule 0 x 185 [97,8]
+  CRUSH rule 0 x 186 [67,116]
+  CRUSH rule 0 x 187 [116,11]
+  CRUSH rule 0 x 188 [69,92]
+  CRUSH rule 0 x 189 [47,84]
+  CRUSH rule 0 x 190 [90,13]
+  CRUSH rule 0 x 191 [49,17]
+  CRUSH rule 0 x 192 [68,93]
+  CRUSH rule 0 x 193 [0,33]
+  CRUSH rule 0 x 194 [17,58]
+  CRUSH rule 0 x 195 [119,41]
+  CRUSH rule 0 x 196 [72,27]
+  CRUSH rule 0 x 197 [106,83]
+  CRUSH rule 0 x 198 [114,95]
+  CRUSH rule 0 x 199 [0,83]
+  CRUSH rule 0 x 200 [35,86]
+  CRUSH rule 0 x 201 [14,29]
+  CRUSH rule 0 x 202 [98,33]
+  CRUSH rule 0 x 203 [36,22]
+  CRUSH rule 0 x 204 [10,98]
+  CRUSH rule 0 x 205 [22,61]
+  CRUSH rule 0 x 206 [49,112]
+  CRUSH rule 0 x 207 [80,39]
+  CRUSH rule 0 x 208 [63,26]
+  CRUSH rule 0 x 209 [85,111]
+  CRUSH rule 0 x 210 [79,18]
+  CRUSH rule 0 x 211 [26,10]
+  CRUSH rule 0 x 212 [28,103]
+  CRUSH rule 0 x 213 [91,0]
+  CRUSH rule 0 x 214 [78,47]
+  CRUSH rule 0 x 215 [61,22]
+  CRUSH rule 0 x 216 [99,3]
+  CRUSH rule 0 x 217 [86,89]
+  CRUSH rule 0 x 218 [93,96]
+  CRUSH rule 0 x 219 [28,59]
+  CRUSH rule 0 x 220 [56,8]
+  CRUSH rule 0 x 221 [0,9]
+  CRUSH rule 0 x 222 [50,63]
+  CRUSH rule 0 x 223 [29,1]
+  CRUSH rule 0 x 224 [52,10]
+  CRUSH rule 0 x 225 [61,11]
+  CRUSH rule 0 x 226 [44,22]
+  CRUSH rule 0 x 227 [42,3]
+  CRUSH rule 0 x 228 [117,49]
+  CRUSH rule 0 x 229 [100,79]
+  CRUSH rule 0 x 230 [41,114]
+  CRUSH rule 0 x 231 [56,95]
+  CRUSH rule 0 x 232 [23,44]
+  CRUSH rule 0 x 233 [88,103]
+  CRUSH rule 0 x 234 [4,101]
+  CRUSH rule 0 x 235 [26,10]
+  CRUSH rule 0 x 236 [32,37]
+  CRUSH rule 0 x 237 [92,3]
+  CRUSH rule 0 x 238 [10,26]
+  CRUSH rule 0 x 239 [15,105]
+  CRUSH rule 0 x 240 [109,85]
+  CRUSH rule 0 x 241 [47,108]
+  CRUSH rule 0 x 242 [24,99]
+  CRUSH rule 0 x 243 [76,8]
+  CRUSH rule 0 x 244 [96,19]
+  CRUSH rule 0 x 245 [27,28]
+  CRUSH rule 0 x 246 [35,82]
+  CRUSH rule 0 x 247 [99,102]
+  CRUSH rule 0 x 248 [8,29]
+  CRUSH rule 0 x 249 [85,1]
+  CRUSH rule 0 x 250 [79,102]
+  CRUSH rule 0 x 251 [28,103]
+  CRUSH rule 0 x 252 [95,22]
+  CRUSH rule 0 x 253 [109,27]
+  CRUSH rule 0 x 254 [80,103]
+  CRUSH rule 0 x 255 [112,22]
+  CRUSH rule 0 x 256 [37,38]
+  CRUSH rule 0 x 257 [69,117]
+  CRUSH rule 0 x 258 [34,55]
+  CRUSH rule 0 x 259 [70,17]
+  CRUSH rule 0 x 260 [98,29]
+  CRUSH rule 0 x 261 [94,83]
+  CRUSH rule 0 x 262 [42,49]
+  CRUSH rule 0 x 263 [65,42]
+  CRUSH rule 0 x 264 [36,17]
+  CRUSH rule 0 x 265 [66,63]
+  CRUSH rule 0 x 266 [75,92]
+  CRUSH rule 0 x 267 [58,35]
+  CRUSH rule 0 x 268 [38,9]
+  CRUSH rule 0 x 269 [43,104]
+  CRUSH rule 0 x 270 [58,37]
+  CRUSH rule 0 x 271 [19,33]
+  CRUSH rule 0 x 272 [73,9]
+  CRUSH rule 0 x 273 [108,29]
+  CRUSH rule 0 x 274 [47,64]
+  CRUSH rule 0 x 275 [92,19]
+  CRUSH rule 0 x 276 [7,79]
+  CRUSH rule 0 x 277 [19,68]
+  CRUSH rule 0 x 278 [116,95]
+  CRUSH rule 0 x 279 [101,3]
+  CRUSH rule 0 x 280 [113,69]
+  CRUSH rule 0 x 281 [14,93]
+  CRUSH rule 0 x 282 [106,7]
+  CRUSH rule 0 x 283 [8,118]
+  CRUSH rule 0 x 284 [10,110]
+  CRUSH rule 0 x 285 [88,63]
+  CRUSH rule 0 x 286 [27,4]
+  CRUSH rule 0 x 287 [84,65]
+  CRUSH rule 0 x 288 [103,8]
+  CRUSH rule 0 x 289 [9,104]
+  CRUSH rule 0 x 290 [115,7]
+  CRUSH rule 0 x 291 [48,45]
+  CRUSH rule 0 x 292 [52,16]
+  CRUSH rule 0 x 293 [27,24]
+  CRUSH rule 0 x 294 [79,36]
+  CRUSH rule 0 x 295 [37,116]
+  CRUSH rule 0 x 296 [56,61]
+  CRUSH rule 0 x 297 [35,40]
+  CRUSH rule 0 x 298 [71,118]
+  CRUSH rule 0 x 299 [79,1]
+  CRUSH rule 0 x 300 [67,5]
+  CRUSH rule 0 x 301 [51,110]
+  CRUSH rule 0 x 302 [78,67]
+  CRUSH rule 0 x 303 [19,94]
+  CRUSH rule 0 x 304 [101,66]
+  CRUSH rule 0 x 305 [81,62]
+  CRUSH rule 0 x 306 [0,23]
+  CRUSH rule 0 x 307 [44,15]
+  CRUSH rule 0 x 308 [91,98]
+  CRUSH rule 0 x 309 [15,18]
+  CRUSH rule 0 x 310 [26,89]
+  CRUSH rule 0 x 311 [36,41]
+  CRUSH rule 0 x 312 [33,22]
+  CRUSH rule 0 x 313 [104,16]
+  CRUSH rule 0 x 314 [28,4]
+  CRUSH rule 0 x 315 [16,8]
+  CRUSH rule 0 x 316 [4,1]
+  CRUSH rule 0 x 317 [118,8]
+  CRUSH rule 0 x 318 [32,47]
+  CRUSH rule 0 x 319 [24,83]
+  CRUSH rule 0 x 320 [36,97]
+  CRUSH rule 0 x 321 [26,85]
+  CRUSH rule 0 x 322 [87,42]
+  CRUSH rule 0 x 323 [73,0]
+  CRUSH rule 0 x 324 [64,37]
+  CRUSH rule 0 x 325 [52,16]
+  CRUSH rule 0 x 326 [111,93]
+  CRUSH rule 0 x 327 [62,16]
+  CRUSH rule 0 x 328 [7,42]
+  CRUSH rule 0 x 329 [93,34]
+  CRUSH rule 0 x 330 [24,4]
+  CRUSH rule 0 x 331 [41,21]
+  CRUSH rule 0 x 332 [61,110]
+  CRUSH rule 0 x 333 [16,8]
+  CRUSH rule 0 x 334 [94,35]
+  CRUSH rule 0 x 335 [71,74]
+  CRUSH rule 0 x 336 [16,19]
+  CRUSH rule 0 x 337 [37,11]
+  CRUSH rule 0 x 338 [109,69]
+  CRUSH rule 0 x 339 [13,64]
+  CRUSH rule 0 x 340 [119,15]
+  CRUSH rule 0 x 341 [63,114]
+  CRUSH rule 0 x 342 [92,25]
+  CRUSH rule 0 x 343 [49,26]
+  CRUSH rule 0 x 344 [103,26]
+  CRUSH rule 0 x 345 [56,25]
+  CRUSH rule 0 x 346 [3,79]
+  CRUSH rule 0 x 347 [106,27]
+  CRUSH rule 0 x 348 [10,117]
+  CRUSH rule 0 x 349 [96,37]
+  CRUSH rule 0 x 350 [63,32]
+  CRUSH rule 0 x 351 [60,85]
+  CRUSH rule 0 x 352 [103,84]
+  CRUSH rule 0 x 353 [10,113]
+  CRUSH rule 0 x 354 [55,52]
+  CRUSH rule 0 x 355 [73,68]
+  CRUSH rule 0 x 356 [114,41]
+  CRUSH rule 0 x 357 [70,13]
+  CRUSH rule 0 x 358 [97,13]
+  CRUSH rule 0 x 359 [4,117]
+  CRUSH rule 0 x 360 [106,69]
+  CRUSH rule 0 x 361 [27,46]
+  CRUSH rule 0 x 362 [28,33]
+  CRUSH rule 0 x 363 [45,26]
+  CRUSH rule 0 x 364 [23,50]
+  CRUSH rule 0 x 365 [57,114]
+  CRUSH rule 0 x 366 [14,58]
+  CRUSH rule 0 x 367 [108,65]
+  CRUSH rule 0 x 368 [103,32]
+  CRUSH rule 0 x 369 [11,57]
+  CRUSH rule 0 x 370 [11,89]
+  CRUSH rule 0 x 371 [34,55]
+  CRUSH rule 0 x 372 [58,10]
+  CRUSH rule 0 x 373 [6,42]
+  CRUSH rule 0 x 374 [110,95]
+  CRUSH rule 0 x 375 [19,92]
+  CRUSH rule 0 x 376 [22,86]
+  CRUSH rule 0 x 377 [93,113]
+  CRUSH rule 0 x 378 [67,36]
+  CRUSH rule 0 x 379 [77,115]
+  CRUSH rule 0 x 380 [3,108]
+  CRUSH rule 0 x 381 [55,1]
+  CRUSH rule 0 x 382 [26,51]
+  CRUSH rule 0 x 383 [48,25]
+  CRUSH rule 0 x 384 [15,100]
+  CRUSH rule 0 x 385 [82,4]
+  CRUSH rule 0 x 386 [108,63]
+  CRUSH rule 0 x 387 [70,41]
+  CRUSH rule 0 x 388 [5,67]
+  CRUSH rule 0 x 389 [14,1]
+  CRUSH rule 0 x 390 [68,10]
+  CRUSH rule 0 x 391 [113,14]
+  CRUSH rule 0 x 392 [72,14]
+  CRUSH rule 0 x 393 [115,6]
+  CRUSH rule 0 x 394 [38,21]
+  CRUSH rule 0 x 395 [0,27]
+  CRUSH rule 0 x 396 [59,92]
+  CRUSH rule 0 x 397 [87,1]
+  CRUSH rule 0 x 398 [44,75]
+  CRUSH rule 0 x 399 [9,2]
+  CRUSH rule 0 x 400 [19,63]
+  CRUSH rule 0 x 401 [79,34]
+  CRUSH rule 0 x 402 [107,98]
+  CRUSH rule 0 x 403 [23,82]
+  CRUSH rule 0 x 404 [76,75]
+  CRUSH rule 0 x 405 [10,32]
+  CRUSH rule 0 x 406 [38,16]
+  CRUSH rule 0 x 407 [70,85]
+  CRUSH rule 0 x 408 [55,72]
+  CRUSH rule 0 x 409 [102,15]
+  CRUSH rule 0 x 410 [59,13]
+  CRUSH rule 0 x 411 [34,29]
+  CRUSH rule 0 x 412 [108,99]
+  CRUSH rule 0 x 413 [54,107]
+  CRUSH rule 0 x 414 [70,4]
+  CRUSH rule 0 x 415 [107,36]
+  CRUSH rule 0 x 416 [21,68]
+  CRUSH rule 0 x 417 [8,70]
+  CRUSH rule 0 x 418 [51,46]
+  CRUSH rule 0 x 419 [8,66]
+  CRUSH rule 0 x 420 [109,105]
+  CRUSH rule 0 x 421 [114,17]
+  CRUSH rule 0 x 422 [109,87]
+  CRUSH rule 0 x 423 [59,98]
+  CRUSH rule 0 x 424 [71,5]
+  CRUSH rule 0 x 425 [101,111]
+  CRUSH rule 0 x 426 [47,46]
+  CRUSH rule 0 x 427 [8,115]
+  CRUSH rule 0 x 428 [68,103]
+  CRUSH rule 0 x 429 [76,6]
+  CRUSH rule 0 x 430 [69,86]
+  CRUSH rule 0 x 431 [70,83]
+  CRUSH rule 0 x 432 [46,37]
+  CRUSH rule 0 x 433 [6,101]
+  CRUSH rule 0 x 434 [64,69]
+  CRUSH rule 0 x 435 [16,50]
+  CRUSH rule 0 x 436 [89,102]
+  CRUSH rule 0 x 437 [29,114]
+  CRUSH rule 0 x 438 [105,98]
+  CRUSH rule 0 x 439 [29,119]
+  CRUSH rule 0 x 440 [38,7]
+  CRUSH rule 0 x 441 [112,105]
+  CRUSH rule 0 x 442 [55,108]
+  CRUSH rule 0 x 443 [44,57]
+  CRUSH rule 0 x 444 [72,27]
+  CRUSH rule 0 x 445 [19,5]
+  CRUSH rule 0 x 446 [40,47]
+  CRUSH rule 0 x 447 [13,61]
+  CRUSH rule 0 x 448 [7,68]
+  CRUSH rule 0 x 449 [67,19]
+  CRUSH rule 0 x 450 [117,79]
+  CRUSH rule 0 x 451 [93,108]
+  CRUSH rule 0 x 452 [70,49]
+  CRUSH rule 0 x 453 [82,22]
+  CRUSH rule 0 x 454 [53,18]
+  CRUSH rule 0 x 455 [91,92]
+  CRUSH rule 0 x 456 [101,104]
+  CRUSH rule 0 x 457 [113,51]
+  CRUSH rule 0 x 458 [53,34]
+  CRUSH rule 0 x 459 [25,115]
+  CRUSH rule 0 x 460 [105,9]
+  CRUSH rule 0 x 461 [102,35]
+  CRUSH rule 0 x 462 [98,107]
+  CRUSH rule 0 x 463 [108,105]
+  CRUSH rule 0 x 464 [19,109]
+  CRUSH rule 0 x 465 [29,86]
+  CRUSH rule 0 x 466 [66,7]
+  CRUSH rule 0 x 467 [6,57]
+  CRUSH rule 0 x 468 [97,26]
+  CRUSH rule 0 x 469 [98,75]
+  CRUSH rule 0 x 470 [50,3]
+  CRUSH rule 0 x 471 [40,79]
+  CRUSH rule 0 x 472 [74,79]
+  CRUSH rule 0 x 473 [95,21]
+  CRUSH rule 0 x 474 [51,32]
+  CRUSH rule 0 x 475 [49,110]
+  CRUSH rule 0 x 476 [110,31]
+  CRUSH rule 0 x 477 [25,106]
+  CRUSH rule 0 x 478 [47,46]
+  CRUSH rule 0 x 479 [70,37]
+  CRUSH rule 0 x 480 [62,57]
+  CRUSH rule 0 x 481 [26,19]
+  CRUSH rule 0 x 482 [84,85]
+  CRUSH rule 0 x 483 [15,116]
+  CRUSH rule 0 x 484 [37,36]
+  CRUSH rule 0 x 485 [47,117]
+  CRUSH rule 0 x 486 [92,10]
+  CRUSH rule 0 x 487 [106,51]
+  CRUSH rule 0 x 488 [42,9]
+  CRUSH rule 0 x 489 [76,16]
+  CRUSH rule 0 x 490 [68,17]
+  CRUSH rule 0 x 491 [80,71]
+  CRUSH rule 0 x 492 [21,57]
+  CRUSH rule 0 x 493 [99,78]
+  CRUSH rule 0 x 494 [4,87]
+  CRUSH rule 0 x 495 [40,43]
+  CRUSH rule 0 x 496 [93,38]
+  CRUSH rule 0 x 497 [102,71]
+  CRUSH rule 0 x 498 [68,83]
+  CRUSH rule 0 x 499 [10,26]
+  CRUSH rule 0 x 500 [50,6]
+  CRUSH rule 0 x 501 [60,9]
+  CRUSH rule 0 x 502 [11,64]
+  CRUSH rule 0 x 503 [117,25]
+  CRUSH rule 0 x 504 [90,41]
+  CRUSH rule 0 x 505 [91,100]
+  CRUSH rule 0 x 506 [82,103]
+  CRUSH rule 0 x 507 [81,54]
+  CRUSH rule 0 x 508 [34,87]
+  CRUSH rule 0 x 509 [88,63]
+  CRUSH rule 0 x 510 [11,73]
+  CRUSH rule 0 x 511 [72,27]
+  CRUSH rule 0 x 512 [118,73]
+  CRUSH rule 0 x 513 [22,76]
+  CRUSH rule 0 x 514 [82,11]
+  CRUSH rule 0 x 515 [27,0]
+  CRUSH rule 0 x 516 [66,13]
+  CRUSH rule 0 x 517 [83,60]
+  CRUSH rule 0 x 518 [18,3]
+  CRUSH rule 0 x 519 [67,119]
+  CRUSH rule 0 x 520 [15,88]
+  CRUSH rule 0 x 521 [63,113]
+  CRUSH rule 0 x 522 [56,73]
+  CRUSH rule 0 x 523 [36,35]
+  CRUSH rule 0 x 524 [33,38]
+  CRUSH rule 0 x 525 [3,119]
+  CRUSH rule 0 x 526 [83,50]
+  CRUSH rule 0 x 527 [37,0]
+  CRUSH rule 0 x 528 [108,87]
+  CRUSH rule 0 x 529 [107,60]
+  CRUSH rule 0 x 530 [49,3]
+  CRUSH rule 0 x 531 [27,104]
+  CRUSH rule 0 x 532 [68,14]
+  CRUSH rule 0 x 533 [5,85]
+  CRUSH rule 0 x 534 [97,24]
+  CRUSH rule 0 x 535 [8,75]
+  CRUSH rule 0 x 536 [3,37]
+  CRUSH rule 0 x 537 [116,7]
+  CRUSH rule 0 x 538 [85,56]
+  CRUSH rule 0 x 539 [10,9]
+  CRUSH rule 0 x 540 [100,101]
+  CRUSH rule 0 x 541 [111,77]
+  CRUSH rule 0 x 542 [50,27]
+  CRUSH rule 0 x 543 [45,21]
+  CRUSH rule 0 x 544 [106,65]
+  CRUSH rule 0 x 545 [43,114]
+  CRUSH rule 0 x 546 [108,79]
+  CRUSH rule 0 x 547 [67,50]
+  CRUSH rule 0 x 548 [58,61]
+  CRUSH rule 0 x 549 [60,22]
+  CRUSH rule 0 x 550 [47,68]
+  CRUSH rule 0 x 551 [14,88]
+  CRUSH rule 0 x 552 [70,65]
+  CRUSH rule 0 x 553 [96,105]
+  CRUSH rule 0 x 554 [61,94]
+  CRUSH rule 0 x 555 [76,37]
+  CRUSH rule 0 x 556 [106,89]
+  CRUSH rule 0 x 557 [39,113]
+  CRUSH rule 0 x 558 [70,79]
+  CRUSH rule 0 x 559 [106,69]
+  CRUSH rule 0 x 560 [94,97]
+  CRUSH rule 0 x 561 [27,76]
+  CRUSH rule 0 x 562 [97,62]
+  CRUSH rule 0 x 563 [64,103]
+  CRUSH rule 0 x 564 [96,41]
+  CRUSH rule 0 x 565 [66,71]
+  CRUSH rule 0 x 566 [27,38]
+  CRUSH rule 0 x 567 [88,8]
+  CRUSH rule 0 x 568 [106,17]
+  CRUSH rule 0 x 569 [102,63]
+  CRUSH rule 0 x 570 [98,27]
+  CRUSH rule 0 x 571 [95,98]
+  CRUSH rule 0 x 572 [62,83]
+  CRUSH rule 0 x 573 [51,118]
+  CRUSH rule 0 x 574 [89,78]
+  CRUSH rule 0 x 575 [87,19]
+  CRUSH rule 0 x 576 [112,73]
+  CRUSH rule 0 x 577 [8,84]
+  CRUSH rule 0 x 578 [64,99]
+  CRUSH rule 0 x 579 [78,77]
+  CRUSH rule 0 x 580 [68,95]
+  CRUSH rule 0 x 581 [55,52]
+  CRUSH rule 0 x 582 [15,113]
+  CRUSH rule 0 x 583 [74,105]
+  CRUSH rule 0 x 584 [22,92]
+  CRUSH rule 0 x 585 [35,1]
+  CRUSH rule 0 x 586 [33,1]
+  CRUSH rule 0 x 587 [106,99]
+  CRUSH rule 0 x 588 [0,83]
+  CRUSH rule 0 x 589 [7,95]
+  CRUSH rule 0 x 590 [40,69]
+  CRUSH rule 0 x 591 [42,23]
+  CRUSH rule 0 x 592 [45,22]
+  CRUSH rule 0 x 593 [89,14]
+  CRUSH rule 0 x 594 [27,76]
+  CRUSH rule 0 x 595 [7,10]
+  CRUSH rule 0 x 596 [82,59]
+  CRUSH rule 0 x 597 [72,83]
+  CRUSH rule 0 x 598 [34,19]
+  CRUSH rule 0 x 599 [119,61]
+  CRUSH rule 0 x 600 [24,27]
+  CRUSH rule 0 x 601 [104,15]
+  CRUSH rule 0 x 602 [48,45]
+  CRUSH rule 0 x 603 [24,13]
+  CRUSH rule 0 x 604 [89,0]
+  CRUSH rule 0 x 605 [104,87]
+  CRUSH rule 0 x 606 [49,34]
+  CRUSH rule 0 x 607 [95,40]
+  CRUSH rule 0 x 608 [112,91]
+  CRUSH rule 0 x 609 [61,66]
+  CRUSH rule 0 x 610 [106,16]
+  CRUSH rule 0 x 611 [66,87]
+  CRUSH rule 0 x 612 [103,8]
+  CRUSH rule 0 x 613 [13,91]
+  CRUSH rule 0 x 614 [81,88]
+  CRUSH rule 0 x 615 [61,19]
+  CRUSH rule 0 x 616 [41,15]
+  CRUSH rule 0 x 617 [111,69]
+  CRUSH rule 0 x 618 [26,99]
+  CRUSH rule 0 x 619 [92,27]
+  CRUSH rule 0 x 620 [108,103]
+  CRUSH rule 0 x 621 [106,99]
+  CRUSH rule 0 x 622 [67,48]
+  CRUSH rule 0 x 623 [94,61]
+  CRUSH rule 0 x 624 [115,59]
+  CRUSH rule 0 x 625 [111,27]
+  CRUSH rule 0 x 626 [3,55]
+  CRUSH rule 0 x 627 [19,29]
+  CRUSH rule 0 x 628 [65,88]
+  CRUSH rule 0 x 629 [6,46]
+  CRUSH rule 0 x 630 [22,72]
+  CRUSH rule 0 x 631 [35,22]
+  CRUSH rule 0 x 632 [81,0]
+  CRUSH rule 0 x 633 [65,68]
+  CRUSH rule 0 x 634 [87,50]
+  CRUSH rule 0 x 635 [40,73]
+  CRUSH rule 0 x 636 [23,70]
+  CRUSH rule 0 x 637 [102,45]
+  CRUSH rule 0 x 638 [43,114]
+  CRUSH rule 0 x 639 [31,78]
+  CRUSH rule 0 x 640 [113,73]
+  CRUSH rule 0 x 641 [45,96]
+  CRUSH rule 0 x 642 [47,66]
+  CRUSH rule 0 x 643 [64,47]
+  CRUSH rule 0 x 644 [31,21]
+  CRUSH rule 0 x 645 [76,43]
+  CRUSH rule 0 x 646 [37,54]
+  CRUSH rule 0 x 647 [58,87]
+  CRUSH rule 0 x 648 [31,21]
+  CRUSH rule 0 x 649 [88,45]
+  CRUSH rule 0 x 650 [116,7]
+  CRUSH rule 0 x 651 [97,106]
+  CRUSH rule 0 x 652 [57,112]
+  CRUSH rule 0 x 653 [8,116]
+  CRUSH rule 0 x 654 [49,32]
+  CRUSH rule 0 x 655 [89,62]
+  CRUSH rule 0 x 656 [0,49]
+  CRUSH rule 0 x 657 [47,17]
+  CRUSH rule 0 x 658 [75,82]
+  CRUSH rule 0 x 659 [26,83]
+  CRUSH rule 0 x 660 [65,112]
+  CRUSH rule 0 x 661 [91,48]
+  CRUSH rule 0 x 662 [111,99]
+  CRUSH rule 0 x 663 [88,35]
+  CRUSH rule 0 x 664 [59,78]
+  CRUSH rule 0 x 665 [78,15]
+  CRUSH rule 0 x 666 [112,4]
+  CRUSH rule 0 x 667 [97,46]
+  CRUSH rule 0 x 668 [97,8]
+  CRUSH rule 0 x 669 [85,66]
+  CRUSH rule 0 x 670 [41,48]
+  CRUSH rule 0 x 671 [116,97]
+  CRUSH rule 0 x 672 [44,55]
+  CRUSH rule 0 x 673 [83,50]
+  CRUSH rule 0 x 674 [36,8]
+  CRUSH rule 0 x 675 [88,14]
+  CRUSH rule 0 x 676 [62,8]
+  CRUSH rule 0 x 677 [88,67]
+  CRUSH rule 0 x 678 [98,83]
+  CRUSH rule 0 x 679 [33,78]
+  CRUSH rule 0 x 680 [55,94]
+  CRUSH rule 0 x 681 [115,95]
+  CRUSH rule 0 x 682 [27,94]
+  CRUSH rule 0 x 683 [57,80]
+  CRUSH rule 0 x 684 [22,65]
+  CRUSH rule 0 x 685 [106,55]
+  CRUSH rule 0 x 686 [86,95]
+  CRUSH rule 0 x 687 [32,57]
+  CRUSH rule 0 x 688 [80,22]
+  CRUSH rule 0 x 689 [6,48]
+  CRUSH rule 0 x 690 [43,70]
+  CRUSH rule 0 x 691 [34,105]
+  CRUSH rule 0 x 692 [40,97]
+  CRUSH rule 0 x 693 [29,84]
+  CRUSH rule 0 x 694 [6,84]
+  CRUSH rule 0 x 695 [19,69]
+  CRUSH rule 0 x 696 [36,75]
+  CRUSH rule 0 x 697 [96,99]
+  CRUSH rule 0 x 698 [61,11]
+  CRUSH rule 0 x 699 [47,62]
+  CRUSH rule 0 x 700 [99,82]
+  CRUSH rule 0 x 701 [42,11]
+  CRUSH rule 0 x 702 [0,71]
+  CRUSH rule 0 x 703 [92,3]
+  CRUSH rule 0 x 704 [10,19]
+  CRUSH rule 0 x 705 [105,21]
+  CRUSH rule 0 x 706 [74,105]
+  CRUSH rule 0 x 707 [0,77]
+  CRUSH rule 0 x 708 [84,8]
+  CRUSH rule 0 x 709 [114,97]
+  CRUSH rule 0 x 710 [94,7]
+  CRUSH rule 0 x 711 [68,49]
+  CRUSH rule 0 x 712 [34,75]
+  CRUSH rule 0 x 713 [29,0]
+  CRUSH rule 0 x 714 [81,115]
+  CRUSH rule 0 x 715 [71,84]
+  CRUSH rule 0 x 716 [40,17]
+  CRUSH rule 0 x 717 [61,62]
+  CRUSH rule 0 x 718 [40,85]
+  CRUSH rule 0 x 719 [59,42]
+  CRUSH rule 0 x 720 [69,72]
+  CRUSH rule 0 x 721 [62,21]
+  CRUSH rule 0 x 722 [115,8]
+  CRUSH rule 0 x 723 [117,41]
+  CRUSH rule 0 x 724 [45,102]
+  CRUSH rule 0 x 725 [53,113]
+  CRUSH rule 0 x 726 [84,19]
+  CRUSH rule 0 x 727 [109,14]
+  CRUSH rule 0 x 728 [76,16]
+  CRUSH rule 0 x 729 [108,47]
+  CRUSH rule 0 x 730 [28,47]
+  CRUSH rule 0 x 731 [78,37]
+  CRUSH rule 0 x 732 [55,90]
+  CRUSH rule 0 x 733 [84,3]
+  CRUSH rule 0 x 734 [27,117]
+  CRUSH rule 0 x 735 [83,4]
+  CRUSH rule 0 x 736 [70,67]
+  CRUSH rule 0 x 737 [117,15]
+  CRUSH rule 0 x 738 [118,22]
+  CRUSH rule 0 x 739 [87,38]
+  CRUSH rule 0 x 740 [29,38]
+  CRUSH rule 0 x 741 [96,73]
+  CRUSH rule 0 x 742 [106,83]
+  CRUSH rule 0 x 743 [105,94]
+  CRUSH rule 0 x 744 [23,14]
+  CRUSH rule 0 x 745 [28,6]
+  CRUSH rule 0 x 746 [56,47]
+  CRUSH rule 0 x 747 [65,70]
+  CRUSH rule 0 x 748 [48,89]
+  CRUSH rule 0 x 749 [102,51]
+  CRUSH rule 0 x 750 [50,3]
+  CRUSH rule 0 x 751 [36,25]
+  CRUSH rule 0 x 752 [69,52]
+  CRUSH rule 0 x 753 [116,65]
+  CRUSH rule 0 x 754 [9,57]
+  CRUSH rule 0 x 755 [98,81]
+  CRUSH rule 0 x 756 [113,8]
+  CRUSH rule 0 x 757 [47,66]
+  CRUSH rule 0 x 758 [57,88]
+  CRUSH rule 0 x 759 [74,97]
+  CRUSH rule 0 x 760 [53,90]
+  CRUSH rule 0 x 761 [78,97]
+  CRUSH rule 0 x 762 [87,104]
+  CRUSH rule 0 x 763 [13,45]
+  CRUSH rule 0 x 764 [106,81]
+  CRUSH rule 0 x 765 [109,91]
+  CRUSH rule 0 x 766 [76,97]
+  CRUSH rule 0 x 767 [41,116]
+  CRUSH rule 0 x 768 [13,114]
+  CRUSH rule 0 x 769 [91,96]
+  CRUSH rule 0 x 770 [105,19]
+  CRUSH rule 0 x 771 [10,76]
+  CRUSH rule 0 x 772 [118,17]
+  CRUSH rule 0 x 773 [116,75]
+  CRUSH rule 0 x 774 [100,43]
+  CRUSH rule 0 x 775 [102,43]
+  CRUSH rule 0 x 776 [69,38]
+  CRUSH rule 0 x 777 [76,49]
+  CRUSH rule 0 x 778 [38,13]
+  CRUSH rule 0 x 779 [46,21]
+  CRUSH rule 0 x 780 [63,102]
+  CRUSH rule 0 x 781 [105,92]
+  CRUSH rule 0 x 782 [117,31]
+  CRUSH rule 0 x 783 [60,93]
+  CRUSH rule 0 x 784 [82,81]
+  CRUSH rule 0 x 785 [27,84]
+  CRUSH rule 0 x 786 [41,80]
+  CRUSH rule 0 x 787 [13,54]
+  CRUSH rule 0 x 788 [4,100]
+  CRUSH rule 0 x 789 [50,37]
+  CRUSH rule 0 x 790 [58,16]
+  CRUSH rule 0 x 791 [96,14]
+  CRUSH rule 0 x 792 [80,4]
+  CRUSH rule 0 x 793 [6,71]
+  CRUSH rule 0 x 794 [14,89]
+  CRUSH rule 0 x 795 [51,3]
+  CRUSH rule 0 x 796 [114,77]
+  CRUSH rule 0 x 797 [79,100]
+  CRUSH rule 0 x 798 [42,10]
+  CRUSH rule 0 x 799 [48,11]
+  CRUSH rule 0 x 800 [91,7]
+  CRUSH rule 0 x 801 [2,6]
+  CRUSH rule 0 x 802 [116,89]
+  CRUSH rule 0 x 803 [37,32]
+  CRUSH rule 0 x 804 [33,4]
+  CRUSH rule 0 x 805 [96,22]
+  CRUSH rule 0 x 806 [67,90]
+  CRUSH rule 0 x 807 [47,42]
+  CRUSH rule 0 x 808 [76,79]
+  CRUSH rule 0 x 809 [27,26]
+  CRUSH rule 0 x 810 [119,61]
+  CRUSH rule 0 x 811 [75,72]
+  CRUSH rule 0 x 812 [25,52]
+  CRUSH rule 0 x 813 [64,13]
+  CRUSH rule 0 x 814 [110,53]
+  CRUSH rule 0 x 815 [84,61]
+  CRUSH rule 0 x 816 [25,22]
+  CRUSH rule 0 x 817 [40,73]
+  CRUSH rule 0 x 818 [34,13]
+  CRUSH rule 0 x 819 [88,19]
+  CRUSH rule 0 x 820 [104,49]
+  CRUSH rule 0 x 821 [58,69]
+  CRUSH rule 0 x 822 [29,72]
+  CRUSH rule 0 x 823 [100,103]
+  CRUSH rule 0 x 824 [102,81]
+  CRUSH rule 0 x 825 [47,17]
+  CRUSH rule 0 x 826 [45,34]
+  CRUSH rule 0 x 827 [101,11]
+  CRUSH rule 0 x 828 [60,27]
+  CRUSH rule 0 x 829 [45,90]
+  CRUSH rule 0 x 830 [51,96]
+  CRUSH rule 0 x 831 [6,64]
+  CRUSH rule 0 x 832 [57,78]
+  CRUSH rule 0 x 833 [34,97]
+  CRUSH rule 0 x 834 [90,33]
+  CRUSH rule 0 x 835 [14,46]
+  CRUSH rule 0 x 836 [38,43]
+  CRUSH rule 0 x 837 [51,74]
+  CRUSH rule 0 x 838 [6,32]
+  CRUSH rule 0 x 839 [106,8]
+  CRUSH rule 0 x 840 [33,109]
+  CRUSH rule 0 x 841 [110,15]
+  CRUSH rule 0 x 842 [66,67]
+  CRUSH rule 0 x 843 [11,63]
+  CRUSH rule 0 x 844 [74,13]
+  CRUSH rule 0 x 845 [74,43]
+  CRUSH rule 0 x 846 [98,107]
+  CRUSH rule 0 x 847 [10,3]
+  CRUSH rule 0 x 848 [89,17]
+  CRUSH rule 0 x 849 [42,59]
+  CRUSH rule 0 x 850 [40,73]
+  CRUSH rule 0 x 851 [65,94]
+  CRUSH rule 0 x 852 [31,94]
+  CRUSH rule 0 x 853 [49,11]
+  CRUSH rule 0 x 854 [90,31]
+  CRUSH rule 0 x 855 [2,19]
+  CRUSH rule 0 x 856 [40,22]
+  CRUSH rule 0 x 857 [15,82]
+  CRUSH rule 0 x 858 [10,80]
+  CRUSH rule 0 x 859 [29,48]
+  CRUSH rule 0 x 860 [114,75]
+  CRUSH rule 0 x 861 [22,33]
+  CRUSH rule 0 x 862 [22,25]
+  CRUSH rule 0 x 863 [79,50]
+  CRUSH rule 0 x 864 [68,6]
+  CRUSH rule 0 x 865 [25,92]
+  CRUSH rule 0 x 866 [18,89]
+  CRUSH rule 0 x 867 [3,78]
+  CRUSH rule 0 x 868 [81,98]
+  CRUSH rule 0 x 869 [22,104]
+  CRUSH rule 0 x 870 [73,98]
+  CRUSH rule 0 x 871 [25,54]
+  CRUSH rule 0 x 872 [39,48]
+  CRUSH rule 0 x 873 [92,9]
+  CRUSH rule 0 x 874 [21,43]
+  CRUSH rule 0 x 875 [27,108]
+  CRUSH rule 0 x 876 [98,75]
+  CRUSH rule 0 x 877 [73,5]
+  CRUSH rule 0 x 878 [64,45]
+  CRUSH rule 0 x 879 [29,18]
+  CRUSH rule 0 x 880 [56,91]
+  CRUSH rule 0 x 881 [109,69]
+  CRUSH rule 0 x 882 [60,33]
+  CRUSH rule 0 x 883 [93,96]
+  CRUSH rule 0 x 884 [67,58]
+  CRUSH rule 0 x 885 [31,8]
+  CRUSH rule 0 x 886 [2,107]
+  CRUSH rule 0 x 887 [5,93]
+  CRUSH rule 0 x 888 [16,13]
+  CRUSH rule 0 x 889 [3,76]
+  CRUSH rule 0 x 890 [48,63]
+  CRUSH rule 0 x 891 [86,79]
+  CRUSH rule 0 x 892 [64,9]
+  CRUSH rule 0 x 893 [118,33]
+  CRUSH rule 0 x 894 [16,111]
+  CRUSH rule 0 x 895 [40,107]
+  CRUSH rule 0 x 896 [97,96]
+  CRUSH rule 0 x 897 [60,67]
+  CRUSH rule 0 x 898 [10,2]
+  CRUSH rule 0 x 899 [75,80]
+  CRUSH rule 0 x 900 [102,81]
+  CRUSH rule 0 x 901 [66,87]
+  CRUSH rule 0 x 902 [102,49]
+  CRUSH rule 0 x 903 [5,14]
+  CRUSH rule 0 x 904 [50,16]
+  CRUSH rule 0 x 905 [19,51]
+  CRUSH rule 0 x 906 [75,119]
+  CRUSH rule 0 x 907 [47,5]
+  CRUSH rule 0 x 908 [96,9]
+  CRUSH rule 0 x 909 [94,75]
+  CRUSH rule 0 x 910 [88,63]
+  CRUSH rule 0 x 911 [102,23]
+  CRUSH rule 0 x 912 [91,60]
+  CRUSH rule 0 x 913 [29,17]
+  CRUSH rule 0 x 914 [84,29]
+  CRUSH rule 0 x 915 [70,22]
+  CRUSH rule 0 x 916 [32,9]
+  CRUSH rule 0 x 917 [43,26]
+  CRUSH rule 0 x 918 [91,98]
+  CRUSH rule 0 x 919 [13,69]
+  CRUSH rule 0 x 920 [18,87]
+  CRUSH rule 0 x 921 [104,33]
+  CRUSH rule 0 x 922 [33,19]
+  CRUSH rule 0 x 923 [28,8]
+  CRUSH rule 0 x 924 [69,88]
+  CRUSH rule 0 x 925 [71,32]
+  CRUSH rule 0 x 926 [64,69]
+  CRUSH rule 0 x 927 [99,106]
+  CRUSH rule 0 x 928 [13,113]
+  CRUSH rule 0 x 929 [117,61]
+  CRUSH rule 0 x 930 [31,82]
+  CRUSH rule 0 x 931 [46,79]
+  CRUSH rule 0 x 932 [60,13]
+  CRUSH rule 0 x 933 [88,31]
+  CRUSH rule 0 x 934 [68,4]
+  CRUSH rule 0 x 935 [31,18]
+  CRUSH rule 0 x 936 [104,57]
+  CRUSH rule 0 x 937 [110,22]
+  CRUSH rule 0 x 938 [29,106]
+  CRUSH rule 0 x 939 [77,13]
+  CRUSH rule 0 x 940 [76,33]
+  CRUSH rule 0 x 941 [66,37]
+  CRUSH rule 0 x 942 [83,94]
+  CRUSH rule 0 x 943 [4,74]
+  CRUSH rule 0 x 944 [113,53]
+  CRUSH rule 0 x 945 [17,52]
+  CRUSH rule 0 x 946 [37,111]
+  CRUSH rule 0 x 947 [107,74]
+  CRUSH rule 0 x 948 [55,98]
+  CRUSH rule 0 x 949 [45,72]
+  CRUSH rule 0 x 950 [96,23]
+  CRUSH rule 0 x 951 [40,93]
+  CRUSH rule 0 x 952 [93,46]
+  CRUSH rule 0 x 953 [55,92]
+  CRUSH rule 0 x 954 [84,57]
+  CRUSH rule 0 x 955 [31,117]
+  CRUSH rule 0 x 956 [72,11]
+  CRUSH rule 0 x 957 [3,74]
+  CRUSH rule 0 x 958 [8,106]
+  CRUSH rule 0 x 959 [42,59]
+  CRUSH rule 0 x 960 [113,107]
+  CRUSH rule 0 x 961 [116,8]
+  CRUSH rule 0 x 962 [13,62]
+  CRUSH rule 0 x 963 [0,99]
+  CRUSH rule 0 x 964 [59,21]
+  CRUSH rule 0 x 965 [47,115]
+  CRUSH rule 0 x 966 [88,63]
+  CRUSH rule 0 x 967 [71,108]
+  CRUSH rule 0 x 968 [73,7]
+  CRUSH rule 0 x 969 [53,6]
+  CRUSH rule 0 x 970 [3,40]
+  CRUSH rule 0 x 971 [87,38]
+  CRUSH rule 0 x 972 [3,37]
+  CRUSH rule 0 x 973 [113,27]
+  CRUSH rule 0 x 974 [114,23]
+  CRUSH rule 0 x 975 [40,59]
+  CRUSH rule 0 x 976 [81,38]
+  CRUSH rule 0 x 977 [95,102]
+  CRUSH rule 0 x 978 [35,56]
+  CRUSH rule 0 x 979 [98,6]
+  CRUSH rule 0 x 980 [52,69]
+  CRUSH rule 0 x 981 [89,117]
+  CRUSH rule 0 x 982 [1,47]
+  CRUSH rule 0 x 983 [34,61]
+  CRUSH rule 0 x 984 [78,25]
+  CRUSH rule 0 x 985 [99,52]
+  CRUSH rule 0 x 986 [4,59]
+  CRUSH rule 0 x 987 [78,21]
+  CRUSH rule 0 x 988 [79,2]
+  CRUSH rule 0 x 989 [87,17]
+  CRUSH rule 0 x 990 [47,118]
+  CRUSH rule 0 x 991 [61,18]
+  CRUSH rule 0 x 992 [83,66]
+  CRUSH rule 0 x 993 [75,62]
+  CRUSH rule 0 x 994 [74,57]
+  CRUSH rule 0 x 995 [100,97]
+  CRUSH rule 0 x 996 [41,6]
+  CRUSH rule 0 x 997 [89,76]
+  CRUSH rule 0 x 998 [92,47]
+  CRUSH rule 0 x 999 [101,11]
+  CRUSH rule 0 x 1000 [9,119]
+  CRUSH rule 0 x 1001 [49,32]
+  CRUSH rule 0 x 1002 [99,113]
+  CRUSH rule 0 x 1003 [43,18]
+  CRUSH rule 0 x 1004 [89,54]
+  CRUSH rule 0 x 1005 [105,84]
+  CRUSH rule 0 x 1006 [45,111]
+  CRUSH rule 0 x 1007 [19,57]
+  CRUSH rule 0 x 1008 [31,24]
+  CRUSH rule 0 x 1009 [19,111]
+  CRUSH rule 0 x 1010 [42,89]
+  CRUSH rule 0 x 1011 [25,114]
+  CRUSH rule 0 x 1012 [68,71]
+  CRUSH rule 0 x 1013 [5,65]
+  CRUSH rule 0 x 1014 [33,4]
+  CRUSH rule 0 x 1015 [106,45]
+  CRUSH rule 0 x 1016 [88,39]
+  CRUSH rule 0 x 1017 [0,89]
+  CRUSH rule 0 x 1018 [63,5]
+  CRUSH rule 0 x 1019 [104,97]
+  CRUSH rule 0 x 1020 [96,9]
+  CRUSH rule 0 x 1021 [117,6]
+  CRUSH rule 0 x 1022 [73,21]
+  CRUSH rule 0 x 1023 [0,16]
+  rule 0 (data) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 3 result size == 2:\t2/1024 (esc)
+  rule 0 (data) num_rep 3 result size == 3:\t1022/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 4 result size == 2:\t2/1024 (esc)
+  rule 0 (data) num_rep 4 result size == 3:\t1022/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 5 result size == 2:\t2/1024 (esc)
+  rule 0 (data) num_rep 5 result size == 3:\t1022/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76,9]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 6 result size == 2:\t1/1024 (esc)
+  rule 0 (data) num_rep 6 result size == 3:\t1023/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76,9]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 7 result size == 2:\t1/1024 (esc)
+  rule 0 (data) num_rep 7 result size == 3:\t1023/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76,9]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 8 result size == 2:\t1/1024 (esc)
+  rule 0 (data) num_rep 8 result size == 3:\t1023/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76,9]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87,19]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 9 result size == 3:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [101,114,14]
+  CRUSH rule 0 x 1 [80,79,17]
+  CRUSH rule 0 x 2 [91,96,4]
+  CRUSH rule 0 x 3 [51,4,109]
+  CRUSH rule 0 x 4 [50,89,8]
+  CRUSH rule 0 x 5 [89,94,11]
+  CRUSH rule 0 x 6 [91,76,7]
+  CRUSH rule 0 x 7 [104,25,17]
+  CRUSH rule 0 x 8 [78,57,8]
+  CRUSH rule 0 x 9 [101,102,4]
+  CRUSH rule 0 x 10 [61,58,22]
+  CRUSH rule 0 x 11 [13,31,26]
+  CRUSH rule 0 x 12 [83,46,13]
+  CRUSH rule 0 x 13 [108,85,17]
+  CRUSH rule 0 x 14 [105,72,13]
+  CRUSH rule 0 x 15 [18,7,29]
+  CRUSH rule 0 x 16 [103,3,50]
+  CRUSH rule 0 x 17 [85,110,9]
+  CRUSH rule 0 x 18 [11,65,52]
+  CRUSH rule 0 x 19 [75,50,22]
+  CRUSH rule 0 x 20 [79,70,15]
+  CRUSH rule 0 x 21 [84,49,9]
+  CRUSH rule 0 x 22 [23,104,21]
+  CRUSH rule 0 x 23 [118,63,6]
+  CRUSH rule 0 x 24 [83,38,8]
+  CRUSH rule 0 x 25 [81,64,3]
+  CRUSH rule 0 x 26 [38,99,3]
+  CRUSH rule 0 x 27 [76,107,17]
+  CRUSH rule 0 x 28 [76,71,15]
+  CRUSH rule 0 x 29 [24,71,19]
+  CRUSH rule 0 x 30 [94,87,19]
+  CRUSH rule 0 x 31 [76,95,22]
+  CRUSH rule 0 x 32 [72,95,19]
+  CRUSH rule 0 x 33 [77,86,3]
+  CRUSH rule 0 x 34 [7,108,83]
+  CRUSH rule 0 x 35 [22,88,83]
+  CRUSH rule 0 x 36 [104,65,15]
+  CRUSH rule 0 x 37 [61,109,11]
+  CRUSH rule 0 x 38 [72,85,3]
+  CRUSH rule 0 x 39 [68,103,8]
+  CRUSH rule 0 x 40 [103,78,3]
+  CRUSH rule 0 x 41 [85,11,110]
+  CRUSH rule 0 x 42 [106,33,9]
+  CRUSH rule 0 x 43 [10,68,11]
+  CRUSH rule 0 x 44 [101,4,109]
+  CRUSH rule 0 x 45 [83,15,24]
+  CRUSH rule 0 x 46 [65,1,7]
+  CRUSH rule 0 x 47 [106,53,7]
+  CRUSH rule 0 x 48 [34,33,14]
+  CRUSH rule 0 x 49 [0,81,4]
+  CRUSH rule 0 x 50 [42,6,101]
+  CRUSH rule 0 x 51 [104,75,9]
+  CRUSH rule 0 x 52 [83,19,58]
+  CRUSH rule 0 x 53 [32,69,7]
+  CRUSH rule 0 x 54 [9,79,104]
+  CRUSH rule 0 x 55 [14,5,37]
+  CRUSH rule 0 x 56 [21,72,63]
+  CRUSH rule 0 x 57 [93,84,3]
+  CRUSH rule 0 x 58 [45,106,13]
+  CRUSH rule 0 x 59 [80,41,15]
+  CRUSH rule 0 x 60 [90,57,15]
+  CRUSH rule 0 x 61 [88,37,3]
+  CRUSH rule 0 x 62 [81,1,9]
+  CRUSH rule 0 x 63 [79,113,9]
+  CRUSH rule 0 x 64 [1,35,9]
+  CRUSH rule 0 x 65 [32,103,15]
+  CRUSH rule 0 x 66 [48,99,9]
+  CRUSH rule 0 x 67 [94,103,15]
+  CRUSH rule 0 x 68 [102,91,6]
+  CRUSH rule 0 x 69 [62,77,11]
+  CRUSH rule 0 x 70 [84,105,4]
+  CRUSH rule 0 x 71 [9,33,38]
+  CRUSH rule 0 x 72 [97,42,22]
+  CRUSH rule 0 x 73 [64,83,6]
+  CRUSH rule 0 x 74 [29,50,11]
+  CRUSH rule 0 x 75 [29,28,4]
+  CRUSH rule 0 x 76 [55,0,7]
+  CRUSH rule 0 x 77 [107,21,0]
+  CRUSH rule 0 x 78 [11,89,102]
+  CRUSH rule 0 x 79 [64,51,7]
+  CRUSH rule 0 x 80 [0,31,14]
+  CRUSH rule 0 x 81 [71,109,19]
+  CRUSH rule 0 x 82 [37,21,74]
+  CRUSH rule 0 x 83 [92,103,3]
+  CRUSH rule 0 x 84 [49,115,7]
+  CRUSH rule 0 x 85 [54,101,19]
+  CRUSH rule 0 x 86 [37,7,109]
+  CRUSH rule 0 x 87 [116,4,33]
+  CRUSH rule 0 x 88 [38,27,17]
+  CRUSH rule 0 x 89 [76,77,19]
+  CRUSH rule 0 x 90 [14,50,39]
+  CRUSH rule 0 x 91 [68,19,105]
+  CRUSH rule 0 x 92 [86,9,73]
+  CRUSH rule 0 x 93 [44,65,19]
+  CRUSH rule 0 x 94 [61,102,22]
+  CRUSH rule 0 x 95 [93,86,21]
+  CRUSH rule 0 x 96 [66,87,17]
+  CRUSH rule 0 x 97 [111,9,89]
+  CRUSH rule 0 x 98 [93,102,6]
+  CRUSH rule 0 x 99 [78,3,81]
+  CRUSH rule 0 x 100 [6,63,104]
+  CRUSH rule 0 x 101 [84,16,17]
+  CRUSH rule 0 x 102 [82,105,7]
+  CRUSH rule 0 x 103 [66,6,49]
+  CRUSH rule 0 x 104 [14,95,50]
+  CRUSH rule 0 x 105 [87,1,7]
+  CRUSH rule 0 x 106 [69,116,4]
+  CRUSH rule 0 x 107 [1,55,4]
+  CRUSH rule 0 x 108 [94,53,4]
+  CRUSH rule 0 x 109 [112,13,25]
+  CRUSH rule 0 x 110 [54,61,13]
+  CRUSH rule 0 x 111 [10,78,3]
+  CRUSH rule 0 x 112 [89,9,109]
+  CRUSH rule 0 x 113 [69,2,9]
+  CRUSH rule 0 x 114 [79,110,9]
+  CRUSH rule 0 x 115 [50,85,6]
+  CRUSH rule 0 x 116 [96,16,4]
+  CRUSH rule 0 x 117 [87,42,13]
+  CRUSH rule 0 x 118 [23,56,13]
+  CRUSH rule 0 x 119 [104,11,71]
+  CRUSH rule 0 x 120 [57,5,22]
+  CRUSH rule 0 x 121 [105,9,114]
+  CRUSH rule 0 x 122 [45,110,4]
+  CRUSH rule 0 x 123 [112,35,14]
+  CRUSH rule 0 x 124 [110,49,17]
+  CRUSH rule 0 x 125 [66,105,13]
+  CRUSH rule 0 x 126 [51,28,4]
+  CRUSH rule 0 x 127 [70,6,65]
+  CRUSH rule 0 x 128 [90,16,8]
+  CRUSH rule 0 x 129 [103,110,8]
+  CRUSH rule 0 x 130 [50,11,63]
+  CRUSH rule 0 x 131 [23,60,9]
+  CRUSH rule 0 x 132 [69,70,19]
+  CRUSH rule 0 x 133 [52,25,6]
+  CRUSH rule 0 x 134 [78,29,8]
+  CRUSH rule 0 x 135 [78,3,29]
+  CRUSH rule 0 x 136 [32,29,17]
+  CRUSH rule 0 x 137 [11,78,75]
+  CRUSH rule 0 x 138 [17,94,85]
+  CRUSH rule 0 x 139 [89,60,8]
+  CRUSH rule 0 x 140 [39,62,13]
+  CRUSH rule 0 x 141 [89,98,3]
+  CRUSH rule 0 x 142 [70,61,4]
+  CRUSH rule 0 x 143 [51,28,7]
+  CRUSH rule 0 x 144 [13,81,60]
+  CRUSH rule 0 x 145 [77,119,17]
+  CRUSH rule 0 x 146 [8,64,53]
+  CRUSH rule 0 x 147 [22,37,94]
+  CRUSH rule 0 x 148 [74,69,11]
+  CRUSH rule 0 x 149 [76,13,81]
+  CRUSH rule 0 x 150 [14,47,110]
+  CRUSH rule 0 x 151 [90,4,65]
+  CRUSH rule 0 x 152 [49,18,15]
+  CRUSH rule 0 x 153 [71,44,9]
+  CRUSH rule 0 x 154 [94,81,13]
+  CRUSH rule 0 x 155 [75,6,70]
+  CRUSH rule 0 x 156 [94,85,7]
+  CRUSH rule 0 x 157 [112,43,3]
+  CRUSH rule 0 x 158 [26,17,99]
+  CRUSH rule 0 x 159 [52,29,3]
+  CRUSH rule 0 x 160 [41,0,7]
+  CRUSH rule 0 x 161 [19,78,95]
+  CRUSH rule 0 x 162 [55,2,9]
+  CRUSH rule 0 x 163 [54,31,9]
+  CRUSH rule 0 x 164 [45,5,14]
+  CRUSH rule 0 x 165 [25,72,7]
+  CRUSH rule 0 x 166 [73,36,7]
+  CRUSH rule 0 x 167 [89,58,14]
+  CRUSH rule 0 x 168 [47,40,15]
+  CRUSH rule 0 x 169 [51,21,0]
+  CRUSH rule 0 x 170 [68,91,17]
+  CRUSH rule 0 x 171 [73,90,13]
+  CRUSH rule 0 x 172 [33,15,102]
+  CRUSH rule 0 x 173 [102,59,19]
+  CRUSH rule 0 x 174 [116,25,15]
+  CRUSH rule 0 x 175 [3,41,102]
+  CRUSH rule 0 x 176 [94,91,3]
+  CRUSH rule 0 x 177 [52,85,8]
+  CRUSH rule 0 x 178 [39,2,15]
+  CRUSH rule 0 x 179 [72,97,15]
+  CRUSH rule 0 x 180 [60,7,99]
+  CRUSH rule 0 x 181 [18,59,15]
+  CRUSH rule 0 x 182 [22,90,25]
+  CRUSH rule 0 x 183 [11,74,103]
+  CRUSH rule 0 x 184 [92,101,6]
+  CRUSH rule 0 x 185 [97,8,24]
+  CRUSH rule 0 x 186 [67,116,4]
+  CRUSH rule 0 x 187 [116,11,31]
+  CRUSH rule 0 x 188 [69,92,9]
+  CRUSH rule 0 x 189 [47,84,3]
+  CRUSH rule 0 x 190 [90,13,23]
+  CRUSH rule 0 x 191 [49,17,60]
+  CRUSH rule 0 x 192 [68,93,7]
+  CRUSH rule 0 x 193 [0,33,6]
+  CRUSH rule 0 x 194 [17,58,61]
+  CRUSH rule 0 x 195 [119,41,9]
+  CRUSH rule 0 x 196 [72,27,22]
+  CRUSH rule 0 x 197 [106,83,13]
+  CRUSH rule 0 x 198 [114,95,14]
+  CRUSH rule 0 x 199 [0,83,11]
+  CRUSH rule 0 x 200 [35,86,14]
+  CRUSH rule 0 x 201 [14,29,109]
+  CRUSH rule 0 x 202 [98,33,17]
+  CRUSH rule 0 x 203 [36,22,101]
+  CRUSH rule 0 x 204 [10,98,17]
+  CRUSH rule 0 x 205 [22,61,72]
+  CRUSH rule 0 x 206 [49,112,15]
+  CRUSH rule 0 x 207 [80,39,14]
+  CRUSH rule 0 x 208 [63,26,7]
+  CRUSH rule 0 x 209 [85,111,8]
+  CRUSH rule 0 x 210 [79,18,11]
+  CRUSH rule 0 x 211 [26,10,19]
+  CRUSH rule 0 x 212 [28,103,15]
+  CRUSH rule 0 x 213 [91,0,8]
+  CRUSH rule 0 x 214 [78,47,13]
+  CRUSH rule 0 x 215 [61,22,102]
+  CRUSH rule 0 x 216 [99,3,104]
+  CRUSH rule 0 x 217 [86,89,15]
+  CRUSH rule 0 x 218 [93,96,4]
+  CRUSH rule 0 x 219 [28,59,6]
+  CRUSH rule 0 x 220 [56,8,83]
+  CRUSH rule 0 x 221 [0,9,71]
+  CRUSH rule 0 x 222 [50,63,21]
+  CRUSH rule 0 x 223 [29,1,15]
+  CRUSH rule 0 x 224 [52,10,19]
+  CRUSH rule 0 x 225 [61,11,64]
+  CRUSH rule 0 x 226 [44,22,93]
+  CRUSH rule 0 x 227 [42,3,81]
+  CRUSH rule 0 x 228 [117,49,22]
+  CRUSH rule 0 x 229 [100,79,9]
+  CRUSH rule 0 x 230 [41,114,11]
+  CRUSH rule 0 x 231 [56,95,8]
+  CRUSH rule 0 x 232 [23,44,11]
+  CRUSH rule 0 x 233 [88,103,21]
+  CRUSH rule 0 x 234 [4,101,18]
+  CRUSH rule 0 x 235 [26,10,11]
+  CRUSH rule 0 x 236 [32,37,3]
+  CRUSH rule 0 x 237 [92,3,61]
+  CRUSH rule 0 x 238 [10,26,22]
+  CRUSH rule 0 x 239 [15,105,2]
+  CRUSH rule 0 x 240 [109,85,14]
+  CRUSH rule 0 x 241 [47,108,15]
+  CRUSH rule 0 x 242 [24,99,9]
+  CRUSH rule 0 x 243 [76,8,99]
+  CRUSH rule 0 x 244 [96,19,105]
+  CRUSH rule 0 x 245 [27,28,19]
+  CRUSH rule 0 x 246 [35,82,19]
+  CRUSH rule 0 x 247 [99,102,4]
+  CRUSH rule 0 x 248 [8,29,42]
+  CRUSH rule 0 x 249 [85,1,13]
+  CRUSH rule 0 x 250 [79,102,13]
+  CRUSH rule 0 x 251 [28,103,19]
+  CRUSH rule 0 x 252 [95,22,92]
+  CRUSH rule 0 x 253 [109,27,17]
+  CRUSH rule 0 x 254 [80,103,3]
+  CRUSH rule 0 x 255 [112,22,85]
+  CRUSH rule 0 x 256 [37,38,11]
+  CRUSH rule 0 x 257 [69,117,9]
+  CRUSH rule 0 x 258 [34,55,19]
+  CRUSH rule 0 x 259 [70,17,91]
+  CRUSH rule 0 x 260 [98,29,4]
+  CRUSH rule 0 x 261 [94,83,22]
+  CRUSH rule 0 x 262 [42,49,14]
+  CRUSH rule 0 x 263 [65,42,14]
+  CRUSH rule 0 x 264 [36,17,107]
+  CRUSH rule 0 x 265 [66,63,4]
+  CRUSH rule 0 x 266 [75,92,7]
+  CRUSH rule 0 x 267 [58,35,6]
+  CRUSH rule 0 x 268 [38,9,63]
+  CRUSH rule 0 x 269 [43,104,7]
+  CRUSH rule 0 x 270 [58,37,4]
+  CRUSH rule 0 x 271 [19,33,114]
+  CRUSH rule 0 x 272 [73,9,100]
+  CRUSH rule 0 x 273 [108,29,22]
+  CRUSH rule 0 x 274 [47,64,22]
+  CRUSH rule 0 x 275 [92,19,43]
+  CRUSH rule 0 x 276 [7,79,118]
+  CRUSH rule 0 x 277 [19,68,10]
+  CRUSH rule 0 x 278 [116,95,19]
+  CRUSH rule 0 x 279 [101,3,76]
+  CRUSH rule 0 x 280 [113,69,4]
+  CRUSH rule 0 x 281 [14,93,96]
+  CRUSH rule 0 x 282 [106,7,47]
+  CRUSH rule 0 x 283 [8,118,101]
+  CRUSH rule 0 x 284 [10,110,22]
+  CRUSH rule 0 x 285 [88,63,15]
+  CRUSH rule 0 x 286 [27,4,18]
+  CRUSH rule 0 x 287 [84,65,4]
+  CRUSH rule 0 x 288 [103,8,70]
+  CRUSH rule 0 x 289 [9,104,45]
+  CRUSH rule 0 x 290 [115,7,101]
+  CRUSH rule 0 x 291 [48,45,13]
+  CRUSH rule 0 x 292 [52,16,14]
+  CRUSH rule 0 x 293 [27,24,17]
+  CRUSH rule 0 x 294 [79,36,13]
+  CRUSH rule 0 x 295 [37,116,7]
+  CRUSH rule 0 x 296 [56,61,7]
+  CRUSH rule 0 x 297 [35,40,9]
+  CRUSH rule 0 x 298 [71,118,8]
+  CRUSH rule 0 x 299 [79,1,19]
+  CRUSH rule 0 x 300 [67,5,9]
+  CRUSH rule 0 x 301 [51,110,8]
+  CRUSH rule 0 x 302 [78,67,19]
+  CRUSH rule 0 x 303 [19,94,31]
+  CRUSH rule 0 x 304 [101,66,13]
+  CRUSH rule 0 x 305 [81,62,6]
+  CRUSH rule 0 x 306 [0,23,9]
+  CRUSH rule 0 x 307 [44,15,95]
+  CRUSH rule 0 x 308 [91,98,21]
+  CRUSH rule 0 x 309 [15,18,99]
+  CRUSH rule 0 x 310 [26,89,11]
+  CRUSH rule 0 x 311 [36,41,9]
+  CRUSH rule 0 x 312 [33,22,113]
+  CRUSH rule 0 x 313 [104,16,3]
+  CRUSH rule 0 x 314 [28,4,23]
+  CRUSH rule 0 x 315 [16,8,96]
+  CRUSH rule 0 x 316 [4,1,79]
+  CRUSH rule 0 x 317 [118,8,31]
+  CRUSH rule 0 x 318 [32,47,7]
+  CRUSH rule 0 x 319 [24,83,4]
+  CRUSH rule 0 x 320 [36,97,17]
+  CRUSH rule 0 x 321 [26,85,11]
+  CRUSH rule 0 x 322 [87,42,21]
+  CRUSH rule 0 x 323 [73,0,13]
+  CRUSH rule 0 x 324 [64,37,21]
+  CRUSH rule 0 x 325 [52,16,3]
+  CRUSH rule 0 x 326 [111,93,13]
+  CRUSH rule 0 x 327 [62,16,19]
+  CRUSH rule 0 x 328 [7,42,67]
+  CRUSH rule 0 x 329 [93,34,11]
+  CRUSH rule 0 x 330 [24,4,63]
+  CRUSH rule 0 x 331 [41,21,111]
+  CRUSH rule 0 x 332 [61,110,3]
+  CRUSH rule 0 x 333 [16,8,116]
+  CRUSH rule 0 x 334 [94,35,15]
+  CRUSH rule 0 x 335 [71,74,7]
+  CRUSH rule 0 x 336 [16,19,66]
+  CRUSH rule 0 x 337 [37,11,52]
+  CRUSH rule 0 x 338 [109,69,13]
+  CRUSH rule 0 x 339 [13,64,93]
+  CRUSH rule 0 x 340 [119,15,107]
+  CRUSH rule 0 x 341 [63,114,14]
+  CRUSH rule 0 x 342 [92,25,17]
+  CRUSH rule 0 x 343 [49,26,17]
+  CRUSH rule 0 x 344 [103,26,7]
+  CRUSH rule 0 x 345 [56,25,8]
+  CRUSH rule 0 x 346 [3,79,24]
+  CRUSH rule 0 x 347 [106,27,21]
+  CRUSH rule 0 x 348 [10,117,19]
+  CRUSH rule 0 x 349 [96,37,8]
+  CRUSH rule 0 x 350 [63,32,9]
+  CRUSH rule 0 x 351 [60,85,22]
+  CRUSH rule 0 x 352 [103,84,17]
+  CRUSH rule 0 x 353 [10,113,13]
+  CRUSH rule 0 x 354 [55,52,11]
+  CRUSH rule 0 x 355 [73,68,14]
+  CRUSH rule 0 x 356 [114,41,14]
+  CRUSH rule 0 x 357 [70,13,75]
+  CRUSH rule 0 x 358 [97,13,42]
+  CRUSH rule 0 x 359 [4,117,87]
+  CRUSH rule 0 x 360 [106,69,15]
+  CRUSH rule 0 x 361 [27,46,6]
+  CRUSH rule 0 x 362 [28,33,17]
+  CRUSH rule 0 x 363 [45,26,6]
+  CRUSH rule 0 x 364 [23,50,4]
+  CRUSH rule 0 x 365 [57,114,19]
+  CRUSH rule 0 x 366 [14,58,16]
+  CRUSH rule 0 x 367 [108,65,8]
+  CRUSH rule 0 x 368 [103,32,3]
+  CRUSH rule 0 x 369 [11,57,110]
+  CRUSH rule 0 x 370 [11,89,66]
+  CRUSH rule 0 x 371 [34,55,19]
+  CRUSH rule 0 x 372 [58,10,9]
+  CRUSH rule 0 x 373 [6,42,27]
+  CRUSH rule 0 x 374 [110,95,4]
+  CRUSH rule 0 x 375 [19,92,103]
+  CRUSH rule 0 x 376 [22,86,91]
+  CRUSH rule 0 x 377 [93,113,11]
+  CRUSH rule 0 x 378 [67,36,15]
+  CRUSH rule 0 x 379 [77,115,7]
+  CRUSH rule 0 x 380 [3,108,83]
+  CRUSH rule 0 x 381 [55,1,14]
+  CRUSH rule 0 x 382 [26,51,17]
+  CRUSH rule 0 x 383 [48,25,13]
+  CRUSH rule 0 x 384 [15,100,81]
+  CRUSH rule 0 x 385 [82,4,67]
+  CRUSH rule 0 x 386 [108,63,11]
+  CRUSH rule 0 x 387 [70,41,21]
+  CRUSH rule 0 x 388 [5,67,19]
+  CRUSH rule 0 x 389 [14,1,45]
+  CRUSH rule 0 x 390 [68,10,13]
+  CRUSH rule 0 x 391 [113,14,27]
+  CRUSH rule 0 x 392 [72,14,77]
+  CRUSH rule 0 x 393 [115,6,81]
+  CRUSH rule 0 x 394 [38,21,16]
+  CRUSH rule 0 x 395 [0,27,13]
+  CRUSH rule 0 x 396 [59,92,11]
+  CRUSH rule 0 x 397 [87,1,7]
+  CRUSH rule 0 x 398 [44,75,14]
+  CRUSH rule 0 x 399 [9,2,95]
+  CRUSH rule 0 x 400 [19,63,98]
+  CRUSH rule 0 x 401 [79,34,11]
+  CRUSH rule 0 x 402 [107,98,8]
+  CRUSH rule 0 x 403 [23,82,13]
+  CRUSH rule 0 x 404 [76,75,7]
+  CRUSH rule 0 x 405 [10,32,15]
+  CRUSH rule 0 x 406 [38,16,7]
+  CRUSH rule 0 x 407 [70,85,9]
+  CRUSH rule 0 x 408 [55,72,14]
+  CRUSH rule 0 x 409 [102,15,73]
+  CRUSH rule 0 x 410 [59,13,118]
+  CRUSH rule 0 x 411 [34,29,21]
+  CRUSH rule 0 x 412 [108,99,9]
+  CRUSH rule 0 x 413 [54,107,8]
+  CRUSH rule 0 x 414 [70,4,73]
+  CRUSH rule 0 x 415 [107,36,13]
+  CRUSH rule 0 x 416 [21,68,57]
+  CRUSH rule 0 x 417 [8,70,61]
+  CRUSH rule 0 x 418 [51,46,3]
+  CRUSH rule 0 x 419 [8,66,79]
+  CRUSH rule 0 x 420 [109,105,7]
+  CRUSH rule 0 x 421 [114,17,67]
+  CRUSH rule 0 x 422 [109,87,17]
+  CRUSH rule 0 x 423 [59,98,9]
+  CRUSH rule 0 x 424 [71,5,17]
+  CRUSH rule 0 x 425 [101,111,15]
+  CRUSH rule 0 x 426 [47,46,19]
+  CRUSH rule 0 x 427 [8,115,65]
+  CRUSH rule 0 x 428 [68,103,21]
+  CRUSH rule 0 x 429 [76,6,75]
+  CRUSH rule 0 x 430 [69,86,13]
+  CRUSH rule 0 x 431 [70,83,17]
+  CRUSH rule 0 x 432 [46,37,19]
+  CRUSH rule 0 x 433 [6,101,68]
+  CRUSH rule 0 x 434 [64,69,4]
+  CRUSH rule 0 x 435 [16,50,6]
+  CRUSH rule 0 x 436 [89,102,21]
+  CRUSH rule 0 x 437 [29,114,9]
+  CRUSH rule 0 x 438 [105,98,6]
+  CRUSH rule 0 x 439 [29,119,7]
+  CRUSH rule 0 x 440 [38,7,87]
+  CRUSH rule 0 x 441 [112,105,13]
+  CRUSH rule 0 x 442 [55,108,21]
+  CRUSH rule 0 x 443 [44,57,9]
+  CRUSH rule 0 x 444 [72,27,9]
+  CRUSH rule 0 x 445 [19,5,39]
+  CRUSH rule 0 x 446 [40,47,7]
+  CRUSH rule 0 x 447 [13,61,90]
+  CRUSH rule 0 x 448 [7,68,55]
+  CRUSH rule 0 x 449 [67,19,66]
+  CRUSH rule 0 x 450 [117,79,17]
+  CRUSH rule 0 x 451 [93,108,8]
+  CRUSH rule 0 x 452 [70,49,11]
+  CRUSH rule 0 x 453 [82,22,59]
+  CRUSH rule 0 x 454 [53,18,21]
+  CRUSH rule 0 x 455 [91,92,3]
+  CRUSH rule 0 x 456 [101,104,9]
+  CRUSH rule 0 x 457 [113,51,4]
+  CRUSH rule 0 x 458 [53,34,21]
+  CRUSH rule 0 x 459 [25,115,11]
+  CRUSH rule 0 x 460 [105,9,74]
+  CRUSH rule 0 x 461 [102,35,13]
+  CRUSH rule 0 x 462 [98,107,8]
+  CRUSH rule 0 x 463 [108,105,11]
+  CRUSH rule 0 x 464 [19,109,105]
+  CRUSH rule 0 x 465 [29,86,21]
+  CRUSH rule 0 x 466 [66,7,16]
+  CRUSH rule 0 x 467 [6,57,44]
+  CRUSH rule 0 x 468 [97,26,7]
+  CRUSH rule 0 x 469 [98,75,9]
+  CRUSH rule 0 x 470 [50,3,45]
+  CRUSH rule 0 x 471 [40,79,17]
+  CRUSH rule 0 x 472 [74,79,6]
+  CRUSH rule 0 x 473 [95,21,36]
+  CRUSH rule 0 x 474 [51,32,15]
+  CRUSH rule 0 x 475 [49,110,22]
+  CRUSH rule 0 x 476 [110,31,11]
+  CRUSH rule 0 x 477 [25,106,7]
+  CRUSH rule 0 x 478 [47,46,6]
+  CRUSH rule 0 x 479 [70,37,6]
+  CRUSH rule 0 x 480 [62,57,6]
+  CRUSH rule 0 x 481 [26,19,49]
+  CRUSH rule 0 x 482 [84,85,11]
+  CRUSH rule 0 x 483 [15,116,63]
+  CRUSH rule 0 x 484 [37,36,8]
+  CRUSH rule 0 x 485 [47,117,17]
+  CRUSH rule 0 x 486 [92,10,6]
+  CRUSH rule 0 x 487 [106,51,11]
+  CRUSH rule 0 x 488 [42,9,87]
+  CRUSH rule 0 x 489 [76,16,21]
+  CRUSH rule 0 x 490 [68,17,101]
+  CRUSH rule 0 x 491 [80,71,8]
+  CRUSH rule 0 x 492 [21,57,86]
+  CRUSH rule 0 x 493 [99,78,14]
+  CRUSH rule 0 x 494 [4,87,114]
+  CRUSH rule 0 x 495 [40,43,17]
+  CRUSH rule 0 x 496 [93,38,3]
+  CRUSH rule 0 x 497 [102,71,6]
+  CRUSH rule 0 x 498 [68,83,3]
+  CRUSH rule 0 x 499 [10,26,7]
+  CRUSH rule 0 x 500 [50,6,95]
+  CRUSH rule 0 x 501 [60,9,103]
+  CRUSH rule 0 x 502 [11,64,53]
+  CRUSH rule 0 x 503 [117,25,14]
+  CRUSH rule 0 x 504 [90,41,9]
+  CRUSH rule 0 x 505 [91,100,21]
+  CRUSH rule 0 x 506 [82,103,14]
+  CRUSH rule 0 x 507 [81,54,6]
+  CRUSH rule 0 x 508 [34,87,19]
+  CRUSH rule 0 x 509 [88,63,8]
+  CRUSH rule 0 x 510 [11,73,106]
+  CRUSH rule 0 x 511 [72,27,21]
+  CRUSH rule 0 x 512 [118,73,13]
+  CRUSH rule 0 x 513 [22,76,77]
+  CRUSH rule 0 x 514 [82,11,29]
+  CRUSH rule 0 x 515 [27,0,22]
+  CRUSH rule 0 x 516 [66,13,43]
+  CRUSH rule 0 x 517 [83,60,8]
+  CRUSH rule 0 x 518 [18,3,83]
+  CRUSH rule 0 x 519 [67,119,14]
+  CRUSH rule 0 x 520 [15,88,53]
+  CRUSH rule 0 x 521 [63,113,7]
+  CRUSH rule 0 x 522 [56,73,19]
+  CRUSH rule 0 x 523 [36,35,3]
+  CRUSH rule 0 x 524 [33,38,13]
+  CRUSH rule 0 x 525 [3,119,45]
+  CRUSH rule 0 x 526 [83,50,3]
+  CRUSH rule 0 x 527 [37,0,11]
+  CRUSH rule 0 x 528 [108,87,15]
+  CRUSH rule 0 x 529 [107,60,4]
+  CRUSH rule 0 x 530 [49,3,56]
+  CRUSH rule 0 x 531 [27,104,21]
+  CRUSH rule 0 x 532 [68,14,107]
+  CRUSH rule 0 x 533 [5,85,3]
+  CRUSH rule 0 x 534 [97,24,19]
+  CRUSH rule 0 x 535 [8,75,88]
+  CRUSH rule 0 x 536 [3,37,86]
+  CRUSH rule 0 x 537 [116,7,59]
+  CRUSH rule 0 x 538 [85,56,17]
+  CRUSH rule 0 x 539 [10,9,117]
+  CRUSH rule 0 x 540 [100,101,14]
+  CRUSH rule 0 x 541 [111,77,11]
+  CRUSH rule 0 x 542 [50,27,13]
+  CRUSH rule 0 x 543 [45,21,109]
+  CRUSH rule 0 x 544 [106,65,21]
+  CRUSH rule 0 x 545 [43,114,17]
+  CRUSH rule 0 x 546 [108,79,17]
+  CRUSH rule 0 x 547 [67,50,4]
+  CRUSH rule 0 x 548 [58,61,6]
+  CRUSH rule 0 x 549 [60,22,89]
+  CRUSH rule 0 x 550 [47,68,21]
+  CRUSH rule 0 x 551 [14,88,59]
+  CRUSH rule 0 x 552 [70,65,22]
+  CRUSH rule 0 x 553 [96,105,9]
+  CRUSH rule 0 x 554 [61,94,22]
+  CRUSH rule 0 x 555 [76,37,9]
+  CRUSH rule 0 x 556 [106,89,9]
+  CRUSH rule 0 x 557 [39,113,17]
+  CRUSH rule 0 x 558 [70,79,8]
+  CRUSH rule 0 x 559 [106,69,14]
+  CRUSH rule 0 x 560 [94,97,8]
+  CRUSH rule 0 x 561 [27,76,9]
+  CRUSH rule 0 x 562 [97,62,7]
+  CRUSH rule 0 x 563 [64,103,15]
+  CRUSH rule 0 x 564 [96,41,14]
+  CRUSH rule 0 x 565 [66,71,19]
+  CRUSH rule 0 x 566 [27,38,11]
+  CRUSH rule 0 x 567 [88,8,25]
+  CRUSH rule 0 x 568 [106,17,33]
+  CRUSH rule 0 x 569 [102,63,17]
+  CRUSH rule 0 x 570 [98,27,19]
+  CRUSH rule 0 x 571 [95,98,4]
+  CRUSH rule 0 x 572 [62,83,7]
+  CRUSH rule 0 x 573 [51,118,4]
+  CRUSH rule 0 x 574 [89,78,13]
+  CRUSH rule 0 x 575 [87,19,38]
+  CRUSH rule 0 x 576 [112,73,19]
+  CRUSH rule 0 x 577 [8,84,41]
+  CRUSH rule 0 x 578 [64,99,7]
+  CRUSH rule 0 x 579 [78,77,17]
+  CRUSH rule 0 x 580 [68,95,7]
+  CRUSH rule 0 x 581 [55,52,7]
+  CRUSH rule 0 x 582 [15,113,77]
+  CRUSH rule 0 x 583 [74,105,15]
+  CRUSH rule 0 x 584 [22,92,87]
+  CRUSH rule 0 x 585 [35,1,15]
+  CRUSH rule 0 x 586 [33,1,13]
+  CRUSH rule 0 x 587 [106,99,22]
+  CRUSH rule 0 x 588 [0,83,7]
+  CRUSH rule 0 x 589 [7,95,90]
+  CRUSH rule 0 x 590 [40,69,4]
+  CRUSH rule 0 x 591 [42,23,11]
+  CRUSH rule 0 x 592 [45,22,108]
+  CRUSH rule 0 x 593 [89,14,42]
+  CRUSH rule 0 x 594 [27,76,9]
+  CRUSH rule 0 x 595 [7,10,34]
+  CRUSH rule 0 x 596 [82,59,19]
+  CRUSH rule 0 x 597 [72,83,9]
+  CRUSH rule 0 x 598 [34,19,69]
+  CRUSH rule 0 x 599 [119,61,7]
+  CRUSH rule 0 x 600 [24,27,21]
+  CRUSH rule 0 x 601 [104,15,49]
+  CRUSH rule 0 x 602 [48,45,3]
+  CRUSH rule 0 x 603 [24,13,41]
+  CRUSH rule 0 x 604 [89,0,14]
+  CRUSH rule 0 x 605 [104,87,13]
+  CRUSH rule 0 x 606 [49,34,13]
+  CRUSH rule 0 x 607 [95,40,15]
+  CRUSH rule 0 x 608 [112,91,6]
+  CRUSH rule 0 x 609 [61,66,11]
+  CRUSH rule 0 x 610 [106,16,14]
+  CRUSH rule 0 x 611 [66,87,3]
+  CRUSH rule 0 x 612 [103,8,44]
+  CRUSH rule 0 x 613 [13,91,96]
+  CRUSH rule 0 x 614 [81,88,11]
+  CRUSH rule 0 x 615 [61,19,64]
+  CRUSH rule 0 x 616 [41,15,106]
+  CRUSH rule 0 x 617 [111,69,15]
+  CRUSH rule 0 x 618 [26,99,9]
+  CRUSH rule 0 x 619 [92,27,19]
+  CRUSH rule 0 x 620 [108,103,15]
+  CRUSH rule 0 x 621 [106,99,3]
+  CRUSH rule 0 x 622 [67,48,14]
+  CRUSH rule 0 x 623 [94,61,15]
+  CRUSH rule 0 x 624 [115,59,15]
+  CRUSH rule 0 x 625 [111,27,19]
+  CRUSH rule 0 x 626 [3,55,80]
+  CRUSH rule 0 x 627 [19,29,90]
+  CRUSH rule 0 x 628 [65,88,7]
+  CRUSH rule 0 x 629 [6,46,87]
+  CRUSH rule 0 x 630 [22,72,55]
+  CRUSH rule 0 x 631 [35,22,94]
+  CRUSH rule 0 x 632 [81,0,14]
+  CRUSH rule 0 x 633 [65,68,13]
+  CRUSH rule 0 x 634 [87,50,7]
+  CRUSH rule 0 x 635 [40,73,13]
+  CRUSH rule 0 x 636 [23,70,3]
+  CRUSH rule 0 x 637 [102,45,3]
+  CRUSH rule 0 x 638 [43,114,19]
+  CRUSH rule 0 x 639 [31,78,11]
+  CRUSH rule 0 x 640 [113,73,22]
+  CRUSH rule 0 x 641 [45,96,3]
+  CRUSH rule 0 x 642 [47,66,3]
+  CRUSH rule 0 x 643 [64,47,21]
+  CRUSH rule 0 x 644 [31,21,119]
+  CRUSH rule 0 x 645 [76,43,6]
+  CRUSH rule 0 x 646 [37,54,8]
+  CRUSH rule 0 x 647 [58,87,19]
+  CRUSH rule 0 x 648 [31,21,102]
+  CRUSH rule 0 x 649 [88,45,14]
+  CRUSH rule 0 x 650 [116,7,107]
+  CRUSH rule 0 x 651 [97,106,3]
+  CRUSH rule 0 x 652 [57,112,9]
+  CRUSH rule 0 x 653 [8,116,97]
+  CRUSH rule 0 x 654 [49,32,7]
+  CRUSH rule 0 x 655 [89,62,17]
+  CRUSH rule 0 x 656 [0,49,22]
+  CRUSH rule 0 x 657 [47,17,58]
+  CRUSH rule 0 x 658 [75,82,17]
+  CRUSH rule 0 x 659 [26,83,8]
+  CRUSH rule 0 x 660 [65,112,13]
+  CRUSH rule 0 x 661 [91,48,3]
+  CRUSH rule 0 x 662 [111,99,17]
+  CRUSH rule 0 x 663 [88,35,3]
+  CRUSH rule 0 x 664 [59,78,8]
+  CRUSH rule 0 x 665 [78,15,67]
+  CRUSH rule 0 x 666 [112,4,61]
+  CRUSH rule 0 x 667 [97,46,8]
+  CRUSH rule 0 x 668 [97,8,56]
+  CRUSH rule 0 x 669 [85,66,3]
+  CRUSH rule 0 x 670 [41,48,14]
+  CRUSH rule 0 x 671 [116,97,13]
+  CRUSH rule 0 x 672 [44,55,17]
+  CRUSH rule 0 x 673 [83,50,14]
+  CRUSH rule 0 x 674 [36,8,65]
+  CRUSH rule 0 x 675 [88,14,43]
+  CRUSH rule 0 x 676 [62,8,99]
+  CRUSH rule 0 x 677 [88,67,8]
+  CRUSH rule 0 x 678 [98,83,3]
+  CRUSH rule 0 x 679 [33,78,3]
+  CRUSH rule 0 x 680 [55,94,17]
+  CRUSH rule 0 x 681 [115,95,3]
+  CRUSH rule 0 x 682 [27,94,15]
+  CRUSH rule 0 x 683 [57,80,9]
+  CRUSH rule 0 x 684 [22,65,44]
+  CRUSH rule 0 x 685 [106,55,8]
+  CRUSH rule 0 x 686 [86,95,4]
+  CRUSH rule 0 x 687 [32,57,13]
+  CRUSH rule 0 x 688 [80,22,49]
+  CRUSH rule 0 x 689 [6,48,71]
+  CRUSH rule 0 x 690 [43,70,14]
+  CRUSH rule 0 x 691 [34,105,4]
+  CRUSH rule 0 x 692 [40,97,13]
+  CRUSH rule 0 x 693 [29,84,21]
+  CRUSH rule 0 x 694 [6,84,57]
+  CRUSH rule 0 x 695 [19,69,112]
+  CRUSH rule 0 x 696 [36,75,11]
+  CRUSH rule 0 x 697 [96,99,14]
+  CRUSH rule 0 x 698 [61,11,84]
+  CRUSH rule 0 x 699 [47,62,15]
+  CRUSH rule 0 x 700 [99,82,22]
+  CRUSH rule 0 x 701 [42,11,91]
+  CRUSH rule 0 x 702 [0,71,22]
+  CRUSH rule 0 x 703 [92,3,89]
+  CRUSH rule 0 x 704 [10,19,88]
+  CRUSH rule 0 x 705 [105,21,2]
+  CRUSH rule 0 x 706 [74,105,13]
+  CRUSH rule 0 x 707 [0,77,15]
+  CRUSH rule 0 x 708 [84,8,39]
+  CRUSH rule 0 x 709 [114,97,19]
+  CRUSH rule 0 x 710 [94,7,33]
+  CRUSH rule 0 x 711 [68,49,8]
+  CRUSH rule 0 x 712 [34,75,11]
+  CRUSH rule 0 x 713 [29,0,21]
+  CRUSH rule 0 x 714 [81,115,3]
+  CRUSH rule 0 x 715 [71,84,6]
+  CRUSH rule 0 x 716 [40,17,69]
+  CRUSH rule 0 x 717 [61,62,14]
+  CRUSH rule 0 x 718 [40,85,13]
+  CRUSH rule 0 x 719 [59,42,3]
+  CRUSH rule 0 x 720 [69,72,14]
+  CRUSH rule 0 x 721 [62,21,35]
+  CRUSH rule 0 x 722 [115,8,43]
+  CRUSH rule 0 x 723 [117,41,13]
+  CRUSH rule 0 x 724 [45,102,4]
+  CRUSH rule 0 x 725 [53,113,13]
+  CRUSH rule 0 x 726 [84,19,103]
+  CRUSH rule 0 x 727 [109,14,31]
+  CRUSH rule 0 x 728 [76,16,11]
+  CRUSH rule 0 x 729 [108,47,11]
+  CRUSH rule 0 x 730 [28,47,21]
+  CRUSH rule 0 x 731 [78,37,14]
+  CRUSH rule 0 x 732 [55,90,4]
+  CRUSH rule 0 x 733 [84,3,99]
+  CRUSH rule 0 x 734 [27,117,4]
+  CRUSH rule 0 x 735 [83,4,54]
+  CRUSH rule 0 x 736 [70,67,21]
+  CRUSH rule 0 x 737 [117,15,101]
+  CRUSH rule 0 x 738 [118,22,65]
+  CRUSH rule 0 x 739 [87,38,11]
+  CRUSH rule 0 x 740 [29,38,19]
+  CRUSH rule 0 x 741 [96,73,4]
+  CRUSH rule 0 x 742 [106,83,8]
+  CRUSH rule 0 x 743 [105,94,9]
+  CRUSH rule 0 x 744 [23,14,78]
+  CRUSH rule 0 x 745 [28,6,87]
+  CRUSH rule 0 x 746 [56,47,13]
+  CRUSH rule 0 x 747 [65,70,19]
+  CRUSH rule 0 x 748 [48,89,17]
+  CRUSH rule 0 x 749 [102,51,6]
+  CRUSH rule 0 x 750 [50,3,59]
+  CRUSH rule 0 x 751 [36,25,9]
+  CRUSH rule 0 x 752 [69,52,15]
+  CRUSH rule 0 x 753 [116,65,21]
+  CRUSH rule 0 x 754 [9,57,40]
+  CRUSH rule 0 x 755 [98,81,4]
+  CRUSH rule 0 x 756 [113,8,43]
+  CRUSH rule 0 x 757 [47,66,14]
+  CRUSH rule 0 x 758 [57,88,4]
+  CRUSH rule 0 x 759 [74,97,6]
+  CRUSH rule 0 x 760 [53,90,8]
+  CRUSH rule 0 x 761 [78,97,7]
+  CRUSH rule 0 x 762 [87,104,8]
+  CRUSH rule 0 x 763 [13,45,92]
+  CRUSH rule 0 x 764 [106,81,22]
+  CRUSH rule 0 x 765 [109,91,6]
+  CRUSH rule 0 x 766 [76,97,7]
+  CRUSH rule 0 x 767 [41,116,6]
+  CRUSH rule 0 x 768 [13,114,57]
+  CRUSH rule 0 x 769 [91,96,13]
+  CRUSH rule 0 x 770 [105,19,104]
+  CRUSH rule 0 x 771 [10,76,17]
+  CRUSH rule 0 x 772 [118,17,69]
+  CRUSH rule 0 x 773 [116,75,6]
+  CRUSH rule 0 x 774 [100,43,19]
+  CRUSH rule 0 x 775 [102,43,13]
+  CRUSH rule 0 x 776 [69,38,14]
+  CRUSH rule 0 x 777 [76,49,17]
+  CRUSH rule 0 x 778 [38,13,89]
+  CRUSH rule 0 x 779 [46,21,29]
+  CRUSH rule 0 x 780 [63,102,6]
+  CRUSH rule 0 x 781 [105,92,22]
+  CRUSH rule 0 x 782 [117,31,13]
+  CRUSH rule 0 x 783 [60,93,13]
+  CRUSH rule 0 x 784 [82,81,15]
+  CRUSH rule 0 x 785 [27,84,8]
+  CRUSH rule 0 x 786 [41,80,19]
+  CRUSH rule 0 x 787 [13,54,43]
+  CRUSH rule 0 x 788 [4,100,41]
+  CRUSH rule 0 x 789 [50,37,14]
+  CRUSH rule 0 x 790 [58,16,15]
+  CRUSH rule 0 x 791 [96,14,105]
+  CRUSH rule 0 x 792 [80,4,35]
+  CRUSH rule 0 x 793 [6,71,82]
+  CRUSH rule 0 x 794 [14,89,52]
+  CRUSH rule 0 x 795 [51,3,78]
+  CRUSH rule 0 x 796 [114,77,19]
+  CRUSH rule 0 x 797 [79,100,15]
+  CRUSH rule 0 x 798 [42,10,7]
+  CRUSH rule 0 x 799 [48,11,101]
+  CRUSH rule 0 x 800 [91,7,18]
+  CRUSH rule 0 x 801 [2,6,73]
+  CRUSH rule 0 x 802 [116,89,7]
+  CRUSH rule 0 x 803 [37,32,7]
+  CRUSH rule 0 x 804 [33,4,106]
+  CRUSH rule 0 x 805 [96,22,41]
+  CRUSH rule 0 x 806 [67,90,9]
+  CRUSH rule 0 x 807 [47,42,17]
+  CRUSH rule 0 x 808 [76,79,14]
+  CRUSH rule 0 x 809 [27,26,3]
+  CRUSH rule 0 x 810 [119,61,8]
+  CRUSH rule 0 x 811 [75,72,15]
+  CRUSH rule 0 x 812 [25,52,13]
+  CRUSH rule 0 x 813 [64,13,77]
+  CRUSH rule 0 x 814 [110,53,3]
+  CRUSH rule 0 x 815 [84,61,4]
+  CRUSH rule 0 x 816 [25,22,84]
+  CRUSH rule 0 x 817 [40,73,13]
+  CRUSH rule 0 x 818 [34,13,45]
+  CRUSH rule 0 x 819 [88,19,85]
+  CRUSH rule 0 x 820 [104,49,11]
+  CRUSH rule 0 x 821 [58,69,14]
+  CRUSH rule 0 x 822 [29,72,6]
+  CRUSH rule 0 x 823 [100,103,17]
+  CRUSH rule 0 x 824 [102,81,4]
+  CRUSH rule 0 x 825 [47,17,94]
+  CRUSH rule 0 x 826 [45,34,22]
+  CRUSH rule 0 x 827 [101,11,66]
+  CRUSH rule 0 x 828 [60,27,19]
+  CRUSH rule 0 x 829 [45,90,9]
+  CRUSH rule 0 x 830 [51,96,17]
+  CRUSH rule 0 x 831 [6,64,73]
+  CRUSH rule 0 x 832 [57,78,13]
+  CRUSH rule 0 x 833 [34,97,3]
+  CRUSH rule 0 x 834 [90,33,6]
+  CRUSH rule 0 x 835 [14,46,25]
+  CRUSH rule 0 x 836 [38,43,7]
+  CRUSH rule 0 x 837 [51,74,15]
+  CRUSH rule 0 x 838 [6,32,107]
+  CRUSH rule 0 x 839 [106,8,39]
+  CRUSH rule 0 x 840 [33,109,3]
+  CRUSH rule 0 x 841 [110,15,71]
+  CRUSH rule 0 x 842 [66,67,13]
+  CRUSH rule 0 x 843 [11,63,48]
+  CRUSH rule 0 x 844 [74,13,59]
+  CRUSH rule 0 x 845 [74,43,22]
+  CRUSH rule 0 x 846 [98,107,19]
+  CRUSH rule 0 x 847 [10,3,88]
+  CRUSH rule 0 x 848 [89,17,111]
+  CRUSH rule 0 x 849 [42,59,14]
+  CRUSH rule 0 x 850 [40,73,13]
+  CRUSH rule 0 x 851 [65,94,11]
+  CRUSH rule 0 x 852 [31,94,7]
+  CRUSH rule 0 x 853 [49,11,114]
+  CRUSH rule 0 x 854 [90,31,21]
+  CRUSH rule 0 x 855 [2,19,81]
+  CRUSH rule 0 x 856 [40,22,61]
+  CRUSH rule 0 x 857 [15,82,91]
+  CRUSH rule 0 x 858 [10,80,19]
+  CRUSH rule 0 x 859 [29,48,4]
+  CRUSH rule 0 x 860 [114,75,21]
+  CRUSH rule 0 x 861 [22,33,98]
+  CRUSH rule 0 x 862 [22,25,76]
+  CRUSH rule 0 x 863 [79,50,11]
+  CRUSH rule 0 x 864 [68,6,41]
+  CRUSH rule 0 x 865 [25,92,14]
+  CRUSH rule 0 x 866 [18,89,22]
+  CRUSH rule 0 x 867 [3,78,41]
+  CRUSH rule 0 x 868 [81,98,11]
+  CRUSH rule 0 x 869 [22,104,89]
+  CRUSH rule 0 x 870 [73,98,3]
+  CRUSH rule 0 x 871 [25,54,19]
+  CRUSH rule 0 x 872 [39,48,11]
+  CRUSH rule 0 x 873 [92,9,75]
+  CRUSH rule 0 x 874 [21,43,66]
+  CRUSH rule 0 x 875 [27,108,7]
+  CRUSH rule 0 x 876 [98,75,13]
+  CRUSH rule 0 x 877 [73,5,4]
+  CRUSH rule 0 x 878 [64,45,22]
+  CRUSH rule 0 x 879 [29,18,9]
+  CRUSH rule 0 x 880 [56,91,13]
+  CRUSH rule 0 x 881 [109,69,4]
+  CRUSH rule 0 x 882 [60,33,11]
+  CRUSH rule 0 x 883 [93,96,11]
+  CRUSH rule 0 x 884 [67,58,4]
+  CRUSH rule 0 x 885 [31,8,104]
+  CRUSH rule 0 x 886 [2,107,9]
+  CRUSH rule 0 x 887 [5,93,19]
+  CRUSH rule 0 x 888 [16,13,26]
+  CRUSH rule 0 x 889 [3,76,93]
+  CRUSH rule 0 x 890 [48,63,4]
+  CRUSH rule 0 x 891 [86,79,22]
+  CRUSH rule 0 x 892 [64,9,10]
+  CRUSH rule 0 x 893 [118,33,22]
+  CRUSH rule 0 x 894 [16,111,11]
+  CRUSH rule 0 x 895 [40,107,4]
+  CRUSH rule 0 x 896 [97,96,14]
+  CRUSH rule 0 x 897 [60,67,22]
+  CRUSH rule 0 x 898 [10,2,21]
+  CRUSH rule 0 x 899 [75,80,4]
+  CRUSH rule 0 x 900 [102,81,8]
+  CRUSH rule 0 x 901 [66,87,14]
+  CRUSH rule 0 x 902 [102,49,8]
+  CRUSH rule 0 x 903 [5,14,33]
+  CRUSH rule 0 x 904 [50,16,4]
+  CRUSH rule 0 x 905 [19,51,110]
+  CRUSH rule 0 x 906 [75,119,13]
+  CRUSH rule 0 x 907 [47,5,7]
+  CRUSH rule 0 x 908 [96,9,29]
+  CRUSH rule 0 x 909 [94,75,19]
+  CRUSH rule 0 x 910 [88,63,15]
+  CRUSH rule 0 x 911 [102,23,3]
+  CRUSH rule 0 x 912 [91,60,13]
+  CRUSH rule 0 x 913 [29,17,96]
+  CRUSH rule 0 x 914 [84,29,17]
+  CRUSH rule 0 x 915 [70,22,107]
+  CRUSH rule 0 x 916 [32,9,57]
+  CRUSH rule 0 x 917 [43,26,3]
+  CRUSH rule 0 x 918 [91,98,6]
+  CRUSH rule 0 x 919 [13,69,56]
+  CRUSH rule 0 x 920 [18,87,11]
+  CRUSH rule 0 x 921 [104,33,14]
+  CRUSH rule 0 x 922 [33,19,117]
+  CRUSH rule 0 x 923 [28,8,101]
+  CRUSH rule 0 x 924 [69,88,9]
+  CRUSH rule 0 x 925 [71,32,17]
+  CRUSH rule 0 x 926 [64,69,15]
+  CRUSH rule 0 x 927 [99,106,13]
+  CRUSH rule 0 x 928 [13,113,95]
+  CRUSH rule 0 x 929 [117,61,21]
+  CRUSH rule 0 x 930 [31,82,3]
+  CRUSH rule 0 x 931 [46,79,22]
+  CRUSH rule 0 x 932 [60,13,103]
+  CRUSH rule 0 x 933 [88,31,6]
+  CRUSH rule 0 x 934 [68,4,99]
+  CRUSH rule 0 x 935 [31,18,4]
+  CRUSH rule 0 x 936 [104,57,6]
+  CRUSH rule 0 x 937 [110,22,95]
+  CRUSH rule 0 x 938 [29,106,13]
+  CRUSH rule 0 x 939 [77,13,52]
+  CRUSH rule 0 x 940 [76,33,7]
+  CRUSH rule 0 x 941 [66,37,8]
+  CRUSH rule 0 x 942 [83,94,9]
+  CRUSH rule 0 x 943 [4,74,89]
+  CRUSH rule 0 x 944 [113,53,21]
+  CRUSH rule 0 x 945 [17,52,16]
+  CRUSH rule 0 x 946 [37,111,11]
+  CRUSH rule 0 x 947 [107,74,7]
+  CRUSH rule 0 x 948 [55,98,9]
+  CRUSH rule 0 x 949 [45,72,21]
+  CRUSH rule 0 x 950 [96,23,3]
+  CRUSH rule 0 x 951 [40,93,7]
+  CRUSH rule 0 x 952 [93,46,6]
+  CRUSH rule 0 x 953 [55,92,6]
+  CRUSH rule 0 x 954 [84,57,7]
+  CRUSH rule 0 x 955 [31,117,13]
+  CRUSH rule 0 x 956 [72,11,55]
+  CRUSH rule 0 x 957 [3,74,87]
+  CRUSH rule 0 x 958 [8,106,43]
+  CRUSH rule 0 x 959 [42,59,22]
+  CRUSH rule 0 x 960 [113,107,11]
+  CRUSH rule 0 x 961 [116,8,53]
+  CRUSH rule 0 x 962 [13,62,79]
+  CRUSH rule 0 x 963 [0,99,14]
+  CRUSH rule 0 x 964 [59,21,32]
+  CRUSH rule 0 x 965 [47,115,9]
+  CRUSH rule 0 x 966 [88,63,13]
+  CRUSH rule 0 x 967 [71,108,14]
+  CRUSH rule 0 x 968 [73,7,54]
+  CRUSH rule 0 x 969 [53,6,2]
+  CRUSH rule 0 x 970 [3,40,65]
+  CRUSH rule 0 x 971 [87,38,9]
+  CRUSH rule 0 x 972 [3,37,109]
+  CRUSH rule 0 x 973 [113,27,4]
+  CRUSH rule 0 x 974 [114,23,13]
+  CRUSH rule 0 x 975 [40,59,8]
+  CRUSH rule 0 x 976 [81,38,19]
+  CRUSH rule 0 x 977 [95,102,11]
+  CRUSH rule 0 x 978 [35,56,15]
+  CRUSH rule 0 x 979 [98,6,45]
+  CRUSH rule 0 x 980 [52,69,3]
+  CRUSH rule 0 x 981 [89,117,15]
+  CRUSH rule 0 x 982 [1,47,22]
+  CRUSH rule 0 x 983 [34,61,13]
+  CRUSH rule 0 x 984 [78,25,8]
+  CRUSH rule 0 x 985 [99,52,6]
+  CRUSH rule 0 x 986 [4,59,84]
+  CRUSH rule 0 x 987 [78,21,27]
+  CRUSH rule 0 x 988 [79,2,11]
+  CRUSH rule 0 x 989 [87,17,32]
+  CRUSH rule 0 x 990 [47,118,9]
+  CRUSH rule 0 x 991 [61,18,6]
+  CRUSH rule 0 x 992 [83,66,17]
+  CRUSH rule 0 x 993 [75,62,8]
+  CRUSH rule 0 x 994 [74,57,9]
+  CRUSH rule 0 x 995 [100,97,7]
+  CRUSH rule 0 x 996 [41,6,58]
+  CRUSH rule 0 x 997 [89,76,7]
+  CRUSH rule 0 x 998 [92,47,13]
+  CRUSH rule 0 x 999 [101,11,66]
+  CRUSH rule 0 x 1000 [9,119,37]
+  CRUSH rule 0 x 1001 [49,32,7]
+  CRUSH rule 0 x 1002 [99,113,7]
+  CRUSH rule 0 x 1003 [43,18,6]
+  CRUSH rule 0 x 1004 [89,54,15]
+  CRUSH rule 0 x 1005 [105,84,8]
+  CRUSH rule 0 x 1006 [45,111,6]
+  CRUSH rule 0 x 1007 [19,57,5]
+  CRUSH rule 0 x 1008 [31,24,13]
+  CRUSH rule 0 x 1009 [19,111,61]
+  CRUSH rule 0 x 1010 [42,89,13]
+  CRUSH rule 0 x 1011 [25,114,6]
+  CRUSH rule 0 x 1012 [68,71,21]
+  CRUSH rule 0 x 1013 [5,65,3]
+  CRUSH rule 0 x 1014 [33,4,109]
+  CRUSH rule 0 x 1015 [106,45,9]
+  CRUSH rule 0 x 1016 [88,39,4]
+  CRUSH rule 0 x 1017 [0,89,7]
+  CRUSH rule 0 x 1018 [63,5,7]
+  CRUSH rule 0 x 1019 [104,97,4]
+  CRUSH rule 0 x 1020 [96,9,91]
+  CRUSH rule 0 x 1021 [117,6,43]
+  CRUSH rule 0 x 1022 [73,21,36]
+  CRUSH rule 0 x 1023 [0,16,3]
+  rule 0 (data) num_rep 10 result size == 3:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-indep.crushmap b/ceph/src/test/cli/crushtool/test-map-indep.crushmap
new file mode 100644
index 0000000000000000000000000000000000000000..b3fd3c21f37ccdc15dde44fb50407dd0e649a911
GIT binary patch
literal 31995
zcmeI*Wwac{y2kMcB)Ge~6S}Lqq;X5|;I2^u#E1}_;F{nTcXxMpcX!9#?LKcfJ-43z
zbU*aEpH9~L)B88sRK0LQ_spK2b;^_}Tc&xLGG)r69r(*GIR1lQ6?T5LqkLJo75wk-
zmfiX2-|)X*?NMeA{J#8>?<nQ~^~)yP@6DzuzU_SxmCeU%6Rio;}fCXm7L+
z+86DI_D2Vx15r^kWb4Q*f97U62#+0%4nc>a!_eXA2y`Sm3LTBIHHtdQEPv!?IR=j%
zi;hFbqZ81H=p=M9It87Iikcx?M`rmwH_K^w>~wSnIuo6R&PJ8cIVeIg%GN0AD6{;Q
zn?>O8Dx++TqK-1l
zFS%K&;IS*vmFOyTHM#~}i>^ae(e7s)wvNp5eQuV=@z@jSN%RzY
z8a;!aMbDw;=y{Z_QPfdp`7Srh3wZ2B^b&d*y@Fmvuc6n`8|Y0`)C}1=GRwEQS>D29
zEl^AJHhKrWi{3-;qYu!BC|jebqs;P6ZkCVm*vIG-^eJkETBA0oEoz6_qoQWW){$Ah
z&dt&Rk99&Pr$=4Kg!$A+R|XgC^yMxs$@G#Z1(qHK+#jxx&^xmm{HvGHgEnusQ$$!H3iil(9I
zsHhpTb!3*$bF<9AV>8h#G#kx9bJ09BA1y!&QMN`=N15fb+$@Xm*kZHtw$TsMzjfSMqAKUl&w+JQD)hmn`N6khIXLO(C6q2
z^dpU}_f7xXLo4P|Q-b(C4Q=4SaFkNtuE
zM1P^b(LYFDl9ff}(9Rcq1-(Yg{#j&}ExB2C!Ct$f3aBF54egHhKzpLS(B3Fpqo||I
zvN<=)KG)0Z9e@r*2cd(}A?Q$4)C}1=GRvmiEQevA!_g7wNOTlB8Xbdxor+FFr=v5_nW(54vUOyZ4Y^s)!airCO6VLEp%^Ko
zQGyK0)+p*Iv#ig}VzG}y9;N79bRIe%U4Sk`7onnN$kvfr*5zio82em;E=8B2%TZ-i
z1zmxzL|36~jiQb+%i7#5S7V=R(6#6~R25y1Za_Dpo6yaus2Q?#WR^9#S#H5Tx1!t7
z?dT44C%OyWjqXABqHK+#jxx*Y+${HDpZn1Rs2Zw{YM`2^7OIUNL`BVzts}Fn%FR*-
z`#gjmMvtJns2-}18lZ;gQIxGw)KO+xnVY2%_Gyfopr+_C)C@h2o|A$xo94mj~1YXXc1bBmY}65TcfC>%rZYW%QEb<
z9IZer(JHhWtwC$iIbKhU4(FO;oO
z)KO-cotx!v?DLQ8RSvtLa;QAo1?`F|po*xd8M1X`mRY%3cEdiqqdm}`XfL!k+6V26
z_Cx!lY>lFhGRw@|EC*np1JOa~U~~vN6di^RM@OI|QBgBw>&PrKaA6`>!9J&=)6nVY40I+s3!RNBp>t4CGi2+?EYotcMA#=r
z3Tc!egDi5$qZDOp6m^tYrsifj7yF!t&PNxZ3(-aBVsr_*6kUdjnju?9W|@+k<#Oy(
z8C5}7pexZ;=xTHgx)xoBvNei2$}E#}vsA@C*P|QIjp!zHGr9%cif%)Z1mzA$k-wLXA-q)D&fF6m^tY#^+{v4Er=gkE18hljte*G&Pr)aL))KO*`otvc{_Gyng
zppK{$>WsReuBaR8j*6NgTSsOYm7App_UVawq28zu>WliJ{%8Oih_W?`I?5~~bF&P>
zK7-K^G!zX(!_f#d5{*KmQBgBw>&Pr4aqZMc+%GN0A
zD6pU}_f7gW>?**Y@Ipxi9KVxQm8@8}QoC;AKh
zjs8J$lV(|ztx?obW*L~9r5yGtk9I-3q6(-Y+70cF_CR~0qGrg}ky!@hX4wn-?2Yz8
z`=b5O{^$U7AUX&gjIuR~I?62lbF&b(C3p=4QDG``nCfLARpY(Cz3BbSJtC
z-HnQxAzMdg>5-e|9_({3x)0rt9zfMlbyNe@M72=1Mo~wZrF(9c+SunoR0lnT9!8I#
zx~Lwij~bw&X2{l&S-RzBX^4FuMU7Bn)C4s}kD+Ghar6Yr)+p*IvvkeP@+9_o3O$XU
zLC>P+P;>M=dI7zNikcx?M`r1go8=|!^D=q`y^3B#ucJ56o9Hdn0%dCyb(C2;=Vob%
zecndzpm))G=za77`Vf7DK1M~&kgX%Lbjr>03HJFEwL+~?8`KuHL+w!q)DdNC6m^tY
zI_75Sgnc@rE~qQ&hPtC3s3+=$dZVId$kvfrI^<^QgMIpyyWA`ju+Ky^2~9>*&{Q-HO-D1(
zOq8ur)KO+>o10}8_L+_5pt)!snvWKsg=i64jEb5eTSsPTlbdA;_F0OSq2*`=T8UPn
z)o2Y`i?TI}I?61qbF-|&KI_p2v=MDWo6#1u6>US?QBgBw>&Ptf`L%L8Kb5wld`(zB
zzb2m>lh13(=cMHGN%FZM`8I+vWMPJWrM9hw{8ko)5|M5V`)#byu!;a-EU$yPQ|$eAXVdLv2wT
zB!5ouD)2x5yZ`t2@1NWM{(JcEzXy4JO2?HHE1Re?R>aFgL<-){JX4Ktx2^Swdyyi)kyxC6ai$tk!bD<|cWjGU%RO60^{l9p3>NlH%cCB<^OFDa4>Kqi;UC7~oQ
z7m1RbTsBIwav>?n$fc#EL@qWZX}SEAq~wBBQY@FKk|MciWwNPU#!B*XVJpeWrLH6^
z7r&B>TnHpxFjc+=aQ^kuuC#>2`?#;
zi+V{~F7qWRx$u`1%Ts}pB6)IrW3?+vd^I8Y@b2P**rrcUt}=yhbn`WT{5`c0{HBbB>SXtwUEDW
z=cdt~q2j$lsl7w#eL^MshBEtwvipZ}2ZZtmhDr|#q@sgE#fOAahlbLJg-Q+&WsV4C
zj|}CG3gwRul^zpFN5_VWj|-)a52a5Cm7EyLoD|BQ9Lk*%%AXo4JuOfYogOMaBa}Kb
zls+p|a&{}Yl#4?7I8>?vnMj9*rlH)qq5OHF((?n^=z>u3
zg`w0%q4dR}l1oCFOGDYqLb=OB`O2ZvDuG;dMX317Q0l5s`sz^0HKEM4q3m^`T-8wi
z`cUZ&fqZmhsQ9K(>gG`TmQcy9q0DWe?Cqi49ijZ4q0+korP1A?;(J1=dqe5_LM8Wy
zG7p5Z)k3-Ip?r-{X-!B@i~pYMqFThbHc>rD)OCo-Lqzj1(LO?Sb&0+nF|8jmjv5f-
zhD7x!Q8ywcjfqAsFQtFp*fu4)$B4ceF?~Fwik=|GPZHHrMEx`|d4_17CEDkRt~t>^
zPfTA3siPN(@k>PYGEu)mOkO3L*NFCYqI-kr-z27Qg-oIr#JD9Y(=(X=7jwnW#C=-U(14k2ySkr;O(s?J2+g_v|D
znr=kfo#=WHeNSTAE2N8h6XQNa)t9LI5tIHzGk|CZ65Sx8A52V#g!Iu+Vmyqfh7V~OdwkZClY7*8Ooi9|h#m`o;`DMUM!=%x|(Km7NENLm
z#_Nb`JyCBUCL4)n6VYxax-CS%m6&b|siW=0cn48^M%14ZlP`$oOQQXX=)NZUZ;0u)
zA(QAkV*EW({Xo<|5|f{Z=4Yb)h3I}I`rnA@?;%a}2QmJWsQx1Azlq5|L{p~h|2}6>
z#+D_zaztOAnC=qNM!OQ@3Pe?rsCOeKyAzH4n6@GYm1l0s+C7PGFQVU@nC=tOMf(!t
z{fKIRqCS9_97r?=5$(Z5cL>oRN=y$6>7&Dm@exFIBvBtlOpYd+V~F-xqC1Z0k0+)l
zgiNCoiSbE9buv+(LQGC2n$w8(bfP1%}sBS0f
zJBZ1hL~|F>-c5A(5dFQx^uCZLx}O+7KvdO;x;inbK{WDiQt7`1V`~vzZK8jWnAQnt
zqlbv`!$kE6QP(9V^@yfE(KaBuhD84;F>MsmMU9DZ6QXKL)Q=I9W<>Ki(LO!(^etls5LQeLsV^vx*aiT
zPc$8fwj{W(LvDB)VBdKbx4&32CFb#CRT2%_r&w#AG4SEF#*)M7MnZd<4r`hnW(oAldVLvjcB(M-43GvjF^5N
zGL60<#$OWES490aG5LmQz9ri4i0*r$|ACnP7|=w~PsI3VqWXoXe6SY#(NOeo16661r2>R_TigqR#kG=~xG;Y4=?(H}`nj|!PYM-$^?
zi0W9PK8~0iPc$bG?TJKp649SbOiu}EqEm_SX+(88QJ+Cf&Lo<%i1uuvt3>qY5Ys56
zjpV+9>?MG)u_7u>)Cn;$L}Q885uGRcl$f3y(naSH
z(GMi1gF>d!U}8LksD={tFk&*CXhsn2NTM4>^rMOCn1C*d#uDRkL^YnMClHf~L^FwK
zCllQiqMu4kr-h88>BM*jQOzXkS;S;E(aa&*xkNXQ=;ssD1tC?mkQgr_s>MXTgqSQP
znq@@0oaj~%{YqlGDx{896XP{RwU(&Y5tH>qvw>(g65S@E-%L!mgiNBX#CRJ~Z71p-
z#N;!g`J8CKAi6J!{wreobx0F^LyW&As_%&Udt&kf(fmlXKM~!}ME?sh{WYYGej~=e
z6V)F?{UuI{#`_S}zC^trG1;GJ4j|eCiS8hxKbV*v5;BbrCB}yl)!{^a1Ti_1XpSP<
zqlxYqqCb|H9v9F@(ecFi1fn{Ts81p$Clk#nM0+aHoksMh6Vo$7#?hI?_$;D2o2V-h
zlXHkBBHEbfXf7w(%0yR%=&vBASB6ZYtBCQ{M0E{OUrS7`Bbus2dp*(JK=d~f)0;w?
z=w@Pk3sK!l)VC3n+ll55qP>&o?jriTiRnEdZFDa&zK^KxC+Y`?Nj0LWPP8?Mt|rmf
zBBr%Ny68b-T!*M0BI<{UiF`b<^q-Y8b&0kf(bXsV2E?>sNFO~)j2jVEW1?A*xr2
z`ZZ$mI?=pAv~LpKTSVW2n6?ZVM{g73cZlj;qJEEZmg@?m|>uiMksx=}t5~h_)xu^&-tq8>m@1`^F6q8&_hLx_GTF&!4tM8k>k2%;KE)T4;WXrdWIv}1{G9MO*_
zrV~QiXd*G5L{yWBdI~X_N;K1mb~@3`Ao`iabXG_g%_hckh-xlT&m$)DiDm)OE+o1|
zM8BArE(z(QrNnp{Q7tFx6~tsE(X1lc)kL?3=+_d{bs^JeJu%)uR2zwU6EWFLG+T&v
xE75Hu`t8IN+WBi2{8c{Pk9RG1{{M~FtJS2M7_Z{W$oDj=UK4-a`Ahxh_&' to write it out.
+  rule 1 (metadata), x = 0..1023, numrep = 1..10
+  CRUSH rule 1 x 0 [36]
+  CRUSH rule 1 x 1 [876]
+  CRUSH rule 1 x 2 [292]
+  CRUSH rule 1 x 3 [623]
+  CRUSH rule 1 x 4 [61]
+  CRUSH rule 1 x 5 [946]
+  CRUSH rule 1 x 6 [576]
+  CRUSH rule 1 x 7 [645]
+  CRUSH rule 1 x 8 [243]
+  CRUSH rule 1 x 9 [22]
+  CRUSH rule 1 x 10 [758]
+  CRUSH rule 1 x 11 [769]
+  CRUSH rule 1 x 12 [780]
+  CRUSH rule 1 x 13 [557]
+  CRUSH rule 1 x 14 [59]
+  CRUSH rule 1 x 15 [718]
+  CRUSH rule 1 x 16 [673]
+  CRUSH rule 1 x 17 [648]
+  CRUSH rule 1 x 18 [654]
+  CRUSH rule 1 x 19 [850]
+  CRUSH rule 1 x 20 [717]
+  CRUSH rule 1 x 21 [420]
+  CRUSH rule 1 x 22 [503]
+  CRUSH rule 1 x 23 [411]
+  CRUSH rule 1 x 24 [266]
+  CRUSH rule 1 x 25 [760]
+  CRUSH rule 1 x 26 [903]
+  CRUSH rule 1 x 27 [946]
+  CRUSH rule 1 x 28 [69]
+  CRUSH rule 1 x 29 [844]
+  CRUSH rule 1 x 30 [621]
+  CRUSH rule 1 x 31 [784]
+  CRUSH rule 1 x 32 [173]
+  CRUSH rule 1 x 33 [698]
+  CRUSH rule 1 x 34 [168]
+  CRUSH rule 1 x 35 [274]
+  CRUSH rule 1 x 36 [318]
+  CRUSH rule 1 x 37 [173]
+  CRUSH rule 1 x 38 [708]
+  CRUSH rule 1 x 39 [662]
+  CRUSH rule 1 x 40 [620]
+  CRUSH rule 1 x 41 [811]
+  CRUSH rule 1 x 42 [863]
+  CRUSH rule 1 x 43 [686]
+  CRUSH rule 1 x 44 [396]
+  CRUSH rule 1 x 45 [991]
+  CRUSH rule 1 x 46 [420]
+  CRUSH rule 1 x 47 [467]
+  CRUSH rule 1 x 48 [955]
+  CRUSH rule 1 x 49 [974]
+  CRUSH rule 1 x 50 [870]
+  CRUSH rule 1 x 51 [182]
+  CRUSH rule 1 x 52 [704]
+  CRUSH rule 1 x 53 [185]
+  CRUSH rule 1 x 54 [270]
+  CRUSH rule 1 x 55 [895]
+  CRUSH rule 1 x 56 [564]
+  CRUSH rule 1 x 57 [738]
+  CRUSH rule 1 x 58 [524]
+  CRUSH rule 1 x 59 [408]
+  CRUSH rule 1 x 60 [228]
+  CRUSH rule 1 x 61 [154]
+  CRUSH rule 1 x 62 [594]
+  CRUSH rule 1 x 63 [646]
+  CRUSH rule 1 x 64 [175]
+  CRUSH rule 1 x 65 [745]
+  CRUSH rule 1 x 66 [275]
+  CRUSH rule 1 x 67 [246]
+  CRUSH rule 1 x 68 [711]
+  CRUSH rule 1 x 69 [493]
+  CRUSH rule 1 x 70 [30]
+  CRUSH rule 1 x 71 [984]
+  CRUSH rule 1 x 72 [71]
+  CRUSH rule 1 x 73 [922]
+  CRUSH rule 1 x 74 [629]
+  CRUSH rule 1 x 75 [222]
+  CRUSH rule 1 x 76 [262]
+  CRUSH rule 1 x 77 [638]
+  CRUSH rule 1 x 78 [324]
+  CRUSH rule 1 x 79 [577]
+  CRUSH rule 1 x 80 [501]
+  CRUSH rule 1 x 81 [506]
+  CRUSH rule 1 x 82 [222]
+  CRUSH rule 1 x 83 [71]
+  CRUSH rule 1 x 84 [49]
+  CRUSH rule 1 x 85 [985]
+  CRUSH rule 1 x 86 [537]
+  CRUSH rule 1 x 87 [997]
+  CRUSH rule 1 x 88 [957]
+  CRUSH rule 1 x 89 [399]
+  CRUSH rule 1 x 90 [943]
+  CRUSH rule 1 x 91 [22]
+  CRUSH rule 1 x 92 [532]
+  CRUSH rule 1 x 93 [218]
+  CRUSH rule 1 x 94 [181]
+  CRUSH rule 1 x 95 [343]
+  CRUSH rule 1 x 96 [861]
+  CRUSH rule 1 x 97 [459]
+  CRUSH rule 1 x 98 [327]
+  CRUSH rule 1 x 99 [974]
+  CRUSH rule 1 x 100 [32]
+  CRUSH rule 1 x 101 [142]
+  CRUSH rule 1 x 102 [172]
+  CRUSH rule 1 x 103 [630]
+  CRUSH rule 1 x 104 [758]
+  CRUSH rule 1 x 105 [843]
+  CRUSH rule 1 x 106 [28]
+  CRUSH rule 1 x 107 [74]
+  CRUSH rule 1 x 108 [875]
+  CRUSH rule 1 x 109 [411]
+  CRUSH rule 1 x 110 [440]
+  CRUSH rule 1 x 111 [405]
+  CRUSH rule 1 x 112 [143]
+  CRUSH rule 1 x 113 [153]
+  CRUSH rule 1 x 114 [804]
+  CRUSH rule 1 x 115 [588]
+  CRUSH rule 1 x 116 [327]
+  CRUSH rule 1 x 117 [95]
+  CRUSH rule 1 x 118 [80]
+  CRUSH rule 1 x 119 [386]
+  CRUSH rule 1 x 120 [366]
+  CRUSH rule 1 x 121 [129]
+  CRUSH rule 1 x 122 [873]
+  CRUSH rule 1 x 123 [533]
+  CRUSH rule 1 x 124 [461]
+  CRUSH rule 1 x 125 [342]
+  CRUSH rule 1 x 126 [819]
+  CRUSH rule 1 x 127 [437]
+  CRUSH rule 1 x 128 [679]
+  CRUSH rule 1 x 129 [380]
+  CRUSH rule 1 x 130 [992]
+  CRUSH rule 1 x 131 [469]
+  CRUSH rule 1 x 132 [571]
+  CRUSH rule 1 x 133 [964]
+  CRUSH rule 1 x 134 [999]
+  CRUSH rule 1 x 135 [634]
+  CRUSH rule 1 x 136 [114]
+  CRUSH rule 1 x 137 [839]
+  CRUSH rule 1 x 138 [967]
+  CRUSH rule 1 x 139 [308]
+  CRUSH rule 1 x 140 [764]
+  CRUSH rule 1 x 141 [423]
+  CRUSH rule 1 x 142 [252]
+  CRUSH rule 1 x 143 [33]
+  CRUSH rule 1 x 144 [472]
+  CRUSH rule 1 x 145 [242]
+  CRUSH rule 1 x 146 [290]
+  CRUSH rule 1 x 147 [447]
+  CRUSH rule 1 x 148 [212]
+  CRUSH rule 1 x 149 [9]
+  CRUSH rule 1 x 150 [166]
+  CRUSH rule 1 x 151 [811]
+  CRUSH rule 1 x 152 [449]
+  CRUSH rule 1 x 153 [523]
+  CRUSH rule 1 x 154 [208]
+  CRUSH rule 1 x 155 [569]
+  CRUSH rule 1 x 156 [488]
+  CRUSH rule 1 x 157 [140]
+  CRUSH rule 1 x 158 [786]
+  CRUSH rule 1 x 159 [134]
+  CRUSH rule 1 x 160 [690]
+  CRUSH rule 1 x 161 [324]
+  CRUSH rule 1 x 162 [748]
+  CRUSH rule 1 x 163 [575]
+  CRUSH rule 1 x 164 [314]
+  CRUSH rule 1 x 165 [116]
+  CRUSH rule 1 x 166 [352]
+  CRUSH rule 1 x 167 [27]
+  CRUSH rule 1 x 168 [953]
+  CRUSH rule 1 x 169 [912]
+  CRUSH rule 1 x 170 [421]
+  CRUSH rule 1 x 171 [488]
+  CRUSH rule 1 x 172 [366]
+  CRUSH rule 1 x 173 [863]
+  CRUSH rule 1 x 174 [263]
+  CRUSH rule 1 x 175 [875]
+  CRUSH rule 1 x 176 [745]
+  CRUSH rule 1 x 177 [128]
+  CRUSH rule 1 x 178 [155]
+  CRUSH rule 1 x 179 [593]
+  CRUSH rule 1 x 180 [154]
+  CRUSH rule 1 x 181 [289]
+  CRUSH rule 1 x 182 [730]
+  CRUSH rule 1 x 183 [639]
+  CRUSH rule 1 x 184 [704]
+  CRUSH rule 1 x 185 [97]
+  CRUSH rule 1 x 186 [26]
+  CRUSH rule 1 x 187 [649]
+  CRUSH rule 1 x 188 [682]
+  CRUSH rule 1 x 189 [325]
+  CRUSH rule 1 x 190 [399]
+  CRUSH rule 1 x 191 [629]
+  CRUSH rule 1 x 192 [503]
+  CRUSH rule 1 x 193 [546]
+  CRUSH rule 1 x 194 [242]
+  CRUSH rule 1 x 195 [625]
+  CRUSH rule 1 x 196 [357]
+  CRUSH rule 1 x 197 [306]
+  CRUSH rule 1 x 198 [863]
+  CRUSH rule 1 x 199 [935]
+  CRUSH rule 1 x 200 [373]
+  CRUSH rule 1 x 201 [659]
+  CRUSH rule 1 x 202 [260]
+  CRUSH rule 1 x 203 [36]
+  CRUSH rule 1 x 204 [92]
+  CRUSH rule 1 x 205 [68]
+  CRUSH rule 1 x 206 [570]
+  CRUSH rule 1 x 207 [834]
+  CRUSH rule 1 x 208 [927]
+  CRUSH rule 1 x 209 [878]
+  CRUSH rule 1 x 210 [572]
+  CRUSH rule 1 x 211 [107]
+  CRUSH rule 1 x 212 [389]
+  CRUSH rule 1 x 213 [497]
+  CRUSH rule 1 x 214 [798]
+  CRUSH rule 1 x 215 [233]
+  CRUSH rule 1 x 216 [494]
+  CRUSH rule 1 x 217 [352]
+  CRUSH rule 1 x 218 [895]
+  CRUSH rule 1 x 219 [222]
+  CRUSH rule 1 x 220 [281]
+  CRUSH rule 1 x 221 [64]
+  CRUSH rule 1 x 222 [40]
+  CRUSH rule 1 x 223 [645]
+  CRUSH rule 1 x 224 [647]
+  CRUSH rule 1 x 225 [219]
+  CRUSH rule 1 x 226 [372]
+  CRUSH rule 1 x 227 [925]
+  CRUSH rule 1 x 228 [682]
+  CRUSH rule 1 x 229 [880]
+  CRUSH rule 1 x 230 [328]
+  CRUSH rule 1 x 231 [320]
+  CRUSH rule 1 x 232 [924]
+  CRUSH rule 1 x 233 [948]
+  CRUSH rule 1 x 234 [484]
+  CRUSH rule 1 x 235 [750]
+  CRUSH rule 1 x 236 [551]
+  CRUSH rule 1 x 237 [390]
+  CRUSH rule 1 x 238 [570]
+  CRUSH rule 1 x 239 [729]
+  CRUSH rule 1 x 240 [981]
+  CRUSH rule 1 x 241 [310]
+  CRUSH rule 1 x 242 [161]
+  CRUSH rule 1 x 243 [180]
+  CRUSH rule 1 x 244 [52]
+  CRUSH rule 1 x 245 [523]
+  CRUSH rule 1 x 246 [362]
+  CRUSH rule 1 x 247 [382]
+  CRUSH rule 1 x 248 [129]
+  CRUSH rule 1 x 249 [159]
+  CRUSH rule 1 x 250 [404]
+  CRUSH rule 1 x 251 [661]
+  CRUSH rule 1 x 252 [961]
+  CRUSH rule 1 x 253 [651]
+  CRUSH rule 1 x 254 [123]
+  CRUSH rule 1 x 255 [314]
+  CRUSH rule 1 x 256 [315]
+  CRUSH rule 1 x 257 [825]
+  CRUSH rule 1 x 258 [624]
+  CRUSH rule 1 x 259 [602]
+  CRUSH rule 1 x 260 [717]
+  CRUSH rule 1 x 261 [145]
+  CRUSH rule 1 x 262 [223]
+  CRUSH rule 1 x 263 [462]
+  CRUSH rule 1 x 264 [654]
+  CRUSH rule 1 x 265 [302]
+  CRUSH rule 1 x 266 [202]
+  CRUSH rule 1 x 267 [282]
+  CRUSH rule 1 x 268 [338]
+  CRUSH rule 1 x 269 [738]
+  CRUSH rule 1 x 270 [707]
+  CRUSH rule 1 x 271 [705]
+  CRUSH rule 1 x 272 [756]
+  CRUSH rule 1 x 273 [197]
+  CRUSH rule 1 x 274 [992]
+  CRUSH rule 1 x 275 [544]
+  CRUSH rule 1 x 276 [658]
+  CRUSH rule 1 x 277 [143]
+  CRUSH rule 1 x 278 [492]
+  CRUSH rule 1 x 279 [517]
+  CRUSH rule 1 x 280 [825]
+  CRUSH rule 1 x 281 [224]
+  CRUSH rule 1 x 282 [298]
+  CRUSH rule 1 x 283 [311]
+  CRUSH rule 1 x 284 [771]
+  CRUSH rule 1 x 285 [693]
+  CRUSH rule 1 x 286 [364]
+  CRUSH rule 1 x 287 [591]
+  CRUSH rule 1 x 288 [965]
+  CRUSH rule 1 x 289 [225]
+  CRUSH rule 1 x 290 [577]
+  CRUSH rule 1 x 291 [160]
+  CRUSH rule 1 x 292 [873]
+  CRUSH rule 1 x 293 [100]
+  CRUSH rule 1 x 294 [285]
+  CRUSH rule 1 x 295 [938]
+  CRUSH rule 1 x 296 [850]
+  CRUSH rule 1 x 297 [951]
+  CRUSH rule 1 x 298 [173]
+  CRUSH rule 1 x 299 [598]
+  CRUSH rule 1 x 300 [531]
+  CRUSH rule 1 x 301 [823]
+  CRUSH rule 1 x 302 [184]
+  CRUSH rule 1 x 303 [521]
+  CRUSH rule 1 x 304 [980]
+  CRUSH rule 1 x 305 [153]
+  CRUSH rule 1 x 306 [423]
+  CRUSH rule 1 x 307 [997]
+  CRUSH rule 1 x 308 [991]
+  CRUSH rule 1 x 309 [860]
+  CRUSH rule 1 x 310 [589]
+  CRUSH rule 1 x 311 [477]
+  CRUSH rule 1 x 312 [887]
+  CRUSH rule 1 x 313 [802]
+  CRUSH rule 1 x 314 [654]
+  CRUSH rule 1 x 315 [767]
+  CRUSH rule 1 x 316 [778]
+  CRUSH rule 1 x 317 [184]
+  CRUSH rule 1 x 318 [525]
+  CRUSH rule 1 x 319 [476]
+  CRUSH rule 1 x 320 [149]
+  CRUSH rule 1 x 321 [710]
+  CRUSH rule 1 x 322 [175]
+  CRUSH rule 1 x 323 [819]
+  CRUSH rule 1 x 324 [16]
+  CRUSH rule 1 x 325 [486]
+  CRUSH rule 1 x 326 [613]
+  CRUSH rule 1 x 327 [125]
+  CRUSH rule 1 x 328 [807]
+  CRUSH rule 1 x 329 [588]
+  CRUSH rule 1 x 330 [932]
+  CRUSH rule 1 x 331 [341]
+  CRUSH rule 1 x 332 [153]
+  CRUSH rule 1 x 333 [745]
+  CRUSH rule 1 x 334 [614]
+  CRUSH rule 1 x 335 [518]
+  CRUSH rule 1 x 336 [389]
+  CRUSH rule 1 x 337 [753]
+  CRUSH rule 1 x 338 [128]
+  CRUSH rule 1 x 339 [430]
+  CRUSH rule 1 x 340 [541]
+  CRUSH rule 1 x 341 [402]
+  CRUSH rule 1 x 342 [982]
+  CRUSH rule 1 x 343 [833]
+  CRUSH rule 1 x 344 [784]
+  CRUSH rule 1 x 345 [546]
+  CRUSH rule 1 x 346 [302]
+  CRUSH rule 1 x 347 [488]
+  CRUSH rule 1 x 348 [903]
+  CRUSH rule 1 x 349 [471]
+  CRUSH rule 1 x 350 [348]
+  CRUSH rule 1 x 351 [961]
+  CRUSH rule 1 x 352 [728]
+  CRUSH rule 1 x 353 [904]
+  CRUSH rule 1 x 354 [345]
+  CRUSH rule 1 x 355 [50]
+  CRUSH rule 1 x 356 [87]
+  CRUSH rule 1 x 357 [762]
+  CRUSH rule 1 x 358 [908]
+  CRUSH rule 1 x 359 [484]
+  CRUSH rule 1 x 360 [173]
+  CRUSH rule 1 x 361 [404]
+  CRUSH rule 1 x 362 [403]
+  CRUSH rule 1 x 363 [639]
+  CRUSH rule 1 x 364 [752]
+  CRUSH rule 1 x 365 [956]
+  CRUSH rule 1 x 366 [860]
+  CRUSH rule 1 x 367 [205]
+  CRUSH rule 1 x 368 [301]
+  CRUSH rule 1 x 369 [452]
+  CRUSH rule 1 x 370 [11]
+  CRUSH rule 1 x 371 [124]
+  CRUSH rule 1 x 372 [253]
+  CRUSH rule 1 x 373 [715]
+  CRUSH rule 1 x 374 [191]
+  CRUSH rule 1 x 375 [711]
+  CRUSH rule 1 x 376 [597]
+  CRUSH rule 1 x 377 [294]
+  CRUSH rule 1 x 378 [34]
+  CRUSH rule 1 x 379 [869]
+  CRUSH rule 1 x 380 [294]
+  CRUSH rule 1 x 381 [119]
+  CRUSH rule 1 x 382 [69]
+  CRUSH rule 1 x 383 [922]
+  CRUSH rule 1 x 384 [221]
+  CRUSH rule 1 x 385 [561]
+  CRUSH rule 1 x 386 [335]
+  CRUSH rule 1 x 387 [514]
+  CRUSH rule 1 x 388 [587]
+  CRUSH rule 1 x 389 [109]
+  CRUSH rule 1 x 390 [925]
+  CRUSH rule 1 x 391 [267]
+  CRUSH rule 1 x 392 [382]
+  CRUSH rule 1 x 393 [425]
+  CRUSH rule 1 x 394 [898]
+  CRUSH rule 1 x 395 [806]
+  CRUSH rule 1 x 396 [790]
+  CRUSH rule 1 x 397 [136]
+  CRUSH rule 1 x 398 [914]
+  CRUSH rule 1 x 399 [261]
+  CRUSH rule 1 x 400 [661]
+  CRUSH rule 1 x 401 [953]
+  CRUSH rule 1 x 402 [738]
+  CRUSH rule 1 x 403 [573]
+  CRUSH rule 1 x 404 [526]
+  CRUSH rule 1 x 405 [582]
+  CRUSH rule 1 x 406 [768]
+  CRUSH rule 1 x 407 [260]
+  CRUSH rule 1 x 408 [657]
+  CRUSH rule 1 x 409 [498]
+  CRUSH rule 1 x 410 [28]
+  CRUSH rule 1 x 411 [684]
+  CRUSH rule 1 x 412 [261]
+  CRUSH rule 1 x 413 [891]
+  CRUSH rule 1 x 414 [127]
+  CRUSH rule 1 x 415 [272]
+  CRUSH rule 1 x 416 [739]
+  CRUSH rule 1 x 417 [106]
+  CRUSH rule 1 x 418 [525]
+  CRUSH rule 1 x 419 [603]
+  CRUSH rule 1 x 420 [988]
+  CRUSH rule 1 x 421 [761]
+  CRUSH rule 1 x 422 [317]
+  CRUSH rule 1 x 423 [137]
+  CRUSH rule 1 x 424 [920]
+  CRUSH rule 1 x 425 [277]
+  CRUSH rule 1 x 426 [485]
+  CRUSH rule 1 x 427 [242]
+  CRUSH rule 1 x 428 [632]
+  CRUSH rule 1 x 429 [641]
+  CRUSH rule 1 x 430 [626]
+  CRUSH rule 1 x 431 [697]
+  CRUSH rule 1 x 432 [590]
+  CRUSH rule 1 x 433 [284]
+  CRUSH rule 1 x 434 [538]
+  CRUSH rule 1 x 435 [30]
+  CRUSH rule 1 x 436 [164]
+  CRUSH rule 1 x 437 [322]
+  CRUSH rule 1 x 438 [142]
+  CRUSH rule 1 x 439 [119]
+  CRUSH rule 1 x 440 [333]
+  CRUSH rule 1 x 441 [477]
+  CRUSH rule 1 x 442 [274]
+  CRUSH rule 1 x 443 [983]
+  CRUSH rule 1 x 444 [536]
+  CRUSH rule 1 x 445 [485]
+  CRUSH rule 1 x 446 [345]
+  CRUSH rule 1 x 447 [61]
+  CRUSH rule 1 x 448 [333]
+  CRUSH rule 1 x 449 [680]
+  CRUSH rule 1 x 450 [235]
+  CRUSH rule 1 x 451 [961]
+  CRUSH rule 1 x 452 [525]
+  CRUSH rule 1 x 453 [138]
+  CRUSH rule 1 x 454 [137]
+  CRUSH rule 1 x 455 [173]
+  CRUSH rule 1 x 456 [235]
+  CRUSH rule 1 x 457 [450]
+  CRUSH rule 1 x 458 [195]
+  CRUSH rule 1 x 459 [381]
+  CRUSH rule 1 x 460 [972]
+  CRUSH rule 1 x 461 [506]
+  CRUSH rule 1 x 462 [692]
+  CRUSH rule 1 x 463 [788]
+  CRUSH rule 1 x 464 [133]
+  CRUSH rule 1 x 465 [971]
+  CRUSH rule 1 x 466 [394]
+  CRUSH rule 1 x 467 [517]
+  CRUSH rule 1 x 468 [829]
+  CRUSH rule 1 x 469 [987]
+  CRUSH rule 1 x 470 [107]
+  CRUSH rule 1 x 471 [181]
+  CRUSH rule 1 x 472 [547]
+  CRUSH rule 1 x 473 [760]
+  CRUSH rule 1 x 474 [787]
+  CRUSH rule 1 x 475 [662]
+  CRUSH rule 1 x 476 [110]
+  CRUSH rule 1 x 477 [393]
+  CRUSH rule 1 x 478 [246]
+  CRUSH rule 1 x 479 [70]
+  CRUSH rule 1 x 480 [753]
+  CRUSH rule 1 x 481 [470]
+  CRUSH rule 1 x 482 [451]
+  CRUSH rule 1 x 483 [816]
+  CRUSH rule 1 x 484 [540]
+  CRUSH rule 1 x 485 [74]
+  CRUSH rule 1 x 486 [958]
+  CRUSH rule 1 x 487 [228]
+  CRUSH rule 1 x 488 [180]
+  CRUSH rule 1 x 489 [47]
+  CRUSH rule 1 x 490 [905]
+  CRUSH rule 1 x 491 [892]
+  CRUSH rule 1 x 492 [588]
+  CRUSH rule 1 x 493 [353]
+  CRUSH rule 1 x 494 [378]
+  CRUSH rule 1 x 495 [845]
+  CRUSH rule 1 x 496 [13]
+  CRUSH rule 1 x 497 [796]
+  CRUSH rule 1 x 498 [412]
+  CRUSH rule 1 x 499 [330]
+  CRUSH rule 1 x 500 [820]
+  CRUSH rule 1 x 501 [110]
+  CRUSH rule 1 x 502 [336]
+  CRUSH rule 1 x 503 [922]
+  CRUSH rule 1 x 504 [483]
+  CRUSH rule 1 x 505 [482]
+  CRUSH rule 1 x 506 [493]
+  CRUSH rule 1 x 507 [12]
+  CRUSH rule 1 x 508 [227]
+  CRUSH rule 1 x 509 [807]
+  CRUSH rule 1 x 510 [134]
+  CRUSH rule 1 x 511 [212]
+  CRUSH rule 1 x 512 [236]
+  CRUSH rule 1 x 513 [994]
+  CRUSH rule 1 x 514 [45]
+  CRUSH rule 1 x 515 [504]
+  CRUSH rule 1 x 516 [285]
+  CRUSH rule 1 x 517 [300]
+  CRUSH rule 1 x 518 [397]
+  CRUSH rule 1 x 519 [86]
+  CRUSH rule 1 x 520 [900]
+  CRUSH rule 1 x 521 [31]
+  CRUSH rule 1 x 522 [390]
+  CRUSH rule 1 x 523 [618]
+  CRUSH rule 1 x 524 [635]
+  CRUSH rule 1 x 525 [311]
+  CRUSH rule 1 x 526 [48]
+  CRUSH rule 1 x 527 [202]
+  CRUSH rule 1 x 528 [565]
+  CRUSH rule 1 x 529 [934]
+  CRUSH rule 1 x 530 [502]
+  CRUSH rule 1 x 531 [681]
+  CRUSH rule 1 x 532 [422]
+  CRUSH rule 1 x 533 [863]
+  CRUSH rule 1 x 534 [962]
+  CRUSH rule 1 x 535 [89]
+  CRUSH rule 1 x 536 [499]
+  CRUSH rule 1 x 537 [676]
+  CRUSH rule 1 x 538 [58]
+  CRUSH rule 1 x 539 [837]
+  CRUSH rule 1 x 540 [831]
+  CRUSH rule 1 x 541 [582]
+  CRUSH rule 1 x 542 [472]
+  CRUSH rule 1 x 543 [382]
+  CRUSH rule 1 x 544 [947]
+  CRUSH rule 1 x 545 [425]
+  CRUSH rule 1 x 546 [18]
+  CRUSH rule 1 x 547 [445]
+  CRUSH rule 1 x 548 [367]
+  CRUSH rule 1 x 549 [125]
+  CRUSH rule 1 x 550 [425]
+  CRUSH rule 1 x 551 [44]
+  CRUSH rule 1 x 552 [246]
+  CRUSH rule 1 x 553 [71]
+  CRUSH rule 1 x 554 [207]
+  CRUSH rule 1 x 555 [570]
+  CRUSH rule 1 x 556 [674]
+  CRUSH rule 1 x 557 [347]
+  CRUSH rule 1 x 558 [627]
+  CRUSH rule 1 x 559 [940]
+  CRUSH rule 1 x 560 [295]
+  CRUSH rule 1 x 561 [506]
+  CRUSH rule 1 x 562 [718]
+  CRUSH rule 1 x 563 [552]
+  CRUSH rule 1 x 564 [835]
+  CRUSH rule 1 x 565 [8]
+  CRUSH rule 1 x 566 [600]
+  CRUSH rule 1 x 567 [999]
+  CRUSH rule 1 x 568 [252]
+  CRUSH rule 1 x 569 [643]
+  CRUSH rule 1 x 570 [617]
+  CRUSH rule 1 x 571 [757]
+  CRUSH rule 1 x 572 [299]
+  CRUSH rule 1 x 573 [25]
+  CRUSH rule 1 x 574 [215]
+  CRUSH rule 1 x 575 [225]
+  CRUSH rule 1 x 576 [627]
+  CRUSH rule 1 x 577 [237]
+  CRUSH rule 1 x 578 [885]
+  CRUSH rule 1 x 579 [924]
+  CRUSH rule 1 x 580 [718]
+  CRUSH rule 1 x 581 [219]
+  CRUSH rule 1 x 582 [893]
+  CRUSH rule 1 x 583 [246]
+  CRUSH rule 1 x 584 [336]
+  CRUSH rule 1 x 585 [324]
+  CRUSH rule 1 x 586 [558]
+  CRUSH rule 1 x 587 [985]
+  CRUSH rule 1 x 588 [211]
+  CRUSH rule 1 x 589 [129]
+  CRUSH rule 1 x 590 [467]
+  CRUSH rule 1 x 591 [758]
+  CRUSH rule 1 x 592 [525]
+  CRUSH rule 1 x 593 [601]
+  CRUSH rule 1 x 594 [227]
+  CRUSH rule 1 x 595 [720]
+  CRUSH rule 1 x 596 [751]
+  CRUSH rule 1 x 597 [129]
+  CRUSH rule 1 x 598 [679]
+  CRUSH rule 1 x 599 [668]
+  CRUSH rule 1 x 600 [143]
+  CRUSH rule 1 x 601 [326]
+  CRUSH rule 1 x 602 [860]
+  CRUSH rule 1 x 603 [709]
+  CRUSH rule 1 x 604 [571]
+  CRUSH rule 1 x 605 [252]
+  CRUSH rule 1 x 606 [339]
+  CRUSH rule 1 x 607 [590]
+  CRUSH rule 1 x 608 [145]
+  CRUSH rule 1 x 609 [973]
+  CRUSH rule 1 x 610 [435]
+  CRUSH rule 1 x 611 [559]
+  CRUSH rule 1 x 612 [273]
+  CRUSH rule 1 x 613 [828]
+  CRUSH rule 1 x 614 [478]
+  CRUSH rule 1 x 615 [392]
+  CRUSH rule 1 x 616 [778]
+  CRUSH rule 1 x 617 [622]
+  CRUSH rule 1 x 618 [149]
+  CRUSH rule 1 x 619 [604]
+  CRUSH rule 1 x 620 [181]
+  CRUSH rule 1 x 621 [735]
+  CRUSH rule 1 x 622 [661]
+  CRUSH rule 1 x 623 [142]
+  CRUSH rule 1 x 624 [360]
+  CRUSH rule 1 x 625 [541]
+  CRUSH rule 1 x 626 [364]
+  CRUSH rule 1 x 627 [458]
+  CRUSH rule 1 x 628 [250]
+  CRUSH rule 1 x 629 [928]
+  CRUSH rule 1 x 630 [243]
+  CRUSH rule 1 x 631 [438]
+  CRUSH rule 1 x 632 [797]
+  CRUSH rule 1 x 633 [993]
+  CRUSH rule 1 x 634 [239]
+  CRUSH rule 1 x 635 [640]
+  CRUSH rule 1 x 636 [173]
+  CRUSH rule 1 x 637 [0]
+  CRUSH rule 1 x 638 [702]
+  CRUSH rule 1 x 639 [475]
+  CRUSH rule 1 x 640 [31]
+  CRUSH rule 1 x 641 [296]
+  CRUSH rule 1 x 642 [894]
+  CRUSH rule 1 x 643 [117]
+  CRUSH rule 1 x 644 [438]
+  CRUSH rule 1 x 645 [982]
+  CRUSH rule 1 x 646 [334]
+  CRUSH rule 1 x 647 [933]
+  CRUSH rule 1 x 648 [22]
+  CRUSH rule 1 x 649 [503]
+  CRUSH rule 1 x 650 [328]
+  CRUSH rule 1 x 651 [3]
+  CRUSH rule 1 x 652 [495]
+  CRUSH rule 1 x 653 [185]
+  CRUSH rule 1 x 654 [130]
+  CRUSH rule 1 x 655 [560]
+  CRUSH rule 1 x 656 [219]
+  CRUSH rule 1 x 657 [233]
+  CRUSH rule 1 x 658 [778]
+  CRUSH rule 1 x 659 [240]
+  CRUSH rule 1 x 660 [244]
+  CRUSH rule 1 x 661 [184]
+  CRUSH rule 1 x 662 [65]
+  CRUSH rule 1 x 663 [323]
+  CRUSH rule 1 x 664 [865]
+  CRUSH rule 1 x 665 [420]
+  CRUSH rule 1 x 666 [319]
+  CRUSH rule 1 x 667 [875]
+  CRUSH rule 1 x 668 [331]
+  CRUSH rule 1 x 669 [915]
+  CRUSH rule 1 x 670 [845]
+  CRUSH rule 1 x 671 [108]
+  CRUSH rule 1 x 672 [578]
+  CRUSH rule 1 x 673 [442]
+  CRUSH rule 1 x 674 [588]
+  CRUSH rule 1 x 675 [489]
+  CRUSH rule 1 x 676 [928]
+  CRUSH rule 1 x 677 [399]
+  CRUSH rule 1 x 678 [546]
+  CRUSH rule 1 x 679 [988]
+  CRUSH rule 1 x 680 [335]
+  CRUSH rule 1 x 681 [690]
+  CRUSH rule 1 x 682 [196]
+  CRUSH rule 1 x 683 [627]
+  CRUSH rule 1 x 684 [38]
+  CRUSH rule 1 x 685 [841]
+  CRUSH rule 1 x 686 [336]
+  CRUSH rule 1 x 687 [20]
+  CRUSH rule 1 x 688 [463]
+  CRUSH rule 1 x 689 [569]
+  CRUSH rule 1 x 690 [551]
+  CRUSH rule 1 x 691 [766]
+  CRUSH rule 1 x 692 [739]
+  CRUSH rule 1 x 693 [339]
+  CRUSH rule 1 x 694 [405]
+  CRUSH rule 1 x 695 [622]
+  CRUSH rule 1 x 696 [558]
+  CRUSH rule 1 x 697 [818]
+  CRUSH rule 1 x 698 [178]
+  CRUSH rule 1 x 699 [450]
+  CRUSH rule 1 x 700 [502]
+  CRUSH rule 1 x 701 [4]
+  CRUSH rule 1 x 702 [177]
+  CRUSH rule 1 x 703 [354]
+  CRUSH rule 1 x 704 [646]
+  CRUSH rule 1 x 705 [921]
+  CRUSH rule 1 x 706 [652]
+  CRUSH rule 1 x 707 [345]
+  CRUSH rule 1 x 708 [333]
+  CRUSH rule 1 x 709 [45]
+  CRUSH rule 1 x 710 [94]
+  CRUSH rule 1 x 711 [227]
+  CRUSH rule 1 x 712 [398]
+  CRUSH rule 1 x 713 [116]
+  CRUSH rule 1 x 714 [111]
+  CRUSH rule 1 x 715 [531]
+  CRUSH rule 1 x 716 [169]
+  CRUSH rule 1 x 717 [417]
+  CRUSH rule 1 x 718 [992]
+  CRUSH rule 1 x 719 [936]
+  CRUSH rule 1 x 720 [370]
+  CRUSH rule 1 x 721 [320]
+  CRUSH rule 1 x 722 [7]
+  CRUSH rule 1 x 723 [270]
+  CRUSH rule 1 x 724 [666]
+  CRUSH rule 1 x 725 [794]
+  CRUSH rule 1 x 726 [420]
+  CRUSH rule 1 x 727 [561]
+  CRUSH rule 1 x 728 [951]
+  CRUSH rule 1 x 729 [656]
+  CRUSH rule 1 x 730 [3]
+  CRUSH rule 1 x 731 [852]
+  CRUSH rule 1 x 732 [983]
+  CRUSH rule 1 x 733 [285]
+  CRUSH rule 1 x 734 [125]
+  CRUSH rule 1 x 735 [417]
+  CRUSH rule 1 x 736 [749]
+  CRUSH rule 1 x 737 [644]
+  CRUSH rule 1 x 738 [449]
+  CRUSH rule 1 x 739 [341]
+  CRUSH rule 1 x 740 [874]
+  CRUSH rule 1 x 741 [189]
+  CRUSH rule 1 x 742 [912]
+  CRUSH rule 1 x 743 [654]
+  CRUSH rule 1 x 744 [725]
+  CRUSH rule 1 x 745 [787]
+  CRUSH rule 1 x 746 [757]
+  CRUSH rule 1 x 747 [700]
+  CRUSH rule 1 x 748 [557]
+  CRUSH rule 1 x 749 [772]
+  CRUSH rule 1 x 750 [946]
+  CRUSH rule 1 x 751 [996]
+  CRUSH rule 1 x 752 [746]
+  CRUSH rule 1 x 753 [741]
+  CRUSH rule 1 x 754 [648]
+  CRUSH rule 1 x 755 [157]
+  CRUSH rule 1 x 756 [416]
+  CRUSH rule 1 x 757 [599]
+  CRUSH rule 1 x 758 [994]
+  CRUSH rule 1 x 759 [959]
+  CRUSH rule 1 x 760 [518]
+  CRUSH rule 1 x 761 [285]
+  CRUSH rule 1 x 762 [591]
+  CRUSH rule 1 x 763 [908]
+  CRUSH rule 1 x 764 [787]
+  CRUSH rule 1 x 765 [327]
+  CRUSH rule 1 x 766 [84]
+  CRUSH rule 1 x 767 [370]
+  CRUSH rule 1 x 768 [826]
+  CRUSH rule 1 x 769 [67]
+  CRUSH rule 1 x 770 [593]
+  CRUSH rule 1 x 771 [309]
+  CRUSH rule 1 x 772 [12]
+  CRUSH rule 1 x 773 [253]
+  CRUSH rule 1 x 774 [164]
+  CRUSH rule 1 x 775 [703]
+  CRUSH rule 1 x 776 [728]
+  CRUSH rule 1 x 777 [981]
+  CRUSH rule 1 x 778 [411]
+  CRUSH rule 1 x 779 [346]
+  CRUSH rule 1 x 780 [476]
+  CRUSH rule 1 x 781 [10]
+  CRUSH rule 1 x 782 [462]
+  CRUSH rule 1 x 783 [580]
+  CRUSH rule 1 x 784 [413]
+  CRUSH rule 1 x 785 [341]
+  CRUSH rule 1 x 786 [411]
+  CRUSH rule 1 x 787 [605]
+  CRUSH rule 1 x 788 [226]
+  CRUSH rule 1 x 789 [545]
+  CRUSH rule 1 x 790 [414]
+  CRUSH rule 1 x 791 [660]
+  CRUSH rule 1 x 792 [287]
+  CRUSH rule 1 x 793 [631]
+  CRUSH rule 1 x 794 [931]
+  CRUSH rule 1 x 795 [551]
+  CRUSH rule 1 x 796 [814]
+  CRUSH rule 1 x 797 [64]
+  CRUSH rule 1 x 798 [422]
+  CRUSH rule 1 x 799 [824]
+  CRUSH rule 1 x 800 [862]
+  CRUSH rule 1 x 801 [145]
+  CRUSH rule 1 x 802 [570]
+  CRUSH rule 1 x 803 [151]
+  CRUSH rule 1 x 804 [467]
+  CRUSH rule 1 x 805 [621]
+  CRUSH rule 1 x 806 [898]
+  CRUSH rule 1 x 807 [354]
+  CRUSH rule 1 x 808 [7]
+  CRUSH rule 1 x 809 [70]
+  CRUSH rule 1 x 810 [701]
+  CRUSH rule 1 x 811 [248]
+  CRUSH rule 1 x 812 [230]
+  CRUSH rule 1 x 813 [805]
+  CRUSH rule 1 x 814 [54]
+  CRUSH rule 1 x 815 [679]
+  CRUSH rule 1 x 816 [919]
+  CRUSH rule 1 x 817 [765]
+  CRUSH rule 1 x 818 [415]
+  CRUSH rule 1 x 819 [721]
+  CRUSH rule 1 x 820 [218]
+  CRUSH rule 1 x 821 [185]
+  CRUSH rule 1 x 822 [356]
+  CRUSH rule 1 x 823 [220]
+  CRUSH rule 1 x 824 [292]
+  CRUSH rule 1 x 825 [949]
+  CRUSH rule 1 x 826 [767]
+  CRUSH rule 1 x 827 [631]
+  CRUSH rule 1 x 828 [288]
+  CRUSH rule 1 x 829 [990]
+  CRUSH rule 1 x 830 [152]
+  CRUSH rule 1 x 831 [814]
+  CRUSH rule 1 x 832 [235]
+  CRUSH rule 1 x 833 [657]
+  CRUSH rule 1 x 834 [907]
+  CRUSH rule 1 x 835 [784]
+  CRUSH rule 1 x 836 [951]
+  CRUSH rule 1 x 837 [556]
+  CRUSH rule 1 x 838 [329]
+  CRUSH rule 1 x 839 [568]
+  CRUSH rule 1 x 840 [45]
+  CRUSH rule 1 x 841 [652]
+  CRUSH rule 1 x 842 [629]
+  CRUSH rule 1 x 843 [799]
+  CRUSH rule 1 x 844 [694]
+  CRUSH rule 1 x 845 [332]
+  CRUSH rule 1 x 846 [452]
+  CRUSH rule 1 x 847 [399]
+  CRUSH rule 1 x 848 [303]
+  CRUSH rule 1 x 849 [666]
+  CRUSH rule 1 x 850 [644]
+  CRUSH rule 1 x 851 [527]
+  CRUSH rule 1 x 852 [31]
+  CRUSH rule 1 x 853 [483]
+  CRUSH rule 1 x 854 [697]
+  CRUSH rule 1 x 855 [837]
+  CRUSH rule 1 x 856 [712]
+  CRUSH rule 1 x 857 [77]
+  CRUSH rule 1 x 858 [412]
+  CRUSH rule 1 x 859 [173]
+  CRUSH rule 1 x 860 [776]
+  CRUSH rule 1 x 861 [705]
+  CRUSH rule 1 x 862 [809]
+  CRUSH rule 1 x 863 [349]
+  CRUSH rule 1 x 864 [717]
+  CRUSH rule 1 x 865 [857]
+  CRUSH rule 1 x 866 [394]
+  CRUSH rule 1 x 867 [640]
+  CRUSH rule 1 x 868 [613]
+  CRUSH rule 1 x 869 [973]
+  CRUSH rule 1 x 870 [505]
+  CRUSH rule 1 x 871 [239]
+  CRUSH rule 1 x 872 [21]
+  CRUSH rule 1 x 873 [954]
+  CRUSH rule 1 x 874 [54]
+  CRUSH rule 1 x 875 [809]
+  CRUSH rule 1 x 876 [483]
+  CRUSH rule 1 x 877 [542]
+  CRUSH rule 1 x 878 [217]
+  CRUSH rule 1 x 879 [999]
+  CRUSH rule 1 x 880 [678]
+  CRUSH rule 1 x 881 [394]
+  CRUSH rule 1 x 882 [467]
+  CRUSH rule 1 x 883 [802]
+  CRUSH rule 1 x 884 [653]
+  CRUSH rule 1 x 885 [898]
+  CRUSH rule 1 x 886 [434]
+  CRUSH rule 1 x 887 [297]
+  CRUSH rule 1 x 888 [863]
+  CRUSH rule 1 x 889 [105]
+  CRUSH rule 1 x 890 [550]
+  CRUSH rule 1 x 891 [575]
+  CRUSH rule 1 x 892 [259]
+  CRUSH rule 1 x 893 [902]
+  CRUSH rule 1 x 894 [180]
+  CRUSH rule 1 x 895 [725]
+  CRUSH rule 1 x 896 [951]
+  CRUSH rule 1 x 897 [810]
+  CRUSH rule 1 x 898 [979]
+  CRUSH rule 1 x 899 [685]
+  CRUSH rule 1 x 900 [530]
+  CRUSH rule 1 x 901 [740]
+  CRUSH rule 1 x 902 [800]
+  CRUSH rule 1 x 903 [230]
+  CRUSH rule 1 x 904 [346]
+  CRUSH rule 1 x 905 [530]
+  CRUSH rule 1 x 906 [80]
+  CRUSH rule 1 x 907 [365]
+  CRUSH rule 1 x 908 [204]
+  CRUSH rule 1 x 909 [883]
+  CRUSH rule 1 x 910 [549]
+  CRUSH rule 1 x 911 [325]
+  CRUSH rule 1 x 912 [874]
+  CRUSH rule 1 x 913 [331]
+  CRUSH rule 1 x 914 [836]
+  CRUSH rule 1 x 915 [245]
+  CRUSH rule 1 x 916 [77]
+  CRUSH rule 1 x 917 [239]
+  CRUSH rule 1 x 918 [988]
+  CRUSH rule 1 x 919 [783]
+  CRUSH rule 1 x 920 [623]
+  CRUSH rule 1 x 921 [105]
+  CRUSH rule 1 x 922 [887]
+  CRUSH rule 1 x 923 [223]
+  CRUSH rule 1 x 924 [25]
+  CRUSH rule 1 x 925 [912]
+  CRUSH rule 1 x 926 [968]
+  CRUSH rule 1 x 927 [277]
+  CRUSH rule 1 x 928 [554]
+  CRUSH rule 1 x 929 [761]
+  CRUSH rule 1 x 930 [814]
+  CRUSH rule 1 x 931 [29]
+  CRUSH rule 1 x 932 [446]
+  CRUSH rule 1 x 933 [352]
+  CRUSH rule 1 x 934 [730]
+  CRUSH rule 1 x 935 [731]
+  CRUSH rule 1 x 936 [322]
+  CRUSH rule 1 x 937 [822]
+  CRUSH rule 1 x 938 [557]
+  CRUSH rule 1 x 939 [150]
+  CRUSH rule 1 x 940 [638]
+  CRUSH rule 1 x 941 [730]
+  CRUSH rule 1 x 942 [62]
+  CRUSH rule 1 x 943 [165]
+  CRUSH rule 1 x 944 [199]
+  CRUSH rule 1 x 945 [946]
+  CRUSH rule 1 x 946 [595]
+  CRUSH rule 1 x 947 [800]
+  CRUSH rule 1 x 948 [132]
+  CRUSH rule 1 x 949 [792]
+  CRUSH rule 1 x 950 [111]
+  CRUSH rule 1 x 951 [414]
+  CRUSH rule 1 x 952 [775]
+  CRUSH rule 1 x 953 [349]
+  CRUSH rule 1 x 954 [570]
+  CRUSH rule 1 x 955 [729]
+  CRUSH rule 1 x 956 [519]
+  CRUSH rule 1 x 957 [242]
+  CRUSH rule 1 x 958 [84]
+  CRUSH rule 1 x 959 [270]
+  CRUSH rule 1 x 960 [458]
+  CRUSH rule 1 x 961 [981]
+  CRUSH rule 1 x 962 [623]
+  CRUSH rule 1 x 963 [291]
+  CRUSH rule 1 x 964 [28]
+  CRUSH rule 1 x 965 [675]
+  CRUSH rule 1 x 966 [836]
+  CRUSH rule 1 x 967 [966]
+  CRUSH rule 1 x 968 [864]
+  CRUSH rule 1 x 969 [729]
+  CRUSH rule 1 x 970 [800]
+  CRUSH rule 1 x 971 [737]
+  CRUSH rule 1 x 972 [952]
+  CRUSH rule 1 x 973 [356]
+  CRUSH rule 1 x 974 [545]
+  CRUSH rule 1 x 975 [336]
+  CRUSH rule 1 x 976 [446]
+  CRUSH rule 1 x 977 [202]
+  CRUSH rule 1 x 978 [612]
+  CRUSH rule 1 x 979 [843]
+  CRUSH rule 1 x 980 [60]
+  CRUSH rule 1 x 981 [702]
+  CRUSH rule 1 x 982 [298]
+  CRUSH rule 1 x 983 [723]
+  CRUSH rule 1 x 984 [723]
+  CRUSH rule 1 x 985 [945]
+  CRUSH rule 1 x 986 [772]
+  CRUSH rule 1 x 987 [88]
+  CRUSH rule 1 x 988 [522]
+  CRUSH rule 1 x 989 [578]
+  CRUSH rule 1 x 990 [638]
+  CRUSH rule 1 x 991 [530]
+  CRUSH rule 1 x 992 [925]
+  CRUSH rule 1 x 993 [991]
+  CRUSH rule 1 x 994 [276]
+  CRUSH rule 1 x 995 [288]
+  CRUSH rule 1 x 996 [887]
+  CRUSH rule 1 x 997 [110]
+  CRUSH rule 1 x 998 [435]
+  CRUSH rule 1 x 999 [876]
+  CRUSH rule 1 x 1000 [178]
+  CRUSH rule 1 x 1001 [99]
+  CRUSH rule 1 x 1002 [515]
+  CRUSH rule 1 x 1003 [104]
+  CRUSH rule 1 x 1004 [269]
+  CRUSH rule 1 x 1005 [369]
+  CRUSH rule 1 x 1006 [40]
+  CRUSH rule 1 x 1007 [978]
+  CRUSH rule 1 x 1008 [965]
+  CRUSH rule 1 x 1009 [598]
+  CRUSH rule 1 x 1010 [767]
+  CRUSH rule 1 x 1011 [289]
+  CRUSH rule 1 x 1012 [128]
+  CRUSH rule 1 x 1013 [979]
+  CRUSH rule 1 x 1014 [979]
+  CRUSH rule 1 x 1015 [277]
+  CRUSH rule 1 x 1016 [262]
+  CRUSH rule 1 x 1017 [150]
+  CRUSH rule 1 x 1018 [555]
+  CRUSH rule 1 x 1019 [513]
+  CRUSH rule 1 x 1020 [158]
+  CRUSH rule 1 x 1021 [915]
+  CRUSH rule 1 x 1022 [967]
+  CRUSH rule 1 x 1023 [488]
+  rule 1 (metadata) num_rep 1 result size == 1:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705]
+  CRUSH rule 1 x 1 [876,250]
+  CRUSH rule 1 x 2 [292,832]
+  CRUSH rule 1 x 3 [623,387]
+  CRUSH rule 1 x 4 [61,334]
+  CRUSH rule 1 x 5 [946,557]
+  CRUSH rule 1 x 6 [576,668]
+  CRUSH rule 1 x 7 [645,753]
+  CRUSH rule 1 x 8 [243,6]
+  CRUSH rule 1 x 9 [22,578]
+  CRUSH rule 1 x 10 [758,828]
+  CRUSH rule 1 x 11 [769,120]
+  CRUSH rule 1 x 12 [780,364]
+  CRUSH rule 1 x 13 [557,18]
+  CRUSH rule 1 x 14 [59,561]
+  CRUSH rule 1 x 15 [718,928]
+  CRUSH rule 1 x 16 [673,632]
+  CRUSH rule 1 x 17 [648,43]
+  CRUSH rule 1 x 18 [654,219]
+  CRUSH rule 1 x 19 [850,545]
+  CRUSH rule 1 x 20 [717,785]
+  CRUSH rule 1 x 21 [420,57]
+  CRUSH rule 1 x 22 [503,998]
+  CRUSH rule 1 x 23 [411,663]
+  CRUSH rule 1 x 24 [266,861]
+  CRUSH rule 1 x 25 [760,483]
+  CRUSH rule 1 x 26 [903,24]
+  CRUSH rule 1 x 27 [946,188]
+  CRUSH rule 1 x 28 [69,312]
+  CRUSH rule 1 x 29 [844,883]
+  CRUSH rule 1 x 30 [621,18]
+  CRUSH rule 1 x 31 [784,943]
+  CRUSH rule 1 x 32 [173,374]
+  CRUSH rule 1 x 33 [698,336]
+  CRUSH rule 1 x 34 [168,836]
+  CRUSH rule 1 x 35 [274,509]
+  CRUSH rule 1 x 36 [318,215]
+  CRUSH rule 1 x 37 [173,604]
+  CRUSH rule 1 x 38 [708,444]
+  CRUSH rule 1 x 39 [662,198]
+  CRUSH rule 1 x 40 [620,801]
+  CRUSH rule 1 x 41 [811,264]
+  CRUSH rule 1 x 42 [863,179]
+  CRUSH rule 1 x 43 [686,822]
+  CRUSH rule 1 x 44 [396,222]
+  CRUSH rule 1 x 45 [991,694]
+  CRUSH rule 1 x 46 [420,909]
+  CRUSH rule 1 x 47 [467,211]
+  CRUSH rule 1 x 48 [955,329]
+  CRUSH rule 1 x 49 [974,891]
+  CRUSH rule 1 x 50 [870,441]
+  CRUSH rule 1 x 51 [182,930]
+  CRUSH rule 1 x 52 [704,812]
+  CRUSH rule 1 x 53 [185,713]
+  CRUSH rule 1 x 54 [270,441]
+  CRUSH rule 1 x 55 [895,734]
+  CRUSH rule 1 x 56 [564,963]
+  CRUSH rule 1 x 57 [738,130]
+  CRUSH rule 1 x 58 [524,113]
+  CRUSH rule 1 x 59 [408,337]
+  CRUSH rule 1 x 60 [228,790]
+  CRUSH rule 1 x 61 [154,843]
+  CRUSH rule 1 x 62 [594,811]
+  CRUSH rule 1 x 63 [646,67]
+  CRUSH rule 1 x 64 [175,542]
+  CRUSH rule 1 x 65 [745,619]
+  CRUSH rule 1 x 66 [275,468]
+  CRUSH rule 1 x 67 [246,958]
+  CRUSH rule 1 x 68 [711,473]
+  CRUSH rule 1 x 69 [493,924]
+  CRUSH rule 1 x 70 [30,499]
+  CRUSH rule 1 x 71 [984,883]
+  CRUSH rule 1 x 72 [71,286]
+  CRUSH rule 1 x 73 [922,618]
+  CRUSH rule 1 x 74 [629,414]
+  CRUSH rule 1 x 75 [222,20]
+  CRUSH rule 1 x 76 [262,366]
+  CRUSH rule 1 x 77 [638,469]
+  CRUSH rule 1 x 78 [324,511]
+  CRUSH rule 1 x 79 [577,990]
+  CRUSH rule 1 x 80 [501,95]
+  CRUSH rule 1 x 81 [506,812]
+  CRUSH rule 1 x 82 [222,145]
+  CRUSH rule 1 x 83 [71,634]
+  CRUSH rule 1 x 84 [49,761]
+  CRUSH rule 1 x 85 [985,896]
+  CRUSH rule 1 x 86 [537,745]
+  CRUSH rule 1 x 87 [997,317]
+  CRUSH rule 1 x 88 [957,350]
+  CRUSH rule 1 x 89 [399,730]
+  CRUSH rule 1 x 90 [943,706]
+  CRUSH rule 1 x 91 [22,368]
+  CRUSH rule 1 x 92 [532,424]
+  CRUSH rule 1 x 93 [218,489]
+  CRUSH rule 1 x 94 [181,96]
+  CRUSH rule 1 x 95 [343,957]
+  CRUSH rule 1 x 96 [861,270]
+  CRUSH rule 1 x 97 [459,706]
+  CRUSH rule 1 x 98 [327,867]
+  CRUSH rule 1 x 99 [974,133]
+  CRUSH rule 1 x 100 [32,445]
+  CRUSH rule 1 x 101 [142,90]
+  CRUSH rule 1 x 102 [172,129]
+  CRUSH rule 1 x 103 [630,47]
+  CRUSH rule 1 x 104 [758,133]
+  CRUSH rule 1 x 105 [843,604]
+  CRUSH rule 1 x 106 [28,681]
+  CRUSH rule 1 x 107 [74,320]
+  CRUSH rule 1 x 108 [875,593]
+  CRUSH rule 1 x 109 [411,985]
+  CRUSH rule 1 x 110 [440,774]
+  CRUSH rule 1 x 111 [405,742]
+  CRUSH rule 1 x 112 [143,181]
+  CRUSH rule 1 x 113 [153,846]
+  CRUSH rule 1 x 114 [804,892]
+  CRUSH rule 1 x 115 [588,508]
+  CRUSH rule 1 x 116 [327,148]
+  CRUSH rule 1 x 117 [95,594]
+  CRUSH rule 1 x 118 [80,957]
+  CRUSH rule 1 x 119 [386,932]
+  CRUSH rule 1 x 120 [366,312]
+  CRUSH rule 1 x 121 [129,154]
+  CRUSH rule 1 x 122 [873,1]
+  CRUSH rule 1 x 123 [533,415]
+  CRUSH rule 1 x 124 [461,691]
+  CRUSH rule 1 x 125 [342,599]
+  CRUSH rule 1 x 126 [819,781]
+  CRUSH rule 1 x 127 [437,893]
+  CRUSH rule 1 x 128 [679,994]
+  CRUSH rule 1 x 129 [380,685]
+  CRUSH rule 1 x 130 [992,52]
+  CRUSH rule 1 x 131 [469,90]
+  CRUSH rule 1 x 132 [571,250]
+  CRUSH rule 1 x 133 [964,728]
+  CRUSH rule 1 x 134 [999,19]
+  CRUSH rule 1 x 135 [634,101]
+  CRUSH rule 1 x 136 [114,889]
+  CRUSH rule 1 x 137 [839,8]
+  CRUSH rule 1 x 138 [967,949]
+  CRUSH rule 1 x 139 [308,711]
+  CRUSH rule 1 x 140 [764,936]
+  CRUSH rule 1 x 141 [423,302]
+  CRUSH rule 1 x 142 [252,821]
+  CRUSH rule 1 x 143 [33,808]
+  CRUSH rule 1 x 144 [472,88]
+  CRUSH rule 1 x 145 [242,208]
+  CRUSH rule 1 x 146 [290,70]
+  CRUSH rule 1 x 147 [447,352]
+  CRUSH rule 1 x 148 [212,644]
+  CRUSH rule 1 x 149 [9,775]
+  CRUSH rule 1 x 150 [166,456]
+  CRUSH rule 1 x 151 [811,875]
+  CRUSH rule 1 x 152 [449,617]
+  CRUSH rule 1 x 153 [523,537]
+  CRUSH rule 1 x 154 [208,559]
+  CRUSH rule 1 x 155 [569,325]
+  CRUSH rule 1 x 156 [488,121]
+  CRUSH rule 1 x 157 [140,723]
+  CRUSH rule 1 x 158 [786,451]
+  CRUSH rule 1 x 159 [134,664]
+  CRUSH rule 1 x 160 [690,112]
+  CRUSH rule 1 x 161 [324,912]
+  CRUSH rule 1 x 162 [748,567]
+  CRUSH rule 1 x 163 [575,499]
+  CRUSH rule 1 x 164 [314,489]
+  CRUSH rule 1 x 165 [116,209]
+  CRUSH rule 1 x 166 [352,706]
+  CRUSH rule 1 x 167 [27,743]
+  CRUSH rule 1 x 168 [953,898]
+  CRUSH rule 1 x 169 [912,147]
+  CRUSH rule 1 x 170 [421,515]
+  CRUSH rule 1 x 171 [488,584]
+  CRUSH rule 1 x 172 [366,443]
+  CRUSH rule 1 x 173 [863,291]
+  CRUSH rule 1 x 174 [263,555]
+  CRUSH rule 1 x 175 [875,961]
+  CRUSH rule 1 x 176 [745,83]
+  CRUSH rule 1 x 177 [128,244]
+  CRUSH rule 1 x 178 [155,41]
+  CRUSH rule 1 x 179 [593,833]
+  CRUSH rule 1 x 180 [154,734]
+  CRUSH rule 1 x 181 [289,675]
+  CRUSH rule 1 x 182 [730,931]
+  CRUSH rule 1 x 183 [639,237]
+  CRUSH rule 1 x 184 [704,312]
+  CRUSH rule 1 x 185 [97,100]
+  CRUSH rule 1 x 186 [26,665]
+  CRUSH rule 1 x 187 [649,14]
+  CRUSH rule 1 x 188 [682,695]
+  CRUSH rule 1 x 189 [325,693]
+  CRUSH rule 1 x 190 [399,933]
+  CRUSH rule 1 x 191 [629,533]
+  CRUSH rule 1 x 192 [503,578]
+  CRUSH rule 1 x 193 [546,333]
+  CRUSH rule 1 x 194 [242,473]
+  CRUSH rule 1 x 195 [625,719]
+  CRUSH rule 1 x 196 [357,114]
+  CRUSH rule 1 x 197 [306,954]
+  CRUSH rule 1 x 198 [863,791]
+  CRUSH rule 1 x 199 [935,906]
+  CRUSH rule 1 x 200 [373,774]
+  CRUSH rule 1 x 201 [659,320]
+  CRUSH rule 1 x 202 [260,433]
+  CRUSH rule 1 x 203 [36,239]
+  CRUSH rule 1 x 204 [92,516]
+  CRUSH rule 1 x 205 [68,395]
+  CRUSH rule 1 x 206 [570,530]
+  CRUSH rule 1 x 207 [834,457]
+  CRUSH rule 1 x 208 [927,484]
+  CRUSH rule 1 x 209 [878,66]
+  CRUSH rule 1 x 210 [572,981]
+  CRUSH rule 1 x 211 [107,597]
+  CRUSH rule 1 x 212 [389,107]
+  CRUSH rule 1 x 213 [497,717]
+  CRUSH rule 1 x 214 [798,65]
+  CRUSH rule 1 x 215 [233,419]
+  CRUSH rule 1 x 216 [494,464]
+  CRUSH rule 1 x 217 [352,396]
+  CRUSH rule 1 x 218 [895,864]
+  CRUSH rule 1 x 219 [222,534]
+  CRUSH rule 1 x 220 [281,19]
+  CRUSH rule 1 x 221 [64,928]
+  CRUSH rule 1 x 222 [40,544]
+  CRUSH rule 1 x 223 [645,556]
+  CRUSH rule 1 x 224 [647,165]
+  CRUSH rule 1 x 225 [219,714]
+  CRUSH rule 1 x 226 [372,511]
+  CRUSH rule 1 x 227 [925,156]
+  CRUSH rule 1 x 228 [682,404]
+  CRUSH rule 1 x 229 [880,838]
+  CRUSH rule 1 x 230 [328,659]
+  CRUSH rule 1 x 231 [320,383]
+  CRUSH rule 1 x 232 [924,846]
+  CRUSH rule 1 x 233 [948,652]
+  CRUSH rule 1 x 234 [484,943]
+  CRUSH rule 1 x 235 [750,65]
+  CRUSH rule 1 x 236 [551,787]
+  CRUSH rule 1 x 237 [390,157]
+  CRUSH rule 1 x 238 [570,6]
+  CRUSH rule 1 x 239 [729,959]
+  CRUSH rule 1 x 240 [981,241]
+  CRUSH rule 1 x 241 [310,816]
+  CRUSH rule 1 x 242 [161,63]
+  CRUSH rule 1 x 243 [180,394]
+  CRUSH rule 1 x 244 [52,174]
+  CRUSH rule 1 x 245 [523,121]
+  CRUSH rule 1 x 246 [362,893]
+  CRUSH rule 1 x 247 [382,184]
+  CRUSH rule 1 x 248 [129,114]
+  CRUSH rule 1 x 249 [159,683]
+  CRUSH rule 1 x 250 [404,945]
+  CRUSH rule 1 x 251 [661,225]
+  CRUSH rule 1 x 252 [961,226]
+  CRUSH rule 1 x 253 [651,97]
+  CRUSH rule 1 x 254 [123,33]
+  CRUSH rule 1 x 255 [314,649]
+  CRUSH rule 1 x 256 [315,215]
+  CRUSH rule 1 x 257 [825,264]
+  CRUSH rule 1 x 258 [624,789]
+  CRUSH rule 1 x 259 [602,542]
+  CRUSH rule 1 x 260 [717,878]
+  CRUSH rule 1 x 261 [145,517]
+  CRUSH rule 1 x 262 [223,1]
+  CRUSH rule 1 x 263 [462,211]
+  CRUSH rule 1 x 264 [654,471]
+  CRUSH rule 1 x 265 [302,794]
+  CRUSH rule 1 x 266 [202,132]
+  CRUSH rule 1 x 267 [282,938]
+  CRUSH rule 1 x 268 [338,309]
+  CRUSH rule 1 x 269 [738,122]
+  CRUSH rule 1 x 270 [707,982]
+  CRUSH rule 1 x 271 [705,432]
+  CRUSH rule 1 x 272 [756,545]
+  CRUSH rule 1 x 273 [197,502]
+  CRUSH rule 1 x 274 [992,44]
+  CRUSH rule 1 x 275 [544,789]
+  CRUSH rule 1 x 276 [658,467]
+  CRUSH rule 1 x 277 [143,490]
+  CRUSH rule 1 x 278 [492,647]
+  CRUSH rule 1 x 279 [517,792]
+  CRUSH rule 1 x 280 [825,740]
+  CRUSH rule 1 x 281 [224,629]
+  CRUSH rule 1 x 282 [298,661]
+  CRUSH rule 1 x 283 [311,606]
+  CRUSH rule 1 x 284 [771,466]
+  CRUSH rule 1 x 285 [693,362]
+  CRUSH rule 1 x 286 [364,477]
+  CRUSH rule 1 x 287 [591,611]
+  CRUSH rule 1 x 288 [965,541]
+  CRUSH rule 1 x 289 [225,551]
+  CRUSH rule 1 x 290 [577,762]
+  CRUSH rule 1 x 291 [160,903]
+  CRUSH rule 1 x 292 [873,598]
+  CRUSH rule 1 x 293 [100,234]
+  CRUSH rule 1 x 294 [285,943]
+  CRUSH rule 1 x 295 [938,262]
+  CRUSH rule 1 x 296 [850,327]
+  CRUSH rule 1 x 297 [951,53]
+  CRUSH rule 1 x 298 [173,336]
+  CRUSH rule 1 x 299 [598,591]
+  CRUSH rule 1 x 300 [531,957]
+  CRUSH rule 1 x 301 [823,628]
+  CRUSH rule 1 x 302 [184,80]
+  CRUSH rule 1 x 303 [521,766]
+  CRUSH rule 1 x 304 [980,127]
+  CRUSH rule 1 x 305 [153,816]
+  CRUSH rule 1 x 306 [423,739]
+  CRUSH rule 1 x 307 [997,557]
+  CRUSH rule 1 x 308 [991,874]
+  CRUSH rule 1 x 309 [860,394]
+  CRUSH rule 1 x 310 [589,818]
+  CRUSH rule 1 x 311 [477,774]
+  CRUSH rule 1 x 312 [887,853]
+  CRUSH rule 1 x 313 [802,646]
+  CRUSH rule 1 x 314 [654,974]
+  CRUSH rule 1 x 315 [767,227]
+  CRUSH rule 1 x 316 [778,83]
+  CRUSH rule 1 x 317 [184,418]
+  CRUSH rule 1 x 318 [525,410]
+  CRUSH rule 1 x 319 [476,724]
+  CRUSH rule 1 x 320 [149,610]
+  CRUSH rule 1 x 321 [710,79]
+  CRUSH rule 1 x 322 [175,275]
+  CRUSH rule 1 x 323 [819,604]
+  CRUSH rule 1 x 324 [16,745]
+  CRUSH rule 1 x 325 [486,400]
+  CRUSH rule 1 x 326 [613,765]
+  CRUSH rule 1 x 327 [125,289]
+  CRUSH rule 1 x 328 [807,383]
+  CRUSH rule 1 x 329 [588,938]
+  CRUSH rule 1 x 330 [932,644]
+  CRUSH rule 1 x 331 [341,953]
+  CRUSH rule 1 x 332 [153,726]
+  CRUSH rule 1 x 333 [745,845]
+  CRUSH rule 1 x 334 [614,751]
+  CRUSH rule 1 x 335 [518,721]
+  CRUSH rule 1 x 336 [389,424]
+  CRUSH rule 1 x 337 [753,508]
+  CRUSH rule 1 x 338 [128,810]
+  CRUSH rule 1 x 339 [430,308]
+  CRUSH rule 1 x 340 [541,44]
+  CRUSH rule 1 x 341 [402,26]
+  CRUSH rule 1 x 342 [982,57]
+  CRUSH rule 1 x 343 [833,412]
+  CRUSH rule 1 x 344 [784,533]
+  CRUSH rule 1 x 345 [546,300]
+  CRUSH rule 1 x 346 [302,420]
+  CRUSH rule 1 x 347 [488,778]
+  CRUSH rule 1 x 348 [903,744]
+  CRUSH rule 1 x 349 [471,547]
+  CRUSH rule 1 x 350 [348,221]
+  CRUSH rule 1 x 351 [961,582]
+  CRUSH rule 1 x 352 [728,137]
+  CRUSH rule 1 x 353 [904,202]
+  CRUSH rule 1 x 354 [345,226]
+  CRUSH rule 1 x 355 [50,430]
+  CRUSH rule 1 x 356 [87,185]
+  CRUSH rule 1 x 357 [762,459]
+  CRUSH rule 1 x 358 [908,25]
+  CRUSH rule 1 x 359 [484,15]
+  CRUSH rule 1 x 360 [173,378]
+  CRUSH rule 1 x 361 [404,577]
+  CRUSH rule 1 x 362 [403,1]
+  CRUSH rule 1 x 363 [639,911]
+  CRUSH rule 1 x 364 [752,689]
+  CRUSH rule 1 x 365 [956,999]
+  CRUSH rule 1 x 366 [860,925]
+  CRUSH rule 1 x 367 [205,609]
+  CRUSH rule 1 x 368 [301,284]
+  CRUSH rule 1 x 369 [452,658]
+  CRUSH rule 1 x 370 [11,467]
+  CRUSH rule 1 x 371 [124,487]
+  CRUSH rule 1 x 372 [253,48]
+  CRUSH rule 1 x 373 [715,605]
+  CRUSH rule 1 x 374 [191,887]
+  CRUSH rule 1 x 375 [711,385]
+  CRUSH rule 1 x 376 [597,818]
+  CRUSH rule 1 x 377 [294,256]
+  CRUSH rule 1 x 378 [34,151]
+  CRUSH rule 1 x 379 [869,136]
+  CRUSH rule 1 x 380 [294,97]
+  CRUSH rule 1 x 381 [119,710]
+  CRUSH rule 1 x 382 [69,631]
+  CRUSH rule 1 x 383 [922,588]
+  CRUSH rule 1 x 384 [221,945]
+  CRUSH rule 1 x 385 [561,737]
+  CRUSH rule 1 x 386 [335,442]
+  CRUSH rule 1 x 387 [514,43]
+  CRUSH rule 1 x 388 [587,89]
+  CRUSH rule 1 x 389 [109,641]
+  CRUSH rule 1 x 390 [925,149]
+  CRUSH rule 1 x 391 [267,87]
+  CRUSH rule 1 x 392 [382,485]
+  CRUSH rule 1 x 393 [425,721]
+  CRUSH rule 1 x 394 [898,18]
+  CRUSH rule 1 x 395 [806,876]
+  CRUSH rule 1 x 396 [790,970]
+  CRUSH rule 1 x 397 [136,363]
+  CRUSH rule 1 x 398 [914,116]
+  CRUSH rule 1 x 399 [261,94]
+  CRUSH rule 1 x 400 [661,197]
+  CRUSH rule 1 x 401 [953,979]
+  CRUSH rule 1 x 402 [738,819]
+  CRUSH rule 1 x 403 [573,238]
+  CRUSH rule 1 x 404 [526,848]
+  CRUSH rule 1 x 405 [582,505]
+  CRUSH rule 1 x 406 [768,324]
+  CRUSH rule 1 x 407 [260,951]
+  CRUSH rule 1 x 408 [657,81]
+  CRUSH rule 1 x 409 [498,89]
+  CRUSH rule 1 x 410 [28,793]
+  CRUSH rule 1 x 411 [684,992]
+  CRUSH rule 1 x 412 [261,958]
+  CRUSH rule 1 x 413 [891,835]
+  CRUSH rule 1 x 414 [127,459]
+  CRUSH rule 1 x 415 [272,540]
+  CRUSH rule 1 x 416 [739,617]
+  CRUSH rule 1 x 417 [106,209]
+  CRUSH rule 1 x 418 [525,441]
+  CRUSH rule 1 x 419 [603,673]
+  CRUSH rule 1 x 420 [988,213]
+  CRUSH rule 1 x 421 [761,521]
+  CRUSH rule 1 x 422 [317,160]
+  CRUSH rule 1 x 423 [137,807]
+  CRUSH rule 1 x 424 [920,37]
+  CRUSH rule 1 x 425 [277,693]
+  CRUSH rule 1 x 426 [485,936]
+  CRUSH rule 1 x 427 [242,515]
+  CRUSH rule 1 x 428 [632,635]
+  CRUSH rule 1 x 429 [641,73]
+  CRUSH rule 1 x 430 [626,585]
+  CRUSH rule 1 x 431 [697,76]
+  CRUSH rule 1 x 432 [590,526]
+  CRUSH rule 1 x 433 [284,387]
+  CRUSH rule 1 x 434 [538,985]
+  CRUSH rule 1 x 435 [30,318]
+  CRUSH rule 1 x 436 [164,919]
+  CRUSH rule 1 x 437 [322,212]
+  CRUSH rule 1 x 438 [142,392]
+  CRUSH rule 1 x 439 [119,370]
+  CRUSH rule 1 x 440 [333,403]
+  CRUSH rule 1 x 441 [477,727]
+  CRUSH rule 1 x 442 [274,590]
+  CRUSH rule 1 x 443 [983,748]
+  CRUSH rule 1 x 444 [536,509]
+  CRUSH rule 1 x 445 [485,209]
+  CRUSH rule 1 x 446 [345,634]
+  CRUSH rule 1 x 447 [61,845]
+  CRUSH rule 1 x 448 [333,232]
+  CRUSH rule 1 x 449 [680,16]
+  CRUSH rule 1 x 450 [235,214]
+  CRUSH rule 1 x 451 [961,468]
+  CRUSH rule 1 x 452 [525,479]
+  CRUSH rule 1 x 453 [138,466]
+  CRUSH rule 1 x 454 [137,625]
+  CRUSH rule 1 x 455 [173,150]
+  CRUSH rule 1 x 456 [235,226]
+  CRUSH rule 1 x 457 [450,577]
+  CRUSH rule 1 x 458 [195,537]
+  CRUSH rule 1 x 459 [381,555]
+  CRUSH rule 1 x 460 [972,730]
+  CRUSH rule 1 x 461 [506,279]
+  CRUSH rule 1 x 462 [692,959]
+  CRUSH rule 1 x 463 [788,667]
+  CRUSH rule 1 x 464 [133,122]
+  CRUSH rule 1 x 465 [971,190]
+  CRUSH rule 1 x 466 [394,576]
+  CRUSH rule 1 x 467 [517,28]
+  CRUSH rule 1 x 468 [829,143]
+  CRUSH rule 1 x 469 [987,936]
+  CRUSH rule 1 x 470 [107,982]
+  CRUSH rule 1 x 471 [181,897]
+  CRUSH rule 1 x 472 [547,512]
+  CRUSH rule 1 x 473 [760,997]
+  CRUSH rule 1 x 474 [787,418]
+  CRUSH rule 1 x 475 [662,312]
+  CRUSH rule 1 x 476 [110,495]
+  CRUSH rule 1 x 477 [393,954]
+  CRUSH rule 1 x 478 [246,483]
+  CRUSH rule 1 x 479 [70,929]
+  CRUSH rule 1 x 480 [753,119]
+  CRUSH rule 1 x 481 [470,429]
+  CRUSH rule 1 x 482 [451,566]
+  CRUSH rule 1 x 483 [816,72]
+  CRUSH rule 1 x 484 [540,454]
+  CRUSH rule 1 x 485 [74,582]
+  CRUSH rule 1 x 486 [958,595]
+  CRUSH rule 1 x 487 [228,302]
+  CRUSH rule 1 x 488 [180,529]
+  CRUSH rule 1 x 489 [47,617]
+  CRUSH rule 1 x 490 [905,822]
+  CRUSH rule 1 x 491 [892,370]
+  CRUSH rule 1 x 492 [588,959]
+  CRUSH rule 1 x 493 [353,461]
+  CRUSH rule 1 x 494 [378,848]
+  CRUSH rule 1 x 495 [845,653]
+  CRUSH rule 1 x 496 [13,988]
+  CRUSH rule 1 x 497 [796,877]
+  CRUSH rule 1 x 498 [412,337]
+  CRUSH rule 1 x 499 [330,695]
+  CRUSH rule 1 x 500 [820,272]
+  CRUSH rule 1 x 501 [110,44]
+  CRUSH rule 1 x 502 [336,595]
+  CRUSH rule 1 x 503 [922,211]
+  CRUSH rule 1 x 504 [483,52]
+  CRUSH rule 1 x 505 [482,598]
+  CRUSH rule 1 x 506 [493,123]
+  CRUSH rule 1 x 507 [12,598]
+  CRUSH rule 1 x 508 [227,157]
+  CRUSH rule 1 x 509 [807,242]
+  CRUSH rule 1 x 510 [134,437]
+  CRUSH rule 1 x 511 [212,54]
+  CRUSH rule 1 x 512 [236,630]
+  CRUSH rule 1 x 513 [994,693]
+  CRUSH rule 1 x 514 [45,508]
+  CRUSH rule 1 x 515 [504,138]
+  CRUSH rule 1 x 516 [285,409]
+  CRUSH rule 1 x 517 [300,232]
+  CRUSH rule 1 x 518 [397,674]
+  CRUSH rule 1 x 519 [86,750]
+  CRUSH rule 1 x 520 [900,833]
+  CRUSH rule 1 x 521 [31,47]
+  CRUSH rule 1 x 522 [390,16]
+  CRUSH rule 1 x 523 [618,308]
+  CRUSH rule 1 x 524 [635,189]
+  CRUSH rule 1 x 525 [311,916]
+  CRUSH rule 1 x 526 [48,738]
+  CRUSH rule 1 x 527 [202,851]
+  CRUSH rule 1 x 528 [565,827]
+  CRUSH rule 1 x 529 [934,864]
+  CRUSH rule 1 x 530 [502,934]
+  CRUSH rule 1 x 531 [681,627]
+  CRUSH rule 1 x 532 [422,6]
+  CRUSH rule 1 x 533 [863,68]
+  CRUSH rule 1 x 534 [962,931]
+  CRUSH rule 1 x 535 [89,565]
+  CRUSH rule 1 x 536 [499,351]
+  CRUSH rule 1 x 537 [676,547]
+  CRUSH rule 1 x 538 [58,644]
+  CRUSH rule 1 x 539 [837,953]
+  CRUSH rule 1 x 540 [831,50]
+  CRUSH rule 1 x 541 [582,757]
+  CRUSH rule 1 x 542 [472,132]
+  CRUSH rule 1 x 543 [382,272]
+  CRUSH rule 1 x 544 [947,930]
+  CRUSH rule 1 x 545 [425,570]
+  CRUSH rule 1 x 546 [18,65]
+  CRUSH rule 1 x 547 [445,715]
+  CRUSH rule 1 x 548 [367,569]
+  CRUSH rule 1 x 549 [125,715]
+  CRUSH rule 1 x 550 [425,599]
+  CRUSH rule 1 x 551 [44,1]
+  CRUSH rule 1 x 552 [246,104]
+  CRUSH rule 1 x 553 [71,703]
+  CRUSH rule 1 x 554 [207,124]
+  CRUSH rule 1 x 555 [570,28]
+  CRUSH rule 1 x 556 [674,152]
+  CRUSH rule 1 x 557 [347,817]
+  CRUSH rule 1 x 558 [627,426]
+  CRUSH rule 1 x 559 [940,630]
+  CRUSH rule 1 x 560 [295,903]
+  CRUSH rule 1 x 561 [506,682]
+  CRUSH rule 1 x 562 [718,529]
+  CRUSH rule 1 x 563 [552,332]
+  CRUSH rule 1 x 564 [835,769]
+  CRUSH rule 1 x 565 [8,167]
+  CRUSH rule 1 x 566 [600,481]
+  CRUSH rule 1 x 567 [999,994]
+  CRUSH rule 1 x 568 [252,431]
+  CRUSH rule 1 x 569 [643,218]
+  CRUSH rule 1 x 570 [617,635]
+  CRUSH rule 1 x 571 [757,80]
+  CRUSH rule 1 x 572 [299,348]
+  CRUSH rule 1 x 573 [25,505]
+  CRUSH rule 1 x 574 [215,431]
+  CRUSH rule 1 x 575 [225,252]
+  CRUSH rule 1 x 576 [627,94]
+  CRUSH rule 1 x 577 [237,809]
+  CRUSH rule 1 x 578 [885,313]
+  CRUSH rule 1 x 579 [924,575]
+  CRUSH rule 1 x 580 [718,51]
+  CRUSH rule 1 x 581 [219,807]
+  CRUSH rule 1 x 582 [893,701]
+  CRUSH rule 1 x 583 [246,930]
+  CRUSH rule 1 x 584 [336,432]
+  CRUSH rule 1 x 585 [324,999]
+  CRUSH rule 1 x 586 [558,230]
+  CRUSH rule 1 x 587 [985,830]
+  CRUSH rule 1 x 588 [211,544]
+  CRUSH rule 1 x 589 [129,21]
+  CRUSH rule 1 x 590 [467,969]
+  CRUSH rule 1 x 591 [758,514]
+  CRUSH rule 1 x 592 [525,253]
+  CRUSH rule 1 x 593 [601,885]
+  CRUSH rule 1 x 594 [227,60]
+  CRUSH rule 1 x 595 [720,854]
+  CRUSH rule 1 x 596 [751,195]
+  CRUSH rule 1 x 597 [129,574]
+  CRUSH rule 1 x 598 [679,207]
+  CRUSH rule 1 x 599 [668,315]
+  CRUSH rule 1 x 600 [143,396]
+  CRUSH rule 1 x 601 [326,573]
+  CRUSH rule 1 x 602 [860,281]
+  CRUSH rule 1 x 603 [709,328]
+  CRUSH rule 1 x 604 [571,62]
+  CRUSH rule 1 x 605 [252,739]
+  CRUSH rule 1 x 606 [339,236]
+  CRUSH rule 1 x 607 [590,248]
+  CRUSH rule 1 x 608 [145,635]
+  CRUSH rule 1 x 609 [973,547]
+  CRUSH rule 1 x 610 [435,816]
+  CRUSH rule 1 x 611 [559,283]
+  CRUSH rule 1 x 612 [273,149]
+  CRUSH rule 1 x 613 [828,614]
+  CRUSH rule 1 x 614 [478,748]
+  CRUSH rule 1 x 615 [392,155]
+  CRUSH rule 1 x 616 [778,637]
+  CRUSH rule 1 x 617 [622,713]
+  CRUSH rule 1 x 618 [149,877]
+  CRUSH rule 1 x 619 [604,163]
+  CRUSH rule 1 x 620 [181,23]
+  CRUSH rule 1 x 621 [735,902]
+  CRUSH rule 1 x 622 [661,824]
+  CRUSH rule 1 x 623 [142,121]
+  CRUSH rule 1 x 624 [360,716]
+  CRUSH rule 1 x 625 [541,167]
+  CRUSH rule 1 x 626 [364,431]
+  CRUSH rule 1 x 627 [458,137]
+  CRUSH rule 1 x 628 [250,350]
+  CRUSH rule 1 x 629 [928,160]
+  CRUSH rule 1 x 630 [243,19]
+  CRUSH rule 1 x 631 [438,221]
+  CRUSH rule 1 x 632 [797,368]
+  CRUSH rule 1 x 633 [993,749]
+  CRUSH rule 1 x 634 [239,351]
+  CRUSH rule 1 x 635 [640,965]
+  CRUSH rule 1 x 636 [173,290]
+  CRUSH rule 1 x 637 [0,918]
+  CRUSH rule 1 x 638 [702,235]
+  CRUSH rule 1 x 639 [475,687]
+  CRUSH rule 1 x 640 [31,664]
+  CRUSH rule 1 x 641 [296,473]
+  CRUSH rule 1 x 642 [894,273]
+  CRUSH rule 1 x 643 [117,111]
+  CRUSH rule 1 x 644 [438,336]
+  CRUSH rule 1 x 645 [982,702]
+  CRUSH rule 1 x 646 [334,804]
+  CRUSH rule 1 x 647 [933,787]
+  CRUSH rule 1 x 648 [22,444]
+  CRUSH rule 1 x 649 [503,229]
+  CRUSH rule 1 x 650 [328,659]
+  CRUSH rule 1 x 651 [3,880]
+  CRUSH rule 1 x 652 [495,977]
+  CRUSH rule 1 x 653 [185,718]
+  CRUSH rule 1 x 654 [130,528]
+  CRUSH rule 1 x 655 [560,872]
+  CRUSH rule 1 x 656 [219,885]
+  CRUSH rule 1 x 657 [233,684]
+  CRUSH rule 1 x 658 [778,6]
+  CRUSH rule 1 x 659 [240,663]
+  CRUSH rule 1 x 660 [244,855]
+  CRUSH rule 1 x 661 [184,270]
+  CRUSH rule 1 x 662 [65,883]
+  CRUSH rule 1 x 663 [323,721]
+  CRUSH rule 1 x 664 [865,113]
+  CRUSH rule 1 x 665 [420,850]
+  CRUSH rule 1 x 666 [319,767]
+  CRUSH rule 1 x 667 [875,39]
+  CRUSH rule 1 x 668 [331,122]
+  CRUSH rule 1 x 669 [915,521]
+  CRUSH rule 1 x 670 [845,659]
+  CRUSH rule 1 x 671 [108,634]
+  CRUSH rule 1 x 672 [578,216]
+  CRUSH rule 1 x 673 [442,74]
+  CRUSH rule 1 x 674 [588,364]
+  CRUSH rule 1 x 675 [489,698]
+  CRUSH rule 1 x 676 [928,911]
+  CRUSH rule 1 x 677 [399,269]
+  CRUSH rule 1 x 678 [546,752]
+  CRUSH rule 1 x 679 [988,25]
+  CRUSH rule 1 x 680 [335,963]
+  CRUSH rule 1 x 681 [690,462]
+  CRUSH rule 1 x 682 [196,588]
+  CRUSH rule 1 x 683 [627,25]
+  CRUSH rule 1 x 684 [38,804]
+  CRUSH rule 1 x 685 [841,368]
+  CRUSH rule 1 x 686 [336,287]
+  CRUSH rule 1 x 687 [20,682]
+  CRUSH rule 1 x 688 [463,371]
+  CRUSH rule 1 x 689 [569,250]
+  CRUSH rule 1 x 690 [551,144]
+  CRUSH rule 1 x 691 [766,464]
+  CRUSH rule 1 x 692 [739,634]
+  CRUSH rule 1 x 693 [339,297]
+  CRUSH rule 1 x 694 [405,26]
+  CRUSH rule 1 x 695 [622,576]
+  CRUSH rule 1 x 696 [558,902]
+  CRUSH rule 1 x 697 [818,222]
+  CRUSH rule 1 x 698 [178,48]
+  CRUSH rule 1 x 699 [450,244]
+  CRUSH rule 1 x 700 [502,771]
+  CRUSH rule 1 x 701 [4,612]
+  CRUSH rule 1 x 702 [177,630]
+  CRUSH rule 1 x 703 [354,178]
+  CRUSH rule 1 x 704 [646,601]
+  CRUSH rule 1 x 705 [921,401]
+  CRUSH rule 1 x 706 [652,877]
+  CRUSH rule 1 x 707 [345,745]
+  CRUSH rule 1 x 708 [333,607]
+  CRUSH rule 1 x 709 [45,187]
+  CRUSH rule 1 x 710 [94,855]
+  CRUSH rule 1 x 711 [227,653]
+  CRUSH rule 1 x 712 [398,953]
+  CRUSH rule 1 x 713 [116,800]
+  CRUSH rule 1 x 714 [111,629]
+  CRUSH rule 1 x 715 [531,291]
+  CRUSH rule 1 x 716 [169,541]
+  CRUSH rule 1 x 717 [417,446]
+  CRUSH rule 1 x 718 [992,383]
+  CRUSH rule 1 x 719 [936,674]
+  CRUSH rule 1 x 720 [370,188]
+  CRUSH rule 1 x 721 [320,859]
+  CRUSH rule 1 x 722 [7,2]
+  CRUSH rule 1 x 723 [270,553]
+  CRUSH rule 1 x 724 [666,822]
+  CRUSH rule 1 x 725 [794,406]
+  CRUSH rule 1 x 726 [420,556]
+  CRUSH rule 1 x 727 [561,461]
+  CRUSH rule 1 x 728 [951,330]
+  CRUSH rule 1 x 729 [656,644]
+  CRUSH rule 1 x 730 [3,558]
+  CRUSH rule 1 x 731 [852,89]
+  CRUSH rule 1 x 732 [983,840]
+  CRUSH rule 1 x 733 [285,396]
+  CRUSH rule 1 x 734 [125,510]
+  CRUSH rule 1 x 735 [417,773]
+  CRUSH rule 1 x 736 [749,396]
+  CRUSH rule 1 x 737 [644,991]
+  CRUSH rule 1 x 738 [449,683]
+  CRUSH rule 1 x 739 [341,220]
+  CRUSH rule 1 x 740 [874,524]
+  CRUSH rule 1 x 741 [189,472]
+  CRUSH rule 1 x 742 [912,581]
+  CRUSH rule 1 x 743 [654,914]
+  CRUSH rule 1 x 744 [725,295]
+  CRUSH rule 1 x 745 [787,858]
+  CRUSH rule 1 x 746 [757,848]
+  CRUSH rule 1 x 747 [700,81]
+  CRUSH rule 1 x 748 [557,436]
+  CRUSH rule 1 x 749 [772,622]
+  CRUSH rule 1 x 750 [946,97]
+  CRUSH rule 1 x 751 [996,618]
+  CRUSH rule 1 x 752 [746,887]
+  CRUSH rule 1 x 753 [741,14]
+  CRUSH rule 1 x 754 [648,349]
+  CRUSH rule 1 x 755 [157,460]
+  CRUSH rule 1 x 756 [416,97]
+  CRUSH rule 1 x 757 [599,839]
+  CRUSH rule 1 x 758 [994,218]
+  CRUSH rule 1 x 759 [959,682]
+  CRUSH rule 1 x 760 [518,943]
+  CRUSH rule 1 x 761 [285,849]
+  CRUSH rule 1 x 762 [591,313]
+  CRUSH rule 1 x 763 [908,411]
+  CRUSH rule 1 x 764 [787,234]
+  CRUSH rule 1 x 765 [327,921]
+  CRUSH rule 1 x 766 [84,161]
+  CRUSH rule 1 x 767 [370,895]
+  CRUSH rule 1 x 768 [826,760]
+  CRUSH rule 1 x 769 [67,768]
+  CRUSH rule 1 x 770 [593,909]
+  CRUSH rule 1 x 771 [309,935]
+  CRUSH rule 1 x 772 [12,125]
+  CRUSH rule 1 x 773 [253,466]
+  CRUSH rule 1 x 774 [164,390]
+  CRUSH rule 1 x 775 [703,47]
+  CRUSH rule 1 x 776 [728,231]
+  CRUSH rule 1 x 777 [981,621]
+  CRUSH rule 1 x 778 [411,456]
+  CRUSH rule 1 x 779 [346,121]
+  CRUSH rule 1 x 780 [476,39]
+  CRUSH rule 1 x 781 [10,130]
+  CRUSH rule 1 x 782 [462,246]
+  CRUSH rule 1 x 783 [580,373]
+  CRUSH rule 1 x 784 [413,113]
+  CRUSH rule 1 x 785 [341,856]
+  CRUSH rule 1 x 786 [411,140]
+  CRUSH rule 1 x 787 [605,522]
+  CRUSH rule 1 x 788 [226,545]
+  CRUSH rule 1 x 789 [545,320]
+  CRUSH rule 1 x 790 [414,748]
+  CRUSH rule 1 x 791 [660,906]
+  CRUSH rule 1 x 792 [287,392]
+  CRUSH rule 1 x 793 [631,133]
+  CRUSH rule 1 x 794 [931,517]
+  CRUSH rule 1 x 795 [551,962]
+  CRUSH rule 1 x 796 [814,4]
+  CRUSH rule 1 x 797 [64,201]
+  CRUSH rule 1 x 798 [422,530]
+  CRUSH rule 1 x 799 [824,32]
+  CRUSH rule 1 x 800 [862,623]
+  CRUSH rule 1 x 801 [145,550]
+  CRUSH rule 1 x 802 [570,19]
+  CRUSH rule 1 x 803 [151,812]
+  CRUSH rule 1 x 804 [467,93]
+  CRUSH rule 1 x 805 [621,223]
+  CRUSH rule 1 x 806 [898,957]
+  CRUSH rule 1 x 807 [354,531]
+  CRUSH rule 1 x 808 [7,96]
+  CRUSH rule 1 x 809 [70,734]
+  CRUSH rule 1 x 810 [701,18]
+  CRUSH rule 1 x 811 [248,547]
+  CRUSH rule 1 x 812 [230,576]
+  CRUSH rule 1 x 813 [805,114]
+  CRUSH rule 1 x 814 [54,619]
+  CRUSH rule 1 x 815 [679,412]
+  CRUSH rule 1 x 816 [919,448]
+  CRUSH rule 1 x 817 [765,830]
+  CRUSH rule 1 x 818 [415,566]
+  CRUSH rule 1 x 819 [721,319]
+  CRUSH rule 1 x 820 [218,301]
+  CRUSH rule 1 x 821 [185,795]
+  CRUSH rule 1 x 822 [356,261]
+  CRUSH rule 1 x 823 [220,281]
+  CRUSH rule 1 x 824 [292,809]
+  CRUSH rule 1 x 825 [949,778]
+  CRUSH rule 1 x 826 [767,818]
+  CRUSH rule 1 x 827 [631,83]
+  CRUSH rule 1 x 828 [288,986]
+  CRUSH rule 1 x 829 [990,667]
+  CRUSH rule 1 x 830 [152,571]
+  CRUSH rule 1 x 831 [814,563]
+  CRUSH rule 1 x 832 [235,641]
+  CRUSH rule 1 x 833 [657,565]
+  CRUSH rule 1 x 834 [907,231]
+  CRUSH rule 1 x 835 [784,262]
+  CRUSH rule 1 x 836 [951,158]
+  CRUSH rule 1 x 837 [556,498]
+  CRUSH rule 1 x 838 [329,274]
+  CRUSH rule 1 x 839 [568,209]
+  CRUSH rule 1 x 840 [45,579]
+  CRUSH rule 1 x 841 [652,702]
+  CRUSH rule 1 x 842 [629,984]
+  CRUSH rule 1 x 843 [799,690]
+  CRUSH rule 1 x 844 [694,600]
+  CRUSH rule 1 x 845 [332,30]
+  CRUSH rule 1 x 846 [452,251]
+  CRUSH rule 1 x 847 [399,681]
+  CRUSH rule 1 x 848 [303,138]
+  CRUSH rule 1 x 849 [666,346]
+  CRUSH rule 1 x 850 [644,511]
+  CRUSH rule 1 x 851 [527,546]
+  CRUSH rule 1 x 852 [31,809]
+  CRUSH rule 1 x 853 [483,330]
+  CRUSH rule 1 x 854 [697,953]
+  CRUSH rule 1 x 855 [837,996]
+  CRUSH rule 1 x 856 [712,40]
+  CRUSH rule 1 x 857 [77,984]
+  CRUSH rule 1 x 858 [412,384]
+  CRUSH rule 1 x 859 [173,760]
+  CRUSH rule 1 x 860 [776,429]
+  CRUSH rule 1 x 861 [705,405]
+  CRUSH rule 1 x 862 [809,44]
+  CRUSH rule 1 x 863 [349,496]
+  CRUSH rule 1 x 864 [717,858]
+  CRUSH rule 1 x 865 [857,603]
+  CRUSH rule 1 x 866 [394,304]
+  CRUSH rule 1 x 867 [640,773]
+  CRUSH rule 1 x 868 [613,950]
+  CRUSH rule 1 x 869 [973,889]
+  CRUSH rule 1 x 870 [505,35]
+  CRUSH rule 1 x 871 [239,264]
+  CRUSH rule 1 x 872 [21,767]
+  CRUSH rule 1 x 873 [954,666]
+  CRUSH rule 1 x 874 [54,510]
+  CRUSH rule 1 x 875 [809,418]
+  CRUSH rule 1 x 876 [483,457]
+  CRUSH rule 1 x 877 [542,531]
+  CRUSH rule 1 x 878 [217,674]
+  CRUSH rule 1 x 879 [999,475]
+  CRUSH rule 1 x 880 [678,573]
+  CRUSH rule 1 x 881 [394,835]
+  CRUSH rule 1 x 882 [467,382]
+  CRUSH rule 1 x 883 [802,744]
+  CRUSH rule 1 x 884 [653,660]
+  CRUSH rule 1 x 885 [898,704]
+  CRUSH rule 1 x 886 [434,357]
+  CRUSH rule 1 x 887 [297,226]
+  CRUSH rule 1 x 888 [863,324]
+  CRUSH rule 1 x 889 [105,102]
+  CRUSH rule 1 x 890 [550,248]
+  CRUSH rule 1 x 891 [575,928]
+  CRUSH rule 1 x 892 [259,862]
+  CRUSH rule 1 x 893 [902,880]
+  CRUSH rule 1 x 894 [180,169]
+  CRUSH rule 1 x 895 [725,849]
+  CRUSH rule 1 x 896 [951,34]
+  CRUSH rule 1 x 897 [810,352]
+  CRUSH rule 1 x 898 [979,433]
+  CRUSH rule 1 x 899 [685,668]
+  CRUSH rule 1 x 900 [530,978]
+  CRUSH rule 1 x 901 [740,107]
+  CRUSH rule 1 x 902 [800,743]
+  CRUSH rule 1 x 903 [230,267]
+  CRUSH rule 1 x 904 [346,949]
+  CRUSH rule 1 x 905 [530,397]
+  CRUSH rule 1 x 906 [80,426]
+  CRUSH rule 1 x 907 [365,968]
+  CRUSH rule 1 x 908 [204,832]
+  CRUSH rule 1 x 909 [883,989]
+  CRUSH rule 1 x 910 [549,593]
+  CRUSH rule 1 x 911 [325,847]
+  CRUSH rule 1 x 912 [874,888]
+  CRUSH rule 1 x 913 [331,463]
+  CRUSH rule 1 x 914 [836,468]
+  CRUSH rule 1 x 915 [245,228]
+  CRUSH rule 1 x 916 [77,967]
+  CRUSH rule 1 x 917 [239,60]
+  CRUSH rule 1 x 918 [988,115]
+  CRUSH rule 1 x 919 [783,139]
+  CRUSH rule 1 x 920 [623,408]
+  CRUSH rule 1 x 921 [105,799]
+  CRUSH rule 1 x 922 [887,505]
+  CRUSH rule 1 x 923 [223,318]
+  CRUSH rule 1 x 924 [25,778]
+  CRUSH rule 1 x 925 [912,601]
+  CRUSH rule 1 x 926 [968,133]
+  CRUSH rule 1 x 927 [277,724]
+  CRUSH rule 1 x 928 [554,203]
+  CRUSH rule 1 x 929 [761,802]
+  CRUSH rule 1 x 930 [814,61]
+  CRUSH rule 1 x 931 [29,193]
+  CRUSH rule 1 x 932 [446,198]
+  CRUSH rule 1 x 933 [352,742]
+  CRUSH rule 1 x 934 [730,2]
+  CRUSH rule 1 x 935 [731,23]
+  CRUSH rule 1 x 936 [322,975]
+  CRUSH rule 1 x 937 [822,221]
+  CRUSH rule 1 x 938 [557,850]
+  CRUSH rule 1 x 939 [150,11]
+  CRUSH rule 1 x 940 [638,398]
+  CRUSH rule 1 x 941 [730,342]
+  CRUSH rule 1 x 942 [62,292]
+  CRUSH rule 1 x 943 [165,314]
+  CRUSH rule 1 x 944 [199,625]
+  CRUSH rule 1 x 945 [946,999]
+  CRUSH rule 1 x 946 [595,93]
+  CRUSH rule 1 x 947 [800,582]
+  CRUSH rule 1 x 948 [132,551]
+  CRUSH rule 1 x 949 [792,920]
+  CRUSH rule 1 x 950 [111,345]
+  CRUSH rule 1 x 951 [414,619]
+  CRUSH rule 1 x 952 [775,469]
+  CRUSH rule 1 x 953 [349,1]
+  CRUSH rule 1 x 954 [570,940]
+  CRUSH rule 1 x 955 [729,774]
+  CRUSH rule 1 x 956 [519,141]
+  CRUSH rule 1 x 957 [242,709]
+  CRUSH rule 1 x 958 [84,217]
+  CRUSH rule 1 x 959 [270,413]
+  CRUSH rule 1 x 960 [458,192]
+  CRUSH rule 1 x 961 [981,388]
+  CRUSH rule 1 x 962 [623,834]
+  CRUSH rule 1 x 963 [291,167]
+  CRUSH rule 1 x 964 [28,156]
+  CRUSH rule 1 x 965 [675,557]
+  CRUSH rule 1 x 966 [836,306]
+  CRUSH rule 1 x 967 [966,386]
+  CRUSH rule 1 x 968 [864,756]
+  CRUSH rule 1 x 969 [729,625]
+  CRUSH rule 1 x 970 [800,362]
+  CRUSH rule 1 x 971 [737,381]
+  CRUSH rule 1 x 972 [952,245]
+  CRUSH rule 1 x 973 [356,455]
+  CRUSH rule 1 x 974 [545,758]
+  CRUSH rule 1 x 975 [336,191]
+  CRUSH rule 1 x 976 [446,208]
+  CRUSH rule 1 x 977 [202,896]
+  CRUSH rule 1 x 978 [612,324]
+  CRUSH rule 1 x 979 [843,457]
+  CRUSH rule 1 x 980 [60,914]
+  CRUSH rule 1 x 981 [702,749]
+  CRUSH rule 1 x 982 [298,928]
+  CRUSH rule 1 x 983 [723,572]
+  CRUSH rule 1 x 984 [723,864]
+  CRUSH rule 1 x 985 [945,459]
+  CRUSH rule 1 x 986 [772,664]
+  CRUSH rule 1 x 987 [88,324]
+  CRUSH rule 1 x 988 [522,927]
+  CRUSH rule 1 x 989 [578,332]
+  CRUSH rule 1 x 990 [638,228]
+  CRUSH rule 1 x 991 [530,221]
+  CRUSH rule 1 x 992 [925,705]
+  CRUSH rule 1 x 993 [991,301]
+  CRUSH rule 1 x 994 [276,51]
+  CRUSH rule 1 x 995 [288,836]
+  CRUSH rule 1 x 996 [887,983]
+  CRUSH rule 1 x 997 [110,924]
+  CRUSH rule 1 x 998 [435,830]
+  CRUSH rule 1 x 999 [876,738]
+  CRUSH rule 1 x 1000 [178,963]
+  CRUSH rule 1 x 1001 [99,519]
+  CRUSH rule 1 x 1002 [515,534]
+  CRUSH rule 1 x 1003 [104,611]
+  CRUSH rule 1 x 1004 [269,638]
+  CRUSH rule 1 x 1005 [369,223]
+  CRUSH rule 1 x 1006 [40,107]
+  CRUSH rule 1 x 1007 [978,111]
+  CRUSH rule 1 x 1008 [965,956]
+  CRUSH rule 1 x 1009 [598,476]
+  CRUSH rule 1 x 1010 [767,523]
+  CRUSH rule 1 x 1011 [289,871]
+  CRUSH rule 1 x 1012 [128,28]
+  CRUSH rule 1 x 1013 [979,765]
+  CRUSH rule 1 x 1014 [979,948]
+  CRUSH rule 1 x 1015 [277,790]
+  CRUSH rule 1 x 1016 [262,73]
+  CRUSH rule 1 x 1017 [150,269]
+  CRUSH rule 1 x 1018 [555,829]
+  CRUSH rule 1 x 1019 [513,356]
+  CRUSH rule 1 x 1020 [158,161]
+  CRUSH rule 1 x 1021 [915,998]
+  CRUSH rule 1 x 1022 [967,829]
+  CRUSH rule 1 x 1023 [488,257]
+  rule 1 (metadata) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536]
+  CRUSH rule 1 x 1 [876,250,334]
+  CRUSH rule 1 x 2 [292,832,53]
+  CRUSH rule 1 x 3 [623,387,124]
+  CRUSH rule 1 x 4 [61,334,710]
+  CRUSH rule 1 x 5 [946,557,713]
+  CRUSH rule 1 x 6 [576,668,212]
+  CRUSH rule 1 x 7 [645,753,906]
+  CRUSH rule 1 x 8 [243,6,863]
+  CRUSH rule 1 x 9 [22,578,251]
+  CRUSH rule 1 x 10 [758,828,360]
+  CRUSH rule 1 x 11 [769,120,124]
+  CRUSH rule 1 x 12 [780,364,689]
+  CRUSH rule 1 x 13 [557,18,351]
+  CRUSH rule 1 x 14 [59,561,249]
+  CRUSH rule 1 x 15 [718,928,993]
+  CRUSH rule 1 x 16 [673,632,841]
+  CRUSH rule 1 x 17 [648,43,560]
+  CRUSH rule 1 x 18 [654,219,181]
+  CRUSH rule 1 x 19 [850,545,377]
+  CRUSH rule 1 x 20 [717,785,974]
+  CRUSH rule 1 x 21 [420,57,519]
+  CRUSH rule 1 x 22 [503,998,193]
+  CRUSH rule 1 x 23 [411,663,168]
+  CRUSH rule 1 x 24 [266,861,353]
+  CRUSH rule 1 x 25 [760,483,818]
+  CRUSH rule 1 x 26 [903,24,573]
+  CRUSH rule 1 x 27 [946,188,289]
+  CRUSH rule 1 x 28 [69,312,73]
+  CRUSH rule 1 x 29 [844,883,337]
+  CRUSH rule 1 x 30 [621,18,613]
+  CRUSH rule 1 x 31 [784,943,814]
+  CRUSH rule 1 x 32 [173,374,369]
+  CRUSH rule 1 x 33 [698,336,357]
+  CRUSH rule 1 x 34 [168,836,210]
+  CRUSH rule 1 x 35 [274,509,534]
+  CRUSH rule 1 x 36 [318,215,153]
+  CRUSH rule 1 x 37 [173,604,109]
+  CRUSH rule 1 x 38 [708,444,683]
+  CRUSH rule 1 x 39 [662,198,417]
+  CRUSH rule 1 x 40 [620,801,414]
+  CRUSH rule 1 x 41 [811,264,177]
+  CRUSH rule 1 x 42 [863,179,527]
+  CRUSH rule 1 x 43 [686,822,988]
+  CRUSH rule 1 x 44 [396,222,46]
+  CRUSH rule 1 x 45 [991,694,253]
+  CRUSH rule 1 x 46 [420,909,184]
+  CRUSH rule 1 x 47 [467,211,605]
+  CRUSH rule 1 x 48 [955,329,368]
+  CRUSH rule 1 x 49 [974,891,931]
+  CRUSH rule 1 x 50 [870,441,691]
+  CRUSH rule 1 x 51 [182,930,25]
+  CRUSH rule 1 x 52 [704,812,894]
+  CRUSH rule 1 x 53 [185,713,631]
+  CRUSH rule 1 x 54 [270,441,100]
+  CRUSH rule 1 x 55 [895,734,958]
+  CRUSH rule 1 x 56 [564,963,683]
+  CRUSH rule 1 x 57 [738,130,208]
+  CRUSH rule 1 x 58 [524,113,806]
+  CRUSH rule 1 x 59 [408,337,668]
+  CRUSH rule 1 x 60 [228,790,857]
+  CRUSH rule 1 x 61 [154,843,717]
+  CRUSH rule 1 x 62 [594,811,549]
+  CRUSH rule 1 x 63 [646,67,884]
+  CRUSH rule 1 x 64 [175,542,155]
+  CRUSH rule 1 x 65 [745,619,131]
+  CRUSH rule 1 x 66 [275,468,23]
+  CRUSH rule 1 x 67 [246,958,524]
+  CRUSH rule 1 x 68 [711,473,403]
+  CRUSH rule 1 x 69 [493,924,850]
+  CRUSH rule 1 x 70 [30,499,644]
+  CRUSH rule 1 x 71 [984,883,574]
+  CRUSH rule 1 x 72 [71,286,942]
+  CRUSH rule 1 x 73 [922,618,3]
+  CRUSH rule 1 x 74 [629,414,185]
+  CRUSH rule 1 x 75 [222,20,174]
+  CRUSH rule 1 x 76 [262,366,339]
+  CRUSH rule 1 x 77 [638,469,992]
+  CRUSH rule 1 x 78 [324,511,788]
+  CRUSH rule 1 x 79 [577,990,64]
+  CRUSH rule 1 x 80 [501,95,278]
+  CRUSH rule 1 x 81 [506,812,9]
+  CRUSH rule 1 x 82 [222,145,80]
+  CRUSH rule 1 x 83 [71,634,61]
+  CRUSH rule 1 x 84 [49,761,773]
+  CRUSH rule 1 x 85 [985,896,708]
+  CRUSH rule 1 x 86 [537,745,93]
+  CRUSH rule 1 x 87 [997,317,463]
+  CRUSH rule 1 x 88 [957,350,890]
+  CRUSH rule 1 x 89 [399,730,148]
+  CRUSH rule 1 x 90 [943,706,683]
+  CRUSH rule 1 x 91 [22,368,149]
+  CRUSH rule 1 x 92 [532,424,426]
+  CRUSH rule 1 x 93 [218,489,405]
+  CRUSH rule 1 x 94 [181,96,102]
+  CRUSH rule 1 x 95 [343,957,820]
+  CRUSH rule 1 x 96 [861,270,87]
+  CRUSH rule 1 x 97 [459,706,45]
+  CRUSH rule 1 x 98 [327,867,353]
+  CRUSH rule 1 x 99 [974,133,468]
+  CRUSH rule 1 x 100 [32,445,547]
+  CRUSH rule 1 x 101 [142,90,337]
+  CRUSH rule 1 x 102 [172,129,139]
+  CRUSH rule 1 x 103 [630,47,161]
+  CRUSH rule 1 x 104 [758,133,278]
+  CRUSH rule 1 x 105 [843,604,47]
+  CRUSH rule 1 x 106 [28,681,193]
+  CRUSH rule 1 x 107 [74,320,85]
+  CRUSH rule 1 x 108 [875,593,575]
+  CRUSH rule 1 x 109 [411,985,811]
+  CRUSH rule 1 x 110 [440,774,799]
+  CRUSH rule 1 x 111 [405,742,276]
+  CRUSH rule 1 x 112 [143,181,922]
+  CRUSH rule 1 x 113 [153,846,160]
+  CRUSH rule 1 x 114 [804,892,939]
+  CRUSH rule 1 x 115 [588,508,958]
+  CRUSH rule 1 x 116 [327,148,637]
+  CRUSH rule 1 x 117 [95,594,989]
+  CRUSH rule 1 x 118 [80,957,897]
+  CRUSH rule 1 x 119 [386,932,951]
+  CRUSH rule 1 x 120 [366,312,653]
+  CRUSH rule 1 x 121 [129,154,847]
+  CRUSH rule 1 x 122 [873,1,110]
+  CRUSH rule 1 x 123 [533,415,789]
+  CRUSH rule 1 x 124 [461,691,898]
+  CRUSH rule 1 x 125 [342,599,830]
+  CRUSH rule 1 x 126 [819,781,822]
+  CRUSH rule 1 x 127 [437,893,585]
+  CRUSH rule 1 x 128 [679,994,982]
+  CRUSH rule 1 x 129 [380,685,947]
+  CRUSH rule 1 x 130 [992,52,466]
+  CRUSH rule 1 x 131 [469,90,208]
+  CRUSH rule 1 x 132 [571,250,316]
+  CRUSH rule 1 x 133 [964,728,329]
+  CRUSH rule 1 x 134 [999,19,716]
+  CRUSH rule 1 x 135 [634,101,52]
+  CRUSH rule 1 x 136 [114,889,692]
+  CRUSH rule 1 x 137 [839,8,959]
+  CRUSH rule 1 x 138 [967,949,138]
+  CRUSH rule 1 x 139 [308,711,736]
+  CRUSH rule 1 x 140 [764,936,926]
+  CRUSH rule 1 x 141 [423,302,112]
+  CRUSH rule 1 x 142 [252,821,715]
+  CRUSH rule 1 x 143 [33,808,518]
+  CRUSH rule 1 x 144 [472,88,969]
+  CRUSH rule 1 x 145 [242,208,252]
+  CRUSH rule 1 x 146 [290,70,570]
+  CRUSH rule 1 x 147 [447,352,657]
+  CRUSH rule 1 x 148 [212,644,432]
+  CRUSH rule 1 x 149 [9,775,87]
+  CRUSH rule 1 x 150 [166,456,582]
+  CRUSH rule 1 x 151 [811,875,307]
+  CRUSH rule 1 x 152 [449,617,223]
+  CRUSH rule 1 x 153 [523,537,695]
+  CRUSH rule 1 x 154 [208,559,874]
+  CRUSH rule 1 x 155 [569,325,192]
+  CRUSH rule 1 x 156 [488,121,521]
+  CRUSH rule 1 x 157 [140,723,633]
+  CRUSH rule 1 x 158 [786,451,320]
+  CRUSH rule 1 x 159 [134,664,517]
+  CRUSH rule 1 x 160 [690,112,414]
+  CRUSH rule 1 x 161 [324,912,397]
+  CRUSH rule 1 x 162 [748,567,284]
+  CRUSH rule 1 x 163 [575,499,31]
+  CRUSH rule 1 x 164 [314,489,308]
+  CRUSH rule 1 x 165 [116,209,750]
+  CRUSH rule 1 x 166 [352,706,701]
+  CRUSH rule 1 x 167 [27,743,174]
+  CRUSH rule 1 x 168 [953,898,880]
+  CRUSH rule 1 x 169 [912,147,266]
+  CRUSH rule 1 x 170 [421,515,828]
+  CRUSH rule 1 x 171 [488,584,880]
+  CRUSH rule 1 x 172 [366,443,957]
+  CRUSH rule 1 x 173 [863,291,625]
+  CRUSH rule 1 x 174 [263,555,650]
+  CRUSH rule 1 x 175 [875,961,361]
+  CRUSH rule 1 x 176 [745,83,701]
+  CRUSH rule 1 x 177 [128,244,41]
+  CRUSH rule 1 x 178 [155,41,264]
+  CRUSH rule 1 x 179 [593,833,202]
+  CRUSH rule 1 x 180 [154,734,17]
+  CRUSH rule 1 x 181 [289,675,723]
+  CRUSH rule 1 x 182 [730,931,560]
+  CRUSH rule 1 x 183 [639,237,794]
+  CRUSH rule 1 x 184 [704,312,685]
+  CRUSH rule 1 x 185 [97,100,762]
+  CRUSH rule 1 x 186 [26,665,554]
+  CRUSH rule 1 x 187 [649,14,740]
+  CRUSH rule 1 x 188 [682,695,590]
+  CRUSH rule 1 x 189 [325,693,726]
+  CRUSH rule 1 x 190 [399,933,136]
+  CRUSH rule 1 x 191 [629,533,17]
+  CRUSH rule 1 x 192 [503,578,38]
+  CRUSH rule 1 x 193 [546,333,651]
+  CRUSH rule 1 x 194 [242,473,58]
+  CRUSH rule 1 x 195 [625,719,135]
+  CRUSH rule 1 x 196 [357,114,125]
+  CRUSH rule 1 x 197 [306,954,453]
+  CRUSH rule 1 x 198 [863,791,311]
+  CRUSH rule 1 x 199 [935,906,929]
+  CRUSH rule 1 x 200 [373,774,229]
+  CRUSH rule 1 x 201 [659,320,477]
+  CRUSH rule 1 x 202 [260,433,524]
+  CRUSH rule 1 x 203 [36,239,675]
+  CRUSH rule 1 x 204 [92,516,993]
+  CRUSH rule 1 x 205 [68,395,473]
+  CRUSH rule 1 x 206 [570,530,642]
+  CRUSH rule 1 x 207 [834,457,850]
+  CRUSH rule 1 x 208 [927,484,640]
+  CRUSH rule 1 x 209 [878,66,58]
+  CRUSH rule 1 x 210 [572,981,484]
+  CRUSH rule 1 x 211 [107,597,780]
+  CRUSH rule 1 x 212 [389,107,838]
+  CRUSH rule 1 x 213 [497,717,567]
+  CRUSH rule 1 x 214 [798,65,254]
+  CRUSH rule 1 x 215 [233,419,283]
+  CRUSH rule 1 x 216 [494,464,742]
+  CRUSH rule 1 x 217 [352,396,309]
+  CRUSH rule 1 x 218 [895,864,988]
+  CRUSH rule 1 x 219 [222,534,277]
+  CRUSH rule 1 x 220 [281,19,584]
+  CRUSH rule 1 x 221 [64,928,963]
+  CRUSH rule 1 x 222 [40,544,161]
+  CRUSH rule 1 x 223 [645,556,159]
+  CRUSH rule 1 x 224 [647,165,957]
+  CRUSH rule 1 x 225 [219,714,858]
+  CRUSH rule 1 x 226 [372,511,181]
+  CRUSH rule 1 x 227 [925,156,714]
+  CRUSH rule 1 x 228 [682,404,839]
+  CRUSH rule 1 x 229 [880,838,770]
+  CRUSH rule 1 x 230 [328,659,916]
+  CRUSH rule 1 x 231 [320,383,669]
+  CRUSH rule 1 x 232 [924,846,394]
+  CRUSH rule 1 x 233 [948,652,575]
+  CRUSH rule 1 x 234 [484,943,42]
+  CRUSH rule 1 x 235 [750,65,590]
+  CRUSH rule 1 x 236 [551,787,490]
+  CRUSH rule 1 x 237 [390,157,166]
+  CRUSH rule 1 x 238 [570,6,989]
+  CRUSH rule 1 x 239 [729,959,376]
+  CRUSH rule 1 x 240 [981,241,156]
+  CRUSH rule 1 x 241 [310,816,641]
+  CRUSH rule 1 x 242 [161,63,642]
+  CRUSH rule 1 x 243 [180,394,33]
+  CRUSH rule 1 x 244 [52,174,685]
+  CRUSH rule 1 x 245 [523,121,915]
+  CRUSH rule 1 x 246 [362,893,390]
+  CRUSH rule 1 x 247 [382,184,116]
+  CRUSH rule 1 x 248 [129,114,852]
+  CRUSH rule 1 x 249 [159,683,91]
+  CRUSH rule 1 x 250 [404,945,569]
+  CRUSH rule 1 x 251 [661,225,738]
+  CRUSH rule 1 x 252 [961,226,542]
+  CRUSH rule 1 x 253 [651,97,225]
+  CRUSH rule 1 x 254 [123,33,741]
+  CRUSH rule 1 x 255 [314,649,891]
+  CRUSH rule 1 x 256 [315,215,651]
+  CRUSH rule 1 x 257 [825,264,867]
+  CRUSH rule 1 x 258 [624,789,370]
+  CRUSH rule 1 x 259 [602,542,70]
+  CRUSH rule 1 x 260 [717,878,43]
+  CRUSH rule 1 x 261 [145,517,20]
+  CRUSH rule 1 x 262 [223,1,561]
+  CRUSH rule 1 x 263 [462,211,405]
+  CRUSH rule 1 x 264 [654,471,266]
+  CRUSH rule 1 x 265 [302,794,704]
+  CRUSH rule 1 x 266 [202,132,884]
+  CRUSH rule 1 x 267 [282,938,657]
+  CRUSH rule 1 x 268 [338,309,356]
+  CRUSH rule 1 x 269 [738,122,266]
+  CRUSH rule 1 x 270 [707,982,946]
+  CRUSH rule 1 x 271 [705,432,364]
+  CRUSH rule 1 x 272 [756,545,942]
+  CRUSH rule 1 x 273 [197,502,527]
+  CRUSH rule 1 x 274 [992,44,653]
+  CRUSH rule 1 x 275 [544,789,170]
+  CRUSH rule 1 x 276 [658,467,577]
+  CRUSH rule 1 x 277 [143,490,880]
+  CRUSH rule 1 x 278 [492,647,355]
+  CRUSH rule 1 x 279 [517,792,604]
+  CRUSH rule 1 x 280 [825,740,27]
+  CRUSH rule 1 x 281 [224,629,120]
+  CRUSH rule 1 x 282 [298,661,380]
+  CRUSH rule 1 x 283 [311,606,208]
+  CRUSH rule 1 x 284 [771,466,371]
+  CRUSH rule 1 x 285 [693,362,404]
+  CRUSH rule 1 x 286 [364,477,285]
+  CRUSH rule 1 x 287 [591,611,828]
+  CRUSH rule 1 x 288 [965,541,848]
+  CRUSH rule 1 x 289 [225,551,948]
+  CRUSH rule 1 x 290 [577,762,777]
+  CRUSH rule 1 x 291 [160,903,477]
+  CRUSH rule 1 x 292 [873,598,216]
+  CRUSH rule 1 x 293 [100,234,874]
+  CRUSH rule 1 x 294 [285,943,379]
+  CRUSH rule 1 x 295 [938,262,880]
+  CRUSH rule 1 x 296 [850,327,86]
+  CRUSH rule 1 x 297 [951,53,99]
+  CRUSH rule 1 x 298 [173,336,85]
+  CRUSH rule 1 x 299 [598,591,315]
+  CRUSH rule 1 x 300 [531,957,62]
+  CRUSH rule 1 x 301 [823,628,23]
+  CRUSH rule 1 x 302 [184,80,780]
+  CRUSH rule 1 x 303 [521,766,222]
+  CRUSH rule 1 x 304 [980,127,807]
+  CRUSH rule 1 x 305 [153,816,22]
+  CRUSH rule 1 x 306 [423,739,664]
+  CRUSH rule 1 x 307 [997,557,682]
+  CRUSH rule 1 x 308 [991,874,534]
+  CRUSH rule 1 x 309 [860,394,724]
+  CRUSH rule 1 x 310 [589,818,546]
+  CRUSH rule 1 x 311 [477,774,225]
+  CRUSH rule 1 x 312 [887,853,950]
+  CRUSH rule 1 x 313 [802,646,447]
+  CRUSH rule 1 x 314 [654,974,229]
+  CRUSH rule 1 x 315 [767,227,28]
+  CRUSH rule 1 x 316 [778,83,733]
+  CRUSH rule 1 x 317 [184,418,642]
+  CRUSH rule 1 x 318 [525,410,500]
+  CRUSH rule 1 x 319 [476,724,569]
+  CRUSH rule 1 x 320 [149,610,697]
+  CRUSH rule 1 x 321 [710,79,667]
+  CRUSH rule 1 x 322 [175,275,323]
+  CRUSH rule 1 x 323 [819,604,638]
+  CRUSH rule 1 x 324 [16,745,511]
+  CRUSH rule 1 x 325 [486,400,872]
+  CRUSH rule 1 x 326 [613,765,207]
+  CRUSH rule 1 x 327 [125,289,738]
+  CRUSH rule 1 x 328 [807,383,476]
+  CRUSH rule 1 x 329 [588,938,599]
+  CRUSH rule 1 x 330 [932,644,41]
+  CRUSH rule 1 x 331 [341,953,950]
+  CRUSH rule 1 x 332 [153,726,459]
+  CRUSH rule 1 x 333 [745,845,853]
+  CRUSH rule 1 x 334 [614,751,807]
+  CRUSH rule 1 x 335 [518,721,221]
+  CRUSH rule 1 x 336 [389,424,77]
+  CRUSH rule 1 x 337 [753,508,765]
+  CRUSH rule 1 x 338 [128,810,490]
+  CRUSH rule 1 x 339 [430,308,58]
+  CRUSH rule 1 x 340 [541,44,630]
+  CRUSH rule 1 x 341 [402,26,631]
+  CRUSH rule 1 x 342 [982,57,992]
+  CRUSH rule 1 x 343 [833,412,572]
+  CRUSH rule 1 x 344 [784,533,792]
+  CRUSH rule 1 x 345 [546,300,304]
+  CRUSH rule 1 x 346 [302,420,428]
+  CRUSH rule 1 x 347 [488,778,101]
+  CRUSH rule 1 x 348 [903,744,937]
+  CRUSH rule 1 x 349 [471,547,582]
+  CRUSH rule 1 x 350 [348,221,823]
+  CRUSH rule 1 x 351 [961,582,705]
+  CRUSH rule 1 x 352 [728,137,461]
+  CRUSH rule 1 x 353 [904,202,184]
+  CRUSH rule 1 x 354 [345,226,319]
+  CRUSH rule 1 x 355 [50,430,175]
+  CRUSH rule 1 x 356 [87,185,55]
+  CRUSH rule 1 x 357 [762,459,921]
+  CRUSH rule 1 x 358 [908,25,280]
+  CRUSH rule 1 x 359 [484,15,132]
+  CRUSH rule 1 x 360 [173,378,337]
+  CRUSH rule 1 x 361 [404,577,115]
+  CRUSH rule 1 x 362 [403,1,422]
+  CRUSH rule 1 x 363 [639,911,510]
+  CRUSH rule 1 x 364 [752,689,610]
+  CRUSH rule 1 x 365 [956,999,212]
+  CRUSH rule 1 x 366 [860,925,924]
+  CRUSH rule 1 x 367 [205,609,647]
+  CRUSH rule 1 x 368 [301,284,810]
+  CRUSH rule 1 x 369 [452,658,339]
+  CRUSH rule 1 x 370 [11,467,695]
+  CRUSH rule 1 x 371 [124,487,55]
+  CRUSH rule 1 x 372 [253,48,979]
+  CRUSH rule 1 x 373 [715,605,775]
+  CRUSH rule 1 x 374 [191,887,920]
+  CRUSH rule 1 x 375 [711,385,651]
+  CRUSH rule 1 x 376 [597,818,49]
+  CRUSH rule 1 x 377 [294,256,933]
+  CRUSH rule 1 x 378 [34,151,681]
+  CRUSH rule 1 x 379 [869,136,315]
+  CRUSH rule 1 x 380 [294,97,575]
+  CRUSH rule 1 x 381 [119,710,219]
+  CRUSH rule 1 x 382 [69,631,508]
+  CRUSH rule 1 x 383 [922,588,589]
+  CRUSH rule 1 x 384 [221,945,671]
+  CRUSH rule 1 x 385 [561,737,953]
+  CRUSH rule 1 x 386 [335,442,788]
+  CRUSH rule 1 x 387 [514,43,353]
+  CRUSH rule 1 x 388 [587,89,157]
+  CRUSH rule 1 x 389 [109,641,255]
+  CRUSH rule 1 x 390 [925,149,421]
+  CRUSH rule 1 x 391 [267,87,387]
+  CRUSH rule 1 x 392 [382,485,370]
+  CRUSH rule 1 x 393 [425,721,221]
+  CRUSH rule 1 x 394 [898,18,38]
+  CRUSH rule 1 x 395 [806,876,269]
+  CRUSH rule 1 x 396 [790,970,437]
+  CRUSH rule 1 x 397 [136,363,507]
+  CRUSH rule 1 x 398 [914,116,558]
+  CRUSH rule 1 x 399 [261,94,299]
+  CRUSH rule 1 x 400 [661,197,338]
+  CRUSH rule 1 x 401 [953,979,287]
+  CRUSH rule 1 x 402 [738,819,618]
+  CRUSH rule 1 x 403 [573,238,425]
+  CRUSH rule 1 x 404 [526,848,790]
+  CRUSH rule 1 x 405 [582,505,330]
+  CRUSH rule 1 x 406 [768,324,493]
+  CRUSH rule 1 x 407 [260,951,437]
+  CRUSH rule 1 x 408 [657,81,770]
+  CRUSH rule 1 x 409 [498,89,182]
+  CRUSH rule 1 x 410 [28,793,737]
+  CRUSH rule 1 x 411 [684,992,60]
+  CRUSH rule 1 x 412 [261,958,699]
+  CRUSH rule 1 x 413 [891,835,297]
+  CRUSH rule 1 x 414 [127,459,119]
+  CRUSH rule 1 x 415 [272,540,631]
+  CRUSH rule 1 x 416 [739,617,115]
+  CRUSH rule 1 x 417 [106,209,157]
+  CRUSH rule 1 x 418 [525,441,147]
+  CRUSH rule 1 x 419 [603,673,615]
+  CRUSH rule 1 x 420 [988,213,251]
+  CRUSH rule 1 x 421 [761,521,748]
+  CRUSH rule 1 x 422 [317,160,924]
+  CRUSH rule 1 x 423 [137,807,168]
+  CRUSH rule 1 x 424 [920,37,146]
+  CRUSH rule 1 x 425 [277,693,285]
+  CRUSH rule 1 x 426 [485,936,407]
+  CRUSH rule 1 x 427 [242,515,9]
+  CRUSH rule 1 x 428 [632,635,26]
+  CRUSH rule 1 x 429 [641,73,465]
+  CRUSH rule 1 x 430 [626,585,6]
+  CRUSH rule 1 x 431 [697,76,753]
+  CRUSH rule 1 x 432 [590,526,306]
+  CRUSH rule 1 x 433 [284,387,149]
+  CRUSH rule 1 x 434 [538,985,79]
+  CRUSH rule 1 x 435 [30,318,593]
+  CRUSH rule 1 x 436 [164,919,851]
+  CRUSH rule 1 x 437 [322,212,163]
+  CRUSH rule 1 x 438 [142,392,85]
+  CRUSH rule 1 x 439 [119,370,68]
+  CRUSH rule 1 x 440 [333,403,187]
+  CRUSH rule 1 x 441 [477,727,906]
+  CRUSH rule 1 x 442 [274,590,933]
+  CRUSH rule 1 x 443 [983,748,574]
+  CRUSH rule 1 x 444 [536,509,431]
+  CRUSH rule 1 x 445 [485,964,528]
+  CRUSH rule 1 x 446 [345,634,42]
+  CRUSH rule 1 x 447 [61,845,767]
+  CRUSH rule 1 x 448 [333,232,292]
+  CRUSH rule 1 x 449 [680,16,484]
+  CRUSH rule 1 x 450 [235,214,79]
+  CRUSH rule 1 x 451 [961,468,333]
+  CRUSH rule 1 x 452 [525,479,153]
+  CRUSH rule 1 x 453 [138,466,302]
+  CRUSH rule 1 x 454 [137,625,215]
+  CRUSH rule 1 x 455 [173,150,997]
+  CRUSH rule 1 x 456 [235,226,238]
+  CRUSH rule 1 x 457 [450,577,253]
+  CRUSH rule 1 x 458 [195,537,91]
+  CRUSH rule 1 x 459 [381,555,312]
+  CRUSH rule 1 x 460 [972,730,534]
+  CRUSH rule 1 x 461 [506,279,142]
+  CRUSH rule 1 x 462 [692,959,578]
+  CRUSH rule 1 x 463 [788,667,949]
+  CRUSH rule 1 x 464 [133,122,588]
+  CRUSH rule 1 x 465 [971,190,230]
+  CRUSH rule 1 x 466 [394,576,148]
+  CRUSH rule 1 x 467 [517,28,366]
+  CRUSH rule 1 x 468 [829,143,874]
+  CRUSH rule 1 x 469 [987,936,106]
+  CRUSH rule 1 x 470 [107,982,56]
+  CRUSH rule 1 x 471 [181,897,629]
+  CRUSH rule 1 x 472 [547,512,172]
+  CRUSH rule 1 x 473 [760,997,824]
+  CRUSH rule 1 x 474 [787,418,743]
+  CRUSH rule 1 x 475 [662,312,253]
+  CRUSH rule 1 x 476 [110,495,185]
+  CRUSH rule 1 x 477 [393,954,834]
+  CRUSH rule 1 x 478 [246,483,480]
+  CRUSH rule 1 x 479 [70,929,697]
+  CRUSH rule 1 x 480 [753,119,961]
+  CRUSH rule 1 x 481 [470,429,677]
+  CRUSH rule 1 x 482 [451,566,961]
+  CRUSH rule 1 x 483 [816,72,371]
+  CRUSH rule 1 x 484 [540,454,389]
+  CRUSH rule 1 x 485 [74,582,624]
+  CRUSH rule 1 x 486 [958,595,199]
+  CRUSH rule 1 x 487 [228,302,804]
+  CRUSH rule 1 x 488 [180,529,722]
+  CRUSH rule 1 x 489 [47,617,812]
+  CRUSH rule 1 x 490 [905,822,479]
+  CRUSH rule 1 x 491 [892,370,609]
+  CRUSH rule 1 x 492 [588,959,127]
+  CRUSH rule 1 x 493 [353,461,593]
+  CRUSH rule 1 x 494 [378,848,443]
+  CRUSH rule 1 x 495 [845,653,768]
+  CRUSH rule 1 x 496 [13,988,0]
+  CRUSH rule 1 x 497 [796,877,788]
+  CRUSH rule 1 x 498 [412,337,270]
+  CRUSH rule 1 x 499 [330,695,8]
+  CRUSH rule 1 x 500 [820,272,547]
+  CRUSH rule 1 x 501 [110,44,132]
+  CRUSH rule 1 x 502 [336,595,650]
+  CRUSH rule 1 x 503 [922,211,157]
+  CRUSH rule 1 x 504 [483,52,122]
+  CRUSH rule 1 x 505 [482,598,224]
+  CRUSH rule 1 x 506 [493,123,43]
+  CRUSH rule 1 x 507 [12,598,264]
+  CRUSH rule 1 x 508 [227,157,611]
+  CRUSH rule 1 x 509 [807,242,363]
+  CRUSH rule 1 x 510 [134,437,227]
+  CRUSH rule 1 x 511 [212,54,83]
+  CRUSH rule 1 x 512 [236,630,758]
+  CRUSH rule 1 x 513 [994,693,644]
+  CRUSH rule 1 x 514 [45,508,831]
+  CRUSH rule 1 x 515 [504,138,480]
+  CRUSH rule 1 x 516 [285,409,136]
+  CRUSH rule 1 x 517 [300,232,23]
+  CRUSH rule 1 x 518 [397,674,98]
+  CRUSH rule 1 x 519 [86,750,772]
+  CRUSH rule 1 x 520 [900,833,614]
+  CRUSH rule 1 x 521 [31,47,236]
+  CRUSH rule 1 x 522 [390,16,280]
+  CRUSH rule 1 x 523 [618,308,424]
+  CRUSH rule 1 x 524 [635,189,687]
+  CRUSH rule 1 x 525 [311,916,699]
+  CRUSH rule 1 x 526 [48,738,227]
+  CRUSH rule 1 x 527 [202,851,889]
+  CRUSH rule 1 x 528 [565,827,590]
+  CRUSH rule 1 x 529 [934,864,241]
+  CRUSH rule 1 x 530 [502,934,298]
+  CRUSH rule 1 x 531 [681,627,942]
+  CRUSH rule 1 x 532 [422,6,147]
+  CRUSH rule 1 x 533 [863,68,364]
+  CRUSH rule 1 x 534 [962,931,775]
+  CRUSH rule 1 x 535 [89,565,397]
+  CRUSH rule 1 x 536 [499,351,760]
+  CRUSH rule 1 x 537 [676,547,787]
+  CRUSH rule 1 x 538 [58,644,571]
+  CRUSH rule 1 x 539 [837,953,457]
+  CRUSH rule 1 x 540 [831,50,132]
+  CRUSH rule 1 x 541 [582,757,121]
+  CRUSH rule 1 x 542 [472,132,790]
+  CRUSH rule 1 x 543 [382,272,797]
+  CRUSH rule 1 x 544 [947,930,496]
+  CRUSH rule 1 x 545 [425,570,305]
+  CRUSH rule 1 x 546 [18,65,529]
+  CRUSH rule 1 x 547 [445,715,600]
+  CRUSH rule 1 x 548 [367,569,980]
+  CRUSH rule 1 x 549 [125,715,671]
+  CRUSH rule 1 x 550 [425,599,744]
+  CRUSH rule 1 x 551 [44,1,528]
+  CRUSH rule 1 x 552 [246,104,68]
+  CRUSH rule 1 x 553 [71,703,615]
+  CRUSH rule 1 x 554 [207,124,217]
+  CRUSH rule 1 x 555 [570,28,317]
+  CRUSH rule 1 x 556 [674,152,421]
+  CRUSH rule 1 x 557 [347,817,191]
+  CRUSH rule 1 x 558 [627,426,369]
+  CRUSH rule 1 x 559 [940,630,924]
+  CRUSH rule 1 x 560 [295,903,541]
+  CRUSH rule 1 x 561 [506,682,384]
+  CRUSH rule 1 x 562 [718,529,87]
+  CRUSH rule 1 x 563 [552,332,747]
+  CRUSH rule 1 x 564 [835,769,736]
+  CRUSH rule 1 x 565 [8,167,539]
+  CRUSH rule 1 x 566 [600,481,301]
+  CRUSH rule 1 x 567 [999,994,509]
+  CRUSH rule 1 x 568 [252,431,157]
+  CRUSH rule 1 x 569 [643,218,943]
+  CRUSH rule 1 x 570 [617,635,765]
+  CRUSH rule 1 x 571 [757,80,59]
+  CRUSH rule 1 x 572 [299,348,575]
+  CRUSH rule 1 x 573 [25,505,270]
+  CRUSH rule 1 x 574 [215,431,624]
+  CRUSH rule 1 x 575 [225,252,611]
+  CRUSH rule 1 x 576 [627,94,159]
+  CRUSH rule 1 x 577 [237,809,778]
+  CRUSH rule 1 x 578 [885,313,120]
+  CRUSH rule 1 x 579 [924,575,787]
+  CRUSH rule 1 x 580 [718,51,766]
+  CRUSH rule 1 x 581 [219,807,129]
+  CRUSH rule 1 x 582 [893,701,598]
+  CRUSH rule 1 x 583 [246,930,964]
+  CRUSH rule 1 x 584 [336,432,680]
+  CRUSH rule 1 x 585 [324,999,397]
+  CRUSH rule 1 x 586 [558,230,976]
+  CRUSH rule 1 x 587 [985,830,597]
+  CRUSH rule 1 x 588 [211,544,57]
+  CRUSH rule 1 x 589 [129,21,112]
+  CRUSH rule 1 x 590 [467,969,652]
+  CRUSH rule 1 x 591 [758,514,316]
+  CRUSH rule 1 x 592 [525,253,190]
+  CRUSH rule 1 x 593 [601,885,339]
+  CRUSH rule 1 x 594 [227,60,450]
+  CRUSH rule 1 x 595 [720,854,496]
+  CRUSH rule 1 x 596 [751,195,997]
+  CRUSH rule 1 x 597 [129,574,714]
+  CRUSH rule 1 x 598 [679,207,604]
+  CRUSH rule 1 x 599 [668,315,683]
+  CRUSH rule 1 x 600 [143,396,464]
+  CRUSH rule 1 x 601 [326,573,873]
+  CRUSH rule 1 x 602 [860,281,875]
+  CRUSH rule 1 x 603 [709,328,445]
+  CRUSH rule 1 x 604 [571,62,814]
+  CRUSH rule 1 x 605 [252,739,860]
+  CRUSH rule 1 x 606 [339,236,759]
+  CRUSH rule 1 x 607 [590,248,759]
+  CRUSH rule 1 x 608 [145,635,309]
+  CRUSH rule 1 x 609 [973,547,223]
+  CRUSH rule 1 x 610 [435,816,961]
+  CRUSH rule 1 x 611 [559,283,422]
+  CRUSH rule 1 x 612 [273,149,123]
+  CRUSH rule 1 x 613 [828,614,642]
+  CRUSH rule 1 x 614 [478,748,393]
+  CRUSH rule 1 x 615 [392,155,144]
+  CRUSH rule 1 x 616 [778,637,452]
+  CRUSH rule 1 x 617 [622,713,996]
+  CRUSH rule 1 x 618 [149,877,270]
+  CRUSH rule 1 x 619 [604,163,656]
+  CRUSH rule 1 x 620 [181,23,409]
+  CRUSH rule 1 x 621 [735,902,386]
+  CRUSH rule 1 x 622 [661,824,717]
+  CRUSH rule 1 x 623 [142,121,643]
+  CRUSH rule 1 x 624 [360,716,420]
+  CRUSH rule 1 x 625 [541,167,385]
+  CRUSH rule 1 x 626 [364,431,610]
+  CRUSH rule 1 x 627 [458,137,557]
+  CRUSH rule 1 x 628 [250,350,556]
+  CRUSH rule 1 x 629 [928,160,710]
+  CRUSH rule 1 x 630 [243,19,918]
+  CRUSH rule 1 x 631 [438,221,574]
+  CRUSH rule 1 x 632 [797,368,247]
+  CRUSH rule 1 x 633 [993,749,525]
+  CRUSH rule 1 x 634 [239,351,633]
+  CRUSH rule 1 x 635 [640,965,25]
+  CRUSH rule 1 x 636 [173,290,297]
+  CRUSH rule 1 x 637 [0,918,98]
+  CRUSH rule 1 x 638 [702,235,424]
+  CRUSH rule 1 x 639 [475,687,31]
+  CRUSH rule 1 x 640 [31,664,399]
+  CRUSH rule 1 x 641 [296,473,108]
+  CRUSH rule 1 x 642 [894,273,427]
+  CRUSH rule 1 x 643 [117,111,732]
+  CRUSH rule 1 x 644 [438,336,327]
+  CRUSH rule 1 x 645 [982,702,351]
+  CRUSH rule 1 x 646 [334,804,146]
+  CRUSH rule 1 x 647 [933,787,185]
+  CRUSH rule 1 x 648 [22,444,400]
+  CRUSH rule 1 x 649 [503,229,213]
+  CRUSH rule 1 x 650 [328,659,420]
+  CRUSH rule 1 x 651 [3,880,823]
+  CRUSH rule 1 x 652 [495,977,563]
+  CRUSH rule 1 x 653 [185,718,804]
+  CRUSH rule 1 x 654 [130,528,380]
+  CRUSH rule 1 x 655 [560,872,454]
+  CRUSH rule 1 x 656 [219,885,178]
+  CRUSH rule 1 x 657 [233,684,813]
+  CRUSH rule 1 x 658 [778,6,756]
+  CRUSH rule 1 x 659 [240,663,306]
+  CRUSH rule 1 x 660 [244,855,196]
+  CRUSH rule 1 x 661 [184,270,128]
+  CRUSH rule 1 x 662 [65,883,921]
+  CRUSH rule 1 x 663 [323,721,594]
+  CRUSH rule 1 x 664 [865,113,512]
+  CRUSH rule 1 x 665 [420,850,591]
+  CRUSH rule 1 x 666 [319,767,246]
+  CRUSH rule 1 x 667 [875,39,343]
+  CRUSH rule 1 x 668 [331,122,263]
+  CRUSH rule 1 x 669 [915,521,402]
+  CRUSH rule 1 x 670 [845,659,943]
+  CRUSH rule 1 x 671 [108,634,527]
+  CRUSH rule 1 x 672 [578,216,110]
+  CRUSH rule 1 x 673 [442,74,579]
+  CRUSH rule 1 x 674 [588,364,281]
+  CRUSH rule 1 x 675 [489,698,744]
+  CRUSH rule 1 x 676 [928,911,40]
+  CRUSH rule 1 x 677 [399,269,692]
+  CRUSH rule 1 x 678 [546,752,544]
+  CRUSH rule 1 x 679 [988,25,275]
+  CRUSH rule 1 x 680 [335,963,382]
+  CRUSH rule 1 x 681 [690,462,623]
+  CRUSH rule 1 x 682 [196,588,154]
+  CRUSH rule 1 x 683 [627,25,421]
+  CRUSH rule 1 x 684 [38,804,592]
+  CRUSH rule 1 x 685 [841,368,548]
+  CRUSH rule 1 x 686 [336,287,525]
+  CRUSH rule 1 x 687 [20,682,924]
+  CRUSH rule 1 x 688 [463,371,780]
+  CRUSH rule 1 x 689 [569,250,78]
+  CRUSH rule 1 x 690 [551,144,587]
+  CRUSH rule 1 x 691 [766,464,446]
+  CRUSH rule 1 x 692 [739,634,18]
+  CRUSH rule 1 x 693 [339,297,118]
+  CRUSH rule 1 x 694 [405,26,830]
+  CRUSH rule 1 x 695 [622,576,597]
+  CRUSH rule 1 x 696 [558,902,689]
+  CRUSH rule 1 x 697 [818,222,406]
+  CRUSH rule 1 x 698 [178,48,402]
+  CRUSH rule 1 x 699 [450,244,180]
+  CRUSH rule 1 x 700 [502,771,987]
+  CRUSH rule 1 x 701 [4,612,782]
+  CRUSH rule 1 x 702 [177,630,232]
+  CRUSH rule 1 x 703 [354,178,389]
+  CRUSH rule 1 x 704 [646,601,156]
+  CRUSH rule 1 x 705 [921,401,890]
+  CRUSH rule 1 x 706 [652,877,562]
+  CRUSH rule 1 x 707 [345,745,67]
+  CRUSH rule 1 x 708 [333,607,180]
+  CRUSH rule 1 x 709 [45,187,302]
+  CRUSH rule 1 x 710 [94,855,43]
+  CRUSH rule 1 x 711 [227,653,731]
+  CRUSH rule 1 x 712 [398,953,136]
+  CRUSH rule 1 x 713 [116,800,503]
+  CRUSH rule 1 x 714 [111,629,866]
+  CRUSH rule 1 x 715 [531,291,486]
+  CRUSH rule 1 x 716 [169,541,291]
+  CRUSH rule 1 x 717 [417,446,994]
+  CRUSH rule 1 x 718 [992,383,298]
+  CRUSH rule 1 x 719 [936,674,324]
+  CRUSH rule 1 x 720 [370,188,174]
+  CRUSH rule 1 x 721 [320,859,278]
+  CRUSH rule 1 x 722 [7,2,673]
+  CRUSH rule 1 x 723 [270,553,831]
+  CRUSH rule 1 x 724 [666,822,708]
+  CRUSH rule 1 x 725 [794,406,875]
+  CRUSH rule 1 x 726 [420,556,341]
+  CRUSH rule 1 x 727 [561,461,129]
+  CRUSH rule 1 x 728 [951,330,196]
+  CRUSH rule 1 x 729 [656,644,436]
+  CRUSH rule 1 x 730 [3,558,629]
+  CRUSH rule 1 x 731 [852,89,75]
+  CRUSH rule 1 x 732 [983,840,869]
+  CRUSH rule 1 x 733 [285,396,388]
+  CRUSH rule 1 x 734 [125,510,402]
+  CRUSH rule 1 x 735 [417,773,686]
+  CRUSH rule 1 x 736 [749,396,632]
+  CRUSH rule 1 x 737 [644,991,946]
+  CRUSH rule 1 x 738 [449,683,290]
+  CRUSH rule 1 x 739 [341,220,641]
+  CRUSH rule 1 x 740 [874,524,674]
+  CRUSH rule 1 x 741 [189,472,712]
+  CRUSH rule 1 x 742 [912,581,114]
+  CRUSH rule 1 x 743 [654,914,425]
+  CRUSH rule 1 x 744 [725,295,579]
+  CRUSH rule 1 x 745 [787,858,850]
+  CRUSH rule 1 x 746 [757,848,704]
+  CRUSH rule 1 x 747 [700,81,867]
+  CRUSH rule 1 x 748 [557,436,238]
+  CRUSH rule 1 x 749 [772,622,337]
+  CRUSH rule 1 x 750 [946,97,376]
+  CRUSH rule 1 x 751 [996,618,343]
+  CRUSH rule 1 x 752 [746,887,695]
+  CRUSH rule 1 x 753 [741,14,463]
+  CRUSH rule 1 x 754 [648,349,333]
+  CRUSH rule 1 x 755 [157,460,466]
+  CRUSH rule 1 x 756 [416,97,197]
+  CRUSH rule 1 x 757 [599,839,776]
+  CRUSH rule 1 x 758 [994,218,620]
+  CRUSH rule 1 x 759 [959,682,514]
+  CRUSH rule 1 x 760 [518,943,215]
+  CRUSH rule 1 x 761 [285,849,420]
+  CRUSH rule 1 x 762 [591,313,41]
+  CRUSH rule 1 x 763 [908,411,200]
+  CRUSH rule 1 x 764 [787,234,894]
+  CRUSH rule 1 x 765 [327,921,882]
+  CRUSH rule 1 x 766 [84,161,878]
+  CRUSH rule 1 x 767 [370,895,702]
+  CRUSH rule 1 x 768 [826,760,879]
+  CRUSH rule 1 x 769 [67,768,663]
+  CRUSH rule 1 x 770 [593,909,482]
+  CRUSH rule 1 x 771 [309,935,121]
+  CRUSH rule 1 x 772 [12,125,797]
+  CRUSH rule 1 x 773 [253,466,820]
+  CRUSH rule 1 x 774 [164,390,705]
+  CRUSH rule 1 x 775 [703,47,43]
+  CRUSH rule 1 x 776 [728,231,80]
+  CRUSH rule 1 x 777 [981,621,568]
+  CRUSH rule 1 x 778 [411,456,544]
+  CRUSH rule 1 x 779 [346,121,519]
+  CRUSH rule 1 x 780 [476,39,288]
+  CRUSH rule 1 x 781 [10,130,585]
+  CRUSH rule 1 x 782 [462,246,581]
+  CRUSH rule 1 x 783 [580,373,153]
+  CRUSH rule 1 x 784 [413,113,978]
+  CRUSH rule 1 x 785 [341,856,332]
+  CRUSH rule 1 x 786 [411,140,313]
+  CRUSH rule 1 x 787 [605,522,211]
+  CRUSH rule 1 x 788 [226,545,35]
+  CRUSH rule 1 x 789 [545,320,414]
+  CRUSH rule 1 x 790 [414,748,816]
+  CRUSH rule 1 x 791 [660,906,406]
+  CRUSH rule 1 x 792 [287,392,514]
+  CRUSH rule 1 x 793 [631,133,850]
+  CRUSH rule 1 x 794 [931,517,543]
+  CRUSH rule 1 x 795 [551,962,477]
+  CRUSH rule 1 x 796 [814,4,95]
+  CRUSH rule 1 x 797 [64,201,299]
+  CRUSH rule 1 x 798 [422,530,114]
+  CRUSH rule 1 x 799 [824,32,679]
+  CRUSH rule 1 x 800 [862,623,489]
+  CRUSH rule 1 x 801 [145,550,329]
+  CRUSH rule 1 x 802 [570,19,847]
+  CRUSH rule 1 x 803 [151,812,662]
+  CRUSH rule 1 x 804 [467,93,264]
+  CRUSH rule 1 x 805 [621,223,938]
+  CRUSH rule 1 x 806 [898,957,805]
+  CRUSH rule 1 x 807 [354,531,422]
+  CRUSH rule 1 x 808 [7,96,76]
+  CRUSH rule 1 x 809 [70,734,719]
+  CRUSH rule 1 x 810 [701,18,972]
+  CRUSH rule 1 x 811 [248,547,103]
+  CRUSH rule 1 x 812 [230,576,821]
+  CRUSH rule 1 x 813 [805,114,683]
+  CRUSH rule 1 x 814 [54,619,973]
+  CRUSH rule 1 x 815 [679,412,613]
+  CRUSH rule 1 x 816 [919,448,826]
+  CRUSH rule 1 x 817 [765,830,436]
+  CRUSH rule 1 x 818 [415,566,644]
+  CRUSH rule 1 x 819 [721,319,865]
+  CRUSH rule 1 x 820 [218,301,333]
+  CRUSH rule 1 x 821 [185,795,680]
+  CRUSH rule 1 x 822 [356,261,54]
+  CRUSH rule 1 x 823 [220,281,549]
+  CRUSH rule 1 x 824 [292,809,887]
+  CRUSH rule 1 x 825 [949,778,101]
+  CRUSH rule 1 x 826 [767,818,833]
+  CRUSH rule 1 x 827 [631,83,406]
+  CRUSH rule 1 x 828 [288,986,445]
+  CRUSH rule 1 x 829 [990,667,915]
+  CRUSH rule 1 x 830 [152,571,778]
+  CRUSH rule 1 x 831 [814,563,630]
+  CRUSH rule 1 x 832 [235,641,616]
+  CRUSH rule 1 x 833 [657,565,922]
+  CRUSH rule 1 x 834 [907,231,644]
+  CRUSH rule 1 x 835 [784,262,771]
+  CRUSH rule 1 x 836 [951,158,366]
+  CRUSH rule 1 x 837 [556,498,334]
+  CRUSH rule 1 x 838 [329,274,964]
+  CRUSH rule 1 x 839 [568,209,939]
+  CRUSH rule 1 x 840 [45,579,842]
+  CRUSH rule 1 x 841 [652,702,24]
+  CRUSH rule 1 x 842 [629,984,314]
+  CRUSH rule 1 x 843 [799,690,688]
+  CRUSH rule 1 x 844 [694,600,534]
+  CRUSH rule 1 x 845 [332,30,179]
+  CRUSH rule 1 x 846 [452,251,712]
+  CRUSH rule 1 x 847 [399,681,847]
+  CRUSH rule 1 x 848 [303,138,440]
+  CRUSH rule 1 x 849 [666,346,708]
+  CRUSH rule 1 x 850 [644,511,345]
+  CRUSH rule 1 x 851 [527,546,737]
+  CRUSH rule 1 x 852 [31,809,94]
+  CRUSH rule 1 x 853 [483,330,869]
+  CRUSH rule 1 x 854 [697,953,968]
+  CRUSH rule 1 x 855 [837,996,239]
+  CRUSH rule 1 x 856 [712,40,547]
+  CRUSH rule 1 x 857 [77,984,576]
+  CRUSH rule 1 x 858 [412,384,841]
+  CRUSH rule 1 x 859 [173,760,26]
+  CRUSH rule 1 x 860 [776,429,328]
+  CRUSH rule 1 x 861 [705,405,477]
+  CRUSH rule 1 x 862 [809,44,788]
+  CRUSH rule 1 x 863 [349,496,963]
+  CRUSH rule 1 x 864 [717,858,101]
+  CRUSH rule 1 x 865 [857,603,586]
+  CRUSH rule 1 x 866 [394,304,71]
+  CRUSH rule 1 x 867 [640,773,663]
+  CRUSH rule 1 x 868 [613,950,712]
+  CRUSH rule 1 x 869 [973,889,524]
+  CRUSH rule 1 x 870 [505,35,386]
+  CRUSH rule 1 x 871 [239,264,262]
+  CRUSH rule 1 x 872 [21,767,456]
+  CRUSH rule 1 x 873 [954,666,980]
+  CRUSH rule 1 x 874 [54,510,947]
+  CRUSH rule 1 x 875 [809,418,452]
+  CRUSH rule 1 x 876 [483,457,61]
+  CRUSH rule 1 x 877 [542,531,952]
+  CRUSH rule 1 x 878 [217,674,857]
+  CRUSH rule 1 x 879 [999,475,134]
+  CRUSH rule 1 x 880 [678,573,935]
+  CRUSH rule 1 x 881 [394,835,789]
+  CRUSH rule 1 x 882 [467,382,353]
+  CRUSH rule 1 x 883 [802,744,237]
+  CRUSH rule 1 x 884 [653,660,638]
+  CRUSH rule 1 x 885 [898,704,307]
+  CRUSH rule 1 x 886 [434,357,938]
+  CRUSH rule 1 x 887 [297,226,711]
+  CRUSH rule 1 x 888 [863,324,443]
+  CRUSH rule 1 x 889 [105,102,308]
+  CRUSH rule 1 x 890 [550,248,606]
+  CRUSH rule 1 x 891 [575,928,880]
+  CRUSH rule 1 x 892 [259,862,133]
+  CRUSH rule 1 x 893 [902,880,543]
+  CRUSH rule 1 x 894 [180,169,916]
+  CRUSH rule 1 x 895 [725,849,182]
+  CRUSH rule 1 x 896 [951,34,874]
+  CRUSH rule 1 x 897 [810,352,73]
+  CRUSH rule 1 x 898 [979,433,719]
+  CRUSH rule 1 x 899 [685,668,534]
+  CRUSH rule 1 x 900 [530,978,41]
+  CRUSH rule 1 x 901 [740,107,336]
+  CRUSH rule 1 x 902 [800,743,693]
+  CRUSH rule 1 x 903 [230,267,842]
+  CRUSH rule 1 x 904 [346,949,460]
+  CRUSH rule 1 x 905 [530,397,619]
+  CRUSH rule 1 x 906 [80,426,138]
+  CRUSH rule 1 x 907 [365,968,475]
+  CRUSH rule 1 x 908 [204,832,742]
+  CRUSH rule 1 x 909 [883,989,146]
+  CRUSH rule 1 x 910 [549,593,249]
+  CRUSH rule 1 x 911 [325,847,352]
+  CRUSH rule 1 x 912 [874,888,582]
+  CRUSH rule 1 x 913 [331,463,342]
+  CRUSH rule 1 x 914 [836,468,601]
+  CRUSH rule 1 x 915 [245,228,100]
+  CRUSH rule 1 x 916 [77,967,364]
+  CRUSH rule 1 x 917 [239,60,866]
+  CRUSH rule 1 x 918 [988,115,922]
+  CRUSH rule 1 x 919 [783,139,696]
+  CRUSH rule 1 x 920 [623,408,685]
+  CRUSH rule 1 x 921 [105,799,144]
+  CRUSH rule 1 x 922 [887,505,652]
+  CRUSH rule 1 x 923 [223,318,552]
+  CRUSH rule 1 x 924 [25,778,366]
+  CRUSH rule 1 x 925 [912,601,297]
+  CRUSH rule 1 x 926 [968,133,155]
+  CRUSH rule 1 x 927 [277,724,214]
+  CRUSH rule 1 x 928 [554,203,658]
+  CRUSH rule 1 x 929 [761,802,367]
+  CRUSH rule 1 x 930 [814,61,788]
+  CRUSH rule 1 x 931 [29,193,61]
+  CRUSH rule 1 x 932 [446,198,862]
+  CRUSH rule 1 x 933 [352,742,216]
+  CRUSH rule 1 x 934 [730,2,332]
+  CRUSH rule 1 x 935 [731,23,736]
+  CRUSH rule 1 x 936 [322,975,20]
+  CRUSH rule 1 x 937 [822,221,841]
+  CRUSH rule 1 x 938 [557,850,66]
+  CRUSH rule 1 x 939 [150,11,971]
+  CRUSH rule 1 x 940 [638,398,169]
+  CRUSH rule 1 x 941 [730,342,929]
+  CRUSH rule 1 x 942 [62,292,166]
+  CRUSH rule 1 x 943 [165,314,519]
+  CRUSH rule 1 x 944 [199,625,766]
+  CRUSH rule 1 x 945 [946,999,699]
+  CRUSH rule 1 x 946 [595,93,852]
+  CRUSH rule 1 x 947 [800,582,356]
+  CRUSH rule 1 x 948 [132,551,139]
+  CRUSH rule 1 x 949 [792,920,466]
+  CRUSH rule 1 x 950 [111,345,176]
+  CRUSH rule 1 x 951 [414,619,648]
+  CRUSH rule 1 x 952 [775,469,500]
+  CRUSH rule 1 x 953 [349,1,5]
+  CRUSH rule 1 x 954 [570,940,410]
+  CRUSH rule 1 x 955 [729,774,823]
+  CRUSH rule 1 x 956 [519,141,575]
+  CRUSH rule 1 x 957 [242,709,611]
+  CRUSH rule 1 x 958 [84,217,227]
+  CRUSH rule 1 x 959 [270,413,918]
+  CRUSH rule 1 x 960 [458,192,307]
+  CRUSH rule 1 x 961 [981,388,777]
+  CRUSH rule 1 x 962 [623,834,277]
+  CRUSH rule 1 x 963 [291,167,714]
+  CRUSH rule 1 x 964 [28,156,788]
+  CRUSH rule 1 x 965 [675,557,290]
+  CRUSH rule 1 x 966 [836,306,946]
+  CRUSH rule 1 x 967 [966,386,735]
+  CRUSH rule 1 x 968 [864,756,690]
+  CRUSH rule 1 x 969 [729,625,480]
+  CRUSH rule 1 x 970 [800,362,646]
+  CRUSH rule 1 x 971 [737,381,153]
+  CRUSH rule 1 x 972 [952,245,720]
+  CRUSH rule 1 x 973 [356,455,579]
+  CRUSH rule 1 x 974 [545,758,586]
+  CRUSH rule 1 x 975 [336,191,202]
+  CRUSH rule 1 x 976 [446,208,757]
+  CRUSH rule 1 x 977 [202,896,196]
+  CRUSH rule 1 x 978 [612,324,996]
+  CRUSH rule 1 x 979 [843,457,675]
+  CRUSH rule 1 x 980 [60,914,881]
+  CRUSH rule 1 x 981 [702,749,937]
+  CRUSH rule 1 x 982 [298,928,738]
+  CRUSH rule 1 x 983 [723,572,395]
+  CRUSH rule 1 x 984 [723,864,804]
+  CRUSH rule 1 x 985 [945,459,868]
+  CRUSH rule 1 x 986 [772,664,535]
+  CRUSH rule 1 x 987 [88,324,312]
+  CRUSH rule 1 x 988 [522,927,131]
+  CRUSH rule 1 x 989 [578,332,208]
+  CRUSH rule 1 x 990 [638,228,414]
+  CRUSH rule 1 x 991 [530,221,451]
+  CRUSH rule 1 x 992 [925,705,275]
+  CRUSH rule 1 x 993 [991,301,43]
+  CRUSH rule 1 x 994 [276,51,868]
+  CRUSH rule 1 x 995 [288,836,753]
+  CRUSH rule 1 x 996 [887,983,252]
+  CRUSH rule 1 x 997 [110,924,386]
+  CRUSH rule 1 x 998 [435,830,485]
+  CRUSH rule 1 x 999 [876,738,357]
+  CRUSH rule 1 x 1000 [178,963,638]
+  CRUSH rule 1 x 1001 [99,519,66]
+  CRUSH rule 1 x 1002 [515,534,468]
+  CRUSH rule 1 x 1003 [104,611,937]
+  CRUSH rule 1 x 1004 [269,638,724]
+  CRUSH rule 1 x 1005 [369,223,309]
+  CRUSH rule 1 x 1006 [40,107,69]
+  CRUSH rule 1 x 1007 [978,111,416]
+  CRUSH rule 1 x 1008 [965,956,624]
+  CRUSH rule 1 x 1009 [598,476,356]
+  CRUSH rule 1 x 1010 [767,523,239]
+  CRUSH rule 1 x 1011 [289,871,207]
+  CRUSH rule 1 x 1012 [128,28,370]
+  CRUSH rule 1 x 1013 [979,765,660]
+  CRUSH rule 1 x 1014 [979,948,513]
+  CRUSH rule 1 x 1015 [277,790,396]
+  CRUSH rule 1 x 1016 [262,73,128]
+  CRUSH rule 1 x 1017 [150,269,61]
+  CRUSH rule 1 x 1018 [555,829,554]
+  CRUSH rule 1 x 1019 [513,356,265]
+  CRUSH rule 1 x 1020 [158,161,877]
+  CRUSH rule 1 x 1021 [915,998,957]
+  CRUSH rule 1 x 1022 [967,829,973]
+  CRUSH rule 1 x 1023 [488,257,614]
+  rule 1 (metadata) num_rep 3 result size == 3:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536,450]
+  CRUSH rule 1 x 1 [876,250,334,633]
+  CRUSH rule 1 x 2 [292,832,53,392]
+  CRUSH rule 1 x 3 [623,387,124,998]
+  CRUSH rule 1 x 4 [61,334,710,4]
+  CRUSH rule 1 x 5 [946,557,713,664]
+  CRUSH rule 1 x 6 [576,668,212,163]
+  CRUSH rule 1 x 7 [645,753,906,393]
+  CRUSH rule 1 x 8 [243,6,863,781]
+  CRUSH rule 1 x 9 [22,578,251,410]
+  CRUSH rule 1 x 10 [758,828,360,477]
+  CRUSH rule 1 x 11 [769,120,124,527]
+  CRUSH rule 1 x 12 [780,364,689,755]
+  CRUSH rule 1 x 13 [557,18,351,719]
+  CRUSH rule 1 x 14 [59,561,249,461]
+  CRUSH rule 1 x 15 [718,928,993,21]
+  CRUSH rule 1 x 16 [673,632,841,954]
+  CRUSH rule 1 x 17 [648,43,560,514]
+  CRUSH rule 1 x 18 [654,219,181,568]
+  CRUSH rule 1 x 19 [850,545,377,848]
+  CRUSH rule 1 x 20 [717,785,974,5]
+  CRUSH rule 1 x 21 [420,57,519,306]
+  CRUSH rule 1 x 22 [503,998,193,821]
+  CRUSH rule 1 x 23 [411,663,168,110]
+  CRUSH rule 1 x 24 [266,861,353,1]
+  CRUSH rule 1 x 25 [760,483,818,600]
+  CRUSH rule 1 x 26 [903,24,573,718]
+  CRUSH rule 1 x 27 [946,188,289,510]
+  CRUSH rule 1 x 28 [69,312,73,198]
+  CRUSH rule 1 x 29 [844,883,337,628]
+  CRUSH rule 1 x 30 [621,18,613,794]
+  CRUSH rule 1 x 31 [784,943,814,539]
+  CRUSH rule 1 x 32 [173,374,369,972]
+  CRUSH rule 1 x 33 [698,336,357,966]
+  CRUSH rule 1 x 34 [168,836,210,798]
+  CRUSH rule 1 x 35 [274,509,534,818]
+  CRUSH rule 1 x 36 [318,215,153,628]
+  CRUSH rule 1 x 37 [173,604,109,935]
+  CRUSH rule 1 x 38 [708,444,683,604]
+  CRUSH rule 1 x 39 [662,198,417,680]
+  CRUSH rule 1 x 40 [620,801,414,78]
+  CRUSH rule 1 x 41 [811,264,177,127]
+  CRUSH rule 1 x 42 [863,179,527,660]
+  CRUSH rule 1 x 43 [686,822,988,228]
+  CRUSH rule 1 x 44 [396,222,46,841]
+  CRUSH rule 1 x 45 [991,694,253,142]
+  CRUSH rule 1 x 46 [420,909,184,285]
+  CRUSH rule 1 x 47 [467,211,605,207]
+  CRUSH rule 1 x 48 [955,329,368,168]
+  CRUSH rule 1 x 49 [974,891,931,29]
+  CRUSH rule 1 x 50 [870,441,691,823]
+  CRUSH rule 1 x 51 [182,930,25,936]
+  CRUSH rule 1 x 52 [704,812,894,794]
+  CRUSH rule 1 x 53 [185,713,631,280]
+  CRUSH rule 1 x 54 [270,441,100,82]
+  CRUSH rule 1 x 55 [895,734,958,793]
+  CRUSH rule 1 x 56 [564,963,683,324]
+  CRUSH rule 1 x 57 [738,130,208,973]
+  CRUSH rule 1 x 58 [524,113,806,903]
+  CRUSH rule 1 x 59 [408,337,668,529]
+  CRUSH rule 1 x 60 [228,790,857,309]
+  CRUSH rule 1 x 61 [154,843,717,467]
+  CRUSH rule 1 x 62 [594,811,549,276]
+  CRUSH rule 1 x 63 [646,67,884,925]
+  CRUSH rule 1 x 64 [175,542,155,837]
+  CRUSH rule 1 x 65 [745,619,131,867]
+  CRUSH rule 1 x 66 [275,468,23,35]
+  CRUSH rule 1 x 67 [246,958,524,493]
+  CRUSH rule 1 x 68 [711,473,403,228]
+  CRUSH rule 1 x 69 [493,924,850,939]
+  CRUSH rule 1 x 70 [30,499,644,33]
+  CRUSH rule 1 x 71 [984,883,574,716]
+  CRUSH rule 1 x 72 [71,286,942,363]
+  CRUSH rule 1 x 73 [922,618,3,371]
+  CRUSH rule 1 x 74 [629,414,185,573]
+  CRUSH rule 1 x 75 [222,20,174,820]
+  CRUSH rule 1 x 76 [262,366,339,290]
+  CRUSH rule 1 x 77 [638,469,992,280]
+  CRUSH rule 1 x 78 [324,511,788,7]
+  CRUSH rule 1 x 79 [577,990,64,94]
+  CRUSH rule 1 x 80 [501,95,278,903]
+  CRUSH rule 1 x 81 [506,812,9,698]
+  CRUSH rule 1 x 82 [222,145,80,785]
+  CRUSH rule 1 x 83 [71,634,61,91]
+  CRUSH rule 1 x 84 [49,761,773,368]
+  CRUSH rule 1 x 85 [985,896,708,861]
+  CRUSH rule 1 x 86 [537,745,93,524]
+  CRUSH rule 1 x 87 [997,317,463,626]
+  CRUSH rule 1 x 88 [957,350,890,857]
+  CRUSH rule 1 x 89 [399,730,148,314]
+  CRUSH rule 1 x 90 [943,706,683,267]
+  CRUSH rule 1 x 91 [22,368,149,928]
+  CRUSH rule 1 x 92 [532,424,426,773]
+  CRUSH rule 1 x 93 [218,489,405,681]
+  CRUSH rule 1 x 94 [181,96,102,515]
+  CRUSH rule 1 x 95 [343,957,820,139]
+  CRUSH rule 1 x 96 [861,270,87,797]
+  CRUSH rule 1 x 97 [459,706,45,328]
+  CRUSH rule 1 x 98 [327,867,353,948]
+  CRUSH rule 1 x 99 [974,133,468,906]
+  CRUSH rule 1 x 100 [32,445,547,371]
+  CRUSH rule 1 x 101 [142,90,337,950]
+  CRUSH rule 1 x 102 [172,129,139,22]
+  CRUSH rule 1 x 103 [630,47,161,356]
+  CRUSH rule 1 x 104 [758,133,278,11]
+  CRUSH rule 1 x 105 [843,604,47,33]
+  CRUSH rule 1 x 106 [28,681,193,679]
+  CRUSH rule 1 x 107 [74,320,85,819]
+  CRUSH rule 1 x 108 [875,593,575,517]
+  CRUSH rule 1 x 109 [411,985,811,720]
+  CRUSH rule 1 x 110 [440,774,799,660]
+  CRUSH rule 1 x 111 [405,742,276,359]
+  CRUSH rule 1 x 112 [143,181,922,545]
+  CRUSH rule 1 x 113 [153,846,160,903]
+  CRUSH rule 1 x 114 [804,892,939,20]
+  CRUSH rule 1 x 115 [588,508,958,580]
+  CRUSH rule 1 x 116 [327,148,637,486]
+  CRUSH rule 1 x 117 [95,594,989,131]
+  CRUSH rule 1 x 118 [80,957,897,239]
+  CRUSH rule 1 x 119 [386,932,951,768]
+  CRUSH rule 1 x 120 [366,312,653,936]
+  CRUSH rule 1 x 121 [129,154,847,16]
+  CRUSH rule 1 x 122 [873,1,110,939]
+  CRUSH rule 1 x 123 [533,415,789,600]
+  CRUSH rule 1 x 124 [461,691,898,723]
+  CRUSH rule 1 x 125 [342,599,830,402]
+  CRUSH rule 1 x 126 [819,781,822,548]
+  CRUSH rule 1 x 127 [437,893,585,707]
+  CRUSH rule 1 x 128 [679,994,982,550]
+  CRUSH rule 1 x 129 [380,685,947,302]
+  CRUSH rule 1 x 130 [992,52,466,867]
+  CRUSH rule 1 x 131 [469,90,208,599]
+  CRUSH rule 1 x 132 [571,250,316,535]
+  CRUSH rule 1 x 133 [964,728,329,902]
+  CRUSH rule 1 x 134 [999,19,716,963]
+  CRUSH rule 1 x 135 [634,101,52,938]
+  CRUSH rule 1 x 136 [114,889,692,768]
+  CRUSH rule 1 x 137 [839,8,959,280]
+  CRUSH rule 1 x 138 [967,949,138,451]
+  CRUSH rule 1 x 139 [308,711,736,247]
+  CRUSH rule 1 x 140 [764,936,926,55]
+  CRUSH rule 1 x 141 [423,302,112,216]
+  CRUSH rule 1 x 142 [252,821,715,340]
+  CRUSH rule 1 x 143 [33,808,518,477]
+  CRUSH rule 1 x 144 [472,88,969,162]
+  CRUSH rule 1 x 145 [242,208,252,604]
+  CRUSH rule 1 x 146 [290,70,570,384]
+  CRUSH rule 1 x 147 [447,352,657,493]
+  CRUSH rule 1 x 148 [212,644,432,658]
+  CRUSH rule 1 x 149 [9,775,87,35]
+  CRUSH rule 1 x 150 [166,456,582,144]
+  CRUSH rule 1 x 151 [811,875,307,20]
+  CRUSH rule 1 x 152 [449,617,223,9]
+  CRUSH rule 1 x 153 [523,537,695,627]
+  CRUSH rule 1 x 154 [208,559,874,597]
+  CRUSH rule 1 x 155 [569,325,192,296]
+  CRUSH rule 1 x 156 [488,121,521,213]
+  CRUSH rule 1 x 157 [140,723,633,260]
+  CRUSH rule 1 x 158 [786,451,320,239]
+  CRUSH rule 1 x 159 [134,664,517,821]
+  CRUSH rule 1 x 160 [690,112,414,990]
+  CRUSH rule 1 x 161 [324,912,397,423]
+  CRUSH rule 1 x 162 [748,567,284,183]
+  CRUSH rule 1 x 163 [575,499,31,816]
+  CRUSH rule 1 x 164 [314,489,308,326]
+  CRUSH rule 1 x 165 [116,209,750,53]
+  CRUSH rule 1 x 166 [352,706,701,810]
+  CRUSH rule 1 x 167 [27,743,174,142]
+  CRUSH rule 1 x 168 [953,898,880,660]
+  CRUSH rule 1 x 169 [912,147,266,547]
+  CRUSH rule 1 x 170 [421,515,828,844]
+  CRUSH rule 1 x 171 [488,584,880,964]
+  CRUSH rule 1 x 172 [366,443,957,66]
+  CRUSH rule 1 x 173 [863,291,625,287]
+  CRUSH rule 1 x 174 [263,555,650,410]
+  CRUSH rule 1 x 175 [875,961,361,575]
+  CRUSH rule 1 x 176 [745,83,701,680]
+  CRUSH rule 1 x 177 [128,244,41,123]
+  CRUSH rule 1 x 178 [155,41,264,777]
+  CRUSH rule 1 x 179 [593,833,202,183]
+  CRUSH rule 1 x 180 [154,734,17,831]
+  CRUSH rule 1 x 181 [289,675,723,800]
+  CRUSH rule 1 x 182 [730,931,560,209]
+  CRUSH rule 1 x 183 [639,237,794,815]
+  CRUSH rule 1 x 184 [704,312,685,645]
+  CRUSH rule 1 x 185 [97,100,762,82]
+  CRUSH rule 1 x 186 [26,665,554,215]
+  CRUSH rule 1 x 187 [649,14,740,494]
+  CRUSH rule 1 x 188 [682,695,590,743]
+  CRUSH rule 1 x 189 [325,693,726,51]
+  CRUSH rule 1 x 190 [399,933,136,955]
+  CRUSH rule 1 x 191 [629,533,17,126]
+  CRUSH rule 1 x 192 [503,578,38,492]
+  CRUSH rule 1 x 193 [546,333,651,678]
+  CRUSH rule 1 x 194 [242,473,58,655]
+  CRUSH rule 1 x 195 [625,719,135,81]
+  CRUSH rule 1 x 196 [357,114,125,867]
+  CRUSH rule 1 x 197 [306,954,453,873]
+  CRUSH rule 1 x 198 [863,791,311,911]
+  CRUSH rule 1 x 199 [935,906,929,252]
+  CRUSH rule 1 x 200 [373,774,229,454]
+  CRUSH rule 1 x 201 [659,320,477,313]
+  CRUSH rule 1 x 202 [260,433,524,880]
+  CRUSH rule 1 x 203 [36,239,675,971]
+  CRUSH rule 1 x 204 [92,516,993,728]
+  CRUSH rule 1 x 205 [68,395,473,45]
+  CRUSH rule 1 x 206 [570,530,642,380]
+  CRUSH rule 1 x 207 [834,457,850,917]
+  CRUSH rule 1 x 208 [927,484,640,976]
+  CRUSH rule 1 x 209 [878,66,58,940]
+  CRUSH rule 1 x 210 [572,981,484,29]
+  CRUSH rule 1 x 211 [107,597,780,857]
+  CRUSH rule 1 x 212 [389,107,838,624]
+  CRUSH rule 1 x 213 [497,717,567,728]
+  CRUSH rule 1 x 214 [798,65,254,572]
+  CRUSH rule 1 x 215 [233,419,283,638]
+  CRUSH rule 1 x 216 [494,464,742,523]
+  CRUSH rule 1 x 217 [352,396,309,938]
+  CRUSH rule 1 x 218 [895,864,988,650]
+  CRUSH rule 1 x 219 [222,534,277,242]
+  CRUSH rule 1 x 220 [281,19,584,563]
+  CRUSH rule 1 x 221 [64,928,963,130]
+  CRUSH rule 1 x 222 [40,544,161,199]
+  CRUSH rule 1 x 223 [645,556,159,417]
+  CRUSH rule 1 x 224 [647,165,957,263]
+  CRUSH rule 1 x 225 [219,714,858,747]
+  CRUSH rule 1 x 226 [372,511,181,277]
+  CRUSH rule 1 x 227 [925,156,714,863]
+  CRUSH rule 1 x 228 [682,404,839,263]
+  CRUSH rule 1 x 229 [880,838,770,891]
+  CRUSH rule 1 x 230 [328,659,916,468]
+  CRUSH rule 1 x 231 [320,383,669,109]
+  CRUSH rule 1 x 232 [924,846,394,319]
+  CRUSH rule 1 x 233 [948,652,575,838]
+  CRUSH rule 1 x 234 [484,943,42,575]
+  CRUSH rule 1 x 235 [750,65,590,168]
+  CRUSH rule 1 x 236 [551,787,490,136]
+  CRUSH rule 1 x 237 [390,157,166,251]
+  CRUSH rule 1 x 238 [570,6,989,707]
+  CRUSH rule 1 x 239 [729,959,376,975]
+  CRUSH rule 1 x 240 [981,241,156,767]
+  CRUSH rule 1 x 241 [310,816,641,177]
+  CRUSH rule 1 x 242 [161,63,642,837]
+  CRUSH rule 1 x 243 [180,394,33,683]
+  CRUSH rule 1 x 244 [52,174,685,189]
+  CRUSH rule 1 x 245 [523,121,915,84]
+  CRUSH rule 1 x 246 [362,893,390,487]
+  CRUSH rule 1 x 247 [382,184,116,34]
+  CRUSH rule 1 x 248 [129,114,852,469]
+  CRUSH rule 1 x 249 [159,683,91,856]
+  CRUSH rule 1 x 250 [404,945,569,955]
+  CRUSH rule 1 x 251 [661,225,738,757]
+  CRUSH rule 1 x 252 [961,226,542,103]
+  CRUSH rule 1 x 253 [651,97,225,364]
+  CRUSH rule 1 x 254 [123,33,741,692]
+  CRUSH rule 1 x 255 [314,649,891,855]
+  CRUSH rule 1 x 256 [315,215,651,126]
+  CRUSH rule 1 x 257 [825,264,867,732]
+  CRUSH rule 1 x 258 [624,789,370,723]
+  CRUSH rule 1 x 259 [602,542,70,563]
+  CRUSH rule 1 x 260 [717,878,43,56]
+  CRUSH rule 1 x 261 [145,517,20,903]
+  CRUSH rule 1 x 262 [223,1,561,420]
+  CRUSH rule 1 x 263 [462,211,405,508]
+  CRUSH rule 1 x 264 [654,471,266,662]
+  CRUSH rule 1 x 265 [302,794,704,798]
+  CRUSH rule 1 x 266 [202,132,884,209]
+  CRUSH rule 1 x 267 [282,938,657,113]
+  CRUSH rule 1 x 268 [338,309,356,278]
+  CRUSH rule 1 x 269 [738,122,266,200]
+  CRUSH rule 1 x 270 [707,982,946,196]
+  CRUSH rule 1 x 271 [705,432,364,735]
+  CRUSH rule 1 x 272 [756,545,942,56]
+  CRUSH rule 1 x 273 [197,502,527,721]
+  CRUSH rule 1 x 274 [992,44,653,573]
+  CRUSH rule 1 x 275 [544,789,170,434]
+  CRUSH rule 1 x 276 [658,467,577,268]
+  CRUSH rule 1 x 277 [143,490,880,483]
+  CRUSH rule 1 x 278 [492,647,355,282]
+  CRUSH rule 1 x 279 [517,792,604,987]
+  CRUSH rule 1 x 280 [825,740,27,848]
+  CRUSH rule 1 x 281 [224,629,120,562]
+  CRUSH rule 1 x 282 [298,661,380,416]
+  CRUSH rule 1 x 283 [311,606,208,50]
+  CRUSH rule 1 x 284 [771,466,371,743]
+  CRUSH rule 1 x 285 [693,362,404,676]
+  CRUSH rule 1 x 286 [364,477,285,167]
+  CRUSH rule 1 x 287 [591,611,828,995]
+  CRUSH rule 1 x 288 [965,541,848,796]
+  CRUSH rule 1 x 289 [225,551,948,877]
+  CRUSH rule 1 x 290 [577,762,777,751]
+  CRUSH rule 1 x 291 [160,903,477,381]
+  CRUSH rule 1 x 292 [873,598,216,666]
+  CRUSH rule 1 x 293 [100,234,874,47]
+  CRUSH rule 1 x 294 [285,943,379,520]
+  CRUSH rule 1 x 295 [938,262,880,327]
+  CRUSH rule 1 x 296 [850,327,86,472]
+  CRUSH rule 1 x 297 [951,53,99,558]
+  CRUSH rule 1 x 298 [173,336,85,766]
+  CRUSH rule 1 x 299 [598,591,315,386]
+  CRUSH rule 1 x 300 [531,957,62,459]
+  CRUSH rule 1 x 301 [823,628,23,858]
+  CRUSH rule 1 x 302 [184,80,780,871]
+  CRUSH rule 1 x 303 [521,766,222,830]
+  CRUSH rule 1 x 304 [980,127,807,507]
+  CRUSH rule 1 x 305 [153,816,22,927]
+  CRUSH rule 1 x 306 [423,739,664,753]
+  CRUSH rule 1 x 307 [997,557,682,456]
+  CRUSH rule 1 x 308 [991,874,534,465]
+  CRUSH rule 1 x 309 [860,394,724,858]
+  CRUSH rule 1 x 310 [589,818,546,201]
+  CRUSH rule 1 x 311 [477,774,225,590]
+  CRUSH rule 1 x 312 [887,853,950,354]
+  CRUSH rule 1 x 313 [802,646,447,416]
+  CRUSH rule 1 x 314 [654,974,229,511]
+  CRUSH rule 1 x 315 [767,227,28,740]
+  CRUSH rule 1 x 316 [778,83,733,359]
+  CRUSH rule 1 x 317 [184,418,642,986]
+  CRUSH rule 1 x 318 [525,410,500,543]
+  CRUSH rule 1 x 319 [476,724,569,382]
+  CRUSH rule 1 x 320 [149,610,697,296]
+  CRUSH rule 1 x 321 [710,79,667,671]
+  CRUSH rule 1 x 322 [175,275,323,333]
+  CRUSH rule 1 x 323 [819,604,638,792]
+  CRUSH rule 1 x 324 [16,745,511,439]
+  CRUSH rule 1 x 325 [486,400,872,873]
+  CRUSH rule 1 x 326 [613,765,207,19]
+  CRUSH rule 1 x 327 [125,289,738,408]
+  CRUSH rule 1 x 328 [807,383,476,583]
+  CRUSH rule 1 x 329 [588,938,599,432]
+  CRUSH rule 1 x 330 [932,644,41,611]
+  CRUSH rule 1 x 331 [341,953,950,537]
+  CRUSH rule 1 x 332 [153,726,459,950]
+  CRUSH rule 1 x 333 [745,845,853,860]
+  CRUSH rule 1 x 334 [614,751,807,58]
+  CRUSH rule 1 x 335 [518,721,221,283]
+  CRUSH rule 1 x 336 [389,424,77,309]
+  CRUSH rule 1 x 337 [753,508,765,720]
+  CRUSH rule 1 x 338 [128,810,490,753]
+  CRUSH rule 1 x 339 [430,308,58,751]
+  CRUSH rule 1 x 340 [541,44,630,231]
+  CRUSH rule 1 x 341 [402,26,631,439]
+  CRUSH rule 1 x 342 [982,57,992,461]
+  CRUSH rule 1 x 343 [833,412,572,732]
+  CRUSH rule 1 x 344 [784,533,792,41]
+  CRUSH rule 1 x 345 [546,300,304,691]
+  CRUSH rule 1 x 346 [302,420,428,891]
+  CRUSH rule 1 x 347 [488,778,101,217]
+  CRUSH rule 1 x 348 [903,744,937,718]
+  CRUSH rule 1 x 349 [471,547,582,306]
+  CRUSH rule 1 x 350 [348,221,823,335]
+  CRUSH rule 1 x 351 [961,582,705,346]
+  CRUSH rule 1 x 352 [728,137,461,298]
+  CRUSH rule 1 x 353 [904,202,184,447]
+  CRUSH rule 1 x 354 [345,226,319,256]
+  CRUSH rule 1 x 355 [50,430,175,43]
+  CRUSH rule 1 x 356 [87,185,55,423]
+  CRUSH rule 1 x 357 [762,459,921,473]
+  CRUSH rule 1 x 358 [908,25,280,6]
+  CRUSH rule 1 x 359 [484,15,132,121]
+  CRUSH rule 1 x 360 [173,378,337,702]
+  CRUSH rule 1 x 361 [404,577,115,25]
+  CRUSH rule 1 x 362 [403,1,422,945]
+  CRUSH rule 1 x 363 [639,911,510,162]
+  CRUSH rule 1 x 364 [752,689,610,990]
+  CRUSH rule 1 x 365 [956,999,212,230]
+  CRUSH rule 1 x 366 [860,925,924,763]
+  CRUSH rule 1 x 367 [205,609,647,665]
+  CRUSH rule 1 x 368 [301,284,810,169]
+  CRUSH rule 1 x 369 [452,658,339,217]
+  CRUSH rule 1 x 370 [11,467,695,989]
+  CRUSH rule 1 x 371 [124,487,55,514]
+  CRUSH rule 1 x 372 [253,48,979,846]
+  CRUSH rule 1 x 373 [715,605,775,748]
+  CRUSH rule 1 x 374 [191,887,920,760]
+  CRUSH rule 1 x 375 [711,385,651,665]
+  CRUSH rule 1 x 376 [597,818,49,458]
+  CRUSH rule 1 x 377 [294,256,933,771]
+  CRUSH rule 1 x 378 [34,151,681,707]
+  CRUSH rule 1 x 379 [869,136,315,378]
+  CRUSH rule 1 x 380 [294,97,575,791]
+  CRUSH rule 1 x 381 [119,710,219,827]
+  CRUSH rule 1 x 382 [69,631,508,706]
+  CRUSH rule 1 x 383 [922,588,589,925]
+  CRUSH rule 1 x 384 [221,945,671,117]
+  CRUSH rule 1 x 385 [561,737,953,723]
+  CRUSH rule 1 x 386 [335,442,788,696]
+  CRUSH rule 1 x 387 [514,43,353,88]
+  CRUSH rule 1 x 388 [587,89,157,996]
+  CRUSH rule 1 x 389 [109,641,255,466]
+  CRUSH rule 1 x 390 [925,149,421,489]
+  CRUSH rule 1 x 391 [267,87,387,527]
+  CRUSH rule 1 x 392 [382,485,370,849]
+  CRUSH rule 1 x 393 [425,721,221,753]
+  CRUSH rule 1 x 394 [898,18,38,793]
+  CRUSH rule 1 x 395 [806,876,269,679]
+  CRUSH rule 1 x 396 [790,970,437,449]
+  CRUSH rule 1 x 397 [136,363,507,613]
+  CRUSH rule 1 x 398 [914,116,558,258]
+  CRUSH rule 1 x 399 [261,94,299,202]
+  CRUSH rule 1 x 400 [661,197,338,461]
+  CRUSH rule 1 x 401 [953,979,287,803]
+  CRUSH rule 1 x 402 [738,819,618,522]
+  CRUSH rule 1 x 403 [573,238,425,546]
+  CRUSH rule 1 x 404 [526,848,790,253]
+  CRUSH rule 1 x 405 [582,505,330,334]
+  CRUSH rule 1 x 406 [768,324,493,60]
+  CRUSH rule 1 x 407 [260,951,437,587]
+  CRUSH rule 1 x 408 [657,81,770,734]
+  CRUSH rule 1 x 409 [498,89,182,423]
+  CRUSH rule 1 x 410 [28,793,737,352]
+  CRUSH rule 1 x 411 [684,992,60,659]
+  CRUSH rule 1 x 412 [261,958,699,950]
+  CRUSH rule 1 x 413 [891,835,297,441]
+  CRUSH rule 1 x 414 [127,459,119,965]
+  CRUSH rule 1 x 415 [272,540,631,328]
+  CRUSH rule 1 x 416 [739,617,115,530]
+  CRUSH rule 1 x 417 [106,209,157,878]
+  CRUSH rule 1 x 418 [525,441,147,390]
+  CRUSH rule 1 x 419 [603,673,615,465]
+  CRUSH rule 1 x 420 [988,213,251,226]
+  CRUSH rule 1 x 421 [761,521,748,368]
+  CRUSH rule 1 x 422 [317,160,924,548]
+  CRUSH rule 1 x 423 [137,807,168,472]
+  CRUSH rule 1 x 424 [920,37,146,263]
+  CRUSH rule 1 x 425 [277,693,285,221]
+  CRUSH rule 1 x 426 [485,936,407,854]
+  CRUSH rule 1 x 427 [242,515,9,564]
+  CRUSH rule 1 x 428 [632,635,26,473]
+  CRUSH rule 1 x 429 [641,73,465,127]
+  CRUSH rule 1 x 430 [626,585,6,387]
+  CRUSH rule 1 x 431 [697,76,753,570]
+  CRUSH rule 1 x 432 [590,526,306,283]
+  CRUSH rule 1 x 433 [284,387,149,817]
+  CRUSH rule 1 x 434 [538,985,79,953]
+  CRUSH rule 1 x 435 [30,318,593,635]
+  CRUSH rule 1 x 436 [164,919,851,693]
+  CRUSH rule 1 x 437 [322,212,163,606]
+  CRUSH rule 1 x 438 [142,392,85,594]
+  CRUSH rule 1 x 439 [119,370,68,443]
+  CRUSH rule 1 x 440 [333,403,187,863]
+  CRUSH rule 1 x 441 [477,727,906,145]
+  CRUSH rule 1 x 442 [274,590,933,244]
+  CRUSH rule 1 x 443 [983,748,574,718]
+  CRUSH rule 1 x 444 [536,509,431,146]
+  CRUSH rule 1 x 445 [485,753,528,209]
+  CRUSH rule 1 x 446 [345,634,42,294]
+  CRUSH rule 1 x 447 [61,845,767,600]
+  CRUSH rule 1 x 448 [333,232,292,846]
+  CRUSH rule 1 x 449 [680,16,484,670]
+  CRUSH rule 1 x 450 [235,214,79,423]
+  CRUSH rule 1 x 451 [961,468,333,640]
+  CRUSH rule 1 x 452 [525,479,153,528]
+  CRUSH rule 1 x 453 [138,466,302,86]
+  CRUSH rule 1 x 454 [137,625,215,402]
+  CRUSH rule 1 x 455 [173,150,997,16]
+  CRUSH rule 1 x 456 [235,226,238,258]
+  CRUSH rule 1 x 457 [450,577,253,413]
+  CRUSH rule 1 x 458 [195,537,91,814]
+  CRUSH rule 1 x 459 [381,555,312,573]
+  CRUSH rule 1 x 460 [972,730,534,678]
+  CRUSH rule 1 x 461 [506,279,142,830]
+  CRUSH rule 1 x 462 [692,959,578,57]
+  CRUSH rule 1 x 463 [788,667,949,550]
+  CRUSH rule 1 x 464 [133,122,588,999]
+  CRUSH rule 1 x 465 [971,190,230,777]
+  CRUSH rule 1 x 466 [394,576,148,157]
+  CRUSH rule 1 x 467 [517,28,366,362]
+  CRUSH rule 1 x 468 [829,143,874,225]
+  CRUSH rule 1 x 469 [987,936,106,725]
+  CRUSH rule 1 x 470 [107,982,56,889]
+  CRUSH rule 1 x 471 [181,897,629,860]
+  CRUSH rule 1 x 472 [547,512,172,24]
+  CRUSH rule 1 x 473 [760,997,824,905]
+  CRUSH rule 1 x 474 [787,418,743,628]
+  CRUSH rule 1 x 475 [662,312,253,617]
+  CRUSH rule 1 x 476 [110,495,185,508]
+  CRUSH rule 1 x 477 [393,954,834,132]
+  CRUSH rule 1 x 478 [246,483,480,644]
+  CRUSH rule 1 x 479 [70,929,697,931]
+  CRUSH rule 1 x 480 [753,119,961,607]
+  CRUSH rule 1 x 481 [470,429,677,242]
+  CRUSH rule 1 x 482 [451,566,961,675]
+  CRUSH rule 1 x 483 [816,72,371,278]
+  CRUSH rule 1 x 484 [540,454,389,31]
+  CRUSH rule 1 x 485 [74,582,624,684]
+  CRUSH rule 1 x 486 [958,595,199,763]
+  CRUSH rule 1 x 487 [228,302,804,833]
+  CRUSH rule 1 x 488 [180,529,722,956]
+  CRUSH rule 1 x 489 [47,617,812,187]
+  CRUSH rule 1 x 490 [905,822,479,124]
+  CRUSH rule 1 x 491 [892,370,609,998]
+  CRUSH rule 1 x 492 [588,959,127,948]
+  CRUSH rule 1 x 493 [353,461,593,291]
+  CRUSH rule 1 x 494 [378,848,443,368]
+  CRUSH rule 1 x 495 [845,653,768,234]
+  CRUSH rule 1 x 496 [13,988,0,691]
+  CRUSH rule 1 x 497 [796,877,788,394]
+  CRUSH rule 1 x 498 [412,337,270,705]
+  CRUSH rule 1 x 499 [330,695,8,74]
+  CRUSH rule 1 x 500 [820,272,547,765]
+  CRUSH rule 1 x 501 [110,44,132,442]
+  CRUSH rule 1 x 502 [336,595,650,274]
+  CRUSH rule 1 x 503 [922,211,157,722]
+  CRUSH rule 1 x 504 [483,52,122,432]
+  CRUSH rule 1 x 505 [482,598,224,279]
+  CRUSH rule 1 x 506 [493,123,43,856]
+  CRUSH rule 1 x 507 [12,598,264,422]
+  CRUSH rule 1 x 508 [227,157,611,301]
+  CRUSH rule 1 x 509 [807,242,363,122]
+  CRUSH rule 1 x 510 [134,437,227,75]
+  CRUSH rule 1 x 511 [212,54,83,799]
+  CRUSH rule 1 x 512 [236,630,758,752]
+  CRUSH rule 1 x 513 [994,693,644,938]
+  CRUSH rule 1 x 514 [45,508,831,19]
+  CRUSH rule 1 x 515 [504,138,480,272]
+  CRUSH rule 1 x 516 [285,409,136,570]
+  CRUSH rule 1 x 517 [300,232,23,906]
+  CRUSH rule 1 x 518 [397,674,98,898]
+  CRUSH rule 1 x 519 [86,750,772,913]
+  CRUSH rule 1 x 520 [900,833,614,130]
+  CRUSH rule 1 x 521 [31,47,236,751]
+  CRUSH rule 1 x 522 [390,16,280,144]
+  CRUSH rule 1 x 523 [618,308,424,590]
+  CRUSH rule 1 x 524 [635,189,687,963]
+  CRUSH rule 1 x 525 [311,916,699,262]
+  CRUSH rule 1 x 526 [48,738,227,718]
+  CRUSH rule 1 x 527 [202,851,889,216]
+  CRUSH rule 1 x 528 [565,827,590,273]
+  CRUSH rule 1 x 529 [934,864,241,43]
+  CRUSH rule 1 x 530 [502,934,298,670]
+  CRUSH rule 1 x 531 [681,627,942,487]
+  CRUSH rule 1 x 532 [422,6,147,205]
+  CRUSH rule 1 x 533 [863,68,364,983]
+  CRUSH rule 1 x 534 [962,931,775,172]
+  CRUSH rule 1 x 535 [89,565,397,693]
+  CRUSH rule 1 x 536 [499,351,760,458]
+  CRUSH rule 1 x 537 [676,547,787,311]
+  CRUSH rule 1 x 538 [58,644,571,649]
+  CRUSH rule 1 x 539 [837,953,457,711]
+  CRUSH rule 1 x 540 [831,50,132,213]
+  CRUSH rule 1 x 541 [582,757,121,525]
+  CRUSH rule 1 x 542 [472,132,790,997]
+  CRUSH rule 1 x 543 [382,272,797,330]
+  CRUSH rule 1 x 544 [947,930,496,883]
+  CRUSH rule 1 x 545 [425,570,305,77]
+  CRUSH rule 1 x 546 [18,65,529,437]
+  CRUSH rule 1 x 547 [445,715,600,472]
+  CRUSH rule 1 x 548 [367,569,980,167]
+  CRUSH rule 1 x 549 [125,715,671,817]
+  CRUSH rule 1 x 550 [425,599,744,199]
+  CRUSH rule 1 x 551 [44,1,528,922]
+  CRUSH rule 1 x 552 [246,104,68,239]
+  CRUSH rule 1 x 553 [71,703,615,28]
+  CRUSH rule 1 x 554 [207,124,217,166]
+  CRUSH rule 1 x 555 [570,28,317,420]
+  CRUSH rule 1 x 556 [674,152,421,79]
+  CRUSH rule 1 x 557 [347,817,191,391]
+  CRUSH rule 1 x 558 [627,426,369,692]
+  CRUSH rule 1 x 559 [940,630,924,242]
+  CRUSH rule 1 x 560 [295,903,541,29]
+  CRUSH rule 1 x 561 [506,682,384,637]
+  CRUSH rule 1 x 562 [718,529,87,729]
+  CRUSH rule 1 x 563 [552,332,747,206]
+  CRUSH rule 1 x 564 [835,769,736,486]
+  CRUSH rule 1 x 565 [8,167,539,182]
+  CRUSH rule 1 x 566 [600,481,301,263]
+  CRUSH rule 1 x 567 [999,994,509,899]
+  CRUSH rule 1 x 568 [252,431,157,62]
+  CRUSH rule 1 x 569 [643,218,943,455]
+  CRUSH rule 1 x 570 [617,635,765,422]
+  CRUSH rule 1 x 571 [757,80,59,98]
+  CRUSH rule 1 x 572 [299,348,575,889]
+  CRUSH rule 1 x 573 [25,505,270,167]
+  CRUSH rule 1 x 574 [215,431,624,177]
+  CRUSH rule 1 x 575 [225,252,611,546]
+  CRUSH rule 1 x 576 [627,94,159,857]
+  CRUSH rule 1 x 577 [237,809,778,636]
+  CRUSH rule 1 x 578 [885,313,120,344]
+  CRUSH rule 1 x 579 [924,575,787,831]
+  CRUSH rule 1 x 580 [718,51,766,121]
+  CRUSH rule 1 x 581 [219,807,129,571]
+  CRUSH rule 1 x 582 [893,701,598,863]
+  CRUSH rule 1 x 583 [246,930,964,170]
+  CRUSH rule 1 x 584 [336,432,680,175]
+  CRUSH rule 1 x 585 [324,999,397,485]
+  CRUSH rule 1 x 586 [558,230,976,541]
+  CRUSH rule 1 x 587 [985,830,597,21]
+  CRUSH rule 1 x 588 [211,544,57,134]
+  CRUSH rule 1 x 589 [129,21,112,190]
+  CRUSH rule 1 x 590 [467,969,652,593]
+  CRUSH rule 1 x 591 [758,514,316,164]
+  CRUSH rule 1 x 592 [525,253,190,443]
+  CRUSH rule 1 x 593 [601,885,339,152]
+  CRUSH rule 1 x 594 [227,60,450,30]
+  CRUSH rule 1 x 595 [720,854,496,912]
+  CRUSH rule 1 x 596 [751,195,997,77]
+  CRUSH rule 1 x 597 [129,574,714,8]
+  CRUSH rule 1 x 598 [679,207,604,396]
+  CRUSH rule 1 x 599 [668,315,683,349]
+  CRUSH rule 1 x 600 [143,396,464,444]
+  CRUSH rule 1 x 601 [326,573,873,902]
+  CRUSH rule 1 x 602 [860,281,875,535]
+  CRUSH rule 1 x 603 [709,328,445,349]
+  CRUSH rule 1 x 604 [571,62,814,95]
+  CRUSH rule 1 x 605 [252,739,860,27]
+  CRUSH rule 1 x 606 [339,236,759,842]
+  CRUSH rule 1 x 607 [590,248,759,868]
+  CRUSH rule 1 x 608 [145,635,309,467]
+  CRUSH rule 1 x 609 [973,547,223,79]
+  CRUSH rule 1 x 610 [435,816,961,983]
+  CRUSH rule 1 x 611 [559,283,422,584]
+  CRUSH rule 1 x 612 [273,149,123,576]
+  CRUSH rule 1 x 613 [828,614,642,674]
+  CRUSH rule 1 x 614 [478,748,393,34]
+  CRUSH rule 1 x 615 [392,155,144,326]
+  CRUSH rule 1 x 616 [778,637,452,248]
+  CRUSH rule 1 x 617 [622,713,996,833]
+  CRUSH rule 1 x 618 [149,877,270,329]
+  CRUSH rule 1 x 619 [604,163,656,409]
+  CRUSH rule 1 x 620 [181,23,409,198]
+  CRUSH rule 1 x 621 [735,902,386,237]
+  CRUSH rule 1 x 622 [661,824,717,568]
+  CRUSH rule 1 x 623 [142,121,643,61]
+  CRUSH rule 1 x 624 [360,716,420,398]
+  CRUSH rule 1 x 625 [541,167,385,1]
+  CRUSH rule 1 x 626 [364,431,610,363]
+  CRUSH rule 1 x 627 [458,137,557,410]
+  CRUSH rule 1 x 628 [250,350,556,497]
+  CRUSH rule 1 x 629 [928,160,710,572]
+  CRUSH rule 1 x 630 [243,19,918,556]
+  CRUSH rule 1 x 631 [438,221,574,676]
+  CRUSH rule 1 x 632 [797,368,247,5]
+  CRUSH rule 1 x 633 [993,749,525,485]
+  CRUSH rule 1 x 634 [239,351,633,299]
+  CRUSH rule 1 x 635 [640,965,25,961]
+  CRUSH rule 1 x 636 [173,290,297,991]
+  CRUSH rule 1 x 637 [0,918,98,108]
+  CRUSH rule 1 x 638 [702,235,424,900]
+  CRUSH rule 1 x 639 [475,687,31,785]
+  CRUSH rule 1 x 640 [31,664,399,677]
+  CRUSH rule 1 x 641 [296,473,108,963]
+  CRUSH rule 1 x 642 [894,273,427,606]
+  CRUSH rule 1 x 643 [117,111,732,191]
+  CRUSH rule 1 x 644 [438,336,327,512]
+  CRUSH rule 1 x 645 [982,702,351,573]
+  CRUSH rule 1 x 646 [334,804,146,842]
+  CRUSH rule 1 x 647 [933,787,185,334]
+  CRUSH rule 1 x 648 [22,444,400,862]
+  CRUSH rule 1 x 649 [503,229,213,460]
+  CRUSH rule 1 x 650 [328,659,420,443]
+  CRUSH rule 1 x 651 [3,880,823,123]
+  CRUSH rule 1 x 652 [495,977,563,733]
+  CRUSH rule 1 x 653 [185,718,804,280]
+  CRUSH rule 1 x 654 [130,528,380,81]
+  CRUSH rule 1 x 655 [560,872,454,504]
+  CRUSH rule 1 x 656 [219,885,178,981]
+  CRUSH rule 1 x 657 [233,684,813,490]
+  CRUSH rule 1 x 658 [778,6,756,380]
+  CRUSH rule 1 x 659 [240,663,306,540]
+  CRUSH rule 1 x 660 [244,855,196,147]
+  CRUSH rule 1 x 661 [184,270,128,398]
+  CRUSH rule 1 x 662 [65,883,921,438]
+  CRUSH rule 1 x 663 [323,721,594,812]
+  CRUSH rule 1 x 664 [865,113,512,51]
+  CRUSH rule 1 x 665 [420,850,591,475]
+  CRUSH rule 1 x 666 [319,767,246,3]
+  CRUSH rule 1 x 667 [875,39,343,100]
+  CRUSH rule 1 x 668 [331,122,263,599]
+  CRUSH rule 1 x 669 [915,521,402,747]
+  CRUSH rule 1 x 670 [845,659,943,447]
+  CRUSH rule 1 x 671 [108,634,527,363]
+  CRUSH rule 1 x 672 [578,216,110,589]
+  CRUSH rule 1 x 673 [442,74,579,797]
+  CRUSH rule 1 x 674 [588,364,281,308]
+  CRUSH rule 1 x 675 [489,698,744,671]
+  CRUSH rule 1 x 676 [928,911,40,180]
+  CRUSH rule 1 x 677 [399,269,692,131]
+  CRUSH rule 1 x 678 [546,752,544,155]
+  CRUSH rule 1 x 679 [988,25,275,433]
+  CRUSH rule 1 x 680 [335,963,382,486]
+  CRUSH rule 1 x 681 [690,462,623,466]
+  CRUSH rule 1 x 682 [196,588,154,257]
+  CRUSH rule 1 x 683 [627,25,421,160]
+  CRUSH rule 1 x 684 [38,804,592,158]
+  CRUSH rule 1 x 685 [841,368,548,362]
+  CRUSH rule 1 x 686 [336,287,525,440]
+  CRUSH rule 1 x 687 [20,682,924,653]
+  CRUSH rule 1 x 688 [463,371,780,556]
+  CRUSH rule 1 x 689 [569,250,78,816]
+  CRUSH rule 1 x 690 [551,144,587,263]
+  CRUSH rule 1 x 691 [766,464,446,533]
+  CRUSH rule 1 x 692 [739,634,18,245]
+  CRUSH rule 1 x 693 [339,297,118,330]
+  CRUSH rule 1 x 694 [405,26,830,181]
+  CRUSH rule 1 x 695 [622,576,597,535]
+  CRUSH rule 1 x 696 [558,902,689,13]
+  CRUSH rule 1 x 697 [818,222,406,691]
+  CRUSH rule 1 x 698 [178,48,402,233]
+  CRUSH rule 1 x 699 [450,244,180,919]
+  CRUSH rule 1 x 700 [502,771,987,706]
+  CRUSH rule 1 x 701 [4,612,782,216]
+  CRUSH rule 1 x 702 [177,630,232,923]
+  CRUSH rule 1 x 703 [354,178,389,393]
+  CRUSH rule 1 x 704 [646,601,156,171]
+  CRUSH rule 1 x 705 [921,401,890,265]
+  CRUSH rule 1 x 706 [652,877,562,452]
+  CRUSH rule 1 x 707 [345,745,67,716]
+  CRUSH rule 1 x 708 [333,607,180,469]
+  CRUSH rule 1 x 709 [45,187,302,115]
+  CRUSH rule 1 x 710 [94,855,43,199]
+  CRUSH rule 1 x 711 [227,653,731,150]
+  CRUSH rule 1 x 712 [398,953,136,870]
+  CRUSH rule 1 x 713 [116,800,503,662]
+  CRUSH rule 1 x 714 [111,629,866,709]
+  CRUSH rule 1 x 715 [531,291,486,382]
+  CRUSH rule 1 x 716 [169,541,291,42]
+  CRUSH rule 1 x 717 [417,446,994,894]
+  CRUSH rule 1 x 718 [992,383,298,844]
+  CRUSH rule 1 x 719 [936,674,324,759]
+  CRUSH rule 1 x 720 [370,188,174,464]
+  CRUSH rule 1 x 721 [320,859,278,259]
+  CRUSH rule 1 x 722 [7,2,673,129]
+  CRUSH rule 1 x 723 [270,553,831,662]
+  CRUSH rule 1 x 724 [666,822,708,895]
+  CRUSH rule 1 x 725 [794,406,875,459]
+  CRUSH rule 1 x 726 [420,556,341,292]
+  CRUSH rule 1 x 727 [561,461,129,635]
+  CRUSH rule 1 x 728 [951,330,196,756]
+  CRUSH rule 1 x 729 [656,644,436,591]
+  CRUSH rule 1 x 730 [3,558,629,184]
+  CRUSH rule 1 x 731 [852,89,75,735]
+  CRUSH rule 1 x 732 [983,840,869,976]
+  CRUSH rule 1 x 733 [285,396,388,122]
+  CRUSH rule 1 x 734 [125,510,402,640]
+  CRUSH rule 1 x 735 [417,773,686,504]
+  CRUSH rule 1 x 736 [749,396,632,550]
+  CRUSH rule 1 x 737 [644,991,946,135]
+  CRUSH rule 1 x 738 [449,683,290,220]
+  CRUSH rule 1 x 739 [341,220,641,454]
+  CRUSH rule 1 x 740 [874,524,674,650]
+  CRUSH rule 1 x 741 [189,472,712,798]
+  CRUSH rule 1 x 742 [912,581,114,81]
+  CRUSH rule 1 x 743 [654,914,425,441]
+  CRUSH rule 1 x 744 [725,295,579,377]
+  CRUSH rule 1 x 745 [787,858,850,506]
+  CRUSH rule 1 x 746 [757,848,704,30]
+  CRUSH rule 1 x 747 [700,81,867,681]
+  CRUSH rule 1 x 748 [557,436,238,664]
+  CRUSH rule 1 x 749 [772,622,337,42]
+  CRUSH rule 1 x 750 [946,97,376,677]
+  CRUSH rule 1 x 751 [996,618,343,911]
+  CRUSH rule 1 x 752 [746,887,695,868]
+  CRUSH rule 1 x 753 [741,14,463,479]
+  CRUSH rule 1 x 754 [648,349,333,355]
+  CRUSH rule 1 x 755 [157,460,466,187]
+  CRUSH rule 1 x 756 [416,97,197,497]
+  CRUSH rule 1 x 757 [599,839,776,410]
+  CRUSH rule 1 x 758 [994,218,620,256]
+  CRUSH rule 1 x 759 [959,682,514,745]
+  CRUSH rule 1 x 760 [518,943,215,83]
+  CRUSH rule 1 x 761 [285,849,420,324]
+  CRUSH rule 1 x 762 [591,313,41,335]
+  CRUSH rule 1 x 763 [908,411,200,740]
+  CRUSH rule 1 x 764 [787,234,894,485]
+  CRUSH rule 1 x 765 [327,921,882,393]
+  CRUSH rule 1 x 766 [84,161,878,704]
+  CRUSH rule 1 x 767 [370,895,702,701]
+  CRUSH rule 1 x 768 [826,760,879,864]
+  CRUSH rule 1 x 769 [67,768,663,735]
+  CRUSH rule 1 x 770 [593,909,482,259]
+  CRUSH rule 1 x 771 [309,935,121,578]
+  CRUSH rule 1 x 772 [12,125,797,301]
+  CRUSH rule 1 x 773 [253,466,820,549]
+  CRUSH rule 1 x 774 [164,390,705,109]
+  CRUSH rule 1 x 775 [703,47,43,973]
+  CRUSH rule 1 x 776 [728,231,80,916]
+  CRUSH rule 1 x 777 [981,621,568,729]
+  CRUSH rule 1 x 778 [411,456,544,597]
+  CRUSH rule 1 x 779 [346,121,519,921]
+  CRUSH rule 1 x 780 [476,39,288,381]
+  CRUSH rule 1 x 781 [10,130,585,844]
+  CRUSH rule 1 x 782 [462,246,581,902]
+  CRUSH rule 1 x 783 [580,373,153,775]
+  CRUSH rule 1 x 784 [413,113,978,990]
+  CRUSH rule 1 x 785 [341,856,332,354]
+  CRUSH rule 1 x 786 [411,140,313,393]
+  CRUSH rule 1 x 787 [605,522,211,813]
+  CRUSH rule 1 x 788 [226,545,35,142]
+  CRUSH rule 1 x 789 [545,320,414,702]
+  CRUSH rule 1 x 790 [414,748,816,327]
+  CRUSH rule 1 x 791 [660,906,406,697]
+  CRUSH rule 1 x 792 [287,392,514,204]
+  CRUSH rule 1 x 793 [631,133,850,713]
+  CRUSH rule 1 x 794 [931,517,543,210]
+  CRUSH rule 1 x 795 [551,962,477,948]
+  CRUSH rule 1 x 796 [814,4,95,27]
+  CRUSH rule 1 x 797 [64,201,299,734]
+  CRUSH rule 1 x 798 [422,530,114,431]
+  CRUSH rule 1 x 799 [824,32,679,562]
+  CRUSH rule 1 x 800 [862,623,489,637]
+  CRUSH rule 1 x 801 [145,550,329,324]
+  CRUSH rule 1 x 802 [570,19,847,308]
+  CRUSH rule 1 x 803 [151,812,662,358]
+  CRUSH rule 1 x 804 [467,93,264,863]
+  CRUSH rule 1 x 805 [621,223,938,809]
+  CRUSH rule 1 x 806 [898,957,805,430]
+  CRUSH rule 1 x 807 [354,531,422,159]
+  CRUSH rule 1 x 808 [7,96,76,897]
+  CRUSH rule 1 x 809 [70,734,719,56]
+  CRUSH rule 1 x 810 [701,18,972,327]
+  CRUSH rule 1 x 811 [248,547,103,728]
+  CRUSH rule 1 x 812 [230,576,821,566]
+  CRUSH rule 1 x 813 [805,114,683,629]
+  CRUSH rule 1 x 814 [54,619,973,741]
+  CRUSH rule 1 x 815 [679,412,613,132]
+  CRUSH rule 1 x 816 [919,448,826,414]
+  CRUSH rule 1 x 817 [765,830,436,521]
+  CRUSH rule 1 x 818 [415,566,644,687]
+  CRUSH rule 1 x 819 [721,319,865,750]
+  CRUSH rule 1 x 820 [218,301,333,190]
+  CRUSH rule 1 x 821 [185,795,680,953]
+  CRUSH rule 1 x 822 [356,261,54,522]
+  CRUSH rule 1 x 823 [220,281,549,456]
+  CRUSH rule 1 x 824 [292,809,887,74]
+  CRUSH rule 1 x 825 [949,778,101,311]
+  CRUSH rule 1 x 826 [767,818,833,927]
+  CRUSH rule 1 x 827 [631,83,406,635]
+  CRUSH rule 1 x 828 [288,986,445,26]
+  CRUSH rule 1 x 829 [990,667,915,694]
+  CRUSH rule 1 x 830 [152,571,778,505]
+  CRUSH rule 1 x 831 [814,563,630,97]
+  CRUSH rule 1 x 832 [235,641,616,110]
+  CRUSH rule 1 x 833 [657,565,922,140]
+  CRUSH rule 1 x 834 [907,231,644,13]
+  CRUSH rule 1 x 835 [784,262,771,264]
+  CRUSH rule 1 x 836 [951,158,366,710]
+  CRUSH rule 1 x 837 [556,498,334,633]
+  CRUSH rule 1 x 838 [329,274,964,547]
+  CRUSH rule 1 x 839 [568,209,939,364]
+  CRUSH rule 1 x 840 [45,579,842,70]
+  CRUSH rule 1 x 841 [652,702,24,605]
+  CRUSH rule 1 x 842 [629,984,314,895]
+  CRUSH rule 1 x 843 [799,690,688,648]
+  CRUSH rule 1 x 844 [694,600,534,700]
+  CRUSH rule 1 x 845 [332,30,179,93]
+  CRUSH rule 1 x 846 [452,251,712,719]
+  CRUSH rule 1 x 847 [399,681,847,739]
+  CRUSH rule 1 x 848 [303,138,440,346]
+  CRUSH rule 1 x 849 [666,346,708,873]
+  CRUSH rule 1 x 850 [644,511,345,844]
+  CRUSH rule 1 x 851 [527,546,737,425]
+  CRUSH rule 1 x 852 [31,809,94,618]
+  CRUSH rule 1 x 853 [483,330,869,184]
+  CRUSH rule 1 x 854 [697,953,968,143]
+  CRUSH rule 1 x 855 [837,996,239,621]
+  CRUSH rule 1 x 856 [712,40,547,430]
+  CRUSH rule 1 x 857 [77,984,576,551]
+  CRUSH rule 1 x 858 [412,384,841,465]
+  CRUSH rule 1 x 859 [173,760,26,300]
+  CRUSH rule 1 x 860 [776,429,328,917]
+  CRUSH rule 1 x 861 [705,405,477,50]
+  CRUSH rule 1 x 862 [809,44,788,938]
+  CRUSH rule 1 x 863 [349,496,963,178]
+  CRUSH rule 1 x 864 [717,858,101,239]
+  CRUSH rule 1 x 865 [857,603,586,262]
+  CRUSH rule 1 x 866 [394,304,71,96]
+  CRUSH rule 1 x 867 [640,773,663,974]
+  CRUSH rule 1 x 868 [613,950,712,663]
+  CRUSH rule 1 x 869 [973,889,524,22]
+  CRUSH rule 1 x 870 [505,35,386,498]
+  CRUSH rule 1 x 871 [239,264,262,773]
+  CRUSH rule 1 x 872 [21,767,456,748]
+  CRUSH rule 1 x 873 [954,666,980,264]
+  CRUSH rule 1 x 874 [54,510,947,1]
+  CRUSH rule 1 x 875 [809,418,452,462]
+  CRUSH rule 1 x 876 [483,457,61,248]
+  CRUSH rule 1 x 877 [542,531,952,939]
+  CRUSH rule 1 x 878 [217,674,857,644]
+  CRUSH rule 1 x 879 [999,475,134,250]
+  CRUSH rule 1 x 880 [678,573,935,385]
+  CRUSH rule 1 x 881 [394,835,789,802]
+  CRUSH rule 1 x 882 [467,382,353,56]
+  CRUSH rule 1 x 883 [802,744,237,337]
+  CRUSH rule 1 x 884 [653,660,638,700]
+  CRUSH rule 1 x 885 [898,704,307,445]
+  CRUSH rule 1 x 886 [434,357,938,641]
+  CRUSH rule 1 x 887 [297,226,711,428]
+  CRUSH rule 1 x 888 [863,324,443,213]
+  CRUSH rule 1 x 889 [105,102,308,163]
+  CRUSH rule 1 x 890 [550,248,606,704]
+  CRUSH rule 1 x 891 [575,928,880,891]
+  CRUSH rule 1 x 892 [259,862,133,271]
+  CRUSH rule 1 x 893 [902,880,543,542]
+  CRUSH rule 1 x 894 [180,169,916,43]
+  CRUSH rule 1 x 895 [725,849,182,129]
+  CRUSH rule 1 x 896 [951,34,874,537]
+  CRUSH rule 1 x 897 [810,352,73,939]
+  CRUSH rule 1 x 898 [979,433,719,411]
+  CRUSH rule 1 x 899 [685,668,534,932]
+  CRUSH rule 1 x 900 [530,978,41,894]
+  CRUSH rule 1 x 901 [740,107,336,175]
+  CRUSH rule 1 x 902 [800,743,693,310]
+  CRUSH rule 1 x 903 [230,267,842,266]
+  CRUSH rule 1 x 904 [346,949,460,973]
+  CRUSH rule 1 x 905 [530,397,619,958]
+  CRUSH rule 1 x 906 [80,426,138,672]
+  CRUSH rule 1 x 907 [365,968,475,297]
+  CRUSH rule 1 x 908 [204,832,742,809]
+  CRUSH rule 1 x 909 [883,989,146,959]
+  CRUSH rule 1 x 910 [549,593,249,853]
+  CRUSH rule 1 x 911 [325,847,352,214]
+  CRUSH rule 1 x 912 [874,888,582,796]
+  CRUSH rule 1 x 913 [331,463,342,574]
+  CRUSH rule 1 x 914 [836,468,601,732]
+  CRUSH rule 1 x 915 [245,228,100,661]
+  CRUSH rule 1 x 916 [77,967,364,435]
+  CRUSH rule 1 x 917 [239,60,866,221]
+  CRUSH rule 1 x 918 [988,115,922,80]
+  CRUSH rule 1 x 919 [783,139,696,1]
+  CRUSH rule 1 x 920 [623,408,685,953]
+  CRUSH rule 1 x 921 [105,799,144,90]
+  CRUSH rule 1 x 922 [887,505,652,348]
+  CRUSH rule 1 x 923 [223,318,552,458]
+  CRUSH rule 1 x 924 [25,778,366,333]
+  CRUSH rule 1 x 925 [912,601,297,682]
+  CRUSH rule 1 x 926 [968,133,709,144]
+  CRUSH rule 1 x 927 [277,724,214,988]
+  CRUSH rule 1 x 928 [554,203,658,789]
+  CRUSH rule 1 x 929 [761,802,367,528]
+  CRUSH rule 1 x 930 [814,61,788,736]
+  CRUSH rule 1 x 931 [29,193,61,41]
+  CRUSH rule 1 x 932 [446,198,862,534]
+  CRUSH rule 1 x 933 [352,742,216,321]
+  CRUSH rule 1 x 934 [730,2,332,631]
+  CRUSH rule 1 x 935 [731,23,736,79]
+  CRUSH rule 1 x 936 [322,975,20,904]
+  CRUSH rule 1 x 937 [822,221,841,161]
+  CRUSH rule 1 x 938 [557,850,66,630]
+  CRUSH rule 1 x 939 [150,11,971,371]
+  CRUSH rule 1 x 940 [638,398,169,616]
+  CRUSH rule 1 x 941 [730,342,929,577]
+  CRUSH rule 1 x 942 [62,292,166,814]
+  CRUSH rule 1 x 943 [165,314,519,548]
+  CRUSH rule 1 x 944 [199,625,766,176]
+  CRUSH rule 1 x 945 [946,999,699,303]
+  CRUSH rule 1 x 946 [595,93,852,142]
+  CRUSH rule 1 x 947 [800,582,356,93]
+  CRUSH rule 1 x 948 [132,551,139,920]
+  CRUSH rule 1 x 949 [792,920,466,380]
+  CRUSH rule 1 x 950 [111,345,176,543]
+  CRUSH rule 1 x 951 [414,619,648,655]
+  CRUSH rule 1 x 952 [775,469,500,356]
+  CRUSH rule 1 x 953 [349,1,5,251]
+  CRUSH rule 1 x 954 [570,940,410,249]
+  CRUSH rule 1 x 955 [729,774,823,800]
+  CRUSH rule 1 x 956 [519,141,575,625]
+  CRUSH rule 1 x 957 [242,709,611,97]
+  CRUSH rule 1 x 958 [84,217,227,253]
+  CRUSH rule 1 x 959 [270,413,918,789]
+  CRUSH rule 1 x 960 [458,192,307,279]
+  CRUSH rule 1 x 961 [981,388,777,546]
+  CRUSH rule 1 x 962 [623,834,277,134]
+  CRUSH rule 1 x 963 [291,167,714,468]
+  CRUSH rule 1 x 964 [28,156,788,127]
+  CRUSH rule 1 x 965 [675,557,290,517]
+  CRUSH rule 1 x 966 [836,306,946,283]
+  CRUSH rule 1 x 967 [966,386,735,837]
+  CRUSH rule 1 x 968 [864,756,690,121]
+  CRUSH rule 1 x 969 [729,625,480,769]
+  CRUSH rule 1 x 970 [800,362,646,582]
+  CRUSH rule 1 x 971 [737,381,153,684]
+  CRUSH rule 1 x 972 [952,245,720,884]
+  CRUSH rule 1 x 973 [356,455,579,857]
+  CRUSH rule 1 x 974 [545,758,586,596]
+  CRUSH rule 1 x 975 [336,191,202,146]
+  CRUSH rule 1 x 976 [446,208,757,620]
+  CRUSH rule 1 x 977 [202,896,196,956]
+  CRUSH rule 1 x 978 [612,324,996,225]
+  CRUSH rule 1 x 979 [843,457,675,650]
+  CRUSH rule 1 x 980 [60,914,881,626]
+  CRUSH rule 1 x 981 [702,749,937,153]
+  CRUSH rule 1 x 982 [298,928,738,167]
+  CRUSH rule 1 x 983 [723,572,395,358]
+  CRUSH rule 1 x 984 [723,864,804,935]
+  CRUSH rule 1 x 985 [945,459,868,211]
+  CRUSH rule 1 x 986 [772,664,535,169]
+  CRUSH rule 1 x 987 [88,324,312,843]
+  CRUSH rule 1 x 988 [522,927,131,996]
+  CRUSH rule 1 x 989 [578,332,208,605]
+  CRUSH rule 1 x 990 [638,228,414,311]
+  CRUSH rule 1 x 991 [530,221,451,422]
+  CRUSH rule 1 x 992 [925,705,275,81]
+  CRUSH rule 1 x 993 [991,301,43,469]
+  CRUSH rule 1 x 994 [276,51,868,683]
+  CRUSH rule 1 x 995 [288,836,753,790]
+  CRUSH rule 1 x 996 [887,983,252,686]
+  CRUSH rule 1 x 997 [110,924,386,79]
+  CRUSH rule 1 x 998 [435,830,485,853]
+  CRUSH rule 1 x 999 [876,738,357,913]
+  CRUSH rule 1 x 1000 [178,963,638,430]
+  CRUSH rule 1 x 1001 [99,519,66,759]
+  CRUSH rule 1 x 1002 [515,534,468,866]
+  CRUSH rule 1 x 1003 [104,611,937,698]
+  CRUSH rule 1 x 1004 [269,638,724,375]
+  CRUSH rule 1 x 1005 [369,223,309,409]
+  CRUSH rule 1 x 1006 [40,107,69,275]
+  CRUSH rule 1 x 1007 [978,111,416,758]
+  CRUSH rule 1 x 1008 [965,956,624,832]
+  CRUSH rule 1 x 1009 [598,476,356,695]
+  CRUSH rule 1 x 1010 [767,523,239,517]
+  CRUSH rule 1 x 1011 [289,871,207,576]
+  CRUSH rule 1 x 1012 [128,28,370,31]
+  CRUSH rule 1 x 1013 [979,765,660,812]
+  CRUSH rule 1 x 1014 [979,948,513,88]
+  CRUSH rule 1 x 1015 [277,790,396,672]
+  CRUSH rule 1 x 1016 [262,73,128,886]
+  CRUSH rule 1 x 1017 [150,269,61,499]
+  CRUSH rule 1 x 1018 [555,829,554,944]
+  CRUSH rule 1 x 1019 [513,356,265,446]
+  CRUSH rule 1 x 1020 [158,161,877,704]
+  CRUSH rule 1 x 1021 [915,998,957,285]
+  CRUSH rule 1 x 1022 [967,829,973,640]
+  CRUSH rule 1 x 1023 [488,257,614,859]
+  rule 1 (metadata) num_rep 4 result size == 4:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536,450,604]
+  CRUSH rule 1 x 1 [876,250,334,633,744]
+  CRUSH rule 1 x 2 [292,832,53,392,386]
+  CRUSH rule 1 x 3 [623,387,124,998,749]
+  CRUSH rule 1 x 4 [61,334,710,4,994]
+  CRUSH rule 1 x 5 [946,557,713,664,141]
+  CRUSH rule 1 x 6 [576,668,212,163,732]
+  CRUSH rule 1 x 7 [645,753,906,393,341]
+  CRUSH rule 1 x 8 [243,6,863,781,211]
+  CRUSH rule 1 x 9 [22,578,251,410,297]
+  CRUSH rule 1 x 10 [758,828,360,477,821]
+  CRUSH rule 1 x 11 [769,120,124,527,119]
+  CRUSH rule 1 x 12 [780,364,689,755,675]
+  CRUSH rule 1 x 13 [557,18,351,719,742]
+  CRUSH rule 1 x 14 [59,561,249,461,971]
+  CRUSH rule 1 x 15 [718,928,993,21,76]
+  CRUSH rule 1 x 16 [673,632,841,954,788]
+  CRUSH rule 1 x 17 [648,43,560,514,142]
+  CRUSH rule 1 x 18 [654,219,181,568,381]
+  CRUSH rule 1 x 19 [850,545,377,848,863]
+  CRUSH rule 1 x 20 [717,785,974,5,225]
+  CRUSH rule 1 x 21 [420,57,519,306,312]
+  CRUSH rule 1 x 22 [503,998,193,821,634]
+  CRUSH rule 1 x 23 [411,663,168,110,899]
+  CRUSH rule 1 x 24 [266,861,353,1,456]
+  CRUSH rule 1 x 25 [760,483,818,600,509]
+  CRUSH rule 1 x 26 [903,24,573,718,112]
+  CRUSH rule 1 x 27 [946,188,289,510,687]
+  CRUSH rule 1 x 28 [69,312,73,198,256]
+  CRUSH rule 1 x 29 [844,883,337,628,496]
+  CRUSH rule 1 x 30 [621,18,613,794,910]
+  CRUSH rule 1 x 31 [784,943,814,539,962]
+  CRUSH rule 1 x 32 [173,374,369,972,315]
+  CRUSH rule 1 x 33 [698,336,357,966,582]
+  CRUSH rule 1 x 34 [168,836,210,798,904]
+  CRUSH rule 1 x 35 [274,509,534,818,912]
+  CRUSH rule 1 x 36 [318,215,153,628,87]
+  CRUSH rule 1 x 37 [173,604,109,935,203]
+  CRUSH rule 1 x 38 [708,444,683,604,722]
+  CRUSH rule 1 x 39 [662,198,417,680,226]
+  CRUSH rule 1 x 40 [620,801,414,78,560]
+  CRUSH rule 1 x 41 [811,264,177,127,148]
+  CRUSH rule 1 x 42 [863,179,527,660,133]
+  CRUSH rule 1 x 43 [686,822,988,228,791]
+  CRUSH rule 1 x 44 [396,222,46,841,536]
+  CRUSH rule 1 x 45 [991,694,253,142,54]
+  CRUSH rule 1 x 46 [420,909,184,285,508]
+  CRUSH rule 1 x 47 [467,211,605,207,241]
+  CRUSH rule 1 x 48 [955,329,368,168,698]
+  CRUSH rule 1 x 49 [974,891,931,29,813]
+  CRUSH rule 1 x 50 [870,441,691,823,761]
+  CRUSH rule 1 x 51 [182,930,25,936,97]
+  CRUSH rule 1 x 52 [704,812,894,794,481]
+  CRUSH rule 1 x 53 [185,713,631,280,345]
+  CRUSH rule 1 x 54 [270,441,100,82,983]
+  CRUSH rule 1 x 55 [895,734,958,793,651]
+  CRUSH rule 1 x 56 [564,963,683,324,40]
+  CRUSH rule 1 x 57 [738,130,208,973,498]
+  CRUSH rule 1 x 58 [524,113,806,903,531]
+  CRUSH rule 1 x 59 [408,337,668,529,34]
+  CRUSH rule 1 x 60 [228,790,857,309,616]
+  CRUSH rule 1 x 61 [154,843,717,467,883]
+  CRUSH rule 1 x 62 [594,811,549,276,693]
+  CRUSH rule 1 x 63 [646,67,884,925,941]
+  CRUSH rule 1 x 64 [175,542,155,837,594]
+  CRUSH rule 1 x 65 [745,619,131,867,269]
+  CRUSH rule 1 x 66 [275,468,23,35,328]
+  CRUSH rule 1 x 67 [246,958,524,493,636]
+  CRUSH rule 1 x 68 [711,473,403,228,835]
+  CRUSH rule 1 x 69 [493,924,850,939,950]
+  CRUSH rule 1 x 70 [30,499,644,33,804]
+  CRUSH rule 1 x 71 [984,883,574,716,575]
+  CRUSH rule 1 x 72 [71,286,942,363,628]
+  CRUSH rule 1 x 73 [922,618,3,371,464]
+  CRUSH rule 1 x 74 [629,414,185,573,678]
+  CRUSH rule 1 x 75 [222,20,174,820,312]
+  CRUSH rule 1 x 76 [262,366,339,290,718]
+  CRUSH rule 1 x 77 [638,469,992,280,773]
+  CRUSH rule 1 x 78 [324,511,788,7,308]
+  CRUSH rule 1 x 79 [577,990,64,94,447]
+  CRUSH rule 1 x 80 [501,95,278,903,631]
+  CRUSH rule 1 x 81 [506,812,9,698,173]
+  CRUSH rule 1 x 82 [222,145,80,785,835]
+  CRUSH rule 1 x 83 [71,634,61,91,856]
+  CRUSH rule 1 x 84 [49,761,773,368,318]
+  CRUSH rule 1 x 85 [985,896,708,861,325]
+  CRUSH rule 1 x 86 [537,745,93,524,466]
+  CRUSH rule 1 x 87 [997,317,463,626,685]
+  CRUSH rule 1 x 88 [957,350,890,857,375]
+  CRUSH rule 1 x 89 [399,730,148,314,159]
+  CRUSH rule 1 x 90 [943,706,683,267,579]
+  CRUSH rule 1 x 91 [22,368,149,928,140]
+  CRUSH rule 1 x 92 [532,424,426,773,623]
+  CRUSH rule 1 x 93 [218,489,405,681,549]
+  CRUSH rule 1 x 94 [181,96,102,515,776]
+  CRUSH rule 1 x 95 [343,957,820,139,334]
+  CRUSH rule 1 x 96 [861,270,87,797,0]
+  CRUSH rule 1 x 97 [459,706,45,328,274]
+  CRUSH rule 1 x 98 [327,867,353,948,728]
+  CRUSH rule 1 x 99 [974,133,468,906,235]
+  CRUSH rule 1 x 100 [32,445,547,371,960]
+  CRUSH rule 1 x 101 [142,90,337,950,970]
+  CRUSH rule 1 x 102 [172,129,139,22,403]
+  CRUSH rule 1 x 103 [630,47,161,356,911]
+  CRUSH rule 1 x 104 [758,133,278,11,947]
+  CRUSH rule 1 x 105 [843,604,47,33,401]
+  CRUSH rule 1 x 106 [28,681,193,679,990]
+  CRUSH rule 1 x 107 [74,320,85,819,315]
+  CRUSH rule 1 x 108 [875,593,575,517,107]
+  CRUSH rule 1 x 109 [411,985,811,720,198]
+  CRUSH rule 1 x 110 [440,774,799,660,715]
+  CRUSH rule 1 x 111 [405,742,276,359,936]
+  CRUSH rule 1 x 112 [143,181,922,545,185]
+  CRUSH rule 1 x 113 [153,846,160,903,789]
+  CRUSH rule 1 x 114 [804,892,939,20,312]
+  CRUSH rule 1 x 115 [588,508,958,580,232]
+  CRUSH rule 1 x 116 [327,148,637,486,712]
+  CRUSH rule 1 x 117 [95,594,989,131,714]
+  CRUSH rule 1 x 118 [80,957,897,239,359]
+  CRUSH rule 1 x 119 [386,932,951,768,679]
+  CRUSH rule 1 x 120 [366,312,653,936,71]
+  CRUSH rule 1 x 121 [129,154,847,16,471]
+  CRUSH rule 1 x 122 [873,1,110,939,90]
+  CRUSH rule 1 x 123 [533,415,789,600,713]
+  CRUSH rule 1 x 124 [461,691,898,723,957]
+  CRUSH rule 1 x 125 [342,599,830,402,615]
+  CRUSH rule 1 x 126 [819,781,822,548,279]
+  CRUSH rule 1 x 127 [437,893,585,707,353]
+  CRUSH rule 1 x 128 [679,994,982,550,991]
+  CRUSH rule 1 x 129 [380,685,947,302,698]
+  CRUSH rule 1 x 130 [992,52,466,867,998]
+  CRUSH rule 1 x 131 [469,90,208,599,829]
+  CRUSH rule 1 x 132 [571,250,316,535,54]
+  CRUSH rule 1 x 133 [964,728,329,902,108]
+  CRUSH rule 1 x 134 [999,19,716,963,323]
+  CRUSH rule 1 x 135 [634,101,52,938,413]
+  CRUSH rule 1 x 136 [114,889,692,768,694]
+  CRUSH rule 1 x 137 [839,8,959,280,922]
+  CRUSH rule 1 x 138 [967,949,138,451,292]
+  CRUSH rule 1 x 139 [308,711,736,247,632]
+  CRUSH rule 1 x 140 [764,936,926,55,331]
+  CRUSH rule 1 x 141 [423,302,112,216,603]
+  CRUSH rule 1 x 142 [252,821,715,340,635]
+  CRUSH rule 1 x 143 [33,808,518,477,325]
+  CRUSH rule 1 x 144 [472,88,969,162,401]
+  CRUSH rule 1 x 145 [242,208,252,604,266]
+  CRUSH rule 1 x 146 [290,70,570,384,934]
+  CRUSH rule 1 x 147 [447,352,657,493,467]
+  CRUSH rule 1 x 148 [212,644,432,658,109]
+  CRUSH rule 1 x 149 [9,775,87,35,260]
+  CRUSH rule 1 x 150 [166,456,582,144,324]
+  CRUSH rule 1 x 151 [811,875,307,20,782]
+  CRUSH rule 1 x 152 [449,617,223,9,182]
+  CRUSH rule 1 x 153 [523,537,695,627,959]
+  CRUSH rule 1 x 154 [208,559,874,597,243]
+  CRUSH rule 1 x 155 [569,325,192,296,367]
+  CRUSH rule 1 x 156 [488,121,521,213,595]
+  CRUSH rule 1 x 157 [140,723,633,260,487]
+  CRUSH rule 1 x 158 [786,451,320,239,667]
+  CRUSH rule 1 x 159 [134,664,517,821,667]
+  CRUSH rule 1 x 160 [690,112,414,990,183]
+  CRUSH rule 1 x 161 [324,912,397,423,991]
+  CRUSH rule 1 x 162 [748,567,284,183,463]
+  CRUSH rule 1 x 163 [575,499,31,816,749]
+  CRUSH rule 1 x 164 [314,489,308,326,51]
+  CRUSH rule 1 x 165 [116,209,750,53,813]
+  CRUSH rule 1 x 166 [352,706,701,810,718]
+  CRUSH rule 1 x 167 [27,743,174,142,551]
+  CRUSH rule 1 x 168 [953,898,880,660,500]
+  CRUSH rule 1 x 169 [912,147,266,547,331]
+  CRUSH rule 1 x 170 [421,515,828,844,151]
+  CRUSH rule 1 x 171 [488,584,880,964,936]
+  CRUSH rule 1 x 172 [366,443,957,66,162]
+  CRUSH rule 1 x 173 [863,291,625,287,158]
+  CRUSH rule 1 x 174 [263,555,650,410,339]
+  CRUSH rule 1 x 175 [875,961,361,575,33]
+  CRUSH rule 1 x 176 [745,83,701,680,250]
+  CRUSH rule 1 x 177 [128,244,41,123,422]
+  CRUSH rule 1 x 178 [155,41,264,777,314]
+  CRUSH rule 1 x 179 [593,833,202,183,971]
+  CRUSH rule 1 x 180 [154,734,17,831,824]
+  CRUSH rule 1 x 181 [289,675,723,800,166]
+  CRUSH rule 1 x 182 [730,931,560,209,943]
+  CRUSH rule 1 x 183 [639,237,794,815,827]
+  CRUSH rule 1 x 184 [704,312,685,645,691]
+  CRUSH rule 1 x 185 [97,100,762,82,999]
+  CRUSH rule 1 x 186 [26,665,554,215,280]
+  CRUSH rule 1 x 187 [649,14,740,494,402]
+  CRUSH rule 1 x 188 [682,695,590,743,927]
+  CRUSH rule 1 x 189 [325,693,726,51,448]
+  CRUSH rule 1 x 190 [399,933,136,955,57]
+  CRUSH rule 1 x 191 [629,533,17,126,60]
+  CRUSH rule 1 x 192 [503,578,38,492,222]
+  CRUSH rule 1 x 193 [546,333,651,678,823]
+  CRUSH rule 1 x 194 [242,473,58,655,449]
+  CRUSH rule 1 x 195 [625,719,135,81,636]
+  CRUSH rule 1 x 196 [357,114,125,867,250]
+  CRUSH rule 1 x 197 [306,954,453,873,211]
+  CRUSH rule 1 x 198 [863,791,311,911,206]
+  CRUSH rule 1 x 199 [935,906,929,252,893]
+  CRUSH rule 1 x 200 [373,774,229,454,909]
+  CRUSH rule 1 x 201 [659,320,477,313,779]
+  CRUSH rule 1 x 202 [260,433,524,880,223]
+  CRUSH rule 1 x 203 [36,239,675,971,703]
+  CRUSH rule 1 x 204 [92,516,993,728,279]
+  CRUSH rule 1 x 205 [68,395,473,45,683]
+  CRUSH rule 1 x 206 [570,530,642,380,311]
+  CRUSH rule 1 x 207 [834,457,850,917,456]
+  CRUSH rule 1 x 208 [927,484,640,976,803]
+  CRUSH rule 1 x 209 [878,66,58,940,48]
+  CRUSH rule 1 x 210 [572,981,484,29,0]
+  CRUSH rule 1 x 211 [107,597,780,857,895]
+  CRUSH rule 1 x 212 [389,107,838,624,698]
+  CRUSH rule 1 x 213 [497,717,567,728,905]
+  CRUSH rule 1 x 214 [798,65,254,572,32]
+  CRUSH rule 1 x 215 [233,419,283,638,520]
+  CRUSH rule 1 x 216 [494,464,742,523,459]
+  CRUSH rule 1 x 217 [352,396,309,938,66]
+  CRUSH rule 1 x 218 [895,864,988,650,593]
+  CRUSH rule 1 x 219 [222,534,277,242,658]
+  CRUSH rule 1 x 220 [281,19,584,563,858]
+  CRUSH rule 1 x 221 [64,928,963,130,312]
+  CRUSH rule 1 x 222 [40,544,161,199,861]
+  CRUSH rule 1 x 223 [645,556,159,417,46]
+  CRUSH rule 1 x 224 [647,165,957,263,961]
+  CRUSH rule 1 x 225 [219,714,858,747,461]
+  CRUSH rule 1 x 226 [372,511,181,277,695]
+  CRUSH rule 1 x 227 [925,156,714,863,257]
+  CRUSH rule 1 x 228 [682,404,839,263,521]
+  CRUSH rule 1 x 229 [880,838,770,891,236]
+  CRUSH rule 1 x 230 [328,659,916,468,646]
+  CRUSH rule 1 x 231 [320,383,669,109,627]
+  CRUSH rule 1 x 232 [924,846,394,319,43]
+  CRUSH rule 1 x 233 [948,652,575,838,498]
+  CRUSH rule 1 x 234 [484,943,42,575,936]
+  CRUSH rule 1 x 235 [750,65,590,168,870]
+  CRUSH rule 1 x 236 [551,787,490,136,370]
+  CRUSH rule 1 x 237 [390,157,166,251,752]
+  CRUSH rule 1 x 238 [570,6,989,707,514]
+  CRUSH rule 1 x 239 [729,959,376,975,496]
+  CRUSH rule 1 x 240 [981,241,156,767,631]
+  CRUSH rule 1 x 241 [310,816,641,177,996]
+  CRUSH rule 1 x 242 [161,63,642,837,763]
+  CRUSH rule 1 x 243 [180,394,33,683,189]
+  CRUSH rule 1 x 244 [52,174,685,189,78]
+  CRUSH rule 1 x 245 [523,121,915,84,386]
+  CRUSH rule 1 x 246 [362,893,390,487,817]
+  CRUSH rule 1 x 247 [382,184,116,34,143]
+  CRUSH rule 1 x 248 [129,114,852,469,359]
+  CRUSH rule 1 x 249 [159,683,91,856,475]
+  CRUSH rule 1 x 250 [404,945,569,955,228]
+  CRUSH rule 1 x 251 [661,225,738,757,37]
+  CRUSH rule 1 x 252 [961,226,542,103,945]
+  CRUSH rule 1 x 253 [651,97,225,364,189]
+  CRUSH rule 1 x 254 [123,33,741,692,599]
+  CRUSH rule 1 x 255 [314,649,891,855,517]
+  CRUSH rule 1 x 256 [315,215,651,126,470]
+  CRUSH rule 1 x 257 [825,264,867,224,529]
+  CRUSH rule 1 x 258 [624,789,370,723,131]
+  CRUSH rule 1 x 259 [602,542,70,563,947]
+  CRUSH rule 1 x 260 [717,878,43,56,377]
+  CRUSH rule 1 x 261 [145,517,20,903,786]
+  CRUSH rule 1 x 262 [223,1,561,420,498]
+  CRUSH rule 1 x 263 [462,211,405,508,787]
+  CRUSH rule 1 x 264 [654,471,266,662,135]
+  CRUSH rule 1 x 265 [302,794,704,798,659]
+  CRUSH rule 1 x 266 [202,132,884,209,551]
+  CRUSH rule 1 x 267 [282,938,657,113,672]
+  CRUSH rule 1 x 268 [338,309,356,278,928]
+  CRUSH rule 1 x 269 [738,122,266,200,894]
+  CRUSH rule 1 x 270 [707,982,946,196,407]
+  CRUSH rule 1 x 271 [705,432,364,735,512]
+  CRUSH rule 1 x 272 [756,545,942,56,542]
+  CRUSH rule 1 x 273 [197,502,527,721,239]
+  CRUSH rule 1 x 274 [992,44,653,573,527]
+  CRUSH rule 1 x 275 [544,789,170,434,23]
+  CRUSH rule 1 x 276 [658,467,577,268,336]
+  CRUSH rule 1 x 277 [143,490,880,483,928]
+  CRUSH rule 1 x 278 [492,647,355,282,834]
+  CRUSH rule 1 x 279 [517,792,604,987,527]
+  CRUSH rule 1 x 280 [825,740,27,848,514]
+  CRUSH rule 1 x 281 [224,629,120,562,616]
+  CRUSH rule 1 x 282 [298,661,380,416,35]
+  CRUSH rule 1 x 283 [311,606,208,50,913]
+  CRUSH rule 1 x 284 [771,466,371,743,672]
+  CRUSH rule 1 x 285 [693,362,404,676,797]
+  CRUSH rule 1 x 286 [364,477,285,167,270]
+  CRUSH rule 1 x 287 [591,611,828,995,170]
+  CRUSH rule 1 x 288 [965,541,848,796,251]
+  CRUSH rule 1 x 289 [225,551,948,877,219]
+  CRUSH rule 1 x 290 [577,762,777,751,291]
+  CRUSH rule 1 x 291 [160,903,477,381,490]
+  CRUSH rule 1 x 292 [873,598,216,666,222]
+  CRUSH rule 1 x 293 [100,234,874,47,28]
+  CRUSH rule 1 x 294 [285,943,379,520,725]
+  CRUSH rule 1 x 295 [938,262,880,327,687]
+  CRUSH rule 1 x 296 [850,327,86,472,1]
+  CRUSH rule 1 x 297 [951,53,99,558,753]
+  CRUSH rule 1 x 298 [173,336,85,766,910]
+  CRUSH rule 1 x 299 [598,591,315,386,895]
+  CRUSH rule 1 x 300 [531,957,62,459,156]
+  CRUSH rule 1 x 301 [823,628,23,858,629]
+  CRUSH rule 1 x 302 [184,80,780,871,531]
+  CRUSH rule 1 x 303 [521,766,222,830,988]
+  CRUSH rule 1 x 304 [980,127,807,507,555]
+  CRUSH rule 1 x 305 [153,816,22,927,696]
+  CRUSH rule 1 x 306 [423,739,664,753,178]
+  CRUSH rule 1 x 307 [997,557,682,456,479]
+  CRUSH rule 1 x 308 [991,874,534,465,330]
+  CRUSH rule 1 x 309 [860,394,724,858,246]
+  CRUSH rule 1 x 310 [589,818,546,201,94]
+  CRUSH rule 1 x 311 [477,774,225,590,830]
+  CRUSH rule 1 x 312 [887,853,950,354,58]
+  CRUSH rule 1 x 313 [802,646,447,416,557]
+  CRUSH rule 1 x 314 [654,974,229,511,562]
+  CRUSH rule 1 x 315 [767,227,28,740,828]
+  CRUSH rule 1 x 316 [778,83,733,359,858]
+  CRUSH rule 1 x 317 [184,418,642,986,939]
+  CRUSH rule 1 x 318 [525,410,500,543,212]
+  CRUSH rule 1 x 319 [476,724,569,382,409]
+  CRUSH rule 1 x 320 [149,610,697,296,818]
+  CRUSH rule 1 x 321 [710,79,667,671,234]
+  CRUSH rule 1 x 322 [175,275,323,333,744]
+  CRUSH rule 1 x 323 [819,604,638,792,316]
+  CRUSH rule 1 x 324 [16,745,511,439,272]
+  CRUSH rule 1 x 325 [486,400,872,873,251]
+  CRUSH rule 1 x 326 [613,765,207,19,359]
+  CRUSH rule 1 x 327 [125,289,738,408,456]
+  CRUSH rule 1 x 328 [807,383,476,583,645]
+  CRUSH rule 1 x 329 [588,938,599,432,446]
+  CRUSH rule 1 x 330 [932,644,41,611,209]
+  CRUSH rule 1 x 331 [341,953,950,537,578]
+  CRUSH rule 1 x 332 [153,726,459,950,466]
+  CRUSH rule 1 x 333 [745,845,853,860,52]
+  CRUSH rule 1 x 334 [614,751,807,58,396]
+  CRUSH rule 1 x 335 [518,721,221,283,454]
+  CRUSH rule 1 x 336 [389,424,77,309,5]
+  CRUSH rule 1 x 337 [753,508,765,720,221]
+  CRUSH rule 1 x 338 [128,810,490,753,406]
+  CRUSH rule 1 x 339 [430,308,58,751,856]
+  CRUSH rule 1 x 340 [541,44,630,231,289]
+  CRUSH rule 1 x 341 [402,26,631,439,165]
+  CRUSH rule 1 x 342 [982,57,992,461,131]
+  CRUSH rule 1 x 343 [833,412,572,732,107]
+  CRUSH rule 1 x 344 [784,533,792,41,642]
+  CRUSH rule 1 x 345 [546,300,304,691,763]
+  CRUSH rule 1 x 346 [302,420,428,891,357]
+  CRUSH rule 1 x 347 [488,778,101,217,366]
+  CRUSH rule 1 x 348 [903,744,937,718,85]
+  CRUSH rule 1 x 349 [471,547,582,306,600]
+  CRUSH rule 1 x 350 [348,221,823,335,383]
+  CRUSH rule 1 x 351 [961,582,705,346,361]
+  CRUSH rule 1 x 352 [728,137,461,298,36]
+  CRUSH rule 1 x 353 [904,202,184,447,58]
+  CRUSH rule 1 x 354 [345,226,319,256,544]
+  CRUSH rule 1 x 355 [50,430,175,43,187]
+  CRUSH rule 1 x 356 [87,185,55,423,829]
+  CRUSH rule 1 x 357 [762,459,921,473,182]
+  CRUSH rule 1 x 358 [908,25,280,6,808]
+  CRUSH rule 1 x 359 [484,15,132,121,394]
+  CRUSH rule 1 x 360 [173,378,337,702,145]
+  CRUSH rule 1 x 361 [404,577,115,25,56]
+  CRUSH rule 1 x 362 [403,1,422,945,132]
+  CRUSH rule 1 x 363 [639,911,510,162,418]
+  CRUSH rule 1 x 364 [752,689,610,990,665]
+  CRUSH rule 1 x 365 [956,999,212,230,624]
+  CRUSH rule 1 x 366 [860,925,924,763,687]
+  CRUSH rule 1 x 367 [205,609,647,665,969]
+  CRUSH rule 1 x 368 [301,284,810,169,78]
+  CRUSH rule 1 x 369 [452,658,339,217,674]
+  CRUSH rule 1 x 370 [11,467,695,989,394]
+  CRUSH rule 1 x 371 [124,487,55,514,313]
+  CRUSH rule 1 x 372 [253,48,979,846,207]
+  CRUSH rule 1 x 373 [715,605,775,748,227]
+  CRUSH rule 1 x 374 [191,887,920,571,223]
+  CRUSH rule 1 x 375 [711,385,651,665,15]
+  CRUSH rule 1 x 376 [597,818,49,458,415]
+  CRUSH rule 1 x 377 [294,256,933,771,184]
+  CRUSH rule 1 x 378 [34,151,681,707,552]
+  CRUSH rule 1 x 379 [869,136,315,378,813]
+  CRUSH rule 1 x 380 [294,97,575,791,690]
+  CRUSH rule 1 x 381 [119,710,219,827,328]
+  CRUSH rule 1 x 382 [69,631,508,706,697]
+  CRUSH rule 1 x 383 [922,588,589,925,471]
+  CRUSH rule 1 x 384 [221,945,671,117,857]
+  CRUSH rule 1 x 385 [561,737,953,723,658]
+  CRUSH rule 1 x 386 [335,442,788,696,507]
+  CRUSH rule 1 x 387 [514,43,353,88,100]
+  CRUSH rule 1 x 388 [587,89,157,996,915]
+  CRUSH rule 1 x 389 [109,641,255,466,372]
+  CRUSH rule 1 x 390 [925,149,421,489,599]
+  CRUSH rule 1 x 391 [267,87,387,527,768]
+  CRUSH rule 1 x 392 [382,485,370,849,936]
+  CRUSH rule 1 x 393 [425,721,221,753,268]
+  CRUSH rule 1 x 394 [898,18,38,793,173]
+  CRUSH rule 1 x 395 [806,876,269,679,32]
+  CRUSH rule 1 x 396 [790,970,437,449,875]
+  CRUSH rule 1 x 397 [136,363,507,613,11]
+  CRUSH rule 1 x 398 [914,116,558,258,722]
+  CRUSH rule 1 x 399 [261,94,299,202,174]
+  CRUSH rule 1 x 400 [661,197,338,461,977]
+  CRUSH rule 1 x 401 [953,979,287,803,41]
+  CRUSH rule 1 x 402 [738,819,618,522,667]
+  CRUSH rule 1 x 403 [573,238,425,546,130]
+  CRUSH rule 1 x 404 [526,848,790,253,922]
+  CRUSH rule 1 x 405 [582,505,330,334,201]
+  CRUSH rule 1 x 406 [768,324,493,60,186]
+  CRUSH rule 1 x 407 [260,951,437,587,692]
+  CRUSH rule 1 x 408 [657,81,770,734,830]
+  CRUSH rule 1 x 409 [498,89,182,423,672]
+  CRUSH rule 1 x 410 [28,793,737,352,166]
+  CRUSH rule 1 x 411 [684,992,60,659,769]
+  CRUSH rule 1 x 412 [261,958,699,950,165]
+  CRUSH rule 1 x 413 [891,835,297,441,384]
+  CRUSH rule 1 x 414 [127,459,119,965,662]
+  CRUSH rule 1 x 415 [272,540,631,328,609]
+  CRUSH rule 1 x 416 [739,617,115,530,339]
+  CRUSH rule 1 x 417 [106,209,157,878,117]
+  CRUSH rule 1 x 418 [525,441,147,390,320]
+  CRUSH rule 1 x 419 [603,673,615,465,266]
+  CRUSH rule 1 x 420 [988,213,251,226,209]
+  CRUSH rule 1 x 421 [761,521,748,368,923]
+  CRUSH rule 1 x 422 [317,160,924,548,198]
+  CRUSH rule 1 x 423 [137,807,168,472,619]
+  CRUSH rule 1 x 424 [920,37,146,263,598]
+  CRUSH rule 1 x 425 [277,693,285,221,478]
+  CRUSH rule 1 x 426 [485,936,407,854,726]
+  CRUSH rule 1 x 427 [242,515,9,564,174]
+  CRUSH rule 1 x 428 [632,635,26,473,494]
+  CRUSH rule 1 x 429 [641,73,465,127,171]
+  CRUSH rule 1 x 430 [626,585,6,387,881]
+  CRUSH rule 1 x 431 [697,76,753,570,964]
+  CRUSH rule 1 x 432 [590,526,306,283,656]
+  CRUSH rule 1 x 433 [284,387,149,817,886]
+  CRUSH rule 1 x 434 [538,985,79,953,770]
+  CRUSH rule 1 x 435 [30,318,593,635,975]
+  CRUSH rule 1 x 436 [164,919,851,693,0]
+  CRUSH rule 1 x 437 [322,212,163,606,302]
+  CRUSH rule 1 x 438 [142,392,85,594,376]
+  CRUSH rule 1 x 439 [119,370,68,443,997]
+  CRUSH rule 1 x 440 [333,403,187,863,475]
+  CRUSH rule 1 x 441 [477,727,906,145,429]
+  CRUSH rule 1 x 442 [274,590,933,244,434]
+  CRUSH rule 1 x 443 [983,748,574,718,700]
+  CRUSH rule 1 x 444 [536,509,431,146,170]
+  CRUSH rule 1 x 445 [485,554,528,209,964]
+  CRUSH rule 1 x 446 [345,634,42,294,711]
+  CRUSH rule 1 x 447 [61,845,767,600,321]
+  CRUSH rule 1 x 448 [333,232,292,846,364]
+  CRUSH rule 1 x 449 [680,16,484,670,851]
+  CRUSH rule 1 x 450 [235,214,79,423,96]
+  CRUSH rule 1 x 451 [961,468,333,640,823]
+  CRUSH rule 1 x 452 [525,479,153,528,570]
+  CRUSH rule 1 x 453 [138,466,302,86,249]
+  CRUSH rule 1 x 454 [137,625,215,402,389]
+  CRUSH rule 1 x 455 [173,150,997,16,846]
+  CRUSH rule 1 x 456 [235,226,238,258,347]
+  CRUSH rule 1 x 457 [450,577,253,413,717]
+  CRUSH rule 1 x 458 [195,537,91,814,351]
+  CRUSH rule 1 x 459 [381,555,312,573,915]
+  CRUSH rule 1 x 460 [972,730,534,678,756]
+  CRUSH rule 1 x 461 [506,279,142,830,784]
+  CRUSH rule 1 x 462 [692,959,578,57,983]
+  CRUSH rule 1 x 463 [788,667,949,550,685]
+  CRUSH rule 1 x 464 [133,122,588,999,270]
+  CRUSH rule 1 x 465 [971,190,230,777,452]
+  CRUSH rule 1 x 466 [394,576,148,157,103]
+  CRUSH rule 1 x 467 [517,28,366,362,984]
+  CRUSH rule 1 x 468 [829,143,874,225,162]
+  CRUSH rule 1 x 469 [987,936,106,725,633]
+  CRUSH rule 1 x 470 [107,982,56,889,67]
+  CRUSH rule 1 x 471 [181,897,629,860,307]
+  CRUSH rule 1 x 472 [547,512,172,24,705]
+  CRUSH rule 1 x 473 [760,997,824,905,888]
+  CRUSH rule 1 x 474 [787,418,743,628,272]
+  CRUSH rule 1 x 475 [662,312,253,617,105]
+  CRUSH rule 1 x 476 [110,495,185,508,961]
+  CRUSH rule 1 x 477 [393,954,834,132,841]
+  CRUSH rule 1 x 478 [246,483,480,644,985]
+  CRUSH rule 1 x 479 [70,929,697,931,744]
+  CRUSH rule 1 x 480 [753,119,961,607,317]
+  CRUSH rule 1 x 481 [470,429,677,242,574]
+  CRUSH rule 1 x 482 [451,566,961,675,354]
+  CRUSH rule 1 x 483 [816,72,371,278,635]
+  CRUSH rule 1 x 484 [540,454,389,31,654]
+  CRUSH rule 1 x 485 [74,582,624,684,566]
+  CRUSH rule 1 x 486 [958,595,199,763,715]
+  CRUSH rule 1 x 487 [228,302,804,833,876]
+  CRUSH rule 1 x 488 [180,529,722,956,353]
+  CRUSH rule 1 x 489 [47,617,812,187,291]
+  CRUSH rule 1 x 490 [905,822,479,124,750]
+  CRUSH rule 1 x 491 [892,370,609,998,433]
+  CRUSH rule 1 x 492 [588,959,127,948,505]
+  CRUSH rule 1 x 493 [353,461,593,291,301]
+  CRUSH rule 1 x 494 [378,848,443,368,507]
+  CRUSH rule 1 x 495 [845,653,768,234,405]
+  CRUSH rule 1 x 496 [13,988,0,691,389]
+  CRUSH rule 1 x 497 [796,877,788,394,648]
+  CRUSH rule 1 x 498 [412,337,270,705,511]
+  CRUSH rule 1 x 499 [330,695,8,74,618]
+  CRUSH rule 1 x 500 [820,272,547,765,755]
+  CRUSH rule 1 x 501 [110,44,132,442,294]
+  CRUSH rule 1 x 502 [336,595,650,274,993]
+  CRUSH rule 1 x 503 [922,211,157,722,502]
+  CRUSH rule 1 x 504 [483,52,122,432,778]
+  CRUSH rule 1 x 505 [482,598,224,279,480]
+  CRUSH rule 1 x 506 [493,123,43,856,936]
+  CRUSH rule 1 x 507 [12,598,264,422,416]
+  CRUSH rule 1 x 508 [227,157,611,301,223]
+  CRUSH rule 1 x 509 [807,242,363,122,582]
+  CRUSH rule 1 x 510 [134,437,227,75,313]
+  CRUSH rule 1 x 511 [212,54,83,799,457]
+  CRUSH rule 1 x 512 [236,630,758,752,361]
+  CRUSH rule 1 x 513 [994,693,644,938,846]
+  CRUSH rule 1 x 514 [45,508,831,19,817]
+  CRUSH rule 1 x 515 [504,138,480,272,530]
+  CRUSH rule 1 x 516 [285,409,136,570,841]
+  CRUSH rule 1 x 517 [300,232,23,906,438]
+  CRUSH rule 1 x 518 [397,674,98,898,967]
+  CRUSH rule 1 x 519 [86,750,772,913,101]
+  CRUSH rule 1 x 520 [900,833,614,130,261]
+  CRUSH rule 1 x 521 [31,47,236,751,911]
+  CRUSH rule 1 x 522 [390,16,280,144,291]
+  CRUSH rule 1 x 523 [618,308,424,590,300]
+  CRUSH rule 1 x 524 [635,189,687,963,601]
+  CRUSH rule 1 x 525 [311,916,699,262,775]
+  CRUSH rule 1 x 526 [48,738,227,718,244]
+  CRUSH rule 1 x 527 [202,851,889,216,763]
+  CRUSH rule 1 x 528 [565,827,590,273,918]
+  CRUSH rule 1 x 529 [934,864,241,43,466]
+  CRUSH rule 1 x 530 [502,934,298,670,986]
+  CRUSH rule 1 x 531 [681,627,942,487,288]
+  CRUSH rule 1 x 532 [422,6,147,205,861]
+  CRUSH rule 1 x 533 [863,68,364,983,247]
+  CRUSH rule 1 x 534 [962,931,775,172,663]
+  CRUSH rule 1 x 535 [89,565,397,693,839]
+  CRUSH rule 1 x 536 [499,351,760,458,918]
+  CRUSH rule 1 x 537 [676,547,787,311,867]
+  CRUSH rule 1 x 538 [58,644,571,649,941]
+  CRUSH rule 1 x 539 [837,953,457,711,458]
+  CRUSH rule 1 x 540 [831,50,132,213,197]
+  CRUSH rule 1 x 541 [582,757,121,525,532]
+  CRUSH rule 1 x 542 [472,132,790,997,948]
+  CRUSH rule 1 x 543 [382,272,797,330,315]
+  CRUSH rule 1 x 544 [947,930,496,883,509]
+  CRUSH rule 1 x 545 [425,570,305,77,821]
+  CRUSH rule 1 x 546 [18,65,529,437,343]
+  CRUSH rule 1 x 547 [445,715,600,472,213]
+  CRUSH rule 1 x 548 [367,569,980,167,627]
+  CRUSH rule 1 x 549 [125,715,671,817,285]
+  CRUSH rule 1 x 550 [425,599,744,199,923]
+  CRUSH rule 1 x 551 [44,1,528,922,944]
+  CRUSH rule 1 x 552 [246,104,68,239,123]
+  CRUSH rule 1 x 553 [71,703,615,28,593]
+  CRUSH rule 1 x 554 [207,124,217,166,525]
+  CRUSH rule 1 x 555 [570,28,317,420,931]
+  CRUSH rule 1 x 556 [674,152,421,79,215]
+  CRUSH rule 1 x 557 [347,817,191,391,741]
+  CRUSH rule 1 x 558 [627,426,369,692,815]
+  CRUSH rule 1 x 559 [940,630,924,242,224]
+  CRUSH rule 1 x 560 [295,903,541,29,245]
+  CRUSH rule 1 x 561 [506,682,384,637,878]
+  CRUSH rule 1 x 562 [718,529,87,729,842]
+  CRUSH rule 1 x 563 [552,332,747,206,274]
+  CRUSH rule 1 x 564 [835,769,736,486,630]
+  CRUSH rule 1 x 565 [8,167,539,182,607]
+  CRUSH rule 1 x 566 [600,481,301,263,90]
+  CRUSH rule 1 x 567 [999,994,509,899,947]
+  CRUSH rule 1 x 568 [252,431,157,62,601]
+  CRUSH rule 1 x 569 [643,218,943,455,83]
+  CRUSH rule 1 x 570 [617,635,765,422,250]
+  CRUSH rule 1 x 571 [757,80,59,98,328]
+  CRUSH rule 1 x 572 [299,348,575,889,943]
+  CRUSH rule 1 x 573 [25,505,270,167,58]
+  CRUSH rule 1 x 574 [215,431,624,177,628]
+  CRUSH rule 1 x 575 [225,252,611,546,32]
+  CRUSH rule 1 x 576 [627,94,159,857,430]
+  CRUSH rule 1 x 577 [237,809,778,636,61]
+  CRUSH rule 1 x 578 [885,313,120,344,771]
+  CRUSH rule 1 x 579 [924,575,787,831,47]
+  CRUSH rule 1 x 580 [718,51,766,121,118]
+  CRUSH rule 1 x 581 [219,807,129,571,856]
+  CRUSH rule 1 x 582 [893,701,598,863,285]
+  CRUSH rule 1 x 583 [246,930,964,170,993]
+  CRUSH rule 1 x 584 [336,432,680,175,495]
+  CRUSH rule 1 x 585 [324,999,397,485,457]
+  CRUSH rule 1 x 586 [558,230,976,541,816]
+  CRUSH rule 1 x 587 [985,830,597,21,308]
+  CRUSH rule 1 x 588 [211,544,57,134,162]
+  CRUSH rule 1 x 589 [129,21,112,190,885]
+  CRUSH rule 1 x 590 [467,969,652,593,287]
+  CRUSH rule 1 x 591 [758,514,316,164,35]
+  CRUSH rule 1 x 592 [525,253,190,443,315]
+  CRUSH rule 1 x 593 [601,885,339,152,297]
+  CRUSH rule 1 x 594 [227,60,450,30,717]
+  CRUSH rule 1 x 595 [720,854,496,912,80]
+  CRUSH rule 1 x 596 [751,195,997,77,261]
+  CRUSH rule 1 x 597 [129,574,714,8,789]
+  CRUSH rule 1 x 598 [679,207,604,396,841]
+  CRUSH rule 1 x 599 [668,315,683,349,681]
+  CRUSH rule 1 x 600 [143,396,464,444,59]
+  CRUSH rule 1 x 601 [326,573,873,902,136]
+  CRUSH rule 1 x 602 [860,281,875,535,672]
+  CRUSH rule 1 x 603 [709,328,445,349,190]
+  CRUSH rule 1 x 604 [571,62,814,95,866]
+  CRUSH rule 1 x 605 [252,739,860,27,313]
+  CRUSH rule 1 x 606 [339,236,759,842,67]
+  CRUSH rule 1 x 607 [590,248,759,868,433]
+  CRUSH rule 1 x 608 [145,635,309,467,875]
+  CRUSH rule 1 x 609 [973,547,223,79,762]
+  CRUSH rule 1 x 610 [435,816,961,983,255]
+  CRUSH rule 1 x 611 [559,283,422,584,176]
+  CRUSH rule 1 x 612 [273,149,123,576,911]
+  CRUSH rule 1 x 613 [828,614,642,674,33]
+  CRUSH rule 1 x 614 [478,748,393,34,171]
+  CRUSH rule 1 x 615 [392,155,144,326,626]
+  CRUSH rule 1 x 616 [778,637,452,248,15]
+  CRUSH rule 1 x 617 [622,713,996,833,611]
+  CRUSH rule 1 x 618 [149,877,270,329,180]
+  CRUSH rule 1 x 619 [604,163,656,409,322]
+  CRUSH rule 1 x 620 [181,23,409,198,64]
+  CRUSH rule 1 x 621 [735,902,386,237,939]
+  CRUSH rule 1 x 622 [661,824,717,568,858]
+  CRUSH rule 1 x 623 [142,121,643,61,695]
+  CRUSH rule 1 x 624 [360,716,420,398,49]
+  CRUSH rule 1 x 625 [541,167,385,1,601]
+  CRUSH rule 1 x 626 [364,431,610,363,535]
+  CRUSH rule 1 x 627 [458,137,557,410,287]
+  CRUSH rule 1 x 628 [250,350,556,497,821]
+  CRUSH rule 1 x 629 [928,160,710,572,365]
+  CRUSH rule 1 x 630 [243,19,918,556,601]
+  CRUSH rule 1 x 631 [438,221,574,676,797]
+  CRUSH rule 1 x 632 [797,368,247,5,32]
+  CRUSH rule 1 x 633 [993,749,525,485,27]
+  CRUSH rule 1 x 634 [239,351,633,299,651]
+  CRUSH rule 1 x 635 [640,965,25,961,306]
+  CRUSH rule 1 x 636 [173,290,297,991,937]
+  CRUSH rule 1 x 637 [0,918,98,108,111]
+  CRUSH rule 1 x 638 [702,235,424,900,983]
+  CRUSH rule 1 x 639 [475,687,31,785,918]
+  CRUSH rule 1 x 640 [31,664,399,677,123]
+  CRUSH rule 1 x 641 [296,473,108,963,341]
+  CRUSH rule 1 x 642 [894,273,427,606,677]
+  CRUSH rule 1 x 643 [117,111,732,191,114]
+  CRUSH rule 1 x 644 [438,336,327,512,599]
+  CRUSH rule 1 x 645 [982,702,351,573,907]
+  CRUSH rule 1 x 646 [334,804,146,842,697]
+  CRUSH rule 1 x 647 [933,787,185,334,752]
+  CRUSH rule 1 x 648 [22,444,400,862,207]
+  CRUSH rule 1 x 649 [503,229,213,460,639]
+  CRUSH rule 1 x 650 [328,659,420,443,739]
+  CRUSH rule 1 x 651 [3,880,823,123,378]
+  CRUSH rule 1 x 652 [495,977,563,733,92]
+  CRUSH rule 1 x 653 [185,718,804,280,975]
+  CRUSH rule 1 x 654 [130,528,380,81,906]
+  CRUSH rule 1 x 655 [560,872,454,504,319]
+  CRUSH rule 1 x 656 [219,885,178,981,863]
+  CRUSH rule 1 x 657 [233,684,813,490,208]
+  CRUSH rule 1 x 658 [778,6,756,380,750]
+  CRUSH rule 1 x 659 [240,663,306,540,789]
+  CRUSH rule 1 x 660 [244,855,196,147,678]
+  CRUSH rule 1 x 661 [184,270,128,398,910]
+  CRUSH rule 1 x 662 [65,883,921,438,79]
+  CRUSH rule 1 x 663 [323,721,594,812,43]
+  CRUSH rule 1 x 664 [865,113,512,51,427]
+  CRUSH rule 1 x 665 [420,850,591,475,202]
+  CRUSH rule 1 x 666 [319,767,246,3,369]
+  CRUSH rule 1 x 667 [875,39,343,100,829]
+  CRUSH rule 1 x 668 [331,122,263,599,355]
+  CRUSH rule 1 x 669 [915,521,402,747,673]
+  CRUSH rule 1 x 670 [845,659,943,447,401]
+  CRUSH rule 1 x 671 [108,634,527,363,856]
+  CRUSH rule 1 x 672 [578,216,110,589,302]
+  CRUSH rule 1 x 673 [442,74,579,797,622]
+  CRUSH rule 1 x 674 [588,364,281,308,645]
+  CRUSH rule 1 x 675 [489,698,744,671,870]
+  CRUSH rule 1 x 676 [928,911,40,180,722]
+  CRUSH rule 1 x 677 [399,269,692,131,615]
+  CRUSH rule 1 x 678 [546,752,544,155,5]
+  CRUSH rule 1 x 679 [988,25,275,433,628]
+  CRUSH rule 1 x 680 [335,963,382,486,749]
+  CRUSH rule 1 x 681 [690,462,623,466,49]
+  CRUSH rule 1 x 682 [196,588,154,257,807]
+  CRUSH rule 1 x 683 [627,25,421,160,873]
+  CRUSH rule 1 x 684 [38,804,592,158,991]
+  CRUSH rule 1 x 685 [841,368,548,362,166]
+  CRUSH rule 1 x 686 [336,287,525,440,166]
+  CRUSH rule 1 x 687 [20,682,924,653,356]
+  CRUSH rule 1 x 688 [463,371,780,556,385]
+  CRUSH rule 1 x 689 [569,250,78,816,847]
+  CRUSH rule 1 x 690 [551,144,587,263,378]
+  CRUSH rule 1 x 691 [766,464,446,533,449]
+  CRUSH rule 1 x 692 [739,634,18,245,624]
+  CRUSH rule 1 x 693 [339,297,118,330,817]
+  CRUSH rule 1 x 694 [405,26,830,181,533]
+  CRUSH rule 1 x 695 [622,576,597,535,600]
+  CRUSH rule 1 x 696 [558,902,689,13,715]
+  CRUSH rule 1 x 697 [818,222,406,691,427]
+  CRUSH rule 1 x 698 [178,48,402,233,841]
+  CRUSH rule 1 x 699 [450,244,180,919,100]
+  CRUSH rule 1 x 700 [502,771,987,706,416]
+  CRUSH rule 1 x 701 [4,612,782,216,853]
+  CRUSH rule 1 x 702 [177,630,232,923,281]
+  CRUSH rule 1 x 703 [354,178,389,393,778]
+  CRUSH rule 1 x 704 [646,601,156,171,603]
+  CRUSH rule 1 x 705 [921,401,890,265,244]
+  CRUSH rule 1 x 706 [652,877,562,452,26]
+  CRUSH rule 1 x 707 [345,745,67,716,789]
+  CRUSH rule 1 x 708 [333,607,180,469,170]
+  CRUSH rule 1 x 709 [45,187,302,115,896]
+  CRUSH rule 1 x 710 [94,855,43,199,18]
+  CRUSH rule 1 x 711 [227,653,731,150,452]
+  CRUSH rule 1 x 712 [398,953,136,870,181]
+  CRUSH rule 1 x 713 [116,800,503,662,635]
+  CRUSH rule 1 x 714 [111,629,866,709,902]
+  CRUSH rule 1 x 715 [531,291,486,382,192]
+  CRUSH rule 1 x 716 [169,541,291,42,343]
+  CRUSH rule 1 x 717 [417,446,994,894,239]
+  CRUSH rule 1 x 718 [992,383,298,844,377]
+  CRUSH rule 1 x 719 [936,674,324,759,194]
+  CRUSH rule 1 x 720 [370,188,174,464,644]
+  CRUSH rule 1 x 721 [320,859,278,259,170]
+  CRUSH rule 1 x 722 [7,2,673,129,96]
+  CRUSH rule 1 x 723 [270,553,831,662,38]
+  CRUSH rule 1 x 724 [666,822,708,895,633]
+  CRUSH rule 1 x 725 [794,406,875,459,981]
+  CRUSH rule 1 x 726 [420,556,341,292,240]
+  CRUSH rule 1 x 727 [561,461,129,635,965]
+  CRUSH rule 1 x 728 [951,330,196,756,589]
+  CRUSH rule 1 x 729 [656,644,436,591,27]
+  CRUSH rule 1 x 730 [3,558,629,184,50]
+  CRUSH rule 1 x 731 [852,89,75,735,713]
+  CRUSH rule 1 x 732 [983,840,869,976,697]
+  CRUSH rule 1 x 733 [285,396,388,122,387]
+  CRUSH rule 1 x 734 [125,510,402,640,676]
+  CRUSH rule 1 x 735 [417,773,686,504,459]
+  CRUSH rule 1 x 736 [749,396,632,550,779]
+  CRUSH rule 1 x 737 [644,991,946,135,448]
+  CRUSH rule 1 x 738 [449,683,290,220,245]
+  CRUSH rule 1 x 739 [341,220,641,454,740]
+  CRUSH rule 1 x 740 [874,524,674,650,472]
+  CRUSH rule 1 x 741 [189,472,712,798,715]
+  CRUSH rule 1 x 742 [912,581,114,145,730]
+  CRUSH rule 1 x 743 [654,914,425,441,763]
+  CRUSH rule 1 x 744 [725,295,579,377,162]
+  CRUSH rule 1 x 745 [787,858,850,506,612]
+  CRUSH rule 1 x 746 [757,848,704,30,47]
+  CRUSH rule 1 x 747 [700,81,867,681,801]
+  CRUSH rule 1 x 748 [557,436,238,664,293]
+  CRUSH rule 1 x 749 [772,622,337,42,156]
+  CRUSH rule 1 x 750 [946,97,376,677,316]
+  CRUSH rule 1 x 751 [996,618,343,911,83]
+  CRUSH rule 1 x 752 [746,887,695,868,610]
+  CRUSH rule 1 x 753 [741,14,463,479,172]
+  CRUSH rule 1 x 754 [648,349,333,355,65]
+  CRUSH rule 1 x 755 [157,460,466,187,959]
+  CRUSH rule 1 x 756 [416,97,197,497,227]
+  CRUSH rule 1 x 757 [599,839,776,410,256]
+  CRUSH rule 1 x 758 [994,218,620,256,361]
+  CRUSH rule 1 x 759 [959,682,514,745,100]
+  CRUSH rule 1 x 760 [518,943,215,83,706]
+  CRUSH rule 1 x 761 [285,849,420,324,987]
+  CRUSH rule 1 x 762 [591,313,41,335,110]
+  CRUSH rule 1 x 763 [908,411,200,740,292]
+  CRUSH rule 1 x 764 [787,234,894,485,883]
+  CRUSH rule 1 x 765 [327,921,882,393,444]
+  CRUSH rule 1 x 766 [84,161,878,704,416]
+  CRUSH rule 1 x 767 [370,895,702,701,890]
+  CRUSH rule 1 x 768 [826,760,879,864,460]
+  CRUSH rule 1 x 769 [67,768,663,735,814]
+  CRUSH rule 1 x 770 [593,909,482,259,5]
+  CRUSH rule 1 x 771 [309,935,121,578,937]
+  CRUSH rule 1 x 772 [12,125,797,301,348]
+  CRUSH rule 1 x 773 [253,466,820,549,591]
+  CRUSH rule 1 x 774 [164,390,705,109,881]
+  CRUSH rule 1 x 775 [703,47,43,973,643]
+  CRUSH rule 1 x 776 [728,231,80,916,2]
+  CRUSH rule 1 x 777 [981,621,568,729,869]
+  CRUSH rule 1 x 778 [411,456,544,597,789]
+  CRUSH rule 1 x 779 [346,121,519,921,587]
+  CRUSH rule 1 x 780 [476,39,288,381,303]
+  CRUSH rule 1 x 781 [10,130,585,844,729]
+  CRUSH rule 1 x 782 [462,246,581,902,623]
+  CRUSH rule 1 x 783 [580,373,153,775,668]
+  CRUSH rule 1 x 784 [413,113,978,990,994]
+  CRUSH rule 1 x 785 [341,856,332,354,59]
+  CRUSH rule 1 x 786 [411,140,313,393,215]
+  CRUSH rule 1 x 787 [605,522,211,813,636]
+  CRUSH rule 1 x 788 [226,545,35,142,726]
+  CRUSH rule 1 x 789 [545,320,414,702,731]
+  CRUSH rule 1 x 790 [414,748,816,327,130]
+  CRUSH rule 1 x 791 [660,906,406,697,916]
+  CRUSH rule 1 x 792 [287,392,514,204,75]
+  CRUSH rule 1 x 793 [631,133,850,713,720]
+  CRUSH rule 1 x 794 [931,517,543,210,963]
+  CRUSH rule 1 x 795 [551,962,477,948,425]
+  CRUSH rule 1 x 796 [814,4,95,27,368]
+  CRUSH rule 1 x 797 [64,201,299,734,605]
+  CRUSH rule 1 x 798 [422,530,114,431,565]
+  CRUSH rule 1 x 799 [824,32,679,562,266]
+  CRUSH rule 1 x 800 [862,623,489,637,861]
+  CRUSH rule 1 x 801 [145,550,329,324,734]
+  CRUSH rule 1 x 802 [570,19,847,308,387]
+  CRUSH rule 1 x 803 [151,812,662,358,880]
+  CRUSH rule 1 x 804 [467,93,264,863,176]
+  CRUSH rule 1 x 805 [621,223,938,809,591]
+  CRUSH rule 1 x 806 [898,957,805,430,499]
+  CRUSH rule 1 x 807 [354,531,422,159,921]
+  CRUSH rule 1 x 808 [7,96,76,897,446]
+  CRUSH rule 1 x 809 [70,734,719,56,687]
+  CRUSH rule 1 x 810 [701,18,972,327,771]
+  CRUSH rule 1 x 811 [248,547,103,728,901]
+  CRUSH rule 1 x 812 [230,576,821,566,993]
+  CRUSH rule 1 x 813 [805,114,683,629,742]
+  CRUSH rule 1 x 814 [54,619,973,741,497]
+  CRUSH rule 1 x 815 [679,412,613,132,969]
+  CRUSH rule 1 x 816 [919,448,826,414,36]
+  CRUSH rule 1 x 817 [765,830,436,521,332]
+  CRUSH rule 1 x 818 [415,566,644,687,692]
+  CRUSH rule 1 x 819 [721,319,865,750,546]
+  CRUSH rule 1 x 820 [218,301,333,190,686]
+  CRUSH rule 1 x 821 [185,795,680,953,329]
+  CRUSH rule 1 x 822 [356,261,54,522,900]
+  CRUSH rule 1 x 823 [220,281,549,456,64]
+  CRUSH rule 1 x 824 [292,809,887,74,776]
+  CRUSH rule 1 x 825 [949,778,101,311,110]
+  CRUSH rule 1 x 826 [767,818,833,927,356]
+  CRUSH rule 1 x 827 [631,83,406,635,657]
+  CRUSH rule 1 x 828 [288,986,445,26,414]
+  CRUSH rule 1 x 829 [990,667,915,694,974]
+  CRUSH rule 1 x 830 [152,571,778,505,685]
+  CRUSH rule 1 x 831 [814,563,630,97,582]
+  CRUSH rule 1 x 832 [235,641,616,110,979]
+  CRUSH rule 1 x 833 [657,565,922,140,825]
+  CRUSH rule 1 x 834 [907,231,644,13,617]
+  CRUSH rule 1 x 835 [784,262,771,264,612]
+  CRUSH rule 1 x 836 [951,158,366,710,43]
+  CRUSH rule 1 x 837 [556,498,334,633,895]
+  CRUSH rule 1 x 838 [329,274,964,547,119]
+  CRUSH rule 1 x 839 [568,209,939,364,658]
+  CRUSH rule 1 x 840 [45,579,842,70,655]
+  CRUSH rule 1 x 841 [652,702,24,605,152]
+  CRUSH rule 1 x 842 [629,984,314,895,408]
+  CRUSH rule 1 x 843 [799,690,688,648,151]
+  CRUSH rule 1 x 844 [694,600,534,700,569]
+  CRUSH rule 1 x 845 [332,30,179,93,951]
+  CRUSH rule 1 x 846 [452,251,712,719,404]
+  CRUSH rule 1 x 847 [399,681,847,739,13]
+  CRUSH rule 1 x 848 [303,138,440,346,547]
+  CRUSH rule 1 x 849 [666,346,708,873,64]
+  CRUSH rule 1 x 850 [644,511,345,844,545]
+  CRUSH rule 1 x 851 [527,546,737,425,100]
+  CRUSH rule 1 x 852 [31,809,94,618,156]
+  CRUSH rule 1 x 853 [483,330,869,184,46]
+  CRUSH rule 1 x 854 [697,953,968,143,502]
+  CRUSH rule 1 x 855 [837,996,239,621,32]
+  CRUSH rule 1 x 856 [712,40,547,430,195]
+  CRUSH rule 1 x 857 [77,984,576,551,568]
+  CRUSH rule 1 x 858 [412,384,841,465,572]
+  CRUSH rule 1 x 859 [173,760,26,300,87]
+  CRUSH rule 1 x 860 [776,429,328,917,658]
+  CRUSH rule 1 x 861 [705,405,477,50,73]
+  CRUSH rule 1 x 862 [809,44,788,938,964]
+  CRUSH rule 1 x 863 [349,496,963,178,675]
+  CRUSH rule 1 x 864 [717,858,101,239,992]
+  CRUSH rule 1 x 865 [857,603,586,262,550]
+  CRUSH rule 1 x 866 [394,304,71,96,642]
+  CRUSH rule 1 x 867 [640,773,663,974,261]
+  CRUSH rule 1 x 868 [613,950,712,663,16]
+  CRUSH rule 1 x 869 [973,889,524,22,671]
+  CRUSH rule 1 x 870 [505,35,386,498,348]
+  CRUSH rule 1 x 871 [239,264,262,773,781]
+  CRUSH rule 1 x 872 [21,767,456,748,783]
+  CRUSH rule 1 x 873 [954,666,980,264,435]
+  CRUSH rule 1 x 874 [54,510,947,1,500]
+  CRUSH rule 1 x 875 [809,418,452,462,88]
+  CRUSH rule 1 x 876 [483,457,61,248,523]
+  CRUSH rule 1 x 877 [542,531,952,939,710]
+  CRUSH rule 1 x 878 [217,674,857,644,678]
+  CRUSH rule 1 x 879 [999,475,134,250,319]
+  CRUSH rule 1 x 880 [678,573,935,385,570]
+  CRUSH rule 1 x 881 [394,835,789,802,587]
+  CRUSH rule 1 x 882 [467,382,353,56,979]
+  CRUSH rule 1 x 883 [802,744,237,337,50]
+  CRUSH rule 1 x 884 [653,660,638,700,31]
+  CRUSH rule 1 x 885 [898,704,307,445,879]
+  CRUSH rule 1 x 886 [434,357,938,641,737]
+  CRUSH rule 1 x 887 [297,226,711,428,370]
+  CRUSH rule 1 x 888 [863,324,443,213,902]
+  CRUSH rule 1 x 889 [105,102,308,163,947]
+  CRUSH rule 1 x 890 [550,248,606,704,615]
+  CRUSH rule 1 x 891 [575,928,880,891,826]
+  CRUSH rule 1 x 892 [259,862,133,271,292]
+  CRUSH rule 1 x 893 [902,880,543,542,37]
+  CRUSH rule 1 x 894 [180,169,916,43,945]
+  CRUSH rule 1 x 895 [725,849,182,129,177]
+  CRUSH rule 1 x 896 [951,34,874,537,969]
+  CRUSH rule 1 x 897 [810,352,73,939,943]
+  CRUSH rule 1 x 898 [979,433,719,411,787]
+  CRUSH rule 1 x 899 [685,668,534,932,399]
+  CRUSH rule 1 x 900 [530,978,41,894,941]
+  CRUSH rule 1 x 901 [740,107,336,175,574]
+  CRUSH rule 1 x 902 [800,743,693,310,67]
+  CRUSH rule 1 x 903 [230,267,842,266,550]
+  CRUSH rule 1 x 904 [346,949,460,973,696]
+  CRUSH rule 1 x 905 [530,397,619,958,576]
+  CRUSH rule 1 x 906 [80,426,138,672,73]
+  CRUSH rule 1 x 907 [365,968,475,297,296]
+  CRUSH rule 1 x 908 [204,832,742,809,862]
+  CRUSH rule 1 x 909 [883,989,146,959,366]
+  CRUSH rule 1 x 910 [549,593,249,853,792]
+  CRUSH rule 1 x 911 [325,847,352,214,851]
+  CRUSH rule 1 x 912 [874,888,582,796,557]
+  CRUSH rule 1 x 913 [331,463,342,574,989]
+  CRUSH rule 1 x 914 [836,468,601,732,607]
+  CRUSH rule 1 x 915 [245,228,100,661,799]
+  CRUSH rule 1 x 916 [77,967,364,435,27]
+  CRUSH rule 1 x 917 [239,60,866,221,772]
+  CRUSH rule 1 x 918 [988,115,922,80,201]
+  CRUSH rule 1 x 919 [783,139,696,1,848]
+  CRUSH rule 1 x 920 [623,408,685,953,974]
+  CRUSH rule 1 x 921 [105,799,144,90,399]
+  CRUSH rule 1 x 922 [887,505,652,348,514]
+  CRUSH rule 1 x 923 [223,318,552,458,743]
+  CRUSH rule 1 x 924 [25,778,366,333,163]
+  CRUSH rule 1 x 925 [912,601,297,682,770]
+  CRUSH rule 1 x 926 [968,133,158,144,814]
+  CRUSH rule 1 x 927 [277,724,214,988,690]
+  CRUSH rule 1 x 928 [554,203,658,789,298]
+  CRUSH rule 1 x 929 [761,802,367,528,758]
+  CRUSH rule 1 x 930 [814,61,788,736,660]
+  CRUSH rule 1 x 931 [29,193,61,41,343]
+  CRUSH rule 1 x 932 [446,198,862,534,168]
+  CRUSH rule 1 x 933 [352,742,216,321,525]
+  CRUSH rule 1 x 934 [730,2,332,631,613]
+  CRUSH rule 1 x 935 [731,23,736,79,361]
+  CRUSH rule 1 x 936 [322,975,20,904,827]
+  CRUSH rule 1 x 937 [822,221,841,161,723]
+  CRUSH rule 1 x 938 [557,850,66,630,499]
+  CRUSH rule 1 x 939 [150,11,971,371,124]
+  CRUSH rule 1 x 940 [638,398,169,616,333]
+  CRUSH rule 1 x 941 [730,342,929,577,451]
+  CRUSH rule 1 x 942 [62,292,166,814,587]
+  CRUSH rule 1 x 943 [165,314,519,548,41]
+  CRUSH rule 1 x 944 [199,625,766,176,194]
+  CRUSH rule 1 x 945 [946,999,699,303,38]
+  CRUSH rule 1 x 946 [595,93,852,142,503]
+  CRUSH rule 1 x 947 [800,582,356,93,716]
+  CRUSH rule 1 x 948 [132,551,139,920,87]
+  CRUSH rule 1 x 949 [792,920,466,380,97]
+  CRUSH rule 1 x 950 [111,345,176,543,879]
+  CRUSH rule 1 x 951 [414,619,648,655,364]
+  CRUSH rule 1 x 952 [775,469,500,356,287]
+  CRUSH rule 1 x 953 [349,1,5,251,168]
+  CRUSH rule 1 x 954 [570,940,410,249,929]
+  CRUSH rule 1 x 955 [729,774,823,800,7]
+  CRUSH rule 1 x 956 [519,141,575,625,738]
+  CRUSH rule 1 x 957 [242,709,611,97,760]
+  CRUSH rule 1 x 958 [84,217,227,253,246]
+  CRUSH rule 1 x 959 [270,413,918,789,703]
+  CRUSH rule 1 x 960 [458,192,307,279,920]
+  CRUSH rule 1 x 961 [981,388,777,546,359]
+  CRUSH rule 1 x 962 [623,834,277,134,729]
+  CRUSH rule 1 x 963 [291,167,714,468,109]
+  CRUSH rule 1 x 964 [28,156,788,127,598]
+  CRUSH rule 1 x 965 [675,557,290,517,840]
+  CRUSH rule 1 x 966 [836,306,946,283,642]
+  CRUSH rule 1 x 967 [966,386,735,837,392]
+  CRUSH rule 1 x 968 [864,756,690,121,328]
+  CRUSH rule 1 x 969 [729,625,480,769,512]
+  CRUSH rule 1 x 970 [800,362,646,582,309]
+  CRUSH rule 1 x 971 [737,381,153,684,298]
+  CRUSH rule 1 x 972 [952,245,720,884,334]
+  CRUSH rule 1 x 973 [356,455,579,857,832]
+  CRUSH rule 1 x 974 [545,758,586,596,405]
+  CRUSH rule 1 x 975 [336,191,202,146,720]
+  CRUSH rule 1 x 976 [446,208,757,620,252]
+  CRUSH rule 1 x 977 [202,896,196,956,763]
+  CRUSH rule 1 x 978 [612,324,996,225,418]
+  CRUSH rule 1 x 979 [843,457,675,650,958]
+  CRUSH rule 1 x 980 [60,914,881,626,850]
+  CRUSH rule 1 x 981 [702,749,937,153,724]
+  CRUSH rule 1 x 982 [298,928,738,167,99]
+  CRUSH rule 1 x 983 [723,572,395,358,900]
+  CRUSH rule 1 x 984 [723,864,804,935,846]
+  CRUSH rule 1 x 985 [945,459,868,211,524]
+  CRUSH rule 1 x 986 [772,664,535,169,297]
+  CRUSH rule 1 x 987 [88,324,312,843,661]
+  CRUSH rule 1 x 988 [522,927,131,996,351]
+  CRUSH rule 1 x 989 [578,332,208,605,975]
+  CRUSH rule 1 x 990 [638,228,414,311,738]
+  CRUSH rule 1 x 991 [530,221,451,422,879]
+  CRUSH rule 1 x 992 [925,705,275,81,234]
+  CRUSH rule 1 x 993 [991,301,43,469,830]
+  CRUSH rule 1 x 994 [276,51,868,683,843]
+  CRUSH rule 1 x 995 [288,836,753,790,758]
+  CRUSH rule 1 x 996 [887,983,252,686,470]
+  CRUSH rule 1 x 997 [110,924,386,79,705]
+  CRUSH rule 1 x 998 [435,830,485,853,926]
+  CRUSH rule 1 x 999 [876,738,357,913,723]
+  CRUSH rule 1 x 1000 [178,963,638,430,845]
+  CRUSH rule 1 x 1001 [99,519,66,759,583]
+  CRUSH rule 1 x 1002 [515,534,468,866,878]
+  CRUSH rule 1 x 1003 [104,611,937,698,94]
+  CRUSH rule 1 x 1004 [269,638,724,375,491]
+  CRUSH rule 1 x 1005 [369,223,309,409,822]
+  CRUSH rule 1 x 1006 [40,107,69,275,79]
+  CRUSH rule 1 x 1007 [978,111,416,758,454]
+  CRUSH rule 1 x 1008 [965,956,624,832,421]
+  CRUSH rule 1 x 1009 [598,476,356,695,919]
+  CRUSH rule 1 x 1010 [767,523,239,517,29]
+  CRUSH rule 1 x 1011 [289,871,207,576,347]
+  CRUSH rule 1 x 1012 [128,28,370,31,341]
+  CRUSH rule 1 x 1013 [979,765,660,812,666]
+  CRUSH rule 1 x 1014 [979,948,513,88,47]
+  CRUSH rule 1 x 1015 [277,790,396,672,542]
+  CRUSH rule 1 x 1016 [262,73,128,886,839]
+  CRUSH rule 1 x 1017 [150,269,61,499,832]
+  CRUSH rule 1 x 1018 [555,829,554,944,406]
+  CRUSH rule 1 x 1019 [513,356,265,446,65]
+  CRUSH rule 1 x 1020 [158,161,877,704,948]
+  CRUSH rule 1 x 1021 [915,998,957,285,546]
+  CRUSH rule 1 x 1022 [967,829,973,640,703]
+  CRUSH rule 1 x 1023 [488,257,614,859,325]
+  rule 1 (metadata) num_rep 5 result size == 5:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536,450,604,380]
+  CRUSH rule 1 x 1 [876,250,334,633,744,843]
+  CRUSH rule 1 x 2 [292,832,53,392,386,787]
+  CRUSH rule 1 x 3 [623,387,124,998,749,211]
+  CRUSH rule 1 x 4 [61,334,710,4,994,982]
+  CRUSH rule 1 x 5 [946,557,713,664,141,817]
+  CRUSH rule 1 x 6 [576,668,212,163,732,381]
+  CRUSH rule 1 x 7 [645,753,906,393,341,44]
+  CRUSH rule 1 x 8 [243,6,863,781,211,100]
+  CRUSH rule 1 x 9 [22,578,251,410,297,430]
+  CRUSH rule 1 x 10 [758,828,360,477,821,801]
+  CRUSH rule 1 x 11 [769,120,124,527,119,504]
+  CRUSH rule 1 x 12 [780,364,689,755,675,199]
+  CRUSH rule 1 x 13 [557,18,351,719,742,780]
+  CRUSH rule 1 x 14 [59,561,249,461,971,835]
+  CRUSH rule 1 x 15 [718,928,993,21,76,313]
+  CRUSH rule 1 x 16 [673,632,841,954,788,90]
+  CRUSH rule 1 x 17 [648,43,560,514,142,289]
+  CRUSH rule 1 x 18 [654,219,181,568,381,253]
+  CRUSH rule 1 x 19 [850,545,377,848,863,543]
+  CRUSH rule 1 x 20 [717,785,974,5,225,552]
+  CRUSH rule 1 x 21 [420,57,519,306,312,983]
+  CRUSH rule 1 x 22 [503,998,193,821,634,684]
+  CRUSH rule 1 x 23 [411,663,168,110,899,488]
+  CRUSH rule 1 x 24 [266,861,353,1,456,128]
+  CRUSH rule 1 x 25 [760,483,818,600,509,951]
+  CRUSH rule 1 x 26 [903,24,573,718,112,694]
+  CRUSH rule 1 x 27 [946,188,289,510,687,827]
+  CRUSH rule 1 x 28 [69,312,73,198,256,629]
+  CRUSH rule 1 x 29 [844,883,337,628,496,405]
+  CRUSH rule 1 x 30 [621,18,613,794,910,936]
+  CRUSH rule 1 x 31 [784,943,814,539,962,392]
+  CRUSH rule 1 x 32 [173,374,369,972,315,83]
+  CRUSH rule 1 x 33 [698,336,357,966,582,407]
+  CRUSH rule 1 x 34 [168,836,210,798,904,190]
+  CRUSH rule 1 x 35 [274,509,534,818,912,671]
+  CRUSH rule 1 x 36 [318,215,153,628,87,407]
+  CRUSH rule 1 x 37 [173,604,109,935,203,401]
+  CRUSH rule 1 x 38 [708,444,683,604,722,900]
+  CRUSH rule 1 x 39 [662,198,417,680,226,342]
+  CRUSH rule 1 x 40 [620,801,414,78,560,766]
+  CRUSH rule 1 x 41 [811,264,177,127,148,791]
+  CRUSH rule 1 x 42 [863,179,527,660,133,529]
+  CRUSH rule 1 x 43 [686,822,988,228,791,549]
+  CRUSH rule 1 x 44 [396,222,46,841,536,140]
+  CRUSH rule 1 x 45 [991,694,253,142,54,422]
+  CRUSH rule 1 x 46 [420,909,184,285,508,458]
+  CRUSH rule 1 x 47 [467,211,605,207,241,881]
+  CRUSH rule 1 x 48 [955,329,368,168,698,787]
+  CRUSH rule 1 x 49 [974,891,931,29,813,506]
+  CRUSH rule 1 x 50 [870,441,691,823,761,6]
+  CRUSH rule 1 x 51 [182,930,25,936,97,260]
+  CRUSH rule 1 x 52 [704,812,894,794,481,37]
+  CRUSH rule 1 x 53 [185,713,631,280,345,558]
+  CRUSH rule 1 x 54 [270,441,100,82,983,930]
+  CRUSH rule 1 x 55 [895,734,958,793,651,572]
+  CRUSH rule 1 x 56 [564,963,683,324,40,189]
+  CRUSH rule 1 x 57 [738,130,208,973,498,861]
+  CRUSH rule 1 x 58 [524,113,806,903,531,334]
+  CRUSH rule 1 x 59 [408,337,668,529,34,384]
+  CRUSH rule 1 x 60 [228,790,857,309,616,895]
+  CRUSH rule 1 x 61 [154,843,717,467,883,536]
+  CRUSH rule 1 x 62 [594,811,549,276,693,917]
+  CRUSH rule 1 x 63 [646,67,884,925,941,434]
+  CRUSH rule 1 x 64 [175,542,155,837,594,197]
+  CRUSH rule 1 x 65 [745,619,131,867,269,62]
+  CRUSH rule 1 x 66 [275,468,23,35,328,432]
+  CRUSH rule 1 x 67 [246,958,524,493,636,227]
+  CRUSH rule 1 x 68 [711,473,403,228,835,126]
+  CRUSH rule 1 x 69 [493,924,850,939,950,105]
+  CRUSH rule 1 x 70 [30,499,644,33,804,654]
+  CRUSH rule 1 x 71 [984,883,574,716,575,391]
+  CRUSH rule 1 x 72 [71,286,942,363,628,632]
+  CRUSH rule 1 x 73 [922,618,3,371,464,442]
+  CRUSH rule 1 x 74 [629,414,185,573,678,338]
+  CRUSH rule 1 x 75 [222,20,174,820,312,361]
+  CRUSH rule 1 x 76 [262,366,339,290,718,143]
+  CRUSH rule 1 x 77 [638,469,992,280,773,892]
+  CRUSH rule 1 x 78 [324,511,788,7,308,228]
+  CRUSH rule 1 x 79 [577,990,64,94,447,924]
+  CRUSH rule 1 x 80 [501,95,278,903,631,842]
+  CRUSH rule 1 x 81 [506,812,9,698,173,664]
+  CRUSH rule 1 x 82 [222,145,80,785,835,745]
+  CRUSH rule 1 x 83 [71,634,61,91,856,529]
+  CRUSH rule 1 x 84 [49,761,773,368,318,708]
+  CRUSH rule 1 x 85 [985,896,708,861,325,307]
+  CRUSH rule 1 x 86 [537,745,93,524,466,356]
+  CRUSH rule 1 x 87 [997,317,463,626,685,429]
+  CRUSH rule 1 x 88 [957,350,890,857,375,176]
+  CRUSH rule 1 x 89 [399,730,148,314,159,982]
+  CRUSH rule 1 x 90 [943,706,683,267,579,141]
+  CRUSH rule 1 x 91 [22,368,149,928,140,529]
+  CRUSH rule 1 x 92 [532,424,426,773,623,197]
+  CRUSH rule 1 x 93 [218,489,405,681,549,201]
+  CRUSH rule 1 x 94 [181,96,102,515,776,365]
+  CRUSH rule 1 x 95 [343,957,820,139,334,37]
+  CRUSH rule 1 x 96 [861,270,87,797,0,245]
+  CRUSH rule 1 x 97 [459,706,45,328,274,605]
+  CRUSH rule 1 x 98 [327,867,353,948,728,280]
+  CRUSH rule 1 x 99 [974,133,468,906,235,988]
+  CRUSH rule 1 x 100 [32,445,547,371,960,885]
+  CRUSH rule 1 x 101 [142,90,337,950,970,570]
+  CRUSH rule 1 x 102 [172,129,139,22,403,867]
+  CRUSH rule 1 x 103 [630,47,161,356,911,421]
+  CRUSH rule 1 x 104 [758,133,278,11,947,799]
+  CRUSH rule 1 x 105 [843,604,47,33,401,632]
+  CRUSH rule 1 x 106 [28,681,193,679,990,343]
+  CRUSH rule 1 x 107 [74,320,85,819,315,253]
+  CRUSH rule 1 x 108 [875,593,575,517,107,153]
+  CRUSH rule 1 x 109 [411,985,811,720,198,666]
+  CRUSH rule 1 x 110 [440,774,799,660,715,167]
+  CRUSH rule 1 x 111 [405,742,276,359,936,360]
+  CRUSH rule 1 x 112 [143,181,922,545,185,303]
+  CRUSH rule 1 x 113 [153,846,160,903,789,897]
+  CRUSH rule 1 x 114 [804,892,939,20,312,692]
+  CRUSH rule 1 x 115 [588,508,958,580,232,722]
+  CRUSH rule 1 x 116 [327,148,637,486,712,464]
+  CRUSH rule 1 x 117 [95,594,989,131,714,275]
+  CRUSH rule 1 x 118 [80,957,897,239,359,432]
+  CRUSH rule 1 x 119 [386,932,951,768,679,300]
+  CRUSH rule 1 x 120 [366,312,653,936,71,241]
+  CRUSH rule 1 x 121 [129,154,847,16,471,481]
+  CRUSH rule 1 x 122 [873,1,110,939,90,412]
+  CRUSH rule 1 x 123 [533,415,789,600,713,800]
+  CRUSH rule 1 x 124 [461,691,898,723,957,759]
+  CRUSH rule 1 x 125 [342,599,830,402,615,994]
+  CRUSH rule 1 x 126 [819,781,822,548,279,255]
+  CRUSH rule 1 x 127 [437,893,585,707,353,189]
+  CRUSH rule 1 x 128 [679,994,982,550,991,324]
+  CRUSH rule 1 x 129 [380,685,947,302,698,144]
+  CRUSH rule 1 x 130 [992,52,466,867,998,777]
+  CRUSH rule 1 x 131 [469,90,208,599,829,656]
+  CRUSH rule 1 x 132 [571,250,316,535,54,418]
+  CRUSH rule 1 x 133 [964,728,329,902,108,118]
+  CRUSH rule 1 x 134 [999,19,716,963,323,559]
+  CRUSH rule 1 x 135 [634,101,52,938,413,573]
+  CRUSH rule 1 x 136 [114,889,692,768,694,279]
+  CRUSH rule 1 x 137 [839,8,959,280,922,870]
+  CRUSH rule 1 x 138 [967,949,138,451,292,548]
+  CRUSH rule 1 x 139 [308,711,736,247,632,126]
+  CRUSH rule 1 x 140 [764,936,926,55,331,115]
+  CRUSH rule 1 x 141 [423,302,112,216,603,873]
+  CRUSH rule 1 x 142 [252,821,715,340,635,668]
+  CRUSH rule 1 x 143 [33,808,518,477,325,316]
+  CRUSH rule 1 x 144 [472,88,969,162,401,771]
+  CRUSH rule 1 x 145 [242,208,252,604,266,743]
+  CRUSH rule 1 x 146 [290,70,570,384,934,856]
+  CRUSH rule 1 x 147 [447,352,657,493,467,918]
+  CRUSH rule 1 x 148 [212,644,432,658,109,275]
+  CRUSH rule 1 x 149 [9,775,87,35,260,646]
+  CRUSH rule 1 x 150 [166,456,582,144,324,340]
+  CRUSH rule 1 x 151 [811,875,307,20,782,229]
+  CRUSH rule 1 x 152 [449,617,223,9,182,407]
+  CRUSH rule 1 x 153 [523,537,695,627,959,613]
+  CRUSH rule 1 x 154 [208,559,874,597,243,706]
+  CRUSH rule 1 x 155 [569,325,192,296,367,848]
+  CRUSH rule 1 x 156 [488,121,521,213,595,837]
+  CRUSH rule 1 x 157 [140,723,633,260,487,856]
+  CRUSH rule 1 x 158 [786,451,320,239,667,632]
+  CRUSH rule 1 x 159 [134,664,517,821,667,944]
+  CRUSH rule 1 x 160 [690,112,414,990,183,590]
+  CRUSH rule 1 x 161 [324,912,397,423,991,284]
+  CRUSH rule 1 x 162 [748,567,284,183,463,336]
+  CRUSH rule 1 x 163 [575,499,31,816,749,737]
+  CRUSH rule 1 x 164 [314,489,308,326,51,568]
+  CRUSH rule 1 x 165 [116,209,750,53,813,640]
+  CRUSH rule 1 x 166 [352,706,701,810,718,527]
+  CRUSH rule 1 x 167 [27,743,174,142,551,1]
+  CRUSH rule 1 x 168 [953,898,880,660,500,799]
+  CRUSH rule 1 x 169 [912,147,266,547,331,770]
+  CRUSH rule 1 x 170 [421,515,828,844,151,981]
+  CRUSH rule 1 x 171 [488,584,880,964,936,196]
+  CRUSH rule 1 x 172 [366,443,957,66,162,693]
+  CRUSH rule 1 x 173 [863,291,625,287,158,496]
+  CRUSH rule 1 x 174 [263,555,650,410,339,616]
+  CRUSH rule 1 x 175 [875,961,361,575,33,109]
+  CRUSH rule 1 x 176 [745,83,701,680,250,420]
+  CRUSH rule 1 x 177 [128,244,41,123,422,902]
+  CRUSH rule 1 x 178 [155,41,264,777,314,564]
+  CRUSH rule 1 x 179 [593,833,202,183,971,38]
+  CRUSH rule 1 x 180 [154,734,17,831,824,522]
+  CRUSH rule 1 x 181 [289,675,723,800,166,712]
+  CRUSH rule 1 x 182 [730,931,560,209,943,261]
+  CRUSH rule 1 x 183 [639,237,794,815,827,400]
+  CRUSH rule 1 x 184 [704,312,685,645,691,778]
+  CRUSH rule 1 x 185 [97,100,762,82,999,542]
+  CRUSH rule 1 x 186 [26,665,554,215,280,421]
+  CRUSH rule 1 x 187 [649,14,740,494,402,684]
+  CRUSH rule 1 x 188 [682,695,590,743,927,945]
+  CRUSH rule 1 x 189 [325,693,726,51,448,169]
+  CRUSH rule 1 x 190 [399,933,136,955,57,504]
+  CRUSH rule 1 x 191 [629,533,17,126,60,146]
+  CRUSH rule 1 x 192 [503,578,38,492,222,251]
+  CRUSH rule 1 x 193 [546,333,651,678,823,652]
+  CRUSH rule 1 x 194 [242,473,58,655,911,277]
+  CRUSH rule 1 x 195 [625,719,135,81,636,513]
+  CRUSH rule 1 x 196 [357,114,125,867,250,522]
+  CRUSH rule 1 x 197 [306,954,453,873,211,334]
+  CRUSH rule 1 x 198 [863,791,311,911,206,61]
+  CRUSH rule 1 x 199 [935,906,929,252,893,75]
+  CRUSH rule 1 x 200 [373,774,229,454,909,611]
+  CRUSH rule 1 x 201 [659,320,477,313,779,16]
+  CRUSH rule 1 x 202 [260,433,524,880,223,818]
+  CRUSH rule 1 x 203 [36,239,675,971,703,209]
+  CRUSH rule 1 x 204 [92,516,993,728,279,478]
+  CRUSH rule 1 x 205 [68,395,473,45,683,662]
+  CRUSH rule 1 x 206 [570,530,642,380,311,398]
+  CRUSH rule 1 x 207 [834,457,850,917,456,296]
+  CRUSH rule 1 x 208 [927,484,640,976,803,626]
+  CRUSH rule 1 x 209 [878,66,58,940,48,233]
+  CRUSH rule 1 x 210 [572,981,484,29,0,426]
+  CRUSH rule 1 x 211 [107,597,780,857,895,57]
+  CRUSH rule 1 x 212 [389,107,838,624,698,562]
+  CRUSH rule 1 x 213 [497,717,567,728,905,134]
+  CRUSH rule 1 x 214 [798,65,254,572,32,393]
+  CRUSH rule 1 x 215 [233,419,283,638,520,891]
+  CRUSH rule 1 x 216 [494,464,742,523,459,174]
+  CRUSH rule 1 x 217 [352,396,309,938,66,41]
+  CRUSH rule 1 x 218 [895,864,988,650,593,740]
+  CRUSH rule 1 x 219 [222,534,277,242,658,482]
+  CRUSH rule 1 x 220 [281,19,584,563,858,965]
+  CRUSH rule 1 x 221 [64,928,963,130,312,394]
+  CRUSH rule 1 x 222 [40,544,161,199,861,644]
+  CRUSH rule 1 x 223 [645,556,159,417,46,135]
+  CRUSH rule 1 x 224 [647,165,957,263,961,576]
+  CRUSH rule 1 x 225 [219,714,858,747,461,175]
+  CRUSH rule 1 x 226 [372,511,181,277,695,404]
+  CRUSH rule 1 x 227 [925,156,714,863,257,74]
+  CRUSH rule 1 x 228 [682,404,839,263,521,195]
+  CRUSH rule 1 x 229 [880,838,770,891,236,542]
+  CRUSH rule 1 x 230 [328,659,916,468,646,572]
+  CRUSH rule 1 x 231 [320,383,669,109,627,621]
+  CRUSH rule 1 x 232 [924,846,394,319,43,519]
+  CRUSH rule 1 x 233 [948,652,575,838,498,395]
+  CRUSH rule 1 x 234 [484,943,42,575,936,180]
+  CRUSH rule 1 x 235 [750,65,590,168,870,308]
+  CRUSH rule 1 x 236 [551,787,490,136,370,833]
+  CRUSH rule 1 x 237 [390,157,166,251,752,75]
+  CRUSH rule 1 x 238 [570,6,989,707,514,905]
+  CRUSH rule 1 x 239 [729,959,376,975,496,49]
+  CRUSH rule 1 x 240 [981,241,156,767,631,576]
+  CRUSH rule 1 x 241 [310,816,641,177,996,454]
+  CRUSH rule 1 x 242 [161,63,642,837,763,458]
+  CRUSH rule 1 x 243 [180,394,33,683,189,419]
+  CRUSH rule 1 x 244 [52,174,685,189,78,310]
+  CRUSH rule 1 x 245 [523,121,915,84,386,409]
+  CRUSH rule 1 x 246 [362,893,390,487,817,88]
+  CRUSH rule 1 x 247 [382,184,116,34,143,15]
+  CRUSH rule 1 x 248 [129,114,852,469,359,291]
+  CRUSH rule 1 x 249 [159,683,91,856,475,369]
+  CRUSH rule 1 x 250 [404,945,569,955,228,910]
+  CRUSH rule 1 x 251 [661,225,738,757,37,642]
+  CRUSH rule 1 x 252 [961,226,542,103,945,885]
+  CRUSH rule 1 x 253 [651,97,225,364,189,248]
+  CRUSH rule 1 x 254 [123,33,741,692,599,11]
+  CRUSH rule 1 x 255 [314,649,891,855,517,344]
+  CRUSH rule 1 x 256 [315,215,651,126,470,849]
+  CRUSH rule 1 x 257 [825,264,867,841,529,409]
+  CRUSH rule 1 x 258 [624,789,370,723,131,982]
+  CRUSH rule 1 x 259 [602,542,70,563,947,723]
+  CRUSH rule 1 x 260 [717,878,43,56,377,481]
+  CRUSH rule 1 x 261 [145,517,20,903,786,939]
+  CRUSH rule 1 x 262 [223,1,561,420,357,16]
+  CRUSH rule 1 x 263 [462,211,405,508,787,669]
+  CRUSH rule 1 x 264 [654,471,266,662,135,564]
+  CRUSH rule 1 x 265 [302,794,704,798,659,487]
+  CRUSH rule 1 x 266 [202,132,884,209,551,984]
+  CRUSH rule 1 x 267 [282,938,657,113,672,993]
+  CRUSH rule 1 x 268 [338,309,356,278,928,797]
+  CRUSH rule 1 x 269 [738,122,266,200,894,118]
+  CRUSH rule 1 x 270 [707,982,946,196,407,804]
+  CRUSH rule 1 x 271 [705,432,364,735,512,595]
+  CRUSH rule 1 x 272 [756,545,942,56,542,449]
+  CRUSH rule 1 x 273 [197,502,527,721,239,648]
+  CRUSH rule 1 x 274 [992,44,653,573,527,702]
+  CRUSH rule 1 x 275 [544,789,170,434,23,926]
+  CRUSH rule 1 x 276 [658,467,577,268,336,5]
+  CRUSH rule 1 x 277 [143,490,880,483,928,272]
+  CRUSH rule 1 x 278 [492,647,355,282,834,64]
+  CRUSH rule 1 x 279 [517,792,604,987,527,894]
+  CRUSH rule 1 x 280 [825,740,27,848,514,750]
+  CRUSH rule 1 x 281 [224,629,120,562,616,200]
+  CRUSH rule 1 x 282 [298,661,380,416,35,585]
+  CRUSH rule 1 x 283 [311,606,208,50,913,678]
+  CRUSH rule 1 x 284 [771,466,371,743,672,119]
+  CRUSH rule 1 x 285 [693,362,404,676,797,531]
+  CRUSH rule 1 x 286 [364,477,285,167,270,617]
+  CRUSH rule 1 x 287 [591,611,828,995,170,987]
+  CRUSH rule 1 x 288 [965,541,848,796,251,668]
+  CRUSH rule 1 x 289 [225,551,948,877,219,167]
+  CRUSH rule 1 x 290 [577,762,777,751,291,349]
+  CRUSH rule 1 x 291 [160,903,477,381,490,559]
+  CRUSH rule 1 x 292 [873,598,216,666,222,228]
+  CRUSH rule 1 x 293 [100,234,874,47,28,452]
+  CRUSH rule 1 x 294 [285,943,379,520,725,547]
+  CRUSH rule 1 x 295 [938,262,880,327,687,3]
+  CRUSH rule 1 x 296 [850,327,86,472,1,776]
+  CRUSH rule 1 x 297 [951,53,99,558,753,228]
+  CRUSH rule 1 x 298 [173,336,85,766,910,657]
+  CRUSH rule 1 x 299 [598,591,315,386,895,296]
+  CRUSH rule 1 x 300 [531,957,62,459,156,538]
+  CRUSH rule 1 x 301 [823,628,23,858,629,808]
+  CRUSH rule 1 x 302 [184,80,780,871,531,211]
+  CRUSH rule 1 x 303 [521,766,222,830,988,275]
+  CRUSH rule 1 x 304 [980,127,807,507,555,245]
+  CRUSH rule 1 x 305 [153,816,22,927,696,911]
+  CRUSH rule 1 x 306 [423,739,664,753,178,431]
+  CRUSH rule 1 x 307 [997,557,682,456,479,631]
+  CRUSH rule 1 x 308 [991,874,534,465,330,284]
+  CRUSH rule 1 x 309 [860,394,724,858,246,866]
+  CRUSH rule 1 x 310 [589,818,546,201,94,653]
+  CRUSH rule 1 x 311 [477,774,225,590,830,559]
+  CRUSH rule 1 x 312 [887,853,950,354,58,23]
+  CRUSH rule 1 x 313 [802,646,447,416,557,118]
+  CRUSH rule 1 x 314 [654,974,229,511,562,916]
+  CRUSH rule 1 x 315 [767,227,28,740,828,156]
+  CRUSH rule 1 x 316 [778,83,733,359,858,319]
+  CRUSH rule 1 x 317 [184,418,642,986,939,675]
+  CRUSH rule 1 x 318 [525,410,500,543,212,95]
+  CRUSH rule 1 x 319 [476,724,569,382,409,521]
+  CRUSH rule 1 x 320 [149,610,697,296,818,955]
+  CRUSH rule 1 x 321 [710,79,667,671,234,4]
+  CRUSH rule 1 x 322 [175,275,323,333,744,718]
+  CRUSH rule 1 x 323 [819,604,638,792,316,544]
+  CRUSH rule 1 x 324 [16,745,511,439,272,205]
+  CRUSH rule 1 x 325 [486,400,872,873,251,68]
+  CRUSH rule 1 x 326 [613,765,207,19,359,370]
+  CRUSH rule 1 x 327 [125,289,738,408,456,784]
+  CRUSH rule 1 x 328 [807,383,476,583,645,141]
+  CRUSH rule 1 x 329 [588,938,599,432,446,840]
+  CRUSH rule 1 x 330 [932,644,41,611,209,406]
+  CRUSH rule 1 x 331 [341,953,950,537,578,862]
+  CRUSH rule 1 x 332 [153,726,459,950,466,804]
+  CRUSH rule 1 x 333 [745,845,853,860,52,615]
+  CRUSH rule 1 x 334 [614,751,807,58,396,159]
+  CRUSH rule 1 x 335 [518,721,221,283,454,187]
+  CRUSH rule 1 x 336 [389,424,77,309,5,898]
+  CRUSH rule 1 x 337 [753,508,765,720,221,807]
+  CRUSH rule 1 x 338 [128,810,490,753,406,760]
+  CRUSH rule 1 x 339 [430,308,58,751,856,823]
+  CRUSH rule 1 x 340 [541,44,630,231,289,966]
+  CRUSH rule 1 x 341 [402,26,631,439,165,928]
+  CRUSH rule 1 x 342 [982,57,992,461,131,32]
+  CRUSH rule 1 x 343 [833,412,572,732,107,805]
+  CRUSH rule 1 x 344 [784,533,792,41,642,869]
+  CRUSH rule 1 x 345 [546,300,304,691,763,556]
+  CRUSH rule 1 x 346 [302,420,428,891,357,124]
+  CRUSH rule 1 x 347 [488,778,101,217,366,442]
+  CRUSH rule 1 x 348 [903,744,937,718,85,314]
+  CRUSH rule 1 x 349 [471,547,582,306,600,486]
+  CRUSH rule 1 x 350 [348,221,823,335,383,708]
+  CRUSH rule 1 x 351 [961,582,705,346,361,32]
+  CRUSH rule 1 x 352 [728,137,461,298,36,903]
+  CRUSH rule 1 x 353 [904,202,184,447,58,294]
+  CRUSH rule 1 x 354 [345,226,319,256,544,311]
+  CRUSH rule 1 x 355 [50,430,175,43,187,458]
+  CRUSH rule 1 x 356 [87,185,55,423,829,1]
+  CRUSH rule 1 x 357 [762,459,921,473,182,231]
+  CRUSH rule 1 x 358 [908,25,280,6,808,676]
+  CRUSH rule 1 x 359 [484,15,132,121,394,423]
+  CRUSH rule 1 x 360 [173,378,337,702,145,499]
+  CRUSH rule 1 x 361 [404,577,115,25,56,914]
+  CRUSH rule 1 x 362 [403,1,422,945,132,685]
+  CRUSH rule 1 x 363 [639,911,510,162,418,294]
+  CRUSH rule 1 x 364 [752,689,610,990,665,222]
+  CRUSH rule 1 x 365 [956,999,212,230,624,84]
+  CRUSH rule 1 x 366 [860,925,924,763,687,851]
+  CRUSH rule 1 x 367 [205,609,647,665,969,720]
+  CRUSH rule 1 x 368 [301,284,810,169,78,340]
+  CRUSH rule 1 x 369 [452,658,339,217,674,210]
+  CRUSH rule 1 x 370 [11,467,695,989,394,576]
+  CRUSH rule 1 x 371 [124,487,55,514,313,411]
+  CRUSH rule 1 x 372 [253,48,979,846,207,631]
+  CRUSH rule 1 x 373 [715,605,775,748,227,493]
+  CRUSH rule 1 x 374 [191,887,920,549,223,714]
+  CRUSH rule 1 x 375 [711,385,651,665,15,71]
+  CRUSH rule 1 x 376 [597,818,49,458,415,755]
+  CRUSH rule 1 x 377 [294,256,933,771,184,861]
+  CRUSH rule 1 x 378 [34,151,681,707,552,127]
+  CRUSH rule 1 x 379 [869,136,315,378,813,153]
+  CRUSH rule 1 x 380 [294,97,575,791,690,482]
+  CRUSH rule 1 x 381 [119,710,219,827,328,886]
+  CRUSH rule 1 x 382 [69,631,508,706,697,168]
+  CRUSH rule 1 x 383 [922,588,589,925,471,601]
+  CRUSH rule 1 x 384 [221,945,671,117,857,655]
+  CRUSH rule 1 x 385 [561,737,953,723,658,368]
+  CRUSH rule 1 x 386 [335,442,788,696,507,716]
+  CRUSH rule 1 x 387 [514,43,353,88,100,842]
+  CRUSH rule 1 x 388 [587,89,157,996,915,927]
+  CRUSH rule 1 x 389 [109,641,255,466,372,563]
+  CRUSH rule 1 x 390 [925,149,421,489,599,810]
+  CRUSH rule 1 x 391 [267,87,387,527,768,873]
+  CRUSH rule 1 x 392 [382,485,370,849,936,636]
+  CRUSH rule 1 x 393 [425,721,221,753,268,463]
+  CRUSH rule 1 x 394 [898,18,38,793,173,738]
+  CRUSH rule 1 x 395 [806,876,269,679,32,744]
+  CRUSH rule 1 x 396 [790,970,437,449,875,395]
+  CRUSH rule 1 x 397 [136,363,507,613,11,30]
+  CRUSH rule 1 x 398 [914,116,558,258,722,904]
+  CRUSH rule 1 x 399 [261,94,299,202,174,622]
+  CRUSH rule 1 x 400 [661,197,338,461,977,848]
+  CRUSH rule 1 x 401 [953,979,287,803,41,349]
+  CRUSH rule 1 x 402 [738,819,618,522,667,334]
+  CRUSH rule 1 x 403 [573,238,425,546,130,68]
+  CRUSH rule 1 x 404 [526,848,790,253,922,820]
+  CRUSH rule 1 x 405 [582,505,330,334,201,110]
+  CRUSH rule 1 x 406 [768,324,493,60,186,165]
+  CRUSH rule 1 x 407 [260,951,437,587,692,648]
+  CRUSH rule 1 x 408 [657,81,770,734,830,821]
+  CRUSH rule 1 x 409 [498,89,182,423,672,152]
+  CRUSH rule 1 x 410 [28,793,737,352,166,645]
+  CRUSH rule 1 x 411 [684,992,60,659,769,267]
+  CRUSH rule 1 x 412 [261,958,699,950,165,14]
+  CRUSH rule 1 x 413 [891,835,297,441,384,979]
+  CRUSH rule 1 x 414 [127,459,119,965,662,594]
+  CRUSH rule 1 x 415 [272,540,631,328,609,568]
+  CRUSH rule 1 x 416 [739,617,115,530,339,371]
+  CRUSH rule 1 x 417 [106,209,157,878,117,128]
+  CRUSH rule 1 x 418 [525,441,147,390,320,300]
+  CRUSH rule 1 x 419 [603,673,615,465,266,855]
+  CRUSH rule 1 x 420 [988,213,251,226,209,245]
+  CRUSH rule 1 x 421 [761,521,748,368,923,992]
+  CRUSH rule 1 x 422 [317,160,924,548,198,709]
+  CRUSH rule 1 x 423 [137,807,168,472,619,443]
+  CRUSH rule 1 x 424 [920,37,146,263,598,748]
+  CRUSH rule 1 x 425 [277,693,285,221,478,165]
+  CRUSH rule 1 x 426 [485,936,407,854,726,524]
+  CRUSH rule 1 x 427 [242,515,9,564,174,453]
+  CRUSH rule 1 x 428 [632,635,26,473,494,478]
+  CRUSH rule 1 x 429 [641,73,465,127,171,397]
+  CRUSH rule 1 x 430 [626,585,6,387,881,583]
+  CRUSH rule 1 x 431 [697,76,753,570,964,339]
+  CRUSH rule 1 x 432 [590,526,306,283,656,728]
+  CRUSH rule 1 x 433 [284,387,149,817,886,714]
+  CRUSH rule 1 x 434 [538,985,79,953,770,468]
+  CRUSH rule 1 x 435 [30,318,593,635,975,833]
+  CRUSH rule 1 x 436 [164,919,851,693,0,874]
+  CRUSH rule 1 x 437 [322,212,163,606,302,282]
+  CRUSH rule 1 x 438 [142,392,85,594,376,419]
+  CRUSH rule 1 x 439 [119,370,68,443,997,837]
+  CRUSH rule 1 x 440 [333,403,187,863,475,844]
+  CRUSH rule 1 x 441 [477,727,906,145,429,91]
+  CRUSH rule 1 x 442 [274,590,933,244,434,49]
+  CRUSH rule 1 x 443 [983,748,574,718,700,442]
+  CRUSH rule 1 x 444 [536,509,431,146,170,149]
+  CRUSH rule 1 x 445 [485,931,528,209,964,753]
+  CRUSH rule 1 x 446 [345,634,42,294,711,376]
+  CRUSH rule 1 x 447 [61,845,767,600,321,716]
+  CRUSH rule 1 x 448 [333,232,292,846,364,951]
+  CRUSH rule 1 x 449 [680,16,484,670,851,500]
+  CRUSH rule 1 x 450 [235,214,79,423,96,822]
+  CRUSH rule 1 x 451 [961,468,333,640,823,151]
+  CRUSH rule 1 x 452 [525,479,153,528,570,806]
+  CRUSH rule 1 x 453 [138,466,302,86,249,154]
+  CRUSH rule 1 x 454 [137,625,215,402,389,914]
+  CRUSH rule 1 x 455 [173,150,997,16,846,888]
+  CRUSH rule 1 x 456 [235,226,238,258,347,784]
+  CRUSH rule 1 x 457 [450,577,253,413,717,609]
+  CRUSH rule 1 x 458 [195,537,91,814,351,90]
+  CRUSH rule 1 x 459 [381,555,312,573,915,623]
+  CRUSH rule 1 x 460 [972,730,534,678,756,692]
+  CRUSH rule 1 x 461 [506,279,142,830,784,124]
+  CRUSH rule 1 x 462 [692,959,578,57,983,299]
+  CRUSH rule 1 x 463 [788,667,949,550,685,702]
+  CRUSH rule 1 x 464 [133,122,588,999,270,880]
+  CRUSH rule 1 x 465 [971,190,230,777,452,914]
+  CRUSH rule 1 x 466 [394,576,148,157,103,822]
+  CRUSH rule 1 x 467 [517,28,366,362,984,521]
+  CRUSH rule 1 x 468 [829,143,874,225,162,413]
+  CRUSH rule 1 x 469 [987,936,106,725,633,238]
+  CRUSH rule 1 x 470 [107,982,56,889,67,65]
+  CRUSH rule 1 x 471 [181,897,629,860,307,116]
+  CRUSH rule 1 x 472 [547,512,172,24,705,837]
+  CRUSH rule 1 x 473 [760,997,824,905,888,755]
+  CRUSH rule 1 x 474 [787,418,743,628,272,341]
+  CRUSH rule 1 x 475 [662,312,253,617,105,58]
+  CRUSH rule 1 x 476 [110,495,185,508,961,837]
+  CRUSH rule 1 x 477 [393,954,834,132,841,367]
+  CRUSH rule 1 x 478 [246,483,480,644,985,420]
+  CRUSH rule 1 x 479 [70,929,697,931,744,487]
+  CRUSH rule 1 x 480 [753,119,961,607,317,717]
+  CRUSH rule 1 x 481 [470,429,677,242,574,757]
+  CRUSH rule 1 x 482 [451,566,961,675,354,746]
+  CRUSH rule 1 x 483 [816,72,371,278,635,30]
+  CRUSH rule 1 x 484 [540,454,389,31,654,494]
+  CRUSH rule 1 x 485 [74,582,624,684,566,677]
+  CRUSH rule 1 x 486 [958,595,199,763,715,973]
+  CRUSH rule 1 x 487 [228,302,804,833,876,647]
+  CRUSH rule 1 x 488 [180,529,722,956,353,890]
+  CRUSH rule 1 x 489 [47,617,812,187,291,828]
+  CRUSH rule 1 x 490 [905,822,479,124,750,843]
+  CRUSH rule 1 x 491 [892,370,609,998,433,957]
+  CRUSH rule 1 x 492 [588,959,127,948,505,936]
+  CRUSH rule 1 x 493 [353,461,593,291,301,830]
+  CRUSH rule 1 x 494 [378,848,443,368,507,423]
+  CRUSH rule 1 x 495 [845,653,768,234,405,367]
+  CRUSH rule 1 x 496 [13,988,0,691,389,757]
+  CRUSH rule 1 x 497 [796,877,788,394,648,829]
+  CRUSH rule 1 x 498 [412,337,270,705,511,227]
+  CRUSH rule 1 x 499 [330,695,8,74,618,101]
+  CRUSH rule 1 x 500 [820,272,547,765,755,96]
+  CRUSH rule 1 x 501 [110,44,132,442,294,423]
+  CRUSH rule 1 x 502 [336,595,650,274,993,312]
+  CRUSH rule 1 x 503 [922,211,157,722,502,971]
+  CRUSH rule 1 x 504 [483,52,122,432,778,461]
+  CRUSH rule 1 x 505 [482,598,224,279,480,310]
+  CRUSH rule 1 x 506 [493,123,43,856,936,622]
+  CRUSH rule 1 x 507 [12,598,264,422,416,947]
+  CRUSH rule 1 x 508 [227,157,611,301,223,746]
+  CRUSH rule 1 x 509 [807,242,363,122,582,530]
+  CRUSH rule 1 x 510 [134,437,227,75,313,351]
+  CRUSH rule 1 x 511 [212,54,83,799,457,218]
+  CRUSH rule 1 x 512 [236,630,758,752,361,249]
+  CRUSH rule 1 x 513 [994,693,644,938,846,685]
+  CRUSH rule 1 x 514 [45,508,831,19,817,52]
+  CRUSH rule 1 x 515 [504,138,480,272,530,377]
+  CRUSH rule 1 x 516 [285,409,136,570,841,610]
+  CRUSH rule 1 x 517 [300,232,23,906,438,236]
+  CRUSH rule 1 x 518 [397,674,98,898,967,113]
+  CRUSH rule 1 x 519 [86,750,772,913,101,864]
+  CRUSH rule 1 x 520 [900,833,614,130,261,885]
+  CRUSH rule 1 x 521 [31,47,236,751,911,599]
+  CRUSH rule 1 x 522 [390,16,280,144,291,175]
+  CRUSH rule 1 x 523 [618,308,424,590,300,206]
+  CRUSH rule 1 x 524 [635,189,687,963,601,518]
+  CRUSH rule 1 x 525 [311,916,699,262,775,32]
+  CRUSH rule 1 x 526 [48,738,227,718,244,942]
+  CRUSH rule 1 x 527 [202,851,889,216,763,351]
+  CRUSH rule 1 x 528 [565,827,590,273,918,106]
+  CRUSH rule 1 x 529 [934,864,241,43,466,924]
+  CRUSH rule 1 x 530 [502,934,298,670,986,360]
+  CRUSH rule 1 x 531 [681,627,942,487,288,561]
+  CRUSH rule 1 x 532 [422,6,147,205,861,141]
+  CRUSH rule 1 x 533 [863,68,364,983,247,199]
+  CRUSH rule 1 x 534 [962,931,775,172,663,119]
+  CRUSH rule 1 x 535 [89,565,397,693,839,632]
+  CRUSH rule 1 x 536 [499,351,760,458,918,86]
+  CRUSH rule 1 x 537 [676,547,787,311,867,748]
+  CRUSH rule 1 x 538 [58,644,571,649,941,7]
+  CRUSH rule 1 x 539 [837,953,457,711,458,621]
+  CRUSH rule 1 x 540 [831,50,132,213,197,709]
+  CRUSH rule 1 x 541 [582,757,121,525,532,963]
+  CRUSH rule 1 x 542 [472,132,790,997,948,269]
+  CRUSH rule 1 x 543 [382,272,797,330,315,748]
+  CRUSH rule 1 x 544 [947,930,496,883,509,219]
+  CRUSH rule 1 x 545 [425,570,305,77,821,422]
+  CRUSH rule 1 x 546 [18,65,529,437,343,547]
+  CRUSH rule 1 x 547 [445,715,600,472,213,851]
+  CRUSH rule 1 x 548 [367,569,980,167,627,442]
+  CRUSH rule 1 x 549 [125,715,671,817,285,420]
+  CRUSH rule 1 x 550 [425,599,744,199,923,222]
+  CRUSH rule 1 x 551 [44,1,528,922,944,115]
+  CRUSH rule 1 x 552 [246,104,68,239,123,427]
+  CRUSH rule 1 x 553 [71,703,615,28,593,724]
+  CRUSH rule 1 x 554 [207,124,217,166,525,226]
+  CRUSH rule 1 x 555 [570,28,317,420,931,413]
+  CRUSH rule 1 x 556 [674,152,421,79,215,347]
+  CRUSH rule 1 x 557 [347,817,191,391,741,571]
+  CRUSH rule 1 x 558 [627,426,369,692,815,371]
+  CRUSH rule 1 x 559 [940,630,924,242,224,912]
+  CRUSH rule 1 x 560 [295,903,541,29,245,753]
+  CRUSH rule 1 x 561 [506,682,384,637,878,991]
+  CRUSH rule 1 x 562 [718,529,87,729,842,341]
+  CRUSH rule 1 x 563 [552,332,747,206,274,871]
+  CRUSH rule 1 x 564 [835,769,736,486,630,209]
+  CRUSH rule 1 x 565 [8,167,539,182,607,62]
+  CRUSH rule 1 x 566 [600,481,301,263,90,450]
+  CRUSH rule 1 x 567 [999,994,509,899,947,24]
+  CRUSH rule 1 x 568 [252,431,157,62,601,863]
+  CRUSH rule 1 x 569 [643,218,943,455,83,969]
+  CRUSH rule 1 x 570 [617,635,765,422,250,156]
+  CRUSH rule 1 x 571 [757,80,59,98,328,700]
+  CRUSH rule 1 x 572 [299,348,575,889,943,675]
+  CRUSH rule 1 x 573 [25,505,270,167,58,901]
+  CRUSH rule 1 x 574 [215,431,624,177,628,814]
+  CRUSH rule 1 x 575 [225,252,611,546,32,815]
+  CRUSH rule 1 x 576 [627,94,159,857,430,691]
+  CRUSH rule 1 x 577 [237,809,778,636,61,167]
+  CRUSH rule 1 x 578 [885,313,120,344,771,614]
+  CRUSH rule 1 x 579 [924,575,787,831,47,996]
+  CRUSH rule 1 x 580 [718,51,766,121,118,471]
+  CRUSH rule 1 x 581 [219,807,129,571,856,179]
+  CRUSH rule 1 x 582 [893,701,598,863,285,829]
+  CRUSH rule 1 x 583 [246,930,964,170,993,409]
+  CRUSH rule 1 x 584 [336,432,680,175,495,839]
+  CRUSH rule 1 x 585 [324,999,397,485,457,527]
+  CRUSH rule 1 x 586 [558,230,976,541,816,72]
+  CRUSH rule 1 x 587 [985,830,597,21,308,890]
+  CRUSH rule 1 x 588 [211,544,57,134,162,496]
+  CRUSH rule 1 x 589 [129,21,112,190,885,844]
+  CRUSH rule 1 x 590 [467,969,652,593,287,76]
+  CRUSH rule 1 x 591 [758,514,316,164,35,110]
+  CRUSH rule 1 x 592 [525,253,190,443,315,603]
+  CRUSH rule 1 x 593 [601,885,339,152,297,223]
+  CRUSH rule 1 x 594 [227,60,450,30,717,840]
+  CRUSH rule 1 x 595 [720,854,496,912,80,655]
+  CRUSH rule 1 x 596 [751,195,997,77,261,490]
+  CRUSH rule 1 x 597 [129,574,714,8,789,847]
+  CRUSH rule 1 x 598 [679,207,604,396,841,284]
+  CRUSH rule 1 x 599 [668,315,683,349,681,253]
+  CRUSH rule 1 x 600 [143,396,464,444,59,57]
+  CRUSH rule 1 x 601 [326,573,873,902,136,921]
+  CRUSH rule 1 x 602 [860,281,875,535,672,474]
+  CRUSH rule 1 x 603 [709,328,445,349,190,455]
+  CRUSH rule 1 x 604 [571,62,814,95,866,978]
+  CRUSH rule 1 x 605 [252,739,860,27,313,362]
+  CRUSH rule 1 x 606 [339,236,759,842,67,644]
+  CRUSH rule 1 x 607 [590,248,759,868,433,398]
+  CRUSH rule 1 x 608 [145,635,309,467,875,115]
+  CRUSH rule 1 x 609 [973,547,223,79,762,863]
+  CRUSH rule 1 x 610 [435,816,961,983,255,886]
+  CRUSH rule 1 x 611 [559,283,422,584,176,429]
+  CRUSH rule 1 x 612 [273,149,123,576,911,270]
+  CRUSH rule 1 x 613 [828,614,642,674,33,361]
+  CRUSH rule 1 x 614 [478,748,393,34,171,80]
+  CRUSH rule 1 x 615 [392,155,144,326,626,134]
+  CRUSH rule 1 x 616 [778,637,452,248,15,888]
+  CRUSH rule 1 x 617 [622,713,996,833,611,407]
+  CRUSH rule 1 x 618 [149,877,270,329,180,327]
+  CRUSH rule 1 x 619 [604,163,656,409,322,848]
+  CRUSH rule 1 x 620 [181,23,409,198,64,898]
+  CRUSH rule 1 x 621 [735,902,386,237,939,475]
+  CRUSH rule 1 x 622 [661,824,717,568,858,583]
+  CRUSH rule 1 x 623 [142,121,643,61,695,852]
+  CRUSH rule 1 x 624 [360,716,420,398,49,717]
+  CRUSH rule 1 x 625 [541,167,385,1,601,481]
+  CRUSH rule 1 x 626 [364,431,610,363,535,747]
+  CRUSH rule 1 x 627 [458,137,557,410,287,749]
+  CRUSH rule 1 x 628 [250,350,556,497,821,65]
+  CRUSH rule 1 x 629 [928,160,710,572,365,772]
+  CRUSH rule 1 x 630 [243,19,918,556,601,16]
+  CRUSH rule 1 x 631 [438,221,574,676,797,580]
+  CRUSH rule 1 x 632 [797,368,247,5,32,102]
+  CRUSH rule 1 x 633 [993,749,525,485,27,330]
+  CRUSH rule 1 x 634 [239,351,633,299,651,678]
+  CRUSH rule 1 x 635 [640,965,25,961,306,172]
+  CRUSH rule 1 x 636 [173,290,297,991,937,823]
+  CRUSH rule 1 x 637 [0,918,98,108,111,495]
+  CRUSH rule 1 x 638 [702,235,424,900,983,754]
+  CRUSH rule 1 x 639 [475,687,31,785,918,611]
+  CRUSH rule 1 x 640 [31,664,399,677,123,609]
+  CRUSH rule 1 x 641 [296,473,108,963,341,876]
+  CRUSH rule 1 x 642 [894,273,427,606,677,670]
+  CRUSH rule 1 x 643 [117,111,732,191,114,153]
+  CRUSH rule 1 x 644 [438,336,327,512,599,862]
+  CRUSH rule 1 x 645 [982,702,351,573,907,915]
+  CRUSH rule 1 x 646 [334,804,146,842,697,638]
+  CRUSH rule 1 x 647 [933,787,185,334,752,285]
+  CRUSH rule 1 x 648 [22,444,400,862,207,842]
+  CRUSH rule 1 x 649 [503,229,213,460,639,760]
+  CRUSH rule 1 x 650 [328,659,420,443,739,950]
+  CRUSH rule 1 x 651 [3,880,823,123,378,585]
+  CRUSH rule 1 x 652 [495,977,563,733,92,997]
+  CRUSH rule 1 x 653 [185,718,804,280,975,912]
+  CRUSH rule 1 x 654 [130,528,380,81,906,511]
+  CRUSH rule 1 x 655 [560,872,454,504,319,284]
+  CRUSH rule 1 x 656 [219,885,178,981,863,508]
+  CRUSH rule 1 x 657 [233,684,813,490,208,941]
+  CRUSH rule 1 x 658 [778,6,756,380,750,836]
+  CRUSH rule 1 x 659 [240,663,306,540,789,902]
+  CRUSH rule 1 x 660 [244,855,196,147,678,323]
+  CRUSH rule 1 x 661 [184,270,128,398,910,230]
+  CRUSH rule 1 x 662 [65,883,921,438,79,957]
+  CRUSH rule 1 x 663 [323,721,594,812,43,992]
+  CRUSH rule 1 x 664 [865,113,512,51,427,123]
+  CRUSH rule 1 x 665 [420,850,591,475,202,733]
+  CRUSH rule 1 x 666 [319,767,246,3,369,493]
+  CRUSH rule 1 x 667 [875,39,343,100,829,2]
+  CRUSH rule 1 x 668 [331,122,263,599,355,484]
+  CRUSH rule 1 x 669 [915,521,402,747,673,445]
+  CRUSH rule 1 x 670 [845,659,943,447,401,322]
+  CRUSH rule 1 x 671 [108,634,527,363,856,238]
+  CRUSH rule 1 x 672 [578,216,110,589,302,137]
+  CRUSH rule 1 x 673 [442,74,579,797,622,950]
+  CRUSH rule 1 x 674 [588,364,281,308,645,631]
+  CRUSH rule 1 x 675 [489,698,744,671,870,174]
+  CRUSH rule 1 x 676 [928,911,40,180,722,729]
+  CRUSH rule 1 x 677 [399,269,692,131,615,136]
+  CRUSH rule 1 x 678 [546,752,544,155,5,463]
+  CRUSH rule 1 x 679 [988,25,275,433,628,57]
+  CRUSH rule 1 x 680 [335,963,382,486,749,257]
+  CRUSH rule 1 x 681 [690,462,623,466,49,471]
+  CRUSH rule 1 x 682 [196,588,154,257,807,776]
+  CRUSH rule 1 x 683 [627,25,421,160,873,102]
+  CRUSH rule 1 x 684 [38,804,592,158,991,264]
+  CRUSH rule 1 x 685 [841,368,548,362,166,211]
+  CRUSH rule 1 x 686 [336,287,525,440,166,993]
+  CRUSH rule 1 x 687 [20,682,924,653,356,16]
+  CRUSH rule 1 x 688 [463,371,780,556,385,883]
+  CRUSH rule 1 x 689 [569,250,78,816,847,775]
+  CRUSH rule 1 x 690 [551,144,587,263,378,394]
+  CRUSH rule 1 x 691 [766,464,446,533,449,541]
+  CRUSH rule 1 x 692 [739,634,18,245,624,35]
+  CRUSH rule 1 x 693 [339,297,118,330,817,91]
+  CRUSH rule 1 x 694 [405,26,830,181,533,166]
+  CRUSH rule 1 x 695 [622,576,597,535,600,593]
+  CRUSH rule 1 x 696 [558,902,689,13,715,28]
+  CRUSH rule 1 x 697 [818,222,406,691,427,863]
+  CRUSH rule 1 x 698 [178,48,402,233,841,604]
+  CRUSH rule 1 x 699 [450,244,180,919,401,332]
+  CRUSH rule 1 x 700 [502,771,987,706,416,240]
+  CRUSH rule 1 x 701 [4,612,782,216,853,303]
+  CRUSH rule 1 x 702 [177,630,232,923,281,708]
+  CRUSH rule 1 x 703 [354,178,389,393,778,803]
+  CRUSH rule 1 x 704 [646,601,156,171,603,116]
+  CRUSH rule 1 x 705 [921,401,890,265,244,690]
+  CRUSH rule 1 x 706 [652,877,562,452,26,323]
+  CRUSH rule 1 x 707 [345,745,67,716,789,576]
+  CRUSH rule 1 x 708 [333,607,180,469,170,555]
+  CRUSH rule 1 x 709 [45,187,302,115,896,579]
+  CRUSH rule 1 x 710 [94,855,43,199,18,948]
+  CRUSH rule 1 x 711 [227,653,731,150,502,842]
+  CRUSH rule 1 x 712 [398,953,136,870,181,408]
+  CRUSH rule 1 x 713 [116,800,503,662,635,579]
+  CRUSH rule 1 x 714 [111,629,866,709,902,557]
+  CRUSH rule 1 x 715 [531,291,486,382,192,807]
+  CRUSH rule 1 x 716 [169,541,291,42,343,724]
+  CRUSH rule 1 x 717 [417,446,994,894,239,494]
+  CRUSH rule 1 x 718 [992,383,298,844,377,463]
+  CRUSH rule 1 x 719 [936,674,324,759,194,409]
+  CRUSH rule 1 x 720 [370,188,174,464,644,218]
+  CRUSH rule 1 x 721 [320,859,278,259,170,957]
+  CRUSH rule 1 x 722 [7,2,673,129,96,445]
+  CRUSH rule 1 x 723 [270,553,831,662,38,101]
+  CRUSH rule 1 x 724 [666,822,708,895,633,800]
+  CRUSH rule 1 x 725 [794,406,875,459,981,751]
+  CRUSH rule 1 x 726 [420,556,341,292,240,68]
+  CRUSH rule 1 x 727 [561,461,129,635,965,610]
+  CRUSH rule 1 x 728 [951,330,196,756,589,849]
+  CRUSH rule 1 x 729 [656,644,436,591,27,119]
+  CRUSH rule 1 x 730 [3,558,629,184,50,765]
+  CRUSH rule 1 x 731 [852,89,75,735,713,113]
+  CRUSH rule 1 x 732 [983,840,869,976,697,307]
+  CRUSH rule 1 x 733 [285,396,388,122,387,364]
+  CRUSH rule 1 x 734 [125,510,402,640,676,501]
+  CRUSH rule 1 x 735 [417,773,686,504,459,912]
+  CRUSH rule 1 x 736 [749,396,632,550,779,109]
+  CRUSH rule 1 x 737 [644,991,946,135,448,903]
+  CRUSH rule 1 x 738 [449,683,290,220,245,525]
+  CRUSH rule 1 x 739 [341,220,641,454,740,661]
+  CRUSH rule 1 x 740 [874,524,674,650,472,282]
+  CRUSH rule 1 x 741 [189,472,712,798,715,757]
+  CRUSH rule 1 x 742 [912,581,114,695,730,21]
+  CRUSH rule 1 x 743 [654,914,425,441,763,39]
+  CRUSH rule 1 x 744 [725,295,579,377,162,447]
+  CRUSH rule 1 x 745 [787,858,850,506,612,735]
+  CRUSH rule 1 x 746 [757,848,704,30,47,940]
+  CRUSH rule 1 x 747 [700,81,867,681,801,64]
+  CRUSH rule 1 x 748 [557,436,238,664,293,865]
+  CRUSH rule 1 x 749 [772,622,337,42,156,302]
+  CRUSH rule 1 x 750 [946,97,376,677,316,670]
+  CRUSH rule 1 x 751 [996,618,343,911,83,22]
+  CRUSH rule 1 x 752 [746,887,695,868,610,950]
+  CRUSH rule 1 x 753 [741,14,463,479,172,192]
+  CRUSH rule 1 x 754 [648,349,333,355,65,63]
+  CRUSH rule 1 x 755 [157,460,466,187,959,674]
+  CRUSH rule 1 x 756 [416,97,197,497,227,3]
+  CRUSH rule 1 x 757 [599,839,776,410,256,823]
+  CRUSH rule 1 x 758 [994,218,620,256,361,749]
+  CRUSH rule 1 x 759 [959,682,514,745,100,519]
+  CRUSH rule 1 x 760 [518,943,215,83,706,137]
+  CRUSH rule 1 x 761 [285,849,420,324,987,338]
+  CRUSH rule 1 x 762 [591,313,41,335,110,696]
+  CRUSH rule 1 x 763 [908,411,200,740,292,295]
+  CRUSH rule 1 x 764 [787,234,894,485,883,711]
+  CRUSH rule 1 x 765 [327,921,882,393,444,792]
+  CRUSH rule 1 x 766 [84,161,878,704,416,144]
+  CRUSH rule 1 x 767 [370,895,702,701,890,2]
+  CRUSH rule 1 x 768 [826,760,879,864,460,474]
+  CRUSH rule 1 x 769 [67,768,663,735,814,66]
+  CRUSH rule 1 x 770 [593,909,482,259,5,550]
+  CRUSH rule 1 x 771 [309,935,121,578,937,685]
+  CRUSH rule 1 x 772 [12,125,797,301,348,419]
+  CRUSH rule 1 x 773 [253,466,820,549,591,193]
+  CRUSH rule 1 x 774 [164,390,705,109,881,505]
+  CRUSH rule 1 x 775 [703,47,43,973,643,406]
+  CRUSH rule 1 x 776 [728,231,80,916,2,850]
+  CRUSH rule 1 x 777 [981,621,568,729,869,952]
+  CRUSH rule 1 x 778 [411,456,544,597,789,784]
+  CRUSH rule 1 x 779 [346,121,519,921,587,48]
+  CRUSH rule 1 x 780 [476,39,288,381,303,29]
+  CRUSH rule 1 x 781 [10,130,585,844,729,705]
+  CRUSH rule 1 x 782 [462,246,581,902,623,877]
+  CRUSH rule 1 x 783 [580,373,153,775,668,661]
+  CRUSH rule 1 x 784 [413,113,978,990,994,56]
+  CRUSH rule 1 x 785 [341,856,332,354,59,581]
+  CRUSH rule 1 x 786 [411,140,313,393,215,618]
+  CRUSH rule 1 x 787 [605,522,211,813,636,224]
+  CRUSH rule 1 x 788 [226,545,35,142,726,851]
+  CRUSH rule 1 x 789 [545,320,414,702,731,277]
+  CRUSH rule 1 x 790 [414,748,816,327,130,115]
+  CRUSH rule 1 x 791 [660,906,406,697,916,322]
+  CRUSH rule 1 x 792 [287,392,514,204,75,789]
+  CRUSH rule 1 x 793 [631,133,850,713,720,487]
+  CRUSH rule 1 x 794 [931,517,543,210,963,898]
+  CRUSH rule 1 x 795 [551,962,477,948,425,434]
+  CRUSH rule 1 x 796 [814,4,95,27,368,300]
+  CRUSH rule 1 x 797 [64,201,299,734,605,864]
+  CRUSH rule 1 x 798 [422,530,114,431,565,716]
+  CRUSH rule 1 x 799 [824,32,679,562,266,549]
+  CRUSH rule 1 x 800 [862,623,489,637,861,196]
+  CRUSH rule 1 x 801 [145,550,329,324,734,160]
+  CRUSH rule 1 x 802 [570,19,847,308,387,518]
+  CRUSH rule 1 x 803 [151,812,662,358,880,349]
+  CRUSH rule 1 x 804 [467,93,264,863,176,842]
+  CRUSH rule 1 x 805 [621,223,938,809,591,686]
+  CRUSH rule 1 x 806 [898,957,805,430,499,584]
+  CRUSH rule 1 x 807 [354,531,422,159,921,431]
+  CRUSH rule 1 x 808 [7,96,76,897,446,2]
+  CRUSH rule 1 x 809 [70,734,719,56,687,21]
+  CRUSH rule 1 x 810 [701,18,972,327,771,649]
+  CRUSH rule 1 x 811 [248,547,103,728,901,264]
+  CRUSH rule 1 x 812 [230,576,821,566,993,762]
+  CRUSH rule 1 x 813 [805,114,683,629,605,462]
+  CRUSH rule 1 x 814 [54,619,973,741,497,894]
+  CRUSH rule 1 x 815 [679,412,613,132,969,411]
+  CRUSH rule 1 x 816 [919,448,826,414,36,289]
+  CRUSH rule 1 x 817 [765,830,436,521,332,458]
+  CRUSH rule 1 x 818 [415,566,644,687,692,414]
+  CRUSH rule 1 x 819 [721,319,865,750,546,859]
+  CRUSH rule 1 x 820 [218,301,333,190,686,179]
+  CRUSH rule 1 x 821 [185,795,680,953,329,750]
+  CRUSH rule 1 x 822 [356,261,54,522,900,103]
+  CRUSH rule 1 x 823 [220,281,549,456,64,306]
+  CRUSH rule 1 x 824 [292,809,887,74,776,788]
+  CRUSH rule 1 x 825 [949,778,101,311,110,480]
+  CRUSH rule 1 x 826 [767,818,833,927,356,954]
+  CRUSH rule 1 x 827 [631,83,406,635,657,713]
+  CRUSH rule 1 x 828 [288,986,445,26,414,607]
+  CRUSH rule 1 x 829 [990,667,915,694,974,453]
+  CRUSH rule 1 x 830 [152,571,778,505,685,209]
+  CRUSH rule 1 x 831 [814,563,630,97,582,107]
+  CRUSH rule 1 x 832 [235,641,616,110,979,844]
+  CRUSH rule 1 x 833 [657,565,922,140,825,457]
+  CRUSH rule 1 x 834 [907,231,644,13,617,130]
+  CRUSH rule 1 x 835 [784,262,771,264,612,238]
+  CRUSH rule 1 x 836 [951,158,366,710,43,427]
+  CRUSH rule 1 x 837 [556,498,334,633,895,627]
+  CRUSH rule 1 x 838 [329,274,964,547,119,342]
+  CRUSH rule 1 x 839 [568,209,939,364,658,747]
+  CRUSH rule 1 x 840 [45,579,842,70,655,862]
+  CRUSH rule 1 x 841 [652,702,24,605,152,93]
+  CRUSH rule 1 x 842 [629,984,314,895,408,897]
+  CRUSH rule 1 x 843 [799,690,688,648,151,812]
+  CRUSH rule 1 x 844 [694,600,534,700,569,11]
+  CRUSH rule 1 x 845 [332,30,179,93,951,324]
+  CRUSH rule 1 x 846 [452,251,712,719,404,739]
+  CRUSH rule 1 x 847 [399,681,847,739,13,555]
+  CRUSH rule 1 x 848 [303,138,440,346,547,216]
+  CRUSH rule 1 x 849 [666,346,708,873,64,694]
+  CRUSH rule 1 x 850 [644,511,345,844,545,337]
+  CRUSH rule 1 x 851 [527,546,737,425,100,331]
+  CRUSH rule 1 x 852 [31,809,94,618,156,853]
+  CRUSH rule 1 x 853 [483,330,869,184,46,942]
+  CRUSH rule 1 x 854 [697,953,968,143,502,955]
+  CRUSH rule 1 x 855 [837,996,239,621,32,191]
+  CRUSH rule 1 x 856 [712,40,547,430,195,857]
+  CRUSH rule 1 x 857 [77,984,576,551,568,96]
+  CRUSH rule 1 x 858 [412,384,841,465,572,576]
+  CRUSH rule 1 x 859 [173,760,26,300,87,567]
+  CRUSH rule 1 x 860 [776,429,328,917,658,783]
+  CRUSH rule 1 x 861 [705,405,477,50,73,714]
+  CRUSH rule 1 x 862 [809,44,788,938,964,177]
+  CRUSH rule 1 x 863 [349,496,963,178,675,853]
+  CRUSH rule 1 x 864 [717,858,101,239,992,244]
+  CRUSH rule 1 x 865 [857,603,586,262,550,289]
+  CRUSH rule 1 x 866 [394,304,71,96,642,155]
+  CRUSH rule 1 x 867 [640,773,663,974,261,296]
+  CRUSH rule 1 x 868 [613,950,712,663,649,460]
+  CRUSH rule 1 x 869 [973,889,524,22,671,477]
+  CRUSH rule 1 x 870 [505,35,386,498,348,503]
+  CRUSH rule 1 x 871 [239,264,262,773,781,734]
+  CRUSH rule 1 x 872 [21,767,456,748,783,797]
+  CRUSH rule 1 x 873 [954,666,980,264,435,233]
+  CRUSH rule 1 x 874 [54,510,947,1,500,119]
+  CRUSH rule 1 x 875 [809,418,452,462,88,673]
+  CRUSH rule 1 x 876 [483,457,61,248,523,277]
+  CRUSH rule 1 x 877 [542,531,952,939,710,179]
+  CRUSH rule 1 x 878 [217,674,857,644,678,809]
+  CRUSH rule 1 x 879 [999,475,134,250,319,357]
+  CRUSH rule 1 x 880 [678,573,935,385,570,651]
+  CRUSH rule 1 x 881 [394,835,789,802,587,155]
+  CRUSH rule 1 x 882 [467,382,353,56,979,674]
+  CRUSH rule 1 x 883 [802,744,237,337,50,96]
+  CRUSH rule 1 x 884 [653,660,638,700,31,558]
+  CRUSH rule 1 x 885 [898,704,307,445,879,872]
+  CRUSH rule 1 x 886 [434,357,938,641,737,8]
+  CRUSH rule 1 x 887 [297,226,711,428,370,318]
+  CRUSH rule 1 x 888 [863,324,443,213,902,25]
+  CRUSH rule 1 x 889 [105,102,308,163,947,548]
+  CRUSH rule 1 x 890 [550,248,606,704,615,708]
+  CRUSH rule 1 x 891 [575,928,880,891,826,763]
+  CRUSH rule 1 x 892 [259,862,133,271,292,162]
+  CRUSH rule 1 x 893 [902,880,543,542,37,942]
+  CRUSH rule 1 x 894 [180,169,916,43,945,713]
+  CRUSH rule 1 x 895 [725,849,182,129,177,272]
+  CRUSH rule 1 x 896 [951,34,874,537,969,123]
+  CRUSH rule 1 x 897 [810,352,73,939,943,895]
+  CRUSH rule 1 x 898 [979,433,719,411,787,359]
+  CRUSH rule 1 x 899 [685,668,534,932,399,156]
+  CRUSH rule 1 x 900 [530,978,41,894,941,681]
+  CRUSH rule 1 x 901 [740,107,336,175,574,706]
+  CRUSH rule 1 x 902 [800,743,693,310,67,111]
+  CRUSH rule 1 x 903 [230,267,842,266,550,769]
+  CRUSH rule 1 x 904 [346,949,460,973,696,91]
+  CRUSH rule 1 x 905 [530,397,619,958,576,973]
+  CRUSH rule 1 x 906 [80,426,138,672,73,776]
+  CRUSH rule 1 x 907 [365,968,475,297,296,724]
+  CRUSH rule 1 x 908 [204,832,742,809,862,745]
+  CRUSH rule 1 x 909 [883,989,146,959,366,59]
+  CRUSH rule 1 x 910 [549,593,249,853,792,769]
+  CRUSH rule 1 x 911 [325,847,352,214,851,732]
+  CRUSH rule 1 x 912 [874,888,582,796,557,601]
+  CRUSH rule 1 x 913 [331,463,342,574,989,362]
+  CRUSH rule 1 x 914 [836,468,601,732,607,275]
+  CRUSH rule 1 x 915 [245,228,100,661,799,13]
+  CRUSH rule 1 x 916 [77,967,364,435,27,474]
+  CRUSH rule 1 x 917 [239,60,866,221,772,967]
+  CRUSH rule 1 x 918 [988,115,922,80,201,544]
+  CRUSH rule 1 x 919 [783,139,696,1,848,169]
+  CRUSH rule 1 x 920 [623,408,685,953,974,696]
+  CRUSH rule 1 x 921 [105,799,144,90,399,373]
+  CRUSH rule 1 x 922 [887,505,652,348,514,806]
+  CRUSH rule 1 x 923 [223,318,552,458,743,871]
+  CRUSH rule 1 x 924 [25,778,366,333,163,801]
+  CRUSH rule 1 x 925 [912,601,297,682,770,173]
+  CRUSH rule 1 x 926 [968,133,96,144,814,155]
+  CRUSH rule 1 x 927 [277,724,214,988,690,342]
+  CRUSH rule 1 x 928 [554,203,658,789,298,299]
+  CRUSH rule 1 x 929 [761,802,367,528,758,522]
+  CRUSH rule 1 x 930 [814,61,788,736,660,491]
+  CRUSH rule 1 x 931 [29,193,61,41,343,664]
+  CRUSH rule 1 x 932 [446,198,862,534,168,35]
+  CRUSH rule 1 x 933 [352,742,216,321,525,44]
+  CRUSH rule 1 x 934 [730,2,332,631,613,249]
+  CRUSH rule 1 x 935 [731,23,736,79,361,992]
+  CRUSH rule 1 x 936 [322,975,20,904,827,603]
+  CRUSH rule 1 x 937 [822,221,841,161,723,137]
+  CRUSH rule 1 x 938 [557,850,66,630,499,404]
+  CRUSH rule 1 x 939 [150,11,971,371,124,785]
+  CRUSH rule 1 x 940 [638,398,169,616,333,751]
+  CRUSH rule 1 x 941 [730,342,929,577,451,838]
+  CRUSH rule 1 x 942 [62,292,166,814,587,172]
+  CRUSH rule 1 x 943 [165,314,519,548,41,726]
+  CRUSH rule 1 x 944 [199,625,766,176,194,297]
+  CRUSH rule 1 x 945 [946,999,699,303,38,81]
+  CRUSH rule 1 x 946 [595,93,852,142,503,647]
+  CRUSH rule 1 x 947 [800,582,356,93,716,117]
+  CRUSH rule 1 x 948 [132,551,139,920,87,46]
+  CRUSH rule 1 x 949 [792,920,466,380,97,568]
+  CRUSH rule 1 x 950 [111,345,176,543,879,954]
+  CRUSH rule 1 x 951 [414,619,648,655,364,971]
+  CRUSH rule 1 x 952 [775,469,500,356,287,4]
+  CRUSH rule 1 x 953 [349,1,5,251,168,680]
+  CRUSH rule 1 x 954 [570,940,410,249,929,394]
+  CRUSH rule 1 x 955 [729,774,823,800,7,127]
+  CRUSH rule 1 x 956 [519,141,575,625,738,475]
+  CRUSH rule 1 x 957 [242,709,611,97,760,309]
+  CRUSH rule 1 x 958 [84,217,227,253,246,604]
+  CRUSH rule 1 x 959 [270,413,918,789,703,608]
+  CRUSH rule 1 x 960 [458,192,307,279,920,139]
+  CRUSH rule 1 x 961 [981,388,777,546,359,660]
+  CRUSH rule 1 x 962 [623,834,277,134,729,246]
+  CRUSH rule 1 x 963 [291,167,714,468,109,373]
+  CRUSH rule 1 x 964 [28,156,788,127,598,215]
+  CRUSH rule 1 x 965 [675,557,290,517,840,510]
+  CRUSH rule 1 x 966 [836,306,946,283,642,606]
+  CRUSH rule 1 x 967 [966,386,735,837,392,116]
+  CRUSH rule 1 x 968 [864,756,690,121,328,122]
+  CRUSH rule 1 x 969 [729,625,480,769,512,882]
+  CRUSH rule 1 x 970 [800,362,646,582,309,102]
+  CRUSH rule 1 x 971 [737,381,153,684,298,166]
+  CRUSH rule 1 x 972 [952,245,720,884,334,311]
+  CRUSH rule 1 x 973 [356,455,579,857,832,596]
+  CRUSH rule 1 x 974 [545,758,586,596,869,790]
+  CRUSH rule 1 x 975 [336,191,202,146,720,897]
+  CRUSH rule 1 x 976 [446,208,757,620,252,846]
+  CRUSH rule 1 x 977 [202,896,196,956,763,126]
+  CRUSH rule 1 x 978 [612,324,996,225,418,583]
+  CRUSH rule 1 x 979 [843,457,675,650,958,657]
+  CRUSH rule 1 x 980 [60,914,881,626,850,759]
+  CRUSH rule 1 x 981 [702,749,937,153,724,514]
+  CRUSH rule 1 x 982 [298,928,738,167,99,668]
+  CRUSH rule 1 x 983 [723,572,395,358,900,37]
+  CRUSH rule 1 x 984 [723,864,804,935,846,993]
+  CRUSH rule 1 x 985 [945,459,868,211,524,954]
+  CRUSH rule 1 x 986 [772,664,535,169,297,996]
+  CRUSH rule 1 x 987 [88,324,312,843,661,580]
+  CRUSH rule 1 x 988 [522,927,131,996,351,685]
+  CRUSH rule 1 x 989 [578,332,208,605,975,207]
+  CRUSH rule 1 x 990 [638,228,414,311,738,698]
+  CRUSH rule 1 x 991 [530,221,451,422,879,916]
+  CRUSH rule 1 x 992 [925,705,275,81,234,310]
+  CRUSH rule 1 x 993 [991,301,43,469,830,242]
+  CRUSH rule 1 x 994 [276,51,868,683,843,815]
+  CRUSH rule 1 x 995 [288,836,753,790,758,120]
+  CRUSH rule 1 x 996 [887,983,252,686,470,345]
+  CRUSH rule 1 x 997 [110,924,386,79,705,697]
+  CRUSH rule 1 x 998 [435,830,485,853,926,730]
+  CRUSH rule 1 x 999 [876,738,357,913,723,51]
+  CRUSH rule 1 x 1000 [178,963,638,430,845,586]
+  CRUSH rule 1 x 1001 [99,519,66,759,583,944]
+  CRUSH rule 1 x 1002 [515,534,468,866,878,717]
+  CRUSH rule 1 x 1003 [104,611,937,698,94,67]
+  CRUSH rule 1 x 1004 [269,638,724,375,491,121]
+  CRUSH rule 1 x 1005 [369,223,309,409,822,39]
+  CRUSH rule 1 x 1006 [40,107,69,275,79,429]
+  CRUSH rule 1 x 1007 [978,111,416,758,454,640]
+  CRUSH rule 1 x 1008 [965,956,624,832,421,96]
+  CRUSH rule 1 x 1009 [598,476,356,695,919,566]
+  CRUSH rule 1 x 1010 [767,523,239,517,29,77]
+  CRUSH rule 1 x 1011 [289,871,207,576,347,698]
+  CRUSH rule 1 x 1012 [128,28,370,31,341,755]
+  CRUSH rule 1 x 1013 [979,765,660,812,666,187]
+  CRUSH rule 1 x 1014 [979,948,513,88,47,825]
+  CRUSH rule 1 x 1015 [277,790,396,672,542,647]
+  CRUSH rule 1 x 1016 [262,73,128,886,839,685]
+  CRUSH rule 1 x 1017 [150,269,61,499,832,591]
+  CRUSH rule 1 x 1018 [555,829,554,944,406,576]
+  CRUSH rule 1 x 1019 [513,356,265,446,65,288]
+  CRUSH rule 1 x 1020 [158,161,877,704,948,570]
+  CRUSH rule 1 x 1021 [915,998,957,285,546,202]
+  CRUSH rule 1 x 1022 [967,829,973,640,703,470]
+  CRUSH rule 1 x 1023 [488,257,614,859,325,419]
+  rule 1 (metadata) num_rep 6 result size == 6:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536,450,604,380,966]
+  CRUSH rule 1 x 1 [876,250,334,633,744,843,672]
+  CRUSH rule 1 x 2 [292,832,53,392,386,787,527]
+  CRUSH rule 1 x 3 [623,387,124,998,749,211,481]
+  CRUSH rule 1 x 4 [61,334,710,4,994,982,847]
+  CRUSH rule 1 x 5 [946,557,713,664,141,817,964]
+  CRUSH rule 1 x 6 [576,668,212,163,732,381,884]
+  CRUSH rule 1 x 7 [645,753,906,393,341,44,578]
+  CRUSH rule 1 x 8 [243,6,863,781,211,100,462]
+  CRUSH rule 1 x 9 [22,578,251,410,297,430,3]
+  CRUSH rule 1 x 10 [758,828,360,477,821,801,811]
+  CRUSH rule 1 x 11 [769,120,124,527,119,504,380]
+  CRUSH rule 1 x 12 [780,364,689,755,675,199,117]
+  CRUSH rule 1 x 13 [557,18,351,719,742,780,78]
+  CRUSH rule 1 x 14 [59,561,249,461,971,835,855]
+  CRUSH rule 1 x 15 [718,928,993,21,76,313,437]
+  CRUSH rule 1 x 16 [673,632,841,954,788,90,786]
+  CRUSH rule 1 x 17 [648,43,560,514,142,289,935]
+  CRUSH rule 1 x 18 [654,219,181,568,381,253,883]
+  CRUSH rule 1 x 19 [850,545,377,848,863,543,51]
+  CRUSH rule 1 x 20 [717,785,974,5,225,552,975]
+  CRUSH rule 1 x 21 [420,57,519,306,312,983,263]
+  CRUSH rule 1 x 22 [503,998,193,821,634,684,557]
+  CRUSH rule 1 x 23 [411,663,168,110,899,488,477]
+  CRUSH rule 1 x 24 [266,861,353,1,456,128,800]
+  CRUSH rule 1 x 25 [760,483,818,600,509,951,248]
+  CRUSH rule 1 x 26 [903,24,573,718,112,694,501]
+  CRUSH rule 1 x 27 [946,188,289,510,687,827,676]
+  CRUSH rule 1 x 28 [69,312,73,198,256,629,770]
+  CRUSH rule 1 x 29 [844,883,337,628,496,405,719]
+  CRUSH rule 1 x 30 [621,18,613,794,910,936,426]
+  CRUSH rule 1 x 31 [784,943,814,539,962,392,813]
+  CRUSH rule 1 x 32 [173,374,369,972,315,83,428]
+  CRUSH rule 1 x 33 [698,336,357,966,582,407,618]
+  CRUSH rule 1 x 34 [168,836,210,798,904,190,663]
+  CRUSH rule 1 x 35 [274,509,534,818,912,671,75]
+  CRUSH rule 1 x 36 [318,215,153,628,87,407,676]
+  CRUSH rule 1 x 37 [173,604,109,935,203,401,311]
+  CRUSH rule 1 x 38 [708,444,683,604,722,900,929]
+  CRUSH rule 1 x 39 [662,198,417,680,226,342,856]
+  CRUSH rule 1 x 40 [620,801,414,78,560,766,980]
+  CRUSH rule 1 x 41 [811,264,177,127,148,791,930]
+  CRUSH rule 1 x 42 [863,179,527,660,133,529,456]
+  CRUSH rule 1 x 43 [686,822,988,228,791,549,514]
+  CRUSH rule 1 x 44 [396,222,46,841,536,140,160]
+  CRUSH rule 1 x 45 [991,694,253,142,54,422,658]
+  CRUSH rule 1 x 46 [420,909,184,285,508,458,45]
+  CRUSH rule 1 x 47 [467,211,605,207,241,881,959]
+  CRUSH rule 1 x 48 [955,329,368,168,698,787,738]
+  CRUSH rule 1 x 49 [974,891,931,29,813,506,822]
+  CRUSH rule 1 x 50 [870,441,691,823,761,6,83]
+  CRUSH rule 1 x 51 [182,930,25,936,97,260,406]
+  CRUSH rule 1 x 52 [704,812,894,794,481,37,304]
+  CRUSH rule 1 x 53 [185,713,631,280,345,558,882]
+  CRUSH rule 1 x 54 [270,441,100,82,983,930,339]
+  CRUSH rule 1 x 55 [895,734,958,793,651,572,508]
+  CRUSH rule 1 x 56 [564,963,683,324,40,189,77]
+  CRUSH rule 1 x 57 [738,130,208,973,498,861,670]
+  CRUSH rule 1 x 58 [524,113,806,903,531,334,8]
+  CRUSH rule 1 x 59 [408,337,668,529,34,384,643]
+  CRUSH rule 1 x 60 [228,790,857,309,616,895,194]
+  CRUSH rule 1 x 61 [154,843,717,467,883,536,812]
+  CRUSH rule 1 x 62 [594,811,549,276,693,917,45]
+  CRUSH rule 1 x 63 [646,67,884,925,941,434,705]
+  CRUSH rule 1 x 64 [175,542,155,837,594,197,451]
+  CRUSH rule 1 x 65 [745,619,131,867,269,62,862]
+  CRUSH rule 1 x 66 [275,468,23,35,328,432,334]
+  CRUSH rule 1 x 67 [246,958,524,493,636,227,783]
+  CRUSH rule 1 x 68 [711,473,403,228,835,126,705]
+  CRUSH rule 1 x 69 [493,924,850,939,950,105,871]
+  CRUSH rule 1 x 70 [30,499,644,33,804,654,684]
+  CRUSH rule 1 x 71 [984,883,574,716,575,391,587]
+  CRUSH rule 1 x 72 [71,286,942,363,628,632,642]
+  CRUSH rule 1 x 73 [922,618,3,371,464,442,835]
+  CRUSH rule 1 x 74 [629,414,185,573,678,338,633]
+  CRUSH rule 1 x 75 [222,20,174,820,312,361,366]
+  CRUSH rule 1 x 76 [262,366,339,290,718,143,735]
+  CRUSH rule 1 x 77 [638,469,992,280,773,892,197]
+  CRUSH rule 1 x 78 [324,511,788,7,308,228,183]
+  CRUSH rule 1 x 79 [577,990,64,94,447,924,339]
+  CRUSH rule 1 x 80 [501,95,278,903,631,842,51]
+  CRUSH rule 1 x 81 [506,812,9,698,173,664,247]
+  CRUSH rule 1 x 82 [222,145,80,785,835,745,580]
+  CRUSH rule 1 x 83 [71,634,61,91,856,529,66]
+  CRUSH rule 1 x 84 [49,761,773,368,318,708,681]
+  CRUSH rule 1 x 85 [985,896,708,861,325,307,567]
+  CRUSH rule 1 x 86 [537,745,93,524,466,356,38]
+  CRUSH rule 1 x 87 [997,317,463,626,685,594,909]
+  CRUSH rule 1 x 88 [957,350,890,857,375,176,99]
+  CRUSH rule 1 x 89 [399,730,148,314,159,982,320]
+  CRUSH rule 1 x 90 [943,706,683,267,579,141,412]
+  CRUSH rule 1 x 91 [22,368,149,928,140,529,495]
+  CRUSH rule 1 x 92 [532,424,426,773,623,197,167]
+  CRUSH rule 1 x 93 [218,489,405,681,549,201,343]
+  CRUSH rule 1 x 94 [181,96,102,515,776,365,82]
+  CRUSH rule 1 x 95 [343,957,820,139,334,37,648]
+  CRUSH rule 1 x 96 [861,270,87,797,0,245,204]
+  CRUSH rule 1 x 97 [459,706,45,328,274,605,83]
+  CRUSH rule 1 x 98 [327,867,353,948,728,280,270]
+  CRUSH rule 1 x 99 [974,133,468,906,235,988,37]
+  CRUSH rule 1 x 100 [32,445,547,371,960,885,9]
+  CRUSH rule 1 x 101 [142,90,337,950,970,570,12]
+  CRUSH rule 1 x 102 [172,129,139,22,403,867,923]
+  CRUSH rule 1 x 103 [630,47,161,356,911,421,933]
+  CRUSH rule 1 x 104 [758,133,278,11,947,799,401]
+  CRUSH rule 1 x 105 [843,604,47,33,401,632,434]
+  CRUSH rule 1 x 106 [28,681,193,679,990,343,878]
+  CRUSH rule 1 x 107 [74,320,85,819,315,253,589]
+  CRUSH rule 1 x 108 [875,593,575,517,107,153,631]
+  CRUSH rule 1 x 109 [411,985,811,720,198,666,856]
+  CRUSH rule 1 x 110 [440,774,799,660,715,167,510]
+  CRUSH rule 1 x 111 [405,742,276,359,936,360,18]
+  CRUSH rule 1 x 112 [143,181,922,545,185,303,725]
+  CRUSH rule 1 x 113 [153,846,160,903,789,897,738]
+  CRUSH rule 1 x 114 [804,892,939,20,312,692,598]
+  CRUSH rule 1 x 115 [588,508,958,580,232,722,421]
+  CRUSH rule 1 x 116 [327,148,637,486,712,464,9]
+  CRUSH rule 1 x 117 [95,594,989,131,714,275,725]
+  CRUSH rule 1 x 118 [80,957,897,239,359,432,766]
+  CRUSH rule 1 x 119 [386,932,951,768,679,300,570]
+  CRUSH rule 1 x 120 [366,312,653,936,71,241,49]
+  CRUSH rule 1 x 121 [129,154,847,16,471,481,424]
+  CRUSH rule 1 x 122 [873,1,110,939,90,412,551]
+  CRUSH rule 1 x 123 [533,415,789,600,713,800,877]
+  CRUSH rule 1 x 124 [461,691,898,723,957,759,482]
+  CRUSH rule 1 x 125 [342,599,830,402,615,994,736]
+  CRUSH rule 1 x 126 [819,781,822,548,279,255,689]
+  CRUSH rule 1 x 127 [437,893,585,707,353,189,909]
+  CRUSH rule 1 x 128 [679,994,982,550,991,324,666]
+  CRUSH rule 1 x 129 [380,685,947,302,698,144,149]
+  CRUSH rule 1 x 130 [992,52,466,867,998,777,270]
+  CRUSH rule 1 x 131 [469,90,208,599,829,656,203]
+  CRUSH rule 1 x 132 [571,250,316,535,54,418,922]
+  CRUSH rule 1 x 133 [964,728,329,902,108,118,14]
+  CRUSH rule 1 x 134 [999,19,716,963,323,559,893]
+  CRUSH rule 1 x 135 [634,101,52,938,413,573,712]
+  CRUSH rule 1 x 136 [114,889,692,768,694,279,846]
+  CRUSH rule 1 x 137 [839,8,959,280,922,870,363]
+  CRUSH rule 1 x 138 [967,949,138,451,292,548,400]
+  CRUSH rule 1 x 139 [308,711,736,247,632,126,384]
+  CRUSH rule 1 x 140 [764,936,926,55,331,115,178]
+  CRUSH rule 1 x 141 [423,302,112,216,603,873,193]
+  CRUSH rule 1 x 142 [252,821,715,340,635,668,424]
+  CRUSH rule 1 x 143 [33,808,518,477,325,316,266]
+  CRUSH rule 1 x 144 [472,88,969,162,401,771,697]
+  CRUSH rule 1 x 145 [242,208,252,604,266,743,577]
+  CRUSH rule 1 x 146 [290,70,570,384,934,856,929]
+  CRUSH rule 1 x 147 [447,352,657,493,467,918,514]
+  CRUSH rule 1 x 148 [212,644,432,658,109,275,352]
+  CRUSH rule 1 x 149 [9,775,87,35,260,646,406]
+  CRUSH rule 1 x 150 [166,456,582,144,324,340,484]
+  CRUSH rule 1 x 151 [811,875,307,20,782,229,671]
+  CRUSH rule 1 x 152 [449,617,223,9,182,407,807]
+  CRUSH rule 1 x 153 [523,537,695,627,959,613,942]
+  CRUSH rule 1 x 154 [208,559,874,597,243,706,443]
+  CRUSH rule 1 x 155 [569,325,192,296,367,848,58]
+  CRUSH rule 1 x 156 [488,121,521,213,595,837,271]
+  CRUSH rule 1 x 157 [140,723,633,260,487,856,384]
+  CRUSH rule 1 x 158 [786,451,320,239,667,632,899]
+  CRUSH rule 1 x 159 [134,664,517,821,667,944,209]
+  CRUSH rule 1 x 160 [690,112,414,990,183,590,242]
+  CRUSH rule 1 x 161 [324,912,397,423,991,284,909]
+  CRUSH rule 1 x 162 [748,567,284,183,463,336,148]
+  CRUSH rule 1 x 163 [575,499,31,816,749,737,587]
+  CRUSH rule 1 x 164 [314,489,308,326,51,568,110]
+  CRUSH rule 1 x 165 [116,209,750,53,813,640,524]
+  CRUSH rule 1 x 166 [352,706,701,810,718,527,548]
+  CRUSH rule 1 x 167 [27,743,174,142,551,1,935]
+  CRUSH rule 1 x 168 [953,898,880,660,500,799,667]
+  CRUSH rule 1 x 169 [912,147,266,547,331,770,601]
+  CRUSH rule 1 x 170 [421,515,828,844,151,981,835]
+  CRUSH rule 1 x 171 [488,584,880,964,936,196,100]
+  CRUSH rule 1 x 172 [366,443,957,66,162,693,36]
+  CRUSH rule 1 x 173 [863,291,625,287,158,496,471]
+  CRUSH rule 1 x 174 [263,555,650,410,339,616,780]
+  CRUSH rule 1 x 175 [875,961,361,575,33,109,51]
+  CRUSH rule 1 x 176 [745,83,701,680,250,420,240]
+  CRUSH rule 1 x 177 [128,244,41,123,422,902,756]
+  CRUSH rule 1 x 178 [155,41,264,777,314,564,856]
+  CRUSH rule 1 x 179 [593,833,202,183,971,38,724]
+  CRUSH rule 1 x 180 [154,734,17,831,824,522,736]
+  CRUSH rule 1 x 181 [289,675,723,800,166,712,168]
+  CRUSH rule 1 x 182 [730,931,560,209,943,261,485]
+  CRUSH rule 1 x 183 [639,237,794,815,827,400,109]
+  CRUSH rule 1 x 184 [704,312,685,645,691,778,74]
+  CRUSH rule 1 x 185 [97,100,762,82,999,542,485]
+  CRUSH rule 1 x 186 [26,665,554,215,280,421,369]
+  CRUSH rule 1 x 187 [649,14,740,494,402,684,566]
+  CRUSH rule 1 x 188 [682,695,590,743,927,945,833]
+  CRUSH rule 1 x 189 [325,693,726,51,448,169,37]
+  CRUSH rule 1 x 190 [399,933,136,955,57,504,527]
+  CRUSH rule 1 x 191 [629,533,17,126,60,146,999]
+  CRUSH rule 1 x 192 [503,578,38,492,222,251,123]
+  CRUSH rule 1 x 193 [546,333,651,678,823,652,359]
+  CRUSH rule 1 x 194 [242,473,58,655,888,277,792]
+  CRUSH rule 1 x 195 [625,719,135,81,636,513,755]
+  CRUSH rule 1 x 196 [357,114,125,867,250,522,413]
+  CRUSH rule 1 x 197 [306,954,453,873,211,334,666]
+  CRUSH rule 1 x 198 [863,791,311,911,206,61,355]
+  CRUSH rule 1 x 199 [935,906,929,252,893,75,960]
+  CRUSH rule 1 x 200 [373,774,229,454,909,611,132]
+  CRUSH rule 1 x 201 [659,320,477,313,779,16,495]
+  CRUSH rule 1 x 202 [260,433,524,880,223,818,153]
+  CRUSH rule 1 x 203 [36,239,675,971,703,209,669]
+  CRUSH rule 1 x 204 [92,516,993,728,279,478,697]
+  CRUSH rule 1 x 205 [68,395,473,45,683,662,776]
+  CRUSH rule 1 x 206 [570,530,642,380,311,398,230]
+  CRUSH rule 1 x 207 [834,457,850,917,456,296,76]
+  CRUSH rule 1 x 208 [927,484,640,976,803,626,96]
+  CRUSH rule 1 x 209 [878,66,58,940,48,233,522]
+  CRUSH rule 1 x 210 [572,981,484,29,0,426,14]
+  CRUSH rule 1 x 211 [107,597,780,857,895,57,922]
+  CRUSH rule 1 x 212 [389,107,838,624,698,562,857]
+  CRUSH rule 1 x 213 [497,717,567,728,905,134,687]
+  CRUSH rule 1 x 214 [798,65,254,572,32,393,579]
+  CRUSH rule 1 x 215 [233,419,283,638,520,891,982]
+  CRUSH rule 1 x 216 [494,464,742,523,459,174,973]
+  CRUSH rule 1 x 217 [352,396,309,938,66,41,264]
+  CRUSH rule 1 x 218 [895,864,988,650,593,740,34]
+  CRUSH rule 1 x 219 [222,534,277,242,658,482,697]
+  CRUSH rule 1 x 220 [281,19,584,563,858,965,686]
+  CRUSH rule 1 x 221 [64,928,963,130,312,394,61]
+  CRUSH rule 1 x 222 [40,544,161,199,861,644,597]
+  CRUSH rule 1 x 223 [645,556,159,417,46,135,465]
+  CRUSH rule 1 x 224 [647,165,957,263,961,576,329]
+  CRUSH rule 1 x 225 [219,714,858,747,461,175,606]
+  CRUSH rule 1 x 226 [372,511,181,277,695,404,876]
+  CRUSH rule 1 x 227 [925,156,714,863,257,74,966]
+  CRUSH rule 1 x 228 [682,404,839,263,521,195,261]
+  CRUSH rule 1 x 229 [880,838,770,891,236,542,262]
+  CRUSH rule 1 x 230 [328,659,916,468,646,572,93]
+  CRUSH rule 1 x 231 [320,383,669,109,627,621,50]
+  CRUSH rule 1 x 232 [924,846,394,319,43,519,106]
+  CRUSH rule 1 x 233 [948,652,575,838,498,395,796]
+  CRUSH rule 1 x 234 [484,943,42,575,936,180,103]
+  CRUSH rule 1 x 235 [750,65,590,168,870,308,471]
+  CRUSH rule 1 x 236 [551,787,490,136,370,833,573]
+  CRUSH rule 1 x 237 [390,157,166,251,752,75,327]
+  CRUSH rule 1 x 238 [570,6,989,707,514,905,894]
+  CRUSH rule 1 x 239 [729,959,376,975,496,49,426]
+  CRUSH rule 1 x 240 [981,241,156,767,631,576,450]
+  CRUSH rule 1 x 241 [310,816,641,177,996,454,413]
+  CRUSH rule 1 x 242 [161,63,642,837,763,458,234]
+  CRUSH rule 1 x 243 [180,394,33,683,189,419,799]
+  CRUSH rule 1 x 244 [52,174,685,189,78,310,785]
+  CRUSH rule 1 x 245 [523,121,915,84,386,409,605]
+  CRUSH rule 1 x 246 [362,893,390,487,817,88,989]
+  CRUSH rule 1 x 247 [382,184,116,34,143,15,590]
+  CRUSH rule 1 x 248 [129,114,852,469,359,291,713]
+  CRUSH rule 1 x 249 [159,683,91,856,475,369,886]
+  CRUSH rule 1 x 250 [404,945,569,955,228,910,270]
+  CRUSH rule 1 x 251 [661,225,738,757,37,642,58]
+  CRUSH rule 1 x 252 [961,226,542,103,945,885,838]
+  CRUSH rule 1 x 253 [651,97,225,364,189,248,797]
+  CRUSH rule 1 x 254 [123,33,741,692,599,11,605]
+  CRUSH rule 1 x 255 [314,649,891,855,517,344,607]
+  CRUSH rule 1 x 256 [315,215,651,126,470,849,189]
+  CRUSH rule 1 x 257 [825,264,867,681,529,409,291]
+  CRUSH rule 1 x 258 [624,789,370,723,131,982,863]
+  CRUSH rule 1 x 259 [602,542,70,563,947,723,77]
+  CRUSH rule 1 x 260 [717,878,43,56,377,481,533]
+  CRUSH rule 1 x 261 [145,517,20,903,786,939,516]
+  CRUSH rule 1 x 262 [223,1,561,420,209,16,88]
+  CRUSH rule 1 x 263 [462,211,405,508,787,669,773]
+  CRUSH rule 1 x 264 [654,471,266,662,135,564,715]
+  CRUSH rule 1 x 265 [302,794,704,798,659,487,833]
+  CRUSH rule 1 x 266 [202,132,884,209,551,984,7]
+  CRUSH rule 1 x 267 [282,938,657,113,672,993,972]
+  CRUSH rule 1 x 268 [338,309,356,278,928,797,715]
+  CRUSH rule 1 x 269 [738,122,266,200,894,118,146]
+  CRUSH rule 1 x 270 [707,982,946,196,407,804,476]
+  CRUSH rule 1 x 271 [705,432,364,735,512,595,263]
+  CRUSH rule 1 x 272 [756,545,942,56,542,449,710]
+  CRUSH rule 1 x 273 [197,502,527,721,239,648,982]
+  CRUSH rule 1 x 274 [992,44,653,573,527,702,370]
+  CRUSH rule 1 x 275 [544,789,170,434,23,926,992]
+  CRUSH rule 1 x 276 [658,467,577,268,336,5,634]
+  CRUSH rule 1 x 277 [143,490,880,483,928,272,783]
+  CRUSH rule 1 x 278 [492,647,355,282,834,64,350]
+  CRUSH rule 1 x 279 [517,792,604,987,527,894,952]
+  CRUSH rule 1 x 280 [825,740,27,848,514,750,895]
+  CRUSH rule 1 x 281 [224,629,120,562,616,200,443]
+  CRUSH rule 1 x 282 [298,661,380,416,35,585,939]
+  CRUSH rule 1 x 283 [311,606,208,50,913,678,369]
+  CRUSH rule 1 x 284 [771,466,371,743,672,119,60]
+  CRUSH rule 1 x 285 [693,362,404,676,797,531,582]
+  CRUSH rule 1 x 286 [364,477,285,167,270,617,699]
+  CRUSH rule 1 x 287 [591,611,828,995,170,987,137]
+  CRUSH rule 1 x 288 [965,541,848,796,251,668,195]
+  CRUSH rule 1 x 289 [225,551,948,877,219,167,795]
+  CRUSH rule 1 x 290 [577,762,777,751,291,349,473]
+  CRUSH rule 1 x 291 [160,903,477,381,490,559,557]
+  CRUSH rule 1 x 292 [873,598,216,666,222,228,806]
+  CRUSH rule 1 x 293 [100,234,874,47,28,452,775]
+  CRUSH rule 1 x 294 [285,943,379,520,725,547,459]
+  CRUSH rule 1 x 295 [938,262,880,327,687,3,440]
+  CRUSH rule 1 x 296 [850,327,86,472,1,776,266]
+  CRUSH rule 1 x 297 [951,53,99,558,753,228,232]
+  CRUSH rule 1 x 298 [173,336,85,766,910,657,213]
+  CRUSH rule 1 x 299 [598,591,315,386,895,296,924]
+  CRUSH rule 1 x 300 [531,957,62,459,156,538,904]
+  CRUSH rule 1 x 301 [823,628,23,858,629,808,220]
+  CRUSH rule 1 x 302 [184,80,780,871,531,211,400]
+  CRUSH rule 1 x 303 [521,766,222,830,988,275,561]
+  CRUSH rule 1 x 304 [980,127,807,507,555,245,214]
+  CRUSH rule 1 x 305 [153,816,22,927,696,911,685]
+  CRUSH rule 1 x 306 [423,739,664,753,178,431,761]
+  CRUSH rule 1 x 307 [997,557,682,456,479,631,459]
+  CRUSH rule 1 x 308 [991,874,534,465,330,284,976]
+  CRUSH rule 1 x 309 [860,394,724,858,246,866,857]
+  CRUSH rule 1 x 310 [589,818,546,201,94,653,90]
+  CRUSH rule 1 x 311 [477,774,225,590,830,559,256]
+  CRUSH rule 1 x 312 [887,853,950,354,58,23,497]
+  CRUSH rule 1 x 313 [802,646,447,416,557,118,24]
+  CRUSH rule 1 x 314 [654,974,229,511,562,916,952]
+  CRUSH rule 1 x 315 [767,227,28,740,828,156,749]
+  CRUSH rule 1 x 316 [778,83,733,359,858,319,761]
+  CRUSH rule 1 x 317 [184,418,642,986,939,675,892]
+  CRUSH rule 1 x 318 [525,410,500,543,212,95,290]
+  CRUSH rule 1 x 319 [476,724,569,382,409,521,800]
+  CRUSH rule 1 x 320 [149,610,697,296,818,955,523]
+  CRUSH rule 1 x 321 [710,79,667,671,234,4,868]
+  CRUSH rule 1 x 322 [175,275,323,333,744,718,187]
+  CRUSH rule 1 x 323 [819,604,638,792,316,544,236]
+  CRUSH rule 1 x 324 [16,745,511,439,272,294,668]
+  CRUSH rule 1 x 325 [486,400,872,873,251,68,462]
+  CRUSH rule 1 x 326 [613,765,207,19,359,370,461]
+  CRUSH rule 1 x 327 [125,289,738,408,456,784,750]
+  CRUSH rule 1 x 328 [807,383,476,583,645,141,33]
+  CRUSH rule 1 x 329 [588,938,599,432,446,840,516]
+  CRUSH rule 1 x 330 [932,644,41,611,209,406,420]
+  CRUSH rule 1 x 331 [341,953,950,537,578,862,624]
+  CRUSH rule 1 x 332 [153,726,459,950,466,804,644]
+  CRUSH rule 1 x 333 [745,845,853,860,52,615,243]
+  CRUSH rule 1 x 334 [614,751,807,58,396,159,408]
+  CRUSH rule 1 x 335 [518,721,221,283,454,187,635]
+  CRUSH rule 1 x 336 [389,424,77,309,5,898,698]
+  CRUSH rule 1 x 337 [753,508,765,720,221,807,956]
+  CRUSH rule 1 x 338 [128,810,490,753,406,760,69]
+  CRUSH rule 1 x 339 [430,308,58,751,856,823,607]
+  CRUSH rule 1 x 340 [541,44,630,231,289,966,707]
+  CRUSH rule 1 x 341 [402,26,631,439,165,928,720]
+  CRUSH rule 1 x 342 [982,57,992,461,131,32,516]
+  CRUSH rule 1 x 343 [833,412,572,732,107,805,660]
+  CRUSH rule 1 x 344 [784,533,792,41,642,869,142]
+  CRUSH rule 1 x 345 [546,300,304,691,763,556,127]
+  CRUSH rule 1 x 346 [302,420,428,891,357,124,419]
+  CRUSH rule 1 x 347 [488,778,101,217,366,442,783]
+  CRUSH rule 1 x 348 [903,744,937,718,85,314,862]
+  CRUSH rule 1 x 349 [471,547,582,306,600,486,795]
+  CRUSH rule 1 x 350 [348,221,823,335,383,708,841]
+  CRUSH rule 1 x 351 [961,582,705,346,361,32,766]
+  CRUSH rule 1 x 352 [728,137,461,298,36,903,899]
+  CRUSH rule 1 x 353 [904,202,184,447,58,294,279]
+  CRUSH rule 1 x 354 [345,226,319,256,544,311,612]
+  CRUSH rule 1 x 355 [50,430,175,43,187,458,985]
+  CRUSH rule 1 x 356 [87,185,55,423,829,1,629]
+  CRUSH rule 1 x 357 [762,459,921,473,182,231,891]
+  CRUSH rule 1 x 358 [908,25,280,6,808,676,874]
+  CRUSH rule 1 x 359 [484,15,132,121,394,423,397]
+  CRUSH rule 1 x 360 [173,378,337,702,145,499,29]
+  CRUSH rule 1 x 361 [404,577,115,25,56,914,643]
+  CRUSH rule 1 x 362 [403,1,422,945,132,685,265]
+  CRUSH rule 1 x 363 [639,911,510,162,418,294,444]
+  CRUSH rule 1 x 364 [752,689,610,990,665,222,203]
+  CRUSH rule 1 x 365 [956,999,212,230,624,84,113]
+  CRUSH rule 1 x 366 [860,925,924,763,687,851,59]
+  CRUSH rule 1 x 367 [205,609,647,665,969,720,685]
+  CRUSH rule 1 x 368 [301,284,810,169,78,340,616]
+  CRUSH rule 1 x 369 [452,658,339,217,674,210,284]
+  CRUSH rule 1 x 370 [11,467,695,989,394,576,850]
+  CRUSH rule 1 x 371 [124,487,55,514,313,411,797]
+  CRUSH rule 1 x 372 [253,48,979,846,207,631,212]
+  CRUSH rule 1 x 373 [715,605,775,748,227,493,128]
+  CRUSH rule 1 x 374 [191,887,920,723,223,714,961]
+  CRUSH rule 1 x 375 [711,385,651,665,15,71,934]
+  CRUSH rule 1 x 376 [597,818,49,458,415,755,446]
+  CRUSH rule 1 x 377 [294,256,933,771,184,861,654]
+  CRUSH rule 1 x 378 [34,151,681,707,552,127,728]
+  CRUSH rule 1 x 379 [869,136,315,378,813,153,115]
+  CRUSH rule 1 x 380 [294,97,575,791,690,482,255]
+  CRUSH rule 1 x 381 [119,710,219,827,328,886,773]
+  CRUSH rule 1 x 382 [69,631,508,706,697,168,276]
+  CRUSH rule 1 x 383 [922,588,589,925,471,601,29]
+  CRUSH rule 1 x 384 [221,945,671,117,857,655,488]
+  CRUSH rule 1 x 385 [561,737,953,723,658,368,910]
+  CRUSH rule 1 x 386 [335,442,788,696,507,716,232]
+  CRUSH rule 1 x 387 [514,43,353,88,100,842,164]
+  CRUSH rule 1 x 388 [587,89,157,996,915,927,474]
+  CRUSH rule 1 x 389 [109,641,255,466,372,563,340]
+  CRUSH rule 1 x 390 [925,149,421,489,599,810,852]
+  CRUSH rule 1 x 391 [267,87,387,527,768,873,735]
+  CRUSH rule 1 x 392 [382,485,370,849,936,636,901]
+  CRUSH rule 1 x 393 [425,721,221,753,268,463,652]
+  CRUSH rule 1 x 394 [898,18,38,793,173,738,15]
+  CRUSH rule 1 x 395 [806,876,269,679,32,744,126]
+  CRUSH rule 1 x 396 [790,970,437,449,875,395,726]
+  CRUSH rule 1 x 397 [136,363,507,613,11,30,996]
+  CRUSH rule 1 x 398 [914,116,558,258,722,904,349]
+  CRUSH rule 1 x 399 [261,94,299,202,174,622,749]
+  CRUSH rule 1 x 400 [661,197,338,461,977,848,536]
+  CRUSH rule 1 x 401 [953,979,287,803,41,349,79]
+  CRUSH rule 1 x 402 [738,819,618,522,667,334,658]
+  CRUSH rule 1 x 403 [573,238,425,546,130,68,202]
+  CRUSH rule 1 x 404 [526,848,790,253,922,820,299]
+  CRUSH rule 1 x 405 [582,505,330,334,201,110,776]
+  CRUSH rule 1 x 406 [768,324,493,60,186,165,718]
+  CRUSH rule 1 x 407 [260,951,437,587,692,648,72]
+  CRUSH rule 1 x 408 [657,81,770,734,830,821,246]
+  CRUSH rule 1 x 409 [498,89,182,423,672,152,213]
+  CRUSH rule 1 x 410 [28,793,737,352,166,645,949]
+  CRUSH rule 1 x 411 [684,992,60,659,769,267,313]
+  CRUSH rule 1 x 412 [261,958,699,950,165,14,560]
+  CRUSH rule 1 x 413 [891,835,297,441,384,979,618]
+  CRUSH rule 1 x 414 [127,459,119,965,662,594,97]
+  CRUSH rule 1 x 415 [272,540,631,328,609,568,694]
+  CRUSH rule 1 x 416 [739,617,115,530,339,371,889]
+  CRUSH rule 1 x 417 [106,209,157,878,117,128,138]
+  CRUSH rule 1 x 418 [525,441,147,390,320,300,848]
+  CRUSH rule 1 x 419 [603,673,615,465,266,855,823]
+  CRUSH rule 1 x 420 [988,213,251,226,209,245,506]
+  CRUSH rule 1 x 421 [761,521,748,368,923,992,764]
+  CRUSH rule 1 x 422 [317,160,924,548,198,709,839]
+  CRUSH rule 1 x 423 [137,807,168,472,619,443,905]
+  CRUSH rule 1 x 424 [920,37,146,263,598,748,785]
+  CRUSH rule 1 x 425 [277,693,285,221,478,165,80]
+  CRUSH rule 1 x 426 [485,936,407,854,726,524,791]
+  CRUSH rule 1 x 427 [242,515,9,564,174,453,334]
+  CRUSH rule 1 x 428 [632,635,26,473,494,478,225]
+  CRUSH rule 1 x 429 [641,73,465,127,171,397,857]
+  CRUSH rule 1 x 430 [626,585,6,387,881,583,859]
+  CRUSH rule 1 x 431 [697,76,753,570,964,339,194]
+  CRUSH rule 1 x 432 [590,526,306,283,656,728,513]
+  CRUSH rule 1 x 433 [284,387,149,817,886,714,52]
+  CRUSH rule 1 x 434 [538,985,79,953,770,468,644]
+  CRUSH rule 1 x 435 [30,318,593,635,975,833,371]
+  CRUSH rule 1 x 436 [164,919,851,693,0,874,10]
+  CRUSH rule 1 x 437 [322,212,163,606,302,282,443]
+  CRUSH rule 1 x 438 [142,392,85,594,376,419,755]
+  CRUSH rule 1 x 439 [119,370,68,443,997,837,414]
+  CRUSH rule 1 x 440 [333,403,187,863,475,844,800]
+  CRUSH rule 1 x 441 [477,727,906,145,429,91,205]
+  CRUSH rule 1 x 442 [274,590,933,244,434,49,864]
+  CRUSH rule 1 x 443 [983,748,574,718,700,442,774]
+  CRUSH rule 1 x 444 [536,509,431,146,170,149,182]
+  CRUSH rule 1 x 445 [485,638,528,209,964,753,554]
+  CRUSH rule 1 x 446 [345,634,42,294,711,376,314]
+  CRUSH rule 1 x 447 [61,845,767,600,321,716,58]
+  CRUSH rule 1 x 448 [333,232,292,846,364,951,807]
+  CRUSH rule 1 x 449 [680,16,484,670,851,500,258]
+  CRUSH rule 1 x 450 [235,214,79,423,96,822,721]
+  CRUSH rule 1 x 451 [961,468,333,640,823,151,878]
+  CRUSH rule 1 x 452 [525,479,153,528,570,806,604]
+  CRUSH rule 1 x 453 [138,466,302,86,249,154,514]
+  CRUSH rule 1 x 454 [137,625,215,402,389,914,106]
+  CRUSH rule 1 x 455 [173,150,997,16,846,888,295]
+  CRUSH rule 1 x 456 [235,226,238,258,347,784,504]
+  CRUSH rule 1 x 457 [450,577,253,413,717,609,762]
+  CRUSH rule 1 x 458 [195,537,91,814,351,90,399]
+  CRUSH rule 1 x 459 [381,555,312,573,915,623,147]
+  CRUSH rule 1 x 460 [972,730,534,678,756,692,841]
+  CRUSH rule 1 x 461 [506,279,142,830,784,124,385]
+  CRUSH rule 1 x 462 [692,959,578,57,983,299,240]
+  CRUSH rule 1 x 463 [788,667,949,550,685,702,538]
+  CRUSH rule 1 x 464 [133,122,588,999,270,880,789]
+  CRUSH rule 1 x 465 [971,190,230,777,452,914,137]
+  CRUSH rule 1 x 466 [394,576,148,157,103,822,659]
+  CRUSH rule 1 x 467 [517,28,366,362,984,521,187]
+  CRUSH rule 1 x 468 [829,143,874,225,162,413,201]
+  CRUSH rule 1 x 469 [987,936,106,725,633,238,681]
+  CRUSH rule 1 x 470 [107,982,56,889,67,65,558]
+  CRUSH rule 1 x 471 [181,897,629,860,307,116,256]
+  CRUSH rule 1 x 472 [547,512,172,24,705,837,809]
+  CRUSH rule 1 x 473 [760,997,824,905,888,755,756]
+  CRUSH rule 1 x 474 [787,418,743,628,272,341,446]
+  CRUSH rule 1 x 475 [662,312,253,617,105,58,237]
+  CRUSH rule 1 x 476 [110,495,185,508,961,837,984]
+  CRUSH rule 1 x 477 [393,954,834,132,841,367,753]
+  CRUSH rule 1 x 478 [246,483,480,644,985,420,941]
+  CRUSH rule 1 x 479 [70,929,697,931,744,487,158]
+  CRUSH rule 1 x 480 [753,119,961,607,317,717,371]
+  CRUSH rule 1 x 481 [470,429,677,242,574,757,135]
+  CRUSH rule 1 x 482 [451,566,961,675,354,746,731]
+  CRUSH rule 1 x 483 [816,72,371,278,635,30,448]
+  CRUSH rule 1 x 484 [540,454,389,31,654,494,283]
+  CRUSH rule 1 x 485 [74,582,624,684,566,677,866]
+  CRUSH rule 1 x 486 [958,595,199,763,715,973,621]
+  CRUSH rule 1 x 487 [228,302,804,833,876,647,857]
+  CRUSH rule 1 x 488 [180,529,722,956,353,890,924]
+  CRUSH rule 1 x 489 [47,617,812,187,291,828,154]
+  CRUSH rule 1 x 490 [905,822,479,124,750,843,566]
+  CRUSH rule 1 x 491 [892,370,609,998,433,957,188]
+  CRUSH rule 1 x 492 [588,959,127,948,505,936,591]
+  CRUSH rule 1 x 493 [353,461,593,291,301,830,231]
+  CRUSH rule 1 x 494 [378,848,443,368,507,423,389]
+  CRUSH rule 1 x 495 [845,653,768,234,405,367,823]
+  CRUSH rule 1 x 496 [13,988,0,691,389,757,129]
+  CRUSH rule 1 x 497 [796,877,788,394,648,829,542]
+  CRUSH rule 1 x 498 [412,337,270,705,511,227,949]
+  CRUSH rule 1 x 499 [330,695,8,74,618,101,440]
+  CRUSH rule 1 x 500 [820,272,547,765,755,96,930]
+  CRUSH rule 1 x 501 [110,44,132,442,294,423,880]
+  CRUSH rule 1 x 502 [336,595,650,274,993,312,490]
+  CRUSH rule 1 x 503 [922,211,157,722,502,971,262]
+  CRUSH rule 1 x 504 [483,52,122,432,778,461,758]
+  CRUSH rule 1 x 505 [482,598,224,279,480,310,764]
+  CRUSH rule 1 x 506 [493,123,43,856,936,622,898]
+  CRUSH rule 1 x 507 [12,598,264,422,416,947,591]
+  CRUSH rule 1 x 508 [227,157,611,301,223,746,313]
+  CRUSH rule 1 x 509 [807,242,363,122,582,530,798]
+  CRUSH rule 1 x 510 [134,437,227,75,313,351,786]
+  CRUSH rule 1 x 511 [212,54,83,799,457,218,600]
+  CRUSH rule 1 x 512 [236,630,758,752,361,249,899]
+  CRUSH rule 1 x 513 [994,693,644,938,846,685,52]
+  CRUSH rule 1 x 514 [45,508,831,19,817,52,374]
+  CRUSH rule 1 x 515 [504,138,480,272,530,377,481]
+  CRUSH rule 1 x 516 [285,409,136,570,841,610,453]
+  CRUSH rule 1 x 517 [300,232,23,906,438,236,519]
+  CRUSH rule 1 x 518 [397,674,98,898,967,113,625]
+  CRUSH rule 1 x 519 [86,750,772,913,101,864,375]
+  CRUSH rule 1 x 520 [900,833,614,130,261,885,558]
+  CRUSH rule 1 x 521 [31,47,236,751,911,599,495]
+  CRUSH rule 1 x 522 [390,16,280,144,291,175,753]
+  CRUSH rule 1 x 523 [618,308,424,590,300,206,834]
+  CRUSH rule 1 x 524 [635,189,687,963,601,518,8]
+  CRUSH rule 1 x 525 [311,916,699,262,775,32,45]
+  CRUSH rule 1 x 526 [48,738,227,718,244,942,853]
+  CRUSH rule 1 x 527 [202,851,889,216,763,351,270]
+  CRUSH rule 1 x 528 [565,827,590,273,918,106,651]
+  CRUSH rule 1 x 529 [934,864,241,43,466,924,278]
+  CRUSH rule 1 x 530 [502,934,298,670,986,360,577]
+  CRUSH rule 1 x 531 [681,627,942,487,288,561,925]
+  CRUSH rule 1 x 532 [422,6,147,205,861,141,949]
+  CRUSH rule 1 x 533 [863,68,364,983,247,199,54]
+  CRUSH rule 1 x 534 [962,931,775,172,663,119,206]
+  CRUSH rule 1 x 535 [89,565,397,693,839,632,859]
+  CRUSH rule 1 x 536 [499,351,760,458,918,86,148]
+  CRUSH rule 1 x 537 [676,547,787,311,867,748,152]
+  CRUSH rule 1 x 538 [58,644,571,649,941,7,37]
+  CRUSH rule 1 x 539 [837,953,457,711,458,621,528]
+  CRUSH rule 1 x 540 [831,50,132,213,197,709,95]
+  CRUSH rule 1 x 541 [582,757,121,525,532,963,738]
+  CRUSH rule 1 x 542 [472,132,790,997,948,269,137]
+  CRUSH rule 1 x 543 [382,272,797,330,315,748,324]
+  CRUSH rule 1 x 544 [947,930,496,883,509,219,250]
+  CRUSH rule 1 x 545 [425,570,305,77,821,422,117]
+  CRUSH rule 1 x 546 [18,65,529,437,343,547,699]
+  CRUSH rule 1 x 547 [445,715,600,472,213,851,428]
+  CRUSH rule 1 x 548 [367,569,980,167,627,442,517]
+  CRUSH rule 1 x 549 [125,715,671,817,285,420,37]
+  CRUSH rule 1 x 550 [425,599,744,199,923,222,915]
+  CRUSH rule 1 x 551 [44,1,528,922,944,115,161]
+  CRUSH rule 1 x 552 [246,104,68,239,123,427,57]
+  CRUSH rule 1 x 553 [71,703,615,28,593,724,218]
+  CRUSH rule 1 x 554 [207,124,217,166,525,226,693]
+  CRUSH rule 1 x 555 [570,28,317,420,931,413,623]
+  CRUSH rule 1 x 556 [674,152,421,79,215,347,830]
+  CRUSH rule 1 x 557 [347,817,191,391,741,571,593]
+  CRUSH rule 1 x 558 [627,426,369,692,815,371,124]
+  CRUSH rule 1 x 559 [940,630,924,242,224,912,185]
+  CRUSH rule 1 x 560 [295,903,541,29,245,753,887]
+  CRUSH rule 1 x 561 [506,682,384,637,878,991,700]
+  CRUSH rule 1 x 562 [718,529,87,729,842,341,62]
+  CRUSH rule 1 x 563 [552,332,747,206,274,871,903]
+  CRUSH rule 1 x 564 [835,769,736,486,630,209,641]
+  CRUSH rule 1 x 565 [8,167,539,182,607,62,738]
+  CRUSH rule 1 x 566 [600,481,301,263,90,450,184]
+  CRUSH rule 1 x 567 [999,994,509,899,947,24,267]
+  CRUSH rule 1 x 568 [252,431,157,62,601,863,398]
+  CRUSH rule 1 x 569 [643,218,943,455,83,969,494]
+  CRUSH rule 1 x 570 [617,635,765,422,250,156,533]
+  CRUSH rule 1 x 571 [757,80,59,98,328,700,329]
+  CRUSH rule 1 x 572 [299,348,575,889,943,675,33]
+  CRUSH rule 1 x 573 [25,505,270,167,58,901,878]
+  CRUSH rule 1 x 574 [215,431,624,177,628,814,333]
+  CRUSH rule 1 x 575 [225,252,611,546,32,815,389]
+  CRUSH rule 1 x 576 [627,94,159,857,430,691,177]
+  CRUSH rule 1 x 577 [237,809,778,636,61,167,700]
+  CRUSH rule 1 x 578 [885,313,120,344,771,614,487]
+  CRUSH rule 1 x 579 [924,575,787,831,47,996,557]
+  CRUSH rule 1 x 580 [718,51,766,121,118,471,608]
+  CRUSH rule 1 x 581 [219,807,129,571,856,179,874]
+  CRUSH rule 1 x 582 [893,701,598,863,285,829,984]
+  CRUSH rule 1 x 583 [246,930,964,170,993,409,469]
+  CRUSH rule 1 x 584 [336,432,680,175,495,839,642]
+  CRUSH rule 1 x 585 [324,999,397,485,457,527,73]
+  CRUSH rule 1 x 586 [558,230,976,541,816,72,794]
+  CRUSH rule 1 x 587 [985,830,597,21,308,890,952]
+  CRUSH rule 1 x 588 [211,544,57,134,162,496,195]
+  CRUSH rule 1 x 589 [129,21,112,190,885,844,753]
+  CRUSH rule 1 x 590 [467,969,652,593,287,76,811]
+  CRUSH rule 1 x 591 [758,514,316,164,35,110,54]
+  CRUSH rule 1 x 592 [525,253,190,443,315,603,667]
+  CRUSH rule 1 x 593 [601,885,339,152,297,223,269]
+  CRUSH rule 1 x 594 [227,60,450,30,717,840,994]
+  CRUSH rule 1 x 595 [720,854,496,912,80,655,917]
+  CRUSH rule 1 x 596 [751,195,997,77,261,490,180]
+  CRUSH rule 1 x 597 [129,574,714,8,789,847,725]
+  CRUSH rule 1 x 598 [679,207,604,396,841,284,286]
+  CRUSH rule 1 x 599 [668,315,683,349,681,253,599]
+  CRUSH rule 1 x 600 [143,396,464,444,59,57,243]
+  CRUSH rule 1 x 601 [326,573,873,902,136,921,633]
+  CRUSH rule 1 x 602 [860,281,875,535,672,474,697]
+  CRUSH rule 1 x 603 [709,328,445,349,190,455,924]
+  CRUSH rule 1 x 604 [571,62,814,95,866,978,983]
+  CRUSH rule 1 x 605 [252,739,860,27,313,362,857]
+  CRUSH rule 1 x 606 [339,236,759,842,67,644,954]
+  CRUSH rule 1 x 607 [590,248,759,868,433,398,578]
+  CRUSH rule 1 x 608 [145,635,309,467,875,115,148]
+  CRUSH rule 1 x 609 [973,547,223,79,762,863,249]
+  CRUSH rule 1 x 610 [435,816,961,983,255,886,160]
+  CRUSH rule 1 x 611 [559,283,422,584,176,429,570]
+  CRUSH rule 1 x 612 [273,149,123,576,911,270,296]
+  CRUSH rule 1 x 613 [828,614,642,674,33,361,958]
+  CRUSH rule 1 x 614 [478,748,393,34,171,80,92]
+  CRUSH rule 1 x 615 [392,155,144,326,626,134,149]
+  CRUSH rule 1 x 616 [778,637,452,248,15,888,74]
+  CRUSH rule 1 x 617 [622,713,996,833,611,407,364]
+  CRUSH rule 1 x 618 [149,877,270,329,180,327,222]
+  CRUSH rule 1 x 619 [604,163,656,409,322,848,519]
+  CRUSH rule 1 x 620 [181,23,409,198,64,898,35]
+  CRUSH rule 1 x 621 [735,902,386,237,939,475,725]
+  CRUSH rule 1 x 622 [661,824,717,568,858,583,446]
+  CRUSH rule 1 x 623 [142,121,643,61,695,852,485]
+  CRUSH rule 1 x 624 [360,716,420,398,49,717,137]
+  CRUSH rule 1 x 625 [541,167,385,1,601,481,308]
+  CRUSH rule 1 x 626 [364,431,610,363,535,747,225]
+  CRUSH rule 1 x 627 [458,137,557,410,287,749,467]
+  CRUSH rule 1 x 628 [250,350,556,497,821,65,205]
+  CRUSH rule 1 x 629 [928,160,710,572,365,772,538]
+  CRUSH rule 1 x 630 [243,19,918,556,601,16,920]
+  CRUSH rule 1 x 631 [438,221,574,676,797,580,219]
+  CRUSH rule 1 x 632 [797,368,247,5,32,102,416]
+  CRUSH rule 1 x 633 [993,749,525,485,27,330,275]
+  CRUSH rule 1 x 634 [239,351,633,299,651,678,296]
+  CRUSH rule 1 x 635 [640,965,25,961,306,172,849]
+  CRUSH rule 1 x 636 [173,290,297,991,937,823,236]
+  CRUSH rule 1 x 637 [0,918,98,108,111,495,887]
+  CRUSH rule 1 x 638 [702,235,424,900,983,754,701]
+  CRUSH rule 1 x 639 [475,687,31,785,918,611,27]
+  CRUSH rule 1 x 640 [31,664,399,677,123,609,858]
+  CRUSH rule 1 x 641 [296,473,108,963,341,876,897]
+  CRUSH rule 1 x 642 [894,273,427,606,677,670,610]
+  CRUSH rule 1 x 643 [117,111,732,191,114,153,500]
+  CRUSH rule 1 x 644 [438,336,327,512,599,862,660]
+  CRUSH rule 1 x 645 [982,702,351,573,907,915,279]
+  CRUSH rule 1 x 646 [334,804,146,842,697,638,720]
+  CRUSH rule 1 x 647 [933,787,185,334,752,285,372]
+  CRUSH rule 1 x 648 [22,444,400,862,207,842,453]
+  CRUSH rule 1 x 649 [503,229,213,460,639,760,722]
+  CRUSH rule 1 x 650 [328,659,420,443,739,950,869]
+  CRUSH rule 1 x 651 [3,880,823,123,378,585,715]
+  CRUSH rule 1 x 652 [495,977,563,733,92,997,119]
+  CRUSH rule 1 x 653 [185,718,804,280,975,912,198]
+  CRUSH rule 1 x 654 [130,528,380,81,906,511,904]
+  CRUSH rule 1 x 655 [560,872,454,504,319,284,605]
+  CRUSH rule 1 x 656 [219,885,178,981,863,508,708]
+  CRUSH rule 1 x 657 [233,684,813,490,208,941,858]
+  CRUSH rule 1 x 658 [778,6,756,380,750,836,547]
+  CRUSH rule 1 x 659 [240,663,306,540,789,902,170]
+  CRUSH rule 1 x 660 [244,855,196,147,678,323,63]
+  CRUSH rule 1 x 661 [184,270,128,398,910,230,402]
+  CRUSH rule 1 x 662 [65,883,921,438,79,957,464]
+  CRUSH rule 1 x 663 [323,721,594,812,43,992,170]
+  CRUSH rule 1 x 664 [865,113,512,51,427,123,585]
+  CRUSH rule 1 x 665 [420,850,591,475,202,733,798]
+  CRUSH rule 1 x 666 [319,767,246,3,369,493,796]
+  CRUSH rule 1 x 667 [875,39,343,100,829,2,795]
+  CRUSH rule 1 x 668 [331,122,263,599,355,484,943]
+  CRUSH rule 1 x 669 [915,521,402,747,673,445,938]
+  CRUSH rule 1 x 670 [845,659,943,447,401,322,168]
+  CRUSH rule 1 x 671 [108,634,527,363,856,238,755]
+  CRUSH rule 1 x 672 [578,216,110,589,302,137,954]
+  CRUSH rule 1 x 673 [442,74,579,797,622,950,371]
+  CRUSH rule 1 x 674 [588,364,281,308,645,631,229]
+  CRUSH rule 1 x 675 [489,698,744,671,870,174,528]
+  CRUSH rule 1 x 676 [928,911,40,180,722,729,673]
+  CRUSH rule 1 x 677 [399,269,692,131,615,136,103]
+  CRUSH rule 1 x 678 [546,752,544,155,5,463,666]
+  CRUSH rule 1 x 679 [988,25,275,433,628,57,247]
+  CRUSH rule 1 x 680 [335,963,382,486,749,257,795]
+  CRUSH rule 1 x 681 [690,462,623,466,49,471,774]
+  CRUSH rule 1 x 682 [196,588,154,257,807,776,367]
+  CRUSH rule 1 x 683 [627,25,421,160,873,102,345]
+  CRUSH rule 1 x 684 [38,804,592,158,991,264,652]
+  CRUSH rule 1 x 685 [841,368,548,362,166,211,154]
+  CRUSH rule 1 x 686 [336,287,525,440,166,993,911]
+  CRUSH rule 1 x 687 [20,682,924,653,356,16,917]
+  CRUSH rule 1 x 688 [463,371,780,556,385,883,115]
+  CRUSH rule 1 x 689 [569,250,78,816,847,775,333]
+  CRUSH rule 1 x 690 [551,144,587,263,378,394,970]
+  CRUSH rule 1 x 691 [766,464,446,533,449,541,451]
+  CRUSH rule 1 x 692 [739,634,18,245,624,35,268]
+  CRUSH rule 1 x 693 [339,297,118,330,817,91,828]
+  CRUSH rule 1 x 694 [405,26,830,181,533,166,488]
+  CRUSH rule 1 x 695 [622,576,597,535,600,593,300]
+  CRUSH rule 1 x 696 [558,902,689,13,715,28,664]
+  CRUSH rule 1 x 697 [818,222,406,691,427,863,153]
+  CRUSH rule 1 x 698 [178,48,402,233,841,604,468]
+  CRUSH rule 1 x 699 [450,244,180,919,335,332,747]
+  CRUSH rule 1 x 700 [502,771,987,706,416,240,68]
+  CRUSH rule 1 x 701 [4,612,782,216,853,303,585]
+  CRUSH rule 1 x 702 [177,630,232,923,281,708,466]
+  CRUSH rule 1 x 703 [354,178,389,393,778,803,796]
+  CRUSH rule 1 x 704 [646,601,156,171,603,116,655]
+  CRUSH rule 1 x 705 [921,401,890,265,244,690,372]
+  CRUSH rule 1 x 706 [652,877,562,452,26,323,923]
+  CRUSH rule 1 x 707 [345,745,67,716,789,576,2]
+  CRUSH rule 1 x 708 [333,607,180,469,170,555,939]
+  CRUSH rule 1 x 709 [45,187,302,115,896,579,733]
+  CRUSH rule 1 x 710 [94,855,43,199,18,948,449]
+  CRUSH rule 1 x 711 [227,653,731,150,356,842,534]
+  CRUSH rule 1 x 712 [398,953,136,870,181,408,895]
+  CRUSH rule 1 x 713 [116,800,503,662,635,579,53]
+  CRUSH rule 1 x 714 [111,629,866,709,902,557,875]
+  CRUSH rule 1 x 715 [531,291,486,382,192,807,322]
+  CRUSH rule 1 x 716 [169,541,291,42,343,724,138]
+  CRUSH rule 1 x 717 [417,446,994,894,239,494,237]
+  CRUSH rule 1 x 718 [992,383,298,844,377,463,544]
+  CRUSH rule 1 x 719 [936,674,324,759,194,409,828]
+  CRUSH rule 1 x 720 [370,188,174,464,644,218,214]
+  CRUSH rule 1 x 721 [320,859,278,259,170,957,177]
+  CRUSH rule 1 x 722 [7,2,673,129,96,445,823]
+  CRUSH rule 1 x 723 [270,553,831,662,38,101,985]
+  CRUSH rule 1 x 724 [666,822,708,895,633,800,616]
+  CRUSH rule 1 x 725 [794,406,875,459,981,751,359]
+  CRUSH rule 1 x 726 [420,556,341,292,240,68,966]
+  CRUSH rule 1 x 727 [561,461,129,635,965,610,105]
+  CRUSH rule 1 x 728 [951,330,196,756,589,849,753]
+  CRUSH rule 1 x 729 [656,644,436,591,27,119,572]
+  CRUSH rule 1 x 730 [3,558,629,184,50,765,760]
+  CRUSH rule 1 x 731 [852,89,75,735,713,113,528]
+  CRUSH rule 1 x 732 [983,840,869,976,697,307,368]
+  CRUSH rule 1 x 733 [285,396,388,122,387,364,880]
+  CRUSH rule 1 x 734 [125,510,402,640,676,501,535]
+  CRUSH rule 1 x 735 [417,773,686,504,459,912,690]
+  CRUSH rule 1 x 736 [749,396,632,550,779,109,845]
+  CRUSH rule 1 x 737 [644,991,946,135,448,903,482]
+  CRUSH rule 1 x 738 [449,683,290,220,245,525,429]
+  CRUSH rule 1 x 739 [341,220,641,454,740,661,146]
+  CRUSH rule 1 x 740 [874,524,674,650,472,282,214]
+  CRUSH rule 1 x 741 [189,472,712,798,715,757,863]
+  CRUSH rule 1 x 742 [912,581,114,245,730,21,687]
+  CRUSH rule 1 x 743 [654,914,425,441,763,39,451]
+  CRUSH rule 1 x 744 [725,295,579,377,162,447,843]
+  CRUSH rule 1 x 745 [787,858,850,506,612,735,926]
+  CRUSH rule 1 x 746 [757,848,704,30,47,940,450]
+  CRUSH rule 1 x 747 [700,81,867,681,801,64,879]
+  CRUSH rule 1 x 748 [557,436,238,664,293,865,304]
+  CRUSH rule 1 x 749 [772,622,337,42,156,302,383]
+  CRUSH rule 1 x 750 [946,97,376,677,316,670,169]
+  CRUSH rule 1 x 751 [996,618,343,911,83,22,388]
+  CRUSH rule 1 x 752 [746,887,695,868,610,950,88]
+  CRUSH rule 1 x 753 [741,14,463,479,172,192,481]
+  CRUSH rule 1 x 754 [648,349,333,355,65,63,336]
+  CRUSH rule 1 x 755 [157,460,466,187,959,674,192]
+  CRUSH rule 1 x 756 [416,97,197,497,227,3,850]
+  CRUSH rule 1 x 757 [599,839,776,410,256,823,121]
+  CRUSH rule 1 x 758 [994,218,620,256,361,749,165]
+  CRUSH rule 1 x 759 [959,682,514,745,100,519,15]
+  CRUSH rule 1 x 760 [518,943,215,83,706,137,345]
+  CRUSH rule 1 x 761 [285,849,420,324,987,338,373]
+  CRUSH rule 1 x 762 [591,313,41,335,110,696,664]
+  CRUSH rule 1 x 763 [908,411,200,740,292,295,387]
+  CRUSH rule 1 x 764 [787,234,894,485,883,711,70]
+  CRUSH rule 1 x 765 [327,921,882,393,444,792,402]
+  CRUSH rule 1 x 766 [84,161,878,704,416,144,357]
+  CRUSH rule 1 x 767 [370,895,702,701,890,2,251]
+  CRUSH rule 1 x 768 [826,760,879,864,460,474,645]
+  CRUSH rule 1 x 769 [67,768,663,735,814,66,213]
+  CRUSH rule 1 x 770 [593,909,482,259,5,550,961]
+  CRUSH rule 1 x 771 [309,935,121,578,937,685,933]
+  CRUSH rule 1 x 772 [12,125,797,301,348,419,891]
+  CRUSH rule 1 x 773 [253,466,820,549,591,193,783]
+  CRUSH rule 1 x 774 [164,390,705,109,881,505,890]
+  CRUSH rule 1 x 775 [703,47,43,973,643,406,885]
+  CRUSH rule 1 x 776 [728,231,80,916,2,850,396]
+  CRUSH rule 1 x 777 [981,621,568,729,869,952,563]
+  CRUSH rule 1 x 778 [411,456,544,597,789,784,65]
+  CRUSH rule 1 x 779 [346,121,519,921,587,48,772]
+  CRUSH rule 1 x 780 [476,39,288,381,303,29,17]
+  CRUSH rule 1 x 781 [10,130,585,844,729,705,714]
+  CRUSH rule 1 x 782 [462,246,581,902,623,877,812]
+  CRUSH rule 1 x 783 [580,373,153,775,668,661,626]
+  CRUSH rule 1 x 784 [413,113,978,990,994,56,481]
+  CRUSH rule 1 x 785 [341,856,332,354,59,581,632]
+  CRUSH rule 1 x 786 [411,140,313,393,215,618,490]
+  CRUSH rule 1 x 787 [605,522,211,813,636,224,600]
+  CRUSH rule 1 x 788 [226,545,35,142,726,851,194]
+  CRUSH rule 1 x 789 [545,320,414,702,731,277,237]
+  CRUSH rule 1 x 790 [414,748,816,327,130,115,788]
+  CRUSH rule 1 x 791 [660,906,406,697,916,322,124]
+  CRUSH rule 1 x 792 [287,392,514,204,75,789,406]
+  CRUSH rule 1 x 793 [631,133,850,713,720,487,376]
+  CRUSH rule 1 x 794 [931,517,543,210,963,898,811]
+  CRUSH rule 1 x 795 [551,962,477,948,425,434,268]
+  CRUSH rule 1 x 796 [814,4,95,27,368,300,646]
+  CRUSH rule 1 x 797 [64,201,299,734,605,864,596]
+  CRUSH rule 1 x 798 [422,530,114,431,565,716,473]
+  CRUSH rule 1 x 799 [824,32,679,562,266,549,859]
+  CRUSH rule 1 x 800 [862,623,489,637,861,196,941]
+  CRUSH rule 1 x 801 [145,550,329,324,734,160,219]
+  CRUSH rule 1 x 802 [570,19,847,308,387,518,846]
+  CRUSH rule 1 x 803 [151,812,662,358,880,349,834]
+  CRUSH rule 1 x 804 [467,93,264,863,176,842,663]
+  CRUSH rule 1 x 805 [621,223,938,809,591,686,121]
+  CRUSH rule 1 x 806 [898,957,805,430,499,584,640]
+  CRUSH rule 1 x 807 [354,531,422,159,921,431,802]
+  CRUSH rule 1 x 808 [7,96,76,897,446,2,166]
+  CRUSH rule 1 x 809 [70,734,719,56,687,21,23]
+  CRUSH rule 1 x 810 [701,18,972,327,771,649,620]
+  CRUSH rule 1 x 811 [248,547,103,728,901,264,948]
+  CRUSH rule 1 x 812 [230,576,821,566,993,762,675]
+  CRUSH rule 1 x 813 [805,114,683,629,845,462,285]
+  CRUSH rule 1 x 814 [54,619,973,741,497,894,401]
+  CRUSH rule 1 x 815 [679,412,613,132,969,411,314]
+  CRUSH rule 1 x 816 [919,448,826,414,36,289,44]
+  CRUSH rule 1 x 817 [765,830,436,521,332,458,260]
+  CRUSH rule 1 x 818 [415,566,644,687,692,414,769]
+  CRUSH rule 1 x 819 [721,319,865,750,546,859,523]
+  CRUSH rule 1 x 820 [218,301,333,190,686,179,535]
+  CRUSH rule 1 x 821 [185,795,680,953,329,750,621]
+  CRUSH rule 1 x 822 [356,261,54,522,900,103,883]
+  CRUSH rule 1 x 823 [220,281,549,456,64,306,282]
+  CRUSH rule 1 x 824 [292,809,887,74,776,788,559]
+  CRUSH rule 1 x 825 [949,778,101,311,110,480,161]
+  CRUSH rule 1 x 826 [767,818,833,927,356,954,910]
+  CRUSH rule 1 x 827 [631,83,406,635,657,713,212]
+  CRUSH rule 1 x 828 [288,986,445,26,414,607,937]
+  CRUSH rule 1 x 829 [990,667,915,694,974,453,669]
+  CRUSH rule 1 x 830 [152,571,778,505,685,209,448]
+  CRUSH rule 1 x 831 [814,563,630,97,582,107,142]
+  CRUSH rule 1 x 832 [235,641,616,110,979,844,656]
+  CRUSH rule 1 x 833 [657,565,922,140,825,457,764]
+  CRUSH rule 1 x 834 [907,231,644,13,617,130,83]
+  CRUSH rule 1 x 835 [784,262,771,264,612,238,537]
+  CRUSH rule 1 x 836 [951,158,366,710,43,427,351]
+  CRUSH rule 1 x 837 [556,498,334,633,895,627,903]
+  CRUSH rule 1 x 838 [329,274,964,547,119,342,983]
+  CRUSH rule 1 x 839 [568,209,939,364,658,747,47]
+  CRUSH rule 1 x 840 [45,579,842,70,655,862,815]
+  CRUSH rule 1 x 841 [652,702,24,605,152,93,226]
+  CRUSH rule 1 x 842 [629,984,314,895,408,897,575]
+  CRUSH rule 1 x 843 [799,690,688,648,151,812,486]
+  CRUSH rule 1 x 844 [694,600,534,700,569,11,899]
+  CRUSH rule 1 x 845 [332,30,179,93,951,324,611]
+  CRUSH rule 1 x 846 [452,251,712,719,404,739,606]
+  CRUSH rule 1 x 847 [399,681,847,739,13,555,363]
+  CRUSH rule 1 x 848 [303,138,440,346,547,216,700]
+  CRUSH rule 1 x 849 [666,346,708,873,64,694,847]
+  CRUSH rule 1 x 850 [644,511,345,844,545,337,358]
+  CRUSH rule 1 x 851 [527,546,737,425,100,331,95]
+  CRUSH rule 1 x 852 [31,809,94,618,156,853,469]
+  CRUSH rule 1 x 853 [483,330,869,184,46,942,774]
+  CRUSH rule 1 x 854 [697,953,968,143,502,955,441]
+  CRUSH rule 1 x 855 [837,996,239,621,32,191,686]
+  CRUSH rule 1 x 856 [712,40,547,430,195,857,224]
+  CRUSH rule 1 x 857 [77,984,576,551,568,96,12]
+  CRUSH rule 1 x 858 [412,384,841,465,572,576,688]
+  CRUSH rule 1 x 859 [173,760,26,300,87,567,463]
+  CRUSH rule 1 x 860 [776,429,328,917,658,783,699]
+  CRUSH rule 1 x 861 [705,405,477,50,73,714,901]
+  CRUSH rule 1 x 862 [809,44,788,938,964,177,490]
+  CRUSH rule 1 x 863 [349,496,963,178,675,853,172]
+  CRUSH rule 1 x 864 [717,858,101,239,992,244,43]
+  CRUSH rule 1 x 865 [857,603,586,262,550,289,850]
+  CRUSH rule 1 x 866 [394,304,71,96,642,155,255]
+  CRUSH rule 1 x 867 [640,773,663,974,261,296,988]
+  CRUSH rule 1 x 868 [613,950,712,663,528,460,643]
+  CRUSH rule 1 x 869 [973,889,524,22,671,477,718]
+  CRUSH rule 1 x 870 [505,35,386,498,348,503,54]
+  CRUSH rule 1 x 871 [239,264,262,773,781,734,387]
+  CRUSH rule 1 x 872 [21,767,456,748,783,797,180]
+  CRUSH rule 1 x 873 [954,666,980,264,435,233,199]
+  CRUSH rule 1 x 874 [54,510,947,1,500,119,93]
+  CRUSH rule 1 x 875 [809,418,452,462,88,673,634]
+  CRUSH rule 1 x 876 [483,457,61,248,523,277,322]
+  CRUSH rule 1 x 877 [542,531,952,939,710,179,181]
+  CRUSH rule 1 x 878 [217,674,857,644,678,809,329]
+  CRUSH rule 1 x 879 [999,475,134,250,319,357,145]
+  CRUSH rule 1 x 880 [678,573,935,385,570,651,319]
+  CRUSH rule 1 x 881 [394,835,789,802,587,155,570]
+  CRUSH rule 1 x 882 [467,382,353,56,979,674,974]
+  CRUSH rule 1 x 883 [802,744,237,337,50,96,202]
+  CRUSH rule 1 x 884 [653,660,638,700,31,558,389]
+  CRUSH rule 1 x 885 [898,704,307,445,879,872,174]
+  CRUSH rule 1 x 886 [434,357,938,641,737,8,56]
+  CRUSH rule 1 x 887 [297,226,711,428,370,318,472]
+  CRUSH rule 1 x 888 [863,324,443,213,902,25,806]
+  CRUSH rule 1 x 889 [105,102,308,163,947,548,399]
+  CRUSH rule 1 x 890 [550,248,606,704,615,708,996]
+  CRUSH rule 1 x 891 [575,928,880,891,826,763,706]
+  CRUSH rule 1 x 892 [259,862,133,271,292,162,53]
+  CRUSH rule 1 x 893 [902,880,543,542,37,942,672]
+  CRUSH rule 1 x 894 [180,169,916,43,945,713,648]
+  CRUSH rule 1 x 895 [725,849,182,129,177,272,599]
+  CRUSH rule 1 x 896 [951,34,874,537,969,123,210]
+  CRUSH rule 1 x 897 [810,352,73,939,943,895,12]
+  CRUSH rule 1 x 898 [979,433,719,411,787,359,342]
+  CRUSH rule 1 x 899 [685,668,534,932,399,156,124]
+  CRUSH rule 1 x 900 [530,978,41,894,941,681,380]
+  CRUSH rule 1 x 901 [740,107,336,175,574,706,157]
+  CRUSH rule 1 x 902 [800,743,693,310,67,111,178]
+  CRUSH rule 1 x 903 [230,267,842,266,550,769,66]
+  CRUSH rule 1 x 904 [346,949,460,973,696,91,957]
+  CRUSH rule 1 x 905 [530,397,619,958,576,973,685]
+  CRUSH rule 1 x 906 [80,426,138,672,73,776,30]
+  CRUSH rule 1 x 907 [365,968,475,297,296,724,664]
+  CRUSH rule 1 x 908 [204,832,742,809,862,745,484]
+  CRUSH rule 1 x 909 [883,989,146,959,366,59,686]
+  CRUSH rule 1 x 910 [549,593,249,853,792,769,824]
+  CRUSH rule 1 x 911 [325,847,352,214,851,732,789]
+  CRUSH rule 1 x 912 [874,888,582,796,557,601,226]
+  CRUSH rule 1 x 913 [331,463,342,574,989,362,925]
+  CRUSH rule 1 x 914 [836,468,601,732,607,275,70]
+  CRUSH rule 1 x 915 [245,228,100,661,799,13,126]
+  CRUSH rule 1 x 916 [77,967,364,435,27,474,255]
+  CRUSH rule 1 x 917 [239,60,866,221,772,967,725]
+  CRUSH rule 1 x 918 [988,115,922,80,201,544,583]
+  CRUSH rule 1 x 919 [783,139,696,1,848,169,888]
+  CRUSH rule 1 x 920 [623,408,685,953,974,696,532]
+  CRUSH rule 1 x 921 [105,799,144,90,399,373,633]
+  CRUSH rule 1 x 922 [887,505,652,348,514,806,952]
+  CRUSH rule 1 x 923 [223,318,552,458,743,871,964]
+  CRUSH rule 1 x 924 [25,778,366,333,163,801,584]
+  CRUSH rule 1 x 925 [912,601,297,682,770,173,969]
+  CRUSH rule 1 x 926 [968,133,739,144,814,155,709]
+  CRUSH rule 1 x 927 [277,724,214,988,690,342,465]
+  CRUSH rule 1 x 928 [554,203,658,789,298,299,847]
+  CRUSH rule 1 x 929 [761,802,367,528,758,522,744]
+  CRUSH rule 1 x 930 [814,61,788,736,660,491,832]
+  CRUSH rule 1 x 931 [29,193,61,41,343,664,487]
+  CRUSH rule 1 x 932 [446,198,862,534,168,35,530]
+  CRUSH rule 1 x 933 [352,742,216,321,525,44,568]
+  CRUSH rule 1 x 934 [730,2,332,631,613,249,533]
+  CRUSH rule 1 x 935 [731,23,736,79,361,992,772]
+  CRUSH rule 1 x 936 [322,975,20,904,827,603,138]
+  CRUSH rule 1 x 937 [822,221,841,161,723,137,630]
+  CRUSH rule 1 x 938 [557,850,66,630,499,404,286]
+  CRUSH rule 1 x 939 [150,11,971,371,124,785,408]
+  CRUSH rule 1 x 940 [638,398,169,616,333,751,25]
+  CRUSH rule 1 x 941 [730,342,929,577,451,838,964]
+  CRUSH rule 1 x 942 [62,292,166,814,587,172,237]
+  CRUSH rule 1 x 943 [165,314,519,548,41,726,759]
+  CRUSH rule 1 x 944 [199,625,766,176,194,297,678]
+  CRUSH rule 1 x 945 [946,999,699,303,38,81,952]
+  CRUSH rule 1 x 946 [595,93,852,142,503,647,933]
+  CRUSH rule 1 x 947 [800,582,356,93,716,117,922]
+  CRUSH rule 1 x 948 [132,551,139,920,87,46,81]
+  CRUSH rule 1 x 949 [792,920,466,380,97,568,799]
+  CRUSH rule 1 x 950 [111,345,176,543,879,954,355]
+  CRUSH rule 1 x 951 [414,619,648,655,364,971,829]
+  CRUSH rule 1 x 952 [775,469,500,356,287,4,16]
+  CRUSH rule 1 x 953 [349,1,5,251,168,680,141]
+  CRUSH rule 1 x 954 [570,940,410,249,929,394,129]
+  CRUSH rule 1 x 955 [729,774,823,800,7,127,536]
+  CRUSH rule 1 x 956 [519,141,575,625,738,475,169]
+  CRUSH rule 1 x 957 [242,709,611,97,760,309,393]
+  CRUSH rule 1 x 958 [84,217,227,253,246,604,346]
+  CRUSH rule 1 x 959 [270,413,918,789,703,608,543]
+  CRUSH rule 1 x 960 [458,192,307,279,920,139,855]
+  CRUSH rule 1 x 961 [981,388,777,546,359,660,455]
+  CRUSH rule 1 x 962 [623,834,277,134,729,246,856]
+  CRUSH rule 1 x 963 [291,167,714,468,109,373,485]
+  CRUSH rule 1 x 964 [28,156,788,127,598,215,361]
+  CRUSH rule 1 x 965 [675,557,290,517,840,510,59]
+  CRUSH rule 1 x 966 [836,306,946,283,642,606,929]
+  CRUSH rule 1 x 967 [966,386,735,837,392,116,19]
+  CRUSH rule 1 x 968 [864,756,690,121,328,122,433]
+  CRUSH rule 1 x 969 [729,625,480,769,512,882,518]
+  CRUSH rule 1 x 970 [800,362,646,582,309,102,576]
+  CRUSH rule 1 x 971 [737,381,153,684,298,166,344]
+  CRUSH rule 1 x 972 [952,245,720,884,334,311,754]
+  CRUSH rule 1 x 973 [356,455,579,857,832,596,549]
+  CRUSH rule 1 x 974 [545,758,586,596,269,790,116]
+  CRUSH rule 1 x 975 [336,191,202,146,720,897,330]
+  CRUSH rule 1 x 976 [446,208,757,620,252,846,397]
+  CRUSH rule 1 x 977 [202,896,196,956,763,126,783]
+  CRUSH rule 1 x 978 [612,324,996,225,418,583,514]
+  CRUSH rule 1 x 979 [843,457,675,650,958,657,677]
+  CRUSH rule 1 x 980 [60,914,881,626,850,759,398]
+  CRUSH rule 1 x 981 [702,749,937,153,724,514,536]
+  CRUSH rule 1 x 982 [298,928,738,167,99,668,395]
+  CRUSH rule 1 x 983 [723,572,395,358,900,37,927]
+  CRUSH rule 1 x 984 [723,864,804,935,846,993,950]
+  CRUSH rule 1 x 985 [945,459,868,211,524,954,911]
+  CRUSH rule 1 x 986 [772,664,535,169,297,996,864]
+  CRUSH rule 1 x 987 [88,324,312,843,661,580,76]
+  CRUSH rule 1 x 988 [522,927,131,996,351,685,865]
+  CRUSH rule 1 x 989 [578,332,208,605,975,207,155]
+  CRUSH rule 1 x 990 [638,228,414,311,738,698,340]
+  CRUSH rule 1 x 991 [530,221,451,422,879,916,754]
+  CRUSH rule 1 x 992 [925,705,275,81,234,310,117]
+  CRUSH rule 1 x 993 [991,301,43,469,830,242,382]
+  CRUSH rule 1 x 994 [276,51,868,683,843,815,557]
+  CRUSH rule 1 x 995 [288,836,753,790,758,120,158]
+  CRUSH rule 1 x 996 [887,983,252,686,470,345,459]
+  CRUSH rule 1 x 997 [110,924,386,79,705,697,210]
+  CRUSH rule 1 x 998 [435,830,485,853,926,730,786]
+  CRUSH rule 1 x 999 [876,738,357,913,723,51,15]
+  CRUSH rule 1 x 1000 [178,963,638,430,845,586,317]
+  CRUSH rule 1 x 1001 [99,519,66,759,583,944,739]
+  CRUSH rule 1 x 1002 [515,534,468,866,878,717,729]
+  CRUSH rule 1 x 1003 [104,611,937,698,94,67,614]
+  CRUSH rule 1 x 1004 [269,638,724,375,491,121,891]
+  CRUSH rule 1 x 1005 [369,223,309,409,822,39,597]
+  CRUSH rule 1 x 1006 [40,107,69,275,79,429,234]
+  CRUSH rule 1 x 1007 [978,111,416,758,454,640,5]
+  CRUSH rule 1 x 1008 [965,956,624,832,421,96,975]
+  CRUSH rule 1 x 1009 [598,476,356,695,919,566,234]
+  CRUSH rule 1 x 1010 [767,523,239,517,29,77,23]
+  CRUSH rule 1 x 1011 [289,871,207,576,347,698,48]
+  CRUSH rule 1 x 1012 [128,28,370,31,341,755,268]
+  CRUSH rule 1 x 1013 [979,765,660,812,666,187,808]
+  CRUSH rule 1 x 1014 [979,948,513,88,47,825,969]
+  CRUSH rule 1 x 1015 [277,790,396,672,542,647,145]
+  CRUSH rule 1 x 1016 [262,73,128,886,839,685,456]
+  CRUSH rule 1 x 1017 [150,269,61,499,832,591,637]
+  CRUSH rule 1 x 1018 [555,829,554,944,406,576,463]
+  CRUSH rule 1 x 1019 [513,356,265,446,65,288,768]
+  CRUSH rule 1 x 1020 [158,161,877,704,948,570,495]
+  CRUSH rule 1 x 1021 [915,998,957,285,546,202,676]
+  CRUSH rule 1 x 1022 [967,829,973,640,703,470,871]
+  CRUSH rule 1 x 1023 [488,257,614,859,325,419,50]
+  rule 1 (metadata) num_rep 7 result size == 7:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536,450,604,380,966,750]
+  CRUSH rule 1 x 1 [876,250,334,633,744,843,672,820]
+  CRUSH rule 1 x 2 [292,832,53,392,386,787,527,901]
+  CRUSH rule 1 x 3 [623,387,124,998,749,211,481,169]
+  CRUSH rule 1 x 4 [61,334,710,4,994,982,847,220]
+  CRUSH rule 1 x 5 [946,557,713,664,141,817,964,872]
+  CRUSH rule 1 x 6 [576,668,212,163,732,381,884,726]
+  CRUSH rule 1 x 7 [645,753,906,393,341,44,578,14]
+  CRUSH rule 1 x 8 [243,6,863,781,211,100,462,207]
+  CRUSH rule 1 x 9 [22,578,251,410,297,430,3,569]
+  CRUSH rule 1 x 10 [758,828,360,477,821,801,811,484]
+  CRUSH rule 1 x 11 [769,120,124,527,119,504,380,821]
+  CRUSH rule 1 x 12 [780,364,689,755,675,199,117,393]
+  CRUSH rule 1 x 13 [557,18,351,719,742,780,78,170]
+  CRUSH rule 1 x 14 [59,561,249,461,971,835,855,76]
+  CRUSH rule 1 x 15 [718,928,993,21,76,313,437,664]
+  CRUSH rule 1 x 16 [673,632,841,954,788,90,786,969]
+  CRUSH rule 1 x 17 [648,43,560,514,142,289,935,605]
+  CRUSH rule 1 x 18 [654,219,181,568,381,253,883,394]
+  CRUSH rule 1 x 19 [850,545,377,848,863,543,51,834]
+  CRUSH rule 1 x 20 [717,785,974,5,225,552,975,636]
+  CRUSH rule 1 x 21 [420,57,519,306,312,983,263,267]
+  CRUSH rule 1 x 22 [503,998,193,821,634,684,557,633]
+  CRUSH rule 1 x 23 [411,663,168,110,899,488,477,468]
+  CRUSH rule 1 x 24 [266,861,353,1,456,128,800,309]
+  CRUSH rule 1 x 25 [760,483,818,600,509,951,248,908]
+  CRUSH rule 1 x 26 [903,24,573,718,112,694,501,909]
+  CRUSH rule 1 x 27 [946,188,289,510,687,827,676,560]
+  CRUSH rule 1 x 28 [69,312,73,198,256,629,770,569]
+  CRUSH rule 1 x 29 [844,883,337,628,496,405,719,581]
+  CRUSH rule 1 x 30 [621,18,613,794,910,936,426,522]
+  CRUSH rule 1 x 31 [784,943,814,539,962,392,813,217]
+  CRUSH rule 1 x 32 [173,374,369,972,315,83,428,63]
+  CRUSH rule 1 x 33 [698,336,357,966,582,407,618,288]
+  CRUSH rule 1 x 34 [168,836,210,798,904,190,663,877]
+  CRUSH rule 1 x 35 [274,509,534,818,912,671,75,580]
+  CRUSH rule 1 x 36 [318,215,153,628,87,407,676,524]
+  CRUSH rule 1 x 37 [173,604,109,935,203,401,311,758]
+  CRUSH rule 1 x 38 [708,444,683,604,722,900,929,910]
+  CRUSH rule 1 x 39 [662,198,417,680,226,342,856,248]
+  CRUSH rule 1 x 40 [620,801,414,78,560,766,980,503]
+  CRUSH rule 1 x 41 [811,264,177,127,148,791,930,74]
+  CRUSH rule 1 x 42 [863,179,527,660,133,529,456,713]
+  CRUSH rule 1 x 43 [686,822,988,228,791,549,514,40]
+  CRUSH rule 1 x 44 [396,222,46,841,536,140,160,527]
+  CRUSH rule 1 x 45 [991,694,253,142,54,422,658,876]
+  CRUSH rule 1 x 46 [420,909,184,285,508,458,45,390]
+  CRUSH rule 1 x 47 [467,211,605,207,241,881,959,800]
+  CRUSH rule 1 x 48 [955,329,368,168,698,787,738,47]
+  CRUSH rule 1 x 49 [974,891,931,29,813,506,822,628]
+  CRUSH rule 1 x 50 [870,441,691,823,761,6,83,344]
+  CRUSH rule 1 x 51 [182,930,25,936,97,260,406,281]
+  CRUSH rule 1 x 52 [704,812,894,794,481,37,304,899]
+  CRUSH rule 1 x 53 [185,713,631,280,345,558,882,503]
+  CRUSH rule 1 x 54 [270,441,100,82,983,930,339,902]
+  CRUSH rule 1 x 55 [895,734,958,793,651,572,508,763]
+  CRUSH rule 1 x 56 [564,963,683,324,40,189,77,500]
+  CRUSH rule 1 x 57 [738,130,208,973,498,861,670,67]
+  CRUSH rule 1 x 58 [524,113,806,903,531,334,8,762]
+  CRUSH rule 1 x 59 [408,337,668,529,34,384,643,511]
+  CRUSH rule 1 x 60 [228,790,857,309,616,895,194,277]
+  CRUSH rule 1 x 61 [154,843,717,467,883,536,812,14]
+  CRUSH rule 1 x 62 [594,811,549,276,693,917,45,723]
+  CRUSH rule 1 x 63 [646,67,884,925,941,434,705,268]
+  CRUSH rule 1 x 64 [175,542,155,837,594,197,451,891]
+  CRUSH rule 1 x 65 [745,619,131,867,269,62,862,221]
+  CRUSH rule 1 x 66 [275,468,23,35,328,432,334,656]
+  CRUSH rule 1 x 67 [246,958,524,493,636,227,783,593]
+  CRUSH rule 1 x 68 [711,473,403,228,835,126,705,114]
+  CRUSH rule 1 x 69 [493,924,850,939,950,105,871,361]
+  CRUSH rule 1 x 70 [30,499,644,33,804,654,684,411]
+  CRUSH rule 1 x 71 [984,883,574,716,575,391,587,264]
+  CRUSH rule 1 x 72 [71,286,942,363,628,632,642,529]
+  CRUSH rule 1 x 73 [922,618,3,371,464,442,835,705]
+  CRUSH rule 1 x 74 [629,414,185,573,678,338,633,560]
+  CRUSH rule 1 x 75 [222,20,174,820,312,361,366,258]
+  CRUSH rule 1 x 76 [262,366,339,290,718,143,735,953]
+  CRUSH rule 1 x 77 [638,469,992,280,773,892,197,690]
+  CRUSH rule 1 x 78 [324,511,788,7,308,228,183,917]
+  CRUSH rule 1 x 79 [577,990,64,94,447,924,339,24]
+  CRUSH rule 1 x 80 [501,95,278,903,631,842,51,766]
+  CRUSH rule 1 x 81 [506,812,9,698,173,664,247,963]
+  CRUSH rule 1 x 82 [222,145,80,785,835,745,580,51]
+  CRUSH rule 1 x 83 [71,634,61,91,856,529,66,197]
+  CRUSH rule 1 x 84 [49,761,773,368,318,708,681,618]
+  CRUSH rule 1 x 85 [985,896,708,861,325,307,567,908]
+  CRUSH rule 1 x 86 [537,745,93,524,466,356,38,326]
+  CRUSH rule 1 x 87 [997,317,463,626,685,458,909,49]
+  CRUSH rule 1 x 88 [957,350,890,857,375,176,99,737]
+  CRUSH rule 1 x 89 [399,730,148,314,159,982,320,921]
+  CRUSH rule 1 x 90 [943,706,683,267,579,141,412,184]
+  CRUSH rule 1 x 91 [22,368,149,928,140,529,495,299]
+  CRUSH rule 1 x 92 [532,424,426,773,623,197,167,634]
+  CRUSH rule 1 x 93 [218,489,405,681,549,201,343,949]
+  CRUSH rule 1 x 94 [181,96,102,515,776,365,82,422]
+  CRUSH rule 1 x 95 [343,957,820,139,334,37,648,661]
+  CRUSH rule 1 x 96 [861,270,87,797,0,245,204,750]
+  CRUSH rule 1 x 97 [459,706,45,328,274,605,83,542]
+  CRUSH rule 1 x 98 [327,867,353,948,728,280,270,511]
+  CRUSH rule 1 x 99 [974,133,468,906,235,988,37,138]
+  CRUSH rule 1 x 100 [32,445,547,371,960,885,9,168]
+  CRUSH rule 1 x 101 [142,90,337,950,970,570,12,369]
+  CRUSH rule 1 x 102 [172,129,139,22,403,867,923,106]
+  CRUSH rule 1 x 103 [630,47,161,356,911,421,933,231]
+  CRUSH rule 1 x 104 [758,133,278,11,947,799,401,85]
+  CRUSH rule 1 x 105 [843,604,47,33,401,632,434,121]
+  CRUSH rule 1 x 106 [28,681,193,679,990,343,878,493]
+  CRUSH rule 1 x 107 [74,320,85,819,315,253,589,614]
+  CRUSH rule 1 x 108 [875,593,575,517,107,153,631,996]
+  CRUSH rule 1 x 109 [411,985,811,720,198,666,856,296]
+  CRUSH rule 1 x 110 [440,774,799,660,715,167,510,472]
+  CRUSH rule 1 x 111 [405,742,276,359,936,360,18,949]
+  CRUSH rule 1 x 112 [143,181,922,545,185,303,725,413]
+  CRUSH rule 1 x 113 [153,846,160,903,789,897,738,253]
+  CRUSH rule 1 x 114 [804,892,939,20,312,692,598,418]
+  CRUSH rule 1 x 115 [588,508,958,580,232,722,421,39]
+  CRUSH rule 1 x 116 [327,148,637,486,712,464,9,448]
+  CRUSH rule 1 x 117 [95,594,989,131,714,275,725,142]
+  CRUSH rule 1 x 118 [80,957,897,239,359,432,766,210]
+  CRUSH rule 1 x 119 [386,932,951,768,679,300,570,278]
+  CRUSH rule 1 x 120 [366,312,653,936,71,241,49,126]
+  CRUSH rule 1 x 121 [129,154,847,16,471,481,424,868]
+  CRUSH rule 1 x 122 [873,1,110,939,90,412,551,43]
+  CRUSH rule 1 x 123 [533,415,789,600,713,800,877,248]
+  CRUSH rule 1 x 124 [461,691,898,723,957,759,482,254]
+  CRUSH rule 1 x 125 [342,599,830,402,615,994,736,737]
+  CRUSH rule 1 x 126 [819,781,822,548,279,255,689,209]
+  CRUSH rule 1 x 127 [437,893,585,707,353,189,909,809]
+  CRUSH rule 1 x 128 [679,994,982,550,991,324,666,691]
+  CRUSH rule 1 x 129 [380,685,947,302,698,144,149,51]
+  CRUSH rule 1 x 130 [992,52,466,867,998,777,270,425]
+  CRUSH rule 1 x 131 [469,90,208,599,829,656,203,667]
+  CRUSH rule 1 x 132 [571,250,316,535,54,418,922,597]
+  CRUSH rule 1 x 133 [964,728,329,902,108,118,14,444]
+  CRUSH rule 1 x 134 [999,19,716,963,323,559,893,281]
+  CRUSH rule 1 x 135 [634,101,52,938,413,573,712,649]
+  CRUSH rule 1 x 136 [114,889,692,768,694,279,846,890]
+  CRUSH rule 1 x 137 [839,8,959,280,922,870,363,323]
+  CRUSH rule 1 x 138 [967,949,138,451,292,548,400,885]
+  CRUSH rule 1 x 139 [308,711,736,247,632,126,384,58]
+  CRUSH rule 1 x 140 [764,936,926,55,331,115,178,532]
+  CRUSH rule 1 x 141 [423,302,112,216,603,873,193,258]
+  CRUSH rule 1 x 142 [252,821,715,340,635,668,424,87]
+  CRUSH rule 1 x 143 [33,808,518,477,325,316,266,70]
+  CRUSH rule 1 x 144 [472,88,969,162,401,771,697,610]
+  CRUSH rule 1 x 145 [242,208,252,604,266,743,577,348]
+  CRUSH rule 1 x 146 [290,70,570,384,934,856,929,196]
+  CRUSH rule 1 x 147 [447,352,657,493,467,918,514,546]
+  CRUSH rule 1 x 148 [212,644,432,658,109,275,352,820]
+  CRUSH rule 1 x 149 [9,775,87,35,260,646,406,556]
+  CRUSH rule 1 x 150 [166,456,582,144,324,340,484,553]
+  CRUSH rule 1 x 151 [811,875,307,20,782,229,671,883]
+  CRUSH rule 1 x 152 [449,617,223,9,182,407,807,50]
+  CRUSH rule 1 x 153 [523,537,695,627,959,613,942,864]
+  CRUSH rule 1 x 154 [208,559,874,597,243,706,443,98]
+  CRUSH rule 1 x 155 [569,325,192,296,367,848,58,641]
+  CRUSH rule 1 x 156 [488,121,521,213,595,837,271,229]
+  CRUSH rule 1 x 157 [140,723,633,260,487,856,384,446]
+  CRUSH rule 1 x 158 [786,451,320,239,667,632,899,902]
+  CRUSH rule 1 x 159 [134,664,517,821,667,944,209,641]
+  CRUSH rule 1 x 160 [690,112,414,990,183,590,242,999]
+  CRUSH rule 1 x 161 [324,912,397,423,991,284,909,642]
+  CRUSH rule 1 x 162 [748,567,284,183,463,336,148,88]
+  CRUSH rule 1 x 163 [575,499,31,816,749,737,587,854]
+  CRUSH rule 1 x 164 [314,489,308,326,51,568,110,329]
+  CRUSH rule 1 x 165 [116,209,750,53,813,640,524,389]
+  CRUSH rule 1 x 166 [352,706,701,810,718,527,548,676]
+  CRUSH rule 1 x 167 [27,743,174,142,551,1,935,266]
+  CRUSH rule 1 x 168 [953,898,880,660,500,799,667,463]
+  CRUSH rule 1 x 169 [912,147,266,547,331,770,601,909]
+  CRUSH rule 1 x 170 [421,515,828,844,151,981,835,840]
+  CRUSH rule 1 x 171 [488,584,880,964,936,196,100,910]
+  CRUSH rule 1 x 172 [366,443,957,66,162,693,36,356]
+  CRUSH rule 1 x 173 [863,291,625,287,158,496,471,529]
+  CRUSH rule 1 x 174 [263,555,650,410,339,616,780,932]
+  CRUSH rule 1 x 175 [875,961,361,575,33,109,51,211]
+  CRUSH rule 1 x 176 [745,83,701,680,250,420,240,316]
+  CRUSH rule 1 x 177 [128,244,41,123,422,902,756,647]
+  CRUSH rule 1 x 178 [155,41,264,777,314,564,856,992]
+  CRUSH rule 1 x 179 [593,833,202,183,971,38,724,923]
+  CRUSH rule 1 x 180 [154,734,17,831,824,522,736,846]
+  CRUSH rule 1 x 181 [289,675,723,800,166,712,168,224]
+  CRUSH rule 1 x 182 [730,931,560,209,943,261,485,571]
+  CRUSH rule 1 x 183 [639,237,794,815,827,400,109,903]
+  CRUSH rule 1 x 184 [704,312,685,645,691,778,74,45]
+  CRUSH rule 1 x 185 [97,100,762,82,999,542,485,511]
+  CRUSH rule 1 x 186 [26,665,554,215,280,421,369,270]
+  CRUSH rule 1 x 187 [649,14,740,494,402,684,566,378]
+  CRUSH rule 1 x 188 [682,695,590,743,927,945,833,650]
+  CRUSH rule 1 x 189 [325,693,726,51,448,169,37,1]
+  CRUSH rule 1 x 190 [399,933,136,955,57,504,527,237]
+  CRUSH rule 1 x 191 [629,533,17,126,60,146,999,754]
+  CRUSH rule 1 x 192 [503,578,38,492,222,251,123,759]
+  CRUSH rule 1 x 193 [546,333,651,678,823,652,359,721]
+  CRUSH rule 1 x 194 [242,473,58,655,311,277,792,887]
+  CRUSH rule 1 x 195 [625,719,135,81,636,513,755,471]
+  CRUSH rule 1 x 196 [357,114,125,867,250,522,413,834]
+  CRUSH rule 1 x 197 [306,954,453,873,211,334,666,316]
+  CRUSH rule 1 x 198 [863,791,311,911,206,61,355,574]
+  CRUSH rule 1 x 199 [935,906,929,252,893,75,960,369]
+  CRUSH rule 1 x 200 [373,774,229,454,909,611,132,271]
+  CRUSH rule 1 x 201 [659,320,477,313,779,16,495,76]
+  CRUSH rule 1 x 202 [260,433,524,880,223,818,153,272]
+  CRUSH rule 1 x 203 [36,239,675,971,703,209,669,676]
+  CRUSH rule 1 x 204 [92,516,993,728,279,478,697,881]
+  CRUSH rule 1 x 205 [68,395,473,45,683,662,776,463]
+  CRUSH rule 1 x 206 [570,530,642,380,311,398,230,367]
+  CRUSH rule 1 x 207 [834,457,850,917,456,296,76,708]
+  CRUSH rule 1 x 208 [927,484,640,976,803,626,96,841]
+  CRUSH rule 1 x 209 [878,66,58,940,48,233,522,185]
+  CRUSH rule 1 x 210 [572,981,484,29,0,426,14,921]
+  CRUSH rule 1 x 211 [107,597,780,857,895,57,922,372]
+  CRUSH rule 1 x 212 [389,107,838,624,698,562,857,894]
+  CRUSH rule 1 x 213 [497,717,567,728,905,134,687,903]
+  CRUSH rule 1 x 214 [798,65,254,572,32,393,579,79]
+  CRUSH rule 1 x 215 [233,419,283,638,520,891,982,826]
+  CRUSH rule 1 x 216 [494,464,742,523,459,174,973,898]
+  CRUSH rule 1 x 217 [352,396,309,938,66,41,264,6]
+  CRUSH rule 1 x 218 [895,864,988,650,593,740,34,497]
+  CRUSH rule 1 x 219 [222,534,277,242,658,482,697,805]
+  CRUSH rule 1 x 220 [281,19,584,563,858,965,686,982]
+  CRUSH rule 1 x 221 [64,928,963,130,312,394,61,559]
+  CRUSH rule 1 x 222 [40,544,161,199,861,644,597,904]
+  CRUSH rule 1 x 223 [645,556,159,417,46,135,465,429]
+  CRUSH rule 1 x 224 [647,165,957,263,961,576,329,320]
+  CRUSH rule 1 x 225 [219,714,858,747,461,175,606,465]
+  CRUSH rule 1 x 226 [372,511,181,277,695,404,876,984]
+  CRUSH rule 1 x 227 [925,156,714,863,257,74,966,217]
+  CRUSH rule 1 x 228 [682,404,839,263,521,195,261,389]
+  CRUSH rule 1 x 229 [880,838,770,891,236,542,262,884]
+  CRUSH rule 1 x 230 [328,659,916,468,646,572,93,880]
+  CRUSH rule 1 x 231 [320,383,669,109,627,621,50,182]
+  CRUSH rule 1 x 232 [924,846,394,319,43,519,106,877]
+  CRUSH rule 1 x 233 [948,652,575,838,498,395,796,835]
+  CRUSH rule 1 x 234 [484,943,42,575,936,180,103,95]
+  CRUSH rule 1 x 235 [750,65,590,168,870,308,471,753]
+  CRUSH rule 1 x 236 [551,787,490,136,370,833,573,128]
+  CRUSH rule 1 x 237 [390,157,166,251,752,75,327,509]
+  CRUSH rule 1 x 238 [570,6,989,707,514,905,894,884]
+  CRUSH rule 1 x 239 [729,959,376,975,496,49,426,427]
+  CRUSH rule 1 x 240 [981,241,156,767,631,576,450,677]
+  CRUSH rule 1 x 241 [310,816,641,177,996,454,413,136]
+  CRUSH rule 1 x 242 [161,63,642,837,763,458,234,756]
+  CRUSH rule 1 x 243 [180,394,33,683,189,419,799,21]
+  CRUSH rule 1 x 244 [52,174,685,189,78,310,785,107]
+  CRUSH rule 1 x 245 [523,121,915,84,386,409,605,837]
+  CRUSH rule 1 x 246 [362,893,390,487,817,88,989,999]
+  CRUSH rule 1 x 247 [382,184,116,34,143,15,590,840]
+  CRUSH rule 1 x 248 [129,114,852,469,359,291,713,237]
+  CRUSH rule 1 x 249 [159,683,91,856,475,369,886,650]
+  CRUSH rule 1 x 250 [404,945,569,955,228,910,270,619]
+  CRUSH rule 1 x 251 [661,225,738,757,37,642,58,354]
+  CRUSH rule 1 x 252 [961,226,542,103,945,885,838,131]
+  CRUSH rule 1 x 253 [651,97,225,364,189,248,797,675]
+  CRUSH rule 1 x 254 [123,33,741,692,599,11,605,453]
+  CRUSH rule 1 x 255 [314,649,891,855,517,344,607,95]
+  CRUSH rule 1 x 256 [315,215,651,126,470,849,189,627]
+  CRUSH rule 1 x 257 [825,264,867,141,529,409,291,732]
+  CRUSH rule 1 x 258 [624,789,370,723,131,982,863,427]
+  CRUSH rule 1 x 259 [602,542,70,563,947,723,77,191]
+  CRUSH rule 1 x 260 [717,878,43,56,377,481,533,646]
+  CRUSH rule 1 x 261 [145,517,20,903,786,939,516,136]
+  CRUSH rule 1 x 262 [223,1,561,420,680,16,88,534]
+  CRUSH rule 1 x 263 [462,211,405,508,787,669,773,979]
+  CRUSH rule 1 x 264 [654,471,266,662,135,564,715,916]
+  CRUSH rule 1 x 265 [302,794,704,798,659,487,833,987]
+  CRUSH rule 1 x 266 [202,132,884,209,551,984,7,557]
+  CRUSH rule 1 x 267 [282,938,657,113,672,993,972,645]
+  CRUSH rule 1 x 268 [338,309,356,278,928,797,715,536]
+  CRUSH rule 1 x 269 [738,122,266,200,894,118,146,14]
+  CRUSH rule 1 x 270 [707,982,946,196,407,804,476,571]
+  CRUSH rule 1 x 271 [705,432,364,735,512,595,263,138]
+  CRUSH rule 1 x 272 [756,545,942,56,542,449,710,779]
+  CRUSH rule 1 x 273 [197,502,527,721,239,648,982,735]
+  CRUSH rule 1 x 274 [992,44,653,573,527,702,370,990]
+  CRUSH rule 1 x 275 [544,789,170,434,23,926,992,823]
+  CRUSH rule 1 x 276 [658,467,577,268,336,5,634,98]
+  CRUSH rule 1 x 277 [143,490,880,483,928,272,783,648]
+  CRUSH rule 1 x 278 [492,647,355,282,834,64,350,600]
+  CRUSH rule 1 x 279 [517,792,604,987,527,894,952,250]
+  CRUSH rule 1 x 280 [825,740,27,848,514,750,895,914]
+  CRUSH rule 1 x 281 [224,629,120,562,616,200,443,604]
+  CRUSH rule 1 x 282 [298,661,380,416,35,585,939,879]
+  CRUSH rule 1 x 283 [311,606,208,50,913,678,369,544]
+  CRUSH rule 1 x 284 [771,466,371,743,672,119,60,546]
+  CRUSH rule 1 x 285 [693,362,404,676,797,531,582,975]
+  CRUSH rule 1 x 286 [364,477,285,167,270,617,699,627]
+  CRUSH rule 1 x 287 [591,611,828,995,170,987,137,890]
+  CRUSH rule 1 x 288 [965,541,848,796,251,668,195,538]
+  CRUSH rule 1 x 289 [225,551,948,877,219,167,795,377]
+  CRUSH rule 1 x 290 [577,762,777,751,291,349,473,209]
+  CRUSH rule 1 x 291 [160,903,477,381,490,559,557,86]
+  CRUSH rule 1 x 292 [873,598,216,666,222,228,806,911]
+  CRUSH rule 1 x 293 [100,234,874,47,28,452,775,636]
+  CRUSH rule 1 x 294 [285,943,379,520,725,547,459,833]
+  CRUSH rule 1 x 295 [938,262,880,327,687,3,440,73]
+  CRUSH rule 1 x 296 [850,327,86,472,1,776,266,82]
+  CRUSH rule 1 x 297 [951,53,99,558,753,228,232,343]
+  CRUSH rule 1 x 298 [173,336,85,766,910,657,213,286]
+  CRUSH rule 1 x 299 [598,591,315,386,895,296,924,106]
+  CRUSH rule 1 x 300 [531,957,62,459,156,538,904,838]
+  CRUSH rule 1 x 301 [823,628,23,858,629,808,220,432]
+  CRUSH rule 1 x 302 [184,80,780,871,531,211,400,365]
+  CRUSH rule 1 x 303 [521,766,222,830,988,275,561,905]
+  CRUSH rule 1 x 304 [980,127,807,507,555,245,214,944]
+  CRUSH rule 1 x 305 [153,816,22,927,696,911,685,838]
+  CRUSH rule 1 x 306 [423,739,664,753,178,431,761,648]
+  CRUSH rule 1 x 307 [997,557,682,456,479,631,459,250]
+  CRUSH rule 1 x 308 [991,874,534,465,330,284,976,551]
+  CRUSH rule 1 x 309 [860,394,724,858,246,866,857,153]
+  CRUSH rule 1 x 310 [589,818,546,201,94,653,90,855]
+  CRUSH rule 1 x 311 [477,774,225,590,830,559,256,798]
+  CRUSH rule 1 x 312 [887,853,950,354,58,23,497,929]
+  CRUSH rule 1 x 313 [802,646,447,416,557,118,24,81]
+  CRUSH rule 1 x 314 [654,974,229,511,562,916,952,599]
+  CRUSH rule 1 x 315 [767,227,28,740,828,156,749,841]
+  CRUSH rule 1 x 316 [778,83,733,359,858,319,761,725]
+  CRUSH rule 1 x 317 [184,418,642,986,939,675,892,86]
+  CRUSH rule 1 x 318 [525,410,500,543,212,95,290,97]
+  CRUSH rule 1 x 319 [476,724,569,382,409,521,800,868]
+  CRUSH rule 1 x 320 [149,610,697,296,818,955,523,366]
+  CRUSH rule 1 x 321 [710,79,667,671,234,4,868,841]
+  CRUSH rule 1 x 322 [175,275,323,333,744,718,187,380]
+  CRUSH rule 1 x 323 [819,604,638,792,316,544,236,404]
+  CRUSH rule 1 x 324 [16,745,511,439,272,95,668,959]
+  CRUSH rule 1 x 325 [486,400,872,873,251,68,462,268]
+  CRUSH rule 1 x 326 [613,765,207,19,359,370,461,509]
+  CRUSH rule 1 x 327 [125,289,738,408,456,784,750,669]
+  CRUSH rule 1 x 328 [807,383,476,583,645,141,33,806]
+  CRUSH rule 1 x 329 [588,938,599,432,446,840,516,713]
+  CRUSH rule 1 x 330 [932,644,41,611,209,406,420,520]
+  CRUSH rule 1 x 331 [341,953,950,537,578,862,624,649]
+  CRUSH rule 1 x 332 [153,726,459,950,466,804,644,821]
+  CRUSH rule 1 x 333 [745,845,853,860,52,615,243,633]
+  CRUSH rule 1 x 334 [614,751,807,58,396,159,408,175]
+  CRUSH rule 1 x 335 [518,721,221,283,454,187,635,367]
+  CRUSH rule 1 x 336 [389,424,77,309,5,898,698,533]
+  CRUSH rule 1 x 337 [753,508,765,720,221,807,956,907]
+  CRUSH rule 1 x 338 [128,810,490,753,406,760,69,11]
+  CRUSH rule 1 x 339 [430,308,58,751,856,823,607,953]
+  CRUSH rule 1 x 340 [541,44,630,231,289,966,707,328]
+  CRUSH rule 1 x 341 [402,26,631,439,165,928,720,503]
+  CRUSH rule 1 x 342 [982,57,992,461,131,32,516,661]
+  CRUSH rule 1 x 343 [833,412,572,732,107,805,660,655]
+  CRUSH rule 1 x 344 [784,533,792,41,642,869,142,114]
+  CRUSH rule 1 x 345 [546,300,304,691,763,556,127,732]
+  CRUSH rule 1 x 346 [302,420,428,891,357,124,419,962]
+  CRUSH rule 1 x 347 [488,778,101,217,366,442,783,661]
+  CRUSH rule 1 x 348 [903,744,937,718,85,314,862,513]
+  CRUSH rule 1 x 349 [471,547,582,306,600,486,795,143]
+  CRUSH rule 1 x 350 [348,221,823,335,383,708,841,164]
+  CRUSH rule 1 x 351 [961,582,705,346,361,32,766,775]
+  CRUSH rule 1 x 352 [728,137,461,298,36,903,899,665]
+  CRUSH rule 1 x 353 [904,202,184,447,58,294,279,616]
+  CRUSH rule 1 x 354 [345,226,319,256,544,311,612,33]
+  CRUSH rule 1 x 355 [50,430,175,43,187,458,985,412]
+  CRUSH rule 1 x 356 [87,185,55,423,829,1,629,228]
+  CRUSH rule 1 x 357 [762,459,921,473,182,231,891,656]
+  CRUSH rule 1 x 358 [908,25,280,6,808,676,874,643]
+  CRUSH rule 1 x 359 [484,15,132,121,394,423,397,52]
+  CRUSH rule 1 x 360 [173,378,337,702,145,499,29,529]
+  CRUSH rule 1 x 361 [404,577,115,25,56,914,643,286]
+  CRUSH rule 1 x 362 [403,1,422,945,132,685,265,35]
+  CRUSH rule 1 x 363 [639,911,510,162,418,294,444,613]
+  CRUSH rule 1 x 364 [752,689,610,990,665,222,203,17]
+  CRUSH rule 1 x 365 [956,999,212,230,624,84,113,373]
+  CRUSH rule 1 x 366 [860,925,924,763,687,851,59,914]
+  CRUSH rule 1 x 367 [205,609,647,665,969,720,685,641]
+  CRUSH rule 1 x 368 [301,284,810,169,78,340,616,93]
+  CRUSH rule 1 x 369 [452,658,339,217,674,210,284,184]
+  CRUSH rule 1 x 370 [11,467,695,989,394,576,850,419]
+  CRUSH rule 1 x 371 [124,487,55,514,313,411,797,547]
+  CRUSH rule 1 x 372 [253,48,979,846,207,631,212,241]
+  CRUSH rule 1 x 373 [715,605,775,748,227,493,128,207]
+  CRUSH rule 1 x 374 [191,887,920,764,223,714,961,760]
+  CRUSH rule 1 x 375 [711,385,651,665,15,71,934,619]
+  CRUSH rule 1 x 376 [597,818,49,458,415,755,446,897]
+  CRUSH rule 1 x 377 [294,256,933,771,184,861,654,487]
+  CRUSH rule 1 x 378 [34,151,681,707,552,127,728,860]
+  CRUSH rule 1 x 379 [869,136,315,378,813,153,115,557]
+  CRUSH rule 1 x 380 [294,97,575,791,690,482,255,806]
+  CRUSH rule 1 x 381 [119,710,219,827,328,886,773,496]
+  CRUSH rule 1 x 382 [69,631,508,706,697,168,276,56]
+  CRUSH rule 1 x 383 [922,588,589,925,471,601,29,197]
+  CRUSH rule 1 x 384 [221,945,671,117,857,655,488,435]
+  CRUSH rule 1 x 385 [561,737,953,723,658,368,910,329]
+  CRUSH rule 1 x 386 [335,442,788,696,507,716,232,692]
+  CRUSH rule 1 x 387 [514,43,353,88,100,842,164,934]
+  CRUSH rule 1 x 388 [587,89,157,996,915,927,474,267]
+  CRUSH rule 1 x 389 [109,641,255,466,372,563,340,222]
+  CRUSH rule 1 x 390 [925,149,421,489,599,810,852,196]
+  CRUSH rule 1 x 391 [267,87,387,527,768,873,481,136]
+  CRUSH rule 1 x 392 [382,485,370,849,936,636,901,82]
+  CRUSH rule 1 x 393 [425,721,221,753,268,463,652,543]
+  CRUSH rule 1 x 394 [898,18,38,793,173,738,15,591]
+  CRUSH rule 1 x 395 [806,876,269,679,32,744,126,179]
+  CRUSH rule 1 x 396 [790,970,437,449,875,395,726,935]
+  CRUSH rule 1 x 397 [136,363,507,613,11,30,996,558]
+  CRUSH rule 1 x 398 [914,116,558,258,722,904,349,672]
+  CRUSH rule 1 x 399 [261,94,299,202,174,622,749,410]
+  CRUSH rule 1 x 400 [661,197,338,461,977,848,536,592]
+  CRUSH rule 1 x 401 [953,979,287,803,41,349,79,32]
+  CRUSH rule 1 x 402 [738,819,618,522,667,334,658,449]
+  CRUSH rule 1 x 403 [573,238,425,546,130,68,202,650]
+  CRUSH rule 1 x 404 [526,848,790,253,922,820,299,577]
+  CRUSH rule 1 x 405 [582,505,330,334,201,110,776,296]
+  CRUSH rule 1 x 406 [768,324,493,60,186,165,718,578]
+  CRUSH rule 1 x 407 [260,951,437,587,692,648,72,345]
+  CRUSH rule 1 x 408 [657,81,770,734,830,821,246,695]
+  CRUSH rule 1 x 409 [498,89,182,423,672,152,213,806]
+  CRUSH rule 1 x 410 [28,793,737,352,166,645,949,507]
+  CRUSH rule 1 x 411 [684,992,60,659,769,267,313,351]
+  CRUSH rule 1 x 412 [261,958,699,950,165,14,560,155]
+  CRUSH rule 1 x 413 [891,835,297,441,384,979,618,907]
+  CRUSH rule 1 x 414 [127,459,119,965,662,594,97,124]
+  CRUSH rule 1 x 415 [272,540,631,328,609,568,694,332]
+  CRUSH rule 1 x 416 [739,617,115,530,339,371,889,344]
+  CRUSH rule 1 x 417 [106,209,157,878,117,128,138,374]
+  CRUSH rule 1 x 418 [525,441,147,390,320,300,848,972]
+  CRUSH rule 1 x 419 [603,673,615,465,266,855,823,884]
+  CRUSH rule 1 x 420 [988,213,251,226,209,245,506,670]
+  CRUSH rule 1 x 421 [761,521,748,368,923,992,764,274]
+  CRUSH rule 1 x 422 [317,160,924,548,198,709,839,547]
+  CRUSH rule 1 x 423 [137,807,168,472,619,443,905,588]
+  CRUSH rule 1 x 424 [920,37,146,263,598,748,785,395]
+  CRUSH rule 1 x 425 [277,693,285,221,478,165,80,236]
+  CRUSH rule 1 x 426 [485,936,407,854,726,524,791,565]
+  CRUSH rule 1 x 427 [242,515,9,564,174,453,334,588]
+  CRUSH rule 1 x 428 [632,635,26,473,494,478,225,94]
+  CRUSH rule 1 x 429 [641,73,465,127,171,397,857,562]
+  CRUSH rule 1 x 430 [626,585,6,387,881,583,859,699]
+  CRUSH rule 1 x 431 [697,76,753,570,964,339,194,366]
+  CRUSH rule 1 x 432 [590,526,306,283,656,728,513,591]
+  CRUSH rule 1 x 433 [284,387,149,817,886,714,52,897]
+  CRUSH rule 1 x 434 [538,985,79,953,770,468,644,646]
+  CRUSH rule 1 x 435 [30,318,593,635,975,833,371,731]
+  CRUSH rule 1 x 436 [164,919,851,693,0,874,10,976]
+  CRUSH rule 1 x 437 [322,212,163,606,302,282,443,23]
+  CRUSH rule 1 x 438 [142,392,85,594,376,419,755,841]
+  CRUSH rule 1 x 439 [119,370,68,443,997,837,414,152]
+  CRUSH rule 1 x 440 [333,403,187,863,475,844,800,174]
+  CRUSH rule 1 x 441 [477,727,906,145,429,91,205,236]
+  CRUSH rule 1 x 442 [274,590,933,244,434,49,864,799]
+  CRUSH rule 1 x 443 [983,748,574,718,700,442,774,350]
+  CRUSH rule 1 x 444 [536,509,431,146,170,149,182,145]
+  CRUSH rule 1 x 445 [485,892,528,209,964,753,554,931]
+  CRUSH rule 1 x 446 [345,634,42,294,711,376,314,714]
+  CRUSH rule 1 x 447 [61,845,767,600,321,716,58,531]
+  CRUSH rule 1 x 448 [333,232,292,846,364,951,807,688]
+  CRUSH rule 1 x 449 [680,16,484,670,851,500,258,548]
+  CRUSH rule 1 x 450 [235,214,79,423,96,822,721,31]
+  CRUSH rule 1 x 451 [961,468,333,640,823,151,878,33]
+  CRUSH rule 1 x 452 [525,479,153,528,570,806,604,49]
+  CRUSH rule 1 x 453 [138,466,302,86,249,154,514,5]
+  CRUSH rule 1 x 454 [137,625,215,402,389,914,106,103]
+  CRUSH rule 1 x 455 [173,150,997,16,846,888,295,967]
+  CRUSH rule 1 x 456 [235,226,238,258,347,784,504,96]
+  CRUSH rule 1 x 457 [450,577,253,413,717,609,762,975]
+  CRUSH rule 1 x 458 [195,537,91,814,351,90,399,558]
+  CRUSH rule 1 x 459 [381,555,312,573,915,623,147,483]
+  CRUSH rule 1 x 460 [972,730,534,678,756,692,841,512]
+  CRUSH rule 1 x 461 [506,279,142,830,784,124,385,797]
+  CRUSH rule 1 x 462 [692,959,578,57,983,299,240,911]
+  CRUSH rule 1 x 463 [788,667,949,550,685,702,538,111]
+  CRUSH rule 1 x 464 [133,122,588,999,270,880,789,0]
+  CRUSH rule 1 x 465 [971,190,230,777,452,914,137,466]
+  CRUSH rule 1 x 466 [394,576,148,157,103,822,659,35]
+  CRUSH rule 1 x 467 [517,28,366,362,984,521,187,640]
+  CRUSH rule 1 x 468 [829,143,874,225,162,413,201,249]
+  CRUSH rule 1 x 469 [987,936,106,725,633,238,681,159]
+  CRUSH rule 1 x 470 [107,982,56,889,67,65,558,71]
+  CRUSH rule 1 x 471 [181,897,629,860,307,116,256,978]
+  CRUSH rule 1 x 472 [547,512,172,24,705,837,809,56]
+  CRUSH rule 1 x 473 [760,997,824,905,888,755,756,663]
+  CRUSH rule 1 x 474 [787,418,743,628,272,341,446,333]
+  CRUSH rule 1 x 475 [662,312,253,617,105,58,237,764]
+  CRUSH rule 1 x 476 [110,495,185,508,961,837,984,226]
+  CRUSH rule 1 x 477 [393,954,834,132,841,367,753,794]
+  CRUSH rule 1 x 478 [246,483,480,644,985,420,941,843]
+  CRUSH rule 1 x 479 [70,929,697,931,744,487,158,489]
+  CRUSH rule 1 x 480 [753,119,961,607,317,717,371,807]
+  CRUSH rule 1 x 481 [470,429,677,242,574,757,135,375]
+  CRUSH rule 1 x 482 [451,566,961,675,354,746,731,233]
+  CRUSH rule 1 x 483 [816,72,371,278,635,30,448,437]
+  CRUSH rule 1 x 484 [540,454,389,31,654,494,283,170]
+  CRUSH rule 1 x 485 [74,582,624,684,566,677,866,661]
+  CRUSH rule 1 x 486 [958,595,199,763,715,973,621,955]
+  CRUSH rule 1 x 487 [228,302,804,833,876,647,857,782]
+  CRUSH rule 1 x 488 [180,529,722,956,353,890,924,965]
+  CRUSH rule 1 x 489 [47,617,812,187,291,828,154,478]
+  CRUSH rule 1 x 490 [905,822,479,124,750,843,566,779]
+  CRUSH rule 1 x 491 [892,370,609,998,433,957,188,563]
+  CRUSH rule 1 x 492 [588,959,127,948,505,936,591,423]
+  CRUSH rule 1 x 493 [353,461,593,291,301,830,231,580]
+  CRUSH rule 1 x 494 [378,848,443,368,507,423,389,819]
+  CRUSH rule 1 x 495 [845,653,768,234,405,367,823,789]
+  CRUSH rule 1 x 496 [13,988,0,691,389,757,129,763]
+  CRUSH rule 1 x 497 [796,877,788,394,648,829,542,745]
+  CRUSH rule 1 x 498 [412,337,270,705,511,227,949,173]
+  CRUSH rule 1 x 499 [330,695,8,74,618,101,440,509]
+  CRUSH rule 1 x 500 [820,272,547,765,755,96,930,573]
+  CRUSH rule 1 x 501 [110,44,132,442,294,423,880,279]
+  CRUSH rule 1 x 502 [336,595,650,274,993,312,490,852]
+  CRUSH rule 1 x 503 [922,211,157,722,502,971,262,926]
+  CRUSH rule 1 x 504 [483,52,122,432,778,461,758,104]
+  CRUSH rule 1 x 505 [482,598,224,279,480,310,764,558]
+  CRUSH rule 1 x 506 [493,123,43,856,936,622,898,161]
+  CRUSH rule 1 x 507 [12,598,264,422,416,947,591,702]
+  CRUSH rule 1 x 508 [227,157,611,301,223,746,313,282]
+  CRUSH rule 1 x 509 [807,242,363,122,582,530,798,808]
+  CRUSH rule 1 x 510 [134,437,227,75,313,351,786,152]
+  CRUSH rule 1 x 511 [212,54,83,799,457,218,600,968]
+  CRUSH rule 1 x 512 [236,630,758,752,361,249,899,451]
+  CRUSH rule 1 x 513 [994,693,644,938,846,685,52,185]
+  CRUSH rule 1 x 514 [45,508,831,19,817,52,374,985]
+  CRUSH rule 1 x 515 [504,138,480,272,530,377,481,820]
+  CRUSH rule 1 x 516 [285,409,136,570,841,610,453,660]
+  CRUSH rule 1 x 517 [300,232,23,906,438,236,519,737]
+  CRUSH rule 1 x 518 [397,674,98,898,967,113,625,434]
+  CRUSH rule 1 x 519 [86,750,772,913,101,864,375,328]
+  CRUSH rule 1 x 520 [900,833,614,130,261,885,558,956]
+  CRUSH rule 1 x 521 [31,47,236,751,911,599,495,354]
+  CRUSH rule 1 x 522 [390,16,280,144,291,175,753,624]
+  CRUSH rule 1 x 523 [618,308,424,590,300,206,834,212]
+  CRUSH rule 1 x 524 [635,189,687,963,601,518,8,550]
+  CRUSH rule 1 x 525 [311,916,699,262,775,32,45,478]
+  CRUSH rule 1 x 526 [48,738,227,718,244,942,853,643]
+  CRUSH rule 1 x 527 [202,851,889,216,763,351,270,35]
+  CRUSH rule 1 x 528 [565,827,590,273,918,106,651,368]
+  CRUSH rule 1 x 529 [934,864,241,43,466,924,278,926]
+  CRUSH rule 1 x 530 [502,934,298,670,986,360,577,509]
+  CRUSH rule 1 x 531 [681,627,942,487,288,561,925,474]
+  CRUSH rule 1 x 532 [422,6,147,205,861,141,949,374]
+  CRUSH rule 1 x 533 [863,68,364,983,247,199,54,931]
+  CRUSH rule 1 x 534 [962,931,775,172,663,119,206,682]
+  CRUSH rule 1 x 535 [89,565,397,693,839,632,859,30]
+  CRUSH rule 1 x 536 [499,351,760,458,918,86,148,668]
+  CRUSH rule 1 x 537 [676,547,787,311,867,748,152,797]
+  CRUSH rule 1 x 538 [58,644,571,649,941,7,37,485]
+  CRUSH rule 1 x 539 [837,953,457,711,458,621,528,722]
+  CRUSH rule 1 x 540 [831,50,132,213,197,709,95,789]
+  CRUSH rule 1 x 541 [582,757,121,525,532,963,738,277]
+  CRUSH rule 1 x 542 [472,132,790,997,948,269,137,934]
+  CRUSH rule 1 x 543 [382,272,797,330,315,748,324,134]
+  CRUSH rule 1 x 544 [947,930,496,883,509,219,250,362]
+  CRUSH rule 1 x 545 [425,570,305,77,821,422,117,172]
+  CRUSH rule 1 x 546 [18,65,529,437,343,547,699,610]
+  CRUSH rule 1 x 547 [445,715,600,472,213,851,428,267]
+  CRUSH rule 1 x 548 [367,569,980,167,627,442,517,684]
+  CRUSH rule 1 x 549 [125,715,671,817,285,420,37,639]
+  CRUSH rule 1 x 550 [425,599,744,199,923,222,915,570]
+  CRUSH rule 1 x 551 [44,1,528,922,944,115,161,901]
+  CRUSH rule 1 x 552 [246,104,68,239,123,427,57,217]
+  CRUSH rule 1 x 553 [71,703,615,28,593,724,218,916]
+  CRUSH rule 1 x 554 [207,124,217,166,525,226,693,953]
+  CRUSH rule 1 x 555 [570,28,317,420,931,413,623,659]
+  CRUSH rule 1 x 556 [674,152,421,79,215,347,830,762]
+  CRUSH rule 1 x 557 [347,817,191,391,741,571,593,267]
+  CRUSH rule 1 x 558 [627,426,369,692,815,371,124,107]
+  CRUSH rule 1 x 559 [940,630,924,242,224,912,185,356]
+  CRUSH rule 1 x 560 [295,903,541,29,245,753,887,376]
+  CRUSH rule 1 x 561 [506,682,384,637,878,991,700,339]
+  CRUSH rule 1 x 562 [718,529,87,729,842,341,62,817]
+  CRUSH rule 1 x 563 [552,332,747,206,274,871,903,900]
+  CRUSH rule 1 x 564 [835,769,736,486,630,209,641,751]
+  CRUSH rule 1 x 565 [8,167,539,182,607,62,738,873]
+  CRUSH rule 1 x 566 [600,481,301,263,90,450,184,127]
+  CRUSH rule 1 x 567 [999,994,509,899,947,24,267,639]
+  CRUSH rule 1 x 568 [252,431,157,62,601,863,398,521]
+  CRUSH rule 1 x 569 [643,218,943,455,83,969,494,624]
+  CRUSH rule 1 x 570 [617,635,765,422,250,156,533,674]
+  CRUSH rule 1 x 571 [757,80,59,98,328,700,329,848]
+  CRUSH rule 1 x 572 [299,348,575,889,943,675,33,312]
+  CRUSH rule 1 x 573 [25,505,270,167,58,901,878,978]
+  CRUSH rule 1 x 574 [215,431,624,177,628,814,333,841]
+  CRUSH rule 1 x 575 [225,252,611,546,32,815,389,486]
+  CRUSH rule 1 x 576 [627,94,159,857,430,691,177,545]
+  CRUSH rule 1 x 577 [237,809,778,636,61,167,700,521]
+  CRUSH rule 1 x 578 [885,313,120,344,771,614,487,976]
+  CRUSH rule 1 x 579 [924,575,787,831,47,996,557,630]
+  CRUSH rule 1 x 580 [718,51,766,121,118,471,608,755]
+  CRUSH rule 1 x 581 [219,807,129,571,856,179,874,902]
+  CRUSH rule 1 x 582 [893,701,598,863,285,829,984,622]
+  CRUSH rule 1 x 583 [246,930,964,170,993,409,469,193]
+  CRUSH rule 1 x 584 [336,432,680,175,495,839,642,226]
+  CRUSH rule 1 x 585 [324,999,397,485,457,527,73,628]
+  CRUSH rule 1 x 586 [558,230,976,541,816,72,794,682]
+  CRUSH rule 1 x 587 [985,830,597,21,308,890,952,421]
+  CRUSH rule 1 x 588 [211,544,57,134,162,496,195,581]
+  CRUSH rule 1 x 589 [129,21,112,190,885,844,753,180]
+  CRUSH rule 1 x 590 [467,969,652,593,287,76,811,413]
+  CRUSH rule 1 x 591 [758,514,316,164,35,110,54,796]
+  CRUSH rule 1 x 592 [525,253,190,443,315,603,667,318]
+  CRUSH rule 1 x 593 [601,885,339,152,297,223,269,455]
+  CRUSH rule 1 x 594 [227,60,450,30,717,840,994,16]
+  CRUSH rule 1 x 595 [720,854,496,912,80,655,917,525]
+  CRUSH rule 1 x 596 [751,195,997,77,261,490,180,482]
+  CRUSH rule 1 x 597 [129,574,714,8,789,847,725,991]
+  CRUSH rule 1 x 598 [679,207,604,396,841,284,286,280]
+  CRUSH rule 1 x 599 [668,315,683,349,681,253,599,364]
+  CRUSH rule 1 x 600 [143,396,464,444,59,57,243,264]
+  CRUSH rule 1 x 601 [326,573,873,902,136,921,633,596]
+  CRUSH rule 1 x 602 [860,281,875,535,672,474,697,763]
+  CRUSH rule 1 x 603 [709,328,445,349,190,455,924,667]
+  CRUSH rule 1 x 604 [571,62,814,95,866,978,983,281]
+  CRUSH rule 1 x 605 [252,739,860,27,313,362,857,899]
+  CRUSH rule 1 x 606 [339,236,759,842,67,644,954,94]
+  CRUSH rule 1 x 607 [590,248,759,868,433,398,578,386]
+  CRUSH rule 1 x 608 [145,635,309,467,875,115,148,33]
+  CRUSH rule 1 x 609 [973,547,223,79,762,863,249,41]
+  CRUSH rule 1 x 610 [435,816,961,983,255,886,160,888]
+  CRUSH rule 1 x 611 [559,283,422,584,176,429,570,43]
+  CRUSH rule 1 x 612 [273,149,123,576,911,270,296,735]
+  CRUSH rule 1 x 613 [828,614,642,674,33,361,958,580]
+  CRUSH rule 1 x 614 [478,748,393,34,171,80,92,12]
+  CRUSH rule 1 x 615 [392,155,144,326,626,134,149,401]
+  CRUSH rule 1 x 616 [778,637,452,248,15,888,74,307]
+  CRUSH rule 1 x 617 [622,713,996,833,611,407,364,8]
+  CRUSH rule 1 x 618 [149,877,270,329,180,327,222,749]
+  CRUSH rule 1 x 619 [604,163,656,409,322,848,519,967]
+  CRUSH rule 1 x 620 [181,23,409,198,64,898,35,620]
+  CRUSH rule 1 x 621 [735,902,386,237,939,475,725,118]
+  CRUSH rule 1 x 622 [661,824,717,568,858,583,446,798]
+  CRUSH rule 1 x 623 [142,121,643,61,695,852,485,478]
+  CRUSH rule 1 x 624 [360,716,420,398,49,717,137,140]
+  CRUSH rule 1 x 625 [541,167,385,1,601,481,308,111]
+  CRUSH rule 1 x 626 [364,431,610,363,535,747,225,841]
+  CRUSH rule 1 x 627 [458,137,557,410,287,749,467,432]
+  CRUSH rule 1 x 628 [250,350,556,497,821,65,205,580]
+  CRUSH rule 1 x 629 [928,160,710,572,365,772,538,46]
+  CRUSH rule 1 x 630 [243,19,918,556,601,16,920,830]
+  CRUSH rule 1 x 631 [438,221,574,676,797,580,219,211]
+  CRUSH rule 1 x 632 [797,368,247,5,32,102,416,45]
+  CRUSH rule 1 x 633 [993,749,525,485,27,330,275,599]
+  CRUSH rule 1 x 634 [239,351,633,299,651,678,296,337]
+  CRUSH rule 1 x 635 [640,965,25,961,306,172,849,357]
+  CRUSH rule 1 x 636 [173,290,297,991,937,823,236,318]
+  CRUSH rule 1 x 637 [0,918,98,108,111,495,887,57]
+  CRUSH rule 1 x 638 [702,235,424,900,983,754,701,887]
+  CRUSH rule 1 x 639 [475,687,31,785,918,611,27,214]
+  CRUSH rule 1 x 640 [31,664,399,677,123,609,858,138]
+  CRUSH rule 1 x 641 [296,473,108,963,341,876,897,449]
+  CRUSH rule 1 x 642 [894,273,427,606,677,670,610,665]
+  CRUSH rule 1 x 643 [117,111,732,191,114,153,500,631]
+  CRUSH rule 1 x 644 [438,336,327,512,599,862,660,857]
+  CRUSH rule 1 x 645 [982,702,351,573,907,915,279,317]
+  CRUSH rule 1 x 646 [334,804,146,842,697,638,720,135]
+  CRUSH rule 1 x 647 [933,787,185,334,752,285,372,890]
+  CRUSH rule 1 x 648 [22,444,400,862,207,842,453,732]
+  CRUSH rule 1 x 649 [503,229,213,460,639,760,722,748]
+  CRUSH rule 1 x 650 [328,659,420,443,739,950,869,150]
+  CRUSH rule 1 x 651 [3,880,823,123,378,585,715,221]
+  CRUSH rule 1 x 652 [495,977,563,733,92,997,119,818]
+  CRUSH rule 1 x 653 [185,718,804,280,975,912,198,291]
+  CRUSH rule 1 x 654 [130,528,380,81,906,511,750,506]
+  CRUSH rule 1 x 655 [560,872,454,504,319,284,605,214]
+  CRUSH rule 1 x 656 [219,885,178,981,863,508,708,6]
+  CRUSH rule 1 x 657 [233,684,813,490,208,941,858,16]
+  CRUSH rule 1 x 658 [778,6,756,380,750,836,547,850]
+  CRUSH rule 1 x 659 [240,663,306,540,789,902,170,954]
+  CRUSH rule 1 x 660 [244,855,196,147,678,323,63,859]
+  CRUSH rule 1 x 661 [184,270,128,398,910,230,402,205]
+  CRUSH rule 1 x 662 [65,883,921,438,79,957,464,902]
+  CRUSH rule 1 x 663 [323,721,594,812,43,992,170,65]
+  CRUSH rule 1 x 664 [865,113,512,51,427,123,585,260]
+  CRUSH rule 1 x 665 [420,850,591,475,202,733,798,658]
+  CRUSH rule 1 x 666 [319,767,246,3,369,493,796,56]
+  CRUSH rule 1 x 667 [875,39,343,100,829,2,795,783]
+  CRUSH rule 1 x 668 [331,122,263,599,355,484,943,554]
+  CRUSH rule 1 x 669 [915,521,402,747,673,445,938,600]
+  CRUSH rule 1 x 670 [845,659,943,447,401,322,168,302]
+  CRUSH rule 1 x 671 [108,634,527,363,856,238,755,330]
+  CRUSH rule 1 x 672 [578,216,110,589,302,137,954,315]
+  CRUSH rule 1 x 673 [442,74,579,797,622,950,371,402]
+  CRUSH rule 1 x 674 [588,364,281,308,645,631,229,506]
+  CRUSH rule 1 x 675 [489,698,744,671,870,174,528,875]
+  CRUSH rule 1 x 676 [928,911,40,180,722,729,673,569]
+  CRUSH rule 1 x 677 [399,269,692,131,615,136,103,763]
+  CRUSH rule 1 x 678 [546,752,544,155,5,463,666,352]
+  CRUSH rule 1 x 679 [988,25,275,433,628,57,247,620]
+  CRUSH rule 1 x 680 [335,963,382,486,749,257,795,347]
+  CRUSH rule 1 x 681 [690,462,623,466,49,471,774,192]
+  CRUSH rule 1 x 682 [196,588,154,257,807,776,367,718]
+  CRUSH rule 1 x 683 [627,25,421,160,873,102,345,599]
+  CRUSH rule 1 x 684 [38,804,592,158,991,264,652,821]
+  CRUSH rule 1 x 685 [841,368,548,362,166,211,154,121]
+  CRUSH rule 1 x 686 [336,287,525,440,166,993,911,638]
+  CRUSH rule 1 x 687 [20,682,924,653,356,16,917,622]
+  CRUSH rule 1 x 688 [463,371,780,556,385,883,115,248]
+  CRUSH rule 1 x 689 [569,250,78,816,847,775,333,161]
+  CRUSH rule 1 x 690 [551,144,587,263,378,394,970,639]
+  CRUSH rule 1 x 691 [766,464,446,533,449,541,451,290]
+  CRUSH rule 1 x 692 [739,634,18,245,624,35,268,525]
+  CRUSH rule 1 x 693 [339,297,118,330,817,91,828,276]
+  CRUSH rule 1 x 694 [405,26,830,181,533,166,488,804]
+  CRUSH rule 1 x 695 [622,576,597,535,600,593,300,989]
+  CRUSH rule 1 x 696 [558,902,689,13,715,28,664,489]
+  CRUSH rule 1 x 697 [818,222,406,691,427,863,153,922]
+  CRUSH rule 1 x 698 [178,48,402,233,841,604,468,180]
+  CRUSH rule 1 x 699 [450,244,180,919,426,332,747,453]
+  CRUSH rule 1 x 700 [502,771,987,706,416,240,68,641]
+  CRUSH rule 1 x 701 [4,612,782,216,853,303,585,513]
+  CRUSH rule 1 x 702 [177,630,232,923,281,708,466,687]
+  CRUSH rule 1 x 703 [354,178,389,393,778,803,796,607]
+  CRUSH rule 1 x 704 [646,601,156,171,603,116,655,595]
+  CRUSH rule 1 x 705 [921,401,890,265,244,690,372,253]
+  CRUSH rule 1 x 706 [652,877,562,452,26,323,923,770]
+  CRUSH rule 1 x 707 [345,745,67,716,789,576,2,133]
+  CRUSH rule 1 x 708 [333,607,180,469,170,555,939,331]
+  CRUSH rule 1 x 709 [45,187,302,115,896,579,733,607]
+  CRUSH rule 1 x 710 [94,855,43,199,18,948,449,28]
+  CRUSH rule 1 x 711 [227,653,731,150,278,842,534,110]
+  CRUSH rule 1 x 712 [398,953,136,870,181,408,895,459]
+  CRUSH rule 1 x 713 [116,800,503,662,635,579,53,839]
+  CRUSH rule 1 x 714 [111,629,866,709,902,557,875,649]
+  CRUSH rule 1 x 715 [531,291,486,382,192,807,322,417]
+  CRUSH rule 1 x 716 [169,541,291,42,343,724,138,197]
+  CRUSH rule 1 x 717 [417,446,994,894,239,494,237,62]
+  CRUSH rule 1 x 718 [992,383,298,844,377,463,544,891]
+  CRUSH rule 1 x 719 [936,674,324,759,194,409,828,975]
+  CRUSH rule 1 x 720 [370,188,174,464,644,218,214,76]
+  CRUSH rule 1 x 721 [320,859,278,259,170,957,177,264]
+  CRUSH rule 1 x 722 [7,2,673,129,96,445,823,833]
+  CRUSH rule 1 x 723 [270,553,831,662,38,101,985,846]
+  CRUSH rule 1 x 724 [666,822,708,895,633,800,616,879]
+  CRUSH rule 1 x 725 [794,406,875,459,981,751,359,926]
+  CRUSH rule 1 x 726 [420,556,341,292,240,68,966,535]
+  CRUSH rule 1 x 727 [561,461,129,635,965,610,105,31]
+  CRUSH rule 1 x 728 [951,330,196,756,589,849,753,760]
+  CRUSH rule 1 x 729 [656,644,436,591,27,119,572,933]
+  CRUSH rule 1 x 730 [3,558,629,184,50,765,760,800]
+  CRUSH rule 1 x 731 [852,89,75,735,713,113,528,890]
+  CRUSH rule 1 x 732 [983,840,869,976,697,307,368,271]
+  CRUSH rule 1 x 733 [285,396,388,122,387,364,880,343]
+  CRUSH rule 1 x 734 [125,510,402,640,676,501,535,627]
+  CRUSH rule 1 x 735 [417,773,686,504,459,912,690,59]
+  CRUSH rule 1 x 736 [749,396,632,550,779,109,845,278]
+  CRUSH rule 1 x 737 [644,991,946,135,448,903,482,564]
+  CRUSH rule 1 x 738 [449,683,290,220,245,525,429,397]
+  CRUSH rule 1 x 739 [341,220,641,454,740,661,146,17]
+  CRUSH rule 1 x 740 [874,524,674,650,472,282,214,494]
+  CRUSH rule 1 x 741 [189,472,712,798,715,757,863,571]
+  CRUSH rule 1 x 742 [912,581,114,377,730,21,687,81]
+  CRUSH rule 1 x 743 [654,914,425,441,763,39,451,631]
+  CRUSH rule 1 x 744 [725,295,579,377,162,447,843,699]
+  CRUSH rule 1 x 745 [787,858,850,506,612,735,926,314]
+  CRUSH rule 1 x 746 [757,848,704,30,47,940,450,651]
+  CRUSH rule 1 x 747 [700,81,867,681,801,64,879,857]
+  CRUSH rule 1 x 748 [557,436,238,664,293,865,304,999]
+  CRUSH rule 1 x 749 [772,622,337,42,156,302,383,506]
+  CRUSH rule 1 x 750 [946,97,376,677,316,670,169,171]
+  CRUSH rule 1 x 751 [996,618,343,911,83,22,388,17]
+  CRUSH rule 1 x 752 [746,887,695,868,610,950,88,315]
+  CRUSH rule 1 x 753 [741,14,463,479,172,192,481,702]
+  CRUSH rule 1 x 754 [648,349,333,355,65,63,336,724]
+  CRUSH rule 1 x 755 [157,460,466,187,959,674,192,279]
+  CRUSH rule 1 x 756 [416,97,197,497,227,3,850,191]
+  CRUSH rule 1 x 757 [599,839,776,410,256,823,121,690]
+  CRUSH rule 1 x 758 [994,218,620,256,361,749,165,686]
+  CRUSH rule 1 x 759 [959,682,514,745,100,519,15,347]
+  CRUSH rule 1 x 760 [518,943,215,83,706,137,345,69]
+  CRUSH rule 1 x 761 [285,849,420,324,987,338,373,361]
+  CRUSH rule 1 x 762 [591,313,41,335,110,696,664,350]
+  CRUSH rule 1 x 763 [908,411,200,740,292,295,387,775]
+  CRUSH rule 1 x 764 [787,234,894,485,883,711,70,202]
+  CRUSH rule 1 x 765 [327,921,882,393,444,792,402,123]
+  CRUSH rule 1 x 766 [84,161,878,704,416,144,357,310]
+  CRUSH rule 1 x 767 [370,895,702,701,890,2,251,951]
+  CRUSH rule 1 x 768 [826,760,879,864,460,474,645,975]
+  CRUSH rule 1 x 769 [67,768,663,735,814,66,213,527]
+  CRUSH rule 1 x 770 [593,909,482,259,5,550,961,324]
+  CRUSH rule 1 x 771 [309,935,121,578,937,685,933,571]
+  CRUSH rule 1 x 772 [12,125,797,301,348,419,891,959]
+  CRUSH rule 1 x 773 [253,466,820,549,591,193,783,951]
+  CRUSH rule 1 x 774 [164,390,705,109,881,505,890,425]
+  CRUSH rule 1 x 775 [703,47,43,973,643,406,885,976]
+  CRUSH rule 1 x 776 [728,231,80,916,2,850,396,76]
+  CRUSH rule 1 x 777 [981,621,568,729,869,952,563,860]
+  CRUSH rule 1 x 778 [411,456,544,597,789,784,65,954]
+  CRUSH rule 1 x 779 [346,121,519,921,587,48,772,645]
+  CRUSH rule 1 x 780 [476,39,288,381,303,29,17,336]
+  CRUSH rule 1 x 781 [10,130,585,844,729,705,714,954]
+  CRUSH rule 1 x 782 [462,246,581,902,623,877,812,516]
+  CRUSH rule 1 x 783 [580,373,153,775,668,661,626,961]
+  CRUSH rule 1 x 784 [413,113,978,990,994,56,481,198]
+  CRUSH rule 1 x 785 [341,856,332,354,59,581,632,151]
+  CRUSH rule 1 x 786 [411,140,313,393,215,618,490,481]
+  CRUSH rule 1 x 787 [605,522,211,813,636,224,600,528]
+  CRUSH rule 1 x 788 [226,545,35,142,726,851,194,216]
+  CRUSH rule 1 x 789 [545,320,414,702,731,277,237,916]
+  CRUSH rule 1 x 790 [414,748,816,327,130,115,788,164]
+  CRUSH rule 1 x 791 [660,906,406,697,916,322,124,711]
+  CRUSH rule 1 x 792 [287,392,514,204,75,789,406,858]
+  CRUSH rule 1 x 793 [631,133,850,713,720,487,376,812]
+  CRUSH rule 1 x 794 [931,517,543,210,963,898,811,459]
+  CRUSH rule 1 x 795 [551,962,477,948,425,434,268,94]
+  CRUSH rule 1 x 796 [814,4,95,27,368,300,646,451]
+  CRUSH rule 1 x 797 [64,201,299,734,605,864,596,196]
+  CRUSH rule 1 x 798 [422,530,114,431,565,716,473,250]
+  CRUSH rule 1 x 799 [824,32,679,562,266,549,859,994]
+  CRUSH rule 1 x 800 [862,623,489,637,861,196,941,643]
+  CRUSH rule 1 x 801 [145,550,329,324,734,160,219,662]
+  CRUSH rule 1 x 802 [570,19,847,308,387,518,846,53]
+  CRUSH rule 1 x 803 [151,812,662,358,880,349,834,881]
+  CRUSH rule 1 x 804 [467,93,264,863,176,842,663,949]
+  CRUSH rule 1 x 805 [621,223,938,809,591,686,121,157]
+  CRUSH rule 1 x 806 [898,957,805,430,499,584,640,607]
+  CRUSH rule 1 x 807 [354,531,422,159,921,431,802,136]
+  CRUSH rule 1 x 808 [7,96,76,897,446,2,166,929]
+  CRUSH rule 1 x 809 [70,734,719,56,687,21,23,145]
+  CRUSH rule 1 x 810 [701,18,972,327,771,649,620,648]
+  CRUSH rule 1 x 811 [248,547,103,728,901,264,948,202]
+  CRUSH rule 1 x 812 [230,576,821,566,993,762,675,28]
+  CRUSH rule 1 x 813 [805,114,683,629,396,462,285,450]
+  CRUSH rule 1 x 814 [54,619,973,741,497,894,401,266]
+  CRUSH rule 1 x 815 [679,412,613,132,969,411,314,670]
+  CRUSH rule 1 x 816 [919,448,826,414,36,289,44,822]
+  CRUSH rule 1 x 817 [765,830,436,521,332,458,260,172]
+  CRUSH rule 1 x 818 [415,566,644,687,692,414,769,826]
+  CRUSH rule 1 x 819 [721,319,865,750,546,859,523,770]
+  CRUSH rule 1 x 820 [218,301,333,190,686,179,535,787]
+  CRUSH rule 1 x 821 [185,795,680,953,329,750,621,815]
+  CRUSH rule 1 x 822 [356,261,54,522,900,103,883,112]
+  CRUSH rule 1 x 823 [220,281,549,456,64,306,282,641]
+  CRUSH rule 1 x 824 [292,809,887,74,776,788,559,886]
+  CRUSH rule 1 x 825 [949,778,101,311,110,480,161,998]
+  CRUSH rule 1 x 826 [767,818,833,927,356,954,910,63]
+  CRUSH rule 1 x 827 [631,83,406,635,657,713,212,916]
+  CRUSH rule 1 x 828 [288,986,445,26,414,607,937,595]
+  CRUSH rule 1 x 829 [990,667,915,694,974,453,669,330]
+  CRUSH rule 1 x 830 [152,571,778,505,685,209,448,55]
+  CRUSH rule 1 x 831 [814,563,630,97,582,107,142,157]
+  CRUSH rule 1 x 832 [235,641,616,110,979,844,656,135]
+  CRUSH rule 1 x 833 [657,565,922,140,825,457,764,766]
+  CRUSH rule 1 x 834 [907,231,644,13,617,130,83,483]
+  CRUSH rule 1 x 835 [784,262,771,264,612,238,537,937]
+  CRUSH rule 1 x 836 [951,158,366,710,43,427,351,961]
+  CRUSH rule 1 x 837 [556,498,334,633,895,627,903,29]
+  CRUSH rule 1 x 838 [329,274,964,547,119,342,983,998]
+  CRUSH rule 1 x 839 [568,209,939,364,658,747,47,859]
+  CRUSH rule 1 x 840 [45,579,842,70,655,862,815,109]
+  CRUSH rule 1 x 841 [652,702,24,605,152,93,226,46]
+  CRUSH rule 1 x 842 [629,984,314,895,408,897,575,1]
+  CRUSH rule 1 x 843 [799,690,688,648,151,812,486,199]
+  CRUSH rule 1 x 844 [694,600,534,700,569,11,899,382]
+  CRUSH rule 1 x 845 [332,30,179,93,951,324,611,512]
+  CRUSH rule 1 x 846 [452,251,712,719,404,739,606,237]
+  CRUSH rule 1 x 847 [399,681,847,739,13,555,363,893]
+  CRUSH rule 1 x 848 [303,138,440,346,547,216,700,249]
+  CRUSH rule 1 x 849 [666,346,708,873,64,694,847,463]
+  CRUSH rule 1 x 850 [644,511,345,844,545,337,358,35]
+  CRUSH rule 1 x 851 [527,546,737,425,100,331,95,337]
+  CRUSH rule 1 x 852 [31,809,94,618,156,853,469,511]
+  CRUSH rule 1 x 853 [483,330,869,184,46,942,774,679]
+  CRUSH rule 1 x 854 [697,953,968,143,502,955,441,302]
+  CRUSH rule 1 x 855 [837,996,239,621,32,191,686,702]
+  CRUSH rule 1 x 856 [712,40,547,430,195,857,224,810]
+  CRUSH rule 1 x 857 [77,984,576,551,568,96,12,763]
+  CRUSH rule 1 x 858 [412,384,841,465,572,576,688,61]
+  CRUSH rule 1 x 859 [173,760,26,300,87,567,463,903]
+  CRUSH rule 1 x 860 [776,429,328,917,658,783,699,907]
+  CRUSH rule 1 x 861 [705,405,477,50,73,714,901,487]
+  CRUSH rule 1 x 862 [809,44,788,938,964,177,490,409]
+  CRUSH rule 1 x 863 [349,496,963,178,675,853,172,980]
+  CRUSH rule 1 x 864 [717,858,101,239,992,244,43,15]
+  CRUSH rule 1 x 865 [857,603,586,262,550,289,850,40]
+  CRUSH rule 1 x 866 [394,304,71,96,642,155,255,481]
+  CRUSH rule 1 x 867 [640,773,663,974,261,296,988,730]
+  CRUSH rule 1 x 868 [613,950,712,663,101,460,643,547]
+  CRUSH rule 1 x 869 [973,889,524,22,671,477,718,431]
+  CRUSH rule 1 x 870 [505,35,386,498,348,503,54,992]
+  CRUSH rule 1 x 871 [239,264,262,773,781,734,387,515]
+  CRUSH rule 1 x 872 [21,767,456,748,783,797,180,800]
+  CRUSH rule 1 x 873 [954,666,980,264,435,233,199,358]
+  CRUSH rule 1 x 874 [54,510,947,1,500,119,93,915]
+  CRUSH rule 1 x 875 [809,418,452,462,88,673,634,435]
+  CRUSH rule 1 x 876 [483,457,61,248,523,277,322,141]
+  CRUSH rule 1 x 877 [542,531,952,939,710,179,181,460]
+  CRUSH rule 1 x 878 [217,674,857,644,678,809,329,591]
+  CRUSH rule 1 x 879 [999,475,134,250,319,357,145,750]
+  CRUSH rule 1 x 880 [678,573,935,385,570,651,319,630]
+  CRUSH rule 1 x 881 [394,835,789,802,587,155,570,109]
+  CRUSH rule 1 x 882 [467,382,353,56,979,674,974,483]
+  CRUSH rule 1 x 883 [802,744,237,337,50,96,202,148]
+  CRUSH rule 1 x 884 [653,660,638,700,31,558,389,381]
+  CRUSH rule 1 x 885 [898,704,307,445,879,872,174,972]
+  CRUSH rule 1 x 886 [434,357,938,641,737,8,56,582]
+  CRUSH rule 1 x 887 [297,226,711,428,370,318,472,947]
+  CRUSH rule 1 x 888 [863,324,443,213,902,25,806,53]
+  CRUSH rule 1 x 889 [105,102,308,163,947,548,399,382]
+  CRUSH rule 1 x 890 [550,248,606,704,615,708,996,561]
+  CRUSH rule 1 x 891 [575,928,880,891,826,763,706,701]
+  CRUSH rule 1 x 892 [259,862,133,271,292,162,53,333]
+  CRUSH rule 1 x 893 [902,880,543,542,37,942,672,320]
+  CRUSH rule 1 x 894 [180,169,916,43,945,713,648,685]
+  CRUSH rule 1 x 895 [725,849,182,129,177,272,599,829]
+  CRUSH rule 1 x 896 [951,34,874,537,969,123,210,529]
+  CRUSH rule 1 x 897 [810,352,73,939,943,895,12,481]
+  CRUSH rule 1 x 898 [979,433,719,411,787,359,342,37]
+  CRUSH rule 1 x 899 [685,668,534,932,399,156,124,653]
+  CRUSH rule 1 x 900 [530,978,41,894,941,681,380,419]
+  CRUSH rule 1 x 901 [740,107,336,175,574,706,157,292]
+  CRUSH rule 1 x 902 [800,743,693,310,67,111,178,624]
+  CRUSH rule 1 x 903 [230,267,842,266,550,769,66,738]
+  CRUSH rule 1 x 904 [346,949,460,973,696,91,957,801]
+  CRUSH rule 1 x 905 [530,397,619,958,576,973,685,6]
+  CRUSH rule 1 x 906 [80,426,138,672,73,776,30,169]
+  CRUSH rule 1 x 907 [365,968,475,297,296,724,664,331]
+  CRUSH rule 1 x 908 [204,832,742,809,862,745,484,391]
+  CRUSH rule 1 x 909 [883,989,146,959,366,59,686,965]
+  CRUSH rule 1 x 910 [549,593,249,853,792,769,824,552]
+  CRUSH rule 1 x 911 [325,847,352,214,851,732,789,255]
+  CRUSH rule 1 x 912 [874,888,582,796,557,601,226,889]
+  CRUSH rule 1 x 913 [331,463,342,574,989,362,925,746]
+  CRUSH rule 1 x 914 [836,468,601,732,607,275,70,280]
+  CRUSH rule 1 x 915 [245,228,100,661,799,13,126,79]
+  CRUSH rule 1 x 916 [77,967,364,435,27,474,255,133]
+  CRUSH rule 1 x 917 [239,60,866,221,772,967,725,707]
+  CRUSH rule 1 x 918 [988,115,922,80,201,544,583,923]
+  CRUSH rule 1 x 919 [783,139,696,1,848,169,888,980]
+  CRUSH rule 1 x 920 [623,408,685,953,974,696,532,124]
+  CRUSH rule 1 x 921 [105,799,144,90,399,373,633,290]
+  CRUSH rule 1 x 922 [887,505,652,348,514,806,952,474]
+  CRUSH rule 1 x 923 [223,318,552,458,743,871,964,384]
+  CRUSH rule 1 x 924 [25,778,366,333,163,801,584,31]
+  CRUSH rule 1 x 925 [912,601,297,682,770,173,969,168]
+  CRUSH rule 1 x 926 [968,133,175,144,814,155,709,158]
+  CRUSH rule 1 x 927 [277,724,214,988,690,342,465,775]
+  CRUSH rule 1 x 928 [554,203,658,789,298,299,847,752]
+  CRUSH rule 1 x 929 [761,802,367,528,758,522,744,171]
+  CRUSH rule 1 x 930 [814,61,788,736,660,491,832,654]
+  CRUSH rule 1 x 931 [29,193,61,41,343,664,487,839]
+  CRUSH rule 1 x 932 [446,198,862,534,168,35,530,462]
+  CRUSH rule 1 x 933 [352,742,216,321,525,44,568,61]
+  CRUSH rule 1 x 934 [730,2,332,631,613,249,533,116]
+  CRUSH rule 1 x 935 [731,23,736,79,361,992,772,49]
+  CRUSH rule 1 x 936 [322,975,20,904,827,603,138,802]
+  CRUSH rule 1 x 937 [822,221,841,161,723,137,630,308]
+  CRUSH rule 1 x 938 [557,850,66,630,499,404,286,395]
+  CRUSH rule 1 x 939 [150,11,971,371,124,785,408,49]
+  CRUSH rule 1 x 940 [638,398,169,616,333,751,25,883]
+  CRUSH rule 1 x 941 [730,342,929,577,451,838,964,28]
+  CRUSH rule 1 x 942 [62,292,166,814,587,172,724,16]
+  CRUSH rule 1 x 943 [165,314,519,548,41,726,759,851]
+  CRUSH rule 1 x 944 [199,625,766,176,194,297,678,915]
+  CRUSH rule 1 x 945 [946,999,699,303,38,81,952,885]
+  CRUSH rule 1 x 946 [595,93,852,142,503,647,933,267]
+  CRUSH rule 1 x 947 [800,582,356,93,716,117,922,868]
+  CRUSH rule 1 x 948 [132,551,139,920,87,46,81,220]
+  CRUSH rule 1 x 949 [792,920,466,380,97,568,799,961]
+  CRUSH rule 1 x 950 [111,345,176,543,879,954,355,220]
+  CRUSH rule 1 x 951 [414,619,648,655,364,971,829,408]
+  CRUSH rule 1 x 952 [775,469,500,356,287,4,16,746]
+  CRUSH rule 1 x 953 [349,1,5,251,168,680,141,619]
+  CRUSH rule 1 x 954 [570,940,410,249,929,394,129,696]
+  CRUSH rule 1 x 955 [729,774,823,800,7,127,536,766]
+  CRUSH rule 1 x 956 [519,141,575,625,738,475,169,751]
+  CRUSH rule 1 x 957 [242,709,611,97,760,309,393,281]
+  CRUSH rule 1 x 958 [84,217,227,253,246,604,346,377]
+  CRUSH rule 1 x 959 [270,413,918,789,703,608,543,519]
+  CRUSH rule 1 x 960 [458,192,307,279,920,139,855,49]
+  CRUSH rule 1 x 961 [981,388,777,546,359,660,455,708]
+  CRUSH rule 1 x 962 [623,834,277,134,729,246,856,477]
+  CRUSH rule 1 x 963 [291,167,714,468,109,373,485,701]
+  CRUSH rule 1 x 964 [28,156,788,127,598,215,361,255]
+  CRUSH rule 1 x 965 [675,557,290,517,840,510,59,229]
+  CRUSH rule 1 x 966 [836,306,946,283,642,606,929,773]
+  CRUSH rule 1 x 967 [966,386,735,837,392,116,19,674]
+  CRUSH rule 1 x 968 [864,756,690,121,328,122,433,520]
+  CRUSH rule 1 x 969 [729,625,480,769,512,882,518,956]
+  CRUSH rule 1 x 970 [800,362,646,582,309,102,576,411]
+  CRUSH rule 1 x 971 [737,381,153,684,298,166,344,520]
+  CRUSH rule 1 x 972 [952,245,720,884,334,311,754,540]
+  CRUSH rule 1 x 973 [356,455,579,857,832,596,549,524]
+  CRUSH rule 1 x 974 [545,758,586,596,46,790,116,993]
+  CRUSH rule 1 x 975 [336,191,202,146,720,897,330,308]
+  CRUSH rule 1 x 976 [446,208,757,620,252,846,397,58]
+  CRUSH rule 1 x 977 [202,896,196,956,763,126,783,828]
+  CRUSH rule 1 x 978 [612,324,996,225,418,583,514,169]
+  CRUSH rule 1 x 979 [843,457,675,650,958,657,677,173]
+  CRUSH rule 1 x 980 [60,914,881,626,850,759,398,943]
+  CRUSH rule 1 x 981 [702,749,937,153,724,514,536,212]
+  CRUSH rule 1 x 982 [298,928,738,167,99,668,395,198]
+  CRUSH rule 1 x 983 [723,572,395,358,900,37,927,597]
+  CRUSH rule 1 x 984 [723,864,804,935,846,993,950,840]
+  CRUSH rule 1 x 985 [945,459,868,211,524,954,911,208]
+  CRUSH rule 1 x 986 [772,664,535,169,297,996,864,555]
+  CRUSH rule 1 x 987 [88,324,312,843,661,580,76,894]
+  CRUSH rule 1 x 988 [522,927,131,996,351,685,865,47]
+  CRUSH rule 1 x 989 [578,332,208,605,975,207,155,380]
+  CRUSH rule 1 x 990 [638,228,414,311,738,698,340,526]
+  CRUSH rule 1 x 991 [530,221,451,422,879,916,754,928]
+  CRUSH rule 1 x 992 [925,705,275,81,234,310,117,546]
+  CRUSH rule 1 x 993 [991,301,43,469,830,242,382,428]
+  CRUSH rule 1 x 994 [276,51,868,683,843,815,557,378]
+  CRUSH rule 1 x 995 [288,836,753,790,758,120,158,265]
+  CRUSH rule 1 x 996 [887,983,252,686,470,345,459,764]
+  CRUSH rule 1 x 997 [110,924,386,79,705,697,210,698]
+  CRUSH rule 1 x 998 [435,830,485,853,926,730,786,762]
+  CRUSH rule 1 x 999 [876,738,357,913,723,51,15,585]
+  CRUSH rule 1 x 1000 [178,963,638,430,845,586,317,102]
+  CRUSH rule 1 x 1001 [99,519,66,759,583,944,739,922]
+  CRUSH rule 1 x 1002 [515,534,468,866,878,717,729,370]
+  CRUSH rule 1 x 1003 [104,611,937,698,94,67,614,783]
+  CRUSH rule 1 x 1004 [269,638,724,375,491,121,891,113]
+  CRUSH rule 1 x 1005 [369,223,309,409,822,39,597,969]
+  CRUSH rule 1 x 1006 [40,107,69,275,79,429,234,945]
+  CRUSH rule 1 x 1007 [978,111,416,758,454,640,5,444]
+  CRUSH rule 1 x 1008 [965,956,624,832,421,96,975,723]
+  CRUSH rule 1 x 1009 [598,476,356,695,919,566,234,383]
+  CRUSH rule 1 x 1010 [767,523,239,517,29,77,23,241]
+  CRUSH rule 1 x 1011 [289,871,207,576,347,698,48,570]
+  CRUSH rule 1 x 1012 [128,28,370,31,341,755,268,647]
+  CRUSH rule 1 x 1013 [979,765,660,812,666,187,808,351]
+  CRUSH rule 1 x 1014 [979,948,513,88,47,825,969,81]
+  CRUSH rule 1 x 1015 [277,790,396,672,542,647,145,11]
+  CRUSH rule 1 x 1016 [262,73,128,886,839,685,456,560]
+  CRUSH rule 1 x 1017 [150,269,61,499,832,591,637,731]
+  CRUSH rule 1 x 1018 [555,829,554,944,406,576,463,926]
+  CRUSH rule 1 x 1019 [513,356,265,446,65,288,768,245]
+  CRUSH rule 1 x 1020 [158,161,877,704,948,570,495,865]
+  CRUSH rule 1 x 1021 [915,998,957,285,546,202,676,322]
+  CRUSH rule 1 x 1022 [967,829,973,640,703,470,871,828]
+  CRUSH rule 1 x 1023 [488,257,614,859,325,419,50,560]
+  rule 1 (metadata) num_rep 8 result size == 8:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536,450,604,380,966,750,695]
+  CRUSH rule 1 x 1 [876,250,334,633,744,843,672,820,782]
+  CRUSH rule 1 x 2 [292,832,53,392,386,787,527,901,106]
+  CRUSH rule 1 x 3 [623,387,124,998,749,211,481,169,816]
+  CRUSH rule 1 x 4 [61,334,710,4,994,982,847,220,87]
+  CRUSH rule 1 x 5 [946,557,713,664,141,817,964,872,66]
+  CRUSH rule 1 x 6 [576,668,212,163,732,381,884,726,456]
+  CRUSH rule 1 x 7 [645,753,906,393,341,44,578,14,543]
+  CRUSH rule 1 x 8 [243,6,863,781,211,100,462,207,759]
+  CRUSH rule 1 x 9 [22,578,251,410,297,430,3,569,603]
+  CRUSH rule 1 x 10 [758,828,360,477,821,801,811,484,296]
+  CRUSH rule 1 x 11 [769,120,124,527,119,504,380,821,470]
+  CRUSH rule 1 x 12 [780,364,689,755,675,199,117,393,435]
+  CRUSH rule 1 x 13 [557,18,351,719,742,780,78,170,333]
+  CRUSH rule 1 x 14 [59,561,249,461,971,835,855,76,269]
+  CRUSH rule 1 x 15 [718,928,993,21,76,313,437,462,680]
+  CRUSH rule 1 x 16 [673,632,841,954,788,90,786,969,378]
+  CRUSH rule 1 x 17 [648,43,560,514,142,289,935,605,228]
+  CRUSH rule 1 x 18 [654,219,181,568,381,253,883,394,188]
+  CRUSH rule 1 x 19 [850,545,377,848,863,543,51,834,690]
+  CRUSH rule 1 x 20 [717,785,974,5,225,552,975,636,387]
+  CRUSH rule 1 x 21 [420,57,519,306,312,983,263,267,128]
+  CRUSH rule 1 x 22 [503,998,193,821,634,684,557,633,812]
+  CRUSH rule 1 x 23 [411,663,168,110,899,488,477,468,303]
+  CRUSH rule 1 x 24 [266,861,353,1,456,128,800,309,622]
+  CRUSH rule 1 x 25 [760,483,818,600,509,951,248,908,624]
+  CRUSH rule 1 x 26 [903,24,573,718,112,694,501,909,219]
+  CRUSH rule 1 x 27 [946,188,289,510,687,827,676,560,753]
+  CRUSH rule 1 x 28 [69,312,73,198,256,629,770,569,359]
+  CRUSH rule 1 x 29 [844,883,337,628,496,405,719,581,816]
+  CRUSH rule 1 x 30 [621,18,613,794,910,936,426,522,208]
+  CRUSH rule 1 x 31 [784,943,814,539,962,392,813,217,750]
+  CRUSH rule 1 x 32 [173,374,369,972,315,83,428,63,801]
+  CRUSH rule 1 x 33 [698,336,357,966,582,407,618,288,846]
+  CRUSH rule 1 x 34 [168,836,210,798,904,190,663,877,177]
+  CRUSH rule 1 x 35 [274,509,534,818,912,671,75,580,568]
+  CRUSH rule 1 x 36 [318,215,153,628,87,407,676,524,510]
+  CRUSH rule 1 x 37 [173,604,109,935,203,401,311,758,201]
+  CRUSH rule 1 x 38 [708,444,683,604,722,900,929,910,698]
+  CRUSH rule 1 x 39 [662,198,417,680,226,342,856,248,279]
+  CRUSH rule 1 x 40 [620,801,414,78,560,766,980,503,287]
+  CRUSH rule 1 x 41 [811,264,177,127,148,791,930,74,844]
+  CRUSH rule 1 x 42 [863,179,527,660,133,529,456,713,348]
+  CRUSH rule 1 x 43 [686,822,988,228,791,549,514,40,261]
+  CRUSH rule 1 x 44 [396,222,46,841,536,140,160,527,250]
+  CRUSH rule 1 x 45 [991,694,253,142,54,422,658,876,201]
+  CRUSH rule 1 x 46 [420,909,184,285,508,458,45,390,546]
+  CRUSH rule 1 x 47 [467,211,605,207,241,881,959,800,743]
+  CRUSH rule 1 x 48 [955,329,368,168,698,787,738,47,812]
+  CRUSH rule 1 x 49 [974,891,931,29,813,506,822,628,696]
+  CRUSH rule 1 x 50 [870,441,691,823,761,6,83,344,713]
+  CRUSH rule 1 x 51 [182,930,25,936,97,260,406,281,991]
+  CRUSH rule 1 x 52 [704,812,894,794,481,37,304,899,629]
+  CRUSH rule 1 x 53 [185,713,631,280,345,558,882,503,327]
+  CRUSH rule 1 x 54 [270,441,100,82,983,930,339,902,81]
+  CRUSH rule 1 x 55 [895,734,958,793,651,572,508,763,108]
+  CRUSH rule 1 x 56 [564,963,683,324,40,189,77,500,553]
+  CRUSH rule 1 x 57 [738,130,208,973,498,861,670,67,114]
+  CRUSH rule 1 x 58 [524,113,806,903,531,334,8,762,842]
+  CRUSH rule 1 x 59 [408,337,668,529,34,384,643,511,370]
+  CRUSH rule 1 x 60 [228,790,857,309,616,895,194,277,985]
+  CRUSH rule 1 x 61 [154,843,717,467,883,536,812,14,55]
+  CRUSH rule 1 x 62 [594,811,549,276,693,917,45,723,926]
+  CRUSH rule 1 x 63 [646,67,884,925,941,434,705,268,140]
+  CRUSH rule 1 x 64 [175,542,155,837,594,197,451,891,654]
+  CRUSH rule 1 x 65 [745,619,131,867,269,62,862,221,66]
+  CRUSH rule 1 x 66 [275,468,23,35,328,432,334,656,719]
+  CRUSH rule 1 x 67 [246,958,524,493,636,227,783,593,814]
+  CRUSH rule 1 x 68 [711,473,403,228,835,126,705,114,981]
+  CRUSH rule 1 x 69 [493,924,850,939,950,105,871,361,533]
+  CRUSH rule 1 x 70 [30,499,644,33,804,654,684,411,114]
+  CRUSH rule 1 x 71 [984,883,574,716,575,391,587,264,446]
+  CRUSH rule 1 x 72 [71,286,942,363,628,632,642,529,966]
+  CRUSH rule 1 x 73 [922,618,3,371,464,442,835,705,260]
+  CRUSH rule 1 x 74 [629,414,185,573,678,338,633,560,565]
+  CRUSH rule 1 x 75 [222,20,174,820,312,361,366,258,711]
+  CRUSH rule 1 x 76 [262,366,339,290,718,143,735,953,188]
+  CRUSH rule 1 x 77 [638,469,992,280,773,892,197,690,426]
+  CRUSH rule 1 x 78 [324,511,788,7,308,228,183,917,464]
+  CRUSH rule 1 x 79 [577,990,64,94,447,924,339,24,581]
+  CRUSH rule 1 x 80 [501,95,278,903,631,842,51,766,822]
+  CRUSH rule 1 x 81 [506,812,9,698,173,664,247,963,0]
+  CRUSH rule 1 x 82 [222,145,80,785,835,745,580,51,939]
+  CRUSH rule 1 x 83 [71,634,61,91,856,529,66,197,698]
+  CRUSH rule 1 x 84 [49,761,773,368,318,708,681,618,723]
+  CRUSH rule 1 x 85 [985,896,708,861,325,307,567,908,514]
+  CRUSH rule 1 x 86 [537,745,93,524,466,356,38,326,385]
+  CRUSH rule 1 x 87 [997,317,463,626,685,162,909,49,28]
+  CRUSH rule 1 x 88 [957,350,890,857,375,176,99,737,942]
+  CRUSH rule 1 x 89 [399,730,148,314,159,982,320,921,812]
+  CRUSH rule 1 x 90 [943,706,683,267,579,141,412,184,529]
+  CRUSH rule 1 x 91 [22,368,149,928,140,529,495,299,812]
+  CRUSH rule 1 x 92 [532,424,426,773,623,197,167,634,781]
+  CRUSH rule 1 x 93 [218,489,405,681,549,201,343,949,51]
+  CRUSH rule 1 x 94 [181,96,102,515,776,365,82,422,738]
+  CRUSH rule 1 x 95 [343,957,820,139,334,37,648,661,46]
+  CRUSH rule 1 x 96 [861,270,87,797,0,245,204,750,322]
+  CRUSH rule 1 x 97 [459,706,45,328,274,605,83,542,131]
+  CRUSH rule 1 x 98 [327,867,353,948,728,280,270,511,586]
+  CRUSH rule 1 x 99 [974,133,468,906,235,988,37,138,326]
+  CRUSH rule 1 x 100 [32,445,547,371,960,885,9,168,590]
+  CRUSH rule 1 x 101 [142,90,337,950,970,570,12,369,9]
+  CRUSH rule 1 x 102 [172,129,139,22,403,867,923,106,653]
+  CRUSH rule 1 x 103 [630,47,161,356,911,421,933,231,520]
+  CRUSH rule 1 x 104 [758,133,278,11,947,799,401,85,139]
+  CRUSH rule 1 x 105 [843,604,47,33,401,632,434,121,488]
+  CRUSH rule 1 x 106 [28,681,193,679,990,343,878,493,550]
+  CRUSH rule 1 x 107 [74,320,85,819,315,253,589,614,814]
+  CRUSH rule 1 x 108 [875,593,575,517,107,153,631,996,630]
+  CRUSH rule 1 x 109 [411,985,811,720,198,666,856,296,122]
+  CRUSH rule 1 x 110 [440,774,799,660,715,167,510,472,270]
+  CRUSH rule 1 x 111 [405,742,276,359,936,360,18,949,341]
+  CRUSH rule 1 x 112 [143,181,922,545,185,303,725,413,187]
+  CRUSH rule 1 x 113 [153,846,160,903,789,897,738,253,213]
+  CRUSH rule 1 x 114 [804,892,939,20,312,692,598,418,641]
+  CRUSH rule 1 x 115 [588,508,958,580,232,722,421,39,241]
+  CRUSH rule 1 x 116 [327,148,637,486,712,464,9,448,816]
+  CRUSH rule 1 x 117 [95,594,989,131,714,275,725,142,304]
+  CRUSH rule 1 x 118 [80,957,897,239,359,432,766,210,528]
+  CRUSH rule 1 x 119 [386,932,951,768,679,300,570,278,867]
+  CRUSH rule 1 x 120 [366,312,653,936,71,241,49,126,410]
+  CRUSH rule 1 x 121 [129,154,847,16,471,481,424,868,469]
+  CRUSH rule 1 x 122 [873,1,110,939,90,412,551,43,590]
+  CRUSH rule 1 x 123 [533,415,789,600,713,800,877,248,753]
+  CRUSH rule 1 x 124 [461,691,898,723,957,759,482,254,158]
+  CRUSH rule 1 x 125 [342,599,830,402,615,994,736,737,508]
+  CRUSH rule 1 x 126 [819,781,822,548,279,255,689,209,99]
+  CRUSH rule 1 x 127 [437,893,585,707,353,189,909,809,553]
+  CRUSH rule 1 x 128 [679,994,982,550,991,324,666,691,899]
+  CRUSH rule 1 x 129 [380,685,947,302,698,144,149,718,904]
+  CRUSH rule 1 x 130 [992,52,466,867,998,777,270,425,864]
+  CRUSH rule 1 x 131 [469,90,208,599,829,656,203,667,528]
+  CRUSH rule 1 x 132 [571,250,316,535,54,418,922,597,680]
+  CRUSH rule 1 x 133 [964,728,329,902,108,118,14,444,709]
+  CRUSH rule 1 x 134 [999,19,716,963,323,559,893,281,226]
+  CRUSH rule 1 x 135 [634,101,52,938,413,573,712,649,27]
+  CRUSH rule 1 x 136 [114,889,692,768,694,279,846,890,151]
+  CRUSH rule 1 x 137 [839,8,959,280,922,870,363,323,153]
+  CRUSH rule 1 x 138 [967,949,138,451,292,548,400,885,907]
+  CRUSH rule 1 x 139 [308,711,736,247,632,126,384,58,373]
+  CRUSH rule 1 x 140 [764,936,926,55,331,115,178,532,883]
+  CRUSH rule 1 x 141 [423,302,112,216,603,873,193,258,445]
+  CRUSH rule 1 x 142 [252,821,715,340,635,668,424,881,751]
+  CRUSH rule 1 x 143 [33,808,518,477,325,316,266,70,210]
+  CRUSH rule 1 x 144 [472,88,969,162,401,771,697,610,203]
+  CRUSH rule 1 x 145 [242,208,252,604,266,743,577,348,1]
+  CRUSH rule 1 x 146 [290,70,570,384,934,856,929,196,880]
+  CRUSH rule 1 x 147 [447,352,657,493,467,918,514,546,861]
+  CRUSH rule 1 x 148 [212,644,432,658,109,275,352,820,857]
+  CRUSH rule 1 x 149 [9,775,87,35,260,646,406,556,532]
+  CRUSH rule 1 x 150 [166,456,582,144,324,340,484,553,315]
+  CRUSH rule 1 x 151 [811,875,307,20,782,229,671,883,204]
+  CRUSH rule 1 x 152 [449,617,223,9,182,407,807,50,206]
+  CRUSH rule 1 x 153 [523,537,695,627,959,613,942,864,388]
+  CRUSH rule 1 x 154 [208,559,874,597,243,706,443,98,27]
+  CRUSH rule 1 x 155 [569,325,192,296,367,848,58,641,186]
+  CRUSH rule 1 x 156 [488,121,521,213,595,837,271,229,961]
+  CRUSH rule 1 x 157 [140,723,633,260,487,856,384,446,836]
+  CRUSH rule 1 x 158 [786,451,320,239,667,632,899,902,956]
+  CRUSH rule 1 x 159 [134,664,517,821,667,944,209,641,228]
+  CRUSH rule 1 x 160 [690,112,414,990,183,590,242,999,974]
+  CRUSH rule 1 x 161 [324,912,397,423,991,284,909,642,188]
+  CRUSH rule 1 x 162 [748,567,284,183,463,336,148,88,764]
+  CRUSH rule 1 x 163 [575,499,31,816,749,737,587,854,482]
+  CRUSH rule 1 x 164 [314,489,308,326,51,568,110,329,361]
+  CRUSH rule 1 x 165 [116,209,750,53,813,640,524,389,185]
+  CRUSH rule 1 x 166 [352,706,701,810,718,527,548,676,448]
+  CRUSH rule 1 x 167 [27,743,174,142,551,1,935,266,883]
+  CRUSH rule 1 x 168 [953,898,880,660,500,799,667,463,818]
+  CRUSH rule 1 x 169 [912,147,266,547,331,770,601,909,60]
+  CRUSH rule 1 x 170 [421,515,828,844,151,981,835,840,548]
+  CRUSH rule 1 x 171 [488,584,880,964,936,196,100,910,446]
+  CRUSH rule 1 x 172 [366,443,957,66,162,693,36,356,274]
+  CRUSH rule 1 x 173 [863,291,625,287,158,496,471,529,359]
+  CRUSH rule 1 x 174 [263,555,650,410,339,616,780,932,573]
+  CRUSH rule 1 x 175 [875,961,361,575,33,109,51,211,409]
+  CRUSH rule 1 x 176 [745,83,701,680,250,420,240,316,337]
+  CRUSH rule 1 x 177 [128,244,41,123,422,902,756,647,45]
+  CRUSH rule 1 x 178 [155,41,264,777,314,564,856,992,696]
+  CRUSH rule 1 x 179 [593,833,202,183,971,38,724,923,450]
+  CRUSH rule 1 x 180 [154,734,17,831,824,522,736,846,926]
+  CRUSH rule 1 x 181 [289,675,723,800,166,712,168,224,705]
+  CRUSH rule 1 x 182 [730,931,560,209,943,261,485,571,796]
+  CRUSH rule 1 x 183 [639,237,794,815,827,400,109,903,96]
+  CRUSH rule 1 x 184 [704,312,685,645,691,778,74,45,438]
+  CRUSH rule 1 x 185 [97,100,762,82,999,542,485,511,14]
+  CRUSH rule 1 x 186 [26,665,554,215,280,421,369,270,16]
+  CRUSH rule 1 x 187 [649,14,740,494,402,684,566,378,816]
+  CRUSH rule 1 x 188 [682,695,590,743,927,945,833,650,761]
+  CRUSH rule 1 x 189 [325,693,726,51,448,169,37,1,939]
+  CRUSH rule 1 x 190 [399,933,136,955,57,504,527,237,295]
+  CRUSH rule 1 x 191 [629,533,17,126,60,146,999,754,339]
+  CRUSH rule 1 x 192 [503,578,38,492,222,251,123,759,147]
+  CRUSH rule 1 x 193 [546,333,651,678,823,652,359,721,996]
+  CRUSH rule 1 x 194 [242,473,58,655,851,277,792,887,561]
+  CRUSH rule 1 x 195 [625,719,135,81,636,513,755,471,629]
+  CRUSH rule 1 x 196 [357,114,125,867,250,522,413,834,832]
+  CRUSH rule 1 x 197 [306,954,453,873,211,334,666,316,243]
+  CRUSH rule 1 x 198 [863,791,311,911,206,61,355,574,781]
+  CRUSH rule 1 x 199 [935,906,929,252,893,75,960,369,584]
+  CRUSH rule 1 x 200 [373,774,229,454,909,611,132,271,128]
+  CRUSH rule 1 x 201 [659,320,477,313,779,16,495,76,598]
+  CRUSH rule 1 x 202 [260,433,524,880,223,818,153,272,944]
+  CRUSH rule 1 x 203 [36,239,675,971,703,209,669,676,762]
+  CRUSH rule 1 x 204 [92,516,993,728,279,478,697,881,64]
+  CRUSH rule 1 x 205 [68,395,473,45,683,662,776,463,327]
+  CRUSH rule 1 x 206 [570,530,642,380,311,398,230,367,890]
+  CRUSH rule 1 x 207 [834,457,850,917,456,296,76,708,101]
+  CRUSH rule 1 x 208 [927,484,640,976,803,626,96,841,811]
+  CRUSH rule 1 x 209 [878,66,58,940,48,233,522,185,949]
+  CRUSH rule 1 x 210 [572,981,484,29,0,426,14,921,544]
+  CRUSH rule 1 x 211 [107,597,780,857,895,57,922,372,581]
+  CRUSH rule 1 x 212 [389,107,838,624,698,562,857,894,60]
+  CRUSH rule 1 x 213 [497,717,567,728,905,134,687,903,620]
+  CRUSH rule 1 x 214 [798,65,254,572,32,393,579,79,258]
+  CRUSH rule 1 x 215 [233,419,283,638,520,891,982,826,488]
+  CRUSH rule 1 x 216 [494,464,742,523,459,174,973,898,556]
+  CRUSH rule 1 x 217 [352,396,309,938,66,41,264,6,603]
+  CRUSH rule 1 x 218 [895,864,988,650,593,740,34,497,108]
+  CRUSH rule 1 x 219 [222,534,277,242,658,482,697,805,976]
+  CRUSH rule 1 x 220 [281,19,584,563,858,965,686,982,0]
+  CRUSH rule 1 x 221 [64,928,963,130,312,394,61,559,846]
+  CRUSH rule 1 x 222 [40,544,161,199,861,644,597,904,897]
+  CRUSH rule 1 x 223 [645,556,159,417,46,135,465,429,614]
+  CRUSH rule 1 x 224 [647,165,957,263,961,576,329,320,645]
+  CRUSH rule 1 x 225 [219,714,858,747,461,175,606,465,354]
+  CRUSH rule 1 x 226 [372,511,181,277,695,404,876,984,491]
+  CRUSH rule 1 x 227 [925,156,714,863,257,74,966,217,501]
+  CRUSH rule 1 x 228 [682,404,839,263,521,195,261,389,281]
+  CRUSH rule 1 x 229 [880,838,770,891,236,542,262,884,215]
+  CRUSH rule 1 x 230 [328,659,916,468,646,572,93,880,959]
+  CRUSH rule 1 x 231 [320,383,669,109,627,621,50,182,541]
+  CRUSH rule 1 x 232 [924,846,394,319,43,519,106,877,130]
+  CRUSH rule 1 x 233 [948,652,575,838,498,395,796,835,714]
+  CRUSH rule 1 x 234 [484,943,42,575,936,180,103,95,634]
+  CRUSH rule 1 x 235 [750,65,590,168,870,308,471,753,350]
+  CRUSH rule 1 x 236 [551,787,490,136,370,833,573,128,154]
+  CRUSH rule 1 x 237 [390,157,166,251,752,75,327,509,325]
+  CRUSH rule 1 x 238 [570,6,989,707,514,905,894,884,824]
+  CRUSH rule 1 x 239 [729,959,376,975,496,49,426,427,736]
+  CRUSH rule 1 x 240 [981,241,156,767,631,576,450,677,659]
+  CRUSH rule 1 x 241 [310,816,641,177,996,454,413,136,411]
+  CRUSH rule 1 x 242 [161,63,642,837,763,458,234,756,496]
+  CRUSH rule 1 x 243 [180,394,33,683,189,419,799,21,13]
+  CRUSH rule 1 x 244 [52,174,685,189,78,310,785,107,816]
+  CRUSH rule 1 x 245 [523,121,915,84,386,409,605,837,1]
+  CRUSH rule 1 x 246 [362,893,390,487,817,88,989,999,138]
+  CRUSH rule 1 x 247 [382,184,116,34,143,15,590,840,586]
+  CRUSH rule 1 x 248 [129,114,852,469,359,291,713,237,468]
+  CRUSH rule 1 x 249 [159,683,91,856,475,369,886,650,827]
+  CRUSH rule 1 x 250 [404,945,569,955,228,910,270,619,450]
+  CRUSH rule 1 x 251 [661,225,738,757,37,642,58,354,16]
+  CRUSH rule 1 x 252 [961,226,542,103,945,885,838,131,387]
+  CRUSH rule 1 x 253 [651,97,225,364,189,248,797,675,452]
+  CRUSH rule 1 x 254 [123,33,741,692,599,11,605,453,987]
+  CRUSH rule 1 x 255 [314,649,891,855,517,344,607,95,121]
+  CRUSH rule 1 x 256 [315,215,651,126,470,849,189,627,592]
+  CRUSH rule 1 x 257 [825,264,867,17,529,409,291,732,224]
+  CRUSH rule 1 x 258 [624,789,370,723,131,982,863,427,873]
+  CRUSH rule 1 x 259 [602,542,70,563,947,723,77,191,669]
+  CRUSH rule 1 x 260 [717,878,43,56,377,481,533,646,475]
+  CRUSH rule 1 x 261 [145,517,20,903,786,939,516,136,87]
+  CRUSH rule 1 x 262 [223,1,561,420,256,16,88,534,289]
+  CRUSH rule 1 x 263 [462,211,405,508,787,669,773,979,719]
+  CRUSH rule 1 x 264 [654,471,266,662,135,564,715,916,633]
+  CRUSH rule 1 x 265 [302,794,704,798,659,487,833,987,445]
+  CRUSH rule 1 x 266 [202,132,884,209,551,984,7,557,76]
+  CRUSH rule 1 x 267 [282,938,657,113,672,993,972,645,882]
+  CRUSH rule 1 x 268 [338,309,356,278,928,797,715,536,983]
+  CRUSH rule 1 x 269 [738,122,266,200,894,118,146,14,414]
+  CRUSH rule 1 x 270 [707,982,946,196,407,804,476,571,314]
+  CRUSH rule 1 x 271 [705,432,364,735,512,595,263,138,526]
+  CRUSH rule 1 x 272 [756,545,942,56,542,449,710,779,161]
+  CRUSH rule 1 x 273 [197,502,527,721,239,648,982,735,58]
+  CRUSH rule 1 x 274 [992,44,653,573,527,702,370,990,320]
+  CRUSH rule 1 x 275 [544,789,170,434,23,926,992,823,321]
+  CRUSH rule 1 x 276 [658,467,577,268,336,5,634,98,457]
+  CRUSH rule 1 x 277 [143,490,880,483,928,272,783,648,927]
+  CRUSH rule 1 x 278 [492,647,355,282,834,64,350,600,283]
+  CRUSH rule 1 x 279 [517,792,604,987,527,894,952,250,206]
+  CRUSH rule 1 x 280 [825,740,27,848,514,750,895,914,892]
+  CRUSH rule 1 x 281 [224,629,120,562,616,200,443,604,859]
+  CRUSH rule 1 x 282 [298,661,380,416,35,585,939,879,338]
+  CRUSH rule 1 x 283 [311,606,208,50,913,678,369,544,721]
+  CRUSH rule 1 x 284 [771,466,371,743,672,119,60,546,39]
+  CRUSH rule 1 x 285 [693,362,404,676,797,531,582,975,810]
+  CRUSH rule 1 x 286 [364,477,285,167,270,617,699,627,725]
+  CRUSH rule 1 x 287 [591,611,828,995,170,987,137,890,487]
+  CRUSH rule 1 x 288 [965,541,848,796,251,668,195,538,356]
+  CRUSH rule 1 x 289 [225,551,948,877,219,167,795,377,825]
+  CRUSH rule 1 x 290 [577,762,777,751,291,349,473,209,59]
+  CRUSH rule 1 x 291 [160,903,477,381,490,559,557,86,89]
+  CRUSH rule 1 x 292 [873,598,216,666,222,228,806,911,738]
+  CRUSH rule 1 x 293 [100,234,874,47,28,452,775,636,232]
+  CRUSH rule 1 x 294 [285,943,379,520,725,547,459,833,503]
+  CRUSH rule 1 x 295 [938,262,880,327,687,3,440,73,29]
+  CRUSH rule 1 x 296 [850,327,86,472,1,776,266,82,671]
+  CRUSH rule 1 x 297 [951,53,99,558,753,228,232,343,831]
+  CRUSH rule 1 x 298 [173,336,85,766,910,657,213,286,61]
+  CRUSH rule 1 x 299 [598,591,315,386,895,296,924,106,63]
+  CRUSH rule 1 x 300 [531,957,62,459,156,538,904,838,458]
+  CRUSH rule 1 x 301 [823,628,23,858,629,808,220,432,393]
+  CRUSH rule 1 x 302 [184,80,780,871,531,211,400,365,697]
+  CRUSH rule 1 x 303 [521,766,222,830,988,275,561,905,522]
+  CRUSH rule 1 x 304 [980,127,807,507,555,245,214,944,845]
+  CRUSH rule 1 x 305 [153,816,22,927,696,911,685,838,3]
+  CRUSH rule 1 x 306 [423,739,664,753,178,431,761,648,867]
+  CRUSH rule 1 x 307 [997,557,682,456,479,631,459,250,415]
+  CRUSH rule 1 x 308 [991,874,534,465,330,284,976,551,126]
+  CRUSH rule 1 x 309 [860,394,724,858,246,866,857,153,970]
+  CRUSH rule 1 x 310 [589,818,546,201,94,653,90,855,441]
+  CRUSH rule 1 x 311 [477,774,225,590,830,559,256,798,743]
+  CRUSH rule 1 x 312 [887,853,950,354,58,23,497,929,92]
+  CRUSH rule 1 x 313 [802,646,447,416,557,118,24,81,215]
+  CRUSH rule 1 x 314 [654,974,229,511,562,916,952,599,398]
+  CRUSH rule 1 x 315 [767,227,28,740,828,156,749,841,969]
+  CRUSH rule 1 x 316 [778,83,733,359,858,319,761,725,923]
+  CRUSH rule 1 x 317 [184,418,642,986,939,675,892,86,214]
+  CRUSH rule 1 x 318 [525,410,500,543,212,95,290,97,529]
+  CRUSH rule 1 x 319 [476,724,569,382,409,521,800,868,364]
+  CRUSH rule 1 x 320 [149,610,697,296,818,955,523,366,891]
+  CRUSH rule 1 x 321 [710,79,667,671,234,4,868,841,563]
+  CRUSH rule 1 x 322 [175,275,323,333,744,718,187,380,947]
+  CRUSH rule 1 x 323 [819,604,638,792,316,544,236,597,969]
+  CRUSH rule 1 x 324 [16,745,511,439,272,417,668,959,845]
+  CRUSH rule 1 x 325 [486,400,872,873,251,68,462,268,124]
+  CRUSH rule 1 x 326 [613,765,207,19,359,370,461,509,75]
+  CRUSH rule 1 x 327 [125,289,738,408,456,784,750,669,296]
+  CRUSH rule 1 x 328 [807,383,476,583,645,141,33,806,181]
+  CRUSH rule 1 x 329 [588,938,599,432,446,840,516,713,223]
+  CRUSH rule 1 x 330 [932,644,41,611,209,406,420,520,395]
+  CRUSH rule 1 x 331 [341,953,950,537,578,862,624,649,626]
+  CRUSH rule 1 x 332 [153,726,459,950,466,804,644,821,238]
+  CRUSH rule 1 x 333 [745,845,853,860,52,615,243,633,309]
+  CRUSH rule 1 x 334 [614,751,807,58,396,159,408,175,189]
+  CRUSH rule 1 x 335 [518,721,221,283,454,187,635,367,997]
+  CRUSH rule 1 x 336 [389,424,77,309,5,898,698,533,683]
+  CRUSH rule 1 x 337 [753,508,765,720,221,807,956,907,464]
+  CRUSH rule 1 x 338 [128,810,490,753,406,760,69,11,624]
+  CRUSH rule 1 x 339 [430,308,58,751,856,823,607,953,125]
+  CRUSH rule 1 x 340 [541,44,630,231,289,966,707,328,325]
+  CRUSH rule 1 x 341 [402,26,631,439,165,928,720,503,209]
+  CRUSH rule 1 x 342 [982,57,992,461,131,32,516,661,985]
+  CRUSH rule 1 x 343 [833,412,572,732,107,805,660,655,149]
+  CRUSH rule 1 x 344 [784,533,792,41,642,869,142,114,108]
+  CRUSH rule 1 x 345 [546,300,304,691,763,556,127,732,290]
+  CRUSH rule 1 x 346 [302,420,428,891,357,124,419,962,304]
+  CRUSH rule 1 x 347 [488,778,101,217,366,442,783,661,622]
+  CRUSH rule 1 x 348 [903,744,937,718,85,314,862,513,112]
+  CRUSH rule 1 x 349 [471,547,582,306,600,486,795,143,529]
+  CRUSH rule 1 x 350 [348,221,823,335,383,708,841,164,765]
+  CRUSH rule 1 x 351 [961,582,705,346,361,32,766,775,518]
+  CRUSH rule 1 x 352 [728,137,461,298,36,903,899,665,802]
+  CRUSH rule 1 x 353 [904,202,184,447,58,294,279,616,892]
+  CRUSH rule 1 x 354 [345,226,319,256,544,311,612,33,122]
+  CRUSH rule 1 x 355 [50,430,175,43,187,458,985,412,599]
+  CRUSH rule 1 x 356 [87,185,55,423,829,1,629,228,150]
+  CRUSH rule 1 x 357 [762,459,921,473,182,231,891,656,196]
+  CRUSH rule 1 x 358 [908,25,280,6,808,676,874,643,550]
+  CRUSH rule 1 x 359 [484,15,132,121,394,423,397,52,702]
+  CRUSH rule 1 x 360 [173,378,337,702,145,499,29,529,156]
+  CRUSH rule 1 x 361 [404,577,115,25,56,914,643,286,552]
+  CRUSH rule 1 x 362 [403,1,422,945,132,685,265,35,662]
+  CRUSH rule 1 x 363 [639,911,510,162,418,294,444,613,813]
+  CRUSH rule 1 x 364 [752,689,610,990,665,222,203,17,743]
+  CRUSH rule 1 x 365 [956,999,212,230,624,84,113,373,426]
+  CRUSH rule 1 x 366 [860,925,924,763,687,851,59,914,521]
+  CRUSH rule 1 x 367 [205,609,647,665,969,720,685,641,894]
+  CRUSH rule 1 x 368 [301,284,810,169,78,340,616,93,283]
+  CRUSH rule 1 x 369 [452,658,339,217,674,210,284,184,718]
+  CRUSH rule 1 x 370 [11,467,695,989,394,576,850,419,307]
+  CRUSH rule 1 x 371 [124,487,55,514,313,411,797,547,778]
+  CRUSH rule 1 x 372 [253,48,979,846,207,631,212,241,346]
+  CRUSH rule 1 x 373 [715,605,775,748,227,493,128,207,88]
+  CRUSH rule 1 x 374 [191,887,920,291,223,714,961,760,571]
+  CRUSH rule 1 x 375 [711,385,651,665,15,71,934,619,527]
+  CRUSH rule 1 x 376 [597,818,49,458,415,755,446,897,460]
+  CRUSH rule 1 x 377 [294,256,933,771,184,861,654,487,891]
+  CRUSH rule 1 x 378 [34,151,681,707,552,127,728,860,968]
+  CRUSH rule 1 x 379 [869,136,315,378,813,153,115,557,165]
+  CRUSH rule 1 x 380 [294,97,575,791,690,482,255,806,429]
+  CRUSH rule 1 x 381 [119,710,219,827,328,886,773,496,433]
+  CRUSH rule 1 x 382 [69,631,508,706,697,168,276,56,278]
+  CRUSH rule 1 x 383 [922,588,589,925,471,601,29,197,822]
+  CRUSH rule 1 x 384 [221,945,671,117,857,655,488,435,223]
+  CRUSH rule 1 x 385 [561,737,953,723,658,368,910,329,396]
+  CRUSH rule 1 x 386 [335,442,788,696,507,716,232,692,742]
+  CRUSH rule 1 x 387 [514,43,353,88,100,842,164,934,297]
+  CRUSH rule 1 x 388 [587,89,157,996,915,927,474,267,640]
+  CRUSH rule 1 x 389 [109,641,255,466,372,563,340,222,74]
+  CRUSH rule 1 x 390 [925,149,421,489,599,810,852,196,469]
+  CRUSH rule 1 x 391 [267,87,387,527,768,873,41,136,818]
+  CRUSH rule 1 x 392 [382,485,370,849,936,636,901,82,695]
+  CRUSH rule 1 x 393 [425,721,221,753,268,463,652,543,10]
+  CRUSH rule 1 x 394 [898,18,38,793,173,738,15,591,420]
+  CRUSH rule 1 x 395 [806,876,269,679,32,744,126,179,607]
+  CRUSH rule 1 x 396 [790,970,437,449,875,395,726,935,278]
+  CRUSH rule 1 x 397 [136,363,507,613,11,30,996,558,602]
+  CRUSH rule 1 x 398 [914,116,558,258,722,904,349,672,826]
+  CRUSH rule 1 x 399 [261,94,299,202,174,622,749,410,815]
+  CRUSH rule 1 x 400 [661,197,338,461,977,848,536,592,886]
+  CRUSH rule 1 x 401 [953,979,287,803,41,349,79,32,343]
+  CRUSH rule 1 x 402 [738,819,618,522,667,334,658,449,886]
+  CRUSH rule 1 x 403 [573,238,425,546,130,68,202,650,501]
+  CRUSH rule 1 x 404 [526,848,790,253,922,820,299,577,563]
+  CRUSH rule 1 x 405 [582,505,330,334,201,110,776,296,19]
+  CRUSH rule 1 x 406 [768,324,493,60,186,165,718,578,580]
+  CRUSH rule 1 x 407 [260,951,437,587,692,648,72,345,709]
+  CRUSH rule 1 x 408 [657,81,770,734,830,821,246,695,76]
+  CRUSH rule 1 x 409 [498,89,182,423,672,152,213,806,168]
+  CRUSH rule 1 x 410 [28,793,737,352,166,645,949,507,361]
+  CRUSH rule 1 x 411 [684,992,60,659,769,267,313,351,497]
+  CRUSH rule 1 x 412 [261,958,699,950,165,14,560,155,661]
+  CRUSH rule 1 x 413 [891,835,297,441,384,979,618,907,9]
+  CRUSH rule 1 x 414 [127,459,119,965,662,594,97,124,229]
+  CRUSH rule 1 x 415 [272,540,631,328,609,568,694,332,572]
+  CRUSH rule 1 x 416 [739,617,115,530,339,371,889,344,838]
+  CRUSH rule 1 x 417 [106,209,157,878,117,128,138,374,470]
+  CRUSH rule 1 x 418 [525,441,147,390,320,300,848,972,781]
+  CRUSH rule 1 x 419 [603,673,615,465,266,855,823,884,832]
+  CRUSH rule 1 x 420 [988,213,251,226,209,245,506,670,285]
+  CRUSH rule 1 x 421 [761,521,748,368,923,992,764,274,623]
+  CRUSH rule 1 x 422 [317,160,924,548,198,709,839,547,599]
+  CRUSH rule 1 x 423 [137,807,168,472,619,443,905,588,312]
+  CRUSH rule 1 x 424 [920,37,146,263,598,748,785,395,884]
+  CRUSH rule 1 x 425 [277,693,285,221,478,165,80,236,988]
+  CRUSH rule 1 x 426 [485,936,407,854,726,524,791,565,352]
+  CRUSH rule 1 x 427 [242,515,9,564,174,453,334,588,571]
+  CRUSH rule 1 x 428 [632,635,26,473,494,478,225,94,303]
+  CRUSH rule 1 x 429 [641,73,465,127,171,397,857,562,976]
+  CRUSH rule 1 x 430 [626,585,6,387,881,583,859,699,91]
+  CRUSH rule 1 x 431 [697,76,753,570,964,339,194,366,279]
+  CRUSH rule 1 x 432 [590,526,306,283,656,728,513,591,599]
+  CRUSH rule 1 x 433 [284,387,149,817,886,714,52,897,705]
+  CRUSH rule 1 x 434 [538,985,79,953,770,468,644,646,747]
+  CRUSH rule 1 x 435 [30,318,593,635,975,833,371,731,906]
+  CRUSH rule 1 x 436 [164,919,851,693,0,874,10,976,284]
+  CRUSH rule 1 x 437 [322,212,163,606,302,282,443,23,696]
+  CRUSH rule 1 x 438 [142,392,85,594,376,419,755,841,94]
+  CRUSH rule 1 x 439 [119,370,68,443,997,837,414,152,331]
+  CRUSH rule 1 x 440 [333,403,187,863,475,844,800,174,117]
+  CRUSH rule 1 x 441 [477,727,906,145,429,91,205,236,86]
+  CRUSH rule 1 x 442 [274,590,933,244,434,49,864,799,762]
+  CRUSH rule 1 x 443 [983,748,574,718,700,442,774,350,37]
+  CRUSH rule 1 x 444 [536,509,431,146,170,149,182,145,347]
+  CRUSH rule 1 x 445 [485,46,528,209,964,753,554,931,638]
+  CRUSH rule 1 x 446 [345,634,42,294,711,376,314,714,212]
+  CRUSH rule 1 x 447 [61,845,767,600,321,716,58,531,827]
+  CRUSH rule 1 x 448 [333,232,292,846,364,951,807,688,21]
+  CRUSH rule 1 x 449 [680,16,484,670,851,500,258,548,905]
+  CRUSH rule 1 x 450 [235,214,79,423,96,822,721,31,312]
+  CRUSH rule 1 x 451 [961,468,333,640,823,151,878,33,3]
+  CRUSH rule 1 x 452 [525,479,153,528,570,806,604,49,922]
+  CRUSH rule 1 x 453 [138,466,302,86,249,154,514,5,494]
+  CRUSH rule 1 x 454 [137,625,215,402,389,914,106,103,511]
+  CRUSH rule 1 x 455 [173,150,997,16,846,888,295,967,132]
+  CRUSH rule 1 x 456 [235,226,238,258,347,784,504,96,890]
+  CRUSH rule 1 x 457 [450,577,253,413,717,609,762,975,485]
+  CRUSH rule 1 x 458 [195,537,91,814,351,90,399,558,15]
+  CRUSH rule 1 x 459 [381,555,312,573,915,623,147,483,517]
+  CRUSH rule 1 x 460 [972,730,534,678,756,692,841,512,70]
+  CRUSH rule 1 x 461 [506,279,142,830,784,124,385,797,917]
+  CRUSH rule 1 x 462 [692,959,578,57,983,299,240,911,375]
+  CRUSH rule 1 x 463 [788,667,949,550,685,702,538,111,232]
+  CRUSH rule 1 x 464 [133,122,588,999,270,880,789,0,653]
+  CRUSH rule 1 x 465 [971,190,230,777,452,914,137,466,531]
+  CRUSH rule 1 x 466 [394,576,148,157,103,822,659,35,797]
+  CRUSH rule 1 x 467 [517,28,366,362,984,521,187,640,601]
+  CRUSH rule 1 x 468 [829,143,874,225,162,413,201,249,555]
+  CRUSH rule 1 x 469 [987,936,106,725,633,238,681,463,551]
+  CRUSH rule 1 x 470 [107,982,56,889,67,65,558,71,676]
+  CRUSH rule 1 x 471 [181,897,629,860,307,116,256,978,409]
+  CRUSH rule 1 x 472 [547,512,172,24,705,837,809,56,476]
+  CRUSH rule 1 x 473 [760,997,824,905,888,755,756,663,167]
+  CRUSH rule 1 x 474 [787,418,743,628,272,341,446,333,245]
+  CRUSH rule 1 x 475 [662,312,253,617,105,58,237,764,682]
+  CRUSH rule 1 x 476 [110,495,185,508,961,837,984,226,333]
+  CRUSH rule 1 x 477 [393,954,834,132,841,367,753,794,237]
+  CRUSH rule 1 x 478 [246,483,480,644,985,420,941,843,751]
+  CRUSH rule 1 x 479 [70,929,697,931,744,487,158,489,515]
+  CRUSH rule 1 x 480 [753,119,961,607,317,717,371,807,687]
+  CRUSH rule 1 x 481 [470,429,677,242,574,757,135,375,613]
+  CRUSH rule 1 x 482 [451,566,961,675,354,746,731,233,640]
+  CRUSH rule 1 x 483 [816,72,371,278,635,30,448,437,219]
+  CRUSH rule 1 x 484 [540,454,389,31,654,494,283,170,278]
+  CRUSH rule 1 x 485 [74,582,624,684,566,677,866,661,581]
+  CRUSH rule 1 x 486 [958,595,199,763,715,973,621,955,400]
+  CRUSH rule 1 x 487 [228,302,804,833,876,647,857,782,24]
+  CRUSH rule 1 x 488 [180,529,722,956,353,890,924,965,25]
+  CRUSH rule 1 x 489 [47,617,812,187,291,828,154,478,512]
+  CRUSH rule 1 x 490 [905,822,479,124,750,843,566,779,963]
+  CRUSH rule 1 x 491 [892,370,609,998,433,957,188,563,490]
+  CRUSH rule 1 x 492 [588,959,127,948,505,936,591,423,668]
+  CRUSH rule 1 x 493 [353,461,593,291,301,830,231,442,474]
+  CRUSH rule 1 x 494 [378,848,443,368,507,423,389,819,956]
+  CRUSH rule 1 x 495 [845,653,768,234,405,367,823,789,217]
+  CRUSH rule 1 x 496 [13,988,0,691,389,757,129,763,39]
+  CRUSH rule 1 x 497 [796,877,788,394,648,829,542,745,131]
+  CRUSH rule 1 x 498 [412,337,270,705,511,227,949,173,398]
+  CRUSH rule 1 x 499 [330,695,8,74,618,101,440,509,295]
+  CRUSH rule 1 x 500 [820,272,547,765,755,96,930,573,357]
+  CRUSH rule 1 x 501 [110,44,132,442,294,423,880,279,616]
+  CRUSH rule 1 x 502 [336,595,650,274,993,312,490,852,962]
+  CRUSH rule 1 x 503 [922,211,157,722,502,971,262,926,316]
+  CRUSH rule 1 x 504 [483,52,122,432,778,461,758,104,831]
+  CRUSH rule 1 x 505 [482,598,224,279,480,310,764,558,891]
+  CRUSH rule 1 x 506 [493,123,43,856,936,622,898,161,78]
+  CRUSH rule 1 x 507 [12,598,264,422,416,947,591,702,346]
+  CRUSH rule 1 x 508 [227,157,611,301,223,746,313,282,207]
+  CRUSH rule 1 x 509 [807,242,363,122,582,530,798,808,139]
+  CRUSH rule 1 x 510 [134,437,227,75,313,351,786,152,921]
+  CRUSH rule 1 x 511 [212,54,83,799,457,218,600,968,355]
+  CRUSH rule 1 x 512 [236,630,758,752,361,249,899,451,415]
+  CRUSH rule 1 x 513 [994,693,644,938,846,685,52,185,197]
+  CRUSH rule 1 x 514 [45,508,831,19,817,52,374,985,944]
+  CRUSH rule 1 x 515 [504,138,480,272,530,377,481,820,517]
+  CRUSH rule 1 x 516 [285,409,136,570,841,610,453,660,93]
+  CRUSH rule 1 x 517 [300,232,23,906,438,236,519,737,20]
+  CRUSH rule 1 x 518 [397,674,98,898,967,113,625,434,527]
+  CRUSH rule 1 x 519 [86,750,772,913,101,864,375,328,3]
+  CRUSH rule 1 x 520 [900,833,614,130,261,885,558,956,664]
+  CRUSH rule 1 x 521 [31,47,236,751,911,599,495,354,665]
+  CRUSH rule 1 x 522 [390,16,280,144,291,175,753,624,769]
+  CRUSH rule 1 x 523 [618,308,424,590,300,206,834,212,906]
+  CRUSH rule 1 x 524 [635,189,687,963,601,518,8,550,769]
+  CRUSH rule 1 x 525 [311,916,699,262,775,32,45,478,911]
+  CRUSH rule 1 x 526 [48,738,227,718,244,942,853,643,625]
+  CRUSH rule 1 x 527 [202,851,889,216,763,351,270,35,809]
+  CRUSH rule 1 x 528 [565,827,590,273,918,106,651,368,118]
+  CRUSH rule 1 x 529 [934,864,241,43,466,924,278,926,280]
+  CRUSH rule 1 x 530 [502,934,298,670,986,360,577,509,195]
+  CRUSH rule 1 x 531 [681,627,942,487,288,561,925,474,669]
+  CRUSH rule 1 x 532 [422,6,147,205,861,141,949,374,988]
+  CRUSH rule 1 x 533 [863,68,364,983,247,199,54,931,4]
+  CRUSH rule 1 x 534 [962,931,775,172,663,119,206,682,627]
+  CRUSH rule 1 x 535 [89,565,397,693,839,632,859,30,61]
+  CRUSH rule 1 x 536 [499,351,760,458,918,86,148,668,436]
+  CRUSH rule 1 x 537 [676,547,787,311,867,748,152,797,492]
+  CRUSH rule 1 x 538 [58,644,571,649,941,7,37,485,88]
+  CRUSH rule 1 x 539 [837,953,457,711,458,621,528,722,59]
+  CRUSH rule 1 x 540 [831,50,132,213,197,709,95,789,348]
+  CRUSH rule 1 x 541 [582,757,121,525,532,963,738,277,225]
+  CRUSH rule 1 x 542 [472,132,790,997,948,269,137,934,547]
+  CRUSH rule 1 x 543 [382,272,797,330,315,748,324,134,839]
+  CRUSH rule 1 x 544 [947,930,496,883,509,219,250,362,614]
+  CRUSH rule 1 x 545 [425,570,305,77,821,422,117,172,764]
+  CRUSH rule 1 x 546 [18,65,529,437,343,547,699,610,785]
+  CRUSH rule 1 x 547 [445,715,600,472,213,851,428,267,229]
+  CRUSH rule 1 x 548 [367,569,980,167,627,442,517,684,154]
+  CRUSH rule 1 x 549 [125,715,671,817,285,420,37,639,934]
+  CRUSH rule 1 x 550 [425,599,744,199,923,222,915,570,546]
+  CRUSH rule 1 x 551 [44,1,528,922,944,115,161,901,342]
+  CRUSH rule 1 x 552 [246,104,68,239,123,427,57,217,21]
+  CRUSH rule 1 x 553 [71,703,615,28,593,724,218,916,561]
+  CRUSH rule 1 x 554 [207,124,217,166,525,226,693,953,606]
+  CRUSH rule 1 x 555 [570,28,317,420,931,413,623,659,403]
+  CRUSH rule 1 x 556 [674,152,421,79,215,347,830,762,691]
+  CRUSH rule 1 x 557 [347,817,191,391,741,571,593,267,17]
+  CRUSH rule 1 x 558 [627,426,369,692,815,371,124,107,766]
+  CRUSH rule 1 x 559 [940,630,924,242,224,912,185,356,87]
+  CRUSH rule 1 x 560 [295,903,541,29,245,753,887,376,658]
+  CRUSH rule 1 x 561 [506,682,384,637,878,991,700,339,687]
+  CRUSH rule 1 x 562 [718,529,87,729,842,341,62,817,766]
+  CRUSH rule 1 x 563 [552,332,747,206,274,871,903,900,812]
+  CRUSH rule 1 x 564 [835,769,736,486,630,209,641,751,930]
+  CRUSH rule 1 x 565 [8,167,539,182,607,62,738,873,47]
+  CRUSH rule 1 x 566 [600,481,301,263,90,450,184,127,448]
+  CRUSH rule 1 x 567 [999,994,509,899,947,24,267,639,646]
+  CRUSH rule 1 x 568 [252,431,157,62,601,863,398,521,59]
+  CRUSH rule 1 x 569 [643,218,943,455,83,969,494,624,352]
+  CRUSH rule 1 x 570 [617,635,765,422,250,156,533,674,23]
+  CRUSH rule 1 x 571 [757,80,59,98,328,700,329,848,235]
+  CRUSH rule 1 x 572 [299,348,575,889,943,675,33,312,202]
+  CRUSH rule 1 x 573 [25,505,270,167,58,901,878,978,1]
+  CRUSH rule 1 x 574 [215,431,624,177,628,814,333,841,193]
+  CRUSH rule 1 x 575 [225,252,611,546,32,815,389,486,10]
+  CRUSH rule 1 x 576 [627,94,159,857,430,691,177,545,839]
+  CRUSH rule 1 x 577 [237,809,778,636,61,167,700,521,825]
+  CRUSH rule 1 x 578 [885,313,120,344,771,614,487,976,977]
+  CRUSH rule 1 x 579 [924,575,787,831,47,996,557,630,468]
+  CRUSH rule 1 x 580 [718,51,766,121,118,471,608,755,326]
+  CRUSH rule 1 x 581 [219,807,129,571,856,179,874,902,958]
+  CRUSH rule 1 x 582 [893,701,598,863,285,829,984,622,175]
+  CRUSH rule 1 x 583 [246,930,964,170,993,409,469,193,737]
+  CRUSH rule 1 x 584 [336,432,680,175,495,839,642,226,122]
+  CRUSH rule 1 x 585 [324,999,397,485,457,527,73,628,884]
+  CRUSH rule 1 x 586 [558,230,976,541,816,72,794,682,127]
+  CRUSH rule 1 x 587 [985,830,597,21,308,890,952,421,875]
+  CRUSH rule 1 x 588 [211,544,57,134,162,496,195,581,649]
+  CRUSH rule 1 x 589 [129,21,112,190,885,844,753,180,160]
+  CRUSH rule 1 x 590 [467,969,652,593,287,76,811,413,436]
+  CRUSH rule 1 x 591 [758,514,316,164,35,110,54,796,369]
+  CRUSH rule 1 x 592 [525,253,190,443,315,603,667,318,496]
+  CRUSH rule 1 x 593 [601,885,339,152,297,223,269,455,168]
+  CRUSH rule 1 x 594 [227,60,450,30,717,840,994,16,777]
+  CRUSH rule 1 x 595 [720,854,496,912,80,655,917,525,945]
+  CRUSH rule 1 x 596 [751,195,997,77,261,490,180,482,449]
+  CRUSH rule 1 x 597 [129,574,714,8,789,847,725,991,955]
+  CRUSH rule 1 x 598 [679,207,604,396,841,284,286,280,507]
+  CRUSH rule 1 x 599 [668,315,683,349,681,253,599,364,546]
+  CRUSH rule 1 x 600 [143,396,464,444,59,57,243,264,31]
+  CRUSH rule 1 x 601 [326,573,873,902,136,921,633,596,988]
+  CRUSH rule 1 x 602 [860,281,875,535,672,474,697,763,442]
+  CRUSH rule 1 x 603 [709,328,445,349,190,455,924,667,356]
+  CRUSH rule 1 x 604 [571,62,814,95,866,978,983,281,292]
+  CRUSH rule 1 x 605 [252,739,860,27,313,362,857,899,349]
+  CRUSH rule 1 x 606 [339,236,759,842,67,644,954,94,88]
+  CRUSH rule 1 x 607 [590,248,759,868,433,398,578,386,226]
+  CRUSH rule 1 x 608 [145,635,309,467,875,115,148,33,420]
+  CRUSH rule 1 x 609 [973,547,223,79,762,863,249,41,778]
+  CRUSH rule 1 x 610 [435,816,961,983,255,886,160,888,685]
+  CRUSH rule 1 x 611 [559,283,422,584,176,429,570,43,362]
+  CRUSH rule 1 x 612 [273,149,123,576,911,270,296,735,245]
+  CRUSH rule 1 x 613 [828,614,642,674,33,361,958,580,197]
+  CRUSH rule 1 x 614 [478,748,393,34,171,80,92,12,62]
+  CRUSH rule 1 x 615 [392,155,144,326,626,134,149,401,14]
+  CRUSH rule 1 x 616 [778,637,452,248,15,888,74,307,976]
+  CRUSH rule 1 x 617 [622,713,996,833,611,407,364,8,342]
+  CRUSH rule 1 x 618 [149,877,270,329,180,327,222,749,697]
+  CRUSH rule 1 x 619 [604,163,656,409,322,848,519,967,737]
+  CRUSH rule 1 x 620 [181,23,409,198,64,898,35,620,268]
+  CRUSH rule 1 x 621 [735,902,386,237,939,475,725,118,875]
+  CRUSH rule 1 x 622 [661,824,717,568,858,583,446,798,869]
+  CRUSH rule 1 x 623 [142,121,643,61,695,852,485,478,185]
+  CRUSH rule 1 x 624 [360,716,420,398,49,717,137,140,488]
+  CRUSH rule 1 x 625 [541,167,385,1,601,481,308,111,207]
+  CRUSH rule 1 x 626 [364,431,610,363,535,747,225,841,868]
+  CRUSH rule 1 x 627 [458,137,557,410,287,749,467,432,944]
+  CRUSH rule 1 x 628 [250,350,556,497,821,65,205,580,972]
+  CRUSH rule 1 x 629 [928,160,710,572,365,772,538,46,300]
+  CRUSH rule 1 x 630 [243,19,918,556,601,16,920,830,171]
+  CRUSH rule 1 x 631 [438,221,574,676,797,580,219,211,157]
+  CRUSH rule 1 x 632 [797,368,247,5,32,102,416,45,624]
+  CRUSH rule 1 x 633 [993,749,525,485,27,330,275,599,219]
+  CRUSH rule 1 x 634 [239,351,633,299,651,678,296,337,676]
+  CRUSH rule 1 x 635 [640,965,25,961,306,172,849,357,317]
+  CRUSH rule 1 x 636 [173,290,297,991,937,823,236,318,228]
+  CRUSH rule 1 x 637 [0,918,98,108,111,495,887,57,16]
+  CRUSH rule 1 x 638 [702,235,424,900,983,754,701,887,355]
+  CRUSH rule 1 x 639 [475,687,31,785,918,611,27,214,226]
+  CRUSH rule 1 x 640 [31,664,399,677,123,609,858,138,726]
+  CRUSH rule 1 x 641 [296,473,108,963,341,876,897,449,42]
+  CRUSH rule 1 x 642 [894,273,427,606,677,670,610,665,299]
+  CRUSH rule 1 x 643 [117,111,732,191,114,153,500,631,833]
+  CRUSH rule 1 x 644 [438,336,327,512,599,862,660,857,123]
+  CRUSH rule 1 x 645 [982,702,351,573,907,915,279,317,414]
+  CRUSH rule 1 x 646 [334,804,146,842,697,638,720,135,369]
+  CRUSH rule 1 x 647 [933,787,185,334,752,285,372,890,30]
+  CRUSH rule 1 x 648 [22,444,400,862,207,842,453,732,262]
+  CRUSH rule 1 x 649 [503,229,213,460,639,760,722,748,599]
+  CRUSH rule 1 x 650 [328,659,420,443,739,950,869,150,743]
+  CRUSH rule 1 x 651 [3,880,823,123,378,585,715,221,31]
+  CRUSH rule 1 x 652 [495,977,563,733,92,997,119,818,459]
+  CRUSH rule 1 x 653 [185,718,804,280,975,912,198,291,71]
+  CRUSH rule 1 x 654 [130,528,380,81,906,511,431,506,546]
+  CRUSH rule 1 x 655 [560,872,454,504,319,284,605,214,833]
+  CRUSH rule 1 x 656 [219,885,178,981,863,508,708,6,746]
+  CRUSH rule 1 x 657 [233,684,813,490,208,941,858,16,128]
+  CRUSH rule 1 x 658 [778,6,756,380,750,836,547,850,499]
+  CRUSH rule 1 x 659 [240,663,306,540,789,902,170,954,22]
+  CRUSH rule 1 x 660 [244,855,196,147,678,323,63,859,215]
+  CRUSH rule 1 x 661 [184,270,128,398,910,230,402,205,609]
+  CRUSH rule 1 x 662 [65,883,921,438,79,957,464,902,276]
+  CRUSH rule 1 x 663 [323,721,594,812,43,992,170,65,906]
+  CRUSH rule 1 x 664 [865,113,512,51,427,123,585,260,254]
+  CRUSH rule 1 x 665 [420,850,591,475,202,733,798,658,28]
+  CRUSH rule 1 x 666 [319,767,246,3,369,493,796,56,736]
+  CRUSH rule 1 x 667 [875,39,343,100,829,2,795,783,386]
+  CRUSH rule 1 x 668 [331,122,263,599,355,484,943,554,395]
+  CRUSH rule 1 x 669 [915,521,402,747,673,445,938,600,517]
+  CRUSH rule 1 x 670 [845,659,943,447,401,322,168,302,681]
+  CRUSH rule 1 x 671 [108,634,527,363,856,238,755,330,584]
+  CRUSH rule 1 x 672 [578,216,110,589,302,137,954,315,735]
+  CRUSH rule 1 x 673 [442,74,579,797,622,950,371,402,725]
+  CRUSH rule 1 x 674 [588,364,281,308,645,631,229,506,565]
+  CRUSH rule 1 x 675 [489,698,744,671,870,174,528,875,982]
+  CRUSH rule 1 x 676 [928,911,40,180,722,729,673,569,701]
+  CRUSH rule 1 x 677 [399,269,692,131,615,136,103,763,527]
+  CRUSH rule 1 x 678 [546,752,544,155,5,463,666,352,576]
+  CRUSH rule 1 x 679 [988,25,275,433,628,57,247,620,437]
+  CRUSH rule 1 x 680 [335,963,382,486,749,257,795,347,831]
+  CRUSH rule 1 x 681 [690,462,623,466,49,471,774,192,454]
+  CRUSH rule 1 x 682 [196,588,154,257,807,776,367,718,345]
+  CRUSH rule 1 x 683 [627,25,421,160,873,102,345,599,30]
+  CRUSH rule 1 x 684 [38,804,592,158,991,264,652,821,641]
+  CRUSH rule 1 x 685 [841,368,548,362,166,211,154,121,51]
+  CRUSH rule 1 x 686 [336,287,525,440,166,993,911,638,690]
+  CRUSH rule 1 x 687 [20,682,924,653,356,16,917,622,156]
+  CRUSH rule 1 x 688 [463,371,780,556,385,883,115,248,566]
+  CRUSH rule 1 x 689 [569,250,78,816,847,775,333,161,74]
+  CRUSH rule 1 x 690 [551,144,587,263,378,394,970,639,835]
+  CRUSH rule 1 x 691 [766,464,446,533,449,541,451,290,789]
+  CRUSH rule 1 x 692 [739,634,18,245,624,35,268,525,425]
+  CRUSH rule 1 x 693 [339,297,118,330,817,91,828,276,264]
+  CRUSH rule 1 x 694 [405,26,830,181,533,166,488,804,501]
+  CRUSH rule 1 x 695 [622,576,597,535,600,593,300,989,804]
+  CRUSH rule 1 x 696 [558,902,689,13,715,28,664,489,598]
+  CRUSH rule 1 x 697 [818,222,406,691,427,863,153,922,986]
+  CRUSH rule 1 x 698 [178,48,402,233,841,604,468,180,783]
+  CRUSH rule 1 x 699 [450,244,180,919,711,332,747,453,519]
+  CRUSH rule 1 x 700 [502,771,987,706,416,240,68,641,109]
+  CRUSH rule 1 x 701 [4,612,782,216,853,303,585,513,907]
+  CRUSH rule 1 x 702 [177,630,232,923,281,708,466,687,742]
+  CRUSH rule 1 x 703 [354,178,389,393,778,803,796,607,894]
+  CRUSH rule 1 x 704 [646,601,156,171,603,116,655,595,888]
+  CRUSH rule 1 x 705 [921,401,890,265,244,690,372,253,807]
+  CRUSH rule 1 x 706 [652,877,562,452,26,323,923,770,516]
+  CRUSH rule 1 x 707 [345,745,67,716,789,576,2,133,256]
+  CRUSH rule 1 x 708 [333,607,180,469,170,555,939,331,41]
+  CRUSH rule 1 x 709 [45,187,302,115,896,579,733,607,763]
+  CRUSH rule 1 x 710 [94,855,43,199,18,948,449,28,731]
+  CRUSH rule 1 x 711 [227,653,731,150,693,842,534,110,639]
+  CRUSH rule 1 x 712 [398,953,136,870,181,408,895,459,341]
+  CRUSH rule 1 x 713 [116,800,503,662,635,579,53,839,56]
+  CRUSH rule 1 x 714 [111,629,866,709,902,557,875,649,23]
+  CRUSH rule 1 x 715 [531,291,486,382,192,807,322,417,973]
+  CRUSH rule 1 x 716 [169,541,291,42,343,724,138,197,32]
+  CRUSH rule 1 x 717 [417,446,994,894,239,494,237,62,327]
+  CRUSH rule 1 x 718 [992,383,298,844,377,463,544,891,210]
+  CRUSH rule 1 x 719 [936,674,324,759,194,409,828,975,119]
+  CRUSH rule 1 x 720 [370,188,174,464,644,218,214,76,870]
+  CRUSH rule 1 x 721 [320,859,278,259,170,957,177,264,867]
+  CRUSH rule 1 x 722 [7,2,673,129,96,445,823,833,1]
+  CRUSH rule 1 x 723 [270,553,831,662,38,101,985,846,77]
+  CRUSH rule 1 x 724 [666,822,708,895,633,800,616,879,480]
+  CRUSH rule 1 x 725 [794,406,875,459,981,751,359,34,720]
+  CRUSH rule 1 x 726 [420,556,341,292,240,68,966,535,669]
+  CRUSH rule 1 x 727 [561,461,129,635,965,610,105,31,506]
+  CRUSH rule 1 x 728 [951,330,196,756,589,849,753,760,254]
+  CRUSH rule 1 x 729 [656,644,436,591,27,119,572,933,434]
+  CRUSH rule 1 x 730 [3,558,629,184,50,765,760,800,945]
+  CRUSH rule 1 x 731 [852,89,75,735,713,113,528,890,625]
+  CRUSH rule 1 x 732 [983,840,869,976,697,307,368,271,778]
+  CRUSH rule 1 x 733 [285,396,388,122,387,364,880,343,590]
+  CRUSH rule 1 x 734 [125,510,402,640,676,501,535,627,224]
+  CRUSH rule 1 x 735 [417,773,686,504,459,912,690,59,294]
+  CRUSH rule 1 x 736 [749,396,632,550,779,109,845,278,559]
+  CRUSH rule 1 x 737 [644,991,946,135,448,903,482,564,259]
+  CRUSH rule 1 x 738 [449,683,290,220,245,525,429,397,872]
+  CRUSH rule 1 x 739 [341,220,641,454,740,661,146,17,314]
+  CRUSH rule 1 x 740 [874,524,674,650,472,282,214,494,593]
+  CRUSH rule 1 x 741 [189,472,712,798,715,757,863,571,876]
+  CRUSH rule 1 x 742 [912,581,114,759,730,21,687,81,145]
+  CRUSH rule 1 x 743 [654,914,425,441,763,39,451,631,911]
+  CRUSH rule 1 x 744 [725,295,579,377,162,447,843,699,24]
+  CRUSH rule 1 x 745 [787,858,850,506,612,735,926,314,771]
+  CRUSH rule 1 x 746 [757,848,704,30,47,940,450,651,105]
+  CRUSH rule 1 x 747 [700,81,867,681,801,64,879,857,727]
+  CRUSH rule 1 x 748 [557,436,238,664,293,865,304,999,685]
+  CRUSH rule 1 x 749 [772,622,337,42,156,302,383,506,570]
+  CRUSH rule 1 x 750 [946,97,376,677,316,670,169,171,9]
+  CRUSH rule 1 x 751 [996,618,343,911,83,22,388,17,892]
+  CRUSH rule 1 x 752 [746,887,695,868,610,950,88,315,728]
+  CRUSH rule 1 x 753 [741,14,463,479,172,192,481,702,431]
+  CRUSH rule 1 x 754 [648,349,333,355,65,63,336,724,262]
+  CRUSH rule 1 x 755 [157,460,466,187,959,674,192,279,371]
+  CRUSH rule 1 x 756 [416,97,197,497,227,3,850,191,991]
+  CRUSH rule 1 x 757 [599,839,776,410,256,823,121,690,544]
+  CRUSH rule 1 x 758 [994,218,620,256,361,749,165,686,449]
+  CRUSH rule 1 x 759 [959,682,514,745,100,519,15,347,311]
+  CRUSH rule 1 x 760 [518,943,215,83,706,137,345,69,39]
+  CRUSH rule 1 x 761 [285,849,420,324,987,338,373,361,684]
+  CRUSH rule 1 x 762 [591,313,41,335,110,696,664,350,339]
+  CRUSH rule 1 x 763 [908,411,200,740,292,295,387,775,797]
+  CRUSH rule 1 x 764 [787,234,894,485,883,711,70,202,557]
+  CRUSH rule 1 x 765 [327,921,882,393,444,792,402,123,902]
+  CRUSH rule 1 x 766 [84,161,878,704,416,144,357,310,890]
+  CRUSH rule 1 x 767 [370,895,702,701,890,2,251,951,675]
+  CRUSH rule 1 x 768 [826,760,879,864,460,474,645,975,947]
+  CRUSH rule 1 x 769 [67,768,663,735,814,66,213,527,546]
+  CRUSH rule 1 x 770 [593,909,482,259,5,550,961,324,309]
+  CRUSH rule 1 x 771 [309,935,121,578,937,685,933,571,822]
+  CRUSH rule 1 x 772 [12,125,797,301,348,419,891,959,487]
+  CRUSH rule 1 x 773 [253,466,820,549,591,193,783,951,982]
+  CRUSH rule 1 x 774 [164,390,705,109,881,505,890,425,599]
+  CRUSH rule 1 x 775 [703,47,43,973,643,406,885,976,936]
+  CRUSH rule 1 x 776 [728,231,80,916,2,850,396,76,680]
+  CRUSH rule 1 x 777 [981,621,568,729,869,952,563,860,388]
+  CRUSH rule 1 x 778 [411,456,544,597,789,784,65,954,125]
+  CRUSH rule 1 x 779 [346,121,519,921,587,48,772,645,254]
+  CRUSH rule 1 x 780 [476,39,288,381,303,29,17,336,147]
+  CRUSH rule 1 x 781 [10,130,585,844,729,705,714,954,271]
+  CRUSH rule 1 x 782 [462,246,581,902,623,877,812,516,774]
+  CRUSH rule 1 x 783 [580,373,153,775,668,661,626,961,576]
+  CRUSH rule 1 x 784 [413,113,978,990,994,56,481,198,171]
+  CRUSH rule 1 x 785 [341,856,332,354,59,581,632,151,586]
+  CRUSH rule 1 x 786 [411,140,313,393,215,618,490,481,627]
+  CRUSH rule 1 x 787 [605,522,211,813,636,224,600,528,966]
+  CRUSH rule 1 x 788 [226,545,35,142,726,851,194,216,486]
+  CRUSH rule 1 x 789 [545,320,414,702,731,277,237,916,374]
+  CRUSH rule 1 x 790 [414,748,816,327,130,115,788,164,691]
+  CRUSH rule 1 x 791 [660,906,406,697,916,322,124,267,742]
+  CRUSH rule 1 x 792 [287,392,514,204,75,789,406,858,694]
+  CRUSH rule 1 x 793 [631,133,850,713,720,487,376,812,886]
+  CRUSH rule 1 x 794 [931,517,543,210,963,898,811,459,344]
+  CRUSH rule 1 x 795 [551,962,477,948,425,434,268,94,648]
+  CRUSH rule 1 x 796 [814,4,95,27,368,300,646,451,67]
+  CRUSH rule 1 x 797 [64,201,299,734,605,864,596,196,93]
+  CRUSH rule 1 x 798 [422,530,114,431,565,716,473,250,839]
+  CRUSH rule 1 x 799 [824,32,679,562,266,549,859,994,831]
+  CRUSH rule 1 x 800 [862,623,489,637,861,196,941,643,398]
+  CRUSH rule 1 x 801 [145,550,329,324,734,160,219,662,142]
+  CRUSH rule 1 x 802 [570,19,847,308,387,518,846,53,783]
+  CRUSH rule 1 x 803 [151,812,662,358,880,349,834,881,23]
+  CRUSH rule 1 x 804 [467,93,264,863,176,842,663,949,380]
+  CRUSH rule 1 x 805 [621,223,938,809,591,686,121,157,934]
+  CRUSH rule 1 x 806 [898,957,805,430,499,584,640,607,790]
+  CRUSH rule 1 x 807 [354,531,422,159,921,431,802,136,305]
+  CRUSH rule 1 x 808 [7,96,76,897,446,2,166,929,234]
+  CRUSH rule 1 x 809 [70,734,719,56,687,21,23,145,184]
+  CRUSH rule 1 x 810 [701,18,972,327,771,649,620,648,433]
+  CRUSH rule 1 x 811 [248,547,103,728,901,264,948,202,521]
+  CRUSH rule 1 x 812 [230,576,821,566,993,762,675,28,263]
+  CRUSH rule 1 x 813 [805,114,683,629,288,462,285,450,948]
+  CRUSH rule 1 x 814 [54,619,973,741,497,894,401,266,905]
+  CRUSH rule 1 x 815 [679,412,613,132,969,411,314,670,928]
+  CRUSH rule 1 x 816 [919,448,826,414,36,289,44,822,332]
+  CRUSH rule 1 x 817 [765,830,436,521,332,458,260,172,193]
+  CRUSH rule 1 x 818 [415,566,644,687,692,414,769,826,519]
+  CRUSH rule 1 x 819 [721,319,865,750,546,859,523,770,56]
+  CRUSH rule 1 x 820 [218,301,333,190,686,179,535,787,267]
+  CRUSH rule 1 x 821 [185,795,680,953,329,750,621,815,313]
+  CRUSH rule 1 x 822 [356,261,54,522,900,103,883,112,601]
+  CRUSH rule 1 x 823 [220,281,549,456,64,306,282,641,216]
+  CRUSH rule 1 x 824 [292,809,887,74,776,788,559,886,753]
+  CRUSH rule 1 x 825 [949,778,101,311,110,480,161,998,370]
+  CRUSH rule 1 x 826 [767,818,833,927,356,954,910,63,288]
+  CRUSH rule 1 x 827 [631,83,406,635,657,713,212,916,692]
+  CRUSH rule 1 x 828 [288,986,445,26,414,607,937,595,935]
+  CRUSH rule 1 x 829 [990,667,915,694,974,453,669,330,822]
+  CRUSH rule 1 x 830 [152,571,778,505,685,209,448,55,965]
+  CRUSH rule 1 x 831 [814,563,630,97,582,107,142,157,957]
+  CRUSH rule 1 x 832 [235,641,616,110,979,844,656,135,341]
+  CRUSH rule 1 x 833 [657,565,922,140,825,457,764,766,853]
+  CRUSH rule 1 x 834 [907,231,644,13,617,130,83,483,811]
+  CRUSH rule 1 x 835 [784,262,771,264,612,238,537,937,101]
+  CRUSH rule 1 x 836 [951,158,366,710,43,427,351,961,52]
+  CRUSH rule 1 x 837 [556,498,334,633,895,627,903,29,454]
+  CRUSH rule 1 x 838 [329,274,964,547,119,342,983,998,320]
+  CRUSH rule 1 x 839 [568,209,939,364,658,747,47,859,402]
+  CRUSH rule 1 x 840 [45,579,842,70,655,862,815,109,762]
+  CRUSH rule 1 x 841 [652,702,24,605,152,93,226,46,918]
+  CRUSH rule 1 x 842 [629,984,314,895,408,897,575,1,312]
+  CRUSH rule 1 x 843 [799,690,688,648,151,812,486,199,966]
+  CRUSH rule 1 x 844 [694,600,534,700,569,11,899,382,851]
+  CRUSH rule 1 x 845 [332,30,179,93,951,324,611,512,855]
+  CRUSH rule 1 x 846 [452,251,712,719,404,739,606,237,414]
+  CRUSH rule 1 x 847 [399,681,847,739,13,555,363,893,592]
+  CRUSH rule 1 x 848 [303,138,440,346,547,216,700,249,214]
+  CRUSH rule 1 x 849 [666,346,708,873,64,694,847,463,995]
+  CRUSH rule 1 x 850 [644,511,345,844,545,337,358,35,913]
+  CRUSH rule 1 x 851 [527,546,737,425,100,331,95,337,677]
+  CRUSH rule 1 x 852 [31,809,94,618,156,853,469,511,999]
+  CRUSH rule 1 x 853 [483,330,869,184,46,942,774,679,616]
+  CRUSH rule 1 x 854 [697,953,968,143,502,955,441,302,437]
+  CRUSH rule 1 x 855 [837,996,239,621,32,191,686,702,919]
+  CRUSH rule 1 x 856 [712,40,547,430,195,857,224,810,404]
+  CRUSH rule 1 x 857 [77,984,576,551,568,96,12,763,594]
+  CRUSH rule 1 x 858 [412,384,841,465,572,576,688,61,545]
+  CRUSH rule 1 x 859 [173,760,26,300,87,567,463,903,272]
+  CRUSH rule 1 x 860 [776,429,328,917,658,783,699,907,532]
+  CRUSH rule 1 x 861 [705,405,477,50,73,714,901,487,725]
+  CRUSH rule 1 x 862 [809,44,788,938,964,177,490,409,15]
+  CRUSH rule 1 x 863 [349,496,963,178,675,853,172,980,772]
+  CRUSH rule 1 x 864 [717,858,101,239,992,244,43,15,29]
+  CRUSH rule 1 x 865 [857,603,586,262,550,289,850,40,170]
+  CRUSH rule 1 x 866 [394,304,71,96,642,155,255,481,435]
+  CRUSH rule 1 x 867 [640,773,663,974,261,296,988,730,753]
+  CRUSH rule 1 x 868 [613,950,712,663,761,460,643,547,734]
+  CRUSH rule 1 x 869 [973,889,524,22,671,477,718,431,968]
+  CRUSH rule 1 x 870 [505,35,386,498,348,503,54,992,726]
+  CRUSH rule 1 x 871 [239,264,262,773,781,734,387,515,98]
+  CRUSH rule 1 x 872 [21,767,456,748,783,797,180,800,521]
+  CRUSH rule 1 x 873 [954,666,980,264,435,233,199,358,805]
+  CRUSH rule 1 x 874 [54,510,947,1,500,119,93,915,801]
+  CRUSH rule 1 x 875 [809,418,452,462,88,673,634,435,778]
+  CRUSH rule 1 x 876 [483,457,61,248,523,277,322,141,82]
+  CRUSH rule 1 x 877 [542,531,952,939,710,179,181,460,459]
+  CRUSH rule 1 x 878 [217,674,857,644,678,809,329,591,59]
+  CRUSH rule 1 x 879 [999,475,134,250,319,357,145,750,54]
+  CRUSH rule 1 x 880 [678,573,935,385,570,651,319,630,888]
+  CRUSH rule 1 x 881 [394,835,789,802,587,155,570,109,896]
+  CRUSH rule 1 x 882 [467,382,353,56,979,674,974,483,412]
+  CRUSH rule 1 x 883 [802,744,237,337,50,96,202,148,129]
+  CRUSH rule 1 x 884 [653,660,638,700,31,558,389,381,347]
+  CRUSH rule 1 x 885 [898,704,307,445,879,872,174,972,544]
+  CRUSH rule 1 x 886 [434,357,938,641,737,8,56,582,915]
+  CRUSH rule 1 x 887 [297,226,711,428,370,318,472,947,35]
+  CRUSH rule 1 x 888 [863,324,443,213,902,25,806,53,385]
+  CRUSH rule 1 x 889 [105,102,308,163,947,548,399,382,761]
+  CRUSH rule 1 x 890 [550,248,606,704,615,708,996,561,485]
+  CRUSH rule 1 x 891 [575,928,880,891,826,763,706,701,501]
+  CRUSH rule 1 x 892 [259,862,133,271,292,162,53,333,458]
+  CRUSH rule 1 x 893 [902,880,543,542,37,942,672,320,394]
+  CRUSH rule 1 x 894 [180,169,916,43,945,713,648,685,895]
+  CRUSH rule 1 x 895 [725,849,182,129,177,272,599,829,809]
+  CRUSH rule 1 x 896 [951,34,874,537,969,123,210,529,491]
+  CRUSH rule 1 x 897 [810,352,73,939,943,895,12,481,539]
+  CRUSH rule 1 x 898 [979,433,719,411,787,359,342,37,303]
+  CRUSH rule 1 x 899 [685,668,534,932,399,156,124,653,574]
+  CRUSH rule 1 x 900 [530,978,41,894,941,681,380,419,667]
+  CRUSH rule 1 x 901 [740,107,336,175,574,706,157,292,724]
+  CRUSH rule 1 x 902 [800,743,693,310,67,111,178,624,733]
+  CRUSH rule 1 x 903 [230,267,842,266,550,769,66,738,419]
+  CRUSH rule 1 x 904 [346,949,460,973,696,91,957,801,74]
+  CRUSH rule 1 x 905 [530,397,619,958,576,973,685,6,689]
+  CRUSH rule 1 x 906 [80,426,138,672,73,776,30,169,506]
+  CRUSH rule 1 x 907 [365,968,475,297,296,724,664,331,184]
+  CRUSH rule 1 x 908 [204,832,742,809,862,745,484,391,841]
+  CRUSH rule 1 x 909 [883,989,146,959,366,59,686,965,515]
+  CRUSH rule 1 x 910 [549,593,249,853,792,769,824,552,717]
+  CRUSH rule 1 x 911 [325,847,352,214,851,732,789,255,896]
+  CRUSH rule 1 x 912 [874,888,582,796,557,601,226,889,69]
+  CRUSH rule 1 x 913 [331,463,342,574,989,362,925,746,664]
+  CRUSH rule 1 x 914 [836,468,601,732,607,275,70,280,837]
+  CRUSH rule 1 x 915 [245,228,100,661,799,13,126,79,652]
+  CRUSH rule 1 x 916 [77,967,364,435,27,474,255,133,892]
+  CRUSH rule 1 x 917 [239,60,866,221,772,967,725,707,47]
+  CRUSH rule 1 x 918 [988,115,922,80,201,544,583,923,863]
+  CRUSH rule 1 x 919 [783,139,696,1,848,169,888,980,33]
+  CRUSH rule 1 x 920 [623,408,685,953,974,696,532,124,911]
+  CRUSH rule 1 x 921 [105,799,144,90,399,373,633,290,155]
+  CRUSH rule 1 x 922 [887,505,652,348,514,806,952,474,67]
+  CRUSH rule 1 x 923 [223,318,552,458,743,871,964,384,454]
+  CRUSH rule 1 x 924 [25,778,366,333,163,801,584,31,151]
+  CRUSH rule 1 x 925 [912,601,297,682,770,173,969,168,500]
+  CRUSH rule 1 x 926 [968,133,135,144,814,155,709,158,96]
+  CRUSH rule 1 x 927 [277,724,214,988,690,342,465,775,725]
+  CRUSH rule 1 x 928 [554,203,658,789,298,299,847,752,780]
+  CRUSH rule 1 x 929 [761,802,367,528,758,522,744,171,144]
+  CRUSH rule 1 x 930 [814,61,788,736,660,491,832,654,567]
+  CRUSH rule 1 x 931 [29,193,61,41,343,664,487,839,776]
+  CRUSH rule 1 x 932 [446,198,862,534,168,35,530,462,202]
+  CRUSH rule 1 x 933 [352,742,216,321,525,44,568,61,945]
+  CRUSH rule 1 x 934 [730,2,332,631,613,249,533,116,254]
+  CRUSH rule 1 x 935 [731,23,736,79,361,992,772,49,567]
+  CRUSH rule 1 x 936 [322,975,20,904,827,603,138,802,885]
+  CRUSH rule 1 x 937 [822,221,841,161,723,137,630,308,973]
+  CRUSH rule 1 x 938 [557,850,66,630,499,404,286,395,927]
+  CRUSH rule 1 x 939 [150,11,971,371,124,785,408,49,977]
+  CRUSH rule 1 x 940 [638,398,169,616,333,751,25,883,867]
+  CRUSH rule 1 x 941 [730,342,929,577,451,838,964,28,633]
+  CRUSH rule 1 x 942 [62,292,166,814,587,172,854,16,440]
+  CRUSH rule 1 x 943 [165,314,519,548,41,726,759,851,617]
+  CRUSH rule 1 x 944 [199,625,766,176,194,297,678,915,619]
+  CRUSH rule 1 x 945 [946,999,699,303,38,81,952,885,987]
+  CRUSH rule 1 x 946 [595,93,852,142,503,647,933,267,846]
+  CRUSH rule 1 x 947 [800,582,356,93,716,117,922,868,413]
+  CRUSH rule 1 x 948 [132,551,139,920,87,46,81,220,725]
+  CRUSH rule 1 x 949 [792,920,466,380,97,568,799,961,564]
+  CRUSH rule 1 x 950 [111,345,176,543,879,954,355,220,528]
+  CRUSH rule 1 x 951 [414,619,648,655,364,971,829,408,568]
+  CRUSH rule 1 x 952 [775,469,500,356,287,4,16,746,835]
+  CRUSH rule 1 x 953 [349,1,5,251,168,680,141,619,234]
+  CRUSH rule 1 x 954 [570,940,410,249,929,394,129,696,115]
+  CRUSH rule 1 x 955 [729,774,823,800,7,127,536,766,579]
+  CRUSH rule 1 x 956 [519,141,575,625,738,475,169,751,667]
+  CRUSH rule 1 x 957 [242,709,611,97,760,309,393,281,227]
+  CRUSH rule 1 x 958 [84,217,227,253,246,604,346,377,425]
+  CRUSH rule 1 x 959 [270,413,918,789,703,608,543,519,496]
+  CRUSH rule 1 x 960 [458,192,307,279,920,139,855,49,548]
+  CRUSH rule 1 x 961 [981,388,777,546,359,660,455,708,649]
+  CRUSH rule 1 x 962 [623,834,277,134,729,246,856,477,895]
+  CRUSH rule 1 x 963 [291,167,714,468,109,373,485,701,76]
+  CRUSH rule 1 x 964 [28,156,788,127,598,215,361,255,507]
+  CRUSH rule 1 x 965 [675,557,290,517,840,510,59,229,819]
+  CRUSH rule 1 x 966 [836,306,946,283,642,606,929,773,928]
+  CRUSH rule 1 x 967 [966,386,735,837,392,116,19,674,395]
+  CRUSH rule 1 x 968 [864,756,690,121,328,122,433,520,916]
+  CRUSH rule 1 x 969 [729,625,480,769,512,882,518,956,398]
+  CRUSH rule 1 x 970 [800,362,646,582,309,102,576,411,416]
+  CRUSH rule 1 x 971 [737,381,153,684,298,166,344,520,546]
+  CRUSH rule 1 x 972 [952,245,720,884,334,311,754,540,79]
+  CRUSH rule 1 x 973 [356,455,579,857,832,596,549,524,109]
+  CRUSH rule 1 x 974 [545,758,586,596,539,790,116,993,644]
+  CRUSH rule 1 x 975 [336,191,202,146,720,897,330,308,744]
+  CRUSH rule 1 x 976 [446,208,757,620,252,846,397,58,57]
+  CRUSH rule 1 x 977 [202,896,196,956,763,126,783,828,409]
+  CRUSH rule 1 x 978 [612,324,996,225,418,583,514,169,99]
+  CRUSH rule 1 x 979 [843,457,675,650,958,657,677,173,903]
+  CRUSH rule 1 x 980 [60,914,881,626,850,759,398,943,764]
+  CRUSH rule 1 x 981 [702,749,937,153,724,514,536,212,247]
+  CRUSH rule 1 x 982 [298,928,738,167,99,668,395,198,100]
+  CRUSH rule 1 x 983 [723,572,395,358,900,37,927,597,103]
+  CRUSH rule 1 x 984 [723,864,804,935,846,993,950,840,427]
+  CRUSH rule 1 x 985 [945,459,868,211,524,954,911,208,91]
+  CRUSH rule 1 x 986 [772,664,535,169,297,996,864,555,687]
+  CRUSH rule 1 x 987 [88,324,312,843,661,580,76,894,480]
+  CRUSH rule 1 x 988 [522,927,131,996,351,685,865,47,116]
+  CRUSH rule 1 x 989 [578,332,208,605,975,207,155,380,797]
+  CRUSH rule 1 x 990 [638,228,414,311,738,698,340,526,728]
+  CRUSH rule 1 x 991 [530,221,451,422,879,916,754,928,288]
+  CRUSH rule 1 x 992 [925,705,275,81,234,310,117,546,798]
+  CRUSH rule 1 x 993 [991,301,43,469,830,242,382,428,451]
+  CRUSH rule 1 x 994 [276,51,868,683,843,815,557,378,936]
+  CRUSH rule 1 x 995 [288,836,753,790,758,120,158,265,110]
+  CRUSH rule 1 x 996 [887,983,252,686,470,345,459,764,859]
+  CRUSH rule 1 x 997 [110,924,386,79,705,697,210,698,273]
+  CRUSH rule 1 x 998 [435,830,485,853,926,730,786,762,444]
+  CRUSH rule 1 x 999 [876,738,357,913,723,51,15,585,898]
+  CRUSH rule 1 x 1000 [178,963,638,430,845,586,317,102,200]
+  CRUSH rule 1 x 1001 [99,519,66,759,583,944,739,922,343]
+  CRUSH rule 1 x 1002 [515,534,468,866,878,717,729,370,326]
+  CRUSH rule 1 x 1003 [104,611,937,698,94,67,614,783,865]
+  CRUSH rule 1 x 1004 [269,638,724,375,491,121,891,113,424]
+  CRUSH rule 1 x 1005 [369,223,309,409,822,39,597,969,911]
+  CRUSH rule 1 x 1006 [40,107,69,275,79,429,234,945,598]
+  CRUSH rule 1 x 1007 [978,111,416,758,454,640,5,444,795]
+  CRUSH rule 1 x 1008 [965,956,624,832,421,96,975,723,909]
+  CRUSH rule 1 x 1009 [598,476,356,695,919,566,234,383,604]
+  CRUSH rule 1 x 1010 [767,523,239,517,29,77,23,241,838]
+  CRUSH rule 1 x 1011 [289,871,207,576,347,698,48,570,639]
+  CRUSH rule 1 x 1012 [128,28,370,31,341,755,268,647,669]
+  CRUSH rule 1 x 1013 [979,765,660,812,666,187,808,351,572]
+  CRUSH rule 1 x 1014 [979,948,513,88,47,825,969,81,586]
+  CRUSH rule 1 x 1015 [277,790,396,672,542,647,145,11,965]
+  CRUSH rule 1 x 1016 [262,73,128,886,839,685,456,560,935]
+  CRUSH rule 1 x 1017 [150,269,61,499,832,591,637,731,738]
+  CRUSH rule 1 x 1018 [555,829,554,944,406,576,463,926,475]
+  CRUSH rule 1 x 1019 [513,356,265,446,65,288,768,245,337]
+  CRUSH rule 1 x 1020 [158,161,877,704,948,570,495,865,698]
+  CRUSH rule 1 x 1021 [915,998,957,285,546,202,676,322,671]
+  CRUSH rule 1 x 1022 [967,829,973,640,703,470,871,828,440]
+  CRUSH rule 1 x 1023 [488,257,614,859,325,419,50,560,595]
+  rule 1 (metadata) num_rep 9 result size == 9:\t1024/1024 (esc)
+  CRUSH rule 1 x 0 [36,705,536,450,604,380,966,750,695,503]
+  CRUSH rule 1 x 1 [876,250,334,633,744,843,672,820,782,802]
+  CRUSH rule 1 x 2 [292,832,53,392,386,787,527,901,106,273]
+  CRUSH rule 1 x 3 [623,387,124,998,749,211,481,169,816,732]
+  CRUSH rule 1 x 4 [61,334,710,4,994,982,847,220,87,254]
+  CRUSH rule 1 x 5 [946,557,713,664,141,817,964,872,66,161]
+  CRUSH rule 1 x 6 [576,668,212,163,732,381,884,726,456,796]
+  CRUSH rule 1 x 7 [645,753,906,393,341,44,578,14,543,287]
+  CRUSH rule 1 x 8 [243,6,863,781,211,100,462,207,759,701]
+  CRUSH rule 1 x 9 [22,578,251,410,297,430,3,569,603,47]
+  CRUSH rule 1 x 10 [758,828,360,477,821,801,811,484,296,320]
+  CRUSH rule 1 x 11 [769,120,124,527,119,504,380,821,470,230]
+  CRUSH rule 1 x 12 [780,364,689,755,675,199,117,393,435,514]
+  CRUSH rule 1 x 13 [557,18,351,719,742,780,78,170,333,295]
+  CRUSH rule 1 x 14 [59,561,249,461,971,835,855,76,269,673]
+  CRUSH rule 1 x 15 [718,928,993,21,76,313,437,797,680,761]
+  CRUSH rule 1 x 16 [673,632,841,954,788,90,786,969,378,246]
+  CRUSH rule 1 x 17 [648,43,560,514,142,289,935,605,228,737]
+  CRUSH rule 1 x 18 [654,219,181,568,381,253,883,394,188,459]
+  CRUSH rule 1 x 19 [850,545,377,848,863,543,51,834,690,375]
+  CRUSH rule 1 x 20 [717,785,974,5,225,552,975,636,387,600]
+  CRUSH rule 1 x 21 [420,57,519,306,312,983,263,267,128,828]
+  CRUSH rule 1 x 22 [503,998,193,821,634,684,557,633,812,521]
+  CRUSH rule 1 x 23 [411,663,168,110,899,488,477,468,303,367]
+  CRUSH rule 1 x 24 [266,861,353,1,456,128,800,309,622,673]
+  CRUSH rule 1 x 25 [760,483,818,600,509,951,248,908,624,643]
+  CRUSH rule 1 x 26 [903,24,573,718,112,694,501,909,219,686]
+  CRUSH rule 1 x 27 [946,188,289,510,687,827,676,560,753,77]
+  CRUSH rule 1 x 28 [69,312,73,198,256,629,770,569,359,733]
+  CRUSH rule 1 x 29 [844,883,337,628,496,405,719,581,816,349]
+  CRUSH rule 1 x 30 [621,18,613,794,910,936,426,522,208,699]
+  CRUSH rule 1 x 31 [784,943,814,539,962,392,813,217,750,155]
+  CRUSH rule 1 x 32 [173,374,369,972,315,83,428,63,801,735]
+  CRUSH rule 1 x 33 [698,336,357,966,582,407,618,288,846,659]
+  CRUSH rule 1 x 34 [168,836,210,798,904,190,663,877,177,567]
+  CRUSH rule 1 x 35 [274,509,534,818,912,671,75,580,568,523]
+  CRUSH rule 1 x 36 [318,215,153,628,87,407,676,524,510,480]
+  CRUSH rule 1 x 37 [173,604,109,935,203,401,311,758,201,999]
+  CRUSH rule 1 x 38 [708,444,683,604,722,900,929,910,698,386]
+  CRUSH rule 1 x 39 [662,198,417,680,226,342,856,248,279,416]
+  CRUSH rule 1 x 40 [620,801,414,78,560,766,980,503,287,564]
+  CRUSH rule 1 x 41 [811,264,177,127,148,791,930,74,844,943]
+  CRUSH rule 1 x 42 [863,179,527,660,133,529,456,713,348,311]
+  CRUSH rule 1 x 43 [686,822,988,228,791,549,514,40,261,223]
+  CRUSH rule 1 x 44 [396,222,46,841,536,140,160,527,250,247]
+  CRUSH rule 1 x 45 [991,694,253,142,54,422,658,876,201,45]
+  CRUSH rule 1 x 46 [420,909,184,285,508,458,45,390,546,908]
+  CRUSH rule 1 x 47 [467,211,605,207,241,881,959,800,743,161]
+  CRUSH rule 1 x 48 [955,329,368,168,698,787,738,47,812,166]
+  CRUSH rule 1 x 49 [974,891,931,29,813,506,822,628,696,407]
+  CRUSH rule 1 x 50 [870,441,691,823,761,6,83,344,713,857]
+  CRUSH rule 1 x 51 [182,930,25,936,97,260,406,281,991,336]
+  CRUSH rule 1 x 52 [704,812,894,794,481,37,304,899,629,701]
+  CRUSH rule 1 x 53 [185,713,631,280,345,558,882,503,327,402]
+  CRUSH rule 1 x 54 [270,441,100,82,983,930,339,902,81,239]
+  CRUSH rule 1 x 55 [895,734,958,793,651,572,508,763,108,185]
+  CRUSH rule 1 x 56 [564,963,683,324,40,189,77,500,553,417]
+  CRUSH rule 1 x 57 [738,130,208,973,498,861,670,67,114,685]
+  CRUSH rule 1 x 58 [524,113,806,903,531,334,8,762,842,884]
+  CRUSH rule 1 x 59 [408,337,668,529,34,384,643,511,370,336]
+  CRUSH rule 1 x 60 [228,790,857,309,616,895,194,277,985,554]
+  CRUSH rule 1 x 61 [154,843,717,467,883,536,812,14,55,752]
+  CRUSH rule 1 x 62 [594,811,549,276,693,917,45,723,926,180]
+  CRUSH rule 1 x 63 [646,67,884,925,941,434,705,268,140,942]
+  CRUSH rule 1 x 64 [175,542,155,837,594,197,451,891,654,294]
+  CRUSH rule 1 x 65 [745,619,131,867,269,62,862,221,66,354]
+  CRUSH rule 1 x 66 [275,468,23,35,328,432,334,656,719,810]
+  CRUSH rule 1 x 67 [246,958,524,493,636,227,783,593,814,970]
+  CRUSH rule 1 x 68 [711,473,403,228,835,126,705,114,981,267]
+  CRUSH rule 1 x 69 [493,924,850,939,950,105,871,361,533,433]
+  CRUSH rule 1 x 70 [30,499,644,33,804,654,684,411,114,42]
+  CRUSH rule 1 x 71 [984,883,574,716,575,391,587,264,446,572]
+  CRUSH rule 1 x 72 [71,286,942,363,628,632,642,529,966,919]
+  CRUSH rule 1 x 73 [922,618,3,371,464,442,835,705,61,745]
+  CRUSH rule 1 x 74 [629,414,185,573,678,338,633,560,565,410]
+  CRUSH rule 1 x 75 [222,20,174,820,312,361,366,258,711,657]
+  CRUSH rule 1 x 76 [262,366,339,290,718,143,735,953,188,516]
+  CRUSH rule 1 x 77 [638,469,992,280,773,892,197,690,426,681]
+  CRUSH rule 1 x 78 [324,511,788,7,308,228,183,917,464,518]
+  CRUSH rule 1 x 79 [577,990,64,94,447,924,339,24,581,969]
+  CRUSH rule 1 x 80 [501,95,278,903,631,842,51,766,822,687]
+  CRUSH rule 1 x 81 [506,812,9,698,173,664,247,963,0,732]
+  CRUSH rule 1 x 82 [222,145,80,785,835,745,580,51,939,278]
+  CRUSH rule 1 x 83 [71,634,61,91,856,529,66,197,698,318]
+  CRUSH rule 1 x 84 [49,761,773,368,318,708,681,618,723,516]
+  CRUSH rule 1 x 85 [985,896,708,861,325,307,567,908,514,355]
+  CRUSH rule 1 x 86 [537,745,93,524,466,356,38,326,385,899]
+  CRUSH rule 1 x 87 [997,317,463,626,685,845,909,49,28,698]
+  CRUSH rule 1 x 88 [957,350,890,857,375,176,99,737,942,647]
+  CRUSH rule 1 x 89 [399,730,148,314,159,982,320,921,812,908]
+  CRUSH rule 1 x 90 [943,706,683,267,579,141,412,184,529,127]
+  CRUSH rule 1 x 91 [22,368,149,928,140,529,495,299,812,743]
+  CRUSH rule 1 x 92 [532,424,426,773,623,197,167,634,781,242]
+  CRUSH rule 1 x 93 [218,489,405,681,549,201,343,949,51,732]
+  CRUSH rule 1 x 94 [181,96,102,515,776,365,82,422,738,933]
+  CRUSH rule 1 x 95 [343,957,820,139,334,37,648,661,46,112]
+  CRUSH rule 1 x 96 [861,270,87,797,0,245,204,750,322,75]
+  CRUSH rule 1 x 97 [459,706,45,328,274,605,83,542,131,240]
+  CRUSH rule 1 x 98 [327,867,353,948,728,280,270,511,586,230]
+  CRUSH rule 1 x 99 [974,133,468,906,235,988,37,138,326,603]
+  CRUSH rule 1 x 100 [32,445,547,371,960,885,9,168,590,873]
+  CRUSH rule 1 x 101 [142,90,337,950,970,570,12,369,9,872]
+  CRUSH rule 1 x 102 [172,129,139,22,403,867,923,106,653,999]
+  CRUSH rule 1 x 103 [630,47,161,356,911,421,933,231,520,303]
+  CRUSH rule 1 x 104 [758,133,278,11,947,799,401,85,139,855]
+  CRUSH rule 1 x 105 [843,604,47,33,401,632,434,121,488,644]
+  CRUSH rule 1 x 106 [28,681,193,679,990,343,878,493,550,484]
+  CRUSH rule 1 x 107 [74,320,85,819,315,253,589,614,814,970]
+  CRUSH rule 1 x 108 [875,593,575,517,107,153,631,996,630,597]
+  CRUSH rule 1 x 109 [411,985,811,720,198,666,856,296,122,477]
+  CRUSH rule 1 x 110 [440,774,799,660,715,167,510,472,270,753]
+  CRUSH rule 1 x 111 [405,742,276,359,936,360,18,949,341,837]
+  CRUSH rule 1 x 112 [143,181,922,545,185,303,725,413,187,840]
+  CRUSH rule 1 x 113 [153,846,160,903,789,897,738,253,213,541]
+  CRUSH rule 1 x 114 [804,892,939,20,312,692,598,418,641,891]
+  CRUSH rule 1 x 115 [588,508,958,580,232,722,421,39,241,881]
+  CRUSH rule 1 x 116 [327,148,637,486,712,464,9,448,816,609]
+  CRUSH rule 1 x 117 [95,594,989,131,714,275,725,142,304,591]
+  CRUSH rule 1 x 118 [80,957,897,239,359,432,766,210,528,252]
+  CRUSH rule 1 x 119 [386,932,951,768,679,300,570,278,867,489]
+  CRUSH rule 1 x 120 [366,312,653,936,71,241,49,126,410,33]
+  CRUSH rule 1 x 121 [129,154,847,16,471,481,424,868,469,183]
+  CRUSH rule 1 x 122 [873,1,110,939,90,412,551,43,590,51]
+  CRUSH rule 1 x 123 [533,415,789,600,713,800,877,248,753,395]
+  CRUSH rule 1 x 124 [461,691,898,723,957,759,482,254,158,641]
+  CRUSH rule 1 x 125 [342,599,830,402,615,994,736,737,508,150]
+  CRUSH rule 1 x 126 [819,781,822,548,279,255,689,209,99,479]
+  CRUSH rule 1 x 127 [437,893,585,707,353,189,909,809,553,785]
+  CRUSH rule 1 x 128 [679,994,982,550,991,324,666,691,899,665]
+  CRUSH rule 1 x 129 [380,685,947,302,698,144,149,983,904,70]
+  CRUSH rule 1 x 130 [992,52,466,867,998,777,270,425,864,38]
+  CRUSH rule 1 x 131 [469,90,208,599,829,656,203,667,528,387]
+  CRUSH rule 1 x 132 [571,250,316,535,54,418,922,597,680,25]
+  CRUSH rule 1 x 133 [964,728,329,902,108,118,14,444,709,592]
+  CRUSH rule 1 x 134 [999,19,716,963,323,559,893,281,226,739]
+  CRUSH rule 1 x 135 [634,101,52,938,413,573,712,649,27,274]
+  CRUSH rule 1 x 136 [114,889,692,768,694,279,846,890,151,872]
+  CRUSH rule 1 x 137 [839,8,959,280,922,870,363,323,153,238]
+  CRUSH rule 1 x 138 [967,949,138,451,292,548,400,885,907,214]
+  CRUSH rule 1 x 139 [308,711,736,247,632,126,384,58,373,121]
+  CRUSH rule 1 x 140 [764,936,926,55,331,115,178,532,883,380]
+  CRUSH rule 1 x 141 [423,302,112,216,603,873,193,258,445,451]
+  CRUSH rule 1 x 142 [252,821,715,340,635,668,424,515,751,746]
+  CRUSH rule 1 x 143 [33,808,518,477,325,316,266,70,210,61]
+  CRUSH rule 1 x 144 [472,88,969,162,401,771,697,610,203,382]
+  CRUSH rule 1 x 145 [242,208,252,604,266,743,577,348,1,323]
+  CRUSH rule 1 x 146 [290,70,570,384,934,856,929,196,880,458]
+  CRUSH rule 1 x 147 [447,352,657,493,467,918,514,546,861,796]
+  CRUSH rule 1 x 148 [212,644,432,658,109,275,352,820,857,282]
+  CRUSH rule 1 x 149 [9,775,87,35,260,646,406,556,532,750]
+  CRUSH rule 1 x 150 [166,456,582,144,324,340,484,553,315,504]
+  CRUSH rule 1 x 151 [811,875,307,20,782,229,671,883,204,12]
+  CRUSH rule 1 x 152 [449,617,223,9,182,407,807,50,206,368]
+  CRUSH rule 1 x 153 [523,537,695,627,959,613,942,864,388,639]
+  CRUSH rule 1 x 154 [208,559,874,597,243,706,443,98,27,120]
+  CRUSH rule 1 x 155 [569,325,192,296,367,848,58,641,186,553]
+  CRUSH rule 1 x 156 [488,121,521,213,595,837,271,229,961,163]
+  CRUSH rule 1 x 157 [140,723,633,260,487,856,384,446,836,917]
+  CRUSH rule 1 x 158 [786,451,320,239,667,632,899,902,956,424]
+  CRUSH rule 1 x 159 [134,664,517,821,667,944,209,641,228,213]
+  CRUSH rule 1 x 160 [690,112,414,990,183,590,242,999,974,652]
+  CRUSH rule 1 x 161 [324,912,397,423,991,284,909,642,188,143]
+  CRUSH rule 1 x 162 [748,567,284,183,463,336,148,88,764,40]
+  CRUSH rule 1 x 163 [575,499,31,816,749,737,587,854,482,522]
+  CRUSH rule 1 x 164 [314,489,308,326,51,568,110,329,361,743]
+  CRUSH rule 1 x 165 [116,209,750,53,813,640,524,389,185,893]
+  CRUSH rule 1 x 166 [352,706,701,810,718,527,548,676,448,991]
+  CRUSH rule 1 x 167 [27,743,174,142,551,1,935,266,883,77]
+  CRUSH rule 1 x 168 [953,898,880,660,500,799,667,463,818,819]
+  CRUSH rule 1 x 169 [912,147,266,547,331,770,601,909,60,255]
+  CRUSH rule 1 x 170 [421,515,828,844,151,981,835,840,548,588]
+  CRUSH rule 1 x 171 [488,584,880,964,936,196,100,910,446,541]
+  CRUSH rule 1 x 172 [366,443,957,66,162,693,36,356,274,976]
+  CRUSH rule 1 x 173 [863,291,625,287,158,496,471,529,359,571]
+  CRUSH rule 1 x 174 [263,555,650,410,339,616,780,932,573,814]
+  CRUSH rule 1 x 175 [875,961,361,575,33,109,51,211,409,865]
+  CRUSH rule 1 x 176 [745,83,701,680,250,420,240,316,337,361]
+  CRUSH rule 1 x 177 [128,244,41,123,422,902,756,647,45,752]
+  CRUSH rule 1 x 178 [155,41,264,777,314,564,856,992,696,784]
+  CRUSH rule 1 x 179 [593,833,202,183,971,38,724,923,450,340]
+  CRUSH rule 1 x 180 [154,734,17,831,824,522,736,846,926,129]
+  CRUSH rule 1 x 181 [289,675,723,800,166,712,168,224,705,185]
+  CRUSH rule 1 x 182 [730,931,560,209,943,261,485,571,796,587]
+  CRUSH rule 1 x 183 [639,237,794,815,827,400,109,903,96,526]
+  CRUSH rule 1 x 184 [704,312,685,645,691,778,74,45,438,26]
+  CRUSH rule 1 x 185 [97,100,762,82,999,542,485,511,14,329]
+  CRUSH rule 1 x 186 [26,665,554,215,280,421,369,270,16,920]
+  CRUSH rule 1 x 187 [649,14,740,494,402,684,566,378,816,553]
+  CRUSH rule 1 x 188 [682,695,590,743,927,945,833,650,761,468]
+  CRUSH rule 1 x 189 [325,693,726,51,448,169,37,1,939,463]
+  CRUSH rule 1 x 190 [399,933,136,955,57,504,527,237,295,716]
+  CRUSH rule 1 x 191 [629,533,17,126,60,146,999,754,339,271]
+  CRUSH rule 1 x 192 [503,578,38,492,222,251,123,759,147,99]
+  CRUSH rule 1 x 193 [546,333,651,678,823,652,359,721,996,318]
+  CRUSH rule 1 x 194 [242,473,58,655,907,277,792,887,561,449]
+  CRUSH rule 1 x 195 [625,719,135,81,636,513,755,471,658,744]
+  CRUSH rule 1 x 196 [357,114,125,867,250,522,413,834,832,368]
+  CRUSH rule 1 x 197 [306,954,453,873,211,334,666,316,243,320]
+  CRUSH rule 1 x 198 [863,791,311,911,206,61,355,574,781,550]
+  CRUSH rule 1 x 199 [935,906,929,252,893,75,960,369,584,612]
+  CRUSH rule 1 x 200 [373,774,229,454,909,611,132,271,128,632]
+  CRUSH rule 1 x 201 [659,320,477,313,779,16,495,76,598,301]
+  CRUSH rule 1 x 202 [260,433,524,880,223,818,153,272,944,741]
+  CRUSH rule 1 x 203 [36,239,675,971,703,209,669,676,762,200]
+  CRUSH rule 1 x 204 [92,516,993,728,279,478,697,881,64,107]
+  CRUSH rule 1 x 205 [68,395,473,45,683,662,776,463,327,721]
+  CRUSH rule 1 x 206 [570,530,642,380,311,398,230,367,890,953]
+  CRUSH rule 1 x 207 [834,457,850,917,456,296,76,708,101,928]
+  CRUSH rule 1 x 208 [927,484,640,976,803,626,96,841,811,979]
+  CRUSH rule 1 x 209 [878,66,58,940,48,233,522,185,949,590]
+  CRUSH rule 1 x 210 [572,981,484,29,0,426,14,921,544,334]
+  CRUSH rule 1 x 211 [107,597,780,857,895,57,922,372,581,629]
+  CRUSH rule 1 x 212 [389,107,838,624,698,562,857,894,60,426]
+  CRUSH rule 1 x 213 [497,717,567,728,905,134,687,903,620,572]
+  CRUSH rule 1 x 214 [798,65,254,572,32,393,579,79,258,42]
+  CRUSH rule 1 x 215 [233,419,283,638,520,891,982,826,488,314]
+  CRUSH rule 1 x 216 [494,464,742,523,459,174,973,898,556,293]
+  CRUSH rule 1 x 217 [352,396,309,938,66,41,264,6,603,317]
+  CRUSH rule 1 x 218 [895,864,988,650,593,740,34,497,108,180]
+  CRUSH rule 1 x 219 [222,534,277,242,658,482,697,805,976,758]
+  CRUSH rule 1 x 220 [281,19,584,563,858,965,686,982,0,32]
+  CRUSH rule 1 x 221 [64,928,963,130,312,394,61,559,846,994]
+  CRUSH rule 1 x 222 [40,544,161,199,861,644,597,904,897,376]
+  CRUSH rule 1 x 223 [645,556,159,417,46,135,465,429,614,711]
+  CRUSH rule 1 x 224 [647,165,957,263,961,576,329,320,645,829]
+  CRUSH rule 1 x 225 [219,714,858,747,461,175,606,465,354,404]
+  CRUSH rule 1 x 226 [372,511,181,277,695,404,876,984,491,784]
+  CRUSH rule 1 x 227 [925,156,714,863,257,74,966,217,501,536]
+  CRUSH rule 1 x 228 [682,404,839,263,521,195,261,389,281,467]
+  CRUSH rule 1 x 229 [880,838,770,891,236,542,262,884,215,687]
+  CRUSH rule 1 x 230 [328,659,916,468,646,572,93,880,959,111]
+  CRUSH rule 1 x 231 [320,383,669,109,627,621,50,182,541,483]
+  CRUSH rule 1 x 232 [924,846,394,319,43,519,106,877,130,387]
+  CRUSH rule 1 x 233 [948,652,575,838,498,395,796,835,714,751]
+  CRUSH rule 1 x 234 [484,943,42,575,936,180,103,95,634,844]
+  CRUSH rule 1 x 235 [750,65,590,168,870,308,471,753,350,224]
+  CRUSH rule 1 x 236 [551,787,490,136,370,833,573,128,154,326]
+  CRUSH rule 1 x 237 [390,157,166,251,752,75,327,509,325,245]
+  CRUSH rule 1 x 238 [570,6,989,707,514,905,894,884,824,343]
+  CRUSH rule 1 x 239 [729,959,376,975,496,49,426,427,736,836]
+  CRUSH rule 1 x 240 [981,241,156,767,631,576,450,677,659,183]
+  CRUSH rule 1 x 241 [310,816,641,177,996,454,413,136,411,549]
+  CRUSH rule 1 x 242 [161,63,642,837,763,458,234,756,496,779]
+  CRUSH rule 1 x 243 [180,394,33,683,189,419,799,21,13,874]
+  CRUSH rule 1 x 244 [52,174,685,189,78,310,785,107,816,89]
+  CRUSH rule 1 x 245 [523,121,915,84,386,409,605,837,1,141]
+  CRUSH rule 1 x 246 [362,893,390,487,817,88,989,999,138,674]
+  CRUSH rule 1 x 247 [382,184,116,34,143,15,590,840,586,594]
+  CRUSH rule 1 x 248 [129,114,852,469,359,291,713,237,468,340]
+  CRUSH rule 1 x 249 [159,683,91,856,475,369,886,650,827,663]
+  CRUSH rule 1 x 250 [404,945,569,955,228,910,270,619,450,707]
+  CRUSH rule 1 x 251 [661,225,738,757,37,642,58,354,16,905]
+  CRUSH rule 1 x 252 [961,226,542,103,945,885,838,131,387,664]
+  CRUSH rule 1 x 253 [651,97,225,364,189,248,797,675,452,129]
+  CRUSH rule 1 x 254 [123,33,741,692,599,11,605,453,987,316]
+  CRUSH rule 1 x 255 [314,649,891,855,517,344,607,95,121,784]
+  CRUSH rule 1 x 256 [315,215,651,126,470,849,189,627,592,241]
+  CRUSH rule 1 x 257 [825,264,867,301,529,409,291,732,224,841]
+  CRUSH rule 1 x 258 [624,789,370,723,131,982,863,427,873,223]
+  CRUSH rule 1 x 259 [602,542,70,563,947,723,77,191,669,61]
+  CRUSH rule 1 x 260 [717,878,43,56,377,481,533,646,475,686]
+  CRUSH rule 1 x 261 [145,517,20,903,786,939,516,136,87,410]
+  CRUSH rule 1 x 262 [223,1,561,420,805,16,88,534,289,498]
+  CRUSH rule 1 x 263 [462,211,405,508,787,669,773,979,719,421]
+  CRUSH rule 1 x 264 [654,471,266,662,135,564,715,916,633,121]
+  CRUSH rule 1 x 265 [302,794,704,798,659,487,833,987,445,23]
+  CRUSH rule 1 x 266 [202,132,884,209,551,984,7,557,76,987]
+  CRUSH rule 1 x 267 [282,938,657,113,672,993,972,645,882,451]
+  CRUSH rule 1 x 268 [338,309,356,278,928,797,715,536,983,688]
+  CRUSH rule 1 x 269 [738,122,266,200,894,118,146,14,414,236]
+  CRUSH rule 1 x 270 [707,982,946,196,407,804,476,571,314,538]
+  CRUSH rule 1 x 271 [705,432,364,735,512,595,263,138,526,607]
+  CRUSH rule 1 x 272 [756,545,942,56,542,449,710,779,161,222]
+  CRUSH rule 1 x 273 [197,502,527,721,239,648,982,735,58,1]
+  CRUSH rule 1 x 274 [992,44,653,573,527,702,370,990,320,52]
+  CRUSH rule 1 x 275 [544,789,170,434,23,926,992,823,321,784]
+  CRUSH rule 1 x 276 [658,467,577,268,336,5,634,98,457,487]
+  CRUSH rule 1 x 277 [143,490,880,483,928,272,783,648,927,285]
+  CRUSH rule 1 x 278 [492,647,355,282,834,64,350,600,283,422]
+  CRUSH rule 1 x 279 [517,792,604,987,527,894,952,250,206,714]
+  CRUSH rule 1 x 280 [825,740,27,848,514,750,895,914,892,149]
+  CRUSH rule 1 x 281 [224,629,120,562,616,200,443,604,52,638]
+  CRUSH rule 1 x 282 [298,661,380,416,35,585,939,879,338,786]
+  CRUSH rule 1 x 283 [311,606,208,50,913,678,369,544,721,267]
+  CRUSH rule 1 x 284 [771,466,371,743,672,119,60,546,39,71]
+  CRUSH rule 1 x 285 [693,362,404,676,797,531,582,975,810,703]
+  CRUSH rule 1 x 286 [364,477,285,167,270,617,699,627,725,389]
+  CRUSH rule 1 x 287 [591,611,828,995,170,987,137,890,487,621]
+  CRUSH rule 1 x 288 [965,541,848,796,251,668,195,538,356,523]
+  CRUSH rule 1 x 289 [225,551,948,877,219,167,795,377,825,874]
+  CRUSH rule 1 x 290 [577,762,777,751,291,349,473,209,59,346]
+  CRUSH rule 1 x 291 [160,903,477,381,490,559,557,86,89,417]
+  CRUSH rule 1 x 292 [873,598,216,666,222,228,806,911,738,969]
+  CRUSH rule 1 x 293 [100,234,874,47,28,452,775,636,232,518]
+  CRUSH rule 1 x 294 [285,943,379,520,725,547,459,833,503,207]
+  CRUSH rule 1 x 295 [938,262,880,327,687,3,440,73,29,38]
+  CRUSH rule 1 x 296 [850,327,86,472,1,776,266,82,671,320]
+  CRUSH rule 1 x 297 [951,53,99,558,753,228,232,343,831,540]
+  CRUSH rule 1 x 298 [173,336,85,766,910,657,213,286,61,961]
+  CRUSH rule 1 x 299 [598,591,315,386,895,296,924,106,63,457]
+  CRUSH rule 1 x 300 [531,957,62,459,156,538,904,838,458,828]
+  CRUSH rule 1 x 301 [823,628,23,858,629,808,220,432,393,433]
+  CRUSH rule 1 x 302 [184,80,780,871,531,211,400,365,697,497]
+  CRUSH rule 1 x 303 [521,766,222,830,988,275,561,905,522,342]
+  CRUSH rule 1 x 304 [980,127,807,507,555,245,214,944,845,895]
+  CRUSH rule 1 x 305 [153,816,22,927,696,911,685,838,3,983]
+  CRUSH rule 1 x 306 [423,739,664,753,178,431,761,648,867,488]
+  CRUSH rule 1 x 307 [997,557,682,456,479,631,459,250,415,194]
+  CRUSH rule 1 x 308 [991,874,534,465,330,284,976,551,126,307]
+  CRUSH rule 1 x 309 [860,394,724,858,246,866,857,153,970,99]
+  CRUSH rule 1 x 310 [589,818,546,201,94,653,90,855,441,736]
+  CRUSH rule 1 x 311 [477,774,225,590,830,559,256,798,743,645]
+  CRUSH rule 1 x 312 [887,853,950,354,58,23,497,929,92,639]
+  CRUSH rule 1 x 313 [802,646,447,416,557,118,24,81,215,850]
+  CRUSH rule 1 x 314 [654,974,229,511,562,916,952,599,201,763]
+  CRUSH rule 1 x 315 [767,227,28,740,828,156,749,841,969,314]
+  CRUSH rule 1 x 316 [778,83,733,359,858,319,761,725,923,461]
+  CRUSH rule 1 x 317 [184,418,642,986,939,675,892,86,214,189]
+  CRUSH rule 1 x 318 [525,410,500,543,212,95,290,97,529,220]
+  CRUSH rule 1 x 319 [476,724,569,382,409,521,800,868,364,427]
+  CRUSH rule 1 x 320 [149,610,697,296,818,955,523,366,891,998]
+  CRUSH rule 1 x 321 [710,79,667,671,234,4,868,841,563,961]
+  CRUSH rule 1 x 322 [175,275,323,333,744,718,187,380,947,952]
+  CRUSH rule 1 x 323 [819,604,638,792,316,544,236,307,969,232]
+  CRUSH rule 1 x 324 [16,745,511,439,272,932,668,959,845,759]
+  CRUSH rule 1 x 325 [486,400,872,873,251,68,462,268,124,431]
+  CRUSH rule 1 x 326 [613,765,207,19,359,370,461,509,75,767]
+  CRUSH rule 1 x 327 [125,289,738,408,456,784,750,669,296,314]
+  CRUSH rule 1 x 328 [807,383,476,583,645,141,33,806,181,597]
+  CRUSH rule 1 x 329 [588,938,599,432,446,840,516,713,223,395]
+  CRUSH rule 1 x 330 [932,644,41,611,209,406,420,520,395,665]
+  CRUSH rule 1 x 331 [341,953,950,537,578,862,624,649,626,928]
+  CRUSH rule 1 x 332 [153,726,459,950,466,804,644,821,238,85]
+  CRUSH rule 1 x 333 [745,845,853,860,52,615,243,633,309,616]
+  CRUSH rule 1 x 334 [614,751,807,58,396,159,408,175,189,500]
+  CRUSH rule 1 x 335 [518,721,221,283,454,187,635,367,997,819]
+  CRUSH rule 1 x 336 [389,424,77,309,5,898,698,533,683,851]
+  CRUSH rule 1 x 337 [753,508,765,720,221,807,956,907,464,39]
+  CRUSH rule 1 x 338 [128,810,490,753,406,760,69,11,624,272]
+  CRUSH rule 1 x 339 [430,308,58,751,856,823,607,953,125,899]
+  CRUSH rule 1 x 340 [541,44,630,231,289,966,707,328,325,81]
+  CRUSH rule 1 x 341 [402,26,631,439,165,928,720,503,209,748]
+  CRUSH rule 1 x 342 [982,57,992,461,131,32,516,661,985,860]
+  CRUSH rule 1 x 343 [833,412,572,732,107,805,660,655,149,994]
+  CRUSH rule 1 x 344 [784,533,792,41,642,869,142,114,108,961]
+  CRUSH rule 1 x 345 [546,300,304,691,763,556,127,732,290,494]
+  CRUSH rule 1 x 346 [302,420,428,891,357,124,419,962,304,12]
+  CRUSH rule 1 x 347 [488,778,101,217,366,442,783,661,622,426]
+  CRUSH rule 1 x 348 [903,744,937,718,85,314,862,513,112,334]
+  CRUSH rule 1 x 349 [471,547,582,306,600,486,795,143,529,765]
+  CRUSH rule 1 x 350 [348,221,823,335,383,708,841,164,765,563]
+  CRUSH rule 1 x 351 [961,582,705,346,361,32,766,775,518,155]
+  CRUSH rule 1 x 352 [728,137,461,298,36,903,899,665,802,620]
+  CRUSH rule 1 x 353 [904,202,184,447,58,294,279,616,892,262]
+  CRUSH rule 1 x 354 [345,226,319,256,544,311,612,33,122,190]
+  CRUSH rule 1 x 355 [50,430,175,43,187,458,985,412,599,375]
+  CRUSH rule 1 x 356 [87,185,55,423,829,1,629,228,150,889]
+  CRUSH rule 1 x 357 [762,459,921,473,182,231,891,656,196,232]
+  CRUSH rule 1 x 358 [908,25,280,6,808,676,874,643,550,633]
+  CRUSH rule 1 x 359 [484,15,132,121,394,423,397,52,702,981]
+  CRUSH rule 1 x 360 [173,378,337,702,145,499,29,529,156,595]
+  CRUSH rule 1 x 361 [404,577,115,25,56,914,643,286,552,985]
+  CRUSH rule 1 x 362 [403,1,422,945,132,685,265,35,662,708]
+  CRUSH rule 1 x 363 [639,911,510,162,418,294,444,613,466,499]
+  CRUSH rule 1 x 364 [752,689,610,990,665,222,203,17,743,570]
+  CRUSH rule 1 x 365 [956,999,212,230,624,84,113,373,426,941]
+  CRUSH rule 1 x 366 [860,925,924,763,687,851,59,914,521,629]
+  CRUSH rule 1 x 367 [205,609,647,665,969,720,685,641,894,813]
+  CRUSH rule 1 x 368 [301,284,810,169,78,340,616,93,283,353]
+  CRUSH rule 1 x 369 [452,658,339,217,674,210,284,184,718,684]
+  CRUSH rule 1 x 370 [11,467,695,989,394,576,850,419,307,965]
+  CRUSH rule 1 x 371 [124,487,55,514,313,411,797,547,778,958]
+  CRUSH rule 1 x 372 [253,48,979,846,207,631,212,241,346,153]
+  CRUSH rule 1 x 373 [715,605,775,748,227,493,128,207,88,641]
+  CRUSH rule 1 x 374 [191,887,920,340,223,714,961,760,571,549]
+  CRUSH rule 1 x 375 [711,385,651,665,15,71,934,619,527,735]
+  CRUSH rule 1 x 376 [597,818,49,458,415,755,446,897,460,869]
+  CRUSH rule 1 x 377 [294,256,933,771,184,861,654,487,891,733]
+  CRUSH rule 1 x 378 [34,151,681,707,552,127,728,860,968,475]
+  CRUSH rule 1 x 379 [869,136,315,378,813,153,115,557,165,292]
+  CRUSH rule 1 x 380 [294,97,575,791,690,482,255,806,429,306]
+  CRUSH rule 1 x 381 [119,710,219,827,328,886,773,496,433,750]
+  CRUSH rule 1 x 382 [69,631,508,706,697,168,276,56,278,772]
+  CRUSH rule 1 x 383 [922,588,589,925,471,601,29,197,822,218]
+  CRUSH rule 1 x 384 [221,945,671,117,857,655,488,435,223,783]
+  CRUSH rule 1 x 385 [561,737,953,723,658,368,910,329,396,482]
+  CRUSH rule 1 x 386 [335,442,788,696,507,716,232,692,742,939]
+  CRUSH rule 1 x 387 [514,43,353,88,100,842,164,934,297,902]
+  CRUSH rule 1 x 388 [587,89,157,996,915,927,474,267,640,53]
+  CRUSH rule 1 x 389 [109,641,255,466,372,563,340,222,74,503]
+  CRUSH rule 1 x 390 [925,149,421,489,599,810,852,196,469,672]
+  CRUSH rule 1 x 391 [267,87,387,527,768,873,886,136,818,516]
+  CRUSH rule 1 x 392 [382,485,370,849,936,636,901,82,695,640]
+  CRUSH rule 1 x 393 [425,721,221,753,268,463,652,543,10,287]
+  CRUSH rule 1 x 394 [898,18,38,793,173,738,15,591,420,525]
+  CRUSH rule 1 x 395 [806,876,269,679,32,744,126,179,607,623]
+  CRUSH rule 1 x 396 [790,970,437,449,875,395,726,935,278,138]
+  CRUSH rule 1 x 397 [136,363,507,613,11,30,996,558,602,528]
+  CRUSH rule 1 x 398 [914,116,558,258,722,904,349,672,826,569]
+  CRUSH rule 1 x 399 [261,94,299,202,174,622,749,410,815,214]
+  CRUSH rule 1 x 400 [661,197,338,461,977,848,536,592,886,981]
+  CRUSH rule 1 x 401 [953,979,287,803,41,349,79,32,343,468]
+  CRUSH rule 1 x 402 [738,819,618,522,667,334,658,449,886,260]
+  CRUSH rule 1 x 403 [573,238,425,546,130,68,202,650,501,628]
+  CRUSH rule 1 x 404 [526,848,790,253,922,820,299,577,563,37]
+  CRUSH rule 1 x 405 [582,505,330,334,201,110,776,296,19,972]
+  CRUSH rule 1 x 406 [768,324,493,60,186,165,718,578,580,249]
+  CRUSH rule 1 x 407 [260,951,437,587,692,648,72,345,709,89]
+  CRUSH rule 1 x 408 [657,81,770,734,830,821,246,695,76,647]
+  CRUSH rule 1 x 409 [498,89,182,423,672,152,213,806,168,907]
+  CRUSH rule 1 x 410 [28,793,737,352,166,645,949,507,361,615]
+  CRUSH rule 1 x 411 [684,992,60,659,769,267,313,351,497,571]
+  CRUSH rule 1 x 412 [261,958,699,950,165,14,560,155,661,678]
+  CRUSH rule 1 x 413 [891,835,297,441,384,979,618,907,9,291]
+  CRUSH rule 1 x 414 [127,459,119,965,662,594,97,124,229,641]
+  CRUSH rule 1 x 415 [272,540,631,328,609,568,694,332,572,681]
+  CRUSH rule 1 x 416 [739,617,115,530,339,371,889,344,838,541]
+  CRUSH rule 1 x 417 [106,209,157,878,117,128,138,374,470,59]
+  CRUSH rule 1 x 418 [525,441,147,390,320,300,848,972,781,361]
+  CRUSH rule 1 x 419 [603,673,615,465,266,855,823,884,832,361]
+  CRUSH rule 1 x 420 [988,213,251,226,209,245,506,670,285,2]
+  CRUSH rule 1 x 421 [761,521,748,368,923,992,764,274,623,613]
+  CRUSH rule 1 x 422 [317,160,924,548,198,709,839,547,599,779]
+  CRUSH rule 1 x 423 [137,807,168,472,619,443,905,588,312,114]
+  CRUSH rule 1 x 424 [920,37,146,263,598,748,785,395,884,360]
+  CRUSH rule 1 x 425 [277,693,285,221,478,165,80,236,988,682]
+  CRUSH rule 1 x 426 [485,936,407,854,726,524,791,565,352,949]
+  CRUSH rule 1 x 427 [242,515,9,564,174,453,334,588,571,428]
+  CRUSH rule 1 x 428 [632,635,26,473,494,478,225,94,303,757]
+  CRUSH rule 1 x 429 [641,73,465,127,171,397,857,562,976,977]
+  CRUSH rule 1 x 430 [626,585,6,387,881,583,859,699,91,148]
+  CRUSH rule 1 x 431 [697,76,753,570,964,339,194,366,279,30]
+  CRUSH rule 1 x 432 [590,526,306,283,656,728,513,591,599,474]
+  CRUSH rule 1 x 433 [284,387,149,817,886,714,52,897,705,256]
+  CRUSH rule 1 x 434 [538,985,79,953,770,468,644,646,747,123]
+  CRUSH rule 1 x 435 [30,318,593,635,975,833,371,731,906,721]
+  CRUSH rule 1 x 436 [164,919,851,693,0,874,10,976,284,126]
+  CRUSH rule 1 x 437 [322,212,163,606,302,282,443,23,696,245]
+  CRUSH rule 1 x 438 [142,392,85,594,376,419,755,841,94,52]
+  CRUSH rule 1 x 439 [119,370,68,443,997,837,414,152,331,985]
+  CRUSH rule 1 x 440 [333,403,187,863,475,844,800,174,117,518]
+  CRUSH rule 1 x 441 [477,727,906,145,429,91,205,236,86,929]
+  CRUSH rule 1 x 442 [274,590,933,244,434,49,864,799,762,611]
+  CRUSH rule 1 x 443 [983,748,574,718,700,442,774,350,37,929]
+  CRUSH rule 1 x 444 [536,509,431,146,170,149,182,145,347,172]
+  CRUSH rule 1 x 445 [485,311,528,209,964,753,554,931,638,892]
+  CRUSH rule 1 x 446 [345,634,42,294,711,376,314,714,212,646]
+  CRUSH rule 1 x 447 [61,845,767,600,321,716,58,531,827,968]
+  CRUSH rule 1 x 448 [333,232,292,846,364,951,807,688,21,841]
+  CRUSH rule 1 x 449 [680,16,484,670,851,500,258,548,905,988]
+  CRUSH rule 1 x 450 [235,214,79,423,96,822,721,31,312,491]
+  CRUSH rule 1 x 451 [961,468,333,640,823,151,878,33,3,917]
+  CRUSH rule 1 x 452 [525,479,153,528,570,806,604,49,922,414]
+  CRUSH rule 1 x 453 [138,466,302,86,249,154,514,5,494,960]
+  CRUSH rule 1 x 454 [137,625,215,402,389,914,106,103,511,624]
+  CRUSH rule 1 x 455 [173,150,997,16,846,888,295,967,132,319]
+  CRUSH rule 1 x 456 [235,226,238,258,347,784,504,96,890,230]
+  CRUSH rule 1 x 457 [450,577,253,413,717,609,762,975,485,228]
+  CRUSH rule 1 x 458 [195,537,91,814,351,90,399,558,15,441]
+  CRUSH rule 1 x 459 [381,555,312,573,915,623,147,483,517,733]
+  CRUSH rule 1 x 460 [972,730,534,678,756,692,841,512,70,914]
+  CRUSH rule 1 x 461 [506,279,142,830,784,124,385,797,917,561]
+  CRUSH rule 1 x 462 [692,959,578,57,983,299,240,911,375,412]
+  CRUSH rule 1 x 463 [788,667,949,550,685,702,538,111,232,539]
+  CRUSH rule 1 x 464 [133,122,588,999,270,880,789,0,653,566]
+  CRUSH rule 1 x 465 [971,190,230,777,452,914,137,466,531,493]
+  CRUSH rule 1 x 466 [394,576,148,157,103,822,659,35,797,235]
+  CRUSH rule 1 x 467 [517,28,366,362,984,521,187,640,601,622]
+  CRUSH rule 1 x 468 [829,143,874,225,162,413,201,249,555,646]
+  CRUSH rule 1 x 469 [987,936,106,725,633,238,681,851,551,768]
+  CRUSH rule 1 x 470 [107,982,56,889,67,65,558,71,676,655]
+  CRUSH rule 1 x 471 [181,897,629,860,307,116,256,978,409,691]
+  CRUSH rule 1 x 472 [547,512,172,24,705,837,809,56,476,137]
+  CRUSH rule 1 x 473 [760,997,824,905,888,755,756,663,167,196]
+  CRUSH rule 1 x 474 [787,418,743,628,272,341,446,333,245,689]
+  CRUSH rule 1 x 475 [662,312,253,617,105,58,237,764,682,318]
+  CRUSH rule 1 x 476 [110,495,185,508,961,837,984,226,333,916]
+  CRUSH rule 1 x 477 [393,954,834,132,841,367,753,794,237,996]
+  CRUSH rule 1 x 478 [246,483,480,644,985,420,941,843,751,451]
+  CRUSH rule 1 x 479 [70,929,697,931,744,487,158,489,515,496]
+  CRUSH rule 1 x 480 [753,119,961,607,317,717,371,807,687,932]
+  CRUSH rule 1 x 481 [470,429,677,242,574,757,135,375,613,657]
+  CRUSH rule 1 x 482 [451,566,961,675,354,746,731,233,640,492]
+  CRUSH rule 1 x 483 [816,72,371,278,635,30,448,437,219,982]
+  CRUSH rule 1 x 484 [540,454,389,31,654,494,283,170,278,77]
+  CRUSH rule 1 x 485 [74,582,624,684,566,677,866,661,581,943]
+  CRUSH rule 1 x 486 [958,595,199,763,715,973,621,955,400,261]
+  CRUSH rule 1 x 487 [228,302,804,833,876,647,857,782,24,970]
+  CRUSH rule 1 x 488 [180,529,722,956,353,890,924,965,25,925]
+  CRUSH rule 1 x 489 [47,617,812,187,291,828,154,478,512,528]
+  CRUSH rule 1 x 490 [905,822,479,124,750,843,566,779,936,507]
+  CRUSH rule 1 x 491 [892,370,609,998,433,957,188,563,490,369]
+  CRUSH rule 1 x 492 [588,959,127,948,505,936,591,423,668,365]
+  CRUSH rule 1 x 493 [353,461,593,291,301,830,231,893,474,946]
+  CRUSH rule 1 x 494 [378,848,443,368,507,423,389,819,956,597]
+  CRUSH rule 1 x 495 [845,653,768,234,405,367,823,789,217,720]
+  CRUSH rule 1 x 496 [13,988,0,691,389,757,129,763,39,651]
+  CRUSH rule 1 x 497 [796,877,788,394,648,829,542,745,131,753]
+  CRUSH rule 1 x 498 [412,337,270,705,511,227,949,173,398,586]
+  CRUSH rule 1 x 499 [330,695,8,74,618,101,440,509,295,921]
+  CRUSH rule 1 x 500 [820,272,547,765,755,96,930,573,357,491]
+  CRUSH rule 1 x 501 [110,44,132,442,294,423,880,279,616,919]
+  CRUSH rule 1 x 502 [336,595,650,274,993,312,490,852,962,387]
+  CRUSH rule 1 x 503 [922,211,157,722,502,971,262,926,316,527]
+  CRUSH rule 1 x 504 [483,52,122,432,778,461,758,104,831,710]
+  CRUSH rule 1 x 505 [482,598,224,279,480,310,764,558,891,406]
+  CRUSH rule 1 x 506 [493,123,43,856,936,622,898,161,78,414]
+  CRUSH rule 1 x 507 [12,598,264,422,416,947,591,702,346,619]
+  CRUSH rule 1 x 508 [227,157,611,301,223,746,313,282,207,626]
+  CRUSH rule 1 x 509 [807,242,363,122,582,530,798,808,139,377]
+  CRUSH rule 1 x 510 [134,437,227,75,313,351,786,152,921,884]
+  CRUSH rule 1 x 511 [212,54,83,799,457,218,600,968,355,109]
+  CRUSH rule 1 x 512 [236,630,758,752,361,249,899,451,415,920]
+  CRUSH rule 1 x 513 [994,693,644,938,846,685,52,185,197,986]
+  CRUSH rule 1 x 514 [45,508,831,19,817,52,374,985,944,101]
+  CRUSH rule 1 x 515 [504,138,480,272,530,377,481,820,517,850]
+  CRUSH rule 1 x 516 [285,409,136,570,841,610,453,660,93,134]
+  CRUSH rule 1 x 517 [300,232,23,906,438,236,519,737,20,892]
+  CRUSH rule 1 x 518 [397,674,98,898,967,113,625,434,527,630]
+  CRUSH rule 1 x 519 [86,750,772,913,101,864,375,328,3,688]
+  CRUSH rule 1 x 520 [900,833,614,130,261,885,558,956,664,468]
+  CRUSH rule 1 x 521 [31,47,236,751,911,599,495,354,665,945]
+  CRUSH rule 1 x 522 [390,16,280,144,291,175,753,624,769,853]
+  CRUSH rule 1 x 523 [618,308,424,590,300,206,834,212,906,305]
+  CRUSH rule 1 x 524 [635,189,687,963,601,518,8,550,769,975]
+  CRUSH rule 1 x 525 [311,916,699,262,775,32,45,478,911,233]
+  CRUSH rule 1 x 526 [48,738,227,718,244,942,853,643,625,43]
+  CRUSH rule 1 x 527 [202,851,889,216,763,351,270,35,809,509]
+  CRUSH rule 1 x 528 [565,827,590,273,918,106,651,368,118,1]
+  CRUSH rule 1 x 529 [934,864,241,43,466,924,278,926,280,321]
+  CRUSH rule 1 x 530 [502,934,298,670,986,360,577,509,195,722]
+  CRUSH rule 1 x 531 [681,627,942,487,288,561,925,474,669,212]
+  CRUSH rule 1 x 532 [422,6,147,205,861,141,949,374,988,367]
+  CRUSH rule 1 x 533 [863,68,364,983,247,199,54,931,4,279]
+  CRUSH rule 1 x 534 [962,931,775,172,663,119,206,682,627,827]
+  CRUSH rule 1 x 535 [89,565,397,693,839,632,859,30,61,75]
+  CRUSH rule 1 x 536 [499,351,760,458,918,86,148,668,436,192]
+  CRUSH rule 1 x 537 [676,547,787,311,867,748,152,797,492,926]
+  CRUSH rule 1 x 538 [58,644,571,649,941,7,37,485,88,273]
+  CRUSH rule 1 x 539 [837,953,457,711,458,621,528,722,59,237]
+  CRUSH rule 1 x 540 [831,50,132,213,197,709,95,789,348,342]
+  CRUSH rule 1 x 541 [582,757,121,525,532,963,738,277,225,142]
+  CRUSH rule 1 x 542 [472,132,790,997,948,269,137,934,547,351]
+  CRUSH rule 1 x 543 [382,272,797,330,315,748,324,134,839,685]
+  CRUSH rule 1 x 544 [947,930,496,883,509,219,250,362,614,123]
+  CRUSH rule 1 x 545 [425,570,305,77,821,422,117,172,764,372]
+  CRUSH rule 1 x 546 [18,65,529,437,343,547,699,610,785,811]
+  CRUSH rule 1 x 547 [445,715,600,472,213,851,428,267,229,379]
+  CRUSH rule 1 x 548 [367,569,980,167,627,442,517,684,154,108]
+  CRUSH rule 1 x 549 [125,715,671,817,285,420,37,639,934,330]
+  CRUSH rule 1 x 550 [425,599,744,199,923,222,915,570,546,724]
+  CRUSH rule 1 x 551 [44,1,528,922,944,115,161,901,342,941]
+  CRUSH rule 1 x 552 [246,104,68,239,123,427,57,217,21,70]
+  CRUSH rule 1 x 553 [71,703,615,28,593,724,218,916,561,416]
+  CRUSH rule 1 x 554 [207,124,217,166,525,226,693,953,606,894]
+  CRUSH rule 1 x 555 [570,28,317,420,931,413,623,659,403,573]
+  CRUSH rule 1 x 556 [674,152,421,79,215,347,830,762,691,951]
+  CRUSH rule 1 x 557 [347,817,191,391,741,571,593,267,17,386]
+  CRUSH rule 1 x 558 [627,426,369,692,815,371,124,107,766,260]
+  CRUSH rule 1 x 559 [940,630,924,242,224,912,185,356,87,113]
+  CRUSH rule 1 x 560 [295,903,541,29,245,753,887,376,658,366]
+  CRUSH rule 1 x 561 [506,682,384,637,878,991,700,339,687,507]
+  CRUSH rule 1 x 562 [718,529,87,729,842,341,62,817,766,376]
+  CRUSH rule 1 x 563 [552,332,747,206,274,871,903,900,812,290]
+  CRUSH rule 1 x 564 [835,769,736,486,630,209,641,751,930,856]
+  CRUSH rule 1 x 565 [8,167,539,182,607,62,738,873,47,84]
+  CRUSH rule 1 x 566 [600,481,301,263,90,450,184,127,448,327]
+  CRUSH rule 1 x 567 [999,994,509,899,947,24,267,639,646,85]
+  CRUSH rule 1 x 568 [252,431,157,62,601,863,398,521,59,250]
+  CRUSH rule 1 x 569 [643,218,943,455,83,969,494,624,352,562]
+  CRUSH rule 1 x 570 [617,635,765,422,250,156,533,674,23,683]
+  CRUSH rule 1 x 571 [757,80,59,98,328,700,329,848,235,502]
+  CRUSH rule 1 x 572 [299,348,575,889,943,675,33,312,202,355]
+  CRUSH rule 1 x 573 [25,505,270,167,58,901,878,978,1,291]
+  CRUSH rule 1 x 574 [215,431,624,177,628,814,333,841,193,146]
+  CRUSH rule 1 x 575 [225,252,611,546,32,815,389,486,10,402]
+  CRUSH rule 1 x 576 [627,94,159,857,430,691,177,545,839,722]
+  CRUSH rule 1 x 577 [237,809,778,636,61,167,700,521,825,444]
+  CRUSH rule 1 x 578 [885,313,120,344,771,614,487,976,977,58]
+  CRUSH rule 1 x 579 [924,575,787,831,47,996,557,630,468,348]
+  CRUSH rule 1 x 580 [718,51,766,121,118,471,608,755,326,604]
+  CRUSH rule 1 x 581 [219,807,129,571,856,179,874,902,958,415]
+  CRUSH rule 1 x 582 [893,701,598,863,285,829,984,622,175,804]
+  CRUSH rule 1 x 583 [246,930,964,170,993,409,469,193,737,681]
+  CRUSH rule 1 x 584 [336,432,680,175,495,839,642,226,122,703]
+  CRUSH rule 1 x 585 [324,999,397,485,457,527,73,628,884,255]
+  CRUSH rule 1 x 586 [558,230,976,541,816,72,794,682,127,372]
+  CRUSH rule 1 x 587 [985,830,597,21,308,890,952,421,875,65]
+  CRUSH rule 1 x 588 [211,544,57,134,162,496,195,581,649,488]
+  CRUSH rule 1 x 589 [129,21,112,190,885,844,753,180,160,465]
+  CRUSH rule 1 x 590 [467,969,652,593,287,76,811,413,436,162]
+  CRUSH rule 1 x 591 [758,514,316,164,35,110,54,796,369,958]
+  CRUSH rule 1 x 592 [525,253,190,443,315,603,667,318,496,74]
+  CRUSH rule 1 x 593 [601,885,339,152,297,223,269,455,168,635]
+  CRUSH rule 1 x 594 [227,60,450,30,717,840,994,16,777,901]
+  CRUSH rule 1 x 595 [720,854,496,912,80,655,917,525,945,715]
+  CRUSH rule 1 x 596 [751,195,997,77,261,490,180,482,449,647]
+  CRUSH rule 1 x 597 [129,574,714,8,789,847,725,991,955,316]
+  CRUSH rule 1 x 598 [679,207,604,396,841,284,286,280,507,912]
+  CRUSH rule 1 x 599 [668,315,683,349,681,253,599,364,546,849]
+  CRUSH rule 1 x 600 [143,396,464,444,59,57,243,264,31,897]
+  CRUSH rule 1 x 601 [326,573,873,902,136,921,633,596,988,727]
+  CRUSH rule 1 x 602 [860,281,875,535,672,474,697,763,442,542]
+  CRUSH rule 1 x 603 [709,328,445,349,190,455,924,667,356,316]
+  CRUSH rule 1 x 604 [571,62,814,95,866,978,983,281,292,953]
+  CRUSH rule 1 x 605 [252,739,860,27,313,362,857,899,349,926]
+  CRUSH rule 1 x 606 [339,236,759,842,67,644,954,94,88,617]
+  CRUSH rule 1 x 607 [590,248,759,868,433,398,578,386,226,269]
+  CRUSH rule 1 x 608 [145,635,309,467,875,115,148,33,420,669]
+  CRUSH rule 1 x 609 [973,547,223,79,762,863,249,41,778,929]
+  CRUSH rule 1 x 610 [435,816,961,983,255,886,160,888,597,767]
+  CRUSH rule 1 x 611 [559,283,422,584,176,429,570,43,362,401]
+  CRUSH rule 1 x 612 [273,149,123,576,911,270,296,735,245,714]
+  CRUSH rule 1 x 613 [828,614,642,674,33,361,958,580,197,897]
+  CRUSH rule 1 x 614 [478,748,393,34,171,80,92,12,62,719]
+  CRUSH rule 1 x 615 [392,155,144,326,626,134,149,401,14,59]
+  CRUSH rule 1 x 616 [778,637,452,248,15,888,74,307,976,613]
+  CRUSH rule 1 x 617 [622,713,996,833,611,407,364,8,342,512]
+  CRUSH rule 1 x 618 [149,877,270,329,180,327,222,749,697,853]
+  CRUSH rule 1 x 619 [604,163,656,409,322,848,519,967,737,892]
+  CRUSH rule 1 x 620 [181,23,409,198,64,898,35,620,268,902]
+  CRUSH rule 1 x 621 [735,902,386,237,939,475,725,118,875,359]
+  CRUSH rule 1 x 622 [661,824,717,568,858,583,446,798,869,586]
+  CRUSH rule 1 x 623 [142,121,643,61,695,852,485,478,185,854]
+  CRUSH rule 1 x 624 [360,716,420,398,49,717,137,140,488,725]
+  CRUSH rule 1 x 625 [541,167,385,1,601,481,308,111,207,48]
+  CRUSH rule 1 x 626 [364,431,610,363,535,747,225,841,868,249]
+  CRUSH rule 1 x 627 [458,137,557,410,287,749,467,432,944,781]
+  CRUSH rule 1 x 628 [250,350,556,497,821,65,205,580,972,427]
+  CRUSH rule 1 x 629 [928,160,710,572,365,772,538,46,300,112]
+  CRUSH rule 1 x 630 [243,19,918,556,601,16,920,830,171,759]
+  CRUSH rule 1 x 631 [438,221,574,676,797,580,219,211,157,614]
+  CRUSH rule 1 x 632 [797,368,247,5,32,102,416,45,624,253]
+  CRUSH rule 1 x 633 [993,749,525,485,27,330,275,599,219,357]
+  CRUSH rule 1 x 634 [239,351,633,299,651,678,296,337,676,416]
+  CRUSH rule 1 x 635 [640,965,25,961,306,172,849,357,317,599]
+  CRUSH rule 1 x 636 [173,290,297,991,937,823,236,318,228,575]
+  CRUSH rule 1 x 637 [0,918,98,108,111,495,887,57,16,319]
+  CRUSH rule 1 x 638 [702,235,424,900,983,754,701,887,355,632]
+  CRUSH rule 1 x 639 [475,687,31,785,918,611,27,214,226,515]
+  CRUSH rule 1 x 640 [31,664,399,677,123,609,858,138,726,1]
+  CRUSH rule 1 x 641 [296,473,108,963,341,876,897,449,42,193]
+  CRUSH rule 1 x 642 [894,273,427,606,677,670,610,665,299,852]
+  CRUSH rule 1 x 643 [117,111,732,191,114,153,500,631,833,439]
+  CRUSH rule 1 x 644 [438,336,327,512,599,862,660,857,123,910]
+  CRUSH rule 1 x 645 [982,702,351,573,907,915,279,317,414,917]
+  CRUSH rule 1 x 646 [334,804,146,842,697,638,720,135,369,711]
+  CRUSH rule 1 x 647 [933,787,185,334,752,285,372,890,30,747]
+  CRUSH rule 1 x 648 [22,444,400,862,207,842,453,732,262,803]
+  CRUSH rule 1 x 649 [503,229,213,460,639,760,722,748,599,556]
+  CRUSH rule 1 x 650 [328,659,420,443,739,950,869,150,743,438]
+  CRUSH rule 1 x 651 [3,880,823,123,378,585,715,221,31,92]
+  CRUSH rule 1 x 652 [495,977,563,733,92,997,119,818,459,782]
+  CRUSH rule 1 x 653 [185,718,804,280,975,912,198,291,71,792]
+  CRUSH rule 1 x 654 [130,528,380,81,906,511,773,506,546,266]
+  CRUSH rule 1 x 655 [560,872,454,504,319,284,605,214,833,862]
+  CRUSH rule 1 x 656 [219,885,178,981,863,508,708,6,746,734]
+  CRUSH rule 1 x 657 [233,684,813,490,208,941,858,16,128,144]
+  CRUSH rule 1 x 658 [778,6,756,380,750,836,547,850,499,125]
+  CRUSH rule 1 x 659 [240,663,306,540,789,902,170,954,22,394]
+  CRUSH rule 1 x 660 [244,855,196,147,678,323,63,859,215,171]
+  CRUSH rule 1 x 661 [184,270,128,398,910,230,402,205,609,831]
+  CRUSH rule 1 x 662 [65,883,921,438,79,957,464,902,276,289]
+  CRUSH rule 1 x 663 [323,721,594,812,43,992,170,65,906,943]
+  CRUSH rule 1 x 664 [865,113,512,51,427,123,585,260,254,209]
+  CRUSH rule 1 x 665 [420,850,591,475,202,733,798,658,28,334]
+  CRUSH rule 1 x 666 [319,767,246,3,369,493,796,56,736,0]
+  CRUSH rule 1 x 667 [875,39,343,100,829,2,795,783,386,956]
+  CRUSH rule 1 x 668 [331,122,263,599,355,484,943,554,395,713]
+  CRUSH rule 1 x 669 [915,521,402,747,673,445,938,600,517,49]
+  CRUSH rule 1 x 670 [845,659,943,447,401,322,168,302,681,978]
+  CRUSH rule 1 x 671 [108,634,527,363,856,238,755,330,584,525]
+  CRUSH rule 1 x 672 [578,216,110,589,302,137,954,315,735,751]
+  CRUSH rule 1 x 673 [442,74,579,797,622,950,371,402,725,870]
+  CRUSH rule 1 x 674 [588,364,281,308,645,631,229,506,565,362]
+  CRUSH rule 1 x 675 [489,698,744,671,870,174,528,875,982,782]
+  CRUSH rule 1 x 676 [928,911,40,180,722,729,673,569,701,403]
+  CRUSH rule 1 x 677 [399,269,692,131,615,136,103,763,527,83]
+  CRUSH rule 1 x 678 [546,752,544,155,5,463,666,352,576,959]
+  CRUSH rule 1 x 679 [988,25,275,433,628,57,247,620,437,29]
+  CRUSH rule 1 x 680 [335,963,382,486,749,257,795,347,831,761]
+  CRUSH rule 1 x 681 [690,462,623,466,49,471,774,192,454,380]
+  CRUSH rule 1 x 682 [196,588,154,257,807,776,367,718,345,677]
+  CRUSH rule 1 x 683 [627,25,421,160,873,102,345,599,30,892]
+  CRUSH rule 1 x 684 [38,804,592,158,991,264,652,821,641,757]
+  CRUSH rule 1 x 685 [841,368,548,362,166,211,154,121,937,804]
+  CRUSH rule 1 x 686 [336,287,525,440,166,993,911,638,690,393]
+  CRUSH rule 1 x 687 [20,682,924,653,356,16,917,622,156,826]
+  CRUSH rule 1 x 688 [463,371,780,556,385,883,115,248,566,11]
+  CRUSH rule 1 x 689 [569,250,78,816,847,775,333,161,74,907]
+  CRUSH rule 1 x 690 [551,144,587,263,378,394,970,639,835,238]
+  CRUSH rule 1 x 691 [766,464,446,533,449,541,451,290,789,853]
+  CRUSH rule 1 x 692 [739,634,18,245,624,35,268,525,425,499]
+  CRUSH rule 1 x 693 [339,297,118,330,817,91,828,276,264,237]
+  CRUSH rule 1 x 694 [405,26,830,181,533,166,488,804,501,885]
+  CRUSH rule 1 x 695 [622,576,597,535,600,593,300,989,804,72]
+  CRUSH rule 1 x 696 [558,902,689,13,715,28,664,489,598,261]
+  CRUSH rule 1 x 697 [818,222,406,691,427,863,153,922,986,480]
+  CRUSH rule 1 x 698 [178,48,402,233,841,604,468,180,783,915]
+  CRUSH rule 1 x 699 [450,244,180,919,276,332,747,453,519,100]
+  CRUSH rule 1 x 700 [502,771,987,706,416,240,68,641,109,182]
+  CRUSH rule 1 x 701 [4,612,782,216,853,303,585,513,907,414]
+  CRUSH rule 1 x 702 [177,630,232,923,281,708,466,687,742,170]
+  CRUSH rule 1 x 703 [354,178,389,393,778,803,796,607,894,1]
+  CRUSH rule 1 x 704 [646,601,156,171,603,116,655,595,888,354]
+  CRUSH rule 1 x 705 [921,401,890,265,244,690,372,253,807,28]
+  CRUSH rule 1 x 706 [652,877,562,452,26,323,923,770,516,982]
+  CRUSH rule 1 x 707 [345,745,67,716,789,576,2,133,256,374]
+  CRUSH rule 1 x 708 [333,607,180,469,170,555,939,331,41,175]
+  CRUSH rule 1 x 709 [45,187,302,115,896,579,733,607,763,845]
+  CRUSH rule 1 x 710 [94,855,43,199,18,948,449,28,731,573]
+  CRUSH rule 1 x 711 [227,653,731,150,380,842,534,110,639,452]
+  CRUSH rule 1 x 712 [398,953,136,870,181,408,895,459,341,833]
+  CRUSH rule 1 x 713 [116,800,503,662,635,579,53,839,56,829]
+  CRUSH rule 1 x 714 [111,629,866,709,902,557,875,649,23,79]
+  CRUSH rule 1 x 715 [531,291,486,382,192,807,322,417,973,582]
+  CRUSH rule 1 x 716 [169,541,291,42,343,724,138,197,32,415]
+  CRUSH rule 1 x 717 [417,446,994,894,239,494,237,62,327,958]
+  CRUSH rule 1 x 718 [992,383,298,844,377,463,544,891,210,370]
+  CRUSH rule 1 x 719 [936,674,324,759,194,409,828,975,119,87]
+  CRUSH rule 1 x 720 [370,188,174,464,644,218,214,76,870,779]
+  CRUSH rule 1 x 721 [320,859,278,259,170,957,177,264,867,327]
+  CRUSH rule 1 x 722 [7,2,673,129,96,445,823,833,1,774]
+  CRUSH rule 1 x 723 [270,553,831,662,38,101,985,846,77,467]
+  CRUSH rule 1 x 724 [666,822,708,895,633,800,616,879,480,309]
+  CRUSH rule 1 x 725 [794,406,875,459,981,751,359,687,720,128]
+  CRUSH rule 1 x 726 [420,556,341,292,240,68,966,535,669,74]
+  CRUSH rule 1 x 727 [561,461,129,635,965,610,105,31,506,430]
+  CRUSH rule 1 x 728 [951,330,196,756,589,849,753,760,254,379]
+  CRUSH rule 1 x 729 [656,644,436,591,27,119,572,933,434,816]
+  CRUSH rule 1 x 730 [3,558,629,184,50,765,760,800,945,743]
+  CRUSH rule 1 x 731 [852,89,75,735,713,113,528,890,625,535]
+  CRUSH rule 1 x 732 [983,840,869,976,697,307,368,271,778,172]
+  CRUSH rule 1 x 733 [285,396,388,122,387,364,880,343,590,539]
+  CRUSH rule 1 x 734 [125,510,402,640,676,501,535,627,224,790]
+  CRUSH rule 1 x 735 [417,773,686,504,459,912,690,59,294,569]
+  CRUSH rule 1 x 736 [749,396,632,550,779,109,845,278,559,613]
+  CRUSH rule 1 x 737 [644,991,946,135,448,903,482,564,259,896]
+  CRUSH rule 1 x 738 [449,683,290,220,245,525,429,397,872,716]
+  CRUSH rule 1 x 739 [341,220,641,454,740,661,146,17,314,156]
+  CRUSH rule 1 x 740 [874,524,674,650,472,282,214,494,593,155]
+  CRUSH rule 1 x 741 [189,472,712,798,715,757,863,571,876,528]
+  CRUSH rule 1 x 742 [912,581,114,447,730,21,687,81,145,695]
+  CRUSH rule 1 x 743 [654,914,425,441,763,39,451,631,911,829]
+  CRUSH rule 1 x 744 [725,295,579,377,162,447,843,699,24,714]
+  CRUSH rule 1 x 745 [787,858,850,506,612,735,926,314,771,910]
+  CRUSH rule 1 x 746 [757,848,704,30,47,940,450,651,105,921]
+  CRUSH rule 1 x 747 [700,81,867,681,801,64,879,857,727,565]
+  CRUSH rule 1 x 748 [557,436,238,664,293,865,304,999,685,843]
+  CRUSH rule 1 x 749 [772,622,337,42,156,302,383,506,570,828]
+  CRUSH rule 1 x 750 [946,97,376,677,316,670,169,171,9,58]
+  CRUSH rule 1 x 751 [996,618,343,911,83,22,388,17,892,537]
+  CRUSH rule 1 x 752 [746,887,695,868,610,950,88,315,728,669]
+  CRUSH rule 1 x 753 [741,14,463,479,172,192,481,702,431,675]
+  CRUSH rule 1 x 754 [648,349,333,355,65,63,336,724,262,61]
+  CRUSH rule 1 x 755 [157,460,466,187,959,674,192,279,371,970]
+  CRUSH rule 1 x 756 [416,97,197,497,227,3,850,191,991,63]
+  CRUSH rule 1 x 757 [599,839,776,410,256,823,121,690,544,28]
+  CRUSH rule 1 x 758 [994,218,620,256,361,749,165,686,449,831]
+  CRUSH rule 1 x 759 [959,682,514,745,100,519,15,347,311,752]
+  CRUSH rule 1 x 760 [518,943,215,83,706,137,345,69,39,199]
+  CRUSH rule 1 x 761 [285,849,420,324,987,338,373,361,684,654]
+  CRUSH rule 1 x 762 [591,313,41,335,110,696,664,350,339,980]
+  CRUSH rule 1 x 763 [908,411,200,740,292,295,387,775,797,990]
+  CRUSH rule 1 x 764 [787,234,894,485,883,711,70,202,557,471]
+  CRUSH rule 1 x 765 [327,921,882,393,444,792,402,123,902,592]
+  CRUSH rule 1 x 766 [84,161,878,704,416,144,357,310,890,850]
+  CRUSH rule 1 x 767 [370,895,702,701,890,2,251,951,675,322]
+  CRUSH rule 1 x 768 [826,760,879,864,460,474,645,975,947,199]
+  CRUSH rule 1 x 769 [67,768,663,735,814,66,213,527,546,42]
+  CRUSH rule 1 x 770 [593,909,482,259,5,550,961,324,309,772]
+  CRUSH rule 1 x 771 [309,935,121,578,937,685,933,571,822,256]
+  CRUSH rule 1 x 772 [12,125,797,301,348,419,891,959,487,355]
+  CRUSH rule 1 x 773 [253,466,820,549,591,193,783,951,982,160]
+  CRUSH rule 1 x 774 [164,390,705,109,881,505,890,425,599,485]
+  CRUSH rule 1 x 775 [703,47,43,973,643,406,885,976,936,221]
+  CRUSH rule 1 x 776 [728,231,80,916,2,850,396,76,680,108]
+  CRUSH rule 1 x 777 [981,621,568,729,869,952,563,860,388,456]
+  CRUSH rule 1 x 778 [411,456,544,597,789,784,65,954,125,358]
+  CRUSH rule 1 x 779 [346,121,519,921,587,48,772,645,254,759]
+  CRUSH rule 1 x 780 [476,39,288,381,303,29,17,336,147,829]
+  CRUSH rule 1 x 781 [10,130,585,844,729,705,714,954,271,58]
+  CRUSH rule 1 x 782 [462,246,581,902,623,877,812,516,774,985]
+  CRUSH rule 1 x 783 [580,373,153,775,668,661,626,961,576,119]
+  CRUSH rule 1 x 784 [413,113,978,990,994,56,481,198,171,944]
+  CRUSH rule 1 x 785 [341,856,332,354,59,581,632,151,586,360]
+  CRUSH rule 1 x 786 [411,140,313,393,215,618,490,481,627,740]
+  CRUSH rule 1 x 787 [605,522,211,813,636,224,600,528,966,556]
+  CRUSH rule 1 x 788 [226,545,35,142,726,851,194,216,486,782]
+  CRUSH rule 1 x 789 [545,320,414,702,731,277,237,916,374,670]
+  CRUSH rule 1 x 790 [414,748,816,327,130,115,788,164,691,329]
+  CRUSH rule 1 x 791 [660,906,406,697,916,322,124,401,742,990]
+  CRUSH rule 1 x 792 [287,392,514,204,75,789,406,858,694,351]
+  CRUSH rule 1 x 793 [631,133,850,713,720,487,376,812,886,264]
+  CRUSH rule 1 x 794 [931,517,543,210,963,898,811,459,344,719]
+  CRUSH rule 1 x 795 [551,962,477,948,425,434,268,94,648,402]
+  CRUSH rule 1 x 796 [814,4,95,27,368,300,646,451,67,738]
+  CRUSH rule 1 x 797 [64,201,299,734,605,864,596,196,93,636]
+  CRUSH rule 1 x 798 [422,530,114,431,565,716,473,250,839,895]
+  CRUSH rule 1 x 799 [824,32,679,562,266,549,859,994,831,60]
+  CRUSH rule 1 x 800 [862,623,489,637,861,196,941,643,398,325]
+  CRUSH rule 1 x 801 [145,550,329,324,734,160,219,662,142,28]
+  CRUSH rule 1 x 802 [570,19,847,308,387,518,846,53,783,511]
+  CRUSH rule 1 x 803 [151,812,662,358,880,349,834,881,23,229]
+  CRUSH rule 1 x 804 [467,93,264,863,176,842,663,949,380,39]
+  CRUSH rule 1 x 805 [621,223,938,809,591,686,121,157,934,660]
+  CRUSH rule 1 x 806 [898,957,805,430,499,584,640,607,790,832]
+  CRUSH rule 1 x 807 [354,531,422,159,921,431,802,136,305,983]
+  CRUSH rule 1 x 808 [7,96,76,897,446,2,166,929,234,460]
+  CRUSH rule 1 x 809 [70,734,719,56,687,21,23,145,184,465]
+  CRUSH rule 1 x 810 [701,18,972,327,771,649,620,648,433,997]
+  CRUSH rule 1 x 811 [248,547,103,728,901,264,948,202,521,278]
+  CRUSH rule 1 x 812 [230,576,821,566,993,762,675,28,263,410]
+  CRUSH rule 1 x 813 [805,114,683,629,99,462,285,450,948,742]
+  CRUSH rule 1 x 814 [54,619,973,741,497,894,401,266,905,320]
+  CRUSH rule 1 x 815 [679,412,613,132,969,411,314,670,928,727]
+  CRUSH rule 1 x 816 [919,448,826,414,36,289,44,822,332,959]
+  CRUSH rule 1 x 817 [765,830,436,521,332,458,260,172,193,516]
+  CRUSH rule 1 x 818 [415,566,644,687,692,414,769,826,519,277]
+  CRUSH rule 1 x 819 [721,319,865,750,546,859,523,770,56,437]
+  CRUSH rule 1 x 820 [218,301,333,190,686,179,535,787,267,46]
+  CRUSH rule 1 x 821 [185,795,680,953,329,750,621,815,313,916]
+  CRUSH rule 1 x 822 [356,261,54,522,900,103,883,112,601,15]
+  CRUSH rule 1 x 823 [220,281,549,456,64,306,282,641,216,929]
+  CRUSH rule 1 x 824 [292,809,887,74,776,788,559,886,753,749]
+  CRUSH rule 1 x 825 [949,778,101,311,110,480,161,998,370,10]
+  CRUSH rule 1 x 826 [767,818,833,927,356,954,910,63,288,836]
+  CRUSH rule 1 x 827 [631,83,406,635,657,713,212,916,692,653]
+  CRUSH rule 1 x 828 [288,986,445,26,414,607,937,595,935,672]
+  CRUSH rule 1 x 829 [990,667,915,694,974,453,669,330,822,36]
+  CRUSH rule 1 x 830 [152,571,778,505,685,209,448,55,965,851]
+  CRUSH rule 1 x 831 [814,563,630,97,582,107,142,157,957,330]
+  CRUSH rule 1 x 832 [235,641,616,110,979,844,656,135,341,922]
+  CRUSH rule 1 x 833 [657,565,922,140,825,457,764,766,853,890]
+  CRUSH rule 1 x 834 [907,231,644,13,617,130,83,483,811,98]
+  CRUSH rule 1 x 835 [784,262,771,264,612,238,537,937,101,507]
+  CRUSH rule 1 x 836 [951,158,366,710,43,427,351,961,52,44]
+  CRUSH rule 1 x 837 [556,498,334,633,895,627,903,29,454,647]
+  CRUSH rule 1 x 838 [329,274,964,547,119,342,983,998,320,935]
+  CRUSH rule 1 x 839 [568,209,939,364,658,747,47,859,402,947]
+  CRUSH rule 1 x 840 [45,579,842,70,655,862,815,109,762,642]
+  CRUSH rule 1 x 841 [652,702,24,605,152,93,226,46,918,220]
+  CRUSH rule 1 x 842 [629,984,314,895,408,897,575,1,312,542]
+  CRUSH rule 1 x 843 [799,690,688,648,151,812,486,199,966,501]
+  CRUSH rule 1 x 844 [694,600,534,700,569,11,899,382,851,472]
+  CRUSH rule 1 x 845 [332,30,179,93,951,324,611,512,855,760]
+  CRUSH rule 1 x 846 [452,251,712,719,404,739,606,237,414,844]
+  CRUSH rule 1 x 847 [399,681,847,739,13,555,363,893,592,634]
+  CRUSH rule 1 x 848 [303,138,440,346,547,216,700,249,214,100]
+  CRUSH rule 1 x 849 [666,346,708,873,64,694,847,463,995,314]
+  CRUSH rule 1 x 850 [644,511,345,844,545,337,358,35,913,310]
+  CRUSH rule 1 x 851 [527,546,737,425,100,331,95,337,677,275]
+  CRUSH rule 1 x 852 [31,809,94,618,156,853,469,511,999,340]
+  CRUSH rule 1 x 853 [483,330,869,184,46,942,774,679,616,492]
+  CRUSH rule 1 x 854 [697,953,968,143,502,955,441,302,437,53]
+  CRUSH rule 1 x 855 [837,996,239,621,32,191,686,702,919,971]
+  CRUSH rule 1 x 856 [712,40,547,430,195,857,224,810,404,126]
+  CRUSH rule 1 x 857 [77,984,576,551,568,96,12,763,594,668]
+  CRUSH rule 1 x 858 [412,384,841,465,572,576,688,61,545,491]
+  CRUSH rule 1 x 859 [173,760,26,300,87,567,463,903,272,8]
+  CRUSH rule 1 x 860 [776,429,328,917,658,783,699,907,532,627]
+  CRUSH rule 1 x 861 [705,405,477,50,73,714,901,487,725,204]
+  CRUSH rule 1 x 862 [809,44,788,938,964,177,490,409,15,842]
+  CRUSH rule 1 x 863 [349,496,963,178,675,853,172,980,772,115]
+  CRUSH rule 1 x 864 [717,858,101,239,992,244,43,15,29,974]
+  CRUSH rule 1 x 865 [857,603,586,262,550,289,850,40,170,31]
+  CRUSH rule 1 x 866 [394,304,71,96,642,155,255,481,435,119]
+  CRUSH rule 1 x 867 [640,773,663,974,261,296,988,730,753,888]
+  CRUSH rule 1 x 868 [613,950,712,663,448,460,643,547,734,16]
+  CRUSH rule 1 x 869 [973,889,524,22,671,477,718,431,968,472]
+  CRUSH rule 1 x 870 [505,35,386,498,348,503,54,992,726,783]
+  CRUSH rule 1 x 871 [239,264,262,773,781,734,387,515,98,232]
+  CRUSH rule 1 x 872 [21,767,456,748,783,797,180,800,521,270]
+  CRUSH rule 1 x 873 [954,666,980,264,435,233,199,358,805,255]
+  CRUSH rule 1 x 874 [54,510,947,1,500,119,93,915,801,43]
+  CRUSH rule 1 x 875 [809,418,452,462,88,673,634,435,778,884]
+  CRUSH rule 1 x 876 [483,457,61,248,523,277,322,141,82,412]
+  CRUSH rule 1 x 877 [542,531,952,939,710,179,181,460,459,527]
+  CRUSH rule 1 x 878 [217,674,857,644,678,809,329,591,59,4]
+  CRUSH rule 1 x 879 [999,475,134,250,319,357,145,750,54,997]
+  CRUSH rule 1 x 880 [678,573,935,385,570,651,319,630,888,970]
+  CRUSH rule 1 x 881 [394,835,789,802,587,155,570,109,896,826]
+  CRUSH rule 1 x 882 [467,382,353,56,979,674,974,483,412,547]
+  CRUSH rule 1 x 883 [802,744,237,337,50,96,202,148,129,72]
+  CRUSH rule 1 x 884 [653,660,638,700,31,558,389,381,347,314]
+  CRUSH rule 1 x 885 [898,704,307,445,879,872,174,972,544,894]
+  CRUSH rule 1 x 886 [434,357,938,641,737,8,56,582,915,541]
+  CRUSH rule 1 x 887 [297,226,711,428,370,318,472,947,35,528]
+  CRUSH rule 1 x 888 [863,324,443,213,902,25,806,53,385,387]
+  CRUSH rule 1 x 889 [105,102,308,163,947,548,399,382,761,907]
+  CRUSH rule 1 x 890 [550,248,606,704,615,708,996,561,485,482]
+  CRUSH rule 1 x 891 [575,928,880,891,826,763,706,701,501,680]
+  CRUSH rule 1 x 892 [259,862,133,271,292,162,53,333,458,77]
+  CRUSH rule 1 x 893 [902,880,543,542,37,942,672,320,394,373]
+  CRUSH rule 1 x 894 [180,169,916,43,945,713,648,685,895,735]
+  CRUSH rule 1 x 895 [725,849,182,129,177,272,599,829,809,713]
+  CRUSH rule 1 x 896 [951,34,874,537,969,123,210,529,491,289]
+  CRUSH rule 1 x 897 [810,352,73,939,943,895,12,481,539,562]
+  CRUSH rule 1 x 898 [979,433,719,411,787,359,342,37,303,70]
+  CRUSH rule 1 x 899 [685,668,534,932,399,156,124,653,574,384]
+  CRUSH rule 1 x 900 [530,978,41,894,941,681,380,419,667,56]
+  CRUSH rule 1 x 901 [740,107,336,175,574,706,157,292,724,805]
+  CRUSH rule 1 x 902 [800,743,693,310,67,111,178,624,733,498]
+  CRUSH rule 1 x 903 [230,267,842,266,550,769,66,738,419,199]
+  CRUSH rule 1 x 904 [346,949,460,973,696,91,957,801,74,934]
+  CRUSH rule 1 x 905 [530,397,619,958,576,973,685,6,689,387]
+  CRUSH rule 1 x 906 [80,426,138,672,73,776,30,169,506,497]
+  CRUSH rule 1 x 907 [365,968,475,297,296,724,664,331,184,461]
+  CRUSH rule 1 x 908 [204,832,742,809,862,745,484,391,841,967]
+  CRUSH rule 1 x 909 [883,989,146,959,366,59,686,965,515,421]
+  CRUSH rule 1 x 910 [549,593,249,853,792,769,824,552,717,159]
+  CRUSH rule 1 x 911 [325,847,352,214,851,732,789,255,896,868]
+  CRUSH rule 1 x 912 [874,888,582,796,557,601,226,889,69,237]
+  CRUSH rule 1 x 913 [331,463,342,574,989,362,925,746,664,533]
+  CRUSH rule 1 x 914 [836,468,601,732,607,275,70,280,837,367]
+  CRUSH rule 1 x 915 [245,228,100,661,799,13,126,79,652,793]
+  CRUSH rule 1 x 916 [77,967,364,435,27,474,255,133,892,524]
+  CRUSH rule 1 x 917 [239,60,866,221,772,967,725,707,47,216]
+  CRUSH rule 1 x 918 [988,115,922,80,201,544,583,923,863,232]
+  CRUSH rule 1 x 919 [783,139,696,1,848,169,888,980,33,261]
+  CRUSH rule 1 x 920 [623,408,685,953,974,696,532,124,911,206]
+  CRUSH rule 1 x 921 [105,799,144,90,399,373,633,290,155,137]
+  CRUSH rule 1 x 922 [887,505,652,348,514,806,952,474,67,938]
+  CRUSH rule 1 x 923 [223,318,552,458,743,871,964,384,454,448]
+  CRUSH rule 1 x 924 [25,778,366,333,163,801,584,31,151,178]
+  CRUSH rule 1 x 925 [912,601,297,682,770,173,969,168,500,68]
+  CRUSH rule 1 x 926 [968,133,690,144,814,155,709,158,96,739]
+  CRUSH rule 1 x 927 [277,724,214,988,690,342,465,775,725,414]
+  CRUSH rule 1 x 928 [554,203,658,789,298,299,847,752,780,738]
+  CRUSH rule 1 x 929 [761,802,367,528,758,522,744,171,144,704]
+  CRUSH rule 1 x 930 [814,61,788,736,660,491,832,654,567,160]
+  CRUSH rule 1 x 931 [29,193,61,41,343,664,487,839,776,117]
+  CRUSH rule 1 x 932 [446,198,862,534,168,35,530,462,202,11]
+  CRUSH rule 1 x 933 [352,742,216,321,525,44,568,61,945,154]
+  CRUSH rule 1 x 934 [730,2,332,631,613,249,533,116,254,569]
+  CRUSH rule 1 x 935 [731,23,736,79,361,992,772,49,567,47]
+  CRUSH rule 1 x 936 [322,975,20,904,827,603,138,802,885,447]
+  CRUSH rule 1 x 937 [822,221,841,161,723,137,630,308,973,934]
+  CRUSH rule 1 x 938 [557,850,66,630,499,404,286,395,927,611]
+  CRUSH rule 1 x 939 [150,11,971,371,124,785,408,49,977,243]
+  CRUSH rule 1 x 940 [638,398,169,616,333,751,25,883,867,270]
+  CRUSH rule 1 x 941 [730,342,929,577,451,838,964,28,633,960]
+  CRUSH rule 1 x 942 [62,292,166,814,587,172,553,16,440,31]
+  CRUSH rule 1 x 943 [165,314,519,548,41,726,759,851,617,420]
+  CRUSH rule 1 x 944 [199,625,766,176,194,297,678,915,619,69]
+  CRUSH rule 1 x 945 [946,999,699,303,38,81,952,885,987,775]
+  CRUSH rule 1 x 946 [595,93,852,142,503,647,933,267,846,866]
+  CRUSH rule 1 x 947 [800,582,356,93,716,117,922,868,413,545]
+  CRUSH rule 1 x 948 [132,551,139,920,87,46,81,220,725,211]
+  CRUSH rule 1 x 949 [792,920,466,380,97,568,799,961,564,71]
+  CRUSH rule 1 x 950 [111,345,176,543,879,954,355,220,528,747]
+  CRUSH rule 1 x 951 [414,619,648,655,364,971,829,408,568,734]
+  CRUSH rule 1 x 952 [775,469,500,356,287,4,16,746,835,529]
+  CRUSH rule 1 x 953 [349,1,5,251,168,680,141,619,234,517]
+  CRUSH rule 1 x 954 [570,940,410,249,929,394,129,696,115,984]
+  CRUSH rule 1 x 955 [729,774,823,800,7,127,536,766,579,398]
+  CRUSH rule 1 x 956 [519,141,575,625,738,475,169,751,667,381]
+  CRUSH rule 1 x 957 [242,709,611,97,760,309,393,281,227,412]
+  CRUSH rule 1 x 958 [84,217,227,253,246,604,346,377,425,533]
+  CRUSH rule 1 x 959 [270,413,918,789,703,608,543,519,496,956]
+  CRUSH rule 1 x 960 [458,192,307,279,920,139,855,49,548,367]
+  CRUSH rule 1 x 961 [981,388,777,546,359,660,455,708,649,93]
+  CRUSH rule 1 x 962 [623,834,277,134,729,246,856,477,895,89]
+  CRUSH rule 1 x 963 [291,167,714,468,109,373,485,701,76,55]
+  CRUSH rule 1 x 964 [28,156,788,127,598,215,361,255,507,540]
+  CRUSH rule 1 x 965 [675,557,290,517,840,510,59,229,819,610]
+  CRUSH rule 1 x 966 [836,306,946,283,642,606,929,773,928,579]
+  CRUSH rule 1 x 967 [966,386,735,837,392,116,19,674,395,483]
+  CRUSH rule 1 x 968 [864,756,690,121,328,122,433,520,916,41]
+  CRUSH rule 1 x 969 [729,625,480,769,512,882,518,956,398,269]
+  CRUSH rule 1 x 970 [800,362,646,582,309,102,576,411,416,523]
+  CRUSH rule 1 x 971 [737,381,153,684,298,166,344,520,546,612]
+  CRUSH rule 1 x 972 [952,245,720,884,334,311,754,540,79,174]
+  CRUSH rule 1 x 973 [356,455,579,857,832,596,549,524,109,364]
+  CRUSH rule 1 x 974 [545,758,586,596,300,790,116,993,644,405]
+  CRUSH rule 1 x 975 [336,191,202,146,720,897,330,308,744,843]
+  CRUSH rule 1 x 976 [446,208,757,620,252,846,397,58,57,603]
+  CRUSH rule 1 x 977 [202,896,196,956,763,126,783,828,409,529]
+  CRUSH rule 1 x 978 [612,324,996,225,418,583,514,169,99,878]
+  CRUSH rule 1 x 979 [843,457,675,650,958,657,677,173,903,781]
+  CRUSH rule 1 x 980 [60,914,881,626,850,759,398,943,764,867]
+  CRUSH rule 1 x 981 [702,749,937,153,724,514,536,212,247,523]
+  CRUSH rule 1 x 982 [298,928,738,167,99,668,395,198,100,580]
+  CRUSH rule 1 x 983 [723,572,395,358,900,37,927,597,103,461]
+  CRUSH rule 1 x 984 [723,864,804,935,846,993,950,840,427,469]
+  CRUSH rule 1 x 985 [945,459,868,211,524,954,911,208,91,999]
+  CRUSH rule 1 x 986 [772,664,535,169,297,996,864,555,687,212]
+  CRUSH rule 1 x 987 [88,324,312,843,661,580,76,894,480,323]
+  CRUSH rule 1 x 988 [522,927,131,996,351,685,865,47,116,230]
+  CRUSH rule 1 x 989 [578,332,208,605,975,207,155,380,797,177]
+  CRUSH rule 1 x 990 [638,228,414,311,738,698,340,526,728,595]
+  CRUSH rule 1 x 991 [530,221,451,422,879,916,754,928,288,668]
+  CRUSH rule 1 x 992 [925,705,275,81,234,310,117,546,798,777]
+  CRUSH rule 1 x 993 [991,301,43,469,830,242,382,428,451,216]
+  CRUSH rule 1 x 994 [276,51,868,683,843,815,557,378,936,192]
+  CRUSH rule 1 x 995 [288,836,753,790,758,120,158,265,110,171]
+  CRUSH rule 1 x 996 [887,983,252,686,470,345,459,764,859,776]
+  CRUSH rule 1 x 997 [110,924,386,79,705,697,210,698,273,955]
+  CRUSH rule 1 x 998 [435,830,485,853,926,730,786,762,444,561]
+  CRUSH rule 1 x 999 [876,738,357,913,723,51,15,585,898,902]
+  CRUSH rule 1 x 1000 [178,963,638,430,845,586,317,102,200,662]
+  CRUSH rule 1 x 1001 [99,519,66,759,583,944,739,922,343,574]
+  CRUSH rule 1 x 1002 [515,534,468,866,878,717,729,370,326,640]
+  CRUSH rule 1 x 1003 [104,611,937,698,94,67,614,783,865,245]
+  CRUSH rule 1 x 1004 [269,638,724,375,491,121,891,113,424,320]
+  CRUSH rule 1 x 1005 [369,223,309,409,822,39,597,969,911,241]
+  CRUSH rule 1 x 1006 [40,107,69,275,79,429,234,945,598,498]
+  CRUSH rule 1 x 1007 [978,111,416,758,454,640,5,444,795,150]
+  CRUSH rule 1 x 1008 [965,956,624,832,421,96,975,723,909,93]
+  CRUSH rule 1 x 1009 [598,476,356,695,919,566,234,383,604,903]
+  CRUSH rule 1 x 1010 [767,523,239,517,29,77,23,241,838,865]
+  CRUSH rule 1 x 1011 [289,871,207,576,347,698,48,570,639,230]
+  CRUSH rule 1 x 1012 [128,28,370,31,341,755,268,647,669,90]
+  CRUSH rule 1 x 1013 [979,765,660,812,666,187,808,351,572,403]
+  CRUSH rule 1 x 1014 [979,948,513,88,47,825,969,81,586,62]
+  CRUSH rule 1 x 1015 [277,790,396,672,542,647,145,11,965,669]
+  CRUSH rule 1 x 1016 [262,73,128,886,839,685,456,560,935,733]
+  CRUSH rule 1 x 1017 [150,269,61,499,832,591,637,731,738,154]
+  CRUSH rule 1 x 1018 [555,829,554,944,406,576,463,926,475,316]
+  CRUSH rule 1 x 1019 [513,356,265,446,65,288,768,245,337,197]
+  CRUSH rule 1 x 1020 [158,161,877,704,948,570,495,865,698,835]
+  CRUSH rule 1 x 1021 [915,998,957,285,546,202,676,322,671,622]
+  CRUSH rule 1 x 1022 [967,829,973,640,703,470,871,828,440,449]
+  CRUSH rule 1 x 1023 [488,257,614,859,325,419,50,560,595,554]
+  rule 1 (metadata) num_rep 10 result size == 10:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-legacy-tunables.t b/ceph/src/test/cli/crushtool/test-map-legacy-tunables.t
new file mode 100644
index 00000000..12bf604a
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-legacy-tunables.t
@@ -0,0 +1,10252 @@
+  $ crushtool -i "$TESTDIR/test-map-a.crushmap" --test --show-statistics --rule 0
+  rule 0 (data), x = 0..1023, numrep = 1..10
+  CRUSH rule 0 x 0 [36]
+  CRUSH rule 0 x 1 [876]
+  CRUSH rule 0 x 2 [292]
+  CRUSH rule 0 x 3 [623]
+  CRUSH rule 0 x 4 [61]
+  CRUSH rule 0 x 5 [946]
+  CRUSH rule 0 x 6 [576]
+  CRUSH rule 0 x 7 [645]
+  CRUSH rule 0 x 8 [243]
+  CRUSH rule 0 x 9 [22]
+  CRUSH rule 0 x 10 [758]
+  CRUSH rule 0 x 11 [769]
+  CRUSH rule 0 x 12 [780]
+  CRUSH rule 0 x 13 [557]
+  CRUSH rule 0 x 14 [59]
+  CRUSH rule 0 x 15 [718]
+  CRUSH rule 0 x 16 [673]
+  CRUSH rule 0 x 17 [648]
+  CRUSH rule 0 x 18 [654]
+  CRUSH rule 0 x 19 [850]
+  CRUSH rule 0 x 20 [717]
+  CRUSH rule 0 x 21 [420]
+  CRUSH rule 0 x 22 [503]
+  CRUSH rule 0 x 23 [411]
+  CRUSH rule 0 x 24 [266]
+  CRUSH rule 0 x 25 [760]
+  CRUSH rule 0 x 26 [903]
+  CRUSH rule 0 x 27 [946]
+  CRUSH rule 0 x 28 [69]
+  CRUSH rule 0 x 29 [844]
+  CRUSH rule 0 x 30 [621]
+  CRUSH rule 0 x 31 [784]
+  CRUSH rule 0 x 32 [173]
+  CRUSH rule 0 x 33 [698]
+  CRUSH rule 0 x 34 [168]
+  CRUSH rule 0 x 35 [274]
+  CRUSH rule 0 x 36 [318]
+  CRUSH rule 0 x 37 [173]
+  CRUSH rule 0 x 38 [708]
+  CRUSH rule 0 x 39 [662]
+  CRUSH rule 0 x 40 [620]
+  CRUSH rule 0 x 41 [811]
+  CRUSH rule 0 x 42 [863]
+  CRUSH rule 0 x 43 [686]
+  CRUSH rule 0 x 44 [396]
+  CRUSH rule 0 x 45 [991]
+  CRUSH rule 0 x 46 [420]
+  CRUSH rule 0 x 47 [467]
+  CRUSH rule 0 x 48 [955]
+  CRUSH rule 0 x 49 [974]
+  CRUSH rule 0 x 50 [870]
+  CRUSH rule 0 x 51 [182]
+  CRUSH rule 0 x 52 [704]
+  CRUSH rule 0 x 53 [185]
+  CRUSH rule 0 x 54 [270]
+  CRUSH rule 0 x 55 [895]
+  CRUSH rule 0 x 56 [564]
+  CRUSH rule 0 x 57 [738]
+  CRUSH rule 0 x 58 [524]
+  CRUSH rule 0 x 59 [408]
+  CRUSH rule 0 x 60 [228]
+  CRUSH rule 0 x 61 [154]
+  CRUSH rule 0 x 62 [594]
+  CRUSH rule 0 x 63 [646]
+  CRUSH rule 0 x 64 [175]
+  CRUSH rule 0 x 65 [745]
+  CRUSH rule 0 x 66 [275]
+  CRUSH rule 0 x 67 [246]
+  CRUSH rule 0 x 68 [711]
+  CRUSH rule 0 x 69 [493]
+  CRUSH rule 0 x 70 [30]
+  CRUSH rule 0 x 71 [984]
+  CRUSH rule 0 x 72 [71]
+  CRUSH rule 0 x 73 [922]
+  CRUSH rule 0 x 74 [629]
+  CRUSH rule 0 x 75 [222]
+  CRUSH rule 0 x 76 [262]
+  CRUSH rule 0 x 77 [638]
+  CRUSH rule 0 x 78 [324]
+  CRUSH rule 0 x 79 [577]
+  CRUSH rule 0 x 80 [501]
+  CRUSH rule 0 x 81 [506]
+  CRUSH rule 0 x 82 [222]
+  CRUSH rule 0 x 83 [71]
+  CRUSH rule 0 x 84 [49]
+  CRUSH rule 0 x 85 [985]
+  CRUSH rule 0 x 86 [537]
+  CRUSH rule 0 x 87 [997]
+  CRUSH rule 0 x 88 [957]
+  CRUSH rule 0 x 89 [399]
+  CRUSH rule 0 x 90 [943]
+  CRUSH rule 0 x 91 [22]
+  CRUSH rule 0 x 92 [532]
+  CRUSH rule 0 x 93 [218]
+  CRUSH rule 0 x 94 [181]
+  CRUSH rule 0 x 95 [343]
+  CRUSH rule 0 x 96 [861]
+  CRUSH rule 0 x 97 [459]
+  CRUSH rule 0 x 98 [327]
+  CRUSH rule 0 x 99 [974]
+  CRUSH rule 0 x 100 [32]
+  CRUSH rule 0 x 101 [142]
+  CRUSH rule 0 x 102 [172]
+  CRUSH rule 0 x 103 [630]
+  CRUSH rule 0 x 104 [758]
+  CRUSH rule 0 x 105 [843]
+  CRUSH rule 0 x 106 [28]
+  CRUSH rule 0 x 107 [74]
+  CRUSH rule 0 x 108 [875]
+  CRUSH rule 0 x 109 [411]
+  CRUSH rule 0 x 110 [440]
+  CRUSH rule 0 x 111 [405]
+  CRUSH rule 0 x 112 [143]
+  CRUSH rule 0 x 113 [153]
+  CRUSH rule 0 x 114 [804]
+  CRUSH rule 0 x 115 [588]
+  CRUSH rule 0 x 116 [327]
+  CRUSH rule 0 x 117 [95]
+  CRUSH rule 0 x 118 [80]
+  CRUSH rule 0 x 119 [386]
+  CRUSH rule 0 x 120 [366]
+  CRUSH rule 0 x 121 [129]
+  CRUSH rule 0 x 122 [873]
+  CRUSH rule 0 x 123 [533]
+  CRUSH rule 0 x 124 [461]
+  CRUSH rule 0 x 125 [342]
+  CRUSH rule 0 x 126 [819]
+  CRUSH rule 0 x 127 [437]
+  CRUSH rule 0 x 128 [679]
+  CRUSH rule 0 x 129 [380]
+  CRUSH rule 0 x 130 [992]
+  CRUSH rule 0 x 131 [469]
+  CRUSH rule 0 x 132 [571]
+  CRUSH rule 0 x 133 [964]
+  CRUSH rule 0 x 134 [999]
+  CRUSH rule 0 x 135 [634]
+  CRUSH rule 0 x 136 [114]
+  CRUSH rule 0 x 137 [839]
+  CRUSH rule 0 x 138 [967]
+  CRUSH rule 0 x 139 [308]
+  CRUSH rule 0 x 140 [764]
+  CRUSH rule 0 x 141 [423]
+  CRUSH rule 0 x 142 [252]
+  CRUSH rule 0 x 143 [33]
+  CRUSH rule 0 x 144 [472]
+  CRUSH rule 0 x 145 [242]
+  CRUSH rule 0 x 146 [290]
+  CRUSH rule 0 x 147 [447]
+  CRUSH rule 0 x 148 [212]
+  CRUSH rule 0 x 149 [9]
+  CRUSH rule 0 x 150 [166]
+  CRUSH rule 0 x 151 [811]
+  CRUSH rule 0 x 152 [449]
+  CRUSH rule 0 x 153 [523]
+  CRUSH rule 0 x 154 [208]
+  CRUSH rule 0 x 155 [569]
+  CRUSH rule 0 x 156 [488]
+  CRUSH rule 0 x 157 [140]
+  CRUSH rule 0 x 158 [786]
+  CRUSH rule 0 x 159 [134]
+  CRUSH rule 0 x 160 [690]
+  CRUSH rule 0 x 161 [324]
+  CRUSH rule 0 x 162 [748]
+  CRUSH rule 0 x 163 [575]
+  CRUSH rule 0 x 164 [314]
+  CRUSH rule 0 x 165 [116]
+  CRUSH rule 0 x 166 [352]
+  CRUSH rule 0 x 167 [27]
+  CRUSH rule 0 x 168 [953]
+  CRUSH rule 0 x 169 [912]
+  CRUSH rule 0 x 170 [421]
+  CRUSH rule 0 x 171 [488]
+  CRUSH rule 0 x 172 [366]
+  CRUSH rule 0 x 173 [863]
+  CRUSH rule 0 x 174 [263]
+  CRUSH rule 0 x 175 [875]
+  CRUSH rule 0 x 176 [745]
+  CRUSH rule 0 x 177 [128]
+  CRUSH rule 0 x 178 [155]
+  CRUSH rule 0 x 179 [593]
+  CRUSH rule 0 x 180 [154]
+  CRUSH rule 0 x 181 [289]
+  CRUSH rule 0 x 182 [730]
+  CRUSH rule 0 x 183 [639]
+  CRUSH rule 0 x 184 [704]
+  CRUSH rule 0 x 185 [97]
+  CRUSH rule 0 x 186 [26]
+  CRUSH rule 0 x 187 [649]
+  CRUSH rule 0 x 188 [682]
+  CRUSH rule 0 x 189 [325]
+  CRUSH rule 0 x 190 [399]
+  CRUSH rule 0 x 191 [629]
+  CRUSH rule 0 x 192 [503]
+  CRUSH rule 0 x 193 [546]
+  CRUSH rule 0 x 194 [242]
+  CRUSH rule 0 x 195 [625]
+  CRUSH rule 0 x 196 [357]
+  CRUSH rule 0 x 197 [306]
+  CRUSH rule 0 x 198 [863]
+  CRUSH rule 0 x 199 [935]
+  CRUSH rule 0 x 200 [373]
+  CRUSH rule 0 x 201 [659]
+  CRUSH rule 0 x 202 [260]
+  CRUSH rule 0 x 203 [36]
+  CRUSH rule 0 x 204 [92]
+  CRUSH rule 0 x 205 [68]
+  CRUSH rule 0 x 206 [570]
+  CRUSH rule 0 x 207 [834]
+  CRUSH rule 0 x 208 [927]
+  CRUSH rule 0 x 209 [878]
+  CRUSH rule 0 x 210 [572]
+  CRUSH rule 0 x 211 [107]
+  CRUSH rule 0 x 212 [389]
+  CRUSH rule 0 x 213 [497]
+  CRUSH rule 0 x 214 [798]
+  CRUSH rule 0 x 215 [233]
+  CRUSH rule 0 x 216 [494]
+  CRUSH rule 0 x 217 [352]
+  CRUSH rule 0 x 218 [895]
+  CRUSH rule 0 x 219 [222]
+  CRUSH rule 0 x 220 [281]
+  CRUSH rule 0 x 221 [64]
+  CRUSH rule 0 x 222 [40]
+  CRUSH rule 0 x 223 [645]
+  CRUSH rule 0 x 224 [647]
+  CRUSH rule 0 x 225 [219]
+  CRUSH rule 0 x 226 [372]
+  CRUSH rule 0 x 227 [925]
+  CRUSH rule 0 x 228 [682]
+  CRUSH rule 0 x 229 [880]
+  CRUSH rule 0 x 230 [328]
+  CRUSH rule 0 x 231 [320]
+  CRUSH rule 0 x 232 [924]
+  CRUSH rule 0 x 233 [948]
+  CRUSH rule 0 x 234 [484]
+  CRUSH rule 0 x 235 [750]
+  CRUSH rule 0 x 236 [551]
+  CRUSH rule 0 x 237 [390]
+  CRUSH rule 0 x 238 [570]
+  CRUSH rule 0 x 239 [729]
+  CRUSH rule 0 x 240 [981]
+  CRUSH rule 0 x 241 [310]
+  CRUSH rule 0 x 242 [161]
+  CRUSH rule 0 x 243 [180]
+  CRUSH rule 0 x 244 [52]
+  CRUSH rule 0 x 245 [523]
+  CRUSH rule 0 x 246 [362]
+  CRUSH rule 0 x 247 [382]
+  CRUSH rule 0 x 248 [129]
+  CRUSH rule 0 x 249 [159]
+  CRUSH rule 0 x 250 [404]
+  CRUSH rule 0 x 251 [661]
+  CRUSH rule 0 x 252 [961]
+  CRUSH rule 0 x 253 [651]
+  CRUSH rule 0 x 254 [123]
+  CRUSH rule 0 x 255 [314]
+  CRUSH rule 0 x 256 [315]
+  CRUSH rule 0 x 257 [825]
+  CRUSH rule 0 x 258 [624]
+  CRUSH rule 0 x 259 [602]
+  CRUSH rule 0 x 260 [717]
+  CRUSH rule 0 x 261 [145]
+  CRUSH rule 0 x 262 [223]
+  CRUSH rule 0 x 263 [462]
+  CRUSH rule 0 x 264 [654]
+  CRUSH rule 0 x 265 [302]
+  CRUSH rule 0 x 266 [202]
+  CRUSH rule 0 x 267 [282]
+  CRUSH rule 0 x 268 [338]
+  CRUSH rule 0 x 269 [738]
+  CRUSH rule 0 x 270 [707]
+  CRUSH rule 0 x 271 [705]
+  CRUSH rule 0 x 272 [756]
+  CRUSH rule 0 x 273 [197]
+  CRUSH rule 0 x 274 [992]
+  CRUSH rule 0 x 275 [544]
+  CRUSH rule 0 x 276 [658]
+  CRUSH rule 0 x 277 [143]
+  CRUSH rule 0 x 278 [492]
+  CRUSH rule 0 x 279 [517]
+  CRUSH rule 0 x 280 [825]
+  CRUSH rule 0 x 281 [224]
+  CRUSH rule 0 x 282 [298]
+  CRUSH rule 0 x 283 [311]
+  CRUSH rule 0 x 284 [771]
+  CRUSH rule 0 x 285 [693]
+  CRUSH rule 0 x 286 [364]
+  CRUSH rule 0 x 287 [591]
+  CRUSH rule 0 x 288 [965]
+  CRUSH rule 0 x 289 [225]
+  CRUSH rule 0 x 290 [577]
+  CRUSH rule 0 x 291 [160]
+  CRUSH rule 0 x 292 [873]
+  CRUSH rule 0 x 293 [100]
+  CRUSH rule 0 x 294 [285]
+  CRUSH rule 0 x 295 [938]
+  CRUSH rule 0 x 296 [850]
+  CRUSH rule 0 x 297 [951]
+  CRUSH rule 0 x 298 [173]
+  CRUSH rule 0 x 299 [598]
+  CRUSH rule 0 x 300 [531]
+  CRUSH rule 0 x 301 [823]
+  CRUSH rule 0 x 302 [184]
+  CRUSH rule 0 x 303 [521]
+  CRUSH rule 0 x 304 [980]
+  CRUSH rule 0 x 305 [153]
+  CRUSH rule 0 x 306 [423]
+  CRUSH rule 0 x 307 [997]
+  CRUSH rule 0 x 308 [991]
+  CRUSH rule 0 x 309 [860]
+  CRUSH rule 0 x 310 [589]
+  CRUSH rule 0 x 311 [477]
+  CRUSH rule 0 x 312 [887]
+  CRUSH rule 0 x 313 [802]
+  CRUSH rule 0 x 314 [654]
+  CRUSH rule 0 x 315 [767]
+  CRUSH rule 0 x 316 [778]
+  CRUSH rule 0 x 317 [184]
+  CRUSH rule 0 x 318 [525]
+  CRUSH rule 0 x 319 [476]
+  CRUSH rule 0 x 320 [149]
+  CRUSH rule 0 x 321 [710]
+  CRUSH rule 0 x 322 [175]
+  CRUSH rule 0 x 323 [819]
+  CRUSH rule 0 x 324 [16]
+  CRUSH rule 0 x 325 [486]
+  CRUSH rule 0 x 326 [613]
+  CRUSH rule 0 x 327 [125]
+  CRUSH rule 0 x 328 [807]
+  CRUSH rule 0 x 329 [588]
+  CRUSH rule 0 x 330 [932]
+  CRUSH rule 0 x 331 [341]
+  CRUSH rule 0 x 332 [153]
+  CRUSH rule 0 x 333 [745]
+  CRUSH rule 0 x 334 [614]
+  CRUSH rule 0 x 335 [518]
+  CRUSH rule 0 x 336 [389]
+  CRUSH rule 0 x 337 [753]
+  CRUSH rule 0 x 338 [128]
+  CRUSH rule 0 x 339 [430]
+  CRUSH rule 0 x 340 [541]
+  CRUSH rule 0 x 341 [402]
+  CRUSH rule 0 x 342 [982]
+  CRUSH rule 0 x 343 [833]
+  CRUSH rule 0 x 344 [784]
+  CRUSH rule 0 x 345 [546]
+  CRUSH rule 0 x 346 [302]
+  CRUSH rule 0 x 347 [488]
+  CRUSH rule 0 x 348 [903]
+  CRUSH rule 0 x 349 [471]
+  CRUSH rule 0 x 350 [348]
+  CRUSH rule 0 x 351 [961]
+  CRUSH rule 0 x 352 [728]
+  CRUSH rule 0 x 353 [904]
+  CRUSH rule 0 x 354 [345]
+  CRUSH rule 0 x 355 [50]
+  CRUSH rule 0 x 356 [87]
+  CRUSH rule 0 x 357 [762]
+  CRUSH rule 0 x 358 [908]
+  CRUSH rule 0 x 359 [484]
+  CRUSH rule 0 x 360 [173]
+  CRUSH rule 0 x 361 [404]
+  CRUSH rule 0 x 362 [403]
+  CRUSH rule 0 x 363 [639]
+  CRUSH rule 0 x 364 [752]
+  CRUSH rule 0 x 365 [956]
+  CRUSH rule 0 x 366 [860]
+  CRUSH rule 0 x 367 [205]
+  CRUSH rule 0 x 368 [301]
+  CRUSH rule 0 x 369 [452]
+  CRUSH rule 0 x 370 [11]
+  CRUSH rule 0 x 371 [124]
+  CRUSH rule 0 x 372 [253]
+  CRUSH rule 0 x 373 [715]
+  CRUSH rule 0 x 374 [191]
+  CRUSH rule 0 x 375 [711]
+  CRUSH rule 0 x 376 [597]
+  CRUSH rule 0 x 377 [294]
+  CRUSH rule 0 x 378 [34]
+  CRUSH rule 0 x 379 [869]
+  CRUSH rule 0 x 380 [294]
+  CRUSH rule 0 x 381 [119]
+  CRUSH rule 0 x 382 [69]
+  CRUSH rule 0 x 383 [922]
+  CRUSH rule 0 x 384 [221]
+  CRUSH rule 0 x 385 [561]
+  CRUSH rule 0 x 386 [335]
+  CRUSH rule 0 x 387 [514]
+  CRUSH rule 0 x 388 [587]
+  CRUSH rule 0 x 389 [109]
+  CRUSH rule 0 x 390 [925]
+  CRUSH rule 0 x 391 [267]
+  CRUSH rule 0 x 392 [382]
+  CRUSH rule 0 x 393 [425]
+  CRUSH rule 0 x 394 [898]
+  CRUSH rule 0 x 395 [806]
+  CRUSH rule 0 x 396 [790]
+  CRUSH rule 0 x 397 [136]
+  CRUSH rule 0 x 398 [914]
+  CRUSH rule 0 x 399 [261]
+  CRUSH rule 0 x 400 [661]
+  CRUSH rule 0 x 401 [953]
+  CRUSH rule 0 x 402 [738]
+  CRUSH rule 0 x 403 [573]
+  CRUSH rule 0 x 404 [526]
+  CRUSH rule 0 x 405 [582]
+  CRUSH rule 0 x 406 [768]
+  CRUSH rule 0 x 407 [260]
+  CRUSH rule 0 x 408 [657]
+  CRUSH rule 0 x 409 [498]
+  CRUSH rule 0 x 410 [28]
+  CRUSH rule 0 x 411 [684]
+  CRUSH rule 0 x 412 [261]
+  CRUSH rule 0 x 413 [891]
+  CRUSH rule 0 x 414 [127]
+  CRUSH rule 0 x 415 [272]
+  CRUSH rule 0 x 416 [739]
+  CRUSH rule 0 x 417 [106]
+  CRUSH rule 0 x 418 [525]
+  CRUSH rule 0 x 419 [603]
+  CRUSH rule 0 x 420 [988]
+  CRUSH rule 0 x 421 [761]
+  CRUSH rule 0 x 422 [317]
+  CRUSH rule 0 x 423 [137]
+  CRUSH rule 0 x 424 [920]
+  CRUSH rule 0 x 425 [277]
+  CRUSH rule 0 x 426 [485]
+  CRUSH rule 0 x 427 [242]
+  CRUSH rule 0 x 428 [632]
+  CRUSH rule 0 x 429 [641]
+  CRUSH rule 0 x 430 [626]
+  CRUSH rule 0 x 431 [697]
+  CRUSH rule 0 x 432 [590]
+  CRUSH rule 0 x 433 [284]
+  CRUSH rule 0 x 434 [538]
+  CRUSH rule 0 x 435 [30]
+  CRUSH rule 0 x 436 [164]
+  CRUSH rule 0 x 437 [322]
+  CRUSH rule 0 x 438 [142]
+  CRUSH rule 0 x 439 [119]
+  CRUSH rule 0 x 440 [333]
+  CRUSH rule 0 x 441 [477]
+  CRUSH rule 0 x 442 [274]
+  CRUSH rule 0 x 443 [983]
+  CRUSH rule 0 x 444 [536]
+  CRUSH rule 0 x 445 [485]
+  CRUSH rule 0 x 446 [345]
+  CRUSH rule 0 x 447 [61]
+  CRUSH rule 0 x 448 [333]
+  CRUSH rule 0 x 449 [680]
+  CRUSH rule 0 x 450 [235]
+  CRUSH rule 0 x 451 [961]
+  CRUSH rule 0 x 452 [525]
+  CRUSH rule 0 x 453 [138]
+  CRUSH rule 0 x 454 [137]
+  CRUSH rule 0 x 455 [173]
+  CRUSH rule 0 x 456 [235]
+  CRUSH rule 0 x 457 [450]
+  CRUSH rule 0 x 458 [195]
+  CRUSH rule 0 x 459 [381]
+  CRUSH rule 0 x 460 [972]
+  CRUSH rule 0 x 461 [506]
+  CRUSH rule 0 x 462 [692]
+  CRUSH rule 0 x 463 [788]
+  CRUSH rule 0 x 464 [133]
+  CRUSH rule 0 x 465 [971]
+  CRUSH rule 0 x 466 [394]
+  CRUSH rule 0 x 467 [517]
+  CRUSH rule 0 x 468 [829]
+  CRUSH rule 0 x 469 [987]
+  CRUSH rule 0 x 470 [107]
+  CRUSH rule 0 x 471 [181]
+  CRUSH rule 0 x 472 [547]
+  CRUSH rule 0 x 473 [760]
+  CRUSH rule 0 x 474 [787]
+  CRUSH rule 0 x 475 [662]
+  CRUSH rule 0 x 476 [110]
+  CRUSH rule 0 x 477 [393]
+  CRUSH rule 0 x 478 [246]
+  CRUSH rule 0 x 479 [70]
+  CRUSH rule 0 x 480 [753]
+  CRUSH rule 0 x 481 [470]
+  CRUSH rule 0 x 482 [451]
+  CRUSH rule 0 x 483 [816]
+  CRUSH rule 0 x 484 [540]
+  CRUSH rule 0 x 485 [74]
+  CRUSH rule 0 x 486 [958]
+  CRUSH rule 0 x 487 [228]
+  CRUSH rule 0 x 488 [180]
+  CRUSH rule 0 x 489 [47]
+  CRUSH rule 0 x 490 [905]
+  CRUSH rule 0 x 491 [892]
+  CRUSH rule 0 x 492 [588]
+  CRUSH rule 0 x 493 [353]
+  CRUSH rule 0 x 494 [378]
+  CRUSH rule 0 x 495 [845]
+  CRUSH rule 0 x 496 [13]
+  CRUSH rule 0 x 497 [796]
+  CRUSH rule 0 x 498 [412]
+  CRUSH rule 0 x 499 [330]
+  CRUSH rule 0 x 500 [820]
+  CRUSH rule 0 x 501 [110]
+  CRUSH rule 0 x 502 [336]
+  CRUSH rule 0 x 503 [922]
+  CRUSH rule 0 x 504 [483]
+  CRUSH rule 0 x 505 [482]
+  CRUSH rule 0 x 506 [493]
+  CRUSH rule 0 x 507 [12]
+  CRUSH rule 0 x 508 [227]
+  CRUSH rule 0 x 509 [807]
+  CRUSH rule 0 x 510 [134]
+  CRUSH rule 0 x 511 [212]
+  CRUSH rule 0 x 512 [236]
+  CRUSH rule 0 x 513 [994]
+  CRUSH rule 0 x 514 [45]
+  CRUSH rule 0 x 515 [504]
+  CRUSH rule 0 x 516 [285]
+  CRUSH rule 0 x 517 [300]
+  CRUSH rule 0 x 518 [397]
+  CRUSH rule 0 x 519 [86]
+  CRUSH rule 0 x 520 [900]
+  CRUSH rule 0 x 521 [31]
+  CRUSH rule 0 x 522 [390]
+  CRUSH rule 0 x 523 [618]
+  CRUSH rule 0 x 524 [635]
+  CRUSH rule 0 x 525 [311]
+  CRUSH rule 0 x 526 [48]
+  CRUSH rule 0 x 527 [202]
+  CRUSH rule 0 x 528 [565]
+  CRUSH rule 0 x 529 [934]
+  CRUSH rule 0 x 530 [502]
+  CRUSH rule 0 x 531 [681]
+  CRUSH rule 0 x 532 [422]
+  CRUSH rule 0 x 533 [863]
+  CRUSH rule 0 x 534 [962]
+  CRUSH rule 0 x 535 [89]
+  CRUSH rule 0 x 536 [499]
+  CRUSH rule 0 x 537 [676]
+  CRUSH rule 0 x 538 [58]
+  CRUSH rule 0 x 539 [837]
+  CRUSH rule 0 x 540 [831]
+  CRUSH rule 0 x 541 [582]
+  CRUSH rule 0 x 542 [472]
+  CRUSH rule 0 x 543 [382]
+  CRUSH rule 0 x 544 [947]
+  CRUSH rule 0 x 545 [425]
+  CRUSH rule 0 x 546 [18]
+  CRUSH rule 0 x 547 [445]
+  CRUSH rule 0 x 548 [367]
+  CRUSH rule 0 x 549 [125]
+  CRUSH rule 0 x 550 [425]
+  CRUSH rule 0 x 551 [44]
+  CRUSH rule 0 x 552 [246]
+  CRUSH rule 0 x 553 [71]
+  CRUSH rule 0 x 554 [207]
+  CRUSH rule 0 x 555 [570]
+  CRUSH rule 0 x 556 [674]
+  CRUSH rule 0 x 557 [347]
+  CRUSH rule 0 x 558 [627]
+  CRUSH rule 0 x 559 [940]
+  CRUSH rule 0 x 560 [295]
+  CRUSH rule 0 x 561 [506]
+  CRUSH rule 0 x 562 [718]
+  CRUSH rule 0 x 563 [552]
+  CRUSH rule 0 x 564 [835]
+  CRUSH rule 0 x 565 [8]
+  CRUSH rule 0 x 566 [600]
+  CRUSH rule 0 x 567 [999]
+  CRUSH rule 0 x 568 [252]
+  CRUSH rule 0 x 569 [643]
+  CRUSH rule 0 x 570 [617]
+  CRUSH rule 0 x 571 [757]
+  CRUSH rule 0 x 572 [299]
+  CRUSH rule 0 x 573 [25]
+  CRUSH rule 0 x 574 [215]
+  CRUSH rule 0 x 575 [225]
+  CRUSH rule 0 x 576 [627]
+  CRUSH rule 0 x 577 [237]
+  CRUSH rule 0 x 578 [885]
+  CRUSH rule 0 x 579 [924]
+  CRUSH rule 0 x 580 [718]
+  CRUSH rule 0 x 581 [219]
+  CRUSH rule 0 x 582 [893]
+  CRUSH rule 0 x 583 [246]
+  CRUSH rule 0 x 584 [336]
+  CRUSH rule 0 x 585 [324]
+  CRUSH rule 0 x 586 [558]
+  CRUSH rule 0 x 587 [985]
+  CRUSH rule 0 x 588 [211]
+  CRUSH rule 0 x 589 [129]
+  CRUSH rule 0 x 590 [467]
+  CRUSH rule 0 x 591 [758]
+  CRUSH rule 0 x 592 [525]
+  CRUSH rule 0 x 593 [601]
+  CRUSH rule 0 x 594 [227]
+  CRUSH rule 0 x 595 [720]
+  CRUSH rule 0 x 596 [751]
+  CRUSH rule 0 x 597 [129]
+  CRUSH rule 0 x 598 [679]
+  CRUSH rule 0 x 599 [668]
+  CRUSH rule 0 x 600 [143]
+  CRUSH rule 0 x 601 [326]
+  CRUSH rule 0 x 602 [860]
+  CRUSH rule 0 x 603 [709]
+  CRUSH rule 0 x 604 [571]
+  CRUSH rule 0 x 605 [252]
+  CRUSH rule 0 x 606 [339]
+  CRUSH rule 0 x 607 [590]
+  CRUSH rule 0 x 608 [145]
+  CRUSH rule 0 x 609 [973]
+  CRUSH rule 0 x 610 [435]
+  CRUSH rule 0 x 611 [559]
+  CRUSH rule 0 x 612 [273]
+  CRUSH rule 0 x 613 [828]
+  CRUSH rule 0 x 614 [478]
+  CRUSH rule 0 x 615 [392]
+  CRUSH rule 0 x 616 [778]
+  CRUSH rule 0 x 617 [622]
+  CRUSH rule 0 x 618 [149]
+  CRUSH rule 0 x 619 [604]
+  CRUSH rule 0 x 620 [181]
+  CRUSH rule 0 x 621 [735]
+  CRUSH rule 0 x 622 [661]
+  CRUSH rule 0 x 623 [142]
+  CRUSH rule 0 x 624 [360]
+  CRUSH rule 0 x 625 [541]
+  CRUSH rule 0 x 626 [364]
+  CRUSH rule 0 x 627 [458]
+  CRUSH rule 0 x 628 [250]
+  CRUSH rule 0 x 629 [928]
+  CRUSH rule 0 x 630 [243]
+  CRUSH rule 0 x 631 [438]
+  CRUSH rule 0 x 632 [797]
+  CRUSH rule 0 x 633 [993]
+  CRUSH rule 0 x 634 [239]
+  CRUSH rule 0 x 635 [640]
+  CRUSH rule 0 x 636 [173]
+  CRUSH rule 0 x 637 [0]
+  CRUSH rule 0 x 638 [702]
+  CRUSH rule 0 x 639 [475]
+  CRUSH rule 0 x 640 [31]
+  CRUSH rule 0 x 641 [296]
+  CRUSH rule 0 x 642 [894]
+  CRUSH rule 0 x 643 [117]
+  CRUSH rule 0 x 644 [438]
+  CRUSH rule 0 x 645 [982]
+  CRUSH rule 0 x 646 [334]
+  CRUSH rule 0 x 647 [933]
+  CRUSH rule 0 x 648 [22]
+  CRUSH rule 0 x 649 [503]
+  CRUSH rule 0 x 650 [328]
+  CRUSH rule 0 x 651 [3]
+  CRUSH rule 0 x 652 [495]
+  CRUSH rule 0 x 653 [185]
+  CRUSH rule 0 x 654 [130]
+  CRUSH rule 0 x 655 [560]
+  CRUSH rule 0 x 656 [219]
+  CRUSH rule 0 x 657 [233]
+  CRUSH rule 0 x 658 [778]
+  CRUSH rule 0 x 659 [240]
+  CRUSH rule 0 x 660 [244]
+  CRUSH rule 0 x 661 [184]
+  CRUSH rule 0 x 662 [65]
+  CRUSH rule 0 x 663 [323]
+  CRUSH rule 0 x 664 [865]
+  CRUSH rule 0 x 665 [420]
+  CRUSH rule 0 x 666 [319]
+  CRUSH rule 0 x 667 [875]
+  CRUSH rule 0 x 668 [331]
+  CRUSH rule 0 x 669 [915]
+  CRUSH rule 0 x 670 [845]
+  CRUSH rule 0 x 671 [108]
+  CRUSH rule 0 x 672 [578]
+  CRUSH rule 0 x 673 [442]
+  CRUSH rule 0 x 674 [588]
+  CRUSH rule 0 x 675 [489]
+  CRUSH rule 0 x 676 [928]
+  CRUSH rule 0 x 677 [399]
+  CRUSH rule 0 x 678 [546]
+  CRUSH rule 0 x 679 [988]
+  CRUSH rule 0 x 680 [335]
+  CRUSH rule 0 x 681 [690]
+  CRUSH rule 0 x 682 [196]
+  CRUSH rule 0 x 683 [627]
+  CRUSH rule 0 x 684 [38]
+  CRUSH rule 0 x 685 [841]
+  CRUSH rule 0 x 686 [336]
+  CRUSH rule 0 x 687 [20]
+  CRUSH rule 0 x 688 [463]
+  CRUSH rule 0 x 689 [569]
+  CRUSH rule 0 x 690 [551]
+  CRUSH rule 0 x 691 [766]
+  CRUSH rule 0 x 692 [739]
+  CRUSH rule 0 x 693 [339]
+  CRUSH rule 0 x 694 [405]
+  CRUSH rule 0 x 695 [622]
+  CRUSH rule 0 x 696 [558]
+  CRUSH rule 0 x 697 [818]
+  CRUSH rule 0 x 698 [178]
+  CRUSH rule 0 x 699 [450]
+  CRUSH rule 0 x 700 [502]
+  CRUSH rule 0 x 701 [4]
+  CRUSH rule 0 x 702 [177]
+  CRUSH rule 0 x 703 [354]
+  CRUSH rule 0 x 704 [646]
+  CRUSH rule 0 x 705 [921]
+  CRUSH rule 0 x 706 [652]
+  CRUSH rule 0 x 707 [345]
+  CRUSH rule 0 x 708 [333]
+  CRUSH rule 0 x 709 [45]
+  CRUSH rule 0 x 710 [94]
+  CRUSH rule 0 x 711 [227]
+  CRUSH rule 0 x 712 [398]
+  CRUSH rule 0 x 713 [116]
+  CRUSH rule 0 x 714 [111]
+  CRUSH rule 0 x 715 [531]
+  CRUSH rule 0 x 716 [169]
+  CRUSH rule 0 x 717 [417]
+  CRUSH rule 0 x 718 [992]
+  CRUSH rule 0 x 719 [936]
+  CRUSH rule 0 x 720 [370]
+  CRUSH rule 0 x 721 [320]
+  CRUSH rule 0 x 722 [7]
+  CRUSH rule 0 x 723 [270]
+  CRUSH rule 0 x 724 [666]
+  CRUSH rule 0 x 725 [794]
+  CRUSH rule 0 x 726 [420]
+  CRUSH rule 0 x 727 [561]
+  CRUSH rule 0 x 728 [951]
+  CRUSH rule 0 x 729 [656]
+  CRUSH rule 0 x 730 [3]
+  CRUSH rule 0 x 731 [852]
+  CRUSH rule 0 x 732 [983]
+  CRUSH rule 0 x 733 [285]
+  CRUSH rule 0 x 734 [125]
+  CRUSH rule 0 x 735 [417]
+  CRUSH rule 0 x 736 [749]
+  CRUSH rule 0 x 737 [644]
+  CRUSH rule 0 x 738 [449]
+  CRUSH rule 0 x 739 [341]
+  CRUSH rule 0 x 740 [874]
+  CRUSH rule 0 x 741 [189]
+  CRUSH rule 0 x 742 [912]
+  CRUSH rule 0 x 743 [654]
+  CRUSH rule 0 x 744 [725]
+  CRUSH rule 0 x 745 [787]
+  CRUSH rule 0 x 746 [757]
+  CRUSH rule 0 x 747 [700]
+  CRUSH rule 0 x 748 [557]
+  CRUSH rule 0 x 749 [772]
+  CRUSH rule 0 x 750 [946]
+  CRUSH rule 0 x 751 [996]
+  CRUSH rule 0 x 752 [746]
+  CRUSH rule 0 x 753 [741]
+  CRUSH rule 0 x 754 [648]
+  CRUSH rule 0 x 755 [157]
+  CRUSH rule 0 x 756 [416]
+  CRUSH rule 0 x 757 [599]
+  CRUSH rule 0 x 758 [994]
+  CRUSH rule 0 x 759 [959]
+  CRUSH rule 0 x 760 [518]
+  CRUSH rule 0 x 761 [285]
+  CRUSH rule 0 x 762 [591]
+  CRUSH rule 0 x 763 [908]
+  CRUSH rule 0 x 764 [787]
+  CRUSH rule 0 x 765 [327]
+  CRUSH rule 0 x 766 [84]
+  CRUSH rule 0 x 767 [370]
+  CRUSH rule 0 x 768 [826]
+  CRUSH rule 0 x 769 [67]
+  CRUSH rule 0 x 770 [593]
+  CRUSH rule 0 x 771 [309]
+  CRUSH rule 0 x 772 [12]
+  CRUSH rule 0 x 773 [253]
+  CRUSH rule 0 x 774 [164]
+  CRUSH rule 0 x 775 [703]
+  CRUSH rule 0 x 776 [728]
+  CRUSH rule 0 x 777 [981]
+  CRUSH rule 0 x 778 [411]
+  CRUSH rule 0 x 779 [346]
+  CRUSH rule 0 x 780 [476]
+  CRUSH rule 0 x 781 [10]
+  CRUSH rule 0 x 782 [462]
+  CRUSH rule 0 x 783 [580]
+  CRUSH rule 0 x 784 [413]
+  CRUSH rule 0 x 785 [341]
+  CRUSH rule 0 x 786 [411]
+  CRUSH rule 0 x 787 [605]
+  CRUSH rule 0 x 788 [226]
+  CRUSH rule 0 x 789 [545]
+  CRUSH rule 0 x 790 [414]
+  CRUSH rule 0 x 791 [660]
+  CRUSH rule 0 x 792 [287]
+  CRUSH rule 0 x 793 [631]
+  CRUSH rule 0 x 794 [931]
+  CRUSH rule 0 x 795 [551]
+  CRUSH rule 0 x 796 [814]
+  CRUSH rule 0 x 797 [64]
+  CRUSH rule 0 x 798 [422]
+  CRUSH rule 0 x 799 [824]
+  CRUSH rule 0 x 800 [862]
+  CRUSH rule 0 x 801 [145]
+  CRUSH rule 0 x 802 [570]
+  CRUSH rule 0 x 803 [151]
+  CRUSH rule 0 x 804 [467]
+  CRUSH rule 0 x 805 [621]
+  CRUSH rule 0 x 806 [898]
+  CRUSH rule 0 x 807 [354]
+  CRUSH rule 0 x 808 [7]
+  CRUSH rule 0 x 809 [70]
+  CRUSH rule 0 x 810 [701]
+  CRUSH rule 0 x 811 [248]
+  CRUSH rule 0 x 812 [230]
+  CRUSH rule 0 x 813 [805]
+  CRUSH rule 0 x 814 [54]
+  CRUSH rule 0 x 815 [679]
+  CRUSH rule 0 x 816 [919]
+  CRUSH rule 0 x 817 [765]
+  CRUSH rule 0 x 818 [415]
+  CRUSH rule 0 x 819 [721]
+  CRUSH rule 0 x 820 [218]
+  CRUSH rule 0 x 821 [185]
+  CRUSH rule 0 x 822 [356]
+  CRUSH rule 0 x 823 [220]
+  CRUSH rule 0 x 824 [292]
+  CRUSH rule 0 x 825 [949]
+  CRUSH rule 0 x 826 [767]
+  CRUSH rule 0 x 827 [631]
+  CRUSH rule 0 x 828 [288]
+  CRUSH rule 0 x 829 [990]
+  CRUSH rule 0 x 830 [152]
+  CRUSH rule 0 x 831 [814]
+  CRUSH rule 0 x 832 [235]
+  CRUSH rule 0 x 833 [657]
+  CRUSH rule 0 x 834 [907]
+  CRUSH rule 0 x 835 [784]
+  CRUSH rule 0 x 836 [951]
+  CRUSH rule 0 x 837 [556]
+  CRUSH rule 0 x 838 [329]
+  CRUSH rule 0 x 839 [568]
+  CRUSH rule 0 x 840 [45]
+  CRUSH rule 0 x 841 [652]
+  CRUSH rule 0 x 842 [629]
+  CRUSH rule 0 x 843 [799]
+  CRUSH rule 0 x 844 [694]
+  CRUSH rule 0 x 845 [332]
+  CRUSH rule 0 x 846 [452]
+  CRUSH rule 0 x 847 [399]
+  CRUSH rule 0 x 848 [303]
+  CRUSH rule 0 x 849 [666]
+  CRUSH rule 0 x 850 [644]
+  CRUSH rule 0 x 851 [527]
+  CRUSH rule 0 x 852 [31]
+  CRUSH rule 0 x 853 [483]
+  CRUSH rule 0 x 854 [697]
+  CRUSH rule 0 x 855 [837]
+  CRUSH rule 0 x 856 [712]
+  CRUSH rule 0 x 857 [77]
+  CRUSH rule 0 x 858 [412]
+  CRUSH rule 0 x 859 [173]
+  CRUSH rule 0 x 860 [776]
+  CRUSH rule 0 x 861 [705]
+  CRUSH rule 0 x 862 [809]
+  CRUSH rule 0 x 863 [349]
+  CRUSH rule 0 x 864 [717]
+  CRUSH rule 0 x 865 [857]
+  CRUSH rule 0 x 866 [394]
+  CRUSH rule 0 x 867 [640]
+  CRUSH rule 0 x 868 [613]
+  CRUSH rule 0 x 869 [973]
+  CRUSH rule 0 x 870 [505]
+  CRUSH rule 0 x 871 [239]
+  CRUSH rule 0 x 872 [21]
+  CRUSH rule 0 x 873 [954]
+  CRUSH rule 0 x 874 [54]
+  CRUSH rule 0 x 875 [809]
+  CRUSH rule 0 x 876 [483]
+  CRUSH rule 0 x 877 [542]
+  CRUSH rule 0 x 878 [217]
+  CRUSH rule 0 x 879 [999]
+  CRUSH rule 0 x 880 [678]
+  CRUSH rule 0 x 881 [394]
+  CRUSH rule 0 x 882 [467]
+  CRUSH rule 0 x 883 [802]
+  CRUSH rule 0 x 884 [653]
+  CRUSH rule 0 x 885 [898]
+  CRUSH rule 0 x 886 [434]
+  CRUSH rule 0 x 887 [297]
+  CRUSH rule 0 x 888 [863]
+  CRUSH rule 0 x 889 [105]
+  CRUSH rule 0 x 890 [550]
+  CRUSH rule 0 x 891 [575]
+  CRUSH rule 0 x 892 [259]
+  CRUSH rule 0 x 893 [902]
+  CRUSH rule 0 x 894 [180]
+  CRUSH rule 0 x 895 [725]
+  CRUSH rule 0 x 896 [951]
+  CRUSH rule 0 x 897 [810]
+  CRUSH rule 0 x 898 [979]
+  CRUSH rule 0 x 899 [685]
+  CRUSH rule 0 x 900 [530]
+  CRUSH rule 0 x 901 [740]
+  CRUSH rule 0 x 902 [800]
+  CRUSH rule 0 x 903 [230]
+  CRUSH rule 0 x 904 [346]
+  CRUSH rule 0 x 905 [530]
+  CRUSH rule 0 x 906 [80]
+  CRUSH rule 0 x 907 [365]
+  CRUSH rule 0 x 908 [204]
+  CRUSH rule 0 x 909 [883]
+  CRUSH rule 0 x 910 [549]
+  CRUSH rule 0 x 911 [325]
+  CRUSH rule 0 x 912 [874]
+  CRUSH rule 0 x 913 [331]
+  CRUSH rule 0 x 914 [836]
+  CRUSH rule 0 x 915 [245]
+  CRUSH rule 0 x 916 [77]
+  CRUSH rule 0 x 917 [239]
+  CRUSH rule 0 x 918 [988]
+  CRUSH rule 0 x 919 [783]
+  CRUSH rule 0 x 920 [623]
+  CRUSH rule 0 x 921 [105]
+  CRUSH rule 0 x 922 [887]
+  CRUSH rule 0 x 923 [223]
+  CRUSH rule 0 x 924 [25]
+  CRUSH rule 0 x 925 [912]
+  CRUSH rule 0 x 926 [968]
+  CRUSH rule 0 x 927 [277]
+  CRUSH rule 0 x 928 [554]
+  CRUSH rule 0 x 929 [761]
+  CRUSH rule 0 x 930 [814]
+  CRUSH rule 0 x 931 [29]
+  CRUSH rule 0 x 932 [446]
+  CRUSH rule 0 x 933 [352]
+  CRUSH rule 0 x 934 [730]
+  CRUSH rule 0 x 935 [731]
+  CRUSH rule 0 x 936 [322]
+  CRUSH rule 0 x 937 [822]
+  CRUSH rule 0 x 938 [557]
+  CRUSH rule 0 x 939 [150]
+  CRUSH rule 0 x 940 [638]
+  CRUSH rule 0 x 941 [730]
+  CRUSH rule 0 x 942 [62]
+  CRUSH rule 0 x 943 [165]
+  CRUSH rule 0 x 944 [199]
+  CRUSH rule 0 x 945 [946]
+  CRUSH rule 0 x 946 [595]
+  CRUSH rule 0 x 947 [800]
+  CRUSH rule 0 x 948 [132]
+  CRUSH rule 0 x 949 [792]
+  CRUSH rule 0 x 950 [111]
+  CRUSH rule 0 x 951 [414]
+  CRUSH rule 0 x 952 [775]
+  CRUSH rule 0 x 953 [349]
+  CRUSH rule 0 x 954 [570]
+  CRUSH rule 0 x 955 [729]
+  CRUSH rule 0 x 956 [519]
+  CRUSH rule 0 x 957 [242]
+  CRUSH rule 0 x 958 [84]
+  CRUSH rule 0 x 959 [270]
+  CRUSH rule 0 x 960 [458]
+  CRUSH rule 0 x 961 [981]
+  CRUSH rule 0 x 962 [623]
+  CRUSH rule 0 x 963 [291]
+  CRUSH rule 0 x 964 [28]
+  CRUSH rule 0 x 965 [675]
+  CRUSH rule 0 x 966 [836]
+  CRUSH rule 0 x 967 [966]
+  CRUSH rule 0 x 968 [864]
+  CRUSH rule 0 x 969 [729]
+  CRUSH rule 0 x 970 [800]
+  CRUSH rule 0 x 971 [737]
+  CRUSH rule 0 x 972 [952]
+  CRUSH rule 0 x 973 [356]
+  CRUSH rule 0 x 974 [545]
+  CRUSH rule 0 x 975 [336]
+  CRUSH rule 0 x 976 [446]
+  CRUSH rule 0 x 977 [202]
+  CRUSH rule 0 x 978 [612]
+  CRUSH rule 0 x 979 [843]
+  CRUSH rule 0 x 980 [60]
+  CRUSH rule 0 x 981 [702]
+  CRUSH rule 0 x 982 [298]
+  CRUSH rule 0 x 983 [723]
+  CRUSH rule 0 x 984 [723]
+  CRUSH rule 0 x 985 [945]
+  CRUSH rule 0 x 986 [772]
+  CRUSH rule 0 x 987 [88]
+  CRUSH rule 0 x 988 [522]
+  CRUSH rule 0 x 989 [578]
+  CRUSH rule 0 x 990 [638]
+  CRUSH rule 0 x 991 [530]
+  CRUSH rule 0 x 992 [925]
+  CRUSH rule 0 x 993 [991]
+  CRUSH rule 0 x 994 [276]
+  CRUSH rule 0 x 995 [288]
+  CRUSH rule 0 x 996 [887]
+  CRUSH rule 0 x 997 [110]
+  CRUSH rule 0 x 998 [435]
+  CRUSH rule 0 x 999 [876]
+  CRUSH rule 0 x 1000 [178]
+  CRUSH rule 0 x 1001 [99]
+  CRUSH rule 0 x 1002 [515]
+  CRUSH rule 0 x 1003 [104]
+  CRUSH rule 0 x 1004 [269]
+  CRUSH rule 0 x 1005 [369]
+  CRUSH rule 0 x 1006 [40]
+  CRUSH rule 0 x 1007 [978]
+  CRUSH rule 0 x 1008 [965]
+  CRUSH rule 0 x 1009 [598]
+  CRUSH rule 0 x 1010 [767]
+  CRUSH rule 0 x 1011 [289]
+  CRUSH rule 0 x 1012 [128]
+  CRUSH rule 0 x 1013 [979]
+  CRUSH rule 0 x 1014 [979]
+  CRUSH rule 0 x 1015 [277]
+  CRUSH rule 0 x 1016 [262]
+  CRUSH rule 0 x 1017 [150]
+  CRUSH rule 0 x 1018 [555]
+  CRUSH rule 0 x 1019 [513]
+  CRUSH rule 0 x 1020 [158]
+  CRUSH rule 0 x 1021 [915]
+  CRUSH rule 0 x 1022 [967]
+  CRUSH rule 0 x 1023 [488]
+  rule 0 (data) num_rep 1 result size == 1:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705]
+  CRUSH rule 0 x 1 [876,250]
+  CRUSH rule 0 x 2 [292,832]
+  CRUSH rule 0 x 3 [623,387]
+  CRUSH rule 0 x 4 [61,334]
+  CRUSH rule 0 x 5 [946,557]
+  CRUSH rule 0 x 6 [576,668]
+  CRUSH rule 0 x 7 [645,753]
+  CRUSH rule 0 x 8 [243,6]
+  CRUSH rule 0 x 9 [22,578]
+  CRUSH rule 0 x 10 [758,828]
+  CRUSH rule 0 x 11 [769,120]
+  CRUSH rule 0 x 12 [780,364]
+  CRUSH rule 0 x 13 [557,18]
+  CRUSH rule 0 x 14 [59,561]
+  CRUSH rule 0 x 15 [718,928]
+  CRUSH rule 0 x 16 [673,632]
+  CRUSH rule 0 x 17 [648,43]
+  CRUSH rule 0 x 18 [654,219]
+  CRUSH rule 0 x 19 [850,545]
+  CRUSH rule 0 x 20 [717,785]
+  CRUSH rule 0 x 21 [420,57]
+  CRUSH rule 0 x 22 [503,998]
+  CRUSH rule 0 x 23 [411,663]
+  CRUSH rule 0 x 24 [266,861]
+  CRUSH rule 0 x 25 [760,483]
+  CRUSH rule 0 x 26 [903,24]
+  CRUSH rule 0 x 27 [946,188]
+  CRUSH rule 0 x 28 [69,312]
+  CRUSH rule 0 x 29 [844,883]
+  CRUSH rule 0 x 30 [621,18]
+  CRUSH rule 0 x 31 [784,943]
+  CRUSH rule 0 x 32 [173,374]
+  CRUSH rule 0 x 33 [698,336]
+  CRUSH rule 0 x 34 [168,836]
+  CRUSH rule 0 x 35 [274,509]
+  CRUSH rule 0 x 36 [318,215]
+  CRUSH rule 0 x 37 [173,604]
+  CRUSH rule 0 x 38 [708,444]
+  CRUSH rule 0 x 39 [662,198]
+  CRUSH rule 0 x 40 [620,801]
+  CRUSH rule 0 x 41 [811,264]
+  CRUSH rule 0 x 42 [863,179]
+  CRUSH rule 0 x 43 [686,822]
+  CRUSH rule 0 x 44 [396,222]
+  CRUSH rule 0 x 45 [991,694]
+  CRUSH rule 0 x 46 [420,909]
+  CRUSH rule 0 x 47 [467,211]
+  CRUSH rule 0 x 48 [955,329]
+  CRUSH rule 0 x 49 [974,891]
+  CRUSH rule 0 x 50 [870,441]
+  CRUSH rule 0 x 51 [182,930]
+  CRUSH rule 0 x 52 [704,812]
+  CRUSH rule 0 x 53 [185,713]
+  CRUSH rule 0 x 54 [270,441]
+  CRUSH rule 0 x 55 [895,734]
+  CRUSH rule 0 x 56 [564,963]
+  CRUSH rule 0 x 57 [738,130]
+  CRUSH rule 0 x 58 [524,113]
+  CRUSH rule 0 x 59 [408,337]
+  CRUSH rule 0 x 60 [228,790]
+  CRUSH rule 0 x 61 [154,843]
+  CRUSH rule 0 x 62 [594,811]
+  CRUSH rule 0 x 63 [646,67]
+  CRUSH rule 0 x 64 [175,542]
+  CRUSH rule 0 x 65 [745,619]
+  CRUSH rule 0 x 66 [275,468]
+  CRUSH rule 0 x 67 [246,958]
+  CRUSH rule 0 x 68 [711,473]
+  CRUSH rule 0 x 69 [493,924]
+  CRUSH rule 0 x 70 [30,499]
+  CRUSH rule 0 x 71 [984,883]
+  CRUSH rule 0 x 72 [71,286]
+  CRUSH rule 0 x 73 [922,618]
+  CRUSH rule 0 x 74 [629,414]
+  CRUSH rule 0 x 75 [222,20]
+  CRUSH rule 0 x 76 [262,366]
+  CRUSH rule 0 x 77 [638,469]
+  CRUSH rule 0 x 78 [324,511]
+  CRUSH rule 0 x 79 [577,990]
+  CRUSH rule 0 x 80 [501,95]
+  CRUSH rule 0 x 81 [506,812]
+  CRUSH rule 0 x 82 [222,145]
+  CRUSH rule 0 x 83 [71,634]
+  CRUSH rule 0 x 84 [49,761]
+  CRUSH rule 0 x 85 [985,896]
+  CRUSH rule 0 x 86 [537,745]
+  CRUSH rule 0 x 87 [997,317]
+  CRUSH rule 0 x 88 [957,350]
+  CRUSH rule 0 x 89 [399,730]
+  CRUSH rule 0 x 90 [943,706]
+  CRUSH rule 0 x 91 [22,368]
+  CRUSH rule 0 x 92 [532,424]
+  CRUSH rule 0 x 93 [218,489]
+  CRUSH rule 0 x 94 [181,96]
+  CRUSH rule 0 x 95 [343,957]
+  CRUSH rule 0 x 96 [861,270]
+  CRUSH rule 0 x 97 [459,706]
+  CRUSH rule 0 x 98 [327,867]
+  CRUSH rule 0 x 99 [974,133]
+  CRUSH rule 0 x 100 [32,445]
+  CRUSH rule 0 x 101 [142,90]
+  CRUSH rule 0 x 102 [172,129]
+  CRUSH rule 0 x 103 [630,47]
+  CRUSH rule 0 x 104 [758,133]
+  CRUSH rule 0 x 105 [843,604]
+  CRUSH rule 0 x 106 [28,681]
+  CRUSH rule 0 x 107 [74,320]
+  CRUSH rule 0 x 108 [875,593]
+  CRUSH rule 0 x 109 [411,985]
+  CRUSH rule 0 x 110 [440,774]
+  CRUSH rule 0 x 111 [405,742]
+  CRUSH rule 0 x 112 [143,181]
+  CRUSH rule 0 x 113 [153,846]
+  CRUSH rule 0 x 114 [804,892]
+  CRUSH rule 0 x 115 [588,508]
+  CRUSH rule 0 x 116 [327,148]
+  CRUSH rule 0 x 117 [95,594]
+  CRUSH rule 0 x 118 [80,957]
+  CRUSH rule 0 x 119 [386,932]
+  CRUSH rule 0 x 120 [366,312]
+  CRUSH rule 0 x 121 [129,154]
+  CRUSH rule 0 x 122 [873,1]
+  CRUSH rule 0 x 123 [533,415]
+  CRUSH rule 0 x 124 [461,691]
+  CRUSH rule 0 x 125 [342,599]
+  CRUSH rule 0 x 126 [819,781]
+  CRUSH rule 0 x 127 [437,893]
+  CRUSH rule 0 x 128 [679,994]
+  CRUSH rule 0 x 129 [380,685]
+  CRUSH rule 0 x 130 [992,52]
+  CRUSH rule 0 x 131 [469,90]
+  CRUSH rule 0 x 132 [571,250]
+  CRUSH rule 0 x 133 [964,728]
+  CRUSH rule 0 x 134 [999,19]
+  CRUSH rule 0 x 135 [634,101]
+  CRUSH rule 0 x 136 [114,889]
+  CRUSH rule 0 x 137 [839,8]
+  CRUSH rule 0 x 138 [967,949]
+  CRUSH rule 0 x 139 [308,711]
+  CRUSH rule 0 x 140 [764,936]
+  CRUSH rule 0 x 141 [423,302]
+  CRUSH rule 0 x 142 [252,821]
+  CRUSH rule 0 x 143 [33,808]
+  CRUSH rule 0 x 144 [472,88]
+  CRUSH rule 0 x 145 [242,208]
+  CRUSH rule 0 x 146 [290,70]
+  CRUSH rule 0 x 147 [447,352]
+  CRUSH rule 0 x 148 [212,644]
+  CRUSH rule 0 x 149 [9,775]
+  CRUSH rule 0 x 150 [166,456]
+  CRUSH rule 0 x 151 [811,875]
+  CRUSH rule 0 x 152 [449,617]
+  CRUSH rule 0 x 153 [523,537]
+  CRUSH rule 0 x 154 [208,559]
+  CRUSH rule 0 x 155 [569,325]
+  CRUSH rule 0 x 156 [488,121]
+  CRUSH rule 0 x 157 [140,723]
+  CRUSH rule 0 x 158 [786,451]
+  CRUSH rule 0 x 159 [134,664]
+  CRUSH rule 0 x 160 [690,112]
+  CRUSH rule 0 x 161 [324,912]
+  CRUSH rule 0 x 162 [748,567]
+  CRUSH rule 0 x 163 [575,499]
+  CRUSH rule 0 x 164 [314,489]
+  CRUSH rule 0 x 165 [116,209]
+  CRUSH rule 0 x 166 [352,706]
+  CRUSH rule 0 x 167 [27,743]
+  CRUSH rule 0 x 168 [953,898]
+  CRUSH rule 0 x 169 [912,147]
+  CRUSH rule 0 x 170 [421,515]
+  CRUSH rule 0 x 171 [488,584]
+  CRUSH rule 0 x 172 [366,443]
+  CRUSH rule 0 x 173 [863,291]
+  CRUSH rule 0 x 174 [263,555]
+  CRUSH rule 0 x 175 [875,961]
+  CRUSH rule 0 x 176 [745,83]
+  CRUSH rule 0 x 177 [128,244]
+  CRUSH rule 0 x 178 [155,41]
+  CRUSH rule 0 x 179 [593,833]
+  CRUSH rule 0 x 180 [154,734]
+  CRUSH rule 0 x 181 [289,675]
+  CRUSH rule 0 x 182 [730,931]
+  CRUSH rule 0 x 183 [639,237]
+  CRUSH rule 0 x 184 [704,312]
+  CRUSH rule 0 x 185 [97,100]
+  CRUSH rule 0 x 186 [26,665]
+  CRUSH rule 0 x 187 [649,14]
+  CRUSH rule 0 x 188 [682,695]
+  CRUSH rule 0 x 189 [325,693]
+  CRUSH rule 0 x 190 [399,933]
+  CRUSH rule 0 x 191 [629,533]
+  CRUSH rule 0 x 192 [503,578]
+  CRUSH rule 0 x 193 [546,333]
+  CRUSH rule 0 x 194 [242,473]
+  CRUSH rule 0 x 195 [625,719]
+  CRUSH rule 0 x 196 [357,114]
+  CRUSH rule 0 x 197 [306,954]
+  CRUSH rule 0 x 198 [863,791]
+  CRUSH rule 0 x 199 [935,906]
+  CRUSH rule 0 x 200 [373,774]
+  CRUSH rule 0 x 201 [659,320]
+  CRUSH rule 0 x 202 [260,433]
+  CRUSH rule 0 x 203 [36,239]
+  CRUSH rule 0 x 204 [92,516]
+  CRUSH rule 0 x 205 [68,395]
+  CRUSH rule 0 x 206 [570,530]
+  CRUSH rule 0 x 207 [834,457]
+  CRUSH rule 0 x 208 [927,484]
+  CRUSH rule 0 x 209 [878,66]
+  CRUSH rule 0 x 210 [572,981]
+  CRUSH rule 0 x 211 [107,597]
+  CRUSH rule 0 x 212 [389,107]
+  CRUSH rule 0 x 213 [497,717]
+  CRUSH rule 0 x 214 [798,65]
+  CRUSH rule 0 x 215 [233,419]
+  CRUSH rule 0 x 216 [494,464]
+  CRUSH rule 0 x 217 [352,396]
+  CRUSH rule 0 x 218 [895,864]
+  CRUSH rule 0 x 219 [222,534]
+  CRUSH rule 0 x 220 [281,19]
+  CRUSH rule 0 x 221 [64,928]
+  CRUSH rule 0 x 222 [40,544]
+  CRUSH rule 0 x 223 [645,556]
+  CRUSH rule 0 x 224 [647,165]
+  CRUSH rule 0 x 225 [219,714]
+  CRUSH rule 0 x 226 [372,511]
+  CRUSH rule 0 x 227 [925,156]
+  CRUSH rule 0 x 228 [682,404]
+  CRUSH rule 0 x 229 [880,838]
+  CRUSH rule 0 x 230 [328,659]
+  CRUSH rule 0 x 231 [320,383]
+  CRUSH rule 0 x 232 [924,846]
+  CRUSH rule 0 x 233 [948,652]
+  CRUSH rule 0 x 234 [484,943]
+  CRUSH rule 0 x 235 [750,65]
+  CRUSH rule 0 x 236 [551,787]
+  CRUSH rule 0 x 237 [390,157]
+  CRUSH rule 0 x 238 [570,6]
+  CRUSH rule 0 x 239 [729,959]
+  CRUSH rule 0 x 240 [981,241]
+  CRUSH rule 0 x 241 [310,816]
+  CRUSH rule 0 x 242 [161,63]
+  CRUSH rule 0 x 243 [180,394]
+  CRUSH rule 0 x 244 [52,174]
+  CRUSH rule 0 x 245 [523,121]
+  CRUSH rule 0 x 246 [362,893]
+  CRUSH rule 0 x 247 [382,184]
+  CRUSH rule 0 x 248 [129,114]
+  CRUSH rule 0 x 249 [159,683]
+  CRUSH rule 0 x 250 [404,945]
+  CRUSH rule 0 x 251 [661,225]
+  CRUSH rule 0 x 252 [961,226]
+  CRUSH rule 0 x 253 [651,97]
+  CRUSH rule 0 x 254 [123,33]
+  CRUSH rule 0 x 255 [314,649]
+  CRUSH rule 0 x 256 [315,215]
+  CRUSH rule 0 x 257 [825,264]
+  CRUSH rule 0 x 258 [624,789]
+  CRUSH rule 0 x 259 [602,542]
+  CRUSH rule 0 x 260 [717,878]
+  CRUSH rule 0 x 261 [145,517]
+  CRUSH rule 0 x 262 [223,1]
+  CRUSH rule 0 x 263 [462,211]
+  CRUSH rule 0 x 264 [654,471]
+  CRUSH rule 0 x 265 [302,794]
+  CRUSH rule 0 x 266 [202,132]
+  CRUSH rule 0 x 267 [282,938]
+  CRUSH rule 0 x 268 [338,309]
+  CRUSH rule 0 x 269 [738,122]
+  CRUSH rule 0 x 270 [707,982]
+  CRUSH rule 0 x 271 [705,432]
+  CRUSH rule 0 x 272 [756,545]
+  CRUSH rule 0 x 273 [197,502]
+  CRUSH rule 0 x 274 [992,44]
+  CRUSH rule 0 x 275 [544,789]
+  CRUSH rule 0 x 276 [658,467]
+  CRUSH rule 0 x 277 [143,490]
+  CRUSH rule 0 x 278 [492,647]
+  CRUSH rule 0 x 279 [517,792]
+  CRUSH rule 0 x 280 [825,740]
+  CRUSH rule 0 x 281 [224,629]
+  CRUSH rule 0 x 282 [298,661]
+  CRUSH rule 0 x 283 [311,606]
+  CRUSH rule 0 x 284 [771,466]
+  CRUSH rule 0 x 285 [693,362]
+  CRUSH rule 0 x 286 [364,477]
+  CRUSH rule 0 x 287 [591,611]
+  CRUSH rule 0 x 288 [965,541]
+  CRUSH rule 0 x 289 [225,551]
+  CRUSH rule 0 x 290 [577,762]
+  CRUSH rule 0 x 291 [160,903]
+  CRUSH rule 0 x 292 [873,598]
+  CRUSH rule 0 x 293 [100,234]
+  CRUSH rule 0 x 294 [285,943]
+  CRUSH rule 0 x 295 [938,262]
+  CRUSH rule 0 x 296 [850,327]
+  CRUSH rule 0 x 297 [951,53]
+  CRUSH rule 0 x 298 [173,336]
+  CRUSH rule 0 x 299 [598,591]
+  CRUSH rule 0 x 300 [531,957]
+  CRUSH rule 0 x 301 [823,628]
+  CRUSH rule 0 x 302 [184,80]
+  CRUSH rule 0 x 303 [521,766]
+  CRUSH rule 0 x 304 [980,127]
+  CRUSH rule 0 x 305 [153,816]
+  CRUSH rule 0 x 306 [423,739]
+  CRUSH rule 0 x 307 [997,557]
+  CRUSH rule 0 x 308 [991,874]
+  CRUSH rule 0 x 309 [860,394]
+  CRUSH rule 0 x 310 [589,818]
+  CRUSH rule 0 x 311 [477,774]
+  CRUSH rule 0 x 312 [887,853]
+  CRUSH rule 0 x 313 [802,646]
+  CRUSH rule 0 x 314 [654,974]
+  CRUSH rule 0 x 315 [767,227]
+  CRUSH rule 0 x 316 [778,83]
+  CRUSH rule 0 x 317 [184,418]
+  CRUSH rule 0 x 318 [525,410]
+  CRUSH rule 0 x 319 [476,724]
+  CRUSH rule 0 x 320 [149,610]
+  CRUSH rule 0 x 321 [710,79]
+  CRUSH rule 0 x 322 [175,275]
+  CRUSH rule 0 x 323 [819,604]
+  CRUSH rule 0 x 324 [16,745]
+  CRUSH rule 0 x 325 [486,400]
+  CRUSH rule 0 x 326 [613,765]
+  CRUSH rule 0 x 327 [125,289]
+  CRUSH rule 0 x 328 [807,383]
+  CRUSH rule 0 x 329 [588,938]
+  CRUSH rule 0 x 330 [932,644]
+  CRUSH rule 0 x 331 [341,953]
+  CRUSH rule 0 x 332 [153,726]
+  CRUSH rule 0 x 333 [745,845]
+  CRUSH rule 0 x 334 [614,751]
+  CRUSH rule 0 x 335 [518,721]
+  CRUSH rule 0 x 336 [389,424]
+  CRUSH rule 0 x 337 [753,508]
+  CRUSH rule 0 x 338 [128,810]
+  CRUSH rule 0 x 339 [430,308]
+  CRUSH rule 0 x 340 [541,44]
+  CRUSH rule 0 x 341 [402,26]
+  CRUSH rule 0 x 342 [982,57]
+  CRUSH rule 0 x 343 [833,412]
+  CRUSH rule 0 x 344 [784,533]
+  CRUSH rule 0 x 345 [546,300]
+  CRUSH rule 0 x 346 [302,420]
+  CRUSH rule 0 x 347 [488,778]
+  CRUSH rule 0 x 348 [903,744]
+  CRUSH rule 0 x 349 [471,547]
+  CRUSH rule 0 x 350 [348,221]
+  CRUSH rule 0 x 351 [961,582]
+  CRUSH rule 0 x 352 [728,137]
+  CRUSH rule 0 x 353 [904,202]
+  CRUSH rule 0 x 354 [345,226]
+  CRUSH rule 0 x 355 [50,430]
+  CRUSH rule 0 x 356 [87,185]
+  CRUSH rule 0 x 357 [762,459]
+  CRUSH rule 0 x 358 [908,25]
+  CRUSH rule 0 x 359 [484,15]
+  CRUSH rule 0 x 360 [173,378]
+  CRUSH rule 0 x 361 [404,577]
+  CRUSH rule 0 x 362 [403,1]
+  CRUSH rule 0 x 363 [639,911]
+  CRUSH rule 0 x 364 [752,689]
+  CRUSH rule 0 x 365 [956,999]
+  CRUSH rule 0 x 366 [860,925]
+  CRUSH rule 0 x 367 [205,609]
+  CRUSH rule 0 x 368 [301,284]
+  CRUSH rule 0 x 369 [452,658]
+  CRUSH rule 0 x 370 [11,467]
+  CRUSH rule 0 x 371 [124,487]
+  CRUSH rule 0 x 372 [253,48]
+  CRUSH rule 0 x 373 [715,605]
+  CRUSH rule 0 x 374 [191,887]
+  CRUSH rule 0 x 375 [711,385]
+  CRUSH rule 0 x 376 [597,818]
+  CRUSH rule 0 x 377 [294,256]
+  CRUSH rule 0 x 378 [34,151]
+  CRUSH rule 0 x 379 [869,136]
+  CRUSH rule 0 x 380 [294,97]
+  CRUSH rule 0 x 381 [119,710]
+  CRUSH rule 0 x 382 [69,631]
+  CRUSH rule 0 x 383 [922,588]
+  CRUSH rule 0 x 384 [221,945]
+  CRUSH rule 0 x 385 [561,737]
+  CRUSH rule 0 x 386 [335,442]
+  CRUSH rule 0 x 387 [514,43]
+  CRUSH rule 0 x 388 [587,89]
+  CRUSH rule 0 x 389 [109,641]
+  CRUSH rule 0 x 390 [925,149]
+  CRUSH rule 0 x 391 [267,87]
+  CRUSH rule 0 x 392 [382,485]
+  CRUSH rule 0 x 393 [425,721]
+  CRUSH rule 0 x 394 [898,18]
+  CRUSH rule 0 x 395 [806,876]
+  CRUSH rule 0 x 396 [790,970]
+  CRUSH rule 0 x 397 [136,363]
+  CRUSH rule 0 x 398 [914,116]
+  CRUSH rule 0 x 399 [261,94]
+  CRUSH rule 0 x 400 [661,197]
+  CRUSH rule 0 x 401 [953,979]
+  CRUSH rule 0 x 402 [738,819]
+  CRUSH rule 0 x 403 [573,238]
+  CRUSH rule 0 x 404 [526,848]
+  CRUSH rule 0 x 405 [582,505]
+  CRUSH rule 0 x 406 [768,324]
+  CRUSH rule 0 x 407 [260,951]
+  CRUSH rule 0 x 408 [657,81]
+  CRUSH rule 0 x 409 [498,89]
+  CRUSH rule 0 x 410 [28,793]
+  CRUSH rule 0 x 411 [684,992]
+  CRUSH rule 0 x 412 [261,958]
+  CRUSH rule 0 x 413 [891,835]
+  CRUSH rule 0 x 414 [127,459]
+  CRUSH rule 0 x 415 [272,540]
+  CRUSH rule 0 x 416 [739,617]
+  CRUSH rule 0 x 417 [106,209]
+  CRUSH rule 0 x 418 [525,441]
+  CRUSH rule 0 x 419 [603,673]
+  CRUSH rule 0 x 420 [988,213]
+  CRUSH rule 0 x 421 [761,521]
+  CRUSH rule 0 x 422 [317,160]
+  CRUSH rule 0 x 423 [137,807]
+  CRUSH rule 0 x 424 [920,37]
+  CRUSH rule 0 x 425 [277,693]
+  CRUSH rule 0 x 426 [485,936]
+  CRUSH rule 0 x 427 [242,515]
+  CRUSH rule 0 x 428 [632,635]
+  CRUSH rule 0 x 429 [641,73]
+  CRUSH rule 0 x 430 [626,585]
+  CRUSH rule 0 x 431 [697,76]
+  CRUSH rule 0 x 432 [590,526]
+  CRUSH rule 0 x 433 [284,387]
+  CRUSH rule 0 x 434 [538,985]
+  CRUSH rule 0 x 435 [30,318]
+  CRUSH rule 0 x 436 [164,919]
+  CRUSH rule 0 x 437 [322,212]
+  CRUSH rule 0 x 438 [142,392]
+  CRUSH rule 0 x 439 [119,370]
+  CRUSH rule 0 x 440 [333,403]
+  CRUSH rule 0 x 441 [477,727]
+  CRUSH rule 0 x 442 [274,590]
+  CRUSH rule 0 x 443 [983,748]
+  CRUSH rule 0 x 444 [536,509]
+  CRUSH rule 0 x 445 [485,482]
+  CRUSH rule 0 x 446 [345,634]
+  CRUSH rule 0 x 447 [61,845]
+  CRUSH rule 0 x 448 [333,232]
+  CRUSH rule 0 x 449 [680,16]
+  CRUSH rule 0 x 450 [235,214]
+  CRUSH rule 0 x 451 [961,468]
+  CRUSH rule 0 x 452 [525,479]
+  CRUSH rule 0 x 453 [138,466]
+  CRUSH rule 0 x 454 [137,625]
+  CRUSH rule 0 x 455 [173,150]
+  CRUSH rule 0 x 456 [235,226]
+  CRUSH rule 0 x 457 [450,577]
+  CRUSH rule 0 x 458 [195,537]
+  CRUSH rule 0 x 459 [381,555]
+  CRUSH rule 0 x 460 [972,730]
+  CRUSH rule 0 x 461 [506,279]
+  CRUSH rule 0 x 462 [692,959]
+  CRUSH rule 0 x 463 [788,667]
+  CRUSH rule 0 x 464 [133,122]
+  CRUSH rule 0 x 465 [971,190]
+  CRUSH rule 0 x 466 [394,576]
+  CRUSH rule 0 x 467 [517,28]
+  CRUSH rule 0 x 468 [829,143]
+  CRUSH rule 0 x 469 [987,936]
+  CRUSH rule 0 x 470 [107,982]
+  CRUSH rule 0 x 471 [181,897]
+  CRUSH rule 0 x 472 [547,512]
+  CRUSH rule 0 x 473 [760,997]
+  CRUSH rule 0 x 474 [787,418]
+  CRUSH rule 0 x 475 [662,312]
+  CRUSH rule 0 x 476 [110,495]
+  CRUSH rule 0 x 477 [393,954]
+  CRUSH rule 0 x 478 [246,483]
+  CRUSH rule 0 x 479 [70,929]
+  CRUSH rule 0 x 480 [753,119]
+  CRUSH rule 0 x 481 [470,429]
+  CRUSH rule 0 x 482 [451,566]
+  CRUSH rule 0 x 483 [816,72]
+  CRUSH rule 0 x 484 [540,454]
+  CRUSH rule 0 x 485 [74,582]
+  CRUSH rule 0 x 486 [958,595]
+  CRUSH rule 0 x 487 [228,302]
+  CRUSH rule 0 x 488 [180,529]
+  CRUSH rule 0 x 489 [47,617]
+  CRUSH rule 0 x 490 [905,822]
+  CRUSH rule 0 x 491 [892,370]
+  CRUSH rule 0 x 492 [588,959]
+  CRUSH rule 0 x 493 [353,461]
+  CRUSH rule 0 x 494 [378,848]
+  CRUSH rule 0 x 495 [845,653]
+  CRUSH rule 0 x 496 [13,988]
+  CRUSH rule 0 x 497 [796,877]
+  CRUSH rule 0 x 498 [412,337]
+  CRUSH rule 0 x 499 [330,695]
+  CRUSH rule 0 x 500 [820,272]
+  CRUSH rule 0 x 501 [110,44]
+  CRUSH rule 0 x 502 [336,595]
+  CRUSH rule 0 x 503 [922,211]
+  CRUSH rule 0 x 504 [483,52]
+  CRUSH rule 0 x 505 [482,598]
+  CRUSH rule 0 x 506 [493,123]
+  CRUSH rule 0 x 507 [12,598]
+  CRUSH rule 0 x 508 [227,157]
+  CRUSH rule 0 x 509 [807,242]
+  CRUSH rule 0 x 510 [134,437]
+  CRUSH rule 0 x 511 [212,54]
+  CRUSH rule 0 x 512 [236,630]
+  CRUSH rule 0 x 513 [994,693]
+  CRUSH rule 0 x 514 [45,508]
+  CRUSH rule 0 x 515 [504,138]
+  CRUSH rule 0 x 516 [285,409]
+  CRUSH rule 0 x 517 [300,232]
+  CRUSH rule 0 x 518 [397,674]
+  CRUSH rule 0 x 519 [86,750]
+  CRUSH rule 0 x 520 [900,833]
+  CRUSH rule 0 x 521 [31,47]
+  CRUSH rule 0 x 522 [390,16]
+  CRUSH rule 0 x 523 [618,308]
+  CRUSH rule 0 x 524 [635,189]
+  CRUSH rule 0 x 525 [311,916]
+  CRUSH rule 0 x 526 [48,738]
+  CRUSH rule 0 x 527 [202,851]
+  CRUSH rule 0 x 528 [565,827]
+  CRUSH rule 0 x 529 [934,864]
+  CRUSH rule 0 x 530 [502,934]
+  CRUSH rule 0 x 531 [681,627]
+  CRUSH rule 0 x 532 [422,6]
+  CRUSH rule 0 x 533 [863,68]
+  CRUSH rule 0 x 534 [962,931]
+  CRUSH rule 0 x 535 [89,565]
+  CRUSH rule 0 x 536 [499,351]
+  CRUSH rule 0 x 537 [676,547]
+  CRUSH rule 0 x 538 [58,644]
+  CRUSH rule 0 x 539 [837,953]
+  CRUSH rule 0 x 540 [831,50]
+  CRUSH rule 0 x 541 [582,757]
+  CRUSH rule 0 x 542 [472,132]
+  CRUSH rule 0 x 543 [382,272]
+  CRUSH rule 0 x 544 [947,930]
+  CRUSH rule 0 x 545 [425,570]
+  CRUSH rule 0 x 546 [18,65]
+  CRUSH rule 0 x 547 [445,715]
+  CRUSH rule 0 x 548 [367,569]
+  CRUSH rule 0 x 549 [125,715]
+  CRUSH rule 0 x 550 [425,599]
+  CRUSH rule 0 x 551 [44,1]
+  CRUSH rule 0 x 552 [246,104]
+  CRUSH rule 0 x 553 [71,703]
+  CRUSH rule 0 x 554 [207,124]
+  CRUSH rule 0 x 555 [570,28]
+  CRUSH rule 0 x 556 [674,152]
+  CRUSH rule 0 x 557 [347,817]
+  CRUSH rule 0 x 558 [627,426]
+  CRUSH rule 0 x 559 [940,630]
+  CRUSH rule 0 x 560 [295,903]
+  CRUSH rule 0 x 561 [506,682]
+  CRUSH rule 0 x 562 [718,529]
+  CRUSH rule 0 x 563 [552,332]
+  CRUSH rule 0 x 564 [835,769]
+  CRUSH rule 0 x 565 [8,167]
+  CRUSH rule 0 x 566 [600,481]
+  CRUSH rule 0 x 567 [999,994]
+  CRUSH rule 0 x 568 [252,431]
+  CRUSH rule 0 x 569 [643,218]
+  CRUSH rule 0 x 570 [617,635]
+  CRUSH rule 0 x 571 [757,80]
+  CRUSH rule 0 x 572 [299,348]
+  CRUSH rule 0 x 573 [25,505]
+  CRUSH rule 0 x 574 [215,431]
+  CRUSH rule 0 x 575 [225,252]
+  CRUSH rule 0 x 576 [627,94]
+  CRUSH rule 0 x 577 [237,809]
+  CRUSH rule 0 x 578 [885,313]
+  CRUSH rule 0 x 579 [924,575]
+  CRUSH rule 0 x 580 [718,51]
+  CRUSH rule 0 x 581 [219,807]
+  CRUSH rule 0 x 582 [893,701]
+  CRUSH rule 0 x 583 [246,930]
+  CRUSH rule 0 x 584 [336,432]
+  CRUSH rule 0 x 585 [324,999]
+  CRUSH rule 0 x 586 [558,230]
+  CRUSH rule 0 x 587 [985,830]
+  CRUSH rule 0 x 588 [211,544]
+  CRUSH rule 0 x 589 [129,21]
+  CRUSH rule 0 x 590 [467,969]
+  CRUSH rule 0 x 591 [758,514]
+  CRUSH rule 0 x 592 [525,253]
+  CRUSH rule 0 x 593 [601,885]
+  CRUSH rule 0 x 594 [227,60]
+  CRUSH rule 0 x 595 [720,854]
+  CRUSH rule 0 x 596 [751,195]
+  CRUSH rule 0 x 597 [129,574]
+  CRUSH rule 0 x 598 [679,207]
+  CRUSH rule 0 x 599 [668,315]
+  CRUSH rule 0 x 600 [143,396]
+  CRUSH rule 0 x 601 [326,573]
+  CRUSH rule 0 x 602 [860,281]
+  CRUSH rule 0 x 603 [709,328]
+  CRUSH rule 0 x 604 [571,62]
+  CRUSH rule 0 x 605 [252,739]
+  CRUSH rule 0 x 606 [339,236]
+  CRUSH rule 0 x 607 [590,248]
+  CRUSH rule 0 x 608 [145,635]
+  CRUSH rule 0 x 609 [973,547]
+  CRUSH rule 0 x 610 [435,816]
+  CRUSH rule 0 x 611 [559,283]
+  CRUSH rule 0 x 612 [273,149]
+  CRUSH rule 0 x 613 [828,614]
+  CRUSH rule 0 x 614 [478,748]
+  CRUSH rule 0 x 615 [392,155]
+  CRUSH rule 0 x 616 [778,637]
+  CRUSH rule 0 x 617 [622,713]
+  CRUSH rule 0 x 618 [149,877]
+  CRUSH rule 0 x 619 [604,163]
+  CRUSH rule 0 x 620 [181,23]
+  CRUSH rule 0 x 621 [735,902]
+  CRUSH rule 0 x 622 [661,824]
+  CRUSH rule 0 x 623 [142,121]
+  CRUSH rule 0 x 624 [360,716]
+  CRUSH rule 0 x 625 [541,167]
+  CRUSH rule 0 x 626 [364,431]
+  CRUSH rule 0 x 627 [458,137]
+  CRUSH rule 0 x 628 [250,350]
+  CRUSH rule 0 x 629 [928,160]
+  CRUSH rule 0 x 630 [243,19]
+  CRUSH rule 0 x 631 [438,221]
+  CRUSH rule 0 x 632 [797,368]
+  CRUSH rule 0 x 633 [993,749]
+  CRUSH rule 0 x 634 [239,351]
+  CRUSH rule 0 x 635 [640,965]
+  CRUSH rule 0 x 636 [173,290]
+  CRUSH rule 0 x 637 [0,918]
+  CRUSH rule 0 x 638 [702,235]
+  CRUSH rule 0 x 639 [475,687]
+  CRUSH rule 0 x 640 [31,664]
+  CRUSH rule 0 x 641 [296,473]
+  CRUSH rule 0 x 642 [894,273]
+  CRUSH rule 0 x 643 [117,111]
+  CRUSH rule 0 x 644 [438,336]
+  CRUSH rule 0 x 645 [982,702]
+  CRUSH rule 0 x 646 [334,804]
+  CRUSH rule 0 x 647 [933,787]
+  CRUSH rule 0 x 648 [22,444]
+  CRUSH rule 0 x 649 [503,229]
+  CRUSH rule 0 x 650 [328,659]
+  CRUSH rule 0 x 651 [3,880]
+  CRUSH rule 0 x 652 [495,977]
+  CRUSH rule 0 x 653 [185,718]
+  CRUSH rule 0 x 654 [130,528]
+  CRUSH rule 0 x 655 [560,872]
+  CRUSH rule 0 x 656 [219,885]
+  CRUSH rule 0 x 657 [233,684]
+  CRUSH rule 0 x 658 [778,6]
+  CRUSH rule 0 x 659 [240,663]
+  CRUSH rule 0 x 660 [244,855]
+  CRUSH rule 0 x 661 [184,270]
+  CRUSH rule 0 x 662 [65,883]
+  CRUSH rule 0 x 663 [323,721]
+  CRUSH rule 0 x 664 [865,113]
+  CRUSH rule 0 x 665 [420,850]
+  CRUSH rule 0 x 666 [319,767]
+  CRUSH rule 0 x 667 [875,39]
+  CRUSH rule 0 x 668 [331,122]
+  CRUSH rule 0 x 669 [915,521]
+  CRUSH rule 0 x 670 [845,659]
+  CRUSH rule 0 x 671 [108,634]
+  CRUSH rule 0 x 672 [578,216]
+  CRUSH rule 0 x 673 [442,74]
+  CRUSH rule 0 x 674 [588,364]
+  CRUSH rule 0 x 675 [489,698]
+  CRUSH rule 0 x 676 [928,911]
+  CRUSH rule 0 x 677 [399,269]
+  CRUSH rule 0 x 678 [546,752]
+  CRUSH rule 0 x 679 [988,25]
+  CRUSH rule 0 x 680 [335,963]
+  CRUSH rule 0 x 681 [690,462]
+  CRUSH rule 0 x 682 [196,588]
+  CRUSH rule 0 x 683 [627,25]
+  CRUSH rule 0 x 684 [38,804]
+  CRUSH rule 0 x 685 [841,368]
+  CRUSH rule 0 x 686 [336,287]
+  CRUSH rule 0 x 687 [20,682]
+  CRUSH rule 0 x 688 [463,371]
+  CRUSH rule 0 x 689 [569,250]
+  CRUSH rule 0 x 690 [551,144]
+  CRUSH rule 0 x 691 [766,464]
+  CRUSH rule 0 x 692 [739,634]
+  CRUSH rule 0 x 693 [339,297]
+  CRUSH rule 0 x 694 [405,26]
+  CRUSH rule 0 x 695 [622,576]
+  CRUSH rule 0 x 696 [558,902]
+  CRUSH rule 0 x 697 [818,222]
+  CRUSH rule 0 x 698 [178,48]
+  CRUSH rule 0 x 699 [450,244]
+  CRUSH rule 0 x 700 [502,771]
+  CRUSH rule 0 x 701 [4,612]
+  CRUSH rule 0 x 702 [177,630]
+  CRUSH rule 0 x 703 [354,178]
+  CRUSH rule 0 x 704 [646,601]
+  CRUSH rule 0 x 705 [921,401]
+  CRUSH rule 0 x 706 [652,877]
+  CRUSH rule 0 x 707 [345,745]
+  CRUSH rule 0 x 708 [333,607]
+  CRUSH rule 0 x 709 [45,187]
+  CRUSH rule 0 x 710 [94,855]
+  CRUSH rule 0 x 711 [227,653]
+  CRUSH rule 0 x 712 [398,953]
+  CRUSH rule 0 x 713 [116,800]
+  CRUSH rule 0 x 714 [111,629]
+  CRUSH rule 0 x 715 [531,291]
+  CRUSH rule 0 x 716 [169,541]
+  CRUSH rule 0 x 717 [417,446]
+  CRUSH rule 0 x 718 [992,383]
+  CRUSH rule 0 x 719 [936,674]
+  CRUSH rule 0 x 720 [370,188]
+  CRUSH rule 0 x 721 [320,859]
+  CRUSH rule 0 x 722 [7,2]
+  CRUSH rule 0 x 723 [270,553]
+  CRUSH rule 0 x 724 [666,822]
+  CRUSH rule 0 x 725 [794,406]
+  CRUSH rule 0 x 726 [420,556]
+  CRUSH rule 0 x 727 [561,461]
+  CRUSH rule 0 x 728 [951,330]
+  CRUSH rule 0 x 729 [656,644]
+  CRUSH rule 0 x 730 [3,558]
+  CRUSH rule 0 x 731 [852,89]
+  CRUSH rule 0 x 732 [983,840]
+  CRUSH rule 0 x 733 [285,396]
+  CRUSH rule 0 x 734 [125,510]
+  CRUSH rule 0 x 735 [417,773]
+  CRUSH rule 0 x 736 [749,396]
+  CRUSH rule 0 x 737 [644,991]
+  CRUSH rule 0 x 738 [449,683]
+  CRUSH rule 0 x 739 [341,220]
+  CRUSH rule 0 x 740 [874,524]
+  CRUSH rule 0 x 741 [189,472]
+  CRUSH rule 0 x 742 [912,581]
+  CRUSH rule 0 x 743 [654,914]
+  CRUSH rule 0 x 744 [725,295]
+  CRUSH rule 0 x 745 [787,858]
+  CRUSH rule 0 x 746 [757,848]
+  CRUSH rule 0 x 747 [700,81]
+  CRUSH rule 0 x 748 [557,436]
+  CRUSH rule 0 x 749 [772,622]
+  CRUSH rule 0 x 750 [946,97]
+  CRUSH rule 0 x 751 [996,618]
+  CRUSH rule 0 x 752 [746,887]
+  CRUSH rule 0 x 753 [741,14]
+  CRUSH rule 0 x 754 [648,349]
+  CRUSH rule 0 x 755 [157,460]
+  CRUSH rule 0 x 756 [416,97]
+  CRUSH rule 0 x 757 [599,839]
+  CRUSH rule 0 x 758 [994,218]
+  CRUSH rule 0 x 759 [959,682]
+  CRUSH rule 0 x 760 [518,943]
+  CRUSH rule 0 x 761 [285,849]
+  CRUSH rule 0 x 762 [591,313]
+  CRUSH rule 0 x 763 [908,411]
+  CRUSH rule 0 x 764 [787,234]
+  CRUSH rule 0 x 765 [327,921]
+  CRUSH rule 0 x 766 [84,161]
+  CRUSH rule 0 x 767 [370,895]
+  CRUSH rule 0 x 768 [826,760]
+  CRUSH rule 0 x 769 [67,768]
+  CRUSH rule 0 x 770 [593,909]
+  CRUSH rule 0 x 771 [309,935]
+  CRUSH rule 0 x 772 [12,125]
+  CRUSH rule 0 x 773 [253,466]
+  CRUSH rule 0 x 774 [164,390]
+  CRUSH rule 0 x 775 [703,47]
+  CRUSH rule 0 x 776 [728,231]
+  CRUSH rule 0 x 777 [981,621]
+  CRUSH rule 0 x 778 [411,456]
+  CRUSH rule 0 x 779 [346,121]
+  CRUSH rule 0 x 780 [476,39]
+  CRUSH rule 0 x 781 [10,130]
+  CRUSH rule 0 x 782 [462,246]
+  CRUSH rule 0 x 783 [580,373]
+  CRUSH rule 0 x 784 [413,113]
+  CRUSH rule 0 x 785 [341,856]
+  CRUSH rule 0 x 786 [411,140]
+  CRUSH rule 0 x 787 [605,522]
+  CRUSH rule 0 x 788 [226,545]
+  CRUSH rule 0 x 789 [545,320]
+  CRUSH rule 0 x 790 [414,748]
+  CRUSH rule 0 x 791 [660,906]
+  CRUSH rule 0 x 792 [287,392]
+  CRUSH rule 0 x 793 [631,133]
+  CRUSH rule 0 x 794 [931,517]
+  CRUSH rule 0 x 795 [551,962]
+  CRUSH rule 0 x 796 [814,4]
+  CRUSH rule 0 x 797 [64,201]
+  CRUSH rule 0 x 798 [422,530]
+  CRUSH rule 0 x 799 [824,32]
+  CRUSH rule 0 x 800 [862,623]
+  CRUSH rule 0 x 801 [145,550]
+  CRUSH rule 0 x 802 [570,19]
+  CRUSH rule 0 x 803 [151,812]
+  CRUSH rule 0 x 804 [467,93]
+  CRUSH rule 0 x 805 [621,223]
+  CRUSH rule 0 x 806 [898,957]
+  CRUSH rule 0 x 807 [354,531]
+  CRUSH rule 0 x 808 [7,96]
+  CRUSH rule 0 x 809 [70,734]
+  CRUSH rule 0 x 810 [701,18]
+  CRUSH rule 0 x 811 [248,547]
+  CRUSH rule 0 x 812 [230,576]
+  CRUSH rule 0 x 813 [805,114]
+  CRUSH rule 0 x 814 [54,619]
+  CRUSH rule 0 x 815 [679,412]
+  CRUSH rule 0 x 816 [919,448]
+  CRUSH rule 0 x 817 [765,830]
+  CRUSH rule 0 x 818 [415,566]
+  CRUSH rule 0 x 819 [721,319]
+  CRUSH rule 0 x 820 [218,301]
+  CRUSH rule 0 x 821 [185,795]
+  CRUSH rule 0 x 822 [356,261]
+  CRUSH rule 0 x 823 [220,281]
+  CRUSH rule 0 x 824 [292,809]
+  CRUSH rule 0 x 825 [949,778]
+  CRUSH rule 0 x 826 [767,818]
+  CRUSH rule 0 x 827 [631,83]
+  CRUSH rule 0 x 828 [288,986]
+  CRUSH rule 0 x 829 [990,667]
+  CRUSH rule 0 x 830 [152,571]
+  CRUSH rule 0 x 831 [814,563]
+  CRUSH rule 0 x 832 [235,641]
+  CRUSH rule 0 x 833 [657,565]
+  CRUSH rule 0 x 834 [907,231]
+  CRUSH rule 0 x 835 [784,262]
+  CRUSH rule 0 x 836 [951,158]
+  CRUSH rule 0 x 837 [556,498]
+  CRUSH rule 0 x 838 [329,274]
+  CRUSH rule 0 x 839 [568,209]
+  CRUSH rule 0 x 840 [45,579]
+  CRUSH rule 0 x 841 [652,702]
+  CRUSH rule 0 x 842 [629,984]
+  CRUSH rule 0 x 843 [799,690]
+  CRUSH rule 0 x 844 [694,600]
+  CRUSH rule 0 x 845 [332,30]
+  CRUSH rule 0 x 846 [452,251]
+  CRUSH rule 0 x 847 [399,681]
+  CRUSH rule 0 x 848 [303,138]
+  CRUSH rule 0 x 849 [666,346]
+  CRUSH rule 0 x 850 [644,511]
+  CRUSH rule 0 x 851 [527,546]
+  CRUSH rule 0 x 852 [31,809]
+  CRUSH rule 0 x 853 [483,330]
+  CRUSH rule 0 x 854 [697,953]
+  CRUSH rule 0 x 855 [837,996]
+  CRUSH rule 0 x 856 [712,40]
+  CRUSH rule 0 x 857 [77,984]
+  CRUSH rule 0 x 858 [412,384]
+  CRUSH rule 0 x 859 [173,760]
+  CRUSH rule 0 x 860 [776,429]
+  CRUSH rule 0 x 861 [705,405]
+  CRUSH rule 0 x 862 [809,44]
+  CRUSH rule 0 x 863 [349,496]
+  CRUSH rule 0 x 864 [717,858]
+  CRUSH rule 0 x 865 [857,603]
+  CRUSH rule 0 x 866 [394,304]
+  CRUSH rule 0 x 867 [640,773]
+  CRUSH rule 0 x 868 [613,950]
+  CRUSH rule 0 x 869 [973,889]
+  CRUSH rule 0 x 870 [505,35]
+  CRUSH rule 0 x 871 [239,264]
+  CRUSH rule 0 x 872 [21,767]
+  CRUSH rule 0 x 873 [954,666]
+  CRUSH rule 0 x 874 [54,510]
+  CRUSH rule 0 x 875 [809,418]
+  CRUSH rule 0 x 876 [483,457]
+  CRUSH rule 0 x 877 [542,531]
+  CRUSH rule 0 x 878 [217,674]
+  CRUSH rule 0 x 879 [999,475]
+  CRUSH rule 0 x 880 [678,573]
+  CRUSH rule 0 x 881 [394,835]
+  CRUSH rule 0 x 882 [467,382]
+  CRUSH rule 0 x 883 [802,744]
+  CRUSH rule 0 x 884 [653,660]
+  CRUSH rule 0 x 885 [898,704]
+  CRUSH rule 0 x 886 [434,357]
+  CRUSH rule 0 x 887 [297,226]
+  CRUSH rule 0 x 888 [863,324]
+  CRUSH rule 0 x 889 [105,102]
+  CRUSH rule 0 x 890 [550,248]
+  CRUSH rule 0 x 891 [575,928]
+  CRUSH rule 0 x 892 [259,862]
+  CRUSH rule 0 x 893 [902,880]
+  CRUSH rule 0 x 894 [180,169]
+  CRUSH rule 0 x 895 [725,849]
+  CRUSH rule 0 x 896 [951,34]
+  CRUSH rule 0 x 897 [810,352]
+  CRUSH rule 0 x 898 [979,433]
+  CRUSH rule 0 x 899 [685,668]
+  CRUSH rule 0 x 900 [530,978]
+  CRUSH rule 0 x 901 [740,107]
+  CRUSH rule 0 x 902 [800,743]
+  CRUSH rule 0 x 903 [230,267]
+  CRUSH rule 0 x 904 [346,949]
+  CRUSH rule 0 x 905 [530,397]
+  CRUSH rule 0 x 906 [80,426]
+  CRUSH rule 0 x 907 [365,968]
+  CRUSH rule 0 x 908 [204,832]
+  CRUSH rule 0 x 909 [883,989]
+  CRUSH rule 0 x 910 [549,593]
+  CRUSH rule 0 x 911 [325,847]
+  CRUSH rule 0 x 912 [874,888]
+  CRUSH rule 0 x 913 [331,463]
+  CRUSH rule 0 x 914 [836,468]
+  CRUSH rule 0 x 915 [245,228]
+  CRUSH rule 0 x 916 [77,967]
+  CRUSH rule 0 x 917 [239,60]
+  CRUSH rule 0 x 918 [988,115]
+  CRUSH rule 0 x 919 [783,139]
+  CRUSH rule 0 x 920 [623,408]
+  CRUSH rule 0 x 921 [105,799]
+  CRUSH rule 0 x 922 [887,505]
+  CRUSH rule 0 x 923 [223,318]
+  CRUSH rule 0 x 924 [25,778]
+  CRUSH rule 0 x 925 [912,601]
+  CRUSH rule 0 x 926 [968,133]
+  CRUSH rule 0 x 927 [277,724]
+  CRUSH rule 0 x 928 [554,203]
+  CRUSH rule 0 x 929 [761,802]
+  CRUSH rule 0 x 930 [814,61]
+  CRUSH rule 0 x 931 [29,193]
+  CRUSH rule 0 x 932 [446,198]
+  CRUSH rule 0 x 933 [352,742]
+  CRUSH rule 0 x 934 [730,2]
+  CRUSH rule 0 x 935 [731,23]
+  CRUSH rule 0 x 936 [322,975]
+  CRUSH rule 0 x 937 [822,221]
+  CRUSH rule 0 x 938 [557,850]
+  CRUSH rule 0 x 939 [150,11]
+  CRUSH rule 0 x 940 [638,398]
+  CRUSH rule 0 x 941 [730,342]
+  CRUSH rule 0 x 942 [62,292]
+  CRUSH rule 0 x 943 [165,314]
+  CRUSH rule 0 x 944 [199,625]
+  CRUSH rule 0 x 945 [946,999]
+  CRUSH rule 0 x 946 [595,93]
+  CRUSH rule 0 x 947 [800,582]
+  CRUSH rule 0 x 948 [132,551]
+  CRUSH rule 0 x 949 [792,920]
+  CRUSH rule 0 x 950 [111,345]
+  CRUSH rule 0 x 951 [414,619]
+  CRUSH rule 0 x 952 [775,469]
+  CRUSH rule 0 x 953 [349,1]
+  CRUSH rule 0 x 954 [570,940]
+  CRUSH rule 0 x 955 [729,774]
+  CRUSH rule 0 x 956 [519,141]
+  CRUSH rule 0 x 957 [242,709]
+  CRUSH rule 0 x 958 [84,217]
+  CRUSH rule 0 x 959 [270,413]
+  CRUSH rule 0 x 960 [458,192]
+  CRUSH rule 0 x 961 [981,388]
+  CRUSH rule 0 x 962 [623,834]
+  CRUSH rule 0 x 963 [291,167]
+  CRUSH rule 0 x 964 [28,156]
+  CRUSH rule 0 x 965 [675,557]
+  CRUSH rule 0 x 966 [836,306]
+  CRUSH rule 0 x 967 [966,386]
+  CRUSH rule 0 x 968 [864,756]
+  CRUSH rule 0 x 969 [729,625]
+  CRUSH rule 0 x 970 [800,362]
+  CRUSH rule 0 x 971 [737,381]
+  CRUSH rule 0 x 972 [952,245]
+  CRUSH rule 0 x 973 [356,455]
+  CRUSH rule 0 x 974 [545,758]
+  CRUSH rule 0 x 975 [336,191]
+  CRUSH rule 0 x 976 [446,208]
+  CRUSH rule 0 x 977 [202,896]
+  CRUSH rule 0 x 978 [612,324]
+  CRUSH rule 0 x 979 [843,457]
+  CRUSH rule 0 x 980 [60,914]
+  CRUSH rule 0 x 981 [702,749]
+  CRUSH rule 0 x 982 [298,928]
+  CRUSH rule 0 x 983 [723,572]
+  CRUSH rule 0 x 984 [723,864]
+  CRUSH rule 0 x 985 [945,459]
+  CRUSH rule 0 x 986 [772,664]
+  CRUSH rule 0 x 987 [88,324]
+  CRUSH rule 0 x 988 [522,927]
+  CRUSH rule 0 x 989 [578,332]
+  CRUSH rule 0 x 990 [638,228]
+  CRUSH rule 0 x 991 [530,221]
+  CRUSH rule 0 x 992 [925,705]
+  CRUSH rule 0 x 993 [991,301]
+  CRUSH rule 0 x 994 [276,51]
+  CRUSH rule 0 x 995 [288,836]
+  CRUSH rule 0 x 996 [887,983]
+  CRUSH rule 0 x 997 [110,924]
+  CRUSH rule 0 x 998 [435,830]
+  CRUSH rule 0 x 999 [876,738]
+  CRUSH rule 0 x 1000 [178,963]
+  CRUSH rule 0 x 1001 [99,519]
+  CRUSH rule 0 x 1002 [515,534]
+  CRUSH rule 0 x 1003 [104,611]
+  CRUSH rule 0 x 1004 [269,638]
+  CRUSH rule 0 x 1005 [369,223]
+  CRUSH rule 0 x 1006 [40,107]
+  CRUSH rule 0 x 1007 [978,111]
+  CRUSH rule 0 x 1008 [965,956]
+  CRUSH rule 0 x 1009 [598,476]
+  CRUSH rule 0 x 1010 [767,523]
+  CRUSH rule 0 x 1011 [289,871]
+  CRUSH rule 0 x 1012 [128,28]
+  CRUSH rule 0 x 1013 [979,765]
+  CRUSH rule 0 x 1014 [979,948]
+  CRUSH rule 0 x 1015 [277,790]
+  CRUSH rule 0 x 1016 [262,73]
+  CRUSH rule 0 x 1017 [150,269]
+  CRUSH rule 0 x 1018 [555,829]
+  CRUSH rule 0 x 1019 [513,356]
+  CRUSH rule 0 x 1020 [158,161]
+  CRUSH rule 0 x 1021 [915,998]
+  CRUSH rule 0 x 1022 [967,829]
+  CRUSH rule 0 x 1023 [488,257]
+  rule 0 (data) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536]
+  CRUSH rule 0 x 1 [876,250,334]
+  CRUSH rule 0 x 2 [292,832,53]
+  CRUSH rule 0 x 3 [623,387,124]
+  CRUSH rule 0 x 4 [61,334,710]
+  CRUSH rule 0 x 5 [946,557,713]
+  CRUSH rule 0 x 6 [576,668,212]
+  CRUSH rule 0 x 7 [645,753,906]
+  CRUSH rule 0 x 8 [243,6,863]
+  CRUSH rule 0 x 9 [22,578,251]
+  CRUSH rule 0 x 10 [758,828,360]
+  CRUSH rule 0 x 11 [769,120,124]
+  CRUSH rule 0 x 12 [780,364,689]
+  CRUSH rule 0 x 13 [557,18,351]
+  CRUSH rule 0 x 14 [59,561,249]
+  CRUSH rule 0 x 15 [718,928,993]
+  CRUSH rule 0 x 16 [673,632,841]
+  CRUSH rule 0 x 17 [648,43,560]
+  CRUSH rule 0 x 18 [654,219,181]
+  CRUSH rule 0 x 19 [850,545,377]
+  CRUSH rule 0 x 20 [717,785,974]
+  CRUSH rule 0 x 21 [420,57,519]
+  CRUSH rule 0 x 22 [503,998,193]
+  CRUSH rule 0 x 23 [411,663,168]
+  CRUSH rule 0 x 24 [266,861,353]
+  CRUSH rule 0 x 25 [760,483,818]
+  CRUSH rule 0 x 26 [903,24,573]
+  CRUSH rule 0 x 27 [946,188,289]
+  CRUSH rule 0 x 28 [69,312,73]
+  CRUSH rule 0 x 29 [844,883,337]
+  CRUSH rule 0 x 30 [621,18,613]
+  CRUSH rule 0 x 31 [784,943,814]
+  CRUSH rule 0 x 32 [173,374,369]
+  CRUSH rule 0 x 33 [698,336,357]
+  CRUSH rule 0 x 34 [168,836,210]
+  CRUSH rule 0 x 35 [274,509,534]
+  CRUSH rule 0 x 36 [318,215,153]
+  CRUSH rule 0 x 37 [173,604,109]
+  CRUSH rule 0 x 38 [708,444,683]
+  CRUSH rule 0 x 39 [662,198,417]
+  CRUSH rule 0 x 40 [620,801,414]
+  CRUSH rule 0 x 41 [811,264,177]
+  CRUSH rule 0 x 42 [863,179,527]
+  CRUSH rule 0 x 43 [686,822,988]
+  CRUSH rule 0 x 44 [396,222,46]
+  CRUSH rule 0 x 45 [991,694,253]
+  CRUSH rule 0 x 46 [420,909,184]
+  CRUSH rule 0 x 47 [467,211,605]
+  CRUSH rule 0 x 48 [955,329,368]
+  CRUSH rule 0 x 49 [974,891,931]
+  CRUSH rule 0 x 50 [870,441,691]
+  CRUSH rule 0 x 51 [182,930,25]
+  CRUSH rule 0 x 52 [704,812,894]
+  CRUSH rule 0 x 53 [185,713,631]
+  CRUSH rule 0 x 54 [270,441,100]
+  CRUSH rule 0 x 55 [895,734,958]
+  CRUSH rule 0 x 56 [564,963,683]
+  CRUSH rule 0 x 57 [738,130,208]
+  CRUSH rule 0 x 58 [524,113,806]
+  CRUSH rule 0 x 59 [408,337,668]
+  CRUSH rule 0 x 60 [228,790,857]
+  CRUSH rule 0 x 61 [154,843,717]
+  CRUSH rule 0 x 62 [594,811,549]
+  CRUSH rule 0 x 63 [646,67,884]
+  CRUSH rule 0 x 64 [175,542,155]
+  CRUSH rule 0 x 65 [745,619,131]
+  CRUSH rule 0 x 66 [275,468,23]
+  CRUSH rule 0 x 67 [246,958,524]
+  CRUSH rule 0 x 68 [711,473,403]
+  CRUSH rule 0 x 69 [493,924,850]
+  CRUSH rule 0 x 70 [30,499,644]
+  CRUSH rule 0 x 71 [984,883,574]
+  CRUSH rule 0 x 72 [71,286,942]
+  CRUSH rule 0 x 73 [922,618,3]
+  CRUSH rule 0 x 74 [629,414,185]
+  CRUSH rule 0 x 75 [222,20,174]
+  CRUSH rule 0 x 76 [262,366,339]
+  CRUSH rule 0 x 77 [638,469,992]
+  CRUSH rule 0 x 78 [324,511,788]
+  CRUSH rule 0 x 79 [577,990,64]
+  CRUSH rule 0 x 80 [501,95,278]
+  CRUSH rule 0 x 81 [506,812,9]
+  CRUSH rule 0 x 82 [222,145,80]
+  CRUSH rule 0 x 83 [71,634,61]
+  CRUSH rule 0 x 84 [49,761,773]
+  CRUSH rule 0 x 85 [985,896,708]
+  CRUSH rule 0 x 86 [537,745,93]
+  CRUSH rule 0 x 87 [997,317,463]
+  CRUSH rule 0 x 88 [957,350,890]
+  CRUSH rule 0 x 89 [399,730,148]
+  CRUSH rule 0 x 90 [943,706,683]
+  CRUSH rule 0 x 91 [22,368,149]
+  CRUSH rule 0 x 92 [532,424,426]
+  CRUSH rule 0 x 93 [218,489,405]
+  CRUSH rule 0 x 94 [181,96,102]
+  CRUSH rule 0 x 95 [343,957,820]
+  CRUSH rule 0 x 96 [861,270,87]
+  CRUSH rule 0 x 97 [459,706,45]
+  CRUSH rule 0 x 98 [327,867,353]
+  CRUSH rule 0 x 99 [974,133,468]
+  CRUSH rule 0 x 100 [32,445,547]
+  CRUSH rule 0 x 101 [142,90,337]
+  CRUSH rule 0 x 102 [172,129,139]
+  CRUSH rule 0 x 103 [630,47,161]
+  CRUSH rule 0 x 104 [758,133,278]
+  CRUSH rule 0 x 105 [843,604,47]
+  CRUSH rule 0 x 106 [28,681,193]
+  CRUSH rule 0 x 107 [74,320,85]
+  CRUSH rule 0 x 108 [875,593,575]
+  CRUSH rule 0 x 109 [411,985,811]
+  CRUSH rule 0 x 110 [440,774,799]
+  CRUSH rule 0 x 111 [405,742,276]
+  CRUSH rule 0 x 112 [143,181,922]
+  CRUSH rule 0 x 113 [153,846,160]
+  CRUSH rule 0 x 114 [804,892,939]
+  CRUSH rule 0 x 115 [588,508,958]
+  CRUSH rule 0 x 116 [327,148,637]
+  CRUSH rule 0 x 117 [95,594,989]
+  CRUSH rule 0 x 118 [80,957,897]
+  CRUSH rule 0 x 119 [386,932,951]
+  CRUSH rule 0 x 120 [366,312,653]
+  CRUSH rule 0 x 121 [129,154,847]
+  CRUSH rule 0 x 122 [873,1,110]
+  CRUSH rule 0 x 123 [533,415,789]
+  CRUSH rule 0 x 124 [461,691,898]
+  CRUSH rule 0 x 125 [342,599,830]
+  CRUSH rule 0 x 126 [819,781,822]
+  CRUSH rule 0 x 127 [437,893,585]
+  CRUSH rule 0 x 128 [679,994,982]
+  CRUSH rule 0 x 129 [380,685,947]
+  CRUSH rule 0 x 130 [992,52,466]
+  CRUSH rule 0 x 131 [469,90,208]
+  CRUSH rule 0 x 132 [571,250,316]
+  CRUSH rule 0 x 133 [964,728,329]
+  CRUSH rule 0 x 134 [999,19,716]
+  CRUSH rule 0 x 135 [634,101,52]
+  CRUSH rule 0 x 136 [114,889,692]
+  CRUSH rule 0 x 137 [839,8,959]
+  CRUSH rule 0 x 138 [967,949,138]
+  CRUSH rule 0 x 139 [308,711,736]
+  CRUSH rule 0 x 140 [764,936,926]
+  CRUSH rule 0 x 141 [423,302,112]
+  CRUSH rule 0 x 142 [252,821,715]
+  CRUSH rule 0 x 143 [33,808,518]
+  CRUSH rule 0 x 144 [472,88,969]
+  CRUSH rule 0 x 145 [242,208,252]
+  CRUSH rule 0 x 146 [290,70,570]
+  CRUSH rule 0 x 147 [447,352,657]
+  CRUSH rule 0 x 148 [212,644,432]
+  CRUSH rule 0 x 149 [9,775,87]
+  CRUSH rule 0 x 150 [166,456,582]
+  CRUSH rule 0 x 151 [811,875,307]
+  CRUSH rule 0 x 152 [449,617,223]
+  CRUSH rule 0 x 153 [523,537,695]
+  CRUSH rule 0 x 154 [208,559,874]
+  CRUSH rule 0 x 155 [569,325,192]
+  CRUSH rule 0 x 156 [488,121,521]
+  CRUSH rule 0 x 157 [140,723,633]
+  CRUSH rule 0 x 158 [786,451,320]
+  CRUSH rule 0 x 159 [134,664,517]
+  CRUSH rule 0 x 160 [690,112,414]
+  CRUSH rule 0 x 161 [324,912,397]
+  CRUSH rule 0 x 162 [748,567,284]
+  CRUSH rule 0 x 163 [575,499,31]
+  CRUSH rule 0 x 164 [314,489,308]
+  CRUSH rule 0 x 165 [116,209,750]
+  CRUSH rule 0 x 166 [352,706,701]
+  CRUSH rule 0 x 167 [27,743,174]
+  CRUSH rule 0 x 168 [953,898,880]
+  CRUSH rule 0 x 169 [912,147,266]
+  CRUSH rule 0 x 170 [421,515,828]
+  CRUSH rule 0 x 171 [488,584,880]
+  CRUSH rule 0 x 172 [366,443,957]
+  CRUSH rule 0 x 173 [863,291,625]
+  CRUSH rule 0 x 174 [263,555,650]
+  CRUSH rule 0 x 175 [875,961,361]
+  CRUSH rule 0 x 176 [745,83,701]
+  CRUSH rule 0 x 177 [128,244,41]
+  CRUSH rule 0 x 178 [155,41,264]
+  CRUSH rule 0 x 179 [593,833,202]
+  CRUSH rule 0 x 180 [154,734,17]
+  CRUSH rule 0 x 181 [289,675,723]
+  CRUSH rule 0 x 182 [730,931,560]
+  CRUSH rule 0 x 183 [639,237,794]
+  CRUSH rule 0 x 184 [704,312,685]
+  CRUSH rule 0 x 185 [97,100,762]
+  CRUSH rule 0 x 186 [26,665,554]
+  CRUSH rule 0 x 187 [649,14,740]
+  CRUSH rule 0 x 188 [682,695,590]
+  CRUSH rule 0 x 189 [325,693,726]
+  CRUSH rule 0 x 190 [399,933,136]
+  CRUSH rule 0 x 191 [629,533,17]
+  CRUSH rule 0 x 192 [503,578,38]
+  CRUSH rule 0 x 193 [546,333,651]
+  CRUSH rule 0 x 194 [242,473,58]
+  CRUSH rule 0 x 195 [625,719,135]
+  CRUSH rule 0 x 196 [357,114,125]
+  CRUSH rule 0 x 197 [306,954,453]
+  CRUSH rule 0 x 198 [863,791,311]
+  CRUSH rule 0 x 199 [935,906,929]
+  CRUSH rule 0 x 200 [373,774,229]
+  CRUSH rule 0 x 201 [659,320,477]
+  CRUSH rule 0 x 202 [260,433,524]
+  CRUSH rule 0 x 203 [36,239,675]
+  CRUSH rule 0 x 204 [92,516,993]
+  CRUSH rule 0 x 205 [68,395,473]
+  CRUSH rule 0 x 206 [570,530,642]
+  CRUSH rule 0 x 207 [834,457,850]
+  CRUSH rule 0 x 208 [927,484,640]
+  CRUSH rule 0 x 209 [878,66,58]
+  CRUSH rule 0 x 210 [572,981,484]
+  CRUSH rule 0 x 211 [107,597,780]
+  CRUSH rule 0 x 212 [389,107,838]
+  CRUSH rule 0 x 213 [497,717,567]
+  CRUSH rule 0 x 214 [798,65,254]
+  CRUSH rule 0 x 215 [233,419,283]
+  CRUSH rule 0 x 216 [494,464,742]
+  CRUSH rule 0 x 217 [352,396,309]
+  CRUSH rule 0 x 218 [895,864,988]
+  CRUSH rule 0 x 219 [222,534,277]
+  CRUSH rule 0 x 220 [281,19,584]
+  CRUSH rule 0 x 221 [64,928,963]
+  CRUSH rule 0 x 222 [40,544,161]
+  CRUSH rule 0 x 223 [645,556,159]
+  CRUSH rule 0 x 224 [647,165,957]
+  CRUSH rule 0 x 225 [219,714,858]
+  CRUSH rule 0 x 226 [372,511,181]
+  CRUSH rule 0 x 227 [925,156,714]
+  CRUSH rule 0 x 228 [682,404,839]
+  CRUSH rule 0 x 229 [880,838,770]
+  CRUSH rule 0 x 230 [328,659,916]
+  CRUSH rule 0 x 231 [320,383,669]
+  CRUSH rule 0 x 232 [924,846,394]
+  CRUSH rule 0 x 233 [948,652,575]
+  CRUSH rule 0 x 234 [484,943,42]
+  CRUSH rule 0 x 235 [750,65,590]
+  CRUSH rule 0 x 236 [551,787,490]
+  CRUSH rule 0 x 237 [390,157,166]
+  CRUSH rule 0 x 238 [570,6,989]
+  CRUSH rule 0 x 239 [729,959,376]
+  CRUSH rule 0 x 240 [981,241,156]
+  CRUSH rule 0 x 241 [310,816,641]
+  CRUSH rule 0 x 242 [161,63,642]
+  CRUSH rule 0 x 243 [180,394,33]
+  CRUSH rule 0 x 244 [52,174,685]
+  CRUSH rule 0 x 245 [523,121,915]
+  CRUSH rule 0 x 246 [362,893,390]
+  CRUSH rule 0 x 247 [382,184,116]
+  CRUSH rule 0 x 248 [129,114,852]
+  CRUSH rule 0 x 249 [159,683,91]
+  CRUSH rule 0 x 250 [404,945,569]
+  CRUSH rule 0 x 251 [661,225,738]
+  CRUSH rule 0 x 252 [961,226,542]
+  CRUSH rule 0 x 253 [651,97,225]
+  CRUSH rule 0 x 254 [123,33,741]
+  CRUSH rule 0 x 255 [314,649,891]
+  CRUSH rule 0 x 256 [315,215,651]
+  CRUSH rule 0 x 257 [825,264,867]
+  CRUSH rule 0 x 258 [624,789,370]
+  CRUSH rule 0 x 259 [602,542,70]
+  CRUSH rule 0 x 260 [717,878,43]
+  CRUSH rule 0 x 261 [145,517,20]
+  CRUSH rule 0 x 262 [223,1,561]
+  CRUSH rule 0 x 263 [462,211,405]
+  CRUSH rule 0 x 264 [654,471,266]
+  CRUSH rule 0 x 265 [302,794,704]
+  CRUSH rule 0 x 266 [202,132,884]
+  CRUSH rule 0 x 267 [282,938,657]
+  CRUSH rule 0 x 268 [338,309,356]
+  CRUSH rule 0 x 269 [738,122,266]
+  CRUSH rule 0 x 270 [707,982,946]
+  CRUSH rule 0 x 271 [705,432,364]
+  CRUSH rule 0 x 272 [756,545,942]
+  CRUSH rule 0 x 273 [197,502,527]
+  CRUSH rule 0 x 274 [992,44,653]
+  CRUSH rule 0 x 275 [544,789,170]
+  CRUSH rule 0 x 276 [658,467,577]
+  CRUSH rule 0 x 277 [143,490,880]
+  CRUSH rule 0 x 278 [492,647,355]
+  CRUSH rule 0 x 279 [517,792,604]
+  CRUSH rule 0 x 280 [825,740,27]
+  CRUSH rule 0 x 281 [224,629,120]
+  CRUSH rule 0 x 282 [298,661,380]
+  CRUSH rule 0 x 283 [311,606,208]
+  CRUSH rule 0 x 284 [771,466,371]
+  CRUSH rule 0 x 285 [693,362,404]
+  CRUSH rule 0 x 286 [364,477,285]
+  CRUSH rule 0 x 287 [591,611,828]
+  CRUSH rule 0 x 288 [965,541,848]
+  CRUSH rule 0 x 289 [225,551,948]
+  CRUSH rule 0 x 290 [577,762,777]
+  CRUSH rule 0 x 291 [160,903,477]
+  CRUSH rule 0 x 292 [873,598,216]
+  CRUSH rule 0 x 293 [100,234,874]
+  CRUSH rule 0 x 294 [285,943,379]
+  CRUSH rule 0 x 295 [938,262,880]
+  CRUSH rule 0 x 296 [850,327,86]
+  CRUSH rule 0 x 297 [951,53,99]
+  CRUSH rule 0 x 298 [173,336,85]
+  CRUSH rule 0 x 299 [598,591,315]
+  CRUSH rule 0 x 300 [531,957,62]
+  CRUSH rule 0 x 301 [823,628,23]
+  CRUSH rule 0 x 302 [184,80,780]
+  CRUSH rule 0 x 303 [521,766,222]
+  CRUSH rule 0 x 304 [980,127,807]
+  CRUSH rule 0 x 305 [153,816,22]
+  CRUSH rule 0 x 306 [423,739,664]
+  CRUSH rule 0 x 307 [997,557,682]
+  CRUSH rule 0 x 308 [991,874,534]
+  CRUSH rule 0 x 309 [860,394,724]
+  CRUSH rule 0 x 310 [589,818,546]
+  CRUSH rule 0 x 311 [477,774,225]
+  CRUSH rule 0 x 312 [887,853,950]
+  CRUSH rule 0 x 313 [802,646,447]
+  CRUSH rule 0 x 314 [654,974,229]
+  CRUSH rule 0 x 315 [767,227,28]
+  CRUSH rule 0 x 316 [778,83,733]
+  CRUSH rule 0 x 317 [184,418,642]
+  CRUSH rule 0 x 318 [525,410,500]
+  CRUSH rule 0 x 319 [476,724,569]
+  CRUSH rule 0 x 320 [149,610,697]
+  CRUSH rule 0 x 321 [710,79,667]
+  CRUSH rule 0 x 322 [175,275,323]
+  CRUSH rule 0 x 323 [819,604,638]
+  CRUSH rule 0 x 324 [16,745,511]
+  CRUSH rule 0 x 325 [486,400,872]
+  CRUSH rule 0 x 326 [613,765,207]
+  CRUSH rule 0 x 327 [125,289,738]
+  CRUSH rule 0 x 328 [807,383,476]
+  CRUSH rule 0 x 329 [588,938,599]
+  CRUSH rule 0 x 330 [932,644,41]
+  CRUSH rule 0 x 331 [341,953,950]
+  CRUSH rule 0 x 332 [153,726,459]
+  CRUSH rule 0 x 333 [745,845,853]
+  CRUSH rule 0 x 334 [614,751,807]
+  CRUSH rule 0 x 335 [518,721,221]
+  CRUSH rule 0 x 336 [389,424,77]
+  CRUSH rule 0 x 337 [753,508,765]
+  CRUSH rule 0 x 338 [128,810,490]
+  CRUSH rule 0 x 339 [430,308,58]
+  CRUSH rule 0 x 340 [541,44,630]
+  CRUSH rule 0 x 341 [402,26,631]
+  CRUSH rule 0 x 342 [982,57,992]
+  CRUSH rule 0 x 343 [833,412,572]
+  CRUSH rule 0 x 344 [784,533,792]
+  CRUSH rule 0 x 345 [546,300,304]
+  CRUSH rule 0 x 346 [302,420,428]
+  CRUSH rule 0 x 347 [488,778,101]
+  CRUSH rule 0 x 348 [903,744,937]
+  CRUSH rule 0 x 349 [471,547,582]
+  CRUSH rule 0 x 350 [348,221,823]
+  CRUSH rule 0 x 351 [961,582,705]
+  CRUSH rule 0 x 352 [728,137,461]
+  CRUSH rule 0 x 353 [904,202,184]
+  CRUSH rule 0 x 354 [345,226,319]
+  CRUSH rule 0 x 355 [50,430,175]
+  CRUSH rule 0 x 356 [87,185,55]
+  CRUSH rule 0 x 357 [762,459,921]
+  CRUSH rule 0 x 358 [908,25,280]
+  CRUSH rule 0 x 359 [484,15,132]
+  CRUSH rule 0 x 360 [173,378,337]
+  CRUSH rule 0 x 361 [404,577,115]
+  CRUSH rule 0 x 362 [403,1,422]
+  CRUSH rule 0 x 363 [639,911,510]
+  CRUSH rule 0 x 364 [752,689,610]
+  CRUSH rule 0 x 365 [956,999,212]
+  CRUSH rule 0 x 366 [860,925,924]
+  CRUSH rule 0 x 367 [205,609,647]
+  CRUSH rule 0 x 368 [301,284,810]
+  CRUSH rule 0 x 369 [452,658,339]
+  CRUSH rule 0 x 370 [11,467,695]
+  CRUSH rule 0 x 371 [124,487,55]
+  CRUSH rule 0 x 372 [253,48,979]
+  CRUSH rule 0 x 373 [715,605,775]
+  CRUSH rule 0 x 374 [191,887,920]
+  CRUSH rule 0 x 375 [711,385,651]
+  CRUSH rule 0 x 376 [597,818,49]
+  CRUSH rule 0 x 377 [294,256,933]
+  CRUSH rule 0 x 378 [34,151,681]
+  CRUSH rule 0 x 379 [869,136,315]
+  CRUSH rule 0 x 380 [294,97,575]
+  CRUSH rule 0 x 381 [119,710,219]
+  CRUSH rule 0 x 382 [69,631,508]
+  CRUSH rule 0 x 383 [922,588,589]
+  CRUSH rule 0 x 384 [221,945,671]
+  CRUSH rule 0 x 385 [561,737,953]
+  CRUSH rule 0 x 386 [335,442,788]
+  CRUSH rule 0 x 387 [514,43,353]
+  CRUSH rule 0 x 388 [587,89,157]
+  CRUSH rule 0 x 389 [109,641,255]
+  CRUSH rule 0 x 390 [925,149,421]
+  CRUSH rule 0 x 391 [267,87,387]
+  CRUSH rule 0 x 392 [382,485,370]
+  CRUSH rule 0 x 393 [425,721,221]
+  CRUSH rule 0 x 394 [898,18,38]
+  CRUSH rule 0 x 395 [806,876,269]
+  CRUSH rule 0 x 396 [790,970,437]
+  CRUSH rule 0 x 397 [136,363,507]
+  CRUSH rule 0 x 398 [914,116,558]
+  CRUSH rule 0 x 399 [261,94,299]
+  CRUSH rule 0 x 400 [661,197,338]
+  CRUSH rule 0 x 401 [953,979,287]
+  CRUSH rule 0 x 402 [738,819,618]
+  CRUSH rule 0 x 403 [573,238,425]
+  CRUSH rule 0 x 404 [526,848,790]
+  CRUSH rule 0 x 405 [582,505,330]
+  CRUSH rule 0 x 406 [768,324,493]
+  CRUSH rule 0 x 407 [260,951,437]
+  CRUSH rule 0 x 408 [657,81,770]
+  CRUSH rule 0 x 409 [498,89,182]
+  CRUSH rule 0 x 410 [28,793,737]
+  CRUSH rule 0 x 411 [684,992,60]
+  CRUSH rule 0 x 412 [261,958,699]
+  CRUSH rule 0 x 413 [891,835,297]
+  CRUSH rule 0 x 414 [127,459,119]
+  CRUSH rule 0 x 415 [272,540,631]
+  CRUSH rule 0 x 416 [739,617,115]
+  CRUSH rule 0 x 417 [106,209,157]
+  CRUSH rule 0 x 418 [525,441,147]
+  CRUSH rule 0 x 419 [603,673,615]
+  CRUSH rule 0 x 420 [988,213,251]
+  CRUSH rule 0 x 421 [761,521,748]
+  CRUSH rule 0 x 422 [317,160,924]
+  CRUSH rule 0 x 423 [137,807,168]
+  CRUSH rule 0 x 424 [920,37,146]
+  CRUSH rule 0 x 425 [277,693,285]
+  CRUSH rule 0 x 426 [485,936,407]
+  CRUSH rule 0 x 427 [242,515,9]
+  CRUSH rule 0 x 428 [632,635,26]
+  CRUSH rule 0 x 429 [641,73,465]
+  CRUSH rule 0 x 430 [626,585,6]
+  CRUSH rule 0 x 431 [697,76,753]
+  CRUSH rule 0 x 432 [590,526,306]
+  CRUSH rule 0 x 433 [284,387,149]
+  CRUSH rule 0 x 434 [538,985,79]
+  CRUSH rule 0 x 435 [30,318,593]
+  CRUSH rule 0 x 436 [164,919,851]
+  CRUSH rule 0 x 437 [322,212,163]
+  CRUSH rule 0 x 438 [142,392,85]
+  CRUSH rule 0 x 439 [119,370,68]
+  CRUSH rule 0 x 440 [333,403,187]
+  CRUSH rule 0 x 441 [477,727,906]
+  CRUSH rule 0 x 442 [274,590,933]
+  CRUSH rule 0 x 443 [983,748,574]
+  CRUSH rule 0 x 444 [536,509,431]
+  CRUSH rule 0 x 445 [485,482,528]
+  CRUSH rule 0 x 446 [345,634,42]
+  CRUSH rule 0 x 447 [61,845,767]
+  CRUSH rule 0 x 448 [333,232,292]
+  CRUSH rule 0 x 449 [680,16,484]
+  CRUSH rule 0 x 450 [235,214,79]
+  CRUSH rule 0 x 451 [961,468,333]
+  CRUSH rule 0 x 452 [525,479,153]
+  CRUSH rule 0 x 453 [138,466,302]
+  CRUSH rule 0 x 454 [137,625,215]
+  CRUSH rule 0 x 455 [173,150,997]
+  CRUSH rule 0 x 456 [235,226,238]
+  CRUSH rule 0 x 457 [450,577,253]
+  CRUSH rule 0 x 458 [195,537,91]
+  CRUSH rule 0 x 459 [381,555,312]
+  CRUSH rule 0 x 460 [972,730,534]
+  CRUSH rule 0 x 461 [506,279,142]
+  CRUSH rule 0 x 462 [692,959,578]
+  CRUSH rule 0 x 463 [788,667,949]
+  CRUSH rule 0 x 464 [133,122,588]
+  CRUSH rule 0 x 465 [971,190,230]
+  CRUSH rule 0 x 466 [394,576,148]
+  CRUSH rule 0 x 467 [517,28,366]
+  CRUSH rule 0 x 468 [829,143,874]
+  CRUSH rule 0 x 469 [987,936,106]
+  CRUSH rule 0 x 470 [107,982,56]
+  CRUSH rule 0 x 471 [181,897,629]
+  CRUSH rule 0 x 472 [547,512,172]
+  CRUSH rule 0 x 473 [760,997,824]
+  CRUSH rule 0 x 474 [787,418,743]
+  CRUSH rule 0 x 475 [662,312,253]
+  CRUSH rule 0 x 476 [110,495,185]
+  CRUSH rule 0 x 477 [393,954,834]
+  CRUSH rule 0 x 478 [246,483,480]
+  CRUSH rule 0 x 479 [70,929,697]
+  CRUSH rule 0 x 480 [753,119,961]
+  CRUSH rule 0 x 481 [470,429,677]
+  CRUSH rule 0 x 482 [451,566,961]
+  CRUSH rule 0 x 483 [816,72,371]
+  CRUSH rule 0 x 484 [540,454,389]
+  CRUSH rule 0 x 485 [74,582,624]
+  CRUSH rule 0 x 486 [958,595,199]
+  CRUSH rule 0 x 487 [228,302,804]
+  CRUSH rule 0 x 488 [180,529,722]
+  CRUSH rule 0 x 489 [47,617,812]
+  CRUSH rule 0 x 490 [905,822,479]
+  CRUSH rule 0 x 491 [892,370,609]
+  CRUSH rule 0 x 492 [588,959,127]
+  CRUSH rule 0 x 493 [353,461,593]
+  CRUSH rule 0 x 494 [378,848,443]
+  CRUSH rule 0 x 495 [845,653,768]
+  CRUSH rule 0 x 496 [13,988,0]
+  CRUSH rule 0 x 497 [796,877,788]
+  CRUSH rule 0 x 498 [412,337,270]
+  CRUSH rule 0 x 499 [330,695,8]
+  CRUSH rule 0 x 500 [820,272,547]
+  CRUSH rule 0 x 501 [110,44,132]
+  CRUSH rule 0 x 502 [336,595,650]
+  CRUSH rule 0 x 503 [922,211,157]
+  CRUSH rule 0 x 504 [483,52,122]
+  CRUSH rule 0 x 505 [482,598,224]
+  CRUSH rule 0 x 506 [493,123,43]
+  CRUSH rule 0 x 507 [12,598,264]
+  CRUSH rule 0 x 508 [227,157,611]
+  CRUSH rule 0 x 509 [807,242,363]
+  CRUSH rule 0 x 510 [134,437,227]
+  CRUSH rule 0 x 511 [212,54,83]
+  CRUSH rule 0 x 512 [236,630,758]
+  CRUSH rule 0 x 513 [994,693,644]
+  CRUSH rule 0 x 514 [45,508,831]
+  CRUSH rule 0 x 515 [504,138,480]
+  CRUSH rule 0 x 516 [285,409,136]
+  CRUSH rule 0 x 517 [300,232,23]
+  CRUSH rule 0 x 518 [397,674,98]
+  CRUSH rule 0 x 519 [86,750,772]
+  CRUSH rule 0 x 520 [900,833,614]
+  CRUSH rule 0 x 521 [31,47,236]
+  CRUSH rule 0 x 522 [390,16,280]
+  CRUSH rule 0 x 523 [618,308,424]
+  CRUSH rule 0 x 524 [635,189,687]
+  CRUSH rule 0 x 525 [311,916,699]
+  CRUSH rule 0 x 526 [48,738,227]
+  CRUSH rule 0 x 527 [202,851,889]
+  CRUSH rule 0 x 528 [565,827,590]
+  CRUSH rule 0 x 529 [934,864,241]
+  CRUSH rule 0 x 530 [502,934,298]
+  CRUSH rule 0 x 531 [681,627,942]
+  CRUSH rule 0 x 532 [422,6,147]
+  CRUSH rule 0 x 533 [863,68,364]
+  CRUSH rule 0 x 534 [962,931,775]
+  CRUSH rule 0 x 535 [89,565,397]
+  CRUSH rule 0 x 536 [499,351,760]
+  CRUSH rule 0 x 537 [676,547,787]
+  CRUSH rule 0 x 538 [58,644,571]
+  CRUSH rule 0 x 539 [837,953,457]
+  CRUSH rule 0 x 540 [831,50,132]
+  CRUSH rule 0 x 541 [582,757,121]
+  CRUSH rule 0 x 542 [472,132,790]
+  CRUSH rule 0 x 543 [382,272,797]
+  CRUSH rule 0 x 544 [947,930,496]
+  CRUSH rule 0 x 545 [425,570,305]
+  CRUSH rule 0 x 546 [18,65,529]
+  CRUSH rule 0 x 547 [445,715,600]
+  CRUSH rule 0 x 548 [367,569,980]
+  CRUSH rule 0 x 549 [125,715,671]
+  CRUSH rule 0 x 550 [425,599,744]
+  CRUSH rule 0 x 551 [44,1,528]
+  CRUSH rule 0 x 552 [246,104,68]
+  CRUSH rule 0 x 553 [71,703,615]
+  CRUSH rule 0 x 554 [207,124,217]
+  CRUSH rule 0 x 555 [570,28,317]
+  CRUSH rule 0 x 556 [674,152,421]
+  CRUSH rule 0 x 557 [347,817,191]
+  CRUSH rule 0 x 558 [627,426,369]
+  CRUSH rule 0 x 559 [940,630,924]
+  CRUSH rule 0 x 560 [295,903,541]
+  CRUSH rule 0 x 561 [506,682,384]
+  CRUSH rule 0 x 562 [718,529,87]
+  CRUSH rule 0 x 563 [552,332,747]
+  CRUSH rule 0 x 564 [835,769,736]
+  CRUSH rule 0 x 565 [8,167,539]
+  CRUSH rule 0 x 566 [600,481,301]
+  CRUSH rule 0 x 567 [999,994,509]
+  CRUSH rule 0 x 568 [252,431,157]
+  CRUSH rule 0 x 569 [643,218,943]
+  CRUSH rule 0 x 570 [617,635,765]
+  CRUSH rule 0 x 571 [757,80,59]
+  CRUSH rule 0 x 572 [299,348,575]
+  CRUSH rule 0 x 573 [25,505,270]
+  CRUSH rule 0 x 574 [215,431,624]
+  CRUSH rule 0 x 575 [225,252,611]
+  CRUSH rule 0 x 576 [627,94,159]
+  CRUSH rule 0 x 577 [237,809,778]
+  CRUSH rule 0 x 578 [885,313,120]
+  CRUSH rule 0 x 579 [924,575,787]
+  CRUSH rule 0 x 580 [718,51,766]
+  CRUSH rule 0 x 581 [219,807,129]
+  CRUSH rule 0 x 582 [893,701,598]
+  CRUSH rule 0 x 583 [246,930,964]
+  CRUSH rule 0 x 584 [336,432,680]
+  CRUSH rule 0 x 585 [324,999,397]
+  CRUSH rule 0 x 586 [558,230,976]
+  CRUSH rule 0 x 587 [985,830,597]
+  CRUSH rule 0 x 588 [211,544,57]
+  CRUSH rule 0 x 589 [129,21,112]
+  CRUSH rule 0 x 590 [467,969,652]
+  CRUSH rule 0 x 591 [758,514,316]
+  CRUSH rule 0 x 592 [525,253,190]
+  CRUSH rule 0 x 593 [601,885,339]
+  CRUSH rule 0 x 594 [227,60,450]
+  CRUSH rule 0 x 595 [720,854,496]
+  CRUSH rule 0 x 596 [751,195,997]
+  CRUSH rule 0 x 597 [129,574,714]
+  CRUSH rule 0 x 598 [679,207,604]
+  CRUSH rule 0 x 599 [668,315,683]
+  CRUSH rule 0 x 600 [143,396,464]
+  CRUSH rule 0 x 601 [326,573,873]
+  CRUSH rule 0 x 602 [860,281,875]
+  CRUSH rule 0 x 603 [709,328,445]
+  CRUSH rule 0 x 604 [571,62,814]
+  CRUSH rule 0 x 605 [252,739,860]
+  CRUSH rule 0 x 606 [339,236,759]
+  CRUSH rule 0 x 607 [590,248,759]
+  CRUSH rule 0 x 608 [145,635,309]
+  CRUSH rule 0 x 609 [973,547,223]
+  CRUSH rule 0 x 610 [435,816,961]
+  CRUSH rule 0 x 611 [559,283,422]
+  CRUSH rule 0 x 612 [273,149,123]
+  CRUSH rule 0 x 613 [828,614,642]
+  CRUSH rule 0 x 614 [478,748,393]
+  CRUSH rule 0 x 615 [392,155,144]
+  CRUSH rule 0 x 616 [778,637,452]
+  CRUSH rule 0 x 617 [622,713,996]
+  CRUSH rule 0 x 618 [149,877,270]
+  CRUSH rule 0 x 619 [604,163,656]
+  CRUSH rule 0 x 620 [181,23,409]
+  CRUSH rule 0 x 621 [735,902,386]
+  CRUSH rule 0 x 622 [661,824,717]
+  CRUSH rule 0 x 623 [142,121,643]
+  CRUSH rule 0 x 624 [360,716,420]
+  CRUSH rule 0 x 625 [541,167,385]
+  CRUSH rule 0 x 626 [364,431,610]
+  CRUSH rule 0 x 627 [458,137,557]
+  CRUSH rule 0 x 628 [250,350,556]
+  CRUSH rule 0 x 629 [928,160,710]
+  CRUSH rule 0 x 630 [243,19,918]
+  CRUSH rule 0 x 631 [438,221,574]
+  CRUSH rule 0 x 632 [797,368,247]
+  CRUSH rule 0 x 633 [993,749,525]
+  CRUSH rule 0 x 634 [239,351,633]
+  CRUSH rule 0 x 635 [640,965,25]
+  CRUSH rule 0 x 636 [173,290,297]
+  CRUSH rule 0 x 637 [0,918,98]
+  CRUSH rule 0 x 638 [702,235,424]
+  CRUSH rule 0 x 639 [475,687,31]
+  CRUSH rule 0 x 640 [31,664,399]
+  CRUSH rule 0 x 641 [296,473,108]
+  CRUSH rule 0 x 642 [894,273,427]
+  CRUSH rule 0 x 643 [117,111,732]
+  CRUSH rule 0 x 644 [438,336,327]
+  CRUSH rule 0 x 645 [982,702,351]
+  CRUSH rule 0 x 646 [334,804,146]
+  CRUSH rule 0 x 647 [933,787,185]
+  CRUSH rule 0 x 648 [22,444,400]
+  CRUSH rule 0 x 649 [503,229,213]
+  CRUSH rule 0 x 650 [328,659,420]
+  CRUSH rule 0 x 651 [3,880,823]
+  CRUSH rule 0 x 652 [495,977,563]
+  CRUSH rule 0 x 653 [185,718,804]
+  CRUSH rule 0 x 654 [130,528,380]
+  CRUSH rule 0 x 655 [560,872,454]
+  CRUSH rule 0 x 656 [219,885,178]
+  CRUSH rule 0 x 657 [233,684,813]
+  CRUSH rule 0 x 658 [778,6,756]
+  CRUSH rule 0 x 659 [240,663,306]
+  CRUSH rule 0 x 660 [244,855,196]
+  CRUSH rule 0 x 661 [184,270,128]
+  CRUSH rule 0 x 662 [65,883,921]
+  CRUSH rule 0 x 663 [323,721,594]
+  CRUSH rule 0 x 664 [865,113,512]
+  CRUSH rule 0 x 665 [420,850,591]
+  CRUSH rule 0 x 666 [319,767,246]
+  CRUSH rule 0 x 667 [875,39,343]
+  CRUSH rule 0 x 668 [331,122,263]
+  CRUSH rule 0 x 669 [915,521,402]
+  CRUSH rule 0 x 670 [845,659,943]
+  CRUSH rule 0 x 671 [108,634,527]
+  CRUSH rule 0 x 672 [578,216,110]
+  CRUSH rule 0 x 673 [442,74,579]
+  CRUSH rule 0 x 674 [588,364,281]
+  CRUSH rule 0 x 675 [489,698,744]
+  CRUSH rule 0 x 676 [928,911,40]
+  CRUSH rule 0 x 677 [399,269,692]
+  CRUSH rule 0 x 678 [546,752,544]
+  CRUSH rule 0 x 679 [988,25,275]
+  CRUSH rule 0 x 680 [335,963,382]
+  CRUSH rule 0 x 681 [690,462,623]
+  CRUSH rule 0 x 682 [196,588,154]
+  CRUSH rule 0 x 683 [627,25,421]
+  CRUSH rule 0 x 684 [38,804,592]
+  CRUSH rule 0 x 685 [841,368,548]
+  CRUSH rule 0 x 686 [336,287,525]
+  CRUSH rule 0 x 687 [20,682,924]
+  CRUSH rule 0 x 688 [463,371,780]
+  CRUSH rule 0 x 689 [569,250,78]
+  CRUSH rule 0 x 690 [551,144,587]
+  CRUSH rule 0 x 691 [766,464,446]
+  CRUSH rule 0 x 692 [739,634,18]
+  CRUSH rule 0 x 693 [339,297,118]
+  CRUSH rule 0 x 694 [405,26,830]
+  CRUSH rule 0 x 695 [622,576,597]
+  CRUSH rule 0 x 696 [558,902,689]
+  CRUSH rule 0 x 697 [818,222,406]
+  CRUSH rule 0 x 698 [178,48,402]
+  CRUSH rule 0 x 699 [450,244,180]
+  CRUSH rule 0 x 700 [502,771,987]
+  CRUSH rule 0 x 701 [4,612,782]
+  CRUSH rule 0 x 702 [177,630,232]
+  CRUSH rule 0 x 703 [354,178,389]
+  CRUSH rule 0 x 704 [646,601,156]
+  CRUSH rule 0 x 705 [921,401,890]
+  CRUSH rule 0 x 706 [652,877,562]
+  CRUSH rule 0 x 707 [345,745,67]
+  CRUSH rule 0 x 708 [333,607,180]
+  CRUSH rule 0 x 709 [45,187,302]
+  CRUSH rule 0 x 710 [94,855,43]
+  CRUSH rule 0 x 711 [227,653,731]
+  CRUSH rule 0 x 712 [398,953,136]
+  CRUSH rule 0 x 713 [116,800,503]
+  CRUSH rule 0 x 714 [111,629,866]
+  CRUSH rule 0 x 715 [531,291,486]
+  CRUSH rule 0 x 716 [169,541,291]
+  CRUSH rule 0 x 717 [417,446,994]
+  CRUSH rule 0 x 718 [992,383,298]
+  CRUSH rule 0 x 719 [936,674,324]
+  CRUSH rule 0 x 720 [370,188,174]
+  CRUSH rule 0 x 721 [320,859,278]
+  CRUSH rule 0 x 722 [7,2,673]
+  CRUSH rule 0 x 723 [270,553,831]
+  CRUSH rule 0 x 724 [666,822,708]
+  CRUSH rule 0 x 725 [794,406,875]
+  CRUSH rule 0 x 726 [420,556,341]
+  CRUSH rule 0 x 727 [561,461,129]
+  CRUSH rule 0 x 728 [951,330,196]
+  CRUSH rule 0 x 729 [656,644,436]
+  CRUSH rule 0 x 730 [3,558,629]
+  CRUSH rule 0 x 731 [852,89,75]
+  CRUSH rule 0 x 732 [983,840,869]
+  CRUSH rule 0 x 733 [285,396,388]
+  CRUSH rule 0 x 734 [125,510,402]
+  CRUSH rule 0 x 735 [417,773,686]
+  CRUSH rule 0 x 736 [749,396,632]
+  CRUSH rule 0 x 737 [644,991,946]
+  CRUSH rule 0 x 738 [449,683,290]
+  CRUSH rule 0 x 739 [341,220,641]
+  CRUSH rule 0 x 740 [874,524,674]
+  CRUSH rule 0 x 741 [189,472,712]
+  CRUSH rule 0 x 742 [912,581,114]
+  CRUSH rule 0 x 743 [654,914,425]
+  CRUSH rule 0 x 744 [725,295,579]
+  CRUSH rule 0 x 745 [787,858,850]
+  CRUSH rule 0 x 746 [757,848,704]
+  CRUSH rule 0 x 747 [700,81,867]
+  CRUSH rule 0 x 748 [557,436,238]
+  CRUSH rule 0 x 749 [772,622,337]
+  CRUSH rule 0 x 750 [946,97,376]
+  CRUSH rule 0 x 751 [996,618,343]
+  CRUSH rule 0 x 752 [746,887,695]
+  CRUSH rule 0 x 753 [741,14,463]
+  CRUSH rule 0 x 754 [648,349,333]
+  CRUSH rule 0 x 755 [157,460,466]
+  CRUSH rule 0 x 756 [416,97,197]
+  CRUSH rule 0 x 757 [599,839,776]
+  CRUSH rule 0 x 758 [994,218,620]
+  CRUSH rule 0 x 759 [959,682,514]
+  CRUSH rule 0 x 760 [518,943,215]
+  CRUSH rule 0 x 761 [285,849,420]
+  CRUSH rule 0 x 762 [591,313,41]
+  CRUSH rule 0 x 763 [908,411,200]
+  CRUSH rule 0 x 764 [787,234,894]
+  CRUSH rule 0 x 765 [327,921,882]
+  CRUSH rule 0 x 766 [84,161,878]
+  CRUSH rule 0 x 767 [370,895,702]
+  CRUSH rule 0 x 768 [826,760,879]
+  CRUSH rule 0 x 769 [67,768,663]
+  CRUSH rule 0 x 770 [593,909,482]
+  CRUSH rule 0 x 771 [309,935,121]
+  CRUSH rule 0 x 772 [12,125,797]
+  CRUSH rule 0 x 773 [253,466,820]
+  CRUSH rule 0 x 774 [164,390,705]
+  CRUSH rule 0 x 775 [703,47,43]
+  CRUSH rule 0 x 776 [728,231,80]
+  CRUSH rule 0 x 777 [981,621,568]
+  CRUSH rule 0 x 778 [411,456,544]
+  CRUSH rule 0 x 779 [346,121,519]
+  CRUSH rule 0 x 780 [476,39,288]
+  CRUSH rule 0 x 781 [10,130,585]
+  CRUSH rule 0 x 782 [462,246,581]
+  CRUSH rule 0 x 783 [580,373,153]
+  CRUSH rule 0 x 784 [413,113,978]
+  CRUSH rule 0 x 785 [341,856,332]
+  CRUSH rule 0 x 786 [411,140,313]
+  CRUSH rule 0 x 787 [605,522,211]
+  CRUSH rule 0 x 788 [226,545,35]
+  CRUSH rule 0 x 789 [545,320,414]
+  CRUSH rule 0 x 790 [414,748,816]
+  CRUSH rule 0 x 791 [660,906,406]
+  CRUSH rule 0 x 792 [287,392,514]
+  CRUSH rule 0 x 793 [631,133,850]
+  CRUSH rule 0 x 794 [931,517,543]
+  CRUSH rule 0 x 795 [551,962,477]
+  CRUSH rule 0 x 796 [814,4,95]
+  CRUSH rule 0 x 797 [64,201,299]
+  CRUSH rule 0 x 798 [422,530,114]
+  CRUSH rule 0 x 799 [824,32,679]
+  CRUSH rule 0 x 800 [862,623,489]
+  CRUSH rule 0 x 801 [145,550,329]
+  CRUSH rule 0 x 802 [570,19,847]
+  CRUSH rule 0 x 803 [151,812,662]
+  CRUSH rule 0 x 804 [467,93,264]
+  CRUSH rule 0 x 805 [621,223,938]
+  CRUSH rule 0 x 806 [898,957,805]
+  CRUSH rule 0 x 807 [354,531,422]
+  CRUSH rule 0 x 808 [7,96,76]
+  CRUSH rule 0 x 809 [70,734,719]
+  CRUSH rule 0 x 810 [701,18,972]
+  CRUSH rule 0 x 811 [248,547,103]
+  CRUSH rule 0 x 812 [230,576,821]
+  CRUSH rule 0 x 813 [805,114,683]
+  CRUSH rule 0 x 814 [54,619,973]
+  CRUSH rule 0 x 815 [679,412,613]
+  CRUSH rule 0 x 816 [919,448,826]
+  CRUSH rule 0 x 817 [765,830,436]
+  CRUSH rule 0 x 818 [415,566,644]
+  CRUSH rule 0 x 819 [721,319,865]
+  CRUSH rule 0 x 820 [218,301,333]
+  CRUSH rule 0 x 821 [185,795,680]
+  CRUSH rule 0 x 822 [356,261,54]
+  CRUSH rule 0 x 823 [220,281,549]
+  CRUSH rule 0 x 824 [292,809,887]
+  CRUSH rule 0 x 825 [949,778,101]
+  CRUSH rule 0 x 826 [767,818,833]
+  CRUSH rule 0 x 827 [631,83,406]
+  CRUSH rule 0 x 828 [288,986,445]
+  CRUSH rule 0 x 829 [990,667,915]
+  CRUSH rule 0 x 830 [152,571,778]
+  CRUSH rule 0 x 831 [814,563,630]
+  CRUSH rule 0 x 832 [235,641,616]
+  CRUSH rule 0 x 833 [657,565,922]
+  CRUSH rule 0 x 834 [907,231,644]
+  CRUSH rule 0 x 835 [784,262,771]
+  CRUSH rule 0 x 836 [951,158,366]
+  CRUSH rule 0 x 837 [556,498,334]
+  CRUSH rule 0 x 838 [329,274,964]
+  CRUSH rule 0 x 839 [568,209,939]
+  CRUSH rule 0 x 840 [45,579,842]
+  CRUSH rule 0 x 841 [652,702,24]
+  CRUSH rule 0 x 842 [629,984,314]
+  CRUSH rule 0 x 843 [799,690,688]
+  CRUSH rule 0 x 844 [694,600,534]
+  CRUSH rule 0 x 845 [332,30,179]
+  CRUSH rule 0 x 846 [452,251,712]
+  CRUSH rule 0 x 847 [399,681,847]
+  CRUSH rule 0 x 848 [303,138,440]
+  CRUSH rule 0 x 849 [666,346,708]
+  CRUSH rule 0 x 850 [644,511,345]
+  CRUSH rule 0 x 851 [527,546,737]
+  CRUSH rule 0 x 852 [31,809,94]
+  CRUSH rule 0 x 853 [483,330,869]
+  CRUSH rule 0 x 854 [697,953,968]
+  CRUSH rule 0 x 855 [837,996,239]
+  CRUSH rule 0 x 856 [712,40,547]
+  CRUSH rule 0 x 857 [77,984,576]
+  CRUSH rule 0 x 858 [412,384,841]
+  CRUSH rule 0 x 859 [173,760,26]
+  CRUSH rule 0 x 860 [776,429,328]
+  CRUSH rule 0 x 861 [705,405,477]
+  CRUSH rule 0 x 862 [809,44,788]
+  CRUSH rule 0 x 863 [349,496,963]
+  CRUSH rule 0 x 864 [717,858,101]
+  CRUSH rule 0 x 865 [857,603,586]
+  CRUSH rule 0 x 866 [394,304,71]
+  CRUSH rule 0 x 867 [640,773,663]
+  CRUSH rule 0 x 868 [613,950,712]
+  CRUSH rule 0 x 869 [973,889,524]
+  CRUSH rule 0 x 870 [505,35,386]
+  CRUSH rule 0 x 871 [239,264,262]
+  CRUSH rule 0 x 872 [21,767,456]
+  CRUSH rule 0 x 873 [954,666,980]
+  CRUSH rule 0 x 874 [54,510,947]
+  CRUSH rule 0 x 875 [809,418,452]
+  CRUSH rule 0 x 876 [483,457,61]
+  CRUSH rule 0 x 877 [542,531,952]
+  CRUSH rule 0 x 878 [217,674,857]
+  CRUSH rule 0 x 879 [999,475,134]
+  CRUSH rule 0 x 880 [678,573,935]
+  CRUSH rule 0 x 881 [394,835,789]
+  CRUSH rule 0 x 882 [467,382,353]
+  CRUSH rule 0 x 883 [802,744,237]
+  CRUSH rule 0 x 884 [653,660,638]
+  CRUSH rule 0 x 885 [898,704,307]
+  CRUSH rule 0 x 886 [434,357,938]
+  CRUSH rule 0 x 887 [297,226,711]
+  CRUSH rule 0 x 888 [863,324,443]
+  CRUSH rule 0 x 889 [105,102,308]
+  CRUSH rule 0 x 890 [550,248,606]
+  CRUSH rule 0 x 891 [575,928,880]
+  CRUSH rule 0 x 892 [259,862,133]
+  CRUSH rule 0 x 893 [902,880,543]
+  CRUSH rule 0 x 894 [180,169,916]
+  CRUSH rule 0 x 895 [725,849,182]
+  CRUSH rule 0 x 896 [951,34,874]
+  CRUSH rule 0 x 897 [810,352,73]
+  CRUSH rule 0 x 898 [979,433,719]
+  CRUSH rule 0 x 899 [685,668,534]
+  CRUSH rule 0 x 900 [530,978,41]
+  CRUSH rule 0 x 901 [740,107,336]
+  CRUSH rule 0 x 902 [800,743,693]
+  CRUSH rule 0 x 903 [230,267,842]
+  CRUSH rule 0 x 904 [346,949,460]
+  CRUSH rule 0 x 905 [530,397,619]
+  CRUSH rule 0 x 906 [80,426,138]
+  CRUSH rule 0 x 907 [365,968,475]
+  CRUSH rule 0 x 908 [204,832,742]
+  CRUSH rule 0 x 909 [883,989,146]
+  CRUSH rule 0 x 910 [549,593,249]
+  CRUSH rule 0 x 911 [325,847,352]
+  CRUSH rule 0 x 912 [874,888,582]
+  CRUSH rule 0 x 913 [331,463,342]
+  CRUSH rule 0 x 914 [836,468,601]
+  CRUSH rule 0 x 915 [245,228,100]
+  CRUSH rule 0 x 916 [77,967,364]
+  CRUSH rule 0 x 917 [239,60,866]
+  CRUSH rule 0 x 918 [988,115,922]
+  CRUSH rule 0 x 919 [783,139,696]
+  CRUSH rule 0 x 920 [623,408,685]
+  CRUSH rule 0 x 921 [105,799,144]
+  CRUSH rule 0 x 922 [887,505,652]
+  CRUSH rule 0 x 923 [223,318,552]
+  CRUSH rule 0 x 924 [25,778,366]
+  CRUSH rule 0 x 925 [912,601,297]
+  CRUSH rule 0 x 926 [968,133,132]
+  CRUSH rule 0 x 927 [277,724,214]
+  CRUSH rule 0 x 928 [554,203,658]
+  CRUSH rule 0 x 929 [761,802,367]
+  CRUSH rule 0 x 930 [814,61,788]
+  CRUSH rule 0 x 931 [29,193,61]
+  CRUSH rule 0 x 932 [446,198,862]
+  CRUSH rule 0 x 933 [352,742,216]
+  CRUSH rule 0 x 934 [730,2,332]
+  CRUSH rule 0 x 935 [731,23,736]
+  CRUSH rule 0 x 936 [322,975,20]
+  CRUSH rule 0 x 937 [822,221,841]
+  CRUSH rule 0 x 938 [557,850,66]
+  CRUSH rule 0 x 939 [150,11,971]
+  CRUSH rule 0 x 940 [638,398,169]
+  CRUSH rule 0 x 941 [730,342,929]
+  CRUSH rule 0 x 942 [62,292,166]
+  CRUSH rule 0 x 943 [165,314,519]
+  CRUSH rule 0 x 944 [199,625,766]
+  CRUSH rule 0 x 945 [946,999,699]
+  CRUSH rule 0 x 946 [595,93,852]
+  CRUSH rule 0 x 947 [800,582,356]
+  CRUSH rule 0 x 948 [132,551,139]
+  CRUSH rule 0 x 949 [792,920,466]
+  CRUSH rule 0 x 950 [111,345,176]
+  CRUSH rule 0 x 951 [414,619,648]
+  CRUSH rule 0 x 952 [775,469,500]
+  CRUSH rule 0 x 953 [349,1,5]
+  CRUSH rule 0 x 954 [570,940,410]
+  CRUSH rule 0 x 955 [729,774,823]
+  CRUSH rule 0 x 956 [519,141,575]
+  CRUSH rule 0 x 957 [242,709,611]
+  CRUSH rule 0 x 958 [84,217,227]
+  CRUSH rule 0 x 959 [270,413,918]
+  CRUSH rule 0 x 960 [458,192,307]
+  CRUSH rule 0 x 961 [981,388,777]
+  CRUSH rule 0 x 962 [623,834,277]
+  CRUSH rule 0 x 963 [291,167,714]
+  CRUSH rule 0 x 964 [28,156,788]
+  CRUSH rule 0 x 965 [675,557,290]
+  CRUSH rule 0 x 966 [836,306,946]
+  CRUSH rule 0 x 967 [966,386,735]
+  CRUSH rule 0 x 968 [864,756,690]
+  CRUSH rule 0 x 969 [729,625,480]
+  CRUSH rule 0 x 970 [800,362,646]
+  CRUSH rule 0 x 971 [737,381,153]
+  CRUSH rule 0 x 972 [952,245,720]
+  CRUSH rule 0 x 973 [356,455,579]
+  CRUSH rule 0 x 974 [545,758,586]
+  CRUSH rule 0 x 975 [336,191,202]
+  CRUSH rule 0 x 976 [446,208,757]
+  CRUSH rule 0 x 977 [202,896,196]
+  CRUSH rule 0 x 978 [612,324,996]
+  CRUSH rule 0 x 979 [843,457,675]
+  CRUSH rule 0 x 980 [60,914,881]
+  CRUSH rule 0 x 981 [702,749,937]
+  CRUSH rule 0 x 982 [298,928,738]
+  CRUSH rule 0 x 983 [723,572,395]
+  CRUSH rule 0 x 984 [723,864,804]
+  CRUSH rule 0 x 985 [945,459,868]
+  CRUSH rule 0 x 986 [772,664,535]
+  CRUSH rule 0 x 987 [88,324,312]
+  CRUSH rule 0 x 988 [522,927,131]
+  CRUSH rule 0 x 989 [578,332,208]
+  CRUSH rule 0 x 990 [638,228,414]
+  CRUSH rule 0 x 991 [530,221,451]
+  CRUSH rule 0 x 992 [925,705,275]
+  CRUSH rule 0 x 993 [991,301,43]
+  CRUSH rule 0 x 994 [276,51,868]
+  CRUSH rule 0 x 995 [288,836,753]
+  CRUSH rule 0 x 996 [887,983,252]
+  CRUSH rule 0 x 997 [110,924,386]
+  CRUSH rule 0 x 998 [435,830,485]
+  CRUSH rule 0 x 999 [876,738,357]
+  CRUSH rule 0 x 1000 [178,963,638]
+  CRUSH rule 0 x 1001 [99,519,66]
+  CRUSH rule 0 x 1002 [515,534,468]
+  CRUSH rule 0 x 1003 [104,611,937]
+  CRUSH rule 0 x 1004 [269,638,724]
+  CRUSH rule 0 x 1005 [369,223,309]
+  CRUSH rule 0 x 1006 [40,107,69]
+  CRUSH rule 0 x 1007 [978,111,416]
+  CRUSH rule 0 x 1008 [965,956,624]
+  CRUSH rule 0 x 1009 [598,476,356]
+  CRUSH rule 0 x 1010 [767,523,239]
+  CRUSH rule 0 x 1011 [289,871,207]
+  CRUSH rule 0 x 1012 [128,28,370]
+  CRUSH rule 0 x 1013 [979,765,660]
+  CRUSH rule 0 x 1014 [979,948,513]
+  CRUSH rule 0 x 1015 [277,790,396]
+  CRUSH rule 0 x 1016 [262,73,128]
+  CRUSH rule 0 x 1017 [150,269,61]
+  CRUSH rule 0 x 1018 [555,829,554]
+  CRUSH rule 0 x 1019 [513,356,265]
+  CRUSH rule 0 x 1020 [158,161,877]
+  CRUSH rule 0 x 1021 [915,998,957]
+  CRUSH rule 0 x 1022 [967,829,973]
+  CRUSH rule 0 x 1023 [488,257,614]
+  rule 0 (data) num_rep 3 result size == 3:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450]
+  CRUSH rule 0 x 1 [876,250,334,633]
+  CRUSH rule 0 x 2 [292,832,53,392]
+  CRUSH rule 0 x 3 [623,387,124,998]
+  CRUSH rule 0 x 4 [61,334,710,4]
+  CRUSH rule 0 x 5 [946,557,713,664]
+  CRUSH rule 0 x 6 [576,668,212,163]
+  CRUSH rule 0 x 7 [645,753,906,393]
+  CRUSH rule 0 x 8 [243,6,863,781]
+  CRUSH rule 0 x 9 [22,578,251,410]
+  CRUSH rule 0 x 10 [758,828,360,477]
+  CRUSH rule 0 x 11 [769,120,124,527]
+  CRUSH rule 0 x 12 [780,364,689,755]
+  CRUSH rule 0 x 13 [557,18,351,719]
+  CRUSH rule 0 x 14 [59,561,249,461]
+  CRUSH rule 0 x 15 [718,928,993,21]
+  CRUSH rule 0 x 16 [673,632,841,954]
+  CRUSH rule 0 x 17 [648,43,560,514]
+  CRUSH rule 0 x 18 [654,219,181,568]
+  CRUSH rule 0 x 19 [850,545,377,848]
+  CRUSH rule 0 x 20 [717,785,974,5]
+  CRUSH rule 0 x 21 [420,57,519,306]
+  CRUSH rule 0 x 22 [503,998,193,821]
+  CRUSH rule 0 x 23 [411,663,168,110]
+  CRUSH rule 0 x 24 [266,861,353,1]
+  CRUSH rule 0 x 25 [760,483,818,600]
+  CRUSH rule 0 x 26 [903,24,573,718]
+  CRUSH rule 0 x 27 [946,188,289,510]
+  CRUSH rule 0 x 28 [69,312,73,198]
+  CRUSH rule 0 x 29 [844,883,337,628]
+  CRUSH rule 0 x 30 [621,18,613,794]
+  CRUSH rule 0 x 31 [784,943,814,539]
+  CRUSH rule 0 x 32 [173,374,369,972]
+  CRUSH rule 0 x 33 [698,336,357,966]
+  CRUSH rule 0 x 34 [168,836,210,798]
+  CRUSH rule 0 x 35 [274,509,534,818]
+  CRUSH rule 0 x 36 [318,215,153,628]
+  CRUSH rule 0 x 37 [173,604,109,935]
+  CRUSH rule 0 x 38 [708,444,683,604]
+  CRUSH rule 0 x 39 [662,198,417,680]
+  CRUSH rule 0 x 40 [620,801,414,78]
+  CRUSH rule 0 x 41 [811,264,177,127]
+  CRUSH rule 0 x 42 [863,179,527,660]
+  CRUSH rule 0 x 43 [686,822,988,228]
+  CRUSH rule 0 x 44 [396,222,46,841]
+  CRUSH rule 0 x 45 [991,694,253,142]
+  CRUSH rule 0 x 46 [420,909,184,285]
+  CRUSH rule 0 x 47 [467,211,605,207]
+  CRUSH rule 0 x 48 [955,329,368,168]
+  CRUSH rule 0 x 49 [974,891,931,29]
+  CRUSH rule 0 x 50 [870,441,691,823]
+  CRUSH rule 0 x 51 [182,930,25,936]
+  CRUSH rule 0 x 52 [704,812,894,794]
+  CRUSH rule 0 x 53 [185,713,631,280]
+  CRUSH rule 0 x 54 [270,441,100,82]
+  CRUSH rule 0 x 55 [895,734,958,793]
+  CRUSH rule 0 x 56 [564,963,683,324]
+  CRUSH rule 0 x 57 [738,130,208,973]
+  CRUSH rule 0 x 58 [524,113,806,903]
+  CRUSH rule 0 x 59 [408,337,668,529]
+  CRUSH rule 0 x 60 [228,790,857,309]
+  CRUSH rule 0 x 61 [154,843,717,467]
+  CRUSH rule 0 x 62 [594,811,549,276]
+  CRUSH rule 0 x 63 [646,67,884,925]
+  CRUSH rule 0 x 64 [175,542,155,837]
+  CRUSH rule 0 x 65 [745,619,131,867]
+  CRUSH rule 0 x 66 [275,468,23,35]
+  CRUSH rule 0 x 67 [246,958,524,493]
+  CRUSH rule 0 x 68 [711,473,403,228]
+  CRUSH rule 0 x 69 [493,924,850,939]
+  CRUSH rule 0 x 70 [30,499,644,33]
+  CRUSH rule 0 x 71 [984,883,574,716]
+  CRUSH rule 0 x 72 [71,286,942,363]
+  CRUSH rule 0 x 73 [922,618,3,371]
+  CRUSH rule 0 x 74 [629,414,185,573]
+  CRUSH rule 0 x 75 [222,20,174,820]
+  CRUSH rule 0 x 76 [262,366,339,290]
+  CRUSH rule 0 x 77 [638,469,992,280]
+  CRUSH rule 0 x 78 [324,511,788,7]
+  CRUSH rule 0 x 79 [577,990,64,94]
+  CRUSH rule 0 x 80 [501,95,278,903]
+  CRUSH rule 0 x 81 [506,812,9,698]
+  CRUSH rule 0 x 82 [222,145,80,785]
+  CRUSH rule 0 x 83 [71,634,61,91]
+  CRUSH rule 0 x 84 [49,761,773,368]
+  CRUSH rule 0 x 85 [985,896,708,861]
+  CRUSH rule 0 x 86 [537,745,93,524]
+  CRUSH rule 0 x 87 [997,317,463,626]
+  CRUSH rule 0 x 88 [957,350,890,857]
+  CRUSH rule 0 x 89 [399,730,148,314]
+  CRUSH rule 0 x 90 [943,706,683,267]
+  CRUSH rule 0 x 91 [22,368,149,928]
+  CRUSH rule 0 x 92 [532,424,426,773]
+  CRUSH rule 0 x 93 [218,489,405,681]
+  CRUSH rule 0 x 94 [181,96,102,515]
+  CRUSH rule 0 x 95 [343,957,820,139]
+  CRUSH rule 0 x 96 [861,270,87,797]
+  CRUSH rule 0 x 97 [459,706,45,328]
+  CRUSH rule 0 x 98 [327,867,353,948]
+  CRUSH rule 0 x 99 [974,133,468,906]
+  CRUSH rule 0 x 100 [32,445,547,371]
+  CRUSH rule 0 x 101 [142,90,337,950]
+  CRUSH rule 0 x 102 [172,129,139,22]
+  CRUSH rule 0 x 103 [630,47,161,356]
+  CRUSH rule 0 x 104 [758,133,278,11]
+  CRUSH rule 0 x 105 [843,604,47,33]
+  CRUSH rule 0 x 106 [28,681,193,679]
+  CRUSH rule 0 x 107 [74,320,85,819]
+  CRUSH rule 0 x 108 [875,593,575,517]
+  CRUSH rule 0 x 109 [411,985,811,720]
+  CRUSH rule 0 x 110 [440,774,799,660]
+  CRUSH rule 0 x 111 [405,742,276,359]
+  CRUSH rule 0 x 112 [143,181,922,545]
+  CRUSH rule 0 x 113 [153,846,160,903]
+  CRUSH rule 0 x 114 [804,892,939,20]
+  CRUSH rule 0 x 115 [588,508,958,580]
+  CRUSH rule 0 x 116 [327,148,637,486]
+  CRUSH rule 0 x 117 [95,594,989,131]
+  CRUSH rule 0 x 118 [80,957,897,239]
+  CRUSH rule 0 x 119 [386,932,951,768]
+  CRUSH rule 0 x 120 [366,312,653,936]
+  CRUSH rule 0 x 121 [129,154,847,16]
+  CRUSH rule 0 x 122 [873,1,110,939]
+  CRUSH rule 0 x 123 [533,415,789,600]
+  CRUSH rule 0 x 124 [461,691,898,723]
+  CRUSH rule 0 x 125 [342,599,830,402]
+  CRUSH rule 0 x 126 [819,781,822,548]
+  CRUSH rule 0 x 127 [437,893,585,707]
+  CRUSH rule 0 x 128 [679,994,982,550]
+  CRUSH rule 0 x 129 [380,685,947,302]
+  CRUSH rule 0 x 130 [992,52,466,867]
+  CRUSH rule 0 x 131 [469,90,208,599]
+  CRUSH rule 0 x 132 [571,250,316,535]
+  CRUSH rule 0 x 133 [964,728,329,902]
+  CRUSH rule 0 x 134 [999,19,716,963]
+  CRUSH rule 0 x 135 [634,101,52,938]
+  CRUSH rule 0 x 136 [114,889,692,768]
+  CRUSH rule 0 x 137 [839,8,959,280]
+  CRUSH rule 0 x 138 [967,949,138,451]
+  CRUSH rule 0 x 139 [308,711,736,247]
+  CRUSH rule 0 x 140 [764,936,926,55]
+  CRUSH rule 0 x 141 [423,302,112,216]
+  CRUSH rule 0 x 142 [252,821,715,340]
+  CRUSH rule 0 x 143 [33,808,518,477]
+  CRUSH rule 0 x 144 [472,88,969,162]
+  CRUSH rule 0 x 145 [242,208,252,604]
+  CRUSH rule 0 x 146 [290,70,570,384]
+  CRUSH rule 0 x 147 [447,352,657,493]
+  CRUSH rule 0 x 148 [212,644,432,658]
+  CRUSH rule 0 x 149 [9,775,87,35]
+  CRUSH rule 0 x 150 [166,456,582,144]
+  CRUSH rule 0 x 151 [811,875,307,20]
+  CRUSH rule 0 x 152 [449,617,223,9]
+  CRUSH rule 0 x 153 [523,537,695,627]
+  CRUSH rule 0 x 154 [208,559,874,597]
+  CRUSH rule 0 x 155 [569,325,192,296]
+  CRUSH rule 0 x 156 [488,121,521,213]
+  CRUSH rule 0 x 157 [140,723,633,260]
+  CRUSH rule 0 x 158 [786,451,320,239]
+  CRUSH rule 0 x 159 [134,664,517,821]
+  CRUSH rule 0 x 160 [690,112,414,990]
+  CRUSH rule 0 x 161 [324,912,397,423]
+  CRUSH rule 0 x 162 [748,567,284,183]
+  CRUSH rule 0 x 163 [575,499,31,816]
+  CRUSH rule 0 x 164 [314,489,308,326]
+  CRUSH rule 0 x 165 [116,209,750,53]
+  CRUSH rule 0 x 166 [352,706,701,810]
+  CRUSH rule 0 x 167 [27,743,174,142]
+  CRUSH rule 0 x 168 [953,898,880,660]
+  CRUSH rule 0 x 169 [912,147,266,547]
+  CRUSH rule 0 x 170 [421,515,828,844]
+  CRUSH rule 0 x 171 [488,584,880,964]
+  CRUSH rule 0 x 172 [366,443,957,66]
+  CRUSH rule 0 x 173 [863,291,625,287]
+  CRUSH rule 0 x 174 [263,555,650,410]
+  CRUSH rule 0 x 175 [875,961,361,575]
+  CRUSH rule 0 x 176 [745,83,701,680]
+  CRUSH rule 0 x 177 [128,244,41,123]
+  CRUSH rule 0 x 178 [155,41,264,777]
+  CRUSH rule 0 x 179 [593,833,202,183]
+  CRUSH rule 0 x 180 [154,734,17,831]
+  CRUSH rule 0 x 181 [289,675,723,800]
+  CRUSH rule 0 x 182 [730,931,560,209]
+  CRUSH rule 0 x 183 [639,237,794,815]
+  CRUSH rule 0 x 184 [704,312,685,645]
+  CRUSH rule 0 x 185 [97,100,762,82]
+  CRUSH rule 0 x 186 [26,665,554,215]
+  CRUSH rule 0 x 187 [649,14,740,494]
+  CRUSH rule 0 x 188 [682,695,590,743]
+  CRUSH rule 0 x 189 [325,693,726,51]
+  CRUSH rule 0 x 190 [399,933,136,955]
+  CRUSH rule 0 x 191 [629,533,17,126]
+  CRUSH rule 0 x 192 [503,578,38,492]
+  CRUSH rule 0 x 193 [546,333,651,678]
+  CRUSH rule 0 x 194 [242,473,58,655]
+  CRUSH rule 0 x 195 [625,719,135,81]
+  CRUSH rule 0 x 196 [357,114,125,867]
+  CRUSH rule 0 x 197 [306,954,453,873]
+  CRUSH rule 0 x 198 [863,791,311,911]
+  CRUSH rule 0 x 199 [935,906,929,252]
+  CRUSH rule 0 x 200 [373,774,229,454]
+  CRUSH rule 0 x 201 [659,320,477,313]
+  CRUSH rule 0 x 202 [260,433,524,880]
+  CRUSH rule 0 x 203 [36,239,675,971]
+  CRUSH rule 0 x 204 [92,516,993,728]
+  CRUSH rule 0 x 205 [68,395,473,45]
+  CRUSH rule 0 x 206 [570,530,642,380]
+  CRUSH rule 0 x 207 [834,457,850,917]
+  CRUSH rule 0 x 208 [927,484,640,976]
+  CRUSH rule 0 x 209 [878,66,58,940]
+  CRUSH rule 0 x 210 [572,981,484,29]
+  CRUSH rule 0 x 211 [107,597,780,857]
+  CRUSH rule 0 x 212 [389,107,838,624]
+  CRUSH rule 0 x 213 [497,717,567,728]
+  CRUSH rule 0 x 214 [798,65,254,572]
+  CRUSH rule 0 x 215 [233,419,283,638]
+  CRUSH rule 0 x 216 [494,464,742,523]
+  CRUSH rule 0 x 217 [352,396,309,938]
+  CRUSH rule 0 x 218 [895,864,988,650]
+  CRUSH rule 0 x 219 [222,534,277,242]
+  CRUSH rule 0 x 220 [281,19,584,563]
+  CRUSH rule 0 x 221 [64,928,963,130]
+  CRUSH rule 0 x 222 [40,544,161,199]
+  CRUSH rule 0 x 223 [645,556,159,417]
+  CRUSH rule 0 x 224 [647,165,957,263]
+  CRUSH rule 0 x 225 [219,714,858,747]
+  CRUSH rule 0 x 226 [372,511,181,277]
+  CRUSH rule 0 x 227 [925,156,714,863]
+  CRUSH rule 0 x 228 [682,404,839,263]
+  CRUSH rule 0 x 229 [880,838,770,891]
+  CRUSH rule 0 x 230 [328,659,916,468]
+  CRUSH rule 0 x 231 [320,383,669,109]
+  CRUSH rule 0 x 232 [924,846,394,319]
+  CRUSH rule 0 x 233 [948,652,575,838]
+  CRUSH rule 0 x 234 [484,943,42,575]
+  CRUSH rule 0 x 235 [750,65,590,168]
+  CRUSH rule 0 x 236 [551,787,490,136]
+  CRUSH rule 0 x 237 [390,157,166,251]
+  CRUSH rule 0 x 238 [570,6,989,707]
+  CRUSH rule 0 x 239 [729,959,376,975]
+  CRUSH rule 0 x 240 [981,241,156,767]
+  CRUSH rule 0 x 241 [310,816,641,177]
+  CRUSH rule 0 x 242 [161,63,642,837]
+  CRUSH rule 0 x 243 [180,394,33,683]
+  CRUSH rule 0 x 244 [52,174,685,189]
+  CRUSH rule 0 x 245 [523,121,915,84]
+  CRUSH rule 0 x 246 [362,893,390,487]
+  CRUSH rule 0 x 247 [382,184,116,34]
+  CRUSH rule 0 x 248 [129,114,852,469]
+  CRUSH rule 0 x 249 [159,683,91,856]
+  CRUSH rule 0 x 250 [404,945,569,955]
+  CRUSH rule 0 x 251 [661,225,738,757]
+  CRUSH rule 0 x 252 [961,226,542,103]
+  CRUSH rule 0 x 253 [651,97,225,364]
+  CRUSH rule 0 x 254 [123,33,741,692]
+  CRUSH rule 0 x 255 [314,649,891,855]
+  CRUSH rule 0 x 256 [315,215,651,126]
+  CRUSH rule 0 x 257 [825,264,867,869]
+  CRUSH rule 0 x 258 [624,789,370,723]
+  CRUSH rule 0 x 259 [602,542,70,563]
+  CRUSH rule 0 x 260 [717,878,43,56]
+  CRUSH rule 0 x 261 [145,517,20,903]
+  CRUSH rule 0 x 262 [223,1,561,420]
+  CRUSH rule 0 x 263 [462,211,405,508]
+  CRUSH rule 0 x 264 [654,471,266,662]
+  CRUSH rule 0 x 265 [302,794,704,798]
+  CRUSH rule 0 x 266 [202,132,884,209]
+  CRUSH rule 0 x 267 [282,938,657,113]
+  CRUSH rule 0 x 268 [338,309,356,278]
+  CRUSH rule 0 x 269 [738,122,266,200]
+  CRUSH rule 0 x 270 [707,982,946,196]
+  CRUSH rule 0 x 271 [705,432,364,735]
+  CRUSH rule 0 x 272 [756,545,942,56]
+  CRUSH rule 0 x 273 [197,502,527,721]
+  CRUSH rule 0 x 274 [992,44,653,573]
+  CRUSH rule 0 x 275 [544,789,170,434]
+  CRUSH rule 0 x 276 [658,467,577,268]
+  CRUSH rule 0 x 277 [143,490,880,483]
+  CRUSH rule 0 x 278 [492,647,355,282]
+  CRUSH rule 0 x 279 [517,792,604,987]
+  CRUSH rule 0 x 280 [825,740,27,848]
+  CRUSH rule 0 x 281 [224,629,120,562]
+  CRUSH rule 0 x 282 [298,661,380,416]
+  CRUSH rule 0 x 283 [311,606,208,50]
+  CRUSH rule 0 x 284 [771,466,371,743]
+  CRUSH rule 0 x 285 [693,362,404,676]
+  CRUSH rule 0 x 286 [364,477,285,167]
+  CRUSH rule 0 x 287 [591,611,828,995]
+  CRUSH rule 0 x 288 [965,541,848,796]
+  CRUSH rule 0 x 289 [225,551,948,877]
+  CRUSH rule 0 x 290 [577,762,777,751]
+  CRUSH rule 0 x 291 [160,903,477,381]
+  CRUSH rule 0 x 292 [873,598,216,666]
+  CRUSH rule 0 x 293 [100,234,874,47]
+  CRUSH rule 0 x 294 [285,943,379,520]
+  CRUSH rule 0 x 295 [938,262,880,327]
+  CRUSH rule 0 x 296 [850,327,86,472]
+  CRUSH rule 0 x 297 [951,53,99,558]
+  CRUSH rule 0 x 298 [173,336,85,766]
+  CRUSH rule 0 x 299 [598,591,315,386]
+  CRUSH rule 0 x 300 [531,957,62,459]
+  CRUSH rule 0 x 301 [823,628,23,858]
+  CRUSH rule 0 x 302 [184,80,780,871]
+  CRUSH rule 0 x 303 [521,766,222,830]
+  CRUSH rule 0 x 304 [980,127,807,507]
+  CRUSH rule 0 x 305 [153,816,22,927]
+  CRUSH rule 0 x 306 [423,739,664,753]
+  CRUSH rule 0 x 307 [997,557,682,456]
+  CRUSH rule 0 x 308 [991,874,534,465]
+  CRUSH rule 0 x 309 [860,394,724,858]
+  CRUSH rule 0 x 310 [589,818,546,201]
+  CRUSH rule 0 x 311 [477,774,225,590]
+  CRUSH rule 0 x 312 [887,853,950,354]
+  CRUSH rule 0 x 313 [802,646,447,416]
+  CRUSH rule 0 x 314 [654,974,229,511]
+  CRUSH rule 0 x 315 [767,227,28,740]
+  CRUSH rule 0 x 316 [778,83,733,359]
+  CRUSH rule 0 x 317 [184,418,642,986]
+  CRUSH rule 0 x 318 [525,410,500,543]
+  CRUSH rule 0 x 319 [476,724,569,382]
+  CRUSH rule 0 x 320 [149,610,697,296]
+  CRUSH rule 0 x 321 [710,79,667,671]
+  CRUSH rule 0 x 322 [175,275,323,333]
+  CRUSH rule 0 x 323 [819,604,638,792]
+  CRUSH rule 0 x 324 [16,745,511,439]
+  CRUSH rule 0 x 325 [486,400,872,873]
+  CRUSH rule 0 x 326 [613,765,207,19]
+  CRUSH rule 0 x 327 [125,289,738,408]
+  CRUSH rule 0 x 328 [807,383,476,583]
+  CRUSH rule 0 x 329 [588,938,599,432]
+  CRUSH rule 0 x 330 [932,644,41,611]
+  CRUSH rule 0 x 331 [341,953,950,537]
+  CRUSH rule 0 x 332 [153,726,459,950]
+  CRUSH rule 0 x 333 [745,845,853,860]
+  CRUSH rule 0 x 334 [614,751,807,58]
+  CRUSH rule 0 x 335 [518,721,221,283]
+  CRUSH rule 0 x 336 [389,424,77,309]
+  CRUSH rule 0 x 337 [753,508,765,720]
+  CRUSH rule 0 x 338 [128,810,490,753]
+  CRUSH rule 0 x 339 [430,308,58,751]
+  CRUSH rule 0 x 340 [541,44,630,231]
+  CRUSH rule 0 x 341 [402,26,631,439]
+  CRUSH rule 0 x 342 [982,57,992,461]
+  CRUSH rule 0 x 343 [833,412,572,732]
+  CRUSH rule 0 x 344 [784,533,792,41]
+  CRUSH rule 0 x 345 [546,300,304,691]
+  CRUSH rule 0 x 346 [302,420,428,891]
+  CRUSH rule 0 x 347 [488,778,101,217]
+  CRUSH rule 0 x 348 [903,744,937,718]
+  CRUSH rule 0 x 349 [471,547,582,306]
+  CRUSH rule 0 x 350 [348,221,823,335]
+  CRUSH rule 0 x 351 [961,582,705,346]
+  CRUSH rule 0 x 352 [728,137,461,298]
+  CRUSH rule 0 x 353 [904,202,184,447]
+  CRUSH rule 0 x 354 [345,226,319,256]
+  CRUSH rule 0 x 355 [50,430,175,43]
+  CRUSH rule 0 x 356 [87,185,55,423]
+  CRUSH rule 0 x 357 [762,459,921,473]
+  CRUSH rule 0 x 358 [908,25,280,6]
+  CRUSH rule 0 x 359 [484,15,132,121]
+  CRUSH rule 0 x 360 [173,378,337,702]
+  CRUSH rule 0 x 361 [404,577,115,25]
+  CRUSH rule 0 x 362 [403,1,422,945]
+  CRUSH rule 0 x 363 [639,911,510,162]
+  CRUSH rule 0 x 364 [752,689,610,990]
+  CRUSH rule 0 x 365 [956,999,212,230]
+  CRUSH rule 0 x 366 [860,925,924,763]
+  CRUSH rule 0 x 367 [205,609,647,665]
+  CRUSH rule 0 x 368 [301,284,810,169]
+  CRUSH rule 0 x 369 [452,658,339,217]
+  CRUSH rule 0 x 370 [11,467,695,989]
+  CRUSH rule 0 x 371 [124,487,55,514]
+  CRUSH rule 0 x 372 [253,48,979,846]
+  CRUSH rule 0 x 373 [715,605,775,748]
+  CRUSH rule 0 x 374 [191,887,920,928]
+  CRUSH rule 0 x 375 [711,385,651,665]
+  CRUSH rule 0 x 376 [597,818,49,458]
+  CRUSH rule 0 x 377 [294,256,933,771]
+  CRUSH rule 0 x 378 [34,151,681,707]
+  CRUSH rule 0 x 379 [869,136,315,378]
+  CRUSH rule 0 x 380 [294,97,575,791]
+  CRUSH rule 0 x 381 [119,710,219,827]
+  CRUSH rule 0 x 382 [69,631,508,706]
+  CRUSH rule 0 x 383 [922,588,589,925]
+  CRUSH rule 0 x 384 [221,945,671,117]
+  CRUSH rule 0 x 385 [561,737,953,723]
+  CRUSH rule 0 x 386 [335,442,788,696]
+  CRUSH rule 0 x 387 [514,43,353,88]
+  CRUSH rule 0 x 388 [587,89,157,996]
+  CRUSH rule 0 x 389 [109,641,255,466]
+  CRUSH rule 0 x 390 [925,149,421,489]
+  CRUSH rule 0 x 391 [267,87,387,527]
+  CRUSH rule 0 x 392 [382,485,370,849]
+  CRUSH rule 0 x 393 [425,721,221,753]
+  CRUSH rule 0 x 394 [898,18,38,793]
+  CRUSH rule 0 x 395 [806,876,269,679]
+  CRUSH rule 0 x 396 [790,970,437,449]
+  CRUSH rule 0 x 397 [136,363,507,613]
+  CRUSH rule 0 x 398 [914,116,558,258]
+  CRUSH rule 0 x 399 [261,94,299,202]
+  CRUSH rule 0 x 400 [661,197,338,461]
+  CRUSH rule 0 x 401 [953,979,287,803]
+  CRUSH rule 0 x 402 [738,819,618,522]
+  CRUSH rule 0 x 403 [573,238,425,546]
+  CRUSH rule 0 x 404 [526,848,790,253]
+  CRUSH rule 0 x 405 [582,505,330,334]
+  CRUSH rule 0 x 406 [768,324,493,60]
+  CRUSH rule 0 x 407 [260,951,437,587]
+  CRUSH rule 0 x 408 [657,81,770,734]
+  CRUSH rule 0 x 409 [498,89,182,423]
+  CRUSH rule 0 x 410 [28,793,737,352]
+  CRUSH rule 0 x 411 [684,992,60,659]
+  CRUSH rule 0 x 412 [261,958,699,950]
+  CRUSH rule 0 x 413 [891,835,297,441]
+  CRUSH rule 0 x 414 [127,459,119,965]
+  CRUSH rule 0 x 415 [272,540,631,328]
+  CRUSH rule 0 x 416 [739,617,115,530]
+  CRUSH rule 0 x 417 [106,209,157,878]
+  CRUSH rule 0 x 418 [525,441,147,390]
+  CRUSH rule 0 x 419 [603,673,615,465]
+  CRUSH rule 0 x 420 [988,213,251,226]
+  CRUSH rule 0 x 421 [761,521,748,368]
+  CRUSH rule 0 x 422 [317,160,924,548]
+  CRUSH rule 0 x 423 [137,807,168,472]
+  CRUSH rule 0 x 424 [920,37,146,263]
+  CRUSH rule 0 x 425 [277,693,285,221]
+  CRUSH rule 0 x 426 [485,936,407,854]
+  CRUSH rule 0 x 427 [242,515,9,564]
+  CRUSH rule 0 x 428 [632,635,26,473]
+  CRUSH rule 0 x 429 [641,73,465,127]
+  CRUSH rule 0 x 430 [626,585,6,387]
+  CRUSH rule 0 x 431 [697,76,753,570]
+  CRUSH rule 0 x 432 [590,526,306,283]
+  CRUSH rule 0 x 433 [284,387,149,817]
+  CRUSH rule 0 x 434 [538,985,79,953]
+  CRUSH rule 0 x 435 [30,318,593,635]
+  CRUSH rule 0 x 436 [164,919,851,693]
+  CRUSH rule 0 x 437 [322,212,163,606]
+  CRUSH rule 0 x 438 [142,392,85,594]
+  CRUSH rule 0 x 439 [119,370,68,443]
+  CRUSH rule 0 x 440 [333,403,187,863]
+  CRUSH rule 0 x 441 [477,727,906,145]
+  CRUSH rule 0 x 442 [274,590,933,244]
+  CRUSH rule 0 x 443 [983,748,574,718]
+  CRUSH rule 0 x 444 [536,509,431,146]
+  CRUSH rule 0 x 445 [485,482,528,209]
+  CRUSH rule 0 x 446 [345,634,42,294]
+  CRUSH rule 0 x 447 [61,845,767,600]
+  CRUSH rule 0 x 448 [333,232,292,846]
+  CRUSH rule 0 x 449 [680,16,484,670]
+  CRUSH rule 0 x 450 [235,214,79,423]
+  CRUSH rule 0 x 451 [961,468,333,640]
+  CRUSH rule 0 x 452 [525,479,153,528]
+  CRUSH rule 0 x 453 [138,466,302,86]
+  CRUSH rule 0 x 454 [137,625,215,402]
+  CRUSH rule 0 x 455 [173,150,997,16]
+  CRUSH rule 0 x 456 [235,226,238,258]
+  CRUSH rule 0 x 457 [450,577,253,413]
+  CRUSH rule 0 x 458 [195,537,91,814]
+  CRUSH rule 0 x 459 [381,555,312,573]
+  CRUSH rule 0 x 460 [972,730,534,678]
+  CRUSH rule 0 x 461 [506,279,142,830]
+  CRUSH rule 0 x 462 [692,959,578,57]
+  CRUSH rule 0 x 463 [788,667,949,550]
+  CRUSH rule 0 x 464 [133,122,588,999]
+  CRUSH rule 0 x 465 [971,190,230,777]
+  CRUSH rule 0 x 466 [394,576,148,157]
+  CRUSH rule 0 x 467 [517,28,366,362]
+  CRUSH rule 0 x 468 [829,143,874,225]
+  CRUSH rule 0 x 469 [987,936,106,725]
+  CRUSH rule 0 x 470 [107,982,56,889]
+  CRUSH rule 0 x 471 [181,897,629,860]
+  CRUSH rule 0 x 472 [547,512,172,24]
+  CRUSH rule 0 x 473 [760,997,824,905]
+  CRUSH rule 0 x 474 [787,418,743,628]
+  CRUSH rule 0 x 475 [662,312,253,617]
+  CRUSH rule 0 x 476 [110,495,185,508]
+  CRUSH rule 0 x 477 [393,954,834,132]
+  CRUSH rule 0 x 478 [246,483,480,644]
+  CRUSH rule 0 x 479 [70,929,697,931]
+  CRUSH rule 0 x 480 [753,119,961,607]
+  CRUSH rule 0 x 481 [470,429,677,242]
+  CRUSH rule 0 x 482 [451,566,961,675]
+  CRUSH rule 0 x 483 [816,72,371,278]
+  CRUSH rule 0 x 484 [540,454,389,31]
+  CRUSH rule 0 x 485 [74,582,624,684]
+  CRUSH rule 0 x 486 [958,595,199,763]
+  CRUSH rule 0 x 487 [228,302,804,833]
+  CRUSH rule 0 x 488 [180,529,722,956]
+  CRUSH rule 0 x 489 [47,617,812,187]
+  CRUSH rule 0 x 490 [905,822,479,124]
+  CRUSH rule 0 x 491 [892,370,609,998]
+  CRUSH rule 0 x 492 [588,959,127,948]
+  CRUSH rule 0 x 493 [353,461,593,291]
+  CRUSH rule 0 x 494 [378,848,443,368]
+  CRUSH rule 0 x 495 [845,653,768,234]
+  CRUSH rule 0 x 496 [13,988,0,691]
+  CRUSH rule 0 x 497 [796,877,788,394]
+  CRUSH rule 0 x 498 [412,337,270,705]
+  CRUSH rule 0 x 499 [330,695,8,74]
+  CRUSH rule 0 x 500 [820,272,547,765]
+  CRUSH rule 0 x 501 [110,44,132,442]
+  CRUSH rule 0 x 502 [336,595,650,274]
+  CRUSH rule 0 x 503 [922,211,157,722]
+  CRUSH rule 0 x 504 [483,52,122,432]
+  CRUSH rule 0 x 505 [482,598,224,279]
+  CRUSH rule 0 x 506 [493,123,43,856]
+  CRUSH rule 0 x 507 [12,598,264,422]
+  CRUSH rule 0 x 508 [227,157,611,301]
+  CRUSH rule 0 x 509 [807,242,363,122]
+  CRUSH rule 0 x 510 [134,437,227,75]
+  CRUSH rule 0 x 511 [212,54,83,799]
+  CRUSH rule 0 x 512 [236,630,758,752]
+  CRUSH rule 0 x 513 [994,693,644,938]
+  CRUSH rule 0 x 514 [45,508,831,19]
+  CRUSH rule 0 x 515 [504,138,480,272]
+  CRUSH rule 0 x 516 [285,409,136,570]
+  CRUSH rule 0 x 517 [300,232,23,906]
+  CRUSH rule 0 x 518 [397,674,98,898]
+  CRUSH rule 0 x 519 [86,750,772,913]
+  CRUSH rule 0 x 520 [900,833,614,130]
+  CRUSH rule 0 x 521 [31,47,236,751]
+  CRUSH rule 0 x 522 [390,16,280,144]
+  CRUSH rule 0 x 523 [618,308,424,590]
+  CRUSH rule 0 x 524 [635,189,687,963]
+  CRUSH rule 0 x 525 [311,916,699,262]
+  CRUSH rule 0 x 526 [48,738,227,718]
+  CRUSH rule 0 x 527 [202,851,889,216]
+  CRUSH rule 0 x 528 [565,827,590,273]
+  CRUSH rule 0 x 529 [934,864,241,43]
+  CRUSH rule 0 x 530 [502,934,298,670]
+  CRUSH rule 0 x 531 [681,627,942,487]
+  CRUSH rule 0 x 532 [422,6,147,205]
+  CRUSH rule 0 x 533 [863,68,364,983]
+  CRUSH rule 0 x 534 [962,931,775,172]
+  CRUSH rule 0 x 535 [89,565,397,693]
+  CRUSH rule 0 x 536 [499,351,760,458]
+  CRUSH rule 0 x 537 [676,547,787,311]
+  CRUSH rule 0 x 538 [58,644,571,649]
+  CRUSH rule 0 x 539 [837,953,457,711]
+  CRUSH rule 0 x 540 [831,50,132,213]
+  CRUSH rule 0 x 541 [582,757,121,525]
+  CRUSH rule 0 x 542 [472,132,790,997]
+  CRUSH rule 0 x 543 [382,272,797,330]
+  CRUSH rule 0 x 544 [947,930,496,883]
+  CRUSH rule 0 x 545 [425,570,305,77]
+  CRUSH rule 0 x 546 [18,65,529,437]
+  CRUSH rule 0 x 547 [445,715,600,472]
+  CRUSH rule 0 x 548 [367,569,980,167]
+  CRUSH rule 0 x 549 [125,715,671,817]
+  CRUSH rule 0 x 550 [425,599,744,199]
+  CRUSH rule 0 x 551 [44,1,528,922]
+  CRUSH rule 0 x 552 [246,104,68,239]
+  CRUSH rule 0 x 553 [71,703,615,28]
+  CRUSH rule 0 x 554 [207,124,217,166]
+  CRUSH rule 0 x 555 [570,28,317,420]
+  CRUSH rule 0 x 556 [674,152,421,79]
+  CRUSH rule 0 x 557 [347,817,191,391]
+  CRUSH rule 0 x 558 [627,426,369,692]
+  CRUSH rule 0 x 559 [940,630,924,242]
+  CRUSH rule 0 x 560 [295,903,541,29]
+  CRUSH rule 0 x 561 [506,682,384,637]
+  CRUSH rule 0 x 562 [718,529,87,729]
+  CRUSH rule 0 x 563 [552,332,747,206]
+  CRUSH rule 0 x 564 [835,769,736,486]
+  CRUSH rule 0 x 565 [8,167,539,182]
+  CRUSH rule 0 x 566 [600,481,301,263]
+  CRUSH rule 0 x 567 [999,994,509,899]
+  CRUSH rule 0 x 568 [252,431,157,62]
+  CRUSH rule 0 x 569 [643,218,943,455]
+  CRUSH rule 0 x 570 [617,635,765,422]
+  CRUSH rule 0 x 571 [757,80,59,98]
+  CRUSH rule 0 x 572 [299,348,575,889]
+  CRUSH rule 0 x 573 [25,505,270,167]
+  CRUSH rule 0 x 574 [215,431,624,177]
+  CRUSH rule 0 x 575 [225,252,611,546]
+  CRUSH rule 0 x 576 [627,94,159,857]
+  CRUSH rule 0 x 577 [237,809,778,636]
+  CRUSH rule 0 x 578 [885,313,120,344]
+  CRUSH rule 0 x 579 [924,575,787,831]
+  CRUSH rule 0 x 580 [718,51,766,121]
+  CRUSH rule 0 x 581 [219,807,129,571]
+  CRUSH rule 0 x 582 [893,701,598,863]
+  CRUSH rule 0 x 583 [246,930,964,170]
+  CRUSH rule 0 x 584 [336,432,680,175]
+  CRUSH rule 0 x 585 [324,999,397,485]
+  CRUSH rule 0 x 586 [558,230,976,541]
+  CRUSH rule 0 x 587 [985,830,597,21]
+  CRUSH rule 0 x 588 [211,544,57,134]
+  CRUSH rule 0 x 589 [129,21,112,190]
+  CRUSH rule 0 x 590 [467,969,652,593]
+  CRUSH rule 0 x 591 [758,514,316,164]
+  CRUSH rule 0 x 592 [525,253,190,443]
+  CRUSH rule 0 x 593 [601,885,339,152]
+  CRUSH rule 0 x 594 [227,60,450,30]
+  CRUSH rule 0 x 595 [720,854,496,912]
+  CRUSH rule 0 x 596 [751,195,997,77]
+  CRUSH rule 0 x 597 [129,574,714,8]
+  CRUSH rule 0 x 598 [679,207,604,396]
+  CRUSH rule 0 x 599 [668,315,683,349]
+  CRUSH rule 0 x 600 [143,396,464,444]
+  CRUSH rule 0 x 601 [326,573,873,902]
+  CRUSH rule 0 x 602 [860,281,875,535]
+  CRUSH rule 0 x 603 [709,328,445,349]
+  CRUSH rule 0 x 604 [571,62,814,95]
+  CRUSH rule 0 x 605 [252,739,860,27]
+  CRUSH rule 0 x 606 [339,236,759,842]
+  CRUSH rule 0 x 607 [590,248,759,868]
+  CRUSH rule 0 x 608 [145,635,309,467]
+  CRUSH rule 0 x 609 [973,547,223,79]
+  CRUSH rule 0 x 610 [435,816,961,983]
+  CRUSH rule 0 x 611 [559,283,422,584]
+  CRUSH rule 0 x 612 [273,149,123,576]
+  CRUSH rule 0 x 613 [828,614,642,674]
+  CRUSH rule 0 x 614 [478,748,393,34]
+  CRUSH rule 0 x 615 [392,155,144,326]
+  CRUSH rule 0 x 616 [778,637,452,248]
+  CRUSH rule 0 x 617 [622,713,996,833]
+  CRUSH rule 0 x 618 [149,877,270,329]
+  CRUSH rule 0 x 619 [604,163,656,409]
+  CRUSH rule 0 x 620 [181,23,409,198]
+  CRUSH rule 0 x 621 [735,902,386,237]
+  CRUSH rule 0 x 622 [661,824,717,568]
+  CRUSH rule 0 x 623 [142,121,643,61]
+  CRUSH rule 0 x 624 [360,716,420,398]
+  CRUSH rule 0 x 625 [541,167,385,1]
+  CRUSH rule 0 x 626 [364,431,610,363]
+  CRUSH rule 0 x 627 [458,137,557,410]
+  CRUSH rule 0 x 628 [250,350,556,497]
+  CRUSH rule 0 x 629 [928,160,710,572]
+  CRUSH rule 0 x 630 [243,19,918,556]
+  CRUSH rule 0 x 631 [438,221,574,676]
+  CRUSH rule 0 x 632 [797,368,247,5]
+  CRUSH rule 0 x 633 [993,749,525,485]
+  CRUSH rule 0 x 634 [239,351,633,299]
+  CRUSH rule 0 x 635 [640,965,25,961]
+  CRUSH rule 0 x 636 [173,290,297,991]
+  CRUSH rule 0 x 637 [0,918,98,108]
+  CRUSH rule 0 x 638 [702,235,424,900]
+  CRUSH rule 0 x 639 [475,687,31,785]
+  CRUSH rule 0 x 640 [31,664,399,677]
+  CRUSH rule 0 x 641 [296,473,108,963]
+  CRUSH rule 0 x 642 [894,273,427,606]
+  CRUSH rule 0 x 643 [117,111,732,191]
+  CRUSH rule 0 x 644 [438,336,327,512]
+  CRUSH rule 0 x 645 [982,702,351,573]
+  CRUSH rule 0 x 646 [334,804,146,842]
+  CRUSH rule 0 x 647 [933,787,185,334]
+  CRUSH rule 0 x 648 [22,444,400,862]
+  CRUSH rule 0 x 649 [503,229,213,460]
+  CRUSH rule 0 x 650 [328,659,420,443]
+  CRUSH rule 0 x 651 [3,880,823,123]
+  CRUSH rule 0 x 652 [495,977,563,733]
+  CRUSH rule 0 x 653 [185,718,804,280]
+  CRUSH rule 0 x 654 [130,528,380,81]
+  CRUSH rule 0 x 655 [560,872,454,504]
+  CRUSH rule 0 x 656 [219,885,178,981]
+  CRUSH rule 0 x 657 [233,684,813,490]
+  CRUSH rule 0 x 658 [778,6,756,380]
+  CRUSH rule 0 x 659 [240,663,306,540]
+  CRUSH rule 0 x 660 [244,855,196,147]
+  CRUSH rule 0 x 661 [184,270,128,398]
+  CRUSH rule 0 x 662 [65,883,921,438]
+  CRUSH rule 0 x 663 [323,721,594,812]
+  CRUSH rule 0 x 664 [865,113,512,51]
+  CRUSH rule 0 x 665 [420,850,591,475]
+  CRUSH rule 0 x 666 [319,767,246,3]
+  CRUSH rule 0 x 667 [875,39,343,100]
+  CRUSH rule 0 x 668 [331,122,263,599]
+  CRUSH rule 0 x 669 [915,521,402,747]
+  CRUSH rule 0 x 670 [845,659,943,447]
+  CRUSH rule 0 x 671 [108,634,527,363]
+  CRUSH rule 0 x 672 [578,216,110,589]
+  CRUSH rule 0 x 673 [442,74,579,797]
+  CRUSH rule 0 x 674 [588,364,281,308]
+  CRUSH rule 0 x 675 [489,698,744,671]
+  CRUSH rule 0 x 676 [928,911,40,180]
+  CRUSH rule 0 x 677 [399,269,692,131]
+  CRUSH rule 0 x 678 [546,752,544,155]
+  CRUSH rule 0 x 679 [988,25,275,433]
+  CRUSH rule 0 x 680 [335,963,382,486]
+  CRUSH rule 0 x 681 [690,462,623,466]
+  CRUSH rule 0 x 682 [196,588,154,257]
+  CRUSH rule 0 x 683 [627,25,421,160]
+  CRUSH rule 0 x 684 [38,804,592,158]
+  CRUSH rule 0 x 685 [841,368,548,362]
+  CRUSH rule 0 x 686 [336,287,525,440]
+  CRUSH rule 0 x 687 [20,682,924,653]
+  CRUSH rule 0 x 688 [463,371,780,556]
+  CRUSH rule 0 x 689 [569,250,78,816]
+  CRUSH rule 0 x 690 [551,144,587,263]
+  CRUSH rule 0 x 691 [766,464,446,533]
+  CRUSH rule 0 x 692 [739,634,18,245]
+  CRUSH rule 0 x 693 [339,297,118,330]
+  CRUSH rule 0 x 694 [405,26,830,181]
+  CRUSH rule 0 x 695 [622,576,597,535]
+  CRUSH rule 0 x 696 [558,902,689,13]
+  CRUSH rule 0 x 697 [818,222,406,691]
+  CRUSH rule 0 x 698 [178,48,402,233]
+  CRUSH rule 0 x 699 [450,244,180,919]
+  CRUSH rule 0 x 700 [502,771,987,706]
+  CRUSH rule 0 x 701 [4,612,782,216]
+  CRUSH rule 0 x 702 [177,630,232,923]
+  CRUSH rule 0 x 703 [354,178,389,393]
+  CRUSH rule 0 x 704 [646,601,156,171]
+  CRUSH rule 0 x 705 [921,401,890,265]
+  CRUSH rule 0 x 706 [652,877,562,452]
+  CRUSH rule 0 x 707 [345,745,67,716]
+  CRUSH rule 0 x 708 [333,607,180,469]
+  CRUSH rule 0 x 709 [45,187,302,115]
+  CRUSH rule 0 x 710 [94,855,43,199]
+  CRUSH rule 0 x 711 [227,653,731,150]
+  CRUSH rule 0 x 712 [398,953,136,870]
+  CRUSH rule 0 x 713 [116,800,503,662]
+  CRUSH rule 0 x 714 [111,629,866,709]
+  CRUSH rule 0 x 715 [531,291,486,382]
+  CRUSH rule 0 x 716 [169,541,291,42]
+  CRUSH rule 0 x 717 [417,446,994,894]
+  CRUSH rule 0 x 718 [992,383,298,844]
+  CRUSH rule 0 x 719 [936,674,324,759]
+  CRUSH rule 0 x 720 [370,188,174,464]
+  CRUSH rule 0 x 721 [320,859,278,259]
+  CRUSH rule 0 x 722 [7,2,673,129]
+  CRUSH rule 0 x 723 [270,553,831,662]
+  CRUSH rule 0 x 724 [666,822,708,895]
+  CRUSH rule 0 x 725 [794,406,875,459]
+  CRUSH rule 0 x 726 [420,556,341,292]
+  CRUSH rule 0 x 727 [561,461,129,635]
+  CRUSH rule 0 x 728 [951,330,196,756]
+  CRUSH rule 0 x 729 [656,644,436,591]
+  CRUSH rule 0 x 730 [3,558,629,184]
+  CRUSH rule 0 x 731 [852,89,75,735]
+  CRUSH rule 0 x 732 [983,840,869,976]
+  CRUSH rule 0 x 733 [285,396,388,122]
+  CRUSH rule 0 x 734 [125,510,402,640]
+  CRUSH rule 0 x 735 [417,773,686,504]
+  CRUSH rule 0 x 736 [749,396,632,550]
+  CRUSH rule 0 x 737 [644,991,946,135]
+  CRUSH rule 0 x 738 [449,683,290,220]
+  CRUSH rule 0 x 739 [341,220,641,454]
+  CRUSH rule 0 x 740 [874,524,674,650]
+  CRUSH rule 0 x 741 [189,472,712,798]
+  CRUSH rule 0 x 742 [912,581,114,117]
+  CRUSH rule 0 x 743 [654,914,425,441]
+  CRUSH rule 0 x 744 [725,295,579,377]
+  CRUSH rule 0 x 745 [787,858,850,506]
+  CRUSH rule 0 x 746 [757,848,704,30]
+  CRUSH rule 0 x 747 [700,81,867,681]
+  CRUSH rule 0 x 748 [557,436,238,664]
+  CRUSH rule 0 x 749 [772,622,337,42]
+  CRUSH rule 0 x 750 [946,97,376,677]
+  CRUSH rule 0 x 751 [996,618,343,911]
+  CRUSH rule 0 x 752 [746,887,695,868]
+  CRUSH rule 0 x 753 [741,14,463,479]
+  CRUSH rule 0 x 754 [648,349,333,355]
+  CRUSH rule 0 x 755 [157,460,466,187]
+  CRUSH rule 0 x 756 [416,97,197,497]
+  CRUSH rule 0 x 757 [599,839,776,410]
+  CRUSH rule 0 x 758 [994,218,620,256]
+  CRUSH rule 0 x 759 [959,682,514,745]
+  CRUSH rule 0 x 760 [518,943,215,83]
+  CRUSH rule 0 x 761 [285,849,420,324]
+  CRUSH rule 0 x 762 [591,313,41,335]
+  CRUSH rule 0 x 763 [908,411,200,740]
+  CRUSH rule 0 x 764 [787,234,894,485]
+  CRUSH rule 0 x 765 [327,921,882,393]
+  CRUSH rule 0 x 766 [84,161,878,704]
+  CRUSH rule 0 x 767 [370,895,702,701]
+  CRUSH rule 0 x 768 [826,760,879,864]
+  CRUSH rule 0 x 769 [67,768,663,735]
+  CRUSH rule 0 x 770 [593,909,482,259]
+  CRUSH rule 0 x 771 [309,935,121,578]
+  CRUSH rule 0 x 772 [12,125,797,301]
+  CRUSH rule 0 x 773 [253,466,820,549]
+  CRUSH rule 0 x 774 [164,390,705,109]
+  CRUSH rule 0 x 775 [703,47,43,973]
+  CRUSH rule 0 x 776 [728,231,80,916]
+  CRUSH rule 0 x 777 [981,621,568,729]
+  CRUSH rule 0 x 778 [411,456,544,597]
+  CRUSH rule 0 x 779 [346,121,519,921]
+  CRUSH rule 0 x 780 [476,39,288,381]
+  CRUSH rule 0 x 781 [10,130,585,844]
+  CRUSH rule 0 x 782 [462,246,581,902]
+  CRUSH rule 0 x 783 [580,373,153,775]
+  CRUSH rule 0 x 784 [413,113,978,990]
+  CRUSH rule 0 x 785 [341,856,332,354]
+  CRUSH rule 0 x 786 [411,140,313,393]
+  CRUSH rule 0 x 787 [605,522,211,813]
+  CRUSH rule 0 x 788 [226,545,35,142]
+  CRUSH rule 0 x 789 [545,320,414,702]
+  CRUSH rule 0 x 790 [414,748,816,327]
+  CRUSH rule 0 x 791 [660,906,406,697]
+  CRUSH rule 0 x 792 [287,392,514,204]
+  CRUSH rule 0 x 793 [631,133,850,713]
+  CRUSH rule 0 x 794 [931,517,543,210]
+  CRUSH rule 0 x 795 [551,962,477,948]
+  CRUSH rule 0 x 796 [814,4,95,27]
+  CRUSH rule 0 x 797 [64,201,299,734]
+  CRUSH rule 0 x 798 [422,530,114,431]
+  CRUSH rule 0 x 799 [824,32,679,562]
+  CRUSH rule 0 x 800 [862,623,489,637]
+  CRUSH rule 0 x 801 [145,550,329,324]
+  CRUSH rule 0 x 802 [570,19,847,308]
+  CRUSH rule 0 x 803 [151,812,662,358]
+  CRUSH rule 0 x 804 [467,93,264,863]
+  CRUSH rule 0 x 805 [621,223,938,809]
+  CRUSH rule 0 x 806 [898,957,805,430]
+  CRUSH rule 0 x 807 [354,531,422,159]
+  CRUSH rule 0 x 808 [7,96,76,897]
+  CRUSH rule 0 x 809 [70,734,719,56]
+  CRUSH rule 0 x 810 [701,18,972,327]
+  CRUSH rule 0 x 811 [248,547,103,728]
+  CRUSH rule 0 x 812 [230,576,821,566]
+  CRUSH rule 0 x 813 [805,114,683,629]
+  CRUSH rule 0 x 814 [54,619,973,741]
+  CRUSH rule 0 x 815 [679,412,613,132]
+  CRUSH rule 0 x 816 [919,448,826,414]
+  CRUSH rule 0 x 817 [765,830,436,521]
+  CRUSH rule 0 x 818 [415,566,644,687]
+  CRUSH rule 0 x 819 [721,319,865,750]
+  CRUSH rule 0 x 820 [218,301,333,190]
+  CRUSH rule 0 x 821 [185,795,680,953]
+  CRUSH rule 0 x 822 [356,261,54,522]
+  CRUSH rule 0 x 823 [220,281,549,456]
+  CRUSH rule 0 x 824 [292,809,887,74]
+  CRUSH rule 0 x 825 [949,778,101,311]
+  CRUSH rule 0 x 826 [767,818,833,927]
+  CRUSH rule 0 x 827 [631,83,406,635]
+  CRUSH rule 0 x 828 [288,986,445,26]
+  CRUSH rule 0 x 829 [990,667,915,694]
+  CRUSH rule 0 x 830 [152,571,778,505]
+  CRUSH rule 0 x 831 [814,563,630,97]
+  CRUSH rule 0 x 832 [235,641,616,110]
+  CRUSH rule 0 x 833 [657,565,922,140]
+  CRUSH rule 0 x 834 [907,231,644,13]
+  CRUSH rule 0 x 835 [784,262,771,264]
+  CRUSH rule 0 x 836 [951,158,366,710]
+  CRUSH rule 0 x 837 [556,498,334,633]
+  CRUSH rule 0 x 838 [329,274,964,547]
+  CRUSH rule 0 x 839 [568,209,939,364]
+  CRUSH rule 0 x 840 [45,579,842,70]
+  CRUSH rule 0 x 841 [652,702,24,605]
+  CRUSH rule 0 x 842 [629,984,314,895]
+  CRUSH rule 0 x 843 [799,690,688,648]
+  CRUSH rule 0 x 844 [694,600,534,700]
+  CRUSH rule 0 x 845 [332,30,179,93]
+  CRUSH rule 0 x 846 [452,251,712,719]
+  CRUSH rule 0 x 847 [399,681,847,739]
+  CRUSH rule 0 x 848 [303,138,440,346]
+  CRUSH rule 0 x 849 [666,346,708,873]
+  CRUSH rule 0 x 850 [644,511,345,844]
+  CRUSH rule 0 x 851 [527,546,737,425]
+  CRUSH rule 0 x 852 [31,809,94,618]
+  CRUSH rule 0 x 853 [483,330,869,184]
+  CRUSH rule 0 x 854 [697,953,968,143]
+  CRUSH rule 0 x 855 [837,996,239,621]
+  CRUSH rule 0 x 856 [712,40,547,430]
+  CRUSH rule 0 x 857 [77,984,576,551]
+  CRUSH rule 0 x 858 [412,384,841,465]
+  CRUSH rule 0 x 859 [173,760,26,300]
+  CRUSH rule 0 x 860 [776,429,328,917]
+  CRUSH rule 0 x 861 [705,405,477,50]
+  CRUSH rule 0 x 862 [809,44,788,938]
+  CRUSH rule 0 x 863 [349,496,963,178]
+  CRUSH rule 0 x 864 [717,858,101,239]
+  CRUSH rule 0 x 865 [857,603,586,262]
+  CRUSH rule 0 x 866 [394,304,71,96]
+  CRUSH rule 0 x 867 [640,773,663,974]
+  CRUSH rule 0 x 868 [613,950,712,663]
+  CRUSH rule 0 x 869 [973,889,524,22]
+  CRUSH rule 0 x 870 [505,35,386,498]
+  CRUSH rule 0 x 871 [239,264,262,773]
+  CRUSH rule 0 x 872 [21,767,456,748]
+  CRUSH rule 0 x 873 [954,666,980,264]
+  CRUSH rule 0 x 874 [54,510,947,1]
+  CRUSH rule 0 x 875 [809,418,452,462]
+  CRUSH rule 0 x 876 [483,457,61,248]
+  CRUSH rule 0 x 877 [542,531,952,939]
+  CRUSH rule 0 x 878 [217,674,857,644]
+  CRUSH rule 0 x 879 [999,475,134,250]
+  CRUSH rule 0 x 880 [678,573,935,385]
+  CRUSH rule 0 x 881 [394,835,789,802]
+  CRUSH rule 0 x 882 [467,382,353,56]
+  CRUSH rule 0 x 883 [802,744,237,337]
+  CRUSH rule 0 x 884 [653,660,638,700]
+  CRUSH rule 0 x 885 [898,704,307,445]
+  CRUSH rule 0 x 886 [434,357,938,641]
+  CRUSH rule 0 x 887 [297,226,711,428]
+  CRUSH rule 0 x 888 [863,324,443,213]
+  CRUSH rule 0 x 889 [105,102,308,163]
+  CRUSH rule 0 x 890 [550,248,606,704]
+  CRUSH rule 0 x 891 [575,928,880,891]
+  CRUSH rule 0 x 892 [259,862,133,271]
+  CRUSH rule 0 x 893 [902,880,543,542]
+  CRUSH rule 0 x 894 [180,169,916,43]
+  CRUSH rule 0 x 895 [725,849,182,129]
+  CRUSH rule 0 x 896 [951,34,874,537]
+  CRUSH rule 0 x 897 [810,352,73,939]
+  CRUSH rule 0 x 898 [979,433,719,411]
+  CRUSH rule 0 x 899 [685,668,534,932]
+  CRUSH rule 0 x 900 [530,978,41,894]
+  CRUSH rule 0 x 901 [740,107,336,175]
+  CRUSH rule 0 x 902 [800,743,693,310]
+  CRUSH rule 0 x 903 [230,267,842,266]
+  CRUSH rule 0 x 904 [346,949,460,973]
+  CRUSH rule 0 x 905 [530,397,619,958]
+  CRUSH rule 0 x 906 [80,426,138,672]
+  CRUSH rule 0 x 907 [365,968,475,297]
+  CRUSH rule 0 x 908 [204,832,742,809]
+  CRUSH rule 0 x 909 [883,989,146,959]
+  CRUSH rule 0 x 910 [549,593,249,853]
+  CRUSH rule 0 x 911 [325,847,352,214]
+  CRUSH rule 0 x 912 [874,888,582,796]
+  CRUSH rule 0 x 913 [331,463,342,574]
+  CRUSH rule 0 x 914 [836,468,601,732]
+  CRUSH rule 0 x 915 [245,228,100,661]
+  CRUSH rule 0 x 916 [77,967,364,435]
+  CRUSH rule 0 x 917 [239,60,866,221]
+  CRUSH rule 0 x 918 [988,115,922,80]
+  CRUSH rule 0 x 919 [783,139,696,1]
+  CRUSH rule 0 x 920 [623,408,685,953]
+  CRUSH rule 0 x 921 [105,799,144,90]
+  CRUSH rule 0 x 922 [887,505,652,348]
+  CRUSH rule 0 x 923 [223,318,552,458]
+  CRUSH rule 0 x 924 [25,778,366,333]
+  CRUSH rule 0 x 925 [912,601,297,682]
+  CRUSH rule 0 x 926 [968,133,132,144]
+  CRUSH rule 0 x 927 [277,724,214,988]
+  CRUSH rule 0 x 928 [554,203,658,789]
+  CRUSH rule 0 x 929 [761,802,367,528]
+  CRUSH rule 0 x 930 [814,61,788,736]
+  CRUSH rule 0 x 931 [29,193,61,41]
+  CRUSH rule 0 x 932 [446,198,862,534]
+  CRUSH rule 0 x 933 [352,742,216,321]
+  CRUSH rule 0 x 934 [730,2,332,631]
+  CRUSH rule 0 x 935 [731,23,736,79]
+  CRUSH rule 0 x 936 [322,975,20,904]
+  CRUSH rule 0 x 937 [822,221,841,161]
+  CRUSH rule 0 x 938 [557,850,66,630]
+  CRUSH rule 0 x 939 [150,11,971,371]
+  CRUSH rule 0 x 940 [638,398,169,616]
+  CRUSH rule 0 x 941 [730,342,929,577]
+  CRUSH rule 0 x 942 [62,292,166,814]
+  CRUSH rule 0 x 943 [165,314,519,548]
+  CRUSH rule 0 x 944 [199,625,766,176]
+  CRUSH rule 0 x 945 [946,999,699,303]
+  CRUSH rule 0 x 946 [595,93,852,142]
+  CRUSH rule 0 x 947 [800,582,356,93]
+  CRUSH rule 0 x 948 [132,551,139,920]
+  CRUSH rule 0 x 949 [792,920,466,380]
+  CRUSH rule 0 x 950 [111,345,176,543]
+  CRUSH rule 0 x 951 [414,619,648,655]
+  CRUSH rule 0 x 952 [775,469,500,356]
+  CRUSH rule 0 x 953 [349,1,5,251]
+  CRUSH rule 0 x 954 [570,940,410,249]
+  CRUSH rule 0 x 955 [729,774,823,800]
+  CRUSH rule 0 x 956 [519,141,575,625]
+  CRUSH rule 0 x 957 [242,709,611,97]
+  CRUSH rule 0 x 958 [84,217,227,253]
+  CRUSH rule 0 x 959 [270,413,918,789]
+  CRUSH rule 0 x 960 [458,192,307,279]
+  CRUSH rule 0 x 961 [981,388,777,546]
+  CRUSH rule 0 x 962 [623,834,277,134]
+  CRUSH rule 0 x 963 [291,167,714,468]
+  CRUSH rule 0 x 964 [28,156,788,127]
+  CRUSH rule 0 x 965 [675,557,290,517]
+  CRUSH rule 0 x 966 [836,306,946,283]
+  CRUSH rule 0 x 967 [966,386,735,837]
+  CRUSH rule 0 x 968 [864,756,690,121]
+  CRUSH rule 0 x 969 [729,625,480,769]
+  CRUSH rule 0 x 970 [800,362,646,582]
+  CRUSH rule 0 x 971 [737,381,153,684]
+  CRUSH rule 0 x 972 [952,245,720,884]
+  CRUSH rule 0 x 973 [356,455,579,857]
+  CRUSH rule 0 x 974 [545,758,586,596]
+  CRUSH rule 0 x 975 [336,191,202,146]
+  CRUSH rule 0 x 976 [446,208,757,620]
+  CRUSH rule 0 x 977 [202,896,196,956]
+  CRUSH rule 0 x 978 [612,324,996,225]
+  CRUSH rule 0 x 979 [843,457,675,650]
+  CRUSH rule 0 x 980 [60,914,881,626]
+  CRUSH rule 0 x 981 [702,749,937,153]
+  CRUSH rule 0 x 982 [298,928,738,167]
+  CRUSH rule 0 x 983 [723,572,395,358]
+  CRUSH rule 0 x 984 [723,864,804,935]
+  CRUSH rule 0 x 985 [945,459,868,211]
+  CRUSH rule 0 x 986 [772,664,535,169]
+  CRUSH rule 0 x 987 [88,324,312,843]
+  CRUSH rule 0 x 988 [522,927,131,996]
+  CRUSH rule 0 x 989 [578,332,208,605]
+  CRUSH rule 0 x 990 [638,228,414,311]
+  CRUSH rule 0 x 991 [530,221,451,422]
+  CRUSH rule 0 x 992 [925,705,275,81]
+  CRUSH rule 0 x 993 [991,301,43,469]
+  CRUSH rule 0 x 994 [276,51,868,683]
+  CRUSH rule 0 x 995 [288,836,753,790]
+  CRUSH rule 0 x 996 [887,983,252,686]
+  CRUSH rule 0 x 997 [110,924,386,79]
+  CRUSH rule 0 x 998 [435,830,485,853]
+  CRUSH rule 0 x 999 [876,738,357,913]
+  CRUSH rule 0 x 1000 [178,963,638,430]
+  CRUSH rule 0 x 1001 [99,519,66,759]
+  CRUSH rule 0 x 1002 [515,534,468,866]
+  CRUSH rule 0 x 1003 [104,611,937,698]
+  CRUSH rule 0 x 1004 [269,638,724,375]
+  CRUSH rule 0 x 1005 [369,223,309,409]
+  CRUSH rule 0 x 1006 [40,107,69,275]
+  CRUSH rule 0 x 1007 [978,111,416,758]
+  CRUSH rule 0 x 1008 [965,956,624,832]
+  CRUSH rule 0 x 1009 [598,476,356,695]
+  CRUSH rule 0 x 1010 [767,523,239,517]
+  CRUSH rule 0 x 1011 [289,871,207,576]
+  CRUSH rule 0 x 1012 [128,28,370,31]
+  CRUSH rule 0 x 1013 [979,765,660,812]
+  CRUSH rule 0 x 1014 [979,948,513,88]
+  CRUSH rule 0 x 1015 [277,790,396,672]
+  CRUSH rule 0 x 1016 [262,73,128,886]
+  CRUSH rule 0 x 1017 [150,269,61,499]
+  CRUSH rule 0 x 1018 [555,829,554,944]
+  CRUSH rule 0 x 1019 [513,356,265,446]
+  CRUSH rule 0 x 1020 [158,161,877,704]
+  CRUSH rule 0 x 1021 [915,998,957,285]
+  CRUSH rule 0 x 1022 [967,829,973,640]
+  CRUSH rule 0 x 1023 [488,257,614,859]
+  rule 0 (data) num_rep 4 result size == 4:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604]
+  CRUSH rule 0 x 1 [876,250,334,633,744]
+  CRUSH rule 0 x 2 [292,832,53,392,386]
+  CRUSH rule 0 x 3 [623,387,124,998,749]
+  CRUSH rule 0 x 4 [61,334,710,4,994]
+  CRUSH rule 0 x 5 [946,557,713,664,141]
+  CRUSH rule 0 x 6 [576,668,212,163,732]
+  CRUSH rule 0 x 7 [645,753,906,393,341]
+  CRUSH rule 0 x 8 [243,6,863,781,211]
+  CRUSH rule 0 x 9 [22,578,251,410,297]
+  CRUSH rule 0 x 10 [758,828,360,477,821]
+  CRUSH rule 0 x 11 [769,120,124,527,119]
+  CRUSH rule 0 x 12 [780,364,689,755,675]
+  CRUSH rule 0 x 13 [557,18,351,719,742]
+  CRUSH rule 0 x 14 [59,561,249,461,971]
+  CRUSH rule 0 x 15 [718,928,993,21,76]
+  CRUSH rule 0 x 16 [673,632,841,954,788]
+  CRUSH rule 0 x 17 [648,43,560,514,142]
+  CRUSH rule 0 x 18 [654,219,181,568,381]
+  CRUSH rule 0 x 19 [850,545,377,848,863]
+  CRUSH rule 0 x 20 [717,785,974,5,225]
+  CRUSH rule 0 x 21 [420,57,519,306,312]
+  CRUSH rule 0 x 22 [503,998,193,821,634]
+  CRUSH rule 0 x 23 [411,663,168,110,899]
+  CRUSH rule 0 x 24 [266,861,353,1,456]
+  CRUSH rule 0 x 25 [760,483,818,600,509]
+  CRUSH rule 0 x 26 [903,24,573,718,112]
+  CRUSH rule 0 x 27 [946,188,289,510,687]
+  CRUSH rule 0 x 28 [69,312,73,198,256]
+  CRUSH rule 0 x 29 [844,883,337,628,496]
+  CRUSH rule 0 x 30 [621,18,613,794,910]
+  CRUSH rule 0 x 31 [784,943,814,539,962]
+  CRUSH rule 0 x 32 [173,374,369,972,315]
+  CRUSH rule 0 x 33 [698,336,357,966,582]
+  CRUSH rule 0 x 34 [168,836,210,798,904]
+  CRUSH rule 0 x 35 [274,509,534,818,912]
+  CRUSH rule 0 x 36 [318,215,153,628,87]
+  CRUSH rule 0 x 37 [173,604,109,935,203]
+  CRUSH rule 0 x 38 [708,444,683,604,722]
+  CRUSH rule 0 x 39 [662,198,417,680,226]
+  CRUSH rule 0 x 40 [620,801,414,78,560]
+  CRUSH rule 0 x 41 [811,264,177,127,148]
+  CRUSH rule 0 x 42 [863,179,527,660,133]
+  CRUSH rule 0 x 43 [686,822,988,228,791]
+  CRUSH rule 0 x 44 [396,222,46,841,536]
+  CRUSH rule 0 x 45 [991,694,253,142,54]
+  CRUSH rule 0 x 46 [420,909,184,285,508]
+  CRUSH rule 0 x 47 [467,211,605,207,241]
+  CRUSH rule 0 x 48 [955,329,368,168,698]
+  CRUSH rule 0 x 49 [974,891,931,29,813]
+  CRUSH rule 0 x 50 [870,441,691,823,761]
+  CRUSH rule 0 x 51 [182,930,25,936,97]
+  CRUSH rule 0 x 52 [704,812,894,794,481]
+  CRUSH rule 0 x 53 [185,713,631,280,345]
+  CRUSH rule 0 x 54 [270,441,100,82,983]
+  CRUSH rule 0 x 55 [895,734,958,793,651]
+  CRUSH rule 0 x 56 [564,963,683,324,40]
+  CRUSH rule 0 x 57 [738,130,208,973,498]
+  CRUSH rule 0 x 58 [524,113,806,903,531]
+  CRUSH rule 0 x 59 [408,337,668,529,34]
+  CRUSH rule 0 x 60 [228,790,857,309,616]
+  CRUSH rule 0 x 61 [154,843,717,467,883]
+  CRUSH rule 0 x 62 [594,811,549,276,693]
+  CRUSH rule 0 x 63 [646,67,884,925,941]
+  CRUSH rule 0 x 64 [175,542,155,837,594]
+  CRUSH rule 0 x 65 [745,619,131,867,269]
+  CRUSH rule 0 x 66 [275,468,23,35,328]
+  CRUSH rule 0 x 67 [246,958,524,493,636]
+  CRUSH rule 0 x 68 [711,473,403,228,835]
+  CRUSH rule 0 x 69 [493,924,850,939,950]
+  CRUSH rule 0 x 70 [30,499,644,33,804]
+  CRUSH rule 0 x 71 [984,883,574,716,575]
+  CRUSH rule 0 x 72 [71,286,942,363,628]
+  CRUSH rule 0 x 73 [922,618,3,371,464]
+  CRUSH rule 0 x 74 [629,414,185,573,678]
+  CRUSH rule 0 x 75 [222,20,174,820,312]
+  CRUSH rule 0 x 76 [262,366,339,290,718]
+  CRUSH rule 0 x 77 [638,469,992,280,773]
+  CRUSH rule 0 x 78 [324,511,788,7,308]
+  CRUSH rule 0 x 79 [577,990,64,94,447]
+  CRUSH rule 0 x 80 [501,95,278,903,631]
+  CRUSH rule 0 x 81 [506,812,9,698,173]
+  CRUSH rule 0 x 82 [222,145,80,785,835]
+  CRUSH rule 0 x 83 [71,634,61,91,856]
+  CRUSH rule 0 x 84 [49,761,773,368,318]
+  CRUSH rule 0 x 85 [985,896,708,861,325]
+  CRUSH rule 0 x 86 [537,745,93,524,466]
+  CRUSH rule 0 x 87 [997,317,463,626,685]
+  CRUSH rule 0 x 88 [957,350,890,857,375]
+  CRUSH rule 0 x 89 [399,730,148,314,159]
+  CRUSH rule 0 x 90 [943,706,683,267,579]
+  CRUSH rule 0 x 91 [22,368,149,928,140]
+  CRUSH rule 0 x 92 [532,424,426,773,623]
+  CRUSH rule 0 x 93 [218,489,405,681,549]
+  CRUSH rule 0 x 94 [181,96,102,515,776]
+  CRUSH rule 0 x 95 [343,957,820,139,334]
+  CRUSH rule 0 x 96 [861,270,87,797,0]
+  CRUSH rule 0 x 97 [459,706,45,328,274]
+  CRUSH rule 0 x 98 [327,867,353,948,728]
+  CRUSH rule 0 x 99 [974,133,468,906,235]
+  CRUSH rule 0 x 100 [32,445,547,371,960]
+  CRUSH rule 0 x 101 [142,90,337,950,970]
+  CRUSH rule 0 x 102 [172,129,139,22,403]
+  CRUSH rule 0 x 103 [630,47,161,356,911]
+  CRUSH rule 0 x 104 [758,133,278,11,947]
+  CRUSH rule 0 x 105 [843,604,47,33,401]
+  CRUSH rule 0 x 106 [28,681,193,679,990]
+  CRUSH rule 0 x 107 [74,320,85,819,315]
+  CRUSH rule 0 x 108 [875,593,575,517,107]
+  CRUSH rule 0 x 109 [411,985,811,720,198]
+  CRUSH rule 0 x 110 [440,774,799,660,715]
+  CRUSH rule 0 x 111 [405,742,276,359,936]
+  CRUSH rule 0 x 112 [143,181,922,545,185]
+  CRUSH rule 0 x 113 [153,846,160,903,789]
+  CRUSH rule 0 x 114 [804,892,939,20,312]
+  CRUSH rule 0 x 115 [588,508,958,580,232]
+  CRUSH rule 0 x 116 [327,148,637,486,712]
+  CRUSH rule 0 x 117 [95,594,989,131,714]
+  CRUSH rule 0 x 118 [80,957,897,239,359]
+  CRUSH rule 0 x 119 [386,932,951,768,679]
+  CRUSH rule 0 x 120 [366,312,653,936,71]
+  CRUSH rule 0 x 121 [129,154,847,16,471]
+  CRUSH rule 0 x 122 [873,1,110,939,90]
+  CRUSH rule 0 x 123 [533,415,789,600,713]
+  CRUSH rule 0 x 124 [461,691,898,723,957]
+  CRUSH rule 0 x 125 [342,599,830,402,615]
+  CRUSH rule 0 x 126 [819,781,822,548,279]
+  CRUSH rule 0 x 127 [437,893,585,707,353]
+  CRUSH rule 0 x 128 [679,994,982,550,991]
+  CRUSH rule 0 x 129 [380,685,947,302,698]
+  CRUSH rule 0 x 130 [992,52,466,867,998]
+  CRUSH rule 0 x 131 [469,90,208,599,829]
+  CRUSH rule 0 x 132 [571,250,316,535,54]
+  CRUSH rule 0 x 133 [964,728,329,902,108]
+  CRUSH rule 0 x 134 [999,19,716,963,323]
+  CRUSH rule 0 x 135 [634,101,52,938,413]
+  CRUSH rule 0 x 136 [114,889,692,768,694]
+  CRUSH rule 0 x 137 [839,8,959,280,922]
+  CRUSH rule 0 x 138 [967,949,138,451,292]
+  CRUSH rule 0 x 139 [308,711,736,247,632]
+  CRUSH rule 0 x 140 [764,936,926,55,331]
+  CRUSH rule 0 x 141 [423,302,112,216,603]
+  CRUSH rule 0 x 142 [252,821,715,340,635]
+  CRUSH rule 0 x 143 [33,808,518,477,325]
+  CRUSH rule 0 x 144 [472,88,969,162,401]
+  CRUSH rule 0 x 145 [242,208,252,604,266]
+  CRUSH rule 0 x 146 [290,70,570,384,934]
+  CRUSH rule 0 x 147 [447,352,657,493,467]
+  CRUSH rule 0 x 148 [212,644,432,658,109]
+  CRUSH rule 0 x 149 [9,775,87,35,260]
+  CRUSH rule 0 x 150 [166,456,582,144,324]
+  CRUSH rule 0 x 151 [811,875,307,20,782]
+  CRUSH rule 0 x 152 [449,617,223,9,182]
+  CRUSH rule 0 x 153 [523,537,695,627,959]
+  CRUSH rule 0 x 154 [208,559,874,597,243]
+  CRUSH rule 0 x 155 [569,325,192,296,367]
+  CRUSH rule 0 x 156 [488,121,521,213,595]
+  CRUSH rule 0 x 157 [140,723,633,260,487]
+  CRUSH rule 0 x 158 [786,451,320,239,667]
+  CRUSH rule 0 x 159 [134,664,517,821,667]
+  CRUSH rule 0 x 160 [690,112,414,990,183]
+  CRUSH rule 0 x 161 [324,912,397,423,991]
+  CRUSH rule 0 x 162 [748,567,284,183,463]
+  CRUSH rule 0 x 163 [575,499,31,816,749]
+  CRUSH rule 0 x 164 [314,489,308,326,51]
+  CRUSH rule 0 x 165 [116,209,750,53,813]
+  CRUSH rule 0 x 166 [352,706,701,810,718]
+  CRUSH rule 0 x 167 [27,743,174,142,551]
+  CRUSH rule 0 x 168 [953,898,880,660,500]
+  CRUSH rule 0 x 169 [912,147,266,547,331]
+  CRUSH rule 0 x 170 [421,515,828,844,151]
+  CRUSH rule 0 x 171 [488,584,880,964,936]
+  CRUSH rule 0 x 172 [366,443,957,66,162]
+  CRUSH rule 0 x 173 [863,291,625,287,158]
+  CRUSH rule 0 x 174 [263,555,650,410,339]
+  CRUSH rule 0 x 175 [875,961,361,575,33]
+  CRUSH rule 0 x 176 [745,83,701,680,250]
+  CRUSH rule 0 x 177 [128,244,41,123,422]
+  CRUSH rule 0 x 178 [155,41,264,777,314]
+  CRUSH rule 0 x 179 [593,833,202,183,971]
+  CRUSH rule 0 x 180 [154,734,17,831,824]
+  CRUSH rule 0 x 181 [289,675,723,800,166]
+  CRUSH rule 0 x 182 [730,931,560,209,943]
+  CRUSH rule 0 x 183 [639,237,794,815,827]
+  CRUSH rule 0 x 184 [704,312,685,645,691]
+  CRUSH rule 0 x 185 [97,100,762,82,999]
+  CRUSH rule 0 x 186 [26,665,554,215,280]
+  CRUSH rule 0 x 187 [649,14,740,494,402]
+  CRUSH rule 0 x 188 [682,695,590,743,927]
+  CRUSH rule 0 x 189 [325,693,726,51,448]
+  CRUSH rule 0 x 190 [399,933,136,955,57]
+  CRUSH rule 0 x 191 [629,533,17,126,60]
+  CRUSH rule 0 x 192 [503,578,38,492,222]
+  CRUSH rule 0 x 193 [546,333,651,678,823]
+  CRUSH rule 0 x 194 [242,473,58,655,653]
+  CRUSH rule 0 x 195 [625,719,135,81,636]
+  CRUSH rule 0 x 196 [357,114,125,867,250]
+  CRUSH rule 0 x 197 [306,954,453,873,211]
+  CRUSH rule 0 x 198 [863,791,311,911,206]
+  CRUSH rule 0 x 199 [935,906,929,252,893]
+  CRUSH rule 0 x 200 [373,774,229,454,909]
+  CRUSH rule 0 x 201 [659,320,477,313,779]
+  CRUSH rule 0 x 202 [260,433,524,880,223]
+  CRUSH rule 0 x 203 [36,239,675,971,703]
+  CRUSH rule 0 x 204 [92,516,993,728,279]
+  CRUSH rule 0 x 205 [68,395,473,45,683]
+  CRUSH rule 0 x 206 [570,530,642,380,311]
+  CRUSH rule 0 x 207 [834,457,850,917,456]
+  CRUSH rule 0 x 208 [927,484,640,976,803]
+  CRUSH rule 0 x 209 [878,66,58,940,48]
+  CRUSH rule 0 x 210 [572,981,484,29,0]
+  CRUSH rule 0 x 211 [107,597,780,857,895]
+  CRUSH rule 0 x 212 [389,107,838,624,698]
+  CRUSH rule 0 x 213 [497,717,567,728,905]
+  CRUSH rule 0 x 214 [798,65,254,572,32]
+  CRUSH rule 0 x 215 [233,419,283,638,520]
+  CRUSH rule 0 x 216 [494,464,742,523,459]
+  CRUSH rule 0 x 217 [352,396,309,938,66]
+  CRUSH rule 0 x 218 [895,864,988,650,593]
+  CRUSH rule 0 x 219 [222,534,277,242,658]
+  CRUSH rule 0 x 220 [281,19,584,563,858]
+  CRUSH rule 0 x 221 [64,928,963,130,312]
+  CRUSH rule 0 x 222 [40,544,161,199,861]
+  CRUSH rule 0 x 223 [645,556,159,417,46]
+  CRUSH rule 0 x 224 [647,165,957,263,961]
+  CRUSH rule 0 x 225 [219,714,858,747,461]
+  CRUSH rule 0 x 226 [372,511,181,277,695]
+  CRUSH rule 0 x 227 [925,156,714,863,257]
+  CRUSH rule 0 x 228 [682,404,839,263,521]
+  CRUSH rule 0 x 229 [880,838,770,891,236]
+  CRUSH rule 0 x 230 [328,659,916,468,646]
+  CRUSH rule 0 x 231 [320,383,669,109,627]
+  CRUSH rule 0 x 232 [924,846,394,319,43]
+  CRUSH rule 0 x 233 [948,652,575,838,498]
+  CRUSH rule 0 x 234 [484,943,42,575,936]
+  CRUSH rule 0 x 235 [750,65,590,168,870]
+  CRUSH rule 0 x 236 [551,787,490,136,370]
+  CRUSH rule 0 x 237 [390,157,166,251,752]
+  CRUSH rule 0 x 238 [570,6,989,707,514]
+  CRUSH rule 0 x 239 [729,959,376,975,496]
+  CRUSH rule 0 x 240 [981,241,156,767,631]
+  CRUSH rule 0 x 241 [310,816,641,177,996]
+  CRUSH rule 0 x 242 [161,63,642,837,763]
+  CRUSH rule 0 x 243 [180,394,33,683,189]
+  CRUSH rule 0 x 244 [52,174,685,189,78]
+  CRUSH rule 0 x 245 [523,121,915,84,386]
+  CRUSH rule 0 x 246 [362,893,390,487,817]
+  CRUSH rule 0 x 247 [382,184,116,34,143]
+  CRUSH rule 0 x 248 [129,114,852,469,359]
+  CRUSH rule 0 x 249 [159,683,91,856,475]
+  CRUSH rule 0 x 250 [404,945,569,955,228]
+  CRUSH rule 0 x 251 [661,225,738,757,37]
+  CRUSH rule 0 x 252 [961,226,542,103,945]
+  CRUSH rule 0 x 253 [651,97,225,364,189]
+  CRUSH rule 0 x 254 [123,33,741,692,599]
+  CRUSH rule 0 x 255 [314,649,891,855,517]
+  CRUSH rule 0 x 256 [315,215,651,126,470]
+  CRUSH rule 0 x 257 [825,264,867,869,529]
+  CRUSH rule 0 x 258 [624,789,370,723,131]
+  CRUSH rule 0 x 259 [602,542,70,563,947]
+  CRUSH rule 0 x 260 [717,878,43,56,377]
+  CRUSH rule 0 x 261 [145,517,20,903,786]
+  CRUSH rule 0 x 262 [223,1,561,420,229]
+  CRUSH rule 0 x 263 [462,211,405,508,787]
+  CRUSH rule 0 x 264 [654,471,266,662,135]
+  CRUSH rule 0 x 265 [302,794,704,798,659]
+  CRUSH rule 0 x 266 [202,132,884,209,551]
+  CRUSH rule 0 x 267 [282,938,657,113,672]
+  CRUSH rule 0 x 268 [338,309,356,278,928]
+  CRUSH rule 0 x 269 [738,122,266,200,894]
+  CRUSH rule 0 x 270 [707,982,946,196,407]
+  CRUSH rule 0 x 271 [705,432,364,735,512]
+  CRUSH rule 0 x 272 [756,545,942,56,542]
+  CRUSH rule 0 x 273 [197,502,527,721,239]
+  CRUSH rule 0 x 274 [992,44,653,573,527]
+  CRUSH rule 0 x 275 [544,789,170,434,23]
+  CRUSH rule 0 x 276 [658,467,577,268,336]
+  CRUSH rule 0 x 277 [143,490,880,483,928]
+  CRUSH rule 0 x 278 [492,647,355,282,834]
+  CRUSH rule 0 x 279 [517,792,604,987,527]
+  CRUSH rule 0 x 280 [825,740,27,848,514]
+  CRUSH rule 0 x 281 [224,629,120,562,616]
+  CRUSH rule 0 x 282 [298,661,380,416,35]
+  CRUSH rule 0 x 283 [311,606,208,50,913]
+  CRUSH rule 0 x 284 [771,466,371,743,672]
+  CRUSH rule 0 x 285 [693,362,404,676,797]
+  CRUSH rule 0 x 286 [364,477,285,167,270]
+  CRUSH rule 0 x 287 [591,611,828,995,170]
+  CRUSH rule 0 x 288 [965,541,848,796,251]
+  CRUSH rule 0 x 289 [225,551,948,877,219]
+  CRUSH rule 0 x 290 [577,762,777,751,291]
+  CRUSH rule 0 x 291 [160,903,477,381,490]
+  CRUSH rule 0 x 292 [873,598,216,666,222]
+  CRUSH rule 0 x 293 [100,234,874,47,28]
+  CRUSH rule 0 x 294 [285,943,379,520,725]
+  CRUSH rule 0 x 295 [938,262,880,327,687]
+  CRUSH rule 0 x 296 [850,327,86,472,1]
+  CRUSH rule 0 x 297 [951,53,99,558,753]
+  CRUSH rule 0 x 298 [173,336,85,766,910]
+  CRUSH rule 0 x 299 [598,591,315,386,895]
+  CRUSH rule 0 x 300 [531,957,62,459,156]
+  CRUSH rule 0 x 301 [823,628,23,858,629]
+  CRUSH rule 0 x 302 [184,80,780,871,531]
+  CRUSH rule 0 x 303 [521,766,222,830,988]
+  CRUSH rule 0 x 304 [980,127,807,507,555]
+  CRUSH rule 0 x 305 [153,816,22,927,696]
+  CRUSH rule 0 x 306 [423,739,664,753,178]
+  CRUSH rule 0 x 307 [997,557,682,456,479]
+  CRUSH rule 0 x 308 [991,874,534,465,330]
+  CRUSH rule 0 x 309 [860,394,724,858,246]
+  CRUSH rule 0 x 310 [589,818,546,201,94]
+  CRUSH rule 0 x 311 [477,774,225,590,830]
+  CRUSH rule 0 x 312 [887,853,950,354,58]
+  CRUSH rule 0 x 313 [802,646,447,416,557]
+  CRUSH rule 0 x 314 [654,974,229,511,562]
+  CRUSH rule 0 x 315 [767,227,28,740,828]
+  CRUSH rule 0 x 316 [778,83,733,359,858]
+  CRUSH rule 0 x 317 [184,418,642,986,939]
+  CRUSH rule 0 x 318 [525,410,500,543,212]
+  CRUSH rule 0 x 319 [476,724,569,382,409]
+  CRUSH rule 0 x 320 [149,610,697,296,818]
+  CRUSH rule 0 x 321 [710,79,667,671,234]
+  CRUSH rule 0 x 322 [175,275,323,333,744]
+  CRUSH rule 0 x 323 [819,604,638,792,316]
+  CRUSH rule 0 x 324 [16,745,511,439,272]
+  CRUSH rule 0 x 325 [486,400,872,873,251]
+  CRUSH rule 0 x 326 [613,765,207,19,359]
+  CRUSH rule 0 x 327 [125,289,738,408,456]
+  CRUSH rule 0 x 328 [807,383,476,583,645]
+  CRUSH rule 0 x 329 [588,938,599,432,446]
+  CRUSH rule 0 x 330 [932,644,41,611,209]
+  CRUSH rule 0 x 331 [341,953,950,537,578]
+  CRUSH rule 0 x 332 [153,726,459,950,466]
+  CRUSH rule 0 x 333 [745,845,853,860,52]
+  CRUSH rule 0 x 334 [614,751,807,58,396]
+  CRUSH rule 0 x 335 [518,721,221,283,454]
+  CRUSH rule 0 x 336 [389,424,77,309,5]
+  CRUSH rule 0 x 337 [753,508,765,720,221]
+  CRUSH rule 0 x 338 [128,810,490,753,406]
+  CRUSH rule 0 x 339 [430,308,58,751,856]
+  CRUSH rule 0 x 340 [541,44,630,231,289]
+  CRUSH rule 0 x 341 [402,26,631,439,165]
+  CRUSH rule 0 x 342 [982,57,992,461,131]
+  CRUSH rule 0 x 343 [833,412,572,732,107]
+  CRUSH rule 0 x 344 [784,533,792,41,642]
+  CRUSH rule 0 x 345 [546,300,304,691,763]
+  CRUSH rule 0 x 346 [302,420,428,891,357]
+  CRUSH rule 0 x 347 [488,778,101,217,366]
+  CRUSH rule 0 x 348 [903,744,937,718,85]
+  CRUSH rule 0 x 349 [471,547,582,306,600]
+  CRUSH rule 0 x 350 [348,221,823,335,383]
+  CRUSH rule 0 x 351 [961,582,705,346,361]
+  CRUSH rule 0 x 352 [728,137,461,298,36]
+  CRUSH rule 0 x 353 [904,202,184,447,58]
+  CRUSH rule 0 x 354 [345,226,319,256,544]
+  CRUSH rule 0 x 355 [50,430,175,43,187]
+  CRUSH rule 0 x 356 [87,185,55,423,829]
+  CRUSH rule 0 x 357 [762,459,921,473,182]
+  CRUSH rule 0 x 358 [908,25,280,6,808]
+  CRUSH rule 0 x 359 [484,15,132,121,394]
+  CRUSH rule 0 x 360 [173,378,337,702,145]
+  CRUSH rule 0 x 361 [404,577,115,25,56]
+  CRUSH rule 0 x 362 [403,1,422,945,132]
+  CRUSH rule 0 x 363 [639,911,510,162,418]
+  CRUSH rule 0 x 364 [752,689,610,990,665]
+  CRUSH rule 0 x 365 [956,999,212,230,624]
+  CRUSH rule 0 x 366 [860,925,924,763,687]
+  CRUSH rule 0 x 367 [205,609,647,665,969]
+  CRUSH rule 0 x 368 [301,284,810,169,78]
+  CRUSH rule 0 x 369 [452,658,339,217,674]
+  CRUSH rule 0 x 370 [11,467,695,989,394]
+  CRUSH rule 0 x 371 [124,487,55,514,313]
+  CRUSH rule 0 x 372 [253,48,979,846,207]
+  CRUSH rule 0 x 373 [715,605,775,748,227]
+  CRUSH rule 0 x 374 [191,887,920,928,223]
+  CRUSH rule 0 x 375 [711,385,651,665,15]
+  CRUSH rule 0 x 376 [597,818,49,458,415]
+  CRUSH rule 0 x 377 [294,256,933,771,184]
+  CRUSH rule 0 x 378 [34,151,681,707,552]
+  CRUSH rule 0 x 379 [869,136,315,378,813]
+  CRUSH rule 0 x 380 [294,97,575,791,690]
+  CRUSH rule 0 x 381 [119,710,219,827,328]
+  CRUSH rule 0 x 382 [69,631,508,706,697]
+  CRUSH rule 0 x 383 [922,588,589,925,471]
+  CRUSH rule 0 x 384 [221,945,671,117,857]
+  CRUSH rule 0 x 385 [561,737,953,723,658]
+  CRUSH rule 0 x 386 [335,442,788,696,507]
+  CRUSH rule 0 x 387 [514,43,353,88,100]
+  CRUSH rule 0 x 388 [587,89,157,996,915]
+  CRUSH rule 0 x 389 [109,641,255,466,372]
+  CRUSH rule 0 x 390 [925,149,421,489,599]
+  CRUSH rule 0 x 391 [267,87,387,527,768]
+  CRUSH rule 0 x 392 [382,485,370,849,936]
+  CRUSH rule 0 x 393 [425,721,221,753,268]
+  CRUSH rule 0 x 394 [898,18,38,793,173]
+  CRUSH rule 0 x 395 [806,876,269,679,32]
+  CRUSH rule 0 x 396 [790,970,437,449,875]
+  CRUSH rule 0 x 397 [136,363,507,613,11]
+  CRUSH rule 0 x 398 [914,116,558,258,722]
+  CRUSH rule 0 x 399 [261,94,299,202,174]
+  CRUSH rule 0 x 400 [661,197,338,461,977]
+  CRUSH rule 0 x 401 [953,979,287,803,41]
+  CRUSH rule 0 x 402 [738,819,618,522,667]
+  CRUSH rule 0 x 403 [573,238,425,546,130]
+  CRUSH rule 0 x 404 [526,848,790,253,922]
+  CRUSH rule 0 x 405 [582,505,330,334,201]
+  CRUSH rule 0 x 406 [768,324,493,60,186]
+  CRUSH rule 0 x 407 [260,951,437,587,692]
+  CRUSH rule 0 x 408 [657,81,770,734,830]
+  CRUSH rule 0 x 409 [498,89,182,423,672]
+  CRUSH rule 0 x 410 [28,793,737,352,166]
+  CRUSH rule 0 x 411 [684,992,60,659,769]
+  CRUSH rule 0 x 412 [261,958,699,950,165]
+  CRUSH rule 0 x 413 [891,835,297,441,384]
+  CRUSH rule 0 x 414 [127,459,119,965,662]
+  CRUSH rule 0 x 415 [272,540,631,328,609]
+  CRUSH rule 0 x 416 [739,617,115,530,339]
+  CRUSH rule 0 x 417 [106,209,157,878,117]
+  CRUSH rule 0 x 418 [525,441,147,390,320]
+  CRUSH rule 0 x 419 [603,673,615,465,266]
+  CRUSH rule 0 x 420 [988,213,251,226,209]
+  CRUSH rule 0 x 421 [761,521,748,368,923]
+  CRUSH rule 0 x 422 [317,160,924,548,198]
+  CRUSH rule 0 x 423 [137,807,168,472,619]
+  CRUSH rule 0 x 424 [920,37,146,263,598]
+  CRUSH rule 0 x 425 [277,693,285,221,478]
+  CRUSH rule 0 x 426 [485,936,407,854,726]
+  CRUSH rule 0 x 427 [242,515,9,564,174]
+  CRUSH rule 0 x 428 [632,635,26,473,494]
+  CRUSH rule 0 x 429 [641,73,465,127,171]
+  CRUSH rule 0 x 430 [626,585,6,387,881]
+  CRUSH rule 0 x 431 [697,76,753,570,964]
+  CRUSH rule 0 x 432 [590,526,306,283,656]
+  CRUSH rule 0 x 433 [284,387,149,817,886]
+  CRUSH rule 0 x 434 [538,985,79,953,770]
+  CRUSH rule 0 x 435 [30,318,593,635,975]
+  CRUSH rule 0 x 436 [164,919,851,693,0]
+  CRUSH rule 0 x 437 [322,212,163,606,302]
+  CRUSH rule 0 x 438 [142,392,85,594,376]
+  CRUSH rule 0 x 439 [119,370,68,443,997]
+  CRUSH rule 0 x 440 [333,403,187,863,475]
+  CRUSH rule 0 x 441 [477,727,906,145,429]
+  CRUSH rule 0 x 442 [274,590,933,244,434]
+  CRUSH rule 0 x 443 [983,748,574,718,700]
+  CRUSH rule 0 x 444 [536,509,431,146,170]
+  CRUSH rule 0 x 445 [485,482,528,209,964]
+  CRUSH rule 0 x 446 [345,634,42,294,711]
+  CRUSH rule 0 x 447 [61,845,767,600,321]
+  CRUSH rule 0 x 448 [333,232,292,846,364]
+  CRUSH rule 0 x 449 [680,16,484,670,851]
+  CRUSH rule 0 x 450 [235,214,79,423,96]
+  CRUSH rule 0 x 451 [961,468,333,640,823]
+  CRUSH rule 0 x 452 [525,479,153,528,570]
+  CRUSH rule 0 x 453 [138,466,302,86,249]
+  CRUSH rule 0 x 454 [137,625,215,402,389]
+  CRUSH rule 0 x 455 [173,150,997,16,846]
+  CRUSH rule 0 x 456 [235,226,238,258,347]
+  CRUSH rule 0 x 457 [450,577,253,413,717]
+  CRUSH rule 0 x 458 [195,537,91,814,351]
+  CRUSH rule 0 x 459 [381,555,312,573,915]
+  CRUSH rule 0 x 460 [972,730,534,678,756]
+  CRUSH rule 0 x 461 [506,279,142,830,784]
+  CRUSH rule 0 x 462 [692,959,578,57,983]
+  CRUSH rule 0 x 463 [788,667,949,550,685]
+  CRUSH rule 0 x 464 [133,122,588,999,270]
+  CRUSH rule 0 x 465 [971,190,230,777,452]
+  CRUSH rule 0 x 466 [394,576,148,157,103]
+  CRUSH rule 0 x 467 [517,28,366,362,984]
+  CRUSH rule 0 x 468 [829,143,874,225,162]
+  CRUSH rule 0 x 469 [987,936,106,725,633]
+  CRUSH rule 0 x 470 [107,982,56,889,67]
+  CRUSH rule 0 x 471 [181,897,629,860,307]
+  CRUSH rule 0 x 472 [547,512,172,24,705]
+  CRUSH rule 0 x 473 [760,997,824,905,888]
+  CRUSH rule 0 x 474 [787,418,743,628,272]
+  CRUSH rule 0 x 475 [662,312,253,617,105]
+  CRUSH rule 0 x 476 [110,495,185,508,961]
+  CRUSH rule 0 x 477 [393,954,834,132,841]
+  CRUSH rule 0 x 478 [246,483,480,644,985]
+  CRUSH rule 0 x 479 [70,929,697,931,744]
+  CRUSH rule 0 x 480 [753,119,961,607,317]
+  CRUSH rule 0 x 481 [470,429,677,242,574]
+  CRUSH rule 0 x 482 [451,566,961,675,354]
+  CRUSH rule 0 x 483 [816,72,371,278,635]
+  CRUSH rule 0 x 484 [540,454,389,31,654]
+  CRUSH rule 0 x 485 [74,582,624,684,566]
+  CRUSH rule 0 x 486 [958,595,199,763,715]
+  CRUSH rule 0 x 487 [228,302,804,833,876]
+  CRUSH rule 0 x 488 [180,529,722,956,353]
+  CRUSH rule 0 x 489 [47,617,812,187,291]
+  CRUSH rule 0 x 490 [905,822,479,124,750]
+  CRUSH rule 0 x 491 [892,370,609,998,433]
+  CRUSH rule 0 x 492 [588,959,127,948,505]
+  CRUSH rule 0 x 493 [353,461,593,291,301]
+  CRUSH rule 0 x 494 [378,848,443,368,507]
+  CRUSH rule 0 x 495 [845,653,768,234,405]
+  CRUSH rule 0 x 496 [13,988,0,691,389]
+  CRUSH rule 0 x 497 [796,877,788,394,648]
+  CRUSH rule 0 x 498 [412,337,270,705,511]
+  CRUSH rule 0 x 499 [330,695,8,74,618]
+  CRUSH rule 0 x 500 [820,272,547,765,755]
+  CRUSH rule 0 x 501 [110,44,132,442,294]
+  CRUSH rule 0 x 502 [336,595,650,274,993]
+  CRUSH rule 0 x 503 [922,211,157,722,502]
+  CRUSH rule 0 x 504 [483,52,122,432,778]
+  CRUSH rule 0 x 505 [482,598,224,279,480]
+  CRUSH rule 0 x 506 [493,123,43,856,936]
+  CRUSH rule 0 x 507 [12,598,264,422,416]
+  CRUSH rule 0 x 508 [227,157,611,301,223]
+  CRUSH rule 0 x 509 [807,242,363,122,582]
+  CRUSH rule 0 x 510 [134,437,227,75,313]
+  CRUSH rule 0 x 511 [212,54,83,799,457]
+  CRUSH rule 0 x 512 [236,630,758,752,361]
+  CRUSH rule 0 x 513 [994,693,644,938,846]
+  CRUSH rule 0 x 514 [45,508,831,19,817]
+  CRUSH rule 0 x 515 [504,138,480,272,530]
+  CRUSH rule 0 x 516 [285,409,136,570,841]
+  CRUSH rule 0 x 517 [300,232,23,906,438]
+  CRUSH rule 0 x 518 [397,674,98,898,967]
+  CRUSH rule 0 x 519 [86,750,772,913,101]
+  CRUSH rule 0 x 520 [900,833,614,130,261]
+  CRUSH rule 0 x 521 [31,47,236,751,911]
+  CRUSH rule 0 x 522 [390,16,280,144,291]
+  CRUSH rule 0 x 523 [618,308,424,590,300]
+  CRUSH rule 0 x 524 [635,189,687,963,601]
+  CRUSH rule 0 x 525 [311,916,699,262,775]
+  CRUSH rule 0 x 526 [48,738,227,718,244]
+  CRUSH rule 0 x 527 [202,851,889,216,763]
+  CRUSH rule 0 x 528 [565,827,590,273,918]
+  CRUSH rule 0 x 529 [934,864,241,43,466]
+  CRUSH rule 0 x 530 [502,934,298,670,986]
+  CRUSH rule 0 x 531 [681,627,942,487,288]
+  CRUSH rule 0 x 532 [422,6,147,205,861]
+  CRUSH rule 0 x 533 [863,68,364,983,247]
+  CRUSH rule 0 x 534 [962,931,775,172,663]
+  CRUSH rule 0 x 535 [89,565,397,693,839]
+  CRUSH rule 0 x 536 [499,351,760,458,918]
+  CRUSH rule 0 x 537 [676,547,787,311,867]
+  CRUSH rule 0 x 538 [58,644,571,649,941]
+  CRUSH rule 0 x 539 [837,953,457,711,458]
+  CRUSH rule 0 x 540 [831,50,132,213,197]
+  CRUSH rule 0 x 541 [582,757,121,525,532]
+  CRUSH rule 0 x 542 [472,132,790,997,948]
+  CRUSH rule 0 x 543 [382,272,797,330,315]
+  CRUSH rule 0 x 544 [947,930,496,883,509]
+  CRUSH rule 0 x 545 [425,570,305,77,821]
+  CRUSH rule 0 x 546 [18,65,529,437,343]
+  CRUSH rule 0 x 547 [445,715,600,472,213]
+  CRUSH rule 0 x 548 [367,569,980,167,627]
+  CRUSH rule 0 x 549 [125,715,671,817,285]
+  CRUSH rule 0 x 550 [425,599,744,199,923]
+  CRUSH rule 0 x 551 [44,1,528,922,944]
+  CRUSH rule 0 x 552 [246,104,68,239,123]
+  CRUSH rule 0 x 553 [71,703,615,28,593]
+  CRUSH rule 0 x 554 [207,124,217,166,525]
+  CRUSH rule 0 x 555 [570,28,317,420,931]
+  CRUSH rule 0 x 556 [674,152,421,79,215]
+  CRUSH rule 0 x 557 [347,817,191,391,741]
+  CRUSH rule 0 x 558 [627,426,369,692,815]
+  CRUSH rule 0 x 559 [940,630,924,242,224]
+  CRUSH rule 0 x 560 [295,903,541,29,245]
+  CRUSH rule 0 x 561 [506,682,384,637,878]
+  CRUSH rule 0 x 562 [718,529,87,729,842]
+  CRUSH rule 0 x 563 [552,332,747,206,274]
+  CRUSH rule 0 x 564 [835,769,736,486,630]
+  CRUSH rule 0 x 565 [8,167,539,182,607]
+  CRUSH rule 0 x 566 [600,481,301,263,90]
+  CRUSH rule 0 x 567 [999,994,509,899,947]
+  CRUSH rule 0 x 568 [252,431,157,62,601]
+  CRUSH rule 0 x 569 [643,218,943,455,83]
+  CRUSH rule 0 x 570 [617,635,765,422,250]
+  CRUSH rule 0 x 571 [757,80,59,98,328]
+  CRUSH rule 0 x 572 [299,348,575,889,943]
+  CRUSH rule 0 x 573 [25,505,270,167,58]
+  CRUSH rule 0 x 574 [215,431,624,177,628]
+  CRUSH rule 0 x 575 [225,252,611,546,32]
+  CRUSH rule 0 x 576 [627,94,159,857,430]
+  CRUSH rule 0 x 577 [237,809,778,636,61]
+  CRUSH rule 0 x 578 [885,313,120,344,771]
+  CRUSH rule 0 x 579 [924,575,787,831,47]
+  CRUSH rule 0 x 580 [718,51,766,121,118]
+  CRUSH rule 0 x 581 [219,807,129,571,856]
+  CRUSH rule 0 x 582 [893,701,598,863,285]
+  CRUSH rule 0 x 583 [246,930,964,170,993]
+  CRUSH rule 0 x 584 [336,432,680,175,495]
+  CRUSH rule 0 x 585 [324,999,397,485,457]
+  CRUSH rule 0 x 586 [558,230,976,541,816]
+  CRUSH rule 0 x 587 [985,830,597,21,308]
+  CRUSH rule 0 x 588 [211,544,57,134,162]
+  CRUSH rule 0 x 589 [129,21,112,190,885]
+  CRUSH rule 0 x 590 [467,969,652,593,287]
+  CRUSH rule 0 x 591 [758,514,316,164,35]
+  CRUSH rule 0 x 592 [525,253,190,443,315]
+  CRUSH rule 0 x 593 [601,885,339,152,297]
+  CRUSH rule 0 x 594 [227,60,450,30,717]
+  CRUSH rule 0 x 595 [720,854,496,912,80]
+  CRUSH rule 0 x 596 [751,195,997,77,261]
+  CRUSH rule 0 x 597 [129,574,714,8,789]
+  CRUSH rule 0 x 598 [679,207,604,396,841]
+  CRUSH rule 0 x 599 [668,315,683,349,681]
+  CRUSH rule 0 x 600 [143,396,464,444,59]
+  CRUSH rule 0 x 601 [326,573,873,902,136]
+  CRUSH rule 0 x 602 [860,281,875,535,672]
+  CRUSH rule 0 x 603 [709,328,445,349,190]
+  CRUSH rule 0 x 604 [571,62,814,95,866]
+  CRUSH rule 0 x 605 [252,739,860,27,313]
+  CRUSH rule 0 x 606 [339,236,759,842,67]
+  CRUSH rule 0 x 607 [590,248,759,868,433]
+  CRUSH rule 0 x 608 [145,635,309,467,875]
+  CRUSH rule 0 x 609 [973,547,223,79,762]
+  CRUSH rule 0 x 610 [435,816,961,983,255]
+  CRUSH rule 0 x 611 [559,283,422,584,176]
+  CRUSH rule 0 x 612 [273,149,123,576,911]
+  CRUSH rule 0 x 613 [828,614,642,674,33]
+  CRUSH rule 0 x 614 [478,748,393,34,171]
+  CRUSH rule 0 x 615 [392,155,144,326,626]
+  CRUSH rule 0 x 616 [778,637,452,248,15]
+  CRUSH rule 0 x 617 [622,713,996,833,611]
+  CRUSH rule 0 x 618 [149,877,270,329,180]
+  CRUSH rule 0 x 619 [604,163,656,409,322]
+  CRUSH rule 0 x 620 [181,23,409,198,64]
+  CRUSH rule 0 x 621 [735,902,386,237,939]
+  CRUSH rule 0 x 622 [661,824,717,568,858]
+  CRUSH rule 0 x 623 [142,121,643,61,695]
+  CRUSH rule 0 x 624 [360,716,420,398,49]
+  CRUSH rule 0 x 625 [541,167,385,1,601]
+  CRUSH rule 0 x 626 [364,431,610,363,535]
+  CRUSH rule 0 x 627 [458,137,557,410,287]
+  CRUSH rule 0 x 628 [250,350,556,497,821]
+  CRUSH rule 0 x 629 [928,160,710,572,365]
+  CRUSH rule 0 x 630 [243,19,918,556,601]
+  CRUSH rule 0 x 631 [438,221,574,676,797]
+  CRUSH rule 0 x 632 [797,368,247,5,32]
+  CRUSH rule 0 x 633 [993,749,525,485,27]
+  CRUSH rule 0 x 634 [239,351,633,299,651]
+  CRUSH rule 0 x 635 [640,965,25,961,306]
+  CRUSH rule 0 x 636 [173,290,297,991,937]
+  CRUSH rule 0 x 637 [0,918,98,108,111]
+  CRUSH rule 0 x 638 [702,235,424,900,983]
+  CRUSH rule 0 x 639 [475,687,31,785,918]
+  CRUSH rule 0 x 640 [31,664,399,677,123]
+  CRUSH rule 0 x 641 [296,473,108,963,341]
+  CRUSH rule 0 x 642 [894,273,427,606,677]
+  CRUSH rule 0 x 643 [117,111,732,191,114]
+  CRUSH rule 0 x 644 [438,336,327,512,599]
+  CRUSH rule 0 x 645 [982,702,351,573,907]
+  CRUSH rule 0 x 646 [334,804,146,842,697]
+  CRUSH rule 0 x 647 [933,787,185,334,752]
+  CRUSH rule 0 x 648 [22,444,400,862,207]
+  CRUSH rule 0 x 649 [503,229,213,460,639]
+  CRUSH rule 0 x 650 [328,659,420,443,739]
+  CRUSH rule 0 x 651 [3,880,823,123,378]
+  CRUSH rule 0 x 652 [495,977,563,733,92]
+  CRUSH rule 0 x 653 [185,718,804,280,975]
+  CRUSH rule 0 x 654 [130,528,380,81,906]
+  CRUSH rule 0 x 655 [560,872,454,504,319]
+  CRUSH rule 0 x 656 [219,885,178,981,863]
+  CRUSH rule 0 x 657 [233,684,813,490,208]
+  CRUSH rule 0 x 658 [778,6,756,380,750]
+  CRUSH rule 0 x 659 [240,663,306,540,789]
+  CRUSH rule 0 x 660 [244,855,196,147,678]
+  CRUSH rule 0 x 661 [184,270,128,398,910]
+  CRUSH rule 0 x 662 [65,883,921,438,79]
+  CRUSH rule 0 x 663 [323,721,594,812,43]
+  CRUSH rule 0 x 664 [865,113,512,51,427]
+  CRUSH rule 0 x 665 [420,850,591,475,202]
+  CRUSH rule 0 x 666 [319,767,246,3,369]
+  CRUSH rule 0 x 667 [875,39,343,100,829]
+  CRUSH rule 0 x 668 [331,122,263,599,355]
+  CRUSH rule 0 x 669 [915,521,402,747,673]
+  CRUSH rule 0 x 670 [845,659,943,447,401]
+  CRUSH rule 0 x 671 [108,634,527,363,856]
+  CRUSH rule 0 x 672 [578,216,110,589,302]
+  CRUSH rule 0 x 673 [442,74,579,797,622]
+  CRUSH rule 0 x 674 [588,364,281,308,645]
+  CRUSH rule 0 x 675 [489,698,744,671,870]
+  CRUSH rule 0 x 676 [928,911,40,180,722]
+  CRUSH rule 0 x 677 [399,269,692,131,615]
+  CRUSH rule 0 x 678 [546,752,544,155,5]
+  CRUSH rule 0 x 679 [988,25,275,433,628]
+  CRUSH rule 0 x 680 [335,963,382,486,749]
+  CRUSH rule 0 x 681 [690,462,623,466,49]
+  CRUSH rule 0 x 682 [196,588,154,257,807]
+  CRUSH rule 0 x 683 [627,25,421,160,873]
+  CRUSH rule 0 x 684 [38,804,592,158,991]
+  CRUSH rule 0 x 685 [841,368,548,362,166]
+  CRUSH rule 0 x 686 [336,287,525,440,166]
+  CRUSH rule 0 x 687 [20,682,924,653,356]
+  CRUSH rule 0 x 688 [463,371,780,556,385]
+  CRUSH rule 0 x 689 [569,250,78,816,847]
+  CRUSH rule 0 x 690 [551,144,587,263,378]
+  CRUSH rule 0 x 691 [766,464,446,533,449]
+  CRUSH rule 0 x 692 [739,634,18,245,624]
+  CRUSH rule 0 x 693 [339,297,118,330,817]
+  CRUSH rule 0 x 694 [405,26,830,181,533]
+  CRUSH rule 0 x 695 [622,576,597,535,600]
+  CRUSH rule 0 x 696 [558,902,689,13,715]
+  CRUSH rule 0 x 697 [818,222,406,691,427]
+  CRUSH rule 0 x 698 [178,48,402,233,841]
+  CRUSH rule 0 x 699 [450,244,180,919,183]
+  CRUSH rule 0 x 700 [502,771,987,706,416]
+  CRUSH rule 0 x 701 [4,612,782,216,853]
+  CRUSH rule 0 x 702 [177,630,232,923,281]
+  CRUSH rule 0 x 703 [354,178,389,393,778]
+  CRUSH rule 0 x 704 [646,601,156,171,603]
+  CRUSH rule 0 x 705 [921,401,890,265,244]
+  CRUSH rule 0 x 706 [652,877,562,452,26]
+  CRUSH rule 0 x 707 [345,745,67,716,789]
+  CRUSH rule 0 x 708 [333,607,180,469,170]
+  CRUSH rule 0 x 709 [45,187,302,115,896]
+  CRUSH rule 0 x 710 [94,855,43,199,18]
+  CRUSH rule 0 x 711 [227,653,731,150,156]
+  CRUSH rule 0 x 712 [398,953,136,870,181]
+  CRUSH rule 0 x 713 [116,800,503,662,635]
+  CRUSH rule 0 x 714 [111,629,866,709,902]
+  CRUSH rule 0 x 715 [531,291,486,382,192]
+  CRUSH rule 0 x 716 [169,541,291,42,343]
+  CRUSH rule 0 x 717 [417,446,994,894,239]
+  CRUSH rule 0 x 718 [992,383,298,844,377]
+  CRUSH rule 0 x 719 [936,674,324,759,194]
+  CRUSH rule 0 x 720 [370,188,174,464,644]
+  CRUSH rule 0 x 721 [320,859,278,259,170]
+  CRUSH rule 0 x 722 [7,2,673,129,96]
+  CRUSH rule 0 x 723 [270,553,831,662,38]
+  CRUSH rule 0 x 724 [666,822,708,895,633]
+  CRUSH rule 0 x 725 [794,406,875,459,981]
+  CRUSH rule 0 x 726 [420,556,341,292,240]
+  CRUSH rule 0 x 727 [561,461,129,635,965]
+  CRUSH rule 0 x 728 [951,330,196,756,589]
+  CRUSH rule 0 x 729 [656,644,436,591,27]
+  CRUSH rule 0 x 730 [3,558,629,184,50]
+  CRUSH rule 0 x 731 [852,89,75,735,713]
+  CRUSH rule 0 x 732 [983,840,869,976,697]
+  CRUSH rule 0 x 733 [285,396,388,122,387]
+  CRUSH rule 0 x 734 [125,510,402,640,676]
+  CRUSH rule 0 x 735 [417,773,686,504,459]
+  CRUSH rule 0 x 736 [749,396,632,550,779]
+  CRUSH rule 0 x 737 [644,991,946,135,448]
+  CRUSH rule 0 x 738 [449,683,290,220,245]
+  CRUSH rule 0 x 739 [341,220,641,454,740]
+  CRUSH rule 0 x 740 [874,524,674,650,472]
+  CRUSH rule 0 x 741 [189,472,712,798,715]
+  CRUSH rule 0 x 742 [912,581,114,117,730]
+  CRUSH rule 0 x 743 [654,914,425,441,763]
+  CRUSH rule 0 x 744 [725,295,579,377,162]
+  CRUSH rule 0 x 745 [787,858,850,506,612]
+  CRUSH rule 0 x 746 [757,848,704,30,47]
+  CRUSH rule 0 x 747 [700,81,867,681,801]
+  CRUSH rule 0 x 748 [557,436,238,664,293]
+  CRUSH rule 0 x 749 [772,622,337,42,156]
+  CRUSH rule 0 x 750 [946,97,376,677,316]
+  CRUSH rule 0 x 751 [996,618,343,911,83]
+  CRUSH rule 0 x 752 [746,887,695,868,610]
+  CRUSH rule 0 x 753 [741,14,463,479,172]
+  CRUSH rule 0 x 754 [648,349,333,355,65]
+  CRUSH rule 0 x 755 [157,460,466,187,959]
+  CRUSH rule 0 x 756 [416,97,197,497,227]
+  CRUSH rule 0 x 757 [599,839,776,410,256]
+  CRUSH rule 0 x 758 [994,218,620,256,361]
+  CRUSH rule 0 x 759 [959,682,514,745,100]
+  CRUSH rule 0 x 760 [518,943,215,83,706]
+  CRUSH rule 0 x 761 [285,849,420,324,987]
+  CRUSH rule 0 x 762 [591,313,41,335,110]
+  CRUSH rule 0 x 763 [908,411,200,740,292]
+  CRUSH rule 0 x 764 [787,234,894,485,883]
+  CRUSH rule 0 x 765 [327,921,882,393,444]
+  CRUSH rule 0 x 766 [84,161,878,704,416]
+  CRUSH rule 0 x 767 [370,895,702,701,890]
+  CRUSH rule 0 x 768 [826,760,879,864,460]
+  CRUSH rule 0 x 769 [67,768,663,735,814]
+  CRUSH rule 0 x 770 [593,909,482,259,5]
+  CRUSH rule 0 x 771 [309,935,121,578,937]
+  CRUSH rule 0 x 772 [12,125,797,301,348]
+  CRUSH rule 0 x 773 [253,466,820,549,591]
+  CRUSH rule 0 x 774 [164,390,705,109,881]
+  CRUSH rule 0 x 775 [703,47,43,973,643]
+  CRUSH rule 0 x 776 [728,231,80,916,2]
+  CRUSH rule 0 x 777 [981,621,568,729,869]
+  CRUSH rule 0 x 778 [411,456,544,597,789]
+  CRUSH rule 0 x 779 [346,121,519,921,587]
+  CRUSH rule 0 x 780 [476,39,288,381,303]
+  CRUSH rule 0 x 781 [10,130,585,844,729]
+  CRUSH rule 0 x 782 [462,246,581,902,623]
+  CRUSH rule 0 x 783 [580,373,153,775,668]
+  CRUSH rule 0 x 784 [413,113,978,990,994]
+  CRUSH rule 0 x 785 [341,856,332,354,59]
+  CRUSH rule 0 x 786 [411,140,313,393,215]
+  CRUSH rule 0 x 787 [605,522,211,813,636]
+  CRUSH rule 0 x 788 [226,545,35,142,726]
+  CRUSH rule 0 x 789 [545,320,414,702,731]
+  CRUSH rule 0 x 790 [414,748,816,327,130]
+  CRUSH rule 0 x 791 [660,906,406,697,916]
+  CRUSH rule 0 x 792 [287,392,514,204,75]
+  CRUSH rule 0 x 793 [631,133,850,713,720]
+  CRUSH rule 0 x 794 [931,517,543,210,963]
+  CRUSH rule 0 x 795 [551,962,477,948,425]
+  CRUSH rule 0 x 796 [814,4,95,27,368]
+  CRUSH rule 0 x 797 [64,201,299,734,605]
+  CRUSH rule 0 x 798 [422,530,114,431,565]
+  CRUSH rule 0 x 799 [824,32,679,562,266]
+  CRUSH rule 0 x 800 [862,623,489,637,861]
+  CRUSH rule 0 x 801 [145,550,329,324,734]
+  CRUSH rule 0 x 802 [570,19,847,308,387]
+  CRUSH rule 0 x 803 [151,812,662,358,880]
+  CRUSH rule 0 x 804 [467,93,264,863,176]
+  CRUSH rule 0 x 805 [621,223,938,809,591]
+  CRUSH rule 0 x 806 [898,957,805,430,499]
+  CRUSH rule 0 x 807 [354,531,422,159,921]
+  CRUSH rule 0 x 808 [7,96,76,897,446]
+  CRUSH rule 0 x 809 [70,734,719,56,687]
+  CRUSH rule 0 x 810 [701,18,972,327,771]
+  CRUSH rule 0 x 811 [248,547,103,728,901]
+  CRUSH rule 0 x 812 [230,576,821,566,993]
+  CRUSH rule 0 x 813 [805,114,683,629,801]
+  CRUSH rule 0 x 814 [54,619,973,741,497]
+  CRUSH rule 0 x 815 [679,412,613,132,969]
+  CRUSH rule 0 x 816 [919,448,826,414,36]
+  CRUSH rule 0 x 817 [765,830,436,521,332]
+  CRUSH rule 0 x 818 [415,566,644,687,692]
+  CRUSH rule 0 x 819 [721,319,865,750,546]
+  CRUSH rule 0 x 820 [218,301,333,190,686]
+  CRUSH rule 0 x 821 [185,795,680,953,329]
+  CRUSH rule 0 x 822 [356,261,54,522,900]
+  CRUSH rule 0 x 823 [220,281,549,456,64]
+  CRUSH rule 0 x 824 [292,809,887,74,776]
+  CRUSH rule 0 x 825 [949,778,101,311,110]
+  CRUSH rule 0 x 826 [767,818,833,927,356]
+  CRUSH rule 0 x 827 [631,83,406,635,657]
+  CRUSH rule 0 x 828 [288,986,445,26,414]
+  CRUSH rule 0 x 829 [990,667,915,694,974]
+  CRUSH rule 0 x 830 [152,571,778,505,685]
+  CRUSH rule 0 x 831 [814,563,630,97,582]
+  CRUSH rule 0 x 832 [235,641,616,110,979]
+  CRUSH rule 0 x 833 [657,565,922,140,825]
+  CRUSH rule 0 x 834 [907,231,644,13,617]
+  CRUSH rule 0 x 835 [784,262,771,264,612]
+  CRUSH rule 0 x 836 [951,158,366,710,43]
+  CRUSH rule 0 x 837 [556,498,334,633,895]
+  CRUSH rule 0 x 838 [329,274,964,547,119]
+  CRUSH rule 0 x 839 [568,209,939,364,658]
+  CRUSH rule 0 x 840 [45,579,842,70,655]
+  CRUSH rule 0 x 841 [652,702,24,605,152]
+  CRUSH rule 0 x 842 [629,984,314,895,408]
+  CRUSH rule 0 x 843 [799,690,688,648,151]
+  CRUSH rule 0 x 844 [694,600,534,700,569]
+  CRUSH rule 0 x 845 [332,30,179,93,951]
+  CRUSH rule 0 x 846 [452,251,712,719,404]
+  CRUSH rule 0 x 847 [399,681,847,739,13]
+  CRUSH rule 0 x 848 [303,138,440,346,547]
+  CRUSH rule 0 x 849 [666,346,708,873,64]
+  CRUSH rule 0 x 850 [644,511,345,844,545]
+  CRUSH rule 0 x 851 [527,546,737,425,100]
+  CRUSH rule 0 x 852 [31,809,94,618,156]
+  CRUSH rule 0 x 853 [483,330,869,184,46]
+  CRUSH rule 0 x 854 [697,953,968,143,502]
+  CRUSH rule 0 x 855 [837,996,239,621,32]
+  CRUSH rule 0 x 856 [712,40,547,430,195]
+  CRUSH rule 0 x 857 [77,984,576,551,568]
+  CRUSH rule 0 x 858 [412,384,841,465,572]
+  CRUSH rule 0 x 859 [173,760,26,300,87]
+  CRUSH rule 0 x 860 [776,429,328,917,658]
+  CRUSH rule 0 x 861 [705,405,477,50,73]
+  CRUSH rule 0 x 862 [809,44,788,938,964]
+  CRUSH rule 0 x 863 [349,496,963,178,675]
+  CRUSH rule 0 x 864 [717,858,101,239,992]
+  CRUSH rule 0 x 865 [857,603,586,262,550]
+  CRUSH rule 0 x 866 [394,304,71,96,642]
+  CRUSH rule 0 x 867 [640,773,663,974,261]
+  CRUSH rule 0 x 868 [613,950,712,663,666]
+  CRUSH rule 0 x 869 [973,889,524,22,671]
+  CRUSH rule 0 x 870 [505,35,386,498,348]
+  CRUSH rule 0 x 871 [239,264,262,773,781]
+  CRUSH rule 0 x 872 [21,767,456,748,783]
+  CRUSH rule 0 x 873 [954,666,980,264,435]
+  CRUSH rule 0 x 874 [54,510,947,1,500]
+  CRUSH rule 0 x 875 [809,418,452,462,88]
+  CRUSH rule 0 x 876 [483,457,61,248,523]
+  CRUSH rule 0 x 877 [542,531,952,939,710]
+  CRUSH rule 0 x 878 [217,674,857,644,678]
+  CRUSH rule 0 x 879 [999,475,134,250,319]
+  CRUSH rule 0 x 880 [678,573,935,385,570]
+  CRUSH rule 0 x 881 [394,835,789,802,587]
+  CRUSH rule 0 x 882 [467,382,353,56,979]
+  CRUSH rule 0 x 883 [802,744,237,337,50]
+  CRUSH rule 0 x 884 [653,660,638,700,31]
+  CRUSH rule 0 x 885 [898,704,307,445,879]
+  CRUSH rule 0 x 886 [434,357,938,641,737]
+  CRUSH rule 0 x 887 [297,226,711,428,370]
+  CRUSH rule 0 x 888 [863,324,443,213,902]
+  CRUSH rule 0 x 889 [105,102,308,163,947]
+  CRUSH rule 0 x 890 [550,248,606,704,615]
+  CRUSH rule 0 x 891 [575,928,880,891,826]
+  CRUSH rule 0 x 892 [259,862,133,271,292]
+  CRUSH rule 0 x 893 [902,880,543,542,37]
+  CRUSH rule 0 x 894 [180,169,916,43,945]
+  CRUSH rule 0 x 895 [725,849,182,129,177]
+  CRUSH rule 0 x 896 [951,34,874,537,969]
+  CRUSH rule 0 x 897 [810,352,73,939,943]
+  CRUSH rule 0 x 898 [979,433,719,411,787]
+  CRUSH rule 0 x 899 [685,668,534,932,399]
+  CRUSH rule 0 x 900 [530,978,41,894,941]
+  CRUSH rule 0 x 901 [740,107,336,175,574]
+  CRUSH rule 0 x 902 [800,743,693,310,67]
+  CRUSH rule 0 x 903 [230,267,842,266,550]
+  CRUSH rule 0 x 904 [346,949,460,973,696]
+  CRUSH rule 0 x 905 [530,397,619,958,576]
+  CRUSH rule 0 x 906 [80,426,138,672,73]
+  CRUSH rule 0 x 907 [365,968,475,297,296]
+  CRUSH rule 0 x 908 [204,832,742,809,862]
+  CRUSH rule 0 x 909 [883,989,146,959,366]
+  CRUSH rule 0 x 910 [549,593,249,853,792]
+  CRUSH rule 0 x 911 [325,847,352,214,851]
+  CRUSH rule 0 x 912 [874,888,582,796,557]
+  CRUSH rule 0 x 913 [331,463,342,574,989]
+  CRUSH rule 0 x 914 [836,468,601,732,607]
+  CRUSH rule 0 x 915 [245,228,100,661,799]
+  CRUSH rule 0 x 916 [77,967,364,435,27]
+  CRUSH rule 0 x 917 [239,60,866,221,772]
+  CRUSH rule 0 x 918 [988,115,922,80,201]
+  CRUSH rule 0 x 919 [783,139,696,1,848]
+  CRUSH rule 0 x 920 [623,408,685,953,974]
+  CRUSH rule 0 x 921 [105,799,144,90,399]
+  CRUSH rule 0 x 922 [887,505,652,348,514]
+  CRUSH rule 0 x 923 [223,318,552,458,743]
+  CRUSH rule 0 x 924 [25,778,366,333,163]
+  CRUSH rule 0 x 925 [912,601,297,682,770]
+  CRUSH rule 0 x 926 [968,133,132,144,814]
+  CRUSH rule 0 x 927 [277,724,214,988,690]
+  CRUSH rule 0 x 928 [554,203,658,789,298]
+  CRUSH rule 0 x 929 [761,802,367,528,758]
+  CRUSH rule 0 x 930 [814,61,788,736,660]
+  CRUSH rule 0 x 931 [29,193,61,41,343]
+  CRUSH rule 0 x 932 [446,198,862,534,168]
+  CRUSH rule 0 x 933 [352,742,216,321,525]
+  CRUSH rule 0 x 934 [730,2,332,631,613]
+  CRUSH rule 0 x 935 [731,23,736,79,361]
+  CRUSH rule 0 x 936 [322,975,20,904,827]
+  CRUSH rule 0 x 937 [822,221,841,161,723]
+  CRUSH rule 0 x 938 [557,850,66,630,499]
+  CRUSH rule 0 x 939 [150,11,971,371,124]
+  CRUSH rule 0 x 940 [638,398,169,616,333]
+  CRUSH rule 0 x 941 [730,342,929,577,451]
+  CRUSH rule 0 x 942 [62,292,166,814,587]
+  CRUSH rule 0 x 943 [165,314,519,548,41]
+  CRUSH rule 0 x 944 [199,625,766,176,194]
+  CRUSH rule 0 x 945 [946,999,699,303,38]
+  CRUSH rule 0 x 946 [595,93,852,142,503]
+  CRUSH rule 0 x 947 [800,582,356,93,716]
+  CRUSH rule 0 x 948 [132,551,139,920,87]
+  CRUSH rule 0 x 949 [792,920,466,380,97]
+  CRUSH rule 0 x 950 [111,345,176,543,879]
+  CRUSH rule 0 x 951 [414,619,648,655,364]
+  CRUSH rule 0 x 952 [775,469,500,356,287]
+  CRUSH rule 0 x 953 [349,1,5,251,168]
+  CRUSH rule 0 x 954 [570,940,410,249,929]
+  CRUSH rule 0 x 955 [729,774,823,800,7]
+  CRUSH rule 0 x 956 [519,141,575,625,738]
+  CRUSH rule 0 x 957 [242,709,611,97,760]
+  CRUSH rule 0 x 958 [84,217,227,253,246]
+  CRUSH rule 0 x 959 [270,413,918,789,703]
+  CRUSH rule 0 x 960 [458,192,307,279,920]
+  CRUSH rule 0 x 961 [981,388,777,546,359]
+  CRUSH rule 0 x 962 [623,834,277,134,729]
+  CRUSH rule 0 x 963 [291,167,714,468,109]
+  CRUSH rule 0 x 964 [28,156,788,127,598]
+  CRUSH rule 0 x 965 [675,557,290,517,840]
+  CRUSH rule 0 x 966 [836,306,946,283,642]
+  CRUSH rule 0 x 967 [966,386,735,837,392]
+  CRUSH rule 0 x 968 [864,756,690,121,328]
+  CRUSH rule 0 x 969 [729,625,480,769,512]
+  CRUSH rule 0 x 970 [800,362,646,582,309]
+  CRUSH rule 0 x 971 [737,381,153,684,298]
+  CRUSH rule 0 x 972 [952,245,720,884,334]
+  CRUSH rule 0 x 973 [356,455,579,857,832]
+  CRUSH rule 0 x 974 [545,758,586,596,756]
+  CRUSH rule 0 x 975 [336,191,202,146,720]
+  CRUSH rule 0 x 976 [446,208,757,620,252]
+  CRUSH rule 0 x 977 [202,896,196,956,763]
+  CRUSH rule 0 x 978 [612,324,996,225,418]
+  CRUSH rule 0 x 979 [843,457,675,650,958]
+  CRUSH rule 0 x 980 [60,914,881,626,850]
+  CRUSH rule 0 x 981 [702,749,937,153,724]
+  CRUSH rule 0 x 982 [298,928,738,167,99]
+  CRUSH rule 0 x 983 [723,572,395,358,900]
+  CRUSH rule 0 x 984 [723,864,804,935,846]
+  CRUSH rule 0 x 985 [945,459,868,211,524]
+  CRUSH rule 0 x 986 [772,664,535,169,297]
+  CRUSH rule 0 x 987 [88,324,312,843,661]
+  CRUSH rule 0 x 988 [522,927,131,996,351]
+  CRUSH rule 0 x 989 [578,332,208,605,975]
+  CRUSH rule 0 x 990 [638,228,414,311,738]
+  CRUSH rule 0 x 991 [530,221,451,422,879]
+  CRUSH rule 0 x 992 [925,705,275,81,234]
+  CRUSH rule 0 x 993 [991,301,43,469,830]
+  CRUSH rule 0 x 994 [276,51,868,683,843]
+  CRUSH rule 0 x 995 [288,836,753,790,758]
+  CRUSH rule 0 x 996 [887,983,252,686,470]
+  CRUSH rule 0 x 997 [110,924,386,79,705]
+  CRUSH rule 0 x 998 [435,830,485,853,926]
+  CRUSH rule 0 x 999 [876,738,357,913,723]
+  CRUSH rule 0 x 1000 [178,963,638,430,845]
+  CRUSH rule 0 x 1001 [99,519,66,759,583]
+  CRUSH rule 0 x 1002 [515,534,468,866,878]
+  CRUSH rule 0 x 1003 [104,611,937,698,94]
+  CRUSH rule 0 x 1004 [269,638,724,375,491]
+  CRUSH rule 0 x 1005 [369,223,309,409,822]
+  CRUSH rule 0 x 1006 [40,107,69,275,79]
+  CRUSH rule 0 x 1007 [978,111,416,758,454]
+  CRUSH rule 0 x 1008 [965,956,624,832,421]
+  CRUSH rule 0 x 1009 [598,476,356,695,919]
+  CRUSH rule 0 x 1010 [767,523,239,517,29]
+  CRUSH rule 0 x 1011 [289,871,207,576,347]
+  CRUSH rule 0 x 1012 [128,28,370,31,341]
+  CRUSH rule 0 x 1013 [979,765,660,812,666]
+  CRUSH rule 0 x 1014 [979,948,513,88,47]
+  CRUSH rule 0 x 1015 [277,790,396,672,542]
+  CRUSH rule 0 x 1016 [262,73,128,886,839]
+  CRUSH rule 0 x 1017 [150,269,61,499,832]
+  CRUSH rule 0 x 1018 [555,829,554,944,406]
+  CRUSH rule 0 x 1019 [513,356,265,446,65]
+  CRUSH rule 0 x 1020 [158,161,877,704,948]
+  CRUSH rule 0 x 1021 [915,998,957,285,546]
+  CRUSH rule 0 x 1022 [967,829,973,640,703]
+  CRUSH rule 0 x 1023 [488,257,614,859,325]
+  rule 0 (data) num_rep 5 result size == 5:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356]
+  CRUSH rule 0 x 87 [997,317,463,626,685,683]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652]
+  CRUSH rule 0 x 194 [242,473,58,655,653,277]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849]
+  CRUSH rule 0 x 257 [825,264,867,869,529,409]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939]
+  CRUSH rule 0 x 262 [223,1,561,420,229,16]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544]
+  CRUSH rule 0 x 324 [16,745,511,439,272,513]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493]
+  CRUSH rule 0 x 374 [191,887,920,928,223,714]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149]
+  CRUSH rule 0 x 445 [485,482,528,209,964,753]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604]
+  CRUSH rule 0 x 699 [450,244,180,919,183,332]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948]
+  CRUSH rule 0 x 711 [227,653,731,150,156,842]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757]
+  CRUSH rule 0 x 742 [912,581,114,117,730,21]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762]
+  CRUSH rule 0 x 813 [805,114,683,629,801,462]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296]
+  CRUSH rule 0 x 868 [613,950,712,663,666,460]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173]
+  CRUSH rule 0 x 926 [968,133,132,144,814,155]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596]
+  CRUSH rule 0 x 974 [545,758,586,596,756,790]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419]
+  rule 0 (data) num_rep 6 result size == 6:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38]
+  CRUSH rule 0 x 87 [997,317,463,626,685,683,909]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359]
+  CRUSH rule 0 x 194 [242,473,58,655,653,277,792]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189]
+  CRUSH rule 0 x 257 [825,264,867,869,529,409,291]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516]
+  CRUSH rule 0 x 262 [223,1,561,420,229,16,88]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236]
+  CRUSH rule 0 x 324 [16,745,511,439,272,513,668]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128]
+  CRUSH rule 0 x 374 [191,887,920,928,223,714,961]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,879]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182]
+  CRUSH rule 0 x 445 [485,482,528,209,964,753,554]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,385]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468]
+  CRUSH rule 0 x 699 [450,244,180,919,183,332,747]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449]
+  CRUSH rule 0 x 711 [227,653,731,150,156,842,534]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863]
+  CRUSH rule 0 x 742 [912,581,114,117,730,21,687]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675]
+  CRUSH rule 0 x 813 [805,114,683,629,801,462,285]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988]
+  CRUSH rule 0 x 868 [613,950,712,663,666,460,643]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969]
+  CRUSH rule 0 x 926 [968,133,132,144,814,155,709]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,171]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549]
+  CRUSH rule 0 x 974 [545,758,586,596,756,790,116]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50]
+  rule 0 (data) num_rep 7 result size == 7:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966,750]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672,820]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527,901]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481,169]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847,220]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964,872]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884,726]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578,14]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462,207]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3,569]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811,484]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380,821]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117,393]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78,170]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855,76]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437,72]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786,969]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935,605]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883,394]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51,834]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975,636]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263,267]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557,633]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477,468]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800,309]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248,908]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501,909]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676,560]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770,569]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719,581]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426,522]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813,217]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428,63]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618,288]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663,877]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75,580]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676,524]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311,758]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929,910]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856,248]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980,503]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930,74]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456,713]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514,40]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160,527]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658,876]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45,390]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959,800]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738,47]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822,628]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83,344]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406,281]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304,899]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882,503]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339,902]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508,763]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77,500]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670,67]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8,762]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643,511]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194,277]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812,14]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45,723]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705,268]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451,891]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862,221]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334,656]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783,593]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705,114]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871,361]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684,411]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587,264]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642,529]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835,705]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633,560]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366,258]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735,953]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197,690]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183,917]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339,24]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51,766]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247,963]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580,51]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66,197]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681,618]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567,908]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38,326]
+  CRUSH rule 0 x 87 [997,317,463,626,685,683,909,49]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99,737]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320,921]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412,184]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495,299]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167,634]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343,949]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82,422]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648,661]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204,750]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83,542]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270,511]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37,138]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9,168]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12,369]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923,106]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933,231]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401,85]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434,121]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878,493]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589,614]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631,996]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856,296]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510,472]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18,949]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725,413]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738,253]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598,418]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421,39]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9,448]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725,142]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766,210]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570,278]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49,126]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424,868]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551,43]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877,248]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482,254]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736,737]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689,209]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909,809]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666,691]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149,146]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270,425]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203,667]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922,597]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14,444]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893,281]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712,649]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846,890]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363,323]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400,885]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384,58]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178,532]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193,258]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424,820]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266,70]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697,610]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577,348]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929,196]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514,546]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352,820]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406,556]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484,553]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671,883]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807,50]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942,864]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443,98]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58,641]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271,229]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384,446]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899,902]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209,641]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242,999]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909,642]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148,88]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587,854]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110,329]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524,389]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548,676]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935,266]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667,463]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601,909]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835,840]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100,910]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36,356]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471,529]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780,932]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51,211]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240,316]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756,647]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856,992]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724,923]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736,846]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168,224]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485,571]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109,903]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74,45]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485,511]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369,270]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566,378]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833,650]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37,1]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527,237]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999,754]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123,759]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359,721]
+  CRUSH rule 0 x 194 [242,473,58,655,653,277,792,887]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755,471]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413,834]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666,316]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355,574]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960,369]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132,271]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495,76]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153,272]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669,676]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697,881]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776,463]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230,367]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76,708]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96,841]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522,185]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14,921]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922,372]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857,894]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687,903]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579,79]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982,826]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973,898]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264,6]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34,497]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697,805]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686,982]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61,559]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597,904]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465,429]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329,320]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606,465]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876,984]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966,217]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261,389]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262,884]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93,880]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50,182]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106,877]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796,835]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103,95]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471,753]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573,128]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327,509]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894,884]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426,427]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450,677]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413,136]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234,756]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799,21]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785,107]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605,837]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989,999]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590,840]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713,237]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886,650]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270,619]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58,354]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838,131]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797,675]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605,453]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607,95]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189,627]
+  CRUSH rule 0 x 257 [825,264,867,869,529,409,291,732]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863,427]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77,191]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533,646]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516,136]
+  CRUSH rule 0 x 262 [223,1,561,420,229,16,88,534]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773,979]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715,916]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833,987]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7,557]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972,645]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715,536]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146,14]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476,571]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263,138]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710,779]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982,735]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370,990]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992,823]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634,98]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783,648]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350,600]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952,250]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895,914]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443,604]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939,879]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369,544]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60,546]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582,975]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699,627]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137,890]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195,538]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795,377]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473,209]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557,86]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806,911]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775,636]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459,833]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440,73]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266,82]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232,343]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213,286]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924,106]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904,838]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220,432]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400,365]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561,905]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214,944]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685,838]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761,648]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459,250]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976,551]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857,153]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90,855]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256,798]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497,929]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24,81]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952,599]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749,841]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761,725]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892,86]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290,97]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800,868]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523,366]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868,841]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187,380]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236,810]
+  CRUSH rule 0 x 324 [16,745,511,439,272,513,668,959]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462,268]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461,509]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750,669]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33,806]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516,713]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420,520]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624,649]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644,821]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243,633]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408,175]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635,367]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698,533]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956,907]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69,11]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607,953]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707,328]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720,503]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516,661]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660,655]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142,114]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127,732]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419,962]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783,661]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862,513]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795,143]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841,164]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766,775]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899,665]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279,616]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612,33]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985,412]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629,228]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891,656]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874,643]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397,52]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29,529]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643,286]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265,35]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444,613]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203,17]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113,373]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59,914]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685,641]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616,93]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284,184]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850,419]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797,547]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212,241]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128,207]
+  CRUSH rule 0 x 374 [191,887,920,928,223,714,961,760]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934,619]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446,897]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654,487]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728,860]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115,557]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255,806]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773,496]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276,56]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29,197]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488,435]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910,329]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232,692]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164,934]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474,267]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340,222]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852,196]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,879,136]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901,82]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652,543]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15,591]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126,179]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726,935]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996,558]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349,672]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749,410]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536,592]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79,32]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658,449]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202,650]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299,577]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776,296]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718,578]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72,345]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246,695]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213,806]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949,507]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313,351]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560,155]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618,907]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97,124]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694,332]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889,344]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138,374]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848,972]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823,884]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506,670]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764,274]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839,547]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905,588]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785,395]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80,236]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791,565]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334,588]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225,94]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857,562]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859,699]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194,366]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513,591]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52,897]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644,646]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371,731]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10,976]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443,23]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755,841]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414,152]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800,174]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205,236]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864,799]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774,350]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182,145]
+  CRUSH rule 0 x 445 [485,482,528,209,964,753,554,931]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314,714]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58,531]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807,688]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258,548]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721,31]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878,33]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604,49]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514,5]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106,103]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295,967]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504,96]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762,975]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399,558]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147,483]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841,512]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385,797]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240,911]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538,111]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789,0]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137,466]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659,35]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187,640]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201,249]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681,683]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558,71]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256,978]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809,56]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756,663]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446,333]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237,764]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984,226]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753,794]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941,843]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158,489]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371,807]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135,375]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731,233]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448,437]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283,170]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866,661]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621,955]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857,782]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924,965]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154,478]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566,779]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188,563]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591,423]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231,308]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389,819]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823,789]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129,763]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542,745]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949,173]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440,509]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930,573]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880,279]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490,852]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262,926]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758,104]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764,558]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898,161]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591,702]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313,282]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798,808]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786,152]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600,968]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899,451]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52,185]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374,985]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481,820]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453,660]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519,737]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625,434]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375,328]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558,956]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495,354]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753,624]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834,212]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8,550]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45,478]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853,643]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270,35]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651,368]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278,926]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577,509]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925,474]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949,374]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54,931]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206,682]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859,30]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148,668]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152,797]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37,485]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528,722]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95,789]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738,277]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137,934]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324,134]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250,362]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117,172]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699,610]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428,267]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517,684]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37,639]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915,570]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161,901]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57,217]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218,916]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693,953]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623,659]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830,762]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593,267]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124,107]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185,356]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887,376]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700,339]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62,817]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903,900]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641,751]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738,873]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184,127]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267,639]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398,521]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494,624]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533,674]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329,848]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33,312]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878,978]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333,841]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389,486]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177,545]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700,521]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487,976]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557,630]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608,755]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874,902]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984,622]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469,193]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642,226]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73,628]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794,682]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952,421]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195,581]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753,180]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811,413]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54,796]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667,318]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269,455]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994,16]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917,525]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180,482]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725,991]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286,280]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599,364]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243,264]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633,596]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697,763]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924,667]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983,281]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857,899]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954,94]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578,386]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148,33]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249,41]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160,888]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570,43]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296,735]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958,580]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92,12]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149,401]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74,307]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364,8]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222,749]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519,967]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35,620]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725,118]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446,798]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485,478]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137,140]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308,111]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225,841]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467,432]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205,580]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538,46]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920,830]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219,211]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416,45]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275,599]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296,337]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849,357]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236,318]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887,57]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701,887]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27,214]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858,138]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897,449]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610,665]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500,631]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660,857]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279,317]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720,135]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372,890]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453,732]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722,748]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869,150]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715,221]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119,818]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198,291]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,385,506]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605,214]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708,6]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858,16]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547,850]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170,954]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63,859]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402,205]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464,902]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170,65]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585,260]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798,658]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796,56]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795,783]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943,554]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938,600]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168,302]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755,330]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954,315]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371,402]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229,506]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528,875]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673,569]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103,763]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666,352]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247,620]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795,347]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774,192]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367,718]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345,599]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652,821]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154,121]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911,638]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917,622]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115,248]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333,161]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970,639]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451,290]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268,525]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828,276]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488,804]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300,989]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664,489]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153,922]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468,180]
+  CRUSH rule 0 x 699 [450,244,180,919,183,332,747,453]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68,641]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585,513]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466,687]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796,607]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655,595]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372,253]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923,770]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2,133]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939,331]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733,607]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449,28]
+  CRUSH rule 0 x 711 [227,653,731,150,156,842,534,110]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895,459]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53,839]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875,649]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322,417]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138,197]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237,62]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544,891]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828,975]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214,76]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177,264]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823,833]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985,846]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616,879]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359,983]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966,535]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105,31]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753,760]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572,933]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760,800]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528,890]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368,271]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880,343]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535,627]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690,59]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845,278]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482,564]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429,397]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146,17]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214,494]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863,571]
+  CRUSH rule 0 x 742 [912,581,114,117,730,21,687,81]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451,631]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843,699]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926,314]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450,651]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879,857]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304,999]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383,506]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169,171]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388,17]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88,315]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481,702]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336,724]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192,279]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850,191]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121,690]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165,686]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15,347]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345,69]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373,361]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664,350]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387,775]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70,202]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402,123]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357,310]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251,951]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645,975]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213,527]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961,324]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933,571]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891,959]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783,951]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890,425]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885,976]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396,76]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563,860]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65,954]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772,645]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17,336]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714,954]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812,516]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626,961]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481,198]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632,151]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490,481]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600,528]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194,216]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237,916]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788,164]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124,128]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406,858]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376,812]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811,459]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268,94]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646,451]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596,196]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473,250]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859,994]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941,643]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219,662]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846,53]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834,881]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663,949]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121,157]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640,607]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802,136]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166,929]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23,145]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620,648]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948,202]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675,28]
+  CRUSH rule 0 x 813 [805,114,683,629,801,462,285,450]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401,266]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314,670]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44,822]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260,172]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769,826]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523,770]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535,787]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621,815]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883,112]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282,641]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559,886]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161,998]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910,63]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212,916]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937,595]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669,330]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448,55]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142,157]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656,135]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764,766]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83,483]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537,937]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351,961]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903,29]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983,998]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47,859]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815,109]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226,46]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575,1]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486,199]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899,382]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611,512]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606,237]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363,893]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700,249]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847,463]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358,35]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95,337]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469,511]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774,679]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441,302]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686,702]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224,810]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12,763]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688,61]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463,903]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699,907]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901,487]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490,409]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172,980]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43,15]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850,40]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255,481]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988,730]
+  CRUSH rule 0 x 868 [613,950,712,663,666,460,643,547]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718,431]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54,992]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387,515]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180,800]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199,358]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93,915]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634,435]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322,141]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181,460]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329,591]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145,750]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319,630]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570,109]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974,483]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202,148]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389,381]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174,972]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56,582]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472,947]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806,53]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399,382]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996,561]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706,701]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53,333]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672,320]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648,685]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599,829]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210,529]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12,481]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342,37]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124,653]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380,419]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157,292]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178,624]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66,738]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957,801]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685,6]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30,169]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664,331]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484,391]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686,965]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824,552]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789,255]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226,889]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925,746]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70,280]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126,79]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255,133]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725,707]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583,923]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888,980]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532,124]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633,290]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952,474]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964,384]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584,31]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969,168]
+  CRUSH rule 0 x 926 [968,133,132,144,814,155,709,158]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465,775]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847,752]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744,171]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832,654]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487,839]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530,462]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568,61]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533,116]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772,49]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138,802]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630,308]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286,395]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408,49]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25,883]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964,28]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,171,16]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759,851]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678,915]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952,885]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933,267]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922,868]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81,220]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799,961]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355,220]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829,408]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16,746]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141,619]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129,696]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536,766]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169,751]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393,281]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346,377]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543,519]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855,49]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455,708]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856,477]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485,701]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361,255]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59,229]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929,773]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19,674]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433,520]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518,956]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576,411]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344,520]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754,540]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549,524]
+  CRUSH rule 0 x 974 [545,758,586,596,756,790,116,993]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330,308]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397,58]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783,828]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514,169]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677,173]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398,943]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536,212]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395,198]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927,597]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950,840]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911,208]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864,555]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76,894]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865,47]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155,380]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340,526]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754,928]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117,546]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382,428]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557,378]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158,265]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459,764]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210,698]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786,762]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15,585]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317,102]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739,922]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729,370]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614,783]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891,113]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597,969]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234,945]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5,444]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975,723]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234,383]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23,241]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48,570]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268,647]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808,351]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969,81]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145,11]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456,560]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637,731]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463,926]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768,245]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495,865]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676,322]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871,828]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50,560]
+  rule 0 (data) num_rep 8 result size == 8:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966,750,695]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672,820,782]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527,901,106]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481,169,816]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847,220,87]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964,872,66]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884,726,456]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578,14,543]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462,207,759]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3,569,603]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811,484,296]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380,821,470]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117,393,435]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78,170,333]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855,76,269]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437,72,680]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786,969,378]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935,605,228]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883,394,188]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51,834,690]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975,636,387]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263,267,128]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557,633,812]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477,468,303]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800,309,622]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248,908,624]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501,909,219]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676,560,753]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770,569,359]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719,581,816]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426,522,208]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813,217,750]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428,63,801]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618,288,846]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663,877,177]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75,580,568]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676,524,510]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311,758,201]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929,910,698]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856,248,279]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980,503,287]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930,74,844]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456,713,348]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514,40,261]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160,527,250]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658,876,201]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45,390,546]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959,800,743]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738,47,812]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822,628,696]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83,344,713]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406,281,991]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304,899,629]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882,503,327]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339,902,81]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508,763,108]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77,500,553]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670,67,114]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8,762,842]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643,511,370]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194,277,985]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812,14,55]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45,723,926]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705,268,140]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451,891,654]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862,221,66]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334,656,719]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783,593,814]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705,114,981]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871,361,533]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684,411,114]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587,264,446]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642,529,966]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835,705,839]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633,560,565]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366,258,711]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735,953,188]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197,690,426]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183,917,464]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339,24,581]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51,766,822]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247,963,0]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580,51,939]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66,197,698]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681,618,723]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567,908,514]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38,326,385]
+  CRUSH rule 0 x 87 [997,317,463,626,685,683,909,49,28]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99,737,942]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320,921,812]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412,184,529]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495,299,812]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167,634,781]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343,949,51]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82,422,738]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648,661,46]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204,750,322]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83,542,131]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270,511,586]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37,138,326]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9,168,590]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12,369,9]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923,106,653]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933,231,520]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401,85,139]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434,121,488]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878,493,550]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589,614,814]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631,996,630]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856,296,122]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510,472,270]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18,949,341]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725,413,187]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738,253,213]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598,418,641]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421,39,241]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9,448,816]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725,142,304]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766,210,528]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570,278,867]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49,126,410]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424,868,469]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551,43,590]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877,248,753]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482,254,158]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736,737,508]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689,209,99]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909,809,553]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666,691,899]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149,146,904]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270,425,864]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203,667,528]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922,597,680]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14,444,709]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893,281,226]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712,649,27]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846,890,151]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363,323,153]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400,885,907]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384,58,373]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178,532,883]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193,258,445]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424,820,751]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266,70,210]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697,610,203]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577,348,1]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929,196,880]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514,546,861]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352,820,857]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406,556,532]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484,553,315]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671,883,204]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807,50,206]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942,864,388]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443,98,27]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58,641,186]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271,229,961]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384,446,836]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899,902,956]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209,641,228]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242,999,974]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909,642,188]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148,88,764]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587,854,482]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110,329,361]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524,389,185]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548,676,448]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935,266,883]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667,463,818]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601,909,60]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835,840,548]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100,910,446]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36,356,274]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471,529,359]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780,932,573]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51,211,409]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240,316,337]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756,647,45]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856,992,696]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724,923,450]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736,846,926]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168,224,705]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485,571,796]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109,903,96]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74,45,438]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485,511,14]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369,270,16]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566,378,816]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833,650,761]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37,1,939]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527,237,295]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999,754,339]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123,759,147]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359,721,996]
+  CRUSH rule 0 x 194 [242,473,58,655,653,277,792,887,561]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755,471,134]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413,834,832]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666,316,243]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355,574,781]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960,369,584]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132,271,128]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495,76,598]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153,272,944]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669,676,762]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697,881,64]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776,463,327]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230,367,890]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76,708,101]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96,841,811]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522,185,949]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14,921,544]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922,372,581]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857,894,60]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687,903,620]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579,79,258]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982,826,488]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973,898,556]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264,6,603]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34,497,108]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697,805,976]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686,982,0]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61,559,846]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597,904,897]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465,429,614]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329,320,645]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606,465,354]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876,984,491]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966,217,501]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261,389,281]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262,884,215]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93,880,959]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50,182,541]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106,877,130]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796,835,714]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103,95,634]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471,753,350]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573,128,154]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327,509,325]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894,884,824]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426,427,736]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450,677,659]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413,136,411]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234,756,496]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799,21,13]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785,107,816]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605,837,1]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989,999,138]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590,840,586]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713,237,468]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886,650,827]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270,619,450]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58,354,16]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838,131,387]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797,675,452]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605,453,987]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607,95,121]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189,627,592]
+  CRUSH rule 0 x 257 [825,264,867,869,529,409,291,732,224]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863,427,873]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77,191,669]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533,646,475]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516,136,87]
+  CRUSH rule 0 x 262 [223,1,561,420,229,16,88,534,289]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773,979,719]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715,916,633]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833,987,445]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7,557,76]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972,645,882]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715,536,983]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146,14,414]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476,571,314]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263,138,526]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710,779,161]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982,735,58]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370,990,320]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992,823,321]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634,98,457]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783,648,927]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350,600,283]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952,250,206]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895,914,892]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443,604,619]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939,879,338]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369,544,721]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60,546,39]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582,975,810]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699,627,725]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137,890,487]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195,538,356]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795,377,825]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473,209,59]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557,86,89]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806,911,738]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775,636,232]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459,833,503]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440,73,29]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266,82,671]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232,343,831]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213,286,61]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924,106,63]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904,838,458]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220,432,393]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400,365,697]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561,905,522]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214,944,845]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685,838,3]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761,648,867]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459,250,415]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976,551,126]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857,153,970]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90,855,441]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256,798,743]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497,929,92]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24,81,215]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952,599,222]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749,841,969]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761,725,923]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892,86,214]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290,97,529]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800,868,364]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523,366,891]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868,841,563]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187,380,947]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236,810,969]
+  CRUSH rule 0 x 324 [16,745,511,439,272,513,668,959,845]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462,268,124]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461,509,75]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750,669,296]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33,806,181]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516,713,223]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420,520,395]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624,649,626]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644,821,238]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243,633,309]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408,175,189]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635,367,997]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698,533,683]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956,907,464]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69,11,624]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607,953,125]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707,328,325]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720,503,209]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516,661,985]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660,655,149]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142,114,108]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127,732,290]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419,962,304]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783,661,622]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862,513,112]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795,143,529]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841,164,765]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766,775,518]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899,665,802]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279,616,892]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612,33,122]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985,412,599]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629,228,150]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891,656,196]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874,643,550]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397,52,702]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29,529,156]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643,286,552]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265,35,662]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444,613,919]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203,17,743]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113,373,426]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59,914,521]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685,641,894]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616,93,283]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284,184,718]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850,419,307]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797,547,778]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212,241,346]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128,207,88]
+  CRUSH rule 0 x 374 [191,887,920,928,223,714,961,760,571]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934,619,527]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446,897,460]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654,487,891]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728,860,968]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115,557,165]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255,806,429]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773,496,433]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276,56,278]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29,197,822]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488,435,223]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910,329,396]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232,692,742]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164,934,297]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474,267,640]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340,222,74]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852,196,469]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,879,136,818]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901,82,695]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652,543,10]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15,591,420]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126,179,607]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726,935,278]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996,558,602]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349,672,826]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749,410,815]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536,592,886]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79,32,343]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658,449,886]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202,650,501]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299,577,563]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776,296,19]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718,578,580]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72,345,709]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246,695,76]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213,806,168]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949,507,361]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313,351,497]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560,155,661]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618,907,9]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97,124,229]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694,332,572]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889,344,838]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138,374,470]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848,972,781]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823,884,832]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506,670,285]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764,274,623]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839,547,599]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905,588,312]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785,395,884]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80,236,988]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791,565,352]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334,588,571]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225,94,303]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857,562,976]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859,699,91]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194,366,279]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513,591,599]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52,897,705]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644,646,747]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371,731,906]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10,976,284]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443,23,696]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755,841,94]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414,152,331]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800,174,117]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205,236,86]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864,799,762]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774,350,37]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182,145,347]
+  CRUSH rule 0 x 445 [485,482,528,209,964,753,554,931,638]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314,714,212]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58,531,827]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807,688,21]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258,548,905]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721,31,312]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878,33,3]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604,49,922]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514,5,494]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106,103,511]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295,967,132]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504,96,890]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762,975,485]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399,558,15]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147,483,517]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841,512,70]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385,797,917]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240,911,375]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538,111,232]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789,0,653]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137,466,531]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659,35,797]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187,640,601]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201,249,555]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681,683,551]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558,71,676]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256,978,409]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809,56,476]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756,663,167]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446,333,245]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237,764,682]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984,226,333]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753,794,237]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941,843,751]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158,489,515]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371,807,687]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135,375,613]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731,233,640]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448,437,219]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283,170,278]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866,661,581]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621,955,400]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857,782,24]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924,965,25]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154,478,512]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566,779,845]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188,563,490]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591,423,668]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231,308,474]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389,819,956]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823,789,217]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129,763,39]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542,745,131]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949,173,398]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440,509,295]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930,573,357]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880,279,616]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490,852,962]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262,926,316]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758,104,831]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764,558,891]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898,161,78]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591,702,346]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313,282,207]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798,808,139]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786,152,921]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600,968,355]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899,451,415]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52,185,197]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374,985,944]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481,820,517]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453,660,93]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519,737,20]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625,434,527]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375,328,3]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558,956,664]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495,354,665]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753,624,769]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834,212,906]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8,550,769]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45,478,911]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853,643,625]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270,35,809]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651,368,118]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278,926,280]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577,509,195]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925,474,669]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949,374,988]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54,931,4]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206,682,627]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859,30,61]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148,668,436]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152,797,492]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37,485,88]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528,722,59]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95,789,348]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738,277,225]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137,934,547]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324,134,839]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250,362,614]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117,172,764]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699,610,785]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428,267,229]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517,684,154]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37,639,934]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915,570,546]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161,901,342]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57,217,21]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218,916,561]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693,953,606]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623,659,403]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830,762,691]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593,267,17]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124,107,766]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185,356,87]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887,376,658]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700,339,687]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62,817,766]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903,900,812]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641,751,930]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738,873,47]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184,127,448]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267,639,646]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398,521,59]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494,624,352]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533,674,23]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329,848,235]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33,312,202]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878,978,1]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333,841,193]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389,486,10]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177,545,839]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700,521,825]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487,976,977]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557,630,468]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608,755,326]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874,902,958]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984,622,175]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469,193,737]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642,226,122]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73,628,884]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794,682,127]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952,421,875]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195,581,649]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753,180,160]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811,413,436]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54,796,369]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667,318,496]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269,455,168]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994,16,777]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917,525,945]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180,482,449]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725,991,955]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286,280,507]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599,364,546]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243,264,31]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633,596,988]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697,763,442]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924,667,356]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983,281,292]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857,899,349]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954,94,88]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578,386,226]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148,33,420]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249,41,778]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160,888,889]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570,43,362]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296,735,245]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958,580,197]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92,12,62]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149,401,14]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74,307,976]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364,8,342]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222,749,697]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519,967,737]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35,620,268]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725,118,875]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446,798,869]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485,478,185]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137,140,488]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308,111,207]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225,841,868]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467,432,944]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205,580,972]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538,46,300]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920,830,171]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219,211,157]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416,45,624]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275,599,219]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296,337,676]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849,357,317]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236,318,228]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887,57,16]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701,887,355]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27,214,226]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858,138,726]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897,449,42]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610,665,299]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500,631,833]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660,857,123]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279,317,414]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720,135,369]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372,890,30]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453,732,262]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722,748,599]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869,150,743]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715,221,31]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119,818,459]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198,291,71]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,385,506,546]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605,214,833]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708,6,746]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858,16,128]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547,850,499]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170,954,22]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63,859,215]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402,205,609]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464,902,276]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170,65,906]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585,260,254]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798,658,28]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796,56,736]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795,783,386]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943,554,395]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938,600,517]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168,302,681]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755,330,584]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954,315,735]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371,402,725]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229,506,565]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528,875,982]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673,569,701]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103,763,527]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666,352,576]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247,620,437]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795,347,831]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774,192,454]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367,718,345]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345,599,30]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652,821,641]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154,121,843]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911,638,690]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917,622,156]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115,248,566]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333,161,74]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970,639,835]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451,290,789]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268,525,425]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828,276,264]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488,804,501]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300,989,804]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664,489,598]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153,922,986]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468,180,783]
+  CRUSH rule 0 x 699 [450,244,180,919,183,332,747,453,519]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68,641,109]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585,513,907]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466,687,742]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796,607,894]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655,595,888]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372,253,807]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923,770,516]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2,133,256]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939,331,41]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733,607,763]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449,28,731]
+  CRUSH rule 0 x 711 [227,653,731,150,156,842,534,110,639]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895,459,341]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53,839,56]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875,649,23]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322,417,973]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138,197,32]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237,62,327]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544,891,210]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828,975,119]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214,76,870]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177,264,867]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823,833,1]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985,846,77]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616,879,480]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359,983,720]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966,535,669]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105,31,506]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753,760,254]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572,933,434]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760,800,945]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528,890,625]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368,271,778]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880,343,590]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535,627,224]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690,59,294]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845,278,559]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482,564,259]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429,397,872]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146,17,314]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214,494,593]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863,571,876]
+  CRUSH rule 0 x 742 [912,581,114,117,730,21,687,81,145]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451,631,911]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843,699,24]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926,314,771]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450,651,105]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879,857,727]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304,999,685]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383,506,570]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169,171,9]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388,17,892]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88,315,728]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481,702,431]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336,724,262]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192,279,371]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850,191,991]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121,690,544]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165,686,449]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15,347,311]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345,69,39]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373,361,684]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664,350,339]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387,775,797]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70,202,557]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402,123,902]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357,310,890]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251,951,675]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645,975,947]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213,527,546]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961,324,309]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933,571,822]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891,959,487]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783,951,982]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890,425,599]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885,976,936]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396,76,680]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563,860,388]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65,954,125]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772,645,254]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17,336,147]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714,954,271]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812,516,774]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626,961,576]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481,198,171]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632,151,586]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490,481,627]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600,528,966]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194,216,486]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237,916,374]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788,164,691]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124,128,742]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406,858,694]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376,812,886]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811,459,344]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268,94,648]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646,451,67]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596,196,93]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473,250,839]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859,994,831]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941,643,398]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219,662,142]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846,53,783]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834,881,23]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663,949,380]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121,157,934]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640,607,790]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802,136,305]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166,929,234]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23,145,184]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620,648,433]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948,202,521]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675,28,263]
+  CRUSH rule 0 x 813 [805,114,683,629,801,462,285,450,948]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401,266,905]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314,670,928]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44,822,332]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260,172,193]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769,826,519]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523,770,56]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535,787,267]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621,815,313]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883,112,601]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282,641,216]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559,886,753]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161,998,370]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910,63,288]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212,916,692]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937,595,935]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669,330,822]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448,55,965]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142,157,957]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656,135,341]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764,766,853]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83,483,811]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537,937,101]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351,961,52]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903,29,454]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983,998,320]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47,859,402]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815,109,762]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226,46,918]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575,1,312]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486,199,966]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899,382,851]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611,512,855]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606,237,414]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363,893,592]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700,249,214]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847,463,995]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358,35,913]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95,337,677]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469,511,999]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774,679,616]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441,302,437]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686,702,919]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224,810,404]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12,763,594]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688,61,545]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463,903,272]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699,907,532]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901,487,725]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490,409,15]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172,980,772]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43,15,29]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850,40,170]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255,481,435]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988,730,753]
+  CRUSH rule 0 x 868 [613,950,712,663,666,460,643,547,734]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718,431,968]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54,992,726]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387,515,98]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180,800,521]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199,358,805]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93,915,801]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634,435,778]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322,141,82]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181,460,459]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329,591,59]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145,750,54]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319,630,888]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570,109,896]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974,483,412]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202,148,129]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389,381,347]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174,972,544]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56,582,915]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472,947,35]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806,53,385]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399,382,761]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996,561,485]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706,701,501]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53,333,458]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672,320,394]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648,685,895]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599,829,809]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210,529,491]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12,481,539]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342,37,303]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124,653,574]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380,419,667]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157,292,724]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178,624,733]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66,738,419]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957,801,74]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685,6,689]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30,169,506]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664,331,184]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484,391,841]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686,965,515]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824,552,717]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789,255,896]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226,889,69]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925,746,664]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70,280,837]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126,79,652]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255,133,892]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725,707,47]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583,923,863]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888,980,33]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532,124,911]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633,290,155]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952,474,67]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964,384,454]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584,31,151]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969,168,500]
+  CRUSH rule 0 x 926 [968,133,132,144,814,155,709,158,96]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465,775,725]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847,752,780]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744,171,144]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832,654,567]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487,839,776]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530,462,202]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568,61,945]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533,116,254]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772,49,567]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138,802,885]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630,308,973]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286,395,927]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408,49,977]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25,883,867]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964,28,633]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,171,16,440]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759,851,617]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678,915,619]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952,885,987]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933,267,846]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922,868,413]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81,220,725]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799,961,564]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355,220,528]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829,408,568]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16,746,835]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141,619,234]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129,696,115]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536,766,579]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169,751,667]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393,281,227]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346,377,425]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543,519,496]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855,49,548]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455,708,649]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856,477,895]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485,701,76]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361,255,507]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59,229,819]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929,773,928]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19,674,395]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433,520,916]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518,956,398]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576,411,416]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344,520,546]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754,540,79]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549,524,109]
+  CRUSH rule 0 x 974 [545,758,586,596,756,790,116,993,644]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330,308,744]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397,58,57]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783,828,409]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514,169,99]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677,173,903]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398,943,764]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536,212,247]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395,198,100]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927,597,103]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950,840,427]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911,208,91]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864,555,687]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76,894,480]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865,47,116]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155,380,797]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340,526,728]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754,928,288]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117,546,798]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382,428,451]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557,378,936]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158,265,110]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459,764,859]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210,698,273]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786,762,444]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15,585,898]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317,102,200]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739,922,343]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729,370,326]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614,783,865]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891,113,424]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597,969,911]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234,945,598]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5,444,795]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975,723,909]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234,383,604]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23,241,838]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48,570,639]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268,647,669]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808,351,572]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969,81,586]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145,11,965]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456,560,935]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637,731,738]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463,926,475]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768,245,337]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495,865,698]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676,322,671]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871,828,440]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50,560,595]
+  rule 0 (data) num_rep 9 result size == 9:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [36,705,536,450,604,380,966,750,695,503]
+  CRUSH rule 0 x 1 [876,250,334,633,744,843,672,820,782,802]
+  CRUSH rule 0 x 2 [292,832,53,392,386,787,527,901,106,273]
+  CRUSH rule 0 x 3 [623,387,124,998,749,211,481,169,816,732]
+  CRUSH rule 0 x 4 [61,334,710,4,994,982,847,220,87,254]
+  CRUSH rule 0 x 5 [946,557,713,664,141,817,964,872,66,161]
+  CRUSH rule 0 x 6 [576,668,212,163,732,381,884,726,456,796]
+  CRUSH rule 0 x 7 [645,753,906,393,341,44,578,14,543,287]
+  CRUSH rule 0 x 8 [243,6,863,781,211,100,462,207,759,701]
+  CRUSH rule 0 x 9 [22,578,251,410,297,430,3,569,603,47]
+  CRUSH rule 0 x 10 [758,828,360,477,821,801,811,484,296,320]
+  CRUSH rule 0 x 11 [769,120,124,527,119,504,380,821,470,230]
+  CRUSH rule 0 x 12 [780,364,689,755,675,199,117,393,435,514]
+  CRUSH rule 0 x 13 [557,18,351,719,742,780,78,170,333,295]
+  CRUSH rule 0 x 14 [59,561,249,461,971,835,855,76,269,673]
+  CRUSH rule 0 x 15 [718,928,993,21,76,313,437,72,680,761]
+  CRUSH rule 0 x 16 [673,632,841,954,788,90,786,969,378,246]
+  CRUSH rule 0 x 17 [648,43,560,514,142,289,935,605,228,737]
+  CRUSH rule 0 x 18 [654,219,181,568,381,253,883,394,188,459]
+  CRUSH rule 0 x 19 [850,545,377,848,863,543,51,834,690,375]
+  CRUSH rule 0 x 20 [717,785,974,5,225,552,975,636,387,600]
+  CRUSH rule 0 x 21 [420,57,519,306,312,983,263,267,128,828]
+  CRUSH rule 0 x 22 [503,998,193,821,634,684,557,633,812,521]
+  CRUSH rule 0 x 23 [411,663,168,110,899,488,477,468,303,367]
+  CRUSH rule 0 x 24 [266,861,353,1,456,128,800,309,622,673]
+  CRUSH rule 0 x 25 [760,483,818,600,509,951,248,908,624,643]
+  CRUSH rule 0 x 26 [903,24,573,718,112,694,501,909,219,686]
+  CRUSH rule 0 x 27 [946,188,289,510,687,827,676,560,753,77]
+  CRUSH rule 0 x 28 [69,312,73,198,256,629,770,569,359,733]
+  CRUSH rule 0 x 29 [844,883,337,628,496,405,719,581,816,349]
+  CRUSH rule 0 x 30 [621,18,613,794,910,936,426,522,208,699]
+  CRUSH rule 0 x 31 [784,943,814,539,962,392,813,217,750,155]
+  CRUSH rule 0 x 32 [173,374,369,972,315,83,428,63,801,735]
+  CRUSH rule 0 x 33 [698,336,357,966,582,407,618,288,846,659]
+  CRUSH rule 0 x 34 [168,836,210,798,904,190,663,877,177,567]
+  CRUSH rule 0 x 35 [274,509,534,818,912,671,75,580,568,523]
+  CRUSH rule 0 x 36 [318,215,153,628,87,407,676,524,510,480]
+  CRUSH rule 0 x 37 [173,604,109,935,203,401,311,758,201,999]
+  CRUSH rule 0 x 38 [708,444,683,604,722,900,929,910,698,386]
+  CRUSH rule 0 x 39 [662,198,417,680,226,342,856,248,279,416]
+  CRUSH rule 0 x 40 [620,801,414,78,560,766,980,503,287,564]
+  CRUSH rule 0 x 41 [811,264,177,127,148,791,930,74,844,943]
+  CRUSH rule 0 x 42 [863,179,527,660,133,529,456,713,348,311]
+  CRUSH rule 0 x 43 [686,822,988,228,791,549,514,40,261,223]
+  CRUSH rule 0 x 44 [396,222,46,841,536,140,160,527,250,247]
+  CRUSH rule 0 x 45 [991,694,253,142,54,422,658,876,201,45]
+  CRUSH rule 0 x 46 [420,909,184,285,508,458,45,390,546,908]
+  CRUSH rule 0 x 47 [467,211,605,207,241,881,959,800,743,161]
+  CRUSH rule 0 x 48 [955,329,368,168,698,787,738,47,812,166]
+  CRUSH rule 0 x 49 [974,891,931,29,813,506,822,628,696,407]
+  CRUSH rule 0 x 50 [870,441,691,823,761,6,83,344,713,857]
+  CRUSH rule 0 x 51 [182,930,25,936,97,260,406,281,991,336]
+  CRUSH rule 0 x 52 [704,812,894,794,481,37,304,899,629,701]
+  CRUSH rule 0 x 53 [185,713,631,280,345,558,882,503,327,402]
+  CRUSH rule 0 x 54 [270,441,100,82,983,930,339,902,81,239]
+  CRUSH rule 0 x 55 [895,734,958,793,651,572,508,763,108,185]
+  CRUSH rule 0 x 56 [564,963,683,324,40,189,77,500,553,417]
+  CRUSH rule 0 x 57 [738,130,208,973,498,861,670,67,114,685]
+  CRUSH rule 0 x 58 [524,113,806,903,531,334,8,762,842,884]
+  CRUSH rule 0 x 59 [408,337,668,529,34,384,643,511,370,336]
+  CRUSH rule 0 x 60 [228,790,857,309,616,895,194,277,985,554]
+  CRUSH rule 0 x 61 [154,843,717,467,883,536,812,14,55,752]
+  CRUSH rule 0 x 62 [594,811,549,276,693,917,45,723,926,180]
+  CRUSH rule 0 x 63 [646,67,884,925,941,434,705,268,140,942]
+  CRUSH rule 0 x 64 [175,542,155,837,594,197,451,891,654,294]
+  CRUSH rule 0 x 65 [745,619,131,867,269,62,862,221,66,354]
+  CRUSH rule 0 x 66 [275,468,23,35,328,432,334,656,719,810]
+  CRUSH rule 0 x 67 [246,958,524,493,636,227,783,593,814,970]
+  CRUSH rule 0 x 68 [711,473,403,228,835,126,705,114,981,267]
+  CRUSH rule 0 x 69 [493,924,850,939,950,105,871,361,533,433]
+  CRUSH rule 0 x 70 [30,499,644,33,804,654,684,411,114,42]
+  CRUSH rule 0 x 71 [984,883,574,716,575,391,587,264,446,572]
+  CRUSH rule 0 x 72 [71,286,942,363,628,632,642,529,966,919]
+  CRUSH rule 0 x 73 [922,618,3,371,464,442,835,705,839,745]
+  CRUSH rule 0 x 74 [629,414,185,573,678,338,633,560,565,410]
+  CRUSH rule 0 x 75 [222,20,174,820,312,361,366,258,711,657]
+  CRUSH rule 0 x 76 [262,366,339,290,718,143,735,953,188,516]
+  CRUSH rule 0 x 77 [638,469,992,280,773,892,197,690,426,681]
+  CRUSH rule 0 x 78 [324,511,788,7,308,228,183,917,464,518]
+  CRUSH rule 0 x 79 [577,990,64,94,447,924,339,24,581,969]
+  CRUSH rule 0 x 80 [501,95,278,903,631,842,51,766,822,687]
+  CRUSH rule 0 x 81 [506,812,9,698,173,664,247,963,0,732]
+  CRUSH rule 0 x 82 [222,145,80,785,835,745,580,51,939,278]
+  CRUSH rule 0 x 83 [71,634,61,91,856,529,66,197,698,318]
+  CRUSH rule 0 x 84 [49,761,773,368,318,708,681,618,723,516]
+  CRUSH rule 0 x 85 [985,896,708,861,325,307,567,908,514,355]
+  CRUSH rule 0 x 86 [537,745,93,524,466,356,38,326,385,899]
+  CRUSH rule 0 x 87 [997,317,463,626,685,683,909,49,28,698]
+  CRUSH rule 0 x 88 [957,350,890,857,375,176,99,737,942,647]
+  CRUSH rule 0 x 89 [399,730,148,314,159,982,320,921,812,908]
+  CRUSH rule 0 x 90 [943,706,683,267,579,141,412,184,529,127]
+  CRUSH rule 0 x 91 [22,368,149,928,140,529,495,299,812,743]
+  CRUSH rule 0 x 92 [532,424,426,773,623,197,167,634,781,242]
+  CRUSH rule 0 x 93 [218,489,405,681,549,201,343,949,51,732]
+  CRUSH rule 0 x 94 [181,96,102,515,776,365,82,422,738,933]
+  CRUSH rule 0 x 95 [343,957,820,139,334,37,648,661,46,112]
+  CRUSH rule 0 x 96 [861,270,87,797,0,245,204,750,322,75]
+  CRUSH rule 0 x 97 [459,706,45,328,274,605,83,542,131,240]
+  CRUSH rule 0 x 98 [327,867,353,948,728,280,270,511,586,230]
+  CRUSH rule 0 x 99 [974,133,468,906,235,988,37,138,326,603]
+  CRUSH rule 0 x 100 [32,445,547,371,960,885,9,168,590,873]
+  CRUSH rule 0 x 101 [142,90,337,950,970,570,12,369,9,872]
+  CRUSH rule 0 x 102 [172,129,139,22,403,867,923,106,653,999]
+  CRUSH rule 0 x 103 [630,47,161,356,911,421,933,231,520,303]
+  CRUSH rule 0 x 104 [758,133,278,11,947,799,401,85,139,855]
+  CRUSH rule 0 x 105 [843,604,47,33,401,632,434,121,488,840]
+  CRUSH rule 0 x 106 [28,681,193,679,990,343,878,493,550,484]
+  CRUSH rule 0 x 107 [74,320,85,819,315,253,589,614,814,970]
+  CRUSH rule 0 x 108 [875,593,575,517,107,153,631,996,630,597]
+  CRUSH rule 0 x 109 [411,985,811,720,198,666,856,296,122,477]
+  CRUSH rule 0 x 110 [440,774,799,660,715,167,510,472,270,753]
+  CRUSH rule 0 x 111 [405,742,276,359,936,360,18,949,341,837]
+  CRUSH rule 0 x 112 [143,181,922,545,185,303,725,413,187,840]
+  CRUSH rule 0 x 113 [153,846,160,903,789,897,738,253,213,541]
+  CRUSH rule 0 x 114 [804,892,939,20,312,692,598,418,641,891]
+  CRUSH rule 0 x 115 [588,508,958,580,232,722,421,39,241,881]
+  CRUSH rule 0 x 116 [327,148,637,486,712,464,9,448,816,609]
+  CRUSH rule 0 x 117 [95,594,989,131,714,275,725,142,304,591]
+  CRUSH rule 0 x 118 [80,957,897,239,359,432,766,210,528,252]
+  CRUSH rule 0 x 119 [386,932,951,768,679,300,570,278,867,489]
+  CRUSH rule 0 x 120 [366,312,653,936,71,241,49,126,410,33]
+  CRUSH rule 0 x 121 [129,154,847,16,471,481,424,868,469,183]
+  CRUSH rule 0 x 122 [873,1,110,939,90,412,551,43,590,51]
+  CRUSH rule 0 x 123 [533,415,789,600,713,800,877,248,753,395]
+  CRUSH rule 0 x 124 [461,691,898,723,957,759,482,254,158,641]
+  CRUSH rule 0 x 125 [342,599,830,402,615,994,736,737,508,150]
+  CRUSH rule 0 x 126 [819,781,822,548,279,255,689,209,99,479]
+  CRUSH rule 0 x 127 [437,893,585,707,353,189,909,809,553,785]
+  CRUSH rule 0 x 128 [679,994,982,550,991,324,666,691,899,665]
+  CRUSH rule 0 x 129 [380,685,947,302,698,144,149,146,904,70]
+  CRUSH rule 0 x 130 [992,52,466,867,998,777,270,425,864,38]
+  CRUSH rule 0 x 131 [469,90,208,599,829,656,203,667,528,387]
+  CRUSH rule 0 x 132 [571,250,316,535,54,418,922,597,680,25]
+  CRUSH rule 0 x 133 [964,728,329,902,108,118,14,444,709,592]
+  CRUSH rule 0 x 134 [999,19,716,963,323,559,893,281,226,739]
+  CRUSH rule 0 x 135 [634,101,52,938,413,573,712,649,27,274]
+  CRUSH rule 0 x 136 [114,889,692,768,694,279,846,890,151,872]
+  CRUSH rule 0 x 137 [839,8,959,280,922,870,363,323,153,238]
+  CRUSH rule 0 x 138 [967,949,138,451,292,548,400,885,907,214]
+  CRUSH rule 0 x 139 [308,711,736,247,632,126,384,58,373,121]
+  CRUSH rule 0 x 140 [764,936,926,55,331,115,178,532,883,380]
+  CRUSH rule 0 x 141 [423,302,112,216,603,873,193,258,445,451]
+  CRUSH rule 0 x 142 [252,821,715,340,635,668,424,820,751,746]
+  CRUSH rule 0 x 143 [33,808,518,477,325,316,266,70,210,61]
+  CRUSH rule 0 x 144 [472,88,969,162,401,771,697,610,203,382]
+  CRUSH rule 0 x 145 [242,208,252,604,266,743,577,348,1,323]
+  CRUSH rule 0 x 146 [290,70,570,384,934,856,929,196,880,458]
+  CRUSH rule 0 x 147 [447,352,657,493,467,918,514,546,861,796]
+  CRUSH rule 0 x 148 [212,644,432,658,109,275,352,820,857,282]
+  CRUSH rule 0 x 149 [9,775,87,35,260,646,406,556,532,750]
+  CRUSH rule 0 x 150 [166,456,582,144,324,340,484,553,315,504]
+  CRUSH rule 0 x 151 [811,875,307,20,782,229,671,883,204,12]
+  CRUSH rule 0 x 152 [449,617,223,9,182,407,807,50,206,368]
+  CRUSH rule 0 x 153 [523,537,695,627,959,613,942,864,388,639]
+  CRUSH rule 0 x 154 [208,559,874,597,243,706,443,98,27,120]
+  CRUSH rule 0 x 155 [569,325,192,296,367,848,58,641,186,553]
+  CRUSH rule 0 x 156 [488,121,521,213,595,837,271,229,961,163]
+  CRUSH rule 0 x 157 [140,723,633,260,487,856,384,446,836,917]
+  CRUSH rule 0 x 158 [786,451,320,239,667,632,899,902,956,424]
+  CRUSH rule 0 x 159 [134,664,517,821,667,944,209,641,228,213]
+  CRUSH rule 0 x 160 [690,112,414,990,183,590,242,999,974,652]
+  CRUSH rule 0 x 161 [324,912,397,423,991,284,909,642,188,143]
+  CRUSH rule 0 x 162 [748,567,284,183,463,336,148,88,764,40]
+  CRUSH rule 0 x 163 [575,499,31,816,749,737,587,854,482,522]
+  CRUSH rule 0 x 164 [314,489,308,326,51,568,110,329,361,743]
+  CRUSH rule 0 x 165 [116,209,750,53,813,640,524,389,185,893]
+  CRUSH rule 0 x 166 [352,706,701,810,718,527,548,676,448,991]
+  CRUSH rule 0 x 167 [27,743,174,142,551,1,935,266,883,77]
+  CRUSH rule 0 x 168 [953,898,880,660,500,799,667,463,818,819]
+  CRUSH rule 0 x 169 [912,147,266,547,331,770,601,909,60,255]
+  CRUSH rule 0 x 170 [421,515,828,844,151,981,835,840,548,588]
+  CRUSH rule 0 x 171 [488,584,880,964,936,196,100,910,446,541]
+  CRUSH rule 0 x 172 [366,443,957,66,162,693,36,356,274,976]
+  CRUSH rule 0 x 173 [863,291,625,287,158,496,471,529,359,571]
+  CRUSH rule 0 x 174 [263,555,650,410,339,616,780,932,573,814]
+  CRUSH rule 0 x 175 [875,961,361,575,33,109,51,211,409,865]
+  CRUSH rule 0 x 176 [745,83,701,680,250,420,240,316,337,361]
+  CRUSH rule 0 x 177 [128,244,41,123,422,902,756,647,45,752]
+  CRUSH rule 0 x 178 [155,41,264,777,314,564,856,992,696,784]
+  CRUSH rule 0 x 179 [593,833,202,183,971,38,724,923,450,340]
+  CRUSH rule 0 x 180 [154,734,17,831,824,522,736,846,926,129]
+  CRUSH rule 0 x 181 [289,675,723,800,166,712,168,224,705,185]
+  CRUSH rule 0 x 182 [730,931,560,209,943,261,485,571,796,587]
+  CRUSH rule 0 x 183 [639,237,794,815,827,400,109,903,96,526]
+  CRUSH rule 0 x 184 [704,312,685,645,691,778,74,45,438,26]
+  CRUSH rule 0 x 185 [97,100,762,82,999,542,485,511,14,329]
+  CRUSH rule 0 x 186 [26,665,554,215,280,421,369,270,16,920]
+  CRUSH rule 0 x 187 [649,14,740,494,402,684,566,378,816,553]
+  CRUSH rule 0 x 188 [682,695,590,743,927,945,833,650,761,468]
+  CRUSH rule 0 x 189 [325,693,726,51,448,169,37,1,939,463]
+  CRUSH rule 0 x 190 [399,933,136,955,57,504,527,237,295,716]
+  CRUSH rule 0 x 191 [629,533,17,126,60,146,999,754,339,271]
+  CRUSH rule 0 x 192 [503,578,38,492,222,251,123,759,147,99]
+  CRUSH rule 0 x 193 [546,333,651,678,823,652,359,721,996,318]
+  CRUSH rule 0 x 194 [242,473,58,655,653,277,792,887,561,449]
+  CRUSH rule 0 x 195 [625,719,135,81,636,513,755,471,134,744]
+  CRUSH rule 0 x 196 [357,114,125,867,250,522,413,834,832,368]
+  CRUSH rule 0 x 197 [306,954,453,873,211,334,666,316,243,320]
+  CRUSH rule 0 x 198 [863,791,311,911,206,61,355,574,781,550]
+  CRUSH rule 0 x 199 [935,906,929,252,893,75,960,369,584,612]
+  CRUSH rule 0 x 200 [373,774,229,454,909,611,132,271,128,632]
+  CRUSH rule 0 x 201 [659,320,477,313,779,16,495,76,598,301]
+  CRUSH rule 0 x 202 [260,433,524,880,223,818,153,272,944,741]
+  CRUSH rule 0 x 203 [36,239,675,971,703,209,669,676,762,200]
+  CRUSH rule 0 x 204 [92,516,993,728,279,478,697,881,64,107]
+  CRUSH rule 0 x 205 [68,395,473,45,683,662,776,463,327,721]
+  CRUSH rule 0 x 206 [570,530,642,380,311,398,230,367,890,953]
+  CRUSH rule 0 x 207 [834,457,850,917,456,296,76,708,101,928]
+  CRUSH rule 0 x 208 [927,484,640,976,803,626,96,841,811,979]
+  CRUSH rule 0 x 209 [878,66,58,940,48,233,522,185,949,590]
+  CRUSH rule 0 x 210 [572,981,484,29,0,426,14,921,544,4]
+  CRUSH rule 0 x 211 [107,597,780,857,895,57,922,372,581,629]
+  CRUSH rule 0 x 212 [389,107,838,624,698,562,857,894,60,426]
+  CRUSH rule 0 x 213 [497,717,567,728,905,134,687,903,620,572]
+  CRUSH rule 0 x 214 [798,65,254,572,32,393,579,79,258,42]
+  CRUSH rule 0 x 215 [233,419,283,638,520,891,982,826,488,314]
+  CRUSH rule 0 x 216 [494,464,742,523,459,174,973,898,556,293]
+  CRUSH rule 0 x 217 [352,396,309,938,66,41,264,6,603,317]
+  CRUSH rule 0 x 218 [895,864,988,650,593,740,34,497,108,180]
+  CRUSH rule 0 x 219 [222,534,277,242,658,482,697,805,976,758]
+  CRUSH rule 0 x 220 [281,19,584,563,858,965,686,982,0,32]
+  CRUSH rule 0 x 221 [64,928,963,130,312,394,61,559,846,994]
+  CRUSH rule 0 x 222 [40,544,161,199,861,644,597,904,897,376]
+  CRUSH rule 0 x 223 [645,556,159,417,46,135,465,429,614,711]
+  CRUSH rule 0 x 224 [647,165,957,263,961,576,329,320,645,829]
+  CRUSH rule 0 x 225 [219,714,858,747,461,175,606,465,354,404]
+  CRUSH rule 0 x 226 [372,511,181,277,695,404,876,984,491,784]
+  CRUSH rule 0 x 227 [925,156,714,863,257,74,966,217,501,536]
+  CRUSH rule 0 x 228 [682,404,839,263,521,195,261,389,281,467]
+  CRUSH rule 0 x 229 [880,838,770,891,236,542,262,884,215,687]
+  CRUSH rule 0 x 230 [328,659,916,468,646,572,93,880,959,111]
+  CRUSH rule 0 x 231 [320,383,669,109,627,621,50,182,541,483]
+  CRUSH rule 0 x 232 [924,846,394,319,43,519,106,877,130,387]
+  CRUSH rule 0 x 233 [948,652,575,838,498,395,796,835,714,751]
+  CRUSH rule 0 x 234 [484,943,42,575,936,180,103,95,634,844]
+  CRUSH rule 0 x 235 [750,65,590,168,870,308,471,753,350,224]
+  CRUSH rule 0 x 236 [551,787,490,136,370,833,573,128,154,326]
+  CRUSH rule 0 x 237 [390,157,166,251,752,75,327,509,325,245]
+  CRUSH rule 0 x 238 [570,6,989,707,514,905,894,884,824,343]
+  CRUSH rule 0 x 239 [729,959,376,975,496,49,426,427,736,836]
+  CRUSH rule 0 x 240 [981,241,156,767,631,576,450,677,659,183]
+  CRUSH rule 0 x 241 [310,816,641,177,996,454,413,136,411,549]
+  CRUSH rule 0 x 242 [161,63,642,837,763,458,234,756,496,779]
+  CRUSH rule 0 x 243 [180,394,33,683,189,419,799,21,13,874]
+  CRUSH rule 0 x 244 [52,174,685,189,78,310,785,107,816,89]
+  CRUSH rule 0 x 245 [523,121,915,84,386,409,605,837,1,141]
+  CRUSH rule 0 x 246 [362,893,390,487,817,88,989,999,138,674]
+  CRUSH rule 0 x 247 [382,184,116,34,143,15,590,840,586,594]
+  CRUSH rule 0 x 248 [129,114,852,469,359,291,713,237,468,340]
+  CRUSH rule 0 x 249 [159,683,91,856,475,369,886,650,827,663]
+  CRUSH rule 0 x 250 [404,945,569,955,228,910,270,619,450,707]
+  CRUSH rule 0 x 251 [661,225,738,757,37,642,58,354,16,905]
+  CRUSH rule 0 x 252 [961,226,542,103,945,885,838,131,387,664]
+  CRUSH rule 0 x 253 [651,97,225,364,189,248,797,675,452,129]
+  CRUSH rule 0 x 254 [123,33,741,692,599,11,605,453,987,316]
+  CRUSH rule 0 x 255 [314,649,891,855,517,344,607,95,121,784]
+  CRUSH rule 0 x 256 [315,215,651,126,470,849,189,627,592,241]
+  CRUSH rule 0 x 257 [825,264,867,869,529,409,291,732,224,841]
+  CRUSH rule 0 x 258 [624,789,370,723,131,982,863,427,873,223]
+  CRUSH rule 0 x 259 [602,542,70,563,947,723,77,191,669,61]
+  CRUSH rule 0 x 260 [717,878,43,56,377,481,533,646,475,686]
+  CRUSH rule 0 x 261 [145,517,20,903,786,939,516,136,87,410]
+  CRUSH rule 0 x 262 [223,1,561,420,229,16,88,534,289,498]
+  CRUSH rule 0 x 263 [462,211,405,508,787,669,773,979,719,421]
+  CRUSH rule 0 x 264 [654,471,266,662,135,564,715,916,633,121]
+  CRUSH rule 0 x 265 [302,794,704,798,659,487,833,987,445,23]
+  CRUSH rule 0 x 266 [202,132,884,209,551,984,7,557,76,987]
+  CRUSH rule 0 x 267 [282,938,657,113,672,993,972,645,882,451]
+  CRUSH rule 0 x 268 [338,309,356,278,928,797,715,536,983,688]
+  CRUSH rule 0 x 269 [738,122,266,200,894,118,146,14,414,236]
+  CRUSH rule 0 x 270 [707,982,946,196,407,804,476,571,314,538]
+  CRUSH rule 0 x 271 [705,432,364,735,512,595,263,138,526,607]
+  CRUSH rule 0 x 272 [756,545,942,56,542,449,710,779,161,222]
+  CRUSH rule 0 x 273 [197,502,527,721,239,648,982,735,58,1]
+  CRUSH rule 0 x 274 [992,44,653,573,527,702,370,990,320,52]
+  CRUSH rule 0 x 275 [544,789,170,434,23,926,992,823,321,784]
+  CRUSH rule 0 x 276 [658,467,577,268,336,5,634,98,457,487]
+  CRUSH rule 0 x 277 [143,490,880,483,928,272,783,648,927,285]
+  CRUSH rule 0 x 278 [492,647,355,282,834,64,350,600,283,422]
+  CRUSH rule 0 x 279 [517,792,604,987,527,894,952,250,206,714]
+  CRUSH rule 0 x 280 [825,740,27,848,514,750,895,914,892,149]
+  CRUSH rule 0 x 281 [224,629,120,562,616,200,443,604,619,638]
+  CRUSH rule 0 x 282 [298,661,380,416,35,585,939,879,338,786]
+  CRUSH rule 0 x 283 [311,606,208,50,913,678,369,544,721,267]
+  CRUSH rule 0 x 284 [771,466,371,743,672,119,60,546,39,71]
+  CRUSH rule 0 x 285 [693,362,404,676,797,531,582,975,810,703]
+  CRUSH rule 0 x 286 [364,477,285,167,270,617,699,627,725,389]
+  CRUSH rule 0 x 287 [591,611,828,995,170,987,137,890,487,621]
+  CRUSH rule 0 x 288 [965,541,848,796,251,668,195,538,356,523]
+  CRUSH rule 0 x 289 [225,551,948,877,219,167,795,377,825,874]
+  CRUSH rule 0 x 290 [577,762,777,751,291,349,473,209,59,346]
+  CRUSH rule 0 x 291 [160,903,477,381,490,559,557,86,89,417]
+  CRUSH rule 0 x 292 [873,598,216,666,222,228,806,911,738,969]
+  CRUSH rule 0 x 293 [100,234,874,47,28,452,775,636,232,518]
+  CRUSH rule 0 x 294 [285,943,379,520,725,547,459,833,503,207]
+  CRUSH rule 0 x 295 [938,262,880,327,687,3,440,73,29,38]
+  CRUSH rule 0 x 296 [850,327,86,472,1,776,266,82,671,320]
+  CRUSH rule 0 x 297 [951,53,99,558,753,228,232,343,831,540]
+  CRUSH rule 0 x 298 [173,336,85,766,910,657,213,286,61,961]
+  CRUSH rule 0 x 299 [598,591,315,386,895,296,924,106,63,457]
+  CRUSH rule 0 x 300 [531,957,62,459,156,538,904,838,458,828]
+  CRUSH rule 0 x 301 [823,628,23,858,629,808,220,432,393,433]
+  CRUSH rule 0 x 302 [184,80,780,871,531,211,400,365,697,497]
+  CRUSH rule 0 x 303 [521,766,222,830,988,275,561,905,522,342]
+  CRUSH rule 0 x 304 [980,127,807,507,555,245,214,944,845,895]
+  CRUSH rule 0 x 305 [153,816,22,927,696,911,685,838,3,983]
+  CRUSH rule 0 x 306 [423,739,664,753,178,431,761,648,867,488]
+  CRUSH rule 0 x 307 [997,557,682,456,479,631,459,250,415,194]
+  CRUSH rule 0 x 308 [991,874,534,465,330,284,976,551,126,307]
+  CRUSH rule 0 x 309 [860,394,724,858,246,866,857,153,970,99]
+  CRUSH rule 0 x 310 [589,818,546,201,94,653,90,855,441,736]
+  CRUSH rule 0 x 311 [477,774,225,590,830,559,256,798,743,645]
+  CRUSH rule 0 x 312 [887,853,950,354,58,23,497,929,92,639]
+  CRUSH rule 0 x 313 [802,646,447,416,557,118,24,81,215,850]
+  CRUSH rule 0 x 314 [654,974,229,511,562,916,952,599,222,763]
+  CRUSH rule 0 x 315 [767,227,28,740,828,156,749,841,969,314]
+  CRUSH rule 0 x 316 [778,83,733,359,858,319,761,725,923,461]
+  CRUSH rule 0 x 317 [184,418,642,986,939,675,892,86,214,189]
+  CRUSH rule 0 x 318 [525,410,500,543,212,95,290,97,529,220]
+  CRUSH rule 0 x 319 [476,724,569,382,409,521,800,868,364,427]
+  CRUSH rule 0 x 320 [149,610,697,296,818,955,523,366,891,998]
+  CRUSH rule 0 x 321 [710,79,667,671,234,4,868,841,563,961]
+  CRUSH rule 0 x 322 [175,275,323,333,744,718,187,380,947,952]
+  CRUSH rule 0 x 323 [819,604,638,792,316,544,236,810,969,232]
+  CRUSH rule 0 x 324 [16,745,511,439,272,513,668,959,845,759]
+  CRUSH rule 0 x 325 [486,400,872,873,251,68,462,268,124,431]
+  CRUSH rule 0 x 326 [613,765,207,19,359,370,461,509,75,767]
+  CRUSH rule 0 x 327 [125,289,738,408,456,784,750,669,296,314]
+  CRUSH rule 0 x 328 [807,383,476,583,645,141,33,806,181,597]
+  CRUSH rule 0 x 329 [588,938,599,432,446,840,516,713,223,395]
+  CRUSH rule 0 x 330 [932,644,41,611,209,406,420,520,395,665]
+  CRUSH rule 0 x 331 [341,953,950,537,578,862,624,649,626,928]
+  CRUSH rule 0 x 332 [153,726,459,950,466,804,644,821,238,85]
+  CRUSH rule 0 x 333 [745,845,853,860,52,615,243,633,309,616]
+  CRUSH rule 0 x 334 [614,751,807,58,396,159,408,175,189,500]
+  CRUSH rule 0 x 335 [518,721,221,283,454,187,635,367,997,819]
+  CRUSH rule 0 x 336 [389,424,77,309,5,898,698,533,683,851]
+  CRUSH rule 0 x 337 [753,508,765,720,221,807,956,907,464,39]
+  CRUSH rule 0 x 338 [128,810,490,753,406,760,69,11,624,272]
+  CRUSH rule 0 x 339 [430,308,58,751,856,823,607,953,125,899]
+  CRUSH rule 0 x 340 [541,44,630,231,289,966,707,328,325,81]
+  CRUSH rule 0 x 341 [402,26,631,439,165,928,720,503,209,748]
+  CRUSH rule 0 x 342 [982,57,992,461,131,32,516,661,985,860]
+  CRUSH rule 0 x 343 [833,412,572,732,107,805,660,655,149,994]
+  CRUSH rule 0 x 344 [784,533,792,41,642,869,142,114,108,961]
+  CRUSH rule 0 x 345 [546,300,304,691,763,556,127,732,290,494]
+  CRUSH rule 0 x 346 [302,420,428,891,357,124,419,962,304,12]
+  CRUSH rule 0 x 347 [488,778,101,217,366,442,783,661,622,426]
+  CRUSH rule 0 x 348 [903,744,937,718,85,314,862,513,112,334]
+  CRUSH rule 0 x 349 [471,547,582,306,600,486,795,143,529,765]
+  CRUSH rule 0 x 350 [348,221,823,335,383,708,841,164,765,563]
+  CRUSH rule 0 x 351 [961,582,705,346,361,32,766,775,518,155]
+  CRUSH rule 0 x 352 [728,137,461,298,36,903,899,665,802,620]
+  CRUSH rule 0 x 353 [904,202,184,447,58,294,279,616,892,262]
+  CRUSH rule 0 x 354 [345,226,319,256,544,311,612,33,122,190]
+  CRUSH rule 0 x 355 [50,430,175,43,187,458,985,412,599,375]
+  CRUSH rule 0 x 356 [87,185,55,423,829,1,629,228,150,889]
+  CRUSH rule 0 x 357 [762,459,921,473,182,231,891,656,196,232]
+  CRUSH rule 0 x 358 [908,25,280,6,808,676,874,643,550,633]
+  CRUSH rule 0 x 359 [484,15,132,121,394,423,397,52,702,981]
+  CRUSH rule 0 x 360 [173,378,337,702,145,499,29,529,156,595]
+  CRUSH rule 0 x 361 [404,577,115,25,56,914,643,286,552,985]
+  CRUSH rule 0 x 362 [403,1,422,945,132,685,265,35,662,708]
+  CRUSH rule 0 x 363 [639,911,510,162,418,294,444,613,919,499]
+  CRUSH rule 0 x 364 [752,689,610,990,665,222,203,17,743,570]
+  CRUSH rule 0 x 365 [956,999,212,230,624,84,113,373,426,941]
+  CRUSH rule 0 x 366 [860,925,924,763,687,851,59,914,521,629]
+  CRUSH rule 0 x 367 [205,609,647,665,969,720,685,641,894,813]
+  CRUSH rule 0 x 368 [301,284,810,169,78,340,616,93,283,353]
+  CRUSH rule 0 x 369 [452,658,339,217,674,210,284,184,718,684]
+  CRUSH rule 0 x 370 [11,467,695,989,394,576,850,419,307,965]
+  CRUSH rule 0 x 371 [124,487,55,514,313,411,797,547,778,958]
+  CRUSH rule 0 x 372 [253,48,979,846,207,631,212,241,346,153]
+  CRUSH rule 0 x 373 [715,605,775,748,227,493,128,207,88,641]
+  CRUSH rule 0 x 374 [191,887,920,928,223,714,961,760,571,549]
+  CRUSH rule 0 x 375 [711,385,651,665,15,71,934,619,527,735]
+  CRUSH rule 0 x 376 [597,818,49,458,415,755,446,897,460,869]
+  CRUSH rule 0 x 377 [294,256,933,771,184,861,654,487,891,733]
+  CRUSH rule 0 x 378 [34,151,681,707,552,127,728,860,968,475]
+  CRUSH rule 0 x 379 [869,136,315,378,813,153,115,557,165,292]
+  CRUSH rule 0 x 380 [294,97,575,791,690,482,255,806,429,306]
+  CRUSH rule 0 x 381 [119,710,219,827,328,886,773,496,433,750]
+  CRUSH rule 0 x 382 [69,631,508,706,697,168,276,56,278,772]
+  CRUSH rule 0 x 383 [922,588,589,925,471,601,29,197,822,218]
+  CRUSH rule 0 x 384 [221,945,671,117,857,655,488,435,223,783]
+  CRUSH rule 0 x 385 [561,737,953,723,658,368,910,329,396,482]
+  CRUSH rule 0 x 386 [335,442,788,696,507,716,232,692,742,939]
+  CRUSH rule 0 x 387 [514,43,353,88,100,842,164,934,297,902]
+  CRUSH rule 0 x 388 [587,89,157,996,915,927,474,267,640,53]
+  CRUSH rule 0 x 389 [109,641,255,466,372,563,340,222,74,503]
+  CRUSH rule 0 x 390 [925,149,421,489,599,810,852,196,469,672]
+  CRUSH rule 0 x 391 [267,87,387,527,768,873,879,136,818,516]
+  CRUSH rule 0 x 392 [382,485,370,849,936,636,901,82,695,640]
+  CRUSH rule 0 x 393 [425,721,221,753,268,463,652,543,10,287]
+  CRUSH rule 0 x 394 [898,18,38,793,173,738,15,591,420,525]
+  CRUSH rule 0 x 395 [806,876,269,679,32,744,126,179,607,623]
+  CRUSH rule 0 x 396 [790,970,437,449,875,395,726,935,278,138]
+  CRUSH rule 0 x 397 [136,363,507,613,11,30,996,558,602,528]
+  CRUSH rule 0 x 398 [914,116,558,258,722,904,349,672,826,569]
+  CRUSH rule 0 x 399 [261,94,299,202,174,622,749,410,815,214]
+  CRUSH rule 0 x 400 [661,197,338,461,977,848,536,592,886,981]
+  CRUSH rule 0 x 401 [953,979,287,803,41,349,79,32,343,468]
+  CRUSH rule 0 x 402 [738,819,618,522,667,334,658,449,886,260]
+  CRUSH rule 0 x 403 [573,238,425,546,130,68,202,650,501,578]
+  CRUSH rule 0 x 404 [526,848,790,253,922,820,299,577,563,37]
+  CRUSH rule 0 x 405 [582,505,330,334,201,110,776,296,19,972]
+  CRUSH rule 0 x 406 [768,324,493,60,186,165,718,578,580,249]
+  CRUSH rule 0 x 407 [260,951,437,587,692,648,72,345,709,89]
+  CRUSH rule 0 x 408 [657,81,770,734,830,821,246,695,76,647]
+  CRUSH rule 0 x 409 [498,89,182,423,672,152,213,806,168,907]
+  CRUSH rule 0 x 410 [28,793,737,352,166,645,949,507,361,615]
+  CRUSH rule 0 x 411 [684,992,60,659,769,267,313,351,497,571]
+  CRUSH rule 0 x 412 [261,958,699,950,165,14,560,155,661,678]
+  CRUSH rule 0 x 413 [891,835,297,441,384,979,618,907,9,291]
+  CRUSH rule 0 x 414 [127,459,119,965,662,594,97,124,229,641]
+  CRUSH rule 0 x 415 [272,540,631,328,609,568,694,332,572,681]
+  CRUSH rule 0 x 416 [739,617,115,530,339,371,889,344,838,541]
+  CRUSH rule 0 x 417 [106,209,157,878,117,128,138,374,470,59]
+  CRUSH rule 0 x 418 [525,441,147,390,320,300,848,972,781,361]
+  CRUSH rule 0 x 419 [603,673,615,465,266,855,823,884,832,361]
+  CRUSH rule 0 x 420 [988,213,251,226,209,245,506,670,285,2]
+  CRUSH rule 0 x 421 [761,521,748,368,923,992,764,274,623,613]
+  CRUSH rule 0 x 422 [317,160,924,548,198,709,839,547,599,779]
+  CRUSH rule 0 x 423 [137,807,168,472,619,443,905,588,312,114]
+  CRUSH rule 0 x 424 [920,37,146,263,598,748,785,395,884,360]
+  CRUSH rule 0 x 425 [277,693,285,221,478,165,80,236,988,682]
+  CRUSH rule 0 x 426 [485,936,407,854,726,524,791,565,352,949]
+  CRUSH rule 0 x 427 [242,515,9,564,174,453,334,588,571,428]
+  CRUSH rule 0 x 428 [632,635,26,473,494,478,225,94,303,757]
+  CRUSH rule 0 x 429 [641,73,465,127,171,397,857,562,976,977]
+  CRUSH rule 0 x 430 [626,585,6,387,881,583,859,699,91,148]
+  CRUSH rule 0 x 431 [697,76,753,570,964,339,194,366,279,30]
+  CRUSH rule 0 x 432 [590,526,306,283,656,728,513,591,599,474]
+  CRUSH rule 0 x 433 [284,387,149,817,886,714,52,897,705,707]
+  CRUSH rule 0 x 434 [538,985,79,953,770,468,644,646,747,123]
+  CRUSH rule 0 x 435 [30,318,593,635,975,833,371,731,906,721]
+  CRUSH rule 0 x 436 [164,919,851,693,0,874,10,976,284,126]
+  CRUSH rule 0 x 437 [322,212,163,606,302,282,443,23,696,245]
+  CRUSH rule 0 x 438 [142,392,85,594,376,419,755,841,94,52]
+  CRUSH rule 0 x 439 [119,370,68,443,997,837,414,152,331,985]
+  CRUSH rule 0 x 440 [333,403,187,863,475,844,800,174,117,518]
+  CRUSH rule 0 x 441 [477,727,906,145,429,91,205,236,86,929]
+  CRUSH rule 0 x 442 [274,590,933,244,434,49,864,799,762,611]
+  CRUSH rule 0 x 443 [983,748,574,718,700,442,774,350,37,929]
+  CRUSH rule 0 x 444 [536,509,431,146,170,149,182,145,347,172]
+  CRUSH rule 0 x 445 [485,482,528,209,964,753,554,931,638,892]
+  CRUSH rule 0 x 446 [345,634,42,294,711,376,314,714,212,646]
+  CRUSH rule 0 x 447 [61,845,767,600,321,716,58,531,827,968]
+  CRUSH rule 0 x 448 [333,232,292,846,364,951,807,688,21,841]
+  CRUSH rule 0 x 449 [680,16,484,670,851,500,258,548,905,682]
+  CRUSH rule 0 x 450 [235,214,79,423,96,822,721,31,312,491]
+  CRUSH rule 0 x 451 [961,468,333,640,823,151,878,33,3,917]
+  CRUSH rule 0 x 452 [525,479,153,528,570,806,604,49,922,414]
+  CRUSH rule 0 x 453 [138,466,302,86,249,154,514,5,494,960]
+  CRUSH rule 0 x 454 [137,625,215,402,389,914,106,103,511,624]
+  CRUSH rule 0 x 455 [173,150,997,16,846,888,295,967,132,319]
+  CRUSH rule 0 x 456 [235,226,238,258,347,784,504,96,890,230]
+  CRUSH rule 0 x 457 [450,577,253,413,717,609,762,975,485,228]
+  CRUSH rule 0 x 458 [195,537,91,814,351,90,399,558,15,441]
+  CRUSH rule 0 x 459 [381,555,312,573,915,623,147,483,517,733]
+  CRUSH rule 0 x 460 [972,730,534,678,756,692,841,512,70,914]
+  CRUSH rule 0 x 461 [506,279,142,830,784,124,385,797,917,561]
+  CRUSH rule 0 x 462 [692,959,578,57,983,299,240,911,375,412]
+  CRUSH rule 0 x 463 [788,667,949,550,685,702,538,111,232,539]
+  CRUSH rule 0 x 464 [133,122,588,999,270,880,789,0,653,566]
+  CRUSH rule 0 x 465 [971,190,230,777,452,914,137,466,531,493]
+  CRUSH rule 0 x 466 [394,576,148,157,103,822,659,35,797,235]
+  CRUSH rule 0 x 467 [517,28,366,362,984,521,187,640,601,622]
+  CRUSH rule 0 x 468 [829,143,874,225,162,413,201,249,555,646]
+  CRUSH rule 0 x 469 [987,936,106,725,633,238,681,683,551,768]
+  CRUSH rule 0 x 470 [107,982,56,889,67,65,558,71,676,655]
+  CRUSH rule 0 x 471 [181,897,629,860,307,116,256,978,409,691]
+  CRUSH rule 0 x 472 [547,512,172,24,705,837,809,56,476,137]
+  CRUSH rule 0 x 473 [760,997,824,905,888,755,756,663,167,196]
+  CRUSH rule 0 x 474 [787,418,743,628,272,341,446,333,245,689]
+  CRUSH rule 0 x 475 [662,312,253,617,105,58,237,764,682,318]
+  CRUSH rule 0 x 476 [110,495,185,508,961,837,984,226,333,916]
+  CRUSH rule 0 x 477 [393,954,834,132,841,367,753,794,237,996]
+  CRUSH rule 0 x 478 [246,483,480,644,985,420,941,843,751,451]
+  CRUSH rule 0 x 479 [70,929,697,931,744,487,158,489,515,496]
+  CRUSH rule 0 x 480 [753,119,961,607,317,717,371,807,687,932]
+  CRUSH rule 0 x 481 [470,429,677,242,574,757,135,375,613,657]
+  CRUSH rule 0 x 482 [451,566,961,675,354,746,731,233,640,492]
+  CRUSH rule 0 x 483 [816,72,371,278,635,30,448,437,219,982]
+  CRUSH rule 0 x 484 [540,454,389,31,654,494,283,170,278,77]
+  CRUSH rule 0 x 485 [74,582,624,684,566,677,866,661,581,943]
+  CRUSH rule 0 x 486 [958,595,199,763,715,973,621,955,400,261]
+  CRUSH rule 0 x 487 [228,302,804,833,876,647,857,782,24,970]
+  CRUSH rule 0 x 488 [180,529,722,956,353,890,924,965,25,925]
+  CRUSH rule 0 x 489 [47,617,812,187,291,828,154,478,512,528]
+  CRUSH rule 0 x 490 [905,822,479,124,750,843,566,779,845,507]
+  CRUSH rule 0 x 491 [892,370,609,998,433,957,188,563,490,369]
+  CRUSH rule 0 x 492 [588,959,127,948,505,936,591,423,668,365]
+  CRUSH rule 0 x 493 [353,461,593,291,301,830,231,308,474,946]
+  CRUSH rule 0 x 494 [378,848,443,368,507,423,389,819,956,597]
+  CRUSH rule 0 x 495 [845,653,768,234,405,367,823,789,217,720]
+  CRUSH rule 0 x 496 [13,988,0,691,389,757,129,763,39,651]
+  CRUSH rule 0 x 497 [796,877,788,394,648,829,542,745,131,753]
+  CRUSH rule 0 x 498 [412,337,270,705,511,227,949,173,398,586]
+  CRUSH rule 0 x 499 [330,695,8,74,618,101,440,509,295,921]
+  CRUSH rule 0 x 500 [820,272,547,765,755,96,930,573,357,491]
+  CRUSH rule 0 x 501 [110,44,132,442,294,423,880,279,616,919]
+  CRUSH rule 0 x 502 [336,595,650,274,993,312,490,852,962,387]
+  CRUSH rule 0 x 503 [922,211,157,722,502,971,262,926,316,527]
+  CRUSH rule 0 x 504 [483,52,122,432,778,461,758,104,831,710]
+  CRUSH rule 0 x 505 [482,598,224,279,480,310,764,558,891,406]
+  CRUSH rule 0 x 506 [493,123,43,856,936,622,898,161,78,414]
+  CRUSH rule 0 x 507 [12,598,264,422,416,947,591,702,346,619]
+  CRUSH rule 0 x 508 [227,157,611,301,223,746,313,282,207,626]
+  CRUSH rule 0 x 509 [807,242,363,122,582,530,798,808,139,377]
+  CRUSH rule 0 x 510 [134,437,227,75,313,351,786,152,921,884]
+  CRUSH rule 0 x 511 [212,54,83,799,457,218,600,968,355,109]
+  CRUSH rule 0 x 512 [236,630,758,752,361,249,899,451,415,920]
+  CRUSH rule 0 x 513 [994,693,644,938,846,685,52,185,197,986]
+  CRUSH rule 0 x 514 [45,508,831,19,817,52,374,985,944,101]
+  CRUSH rule 0 x 515 [504,138,480,272,530,377,481,820,517,850]
+  CRUSH rule 0 x 516 [285,409,136,570,841,610,453,660,93,134]
+  CRUSH rule 0 x 517 [300,232,23,906,438,236,519,737,20,892]
+  CRUSH rule 0 x 518 [397,674,98,898,967,113,625,434,527,630]
+  CRUSH rule 0 x 519 [86,750,772,913,101,864,375,328,3,688]
+  CRUSH rule 0 x 520 [900,833,614,130,261,885,558,956,664,468]
+  CRUSH rule 0 x 521 [31,47,236,751,911,599,495,354,665,945]
+  CRUSH rule 0 x 522 [390,16,280,144,291,175,753,624,769,853]
+  CRUSH rule 0 x 523 [618,308,424,590,300,206,834,212,906,305]
+  CRUSH rule 0 x 524 [635,189,687,963,601,518,8,550,769,975]
+  CRUSH rule 0 x 525 [311,916,699,262,775,32,45,478,911,233]
+  CRUSH rule 0 x 526 [48,738,227,718,244,942,853,643,625,43]
+  CRUSH rule 0 x 527 [202,851,889,216,763,351,270,35,809,509]
+  CRUSH rule 0 x 528 [565,827,590,273,918,106,651,368,118,1]
+  CRUSH rule 0 x 529 [934,864,241,43,466,924,278,926,280,321]
+  CRUSH rule 0 x 530 [502,934,298,670,986,360,577,509,195,722]
+  CRUSH rule 0 x 531 [681,627,942,487,288,561,925,474,669,212]
+  CRUSH rule 0 x 532 [422,6,147,205,861,141,949,374,988,367]
+  CRUSH rule 0 x 533 [863,68,364,983,247,199,54,931,4,279]
+  CRUSH rule 0 x 534 [962,931,775,172,663,119,206,682,627,827]
+  CRUSH rule 0 x 535 [89,565,397,693,839,632,859,30,61,75]
+  CRUSH rule 0 x 536 [499,351,760,458,918,86,148,668,436,192]
+  CRUSH rule 0 x 537 [676,547,787,311,867,748,152,797,492,862]
+  CRUSH rule 0 x 538 [58,644,571,649,941,7,37,485,88,273]
+  CRUSH rule 0 x 539 [837,953,457,711,458,621,528,722,59,237]
+  CRUSH rule 0 x 540 [831,50,132,213,197,709,95,789,348,342]
+  CRUSH rule 0 x 541 [582,757,121,525,532,963,738,277,225,142]
+  CRUSH rule 0 x 542 [472,132,790,997,948,269,137,934,547,351]
+  CRUSH rule 0 x 543 [382,272,797,330,315,748,324,134,839,685]
+  CRUSH rule 0 x 544 [947,930,496,883,509,219,250,362,614,123]
+  CRUSH rule 0 x 545 [425,570,305,77,821,422,117,172,764,372]
+  CRUSH rule 0 x 546 [18,65,529,437,343,547,699,610,785,811]
+  CRUSH rule 0 x 547 [445,715,600,472,213,851,428,267,229,379]
+  CRUSH rule 0 x 548 [367,569,980,167,627,442,517,684,154,108]
+  CRUSH rule 0 x 549 [125,715,671,817,285,420,37,639,934,330]
+  CRUSH rule 0 x 550 [425,599,744,199,923,222,915,570,546,724]
+  CRUSH rule 0 x 551 [44,1,528,922,944,115,161,901,342,941]
+  CRUSH rule 0 x 552 [246,104,68,239,123,427,57,217,21,70]
+  CRUSH rule 0 x 553 [71,703,615,28,593,724,218,916,561,416]
+  CRUSH rule 0 x 554 [207,124,217,166,525,226,693,953,606,894]
+  CRUSH rule 0 x 555 [570,28,317,420,931,413,623,659,403,573]
+  CRUSH rule 0 x 556 [674,152,421,79,215,347,830,762,691,951]
+  CRUSH rule 0 x 557 [347,817,191,391,741,571,593,267,17,386]
+  CRUSH rule 0 x 558 [627,426,369,692,815,371,124,107,766,260]
+  CRUSH rule 0 x 559 [940,630,924,242,224,912,185,356,87,113]
+  CRUSH rule 0 x 560 [295,903,541,29,245,753,887,376,658,366]
+  CRUSH rule 0 x 561 [506,682,384,637,878,991,700,339,687,507]
+  CRUSH rule 0 x 562 [718,529,87,729,842,341,62,817,766,376]
+  CRUSH rule 0 x 563 [552,332,747,206,274,871,903,900,812,290]
+  CRUSH rule 0 x 564 [835,769,736,486,630,209,641,751,930,856]
+  CRUSH rule 0 x 565 [8,167,539,182,607,62,738,873,47,84]
+  CRUSH rule 0 x 566 [600,481,301,263,90,450,184,127,448,327]
+  CRUSH rule 0 x 567 [999,994,509,899,947,24,267,639,646,85]
+  CRUSH rule 0 x 568 [252,431,157,62,601,863,398,521,59,250]
+  CRUSH rule 0 x 569 [643,218,943,455,83,969,494,624,352,562]
+  CRUSH rule 0 x 570 [617,635,765,422,250,156,533,674,23,683]
+  CRUSH rule 0 x 571 [757,80,59,98,328,700,329,848,235,502]
+  CRUSH rule 0 x 572 [299,348,575,889,943,675,33,312,202,355]
+  CRUSH rule 0 x 573 [25,505,270,167,58,901,878,978,1,291]
+  CRUSH rule 0 x 574 [215,431,624,177,628,814,333,841,193,146]
+  CRUSH rule 0 x 575 [225,252,611,546,32,815,389,486,10,402]
+  CRUSH rule 0 x 576 [627,94,159,857,430,691,177,545,839,722]
+  CRUSH rule 0 x 577 [237,809,778,636,61,167,700,521,825,444]
+  CRUSH rule 0 x 578 [885,313,120,344,771,614,487,976,977,58]
+  CRUSH rule 0 x 579 [924,575,787,831,47,996,557,630,468,348]
+  CRUSH rule 0 x 580 [718,51,766,121,118,471,608,755,326,604]
+  CRUSH rule 0 x 581 [219,807,129,571,856,179,874,902,958,415]
+  CRUSH rule 0 x 582 [893,701,598,863,285,829,984,622,175,804]
+  CRUSH rule 0 x 583 [246,930,964,170,993,409,469,193,737,681]
+  CRUSH rule 0 x 584 [336,432,680,175,495,839,642,226,122,703]
+  CRUSH rule 0 x 585 [324,999,397,485,457,527,73,628,884,255]
+  CRUSH rule 0 x 586 [558,230,976,541,816,72,794,682,127,372]
+  CRUSH rule 0 x 587 [985,830,597,21,308,890,952,421,875,65]
+  CRUSH rule 0 x 588 [211,544,57,134,162,496,195,581,649,488]
+  CRUSH rule 0 x 589 [129,21,112,190,885,844,753,180,160,465]
+  CRUSH rule 0 x 590 [467,969,652,593,287,76,811,413,436,162]
+  CRUSH rule 0 x 591 [758,514,316,164,35,110,54,796,369,958]
+  CRUSH rule 0 x 592 [525,253,190,443,315,603,667,318,496,74]
+  CRUSH rule 0 x 593 [601,885,339,152,297,223,269,455,168,635]
+  CRUSH rule 0 x 594 [227,60,450,30,717,840,994,16,777,901]
+  CRUSH rule 0 x 595 [720,854,496,912,80,655,917,525,945,715]
+  CRUSH rule 0 x 596 [751,195,997,77,261,490,180,482,449,647]
+  CRUSH rule 0 x 597 [129,574,714,8,789,847,725,991,955,316]
+  CRUSH rule 0 x 598 [679,207,604,396,841,284,286,280,507,912]
+  CRUSH rule 0 x 599 [668,315,683,349,681,253,599,364,546,849]
+  CRUSH rule 0 x 600 [143,396,464,444,59,57,243,264,31,897]
+  CRUSH rule 0 x 601 [326,573,873,902,136,921,633,596,988,727]
+  CRUSH rule 0 x 602 [860,281,875,535,672,474,697,763,442,542]
+  CRUSH rule 0 x 603 [709,328,445,349,190,455,924,667,356,316]
+  CRUSH rule 0 x 604 [571,62,814,95,866,978,983,281,292,953]
+  CRUSH rule 0 x 605 [252,739,860,27,313,362,857,899,349,926]
+  CRUSH rule 0 x 606 [339,236,759,842,67,644,954,94,88,617]
+  CRUSH rule 0 x 607 [590,248,759,868,433,398,578,386,226,269]
+  CRUSH rule 0 x 608 [145,635,309,467,875,115,148,33,420,669]
+  CRUSH rule 0 x 609 [973,547,223,79,762,863,249,41,778,929]
+  CRUSH rule 0 x 610 [435,816,961,983,255,886,160,888,889,767]
+  CRUSH rule 0 x 611 [559,283,422,584,176,429,570,43,362,401]
+  CRUSH rule 0 x 612 [273,149,123,576,911,270,296,735,245,714]
+  CRUSH rule 0 x 613 [828,614,642,674,33,361,958,580,197,897]
+  CRUSH rule 0 x 614 [478,748,393,34,171,80,92,12,62,719]
+  CRUSH rule 0 x 615 [392,155,144,326,626,134,149,401,14,59]
+  CRUSH rule 0 x 616 [778,637,452,248,15,888,74,307,976,613]
+  CRUSH rule 0 x 617 [622,713,996,833,611,407,364,8,342,512]
+  CRUSH rule 0 x 618 [149,877,270,329,180,327,222,749,697,853]
+  CRUSH rule 0 x 619 [604,163,656,409,322,848,519,967,737,892]
+  CRUSH rule 0 x 620 [181,23,409,198,64,898,35,620,268,902]
+  CRUSH rule 0 x 621 [735,902,386,237,939,475,725,118,875,359]
+  CRUSH rule 0 x 622 [661,824,717,568,858,583,446,798,869,586]
+  CRUSH rule 0 x 623 [142,121,643,61,695,852,485,478,185,854]
+  CRUSH rule 0 x 624 [360,716,420,398,49,717,137,140,488,725]
+  CRUSH rule 0 x 625 [541,167,385,1,601,481,308,111,207,48]
+  CRUSH rule 0 x 626 [364,431,610,363,535,747,225,841,868,249]
+  CRUSH rule 0 x 627 [458,137,557,410,287,749,467,432,944,781]
+  CRUSH rule 0 x 628 [250,350,556,497,821,65,205,580,972,427]
+  CRUSH rule 0 x 629 [928,160,710,572,365,772,538,46,300,112]
+  CRUSH rule 0 x 630 [243,19,918,556,601,16,920,830,171,759]
+  CRUSH rule 0 x 631 [438,221,574,676,797,580,219,211,157,614]
+  CRUSH rule 0 x 632 [797,368,247,5,32,102,416,45,624,253]
+  CRUSH rule 0 x 633 [993,749,525,485,27,330,275,599,219,357]
+  CRUSH rule 0 x 634 [239,351,633,299,651,678,296,337,676,416]
+  CRUSH rule 0 x 635 [640,965,25,961,306,172,849,357,317,599]
+  CRUSH rule 0 x 636 [173,290,297,991,937,823,236,318,228,575]
+  CRUSH rule 0 x 637 [0,918,98,108,111,495,887,57,16,319]
+  CRUSH rule 0 x 638 [702,235,424,900,983,754,701,887,355,632]
+  CRUSH rule 0 x 639 [475,687,31,785,918,611,27,214,226,515]
+  CRUSH rule 0 x 640 [31,664,399,677,123,609,858,138,726,1]
+  CRUSH rule 0 x 641 [296,473,108,963,341,876,897,449,42,193]
+  CRUSH rule 0 x 642 [894,273,427,606,677,670,610,665,299,852]
+  CRUSH rule 0 x 643 [117,111,732,191,114,153,500,631,833,439]
+  CRUSH rule 0 x 644 [438,336,327,512,599,862,660,857,123,910]
+  CRUSH rule 0 x 645 [982,702,351,573,907,915,279,317,414,917]
+  CRUSH rule 0 x 646 [334,804,146,842,697,638,720,135,369,711]
+  CRUSH rule 0 x 647 [933,787,185,334,752,285,372,890,30,747]
+  CRUSH rule 0 x 648 [22,444,400,862,207,842,453,732,262,803]
+  CRUSH rule 0 x 649 [503,229,213,460,639,760,722,748,599,556]
+  CRUSH rule 0 x 650 [328,659,420,443,739,950,869,150,743,438]
+  CRUSH rule 0 x 651 [3,880,823,123,378,585,715,221,31,92]
+  CRUSH rule 0 x 652 [495,977,563,733,92,997,119,818,459,782]
+  CRUSH rule 0 x 653 [185,718,804,280,975,912,198,291,71,792]
+  CRUSH rule 0 x 654 [130,528,380,81,906,511,385,506,546,266]
+  CRUSH rule 0 x 655 [560,872,454,504,319,284,605,214,833,862]
+  CRUSH rule 0 x 656 [219,885,178,981,863,508,708,6,746,734]
+  CRUSH rule 0 x 657 [233,684,813,490,208,941,858,16,128,144]
+  CRUSH rule 0 x 658 [778,6,756,380,750,836,547,850,499,125]
+  CRUSH rule 0 x 659 [240,663,306,540,789,902,170,954,22,394]
+  CRUSH rule 0 x 660 [244,855,196,147,678,323,63,859,215,171]
+  CRUSH rule 0 x 661 [184,270,128,398,910,230,402,205,609,831]
+  CRUSH rule 0 x 662 [65,883,921,438,79,957,464,902,276,289]
+  CRUSH rule 0 x 663 [323,721,594,812,43,992,170,65,906,943]
+  CRUSH rule 0 x 664 [865,113,512,51,427,123,585,260,254,209]
+  CRUSH rule 0 x 665 [420,850,591,475,202,733,798,658,28,334]
+  CRUSH rule 0 x 666 [319,767,246,3,369,493,796,56,736,0]
+  CRUSH rule 0 x 667 [875,39,343,100,829,2,795,783,386,956]
+  CRUSH rule 0 x 668 [331,122,263,599,355,484,943,554,395,713]
+  CRUSH rule 0 x 669 [915,521,402,747,673,445,938,600,517,49]
+  CRUSH rule 0 x 670 [845,659,943,447,401,322,168,302,681,978]
+  CRUSH rule 0 x 671 [108,634,527,363,856,238,755,330,584,525]
+  CRUSH rule 0 x 672 [578,216,110,589,302,137,954,315,735,751]
+  CRUSH rule 0 x 673 [442,74,579,797,622,950,371,402,725,870]
+  CRUSH rule 0 x 674 [588,364,281,308,645,631,229,506,565,362]
+  CRUSH rule 0 x 675 [489,698,744,671,870,174,528,875,982,782]
+  CRUSH rule 0 x 676 [928,911,40,180,722,729,673,569,701,403]
+  CRUSH rule 0 x 677 [399,269,692,131,615,136,103,763,527,83]
+  CRUSH rule 0 x 678 [546,752,544,155,5,463,666,352,576,959]
+  CRUSH rule 0 x 679 [988,25,275,433,628,57,247,620,437,29]
+  CRUSH rule 0 x 680 [335,963,382,486,749,257,795,347,831,761]
+  CRUSH rule 0 x 681 [690,462,623,466,49,471,774,192,454,380]
+  CRUSH rule 0 x 682 [196,588,154,257,807,776,367,718,345,677]
+  CRUSH rule 0 x 683 [627,25,421,160,873,102,345,599,30,892]
+  CRUSH rule 0 x 684 [38,804,592,158,991,264,652,821,641,757]
+  CRUSH rule 0 x 685 [841,368,548,362,166,211,154,121,843,804]
+  CRUSH rule 0 x 686 [336,287,525,440,166,993,911,638,690,393]
+  CRUSH rule 0 x 687 [20,682,924,653,356,16,917,622,156,826]
+  CRUSH rule 0 x 688 [463,371,780,556,385,883,115,248,566,11]
+  CRUSH rule 0 x 689 [569,250,78,816,847,775,333,161,74,907]
+  CRUSH rule 0 x 690 [551,144,587,263,378,394,970,639,835,238]
+  CRUSH rule 0 x 691 [766,464,446,533,449,541,451,290,789,853]
+  CRUSH rule 0 x 692 [739,634,18,245,624,35,268,525,425,499]
+  CRUSH rule 0 x 693 [339,297,118,330,817,91,828,276,264,237]
+  CRUSH rule 0 x 694 [405,26,830,181,533,166,488,804,501,885]
+  CRUSH rule 0 x 695 [622,576,597,535,600,593,300,989,804,72]
+  CRUSH rule 0 x 696 [558,902,689,13,715,28,664,489,598,261]
+  CRUSH rule 0 x 697 [818,222,406,691,427,863,153,922,986,480]
+  CRUSH rule 0 x 698 [178,48,402,233,841,604,468,180,783,915]
+  CRUSH rule 0 x 699 [450,244,180,919,183,332,747,453,519,100]
+  CRUSH rule 0 x 700 [502,771,987,706,416,240,68,641,109,182]
+  CRUSH rule 0 x 701 [4,612,782,216,853,303,585,513,907,414]
+  CRUSH rule 0 x 702 [177,630,232,923,281,708,466,687,742,170]
+  CRUSH rule 0 x 703 [354,178,389,393,778,803,796,607,894,1]
+  CRUSH rule 0 x 704 [646,601,156,171,603,116,655,595,888,354]
+  CRUSH rule 0 x 705 [921,401,890,265,244,690,372,253,807,28]
+  CRUSH rule 0 x 706 [652,877,562,452,26,323,923,770,516,982]
+  CRUSH rule 0 x 707 [345,745,67,716,789,576,2,133,256,374]
+  CRUSH rule 0 x 708 [333,607,180,469,170,555,939,331,41,175]
+  CRUSH rule 0 x 709 [45,187,302,115,896,579,733,607,763,845]
+  CRUSH rule 0 x 710 [94,855,43,199,18,948,449,28,731,573]
+  CRUSH rule 0 x 711 [227,653,731,150,156,842,534,110,639,452]
+  CRUSH rule 0 x 712 [398,953,136,870,181,408,895,459,341,833]
+  CRUSH rule 0 x 713 [116,800,503,662,635,579,53,839,56,829]
+  CRUSH rule 0 x 714 [111,629,866,709,902,557,875,649,23,79]
+  CRUSH rule 0 x 715 [531,291,486,382,192,807,322,417,973,582]
+  CRUSH rule 0 x 716 [169,541,291,42,343,724,138,197,32,415]
+  CRUSH rule 0 x 717 [417,446,994,894,239,494,237,62,327,958]
+  CRUSH rule 0 x 718 [992,383,298,844,377,463,544,891,210,370]
+  CRUSH rule 0 x 719 [936,674,324,759,194,409,828,975,119,87]
+  CRUSH rule 0 x 720 [370,188,174,464,644,218,214,76,870,779]
+  CRUSH rule 0 x 721 [320,859,278,259,170,957,177,264,867,327]
+  CRUSH rule 0 x 722 [7,2,673,129,96,445,823,833,1,774]
+  CRUSH rule 0 x 723 [270,553,831,662,38,101,985,846,77,467]
+  CRUSH rule 0 x 724 [666,822,708,895,633,800,616,879,480,309]
+  CRUSH rule 0 x 725 [794,406,875,459,981,751,359,983,720,128]
+  CRUSH rule 0 x 726 [420,556,341,292,240,68,966,535,669,74]
+  CRUSH rule 0 x 727 [561,461,129,635,965,610,105,31,506,430]
+  CRUSH rule 0 x 728 [951,330,196,756,589,849,753,760,254,379]
+  CRUSH rule 0 x 729 [656,644,436,591,27,119,572,933,434,816]
+  CRUSH rule 0 x 730 [3,558,629,184,50,765,760,800,945,743]
+  CRUSH rule 0 x 731 [852,89,75,735,713,113,528,890,625,535]
+  CRUSH rule 0 x 732 [983,840,869,976,697,307,368,271,778,172]
+  CRUSH rule 0 x 733 [285,396,388,122,387,364,880,343,590,539]
+  CRUSH rule 0 x 734 [125,510,402,640,676,501,535,627,224,790]
+  CRUSH rule 0 x 735 [417,773,686,504,459,912,690,59,294,569]
+  CRUSH rule 0 x 736 [749,396,632,550,779,109,845,278,559,613]
+  CRUSH rule 0 x 737 [644,991,946,135,448,903,482,564,259,896]
+  CRUSH rule 0 x 738 [449,683,290,220,245,525,429,397,872,716]
+  CRUSH rule 0 x 739 [341,220,641,454,740,661,146,17,314,156]
+  CRUSH rule 0 x 740 [874,524,674,650,472,282,214,494,593,155]
+  CRUSH rule 0 x 741 [189,472,712,798,715,757,863,571,876,528]
+  CRUSH rule 0 x 742 [912,581,114,117,730,21,687,81,145,695]
+  CRUSH rule 0 x 743 [654,914,425,441,763,39,451,631,911,829]
+  CRUSH rule 0 x 744 [725,295,579,377,162,447,843,699,24,714]
+  CRUSH rule 0 x 745 [787,858,850,506,612,735,926,314,771,910]
+  CRUSH rule 0 x 746 [757,848,704,30,47,940,450,651,105,921]
+  CRUSH rule 0 x 747 [700,81,867,681,801,64,879,857,727,565]
+  CRUSH rule 0 x 748 [557,436,238,664,293,865,304,999,685,843]
+  CRUSH rule 0 x 749 [772,622,337,42,156,302,383,506,570,828]
+  CRUSH rule 0 x 750 [946,97,376,677,316,670,169,171,9,58]
+  CRUSH rule 0 x 751 [996,618,343,911,83,22,388,17,892,537]
+  CRUSH rule 0 x 752 [746,887,695,868,610,950,88,315,728,669]
+  CRUSH rule 0 x 753 [741,14,463,479,172,192,481,702,431,675]
+  CRUSH rule 0 x 754 [648,349,333,355,65,63,336,724,262,61]
+  CRUSH rule 0 x 755 [157,460,466,187,959,674,192,279,371,970]
+  CRUSH rule 0 x 756 [416,97,197,497,227,3,850,191,991,63]
+  CRUSH rule 0 x 757 [599,839,776,410,256,823,121,690,544,28]
+  CRUSH rule 0 x 758 [994,218,620,256,361,749,165,686,449,831]
+  CRUSH rule 0 x 759 [959,682,514,745,100,519,15,347,311,512]
+  CRUSH rule 0 x 760 [518,943,215,83,706,137,345,69,39,199]
+  CRUSH rule 0 x 761 [285,849,420,324,987,338,373,361,684,654]
+  CRUSH rule 0 x 762 [591,313,41,335,110,696,664,350,339,980]
+  CRUSH rule 0 x 763 [908,411,200,740,292,295,387,775,797,990]
+  CRUSH rule 0 x 764 [787,234,894,485,883,711,70,202,557,471]
+  CRUSH rule 0 x 765 [327,921,882,393,444,792,402,123,902,592]
+  CRUSH rule 0 x 766 [84,161,878,704,416,144,357,310,890,850]
+  CRUSH rule 0 x 767 [370,895,702,701,890,2,251,951,675,322]
+  CRUSH rule 0 x 768 [826,760,879,864,460,474,645,975,947,199]
+  CRUSH rule 0 x 769 [67,768,663,735,814,66,213,527,546,42]
+  CRUSH rule 0 x 770 [593,909,482,259,5,550,961,324,309,772]
+  CRUSH rule 0 x 771 [309,935,121,578,937,685,933,571,822,256]
+  CRUSH rule 0 x 772 [12,125,797,301,348,419,891,959,487,355]
+  CRUSH rule 0 x 773 [253,466,820,549,591,193,783,951,982,160]
+  CRUSH rule 0 x 774 [164,390,705,109,881,505,890,425,599,485]
+  CRUSH rule 0 x 775 [703,47,43,973,643,406,885,976,936,221]
+  CRUSH rule 0 x 776 [728,231,80,916,2,850,396,76,680,108]
+  CRUSH rule 0 x 777 [981,621,568,729,869,952,563,860,388,456]
+  CRUSH rule 0 x 778 [411,456,544,597,789,784,65,954,125,358]
+  CRUSH rule 0 x 779 [346,121,519,921,587,48,772,645,254,759]
+  CRUSH rule 0 x 780 [476,39,288,381,303,29,17,336,147,829]
+  CRUSH rule 0 x 781 [10,130,585,844,729,705,714,954,271,58]
+  CRUSH rule 0 x 782 [462,246,581,902,623,877,812,516,774,985]
+  CRUSH rule 0 x 783 [580,373,153,775,668,661,626,961,576,119]
+  CRUSH rule 0 x 784 [413,113,978,990,994,56,481,198,171,944]
+  CRUSH rule 0 x 785 [341,856,332,354,59,581,632,151,586,360]
+  CRUSH rule 0 x 786 [411,140,313,393,215,618,490,481,627,740]
+  CRUSH rule 0 x 787 [605,522,211,813,636,224,600,528,966,556]
+  CRUSH rule 0 x 788 [226,545,35,142,726,851,194,216,486,782]
+  CRUSH rule 0 x 789 [545,320,414,702,731,277,237,916,374,670]
+  CRUSH rule 0 x 790 [414,748,816,327,130,115,788,164,691,329]
+  CRUSH rule 0 x 791 [660,906,406,697,916,322,124,128,742,990]
+  CRUSH rule 0 x 792 [287,392,514,204,75,789,406,858,694,351]
+  CRUSH rule 0 x 793 [631,133,850,713,720,487,376,812,886,264]
+  CRUSH rule 0 x 794 [931,517,543,210,963,898,811,459,344,719]
+  CRUSH rule 0 x 795 [551,962,477,948,425,434,268,94,648,402]
+  CRUSH rule 0 x 796 [814,4,95,27,368,300,646,451,67,738]
+  CRUSH rule 0 x 797 [64,201,299,734,605,864,596,196,93,636]
+  CRUSH rule 0 x 798 [422,530,114,431,565,716,473,250,839,895]
+  CRUSH rule 0 x 799 [824,32,679,562,266,549,859,994,831,60]
+  CRUSH rule 0 x 800 [862,623,489,637,861,196,941,643,398,325]
+  CRUSH rule 0 x 801 [145,550,329,324,734,160,219,662,142,28]
+  CRUSH rule 0 x 802 [570,19,847,308,387,518,846,53,783,511]
+  CRUSH rule 0 x 803 [151,812,662,358,880,349,834,881,23,229]
+  CRUSH rule 0 x 804 [467,93,264,863,176,842,663,949,380,39]
+  CRUSH rule 0 x 805 [621,223,938,809,591,686,121,157,934,660]
+  CRUSH rule 0 x 806 [898,957,805,430,499,584,640,607,790,832]
+  CRUSH rule 0 x 807 [354,531,422,159,921,431,802,136,305,983]
+  CRUSH rule 0 x 808 [7,96,76,897,446,2,166,929,234,460]
+  CRUSH rule 0 x 809 [70,734,719,56,687,21,23,145,184,465]
+  CRUSH rule 0 x 810 [701,18,972,327,771,649,620,648,433,997]
+  CRUSH rule 0 x 811 [248,547,103,728,901,264,948,202,521,278]
+  CRUSH rule 0 x 812 [230,576,821,566,993,762,675,28,263,410]
+  CRUSH rule 0 x 813 [805,114,683,629,801,462,285,450,948,742]
+  CRUSH rule 0 x 814 [54,619,973,741,497,894,401,266,905,320]
+  CRUSH rule 0 x 815 [679,412,613,132,969,411,314,670,928,727]
+  CRUSH rule 0 x 816 [919,448,826,414,36,289,44,822,332,959]
+  CRUSH rule 0 x 817 [765,830,436,521,332,458,260,172,193,516]
+  CRUSH rule 0 x 818 [415,566,644,687,692,414,769,826,519,277]
+  CRUSH rule 0 x 819 [721,319,865,750,546,859,523,770,56,437]
+  CRUSH rule 0 x 820 [218,301,333,190,686,179,535,787,267,46]
+  CRUSH rule 0 x 821 [185,795,680,953,329,750,621,815,313,916]
+  CRUSH rule 0 x 822 [356,261,54,522,900,103,883,112,601,15]
+  CRUSH rule 0 x 823 [220,281,549,456,64,306,282,641,216,929]
+  CRUSH rule 0 x 824 [292,809,887,74,776,788,559,886,753,749]
+  CRUSH rule 0 x 825 [949,778,101,311,110,480,161,998,370,10]
+  CRUSH rule 0 x 826 [767,818,833,927,356,954,910,63,288,836]
+  CRUSH rule 0 x 827 [631,83,406,635,657,713,212,916,692,653]
+  CRUSH rule 0 x 828 [288,986,445,26,414,607,937,595,935,672]
+  CRUSH rule 0 x 829 [990,667,915,694,974,453,669,330,822,36]
+  CRUSH rule 0 x 830 [152,571,778,505,685,209,448,55,965,851]
+  CRUSH rule 0 x 831 [814,563,630,97,582,107,142,157,957,105]
+  CRUSH rule 0 x 832 [235,641,616,110,979,844,656,135,341,922]
+  CRUSH rule 0 x 833 [657,565,922,140,825,457,764,766,853,890]
+  CRUSH rule 0 x 834 [907,231,644,13,617,130,83,483,811,98]
+  CRUSH rule 0 x 835 [784,262,771,264,612,238,537,937,101,507]
+  CRUSH rule 0 x 836 [951,158,366,710,43,427,351,961,52,44]
+  CRUSH rule 0 x 837 [556,498,334,633,895,627,903,29,454,647]
+  CRUSH rule 0 x 838 [329,274,964,547,119,342,983,998,320,935]
+  CRUSH rule 0 x 839 [568,209,939,364,658,747,47,859,402,947]
+  CRUSH rule 0 x 840 [45,579,842,70,655,862,815,109,762,642]
+  CRUSH rule 0 x 841 [652,702,24,605,152,93,226,46,918,220]
+  CRUSH rule 0 x 842 [629,984,314,895,408,897,575,1,312,542]
+  CRUSH rule 0 x 843 [799,690,688,648,151,812,486,199,966,501]
+  CRUSH rule 0 x 844 [694,600,534,700,569,11,899,382,851,472]
+  CRUSH rule 0 x 845 [332,30,179,93,951,324,611,512,855,760]
+  CRUSH rule 0 x 846 [452,251,712,719,404,739,606,237,414,844]
+  CRUSH rule 0 x 847 [399,681,847,739,13,555,363,893,592,634]
+  CRUSH rule 0 x 848 [303,138,440,346,547,216,700,249,214,100]
+  CRUSH rule 0 x 849 [666,346,708,873,64,694,847,463,995,314]
+  CRUSH rule 0 x 850 [644,511,345,844,545,337,358,35,913,310]
+  CRUSH rule 0 x 851 [527,546,737,425,100,331,95,337,677,275]
+  CRUSH rule 0 x 852 [31,809,94,618,156,853,469,511,999,340]
+  CRUSH rule 0 x 853 [483,330,869,184,46,942,774,679,616,492]
+  CRUSH rule 0 x 854 [697,953,968,143,502,955,441,302,437,53]
+  CRUSH rule 0 x 855 [837,996,239,621,32,191,686,702,919,971]
+  CRUSH rule 0 x 856 [712,40,547,430,195,857,224,810,404,126]
+  CRUSH rule 0 x 857 [77,984,576,551,568,96,12,763,594,668]
+  CRUSH rule 0 x 858 [412,384,841,465,572,576,688,61,545,491]
+  CRUSH rule 0 x 859 [173,760,26,300,87,567,463,903,272,8]
+  CRUSH rule 0 x 860 [776,429,328,917,658,783,699,907,532,627]
+  CRUSH rule 0 x 861 [705,405,477,50,73,714,901,487,725,204]
+  CRUSH rule 0 x 862 [809,44,788,938,964,177,490,409,15,842]
+  CRUSH rule 0 x 863 [349,496,963,178,675,853,172,980,772,115]
+  CRUSH rule 0 x 864 [717,858,101,239,992,244,43,15,29,974]
+  CRUSH rule 0 x 865 [857,603,586,262,550,289,850,40,170,31]
+  CRUSH rule 0 x 866 [394,304,71,96,642,155,255,481,435,119]
+  CRUSH rule 0 x 867 [640,773,663,974,261,296,988,730,753,888]
+  CRUSH rule 0 x 868 [613,950,712,663,666,460,643,547,734,16]
+  CRUSH rule 0 x 869 [973,889,524,22,671,477,718,431,968,472]
+  CRUSH rule 0 x 870 [505,35,386,498,348,503,54,992,726,783]
+  CRUSH rule 0 x 871 [239,264,262,773,781,734,387,515,98,232]
+  CRUSH rule 0 x 872 [21,767,456,748,783,797,180,800,521,270]
+  CRUSH rule 0 x 873 [954,666,980,264,435,233,199,358,805,255]
+  CRUSH rule 0 x 874 [54,510,947,1,500,119,93,915,801,43]
+  CRUSH rule 0 x 875 [809,418,452,462,88,673,634,435,778,884]
+  CRUSH rule 0 x 876 [483,457,61,248,523,277,322,141,82,412]
+  CRUSH rule 0 x 877 [542,531,952,939,710,179,181,460,459,527]
+  CRUSH rule 0 x 878 [217,674,857,644,678,809,329,591,59,4]
+  CRUSH rule 0 x 879 [999,475,134,250,319,357,145,750,54,997]
+  CRUSH rule 0 x 880 [678,573,935,385,570,651,319,630,888,970]
+  CRUSH rule 0 x 881 [394,835,789,802,587,155,570,109,896,826]
+  CRUSH rule 0 x 882 [467,382,353,56,979,674,974,483,412,547]
+  CRUSH rule 0 x 883 [802,744,237,337,50,96,202,148,129,72]
+  CRUSH rule 0 x 884 [653,660,638,700,31,558,389,381,347,314]
+  CRUSH rule 0 x 885 [898,704,307,445,879,872,174,972,544,894]
+  CRUSH rule 0 x 886 [434,357,938,641,737,8,56,582,915,541]
+  CRUSH rule 0 x 887 [297,226,711,428,370,318,472,947,35,528]
+  CRUSH rule 0 x 888 [863,324,443,213,902,25,806,53,385,387]
+  CRUSH rule 0 x 889 [105,102,308,163,947,548,399,382,761,907]
+  CRUSH rule 0 x 890 [550,248,606,704,615,708,996,561,485,482]
+  CRUSH rule 0 x 891 [575,928,880,891,826,763,706,701,501,680]
+  CRUSH rule 0 x 892 [259,862,133,271,292,162,53,333,458,77]
+  CRUSH rule 0 x 893 [902,880,543,542,37,942,672,320,394,373]
+  CRUSH rule 0 x 894 [180,169,916,43,945,713,648,685,895,735]
+  CRUSH rule 0 x 895 [725,849,182,129,177,272,599,829,809,713]
+  CRUSH rule 0 x 896 [951,34,874,537,969,123,210,529,491,289]
+  CRUSH rule 0 x 897 [810,352,73,939,943,895,12,481,539,562]
+  CRUSH rule 0 x 898 [979,433,719,411,787,359,342,37,303,70]
+  CRUSH rule 0 x 899 [685,668,534,932,399,156,124,653,574,384]
+  CRUSH rule 0 x 900 [530,978,41,894,941,681,380,419,667,56]
+  CRUSH rule 0 x 901 [740,107,336,175,574,706,157,292,724,805]
+  CRUSH rule 0 x 902 [800,743,693,310,67,111,178,624,733,498]
+  CRUSH rule 0 x 903 [230,267,842,266,550,769,66,738,419,199]
+  CRUSH rule 0 x 904 [346,949,460,973,696,91,957,801,74,934]
+  CRUSH rule 0 x 905 [530,397,619,958,576,973,685,6,689,387]
+  CRUSH rule 0 x 906 [80,426,138,672,73,776,30,169,506,497]
+  CRUSH rule 0 x 907 [365,968,475,297,296,724,664,331,184,461]
+  CRUSH rule 0 x 908 [204,832,742,809,862,745,484,391,841,967]
+  CRUSH rule 0 x 909 [883,989,146,959,366,59,686,965,515,421]
+  CRUSH rule 0 x 910 [549,593,249,853,792,769,824,552,717,159]
+  CRUSH rule 0 x 911 [325,847,352,214,851,732,789,255,896,868]
+  CRUSH rule 0 x 912 [874,888,582,796,557,601,226,889,69,237]
+  CRUSH rule 0 x 913 [331,463,342,574,989,362,925,746,664,533]
+  CRUSH rule 0 x 914 [836,468,601,732,607,275,70,280,837,367]
+  CRUSH rule 0 x 915 [245,228,100,661,799,13,126,79,652,793]
+  CRUSH rule 0 x 916 [77,967,364,435,27,474,255,133,892,524]
+  CRUSH rule 0 x 917 [239,60,866,221,772,967,725,707,47,216]
+  CRUSH rule 0 x 918 [988,115,922,80,201,544,583,923,863,232]
+  CRUSH rule 0 x 919 [783,139,696,1,848,169,888,980,33,261]
+  CRUSH rule 0 x 920 [623,408,685,953,974,696,532,124,911,206]
+  CRUSH rule 0 x 921 [105,799,144,90,399,373,633,290,155,137]
+  CRUSH rule 0 x 922 [887,505,652,348,514,806,952,474,67,938]
+  CRUSH rule 0 x 923 [223,318,552,458,743,871,964,384,454,448]
+  CRUSH rule 0 x 924 [25,778,366,333,163,801,584,31,151,178]
+  CRUSH rule 0 x 925 [912,601,297,682,770,173,969,168,500,68]
+  CRUSH rule 0 x 926 [968,133,132,144,814,155,709,158,96,739]
+  CRUSH rule 0 x 927 [277,724,214,988,690,342,465,775,725,414]
+  CRUSH rule 0 x 928 [554,203,658,789,298,299,847,752,780,738]
+  CRUSH rule 0 x 929 [761,802,367,528,758,522,744,171,144,704]
+  CRUSH rule 0 x 930 [814,61,788,736,660,491,832,654,567,160]
+  CRUSH rule 0 x 931 [29,193,61,41,343,664,487,839,776,117]
+  CRUSH rule 0 x 932 [446,198,862,534,168,35,530,462,202,11]
+  CRUSH rule 0 x 933 [352,742,216,321,525,44,568,61,945,154]
+  CRUSH rule 0 x 934 [730,2,332,631,613,249,533,116,254,569]
+  CRUSH rule 0 x 935 [731,23,736,79,361,992,772,49,567,47]
+  CRUSH rule 0 x 936 [322,975,20,904,827,603,138,802,885,447]
+  CRUSH rule 0 x 937 [822,221,841,161,723,137,630,308,973,934]
+  CRUSH rule 0 x 938 [557,850,66,630,499,404,286,395,927,611]
+  CRUSH rule 0 x 939 [150,11,971,371,124,785,408,49,977,243]
+  CRUSH rule 0 x 940 [638,398,169,616,333,751,25,883,867,270]
+  CRUSH rule 0 x 941 [730,342,929,577,451,838,964,28,633,960]
+  CRUSH rule 0 x 942 [62,292,166,814,587,172,171,16,440,31]
+  CRUSH rule 0 x 943 [165,314,519,548,41,726,759,851,617,420]
+  CRUSH rule 0 x 944 [199,625,766,176,194,297,678,915,619,69]
+  CRUSH rule 0 x 945 [946,999,699,303,38,81,952,885,987,775]
+  CRUSH rule 0 x 946 [595,93,852,142,503,647,933,267,846,866]
+  CRUSH rule 0 x 947 [800,582,356,93,716,117,922,868,413,545]
+  CRUSH rule 0 x 948 [132,551,139,920,87,46,81,220,725,211]
+  CRUSH rule 0 x 949 [792,920,466,380,97,568,799,961,564,71]
+  CRUSH rule 0 x 950 [111,345,176,543,879,954,355,220,528,747]
+  CRUSH rule 0 x 951 [414,619,648,655,364,971,829,408,568,734]
+  CRUSH rule 0 x 952 [775,469,500,356,287,4,16,746,835,529]
+  CRUSH rule 0 x 953 [349,1,5,251,168,680,141,619,234,517]
+  CRUSH rule 0 x 954 [570,940,410,249,929,394,129,696,115,984]
+  CRUSH rule 0 x 955 [729,774,823,800,7,127,536,766,579,398]
+  CRUSH rule 0 x 956 [519,141,575,625,738,475,169,751,667,381]
+  CRUSH rule 0 x 957 [242,709,611,97,760,309,393,281,227,412]
+  CRUSH rule 0 x 958 [84,217,227,253,246,604,346,377,425,533]
+  CRUSH rule 0 x 959 [270,413,918,789,703,608,543,519,496,956]
+  CRUSH rule 0 x 960 [458,192,307,279,920,139,855,49,548,304]
+  CRUSH rule 0 x 961 [981,388,777,546,359,660,455,708,649,93]
+  CRUSH rule 0 x 962 [623,834,277,134,729,246,856,477,895,89]
+  CRUSH rule 0 x 963 [291,167,714,468,109,373,485,701,76,55]
+  CRUSH rule 0 x 964 [28,156,788,127,598,215,361,255,507,540]
+  CRUSH rule 0 x 965 [675,557,290,517,840,510,59,229,819,610]
+  CRUSH rule 0 x 966 [836,306,946,283,642,606,929,773,928,579]
+  CRUSH rule 0 x 967 [966,386,735,837,392,116,19,674,395,483]
+  CRUSH rule 0 x 968 [864,756,690,121,328,122,433,520,916,41]
+  CRUSH rule 0 x 969 [729,625,480,769,512,882,518,956,398,269]
+  CRUSH rule 0 x 970 [800,362,646,582,309,102,576,411,416,523]
+  CRUSH rule 0 x 971 [737,381,153,684,298,166,344,520,546,612]
+  CRUSH rule 0 x 972 [952,245,720,884,334,311,754,540,79,174]
+  CRUSH rule 0 x 973 [356,455,579,857,832,596,549,524,109,364]
+  CRUSH rule 0 x 974 [545,758,586,596,756,790,116,993,644,405]
+  CRUSH rule 0 x 975 [336,191,202,146,720,897,330,308,744,843]
+  CRUSH rule 0 x 976 [446,208,757,620,252,846,397,58,57,603]
+  CRUSH rule 0 x 977 [202,896,196,956,763,126,783,828,409,529]
+  CRUSH rule 0 x 978 [612,324,996,225,418,583,514,169,99,878]
+  CRUSH rule 0 x 979 [843,457,675,650,958,657,677,173,903,781]
+  CRUSH rule 0 x 980 [60,914,881,626,850,759,398,943,764,867]
+  CRUSH rule 0 x 981 [702,749,937,153,724,514,536,212,247,523]
+  CRUSH rule 0 x 982 [298,928,738,167,99,668,395,198,100,580]
+  CRUSH rule 0 x 983 [723,572,395,358,900,37,927,597,103,461]
+  CRUSH rule 0 x 984 [723,864,804,935,846,993,950,840,427,469]
+  CRUSH rule 0 x 985 [945,459,868,211,524,954,911,208,91,999]
+  CRUSH rule 0 x 986 [772,664,535,169,297,996,864,555,687,212]
+  CRUSH rule 0 x 987 [88,324,312,843,661,580,76,894,480,323]
+  CRUSH rule 0 x 988 [522,927,131,996,351,685,865,47,116,230]
+  CRUSH rule 0 x 989 [578,332,208,605,975,207,155,380,797,177]
+  CRUSH rule 0 x 990 [638,228,414,311,738,698,340,526,728,595]
+  CRUSH rule 0 x 991 [530,221,451,422,879,916,754,928,288,668]
+  CRUSH rule 0 x 992 [925,705,275,81,234,310,117,546,798,777]
+  CRUSH rule 0 x 993 [991,301,43,469,830,242,382,428,451,216]
+  CRUSH rule 0 x 994 [276,51,868,683,843,815,557,378,936,192]
+  CRUSH rule 0 x 995 [288,836,753,790,758,120,158,265,110,171]
+  CRUSH rule 0 x 996 [887,983,252,686,470,345,459,764,859,776]
+  CRUSH rule 0 x 997 [110,924,386,79,705,697,210,698,273,955]
+  CRUSH rule 0 x 998 [435,830,485,853,926,730,786,762,444,561]
+  CRUSH rule 0 x 999 [876,738,357,913,723,51,15,585,898,902]
+  CRUSH rule 0 x 1000 [178,963,638,430,845,586,317,102,200,662]
+  CRUSH rule 0 x 1001 [99,519,66,759,583,944,739,922,343,574]
+  CRUSH rule 0 x 1002 [515,534,468,866,878,717,729,370,326,640]
+  CRUSH rule 0 x 1003 [104,611,937,698,94,67,614,783,865,245]
+  CRUSH rule 0 x 1004 [269,638,724,375,491,121,891,113,424,320]
+  CRUSH rule 0 x 1005 [369,223,309,409,822,39,597,969,911,241]
+  CRUSH rule 0 x 1006 [40,107,69,275,79,429,234,945,598,498]
+  CRUSH rule 0 x 1007 [978,111,416,758,454,640,5,444,795,150]
+  CRUSH rule 0 x 1008 [965,956,624,832,421,96,975,723,909,93]
+  CRUSH rule 0 x 1009 [598,476,356,695,919,566,234,383,604,903]
+  CRUSH rule 0 x 1010 [767,523,239,517,29,77,23,241,838,865]
+  CRUSH rule 0 x 1011 [289,871,207,576,347,698,48,570,639,230]
+  CRUSH rule 0 x 1012 [128,28,370,31,341,755,268,647,669,90]
+  CRUSH rule 0 x 1013 [979,765,660,812,666,187,808,351,572,403]
+  CRUSH rule 0 x 1014 [979,948,513,88,47,825,969,81,586,62]
+  CRUSH rule 0 x 1015 [277,790,396,672,542,647,145,11,965,669]
+  CRUSH rule 0 x 1016 [262,73,128,886,839,685,456,560,935,733]
+  CRUSH rule 0 x 1017 [150,269,61,499,832,591,637,731,738,154]
+  CRUSH rule 0 x 1018 [555,829,554,944,406,576,463,926,475,316]
+  CRUSH rule 0 x 1019 [513,356,265,446,65,288,768,245,337,197]
+  CRUSH rule 0 x 1020 [158,161,877,704,948,570,495,865,698,872]
+  CRUSH rule 0 x 1021 [915,998,957,285,546,202,676,322,671,622]
+  CRUSH rule 0 x 1022 [967,829,973,640,703,470,871,828,440,449]
+  CRUSH rule 0 x 1023 [488,257,614,859,325,419,50,560,595,554]
+  rule 0 (data) num_rep 10 result size == 10:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-tries-vs-retries.crushmap b/ceph/src/test/cli/crushtool/test-map-tries-vs-retries.crushmap
new file mode 100644
index 0000000000000000000000000000000000000000..e36c0eb9fda03473ddf9761c76c60a88fdaa69ed
GIT binary patch
literal 808
zcmbV}*>1ux5JerJ6zGQdlCZRV4nh!#sz^a0{5pF}@3EOanC68+j(x{R!}yMKiR(J2
zuk(9D`$dsBPBr~Pa33g6VL|*vEPP$d7~5gLBbM9)=(V2hAJX*1W%6N3W6>e8eB#&3ovR|EDf<2=Bgc$Iq?m
z)UW@^^ync>JWN)po8ZRJGD$j|sR6_+2y)c6fVJErUzc>teKDPVap%u@Ug9~mn0+Cg
z(u@}ZQ(laq!WauoeBgzR}+1m|3pfsL{F>8R?E-Xw9Mv{VzbQF
MS?RCz8E8TN0y-?2G5`Po

literal 0
HcmV?d00001

diff --git a/ceph/src/test/cli/crushtool/test-map-tries-vs-retries.t b/ceph/src/test/cli/crushtool/test-map-tries-vs-retries.t
new file mode 100644
index 00000000..8eac2556
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-tries-vs-retries.t
@@ -0,0 +1,10259 @@
+  $ crushtool -i "$TESTDIR/test-map-tries-vs-retries.crushmap" --test --show-statistics --weight 0 0 --weight 8 0
+  rule 0 (replicated_ruleset), x = 0..1023, numrep = 1..10
+  CRUSH rule 0 x 0 [7]
+  CRUSH rule 0 x 1 [10]
+  CRUSH rule 0 x 2 [1]
+  CRUSH rule 0 x 3 [15]
+  CRUSH rule 0 x 4 [14]
+  CRUSH rule 0 x 5 [7]
+  CRUSH rule 0 x 6 [12]
+  CRUSH rule 0 x 7 [9]
+  CRUSH rule 0 x 8 [10]
+  CRUSH rule 0 x 9 [7]
+  CRUSH rule 0 x 10 [10]
+  CRUSH rule 0 x 11 [13]
+  CRUSH rule 0 x 12 [7]
+  CRUSH rule 0 x 13 [3]
+  CRUSH rule 0 x 14 [13]
+  CRUSH rule 0 x 15 [15]
+  CRUSH rule 0 x 16 [7]
+  CRUSH rule 0 x 17 [10]
+  CRUSH rule 0 x 18 [1]
+  CRUSH rule 0 x 19 [7]
+  CRUSH rule 0 x 20 [14]
+  CRUSH rule 0 x 21 [3]
+  CRUSH rule 0 x 22 [6]
+  CRUSH rule 0 x 23 [10]
+  CRUSH rule 0 x 24 [12]
+  CRUSH rule 0 x 25 [7]
+  CRUSH rule 0 x 26 [1]
+  CRUSH rule 0 x 27 [3]
+  CRUSH rule 0 x 28 [14]
+  CRUSH rule 0 x 29 [5]
+  CRUSH rule 0 x 30 [2]
+  CRUSH rule 0 x 31 [5]
+  CRUSH rule 0 x 32 [9]
+  CRUSH rule 0 x 33 [13]
+  CRUSH rule 0 x 34 [13]
+  CRUSH rule 0 x 35 [4]
+  CRUSH rule 0 x 36 [3]
+  CRUSH rule 0 x 37 [9]
+  CRUSH rule 0 x 38 [3]
+  CRUSH rule 0 x 39 [12]
+  CRUSH rule 0 x 40 [10]
+  CRUSH rule 0 x 41 [4]
+  CRUSH rule 0 x 42 [3]
+  CRUSH rule 0 x 43 [10]
+  CRUSH rule 0 x 44 [11]
+  CRUSH rule 0 x 45 [11]
+  CRUSH rule 0 x 46 [6]
+  CRUSH rule 0 x 47 [3]
+  CRUSH rule 0 x 48 [4]
+  CRUSH rule 0 x 49 [9]
+  CRUSH rule 0 x 50 [14]
+  CRUSH rule 0 x 51 [10]
+  CRUSH rule 0 x 52 [12]
+  CRUSH rule 0 x 53 [3]
+  CRUSH rule 0 x 54 [4]
+  CRUSH rule 0 x 55 [4]
+  CRUSH rule 0 x 56 [5]
+  CRUSH rule 0 x 57 [6]
+  CRUSH rule 0 x 58 [7]
+  CRUSH rule 0 x 59 [2]
+  CRUSH rule 0 x 60 [3]
+  CRUSH rule 0 x 61 [3]
+  CRUSH rule 0 x 62 [15]
+  CRUSH rule 0 x 63 [10]
+  CRUSH rule 0 x 64 [3]
+  CRUSH rule 0 x 65 [4]
+  CRUSH rule 0 x 66 [15]
+  CRUSH rule 0 x 67 [2]
+  CRUSH rule 0 x 68 [15]
+  CRUSH rule 0 x 69 [2]
+  CRUSH rule 0 x 70 [9]
+  CRUSH rule 0 x 71 [15]
+  CRUSH rule 0 x 72 [9]
+  CRUSH rule 0 x 73 [5]
+  CRUSH rule 0 x 74 [11]
+  CRUSH rule 0 x 75 [9]
+  CRUSH rule 0 x 76 [6]
+  CRUSH rule 0 x 77 [7]
+  CRUSH rule 0 x 78 [9]
+  CRUSH rule 0 x 79 [13]
+  CRUSH rule 0 x 80 [15]
+  CRUSH rule 0 x 81 [15]
+  CRUSH rule 0 x 82 [14]
+  CRUSH rule 0 x 83 [4]
+  CRUSH rule 0 x 84 [10]
+  CRUSH rule 0 x 85 [3]
+  CRUSH rule 0 x 86 [10]
+  CRUSH rule 0 x 87 [15]
+  CRUSH rule 0 x 88 [4]
+  CRUSH rule 0 x 89 [3]
+  CRUSH rule 0 x 90 [4]
+  CRUSH rule 0 x 91 [6]
+  CRUSH rule 0 x 92 [1]
+  CRUSH rule 0 x 93 [9]
+  CRUSH rule 0 x 94 [9]
+  CRUSH rule 0 x 95 [7]
+  CRUSH rule 0 x 96 [2]
+  CRUSH rule 0 x 97 [4]
+  CRUSH rule 0 x 98 [11]
+  CRUSH rule 0 x 99 [12]
+  CRUSH rule 0 x 100 [9]
+  CRUSH rule 0 x 101 [15]
+  CRUSH rule 0 x 102 [3]
+  CRUSH rule 0 x 103 [13]
+  CRUSH rule 0 x 104 [14]
+  CRUSH rule 0 x 105 [14]
+  CRUSH rule 0 x 106 [6]
+  CRUSH rule 0 x 107 [3]
+  CRUSH rule 0 x 108 [5]
+  CRUSH rule 0 x 109 [9]
+  CRUSH rule 0 x 110 [5]
+  CRUSH rule 0 x 111 [10]
+  CRUSH rule 0 x 112 [1]
+  CRUSH rule 0 x 113 [6]
+  CRUSH rule 0 x 114 [5]
+  CRUSH rule 0 x 115 [10]
+  CRUSH rule 0 x 116 [1]
+  CRUSH rule 0 x 117 [5]
+  CRUSH rule 0 x 118 [10]
+  CRUSH rule 0 x 119 [14]
+  CRUSH rule 0 x 120 [11]
+  CRUSH rule 0 x 121 [9]
+  CRUSH rule 0 x 122 [4]
+  CRUSH rule 0 x 123 [3]
+  CRUSH rule 0 x 124 [12]
+  CRUSH rule 0 x 125 [9]
+  CRUSH rule 0 x 126 [7]
+  CRUSH rule 0 x 127 [4]
+  CRUSH rule 0 x 128 [3]
+  CRUSH rule 0 x 129 [11]
+  CRUSH rule 0 x 130 [3]
+  CRUSH rule 0 x 131 [12]
+  CRUSH rule 0 x 132 [11]
+  CRUSH rule 0 x 133 [3]
+  CRUSH rule 0 x 134 [12]
+  CRUSH rule 0 x 135 [3]
+  CRUSH rule 0 x 136 [15]
+  CRUSH rule 0 x 137 [14]
+  CRUSH rule 0 x 138 [13]
+  CRUSH rule 0 x 139 [11]
+  CRUSH rule 0 x 140 [11]
+  CRUSH rule 0 x 141 [6]
+  CRUSH rule 0 x 142 [3]
+  CRUSH rule 0 x 143 [9]
+  CRUSH rule 0 x 144 [13]
+  CRUSH rule 0 x 145 [12]
+  CRUSH rule 0 x 146 [1]
+  CRUSH rule 0 x 147 [1]
+  CRUSH rule 0 x 148 [12]
+  CRUSH rule 0 x 149 [2]
+  CRUSH rule 0 x 150 [1]
+  CRUSH rule 0 x 151 [2]
+  CRUSH rule 0 x 152 [5]
+  CRUSH rule 0 x 153 [6]
+  CRUSH rule 0 x 154 [3]
+  CRUSH rule 0 x 155 [14]
+  CRUSH rule 0 x 156 [7]
+  CRUSH rule 0 x 157 [15]
+  CRUSH rule 0 x 158 [15]
+  CRUSH rule 0 x 159 [4]
+  CRUSH rule 0 x 160 [5]
+  CRUSH rule 0 x 161 [1]
+  CRUSH rule 0 x 162 [10]
+  CRUSH rule 0 x 163 [15]
+  CRUSH rule 0 x 164 [9]
+  CRUSH rule 0 x 165 [11]
+  CRUSH rule 0 x 166 [1]
+  CRUSH rule 0 x 167 [9]
+  CRUSH rule 0 x 168 [13]
+  CRUSH rule 0 x 169 [1]
+  CRUSH rule 0 x 170 [1]
+  CRUSH rule 0 x 171 [9]
+  CRUSH rule 0 x 172 [14]
+  CRUSH rule 0 x 173 [5]
+  CRUSH rule 0 x 174 [15]
+  CRUSH rule 0 x 175 [5]
+  CRUSH rule 0 x 176 [9]
+  CRUSH rule 0 x 177 [2]
+  CRUSH rule 0 x 178 [12]
+  CRUSH rule 0 x 179 [2]
+  CRUSH rule 0 x 180 [3]
+  CRUSH rule 0 x 181 [9]
+  CRUSH rule 0 x 182 [5]
+  CRUSH rule 0 x 183 [5]
+  CRUSH rule 0 x 184 [2]
+  CRUSH rule 0 x 185 [13]
+  CRUSH rule 0 x 186 [6]
+  CRUSH rule 0 x 187 [1]
+  CRUSH rule 0 x 188 [9]
+  CRUSH rule 0 x 189 [6]
+  CRUSH rule 0 x 190 [9]
+  CRUSH rule 0 x 191 [7]
+  CRUSH rule 0 x 192 [2]
+  CRUSH rule 0 x 193 [3]
+  CRUSH rule 0 x 194 [3]
+  CRUSH rule 0 x 195 [5]
+  CRUSH rule 0 x 196 [4]
+  CRUSH rule 0 x 197 [14]
+  CRUSH rule 0 x 198 [2]
+  CRUSH rule 0 x 199 [2]
+  CRUSH rule 0 x 200 [7]
+  CRUSH rule 0 x 201 [9]
+  CRUSH rule 0 x 202 [14]
+  CRUSH rule 0 x 203 [12]
+  CRUSH rule 0 x 204 [6]
+  CRUSH rule 0 x 205 [15]
+  CRUSH rule 0 x 206 [13]
+  CRUSH rule 0 x 207 [2]
+  CRUSH rule 0 x 208 [13]
+  CRUSH rule 0 x 209 [6]
+  CRUSH rule 0 x 210 [13]
+  CRUSH rule 0 x 211 [2]
+  CRUSH rule 0 x 212 [10]
+  CRUSH rule 0 x 213 [3]
+  CRUSH rule 0 x 214 [7]
+  CRUSH rule 0 x 215 [6]
+  CRUSH rule 0 x 216 [12]
+  CRUSH rule 0 x 217 [12]
+  CRUSH rule 0 x 218 [12]
+  CRUSH rule 0 x 219 [3]
+  CRUSH rule 0 x 220 [14]
+  CRUSH rule 0 x 221 [15]
+  CRUSH rule 0 x 222 [10]
+  CRUSH rule 0 x 223 [9]
+  CRUSH rule 0 x 224 [1]
+  CRUSH rule 0 x 225 [10]
+  CRUSH rule 0 x 226 [4]
+  CRUSH rule 0 x 227 [7]
+  CRUSH rule 0 x 228 [2]
+  CRUSH rule 0 x 229 [9]
+  CRUSH rule 0 x 230 [10]
+  CRUSH rule 0 x 231 [2]
+  CRUSH rule 0 x 232 [10]
+  CRUSH rule 0 x 233 [6]
+  CRUSH rule 0 x 234 [10]
+  CRUSH rule 0 x 235 [13]
+  CRUSH rule 0 x 236 [2]
+  CRUSH rule 0 x 237 [3]
+  CRUSH rule 0 x 238 [2]
+  CRUSH rule 0 x 239 [4]
+  CRUSH rule 0 x 240 [15]
+  CRUSH rule 0 x 241 [7]
+  CRUSH rule 0 x 242 [14]
+  CRUSH rule 0 x 243 [2]
+  CRUSH rule 0 x 244 [13]
+  CRUSH rule 0 x 245 [12]
+  CRUSH rule 0 x 246 [15]
+  CRUSH rule 0 x 247 [6]
+  CRUSH rule 0 x 248 [5]
+  CRUSH rule 0 x 249 [10]
+  CRUSH rule 0 x 250 [12]
+  CRUSH rule 0 x 251 [13]
+  CRUSH rule 0 x 252 [7]
+  CRUSH rule 0 x 253 [3]
+  CRUSH rule 0 x 254 [2]
+  CRUSH rule 0 x 255 [1]
+  CRUSH rule 0 x 256 [6]
+  CRUSH rule 0 x 257 [15]
+  CRUSH rule 0 x 258 [12]
+  CRUSH rule 0 x 259 [9]
+  CRUSH rule 0 x 260 [10]
+  CRUSH rule 0 x 261 [13]
+  CRUSH rule 0 x 262 [15]
+  CRUSH rule 0 x 263 [12]
+  CRUSH rule 0 x 264 [13]
+  CRUSH rule 0 x 265 [12]
+  CRUSH rule 0 x 266 [14]
+  CRUSH rule 0 x 267 [12]
+  CRUSH rule 0 x 268 [4]
+  CRUSH rule 0 x 269 [11]
+  CRUSH rule 0 x 270 [7]
+  CRUSH rule 0 x 271 [4]
+  CRUSH rule 0 x 272 [15]
+  CRUSH rule 0 x 273 [2]
+  CRUSH rule 0 x 274 [10]
+  CRUSH rule 0 x 275 [10]
+  CRUSH rule 0 x 276 [5]
+  CRUSH rule 0 x 277 [14]
+  CRUSH rule 0 x 278 [5]
+  CRUSH rule 0 x 279 [6]
+  CRUSH rule 0 x 280 [7]
+  CRUSH rule 0 x 281 [5]
+  CRUSH rule 0 x 282 [2]
+  CRUSH rule 0 x 283 [4]
+  CRUSH rule 0 x 284 [5]
+  CRUSH rule 0 x 285 [15]
+  CRUSH rule 0 x 286 [10]
+  CRUSH rule 0 x 287 [12]
+  CRUSH rule 0 x 288 [4]
+  CRUSH rule 0 x 289 [2]
+  CRUSH rule 0 x 290 [12]
+  CRUSH rule 0 x 291 [7]
+  CRUSH rule 0 x 292 [4]
+  CRUSH rule 0 x 293 [6]
+  CRUSH rule 0 x 294 [9]
+  CRUSH rule 0 x 295 [6]
+  CRUSH rule 0 x 296 [3]
+  CRUSH rule 0 x 297 [6]
+  CRUSH rule 0 x 298 [14]
+  CRUSH rule 0 x 299 [14]
+  CRUSH rule 0 x 300 [15]
+  CRUSH rule 0 x 301 [9]
+  CRUSH rule 0 x 302 [9]
+  CRUSH rule 0 x 303 [4]
+  CRUSH rule 0 x 304 [6]
+  CRUSH rule 0 x 305 [13]
+  CRUSH rule 0 x 306 [10]
+  CRUSH rule 0 x 307 [11]
+  CRUSH rule 0 x 308 [12]
+  CRUSH rule 0 x 309 [9]
+  CRUSH rule 0 x 310 [3]
+  CRUSH rule 0 x 311 [3]
+  CRUSH rule 0 x 312 [15]
+  CRUSH rule 0 x 313 [9]
+  CRUSH rule 0 x 314 [2]
+  CRUSH rule 0 x 315 [15]
+  CRUSH rule 0 x 316 [4]
+  CRUSH rule 0 x 317 [1]
+  CRUSH rule 0 x 318 [4]
+  CRUSH rule 0 x 319 [2]
+  CRUSH rule 0 x 320 [5]
+  CRUSH rule 0 x 321 [1]
+  CRUSH rule 0 x 322 [13]
+  CRUSH rule 0 x 323 [7]
+  CRUSH rule 0 x 324 [5]
+  CRUSH rule 0 x 325 [9]
+  CRUSH rule 0 x 326 [11]
+  CRUSH rule 0 x 327 [12]
+  CRUSH rule 0 x 328 [5]
+  CRUSH rule 0 x 329 [2]
+  CRUSH rule 0 x 330 [3]
+  CRUSH rule 0 x 331 [12]
+  CRUSH rule 0 x 332 [10]
+  CRUSH rule 0 x 333 [6]
+  CRUSH rule 0 x 334 [4]
+  CRUSH rule 0 x 335 [11]
+  CRUSH rule 0 x 336 [6]
+  CRUSH rule 0 x 337 [15]
+  CRUSH rule 0 x 338 [10]
+  CRUSH rule 0 x 339 [11]
+  CRUSH rule 0 x 340 [11]
+  CRUSH rule 0 x 341 [7]
+  CRUSH rule 0 x 342 [12]
+  CRUSH rule 0 x 343 [12]
+  CRUSH rule 0 x 344 [9]
+  CRUSH rule 0 x 345 [14]
+  CRUSH rule 0 x 346 [5]
+  CRUSH rule 0 x 347 [10]
+  CRUSH rule 0 x 348 [7]
+  CRUSH rule 0 x 349 [9]
+  CRUSH rule 0 x 350 [13]
+  CRUSH rule 0 x 351 [13]
+  CRUSH rule 0 x 352 [1]
+  CRUSH rule 0 x 353 [10]
+  CRUSH rule 0 x 354 [6]
+  CRUSH rule 0 x 355 [13]
+  CRUSH rule 0 x 356 [15]
+  CRUSH rule 0 x 357 [4]
+  CRUSH rule 0 x 358 [12]
+  CRUSH rule 0 x 359 [5]
+  CRUSH rule 0 x 360 [13]
+  CRUSH rule 0 x 361 [5]
+  CRUSH rule 0 x 362 [2]
+  CRUSH rule 0 x 363 [7]
+  CRUSH rule 0 x 364 [2]
+  CRUSH rule 0 x 365 [13]
+  CRUSH rule 0 x 366 [12]
+  CRUSH rule 0 x 367 [7]
+  CRUSH rule 0 x 368 [7]
+  CRUSH rule 0 x 369 [7]
+  CRUSH rule 0 x 370 [4]
+  CRUSH rule 0 x 371 [1]
+  CRUSH rule 0 x 372 [10]
+  CRUSH rule 0 x 373 [15]
+  CRUSH rule 0 x 374 [3]
+  CRUSH rule 0 x 375 [5]
+  CRUSH rule 0 x 376 [5]
+  CRUSH rule 0 x 377 [1]
+  CRUSH rule 0 x 378 [9]
+  CRUSH rule 0 x 379 [11]
+  CRUSH rule 0 x 380 [6]
+  CRUSH rule 0 x 381 [15]
+  CRUSH rule 0 x 382 [14]
+  CRUSH rule 0 x 383 [3]
+  CRUSH rule 0 x 384 [4]
+  CRUSH rule 0 x 385 [4]
+  CRUSH rule 0 x 386 [14]
+  CRUSH rule 0 x 387 [1]
+  CRUSH rule 0 x 388 [2]
+  CRUSH rule 0 x 389 [12]
+  CRUSH rule 0 x 390 [2]
+  CRUSH rule 0 x 391 [3]
+  CRUSH rule 0 x 392 [11]
+  CRUSH rule 0 x 393 [2]
+  CRUSH rule 0 x 394 [4]
+  CRUSH rule 0 x 395 [10]
+  CRUSH rule 0 x 396 [2]
+  CRUSH rule 0 x 397 [1]
+  CRUSH rule 0 x 398 [9]
+  CRUSH rule 0 x 399 [5]
+  CRUSH rule 0 x 400 [10]
+  CRUSH rule 0 x 401 [6]
+  CRUSH rule 0 x 402 [4]
+  CRUSH rule 0 x 403 [7]
+  CRUSH rule 0 x 404 [14]
+  CRUSH rule 0 x 405 [9]
+  CRUSH rule 0 x 406 [12]
+  CRUSH rule 0 x 407 [9]
+  CRUSH rule 0 x 408 [7]
+  CRUSH rule 0 x 409 [11]
+  CRUSH rule 0 x 410 [6]
+  CRUSH rule 0 x 411 [13]
+  CRUSH rule 0 x 412 [5]
+  CRUSH rule 0 x 413 [13]
+  CRUSH rule 0 x 414 [3]
+  CRUSH rule 0 x 415 [6]
+  CRUSH rule 0 x 416 [13]
+  CRUSH rule 0 x 417 [4]
+  CRUSH rule 0 x 418 [14]
+  CRUSH rule 0 x 419 [5]
+  CRUSH rule 0 x 420 [2]
+  CRUSH rule 0 x 421 [15]
+  CRUSH rule 0 x 422 [4]
+  CRUSH rule 0 x 423 [3]
+  CRUSH rule 0 x 424 [6]
+  CRUSH rule 0 x 425 [11]
+  CRUSH rule 0 x 426 [12]
+  CRUSH rule 0 x 427 [14]
+  CRUSH rule 0 x 428 [12]
+  CRUSH rule 0 x 429 [3]
+  CRUSH rule 0 x 430 [3]
+  CRUSH rule 0 x 431 [9]
+  CRUSH rule 0 x 432 [4]
+  CRUSH rule 0 x 433 [4]
+  CRUSH rule 0 x 434 [2]
+  CRUSH rule 0 x 435 [13]
+  CRUSH rule 0 x 436 [9]
+  CRUSH rule 0 x 437 [9]
+  CRUSH rule 0 x 438 [7]
+  CRUSH rule 0 x 439 [7]
+  CRUSH rule 0 x 440 [14]
+  CRUSH rule 0 x 441 [2]
+  CRUSH rule 0 x 442 [10]
+  CRUSH rule 0 x 443 [12]
+  CRUSH rule 0 x 444 [4]
+  CRUSH rule 0 x 445 [4]
+  CRUSH rule 0 x 446 [12]
+  CRUSH rule 0 x 447 [15]
+  CRUSH rule 0 x 448 [5]
+  CRUSH rule 0 x 449 [14]
+  CRUSH rule 0 x 450 [2]
+  CRUSH rule 0 x 451 [6]
+  CRUSH rule 0 x 452 [14]
+  CRUSH rule 0 x 453 [5]
+  CRUSH rule 0 x 454 [10]
+  CRUSH rule 0 x 455 [6]
+  CRUSH rule 0 x 456 [5]
+  CRUSH rule 0 x 457 [9]
+  CRUSH rule 0 x 458 [9]
+  CRUSH rule 0 x 459 [13]
+  CRUSH rule 0 x 460 [5]
+  CRUSH rule 0 x 461 [4]
+  CRUSH rule 0 x 462 [4]
+  CRUSH rule 0 x 463 [4]
+  CRUSH rule 0 x 464 [4]
+  CRUSH rule 0 x 465 [5]
+  CRUSH rule 0 x 466 [13]
+  CRUSH rule 0 x 467 [13]
+  CRUSH rule 0 x 468 [10]
+  CRUSH rule 0 x 469 [4]
+  CRUSH rule 0 x 470 [3]
+  CRUSH rule 0 x 471 [6]
+  CRUSH rule 0 x 472 [2]
+  CRUSH rule 0 x 473 [15]
+  CRUSH rule 0 x 474 [15]
+  CRUSH rule 0 x 475 [10]
+  CRUSH rule 0 x 476 [3]
+  CRUSH rule 0 x 477 [6]
+  CRUSH rule 0 x 478 [4]
+  CRUSH rule 0 x 479 [13]
+  CRUSH rule 0 x 480 [1]
+  CRUSH rule 0 x 481 [15]
+  CRUSH rule 0 x 482 [2]
+  CRUSH rule 0 x 483 [10]
+  CRUSH rule 0 x 484 [1]
+  CRUSH rule 0 x 485 [9]
+  CRUSH rule 0 x 486 [3]
+  CRUSH rule 0 x 487 [12]
+  CRUSH rule 0 x 488 [14]
+  CRUSH rule 0 x 489 [11]
+  CRUSH rule 0 x 490 [4]
+  CRUSH rule 0 x 491 [1]
+  CRUSH rule 0 x 492 [5]
+  CRUSH rule 0 x 493 [12]
+  CRUSH rule 0 x 494 [1]
+  CRUSH rule 0 x 495 [3]
+  CRUSH rule 0 x 496 [5]
+  CRUSH rule 0 x 497 [13]
+  CRUSH rule 0 x 498 [10]
+  CRUSH rule 0 x 499 [14]
+  CRUSH rule 0 x 500 [15]
+  CRUSH rule 0 x 501 [10]
+  CRUSH rule 0 x 502 [5]
+  CRUSH rule 0 x 503 [15]
+  CRUSH rule 0 x 504 [13]
+  CRUSH rule 0 x 505 [12]
+  CRUSH rule 0 x 506 [11]
+  CRUSH rule 0 x 507 [4]
+  CRUSH rule 0 x 508 [12]
+  CRUSH rule 0 x 509 [4]
+  CRUSH rule 0 x 510 [5]
+  CRUSH rule 0 x 511 [2]
+  CRUSH rule 0 x 512 [15]
+  CRUSH rule 0 x 513 [4]
+  CRUSH rule 0 x 514 [11]
+  CRUSH rule 0 x 515 [12]
+  CRUSH rule 0 x 516 [14]
+  CRUSH rule 0 x 517 [11]
+  CRUSH rule 0 x 518 [3]
+  CRUSH rule 0 x 519 [12]
+  CRUSH rule 0 x 520 [12]
+  CRUSH rule 0 x 521 [11]
+  CRUSH rule 0 x 522 [4]
+  CRUSH rule 0 x 523 [3]
+  CRUSH rule 0 x 524 [15]
+  CRUSH rule 0 x 525 [3]
+  CRUSH rule 0 x 526 [10]
+  CRUSH rule 0 x 527 [3]
+  CRUSH rule 0 x 528 [12]
+  CRUSH rule 0 x 529 [6]
+  CRUSH rule 0 x 530 [11]
+  CRUSH rule 0 x 531 [9]
+  CRUSH rule 0 x 532 [5]
+  CRUSH rule 0 x 533 [12]
+  CRUSH rule 0 x 534 [11]
+  CRUSH rule 0 x 535 [11]
+  CRUSH rule 0 x 536 [9]
+  CRUSH rule 0 x 537 [15]
+  CRUSH rule 0 x 538 [13]
+  CRUSH rule 0 x 539 [10]
+  CRUSH rule 0 x 540 [12]
+  CRUSH rule 0 x 541 [2]
+  CRUSH rule 0 x 542 [3]
+  CRUSH rule 0 x 543 [4]
+  CRUSH rule 0 x 544 [3]
+  CRUSH rule 0 x 545 [14]
+  CRUSH rule 0 x 546 [5]
+  CRUSH rule 0 x 547 [5]
+  CRUSH rule 0 x 548 [11]
+  CRUSH rule 0 x 549 [14]
+  CRUSH rule 0 x 550 [9]
+  CRUSH rule 0 x 551 [11]
+  CRUSH rule 0 x 552 [2]
+  CRUSH rule 0 x 553 [11]
+  CRUSH rule 0 x 554 [11]
+  CRUSH rule 0 x 555 [6]
+  CRUSH rule 0 x 556 [15]
+  CRUSH rule 0 x 557 [12]
+  CRUSH rule 0 x 558 [12]
+  CRUSH rule 0 x 559 [2]
+  CRUSH rule 0 x 560 [4]
+  CRUSH rule 0 x 561 [12]
+  CRUSH rule 0 x 562 [7]
+  CRUSH rule 0 x 563 [15]
+  CRUSH rule 0 x 564 [2]
+  CRUSH rule 0 x 565 [3]
+  CRUSH rule 0 x 566 [6]
+  CRUSH rule 0 x 567 [15]
+  CRUSH rule 0 x 568 [4]
+  CRUSH rule 0 x 569 [11]
+  CRUSH rule 0 x 570 [1]
+  CRUSH rule 0 x 571 [10]
+  CRUSH rule 0 x 572 [12]
+  CRUSH rule 0 x 573 [7]
+  CRUSH rule 0 x 574 [11]
+  CRUSH rule 0 x 575 [5]
+  CRUSH rule 0 x 576 [3]
+  CRUSH rule 0 x 577 [13]
+  CRUSH rule 0 x 578 [4]
+  CRUSH rule 0 x 579 [13]
+  CRUSH rule 0 x 580 [3]
+  CRUSH rule 0 x 581 [7]
+  CRUSH rule 0 x 582 [10]
+  CRUSH rule 0 x 583 [4]
+  CRUSH rule 0 x 584 [10]
+  CRUSH rule 0 x 585 [5]
+  CRUSH rule 0 x 586 [7]
+  CRUSH rule 0 x 587 [11]
+  CRUSH rule 0 x 588 [3]
+  CRUSH rule 0 x 589 [9]
+  CRUSH rule 0 x 590 [12]
+  CRUSH rule 0 x 591 [2]
+  CRUSH rule 0 x 592 [15]
+  CRUSH rule 0 x 593 [13]
+  CRUSH rule 0 x 594 [12]
+  CRUSH rule 0 x 595 [12]
+  CRUSH rule 0 x 596 [2]
+  CRUSH rule 0 x 597 [15]
+  CRUSH rule 0 x 598 [11]
+  CRUSH rule 0 x 599 [13]
+  CRUSH rule 0 x 600 [4]
+  CRUSH rule 0 x 601 [13]
+  CRUSH rule 0 x 602 [3]
+  CRUSH rule 0 x 603 [3]
+  CRUSH rule 0 x 604 [14]
+  CRUSH rule 0 x 605 [2]
+  CRUSH rule 0 x 606 [12]
+  CRUSH rule 0 x 607 [3]
+  CRUSH rule 0 x 608 [13]
+  CRUSH rule 0 x 609 [14]
+  CRUSH rule 0 x 610 [7]
+  CRUSH rule 0 x 611 [13]
+  CRUSH rule 0 x 612 [7]
+  CRUSH rule 0 x 613 [10]
+  CRUSH rule 0 x 614 [9]
+  CRUSH rule 0 x 615 [9]
+  CRUSH rule 0 x 616 [10]
+  CRUSH rule 0 x 617 [15]
+  CRUSH rule 0 x 618 [4]
+  CRUSH rule 0 x 619 [15]
+  CRUSH rule 0 x 620 [3]
+  CRUSH rule 0 x 621 [3]
+  CRUSH rule 0 x 622 [10]
+  CRUSH rule 0 x 623 [4]
+  CRUSH rule 0 x 624 [3]
+  CRUSH rule 0 x 625 [11]
+  CRUSH rule 0 x 626 [10]
+  CRUSH rule 0 x 627 [1]
+  CRUSH rule 0 x 628 [15]
+  CRUSH rule 0 x 629 [5]
+  CRUSH rule 0 x 630 [1]
+  CRUSH rule 0 x 631 [5]
+  CRUSH rule 0 x 632 [12]
+  CRUSH rule 0 x 633 [14]
+  CRUSH rule 0 x 634 [6]
+  CRUSH rule 0 x 635 [6]
+  CRUSH rule 0 x 636 [13]
+  CRUSH rule 0 x 637 [3]
+  CRUSH rule 0 x 638 [10]
+  CRUSH rule 0 x 639 [6]
+  CRUSH rule 0 x 640 [9]
+  CRUSH rule 0 x 641 [10]
+  CRUSH rule 0 x 642 [1]
+  CRUSH rule 0 x 643 [3]
+  CRUSH rule 0 x 644 [15]
+  CRUSH rule 0 x 645 [14]
+  CRUSH rule 0 x 646 [5]
+  CRUSH rule 0 x 647 [10]
+  CRUSH rule 0 x 648 [6]
+  CRUSH rule 0 x 649 [3]
+  CRUSH rule 0 x 650 [10]
+  CRUSH rule 0 x 651 [3]
+  CRUSH rule 0 x 652 [15]
+  CRUSH rule 0 x 653 [11]
+  CRUSH rule 0 x 654 [13]
+  CRUSH rule 0 x 655 [6]
+  CRUSH rule 0 x 656 [3]
+  CRUSH rule 0 x 657 [11]
+  CRUSH rule 0 x 658 [7]
+  CRUSH rule 0 x 659 [2]
+  CRUSH rule 0 x 660 [13]
+  CRUSH rule 0 x 661 [7]
+  CRUSH rule 0 x 662 [15]
+  CRUSH rule 0 x 663 [14]
+  CRUSH rule 0 x 664 [6]
+  CRUSH rule 0 x 665 [2]
+  CRUSH rule 0 x 666 [12]
+  CRUSH rule 0 x 667 [1]
+  CRUSH rule 0 x 668 [9]
+  CRUSH rule 0 x 669 [9]
+  CRUSH rule 0 x 670 [6]
+  CRUSH rule 0 x 671 [6]
+  CRUSH rule 0 x 672 [2]
+  CRUSH rule 0 x 673 [7]
+  CRUSH rule 0 x 674 [7]
+  CRUSH rule 0 x 675 [9]
+  CRUSH rule 0 x 676 [10]
+  CRUSH rule 0 x 677 [2]
+  CRUSH rule 0 x 678 [1]
+  CRUSH rule 0 x 679 [5]
+  CRUSH rule 0 x 680 [7]
+  CRUSH rule 0 x 681 [6]
+  CRUSH rule 0 x 682 [6]
+  CRUSH rule 0 x 683 [6]
+  CRUSH rule 0 x 684 [9]
+  CRUSH rule 0 x 685 [5]
+  CRUSH rule 0 x 686 [1]
+  CRUSH rule 0 x 687 [7]
+  CRUSH rule 0 x 688 [11]
+  CRUSH rule 0 x 689 [5]
+  CRUSH rule 0 x 690 [9]
+  CRUSH rule 0 x 691 [11]
+  CRUSH rule 0 x 692 [15]
+  CRUSH rule 0 x 693 [5]
+  CRUSH rule 0 x 694 [4]
+  CRUSH rule 0 x 695 [6]
+  CRUSH rule 0 x 696 [1]
+  CRUSH rule 0 x 697 [13]
+  CRUSH rule 0 x 698 [11]
+  CRUSH rule 0 x 699 [7]
+  CRUSH rule 0 x 700 [12]
+  CRUSH rule 0 x 701 [3]
+  CRUSH rule 0 x 702 [3]
+  CRUSH rule 0 x 703 [15]
+  CRUSH rule 0 x 704 [6]
+  CRUSH rule 0 x 705 [14]
+  CRUSH rule 0 x 706 [1]
+  CRUSH rule 0 x 707 [4]
+  CRUSH rule 0 x 708 [3]
+  CRUSH rule 0 x 709 [11]
+  CRUSH rule 0 x 710 [14]
+  CRUSH rule 0 x 711 [14]
+  CRUSH rule 0 x 712 [12]
+  CRUSH rule 0 x 713 [11]
+  CRUSH rule 0 x 714 [12]
+  CRUSH rule 0 x 715 [6]
+  CRUSH rule 0 x 716 [11]
+  CRUSH rule 0 x 717 [12]
+  CRUSH rule 0 x 718 [7]
+  CRUSH rule 0 x 719 [5]
+  CRUSH rule 0 x 720 [4]
+  CRUSH rule 0 x 721 [11]
+  CRUSH rule 0 x 722 [2]
+  CRUSH rule 0 x 723 [2]
+  CRUSH rule 0 x 724 [7]
+  CRUSH rule 0 x 725 [11]
+  CRUSH rule 0 x 726 [7]
+  CRUSH rule 0 x 727 [2]
+  CRUSH rule 0 x 728 [13]
+  CRUSH rule 0 x 729 [15]
+  CRUSH rule 0 x 730 [3]
+  CRUSH rule 0 x 731 [9]
+  CRUSH rule 0 x 732 [1]
+  CRUSH rule 0 x 733 [11]
+  CRUSH rule 0 x 734 [14]
+  CRUSH rule 0 x 735 [6]
+  CRUSH rule 0 x 736 [3]
+  CRUSH rule 0 x 737 [1]
+  CRUSH rule 0 x 738 [11]
+  CRUSH rule 0 x 739 [11]
+  CRUSH rule 0 x 740 [7]
+  CRUSH rule 0 x 741 [12]
+  CRUSH rule 0 x 742 [9]
+  CRUSH rule 0 x 743 [5]
+  CRUSH rule 0 x 744 [6]
+  CRUSH rule 0 x 745 [3]
+  CRUSH rule 0 x 746 [3]
+  CRUSH rule 0 x 747 [15]
+  CRUSH rule 0 x 748 [6]
+  CRUSH rule 0 x 749 [14]
+  CRUSH rule 0 x 750 [1]
+  CRUSH rule 0 x 751 [15]
+  CRUSH rule 0 x 752 [13]
+  CRUSH rule 0 x 753 [4]
+  CRUSH rule 0 x 754 [14]
+  CRUSH rule 0 x 755 [13]
+  CRUSH rule 0 x 756 [3]
+  CRUSH rule 0 x 757 [10]
+  CRUSH rule 0 x 758 [6]
+  CRUSH rule 0 x 759 [5]
+  CRUSH rule 0 x 760 [1]
+  CRUSH rule 0 x 761 [2]
+  CRUSH rule 0 x 762 [1]
+  CRUSH rule 0 x 763 [4]
+  CRUSH rule 0 x 764 [1]
+  CRUSH rule 0 x 765 [9]
+  CRUSH rule 0 x 766 [11]
+  CRUSH rule 0 x 767 [6]
+  CRUSH rule 0 x 768 [2]
+  CRUSH rule 0 x 769 [15]
+  CRUSH rule 0 x 770 [15]
+  CRUSH rule 0 x 771 [9]
+  CRUSH rule 0 x 772 [4]
+  CRUSH rule 0 x 773 [3]
+  CRUSH rule 0 x 774 [12]
+  CRUSH rule 0 x 775 [5]
+  CRUSH rule 0 x 776 [10]
+  CRUSH rule 0 x 777 [11]
+  CRUSH rule 0 x 778 [13]
+  CRUSH rule 0 x 779 [5]
+  CRUSH rule 0 x 780 [13]
+  CRUSH rule 0 x 781 [5]
+  CRUSH rule 0 x 782 [2]
+  CRUSH rule 0 x 783 [12]
+  CRUSH rule 0 x 784 [14]
+  CRUSH rule 0 x 785 [6]
+  CRUSH rule 0 x 786 [10]
+  CRUSH rule 0 x 787 [1]
+  CRUSH rule 0 x 788 [4]
+  CRUSH rule 0 x 789 [9]
+  CRUSH rule 0 x 790 [15]
+  CRUSH rule 0 x 791 [9]
+  CRUSH rule 0 x 792 [6]
+  CRUSH rule 0 x 793 [15]
+  CRUSH rule 0 x 794 [5]
+  CRUSH rule 0 x 795 [6]
+  CRUSH rule 0 x 796 [11]
+  CRUSH rule 0 x 797 [14]
+  CRUSH rule 0 x 798 [5]
+  CRUSH rule 0 x 799 [2]
+  CRUSH rule 0 x 800 [6]
+  CRUSH rule 0 x 801 [2]
+  CRUSH rule 0 x 802 [1]
+  CRUSH rule 0 x 803 [7]
+  CRUSH rule 0 x 804 [5]
+  CRUSH rule 0 x 805 [13]
+  CRUSH rule 0 x 806 [6]
+  CRUSH rule 0 x 807 [14]
+  CRUSH rule 0 x 808 [2]
+  CRUSH rule 0 x 809 [1]
+  CRUSH rule 0 x 810 [2]
+  CRUSH rule 0 x 811 [15]
+  CRUSH rule 0 x 812 [7]
+  CRUSH rule 0 x 813 [4]
+  CRUSH rule 0 x 814 [13]
+  CRUSH rule 0 x 815 [15]
+  CRUSH rule 0 x 816 [14]
+  CRUSH rule 0 x 817 [10]
+  CRUSH rule 0 x 818 [15]
+  CRUSH rule 0 x 819 [5]
+  CRUSH rule 0 x 820 [3]
+  CRUSH rule 0 x 821 [15]
+  CRUSH rule 0 x 822 [10]
+  CRUSH rule 0 x 823 [2]
+  CRUSH rule 0 x 824 [3]
+  CRUSH rule 0 x 825 [10]
+  CRUSH rule 0 x 826 [5]
+  CRUSH rule 0 x 827 [13]
+  CRUSH rule 0 x 828 [12]
+  CRUSH rule 0 x 829 [13]
+  CRUSH rule 0 x 830 [15]
+  CRUSH rule 0 x 831 [1]
+  CRUSH rule 0 x 832 [14]
+  CRUSH rule 0 x 833 [9]
+  CRUSH rule 0 x 834 [9]
+  CRUSH rule 0 x 835 [14]
+  CRUSH rule 0 x 836 [3]
+  CRUSH rule 0 x 837 [15]
+  CRUSH rule 0 x 838 [12]
+  CRUSH rule 0 x 839 [3]
+  CRUSH rule 0 x 840 [10]
+  CRUSH rule 0 x 841 [3]
+  CRUSH rule 0 x 842 [9]
+  CRUSH rule 0 x 843 [14]
+  CRUSH rule 0 x 844 [7]
+  CRUSH rule 0 x 845 [13]
+  CRUSH rule 0 x 846 [3]
+  CRUSH rule 0 x 847 [12]
+  CRUSH rule 0 x 848 [11]
+  CRUSH rule 0 x 849 [3]
+  CRUSH rule 0 x 850 [1]
+  CRUSH rule 0 x 851 [14]
+  CRUSH rule 0 x 852 [9]
+  CRUSH rule 0 x 853 [13]
+  CRUSH rule 0 x 854 [7]
+  CRUSH rule 0 x 855 [14]
+  CRUSH rule 0 x 856 [5]
+  CRUSH rule 0 x 857 [4]
+  CRUSH rule 0 x 858 [5]
+  CRUSH rule 0 x 859 [5]
+  CRUSH rule 0 x 860 [11]
+  CRUSH rule 0 x 861 [13]
+  CRUSH rule 0 x 862 [5]
+  CRUSH rule 0 x 863 [11]
+  CRUSH rule 0 x 864 [6]
+  CRUSH rule 0 x 865 [4]
+  CRUSH rule 0 x 866 [2]
+  CRUSH rule 0 x 867 [12]
+  CRUSH rule 0 x 868 [14]
+  CRUSH rule 0 x 869 [10]
+  CRUSH rule 0 x 870 [14]
+  CRUSH rule 0 x 871 [6]
+  CRUSH rule 0 x 872 [6]
+  CRUSH rule 0 x 873 [2]
+  CRUSH rule 0 x 874 [12]
+  CRUSH rule 0 x 875 [10]
+  CRUSH rule 0 x 876 [14]
+  CRUSH rule 0 x 877 [15]
+  CRUSH rule 0 x 878 [7]
+  CRUSH rule 0 x 879 [12]
+  CRUSH rule 0 x 880 [2]
+  CRUSH rule 0 x 881 [6]
+  CRUSH rule 0 x 882 [11]
+  CRUSH rule 0 x 883 [13]
+  CRUSH rule 0 x 884 [6]
+  CRUSH rule 0 x 885 [14]
+  CRUSH rule 0 x 886 [13]
+  CRUSH rule 0 x 887 [14]
+  CRUSH rule 0 x 888 [10]
+  CRUSH rule 0 x 889 [15]
+  CRUSH rule 0 x 890 [10]
+  CRUSH rule 0 x 891 [9]
+  CRUSH rule 0 x 892 [12]
+  CRUSH rule 0 x 893 [1]
+  CRUSH rule 0 x 894 [7]
+  CRUSH rule 0 x 895 [2]
+  CRUSH rule 0 x 896 [9]
+  CRUSH rule 0 x 897 [7]
+  CRUSH rule 0 x 898 [10]
+  CRUSH rule 0 x 899 [1]
+  CRUSH rule 0 x 900 [2]
+  CRUSH rule 0 x 901 [9]
+  CRUSH rule 0 x 902 [4]
+  CRUSH rule 0 x 903 [14]
+  CRUSH rule 0 x 904 [15]
+  CRUSH rule 0 x 905 [12]
+  CRUSH rule 0 x 906 [14]
+  CRUSH rule 0 x 907 [7]
+  CRUSH rule 0 x 908 [2]
+  CRUSH rule 0 x 909 [10]
+  CRUSH rule 0 x 910 [12]
+  CRUSH rule 0 x 911 [11]
+  CRUSH rule 0 x 912 [6]
+  CRUSH rule 0 x 913 [4]
+  CRUSH rule 0 x 914 [4]
+  CRUSH rule 0 x 915 [12]
+  CRUSH rule 0 x 916 [3]
+  CRUSH rule 0 x 917 [1]
+  CRUSH rule 0 x 918 [7]
+  CRUSH rule 0 x 919 [10]
+  CRUSH rule 0 x 920 [4]
+  CRUSH rule 0 x 921 [1]
+  CRUSH rule 0 x 922 [6]
+  CRUSH rule 0 x 923 [12]
+  CRUSH rule 0 x 924 [6]
+  CRUSH rule 0 x 925 [12]
+  CRUSH rule 0 x 926 [3]
+  CRUSH rule 0 x 927 [6]
+  CRUSH rule 0 x 928 [13]
+  CRUSH rule 0 x 929 [10]
+  CRUSH rule 0 x 930 [7]
+  CRUSH rule 0 x 931 [6]
+  CRUSH rule 0 x 932 [13]
+  CRUSH rule 0 x 933 [12]
+  CRUSH rule 0 x 934 [12]
+  CRUSH rule 0 x 935 [6]
+  CRUSH rule 0 x 936 [9]
+  CRUSH rule 0 x 937 [14]
+  CRUSH rule 0 x 938 [14]
+  CRUSH rule 0 x 939 [6]
+  CRUSH rule 0 x 940 [13]
+  CRUSH rule 0 x 941 [3]
+  CRUSH rule 0 x 942 [15]
+  CRUSH rule 0 x 943 [10]
+  CRUSH rule 0 x 944 [2]
+  CRUSH rule 0 x 945 [10]
+  CRUSH rule 0 x 946 [11]
+  CRUSH rule 0 x 947 [11]
+  CRUSH rule 0 x 948 [7]
+  CRUSH rule 0 x 949 [9]
+  CRUSH rule 0 x 950 [9]
+  CRUSH rule 0 x 951 [2]
+  CRUSH rule 0 x 952 [9]
+  CRUSH rule 0 x 953 [1]
+  CRUSH rule 0 x 954 [10]
+  CRUSH rule 0 x 955 [7]
+  CRUSH rule 0 x 956 [1]
+  CRUSH rule 0 x 957 [14]
+  CRUSH rule 0 x 958 [15]
+  CRUSH rule 0 x 959 [2]
+  CRUSH rule 0 x 960 [2]
+  CRUSH rule 0 x 961 [3]
+  CRUSH rule 0 x 962 [5]
+  CRUSH rule 0 x 963 [13]
+  CRUSH rule 0 x 964 [7]
+  CRUSH rule 0 x 965 [12]
+  CRUSH rule 0 x 966 [12]
+  CRUSH rule 0 x 967 [7]
+  CRUSH rule 0 x 968 [12]
+  CRUSH rule 0 x 969 [11]
+  CRUSH rule 0 x 970 [5]
+  CRUSH rule 0 x 971 [1]
+  CRUSH rule 0 x 972 [12]
+  CRUSH rule 0 x 973 [1]
+  CRUSH rule 0 x 974 [7]
+  CRUSH rule 0 x 975 [7]
+  CRUSH rule 0 x 976 [7]
+  CRUSH rule 0 x 977 [14]
+  CRUSH rule 0 x 978 [12]
+  CRUSH rule 0 x 979 [5]
+  CRUSH rule 0 x 980 [15]
+  CRUSH rule 0 x 981 [5]
+  CRUSH rule 0 x 982 [2]
+  CRUSH rule 0 x 983 [3]
+  CRUSH rule 0 x 984 [15]
+  CRUSH rule 0 x 985 [11]
+  CRUSH rule 0 x 986 [6]
+  CRUSH rule 0 x 987 [13]
+  CRUSH rule 0 x 988 [12]
+  CRUSH rule 0 x 989 [7]
+  CRUSH rule 0 x 990 [1]
+  CRUSH rule 0 x 991 [7]
+  CRUSH rule 0 x 992 [9]
+  CRUSH rule 0 x 993 [6]
+  CRUSH rule 0 x 994 [3]
+  CRUSH rule 0 x 995 [15]
+  CRUSH rule 0 x 996 [15]
+  CRUSH rule 0 x 997 [15]
+  CRUSH rule 0 x 998 [6]
+  CRUSH rule 0 x 999 [9]
+  CRUSH rule 0 x 1000 [14]
+  CRUSH rule 0 x 1001 [11]
+  CRUSH rule 0 x 1002 [1]
+  CRUSH rule 0 x 1003 [10]
+  CRUSH rule 0 x 1004 [15]
+  CRUSH rule 0 x 1005 [6]
+  CRUSH rule 0 x 1006 [10]
+  CRUSH rule 0 x 1007 [1]
+  CRUSH rule 0 x 1008 [7]
+  CRUSH rule 0 x 1009 [5]
+  CRUSH rule 0 x 1010 [10]
+  CRUSH rule 0 x 1011 [6]
+  CRUSH rule 0 x 1012 [12]
+  CRUSH rule 0 x 1013 [2]
+  CRUSH rule 0 x 1014 [1]
+  CRUSH rule 0 x 1015 [12]
+  CRUSH rule 0 x 1016 [10]
+  CRUSH rule 0 x 1017 [5]
+  CRUSH rule 0 x 1018 [13]
+  CRUSH rule 0 x 1019 [10]
+  CRUSH rule 0 x 1020 [3]
+  CRUSH rule 0 x 1021 [2]
+  CRUSH rule 0 x 1022 [15]
+  CRUSH rule 0 x 1023 [15]
+  rule 0 (replicated_ruleset) num_rep 1 result size == 1:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [7,10]
+  CRUSH rule 0 x 1 [10,15]
+  CRUSH rule 0 x 2 [1,12]
+  CRUSH rule 0 x 3 [15,4]
+  CRUSH rule 0 x 4 [14,2]
+  CRUSH rule 0 x 5 [7,4]
+  CRUSH rule 0 x 6 [12,6]
+  CRUSH rule 0 x 7 [9,2]
+  CRUSH rule 0 x 8 [10,2]
+  CRUSH rule 0 x 9 [7,1]
+  CRUSH rule 0 x 10 [10,14]
+  CRUSH rule 0 x 11 [13,9]
+  CRUSH rule 0 x 12 [7,1]
+  CRUSH rule 0 x 13 [3,5]
+  CRUSH rule 0 x 14 [13,5]
+  CRUSH rule 0 x 15 [15,1]
+  CRUSH rule 0 x 16 [7,11]
+  CRUSH rule 0 x 17 [10,1]
+  CRUSH rule 0 x 18 [1,7]
+  CRUSH rule 0 x 19 [7,12]
+  CRUSH rule 0 x 20 [14,12]
+  CRUSH rule 0 x 21 [3,12]
+  CRUSH rule 0 x 22 [6,3]
+  CRUSH rule 0 x 23 [10,5]
+  CRUSH rule 0 x 24 [12,11]
+  CRUSH rule 0 x 25 [7,12]
+  CRUSH rule 0 x 26 [1,7]
+  CRUSH rule 0 x 27 [3,6]
+  CRUSH rule 0 x 28 [14,4]
+  CRUSH rule 0 x 29 [5,14]
+  CRUSH rule 0 x 30 [2,5]
+  CRUSH rule 0 x 31 [5,15]
+  CRUSH rule 0 x 32 [9,10]
+  CRUSH rule 0 x 33 [13,4]
+  CRUSH rule 0 x 34 [13,15]
+  CRUSH rule 0 x 35 [4,14]
+  CRUSH rule 0 x 36 [3,12]
+  CRUSH rule 0 x 37 [9,2]
+  CRUSH rule 0 x 38 [3,4]
+  CRUSH rule 0 x 39 [12,7]
+  CRUSH rule 0 x 40 [10,1]
+  CRUSH rule 0 x 41 [4,9]
+  CRUSH rule 0 x 42 [3,6]
+  CRUSH rule 0 x 43 [10,5]
+  CRUSH rule 0 x 44 [11,4]
+  CRUSH rule 0 x 45 [11,12]
+  CRUSH rule 0 x 46 [6,9]
+  CRUSH rule 0 x 47 [3,9]
+  CRUSH rule 0 x 48 [4,6]
+  CRUSH rule 0 x 49 [9,15]
+  CRUSH rule 0 x 50 [14,12]
+  CRUSH rule 0 x 51 [10,6]
+  CRUSH rule 0 x 52 [12,1]
+  CRUSH rule 0 x 53 [3,6]
+  CRUSH rule 0 x 54 [4,13]
+  CRUSH rule 0 x 55 [4,11]
+  CRUSH rule 0 x 56 [5,9]
+  CRUSH rule 0 x 57 [6,2]
+  CRUSH rule 0 x 58 [7,1]
+  CRUSH rule 0 x 59 [2,13]
+  CRUSH rule 0 x 60 [3,6]
+  CRUSH rule 0 x 61 [3,15]
+  CRUSH rule 0 x 62 [15,11]
+  CRUSH rule 0 x 63 [10,14]
+  CRUSH rule 0 x 64 [3,9]
+  CRUSH rule 0 x 65 [4,12]
+  CRUSH rule 0 x 66 [15,11]
+  CRUSH rule 0 x 67 [2,6]
+  CRUSH rule 0 x 68 [15,7]
+  CRUSH rule 0 x 69 [2,1]
+  CRUSH rule 0 x 70 [9,6]
+  CRUSH rule 0 x 71 [15,5]
+  CRUSH rule 0 x 72 [9,10]
+  CRUSH rule 0 x 73 [5,3]
+  CRUSH rule 0 x 74 [11,7]
+  CRUSH rule 0 x 75 [9,7]
+  CRUSH rule 0 x 76 [6,1]
+  CRUSH rule 0 x 77 [7,4]
+  CRUSH rule 0 x 78 [9,3]
+  CRUSH rule 0 x 79 [13,2]
+  CRUSH rule 0 x 80 [15,2]
+  CRUSH rule 0 x 81 [15,2]
+  CRUSH rule 0 x 82 [14,13]
+  CRUSH rule 0 x 83 [4,15]
+  CRUSH rule 0 x 84 [10,7]
+  CRUSH rule 0 x 85 [3,15]
+  CRUSH rule 0 x 86 [10,9]
+  CRUSH rule 0 x 87 [15,10]
+  CRUSH rule 0 x 88 [4,13]
+  CRUSH rule 0 x 89 [3,9]
+  CRUSH rule 0 x 90 [4,9]
+  CRUSH rule 0 x 91 [6,11]
+  CRUSH rule 0 x 92 [1,5]
+  CRUSH rule 0 x 93 [9,3]
+  CRUSH rule 0 x 94 [9,2]
+  CRUSH rule 0 x 95 [7,15]
+  CRUSH rule 0 x 96 [2,15]
+  CRUSH rule 0 x 97 [4,11]
+  CRUSH rule 0 x 98 [11,13]
+  CRUSH rule 0 x 99 [12,4]
+  CRUSH rule 0 x 100 [9,4]
+  CRUSH rule 0 x 101 [15,7]
+  CRUSH rule 0 x 102 [3,11]
+  CRUSH rule 0 x 103 [13,11]
+  CRUSH rule 0 x 104 [14,6]
+  CRUSH rule 0 x 105 [14,10]
+  CRUSH rule 0 x 106 [6,5]
+  CRUSH rule 0 x 107 [3,1]
+  CRUSH rule 0 x 108 [5,10]
+  CRUSH rule 0 x 109 [9,1]
+  CRUSH rule 0 x 110 [5,1]
+  CRUSH rule 0 x 111 [10,1]
+  CRUSH rule 0 x 112 [1,10]
+  CRUSH rule 0 x 113 [6,10]
+  CRUSH rule 0 x 114 [5,13]
+  CRUSH rule 0 x 115 [10,13]
+  CRUSH rule 0 x 116 [1,14]
+  CRUSH rule 0 x 117 [5,6]
+  CRUSH rule 0 x 118 [10,4]
+  CRUSH rule 0 x 119 [14,12]
+  CRUSH rule 0 x 120 [11,3]
+  CRUSH rule 0 x 121 [9,5]
+  CRUSH rule 0 x 122 [4,3]
+  CRUSH rule 0 x 123 [3,10]
+  CRUSH rule 0 x 124 [12,2]
+  CRUSH rule 0 x 125 [9,12]
+  CRUSH rule 0 x 126 [7,15]
+  CRUSH rule 0 x 127 [4,14]
+  CRUSH rule 0 x 128 [3,12]
+  CRUSH rule 0 x 129 [11,13]
+  CRUSH rule 0 x 130 [3,13]
+  CRUSH rule 0 x 131 [12,1]
+  CRUSH rule 0 x 132 [11,15]
+  CRUSH rule 0 x 133 [3,6]
+  CRUSH rule 0 x 134 [12,5]
+  CRUSH rule 0 x 135 [3,14]
+  CRUSH rule 0 x 136 [15,6]
+  CRUSH rule 0 x 137 [14,3]
+  CRUSH rule 0 x 138 [13,15]
+  CRUSH rule 0 x 139 [11,2]
+  CRUSH rule 0 x 140 [11,4]
+  CRUSH rule 0 x 141 [6,12]
+  CRUSH rule 0 x 142 [3,14]
+  CRUSH rule 0 x 143 [9,6]
+  CRUSH rule 0 x 144 [13,7]
+  CRUSH rule 0 x 145 [12,2]
+  CRUSH rule 0 x 146 [1,5]
+  CRUSH rule 0 x 147 [1,4]
+  CRUSH rule 0 x 148 [12,7]
+  CRUSH rule 0 x 149 [2,5]
+  CRUSH rule 0 x 150 [1,15]
+  CRUSH rule 0 x 151 [2,9]
+  CRUSH rule 0 x 152 [5,9]
+  CRUSH rule 0 x 153 [6,9]
+  CRUSH rule 0 x 154 [3,11]
+  CRUSH rule 0 x 155 [14,12]
+  CRUSH rule 0 x 156 [7,13]
+  CRUSH rule 0 x 157 [15,1]
+  CRUSH rule 0 x 158 [15,1]
+  CRUSH rule 0 x 159 [4,14]
+  CRUSH rule 0 x 160 [5,7]
+  CRUSH rule 0 x 161 [1,2]
+  CRUSH rule 0 x 162 [10,6]
+  CRUSH rule 0 x 163 [15,1]
+  CRUSH rule 0 x 164 [9,14]
+  CRUSH rule 0 x 165 [11,7]
+  CRUSH rule 0 x 166 [1,2]
+  CRUSH rule 0 x 167 [9,7]
+  CRUSH rule 0 x 168 [13,2]
+  CRUSH rule 0 x 169 [1,4]
+  CRUSH rule 0 x 170 [1,15]
+  CRUSH rule 0 x 171 [9,2]
+  CRUSH rule 0 x 172 [14,4]
+  CRUSH rule 0 x 173 [5,10]
+  CRUSH rule 0 x 174 [15,6]
+  CRUSH rule 0 x 175 [5,7]
+  CRUSH rule 0 x 176 [9,6]
+  CRUSH rule 0 x 177 [2,9]
+  CRUSH rule 0 x 178 [12,11]
+  CRUSH rule 0 x 179 [2,10]
+  CRUSH rule 0 x 180 [3,11]
+  CRUSH rule 0 x 181 [9,12]
+  CRUSH rule 0 x 182 [5,13]
+  CRUSH rule 0 x 183 [5,7]
+  CRUSH rule 0 x 184 [2,5]
+  CRUSH rule 0 x 185 [13,5]
+  CRUSH rule 0 x 186 [6,14]
+  CRUSH rule 0 x 187 [1,4]
+  CRUSH rule 0 x 188 [9,13]
+  CRUSH rule 0 x 189 [6,12]
+  CRUSH rule 0 x 190 [9,13]
+  CRUSH rule 0 x 191 [7,11]
+  CRUSH rule 0 x 192 [2,11]
+  CRUSH rule 0 x 193 [3,13]
+  CRUSH rule 0 x 194 [3,13]
+  CRUSH rule 0 x 195 [5,7]
+  CRUSH rule 0 x 196 [4,15]
+  CRUSH rule 0 x 197 [14,10]
+  CRUSH rule 0 x 198 [2,5]
+  CRUSH rule 0 x 199 [2,10]
+  CRUSH rule 0 x 200 [7,14]
+  CRUSH rule 0 x 201 [9,14]
+  CRUSH rule 0 x 202 [14,11]
+  CRUSH rule 0 x 203 [12,5]
+  CRUSH rule 0 x 204 [6,11]
+  CRUSH rule 0 x 205 [15,4]
+  CRUSH rule 0 x 206 [13,11]
+  CRUSH rule 0 x 207 [2,11]
+  CRUSH rule 0 x 208 [13,1]
+  CRUSH rule 0 x 209 [6,15]
+  CRUSH rule 0 x 210 [13,11]
+  CRUSH rule 0 x 211 [2,14]
+  CRUSH rule 0 x 212 [10,1]
+  CRUSH rule 0 x 213 [3,9]
+  CRUSH rule 0 x 214 [7,15]
+  CRUSH rule 0 x 215 [6,1]
+  CRUSH rule 0 x 216 [12,9]
+  CRUSH rule 0 x 217 [12,11]
+  CRUSH rule 0 x 218 [12,10]
+  CRUSH rule 0 x 219 [3,11]
+  CRUSH rule 0 x 220 [14,4]
+  CRUSH rule 0 x 221 [15,5]
+  CRUSH rule 0 x 222 [10,4]
+  CRUSH rule 0 x 223 [9,7]
+  CRUSH rule 0 x 224 [1,7]
+  CRUSH rule 0 x 225 [10,5]
+  CRUSH rule 0 x 226 [4,1]
+  CRUSH rule 0 x 227 [7,2]
+  CRUSH rule 0 x 228 [2,15]
+  CRUSH rule 0 x 229 [9,3]
+  CRUSH rule 0 x 230 [10,5]
+  CRUSH rule 0 x 231 [2,7]
+  CRUSH rule 0 x 232 [10,5]
+  CRUSH rule 0 x 233 [6,12]
+  CRUSH rule 0 x 234 [10,1]
+  CRUSH rule 0 x 235 [13,14]
+  CRUSH rule 0 x 236 [2,15]
+  CRUSH rule 0 x 237 [3,12]
+  CRUSH rule 0 x 238 [2,10]
+  CRUSH rule 0 x 239 [4,15]
+  CRUSH rule 0 x 240 [15,5]
+  CRUSH rule 0 x 241 [7,9]
+  CRUSH rule 0 x 242 [14,2]
+  CRUSH rule 0 x 243 [2,11]
+  CRUSH rule 0 x 244 [13,9]
+  CRUSH rule 0 x 245 [12,9]
+  CRUSH rule 0 x 246 [15,3]
+  CRUSH rule 0 x 247 [6,4]
+  CRUSH rule 0 x 248 [5,13]
+  CRUSH rule 0 x 249 [10,14]
+  CRUSH rule 0 x 250 [12,15]
+  CRUSH rule 0 x 251 [13,2]
+  CRUSH rule 0 x 252 [7,5]
+  CRUSH rule 0 x 253 [3,13]
+  CRUSH rule 0 x 254 [2,9]
+  CRUSH rule 0 x 255 [1,9]
+  CRUSH rule 0 x 256 [6,9]
+  CRUSH rule 0 x 257 [15,12]
+  CRUSH rule 0 x 258 [12,5]
+  CRUSH rule 0 x 259 [9,10]
+  CRUSH rule 0 x 260 [10,12]
+  CRUSH rule 0 x 261 [13,7]
+  CRUSH rule 0 x 262 [15,3]
+  CRUSH rule 0 x 263 [12,6]
+  CRUSH rule 0 x 264 [13,14]
+  CRUSH rule 0 x 265 [12,10]
+  CRUSH rule 0 x 266 [14,7]
+  CRUSH rule 0 x 267 [12,11]
+  CRUSH rule 0 x 268 [4,1]
+  CRUSH rule 0 x 269 [11,1]
+  CRUSH rule 0 x 270 [7,11]
+  CRUSH rule 0 x 271 [4,7]
+  CRUSH rule 0 x 272 [15,5]
+  CRUSH rule 0 x 273 [2,10]
+  CRUSH rule 0 x 274 [10,2]
+  CRUSH rule 0 x 275 [10,3]
+  CRUSH rule 0 x 276 [5,12]
+  CRUSH rule 0 x 277 [14,3]
+  CRUSH rule 0 x 278 [5,6]
+  CRUSH rule 0 x 279 [6,10]
+  CRUSH rule 0 x 280 [7,3]
+  CRUSH rule 0 x 281 [5,11]
+  CRUSH rule 0 x 282 [2,1]
+  CRUSH rule 0 x 283 [4,1]
+  CRUSH rule 0 x 284 [5,11]
+  CRUSH rule 0 x 285 [15,5]
+  CRUSH rule 0 x 286 [10,4]
+  CRUSH rule 0 x 287 [12,4]
+  CRUSH rule 0 x 288 [4,12]
+  CRUSH rule 0 x 289 [2,5]
+  CRUSH rule 0 x 290 [12,2]
+  CRUSH rule 0 x 291 [7,11]
+  CRUSH rule 0 x 292 [4,10]
+  CRUSH rule 0 x 293 [6,5]
+  CRUSH rule 0 x 294 [9,12]
+  CRUSH rule 0 x 295 [6,10]
+  CRUSH rule 0 x 296 [3,1]
+  CRUSH rule 0 x 297 [6,13]
+  CRUSH rule 0 x 298 [14,9]
+  CRUSH rule 0 x 299 [14,12]
+  CRUSH rule 0 x 300 [15,7]
+  CRUSH rule 0 x 301 [9,11]
+  CRUSH rule 0 x 302 [9,7]
+  CRUSH rule 0 x 303 [4,13]
+  CRUSH rule 0 x 304 [6,9]
+  CRUSH rule 0 x 305 [13,7]
+  CRUSH rule 0 x 306 [10,12]
+  CRUSH rule 0 x 307 [11,12]
+  CRUSH rule 0 x 308 [12,14]
+  CRUSH rule 0 x 309 [9,3]
+  CRUSH rule 0 x 310 [3,1]
+  CRUSH rule 0 x 311 [3,9]
+  CRUSH rule 0 x 312 [15,13]
+  CRUSH rule 0 x 313 [9,15]
+  CRUSH rule 0 x 314 [2,15]
+  CRUSH rule 0 x 315 [15,2]
+  CRUSH rule 0 x 316 [4,9]
+  CRUSH rule 0 x 317 [1,5]
+  CRUSH rule 0 x 318 [4,1]
+  CRUSH rule 0 x 319 [2,15]
+  CRUSH rule 0 x 320 [5,7]
+  CRUSH rule 0 x 321 [1,6]
+  CRUSH rule 0 x 322 [13,7]
+  CRUSH rule 0 x 323 [7,4]
+  CRUSH rule 0 x 324 [5,6]
+  CRUSH rule 0 x 325 [9,10]
+  CRUSH rule 0 x 326 [11,7]
+  CRUSH rule 0 x 327 [12,5]
+  CRUSH rule 0 x 328 [5,2]
+  CRUSH rule 0 x 329 [2,6]
+  CRUSH rule 0 x 330 [3,9]
+  CRUSH rule 0 x 331 [12,14]
+  CRUSH rule 0 x 332 [10,12]
+  CRUSH rule 0 x 333 [6,5]
+  CRUSH rule 0 x 334 [4,9]
+  CRUSH rule 0 x 335 [11,7]
+  CRUSH rule 0 x 336 [6,14]
+  CRUSH rule 0 x 337 [15,11]
+  CRUSH rule 0 x 338 [10,5]
+  CRUSH rule 0 x 339 [11,14]
+  CRUSH rule 0 x 340 [11,6]
+  CRUSH rule 0 x 341 [7,5]
+  CRUSH rule 0 x 342 [12,14]
+  CRUSH rule 0 x 343 [12,14]
+  CRUSH rule 0 x 344 [9,11]
+  CRUSH rule 0 x 345 [14,2]
+  CRUSH rule 0 x 346 [5,3]
+  CRUSH rule 0 x 347 [10,2]
+  CRUSH rule 0 x 348 [7,9]
+  CRUSH rule 0 x 349 [9,6]
+  CRUSH rule 0 x 350 [13,9]
+  CRUSH rule 0 x 351 [13,5]
+  CRUSH rule 0 x 352 [1,12]
+  CRUSH rule 0 x 353 [10,14]
+  CRUSH rule 0 x 354 [6,3]
+  CRUSH rule 0 x 355 [13,14]
+  CRUSH rule 0 x 356 [15,13]
+  CRUSH rule 0 x 357 [4,11]
+  CRUSH rule 0 x 358 [12,7]
+  CRUSH rule 0 x 359 [5,15]
+  CRUSH rule 0 x 360 [13,10]
+  CRUSH rule 0 x 361 [5,3]
+  CRUSH rule 0 x 362 [2,9]
+  CRUSH rule 0 x 363 [7,12]
+  CRUSH rule 0 x 364 [2,12]
+  CRUSH rule 0 x 365 [13,5]
+  CRUSH rule 0 x 366 [12,7]
+  CRUSH rule 0 x 367 [7,13]
+  CRUSH rule 0 x 368 [7,9]
+  CRUSH rule 0 x 369 [7,5]
+  CRUSH rule 0 x 370 [4,7]
+  CRUSH rule 0 x 371 [1,7]
+  CRUSH rule 0 x 372 [10,4]
+  CRUSH rule 0 x 373 [15,5]
+  CRUSH rule 0 x 374 [3,15]
+  CRUSH rule 0 x 375 [5,2]
+  CRUSH rule 0 x 376 [5,14]
+  CRUSH rule 0 x 377 [1,15]
+  CRUSH rule 0 x 378 [9,12]
+  CRUSH rule 0 x 379 [11,2]
+  CRUSH rule 0 x 380 [6,1]
+  CRUSH rule 0 x 381 [15,13]
+  CRUSH rule 0 x 382 [14,3]
+  CRUSH rule 0 x 383 [3,6]
+  CRUSH rule 0 x 384 [4,13]
+  CRUSH rule 0 x 385 [4,6]
+  CRUSH rule 0 x 386 [14,3]
+  CRUSH rule 0 x 387 [1,11]
+  CRUSH rule 0 x 388 [2,6]
+  CRUSH rule 0 x 389 [12,7]
+  CRUSH rule 0 x 390 [2,11]
+  CRUSH rule 0 x 391 [3,4]
+  CRUSH rule 0 x 392 [11,5]
+  CRUSH rule 0 x 393 [2,14]
+  CRUSH rule 0 x 394 [4,9]
+  CRUSH rule 0 x 395 [10,13]
+  CRUSH rule 0 x 396 [2,12]
+  CRUSH rule 0 x 397 [1,14]
+  CRUSH rule 0 x 398 [9,2]
+  CRUSH rule 0 x 399 [5,9]
+  CRUSH rule 0 x 400 [10,6]
+  CRUSH rule 0 x 401 [6,9]
+  CRUSH rule 0 x 402 [4,7]
+  CRUSH rule 0 x 403 [7,15]
+  CRUSH rule 0 x 404 [14,12]
+  CRUSH rule 0 x 405 [9,15]
+  CRUSH rule 0 x 406 [12,14]
+  CRUSH rule 0 x 407 [9,5]
+  CRUSH rule 0 x 408 [7,1]
+  CRUSH rule 0 x 409 [11,2]
+  CRUSH rule 0 x 410 [6,4]
+  CRUSH rule 0 x 411 [13,11]
+  CRUSH rule 0 x 412 [5,9]
+  CRUSH rule 0 x 413 [13,5]
+  CRUSH rule 0 x 414 [3,11]
+  CRUSH rule 0 x 415 [6,10]
+  CRUSH rule 0 x 416 [13,1]
+  CRUSH rule 0 x 417 [4,12]
+  CRUSH rule 0 x 418 [14,5]
+  CRUSH rule 0 x 419 [5,14]
+  CRUSH rule 0 x 420 [2,4]
+  CRUSH rule 0 x 421 [15,4]
+  CRUSH rule 0 x 422 [4,11]
+  CRUSH rule 0 x 423 [3,15]
+  CRUSH rule 0 x 424 [6,10]
+  CRUSH rule 0 x 425 [11,15]
+  CRUSH rule 0 x 426 [12,4]
+  CRUSH rule 0 x 427 [14,10]
+  CRUSH rule 0 x 428 [12,7]
+  CRUSH rule 0 x 429 [3,4]
+  CRUSH rule 0 x 430 [3,5]
+  CRUSH rule 0 x 431 [9,3]
+  CRUSH rule 0 x 432 [4,1]
+  CRUSH rule 0 x 433 [4,11]
+  CRUSH rule 0 x 434 [2,14]
+  CRUSH rule 0 x 435 [13,11]
+  CRUSH rule 0 x 436 [9,15]
+  CRUSH rule 0 x 437 [9,6]
+  CRUSH rule 0 x 438 [7,2]
+  CRUSH rule 0 x 439 [7,14]
+  CRUSH rule 0 x 440 [14,11]
+  CRUSH rule 0 x 441 [2,4]
+  CRUSH rule 0 x 442 [10,13]
+  CRUSH rule 0 x 443 [12,15]
+  CRUSH rule 0 x 444 [4,13]
+  CRUSH rule 0 x 445 [4,2]
+  CRUSH rule 0 x 446 [12,10]
+  CRUSH rule 0 x 447 [15,7]
+  CRUSH rule 0 x 448 [5,2]
+  CRUSH rule 0 x 449 [14,5]
+  CRUSH rule 0 x 450 [2,4]
+  CRUSH rule 0 x 451 [6,14]
+  CRUSH rule 0 x 452 [14,9]
+  CRUSH rule 0 x 453 [5,15]
+  CRUSH rule 0 x 454 [10,4]
+  CRUSH rule 0 x 455 [6,13]
+  CRUSH rule 0 x 456 [5,7]
+  CRUSH rule 0 x 457 [9,1]
+  CRUSH rule 0 x 458 [9,11]
+  CRUSH rule 0 x 459 [13,15]
+  CRUSH rule 0 x 460 [5,12]
+  CRUSH rule 0 x 461 [4,3]
+  CRUSH rule 0 x 462 [4,7]
+  CRUSH rule 0 x 463 [4,12]
+  CRUSH rule 0 x 464 [4,2]
+  CRUSH rule 0 x 465 [5,10]
+  CRUSH rule 0 x 466 [13,5]
+  CRUSH rule 0 x 467 [13,6]
+  CRUSH rule 0 x 468 [10,7]
+  CRUSH rule 0 x 469 [4,9]
+  CRUSH rule 0 x 470 [3,9]
+  CRUSH rule 0 x 471 [6,1]
+  CRUSH rule 0 x 472 [2,14]
+  CRUSH rule 0 x 473 [15,10]
+  CRUSH rule 0 x 474 [15,10]
+  CRUSH rule 0 x 475 [10,5]
+  CRUSH rule 0 x 476 [3,6]
+  CRUSH rule 0 x 477 [6,13]
+  CRUSH rule 0 x 478 [4,15]
+  CRUSH rule 0 x 479 [13,11]
+  CRUSH rule 0 x 480 [1,13]
+  CRUSH rule 0 x 481 [15,12]
+  CRUSH rule 0 x 482 [2,12]
+  CRUSH rule 0 x 483 [10,1]
+  CRUSH rule 0 x 484 [1,4]
+  CRUSH rule 0 x 485 [9,4]
+  CRUSH rule 0 x 486 [3,10]
+  CRUSH rule 0 x 487 [12,11]
+  CRUSH rule 0 x 488 [14,4]
+  CRUSH rule 0 x 489 [11,4]
+  CRUSH rule 0 x 490 [4,9]
+  CRUSH rule 0 x 491 [1,12]
+  CRUSH rule 0 x 492 [5,7]
+  CRUSH rule 0 x 493 [12,1]
+  CRUSH rule 0 x 494 [1,7]
+  CRUSH rule 0 x 495 [3,15]
+  CRUSH rule 0 x 496 [5,3]
+  CRUSH rule 0 x 497 [13,10]
+  CRUSH rule 0 x 498 [10,6]
+  CRUSH rule 0 x 499 [14,3]
+  CRUSH rule 0 x 500 [15,9]
+  CRUSH rule 0 x 501 [10,13]
+  CRUSH rule 0 x 502 [5,1]
+  CRUSH rule 0 x 503 [15,10]
+  CRUSH rule 0 x 504 [13,2]
+  CRUSH rule 0 x 505 [12,7]
+  CRUSH rule 0 x 506 [11,7]
+  CRUSH rule 0 x 507 [4,14]
+  CRUSH rule 0 x 508 [12,1]
+  CRUSH rule 0 x 509 [4,2]
+  CRUSH rule 0 x 510 [5,3]
+  CRUSH rule 0 x 511 [2,12]
+  CRUSH rule 0 x 512 [15,11]
+  CRUSH rule 0 x 513 [4,9]
+  CRUSH rule 0 x 514 [11,9]
+  CRUSH rule 0 x 515 [12,14]
+  CRUSH rule 0 x 516 [14,11]
+  CRUSH rule 0 x 517 [11,5]
+  CRUSH rule 0 x 518 [3,5]
+  CRUSH rule 0 x 519 [12,14]
+  CRUSH rule 0 x 520 [12,4]
+  CRUSH rule 0 x 521 [11,5]
+  CRUSH rule 0 x 522 [4,12]
+  CRUSH rule 0 x 523 [3,1]
+  CRUSH rule 0 x 524 [15,9]
+  CRUSH rule 0 x 525 [3,15]
+  CRUSH rule 0 x 526 [10,2]
+  CRUSH rule 0 x 527 [3,13]
+  CRUSH rule 0 x 528 [12,7]
+  CRUSH rule 0 x 529 [6,4]
+  CRUSH rule 0 x 530 [11,9]
+  CRUSH rule 0 x 531 [9,15]
+  CRUSH rule 0 x 532 [5,3]
+  CRUSH rule 0 x 533 [12,15]
+  CRUSH rule 0 x 534 [11,9]
+  CRUSH rule 0 x 535 [11,1]
+  CRUSH rule 0 x 536 [9,1]
+  CRUSH rule 0 x 537 [15,5]
+  CRUSH rule 0 x 538 [13,5]
+  CRUSH rule 0 x 539 [10,12]
+  CRUSH rule 0 x 540 [12,15]
+  CRUSH rule 0 x 541 [2,1]
+  CRUSH rule 0 x 542 [3,9]
+  CRUSH rule 0 x 543 [4,10]
+  CRUSH rule 0 x 544 [3,15]
+  CRUSH rule 0 x 545 [14,10]
+  CRUSH rule 0 x 546 [5,15]
+  CRUSH rule 0 x 547 [5,13]
+  CRUSH rule 0 x 548 [11,7]
+  CRUSH rule 0 x 549 [14,1]
+  CRUSH rule 0 x 550 [9,15]
+  CRUSH rule 0 x 551 [11,2]
+  CRUSH rule 0 x 552 [2,11]
+  CRUSH rule 0 x 553 [11,9]
+  CRUSH rule 0 x 554 [11,14]
+  CRUSH rule 0 x 555 [6,5]
+  CRUSH rule 0 x 556 [15,6]
+  CRUSH rule 0 x 557 [12,2]
+  CRUSH rule 0 x 558 [12,1]
+  CRUSH rule 0 x 559 [2,13]
+  CRUSH rule 0 x 560 [4,9]
+  CRUSH rule 0 x 561 [12,7]
+  CRUSH rule 0 x 562 [7,13]
+  CRUSH rule 0 x 563 [15,4]
+  CRUSH rule 0 x 564 [2,13]
+  CRUSH rule 0 x 565 [3,12]
+  CRUSH rule 0 x 566 [6,14]
+  CRUSH rule 0 x 567 [15,4]
+  CRUSH rule 0 x 568 [4,14]
+  CRUSH rule 0 x 569 [11,3]
+  CRUSH rule 0 x 570 [1,10]
+  CRUSH rule 0 x 571 [10,12]
+  CRUSH rule 0 x 572 [12,14]
+  CRUSH rule 0 x 573 [7,15]
+  CRUSH rule 0 x 574 [11,14]
+  CRUSH rule 0 x 575 [5,13]
+  CRUSH rule 0 x 576 [3,15]
+  CRUSH rule 0 x 577 [13,9]
+  CRUSH rule 0 x 578 [4,10]
+  CRUSH rule 0 x 579 [13,1]
+  CRUSH rule 0 x 580 [3,12]
+  CRUSH rule 0 x 581 [7,14]
+  CRUSH rule 0 x 582 [10,5]
+  CRUSH rule 0 x 583 [4,15]
+  CRUSH rule 0 x 584 [10,1]
+  CRUSH rule 0 x 585 [5,3]
+  CRUSH rule 0 x 586 [7,10]
+  CRUSH rule 0 x 587 [11,6]
+  CRUSH rule 0 x 588 [3,12]
+  CRUSH rule 0 x 589 [9,7]
+  CRUSH rule 0 x 590 [12,1]
+  CRUSH rule 0 x 591 [2,6]
+  CRUSH rule 0 x 592 [15,12]
+  CRUSH rule 0 x 593 [13,14]
+  CRUSH rule 0 x 594 [12,14]
+  CRUSH rule 0 x 595 [12,7]
+  CRUSH rule 0 x 596 [2,7]
+  CRUSH rule 0 x 597 [15,1]
+  CRUSH rule 0 x 598 [11,5]
+  CRUSH rule 0 x 599 [13,11]
+  CRUSH rule 0 x 600 [4,12]
+  CRUSH rule 0 x 601 [13,5]
+  CRUSH rule 0 x 602 [3,11]
+  CRUSH rule 0 x 603 [3,1]
+  CRUSH rule 0 x 604 [14,2]
+  CRUSH rule 0 x 605 [2,7]
+  CRUSH rule 0 x 606 [12,15]
+  CRUSH rule 0 x 607 [3,9]
+  CRUSH rule 0 x 608 [13,10]
+  CRUSH rule 0 x 609 [14,3]
+  CRUSH rule 0 x 610 [7,10]
+  CRUSH rule 0 x 611 [13,1]
+  CRUSH rule 0 x 612 [7,1]
+  CRUSH rule 0 x 613 [10,7]
+  CRUSH rule 0 x 614 [9,4]
+  CRUSH rule 0 x 615 [9,4]
+  CRUSH rule 0 x 616 [10,14]
+  CRUSH rule 0 x 617 [15,7]
+  CRUSH rule 0 x 618 [4,2]
+  CRUSH rule 0 x 619 [15,4]
+  CRUSH rule 0 x 620 [3,7]
+  CRUSH rule 0 x 621 [3,6]
+  CRUSH rule 0 x 622 [10,2]
+  CRUSH rule 0 x 623 [4,9]
+  CRUSH rule 0 x 624 [3,9]
+  CRUSH rule 0 x 625 [11,7]
+  CRUSH rule 0 x 626 [10,12]
+  CRUSH rule 0 x 627 [1,12]
+  CRUSH rule 0 x 628 [15,13]
+  CRUSH rule 0 x 629 [5,6]
+  CRUSH rule 0 x 630 [1,4]
+  CRUSH rule 0 x 631 [5,7]
+  CRUSH rule 0 x 632 [12,3]
+  CRUSH rule 0 x 633 [14,4]
+  CRUSH rule 0 x 634 [6,9]
+  CRUSH rule 0 x 635 [6,5]
+  CRUSH rule 0 x 636 [13,6]
+  CRUSH rule 0 x 637 [3,1]
+  CRUSH rule 0 x 638 [10,15]
+  CRUSH rule 0 x 639 [6,9]
+  CRUSH rule 0 x 640 [9,6]
+  CRUSH rule 0 x 641 [10,6]
+  CRUSH rule 0 x 642 [1,15]
+  CRUSH rule 0 x 643 [3,7]
+  CRUSH rule 0 x 644 [15,13]
+  CRUSH rule 0 x 645 [14,2]
+  CRUSH rule 0 x 646 [5,13]
+  CRUSH rule 0 x 647 [10,1]
+  CRUSH rule 0 x 648 [6,5]
+  CRUSH rule 0 x 649 [3,9]
+  CRUSH rule 0 x 650 [10,9]
+  CRUSH rule 0 x 651 [3,9]
+  CRUSH rule 0 x 652 [15,9]
+  CRUSH rule 0 x 653 [11,14]
+  CRUSH rule 0 x 654 [13,6]
+  CRUSH rule 0 x 655 [6,3]
+  CRUSH rule 0 x 656 [3,15]
+  CRUSH rule 0 x 657 [11,15]
+  CRUSH rule 0 x 658 [7,2]
+  CRUSH rule 0 x 659 [2,5]
+  CRUSH rule 0 x 660 [13,14]
+  CRUSH rule 0 x 661 [7,15]
+  CRUSH rule 0 x 662 [15,2]
+  CRUSH rule 0 x 663 [14,9]
+  CRUSH rule 0 x 664 [6,10]
+  CRUSH rule 0 x 665 [2,9]
+  CRUSH rule 0 x 666 [12,3]
+  CRUSH rule 0 x 667 [1,9]
+  CRUSH rule 0 x 668 [9,5]
+  CRUSH rule 0 x 669 [9,7]
+  CRUSH rule 0 x 670 [6,10]
+  CRUSH rule 0 x 671 [6,15]
+  CRUSH rule 0 x 672 [2,9]
+  CRUSH rule 0 x 673 [7,10]
+  CRUSH rule 0 x 674 [7,12]
+  CRUSH rule 0 x 675 [9,5]
+  CRUSH rule 0 x 676 [10,12]
+  CRUSH rule 0 x 677 [2,12]
+  CRUSH rule 0 x 678 [1,2]
+  CRUSH rule 0 x 679 [5,6]
+  CRUSH rule 0 x 680 [7,11]
+  CRUSH rule 0 x 681 [6,4]
+  CRUSH rule 0 x 682 [6,1]
+  CRUSH rule 0 x 683 [6,13]
+  CRUSH rule 0 x 684 [9,11]
+  CRUSH rule 0 x 685 [5,1]
+  CRUSH rule 0 x 686 [1,9]
+  CRUSH rule 0 x 687 [7,13]
+  CRUSH rule 0 x 688 [11,9]
+  CRUSH rule 0 x 689 [5,2]
+  CRUSH rule 0 x 690 [9,7]
+  CRUSH rule 0 x 691 [11,15]
+  CRUSH rule 0 x 692 [15,5]
+  CRUSH rule 0 x 693 [5,6]
+  CRUSH rule 0 x 694 [4,7]
+  CRUSH rule 0 x 695 [6,13]
+  CRUSH rule 0 x 696 [1,2]
+  CRUSH rule 0 x 697 [13,11]
+  CRUSH rule 0 x 698 [11,13]
+  CRUSH rule 0 x 699 [7,14]
+  CRUSH rule 0 x 700 [12,14]
+  CRUSH rule 0 x 701 [3,13]
+  CRUSH rule 0 x 702 [3,12]
+  CRUSH rule 0 x 703 [15,11]
+  CRUSH rule 0 x 704 [6,4]
+  CRUSH rule 0 x 705 [14,6]
+  CRUSH rule 0 x 706 [1,12]
+  CRUSH rule 0 x 707 [4,7]
+  CRUSH rule 0 x 708 [3,10]
+  CRUSH rule 0 x 709 [11,12]
+  CRUSH rule 0 x 710 [14,2]
+  CRUSH rule 0 x 711 [14,3]
+  CRUSH rule 0 x 712 [12,3]
+  CRUSH rule 0 x 713 [11,9]
+  CRUSH rule 0 x 714 [12,1]
+  CRUSH rule 0 x 715 [6,1]
+  CRUSH rule 0 x 716 [11,13]
+  CRUSH rule 0 x 717 [12,4]
+  CRUSH rule 0 x 718 [7,15]
+  CRUSH rule 0 x 719 [5,15]
+  CRUSH rule 0 x 720 [4,13]
+  CRUSH rule 0 x 721 [11,3]
+  CRUSH rule 0 x 722 [2,4]
+  CRUSH rule 0 x 723 [2,1]
+  CRUSH rule 0 x 724 [7,1]
+  CRUSH rule 0 x 725 [11,12]
+  CRUSH rule 0 x 726 [7,14]
+  CRUSH rule 0 x 727 [2,5]
+  CRUSH rule 0 x 728 [13,11]
+  CRUSH rule 0 x 729 [15,11]
+  CRUSH rule 0 x 730 [3,7]
+  CRUSH rule 0 x 731 [9,1]
+  CRUSH rule 0 x 732 [1,2]
+  CRUSH rule 0 x 733 [11,3]
+  CRUSH rule 0 x 734 [14,3]
+  CRUSH rule 0 x 735 [6,9]
+  CRUSH rule 0 x 736 [3,9]
+  CRUSH rule 0 x 737 [1,4]
+  CRUSH rule 0 x 738 [11,15]
+  CRUSH rule 0 x 739 [11,12]
+  CRUSH rule 0 x 740 [7,9]
+  CRUSH rule 0 x 741 [12,11]
+  CRUSH rule 0 x 742 [9,7]
+  CRUSH rule 0 x 743 [5,13]
+  CRUSH rule 0 x 744 [6,2]
+  CRUSH rule 0 x 745 [3,6]
+  CRUSH rule 0 x 746 [3,7]
+  CRUSH rule 0 x 747 [15,11]
+  CRUSH rule 0 x 748 [6,10]
+  CRUSH rule 0 x 749 [14,9]
+  CRUSH rule 0 x 750 [1,14]
+  CRUSH rule 0 x 751 [15,1]
+  CRUSH rule 0 x 752 [13,1]
+  CRUSH rule 0 x 753 [4,11]
+  CRUSH rule 0 x 754 [14,12]
+  CRUSH rule 0 x 755 [13,6]
+  CRUSH rule 0 x 756 [3,4]
+  CRUSH rule 0 x 757 [10,6]
+  CRUSH rule 0 x 758 [6,3]
+  CRUSH rule 0 x 759 [5,7]
+  CRUSH rule 0 x 760 [1,15]
+  CRUSH rule 0 x 761 [2,12]
+  CRUSH rule 0 x 762 [1,4]
+  CRUSH rule 0 x 763 [4,13]
+  CRUSH rule 0 x 764 [1,14]
+  CRUSH rule 0 x 765 [9,15]
+  CRUSH rule 0 x 766 [11,2]
+  CRUSH rule 0 x 767 [6,11]
+  CRUSH rule 0 x 768 [2,12]
+  CRUSH rule 0 x 769 [15,1]
+  CRUSH rule 0 x 770 [15,13]
+  CRUSH rule 0 x 771 [9,2]
+  CRUSH rule 0 x 772 [4,3]
+  CRUSH rule 0 x 773 [3,7]
+  CRUSH rule 0 x 774 [12,6]
+  CRUSH rule 0 x 775 [5,10]
+  CRUSH rule 0 x 776 [10,15]
+  CRUSH rule 0 x 777 [11,13]
+  CRUSH rule 0 x 778 [13,1]
+  CRUSH rule 0 x 779 [5,11]
+  CRUSH rule 0 x 780 [13,9]
+  CRUSH rule 0 x 781 [5,7]
+  CRUSH rule 0 x 782 [2,15]
+  CRUSH rule 0 x 783 [12,7]
+  CRUSH rule 0 x 784 [14,1]
+  CRUSH rule 0 x 785 [6,12]
+  CRUSH rule 0 x 786 [10,5]
+  CRUSH rule 0 x 787 [1,12]
+  CRUSH rule 0 x 788 [4,2]
+  CRUSH rule 0 x 789 [9,2]
+  CRUSH rule 0 x 790 [15,2]
+  CRUSH rule 0 x 791 [9,4]
+  CRUSH rule 0 x 792 [6,4]
+  CRUSH rule 0 x 793 [15,9]
+  CRUSH rule 0 x 794 [5,12]
+  CRUSH rule 0 x 795 [6,14]
+  CRUSH rule 0 x 796 [11,2]
+  CRUSH rule 0 x 797 [14,3]
+  CRUSH rule 0 x 798 [5,11]
+  CRUSH rule 0 x 799 [2,9]
+  CRUSH rule 0 x 800 [6,3]
+  CRUSH rule 0 x 801 [2,5]
+  CRUSH rule 0 x 802 [1,4]
+  CRUSH rule 0 x 803 [7,2]
+  CRUSH rule 0 x 804 [5,14]
+  CRUSH rule 0 x 805 [13,4]
+  CRUSH rule 0 x 806 [6,2]
+  CRUSH rule 0 x 807 [14,2]
+  CRUSH rule 0 x 808 [2,15]
+  CRUSH rule 0 x 809 [1,11]
+  CRUSH rule 0 x 810 [2,5]
+  CRUSH rule 0 x 811 [15,6]
+  CRUSH rule 0 x 812 [7,11]
+  CRUSH rule 0 x 813 [4,10]
+  CRUSH rule 0 x 814 [13,4]
+  CRUSH rule 0 x 815 [15,12]
+  CRUSH rule 0 x 816 [14,10]
+  CRUSH rule 0 x 817 [10,7]
+  CRUSH rule 0 x 818 [15,2]
+  CRUSH rule 0 x 819 [5,12]
+  CRUSH rule 0 x 820 [3,6]
+  CRUSH rule 0 x 821 [15,10]
+  CRUSH rule 0 x 822 [10,13]
+  CRUSH rule 0 x 823 [2,6]
+  CRUSH rule 0 x 824 [3,7]
+  CRUSH rule 0 x 825 [10,5]
+  CRUSH rule 0 x 826 [5,2]
+  CRUSH rule 0 x 827 [13,5]
+  CRUSH rule 0 x 828 [12,6]
+  CRUSH rule 0 x 829 [13,6]
+  CRUSH rule 0 x 830 [15,13]
+  CRUSH rule 0 x 831 [1,4]
+  CRUSH rule 0 x 832 [14,11]
+  CRUSH rule 0 x 833 [9,13]
+  CRUSH rule 0 x 834 [9,7]
+  CRUSH rule 0 x 835 [14,3]
+  CRUSH rule 0 x 836 [3,9]
+  CRUSH rule 0 x 837 [15,12]
+  CRUSH rule 0 x 838 [12,14]
+  CRUSH rule 0 x 839 [3,4]
+  CRUSH rule 0 x 840 [10,15]
+  CRUSH rule 0 x 841 [3,5]
+  CRUSH rule 0 x 842 [9,13]
+  CRUSH rule 0 x 843 [14,7]
+  CRUSH rule 0 x 844 [7,1]
+  CRUSH rule 0 x 845 [13,6]
+  CRUSH rule 0 x 846 [3,7]
+  CRUSH rule 0 x 847 [12,15]
+  CRUSH rule 0 x 848 [11,13]
+  CRUSH rule 0 x 849 [3,15]
+  CRUSH rule 0 x 850 [1,3]
+  CRUSH rule 0 x 851 [14,4]
+  CRUSH rule 0 x 852 [9,12]
+  CRUSH rule 0 x 853 [13,14]
+  CRUSH rule 0 x 854 [7,11]
+  CRUSH rule 0 x 855 [14,4]
+  CRUSH rule 0 x 856 [5,10]
+  CRUSH rule 0 x 857 [4,3]
+  CRUSH rule 0 x 858 [5,15]
+  CRUSH rule 0 x 859 [5,15]
+  CRUSH rule 0 x 860 [11,14]
+  CRUSH rule 0 x 861 [13,7]
+  CRUSH rule 0 x 862 [5,10]
+  CRUSH rule 0 x 863 [11,6]
+  CRUSH rule 0 x 864 [6,13]
+  CRUSH rule 0 x 865 [4,1]
+  CRUSH rule 0 x 866 [2,13]
+  CRUSH rule 0 x 867 [12,2]
+  CRUSH rule 0 x 868 [14,11]
+  CRUSH rule 0 x 869 [10,13]
+  CRUSH rule 0 x 870 [14,9]
+  CRUSH rule 0 x 871 [6,2]
+  CRUSH rule 0 x 872 [6,1]
+  CRUSH rule 0 x 873 [2,5]
+  CRUSH rule 0 x 874 [12,4]
+  CRUSH rule 0 x 875 [10,6]
+  CRUSH rule 0 x 876 [14,7]
+  CRUSH rule 0 x 877 [15,11]
+  CRUSH rule 0 x 878 [7,14]
+  CRUSH rule 0 x 879 [12,2]
+  CRUSH rule 0 x 880 [2,12]
+  CRUSH rule 0 x 881 [6,3]
+  CRUSH rule 0 x 882 [11,13]
+  CRUSH rule 0 x 883 [13,1]
+  CRUSH rule 0 x 884 [6,15]
+  CRUSH rule 0 x 885 [14,7]
+  CRUSH rule 0 x 886 [13,11]
+  CRUSH rule 0 x 887 [14,4]
+  CRUSH rule 0 x 888 [10,12]
+  CRUSH rule 0 x 889 [15,13]
+  CRUSH rule 0 x 890 [10,12]
+  CRUSH rule 0 x 891 [9,5]
+  CRUSH rule 0 x 892 [12,15]
+  CRUSH rule 0 x 893 [1,3]
+  CRUSH rule 0 x 894 [7,2]
+  CRUSH rule 0 x 895 [2,1]
+  CRUSH rule 0 x 896 [9,1]
+  CRUSH rule 0 x 897 [7,5]
+  CRUSH rule 0 x 898 [10,6]
+  CRUSH rule 0 x 899 [1,11]
+  CRUSH rule 0 x 900 [2,9]
+  CRUSH rule 0 x 901 [9,12]
+  CRUSH rule 0 x 902 [4,2]
+  CRUSH rule 0 x 903 [14,10]
+  CRUSH rule 0 x 904 [15,12]
+  CRUSH rule 0 x 905 [12,6]
+  CRUSH rule 0 x 906 [14,11]
+  CRUSH rule 0 x 907 [7,12]
+  CRUSH rule 0 x 908 [2,15]
+  CRUSH rule 0 x 909 [10,14]
+  CRUSH rule 0 x 910 [12,7]
+  CRUSH rule 0 x 911 [11,15]
+  CRUSH rule 0 x 912 [6,4]
+  CRUSH rule 0 x 913 [4,6]
+  CRUSH rule 0 x 914 [4,15]
+  CRUSH rule 0 x 915 [12,14]
+  CRUSH rule 0 x 916 [3,1]
+  CRUSH rule 0 x 917 [1,15]
+  CRUSH rule 0 x 918 [7,14]
+  CRUSH rule 0 x 919 [10,7]
+  CRUSH rule 0 x 920 [4,2]
+  CRUSH rule 0 x 921 [1,11]
+  CRUSH rule 0 x 922 [6,4]
+  CRUSH rule 0 x 923 [12,2]
+  CRUSH rule 0 x 924 [6,2]
+  CRUSH rule 0 x 925 [12,15]
+  CRUSH rule 0 x 926 [3,13]
+  CRUSH rule 0 x 927 [6,5]
+  CRUSH rule 0 x 928 [13,1]
+  CRUSH rule 0 x 929 [10,7]
+  CRUSH rule 0 x 930 [7,15]
+  CRUSH rule 0 x 931 [6,15]
+  CRUSH rule 0 x 932 [13,2]
+  CRUSH rule 0 x 933 [12,7]
+  CRUSH rule 0 x 934 [12,2]
+  CRUSH rule 0 x 935 [6,11]
+  CRUSH rule 0 x 936 [9,12]
+  CRUSH rule 0 x 937 [14,2]
+  CRUSH rule 0 x 938 [14,3]
+  CRUSH rule 0 x 939 [6,4]
+  CRUSH rule 0 x 940 [13,11]
+  CRUSH rule 0 x 941 [3,12]
+  CRUSH rule 0 x 942 [15,12]
+  CRUSH rule 0 x 943 [10,2]
+  CRUSH rule 0 x 944 [2,9]
+  CRUSH rule 0 x 945 [10,15]
+  CRUSH rule 0 x 946 [11,15]
+  CRUSH rule 0 x 947 [11,3]
+  CRUSH rule 0 x 948 [7,13]
+  CRUSH rule 0 x 949 [9,1]
+  CRUSH rule 0 x 950 [9,15]
+  CRUSH rule 0 x 951 [2,6]
+  CRUSH rule 0 x 952 [9,7]
+  CRUSH rule 0 x 953 [1,3]
+  CRUSH rule 0 x 954 [10,2]
+  CRUSH rule 0 x 955 [7,14]
+  CRUSH rule 0 x 956 [1,6]
+  CRUSH rule 0 x 957 [14,11]
+  CRUSH rule 0 x 958 [15,4]
+  CRUSH rule 0 x 959 [2,1]
+  CRUSH rule 0 x 960 [2,6]
+  CRUSH rule 0 x 961 [3,13]
+  CRUSH rule 0 x 962 [5,11]
+  CRUSH rule 0 x 963 [13,10]
+  CRUSH rule 0 x 964 [7,11]
+  CRUSH rule 0 x 965 [12,2]
+  CRUSH rule 0 x 966 [12,14]
+  CRUSH rule 0 x 967 [7,5]
+  CRUSH rule 0 x 968 [12,15]
+  CRUSH rule 0 x 969 [11,4]
+  CRUSH rule 0 x 970 [5,12]
+  CRUSH rule 0 x 971 [1,9]
+  CRUSH rule 0 x 972 [12,3]
+  CRUSH rule 0 x 973 [1,10]
+  CRUSH rule 0 x 974 [7,11]
+  CRUSH rule 0 x 975 [7,9]
+  CRUSH rule 0 x 976 [7,3]
+  CRUSH rule 0 x 977 [14,3]
+  CRUSH rule 0 x 978 [12,5]
+  CRUSH rule 0 x 979 [5,1]
+  CRUSH rule 0 x 980 [15,11]
+  CRUSH rule 0 x 981 [5,11]
+  CRUSH rule 0 x 982 [2,6]
+  CRUSH rule 0 x 983 [3,12]
+  CRUSH rule 0 x 984 [15,13]
+  CRUSH rule 0 x 985 [11,2]
+  CRUSH rule 0 x 986 [6,13]
+  CRUSH rule 0 x 987 [13,14]
+  CRUSH rule 0 x 988 [12,9]
+  CRUSH rule 0 x 989 [7,4]
+  CRUSH rule 0 x 990 [1,10]
+  CRUSH rule 0 x 991 [7,11]
+  CRUSH rule 0 x 992 [9,10]
+  CRUSH rule 0 x 993 [6,10]
+  CRUSH rule 0 x 994 [3,13]
+  CRUSH rule 0 x 995 [15,6]
+  CRUSH rule 0 x 996 [15,10]
+  CRUSH rule 0 x 997 [15,2]
+  CRUSH rule 0 x 998 [6,1]
+  CRUSH rule 0 x 999 [9,10]
+  CRUSH rule 0 x 1000 [14,2]
+  CRUSH rule 0 x 1001 [11,14]
+  CRUSH rule 0 x 1002 [1,10]
+  CRUSH rule 0 x 1003 [10,7]
+  CRUSH rule 0 x 1004 [15,1]
+  CRUSH rule 0 x 1005 [6,12]
+  CRUSH rule 0 x 1006 [10,12]
+  CRUSH rule 0 x 1007 [1,7]
+  CRUSH rule 0 x 1008 [7,4]
+  CRUSH rule 0 x 1009 [5,2]
+  CRUSH rule 0 x 1010 [10,2]
+  CRUSH rule 0 x 1011 [6,3]
+  CRUSH rule 0 x 1012 [12,6]
+  CRUSH rule 0 x 1013 [2,14]
+  CRUSH rule 0 x 1014 [1,13]
+  CRUSH rule 0 x 1015 [12,6]
+  CRUSH rule 0 x 1016 [10,13]
+  CRUSH rule 0 x 1017 [5,11]
+  CRUSH rule 0 x 1018 [13,11]
+  CRUSH rule 0 x 1019 [10,13]
+  CRUSH rule 0 x 1020 [3,1]
+  CRUSH rule 0 x 1021 [2,11]
+  CRUSH rule 0 x 1022 [15,5]
+  CRUSH rule 0 x 1023 [15,2]
+  rule 0 (replicated_ruleset) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3]
+  CRUSH rule 0 x 1 [10,15,1]
+  CRUSH rule 0 x 2 [1,12,2]
+  CRUSH rule 0 x 3 [15,4,10]
+  CRUSH rule 0 x 4 [14,2,10]
+  CRUSH rule 0 x 5 [7,4,11]
+  CRUSH rule 0 x 6 [12,6,10]
+  CRUSH rule 0 x 7 [9,2,6]
+  CRUSH rule 0 x 8 [10,2,15]
+  CRUSH rule 0 x 9 [7,1,14]
+  CRUSH rule 0 x 10 [10,14,4]
+  CRUSH rule 0 x 11 [13,9,14]
+  CRUSH rule 0 x 12 [7,1,2]
+  CRUSH rule 0 x 13 [3,5,12]
+  CRUSH rule 0 x 14 [13,5,2]
+  CRUSH rule 0 x 15 [15,1,9]
+  CRUSH rule 0 x 16 [7,11,14]
+  CRUSH rule 0 x 17 [10,1,13]
+  CRUSH rule 0 x 18 [1,7,3]
+  CRUSH rule 0 x 19 [7,12,2]
+  CRUSH rule 0 x 20 [14,12,3]
+  CRUSH rule 0 x 21 [3,12,1]
+  CRUSH rule 0 x 22 [6,3,13]
+  CRUSH rule 0 x 23 [10,5,13]
+  CRUSH rule 0 x 24 [12,11,3]
+  CRUSH rule 0 x 25 [7,12,15]
+  CRUSH rule 0 x 26 [1,7,13]
+  CRUSH rule 0 x 27 [3,6,15]
+  CRUSH rule 0 x 28 [14,4,3]
+  CRUSH rule 0 x 29 [5,14,12]
+  CRUSH rule 0 x 30 [2,5,6]
+  CRUSH rule 0 x 31 [5,15,10]
+  CRUSH rule 0 x 32 [9,10,2]
+  CRUSH rule 0 x 33 [13,4,9]
+  CRUSH rule 0 x 34 [13,15,2]
+  CRUSH rule 0 x 35 [4,14,3]
+  CRUSH rule 0 x 36 [3,12,9]
+  CRUSH rule 0 x 37 [9,2,6]
+  CRUSH rule 0 x 38 [3,4,13]
+  CRUSH rule 0 x 39 [12,7,14]
+  CRUSH rule 0 x 40 [10,1,9]
+  CRUSH rule 0 x 41 [4,9,11]
+  CRUSH rule 0 x 42 [3,6,14]
+  CRUSH rule 0 x 43 [10,5,15]
+  CRUSH rule 0 x 44 [11,4,13]
+  CRUSH rule 0 x 45 [11,12,15]
+  CRUSH rule 0 x 46 [6,9,2]
+  CRUSH rule 0 x 47 [3,9,6]
+  CRUSH rule 0 x 48 [4,6,2]
+  CRUSH rule 0 x 49 [9,15,10]
+  CRUSH rule 0 x 50 [14,12,1]
+  CRUSH rule 0 x 51 [10,6,5]
+  CRUSH rule 0 x 52 [12,1,9]
+  CRUSH rule 0 x 53 [3,6,13]
+  CRUSH rule 0 x 54 [4,13,9]
+  CRUSH rule 0 x 55 [4,11,2]
+  CRUSH rule 0 x 56 [5,9,10]
+  CRUSH rule 0 x 57 [6,2,1]
+  CRUSH rule 0 x 58 [7,1,11]
+  CRUSH rule 0 x 59 [2,13,1]
+  CRUSH rule 0 x 60 [3,6,11]
+  CRUSH rule 0 x 61 [3,15,13]
+  CRUSH rule 0 x 62 [15,11,7]
+  CRUSH rule 0 x 63 [10,14,12]
+  CRUSH rule 0 x 64 [3,9,1]
+  CRUSH rule 0 x 65 [4,12,11]
+  CRUSH rule 0 x 66 [15,11,6]
+  CRUSH rule 0 x 67 [2,6,4]
+  CRUSH rule 0 x 68 [15,7,4]
+  CRUSH rule 0 x 69 [2,1,15]
+  CRUSH rule 0 x 70 [9,6,1]
+  CRUSH rule 0 x 71 [15,5,1]
+  CRUSH rule 0 x 72 [9,10,3]
+  CRUSH rule 0 x 73 [5,3,11]
+  CRUSH rule 0 x 74 [11,7,9]
+  CRUSH rule 0 x 75 [9,7,11]
+  CRUSH rule 0 x 76 [6,1,3]
+  CRUSH rule 0 x 77 [7,4,2]
+  CRUSH rule 0 x 78 [9,3,1]
+  CRUSH rule 0 x 79 [13,2,15]
+  CRUSH rule 0 x 80 [15,2,6]
+  CRUSH rule 0 x 81 [15,2,1]
+  CRUSH rule 0 x 82 [14,13,5]
+  CRUSH rule 0 x 83 [4,15,3]
+  CRUSH rule 0 x 84 [10,7,9]
+  CRUSH rule 0 x 85 [3,15,9]
+  CRUSH rule 0 x 86 [10,9,14]
+  CRUSH rule 0 x 87 [15,10,7]
+  CRUSH rule 0 x 88 [4,13,3]
+  CRUSH rule 0 x 89 [3,9,7]
+  CRUSH rule 0 x 90 [4,9,7]
+  CRUSH rule 0 x 91 [6,11,9]
+  CRUSH rule 0 x 92 [1,5,10]
+  CRUSH rule 0 x 93 [9,3,15]
+  CRUSH rule 0 x 94 [9,2,12]
+  CRUSH rule 0 x 95 [7,15,4]
+  CRUSH rule 0 x 96 [2,15,11]
+  CRUSH rule 0 x 97 [4,11,2]
+  CRUSH rule 0 x 98 [11,13,9]
+  CRUSH rule 0 x 99 [12,4,11]
+  CRUSH rule 0 x 100 [9,4,10]
+  CRUSH rule 0 x 101 [15,7,1]
+  CRUSH rule 0 x 102 [3,11,14]
+  CRUSH rule 0 x 103 [13,11,6]
+  CRUSH rule 0 x 104 [14,6,3]
+  CRUSH rule 0 x 105 [14,10,1]
+  CRUSH rule 0 x 106 [6,5,13]
+  CRUSH rule 0 x 107 [3,1,10]
+  CRUSH rule 0 x 108 [5,10,7]
+  CRUSH rule 0 x 109 [9,1,13]
+  CRUSH rule 0 x 110 [5,1,11]
+  CRUSH rule 0 x 111 [10,1,9]
+  CRUSH rule 0 x 112 [1,10,4]
+  CRUSH rule 0 x 113 [6,10,13]
+  CRUSH rule 0 x 114 [5,13,6]
+  CRUSH rule 0 x 115 [10,13,14]
+  CRUSH rule 0 x 116 [1,14,13]
+  CRUSH rule 0 x 117 [5,6,1]
+  CRUSH rule 0 x 118 [10,4,13]
+  CRUSH rule 0 x 119 [14,12,11]
+  CRUSH rule 0 x 120 [11,3,14]
+  CRUSH rule 0 x 121 [9,5,1]
+  CRUSH rule 0 x 122 [4,3,14]
+  CRUSH rule 0 x 123 [3,10,5]
+  CRUSH rule 0 x 124 [12,2,1]
+  CRUSH rule 0 x 125 [9,12,15]
+  CRUSH rule 0 x 126 [7,15,10]
+  CRUSH rule 0 x 127 [4,14,9]
+  CRUSH rule 0 x 128 [3,12,1]
+  CRUSH rule 0 x 129 [11,13,14]
+  CRUSH rule 0 x 130 [3,13,5]
+  CRUSH rule 0 x 131 [12,1,6]
+  CRUSH rule 0 x 132 [11,15,13]
+  CRUSH rule 0 x 133 [3,6,9]
+  CRUSH rule 0 x 134 [12,5,6]
+  CRUSH rule 0 x 135 [3,14,12]
+  CRUSH rule 0 x 136 [15,6,9]
+  CRUSH rule 0 x 137 [14,3,6]
+  CRUSH rule 0 x 138 [13,15,4]
+  CRUSH rule 0 x 139 [11,2,13]
+  CRUSH rule 0 x 140 [11,4,12]
+  CRUSH rule 0 x 141 [6,12,15]
+  CRUSH rule 0 x 142 [3,14,7]
+  CRUSH rule 0 x 143 [9,6,4]
+  CRUSH rule 0 x 144 [13,7,11]
+  CRUSH rule 0 x 145 [12,2,6]
+  CRUSH rule 0 x 146 [1,5,9]
+  CRUSH rule 0 x 147 [1,4,9]
+  CRUSH rule 0 x 148 [12,7,9]
+  CRUSH rule 0 x 149 [2,5,9]
+  CRUSH rule 0 x 150 [1,15,2]
+  CRUSH rule 0 x 151 [2,9,14]
+  CRUSH rule 0 x 152 [5,9,2]
+  CRUSH rule 0 x 153 [6,9,4]
+  CRUSH rule 0 x 154 [3,11,7]
+  CRUSH rule 0 x 155 [14,12,7]
+  CRUSH rule 0 x 156 [7,13,3]
+  CRUSH rule 0 x 157 [15,1,6]
+  CRUSH rule 0 x 158 [15,1,10]
+  CRUSH rule 0 x 159 [4,14,3]
+  CRUSH rule 0 x 160 [5,7,3]
+  CRUSH rule 0 x 161 [1,2,11]
+  CRUSH rule 0 x 162 [10,6,1]
+  CRUSH rule 0 x 163 [15,1,10]
+  CRUSH rule 0 x 164 [9,14,10]
+  CRUSH rule 0 x 165 [11,7,2]
+  CRUSH rule 0 x 166 [1,2,12]
+  CRUSH rule 0 x 167 [9,7,3]
+  CRUSH rule 0 x 168 [13,2,4]
+  CRUSH rule 0 x 169 [1,4,9]
+  CRUSH rule 0 x 170 [1,15,7]
+  CRUSH rule 0 x 171 [9,2,10]
+  CRUSH rule 0 x 172 [14,4,10]
+  CRUSH rule 0 x 173 [5,10,12]
+  CRUSH rule 0 x 174 [15,6,4]
+  CRUSH rule 0 x 175 [5,7,9]
+  CRUSH rule 0 x 176 [9,6,3]
+  CRUSH rule 0 x 177 [2,9,10]
+  CRUSH rule 0 x 178 [12,11,7]
+  CRUSH rule 0 x 179 [2,10,13]
+  CRUSH rule 0 x 180 [3,11,5]
+  CRUSH rule 0 x 181 [9,12,6]
+  CRUSH rule 0 x 182 [5,13,11]
+  CRUSH rule 0 x 183 [5,7,10]
+  CRUSH rule 0 x 184 [2,5,11]
+  CRUSH rule 0 x 185 [13,5,7]
+  CRUSH rule 0 x 186 [6,14,13]
+  CRUSH rule 0 x 187 [1,4,11]
+  CRUSH rule 0 x 188 [9,13,5]
+  CRUSH rule 0 x 189 [6,12,4]
+  CRUSH rule 0 x 190 [9,13,15]
+  CRUSH rule 0 x 191 [7,11,4]
+  CRUSH rule 0 x 192 [2,11,5]
+  CRUSH rule 0 x 193 [3,13,6]
+  CRUSH rule 0 x 194 [3,13,4]
+  CRUSH rule 0 x 195 [5,7,10]
+  CRUSH rule 0 x 196 [4,15,1]
+  CRUSH rule 0 x 197 [14,10,13]
+  CRUSH rule 0 x 198 [2,5,6]
+  CRUSH rule 0 x 199 [2,10,4]
+  CRUSH rule 0 x 200 [7,14,11]
+  CRUSH rule 0 x 201 [9,14,1]
+  CRUSH rule 0 x 202 [14,11,7]
+  CRUSH rule 0 x 203 [12,5,7]
+  CRUSH rule 0 x 204 [6,11,3]
+  CRUSH rule 0 x 205 [15,4,6]
+  CRUSH rule 0 x 206 [13,11,2]
+  CRUSH rule 0 x 207 [2,11,7]
+  CRUSH rule 0 x 208 [13,1,6]
+  CRUSH rule 0 x 209 [6,15,13]
+  CRUSH rule 0 x 210 [13,11,2]
+  CRUSH rule 0 x 211 [2,14,1]
+  CRUSH rule 0 x 212 [10,1,12]
+  CRUSH rule 0 x 213 [3,9,6]
+  CRUSH rule 0 x 214 [7,15,4]
+  CRUSH rule 0 x 215 [6,1,4]
+  CRUSH rule 0 x 216 [12,9,6]
+  CRUSH rule 0 x 217 [12,11,1]
+  CRUSH rule 0 x 218 [12,10,15]
+  CRUSH rule 0 x 219 [3,11,14]
+  CRUSH rule 0 x 220 [14,4,3]
+  CRUSH rule 0 x 221 [15,5,2]
+  CRUSH rule 0 x 222 [10,4,3]
+  CRUSH rule 0 x 223 [9,7,11]
+  CRUSH rule 0 x 224 [1,7,10]
+  CRUSH rule 0 x 225 [10,5,2]
+  CRUSH rule 0 x 226 [4,1,9]
+  CRUSH rule 0 x 227 [7,2,12]
+  CRUSH rule 0 x 228 [2,15,11]
+  CRUSH rule 0 x 229 [9,3,7]
+  CRUSH rule 0 x 230 [10,5,7]
+  CRUSH rule 0 x 231 [2,7,5]
+  CRUSH rule 0 x 232 [10,5,13]
+  CRUSH rule 0 x 233 [6,12,11]
+  CRUSH rule 0 x 234 [10,1,2]
+  CRUSH rule 0 x 235 [13,14,7]
+  CRUSH rule 0 x 236 [2,15,9]
+  CRUSH rule 0 x 237 [3,12,9]
+  CRUSH rule 0 x 238 [2,10,4]
+  CRUSH rule 0 x 239 [4,15,10]
+  CRUSH rule 0 x 240 [15,5,13]
+  CRUSH rule 0 x 241 [7,9,15]
+  CRUSH rule 0 x 242 [14,2,6]
+  CRUSH rule 0 x 243 [2,11,5]
+  CRUSH rule 0 x 244 [13,9,15]
+  CRUSH rule 0 x 245 [12,9,15]
+  CRUSH rule 0 x 246 [15,3,5]
+  CRUSH rule 0 x 247 [6,4,9]
+  CRUSH rule 0 x 248 [5,13,7]
+  CRUSH rule 0 x 249 [10,14,7]
+  CRUSH rule 0 x 250 [12,15,1]
+  CRUSH rule 0 x 251 [13,2,15]
+  CRUSH rule 0 x 252 [7,5,13]
+  CRUSH rule 0 x 253 [3,13,15]
+  CRUSH rule 0 x 254 [2,9,13]
+  CRUSH rule 0 x 255 [1,9,13]
+  CRUSH rule 0 x 256 [6,9,13]
+  CRUSH rule 0 x 257 [15,12,3]
+  CRUSH rule 0 x 258 [12,5,6]
+  CRUSH rule 0 x 259 [9,10,4]
+  CRUSH rule 0 x 260 [10,12,6]
+  CRUSH rule 0 x 261 [13,7,2]
+  CRUSH rule 0 x 262 [15,3,12]
+  CRUSH rule 0 x 263 [12,6,10]
+  CRUSH rule 0 x 264 [13,14,11]
+  CRUSH rule 0 x 265 [12,10,14]
+  CRUSH rule 0 x 266 [14,7,11]
+  CRUSH rule 0 x 267 [12,11,6]
+  CRUSH rule 0 x 268 [4,1,15]
+  CRUSH rule 0 x 269 [11,1,15]
+  CRUSH rule 0 x 270 [7,11,12]
+  CRUSH rule 0 x 271 [4,7,3]
+  CRUSH rule 0 x 272 [15,5,13]
+  CRUSH rule 0 x 273 [2,10,7]
+  CRUSH rule 0 x 274 [10,2,5]
+  CRUSH rule 0 x 275 [10,3,4]
+  CRUSH rule 0 x 276 [5,12,9]
+  CRUSH rule 0 x 277 [14,3,13]
+  CRUSH rule 0 x 278 [5,6,14]
+  CRUSH rule 0 x 279 [6,10,13]
+  CRUSH rule 0 x 280 [7,3,14]
+  CRUSH rule 0 x 281 [5,11,14]
+  CRUSH rule 0 x 282 [2,1,13]
+  CRUSH rule 0 x 283 [4,1,12]
+  CRUSH rule 0 x 284 [5,11,7]
+  CRUSH rule 0 x 285 [15,5,3]
+  CRUSH rule 0 x 286 [10,4,3]
+  CRUSH rule 0 x 287 [12,4,9]
+  CRUSH rule 0 x 288 [4,12,10]
+  CRUSH rule 0 x 289 [2,5,14]
+  CRUSH rule 0 x 290 [12,2,5]
+  CRUSH rule 0 x 291 [7,11,1]
+  CRUSH rule 0 x 292 [4,10,6]
+  CRUSH rule 0 x 293 [6,5,11]
+  CRUSH rule 0 x 294 [9,12,3]
+  CRUSH rule 0 x 295 [6,10,3]
+  CRUSH rule 0 x 296 [3,1,13]
+  CRUSH rule 0 x 297 [6,13,4]
+  CRUSH rule 0 x 298 [14,9,13]
+  CRUSH rule 0 x 299 [14,12,11]
+  CRUSH rule 0 x 300 [15,7,10]
+  CRUSH rule 0 x 301 [9,11,7]
+  CRUSH rule 0 x 302 [9,7,1]
+  CRUSH rule 0 x 303 [4,13,3]
+  CRUSH rule 0 x 304 [6,9,2]
+  CRUSH rule 0 x 305 [13,7,5]
+  CRUSH rule 0 x 306 [10,12,4]
+  CRUSH rule 0 x 307 [11,12,15]
+  CRUSH rule 0 x 308 [12,14,10]
+  CRUSH rule 0 x 309 [9,3,12]
+  CRUSH rule 0 x 310 [3,1,5]
+  CRUSH rule 0 x 311 [3,9,7]
+  CRUSH rule 0 x 312 [15,13,9]
+  CRUSH rule 0 x 313 [9,15,3]
+  CRUSH rule 0 x 314 [2,15,9]
+  CRUSH rule 0 x 315 [15,2,13]
+  CRUSH rule 0 x 316 [4,9,11]
+  CRUSH rule 0 x 317 [1,5,3]
+  CRUSH rule 0 x 318 [4,1,15]
+  CRUSH rule 0 x 319 [2,15,4]
+  CRUSH rule 0 x 320 [5,7,13]
+  CRUSH rule 0 x 321 [1,6,11]
+  CRUSH rule 0 x 322 [13,7,5]
+  CRUSH rule 0 x 323 [7,4,10]
+  CRUSH rule 0 x 324 [5,6,10]
+  CRUSH rule 0 x 325 [9,10,14]
+  CRUSH rule 0 x 326 [11,7,13]
+  CRUSH rule 0 x 327 [12,5,10]
+  CRUSH rule 0 x 328 [5,2,6]
+  CRUSH rule 0 x 329 [2,6,15]
+  CRUSH rule 0 x 330 [3,9,11]
+  CRUSH rule 0 x 331 [12,14,6]
+  CRUSH rule 0 x 332 [10,12,6]
+  CRUSH rule 0 x 333 [6,5,3]
+  CRUSH rule 0 x 334 [4,9,2]
+  CRUSH rule 0 x 335 [11,7,1]
+  CRUSH rule 0 x 336 [6,14,13]
+  CRUSH rule 0 x 337 [15,11,3]
+  CRUSH rule 0 x 338 [10,5,3]
+  CRUSH rule 0 x 339 [11,14,13]
+  CRUSH rule 0 x 340 [11,6,12]
+  CRUSH rule 0 x 341 [7,5,2]
+  CRUSH rule 0 x 342 [12,14,1]
+  CRUSH rule 0 x 343 [12,14,9]
+  CRUSH rule 0 x 344 [9,11,5]
+  CRUSH rule 0 x 345 [14,2,11]
+  CRUSH rule 0 x 346 [5,3,14]
+  CRUSH rule 0 x 347 [10,2,12]
+  CRUSH rule 0 x 348 [7,9,10]
+  CRUSH rule 0 x 349 [9,6,10]
+  CRUSH rule 0 x 350 [13,9,15]
+  CRUSH rule 0 x 351 [13,5,15]
+  CRUSH rule 0 x 352 [1,12,11]
+  CRUSH rule 0 x 353 [10,14,12]
+  CRUSH rule 0 x 354 [6,3,15]
+  CRUSH rule 0 x 355 [13,14,6]
+  CRUSH rule 0 x 356 [15,13,2]
+  CRUSH rule 0 x 357 [4,11,1]
+  CRUSH rule 0 x 358 [12,7,2]
+  CRUSH rule 0 x 359 [5,15,7]
+  CRUSH rule 0 x 360 [13,10,1]
+  CRUSH rule 0 x 361 [5,3,13]
+  CRUSH rule 0 x 362 [2,9,11]
+  CRUSH rule 0 x 363 [7,12,3]
+  CRUSH rule 0 x 364 [2,12,6]
+  CRUSH rule 0 x 365 [13,5,11]
+  CRUSH rule 0 x 366 [12,7,3]
+  CRUSH rule 0 x 367 [7,13,3]
+  CRUSH rule 0 x 368 [7,9,10]
+  CRUSH rule 0 x 369 [7,5,3]
+  CRUSH rule 0 x 370 [4,7,14]
+  CRUSH rule 0 x 371 [1,7,12]
+  CRUSH rule 0 x 372 [10,4,3]
+  CRUSH rule 0 x 373 [15,5,2]
+  CRUSH rule 0 x 374 [3,15,12]
+  CRUSH rule 0 x 375 [5,2,14]
+  CRUSH rule 0 x 376 [5,14,10]
+  CRUSH rule 0 x 377 [1,15,2]
+  CRUSH rule 0 x 378 [9,12,2]
+  CRUSH rule 0 x 379 [11,2,15]
+  CRUSH rule 0 x 380 [6,1,12]
+  CRUSH rule 0 x 381 [15,13,7]
+  CRUSH rule 0 x 382 [14,3,1]
+  CRUSH rule 0 x 383 [3,6,11]
+  CRUSH rule 0 x 384 [4,13,6]
+  CRUSH rule 0 x 385 [4,6,15]
+  CRUSH rule 0 x 386 [14,3,11]
+  CRUSH rule 0 x 387 [1,11,5]
+  CRUSH rule 0 x 388 [2,6,11]
+  CRUSH rule 0 x 389 [12,7,2]
+  CRUSH rule 0 x 390 [2,11,13]
+  CRUSH rule 0 x 391 [3,4,9]
+  CRUSH rule 0 x 392 [11,5,14]
+  CRUSH rule 0 x 393 [2,14,5]
+  CRUSH rule 0 x 394 [4,9,3]
+  CRUSH rule 0 x 395 [10,13,5]
+  CRUSH rule 0 x 396 [2,12,15]
+  CRUSH rule 0 x 397 [1,14,9]
+  CRUSH rule 0 x 398 [9,2,1]
+  CRUSH rule 0 x 399 [5,9,14]
+  CRUSH rule 0 x 400 [10,6,2]
+  CRUSH rule 0 x 401 [6,9,11]
+  CRUSH rule 0 x 402 [4,7,9]
+  CRUSH rule 0 x 403 [7,15,13]
+  CRUSH rule 0 x 404 [14,12,7]
+  CRUSH rule 0 x 405 [9,15,11]
+  CRUSH rule 0 x 406 [12,14,9]
+  CRUSH rule 0 x 407 [9,5,12]
+  CRUSH rule 0 x 408 [7,1,5]
+  CRUSH rule 0 x 409 [11,2,4]
+  CRUSH rule 0 x 410 [6,4,14]
+  CRUSH rule 0 x 411 [13,11,15]
+  CRUSH rule 0 x 412 [5,9,6]
+  CRUSH rule 0 x 413 [13,5,3]
+  CRUSH rule 0 x 414 [3,11,9]
+  CRUSH rule 0 x 415 [6,10,14]
+  CRUSH rule 0 x 416 [13,1,4]
+  CRUSH rule 0 x 417 [4,12,1]
+  CRUSH rule 0 x 418 [14,5,10]
+  CRUSH rule 0 x 419 [5,14,10]
+  CRUSH rule 0 x 420 [2,4,9]
+  CRUSH rule 0 x 421 [15,4,10]
+  CRUSH rule 0 x 422 [4,11,2]
+  CRUSH rule 0 x 423 [3,15,12]
+  CRUSH rule 0 x 424 [6,10,12]
+  CRUSH rule 0 x 425 [11,15,2]
+  CRUSH rule 0 x 426 [12,4,7]
+  CRUSH rule 0 x 427 [14,10,3]
+  CRUSH rule 0 x 428 [12,7,9]
+  CRUSH rule 0 x 429 [3,4,9]
+  CRUSH rule 0 x 430 [3,5,10]
+  CRUSH rule 0 x 431 [9,3,7]
+  CRUSH rule 0 x 432 [4,1,12]
+  CRUSH rule 0 x 433 [4,11,12]
+  CRUSH rule 0 x 434 [2,14,9]
+  CRUSH rule 0 x 435 [13,11,5]
+  CRUSH rule 0 x 436 [9,15,10]
+  CRUSH rule 0 x 437 [9,6,3]
+  CRUSH rule 0 x 438 [7,2,13]
+  CRUSH rule 0 x 439 [7,14,4]
+  CRUSH rule 0 x 440 [14,11,9]
+  CRUSH rule 0 x 441 [2,4,11]
+  CRUSH rule 0 x 442 [10,13,9]
+  CRUSH rule 0 x 443 [12,15,10]
+  CRUSH rule 0 x 444 [4,13,7]
+  CRUSH rule 0 x 445 [4,2,15]
+  CRUSH rule 0 x 446 [12,10,6]
+  CRUSH rule 0 x 447 [15,7,13]
+  CRUSH rule 0 x 448 [5,2,13]
+  CRUSH rule 0 x 449 [14,5,3]
+  CRUSH rule 0 x 450 [2,4,6]
+  CRUSH rule 0 x 451 [6,14,11]
+  CRUSH rule 0 x 452 [14,9,10]
+  CRUSH rule 0 x 453 [5,15,13]
+  CRUSH rule 0 x 454 [10,4,2]
+  CRUSH rule 0 x 455 [6,13,2]
+  CRUSH rule 0 x 456 [5,7,13]
+  CRUSH rule 0 x 457 [9,1,5]
+  CRUSH rule 0 x 458 [9,11,15]
+  CRUSH rule 0 x 459 [13,15,11]
+  CRUSH rule 0 x 460 [5,12,10]
+  CRUSH rule 0 x 461 [4,3,9]
+  CRUSH rule 0 x 462 [4,7,12]
+  CRUSH rule 0 x 463 [4,12,14]
+  CRUSH rule 0 x 464 [4,2,15]
+  CRUSH rule 0 x 465 [5,10,9]
+  CRUSH rule 0 x 466 [13,5,2]
+  CRUSH rule 0 x 467 [13,6,14]
+  CRUSH rule 0 x 468 [10,7,12]
+  CRUSH rule 0 x 469 [4,9,6]
+  CRUSH rule 0 x 470 [3,9,12]
+  CRUSH rule 0 x 471 [6,1,5]
+  CRUSH rule 0 x 472 [2,14,7]
+  CRUSH rule 0 x 473 [15,10,6]
+  CRUSH rule 0 x 474 [15,10,4]
+  CRUSH rule 0 x 475 [10,5,12]
+  CRUSH rule 0 x 476 [3,6,10]
+  CRUSH rule 0 x 477 [6,13,5]
+  CRUSH rule 0 x 478 [4,15,1]
+  CRUSH rule 0 x 479 [13,11,1]
+  CRUSH rule 0 x 480 [1,13,6]
+  CRUSH rule 0 x 481 [15,12,7]
+  CRUSH rule 0 x 482 [2,12,9]
+  CRUSH rule 0 x 483 [10,1,4]
+  CRUSH rule 0 x 484 [1,4,10]
+  CRUSH rule 0 x 485 [9,4,3]
+  CRUSH rule 0 x 486 [3,10,15]
+  CRUSH rule 0 x 487 [12,11,4]
+  CRUSH rule 0 x 488 [14,4,1]
+  CRUSH rule 0 x 489 [11,4,2]
+  CRUSH rule 0 x 490 [4,9,1]
+  CRUSH rule 0 x 491 [1,12,5]
+  CRUSH rule 0 x 492 [5,7,11]
+  CRUSH rule 0 x 493 [12,1,4]
+  CRUSH rule 0 x 494 [1,7,13]
+  CRUSH rule 0 x 495 [3,15,7]
+  CRUSH rule 0 x 496 [5,3,7]
+  CRUSH rule 0 x 497 [13,10,3]
+  CRUSH rule 0 x 498 [10,6,1]
+  CRUSH rule 0 x 499 [14,3,12]
+  CRUSH rule 0 x 500 [15,9,6]
+  CRUSH rule 0 x 501 [10,13,1]
+  CRUSH rule 0 x 502 [5,1,14]
+  CRUSH rule 0 x 503 [15,10,7]
+  CRUSH rule 0 x 504 [13,2,7]
+  CRUSH rule 0 x 505 [12,7,5]
+  CRUSH rule 0 x 506 [11,7,9]
+  CRUSH rule 0 x 507 [4,14,13]
+  CRUSH rule 0 x 508 [12,1,4]
+  CRUSH rule 0 x 509 [4,2,6]
+  CRUSH rule 0 x 510 [5,3,1]
+  CRUSH rule 0 x 511 [2,12,10]
+  CRUSH rule 0 x 512 [15,11,3]
+  CRUSH rule 0 x 513 [4,9,11]
+  CRUSH rule 0 x 514 [11,9,3]
+  CRUSH rule 0 x 515 [12,14,6]
+  CRUSH rule 0 x 516 [14,11,1]
+  CRUSH rule 0 x 517 [11,5,6]
+  CRUSH rule 0 x 518 [3,5,7]
+  CRUSH rule 0 x 519 [12,14,2]
+  CRUSH rule 0 x 520 [12,4,2]
+  CRUSH rule 0 x 521 [11,5,9]
+  CRUSH rule 0 x 522 [4,12,11]
+  CRUSH rule 0 x 523 [3,1,5]
+  CRUSH rule 0 x 524 [15,9,3]
+  CRUSH rule 0 x 525 [3,15,11]
+  CRUSH rule 0 x 526 [10,2,5]
+  CRUSH rule 0 x 527 [3,13,4]
+  CRUSH rule 0 x 528 [12,7,15]
+  CRUSH rule 0 x 529 [6,4,10]
+  CRUSH rule 0 x 530 [11,9,12]
+  CRUSH rule 0 x 531 [9,15,4]
+  CRUSH rule 0 x 532 [5,3,13]
+  CRUSH rule 0 x 533 [12,15,1]
+  CRUSH rule 0 x 534 [11,9,3]
+  CRUSH rule 0 x 535 [11,1,3]
+  CRUSH rule 0 x 536 [9,1,14]
+  CRUSH rule 0 x 537 [15,5,13]
+  CRUSH rule 0 x 538 [13,5,11]
+  CRUSH rule 0 x 539 [10,12,6]
+  CRUSH rule 0 x 540 [12,15,7]
+  CRUSH rule 0 x 541 [2,1,6]
+  CRUSH rule 0 x 542 [3,9,15]
+  CRUSH rule 0 x 543 [4,10,9]
+  CRUSH rule 0 x 544 [3,15,9]
+  CRUSH rule 0 x 545 [14,10,7]
+  CRUSH rule 0 x 546 [5,15,13]
+  CRUSH rule 0 x 547 [5,13,7]
+  CRUSH rule 0 x 548 [11,7,12]
+  CRUSH rule 0 x 549 [14,1,4]
+  CRUSH rule 0 x 550 [9,15,3]
+  CRUSH rule 0 x 551 [11,2,15]
+  CRUSH rule 0 x 552 [2,11,14]
+  CRUSH rule 0 x 553 [11,9,14]
+  CRUSH rule 0 x 554 [11,14,6]
+  CRUSH rule 0 x 555 [6,5,10]
+  CRUSH rule 0 x 556 [15,6,3]
+  CRUSH rule 0 x 557 [12,2,5]
+  CRUSH rule 0 x 558 [12,1,6]
+  CRUSH rule 0 x 559 [2,13,5]
+  CRUSH rule 0 x 560 [4,9,12]
+  CRUSH rule 0 x 561 [12,7,1]
+  CRUSH rule 0 x 562 [7,13,9]
+  CRUSH rule 0 x 563 [15,4,3]
+  CRUSH rule 0 x 564 [2,13,7]
+  CRUSH rule 0 x 565 [3,12,4]
+  CRUSH rule 0 x 566 [6,14,4]
+  CRUSH rule 0 x 567 [15,4,11]
+  CRUSH rule 0 x 568 [4,14,1]
+  CRUSH rule 0 x 569 [11,3,15]
+  CRUSH rule 0 x 570 [1,10,13]
+  CRUSH rule 0 x 571 [10,12,14]
+  CRUSH rule 0 x 572 [12,14,3]
+  CRUSH rule 0 x 573 [7,15,11]
+  CRUSH rule 0 x 574 [11,14,13]
+  CRUSH rule 0 x 575 [5,13,15]
+  CRUSH rule 0 x 576 [3,15,11]
+  CRUSH rule 0 x 577 [13,9,6]
+  CRUSH rule 0 x 578 [4,10,1]
+  CRUSH rule 0 x 579 [13,1,15]
+  CRUSH rule 0 x 580 [3,12,4]
+  CRUSH rule 0 x 581 [7,14,12]
+  CRUSH rule 0 x 582 [10,5,13]
+  CRUSH rule 0 x 583 [4,15,1]
+  CRUSH rule 0 x 584 [10,1,5]
+  CRUSH rule 0 x 585 [5,3,6]
+  CRUSH rule 0 x 586 [7,10,14]
+  CRUSH rule 0 x 587 [11,6,9]
+  CRUSH rule 0 x 588 [3,12,7]
+  CRUSH rule 0 x 589 [9,7,12]
+  CRUSH rule 0 x 590 [12,1,3]
+  CRUSH rule 0 x 591 [2,6,14]
+  CRUSH rule 0 x 592 [15,12,9]
+  CRUSH rule 0 x 593 [13,14,5]
+  CRUSH rule 0 x 594 [12,14,2]
+  CRUSH rule 0 x 595 [12,7,10]
+  CRUSH rule 0 x 596 [2,7,12]
+  CRUSH rule 0 x 597 [15,1,2]
+  CRUSH rule 0 x 598 [11,5,9]
+  CRUSH rule 0 x 599 [13,11,1]
+  CRUSH rule 0 x 600 [4,12,3]
+  CRUSH rule 0 x 601 [13,5,15]
+  CRUSH rule 0 x 602 [3,11,7]
+  CRUSH rule 0 x 603 [3,1,4]
+  CRUSH rule 0 x 604 [14,2,6]
+  CRUSH rule 0 x 605 [2,7,12]
+  CRUSH rule 0 x 606 [12,15,1]
+  CRUSH rule 0 x 607 [3,9,10]
+  CRUSH rule 0 x 608 [13,10,1]
+  CRUSH rule 0 x 609 [14,3,7]
+  CRUSH rule 0 x 610 [7,10,5]
+  CRUSH rule 0 x 611 [13,1,5]
+  CRUSH rule 0 x 612 [7,1,2]
+  CRUSH rule 0 x 613 [10,7,14]
+  CRUSH rule 0 x 614 [9,4,15]
+  CRUSH rule 0 x 615 [9,4,11]
+  CRUSH rule 0 x 616 [10,14,1]
+  CRUSH rule 0 x 617 [15,7,2]
+  CRUSH rule 0 x 618 [4,2,10]
+  CRUSH rule 0 x 619 [15,4,3]
+  CRUSH rule 0 x 620 [3,7,11]
+  CRUSH rule 0 x 621 [3,6,4]
+  CRUSH rule 0 x 622 [10,2,13]
+  CRUSH rule 0 x 623 [4,9,14]
+  CRUSH rule 0 x 624 [3,9,15]
+  CRUSH rule 0 x 625 [11,7,3]
+  CRUSH rule 0 x 626 [10,12,2]
+  CRUSH rule 0 x 627 [1,12,10]
+  CRUSH rule 0 x 628 [15,13,11]
+  CRUSH rule 0 x 629 [5,6,15]
+  CRUSH rule 0 x 630 [1,4,12]
+  CRUSH rule 0 x 631 [5,7,1]
+  CRUSH rule 0 x 632 [12,3,11]
+  CRUSH rule 0 x 633 [14,4,3]
+  CRUSH rule 0 x 634 [6,9,5]
+  CRUSH rule 0 x 635 [6,5,2]
+  CRUSH rule 0 x 636 [13,6,11]
+  CRUSH rule 0 x 637 [3,1,10]
+  CRUSH rule 0 x 638 [10,15,3]
+  CRUSH rule 0 x 639 [6,9,14]
+  CRUSH rule 0 x 640 [9,6,1]
+  CRUSH rule 0 x 641 [10,6,5]
+  CRUSH rule 0 x 642 [1,15,4]
+  CRUSH rule 0 x 643 [3,7,5]
+  CRUSH rule 0 x 644 [15,13,6]
+  CRUSH rule 0 x 645 [14,2,4]
+  CRUSH rule 0 x 646 [5,13,14]
+  CRUSH rule 0 x 647 [10,1,9]
+  CRUSH rule 0 x 648 [6,5,2]
+  CRUSH rule 0 x 649 [3,9,13]
+  CRUSH rule 0 x 650 [10,9,4]
+  CRUSH rule 0 x 651 [3,9,5]
+  CRUSH rule 0 x 652 [15,9,4]
+  CRUSH rule 0 x 653 [11,14,1]
+  CRUSH rule 0 x 654 [13,6,2]
+  CRUSH rule 0 x 655 [6,3,4]
+  CRUSH rule 0 x 656 [3,15,1]
+  CRUSH rule 0 x 657 [11,15,3]
+  CRUSH rule 0 x 658 [7,2,10]
+  CRUSH rule 0 x 659 [2,5,14]
+  CRUSH rule 0 x 660 [13,14,10]
+  CRUSH rule 0 x 661 [7,15,3]
+  CRUSH rule 0 x 662 [15,2,12]
+  CRUSH rule 0 x 663 [14,9,13]
+  CRUSH rule 0 x 664 [6,10,12]
+  CRUSH rule 0 x 665 [2,9,12]
+  CRUSH rule 0 x 666 [12,3,6]
+  CRUSH rule 0 x 667 [1,9,12]
+  CRUSH rule 0 x 668 [9,5,1]
+  CRUSH rule 0 x 669 [9,7,14]
+  CRUSH rule 0 x 670 [6,10,9]
+  CRUSH rule 0 x 671 [6,15,5]
+  CRUSH rule 0 x 672 [2,9,13]
+  CRUSH rule 0 x 673 [7,10,5]
+  CRUSH rule 0 x 674 [7,12,10]
+  CRUSH rule 0 x 675 [9,5,1]
+  CRUSH rule 0 x 676 [10,12,2]
+  CRUSH rule 0 x 677 [2,12,1]
+  CRUSH rule 0 x 678 [1,2,4]
+  CRUSH rule 0 x 679 [5,6,12]
+  CRUSH rule 0 x 680 [7,11,3]
+  CRUSH rule 0 x 681 [6,4,3]
+  CRUSH rule 0 x 682 [6,1,11]
+  CRUSH rule 0 x 683 [6,13,2]
+  CRUSH rule 0 x 684 [9,11,3]
+  CRUSH rule 0 x 685 [5,1,15]
+  CRUSH rule 0 x 686 [1,9,11]
+  CRUSH rule 0 x 687 [7,13,3]
+  CRUSH rule 0 x 688 [11,9,1]
+  CRUSH rule 0 x 689 [5,2,9]
+  CRUSH rule 0 x 690 [9,7,10]
+  CRUSH rule 0 x 691 [11,15,9]
+  CRUSH rule 0 x 692 [15,5,1]
+  CRUSH rule 0 x 693 [5,6,12]
+  CRUSH rule 0 x 694 [4,7,1]
+  CRUSH rule 0 x 695 [6,13,14]
+  CRUSH rule 0 x 696 [1,2,4]
+  CRUSH rule 0 x 697 [13,11,3]
+  CRUSH rule 0 x 698 [11,13,4]
+  CRUSH rule 0 x 699 [7,14,12]
+  CRUSH rule 0 x 700 [12,14,11]
+  CRUSH rule 0 x 701 [3,13,1]
+  CRUSH rule 0 x 702 [3,12,15]
+  CRUSH rule 0 x 703 [15,11,13]
+  CRUSH rule 0 x 704 [6,4,2]
+  CRUSH rule 0 x 705 [14,6,11]
+  CRUSH rule 0 x 706 [1,12,3]
+  CRUSH rule 0 x 707 [4,7,14]
+  CRUSH rule 0 x 708 [3,10,5]
+  CRUSH rule 0 x 709 [11,12,3]
+  CRUSH rule 0 x 710 [14,2,11]
+  CRUSH rule 0 x 711 [14,3,9]
+  CRUSH rule 0 x 712 [12,3,11]
+  CRUSH rule 0 x 713 [11,9,3]
+  CRUSH rule 0 x 714 [12,1,9]
+  CRUSH rule 0 x 715 [6,1,14]
+  CRUSH rule 0 x 716 [11,13,9]
+  CRUSH rule 0 x 717 [12,4,10]
+  CRUSH rule 0 x 718 [7,15,5]
+  CRUSH rule 0 x 719 [5,15,13]
+  CRUSH rule 0 x 720 [4,13,10]
+  CRUSH rule 0 x 721 [11,3,14]
+  CRUSH rule 0 x 722 [2,4,6]
+  CRUSH rule 0 x 723 [2,1,12]
+  CRUSH rule 0 x 724 [7,1,9]
+  CRUSH rule 0 x 725 [11,12,7]
+  CRUSH rule 0 x 726 [7,14,4]
+  CRUSH rule 0 x 727 [2,5,1]
+  CRUSH rule 0 x 728 [13,11,4]
+  CRUSH rule 0 x 729 [15,11,4]
+  CRUSH rule 0 x 730 [3,7,1]
+  CRUSH rule 0 x 731 [9,1,6]
+  CRUSH rule 0 x 732 [1,2,10]
+  CRUSH rule 0 x 733 [11,3,5]
+  CRUSH rule 0 x 734 [14,3,11]
+  CRUSH rule 0 x 735 [6,9,2]
+  CRUSH rule 0 x 736 [3,9,1]
+  CRUSH rule 0 x 737 [1,4,2]
+  CRUSH rule 0 x 738 [11,15,7]
+  CRUSH rule 0 x 739 [11,12,6]
+  CRUSH rule 0 x 740 [7,9,10]
+  CRUSH rule 0 x 741 [12,11,7]
+  CRUSH rule 0 x 742 [9,7,4]
+  CRUSH rule 0 x 743 [5,13,9]
+  CRUSH rule 0 x 744 [6,2,13]
+  CRUSH rule 0 x 745 [3,6,1]
+  CRUSH rule 0 x 746 [3,7,9]
+  CRUSH rule 0 x 747 [15,11,5]
+  CRUSH rule 0 x 748 [6,10,13]
+  CRUSH rule 0 x 749 [14,9,10]
+  CRUSH rule 0 x 750 [1,14,6]
+  CRUSH rule 0 x 751 [15,1,6]
+  CRUSH rule 0 x 752 [13,1,7]
+  CRUSH rule 0 x 753 [4,11,1]
+  CRUSH rule 0 x 754 [14,12,11]
+  CRUSH rule 0 x 755 [13,6,1]
+  CRUSH rule 0 x 756 [3,4,14]
+  CRUSH rule 0 x 757 [10,6,1]
+  CRUSH rule 0 x 758 [6,3,4]
+  CRUSH rule 0 x 759 [5,7,3]
+  CRUSH rule 0 x 760 [1,15,10]
+  CRUSH rule 0 x 761 [2,12,1]
+  CRUSH rule 0 x 762 [1,4,10]
+  CRUSH rule 0 x 763 [4,13,1]
+  CRUSH rule 0 x 764 [1,14,6]
+  CRUSH rule 0 x 765 [9,15,2]
+  CRUSH rule 0 x 766 [11,2,7]
+  CRUSH rule 0 x 767 [6,11,4]
+  CRUSH rule 0 x 768 [2,12,15]
+  CRUSH rule 0 x 769 [15,1,9]
+  CRUSH rule 0 x 770 [15,13,4]
+  CRUSH rule 0 x 771 [9,2,12]
+  CRUSH rule 0 x 772 [4,3,13]
+  CRUSH rule 0 x 773 [3,7,4]
+  CRUSH rule 0 x 774 [12,6,3]
+  CRUSH rule 0 x 775 [5,10,14]
+  CRUSH rule 0 x 776 [10,15,3]
+  CRUSH rule 0 x 777 [11,13,4]
+  CRUSH rule 0 x 778 [13,1,9]
+  CRUSH rule 0 x 779 [5,11,1]
+  CRUSH rule 0 x 780 [13,9,3]
+  CRUSH rule 0 x 781 [5,7,14]
+  CRUSH rule 0 x 782 [2,15,9]
+  CRUSH rule 0 x 783 [12,7,5]
+  CRUSH rule 0 x 784 [14,1,10]
+  CRUSH rule 0 x 785 [6,12,1]
+  CRUSH rule 0 x 786 [10,5,2]
+  CRUSH rule 0 x 787 [1,12,10]
+  CRUSH rule 0 x 788 [4,2,9]
+  CRUSH rule 0 x 789 [9,2,14]
+  CRUSH rule 0 x 790 [15,2,7]
+  CRUSH rule 0 x 791 [9,4,7]
+  CRUSH rule 0 x 792 [6,4,15]
+  CRUSH rule 0 x 793 [15,9,6]
+  CRUSH rule 0 x 794 [5,12,2]
+  CRUSH rule 0 x 795 [6,14,12]
+  CRUSH rule 0 x 796 [11,2,12]
+  CRUSH rule 0 x 797 [14,3,7]
+  CRUSH rule 0 x 798 [5,11,6]
+  CRUSH rule 0 x 799 [2,9,14]
+  CRUSH rule 0 x 800 [6,3,4]
+  CRUSH rule 0 x 801 [2,5,6]
+  CRUSH rule 0 x 802 [1,4,12]
+  CRUSH rule 0 x 803 [7,2,4]
+  CRUSH rule 0 x 804 [5,14,9]
+  CRUSH rule 0 x 805 [13,4,3]
+  CRUSH rule 0 x 806 [6,2,13]
+  CRUSH rule 0 x 807 [14,2,7]
+  CRUSH rule 0 x 808 [2,15,12]
+  CRUSH rule 0 x 809 [1,11,7]
+  CRUSH rule 0 x 810 [2,5,9]
+  CRUSH rule 0 x 811 [15,6,3]
+  CRUSH rule 0 x 812 [7,11,2]
+  CRUSH rule 0 x 813 [4,10,13]
+  CRUSH rule 0 x 814 [13,4,9]
+  CRUSH rule 0 x 815 [15,12,9]
+  CRUSH rule 0 x 816 [14,10,13]
+  CRUSH rule 0 x 817 [10,7,2]
+  CRUSH rule 0 x 818 [15,2,11]
+  CRUSH rule 0 x 819 [5,12,10]
+  CRUSH rule 0 x 820 [3,6,9]
+  CRUSH rule 0 x 821 [15,10,9]
+  CRUSH rule 0 x 822 [10,13,2]
+  CRUSH rule 0 x 823 [2,6,12]
+  CRUSH rule 0 x 824 [3,7,9]
+  CRUSH rule 0 x 825 [10,5,14]
+  CRUSH rule 0 x 826 [5,2,11]
+  CRUSH rule 0 x 827 [13,5,1]
+  CRUSH rule 0 x 828 [12,6,10]
+  CRUSH rule 0 x 829 [13,6,15]
+  CRUSH rule 0 x 830 [15,13,2]
+  CRUSH rule 0 x 831 [1,4,11]
+  CRUSH rule 0 x 832 [14,11,13]
+  CRUSH rule 0 x 833 [9,13,3]
+  CRUSH rule 0 x 834 [9,7,5]
+  CRUSH rule 0 x 835 [14,3,13]
+  CRUSH rule 0 x 836 [3,9,10]
+  CRUSH rule 0 x 837 [15,12,11]
+  CRUSH rule 0 x 838 [12,14,9]
+  CRUSH rule 0 x 839 [3,4,6]
+  CRUSH rule 0 x 840 [10,15,12]
+  CRUSH rule 0 x 841 [3,5,7]
+  CRUSH rule 0 x 842 [9,13,2]
+  CRUSH rule 0 x 843 [14,7,4]
+  CRUSH rule 0 x 844 [7,1,4]
+  CRUSH rule 0 x 845 [13,6,1]
+  CRUSH rule 0 x 846 [3,7,15]
+  CRUSH rule 0 x 847 [12,15,11]
+  CRUSH rule 0 x 848 [11,13,1]
+  CRUSH rule 0 x 849 [3,15,11]
+  CRUSH rule 0 x 850 [1,3,10]
+  CRUSH rule 0 x 851 [14,4,3]
+  CRUSH rule 0 x 852 [9,12,4]
+  CRUSH rule 0 x 853 [13,14,6]
+  CRUSH rule 0 x 854 [7,11,12]
+  CRUSH rule 0 x 855 [14,4,12]
+  CRUSH rule 0 x 856 [5,10,7]
+  CRUSH rule 0 x 857 [4,3,13]
+  CRUSH rule 0 x 858 [5,15,6]
+  CRUSH rule 0 x 859 [5,15,6]
+  CRUSH rule 0 x 860 [11,14,1]
+  CRUSH rule 0 x 861 [13,7,4]
+  CRUSH rule 0 x 862 [5,10,9]
+  CRUSH rule 0 x 863 [11,6,3]
+  CRUSH rule 0 x 864 [6,13,4]
+  CRUSH rule 0 x 865 [4,1,14]
+  CRUSH rule 0 x 866 [2,13,4]
+  CRUSH rule 0 x 867 [12,2,9]
+  CRUSH rule 0 x 868 [14,11,7]
+  CRUSH rule 0 x 869 [10,13,7]
+  CRUSH rule 0 x 870 [14,9,11]
+  CRUSH rule 0 x 871 [6,2,1]
+  CRUSH rule 0 x 872 [6,1,15]
+  CRUSH rule 0 x 873 [2,5,12]
+  CRUSH rule 0 x 874 [12,4,7]
+  CRUSH rule 0 x 875 [10,6,14]
+  CRUSH rule 0 x 876 [14,7,13]
+  CRUSH rule 0 x 877 [15,11,13]
+  CRUSH rule 0 x 878 [7,14,3]
+  CRUSH rule 0 x 879 [12,2,7]
+  CRUSH rule 0 x 880 [2,12,10]
+  CRUSH rule 0 x 881 [6,3,1]
+  CRUSH rule 0 x 882 [11,13,7]
+  CRUSH rule 0 x 883 [13,1,3]
+  CRUSH rule 0 x 884 [6,15,4]
+  CRUSH rule 0 x 885 [14,7,9]
+  CRUSH rule 0 x 886 [13,11,4]
+  CRUSH rule 0 x 887 [14,4,12]
+  CRUSH rule 0 x 888 [10,12,7]
+  CRUSH rule 0 x 889 [15,13,4]
+  CRUSH rule 0 x 890 [10,12,14]
+  CRUSH rule 0 x 891 [9,5,11]
+  CRUSH rule 0 x 892 [12,15,2]
+  CRUSH rule 0 x 893 [1,3,5]
+  CRUSH rule 0 x 894 [7,2,11]
+  CRUSH rule 0 x 895 [2,1,11]
+  CRUSH rule 0 x 896 [9,1,14]
+  CRUSH rule 0 x 897 [7,5,14]
+  CRUSH rule 0 x 898 [10,6,12]
+  CRUSH rule 0 x 899 [1,11,5]
+  CRUSH rule 0 x 900 [2,9,10]
+  CRUSH rule 0 x 901 [9,12,11]
+  CRUSH rule 0 x 902 [4,2,6]
+  CRUSH rule 0 x 903 [14,10,3]
+  CRUSH rule 0 x 904 [15,12,4]
+  CRUSH rule 0 x 905 [12,6,11]
+  CRUSH rule 0 x 906 [14,11,12]
+  CRUSH rule 0 x 907 [7,12,3]
+  CRUSH rule 0 x 908 [2,15,9]
+  CRUSH rule 0 x 909 [10,14,1]
+  CRUSH rule 0 x 910 [12,7,4]
+  CRUSH rule 0 x 911 [11,15,2]
+  CRUSH rule 0 x 912 [6,4,14]
+  CRUSH rule 0 x 913 [4,6,10]
+  CRUSH rule 0 x 914 [4,15,2]
+  CRUSH rule 0 x 915 [12,14,1]
+  CRUSH rule 0 x 916 [3,1,11]
+  CRUSH rule 0 x 917 [1,15,6]
+  CRUSH rule 0 x 918 [7,14,11]
+  CRUSH rule 0 x 919 [10,7,3]
+  CRUSH rule 0 x 920 [4,2,10]
+  CRUSH rule 0 x 921 [1,11,6]
+  CRUSH rule 0 x 922 [6,4,14]
+  CRUSH rule 0 x 923 [12,2,5]
+  CRUSH rule 0 x 924 [6,2,14]
+  CRUSH rule 0 x 925 [12,15,2]
+  CRUSH rule 0 x 926 [3,13,10]
+  CRUSH rule 0 x 927 [6,5,1]
+  CRUSH rule 0 x 928 [13,1,3]
+  CRUSH rule 0 x 929 [10,7,1]
+  CRUSH rule 0 x 930 [7,15,10]
+  CRUSH rule 0 x 931 [6,15,11]
+  CRUSH rule 0 x 932 [13,2,5]
+  CRUSH rule 0 x 933 [12,7,14]
+  CRUSH rule 0 x 934 [12,2,5]
+  CRUSH rule 0 x 935 [6,11,1]
+  CRUSH rule 0 x 936 [9,12,7]
+  CRUSH rule 0 x 937 [14,2,11]
+  CRUSH rule 0 x 938 [14,3,5]
+  CRUSH rule 0 x 939 [6,4,14]
+  CRUSH rule 0 x 940 [13,11,4]
+  CRUSH rule 0 x 941 [3,12,4]
+  CRUSH rule 0 x 942 [15,12,10]
+  CRUSH rule 0 x 943 [10,2,4]
+  CRUSH rule 0 x 944 [2,9,4]
+  CRUSH rule 0 x 945 [10,15,2]
+  CRUSH rule 0 x 946 [11,15,7]
+  CRUSH rule 0 x 947 [11,3,14]
+  CRUSH rule 0 x 948 [7,13,11]
+  CRUSH rule 0 x 949 [9,1,12]
+  CRUSH rule 0 x 950 [9,15,13]
+  CRUSH rule 0 x 951 [2,6,12]
+  CRUSH rule 0 x 952 [9,7,15]
+  CRUSH rule 0 x 953 [1,3,6]
+  CRUSH rule 0 x 954 [10,2,14]
+  CRUSH rule 0 x 955 [7,14,3]
+  CRUSH rule 0 x 956 [1,6,11]
+  CRUSH rule 0 x 957 [14,11,1]
+  CRUSH rule 0 x 958 [15,4,3]
+  CRUSH rule 0 x 959 [2,1,12]
+  CRUSH rule 0 x 960 [2,6,11]
+  CRUSH rule 0 x 961 [3,13,11]
+  CRUSH rule 0 x 962 [5,11,3]
+  CRUSH rule 0 x 963 [13,10,15]
+  CRUSH rule 0 x 964 [7,11,4]
+  CRUSH rule 0 x 965 [12,2,9]
+  CRUSH rule 0 x 966 [12,14,9]
+  CRUSH rule 0 x 967 [7,5,3]
+  CRUSH rule 0 x 968 [12,15,4]
+  CRUSH rule 0 x 969 [11,4,7]
+  CRUSH rule 0 x 970 [5,12,10]
+  CRUSH rule 0 x 971 [1,9,4]
+  CRUSH rule 0 x 972 [12,3,14]
+  CRUSH rule 0 x 973 [1,10,4]
+  CRUSH rule 0 x 974 [7,11,1]
+  CRUSH rule 0 x 975 [7,9,15]
+  CRUSH rule 0 x 976 [7,3,15]
+  CRUSH rule 0 x 977 [14,3,6]
+  CRUSH rule 0 x 978 [12,5,11]
+  CRUSH rule 0 x 979 [5,1,13]
+  CRUSH rule 0 x 980 [15,11,5]
+  CRUSH rule 0 x 981 [5,11,15]
+  CRUSH rule 0 x 982 [2,6,14]
+  CRUSH rule 0 x 983 [3,12,10]
+  CRUSH rule 0 x 984 [15,13,1]
+  CRUSH rule 0 x 985 [11,2,15]
+  CRUSH rule 0 x 986 [6,13,9]
+  CRUSH rule 0 x 987 [13,14,5]
+  CRUSH rule 0 x 988 [12,9,10]
+  CRUSH rule 0 x 989 [7,4,3]
+  CRUSH rule 0 x 990 [1,10,9]
+  CRUSH rule 0 x 991 [7,11,1]
+  CRUSH rule 0 x 992 [9,10,2]
+  CRUSH rule 0 x 993 [6,10,14]
+  CRUSH rule 0 x 994 [3,13,15]
+  CRUSH rule 0 x 995 [15,6,12]
+  CRUSH rule 0 x 996 [15,10,5]
+  CRUSH rule 0 x 997 [15,2,1]
+  CRUSH rule 0 x 998 [6,1,9]
+  CRUSH rule 0 x 999 [9,10,15]
+  CRUSH rule 0 x 1000 [14,2,9]
+  CRUSH rule 0 x 1001 [11,14,4]
+  CRUSH rule 0 x 1002 [1,10,14]
+  CRUSH rule 0 x 1003 [10,7,5]
+  CRUSH rule 0 x 1004 [15,1,4]
+  CRUSH rule 0 x 1005 [6,12,2]
+  CRUSH rule 0 x 1006 [10,12,15]
+  CRUSH rule 0 x 1007 [1,7,13]
+  CRUSH rule 0 x 1008 [7,4,9]
+  CRUSH rule 0 x 1009 [5,2,11]
+  CRUSH rule 0 x 1010 [10,2,15]
+  CRUSH rule 0 x 1011 [6,3,12]
+  CRUSH rule 0 x 1012 [12,6,9]
+  CRUSH rule 0 x 1013 [2,14,12]
+  CRUSH rule 0 x 1014 [1,13,7]
+  CRUSH rule 0 x 1015 [12,6,10]
+  CRUSH rule 0 x 1016 [10,13,14]
+  CRUSH rule 0 x 1017 [5,11,14]
+  CRUSH rule 0 x 1018 [13,11,14]
+  CRUSH rule 0 x 1019 [10,13,14]
+  CRUSH rule 0 x 1020 [3,1,13]
+  CRUSH rule 0 x 1021 [2,11,14]
+  CRUSH rule 0 x 1022 [15,5,7]
+  CRUSH rule 0 x 1023 [15,2,9]
+  rule 0 (replicated_ruleset) num_rep 3 result size == 3:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3,15]
+  CRUSH rule 0 x 1 [10,15,1,2]
+  CRUSH rule 0 x 2 [1,12,2,6]
+  CRUSH rule 0 x 3 [15,4,10,2]
+  CRUSH rule 0 x 4 [14,2,10,1]
+  CRUSH rule 0 x 5 [7,4,11,2]
+  CRUSH rule 0 x 6 [12,6,10,9]
+  CRUSH rule 0 x 7 [9,2,6,12]
+  CRUSH rule 0 x 8 [10,2,15,1]
+  CRUSH rule 0 x 9 [7,1,14,2]
+  CRUSH rule 0 x 10 [10,14,4,1]
+  CRUSH rule 0 x 11 [13,9,14,7]
+  CRUSH rule 0 x 12 [7,1,2,5]
+  CRUSH rule 0 x 13 [3,5,12,7]
+  CRUSH rule 0 x 14 [13,5,2,7]
+  CRUSH rule 0 x 15 [15,1,9,6]
+  CRUSH rule 0 x 16 [7,11,14,2]
+  CRUSH rule 0 x 17 [10,1,13,2]
+  CRUSH rule 0 x 18 [1,7,3,10]
+  CRUSH rule 0 x 19 [7,12,2,4]
+  CRUSH rule 0 x 20 [14,12,3,10]
+  CRUSH rule 0 x 21 [3,12,1,10]
+  CRUSH rule 0 x 22 [6,3,13,11]
+  CRUSH rule 0 x 23 [10,5,13,9]
+  CRUSH rule 0 x 24 [12,11,3,1]
+  CRUSH rule 0 x 25 [7,12,15,1]
+  CRUSH rule 0 x 26 [1,7,13,2]
+  CRUSH rule 0 x 27 [3,6,15,4]
+  CRUSH rule 0 x 28 [14,4,3,9]
+  CRUSH rule 0 x 29 [5,14,12,11]
+  CRUSH rule 0 x 30 [2,5,6,9]
+  CRUSH rule 0 x 31 [5,15,10,1]
+  CRUSH rule 0 x 32 [9,10,2,1]
+  CRUSH rule 0 x 33 [13,4,9,2]
+  CRUSH rule 0 x 34 [13,15,2,4]
+  CRUSH rule 0 x 35 [4,14,3,13]
+  CRUSH rule 0 x 36 [3,12,9,7]
+  CRUSH rule 0 x 37 [9,2,6,14]
+  CRUSH rule 0 x 38 [3,4,13,10]
+  CRUSH rule 0 x 39 [12,7,14,11]
+  CRUSH rule 0 x 40 [10,1,9,5]
+  CRUSH rule 0 x 41 [4,9,11,1]
+  CRUSH rule 0 x 42 [3,6,14,10]
+  CRUSH rule 0 x 43 [10,5,15,7]
+  CRUSH rule 0 x 44 [11,4,13,3]
+  CRUSH rule 0 x 45 [11,12,15,9]
+  CRUSH rule 0 x 46 [6,9,2,14]
+  CRUSH rule 0 x 47 [3,9,6,4]
+  CRUSH rule 0 x 48 [4,6,2,1]
+  CRUSH rule 0 x 49 [9,15,10,7]
+  CRUSH rule 0 x 50 [14,12,1,4]
+  CRUSH rule 0 x 51 [10,6,5,12]
+  CRUSH rule 0 x 52 [12,1,9,11]
+  CRUSH rule 0 x 53 [3,6,13,9]
+  CRUSH rule 0 x 54 [4,13,9,2]
+  CRUSH rule 0 x 55 [4,11,2,7]
+  CRUSH rule 0 x 56 [5,9,10,1]
+  CRUSH rule 0 x 57 [6,2,1,15]
+  CRUSH rule 0 x 58 [7,1,11,4]
+  CRUSH rule 0 x 59 [2,13,1,10]
+  CRUSH rule 0 x 60 [3,6,11,1]
+  CRUSH rule 0 x 61 [3,15,13,7]
+  CRUSH rule 0 x 62 [15,11,7,12]
+  CRUSH rule 0 x 63 [10,14,12,1]
+  CRUSH rule 0 x 64 [3,9,1,4]
+  CRUSH rule 0 x 65 [4,12,11,7]
+  CRUSH rule 0 x 66 [15,11,6,9]
+  CRUSH rule 0 x 67 [2,6,4,14]
+  CRUSH rule 0 x 68 [15,7,4,2]
+  CRUSH rule 0 x 69 [2,1,15,10]
+  CRUSH rule 0 x 70 [9,6,1,3]
+  CRUSH rule 0 x 71 [15,5,1,3]
+  CRUSH rule 0 x 72 [9,10,3,5]
+  CRUSH rule 0 x 73 [5,3,11,1]
+  CRUSH rule 0 x 74 [11,7,9,5]
+  CRUSH rule 0 x 75 [9,7,11,14]
+  CRUSH rule 0 x 76 [6,1,3,5]
+  CRUSH rule 0 x 77 [7,4,2,13]
+  CRUSH rule 0 x 78 [9,3,1,5]
+  CRUSH rule 0 x 79 [13,2,15,5]
+  CRUSH rule 0 x 80 [15,2,6,4]
+  CRUSH rule 0 x 81 [15,2,1,11]
+  CRUSH rule 0 x 82 [14,13,5,11]
+  CRUSH rule 0 x 83 [4,15,3,9]
+  CRUSH rule 0 x 84 [10,7,9,15]
+  CRUSH rule 0 x 85 [3,15,9,7]
+  CRUSH rule 0 x 86 [10,9,14,1]
+  CRUSH rule 0 x 87 [15,10,7,12]
+  CRUSH rule 0 x 88 [4,13,3,1]
+  CRUSH rule 0 x 89 [3,9,7,4]
+  CRUSH rule 0 x 90 [4,9,7,12]
+  CRUSH rule 0 x 91 [6,11,9,1]
+  CRUSH rule 0 x 92 [1,5,10,9]
+  CRUSH rule 0 x 93 [9,3,15,13]
+  CRUSH rule 0 x 94 [9,2,12,5]
+  CRUSH rule 0 x 95 [7,15,4,10]
+  CRUSH rule 0 x 96 [2,15,11,7]
+  CRUSH rule 0 x 97 [4,11,2,13]
+  CRUSH rule 0 x 98 [11,13,9,3]
+  CRUSH rule 0 x 99 [12,4,11,7]
+  CRUSH rule 0 x 100 [9,4,10,15]
+  CRUSH rule 0 x 101 [15,7,1,9]
+  CRUSH rule 0 x 102 [3,11,14,6]
+  CRUSH rule 0 x 103 [13,11,6,14]
+  CRUSH rule 0 x 104 [14,6,3,5]
+  CRUSH rule 0 x 105 [14,10,1,9]
+  CRUSH rule 0 x 106 [6,5,13,2]
+  CRUSH rule 0 x 107 [3,1,10,14]
+  CRUSH rule 0 x 108 [5,10,7,2]
+  CRUSH rule 0 x 109 [9,1,13,7]
+  CRUSH rule 0 x 110 [5,1,11,3]
+  CRUSH rule 0 x 111 [10,1,9,7]
+  CRUSH rule 0 x 112 [1,10,4,14]
+  CRUSH rule 0 x 113 [6,10,13,9]
+  CRUSH rule 0 x 114 [5,13,6,2]
+  CRUSH rule 0 x 115 [10,13,14,3]
+  CRUSH rule 0 x 116 [1,14,13,2]
+  CRUSH rule 0 x 117 [5,6,1,12]
+  CRUSH rule 0 x 118 [10,4,13,15]
+  CRUSH rule 0 x 119 [14,12,11,4]
+  CRUSH rule 0 x 120 [11,3,14,13]
+  CRUSH rule 0 x 121 [9,5,1,11]
+  CRUSH rule 0 x 122 [4,3,14,1]
+  CRUSH rule 0 x 123 [3,10,5,6]
+  CRUSH rule 0 x 124 [12,2,1,5]
+  CRUSH rule 0 x 125 [9,12,15,1]
+  CRUSH rule 0 x 126 [7,15,10,9]
+  CRUSH rule 0 x 127 [4,14,9,13]
+  CRUSH rule 0 x 128 [3,12,1,10]
+  CRUSH rule 0 x 129 [11,13,14,2]
+  CRUSH rule 0 x 130 [3,13,5,14]
+  CRUSH rule 0 x 131 [12,1,6,15]
+  CRUSH rule 0 x 132 [11,15,13,9]
+  CRUSH rule 0 x 133 [3,6,9,11]
+  CRUSH rule 0 x 134 [12,5,6,15]
+  CRUSH rule 0 x 135 [3,14,12,4]
+  CRUSH rule 0 x 136 [15,6,9,4]
+  CRUSH rule 0 x 137 [14,3,6,11]
+  CRUSH rule 0 x 138 [13,15,4,10]
+  CRUSH rule 0 x 139 [11,2,13,9]
+  CRUSH rule 0 x 140 [11,4,12,15]
+  CRUSH rule 0 x 141 [6,12,15,11]
+  CRUSH rule 0 x 142 [3,14,7,9]
+  CRUSH rule 0 x 143 [9,6,4,2]
+  CRUSH rule 0 x 144 [13,7,11,2]
+  CRUSH rule 0 x 145 [12,2,6,10]
+  CRUSH rule 0 x 146 [1,5,9,2]
+  CRUSH rule 0 x 147 [1,4,9,11]
+  CRUSH rule 0 x 148 [12,7,9,2]
+  CRUSH rule 0 x 149 [2,5,9,12]
+  CRUSH rule 0 x 150 [1,15,2,10]
+  CRUSH rule 0 x 151 [2,9,14,7]
+  CRUSH rule 0 x 152 [5,9,2,6]
+  CRUSH rule 0 x 153 [6,9,4,15]
+  CRUSH rule 0 x 154 [3,11,7,1]
+  CRUSH rule 0 x 155 [14,12,7,3]
+  CRUSH rule 0 x 156 [7,13,3,10]
+  CRUSH rule 0 x 157 [15,1,6,4]
+  CRUSH rule 0 x 158 [15,1,10,6]
+  CRUSH rule 0 x 159 [4,14,3,12]
+  CRUSH rule 0 x 160 [5,7,3,14]
+  CRUSH rule 0 x 161 [1,2,11,4]
+  CRUSH rule 0 x 162 [10,6,1,12]
+  CRUSH rule 0 x 163 [15,1,10,2]
+  CRUSH rule 0 x 164 [9,14,10,7]
+  CRUSH rule 0 x 165 [11,7,2,13]
+  CRUSH rule 0 x 166 [1,2,12,14]
+  CRUSH rule 0 x 167 [9,7,3,4]
+  CRUSH rule 0 x 168 [13,2,4,1]
+  CRUSH rule 0 x 169 [1,4,9,14]
+  CRUSH rule 0 x 170 [1,15,7,9]
+  CRUSH rule 0 x 171 [9,2,10,7]
+  CRUSH rule 0 x 172 [14,4,10,12]
+  CRUSH rule 0 x 173 [5,10,12,15]
+  CRUSH rule 0 x 174 [15,6,4,12]
+  CRUSH rule 0 x 175 [5,7,9,3]
+  CRUSH rule 0 x 176 [9,6,3,14]
+  CRUSH rule 0 x 177 [2,9,10,13]
+  CRUSH rule 0 x 178 [12,11,7,14]
+  CRUSH rule 0 x 179 [2,10,13,9]
+  CRUSH rule 0 x 180 [3,11,5,15]
+  CRUSH rule 0 x 181 [9,12,6,5]
+  CRUSH rule 0 x 182 [5,13,11,2]
+  CRUSH rule 0 x 183 [5,7,10,13]
+  CRUSH rule 0 x 184 [2,5,11,12]
+  CRUSH rule 0 x 185 [13,5,7,11]
+  CRUSH rule 0 x 186 [6,14,13,5]
+  CRUSH rule 0 x 187 [1,4,11,13]
+  CRUSH rule 0 x 188 [9,13,5,14]
+  CRUSH rule 0 x 189 [6,12,4,9]
+  CRUSH rule 0 x 190 [9,13,15,10]
+  CRUSH rule 0 x 191 [7,11,4,1]
+  CRUSH rule 0 x 192 [2,11,5,15]
+  CRUSH rule 0 x 193 [3,13,6,10]
+  CRUSH rule 0 x 194 [3,13,4,14]
+  CRUSH rule 0 x 195 [5,7,10,12]
+  CRUSH rule 0 x 196 [4,15,1,10]
+  CRUSH rule 0 x 197 [14,10,13,4]
+  CRUSH rule 0 x 198 [2,5,6,15]
+  CRUSH rule 0 x 199 [2,10,4,15]
+  CRUSH rule 0 x 200 [7,14,11,4]
+  CRUSH rule 0 x 201 [9,14,1,7]
+  CRUSH rule 0 x 202 [14,11,7,3]
+  CRUSH rule 0 x 203 [12,5,7,15]
+  CRUSH rule 0 x 204 [6,11,3,12]
+  CRUSH rule 0 x 205 [15,4,6,10]
+  CRUSH rule 0 x 206 [13,11,2,15]
+  CRUSH rule 0 x 207 [2,11,7,4]
+  CRUSH rule 0 x 208 [13,1,6,14]
+  CRUSH rule 0 x 209 [6,15,13,1]
+  CRUSH rule 0 x 210 [13,11,2,7]
+  CRUSH rule 0 x 211 [2,14,1,13]
+  CRUSH rule 0 x 212 [10,1,12,15]
+  CRUSH rule 0 x 213 [3,9,6,5]
+  CRUSH rule 0 x 214 [7,15,4,1]
+  CRUSH rule 0 x 215 [6,1,4,13]
+  CRUSH rule 0 x 216 [12,9,6,2]
+  CRUSH rule 0 x 217 [12,11,1,14]
+  CRUSH rule 0 x 218 [12,10,15,6]
+  CRUSH rule 0 x 219 [3,11,14,6]
+  CRUSH rule 0 x 220 [14,4,3,12]
+  CRUSH rule 0 x 221 [15,5,2,6]
+  CRUSH rule 0 x 222 [10,4,3,15]
+  CRUSH rule 0 x 223 [9,7,11,1]
+  CRUSH rule 0 x 224 [1,7,10,2]
+  CRUSH rule 0 x 225 [10,5,2,6]
+  CRUSH rule 0 x 226 [4,1,9,3]
+  CRUSH rule 0 x 227 [7,2,12,15]
+  CRUSH rule 0 x 228 [2,15,11,1]
+  CRUSH rule 0 x 229 [9,3,7,14]
+  CRUSH rule 0 x 230 [10,5,7,2]
+  CRUSH rule 0 x 231 [2,7,5,13]
+  CRUSH rule 0 x 232 [10,5,13,1]
+  CRUSH rule 0 x 233 [6,12,11,4]
+  CRUSH rule 0 x 234 [10,1,2,12]
+  CRUSH rule 0 x 235 [13,14,7,10]
+  CRUSH rule 0 x 236 [2,15,9,12]
+  CRUSH rule 0 x 237 [3,12,9,10]
+  CRUSH rule 0 x 238 [2,10,4,15]
+  CRUSH rule 0 x 239 [4,15,10,7]
+  CRUSH rule 0 x 240 [15,5,13,7]
+  CRUSH rule 0 x 241 [7,9,15,12]
+  CRUSH rule 0 x 242 [14,2,6,9]
+  CRUSH rule 0 x 243 [2,11,5,1]
+  CRUSH rule 0 x 244 [13,9,15,3]
+  CRUSH rule 0 x 245 [12,9,15,3]
+  CRUSH rule 0 x 246 [15,3,5,11]
+  CRUSH rule 0 x 247 [6,4,9,12]
+  CRUSH rule 0 x 248 [5,13,7,11]
+  CRUSH rule 0 x 249 [10,14,7,3]
+  CRUSH rule 0 x 250 [12,15,1,10]
+  CRUSH rule 0 x 251 [13,2,15,5]
+  CRUSH rule 0 x 252 [7,5,13,9]
+  CRUSH rule 0 x 253 [3,13,15,10]
+  CRUSH rule 0 x 254 [2,9,13,14]
+  CRUSH rule 0 x 255 [1,9,13,2]
+  CRUSH rule 0 x 256 [6,9,13,1]
+  CRUSH rule 0 x 257 [15,12,3,9]
+  CRUSH rule 0 x 258 [12,5,6,10]
+  CRUSH rule 0 x 259 [9,10,4,3]
+  CRUSH rule 0 x 260 [10,12,6,9]
+  CRUSH rule 0 x 261 [13,7,2,1]
+  CRUSH rule 0 x 262 [15,3,12,7]
+  CRUSH rule 0 x 263 [12,6,10,9]
+  CRUSH rule 0 x 264 [13,14,11,3]
+  CRUSH rule 0 x 265 [12,10,14,5]
+  CRUSH rule 0 x 266 [14,7,11,1]
+  CRUSH rule 0 x 267 [12,11,6,5]
+  CRUSH rule 0 x 268 [4,1,15,12]
+  CRUSH rule 0 x 269 [11,1,15,5]
+  CRUSH rule 0 x 270 [7,11,12,3]
+  CRUSH rule 0 x 271 [4,7,3,13]
+  CRUSH rule 0 x 272 [15,5,13,10]
+  CRUSH rule 0 x 273 [2,10,7,12]
+  CRUSH rule 0 x 274 [10,2,5,6]
+  CRUSH rule 0 x 275 [10,3,4,7]
+  CRUSH rule 0 x 276 [5,12,9,2]
+  CRUSH rule 0 x 277 [14,3,13,4]
+  CRUSH rule 0 x 278 [5,6,14,3]
+  CRUSH rule 0 x 279 [6,10,13,3]
+  CRUSH rule 0 x 280 [7,3,14,9]
+  CRUSH rule 0 x 281 [5,11,14,7]
+  CRUSH rule 0 x 282 [2,1,13,14]
+  CRUSH rule 0 x 283 [4,1,12,3]
+  CRUSH rule 0 x 284 [5,11,7,15]
+  CRUSH rule 0 x 285 [15,5,3,1]
+  CRUSH rule 0 x 286 [10,4,3,6]
+  CRUSH rule 0 x 287 [12,4,9,1]
+  CRUSH rule 0 x 288 [4,12,10,7]
+  CRUSH rule 0 x 289 [2,5,14,9]
+  CRUSH rule 0 x 290 [12,2,5,6]
+  CRUSH rule 0 x 291 [7,11,1,14]
+  CRUSH rule 0 x 292 [4,10,6,3]
+  CRUSH rule 0 x 293 [6,5,11,1]
+  CRUSH rule 0 x 294 [9,12,3,14]
+  CRUSH rule 0 x 295 [6,10,3,14]
+  CRUSH rule 0 x 296 [3,1,13,7]
+  CRUSH rule 0 x 297 [6,13,4,14]
+  CRUSH rule 0 x 298 [14,9,13,1]
+  CRUSH rule 0 x 299 [14,12,11,6]
+  CRUSH rule 0 x 300 [15,7,10,5]
+  CRUSH rule 0 x 301 [9,11,7,1]
+  CRUSH rule 0 x 302 [9,7,1,13]
+  CRUSH rule 0 x 303 [4,13,3,7]
+  CRUSH rule 0 x 304 [6,9,2,11]
+  CRUSH rule 0 x 305 [13,7,5,11]
+  CRUSH rule 0 x 306 [10,12,4,6]
+  CRUSH rule 0 x 307 [11,12,15,5]
+  CRUSH rule 0 x 308 [12,14,10,9]
+  CRUSH rule 0 x 309 [9,3,12,5]
+  CRUSH rule 0 x 310 [3,1,5,10]
+  CRUSH rule 0 x 311 [3,9,7,1]
+  CRUSH rule 0 x 312 [15,13,9,7]
+  CRUSH rule 0 x 313 [9,15,3,7]
+  CRUSH rule 0 x 314 [2,15,9,5]
+  CRUSH rule 0 x 315 [15,2,13,1]
+  CRUSH rule 0 x 316 [4,9,11,2]
+  CRUSH rule 0 x 317 [1,5,3,13]
+  CRUSH rule 0 x 318 [4,1,15,11]
+  CRUSH rule 0 x 319 [2,15,4,1]
+  CRUSH rule 0 x 320 [5,7,13,9]
+  CRUSH rule 0 x 321 [1,6,11,15]
+  CRUSH rule 0 x 322 [13,7,5,3]
+  CRUSH rule 0 x 323 [7,4,10,1]
+  CRUSH rule 0 x 324 [5,6,10,15]
+  CRUSH rule 0 x 325 [9,10,14,5]
+  CRUSH rule 0 x 326 [11,7,13,4]
+  CRUSH rule 0 x 327 [12,5,10,14]
+  CRUSH rule 0 x 328 [5,2,6,14]
+  CRUSH rule 0 x 329 [2,6,15,5]
+  CRUSH rule 0 x 330 [3,9,11,13]
+  CRUSH rule 0 x 331 [12,14,6,3]
+  CRUSH rule 0 x 332 [10,12,6,15]
+  CRUSH rule 0 x 333 [6,5,3,12]
+  CRUSH rule 0 x 334 [4,9,2,12]
+  CRUSH rule 0 x 335 [11,7,1,5]
+  CRUSH rule 0 x 336 [6,14,13,2]
+  CRUSH rule 0 x 337 [15,11,3,7]
+  CRUSH rule 0 x 338 [10,5,3,6]
+  CRUSH rule 0 x 339 [11,14,13,5]
+  CRUSH rule 0 x 340 [11,6,12,4]
+  CRUSH rule 0 x 341 [7,5,2,10]
+  CRUSH rule 0 x 342 [12,14,1,9]
+  CRUSH rule 0 x 343 [12,14,9,6]
+  CRUSH rule 0 x 344 [9,11,5,2]
+  CRUSH rule 0 x 345 [14,2,11,9]
+  CRUSH rule 0 x 346 [5,3,14,10]
+  CRUSH rule 0 x 347 [10,2,12,6]
+  CRUSH rule 0 x 348 [7,9,10,1]
+  CRUSH rule 0 x 349 [9,6,10,12]
+  CRUSH rule 0 x 350 [13,9,15,4]
+  CRUSH rule 0 x 351 [13,5,15,3]
+  CRUSH rule 0 x 352 [1,12,11,9]
+  CRUSH rule 0 x 353 [10,14,12,2]
+  CRUSH rule 0 x 354 [6,3,15,10]
+  CRUSH rule 0 x 355 [13,14,6,10]
+  CRUSH rule 0 x 356 [15,13,2,9]
+  CRUSH rule 0 x 357 [4,11,1,13]
+  CRUSH rule 0 x 358 [12,7,2,9]
+  CRUSH rule 0 x 359 [5,15,7,11]
+  CRUSH rule 0 x 360 [13,10,1,2]
+  CRUSH rule 0 x 361 [5,3,13,6]
+  CRUSH rule 0 x 362 [2,9,11,13]
+  CRUSH rule 0 x 363 [7,12,3,9]
+  CRUSH rule 0 x 364 [2,12,6,9]
+  CRUSH rule 0 x 365 [13,5,11,15]
+  CRUSH rule 0 x 366 [12,7,3,14]
+  CRUSH rule 0 x 367 [7,13,3,1]
+  CRUSH rule 0 x 368 [7,9,10,15]
+  CRUSH rule 0 x 369 [7,5,3,13]
+  CRUSH rule 0 x 370 [4,7,14,1]
+  CRUSH rule 0 x 371 [1,7,12,3]
+  CRUSH rule 0 x 372 [10,4,3,14]
+  CRUSH rule 0 x 373 [15,5,2,6]
+  CRUSH rule 0 x 374 [3,15,12,5]
+  CRUSH rule 0 x 375 [5,2,14,1]
+  CRUSH rule 0 x 376 [5,14,10,13]
+  CRUSH rule 0 x 377 [1,15,2,4]
+  CRUSH rule 0 x 378 [9,12,2,15]
+  CRUSH rule 0 x 379 [11,2,15,5]
+  CRUSH rule 0 x 380 [6,1,12,11]
+  CRUSH rule 0 x 381 [15,13,7,5]
+  CRUSH rule 0 x 382 [14,3,1,4]
+  CRUSH rule 0 x 383 [3,6,11,4]
+  CRUSH rule 0 x 384 [4,13,6,3]
+  CRUSH rule 0 x 385 [4,6,15,3]
+  CRUSH rule 0 x 386 [14,3,11,13]
+  CRUSH rule 0 x 387 [1,11,5,7]
+  CRUSH rule 0 x 388 [2,6,11,9]
+  CRUSH rule 0 x 389 [12,7,2,4]
+  CRUSH rule 0 x 390 [2,11,13,7]
+  CRUSH rule 0 x 391 [3,4,9,13]
+  CRUSH rule 0 x 392 [11,5,14,7]
+  CRUSH rule 0 x 393 [2,14,5,9]
+  CRUSH rule 0 x 394 [4,9,3,15]
+  CRUSH rule 0 x 395 [10,13,5,15]
+  CRUSH rule 0 x 396 [2,12,15,9]
+  CRUSH rule 0 x 397 [1,14,9,4]
+  CRUSH rule 0 x 398 [9,2,1,5]
+  CRUSH rule 0 x 399 [5,9,14,3]
+  CRUSH rule 0 x 400 [10,6,2,4]
+  CRUSH rule 0 x 401 [6,9,11,12]
+  CRUSH rule 0 x 402 [4,7,9,2]
+  CRUSH rule 0 x 403 [7,15,13,3]
+  CRUSH rule 0 x 404 [14,12,7,9]
+  CRUSH rule 0 x 405 [9,15,11,2]
+  CRUSH rule 0 x 406 [12,14,9,2]
+  CRUSH rule 0 x 407 [9,5,12,10]
+  CRUSH rule 0 x 408 [7,1,5,2]
+  CRUSH rule 0 x 409 [11,2,4,13]
+  CRUSH rule 0 x 410 [6,4,14,2]
+  CRUSH rule 0 x 411 [13,11,15,6]
+  CRUSH rule 0 x 412 [5,9,6,11]
+  CRUSH rule 0 x 413 [13,5,3,11]
+  CRUSH rule 0 x 414 [3,11,9,13]
+  CRUSH rule 0 x 415 [6,10,14,5]
+  CRUSH rule 0 x 416 [13,1,4,7]
+  CRUSH rule 0 x 417 [4,12,1,15]
+  CRUSH rule 0 x 418 [14,5,10,2]
+  CRUSH rule 0 x 419 [5,14,10,9]
+  CRUSH rule 0 x 420 [2,4,9,11]
+  CRUSH rule 0 x 421 [15,4,10,3]
+  CRUSH rule 0 x 422 [4,11,2,7]
+  CRUSH rule 0 x 423 [3,15,12,6]
+  CRUSH rule 0 x 424 [6,10,12,2]
+  CRUSH rule 0 x 425 [11,15,2,13]
+  CRUSH rule 0 x 426 [12,4,7,1]
+  CRUSH rule 0 x 427 [14,10,3,1]
+  CRUSH rule 0 x 428 [12,7,9,4]
+  CRUSH rule 0 x 429 [3,4,9,7]
+  CRUSH rule 0 x 430 [3,5,10,13]
+  CRUSH rule 0 x 431 [9,3,7,1]
+  CRUSH rule 0 x 432 [4,1,12,7]
+  CRUSH rule 0 x 433 [4,11,12,15]
+  CRUSH rule 0 x 434 [2,14,9,1]
+  CRUSH rule 0 x 435 [13,11,5,6]
+  CRUSH rule 0 x 436 [9,15,10,2]
+  CRUSH rule 0 x 437 [9,6,3,14]
+  CRUSH rule 0 x 438 [7,2,13,4]
+  CRUSH rule 0 x 439 [7,14,4,3]
+  CRUSH rule 0 x 440 [14,11,9,2]
+  CRUSH rule 0 x 441 [2,4,11,9]
+  CRUSH rule 0 x 442 [10,13,9,7]
+  CRUSH rule 0 x 443 [12,15,10,9]
+  CRUSH rule 0 x 444 [4,13,7,14]
+  CRUSH rule 0 x 445 [4,2,15,7]
+  CRUSH rule 0 x 446 [12,10,6,9]
+  CRUSH rule 0 x 447 [15,7,13,1]
+  CRUSH rule 0 x 448 [5,2,13,7]
+  CRUSH rule 0 x 449 [14,5,3,12]
+  CRUSH rule 0 x 450 [2,4,6,9]
+  CRUSH rule 0 x 451 [6,14,11,3]
+  CRUSH rule 0 x 452 [14,9,10,4]
+  CRUSH rule 0 x 453 [5,15,13,2]
+  CRUSH rule 0 x 454 [10,4,2,6]
+  CRUSH rule 0 x 455 [6,13,2,4]
+  CRUSH rule 0 x 456 [5,7,13,1]
+  CRUSH rule 0 x 457 [9,1,5,7]
+  CRUSH rule 0 x 458 [9,11,15,4]
+  CRUSH rule 0 x 459 [13,15,11,1]
+  CRUSH rule 0 x 460 [5,12,10,15]
+  CRUSH rule 0 x 461 [4,3,9,13]
+  CRUSH rule 0 x 462 [4,7,12,14]
+  CRUSH rule 0 x 463 [4,12,14,11]
+  CRUSH rule 0 x 464 [4,2,15,10]
+  CRUSH rule 0 x 465 [5,10,9,7]
+  CRUSH rule 0 x 466 [13,5,2,15]
+  CRUSH rule 0 x 467 [13,6,14,3]
+  CRUSH rule 0 x 468 [10,7,12,14]
+  CRUSH rule 0 x 469 [4,9,6,14]
+  CRUSH rule 0 x 470 [3,9,12,15]
+  CRUSH rule 0 x 471 [6,1,5,14]
+  CRUSH rule 0 x 472 [2,14,7,5]
+  CRUSH rule 0 x 473 [15,10,6,9]
+  CRUSH rule 0 x 474 [15,10,4,12]
+  CRUSH rule 0 x 475 [10,5,12,9]
+  CRUSH rule 0 x 476 [3,6,10,12]
+  CRUSH rule 0 x 477 [6,13,5,15]
+  CRUSH rule 0 x 478 [4,15,1,3]
+  CRUSH rule 0 x 479 [13,11,1,6]
+  CRUSH rule 0 x 480 [1,13,6,4]
+  CRUSH rule 0 x 481 [15,12,7,9]
+  CRUSH rule 0 x 482 [2,12,9,1]
+  CRUSH rule 0 x 483 [10,1,4,15]
+  CRUSH rule 0 x 484 [1,4,10,13]
+  CRUSH rule 0 x 485 [9,4,3,1]
+  CRUSH rule 0 x 486 [3,10,15,9]
+  CRUSH rule 0 x 487 [12,11,4,14]
+  CRUSH rule 0 x 488 [14,4,1,9]
+  CRUSH rule 0 x 489 [11,4,2,13]
+  CRUSH rule 0 x 490 [4,9,1,3]
+  CRUSH rule 0 x 491 [1,12,5,2]
+  CRUSH rule 0 x 492 [5,7,11,3]
+  CRUSH rule 0 x 493 [12,1,4,15]
+  CRUSH rule 0 x 494 [1,7,13,4]
+  CRUSH rule 0 x 495 [3,15,7,1]
+  CRUSH rule 0 x 496 [5,3,7,13]
+  CRUSH rule 0 x 497 [13,10,3,6]
+  CRUSH rule 0 x 498 [10,6,1,5]
+  CRUSH rule 0 x 499 [14,3,12,5]
+  CRUSH rule 0 x 500 [15,9,6,12]
+  CRUSH rule 0 x 501 [10,13,1,9]
+  CRUSH rule 0 x 502 [5,1,14,11]
+  CRUSH rule 0 x 503 [15,10,7,9]
+  CRUSH rule 0 x 504 [13,2,7,1]
+  CRUSH rule 0 x 505 [12,7,5,2]
+  CRUSH rule 0 x 506 [11,7,9,14]
+  CRUSH rule 0 x 507 [4,14,13,3]
+  CRUSH rule 0 x 508 [12,1,4,9]
+  CRUSH rule 0 x 509 [4,2,6,9]
+  CRUSH rule 0 x 510 [5,3,1,12]
+  CRUSH rule 0 x 511 [2,12,10,6]
+  CRUSH rule 0 x 512 [15,11,3,5]
+  CRUSH rule 0 x 513 [4,9,11,3]
+  CRUSH rule 0 x 514 [11,9,3,4]
+  CRUSH rule 0 x 515 [12,14,6,5]
+  CRUSH rule 0 x 516 [14,11,1,12]
+  CRUSH rule 0 x 517 [11,5,6,13]
+  CRUSH rule 0 x 518 [3,5,7,12]
+  CRUSH rule 0 x 519 [12,14,2,1]
+  CRUSH rule 0 x 520 [12,4,2,10]
+  CRUSH rule 0 x 521 [11,5,9,6]
+  CRUSH rule 0 x 522 [4,12,11,1]
+  CRUSH rule 0 x 523 [3,1,5,9]
+  CRUSH rule 0 x 524 [15,9,3,11]
+  CRUSH rule 0 x 525 [3,15,11,6]
+  CRUSH rule 0 x 526 [10,2,5,13]
+  CRUSH rule 0 x 527 [3,13,4,1]
+  CRUSH rule 0 x 528 [12,7,15,10]
+  CRUSH rule 0 x 529 [6,4,10,12]
+  CRUSH rule 0 x 530 [11,9,12,7]
+  CRUSH rule 0 x 531 [9,15,4,7]
+  CRUSH rule 0 x 532 [5,3,13,7]
+  CRUSH rule 0 x 533 [12,15,1,2]
+  CRUSH rule 0 x 534 [11,9,3,7]
+  CRUSH rule 0 x 535 [11,1,3,5]
+  CRUSH rule 0 x 536 [9,1,14,13]
+  CRUSH rule 0 x 537 [15,5,13,2]
+  CRUSH rule 0 x 538 [13,5,11,2]
+  CRUSH rule 0 x 539 [10,12,6,14]
+  CRUSH rule 0 x 540 [12,15,7,3]
+  CRUSH rule 0 x 541 [2,1,6,11]
+  CRUSH rule 0 x 542 [3,9,15,5]
+  CRUSH rule 0 x 543 [4,10,9,3]
+  CRUSH rule 0 x 544 [3,15,9,11]
+  CRUSH rule 0 x 545 [14,10,7,12]
+  CRUSH rule 0 x 546 [5,15,13,7]
+  CRUSH rule 0 x 547 [5,13,7,9]
+  CRUSH rule 0 x 548 [11,7,12,15]
+  CRUSH rule 0 x 549 [14,1,4,9]
+  CRUSH rule 0 x 550 [9,15,3,13]
+  CRUSH rule 0 x 551 [11,2,15,6]
+  CRUSH rule 0 x 552 [2,11,14,1]
+  CRUSH rule 0 x 553 [11,9,14,6]
+  CRUSH rule 0 x 554 [11,14,6,4]
+  CRUSH rule 0 x 555 [6,5,10,9]
+  CRUSH rule 0 x 556 [15,6,3,13]
+  CRUSH rule 0 x 557 [12,2,5,14]
+  CRUSH rule 0 x 558 [12,1,6,15]
+  CRUSH rule 0 x 559 [2,13,5,10]
+  CRUSH rule 0 x 560 [4,9,12,6]
+  CRUSH rule 0 x 561 [12,7,1,2]
+  CRUSH rule 0 x 562 [7,13,9,14]
+  CRUSH rule 0 x 563 [15,4,3,10]
+  CRUSH rule 0 x 564 [2,13,7,1]
+  CRUSH rule 0 x 565 [3,12,4,1]
+  CRUSH rule 0 x 566 [6,14,4,2]
+  CRUSH rule 0 x 567 [15,4,11,6]
+  CRUSH rule 0 x 568 [4,14,1,6]
+  CRUSH rule 0 x 569 [11,3,15,13]
+  CRUSH rule 0 x 570 [1,10,13,4]
+  CRUSH rule 0 x 571 [10,12,14,9]
+  CRUSH rule 0 x 572 [12,14,3,10]
+  CRUSH rule 0 x 573 [7,15,11,2]
+  CRUSH rule 0 x 574 [11,14,13,1]
+  CRUSH rule 0 x 575 [5,13,15,9]
+  CRUSH rule 0 x 576 [3,15,11,9]
+  CRUSH rule 0 x 577 [13,9,6,15]
+  CRUSH rule 0 x 578 [4,10,1,2]
+  CRUSH rule 0 x 579 [13,1,15,2]
+  CRUSH rule 0 x 580 [3,12,4,1]
+  CRUSH rule 0 x 581 [7,14,12,10]
+  CRUSH rule 0 x 582 [10,5,13,14]
+  CRUSH rule 0 x 583 [4,15,1,9]
+  CRUSH rule 0 x 584 [10,1,5,13]
+  CRUSH rule 0 x 585 [5,3,6,1]
+  CRUSH rule 0 x 586 [7,10,14,12]
+  CRUSH rule 0 x 587 [11,6,9,4]
+  CRUSH rule 0 x 588 [3,12,7,15]
+  CRUSH rule 0 x 589 [9,7,12,1]
+  CRUSH rule 0 x 590 [12,1,3,9]
+  CRUSH rule 0 x 591 [2,6,14,13]
+  CRUSH rule 0 x 592 [15,12,9,7]
+  CRUSH rule 0 x 593 [13,14,5,11]
+  CRUSH rule 0 x 594 [12,14,2,9]
+  CRUSH rule 0 x 595 [12,7,10,3]
+  CRUSH rule 0 x 596 [2,7,12,11]
+  CRUSH rule 0 x 597 [15,1,2,10]
+  CRUSH rule 0 x 598 [11,5,9,14]
+  CRUSH rule 0 x 599 [13,11,1,5]
+  CRUSH rule 0 x 600 [4,12,3,10]
+  CRUSH rule 0 x 601 [13,5,15,2]
+  CRUSH rule 0 x 602 [3,11,7,1]
+  CRUSH rule 0 x 603 [3,1,4,14]
+  CRUSH rule 0 x 604 [14,2,6,1]
+  CRUSH rule 0 x 605 [2,7,12,5]
+  CRUSH rule 0 x 606 [12,15,1,5]
+  CRUSH rule 0 x 607 [3,9,10,14]
+  CRUSH rule 0 x 608 [13,10,1,7]
+  CRUSH rule 0 x 609 [14,3,7,9]
+  CRUSH rule 0 x 610 [7,10,5,1]
+  CRUSH rule 0 x 611 [13,1,5,3]
+  CRUSH rule 0 x 612 [7,1,2,13]
+  CRUSH rule 0 x 613 [10,7,14,9]
+  CRUSH rule 0 x 614 [9,4,15,3]
+  CRUSH rule 0 x 615 [9,4,11,2]
+  CRUSH rule 0 x 616 [10,14,1,5]
+  CRUSH rule 0 x 617 [15,7,2,11]
+  CRUSH rule 0 x 618 [4,2,10,6]
+  CRUSH rule 0 x 619 [15,4,3,9]
+  CRUSH rule 0 x 620 [3,7,11,14]
+  CRUSH rule 0 x 621 [3,6,4,14]
+  CRUSH rule 0 x 622 [10,2,13,5]
+  CRUSH rule 0 x 623 [4,9,14,7]
+  CRUSH rule 0 x 624 [3,9,15,6]
+  CRUSH rule 0 x 625 [11,7,3,5]
+  CRUSH rule 0 x 626 [10,12,2,1]
+  CRUSH rule 0 x 627 [1,12,10,14]
+  CRUSH rule 0 x 628 [15,13,11,4]
+  CRUSH rule 0 x 629 [5,6,15,12]
+  CRUSH rule 0 x 630 [1,4,12,9]
+  CRUSH rule 0 x 631 [5,7,1,15]
+  CRUSH rule 0 x 632 [12,3,11,9]
+  CRUSH rule 0 x 633 [14,4,3,7]
+  CRUSH rule 0 x 634 [6,9,5,3]
+  CRUSH rule 0 x 635 [6,5,2,15]
+  CRUSH rule 0 x 636 [13,6,11,3]
+  CRUSH rule 0 x 637 [3,1,10,6]
+  CRUSH rule 0 x 638 [10,15,3,5]
+  CRUSH rule 0 x 639 [6,9,14,4]
+  CRUSH rule 0 x 640 [9,6,1,11]
+  CRUSH rule 0 x 641 [10,6,5,14]
+  CRUSH rule 0 x 642 [1,15,4,6]
+  CRUSH rule 0 x 643 [3,7,5,1]
+  CRUSH rule 0 x 644 [15,13,6,9]
+  CRUSH rule 0 x 645 [14,2,4,9]
+  CRUSH rule 0 x 646 [5,13,14,1]
+  CRUSH rule 0 x 647 [10,1,9,13]
+  CRUSH rule 0 x 648 [6,5,2,14]
+  CRUSH rule 0 x 649 [3,9,13,11]
+  CRUSH rule 0 x 650 [10,9,4,15]
+  CRUSH rule 0 x 651 [3,9,5,7]
+  CRUSH rule 0 x 652 [15,9,4,6]
+  CRUSH rule 0 x 653 [11,14,1,3]
+  CRUSH rule 0 x 654 [13,6,2,10]
+  CRUSH rule 0 x 655 [6,3,4,15]
+  CRUSH rule 0 x 656 [3,15,1,4]
+  CRUSH rule 0 x 657 [11,15,3,5]
+  CRUSH rule 0 x 658 [7,2,10,12]
+  CRUSH rule 0 x 659 [2,5,14,6]
+  CRUSH rule 0 x 660 [13,14,10,6]
+  CRUSH rule 0 x 661 [7,15,3,12]
+  CRUSH rule 0 x 662 [15,2,12,5]
+  CRUSH rule 0 x 663 [14,9,13,10]
+  CRUSH rule 0 x 664 [6,10,12,4]
+  CRUSH rule 0 x 665 [2,9,12,1]
+  CRUSH rule 0 x 666 [12,3,6,1]
+  CRUSH rule 0 x 667 [1,9,12,10]
+  CRUSH rule 0 x 668 [9,5,1,2]
+  CRUSH rule 0 x 669 [9,7,14,5]
+  CRUSH rule 0 x 670 [6,10,9,13]
+  CRUSH rule 0 x 671 [6,15,5,10]
+  CRUSH rule 0 x 672 [2,9,13,1]
+  CRUSH rule 0 x 673 [7,10,5,9]
+  CRUSH rule 0 x 674 [7,12,10,1]
+  CRUSH rule 0 x 675 [9,5,1,10]
+  CRUSH rule 0 x 676 [10,12,2,1]
+  CRUSH rule 0 x 677 [2,12,1,4]
+  CRUSH rule 0 x 678 [1,2,4,10]
+  CRUSH rule 0 x 679 [5,6,12,15]
+  CRUSH rule 0 x 680 [7,11,3,1]
+  CRUSH rule 0 x 681 [6,4,3,11]
+  CRUSH rule 0 x 682 [6,1,11,15]
+  CRUSH rule 0 x 683 [6,13,2,4]
+  CRUSH rule 0 x 684 [9,11,3,7]
+  CRUSH rule 0 x 685 [5,1,15,7]
+  CRUSH rule 0 x 686 [1,9,11,14]
+  CRUSH rule 0 x 687 [7,13,3,5]
+  CRUSH rule 0 x 688 [11,9,1,14]
+  CRUSH rule 0 x 689 [5,2,9,12]
+  CRUSH rule 0 x 690 [9,7,10,3]
+  CRUSH rule 0 x 691 [11,15,9,5]
+  CRUSH rule 0 x 692 [15,5,1,2]
+  CRUSH rule 0 x 693 [5,6,12,15]
+  CRUSH rule 0 x 694 [4,7,1,10]
+  CRUSH rule 0 x 695 [6,13,14,10]
+  CRUSH rule 0 x 696 [1,2,4,14]
+  CRUSH rule 0 x 697 [13,11,3,6]
+  CRUSH rule 0 x 698 [11,13,4,2]
+  CRUSH rule 0 x 699 [7,14,12,4]
+  CRUSH rule 0 x 700 [12,14,11,9]
+  CRUSH rule 0 x 701 [3,13,1,14]
+  CRUSH rule 0 x 702 [3,12,15,6]
+  CRUSH rule 0 x 703 [15,11,13,3]
+  CRUSH rule 0 x 704 [6,4,2,15]
+  CRUSH rule 0 x 705 [14,6,11,5]
+  CRUSH rule 0 x 706 [1,12,3,6]
+  CRUSH rule 0 x 707 [4,7,14,3]
+  CRUSH rule 0 x 708 [3,10,5,1]
+  CRUSH rule 0 x 709 [11,12,3,7]
+  CRUSH rule 0 x 710 [14,2,11,9]
+  CRUSH rule 0 x 711 [14,3,9,10]
+  CRUSH rule 0 x 712 [12,3,11,15]
+  CRUSH rule 0 x 713 [11,9,3,15]
+  CRUSH rule 0 x 714 [12,1,9,7]
+  CRUSH rule 0 x 715 [6,1,14,4]
+  CRUSH rule 0 x 716 [11,13,9,14]
+  CRUSH rule 0 x 717 [12,4,10,9]
+  CRUSH rule 0 x 718 [7,15,5,2]
+  CRUSH rule 0 x 719 [5,15,13,3]
+  CRUSH rule 0 x 720 [4,13,10,2]
+  CRUSH rule 0 x 721 [11,3,14,9]
+  CRUSH rule 0 x 722 [2,4,6,1]
+  CRUSH rule 0 x 723 [2,1,12,15]
+  CRUSH rule 0 x 724 [7,1,9,10]
+  CRUSH rule 0 x 725 [11,12,7,15]
+  CRUSH rule 0 x 726 [7,14,4,3]
+  CRUSH rule 0 x 727 [2,5,1,11]
+  CRUSH rule 0 x 728 [13,11,4,6]
+  CRUSH rule 0 x 729 [15,11,4,6]
+  CRUSH rule 0 x 730 [3,7,1,13]
+  CRUSH rule 0 x 731 [9,1,6,5]
+  CRUSH rule 0 x 732 [1,2,10,13]
+  CRUSH rule 0 x 733 [11,3,5,6]
+  CRUSH rule 0 x 734 [14,3,11,7]
+  CRUSH rule 0 x 735 [6,9,2,10]
+  CRUSH rule 0 x 736 [3,9,1,11]
+  CRUSH rule 0 x 737 [1,4,2,12]
+  CRUSH rule 0 x 738 [11,15,7,4]
+  CRUSH rule 0 x 739 [11,12,6,2]
+  CRUSH rule 0 x 740 [7,9,10,13]
+  CRUSH rule 0 x 741 [12,11,7,15]
+  CRUSH rule 0 x 742 [9,7,4,11]
+  CRUSH rule 0 x 743 [5,13,9,15]
+  CRUSH rule 0 x 744 [6,2,13,1]
+  CRUSH rule 0 x 745 [3,6,1,4]
+  CRUSH rule 0 x 746 [3,7,9,10]
+  CRUSH rule 0 x 747 [15,11,5,2]
+  CRUSH rule 0 x 748 [6,10,13,2]
+  CRUSH rule 0 x 749 [14,9,10,7]
+  CRUSH rule 0 x 750 [1,14,6,5]
+  CRUSH rule 0 x 751 [15,1,6,9]
+  CRUSH rule 0 x 752 [13,1,7,3]
+  CRUSH rule 0 x 753 [4,11,1,3]
+  CRUSH rule 0 x 754 [14,12,11,4]
+  CRUSH rule 0 x 755 [13,6,1,10]
+  CRUSH rule 0 x 756 [3,4,14,6]
+  CRUSH rule 0 x 757 [10,6,1,4]
+  CRUSH rule 0 x 758 [6,3,4,10]
+  CRUSH rule 0 x 759 [5,7,3,14]
+  CRUSH rule 0 x 760 [1,15,10,12]
+  CRUSH rule 0 x 761 [2,12,1,14]
+  CRUSH rule 0 x 762 [1,4,10,9]
+  CRUSH rule 0 x 763 [4,13,1,14]
+  CRUSH rule 0 x 764 [1,14,6,13]
+  CRUSH rule 0 x 765 [9,15,2,13]
+  CRUSH rule 0 x 766 [11,2,7,15]
+  CRUSH rule 0 x 767 [6,11,4,3]
+  CRUSH rule 0 x 768 [2,12,15,7]
+  CRUSH rule 0 x 769 [15,1,9,2]
+  CRUSH rule 0 x 770 [15,13,4,6]
+  CRUSH rule 0 x 771 [9,2,12,11]
+  CRUSH rule 0 x 772 [4,3,13,11]
+  CRUSH rule 0 x 773 [3,7,4,15]
+  CRUSH rule 0 x 774 [12,6,3,15]
+  CRUSH rule 0 x 775 [5,10,14,2]
+  CRUSH rule 0 x 776 [10,15,3,9]
+  CRUSH rule 0 x 777 [11,13,4,7]
+  CRUSH rule 0 x 778 [13,1,9,11]
+  CRUSH rule 0 x 779 [5,11,1,14]
+  CRUSH rule 0 x 780 [13,9,3,6]
+  CRUSH rule 0 x 781 [5,7,14,3]
+  CRUSH rule 0 x 782 [2,15,9,7]
+  CRUSH rule 0 x 783 [12,7,5,14]
+  CRUSH rule 0 x 784 [14,1,10,13]
+  CRUSH rule 0 x 785 [6,12,1,2]
+  CRUSH rule 0 x 786 [10,5,2,15]
+  CRUSH rule 0 x 787 [1,12,10,2]
+  CRUSH rule 0 x 788 [4,2,9,13]
+  CRUSH rule 0 x 789 [9,2,14,7]
+  CRUSH rule 0 x 790 [15,2,7,4]
+  CRUSH rule 0 x 791 [9,4,7,13]
+  CRUSH rule 0 x 792 [6,4,15,10]
+  CRUSH rule 0 x 793 [15,9,6,2]
+  CRUSH rule 0 x 794 [5,12,2,14]
+  CRUSH rule 0 x 795 [6,14,12,4]
+  CRUSH rule 0 x 796 [11,2,12,6]
+  CRUSH rule 0 x 797 [14,3,7,1]
+  CRUSH rule 0 x 798 [5,11,6,13]
+  CRUSH rule 0 x 799 [2,9,14,4]
+  CRUSH rule 0 x 800 [6,3,4,11]
+  CRUSH rule 0 x 801 [2,5,6,13]
+  CRUSH rule 0 x 802 [1,4,12,7]
+  CRUSH rule 0 x 803 [7,2,4,1]
+  CRUSH rule 0 x 804 [5,14,9,7]
+  CRUSH rule 0 x 805 [13,4,3,1]
+  CRUSH rule 0 x 806 [6,2,13,4]
+  CRUSH rule 0 x 807 [14,2,7,4]
+  CRUSH rule 0 x 808 [2,15,12,7]
+  CRUSH rule 0 x 809 [1,11,7,12]
+  CRUSH rule 0 x 810 [2,5,9,12]
+  CRUSH rule 0 x 811 [15,6,3,10]
+  CRUSH rule 0 x 812 [7,11,2,14]
+  CRUSH rule 0 x 813 [4,10,13,14]
+  CRUSH rule 0 x 814 [13,4,9,3]
+  CRUSH rule 0 x 815 [15,12,9,4]
+  CRUSH rule 0 x 816 [14,10,13,7]
+  CRUSH rule 0 x 817 [10,7,2,15]
+  CRUSH rule 0 x 818 [15,2,11,4]
+  CRUSH rule 0 x 819 [5,12,10,6]
+  CRUSH rule 0 x 820 [3,6,9,12]
+  CRUSH rule 0 x 821 [15,10,9,13]
+  CRUSH rule 0 x 822 [10,13,2,9]
+  CRUSH rule 0 x 823 [2,6,12,10]
+  CRUSH rule 0 x 824 [3,7,9,13]
+  CRUSH rule 0 x 825 [10,5,14,6]
+  CRUSH rule 0 x 826 [5,2,11,15]
+  CRUSH rule 0 x 827 [13,5,1,3]
+  CRUSH rule 0 x 828 [12,6,10,5]
+  CRUSH rule 0 x 829 [13,6,15,10]
+  CRUSH rule 0 x 830 [15,13,2,9]
+  CRUSH rule 0 x 831 [1,4,11,12]
+  CRUSH rule 0 x 832 [14,11,13,2]
+  CRUSH rule 0 x 833 [9,13,3,11]
+  CRUSH rule 0 x 834 [9,7,5,1]
+  CRUSH rule 0 x 835 [14,3,13,6]
+  CRUSH rule 0 x 836 [3,9,10,13]
+  CRUSH rule 0 x 837 [15,12,11,2]
+  CRUSH rule 0 x 838 [12,14,9,2]
+  CRUSH rule 0 x 839 [3,4,6,10]
+  CRUSH rule 0 x 840 [10,15,12,4]
+  CRUSH rule 0 x 841 [3,5,7,12]
+  CRUSH rule 0 x 842 [9,13,2,6]
+  CRUSH rule 0 x 843 [14,7,4,9]
+  CRUSH rule 0 x 844 [7,1,4,15]
+  CRUSH rule 0 x 845 [13,6,1,15]
+  CRUSH rule 0 x 846 [3,7,15,13]
+  CRUSH rule 0 x 847 [12,15,11,5]
+  CRUSH rule 0 x 848 [11,13,1,14]
+  CRUSH rule 0 x 849 [3,15,11,9]
+  CRUSH rule 0 x 850 [1,3,10,6]
+  CRUSH rule 0 x 851 [14,4,3,6]
+  CRUSH rule 0 x 852 [9,12,4,7]
+  CRUSH rule 0 x 853 [13,14,6,11]
+  CRUSH rule 0 x 854 [7,11,12,1]
+  CRUSH rule 0 x 855 [14,4,12,6]
+  CRUSH rule 0 x 856 [5,10,7,3]
+  CRUSH rule 0 x 857 [4,3,13,11]
+  CRUSH rule 0 x 858 [5,15,6,3]
+  CRUSH rule 0 x 859 [5,15,6,2]
+  CRUSH rule 0 x 860 [11,14,1,12]
+  CRUSH rule 0 x 861 [13,7,4,10]
+  CRUSH rule 0 x 862 [5,10,9,7]
+  CRUSH rule 0 x 863 [11,6,3,9]
+  CRUSH rule 0 x 864 [6,13,4,2]
+  CRUSH rule 0 x 865 [4,1,14,11]
+  CRUSH rule 0 x 866 [2,13,4,15]
+  CRUSH rule 0 x 867 [12,2,9,10]
+  CRUSH rule 0 x 868 [14,11,7,2]
+  CRUSH rule 0 x 869 [10,13,7,14]
+  CRUSH rule 0 x 870 [14,9,11,4]
+  CRUSH rule 0 x 871 [6,2,1,4]
+  CRUSH rule 0 x 872 [6,1,15,3]
+  CRUSH rule 0 x 873 [2,5,12,10]
+  CRUSH rule 0 x 874 [12,4,7,2]
+  CRUSH rule 0 x 875 [10,6,14,1]
+  CRUSH rule 0 x 876 [14,7,13,3]
+  CRUSH rule 0 x 877 [15,11,13,9]
+  CRUSH rule 0 x 878 [7,14,3,13]
+  CRUSH rule 0 x 879 [12,2,7,4]
+  CRUSH rule 0 x 880 [2,12,10,7]
+  CRUSH rule 0 x 881 [6,3,1,11]
+  CRUSH rule 0 x 882 [11,13,7,1]
+  CRUSH rule 0 x 883 [13,1,3,10]
+  CRUSH rule 0 x 884 [6,15,4,9]
+  CRUSH rule 0 x 885 [14,7,9,4]
+  CRUSH rule 0 x 886 [13,11,4,2]
+  CRUSH rule 0 x 887 [14,4,12,11]
+  CRUSH rule 0 x 888 [10,12,7,15]
+  CRUSH rule 0 x 889 [15,13,4,1]
+  CRUSH rule 0 x 890 [10,12,14,2]
+  CRUSH rule 0 x 891 [9,5,11,6]
+  CRUSH rule 0 x 892 [12,15,2,4]
+  CRUSH rule 0 x 893 [1,3,5,9]
+  CRUSH rule 0 x 894 [7,2,11,13]
+  CRUSH rule 0 x 895 [2,1,11,5]
+  CRUSH rule 0 x 896 [9,1,14,10]
+  CRUSH rule 0 x 897 [7,5,14,3]
+  CRUSH rule 0 x 898 [10,6,12,9]
+  CRUSH rule 0 x 899 [1,11,5,3]
+  CRUSH rule 0 x 900 [2,9,10,7]
+  CRUSH rule 0 x 901 [9,12,11,3]
+  CRUSH rule 0 x 902 [4,2,6,15]
+  CRUSH rule 0 x 903 [14,10,3,1]
+  CRUSH rule 0 x 904 [15,12,4,9]
+  CRUSH rule 0 x 905 [12,6,11,3]
+  CRUSH rule 0 x 906 [14,11,12,2]
+  CRUSH rule 0 x 907 [7,12,3,9]
+  CRUSH rule 0 x 908 [2,15,9,6]
+  CRUSH rule 0 x 909 [10,14,1,13]
+  CRUSH rule 0 x 910 [12,7,4,15]
+  CRUSH rule 0 x 911 [11,15,2,4]
+  CRUSH rule 0 x 912 [6,4,14,13]
+  CRUSH rule 0 x 913 [4,6,10,1]
+  CRUSH rule 0 x 914 [4,15,2,10]
+  CRUSH rule 0 x 915 [12,14,1,9]
+  CRUSH rule 0 x 916 [3,1,11,5]
+  CRUSH rule 0 x 917 [1,15,6,5]
+  CRUSH rule 0 x 918 [7,14,11,4]
+  CRUSH rule 0 x 919 [10,7,3,13]
+  CRUSH rule 0 x 920 [4,2,10,15]
+  CRUSH rule 0 x 921 [1,11,6,13]
+  CRUSH rule 0 x 922 [6,4,14,13]
+  CRUSH rule 0 x 923 [12,2,5,14]
+  CRUSH rule 0 x 924 [6,2,14,13]
+  CRUSH rule 0 x 925 [12,15,2,10]
+  CRUSH rule 0 x 926 [3,13,10,1]
+  CRUSH rule 0 x 927 [6,5,1,11]
+  CRUSH rule 0 x 928 [13,1,3,9]
+  CRUSH rule 0 x 929 [10,7,1,5]
+  CRUSH rule 0 x 930 [7,15,10,5]
+  CRUSH rule 0 x 931 [6,15,11,9]
+  CRUSH rule 0 x 932 [13,2,5,11]
+  CRUSH rule 0 x 933 [12,7,14,10]
+  CRUSH rule 0 x 934 [12,2,5,7]
+  CRUSH rule 0 x 935 [6,11,1,14]
+  CRUSH rule 0 x 936 [9,12,7,5]
+  CRUSH rule 0 x 937 [14,2,11,1]
+  CRUSH rule 0 x 938 [14,3,5,11]
+  CRUSH rule 0 x 939 [6,4,14,9]
+  CRUSH rule 0 x 940 [13,11,4,2]
+  CRUSH rule 0 x 941 [3,12,4,7]
+  CRUSH rule 0 x 942 [15,12,10,4]
+  CRUSH rule 0 x 943 [10,2,4,9]
+  CRUSH rule 0 x 944 [2,9,4,7]
+  CRUSH rule 0 x 945 [10,15,2,9]
+  CRUSH rule 0 x 946 [11,15,7,12]
+  CRUSH rule 0 x 947 [11,3,14,1]
+  CRUSH rule 0 x 948 [7,13,11,5]
+  CRUSH rule 0 x 949 [9,1,12,5]
+  CRUSH rule 0 x 950 [9,15,13,6]
+  CRUSH rule 0 x 951 [2,6,12,9]
+  CRUSH rule 0 x 952 [9,7,15,3]
+  CRUSH rule 0 x 953 [1,3,6,10]
+  CRUSH rule 0 x 954 [10,2,14,9]
+  CRUSH rule 0 x 955 [7,14,3,1]
+  CRUSH rule 0 x 956 [1,6,11,5]
+  CRUSH rule 0 x 957 [14,11,1,12]
+  CRUSH rule 0 x 958 [15,4,3,11]
+  CRUSH rule 0 x 959 [2,1,12,15]
+  CRUSH rule 0 x 960 [2,6,11,13]
+  CRUSH rule 0 x 961 [3,13,11,9]
+  CRUSH rule 0 x 962 [5,11,3,14]
+  CRUSH rule 0 x 963 [13,10,15,4]
+  CRUSH rule 0 x 964 [7,11,4,9]
+  CRUSH rule 0 x 965 [12,2,9,7]
+  CRUSH rule 0 x 966 [12,14,9,4]
+  CRUSH rule 0 x 967 [7,5,3,10]
+  CRUSH rule 0 x 968 [12,15,4,9]
+  CRUSH rule 0 x 969 [11,4,7,1]
+  CRUSH rule 0 x 970 [5,12,10,1]
+  CRUSH rule 0 x 971 [1,9,4,12]
+  CRUSH rule 0 x 972 [12,3,14,5]
+  CRUSH rule 0 x 973 [1,10,4,12]
+  CRUSH rule 0 x 974 [7,11,1,2]
+  CRUSH rule 0 x 975 [7,9,15,12]
+  CRUSH rule 0 x 976 [7,3,15,5]
+  CRUSH rule 0 x 977 [14,3,6,10]
+  CRUSH rule 0 x 978 [12,5,11,1]
+  CRUSH rule 0 x 979 [5,1,13,6]
+  CRUSH rule 0 x 980 [15,11,5,6]
+  CRUSH rule 0 x 981 [5,11,15,12]
+  CRUSH rule 0 x 982 [2,6,14,11]
+  CRUSH rule 0 x 983 [3,12,10,9]
+  CRUSH rule 0 x 984 [15,13,1,10]
+  CRUSH rule 0 x 985 [11,2,15,1]
+  CRUSH rule 0 x 986 [6,13,9,1]
+  CRUSH rule 0 x 987 [13,14,5,10]
+  CRUSH rule 0 x 988 [12,9,10,14]
+  CRUSH rule 0 x 989 [7,4,3,15]
+  CRUSH rule 0 x 990 [1,10,9,13]
+  CRUSH rule 0 x 991 [7,11,1,14]
+  CRUSH rule 0 x 992 [9,10,2,13]
+  CRUSH rule 0 x 993 [6,10,14,12]
+  CRUSH rule 0 x 994 [3,13,15,4]
+  CRUSH rule 0 x 995 [15,6,12,2]
+  CRUSH rule 0 x 996 [15,10,5,3]
+  CRUSH rule 0 x 997 [15,2,1,12]
+  CRUSH rule 0 x 998 [6,1,9,5]
+  CRUSH rule 0 x 999 [9,10,15,5]
+  CRUSH rule 0 x 1000 [14,2,9,4]
+  CRUSH rule 0 x 1001 [11,14,4,2]
+  CRUSH rule 0 x 1002 [1,10,14,2]
+  CRUSH rule 0 x 1003 [10,7,5,14]
+  CRUSH rule 0 x 1004 [15,1,4,6]
+  CRUSH rule 0 x 1005 [6,12,2,10]
+  CRUSH rule 0 x 1006 [10,12,15,1]
+  CRUSH rule 0 x 1007 [1,7,13,14]
+  CRUSH rule 0 x 1008 [7,4,9,11]
+  CRUSH rule 0 x 1009 [5,2,11,7]
+  CRUSH rule 0 x 1010 [10,2,15,6]
+  CRUSH rule 0 x 1011 [6,3,12,1]
+  CRUSH rule 0 x 1012 [12,6,9,15]
+  CRUSH rule 0 x 1013 [2,14,12,4]
+  CRUSH rule 0 x 1014 [1,13,7,2]
+  CRUSH rule 0 x 1015 [12,6,10,1]
+  CRUSH rule 0 x 1016 [10,13,14,3]
+  CRUSH rule 0 x 1017 [5,11,14,7]
+  CRUSH rule 0 x 1018 [13,11,14,1]
+  CRUSH rule 0 x 1019 [10,13,14,7]
+  CRUSH rule 0 x 1020 [3,1,13,4]
+  CRUSH rule 0 x 1021 [2,11,14,9]
+  CRUSH rule 0 x 1022 [15,5,7,2]
+  CRUSH rule 0 x 1023 [15,2,9,12]
+  rule 0 (replicated_ruleset) num_rep 4 result size == 4:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3,15,12]
+  CRUSH rule 0 x 1 [10,15,1,2,13]
+  CRUSH rule 0 x 2 [1,12,2,6,5]
+  CRUSH rule 0 x 3 [15,4,10,2,9]
+  CRUSH rule 0 x 4 [14,2,10,1,9]
+  CRUSH rule 0 x 5 [7,4,11,2,13]
+  CRUSH rule 0 x 6 [12,6,10,9,3]
+  CRUSH rule 0 x 7 [9,2,6,12,11]
+  CRUSH rule 0 x 8 [10,2,15,1,4]
+  CRUSH rule 0 x 9 [7,1,14,2,11]
+  CRUSH rule 0 x 10 [10,14,4,1,2]
+  CRUSH rule 0 x 11 [13,9,14,7,5]
+  CRUSH rule 0 x 12 [7,1,2,5,13]
+  CRUSH rule 0 x 13 [3,5,12,7,9]
+  CRUSH rule 0 x 14 [13,5,2,7,10]
+  CRUSH rule 0 x 15 [15,1,9,6,13]
+  CRUSH rule 0 x 16 [7,11,14,2,13]
+  CRUSH rule 0 x 17 [10,1,13,2,4]
+  CRUSH rule 0 x 18 [1,7,3,10,5]
+  CRUSH rule 0 x 19 [7,12,2,4,15]
+  CRUSH rule 0 x 20 [14,12,3,10,9]
+  CRUSH rule 0 x 21 [3,12,1,10,4]
+  CRUSH rule 0 x 22 [6,3,13,11,4]
+  CRUSH rule 0 x 23 [10,5,13,9,3]
+  CRUSH rule 0 x 24 [12,11,3,1,9]
+  CRUSH rule 0 x 25 [7,12,15,1,3]
+  CRUSH rule 0 x 26 [1,7,13,2,14]
+  CRUSH rule 0 x 27 [3,6,15,4,13]
+  CRUSH rule 0 x 28 [14,4,3,9,6]
+  CRUSH rule 0 x 29 [5,14,12,11,6]
+  CRUSH rule 0 x 30 [2,5,6,9,1]
+  CRUSH rule 0 x 31 [5,15,10,1,9]
+  CRUSH rule 0 x 32 [9,10,2,1,13]
+  CRUSH rule 0 x 33 [13,4,9,2,7]
+  CRUSH rule 0 x 34 [13,15,2,4,1]
+  CRUSH rule 0 x 35 [4,14,3,13,10]
+  CRUSH rule 0 x 36 [3,12,9,7,5]
+  CRUSH rule 0 x 37 [9,2,6,14,11]
+  CRUSH rule 0 x 38 [3,4,13,10,9]
+  CRUSH rule 0 x 39 [12,7,14,11,1]
+  CRUSH rule 0 x 40 [10,1,9,5,15]
+  CRUSH rule 0 x 41 [4,9,11,1,14]
+  CRUSH rule 0 x 42 [3,6,14,10,12]
+  CRUSH rule 0 x 43 [10,5,15,7,2]
+  CRUSH rule 0 x 44 [11,4,13,3,7]
+  CRUSH rule 0 x 45 [11,12,15,9,1]
+  CRUSH rule 0 x 46 [6,9,2,14,11]
+  CRUSH rule 0 x 47 [3,9,6,4,13]
+  CRUSH rule 0 x 48 [4,6,2,1,10]
+  CRUSH rule 0 x 49 [9,15,10,7,4]
+  CRUSH rule 0 x 50 [14,12,1,4,2]
+  CRUSH rule 0 x 51 [10,6,5,12,15]
+  CRUSH rule 0 x 52 [12,1,9,11,7]
+  CRUSH rule 0 x 53 [3,6,13,9,5]
+  CRUSH rule 0 x 54 [4,13,9,2,14]
+  CRUSH rule 0 x 55 [4,11,2,7,1]
+  CRUSH rule 0 x 56 [5,9,10,1,3]
+  CRUSH rule 0 x 57 [6,2,1,15,10]
+  CRUSH rule 0 x 58 [7,1,11,4,3]
+  CRUSH rule 0 x 59 [2,13,1,10,9]
+  CRUSH rule 0 x 60 [3,6,11,1,4]
+  CRUSH rule 0 x 61 [3,15,13,7,4]
+  CRUSH rule 0 x 62 [15,11,7,12,5]
+  CRUSH rule 0 x 63 [10,14,12,1,7]
+  CRUSH rule 0 x 64 [3,9,1,4,7]
+  CRUSH rule 0 x 65 [4,12,11,7,14]
+  CRUSH rule 0 x 66 [15,11,6,9,4]
+  CRUSH rule 0 x 67 [2,6,4,14,1]
+  CRUSH rule 0 x 68 [15,7,4,2,9]
+  CRUSH rule 0 x 69 [2,1,15,10,4]
+  CRUSH rule 0 x 70 [9,6,1,3,13]
+  CRUSH rule 0 x 71 [15,5,1,3,13]
+  CRUSH rule 0 x 72 [9,10,3,5,7]
+  CRUSH rule 0 x 73 [5,3,11,1,7]
+  CRUSH rule 0 x 74 [11,7,9,5,1]
+  CRUSH rule 0 x 75 [9,7,11,14,12]
+  CRUSH rule 0 x 76 [6,1,3,5,14]
+  CRUSH rule 0 x 77 [7,4,2,13,9]
+  CRUSH rule 0 x 78 [9,3,1,5,6]
+  CRUSH rule 0 x 79 [13,2,15,5,7]
+  CRUSH rule 0 x 80 [15,2,6,4,13]
+  CRUSH rule 0 x 81 [15,2,1,11,4]
+  CRUSH rule 0 x 82 [14,13,5,11,6]
+  CRUSH rule 0 x 83 [4,15,3,9,10]
+  CRUSH rule 0 x 84 [10,7,9,15,3]
+  CRUSH rule 0 x 85 [3,15,9,7,4]
+  CRUSH rule 0 x 86 [10,9,14,1,13]
+  CRUSH rule 0 x 87 [15,10,7,12,5]
+  CRUSH rule 0 x 88 [4,13,3,1,9]
+  CRUSH rule 0 x 89 [3,9,7,4,1]
+  CRUSH rule 0 x 90 [4,9,7,12,11]
+  CRUSH rule 0 x 91 [6,11,9,1,2]
+  CRUSH rule 0 x 92 [1,5,10,9,13]
+  CRUSH rule 0 x 93 [9,3,15,13,7]
+  CRUSH rule 0 x 94 [9,2,12,5,6]
+  CRUSH rule 0 x 95 [7,15,4,10,9]
+  CRUSH rule 0 x 96 [2,15,11,7,5]
+  CRUSH rule 0 x 97 [4,11,2,13,1]
+  CRUSH rule 0 x 98 [11,13,9,3,15]
+  CRUSH rule 0 x 99 [12,4,11,7,3]
+  CRUSH rule 0 x 100 [9,4,10,15,7]
+  CRUSH rule 0 x 101 [15,7,1,9,10]
+  CRUSH rule 0 x 102 [3,11,14,6,13]
+  CRUSH rule 0 x 103 [13,11,6,14,4]
+  CRUSH rule 0 x 104 [14,6,3,5,9]
+  CRUSH rule 0 x 105 [14,10,1,9,3]
+  CRUSH rule 0 x 106 [6,5,13,2,14]
+  CRUSH rule 0 x 107 [3,1,10,14,13]
+  CRUSH rule 0 x 108 [5,10,7,2,15]
+  CRUSH rule 0 x 109 [9,1,13,7,15]
+  CRUSH rule 0 x 110 [5,1,11,3,7]
+  CRUSH rule 0 x 111 [10,1,9,7,5]
+  CRUSH rule 0 x 112 [1,10,4,14,2]
+  CRUSH rule 0 x 113 [6,10,13,9,1]
+  CRUSH rule 0 x 114 [5,13,6,2,1]
+  CRUSH rule 0 x 115 [10,13,14,3,9]
+  CRUSH rule 0 x 116 [1,14,13,2,11]
+  CRUSH rule 0 x 117 [5,6,1,12,15]
+  CRUSH rule 0 x 118 [10,4,13,15,9]
+  CRUSH rule 0 x 119 [14,12,11,4,6]
+  CRUSH rule 0 x 120 [11,3,14,13,4]
+  CRUSH rule 0 x 121 [9,5,1,11,7]
+  CRUSH rule 0 x 122 [4,3,14,1,11]
+  CRUSH rule 0 x 123 [3,10,5,6,9]
+  CRUSH rule 0 x 124 [12,2,1,5,14]
+  CRUSH rule 0 x 125 [9,12,15,1,6]
+  CRUSH rule 0 x 126 [7,15,10,9,2]
+  CRUSH rule 0 x 127 [4,14,9,13,1]
+  CRUSH rule 0 x 128 [3,12,1,10,4]
+  CRUSH rule 0 x 129 [11,13,14,2,9]
+  CRUSH rule 0 x 130 [3,13,5,14,10]
+  CRUSH rule 0 x 131 [12,1,6,15,4]
+  CRUSH rule 0 x 132 [11,15,13,9,2]
+  CRUSH rule 0 x 133 [3,6,9,11,15]
+  CRUSH rule 0 x 134 [12,5,6,15,3]
+  CRUSH rule 0 x 135 [3,14,12,4,6]
+  CRUSH rule 0 x 136 [15,6,9,4,10]
+  CRUSH rule 0 x 137 [14,3,6,11,1]
+  CRUSH rule 0 x 138 [13,15,4,10,2]
+  CRUSH rule 0 x 139 [11,2,13,9,1]
+  CRUSH rule 0 x 140 [11,4,12,15,2]
+  CRUSH rule 0 x 141 [6,12,15,11,3]
+  CRUSH rule 0 x 142 [3,14,7,9,11]
+  CRUSH rule 0 x 143 [9,6,4,2,14]
+  CRUSH rule 0 x 144 [13,7,11,2,14]
+  CRUSH rule 0 x 145 [12,2,6,10,9]
+  CRUSH rule 0 x 146 [1,5,9,2,6]
+  CRUSH rule 0 x 147 [1,4,9,11,2]
+  CRUSH rule 0 x 148 [12,7,9,2,14]
+  CRUSH rule 0 x 149 [2,5,9,12,11]
+  CRUSH rule 0 x 150 [1,15,2,10,7]
+  CRUSH rule 0 x 151 [2,9,14,7,1]
+  CRUSH rule 0 x 152 [5,9,2,6,10]
+  CRUSH rule 0 x 153 [6,9,4,15,2]
+  CRUSH rule 0 x 154 [3,11,7,1,4]
+  CRUSH rule 0 x 155 [14,12,7,3,5]
+  CRUSH rule 0 x 156 [7,13,3,10,15]
+  CRUSH rule 0 x 157 [15,1,6,4,3]
+  CRUSH rule 0 x 158 [15,1,10,6,12]
+  CRUSH rule 0 x 159 [4,14,3,12,10]
+  CRUSH rule 0 x 160 [5,7,3,14,11]
+  CRUSH rule 0 x 161 [1,2,11,4,6]
+  CRUSH rule 0 x 162 [10,6,1,12,2]
+  CRUSH rule 0 x 163 [15,1,10,2,6]
+  CRUSH rule 0 x 164 [9,14,10,7,12]
+  CRUSH rule 0 x 165 [11,7,2,13,9]
+  CRUSH rule 0 x 166 [1,2,12,14,4]
+  CRUSH rule 0 x 167 [9,7,3,4,11]
+  CRUSH rule 0 x 168 [13,2,4,1,6]
+  CRUSH rule 0 x 169 [1,4,9,14,13]
+  CRUSH rule 0 x 170 [1,15,7,9,12]
+  CRUSH rule 0 x 171 [9,2,10,7,1]
+  CRUSH rule 0 x 172 [14,4,10,12,9]
+  CRUSH rule 0 x 173 [5,10,12,15,6]
+  CRUSH rule 0 x 174 [15,6,4,12,1]
+  CRUSH rule 0 x 175 [5,7,9,3,10]
+  CRUSH rule 0 x 176 [9,6,3,14,13]
+  CRUSH rule 0 x 177 [2,9,10,13,4]
+  CRUSH rule 0 x 178 [12,11,7,14,3]
+  CRUSH rule 0 x 179 [2,10,13,9,5]
+  CRUSH rule 0 x 180 [3,11,5,15,7]
+  CRUSH rule 0 x 181 [9,12,6,5,1]
+  CRUSH rule 0 x 182 [5,13,11,2,1]
+  CRUSH rule 0 x 183 [5,7,10,13,3]
+  CRUSH rule 0 x 184 [2,5,11,12,7]
+  CRUSH rule 0 x 185 [13,5,7,11,2]
+  CRUSH rule 0 x 186 [6,14,13,5,10]
+  CRUSH rule 0 x 187 [1,4,11,13,6]
+  CRUSH rule 0 x 188 [9,13,5,14,10]
+  CRUSH rule 0 x 189 [6,12,4,9,2]
+  CRUSH rule 0 x 190 [9,13,15,10,3]
+  CRUSH rule 0 x 191 [7,11,4,1,15]
+  CRUSH rule 0 x 192 [2,11,5,15,6]
+  CRUSH rule 0 x 193 [3,13,6,10,4]
+  CRUSH rule 0 x 194 [3,13,4,14,6]
+  CRUSH rule 0 x 195 [5,7,10,12,1]
+  CRUSH rule 0 x 196 [4,15,1,10,9]
+  CRUSH rule 0 x 197 [14,10,13,4,6]
+  CRUSH rule 0 x 198 [2,5,6,15,9]
+  CRUSH rule 0 x 199 [2,10,4,15,1]
+  CRUSH rule 0 x 200 [7,14,11,4,1]
+  CRUSH rule 0 x 201 [9,14,1,7,4]
+  CRUSH rule 0 x 202 [14,11,7,3,5]
+  CRUSH rule 0 x 203 [12,5,7,15,1]
+  CRUSH rule 0 x 204 [6,11,3,12,14]
+  CRUSH rule 0 x 205 [15,4,6,10,13]
+  CRUSH rule 0 x 206 [13,11,2,15,7]
+  CRUSH rule 0 x 207 [2,11,7,4,14]
+  CRUSH rule 0 x 208 [13,1,6,14,9]
+  CRUSH rule 0 x 209 [6,15,13,1,11]
+  CRUSH rule 0 x 210 [13,11,2,7,5]
+  CRUSH rule 0 x 211 [2,14,1,13,11]
+  CRUSH rule 0 x 212 [10,1,12,15,5]
+  CRUSH rule 0 x 213 [3,9,6,5,15]
+  CRUSH rule 0 x 214 [7,15,4,1,10]
+  CRUSH rule 0 x 215 [6,1,4,13,3]
+  CRUSH rule 0 x 216 [12,9,6,2,1]
+  CRUSH rule 0 x 217 [12,11,1,14,2]
+  CRUSH rule 0 x 218 [12,10,15,6,1]
+  CRUSH rule 0 x 219 [3,11,14,6,4]
+  CRUSH rule 0 x 220 [14,4,3,12,10]
+  CRUSH rule 0 x 221 [15,5,2,6,12]
+  CRUSH rule 0 x 222 [10,4,3,15,7]
+  CRUSH rule 0 x 223 [9,7,11,1,4]
+  CRUSH rule 0 x 224 [1,7,10,2,12]
+  CRUSH rule 0 x 225 [10,5,2,6,1]
+  CRUSH rule 0 x 226 [4,1,9,3,13]
+  CRUSH rule 0 x 227 [7,2,12,15,5]
+  CRUSH rule 0 x 228 [2,15,11,1,6]
+  CRUSH rule 0 x 229 [9,3,7,14,1]
+  CRUSH rule 0 x 230 [10,5,7,2,15]
+  CRUSH rule 0 x 231 [2,7,5,13,9]
+  CRUSH rule 0 x 232 [10,5,13,1,9]
+  CRUSH rule 0 x 233 [6,12,11,4,9]
+  CRUSH rule 0 x 234 [10,1,2,12,5]
+  CRUSH rule 0 x 235 [13,14,7,10,1]
+  CRUSH rule 0 x 236 [2,15,9,12,1]
+  CRUSH rule 0 x 237 [3,12,9,10,4]
+  CRUSH rule 0 x 238 [2,10,4,15,6]
+  CRUSH rule 0 x 239 [4,15,10,7,9]
+  CRUSH rule 0 x 240 [15,5,13,7,2]
+  CRUSH rule 0 x 241 [7,9,15,12,1]
+  CRUSH rule 0 x 242 [14,2,6,9,10]
+  CRUSH rule 0 x 243 [2,11,5,1,15]
+  CRUSH rule 0 x 244 [13,9,15,3,11]
+  CRUSH rule 0 x 245 [12,9,15,3,1]
+  CRUSH rule 0 x 246 [15,3,5,11,7]
+  CRUSH rule 0 x 247 [6,4,9,12,1]
+  CRUSH rule 0 x 248 [5,13,7,11,9]
+  CRUSH rule 0 x 249 [10,14,7,3,9]
+  CRUSH rule 0 x 250 [12,15,1,10,5]
+  CRUSH rule 0 x 251 [13,2,15,5,6]
+  CRUSH rule 0 x 252 [7,5,13,9,3]
+  CRUSH rule 0 x 253 [3,13,15,10,7]
+  CRUSH rule 0 x 254 [2,9,13,14,4]
+  CRUSH rule 0 x 255 [1,9,13,2,6]
+  CRUSH rule 0 x 256 [6,9,13,1,3]
+  CRUSH rule 0 x 257 [15,12,3,9,6]
+  CRUSH rule 0 x 258 [12,5,6,10,2]
+  CRUSH rule 0 x 259 [9,10,4,3,14]
+  CRUSH rule 0 x 260 [10,12,6,9,3]
+  CRUSH rule 0 x 261 [13,7,2,1,15]
+  CRUSH rule 0 x 262 [15,3,12,7,4]
+  CRUSH rule 0 x 263 [12,6,10,9,5]
+  CRUSH rule 0 x 264 [13,14,11,3,1]
+  CRUSH rule 0 x 265 [12,10,14,5,7]
+  CRUSH rule 0 x 266 [14,7,11,1,2]
+  CRUSH rule 0 x 267 [12,11,6,5,1]
+  CRUSH rule 0 x 268 [4,1,15,12,6]
+  CRUSH rule 0 x 269 [11,1,15,5,13]
+  CRUSH rule 0 x 270 [7,11,12,3,1]
+  CRUSH rule 0 x 271 [4,7,3,13,15]
+  CRUSH rule 0 x 272 [15,5,13,10,6]
+  CRUSH rule 0 x 273 [2,10,7,12,1]
+  CRUSH rule 0 x 274 [10,2,5,6,13]
+  CRUSH rule 0 x 275 [10,3,4,7,14]
+  CRUSH rule 0 x 276 [5,12,9,2,11]
+  CRUSH rule 0 x 277 [14,3,13,4,1]
+  CRUSH rule 0 x 278 [5,6,14,3,1]
+  CRUSH rule 0 x 279 [6,10,13,3,9]
+  CRUSH rule 0 x 280 [7,3,14,9,1]
+  CRUSH rule 0 x 281 [5,11,14,7,9]
+  CRUSH rule 0 x 282 [2,1,13,14,9]
+  CRUSH rule 0 x 283 [4,1,12,3,10]
+  CRUSH rule 0 x 284 [5,11,7,15,3]
+  CRUSH rule 0 x 285 [15,5,3,1,6]
+  CRUSH rule 0 x 286 [10,4,3,6,12]
+  CRUSH rule 0 x 287 [12,4,9,1,3]
+  CRUSH rule 0 x 288 [4,12,10,7,1]
+  CRUSH rule 0 x 289 [2,5,14,9,13]
+  CRUSH rule 0 x 290 [12,2,5,6,15]
+  CRUSH rule 0 x 291 [7,11,1,14,5]
+  CRUSH rule 0 x 292 [4,10,6,3,14]
+  CRUSH rule 0 x 293 [6,5,11,1,2]
+  CRUSH rule 0 x 294 [9,12,3,14,6]
+  CRUSH rule 0 x 295 [6,10,3,14,9]
+  CRUSH rule 0 x 296 [3,1,13,7,14]
+  CRUSH rule 0 x 297 [6,13,4,14,10]
+  CRUSH rule 0 x 298 [14,9,13,1,4]
+  CRUSH rule 0 x 299 [14,12,11,6,4]
+  CRUSH rule 0 x 300 [15,7,10,5,1]
+  CRUSH rule 0 x 301 [9,11,7,1,13]
+  CRUSH rule 0 x 302 [9,7,1,13,5]
+  CRUSH rule 0 x 303 [4,13,3,7,10]
+  CRUSH rule 0 x 304 [6,9,2,11,15]
+  CRUSH rule 0 x 305 [13,7,5,11,2]
+  CRUSH rule 0 x 306 [10,12,4,6,9]
+  CRUSH rule 0 x 307 [11,12,15,5,6]
+  CRUSH rule 0 x 308 [12,14,10,9,1]
+  CRUSH rule 0 x 309 [9,3,12,5,11]
+  CRUSH rule 0 x 310 [3,1,5,10,14]
+  CRUSH rule 0 x 311 [3,9,7,1,14]
+  CRUSH rule 0 x 312 [15,13,9,7,5]
+  CRUSH rule 0 x 313 [9,15,3,7,5]
+  CRUSH rule 0 x 314 [2,15,9,5,6]
+  CRUSH rule 0 x 315 [15,2,13,1,11]
+  CRUSH rule 0 x 316 [4,9,11,2,12]
+  CRUSH rule 0 x 317 [1,5,3,13,15]
+  CRUSH rule 0 x 318 [4,1,15,11,9]
+  CRUSH rule 0 x 319 [2,15,4,1,11]
+  CRUSH rule 0 x 320 [5,7,13,9,11]
+  CRUSH rule 0 x 321 [1,6,11,15,5]
+  CRUSH rule 0 x 322 [13,7,5,3,14]
+  CRUSH rule 0 x 323 [7,4,10,1,2]
+  CRUSH rule 0 x 324 [5,6,10,15,2]
+  CRUSH rule 0 x 325 [9,10,14,5,1]
+  CRUSH rule 0 x 326 [11,7,13,4,2]
+  CRUSH rule 0 x 327 [12,5,10,14,3]
+  CRUSH rule 0 x 328 [5,2,6,14,1]
+  CRUSH rule 0 x 329 [2,6,15,5,9]
+  CRUSH rule 0 x 330 [3,9,11,13,1]
+  CRUSH rule 0 x 331 [12,14,6,3,1]
+  CRUSH rule 0 x 332 [10,12,6,15,9]
+  CRUSH rule 0 x 333 [6,5,3,12,14]
+  CRUSH rule 0 x 334 [4,9,2,12,7]
+  CRUSH rule 0 x 335 [11,7,1,5,13]
+  CRUSH rule 0 x 336 [6,14,13,2,5]
+  CRUSH rule 0 x 337 [15,11,3,7,12]
+  CRUSH rule 0 x 338 [10,5,3,6,15]
+  CRUSH rule 0 x 339 [11,14,13,5,3]
+  CRUSH rule 0 x 340 [11,6,12,4,9]
+  CRUSH rule 0 x 341 [7,5,2,10,14]
+  CRUSH rule 0 x 342 [12,14,1,9,2]
+  CRUSH rule 0 x 343 [12,14,9,6,10]
+  CRUSH rule 0 x 344 [9,11,5,2,14]
+  CRUSH rule 0 x 345 [14,2,11,9,6]
+  CRUSH rule 0 x 346 [5,3,14,10,7]
+  CRUSH rule 0 x 347 [10,2,12,6,9]
+  CRUSH rule 0 x 348 [7,9,10,1,14]
+  CRUSH rule 0 x 349 [9,6,10,12,1]
+  CRUSH rule 0 x 350 [13,9,15,4,10]
+  CRUSH rule 0 x 351 [13,5,15,3,1]
+  CRUSH rule 0 x 352 [1,12,11,9,4]
+  CRUSH rule 0 x 353 [10,14,12,2,9]
+  CRUSH rule 0 x 354 [6,3,15,10,9]
+  CRUSH rule 0 x 355 [13,14,6,10,2]
+  CRUSH rule 0 x 356 [15,13,2,9,6]
+  CRUSH rule 0 x 357 [4,11,1,13,3]
+  CRUSH rule 0 x 358 [12,7,2,9,1]
+  CRUSH rule 0 x 359 [5,15,7,11,3]
+  CRUSH rule 0 x 360 [13,10,1,2,6]
+  CRUSH rule 0 x 361 [5,3,13,6,1]
+  CRUSH rule 0 x 362 [2,9,11,13,1]
+  CRUSH rule 0 x 363 [7,12,3,9,15]
+  CRUSH rule 0 x 364 [2,12,6,9,5]
+  CRUSH rule 0 x 365 [13,5,11,15,6]
+  CRUSH rule 0 x 366 [12,7,3,14,5]
+  CRUSH rule 0 x 367 [7,13,3,1,5]
+  CRUSH rule 0 x 368 [7,9,10,15,3]
+  CRUSH rule 0 x 369 [7,5,3,13,14]
+  CRUSH rule 0 x 370 [4,7,14,1,2]
+  CRUSH rule 0 x 371 [1,7,12,3,4]
+  CRUSH rule 0 x 372 [10,4,3,14,6]
+  CRUSH rule 0 x 373 [15,5,2,6,13]
+  CRUSH rule 0 x 374 [3,15,12,5,1]
+  CRUSH rule 0 x 375 [5,2,14,1,6]
+  CRUSH rule 0 x 376 [5,14,10,13,3]
+  CRUSH rule 0 x 377 [1,15,2,4,9]
+  CRUSH rule 0 x 378 [9,12,2,15,1]
+  CRUSH rule 0 x 379 [11,2,15,5,7]
+  CRUSH rule 0 x 380 [6,1,12,11,2]
+  CRUSH rule 0 x 381 [15,13,7,5,10]
+  CRUSH rule 0 x 382 [14,3,1,4,13]
+  CRUSH rule 0 x 383 [3,6,11,4,13]
+  CRUSH rule 0 x 384 [4,13,6,3,15]
+  CRUSH rule 0 x 385 [4,6,15,3,10]
+  CRUSH rule 0 x 386 [14,3,11,13,5]
+  CRUSH rule 0 x 387 [1,11,5,7,9]
+  CRUSH rule 0 x 388 [2,6,11,9,15]
+  CRUSH rule 0 x 389 [12,7,2,4,15]
+  CRUSH rule 0 x 390 [2,11,13,7,5]
+  CRUSH rule 0 x 391 [3,4,9,13,7]
+  CRUSH rule 0 x 392 [11,5,14,7,1]
+  CRUSH rule 0 x 393 [2,14,5,9,7]
+  CRUSH rule 0 x 394 [4,9,3,15,13]
+  CRUSH rule 0 x 395 [10,13,5,15,6]
+  CRUSH rule 0 x 396 [2,12,15,9,4]
+  CRUSH rule 0 x 397 [1,14,9,4,12]
+  CRUSH rule 0 x 398 [9,2,1,5,12]
+  CRUSH rule 0 x 399 [5,9,14,3,1]
+  CRUSH rule 0 x 400 [10,6,2,4,15]
+  CRUSH rule 0 x 401 [6,9,11,12,4]
+  CRUSH rule 0 x 402 [4,7,9,2,13]
+  CRUSH rule 0 x 403 [7,15,13,3,5]
+  CRUSH rule 0 x 404 [14,12,7,9,2]
+  CRUSH rule 0 x 405 [9,15,11,2,4]
+  CRUSH rule 0 x 406 [12,14,9,2,7]
+  CRUSH rule 0 x 407 [9,5,12,10,15]
+  CRUSH rule 0 x 408 [7,1,5,2,10]
+  CRUSH rule 0 x 409 [11,2,4,13,1]
+  CRUSH rule 0 x 410 [6,4,14,2,12]
+  CRUSH rule 0 x 411 [13,11,15,6,4]
+  CRUSH rule 0 x 412 [5,9,6,11,14]
+  CRUSH rule 0 x 413 [13,5,3,11,6]
+  CRUSH rule 0 x 414 [3,11,9,13,4]
+  CRUSH rule 0 x 415 [6,10,14,5,1]
+  CRUSH rule 0 x 416 [13,1,4,7,2]
+  CRUSH rule 0 x 417 [4,12,1,15,2]
+  CRUSH rule 0 x 418 [14,5,10,2,6]
+  CRUSH rule 0 x 419 [5,14,10,9,2]
+  CRUSH rule 0 x 420 [2,4,9,11,6]
+  CRUSH rule 0 x 421 [15,4,10,3,9]
+  CRUSH rule 0 x 422 [4,11,2,7,13]
+  CRUSH rule 0 x 423 [3,15,12,6,5]
+  CRUSH rule 0 x 424 [6,10,12,2,5]
+  CRUSH rule 0 x 425 [11,15,2,13,5]
+  CRUSH rule 0 x 426 [12,4,7,1,9]
+  CRUSH rule 0 x 427 [14,10,3,1,9]
+  CRUSH rule 0 x 428 [12,7,9,4,2]
+  CRUSH rule 0 x 429 [3,4,9,7,11]
+  CRUSH rule 0 x 430 [3,5,10,13,1]
+  CRUSH rule 0 x 431 [9,3,7,1,12]
+  CRUSH rule 0 x 432 [4,1,12,7,15]
+  CRUSH rule 0 x 433 [4,11,12,15,7]
+  CRUSH rule 0 x 434 [2,14,9,1,5]
+  CRUSH rule 0 x 435 [13,11,5,6,9]
+  CRUSH rule 0 x 436 [9,15,10,2,4]
+  CRUSH rule 0 x 437 [9,6,3,14,10]
+  CRUSH rule 0 x 438 [7,2,13,4,11]
+  CRUSH rule 0 x 439 [7,14,4,3,12]
+  CRUSH rule 0 x 440 [14,11,9,2,7]
+  CRUSH rule 0 x 441 [2,4,11,9,13]
+  CRUSH rule 0 x 442 [10,13,9,7,15]
+  CRUSH rule 0 x 443 [12,15,10,9,2]
+  CRUSH rule 0 x 444 [4,13,7,14,3]
+  CRUSH rule 0 x 445 [4,2,15,7,1]
+  CRUSH rule 0 x 446 [12,10,6,9,4]
+  CRUSH rule 0 x 447 [15,7,13,1,4]
+  CRUSH rule 0 x 448 [5,2,13,7,15]
+  CRUSH rule 0 x 449 [14,5,3,12,10]
+  CRUSH rule 0 x 450 [2,4,6,9,15]
+  CRUSH rule 0 x 451 [6,14,11,3,9]
+  CRUSH rule 0 x 452 [14,9,10,4,2]
+  CRUSH rule 0 x 453 [5,15,13,2,6]
+  CRUSH rule 0 x 454 [10,4,2,6,15]
+  CRUSH rule 0 x 455 [6,13,2,4,10]
+  CRUSH rule 0 x 456 [5,7,13,1,11]
+  CRUSH rule 0 x 457 [9,1,5,7,11]
+  CRUSH rule 0 x 458 [9,11,15,4,7]
+  CRUSH rule 0 x 459 [13,15,11,1,5]
+  CRUSH rule 0 x 460 [5,12,10,15,7]
+  CRUSH rule 0 x 461 [4,3,9,13,15]
+  CRUSH rule 0 x 462 [4,7,12,14,11]
+  CRUSH rule 0 x 463 [4,12,14,11,2]
+  CRUSH rule 0 x 464 [4,2,15,10,1]
+  CRUSH rule 0 x 465 [5,10,9,7,13]
+  CRUSH rule 0 x 466 [13,5,2,15,9]
+  CRUSH rule 0 x 467 [13,6,14,3,9]
+  CRUSH rule 0 x 468 [10,7,12,14,4]
+  CRUSH rule 0 x 469 [4,9,6,14,12]
+  CRUSH rule 0 x 470 [3,9,12,15,5]
+  CRUSH rule 0 x 471 [6,1,5,14,13]
+  CRUSH rule 0 x 472 [2,14,7,5,13]
+  CRUSH rule 0 x 473 [15,10,6,9,4]
+  CRUSH rule 0 x 474 [15,10,4,12,6]
+  CRUSH rule 0 x 475 [10,5,12,9,14]
+  CRUSH rule 0 x 476 [3,6,10,12,1]
+  CRUSH rule 0 x 477 [6,13,5,15,11]
+  CRUSH rule 0 x 478 [4,15,1,3,7]
+  CRUSH rule 0 x 479 [13,11,1,6,14]
+  CRUSH rule 0 x 480 [1,13,6,4,9]
+  CRUSH rule 0 x 481 [15,12,7,9,1]
+  CRUSH rule 0 x 482 [2,12,9,1,7]
+  CRUSH rule 0 x 483 [10,1,4,15,9]
+  CRUSH rule 0 x 484 [1,4,10,13,7]
+  CRUSH rule 0 x 485 [9,4,3,1,14]
+  CRUSH rule 0 x 486 [3,10,15,9,7]
+  CRUSH rule 0 x 487 [12,11,4,14,7]
+  CRUSH rule 0 x 488 [14,4,1,9,2]
+  CRUSH rule 0 x 489 [11,4,2,13,15]
+  CRUSH rule 0 x 490 [4,9,1,3,13]
+  CRUSH rule 0 x 491 [1,12,5,2,14]
+  CRUSH rule 0 x 492 [5,7,11,3,14]
+  CRUSH rule 0 x 493 [12,1,4,15,3]
+  CRUSH rule 0 x 494 [1,7,13,4,15]
+  CRUSH rule 0 x 495 [3,15,7,1,9]
+  CRUSH rule 0 x 496 [5,3,7,13,9]
+  CRUSH rule 0 x 497 [13,10,3,6,5]
+  CRUSH rule 0 x 498 [10,6,1,5,9]
+  CRUSH rule 0 x 499 [14,3,12,5,1]
+  CRUSH rule 0 x 500 [15,9,6,12,11]
+  CRUSH rule 0 x 501 [10,13,1,9,3]
+  CRUSH rule 0 x 502 [5,1,14,11,7]
+  CRUSH rule 0 x 503 [15,10,7,9,1]
+  CRUSH rule 0 x 504 [13,2,7,1,14]
+  CRUSH rule 0 x 505 [12,7,5,2,14]
+  CRUSH rule 0 x 506 [11,7,9,14,12]
+  CRUSH rule 0 x 507 [4,14,13,3,9]
+  CRUSH rule 0 x 508 [12,1,4,9,2]
+  CRUSH rule 0 x 509 [4,2,6,9,14]
+  CRUSH rule 0 x 510 [5,3,1,12,11]
+  CRUSH rule 0 x 511 [2,12,10,6,14]
+  CRUSH rule 0 x 512 [15,11,3,5,7]
+  CRUSH rule 0 x 513 [4,9,11,3,13]
+  CRUSH rule 0 x 514 [11,9,3,4,12]
+  CRUSH rule 0 x 515 [12,14,6,5,3]
+  CRUSH rule 0 x 516 [14,11,1,12,3]
+  CRUSH rule 0 x 517 [11,5,6,13,9]
+  CRUSH rule 0 x 518 [3,5,7,12,15]
+  CRUSH rule 0 x 519 [12,14,2,1,4]
+  CRUSH rule 0 x 520 [12,4,2,10,6]
+  CRUSH rule 0 x 521 [11,5,9,6,15]
+  CRUSH rule 0 x 522 [4,12,11,1,15]
+  CRUSH rule 0 x 523 [3,1,5,9,15]
+  CRUSH rule 0 x 524 [15,9,3,11,13]
+  CRUSH rule 0 x 525 [3,15,11,6,9]
+  CRUSH rule 0 x 526 [10,2,5,13,6]
+  CRUSH rule 0 x 527 [3,13,4,1,9]
+  CRUSH rule 0 x 528 [12,7,15,10,2]
+  CRUSH rule 0 x 529 [6,4,10,12,2]
+  CRUSH rule 0 x 530 [11,9,12,7,5]
+  CRUSH rule 0 x 531 [9,15,4,7,2]
+  CRUSH rule 0 x 532 [5,3,13,7,9]
+  CRUSH rule 0 x 533 [12,15,1,2,7]
+  CRUSH rule 0 x 534 [11,9,3,7,15]
+  CRUSH rule 0 x 535 [11,1,3,5,14]
+  CRUSH rule 0 x 536 [9,1,14,13,4]
+  CRUSH rule 0 x 537 [15,5,13,2,7]
+  CRUSH rule 0 x 538 [13,5,11,2,6]
+  CRUSH rule 0 x 539 [10,12,6,14,1]
+  CRUSH rule 0 x 540 [12,15,7,3,9]
+  CRUSH rule 0 x 541 [2,1,6,11,14]
+  CRUSH rule 0 x 542 [3,9,15,5,11]
+  CRUSH rule 0 x 543 [4,10,9,3,6]
+  CRUSH rule 0 x 544 [3,15,9,11,7]
+  CRUSH rule 0 x 545 [14,10,7,12,4]
+  CRUSH rule 0 x 546 [5,15,13,7,1]
+  CRUSH rule 0 x 547 [5,13,7,9,3]
+  CRUSH rule 0 x 548 [11,7,12,15,4]
+  CRUSH rule 0 x 549 [14,1,4,9,13]
+  CRUSH rule 0 x 550 [9,15,3,13,1]
+  CRUSH rule 0 x 551 [11,2,15,6,13]
+  CRUSH rule 0 x 552 [2,11,14,1,9]
+  CRUSH rule 0 x 553 [11,9,14,6,4]
+  CRUSH rule 0 x 554 [11,14,6,4,13]
+  CRUSH rule 0 x 555 [6,5,10,9,14]
+  CRUSH rule 0 x 556 [15,6,3,13,11]
+  CRUSH rule 0 x 557 [12,2,5,14,10]
+  CRUSH rule 0 x 558 [12,1,6,15,5]
+  CRUSH rule 0 x 559 [2,13,5,10,14]
+  CRUSH rule 0 x 560 [4,9,12,6,3]
+  CRUSH rule 0 x 561 [12,7,1,2,5]
+  CRUSH rule 0 x 562 [7,13,9,14,2]
+  CRUSH rule 0 x 563 [15,4,3,10,13]
+  CRUSH rule 0 x 564 [2,13,7,1,15]
+  CRUSH rule 0 x 565 [3,12,4,1,14]
+  CRUSH rule 0 x 566 [6,14,4,2,13]
+  CRUSH rule 0 x 567 [15,4,11,6,3]
+  CRUSH rule 0 x 568 [4,14,1,6,10]
+  CRUSH rule 0 x 569 [11,3,15,13,5]
+  CRUSH rule 0 x 570 [1,10,13,4,7]
+  CRUSH rule 0 x 571 [10,12,14,9,4]
+  CRUSH rule 0 x 572 [12,14,3,10,6]
+  CRUSH rule 0 x 573 [7,15,11,2,12]
+  CRUSH rule 0 x 574 [11,14,13,1,3]
+  CRUSH rule 0 x 575 [5,13,15,9,6]
+  CRUSH rule 0 x 576 [3,15,11,9,1]
+  CRUSH rule 0 x 577 [13,9,6,15,3]
+  CRUSH rule 0 x 578 [4,10,1,2,7]
+  CRUSH rule 0 x 579 [13,1,15,2,10]
+  CRUSH rule 0 x 580 [3,12,4,1,10]
+  CRUSH rule 0 x 581 [7,14,12,10,1]
+  CRUSH rule 0 x 582 [10,5,13,14,1]
+  CRUSH rule 0 x 583 [4,15,1,9,10]
+  CRUSH rule 0 x 584 [10,1,5,13,6]
+  CRUSH rule 0 x 585 [5,3,6,1,11]
+  CRUSH rule 0 x 586 [7,10,14,12,9]
+  CRUSH rule 0 x 587 [11,6,9,4,1]
+  CRUSH rule 0 x 588 [3,12,7,15,4]
+  CRUSH rule 0 x 589 [9,7,12,1,10]
+  CRUSH rule 0 x 590 [12,1,3,9,10]
+  CRUSH rule 0 x 591 [2,6,14,13,9]
+  CRUSH rule 0 x 592 [15,12,9,7,5]
+  CRUSH rule 0 x 593 [13,14,5,11,9]
+  CRUSH rule 0 x 594 [12,14,2,9,7]
+  CRUSH rule 0 x 595 [12,7,10,3,1]
+  CRUSH rule 0 x 596 [2,7,12,11,1]
+  CRUSH rule 0 x 597 [15,1,2,10,7]
+  CRUSH rule 0 x 598 [11,5,9,14,12]
+  CRUSH rule 0 x 599 [13,11,1,5,6]
+  CRUSH rule 0 x 600 [4,12,3,10,9]
+  CRUSH rule 0 x 601 [13,5,15,2,1]
+  CRUSH rule 0 x 602 [3,11,7,1,13]
+  CRUSH rule 0 x 603 [3,1,4,14,10]
+  CRUSH rule 0 x 604 [14,2,6,1,11]
+  CRUSH rule 0 x 605 [2,7,12,5,14]
+  CRUSH rule 0 x 606 [12,15,1,5,7]
+  CRUSH rule 0 x 607 [3,9,10,14,7]
+  CRUSH rule 0 x 608 [13,10,1,7,9]
+  CRUSH rule 0 x 609 [14,3,7,9,11]
+  CRUSH rule 0 x 610 [7,10,5,1,12]
+  CRUSH rule 0 x 611 [13,1,5,3,10]
+  CRUSH rule 0 x 612 [7,1,2,13,9]
+  CRUSH rule 0 x 613 [10,7,14,9,5]
+  CRUSH rule 0 x 614 [9,4,15,3,1]
+  CRUSH rule 0 x 615 [9,4,11,2,1]
+  CRUSH rule 0 x 616 [10,14,1,5,3]
+  CRUSH rule 0 x 617 [15,7,2,11,12]
+  CRUSH rule 0 x 618 [4,2,10,6,14]
+  CRUSH rule 0 x 619 [15,4,3,9,6]
+  CRUSH rule 0 x 620 [3,7,11,14,13]
+  CRUSH rule 0 x 621 [3,6,4,14,1]
+  CRUSH rule 0 x 622 [10,2,13,5,15]
+  CRUSH rule 0 x 623 [4,9,14,7,3]
+  CRUSH rule 0 x 624 [3,9,15,6,10]
+  CRUSH rule 0 x 625 [11,7,3,5,13]
+  CRUSH rule 0 x 626 [10,12,2,1,9]
+  CRUSH rule 0 x 627 [1,12,10,14,3]
+  CRUSH rule 0 x 628 [15,13,11,4,2]
+  CRUSH rule 0 x 629 [5,6,15,12,1]
+  CRUSH rule 0 x 630 [1,4,12,9,3]
+  CRUSH rule 0 x 631 [5,7,1,15,12]
+  CRUSH rule 0 x 632 [12,3,11,9,6]
+  CRUSH rule 0 x 633 [14,4,3,7,10]
+  CRUSH rule 0 x 634 [6,9,5,3,13]
+  CRUSH rule 0 x 635 [6,5,2,15,9]
+  CRUSH rule 0 x 636 [13,6,11,3,15]
+  CRUSH rule 0 x 637 [3,1,10,6,9]
+  CRUSH rule 0 x 638 [10,15,3,5,13]
+  CRUSH rule 0 x 639 [6,9,14,4,3]
+  CRUSH rule 0 x 640 [9,6,1,11,14]
+  CRUSH rule 0 x 641 [10,6,5,14,1]
+  CRUSH rule 0 x 642 [1,15,4,6,2]
+  CRUSH rule 0 x 643 [3,7,5,1,10]
+  CRUSH rule 0 x 644 [15,13,6,9,3]
+  CRUSH rule 0 x 645 [14,2,4,9,10]
+  CRUSH rule 0 x 646 [5,13,14,1,6]
+  CRUSH rule 0 x 647 [10,1,9,13,6]
+  CRUSH rule 0 x 648 [6,5,2,14,11]
+  CRUSH rule 0 x 649 [3,9,13,11,4]
+  CRUSH rule 0 x 650 [10,9,4,15,12]
+  CRUSH rule 0 x 651 [3,9,5,7,14]
+  CRUSH rule 0 x 652 [15,9,4,6,13]
+  CRUSH rule 0 x 653 [11,14,1,3,6]
+  CRUSH rule 0 x 654 [13,6,2,10,15]
+  CRUSH rule 0 x 655 [6,3,4,15,12]
+  CRUSH rule 0 x 656 [3,15,1,4,6]
+  CRUSH rule 0 x 657 [11,15,3,5,7]
+  CRUSH rule 0 x 658 [7,2,10,12,1]
+  CRUSH rule 0 x 659 [2,5,14,6,10]
+  CRUSH rule 0 x 660 [13,14,10,6,4]
+  CRUSH rule 0 x 661 [7,15,3,12,11]
+  CRUSH rule 0 x 662 [15,2,12,5,1]
+  CRUSH rule 0 x 663 [14,9,13,10,5]
+  CRUSH rule 0 x 664 [6,10,12,4,9]
+  CRUSH rule 0 x 665 [2,9,12,1,7]
+  CRUSH rule 0 x 666 [12,3,6,1,15]
+  CRUSH rule 0 x 667 [1,9,12,10,2]
+  CRUSH rule 0 x 668 [9,5,1,2,6]
+  CRUSH rule 0 x 669 [9,7,14,5,11]
+  CRUSH rule 0 x 670 [6,10,9,13,1]
+  CRUSH rule 0 x 671 [6,15,5,10,13]
+  CRUSH rule 0 x 672 [2,9,13,1,4]
+  CRUSH rule 0 x 673 [7,10,5,9,15]
+  CRUSH rule 0 x 674 [7,12,10,1,14]
+  CRUSH rule 0 x 675 [9,5,1,10,6]
+  CRUSH rule 0 x 676 [10,12,2,1,4]
+  CRUSH rule 0 x 677 [2,12,1,4,10]
+  CRUSH rule 0 x 678 [1,2,4,10,12]
+  CRUSH rule 0 x 679 [5,6,12,15,9]
+  CRUSH rule 0 x 680 [7,11,3,1,15]
+  CRUSH rule 0 x 681 [6,4,3,11,14]
+  CRUSH rule 0 x 682 [6,1,11,15,12]
+  CRUSH rule 0 x 683 [6,13,2,4,9]
+  CRUSH rule 0 x 684 [9,11,3,7,15]
+  CRUSH rule 0 x 685 [5,1,15,7,9]
+  CRUSH rule 0 x 686 [1,9,11,14,6]
+  CRUSH rule 0 x 687 [7,13,3,5,11]
+  CRUSH rule 0 x 688 [11,9,1,14,3]
+  CRUSH rule 0 x 689 [5,2,9,12,1]
+  CRUSH rule 0 x 690 [9,7,10,3,13]
+  CRUSH rule 0 x 691 [11,15,9,5,7]
+  CRUSH rule 0 x 692 [15,5,1,2,9]
+  CRUSH rule 0 x 693 [5,6,12,15,2]
+  CRUSH rule 0 x 694 [4,7,1,10,12]
+  CRUSH rule 0 x 695 [6,13,14,10,9]
+  CRUSH rule 0 x 696 [1,2,4,14,7]
+  CRUSH rule 0 x 697 [13,11,3,6,4]
+  CRUSH rule 0 x 698 [11,13,4,2,6]
+  CRUSH rule 0 x 699 [7,14,12,4,2]
+  CRUSH rule 0 x 700 [12,14,11,9,4]
+  CRUSH rule 0 x 701 [3,13,1,14,4]
+  CRUSH rule 0 x 702 [3,12,15,6,5]
+  CRUSH rule 0 x 703 [15,11,13,3,4]
+  CRUSH rule 0 x 704 [6,4,2,15,11]
+  CRUSH rule 0 x 705 [14,6,11,5,1]
+  CRUSH rule 0 x 706 [1,12,3,6,4]
+  CRUSH rule 0 x 707 [4,7,14,3,10]
+  CRUSH rule 0 x 708 [3,10,5,1,15]
+  CRUSH rule 0 x 709 [11,12,3,7,5]
+  CRUSH rule 0 x 710 [14,2,11,9,5]
+  CRUSH rule 0 x 711 [14,3,9,10,12]
+  CRUSH rule 0 x 712 [12,3,11,15,9]
+  CRUSH rule 0 x 713 [11,9,3,15,13]
+  CRUSH rule 0 x 714 [12,1,9,7,2]
+  CRUSH rule 0 x 715 [6,1,14,4,11]
+  CRUSH rule 0 x 716 [11,13,9,14,5]
+  CRUSH rule 0 x 717 [12,4,10,9,15]
+  CRUSH rule 0 x 718 [7,15,5,2,11]
+  CRUSH rule 0 x 719 [5,15,13,3,1]
+  CRUSH rule 0 x 720 [4,13,10,2,7]
+  CRUSH rule 0 x 721 [11,3,14,9,1]
+  CRUSH rule 0 x 722 [2,4,6,1,9]
+  CRUSH rule 0 x 723 [2,1,12,15,11]
+  CRUSH rule 0 x 724 [7,1,9,10,5]
+  CRUSH rule 0 x 725 [11,12,7,15,4]
+  CRUSH rule 0 x 726 [7,14,4,3,11]
+  CRUSH rule 0 x 727 [2,5,1,11,15]
+  CRUSH rule 0 x 728 [13,11,4,6,15]
+  CRUSH rule 0 x 729 [15,11,4,6,2]
+  CRUSH rule 0 x 730 [3,7,1,13,11]
+  CRUSH rule 0 x 731 [9,1,6,5,2]
+  CRUSH rule 0 x 732 [1,2,10,13,9]
+  CRUSH rule 0 x 733 [11,3,5,6,1]
+  CRUSH rule 0 x 734 [14,3,11,7,12]
+  CRUSH rule 0 x 735 [6,9,2,10,13]
+  CRUSH rule 0 x 736 [3,9,1,11,7]
+  CRUSH rule 0 x 737 [1,4,2,12,9]
+  CRUSH rule 0 x 738 [11,15,7,4,9]
+  CRUSH rule 0 x 739 [11,12,6,2,4]
+  CRUSH rule 0 x 740 [7,9,10,13,1]
+  CRUSH rule 0 x 741 [12,11,7,15,2]
+  CRUSH rule 0 x 742 [9,7,4,11,12]
+  CRUSH rule 0 x 743 [5,13,9,15,10]
+  CRUSH rule 0 x 744 [6,2,13,1,14]
+  CRUSH rule 0 x 745 [3,6,1,4,11]
+  CRUSH rule 0 x 746 [3,7,9,10,14]
+  CRUSH rule 0 x 747 [15,11,5,2,13]
+  CRUSH rule 0 x 748 [6,10,13,2,14]
+  CRUSH rule 0 x 749 [14,9,10,7,5]
+  CRUSH rule 0 x 750 [1,14,6,5,11]
+  CRUSH rule 0 x 751 [15,1,6,9,5]
+  CRUSH rule 0 x 752 [13,1,7,3,11]
+  CRUSH rule 0 x 753 [4,11,1,3,15]
+  CRUSH rule 0 x 754 [14,12,11,4,2]
+  CRUSH rule 0 x 755 [13,6,1,10,4]
+  CRUSH rule 0 x 756 [3,4,14,6,1]
+  CRUSH rule 0 x 757 [10,6,1,4,13]
+  CRUSH rule 0 x 758 [6,3,4,10,15]
+  CRUSH rule 0 x 759 [5,7,3,14,11]
+  CRUSH rule 0 x 760 [1,15,10,12,4]
+  CRUSH rule 0 x 761 [2,12,1,14,5]
+  CRUSH rule 0 x 762 [1,4,10,9,3]
+  CRUSH rule 0 x 763 [4,13,1,14,7]
+  CRUSH rule 0 x 764 [1,14,6,13,9]
+  CRUSH rule 0 x 765 [9,15,2,13,4]
+  CRUSH rule 0 x 766 [11,2,7,15,9]
+  CRUSH rule 0 x 767 [6,11,4,3,12]
+  CRUSH rule 0 x 768 [2,12,15,7,1]
+  CRUSH rule 0 x 769 [15,1,9,2,11]
+  CRUSH rule 0 x 770 [15,13,4,6,3]
+  CRUSH rule 0 x 771 [9,2,12,11,6]
+  CRUSH rule 0 x 772 [4,3,13,11,14]
+  CRUSH rule 0 x 773 [3,7,4,15,1]
+  CRUSH rule 0 x 774 [12,6,3,15,5]
+  CRUSH rule 0 x 775 [5,10,14,2,6]
+  CRUSH rule 0 x 776 [10,15,3,9,6]
+  CRUSH rule 0 x 777 [11,13,4,7,1]
+  CRUSH rule 0 x 778 [13,1,9,11,15]
+  CRUSH rule 0 x 779 [5,11,1,14,2]
+  CRUSH rule 0 x 780 [13,9,3,6,4]
+  CRUSH rule 0 x 781 [5,7,14,3,1]
+  CRUSH rule 0 x 782 [2,15,9,7,11]
+  CRUSH rule 0 x 783 [12,7,5,14,9]
+  CRUSH rule 0 x 784 [14,1,10,13,3]
+  CRUSH rule 0 x 785 [6,12,1,2,4]
+  CRUSH rule 0 x 786 [10,5,2,15,1]
+  CRUSH rule 0 x 787 [1,12,10,2,9]
+  CRUSH rule 0 x 788 [4,2,9,13,6]
+  CRUSH rule 0 x 789 [9,2,14,7,4]
+  CRUSH rule 0 x 790 [15,2,7,4,1]
+  CRUSH rule 0 x 791 [9,4,7,13,14]
+  CRUSH rule 0 x 792 [6,4,15,10,12]
+  CRUSH rule 0 x 793 [15,9,6,2,13]
+  CRUSH rule 0 x 794 [5,12,2,14,9]
+  CRUSH rule 0 x 795 [6,14,12,4,10]
+  CRUSH rule 0 x 796 [11,2,12,6,15]
+  CRUSH rule 0 x 797 [14,3,7,1,5]
+  CRUSH rule 0 x 798 [5,11,6,13,1]
+  CRUSH rule 0 x 799 [2,9,14,4,13]
+  CRUSH rule 0 x 800 [6,3,4,11,15]
+  CRUSH rule 0 x 801 [2,5,6,13,9]
+  CRUSH rule 0 x 802 [1,4,12,7,3]
+  CRUSH rule 0 x 803 [7,2,4,1,11]
+  CRUSH rule 0 x 804 [5,14,9,7,3]
+  CRUSH rule 0 x 805 [13,4,3,1,10]
+  CRUSH rule 0 x 806 [6,2,13,4,15]
+  CRUSH rule 0 x 807 [14,2,7,4,9]
+  CRUSH rule 0 x 808 [2,15,12,7,9]
+  CRUSH rule 0 x 809 [1,11,7,12,4]
+  CRUSH rule 0 x 810 [2,5,9,12,15]
+  CRUSH rule 0 x 811 [15,6,3,10,1]
+  CRUSH rule 0 x 812 [7,11,2,14,9]
+  CRUSH rule 0 x 813 [4,10,13,14,2]
+  CRUSH rule 0 x 814 [13,4,9,3,10]
+  CRUSH rule 0 x 815 [15,12,9,4,10]
+  CRUSH rule 0 x 816 [14,10,13,7,3]
+  CRUSH rule 0 x 817 [10,7,2,15,13]
+  CRUSH rule 0 x 818 [15,2,11,4,1]
+  CRUSH rule 0 x 819 [5,12,10,6,1]
+  CRUSH rule 0 x 820 [3,6,9,12,11]
+  CRUSH rule 0 x 821 [15,10,9,13,3]
+  CRUSH rule 0 x 822 [10,13,2,9,7]
+  CRUSH rule 0 x 823 [2,6,12,10,15]
+  CRUSH rule 0 x 824 [3,7,9,13,15]
+  CRUSH rule 0 x 825 [10,5,14,6,12]
+  CRUSH rule 0 x 826 [5,2,11,15,1]
+  CRUSH rule 0 x 827 [13,5,1,3,7]
+  CRUSH rule 0 x 828 [12,6,10,5,1]
+  CRUSH rule 0 x 829 [13,6,15,10,5]
+  CRUSH rule 0 x 830 [15,13,2,9,7]
+  CRUSH rule 0 x 831 [1,4,11,12,6]
+  CRUSH rule 0 x 832 [14,11,13,2,9]
+  CRUSH rule 0 x 833 [9,13,3,11,7]
+  CRUSH rule 0 x 834 [9,7,5,1,11]
+  CRUSH rule 0 x 835 [14,3,13,6,4]
+  CRUSH rule 0 x 836 [3,9,10,13,1]
+  CRUSH rule 0 x 837 [15,12,11,2,7]
+  CRUSH rule 0 x 838 [12,14,9,2,5]
+  CRUSH rule 0 x 839 [3,4,6,10,15]
+  CRUSH rule 0 x 840 [10,15,12,4,7]
+  CRUSH rule 0 x 841 [3,5,7,12,11]
+  CRUSH rule 0 x 842 [9,13,2,6,5]
+  CRUSH rule 0 x 843 [14,7,4,9,3]
+  CRUSH rule 0 x 844 [7,1,4,15,9]
+  CRUSH rule 0 x 845 [13,6,1,15,4]
+  CRUSH rule 0 x 846 [3,7,15,13,1]
+  CRUSH rule 0 x 847 [12,15,11,5,2]
+  CRUSH rule 0 x 848 [11,13,1,14,5]
+  CRUSH rule 0 x 849 [3,15,11,9,6]
+  CRUSH rule 0 x 850 [1,3,10,6,14]
+  CRUSH rule 0 x 851 [14,4,3,6,11]
+  CRUSH rule 0 x 852 [9,12,4,7,15]
+  CRUSH rule 0 x 853 [13,14,6,11,2]
+  CRUSH rule 0 x 854 [7,11,12,1,4]
+  CRUSH rule 0 x 855 [14,4,12,6,3]
+  CRUSH rule 0 x 856 [5,10,7,3,15]
+  CRUSH rule 0 x 857 [4,3,13,11,9]
+  CRUSH rule 0 x 858 [5,15,6,3,9]
+  CRUSH rule 0 x 859 [5,15,6,2,1]
+  CRUSH rule 0 x 860 [11,14,1,12,6]
+  CRUSH rule 0 x 861 [13,7,4,10,1]
+  CRUSH rule 0 x 862 [5,10,9,7,3]
+  CRUSH rule 0 x 863 [11,6,3,9,4]
+  CRUSH rule 0 x 864 [6,13,4,2,10]
+  CRUSH rule 0 x 865 [4,1,14,11,6]
+  CRUSH rule 0 x 866 [2,13,4,15,9]
+  CRUSH rule 0 x 867 [12,2,9,10,4]
+  CRUSH rule 0 x 868 [14,11,7,2,1]
+  CRUSH rule 0 x 869 [10,13,7,14,3]
+  CRUSH rule 0 x 870 [14,9,11,4,3]
+  CRUSH rule 0 x 871 [6,2,1,4,15]
+  CRUSH rule 0 x 872 [6,1,15,3,10]
+  CRUSH rule 0 x 873 [2,5,12,10,1]
+  CRUSH rule 0 x 874 [12,4,7,2,15]
+  CRUSH rule 0 x 875 [10,6,14,1,12]
+  CRUSH rule 0 x 876 [14,7,13,3,9]
+  CRUSH rule 0 x 877 [15,11,13,9,5]
+  CRUSH rule 0 x 878 [7,14,3,13,9]
+  CRUSH rule 0 x 879 [12,2,7,4,10]
+  CRUSH rule 0 x 880 [2,12,10,7,1]
+  CRUSH rule 0 x 881 [6,3,1,11,4]
+  CRUSH rule 0 x 882 [11,13,7,1,2]
+  CRUSH rule 0 x 883 [13,1,3,10,6]
+  CRUSH rule 0 x 884 [6,15,4,9,3]
+  CRUSH rule 0 x 885 [14,7,9,4,2]
+  CRUSH rule 0 x 886 [13,11,4,2,1]
+  CRUSH rule 0 x 887 [14,4,12,11,2]
+  CRUSH rule 0 x 888 [10,12,7,15,9]
+  CRUSH rule 0 x 889 [15,13,4,1,6]
+  CRUSH rule 0 x 890 [10,12,14,2,9]
+  CRUSH rule 0 x 891 [9,5,11,6,3]
+  CRUSH rule 0 x 892 [12,15,2,4,7]
+  CRUSH rule 0 x 893 [1,3,5,9,6]
+  CRUSH rule 0 x 894 [7,2,11,13,4]
+  CRUSH rule 0 x 895 [2,1,11,5,7]
+  CRUSH rule 0 x 896 [9,1,14,10,4]
+  CRUSH rule 0 x 897 [7,5,14,3,1]
+  CRUSH rule 0 x 898 [10,6,12,9,15]
+  CRUSH rule 0 x 899 [1,11,5,3,13]
+  CRUSH rule 0 x 900 [2,9,10,7,13]
+  CRUSH rule 0 x 901 [9,12,11,3,14]
+  CRUSH rule 0 x 902 [4,2,6,15,12]
+  CRUSH rule 0 x 903 [14,10,3,1,12]
+  CRUSH rule 0 x 904 [15,12,4,9,6]
+  CRUSH rule 0 x 905 [12,6,11,3,9]
+  CRUSH rule 0 x 906 [14,11,12,2,4]
+  CRUSH rule 0 x 907 [7,12,3,9,10]
+  CRUSH rule 0 x 908 [2,15,9,6,10]
+  CRUSH rule 0 x 909 [10,14,1,13,2]
+  CRUSH rule 0 x 910 [12,7,4,15,10]
+  CRUSH rule 0 x 911 [11,15,2,4,9]
+  CRUSH rule 0 x 912 [6,4,14,13,3]
+  CRUSH rule 0 x 913 [4,6,10,1,12]
+  CRUSH rule 0 x 914 [4,15,2,10,1]
+  CRUSH rule 0 x 915 [12,14,1,9,4]
+  CRUSH rule 0 x 916 [3,1,11,5,6]
+  CRUSH rule 0 x 917 [1,15,6,5,10]
+  CRUSH rule 0 x 918 [7,14,11,4,9]
+  CRUSH rule 0 x 919 [10,7,3,13,15]
+  CRUSH rule 0 x 920 [4,2,10,15,1]
+  CRUSH rule 0 x 921 [1,11,6,13,4]
+  CRUSH rule 0 x 922 [6,4,14,13,3]
+  CRUSH rule 0 x 923 [12,2,5,14,10]
+  CRUSH rule 0 x 924 [6,2,14,13,9]
+  CRUSH rule 0 x 925 [12,15,2,10,1]
+  CRUSH rule 0 x 926 [3,13,10,1,14]
+  CRUSH rule 0 x 927 [6,5,1,11,14]
+  CRUSH rule 0 x 928 [13,1,3,9,6]
+  CRUSH rule 0 x 929 [10,7,1,5,2]
+  CRUSH rule 0 x 930 [7,15,10,5,1]
+  CRUSH rule 0 x 931 [6,15,11,9,5]
+  CRUSH rule 0 x 932 [13,2,5,11,9]
+  CRUSH rule 0 x 933 [12,7,14,10,4]
+  CRUSH rule 0 x 934 [12,2,5,7,9]
+  CRUSH rule 0 x 935 [6,11,1,14,5]
+  CRUSH rule 0 x 936 [9,12,7,5,1]
+  CRUSH rule 0 x 937 [14,2,11,1,13]
+  CRUSH rule 0 x 938 [14,3,5,11,7]
+  CRUSH rule 0 x 939 [6,4,14,9,12]
+  CRUSH rule 0 x 940 [13,11,4,2,1]
+  CRUSH rule 0 x 941 [3,12,4,7,14]
+  CRUSH rule 0 x 942 [15,12,10,4,1]
+  CRUSH rule 0 x 943 [10,2,4,9,6]
+  CRUSH rule 0 x 944 [2,9,4,7,1]
+  CRUSH rule 0 x 945 [10,15,2,9,5]
+  CRUSH rule 0 x 946 [11,15,7,12,5]
+  CRUSH rule 0 x 947 [11,3,14,1,12]
+  CRUSH rule 0 x 948 [7,13,11,5,14]
+  CRUSH rule 0 x 949 [9,1,12,5,15]
+  CRUSH rule 0 x 950 [9,15,13,6,4]
+  CRUSH rule 0 x 951 [2,6,12,9,10]
+  CRUSH rule 0 x 952 [9,7,15,3,5]
+  CRUSH rule 0 x 953 [1,3,6,10,12]
+  CRUSH rule 0 x 954 [10,2,14,9,4]
+  CRUSH rule 0 x 955 [7,14,3,1,10]
+  CRUSH rule 0 x 956 [1,6,11,5,14]
+  CRUSH rule 0 x 957 [14,11,1,12,6]
+  CRUSH rule 0 x 958 [15,4,3,11,1]
+  CRUSH rule 0 x 959 [2,1,12,15,10]
+  CRUSH rule 0 x 960 [2,6,11,13,15]
+  CRUSH rule 0 x 961 [3,13,11,9,6]
+  CRUSH rule 0 x 962 [5,11,3,14,1]
+  CRUSH rule 0 x 963 [13,10,15,4,6]
+  CRUSH rule 0 x 964 [7,11,4,9,2]
+  CRUSH rule 0 x 965 [12,2,9,7,4]
+  CRUSH rule 0 x 966 [12,14,9,4,1]
+  CRUSH rule 0 x 967 [7,5,3,10,12]
+  CRUSH rule 0 x 968 [12,15,4,9,11]
+  CRUSH rule 0 x 969 [11,4,7,1,9]
+  CRUSH rule 0 x 970 [5,12,10,1,3]
+  CRUSH rule 0 x 971 [1,9,4,12,7]
+  CRUSH rule 0 x 972 [12,3,14,5,1]
+  CRUSH rule 0 x 973 [1,10,4,12,2]
+  CRUSH rule 0 x 974 [7,11,1,2,15]
+  CRUSH rule 0 x 975 [7,9,15,12,2]
+  CRUSH rule 0 x 976 [7,3,15,5,12]
+  CRUSH rule 0 x 977 [14,3,6,10,4]
+  CRUSH rule 0 x 978 [12,5,11,1,15]
+  CRUSH rule 0 x 979 [5,1,13,6,15]
+  CRUSH rule 0 x 980 [15,11,5,6,1]
+  CRUSH rule 0 x 981 [5,11,15,12,7]
+  CRUSH rule 0 x 982 [2,6,14,11,12]
+  CRUSH rule 0 x 983 [3,12,10,9,14]
+  CRUSH rule 0 x 984 [15,13,1,10,2]
+  CRUSH rule 0 x 985 [11,2,15,1,4]
+  CRUSH rule 0 x 986 [6,13,9,1,15]
+  CRUSH rule 0 x 987 [13,14,5,10,6]
+  CRUSH rule 0 x 988 [12,9,10,14,3]
+  CRUSH rule 0 x 989 [7,4,3,15,9]
+  CRUSH rule 0 x 990 [1,10,9,13,3]
+  CRUSH rule 0 x 991 [7,11,1,14,2]
+  CRUSH rule 0 x 992 [9,10,2,13,7]
+  CRUSH rule 0 x 993 [6,10,14,12,4]
+  CRUSH rule 0 x 994 [3,13,15,4,11]
+  CRUSH rule 0 x 995 [15,6,12,2,5]
+  CRUSH rule 0 x 996 [15,10,5,3,13]
+  CRUSH rule 0 x 997 [15,2,1,12,7]
+  CRUSH rule 0 x 998 [6,1,9,5,12]
+  CRUSH rule 0 x 999 [9,10,15,5,13]
+  CRUSH rule 0 x 1000 [14,2,9,4,12]
+  CRUSH rule 0 x 1001 [11,14,4,2,6]
+  CRUSH rule 0 x 1002 [1,10,14,2,9]
+  CRUSH rule 0 x 1003 [10,7,5,14,2]
+  CRUSH rule 0 x 1004 [15,1,4,6,10]
+  CRUSH rule 0 x 1005 [6,12,2,10,9]
+  CRUSH rule 0 x 1006 [10,12,15,1,2]
+  CRUSH rule 0 x 1007 [1,7,13,14,3]
+  CRUSH rule 0 x 1008 [7,4,9,11,3]
+  CRUSH rule 0 x 1009 [5,2,11,7,15]
+  CRUSH rule 0 x 1010 [10,2,15,6,9]
+  CRUSH rule 0 x 1011 [6,3,12,1,10]
+  CRUSH rule 0 x 1012 [12,6,9,15,3]
+  CRUSH rule 0 x 1013 [2,14,12,4,9]
+  CRUSH rule 0 x 1014 [1,13,7,2,10]
+  CRUSH rule 0 x 1015 [12,6,10,1,4]
+  CRUSH rule 0 x 1016 [10,13,14,3,5]
+  CRUSH rule 0 x 1017 [5,11,14,7,13]
+  CRUSH rule 0 x 1018 [13,11,14,1,9]
+  CRUSH rule 0 x 1019 [10,13,14,7,5]
+  CRUSH rule 0 x 1020 [3,1,13,4,10]
+  CRUSH rule 0 x 1021 [2,11,14,9,4]
+  CRUSH rule 0 x 1022 [15,5,7,2,12]
+  CRUSH rule 0 x 1023 [15,2,9,12,1]
+  rule 0 (replicated_ruleset) num_rep 5 result size == 5:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3,15,12,1]
+  CRUSH rule 0 x 1 [10,15,1,2,13,4]
+  CRUSH rule 0 x 2 [1,12,2,6,5,10]
+  CRUSH rule 0 x 3 [15,4,10,2,9,6]
+  CRUSH rule 0 x 4 [14,2,10,1,9,4]
+  CRUSH rule 0 x 5 [7,4,11,2,13,15]
+  CRUSH rule 0 x 6 [12,6,10,9,3,4]
+  CRUSH rule 0 x 7 [9,2,6,12,11,4]
+  CRUSH rule 0 x 8 [10,2,15,1,4,13]
+  CRUSH rule 0 x 9 [7,1,14,2,11,9]
+  CRUSH rule 0 x 10 [10,14,4,1,2,7]
+  CRUSH rule 0 x 11 [13,9,14,7,5,11]
+  CRUSH rule 0 x 12 [7,1,2,5,13,15]
+  CRUSH rule 0 x 13 [3,5,12,7,9,1]
+  CRUSH rule 0 x 14 [13,5,2,7,10,15]
+  CRUSH rule 0 x 15 [15,1,9,6,13,3]
+  CRUSH rule 0 x 16 [7,11,14,2,13,1]
+  CRUSH rule 0 x 17 [10,1,13,2,4,6]
+  CRUSH rule 0 x 18 [1,7,3,10,5,12]
+  CRUSH rule 0 x 19 [7,12,2,4,15,10]
+  CRUSH rule 0 x 20 [14,12,3,10,9,4]
+  CRUSH rule 0 x 21 [3,12,1,10,4,15]
+  CRUSH rule 0 x 22 [6,3,13,11,4,1]
+  CRUSH rule 0 x 23 [10,5,13,9,3,15]
+  CRUSH rule 0 x 24 [12,11,3,1,9,4]
+  CRUSH rule 0 x 25 [7,12,15,1,3,10]
+  CRUSH rule 0 x 26 [1,7,13,2,14,5]
+  CRUSH rule 0 x 27 [3,6,15,4,13,9]
+  CRUSH rule 0 x 28 [14,4,3,9,6,11]
+  CRUSH rule 0 x 29 [5,14,12,11,6,3]
+  CRUSH rule 0 x 30 [2,5,6,9,1,11]
+  CRUSH rule 0 x 31 [5,15,10,1,9,13]
+  CRUSH rule 0 x 32 [9,10,2,1,13,14]
+  CRUSH rule 0 x 33 [13,4,9,2,7,1]
+  CRUSH rule 0 x 34 [13,15,2,4,1,10]
+  CRUSH rule 0 x 35 [4,14,3,13,10,9]
+  CRUSH rule 0 x 36 [3,12,9,7,5,10]
+  CRUSH rule 0 x 37 [9,2,6,14,11,1]
+  CRUSH rule 0 x 38 [3,4,13,10,9,1]
+  CRUSH rule 0 x 39 [12,7,14,11,1,9]
+  CRUSH rule 0 x 40 [10,1,9,5,15,2]
+  CRUSH rule 0 x 41 [4,9,11,1,14,13]
+  CRUSH rule 0 x 42 [3,6,14,10,12,5]
+  CRUSH rule 0 x 43 [10,5,15,7,2,9]
+  CRUSH rule 0 x 44 [11,4,13,3,7,14]
+  CRUSH rule 0 x 45 [11,12,15,9,1,5]
+  CRUSH rule 0 x 46 [6,9,2,14,11,13]
+  CRUSH rule 0 x 47 [3,9,6,4,13,1]
+  CRUSH rule 0 x 48 [4,6,2,1,10,14]
+  CRUSH rule 0 x 49 [9,15,10,7,4,3]
+  CRUSH rule 0 x 50 [14,12,1,4,2,11]
+  CRUSH rule 0 x 51 [10,6,5,12,15,2]
+  CRUSH rule 0 x 52 [12,1,9,11,7,3]
+  CRUSH rule 0 x 53 [3,6,13,9,5,1]
+  CRUSH rule 0 x 54 [4,13,9,2,14,10]
+  CRUSH rule 0 x 55 [4,11,2,7,1,13]
+  CRUSH rule 0 x 56 [5,9,10,1,3,13]
+  CRUSH rule 0 x 57 [6,2,1,15,10,12]
+  CRUSH rule 0 x 58 [7,1,11,4,3,14]
+  CRUSH rule 0 x 59 [2,13,1,10,9,5]
+  CRUSH rule 0 x 60 [3,6,11,1,4,9]
+  CRUSH rule 0 x 61 [3,15,13,7,4,1]
+  CRUSH rule 0 x 62 [15,11,7,12,5,9]
+  CRUSH rule 0 x 63 [10,14,12,1,7,3]
+  CRUSH rule 0 x 64 [3,9,1,4,7,12]
+  CRUSH rule 0 x 65 [4,12,11,7,14,3]
+  CRUSH rule 0 x 66 [15,11,6,9,4,1]
+  CRUSH rule 0 x 67 [2,6,4,14,1,11]
+  CRUSH rule 0 x 68 [15,7,4,2,9,12]
+  CRUSH rule 0 x 69 [2,1,15,10,4,9]
+  CRUSH rule 0 x 70 [9,6,1,3,13,15]
+  CRUSH rule 0 x 71 [15,5,1,3,13,10]
+  CRUSH rule 0 x 72 [9,10,3,5,7,12]
+  CRUSH rule 0 x 73 [5,3,11,1,7,12]
+  CRUSH rule 0 x 74 [11,7,9,5,1,15]
+  CRUSH rule 0 x 75 [9,7,11,14,12,1]
+  CRUSH rule 0 x 76 [6,1,3,5,14,10]
+  CRUSH rule 0 x 77 [7,4,2,13,9,1]
+  CRUSH rule 0 x 78 [9,3,1,5,6,13]
+  CRUSH rule 0 x 79 [13,2,15,5,7,9]
+  CRUSH rule 0 x 80 [15,2,6,4,13,10]
+  CRUSH rule 0 x 81 [15,2,1,11,4,6]
+  CRUSH rule 0 x 82 [14,13,5,11,6,2]
+  CRUSH rule 0 x 83 [4,15,3,9,10,13]
+  CRUSH rule 0 x 84 [10,7,9,15,3,4]
+  CRUSH rule 0 x 85 [3,15,9,7,4,11]
+  CRUSH rule 0 x 86 [10,9,14,1,13,4]
+  CRUSH rule 0 x 87 [15,10,7,12,5,3]
+  CRUSH rule 0 x 88 [4,13,3,1,9,15]
+  CRUSH rule 0 x 89 [3,9,7,4,1,14]
+  CRUSH rule 0 x 90 [4,9,7,12,11,14]
+  CRUSH rule 0 x 91 [6,11,9,1,2,4]
+  CRUSH rule 0 x 92 [1,5,10,9,13,15]
+  CRUSH rule 0 x 93 [9,3,15,13,7,5]
+  CRUSH rule 0 x 94 [9,2,12,5,6,11]
+  CRUSH rule 0 x 95 [7,15,4,10,9,13]
+  CRUSH rule 0 x 96 [2,15,11,7,5,1]
+  CRUSH rule 0 x 97 [4,11,2,13,1,7]
+  CRUSH rule 0 x 98 [11,13,9,3,15,1]
+  CRUSH rule 0 x 99 [12,4,11,7,3,14]
+  CRUSH rule 0 x 100 [9,4,10,15,7,3]
+  CRUSH rule 0 x 101 [15,7,1,9,10,5]
+  CRUSH rule 0 x 102 [3,11,14,6,13,4]
+  CRUSH rule 0 x 103 [13,11,6,14,4,3]
+  CRUSH rule 0 x 104 [14,6,3,5,9,1]
+  CRUSH rule 0 x 105 [14,10,1,9,3,5]
+  CRUSH rule 0 x 106 [6,5,13,2,14,11]
+  CRUSH rule 0 x 107 [3,1,10,14,13,5]
+  CRUSH rule 0 x 108 [5,10,7,2,15,9]
+  CRUSH rule 0 x 109 [9,1,13,7,15,5]
+  CRUSH rule 0 x 110 [5,1,11,3,7,14]
+  CRUSH rule 0 x 111 [10,1,9,7,5,2]
+  CRUSH rule 0 x 112 [1,10,4,14,2,12]
+  CRUSH rule 0 x 113 [6,10,13,9,1,5]
+  CRUSH rule 0 x 114 [5,13,6,2,1,14]
+  CRUSH rule 0 x 115 [10,13,14,3,9,1]
+  CRUSH rule 0 x 116 [1,14,13,2,11,5]
+  CRUSH rule 0 x 117 [5,6,1,12,15,9]
+  CRUSH rule 0 x 118 [10,4,13,15,9,3]
+  CRUSH rule 0 x 119 [14,12,11,4,6,9]
+  CRUSH rule 0 x 120 [11,3,14,13,4,7]
+  CRUSH rule 0 x 121 [9,5,1,11,7,3]
+  CRUSH rule 0 x 122 [4,3,14,1,11,13]
+  CRUSH rule 0 x 123 [3,10,5,6,9,1]
+  CRUSH rule 0 x 124 [12,2,1,5,14,7]
+  CRUSH rule 0 x 125 [9,12,15,1,6,5]
+  CRUSH rule 0 x 126 [7,15,10,9,2,12]
+  CRUSH rule 0 x 127 [4,14,9,13,1,3]
+  CRUSH rule 0 x 128 [3,12,1,10,4,9]
+  CRUSH rule 0 x 129 [11,13,14,2,9,4]
+  CRUSH rule 0 x 130 [3,13,5,14,10,1]
+  CRUSH rule 0 x 131 [12,1,6,15,4,2]
+  CRUSH rule 0 x 132 [11,15,13,9,2,5]
+  CRUSH rule 0 x 133 [3,6,9,11,15,12]
+  CRUSH rule 0 x 134 [12,5,6,15,3,9]
+  CRUSH rule 0 x 135 [3,14,12,4,6,11]
+  CRUSH rule 0 x 136 [15,6,9,4,10,3]
+  CRUSH rule 0 x 137 [14,3,6,11,1,9]
+  CRUSH rule 0 x 138 [13,15,4,10,2,7]
+  CRUSH rule 0 x 139 [11,2,13,9,1,15]
+  CRUSH rule 0 x 140 [11,4,12,15,2,6]
+  CRUSH rule 0 x 141 [6,12,15,11,3,5]
+  CRUSH rule 0 x 142 [3,14,7,9,11,1]
+  CRUSH rule 0 x 143 [9,6,4,2,14,10]
+  CRUSH rule 0 x 144 [13,7,11,2,14,4]
+  CRUSH rule 0 x 145 [12,2,6,10,9,4]
+  CRUSH rule 0 x 146 [1,5,9,2,6,13]
+  CRUSH rule 0 x 147 [1,4,9,11,2,7]
+  CRUSH rule 0 x 148 [12,7,9,2,14,11]
+  CRUSH rule 0 x 149 [2,5,9,12,11,1]
+  CRUSH rule 0 x 150 [1,15,2,10,7,9]
+  CRUSH rule 0 x 151 [2,9,14,7,1,10]
+  CRUSH rule 0 x 152 [5,9,2,6,10,13]
+  CRUSH rule 0 x 153 [6,9,4,15,2,1]
+  CRUSH rule 0 x 154 [3,11,7,1,4,12]
+  CRUSH rule 0 x 155 [14,12,7,3,5,1]
+  CRUSH rule 0 x 156 [7,13,3,10,15,5]
+  CRUSH rule 0 x 157 [15,1,6,4,3,10]
+  CRUSH rule 0 x 158 [15,1,10,6,12,2]
+  CRUSH rule 0 x 159 [4,14,3,12,10,6]
+  CRUSH rule 0 x 160 [5,7,3,14,11,1]
+  CRUSH rule 0 x 161 [1,2,11,4,6,13]
+  CRUSH rule 0 x 162 [10,6,1,12,2,4]
+  CRUSH rule 0 x 163 [15,1,10,2,6,4]
+  CRUSH rule 0 x 164 [9,14,10,7,12,2]
+  CRUSH rule 0 x 165 [11,7,2,13,9,15]
+  CRUSH rule 0 x 166 [1,2,12,14,4,11]
+  CRUSH rule 0 x 167 [9,7,3,4,11,13]
+  CRUSH rule 0 x 168 [13,2,4,1,6,15]
+  CRUSH rule 0 x 169 [1,4,9,14,13,10]
+  CRUSH rule 0 x 170 [1,15,7,9,12,10]
+  CRUSH rule 0 x 171 [9,2,10,7,1,5]
+  CRUSH rule 0 x 172 [14,4,10,12,9,3]
+  CRUSH rule 0 x 173 [5,10,12,15,6,1]
+  CRUSH rule 0 x 174 [15,6,4,12,1,11]
+  CRUSH rule 0 x 175 [5,7,9,3,10,1]
+  CRUSH rule 0 x 176 [9,6,3,14,13,10]
+  CRUSH rule 0 x 177 [2,9,10,13,4,1]
+  CRUSH rule 0 x 178 [12,11,7,14,3,4]
+  CRUSH rule 0 x 179 [2,10,13,9,5,1]
+  CRUSH rule 0 x 180 [3,11,5,15,7,12]
+  CRUSH rule 0 x 181 [9,12,6,5,1,10]
+  CRUSH rule 0 x 182 [5,13,11,2,1,6]
+  CRUSH rule 0 x 183 [5,7,10,13,3,9]
+  CRUSH rule 0 x 184 [2,5,11,12,7,1]
+  CRUSH rule 0 x 185 [13,5,7,11,2,14]
+  CRUSH rule 0 x 186 [6,14,13,5,10,1]
+  CRUSH rule 0 x 187 [1,4,11,13,6,14]
+  CRUSH rule 0 x 188 [9,13,5,14,10,6]
+  CRUSH rule 0 x 189 [6,12,4,9,2,1]
+  CRUSH rule 0 x 190 [9,13,15,10,3,1]
+  CRUSH rule 0 x 191 [7,11,4,1,15,12]
+  CRUSH rule 0 x 192 [2,11,5,15,6,1]
+  CRUSH rule 0 x 193 [3,13,6,10,4,1]
+  CRUSH rule 0 x 194 [3,13,4,14,6,9]
+  CRUSH rule 0 x 195 [5,7,10,12,1,3]
+  CRUSH rule 0 x 196 [4,15,1,10,9,2]
+  CRUSH rule 0 x 197 [14,10,13,4,6,3]
+  CRUSH rule 0 x 198 [2,5,6,15,9,13]
+  CRUSH rule 0 x 199 [2,10,4,15,1,9]
+  CRUSH rule 0 x 200 [7,14,11,4,1,3]
+  CRUSH rule 0 x 201 [9,14,1,7,4,3]
+  CRUSH rule 0 x 202 [14,11,7,3,5,1]
+  CRUSH rule 0 x 203 [12,5,7,15,1,2]
+  CRUSH rule 0 x 204 [6,11,3,12,14,1]
+  CRUSH rule 0 x 205 [15,4,6,10,13,9]
+  CRUSH rule 0 x 206 [13,11,2,15,7,1]
+  CRUSH rule 0 x 207 [2,11,7,4,14,1]
+  CRUSH rule 0 x 208 [13,1,6,14,9,11]
+  CRUSH rule 0 x 209 [6,15,13,1,11,4]
+  CRUSH rule 0 x 210 [13,11,2,7,5,14]
+  CRUSH rule 0 x 211 [2,14,1,13,11,7]
+  CRUSH rule 0 x 212 [10,1,12,15,5,6]
+  CRUSH rule 0 x 213 [3,9,6,5,15,13]
+  CRUSH rule 0 x 214 [7,15,4,1,10,2]
+  CRUSH rule 0 x 215 [6,1,4,13,3,11]
+  CRUSH rule 0 x 216 [12,9,6,2,1,11]
+  CRUSH rule 0 x 217 [12,11,1,14,2,4]
+  CRUSH rule 0 x 218 [12,10,15,6,1,4]
+  CRUSH rule 0 x 219 [3,11,14,6,4,1]
+  CRUSH rule 0 x 220 [14,4,3,12,10,9]
+  CRUSH rule 0 x 221 [15,5,2,6,12,11]
+  CRUSH rule 0 x 222 [10,4,3,15,7,12]
+  CRUSH rule 0 x 223 [9,7,11,1,4,14]
+  CRUSH rule 0 x 224 [1,7,10,2,12,9]
+  CRUSH rule 0 x 225 [10,5,2,6,1,13]
+  CRUSH rule 0 x 226 [4,1,9,3,13,10]
+  CRUSH rule 0 x 227 [7,2,12,15,5,11]
+  CRUSH rule 0 x 228 [2,15,11,1,6,13]
+  CRUSH rule 0 x 229 [9,3,7,14,1,12]
+  CRUSH rule 0 x 230 [10,5,7,2,15,1]
+  CRUSH rule 0 x 231 [2,7,5,13,9,15]
+  CRUSH rule 0 x 232 [10,5,13,1,9,2]
+  CRUSH rule 0 x 233 [6,12,11,4,9,14]
+  CRUSH rule 0 x 234 [10,1,2,12,5,9]
+  CRUSH rule 0 x 235 [13,14,7,10,1,9]
+  CRUSH rule 0 x 236 [2,15,9,12,1,7]
+  CRUSH rule 0 x 237 [3,12,9,10,4,7]
+  CRUSH rule 0 x 238 [2,10,4,15,6,12]
+  CRUSH rule 0 x 239 [4,15,10,7,9,13]
+  CRUSH rule 0 x 240 [15,5,13,7,2,9]
+  CRUSH rule 0 x 241 [7,9,15,12,1,5]
+  CRUSH rule 0 x 242 [14,2,6,9,10,12]
+  CRUSH rule 0 x 243 [2,11,5,1,15,6]
+  CRUSH rule 0 x 244 [13,9,15,3,11,7]
+  CRUSH rule 0 x 245 [12,9,15,3,1,5]
+  CRUSH rule 0 x 246 [15,3,5,11,7,1]
+  CRUSH rule 0 x 247 [6,4,9,12,1,2]
+  CRUSH rule 0 x 248 [5,13,7,11,9,15]
+  CRUSH rule 0 x 249 [10,14,7,3,9,13]
+  CRUSH rule 0 x 250 [12,15,1,10,5,6]
+  CRUSH rule 0 x 251 [13,2,15,5,6,1]
+  CRUSH rule 0 x 252 [7,5,13,9,3,10]
+  CRUSH rule 0 x 253 [3,13,15,10,7,4]
+  CRUSH rule 0 x 254 [2,9,13,14,4,6]
+  CRUSH rule 0 x 255 [1,9,13,2,6,10]
+  CRUSH rule 0 x 256 [6,9,13,1,3,14]
+  CRUSH rule 0 x 257 [15,12,3,9,6,4]
+  CRUSH rule 0 x 258 [12,5,6,10,2,1]
+  CRUSH rule 0 x 259 [9,10,4,3,14,13]
+  CRUSH rule 0 x 260 [10,12,6,9,3,15]
+  CRUSH rule 0 x 261 [13,7,2,1,15,5]
+  CRUSH rule 0 x 262 [15,3,12,7,4,9]
+  CRUSH rule 0 x 263 [12,6,10,9,5,15]
+  CRUSH rule 0 x 264 [13,14,11,3,1,4]
+  CRUSH rule 0 x 265 [12,10,14,5,7,1]
+  CRUSH rule 0 x 266 [14,7,11,1,2,9]
+  CRUSH rule 0 x 267 [12,11,6,5,1,2]
+  CRUSH rule 0 x 268 [4,1,15,12,6,11]
+  CRUSH rule 0 x 269 [11,1,15,5,13,9]
+  CRUSH rule 0 x 270 [7,11,12,3,1,14]
+  CRUSH rule 0 x 271 [4,7,3,13,15,10]
+  CRUSH rule 0 x 272 [15,5,13,10,6,2]
+  CRUSH rule 0 x 273 [2,10,7,12,1,15]
+  CRUSH rule 0 x 274 [10,2,5,6,13,9]
+  CRUSH rule 0 x 275 [10,3,4,7,14,13]
+  CRUSH rule 0 x 276 [5,12,9,2,11,7]
+  CRUSH rule 0 x 277 [14,3,13,4,1,9]
+  CRUSH rule 0 x 278 [5,6,14,3,1,11]
+  CRUSH rule 0 x 279 [6,10,13,3,9,4]
+  CRUSH rule 0 x 280 [7,3,14,9,1,11]
+  CRUSH rule 0 x 281 [5,11,14,7,9,13]
+  CRUSH rule 0 x 282 [2,1,13,14,9,7]
+  CRUSH rule 0 x 283 [4,1,12,3,10,7]
+  CRUSH rule 0 x 284 [5,11,7,15,3,13]
+  CRUSH rule 0 x 285 [15,5,3,1,6,13]
+  CRUSH rule 0 x 286 [10,4,3,6,12,15]
+  CRUSH rule 0 x 287 [12,4,9,1,3,11]
+  CRUSH rule 0 x 288 [4,12,10,7,1,3]
+  CRUSH rule 0 x 289 [2,5,14,9,13,6]
+  CRUSH rule 0 x 290 [12,2,5,6,15,9]
+  CRUSH rule 0 x 291 [7,11,1,14,5,9]
+  CRUSH rule 0 x 292 [4,10,6,3,14,9]
+  CRUSH rule 0 x 293 [6,5,11,1,2,14]
+  CRUSH rule 0 x 294 [9,12,3,14,6,11]
+  CRUSH rule 0 x 295 [6,10,3,14,9,4]
+  CRUSH rule 0 x 296 [3,1,13,7,14,9]
+  CRUSH rule 0 x 297 [6,13,4,14,10,1]
+  CRUSH rule 0 x 298 [14,9,13,1,4,2]
+  CRUSH rule 0 x 299 [14,12,11,6,4,2]
+  CRUSH rule 0 x 300 [15,7,10,5,1,3]
+  CRUSH rule 0 x 301 [9,11,7,1,13,14]
+  CRUSH rule 0 x 302 [9,7,1,13,5,10]
+  CRUSH rule 0 x 303 [4,13,3,7,10,15]
+  CRUSH rule 0 x 304 [6,9,2,11,15,13]
+  CRUSH rule 0 x 305 [13,7,5,11,2,15]
+  CRUSH rule 0 x 306 [10,12,4,6,9,2]
+  CRUSH rule 0 x 307 [11,12,15,5,6,2]
+  CRUSH rule 0 x 308 [12,14,10,9,1,2]
+  CRUSH rule 0 x 309 [9,3,12,5,11,15]
+  CRUSH rule 0 x 310 [3,1,5,10,14,9]
+  CRUSH rule 0 x 311 [3,9,7,1,14,13]
+  CRUSH rule 0 x 312 [15,13,9,7,5,10]
+  CRUSH rule 0 x 313 [9,15,3,7,5,13]
+  CRUSH rule 0 x 314 [2,15,9,5,6,12]
+  CRUSH rule 0 x 315 [15,2,13,1,11,9]
+  CRUSH rule 0 x 316 [4,9,11,2,12,14]
+  CRUSH rule 0 x 317 [1,5,3,13,15,7]
+  CRUSH rule 0 x 318 [4,1,15,11,9,13]
+  CRUSH rule 0 x 319 [2,15,4,1,11,9]
+  CRUSH rule 0 x 320 [5,7,13,9,11,2]
+  CRUSH rule 0 x 321 [1,6,11,15,5,3]
+  CRUSH rule 0 x 322 [13,7,5,3,14,11]
+  CRUSH rule 0 x 323 [7,4,10,1,2,13]
+  CRUSH rule 0 x 324 [5,6,10,15,2,13]
+  CRUSH rule 0 x 325 [9,10,14,5,1,6]
+  CRUSH rule 0 x 326 [11,7,13,4,2,15]
+  CRUSH rule 0 x 327 [12,5,10,14,3,7]
+  CRUSH rule 0 x 328 [5,2,6,14,1,11]
+  CRUSH rule 0 x 329 [2,6,15,5,9,10]
+  CRUSH rule 0 x 330 [3,9,11,13,1,6]
+  CRUSH rule 0 x 331 [12,14,6,3,1,4]
+  CRUSH rule 0 x 332 [10,12,6,15,9,2]
+  CRUSH rule 0 x 333 [6,5,3,12,14,10]
+  CRUSH rule 0 x 334 [4,9,2,12,7,11]
+  CRUSH rule 0 x 335 [11,7,1,5,13,2]
+  CRUSH rule 0 x 336 [6,14,13,2,5,9]
+  CRUSH rule 0 x 337 [15,11,3,7,12,5]
+  CRUSH rule 0 x 338 [10,5,3,6,15,1]
+  CRUSH rule 0 x 339 [11,14,13,5,3,7]
+  CRUSH rule 0 x 340 [11,6,12,4,9,3]
+  CRUSH rule 0 x 341 [7,5,2,10,14,9]
+  CRUSH rule 0 x 342 [12,14,1,9,2,11]
+  CRUSH rule 0 x 343 [12,14,9,6,10,2]
+  CRUSH rule 0 x 344 [9,11,5,2,14,13]
+  CRUSH rule 0 x 345 [14,2,11,9,6,12]
+  CRUSH rule 0 x 346 [5,3,14,10,7,1]
+  CRUSH rule 0 x 347 [10,2,12,6,9,1]
+  CRUSH rule 0 x 348 [7,9,10,1,14,13]
+  CRUSH rule 0 x 349 [9,6,10,12,1,5]
+  CRUSH rule 0 x 350 [13,9,15,4,10,7]
+  CRUSH rule 0 x 351 [13,5,15,3,1,6]
+  CRUSH rule 0 x 352 [1,12,11,9,4,7]
+  CRUSH rule 0 x 353 [10,14,12,2,9,1]
+  CRUSH rule 0 x 354 [6,3,15,10,9,4]
+  CRUSH rule 0 x 355 [13,14,6,10,2,5]
+  CRUSH rule 0 x 356 [15,13,2,9,6,5]
+  CRUSH rule 0 x 357 [4,11,1,13,3,14]
+  CRUSH rule 0 x 358 [12,7,2,9,1,14]
+  CRUSH rule 0 x 359 [5,15,7,11,3,13]
+  CRUSH rule 0 x 360 [13,10,1,2,6,14]
+  CRUSH rule 0 x 361 [5,3,13,6,1,14]
+  CRUSH rule 0 x 362 [2,9,11,13,1,6]
+  CRUSH rule 0 x 363 [7,12,3,9,15,4]
+  CRUSH rule 0 x 364 [2,12,6,9,5,10]
+  CRUSH rule 0 x 365 [13,5,11,15,6,2]
+  CRUSH rule 0 x 366 [12,7,3,14,5,10]
+  CRUSH rule 0 x 367 [7,13,3,1,5,11]
+  CRUSH rule 0 x 368 [7,9,10,15,3,4]
+  CRUSH rule 0 x 369 [7,5,3,13,14,9]
+  CRUSH rule 0 x 370 [4,7,14,1,2,9]
+  CRUSH rule 0 x 371 [1,7,12,3,4,15]
+  CRUSH rule 0 x 372 [10,4,3,14,6,1]
+  CRUSH rule 0 x 373 [15,5,2,6,13,1]
+  CRUSH rule 0 x 374 [3,15,12,5,1,6]
+  CRUSH rule 0 x 375 [5,2,14,1,6,13]
+  CRUSH rule 0 x 376 [5,14,10,13,3,6]
+  CRUSH rule 0 x 377 [1,15,2,4,9,11]
+  CRUSH rule 0 x 378 [9,12,2,15,1,5]
+  CRUSH rule 0 x 379 [11,2,15,5,7,9]
+  CRUSH rule 0 x 380 [6,1,12,11,2,9]
+  CRUSH rule 0 x 381 [15,13,7,5,10,2]
+  CRUSH rule 0 x 382 [14,3,1,4,13,7]
+  CRUSH rule 0 x 383 [3,6,11,4,13,15]
+  CRUSH rule 0 x 384 [4,13,6,3,15,11]
+  CRUSH rule 0 x 385 [4,6,15,3,10,9]
+  CRUSH rule 0 x 386 [14,3,11,13,5,6]
+  CRUSH rule 0 x 387 [1,11,5,7,9,2]
+  CRUSH rule 0 x 388 [2,6,11,9,15,4]
+  CRUSH rule 0 x 389 [12,7,2,4,15,10]
+  CRUSH rule 0 x 390 [2,11,13,7,5,9]
+  CRUSH rule 0 x 391 [3,4,9,13,7,10]
+  CRUSH rule 0 x 392 [11,5,14,7,1,9]
+  CRUSH rule 0 x 393 [2,14,5,9,7,13]
+  CRUSH rule 0 x 394 [4,9,3,15,13,6]
+  CRUSH rule 0 x 395 [10,13,5,15,6,9]
+  CRUSH rule 0 x 396 [2,12,15,9,4,6]
+  CRUSH rule 0 x 397 [1,14,9,4,12,10]
+  CRUSH rule 0 x 398 [9,2,1,5,12,6]
+  CRUSH rule 0 x 399 [5,9,14,3,1,10]
+  CRUSH rule 0 x 400 [10,6,2,4,15,12]
+  CRUSH rule 0 x 401 [6,9,11,12,4,3]
+  CRUSH rule 0 x 402 [4,7,9,2,13,1]
+  CRUSH rule 0 x 403 [7,15,13,3,5,9]
+  CRUSH rule 0 x 404 [14,12,7,9,2,1]
+  CRUSH rule 0 x 405 [9,15,11,2,4,7]
+  CRUSH rule 0 x 406 [12,14,9,2,7,10]
+  CRUSH rule 0 x 407 [9,5,12,10,15,6]
+  CRUSH rule 0 x 408 [7,1,5,2,10,15]
+  CRUSH rule 0 x 409 [11,2,4,13,1,15]
+  CRUSH rule 0 x 410 [6,4,14,2,12,9]
+  CRUSH rule 0 x 411 [13,11,15,6,4,1]
+  CRUSH rule 0 x 412 [5,9,6,11,14,2]
+  CRUSH rule 0 x 413 [13,5,3,11,6,9]
+  CRUSH rule 0 x 414 [3,11,9,13,4,1]
+  CRUSH rule 0 x 415 [6,10,14,5,1,13]
+  CRUSH rule 0 x 416 [13,1,4,7,2,9]
+  CRUSH rule 0 x 417 [4,12,1,15,2,11]
+  CRUSH rule 0 x 418 [14,5,10,2,6,9]
+  CRUSH rule 0 x 419 [5,14,10,9,2,12]
+  CRUSH rule 0 x 420 [2,4,9,11,6,14]
+  CRUSH rule 0 x 421 [15,4,10,3,9,12]
+  CRUSH rule 0 x 422 [4,11,2,7,13,9]
+  CRUSH rule 0 x 423 [3,15,12,6,5,1]
+  CRUSH rule 0 x 424 [6,10,12,2,5,1]
+  CRUSH rule 0 x 425 [11,15,2,13,5,7]
+  CRUSH rule 0 x 426 [12,4,7,1,9,10]
+  CRUSH rule 0 x 427 [14,10,3,1,9,7]
+  CRUSH rule 0 x 428 [12,7,9,4,2,1]
+  CRUSH rule 0 x 429 [3,4,9,7,11,12]
+  CRUSH rule 0 x 430 [3,5,10,13,1,15]
+  CRUSH rule 0 x 431 [9,3,7,1,12,5]
+  CRUSH rule 0 x 432 [4,1,12,7,15,2]
+  CRUSH rule 0 x 433 [4,11,12,15,7,3]
+  CRUSH rule 0 x 434 [2,14,9,1,5,11]
+  CRUSH rule 0 x 435 [13,11,5,6,9,2]
+  CRUSH rule 0 x 436 [9,15,10,2,4,1]
+  CRUSH rule 0 x 437 [9,6,3,14,10,12]
+  CRUSH rule 0 x 438 [7,2,13,4,11,1]
+  CRUSH rule 0 x 439 [7,14,4,3,12,10]
+  CRUSH rule 0 x 440 [14,11,9,2,7,12]
+  CRUSH rule 0 x 441 [2,4,11,9,13,6]
+  CRUSH rule 0 x 442 [10,13,9,7,15,1]
+  CRUSH rule 0 x 443 [12,15,10,9,2,1]
+  CRUSH rule 0 x 444 [4,13,7,14,3,1]
+  CRUSH rule 0 x 445 [4,2,15,7,1,9]
+  CRUSH rule 0 x 446 [12,10,6,9,4,1]
+  CRUSH rule 0 x 447 [15,7,13,1,4,9]
+  CRUSH rule 0 x 448 [5,2,13,7,15,10]
+  CRUSH rule 0 x 449 [14,5,3,12,10,9]
+  CRUSH rule 0 x 450 [2,4,6,9,15,1]
+  CRUSH rule 0 x 451 [6,14,11,3,9,1]
+  CRUSH rule 0 x 452 [14,9,10,4,2,13]
+  CRUSH rule 0 x 453 [5,15,13,2,6,9]
+  CRUSH rule 0 x 454 [10,4,2,6,15,12]
+  CRUSH rule 0 x 455 [6,13,2,4,10,1]
+  CRUSH rule 0 x 456 [5,7,13,1,11,3]
+  CRUSH rule 0 x 457 [9,1,5,7,11,13]
+  CRUSH rule 0 x 458 [9,11,15,4,7,2]
+  CRUSH rule 0 x 459 [13,15,11,1,5,2]
+  CRUSH rule 0 x 460 [5,12,10,15,7,3]
+  CRUSH rule 0 x 461 [4,3,9,13,15,6]
+  CRUSH rule 0 x 462 [4,7,12,14,11,1]
+  CRUSH rule 0 x 463 [4,12,14,11,2,7]
+  CRUSH rule 0 x 464 [4,2,15,10,1,9]
+  CRUSH rule 0 x 465 [5,10,9,7,13,1]
+  CRUSH rule 0 x 466 [13,5,2,15,9,11]
+  CRUSH rule 0 x 467 [13,6,14,3,9,1]
+  CRUSH rule 0 x 468 [10,7,12,14,4,1]
+  CRUSH rule 0 x 469 [4,9,6,14,12,11]
+  CRUSH rule 0 x 470 [3,9,12,15,5,6]
+  CRUSH rule 0 x 471 [6,1,5,14,13,10]
+  CRUSH rule 0 x 472 [2,14,7,5,13,1]
+  CRUSH rule 0 x 473 [15,10,6,9,4,12]
+  CRUSH rule 0 x 474 [15,10,4,12,6,9]
+  CRUSH rule 0 x 475 [10,5,12,9,14,3]
+  CRUSH rule 0 x 476 [3,6,10,12,1,15]
+  CRUSH rule 0 x 477 [6,13,5,15,11,9]
+  CRUSH rule 0 x 478 [4,15,1,3,7,12]
+  CRUSH rule 0 x 479 [13,11,1,6,14,5]
+  CRUSH rule 0 x 480 [1,13,6,4,9,14]
+  CRUSH rule 0 x 481 [15,12,7,9,1,3]
+  CRUSH rule 0 x 482 [2,12,9,1,7,11]
+  CRUSH rule 0 x 483 [10,1,4,15,9,7]
+  CRUSH rule 0 x 484 [1,4,10,13,7,14]
+  CRUSH rule 0 x 485 [9,4,3,1,14,12]
+  CRUSH rule 0 x 486 [3,10,15,9,7,13]
+  CRUSH rule 0 x 487 [12,11,4,14,7,2]
+  CRUSH rule 0 x 488 [14,4,1,9,2,6]
+  CRUSH rule 0 x 489 [11,4,2,13,15,7]
+  CRUSH rule 0 x 490 [4,9,1,3,13,15]
+  CRUSH rule 0 x 491 [1,12,5,2,14,11]
+  CRUSH rule 0 x 492 [5,7,11,3,14,9]
+  CRUSH rule 0 x 493 [12,1,4,15,3,11]
+  CRUSH rule 0 x 494 [1,7,13,4,15,9]
+  CRUSH rule 0 x 495 [3,15,7,1,9,5]
+  CRUSH rule 0 x 496 [5,3,7,13,9,14]
+  CRUSH rule 0 x 497 [13,10,3,6,5,14]
+  CRUSH rule 0 x 498 [10,6,1,5,9,12]
+  CRUSH rule 0 x 499 [14,3,12,5,1,11]
+  CRUSH rule 0 x 500 [15,9,6,12,11,2]
+  CRUSH rule 0 x 501 [10,13,1,9,3,14]
+  CRUSH rule 0 x 502 [5,1,14,11,7,12]
+  CRUSH rule 0 x 503 [15,10,7,9,1,12]
+  CRUSH rule 0 x 504 [13,2,7,1,14,11]
+  CRUSH rule 0 x 505 [12,7,5,2,14,10]
+  CRUSH rule 0 x 506 [11,7,9,14,12,1]
+  CRUSH rule 0 x 507 [4,14,13,3,9,7]
+  CRUSH rule 0 x 508 [12,1,4,9,2,11]
+  CRUSH rule 0 x 509 [4,2,6,9,14,1]
+  CRUSH rule 0 x 510 [5,3,1,12,11,14]
+  CRUSH rule 0 x 511 [2,12,10,6,14,5]
+  CRUSH rule 0 x 512 [15,11,3,5,7,1]
+  CRUSH rule 0 x 513 [4,9,11,3,13,7]
+  CRUSH rule 0 x 514 [11,9,3,4,12,15]
+  CRUSH rule 0 x 515 [12,14,6,5,3,9]
+  CRUSH rule 0 x 516 [14,11,1,12,3,7]
+  CRUSH rule 0 x 517 [11,5,6,13,9,3]
+  CRUSH rule 0 x 518 [3,5,7,12,15,11]
+  CRUSH rule 0 x 519 [12,14,2,1,4,6]
+  CRUSH rule 0 x 520 [12,4,2,10,6,15]
+  CRUSH rule 0 x 521 [11,5,9,6,15,3]
+  CRUSH rule 0 x 522 [4,12,11,1,15,3]
+  CRUSH rule 0 x 523 [3,1,5,9,15,10]
+  CRUSH rule 0 x 524 [15,9,3,11,13,7]
+  CRUSH rule 0 x 525 [3,15,11,6,9,12]
+  CRUSH rule 0 x 526 [10,2,5,13,6,15]
+  CRUSH rule 0 x 527 [3,13,4,1,9,10]
+  CRUSH rule 0 x 528 [12,7,15,10,2,5]
+  CRUSH rule 0 x 529 [6,4,10,12,2,9]
+  CRUSH rule 0 x 530 [11,9,12,7,5,1]
+  CRUSH rule 0 x 531 [9,15,4,7,2,13]
+  CRUSH rule 0 x 532 [5,3,13,7,9,14]
+  CRUSH rule 0 x 533 [12,15,1,2,7,5]
+  CRUSH rule 0 x 534 [11,9,3,7,15,4]
+  CRUSH rule 0 x 535 [11,1,3,5,14,9]
+  CRUSH rule 0 x 536 [9,1,14,13,4,6]
+  CRUSH rule 0 x 537 [15,5,13,2,7,11]
+  CRUSH rule 0 x 538 [13,5,11,2,6,15]
+  CRUSH rule 0 x 539 [10,12,6,14,1,2]
+  CRUSH rule 0 x 540 [12,15,7,3,9,11]
+  CRUSH rule 0 x 541 [2,1,6,11,14,13]
+  CRUSH rule 0 x 542 [3,9,15,5,11,12]
+  CRUSH rule 0 x 543 [4,10,9,3,6,13]
+  CRUSH rule 0 x 544 [3,15,9,11,7,4]
+  CRUSH rule 0 x 545 [14,10,7,12,4,9]
+  CRUSH rule 0 x 546 [5,15,13,7,1,10]
+  CRUSH rule 0 x 547 [5,13,7,9,3,14]
+  CRUSH rule 0 x 548 [11,7,12,15,4,2]
+  CRUSH rule 0 x 549 [14,1,4,9,13,6]
+  CRUSH rule 0 x 550 [9,15,3,13,1,6]
+  CRUSH rule 0 x 551 [11,2,15,6,13,5]
+  CRUSH rule 0 x 552 [2,11,14,1,9,6]
+  CRUSH rule 0 x 553 [11,9,14,6,4,13]
+  CRUSH rule 0 x 554 [11,14,6,4,13,9]
+  CRUSH rule 0 x 555 [6,5,10,9,14,2]
+  CRUSH rule 0 x 556 [15,6,3,13,11,4]
+  CRUSH rule 0 x 557 [12,2,5,14,10,9]
+  CRUSH rule 0 x 558 [12,1,6,15,5,10]
+  CRUSH rule 0 x 559 [2,13,5,10,14,7]
+  CRUSH rule 0 x 560 [4,9,12,6,3,10]
+  CRUSH rule 0 x 561 [12,7,1,2,5,15]
+  CRUSH rule 0 x 562 [7,13,9,14,2,1]
+  CRUSH rule 0 x 563 [15,4,3,10,13,9]
+  CRUSH rule 0 x 564 [2,13,7,1,15,10]
+  CRUSH rule 0 x 565 [3,12,4,1,14,7]
+  CRUSH rule 0 x 566 [6,14,4,2,13,11]
+  CRUSH rule 0 x 567 [15,4,11,6,3,12]
+  CRUSH rule 0 x 568 [4,14,1,6,10,13]
+  CRUSH rule 0 x 569 [11,3,15,13,5,1]
+  CRUSH rule 0 x 570 [1,10,13,4,7,2]
+  CRUSH rule 0 x 571 [10,12,14,9,4,2]
+  CRUSH rule 0 x 572 [12,14,3,10,6,1]
+  CRUSH rule 0 x 573 [7,15,11,2,12,9]
+  CRUSH rule 0 x 574 [11,14,13,1,3,7]
+  CRUSH rule 0 x 575 [5,13,15,9,6,10]
+  CRUSH rule 0 x 576 [3,15,11,9,1,6]
+  CRUSH rule 0 x 577 [13,9,6,15,3,11]
+  CRUSH rule 0 x 578 [4,10,1,2,7,13]
+  CRUSH rule 0 x 579 [13,1,15,2,10,7]
+  CRUSH rule 0 x 580 [3,12,4,1,10,15]
+  CRUSH rule 0 x 581 [7,14,12,10,1,2]
+  CRUSH rule 0 x 582 [10,5,13,14,1,2]
+  CRUSH rule 0 x 583 [4,15,1,9,10,12]
+  CRUSH rule 0 x 584 [10,1,5,13,6,9]
+  CRUSH rule 0 x 585 [5,3,6,1,11,14]
+  CRUSH rule 0 x 586 [7,10,14,12,9,3]
+  CRUSH rule 0 x 587 [11,6,9,4,1,14]
+  CRUSH rule 0 x 588 [3,12,7,15,4,9]
+  CRUSH rule 0 x 589 [9,7,12,1,10,3]
+  CRUSH rule 0 x 590 [12,1,3,9,10,6]
+  CRUSH rule 0 x 591 [2,6,14,13,9,11]
+  CRUSH rule 0 x 592 [15,12,9,7,5,2]
+  CRUSH rule 0 x 593 [13,14,5,11,9,6]
+  CRUSH rule 0 x 594 [12,14,2,9,7,4]
+  CRUSH rule 0 x 595 [12,7,10,3,1,14]
+  CRUSH rule 0 x 596 [2,7,12,11,1,5]
+  CRUSH rule 0 x 597 [15,1,2,10,7,13]
+  CRUSH rule 0 x 598 [11,5,9,14,12,7]
+  CRUSH rule 0 x 599 [13,11,1,5,6,2]
+  CRUSH rule 0 x 600 [4,12,3,10,9,7]
+  CRUSH rule 0 x 601 [13,5,15,2,1,7]
+  CRUSH rule 0 x 602 [3,11,7,1,13,15]
+  CRUSH rule 0 x 603 [3,1,4,14,10,9]
+  CRUSH rule 0 x 604 [14,2,6,1,11,13]
+  CRUSH rule 0 x 605 [2,7,12,5,14,10]
+  CRUSH rule 0 x 606 [12,15,1,5,7,9]
+  CRUSH rule 0 x 607 [3,9,10,14,7,1]
+  CRUSH rule 0 x 608 [13,10,1,7,9,15]
+  CRUSH rule 0 x 609 [14,3,7,9,11,12]
+  CRUSH rule 0 x 610 [7,10,5,1,12,2]
+  CRUSH rule 0 x 611 [13,1,5,3,10,7]
+  CRUSH rule 0 x 612 [7,1,2,13,9,15]
+  CRUSH rule 0 x 613 [10,7,14,9,5,2]
+  CRUSH rule 0 x 614 [9,4,15,3,1,11]
+  CRUSH rule 0 x 615 [9,4,11,2,1,12]
+  CRUSH rule 0 x 616 [10,14,1,5,3,6]
+  CRUSH rule 0 x 617 [15,7,2,11,12,1]
+  CRUSH rule 0 x 618 [4,2,10,6,14,9]
+  CRUSH rule 0 x 619 [15,4,3,9,6,1]
+  CRUSH rule 0 x 620 [3,7,11,14,13,1]
+  CRUSH rule 0 x 621 [3,6,4,14,1,11]
+  CRUSH rule 0 x 622 [10,2,13,5,15,9]
+  CRUSH rule 0 x 623 [4,9,14,7,3,13]
+  CRUSH rule 0 x 624 [3,9,15,6,10,1]
+  CRUSH rule 0 x 625 [11,7,3,5,13,15]
+  CRUSH rule 0 x 626 [10,12,2,1,9,7]
+  CRUSH rule 0 x 627 [1,12,10,14,3,5]
+  CRUSH rule 0 x 628 [15,13,11,4,2,1]
+  CRUSH rule 0 x 629 [5,6,15,12,1,10]
+  CRUSH rule 0 x 630 [1,4,12,9,3,7]
+  CRUSH rule 0 x 631 [5,7,1,15,12,11]
+  CRUSH rule 0 x 632 [12,3,11,9,6,1]
+  CRUSH rule 0 x 633 [14,4,3,7,10,12]
+  CRUSH rule 0 x 634 [6,9,5,3,13,11]
+  CRUSH rule 0 x 635 [6,5,2,15,9,12]
+  CRUSH rule 0 x 636 [13,6,11,3,15,9]
+  CRUSH rule 0 x 637 [3,1,10,6,9,12]
+  CRUSH rule 0 x 638 [10,15,3,5,13,1]
+  CRUSH rule 0 x 639 [6,9,14,4,3,1]
+  CRUSH rule 0 x 640 [9,6,1,11,14,2]
+  CRUSH rule 0 x 641 [10,6,5,14,1,9]
+  CRUSH rule 0 x 642 [1,15,4,6,2,10]
+  CRUSH rule 0 x 643 [3,7,5,1,10,15]
+  CRUSH rule 0 x 644 [15,13,6,9,3,11]
+  CRUSH rule 0 x 645 [14,2,4,9,10,1]
+  CRUSH rule 0 x 646 [5,13,14,1,6,9]
+  CRUSH rule 0 x 647 [10,1,9,13,6,2]
+  CRUSH rule 0 x 648 [6,5,2,14,11,1]
+  CRUSH rule 0 x 649 [3,9,13,11,4,14]
+  CRUSH rule 0 x 650 [10,9,4,15,12,7]
+  CRUSH rule 0 x 651 [3,9,5,7,14,1]
+  CRUSH rule 0 x 652 [15,9,4,6,13,1]
+  CRUSH rule 0 x 653 [11,14,1,3,6,9]
+  CRUSH rule 0 x 654 [13,6,2,10,15,4]
+  CRUSH rule 0 x 655 [6,3,4,15,12,11]
+  CRUSH rule 0 x 656 [3,15,1,4,6,12]
+  CRUSH rule 0 x 657 [11,15,3,5,7,13]
+  CRUSH rule 0 x 658 [7,2,10,12,1,4]
+  CRUSH rule 0 x 659 [2,5,14,6,10,12]
+  CRUSH rule 0 x 660 [13,14,10,6,4,9]
+  CRUSH rule 0 x 661 [7,15,3,12,11,4]
+  CRUSH rule 0 x 662 [15,2,12,5,1,10]
+  CRUSH rule 0 x 663 [14,9,13,10,5,3]
+  CRUSH rule 0 x 664 [6,10,12,4,9,2]
+  CRUSH rule 0 x 665 [2,9,12,1,7,10]
+  CRUSH rule 0 x 666 [12,3,6,1,15,9]
+  CRUSH rule 0 x 667 [1,9,12,10,2,14]
+  CRUSH rule 0 x 668 [9,5,1,2,6,11]
+  CRUSH rule 0 x 669 [9,7,14,5,11,13]
+  CRUSH rule 0 x 670 [6,10,9,13,1,2]
+  CRUSH rule 0 x 671 [6,15,5,10,13,3]
+  CRUSH rule 0 x 672 [2,9,13,1,4,14]
+  CRUSH rule 0 x 673 [7,10,5,9,15,13]
+  CRUSH rule 0 x 674 [7,12,10,1,14,9]
+  CRUSH rule 0 x 675 [9,5,1,10,6,14]
+  CRUSH rule 0 x 676 [10,12,2,1,4,15]
+  CRUSH rule 0 x 677 [2,12,1,4,10,6]
+  CRUSH rule 0 x 678 [1,2,4,10,12,14]
+  CRUSH rule 0 x 679 [5,6,12,15,9,11]
+  CRUSH rule 0 x 680 [7,11,3,1,15,4]
+  CRUSH rule 0 x 681 [6,4,3,11,14,13]
+  CRUSH rule 0 x 682 [6,1,11,15,12,2]
+  CRUSH rule 0 x 683 [6,13,2,4,9,14]
+  CRUSH rule 0 x 684 [9,11,3,7,15,4]
+  CRUSH rule 0 x 685 [5,1,15,7,9,2]
+  CRUSH rule 0 x 686 [1,9,11,14,6,13]
+  CRUSH rule 0 x 687 [7,13,3,5,11,9]
+  CRUSH rule 0 x 688 [11,9,1,14,3,5]
+  CRUSH rule 0 x 689 [5,2,9,12,1,14]
+  CRUSH rule 0 x 690 [9,7,10,3,13,15]
+  CRUSH rule 0 x 691 [11,15,9,5,7,13]
+  CRUSH rule 0 x 692 [15,5,1,2,9,11]
+  CRUSH rule 0 x 693 [5,6,12,15,2,10]
+  CRUSH rule 0 x 694 [4,7,1,10,12,3]
+  CRUSH rule 0 x 695 [6,13,14,10,9,5]
+  CRUSH rule 0 x 696 [1,2,4,14,7,11]
+  CRUSH rule 0 x 697 [13,11,3,6,4,14]
+  CRUSH rule 0 x 698 [11,13,4,2,6,1]
+  CRUSH rule 0 x 699 [7,14,12,4,2,11]
+  CRUSH rule 0 x 700 [12,14,11,9,4,6]
+  CRUSH rule 0 x 701 [3,13,1,14,4,7]
+  CRUSH rule 0 x 702 [3,12,15,6,5,11]
+  CRUSH rule 0 x 703 [15,11,13,3,4,7]
+  CRUSH rule 0 x 704 [6,4,2,15,11,1]
+  CRUSH rule 0 x 705 [14,6,11,5,1,13]
+  CRUSH rule 0 x 706 [1,12,3,6,4,10]
+  CRUSH rule 0 x 707 [4,7,14,3,10,9]
+  CRUSH rule 0 x 708 [3,10,5,1,15,9]
+  CRUSH rule 0 x 709 [11,12,3,7,5,14]
+  CRUSH rule 0 x 710 [14,2,11,9,5,7]
+  CRUSH rule 0 x 711 [14,3,9,10,12,5]
+  CRUSH rule 0 x 712 [12,3,11,15,9,1]
+  CRUSH rule 0 x 713 [11,9,3,15,13,6]
+  CRUSH rule 0 x 714 [12,1,9,7,2,15]
+  CRUSH rule 0 x 715 [6,1,14,4,11,12]
+  CRUSH rule 0 x 716 [11,13,9,14,5,2]
+  CRUSH rule 0 x 717 [12,4,10,9,15,1]
+  CRUSH rule 0 x 718 [7,15,5,2,11,13]
+  CRUSH rule 0 x 719 [5,15,13,3,1,7]
+  CRUSH rule 0 x 720 [4,13,10,2,7,9]
+  CRUSH rule 0 x 721 [11,3,14,9,1,12]
+  CRUSH rule 0 x 722 [2,4,6,1,9,15]
+  CRUSH rule 0 x 723 [2,1,12,15,11,7]
+  CRUSH rule 0 x 724 [7,1,9,10,5,15]
+  CRUSH rule 0 x 725 [11,12,7,15,4,1]
+  CRUSH rule 0 x 726 [7,14,4,3,11,13]
+  CRUSH rule 0 x 727 [2,5,1,11,15,7]
+  CRUSH rule 0 x 728 [13,11,4,6,15,2]
+  CRUSH rule 0 x 729 [15,11,4,6,2,9]
+  CRUSH rule 0 x 730 [3,7,1,13,11,15]
+  CRUSH rule 0 x 731 [9,1,6,5,2,11]
+  CRUSH rule 0 x 732 [1,2,10,13,9,4]
+  CRUSH rule 0 x 733 [11,3,5,6,1,9]
+  CRUSH rule 0 x 734 [14,3,11,7,12,9]
+  CRUSH rule 0 x 735 [6,9,2,10,13,14]
+  CRUSH rule 0 x 736 [3,9,1,11,7,5]
+  CRUSH rule 0 x 737 [1,4,2,12,9,10]
+  CRUSH rule 0 x 738 [11,15,7,4,9,2]
+  CRUSH rule 0 x 739 [11,12,6,2,4,1]
+  CRUSH rule 0 x 740 [7,9,10,13,1,15]
+  CRUSH rule 0 x 741 [12,11,7,15,2,5]
+  CRUSH rule 0 x 742 [9,7,4,11,12,1]
+  CRUSH rule 0 x 743 [5,13,9,15,10,7]
+  CRUSH rule 0 x 744 [6,2,13,1,14,11]
+  CRUSH rule 0 x 745 [3,6,1,4,11,12]
+  CRUSH rule 0 x 746 [3,7,9,10,14,5]
+  CRUSH rule 0 x 747 [15,11,5,2,13,9]
+  CRUSH rule 0 x 748 [6,10,13,2,14,5]
+  CRUSH rule 0 x 749 [14,9,10,7,5,1]
+  CRUSH rule 0 x 750 [1,14,6,5,11,2]
+  CRUSH rule 0 x 751 [15,1,6,9,5,11]
+  CRUSH rule 0 x 752 [13,1,7,3,11,15]
+  CRUSH rule 0 x 753 [4,11,1,3,15,7]
+  CRUSH rule 0 x 754 [14,12,11,4,2,1]
+  CRUSH rule 0 x 755 [13,6,1,10,4,2]
+  CRUSH rule 0 x 756 [3,4,14,6,1,10]
+  CRUSH rule 0 x 757 [10,6,1,4,13,15]
+  CRUSH rule 0 x 758 [6,3,4,10,15,13]
+  CRUSH rule 0 x 759 [5,7,3,14,11,1]
+  CRUSH rule 0 x 760 [1,15,10,12,4,3]
+  CRUSH rule 0 x 761 [2,12,1,14,5,7]
+  CRUSH rule 0 x 762 [1,4,10,9,3,7]
+  CRUSH rule 0 x 763 [4,13,1,14,7,10]
+  CRUSH rule 0 x 764 [1,14,6,13,9,5]
+  CRUSH rule 0 x 765 [9,15,2,13,4,1]
+  CRUSH rule 0 x 766 [11,2,7,15,9,12]
+  CRUSH rule 0 x 767 [6,11,4,3,12,14]
+  CRUSH rule 0 x 768 [2,12,15,7,1,11]
+  CRUSH rule 0 x 769 [15,1,9,2,11,12]
+  CRUSH rule 0 x 770 [15,13,4,6,3,10]
+  CRUSH rule 0 x 771 [9,2,12,11,6,14]
+  CRUSH rule 0 x 772 [4,3,13,11,14,1]
+  CRUSH rule 0 x 773 [3,7,4,15,1,12]
+  CRUSH rule 0 x 774 [12,6,3,15,5,9]
+  CRUSH rule 0 x 775 [5,10,14,2,6,1]
+  CRUSH rule 0 x 776 [10,15,3,9,6,13]
+  CRUSH rule 0 x 777 [11,13,4,7,1,14]
+  CRUSH rule 0 x 778 [13,1,9,11,15,6]
+  CRUSH rule 0 x 779 [5,11,1,14,2,9]
+  CRUSH rule 0 x 780 [13,9,3,6,4,1]
+  CRUSH rule 0 x 781 [5,7,14,3,1,12]
+  CRUSH rule 0 x 782 [2,15,9,7,11,13]
+  CRUSH rule 0 x 783 [12,7,5,14,9,1]
+  CRUSH rule 0 x 784 [14,1,10,13,3,4]
+  CRUSH rule 0 x 785 [6,12,1,2,4,9]
+  CRUSH rule 0 x 786 [10,5,2,15,1,7]
+  CRUSH rule 0 x 787 [1,12,10,2,9,4]
+  CRUSH rule 0 x 788 [4,2,9,13,6,15]
+  CRUSH rule 0 x 789 [9,2,14,7,4,12]
+  CRUSH rule 0 x 790 [15,2,7,4,1,10]
+  CRUSH rule 0 x 791 [9,4,7,13,14,11]
+  CRUSH rule 0 x 792 [6,4,15,10,12,3]
+  CRUSH rule 0 x 793 [15,9,6,2,13,11]
+  CRUSH rule 0 x 794 [5,12,2,14,9,10]
+  CRUSH rule 0 x 795 [6,14,12,4,10,1]
+  CRUSH rule 0 x 796 [11,2,12,6,15,4]
+  CRUSH rule 0 x 797 [14,3,7,1,5,13]
+  CRUSH rule 0 x 798 [5,11,6,13,1,3]
+  CRUSH rule 0 x 799 [2,9,14,4,13,6]
+  CRUSH rule 0 x 800 [6,3,4,11,15,13]
+  CRUSH rule 0 x 801 [2,5,6,13,9,1]
+  CRUSH rule 0 x 802 [1,4,12,7,3,9]
+  CRUSH rule 0 x 803 [7,2,4,1,11,13]
+  CRUSH rule 0 x 804 [5,14,9,7,3,1]
+  CRUSH rule 0 x 805 [13,4,3,1,10,15]
+  CRUSH rule 0 x 806 [6,2,13,4,15,1]
+  CRUSH rule 0 x 807 [14,2,7,4,9,12]
+  CRUSH rule 0 x 808 [2,15,12,7,9,1]
+  CRUSH rule 0 x 809 [1,11,7,12,4,2]
+  CRUSH rule 0 x 810 [2,5,9,12,15,1]
+  CRUSH rule 0 x 811 [15,6,3,10,1,5]
+  CRUSH rule 0 x 812 [7,11,2,14,9,5]
+  CRUSH rule 0 x 813 [4,10,13,14,2,6]
+  CRUSH rule 0 x 814 [13,4,9,3,10,6]
+  CRUSH rule 0 x 815 [15,12,9,4,10,6]
+  CRUSH rule 0 x 816 [14,10,13,7,3,9]
+  CRUSH rule 0 x 817 [10,7,2,15,13,9]
+  CRUSH rule 0 x 818 [15,2,11,4,1,12]
+  CRUSH rule 0 x 819 [5,12,10,6,1,14]
+  CRUSH rule 0 x 820 [3,6,9,12,11,15]
+  CRUSH rule 0 x 821 [15,10,9,13,3,4]
+  CRUSH rule 0 x 822 [10,13,2,9,7,4]
+  CRUSH rule 0 x 823 [2,6,12,10,15,4]
+  CRUSH rule 0 x 824 [3,7,9,13,15,5]
+  CRUSH rule 0 x 825 [10,5,14,6,12,9]
+  CRUSH rule 0 x 826 [5,2,11,15,1,12]
+  CRUSH rule 0 x 827 [13,5,1,3,7,9]
+  CRUSH rule 0 x 828 [12,6,10,5,1,9]
+  CRUSH rule 0 x 829 [13,6,15,10,5,3]
+  CRUSH rule 0 x 830 [15,13,2,9,7,11]
+  CRUSH rule 0 x 831 [1,4,11,12,6,3]
+  CRUSH rule 0 x 832 [14,11,13,2,9,4]
+  CRUSH rule 0 x 833 [9,13,3,11,7,5]
+  CRUSH rule 0 x 834 [9,7,5,1,11,2]
+  CRUSH rule 0 x 835 [14,3,13,6,4,9]
+  CRUSH rule 0 x 836 [3,9,10,13,1,5]
+  CRUSH rule 0 x 837 [15,12,11,2,7,9]
+  CRUSH rule 0 x 838 [12,14,9,2,5,7]
+  CRUSH rule 0 x 839 [3,4,6,10,15,1]
+  CRUSH rule 0 x 840 [10,15,12,4,7,1]
+  CRUSH rule 0 x 841 [3,5,7,12,11,15]
+  CRUSH rule 0 x 842 [9,13,2,6,5,14]
+  CRUSH rule 0 x 843 [14,7,4,9,3,12]
+  CRUSH rule 0 x 844 [7,1,4,15,9,2]
+  CRUSH rule 0 x 845 [13,6,1,15,4,2]
+  CRUSH rule 0 x 846 [3,7,15,13,1,9]
+  CRUSH rule 0 x 847 [12,15,11,5,2,7]
+  CRUSH rule 0 x 848 [11,13,1,14,5,9]
+  CRUSH rule 0 x 849 [3,15,11,9,6,1]
+  CRUSH rule 0 x 850 [1,3,10,6,14,4]
+  CRUSH rule 0 x 851 [14,4,3,6,11,1]
+  CRUSH rule 0 x 852 [9,12,4,7,15,2]
+  CRUSH rule 0 x 853 [13,14,6,11,2,4]
+  CRUSH rule 0 x 854 [7,11,12,1,4,15]
+  CRUSH rule 0 x 855 [14,4,12,6,3,1]
+  CRUSH rule 0 x 856 [5,10,7,3,15,9]
+  CRUSH rule 0 x 857 [4,3,13,11,9,1]
+  CRUSH rule 0 x 858 [5,15,6,3,9,12]
+  CRUSH rule 0 x 859 [5,15,6,2,1,11]
+  CRUSH rule 0 x 860 [11,14,1,12,6,9]
+  CRUSH rule 0 x 861 [13,7,4,10,1,14]
+  CRUSH rule 0 x 862 [5,10,9,7,3,12]
+  CRUSH rule 0 x 863 [11,6,3,9,4,12]
+  CRUSH rule 0 x 864 [6,13,4,2,10,15]
+  CRUSH rule 0 x 865 [4,1,14,11,6,9]
+  CRUSH rule 0 x 866 [2,13,4,15,9,6]
+  CRUSH rule 0 x 867 [12,2,9,10,4,14]
+  CRUSH rule 0 x 868 [14,11,7,2,1,4]
+  CRUSH rule 0 x 869 [10,13,7,14,3,5]
+  CRUSH rule 0 x 870 [14,9,11,4,3,12]
+  CRUSH rule 0 x 871 [6,2,1,4,15,13]
+  CRUSH rule 0 x 872 [6,1,15,3,10,12]
+  CRUSH rule 0 x 873 [2,5,12,10,1,9]
+  CRUSH rule 0 x 874 [12,4,7,2,15,10]
+  CRUSH rule 0 x 875 [10,6,14,1,12,5]
+  CRUSH rule 0 x 876 [14,7,13,3,9,1]
+  CRUSH rule 0 x 877 [15,11,13,9,5,1]
+  CRUSH rule 0 x 878 [7,14,3,13,9,1]
+  CRUSH rule 0 x 879 [12,2,7,4,10,15]
+  CRUSH rule 0 x 880 [2,12,10,7,1,4]
+  CRUSH rule 0 x 881 [6,3,1,11,4,15]
+  CRUSH rule 0 x 882 [11,13,7,1,2,15]
+  CRUSH rule 0 x 883 [13,1,3,10,6,5]
+  CRUSH rule 0 x 884 [6,15,4,9,3,11]
+  CRUSH rule 0 x 885 [14,7,9,4,2,13]
+  CRUSH rule 0 x 886 [13,11,4,2,1,14]
+  CRUSH rule 0 x 887 [14,4,12,11,2,6]
+  CRUSH rule 0 x 888 [10,12,7,15,9,2]
+  CRUSH rule 0 x 889 [15,13,4,1,6,2]
+  CRUSH rule 0 x 890 [10,12,14,2,9,5]
+  CRUSH rule 0 x 891 [9,5,11,6,3,15]
+  CRUSH rule 0 x 892 [12,15,2,4,7,9]
+  CRUSH rule 0 x 893 [1,3,5,9,6,10]
+  CRUSH rule 0 x 894 [7,2,11,13,4,1]
+  CRUSH rule 0 x 895 [2,1,11,5,7,15]
+  CRUSH rule 0 x 896 [9,1,14,10,4,12]
+  CRUSH rule 0 x 897 [7,5,14,3,1,9]
+  CRUSH rule 0 x 898 [10,6,12,9,15,5]
+  CRUSH rule 0 x 899 [1,11,5,3,13,14]
+  CRUSH rule 0 x 900 [2,9,10,7,13,14]
+  CRUSH rule 0 x 901 [9,12,11,3,14,4]
+  CRUSH rule 0 x 902 [4,2,6,15,12,10]
+  CRUSH rule 0 x 903 [14,10,3,1,12,6]
+  CRUSH rule 0 x 904 [15,12,4,9,6,3]
+  CRUSH rule 0 x 905 [12,6,11,3,9,4]
+  CRUSH rule 0 x 906 [14,11,12,2,4,9]
+  CRUSH rule 0 x 907 [7,12,3,9,10,5]
+  CRUSH rule 0 x 908 [2,15,9,6,10,13]
+  CRUSH rule 0 x 909 [10,14,1,13,2,9]
+  CRUSH rule 0 x 910 [12,7,4,15,10,3]
+  CRUSH rule 0 x 911 [11,15,2,4,9,13]
+  CRUSH rule 0 x 912 [6,4,14,13,3,1]
+  CRUSH rule 0 x 913 [4,6,10,1,12,3]
+  CRUSH rule 0 x 914 [4,15,2,10,1,13]
+  CRUSH rule 0 x 915 [12,14,1,9,4,3]
+  CRUSH rule 0 x 916 [3,1,11,5,6,13]
+  CRUSH rule 0 x 917 [1,15,6,5,10,3]
+  CRUSH rule 0 x 918 [7,14,11,4,9,2]
+  CRUSH rule 0 x 919 [10,7,3,13,15,1]
+  CRUSH rule 0 x 920 [4,2,10,15,1,13]
+  CRUSH rule 0 x 921 [1,11,6,13,4,2]
+  CRUSH rule 0 x 922 [6,4,14,13,3,1]
+  CRUSH rule 0 x 923 [12,2,5,14,10,1]
+  CRUSH rule 0 x 924 [6,2,14,13,9,1]
+  CRUSH rule 0 x 925 [12,15,2,10,1,5]
+  CRUSH rule 0 x 926 [3,13,10,1,14,9]
+  CRUSH rule 0 x 927 [6,5,1,11,14,2]
+  CRUSH rule 0 x 928 [13,1,3,9,6,11]
+  CRUSH rule 0 x 929 [10,7,1,5,2,12]
+  CRUSH rule 0 x 930 [7,15,10,5,1,13]
+  CRUSH rule 0 x 931 [6,15,11,9,5,3]
+  CRUSH rule 0 x 932 [13,2,5,11,9,1]
+  CRUSH rule 0 x 933 [12,7,14,10,4,1]
+  CRUSH rule 0 x 934 [12,2,5,7,9,1]
+  CRUSH rule 0 x 935 [6,11,1,14,5,13]
+  CRUSH rule 0 x 936 [9,12,7,5,1,2]
+  CRUSH rule 0 x 937 [14,2,11,1,13,4]
+  CRUSH rule 0 x 938 [14,3,5,11,7,9]
+  CRUSH rule 0 x 939 [6,4,14,9,12,1]
+  CRUSH rule 0 x 940 [13,11,4,2,1,6]
+  CRUSH rule 0 x 941 [3,12,4,7,14,10]
+  CRUSH rule 0 x 942 [15,12,10,4,1,9]
+  CRUSH rule 0 x 943 [10,2,4,9,6,15]
+  CRUSH rule 0 x 944 [2,9,4,7,1,14]
+  CRUSH rule 0 x 945 [10,15,2,9,5,12]
+  CRUSH rule 0 x 946 [11,15,7,12,5,9]
+  CRUSH rule 0 x 947 [11,3,14,1,12,5]
+  CRUSH rule 0 x 948 [7,13,11,5,14,2]
+  CRUSH rule 0 x 949 [9,1,12,5,15,10]
+  CRUSH rule 0 x 950 [9,15,13,6,4,2]
+  CRUSH rule 0 x 951 [2,6,12,9,10,4]
+  CRUSH rule 0 x 952 [9,7,15,3,5,13]
+  CRUSH rule 0 x 953 [1,3,6,10,12,14]
+  CRUSH rule 0 x 954 [10,2,14,9,4,6]
+  CRUSH rule 0 x 955 [7,14,3,1,10,4]
+  CRUSH rule 0 x 956 [1,6,11,5,14,3]
+  CRUSH rule 0 x 957 [14,11,1,12,6,9]
+  CRUSH rule 0 x 958 [15,4,3,11,1,6]
+  CRUSH rule 0 x 959 [2,1,12,15,10,9]
+  CRUSH rule 0 x 960 [2,6,11,13,15,4]
+  CRUSH rule 0 x 961 [3,13,11,9,6,1]
+  CRUSH rule 0 x 962 [5,11,3,14,1,6]
+  CRUSH rule 0 x 963 [13,10,15,4,6,9]
+  CRUSH rule 0 x 964 [7,11,4,9,2,12]
+  CRUSH rule 0 x 965 [12,2,9,7,4,15]
+  CRUSH rule 0 x 966 [12,14,9,4,1,2]
+  CRUSH rule 0 x 967 [7,5,3,10,12,14]
+  CRUSH rule 0 x 968 [12,15,4,9,11,6]
+  CRUSH rule 0 x 969 [11,4,7,1,9,14]
+  CRUSH rule 0 x 970 [5,12,10,1,3,14]
+  CRUSH rule 0 x 971 [1,9,4,12,7,2]
+  CRUSH rule 0 x 972 [12,3,14,5,1,9]
+  CRUSH rule 0 x 973 [1,10,4,12,2,7]
+  CRUSH rule 0 x 974 [7,11,1,2,15,4]
+  CRUSH rule 0 x 975 [7,9,15,12,2,11]
+  CRUSH rule 0 x 976 [7,3,15,5,12,11]
+  CRUSH rule 0 x 977 [14,3,6,10,4,1]
+  CRUSH rule 0 x 978 [12,5,11,1,15,3]
+  CRUSH rule 0 x 979 [5,1,13,6,15,10]
+  CRUSH rule 0 x 980 [15,11,5,6,1,3]
+  CRUSH rule 0 x 981 [5,11,15,12,7,1]
+  CRUSH rule 0 x 982 [2,6,14,11,12,9]
+  CRUSH rule 0 x 983 [3,12,10,9,14,5]
+  CRUSH rule 0 x 984 [15,13,1,10,2,5]
+  CRUSH rule 0 x 985 [11,2,15,1,4,13]
+  CRUSH rule 0 x 986 [6,13,9,1,15,10]
+  CRUSH rule 0 x 987 [13,14,5,10,6,1]
+  CRUSH rule 0 x 988 [12,9,10,14,3,1]
+  CRUSH rule 0 x 989 [7,4,3,15,9,13]
+  CRUSH rule 0 x 990 [1,10,9,13,3,4]
+  CRUSH rule 0 x 991 [7,11,1,14,2,5]
+  CRUSH rule 0 x 992 [9,10,2,13,7,4]
+  CRUSH rule 0 x 993 [6,10,14,12,4,1]
+  CRUSH rule 0 x 994 [3,13,15,4,11,7]
+  CRUSH rule 0 x 995 [15,6,12,2,5,11]
+  CRUSH rule 0 x 996 [15,10,5,3,13,1]
+  CRUSH rule 0 x 997 [15,2,1,12,7,9]
+  CRUSH rule 0 x 998 [6,1,9,5,12,11]
+  CRUSH rule 0 x 999 [9,10,15,5,13,3]
+  CRUSH rule 0 x 1000 [14,2,9,4,12,1]
+  CRUSH rule 0 x 1001 [11,14,4,2,6,9]
+  CRUSH rule 0 x 1002 [1,10,14,2,9,5]
+  CRUSH rule 0 x 1003 [10,7,5,14,2,1]
+  CRUSH rule 0 x 1004 [15,1,4,6,10,12]
+  CRUSH rule 0 x 1005 [6,12,2,10,9,15]
+  CRUSH rule 0 x 1006 [10,12,15,1,2,6]
+  CRUSH rule 0 x 1007 [1,7,13,14,3,4]
+  CRUSH rule 0 x 1008 [7,4,9,11,3,15]
+  CRUSH rule 0 x 1009 [5,2,11,7,15,9]
+  CRUSH rule 0 x 1010 [10,2,15,6,9,13]
+  CRUSH rule 0 x 1011 [6,3,12,1,10,4]
+  CRUSH rule 0 x 1012 [12,6,9,15,3,1]
+  CRUSH rule 0 x 1013 [2,14,12,4,9,1]
+  CRUSH rule 0 x 1014 [1,13,7,2,10,14]
+  CRUSH rule 0 x 1015 [12,6,10,1,4,15]
+  CRUSH rule 0 x 1016 [10,13,14,3,5,6]
+  CRUSH rule 0 x 1017 [5,11,14,7,13,9]
+  CRUSH rule 0 x 1018 [13,11,14,1,9,3]
+  CRUSH rule 0 x 1019 [10,13,14,7,5,1]
+  CRUSH rule 0 x 1020 [3,1,13,4,10,9]
+  CRUSH rule 0 x 1021 [2,11,14,9,4,6]
+  CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+  CRUSH rule 0 x 1023 [15,2,9,12,1,7]
+  rule 0 (replicated_ruleset) num_rep 6 result size == 6:\t1024/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3,15,12,1,4]
+  CRUSH rule 0 x 1 [10,15,1,2,13,4,7]
+  CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+  CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+  CRUSH rule 0 x 4 [14,2,10,1,9,4,7]
+  CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+  CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+  CRUSH rule 0 x 7 [9,2,6,12,11,4,1]
+  CRUSH rule 0 x 8 [10,2,15,1,4,13,6]
+  CRUSH rule 0 x 9 [7,1,14,2,11,9,12]
+  CRUSH rule 0 x 10 [10,14,4,1,2,7,13]
+  CRUSH rule 0 x 11 [13,9,14,7,5,11,2]
+  CRUSH rule 0 x 12 [7,1,2,5,13,15,11]
+  CRUSH rule 0 x 13 [3,5,12,7,9,1,14]
+  CRUSH rule 0 x 14 [13,5,2,7,10,15,1]
+  CRUSH rule 0 x 15 [15,1,9,6,13,3,5]
+  CRUSH rule 0 x 16 [7,11,14,2,13,1,9]
+  CRUSH rule 0 x 17 [10,1,13,2,4,6,14]
+  CRUSH rule 0 x 18 [1,7,3,10,5,12,9]
+  CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+  CRUSH rule 0 x 20 [14,12,3,10,9,4,7]
+  CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+  CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+  CRUSH rule 0 x 23 [10,5,13,9,3,15,1]
+  CRUSH rule 0 x 24 [12,11,3,1,9,4,7]
+  CRUSH rule 0 x 25 [7,12,15,1,3,10,4]
+  CRUSH rule 0 x 26 [1,7,13,2,14,5,9]
+  CRUSH rule 0 x 27 [3,6,15,4,13,9,11]
+  CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+  CRUSH rule 0 x 29 [5,14,12,11,6,3,1]
+  CRUSH rule 0 x 30 [2,5,6,9,1,11,13]
+  CRUSH rule 0 x 31 [5,15,10,1,9,13,6]
+  CRUSH rule 0 x 32 [9,10,2,1,13,14,6]
+  CRUSH rule 0 x 33 [13,4,9,2,7,1,10]
+  CRUSH rule 0 x 34 [13,15,2,4,1,10,9]
+  CRUSH rule 0 x 35 [4,14,3,13,10,9,1]
+  CRUSH rule 0 x 36 [3,12,9,7,5,10,14]
+  CRUSH rule 0 x 37 [9,2,6,14,11,1,4]
+  CRUSH rule 0 x 38 [3,4,13,10,9,1,14]
+  CRUSH rule 0 x 39 [12,7,14,11,1,9,5]
+  CRUSH rule 0 x 40 [10,1,9,5,15,2,6]
+  CRUSH rule 0 x 41 [4,9,11,1,14,13,6]
+  CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+  CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+  CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+  CRUSH rule 0 x 45 [11,12,15,9,1,5,6]
+  CRUSH rule 0 x 46 [6,9,2,14,11,13,1]
+  CRUSH rule 0 x 47 [3,9,6,4,13,1,11]
+  CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+  CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+  CRUSH rule 0 x 50 [14,12,1,4,2,11,6]
+  CRUSH rule 0 x 51 [10,6,5,12,15,2,1]
+  CRUSH rule 0 x 52 [12,1,9,11,7,3,14]
+  CRUSH rule 0 x 53 [3,6,13,9,5,1,11]
+  CRUSH rule 0 x 54 [4,13,9,2,14,10,6]
+  CRUSH rule 0 x 55 [4,11,2,7,1,13,9]
+  CRUSH rule 0 x 56 [5,9,10,1,3,13,14]
+  CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+  CRUSH rule 0 x 58 [7,1,11,4,3,14,12]
+  CRUSH rule 0 x 59 [2,13,1,10,9,5,14]
+  CRUSH rule 0 x 60 [3,6,11,1,4,9,12]
+  CRUSH rule 0 x 61 [3,15,13,7,4,1,10]
+  CRUSH rule 0 x 62 [15,11,7,12,5,9,2]
+  CRUSH rule 0 x 63 [10,14,12,1,7,3]
+  CRUSH rule 0 x 64 [3,9,1,4,7,12,11]
+  CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+  CRUSH rule 0 x 66 [15,11,6,9,4,1,3]
+  CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+  CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+  CRUSH rule 0 x 69 [2,1,15,10,4,9,13]
+  CRUSH rule 0 x 70 [9,6,1,3,13,15,11]
+  CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+  CRUSH rule 0 x 72 [9,10,3,5,7,12,15]
+  CRUSH rule 0 x 73 [5,3,11,1,7,12,15]
+  CRUSH rule 0 x 74 [11,7,9,5,1,15,3]
+  CRUSH rule 0 x 75 [9,7,11,14,12,1,2]
+  CRUSH rule 0 x 76 [6,1,3,5,14,10,12]
+  CRUSH rule 0 x 77 [7,4,2,13,9,1,11]
+  CRUSH rule 0 x 78 [9,3,1,5,6,13,14]
+  CRUSH rule 0 x 79 [13,2,15,5,7,9,11]
+  CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+  CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+  CRUSH rule 0 x 82 [14,13,5,11,6,2,1]
+  CRUSH rule 0 x 83 [4,15,3,9,10,13,6]
+  CRUSH rule 0 x 84 [10,7,9,15,3,4,1]
+  CRUSH rule 0 x 85 [3,15,9,7,4,11,1]
+  CRUSH rule 0 x 86 [10,9,14,1,13,4,2]
+  CRUSH rule 0 x 87 [15,10,7,12,5,3,9]
+  CRUSH rule 0 x 88 [4,13,3,1,9,15,11]
+  CRUSH rule 0 x 89 [3,9,7,4,1,14,10]
+  CRUSH rule 0 x 90 [4,9,7,12,11,14,2]
+  CRUSH rule 0 x 91 [6,11,9,1,2,4,14]
+  CRUSH rule 0 x 92 [1,5,10,9,13,15,6]
+  CRUSH rule 0 x 93 [9,3,15,13,7,5,1]
+  CRUSH rule 0 x 94 [9,2,12,5,6,11,1]
+  CRUSH rule 0 x 95 [7,15,4,10,9,13,2]
+  CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+  CRUSH rule 0 x 97 [4,11,2,13,1,7,9]
+  CRUSH rule 0 x 98 [11,13,9,3,15,1,5]
+  CRUSH rule 0 x 99 [12,4,11,7,3,14,9]
+  CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+  CRUSH rule 0 x 101 [15,7,1,9,10,5,2]
+  CRUSH rule 0 x 102 [3,11,14,6,13,4,9]
+  CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+  CRUSH rule 0 x 104 [14,6,3,5,9,1,10]
+  CRUSH rule 0 x 105 [14,10,1,9,3,5,6]
+  CRUSH rule 0 x 106 [6,5,13,2,14,11,1]
+  CRUSH rule 0 x 107 [3,1,10,14,13,5,9]
+  CRUSH rule 0 x 108 [5,10,7,2,15,9,12]
+  CRUSH rule 0 x 109 [9,1,13,7,15,5,11]
+  CRUSH rule 0 x 110 [5,1,11,3,7,14,13]
+  CRUSH rule 0 x 111 [10,1,9,7,5,2,13]
+  CRUSH rule 0 x 112 [1,10,4,14,2,12,6]
+  CRUSH rule 0 x 113 [6,10,13,9,1,5,2]
+  CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+  CRUSH rule 0 x 115 [10,13,14,3,9,1,6]
+  CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+  CRUSH rule 0 x 117 [5,6,1,12,15,9,11]
+  CRUSH rule 0 x 118 [10,4,13,15,9,3,1]
+  CRUSH rule 0 x 119 [14,12,11,4,6,9,3]
+  CRUSH rule 0 x 120 [11,3,14,13,4,7]
+  CRUSH rule 0 x 121 [9,5,1,11,7,3,15]
+  CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+  CRUSH rule 0 x 123 [3,10,5,6,9,1,12]
+  CRUSH rule 0 x 124 [12,2,1,5,14,7,11]
+  CRUSH rule 0 x 125 [9,12,15,1,6,5,3]
+  CRUSH rule 0 x 126 [7,15,10,9,2,12,5]
+  CRUSH rule 0 x 127 [4,14,9,13,1,3,7]
+  CRUSH rule 0 x 128 [3,12,1,10,4,9,7]
+  CRUSH rule 0 x 129 [11,13,14,2,9,4,6]
+  CRUSH rule 0 x 130 [3,13,5,14,10,1,9]
+  CRUSH rule 0 x 131 [12,1,6,15,4,2,11]
+  CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+  CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+  CRUSH rule 0 x 134 [12,5,6,15,3,9,10]
+  CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+  CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+  CRUSH rule 0 x 137 [14,3,6,11,1,9,4]
+  CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+  CRUSH rule 0 x 139 [11,2,13,9,1,15,7]
+  CRUSH rule 0 x 140 [11,4,12,15,2,6,9]
+  CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+  CRUSH rule 0 x 142 [3,14,7,9,11,1,4]
+  CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+  CRUSH rule 0 x 144 [13,7,11,2,14,4,1]
+  CRUSH rule 0 x 145 [12,2,6,10,9,4,14]
+  CRUSH rule 0 x 146 [1,5,9,2,6,13,14]
+  CRUSH rule 0 x 147 [1,4,9,11,2,7,15]
+  CRUSH rule 0 x 148 [12,7,9,2,14,11,1]
+  CRUSH rule 0 x 149 [2,5,9,12,11,1,7]
+  CRUSH rule 0 x 150 [1,15,2,10,7,9,5]
+  CRUSH rule 0 x 151 [2,9,14,7,1,10,5]
+  CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+  CRUSH rule 0 x 153 [6,9,4,15,2,1,10]
+  CRUSH rule 0 x 154 [3,11,7,1,4,12,15]
+  CRUSH rule 0 x 155 [14,12,7,3,5,1,9]
+  CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+  CRUSH rule 0 x 157 [15,1,6,4,3,10,9]
+  CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+  CRUSH rule 0 x 159 [4,14,3,12,10,6,1]
+  CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+  CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+  CRUSH rule 0 x 162 [10,6,1,12,2,4,14]
+  CRUSH rule 0 x 163 [15,1,10,2,6,4,13]
+  CRUSH rule 0 x 164 [9,14,10,7,12,2,5]
+  CRUSH rule 0 x 165 [11,7,2,13,9,15,1]
+  CRUSH rule 0 x 166 [1,2,12,14,4,11,7]
+  CRUSH rule 0 x 167 [9,7,3,4,11,13,15]
+  CRUSH rule 0 x 168 [13,2,4,1,6,15,10]
+  CRUSH rule 0 x 169 [1,4,9,14,13,10,2]
+  CRUSH rule 0 x 170 [1,15,7,9,12,10,3]
+  CRUSH rule 0 x 171 [9,2,10,7,1,5,15]
+  CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+  CRUSH rule 0 x 173 [5,10,12,15,6,1,2]
+  CRUSH rule 0 x 174 [15,6,4,12,1,11,9]
+  CRUSH rule 0 x 175 [5,7,9,3,10,1,14]
+  CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+  CRUSH rule 0 x 177 [2,9,10,13,4,1,14]
+  CRUSH rule 0 x 178 [12,11,7,14,3,4]
+  CRUSH rule 0 x 179 [2,10,13,9,5,1,7]
+  CRUSH rule 0 x 180 [3,11,5,15,7,12]
+  CRUSH rule 0 x 181 [9,12,6,5,1,10,14]
+  CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+  CRUSH rule 0 x 183 [5,7,10,13,3,9,14]
+  CRUSH rule 0 x 184 [2,5,11,12,7,1,9]
+  CRUSH rule 0 x 185 [13,5,7,11,2,14]
+  CRUSH rule 0 x 186 [6,14,13,5,10,1,3]
+  CRUSH rule 0 x 187 [1,4,11,13,6,14,9]
+  CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+  CRUSH rule 0 x 189 [6,12,4,9,2,1,11]
+  CRUSH rule 0 x 190 [9,13,15,10,3,1,5]
+  CRUSH rule 0 x 191 [7,11,4,1,15,12,9]
+  CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+  CRUSH rule 0 x 193 [3,13,6,10,4,1,9]
+  CRUSH rule 0 x 194 [3,13,4,14,6,9,1]
+  CRUSH rule 0 x 195 [5,7,10,12,1,3,15]
+  CRUSH rule 0 x 196 [4,15,1,10,9,2,13]
+  CRUSH rule 0 x 197 [14,10,13,4,6,3,1]
+  CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+  CRUSH rule 0 x 199 [2,10,4,15,1,9,6]
+  CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+  CRUSH rule 0 x 201 [9,14,1,7,4,3,10]
+  CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+  CRUSH rule 0 x 203 [12,5,7,15,1,2,10]
+  CRUSH rule 0 x 204 [6,11,3,12,14,1,9]
+  CRUSH rule 0 x 205 [15,4,6,10,13,9,2]
+  CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+  CRUSH rule 0 x 207 [2,11,7,4,14,1,12]
+  CRUSH rule 0 x 208 [13,1,6,14,9,11,3]
+  CRUSH rule 0 x 209 [6,15,13,1,11,4,9]
+  CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+  CRUSH rule 0 x 211 [2,14,1,13,11,7,9]
+  CRUSH rule 0 x 212 [10,1,12,15,5,6,2]
+  CRUSH rule 0 x 213 [3,9,6,5,15,13,1]
+  CRUSH rule 0 x 214 [7,15,4,1,10,2,13]
+  CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+  CRUSH rule 0 x 216 [12,9,6,2,1,11,5]
+  CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+  CRUSH rule 0 x 218 [12,10,15,6,1,4,9]
+  CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+  CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+  CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+  CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+  CRUSH rule 0 x 223 [9,7,11,1,4,14,13]
+  CRUSH rule 0 x 224 [1,7,10,2,12,9,14]
+  CRUSH rule 0 x 225 [10,5,2,6,1,13,9]
+  CRUSH rule 0 x 226 [4,1,9,3,13,10,15]
+  CRUSH rule 0 x 227 [7,2,12,15,5,11]
+  CRUSH rule 0 x 228 [2,15,11,1,6,13,9]
+  CRUSH rule 0 x 229 [9,3,7,14,1,12,4]
+  CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+  CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+  CRUSH rule 0 x 232 [10,5,13,1,9,2,7]
+  CRUSH rule 0 x 233 [6,12,11,4,9,14,1]
+  CRUSH rule 0 x 234 [10,1,2,12,5,9,15]
+  CRUSH rule 0 x 235 [13,14,7,10,1,9,5]
+  CRUSH rule 0 x 236 [2,15,9,12,1,7,4]
+  CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+  CRUSH rule 0 x 238 [2,10,4,15,6,12,9]
+  CRUSH rule 0 x 239 [4,15,10,7,9,13,3]
+  CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+  CRUSH rule 0 x 241 [7,9,15,12,1,5,2]
+  CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+  CRUSH rule 0 x 243 [2,11,5,1,15,6,9]
+  CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+  CRUSH rule 0 x 245 [12,9,15,3,1,5,10]
+  CRUSH rule 0 x 246 [15,3,5,11,7,1,12]
+  CRUSH rule 0 x 247 [6,4,9,12,1,2,10]
+  CRUSH rule 0 x 248 [5,13,7,11,9,15,3]
+  CRUSH rule 0 x 249 [10,14,7,3,9,13,1]
+  CRUSH rule 0 x 250 [12,15,1,10,5,6,3]
+  CRUSH rule 0 x 251 [13,2,15,5,6,1,9]
+  CRUSH rule 0 x 252 [7,5,13,9,3,10,14]
+  CRUSH rule 0 x 253 [3,13,15,10,7,4]
+  CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+  CRUSH rule 0 x 255 [1,9,13,2,6,10,4]
+  CRUSH rule 0 x 256 [6,9,13,1,3,14,5]
+  CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+  CRUSH rule 0 x 258 [12,5,6,10,2,1,14]
+  CRUSH rule 0 x 259 [9,10,4,3,14,13,1]
+  CRUSH rule 0 x 260 [10,12,6,9,3,15,1]
+  CRUSH rule 0 x 261 [13,7,2,1,15,5,11]
+  CRUSH rule 0 x 262 [15,3,12,7,4,9,1]
+  CRUSH rule 0 x 263 [12,6,10,9,5,15,3]
+  CRUSH rule 0 x 264 [13,14,11,3,1,4,7]
+  CRUSH rule 0 x 265 [12,10,14,5,7,1,9]
+  CRUSH rule 0 x 266 [14,7,11,1,2,9,4]
+  CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+  CRUSH rule 0 x 268 [4,1,15,12,6,11,3]
+  CRUSH rule 0 x 269 [11,1,15,5,13,9,7]
+  CRUSH rule 0 x 270 [7,11,12,3,1,14,9]
+  CRUSH rule 0 x 271 [4,7,3,13,15,10,9]
+  CRUSH rule 0 x 272 [15,5,13,10,6,2]
+  CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+  CRUSH rule 0 x 274 [10,2,5,6,13,9,15]
+  CRUSH rule 0 x 275 [10,3,4,7,14,13]
+  CRUSH rule 0 x 276 [5,12,9,2,11,7,15]
+  CRUSH rule 0 x 277 [14,3,13,4,1,9,11]
+  CRUSH rule 0 x 278 [5,6,14,3,1,11,13]
+  CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+  CRUSH rule 0 x 280 [7,3,14,9,1,11,4]
+  CRUSH rule 0 x 281 [5,11,14,7,9,13,2]
+  CRUSH rule 0 x 282 [2,1,13,14,9,7,5]
+  CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+  CRUSH rule 0 x 284 [5,11,7,15,3,13,1]
+  CRUSH rule 0 x 285 [15,5,3,1,6,13,11]
+  CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+  CRUSH rule 0 x 287 [12,4,9,1,3,11,15]
+  CRUSH rule 0 x 288 [4,12,10,7,1,3,14]
+  CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+  CRUSH rule 0 x 290 [12,2,5,6,15,9,1]
+  CRUSH rule 0 x 291 [7,11,1,14,5,9,2]
+  CRUSH rule 0 x 292 [4,10,6,3,14,9,12]
+  CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+  CRUSH rule 0 x 294 [9,12,3,14,6,11,5]
+  CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+  CRUSH rule 0 x 296 [3,1,13,7,14,9,10]
+  CRUSH rule 0 x 297 [6,13,4,14,10,1,2]
+  CRUSH rule 0 x 298 [14,9,13,1,4,2,7]
+  CRUSH rule 0 x 299 [14,12,11,6,4,2,1]
+  CRUSH rule 0 x 300 [15,7,10,5,1,3,13]
+  CRUSH rule 0 x 301 [9,11,7,1,13,14,4]
+  CRUSH rule 0 x 302 [9,7,1,13,5,10,3]
+  CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+  CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+  CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+  CRUSH rule 0 x 306 [10,12,4,6,9,2,15]
+  CRUSH rule 0 x 307 [11,12,15,5,6,2,1]
+  CRUSH rule 0 x 308 [12,14,10,9,1,2,5]
+  CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+  CRUSH rule 0 x 310 [3,1,5,10,14,9,7]
+  CRUSH rule 0 x 311 [3,9,7,1,14,13,10]
+  CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+  CRUSH rule 0 x 313 [9,15,3,7,5,13,1]
+  CRUSH rule 0 x 314 [2,15,9,5,6,12,1]
+  CRUSH rule 0 x 315 [15,2,13,1,11,9,6]
+  CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+  CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+  CRUSH rule 0 x 318 [4,1,15,11,9,13,6]
+  CRUSH rule 0 x 319 [2,15,4,1,11,9,7]
+  CRUSH rule 0 x 320 [5,7,13,9,11,2,1]
+  CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+  CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+  CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+  CRUSH rule 0 x 324 [5,6,10,15,2,13]
+  CRUSH rule 0 x 325 [9,10,14,5,1,6,2]
+  CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+  CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+  CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+  CRUSH rule 0 x 329 [2,6,15,5,9,10,13]
+  CRUSH rule 0 x 330 [3,9,11,13,1,6,5]
+  CRUSH rule 0 x 331 [12,14,6,3,1,4,10]
+  CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+  CRUSH rule 0 x 333 [6,5,3,12,14,10,9]
+  CRUSH rule 0 x 334 [4,9,2,12,7,11,15]
+  CRUSH rule 0 x 335 [11,7,1,5,13,2,9]
+  CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+  CRUSH rule 0 x 337 [15,11,3,7,12,5]
+  CRUSH rule 0 x 338 [10,5,3,6,15,1,9]
+  CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+  CRUSH rule 0 x 340 [11,6,12,4,9,3,14]
+  CRUSH rule 0 x 341 [7,5,2,10,14,9,1]
+  CRUSH rule 0 x 342 [12,14,1,9,2,11,4]
+  CRUSH rule 0 x 343 [12,14,9,6,10,2,4]
+  CRUSH rule 0 x 344 [9,11,5,2,14,13,1]
+  CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+  CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+  CRUSH rule 0 x 347 [10,2,12,6,9,1,14]
+  CRUSH rule 0 x 348 [7,9,10,1,14,13,3]
+  CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+  CRUSH rule 0 x 350 [13,9,15,4,10,7,2]
+  CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+  CRUSH rule 0 x 352 [1,12,11,9,4,7,3]
+  CRUSH rule 0 x 353 [10,14,12,2,9,1,4]
+  CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+  CRUSH rule 0 x 355 [13,14,6,10,2,5,1]
+  CRUSH rule 0 x 356 [15,13,2,9,6,5,1]
+  CRUSH rule 0 x 357 [4,11,1,13,3,14,6]
+  CRUSH rule 0 x 358 [12,7,2,9,1,14,10]
+  CRUSH rule 0 x 359 [5,15,7,11,3,13]
+  CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+  CRUSH rule 0 x 361 [5,3,13,6,1,14,11]
+  CRUSH rule 0 x 362 [2,9,11,13,1,6,5]
+  CRUSH rule 0 x 363 [7,12,3,9,15,4,1]
+  CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+  CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+  CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+  CRUSH rule 0 x 367 [7,13,3,1,5,11,15]
+  CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+  CRUSH rule 0 x 369 [7,5,3,13,14,9,11]
+  CRUSH rule 0 x 370 [4,7,14,1,2,9,11]
+  CRUSH rule 0 x 371 [1,7,12,3,4,15,10]
+  CRUSH rule 0 x 372 [10,4,3,14,6,1,12]
+  CRUSH rule 0 x 373 [15,5,2,6,13,1,9]
+  CRUSH rule 0 x 374 [3,15,12,5,1,6,10]
+  CRUSH rule 0 x 375 [5,2,14,1,6,13,11]
+  CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+  CRUSH rule 0 x 377 [1,15,2,4,9,11,12]
+  CRUSH rule 0 x 378 [9,12,2,15,1,5,11]
+  CRUSH rule 0 x 379 [11,2,15,5,7,9,13]
+  CRUSH rule 0 x 380 [6,1,12,11,2,9,5]
+  CRUSH rule 0 x 381 [15,13,7,5,10,2,1]
+  CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+  CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+  CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+  CRUSH rule 0 x 385 [4,6,15,3,10,9,1]
+  CRUSH rule 0 x 386 [14,3,11,13,5,6,9]
+  CRUSH rule 0 x 387 [1,11,5,7,9,2,14]
+  CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+  CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+  CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+  CRUSH rule 0 x 391 [3,4,9,13,7,10,1]
+  CRUSH rule 0 x 392 [11,5,14,7,1,9,2]
+  CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+  CRUSH rule 0 x 394 [4,9,3,15,13,6,1]
+  CRUSH rule 0 x 395 [10,13,5,15,6,9,3]
+  CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+  CRUSH rule 0 x 397 [1,14,9,4,12,10,3]
+  CRUSH rule 0 x 398 [9,2,1,5,12,6,11]
+  CRUSH rule 0 x 399 [5,9,14,3,1,10,13]
+  CRUSH rule 0 x 400 [10,6,2,4,15,12,1]
+  CRUSH rule 0 x 401 [6,9,11,12,4,3,15]
+  CRUSH rule 0 x 402 [4,7,9,2,13,1,15]
+  CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+  CRUSH rule 0 x 404 [14,12,7,9,2,1,5]
+  CRUSH rule 0 x 405 [9,15,11,2,4,7,13]
+  CRUSH rule 0 x 406 [12,14,9,2,7,10,4]
+  CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+  CRUSH rule 0 x 408 [7,1,5,2,10,15,13]
+  CRUSH rule 0 x 409 [11,2,4,13,1,15,7]
+  CRUSH rule 0 x 410 [6,4,14,2,12,9,10]
+  CRUSH rule 0 x 411 [13,11,15,6,4,1,9]
+  CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+  CRUSH rule 0 x 413 [13,5,3,11,6,9,1]
+  CRUSH rule 0 x 414 [3,11,9,13,4,1,6]
+  CRUSH rule 0 x 415 [6,10,14,5,1,13,3]
+  CRUSH rule 0 x 416 [13,1,4,7,2,9,14]
+  CRUSH rule 0 x 417 [4,12,1,15,2,11,9]
+  CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+  CRUSH rule 0 x 419 [5,14,10,9,2,12,6]
+  CRUSH rule 0 x 420 [2,4,9,11,6,14,13]
+  CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+  CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+  CRUSH rule 0 x 423 [3,15,12,6,5,1,9]
+  CRUSH rule 0 x 424 [6,10,12,2,5,1,14]
+  CRUSH rule 0 x 425 [11,15,2,13,5,7,9]
+  CRUSH rule 0 x 426 [12,4,7,1,9,10,14]
+  CRUSH rule 0 x 427 [14,10,3,1,9,7,5]
+  CRUSH rule 0 x 428 [12,7,9,4,2,1,14]
+  CRUSH rule 0 x 429 [3,4,9,7,11,12,1]
+  CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+  CRUSH rule 0 x 431 [9,3,7,1,12,5,14]
+  CRUSH rule 0 x 432 [4,1,12,7,15,2,10]
+  CRUSH rule 0 x 433 [4,11,12,15,7,3]
+  CRUSH rule 0 x 434 [2,14,9,1,5,11,7]
+  CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+  CRUSH rule 0 x 436 [9,15,10,2,4,1,12]
+  CRUSH rule 0 x 437 [9,6,3,14,10,12,5]
+  CRUSH rule 0 x 438 [7,2,13,4,11,1,9]
+  CRUSH rule 0 x 439 [7,14,4,3,12,10]
+  CRUSH rule 0 x 440 [14,11,9,2,7,12,1]
+  CRUSH rule 0 x 441 [2,4,11,9,13,6,1]
+  CRUSH rule 0 x 442 [10,13,9,7,15,1,4]
+  CRUSH rule 0 x 443 [12,15,10,9,2,1,6]
+  CRUSH rule 0 x 444 [4,13,7,14,3,1,9]
+  CRUSH rule 0 x 445 [4,2,15,7,1,9,11]
+  CRUSH rule 0 x 446 [12,10,6,9,4,1,15]
+  CRUSH rule 0 x 447 [15,7,13,1,4,9,3]
+  CRUSH rule 0 x 448 [5,2,13,7,15,10]
+  CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+  CRUSH rule 0 x 450 [2,4,6,9,15,1,13]
+  CRUSH rule 0 x 451 [6,14,11,3,9,1,12]
+  CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+  CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+  CRUSH rule 0 x 454 [10,4,2,6,15,12,9]
+  CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+  CRUSH rule 0 x 456 [5,7,13,1,11,3,9]
+  CRUSH rule 0 x 457 [9,1,5,7,11,13,15]
+  CRUSH rule 0 x 458 [9,11,15,4,7,2,12]
+  CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+  CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+  CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+  CRUSH rule 0 x 462 [4,7,12,14,11,1,3]
+  CRUSH rule 0 x 463 [4,12,14,11,2,7,1]
+  CRUSH rule 0 x 464 [4,2,15,10,1,9,13]
+  CRUSH rule 0 x 465 [5,10,9,7,13,1,3]
+  CRUSH rule 0 x 466 [13,5,2,15,9,11,6]
+  CRUSH rule 0 x 467 [13,6,14,3,9,1,11]
+  CRUSH rule 0 x 468 [10,7,12,14,4,1,9]
+  CRUSH rule 0 x 469 [4,9,6,14,12,11,3]
+  CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+  CRUSH rule 0 x 471 [6,1,5,14,13,10,9]
+  CRUSH rule 0 x 472 [2,14,7,5,13,1,11]
+  CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+  CRUSH rule 0 x 474 [15,10,4,12,6,9,2]
+  CRUSH rule 0 x 475 [10,5,12,9,14,3,6]
+  CRUSH rule 0 x 476 [3,6,10,12,1,15,9]
+  CRUSH rule 0 x 477 [6,13,5,15,11,9,2]
+  CRUSH rule 0 x 478 [4,15,1,3,7,12,10]
+  CRUSH rule 0 x 479 [13,11,1,6,14,5,9]
+  CRUSH rule 0 x 480 [1,13,6,4,9,14,11]
+  CRUSH rule 0 x 481 [15,12,7,9,1,3,10]
+  CRUSH rule 0 x 482 [2,12,9,1,7,11,14]
+  CRUSH rule 0 x 483 [10,1,4,15,9,7,13]
+  CRUSH rule 0 x 484 [1,4,10,13,7,14,2]
+  CRUSH rule 0 x 485 [9,4,3,1,14,12,7]
+  CRUSH rule 0 x 486 [3,10,15,9,7,13,4]
+  CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+  CRUSH rule 0 x 488 [14,4,1,9,2,6,10]
+  CRUSH rule 0 x 489 [11,4,2,13,15,7]
+  CRUSH rule 0 x 490 [4,9,1,3,13,15,6]
+  CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+  CRUSH rule 0 x 492 [5,7,11,3,14,9,1]
+  CRUSH rule 0 x 493 [12,1,4,15,3,11,9]
+  CRUSH rule 0 x 494 [1,7,13,4,15,9,10]
+  CRUSH rule 0 x 495 [3,15,7,1,9,5,12]
+  CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+  CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+  CRUSH rule 0 x 498 [10,6,1,5,9,12,3]
+  CRUSH rule 0 x 499 [14,3,12,5,1,11,9]
+  CRUSH rule 0 x 500 [15,9,6,12,11,2,1]
+  CRUSH rule 0 x 501 [10,13,1,9,3,14,5]
+  CRUSH rule 0 x 502 [5,1,14,11,7,12,9]
+  CRUSH rule 0 x 503 [15,10,7,9,1,12,4]
+  CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+  CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+  CRUSH rule 0 x 506 [11,7,9,14,12,1,2]
+  CRUSH rule 0 x 507 [4,14,13,3,9,7,1]
+  CRUSH rule 0 x 508 [12,1,4,9,2,11,15]
+  CRUSH rule 0 x 509 [4,2,6,9,14,1,10]
+  CRUSH rule 0 x 510 [5,3,1,12,11,14,9]
+  CRUSH rule 0 x 511 [2,12,10,6,14,5]
+  CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+  CRUSH rule 0 x 513 [4,9,11,3,13,7,1]
+  CRUSH rule 0 x 514 [11,9,3,4,12,15,6]
+  CRUSH rule 0 x 515 [12,14,6,5,3,9,1]
+  CRUSH rule 0 x 516 [14,11,1,12,3,7,4]
+  CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+  CRUSH rule 0 x 518 [3,5,7,12,15,11,9]
+  CRUSH rule 0 x 519 [12,14,2,1,4,6,9]
+  CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+  CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+  CRUSH rule 0 x 522 [4,12,11,1,15,3,9]
+  CRUSH rule 0 x 523 [3,1,5,9,15,10,13]
+  CRUSH rule 0 x 524 [15,9,3,11,13,7,4]
+  CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+  CRUSH rule 0 x 526 [10,2,5,13,6,15,1]
+  CRUSH rule 0 x 527 [3,13,4,1,9,10,14]
+  CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+  CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+  CRUSH rule 0 x 530 [11,9,12,7,5,1,3]
+  CRUSH rule 0 x 531 [9,15,4,7,2,13,1]
+  CRUSH rule 0 x 532 [5,3,13,7,9,14,1]
+  CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+  CRUSH rule 0 x 534 [11,9,3,7,15,4,1]
+  CRUSH rule 0 x 535 [11,1,3,5,14,9,12]
+  CRUSH rule 0 x 536 [9,1,14,13,4,6,2]
+  CRUSH rule 0 x 537 [15,5,13,2,7,11]
+  CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+  CRUSH rule 0 x 539 [10,12,6,14,1,2,9]
+  CRUSH rule 0 x 540 [12,15,7,3,9,11,1]
+  CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+  CRUSH rule 0 x 542 [3,9,15,5,11,12,7]
+  CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+  CRUSH rule 0 x 544 [3,15,9,11,7,4,12]
+  CRUSH rule 0 x 545 [14,10,7,12,4,9,1]
+  CRUSH rule 0 x 546 [5,15,13,7,1,10,9]
+  CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+  CRUSH rule 0 x 548 [11,7,12,15,4,2]
+  CRUSH rule 0 x 549 [14,1,4,9,13,6,3]
+  CRUSH rule 0 x 550 [9,15,3,13,1,6,4]
+  CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+  CRUSH rule 0 x 552 [2,11,14,1,9,6,5]
+  CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+  CRUSH rule 0 x 554 [11,14,6,4,13,9,3]
+  CRUSH rule 0 x 555 [6,5,10,9,14,2,13]
+  CRUSH rule 0 x 556 [15,6,3,13,11,4,1]
+  CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+  CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+  CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+  CRUSH rule 0 x 560 [4,9,12,6,3,10,1]
+  CRUSH rule 0 x 561 [12,7,1,2,5,15,11]
+  CRUSH rule 0 x 562 [7,13,9,14,2,1,11]
+  CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+  CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+  CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+  CRUSH rule 0 x 566 [6,14,4,2,13,11]
+  CRUSH rule 0 x 567 [15,4,11,6,3,12]
+  CRUSH rule 0 x 568 [4,14,1,6,10,13,3]
+  CRUSH rule 0 x 569 [11,3,15,13,5,1,9]
+  CRUSH rule 0 x 570 [1,10,13,4,7,2,9]
+  CRUSH rule 0 x 571 [10,12,14,9,4,2,1]
+  CRUSH rule 0 x 572 [12,14,3,10,6,1,4]
+  CRUSH rule 0 x 573 [7,15,11,2,12,9,4]
+  CRUSH rule 0 x 574 [11,14,13,1,3,7,4]
+  CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+  CRUSH rule 0 x 576 [3,15,11,9,1,6,5]
+  CRUSH rule 0 x 577 [13,9,6,15,3,11,4]
+  CRUSH rule 0 x 578 [4,10,1,2,7,13,14]
+  CRUSH rule 0 x 579 [13,1,15,2,10,7,5]
+  CRUSH rule 0 x 580 [3,12,4,1,10,15,7]
+  CRUSH rule 0 x 581 [7,14,12,10,1,2,9]
+  CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+  CRUSH rule 0 x 583 [4,15,1,9,10,12,2]
+  CRUSH rule 0 x 584 [10,1,5,13,6,9,2]
+  CRUSH rule 0 x 585 [5,3,6,1,11,14,13]
+  CRUSH rule 0 x 586 [7,10,14,12,9,3,5]
+  CRUSH rule 0 x 587 [11,6,9,4,1,14,13]
+  CRUSH rule 0 x 588 [3,12,7,15,4,9,1]
+  CRUSH rule 0 x 589 [9,7,12,1,10,3,4]
+  CRUSH rule 0 x 590 [12,1,3,9,10,6,4]
+  CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+  CRUSH rule 0 x 592 [15,12,9,7,5,2,11]
+  CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+  CRUSH rule 0 x 594 [12,14,2,9,7,4,1]
+  CRUSH rule 0 x 595 [12,7,10,3,1,14,9]
+  CRUSH rule 0 x 596 [2,7,12,11,1,5,15]
+  CRUSH rule 0 x 597 [15,1,2,10,7,13,5]
+  CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+  CRUSH rule 0 x 599 [13,11,1,5,6,2,15]
+  CRUSH rule 0 x 600 [4,12,3,10,9,7,1]
+  CRUSH rule 0 x 601 [13,5,15,2,1,7,9]
+  CRUSH rule 0 x 602 [3,11,7,1,13,15,5]
+  CRUSH rule 0 x 603 [3,1,4,14,10,9,6]
+  CRUSH rule 0 x 604 [14,2,6,1,11,13,9]
+  CRUSH rule 0 x 605 [2,7,12,5,14,10,1]
+  CRUSH rule 0 x 606 [12,15,1,5,7,9,3]
+  CRUSH rule 0 x 607 [3,9,10,14,7,1,4]
+  CRUSH rule 0 x 608 [13,10,1,7,9,15,5]
+  CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+  CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+  CRUSH rule 0 x 611 [13,1,5,3,10,7,15]
+  CRUSH rule 0 x 612 [7,1,2,13,9,15,4]
+  CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+  CRUSH rule 0 x 614 [9,4,15,3,1,11,6]
+  CRUSH rule 0 x 615 [9,4,11,2,1,12,6]
+  CRUSH rule 0 x 616 [10,14,1,5,3,6,12]
+  CRUSH rule 0 x 617 [15,7,2,11,12,1,9]
+  CRUSH rule 0 x 618 [4,2,10,6,14,9,1]
+  CRUSH rule 0 x 619 [15,4,3,9,6,1,13]
+  CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+  CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+  CRUSH rule 0 x 622 [10,2,13,5,15,9,1]
+  CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+  CRUSH rule 0 x 624 [3,9,15,6,10,1,5]
+  CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+  CRUSH rule 0 x 626 [10,12,2,1,9,7,5]
+  CRUSH rule 0 x 627 [1,12,10,14,3,5,9]
+  CRUSH rule 0 x 628 [15,13,11,4,2,1,7]
+  CRUSH rule 0 x 629 [5,6,15,12,1,10,3]
+  CRUSH rule 0 x 630 [1,4,12,9,3,7,15]
+  CRUSH rule 0 x 631 [5,7,1,15,12,11,3]
+  CRUSH rule 0 x 632 [12,3,11,9,6,1,15]
+  CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+  CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+  CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+  CRUSH rule 0 x 636 [13,6,11,3,15,9,1]
+  CRUSH rule 0 x 637 [3,1,10,6,9,12,4]
+  CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+  CRUSH rule 0 x 639 [6,9,14,4,3,1,10]
+  CRUSH rule 0 x 640 [9,6,1,11,14,2,4]
+  CRUSH rule 0 x 641 [10,6,5,14,1,9,12]
+  CRUSH rule 0 x 642 [1,15,4,6,2,10,9]
+  CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+  CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+  CRUSH rule 0 x 645 [14,2,4,9,10,1,7]
+  CRUSH rule 0 x 646 [5,13,14,1,6,9,2]
+  CRUSH rule 0 x 647 [10,1,9,13,6,2,14]
+  CRUSH rule 0 x 648 [6,5,2,14,11,1,12]
+  CRUSH rule 0 x 649 [3,9,13,11,4,14,1]
+  CRUSH rule 0 x 650 [10,9,4,15,12,7,1]
+  CRUSH rule 0 x 651 [3,9,5,7,14,1,13]
+  CRUSH rule 0 x 652 [15,9,4,6,13,1,2]
+  CRUSH rule 0 x 653 [11,14,1,3,6,9,12]
+  CRUSH rule 0 x 654 [13,6,2,10,15,4,1]
+  CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+  CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+  CRUSH rule 0 x 657 [11,15,3,5,7,13,1]
+  CRUSH rule 0 x 658 [7,2,10,12,1,4,9]
+  CRUSH rule 0 x 659 [2,5,14,6,10,12]
+  CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+  CRUSH rule 0 x 661 [7,15,3,12,11,4,9]
+  CRUSH rule 0 x 662 [15,2,12,5,1,10,9]
+  CRUSH rule 0 x 663 [14,9,13,10,5,3,1]
+  CRUSH rule 0 x 664 [6,10,12,4,9,2,1]
+  CRUSH rule 0 x 665 [2,9,12,1,7,10,4]
+  CRUSH rule 0 x 666 [12,3,6,1,15,9,10]
+  CRUSH rule 0 x 667 [1,9,12,10,2,14,7]
+  CRUSH rule 0 x 668 [9,5,1,2,6,11,13]
+  CRUSH rule 0 x 669 [9,7,14,5,11,13,1]
+  CRUSH rule 0 x 670 [6,10,9,13,1,2,15]
+  CRUSH rule 0 x 671 [6,15,5,10,13,3]
+  CRUSH rule 0 x 672 [2,9,13,1,4,14,6]
+  CRUSH rule 0 x 673 [7,10,5,9,15,13,2]
+  CRUSH rule 0 x 674 [7,12,10,1,14,9,3]
+  CRUSH rule 0 x 675 [9,5,1,10,6,14,12]
+  CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+  CRUSH rule 0 x 677 [2,12,1,4,10,6,15]
+  CRUSH rule 0 x 678 [1,2,4,10,12,14,9]
+  CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+  CRUSH rule 0 x 680 [7,11,3,1,15,4,9]
+  CRUSH rule 0 x 681 [6,4,3,11,14,13,1]
+  CRUSH rule 0 x 682 [6,1,11,15,12,2,5]
+  CRUSH rule 0 x 683 [6,13,2,4,9,14,10]
+  CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+  CRUSH rule 0 x 685 [5,1,15,7,9,2,10]
+  CRUSH rule 0 x 686 [1,9,11,14,6,13,4]
+  CRUSH rule 0 x 687 [7,13,3,5,11,9,15]
+  CRUSH rule 0 x 688 [11,9,1,14,3,5,7]
+  CRUSH rule 0 x 689 [5,2,9,12,1,14,11]
+  CRUSH rule 0 x 690 [9,7,10,3,13,15,5]
+  CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+  CRUSH rule 0 x 692 [15,5,1,2,9,11,12]
+  CRUSH rule 0 x 693 [5,6,12,15,2,10,9]
+  CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+  CRUSH rule 0 x 695 [6,13,14,10,9,5,1]
+  CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+  CRUSH rule 0 x 697 [13,11,3,6,4,14,9]
+  CRUSH rule 0 x 698 [11,13,4,2,6,1,9]
+  CRUSH rule 0 x 699 [7,14,12,4,2,11]
+  CRUSH rule 0 x 700 [12,14,11,9,4,6,3]
+  CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+  CRUSH rule 0 x 702 [3,12,15,6,5,11,1]
+  CRUSH rule 0 x 703 [15,11,13,3,4,7,1]
+  CRUSH rule 0 x 704 [6,4,2,15,11,1,13]
+  CRUSH rule 0 x 705 [14,6,11,5,1,13,9]
+  CRUSH rule 0 x 706 [1,12,3,6,4,10,15]
+  CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+  CRUSH rule 0 x 708 [3,10,5,1,15,9,7]
+  CRUSH rule 0 x 709 [11,12,3,7,5,14,1]
+  CRUSH rule 0 x 710 [14,2,11,9,5,7,12]
+  CRUSH rule 0 x 711 [14,3,9,10,12,5,6]
+  CRUSH rule 0 x 712 [12,3,11,15,9,1,6]
+  CRUSH rule 0 x 713 [11,9,3,15,13,6,4]
+  CRUSH rule 0 x 714 [12,1,9,7,2,15,10]
+  CRUSH rule 0 x 715 [6,1,14,4,11,12,3]
+  CRUSH rule 0 x 716 [11,13,9,14,5,2,1]
+  CRUSH rule 0 x 717 [12,4,10,9,15,1,2]
+  CRUSH rule 0 x 718 [7,15,5,2,11,13]
+  CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+  CRUSH rule 0 x 720 [4,13,10,2,7,9,1]
+  CRUSH rule 0 x 721 [11,3,14,9,1,12,4]
+  CRUSH rule 0 x 722 [2,4,6,1,9,15,13]
+  CRUSH rule 0 x 723 [2,1,12,15,11,7,5]
+  CRUSH rule 0 x 724 [7,1,9,10,5,15,13]
+  CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+  CRUSH rule 0 x 726 [7,14,4,3,11,13,9]
+  CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+  CRUSH rule 0 x 728 [13,11,4,6,15,2]
+  CRUSH rule 0 x 729 [15,11,4,6,2,9,1]
+  CRUSH rule 0 x 730 [3,7,1,13,11,15,9]
+  CRUSH rule 0 x 731 [9,1,6,5,2,11,13]
+  CRUSH rule 0 x 732 [1,2,10,13,9,4,7]
+  CRUSH rule 0 x 733 [11,3,5,6,1,9,12]
+  CRUSH rule 0 x 734 [14,3,11,7,12,9,4]
+  CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+  CRUSH rule 0 x 736 [3,9,1,11,7,5,13]
+  CRUSH rule 0 x 737 [1,4,2,12,9,10,6]
+  CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+  CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+  CRUSH rule 0 x 740 [7,9,10,13,1,15,2]
+  CRUSH rule 0 x 741 [12,11,7,15,2,5]
+  CRUSH rule 0 x 742 [9,7,4,11,12,1,14]
+  CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+  CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+  CRUSH rule 0 x 745 [3,6,1,4,11,12,14]
+  CRUSH rule 0 x 746 [3,7,9,10,14,5,1]
+  CRUSH rule 0 x 747 [15,11,5,2,13,9,1]
+  CRUSH rule 0 x 748 [6,10,13,2,14,5,9]
+  CRUSH rule 0 x 749 [14,9,10,7,5,1,2]
+  CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+  CRUSH rule 0 x 751 [15,1,6,9,5,11,12]
+  CRUSH rule 0 x 752 [13,1,7,3,11,15,9]
+  CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+  CRUSH rule 0 x 754 [14,12,11,4,2,1,9]
+  CRUSH rule 0 x 755 [13,6,1,10,4,2,14]
+  CRUSH rule 0 x 756 [3,4,14,6,1,10,13]
+  CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+  CRUSH rule 0 x 758 [6,3,4,10,15,13,9]
+  CRUSH rule 0 x 759 [5,7,3,14,11,1,9]
+  CRUSH rule 0 x 760 [1,15,10,12,4,3,9]
+  CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+  CRUSH rule 0 x 762 [1,4,10,9,3,7,14]
+  CRUSH rule 0 x 763 [4,13,1,14,7,10,2]
+  CRUSH rule 0 x 764 [1,14,6,13,9,5,2]
+  CRUSH rule 0 x 765 [9,15,2,13,4,1,11]
+  CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+  CRUSH rule 0 x 767 [6,11,4,3,12,14]
+  CRUSH rule 0 x 768 [2,12,15,7,1,11,9]
+  CRUSH rule 0 x 769 [15,1,9,2,11,12,7]
+  CRUSH rule 0 x 770 [15,13,4,6,3,10,1]
+  CRUSH rule 0 x 771 [9,2,12,11,6,14,5]
+  CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+  CRUSH rule 0 x 773 [3,7,4,15,1,12,11]
+  CRUSH rule 0 x 774 [12,6,3,15,5,9,10]
+  CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+  CRUSH rule 0 x 776 [10,15,3,9,6,13,1]
+  CRUSH rule 0 x 777 [11,13,4,7,1,14,9]
+  CRUSH rule 0 x 778 [13,1,9,11,15,6,3]
+  CRUSH rule 0 x 779 [5,11,1,14,2,9,13]
+  CRUSH rule 0 x 780 [13,9,3,6,4,1,14]
+  CRUSH rule 0 x 781 [5,7,14,3,1,12,11]
+  CRUSH rule 0 x 782 [2,15,9,7,11,13,4]
+  CRUSH rule 0 x 783 [12,7,5,14,9,1,2]
+  CRUSH rule 0 x 784 [14,1,10,13,3,4,7]
+  CRUSH rule 0 x 785 [6,12,1,2,4,9,15]
+  CRUSH rule 0 x 786 [10,5,2,15,1,7,12]
+  CRUSH rule 0 x 787 [1,12,10,2,9,4,14]
+  CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+  CRUSH rule 0 x 789 [9,2,14,7,4,12,1]
+  CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+  CRUSH rule 0 x 791 [9,4,7,13,14,11,1]
+  CRUSH rule 0 x 792 [6,4,15,10,12,3]
+  CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+  CRUSH rule 0 x 794 [5,12,2,14,9,10,1]
+  CRUSH rule 0 x 795 [6,14,12,4,10,1,2]
+  CRUSH rule 0 x 796 [11,2,12,6,15,4]
+  CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+  CRUSH rule 0 x 798 [5,11,6,13,1,3,15]
+  CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+  CRUSH rule 0 x 800 [6,3,4,11,15,13]
+  CRUSH rule 0 x 801 [2,5,6,13,9,1,10]
+  CRUSH rule 0 x 802 [1,4,12,7,3,9,10]
+  CRUSH rule 0 x 803 [7,2,4,1,11,13,9]
+  CRUSH rule 0 x 804 [5,14,9,7,3,1,12]
+  CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+  CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+  CRUSH rule 0 x 807 [14,2,7,4,9,12,1]
+  CRUSH rule 0 x 808 [2,15,12,7,9,1,5]
+  CRUSH rule 0 x 809 [1,11,7,12,4,2,15]
+  CRUSH rule 0 x 810 [2,5,9,12,15,1,7]
+  CRUSH rule 0 x 811 [15,6,3,10,1,5,9]
+  CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+  CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+  CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+  CRUSH rule 0 x 815 [15,12,9,4,10,6,1]
+  CRUSH rule 0 x 816 [14,10,13,7,3,9,4]
+  CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+  CRUSH rule 0 x 818 [15,2,11,4,1,12,6]
+  CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+  CRUSH rule 0 x 820 [3,6,9,12,11,15,4]
+  CRUSH rule 0 x 821 [15,10,9,13,3,4,7]
+  CRUSH rule 0 x 822 [10,13,2,9,7,4,14]
+  CRUSH rule 0 x 823 [2,6,12,10,15,4,1]
+  CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+  CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+  CRUSH rule 0 x 826 [5,2,11,15,1,12,9]
+  CRUSH rule 0 x 827 [13,5,1,3,7,9,11]
+  CRUSH rule 0 x 828 [12,6,10,5,1,9,2]
+  CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+  CRUSH rule 0 x 830 [15,13,2,9,7,11,1]
+  CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+  CRUSH rule 0 x 832 [14,11,13,2,9,4,6]
+  CRUSH rule 0 x 833 [9,13,3,11,7,5,15]
+  CRUSH rule 0 x 834 [9,7,5,1,11,2,13]
+  CRUSH rule 0 x 835 [14,3,13,6,4,9,1]
+  CRUSH rule 0 x 836 [3,9,10,13,1,5,14]
+  CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+  CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+  CRUSH rule 0 x 839 [3,4,6,10,15,1,13]
+  CRUSH rule 0 x 840 [10,15,12,4,7,1,2]
+  CRUSH rule 0 x 841 [3,5,7,12,11,15,1]
+  CRUSH rule 0 x 842 [9,13,2,6,5,14,10]
+  CRUSH rule 0 x 843 [14,7,4,9,3,12,1]
+  CRUSH rule 0 x 844 [7,1,4,15,9,2,11]
+  CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+  CRUSH rule 0 x 846 [3,7,15,13,1,9,10]
+  CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+  CRUSH rule 0 x 848 [11,13,1,14,5,9,2]
+  CRUSH rule 0 x 849 [3,15,11,9,6,1,13]
+  CRUSH rule 0 x 850 [1,3,10,6,14,4,9]
+  CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+  CRUSH rule 0 x 852 [9,12,4,7,15,2,11]
+  CRUSH rule 0 x 853 [13,14,6,11,2,4,9]
+  CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+  CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+  CRUSH rule 0 x 856 [5,10,7,3,15,9,12]
+  CRUSH rule 0 x 857 [4,3,13,11,9,1,7]
+  CRUSH rule 0 x 858 [5,15,6,3,9,12,1]
+  CRUSH rule 0 x 859 [5,15,6,2,1,11,12]
+  CRUSH rule 0 x 860 [11,14,1,12,6,9,2]
+  CRUSH rule 0 x 861 [13,7,4,10,1,14,3]
+  CRUSH rule 0 x 862 [5,10,9,7,3,12,1]
+  CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+  CRUSH rule 0 x 864 [6,13,4,2,10,15,1]
+  CRUSH rule 0 x 865 [4,1,14,11,6,9,3]
+  CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+  CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+  CRUSH rule 0 x 868 [14,11,7,2,1,4,9]
+  CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+  CRUSH rule 0 x 870 [14,9,11,4,3,12,6]
+  CRUSH rule 0 x 871 [6,2,1,4,15,13,11]
+  CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+  CRUSH rule 0 x 873 [2,5,12,10,1,9,15]
+  CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+  CRUSH rule 0 x 875 [10,6,14,1,12,5,9]
+  CRUSH rule 0 x 876 [14,7,13,3,9,1,11]
+  CRUSH rule 0 x 877 [15,11,13,9,5,1,6]
+  CRUSH rule 0 x 878 [7,14,3,13,9,1,11]
+  CRUSH rule 0 x 879 [12,2,7,4,10,15]
+  CRUSH rule 0 x 880 [2,12,10,7,1,4,9]
+  CRUSH rule 0 x 881 [6,3,1,11,4,15,9]
+  CRUSH rule 0 x 882 [11,13,7,1,2,15,4]
+  CRUSH rule 0 x 883 [13,1,3,10,6,5,9]
+  CRUSH rule 0 x 884 [6,15,4,9,3,11,12]
+  CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+  CRUSH rule 0 x 886 [13,11,4,2,1,14,9]
+  CRUSH rule 0 x 887 [14,4,12,11,2,6,9]
+  CRUSH rule 0 x 888 [10,12,7,15,9,2,1]
+  CRUSH rule 0 x 889 [15,13,4,1,6,2,10]
+  CRUSH rule 0 x 890 [10,12,14,2,9,5,6]
+  CRUSH rule 0 x 891 [9,5,11,6,3,15,12]
+  CRUSH rule 0 x 892 [12,15,2,4,7,9,11]
+  CRUSH rule 0 x 893 [1,3,5,9,6,10,14]
+  CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+  CRUSH rule 0 x 895 [2,1,11,5,7,15,13]
+  CRUSH rule 0 x 896 [9,1,14,10,4,12,2]
+  CRUSH rule 0 x 897 [7,5,14,3,1,9,11]
+  CRUSH rule 0 x 898 [10,6,12,9,15,5,2]
+  CRUSH rule 0 x 899 [1,11,5,3,13,14,9]
+  CRUSH rule 0 x 900 [2,9,10,7,13,14,5]
+  CRUSH rule 0 x 901 [9,12,11,3,14,4,1]
+  CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+  CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+  CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+  CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+  CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+  CRUSH rule 0 x 907 [7,12,3,9,10,5,14]
+  CRUSH rule 0 x 908 [2,15,9,6,10,13,5]
+  CRUSH rule 0 x 909 [10,14,1,13,2,9,7]
+  CRUSH rule 0 x 910 [12,7,4,15,10,3,1]
+  CRUSH rule 0 x 911 [11,15,2,4,9,13,6]
+  CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+  CRUSH rule 0 x 913 [4,6,10,1,12,3,9]
+  CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+  CRUSH rule 0 x 915 [12,14,1,9,4,3,11]
+  CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+  CRUSH rule 0 x 917 [1,15,6,5,10,3,13]
+  CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+  CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+  CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+  CRUSH rule 0 x 921 [1,11,6,13,4,2,9]
+  CRUSH rule 0 x 922 [6,4,14,13,3,1,10]
+  CRUSH rule 0 x 923 [12,2,5,14,10,1,9]
+  CRUSH rule 0 x 924 [6,2,14,13,9,1,11]
+  CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+  CRUSH rule 0 x 926 [3,13,10,1,14,9,6]
+  CRUSH rule 0 x 927 [6,5,1,11,14,2,13]
+  CRUSH rule 0 x 928 [13,1,3,9,6,11,15]
+  CRUSH rule 0 x 929 [10,7,1,5,2,12,9]
+  CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+  CRUSH rule 0 x 931 [6,15,11,9,5,3,1]
+  CRUSH rule 0 x 932 [13,2,5,11,9,1,6]
+  CRUSH rule 0 x 933 [12,7,14,10,4,1,2]
+  CRUSH rule 0 x 934 [12,2,5,7,9,1,15]
+  CRUSH rule 0 x 935 [6,11,1,14,5,13,3]
+  CRUSH rule 0 x 936 [9,12,7,5,1,2,14]
+  CRUSH rule 0 x 937 [14,2,11,1,13,4,9]
+  CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+  CRUSH rule 0 x 939 [6,4,14,9,12,1,11]
+  CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+  CRUSH rule 0 x 941 [3,12,4,7,14,10]
+  CRUSH rule 0 x 942 [15,12,10,4,1,9,3]
+  CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+  CRUSH rule 0 x 944 [2,9,4,7,1,14,12]
+  CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+  CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+  CRUSH rule 0 x 947 [11,3,14,1,12,5,6]
+  CRUSH rule 0 x 948 [7,13,11,5,14,2,1]
+  CRUSH rule 0 x 949 [9,1,12,5,15,10,2]
+  CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+  CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+  CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+  CRUSH rule 0 x 953 [1,3,6,10,12,14,4]
+  CRUSH rule 0 x 954 [10,2,14,9,4,6,12]
+  CRUSH rule 0 x 955 [7,14,3,1,10,4,9]
+  CRUSH rule 0 x 956 [1,6,11,5,14,3,9]
+  CRUSH rule 0 x 957 [14,11,1,12,6,9,4]
+  CRUSH rule 0 x 958 [15,4,3,11,1,6,12]
+  CRUSH rule 0 x 959 [2,1,12,15,10,9,4]
+  CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+  CRUSH rule 0 x 961 [3,13,11,9,6,1,4]
+  CRUSH rule 0 x 962 [5,11,3,14,1,6,13]
+  CRUSH rule 0 x 963 [13,10,15,4,6,9,1]
+  CRUSH rule 0 x 964 [7,11,4,9,2,12,1]
+  CRUSH rule 0 x 965 [12,2,9,7,4,15,11]
+  CRUSH rule 0 x 966 [12,14,9,4,1,2,11]
+  CRUSH rule 0 x 967 [7,5,3,10,12,14]
+  CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+  CRUSH rule 0 x 969 [11,4,7,1,9,14,13]
+  CRUSH rule 0 x 970 [5,12,10,1,3,14,9]
+  CRUSH rule 0 x 971 [1,9,4,12,7,2,10]
+  CRUSH rule 0 x 972 [12,3,14,5,1,9,7]
+  CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+  CRUSH rule 0 x 974 [7,11,1,2,15,4,12]
+  CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+  CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+  CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+  CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+  CRUSH rule 0 x 979 [5,1,13,6,15,10,3]
+  CRUSH rule 0 x 980 [15,11,5,6,1,3,13]
+  CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+  CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+  CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+  CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+  CRUSH rule 0 x 985 [11,2,15,1,4,13,6]
+  CRUSH rule 0 x 986 [6,13,9,1,15,10,5]
+  CRUSH rule 0 x 987 [13,14,5,10,6,1,3]
+  CRUSH rule 0 x 988 [12,9,10,14,3,1,4]
+  CRUSH rule 0 x 989 [7,4,3,15,9,13,10]
+  CRUSH rule 0 x 990 [1,10,9,13,3,4,6]
+  CRUSH rule 0 x 991 [7,11,1,14,2,5,9]
+  CRUSH rule 0 x 992 [9,10,2,13,7,4,1]
+  CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+  CRUSH rule 0 x 994 [3,13,15,4,11,7,1]
+  CRUSH rule 0 x 995 [15,6,12,2,5,11]
+  CRUSH rule 0 x 996 [15,10,5,3,13,1,9]
+  CRUSH rule 0 x 997 [15,2,1,12,7,9,4]
+  CRUSH rule 0 x 998 [6,1,9,5,12,11,15]
+  CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+  CRUSH rule 0 x 1000 [14,2,9,4,12,1,6]
+  CRUSH rule 0 x 1001 [11,14,4,2,6,9,1]
+  CRUSH rule 0 x 1002 [1,10,14,2,9,5,13]
+  CRUSH rule 0 x 1003 [10,7,5,14,2,1,9]
+  CRUSH rule 0 x 1004 [15,1,4,6,10,12,9]
+  CRUSH rule 0 x 1005 [6,12,2,10,9,15,5]
+  CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+  CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+  CRUSH rule 0 x 1008 [7,4,9,11,3,15,1]
+  CRUSH rule 0 x 1009 [5,2,11,7,15,9,1]
+  CRUSH rule 0 x 1010 [10,2,15,6,9,13,4]
+  CRUSH rule 0 x 1011 [6,3,12,1,10,4,9]
+  CRUSH rule 0 x 1012 [12,6,9,15,3,1,5]
+  CRUSH rule 0 x 1013 [2,14,12,4,9,1,6]
+  CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+  CRUSH rule 0 x 1015 [12,6,10,1,4,15,9]
+  CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+  CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+  CRUSH rule 0 x 1018 [13,11,14,1,9,3,5]
+  CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+  CRUSH rule 0 x 1020 [3,1,13,4,10,9,14]
+  CRUSH rule 0 x 1021 [2,11,14,9,4,6,1]
+  CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+  CRUSH rule 0 x 1023 [15,2,9,12,1,7,4]
+  rule 0 (replicated_ruleset) num_rep 7 result size == 6:\t36/1024 (esc)
+  rule 0 (replicated_ruleset) num_rep 7 result size == 7:\t988/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3,15,12,1,4,9]
+  CRUSH rule 0 x 1 [10,15,1,2,13,4,7,9]
+  CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+  CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+  CRUSH rule 0 x 4 [14,2,10,1,9,4,7,13]
+  CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+  CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+  CRUSH rule 0 x 7 [9,2,6,12,11,4,1,14]
+  CRUSH rule 0 x 8 [10,2,15,1,4,13,6,9]
+  CRUSH rule 0 x 9 [7,1,14,2,11,9,12,4]
+  CRUSH rule 0 x 10 [10,14,4,1,2,7,13,9]
+  CRUSH rule 0 x 11 [13,9,14,7,5,11,2,1]
+  CRUSH rule 0 x 12 [7,1,2,5,13,15,11,9]
+  CRUSH rule 0 x 13 [3,5,12,7,9,1,14,11]
+  CRUSH rule 0 x 14 [13,5,2,7,10,15,1,9]
+  CRUSH rule 0 x 15 [15,1,9,6,13,3,5,11]
+  CRUSH rule 0 x 16 [7,11,14,2,13,1,9,4]
+  CRUSH rule 0 x 17 [10,1,13,2,4,6,14,9]
+  CRUSH rule 0 x 18 [1,7,3,10,5,12,9,14]
+  CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+  CRUSH rule 0 x 20 [14,12,3,10,9,4,7,1]
+  CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+  CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+  CRUSH rule 0 x 23 [10,5,13,9,3,15,1,6]
+  CRUSH rule 0 x 24 [12,11,3,1,9,4,7,15]
+  CRUSH rule 0 x 25 [7,12,15,1,3,10,4,9]
+  CRUSH rule 0 x 26 [1,7,13,2,14,5,9,11]
+  CRUSH rule 0 x 27 [3,6,15,4,13,9,11,1]
+  CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+  CRUSH rule 0 x 29 [5,14,12,11,6,3,1,9]
+  CRUSH rule 0 x 30 [2,5,6,9,1,11,13,14]
+  CRUSH rule 0 x 31 [5,15,10,1,9,13,6,3]
+  CRUSH rule 0 x 32 [9,10,2,1,13,14,6,4]
+  CRUSH rule 0 x 33 [13,4,9,2,7,1,10,14]
+  CRUSH rule 0 x 34 [13,15,2,4,1,10,9,6]
+  CRUSH rule 0 x 35 [4,14,3,13,10,9,1,6]
+  CRUSH rule 0 x 36 [3,12,9,7,5,10,14,1]
+  CRUSH rule 0 x 37 [9,2,6,14,11,1,4,13]
+  CRUSH rule 0 x 38 [3,4,13,10,9,1,14,6]
+  CRUSH rule 0 x 39 [12,7,14,11,1,9,5,2]
+  CRUSH rule 0 x 40 [10,1,9,5,15,2,6,13]
+  CRUSH rule 0 x 41 [4,9,11,1,14,13,6,3]
+  CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+  CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+  CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+  CRUSH rule 0 x 45 [11,12,15,9,1,5,6,3]
+  CRUSH rule 0 x 46 [6,9,2,14,11,13,1,5]
+  CRUSH rule 0 x 47 [3,9,6,4,13,1,11,15]
+  CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+  CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+  CRUSH rule 0 x 50 [14,12,1,4,2,11,6,9]
+  CRUSH rule 0 x 51 [10,6,5,12,15,2,1,9]
+  CRUSH rule 0 x 52 [12,1,9,11,7,3,14,4]
+  CRUSH rule 0 x 53 [3,6,13,9,5,1,11,15]
+  CRUSH rule 0 x 54 [4,13,9,2,14,10,6,1]
+  CRUSH rule 0 x 55 [4,11,2,7,1,13,9,15]
+  CRUSH rule 0 x 56 [5,9,10,1,3,13,14,7]
+  CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+  CRUSH rule 0 x 58 [7,1,11,4,3,14,12,9]
+  CRUSH rule 0 x 59 [2,13,1,10,9,5,14,6]
+  CRUSH rule 0 x 60 [3,6,11,1,4,9,12,15]
+  CRUSH rule 0 x 61 [3,15,13,7,4,1,10,9]
+  CRUSH rule 0 x 62 [15,11,7,12,5,9,2,1]
+  CRUSH rule 0 x 63 [10,14,12,1,7,3]
+  CRUSH rule 0 x 64 [3,9,1,4,7,12,11,15]
+  CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+  CRUSH rule 0 x 66 [15,11,6,9,4,1,3,12]
+  CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+  CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+  CRUSH rule 0 x 69 [2,1,15,10,4,9,13,7]
+  CRUSH rule 0 x 70 [9,6,1,3,13,15,11,5]
+  CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+  CRUSH rule 0 x 72 [9,10,3,5,7,12,15,1]
+  CRUSH rule 0 x 73 [5,3,11,1,7,12,15,9]
+  CRUSH rule 0 x 74 [11,7,9,5,1,15,3,12]
+  CRUSH rule 0 x 75 [9,7,11,14,12,1,2,5]
+  CRUSH rule 0 x 76 [6,1,3,5,14,10,12,9]
+  CRUSH rule 0 x 77 [7,4,2,13,9,1,11,15]
+  CRUSH rule 0 x 78 [9,3,1,5,6,13,14,11]
+  CRUSH rule 0 x 79 [13,2,15,5,7,9,11,1]
+  CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+  CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+  CRUSH rule 0 x 82 [14,13,5,11,6,2,1,9]
+  CRUSH rule 0 x 83 [4,15,3,9,10,13,6,1]
+  CRUSH rule 0 x 84 [10,7,9,15,3,4,1,13]
+  CRUSH rule 0 x 85 [3,15,9,7,4,11,1,12]
+  CRUSH rule 0 x 86 [10,9,14,1,13,4,2,7]
+  CRUSH rule 0 x 87 [15,10,7,12,5,3,9,1]
+  CRUSH rule 0 x 88 [4,13,3,1,9,15,11,7]
+  CRUSH rule 0 x 89 [3,9,7,4,1,14,10,12]
+  CRUSH rule 0 x 90 [4,9,7,12,11,14,2,1]
+  CRUSH rule 0 x 91 [6,11,9,1,2,4,14,13]
+  CRUSH rule 0 x 92 [1,5,10,9,13,15,6,2]
+  CRUSH rule 0 x 93 [9,3,15,13,7,5,1,10]
+  CRUSH rule 0 x 94 [9,2,12,5,6,11,1,14]
+  CRUSH rule 0 x 95 [7,15,4,10,9,13,2,1]
+  CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+  CRUSH rule 0 x 97 [4,11,2,13,1,7,9,14]
+  CRUSH rule 0 x 98 [11,13,9,3,15,1,5,6]
+  CRUSH rule 0 x 99 [12,4,11,7,3,14,9,1]
+  CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+  CRUSH rule 0 x 101 [15,7,1,9,10,5,2,12]
+  CRUSH rule 0 x 102 [3,11,14,6,13,4,9,1]
+  CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+  CRUSH rule 0 x 104 [14,6,3,5,9,1,10,13]
+  CRUSH rule 0 x 105 [14,10,1,9,3,5,6,13]
+  CRUSH rule 0 x 106 [6,5,13,2,14,11,1,9]
+  CRUSH rule 0 x 107 [3,1,10,14,13,5,9,6]
+  CRUSH rule 0 x 108 [5,10,7,2,15,9,12,1]
+  CRUSH rule 0 x 109 [9,1,13,7,15,5,11,3]
+  CRUSH rule 0 x 110 [5,1,11,3,7,14,13,9]
+  CRUSH rule 0 x 111 [10,1,9,7,5,2,13,14]
+  CRUSH rule 0 x 112 [1,10,4,14,2,12,6,9]
+  CRUSH rule 0 x 113 [6,10,13,9,1,5,2,14]
+  CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+  CRUSH rule 0 x 115 [10,13,14,3,9,1,6,5]
+  CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+  CRUSH rule 0 x 117 [5,6,1,12,15,9,11,3]
+  CRUSH rule 0 x 118 [10,4,13,15,9,3,1,6]
+  CRUSH rule 0 x 119 [14,12,11,4,6,9,3,1]
+  CRUSH rule 0 x 120 [11,3,14,13,4,7]
+  CRUSH rule 0 x 121 [9,5,1,11,7,3,15,12]
+  CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+  CRUSH rule 0 x 123 [3,10,5,6,9,1,12,15]
+  CRUSH rule 0 x 124 [12,2,1,5,14,7,11,9]
+  CRUSH rule 0 x 125 [9,12,15,1,6,5,3,10]
+  CRUSH rule 0 x 126 [7,15,10,9,2,12,5,1]
+  CRUSH rule 0 x 127 [4,14,9,13,1,3,7,10]
+  CRUSH rule 0 x 128 [3,12,1,10,4,9,7,14]
+  CRUSH rule 0 x 129 [11,13,14,2,9,4,6,1]
+  CRUSH rule 0 x 130 [3,13,5,14,10,1,9,6]
+  CRUSH rule 0 x 131 [12,1,6,15,4,2,11,9]
+  CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+  CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+  CRUSH rule 0 x 134 [12,5,6,15,3,9,10,1]
+  CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+  CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+  CRUSH rule 0 x 137 [14,3,6,11,1,9,4,13]
+  CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+  CRUSH rule 0 x 139 [11,2,13,9,1,15,7,5]
+  CRUSH rule 0 x 140 [11,4,12,15,2,6,9,1]
+  CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+  CRUSH rule 0 x 142 [3,14,7,9,11,1,4,13]
+  CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+  CRUSH rule 0 x 144 [13,7,11,2,14,4,1,9]
+  CRUSH rule 0 x 145 [12,2,6,10,9,4,14,1]
+  CRUSH rule 0 x 146 [1,5,9,2,6,13,14,10]
+  CRUSH rule 0 x 147 [1,4,9,11,2,7,15,12]
+  CRUSH rule 0 x 148 [12,7,9,2,14,11,1,4]
+  CRUSH rule 0 x 149 [2,5,9,12,11,1,7,14]
+  CRUSH rule 0 x 150 [1,15,2,10,7,9,5,12]
+  CRUSH rule 0 x 151 [2,9,14,7,1,10,5,13]
+  CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+  CRUSH rule 0 x 153 [6,9,4,15,2,1,10,12]
+  CRUSH rule 0 x 154 [3,11,7,1,4,12,15,9]
+  CRUSH rule 0 x 155 [14,12,7,3,5,1,9,11]
+  CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+  CRUSH rule 0 x 157 [15,1,6,4,3,10,9,13]
+  CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+  CRUSH rule 0 x 159 [4,14,3,12,10,6,1,9]
+  CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+  CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+  CRUSH rule 0 x 162 [10,6,1,12,2,4,14,9]
+  CRUSH rule 0 x 163 [15,1,10,2,6,4,13,9]
+  CRUSH rule 0 x 164 [9,14,10,7,12,2,5,1]
+  CRUSH rule 0 x 165 [11,7,2,13,9,15,1,5]
+  CRUSH rule 0 x 166 [1,2,12,14,4,11,7,9]
+  CRUSH rule 0 x 167 [9,7,3,4,11,13,15,1]
+  CRUSH rule 0 x 168 [13,2,4,1,6,15,10,9]
+  CRUSH rule 0 x 169 [1,4,9,14,13,10,2,7]
+  CRUSH rule 0 x 170 [1,15,7,9,12,10,3,5]
+  CRUSH rule 0 x 171 [9,2,10,7,1,5,15,12]
+  CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+  CRUSH rule 0 x 173 [5,10,12,15,6,1,2,9]
+  CRUSH rule 0 x 174 [15,6,4,12,1,11,9,2]
+  CRUSH rule 0 x 175 [5,7,9,3,10,1,14,13]
+  CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+  CRUSH rule 0 x 177 [2,9,10,13,4,1,14,7]
+  CRUSH rule 0 x 178 [12,11,7,14,3,4]
+  CRUSH rule 0 x 179 [2,10,13,9,5,1,7,14]
+  CRUSH rule 0 x 180 [3,11,5,15,7,12]
+  CRUSH rule 0 x 181 [9,12,6,5,1,10,14,2]
+  CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+  CRUSH rule 0 x 183 [5,7,10,13,3,9,14,1]
+  CRUSH rule 0 x 184 [2,5,11,12,7,1,9,15]
+  CRUSH rule 0 x 185 [13,5,7,11,2,14]
+  CRUSH rule 0 x 186 [6,14,13,5,10,1,3,9]
+  CRUSH rule 0 x 187 [1,4,11,13,6,14,9,3]
+  CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+  CRUSH rule 0 x 189 [6,12,4,9,2,1,11,15]
+  CRUSH rule 0 x 190 [9,13,15,10,3,1,5,6]
+  CRUSH rule 0 x 191 [7,11,4,1,15,12,9,2]
+  CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+  CRUSH rule 0 x 193 [3,13,6,10,4,1,9,14]
+  CRUSH rule 0 x 194 [3,13,4,14,6,9,1,11]
+  CRUSH rule 0 x 195 [5,7,10,12,1,3,15,9]
+  CRUSH rule 0 x 196 [4,15,1,10,9,2,13,7]
+  CRUSH rule 0 x 197 [14,10,13,4,6,3,1,9]
+  CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+  CRUSH rule 0 x 199 [2,10,4,15,1,9,6,12]
+  CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+  CRUSH rule 0 x 201 [9,14,1,7,4,3,10,13]
+  CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+  CRUSH rule 0 x 203 [12,5,7,15,1,2,10,9]
+  CRUSH rule 0 x 204 [6,11,3,12,14,1,9,4]
+  CRUSH rule 0 x 205 [15,4,6,10,13,9,2,1]
+  CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+  CRUSH rule 0 x 207 [2,11,7,4,14,1,12,9]
+  CRUSH rule 0 x 208 [13,1,6,14,9,11,3,5]
+  CRUSH rule 0 x 209 [6,15,13,1,11,4,9,2]
+  CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+  CRUSH rule 0 x 211 [2,14,1,13,11,7,9,5]
+  CRUSH rule 0 x 212 [10,1,12,15,5,6,2,9]
+  CRUSH rule 0 x 213 [3,9,6,5,15,13,1,11]
+  CRUSH rule 0 x 214 [7,15,4,1,10,2,13,9]
+  CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+  CRUSH rule 0 x 216 [12,9,6,2,1,11,5,15]
+  CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+  CRUSH rule 0 x 218 [12,10,15,6,1,4,9,2]
+  CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+  CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+  CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+  CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+  CRUSH rule 0 x 223 [9,7,11,1,4,14,13,3]
+  CRUSH rule 0 x 224 [1,7,10,2,12,9,14,5]
+  CRUSH rule 0 x 225 [10,5,2,6,1,13,9,15]
+  CRUSH rule 0 x 226 [4,1,9,3,13,10,15,7]
+  CRUSH rule 0 x 227 [7,2,12,15,5,11]
+  CRUSH rule 0 x 228 [2,15,11,1,6,13,9,4]
+  CRUSH rule 0 x 229 [9,3,7,14,1,12,4,10]
+  CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+  CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+  CRUSH rule 0 x 232 [10,5,13,1,9,2,7,14]
+  CRUSH rule 0 x 233 [6,12,11,4,9,14,1,3]
+  CRUSH rule 0 x 234 [10,1,2,12,5,9,15,7]
+  CRUSH rule 0 x 235 [13,14,7,10,1,9,5,3]
+  CRUSH rule 0 x 236 [2,15,9,12,1,7,4,10]
+  CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+  CRUSH rule 0 x 238 [2,10,4,15,6,12,9,1]
+  CRUSH rule 0 x 239 [4,15,10,7,9,13,3,1]
+  CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+  CRUSH rule 0 x 241 [7,9,15,12,1,5,2,10]
+  CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+  CRUSH rule 0 x 243 [2,11,5,1,15,6,9,13]
+  CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+  CRUSH rule 0 x 245 [12,9,15,3,1,5,10,7]
+  CRUSH rule 0 x 246 [15,3,5,11,7,1,12,9]
+  CRUSH rule 0 x 247 [6,4,9,12,1,2,10,14]
+  CRUSH rule 0 x 248 [5,13,7,11,9,15,3,1]
+  CRUSH rule 0 x 249 [10,14,7,3,9,13,1,4]
+  CRUSH rule 0 x 250 [12,15,1,10,5,6,3,9]
+  CRUSH rule 0 x 251 [13,2,15,5,6,1,9,10]
+  CRUSH rule 0 x 252 [7,5,13,9,3,10,14,1]
+  CRUSH rule 0 x 253 [3,13,15,10,7,4]
+  CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+  CRUSH rule 0 x 255 [1,9,13,2,6,10,4,15]
+  CRUSH rule 0 x 256 [6,9,13,1,3,14,5,10]
+  CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+  CRUSH rule 0 x 258 [12,5,6,10,2,1,14,9]
+  CRUSH rule 0 x 259 [9,10,4,3,14,13,1,7]
+  CRUSH rule 0 x 260 [10,12,6,9,3,15,1,4]
+  CRUSH rule 0 x 261 [13,7,2,1,15,5,11,9]
+  CRUSH rule 0 x 262 [15,3,12,7,4,9,1,11]
+  CRUSH rule 0 x 263 [12,6,10,9,5,15,3,1]
+  CRUSH rule 0 x 264 [13,14,11,3,1,4,7,9]
+  CRUSH rule 0 x 265 [12,10,14,5,7,1,9,3]
+  CRUSH rule 0 x 266 [14,7,11,1,2,9,4,12]
+  CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+  CRUSH rule 0 x 268 [4,1,15,12,6,11,3,9]
+  CRUSH rule 0 x 269 [11,1,15,5,13,9,7,2]
+  CRUSH rule 0 x 270 [7,11,12,3,1,14,9,4]
+  CRUSH rule 0 x 271 [4,7,3,13,15,10,9,1]
+  CRUSH rule 0 x 272 [15,5,13,10,6,2]
+  CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+  CRUSH rule 0 x 274 [10,2,5,6,13,9,15,1]
+  CRUSH rule 0 x 275 [10,3,4,7,14,13]
+  CRUSH rule 0 x 276 [5,12,9,2,11,7,15,1]
+  CRUSH rule 0 x 277 [14,3,13,4,1,9,11,7]
+  CRUSH rule 0 x 278 [5,6,14,3,1,11,13,9]
+  CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+  CRUSH rule 0 x 280 [7,3,14,9,1,11,4,13]
+  CRUSH rule 0 x 281 [5,11,14,7,9,13,2,1]
+  CRUSH rule 0 x 282 [2,1,13,14,9,7,5,10]
+  CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+  CRUSH rule 0 x 284 [5,11,7,15,3,13,1,9]
+  CRUSH rule 0 x 285 [15,5,3,1,6,13,11,9]
+  CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+  CRUSH rule 0 x 287 [12,4,9,1,3,11,15,7]
+  CRUSH rule 0 x 288 [4,12,10,7,1,3,14,9]
+  CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+  CRUSH rule 0 x 290 [12,2,5,6,15,9,1,10]
+  CRUSH rule 0 x 291 [7,11,1,14,5,9,2,12]
+  CRUSH rule 0 x 292 [4,10,6,3,14,9,12,1]
+  CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+  CRUSH rule 0 x 294 [9,12,3,14,6,11,5,1]
+  CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+  CRUSH rule 0 x 296 [3,1,13,7,14,9,10,4]
+  CRUSH rule 0 x 297 [6,13,4,14,10,1,2,9]
+  CRUSH rule 0 x 298 [14,9,13,1,4,2,7,10]
+  CRUSH rule 0 x 299 [14,12,11,6,4,2,1,9]
+  CRUSH rule 0 x 300 [15,7,10,5,1,3,13,9]
+  CRUSH rule 0 x 301 [9,11,7,1,13,14,4,2]
+  CRUSH rule 0 x 302 [9,7,1,13,5,10,3,15]
+  CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+  CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+  CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+  CRUSH rule 0 x 306 [10,12,4,6,9,2,15,1]
+  CRUSH rule 0 x 307 [11,12,15,5,6,2,1,9]
+  CRUSH rule 0 x 308 [12,14,10,9,1,2,5,7]
+  CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+  CRUSH rule 0 x 310 [3,1,5,10,14,9,7,12]
+  CRUSH rule 0 x 311 [3,9,7,1,14,13,10,5]
+  CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+  CRUSH rule 0 x 313 [9,15,3,7,5,13,1,11]
+  CRUSH rule 0 x 314 [2,15,9,5,6,12,1,11]
+  CRUSH rule 0 x 315 [15,2,13,1,11,9,6,4]
+  CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+  CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+  CRUSH rule 0 x 318 [4,1,15,11,9,13,6,2]
+  CRUSH rule 0 x 319 [2,15,4,1,11,9,7,12]
+  CRUSH rule 0 x 320 [5,7,13,9,11,2,1,15]
+  CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+  CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+  CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+  CRUSH rule 0 x 324 [5,6,10,15,2,13]
+  CRUSH rule 0 x 325 [9,10,14,5,1,6,2,13]
+  CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+  CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+  CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+  CRUSH rule 0 x 329 [2,6,15,5,9,10,13,1]
+  CRUSH rule 0 x 330 [3,9,11,13,1,6,5,14]
+  CRUSH rule 0 x 331 [12,14,6,3,1,4,10,9]
+  CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+  CRUSH rule 0 x 333 [6,5,3,12,14,10,9,1]
+  CRUSH rule 0 x 334 [4,9,2,12,7,11,15,1]
+  CRUSH rule 0 x 335 [11,7,1,5,13,2,9,15]
+  CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+  CRUSH rule 0 x 337 [15,11,3,7,12,5]
+  CRUSH rule 0 x 338 [10,5,3,6,15,1,9,13]
+  CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+  CRUSH rule 0 x 340 [11,6,12,4,9,3,14,1]
+  CRUSH rule 0 x 341 [7,5,2,10,14,9,1,12]
+  CRUSH rule 0 x 342 [12,14,1,9,2,11,4,7]
+  CRUSH rule 0 x 343 [12,14,9,6,10,2,4,1]
+  CRUSH rule 0 x 344 [9,11,5,2,14,13,1,7]
+  CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+  CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+  CRUSH rule 0 x 347 [10,2,12,6,9,1,14,5]
+  CRUSH rule 0 x 348 [7,9,10,1,14,13,3,4]
+  CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+  CRUSH rule 0 x 350 [13,9,15,4,10,7,2,1]
+  CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+  CRUSH rule 0 x 352 [1,12,11,9,4,7,3,15]
+  CRUSH rule 0 x 353 [10,14,12,2,9,1,4,6]
+  CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+  CRUSH rule 0 x 355 [13,14,6,10,2,5,1,9]
+  CRUSH rule 0 x 356 [15,13,2,9,6,5,1,11]
+  CRUSH rule 0 x 357 [4,11,1,13,3,14,6,9]
+  CRUSH rule 0 x 358 [12,7,2,9,1,14,10,4]
+  CRUSH rule 0 x 359 [5,15,7,11,3,13]
+  CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+  CRUSH rule 0 x 361 [5,3,13,6,1,14,11,9]
+  CRUSH rule 0 x 362 [2,9,11,13,1,6,5,15]
+  CRUSH rule 0 x 363 [7,12,3,9,15,4,1,10]
+  CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+  CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+  CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+  CRUSH rule 0 x 367 [7,13,3,1,5,11,15,9]
+  CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+  CRUSH rule 0 x 369 [7,5,3,13,14,9,11,1]
+  CRUSH rule 0 x 370 [4,7,14,1,2,9,11,12]
+  CRUSH rule 0 x 371 [1,7,12,3,4,15,10,9]
+  CRUSH rule 0 x 372 [10,4,3,14,6,1,12,9]
+  CRUSH rule 0 x 373 [15,5,2,6,13,1,9,10]
+  CRUSH rule 0 x 374 [3,15,12,5,1,6,10,9]
+  CRUSH rule 0 x 375 [5,2,14,1,6,13,11,9]
+  CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+  CRUSH rule 0 x 377 [1,15,2,4,9,11,12,6]
+  CRUSH rule 0 x 378 [9,12,2,15,1,5,11,6]
+  CRUSH rule 0 x 379 [11,2,15,5,7,9,13,1]
+  CRUSH rule 0 x 380 [6,1,12,11,2,9,5,14]
+  CRUSH rule 0 x 381 [15,13,7,5,10,2,1,9]
+  CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+  CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+  CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+  CRUSH rule 0 x 385 [4,6,15,3,10,9,1,13]
+  CRUSH rule 0 x 386 [14,3,11,13,5,6,9,1]
+  CRUSH rule 0 x 387 [1,11,5,7,9,2,14,12]
+  CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+  CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+  CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+  CRUSH rule 0 x 391 [3,4,9,13,7,10,1,14]
+  CRUSH rule 0 x 392 [11,5,14,7,1,9,2,12]
+  CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+  CRUSH rule 0 x 394 [4,9,3,15,13,6,1,11]
+  CRUSH rule 0 x 395 [10,13,5,15,6,9,3,1]
+  CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+  CRUSH rule 0 x 397 [1,14,9,4,12,10,3,7]
+  CRUSH rule 0 x 398 [9,2,1,5,12,6,11,15]
+  CRUSH rule 0 x 399 [5,9,14,3,1,10,13,7]
+  CRUSH rule 0 x 400 [10,6,2,4,15,12,1,9]
+  CRUSH rule 0 x 401 [6,9,11,12,4,3,15,1]
+  CRUSH rule 0 x 402 [4,7,9,2,13,1,15,11]
+  CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+  CRUSH rule 0 x 404 [14,12,7,9,2,1,5,11]
+  CRUSH rule 0 x 405 [9,15,11,2,4,7,13,1]
+  CRUSH rule 0 x 406 [12,14,9,2,7,10,4,1]
+  CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+  CRUSH rule 0 x 408 [7,1,5,2,10,15,13,9]
+  CRUSH rule 0 x 409 [11,2,4,13,1,15,7,9]
+  CRUSH rule 0 x 410 [6,4,14,2,12,9,10,1]
+  CRUSH rule 0 x 411 [13,11,15,6,4,1,9,2]
+  CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+  CRUSH rule 0 x 413 [13,5,3,11,6,9,1,14]
+  CRUSH rule 0 x 414 [3,11,9,13,4,1,6,15]
+  CRUSH rule 0 x 415 [6,10,14,5,1,13,3,9]
+  CRUSH rule 0 x 416 [13,1,4,7,2,9,14,11]
+  CRUSH rule 0 x 417 [4,12,1,15,2,11,9,6]
+  CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+  CRUSH rule 0 x 419 [5,14,10,9,2,12,6,1]
+  CRUSH rule 0 x 420 [2,4,9,11,6,14,13,1]
+  CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+  CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+  CRUSH rule 0 x 423 [3,15,12,6,5,1,9,10]
+  CRUSH rule 0 x 424 [6,10,12,2,5,1,14,9]
+  CRUSH rule 0 x 425 [11,15,2,13,5,7,9,1]
+  CRUSH rule 0 x 426 [12,4,7,1,9,10,14,2]
+  CRUSH rule 0 x 427 [14,10,3,1,9,7,5,13]
+  CRUSH rule 0 x 428 [12,7,9,4,2,1,14,10]
+  CRUSH rule 0 x 429 [3,4,9,7,11,12,1,14]
+  CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+  CRUSH rule 0 x 431 [9,3,7,1,12,5,14,11]
+  CRUSH rule 0 x 432 [4,1,12,7,15,2,10,9]
+  CRUSH rule 0 x 433 [4,11,12,15,7,3]
+  CRUSH rule 0 x 434 [2,14,9,1,5,11,7,13]
+  CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+  CRUSH rule 0 x 436 [9,15,10,2,4,1,12,7]
+  CRUSH rule 0 x 437 [9,6,3,14,10,12,5,1]
+  CRUSH rule 0 x 438 [7,2,13,4,11,1,9,14]
+  CRUSH rule 0 x 439 [7,14,4,3,12,10]
+  CRUSH rule 0 x 440 [14,11,9,2,7,12,1,5]
+  CRUSH rule 0 x 441 [2,4,11,9,13,6,1,14]
+  CRUSH rule 0 x 442 [10,13,9,7,15,1,4,2]
+  CRUSH rule 0 x 443 [12,15,10,9,2,1,6,4]
+  CRUSH rule 0 x 444 [4,13,7,14,3,1,9,11]
+  CRUSH rule 0 x 445 [4,2,15,7,1,9,11,12]
+  CRUSH rule 0 x 446 [12,10,6,9,4,1,15,3]
+  CRUSH rule 0 x 447 [15,7,13,1,4,9,3,11]
+  CRUSH rule 0 x 448 [5,2,13,7,15,10]
+  CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+  CRUSH rule 0 x 450 [2,4,6,9,15,1,13,10]
+  CRUSH rule 0 x 451 [6,14,11,3,9,1,12,5]
+  CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+  CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+  CRUSH rule 0 x 454 [10,4,2,6,15,12,9,1]
+  CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+  CRUSH rule 0 x 456 [5,7,13,1,11,3,9,15]
+  CRUSH rule 0 x 457 [9,1,5,7,11,13,15]
+  CRUSH rule 0 x 458 [9,11,15,4,7,2,12,1]
+  CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+  CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+  CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+  CRUSH rule 0 x 462 [4,7,12,14,11,1,3,9]
+  CRUSH rule 0 x 463 [4,12,14,11,2,7,1,9]
+  CRUSH rule 0 x 464 [4,2,15,10,1,9,13,7]
+  CRUSH rule 0 x 465 [5,10,9,7,13,1,3,14]
+  CRUSH rule 0 x 466 [13,5,2,15,9,11,6,1]
+  CRUSH rule 0 x 467 [13,6,14,3,9,1,11,5]
+  CRUSH rule 0 x 468 [10,7,12,14,4,1,9,2]
+  CRUSH rule 0 x 469 [4,9,6,14,12,11,3,1]
+  CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+  CRUSH rule 0 x 471 [6,1,5,14,13,10,9,3]
+  CRUSH rule 0 x 472 [2,14,7,5,13,1,11,9]
+  CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+  CRUSH rule 0 x 474 [15,10,4,12,6,9,2,1]
+  CRUSH rule 0 x 475 [10,5,12,9,14,3,6,1]
+  CRUSH rule 0 x 476 [3,6,10,12,1,15,9,4]
+  CRUSH rule 0 x 477 [6,13,5,15,11,9,2,1]
+  CRUSH rule 0 x 478 [4,15,1,3,7,12,10,9]
+  CRUSH rule 0 x 479 [13,11,1,6,14,5,9,3]
+  CRUSH rule 0 x 480 [1,13,6,4,9,14,11,3]
+  CRUSH rule 0 x 481 [15,12,7,9,1,3,10,4]
+  CRUSH rule 0 x 482 [2,12,9,1,7,11,14,4]
+  CRUSH rule 0 x 483 [10,1,4,15,9,7,13,2]
+  CRUSH rule 0 x 484 [1,4,10,13,7,14,2,9]
+  CRUSH rule 0 x 485 [9,4,3,1,14,12,7,10]
+  CRUSH rule 0 x 486 [3,10,15,9,7,13,4,1]
+  CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+  CRUSH rule 0 x 488 [14,4,1,9,2,6,10,12]
+  CRUSH rule 0 x 489 [11,4,2,13,15,7]
+  CRUSH rule 0 x 490 [4,9,1,3,13,15,6,11]
+  CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+  CRUSH rule 0 x 492 [5,7,11,3,14,9,1,13]
+  CRUSH rule 0 x 493 [12,1,4,15,3,11,9,6]
+  CRUSH rule 0 x 494 [1,7,13,4,15,9,10,3]
+  CRUSH rule 0 x 495 [3,15,7,1,9,5,12,11]
+  CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+  CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+  CRUSH rule 0 x 498 [10,6,1,5,9,12,3,15]
+  CRUSH rule 0 x 499 [14,3,12,5,1,11,9,7]
+  CRUSH rule 0 x 500 [15,9,6,12,11,2,1,5]
+  CRUSH rule 0 x 501 [10,13,1,9,3,14,5,7]
+  CRUSH rule 0 x 502 [5,1,14,11,7,12,9,2]
+  CRUSH rule 0 x 503 [15,10,7,9,1,12,4,2]
+  CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+  CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+  CRUSH rule 0 x 506 [11,7,9,14,12,1,2,5]
+  CRUSH rule 0 x 507 [4,14,13,3,9,7,1,10]
+  CRUSH rule 0 x 508 [12,1,4,9,2,11,15,7]
+  CRUSH rule 0 x 509 [4,2,6,9,14,1,10,13]
+  CRUSH rule 0 x 510 [5,3,1,12,11,14,9,7]
+  CRUSH rule 0 x 511 [2,12,10,6,14,5]
+  CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+  CRUSH rule 0 x 513 [4,9,11,3,13,7,1,14]
+  CRUSH rule 0 x 514 [11,9,3,4,12,15,6,1]
+  CRUSH rule 0 x 515 [12,14,6,5,3,9,1,10]
+  CRUSH rule 0 x 516 [14,11,1,12,3,7,4,9]
+  CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+  CRUSH rule 0 x 518 [3,5,7,12,15,11,9,1]
+  CRUSH rule 0 x 519 [12,14,2,1,4,6,9,10]
+  CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+  CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+  CRUSH rule 0 x 522 [4,12,11,1,15,3,9,6]
+  CRUSH rule 0 x 523 [3,1,5,9,15,10,13,7]
+  CRUSH rule 0 x 524 [15,9,3,11,13,7,4,1]
+  CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+  CRUSH rule 0 x 526 [10,2,5,13,6,15,1,9]
+  CRUSH rule 0 x 527 [3,13,4,1,9,10,14,7]
+  CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+  CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+  CRUSH rule 0 x 530 [11,9,12,7,5,1,3,15]
+  CRUSH rule 0 x 531 [9,15,4,7,2,13,1,11]
+  CRUSH rule 0 x 532 [5,3,13,7,9,14,1,10]
+  CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+  CRUSH rule 0 x 534 [11,9,3,7,15,4,1,12]
+  CRUSH rule 0 x 535 [11,1,3,5,14,9,12,7]
+  CRUSH rule 0 x 536 [9,1,14,13,4,6,2,11]
+  CRUSH rule 0 x 537 [15,5,13,2,7,11]
+  CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+  CRUSH rule 0 x 539 [10,12,6,14,1,2,9,5]
+  CRUSH rule 0 x 540 [12,15,7,3,9,11,1,4]
+  CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+  CRUSH rule 0 x 542 [3,9,15,5,11,12,7,1]
+  CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+  CRUSH rule 0 x 544 [3,15,9,11,7,4,12,1]
+  CRUSH rule 0 x 545 [14,10,7,12,4,9,1,3]
+  CRUSH rule 0 x 546 [5,15,13,7,1,10,9,2]
+  CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+  CRUSH rule 0 x 548 [11,7,12,15,4,2]
+  CRUSH rule 0 x 549 [14,1,4,9,13,6,3,10]
+  CRUSH rule 0 x 550 [9,15,3,13,1,6,4,11]
+  CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+  CRUSH rule 0 x 552 [2,11,14,1,9,6,5,12]
+  CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+  CRUSH rule 0 x 554 [11,14,6,4,13,9,3,1]
+  CRUSH rule 0 x 555 [6,5,10,9,14,2,13,1]
+  CRUSH rule 0 x 556 [15,6,3,13,11,4,1,9]
+  CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+  CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+  CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+  CRUSH rule 0 x 560 [4,9,12,6,3,10,1,15]
+  CRUSH rule 0 x 561 [12,7,1,2,5,15,11,9]
+  CRUSH rule 0 x 562 [7,13,9,14,2,1,11,4]
+  CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+  CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+  CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+  CRUSH rule 0 x 566 [6,14,4,2,13,11]
+  CRUSH rule 0 x 567 [15,4,11,6,3,12]
+  CRUSH rule 0 x 568 [4,14,1,6,10,13,3,9]
+  CRUSH rule 0 x 569 [11,3,15,13,5,1,9,7]
+  CRUSH rule 0 x 570 [1,10,13,4,7,2,9,14]
+  CRUSH rule 0 x 571 [10,12,14,9,4,2,1,6]
+  CRUSH rule 0 x 572 [12,14,3,10,6,1,4,9]
+  CRUSH rule 0 x 573 [7,15,11,2,12,9,4,1]
+  CRUSH rule 0 x 574 [11,14,13,1,3,7,4,9]
+  CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+  CRUSH rule 0 x 576 [3,15,11,9,1,6,5,13]
+  CRUSH rule 0 x 577 [13,9,6,15,3,11,4,1]
+  CRUSH rule 0 x 578 [4,10,1,2,7,13,14,9]
+  CRUSH rule 0 x 579 [13,1,15,2,10,7,5,9]
+  CRUSH rule 0 x 580 [3,12,4,1,10,15,7,9]
+  CRUSH rule 0 x 581 [7,14,12,10,1,2,9,5]
+  CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+  CRUSH rule 0 x 583 [4,15,1,9,10,12,2,6]
+  CRUSH rule 0 x 584 [10,1,5,13,6,9,2,15]
+  CRUSH rule 0 x 585 [5,3,6,1,11,14,13,9]
+  CRUSH rule 0 x 586 [7,10,14,12,9,3,5,1]
+  CRUSH rule 0 x 587 [11,6,9,4,1,14,13,2]
+  CRUSH rule 0 x 588 [3,12,7,15,4,9,1,10]
+  CRUSH rule 0 x 589 [9,7,12,1,10,3,4,15]
+  CRUSH rule 0 x 590 [12,1,3,9,10,6,4,14]
+  CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+  CRUSH rule 0 x 592 [15,12,9,7,5,2,11,1]
+  CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+  CRUSH rule 0 x 594 [12,14,2,9,7,4,1,11]
+  CRUSH rule 0 x 595 [12,7,10,3,1,14,9,4]
+  CRUSH rule 0 x 596 [2,7,12,11,1,5,15,9]
+  CRUSH rule 0 x 597 [15,1,2,10,7,13,5,9]
+  CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+  CRUSH rule 0 x 599 [13,11,1,5,6,2,15,9]
+  CRUSH rule 0 x 600 [4,12,3,10,9,7,1,14]
+  CRUSH rule 0 x 601 [13,5,15,2,1,7,9,10]
+  CRUSH rule 0 x 602 [3,11,7,1,13,15,5,9]
+  CRUSH rule 0 x 603 [3,1,4,14,10,9,6,12]
+  CRUSH rule 0 x 604 [14,2,6,1,11,13,9,4]
+  CRUSH rule 0 x 605 [2,7,12,5,14,10,1,9]
+  CRUSH rule 0 x 606 [12,15,1,5,7,9,3,11]
+  CRUSH rule 0 x 607 [3,9,10,14,7,1,4,12]
+  CRUSH rule 0 x 608 [13,10,1,7,9,15,5,2]
+  CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+  CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+  CRUSH rule 0 x 611 [13,1,5,3,10,7,15,9]
+  CRUSH rule 0 x 612 [7,1,2,13,9,15,4,11]
+  CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+  CRUSH rule 0 x 614 [9,4,15,3,1,11,6,12]
+  CRUSH rule 0 x 615 [9,4,11,2,1,12,6,15]
+  CRUSH rule 0 x 616 [10,14,1,5,3,6,12,9]
+  CRUSH rule 0 x 617 [15,7,2,11,12,1,9,4]
+  CRUSH rule 0 x 618 [4,2,10,6,14,9,1,12]
+  CRUSH rule 0 x 619 [15,4,3,9,6,1,13,11]
+  CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+  CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+  CRUSH rule 0 x 622 [10,2,13,5,15,9,1,7]
+  CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+  CRUSH rule 0 x 624 [3,9,15,6,10,1,5,12]
+  CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+  CRUSH rule 0 x 626 [10,12,2,1,9,7,5,14]
+  CRUSH rule 0 x 627 [1,12,10,14,3,5,9,7]
+  CRUSH rule 0 x 628 [15,13,11,4,2,1,7,9]
+  CRUSH rule 0 x 629 [5,6,15,12,1,10,3,9]
+  CRUSH rule 0 x 630 [1,4,12,9,3,7,15,11]
+  CRUSH rule 0 x 631 [5,7,1,15,12,11,3,9]
+  CRUSH rule 0 x 632 [12,3,11,9,6,1,15,5]
+  CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+  CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+  CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+  CRUSH rule 0 x 636 [13,6,11,3,15,9,1,4]
+  CRUSH rule 0 x 637 [3,1,10,6,9,12,4,14]
+  CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+  CRUSH rule 0 x 639 [6,9,14,4,3,1,10,13]
+  CRUSH rule 0 x 640 [9,6,1,11,14,2,4,13]
+  CRUSH rule 0 x 641 [10,6,5,14,1,9,12,2]
+  CRUSH rule 0 x 642 [1,15,4,6,2,10,9,12]
+  CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+  CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+  CRUSH rule 0 x 645 [14,2,4,9,10,1,7,13]
+  CRUSH rule 0 x 646 [5,13,14,1,6,9,2,11]
+  CRUSH rule 0 x 647 [10,1,9,13,6,2,14,5]
+  CRUSH rule 0 x 648 [6,5,2,14,11,1,12,9]
+  CRUSH rule 0 x 649 [3,9,13,11,4,14,1,7]
+  CRUSH rule 0 x 650 [10,9,4,15,12,7,1,2]
+  CRUSH rule 0 x 651 [3,9,5,7,14,1,13,11]
+  CRUSH rule 0 x 652 [15,9,4,6,13,1,2,11]
+  CRUSH rule 0 x 653 [11,14,1,3,6,9,12,5]
+  CRUSH rule 0 x 654 [13,6,2,10,15,4,1,9]
+  CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+  CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+  CRUSH rule 0 x 657 [11,15,3,5,7,13,1,9]
+  CRUSH rule 0 x 658 [7,2,10,12,1,4,9,14]
+  CRUSH rule 0 x 659 [2,5,14,6,10,12]
+  CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+  CRUSH rule 0 x 661 [7,15,3,12,11,4,9,1]
+  CRUSH rule 0 x 662 [15,2,12,5,1,10,9,7]
+  CRUSH rule 0 x 663 [14,9,13,10,5,3,1,6]
+  CRUSH rule 0 x 664 [6,10,12,4,9,2,1,15]
+  CRUSH rule 0 x 665 [2,9,12,1,7,10,4,15]
+  CRUSH rule 0 x 666 [12,3,6,1,15,9,10,4]
+  CRUSH rule 0 x 667 [1,9,12,10,2,14,7,4]
+  CRUSH rule 0 x 668 [9,5,1,2,6,11,13,15]
+  CRUSH rule 0 x 669 [9,7,14,5,11,13,1,2]
+  CRUSH rule 0 x 670 [6,10,9,13,1,2,15,4]
+  CRUSH rule 0 x 671 [6,15,5,10,13,3]
+  CRUSH rule 0 x 672 [2,9,13,1,4,14,6,10]
+  CRUSH rule 0 x 673 [7,10,5,9,15,13,2,1]
+  CRUSH rule 0 x 674 [7,12,10,1,14,9,3,4]
+  CRUSH rule 0 x 675 [9,5,1,10,6,14,12,2]
+  CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+  CRUSH rule 0 x 677 [2,12,1,4,10,6,15,9]
+  CRUSH rule 0 x 678 [1,2,4,10,12,14,9,6]
+  CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+  CRUSH rule 0 x 680 [7,11,3,1,15,4,9,12]
+  CRUSH rule 0 x 681 [6,4,3,11,14,13,1,9]
+  CRUSH rule 0 x 682 [6,1,11,15,12,2,5,9]
+  CRUSH rule 0 x 683 [6,13,2,4,9,14,10,1]
+  CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+  CRUSH rule 0 x 685 [5,1,15,7,9,2,10,13]
+  CRUSH rule 0 x 686 [1,9,11,14,6,13,4,3]
+  CRUSH rule 0 x 687 [7,13,3,5,11,9,15,1]
+  CRUSH rule 0 x 688 [11,9,1,14,3,5,7,12]
+  CRUSH rule 0 x 689 [5,2,9,12,1,14,11,7]
+  CRUSH rule 0 x 690 [9,7,10,3,13,15,5,1]
+  CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+  CRUSH rule 0 x 692 [15,5,1,2,9,11,12,7]
+  CRUSH rule 0 x 693 [5,6,12,15,2,10,9,1]
+  CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+  CRUSH rule 0 x 695 [6,13,14,10,9,5,1,3]
+  CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+  CRUSH rule 0 x 697 [13,11,3,6,4,14,9,1]
+  CRUSH rule 0 x 698 [11,13,4,2,6,1,9,15]
+  CRUSH rule 0 x 699 [7,14,12,4,2,11]
+  CRUSH rule 0 x 700 [12,14,11,9,4,6,3,1]
+  CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+  CRUSH rule 0 x 702 [3,12,15,6,5,11,1,9]
+  CRUSH rule 0 x 703 [15,11,13,3,4,7,1,9]
+  CRUSH rule 0 x 704 [6,4,2,15,11,1,13,9]
+  CRUSH rule 0 x 705 [14,6,11,5,1,13,9,3]
+  CRUSH rule 0 x 706 [1,12,3,6,4,10,15,9]
+  CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+  CRUSH rule 0 x 708 [3,10,5,1,15,9,7,13]
+  CRUSH rule 0 x 709 [11,12,3,7,5,14,1,9]
+  CRUSH rule 0 x 710 [14,2,11,9,5,7,12,1]
+  CRUSH rule 0 x 711 [14,3,9,10,12,5,6,1]
+  CRUSH rule 0 x 712 [12,3,11,15,9,1,6,4]
+  CRUSH rule 0 x 713 [11,9,3,15,13,6,4,1]
+  CRUSH rule 0 x 714 [12,1,9,7,2,15,10,5]
+  CRUSH rule 0 x 715 [6,1,14,4,11,12,3,9]
+  CRUSH rule 0 x 716 [11,13,9,14,5,2,1,7]
+  CRUSH rule 0 x 717 [12,4,10,9,15,1,2,7]
+  CRUSH rule 0 x 718 [7,15,5,2,11,13]
+  CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+  CRUSH rule 0 x 720 [4,13,10,2,7,9,1,14]
+  CRUSH rule 0 x 721 [11,3,14,9,1,12,4,6]
+  CRUSH rule 0 x 722 [2,4,6,1,9,15,13,10]
+  CRUSH rule 0 x 723 [2,1,12,15,11,7,5,9]
+  CRUSH rule 0 x 724 [7,1,9,10,5,15,13,2]
+  CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+  CRUSH rule 0 x 726 [7,14,4,3,11,13,9,1]
+  CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+  CRUSH rule 0 x 728 [13,11,4,6,15,2]
+  CRUSH rule 0 x 729 [15,11,4,6,2,9,1,13]
+  CRUSH rule 0 x 730 [3,7,1,13,11,15,9,5]
+  CRUSH rule 0 x 731 [9,1,6,5,2,11,13,15]
+  CRUSH rule 0 x 732 [1,2,10,13,9,4,7,15]
+  CRUSH rule 0 x 733 [11,3,5,6,1,9,12,15]
+  CRUSH rule 0 x 734 [14,3,11,7,12,9,4,1]
+  CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+  CRUSH rule 0 x 736 [3,9,1,11,7,5,13,14]
+  CRUSH rule 0 x 737 [1,4,2,12,9,10,6,15]
+  CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+  CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+  CRUSH rule 0 x 740 [7,9,10,13,1,15,2,5]
+  CRUSH rule 0 x 741 [12,11,7,15,2,5]
+  CRUSH rule 0 x 742 [9,7,4,11,12,1,14,3]
+  CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+  CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+  CRUSH rule 0 x 745 [3,6,1,4,11,12,14,9]
+  CRUSH rule 0 x 746 [3,7,9,10,14,5,1,13]
+  CRUSH rule 0 x 747 [15,11,5,2,13,9,1,7]
+  CRUSH rule 0 x 748 [6,10,13,2,14,5,9,1]
+  CRUSH rule 0 x 749 [14,9,10,7,5,1,2,12]
+  CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+  CRUSH rule 0 x 751 [15,1,6,9,5,11,12,3]
+  CRUSH rule 0 x 752 [13,1,7,3,11,15,9,4]
+  CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+  CRUSH rule 0 x 754 [14,12,11,4,2,1,9,6]
+  CRUSH rule 0 x 755 [13,6,1,10,4,2,14,9]
+  CRUSH rule 0 x 756 [3,4,14,6,1,10,13,9]
+  CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+  CRUSH rule 0 x 758 [6,3,4,10,15,13,9,1]
+  CRUSH rule 0 x 759 [5,7,3,14,11,1,9,13]
+  CRUSH rule 0 x 760 [1,15,10,12,4,3,9,7]
+  CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+  CRUSH rule 0 x 762 [1,4,10,9,3,7,14,12]
+  CRUSH rule 0 x 763 [4,13,1,14,7,10,2,9]
+  CRUSH rule 0 x 764 [1,14,6,13,9,5,2,10]
+  CRUSH rule 0 x 765 [9,15,2,13,4,1,11,7]
+  CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+  CRUSH rule 0 x 767 [6,11,4,3,12,14]
+  CRUSH rule 0 x 768 [2,12,15,7,1,11,9,4]
+  CRUSH rule 0 x 769 [15,1,9,2,11,12,7,4]
+  CRUSH rule 0 x 770 [15,13,4,6,3,10,1,9]
+  CRUSH rule 0 x 771 [9,2,12,11,6,14,5,1]
+  CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+  CRUSH rule 0 x 773 [3,7,4,15,1,12,11,9]
+  CRUSH rule 0 x 774 [12,6,3,15,5,9,10,1]
+  CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+  CRUSH rule 0 x 776 [10,15,3,9,6,13,1,5]
+  CRUSH rule 0 x 777 [11,13,4,7,1,14,9,2]
+  CRUSH rule 0 x 778 [13,1,9,11,15,6,3,5]
+  CRUSH rule 0 x 779 [5,11,1,14,2,9,13,6]
+  CRUSH rule 0 x 780 [13,9,3,6,4,1,14,10]
+  CRUSH rule 0 x 781 [5,7,14,3,1,12,11,9]
+  CRUSH rule 0 x 782 [2,15,9,7,11,13,4,1]
+  CRUSH rule 0 x 783 [12,7,5,14,9,1,2,10]
+  CRUSH rule 0 x 784 [14,1,10,13,3,4,7,9]
+  CRUSH rule 0 x 785 [6,12,1,2,4,9,15,10]
+  CRUSH rule 0 x 786 [10,5,2,15,1,7,12,9]
+  CRUSH rule 0 x 787 [1,12,10,2,9,4,14,6]
+  CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+  CRUSH rule 0 x 789 [9,2,14,7,4,12,1,10]
+  CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+  CRUSH rule 0 x 791 [9,4,7,13,14,11,1,3]
+  CRUSH rule 0 x 792 [6,4,15,10,12,3]
+  CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+  CRUSH rule 0 x 794 [5,12,2,14,9,10,1,6]
+  CRUSH rule 0 x 795 [6,14,12,4,10,1,2,9]
+  CRUSH rule 0 x 796 [11,2,12,6,15,4]
+  CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+  CRUSH rule 0 x 798 [5,11,6,13,1,3,15,9]
+  CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+  CRUSH rule 0 x 800 [6,3,4,11,15,13]
+  CRUSH rule 0 x 801 [2,5,6,13,9,1,10,15]
+  CRUSH rule 0 x 802 [1,4,12,7,3,9,10,14]
+  CRUSH rule 0 x 803 [7,2,4,1,11,13,9,14]
+  CRUSH rule 0 x 804 [5,14,9,7,3,1,12,10]
+  CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+  CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+  CRUSH rule 0 x 807 [14,2,7,4,9,12,1,10]
+  CRUSH rule 0 x 808 [2,15,12,7,9,1,5,10]
+  CRUSH rule 0 x 809 [1,11,7,12,4,2,15,9]
+  CRUSH rule 0 x 810 [2,5,9,12,15,1,7,11]
+  CRUSH rule 0 x 811 [15,6,3,10,1,5,9,12]
+  CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+  CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+  CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+  CRUSH rule 0 x 815 [15,12,9,4,10,6,1,2]
+  CRUSH rule 0 x 816 [14,10,13,7,3,9,4,1]
+  CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+  CRUSH rule 0 x 818 [15,2,11,4,1,12,6,9]
+  CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+  CRUSH rule 0 x 820 [3,6,9,12,11,15,4,1]
+  CRUSH rule 0 x 821 [15,10,9,13,3,4,7,1]
+  CRUSH rule 0 x 822 [10,13,2,9,7,4,14,1]
+  CRUSH rule 0 x 823 [2,6,12,10,15,4,1,9]
+  CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+  CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+  CRUSH rule 0 x 826 [5,2,11,15,1,12,9,7]
+  CRUSH rule 0 x 827 [13,5,1,3,7,9,11,14]
+  CRUSH rule 0 x 828 [12,6,10,5,1,9,2,15]
+  CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+  CRUSH rule 0 x 830 [15,13,2,9,7,11,1,5]
+  CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+  CRUSH rule 0 x 832 [14,11,13,2,9,4,6,1]
+  CRUSH rule 0 x 833 [9,13,3,11,7,5,15,1]
+  CRUSH rule 0 x 834 [9,7,5,1,11,2,13,14]
+  CRUSH rule 0 x 835 [14,3,13,6,4,9,1,10]
+  CRUSH rule 0 x 836 [3,9,10,13,1,5,14,7]
+  CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+  CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+  CRUSH rule 0 x 839 [3,4,6,10,15,1,13,9]
+  CRUSH rule 0 x 840 [10,15,12,4,7,1,2,9]
+  CRUSH rule 0 x 841 [3,5,7,12,11,15,1,9]
+  CRUSH rule 0 x 842 [9,13,2,6,5,14,10,1]
+  CRUSH rule 0 x 843 [14,7,4,9,3,12,1,10]
+  CRUSH rule 0 x 844 [7,1,4,15,9,2,11,12]
+  CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+  CRUSH rule 0 x 846 [3,7,15,13,1,9,10,4]
+  CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+  CRUSH rule 0 x 848 [11,13,1,14,5,9,2,7]
+  CRUSH rule 0 x 849 [3,15,11,9,6,1,13,5]
+  CRUSH rule 0 x 850 [1,3,10,6,14,4,9,12]
+  CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+  CRUSH rule 0 x 852 [9,12,4,7,15,2,11,1]
+  CRUSH rule 0 x 853 [13,14,6,11,2,4,9,1]
+  CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+  CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+  CRUSH rule 0 x 856 [5,10,7,3,15,9,12,1]
+  CRUSH rule 0 x 857 [4,3,13,11,9,1,7,14]
+  CRUSH rule 0 x 858 [5,15,6,3,9,12,1,10]
+  CRUSH rule 0 x 859 [5,15,6,2,1,11,12,9]
+  CRUSH rule 0 x 860 [11,14,1,12,6,9,2,4]
+  CRUSH rule 0 x 861 [13,7,4,10,1,14,3,9]
+  CRUSH rule 0 x 862 [5,10,9,7,3,12,1,15]
+  CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+  CRUSH rule 0 x 864 [6,13,4,2,10,15,1,9]
+  CRUSH rule 0 x 865 [4,1,14,11,6,9,3,13]
+  CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+  CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+  CRUSH rule 0 x 868 [14,11,7,2,1,4,9,12]
+  CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+  CRUSH rule 0 x 870 [14,9,11,4,3,12,6,1]
+  CRUSH rule 0 x 871 [6,2,1,4,15,13,11,9]
+  CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+  CRUSH rule 0 x 873 [2,5,12,10,1,9,15,7]
+  CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+  CRUSH rule 0 x 875 [10,6,14,1,12,5,9,3]
+  CRUSH rule 0 x 876 [14,7,13,3,9,1,11,4]
+  CRUSH rule 0 x 877 [15,11,13,9,5,1,6,3]
+  CRUSH rule 0 x 878 [7,14,3,13,9,1,11,4]
+  CRUSH rule 0 x 879 [12,2,7,4,10,15]
+  CRUSH rule 0 x 880 [2,12,10,7,1,4,9,14]
+  CRUSH rule 0 x 881 [6,3,1,11,4,15,9,13]
+  CRUSH rule 0 x 882 [11,13,7,1,2,15,4,9]
+  CRUSH rule 0 x 883 [13,1,3,10,6,5,9,15]
+  CRUSH rule 0 x 884 [6,15,4,9,3,11,12,1]
+  CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+  CRUSH rule 0 x 886 [13,11,4,2,1,14,9,6]
+  CRUSH rule 0 x 887 [14,4,12,11,2,6,9,1]
+  CRUSH rule 0 x 888 [10,12,7,15,9,2,1,5]
+  CRUSH rule 0 x 889 [15,13,4,1,6,2,10,9]
+  CRUSH rule 0 x 890 [10,12,14,2,9,5,6,1]
+  CRUSH rule 0 x 891 [9,5,11,6,3,15,12,1]
+  CRUSH rule 0 x 892 [12,15,2,4,7,9,11,1]
+  CRUSH rule 0 x 893 [1,3,5,9,6,10,14,12]
+  CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+  CRUSH rule 0 x 895 [2,1,11,5,7,15,13,9]
+  CRUSH rule 0 x 896 [9,1,14,10,4,12,2,7]
+  CRUSH rule 0 x 897 [7,5,14,3,1,9,11,12]
+  CRUSH rule 0 x 898 [10,6,12,9,15,5,2,1]
+  CRUSH rule 0 x 899 [1,11,5,3,13,14,9,6]
+  CRUSH rule 0 x 900 [2,9,10,7,13,14,5,1]
+  CRUSH rule 0 x 901 [9,12,11,3,14,4,1,6]
+  CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+  CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+  CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+  CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+  CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+  CRUSH rule 0 x 907 [7,12,3,9,10,5,14,1]
+  CRUSH rule 0 x 908 [2,15,9,6,10,13,5,1]
+  CRUSH rule 0 x 909 [10,14,1,13,2,9,7,4]
+  CRUSH rule 0 x 910 [12,7,4,15,10,3,1,9]
+  CRUSH rule 0 x 911 [11,15,2,4,9,13,6,1]
+  CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+  CRUSH rule 0 x 913 [4,6,10,1,12,3,9,14]
+  CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+  CRUSH rule 0 x 915 [12,14,1,9,4,3,11,6]
+  CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+  CRUSH rule 0 x 917 [1,15,6,5,10,3,13,9]
+  CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+  CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+  CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+  CRUSH rule 0 x 921 [1,11,6,13,4,2,9,14]
+  CRUSH rule 0 x 922 [6,4,14,13,3,1,10,9]
+  CRUSH rule 0 x 923 [12,2,5,14,10,1,9,6]
+  CRUSH rule 0 x 924 [6,2,14,13,9,1,11,5]
+  CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+  CRUSH rule 0 x 926 [3,13,10,1,14,9,6,5]
+  CRUSH rule 0 x 927 [6,5,1,11,14,2,13,9]
+  CRUSH rule 0 x 928 [13,1,3,9,6,11,15,5]
+  CRUSH rule 0 x 929 [10,7,1,5,2,12,9,14]
+  CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+  CRUSH rule 0 x 931 [6,15,11,9,5,3,1,13]
+  CRUSH rule 0 x 932 [13,2,5,11,9,1,6,15]
+  CRUSH rule 0 x 933 [12,7,14,10,4,1,2,9]
+  CRUSH rule 0 x 934 [12,2,5,7,9,1,15,11]
+  CRUSH rule 0 x 935 [6,11,1,14,5,13,3,9]
+  CRUSH rule 0 x 936 [9,12,7,5,1,2,14,11]
+  CRUSH rule 0 x 937 [14,2,11,1,13,4,9,6]
+  CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+  CRUSH rule 0 x 939 [6,4,14,9,12,1,11,2]
+  CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+  CRUSH rule 0 x 941 [3,12,4,7,14,10]
+  CRUSH rule 0 x 942 [15,12,10,4,1,9,3,7]
+  CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+  CRUSH rule 0 x 944 [2,9,4,7,1,14,12,11]
+  CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+  CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+  CRUSH rule 0 x 947 [11,3,14,1,12,5,6,9]
+  CRUSH rule 0 x 948 [7,13,11,5,14,2,1,9]
+  CRUSH rule 0 x 949 [9,1,12,5,15,10,2,6]
+  CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+  CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+  CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+  CRUSH rule 0 x 953 [1,3,6,10,12,14,4,9]
+  CRUSH rule 0 x 954 [10,2,14,9,4,6,12,1]
+  CRUSH rule 0 x 955 [7,14,3,1,10,4,9,12]
+  CRUSH rule 0 x 956 [1,6,11,5,14,3,9,13]
+  CRUSH rule 0 x 957 [14,11,1,12,6,9,4,3]
+  CRUSH rule 0 x 958 [15,4,3,11,1,6,12,9]
+  CRUSH rule 0 x 959 [2,1,12,15,10,9,4,6]
+  CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+  CRUSH rule 0 x 961 [3,13,11,9,6,1,4,15]
+  CRUSH rule 0 x 962 [5,11,3,14,1,6,13,9]
+  CRUSH rule 0 x 963 [13,10,15,4,6,9,1,3]
+  CRUSH rule 0 x 964 [7,11,4,9,2,12,1,15]
+  CRUSH rule 0 x 965 [12,2,9,7,4,15,11,1]
+  CRUSH rule 0 x 966 [12,14,9,4,1,2,11,7]
+  CRUSH rule 0 x 967 [7,5,3,10,12,14]
+  CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+  CRUSH rule 0 x 969 [11,4,7,1,9,14,13,2]
+  CRUSH rule 0 x 970 [5,12,10,1,3,14,9,6]
+  CRUSH rule 0 x 971 [1,9,4,12,7,2,10,15]
+  CRUSH rule 0 x 972 [12,3,14,5,1,9,7,11]
+  CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+  CRUSH rule 0 x 974 [7,11,1,2,15,4,12,9]
+  CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+  CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+  CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+  CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+  CRUSH rule 0 x 979 [5,1,13,6,15,10,3,9]
+  CRUSH rule 0 x 980 [15,11,5,6,1,3,13,9]
+  CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+  CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+  CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+  CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+  CRUSH rule 0 x 985 [11,2,15,1,4,13,6,9]
+  CRUSH rule 0 x 986 [6,13,9,1,15,10,5,2]
+  CRUSH rule 0 x 987 [13,14,5,10,6,1,3,9]
+  CRUSH rule 0 x 988 [12,9,10,14,3,1,4,7]
+  CRUSH rule 0 x 989 [7,4,3,15,9,13,10,1]
+  CRUSH rule 0 x 990 [1,10,9,13,3,4,6,15]
+  CRUSH rule 0 x 991 [7,11,1,14,2,5,9,12]
+  CRUSH rule 0 x 992 [9,10,2,13,7,4,1,15]
+  CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+  CRUSH rule 0 x 994 [3,13,15,4,11,7,1,9]
+  CRUSH rule 0 x 995 [15,6,12,2,5,11]
+  CRUSH rule 0 x 996 [15,10,5,3,13,1,9,7]
+  CRUSH rule 0 x 997 [15,2,1,12,7,9,4,10]
+  CRUSH rule 0 x 998 [6,1,9,5,12,11,15,2]
+  CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+  CRUSH rule 0 x 1000 [14,2,9,4,12,1,6,11]
+  CRUSH rule 0 x 1001 [11,14,4,2,6,9,1,13]
+  CRUSH rule 0 x 1002 [1,10,14,2,9,5,13,7]
+  CRUSH rule 0 x 1003 [10,7,5,14,2,1,9,12]
+  CRUSH rule 0 x 1004 [15,1,4,6,10,12,9,3]
+  CRUSH rule 0 x 1005 [6,12,2,10,9,15,5,1]
+  CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+  CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+  CRUSH rule 0 x 1008 [7,4,9,11,3,15,1,13]
+  CRUSH rule 0 x 1009 [5,2,11,7,15,9,1,12]
+  CRUSH rule 0 x 1010 [10,2,15,6,9,13,4,1]
+  CRUSH rule 0 x 1011 [6,3,12,1,10,4,9,14]
+  CRUSH rule 0 x 1012 [12,6,9,15,3,1,5,11]
+  CRUSH rule 0 x 1013 [2,14,12,4,9,1,6,10]
+  CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+  CRUSH rule 0 x 1015 [12,6,10,1,4,15,9,2]
+  CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+  CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+  CRUSH rule 0 x 1018 [13,11,14,1,9,3,5,7]
+  CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+  CRUSH rule 0 x 1020 [3,1,13,4,10,9,14,6]
+  CRUSH rule 0 x 1021 [2,11,14,9,4,6,1,13]
+  CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+  CRUSH rule 0 x 1023 [15,2,9,12,1,7,4,11]
+  rule 0 (replicated_ruleset) num_rep 8 result size == 6:\t36/1024 (esc)
+  rule 0 (replicated_ruleset) num_rep 8 result size == 7:\t264/1024 (esc)
+  rule 0 (replicated_ruleset) num_rep 8 result size == 8:\t724/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3,15,12,1,4,9]
+  CRUSH rule 0 x 1 [10,15,1,2,13,4,7,9]
+  CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+  CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+  CRUSH rule 0 x 4 [14,2,10,1,9,4,7,13]
+  CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+  CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+  CRUSH rule 0 x 7 [9,2,6,12,11,4,1,14]
+  CRUSH rule 0 x 8 [10,2,15,1,4,13,6,9]
+  CRUSH rule 0 x 9 [7,1,14,2,11,9,12,4]
+  CRUSH rule 0 x 10 [10,14,4,1,2,7,13,9]
+  CRUSH rule 0 x 11 [13,9,14,7,5,11,2,1]
+  CRUSH rule 0 x 12 [7,1,2,5,13,15,11,9]
+  CRUSH rule 0 x 13 [3,5,12,7,9,1,14,11]
+  CRUSH rule 0 x 14 [13,5,2,7,10,15,1,9]
+  CRUSH rule 0 x 15 [15,1,9,6,13,3,5,11]
+  CRUSH rule 0 x 16 [7,11,14,2,13,1,9,4]
+  CRUSH rule 0 x 17 [10,1,13,2,4,6,14,9]
+  CRUSH rule 0 x 18 [1,7,3,10,5,12,9,14]
+  CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+  CRUSH rule 0 x 20 [14,12,3,10,9,4,7,1]
+  CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+  CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+  CRUSH rule 0 x 23 [10,5,13,9,3,15,1,6]
+  CRUSH rule 0 x 24 [12,11,3,1,9,4,7,15]
+  CRUSH rule 0 x 25 [7,12,15,1,3,10,4,9]
+  CRUSH rule 0 x 26 [1,7,13,2,14,5,9,11]
+  CRUSH rule 0 x 27 [3,6,15,4,13,9,11,1]
+  CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+  CRUSH rule 0 x 29 [5,14,12,11,6,3,1,9]
+  CRUSH rule 0 x 30 [2,5,6,9,1,11,13,14]
+  CRUSH rule 0 x 31 [5,15,10,1,9,13,6,3]
+  CRUSH rule 0 x 32 [9,10,2,1,13,14,6,4]
+  CRUSH rule 0 x 33 [13,4,9,2,7,1,10,14]
+  CRUSH rule 0 x 34 [13,15,2,4,1,10,9,6]
+  CRUSH rule 0 x 35 [4,14,3,13,10,9,1,6]
+  CRUSH rule 0 x 36 [3,12,9,7,5,10,14,1]
+  CRUSH rule 0 x 37 [9,2,6,14,11,1,4,13]
+  CRUSH rule 0 x 38 [3,4,13,10,9,1,14,6]
+  CRUSH rule 0 x 39 [12,7,14,11,1,9,5,2]
+  CRUSH rule 0 x 40 [10,1,9,5,15,2,6,13]
+  CRUSH rule 0 x 41 [4,9,11,1,14,13,6,3]
+  CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+  CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+  CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+  CRUSH rule 0 x 45 [11,12,15,9,1,5,6,3]
+  CRUSH rule 0 x 46 [6,9,2,14,11,13,1,5]
+  CRUSH rule 0 x 47 [3,9,6,4,13,1,11,15]
+  CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+  CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+  CRUSH rule 0 x 50 [14,12,1,4,2,11,6,9]
+  CRUSH rule 0 x 51 [10,6,5,12,15,2,1,9]
+  CRUSH rule 0 x 52 [12,1,9,11,7,3,14,4]
+  CRUSH rule 0 x 53 [3,6,13,9,5,1,11,15]
+  CRUSH rule 0 x 54 [4,13,9,2,14,10,6,1]
+  CRUSH rule 0 x 55 [4,11,2,7,1,13,9,15]
+  CRUSH rule 0 x 56 [5,9,10,1,3,13,14,7]
+  CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+  CRUSH rule 0 x 58 [7,1,11,4,3,14,12,9]
+  CRUSH rule 0 x 59 [2,13,1,10,9,5,14,6]
+  CRUSH rule 0 x 60 [3,6,11,1,4,9,12,15]
+  CRUSH rule 0 x 61 [3,15,13,7,4,1,10,9]
+  CRUSH rule 0 x 62 [15,11,7,12,5,9,2,1]
+  CRUSH rule 0 x 63 [10,14,12,1,7,3]
+  CRUSH rule 0 x 64 [3,9,1,4,7,12,11,15]
+  CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+  CRUSH rule 0 x 66 [15,11,6,9,4,1,3,12]
+  CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+  CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+  CRUSH rule 0 x 69 [2,1,15,10,4,9,13,7]
+  CRUSH rule 0 x 70 [9,6,1,3,13,15,11,5]
+  CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+  CRUSH rule 0 x 72 [9,10,3,5,7,12,15,1]
+  CRUSH rule 0 x 73 [5,3,11,1,7,12,15,9]
+  CRUSH rule 0 x 74 [11,7,9,5,1,15,3,12]
+  CRUSH rule 0 x 75 [9,7,11,14,12,1,2,5]
+  CRUSH rule 0 x 76 [6,1,3,5,14,10,12,9]
+  CRUSH rule 0 x 77 [7,4,2,13,9,1,11,15]
+  CRUSH rule 0 x 78 [9,3,1,5,6,13,14,11]
+  CRUSH rule 0 x 79 [13,2,15,5,7,9,11,1]
+  CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+  CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+  CRUSH rule 0 x 82 [14,13,5,11,6,2,1,9]
+  CRUSH rule 0 x 83 [4,15,3,9,10,13,6,1]
+  CRUSH rule 0 x 84 [10,7,9,15,3,4,1,13]
+  CRUSH rule 0 x 85 [3,15,9,7,4,11,1,12]
+  CRUSH rule 0 x 86 [10,9,14,1,13,4,2,7]
+  CRUSH rule 0 x 87 [15,10,7,12,5,3,9,1]
+  CRUSH rule 0 x 88 [4,13,3,1,9,15,11,7]
+  CRUSH rule 0 x 89 [3,9,7,4,1,14,10,12]
+  CRUSH rule 0 x 90 [4,9,7,12,11,14,2,1]
+  CRUSH rule 0 x 91 [6,11,9,1,2,4,14,13]
+  CRUSH rule 0 x 92 [1,5,10,9,13,15,6,2]
+  CRUSH rule 0 x 93 [9,3,15,13,7,5,1,10]
+  CRUSH rule 0 x 94 [9,2,12,5,6,11,1,14]
+  CRUSH rule 0 x 95 [7,15,4,10,9,13,2,1]
+  CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+  CRUSH rule 0 x 97 [4,11,2,13,1,7,9,14]
+  CRUSH rule 0 x 98 [11,13,9,3,15,1,5,6]
+  CRUSH rule 0 x 99 [12,4,11,7,3,14,9,1]
+  CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+  CRUSH rule 0 x 101 [15,7,1,9,10,5,2,12]
+  CRUSH rule 0 x 102 [3,11,14,6,13,4,9,1]
+  CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+  CRUSH rule 0 x 104 [14,6,3,5,9,1,10,13]
+  CRUSH rule 0 x 105 [14,10,1,9,3,5,6,13]
+  CRUSH rule 0 x 106 [6,5,13,2,14,11,1,9]
+  CRUSH rule 0 x 107 [3,1,10,14,13,5,9,6]
+  CRUSH rule 0 x 108 [5,10,7,2,15,9,12,1]
+  CRUSH rule 0 x 109 [9,1,13,7,15,5,11,3]
+  CRUSH rule 0 x 110 [5,1,11,3,7,14,13,9]
+  CRUSH rule 0 x 111 [10,1,9,7,5,2,13,14]
+  CRUSH rule 0 x 112 [1,10,4,14,2,12,6,9]
+  CRUSH rule 0 x 113 [6,10,13,9,1,5,2,14]
+  CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+  CRUSH rule 0 x 115 [10,13,14,3,9,1,6,5]
+  CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+  CRUSH rule 0 x 117 [5,6,1,12,15,9,11,3]
+  CRUSH rule 0 x 118 [10,4,13,15,9,3,1,6]
+  CRUSH rule 0 x 119 [14,12,11,4,6,9,3,1]
+  CRUSH rule 0 x 120 [11,3,14,13,4,7]
+  CRUSH rule 0 x 121 [9,5,1,11,7,3,15,12]
+  CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+  CRUSH rule 0 x 123 [3,10,5,6,9,1,12,15]
+  CRUSH rule 0 x 124 [12,2,1,5,14,7,11,9]
+  CRUSH rule 0 x 125 [9,12,15,1,6,5,3,10]
+  CRUSH rule 0 x 126 [7,15,10,9,2,12,5,1]
+  CRUSH rule 0 x 127 [4,14,9,13,1,3,7,10]
+  CRUSH rule 0 x 128 [3,12,1,10,4,9,7,14]
+  CRUSH rule 0 x 129 [11,13,14,2,9,4,6,1]
+  CRUSH rule 0 x 130 [3,13,5,14,10,1,9,6]
+  CRUSH rule 0 x 131 [12,1,6,15,4,2,11,9]
+  CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+  CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+  CRUSH rule 0 x 134 [12,5,6,15,3,9,10,1]
+  CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+  CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+  CRUSH rule 0 x 137 [14,3,6,11,1,9,4,13]
+  CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+  CRUSH rule 0 x 139 [11,2,13,9,1,15,7,5]
+  CRUSH rule 0 x 140 [11,4,12,15,2,6,9,1]
+  CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+  CRUSH rule 0 x 142 [3,14,7,9,11,1,4,13]
+  CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+  CRUSH rule 0 x 144 [13,7,11,2,14,4,1,9]
+  CRUSH rule 0 x 145 [12,2,6,10,9,4,14,1]
+  CRUSH rule 0 x 146 [1,5,9,2,6,13,14,10]
+  CRUSH rule 0 x 147 [1,4,9,11,2,7,15,12]
+  CRUSH rule 0 x 148 [12,7,9,2,14,11,1,4]
+  CRUSH rule 0 x 149 [2,5,9,12,11,1,7,14]
+  CRUSH rule 0 x 150 [1,15,2,10,7,9,5,12]
+  CRUSH rule 0 x 151 [2,9,14,7,1,10,5,13]
+  CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+  CRUSH rule 0 x 153 [6,9,4,15,2,1,10,12]
+  CRUSH rule 0 x 154 [3,11,7,1,4,12,15,9]
+  CRUSH rule 0 x 155 [14,12,7,3,5,1,9,11]
+  CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+  CRUSH rule 0 x 157 [15,1,6,4,3,10,9,13]
+  CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+  CRUSH rule 0 x 159 [4,14,3,12,10,6,1,9]
+  CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+  CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+  CRUSH rule 0 x 162 [10,6,1,12,2,4,14,9]
+  CRUSH rule 0 x 163 [15,1,10,2,6,4,13,9]
+  CRUSH rule 0 x 164 [9,14,10,7,12,2,5,1]
+  CRUSH rule 0 x 165 [11,7,2,13,9,15,1,5]
+  CRUSH rule 0 x 166 [1,2,12,14,4,11,7,9]
+  CRUSH rule 0 x 167 [9,7,3,4,11,13,15,1]
+  CRUSH rule 0 x 168 [13,2,4,1,6,15,10,9]
+  CRUSH rule 0 x 169 [1,4,9,14,13,10,2,7]
+  CRUSH rule 0 x 170 [1,15,7,9,12,10,3,5]
+  CRUSH rule 0 x 171 [9,2,10,7,1,5,15,12]
+  CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+  CRUSH rule 0 x 173 [5,10,12,15,6,1,2,9]
+  CRUSH rule 0 x 174 [15,6,4,12,1,11,9,2]
+  CRUSH rule 0 x 175 [5,7,9,3,10,1,14,13]
+  CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+  CRUSH rule 0 x 177 [2,9,10,13,4,1,14,7]
+  CRUSH rule 0 x 178 [12,11,7,14,3,4]
+  CRUSH rule 0 x 179 [2,10,13,9,5,1,7,14]
+  CRUSH rule 0 x 180 [3,11,5,15,7,12]
+  CRUSH rule 0 x 181 [9,12,6,5,1,10,14,2]
+  CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+  CRUSH rule 0 x 183 [5,7,10,13,3,9,14,1]
+  CRUSH rule 0 x 184 [2,5,11,12,7,1,9,15]
+  CRUSH rule 0 x 185 [13,5,7,11,2,14]
+  CRUSH rule 0 x 186 [6,14,13,5,10,1,3,9]
+  CRUSH rule 0 x 187 [1,4,11,13,6,14,9,3]
+  CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+  CRUSH rule 0 x 189 [6,12,4,9,2,1,11,15]
+  CRUSH rule 0 x 190 [9,13,15,10,3,1,5,6]
+  CRUSH rule 0 x 191 [7,11,4,1,15,12,9,2]
+  CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+  CRUSH rule 0 x 193 [3,13,6,10,4,1,9,14]
+  CRUSH rule 0 x 194 [3,13,4,14,6,9,1,11]
+  CRUSH rule 0 x 195 [5,7,10,12,1,3,15,9]
+  CRUSH rule 0 x 196 [4,15,1,10,9,2,13,7]
+  CRUSH rule 0 x 197 [14,10,13,4,6,3,1,9]
+  CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+  CRUSH rule 0 x 199 [2,10,4,15,1,9,6,12]
+  CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+  CRUSH rule 0 x 201 [9,14,1,7,4,3,10,13]
+  CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+  CRUSH rule 0 x 203 [12,5,7,15,1,2,10,9]
+  CRUSH rule 0 x 204 [6,11,3,12,14,1,9,4]
+  CRUSH rule 0 x 205 [15,4,6,10,13,9,2,1]
+  CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+  CRUSH rule 0 x 207 [2,11,7,4,14,1,12,9]
+  CRUSH rule 0 x 208 [13,1,6,14,9,11,3,5]
+  CRUSH rule 0 x 209 [6,15,13,1,11,4,9,2]
+  CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+  CRUSH rule 0 x 211 [2,14,1,13,11,7,9,5]
+  CRUSH rule 0 x 212 [10,1,12,15,5,6,2,9]
+  CRUSH rule 0 x 213 [3,9,6,5,15,13,1,11]
+  CRUSH rule 0 x 214 [7,15,4,1,10,2,13,9]
+  CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+  CRUSH rule 0 x 216 [12,9,6,2,1,11,5,15]
+  CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+  CRUSH rule 0 x 218 [12,10,15,6,1,4,9,2]
+  CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+  CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+  CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+  CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+  CRUSH rule 0 x 223 [9,7,11,1,4,14,13,3]
+  CRUSH rule 0 x 224 [1,7,10,2,12,9,14,5]
+  CRUSH rule 0 x 225 [10,5,2,6,1,13,9,15]
+  CRUSH rule 0 x 226 [4,1,9,3,13,10,15,7]
+  CRUSH rule 0 x 227 [7,2,12,15,5,11]
+  CRUSH rule 0 x 228 [2,15,11,1,6,13,9,4]
+  CRUSH rule 0 x 229 [9,3,7,14,1,12,4,10]
+  CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+  CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+  CRUSH rule 0 x 232 [10,5,13,1,9,2,7,14]
+  CRUSH rule 0 x 233 [6,12,11,4,9,14,1,3]
+  CRUSH rule 0 x 234 [10,1,2,12,5,9,15,7]
+  CRUSH rule 0 x 235 [13,14,7,10,1,9,5,3]
+  CRUSH rule 0 x 236 [2,15,9,12,1,7,4,10]
+  CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+  CRUSH rule 0 x 238 [2,10,4,15,6,12,9,1]
+  CRUSH rule 0 x 239 [4,15,10,7,9,13,3,1]
+  CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+  CRUSH rule 0 x 241 [7,9,15,12,1,5,2,10]
+  CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+  CRUSH rule 0 x 243 [2,11,5,1,15,6,9,13]
+  CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+  CRUSH rule 0 x 245 [12,9,15,3,1,5,10,7]
+  CRUSH rule 0 x 246 [15,3,5,11,7,1,12,9]
+  CRUSH rule 0 x 247 [6,4,9,12,1,2,10,14]
+  CRUSH rule 0 x 248 [5,13,7,11,9,15,3,1]
+  CRUSH rule 0 x 249 [10,14,7,3,9,13,1,4]
+  CRUSH rule 0 x 250 [12,15,1,10,5,6,3,9]
+  CRUSH rule 0 x 251 [13,2,15,5,6,1,9,10]
+  CRUSH rule 0 x 252 [7,5,13,9,3,10,14,1]
+  CRUSH rule 0 x 253 [3,13,15,10,7,4]
+  CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+  CRUSH rule 0 x 255 [1,9,13,2,6,10,4,15]
+  CRUSH rule 0 x 256 [6,9,13,1,3,14,5,10]
+  CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+  CRUSH rule 0 x 258 [12,5,6,10,2,1,14,9]
+  CRUSH rule 0 x 259 [9,10,4,3,14,13,1,7]
+  CRUSH rule 0 x 260 [10,12,6,9,3,15,1,4]
+  CRUSH rule 0 x 261 [13,7,2,1,15,5,11,9]
+  CRUSH rule 0 x 262 [15,3,12,7,4,9,1,11]
+  CRUSH rule 0 x 263 [12,6,10,9,5,15,3,1]
+  CRUSH rule 0 x 264 [13,14,11,3,1,4,7,9]
+  CRUSH rule 0 x 265 [12,10,14,5,7,1,9,3]
+  CRUSH rule 0 x 266 [14,7,11,1,2,9,4,12]
+  CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+  CRUSH rule 0 x 268 [4,1,15,12,6,11,3,9]
+  CRUSH rule 0 x 269 [11,1,15,5,13,9,7,2]
+  CRUSH rule 0 x 270 [7,11,12,3,1,14,9,4]
+  CRUSH rule 0 x 271 [4,7,3,13,15,10,9,1]
+  CRUSH rule 0 x 272 [15,5,13,10,6,2]
+  CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+  CRUSH rule 0 x 274 [10,2,5,6,13,9,15,1]
+  CRUSH rule 0 x 275 [10,3,4,7,14,13]
+  CRUSH rule 0 x 276 [5,12,9,2,11,7,15,1]
+  CRUSH rule 0 x 277 [14,3,13,4,1,9,11,7]
+  CRUSH rule 0 x 278 [5,6,14,3,1,11,13,9]
+  CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+  CRUSH rule 0 x 280 [7,3,14,9,1,11,4,13]
+  CRUSH rule 0 x 281 [5,11,14,7,9,13,2,1]
+  CRUSH rule 0 x 282 [2,1,13,14,9,7,5,10]
+  CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+  CRUSH rule 0 x 284 [5,11,7,15,3,13,1,9]
+  CRUSH rule 0 x 285 [15,5,3,1,6,13,11,9]
+  CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+  CRUSH rule 0 x 287 [12,4,9,1,3,11,15,7]
+  CRUSH rule 0 x 288 [4,12,10,7,1,3,14,9]
+  CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+  CRUSH rule 0 x 290 [12,2,5,6,15,9,1,10]
+  CRUSH rule 0 x 291 [7,11,1,14,5,9,2,12]
+  CRUSH rule 0 x 292 [4,10,6,3,14,9,12,1]
+  CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+  CRUSH rule 0 x 294 [9,12,3,14,6,11,5,1]
+  CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+  CRUSH rule 0 x 296 [3,1,13,7,14,9,10,4]
+  CRUSH rule 0 x 297 [6,13,4,14,10,1,2,9]
+  CRUSH rule 0 x 298 [14,9,13,1,4,2,7,10]
+  CRUSH rule 0 x 299 [14,12,11,6,4,2,1,9]
+  CRUSH rule 0 x 300 [15,7,10,5,1,3,13,9]
+  CRUSH rule 0 x 301 [9,11,7,1,13,14,4,2]
+  CRUSH rule 0 x 302 [9,7,1,13,5,10,3,15]
+  CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+  CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+  CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+  CRUSH rule 0 x 306 [10,12,4,6,9,2,15,1]
+  CRUSH rule 0 x 307 [11,12,15,5,6,2,1,9]
+  CRUSH rule 0 x 308 [12,14,10,9,1,2,5,7]
+  CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+  CRUSH rule 0 x 310 [3,1,5,10,14,9,7,12]
+  CRUSH rule 0 x 311 [3,9,7,1,14,13,10,5]
+  CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+  CRUSH rule 0 x 313 [9,15,3,7,5,13,1,11]
+  CRUSH rule 0 x 314 [2,15,9,5,6,12,1,11]
+  CRUSH rule 0 x 315 [15,2,13,1,11,9,6,4]
+  CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+  CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+  CRUSH rule 0 x 318 [4,1,15,11,9,13,6,2]
+  CRUSH rule 0 x 319 [2,15,4,1,11,9,7,12]
+  CRUSH rule 0 x 320 [5,7,13,9,11,2,1,15]
+  CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+  CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+  CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+  CRUSH rule 0 x 324 [5,6,10,15,2,13]
+  CRUSH rule 0 x 325 [9,10,14,5,1,6,2,13]
+  CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+  CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+  CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+  CRUSH rule 0 x 329 [2,6,15,5,9,10,13,1]
+  CRUSH rule 0 x 330 [3,9,11,13,1,6,5,14]
+  CRUSH rule 0 x 331 [12,14,6,3,1,4,10,9]
+  CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+  CRUSH rule 0 x 333 [6,5,3,12,14,10,9,1]
+  CRUSH rule 0 x 334 [4,9,2,12,7,11,15,1]
+  CRUSH rule 0 x 335 [11,7,1,5,13,2,9,15]
+  CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+  CRUSH rule 0 x 337 [15,11,3,7,12,5]
+  CRUSH rule 0 x 338 [10,5,3,6,15,1,9,13]
+  CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+  CRUSH rule 0 x 340 [11,6,12,4,9,3,14,1]
+  CRUSH rule 0 x 341 [7,5,2,10,14,9,1,12]
+  CRUSH rule 0 x 342 [12,14,1,9,2,11,4,7]
+  CRUSH rule 0 x 343 [12,14,9,6,10,2,4,1]
+  CRUSH rule 0 x 344 [9,11,5,2,14,13,1,7]
+  CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+  CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+  CRUSH rule 0 x 347 [10,2,12,6,9,1,14,5]
+  CRUSH rule 0 x 348 [7,9,10,1,14,13,3,4]
+  CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+  CRUSH rule 0 x 350 [13,9,15,4,10,7,2,1]
+  CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+  CRUSH rule 0 x 352 [1,12,11,9,4,7,3,15]
+  CRUSH rule 0 x 353 [10,14,12,2,9,1,4,6]
+  CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+  CRUSH rule 0 x 355 [13,14,6,10,2,5,1,9]
+  CRUSH rule 0 x 356 [15,13,2,9,6,5,1,11]
+  CRUSH rule 0 x 357 [4,11,1,13,3,14,6,9]
+  CRUSH rule 0 x 358 [12,7,2,9,1,14,10,4]
+  CRUSH rule 0 x 359 [5,15,7,11,3,13]
+  CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+  CRUSH rule 0 x 361 [5,3,13,6,1,14,11,9]
+  CRUSH rule 0 x 362 [2,9,11,13,1,6,5,15]
+  CRUSH rule 0 x 363 [7,12,3,9,15,4,1,10]
+  CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+  CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+  CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+  CRUSH rule 0 x 367 [7,13,3,1,5,11,15,9]
+  CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+  CRUSH rule 0 x 369 [7,5,3,13,14,9,11,1]
+  CRUSH rule 0 x 370 [4,7,14,1,2,9,11,12]
+  CRUSH rule 0 x 371 [1,7,12,3,4,15,10,9]
+  CRUSH rule 0 x 372 [10,4,3,14,6,1,12,9]
+  CRUSH rule 0 x 373 [15,5,2,6,13,1,9,10]
+  CRUSH rule 0 x 374 [3,15,12,5,1,6,10,9]
+  CRUSH rule 0 x 375 [5,2,14,1,6,13,11,9]
+  CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+  CRUSH rule 0 x 377 [1,15,2,4,9,11,12,6]
+  CRUSH rule 0 x 378 [9,12,2,15,1,5,11,6]
+  CRUSH rule 0 x 379 [11,2,15,5,7,9,13,1]
+  CRUSH rule 0 x 380 [6,1,12,11,2,9,5,14]
+  CRUSH rule 0 x 381 [15,13,7,5,10,2,1,9]
+  CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+  CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+  CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+  CRUSH rule 0 x 385 [4,6,15,3,10,9,1,13]
+  CRUSH rule 0 x 386 [14,3,11,13,5,6,9,1]
+  CRUSH rule 0 x 387 [1,11,5,7,9,2,14,12]
+  CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+  CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+  CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+  CRUSH rule 0 x 391 [3,4,9,13,7,10,1,14]
+  CRUSH rule 0 x 392 [11,5,14,7,1,9,2,12]
+  CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+  CRUSH rule 0 x 394 [4,9,3,15,13,6,1,11]
+  CRUSH rule 0 x 395 [10,13,5,15,6,9,3,1]
+  CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+  CRUSH rule 0 x 397 [1,14,9,4,12,10,3,7]
+  CRUSH rule 0 x 398 [9,2,1,5,12,6,11,15]
+  CRUSH rule 0 x 399 [5,9,14,3,1,10,13,7]
+  CRUSH rule 0 x 400 [10,6,2,4,15,12,1,9]
+  CRUSH rule 0 x 401 [6,9,11,12,4,3,15,1]
+  CRUSH rule 0 x 402 [4,7,9,2,13,1,15,11]
+  CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+  CRUSH rule 0 x 404 [14,12,7,9,2,1,5,11]
+  CRUSH rule 0 x 405 [9,15,11,2,4,7,13,1]
+  CRUSH rule 0 x 406 [12,14,9,2,7,10,4,1]
+  CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+  CRUSH rule 0 x 408 [7,1,5,2,10,15,13,9]
+  CRUSH rule 0 x 409 [11,2,4,13,1,15,7,9]
+  CRUSH rule 0 x 410 [6,4,14,2,12,9,10,1]
+  CRUSH rule 0 x 411 [13,11,15,6,4,1,9,2]
+  CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+  CRUSH rule 0 x 413 [13,5,3,11,6,9,1,14]
+  CRUSH rule 0 x 414 [3,11,9,13,4,1,6,15]
+  CRUSH rule 0 x 415 [6,10,14,5,1,13,3,9]
+  CRUSH rule 0 x 416 [13,1,4,7,2,9,14,11]
+  CRUSH rule 0 x 417 [4,12,1,15,2,11,9,6]
+  CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+  CRUSH rule 0 x 419 [5,14,10,9,2,12,6,1]
+  CRUSH rule 0 x 420 [2,4,9,11,6,14,13,1]
+  CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+  CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+  CRUSH rule 0 x 423 [3,15,12,6,5,1,9,10]
+  CRUSH rule 0 x 424 [6,10,12,2,5,1,14,9]
+  CRUSH rule 0 x 425 [11,15,2,13,5,7,9,1]
+  CRUSH rule 0 x 426 [12,4,7,1,9,10,14,2]
+  CRUSH rule 0 x 427 [14,10,3,1,9,7,5,13]
+  CRUSH rule 0 x 428 [12,7,9,4,2,1,14,10]
+  CRUSH rule 0 x 429 [3,4,9,7,11,12,1,14]
+  CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+  CRUSH rule 0 x 431 [9,3,7,1,12,5,14,11]
+  CRUSH rule 0 x 432 [4,1,12,7,15,2,10,9]
+  CRUSH rule 0 x 433 [4,11,12,15,7,3]
+  CRUSH rule 0 x 434 [2,14,9,1,5,11,7,13]
+  CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+  CRUSH rule 0 x 436 [9,15,10,2,4,1,12,7]
+  CRUSH rule 0 x 437 [9,6,3,14,10,12,5,1]
+  CRUSH rule 0 x 438 [7,2,13,4,11,1,9,14]
+  CRUSH rule 0 x 439 [7,14,4,3,12,10]
+  CRUSH rule 0 x 440 [14,11,9,2,7,12,1,5]
+  CRUSH rule 0 x 441 [2,4,11,9,13,6,1,14]
+  CRUSH rule 0 x 442 [10,13,9,7,15,1,4,2]
+  CRUSH rule 0 x 443 [12,15,10,9,2,1,6,4]
+  CRUSH rule 0 x 444 [4,13,7,14,3,1,9,11]
+  CRUSH rule 0 x 445 [4,2,15,7,1,9,11,12]
+  CRUSH rule 0 x 446 [12,10,6,9,4,1,15,3]
+  CRUSH rule 0 x 447 [15,7,13,1,4,9,3,11]
+  CRUSH rule 0 x 448 [5,2,13,7,15,10]
+  CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+  CRUSH rule 0 x 450 [2,4,6,9,15,1,13,10]
+  CRUSH rule 0 x 451 [6,14,11,3,9,1,12,5]
+  CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+  CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+  CRUSH rule 0 x 454 [10,4,2,6,15,12,9,1]
+  CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+  CRUSH rule 0 x 456 [5,7,13,1,11,3,9,15]
+  CRUSH rule 0 x 457 [9,1,5,7,11,13,15,2]
+  CRUSH rule 0 x 458 [9,11,15,4,7,2,12,1]
+  CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+  CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+  CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+  CRUSH rule 0 x 462 [4,7,12,14,11,1,3,9]
+  CRUSH rule 0 x 463 [4,12,14,11,2,7,1,9]
+  CRUSH rule 0 x 464 [4,2,15,10,1,9,13,7]
+  CRUSH rule 0 x 465 [5,10,9,7,13,1,3,14]
+  CRUSH rule 0 x 466 [13,5,2,15,9,11,6,1]
+  CRUSH rule 0 x 467 [13,6,14,3,9,1,11,5]
+  CRUSH rule 0 x 468 [10,7,12,14,4,1,9,2]
+  CRUSH rule 0 x 469 [4,9,6,14,12,11,3,1]
+  CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+  CRUSH rule 0 x 471 [6,1,5,14,13,10,9,3]
+  CRUSH rule 0 x 472 [2,14,7,5,13,1,11,9]
+  CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+  CRUSH rule 0 x 474 [15,10,4,12,6,9,2,1]
+  CRUSH rule 0 x 475 [10,5,12,9,14,3,6,1]
+  CRUSH rule 0 x 476 [3,6,10,12,1,15,9,4]
+  CRUSH rule 0 x 477 [6,13,5,15,11,9,2,1]
+  CRUSH rule 0 x 478 [4,15,1,3,7,12,10,9]
+  CRUSH rule 0 x 479 [13,11,1,6,14,5,9,3]
+  CRUSH rule 0 x 480 [1,13,6,4,9,14,11,3]
+  CRUSH rule 0 x 481 [15,12,7,9,1,3,10,4]
+  CRUSH rule 0 x 482 [2,12,9,1,7,11,14,4]
+  CRUSH rule 0 x 483 [10,1,4,15,9,7,13,2]
+  CRUSH rule 0 x 484 [1,4,10,13,7,14,2,9]
+  CRUSH rule 0 x 485 [9,4,3,1,14,12,7,10]
+  CRUSH rule 0 x 486 [3,10,15,9,7,13,4,1]
+  CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+  CRUSH rule 0 x 488 [14,4,1,9,2,6,10,12]
+  CRUSH rule 0 x 489 [11,4,2,13,15,7]
+  CRUSH rule 0 x 490 [4,9,1,3,13,15,6,11]
+  CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+  CRUSH rule 0 x 492 [5,7,11,3,14,9,1,13]
+  CRUSH rule 0 x 493 [12,1,4,15,3,11,9,6]
+  CRUSH rule 0 x 494 [1,7,13,4,15,9,10,3]
+  CRUSH rule 0 x 495 [3,15,7,1,9,5,12,11]
+  CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+  CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+  CRUSH rule 0 x 498 [10,6,1,5,9,12,3,15]
+  CRUSH rule 0 x 499 [14,3,12,5,1,11,9,7]
+  CRUSH rule 0 x 500 [15,9,6,12,11,2,1,5]
+  CRUSH rule 0 x 501 [10,13,1,9,3,14,5,7]
+  CRUSH rule 0 x 502 [5,1,14,11,7,12,9,2]
+  CRUSH rule 0 x 503 [15,10,7,9,1,12,4,2]
+  CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+  CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+  CRUSH rule 0 x 506 [11,7,9,14,12,1,2,5]
+  CRUSH rule 0 x 507 [4,14,13,3,9,7,1,10]
+  CRUSH rule 0 x 508 [12,1,4,9,2,11,15,7]
+  CRUSH rule 0 x 509 [4,2,6,9,14,1,10,13]
+  CRUSH rule 0 x 510 [5,3,1,12,11,14,9,7]
+  CRUSH rule 0 x 511 [2,12,10,6,14,5]
+  CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+  CRUSH rule 0 x 513 [4,9,11,3,13,7,1,14]
+  CRUSH rule 0 x 514 [11,9,3,4,12,15,6,1]
+  CRUSH rule 0 x 515 [12,14,6,5,3,9,1,10]
+  CRUSH rule 0 x 516 [14,11,1,12,3,7,4,9]
+  CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+  CRUSH rule 0 x 518 [3,5,7,12,15,11,9,1]
+  CRUSH rule 0 x 519 [12,14,2,1,4,6,9,10]
+  CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+  CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+  CRUSH rule 0 x 522 [4,12,11,1,15,3,9,6]
+  CRUSH rule 0 x 523 [3,1,5,9,15,10,13,7]
+  CRUSH rule 0 x 524 [15,9,3,11,13,7,4,1]
+  CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+  CRUSH rule 0 x 526 [10,2,5,13,6,15,1,9]
+  CRUSH rule 0 x 527 [3,13,4,1,9,10,14,7]
+  CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+  CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+  CRUSH rule 0 x 530 [11,9,12,7,5,1,3,15]
+  CRUSH rule 0 x 531 [9,15,4,7,2,13,1,11]
+  CRUSH rule 0 x 532 [5,3,13,7,9,14,1,10]
+  CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+  CRUSH rule 0 x 534 [11,9,3,7,15,4,1,12]
+  CRUSH rule 0 x 535 [11,1,3,5,14,9,12,7]
+  CRUSH rule 0 x 536 [9,1,14,13,4,6,2,11]
+  CRUSH rule 0 x 537 [15,5,13,2,7,11]
+  CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+  CRUSH rule 0 x 539 [10,12,6,14,1,2,9,5]
+  CRUSH rule 0 x 540 [12,15,7,3,9,11,1,4]
+  CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+  CRUSH rule 0 x 542 [3,9,15,5,11,12,7,1]
+  CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+  CRUSH rule 0 x 544 [3,15,9,11,7,4,12,1]
+  CRUSH rule 0 x 545 [14,10,7,12,4,9,1,3]
+  CRUSH rule 0 x 546 [5,15,13,7,1,10,9,2]
+  CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+  CRUSH rule 0 x 548 [11,7,12,15,4,2]
+  CRUSH rule 0 x 549 [14,1,4,9,13,6,3,10]
+  CRUSH rule 0 x 550 [9,15,3,13,1,6,4,11]
+  CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+  CRUSH rule 0 x 552 [2,11,14,1,9,6,5,12]
+  CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+  CRUSH rule 0 x 554 [11,14,6,4,13,9,3,1]
+  CRUSH rule 0 x 555 [6,5,10,9,14,2,13,1]
+  CRUSH rule 0 x 556 [15,6,3,13,11,4,1,9]
+  CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+  CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+  CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+  CRUSH rule 0 x 560 [4,9,12,6,3,10,1,15]
+  CRUSH rule 0 x 561 [12,7,1,2,5,15,11,9]
+  CRUSH rule 0 x 562 [7,13,9,14,2,1,11,4]
+  CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+  CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+  CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+  CRUSH rule 0 x 566 [6,14,4,2,13,11]
+  CRUSH rule 0 x 567 [15,4,11,6,3,12]
+  CRUSH rule 0 x 568 [4,14,1,6,10,13,3,9]
+  CRUSH rule 0 x 569 [11,3,15,13,5,1,9,7]
+  CRUSH rule 0 x 570 [1,10,13,4,7,2,9,14]
+  CRUSH rule 0 x 571 [10,12,14,9,4,2,1,6]
+  CRUSH rule 0 x 572 [12,14,3,10,6,1,4,9]
+  CRUSH rule 0 x 573 [7,15,11,2,12,9,4,1]
+  CRUSH rule 0 x 574 [11,14,13,1,3,7,4,9]
+  CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+  CRUSH rule 0 x 576 [3,15,11,9,1,6,5,13]
+  CRUSH rule 0 x 577 [13,9,6,15,3,11,4,1]
+  CRUSH rule 0 x 578 [4,10,1,2,7,13,14,9]
+  CRUSH rule 0 x 579 [13,1,15,2,10,7,5,9]
+  CRUSH rule 0 x 580 [3,12,4,1,10,15,7,9]
+  CRUSH rule 0 x 581 [7,14,12,10,1,2,9,5]
+  CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+  CRUSH rule 0 x 583 [4,15,1,9,10,12,2,6]
+  CRUSH rule 0 x 584 [10,1,5,13,6,9,2,15]
+  CRUSH rule 0 x 585 [5,3,6,1,11,14,13,9]
+  CRUSH rule 0 x 586 [7,10,14,12,9,3,5,1]
+  CRUSH rule 0 x 587 [11,6,9,4,1,14,13,2]
+  CRUSH rule 0 x 588 [3,12,7,15,4,9,1,10]
+  CRUSH rule 0 x 589 [9,7,12,1,10,3,4,15]
+  CRUSH rule 0 x 590 [12,1,3,9,10,6,4,14]
+  CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+  CRUSH rule 0 x 592 [15,12,9,7,5,2,11,1]
+  CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+  CRUSH rule 0 x 594 [12,14,2,9,7,4,1,11]
+  CRUSH rule 0 x 595 [12,7,10,3,1,14,9,4]
+  CRUSH rule 0 x 596 [2,7,12,11,1,5,15,9]
+  CRUSH rule 0 x 597 [15,1,2,10,7,13,5,9]
+  CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+  CRUSH rule 0 x 599 [13,11,1,5,6,2,15,9]
+  CRUSH rule 0 x 600 [4,12,3,10,9,7,1,14]
+  CRUSH rule 0 x 601 [13,5,15,2,1,7,9,10]
+  CRUSH rule 0 x 602 [3,11,7,1,13,15,5,9]
+  CRUSH rule 0 x 603 [3,1,4,14,10,9,6,12]
+  CRUSH rule 0 x 604 [14,2,6,1,11,13,9,4]
+  CRUSH rule 0 x 605 [2,7,12,5,14,10,1,9]
+  CRUSH rule 0 x 606 [12,15,1,5,7,9,3,11]
+  CRUSH rule 0 x 607 [3,9,10,14,7,1,4,12]
+  CRUSH rule 0 x 608 [13,10,1,7,9,15,5,2]
+  CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+  CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+  CRUSH rule 0 x 611 [13,1,5,3,10,7,15,9]
+  CRUSH rule 0 x 612 [7,1,2,13,9,15,4,11]
+  CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+  CRUSH rule 0 x 614 [9,4,15,3,1,11,6,12]
+  CRUSH rule 0 x 615 [9,4,11,2,1,12,6,15]
+  CRUSH rule 0 x 616 [10,14,1,5,3,6,12,9]
+  CRUSH rule 0 x 617 [15,7,2,11,12,1,9,4]
+  CRUSH rule 0 x 618 [4,2,10,6,14,9,1,12]
+  CRUSH rule 0 x 619 [15,4,3,9,6,1,13,11]
+  CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+  CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+  CRUSH rule 0 x 622 [10,2,13,5,15,9,1,7]
+  CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+  CRUSH rule 0 x 624 [3,9,15,6,10,1,5,12]
+  CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+  CRUSH rule 0 x 626 [10,12,2,1,9,7,5,14]
+  CRUSH rule 0 x 627 [1,12,10,14,3,5,9,7]
+  CRUSH rule 0 x 628 [15,13,11,4,2,1,7,9]
+  CRUSH rule 0 x 629 [5,6,15,12,1,10,3,9]
+  CRUSH rule 0 x 630 [1,4,12,9,3,7,15,11]
+  CRUSH rule 0 x 631 [5,7,1,15,12,11,3,9]
+  CRUSH rule 0 x 632 [12,3,11,9,6,1,15,5]
+  CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+  CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+  CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+  CRUSH rule 0 x 636 [13,6,11,3,15,9,1,4]
+  CRUSH rule 0 x 637 [3,1,10,6,9,12,4,14]
+  CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+  CRUSH rule 0 x 639 [6,9,14,4,3,1,10,13]
+  CRUSH rule 0 x 640 [9,6,1,11,14,2,4,13]
+  CRUSH rule 0 x 641 [10,6,5,14,1,9,12,2]
+  CRUSH rule 0 x 642 [1,15,4,6,2,10,9,12]
+  CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+  CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+  CRUSH rule 0 x 645 [14,2,4,9,10,1,7,13]
+  CRUSH rule 0 x 646 [5,13,14,1,6,9,2,11]
+  CRUSH rule 0 x 647 [10,1,9,13,6,2,14,5]
+  CRUSH rule 0 x 648 [6,5,2,14,11,1,12,9]
+  CRUSH rule 0 x 649 [3,9,13,11,4,14,1,7]
+  CRUSH rule 0 x 650 [10,9,4,15,12,7,1,2]
+  CRUSH rule 0 x 651 [3,9,5,7,14,1,13,11]
+  CRUSH rule 0 x 652 [15,9,4,6,13,1,2,11]
+  CRUSH rule 0 x 653 [11,14,1,3,6,9,12,5]
+  CRUSH rule 0 x 654 [13,6,2,10,15,4,1,9]
+  CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+  CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+  CRUSH rule 0 x 657 [11,15,3,5,7,13,1,9]
+  CRUSH rule 0 x 658 [7,2,10,12,1,4,9,14]
+  CRUSH rule 0 x 659 [2,5,14,6,10,12]
+  CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+  CRUSH rule 0 x 661 [7,15,3,12,11,4,9,1]
+  CRUSH rule 0 x 662 [15,2,12,5,1,10,9,7]
+  CRUSH rule 0 x 663 [14,9,13,10,5,3,1,6]
+  CRUSH rule 0 x 664 [6,10,12,4,9,2,1,15]
+  CRUSH rule 0 x 665 [2,9,12,1,7,10,4,15]
+  CRUSH rule 0 x 666 [12,3,6,1,15,9,10,4]
+  CRUSH rule 0 x 667 [1,9,12,10,2,14,7,4]
+  CRUSH rule 0 x 668 [9,5,1,2,6,11,13,15]
+  CRUSH rule 0 x 669 [9,7,14,5,11,13,1,2]
+  CRUSH rule 0 x 670 [6,10,9,13,1,2,15,4]
+  CRUSH rule 0 x 671 [6,15,5,10,13,3]
+  CRUSH rule 0 x 672 [2,9,13,1,4,14,6,10]
+  CRUSH rule 0 x 673 [7,10,5,9,15,13,2,1]
+  CRUSH rule 0 x 674 [7,12,10,1,14,9,3,4]
+  CRUSH rule 0 x 675 [9,5,1,10,6,14,12,2]
+  CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+  CRUSH rule 0 x 677 [2,12,1,4,10,6,15,9]
+  CRUSH rule 0 x 678 [1,2,4,10,12,14,9,6]
+  CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+  CRUSH rule 0 x 680 [7,11,3,1,15,4,9,12]
+  CRUSH rule 0 x 681 [6,4,3,11,14,13,1,9]
+  CRUSH rule 0 x 682 [6,1,11,15,12,2,5,9]
+  CRUSH rule 0 x 683 [6,13,2,4,9,14,10,1]
+  CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+  CRUSH rule 0 x 685 [5,1,15,7,9,2,10,13]
+  CRUSH rule 0 x 686 [1,9,11,14,6,13,4,3]
+  CRUSH rule 0 x 687 [7,13,3,5,11,9,15,1]
+  CRUSH rule 0 x 688 [11,9,1,14,3,5,7,12]
+  CRUSH rule 0 x 689 [5,2,9,12,1,14,11,7]
+  CRUSH rule 0 x 690 [9,7,10,3,13,15,5,1]
+  CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+  CRUSH rule 0 x 692 [15,5,1,2,9,11,12,7]
+  CRUSH rule 0 x 693 [5,6,12,15,2,10,9,1]
+  CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+  CRUSH rule 0 x 695 [6,13,14,10,9,5,1,3]
+  CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+  CRUSH rule 0 x 697 [13,11,3,6,4,14,9,1]
+  CRUSH rule 0 x 698 [11,13,4,2,6,1,9,15]
+  CRUSH rule 0 x 699 [7,14,12,4,2,11]
+  CRUSH rule 0 x 700 [12,14,11,9,4,6,3,1]
+  CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+  CRUSH rule 0 x 702 [3,12,15,6,5,11,1,9]
+  CRUSH rule 0 x 703 [15,11,13,3,4,7,1,9]
+  CRUSH rule 0 x 704 [6,4,2,15,11,1,13,9]
+  CRUSH rule 0 x 705 [14,6,11,5,1,13,9,3]
+  CRUSH rule 0 x 706 [1,12,3,6,4,10,15,9]
+  CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+  CRUSH rule 0 x 708 [3,10,5,1,15,9,7,13]
+  CRUSH rule 0 x 709 [11,12,3,7,5,14,1,9]
+  CRUSH rule 0 x 710 [14,2,11,9,5,7,12,1]
+  CRUSH rule 0 x 711 [14,3,9,10,12,5,6,1]
+  CRUSH rule 0 x 712 [12,3,11,15,9,1,6,4]
+  CRUSH rule 0 x 713 [11,9,3,15,13,6,4,1]
+  CRUSH rule 0 x 714 [12,1,9,7,2,15,10,5]
+  CRUSH rule 0 x 715 [6,1,14,4,11,12,3,9]
+  CRUSH rule 0 x 716 [11,13,9,14,5,2,1,7]
+  CRUSH rule 0 x 717 [12,4,10,9,15,1,2,7]
+  CRUSH rule 0 x 718 [7,15,5,2,11,13]
+  CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+  CRUSH rule 0 x 720 [4,13,10,2,7,9,1,14]
+  CRUSH rule 0 x 721 [11,3,14,9,1,12,4,6]
+  CRUSH rule 0 x 722 [2,4,6,1,9,15,13,10]
+  CRUSH rule 0 x 723 [2,1,12,15,11,7,5,9]
+  CRUSH rule 0 x 724 [7,1,9,10,5,15,13,2]
+  CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+  CRUSH rule 0 x 726 [7,14,4,3,11,13,9,1]
+  CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+  CRUSH rule 0 x 728 [13,11,4,6,15,2]
+  CRUSH rule 0 x 729 [15,11,4,6,2,9,1,13]
+  CRUSH rule 0 x 730 [3,7,1,13,11,15,9,5]
+  CRUSH rule 0 x 731 [9,1,6,5,2,11,13,15]
+  CRUSH rule 0 x 732 [1,2,10,13,9,4,7,15]
+  CRUSH rule 0 x 733 [11,3,5,6,1,9,12,15]
+  CRUSH rule 0 x 734 [14,3,11,7,12,9,4,1]
+  CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+  CRUSH rule 0 x 736 [3,9,1,11,7,5,13,14]
+  CRUSH rule 0 x 737 [1,4,2,12,9,10,6,15]
+  CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+  CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+  CRUSH rule 0 x 740 [7,9,10,13,1,15,2,5]
+  CRUSH rule 0 x 741 [12,11,7,15,2,5]
+  CRUSH rule 0 x 742 [9,7,4,11,12,1,14,3]
+  CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+  CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+  CRUSH rule 0 x 745 [3,6,1,4,11,12,14,9]
+  CRUSH rule 0 x 746 [3,7,9,10,14,5,1,13]
+  CRUSH rule 0 x 747 [15,11,5,2,13,9,1,7]
+  CRUSH rule 0 x 748 [6,10,13,2,14,5,9,1]
+  CRUSH rule 0 x 749 [14,9,10,7,5,1,2,12]
+  CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+  CRUSH rule 0 x 751 [15,1,6,9,5,11,12,3]
+  CRUSH rule 0 x 752 [13,1,7,3,11,15,9,4]
+  CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+  CRUSH rule 0 x 754 [14,12,11,4,2,1,9,6]
+  CRUSH rule 0 x 755 [13,6,1,10,4,2,14,9]
+  CRUSH rule 0 x 756 [3,4,14,6,1,10,13,9]
+  CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+  CRUSH rule 0 x 758 [6,3,4,10,15,13,9,1]
+  CRUSH rule 0 x 759 [5,7,3,14,11,1,9,13]
+  CRUSH rule 0 x 760 [1,15,10,12,4,3,9,7]
+  CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+  CRUSH rule 0 x 762 [1,4,10,9,3,7,14,12]
+  CRUSH rule 0 x 763 [4,13,1,14,7,10,2,9]
+  CRUSH rule 0 x 764 [1,14,6,13,9,5,2,10]
+  CRUSH rule 0 x 765 [9,15,2,13,4,1,11,7]
+  CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+  CRUSH rule 0 x 767 [6,11,4,3,12,14]
+  CRUSH rule 0 x 768 [2,12,15,7,1,11,9,4]
+  CRUSH rule 0 x 769 [15,1,9,2,11,12,7,4]
+  CRUSH rule 0 x 770 [15,13,4,6,3,10,1,9]
+  CRUSH rule 0 x 771 [9,2,12,11,6,14,5,1]
+  CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+  CRUSH rule 0 x 773 [3,7,4,15,1,12,11,9]
+  CRUSH rule 0 x 774 [12,6,3,15,5,9,10,1]
+  CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+  CRUSH rule 0 x 776 [10,15,3,9,6,13,1,5]
+  CRUSH rule 0 x 777 [11,13,4,7,1,14,9,2]
+  CRUSH rule 0 x 778 [13,1,9,11,15,6,3,5]
+  CRUSH rule 0 x 779 [5,11,1,14,2,9,13,6]
+  CRUSH rule 0 x 780 [13,9,3,6,4,1,14,10]
+  CRUSH rule 0 x 781 [5,7,14,3,1,12,11,9]
+  CRUSH rule 0 x 782 [2,15,9,7,11,13,4,1]
+  CRUSH rule 0 x 783 [12,7,5,14,9,1,2,10]
+  CRUSH rule 0 x 784 [14,1,10,13,3,4,7,9]
+  CRUSH rule 0 x 785 [6,12,1,2,4,9,15,10]
+  CRUSH rule 0 x 786 [10,5,2,15,1,7,12,9]
+  CRUSH rule 0 x 787 [1,12,10,2,9,4,14,6]
+  CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+  CRUSH rule 0 x 789 [9,2,14,7,4,12,1,10]
+  CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+  CRUSH rule 0 x 791 [9,4,7,13,14,11,1,3]
+  CRUSH rule 0 x 792 [6,4,15,10,12,3]
+  CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+  CRUSH rule 0 x 794 [5,12,2,14,9,10,1,6]
+  CRUSH rule 0 x 795 [6,14,12,4,10,1,2,9]
+  CRUSH rule 0 x 796 [11,2,12,6,15,4]
+  CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+  CRUSH rule 0 x 798 [5,11,6,13,1,3,15,9]
+  CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+  CRUSH rule 0 x 800 [6,3,4,11,15,13]
+  CRUSH rule 0 x 801 [2,5,6,13,9,1,10,15]
+  CRUSH rule 0 x 802 [1,4,12,7,3,9,10,14]
+  CRUSH rule 0 x 803 [7,2,4,1,11,13,9,14]
+  CRUSH rule 0 x 804 [5,14,9,7,3,1,12,10]
+  CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+  CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+  CRUSH rule 0 x 807 [14,2,7,4,9,12,1,10]
+  CRUSH rule 0 x 808 [2,15,12,7,9,1,5,10]
+  CRUSH rule 0 x 809 [1,11,7,12,4,2,15,9]
+  CRUSH rule 0 x 810 [2,5,9,12,15,1,7,11]
+  CRUSH rule 0 x 811 [15,6,3,10,1,5,9,12]
+  CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+  CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+  CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+  CRUSH rule 0 x 815 [15,12,9,4,10,6,1,2]
+  CRUSH rule 0 x 816 [14,10,13,7,3,9,4,1]
+  CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+  CRUSH rule 0 x 818 [15,2,11,4,1,12,6,9]
+  CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+  CRUSH rule 0 x 820 [3,6,9,12,11,15,4,1]
+  CRUSH rule 0 x 821 [15,10,9,13,3,4,7,1]
+  CRUSH rule 0 x 822 [10,13,2,9,7,4,14,1]
+  CRUSH rule 0 x 823 [2,6,12,10,15,4,1,9]
+  CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+  CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+  CRUSH rule 0 x 826 [5,2,11,15,1,12,9,7]
+  CRUSH rule 0 x 827 [13,5,1,3,7,9,11,14]
+  CRUSH rule 0 x 828 [12,6,10,5,1,9,2,15]
+  CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+  CRUSH rule 0 x 830 [15,13,2,9,7,11,1,5]
+  CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+  CRUSH rule 0 x 832 [14,11,13,2,9,4,6,1]
+  CRUSH rule 0 x 833 [9,13,3,11,7,5,15,1]
+  CRUSH rule 0 x 834 [9,7,5,1,11,2,13,14]
+  CRUSH rule 0 x 835 [14,3,13,6,4,9,1,10]
+  CRUSH rule 0 x 836 [3,9,10,13,1,5,14,7]
+  CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+  CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+  CRUSH rule 0 x 839 [3,4,6,10,15,1,13,9]
+  CRUSH rule 0 x 840 [10,15,12,4,7,1,2,9]
+  CRUSH rule 0 x 841 [3,5,7,12,11,15,1,9]
+  CRUSH rule 0 x 842 [9,13,2,6,5,14,10,1]
+  CRUSH rule 0 x 843 [14,7,4,9,3,12,1,10]
+  CRUSH rule 0 x 844 [7,1,4,15,9,2,11,12]
+  CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+  CRUSH rule 0 x 846 [3,7,15,13,1,9,10,4]
+  CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+  CRUSH rule 0 x 848 [11,13,1,14,5,9,2,7]
+  CRUSH rule 0 x 849 [3,15,11,9,6,1,13,5]
+  CRUSH rule 0 x 850 [1,3,10,6,14,4,9,12]
+  CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+  CRUSH rule 0 x 852 [9,12,4,7,15,2,11,1]
+  CRUSH rule 0 x 853 [13,14,6,11,2,4,9,1]
+  CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+  CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+  CRUSH rule 0 x 856 [5,10,7,3,15,9,12,1]
+  CRUSH rule 0 x 857 [4,3,13,11,9,1,7,14]
+  CRUSH rule 0 x 858 [5,15,6,3,9,12,1,10]
+  CRUSH rule 0 x 859 [5,15,6,2,1,11,12,9]
+  CRUSH rule 0 x 860 [11,14,1,12,6,9,2,4]
+  CRUSH rule 0 x 861 [13,7,4,10,1,14,3,9]
+  CRUSH rule 0 x 862 [5,10,9,7,3,12,1,15]
+  CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+  CRUSH rule 0 x 864 [6,13,4,2,10,15,1,9]
+  CRUSH rule 0 x 865 [4,1,14,11,6,9,3,13]
+  CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+  CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+  CRUSH rule 0 x 868 [14,11,7,2,1,4,9,12]
+  CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+  CRUSH rule 0 x 870 [14,9,11,4,3,12,6,1]
+  CRUSH rule 0 x 871 [6,2,1,4,15,13,11,9]
+  CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+  CRUSH rule 0 x 873 [2,5,12,10,1,9,15,7]
+  CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+  CRUSH rule 0 x 875 [10,6,14,1,12,5,9,3]
+  CRUSH rule 0 x 876 [14,7,13,3,9,1,11,4]
+  CRUSH rule 0 x 877 [15,11,13,9,5,1,6,3]
+  CRUSH rule 0 x 878 [7,14,3,13,9,1,11,4]
+  CRUSH rule 0 x 879 [12,2,7,4,10,15]
+  CRUSH rule 0 x 880 [2,12,10,7,1,4,9,14]
+  CRUSH rule 0 x 881 [6,3,1,11,4,15,9,13]
+  CRUSH rule 0 x 882 [11,13,7,1,2,15,4,9]
+  CRUSH rule 0 x 883 [13,1,3,10,6,5,9,15]
+  CRUSH rule 0 x 884 [6,15,4,9,3,11,12,1]
+  CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+  CRUSH rule 0 x 886 [13,11,4,2,1,14,9,6]
+  CRUSH rule 0 x 887 [14,4,12,11,2,6,9,1]
+  CRUSH rule 0 x 888 [10,12,7,15,9,2,1,5]
+  CRUSH rule 0 x 889 [15,13,4,1,6,2,10,9]
+  CRUSH rule 0 x 890 [10,12,14,2,9,5,6,1]
+  CRUSH rule 0 x 891 [9,5,11,6,3,15,12,1]
+  CRUSH rule 0 x 892 [12,15,2,4,7,9,11,1]
+  CRUSH rule 0 x 893 [1,3,5,9,6,10,14,12]
+  CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+  CRUSH rule 0 x 895 [2,1,11,5,7,15,13,9]
+  CRUSH rule 0 x 896 [9,1,14,10,4,12,2,7]
+  CRUSH rule 0 x 897 [7,5,14,3,1,9,11,12]
+  CRUSH rule 0 x 898 [10,6,12,9,15,5,2,1]
+  CRUSH rule 0 x 899 [1,11,5,3,13,14,9,6]
+  CRUSH rule 0 x 900 [2,9,10,7,13,14,5,1]
+  CRUSH rule 0 x 901 [9,12,11,3,14,4,1,6]
+  CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+  CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+  CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+  CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+  CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+  CRUSH rule 0 x 907 [7,12,3,9,10,5,14,1]
+  CRUSH rule 0 x 908 [2,15,9,6,10,13,5,1]
+  CRUSH rule 0 x 909 [10,14,1,13,2,9,7,4]
+  CRUSH rule 0 x 910 [12,7,4,15,10,3,1,9]
+  CRUSH rule 0 x 911 [11,15,2,4,9,13,6,1]
+  CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+  CRUSH rule 0 x 913 [4,6,10,1,12,3,9,14]
+  CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+  CRUSH rule 0 x 915 [12,14,1,9,4,3,11,6]
+  CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+  CRUSH rule 0 x 917 [1,15,6,5,10,3,13,9]
+  CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+  CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+  CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+  CRUSH rule 0 x 921 [1,11,6,13,4,2,9,14]
+  CRUSH rule 0 x 922 [6,4,14,13,3,1,10,9]
+  CRUSH rule 0 x 923 [12,2,5,14,10,1,9,6]
+  CRUSH rule 0 x 924 [6,2,14,13,9,1,11,5]
+  CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+  CRUSH rule 0 x 926 [3,13,10,1,14,9,6,5]
+  CRUSH rule 0 x 927 [6,5,1,11,14,2,13,9]
+  CRUSH rule 0 x 928 [13,1,3,9,6,11,15,5]
+  CRUSH rule 0 x 929 [10,7,1,5,2,12,9,14]
+  CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+  CRUSH rule 0 x 931 [6,15,11,9,5,3,1,13]
+  CRUSH rule 0 x 932 [13,2,5,11,9,1,6,15]
+  CRUSH rule 0 x 933 [12,7,14,10,4,1,2,9]
+  CRUSH rule 0 x 934 [12,2,5,7,9,1,15,11]
+  CRUSH rule 0 x 935 [6,11,1,14,5,13,3,9]
+  CRUSH rule 0 x 936 [9,12,7,5,1,2,14,11]
+  CRUSH rule 0 x 937 [14,2,11,1,13,4,9,6]
+  CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+  CRUSH rule 0 x 939 [6,4,14,9,12,1,11,2]
+  CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+  CRUSH rule 0 x 941 [3,12,4,7,14,10]
+  CRUSH rule 0 x 942 [15,12,10,4,1,9,3,7]
+  CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+  CRUSH rule 0 x 944 [2,9,4,7,1,14,12,11]
+  CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+  CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+  CRUSH rule 0 x 947 [11,3,14,1,12,5,6,9]
+  CRUSH rule 0 x 948 [7,13,11,5,14,2,1,9]
+  CRUSH rule 0 x 949 [9,1,12,5,15,10,2,6]
+  CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+  CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+  CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+  CRUSH rule 0 x 953 [1,3,6,10,12,14,4,9]
+  CRUSH rule 0 x 954 [10,2,14,9,4,6,12,1]
+  CRUSH rule 0 x 955 [7,14,3,1,10,4,9,12]
+  CRUSH rule 0 x 956 [1,6,11,5,14,3,9,13]
+  CRUSH rule 0 x 957 [14,11,1,12,6,9,4,3]
+  CRUSH rule 0 x 958 [15,4,3,11,1,6,12,9]
+  CRUSH rule 0 x 959 [2,1,12,15,10,9,4,6]
+  CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+  CRUSH rule 0 x 961 [3,13,11,9,6,1,4,15]
+  CRUSH rule 0 x 962 [5,11,3,14,1,6,13,9]
+  CRUSH rule 0 x 963 [13,10,15,4,6,9,1,3]
+  CRUSH rule 0 x 964 [7,11,4,9,2,12,1,15]
+  CRUSH rule 0 x 965 [12,2,9,7,4,15,11,1]
+  CRUSH rule 0 x 966 [12,14,9,4,1,2,11,7]
+  CRUSH rule 0 x 967 [7,5,3,10,12,14]
+  CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+  CRUSH rule 0 x 969 [11,4,7,1,9,14,13,2]
+  CRUSH rule 0 x 970 [5,12,10,1,3,14,9,6]
+  CRUSH rule 0 x 971 [1,9,4,12,7,2,10,15]
+  CRUSH rule 0 x 972 [12,3,14,5,1,9,7,11]
+  CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+  CRUSH rule 0 x 974 [7,11,1,2,15,4,12,9]
+  CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+  CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+  CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+  CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+  CRUSH rule 0 x 979 [5,1,13,6,15,10,3,9]
+  CRUSH rule 0 x 980 [15,11,5,6,1,3,13,9]
+  CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+  CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+  CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+  CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+  CRUSH rule 0 x 985 [11,2,15,1,4,13,6,9]
+  CRUSH rule 0 x 986 [6,13,9,1,15,10,5,2]
+  CRUSH rule 0 x 987 [13,14,5,10,6,1,3,9]
+  CRUSH rule 0 x 988 [12,9,10,14,3,1,4,7]
+  CRUSH rule 0 x 989 [7,4,3,15,9,13,10,1]
+  CRUSH rule 0 x 990 [1,10,9,13,3,4,6,15]
+  CRUSH rule 0 x 991 [7,11,1,14,2,5,9,12]
+  CRUSH rule 0 x 992 [9,10,2,13,7,4,1,15]
+  CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+  CRUSH rule 0 x 994 [3,13,15,4,11,7,1,9]
+  CRUSH rule 0 x 995 [15,6,12,2,5,11]
+  CRUSH rule 0 x 996 [15,10,5,3,13,1,9,7]
+  CRUSH rule 0 x 997 [15,2,1,12,7,9,4,10]
+  CRUSH rule 0 x 998 [6,1,9,5,12,11,15,2]
+  CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+  CRUSH rule 0 x 1000 [14,2,9,4,12,1,6,11]
+  CRUSH rule 0 x 1001 [11,14,4,2,6,9,1,13]
+  CRUSH rule 0 x 1002 [1,10,14,2,9,5,13,7]
+  CRUSH rule 0 x 1003 [10,7,5,14,2,1,9,12]
+  CRUSH rule 0 x 1004 [15,1,4,6,10,12,9,3]
+  CRUSH rule 0 x 1005 [6,12,2,10,9,15,5,1]
+  CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+  CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+  CRUSH rule 0 x 1008 [7,4,9,11,3,15,1,13]
+  CRUSH rule 0 x 1009 [5,2,11,7,15,9,1,12]
+  CRUSH rule 0 x 1010 [10,2,15,6,9,13,4,1]
+  CRUSH rule 0 x 1011 [6,3,12,1,10,4,9,14]
+  CRUSH rule 0 x 1012 [12,6,9,15,3,1,5,11]
+  CRUSH rule 0 x 1013 [2,14,12,4,9,1,6,10]
+  CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+  CRUSH rule 0 x 1015 [12,6,10,1,4,15,9,2]
+  CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+  CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+  CRUSH rule 0 x 1018 [13,11,14,1,9,3,5,7]
+  CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+  CRUSH rule 0 x 1020 [3,1,13,4,10,9,14,6]
+  CRUSH rule 0 x 1021 [2,11,14,9,4,6,1,13]
+  CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+  CRUSH rule 0 x 1023 [15,2,9,12,1,7,4,11]
+  rule 0 (replicated_ruleset) num_rep 9 result size == 6:\t36/1024 (esc)
+  rule 0 (replicated_ruleset) num_rep 9 result size == 7:\t263/1024 (esc)
+  rule 0 (replicated_ruleset) num_rep 9 result size == 8:\t725/1024 (esc)
+  CRUSH rule 0 x 0 [7,10,3,15,12,1,4,9]
+  CRUSH rule 0 x 1 [10,15,1,2,13,4,7,9]
+  CRUSH rule 0 x 2 [1,12,2,6,5,10,15]
+  CRUSH rule 0 x 3 [15,4,10,2,9,6,13]
+  CRUSH rule 0 x 4 [14,2,10,1,9,4,7,13]
+  CRUSH rule 0 x 5 [7,4,11,2,13,15,9]
+  CRUSH rule 0 x 6 [12,6,10,9,3,4,14]
+  CRUSH rule 0 x 7 [9,2,6,12,11,4,1,14]
+  CRUSH rule 0 x 8 [10,2,15,1,4,13,6,9]
+  CRUSH rule 0 x 9 [7,1,14,2,11,9,12,4]
+  CRUSH rule 0 x 10 [10,14,4,1,2,7,13,9]
+  CRUSH rule 0 x 11 [13,9,14,7,5,11,2,1]
+  CRUSH rule 0 x 12 [7,1,2,5,13,15,11,9]
+  CRUSH rule 0 x 13 [3,5,12,7,9,1,14,11]
+  CRUSH rule 0 x 14 [13,5,2,7,10,15,1,9]
+  CRUSH rule 0 x 15 [15,1,9,6,13,3,5,11]
+  CRUSH rule 0 x 16 [7,11,14,2,13,1,9,4]
+  CRUSH rule 0 x 17 [10,1,13,2,4,6,14,9]
+  CRUSH rule 0 x 18 [1,7,3,10,5,12,9,14]
+  CRUSH rule 0 x 19 [7,12,2,4,15,10,1]
+  CRUSH rule 0 x 20 [14,12,3,10,9,4,7,1]
+  CRUSH rule 0 x 21 [3,12,1,10,4,15,6]
+  CRUSH rule 0 x 22 [6,3,13,11,4,1,15]
+  CRUSH rule 0 x 23 [10,5,13,9,3,15,1,6]
+  CRUSH rule 0 x 24 [12,11,3,1,9,4,7,15]
+  CRUSH rule 0 x 25 [7,12,15,1,3,10,4,9]
+  CRUSH rule 0 x 26 [1,7,13,2,14,5,9,11]
+  CRUSH rule 0 x 27 [3,6,15,4,13,9,11,1]
+  CRUSH rule 0 x 28 [14,4,3,9,6,11,13]
+  CRUSH rule 0 x 29 [5,14,12,11,6,3,1,9]
+  CRUSH rule 0 x 30 [2,5,6,9,1,11,13,14]
+  CRUSH rule 0 x 31 [5,15,10,1,9,13,6,3]
+  CRUSH rule 0 x 32 [9,10,2,1,13,14,6,4]
+  CRUSH rule 0 x 33 [13,4,9,2,7,1,10,14]
+  CRUSH rule 0 x 34 [13,15,2,4,1,10,9,6]
+  CRUSH rule 0 x 35 [4,14,3,13,10,9,1,6]
+  CRUSH rule 0 x 36 [3,12,9,7,5,10,14,1]
+  CRUSH rule 0 x 37 [9,2,6,14,11,1,4,13]
+  CRUSH rule 0 x 38 [3,4,13,10,9,1,14,6]
+  CRUSH rule 0 x 39 [12,7,14,11,1,9,5,2]
+  CRUSH rule 0 x 40 [10,1,9,5,15,2,6,13]
+  CRUSH rule 0 x 41 [4,9,11,1,14,13,6,3]
+  CRUSH rule 0 x 42 [3,6,14,10,12,5,1]
+  CRUSH rule 0 x 43 [10,5,15,7,2,9,12]
+  CRUSH rule 0 x 44 [11,4,13,3,7,14,9]
+  CRUSH rule 0 x 45 [11,12,15,9,1,5,6,3]
+  CRUSH rule 0 x 46 [6,9,2,14,11,13,1,5]
+  CRUSH rule 0 x 47 [3,9,6,4,13,1,11,15]
+  CRUSH rule 0 x 48 [4,6,2,1,10,14,13]
+  CRUSH rule 0 x 49 [9,15,10,7,4,3,13]
+  CRUSH rule 0 x 50 [14,12,1,4,2,11,6,9]
+  CRUSH rule 0 x 51 [10,6,5,12,15,2,1,9]
+  CRUSH rule 0 x 52 [12,1,9,11,7,3,14,4]
+  CRUSH rule 0 x 53 [3,6,13,9,5,1,11,15]
+  CRUSH rule 0 x 54 [4,13,9,2,14,10,6,1]
+  CRUSH rule 0 x 55 [4,11,2,7,1,13,9,15]
+  CRUSH rule 0 x 56 [5,9,10,1,3,13,14,7]
+  CRUSH rule 0 x 57 [6,2,1,15,10,12,5]
+  CRUSH rule 0 x 58 [7,1,11,4,3,14,12,9]
+  CRUSH rule 0 x 59 [2,13,1,10,9,5,14,6]
+  CRUSH rule 0 x 60 [3,6,11,1,4,9,12,15]
+  CRUSH rule 0 x 61 [3,15,13,7,4,1,10,9]
+  CRUSH rule 0 x 62 [15,11,7,12,5,9,2,1]
+  CRUSH rule 0 x 63 [10,14,12,1,7,3]
+  CRUSH rule 0 x 64 [3,9,1,4,7,12,11,15]
+  CRUSH rule 0 x 65 [4,12,11,7,14,3,1]
+  CRUSH rule 0 x 66 [15,11,6,9,4,1,3,12]
+  CRUSH rule 0 x 67 [2,6,4,14,1,11,12]
+  CRUSH rule 0 x 68 [15,7,4,2,9,12,11]
+  CRUSH rule 0 x 69 [2,1,15,10,4,9,13,7]
+  CRUSH rule 0 x 70 [9,6,1,3,13,15,11,5]
+  CRUSH rule 0 x 71 [15,5,1,3,13,10,7]
+  CRUSH rule 0 x 72 [9,10,3,5,7,12,15,1]
+  CRUSH rule 0 x 73 [5,3,11,1,7,12,15,9]
+  CRUSH rule 0 x 74 [11,7,9,5,1,15,3,12]
+  CRUSH rule 0 x 75 [9,7,11,14,12,1,2,5]
+  CRUSH rule 0 x 76 [6,1,3,5,14,10,12,9]
+  CRUSH rule 0 x 77 [7,4,2,13,9,1,11,15]
+  CRUSH rule 0 x 78 [9,3,1,5,6,13,14,11]
+  CRUSH rule 0 x 79 [13,2,15,5,7,9,11,1]
+  CRUSH rule 0 x 80 [15,2,6,4,13,10,1]
+  CRUSH rule 0 x 81 [15,2,1,11,4,6,13]
+  CRUSH rule 0 x 82 [14,13,5,11,6,2,1,9]
+  CRUSH rule 0 x 83 [4,15,3,9,10,13,6,1]
+  CRUSH rule 0 x 84 [10,7,9,15,3,4,1,13]
+  CRUSH rule 0 x 85 [3,15,9,7,4,11,1,12]
+  CRUSH rule 0 x 86 [10,9,14,1,13,4,2,7]
+  CRUSH rule 0 x 87 [15,10,7,12,5,3,9,1]
+  CRUSH rule 0 x 88 [4,13,3,1,9,15,11,7]
+  CRUSH rule 0 x 89 [3,9,7,4,1,14,10,12]
+  CRUSH rule 0 x 90 [4,9,7,12,11,14,2,1]
+  CRUSH rule 0 x 91 [6,11,9,1,2,4,14,13]
+  CRUSH rule 0 x 92 [1,5,10,9,13,15,6,2]
+  CRUSH rule 0 x 93 [9,3,15,13,7,5,1,10]
+  CRUSH rule 0 x 94 [9,2,12,5,6,11,1,14]
+  CRUSH rule 0 x 95 [7,15,4,10,9,13,2,1]
+  CRUSH rule 0 x 96 [2,15,11,7,5,1,12]
+  CRUSH rule 0 x 97 [4,11,2,13,1,7,9,14]
+  CRUSH rule 0 x 98 [11,13,9,3,15,1,5,6]
+  CRUSH rule 0 x 99 [12,4,11,7,3,14,9,1]
+  CRUSH rule 0 x 100 [9,4,10,15,7,3,13]
+  CRUSH rule 0 x 101 [15,7,1,9,10,5,2,12]
+  CRUSH rule 0 x 102 [3,11,14,6,13,4,9,1]
+  CRUSH rule 0 x 103 [13,11,6,14,4,3,1]
+  CRUSH rule 0 x 104 [14,6,3,5,9,1,10,13]
+  CRUSH rule 0 x 105 [14,10,1,9,3,5,6,13]
+  CRUSH rule 0 x 106 [6,5,13,2,14,11,1,9]
+  CRUSH rule 0 x 107 [3,1,10,14,13,5,9,6]
+  CRUSH rule 0 x 108 [5,10,7,2,15,9,12,1]
+  CRUSH rule 0 x 109 [9,1,13,7,15,5,11,3]
+  CRUSH rule 0 x 110 [5,1,11,3,7,14,13,9]
+  CRUSH rule 0 x 111 [10,1,9,7,5,2,13,14]
+  CRUSH rule 0 x 112 [1,10,4,14,2,12,6,9]
+  CRUSH rule 0 x 113 [6,10,13,9,1,5,2,14]
+  CRUSH rule 0 x 114 [5,13,6,2,1,14,11]
+  CRUSH rule 0 x 115 [10,13,14,3,9,1,6,5]
+  CRUSH rule 0 x 116 [1,14,13,2,11,5,7]
+  CRUSH rule 0 x 117 [5,6,1,12,15,9,11,3]
+  CRUSH rule 0 x 118 [10,4,13,15,9,3,1,6]
+  CRUSH rule 0 x 119 [14,12,11,4,6,9,3,1]
+  CRUSH rule 0 x 120 [11,3,14,13,4,7]
+  CRUSH rule 0 x 121 [9,5,1,11,7,3,15,12]
+  CRUSH rule 0 x 122 [4,3,14,1,11,13,7]
+  CRUSH rule 0 x 123 [3,10,5,6,9,1,12,15]
+  CRUSH rule 0 x 124 [12,2,1,5,14,7,11,9]
+  CRUSH rule 0 x 125 [9,12,15,1,6,5,3,10]
+  CRUSH rule 0 x 126 [7,15,10,9,2,12,5,1]
+  CRUSH rule 0 x 127 [4,14,9,13,1,3,7,10]
+  CRUSH rule 0 x 128 [3,12,1,10,4,9,7,14]
+  CRUSH rule 0 x 129 [11,13,14,2,9,4,6,1]
+  CRUSH rule 0 x 130 [3,13,5,14,10,1,9,6]
+  CRUSH rule 0 x 131 [12,1,6,15,4,2,11,9]
+  CRUSH rule 0 x 132 [11,15,13,9,2,5,7]
+  CRUSH rule 0 x 133 [3,6,9,11,15,12,5]
+  CRUSH rule 0 x 134 [12,5,6,15,3,9,10,1]
+  CRUSH rule 0 x 135 [3,14,12,4,6,11,9]
+  CRUSH rule 0 x 136 [15,6,9,4,10,3,12]
+  CRUSH rule 0 x 137 [14,3,6,11,1,9,4,13]
+  CRUSH rule 0 x 138 [13,15,4,10,2,7,1]
+  CRUSH rule 0 x 139 [11,2,13,9,1,15,7,5]
+  CRUSH rule 0 x 140 [11,4,12,15,2,6,9,1]
+  CRUSH rule 0 x 141 [6,12,15,11,3,5,1]
+  CRUSH rule 0 x 142 [3,14,7,9,11,1,4,13]
+  CRUSH rule 0 x 143 [9,6,4,2,14,10,12]
+  CRUSH rule 0 x 144 [13,7,11,2,14,4,1,9]
+  CRUSH rule 0 x 145 [12,2,6,10,9,4,14,1]
+  CRUSH rule 0 x 146 [1,5,9,2,6,13,14,10]
+  CRUSH rule 0 x 147 [1,4,9,11,2,7,15,12]
+  CRUSH rule 0 x 148 [12,7,9,2,14,11,1,4]
+  CRUSH rule 0 x 149 [2,5,9,12,11,1,7,14]
+  CRUSH rule 0 x 150 [1,15,2,10,7,9,5,12]
+  CRUSH rule 0 x 151 [2,9,14,7,1,10,5,13]
+  CRUSH rule 0 x 152 [5,9,2,6,10,13,14]
+  CRUSH rule 0 x 153 [6,9,4,15,2,1,10,12]
+  CRUSH rule 0 x 154 [3,11,7,1,4,12,15,9]
+  CRUSH rule 0 x 155 [14,12,7,3,5,1,9,11]
+  CRUSH rule 0 x 156 [7,13,3,10,15,5,1]
+  CRUSH rule 0 x 157 [15,1,6,4,3,10,9,13]
+  CRUSH rule 0 x 158 [15,1,10,6,12,2,4]
+  CRUSH rule 0 x 159 [4,14,3,12,10,6,1,9]
+  CRUSH rule 0 x 160 [5,7,3,14,11,1,12]
+  CRUSH rule 0 x 161 [1,2,11,4,6,13,14]
+  CRUSH rule 0 x 162 [10,6,1,12,2,4,14,9]
+  CRUSH rule 0 x 163 [15,1,10,2,6,4,13,9]
+  CRUSH rule 0 x 164 [9,14,10,7,12,2,5,1]
+  CRUSH rule 0 x 165 [11,7,2,13,9,15,1,5]
+  CRUSH rule 0 x 166 [1,2,12,14,4,11,7,9]
+  CRUSH rule 0 x 167 [9,7,3,4,11,13,15,1]
+  CRUSH rule 0 x 168 [13,2,4,1,6,15,10,9]
+  CRUSH rule 0 x 169 [1,4,9,14,13,10,2,7]
+  CRUSH rule 0 x 170 [1,15,7,9,12,10,3,5]
+  CRUSH rule 0 x 171 [9,2,10,7,1,5,15,12]
+  CRUSH rule 0 x 172 [14,4,10,12,9,3,6]
+  CRUSH rule 0 x 173 [5,10,12,15,6,1,2,9]
+  CRUSH rule 0 x 174 [15,6,4,12,1,11,9,2]
+  CRUSH rule 0 x 175 [5,7,9,3,10,1,14,13]
+  CRUSH rule 0 x 176 [9,6,3,14,13,10,4]
+  CRUSH rule 0 x 177 [2,9,10,13,4,1,14,7]
+  CRUSH rule 0 x 178 [12,11,7,14,3,4]
+  CRUSH rule 0 x 179 [2,10,13,9,5,1,7,14]
+  CRUSH rule 0 x 180 [3,11,5,15,7,12]
+  CRUSH rule 0 x 181 [9,12,6,5,1,10,14,2]
+  CRUSH rule 0 x 182 [5,13,11,2,1,6,14]
+  CRUSH rule 0 x 183 [5,7,10,13,3,9,14,1]
+  CRUSH rule 0 x 184 [2,5,11,12,7,1,9,15]
+  CRUSH rule 0 x 185 [13,5,7,11,2,14]
+  CRUSH rule 0 x 186 [6,14,13,5,10,1,3,9]
+  CRUSH rule 0 x 187 [1,4,11,13,6,14,9,3]
+  CRUSH rule 0 x 188 [9,13,5,14,10,6,2]
+  CRUSH rule 0 x 189 [6,12,4,9,2,1,11,15]
+  CRUSH rule 0 x 190 [9,13,15,10,3,1,5,6]
+  CRUSH rule 0 x 191 [7,11,4,1,15,12,9,2]
+  CRUSH rule 0 x 192 [2,11,5,15,6,1,13]
+  CRUSH rule 0 x 193 [3,13,6,10,4,1,9,14]
+  CRUSH rule 0 x 194 [3,13,4,14,6,9,1,11]
+  CRUSH rule 0 x 195 [5,7,10,12,1,3,15,9]
+  CRUSH rule 0 x 196 [4,15,1,10,9,2,13,7]
+  CRUSH rule 0 x 197 [14,10,13,4,6,3,1,9]
+  CRUSH rule 0 x 198 [2,5,6,15,9,13,10]
+  CRUSH rule 0 x 199 [2,10,4,15,1,9,6,12]
+  CRUSH rule 0 x 200 [7,14,11,4,1,3,13]
+  CRUSH rule 0 x 201 [9,14,1,7,4,3,10,13]
+  CRUSH rule 0 x 202 [14,11,7,3,5,1,12]
+  CRUSH rule 0 x 203 [12,5,7,15,1,2,10,9]
+  CRUSH rule 0 x 204 [6,11,3,12,14,1,9,4]
+  CRUSH rule 0 x 205 [15,4,6,10,13,9,2,1]
+  CRUSH rule 0 x 206 [13,11,2,15,7,1,5]
+  CRUSH rule 0 x 207 [2,11,7,4,14,1,12,9]
+  CRUSH rule 0 x 208 [13,1,6,14,9,11,3,5]
+  CRUSH rule 0 x 209 [6,15,13,1,11,4,9,2]
+  CRUSH rule 0 x 210 [13,11,2,7,5,14,9]
+  CRUSH rule 0 x 211 [2,14,1,13,11,7,9,5]
+  CRUSH rule 0 x 212 [10,1,12,15,5,6,2,9]
+  CRUSH rule 0 x 213 [3,9,6,5,15,13,1,11]
+  CRUSH rule 0 x 214 [7,15,4,1,10,2,13,9]
+  CRUSH rule 0 x 215 [6,1,4,13,3,11,14]
+  CRUSH rule 0 x 216 [12,9,6,2,1,11,5,15]
+  CRUSH rule 0 x 217 [12,11,1,14,2,4,7]
+  CRUSH rule 0 x 218 [12,10,15,6,1,4,9,2]
+  CRUSH rule 0 x 219 [3,11,14,6,4,1,13]
+  CRUSH rule 0 x 220 [14,4,3,12,10,9,6]
+  CRUSH rule 0 x 221 [15,5,2,6,12,11,9]
+  CRUSH rule 0 x 222 [10,4,3,15,7,12,1]
+  CRUSH rule 0 x 223 [9,7,11,1,4,14,13,3]
+  CRUSH rule 0 x 224 [1,7,10,2,12,9,14,5]
+  CRUSH rule 0 x 225 [10,5,2,6,1,13,9,15]
+  CRUSH rule 0 x 226 [4,1,9,3,13,10,15,7]
+  CRUSH rule 0 x 227 [7,2,12,15,5,11]
+  CRUSH rule 0 x 228 [2,15,11,1,6,13,9,4]
+  CRUSH rule 0 x 229 [9,3,7,14,1,12,4,10]
+  CRUSH rule 0 x 230 [10,5,7,2,15,1,13]
+  CRUSH rule 0 x 231 [2,7,5,13,9,15,10]
+  CRUSH rule 0 x 232 [10,5,13,1,9,2,7,14]
+  CRUSH rule 0 x 233 [6,12,11,4,9,14,1,3]
+  CRUSH rule 0 x 234 [10,1,2,12,5,9,15,7]
+  CRUSH rule 0 x 235 [13,14,7,10,1,9,5,3]
+  CRUSH rule 0 x 236 [2,15,9,12,1,7,4,10]
+  CRUSH rule 0 x 237 [3,12,9,10,4,7,15]
+  CRUSH rule 0 x 238 [2,10,4,15,6,12,9,1]
+  CRUSH rule 0 x 239 [4,15,10,7,9,13,3,1]
+  CRUSH rule 0 x 240 [15,5,13,7,2,9,10]
+  CRUSH rule 0 x 241 [7,9,15,12,1,5,2,10]
+  CRUSH rule 0 x 242 [14,2,6,9,10,12,5]
+  CRUSH rule 0 x 243 [2,11,5,1,15,6,9,13]
+  CRUSH rule 0 x 244 [13,9,15,3,11,7,5]
+  CRUSH rule 0 x 245 [12,9,15,3,1,5,10,7]
+  CRUSH rule 0 x 246 [15,3,5,11,7,1,12,9]
+  CRUSH rule 0 x 247 [6,4,9,12,1,2,10,14]
+  CRUSH rule 0 x 248 [5,13,7,11,9,15,3,1]
+  CRUSH rule 0 x 249 [10,14,7,3,9,13,1,4]
+  CRUSH rule 0 x 250 [12,15,1,10,5,6,3,9]
+  CRUSH rule 0 x 251 [13,2,15,5,6,1,9,10]
+  CRUSH rule 0 x 252 [7,5,13,9,3,10,14,1]
+  CRUSH rule 0 x 253 [3,13,15,10,7,4]
+  CRUSH rule 0 x 254 [2,9,13,14,4,6,10]
+  CRUSH rule 0 x 255 [1,9,13,2,6,10,4,15]
+  CRUSH rule 0 x 256 [6,9,13,1,3,14,5,10]
+  CRUSH rule 0 x 257 [15,12,3,9,6,4,11]
+  CRUSH rule 0 x 258 [12,5,6,10,2,1,14,9]
+  CRUSH rule 0 x 259 [9,10,4,3,14,13,1,7]
+  CRUSH rule 0 x 260 [10,12,6,9,3,15,1,4]
+  CRUSH rule 0 x 261 [13,7,2,1,15,5,11,9]
+  CRUSH rule 0 x 262 [15,3,12,7,4,9,1,11]
+  CRUSH rule 0 x 263 [12,6,10,9,5,15,3,1]
+  CRUSH rule 0 x 264 [13,14,11,3,1,4,7,9]
+  CRUSH rule 0 x 265 [12,10,14,5,7,1,9,3]
+  CRUSH rule 0 x 266 [14,7,11,1,2,9,4,12]
+  CRUSH rule 0 x 267 [12,11,6,5,1,2,15]
+  CRUSH rule 0 x 268 [4,1,15,12,6,11,3,9]
+  CRUSH rule 0 x 269 [11,1,15,5,13,9,7,2]
+  CRUSH rule 0 x 270 [7,11,12,3,1,14,9,4]
+  CRUSH rule 0 x 271 [4,7,3,13,15,10,9,1]
+  CRUSH rule 0 x 272 [15,5,13,10,6,2]
+  CRUSH rule 0 x 273 [2,10,7,12,1,15,5]
+  CRUSH rule 0 x 274 [10,2,5,6,13,9,15,1]
+  CRUSH rule 0 x 275 [10,3,4,7,14,13]
+  CRUSH rule 0 x 276 [5,12,9,2,11,7,15,1]
+  CRUSH rule 0 x 277 [14,3,13,4,1,9,11,7]
+  CRUSH rule 0 x 278 [5,6,14,3,1,11,13,9]
+  CRUSH rule 0 x 279 [6,10,13,3,9,4,15]
+  CRUSH rule 0 x 280 [7,3,14,9,1,11,4,13]
+  CRUSH rule 0 x 281 [5,11,14,7,9,13,2,1]
+  CRUSH rule 0 x 282 [2,1,13,14,9,7,5,10]
+  CRUSH rule 0 x 283 [4,1,12,3,10,7,15]
+  CRUSH rule 0 x 284 [5,11,7,15,3,13,1,9]
+  CRUSH rule 0 x 285 [15,5,3,1,6,13,11,9]
+  CRUSH rule 0 x 286 [10,4,3,6,12,15,1]
+  CRUSH rule 0 x 287 [12,4,9,1,3,11,15,7]
+  CRUSH rule 0 x 288 [4,12,10,7,1,3,14,9]
+  CRUSH rule 0 x 289 [2,5,14,9,13,6,10]
+  CRUSH rule 0 x 290 [12,2,5,6,15,9,1,10]
+  CRUSH rule 0 x 291 [7,11,1,14,5,9,2,12]
+  CRUSH rule 0 x 292 [4,10,6,3,14,9,12,1]
+  CRUSH rule 0 x 293 [6,5,11,1,2,14,12]
+  CRUSH rule 0 x 294 [9,12,3,14,6,11,5,1]
+  CRUSH rule 0 x 295 [6,10,3,14,9,4,13]
+  CRUSH rule 0 x 296 [3,1,13,7,14,9,10,4]
+  CRUSH rule 0 x 297 [6,13,4,14,10,1,2,9]
+  CRUSH rule 0 x 298 [14,9,13,1,4,2,7,10]
+  CRUSH rule 0 x 299 [14,12,11,6,4,2,1,9]
+  CRUSH rule 0 x 300 [15,7,10,5,1,3,13,9]
+  CRUSH rule 0 x 301 [9,11,7,1,13,14,4,2]
+  CRUSH rule 0 x 302 [9,7,1,13,5,10,3,15]
+  CRUSH rule 0 x 303 [4,13,3,7,10,15,1]
+  CRUSH rule 0 x 304 [6,9,2,11,15,13,4]
+  CRUSH rule 0 x 305 [13,7,5,11,2,15,9]
+  CRUSH rule 0 x 306 [10,12,4,6,9,2,15,1]
+  CRUSH rule 0 x 307 [11,12,15,5,6,2,1,9]
+  CRUSH rule 0 x 308 [12,14,10,9,1,2,5,7]
+  CRUSH rule 0 x 309 [9,3,12,5,11,15,7]
+  CRUSH rule 0 x 310 [3,1,5,10,14,9,7,12]
+  CRUSH rule 0 x 311 [3,9,7,1,14,13,10,5]
+  CRUSH rule 0 x 312 [15,13,9,7,5,10,2]
+  CRUSH rule 0 x 313 [9,15,3,7,5,13,1,11]
+  CRUSH rule 0 x 314 [2,15,9,5,6,12,1,11]
+  CRUSH rule 0 x 315 [15,2,13,1,11,9,6,4]
+  CRUSH rule 0 x 316 [4,9,11,2,12,14,6]
+  CRUSH rule 0 x 317 [1,5,3,13,15,7,10]
+  CRUSH rule 0 x 318 [4,1,15,11,9,13,6,2]
+  CRUSH rule 0 x 319 [2,15,4,1,11,9,7,12]
+  CRUSH rule 0 x 320 [5,7,13,9,11,2,1,15]
+  CRUSH rule 0 x 321 [1,6,11,15,5,3,13]
+  CRUSH rule 0 x 322 [13,7,5,3,14,11,1]
+  CRUSH rule 0 x 323 [7,4,10,1,2,13,14]
+  CRUSH rule 0 x 324 [5,6,10,15,2,13]
+  CRUSH rule 0 x 325 [9,10,14,5,1,6,2,13]
+  CRUSH rule 0 x 326 [11,7,13,4,2,15,1]
+  CRUSH rule 0 x 327 [12,5,10,14,3,7,9]
+  CRUSH rule 0 x 328 [5,2,6,14,1,11,12]
+  CRUSH rule 0 x 329 [2,6,15,5,9,10,13,1]
+  CRUSH rule 0 x 330 [3,9,11,13,1,6,5,14]
+  CRUSH rule 0 x 331 [12,14,6,3,1,4,10,9]
+  CRUSH rule 0 x 332 [10,12,6,15,9,2,5]
+  CRUSH rule 0 x 333 [6,5,3,12,14,10,9,1]
+  CRUSH rule 0 x 334 [4,9,2,12,7,11,15,1]
+  CRUSH rule 0 x 335 [11,7,1,5,13,2,9,15]
+  CRUSH rule 0 x 336 [6,14,13,2,5,9,11]
+  CRUSH rule 0 x 337 [15,11,3,7,12,5]
+  CRUSH rule 0 x 338 [10,5,3,6,15,1,9,13]
+  CRUSH rule 0 x 339 [11,14,13,5,3,7,1]
+  CRUSH rule 0 x 340 [11,6,12,4,9,3,14,1]
+  CRUSH rule 0 x 341 [7,5,2,10,14,9,1,12]
+  CRUSH rule 0 x 342 [12,14,1,9,2,11,4,7]
+  CRUSH rule 0 x 343 [12,14,9,6,10,2,4,1]
+  CRUSH rule 0 x 344 [9,11,5,2,14,13,1,7]
+  CRUSH rule 0 x 345 [14,2,11,9,6,12,4]
+  CRUSH rule 0 x 346 [5,3,14,10,7,1,13]
+  CRUSH rule 0 x 347 [10,2,12,6,9,1,14,5]
+  CRUSH rule 0 x 348 [7,9,10,1,14,13,3,4]
+  CRUSH rule 0 x 349 [9,6,10,12,1,5,14]
+  CRUSH rule 0 x 350 [13,9,15,4,10,7,2,1]
+  CRUSH rule 0 x 351 [13,5,15,3,1,6,11]
+  CRUSH rule 0 x 352 [1,12,11,9,4,7,3,15]
+  CRUSH rule 0 x 353 [10,14,12,2,9,1,4,6]
+  CRUSH rule 0 x 354 [6,3,15,10,9,4,13]
+  CRUSH rule 0 x 355 [13,14,6,10,2,5,1,9]
+  CRUSH rule 0 x 356 [15,13,2,9,6,5,1,11]
+  CRUSH rule 0 x 357 [4,11,1,13,3,14,6,9]
+  CRUSH rule 0 x 358 [12,7,2,9,1,14,10,4]
+  CRUSH rule 0 x 359 [5,15,7,11,3,13]
+  CRUSH rule 0 x 360 [13,10,1,2,6,14,5]
+  CRUSH rule 0 x 361 [5,3,13,6,1,14,11,9]
+  CRUSH rule 0 x 362 [2,9,11,13,1,6,5,15]
+  CRUSH rule 0 x 363 [7,12,3,9,15,4,1,10]
+  CRUSH rule 0 x 364 [2,12,6,9,5,10,15]
+  CRUSH rule 0 x 365 [13,5,11,15,6,2,9]
+  CRUSH rule 0 x 366 [12,7,3,14,5,10,9]
+  CRUSH rule 0 x 367 [7,13,3,1,5,11,15,9]
+  CRUSH rule 0 x 368 [7,9,10,15,3,4,13]
+  CRUSH rule 0 x 369 [7,5,3,13,14,9,11,1]
+  CRUSH rule 0 x 370 [4,7,14,1,2,9,11,12]
+  CRUSH rule 0 x 371 [1,7,12,3,4,15,10,9]
+  CRUSH rule 0 x 372 [10,4,3,14,6,1,12,9]
+  CRUSH rule 0 x 373 [15,5,2,6,13,1,9,10]
+  CRUSH rule 0 x 374 [3,15,12,5,1,6,10,9]
+  CRUSH rule 0 x 375 [5,2,14,1,6,13,11,9]
+  CRUSH rule 0 x 376 [5,14,10,13,3,6,1]
+  CRUSH rule 0 x 377 [1,15,2,4,9,11,12,6]
+  CRUSH rule 0 x 378 [9,12,2,15,1,5,11,6]
+  CRUSH rule 0 x 379 [11,2,15,5,7,9,13,1]
+  CRUSH rule 0 x 380 [6,1,12,11,2,9,5,14]
+  CRUSH rule 0 x 381 [15,13,7,5,10,2,1,9]
+  CRUSH rule 0 x 382 [14,3,1,4,13,7,10]
+  CRUSH rule 0 x 383 [3,6,11,4,13,15,1]
+  CRUSH rule 0 x 384 [4,13,6,3,15,11,9]
+  CRUSH rule 0 x 385 [4,6,15,3,10,9,1,13]
+  CRUSH rule 0 x 386 [14,3,11,13,5,6,9,1]
+  CRUSH rule 0 x 387 [1,11,5,7,9,2,14,12]
+  CRUSH rule 0 x 388 [2,6,11,9,15,4,12]
+  CRUSH rule 0 x 389 [12,7,2,4,15,10,1]
+  CRUSH rule 0 x 390 [2,11,13,7,5,9,15]
+  CRUSH rule 0 x 391 [3,4,9,13,7,10,1,14]
+  CRUSH rule 0 x 392 [11,5,14,7,1,9,2,12]
+  CRUSH rule 0 x 393 [2,14,5,9,7,13,11]
+  CRUSH rule 0 x 394 [4,9,3,15,13,6,1,11]
+  CRUSH rule 0 x 395 [10,13,5,15,6,9,3,1]
+  CRUSH rule 0 x 396 [2,12,15,9,4,6,11]
+  CRUSH rule 0 x 397 [1,14,9,4,12,10,3,7]
+  CRUSH rule 0 x 398 [9,2,1,5,12,6,11,15]
+  CRUSH rule 0 x 399 [5,9,14,3,1,10,13,7]
+  CRUSH rule 0 x 400 [10,6,2,4,15,12,1,9]
+  CRUSH rule 0 x 401 [6,9,11,12,4,3,15,1]
+  CRUSH rule 0 x 402 [4,7,9,2,13,1,15,11]
+  CRUSH rule 0 x 403 [7,15,13,3,5,9,10]
+  CRUSH rule 0 x 404 [14,12,7,9,2,1,5,11]
+  CRUSH rule 0 x 405 [9,15,11,2,4,7,13,1]
+  CRUSH rule 0 x 406 [12,14,9,2,7,10,4,1]
+  CRUSH rule 0 x 407 [9,5,12,10,15,6,3]
+  CRUSH rule 0 x 408 [7,1,5,2,10,15,13,9]
+  CRUSH rule 0 x 409 [11,2,4,13,1,15,7,9]
+  CRUSH rule 0 x 410 [6,4,14,2,12,9,10,1]
+  CRUSH rule 0 x 411 [13,11,15,6,4,1,9,2]
+  CRUSH rule 0 x 412 [5,9,6,11,14,2,12]
+  CRUSH rule 0 x 413 [13,5,3,11,6,9,1,14]
+  CRUSH rule 0 x 414 [3,11,9,13,4,1,6,15]
+  CRUSH rule 0 x 415 [6,10,14,5,1,13,3,9]
+  CRUSH rule 0 x 416 [13,1,4,7,2,9,14,11]
+  CRUSH rule 0 x 417 [4,12,1,15,2,11,9,6]
+  CRUSH rule 0 x 418 [14,5,10,2,6,9,13]
+  CRUSH rule 0 x 419 [5,14,10,9,2,12,6,1]
+  CRUSH rule 0 x 420 [2,4,9,11,6,14,13,1]
+  CRUSH rule 0 x 421 [15,4,10,3,9,12,7]
+  CRUSH rule 0 x 422 [4,11,2,7,13,9,15]
+  CRUSH rule 0 x 423 [3,15,12,6,5,1,9,10]
+  CRUSH rule 0 x 424 [6,10,12,2,5,1,14,9]
+  CRUSH rule 0 x 425 [11,15,2,13,5,7,9,1]
+  CRUSH rule 0 x 426 [12,4,7,1,9,10,14,2]
+  CRUSH rule 0 x 427 [14,10,3,1,9,7,5,13]
+  CRUSH rule 0 x 428 [12,7,9,4,2,1,14,10]
+  CRUSH rule 0 x 429 [3,4,9,7,11,12,1,14]
+  CRUSH rule 0 x 430 [3,5,10,13,1,15,6]
+  CRUSH rule 0 x 431 [9,3,7,1,12,5,14,11]
+  CRUSH rule 0 x 432 [4,1,12,7,15,2,10,9]
+  CRUSH rule 0 x 433 [4,11,12,15,7,3]
+  CRUSH rule 0 x 434 [2,14,9,1,5,11,7,13]
+  CRUSH rule 0 x 435 [13,11,5,6,9,2,15]
+  CRUSH rule 0 x 436 [9,15,10,2,4,1,12,7]
+  CRUSH rule 0 x 437 [9,6,3,14,10,12,5,1]
+  CRUSH rule 0 x 438 [7,2,13,4,11,1,9,14]
+  CRUSH rule 0 x 439 [7,14,4,3,12,10]
+  CRUSH rule 0 x 440 [14,11,9,2,7,12,1,5]
+  CRUSH rule 0 x 441 [2,4,11,9,13,6,1,14]
+  CRUSH rule 0 x 442 [10,13,9,7,15,1,4,2]
+  CRUSH rule 0 x 443 [12,15,10,9,2,1,6,4]
+  CRUSH rule 0 x 444 [4,13,7,14,3,1,9,11]
+  CRUSH rule 0 x 445 [4,2,15,7,1,9,11,12]
+  CRUSH rule 0 x 446 [12,10,6,9,4,1,15,3]
+  CRUSH rule 0 x 447 [15,7,13,1,4,9,3,11]
+  CRUSH rule 0 x 448 [5,2,13,7,15,10]
+  CRUSH rule 0 x 449 [14,5,3,12,10,9,6]
+  CRUSH rule 0 x 450 [2,4,6,9,15,1,13,10]
+  CRUSH rule 0 x 451 [6,14,11,3,9,1,12,5]
+  CRUSH rule 0 x 452 [14,9,10,4,2,13,7]
+  CRUSH rule 0 x 453 [5,15,13,2,6,9,11]
+  CRUSH rule 0 x 454 [10,4,2,6,15,12,9,1]
+  CRUSH rule 0 x 455 [6,13,2,4,10,1,15]
+  CRUSH rule 0 x 456 [5,7,13,1,11,3,9,15]
+  CRUSH rule 0 x 457 [9,1,5,7,11,13,15,2]
+  CRUSH rule 0 x 458 [9,11,15,4,7,2,12,1]
+  CRUSH rule 0 x 459 [13,15,11,1,5,2,6]
+  CRUSH rule 0 x 460 [5,12,10,15,7,3,9]
+  CRUSH rule 0 x 461 [4,3,9,13,15,6,10]
+  CRUSH rule 0 x 462 [4,7,12,14,11,1,3,9]
+  CRUSH rule 0 x 463 [4,12,14,11,2,7,1,9]
+  CRUSH rule 0 x 464 [4,2,15,10,1,9,13,7]
+  CRUSH rule 0 x 465 [5,10,9,7,13,1,3,14]
+  CRUSH rule 0 x 466 [13,5,2,15,9,11,6,1]
+  CRUSH rule 0 x 467 [13,6,14,3,9,1,11,5]
+  CRUSH rule 0 x 468 [10,7,12,14,4,1,9,2]
+  CRUSH rule 0 x 469 [4,9,6,14,12,11,3,1]
+  CRUSH rule 0 x 470 [3,9,12,15,5,6,10]
+  CRUSH rule 0 x 471 [6,1,5,14,13,10,9,3]
+  CRUSH rule 0 x 472 [2,14,7,5,13,1,11,9]
+  CRUSH rule 0 x 473 [15,10,6,9,4,12,2]
+  CRUSH rule 0 x 474 [15,10,4,12,6,9,2,1]
+  CRUSH rule 0 x 475 [10,5,12,9,14,3,6,1]
+  CRUSH rule 0 x 476 [3,6,10,12,1,15,9,4]
+  CRUSH rule 0 x 477 [6,13,5,15,11,9,2,1]
+  CRUSH rule 0 x 478 [4,15,1,3,7,12,10,9]
+  CRUSH rule 0 x 479 [13,11,1,6,14,5,9,3]
+  CRUSH rule 0 x 480 [1,13,6,4,9,14,11,3]
+  CRUSH rule 0 x 481 [15,12,7,9,1,3,10,4]
+  CRUSH rule 0 x 482 [2,12,9,1,7,11,14,4]
+  CRUSH rule 0 x 483 [10,1,4,15,9,7,13,2]
+  CRUSH rule 0 x 484 [1,4,10,13,7,14,2,9]
+  CRUSH rule 0 x 485 [9,4,3,1,14,12,7,10]
+  CRUSH rule 0 x 486 [3,10,15,9,7,13,4,1]
+  CRUSH rule 0 x 487 [12,11,4,14,7,2,1]
+  CRUSH rule 0 x 488 [14,4,1,9,2,6,10,12]
+  CRUSH rule 0 x 489 [11,4,2,13,15,7]
+  CRUSH rule 0 x 490 [4,9,1,3,13,15,6,11]
+  CRUSH rule 0 x 491 [1,12,5,2,14,11,6]
+  CRUSH rule 0 x 492 [5,7,11,3,14,9,1,13]
+  CRUSH rule 0 x 493 [12,1,4,15,3,11,9,6]
+  CRUSH rule 0 x 494 [1,7,13,4,15,9,10,3]
+  CRUSH rule 0 x 495 [3,15,7,1,9,5,12,11]
+  CRUSH rule 0 x 496 [5,3,7,13,9,14,10]
+  CRUSH rule 0 x 497 [13,10,3,6,5,14,1]
+  CRUSH rule 0 x 498 [10,6,1,5,9,12,3,15]
+  CRUSH rule 0 x 499 [14,3,12,5,1,11,9,7]
+  CRUSH rule 0 x 500 [15,9,6,12,11,2,1,5]
+  CRUSH rule 0 x 501 [10,13,1,9,3,14,5,7]
+  CRUSH rule 0 x 502 [5,1,14,11,7,12,9,2]
+  CRUSH rule 0 x 503 [15,10,7,9,1,12,4,2]
+  CRUSH rule 0 x 504 [13,2,7,1,14,11,5]
+  CRUSH rule 0 x 505 [12,7,5,2,14,10,9]
+  CRUSH rule 0 x 506 [11,7,9,14,12,1,2,5]
+  CRUSH rule 0 x 507 [4,14,13,3,9,7,1,10]
+  CRUSH rule 0 x 508 [12,1,4,9,2,11,15,7]
+  CRUSH rule 0 x 509 [4,2,6,9,14,1,10,13]
+  CRUSH rule 0 x 510 [5,3,1,12,11,14,9,7]
+  CRUSH rule 0 x 511 [2,12,10,6,14,5]
+  CRUSH rule 0 x 512 [15,11,3,5,7,1,13]
+  CRUSH rule 0 x 513 [4,9,11,3,13,7,1,14]
+  CRUSH rule 0 x 514 [11,9,3,4,12,15,6,1]
+  CRUSH rule 0 x 515 [12,14,6,5,3,9,1,10]
+  CRUSH rule 0 x 516 [14,11,1,12,3,7,4,9]
+  CRUSH rule 0 x 517 [11,5,6,13,9,3,14]
+  CRUSH rule 0 x 518 [3,5,7,12,15,11,9,1]
+  CRUSH rule 0 x 519 [12,14,2,1,4,6,9,10]
+  CRUSH rule 0 x 520 [12,4,2,10,6,15,9]
+  CRUSH rule 0 x 521 [11,5,9,6,15,3,13]
+  CRUSH rule 0 x 522 [4,12,11,1,15,3,9,6]
+  CRUSH rule 0 x 523 [3,1,5,9,15,10,13,7]
+  CRUSH rule 0 x 524 [15,9,3,11,13,7,4,1]
+  CRUSH rule 0 x 525 [3,15,11,6,9,12,4]
+  CRUSH rule 0 x 526 [10,2,5,13,6,15,1,9]
+  CRUSH rule 0 x 527 [3,13,4,1,9,10,14,7]
+  CRUSH rule 0 x 528 [12,7,15,10,2,5,9]
+  CRUSH rule 0 x 529 [6,4,10,12,2,9,14]
+  CRUSH rule 0 x 530 [11,9,12,7,5,1,3,15]
+  CRUSH rule 0 x 531 [9,15,4,7,2,13,1,11]
+  CRUSH rule 0 x 532 [5,3,13,7,9,14,1,10]
+  CRUSH rule 0 x 533 [12,15,1,2,7,5,10]
+  CRUSH rule 0 x 534 [11,9,3,7,15,4,1,12]
+  CRUSH rule 0 x 535 [11,1,3,5,14,9,12,7]
+  CRUSH rule 0 x 536 [9,1,14,13,4,6,2,11]
+  CRUSH rule 0 x 537 [15,5,13,2,7,11]
+  CRUSH rule 0 x 538 [13,5,11,2,6,15,9]
+  CRUSH rule 0 x 539 [10,12,6,14,1,2,9,5]
+  CRUSH rule 0 x 540 [12,15,7,3,9,11,1,4]
+  CRUSH rule 0 x 541 [2,1,6,11,14,13,4]
+  CRUSH rule 0 x 542 [3,9,15,5,11,12,7,1]
+  CRUSH rule 0 x 543 [4,10,9,3,6,13,14]
+  CRUSH rule 0 x 544 [3,15,9,11,7,4,12,1]
+  CRUSH rule 0 x 545 [14,10,7,12,4,9,1,3]
+  CRUSH rule 0 x 546 [5,15,13,7,1,10,9,2]
+  CRUSH rule 0 x 547 [5,13,7,9,3,14,10]
+  CRUSH rule 0 x 548 [11,7,12,15,4,2]
+  CRUSH rule 0 x 549 [14,1,4,9,13,6,3,10]
+  CRUSH rule 0 x 550 [9,15,3,13,1,6,4,11]
+  CRUSH rule 0 x 551 [11,2,15,6,13,5,1]
+  CRUSH rule 0 x 552 [2,11,14,1,9,6,5,12]
+  CRUSH rule 0 x 553 [11,9,14,6,4,13,3]
+  CRUSH rule 0 x 554 [11,14,6,4,13,9,3,1]
+  CRUSH rule 0 x 555 [6,5,10,9,14,2,13,1]
+  CRUSH rule 0 x 556 [15,6,3,13,11,4,1,9]
+  CRUSH rule 0 x 557 [12,2,5,14,10,9,6]
+  CRUSH rule 0 x 558 [12,1,6,15,5,10,3]
+  CRUSH rule 0 x 559 [2,13,5,10,14,7,1]
+  CRUSH rule 0 x 560 [4,9,12,6,3,10,1,15]
+  CRUSH rule 0 x 561 [12,7,1,2,5,15,11,9]
+  CRUSH rule 0 x 562 [7,13,9,14,2,1,11,4]
+  CRUSH rule 0 x 563 [15,4,3,10,13,9,7]
+  CRUSH rule 0 x 564 [2,13,7,1,15,10,4]
+  CRUSH rule 0 x 565 [3,12,4,1,14,7,11]
+  CRUSH rule 0 x 566 [6,14,4,2,13,11]
+  CRUSH rule 0 x 567 [15,4,11,6,3,12]
+  CRUSH rule 0 x 568 [4,14,1,6,10,13,3,9]
+  CRUSH rule 0 x 569 [11,3,15,13,5,1,9,7]
+  CRUSH rule 0 x 570 [1,10,13,4,7,2,9,14]
+  CRUSH rule 0 x 571 [10,12,14,9,4,2,1,6]
+  CRUSH rule 0 x 572 [12,14,3,10,6,1,4,9]
+  CRUSH rule 0 x 573 [7,15,11,2,12,9,4,1]
+  CRUSH rule 0 x 574 [11,14,13,1,3,7,4,9]
+  CRUSH rule 0 x 575 [5,13,15,9,6,10,2]
+  CRUSH rule 0 x 576 [3,15,11,9,1,6,5,13]
+  CRUSH rule 0 x 577 [13,9,6,15,3,11,4,1]
+  CRUSH rule 0 x 578 [4,10,1,2,7,13,14,9]
+  CRUSH rule 0 x 579 [13,1,15,2,10,7,5,9]
+  CRUSH rule 0 x 580 [3,12,4,1,10,15,7,9]
+  CRUSH rule 0 x 581 [7,14,12,10,1,2,9,5]
+  CRUSH rule 0 x 582 [10,5,13,14,1,2,7]
+  CRUSH rule 0 x 583 [4,15,1,9,10,12,2,6]
+  CRUSH rule 0 x 584 [10,1,5,13,6,9,2,15]
+  CRUSH rule 0 x 585 [5,3,6,1,11,14,13,9]
+  CRUSH rule 0 x 586 [7,10,14,12,9,3,5,1]
+  CRUSH rule 0 x 587 [11,6,9,4,1,14,13,2]
+  CRUSH rule 0 x 588 [3,12,7,15,4,9,1,10]
+  CRUSH rule 0 x 589 [9,7,12,1,10,3,4,15]
+  CRUSH rule 0 x 590 [12,1,3,9,10,6,4,14]
+  CRUSH rule 0 x 591 [2,6,14,13,9,11,4]
+  CRUSH rule 0 x 592 [15,12,9,7,5,2,11,1]
+  CRUSH rule 0 x 593 [13,14,5,11,9,6,2]
+  CRUSH rule 0 x 594 [12,14,2,9,7,4,1,11]
+  CRUSH rule 0 x 595 [12,7,10,3,1,14,9,4]
+  CRUSH rule 0 x 596 [2,7,12,11,1,5,15,9]
+  CRUSH rule 0 x 597 [15,1,2,10,7,13,5,9]
+  CRUSH rule 0 x 598 [11,5,9,14,12,7,3]
+  CRUSH rule 0 x 599 [13,11,1,5,6,2,15,9]
+  CRUSH rule 0 x 600 [4,12,3,10,9,7,1,14]
+  CRUSH rule 0 x 601 [13,5,15,2,1,7,9,10]
+  CRUSH rule 0 x 602 [3,11,7,1,13,15,5,9]
+  CRUSH rule 0 x 603 [3,1,4,14,10,9,6,12]
+  CRUSH rule 0 x 604 [14,2,6,1,11,13,9,4]
+  CRUSH rule 0 x 605 [2,7,12,5,14,10,1,9]
+  CRUSH rule 0 x 606 [12,15,1,5,7,9,3,11]
+  CRUSH rule 0 x 607 [3,9,10,14,7,1,4,12]
+  CRUSH rule 0 x 608 [13,10,1,7,9,15,5,2]
+  CRUSH rule 0 x 609 [14,3,7,9,11,12,5]
+  CRUSH rule 0 x 610 [7,10,5,1,12,2,15]
+  CRUSH rule 0 x 611 [13,1,5,3,10,7,15,9]
+  CRUSH rule 0 x 612 [7,1,2,13,9,15,4,11]
+  CRUSH rule 0 x 613 [10,7,14,9,5,2,13]
+  CRUSH rule 0 x 614 [9,4,15,3,1,11,6,12]
+  CRUSH rule 0 x 615 [9,4,11,2,1,12,6,15]
+  CRUSH rule 0 x 616 [10,14,1,5,3,6,12,9]
+  CRUSH rule 0 x 617 [15,7,2,11,12,1,9,4]
+  CRUSH rule 0 x 618 [4,2,10,6,14,9,1,12]
+  CRUSH rule 0 x 619 [15,4,3,9,6,1,13,11]
+  CRUSH rule 0 x 620 [3,7,11,14,13,1,5]
+  CRUSH rule 0 x 621 [3,6,4,14,1,11,13]
+  CRUSH rule 0 x 622 [10,2,13,5,15,9,1,7]
+  CRUSH rule 0 x 623 [4,9,14,7,3,13,11]
+  CRUSH rule 0 x 624 [3,9,15,6,10,1,5,12]
+  CRUSH rule 0 x 625 [11,7,3,5,13,15,9]
+  CRUSH rule 0 x 626 [10,12,2,1,9,7,5,14]
+  CRUSH rule 0 x 627 [1,12,10,14,3,5,9,7]
+  CRUSH rule 0 x 628 [15,13,11,4,2,1,7,9]
+  CRUSH rule 0 x 629 [5,6,15,12,1,10,3,9]
+  CRUSH rule 0 x 630 [1,4,12,9,3,7,15,11]
+  CRUSH rule 0 x 631 [5,7,1,15,12,11,3,9]
+  CRUSH rule 0 x 632 [12,3,11,9,6,1,15,5]
+  CRUSH rule 0 x 633 [14,4,3,7,10,12,9]
+  CRUSH rule 0 x 634 [6,9,5,3,13,11,14]
+  CRUSH rule 0 x 635 [6,5,2,15,9,12,11]
+  CRUSH rule 0 x 636 [13,6,11,3,15,9,1,4]
+  CRUSH rule 0 x 637 [3,1,10,6,9,12,4,14]
+  CRUSH rule 0 x 638 [10,15,3,5,13,1,7]
+  CRUSH rule 0 x 639 [6,9,14,4,3,1,10,13]
+  CRUSH rule 0 x 640 [9,6,1,11,14,2,4,13]
+  CRUSH rule 0 x 641 [10,6,5,14,1,9,12,2]
+  CRUSH rule 0 x 642 [1,15,4,6,2,10,9,12]
+  CRUSH rule 0 x 643 [3,7,5,1,10,15,13]
+  CRUSH rule 0 x 644 [15,13,6,9,3,11,5]
+  CRUSH rule 0 x 645 [14,2,4,9,10,1,7,13]
+  CRUSH rule 0 x 646 [5,13,14,1,6,9,2,11]
+  CRUSH rule 0 x 647 [10,1,9,13,6,2,14,5]
+  CRUSH rule 0 x 648 [6,5,2,14,11,1,12,9]
+  CRUSH rule 0 x 649 [3,9,13,11,4,14,1,7]
+  CRUSH rule 0 x 650 [10,9,4,15,12,7,1,2]
+  CRUSH rule 0 x 651 [3,9,5,7,14,1,13,11]
+  CRUSH rule 0 x 652 [15,9,4,6,13,1,2,11]
+  CRUSH rule 0 x 653 [11,14,1,3,6,9,12,5]
+  CRUSH rule 0 x 654 [13,6,2,10,15,4,1,9]
+  CRUSH rule 0 x 655 [6,3,4,15,12,11,1]
+  CRUSH rule 0 x 656 [3,15,1,4,6,12,11]
+  CRUSH rule 0 x 657 [11,15,3,5,7,13,1,9]
+  CRUSH rule 0 x 658 [7,2,10,12,1,4,9,14]
+  CRUSH rule 0 x 659 [2,5,14,6,10,12]
+  CRUSH rule 0 x 660 [13,14,10,6,4,9,3]
+  CRUSH rule 0 x 661 [7,15,3,12,11,4,9,1]
+  CRUSH rule 0 x 662 [15,2,12,5,1,10,9,7]
+  CRUSH rule 0 x 663 [14,9,13,10,5,3,1,6]
+  CRUSH rule 0 x 664 [6,10,12,4,9,2,1,15]
+  CRUSH rule 0 x 665 [2,9,12,1,7,10,4,15]
+  CRUSH rule 0 x 666 [12,3,6,1,15,9,10,4]
+  CRUSH rule 0 x 667 [1,9,12,10,2,14,7,4]
+  CRUSH rule 0 x 668 [9,5,1,2,6,11,13,15]
+  CRUSH rule 0 x 669 [9,7,14,5,11,13,1,2]
+  CRUSH rule 0 x 670 [6,10,9,13,1,2,15,4]
+  CRUSH rule 0 x 671 [6,15,5,10,13,3]
+  CRUSH rule 0 x 672 [2,9,13,1,4,14,6,10]
+  CRUSH rule 0 x 673 [7,10,5,9,15,13,2,1]
+  CRUSH rule 0 x 674 [7,12,10,1,14,9,3,4]
+  CRUSH rule 0 x 675 [9,5,1,10,6,14,12,2]
+  CRUSH rule 0 x 676 [10,12,2,1,4,15,7]
+  CRUSH rule 0 x 677 [2,12,1,4,10,6,15,9]
+  CRUSH rule 0 x 678 [1,2,4,10,12,14,9,6]
+  CRUSH rule 0 x 679 [5,6,12,15,9,11,3]
+  CRUSH rule 0 x 680 [7,11,3,1,15,4,9,12]
+  CRUSH rule 0 x 681 [6,4,3,11,14,13,1,9]
+  CRUSH rule 0 x 682 [6,1,11,15,12,2,5,9]
+  CRUSH rule 0 x 683 [6,13,2,4,9,14,10,1]
+  CRUSH rule 0 x 684 [9,11,3,7,15,4,13]
+  CRUSH rule 0 x 685 [5,1,15,7,9,2,10,13]
+  CRUSH rule 0 x 686 [1,9,11,14,6,13,4,3]
+  CRUSH rule 0 x 687 [7,13,3,5,11,9,15,1]
+  CRUSH rule 0 x 688 [11,9,1,14,3,5,7,12]
+  CRUSH rule 0 x 689 [5,2,9,12,1,14,11,7]
+  CRUSH rule 0 x 690 [9,7,10,3,13,15,5,1]
+  CRUSH rule 0 x 691 [11,15,9,5,7,13,2]
+  CRUSH rule 0 x 692 [15,5,1,2,9,11,12,7]
+  CRUSH rule 0 x 693 [5,6,12,15,2,10,9,1]
+  CRUSH rule 0 x 694 [4,7,1,10,12,3,14]
+  CRUSH rule 0 x 695 [6,13,14,10,9,5,1,3]
+  CRUSH rule 0 x 696 [1,2,4,14,7,11,13]
+  CRUSH rule 0 x 697 [13,11,3,6,4,14,9,1]
+  CRUSH rule 0 x 698 [11,13,4,2,6,1,9,15]
+  CRUSH rule 0 x 699 [7,14,12,4,2,11]
+  CRUSH rule 0 x 700 [12,14,11,9,4,6,3,1]
+  CRUSH rule 0 x 701 [3,13,1,14,4,7,11]
+  CRUSH rule 0 x 702 [3,12,15,6,5,11,1,9]
+  CRUSH rule 0 x 703 [15,11,13,3,4,7,1,9]
+  CRUSH rule 0 x 704 [6,4,2,15,11,1,13,9]
+  CRUSH rule 0 x 705 [14,6,11,5,1,13,9,3]
+  CRUSH rule 0 x 706 [1,12,3,6,4,10,15,9]
+  CRUSH rule 0 x 707 [4,7,14,3,10,9,13]
+  CRUSH rule 0 x 708 [3,10,5,1,15,9,7,13]
+  CRUSH rule 0 x 709 [11,12,3,7,5,14,1,9]
+  CRUSH rule 0 x 710 [14,2,11,9,5,7,12,1]
+  CRUSH rule 0 x 711 [14,3,9,10,12,5,6,1]
+  CRUSH rule 0 x 712 [12,3,11,15,9,1,6,4]
+  CRUSH rule 0 x 713 [11,9,3,15,13,6,4,1]
+  CRUSH rule 0 x 714 [12,1,9,7,2,15,10,5]
+  CRUSH rule 0 x 715 [6,1,14,4,11,12,3,9]
+  CRUSH rule 0 x 716 [11,13,9,14,5,2,1,7]
+  CRUSH rule 0 x 717 [12,4,10,9,15,1,2,7]
+  CRUSH rule 0 x 718 [7,15,5,2,11,13]
+  CRUSH rule 0 x 719 [5,15,13,3,1,7,11]
+  CRUSH rule 0 x 720 [4,13,10,2,7,9,1,14]
+  CRUSH rule 0 x 721 [11,3,14,9,1,12,4,6]
+  CRUSH rule 0 x 722 [2,4,6,1,9,15,13,10]
+  CRUSH rule 0 x 723 [2,1,12,15,11,7,5,9]
+  CRUSH rule 0 x 724 [7,1,9,10,5,15,13,2]
+  CRUSH rule 0 x 725 [11,12,7,15,4,1,2]
+  CRUSH rule 0 x 726 [7,14,4,3,11,13,9,1]
+  CRUSH rule 0 x 727 [2,5,1,11,15,7,12]
+  CRUSH rule 0 x 728 [13,11,4,6,15,2]
+  CRUSH rule 0 x 729 [15,11,4,6,2,9,1,13]
+  CRUSH rule 0 x 730 [3,7,1,13,11,15,9,5]
+  CRUSH rule 0 x 731 [9,1,6,5,2,11,13,15]
+  CRUSH rule 0 x 732 [1,2,10,13,9,4,7,15]
+  CRUSH rule 0 x 733 [11,3,5,6,1,9,12,15]
+  CRUSH rule 0 x 734 [14,3,11,7,12,9,4,1]
+  CRUSH rule 0 x 735 [6,9,2,10,13,14,5]
+  CRUSH rule 0 x 736 [3,9,1,11,7,5,13,14]
+  CRUSH rule 0 x 737 [1,4,2,12,9,10,6,15]
+  CRUSH rule 0 x 738 [11,15,7,4,9,2,12]
+  CRUSH rule 0 x 739 [11,12,6,2,4,1,14]
+  CRUSH rule 0 x 740 [7,9,10,13,1,15,2,5]
+  CRUSH rule 0 x 741 [12,11,7,15,2,5]
+  CRUSH rule 0 x 742 [9,7,4,11,12,1,14,3]
+  CRUSH rule 0 x 743 [5,13,9,15,10,7,3]
+  CRUSH rule 0 x 744 [6,2,13,1,14,11,4]
+  CRUSH rule 0 x 745 [3,6,1,4,11,12,14,9]
+  CRUSH rule 0 x 746 [3,7,9,10,14,5,1,13]
+  CRUSH rule 0 x 747 [15,11,5,2,13,9,1,7]
+  CRUSH rule 0 x 748 [6,10,13,2,14,5,9,1]
+  CRUSH rule 0 x 749 [14,9,10,7,5,1,2,12]
+  CRUSH rule 0 x 750 [1,14,6,5,11,2,13]
+  CRUSH rule 0 x 751 [15,1,6,9,5,11,12,3]
+  CRUSH rule 0 x 752 [13,1,7,3,11,15,9,4]
+  CRUSH rule 0 x 753 [4,11,1,3,15,7,13]
+  CRUSH rule 0 x 754 [14,12,11,4,2,1,9,6]
+  CRUSH rule 0 x 755 [13,6,1,10,4,2,14,9]
+  CRUSH rule 0 x 756 [3,4,14,6,1,10,13,9]
+  CRUSH rule 0 x 757 [10,6,1,4,13,15,2]
+  CRUSH rule 0 x 758 [6,3,4,10,15,13,9,1]
+  CRUSH rule 0 x 759 [5,7,3,14,11,1,9,13]
+  CRUSH rule 0 x 760 [1,15,10,12,4,3,9,7]
+  CRUSH rule 0 x 761 [2,12,1,14,5,7,10]
+  CRUSH rule 0 x 762 [1,4,10,9,3,7,14,12]
+  CRUSH rule 0 x 763 [4,13,1,14,7,10,2,9]
+  CRUSH rule 0 x 764 [1,14,6,13,9,5,2,10]
+  CRUSH rule 0 x 765 [9,15,2,13,4,1,11,7]
+  CRUSH rule 0 x 766 [11,2,7,15,9,12,4]
+  CRUSH rule 0 x 767 [6,11,4,3,12,14]
+  CRUSH rule 0 x 768 [2,12,15,7,1,11,9,4]
+  CRUSH rule 0 x 769 [15,1,9,2,11,12,7,4]
+  CRUSH rule 0 x 770 [15,13,4,6,3,10,1,9]
+  CRUSH rule 0 x 771 [9,2,12,11,6,14,5,1]
+  CRUSH rule 0 x 772 [4,3,13,11,14,1,7]
+  CRUSH rule 0 x 773 [3,7,4,15,1,12,11,9]
+  CRUSH rule 0 x 774 [12,6,3,15,5,9,10,1]
+  CRUSH rule 0 x 775 [5,10,14,2,6,1,13]
+  CRUSH rule 0 x 776 [10,15,3,9,6,13,1,5]
+  CRUSH rule 0 x 777 [11,13,4,7,1,14,9,2]
+  CRUSH rule 0 x 778 [13,1,9,11,15,6,3,5]
+  CRUSH rule 0 x 779 [5,11,1,14,2,9,13,6]
+  CRUSH rule 0 x 780 [13,9,3,6,4,1,14,10]
+  CRUSH rule 0 x 781 [5,7,14,3,1,12,11,9]
+  CRUSH rule 0 x 782 [2,15,9,7,11,13,4,1]
+  CRUSH rule 0 x 783 [12,7,5,14,9,1,2,10]
+  CRUSH rule 0 x 784 [14,1,10,13,3,4,7,9]
+  CRUSH rule 0 x 785 [6,12,1,2,4,9,15,10]
+  CRUSH rule 0 x 786 [10,5,2,15,1,7,12,9]
+  CRUSH rule 0 x 787 [1,12,10,2,9,4,14,6]
+  CRUSH rule 0 x 788 [4,2,9,13,6,15,11]
+  CRUSH rule 0 x 789 [9,2,14,7,4,12,1,10]
+  CRUSH rule 0 x 790 [15,2,7,4,1,10,13]
+  CRUSH rule 0 x 791 [9,4,7,13,14,11,1,3]
+  CRUSH rule 0 x 792 [6,4,15,10,12,3]
+  CRUSH rule 0 x 793 [15,9,6,2,13,11,4]
+  CRUSH rule 0 x 794 [5,12,2,14,9,10,1,6]
+  CRUSH rule 0 x 795 [6,14,12,4,10,1,2,9]
+  CRUSH rule 0 x 796 [11,2,12,6,15,4]
+  CRUSH rule 0 x 797 [14,3,7,1,5,13,11]
+  CRUSH rule 0 x 798 [5,11,6,13,1,3,15,9]
+  CRUSH rule 0 x 799 [2,9,14,4,13,6,11]
+  CRUSH rule 0 x 800 [6,3,4,11,15,13]
+  CRUSH rule 0 x 801 [2,5,6,13,9,1,10,15]
+  CRUSH rule 0 x 802 [1,4,12,7,3,9,10,14]
+  CRUSH rule 0 x 803 [7,2,4,1,11,13,9,14]
+  CRUSH rule 0 x 804 [5,14,9,7,3,1,12,10]
+  CRUSH rule 0 x 805 [13,4,3,1,10,15,7]
+  CRUSH rule 0 x 806 [6,2,13,4,15,1,10]
+  CRUSH rule 0 x 807 [14,2,7,4,9,12,1,10]
+  CRUSH rule 0 x 808 [2,15,12,7,9,1,5,10]
+  CRUSH rule 0 x 809 [1,11,7,12,4,2,15,9]
+  CRUSH rule 0 x 810 [2,5,9,12,15,1,7,11]
+  CRUSH rule 0 x 811 [15,6,3,10,1,5,9,12]
+  CRUSH rule 0 x 812 [7,11,2,14,9,5,12]
+  CRUSH rule 0 x 813 [4,10,13,14,2,6,9]
+  CRUSH rule 0 x 814 [13,4,9,3,10,6,15]
+  CRUSH rule 0 x 815 [15,12,9,4,10,6,1,2]
+  CRUSH rule 0 x 816 [14,10,13,7,3,9,4,1]
+  CRUSH rule 0 x 817 [10,7,2,15,13,9,5]
+  CRUSH rule 0 x 818 [15,2,11,4,1,12,6,9]
+  CRUSH rule 0 x 819 [5,12,10,6,1,14,3]
+  CRUSH rule 0 x 820 [3,6,9,12,11,15,4,1]
+  CRUSH rule 0 x 821 [15,10,9,13,3,4,7,1]
+  CRUSH rule 0 x 822 [10,13,2,9,7,4,14,1]
+  CRUSH rule 0 x 823 [2,6,12,10,15,4,1,9]
+  CRUSH rule 0 x 824 [3,7,9,13,15,5,10]
+  CRUSH rule 0 x 825 [10,5,14,6,12,9,3]
+  CRUSH rule 0 x 826 [5,2,11,15,1,12,9,7]
+  CRUSH rule 0 x 827 [13,5,1,3,7,9,11,14]
+  CRUSH rule 0 x 828 [12,6,10,5,1,9,2,15]
+  CRUSH rule 0 x 829 [13,6,15,10,5,3,9]
+  CRUSH rule 0 x 830 [15,13,2,9,7,11,1,5]
+  CRUSH rule 0 x 831 [1,4,11,12,6,3,15]
+  CRUSH rule 0 x 832 [14,11,13,2,9,4,6,1]
+  CRUSH rule 0 x 833 [9,13,3,11,7,5,15,1]
+  CRUSH rule 0 x 834 [9,7,5,1,11,2,13,14]
+  CRUSH rule 0 x 835 [14,3,13,6,4,9,1,10]
+  CRUSH rule 0 x 836 [3,9,10,13,1,5,14,7]
+  CRUSH rule 0 x 837 [15,12,11,2,7,9,5]
+  CRUSH rule 0 x 838 [12,14,9,2,5,7,11]
+  CRUSH rule 0 x 839 [3,4,6,10,15,1,13,9]
+  CRUSH rule 0 x 840 [10,15,12,4,7,1,2,9]
+  CRUSH rule 0 x 841 [3,5,7,12,11,15,1,9]
+  CRUSH rule 0 x 842 [9,13,2,6,5,14,10,1]
+  CRUSH rule 0 x 843 [14,7,4,9,3,12,1,10]
+  CRUSH rule 0 x 844 [7,1,4,15,9,2,11,12]
+  CRUSH rule 0 x 845 [13,6,1,15,4,2,11]
+  CRUSH rule 0 x 846 [3,7,15,13,1,9,10,4]
+  CRUSH rule 0 x 847 [12,15,11,5,2,7,1]
+  CRUSH rule 0 x 848 [11,13,1,14,5,9,2,7]
+  CRUSH rule 0 x 849 [3,15,11,9,6,1,13,5]
+  CRUSH rule 0 x 850 [1,3,10,6,14,4,9,12]
+  CRUSH rule 0 x 851 [14,4,3,6,11,1,13]
+  CRUSH rule 0 x 852 [9,12,4,7,15,2,11,1]
+  CRUSH rule 0 x 853 [13,14,6,11,2,4,9,1]
+  CRUSH rule 0 x 854 [7,11,12,1,4,15,3]
+  CRUSH rule 0 x 855 [14,4,12,6,3,1,10]
+  CRUSH rule 0 x 856 [5,10,7,3,15,9,12,1]
+  CRUSH rule 0 x 857 [4,3,13,11,9,1,7,14]
+  CRUSH rule 0 x 858 [5,15,6,3,9,12,1,10]
+  CRUSH rule 0 x 859 [5,15,6,2,1,11,12,9]
+  CRUSH rule 0 x 860 [11,14,1,12,6,9,2,4]
+  CRUSH rule 0 x 861 [13,7,4,10,1,14,3,9]
+  CRUSH rule 0 x 862 [5,10,9,7,3,12,1,15]
+  CRUSH rule 0 x 863 [11,6,3,9,4,12,15]
+  CRUSH rule 0 x 864 [6,13,4,2,10,15,1,9]
+  CRUSH rule 0 x 865 [4,1,14,11,6,9,3,13]
+  CRUSH rule 0 x 866 [2,13,4,15,9,6,11]
+  CRUSH rule 0 x 867 [12,2,9,10,4,14,6]
+  CRUSH rule 0 x 868 [14,11,7,2,1,4,9,12]
+  CRUSH rule 0 x 869 [10,13,7,14,3,5,1]
+  CRUSH rule 0 x 870 [14,9,11,4,3,12,6,1]
+  CRUSH rule 0 x 871 [6,2,1,4,15,13,11,9]
+  CRUSH rule 0 x 872 [6,1,15,3,10,12,5]
+  CRUSH rule 0 x 873 [2,5,12,10,1,9,15,7]
+  CRUSH rule 0 x 874 [12,4,7,2,15,10,1]
+  CRUSH rule 0 x 875 [10,6,14,1,12,5,9,3]
+  CRUSH rule 0 x 876 [14,7,13,3,9,1,11,4]
+  CRUSH rule 0 x 877 [15,11,13,9,5,1,6,3]
+  CRUSH rule 0 x 878 [7,14,3,13,9,1,11,4]
+  CRUSH rule 0 x 879 [12,2,7,4,10,15]
+  CRUSH rule 0 x 880 [2,12,10,7,1,4,9,14]
+  CRUSH rule 0 x 881 [6,3,1,11,4,15,9,13]
+  CRUSH rule 0 x 882 [11,13,7,1,2,15,4,9]
+  CRUSH rule 0 x 883 [13,1,3,10,6,5,9,15]
+  CRUSH rule 0 x 884 [6,15,4,9,3,11,12,1]
+  CRUSH rule 0 x 885 [14,7,9,4,2,13,11]
+  CRUSH rule 0 x 886 [13,11,4,2,1,14,9,6]
+  CRUSH rule 0 x 887 [14,4,12,11,2,6,9,1]
+  CRUSH rule 0 x 888 [10,12,7,15,9,2,1,5]
+  CRUSH rule 0 x 889 [15,13,4,1,6,2,10,9]
+  CRUSH rule 0 x 890 [10,12,14,2,9,5,6,1]
+  CRUSH rule 0 x 891 [9,5,11,6,3,15,12,1]
+  CRUSH rule 0 x 892 [12,15,2,4,7,9,11,1]
+  CRUSH rule 0 x 893 [1,3,5,9,6,10,14,12]
+  CRUSH rule 0 x 894 [7,2,11,13,4,1,14]
+  CRUSH rule 0 x 895 [2,1,11,5,7,15,13,9]
+  CRUSH rule 0 x 896 [9,1,14,10,4,12,2,7]
+  CRUSH rule 0 x 897 [7,5,14,3,1,9,11,12]
+  CRUSH rule 0 x 898 [10,6,12,9,15,5,2,1]
+  CRUSH rule 0 x 899 [1,11,5,3,13,14,9,6]
+  CRUSH rule 0 x 900 [2,9,10,7,13,14,5,1]
+  CRUSH rule 0 x 901 [9,12,11,3,14,4,1,6]
+  CRUSH rule 0 x 902 [4,2,6,15,12,10,1]
+  CRUSH rule 0 x 903 [14,10,3,1,12,6,5]
+  CRUSH rule 0 x 904 [15,12,4,9,6,3,11]
+  CRUSH rule 0 x 905 [12,6,11,3,9,4,15]
+  CRUSH rule 0 x 906 [14,11,12,2,4,9,6]
+  CRUSH rule 0 x 907 [7,12,3,9,10,5,14,1]
+  CRUSH rule 0 x 908 [2,15,9,6,10,13,5,1]
+  CRUSH rule 0 x 909 [10,14,1,13,2,9,7,4]
+  CRUSH rule 0 x 910 [12,7,4,15,10,3,1,9]
+  CRUSH rule 0 x 911 [11,15,2,4,9,13,6,1]
+  CRUSH rule 0 x 912 [6,4,14,13,3,1,11]
+  CRUSH rule 0 x 913 [4,6,10,1,12,3,9,14]
+  CRUSH rule 0 x 914 [4,15,2,10,1,13,7]
+  CRUSH rule 0 x 915 [12,14,1,9,4,3,11,6]
+  CRUSH rule 0 x 916 [3,1,11,5,6,13,14]
+  CRUSH rule 0 x 917 [1,15,6,5,10,3,13,9]
+  CRUSH rule 0 x 918 [7,14,11,4,9,2,13]
+  CRUSH rule 0 x 919 [10,7,3,13,15,1,4]
+  CRUSH rule 0 x 920 [4,2,10,15,1,13,6]
+  CRUSH rule 0 x 921 [1,11,6,13,4,2,9,14]
+  CRUSH rule 0 x 922 [6,4,14,13,3,1,10,9]
+  CRUSH rule 0 x 923 [12,2,5,14,10,1,9,6]
+  CRUSH rule 0 x 924 [6,2,14,13,9,1,11,5]
+  CRUSH rule 0 x 925 [12,15,2,10,1,5,7]
+  CRUSH rule 0 x 926 [3,13,10,1,14,9,6,5]
+  CRUSH rule 0 x 927 [6,5,1,11,14,2,13,9]
+  CRUSH rule 0 x 928 [13,1,3,9,6,11,15,5]
+  CRUSH rule 0 x 929 [10,7,1,5,2,12,9,14]
+  CRUSH rule 0 x 930 [7,15,10,5,1,13,2]
+  CRUSH rule 0 x 931 [6,15,11,9,5,3,1,13]
+  CRUSH rule 0 x 932 [13,2,5,11,9,1,6,15]
+  CRUSH rule 0 x 933 [12,7,14,10,4,1,2,9]
+  CRUSH rule 0 x 934 [12,2,5,7,9,1,15,11]
+  CRUSH rule 0 x 935 [6,11,1,14,5,13,3,9]
+  CRUSH rule 0 x 936 [9,12,7,5,1,2,14,11]
+  CRUSH rule 0 x 937 [14,2,11,1,13,4,9,6]
+  CRUSH rule 0 x 938 [14,3,5,11,7,9,13]
+  CRUSH rule 0 x 939 [6,4,14,9,12,1,11,2]
+  CRUSH rule 0 x 940 [13,11,4,2,1,6,15]
+  CRUSH rule 0 x 941 [3,12,4,7,14,10]
+  CRUSH rule 0 x 942 [15,12,10,4,1,9,3,7]
+  CRUSH rule 0 x 943 [10,2,4,9,6,15,12]
+  CRUSH rule 0 x 944 [2,9,4,7,1,14,12,11]
+  CRUSH rule 0 x 945 [10,15,2,9,5,12,7]
+  CRUSH rule 0 x 946 [11,15,7,12,5,9,2]
+  CRUSH rule 0 x 947 [11,3,14,1,12,5,6,9]
+  CRUSH rule 0 x 948 [7,13,11,5,14,2,1,9]
+  CRUSH rule 0 x 949 [9,1,12,5,15,10,2,6]
+  CRUSH rule 0 x 950 [9,15,13,6,4,2,10]
+  CRUSH rule 0 x 951 [2,6,12,9,10,4,14]
+  CRUSH rule 0 x 952 [9,7,15,3,5,13,11]
+  CRUSH rule 0 x 953 [1,3,6,10,12,14,4,9]
+  CRUSH rule 0 x 954 [10,2,14,9,4,6,12,1]
+  CRUSH rule 0 x 955 [7,14,3,1,10,4,9,12]
+  CRUSH rule 0 x 956 [1,6,11,5,14,3,9,13]
+  CRUSH rule 0 x 957 [14,11,1,12,6,9,4,3]
+  CRUSH rule 0 x 958 [15,4,3,11,1,6,12,9]
+  CRUSH rule 0 x 959 [2,1,12,15,10,9,4,6]
+  CRUSH rule 0 x 960 [2,6,11,13,15,4,9]
+  CRUSH rule 0 x 961 [3,13,11,9,6,1,4,15]
+  CRUSH rule 0 x 962 [5,11,3,14,1,6,13,9]
+  CRUSH rule 0 x 963 [13,10,15,4,6,9,1,3]
+  CRUSH rule 0 x 964 [7,11,4,9,2,12,1,15]
+  CRUSH rule 0 x 965 [12,2,9,7,4,15,11,1]
+  CRUSH rule 0 x 966 [12,14,9,4,1,2,11,7]
+  CRUSH rule 0 x 967 [7,5,3,10,12,14]
+  CRUSH rule 0 x 968 [12,15,4,9,11,6,3]
+  CRUSH rule 0 x 969 [11,4,7,1,9,14,13,2]
+  CRUSH rule 0 x 970 [5,12,10,1,3,14,9,6]
+  CRUSH rule 0 x 971 [1,9,4,12,7,2,10,15]
+  CRUSH rule 0 x 972 [12,3,14,5,1,9,7,11]
+  CRUSH rule 0 x 973 [1,10,4,12,2,7,15]
+  CRUSH rule 0 x 974 [7,11,1,2,15,4,12,9]
+  CRUSH rule 0 x 975 [7,9,15,12,2,11,4]
+  CRUSH rule 0 x 976 [7,3,15,5,12,11,1]
+  CRUSH rule 0 x 977 [14,3,6,10,4,1,12]
+  CRUSH rule 0 x 978 [12,5,11,1,15,3,6]
+  CRUSH rule 0 x 979 [5,1,13,6,15,10,3,9]
+  CRUSH rule 0 x 980 [15,11,5,6,1,3,13,9]
+  CRUSH rule 0 x 981 [5,11,15,12,7,1,2]
+  CRUSH rule 0 x 982 [2,6,14,11,12,9,5]
+  CRUSH rule 0 x 983 [3,12,10,9,14,5,6]
+  CRUSH rule 0 x 984 [15,13,1,10,2,5,7]
+  CRUSH rule 0 x 985 [11,2,15,1,4,13,6,9]
+  CRUSH rule 0 x 986 [6,13,9,1,15,10,5,2]
+  CRUSH rule 0 x 987 [13,14,5,10,6,1,3,9]
+  CRUSH rule 0 x 988 [12,9,10,14,3,1,4,7]
+  CRUSH rule 0 x 989 [7,4,3,15,9,13,10,1]
+  CRUSH rule 0 x 990 [1,10,9,13,3,4,6,15]
+  CRUSH rule 0 x 991 [7,11,1,14,2,5,9,12]
+  CRUSH rule 0 x 992 [9,10,2,13,7,4,1,15]
+  CRUSH rule 0 x 993 [6,10,14,12,4,1,2]
+  CRUSH rule 0 x 994 [3,13,15,4,11,7,1,9]
+  CRUSH rule 0 x 995 [15,6,12,2,5,11]
+  CRUSH rule 0 x 996 [15,10,5,3,13,1,9,7]
+  CRUSH rule 0 x 997 [15,2,1,12,7,9,4,10]
+  CRUSH rule 0 x 998 [6,1,9,5,12,11,15,2]
+  CRUSH rule 0 x 999 [9,10,15,5,13,3,7]
+  CRUSH rule 0 x 1000 [14,2,9,4,12,1,6,11]
+  CRUSH rule 0 x 1001 [11,14,4,2,6,9,1,13]
+  CRUSH rule 0 x 1002 [1,10,14,2,9,5,13,7]
+  CRUSH rule 0 x 1003 [10,7,5,14,2,1,9,12]
+  CRUSH rule 0 x 1004 [15,1,4,6,10,12,9,3]
+  CRUSH rule 0 x 1005 [6,12,2,10,9,15,5,1]
+  CRUSH rule 0 x 1006 [10,12,15,1,2,6,5]
+  CRUSH rule 0 x 1007 [1,7,13,14,3,4,10]
+  CRUSH rule 0 x 1008 [7,4,9,11,3,15,1,13]
+  CRUSH rule 0 x 1009 [5,2,11,7,15,9,1,12]
+  CRUSH rule 0 x 1010 [10,2,15,6,9,13,4,1]
+  CRUSH rule 0 x 1011 [6,3,12,1,10,4,9,14]
+  CRUSH rule 0 x 1012 [12,6,9,15,3,1,5,11]
+  CRUSH rule 0 x 1013 [2,14,12,4,9,1,6,10]
+  CRUSH rule 0 x 1014 [1,13,7,2,10,14,5]
+  CRUSH rule 0 x 1015 [12,6,10,1,4,15,9,2]
+  CRUSH rule 0 x 1016 [10,13,14,3,5,6,1]
+  CRUSH rule 0 x 1017 [5,11,14,7,13,9,2]
+  CRUSH rule 0 x 1018 [13,11,14,1,9,3,5,7]
+  CRUSH rule 0 x 1019 [10,13,14,7,5,1,2]
+  CRUSH rule 0 x 1020 [3,1,13,4,10,9,14,6]
+  CRUSH rule 0 x 1021 [2,11,14,9,4,6,1,13]
+  CRUSH rule 0 x 1022 [15,5,7,2,12,10]
+  CRUSH rule 0 x 1023 [15,2,9,12,1,7,4,11]
+  rule 0 (replicated_ruleset) num_rep 10 result size == 6:\t36/1024 (esc)
+  rule 0 (replicated_ruleset) num_rep 10 result size == 7:\t263/1024 (esc)
+  rule 0 (replicated_ruleset) num_rep 10 result size == 8:\t725/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-vary-r-0.t b/ceph/src/test/cli/crushtool/test-map-vary-r-0.t
new file mode 100644
index 00000000..663ef650
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-vary-r-0.t
@@ -0,0 +1,3081 @@
+  $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 0 --weight 0 0 --weight 4 0 --weight 9 0
+  crushtool successfully built or modified map.  Use '-o ' to write it out.
+  rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,12]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,52]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,82]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,40]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,107]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38]
+  CRUSH rule 3 x 38 [72,57]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,2]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,29]
+  CRUSH rule 3 x 54 [28,77]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,105]
+  CRUSH rule 3 x 69 [62]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,88]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,43]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,39]
+  CRUSH rule 3 x 79 [64,65]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,83]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,68]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,26]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,17]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,83]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,100]
+  CRUSH rule 3 x 146 [12,15]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,81]
+  CRUSH rule 3 x 154 [19,56]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,22]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,103]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,100]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,85]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90]
+  CRUSH rule 3 x 191 [49,113]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,66]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,86]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,118]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,32]
+  CRUSH rule 3 x 234 [32,55]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,10]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,87]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,31]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,54]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,75]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,18]
+  CRUSH rule 3 x 291 [35,117]
+  CRUSH rule 3 x 292 [20,74]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,36]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,105]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,119]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,15]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,17]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,116]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,118]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,112]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,32]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,81]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,12]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,113]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,70]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,17]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,44]
+  CRUSH rule 3 x 410 [70,8]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,15]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,41]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,21]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,68]
+  CRUSH rule 3 x 456 [101,60]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,106]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,84]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,61]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99]
+  CRUSH rule 3 x 492 [21,58]
+  CRUSH rule 3 x 493 [94,89]
+  CRUSH rule 3 x 494 [56,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,82]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,6]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,77]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,30]
+  CRUSH rule 3 x 518 [18,37]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,104]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,92]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,19]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,116]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,35]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,96]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,24]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,44]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,50]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,114]
+  CRUSH rule 3 x 638 [43,78]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,99]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,39]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,107]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,102]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,100]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,116]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,26]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36]
+  CRUSH rule 3 x 697 [19,38]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,60]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,113]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,38]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,26]
+  CRUSH rule 3 x 723 [117,75]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,38]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,94]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,56]
+  CRUSH rule 3 x 752 [82,16]
+  CRUSH rule 3 x 753 [116,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,65]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,90]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,53]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,93]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,35]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,94]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,19]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,102]
+  CRUSH rule 3 x 824 [102,95]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,33]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,75]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,39]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,31]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,29]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,66]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,62]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,80]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,25]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,60]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,107]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,10]
+  CRUSH rule 3 x 942 [80,37]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,111]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,16]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,46]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,103]
+  CRUSH rule 3 x 969 [53]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,19]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,100]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,113]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,33]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,22]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,35]
+  CRUSH rule 3 x 1014 [33,48]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,111]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [59,88]
+  rule 3 (delltestrule) num_rep 2 result size == 1:\t27/1024 (esc)
+  rule 3 (delltestrule) num_rep 2 result size == 2:\t997/1024 (esc)
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,12]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,52]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,82]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,40]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,107]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38]
+  CRUSH rule 3 x 38 [72,57]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,2]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,29]
+  CRUSH rule 3 x 54 [28,77]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,105]
+  CRUSH rule 3 x 69 [62]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,88]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,43]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,39]
+  CRUSH rule 3 x 79 [64,65]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,83]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,68]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,26]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,17]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,83]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,100]
+  CRUSH rule 3 x 146 [12,15]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,81]
+  CRUSH rule 3 x 154 [19,56]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,22]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,103]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,100]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,85]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90]
+  CRUSH rule 3 x 191 [49,113]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,66]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,86]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,118]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,32]
+  CRUSH rule 3 x 234 [32,55]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,10]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,87]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,31]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,54]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,75]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,18]
+  CRUSH rule 3 x 291 [35,117]
+  CRUSH rule 3 x 292 [20,74]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,36]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,105]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,119]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,15]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,17]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,116]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,118]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,112]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,32]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,81]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,12]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,113]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,70]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,17]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,44]
+  CRUSH rule 3 x 410 [70,8]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,15]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,41]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,21]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,68]
+  CRUSH rule 3 x 456 [101,60]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,106]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,84]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,61]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99]
+  CRUSH rule 3 x 492 [21,58]
+  CRUSH rule 3 x 493 [94,89]
+  CRUSH rule 3 x 494 [56,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,82]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,6]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,77]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,30]
+  CRUSH rule 3 x 518 [18,37]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,104]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,92]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,19]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,116]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,35]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,96]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,24]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,44]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,50]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,114]
+  CRUSH rule 3 x 638 [43,78]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,99]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,39]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,107]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,102]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,100]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,116]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,26]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36]
+  CRUSH rule 3 x 697 [19,38]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,60]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,113]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,38]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,26]
+  CRUSH rule 3 x 723 [117,75]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,38]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,94]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,56]
+  CRUSH rule 3 x 752 [82,16]
+  CRUSH rule 3 x 753 [116,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,65]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,90]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,53]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,93]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,35]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,94]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,19]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,102]
+  CRUSH rule 3 x 824 [102,95]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,33]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,75]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,39]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,31]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,29]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,66]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,62]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,80]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,25]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,60]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,107]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,10]
+  CRUSH rule 3 x 942 [80,37]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,111]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,16]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,46]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,103]
+  CRUSH rule 3 x 969 [53]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,19]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,100]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,113]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,33]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,22]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,35]
+  CRUSH rule 3 x 1014 [33,48]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,111]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [59,88]
+  rule 3 (delltestrule) num_rep 3 result size == 1:\t27/1024 (esc)
+  rule 3 (delltestrule) num_rep 3 result size == 2:\t997/1024 (esc)
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,12]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,52]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,82]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,40]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,107]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38]
+  CRUSH rule 3 x 38 [72,57]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,2]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,29]
+  CRUSH rule 3 x 54 [28,77]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,105]
+  CRUSH rule 3 x 69 [62]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,88]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,43]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,39]
+  CRUSH rule 3 x 79 [64,65]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,83]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,68]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,26]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,17]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,83]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,100]
+  CRUSH rule 3 x 146 [12,15]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,81]
+  CRUSH rule 3 x 154 [19,56]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,22]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,103]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,100]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,85]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90]
+  CRUSH rule 3 x 191 [49,113]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,66]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,86]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,118]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,32]
+  CRUSH rule 3 x 234 [32,55]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,10]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,87]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,31]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,54]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,75]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,18]
+  CRUSH rule 3 x 291 [35,117]
+  CRUSH rule 3 x 292 [20,74]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,36]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,105]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,119]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,15]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,17]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,116]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,118]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,112]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,32]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,81]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,12]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,113]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,70]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,17]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,44]
+  CRUSH rule 3 x 410 [70,8]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,15]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,41]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,21]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,68]
+  CRUSH rule 3 x 456 [101,60]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,106]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,84]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,61]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99]
+  CRUSH rule 3 x 492 [21,58]
+  CRUSH rule 3 x 493 [94,89]
+  CRUSH rule 3 x 494 [56,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,82]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,6]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,77]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,30]
+  CRUSH rule 3 x 518 [18,37]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,104]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,92]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,19]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,116]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,35]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,96]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,24]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,44]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,50]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,114]
+  CRUSH rule 3 x 638 [43,78]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,99]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,39]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,107]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,102]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,100]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,116]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,26]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36]
+  CRUSH rule 3 x 697 [19,38]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,60]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,113]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,38]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,26]
+  CRUSH rule 3 x 723 [117,75]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,38]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,94]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,56]
+  CRUSH rule 3 x 752 [82,16]
+  CRUSH rule 3 x 753 [116,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,65]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,90]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,53]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,93]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,35]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,94]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,19]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,102]
+  CRUSH rule 3 x 824 [102,95]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,33]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,75]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,39]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,31]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,29]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,66]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,62]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,80]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,25]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,60]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,107]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,10]
+  CRUSH rule 3 x 942 [80,37]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,111]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,16]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,46]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,103]
+  CRUSH rule 3 x 969 [53]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,19]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,100]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,113]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,33]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,22]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,35]
+  CRUSH rule 3 x 1014 [33,48]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,111]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [59,88]
+  rule 3 (delltestrule) num_rep 4 result size == 1:\t27/1024 (esc)
+  rule 3 (delltestrule) num_rep 4 result size == 2:\t997/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-vary-r-1.t b/ceph/src/test/cli/crushtool/test-map-vary-r-1.t
new file mode 100644
index 00000000..4ac4c222
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-vary-r-1.t
@@ -0,0 +1,3078 @@
+  $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 1 --weight 0 0 --weight 4 0 --weight 9 0
+  crushtool successfully built or modified map.  Use '-o ' to write it out.
+  rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+  CRUSH rule 3 x 0 [94,6]
+  CRUSH rule 3 x 1 [73,52]
+  CRUSH rule 3 x 2 [91,48]
+  CRUSH rule 3 x 3 [51,48]
+  CRUSH rule 3 x 4 [45,114]
+  CRUSH rule 3 x 5 [89,94]
+  CRUSH rule 3 x 6 [91,76]
+  CRUSH rule 3 x 7 [104,73]
+  CRUSH rule 3 x 8 [41,98]
+  CRUSH rule 3 x 9 [46,47]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,40]
+  CRUSH rule 3 x 12 [83,26]
+  CRUSH rule 3 x 13 [27,28]
+  CRUSH rule 3 x 14 [105,64]
+  CRUSH rule 3 x 15 [18,7]
+  CRUSH rule 3 x 16 [103,30]
+  CRUSH rule 3 x 17 [85,118]
+  CRUSH rule 3 x 18 [11,106]
+  CRUSH rule 3 x 19 [75,50]
+  CRUSH rule 3 x 20 [111,67]
+  CRUSH rule 3 x 21 [84,61]
+  CRUSH rule 3 x 22 [23,104]
+  CRUSH rule 3 x 23 [19,86]
+  CRUSH rule 3 x 24 [83,60]
+  CRUSH rule 3 x 25 [81,64]
+  CRUSH rule 3 x 26 [17,38]
+  CRUSH rule 3 x 27 [33,84]
+  CRUSH rule 3 x 28 [45,90]
+  CRUSH rule 3 x 29 [8,109]
+  CRUSH rule 3 x 30 [55,42]
+  CRUSH rule 3 x 31 [76,95]
+  CRUSH rule 3 x 32 [72,11]
+  CRUSH rule 3 x 33 [86,53]
+  CRUSH rule 3 x 34 [7,108]
+  CRUSH rule 3 x 35 [108,13]
+  CRUSH rule 3 x 36 [67,66]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,105]
+  CRUSH rule 3 x 39 [68,103]
+  CRUSH rule 3 x 40 [30,85]
+  CRUSH rule 3 x 41 [52,11]
+  CRUSH rule 3 x 42 [106,75]
+  CRUSH rule 3 x 43 [10,104]
+  CRUSH rule 3 x 44 [101,28]
+  CRUSH rule 3 x 45 [83,64]
+  CRUSH rule 3 x 46 [54,31]
+  CRUSH rule 3 x 47 [106,61]
+  CRUSH rule 3 x 48 [34,41]
+  CRUSH rule 3 x 49 [79,110]
+  CRUSH rule 3 x 50 [42,13]
+  CRUSH rule 3 x 51 [6,94]
+  CRUSH rule 3 x 52 [82,19]
+  CRUSH rule 3 x 53 [32,91]
+  CRUSH rule 3 x 54 [108,8]
+  CRUSH rule 3 x 55 [14,94]
+  CRUSH rule 3 x 56 [21,72]
+  CRUSH rule 3 x 57 [69,88]
+  CRUSH rule 3 x 58 [48,87]
+  CRUSH rule 3 x 59 [21,113]
+  CRUSH rule 3 x 60 [90,73]
+  CRUSH rule 3 x 61 [88,63]
+  CRUSH rule 3 x 62 [100,13]
+  CRUSH rule 3 x 63 [79,5]
+  CRUSH rule 3 x 64 [1,89]
+  CRUSH rule 3 x 65 [32,103]
+  CRUSH rule 3 x 66 [48,79]
+  CRUSH rule 3 x 67 [94,11]
+  CRUSH rule 3 x 68 [102,15]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,11]
+  CRUSH rule 3 x 71 [12,33]
+  CRUSH rule 3 x 72 [26,99]
+  CRUSH rule 3 x 73 [29,114]
+  CRUSH rule 3 x 74 [29,1]
+  CRUSH rule 3 x 75 [60,65]
+  CRUSH rule 3 x 76 [55,62]
+  CRUSH rule 3 x 77 [107,100]
+  CRUSH rule 3 x 78 [86,107]
+  CRUSH rule 3 x 79 [64,16]
+  CRUSH rule 3 x 80 [19,100]
+  CRUSH rule 3 x 81 [64,16]
+  CRUSH rule 3 x 82 [37,40]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,115]
+  CRUSH rule 3 x 85 [87,88]
+  CRUSH rule 3 x 86 [37,68]
+  CRUSH rule 3 x 87 [116,77]
+  CRUSH rule 3 x 88 [38,55]
+  CRUSH rule 3 x 89 [76,25]
+  CRUSH rule 3 x 90 [14,50]
+  CRUSH rule 3 x 91 [68,61]
+  CRUSH rule 3 x 92 [86,95]
+  CRUSH rule 3 x 93 [44,35]
+  CRUSH rule 3 x 94 [46,71]
+  CRUSH rule 3 x 95 [108,53]
+  CRUSH rule 3 x 96 [66,87]
+  CRUSH rule 3 x 97 [111,45]
+  CRUSH rule 3 x 98 [93,110]
+  CRUSH rule 3 x 99 [78,43]
+  CRUSH rule 3 x 100 [28,61]
+  CRUSH rule 3 x 101 [91,110]
+  CRUSH rule 3 x 102 [82,7]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,79]
+  CRUSH rule 3 x 105 [34,87]
+  CRUSH rule 3 x 106 [69,12]
+  CRUSH rule 3 x 107 [1,59]
+  CRUSH rule 3 x 108 [7,109]
+  CRUSH rule 3 x 109 [112,67]
+  CRUSH rule 3 x 110 [54,61]
+  CRUSH rule 3 x 111 [10,92]
+  CRUSH rule 3 x 112 [80,11]
+  CRUSH rule 3 x 113 [69,38]
+  CRUSH rule 3 x 114 [79,38]
+  CRUSH rule 3 x 115 [10,48]
+  CRUSH rule 3 x 116 [37,108]
+  CRUSH rule 3 x 117 [87,56]
+  CRUSH rule 3 x 118 [23,56]
+  CRUSH rule 3 x 119 [104,31]
+  CRUSH rule 3 x 120 [44,93]
+  CRUSH rule 3 x 121 [80,16]
+  CRUSH rule 3 x 122 [45,54]
+  CRUSH rule 3 x 123 [22,112]
+  CRUSH rule 3 x 124 [97,50]
+  CRUSH rule 3 x 125 [66,6]
+  CRUSH rule 3 x 126 [70,39]
+  CRUSH rule 3 x 127 [70,75]
+  CRUSH rule 3 x 128 [11,111]
+  CRUSH rule 3 x 129 [103,46]
+  CRUSH rule 3 x 130 [50,73]
+  CRUSH rule 3 x 131 [44,15]
+  CRUSH rule 3 x 132 [69,58]
+  CRUSH rule 3 x 133 [67,115]
+  CRUSH rule 3 x 134 [37,60]
+  CRUSH rule 3 x 135 [78,61]
+  CRUSH rule 3 x 136 [32,29]
+  CRUSH rule 3 x 137 [92,87]
+  CRUSH rule 3 x 138 [54,8]
+  CRUSH rule 3 x 139 [89,60]
+  CRUSH rule 3 x 140 [39,50]
+  CRUSH rule 3 x 141 [89,62]
+  CRUSH rule 3 x 142 [22,86]
+  CRUSH rule 3 x 143 [96,16]
+  CRUSH rule 3 x 144 [13,1]
+  CRUSH rule 3 x 145 [77,54]
+  CRUSH rule 3 x 146 [12,43]
+  CRUSH rule 3 x 147 [2,59]
+  CRUSH rule 3 x 148 [85,50]
+  CRUSH rule 3 x 149 [103,68]
+  CRUSH rule 3 x 150 [14,50]
+  CRUSH rule 3 x 151 [75,56]
+  CRUSH rule 3 x 152 [49,18]
+  CRUSH rule 3 x 153 [92,79]
+  CRUSH rule 3 x 154 [19,26]
+  CRUSH rule 3 x 155 [12,13]
+  CRUSH rule 3 x 156 [107,18]
+  CRUSH rule 3 x 157 [15,78]
+  CRUSH rule 3 x 158 [11,28]
+  CRUSH rule 3 x 159 [33,88]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,78]
+  CRUSH rule 3 x 162 [55,96]
+  CRUSH rule 3 x 163 [54,55]
+  CRUSH rule 3 x 164 [72,99]
+  CRUSH rule 3 x 165 [25,116]
+  CRUSH rule 3 x 166 [2,23]
+  CRUSH rule 3 x 167 [89,40]
+  CRUSH rule 3 x 168 [68,49]
+  CRUSH rule 3 x 169 [51,50]
+  CRUSH rule 3 x 170 [68,91]
+  CRUSH rule 3 x 171 [88,51]
+  CRUSH rule 3 x 172 [117,23]
+  CRUSH rule 3 x 173 [29,1]
+  CRUSH rule 3 x 174 [67,40]
+  CRUSH rule 3 x 175 [48,41]
+  CRUSH rule 3 x 176 [94,91]
+  CRUSH rule 3 x 177 [53,70]
+  CRUSH rule 3 x 178 [39,110]
+  CRUSH rule 3 x 179 [72,89]
+  CRUSH rule 3 x 180 [3,38]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,46]
+  CRUSH rule 3 x 183 [11,78]
+  CRUSH rule 3 x 184 [79,92]
+  CRUSH rule 3 x 185 [97,92]
+  CRUSH rule 3 x 186 [67,116]
+  CRUSH rule 3 x 187 [6,96]
+  CRUSH rule 3 x 188 [76,16]
+  CRUSH rule 3 x 189 [96,71]
+  CRUSH rule 3 x 190 [90,6]
+  CRUSH rule 3 x 191 [49,84]
+  CRUSH rule 3 x 192 [93,114]
+  CRUSH rule 3 x 193 [89,60]
+  CRUSH rule 3 x 194 [62,61]
+  CRUSH rule 3 x 195 [119,95]
+  CRUSH rule 3 x 196 [20,28]
+  CRUSH rule 3 x 197 [6,64]
+  CRUSH rule 3 x 198 [55,112]
+  CRUSH rule 3 x 199 [66,17]
+  CRUSH rule 3 x 200 [12,63]
+  CRUSH rule 3 x 201 [52,23]
+  CRUSH rule 3 x 202 [98,33]
+  CRUSH rule 3 x 203 [36,22]
+  CRUSH rule 3 x 204 [10,100]
+  CRUSH rule 3 x 205 [38,29]
+  CRUSH rule 3 x 206 [38,27]
+  CRUSH rule 3 x 207 [19,108]
+  CRUSH rule 3 x 208 [63,26]
+  CRUSH rule 3 x 209 [70,11]
+  CRUSH rule 3 x 210 [79,58]
+  CRUSH rule 3 x 211 [26,59]
+  CRUSH rule 3 x 212 [107,114]
+  CRUSH rule 3 x 213 [100,3]
+  CRUSH rule 3 x 214 [91,118]
+  CRUSH rule 3 x 215 [92,16]
+  CRUSH rule 3 x 216 [99,94]
+  CRUSH rule 3 x 217 [86,99]
+  CRUSH rule 3 x 218 [70,15]
+  CRUSH rule 3 x 219 [61,58]
+  CRUSH rule 3 x 220 [23,56]
+  CRUSH rule 3 x 221 [21,92]
+  CRUSH rule 3 x 222 [102,29]
+  CRUSH rule 3 x 223 [34,73]
+  CRUSH rule 3 x 224 [107,68]
+  CRUSH rule 3 x 225 [61,98]
+  CRUSH rule 3 x 226 [44,13]
+  CRUSH rule 3 x 227 [55,60]
+  CRUSH rule 3 x 228 [117,55]
+  CRUSH rule 3 x 229 [100,67]
+  CRUSH rule 3 x 230 [41,109]
+  CRUSH rule 3 x 231 [30,71]
+  CRUSH rule 3 x 232 [23,1]
+  CRUSH rule 3 x 233 [47,90]
+  CRUSH rule 3 x 234 [55,62]
+  CRUSH rule 3 x 235 [20,60]
+  CRUSH rule 3 x 236 [95,24]
+  CRUSH rule 3 x 237 [21,106]
+  CRUSH rule 3 x 238 [109,15]
+  CRUSH rule 3 x 239 [40,101]
+  CRUSH rule 3 x 240 [63,60]
+  CRUSH rule 3 x 241 [47,12]
+  CRUSH rule 3 x 242 [73,74]
+  CRUSH rule 3 x 243 [76,8]
+  CRUSH rule 3 x 244 [103,50]
+  CRUSH rule 3 x 245 [106,95]
+  CRUSH rule 3 x 246 [35,82]
+  CRUSH rule 3 x 247 [116,101]
+  CRUSH rule 3 x 248 [8,119]
+  CRUSH rule 3 x 249 [2,17]
+  CRUSH rule 3 x 250 [34,89]
+  CRUSH rule 3 x 251 [28,69]
+  CRUSH rule 3 x 252 [95,80]
+  CRUSH rule 3 x 253 [109,3]
+  CRUSH rule 3 x 254 [99,80]
+  CRUSH rule 3 x 255 [112,85]
+  CRUSH rule 3 x 256 [94,63]
+  CRUSH rule 3 x 257 [100,87]
+  CRUSH rule 3 x 258 [34,63]
+  CRUSH rule 3 x 259 [70,107]
+  CRUSH rule 3 x 260 [89,115]
+  CRUSH rule 3 x 261 [94,83]
+  CRUSH rule 3 x 262 [42,45]
+  CRUSH rule 3 x 263 [113,101]
+  CRUSH rule 3 x 264 [36,81]
+  CRUSH rule 3 x 265 [14,88]
+  CRUSH rule 3 x 266 [75,96]
+  CRUSH rule 3 x 267 [6,5]
+  CRUSH rule 3 x 268 [38,47]
+  CRUSH rule 3 x 269 [86,59]
+  CRUSH rule 3 x 270 [87,70]
+  CRUSH rule 3 x 271 [19,108]
+  CRUSH rule 3 x 272 [73,5]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,64]
+  CRUSH rule 3 x 275 [29,34]
+  CRUSH rule 3 x 276 [7,100]
+  CRUSH rule 3 x 277 [74,6]
+  CRUSH rule 3 x 278 [107,115]
+  CRUSH rule 3 x 279 [112,20]
+  CRUSH rule 3 x 280 [113,15]
+  CRUSH rule 3 x 281 [89,56]
+  CRUSH rule 3 x 282 [20,38]
+  CRUSH rule 3 x 283 [8,114]
+  CRUSH rule 3 x 284 [66,75]
+  CRUSH rule 3 x 285 [99,94]
+  CRUSH rule 3 x 286 [78,6]
+  CRUSH rule 3 x 287 [12,27]
+  CRUSH rule 3 x 288 [24,22]
+  CRUSH rule 3 x 289 [105,64]
+  CRUSH rule 3 x 290 [25,46]
+  CRUSH rule 3 x 291 [35,116]
+  CRUSH rule 3 x 292 [20,109]
+  CRUSH rule 3 x 293 [27,92]
+  CRUSH rule 3 x 294 [60,93]
+  CRUSH rule 3 x 295 [37,2]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,55]
+  CRUSH rule 3 x 298 [70,53]
+  CRUSH rule 3 x 299 [116,10]
+  CRUSH rule 3 x 300 [67,26]
+  CRUSH rule 3 x 301 [117,23]
+  CRUSH rule 3 x 302 [78,67]
+  CRUSH rule 3 x 303 [19,5]
+  CRUSH rule 3 x 304 [101,50]
+  CRUSH rule 3 x 305 [5,59]
+  CRUSH rule 3 x 306 [41,18]
+  CRUSH rule 3 x 307 [65,5]
+  CRUSH rule 3 x 308 [91,2]
+  CRUSH rule 3 x 309 [38,53]
+  CRUSH rule 3 x 310 [26,15]
+  CRUSH rule 3 x 311 [36,95]
+  CRUSH rule 3 x 312 [114,61]
+  CRUSH rule 3 x 313 [104,65]
+  CRUSH rule 3 x 314 [28,6]
+  CRUSH rule 3 x 315 [118,31]
+  CRUSH rule 3 x 316 [98,95]
+  CRUSH rule 3 x 317 [118,13]
+  CRUSH rule 3 x 318 [17,30]
+  CRUSH rule 3 x 319 [53,1]
+  CRUSH rule 3 x 320 [36,41]
+  CRUSH rule 3 x 321 [33,5]
+  CRUSH rule 3 x 322 [68,10]
+  CRUSH rule 3 x 323 [66,29]
+  CRUSH rule 3 x 324 [21,72]
+  CRUSH rule 3 x 325 [52,16]
+  CRUSH rule 3 x 326 [7,109]
+  CRUSH rule 3 x 327 [62,17]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,100]
+  CRUSH rule 3 x 330 [24,11]
+  CRUSH rule 3 x 331 [84,95]
+  CRUSH rule 3 x 332 [61,111]
+  CRUSH rule 3 x 333 [116,25]
+  CRUSH rule 3 x 334 [94,103]
+  CRUSH rule 3 x 335 [71,118]
+  CRUSH rule 3 x 336 [24,99]
+  CRUSH rule 3 x 337 [18,83]
+  CRUSH rule 3 x 338 [43,28]
+  CRUSH rule 3 x 339 [13,64]
+  CRUSH rule 3 x 340 [81,111]
+  CRUSH rule 3 x 341 [46,105]
+  CRUSH rule 3 x 342 [92,23]
+  CRUSH rule 3 x 343 [49,112]
+  CRUSH rule 3 x 344 [1,87]
+  CRUSH rule 3 x 345 [56,35]
+  CRUSH rule 3 x 346 [3,54]
+  CRUSH rule 3 x 347 [106,27]
+  CRUSH rule 3 x 348 [10,117]
+  CRUSH rule 3 x 349 [96,87]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,41]
+  CRUSH rule 3 x 352 [36,13]
+  CRUSH rule 3 x 353 [10,82]
+  CRUSH rule 3 x 354 [55,52]
+  CRUSH rule 3 x 355 [73,94]
+  CRUSH rule 3 x 356 [75,66]
+  CRUSH rule 3 x 357 [70,93]
+  CRUSH rule 3 x 358 [97,56]
+  CRUSH rule 3 x 359 [110,105]
+  CRUSH rule 3 x 360 [106,57]
+  CRUSH rule 3 x 361 [27,42]
+  CRUSH rule 3 x 362 [28,55]
+  CRUSH rule 3 x 363 [68,20]
+  CRUSH rule 3 x 364 [23,50]
+  CRUSH rule 3 x 365 [57,76]
+  CRUSH rule 3 x 366 [42,75]
+  CRUSH rule 3 x 367 [103,82]
+  CRUSH rule 3 x 368 [103,104]
+  CRUSH rule 3 x 369 [12,57]
+  CRUSH rule 3 x 370 [11,26]
+  CRUSH rule 3 x 371 [34,55]
+  CRUSH rule 3 x 372 [58,14]
+  CRUSH rule 3 x 373 [6,42]
+  CRUSH rule 3 x 374 [110,95]
+  CRUSH rule 3 x 375 [5,43]
+  CRUSH rule 3 x 376 [91,86]
+  CRUSH rule 3 x 377 [93,116]
+  CRUSH rule 3 x 378 [68,6]
+  CRUSH rule 3 x 379 [77,44]
+  CRUSH rule 3 x 380 [76,83]
+  CRUSH rule 3 x 381 [36,27]
+  CRUSH rule 3 x 382 [26,77]
+  CRUSH rule 3 x 383 [76,99]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,93]
+  CRUSH rule 3 x 386 [83,92]
+  CRUSH rule 3 x 387 [16,26]
+  CRUSH rule 3 x 388 [29,113]
+  CRUSH rule 3 x 389 [92,29]
+  CRUSH rule 3 x 390 [68,77]
+  CRUSH rule 3 x 391 [15,88]
+  CRUSH rule 3 x 392 [21,32]
+  CRUSH rule 3 x 393 [91,18]
+  CRUSH rule 3 x 394 [38,73]
+  CRUSH rule 3 x 395 [21,119]
+  CRUSH rule 3 x 396 [12,13]
+  CRUSH rule 3 x 397 [40,63]
+  CRUSH rule 3 x 398 [44,3]
+  CRUSH rule 3 x 399 [5,95]
+  CRUSH rule 3 x 400 [19,102]
+  CRUSH rule 3 x 401 [79,52]
+  CRUSH rule 3 x 402 [107,98]
+  CRUSH rule 3 x 403 [23,82]
+  CRUSH rule 3 x 404 [87,68]
+  CRUSH rule 3 x 405 [90,97]
+  CRUSH rule 3 x 406 [15,117]
+  CRUSH rule 3 x 407 [70,35]
+  CRUSH rule 3 x 408 [55,72]
+  CRUSH rule 3 x 409 [73,62]
+  CRUSH rule 3 x 410 [70,73]
+  CRUSH rule 3 x 411 [34,25]
+  CRUSH rule 3 x 412 [105,117]
+  CRUSH rule 3 x 413 [41,110]
+  CRUSH rule 3 x 414 [70,65]
+  CRUSH rule 3 x 415 [107,5]
+  CRUSH rule 3 x 416 [2,22]
+  CRUSH rule 3 x 417 [26,14]
+  CRUSH rule 3 x 418 [51,46]
+  CRUSH rule 3 x 419 [8,82]
+  CRUSH rule 3 x 420 [109,105]
+  CRUSH rule 3 x 421 [114,75]
+  CRUSH rule 3 x 422 [109,87]
+  CRUSH rule 3 x 423 [59,24]
+  CRUSH rule 3 x 424 [92,51]
+  CRUSH rule 3 x 425 [101,111]
+  CRUSH rule 3 x 426 [36,6]
+  CRUSH rule 3 x 427 [8,24]
+  CRUSH rule 3 x 428 [68,35]
+  CRUSH rule 3 x 429 [76,75]
+  CRUSH rule 3 x 430 [67,117]
+  CRUSH rule 3 x 431 [70,25]
+  CRUSH rule 3 x 432 [7,34]
+  CRUSH rule 3 x 433 [49,84]
+  CRUSH rule 3 x 434 [64,31]
+  CRUSH rule 3 x 435 [110,13]
+  CRUSH rule 3 x 436 [106,89]
+  CRUSH rule 3 x 437 [26,65]
+  CRUSH rule 3 x 438 [118,63]
+  CRUSH rule 3 x 439 [40,21]
+  CRUSH rule 3 x 440 [45,119]
+  CRUSH rule 3 x 441 [112,105]
+  CRUSH rule 3 x 442 [55,113]
+  CRUSH rule 3 x 443 [44,33]
+  CRUSH rule 3 x 444 [71,38]
+  CRUSH rule 3 x 445 [58,81]
+  CRUSH rule 3 x 446 [40,10]
+  CRUSH rule 3 x 447 [100,61]
+  CRUSH rule 3 x 448 [111,73]
+  CRUSH rule 3 x 449 [67,66]
+  CRUSH rule 3 x 450 [117,61]
+  CRUSH rule 3 x 451 [66,81]
+  CRUSH rule 3 x 452 [70,8]
+  CRUSH rule 3 x 453 [82,85]
+  CRUSH rule 3 x 454 [53,18]
+  CRUSH rule 3 x 455 [91,42]
+  CRUSH rule 3 x 456 [101,46]
+  CRUSH rule 3 x 457 [113,51]
+  CRUSH rule 3 x 458 [119,25]
+  CRUSH rule 3 x 459 [50,67]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,51]
+  CRUSH rule 3 x 462 [98,107]
+  CRUSH rule 3 x 463 [108,105]
+  CRUSH rule 3 x 464 [19,109]
+  CRUSH rule 3 x 465 [62,23]
+  CRUSH rule 3 x 466 [53,12]
+  CRUSH rule 3 x 467 [40,57]
+  CRUSH rule 3 x 468 [97,44]
+  CRUSH rule 3 x 469 [98,75]
+  CRUSH rule 3 x 470 [50,29]
+  CRUSH rule 3 x 471 [40,13]
+  CRUSH rule 3 x 472 [27,18]
+  CRUSH rule 3 x 473 [48,35]
+  CRUSH rule 3 x 474 [51,32]
+  CRUSH rule 3 x 475 [49,117]
+  CRUSH rule 3 x 476 [110,31]
+  CRUSH rule 3 x 477 [80,97]
+  CRUSH rule 3 x 478 [78,99]
+  CRUSH rule 3 x 479 [31,66]
+  CRUSH rule 3 x 480 [75,88]
+  CRUSH rule 3 x 481 [26,20]
+  CRUSH rule 3 x 482 [84,53]
+  CRUSH rule 3 x 483 [15,116]
+  CRUSH rule 3 x 484 [37,114]
+  CRUSH rule 3 x 485 [84,8]
+  CRUSH rule 3 x 486 [92,10]
+  CRUSH rule 3 x 487 [106,17]
+  CRUSH rule 3 x 488 [42,20]
+  CRUSH rule 3 x 489 [89,2]
+  CRUSH rule 3 x 490 [22,114]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,66]
+  CRUSH rule 3 x 493 [94,14]
+  CRUSH rule 3 x 494 [59,86]
+  CRUSH rule 3 x 495 [95,58]
+  CRUSH rule 3 x 496 [46,41]
+  CRUSH rule 3 x 497 [102,27]
+  CRUSH rule 3 x 498 [21,116]
+  CRUSH rule 3 x 499 [5,49]
+  CRUSH rule 3 x 500 [50,49]
+  CRUSH rule 3 x 501 [60,3]
+  CRUSH rule 3 x 502 [65,110]
+  CRUSH rule 3 x 503 [21,112]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,93]
+  CRUSH rule 3 x 506 [79,64]
+  CRUSH rule 3 x 507 [34,107]
+  CRUSH rule 3 x 508 [45,114]
+  CRUSH rule 3 x 509 [19,88]
+  CRUSH rule 3 x 510 [117,45]
+  CRUSH rule 3 x 511 [14,104]
+  CRUSH rule 3 x 512 [59,26]
+  CRUSH rule 3 x 513 [102,93]
+  CRUSH rule 3 x 514 [75,72]
+  CRUSH rule 3 x 515 [84,41]
+  CRUSH rule 3 x 516 [37,30]
+  CRUSH rule 3 x 517 [83,115]
+  CRUSH rule 3 x 518 [18,83]
+  CRUSH rule 3 x 519 [67,88]
+  CRUSH rule 3 x 520 [15,114]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,51]
+  CRUSH rule 3 x 523 [68,101]
+  CRUSH rule 3 x 524 [33,38]
+  CRUSH rule 3 x 525 [63,115]
+  CRUSH rule 3 x 526 [83,50]
+  CRUSH rule 3 x 527 [37,56]
+  CRUSH rule 3 x 528 [108,81]
+  CRUSH rule 3 x 529 [74,33]
+  CRUSH rule 3 x 530 [49,92]
+  CRUSH rule 3 x 531 [117,105]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,85]
+  CRUSH rule 3 x 534 [97,24]
+  CRUSH rule 3 x 535 [48,75]
+  CRUSH rule 3 x 536 [113,101]
+  CRUSH rule 3 x 537 [116,47]
+  CRUSH rule 3 x 538 [85,74]
+  CRUSH rule 3 x 539 [72,43]
+  CRUSH rule 3 x 540 [39,34]
+  CRUSH rule 3 x 541 [53,84]
+  CRUSH rule 3 x 542 [27,32]
+  CRUSH rule 3 x 543 [45,113]
+  CRUSH rule 3 x 544 [59,42]
+  CRUSH rule 3 x 545 [118,95]
+  CRUSH rule 3 x 546 [18,79]
+  CRUSH rule 3 x 547 [67,30]
+  CRUSH rule 3 x 548 [53,100]
+  CRUSH rule 3 x 549 [60,45]
+  CRUSH rule 3 x 550 [92,101]
+  CRUSH rule 3 x 551 [77,88]
+  CRUSH rule 3 x 552 [61,94]
+  CRUSH rule 3 x 553 [71,78]
+  CRUSH rule 3 x 554 [61,115]
+  CRUSH rule 3 x 555 [76,77]
+  CRUSH rule 3 x 556 [106,55]
+  CRUSH rule 3 x 557 [26,22]
+  CRUSH rule 3 x 558 [41,84]
+  CRUSH rule 3 x 559 [65,24]
+  CRUSH rule 3 x 560 [94,16]
+  CRUSH rule 3 x 561 [27,5]
+  CRUSH rule 3 x 562 [78,59]
+  CRUSH rule 3 x 563 [59,70]
+  CRUSH rule 3 x 564 [96,8]
+  CRUSH rule 3 x 565 [8,48]
+  CRUSH rule 3 x 566 [119,17]
+  CRUSH rule 3 x 567 [7,38]
+  CRUSH rule 3 x 568 [57,94]
+  CRUSH rule 3 x 569 [65,26]
+  CRUSH rule 3 x 570 [98,27]
+  CRUSH rule 3 x 571 [95,30]
+  CRUSH rule 3 x 572 [62,83]
+  CRUSH rule 3 x 573 [1,79]
+  CRUSH rule 3 x 574 [89,42]
+  CRUSH rule 3 x 575 [87,113]
+  CRUSH rule 3 x 576 [21,68]
+  CRUSH rule 3 x 577 [8,84]
+  CRUSH rule 3 x 578 [75,115]
+  CRUSH rule 3 x 579 [105,68]
+  CRUSH rule 3 x 580 [51,28]
+  CRUSH rule 3 x 581 [55,113]
+  CRUSH rule 3 x 582 [27,113]
+  CRUSH rule 3 x 583 [6,78]
+  CRUSH rule 3 x 584 [10,30]
+  CRUSH rule 3 x 585 [20,111]
+  CRUSH rule 3 x 586 [48,27]
+  CRUSH rule 3 x 587 [29,94]
+  CRUSH rule 3 x 588 [103,90]
+  CRUSH rule 3 x 589 [88,95]
+  CRUSH rule 3 x 590 [76,101]
+  CRUSH rule 3 x 591 [42,43]
+  CRUSH rule 3 x 592 [78,51]
+  CRUSH rule 3 x 593 [82,71]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,59]
+  CRUSH rule 3 x 597 [16,36]
+  CRUSH rule 3 x 598 [37,56]
+  CRUSH rule 3 x 599 [10,84]
+  CRUSH rule 3 x 600 [24,69]
+  CRUSH rule 3 x 601 [104,14]
+  CRUSH rule 3 x 602 [48,45]
+  CRUSH rule 3 x 603 [93,32]
+  CRUSH rule 3 x 604 [118,79]
+  CRUSH rule 3 x 605 [104,53]
+  CRUSH rule 3 x 606 [90,83]
+  CRUSH rule 3 x 607 [95,110]
+  CRUSH rule 3 x 608 [112,101]
+  CRUSH rule 3 x 609 [34,99]
+  CRUSH rule 3 x 610 [106,16]
+  CRUSH rule 3 x 611 [66,87]
+  CRUSH rule 3 x 612 [2,81]
+  CRUSH rule 3 x 613 [13,86]
+  CRUSH rule 3 x 614 [50,3]
+  CRUSH rule 3 x 615 [24,73]
+  CRUSH rule 3 x 616 [41,119]
+  CRUSH rule 3 x 617 [81,106]
+  CRUSH rule 3 x 618 [3,104]
+  CRUSH rule 3 x 619 [92,7]
+  CRUSH rule 3 x 620 [108,11]
+  CRUSH rule 3 x 621 [105,115]
+  CRUSH rule 3 x 622 [67,48]
+  CRUSH rule 3 x 623 [69,74]
+  CRUSH rule 3 x 624 [115,49]
+  CRUSH rule 3 x 625 [73,109]
+  CRUSH rule 3 x 626 [52,3]
+  CRUSH rule 3 x 627 [116,3]
+  CRUSH rule 3 x 628 [98,91]
+  CRUSH rule 3 x 629 [6,112]
+  CRUSH rule 3 x 630 [22,72]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,71]
+  CRUSH rule 3 x 633 [65,12]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,46]
+  CRUSH rule 3 x 636 [23,70]
+  CRUSH rule 3 x 637 [99,24]
+  CRUSH rule 3 x 638 [43,114]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,73]
+  CRUSH rule 3 x 641 [45,84]
+  CRUSH rule 3 x 642 [47,66]
+  CRUSH rule 3 x 643 [64,8]
+  CRUSH rule 3 x 644 [31,82]
+  CRUSH rule 3 x 645 [77,64]
+  CRUSH rule 3 x 646 [37,86]
+  CRUSH rule 3 x 647 [65,56]
+  CRUSH rule 3 x 648 [84,13]
+  CRUSH rule 3 x 649 [88,55]
+  CRUSH rule 3 x 650 [21,76]
+  CRUSH rule 3 x 651 [63,116]
+  CRUSH rule 3 x 652 [57,112]
+  CRUSH rule 3 x 653 [38,61]
+  CRUSH rule 3 x 654 [104,67]
+  CRUSH rule 3 x 655 [89,54]
+  CRUSH rule 3 x 656 [84,49]
+  CRUSH rule 3 x 657 [47,32]
+  CRUSH rule 3 x 658 [80,29]
+  CRUSH rule 3 x 659 [11,112]
+  CRUSH rule 3 x 660 [65,111]
+  CRUSH rule 3 x 661 [96,73]
+  CRUSH rule 3 x 662 [111,73]
+  CRUSH rule 3 x 663 [83,60]
+  CRUSH rule 3 x 664 [59,80]
+  CRUSH rule 3 x 665 [31,117]
+  CRUSH rule 3 x 666 [112,101]
+  CRUSH rule 3 x 667 [70,47]
+  CRUSH rule 3 x 668 [96,57]
+  CRUSH rule 3 x 669 [56,39]
+  CRUSH rule 3 x 670 [98,105]
+  CRUSH rule 3 x 671 [57,48]
+  CRUSH rule 3 x 672 [37,36]
+  CRUSH rule 3 x 673 [83,2]
+  CRUSH rule 3 x 674 [36,25]
+  CRUSH rule 3 x 675 [88,14]
+  CRUSH rule 3 x 676 [3,110]
+  CRUSH rule 3 x 677 [88,67]
+  CRUSH rule 3 x 678 [27,44]
+  CRUSH rule 3 x 679 [33,116]
+  CRUSH rule 3 x 680 [111,39]
+  CRUSH rule 3 x 681 [53,12]
+  CRUSH rule 3 x 682 [12,87]
+  CRUSH rule 3 x 683 [24,85]
+  CRUSH rule 3 x 684 [98,65]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,72]
+  CRUSH rule 3 x 688 [16,114]
+  CRUSH rule 3 x 689 [32,31]
+  CRUSH rule 3 x 690 [96,33]
+  CRUSH rule 3 x 691 [34,6]
+  CRUSH rule 3 x 692 [97,84]
+  CRUSH rule 3 x 693 [29,118]
+  CRUSH rule 3 x 694 [6,30]
+  CRUSH rule 3 x 695 [31,72]
+  CRUSH rule 3 x 696 [104,97]
+  CRUSH rule 3 x 697 [19,96]
+  CRUSH rule 3 x 698 [30,69]
+  CRUSH rule 3 x 699 [47,76]
+  CRUSH rule 3 x 700 [82,55]
+  CRUSH rule 3 x 701 [53,80]
+  CRUSH rule 3 x 702 [95,98]
+  CRUSH rule 3 x 703 [92,65]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,1]
+  CRUSH rule 3 x 706 [74,35]
+  CRUSH rule 3 x 707 [91,115]
+  CRUSH rule 3 x 708 [95,112]
+  CRUSH rule 3 x 709 [73,72]
+  CRUSH rule 3 x 710 [94,47]
+  CRUSH rule 3 x 711 [68,41]
+  CRUSH rule 3 x 712 [107,18]
+  CRUSH rule 3 x 713 [29,109]
+  CRUSH rule 3 x 714 [86,61]
+  CRUSH rule 3 x 715 [74,13]
+  CRUSH rule 3 x 716 [101,56]
+  CRUSH rule 3 x 717 [12,29]
+  CRUSH rule 3 x 718 [83,24]
+  CRUSH rule 3 x 719 [26,10]
+  CRUSH rule 3 x 720 [69,2]
+  CRUSH rule 3 x 721 [51,42]
+  CRUSH rule 3 x 722 [15,74]
+  CRUSH rule 3 x 723 [117,14]
+  CRUSH rule 3 x 724 [45,38]
+  CRUSH rule 3 x 725 [53,110]
+  CRUSH rule 3 x 726 [103,68]
+  CRUSH rule 3 x 727 [89,100]
+  CRUSH rule 3 x 728 [76,16]
+  CRUSH rule 3 x 729 [35,90]
+  CRUSH rule 3 x 730 [28,103]
+  CRUSH rule 3 x 731 [78,41]
+  CRUSH rule 3 x 732 [1,27]
+  CRUSH rule 3 x 733 [35,100]
+  CRUSH rule 3 x 734 [119,85]
+  CRUSH rule 3 x 735 [102,43]
+  CRUSH rule 3 x 736 [37,92]
+  CRUSH rule 3 x 737 [117,11]
+  CRUSH rule 3 x 738 [57,32]
+  CRUSH rule 3 x 739 [87,1]
+  CRUSH rule 3 x 740 [29,80]
+  CRUSH rule 3 x 741 [47,111]
+  CRUSH rule 3 x 742 [106,83]
+  CRUSH rule 3 x 743 [105,94]
+  CRUSH rule 3 x 744 [23,64]
+  CRUSH rule 3 x 745 [37,112]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,95]
+  CRUSH rule 3 x 748 [48,14]
+  CRUSH rule 3 x 749 [102,101]
+  CRUSH rule 3 x 750 [83,78]
+  CRUSH rule 3 x 751 [25,104]
+  CRUSH rule 3 x 752 [82,95]
+  CRUSH rule 3 x 753 [14,113]
+  CRUSH rule 3 x 754 [114,51]
+  CRUSH rule 3 x 755 [87,26]
+  CRUSH rule 3 x 756 [113,87]
+  CRUSH rule 3 x 757 [47,66]
+  CRUSH rule 3 x 758 [54,63]
+  CRUSH rule 3 x 759 [74,20]
+  CRUSH rule 3 x 760 [88,22]
+  CRUSH rule 3 x 761 [73,86]
+  CRUSH rule 3 x 762 [34,17]
+  CRUSH rule 3 x 763 [13,78]
+  CRUSH rule 3 x 764 [89,42]
+  CRUSH rule 3 x 765 [109,91]
+  CRUSH rule 3 x 766 [19,66]
+  CRUSH rule 3 x 767 [41,26]
+  CRUSH rule 3 x 768 [106,57]
+  CRUSH rule 3 x 769 [91,104]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,35]
+  CRUSH rule 3 x 772 [97,108]
+  CRUSH rule 3 x 773 [116,47]
+  CRUSH rule 3 x 774 [100,31]
+  CRUSH rule 3 x 775 [102,43]
+  CRUSH rule 3 x 776 [69,38]
+  CRUSH rule 3 x 777 [91,52]
+  CRUSH rule 3 x 778 [83,119]
+  CRUSH rule 3 x 779 [47,60]
+  CRUSH rule 3 x 780 [63,70]
+  CRUSH rule 3 x 781 [105,2]
+  CRUSH rule 3 x 782 [117,59]
+  CRUSH rule 3 x 783 [19,109]
+  CRUSH rule 3 x 784 [63,114]
+  CRUSH rule 3 x 785 [27,84]
+  CRUSH rule 3 x 786 [41,110]
+  CRUSH rule 3 x 787 [108,73]
+  CRUSH rule 3 x 788 [74,103]
+  CRUSH rule 3 x 789 [50,17]
+  CRUSH rule 3 x 790 [20,106]
+  CRUSH rule 3 x 791 [96,87]
+  CRUSH rule 3 x 792 [80,97]
+  CRUSH rule 3 x 793 [6,26]
+  CRUSH rule 3 x 794 [14,42]
+  CRUSH rule 3 x 795 [30,8]
+  CRUSH rule 3 x 796 [87,36]
+  CRUSH rule 3 x 797 [64,61]
+  CRUSH rule 3 x 798 [42,69]
+  CRUSH rule 3 x 799 [19,117]
+  CRUSH rule 3 x 800 [106,8]
+  CRUSH rule 3 x 801 [2,57]
+  CRUSH rule 3 x 802 [63,68]
+  CRUSH rule 3 x 803 [46,35]
+  CRUSH rule 3 x 804 [33,26]
+  CRUSH rule 3 x 805 [96,49]
+  CRUSH rule 3 x 806 [48,25]
+  CRUSH rule 3 x 807 [48,83]
+  CRUSH rule 3 x 808 [76,31]
+  CRUSH rule 3 x 809 [27,48]
+  CRUSH rule 3 x 810 [119,71]
+  CRUSH rule 3 x 811 [111,91]
+  CRUSH rule 3 x 812 [25,111]
+  CRUSH rule 3 x 813 [81,28]
+  CRUSH rule 3 x 814 [95,42]
+  CRUSH rule 3 x 815 [84,61]
+  CRUSH rule 3 x 816 [64,35]
+  CRUSH rule 3 x 817 [63,60]
+  CRUSH rule 3 x 818 [69,46]
+  CRUSH rule 3 x 819 [88,75]
+  CRUSH rule 3 x 820 [104,57]
+  CRUSH rule 3 x 821 [58,21]
+  CRUSH rule 3 x 822 [20,80]
+  CRUSH rule 3 x 823 [63,118]
+  CRUSH rule 3 x 824 [102,13]
+  CRUSH rule 3 x 825 [47,118]
+  CRUSH rule 3 x 826 [44,7]
+  CRUSH rule 3 x 827 [101,88]
+  CRUSH rule 3 x 828 [60,41]
+  CRUSH rule 3 x 829 [45,102]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,75]
+  CRUSH rule 3 x 833 [57,32]
+  CRUSH rule 3 x 834 [90,33]
+  CRUSH rule 3 x 835 [6,1]
+  CRUSH rule 3 x 836 [63,68]
+  CRUSH rule 3 x 837 [76,71]
+  CRUSH rule 3 x 838 [106,20]
+  CRUSH rule 3 x 839 [87,96]
+  CRUSH rule 3 x 840 [33,32]
+  CRUSH rule 3 x 841 [110,55]
+  CRUSH rule 3 x 842 [66,87]
+  CRUSH rule 3 x 843 [11,80]
+  CRUSH rule 3 x 844 [74,103]
+  CRUSH rule 3 x 845 [74,43]
+  CRUSH rule 3 x 846 [43,76]
+  CRUSH rule 3 x 847 [62,20]
+  CRUSH rule 3 x 848 [92,17]
+  CRUSH rule 3 x 849 [93,36]
+  CRUSH rule 3 x 850 [83,82]
+  CRUSH rule 3 x 851 [65,94]
+  CRUSH rule 3 x 852 [60,22]
+  CRUSH rule 3 x 853 [88,29]
+  CRUSH rule 3 x 854 [83,54]
+  CRUSH rule 3 x 855 [2,101]
+  CRUSH rule 3 x 856 [40,41]
+  CRUSH rule 3 x 857 [69,82]
+  CRUSH rule 3 x 858 [98,81]
+  CRUSH rule 3 x 859 [56,43]
+  CRUSH rule 3 x 860 [11,26]
+  CRUSH rule 3 x 861 [22,110]
+  CRUSH rule 3 x 862 [22,70]
+  CRUSH rule 3 x 863 [79,84]
+  CRUSH rule 3 x 864 [77,24]
+  CRUSH rule 3 x 865 [119,17]
+  CRUSH rule 3 x 866 [18,49]
+  CRUSH rule 3 x 867 [3,84]
+  CRUSH rule 3 x 868 [100,107]
+  CRUSH rule 3 x 869 [22,104]
+  CRUSH rule 3 x 870 [73,30]
+  CRUSH rule 3 x 871 [84,105]
+  CRUSH rule 3 x 872 [72,75]
+  CRUSH rule 3 x 873 [81,96]
+  CRUSH rule 3 x 874 [21,72]
+  CRUSH rule 3 x 875 [115,59]
+  CRUSH rule 3 x 876 [98,49]
+  CRUSH rule 3 x 877 [80,79]
+  CRUSH rule 3 x 878 [87,94]
+  CRUSH rule 3 x 879 [29,18]
+  CRUSH rule 3 x 880 [23,40]
+  CRUSH rule 3 x 881 [109,69]
+  CRUSH rule 3 x 882 [31,118]
+  CRUSH rule 3 x 883 [102,8]
+  CRUSH rule 3 x 884 [80,19]
+  CRUSH rule 3 x 885 [46,101]
+  CRUSH rule 3 x 886 [2,65]
+  CRUSH rule 3 x 887 [5,99]
+  CRUSH rule 3 x 888 [16,70]
+  CRUSH rule 3 x 889 [84,93]
+  CRUSH rule 3 x 890 [65,118]
+  CRUSH rule 3 x 891 [86,105]
+  CRUSH rule 3 x 892 [64,10]
+  CRUSH rule 3 x 893 [20,110]
+  CRUSH rule 3 x 894 [32,47]
+  CRUSH rule 3 x 895 [40,21]
+  CRUSH rule 3 x 896 [113,14]
+  CRUSH rule 3 x 897 [107,80]
+  CRUSH rule 3 x 898 [76,71]
+  CRUSH rule 3 x 899 [75,82]
+  CRUSH rule 3 x 900 [83,82]
+  CRUSH rule 3 x 901 [66,61]
+  CRUSH rule 3 x 902 [25,56]
+  CRUSH rule 3 x 903 [53,46]
+  CRUSH rule 3 x 904 [50,101]
+  CRUSH rule 3 x 905 [99,110]
+  CRUSH rule 3 x 906 [68,27]
+  CRUSH rule 3 x 907 [109,47]
+  CRUSH rule 3 x 908 [47,1]
+  CRUSH rule 3 x 909 [73,2]
+  CRUSH rule 3 x 910 [71,74]
+  CRUSH rule 3 x 911 [39,42]
+  CRUSH rule 3 x 912 [90,7]
+  CRUSH rule 3 x 913 [29,96]
+  CRUSH rule 3 x 914 [84,45]
+  CRUSH rule 3 x 915 [49,115]
+  CRUSH rule 3 x 916 [32,77]
+  CRUSH rule 3 x 917 [46,23]
+  CRUSH rule 3 x 918 [82,73]
+  CRUSH rule 3 x 919 [13,28]
+  CRUSH rule 3 x 920 [25,26]
+  CRUSH rule 3 x 921 [55,119]
+  CRUSH rule 3 x 922 [33,32]
+  CRUSH rule 3 x 923 [28,15]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,22]
+  CRUSH rule 3 x 926 [64,69]
+  CRUSH rule 3 x 927 [32,16]
+  CRUSH rule 3 x 928 [13,113]
+  CRUSH rule 3 x 929 [85,115]
+  CRUSH rule 3 x 930 [104,20]
+  CRUSH rule 3 x 931 [46,79]
+  CRUSH rule 3 x 932 [43,52]
+  CRUSH rule 3 x 933 [18,55]
+  CRUSH rule 3 x 934 [68,81]
+  CRUSH rule 3 x 935 [28,57]
+  CRUSH rule 3 x 936 [104,57]
+  CRUSH rule 3 x 937 [110,10]
+  CRUSH rule 3 x 938 [48,23]
+  CRUSH rule 3 x 939 [77,42]
+  CRUSH rule 3 x 940 [76,49]
+  CRUSH rule 3 x 941 [66,101]
+  CRUSH rule 3 x 942 [80,87]
+  CRUSH rule 3 x 943 [75,74]
+  CRUSH rule 3 x 944 [82,53]
+  CRUSH rule 3 x 945 [71,74]
+  CRUSH rule 3 x 946 [37,42]
+  CRUSH rule 3 x 947 [107,30]
+  CRUSH rule 3 x 948 [108,95]
+  CRUSH rule 3 x 949 [46,103]
+  CRUSH rule 3 x 950 [96,101]
+  CRUSH rule 3 x 951 [40,14]
+  CRUSH rule 3 x 952 [114,21]
+  CRUSH rule 3 x 953 [62,23]
+  CRUSH rule 3 x 954 [103,5]
+  CRUSH rule 3 x 955 [42,73]
+  CRUSH rule 3 x 956 [72,103]
+  CRUSH rule 3 x 957 [117,22]
+  CRUSH rule 3 x 958 [23,106]
+  CRUSH rule 3 x 959 [42,93]
+  CRUSH rule 3 x 960 [113,8]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,51]
+  CRUSH rule 3 x 963 [101,106]
+  CRUSH rule 3 x 964 [66,89]
+  CRUSH rule 3 x 965 [47,102]
+  CRUSH rule 3 x 966 [88,63]
+  CRUSH rule 3 x 967 [71,46]
+  CRUSH rule 3 x 968 [74,51]
+  CRUSH rule 3 x 969 [53,78]
+  CRUSH rule 3 x 970 [3,30]
+  CRUSH rule 3 x 971 [66,107]
+  CRUSH rule 3 x 972 [3,66]
+  CRUSH rule 3 x 973 [113,20]
+  CRUSH rule 3 x 974 [114,35]
+  CRUSH rule 3 x 975 [83,58]
+  CRUSH rule 3 x 976 [81,48]
+  CRUSH rule 3 x 977 [95,102]
+  CRUSH rule 3 x 978 [119,41]
+  CRUSH rule 3 x 979 [98,6]
+  CRUSH rule 3 x 980 [39,108]
+  CRUSH rule 3 x 981 [89,84]
+  CRUSH rule 3 x 982 [19,94]
+  CRUSH rule 3 x 983 [34,45]
+  CRUSH rule 3 x 984 [78,63]
+  CRUSH rule 3 x 985 [99,52]
+  CRUSH rule 3 x 986 [44,99]
+  CRUSH rule 3 x 987 [25,32]
+  CRUSH rule 3 x 988 [79,2]
+  CRUSH rule 3 x 989 [87,26]
+  CRUSH rule 3 x 990 [72,69]
+  CRUSH rule 3 x 991 [90,8]
+  CRUSH rule 3 x 992 [30,67]
+  CRUSH rule 3 x 993 [74,49]
+  CRUSH rule 3 x 994 [74,105]
+  CRUSH rule 3 x 995 [100,97]
+  CRUSH rule 3 x 996 [41,58]
+  CRUSH rule 3 x 997 [89,76]
+  CRUSH rule 3 x 998 [92,47]
+  CRUSH rule 3 x 999 [117,16]
+  CRUSH rule 3 x 1000 [50,47]
+  CRUSH rule 3 x 1001 [83,102]
+  CRUSH rule 3 x 1002 [94,37]
+  CRUSH rule 3 x 1003 [43,88]
+  CRUSH rule 3 x 1004 [89,54]
+  CRUSH rule 3 x 1005 [105,84]
+  CRUSH rule 3 x 1006 [45,111]
+  CRUSH rule 3 x 1007 [19,66]
+  CRUSH rule 3 x 1008 [31,76]
+  CRUSH rule 3 x 1009 [1,95]
+  CRUSH rule 3 x 1010 [31,113]
+  CRUSH rule 3 x 1011 [64,81]
+  CRUSH rule 3 x 1012 [68,49]
+  CRUSH rule 3 x 1013 [5,93]
+  CRUSH rule 3 x 1014 [33,66]
+  CRUSH rule 3 x 1015 [106,45]
+  CRUSH rule 3 x 1016 [107,86]
+  CRUSH rule 3 x 1017 [12,61]
+  CRUSH rule 3 x 1018 [61,26]
+  CRUSH rule 3 x 1019 [27,104]
+  CRUSH rule 3 x 1020 [31,86]
+  CRUSH rule 3 x 1021 [22,26]
+  CRUSH rule 3 x 1022 [73,34]
+  CRUSH rule 3 x 1023 [88,79]
+  rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,6]
+  CRUSH rule 3 x 1 [73,52]
+  CRUSH rule 3 x 2 [91,48]
+  CRUSH rule 3 x 3 [51,48]
+  CRUSH rule 3 x 4 [45,114]
+  CRUSH rule 3 x 5 [89,94]
+  CRUSH rule 3 x 6 [91,76]
+  CRUSH rule 3 x 7 [104,73]
+  CRUSH rule 3 x 8 [41,98]
+  CRUSH rule 3 x 9 [46,47]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,40]
+  CRUSH rule 3 x 12 [83,26]
+  CRUSH rule 3 x 13 [27,28]
+  CRUSH rule 3 x 14 [105,64]
+  CRUSH rule 3 x 15 [18,7]
+  CRUSH rule 3 x 16 [103,30]
+  CRUSH rule 3 x 17 [85,118]
+  CRUSH rule 3 x 18 [11,106]
+  CRUSH rule 3 x 19 [75,50]
+  CRUSH rule 3 x 20 [111,67]
+  CRUSH rule 3 x 21 [84,61]
+  CRUSH rule 3 x 22 [23,104]
+  CRUSH rule 3 x 23 [19,86]
+  CRUSH rule 3 x 24 [83,60]
+  CRUSH rule 3 x 25 [81,64]
+  CRUSH rule 3 x 26 [17,38]
+  CRUSH rule 3 x 27 [33,84]
+  CRUSH rule 3 x 28 [45,90]
+  CRUSH rule 3 x 29 [8,109]
+  CRUSH rule 3 x 30 [55,42]
+  CRUSH rule 3 x 31 [76,95]
+  CRUSH rule 3 x 32 [72,11]
+  CRUSH rule 3 x 33 [86,53]
+  CRUSH rule 3 x 34 [7,108]
+  CRUSH rule 3 x 35 [108,13]
+  CRUSH rule 3 x 36 [67,66]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,105]
+  CRUSH rule 3 x 39 [68,103]
+  CRUSH rule 3 x 40 [30,85]
+  CRUSH rule 3 x 41 [52,11]
+  CRUSH rule 3 x 42 [106,75]
+  CRUSH rule 3 x 43 [10,104]
+  CRUSH rule 3 x 44 [101,28]
+  CRUSH rule 3 x 45 [83,64]
+  CRUSH rule 3 x 46 [54,31]
+  CRUSH rule 3 x 47 [106,61]
+  CRUSH rule 3 x 48 [34,41]
+  CRUSH rule 3 x 49 [79,110]
+  CRUSH rule 3 x 50 [42,13]
+  CRUSH rule 3 x 51 [6,94]
+  CRUSH rule 3 x 52 [82,19]
+  CRUSH rule 3 x 53 [32,91]
+  CRUSH rule 3 x 54 [108,8]
+  CRUSH rule 3 x 55 [14,94]
+  CRUSH rule 3 x 56 [21,72]
+  CRUSH rule 3 x 57 [69,88]
+  CRUSH rule 3 x 58 [48,87]
+  CRUSH rule 3 x 59 [21,113]
+  CRUSH rule 3 x 60 [90,73]
+  CRUSH rule 3 x 61 [88,63]
+  CRUSH rule 3 x 62 [100,13]
+  CRUSH rule 3 x 63 [79,5]
+  CRUSH rule 3 x 64 [1,89]
+  CRUSH rule 3 x 65 [32,103]
+  CRUSH rule 3 x 66 [48,79]
+  CRUSH rule 3 x 67 [94,11]
+  CRUSH rule 3 x 68 [102,15]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,11]
+  CRUSH rule 3 x 71 [12,33]
+  CRUSH rule 3 x 72 [26,99]
+  CRUSH rule 3 x 73 [29,114]
+  CRUSH rule 3 x 74 [29,1]
+  CRUSH rule 3 x 75 [60,65]
+  CRUSH rule 3 x 76 [55,62]
+  CRUSH rule 3 x 77 [107,100]
+  CRUSH rule 3 x 78 [86,107]
+  CRUSH rule 3 x 79 [64,16]
+  CRUSH rule 3 x 80 [19,100]
+  CRUSH rule 3 x 81 [64,16]
+  CRUSH rule 3 x 82 [37,40]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,115]
+  CRUSH rule 3 x 85 [87,88]
+  CRUSH rule 3 x 86 [37,68]
+  CRUSH rule 3 x 87 [116,77]
+  CRUSH rule 3 x 88 [38,55]
+  CRUSH rule 3 x 89 [76,25]
+  CRUSH rule 3 x 90 [14,50]
+  CRUSH rule 3 x 91 [68,61]
+  CRUSH rule 3 x 92 [86,95]
+  CRUSH rule 3 x 93 [44,35]
+  CRUSH rule 3 x 94 [46,71]
+  CRUSH rule 3 x 95 [108,53]
+  CRUSH rule 3 x 96 [66,87]
+  CRUSH rule 3 x 97 [111,45]
+  CRUSH rule 3 x 98 [93,110]
+  CRUSH rule 3 x 99 [78,43]
+  CRUSH rule 3 x 100 [28,61]
+  CRUSH rule 3 x 101 [91,110]
+  CRUSH rule 3 x 102 [82,7]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,79]
+  CRUSH rule 3 x 105 [34,87]
+  CRUSH rule 3 x 106 [69,12]
+  CRUSH rule 3 x 107 [1,59]
+  CRUSH rule 3 x 108 [7,109]
+  CRUSH rule 3 x 109 [112,67]
+  CRUSH rule 3 x 110 [54,61]
+  CRUSH rule 3 x 111 [10,92]
+  CRUSH rule 3 x 112 [80,11]
+  CRUSH rule 3 x 113 [69,38]
+  CRUSH rule 3 x 114 [79,38]
+  CRUSH rule 3 x 115 [10,48]
+  CRUSH rule 3 x 116 [37,108]
+  CRUSH rule 3 x 117 [87,56]
+  CRUSH rule 3 x 118 [23,56]
+  CRUSH rule 3 x 119 [104,31]
+  CRUSH rule 3 x 120 [44,93]
+  CRUSH rule 3 x 121 [80,16]
+  CRUSH rule 3 x 122 [45,54]
+  CRUSH rule 3 x 123 [22,112]
+  CRUSH rule 3 x 124 [97,50]
+  CRUSH rule 3 x 125 [66,6]
+  CRUSH rule 3 x 126 [70,39]
+  CRUSH rule 3 x 127 [70,75]
+  CRUSH rule 3 x 128 [11,111]
+  CRUSH rule 3 x 129 [103,46]
+  CRUSH rule 3 x 130 [50,73]
+  CRUSH rule 3 x 131 [44,15]
+  CRUSH rule 3 x 132 [69,58]
+  CRUSH rule 3 x 133 [67,115]
+  CRUSH rule 3 x 134 [37,60]
+  CRUSH rule 3 x 135 [78,61]
+  CRUSH rule 3 x 136 [32,29]
+  CRUSH rule 3 x 137 [92,87]
+  CRUSH rule 3 x 138 [54,8]
+  CRUSH rule 3 x 139 [89,60]
+  CRUSH rule 3 x 140 [39,50]
+  CRUSH rule 3 x 141 [89,62]
+  CRUSH rule 3 x 142 [22,86]
+  CRUSH rule 3 x 143 [96,16]
+  CRUSH rule 3 x 144 [13,1]
+  CRUSH rule 3 x 145 [77,54]
+  CRUSH rule 3 x 146 [12,43]
+  CRUSH rule 3 x 147 [2,59]
+  CRUSH rule 3 x 148 [85,50]
+  CRUSH rule 3 x 149 [103,68]
+  CRUSH rule 3 x 150 [14,50]
+  CRUSH rule 3 x 151 [75,56]
+  CRUSH rule 3 x 152 [49,18]
+  CRUSH rule 3 x 153 [92,79]
+  CRUSH rule 3 x 154 [19,26]
+  CRUSH rule 3 x 155 [12,13]
+  CRUSH rule 3 x 156 [107,18]
+  CRUSH rule 3 x 157 [15,78]
+  CRUSH rule 3 x 158 [11,28]
+  CRUSH rule 3 x 159 [33,88]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,78]
+  CRUSH rule 3 x 162 [55,96]
+  CRUSH rule 3 x 163 [54,55]
+  CRUSH rule 3 x 164 [72,99]
+  CRUSH rule 3 x 165 [25,116]
+  CRUSH rule 3 x 166 [2,23]
+  CRUSH rule 3 x 167 [89,40]
+  CRUSH rule 3 x 168 [68,49]
+  CRUSH rule 3 x 169 [51,50]
+  CRUSH rule 3 x 170 [68,91]
+  CRUSH rule 3 x 171 [88,51]
+  CRUSH rule 3 x 172 [117,23]
+  CRUSH rule 3 x 173 [29,1]
+  CRUSH rule 3 x 174 [67,40]
+  CRUSH rule 3 x 175 [48,41]
+  CRUSH rule 3 x 176 [94,91]
+  CRUSH rule 3 x 177 [53,70]
+  CRUSH rule 3 x 178 [39,110]
+  CRUSH rule 3 x 179 [72,89]
+  CRUSH rule 3 x 180 [3,38]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,46]
+  CRUSH rule 3 x 183 [11,78]
+  CRUSH rule 3 x 184 [79,92]
+  CRUSH rule 3 x 185 [97,92]
+  CRUSH rule 3 x 186 [67,116]
+  CRUSH rule 3 x 187 [6,96]
+  CRUSH rule 3 x 188 [76,16]
+  CRUSH rule 3 x 189 [96,71]
+  CRUSH rule 3 x 190 [90,6]
+  CRUSH rule 3 x 191 [49,84]
+  CRUSH rule 3 x 192 [93,114]
+  CRUSH rule 3 x 193 [89,60]
+  CRUSH rule 3 x 194 [62,61]
+  CRUSH rule 3 x 195 [119,95]
+  CRUSH rule 3 x 196 [20,28]
+  CRUSH rule 3 x 197 [6,64]
+  CRUSH rule 3 x 198 [55,112]
+  CRUSH rule 3 x 199 [66,17]
+  CRUSH rule 3 x 200 [12,63]
+  CRUSH rule 3 x 201 [52,23]
+  CRUSH rule 3 x 202 [98,33]
+  CRUSH rule 3 x 203 [36,22]
+  CRUSH rule 3 x 204 [10,100]
+  CRUSH rule 3 x 205 [38,29]
+  CRUSH rule 3 x 206 [38,27]
+  CRUSH rule 3 x 207 [19,108]
+  CRUSH rule 3 x 208 [63,26]
+  CRUSH rule 3 x 209 [70,11]
+  CRUSH rule 3 x 210 [79,58]
+  CRUSH rule 3 x 211 [26,59]
+  CRUSH rule 3 x 212 [107,114]
+  CRUSH rule 3 x 213 [100,3]
+  CRUSH rule 3 x 214 [91,118]
+  CRUSH rule 3 x 215 [92,16]
+  CRUSH rule 3 x 216 [99,94]
+  CRUSH rule 3 x 217 [86,99]
+  CRUSH rule 3 x 218 [70,15]
+  CRUSH rule 3 x 219 [61,58]
+  CRUSH rule 3 x 220 [23,56]
+  CRUSH rule 3 x 221 [21,92]
+  CRUSH rule 3 x 222 [102,29]
+  CRUSH rule 3 x 223 [34,73]
+  CRUSH rule 3 x 224 [107,68]
+  CRUSH rule 3 x 225 [61,98]
+  CRUSH rule 3 x 226 [44,13]
+  CRUSH rule 3 x 227 [55,60]
+  CRUSH rule 3 x 228 [117,55]
+  CRUSH rule 3 x 229 [100,67]
+  CRUSH rule 3 x 230 [41,109]
+  CRUSH rule 3 x 231 [30,71]
+  CRUSH rule 3 x 232 [23,1]
+  CRUSH rule 3 x 233 [47,90]
+  CRUSH rule 3 x 234 [55,62]
+  CRUSH rule 3 x 235 [20,60]
+  CRUSH rule 3 x 236 [95,24]
+  CRUSH rule 3 x 237 [21,106]
+  CRUSH rule 3 x 238 [109,15]
+  CRUSH rule 3 x 239 [40,101]
+  CRUSH rule 3 x 240 [63,60]
+  CRUSH rule 3 x 241 [47,12]
+  CRUSH rule 3 x 242 [73,74]
+  CRUSH rule 3 x 243 [76,8]
+  CRUSH rule 3 x 244 [103,50]
+  CRUSH rule 3 x 245 [106,95]
+  CRUSH rule 3 x 246 [35,82]
+  CRUSH rule 3 x 247 [116,101]
+  CRUSH rule 3 x 248 [8,119]
+  CRUSH rule 3 x 249 [2,17]
+  CRUSH rule 3 x 250 [34,89]
+  CRUSH rule 3 x 251 [28,69]
+  CRUSH rule 3 x 252 [95,80]
+  CRUSH rule 3 x 253 [109,3]
+  CRUSH rule 3 x 254 [99,80]
+  CRUSH rule 3 x 255 [112,85]
+  CRUSH rule 3 x 256 [94,63]
+  CRUSH rule 3 x 257 [100,87]
+  CRUSH rule 3 x 258 [34,63]
+  CRUSH rule 3 x 259 [70,107]
+  CRUSH rule 3 x 260 [89,115]
+  CRUSH rule 3 x 261 [94,83]
+  CRUSH rule 3 x 262 [42,45]
+  CRUSH rule 3 x 263 [113,101]
+  CRUSH rule 3 x 264 [36,81]
+  CRUSH rule 3 x 265 [14,88]
+  CRUSH rule 3 x 266 [75,96]
+  CRUSH rule 3 x 267 [6,5]
+  CRUSH rule 3 x 268 [38,47]
+  CRUSH rule 3 x 269 [86,59]
+  CRUSH rule 3 x 270 [87,70]
+  CRUSH rule 3 x 271 [19,108]
+  CRUSH rule 3 x 272 [73,5]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,64]
+  CRUSH rule 3 x 275 [29,34]
+  CRUSH rule 3 x 276 [7,100]
+  CRUSH rule 3 x 277 [74,6]
+  CRUSH rule 3 x 278 [107,115]
+  CRUSH rule 3 x 279 [112,20]
+  CRUSH rule 3 x 280 [113,15]
+  CRUSH rule 3 x 281 [89,56]
+  CRUSH rule 3 x 282 [20,38]
+  CRUSH rule 3 x 283 [8,114]
+  CRUSH rule 3 x 284 [66,75]
+  CRUSH rule 3 x 285 [99,94]
+  CRUSH rule 3 x 286 [78,6]
+  CRUSH rule 3 x 287 [12,27]
+  CRUSH rule 3 x 288 [24,22]
+  CRUSH rule 3 x 289 [105,64]
+  CRUSH rule 3 x 290 [25,46]
+  CRUSH rule 3 x 291 [35,116]
+  CRUSH rule 3 x 292 [20,109]
+  CRUSH rule 3 x 293 [27,92]
+  CRUSH rule 3 x 294 [60,93]
+  CRUSH rule 3 x 295 [37,2]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,55]
+  CRUSH rule 3 x 298 [70,53]
+  CRUSH rule 3 x 299 [116,10]
+  CRUSH rule 3 x 300 [67,26]
+  CRUSH rule 3 x 301 [117,23]
+  CRUSH rule 3 x 302 [78,67]
+  CRUSH rule 3 x 303 [19,5]
+  CRUSH rule 3 x 304 [101,50]
+  CRUSH rule 3 x 305 [5,59]
+  CRUSH rule 3 x 306 [41,18]
+  CRUSH rule 3 x 307 [65,5]
+  CRUSH rule 3 x 308 [91,2]
+  CRUSH rule 3 x 309 [38,53]
+  CRUSH rule 3 x 310 [26,15]
+  CRUSH rule 3 x 311 [36,95]
+  CRUSH rule 3 x 312 [114,61]
+  CRUSH rule 3 x 313 [104,65]
+  CRUSH rule 3 x 314 [28,6]
+  CRUSH rule 3 x 315 [118,31]
+  CRUSH rule 3 x 316 [98,95]
+  CRUSH rule 3 x 317 [118,13]
+  CRUSH rule 3 x 318 [17,30]
+  CRUSH rule 3 x 319 [53,1]
+  CRUSH rule 3 x 320 [36,41]
+  CRUSH rule 3 x 321 [33,5]
+  CRUSH rule 3 x 322 [68,10]
+  CRUSH rule 3 x 323 [66,29]
+  CRUSH rule 3 x 324 [21,72]
+  CRUSH rule 3 x 325 [52,16]
+  CRUSH rule 3 x 326 [7,109]
+  CRUSH rule 3 x 327 [62,17]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,100]
+  CRUSH rule 3 x 330 [24,11]
+  CRUSH rule 3 x 331 [84,95]
+  CRUSH rule 3 x 332 [61,111]
+  CRUSH rule 3 x 333 [116,25]
+  CRUSH rule 3 x 334 [94,103]
+  CRUSH rule 3 x 335 [71,118]
+  CRUSH rule 3 x 336 [24,99]
+  CRUSH rule 3 x 337 [18,83]
+  CRUSH rule 3 x 338 [43,28]
+  CRUSH rule 3 x 339 [13,64]
+  CRUSH rule 3 x 340 [81,111]
+  CRUSH rule 3 x 341 [46,105]
+  CRUSH rule 3 x 342 [92,23]
+  CRUSH rule 3 x 343 [49,112]
+  CRUSH rule 3 x 344 [1,87]
+  CRUSH rule 3 x 345 [56,35]
+  CRUSH rule 3 x 346 [3,54]
+  CRUSH rule 3 x 347 [106,27]
+  CRUSH rule 3 x 348 [10,117]
+  CRUSH rule 3 x 349 [96,87]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,41]
+  CRUSH rule 3 x 352 [36,13]
+  CRUSH rule 3 x 353 [10,82]
+  CRUSH rule 3 x 354 [55,52]
+  CRUSH rule 3 x 355 [73,94]
+  CRUSH rule 3 x 356 [75,66]
+  CRUSH rule 3 x 357 [70,93]
+  CRUSH rule 3 x 358 [97,56]
+  CRUSH rule 3 x 359 [110,105]
+  CRUSH rule 3 x 360 [106,57]
+  CRUSH rule 3 x 361 [27,42]
+  CRUSH rule 3 x 362 [28,55]
+  CRUSH rule 3 x 363 [68,20]
+  CRUSH rule 3 x 364 [23,50]
+  CRUSH rule 3 x 365 [57,76]
+  CRUSH rule 3 x 366 [42,75]
+  CRUSH rule 3 x 367 [103,82]
+  CRUSH rule 3 x 368 [103,104]
+  CRUSH rule 3 x 369 [12,57]
+  CRUSH rule 3 x 370 [11,26]
+  CRUSH rule 3 x 371 [34,55]
+  CRUSH rule 3 x 372 [58,14]
+  CRUSH rule 3 x 373 [6,42]
+  CRUSH rule 3 x 374 [110,95]
+  CRUSH rule 3 x 375 [5,43]
+  CRUSH rule 3 x 376 [91,86]
+  CRUSH rule 3 x 377 [93,116]
+  CRUSH rule 3 x 378 [68,6]
+  CRUSH rule 3 x 379 [77,44]
+  CRUSH rule 3 x 380 [76,83]
+  CRUSH rule 3 x 381 [36,27]
+  CRUSH rule 3 x 382 [26,77]
+  CRUSH rule 3 x 383 [76,99]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,93]
+  CRUSH rule 3 x 386 [83,92]
+  CRUSH rule 3 x 387 [16,26]
+  CRUSH rule 3 x 388 [29,113]
+  CRUSH rule 3 x 389 [92,29]
+  CRUSH rule 3 x 390 [68,77]
+  CRUSH rule 3 x 391 [15,88]
+  CRUSH rule 3 x 392 [21,32]
+  CRUSH rule 3 x 393 [91,18]
+  CRUSH rule 3 x 394 [38,73]
+  CRUSH rule 3 x 395 [21,119]
+  CRUSH rule 3 x 396 [12,13]
+  CRUSH rule 3 x 397 [40,63]
+  CRUSH rule 3 x 398 [44,3]
+  CRUSH rule 3 x 399 [5,95]
+  CRUSH rule 3 x 400 [19,102]
+  CRUSH rule 3 x 401 [79,52]
+  CRUSH rule 3 x 402 [107,98]
+  CRUSH rule 3 x 403 [23,82]
+  CRUSH rule 3 x 404 [87,68]
+  CRUSH rule 3 x 405 [90,97]
+  CRUSH rule 3 x 406 [15,117]
+  CRUSH rule 3 x 407 [70,35]
+  CRUSH rule 3 x 408 [55,72]
+  CRUSH rule 3 x 409 [73,62]
+  CRUSH rule 3 x 410 [70,73]
+  CRUSH rule 3 x 411 [34,25]
+  CRUSH rule 3 x 412 [105,117]
+  CRUSH rule 3 x 413 [41,110]
+  CRUSH rule 3 x 414 [70,65]
+  CRUSH rule 3 x 415 [107,5]
+  CRUSH rule 3 x 416 [2,22]
+  CRUSH rule 3 x 417 [26,14]
+  CRUSH rule 3 x 418 [51,46]
+  CRUSH rule 3 x 419 [8,82]
+  CRUSH rule 3 x 420 [109,105]
+  CRUSH rule 3 x 421 [114,75]
+  CRUSH rule 3 x 422 [109,87]
+  CRUSH rule 3 x 423 [59,24]
+  CRUSH rule 3 x 424 [92,51]
+  CRUSH rule 3 x 425 [101,111]
+  CRUSH rule 3 x 426 [36,6]
+  CRUSH rule 3 x 427 [8,24]
+  CRUSH rule 3 x 428 [68,35]
+  CRUSH rule 3 x 429 [76,75]
+  CRUSH rule 3 x 430 [67,117]
+  CRUSH rule 3 x 431 [70,25]
+  CRUSH rule 3 x 432 [7,34]
+  CRUSH rule 3 x 433 [49,84]
+  CRUSH rule 3 x 434 [64,31]
+  CRUSH rule 3 x 435 [110,13]
+  CRUSH rule 3 x 436 [106,89]
+  CRUSH rule 3 x 437 [26,65]
+  CRUSH rule 3 x 438 [118,63]
+  CRUSH rule 3 x 439 [40,21]
+  CRUSH rule 3 x 440 [45,119]
+  CRUSH rule 3 x 441 [112,105]
+  CRUSH rule 3 x 442 [55,113]
+  CRUSH rule 3 x 443 [44,33]
+  CRUSH rule 3 x 444 [71,38]
+  CRUSH rule 3 x 445 [58,81]
+  CRUSH rule 3 x 446 [40,10]
+  CRUSH rule 3 x 447 [100,61]
+  CRUSH rule 3 x 448 [111,73]
+  CRUSH rule 3 x 449 [67,66]
+  CRUSH rule 3 x 450 [117,61]
+  CRUSH rule 3 x 451 [66,81]
+  CRUSH rule 3 x 452 [70,8]
+  CRUSH rule 3 x 453 [82,85]
+  CRUSH rule 3 x 454 [53,18]
+  CRUSH rule 3 x 455 [91,42]
+  CRUSH rule 3 x 456 [101,46]
+  CRUSH rule 3 x 457 [113,51]
+  CRUSH rule 3 x 458 [119,25]
+  CRUSH rule 3 x 459 [50,67]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,51]
+  CRUSH rule 3 x 462 [98,107]
+  CRUSH rule 3 x 463 [108,105]
+  CRUSH rule 3 x 464 [19,109]
+  CRUSH rule 3 x 465 [62,23]
+  CRUSH rule 3 x 466 [53,12]
+  CRUSH rule 3 x 467 [40,57]
+  CRUSH rule 3 x 468 [97,44]
+  CRUSH rule 3 x 469 [98,75]
+  CRUSH rule 3 x 470 [50,29]
+  CRUSH rule 3 x 471 [40,13]
+  CRUSH rule 3 x 472 [27,18]
+  CRUSH rule 3 x 473 [48,35]
+  CRUSH rule 3 x 474 [51,32]
+  CRUSH rule 3 x 475 [49,117]
+  CRUSH rule 3 x 476 [110,31]
+  CRUSH rule 3 x 477 [80,97]
+  CRUSH rule 3 x 478 [78,99]
+  CRUSH rule 3 x 479 [31,66]
+  CRUSH rule 3 x 480 [75,88]
+  CRUSH rule 3 x 481 [26,20]
+  CRUSH rule 3 x 482 [84,53]
+  CRUSH rule 3 x 483 [15,116]
+  CRUSH rule 3 x 484 [37,114]
+  CRUSH rule 3 x 485 [84,8]
+  CRUSH rule 3 x 486 [92,10]
+  CRUSH rule 3 x 487 [106,17]
+  CRUSH rule 3 x 488 [42,20]
+  CRUSH rule 3 x 489 [89,2]
+  CRUSH rule 3 x 490 [22,114]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,66]
+  CRUSH rule 3 x 493 [94,14]
+  CRUSH rule 3 x 494 [59,86]
+  CRUSH rule 3 x 495 [95,58]
+  CRUSH rule 3 x 496 [46,41]
+  CRUSH rule 3 x 497 [102,27]
+  CRUSH rule 3 x 498 [21,116]
+  CRUSH rule 3 x 499 [5,49]
+  CRUSH rule 3 x 500 [50,49]
+  CRUSH rule 3 x 501 [60,3]
+  CRUSH rule 3 x 502 [65,110]
+  CRUSH rule 3 x 503 [21,112]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,93]
+  CRUSH rule 3 x 506 [79,64]
+  CRUSH rule 3 x 507 [34,107]
+  CRUSH rule 3 x 508 [45,114]
+  CRUSH rule 3 x 509 [19,88]
+  CRUSH rule 3 x 510 [117,45]
+  CRUSH rule 3 x 511 [14,104]
+  CRUSH rule 3 x 512 [59,26]
+  CRUSH rule 3 x 513 [102,93]
+  CRUSH rule 3 x 514 [75,72]
+  CRUSH rule 3 x 515 [84,41]
+  CRUSH rule 3 x 516 [37,30]
+  CRUSH rule 3 x 517 [83,115]
+  CRUSH rule 3 x 518 [18,83]
+  CRUSH rule 3 x 519 [67,88]
+  CRUSH rule 3 x 520 [15,114]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,51]
+  CRUSH rule 3 x 523 [68,101]
+  CRUSH rule 3 x 524 [33,38]
+  CRUSH rule 3 x 525 [63,115]
+  CRUSH rule 3 x 526 [83,50]
+  CRUSH rule 3 x 527 [37,56]
+  CRUSH rule 3 x 528 [108,81]
+  CRUSH rule 3 x 529 [74,33]
+  CRUSH rule 3 x 530 [49,92]
+  CRUSH rule 3 x 531 [117,105]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,85]
+  CRUSH rule 3 x 534 [97,24]
+  CRUSH rule 3 x 535 [48,75]
+  CRUSH rule 3 x 536 [113,101]
+  CRUSH rule 3 x 537 [116,47]
+  CRUSH rule 3 x 538 [85,74]
+  CRUSH rule 3 x 539 [72,43]
+  CRUSH rule 3 x 540 [39,34]
+  CRUSH rule 3 x 541 [53,84]
+  CRUSH rule 3 x 542 [27,32]
+  CRUSH rule 3 x 543 [45,113]
+  CRUSH rule 3 x 544 [59,42]
+  CRUSH rule 3 x 545 [118,95]
+  CRUSH rule 3 x 546 [18,79]
+  CRUSH rule 3 x 547 [67,30]
+  CRUSH rule 3 x 548 [53,100]
+  CRUSH rule 3 x 549 [60,45]
+  CRUSH rule 3 x 550 [92,101]
+  CRUSH rule 3 x 551 [77,88]
+  CRUSH rule 3 x 552 [61,94]
+  CRUSH rule 3 x 553 [71,78]
+  CRUSH rule 3 x 554 [61,115]
+  CRUSH rule 3 x 555 [76,77]
+  CRUSH rule 3 x 556 [106,55]
+  CRUSH rule 3 x 557 [26,22]
+  CRUSH rule 3 x 558 [41,84]
+  CRUSH rule 3 x 559 [65,24]
+  CRUSH rule 3 x 560 [94,16]
+  CRUSH rule 3 x 561 [27,5]
+  CRUSH rule 3 x 562 [78,59]
+  CRUSH rule 3 x 563 [59,70]
+  CRUSH rule 3 x 564 [96,8]
+  CRUSH rule 3 x 565 [8,48]
+  CRUSH rule 3 x 566 [119,17]
+  CRUSH rule 3 x 567 [7,38]
+  CRUSH rule 3 x 568 [57,94]
+  CRUSH rule 3 x 569 [65,26]
+  CRUSH rule 3 x 570 [98,27]
+  CRUSH rule 3 x 571 [95,30]
+  CRUSH rule 3 x 572 [62,83]
+  CRUSH rule 3 x 573 [1,79]
+  CRUSH rule 3 x 574 [89,42]
+  CRUSH rule 3 x 575 [87,113]
+  CRUSH rule 3 x 576 [21,68]
+  CRUSH rule 3 x 577 [8,84]
+  CRUSH rule 3 x 578 [75,115]
+  CRUSH rule 3 x 579 [105,68]
+  CRUSH rule 3 x 580 [51,28]
+  CRUSH rule 3 x 581 [55,113]
+  CRUSH rule 3 x 582 [27,113]
+  CRUSH rule 3 x 583 [6,78]
+  CRUSH rule 3 x 584 [10,30]
+  CRUSH rule 3 x 585 [20,111]
+  CRUSH rule 3 x 586 [48,27]
+  CRUSH rule 3 x 587 [29,94]
+  CRUSH rule 3 x 588 [103,90]
+  CRUSH rule 3 x 589 [88,95]
+  CRUSH rule 3 x 590 [76,101]
+  CRUSH rule 3 x 591 [42,43]
+  CRUSH rule 3 x 592 [78,51]
+  CRUSH rule 3 x 593 [82,71]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,59]
+  CRUSH rule 3 x 597 [16,36]
+  CRUSH rule 3 x 598 [37,56]
+  CRUSH rule 3 x 599 [10,84]
+  CRUSH rule 3 x 600 [24,69]
+  CRUSH rule 3 x 601 [104,14]
+  CRUSH rule 3 x 602 [48,45]
+  CRUSH rule 3 x 603 [93,32]
+  CRUSH rule 3 x 604 [118,79]
+  CRUSH rule 3 x 605 [104,53]
+  CRUSH rule 3 x 606 [90,83]
+  CRUSH rule 3 x 607 [95,110]
+  CRUSH rule 3 x 608 [112,101]
+  CRUSH rule 3 x 609 [34,99]
+  CRUSH rule 3 x 610 [106,16]
+  CRUSH rule 3 x 611 [66,87]
+  CRUSH rule 3 x 612 [2,81]
+  CRUSH rule 3 x 613 [13,86]
+  CRUSH rule 3 x 614 [50,3]
+  CRUSH rule 3 x 615 [24,73]
+  CRUSH rule 3 x 616 [41,119]
+  CRUSH rule 3 x 617 [81,106]
+  CRUSH rule 3 x 618 [3,104]
+  CRUSH rule 3 x 619 [92,7]
+  CRUSH rule 3 x 620 [108,11]
+  CRUSH rule 3 x 621 [105,115]
+  CRUSH rule 3 x 622 [67,48]
+  CRUSH rule 3 x 623 [69,74]
+  CRUSH rule 3 x 624 [115,49]
+  CRUSH rule 3 x 625 [73,109]
+  CRUSH rule 3 x 626 [52,3]
+  CRUSH rule 3 x 627 [116,3]
+  CRUSH rule 3 x 628 [98,91]
+  CRUSH rule 3 x 629 [6,112]
+  CRUSH rule 3 x 630 [22,72]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,71]
+  CRUSH rule 3 x 633 [65,12]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,46]
+  CRUSH rule 3 x 636 [23,70]
+  CRUSH rule 3 x 637 [99,24]
+  CRUSH rule 3 x 638 [43,114]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,73]
+  CRUSH rule 3 x 641 [45,84]
+  CRUSH rule 3 x 642 [47,66]
+  CRUSH rule 3 x 643 [64,8]
+  CRUSH rule 3 x 644 [31,82]
+  CRUSH rule 3 x 645 [77,64]
+  CRUSH rule 3 x 646 [37,86]
+  CRUSH rule 3 x 647 [65,56]
+  CRUSH rule 3 x 648 [84,13]
+  CRUSH rule 3 x 649 [88,55]
+  CRUSH rule 3 x 650 [21,76]
+  CRUSH rule 3 x 651 [63,116]
+  CRUSH rule 3 x 652 [57,112]
+  CRUSH rule 3 x 653 [38,61]
+  CRUSH rule 3 x 654 [104,67]
+  CRUSH rule 3 x 655 [89,54]
+  CRUSH rule 3 x 656 [84,49]
+  CRUSH rule 3 x 657 [47,32]
+  CRUSH rule 3 x 658 [80,29]
+  CRUSH rule 3 x 659 [11,112]
+  CRUSH rule 3 x 660 [65,111]
+  CRUSH rule 3 x 661 [96,73]
+  CRUSH rule 3 x 662 [111,73]
+  CRUSH rule 3 x 663 [83,60]
+  CRUSH rule 3 x 664 [59,80]
+  CRUSH rule 3 x 665 [31,117]
+  CRUSH rule 3 x 666 [112,101]
+  CRUSH rule 3 x 667 [70,47]
+  CRUSH rule 3 x 668 [96,57]
+  CRUSH rule 3 x 669 [56,39]
+  CRUSH rule 3 x 670 [98,105]
+  CRUSH rule 3 x 671 [57,48]
+  CRUSH rule 3 x 672 [37,36]
+  CRUSH rule 3 x 673 [83,2]
+  CRUSH rule 3 x 674 [36,25]
+  CRUSH rule 3 x 675 [88,14]
+  CRUSH rule 3 x 676 [3,110]
+  CRUSH rule 3 x 677 [88,67]
+  CRUSH rule 3 x 678 [27,44]
+  CRUSH rule 3 x 679 [33,116]
+  CRUSH rule 3 x 680 [111,39]
+  CRUSH rule 3 x 681 [53,12]
+  CRUSH rule 3 x 682 [12,87]
+  CRUSH rule 3 x 683 [24,85]
+  CRUSH rule 3 x 684 [98,65]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,72]
+  CRUSH rule 3 x 688 [16,114]
+  CRUSH rule 3 x 689 [32,31]
+  CRUSH rule 3 x 690 [96,33]
+  CRUSH rule 3 x 691 [34,6]
+  CRUSH rule 3 x 692 [97,84]
+  CRUSH rule 3 x 693 [29,118]
+  CRUSH rule 3 x 694 [6,30]
+  CRUSH rule 3 x 695 [31,72]
+  CRUSH rule 3 x 696 [104,97]
+  CRUSH rule 3 x 697 [19,96]
+  CRUSH rule 3 x 698 [30,69]
+  CRUSH rule 3 x 699 [47,76]
+  CRUSH rule 3 x 700 [82,55]
+  CRUSH rule 3 x 701 [53,80]
+  CRUSH rule 3 x 702 [95,98]
+  CRUSH rule 3 x 703 [92,65]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,1]
+  CRUSH rule 3 x 706 [74,35]
+  CRUSH rule 3 x 707 [91,115]
+  CRUSH rule 3 x 708 [95,112]
+  CRUSH rule 3 x 709 [73,72]
+  CRUSH rule 3 x 710 [94,47]
+  CRUSH rule 3 x 711 [68,41]
+  CRUSH rule 3 x 712 [107,18]
+  CRUSH rule 3 x 713 [29,109]
+  CRUSH rule 3 x 714 [86,61]
+  CRUSH rule 3 x 715 [74,13]
+  CRUSH rule 3 x 716 [101,56]
+  CRUSH rule 3 x 717 [12,29]
+  CRUSH rule 3 x 718 [83,24]
+  CRUSH rule 3 x 719 [26,10]
+  CRUSH rule 3 x 720 [69,2]
+  CRUSH rule 3 x 721 [51,42]
+  CRUSH rule 3 x 722 [15,74]
+  CRUSH rule 3 x 723 [117,14]
+  CRUSH rule 3 x 724 [45,38]
+  CRUSH rule 3 x 725 [53,110]
+  CRUSH rule 3 x 726 [103,68]
+  CRUSH rule 3 x 727 [89,100]
+  CRUSH rule 3 x 728 [76,16]
+  CRUSH rule 3 x 729 [35,90]
+  CRUSH rule 3 x 730 [28,103]
+  CRUSH rule 3 x 731 [78,41]
+  CRUSH rule 3 x 732 [1,27]
+  CRUSH rule 3 x 733 [35,100]
+  CRUSH rule 3 x 734 [119,85]
+  CRUSH rule 3 x 735 [102,43]
+  CRUSH rule 3 x 736 [37,92]
+  CRUSH rule 3 x 737 [117,11]
+  CRUSH rule 3 x 738 [57,32]
+  CRUSH rule 3 x 739 [87,1]
+  CRUSH rule 3 x 740 [29,80]
+  CRUSH rule 3 x 741 [47,111]
+  CRUSH rule 3 x 742 [106,83]
+  CRUSH rule 3 x 743 [105,94]
+  CRUSH rule 3 x 744 [23,64]
+  CRUSH rule 3 x 745 [37,112]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,95]
+  CRUSH rule 3 x 748 [48,14]
+  CRUSH rule 3 x 749 [102,101]
+  CRUSH rule 3 x 750 [83,78]
+  CRUSH rule 3 x 751 [25,104]
+  CRUSH rule 3 x 752 [82,95]
+  CRUSH rule 3 x 753 [14,113]
+  CRUSH rule 3 x 754 [114,51]
+  CRUSH rule 3 x 755 [87,26]
+  CRUSH rule 3 x 756 [113,87]
+  CRUSH rule 3 x 757 [47,66]
+  CRUSH rule 3 x 758 [54,63]
+  CRUSH rule 3 x 759 [74,20]
+  CRUSH rule 3 x 760 [88,22]
+  CRUSH rule 3 x 761 [73,86]
+  CRUSH rule 3 x 762 [34,17]
+  CRUSH rule 3 x 763 [13,78]
+  CRUSH rule 3 x 764 [89,42]
+  CRUSH rule 3 x 765 [109,91]
+  CRUSH rule 3 x 766 [19,66]
+  CRUSH rule 3 x 767 [41,26]
+  CRUSH rule 3 x 768 [106,57]
+  CRUSH rule 3 x 769 [91,104]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,35]
+  CRUSH rule 3 x 772 [97,108]
+  CRUSH rule 3 x 773 [116,47]
+  CRUSH rule 3 x 774 [100,31]
+  CRUSH rule 3 x 775 [102,43]
+  CRUSH rule 3 x 776 [69,38]
+  CRUSH rule 3 x 777 [91,52]
+  CRUSH rule 3 x 778 [83,119]
+  CRUSH rule 3 x 779 [47,60]
+  CRUSH rule 3 x 780 [63,70]
+  CRUSH rule 3 x 781 [105,2]
+  CRUSH rule 3 x 782 [117,59]
+  CRUSH rule 3 x 783 [19,109]
+  CRUSH rule 3 x 784 [63,114]
+  CRUSH rule 3 x 785 [27,84]
+  CRUSH rule 3 x 786 [41,110]
+  CRUSH rule 3 x 787 [108,73]
+  CRUSH rule 3 x 788 [74,103]
+  CRUSH rule 3 x 789 [50,17]
+  CRUSH rule 3 x 790 [20,106]
+  CRUSH rule 3 x 791 [96,87]
+  CRUSH rule 3 x 792 [80,97]
+  CRUSH rule 3 x 793 [6,26]
+  CRUSH rule 3 x 794 [14,42]
+  CRUSH rule 3 x 795 [30,8]
+  CRUSH rule 3 x 796 [87,36]
+  CRUSH rule 3 x 797 [64,61]
+  CRUSH rule 3 x 798 [42,69]
+  CRUSH rule 3 x 799 [19,117]
+  CRUSH rule 3 x 800 [106,8]
+  CRUSH rule 3 x 801 [2,57]
+  CRUSH rule 3 x 802 [63,68]
+  CRUSH rule 3 x 803 [46,35]
+  CRUSH rule 3 x 804 [33,26]
+  CRUSH rule 3 x 805 [96,49]
+  CRUSH rule 3 x 806 [48,25]
+  CRUSH rule 3 x 807 [48,83]
+  CRUSH rule 3 x 808 [76,31]
+  CRUSH rule 3 x 809 [27,48]
+  CRUSH rule 3 x 810 [119,71]
+  CRUSH rule 3 x 811 [111,91]
+  CRUSH rule 3 x 812 [25,111]
+  CRUSH rule 3 x 813 [81,28]
+  CRUSH rule 3 x 814 [95,42]
+  CRUSH rule 3 x 815 [84,61]
+  CRUSH rule 3 x 816 [64,35]
+  CRUSH rule 3 x 817 [63,60]
+  CRUSH rule 3 x 818 [69,46]
+  CRUSH rule 3 x 819 [88,75]
+  CRUSH rule 3 x 820 [104,57]
+  CRUSH rule 3 x 821 [58,21]
+  CRUSH rule 3 x 822 [20,80]
+  CRUSH rule 3 x 823 [63,118]
+  CRUSH rule 3 x 824 [102,13]
+  CRUSH rule 3 x 825 [47,118]
+  CRUSH rule 3 x 826 [44,7]
+  CRUSH rule 3 x 827 [101,88]
+  CRUSH rule 3 x 828 [60,41]
+  CRUSH rule 3 x 829 [45,102]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,75]
+  CRUSH rule 3 x 833 [57,32]
+  CRUSH rule 3 x 834 [90,33]
+  CRUSH rule 3 x 835 [6,1]
+  CRUSH rule 3 x 836 [63,68]
+  CRUSH rule 3 x 837 [76,71]
+  CRUSH rule 3 x 838 [106,20]
+  CRUSH rule 3 x 839 [87,96]
+  CRUSH rule 3 x 840 [33,32]
+  CRUSH rule 3 x 841 [110,55]
+  CRUSH rule 3 x 842 [66,87]
+  CRUSH rule 3 x 843 [11,80]
+  CRUSH rule 3 x 844 [74,103]
+  CRUSH rule 3 x 845 [74,43]
+  CRUSH rule 3 x 846 [43,76]
+  CRUSH rule 3 x 847 [62,20]
+  CRUSH rule 3 x 848 [92,17]
+  CRUSH rule 3 x 849 [93,36]
+  CRUSH rule 3 x 850 [83,82]
+  CRUSH rule 3 x 851 [65,94]
+  CRUSH rule 3 x 852 [60,22]
+  CRUSH rule 3 x 853 [88,29]
+  CRUSH rule 3 x 854 [83,54]
+  CRUSH rule 3 x 855 [2,101]
+  CRUSH rule 3 x 856 [40,41]
+  CRUSH rule 3 x 857 [69,82]
+  CRUSH rule 3 x 858 [98,81]
+  CRUSH rule 3 x 859 [56,43]
+  CRUSH rule 3 x 860 [11,26]
+  CRUSH rule 3 x 861 [22,110]
+  CRUSH rule 3 x 862 [22,70]
+  CRUSH rule 3 x 863 [79,84]
+  CRUSH rule 3 x 864 [77,24]
+  CRUSH rule 3 x 865 [119,17]
+  CRUSH rule 3 x 866 [18,49]
+  CRUSH rule 3 x 867 [3,84]
+  CRUSH rule 3 x 868 [100,107]
+  CRUSH rule 3 x 869 [22,104]
+  CRUSH rule 3 x 870 [73,30]
+  CRUSH rule 3 x 871 [84,105]
+  CRUSH rule 3 x 872 [72,75]
+  CRUSH rule 3 x 873 [81,96]
+  CRUSH rule 3 x 874 [21,72]
+  CRUSH rule 3 x 875 [115,59]
+  CRUSH rule 3 x 876 [98,49]
+  CRUSH rule 3 x 877 [80,79]
+  CRUSH rule 3 x 878 [87,94]
+  CRUSH rule 3 x 879 [29,18]
+  CRUSH rule 3 x 880 [23,40]
+  CRUSH rule 3 x 881 [109,69]
+  CRUSH rule 3 x 882 [31,118]
+  CRUSH rule 3 x 883 [102,8]
+  CRUSH rule 3 x 884 [80,19]
+  CRUSH rule 3 x 885 [46,101]
+  CRUSH rule 3 x 886 [2,65]
+  CRUSH rule 3 x 887 [5,99]
+  CRUSH rule 3 x 888 [16,70]
+  CRUSH rule 3 x 889 [84,93]
+  CRUSH rule 3 x 890 [65,118]
+  CRUSH rule 3 x 891 [86,105]
+  CRUSH rule 3 x 892 [64,10]
+  CRUSH rule 3 x 893 [20,110]
+  CRUSH rule 3 x 894 [32,47]
+  CRUSH rule 3 x 895 [40,21]
+  CRUSH rule 3 x 896 [113,14]
+  CRUSH rule 3 x 897 [107,80]
+  CRUSH rule 3 x 898 [76,71]
+  CRUSH rule 3 x 899 [75,82]
+  CRUSH rule 3 x 900 [83,82]
+  CRUSH rule 3 x 901 [66,61]
+  CRUSH rule 3 x 902 [25,56]
+  CRUSH rule 3 x 903 [53,46]
+  CRUSH rule 3 x 904 [50,101]
+  CRUSH rule 3 x 905 [99,110]
+  CRUSH rule 3 x 906 [68,27]
+  CRUSH rule 3 x 907 [109,47]
+  CRUSH rule 3 x 908 [47,1]
+  CRUSH rule 3 x 909 [73,2]
+  CRUSH rule 3 x 910 [71,74]
+  CRUSH rule 3 x 911 [39,42]
+  CRUSH rule 3 x 912 [90,7]
+  CRUSH rule 3 x 913 [29,96]
+  CRUSH rule 3 x 914 [84,45]
+  CRUSH rule 3 x 915 [49,115]
+  CRUSH rule 3 x 916 [32,77]
+  CRUSH rule 3 x 917 [46,23]
+  CRUSH rule 3 x 918 [82,73]
+  CRUSH rule 3 x 919 [13,28]
+  CRUSH rule 3 x 920 [25,26]
+  CRUSH rule 3 x 921 [55,119]
+  CRUSH rule 3 x 922 [33,32]
+  CRUSH rule 3 x 923 [28,15]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,22]
+  CRUSH rule 3 x 926 [64,69]
+  CRUSH rule 3 x 927 [32,16]
+  CRUSH rule 3 x 928 [13,113]
+  CRUSH rule 3 x 929 [85,115]
+  CRUSH rule 3 x 930 [104,20]
+  CRUSH rule 3 x 931 [46,79]
+  CRUSH rule 3 x 932 [43,52]
+  CRUSH rule 3 x 933 [18,55]
+  CRUSH rule 3 x 934 [68,81]
+  CRUSH rule 3 x 935 [28,57]
+  CRUSH rule 3 x 936 [104,57]
+  CRUSH rule 3 x 937 [110,10]
+  CRUSH rule 3 x 938 [48,23]
+  CRUSH rule 3 x 939 [77,42]
+  CRUSH rule 3 x 940 [76,49]
+  CRUSH rule 3 x 941 [66,101]
+  CRUSH rule 3 x 942 [80,87]
+  CRUSH rule 3 x 943 [75,74]
+  CRUSH rule 3 x 944 [82,53]
+  CRUSH rule 3 x 945 [71,74]
+  CRUSH rule 3 x 946 [37,42]
+  CRUSH rule 3 x 947 [107,30]
+  CRUSH rule 3 x 948 [108,95]
+  CRUSH rule 3 x 949 [46,103]
+  CRUSH rule 3 x 950 [96,101]
+  CRUSH rule 3 x 951 [40,14]
+  CRUSH rule 3 x 952 [114,21]
+  CRUSH rule 3 x 953 [62,23]
+  CRUSH rule 3 x 954 [103,5]
+  CRUSH rule 3 x 955 [42,73]
+  CRUSH rule 3 x 956 [72,103]
+  CRUSH rule 3 x 957 [117,22]
+  CRUSH rule 3 x 958 [23,106]
+  CRUSH rule 3 x 959 [42,93]
+  CRUSH rule 3 x 960 [113,8]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,51]
+  CRUSH rule 3 x 963 [101,106]
+  CRUSH rule 3 x 964 [66,89]
+  CRUSH rule 3 x 965 [47,102]
+  CRUSH rule 3 x 966 [88,63]
+  CRUSH rule 3 x 967 [71,46]
+  CRUSH rule 3 x 968 [74,51]
+  CRUSH rule 3 x 969 [53,78]
+  CRUSH rule 3 x 970 [3,30]
+  CRUSH rule 3 x 971 [66,107]
+  CRUSH rule 3 x 972 [3,66]
+  CRUSH rule 3 x 973 [113,20]
+  CRUSH rule 3 x 974 [114,35]
+  CRUSH rule 3 x 975 [83,58]
+  CRUSH rule 3 x 976 [81,48]
+  CRUSH rule 3 x 977 [95,102]
+  CRUSH rule 3 x 978 [119,41]
+  CRUSH rule 3 x 979 [98,6]
+  CRUSH rule 3 x 980 [39,108]
+  CRUSH rule 3 x 981 [89,84]
+  CRUSH rule 3 x 982 [19,94]
+  CRUSH rule 3 x 983 [34,45]
+  CRUSH rule 3 x 984 [78,63]
+  CRUSH rule 3 x 985 [99,52]
+  CRUSH rule 3 x 986 [44,99]
+  CRUSH rule 3 x 987 [25,32]
+  CRUSH rule 3 x 988 [79,2]
+  CRUSH rule 3 x 989 [87,26]
+  CRUSH rule 3 x 990 [72,69]
+  CRUSH rule 3 x 991 [90,8]
+  CRUSH rule 3 x 992 [30,67]
+  CRUSH rule 3 x 993 [74,49]
+  CRUSH rule 3 x 994 [74,105]
+  CRUSH rule 3 x 995 [100,97]
+  CRUSH rule 3 x 996 [41,58]
+  CRUSH rule 3 x 997 [89,76]
+  CRUSH rule 3 x 998 [92,47]
+  CRUSH rule 3 x 999 [117,16]
+  CRUSH rule 3 x 1000 [50,47]
+  CRUSH rule 3 x 1001 [83,102]
+  CRUSH rule 3 x 1002 [94,37]
+  CRUSH rule 3 x 1003 [43,88]
+  CRUSH rule 3 x 1004 [89,54]
+  CRUSH rule 3 x 1005 [105,84]
+  CRUSH rule 3 x 1006 [45,111]
+  CRUSH rule 3 x 1007 [19,66]
+  CRUSH rule 3 x 1008 [31,76]
+  CRUSH rule 3 x 1009 [1,95]
+  CRUSH rule 3 x 1010 [31,113]
+  CRUSH rule 3 x 1011 [64,81]
+  CRUSH rule 3 x 1012 [68,49]
+  CRUSH rule 3 x 1013 [5,93]
+  CRUSH rule 3 x 1014 [33,66]
+  CRUSH rule 3 x 1015 [106,45]
+  CRUSH rule 3 x 1016 [107,86]
+  CRUSH rule 3 x 1017 [12,61]
+  CRUSH rule 3 x 1018 [61,26]
+  CRUSH rule 3 x 1019 [27,104]
+  CRUSH rule 3 x 1020 [31,86]
+  CRUSH rule 3 x 1021 [22,26]
+  CRUSH rule 3 x 1022 [73,34]
+  CRUSH rule 3 x 1023 [88,79]
+  rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,6]
+  CRUSH rule 3 x 1 [73,52]
+  CRUSH rule 3 x 2 [91,48]
+  CRUSH rule 3 x 3 [51,48]
+  CRUSH rule 3 x 4 [45,114]
+  CRUSH rule 3 x 5 [89,94]
+  CRUSH rule 3 x 6 [91,76]
+  CRUSH rule 3 x 7 [104,73]
+  CRUSH rule 3 x 8 [41,98]
+  CRUSH rule 3 x 9 [46,47]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,40]
+  CRUSH rule 3 x 12 [83,26]
+  CRUSH rule 3 x 13 [27,28]
+  CRUSH rule 3 x 14 [105,64]
+  CRUSH rule 3 x 15 [18,7]
+  CRUSH rule 3 x 16 [103,30]
+  CRUSH rule 3 x 17 [85,118]
+  CRUSH rule 3 x 18 [11,106]
+  CRUSH rule 3 x 19 [75,50]
+  CRUSH rule 3 x 20 [111,67]
+  CRUSH rule 3 x 21 [84,61]
+  CRUSH rule 3 x 22 [23,104]
+  CRUSH rule 3 x 23 [19,86]
+  CRUSH rule 3 x 24 [83,60]
+  CRUSH rule 3 x 25 [81,64]
+  CRUSH rule 3 x 26 [17,38]
+  CRUSH rule 3 x 27 [33,84]
+  CRUSH rule 3 x 28 [45,90]
+  CRUSH rule 3 x 29 [8,109]
+  CRUSH rule 3 x 30 [55,42]
+  CRUSH rule 3 x 31 [76,95]
+  CRUSH rule 3 x 32 [72,11]
+  CRUSH rule 3 x 33 [86,53]
+  CRUSH rule 3 x 34 [7,108]
+  CRUSH rule 3 x 35 [108,13]
+  CRUSH rule 3 x 36 [67,66]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,105]
+  CRUSH rule 3 x 39 [68,103]
+  CRUSH rule 3 x 40 [30,85]
+  CRUSH rule 3 x 41 [52,11]
+  CRUSH rule 3 x 42 [106,75]
+  CRUSH rule 3 x 43 [10,104]
+  CRUSH rule 3 x 44 [101,28]
+  CRUSH rule 3 x 45 [83,64]
+  CRUSH rule 3 x 46 [54,31]
+  CRUSH rule 3 x 47 [106,61]
+  CRUSH rule 3 x 48 [34,41]
+  CRUSH rule 3 x 49 [79,110]
+  CRUSH rule 3 x 50 [42,13]
+  CRUSH rule 3 x 51 [6,94]
+  CRUSH rule 3 x 52 [82,19]
+  CRUSH rule 3 x 53 [32,91]
+  CRUSH rule 3 x 54 [108,8]
+  CRUSH rule 3 x 55 [14,94]
+  CRUSH rule 3 x 56 [21,72]
+  CRUSH rule 3 x 57 [69,88]
+  CRUSH rule 3 x 58 [48,87]
+  CRUSH rule 3 x 59 [21,113]
+  CRUSH rule 3 x 60 [90,73]
+  CRUSH rule 3 x 61 [88,63]
+  CRUSH rule 3 x 62 [100,13]
+  CRUSH rule 3 x 63 [79,5]
+  CRUSH rule 3 x 64 [1,89]
+  CRUSH rule 3 x 65 [32,103]
+  CRUSH rule 3 x 66 [48,79]
+  CRUSH rule 3 x 67 [94,11]
+  CRUSH rule 3 x 68 [102,15]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,11]
+  CRUSH rule 3 x 71 [12,33]
+  CRUSH rule 3 x 72 [26,99]
+  CRUSH rule 3 x 73 [29,114]
+  CRUSH rule 3 x 74 [29,1]
+  CRUSH rule 3 x 75 [60,65]
+  CRUSH rule 3 x 76 [55,62]
+  CRUSH rule 3 x 77 [107,100]
+  CRUSH rule 3 x 78 [86,107]
+  CRUSH rule 3 x 79 [64,16]
+  CRUSH rule 3 x 80 [19,100]
+  CRUSH rule 3 x 81 [64,16]
+  CRUSH rule 3 x 82 [37,40]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,115]
+  CRUSH rule 3 x 85 [87,88]
+  CRUSH rule 3 x 86 [37,68]
+  CRUSH rule 3 x 87 [116,77]
+  CRUSH rule 3 x 88 [38,55]
+  CRUSH rule 3 x 89 [76,25]
+  CRUSH rule 3 x 90 [14,50]
+  CRUSH rule 3 x 91 [68,61]
+  CRUSH rule 3 x 92 [86,95]
+  CRUSH rule 3 x 93 [44,35]
+  CRUSH rule 3 x 94 [46,71]
+  CRUSH rule 3 x 95 [108,53]
+  CRUSH rule 3 x 96 [66,87]
+  CRUSH rule 3 x 97 [111,45]
+  CRUSH rule 3 x 98 [93,110]
+  CRUSH rule 3 x 99 [78,43]
+  CRUSH rule 3 x 100 [28,61]
+  CRUSH rule 3 x 101 [91,110]
+  CRUSH rule 3 x 102 [82,7]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,79]
+  CRUSH rule 3 x 105 [34,87]
+  CRUSH rule 3 x 106 [69,12]
+  CRUSH rule 3 x 107 [1,59]
+  CRUSH rule 3 x 108 [7,109]
+  CRUSH rule 3 x 109 [112,67]
+  CRUSH rule 3 x 110 [54,61]
+  CRUSH rule 3 x 111 [10,92]
+  CRUSH rule 3 x 112 [80,11]
+  CRUSH rule 3 x 113 [69,38]
+  CRUSH rule 3 x 114 [79,38]
+  CRUSH rule 3 x 115 [10,48]
+  CRUSH rule 3 x 116 [37,108]
+  CRUSH rule 3 x 117 [87,56]
+  CRUSH rule 3 x 118 [23,56]
+  CRUSH rule 3 x 119 [104,31]
+  CRUSH rule 3 x 120 [44,93]
+  CRUSH rule 3 x 121 [80,16]
+  CRUSH rule 3 x 122 [45,54]
+  CRUSH rule 3 x 123 [22,112]
+  CRUSH rule 3 x 124 [97,50]
+  CRUSH rule 3 x 125 [66,6]
+  CRUSH rule 3 x 126 [70,39]
+  CRUSH rule 3 x 127 [70,75]
+  CRUSH rule 3 x 128 [11,111]
+  CRUSH rule 3 x 129 [103,46]
+  CRUSH rule 3 x 130 [50,73]
+  CRUSH rule 3 x 131 [44,15]
+  CRUSH rule 3 x 132 [69,58]
+  CRUSH rule 3 x 133 [67,115]
+  CRUSH rule 3 x 134 [37,60]
+  CRUSH rule 3 x 135 [78,61]
+  CRUSH rule 3 x 136 [32,29]
+  CRUSH rule 3 x 137 [92,87]
+  CRUSH rule 3 x 138 [54,8]
+  CRUSH rule 3 x 139 [89,60]
+  CRUSH rule 3 x 140 [39,50]
+  CRUSH rule 3 x 141 [89,62]
+  CRUSH rule 3 x 142 [22,86]
+  CRUSH rule 3 x 143 [96,16]
+  CRUSH rule 3 x 144 [13,1]
+  CRUSH rule 3 x 145 [77,54]
+  CRUSH rule 3 x 146 [12,43]
+  CRUSH rule 3 x 147 [2,59]
+  CRUSH rule 3 x 148 [85,50]
+  CRUSH rule 3 x 149 [103,68]
+  CRUSH rule 3 x 150 [14,50]
+  CRUSH rule 3 x 151 [75,56]
+  CRUSH rule 3 x 152 [49,18]
+  CRUSH rule 3 x 153 [92,79]
+  CRUSH rule 3 x 154 [19,26]
+  CRUSH rule 3 x 155 [12,13]
+  CRUSH rule 3 x 156 [107,18]
+  CRUSH rule 3 x 157 [15,78]
+  CRUSH rule 3 x 158 [11,28]
+  CRUSH rule 3 x 159 [33,88]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,78]
+  CRUSH rule 3 x 162 [55,96]
+  CRUSH rule 3 x 163 [54,55]
+  CRUSH rule 3 x 164 [72,99]
+  CRUSH rule 3 x 165 [25,116]
+  CRUSH rule 3 x 166 [2,23]
+  CRUSH rule 3 x 167 [89,40]
+  CRUSH rule 3 x 168 [68,49]
+  CRUSH rule 3 x 169 [51,50]
+  CRUSH rule 3 x 170 [68,91]
+  CRUSH rule 3 x 171 [88,51]
+  CRUSH rule 3 x 172 [117,23]
+  CRUSH rule 3 x 173 [29,1]
+  CRUSH rule 3 x 174 [67,40]
+  CRUSH rule 3 x 175 [48,41]
+  CRUSH rule 3 x 176 [94,91]
+  CRUSH rule 3 x 177 [53,70]
+  CRUSH rule 3 x 178 [39,110]
+  CRUSH rule 3 x 179 [72,89]
+  CRUSH rule 3 x 180 [3,38]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,46]
+  CRUSH rule 3 x 183 [11,78]
+  CRUSH rule 3 x 184 [79,92]
+  CRUSH rule 3 x 185 [97,92]
+  CRUSH rule 3 x 186 [67,116]
+  CRUSH rule 3 x 187 [6,96]
+  CRUSH rule 3 x 188 [76,16]
+  CRUSH rule 3 x 189 [96,71]
+  CRUSH rule 3 x 190 [90,6]
+  CRUSH rule 3 x 191 [49,84]
+  CRUSH rule 3 x 192 [93,114]
+  CRUSH rule 3 x 193 [89,60]
+  CRUSH rule 3 x 194 [62,61]
+  CRUSH rule 3 x 195 [119,95]
+  CRUSH rule 3 x 196 [20,28]
+  CRUSH rule 3 x 197 [6,64]
+  CRUSH rule 3 x 198 [55,112]
+  CRUSH rule 3 x 199 [66,17]
+  CRUSH rule 3 x 200 [12,63]
+  CRUSH rule 3 x 201 [52,23]
+  CRUSH rule 3 x 202 [98,33]
+  CRUSH rule 3 x 203 [36,22]
+  CRUSH rule 3 x 204 [10,100]
+  CRUSH rule 3 x 205 [38,29]
+  CRUSH rule 3 x 206 [38,27]
+  CRUSH rule 3 x 207 [19,108]
+  CRUSH rule 3 x 208 [63,26]
+  CRUSH rule 3 x 209 [70,11]
+  CRUSH rule 3 x 210 [79,58]
+  CRUSH rule 3 x 211 [26,59]
+  CRUSH rule 3 x 212 [107,114]
+  CRUSH rule 3 x 213 [100,3]
+  CRUSH rule 3 x 214 [91,118]
+  CRUSH rule 3 x 215 [92,16]
+  CRUSH rule 3 x 216 [99,94]
+  CRUSH rule 3 x 217 [86,99]
+  CRUSH rule 3 x 218 [70,15]
+  CRUSH rule 3 x 219 [61,58]
+  CRUSH rule 3 x 220 [23,56]
+  CRUSH rule 3 x 221 [21,92]
+  CRUSH rule 3 x 222 [102,29]
+  CRUSH rule 3 x 223 [34,73]
+  CRUSH rule 3 x 224 [107,68]
+  CRUSH rule 3 x 225 [61,98]
+  CRUSH rule 3 x 226 [44,13]
+  CRUSH rule 3 x 227 [55,60]
+  CRUSH rule 3 x 228 [117,55]
+  CRUSH rule 3 x 229 [100,67]
+  CRUSH rule 3 x 230 [41,109]
+  CRUSH rule 3 x 231 [30,71]
+  CRUSH rule 3 x 232 [23,1]
+  CRUSH rule 3 x 233 [47,90]
+  CRUSH rule 3 x 234 [55,62]
+  CRUSH rule 3 x 235 [20,60]
+  CRUSH rule 3 x 236 [95,24]
+  CRUSH rule 3 x 237 [21,106]
+  CRUSH rule 3 x 238 [109,15]
+  CRUSH rule 3 x 239 [40,101]
+  CRUSH rule 3 x 240 [63,60]
+  CRUSH rule 3 x 241 [47,12]
+  CRUSH rule 3 x 242 [73,74]
+  CRUSH rule 3 x 243 [76,8]
+  CRUSH rule 3 x 244 [103,50]
+  CRUSH rule 3 x 245 [106,95]
+  CRUSH rule 3 x 246 [35,82]
+  CRUSH rule 3 x 247 [116,101]
+  CRUSH rule 3 x 248 [8,119]
+  CRUSH rule 3 x 249 [2,17]
+  CRUSH rule 3 x 250 [34,89]
+  CRUSH rule 3 x 251 [28,69]
+  CRUSH rule 3 x 252 [95,80]
+  CRUSH rule 3 x 253 [109,3]
+  CRUSH rule 3 x 254 [99,80]
+  CRUSH rule 3 x 255 [112,85]
+  CRUSH rule 3 x 256 [94,63]
+  CRUSH rule 3 x 257 [100,87]
+  CRUSH rule 3 x 258 [34,63]
+  CRUSH rule 3 x 259 [70,107]
+  CRUSH rule 3 x 260 [89,115]
+  CRUSH rule 3 x 261 [94,83]
+  CRUSH rule 3 x 262 [42,45]
+  CRUSH rule 3 x 263 [113,101]
+  CRUSH rule 3 x 264 [36,81]
+  CRUSH rule 3 x 265 [14,88]
+  CRUSH rule 3 x 266 [75,96]
+  CRUSH rule 3 x 267 [6,5]
+  CRUSH rule 3 x 268 [38,47]
+  CRUSH rule 3 x 269 [86,59]
+  CRUSH rule 3 x 270 [87,70]
+  CRUSH rule 3 x 271 [19,108]
+  CRUSH rule 3 x 272 [73,5]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,64]
+  CRUSH rule 3 x 275 [29,34]
+  CRUSH rule 3 x 276 [7,100]
+  CRUSH rule 3 x 277 [74,6]
+  CRUSH rule 3 x 278 [107,115]
+  CRUSH rule 3 x 279 [112,20]
+  CRUSH rule 3 x 280 [113,15]
+  CRUSH rule 3 x 281 [89,56]
+  CRUSH rule 3 x 282 [20,38]
+  CRUSH rule 3 x 283 [8,114]
+  CRUSH rule 3 x 284 [66,75]
+  CRUSH rule 3 x 285 [99,94]
+  CRUSH rule 3 x 286 [78,6]
+  CRUSH rule 3 x 287 [12,27]
+  CRUSH rule 3 x 288 [24,22]
+  CRUSH rule 3 x 289 [105,64]
+  CRUSH rule 3 x 290 [25,46]
+  CRUSH rule 3 x 291 [35,116]
+  CRUSH rule 3 x 292 [20,109]
+  CRUSH rule 3 x 293 [27,92]
+  CRUSH rule 3 x 294 [60,93]
+  CRUSH rule 3 x 295 [37,2]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,55]
+  CRUSH rule 3 x 298 [70,53]
+  CRUSH rule 3 x 299 [116,10]
+  CRUSH rule 3 x 300 [67,26]
+  CRUSH rule 3 x 301 [117,23]
+  CRUSH rule 3 x 302 [78,67]
+  CRUSH rule 3 x 303 [19,5]
+  CRUSH rule 3 x 304 [101,50]
+  CRUSH rule 3 x 305 [5,59]
+  CRUSH rule 3 x 306 [41,18]
+  CRUSH rule 3 x 307 [65,5]
+  CRUSH rule 3 x 308 [91,2]
+  CRUSH rule 3 x 309 [38,53]
+  CRUSH rule 3 x 310 [26,15]
+  CRUSH rule 3 x 311 [36,95]
+  CRUSH rule 3 x 312 [114,61]
+  CRUSH rule 3 x 313 [104,65]
+  CRUSH rule 3 x 314 [28,6]
+  CRUSH rule 3 x 315 [118,31]
+  CRUSH rule 3 x 316 [98,95]
+  CRUSH rule 3 x 317 [118,13]
+  CRUSH rule 3 x 318 [17,30]
+  CRUSH rule 3 x 319 [53,1]
+  CRUSH rule 3 x 320 [36,41]
+  CRUSH rule 3 x 321 [33,5]
+  CRUSH rule 3 x 322 [68,10]
+  CRUSH rule 3 x 323 [66,29]
+  CRUSH rule 3 x 324 [21,72]
+  CRUSH rule 3 x 325 [52,16]
+  CRUSH rule 3 x 326 [7,109]
+  CRUSH rule 3 x 327 [62,17]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,100]
+  CRUSH rule 3 x 330 [24,11]
+  CRUSH rule 3 x 331 [84,95]
+  CRUSH rule 3 x 332 [61,111]
+  CRUSH rule 3 x 333 [116,25]
+  CRUSH rule 3 x 334 [94,103]
+  CRUSH rule 3 x 335 [71,118]
+  CRUSH rule 3 x 336 [24,99]
+  CRUSH rule 3 x 337 [18,83]
+  CRUSH rule 3 x 338 [43,28]
+  CRUSH rule 3 x 339 [13,64]
+  CRUSH rule 3 x 340 [81,111]
+  CRUSH rule 3 x 341 [46,105]
+  CRUSH rule 3 x 342 [92,23]
+  CRUSH rule 3 x 343 [49,112]
+  CRUSH rule 3 x 344 [1,87]
+  CRUSH rule 3 x 345 [56,35]
+  CRUSH rule 3 x 346 [3,54]
+  CRUSH rule 3 x 347 [106,27]
+  CRUSH rule 3 x 348 [10,117]
+  CRUSH rule 3 x 349 [96,87]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,41]
+  CRUSH rule 3 x 352 [36,13]
+  CRUSH rule 3 x 353 [10,82]
+  CRUSH rule 3 x 354 [55,52]
+  CRUSH rule 3 x 355 [73,94]
+  CRUSH rule 3 x 356 [75,66]
+  CRUSH rule 3 x 357 [70,93]
+  CRUSH rule 3 x 358 [97,56]
+  CRUSH rule 3 x 359 [110,105]
+  CRUSH rule 3 x 360 [106,57]
+  CRUSH rule 3 x 361 [27,42]
+  CRUSH rule 3 x 362 [28,55]
+  CRUSH rule 3 x 363 [68,20]
+  CRUSH rule 3 x 364 [23,50]
+  CRUSH rule 3 x 365 [57,76]
+  CRUSH rule 3 x 366 [42,75]
+  CRUSH rule 3 x 367 [103,82]
+  CRUSH rule 3 x 368 [103,104]
+  CRUSH rule 3 x 369 [12,57]
+  CRUSH rule 3 x 370 [11,26]
+  CRUSH rule 3 x 371 [34,55]
+  CRUSH rule 3 x 372 [58,14]
+  CRUSH rule 3 x 373 [6,42]
+  CRUSH rule 3 x 374 [110,95]
+  CRUSH rule 3 x 375 [5,43]
+  CRUSH rule 3 x 376 [91,86]
+  CRUSH rule 3 x 377 [93,116]
+  CRUSH rule 3 x 378 [68,6]
+  CRUSH rule 3 x 379 [77,44]
+  CRUSH rule 3 x 380 [76,83]
+  CRUSH rule 3 x 381 [36,27]
+  CRUSH rule 3 x 382 [26,77]
+  CRUSH rule 3 x 383 [76,99]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,93]
+  CRUSH rule 3 x 386 [83,92]
+  CRUSH rule 3 x 387 [16,26]
+  CRUSH rule 3 x 388 [29,113]
+  CRUSH rule 3 x 389 [92,29]
+  CRUSH rule 3 x 390 [68,77]
+  CRUSH rule 3 x 391 [15,88]
+  CRUSH rule 3 x 392 [21,32]
+  CRUSH rule 3 x 393 [91,18]
+  CRUSH rule 3 x 394 [38,73]
+  CRUSH rule 3 x 395 [21,119]
+  CRUSH rule 3 x 396 [12,13]
+  CRUSH rule 3 x 397 [40,63]
+  CRUSH rule 3 x 398 [44,3]
+  CRUSH rule 3 x 399 [5,95]
+  CRUSH rule 3 x 400 [19,102]
+  CRUSH rule 3 x 401 [79,52]
+  CRUSH rule 3 x 402 [107,98]
+  CRUSH rule 3 x 403 [23,82]
+  CRUSH rule 3 x 404 [87,68]
+  CRUSH rule 3 x 405 [90,97]
+  CRUSH rule 3 x 406 [15,117]
+  CRUSH rule 3 x 407 [70,35]
+  CRUSH rule 3 x 408 [55,72]
+  CRUSH rule 3 x 409 [73,62]
+  CRUSH rule 3 x 410 [70,73]
+  CRUSH rule 3 x 411 [34,25]
+  CRUSH rule 3 x 412 [105,117]
+  CRUSH rule 3 x 413 [41,110]
+  CRUSH rule 3 x 414 [70,65]
+  CRUSH rule 3 x 415 [107,5]
+  CRUSH rule 3 x 416 [2,22]
+  CRUSH rule 3 x 417 [26,14]
+  CRUSH rule 3 x 418 [51,46]
+  CRUSH rule 3 x 419 [8,82]
+  CRUSH rule 3 x 420 [109,105]
+  CRUSH rule 3 x 421 [114,75]
+  CRUSH rule 3 x 422 [109,87]
+  CRUSH rule 3 x 423 [59,24]
+  CRUSH rule 3 x 424 [92,51]
+  CRUSH rule 3 x 425 [101,111]
+  CRUSH rule 3 x 426 [36,6]
+  CRUSH rule 3 x 427 [8,24]
+  CRUSH rule 3 x 428 [68,35]
+  CRUSH rule 3 x 429 [76,75]
+  CRUSH rule 3 x 430 [67,117]
+  CRUSH rule 3 x 431 [70,25]
+  CRUSH rule 3 x 432 [7,34]
+  CRUSH rule 3 x 433 [49,84]
+  CRUSH rule 3 x 434 [64,31]
+  CRUSH rule 3 x 435 [110,13]
+  CRUSH rule 3 x 436 [106,89]
+  CRUSH rule 3 x 437 [26,65]
+  CRUSH rule 3 x 438 [118,63]
+  CRUSH rule 3 x 439 [40,21]
+  CRUSH rule 3 x 440 [45,119]
+  CRUSH rule 3 x 441 [112,105]
+  CRUSH rule 3 x 442 [55,113]
+  CRUSH rule 3 x 443 [44,33]
+  CRUSH rule 3 x 444 [71,38]
+  CRUSH rule 3 x 445 [58,81]
+  CRUSH rule 3 x 446 [40,10]
+  CRUSH rule 3 x 447 [100,61]
+  CRUSH rule 3 x 448 [111,73]
+  CRUSH rule 3 x 449 [67,66]
+  CRUSH rule 3 x 450 [117,61]
+  CRUSH rule 3 x 451 [66,81]
+  CRUSH rule 3 x 452 [70,8]
+  CRUSH rule 3 x 453 [82,85]
+  CRUSH rule 3 x 454 [53,18]
+  CRUSH rule 3 x 455 [91,42]
+  CRUSH rule 3 x 456 [101,46]
+  CRUSH rule 3 x 457 [113,51]
+  CRUSH rule 3 x 458 [119,25]
+  CRUSH rule 3 x 459 [50,67]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,51]
+  CRUSH rule 3 x 462 [98,107]
+  CRUSH rule 3 x 463 [108,105]
+  CRUSH rule 3 x 464 [19,109]
+  CRUSH rule 3 x 465 [62,23]
+  CRUSH rule 3 x 466 [53,12]
+  CRUSH rule 3 x 467 [40,57]
+  CRUSH rule 3 x 468 [97,44]
+  CRUSH rule 3 x 469 [98,75]
+  CRUSH rule 3 x 470 [50,29]
+  CRUSH rule 3 x 471 [40,13]
+  CRUSH rule 3 x 472 [27,18]
+  CRUSH rule 3 x 473 [48,35]
+  CRUSH rule 3 x 474 [51,32]
+  CRUSH rule 3 x 475 [49,117]
+  CRUSH rule 3 x 476 [110,31]
+  CRUSH rule 3 x 477 [80,97]
+  CRUSH rule 3 x 478 [78,99]
+  CRUSH rule 3 x 479 [31,66]
+  CRUSH rule 3 x 480 [75,88]
+  CRUSH rule 3 x 481 [26,20]
+  CRUSH rule 3 x 482 [84,53]
+  CRUSH rule 3 x 483 [15,116]
+  CRUSH rule 3 x 484 [37,114]
+  CRUSH rule 3 x 485 [84,8]
+  CRUSH rule 3 x 486 [92,10]
+  CRUSH rule 3 x 487 [106,17]
+  CRUSH rule 3 x 488 [42,20]
+  CRUSH rule 3 x 489 [89,2]
+  CRUSH rule 3 x 490 [22,114]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,66]
+  CRUSH rule 3 x 493 [94,14]
+  CRUSH rule 3 x 494 [59,86]
+  CRUSH rule 3 x 495 [95,58]
+  CRUSH rule 3 x 496 [46,41]
+  CRUSH rule 3 x 497 [102,27]
+  CRUSH rule 3 x 498 [21,116]
+  CRUSH rule 3 x 499 [5,49]
+  CRUSH rule 3 x 500 [50,49]
+  CRUSH rule 3 x 501 [60,3]
+  CRUSH rule 3 x 502 [65,110]
+  CRUSH rule 3 x 503 [21,112]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,93]
+  CRUSH rule 3 x 506 [79,64]
+  CRUSH rule 3 x 507 [34,107]
+  CRUSH rule 3 x 508 [45,114]
+  CRUSH rule 3 x 509 [19,88]
+  CRUSH rule 3 x 510 [117,45]
+  CRUSH rule 3 x 511 [14,104]
+  CRUSH rule 3 x 512 [59,26]
+  CRUSH rule 3 x 513 [102,93]
+  CRUSH rule 3 x 514 [75,72]
+  CRUSH rule 3 x 515 [84,41]
+  CRUSH rule 3 x 516 [37,30]
+  CRUSH rule 3 x 517 [83,115]
+  CRUSH rule 3 x 518 [18,83]
+  CRUSH rule 3 x 519 [67,88]
+  CRUSH rule 3 x 520 [15,114]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,51]
+  CRUSH rule 3 x 523 [68,101]
+  CRUSH rule 3 x 524 [33,38]
+  CRUSH rule 3 x 525 [63,115]
+  CRUSH rule 3 x 526 [83,50]
+  CRUSH rule 3 x 527 [37,56]
+  CRUSH rule 3 x 528 [108,81]
+  CRUSH rule 3 x 529 [74,33]
+  CRUSH rule 3 x 530 [49,92]
+  CRUSH rule 3 x 531 [117,105]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,85]
+  CRUSH rule 3 x 534 [97,24]
+  CRUSH rule 3 x 535 [48,75]
+  CRUSH rule 3 x 536 [113,101]
+  CRUSH rule 3 x 537 [116,47]
+  CRUSH rule 3 x 538 [85,74]
+  CRUSH rule 3 x 539 [72,43]
+  CRUSH rule 3 x 540 [39,34]
+  CRUSH rule 3 x 541 [53,84]
+  CRUSH rule 3 x 542 [27,32]
+  CRUSH rule 3 x 543 [45,113]
+  CRUSH rule 3 x 544 [59,42]
+  CRUSH rule 3 x 545 [118,95]
+  CRUSH rule 3 x 546 [18,79]
+  CRUSH rule 3 x 547 [67,30]
+  CRUSH rule 3 x 548 [53,100]
+  CRUSH rule 3 x 549 [60,45]
+  CRUSH rule 3 x 550 [92,101]
+  CRUSH rule 3 x 551 [77,88]
+  CRUSH rule 3 x 552 [61,94]
+  CRUSH rule 3 x 553 [71,78]
+  CRUSH rule 3 x 554 [61,115]
+  CRUSH rule 3 x 555 [76,77]
+  CRUSH rule 3 x 556 [106,55]
+  CRUSH rule 3 x 557 [26,22]
+  CRUSH rule 3 x 558 [41,84]
+  CRUSH rule 3 x 559 [65,24]
+  CRUSH rule 3 x 560 [94,16]
+  CRUSH rule 3 x 561 [27,5]
+  CRUSH rule 3 x 562 [78,59]
+  CRUSH rule 3 x 563 [59,70]
+  CRUSH rule 3 x 564 [96,8]
+  CRUSH rule 3 x 565 [8,48]
+  CRUSH rule 3 x 566 [119,17]
+  CRUSH rule 3 x 567 [7,38]
+  CRUSH rule 3 x 568 [57,94]
+  CRUSH rule 3 x 569 [65,26]
+  CRUSH rule 3 x 570 [98,27]
+  CRUSH rule 3 x 571 [95,30]
+  CRUSH rule 3 x 572 [62,83]
+  CRUSH rule 3 x 573 [1,79]
+  CRUSH rule 3 x 574 [89,42]
+  CRUSH rule 3 x 575 [87,113]
+  CRUSH rule 3 x 576 [21,68]
+  CRUSH rule 3 x 577 [8,84]
+  CRUSH rule 3 x 578 [75,115]
+  CRUSH rule 3 x 579 [105,68]
+  CRUSH rule 3 x 580 [51,28]
+  CRUSH rule 3 x 581 [55,113]
+  CRUSH rule 3 x 582 [27,113]
+  CRUSH rule 3 x 583 [6,78]
+  CRUSH rule 3 x 584 [10,30]
+  CRUSH rule 3 x 585 [20,111]
+  CRUSH rule 3 x 586 [48,27]
+  CRUSH rule 3 x 587 [29,94]
+  CRUSH rule 3 x 588 [103,90]
+  CRUSH rule 3 x 589 [88,95]
+  CRUSH rule 3 x 590 [76,101]
+  CRUSH rule 3 x 591 [42,43]
+  CRUSH rule 3 x 592 [78,51]
+  CRUSH rule 3 x 593 [82,71]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,59]
+  CRUSH rule 3 x 597 [16,36]
+  CRUSH rule 3 x 598 [37,56]
+  CRUSH rule 3 x 599 [10,84]
+  CRUSH rule 3 x 600 [24,69]
+  CRUSH rule 3 x 601 [104,14]
+  CRUSH rule 3 x 602 [48,45]
+  CRUSH rule 3 x 603 [93,32]
+  CRUSH rule 3 x 604 [118,79]
+  CRUSH rule 3 x 605 [104,53]
+  CRUSH rule 3 x 606 [90,83]
+  CRUSH rule 3 x 607 [95,110]
+  CRUSH rule 3 x 608 [112,101]
+  CRUSH rule 3 x 609 [34,99]
+  CRUSH rule 3 x 610 [106,16]
+  CRUSH rule 3 x 611 [66,87]
+  CRUSH rule 3 x 612 [2,81]
+  CRUSH rule 3 x 613 [13,86]
+  CRUSH rule 3 x 614 [50,3]
+  CRUSH rule 3 x 615 [24,73]
+  CRUSH rule 3 x 616 [41,119]
+  CRUSH rule 3 x 617 [81,106]
+  CRUSH rule 3 x 618 [3,104]
+  CRUSH rule 3 x 619 [92,7]
+  CRUSH rule 3 x 620 [108,11]
+  CRUSH rule 3 x 621 [105,115]
+  CRUSH rule 3 x 622 [67,48]
+  CRUSH rule 3 x 623 [69,74]
+  CRUSH rule 3 x 624 [115,49]
+  CRUSH rule 3 x 625 [73,109]
+  CRUSH rule 3 x 626 [52,3]
+  CRUSH rule 3 x 627 [116,3]
+  CRUSH rule 3 x 628 [98,91]
+  CRUSH rule 3 x 629 [6,112]
+  CRUSH rule 3 x 630 [22,72]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,71]
+  CRUSH rule 3 x 633 [65,12]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,46]
+  CRUSH rule 3 x 636 [23,70]
+  CRUSH rule 3 x 637 [99,24]
+  CRUSH rule 3 x 638 [43,114]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,73]
+  CRUSH rule 3 x 641 [45,84]
+  CRUSH rule 3 x 642 [47,66]
+  CRUSH rule 3 x 643 [64,8]
+  CRUSH rule 3 x 644 [31,82]
+  CRUSH rule 3 x 645 [77,64]
+  CRUSH rule 3 x 646 [37,86]
+  CRUSH rule 3 x 647 [65,56]
+  CRUSH rule 3 x 648 [84,13]
+  CRUSH rule 3 x 649 [88,55]
+  CRUSH rule 3 x 650 [21,76]
+  CRUSH rule 3 x 651 [63,116]
+  CRUSH rule 3 x 652 [57,112]
+  CRUSH rule 3 x 653 [38,61]
+  CRUSH rule 3 x 654 [104,67]
+  CRUSH rule 3 x 655 [89,54]
+  CRUSH rule 3 x 656 [84,49]
+  CRUSH rule 3 x 657 [47,32]
+  CRUSH rule 3 x 658 [80,29]
+  CRUSH rule 3 x 659 [11,112]
+  CRUSH rule 3 x 660 [65,111]
+  CRUSH rule 3 x 661 [96,73]
+  CRUSH rule 3 x 662 [111,73]
+  CRUSH rule 3 x 663 [83,60]
+  CRUSH rule 3 x 664 [59,80]
+  CRUSH rule 3 x 665 [31,117]
+  CRUSH rule 3 x 666 [112,101]
+  CRUSH rule 3 x 667 [70,47]
+  CRUSH rule 3 x 668 [96,57]
+  CRUSH rule 3 x 669 [56,39]
+  CRUSH rule 3 x 670 [98,105]
+  CRUSH rule 3 x 671 [57,48]
+  CRUSH rule 3 x 672 [37,36]
+  CRUSH rule 3 x 673 [83,2]
+  CRUSH rule 3 x 674 [36,25]
+  CRUSH rule 3 x 675 [88,14]
+  CRUSH rule 3 x 676 [3,110]
+  CRUSH rule 3 x 677 [88,67]
+  CRUSH rule 3 x 678 [27,44]
+  CRUSH rule 3 x 679 [33,116]
+  CRUSH rule 3 x 680 [111,39]
+  CRUSH rule 3 x 681 [53,12]
+  CRUSH rule 3 x 682 [12,87]
+  CRUSH rule 3 x 683 [24,85]
+  CRUSH rule 3 x 684 [98,65]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,72]
+  CRUSH rule 3 x 688 [16,114]
+  CRUSH rule 3 x 689 [32,31]
+  CRUSH rule 3 x 690 [96,33]
+  CRUSH rule 3 x 691 [34,6]
+  CRUSH rule 3 x 692 [97,84]
+  CRUSH rule 3 x 693 [29,118]
+  CRUSH rule 3 x 694 [6,30]
+  CRUSH rule 3 x 695 [31,72]
+  CRUSH rule 3 x 696 [104,97]
+  CRUSH rule 3 x 697 [19,96]
+  CRUSH rule 3 x 698 [30,69]
+  CRUSH rule 3 x 699 [47,76]
+  CRUSH rule 3 x 700 [82,55]
+  CRUSH rule 3 x 701 [53,80]
+  CRUSH rule 3 x 702 [95,98]
+  CRUSH rule 3 x 703 [92,65]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,1]
+  CRUSH rule 3 x 706 [74,35]
+  CRUSH rule 3 x 707 [91,115]
+  CRUSH rule 3 x 708 [95,112]
+  CRUSH rule 3 x 709 [73,72]
+  CRUSH rule 3 x 710 [94,47]
+  CRUSH rule 3 x 711 [68,41]
+  CRUSH rule 3 x 712 [107,18]
+  CRUSH rule 3 x 713 [29,109]
+  CRUSH rule 3 x 714 [86,61]
+  CRUSH rule 3 x 715 [74,13]
+  CRUSH rule 3 x 716 [101,56]
+  CRUSH rule 3 x 717 [12,29]
+  CRUSH rule 3 x 718 [83,24]
+  CRUSH rule 3 x 719 [26,10]
+  CRUSH rule 3 x 720 [69,2]
+  CRUSH rule 3 x 721 [51,42]
+  CRUSH rule 3 x 722 [15,74]
+  CRUSH rule 3 x 723 [117,14]
+  CRUSH rule 3 x 724 [45,38]
+  CRUSH rule 3 x 725 [53,110]
+  CRUSH rule 3 x 726 [103,68]
+  CRUSH rule 3 x 727 [89,100]
+  CRUSH rule 3 x 728 [76,16]
+  CRUSH rule 3 x 729 [35,90]
+  CRUSH rule 3 x 730 [28,103]
+  CRUSH rule 3 x 731 [78,41]
+  CRUSH rule 3 x 732 [1,27]
+  CRUSH rule 3 x 733 [35,100]
+  CRUSH rule 3 x 734 [119,85]
+  CRUSH rule 3 x 735 [102,43]
+  CRUSH rule 3 x 736 [37,92]
+  CRUSH rule 3 x 737 [117,11]
+  CRUSH rule 3 x 738 [57,32]
+  CRUSH rule 3 x 739 [87,1]
+  CRUSH rule 3 x 740 [29,80]
+  CRUSH rule 3 x 741 [47,111]
+  CRUSH rule 3 x 742 [106,83]
+  CRUSH rule 3 x 743 [105,94]
+  CRUSH rule 3 x 744 [23,64]
+  CRUSH rule 3 x 745 [37,112]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,95]
+  CRUSH rule 3 x 748 [48,14]
+  CRUSH rule 3 x 749 [102,101]
+  CRUSH rule 3 x 750 [83,78]
+  CRUSH rule 3 x 751 [25,104]
+  CRUSH rule 3 x 752 [82,95]
+  CRUSH rule 3 x 753 [14,113]
+  CRUSH rule 3 x 754 [114,51]
+  CRUSH rule 3 x 755 [87,26]
+  CRUSH rule 3 x 756 [113,87]
+  CRUSH rule 3 x 757 [47,66]
+  CRUSH rule 3 x 758 [54,63]
+  CRUSH rule 3 x 759 [74,20]
+  CRUSH rule 3 x 760 [88,22]
+  CRUSH rule 3 x 761 [73,86]
+  CRUSH rule 3 x 762 [34,17]
+  CRUSH rule 3 x 763 [13,78]
+  CRUSH rule 3 x 764 [89,42]
+  CRUSH rule 3 x 765 [109,91]
+  CRUSH rule 3 x 766 [19,66]
+  CRUSH rule 3 x 767 [41,26]
+  CRUSH rule 3 x 768 [106,57]
+  CRUSH rule 3 x 769 [91,104]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,35]
+  CRUSH rule 3 x 772 [97,108]
+  CRUSH rule 3 x 773 [116,47]
+  CRUSH rule 3 x 774 [100,31]
+  CRUSH rule 3 x 775 [102,43]
+  CRUSH rule 3 x 776 [69,38]
+  CRUSH rule 3 x 777 [91,52]
+  CRUSH rule 3 x 778 [83,119]
+  CRUSH rule 3 x 779 [47,60]
+  CRUSH rule 3 x 780 [63,70]
+  CRUSH rule 3 x 781 [105,2]
+  CRUSH rule 3 x 782 [117,59]
+  CRUSH rule 3 x 783 [19,109]
+  CRUSH rule 3 x 784 [63,114]
+  CRUSH rule 3 x 785 [27,84]
+  CRUSH rule 3 x 786 [41,110]
+  CRUSH rule 3 x 787 [108,73]
+  CRUSH rule 3 x 788 [74,103]
+  CRUSH rule 3 x 789 [50,17]
+  CRUSH rule 3 x 790 [20,106]
+  CRUSH rule 3 x 791 [96,87]
+  CRUSH rule 3 x 792 [80,97]
+  CRUSH rule 3 x 793 [6,26]
+  CRUSH rule 3 x 794 [14,42]
+  CRUSH rule 3 x 795 [30,8]
+  CRUSH rule 3 x 796 [87,36]
+  CRUSH rule 3 x 797 [64,61]
+  CRUSH rule 3 x 798 [42,69]
+  CRUSH rule 3 x 799 [19,117]
+  CRUSH rule 3 x 800 [106,8]
+  CRUSH rule 3 x 801 [2,57]
+  CRUSH rule 3 x 802 [63,68]
+  CRUSH rule 3 x 803 [46,35]
+  CRUSH rule 3 x 804 [33,26]
+  CRUSH rule 3 x 805 [96,49]
+  CRUSH rule 3 x 806 [48,25]
+  CRUSH rule 3 x 807 [48,83]
+  CRUSH rule 3 x 808 [76,31]
+  CRUSH rule 3 x 809 [27,48]
+  CRUSH rule 3 x 810 [119,71]
+  CRUSH rule 3 x 811 [111,91]
+  CRUSH rule 3 x 812 [25,111]
+  CRUSH rule 3 x 813 [81,28]
+  CRUSH rule 3 x 814 [95,42]
+  CRUSH rule 3 x 815 [84,61]
+  CRUSH rule 3 x 816 [64,35]
+  CRUSH rule 3 x 817 [63,60]
+  CRUSH rule 3 x 818 [69,46]
+  CRUSH rule 3 x 819 [88,75]
+  CRUSH rule 3 x 820 [104,57]
+  CRUSH rule 3 x 821 [58,21]
+  CRUSH rule 3 x 822 [20,80]
+  CRUSH rule 3 x 823 [63,118]
+  CRUSH rule 3 x 824 [102,13]
+  CRUSH rule 3 x 825 [47,118]
+  CRUSH rule 3 x 826 [44,7]
+  CRUSH rule 3 x 827 [101,88]
+  CRUSH rule 3 x 828 [60,41]
+  CRUSH rule 3 x 829 [45,102]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,75]
+  CRUSH rule 3 x 833 [57,32]
+  CRUSH rule 3 x 834 [90,33]
+  CRUSH rule 3 x 835 [6,1]
+  CRUSH rule 3 x 836 [63,68]
+  CRUSH rule 3 x 837 [76,71]
+  CRUSH rule 3 x 838 [106,20]
+  CRUSH rule 3 x 839 [87,96]
+  CRUSH rule 3 x 840 [33,32]
+  CRUSH rule 3 x 841 [110,55]
+  CRUSH rule 3 x 842 [66,87]
+  CRUSH rule 3 x 843 [11,80]
+  CRUSH rule 3 x 844 [74,103]
+  CRUSH rule 3 x 845 [74,43]
+  CRUSH rule 3 x 846 [43,76]
+  CRUSH rule 3 x 847 [62,20]
+  CRUSH rule 3 x 848 [92,17]
+  CRUSH rule 3 x 849 [93,36]
+  CRUSH rule 3 x 850 [83,82]
+  CRUSH rule 3 x 851 [65,94]
+  CRUSH rule 3 x 852 [60,22]
+  CRUSH rule 3 x 853 [88,29]
+  CRUSH rule 3 x 854 [83,54]
+  CRUSH rule 3 x 855 [2,101]
+  CRUSH rule 3 x 856 [40,41]
+  CRUSH rule 3 x 857 [69,82]
+  CRUSH rule 3 x 858 [98,81]
+  CRUSH rule 3 x 859 [56,43]
+  CRUSH rule 3 x 860 [11,26]
+  CRUSH rule 3 x 861 [22,110]
+  CRUSH rule 3 x 862 [22,70]
+  CRUSH rule 3 x 863 [79,84]
+  CRUSH rule 3 x 864 [77,24]
+  CRUSH rule 3 x 865 [119,17]
+  CRUSH rule 3 x 866 [18,49]
+  CRUSH rule 3 x 867 [3,84]
+  CRUSH rule 3 x 868 [100,107]
+  CRUSH rule 3 x 869 [22,104]
+  CRUSH rule 3 x 870 [73,30]
+  CRUSH rule 3 x 871 [84,105]
+  CRUSH rule 3 x 872 [72,75]
+  CRUSH rule 3 x 873 [81,96]
+  CRUSH rule 3 x 874 [21,72]
+  CRUSH rule 3 x 875 [115,59]
+  CRUSH rule 3 x 876 [98,49]
+  CRUSH rule 3 x 877 [80,79]
+  CRUSH rule 3 x 878 [87,94]
+  CRUSH rule 3 x 879 [29,18]
+  CRUSH rule 3 x 880 [23,40]
+  CRUSH rule 3 x 881 [109,69]
+  CRUSH rule 3 x 882 [31,118]
+  CRUSH rule 3 x 883 [102,8]
+  CRUSH rule 3 x 884 [80,19]
+  CRUSH rule 3 x 885 [46,101]
+  CRUSH rule 3 x 886 [2,65]
+  CRUSH rule 3 x 887 [5,99]
+  CRUSH rule 3 x 888 [16,70]
+  CRUSH rule 3 x 889 [84,93]
+  CRUSH rule 3 x 890 [65,118]
+  CRUSH rule 3 x 891 [86,105]
+  CRUSH rule 3 x 892 [64,10]
+  CRUSH rule 3 x 893 [20,110]
+  CRUSH rule 3 x 894 [32,47]
+  CRUSH rule 3 x 895 [40,21]
+  CRUSH rule 3 x 896 [113,14]
+  CRUSH rule 3 x 897 [107,80]
+  CRUSH rule 3 x 898 [76,71]
+  CRUSH rule 3 x 899 [75,82]
+  CRUSH rule 3 x 900 [83,82]
+  CRUSH rule 3 x 901 [66,61]
+  CRUSH rule 3 x 902 [25,56]
+  CRUSH rule 3 x 903 [53,46]
+  CRUSH rule 3 x 904 [50,101]
+  CRUSH rule 3 x 905 [99,110]
+  CRUSH rule 3 x 906 [68,27]
+  CRUSH rule 3 x 907 [109,47]
+  CRUSH rule 3 x 908 [47,1]
+  CRUSH rule 3 x 909 [73,2]
+  CRUSH rule 3 x 910 [71,74]
+  CRUSH rule 3 x 911 [39,42]
+  CRUSH rule 3 x 912 [90,7]
+  CRUSH rule 3 x 913 [29,96]
+  CRUSH rule 3 x 914 [84,45]
+  CRUSH rule 3 x 915 [49,115]
+  CRUSH rule 3 x 916 [32,77]
+  CRUSH rule 3 x 917 [46,23]
+  CRUSH rule 3 x 918 [82,73]
+  CRUSH rule 3 x 919 [13,28]
+  CRUSH rule 3 x 920 [25,26]
+  CRUSH rule 3 x 921 [55,119]
+  CRUSH rule 3 x 922 [33,32]
+  CRUSH rule 3 x 923 [28,15]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,22]
+  CRUSH rule 3 x 926 [64,69]
+  CRUSH rule 3 x 927 [32,16]
+  CRUSH rule 3 x 928 [13,113]
+  CRUSH rule 3 x 929 [85,115]
+  CRUSH rule 3 x 930 [104,20]
+  CRUSH rule 3 x 931 [46,79]
+  CRUSH rule 3 x 932 [43,52]
+  CRUSH rule 3 x 933 [18,55]
+  CRUSH rule 3 x 934 [68,81]
+  CRUSH rule 3 x 935 [28,57]
+  CRUSH rule 3 x 936 [104,57]
+  CRUSH rule 3 x 937 [110,10]
+  CRUSH rule 3 x 938 [48,23]
+  CRUSH rule 3 x 939 [77,42]
+  CRUSH rule 3 x 940 [76,49]
+  CRUSH rule 3 x 941 [66,101]
+  CRUSH rule 3 x 942 [80,87]
+  CRUSH rule 3 x 943 [75,74]
+  CRUSH rule 3 x 944 [82,53]
+  CRUSH rule 3 x 945 [71,74]
+  CRUSH rule 3 x 946 [37,42]
+  CRUSH rule 3 x 947 [107,30]
+  CRUSH rule 3 x 948 [108,95]
+  CRUSH rule 3 x 949 [46,103]
+  CRUSH rule 3 x 950 [96,101]
+  CRUSH rule 3 x 951 [40,14]
+  CRUSH rule 3 x 952 [114,21]
+  CRUSH rule 3 x 953 [62,23]
+  CRUSH rule 3 x 954 [103,5]
+  CRUSH rule 3 x 955 [42,73]
+  CRUSH rule 3 x 956 [72,103]
+  CRUSH rule 3 x 957 [117,22]
+  CRUSH rule 3 x 958 [23,106]
+  CRUSH rule 3 x 959 [42,93]
+  CRUSH rule 3 x 960 [113,8]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,51]
+  CRUSH rule 3 x 963 [101,106]
+  CRUSH rule 3 x 964 [66,89]
+  CRUSH rule 3 x 965 [47,102]
+  CRUSH rule 3 x 966 [88,63]
+  CRUSH rule 3 x 967 [71,46]
+  CRUSH rule 3 x 968 [74,51]
+  CRUSH rule 3 x 969 [53,78]
+  CRUSH rule 3 x 970 [3,30]
+  CRUSH rule 3 x 971 [66,107]
+  CRUSH rule 3 x 972 [3,66]
+  CRUSH rule 3 x 973 [113,20]
+  CRUSH rule 3 x 974 [114,35]
+  CRUSH rule 3 x 975 [83,58]
+  CRUSH rule 3 x 976 [81,48]
+  CRUSH rule 3 x 977 [95,102]
+  CRUSH rule 3 x 978 [119,41]
+  CRUSH rule 3 x 979 [98,6]
+  CRUSH rule 3 x 980 [39,108]
+  CRUSH rule 3 x 981 [89,84]
+  CRUSH rule 3 x 982 [19,94]
+  CRUSH rule 3 x 983 [34,45]
+  CRUSH rule 3 x 984 [78,63]
+  CRUSH rule 3 x 985 [99,52]
+  CRUSH rule 3 x 986 [44,99]
+  CRUSH rule 3 x 987 [25,32]
+  CRUSH rule 3 x 988 [79,2]
+  CRUSH rule 3 x 989 [87,26]
+  CRUSH rule 3 x 990 [72,69]
+  CRUSH rule 3 x 991 [90,8]
+  CRUSH rule 3 x 992 [30,67]
+  CRUSH rule 3 x 993 [74,49]
+  CRUSH rule 3 x 994 [74,105]
+  CRUSH rule 3 x 995 [100,97]
+  CRUSH rule 3 x 996 [41,58]
+  CRUSH rule 3 x 997 [89,76]
+  CRUSH rule 3 x 998 [92,47]
+  CRUSH rule 3 x 999 [117,16]
+  CRUSH rule 3 x 1000 [50,47]
+  CRUSH rule 3 x 1001 [83,102]
+  CRUSH rule 3 x 1002 [94,37]
+  CRUSH rule 3 x 1003 [43,88]
+  CRUSH rule 3 x 1004 [89,54]
+  CRUSH rule 3 x 1005 [105,84]
+  CRUSH rule 3 x 1006 [45,111]
+  CRUSH rule 3 x 1007 [19,66]
+  CRUSH rule 3 x 1008 [31,76]
+  CRUSH rule 3 x 1009 [1,95]
+  CRUSH rule 3 x 1010 [31,113]
+  CRUSH rule 3 x 1011 [64,81]
+  CRUSH rule 3 x 1012 [68,49]
+  CRUSH rule 3 x 1013 [5,93]
+  CRUSH rule 3 x 1014 [33,66]
+  CRUSH rule 3 x 1015 [106,45]
+  CRUSH rule 3 x 1016 [107,86]
+  CRUSH rule 3 x 1017 [12,61]
+  CRUSH rule 3 x 1018 [61,26]
+  CRUSH rule 3 x 1019 [27,104]
+  CRUSH rule 3 x 1020 [31,86]
+  CRUSH rule 3 x 1021 [22,26]
+  CRUSH rule 3 x 1022 [73,34]
+  CRUSH rule 3 x 1023 [88,79]
+  rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-vary-r-2.t b/ceph/src/test/cli/crushtool/test-map-vary-r-2.t
new file mode 100644
index 00000000..c9e78c68
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-vary-r-2.t
@@ -0,0 +1,3078 @@
+  $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 2 --weight 0 0 --weight 4 0 --weight 9 0
+  crushtool successfully built or modified map.  Use '-o ' to write it out.
+  rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+  CRUSH rule 3 x 0 [94,45]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,118]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,96]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,70]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,76]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,100]
+  CRUSH rule 3 x 17 [85,110]
+  CRUSH rule 3 x 18 [11,119]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,47]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,90]
+  CRUSH rule 3 x 24 [83,38]
+  CRUSH rule 3 x 25 [81,5]
+  CRUSH rule 3 x 26 [17,100]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,119]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,69]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,52]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,29]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,11]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,68]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,21]
+  CRUSH rule 3 x 47 [106,53]
+  CRUSH rule 3 x 48 [34,33]
+  CRUSH rule 3 x 49 [99,104]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,52]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,75]
+  CRUSH rule 3 x 54 [28,93]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,40]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,33]
+  CRUSH rule 3 x 61 [88,3]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,113]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,61]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,7]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,44]
+  CRUSH rule 3 x 74 [29,74]
+  CRUSH rule 3 x 75 [60,89]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,7]
+  CRUSH rule 3 x 79 [64,41]
+  CRUSH rule 3 x 80 [73,104]
+  CRUSH rule 3 x 81 [64,19]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,68]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,95]
+  CRUSH rule 3 x 89 [76,3]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,65]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,23]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,100]
+  CRUSH rule 3 x 99 [78,22]
+  CRUSH rule 3 x 100 [28,63]
+  CRUSH rule 3 x 101 [91,36]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,95]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,116]
+  CRUSH rule 3 x 107 [1,55]
+  CRUSH rule 3 x 108 [7,72]
+  CRUSH rule 3 x 109 [112,63]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,78]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,54]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,42]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,16]
+  CRUSH rule 3 x 122 [45,110]
+  CRUSH rule 3 x 123 [22,86]
+  CRUSH rule 3 x 124 [97,72]
+  CRUSH rule 3 x 125 [66,71]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,97]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,110]
+  CRUSH rule 3 x 130 [50,63]
+  CRUSH rule 3 x 131 [44,25]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,2]
+  CRUSH rule 3 x 134 [37,94]
+  CRUSH rule 3 x 135 [78,17]
+  CRUSH rule 3 x 136 [32,91]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,62]
+  CRUSH rule 3 x 141 [89,96]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,87]
+  CRUSH rule 3 x 144 [13,86]
+  CRUSH rule 3 x 145 [77,60]
+  CRUSH rule 3 x 146 [12,53]
+  CRUSH rule 3 x 147 [2,59]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,58]
+  CRUSH rule 3 x 150 [14,54]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,33]
+  CRUSH rule 3 x 154 [19,111]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,5]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,58]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,31]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,47]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,107]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,34]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,2]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,1]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,90]
+  CRUSH rule 3 x 183 [11,74]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,113]
+  CRUSH rule 3 x 186 [67,96]
+  CRUSH rule 3 x 187 [6,36]
+  CRUSH rule 3 x 188 [76,20]
+  CRUSH rule 3 x 189 [96,89]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,60]
+  CRUSH rule 3 x 192 [93,64]
+  CRUSH rule 3 x 193 [89,112]
+  CRUSH rule 3 x 194 [62,63]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [97,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,29]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,98]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,35]
+  CRUSH rule 3 x 207 [19,36]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,76]
+  CRUSH rule 3 x 211 [26,89]
+  CRUSH rule 3 x 212 [115,107]
+  CRUSH rule 3 x 213 [100,8]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,89]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,2]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,12]
+  CRUSH rule 3 x 222 [50,63]
+  CRUSH rule 3 x 223 [34,59]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,102]
+  CRUSH rule 3 x 226 [44,105]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,78]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,44]
+  CRUSH rule 3 x 233 [47,113]
+  CRUSH rule 3 x 234 [55,18]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,53]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,38]
+  CRUSH rule 3 x 242 [73,52]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,71]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,81]
+  CRUSH rule 3 x 250 [34,41]
+  CRUSH rule 3 x 251 [28,15]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,77]
+  CRUSH rule 3 x 256 [94,45]
+  CRUSH rule 3 x 257 [100,81]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,80]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,53]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,18]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,86]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,70]
+  CRUSH rule 3 x 271 [19,88]
+  CRUSH rule 3 x 272 [73,52]
+  CRUSH rule 3 x 273 [69,92]
+  CRUSH rule 3 x 274 [47,88]
+  CRUSH rule 3 x 275 [112,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,87]
+  CRUSH rule 3 x 278 [107,44]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,10]
+  CRUSH rule 3 x 281 [89,109]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,118]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,101]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,26]
+  CRUSH rule 3 x 290 [25,12]
+  CRUSH rule 3 x 291 [35,46]
+  CRUSH rule 3 x 292 [20,28]
+  CRUSH rule 3 x 293 [27,24]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,116]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,63]
+  CRUSH rule 3 x 298 [70,87]
+  CRUSH rule 3 x 299 [116,61]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,94]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,53]
+  CRUSH rule 3 x 306 [41,12]
+  CRUSH rule 3 x 307 [65,64]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,69]
+  CRUSH rule 3 x 311 [36,83]
+  CRUSH rule 3 x 312 [114,97]
+  CRUSH rule 3 x 313 [104,31]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,22]
+  CRUSH rule 3 x 316 [98,8]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,48]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,30]
+  CRUSH rule 3 x 330 [24,55]
+  CRUSH rule 3 x 331 [84,91]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,35]
+  CRUSH rule 3 x 335 [71,66]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,74]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,52]
+  CRUSH rule 3 x 341 [46,81]
+  CRUSH rule 3 x 342 [92,6]
+  CRUSH rule 3 x 343 [49,26]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,87]
+  CRUSH rule 3 x 346 [3,70]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,103]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,91]
+  CRUSH rule 3 x 353 [10,118]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,5]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,89]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,46]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,75]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,115]
+  CRUSH rule 3 x 366 [42,33]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,23]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,76]
+  CRUSH rule 3 x 378 [68,10]
+  CRUSH rule 3 x 379 [77,30]
+  CRUSH rule 3 x 380 [76,25]
+  CRUSH rule 3 x 381 [36,55]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,25]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,15]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,118]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,2]
+  CRUSH rule 3 x 393 [91,112]
+  CRUSH rule 3 x 394 [38,13]
+  CRUSH rule 3 x 395 [21,117]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,103]
+  CRUSH rule 3 x 400 [19,100]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,110]
+  CRUSH rule 3 x 403 [23,92]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,52]
+  CRUSH rule 3 x 407 [70,85]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,1]
+  CRUSH rule 3 x 410 [70,107]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,68]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,66]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,17]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,21]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,73]
+  CRUSH rule 3 x 427 [8,115]
+  CRUSH rule 3 x 428 [68,31]
+  CRUSH rule 3 x 429 [76,6]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,1]
+  CRUSH rule 3 x 434 [64,31]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,25]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,85]
+  CRUSH rule 3 x 439 [40,21]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,57]
+  CRUSH rule 3 x 442 [55,108]
+  CRUSH rule 3 x 443 [44,14]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,19]
+  CRUSH rule 3 x 449 [67,66]
+  CRUSH rule 3 x 450 [117,97]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,41]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,92]
+  CRUSH rule 3 x 456 [101,104]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,43]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,64]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,60]
+  CRUSH rule 3 x 469 [98,71]
+  CRUSH rule 3 x 470 [50,27]
+  CRUSH rule 3 x 471 [40,31]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,75]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,90]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,36]
+  CRUSH rule 3 x 485 [84,43]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,51]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,86]
+  CRUSH rule 3 x 493 [94,21]
+  CRUSH rule 3 x 494 [59,98]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,12]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,37]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,28]
+  CRUSH rule 3 x 503 [21,96]
+  CRUSH rule 3 x 504 [67,1]
+  CRUSH rule 3 x 505 [12,10]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,67]
+  CRUSH rule 3 x 508 [45,56]
+  CRUSH rule 3 x 509 [19,70]
+  CRUSH rule 3 x 510 [117,73]
+  CRUSH rule 3 x 511 [14,117]
+  CRUSH rule 3 x 512 [59,26]
+  CRUSH rule 3 x 513 [102,93]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,39]
+  CRUSH rule 3 x 516 [37,12]
+  CRUSH rule 3 x 517 [83,68]
+  CRUSH rule 3 x 518 [18,107]
+  CRUSH rule 3 x 519 [67,119]
+  CRUSH rule 3 x 520 [15,88]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,3]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,113]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,56]
+  CRUSH rule 3 x 528 [108,35]
+  CRUSH rule 3 x 529 [74,15]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,104]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,37]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,56]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,94]
+  CRUSH rule 3 x 541 [53,86]
+  CRUSH rule 3 x 542 [27,114]
+  CRUSH rule 3 x 543 [45,78]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,49]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,82]
+  CRUSH rule 3 x 549 [60,71]
+  CRUSH rule 3 x 550 [92,71]
+  CRUSH rule 3 x 551 [77,88]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,20]
+  CRUSH rule 3 x 556 [106,89]
+  CRUSH rule 3 x 557 [26,45]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,105]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,25]
+  CRUSH rule 3 x 563 [59,72]
+  CRUSH rule 3 x 564 [96,59]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,35]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,38]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,11]
+  CRUSH rule 3 x 574 [89,78]
+  CRUSH rule 3 x 575 [87,50]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,113]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,116]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,66]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,116]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,63]
+  CRUSH rule 3 x 591 [42,77]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,31]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,94]
+  CRUSH rule 3 x 598 [37,60]
+  CRUSH rule 3 x 599 [10,76]
+  CRUSH rule 3 x 600 [24,7]
+  CRUSH rule 3 x 601 [104,87]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,60]
+  CRUSH rule 3 x 604 [118,71]
+  CRUSH rule 3 x 605 [104,87]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,40]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,7]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [81,54]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,27]
+  CRUSH rule 3 x 620 [108,103]
+  CRUSH rule 3 x 621 [105,110]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,64]
+  CRUSH rule 3 x 624 [115,29]
+  CRUSH rule 3 x 625 [73,98]
+  CRUSH rule 3 x 626 [52,55]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,35]
+  CRUSH rule 3 x 629 [6,46]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,80]
+  CRUSH rule 3 x 632 [80,95]
+  CRUSH rule 3 x 633 [65,68]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,72]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,68]
+  CRUSH rule 3 x 638 [43,109]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,96]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,7]
+  CRUSH rule 3 x 644 [31,5]
+  CRUSH rule 3 x 645 [77,1]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,56]
+  CRUSH rule 3 x 648 [56,79]
+  CRUSH rule 3 x 649 [88,103]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,14]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [7,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,2]
+  CRUSH rule 3 x 660 [65,110]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,78]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,101]
+  CRUSH rule 3 x 667 [70,11]
+  CRUSH rule 3 x 668 [96,63]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,99]
+  CRUSH rule 3 x 671 [57,2]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,62]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,17]
+  CRUSH rule 3 x 676 [3,100]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,26]
+  CRUSH rule 3 x 680 [111,43]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,95]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,41]
+  CRUSH rule 3 x 690 [96,103]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,72]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,70]
+  CRUSH rule 3 x 695 [31,62]
+  CRUSH rule 3 x 696 [42,97]
+  CRUSH rule 3 x 697 [19,86]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,106]
+  CRUSH rule 3 x 700 [35,82]
+  CRUSH rule 3 x 701 [53,30]
+  CRUSH rule 3 x 702 [101,32]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,8]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,42]
+  CRUSH rule 3 x 708 [95,84]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,23]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,26]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,21]
+  CRUSH rule 3 x 716 [101,72]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,96]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,58]
+  CRUSH rule 3 x 722 [15,80]
+  CRUSH rule 3 x 723 [117,41]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,116]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,42]
+  CRUSH rule 3 x 730 [28,47]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,89]
+  CRUSH rule 3 x 733 [35,62]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,73]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,90]
+  CRUSH rule 3 x 742 [106,31]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,89]
+  CRUSH rule 3 x 749 [102,71]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,74]
+  CRUSH rule 3 x 752 [82,83]
+  CRUSH rule 3 x 753 [14,32]
+  CRUSH rule 3 x 754 [114,57]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,83]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,29]
+  CRUSH rule 3 x 760 [88,105]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,41]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,116]
+  CRUSH rule 3 x 768 [106,71]
+  CRUSH rule 3 x 769 [91,48]
+  CRUSH rule 3 x 770 [72,43]
+  CRUSH rule 3 x 771 [115,97]
+  CRUSH rule 3 x 772 [97,111]
+  CRUSH rule 3 x 773 [116,75]
+  CRUSH rule 3 x 774 [100,43]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,118]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,40]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,58]
+  CRUSH rule 3 x 787 [108,16]
+  CRUSH rule 3 x 788 [74,6]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,97]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,90]
+  CRUSH rule 3 x 797 [64,63]
+  CRUSH rule 3 x 798 [42,91]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,49]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,5]
+  CRUSH rule 3 x 803 [57,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,63]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,59]
+  CRUSH rule 3 x 809 [27,26]
+  CRUSH rule 3 x 810 [119,107]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,52]
+  CRUSH rule 3 x 813 [81,72]
+  CRUSH rule 3 x 814 [95,32]
+  CRUSH rule 3 x 815 [84,15]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,15]
+  CRUSH rule 3 x 820 [104,49]
+  CRUSH rule 3 x 821 [58,85]
+  CRUSH rule 3 x 822 [20,98]
+  CRUSH rule 3 x 823 [63,90]
+  CRUSH rule 3 x 824 [102,81]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,3]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,3]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,46]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,17]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,109]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,57]
+  CRUSH rule 3 x 845 [74,63]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,117]
+  CRUSH rule 3 x 852 [60,27]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,84]
+  CRUSH rule 3 x 855 [2,105]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,79]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,54]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,68]
+  CRUSH rule 3 x 865 [119,14]
+  CRUSH rule 3 x 866 [18,89]
+  CRUSH rule 3 x 867 [3,78]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,21]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,112]
+  CRUSH rule 3 x 874 [21,44]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,75]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,96]
+  CRUSH rule 3 x 881 [109,69]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,51]
+  CRUSH rule 3 x 884 [80,103]
+  CRUSH rule 3 x 885 [46,7]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,40]
+  CRUSH rule 3 x 889 [84,93]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,105]
+  CRUSH rule 3 x 892 [64,87]
+  CRUSH rule 3 x 893 [20,115]
+  CRUSH rule 3 x 894 [32,3]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,93]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,80]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,104]
+  CRUSH rule 3 x 903 [53,64]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,5]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,86]
+  CRUSH rule 3 x 911 [39,84]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,12]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,54]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,86]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,23]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,33]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,80]
+  CRUSH rule 3 x 930 [104,61]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,42]
+  CRUSH rule 3 x 933 [18,63]
+  CRUSH rule 3 x 934 [68,51]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,11]
+  CRUSH rule 3 x 938 [48,73]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,22]
+  CRUSH rule 3 x 942 [80,8]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,7]
+  CRUSH rule 3 x 945 [71,56]
+  CRUSH rule 3 x 946 [37,114]
+  CRUSH rule 3 x 947 [107,74]
+  CRUSH rule 3 x 948 [108,79]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,87]
+  CRUSH rule 3 x 953 [62,105]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,91]
+  CRUSH rule 3 x 957 [117,16]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,107]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,92]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,108]
+  CRUSH rule 3 x 968 [74,6]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,10]
+  CRUSH rule 3 x 972 [3,58]
+  CRUSH rule 3 x 973 [113,81]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,117]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [119,93]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,26]
+  CRUSH rule 3 x 981 [89,117]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,21]
+  CRUSH rule 3 x 984 [78,15]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,103]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,86]
+  CRUSH rule 3 x 990 [72,35]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,53]
+  CRUSH rule 3 x 994 [74,20]
+  CRUSH rule 3 x 995 [100,17]
+  CRUSH rule 3 x 996 [41,30]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,65]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,36]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,118]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,113]
+  CRUSH rule 3 x 1008 [31,36]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,34]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,71]
+  CRUSH rule 3 x 1013 [5,39]
+  CRUSH rule 3 x 1014 [33,80]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,109]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,109]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,108]
+  CRUSH rule 3 x 1022 [73,106]
+  CRUSH rule 3 x 1023 [88,89]
+  rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,45]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,118]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,96]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,70]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,76]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,100]
+  CRUSH rule 3 x 17 [85,110]
+  CRUSH rule 3 x 18 [11,119]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,47]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,90]
+  CRUSH rule 3 x 24 [83,38]
+  CRUSH rule 3 x 25 [81,5]
+  CRUSH rule 3 x 26 [17,100]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,119]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,69]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,52]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,29]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,11]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,68]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,21]
+  CRUSH rule 3 x 47 [106,53]
+  CRUSH rule 3 x 48 [34,33]
+  CRUSH rule 3 x 49 [99,104]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,52]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,75]
+  CRUSH rule 3 x 54 [28,93]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,40]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,33]
+  CRUSH rule 3 x 61 [88,3]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,113]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,61]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,7]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,44]
+  CRUSH rule 3 x 74 [29,74]
+  CRUSH rule 3 x 75 [60,89]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,7]
+  CRUSH rule 3 x 79 [64,41]
+  CRUSH rule 3 x 80 [73,104]
+  CRUSH rule 3 x 81 [64,19]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,68]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,95]
+  CRUSH rule 3 x 89 [76,3]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,65]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,23]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,100]
+  CRUSH rule 3 x 99 [78,22]
+  CRUSH rule 3 x 100 [28,63]
+  CRUSH rule 3 x 101 [91,36]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,95]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,116]
+  CRUSH rule 3 x 107 [1,55]
+  CRUSH rule 3 x 108 [7,72]
+  CRUSH rule 3 x 109 [112,63]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,78]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,54]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,42]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,16]
+  CRUSH rule 3 x 122 [45,110]
+  CRUSH rule 3 x 123 [22,86]
+  CRUSH rule 3 x 124 [97,72]
+  CRUSH rule 3 x 125 [66,71]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,97]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,110]
+  CRUSH rule 3 x 130 [50,63]
+  CRUSH rule 3 x 131 [44,25]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,2]
+  CRUSH rule 3 x 134 [37,94]
+  CRUSH rule 3 x 135 [78,17]
+  CRUSH rule 3 x 136 [32,91]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,62]
+  CRUSH rule 3 x 141 [89,96]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,87]
+  CRUSH rule 3 x 144 [13,86]
+  CRUSH rule 3 x 145 [77,60]
+  CRUSH rule 3 x 146 [12,53]
+  CRUSH rule 3 x 147 [2,59]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,58]
+  CRUSH rule 3 x 150 [14,54]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,33]
+  CRUSH rule 3 x 154 [19,111]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,5]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,58]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,31]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,47]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,107]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,34]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,2]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,1]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,90]
+  CRUSH rule 3 x 183 [11,74]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,113]
+  CRUSH rule 3 x 186 [67,96]
+  CRUSH rule 3 x 187 [6,36]
+  CRUSH rule 3 x 188 [76,20]
+  CRUSH rule 3 x 189 [96,89]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,60]
+  CRUSH rule 3 x 192 [93,64]
+  CRUSH rule 3 x 193 [89,112]
+  CRUSH rule 3 x 194 [62,63]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [97,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,29]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,98]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,35]
+  CRUSH rule 3 x 207 [19,36]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,76]
+  CRUSH rule 3 x 211 [26,89]
+  CRUSH rule 3 x 212 [115,107]
+  CRUSH rule 3 x 213 [100,8]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,89]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,2]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,12]
+  CRUSH rule 3 x 222 [50,63]
+  CRUSH rule 3 x 223 [34,59]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,102]
+  CRUSH rule 3 x 226 [44,105]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,78]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,44]
+  CRUSH rule 3 x 233 [47,113]
+  CRUSH rule 3 x 234 [55,18]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,53]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,38]
+  CRUSH rule 3 x 242 [73,52]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,71]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,81]
+  CRUSH rule 3 x 250 [34,41]
+  CRUSH rule 3 x 251 [28,15]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,77]
+  CRUSH rule 3 x 256 [94,45]
+  CRUSH rule 3 x 257 [100,81]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,80]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,53]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,18]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,86]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,70]
+  CRUSH rule 3 x 271 [19,88]
+  CRUSH rule 3 x 272 [73,52]
+  CRUSH rule 3 x 273 [69,92]
+  CRUSH rule 3 x 274 [47,88]
+  CRUSH rule 3 x 275 [112,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,87]
+  CRUSH rule 3 x 278 [107,44]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,10]
+  CRUSH rule 3 x 281 [89,109]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,118]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,101]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,26]
+  CRUSH rule 3 x 290 [25,12]
+  CRUSH rule 3 x 291 [35,46]
+  CRUSH rule 3 x 292 [20,28]
+  CRUSH rule 3 x 293 [27,24]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,116]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,63]
+  CRUSH rule 3 x 298 [70,87]
+  CRUSH rule 3 x 299 [116,61]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,94]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,53]
+  CRUSH rule 3 x 306 [41,12]
+  CRUSH rule 3 x 307 [65,64]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,69]
+  CRUSH rule 3 x 311 [36,83]
+  CRUSH rule 3 x 312 [114,97]
+  CRUSH rule 3 x 313 [104,31]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,22]
+  CRUSH rule 3 x 316 [98,8]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,48]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,30]
+  CRUSH rule 3 x 330 [24,55]
+  CRUSH rule 3 x 331 [84,91]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,35]
+  CRUSH rule 3 x 335 [71,66]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,74]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,52]
+  CRUSH rule 3 x 341 [46,81]
+  CRUSH rule 3 x 342 [92,6]
+  CRUSH rule 3 x 343 [49,26]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,87]
+  CRUSH rule 3 x 346 [3,70]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,103]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,91]
+  CRUSH rule 3 x 353 [10,118]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,5]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,89]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,46]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,75]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,115]
+  CRUSH rule 3 x 366 [42,33]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,23]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,76]
+  CRUSH rule 3 x 378 [68,10]
+  CRUSH rule 3 x 379 [77,30]
+  CRUSH rule 3 x 380 [76,25]
+  CRUSH rule 3 x 381 [36,55]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,25]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,15]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,118]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,2]
+  CRUSH rule 3 x 393 [91,112]
+  CRUSH rule 3 x 394 [38,13]
+  CRUSH rule 3 x 395 [21,117]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,103]
+  CRUSH rule 3 x 400 [19,100]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,110]
+  CRUSH rule 3 x 403 [23,92]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,52]
+  CRUSH rule 3 x 407 [70,85]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,1]
+  CRUSH rule 3 x 410 [70,107]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,68]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,66]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,17]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,21]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,73]
+  CRUSH rule 3 x 427 [8,115]
+  CRUSH rule 3 x 428 [68,31]
+  CRUSH rule 3 x 429 [76,6]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,1]
+  CRUSH rule 3 x 434 [64,31]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,25]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,85]
+  CRUSH rule 3 x 439 [40,21]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,57]
+  CRUSH rule 3 x 442 [55,108]
+  CRUSH rule 3 x 443 [44,14]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,19]
+  CRUSH rule 3 x 449 [67,66]
+  CRUSH rule 3 x 450 [117,97]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,41]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,92]
+  CRUSH rule 3 x 456 [101,104]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,43]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,64]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,60]
+  CRUSH rule 3 x 469 [98,71]
+  CRUSH rule 3 x 470 [50,27]
+  CRUSH rule 3 x 471 [40,31]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,75]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,90]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,36]
+  CRUSH rule 3 x 485 [84,43]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,51]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,86]
+  CRUSH rule 3 x 493 [94,21]
+  CRUSH rule 3 x 494 [59,98]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,12]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,37]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,28]
+  CRUSH rule 3 x 503 [21,96]
+  CRUSH rule 3 x 504 [67,1]
+  CRUSH rule 3 x 505 [12,10]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,67]
+  CRUSH rule 3 x 508 [45,56]
+  CRUSH rule 3 x 509 [19,70]
+  CRUSH rule 3 x 510 [117,73]
+  CRUSH rule 3 x 511 [14,117]
+  CRUSH rule 3 x 512 [59,26]
+  CRUSH rule 3 x 513 [102,93]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,39]
+  CRUSH rule 3 x 516 [37,12]
+  CRUSH rule 3 x 517 [83,68]
+  CRUSH rule 3 x 518 [18,107]
+  CRUSH rule 3 x 519 [67,119]
+  CRUSH rule 3 x 520 [15,88]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,3]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,113]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,56]
+  CRUSH rule 3 x 528 [108,35]
+  CRUSH rule 3 x 529 [74,15]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,104]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,37]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,56]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,94]
+  CRUSH rule 3 x 541 [53,86]
+  CRUSH rule 3 x 542 [27,114]
+  CRUSH rule 3 x 543 [45,78]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,49]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,82]
+  CRUSH rule 3 x 549 [60,71]
+  CRUSH rule 3 x 550 [92,71]
+  CRUSH rule 3 x 551 [77,88]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,20]
+  CRUSH rule 3 x 556 [106,89]
+  CRUSH rule 3 x 557 [26,45]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,105]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,25]
+  CRUSH rule 3 x 563 [59,72]
+  CRUSH rule 3 x 564 [96,59]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,35]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,38]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,11]
+  CRUSH rule 3 x 574 [89,78]
+  CRUSH rule 3 x 575 [87,50]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,113]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,116]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,66]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,116]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,63]
+  CRUSH rule 3 x 591 [42,77]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,31]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,94]
+  CRUSH rule 3 x 598 [37,60]
+  CRUSH rule 3 x 599 [10,76]
+  CRUSH rule 3 x 600 [24,7]
+  CRUSH rule 3 x 601 [104,87]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,60]
+  CRUSH rule 3 x 604 [118,71]
+  CRUSH rule 3 x 605 [104,87]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,40]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,7]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [81,54]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,27]
+  CRUSH rule 3 x 620 [108,103]
+  CRUSH rule 3 x 621 [105,110]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,64]
+  CRUSH rule 3 x 624 [115,29]
+  CRUSH rule 3 x 625 [73,98]
+  CRUSH rule 3 x 626 [52,55]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,35]
+  CRUSH rule 3 x 629 [6,46]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,80]
+  CRUSH rule 3 x 632 [80,95]
+  CRUSH rule 3 x 633 [65,68]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,72]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,68]
+  CRUSH rule 3 x 638 [43,109]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,96]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,7]
+  CRUSH rule 3 x 644 [31,5]
+  CRUSH rule 3 x 645 [77,1]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,56]
+  CRUSH rule 3 x 648 [56,79]
+  CRUSH rule 3 x 649 [88,103]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,14]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [7,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,2]
+  CRUSH rule 3 x 660 [65,110]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,78]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,101]
+  CRUSH rule 3 x 667 [70,11]
+  CRUSH rule 3 x 668 [96,63]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,99]
+  CRUSH rule 3 x 671 [57,2]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,62]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,17]
+  CRUSH rule 3 x 676 [3,100]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,26]
+  CRUSH rule 3 x 680 [111,43]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,95]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,41]
+  CRUSH rule 3 x 690 [96,103]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,72]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,70]
+  CRUSH rule 3 x 695 [31,62]
+  CRUSH rule 3 x 696 [42,97]
+  CRUSH rule 3 x 697 [19,86]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,106]
+  CRUSH rule 3 x 700 [35,82]
+  CRUSH rule 3 x 701 [53,30]
+  CRUSH rule 3 x 702 [101,32]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,8]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,42]
+  CRUSH rule 3 x 708 [95,84]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,23]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,26]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,21]
+  CRUSH rule 3 x 716 [101,72]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,96]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,58]
+  CRUSH rule 3 x 722 [15,80]
+  CRUSH rule 3 x 723 [117,41]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,116]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,42]
+  CRUSH rule 3 x 730 [28,47]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,89]
+  CRUSH rule 3 x 733 [35,62]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,73]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,90]
+  CRUSH rule 3 x 742 [106,31]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,89]
+  CRUSH rule 3 x 749 [102,71]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,74]
+  CRUSH rule 3 x 752 [82,83]
+  CRUSH rule 3 x 753 [14,32]
+  CRUSH rule 3 x 754 [114,57]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,83]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,29]
+  CRUSH rule 3 x 760 [88,105]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,41]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,116]
+  CRUSH rule 3 x 768 [106,71]
+  CRUSH rule 3 x 769 [91,48]
+  CRUSH rule 3 x 770 [72,43]
+  CRUSH rule 3 x 771 [115,97]
+  CRUSH rule 3 x 772 [97,111]
+  CRUSH rule 3 x 773 [116,75]
+  CRUSH rule 3 x 774 [100,43]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,118]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,40]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,58]
+  CRUSH rule 3 x 787 [108,16]
+  CRUSH rule 3 x 788 [74,6]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,97]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,90]
+  CRUSH rule 3 x 797 [64,63]
+  CRUSH rule 3 x 798 [42,91]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,49]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,5]
+  CRUSH rule 3 x 803 [57,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,63]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,59]
+  CRUSH rule 3 x 809 [27,26]
+  CRUSH rule 3 x 810 [119,107]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,52]
+  CRUSH rule 3 x 813 [81,72]
+  CRUSH rule 3 x 814 [95,32]
+  CRUSH rule 3 x 815 [84,15]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,15]
+  CRUSH rule 3 x 820 [104,49]
+  CRUSH rule 3 x 821 [58,85]
+  CRUSH rule 3 x 822 [20,98]
+  CRUSH rule 3 x 823 [63,90]
+  CRUSH rule 3 x 824 [102,81]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,3]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,3]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,46]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,17]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,109]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,57]
+  CRUSH rule 3 x 845 [74,63]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,117]
+  CRUSH rule 3 x 852 [60,27]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,84]
+  CRUSH rule 3 x 855 [2,105]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,79]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,54]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,68]
+  CRUSH rule 3 x 865 [119,14]
+  CRUSH rule 3 x 866 [18,89]
+  CRUSH rule 3 x 867 [3,78]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,21]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,112]
+  CRUSH rule 3 x 874 [21,44]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,75]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,96]
+  CRUSH rule 3 x 881 [109,69]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,51]
+  CRUSH rule 3 x 884 [80,103]
+  CRUSH rule 3 x 885 [46,7]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,40]
+  CRUSH rule 3 x 889 [84,93]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,105]
+  CRUSH rule 3 x 892 [64,87]
+  CRUSH rule 3 x 893 [20,115]
+  CRUSH rule 3 x 894 [32,3]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,93]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,80]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,104]
+  CRUSH rule 3 x 903 [53,64]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,5]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,86]
+  CRUSH rule 3 x 911 [39,84]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,12]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,54]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,86]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,23]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,33]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,80]
+  CRUSH rule 3 x 930 [104,61]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,42]
+  CRUSH rule 3 x 933 [18,63]
+  CRUSH rule 3 x 934 [68,51]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,11]
+  CRUSH rule 3 x 938 [48,73]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,22]
+  CRUSH rule 3 x 942 [80,8]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,7]
+  CRUSH rule 3 x 945 [71,56]
+  CRUSH rule 3 x 946 [37,114]
+  CRUSH rule 3 x 947 [107,74]
+  CRUSH rule 3 x 948 [108,79]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,87]
+  CRUSH rule 3 x 953 [62,105]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,91]
+  CRUSH rule 3 x 957 [117,16]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,107]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,92]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,108]
+  CRUSH rule 3 x 968 [74,6]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,10]
+  CRUSH rule 3 x 972 [3,58]
+  CRUSH rule 3 x 973 [113,81]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,117]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [119,93]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,26]
+  CRUSH rule 3 x 981 [89,117]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,21]
+  CRUSH rule 3 x 984 [78,15]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,103]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,86]
+  CRUSH rule 3 x 990 [72,35]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,53]
+  CRUSH rule 3 x 994 [74,20]
+  CRUSH rule 3 x 995 [100,17]
+  CRUSH rule 3 x 996 [41,30]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,65]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,36]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,118]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,113]
+  CRUSH rule 3 x 1008 [31,36]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,34]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,71]
+  CRUSH rule 3 x 1013 [5,39]
+  CRUSH rule 3 x 1014 [33,80]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,109]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,109]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,108]
+  CRUSH rule 3 x 1022 [73,106]
+  CRUSH rule 3 x 1023 [88,89]
+  rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,45]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,118]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,96]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,70]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,76]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,100]
+  CRUSH rule 3 x 17 [85,110]
+  CRUSH rule 3 x 18 [11,119]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,47]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,90]
+  CRUSH rule 3 x 24 [83,38]
+  CRUSH rule 3 x 25 [81,5]
+  CRUSH rule 3 x 26 [17,100]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,119]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,69]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,52]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,29]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,11]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,68]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,21]
+  CRUSH rule 3 x 47 [106,53]
+  CRUSH rule 3 x 48 [34,33]
+  CRUSH rule 3 x 49 [99,104]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,52]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,75]
+  CRUSH rule 3 x 54 [28,93]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,40]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,33]
+  CRUSH rule 3 x 61 [88,3]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,113]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,61]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,7]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,44]
+  CRUSH rule 3 x 74 [29,74]
+  CRUSH rule 3 x 75 [60,89]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,7]
+  CRUSH rule 3 x 79 [64,41]
+  CRUSH rule 3 x 80 [73,104]
+  CRUSH rule 3 x 81 [64,19]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,68]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,95]
+  CRUSH rule 3 x 89 [76,3]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,65]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,23]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,100]
+  CRUSH rule 3 x 99 [78,22]
+  CRUSH rule 3 x 100 [28,63]
+  CRUSH rule 3 x 101 [91,36]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,95]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,116]
+  CRUSH rule 3 x 107 [1,55]
+  CRUSH rule 3 x 108 [7,72]
+  CRUSH rule 3 x 109 [112,63]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,78]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,54]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,42]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,16]
+  CRUSH rule 3 x 122 [45,110]
+  CRUSH rule 3 x 123 [22,86]
+  CRUSH rule 3 x 124 [97,72]
+  CRUSH rule 3 x 125 [66,71]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,97]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,110]
+  CRUSH rule 3 x 130 [50,63]
+  CRUSH rule 3 x 131 [44,25]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,2]
+  CRUSH rule 3 x 134 [37,94]
+  CRUSH rule 3 x 135 [78,17]
+  CRUSH rule 3 x 136 [32,91]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,62]
+  CRUSH rule 3 x 141 [89,96]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,87]
+  CRUSH rule 3 x 144 [13,86]
+  CRUSH rule 3 x 145 [77,60]
+  CRUSH rule 3 x 146 [12,53]
+  CRUSH rule 3 x 147 [2,59]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,58]
+  CRUSH rule 3 x 150 [14,54]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,33]
+  CRUSH rule 3 x 154 [19,111]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,5]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,58]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,31]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,47]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,107]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,34]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,2]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,1]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,90]
+  CRUSH rule 3 x 183 [11,74]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,113]
+  CRUSH rule 3 x 186 [67,96]
+  CRUSH rule 3 x 187 [6,36]
+  CRUSH rule 3 x 188 [76,20]
+  CRUSH rule 3 x 189 [96,89]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,60]
+  CRUSH rule 3 x 192 [93,64]
+  CRUSH rule 3 x 193 [89,112]
+  CRUSH rule 3 x 194 [62,63]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [97,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,29]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,98]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,35]
+  CRUSH rule 3 x 207 [19,36]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,76]
+  CRUSH rule 3 x 211 [26,89]
+  CRUSH rule 3 x 212 [115,107]
+  CRUSH rule 3 x 213 [100,8]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,89]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,2]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,12]
+  CRUSH rule 3 x 222 [50,63]
+  CRUSH rule 3 x 223 [34,59]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,102]
+  CRUSH rule 3 x 226 [44,105]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,78]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,44]
+  CRUSH rule 3 x 233 [47,113]
+  CRUSH rule 3 x 234 [55,18]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,53]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,38]
+  CRUSH rule 3 x 242 [73,52]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,71]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,81]
+  CRUSH rule 3 x 250 [34,41]
+  CRUSH rule 3 x 251 [28,15]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,77]
+  CRUSH rule 3 x 256 [94,45]
+  CRUSH rule 3 x 257 [100,81]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,80]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,53]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,18]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,86]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,70]
+  CRUSH rule 3 x 271 [19,88]
+  CRUSH rule 3 x 272 [73,52]
+  CRUSH rule 3 x 273 [69,92]
+  CRUSH rule 3 x 274 [47,88]
+  CRUSH rule 3 x 275 [112,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,87]
+  CRUSH rule 3 x 278 [107,44]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,10]
+  CRUSH rule 3 x 281 [89,109]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,118]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,101]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,26]
+  CRUSH rule 3 x 290 [25,12]
+  CRUSH rule 3 x 291 [35,46]
+  CRUSH rule 3 x 292 [20,28]
+  CRUSH rule 3 x 293 [27,24]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,116]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,63]
+  CRUSH rule 3 x 298 [70,87]
+  CRUSH rule 3 x 299 [116,61]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,94]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,53]
+  CRUSH rule 3 x 306 [41,12]
+  CRUSH rule 3 x 307 [65,64]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,69]
+  CRUSH rule 3 x 311 [36,83]
+  CRUSH rule 3 x 312 [114,97]
+  CRUSH rule 3 x 313 [104,31]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,22]
+  CRUSH rule 3 x 316 [98,8]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,48]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,30]
+  CRUSH rule 3 x 330 [24,55]
+  CRUSH rule 3 x 331 [84,91]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,35]
+  CRUSH rule 3 x 335 [71,66]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,74]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,52]
+  CRUSH rule 3 x 341 [46,81]
+  CRUSH rule 3 x 342 [92,6]
+  CRUSH rule 3 x 343 [49,26]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,87]
+  CRUSH rule 3 x 346 [3,70]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,103]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,91]
+  CRUSH rule 3 x 353 [10,118]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,5]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,89]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,46]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,75]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,115]
+  CRUSH rule 3 x 366 [42,33]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,23]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,76]
+  CRUSH rule 3 x 378 [68,10]
+  CRUSH rule 3 x 379 [77,30]
+  CRUSH rule 3 x 380 [76,25]
+  CRUSH rule 3 x 381 [36,55]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,25]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,15]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,118]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,2]
+  CRUSH rule 3 x 393 [91,112]
+  CRUSH rule 3 x 394 [38,13]
+  CRUSH rule 3 x 395 [21,117]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,103]
+  CRUSH rule 3 x 400 [19,100]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,110]
+  CRUSH rule 3 x 403 [23,92]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,52]
+  CRUSH rule 3 x 407 [70,85]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,1]
+  CRUSH rule 3 x 410 [70,107]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,68]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,66]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,17]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,21]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,73]
+  CRUSH rule 3 x 427 [8,115]
+  CRUSH rule 3 x 428 [68,31]
+  CRUSH rule 3 x 429 [76,6]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,1]
+  CRUSH rule 3 x 434 [64,31]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,25]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,85]
+  CRUSH rule 3 x 439 [40,21]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,57]
+  CRUSH rule 3 x 442 [55,108]
+  CRUSH rule 3 x 443 [44,14]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,19]
+  CRUSH rule 3 x 449 [67,66]
+  CRUSH rule 3 x 450 [117,97]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,41]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,92]
+  CRUSH rule 3 x 456 [101,104]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,43]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,64]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,60]
+  CRUSH rule 3 x 469 [98,71]
+  CRUSH rule 3 x 470 [50,27]
+  CRUSH rule 3 x 471 [40,31]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,75]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,90]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,36]
+  CRUSH rule 3 x 485 [84,43]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,51]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,86]
+  CRUSH rule 3 x 493 [94,21]
+  CRUSH rule 3 x 494 [59,98]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,12]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,37]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,28]
+  CRUSH rule 3 x 503 [21,96]
+  CRUSH rule 3 x 504 [67,1]
+  CRUSH rule 3 x 505 [12,10]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,67]
+  CRUSH rule 3 x 508 [45,56]
+  CRUSH rule 3 x 509 [19,70]
+  CRUSH rule 3 x 510 [117,73]
+  CRUSH rule 3 x 511 [14,117]
+  CRUSH rule 3 x 512 [59,26]
+  CRUSH rule 3 x 513 [102,93]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,39]
+  CRUSH rule 3 x 516 [37,12]
+  CRUSH rule 3 x 517 [83,68]
+  CRUSH rule 3 x 518 [18,107]
+  CRUSH rule 3 x 519 [67,119]
+  CRUSH rule 3 x 520 [15,88]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,3]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,113]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,56]
+  CRUSH rule 3 x 528 [108,35]
+  CRUSH rule 3 x 529 [74,15]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,104]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,37]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,56]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,94]
+  CRUSH rule 3 x 541 [53,86]
+  CRUSH rule 3 x 542 [27,114]
+  CRUSH rule 3 x 543 [45,78]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,49]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,82]
+  CRUSH rule 3 x 549 [60,71]
+  CRUSH rule 3 x 550 [92,71]
+  CRUSH rule 3 x 551 [77,88]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,20]
+  CRUSH rule 3 x 556 [106,89]
+  CRUSH rule 3 x 557 [26,45]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,105]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,25]
+  CRUSH rule 3 x 563 [59,72]
+  CRUSH rule 3 x 564 [96,59]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,35]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,38]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,11]
+  CRUSH rule 3 x 574 [89,78]
+  CRUSH rule 3 x 575 [87,50]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,113]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,116]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,66]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,116]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,63]
+  CRUSH rule 3 x 591 [42,77]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,31]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,94]
+  CRUSH rule 3 x 598 [37,60]
+  CRUSH rule 3 x 599 [10,76]
+  CRUSH rule 3 x 600 [24,7]
+  CRUSH rule 3 x 601 [104,87]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,60]
+  CRUSH rule 3 x 604 [118,71]
+  CRUSH rule 3 x 605 [104,87]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,40]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,7]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [81,54]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,27]
+  CRUSH rule 3 x 620 [108,103]
+  CRUSH rule 3 x 621 [105,110]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,64]
+  CRUSH rule 3 x 624 [115,29]
+  CRUSH rule 3 x 625 [73,98]
+  CRUSH rule 3 x 626 [52,55]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,35]
+  CRUSH rule 3 x 629 [6,46]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,80]
+  CRUSH rule 3 x 632 [80,95]
+  CRUSH rule 3 x 633 [65,68]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,72]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,68]
+  CRUSH rule 3 x 638 [43,109]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,96]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,7]
+  CRUSH rule 3 x 644 [31,5]
+  CRUSH rule 3 x 645 [77,1]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,56]
+  CRUSH rule 3 x 648 [56,79]
+  CRUSH rule 3 x 649 [88,103]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,14]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [7,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,2]
+  CRUSH rule 3 x 660 [65,110]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,78]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,101]
+  CRUSH rule 3 x 667 [70,11]
+  CRUSH rule 3 x 668 [96,63]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,99]
+  CRUSH rule 3 x 671 [57,2]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,62]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,17]
+  CRUSH rule 3 x 676 [3,100]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,26]
+  CRUSH rule 3 x 680 [111,43]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,95]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,41]
+  CRUSH rule 3 x 690 [96,103]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,72]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,70]
+  CRUSH rule 3 x 695 [31,62]
+  CRUSH rule 3 x 696 [42,97]
+  CRUSH rule 3 x 697 [19,86]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,106]
+  CRUSH rule 3 x 700 [35,82]
+  CRUSH rule 3 x 701 [53,30]
+  CRUSH rule 3 x 702 [101,32]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,8]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,42]
+  CRUSH rule 3 x 708 [95,84]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,23]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,26]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,21]
+  CRUSH rule 3 x 716 [101,72]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,96]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,58]
+  CRUSH rule 3 x 722 [15,80]
+  CRUSH rule 3 x 723 [117,41]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,116]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,42]
+  CRUSH rule 3 x 730 [28,47]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,89]
+  CRUSH rule 3 x 733 [35,62]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,73]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,90]
+  CRUSH rule 3 x 742 [106,31]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,89]
+  CRUSH rule 3 x 749 [102,71]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,74]
+  CRUSH rule 3 x 752 [82,83]
+  CRUSH rule 3 x 753 [14,32]
+  CRUSH rule 3 x 754 [114,57]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,83]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,29]
+  CRUSH rule 3 x 760 [88,105]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,41]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,116]
+  CRUSH rule 3 x 768 [106,71]
+  CRUSH rule 3 x 769 [91,48]
+  CRUSH rule 3 x 770 [72,43]
+  CRUSH rule 3 x 771 [115,97]
+  CRUSH rule 3 x 772 [97,111]
+  CRUSH rule 3 x 773 [116,75]
+  CRUSH rule 3 x 774 [100,43]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,118]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,40]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,58]
+  CRUSH rule 3 x 787 [108,16]
+  CRUSH rule 3 x 788 [74,6]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,97]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,90]
+  CRUSH rule 3 x 797 [64,63]
+  CRUSH rule 3 x 798 [42,91]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,49]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,5]
+  CRUSH rule 3 x 803 [57,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,63]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,59]
+  CRUSH rule 3 x 809 [27,26]
+  CRUSH rule 3 x 810 [119,107]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,52]
+  CRUSH rule 3 x 813 [81,72]
+  CRUSH rule 3 x 814 [95,32]
+  CRUSH rule 3 x 815 [84,15]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,15]
+  CRUSH rule 3 x 820 [104,49]
+  CRUSH rule 3 x 821 [58,85]
+  CRUSH rule 3 x 822 [20,98]
+  CRUSH rule 3 x 823 [63,90]
+  CRUSH rule 3 x 824 [102,81]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,3]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,3]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,46]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,17]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,109]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,57]
+  CRUSH rule 3 x 845 [74,63]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,117]
+  CRUSH rule 3 x 852 [60,27]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,84]
+  CRUSH rule 3 x 855 [2,105]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,79]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,54]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,68]
+  CRUSH rule 3 x 865 [119,14]
+  CRUSH rule 3 x 866 [18,89]
+  CRUSH rule 3 x 867 [3,78]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,21]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,112]
+  CRUSH rule 3 x 874 [21,44]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,75]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,96]
+  CRUSH rule 3 x 881 [109,69]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,51]
+  CRUSH rule 3 x 884 [80,103]
+  CRUSH rule 3 x 885 [46,7]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,40]
+  CRUSH rule 3 x 889 [84,93]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,105]
+  CRUSH rule 3 x 892 [64,87]
+  CRUSH rule 3 x 893 [20,115]
+  CRUSH rule 3 x 894 [32,3]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,93]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,80]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,104]
+  CRUSH rule 3 x 903 [53,64]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,5]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,86]
+  CRUSH rule 3 x 911 [39,84]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,12]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,54]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,86]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,23]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,33]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,80]
+  CRUSH rule 3 x 930 [104,61]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,42]
+  CRUSH rule 3 x 933 [18,63]
+  CRUSH rule 3 x 934 [68,51]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,11]
+  CRUSH rule 3 x 938 [48,73]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,22]
+  CRUSH rule 3 x 942 [80,8]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,7]
+  CRUSH rule 3 x 945 [71,56]
+  CRUSH rule 3 x 946 [37,114]
+  CRUSH rule 3 x 947 [107,74]
+  CRUSH rule 3 x 948 [108,79]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,87]
+  CRUSH rule 3 x 953 [62,105]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,91]
+  CRUSH rule 3 x 957 [117,16]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,107]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,92]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,108]
+  CRUSH rule 3 x 968 [74,6]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,10]
+  CRUSH rule 3 x 972 [3,58]
+  CRUSH rule 3 x 973 [113,81]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,117]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [119,93]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,26]
+  CRUSH rule 3 x 981 [89,117]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,21]
+  CRUSH rule 3 x 984 [78,15]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,103]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,86]
+  CRUSH rule 3 x 990 [72,35]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,53]
+  CRUSH rule 3 x 994 [74,20]
+  CRUSH rule 3 x 995 [100,17]
+  CRUSH rule 3 x 996 [41,30]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,65]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,36]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,118]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,113]
+  CRUSH rule 3 x 1008 [31,36]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,34]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,71]
+  CRUSH rule 3 x 1013 [5,39]
+  CRUSH rule 3 x 1014 [33,80]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,109]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,109]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,108]
+  CRUSH rule 3 x 1022 [73,106]
+  CRUSH rule 3 x 1023 [88,89]
+  rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-vary-r-3.t b/ceph/src/test/cli/crushtool/test-map-vary-r-3.t
new file mode 100644
index 00000000..ad02e737
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-vary-r-3.t
@@ -0,0 +1,3078 @@
+  $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 3 --weight 0 0 --weight 4 0 --weight 9 0
+  crushtool successfully built or modified map.  Use '-o ' to write it out.
+  rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,62]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,60]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,48]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,111]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,20]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,85]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,12]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,93]
+  CRUSH rule 3 x 54 [28,3]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,91]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,92]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,16]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,89]
+  CRUSH rule 3 x 79 [64,75]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,57]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,117]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,2]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,55]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,29]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,119]
+  CRUSH rule 3 x 146 [12,73]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,8]
+  CRUSH rule 3 x 154 [19,5]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,10]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,47]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,86]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,17]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,38]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,118]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,32]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,54]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,104]
+  CRUSH rule 3 x 234 [55,94]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,105]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,41]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,22]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,66]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,74]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,69]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,112]
+  CRUSH rule 3 x 291 [35,52]
+  CRUSH rule 3 x 292 [20,60]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,66]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,55]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,80]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,16]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,16]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,92]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,88]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,40]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,113]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,101]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,58]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,94]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,114]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,20]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,92]
+  CRUSH rule 3 x 410 [70,11]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,21]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,15]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,22]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,34]
+  CRUSH rule 3 x 456 [101,119]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,2]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,109]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,13]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,113]
+  CRUSH rule 3 x 493 [94,105]
+  CRUSH rule 3 x 494 [66,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,78]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,81]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,54]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,17]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59,111]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,60]
+  CRUSH rule 3 x 518 [18,13]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,119]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,66]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,21]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,18]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,29]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,108]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,62]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,40]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,64]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,30]
+  CRUSH rule 3 x 638 [43,113]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,97]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,45]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,87]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,82]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,86]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,50]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3,40]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,84]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36,97]
+  CRUSH rule 3 x 697 [19,60]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,62]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,94]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,30]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,56]
+  CRUSH rule 3 x 723 [117,25]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,48]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,44]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,119]
+  CRUSH rule 3 x 752 [82,47]
+  CRUSH rule 3 x 753 [24,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,27]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,80]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,37]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,75]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,29]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,30]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,91]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,88]
+  CRUSH rule 3 x 824 [102,81]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,11]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,35]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,85]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,77]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,16]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,98]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,58]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,12]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,41]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,24]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,21]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,37]
+  CRUSH rule 3 x 942 [80,8]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,52]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,55]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,78]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,37]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,22]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,38]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,98]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,59]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,31]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,10]
+  CRUSH rule 3 x 1014 [33,98]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,44]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [83,88]
+  rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,62]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,60]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,48]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,111]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,20]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,85]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,12]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,93]
+  CRUSH rule 3 x 54 [28,3]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,91]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,92]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,16]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,89]
+  CRUSH rule 3 x 79 [64,75]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,57]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,117]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,2]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,55]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,29]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,119]
+  CRUSH rule 3 x 146 [12,73]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,8]
+  CRUSH rule 3 x 154 [19,5]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,10]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,47]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,86]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,17]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,38]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,118]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,32]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,54]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,104]
+  CRUSH rule 3 x 234 [55,94]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,105]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,41]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,22]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,66]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,74]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,69]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,112]
+  CRUSH rule 3 x 291 [35,52]
+  CRUSH rule 3 x 292 [20,60]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,66]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,55]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,80]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,16]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,16]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,92]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,88]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,40]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,113]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,101]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,58]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,94]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,114]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,20]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,92]
+  CRUSH rule 3 x 410 [70,11]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,21]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,15]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,22]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,34]
+  CRUSH rule 3 x 456 [101,119]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,2]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,109]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,13]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,113]
+  CRUSH rule 3 x 493 [94,105]
+  CRUSH rule 3 x 494 [66,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,78]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,81]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,54]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,17]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59,111]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,60]
+  CRUSH rule 3 x 518 [18,13]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,119]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,66]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,21]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,18]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,29]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,108]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,62]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,40]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,64]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,30]
+  CRUSH rule 3 x 638 [43,113]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,97]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,45]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,87]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,82]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,86]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,50]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3,40]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,84]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36,97]
+  CRUSH rule 3 x 697 [19,60]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,62]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,94]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,30]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,56]
+  CRUSH rule 3 x 723 [117,25]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,48]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,44]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,119]
+  CRUSH rule 3 x 752 [82,47]
+  CRUSH rule 3 x 753 [24,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,27]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,80]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,37]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,75]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,29]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,30]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,91]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,88]
+  CRUSH rule 3 x 824 [102,81]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,11]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,35]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,85]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,77]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,16]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,98]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,58]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,12]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,41]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,24]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,21]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,37]
+  CRUSH rule 3 x 942 [80,8]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,52]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,55]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,78]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,37]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,22]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,38]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,98]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,59]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,31]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,10]
+  CRUSH rule 3 x 1014 [33,98]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,44]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [83,88]
+  rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,62]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,60]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,48]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,111]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,20]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,85]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,12]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,93]
+  CRUSH rule 3 x 54 [28,3]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,91]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,92]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,16]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,89]
+  CRUSH rule 3 x 79 [64,75]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,57]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,117]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,2]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,55]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,29]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,119]
+  CRUSH rule 3 x 146 [12,73]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,8]
+  CRUSH rule 3 x 154 [19,5]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,10]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,47]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,86]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,17]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,38]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,118]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,32]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,54]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,104]
+  CRUSH rule 3 x 234 [55,94]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,105]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,41]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,22]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,66]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,74]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,69]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,112]
+  CRUSH rule 3 x 291 [35,52]
+  CRUSH rule 3 x 292 [20,60]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,66]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,55]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,80]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,16]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,16]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,92]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,88]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,40]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,113]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,101]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,58]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,94]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,114]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,20]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,92]
+  CRUSH rule 3 x 410 [70,11]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,21]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,15]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,22]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,34]
+  CRUSH rule 3 x 456 [101,119]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,2]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,109]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,13]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,113]
+  CRUSH rule 3 x 493 [94,105]
+  CRUSH rule 3 x 494 [66,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,78]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,81]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,54]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,17]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59,111]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,60]
+  CRUSH rule 3 x 518 [18,13]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,119]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,66]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,21]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,18]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,29]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,108]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,62]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,40]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,64]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,30]
+  CRUSH rule 3 x 638 [43,113]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,97]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,45]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,87]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,82]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,86]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,50]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3,40]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,84]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36,97]
+  CRUSH rule 3 x 697 [19,60]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,62]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,94]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,30]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,56]
+  CRUSH rule 3 x 723 [117,25]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,48]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,44]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,119]
+  CRUSH rule 3 x 752 [82,47]
+  CRUSH rule 3 x 753 [24,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,27]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,80]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,37]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,75]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,29]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,30]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,91]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,88]
+  CRUSH rule 3 x 824 [102,81]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,11]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,35]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,85]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,77]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,16]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,98]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,58]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,12]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,41]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,24]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,21]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,37]
+  CRUSH rule 3 x 942 [80,8]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,52]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,55]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,78]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,37]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,22]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,38]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,98]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,59]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,31]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,10]
+  CRUSH rule 3 x 1014 [33,98]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,44]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [83,88]
+  rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-vary-r-4.t b/ceph/src/test/cli/crushtool/test-map-vary-r-4.t
new file mode 100644
index 00000000..059da771
--- /dev/null
+++ b/ceph/src/test/cli/crushtool/test-map-vary-r-4.t
@@ -0,0 +1,3078 @@
+  $ crushtool -i "$TESTDIR/test-map-vary-r.crushmap" --test --show-statistics --rule 3 --set-chooseleaf-vary-r 4 --weight 0 0 --weight 4 0 --weight 9 0
+  crushtool successfully built or modified map.  Use '-o ' to write it out.
+  rule 3 (delltestrule), x = 0..1023, numrep = 2..4
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,12]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,52]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,46]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,40]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,107]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,57]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,114]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,29]
+  CRUSH rule 3 x 54 [28,77]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,105]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,88]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,43]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,39]
+  CRUSH rule 3 x 79 [64,65]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,83]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,68]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,26]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,17]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,83]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,100]
+  CRUSH rule 3 x 146 [12,15]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,81]
+  CRUSH rule 3 x 154 [19,56]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,22]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,103]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,100]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,85]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,113]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,66]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,86]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,102]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,32]
+  CRUSH rule 3 x 234 [55,78]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,10]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,87]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,31]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,54]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,75]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,18]
+  CRUSH rule 3 x 291 [35,42]
+  CRUSH rule 3 x 292 [20,74]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,36]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,105]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,119]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,15]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,17]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,116]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,118]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,112]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,32]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,81]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,12]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,113]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,70]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,21]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,44]
+  CRUSH rule 3 x 410 [70,47]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,15]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,79]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,21]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,68]
+  CRUSH rule 3 x 456 [101,60]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,106]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,106]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,61]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,58]
+  CRUSH rule 3 x 493 [94,89]
+  CRUSH rule 3 x 494 [56,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,82]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,6]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,77]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59,111]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,30]
+  CRUSH rule 3 x 518 [18,37]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,104]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,92]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,19]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,116]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,35]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,32]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,24]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,44]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,50]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,114]
+  CRUSH rule 3 x 638 [43,78]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,99]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,39]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,107]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,102]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,100]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,116]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3,40]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,26]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36,97]
+  CRUSH rule 3 x 697 [19,38]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,60]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,113]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,38]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,26]
+  CRUSH rule 3 x 723 [117,75]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,38]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,94]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,56]
+  CRUSH rule 3 x 752 [82,16]
+  CRUSH rule 3 x 753 [116,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,65]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,90]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,53]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,93]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,61]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,94]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,19]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,102]
+  CRUSH rule 3 x 824 [102,95]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,33]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,75]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,39]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,31]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,29]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,66]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,62]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,80]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,25]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,60]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,107]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,10]
+  CRUSH rule 3 x 942 [80,37]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,111]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,16]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,46]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,75]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,19]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,100]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,113]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,33]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,22]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,35]
+  CRUSH rule 3 x 1014 [33,48]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,111]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [59,88]
+  rule 3 (delltestrule) num_rep 2 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,12]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,52]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,46]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,40]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,107]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,57]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,114]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,29]
+  CRUSH rule 3 x 54 [28,77]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,105]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,88]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,43]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,39]
+  CRUSH rule 3 x 79 [64,65]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,83]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,68]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,26]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,17]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,83]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,100]
+  CRUSH rule 3 x 146 [12,15]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,81]
+  CRUSH rule 3 x 154 [19,56]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,22]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,103]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,100]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,85]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,113]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,66]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,86]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,102]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,32]
+  CRUSH rule 3 x 234 [55,78]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,10]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,87]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,31]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,54]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,75]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,18]
+  CRUSH rule 3 x 291 [35,42]
+  CRUSH rule 3 x 292 [20,74]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,36]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,105]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,119]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,15]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,17]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,116]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,118]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,112]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,32]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,81]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,12]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,113]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,70]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,21]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,44]
+  CRUSH rule 3 x 410 [70,47]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,15]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,79]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,21]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,68]
+  CRUSH rule 3 x 456 [101,60]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,106]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,106]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,61]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,58]
+  CRUSH rule 3 x 493 [94,89]
+  CRUSH rule 3 x 494 [56,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,82]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,6]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,77]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59,111]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,30]
+  CRUSH rule 3 x 518 [18,37]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,104]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,92]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,19]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,116]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,35]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,32]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,24]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,44]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,50]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,114]
+  CRUSH rule 3 x 638 [43,78]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,99]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,39]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,107]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,102]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,100]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,116]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3,40]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,26]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36,97]
+  CRUSH rule 3 x 697 [19,38]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,60]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,113]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,38]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,26]
+  CRUSH rule 3 x 723 [117,75]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,38]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,94]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,56]
+  CRUSH rule 3 x 752 [82,16]
+  CRUSH rule 3 x 753 [116,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,65]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,90]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,53]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,93]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,61]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,94]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,19]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,102]
+  CRUSH rule 3 x 824 [102,95]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,33]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,75]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,39]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,31]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,29]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,66]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,62]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,80]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,25]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,60]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,107]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,10]
+  CRUSH rule 3 x 942 [80,37]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,111]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,16]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,46]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,75]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,19]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,100]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,113]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,33]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,22]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,35]
+  CRUSH rule 3 x 1014 [33,48]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,111]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [59,88]
+  rule 3 (delltestrule) num_rep 3 result size == 2:\t1024/1024 (esc)
+  CRUSH rule 3 x 0 [94,85]
+  CRUSH rule 3 x 1 [73,78]
+  CRUSH rule 3 x 2 [91,104]
+  CRUSH rule 3 x 3 [51,94]
+  CRUSH rule 3 x 4 [45,28]
+  CRUSH rule 3 x 5 [89,113]
+  CRUSH rule 3 x 6 [91,12]
+  CRUSH rule 3 x 7 [104,71]
+  CRUSH rule 3 x 8 [41,12]
+  CRUSH rule 3 x 9 [46,35]
+  CRUSH rule 3 x 10 [61,60]
+  CRUSH rule 3 x 11 [13,74]
+  CRUSH rule 3 x 12 [83,62]
+  CRUSH rule 3 x 13 [27,117]
+  CRUSH rule 3 x 14 [105,115]
+  CRUSH rule 3 x 15 [18,87]
+  CRUSH rule 3 x 16 [103,52]
+  CRUSH rule 3 x 17 [85,80]
+  CRUSH rule 3 x 18 [11,46]
+  CRUSH rule 3 x 19 [75,114]
+  CRUSH rule 3 x 20 [111,27]
+  CRUSH rule 3 x 21 [84,7]
+  CRUSH rule 3 x 22 [23,66]
+  CRUSH rule 3 x 23 [19,84]
+  CRUSH rule 3 x 24 [83,40]
+  CRUSH rule 3 x 25 [81,108]
+  CRUSH rule 3 x 26 [17,117]
+  CRUSH rule 3 x 27 [33,58]
+  CRUSH rule 3 x 28 [45,98]
+  CRUSH rule 3 x 29 [8,46]
+  CRUSH rule 3 x 30 [55,119]
+  CRUSH rule 3 x 31 [76,35]
+  CRUSH rule 3 x 32 [72,13]
+  CRUSH rule 3 x 33 [86,107]
+  CRUSH rule 3 x 34 [7,38]
+  CRUSH rule 3 x 35 [108,31]
+  CRUSH rule 3 x 36 [67,24]
+  CRUSH rule 3 x 37 [38,17]
+  CRUSH rule 3 x 38 [72,57]
+  CRUSH rule 3 x 39 [68,73]
+  CRUSH rule 3 x 40 [30,25]
+  CRUSH rule 3 x 41 [52,91]
+  CRUSH rule 3 x 42 [106,39]
+  CRUSH rule 3 x 43 [10,115]
+  CRUSH rule 3 x 44 [101,115]
+  CRUSH rule 3 x 45 [83,80]
+  CRUSH rule 3 x 46 [54,33]
+  CRUSH rule 3 x 47 [106,41]
+  CRUSH rule 3 x 48 [34,65]
+  CRUSH rule 3 x 49 [99,46]
+  CRUSH rule 3 x 50 [42,85]
+  CRUSH rule 3 x 51 [6,114]
+  CRUSH rule 3 x 52 [82,14]
+  CRUSH rule 3 x 53 [32,29]
+  CRUSH rule 3 x 54 [28,77]
+  CRUSH rule 3 x 55 [14,44]
+  CRUSH rule 3 x 56 [21,112]
+  CRUSH rule 3 x 57 [93,26]
+  CRUSH rule 3 x 58 [48,95]
+  CRUSH rule 3 x 59 [21,104]
+  CRUSH rule 3 x 60 [90,75]
+  CRUSH rule 3 x 61 [88,39]
+  CRUSH rule 3 x 62 [100,8]
+  CRUSH rule 3 x 63 [79,96]
+  CRUSH rule 3 x 64 [1,77]
+  CRUSH rule 3 x 65 [32,25]
+  CRUSH rule 3 x 66 [48,93]
+  CRUSH rule 3 x 67 [94,91]
+  CRUSH rule 3 x 68 [102,105]
+  CRUSH rule 3 x 69 [62,20]
+  CRUSH rule 3 x 70 [84,27]
+  CRUSH rule 3 x 71 [12,99]
+  CRUSH rule 3 x 72 [26,69]
+  CRUSH rule 3 x 73 [29,88]
+  CRUSH rule 3 x 74 [29,60]
+  CRUSH rule 3 x 75 [60,43]
+  CRUSH rule 3 x 76 [55,60]
+  CRUSH rule 3 x 77 [107,78]
+  CRUSH rule 3 x 78 [86,39]
+  CRUSH rule 3 x 79 [64,65]
+  CRUSH rule 3 x 80 [73,26]
+  CRUSH rule 3 x 81 [64,57]
+  CRUSH rule 3 x 82 [37,1]
+  CRUSH rule 3 x 83 [92,22]
+  CRUSH rule 3 x 84 [49,40]
+  CRUSH rule 3 x 85 [87,30]
+  CRUSH rule 3 x 86 [37,119]
+  CRUSH rule 3 x 87 [116,3]
+  CRUSH rule 3 x 88 [38,22]
+  CRUSH rule 3 x 89 [76,41]
+  CRUSH rule 3 x 90 [14,98]
+  CRUSH rule 3 x 91 [68,27]
+  CRUSH rule 3 x 92 [86,13]
+  CRUSH rule 3 x 93 [44,83]
+  CRUSH rule 3 x 94 [46,15]
+  CRUSH rule 3 x 95 [108,6]
+  CRUSH rule 3 x 96 [66,25]
+  CRUSH rule 3 x 97 [111,33]
+  CRUSH rule 3 x 98 [93,36]
+  CRUSH rule 3 x 99 [78,17]
+  CRUSH rule 3 x 100 [28,55]
+  CRUSH rule 3 x 101 [91,34]
+  CRUSH rule 3 x 102 [82,93]
+  CRUSH rule 3 x 103 [66,105]
+  CRUSH rule 3 x 104 [116,10]
+  CRUSH rule 3 x 105 [34,69]
+  CRUSH rule 3 x 106 [69,66]
+  CRUSH rule 3 x 107 [1,41]
+  CRUSH rule 3 x 108 [7,68]
+  CRUSH rule 3 x 109 [112,87]
+  CRUSH rule 3 x 110 [54,10]
+  CRUSH rule 3 x 111 [10,86]
+  CRUSH rule 3 x 112 [80,29]
+  CRUSH rule 3 x 113 [69,26]
+  CRUSH rule 3 x 114 [79,46]
+  CRUSH rule 3 x 115 [10,111]
+  CRUSH rule 3 x 116 [37,86]
+  CRUSH rule 3 x 117 [87,50]
+  CRUSH rule 3 x 118 [23,106]
+  CRUSH rule 3 x 119 [104,14]
+  CRUSH rule 3 x 120 [44,3]
+  CRUSH rule 3 x 121 [80,14]
+  CRUSH rule 3 x 122 [45,68]
+  CRUSH rule 3 x 123 [112,22]
+  CRUSH rule 3 x 124 [97,118]
+  CRUSH rule 3 x 125 [66,7]
+  CRUSH rule 3 x 126 [70,23]
+  CRUSH rule 3 x 127 [70,13]
+  CRUSH rule 3 x 128 [11,119]
+  CRUSH rule 3 x 129 [103,108]
+  CRUSH rule 3 x 130 [50,17]
+  CRUSH rule 3 x 131 [44,55]
+  CRUSH rule 3 x 132 [69,1]
+  CRUSH rule 3 x 133 [67,104]
+  CRUSH rule 3 x 134 [37,66]
+  CRUSH rule 3 x 135 [78,101]
+  CRUSH rule 3 x 136 [32,83]
+  CRUSH rule 3 x 137 [92,81]
+  CRUSH rule 3 x 138 [54,17]
+  CRUSH rule 3 x 139 [89,92]
+  CRUSH rule 3 x 140 [39,1]
+  CRUSH rule 3 x 141 [89,28]
+  CRUSH rule 3 x 142 [22,26]
+  CRUSH rule 3 x 143 [96,77]
+  CRUSH rule 3 x 144 [13,111]
+  CRUSH rule 3 x 145 [77,100]
+  CRUSH rule 3 x 146 [12,15]
+  CRUSH rule 3 x 147 [2,11]
+  CRUSH rule 3 x 148 [85,108]
+  CRUSH rule 3 x 149 [103,62]
+  CRUSH rule 3 x 150 [14,78]
+  CRUSH rule 3 x 151 [75,119]
+  CRUSH rule 3 x 152 [49,84]
+  CRUSH rule 3 x 153 [92,81]
+  CRUSH rule 3 x 154 [19,56]
+  CRUSH rule 3 x 155 [12,75]
+  CRUSH rule 3 x 156 [107,112]
+  CRUSH rule 3 x 157 [15,28]
+  CRUSH rule 3 x 158 [11,113]
+  CRUSH rule 3 x 159 [33,52]
+  CRUSH rule 3 x 160 [86,35]
+  CRUSH rule 3 x 161 [19,117]
+  CRUSH rule 3 x 162 [55,113]
+  CRUSH rule 3 x 163 [54,87]
+  CRUSH rule 3 x 164 [72,8]
+  CRUSH rule 3 x 165 [25,74]
+  CRUSH rule 3 x 166 [2,22]
+  CRUSH rule 3 x 167 [89,56]
+  CRUSH rule 3 x 168 [68,103]
+  CRUSH rule 3 x 169 [51,12]
+  CRUSH rule 3 x 170 [68,53]
+  CRUSH rule 3 x 171 [88,79]
+  CRUSH rule 3 x 172 [117,89]
+  CRUSH rule 3 x 173 [29,40]
+  CRUSH rule 3 x 174 [67,86]
+  CRUSH rule 3 x 175 [48,85]
+  CRUSH rule 3 x 176 [94,83]
+  CRUSH rule 3 x 177 [53,18]
+  CRUSH rule 3 x 178 [39,30]
+  CRUSH rule 3 x 179 [72,17]
+  CRUSH rule 3 x 180 [3,114]
+  CRUSH rule 3 x 181 [18,16]
+  CRUSH rule 3 x 182 [75,5]
+  CRUSH rule 3 x 183 [11,110]
+  CRUSH rule 3 x 184 [79,48]
+  CRUSH rule 3 x 185 [97,100]
+  CRUSH rule 3 x 186 [67,44]
+  CRUSH rule 3 x 187 [6,50]
+  CRUSH rule 3 x 188 [76,85]
+  CRUSH rule 3 x 189 [96,7]
+  CRUSH rule 3 x 190 [90,95]
+  CRUSH rule 3 x 191 [49,113]
+  CRUSH rule 3 x 192 [93,58]
+  CRUSH rule 3 x 193 [89,66]
+  CRUSH rule 3 x 194 [62,3]
+  CRUSH rule 3 x 195 [119,85]
+  CRUSH rule 3 x 196 [20,72]
+  CRUSH rule 3 x 197 [6,116]
+  CRUSH rule 3 x 198 [55,92]
+  CRUSH rule 3 x 199 [77,66]
+  CRUSH rule 3 x 200 [12,81]
+  CRUSH rule 3 x 201 [52,71]
+  CRUSH rule 3 x 202 [98,59]
+  CRUSH rule 3 x 203 [36,19]
+  CRUSH rule 3 x 204 [10,113]
+  CRUSH rule 3 x 205 [38,79]
+  CRUSH rule 3 x 206 [38,105]
+  CRUSH rule 3 x 207 [19,86]
+  CRUSH rule 3 x 208 [63,92]
+  CRUSH rule 3 x 209 [70,99]
+  CRUSH rule 3 x 210 [79,102]
+  CRUSH rule 3 x 211 [26,27]
+  CRUSH rule 3 x 212 [28,107]
+  CRUSH rule 3 x 213 [100,49]
+  CRUSH rule 3 x 214 [91,88]
+  CRUSH rule 3 x 215 [92,7]
+  CRUSH rule 3 x 216 [99,108]
+  CRUSH rule 3 x 217 [86,97]
+  CRUSH rule 3 x 218 [70,10]
+  CRUSH rule 3 x 219 [61,112]
+  CRUSH rule 3 x 220 [23,66]
+  CRUSH rule 3 x 221 [51,66]
+  CRUSH rule 3 x 222 [50,65]
+  CRUSH rule 3 x 223 [34,45]
+  CRUSH rule 3 x 224 [107,44]
+  CRUSH rule 3 x 225 [61,102]
+  CRUSH rule 3 x 226 [44,87]
+  CRUSH rule 3 x 227 [55,66]
+  CRUSH rule 3 x 228 [117,103]
+  CRUSH rule 3 x 229 [100,27]
+  CRUSH rule 3 x 230 [41,32]
+  CRUSH rule 3 x 231 [30,16]
+  CRUSH rule 3 x 232 [23,102]
+  CRUSH rule 3 x 233 [47,32]
+  CRUSH rule 3 x 234 [55,78]
+  CRUSH rule 3 x 235 [20,32]
+  CRUSH rule 3 x 236 [95,118]
+  CRUSH rule 3 x 237 [21,72]
+  CRUSH rule 3 x 238 [109,53]
+  CRUSH rule 3 x 239 [40,10]
+  CRUSH rule 3 x 240 [63,96]
+  CRUSH rule 3 x 241 [47,1]
+  CRUSH rule 3 x 242 [73,24]
+  CRUSH rule 3 x 243 [76,79]
+  CRUSH rule 3 x 244 [103,115]
+  CRUSH rule 3 x 245 [106,29]
+  CRUSH rule 3 x 246 [35,5]
+  CRUSH rule 3 x 247 [116,37]
+  CRUSH rule 3 x 248 [8,34]
+  CRUSH rule 3 x 249 [2,105]
+  CRUSH rule 3 x 250 [34,79]
+  CRUSH rule 3 x 251 [28,87]
+  CRUSH rule 3 x 252 [95,24]
+  CRUSH rule 3 x 253 [109,97]
+  CRUSH rule 3 x 254 [99,56]
+  CRUSH rule 3 x 255 [112,31]
+  CRUSH rule 3 x 256 [94,31]
+  CRUSH rule 3 x 257 [100,39]
+  CRUSH rule 3 x 258 [34,83]
+  CRUSH rule 3 x 259 [70,87]
+  CRUSH rule 3 x 260 [89,24]
+  CRUSH rule 3 x 261 [94,77]
+  CRUSH rule 3 x 262 [42,97]
+  CRUSH rule 3 x 263 [113,37]
+  CRUSH rule 3 x 264 [36,89]
+  CRUSH rule 3 x 265 [14,46]
+  CRUSH rule 3 x 266 [75,48]
+  CRUSH rule 3 x 267 [6,46]
+  CRUSH rule 3 x 268 [38,3]
+  CRUSH rule 3 x 269 [86,91]
+  CRUSH rule 3 x 270 [87,54]
+  CRUSH rule 3 x 271 [19,78]
+  CRUSH rule 3 x 272 [73,110]
+  CRUSH rule 3 x 273 [69,113]
+  CRUSH rule 3 x 274 [47,26]
+  CRUSH rule 3 x 275 [92,29]
+  CRUSH rule 3 x 276 [7,38]
+  CRUSH rule 3 x 277 [74,95]
+  CRUSH rule 3 x 278 [107,62]
+  CRUSH rule 3 x 279 [112,53]
+  CRUSH rule 3 x 280 [113,75]
+  CRUSH rule 3 x 281 [89,40]
+  CRUSH rule 3 x 282 [20,46]
+  CRUSH rule 3 x 283 [8,36]
+  CRUSH rule 3 x 284 [66,85]
+  CRUSH rule 3 x 285 [99,109]
+  CRUSH rule 3 x 286 [78,89]
+  CRUSH rule 3 x 287 [12,79]
+  CRUSH rule 3 x 288 [24,37]
+  CRUSH rule 3 x 289 [105,74]
+  CRUSH rule 3 x 290 [25,18]
+  CRUSH rule 3 x 291 [35,42]
+  CRUSH rule 3 x 292 [20,74]
+  CRUSH rule 3 x 293 [27,118]
+  CRUSH rule 3 x 294 [60,75]
+  CRUSH rule 3 x 295 [37,36]
+  CRUSH rule 3 x 296 [16,28]
+  CRUSH rule 3 x 297 [36,29]
+  CRUSH rule 3 x 298 [70,105]
+  CRUSH rule 3 x 299 [116,85]
+  CRUSH rule 3 x 300 [67,36]
+  CRUSH rule 3 x 301 [117,71]
+  CRUSH rule 3 x 302 [78,105]
+  CRUSH rule 3 x 303 [19,82]
+  CRUSH rule 3 x 304 [101,38]
+  CRUSH rule 3 x 305 [5,49]
+  CRUSH rule 3 x 306 [41,64]
+  CRUSH rule 3 x 307 [65,119]
+  CRUSH rule 3 x 308 [91,115]
+  CRUSH rule 3 x 309 [38,41]
+  CRUSH rule 3 x 310 [26,43]
+  CRUSH rule 3 x 311 [36,75]
+  CRUSH rule 3 x 312 [114,15]
+  CRUSH rule 3 x 313 [104,79]
+  CRUSH rule 3 x 314 [28,43]
+  CRUSH rule 3 x 315 [118,17]
+  CRUSH rule 3 x 316 [98,39]
+  CRUSH rule 3 x 317 [118,21]
+  CRUSH rule 3 x 318 [17,94]
+  CRUSH rule 3 x 319 [53,62]
+  CRUSH rule 3 x 320 [36,3]
+  CRUSH rule 3 x 321 [33,60]
+  CRUSH rule 3 x 322 [68,3]
+  CRUSH rule 3 x 323 [66,95]
+  CRUSH rule 3 x 324 [21,42]
+  CRUSH rule 3 x 325 [52,43]
+  CRUSH rule 3 x 326 [7,90]
+  CRUSH rule 3 x 327 [62,3]
+  CRUSH rule 3 x 328 [61,42]
+  CRUSH rule 3 x 329 [19,115]
+  CRUSH rule 3 x 330 [24,15]
+  CRUSH rule 3 x 331 [84,14]
+  CRUSH rule 3 x 332 [61,72]
+  CRUSH rule 3 x 333 [116,6]
+  CRUSH rule 3 x 334 [94,29]
+  CRUSH rule 3 x 335 [71,116]
+  CRUSH rule 3 x 336 [24,11]
+  CRUSH rule 3 x 337 [18,23]
+  CRUSH rule 3 x 338 [43,118]
+  CRUSH rule 3 x 339 [13,50]
+  CRUSH rule 3 x 340 [81,115]
+  CRUSH rule 3 x 341 [46,65]
+  CRUSH rule 3 x 342 [92,71]
+  CRUSH rule 3 x 343 [49,56]
+  CRUSH rule 3 x 344 [1,25]
+  CRUSH rule 3 x 345 [56,11]
+  CRUSH rule 3 x 346 [3,112]
+  CRUSH rule 3 x 347 [106,85]
+  CRUSH rule 3 x 348 [10,114]
+  CRUSH rule 3 x 349 [96,51]
+  CRUSH rule 3 x 350 [63,32]
+  CRUSH rule 3 x 351 [60,20]
+  CRUSH rule 3 x 352 [36,21]
+  CRUSH rule 3 x 353 [10,32]
+  CRUSH rule 3 x 354 [55,74]
+  CRUSH rule 3 x 355 [73,80]
+  CRUSH rule 3 x 356 [75,96]
+  CRUSH rule 3 x 357 [70,89]
+  CRUSH rule 3 x 358 [97,92]
+  CRUSH rule 3 x 359 [119,20]
+  CRUSH rule 3 x 360 [106,15]
+  CRUSH rule 3 x 361 [27,56]
+  CRUSH rule 3 x 362 [28,22]
+  CRUSH rule 3 x 363 [68,81]
+  CRUSH rule 3 x 364 [23,2]
+  CRUSH rule 3 x 365 [57,12]
+  CRUSH rule 3 x 366 [42,61]
+  CRUSH rule 3 x 367 [103,108]
+  CRUSH rule 3 x 368 [103,119]
+  CRUSH rule 3 x 369 [12,11]
+  CRUSH rule 3 x 370 [11,109]
+  CRUSH rule 3 x 371 [34,65]
+  CRUSH rule 3 x 372 [58,29]
+  CRUSH rule 3 x 373 [6,64]
+  CRUSH rule 3 x 374 [110,89]
+  CRUSH rule 3 x 375 [5,89]
+  CRUSH rule 3 x 376 [91,98]
+  CRUSH rule 3 x 377 [93,113]
+  CRUSH rule 3 x 378 [68,41]
+  CRUSH rule 3 x 379 [77,94]
+  CRUSH rule 3 x 380 [76,107]
+  CRUSH rule 3 x 381 [36,20]
+  CRUSH rule 3 x 382 [26,107]
+  CRUSH rule 3 x 383 [48,93]
+  CRUSH rule 3 x 384 [15,100]
+  CRUSH rule 3 x 385 [82,27]
+  CRUSH rule 3 x 386 [83,24]
+  CRUSH rule 3 x 387 [16,70]
+  CRUSH rule 3 x 388 [29,66]
+  CRUSH rule 3 x 389 [92,67]
+  CRUSH rule 3 x 390 [68,13]
+  CRUSH rule 3 x 391 [15,2]
+  CRUSH rule 3 x 392 [21,110]
+  CRUSH rule 3 x 393 [91,113]
+  CRUSH rule 3 x 394 [38,21]
+  CRUSH rule 3 x 395 [21,92]
+  CRUSH rule 3 x 396 [12,59]
+  CRUSH rule 3 x 397 [40,51]
+  CRUSH rule 3 x 398 [44,21]
+  CRUSH rule 3 x 399 [5,33]
+  CRUSH rule 3 x 400 [19,64]
+  CRUSH rule 3 x 401 [79,109]
+  CRUSH rule 3 x 402 [107,72]
+  CRUSH rule 3 x 403 [23,74]
+  CRUSH rule 3 x 404 [87,78]
+  CRUSH rule 3 x 405 [90,93]
+  CRUSH rule 3 x 406 [15,98]
+  CRUSH rule 3 x 407 [70,25]
+  CRUSH rule 3 x 408 [55,104]
+  CRUSH rule 3 x 409 [73,44]
+  CRUSH rule 3 x 410 [70,47]
+  CRUSH rule 3 x 411 [34,15]
+  CRUSH rule 3 x 412 [105,44]
+  CRUSH rule 3 x 413 [41,86]
+  CRUSH rule 3 x 414 [70,71]
+  CRUSH rule 3 x 415 [107,80]
+  CRUSH rule 3 x 416 [2,23]
+  CRUSH rule 3 x 417 [26,23]
+  CRUSH rule 3 x 418 [51,114]
+  CRUSH rule 3 x 419 [8,94]
+  CRUSH rule 3 x 420 [109,15]
+  CRUSH rule 3 x 421 [114,77]
+  CRUSH rule 3 x 422 [109,39]
+  CRUSH rule 3 x 423 [59,98]
+  CRUSH rule 3 x 424 [92,65]
+  CRUSH rule 3 x 425 [101,50]
+  CRUSH rule 3 x 426 [36,57]
+  CRUSH rule 3 x 427 [8,38]
+  CRUSH rule 3 x 428 [68,63]
+  CRUSH rule 3 x 429 [76,13]
+  CRUSH rule 3 x 430 [67,100]
+  CRUSH rule 3 x 431 [70,53]
+  CRUSH rule 3 x 432 [7,50]
+  CRUSH rule 3 x 433 [49,24]
+  CRUSH rule 3 x 434 [64,59]
+  CRUSH rule 3 x 435 [110,71]
+  CRUSH rule 3 x 436 [106,47]
+  CRUSH rule 3 x 437 [26,29]
+  CRUSH rule 3 x 438 [118,95]
+  CRUSH rule 3 x 439 [40,83]
+  CRUSH rule 3 x 440 [45,68]
+  CRUSH rule 3 x 441 [112,15]
+  CRUSH rule 3 x 442 [55,18]
+  CRUSH rule 3 x 443 [44,37]
+  CRUSH rule 3 x 444 [71,119]
+  CRUSH rule 3 x 445 [58,63]
+  CRUSH rule 3 x 446 [40,20]
+  CRUSH rule 3 x 447 [100,43]
+  CRUSH rule 3 x 448 [111,15]
+  CRUSH rule 3 x 449 [67,102]
+  CRUSH rule 3 x 450 [117,79]
+  CRUSH rule 3 x 451 [66,75]
+  CRUSH rule 3 x 452 [70,33]
+  CRUSH rule 3 x 453 [82,21]
+  CRUSH rule 3 x 454 [53,28]
+  CRUSH rule 3 x 455 [91,68]
+  CRUSH rule 3 x 456 [101,60]
+  CRUSH rule 3 x 457 [113,97]
+  CRUSH rule 3 x 458 [119,41]
+  CRUSH rule 3 x 459 [50,55]
+  CRUSH rule 3 x 460 [105,30]
+  CRUSH rule 3 x 461 [102,45]
+  CRUSH rule 3 x 462 [98,25]
+  CRUSH rule 3 x 463 [108,57]
+  CRUSH rule 3 x 464 [19,50]
+  CRUSH rule 3 x 465 [62,95]
+  CRUSH rule 3 x 466 [53,106]
+  CRUSH rule 3 x 467 [40,95]
+  CRUSH rule 3 x 468 [97,108]
+  CRUSH rule 3 x 469 [98,16]
+  CRUSH rule 3 x 470 [50,3]
+  CRUSH rule 3 x 471 [40,14]
+  CRUSH rule 3 x 472 [27,28]
+  CRUSH rule 3 x 473 [48,17]
+  CRUSH rule 3 x 474 [51,113]
+  CRUSH rule 3 x 475 [49,66]
+  CRUSH rule 3 x 476 [110,55]
+  CRUSH rule 3 x 477 [80,8]
+  CRUSH rule 3 x 478 [78,25]
+  CRUSH rule 3 x 479 [31,106]
+  CRUSH rule 3 x 480 [75,5]
+  CRUSH rule 3 x 481 [26,37]
+  CRUSH rule 3 x 482 [84,87]
+  CRUSH rule 3 x 483 [15,113]
+  CRUSH rule 3 x 484 [37,28]
+  CRUSH rule 3 x 485 [84,61]
+  CRUSH rule 3 x 486 [92,61]
+  CRUSH rule 3 x 487 [106,53]
+  CRUSH rule 3 x 488 [42,7]
+  CRUSH rule 3 x 489 [89,98]
+  CRUSH rule 3 x 490 [22,119]
+  CRUSH rule 3 x 491 [99,5]
+  CRUSH rule 3 x 492 [21,58]
+  CRUSH rule 3 x 493 [94,89]
+  CRUSH rule 3 x 494 [56,59]
+  CRUSH rule 3 x 495 [95,119]
+  CRUSH rule 3 x 496 [46,43]
+  CRUSH rule 3 x 497 [102,89]
+  CRUSH rule 3 x 498 [21,82]
+  CRUSH rule 3 x 499 [5,95]
+  CRUSH rule 3 x 500 [50,6]
+  CRUSH rule 3 x 501 [60,75]
+  CRUSH rule 3 x 502 [65,1]
+  CRUSH rule 3 x 503 [21,115]
+  CRUSH rule 3 x 504 [67,5]
+  CRUSH rule 3 x 505 [12,91]
+  CRUSH rule 3 x 506 [79,110]
+  CRUSH rule 3 x 507 [34,77]
+  CRUSH rule 3 x 508 [34,45]
+  CRUSH rule 3 x 509 [19,74]
+  CRUSH rule 3 x 510 [117,69]
+  CRUSH rule 3 x 511 [14,34]
+  CRUSH rule 3 x 512 [59,111]
+  CRUSH rule 3 x 513 [102,13]
+  CRUSH rule 3 x 514 [75,111]
+  CRUSH rule 3 x 515 [84,83]
+  CRUSH rule 3 x 516 [37,80]
+  CRUSH rule 3 x 517 [83,30]
+  CRUSH rule 3 x 518 [18,37]
+  CRUSH rule 3 x 519 [67,52]
+  CRUSH rule 3 x 520 [15,70]
+  CRUSH rule 3 x 521 [70,22]
+  CRUSH rule 3 x 522 [56,3]
+  CRUSH rule 3 x 523 [36,23]
+  CRUSH rule 3 x 524 [33,94]
+  CRUSH rule 3 x 525 [63,104]
+  CRUSH rule 3 x 526 [83,118]
+  CRUSH rule 3 x 527 [37,5]
+  CRUSH rule 3 x 528 [108,43]
+  CRUSH rule 3 x 529 [74,7]
+  CRUSH rule 3 x 530 [49,12]
+  CRUSH rule 3 x 531 [117,107]
+  CRUSH rule 3 x 532 [31,68]
+  CRUSH rule 3 x 533 [5,73]
+  CRUSH rule 3 x 534 [97,104]
+  CRUSH rule 3 x 535 [48,41]
+  CRUSH rule 3 x 536 [113,71]
+  CRUSH rule 3 x 537 [116,7]
+  CRUSH rule 3 x 538 [85,40]
+  CRUSH rule 3 x 539 [72,85]
+  CRUSH rule 3 x 540 [39,12]
+  CRUSH rule 3 x 541 [53,64]
+  CRUSH rule 3 x 542 [27,54]
+  CRUSH rule 3 x 543 [45,106]
+  CRUSH rule 3 x 544 [59,26]
+  CRUSH rule 3 x 545 [118,15]
+  CRUSH rule 3 x 546 [18,71]
+  CRUSH rule 3 x 547 [67,80]
+  CRUSH rule 3 x 548 [53,92]
+  CRUSH rule 3 x 549 [60,51]
+  CRUSH rule 3 x 550 [92,37]
+  CRUSH rule 3 x 551 [77,52]
+  CRUSH rule 3 x 552 [61,80]
+  CRUSH rule 3 x 553 [71,84]
+  CRUSH rule 3 x 554 [61,52]
+  CRUSH rule 3 x 555 [76,69]
+  CRUSH rule 3 x 556 [106,10]
+  CRUSH rule 3 x 557 [26,35]
+  CRUSH rule 3 x 558 [41,46]
+  CRUSH rule 3 x 559 [65,86]
+  CRUSH rule 3 x 560 [94,91]
+  CRUSH rule 3 x 561 [27,98]
+  CRUSH rule 3 x 562 [78,19]
+  CRUSH rule 3 x 563 [59,82]
+  CRUSH rule 3 x 564 [96,15]
+  CRUSH rule 3 x 565 [8,92]
+  CRUSH rule 3 x 566 [119,81]
+  CRUSH rule 3 x 567 [7,46]
+  CRUSH rule 3 x 568 [57,96]
+  CRUSH rule 3 x 569 [65,100]
+  CRUSH rule 3 x 570 [98,103]
+  CRUSH rule 3 x 571 [95,110]
+  CRUSH rule 3 x 572 [62,75]
+  CRUSH rule 3 x 573 [1,20]
+  CRUSH rule 3 x 574 [89,64]
+  CRUSH rule 3 x 575 [87,54]
+  CRUSH rule 3 x 576 [21,113]
+  CRUSH rule 3 x 577 [8,113]
+  CRUSH rule 3 x 578 [75,116]
+  CRUSH rule 3 x 579 [105,96]
+  CRUSH rule 3 x 580 [51,12]
+  CRUSH rule 3 x 581 [55,40]
+  CRUSH rule 3 x 582 [27,106]
+  CRUSH rule 3 x 583 [6,102]
+  CRUSH rule 3 x 584 [10,90]
+  CRUSH rule 3 x 585 [20,88]
+  CRUSH rule 3 x 586 [48,67]
+  CRUSH rule 3 x 587 [29,5]
+  CRUSH rule 3 x 588 [103,40]
+  CRUSH rule 3 x 589 [88,85]
+  CRUSH rule 3 x 590 [76,11]
+  CRUSH rule 3 x 591 [42,17]
+  CRUSH rule 3 x 592 [78,6]
+  CRUSH rule 3 x 593 [82,35]
+  CRUSH rule 3 x 594 [27,76]
+  CRUSH rule 3 x 595 [52,10]
+  CRUSH rule 3 x 596 [82,99]
+  CRUSH rule 3 x 597 [16,32]
+  CRUSH rule 3 x 598 [37,36]
+  CRUSH rule 3 x 599 [10,24]
+  CRUSH rule 3 x 600 [24,37]
+  CRUSH rule 3 x 601 [104,21]
+  CRUSH rule 3 x 602 [48,39]
+  CRUSH rule 3 x 603 [93,44]
+  CRUSH rule 3 x 604 [118,87]
+  CRUSH rule 3 x 605 [104,63]
+  CRUSH rule 3 x 606 [90,103]
+  CRUSH rule 3 x 607 [95,72]
+  CRUSH rule 3 x 608 [112,71]
+  CRUSH rule 3 x 609 [34,16]
+  CRUSH rule 3 x 610 [106,73]
+  CRUSH rule 3 x 611 [66,37]
+  CRUSH rule 3 x 612 [2,20]
+  CRUSH rule 3 x 613 [13,92]
+  CRUSH rule 3 x 614 [50,65]
+  CRUSH rule 3 x 615 [24,39]
+  CRUSH rule 3 x 616 [41,46]
+  CRUSH rule 3 x 617 [111,81]
+  CRUSH rule 3 x 618 [3,72]
+  CRUSH rule 3 x 619 [92,31]
+  CRUSH rule 3 x 620 [108,31]
+  CRUSH rule 3 x 621 [105,50]
+  CRUSH rule 3 x 622 [67,102]
+  CRUSH rule 3 x 623 [69,117]
+  CRUSH rule 3 x 624 [115,79]
+  CRUSH rule 3 x 625 [73,94]
+  CRUSH rule 3 x 626 [52,25]
+  CRUSH rule 3 x 627 [116,105]
+  CRUSH rule 3 x 628 [98,87]
+  CRUSH rule 3 x 629 [6,116]
+  CRUSH rule 3 x 630 [22,50]
+  CRUSH rule 3 x 631 [35,96]
+  CRUSH rule 3 x 632 [80,53]
+  CRUSH rule 3 x 633 [65,110]
+  CRUSH rule 3 x 634 [87,50]
+  CRUSH rule 3 x 635 [107,111]
+  CRUSH rule 3 x 636 [23,30]
+  CRUSH rule 3 x 637 [99,114]
+  CRUSH rule 3 x 638 [43,78]
+  CRUSH rule 3 x 639 [30,31]
+  CRUSH rule 3 x 640 [113,87]
+  CRUSH rule 3 x 641 [45,58]
+  CRUSH rule 3 x 642 [47,30]
+  CRUSH rule 3 x 643 [64,99]
+  CRUSH rule 3 x 644 [31,119]
+  CRUSH rule 3 x 645 [77,90]
+  CRUSH rule 3 x 646 [37,26]
+  CRUSH rule 3 x 647 [65,112]
+  CRUSH rule 3 x 648 [31,84]
+  CRUSH rule 3 x 649 [88,39]
+  CRUSH rule 3 x 650 [21,44]
+  CRUSH rule 3 x 651 [63,12]
+  CRUSH rule 3 x 652 [57,28]
+  CRUSH rule 3 x 653 [38,63]
+  CRUSH rule 3 x 654 [104,107]
+  CRUSH rule 3 x 655 [89,109]
+  CRUSH rule 3 x 656 [79,84]
+  CRUSH rule 3 x 657 [47,18]
+  CRUSH rule 3 x 658 [80,49]
+  CRUSH rule 3 x 659 [11,104]
+  CRUSH rule 3 x 660 [65,102]
+  CRUSH rule 3 x 661 [96,67]
+  CRUSH rule 3 x 662 [111,43]
+  CRUSH rule 3 x 663 [83,115]
+  CRUSH rule 3 x 664 [59,52]
+  CRUSH rule 3 x 665 [31,86]
+  CRUSH rule 3 x 666 [112,8]
+  CRUSH rule 3 x 667 [70,107]
+  CRUSH rule 3 x 668 [96,43]
+  CRUSH rule 3 x 669 [56,25]
+  CRUSH rule 3 x 670 [98,83]
+  CRUSH rule 3 x 671 [57,100]
+  CRUSH rule 3 x 672 [37,98]
+  CRUSH rule 3 x 673 [83,116]
+  CRUSH rule 3 x 674 [36,95]
+  CRUSH rule 3 x 675 [88,91]
+  CRUSH rule 3 x 676 [3,40]
+  CRUSH rule 3 x 677 [88,105]
+  CRUSH rule 3 x 678 [27,100]
+  CRUSH rule 3 x 679 [33,118]
+  CRUSH rule 3 x 680 [111,81]
+  CRUSH rule 3 x 681 [53,68]
+  CRUSH rule 3 x 682 [12,83]
+  CRUSH rule 3 x 683 [24,67]
+  CRUSH rule 3 x 684 [98,45]
+  CRUSH rule 3 x 685 [106,25]
+  CRUSH rule 3 x 686 [86,45]
+  CRUSH rule 3 x 687 [49,102]
+  CRUSH rule 3 x 688 [16,52]
+  CRUSH rule 3 x 689 [32,101]
+  CRUSH rule 3 x 690 [96,79]
+  CRUSH rule 3 x 691 [34,99]
+  CRUSH rule 3 x 692 [97,68]
+  CRUSH rule 3 x 693 [29,38]
+  CRUSH rule 3 x 694 [6,26]
+  CRUSH rule 3 x 695 [31,112]
+  CRUSH rule 3 x 696 [36,97]
+  CRUSH rule 3 x 697 [19,38]
+  CRUSH rule 3 x 698 [30,103]
+  CRUSH rule 3 x 699 [47,60]
+  CRUSH rule 3 x 700 [99,82]
+  CRUSH rule 3 x 701 [53,72]
+  CRUSH rule 3 x 702 [101,113]
+  CRUSH rule 3 x 703 [92,20]
+  CRUSH rule 3 x 704 [34,47]
+  CRUSH rule 3 x 705 [105,88]
+  CRUSH rule 3 x 706 [74,20]
+  CRUSH rule 3 x 707 [95,40]
+  CRUSH rule 3 x 708 [95,38]
+  CRUSH rule 3 x 709 [73,94]
+  CRUSH rule 3 x 710 [94,7]
+  CRUSH rule 3 x 711 [68,16]
+  CRUSH rule 3 x 712 [107,64]
+  CRUSH rule 3 x 713 [29,2]
+  CRUSH rule 3 x 714 [86,97]
+  CRUSH rule 3 x 715 [74,95]
+  CRUSH rule 3 x 716 [101,74]
+  CRUSH rule 3 x 717 [12,57]
+  CRUSH rule 3 x 718 [83,106]
+  CRUSH rule 3 x 719 [26,39]
+  CRUSH rule 3 x 720 [69,64]
+  CRUSH rule 3 x 721 [51,119]
+  CRUSH rule 3 x 722 [15,26]
+  CRUSH rule 3 x 723 [117,75]
+  CRUSH rule 3 x 724 [45,106]
+  CRUSH rule 3 x 725 [53,66]
+  CRUSH rule 3 x 726 [103,38]
+  CRUSH rule 3 x 727 [89,115]
+  CRUSH rule 3 x 728 [76,65]
+  CRUSH rule 3 x 729 [35,48]
+  CRUSH rule 3 x 730 [28,37]
+  CRUSH rule 3 x 731 [78,6]
+  CRUSH rule 3 x 732 [1,93]
+  CRUSH rule 3 x 733 [35,44]
+  CRUSH rule 3 x 734 [119,93]
+  CRUSH rule 3 x 735 [102,17]
+  CRUSH rule 3 x 736 [37,78]
+  CRUSH rule 3 x 737 [117,35]
+  CRUSH rule 3 x 738 [57,56]
+  CRUSH rule 3 x 739 [87,24]
+  CRUSH rule 3 x 740 [29,34]
+  CRUSH rule 3 x 741 [47,94]
+  CRUSH rule 3 x 742 [106,107]
+  CRUSH rule 3 x 743 [105,5]
+  CRUSH rule 3 x 744 [23,30]
+  CRUSH rule 3 x 745 [37,106]
+  CRUSH rule 3 x 746 [56,47]
+  CRUSH rule 3 x 747 [56,107]
+  CRUSH rule 3 x 748 [48,25]
+  CRUSH rule 3 x 749 [102,93]
+  CRUSH rule 3 x 750 [83,102]
+  CRUSH rule 3 x 751 [25,56]
+  CRUSH rule 3 x 752 [82,16]
+  CRUSH rule 3 x 753 [116,14]
+  CRUSH rule 3 x 754 [114,39]
+  CRUSH rule 3 x 755 [87,60]
+  CRUSH rule 3 x 756 [113,77]
+  CRUSH rule 3 x 757 [47,112]
+  CRUSH rule 3 x 758 [54,107]
+  CRUSH rule 3 x 759 [74,65]
+  CRUSH rule 3 x 760 [88,47]
+  CRUSH rule 3 x 761 [73,98]
+  CRUSH rule 3 x 762 [34,33]
+  CRUSH rule 3 x 763 [13,116]
+  CRUSH rule 3 x 764 [89,2]
+  CRUSH rule 3 x 765 [109,77]
+  CRUSH rule 3 x 766 [19,92]
+  CRUSH rule 3 x 767 [41,80]
+  CRUSH rule 3 x 768 [106,16]
+  CRUSH rule 3 x 769 [91,2]
+  CRUSH rule 3 x 770 [72,19]
+  CRUSH rule 3 x 771 [115,63]
+  CRUSH rule 3 x 772 [97,102]
+  CRUSH rule 3 x 773 [116,91]
+  CRUSH rule 3 x 774 [100,105]
+  CRUSH rule 3 x 775 [102,95]
+  CRUSH rule 3 x 776 [69,44]
+  CRUSH rule 3 x 777 [91,102]
+  CRUSH rule 3 x 778 [83,110]
+  CRUSH rule 3 x 779 [47,80]
+  CRUSH rule 3 x 780 [63,117]
+  CRUSH rule 3 x 781 [105,106]
+  CRUSH rule 3 x 782 [117,107]
+  CRUSH rule 3 x 783 [19,30]
+  CRUSH rule 3 x 784 [63,82]
+  CRUSH rule 3 x 785 [27,50]
+  CRUSH rule 3 x 786 [41,90]
+  CRUSH rule 3 x 787 [108,27]
+  CRUSH rule 3 x 788 [74,75]
+  CRUSH rule 3 x 789 [50,67]
+  CRUSH rule 3 x 790 [20,108]
+  CRUSH rule 3 x 791 [96,53]
+  CRUSH rule 3 x 792 [80,13]
+  CRUSH rule 3 x 793 [6,82]
+  CRUSH rule 3 x 794 [14,90]
+  CRUSH rule 3 x 795 [30,67]
+  CRUSH rule 3 x 796 [87,60]
+  CRUSH rule 3 x 797 [64,93]
+  CRUSH rule 3 x 798 [42,19]
+  CRUSH rule 3 x 799 [19,113]
+  CRUSH rule 3 x 800 [106,22]
+  CRUSH rule 3 x 801 [2,11]
+  CRUSH rule 3 x 802 [63,1]
+  CRUSH rule 3 x 803 [37,46]
+  CRUSH rule 3 x 804 [33,66]
+  CRUSH rule 3 x 805 [96,3]
+  CRUSH rule 3 x 806 [48,57]
+  CRUSH rule 3 x 807 [48,85]
+  CRUSH rule 3 x 808 [76,15]
+  CRUSH rule 3 x 809 [27,90]
+  CRUSH rule 3 x 810 [119,61]
+  CRUSH rule 3 x 811 [111,93]
+  CRUSH rule 3 x 812 [25,94]
+  CRUSH rule 3 x 813 [81,50]
+  CRUSH rule 3 x 814 [95,48]
+  CRUSH rule 3 x 815 [84,6]
+  CRUSH rule 3 x 816 [64,3]
+  CRUSH rule 3 x 817 [63,117]
+  CRUSH rule 3 x 818 [69,52]
+  CRUSH rule 3 x 819 [88,19]
+  CRUSH rule 3 x 820 [104,29]
+  CRUSH rule 3 x 821 [58,107]
+  CRUSH rule 3 x 822 [20,18]
+  CRUSH rule 3 x 823 [63,102]
+  CRUSH rule 3 x 824 [102,95]
+  CRUSH rule 3 x 825 [47,46]
+  CRUSH rule 3 x 826 [44,33]
+  CRUSH rule 3 x 827 [101,115]
+  CRUSH rule 3 x 828 [60,39]
+  CRUSH rule 3 x 829 [45,24]
+  CRUSH rule 3 x 830 [51,96]
+  CRUSH rule 3 x 831 [78,53]
+  CRUSH rule 3 x 832 [28,15]
+  CRUSH rule 3 x 833 [57,72]
+  CRUSH rule 3 x 834 [90,77]
+  CRUSH rule 3 x 835 [14,50]
+  CRUSH rule 3 x 836 [63,100]
+  CRUSH rule 3 x 837 [76,85]
+  CRUSH rule 3 x 838 [106,75]
+  CRUSH rule 3 x 839 [87,12]
+  CRUSH rule 3 x 840 [33,117]
+  CRUSH rule 3 x 841 [110,13]
+  CRUSH rule 3 x 842 [66,97]
+  CRUSH rule 3 x 843 [11,50]
+  CRUSH rule 3 x 844 [74,22]
+  CRUSH rule 3 x 845 [74,20]
+  CRUSH rule 3 x 846 [43,113]
+  CRUSH rule 3 x 847 [62,105]
+  CRUSH rule 3 x 848 [92,19]
+  CRUSH rule 3 x 849 [93,118]
+  CRUSH rule 3 x 850 [83,119]
+  CRUSH rule 3 x 851 [65,56]
+  CRUSH rule 3 x 852 [60,11]
+  CRUSH rule 3 x 853 [88,11]
+  CRUSH rule 3 x 854 [83,52]
+  CRUSH rule 3 x 855 [2,22]
+  CRUSH rule 3 x 856 [40,13]
+  CRUSH rule 3 x 857 [69,110]
+  CRUSH rule 3 x 858 [98,27]
+  CRUSH rule 3 x 859 [56,41]
+  CRUSH rule 3 x 860 [11,30]
+  CRUSH rule 3 x 861 [22,68]
+  CRUSH rule 3 x 862 [22,52]
+  CRUSH rule 3 x 863 [79,32]
+  CRUSH rule 3 x 864 [77,32]
+  CRUSH rule 3 x 865 [119,99]
+  CRUSH rule 3 x 866 [18,39]
+  CRUSH rule 3 x 867 [3,58]
+  CRUSH rule 3 x 868 [100,22]
+  CRUSH rule 3 x 869 [22,86]
+  CRUSH rule 3 x 870 [73,94]
+  CRUSH rule 3 x 871 [84,51]
+  CRUSH rule 3 x 872 [72,91]
+  CRUSH rule 3 x 873 [81,72]
+  CRUSH rule 3 x 874 [21,38]
+  CRUSH rule 3 x 875 [115,27]
+  CRUSH rule 3 x 876 [98,16]
+  CRUSH rule 3 x 877 [80,25]
+  CRUSH rule 3 x 878 [87,114]
+  CRUSH rule 3 x 879 [29,1]
+  CRUSH rule 3 x 880 [23,2]
+  CRUSH rule 3 x 881 [109,97]
+  CRUSH rule 3 x 882 [31,36]
+  CRUSH rule 3 x 883 [102,17]
+  CRUSH rule 3 x 884 [80,23]
+  CRUSH rule 3 x 885 [46,31]
+  CRUSH rule 3 x 886 [2,11]
+  CRUSH rule 3 x 887 [5,85]
+  CRUSH rule 3 x 888 [16,64]
+  CRUSH rule 3 x 889 [84,45]
+  CRUSH rule 3 x 890 [65,50]
+  CRUSH rule 3 x 891 [86,59]
+  CRUSH rule 3 x 892 [64,11]
+  CRUSH rule 3 x 893 [20,118]
+  CRUSH rule 3 x 894 [32,14]
+  CRUSH rule 3 x 895 [40,91]
+  CRUSH rule 3 x 896 [113,29]
+  CRUSH rule 3 x 897 [107,112]
+  CRUSH rule 3 x 898 [76,51]
+  CRUSH rule 3 x 899 [75,66]
+  CRUSH rule 3 x 900 [83,111]
+  CRUSH rule 3 x 901 [66,17]
+  CRUSH rule 3 x 902 [25,5]
+  CRUSH rule 3 x 903 [53,54]
+  CRUSH rule 3 x 904 [50,10]
+  CRUSH rule 3 x 905 [99,106]
+  CRUSH rule 3 x 906 [68,73]
+  CRUSH rule 3 x 907 [109,45]
+  CRUSH rule 3 x 908 [47,24]
+  CRUSH rule 3 x 909 [73,94]
+  CRUSH rule 3 x 910 [71,26]
+  CRUSH rule 3 x 911 [39,62]
+  CRUSH rule 3 x 912 [90,39]
+  CRUSH rule 3 x 913 [29,80]
+  CRUSH rule 3 x 914 [84,99]
+  CRUSH rule 3 x 915 [49,62]
+  CRUSH rule 3 x 916 [32,7]
+  CRUSH rule 3 x 917 [46,91]
+  CRUSH rule 3 x 918 [82,71]
+  CRUSH rule 3 x 919 [13,109]
+  CRUSH rule 3 x 920 [25,100]
+  CRUSH rule 3 x 921 [55,32]
+  CRUSH rule 3 x 922 [33,96]
+  CRUSH rule 3 x 923 [28,79]
+  CRUSH rule 3 x 924 [1,41]
+  CRUSH rule 3 x 925 [113,25]
+  CRUSH rule 3 x 926 [64,65]
+  CRUSH rule 3 x 927 [32,23]
+  CRUSH rule 3 x 928 [13,94]
+  CRUSH rule 3 x 929 [85,60]
+  CRUSH rule 3 x 930 [104,55]
+  CRUSH rule 3 x 931 [46,91]
+  CRUSH rule 3 x 932 [43,54]
+  CRUSH rule 3 x 933 [18,93]
+  CRUSH rule 3 x 934 [68,107]
+  CRUSH rule 3 x 935 [28,23]
+  CRUSH rule 3 x 936 [104,51]
+  CRUSH rule 3 x 937 [110,37]
+  CRUSH rule 3 x 938 [48,69]
+  CRUSH rule 3 x 939 [77,32]
+  CRUSH rule 3 x 940 [76,19]
+  CRUSH rule 3 x 941 [66,10]
+  CRUSH rule 3 x 942 [80,37]
+  CRUSH rule 3 x 943 [75,82]
+  CRUSH rule 3 x 944 [113,15]
+  CRUSH rule 3 x 945 [71,111]
+  CRUSH rule 3 x 946 [37,115]
+  CRUSH rule 3 x 947 [107,48]
+  CRUSH rule 3 x 948 [108,8]
+  CRUSH rule 3 x 949 [46,14]
+  CRUSH rule 3 x 950 [96,13]
+  CRUSH rule 3 x 951 [40,63]
+  CRUSH rule 3 x 952 [114,16]
+  CRUSH rule 3 x 953 [62,53]
+  CRUSH rule 3 x 954 [103,68]
+  CRUSH rule 3 x 955 [42,63]
+  CRUSH rule 3 x 956 [72,6]
+  CRUSH rule 3 x 957 [117,6]
+  CRUSH rule 3 x 958 [23,74]
+  CRUSH rule 3 x 959 [42,87]
+  CRUSH rule 3 x 960 [113,91]
+  CRUSH rule 3 x 961 [116,61]
+  CRUSH rule 3 x 962 [60,41]
+  CRUSH rule 3 x 963 [103,46]
+  CRUSH rule 3 x 964 [66,15]
+  CRUSH rule 3 x 965 [47,108]
+  CRUSH rule 3 x 966 [88,69]
+  CRUSH rule 3 x 967 [71,74]
+  CRUSH rule 3 x 968 [74,75]
+  CRUSH rule 3 x 969 [53,30]
+  CRUSH rule 3 x 970 [3,2]
+  CRUSH rule 3 x 971 [66,19]
+  CRUSH rule 3 x 972 [3,115]
+  CRUSH rule 3 x 973 [113,89]
+  CRUSH rule 3 x 974 [114,73]
+  CRUSH rule 3 x 975 [83,96]
+  CRUSH rule 3 x 976 [81,100]
+  CRUSH rule 3 x 977 [95,76]
+  CRUSH rule 3 x 978 [35,119]
+  CRUSH rule 3 x 979 [98,13]
+  CRUSH rule 3 x 980 [39,113]
+  CRUSH rule 3 x 981 [89,46]
+  CRUSH rule 3 x 982 [19,66]
+  CRUSH rule 3 x 983 [34,107]
+  CRUSH rule 3 x 984 [78,23]
+  CRUSH rule 3 x 985 [99,24]
+  CRUSH rule 3 x 986 [44,33]
+  CRUSH rule 3 x 987 [25,98]
+  CRUSH rule 3 x 988 [79,84]
+  CRUSH rule 3 x 989 [87,60]
+  CRUSH rule 3 x 990 [72,22]
+  CRUSH rule 3 x 991 [90,71]
+  CRUSH rule 3 x 992 [30,75]
+  CRUSH rule 3 x 993 [74,27]
+  CRUSH rule 3 x 994 [74,75]
+  CRUSH rule 3 x 995 [100,45]
+  CRUSH rule 3 x 996 [41,34]
+  CRUSH rule 3 x 997 [89,32]
+  CRUSH rule 3 x 998 [92,41]
+  CRUSH rule 3 x 999 [117,13]
+  CRUSH rule 3 x 1000 [50,31]
+  CRUSH rule 3 x 1001 [83,116]
+  CRUSH rule 3 x 1002 [94,13]
+  CRUSH rule 3 x 1003 [43,54]
+  CRUSH rule 3 x 1004 [89,106]
+  CRUSH rule 3 x 1005 [105,76]
+  CRUSH rule 3 x 1006 [45,5]
+  CRUSH rule 3 x 1007 [19,111]
+  CRUSH rule 3 x 1008 [31,74]
+  CRUSH rule 3 x 1009 [1,51]
+  CRUSH rule 3 x 1010 [31,108]
+  CRUSH rule 3 x 1011 [64,3]
+  CRUSH rule 3 x 1012 [68,81]
+  CRUSH rule 3 x 1013 [5,35]
+  CRUSH rule 3 x 1014 [33,48]
+  CRUSH rule 3 x 1015 [106,99]
+  CRUSH rule 3 x 1016 [107,111]
+  CRUSH rule 3 x 1017 [12,69]
+  CRUSH rule 3 x 1018 [61,60]
+  CRUSH rule 3 x 1019 [27,88]
+  CRUSH rule 3 x 1020 [31,111]
+  CRUSH rule 3 x 1021 [22,36]
+  CRUSH rule 3 x 1022 [73,28]
+  CRUSH rule 3 x 1023 [59,88]
+  rule 3 (delltestrule) num_rep 4 result size == 2:\t1024/1024 (esc)
diff --git a/ceph/src/test/cli/crushtool/test-map-vary-r.crushmap b/ceph/src/test/cli/crushtool/test-map-vary-r.crushmap
new file mode 100644
index 0000000000000000000000000000000000000000..41886aefcbf23a6aed73f3c44c571893fedfaccb
GIT binary patch
literal 3892
zcmeH~$#&dC6ow_o1W14o<}hU*Nt-IoV#*w6CO`<7Dd|BT;%+PHR@Dc}_JIT75I78`fB^xB0B6m>8-R1X1CD~@;3PN=&VX~^0=NV&gZIG|
za20$EJ^`PBFTi!s0v#|5=D`in1wF6?N^le00(~$56&QkLumYyPS|u-_V!&vQ`|{Wv|LfZEc#Lu87=Q8^^L&33=S8mQjGC
z+k;NA+!z|$GBN+T@xDfDVQv=3Gco^dMzZ#x!GpH?3++W?v2&H{>lCxiVVBu(l;*L_
zDZ_>G>?xKjbrNuyPC_olNyHycVlLE4!daZ846#l!##$#iFBQj)CN&JfVwOu13wR~O
zLM}}#Vz?HI85G45hHYUxM`F=7Z83``X-gQQ
z#ZneiVj07wSk5BLS!lLvvpci}JGF(ov_-qM#e1|Rd$pzev}OCXdN>nt%$qhgjtu2^tHTX7+-if`03VFk9{ns#fN^=g_gX|<-Tsl8d#;8smTzE;UZ
zHX774W_np~O@=j1mus4>)HG+7;l^*KQYI&6kzP?X>C8_|OZ`@dUrZH}JBM9f+R=6w
HjU0Lpzt&9h

literal 0
HcmV?d00001

diff --git a/ceph/src/test/cli/monmaptool/add-exists.t b/ceph/src/test/cli/monmaptool/add-exists.t
new file mode 100644
index 00000000..c23d51f2
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/add-exists.t
@@ -0,0 +1,26 @@
+  $ monmaptool --create mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (0 monitors)
+
+  $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+
+  $ monmaptool --add foo 2.3.4.5:6789 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: writing epoch 0 to mymonmap (1 monitors)
+  $ monmaptool --add foo 3.4.5.6:7890 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: map already contains mon.foo
+   usage: [--print] [--create [--clobber][--fsid uuid]] [--generate] [--set-initial-members] [--add name 1.2.3.4:567] [--rm name] 
+  [1]
+
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  0: 2.3.4.5:6789/0 mon.foo
+
+  $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+  $ [ "$ORIG_FSID" = "$NEW_FSID" ]
diff --git a/ceph/src/test/cli/monmaptool/add-many.t b/ceph/src/test/cli/monmaptool/add-many.t
new file mode 100644
index 00000000..187adf8f
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/add-many.t
@@ -0,0 +1,29 @@
+  $ monmaptool --create mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (0 monitors)
+
+  $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+
+  $ monmaptool --add foo 2.3.4.5:6789 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: writing epoch 0 to mymonmap (1 monitors)
+  $ monmaptool --add bar 3.4.5.6:7890 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: writing epoch 0 to mymonmap (2 monitors)
+  $ monmaptool --add baz 4.5.6.7:8901 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: writing epoch 0 to mymonmap (3 monitors)
+
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  0: 2.3.4.5:6789/0 mon.foo
+  1: 3.4.5.6:7890/0 mon.bar
+  2: 4.5.6.7:8901/0 mon.baz
+
+  $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+  $ [ "$ORIG_FSID" = "$NEW_FSID" ]
diff --git a/ceph/src/test/cli/monmaptool/clobber.t b/ceph/src/test/cli/monmaptool/clobber.t
new file mode 100644
index 00000000..9f038abf
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/clobber.t
@@ -0,0 +1,38 @@
+  $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (1 monitors)
+
+  $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+
+  $ monmaptool --create mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: mymonmap exists, --clobber to overwrite
+  [255]
+
+# hasn't changed yet
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  0: 2.3.4.5:6789/0 mon.foo
+
+  $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+  $ [ "$ORIG_FSID" = "$NEW_FSID" ]
+
+  $ monmaptool --create --clobber mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (0 monitors)
+
+  $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+  $ [ "$ORIG_FSID" != "$NEW_FSID" ]
+
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
diff --git a/ceph/src/test/cli/monmaptool/create-print.t b/ceph/src/test/cli/monmaptool/create-print.t
new file mode 100644
index 00000000..3059b791
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/create-print.t
@@ -0,0 +1,18 @@
+  $ monmaptool --create mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (0 monitors)
+
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+
+  $ monmaptool --print -- mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
diff --git a/ceph/src/test/cli/monmaptool/create-with-add.t b/ceph/src/test/cli/monmaptool/create-with-add.t
new file mode 100644
index 00000000..783642a1
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/create-with-add.t
@@ -0,0 +1,12 @@
+  $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (1 monitors)
+
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  0: 2.3.4.5:6789/0 mon.foo
diff --git a/ceph/src/test/cli/monmaptool/help.t b/ceph/src/test/cli/monmaptool/help.t
new file mode 100644
index 00000000..31a372a6
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/help.t
@@ -0,0 +1,3 @@
+  $ monmaptool --help
+   usage: [--print] [--create [--clobber][--fsid uuid]] [--generate] [--set-initial-members] [--add name 1.2.3.4:567] [--rm name] 
+  [1]
diff --git a/ceph/src/test/cli/monmaptool/print-empty.t b/ceph/src/test/cli/monmaptool/print-empty.t
new file mode 100644
index 00000000..cd67db48
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/print-empty.t
@@ -0,0 +1,5 @@
+  $ touch empty
+  $ monmaptool --print empty
+  monmaptool: monmap file empty
+  monmaptool: unable to read monmap file
+  [255]
diff --git a/ceph/src/test/cli/monmaptool/print-nonexistent.t b/ceph/src/test/cli/monmaptool/print-nonexistent.t
new file mode 100644
index 00000000..ae366c1e
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/print-nonexistent.t
@@ -0,0 +1,4 @@
+  $ monmaptool --print nonexistent
+  monmaptool: monmap file nonexistent
+  monmaptool: couldn't open nonexistent: (2) No such file or directory
+  [255]
diff --git a/ceph/src/test/cli/monmaptool/rm-nonexistent.t b/ceph/src/test/cli/monmaptool/rm-nonexistent.t
new file mode 100644
index 00000000..e23ebf3e
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/rm-nonexistent.t
@@ -0,0 +1,24 @@
+  $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (1 monitors)
+
+  $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+
+  $ monmaptool --rm doesnotexist mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: removing doesnotexist
+  monmaptool: map does not contain doesnotexist
+   usage: [--print] [--create [--clobber][--fsid uuid]] [--generate] [--set-initial-members] [--add name 1.2.3.4:567] [--rm name] 
+  [1]
+
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  0: 2.3.4.5:6789/0 mon.foo
+
+  $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+  $ [ "$ORIG_FSID" = "$NEW_FSID" ]
diff --git a/ceph/src/test/cli/monmaptool/rm.t b/ceph/src/test/cli/monmaptool/rm.t
new file mode 100644
index 00000000..5155f24b
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/rm.t
@@ -0,0 +1,21 @@
+  $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  monmaptool: writing epoch 0 to mymonmap (1 monitors)
+
+  $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+
+  $ monmaptool --rm foo mymonmap
+  monmaptool: monmap file mymonmap
+  monmaptool: removing foo
+  monmaptool: writing epoch 0 to mymonmap (0 monitors)
+
+  $ monmaptool --print mymonmap
+  monmaptool: monmap file mymonmap
+  epoch 0
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  last_changed \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+
+  $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
+  $ [ "$ORIG_FSID" = "$NEW_FSID" ]
diff --git a/ceph/src/test/cli/monmaptool/simple.t b/ceph/src/test/cli/monmaptool/simple.t
new file mode 100644
index 00000000..ee7d6304
--- /dev/null
+++ b/ceph/src/test/cli/monmaptool/simple.t
@@ -0,0 +1,4 @@
+  $ monmaptool
+  monmaptool: must specify monmap filename
+   usage: [--print] [--create [--clobber][--fsid uuid]] [--generate] [--set-initial-members] [--add name 1.2.3.4:567] [--rm name] 
+  [1]
diff --git a/ceph/src/test/cli/osdmaptool/ceph.conf.withracks b/ceph/src/test/cli/osdmaptool/ceph.conf.withracks
new file mode 100644
index 00000000..09399e95
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/ceph.conf.withracks
@@ -0,0 +1,1480 @@
+
+[global]
+  auth supported = cephx
+  ms bind ipv6 = true
+
+[mon]
+  mon data = /var/ceph/mon
+  mon clock drift allowed = 0.1
+
+  osd pool default size = 3
+  osd pool default crush rule = 0
+
+  ; don't mark down osds out automatically; wait for an admin!
+  mon osd down out interval = 0
+
+[mon.alpha]
+  host = peon5752
+  mon addr = [2607:f298:4:2243::5752]:6789
+
+[mon.beta]
+  host = peon5753
+  mon addr = [2607:f298:4:2243::5753]:6789
+
+[mon.charlie]
+  host = peon5754
+  mon addr = [2607:f298:4:2243::5754]:6789
+
+[client]
+  rgw socket path = /var/run/ceph/radosgw.$name
+  rgw cache enabled = true
+  rgw dns name = objects.dreamhost.com
+  rgw swift url = https://objects.dreamhost.com
+
+[client.radosgw.peon5751]
+  host = peon5751
+  log file = /var/log/ceph/$name.log
+  debug rgw = 40
+  debug ms = 1
+
+
+[osd]
+  keyring = /mnt/osd.$id/keyring
+  osd data = /mnt/osd.$id
+  osd journal = /dev/disk/by-label/osd.$id.journal
+  osd mkfs type = btrfs
+  osd mount options btrfs = rw,noatime
+  devs = /dev/disk/by-label/osd.$id.data
+; temp sage
+  debug osd = 20
+  debug ms = 1
+  debug filestore = 20
+
+[osd.1]
+  host = cephstore5522
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5522]
+  public addr = [2607:f298:4:2243::5522]
+
+[osd.2]
+  host = cephstore5522
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5522]
+  public addr = [2607:f298:4:2243::5522]
+
+[osd.3]
+  host = cephstore5522
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5522]
+  public addr = [2607:f298:4:2243::5522]
+
+[osd.4]
+  host = cephstore5522
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5522]
+  public addr = [2607:f298:4:2243::5522]
+
+[osd.5]
+  host = cephstore5522
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5522]
+  public addr = [2607:f298:4:2243::5522]
+
+[osd.6]
+  host = cephstore5522
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5522]
+  public addr = [2607:f298:4:2243::5522]
+
+[osd.7]
+  host = cephstore5522
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5522]
+  public addr = [2607:f298:4:2243::5522]
+
+[osd.8]
+  host = cephstore5523
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5523]
+  public addr = [2607:f298:4:2243::5523]
+
+[osd.9]
+  host = cephstore5523
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5523]
+  public addr = [2607:f298:4:2243::5523]
+
+[osd.10]
+  host = cephstore5523
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5523]
+  public addr = [2607:f298:4:2243::5523]
+
+[osd.11]
+  host = cephstore5523
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5523]
+  public addr = [2607:f298:4:2243::5523]
+
+[osd.12]
+  host = cephstore5523
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5523]
+  public addr = [2607:f298:4:2243::5523]
+
+[osd.13]
+  host = cephstore5523
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5523]
+  public addr = [2607:f298:4:2243::5523]
+
+[osd.14]
+  host = cephstore5523
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5523]
+  public addr = [2607:f298:4:2243::5523]
+
+[osd.15]
+  host = cephstore5524
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5524]
+  public addr = [2607:f298:4:2243::5524]
+
+[osd.16]
+  host = cephstore5524
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5524]
+  public addr = [2607:f298:4:2243::5524]
+
+[osd.17]
+  host = cephstore5524
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5524]
+  public addr = [2607:f298:4:2243::5524]
+
+[osd.18]
+  host = cephstore5524
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5524]
+  public addr = [2607:f298:4:2243::5524]
+
+[osd.19]
+  host = cephstore5524
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5524]
+  public addr = [2607:f298:4:2243::5524]
+
+[osd.20]
+  host = cephstore5524
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5524]
+  public addr = [2607:f298:4:2243::5524]
+
+[osd.21]
+  host = cephstore5524
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5524]
+  public addr = [2607:f298:4:2243::5524]
+
+[osd.22]
+  host = cephstore5525
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5525]
+  public addr = [2607:f298:4:2243::5525]
+
+[osd.23]
+  host = cephstore5525
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5525]
+  public addr = [2607:f298:4:2243::5525]
+
+[osd.24]
+  host = cephstore5525
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5525]
+  public addr = [2607:f298:4:2243::5525]
+
+[osd.25]
+  host = cephstore5525
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5525]
+  public addr = [2607:f298:4:2243::5525]
+
+[osd.26]
+  host = cephstore5525
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5525]
+  public addr = [2607:f298:4:2243::5525]
+
+[osd.27]
+  host = cephstore5525
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5525]
+  public addr = [2607:f298:4:2243::5525]
+
+[osd.28]
+  host = cephstore5525
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5525]
+  public addr = [2607:f298:4:2243::5525]
+
+[osd.29]
+  host = cephstore5526
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5526]
+  public addr = [2607:f298:4:2243::5526]
+
+[osd.30]
+  host = cephstore5526
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5526]
+  public addr = [2607:f298:4:2243::5526]
+
+[osd.31]
+  host = cephstore5526
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5526]
+  public addr = [2607:f298:4:2243::5526]
+
+[osd.32]
+  host = cephstore5526
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5526]
+  public addr = [2607:f298:4:2243::5526]
+
+[osd.33]
+  host = cephstore5526
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5526]
+  public addr = [2607:f298:4:2243::5526]
+
+[osd.34]
+  host = cephstore5526
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5526]
+  public addr = [2607:f298:4:2243::5526]
+
+[osd.35]
+  host = cephstore5526
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5526]
+  public addr = [2607:f298:4:2243::5526]
+
+[osd.36]
+  host = cephstore5527
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5527]
+  public addr = [2607:f298:4:2243::5527]
+
+[osd.37]
+  host = cephstore5527
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5527]
+  public addr = [2607:f298:4:2243::5527]
+
+[osd.38]
+  host = cephstore5527
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5527]
+  public addr = [2607:f298:4:2243::5527]
+
+[osd.39]
+  host = cephstore5527
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5527]
+  public addr = [2607:f298:4:2243::5527]
+
+[osd.40]
+  host = cephstore5527
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5527]
+  public addr = [2607:f298:4:2243::5527]
+
+[osd.41]
+  host = cephstore5527
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5527]
+  public addr = [2607:f298:4:2243::5527]
+
+[osd.42]
+  host = cephstore5527
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5527]
+  public addr = [2607:f298:4:2243::5527]
+
+[osd.43]
+  host = cephstore5529
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5529]
+  public addr = [2607:f298:4:2243::5529]
+
+[osd.44]
+  host = cephstore5529
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5529]
+  public addr = [2607:f298:4:2243::5529]
+
+[osd.45]
+  host = cephstore5529
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5529]
+  public addr = [2607:f298:4:2243::5529]
+
+[osd.46]
+  host = cephstore5529
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5529]
+  public addr = [2607:f298:4:2243::5529]
+
+[osd.47]
+  host = cephstore5529
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5529]
+  public addr = [2607:f298:4:2243::5529]
+
+[osd.48]
+  host = cephstore5529
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5529]
+  public addr = [2607:f298:4:2243::5529]
+
+[osd.49]
+  host = cephstore5529
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5529]
+  public addr = [2607:f298:4:2243::5529]
+
+[osd.50]
+  host = cephstore5530
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5530]
+  public addr = [2607:f298:4:2243::5530]
+
+[osd.51]
+  host = cephstore5530
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5530]
+  public addr = [2607:f298:4:2243::5530]
+
+[osd.52]
+  host = cephstore5530
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5530]
+  public addr = [2607:f298:4:2243::5530]
+
+[osd.53]
+  host = cephstore5530
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5530]
+  public addr = [2607:f298:4:2243::5530]
+
+[osd.54]
+  host = cephstore5530
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5530]
+  public addr = [2607:f298:4:2243::5530]
+
+[osd.55]
+  host = cephstore5530
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5530]
+  public addr = [2607:f298:4:2243::5530]
+
+[osd.56]
+  host = cephstore5530
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::5530]
+  public addr = [2607:f298:4:2243::5530]
+
+[osd.57]
+  host = cephstore6230
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6230]
+  public addr = [2607:f298:4:2243::6230]
+
+[osd.58]
+  host = cephstore6230
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6230]
+  public addr = [2607:f298:4:2243::6230]
+
+[osd.59]
+  host = cephstore6230
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6230]
+  public addr = [2607:f298:4:2243::6230]
+
+[osd.60]
+  host = cephstore6230
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6230]
+  public addr = [2607:f298:4:2243::6230]
+
+[osd.61]
+  host = cephstore6230
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6230]
+  public addr = [2607:f298:4:2243::6230]
+
+[osd.62]
+  host = cephstore6230
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6230]
+  public addr = [2607:f298:4:2243::6230]
+
+[osd.63]
+  host = cephstore6230
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6230]
+  public addr = [2607:f298:4:2243::6230]
+
+[osd.64]
+  host = cephstore6231
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6231]
+  public addr = [2607:f298:4:2243::6231]
+
+[osd.65]
+  host = cephstore6231
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6231]
+  public addr = [2607:f298:4:2243::6231]
+
+[osd.66]
+  host = cephstore6231
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6231]
+  public addr = [2607:f298:4:2243::6231]
+
+[osd.67]
+  host = cephstore6231
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6231]
+  public addr = [2607:f298:4:2243::6231]
+
+[osd.68]
+  host = cephstore6231
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6231]
+  public addr = [2607:f298:4:2243::6231]
+
+[osd.69]
+  host = cephstore6231
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6231]
+  public addr = [2607:f298:4:2243::6231]
+
+[osd.70]
+  host = cephstore6231
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6231]
+  public addr = [2607:f298:4:2243::6231]
+
+[osd.71]
+  host = cephstore6232
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6232]
+  public addr = [2607:f298:4:2243::6232]
+
+[osd.72]
+  host = cephstore6232
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6232]
+  public addr = [2607:f298:4:2243::6232]
+
+[osd.73]
+  host = cephstore6232
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6232]
+  public addr = [2607:f298:4:2243::6232]
+
+[osd.74]
+  host = cephstore6232
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6232]
+  public addr = [2607:f298:4:2243::6232]
+
+[osd.75]
+  host = cephstore6232
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6232]
+  public addr = [2607:f298:4:2243::6232]
+
+[osd.76]
+  host = cephstore6232
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6232]
+  public addr = [2607:f298:4:2243::6232]
+
+[osd.77]
+  host = cephstore6232
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6232]
+  public addr = [2607:f298:4:2243::6232]
+
+[osd.78]
+  host = cephstore6233
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6233]
+  public addr = [2607:f298:4:2243::6233]
+
+[osd.79]
+  host = cephstore6233
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6233]
+  public addr = [2607:f298:4:2243::6233]
+
+[osd.80]
+  host = cephstore6233
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6233]
+  public addr = [2607:f298:4:2243::6233]
+
+[osd.81]
+  host = cephstore6233
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6233]
+  public addr = [2607:f298:4:2243::6233]
+
+[osd.82]
+  host = cephstore6233
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6233]
+  public addr = [2607:f298:4:2243::6233]
+
+[osd.83]
+  host = cephstore6233
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6233]
+  public addr = [2607:f298:4:2243::6233]
+
+[osd.84]
+  host = cephstore6233
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6233]
+  public addr = [2607:f298:4:2243::6233]
+
+[osd.85]
+  host = cephstore6234
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6234]
+  public addr = [2607:f298:4:2243::6234]
+
+[osd.86]
+  host = cephstore6234
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6234]
+  public addr = [2607:f298:4:2243::6234]
+
+[osd.87]
+  host = cephstore6234
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6234]
+  public addr = [2607:f298:4:2243::6234]
+
+[osd.88]
+  host = cephstore6234
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6234]
+  public addr = [2607:f298:4:2243::6234]
+
+[osd.89]
+  host = cephstore6234
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6234]
+  public addr = [2607:f298:4:2243::6234]
+
+[osd.90]
+  host = cephstore6234
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6234]
+  public addr = [2607:f298:4:2243::6234]
+
+[osd.91]
+  host = cephstore6234
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6234]
+  public addr = [2607:f298:4:2243::6234]
+
+[osd.92]
+  host = cephstore6235
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6235]
+  public addr = [2607:f298:4:2243::6235]
+
+[osd.93]
+  host = cephstore6235
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6235]
+  public addr = [2607:f298:4:2243::6235]
+
+[osd.94]
+  host = cephstore6235
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6235]
+  public addr = [2607:f298:4:2243::6235]
+
+[osd.95]
+  host = cephstore6235
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6235]
+  public addr = [2607:f298:4:2243::6235]
+
+[osd.96]
+  host = cephstore6235
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6235]
+  public addr = [2607:f298:4:2243::6235]
+
+[osd.97]
+  host = cephstore6235
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6235]
+  public addr = [2607:f298:4:2243::6235]
+
+[osd.98]
+  host = cephstore6235
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6235]
+  public addr = [2607:f298:4:2243::6235]
+
+[osd.99]
+  host = cephstore6236
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6236]
+  public addr = [2607:f298:4:2243::6236]
+
+[osd.100]
+  host = cephstore6236
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6236]
+  public addr = [2607:f298:4:2243::6236]
+
+[osd.101]
+  host = cephstore6236
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6236]
+  public addr = [2607:f298:4:2243::6236]
+
+[osd.102]
+  host = cephstore6236
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6236]
+  public addr = [2607:f298:4:2243::6236]
+
+[osd.103]
+  host = cephstore6236
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6236]
+  public addr = [2607:f298:4:2243::6236]
+
+[osd.104]
+  host = cephstore6236
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6236]
+  public addr = [2607:f298:4:2243::6236]
+
+[osd.105]
+  host = cephstore6236
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6236]
+  public addr = [2607:f298:4:2243::6236]
+
+[osd.106]
+  host = cephstore6237
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6237]
+  public addr = [2607:f298:4:2243::6237]
+
+[osd.107]
+  host = cephstore6237
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6237]
+  public addr = [2607:f298:4:2243::6237]
+
+[osd.108]
+  host = cephstore6237
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6237]
+  public addr = [2607:f298:4:2243::6237]
+
+[osd.109]
+  host = cephstore6237
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6237]
+  public addr = [2607:f298:4:2243::6237]
+
+[osd.110]
+  host = cephstore6237
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6237]
+  public addr = [2607:f298:4:2243::6237]
+
+[osd.111]
+  host = cephstore6237
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6237]
+  public addr = [2607:f298:4:2243::6237]
+
+[osd.112]
+  host = cephstore6237
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6237]
+  public addr = [2607:f298:4:2243::6237]
+
+[osd.113]
+  host = cephstore6238
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6238]
+  public addr = [2607:f298:4:2243::6238]
+
+[osd.114]
+  host = cephstore6238
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6238]
+  public addr = [2607:f298:4:2243::6238]
+
+[osd.115]
+  host = cephstore6238
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6238]
+  public addr = [2607:f298:4:2243::6238]
+
+[osd.116]
+  host = cephstore6238
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6238]
+  public addr = [2607:f298:4:2243::6238]
+
+[osd.117]
+  host = cephstore6238
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6238]
+  public addr = [2607:f298:4:2243::6238]
+
+[osd.118]
+  host = cephstore6238
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6238]
+  public addr = [2607:f298:4:2243::6238]
+
+[osd.119]
+  host = cephstore6238
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6238]
+  public addr = [2607:f298:4:2243::6238]
+
+[osd.120]
+  host = cephstore6239
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6239]
+  public addr = [2607:f298:4:2243::6239]
+
+[osd.121]
+  host = cephstore6239
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6239]
+  public addr = [2607:f298:4:2243::6239]
+
+[osd.122]
+  host = cephstore6239
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6239]
+  public addr = [2607:f298:4:2243::6239]
+
+[osd.123]
+  host = cephstore6239
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6239]
+  public addr = [2607:f298:4:2243::6239]
+
+[osd.124]
+  host = cephstore6239
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6239]
+  public addr = [2607:f298:4:2243::6239]
+
+[osd.125]
+  host = cephstore6239
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6239]
+  public addr = [2607:f298:4:2243::6239]
+
+[osd.126]
+  host = cephstore6239
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6239]
+  public addr = [2607:f298:4:2243::6239]
+
+[osd.127]
+  host = cephstore6240
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6240]
+  public addr = [2607:f298:4:2243::6240]
+
+[osd.128]
+  host = cephstore6240
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6240]
+  public addr = [2607:f298:4:2243::6240]
+
+[osd.129]
+  host = cephstore6240
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6240]
+  public addr = [2607:f298:4:2243::6240]
+
+[osd.130]
+  host = cephstore6240
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6240]
+  public addr = [2607:f298:4:2243::6240]
+
+[osd.131]
+  host = cephstore6240
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6240]
+  public addr = [2607:f298:4:2243::6240]
+
+[osd.132]
+  host = cephstore6240
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6240]
+  public addr = [2607:f298:4:2243::6240]
+
+[osd.133]
+  host = cephstore6240
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6240]
+  public addr = [2607:f298:4:2243::6240]
+
+[osd.134]
+  host = cephstore6241
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6241]
+  public addr = [2607:f298:4:2243::6241]
+
+[osd.135]
+  host = cephstore6241
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6241]
+  public addr = [2607:f298:4:2243::6241]
+
+[osd.136]
+  host = cephstore6241
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6241]
+  public addr = [2607:f298:4:2243::6241]
+
+[osd.137]
+  host = cephstore6241
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6241]
+  public addr = [2607:f298:4:2243::6241]
+
+[osd.138]
+  host = cephstore6241
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6241]
+  public addr = [2607:f298:4:2243::6241]
+
+[osd.139]
+  host = cephstore6241
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6241]
+  public addr = [2607:f298:4:2243::6241]
+
+[osd.140]
+  host = cephstore6241
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6241]
+  public addr = [2607:f298:4:2243::6241]
+
+[osd.141]
+  host = cephstore6242
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6242]
+  public addr = [2607:f298:4:2243::6242]
+
+[osd.142]
+  host = cephstore6242
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6242]
+  public addr = [2607:f298:4:2243::6242]
+
+[osd.143]
+  host = cephstore6242
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6242]
+  public addr = [2607:f298:4:2243::6242]
+
+[osd.144]
+  host = cephstore6242
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6242]
+  public addr = [2607:f298:4:2243::6242]
+
+[osd.145]
+  host = cephstore6242
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6242]
+  public addr = [2607:f298:4:2243::6242]
+
+[osd.146]
+  host = cephstore6242
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6242]
+  public addr = [2607:f298:4:2243::6242]
+
+[osd.147]
+  host = cephstore6242
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6242]
+  public addr = [2607:f298:4:2243::6242]
+
+[osd.148]
+  host = cephstore6243
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6243]
+  public addr = [2607:f298:4:2243::6243]
+
+[osd.149]
+  host = cephstore6243
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6243]
+  public addr = [2607:f298:4:2243::6243]
+
+[osd.150]
+  host = cephstore6243
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6243]
+  public addr = [2607:f298:4:2243::6243]
+
+[osd.151]
+  host = cephstore6243
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6243]
+  public addr = [2607:f298:4:2243::6243]
+
+[osd.152]
+  host = cephstore6243
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6243]
+  public addr = [2607:f298:4:2243::6243]
+
+[osd.153]
+  host = cephstore6243
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6243]
+  public addr = [2607:f298:4:2243::6243]
+
+[osd.154]
+  host = cephstore6243
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6243]
+  public addr = [2607:f298:4:2243::6243]
+
+[osd.155]
+  host = cephstore6244
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6244]
+  public addr = [2607:f298:4:2243::6244]
+
+[osd.156]
+  host = cephstore6244
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6244]
+  public addr = [2607:f298:4:2243::6244]
+
+[osd.157]
+  host = cephstore6244
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6244]
+  public addr = [2607:f298:4:2243::6244]
+
+[osd.158]
+  host = cephstore6244
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6244]
+  public addr = [2607:f298:4:2243::6244]
+
+[osd.159]
+  host = cephstore6244
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6244]
+  public addr = [2607:f298:4:2243::6244]
+
+[osd.160]
+  host = cephstore6244
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6244]
+  public addr = [2607:f298:4:2243::6244]
+
+[osd.161]
+  host = cephstore6244
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6244]
+  public addr = [2607:f298:4:2243::6244]
+
+[osd.162]
+  host = cephstore6245
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6245]
+  public addr = [2607:f298:4:2243::6245]
+
+[osd.163]
+  host = cephstore6245
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6245]
+  public addr = [2607:f298:4:2243::6245]
+
+[osd.164]
+  host = cephstore6245
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6245]
+  public addr = [2607:f298:4:2243::6245]
+
+[osd.165]
+  host = cephstore6245
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6245]
+  public addr = [2607:f298:4:2243::6245]
+
+[osd.166]
+  host = cephstore6245
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6245]
+  public addr = [2607:f298:4:2243::6245]
+
+[osd.167]
+  host = cephstore6245
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6245]
+  public addr = [2607:f298:4:2243::6245]
+
+[osd.168]
+  host = cephstore6245
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6245]
+  public addr = [2607:f298:4:2243::6245]
+
+[osd.169]
+  host = cephstore6246
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6246]
+  public addr = [2607:f298:4:2243::6246]
+
+[osd.170]
+  host = cephstore6246
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6246]
+  public addr = [2607:f298:4:2243::6246]
+
+[osd.171]
+  host = cephstore6246
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6246]
+  public addr = [2607:f298:4:2243::6246]
+
+[osd.172]
+  host = cephstore6246
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6246]
+  public addr = [2607:f298:4:2243::6246]
+
+[osd.173]
+  host = cephstore6246
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6246]
+  public addr = [2607:f298:4:2243::6246]
+
+[osd.174]
+  host = cephstore6246
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6246]
+  public addr = [2607:f298:4:2243::6246]
+
+[osd.175]
+  host = cephstore6246
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6246]
+  public addr = [2607:f298:4:2243::6246]
+
+[osd.176]
+  host = cephstore6336
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6336]
+  public addr = [2607:f298:4:2243::6336]
+
+[osd.177]
+  host = cephstore6336
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6336]
+  public addr = [2607:f298:4:2243::6336]
+
+[osd.178]
+  host = cephstore6336
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6336]
+  public addr = [2607:f298:4:2243::6336]
+
+[osd.179]
+  host = cephstore6336
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6336]
+  public addr = [2607:f298:4:2243::6336]
+
+[osd.180]
+  host = cephstore6336
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6336]
+  public addr = [2607:f298:4:2243::6336]
+
+[osd.181]
+  host = cephstore6336
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6336]
+  public addr = [2607:f298:4:2243::6336]
+
+[osd.182]
+  host = cephstore6336
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6336]
+  public addr = [2607:f298:4:2243::6336]
+
+[osd.183]
+  host = cephstore6337
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6337]
+  public addr = [2607:f298:4:2243::6337]
+
+[osd.184]
+  host = cephstore6337
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6337]
+  public addr = [2607:f298:4:2243::6337]
+
+[osd.185]
+  host = cephstore6337
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6337]
+  public addr = [2607:f298:4:2243::6337]
+
+[osd.186]
+  host = cephstore6337
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6337]
+  public addr = [2607:f298:4:2243::6337]
+
+[osd.187]
+  host = cephstore6337
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6337]
+  public addr = [2607:f298:4:2243::6337]
+
+[osd.188]
+  host = cephstore6337
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6337]
+  public addr = [2607:f298:4:2243::6337]
+
+[osd.189]
+  host = cephstore6337
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6337]
+  public addr = [2607:f298:4:2243::6337]
+
+[osd.190]
+  host = cephstore6338
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6338]
+  public addr = [2607:f298:4:2243::6338]
+
+[osd.191]
+  host = cephstore6338
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6338]
+  public addr = [2607:f298:4:2243::6338]
+
+[osd.192]
+  host = cephstore6338
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6338]
+  public addr = [2607:f298:4:2243::6338]
+
+[osd.193]
+  host = cephstore6338
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6338]
+  public addr = [2607:f298:4:2243::6338]
+
+[osd.194]
+  host = cephstore6338
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6338]
+  public addr = [2607:f298:4:2243::6338]
+
+[osd.195]
+  host = cephstore6338
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6338]
+  public addr = [2607:f298:4:2243::6338]
+
+[osd.196]
+  host = cephstore6338
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6338]
+  public addr = [2607:f298:4:2243::6338]
+
+[osd.197]
+  host = cephstore6339
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6339]
+  public addr = [2607:f298:4:2243::6339]
+
+[osd.198]
+  host = cephstore6339
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6339]
+  public addr = [2607:f298:4:2243::6339]
+
+[osd.199]
+  host = cephstore6339
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6339]
+  public addr = [2607:f298:4:2243::6339]
+
+[osd.200]
+  host = cephstore6339
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6339]
+  public addr = [2607:f298:4:2243::6339]
+
+[osd.201]
+  host = cephstore6339
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6339]
+  public addr = [2607:f298:4:2243::6339]
+
+[osd.202]
+  host = cephstore6339
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6339]
+  public addr = [2607:f298:4:2243::6339]
+
+[osd.203]
+  host = cephstore6339
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6339]
+  public addr = [2607:f298:4:2243::6339]
+
+[osd.204]
+  host = cephstore6340
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6340]
+  public addr = [2607:f298:4:2243::6340]
+
+[osd.205]
+  host = cephstore6340
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6340]
+  public addr = [2607:f298:4:2243::6340]
+
+[osd.206]
+  host = cephstore6340
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6340]
+  public addr = [2607:f298:4:2243::6340]
+
+[osd.207]
+  host = cephstore6340
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6340]
+  public addr = [2607:f298:4:2243::6340]
+
+[osd.208]
+  host = cephstore6340
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6340]
+  public addr = [2607:f298:4:2243::6340]
+
+[osd.209]
+  host = cephstore6340
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6340]
+  public addr = [2607:f298:4:2243::6340]
+
+[osd.210]
+  host = cephstore6340
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6340]
+  public addr = [2607:f298:4:2243::6340]
+
+[osd.211]
+  host = cephstore6341
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6341]
+  public addr = [2607:f298:4:2243::6341]
+
+[osd.212]
+  host = cephstore6341
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6341]
+  public addr = [2607:f298:4:2243::6341]
+
+[osd.213]
+  host = cephstore6341
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6341]
+  public addr = [2607:f298:4:2243::6341]
+
+[osd.214]
+  host = cephstore6341
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6341]
+  public addr = [2607:f298:4:2243::6341]
+
+[osd.215]
+  host = cephstore6341
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6341]
+  public addr = [2607:f298:4:2243::6341]
+
+[osd.216]
+  host = cephstore6341
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6341]
+  public addr = [2607:f298:4:2243::6341]
+
+[osd.217]
+  host = cephstore6341
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6341]
+  public addr = [2607:f298:4:2243::6341]
+
+[osd.218]
+  host = cephstore6342
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6342]
+  public addr = [2607:f298:4:2243::6342]
+
+[osd.219]
+  host = cephstore6342
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6342]
+  public addr = [2607:f298:4:2243::6342]
+
+[osd.220]
+  host = cephstore6342
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6342]
+  public addr = [2607:f298:4:2243::6342]
+
+[osd.221]
+  host = cephstore6342
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6342]
+  public addr = [2607:f298:4:2243::6342]
+
+[osd.222]
+  host = cephstore6342
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6342]
+  public addr = [2607:f298:4:2243::6342]
+
+[osd.223]
+  host = cephstore6342
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6342]
+  public addr = [2607:f298:4:2243::6342]
+
+[osd.224]
+  host = cephstore6342
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6342]
+  public addr = [2607:f298:4:2243::6342]
+
+[osd.225]
+  host = cephstore6343
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6343]
+  public addr = [2607:f298:4:2243::6343]
+
+[osd.226]
+  host = cephstore6343
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6343]
+  public addr = [2607:f298:4:2243::6343]
+
+[osd.227]
+  host = cephstore6343
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6343]
+  public addr = [2607:f298:4:2243::6343]
+
+[osd.228]
+  host = cephstore6343
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6343]
+  public addr = [2607:f298:4:2243::6343]
+
+[osd.229]
+  host = cephstore6343
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6343]
+  public addr = [2607:f298:4:2243::6343]
+
+[osd.230]
+  host = cephstore6343
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6343]
+  public addr = [2607:f298:4:2243::6343]
+
+[osd.231]
+  host = cephstore6343
+  rack = irv-n1
+  cluster addr = [2607:f298:4:3243::6343]
+  public addr = [2607:f298:4:2243::6343]
+
+[osd.232]
+  host = cephstore6345
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6345]
+  public addr = [2607:f298:4:2243::6345]
+
+[osd.233]
+  host = cephstore6345
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6345]
+  public addr = [2607:f298:4:2243::6345]
+
+[osd.234]
+  host = cephstore6345
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6345]
+  public addr = [2607:f298:4:2243::6345]
+
+[osd.235]
+  host = cephstore6345
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6345]
+  public addr = [2607:f298:4:2243::6345]
+
+[osd.236]
+  host = cephstore6345
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6345]
+  public addr = [2607:f298:4:2243::6345]
+
+[osd.237]
+  host = cephstore6345
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6345]
+  public addr = [2607:f298:4:2243::6345]
+
+[osd.238]
+  host = cephstore6345
+  rack = irv-n2
+  cluster addr = [2607:f298:4:3243::6345]
+  public addr = [2607:f298:4:2243::6345]
+
diff --git a/ceph/src/test/cli/osdmaptool/clobber.t b/ceph/src/test/cli/osdmaptool/clobber.t
new file mode 100644
index 00000000..5f37f034
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/clobber.t
@@ -0,0 +1,57 @@
+  $ osdmaptool --createsimple 3 myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: writing epoch 1 to myosdmap
+
+  $ ORIG_FSID="$(osdmaptool --print myosdmap|grep ^fsid)"
+  osdmaptool: osdmap file 'myosdmap'
+
+  $ osdmaptool --createsimple 3 myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: myosdmap exists, --clobber to overwrite
+  [255]
+
+# hasn't changed yet
+#TODO typo
+  $ osdmaptool --print myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  epoch 1
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  flags 
+  
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool stripe_width 0
+  pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool stripe_width 0
+  
+  max_osd 3
+  
+
+  $ NEW_FSID="$(osdmaptool --print myosdmap|grep ^fsid)"
+  osdmaptool: osdmap file 'myosdmap'
+  $ [ "$ORIG_FSID" = "$NEW_FSID" ]
+
+  $ osdmaptool --createsimple 1 --clobber myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: writing epoch 1 to myosdmap
+
+  $ osdmaptool --print myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  epoch 1
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  flags 
+  
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 flags hashpspool stripe_width 0
+  pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 0 flags hashpspool stripe_width 0
+  
+  max_osd 1
+  
+
+  $ NEW_FSID="$(osdmaptool --print myosdmap|grep ^fsid)"
+  osdmaptool: osdmap file 'myosdmap'
+#TODO --clobber should probably set new fsid, remove the [1]
+  $ [ "$ORIG_FSID" != "$NEW_FSID" ]
+  [1]
diff --git a/ceph/src/test/cli/osdmaptool/create-print.t b/ceph/src/test/cli/osdmaptool/create-print.t
new file mode 100644
index 00000000..9ebd2747
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/create-print.t
@@ -0,0 +1,104 @@
+  $ osdmaptool --createsimple 3 myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: writing epoch 1 to myosdmap
+
+  $ osdmaptool --export-crush oc myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: exported crush map to oc
+  $ crushtool --decompile oc
+  # begin crush map
+  tunable choose_local_tries 0
+  tunable choose_local_fallback_tries 0
+  tunable choose_total_tries 50
+  tunable chooseleaf_descend_once 1
+  
+  # devices
+  device 0 osd.0
+  device 1 osd.1
+  device 2 osd.2
+  
+  # types
+  type 0 osd
+  type 1 host
+  type 2 chassis
+  type 3 rack
+  type 4 row
+  type 5 pdu
+  type 6 pod
+  type 7 room
+  type 8 datacenter
+  type 9 region
+  type 10 root
+  
+  # buckets
+  host localhost {
+  \tid -2\t\t# do not change unnecessarily (esc)
+  \t# weight 3.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.0 weight 1.000 (esc)
+  \titem osd.1 weight 1.000 (esc)
+  \titem osd.2 weight 1.000 (esc)
+  }
+  rack localrack {
+  \tid -3\t\t# do not change unnecessarily (esc)
+  \t# weight 3.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem localhost weight 3.000 (esc)
+  }
+  root default {
+  \tid -1\t\t# do not change unnecessarily (esc)
+  \t# weight 3.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem localrack weight 3.000 (esc)
+  }
+  
+  # rules
+  rule replicated_ruleset {
+  \truleset 0 (esc)
+  \ttype replicated (esc)
+  \tmin_size 1 (esc)
+  \tmax_size 10 (esc)
+  \tstep take default (esc)
+  \tstep chooseleaf firstn 0 type host (esc)
+  \tstep emit (esc)
+  }
+  
+  # end crush map
+  $ osdmaptool --print myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  epoch 1
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  flags 
+  
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool stripe_width 0
+  pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool stripe_width 0
+  
+  max_osd 3
+  
+  $ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_replicated_ruleset 66 myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: writing epoch 1 to myosdmap
+  $ osdmaptool --print myosdmap | grep 'pool 0'
+  osdmaptool: osdmap file 'myosdmap'
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 66 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  $ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_rule 55 myosdmap 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+  osdmaptool: osdmap file 'myosdmap'
+  default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+  default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+  $ osdmaptool --print myosdmap | grep 'pool 0'
+  osdmaptool: osdmap file 'myosdmap'
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  $ osdmaptool --clobber --createsimple 3 --osd_pool_default_crush_replicated_ruleset 66 --osd_pool_default_crush_rule 55 myosdmap 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+  osdmaptool: osdmap file 'myosdmap'
+  default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+  default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+  $ osdmaptool --print myosdmap | grep 'pool 0'
+  osdmaptool: osdmap file 'myosdmap'
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 192 pgp_num 192 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  $ rm -f myosdmap
diff --git a/ceph/src/test/cli/osdmaptool/create-racks.t b/ceph/src/test/cli/osdmaptool/create-racks.t
new file mode 100644
index 00000000..33fa9eef
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/create-racks.t
@@ -0,0 +1,818 @@
+  $ osdmaptool --create-from-conf om -c $TESTDIR/ceph.conf.withracks
+  osdmaptool: osdmap file 'om'
+  osdmaptool: writing epoch 1 to om
+  $ osdmaptool --export-crush oc om
+  osdmaptool: osdmap file 'om'
+  osdmaptool: exported crush map to oc
+  $ crushtool --decompile oc
+  # begin crush map
+  tunable choose_local_tries 0
+  tunable choose_local_fallback_tries 0
+  tunable choose_total_tries 50
+  tunable chooseleaf_descend_once 1
+  
+  # devices
+  device 0 device0
+  device 1 osd.1
+  device 2 osd.2
+  device 3 osd.3
+  device 4 osd.4
+  device 5 osd.5
+  device 6 osd.6
+  device 7 osd.7
+  device 8 osd.8
+  device 9 osd.9
+  device 10 osd.10
+  device 11 osd.11
+  device 12 osd.12
+  device 13 osd.13
+  device 14 osd.14
+  device 15 osd.15
+  device 16 osd.16
+  device 17 osd.17
+  device 18 osd.18
+  device 19 osd.19
+  device 20 osd.20
+  device 21 osd.21
+  device 22 osd.22
+  device 23 osd.23
+  device 24 osd.24
+  device 25 osd.25
+  device 26 osd.26
+  device 27 osd.27
+  device 28 osd.28
+  device 29 osd.29
+  device 30 osd.30
+  device 31 osd.31
+  device 32 osd.32
+  device 33 osd.33
+  device 34 osd.34
+  device 35 osd.35
+  device 36 osd.36
+  device 37 osd.37
+  device 38 osd.38
+  device 39 osd.39
+  device 40 osd.40
+  device 41 osd.41
+  device 42 osd.42
+  device 43 osd.43
+  device 44 osd.44
+  device 45 osd.45
+  device 46 osd.46
+  device 47 osd.47
+  device 48 osd.48
+  device 49 osd.49
+  device 50 osd.50
+  device 51 osd.51
+  device 52 osd.52
+  device 53 osd.53
+  device 54 osd.54
+  device 55 osd.55
+  device 56 osd.56
+  device 57 osd.57
+  device 58 osd.58
+  device 59 osd.59
+  device 60 osd.60
+  device 61 osd.61
+  device 62 osd.62
+  device 63 osd.63
+  device 64 osd.64
+  device 65 osd.65
+  device 66 osd.66
+  device 67 osd.67
+  device 68 osd.68
+  device 69 osd.69
+  device 70 osd.70
+  device 71 osd.71
+  device 72 osd.72
+  device 73 osd.73
+  device 74 osd.74
+  device 75 osd.75
+  device 76 osd.76
+  device 77 osd.77
+  device 78 osd.78
+  device 79 osd.79
+  device 80 osd.80
+  device 81 osd.81
+  device 82 osd.82
+  device 83 osd.83
+  device 84 osd.84
+  device 85 osd.85
+  device 86 osd.86
+  device 87 osd.87
+  device 88 osd.88
+  device 89 osd.89
+  device 90 osd.90
+  device 91 osd.91
+  device 92 osd.92
+  device 93 osd.93
+  device 94 osd.94
+  device 95 osd.95
+  device 96 osd.96
+  device 97 osd.97
+  device 98 osd.98
+  device 99 osd.99
+  device 100 osd.100
+  device 101 osd.101
+  device 102 osd.102
+  device 103 osd.103
+  device 104 osd.104
+  device 105 osd.105
+  device 106 osd.106
+  device 107 osd.107
+  device 108 osd.108
+  device 109 osd.109
+  device 110 osd.110
+  device 111 osd.111
+  device 112 osd.112
+  device 113 osd.113
+  device 114 osd.114
+  device 115 osd.115
+  device 116 osd.116
+  device 117 osd.117
+  device 118 osd.118
+  device 119 osd.119
+  device 120 osd.120
+  device 121 osd.121
+  device 122 osd.122
+  device 123 osd.123
+  device 124 osd.124
+  device 125 osd.125
+  device 126 osd.126
+  device 127 osd.127
+  device 128 osd.128
+  device 129 osd.129
+  device 130 osd.130
+  device 131 osd.131
+  device 132 osd.132
+  device 133 osd.133
+  device 134 osd.134
+  device 135 osd.135
+  device 136 osd.136
+  device 137 osd.137
+  device 138 osd.138
+  device 139 osd.139
+  device 140 osd.140
+  device 141 osd.141
+  device 142 osd.142
+  device 143 osd.143
+  device 144 osd.144
+  device 145 osd.145
+  device 146 osd.146
+  device 147 osd.147
+  device 148 osd.148
+  device 149 osd.149
+  device 150 osd.150
+  device 151 osd.151
+  device 152 osd.152
+  device 153 osd.153
+  device 154 osd.154
+  device 155 osd.155
+  device 156 osd.156
+  device 157 osd.157
+  device 158 osd.158
+  device 159 osd.159
+  device 160 osd.160
+  device 161 osd.161
+  device 162 osd.162
+  device 163 osd.163
+  device 164 osd.164
+  device 165 osd.165
+  device 166 osd.166
+  device 167 osd.167
+  device 168 osd.168
+  device 169 osd.169
+  device 170 osd.170
+  device 171 osd.171
+  device 172 osd.172
+  device 173 osd.173
+  device 174 osd.174
+  device 175 osd.175
+  device 176 osd.176
+  device 177 osd.177
+  device 178 osd.178
+  device 179 osd.179
+  device 180 osd.180
+  device 181 osd.181
+  device 182 osd.182
+  device 183 osd.183
+  device 184 osd.184
+  device 185 osd.185
+  device 186 osd.186
+  device 187 osd.187
+  device 188 osd.188
+  device 189 osd.189
+  device 190 osd.190
+  device 191 osd.191
+  device 192 osd.192
+  device 193 osd.193
+  device 194 osd.194
+  device 195 osd.195
+  device 196 osd.196
+  device 197 osd.197
+  device 198 osd.198
+  device 199 osd.199
+  device 200 osd.200
+  device 201 osd.201
+  device 202 osd.202
+  device 203 osd.203
+  device 204 osd.204
+  device 205 osd.205
+  device 206 osd.206
+  device 207 osd.207
+  device 208 osd.208
+  device 209 osd.209
+  device 210 osd.210
+  device 211 osd.211
+  device 212 osd.212
+  device 213 osd.213
+  device 214 osd.214
+  device 215 osd.215
+  device 216 osd.216
+  device 217 osd.217
+  device 218 osd.218
+  device 219 osd.219
+  device 220 osd.220
+  device 221 osd.221
+  device 222 osd.222
+  device 223 osd.223
+  device 224 osd.224
+  device 225 osd.225
+  device 226 osd.226
+  device 227 osd.227
+  device 228 osd.228
+  device 229 osd.229
+  device 230 osd.230
+  device 231 osd.231
+  device 232 osd.232
+  device 233 osd.233
+  device 234 osd.234
+  device 235 osd.235
+  device 236 osd.236
+  device 237 osd.237
+  device 238 osd.238
+  
+  # types
+  type 0 osd
+  type 1 host
+  type 2 chassis
+  type 3 rack
+  type 4 row
+  type 5 pdu
+  type 6 pod
+  type 7 room
+  type 8 datacenter
+  type 9 region
+  type 10 root
+  
+  # buckets
+  host cephstore5522 {
+  \tid -2\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.1 weight 1.000 (esc)
+  \titem osd.2 weight 1.000 (esc)
+  \titem osd.3 weight 1.000 (esc)
+  \titem osd.4 weight 1.000 (esc)
+  \titem osd.5 weight 1.000 (esc)
+  \titem osd.6 weight 1.000 (esc)
+  \titem osd.7 weight 1.000 (esc)
+  }
+  host cephstore5523 {
+  \tid -4\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.10 weight 1.000 (esc)
+  \titem osd.11 weight 1.000 (esc)
+  \titem osd.12 weight 1.000 (esc)
+  \titem osd.13 weight 1.000 (esc)
+  \titem osd.14 weight 1.000 (esc)
+  \titem osd.8 weight 1.000 (esc)
+  \titem osd.9 weight 1.000 (esc)
+  }
+  host cephstore6238 {
+  \tid -8\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.113 weight 1.000 (esc)
+  \titem osd.114 weight 1.000 (esc)
+  \titem osd.115 weight 1.000 (esc)
+  \titem osd.116 weight 1.000 (esc)
+  \titem osd.117 weight 1.000 (esc)
+  \titem osd.118 weight 1.000 (esc)
+  \titem osd.119 weight 1.000 (esc)
+  }
+  host cephstore6240 {
+  \tid -10\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.127 weight 1.000 (esc)
+  \titem osd.128 weight 1.000 (esc)
+  \titem osd.129 weight 1.000 (esc)
+  \titem osd.130 weight 1.000 (esc)
+  \titem osd.131 weight 1.000 (esc)
+  \titem osd.132 weight 1.000 (esc)
+  \titem osd.133 weight 1.000 (esc)
+  }
+  host cephstore6242 {
+  \tid -12\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.141 weight 1.000 (esc)
+  \titem osd.142 weight 1.000 (esc)
+  \titem osd.143 weight 1.000 (esc)
+  \titem osd.144 weight 1.000 (esc)
+  \titem osd.145 weight 1.000 (esc)
+  \titem osd.146 weight 1.000 (esc)
+  \titem osd.147 weight 1.000 (esc)
+  }
+  host cephstore5524 {
+  \tid -14\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.15 weight 1.000 (esc)
+  \titem osd.16 weight 1.000 (esc)
+  \titem osd.17 weight 1.000 (esc)
+  \titem osd.18 weight 1.000 (esc)
+  \titem osd.19 weight 1.000 (esc)
+  \titem osd.20 weight 1.000 (esc)
+  \titem osd.21 weight 1.000 (esc)
+  }
+  host cephstore6244 {
+  \tid -15\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.155 weight 1.000 (esc)
+  \titem osd.156 weight 1.000 (esc)
+  \titem osd.157 weight 1.000 (esc)
+  \titem osd.158 weight 1.000 (esc)
+  \titem osd.159 weight 1.000 (esc)
+  \titem osd.160 weight 1.000 (esc)
+  \titem osd.161 weight 1.000 (esc)
+  }
+  host cephstore6246 {
+  \tid -17\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.169 weight 1.000 (esc)
+  \titem osd.170 weight 1.000 (esc)
+  \titem osd.171 weight 1.000 (esc)
+  \titem osd.172 weight 1.000 (esc)
+  \titem osd.173 weight 1.000 (esc)
+  \titem osd.174 weight 1.000 (esc)
+  \titem osd.175 weight 1.000 (esc)
+  }
+  host cephstore6337 {
+  \tid -19\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.183 weight 1.000 (esc)
+  \titem osd.184 weight 1.000 (esc)
+  \titem osd.185 weight 1.000 (esc)
+  \titem osd.186 weight 1.000 (esc)
+  \titem osd.187 weight 1.000 (esc)
+  \titem osd.188 weight 1.000 (esc)
+  \titem osd.189 weight 1.000 (esc)
+  }
+  host cephstore6341 {
+  \tid -23\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.211 weight 1.000 (esc)
+  \titem osd.212 weight 1.000 (esc)
+  \titem osd.213 weight 1.000 (esc)
+  \titem osd.214 weight 1.000 (esc)
+  \titem osd.215 weight 1.000 (esc)
+  \titem osd.216 weight 1.000 (esc)
+  \titem osd.217 weight 1.000 (esc)
+  }
+  host cephstore6342 {
+  \tid -24\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.218 weight 1.000 (esc)
+  \titem osd.219 weight 1.000 (esc)
+  \titem osd.220 weight 1.000 (esc)
+  \titem osd.221 weight 1.000 (esc)
+  \titem osd.222 weight 1.000 (esc)
+  \titem osd.223 weight 1.000 (esc)
+  \titem osd.224 weight 1.000 (esc)
+  }
+  host cephstore5525 {
+  \tid -25\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.22 weight 1.000 (esc)
+  \titem osd.23 weight 1.000 (esc)
+  \titem osd.24 weight 1.000 (esc)
+  \titem osd.25 weight 1.000 (esc)
+  \titem osd.26 weight 1.000 (esc)
+  \titem osd.27 weight 1.000 (esc)
+  \titem osd.28 weight 1.000 (esc)
+  }
+  host cephstore6345 {
+  \tid -27\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.232 weight 1.000 (esc)
+  \titem osd.233 weight 1.000 (esc)
+  \titem osd.234 weight 1.000 (esc)
+  \titem osd.235 weight 1.000 (esc)
+  \titem osd.236 weight 1.000 (esc)
+  \titem osd.237 weight 1.000 (esc)
+  \titem osd.238 weight 1.000 (esc)
+  }
+  host cephstore5526 {
+  \tid -28\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.29 weight 1.000 (esc)
+  \titem osd.30 weight 1.000 (esc)
+  \titem osd.31 weight 1.000 (esc)
+  \titem osd.32 weight 1.000 (esc)
+  \titem osd.33 weight 1.000 (esc)
+  \titem osd.34 weight 1.000 (esc)
+  \titem osd.35 weight 1.000 (esc)
+  }
+  host cephstore5527 {
+  \tid -29\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.36 weight 1.000 (esc)
+  \titem osd.37 weight 1.000 (esc)
+  \titem osd.38 weight 1.000 (esc)
+  \titem osd.39 weight 1.000 (esc)
+  \titem osd.40 weight 1.000 (esc)
+  \titem osd.41 weight 1.000 (esc)
+  \titem osd.42 weight 1.000 (esc)
+  }
+  host cephstore5529 {
+  \tid -30\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.43 weight 1.000 (esc)
+  \titem osd.44 weight 1.000 (esc)
+  \titem osd.45 weight 1.000 (esc)
+  \titem osd.46 weight 1.000 (esc)
+  \titem osd.47 weight 1.000 (esc)
+  \titem osd.48 weight 1.000 (esc)
+  \titem osd.49 weight 1.000 (esc)
+  }
+  host cephstore5530 {
+  \tid -31\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.50 weight 1.000 (esc)
+  \titem osd.51 weight 1.000 (esc)
+  \titem osd.52 weight 1.000 (esc)
+  \titem osd.53 weight 1.000 (esc)
+  \titem osd.54 weight 1.000 (esc)
+  \titem osd.55 weight 1.000 (esc)
+  \titem osd.56 weight 1.000 (esc)
+  }
+  rack irv-n2 {
+  \tid -3\t\t# do not change unnecessarily (esc)
+  \t# weight 119.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem cephstore5522 weight 7.000 (esc)
+  \titem cephstore5523 weight 7.000 (esc)
+  \titem cephstore6238 weight 7.000 (esc)
+  \titem cephstore6240 weight 7.000 (esc)
+  \titem cephstore6242 weight 7.000 (esc)
+  \titem cephstore5524 weight 7.000 (esc)
+  \titem cephstore6244 weight 7.000 (esc)
+  \titem cephstore6246 weight 7.000 (esc)
+  \titem cephstore6337 weight 7.000 (esc)
+  \titem cephstore6341 weight 7.000 (esc)
+  \titem cephstore6342 weight 7.000 (esc)
+  \titem cephstore5525 weight 7.000 (esc)
+  \titem cephstore6345 weight 7.000 (esc)
+  \titem cephstore5526 weight 7.000 (esc)
+  \titem cephstore5527 weight 7.000 (esc)
+  \titem cephstore5529 weight 7.000 (esc)
+  \titem cephstore5530 weight 7.000 (esc)
+  }
+  host cephstore6236 {
+  \tid -5\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.100 weight 1.000 (esc)
+  \titem osd.101 weight 1.000 (esc)
+  \titem osd.102 weight 1.000 (esc)
+  \titem osd.103 weight 1.000 (esc)
+  \titem osd.104 weight 1.000 (esc)
+  \titem osd.105 weight 1.000 (esc)
+  \titem osd.99 weight 1.000 (esc)
+  }
+  host cephstore6237 {
+  \tid -7\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.106 weight 1.000 (esc)
+  \titem osd.107 weight 1.000 (esc)
+  \titem osd.108 weight 1.000 (esc)
+  \titem osd.109 weight 1.000 (esc)
+  \titem osd.110 weight 1.000 (esc)
+  \titem osd.111 weight 1.000 (esc)
+  \titem osd.112 weight 1.000 (esc)
+  }
+  host cephstore6239 {
+  \tid -9\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.120 weight 1.000 (esc)
+  \titem osd.121 weight 1.000 (esc)
+  \titem osd.122 weight 1.000 (esc)
+  \titem osd.123 weight 1.000 (esc)
+  \titem osd.124 weight 1.000 (esc)
+  \titem osd.125 weight 1.000 (esc)
+  \titem osd.126 weight 1.000 (esc)
+  }
+  host cephstore6241 {
+  \tid -11\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.134 weight 1.000 (esc)
+  \titem osd.135 weight 1.000 (esc)
+  \titem osd.136 weight 1.000 (esc)
+  \titem osd.137 weight 1.000 (esc)
+  \titem osd.138 weight 1.000 (esc)
+  \titem osd.139 weight 1.000 (esc)
+  \titem osd.140 weight 1.000 (esc)
+  }
+  host cephstore6243 {
+  \tid -13\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.148 weight 1.000 (esc)
+  \titem osd.149 weight 1.000 (esc)
+  \titem osd.150 weight 1.000 (esc)
+  \titem osd.151 weight 1.000 (esc)
+  \titem osd.152 weight 1.000 (esc)
+  \titem osd.153 weight 1.000 (esc)
+  \titem osd.154 weight 1.000 (esc)
+  }
+  host cephstore6245 {
+  \tid -16\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.162 weight 1.000 (esc)
+  \titem osd.163 weight 1.000 (esc)
+  \titem osd.164 weight 1.000 (esc)
+  \titem osd.165 weight 1.000 (esc)
+  \titem osd.166 weight 1.000 (esc)
+  \titem osd.167 weight 1.000 (esc)
+  \titem osd.168 weight 1.000 (esc)
+  }
+  host cephstore6336 {
+  \tid -18\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.176 weight 1.000 (esc)
+  \titem osd.177 weight 1.000 (esc)
+  \titem osd.178 weight 1.000 (esc)
+  \titem osd.179 weight 1.000 (esc)
+  \titem osd.180 weight 1.000 (esc)
+  \titem osd.181 weight 1.000 (esc)
+  \titem osd.182 weight 1.000 (esc)
+  }
+  host cephstore6338 {
+  \tid -20\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.190 weight 1.000 (esc)
+  \titem osd.191 weight 1.000 (esc)
+  \titem osd.192 weight 1.000 (esc)
+  \titem osd.193 weight 1.000 (esc)
+  \titem osd.194 weight 1.000 (esc)
+  \titem osd.195 weight 1.000 (esc)
+  \titem osd.196 weight 1.000 (esc)
+  }
+  host cephstore6339 {
+  \tid -21\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.197 weight 1.000 (esc)
+  \titem osd.198 weight 1.000 (esc)
+  \titem osd.199 weight 1.000 (esc)
+  \titem osd.200 weight 1.000 (esc)
+  \titem osd.201 weight 1.000 (esc)
+  \titem osd.202 weight 1.000 (esc)
+  \titem osd.203 weight 1.000 (esc)
+  }
+  host cephstore6340 {
+  \tid -22\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.204 weight 1.000 (esc)
+  \titem osd.205 weight 1.000 (esc)
+  \titem osd.206 weight 1.000 (esc)
+  \titem osd.207 weight 1.000 (esc)
+  \titem osd.208 weight 1.000 (esc)
+  \titem osd.209 weight 1.000 (esc)
+  \titem osd.210 weight 1.000 (esc)
+  }
+  host cephstore6343 {
+  \tid -26\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.225 weight 1.000 (esc)
+  \titem osd.226 weight 1.000 (esc)
+  \titem osd.227 weight 1.000 (esc)
+  \titem osd.228 weight 1.000 (esc)
+  \titem osd.229 weight 1.000 (esc)
+  \titem osd.230 weight 1.000 (esc)
+  \titem osd.231 weight 1.000 (esc)
+  }
+  host cephstore6230 {
+  \tid -32\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.57 weight 1.000 (esc)
+  \titem osd.58 weight 1.000 (esc)
+  \titem osd.59 weight 1.000 (esc)
+  \titem osd.60 weight 1.000 (esc)
+  \titem osd.61 weight 1.000 (esc)
+  \titem osd.62 weight 1.000 (esc)
+  \titem osd.63 weight 1.000 (esc)
+  }
+  host cephstore6231 {
+  \tid -33\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.64 weight 1.000 (esc)
+  \titem osd.65 weight 1.000 (esc)
+  \titem osd.66 weight 1.000 (esc)
+  \titem osd.67 weight 1.000 (esc)
+  \titem osd.68 weight 1.000 (esc)
+  \titem osd.69 weight 1.000 (esc)
+  \titem osd.70 weight 1.000 (esc)
+  }
+  host cephstore6232 {
+  \tid -34\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.71 weight 1.000 (esc)
+  \titem osd.72 weight 1.000 (esc)
+  \titem osd.73 weight 1.000 (esc)
+  \titem osd.74 weight 1.000 (esc)
+  \titem osd.75 weight 1.000 (esc)
+  \titem osd.76 weight 1.000 (esc)
+  \titem osd.77 weight 1.000 (esc)
+  }
+  host cephstore6233 {
+  \tid -35\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.78 weight 1.000 (esc)
+  \titem osd.79 weight 1.000 (esc)
+  \titem osd.80 weight 1.000 (esc)
+  \titem osd.81 weight 1.000 (esc)
+  \titem osd.82 weight 1.000 (esc)
+  \titem osd.83 weight 1.000 (esc)
+  \titem osd.84 weight 1.000 (esc)
+  }
+  host cephstore6234 {
+  \tid -36\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.85 weight 1.000 (esc)
+  \titem osd.86 weight 1.000 (esc)
+  \titem osd.87 weight 1.000 (esc)
+  \titem osd.88 weight 1.000 (esc)
+  \titem osd.89 weight 1.000 (esc)
+  \titem osd.90 weight 1.000 (esc)
+  \titem osd.91 weight 1.000 (esc)
+  }
+  host cephstore6235 {
+  \tid -37\t\t# do not change unnecessarily (esc)
+  \t# weight 7.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem osd.92 weight 1.000 (esc)
+  \titem osd.93 weight 1.000 (esc)
+  \titem osd.94 weight 1.000 (esc)
+  \titem osd.95 weight 1.000 (esc)
+  \titem osd.96 weight 1.000 (esc)
+  \titem osd.97 weight 1.000 (esc)
+  \titem osd.98 weight 1.000 (esc)
+  }
+  rack irv-n1 {
+  \tid -6\t\t# do not change unnecessarily (esc)
+  \t# weight 119.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem cephstore6236 weight 7.000 (esc)
+  \titem cephstore6237 weight 7.000 (esc)
+  \titem cephstore6239 weight 7.000 (esc)
+  \titem cephstore6241 weight 7.000 (esc)
+  \titem cephstore6243 weight 7.000 (esc)
+  \titem cephstore6245 weight 7.000 (esc)
+  \titem cephstore6336 weight 7.000 (esc)
+  \titem cephstore6338 weight 7.000 (esc)
+  \titem cephstore6339 weight 7.000 (esc)
+  \titem cephstore6340 weight 7.000 (esc)
+  \titem cephstore6343 weight 7.000 (esc)
+  \titem cephstore6230 weight 7.000 (esc)
+  \titem cephstore6231 weight 7.000 (esc)
+  \titem cephstore6232 weight 7.000 (esc)
+  \titem cephstore6233 weight 7.000 (esc)
+  \titem cephstore6234 weight 7.000 (esc)
+  \titem cephstore6235 weight 7.000 (esc)
+  }
+  root default {
+  \tid -1\t\t# do not change unnecessarily (esc)
+  \t# weight 238.000 (esc)
+  \talg straw (esc)
+  \thash 0\t# rjenkins1 (esc)
+  \titem irv-n2 weight 119.000 (esc)
+  \titem irv-n1 weight 119.000 (esc)
+  }
+  
+  # rules
+  rule replicated_ruleset {
+  \truleset 0 (esc)
+  \ttype replicated (esc)
+  \tmin_size 1 (esc)
+  \tmax_size 10 (esc)
+  \tstep take default (esc)
+  \tstep chooseleaf firstn 0 type host (esc)
+  \tstep emit (esc)
+  }
+  
+  # end crush map
+  $ rm oc
+  $ osdmaptool --test-map-pg 0.0 om
+  osdmaptool: osdmap file 'om'
+   parsed '0.0' -> 0.0
+  0.0 raw ([], p-1) up ([], p-1) acting ([], p-1)
+  $ osdmaptool --print om
+  osdmaptool: osdmap file 'om'
+  epoch 1
+  fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
+  created \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  modified \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ (re)
+  flags 
+  
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  pool 1 'metadata' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 flags hashpspool stripe_width 0
+  pool 2 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 flags hashpspool stripe_width 0
+  
+  max_osd 239
+  
+
+  $ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_replicated_ruleset 55 om -c $TESTDIR/ceph.conf.withracks  
+  osdmaptool: osdmap file 'om'
+  osdmaptool: writing epoch 1 to om
+  $ osdmaptool --print om | grep 'pool 0'
+  osdmaptool: osdmap file 'om'
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  $ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_rule 55 om -c $TESTDIR/ceph.conf.withracks 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+  osdmaptool: osdmap file 'om'
+  default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+  default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 0
+  $ osdmaptool --print om | grep 'pool 0'
+  osdmaptool: osdmap file 'om'
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  $ osdmaptool --clobber --create-from-conf --osd_pool_default_crush_replicated_ruleset 66 --osd_pool_default_crush_rule 55 om -c $TESTDIR/ceph.conf.withracks 2>&1 >/dev/null | sed -e 's/^.* 0 osd_pool_//'
+  osdmaptool: osdmap file 'om'
+  default_crush_rule is deprecated use osd_pool_default_crush_replicated_ruleset instead
+  default_crush_rule = 55 overrides osd_pool_default_crush_replicated_ruleset = 66
+  $ osdmaptool --print om | grep 'pool 0'
+  osdmaptool: osdmap file 'om'
+  pool 0 'data' replicated size 3 min_size 2 crush_ruleset 55 object_hash rjenkins pg_num 15296 pgp_num 15296 last_change 0 flags hashpspool crash_replay_interval 45 stripe_width 0
+  $ rm -f om
diff --git a/ceph/src/test/cli/osdmaptool/crush.t b/ceph/src/test/cli/osdmaptool/crush.t
new file mode 100644
index 00000000..5833da8a
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/crush.t
@@ -0,0 +1,10 @@
+  $ osdmaptool --createsimple 3 myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: writing epoch 1 to myosdmap
+  $ osdmaptool --export-crush oc myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: exported crush map to oc
+  $ osdmaptool --import-crush oc myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: imported 486 byte crush map from oc
+  osdmaptool: writing epoch 3 to myosdmap
diff --git a/ceph/src/test/cli/osdmaptool/help.t b/ceph/src/test/cli/osdmaptool/help.t
new file mode 100644
index 00000000..2c5a41de
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/help.t
@@ -0,0 +1,12 @@
+# TODO be user-friendly
+  $ osdmaptool --help
+   usage: [--print] [--createsimple  [--clobber] [--pg_bits ]] 
+     --export-crush    write osdmap's crush map to 
+     --import-crush    replace osdmap's crush map with 
+     --test-map-pgs [--pool ] map all pgs
+     --mark-up-in            mark osds up and in (but do not persist)
+     --clear-temp            clear pg_temp and primary_temp
+     --test-random           do random placements
+     --test-map-pg     map a pgid to osds
+     --test-map-object  [--pool ] map an object to osds
+  [1]
diff --git a/ceph/src/test/cli/osdmaptool/missing-argument.t b/ceph/src/test/cli/osdmaptool/missing-argument.t
new file mode 100644
index 00000000..d0740ab1
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/missing-argument.t
@@ -0,0 +1,12 @@
+  $ osdmaptool
+  osdmaptool: must specify osdmap filename
+   usage: [--print] [--createsimple  [--clobber] [--pg_bits ]] 
+     --export-crush    write osdmap's crush map to 
+     --import-crush    replace osdmap's crush map with 
+     --test-map-pgs [--pool ] map all pgs
+     --mark-up-in            mark osds up and in (but do not persist)
+     --clear-temp            clear pg_temp and primary_temp
+     --test-random           do random placements
+     --test-map-pg     map a pgid to osds
+     --test-map-object  [--pool ] map an object to osds
+  [1]
diff --git a/ceph/src/test/cli/osdmaptool/pool.t b/ceph/src/test/cli/osdmaptool/pool.t
new file mode 100644
index 00000000..0adb2408
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/pool.t
@@ -0,0 +1,54 @@
+  $ osdmaptool --createsimple 3 myosdmap
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: writing epoch 1 to myosdmap
+
+#
+# --test-map-object / --pool
+#
+  $ osdmaptool myosdmap --test-map-object foo --pool
+  Option --pool requires an argument.
+  [1]
+
+  $ osdmaptool myosdmap --test-map-object foo --pool bar
+  strict_strtoll: expected integer, got: 'bar'
+  [1]
+
+  $ osdmaptool myosdmap --test-map-object foo --pool 123
+  osdmaptool: osdmap file 'myosdmap'
+  There is no pool 123
+  [1]
+
+  $ osdmaptool myosdmap --test-map-object foo --pool 2
+  osdmaptool: osdmap file 'myosdmap'
+   object 'foo' \-\> 2\..* (re)
+
+  $ osdmaptool myosdmap --test-map-object foo
+  osdmaptool: osdmap file 'myosdmap'
+  osdmaptool: assuming pool 0 (use --pool to override)
+   object 'foo' \-\> 0\..* (re)
+
+#
+# --test-map-pgs / --pool
+#
+  $ osdmaptool myosdmap --test-map-pgs --pool
+  Option --pool requires an argument.
+  [1]
+
+  $ osdmaptool myosdmap --test-map-pgs --pool baz
+  strict_strtoll: expected integer, got: 'baz'
+  [1]
+
+  $ osdmaptool myosdmap --test-map-pgs --pool 123
+  osdmaptool: osdmap file 'myosdmap'
+  There is no pool 123
+  [1]
+
+  $ osdmaptool myosdmap --mark-up-in --test-map-pgs --pool 2 | grep pool
+  osdmaptool: osdmap file 'myosdmap'
+  pool 2 pg_num .* (re)
+
+  $ osdmaptool myosdmap --mark-up-in --test-map-pgs | grep pool
+  osdmaptool: osdmap file 'myosdmap'
+  pool 0 pg_num .* (re)
+  pool 1 pg_num .* (re)
+  pool 2 pg_num .* (re)
diff --git a/ceph/src/test/cli/osdmaptool/print-empty.t b/ceph/src/test/cli/osdmaptool/print-empty.t
new file mode 100644
index 00000000..a629f771
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/print-empty.t
@@ -0,0 +1,5 @@
+  $ touch empty
+  $ osdmaptool --print empty
+  osdmaptool: osdmap file 'empty'
+  osdmaptool: error decoding osdmap 'empty'
+  [255]
diff --git a/ceph/src/test/cli/osdmaptool/print-nonexistent.t b/ceph/src/test/cli/osdmaptool/print-nonexistent.t
new file mode 100644
index 00000000..88f7e618
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/print-nonexistent.t
@@ -0,0 +1,4 @@
+  $ osdmaptool --print nonexistent
+  osdmaptool: osdmap file 'nonexistent'
+  osdmaptool: couldn't open nonexistent: can't open nonexistent: (2) No such file or directory
+  [255]
diff --git a/ceph/src/test/cli/osdmaptool/test-map-pgs.t b/ceph/src/test/cli/osdmaptool/test-map-pgs.t
new file mode 100644
index 00000000..b64f2d99
--- /dev/null
+++ b/ceph/src/test/cli/osdmaptool/test-map-pgs.t
@@ -0,0 +1,52 @@
+  $ NUM_OSDS=500
+  $ POOL_COUNT=3 # data + metadata + rbd
+  $ SIZE=3
+  $ PG_BITS=4
+#
+# create an osdmap with a few hundred devices and a realistic crushmap
+#
+  $ OSD_MAP="osdmap"
+  $ osdmaptool --osd_pool_default_size $SIZE --pg_bits $PG_BITS --createsimple $NUM_OSDS "$OSD_MAP" > /dev/null
+  osdmaptool: osdmap file 'osdmap'
+  $ CRUSH_MAP="crushmap"
+  $ CEPH_ARGS="--debug-crush 0" crushtool --outfn "$CRUSH_MAP" --build --num_osds $NUM_OSDS node straw 10 rack straw 10 root straw 0
+  $ osdmaptool --import-crush "$CRUSH_MAP" "$OSD_MAP" > /dev/null
+  osdmaptool: osdmap file 'osdmap'
+  $ OUT="$TESTDIR/out"
+# 
+# --test-map-pgs
+#
+  $ osdmaptool --mark-up-in --test-map-pgs "$OSD_MAP" > "$OUT"
+  osdmaptool: osdmap file 'osdmap'
+  $ PG_NUM=$(($NUM_OSDS << $PG_BITS))
+  $ grep "pg_num $PG_NUM" "$OUT" || cat $OUT
+  pool 0 pg_num 8000
+  pool 1 pg_num 8000
+  pool 2 pg_num 8000
+  $ TOTAL=$((POOL_COUNT * $PG_NUM))
+  $ PATTERN=$(echo "size $SIZE\t$TOTAL")
+  $ grep "$PATTERN" $OUT || cat "$OUT"
+  size 3\t24000 (esc)
+  $ STATS_CRUSH=$(grep '^ avg ' "$OUT")
+# 
+# --test-map-pgs --test-random is expected to change nothing regarding the totals
+#
+  $ osdmaptool --mark-up-in --test-random --test-map-pgs "$OSD_MAP" > "$OUT"
+  osdmaptool: osdmap file 'osdmap'
+  $ PG_NUM=$(($NUM_OSDS << $PG_BITS))
+  $ grep "pg_num $PG_NUM" "$OUT" || cat $OUT
+  pool 0 pg_num 8000
+  pool 1 pg_num 8000
+  pool 2 pg_num 8000
+  $ TOTAL=$((POOL_COUNT * $PG_NUM))
+  $ PATTERN=$(echo "size $SIZE\t$TOTAL")
+  $ grep "$PATTERN" $OUT || cat "$OUT"
+  size 3\t24000 (esc)
+  $ STATS_RANDOM=$(grep '^ avg ' "$OUT")
+# it is almost impossible to get the same stats with random and crush
+# if they are, it most probably means something went wrong somewhere
+  $ test "$STATS_CRUSH" != "$STATS_RANDOM"
+#
+# cleanup
+#
+  $ rm -f "$CRUSH_MAP" "$OSD_MAP" "$OUT"
diff --git a/ceph/src/test/cli/radosgw-admin/help.t b/ceph/src/test/cli/radosgw-admin/help.t
new file mode 100644
index 00000000..a46d240d
--- /dev/null
+++ b/ceph/src/test/cli/radosgw-admin/help.t
@@ -0,0 +1,139 @@
+  $ radosgw-admin --help
+  usage: radosgw-admin  [options...]
+  commands:
+    user create                create a new user
+    user modify                modify user
+    user info                  get user info
+    user rm                    remove user
+    user suspend               suspend a user
+    user enable                re-enable user after suspension
+    user check                 check user info
+    user stats                 show user stats as accounted by quota subsystem
+    caps add                   add user capabilities
+    caps rm                    remove user capabilities
+    subuser create             create a new subuser
+    subuser modify             modify subuser
+    subuser rm                 remove subuser
+    key create                 create access key
+    key rm                     remove access key
+    bucket list                list buckets
+    bucket link                link bucket to specified user
+    bucket unlink              unlink bucket from specified user
+    bucket stats               returns bucket statistics
+    bucket rm                  remove bucket
+    bucket check               check bucket index
+    object rm                  remove object
+    object unlink              unlink object from bucket index
+    quota set                  set quota params
+    quota enable               enable quota
+    quota disable              disable quota
+    region get                 show region info
+    regions list               list all regions set on this cluster
+    region set                 set region info (requires infile)
+    region default             set default region
+    region-map get             show region-map
+    region-map set             set region-map (requires infile)
+    zone get                   show zone cluster params
+    zone set                   set zone cluster params (requires infile)
+    zone list                  list all zones set on this cluster
+    pool add                   add an existing pool for data placement
+    pool rm                    remove an existing pool from data placement set
+    pools list                 list placement active set
+    policy                     read bucket/object policy
+    log list                   list log objects
+    log show                   dump a log from specific object or (bucket + date
+                               + bucket-id)
+    log rm                     remove log object
+    usage show                 show usage (by user, date range)
+    usage trim                 trim usage (by user, date range)
+    temp remove                remove temporary objects that were created up to
+                               specified date (and optional time)
+    gc list                    dump expired garbage collection objects (specify
+                               --include-all to list all entries, including unexpired)
+    gc process                 manually process garbage
+    metadata get               get metadata info
+    metadata put               put metadata info
+    metadata rm                remove metadata info
+    metadata list              list metadata info
+    mdlog list                 list metadata log
+    mdlog trim                 trim metadata log
+    bilog list                 list bucket index log
+    bilog trim                 trim bucket index log (use start-marker, end-marker)
+    datalog list               list data log
+    datalog trim               trim data log
+    opstate list               list stateful operations entries (use client_id,
+                               op_id, object)
+    opstate set                set state on an entry (use client_id, op_id, object, state)
+    opstate renew              renew state on an entry (use client_id, op_id, object)
+    opstate rm                 remove entry (use client_id, op_id, object)
+    replicalog get             get replica metadata log entry
+    replicalog delete          delete replica metadata log entry
+  options:
+     --uid=                user id
+     --subuser=          subuser name
+     --access-key=        S3 access key
+     --email=
+     --secret=            specify secret key
+     --gen-access-key          generate random access key (for S3)
+     --gen-secret              generate random secret key
+     --key-type=         key type, options are: swift, s3
+     --temp-url-key[-2]=  temp url key
+     --access=         Set access permissions for sub-user, should be one
+                               of read, write, readwrite, full
+     --display-name=
+     --system                  set the system flag on the user
+     --bucket=
+     --pool=
+     --object=
+     --date=
+     --start-date=
+     --end-date=
+     --bucket-id=
+     --shard-id=     optional for mdlog list
+                               required for: 
+                                 mdlog trim
+                                 replica mdlog get/delete
+                                 replica datalog get/delete
+     --metadata-key=      key to retrieve metadata from with metadata get
+     --rgw-region=     region in which radosgw is running
+     --rgw-zone=         zone in which radosgw is running
+     --fix                     besides checking bucket index, will also fix it
+     --check-objects           bucket check: rebuilds bucket index according to
+                               actual objects state
+     --format=         specify output format for certain operations: xml,
+                               json
+     --purge-data              when specified, user removal will also purge all the
+                               user data
+     --purge-keys              when specified, subuser removal will also purge all the
+                               subuser keys
+     --purge-objects           remove a bucket's objects before deleting it
+                               (NOTE: required to delete a non-empty bucket)
+     --sync-stats              option to 'user stats', update user stats with current
+                               stats reported by user's buckets indexes
+     --show-log-entries= enable/disable dump of log entries on log show
+     --show-log-sum=     enable/disable dump of log summation on log show
+     --skip-zero-entries       log show only dumps entries that don't have zero value
+                               in one of the numeric field
+     --infile                  specify a file to read in when setting data
+     --state=    specify a state for the opstate set command
+     --replica-log-type        replica log type (metadata, data, bucket), required for
+                               replica log operations
+     --categories=       comma separated list of categories, used in usage show
+     --caps=             list of caps (e.g., "usage=read, write; user=read"
+     --yes-i-really-mean-it    required for certain operations
+  
+   := "YYYY-MM-DD[ hh:mm:ss]"
+  
+  Quota options:
+     --bucket                  specified bucket for quota command
+     --max-objects             specify max objects (negative value to disable)
+     --max-size                specify max size (in bytes, negative value to disable)
+     --quota-scope             scope of quota (bucket, user)
+  
+    --conf/-c FILE    read configuration from the given configuration file
+    --id/-i ID        set ID portion of my name
+    --name/-n TYPE.ID set name
+    --cluster NAME    set cluster name (default: ceph)
+    --version         show version and quit
+  
+  [1]
diff --git a/ceph/src/test/cli/rbd/help.t b/ceph/src/test/cli/rbd/help.t
new file mode 100644
index 00000000..5723bef8
--- /dev/null
+++ b/ceph/src/test/cli/rbd/help.t
@@ -0,0 +1,81 @@
+  $ rbd --help
+  usage: rbd [-n ] [OPTIONS]  ...
+  where 'pool' is a rados pool name (default is 'rbd') and 'cmd' is one of:
+    (ls | list) [-l | --long ] [pool-name] list rbd images
+                                                (-l includes snapshots/clones)
+    info                            show information about image size,
+                                                striping, etc.
+    create [--order ] --size    create an empty image
+    clone [--order ]  
+                                                clone a snapshot into a COW
+                                                child image
+    children                         display children of snapshot
+    flatten                         fill clone with parent data
+                                                (make it independent)
+    resize --size               resize (expand or contract) image
+    rm                              delete an image
+    export                    export image to file
+                                                "-" for stdout
+    import                    import image from file
+                                                (dest defaults
+                                                 as the filename part of file)
+                                                "-" for stdin
+    diff  [--from-snap ] print extents that differ since
+                                                a previous snap, or image creation
+    export-diff  [--from-snap ] 
+                                                export an incremental diff to
+                                                path, or "-" for stdout
+    import-diff               import an incremental diff from
+                                                path or "-" for stdin
+    (cp | copy)                      copy src image to dest
+    (mv | rename)                    rename src image to dest
+    snap ls                         dump list of image snapshots
+    snap create                      create a snapshot
+    snap rollback                    rollback image to snapshot
+    snap rm                          deletes a snapshot
+    snap purge                      deletes all snapshots
+    snap protect                     prevent a snapshot from being deleted
+    snap unprotect                   allow a snapshot to be deleted
+    watch                           watch events on image
+    map                             map image to a block device
+                                                using the kernel
+    unmap                               unmap a rbd device that was
+                                                mapped by the kernel
+    showmapped                                  show the rbd images mapped
+                                                by the kernel
+    lock list                       show locks held on an image
+    lock add   [--shared ] take a lock called id on an image
+    lock remove         release a lock on an image
+    bench-write                     simple write benchmark
+                   --io-size               write size
+                   --io-threads              ios in flight
+                   --io-total              total bytes to write
+                   --io-pattern         write pattern
+  
+  ,  are [pool/]name[@snap], or you may specify
+  individual pieces of names with -p/--pool, --image, and/or --snap.
+  
+  Other input options:
+    -p, --pool                   source pool name
+    --image                image name
+    --dest                 destination [pool and] image name
+    --snap                  snapshot name
+    --dest-pool                  destination pool name
+    --path                  path name for import/export
+    --size                 size of image for create and resize
+    --order                      the object size in bits; object size will be
+                                       (1 << order) bytes. Default is 22 (4 MB).
+    --image-format      format to use when creating an image
+                                       format 1 is the original format (default)
+                                       format 2 supports cloning
+    --id                     rados user (without 'client.'prefix) to
+                                       authenticate as
+    --keyfile                    file containing secret key for use with cephx
+    --shared                      take a shared (rather than exclusive) lock
+    --format            output format (default: plain, json, xml)
+    --pretty-format                    make json or xml output more readable
+    --no-settle                        do not wait for udevadm to settle on map/unmap
+    --no-progress                      do not show progress for long-running commands
+    -o, --options         options to use when mapping an image
+    --read-only                        set device readonly when mapping image
+    --allow-shrink                     allow shrinking of an image when resizing
diff --git a/ceph/src/test/cli/rbd/invalid-snap-usage.t b/ceph/src/test/cli/rbd/invalid-snap-usage.t
new file mode 100644
index 00000000..40fc606b
--- /dev/null
+++ b/ceph/src/test/cli/rbd/invalid-snap-usage.t
@@ -0,0 +1,36 @@
+  $ rbd resize --snap=snap1 img
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd resize img@snap
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd import --snap=snap1 /bin/ls ls
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd create --snap=snap img
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd rm --snap=snap img
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd rename --snap=snap img
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd ls --snap=snap rbd
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd snap ls --snap=snap img
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd watch --snap=snap img
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd lock list --snap=snap img
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd lock add --snap=snap img id
+  rbd: snapname specified for a command that doesn't use it
+  [1]
+  $ rbd lock remove --snap=snap img id client.1234
+  rbd: snapname specified for a command that doesn't use it
+  [1]
diff --git a/ceph/src/test/cli/rbd/not-enough-args.t b/ceph/src/test/cli/rbd/not-enough-args.t
new file mode 100644
index 00000000..40fe0d12
--- /dev/null
+++ b/ceph/src/test/cli/rbd/not-enough-args.t
@@ -0,0 +1,33 @@
+  $ rbd map
+  rbd: image name was not specified
+  [1]
+  $ rbd unmap
+  rbd: device path was not specified
+  [1]
+  $ rbd clone foo@snap bar@snap
+  rbd: cannot clone to a snapshot
+  [1]
+  $ rbd cp foo
+  rbd: destination image name was not specified
+  [1]
+  $ rbd cp foo@bar
+  rbd: destination image name was not specified
+  [1]
+  $ rbd copy foo
+  rbd: destination image name was not specified
+  [1]
+  $ rbd copy foo@bar
+  rbd: destination image name was not specified
+  [1]
+  $ rbd mv foo
+  rbd: destination image name was not specified
+  [1]
+  $ rbd rename foo
+  rbd: destination image name was not specified
+  [1]
+  $ rbd clone foo@bar
+  rbd: destination image name was not specified
+  [1]
+  $ rbd clone foo
+  rbd: snap name was not specified
+  [1]
diff --git a/ceph/src/test/cls_hello/test_cls_hello.cc b/ceph/src/test/cls_hello/test_cls_hello.cc
new file mode 100644
index 00000000..58ecb97f
--- /dev/null
+++ b/ceph/src/test/cls_hello/test_cls_hello.cc
@@ -0,0 +1,133 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+#include "gtest/gtest.h"
+
+using namespace librados;
+
+TEST(ClsHello, SayHello) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+  ASSERT_EQ(-ENOENT, ioctx.exec("myobject", "hello", "say_hello", in, out));
+  ASSERT_EQ(0, ioctx.write_full("myobject", in));
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "say_hello", in, out));
+  ASSERT_EQ(std::string("Hello, world!"), std::string(out.c_str(), out.length()));
+
+  out.clear();
+  in.append("Tester");
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "say_hello", in, out));
+  ASSERT_EQ(std::string("Hello, Tester!"), std::string(out.c_str(), out.length()));
+
+  out.clear();
+  in.clear();
+  char buf[4096];
+  memset(buf, 1, sizeof(buf));
+  in.append(buf, sizeof(buf));
+  ASSERT_EQ(-EINVAL, ioctx.exec("myobject", "hello", "say_hello", in, out));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsHello, RecordHello) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "record_hello", in, out));
+  ASSERT_EQ(-EEXIST, ioctx.exec("myobject", "hello", "record_hello", in, out));
+
+  in.append("Tester");
+  ASSERT_EQ(0, ioctx.exec("myobject2", "hello", "record_hello", in, out));
+  ASSERT_EQ(-EEXIST, ioctx.exec("myobject2", "hello", "record_hello", in, out));
+  ASSERT_EQ(0u, out.length());
+
+  in.clear();
+  out.clear();
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "replay", in, out));
+  ASSERT_EQ(std::string("Hello, world!"), std::string(out.c_str(), out.length()));
+  out.clear();
+  ASSERT_EQ(0, ioctx.exec("myobject2", "hello", "replay", in, out));
+  ASSERT_EQ(std::string("Hello, Tester!"), std::string(out.c_str(), out.length()));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsHello, WriteReturnData) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "writes_dont_return_data", in, out));
+  ASSERT_EQ(std::string(), std::string(out.c_str(), out.length()));
+
+  char buf[4096];
+  memset(buf, 1, sizeof(buf));
+  in.append(buf, sizeof(buf));
+  ASSERT_EQ(-EINVAL, ioctx.exec("myobject2", "hello", "writes_dont_return_data", in, out));
+  ASSERT_EQ(std::string("too much input data!"), std::string(out.c_str(), out.length()));
+  ASSERT_EQ(-ENOENT, ioctx.getxattr("myobject2", "foo", out));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsHello, Loud) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "record_hello", in, out));
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "replay", in, out));
+  ASSERT_EQ(std::string("Hello, world!"), std::string(out.c_str(), out.length()));
+
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "turn_it_to_11", in, out));
+  ASSERT_EQ(0, ioctx.exec("myobject", "hello", "replay", in, out));
+  ASSERT_EQ(std::string("HELLO, WORLD!"), std::string(out.c_str(), out.length()));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsHello, BadMethods) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+
+  ASSERT_EQ(0, ioctx.write_full("myobject", in));
+  ASSERT_EQ(-EIO, ioctx.exec("myobject", "hello", "bad_reader", in, out));
+  ASSERT_EQ(-EIO, ioctx.exec("myobject", "hello", "bad_writer", in, out));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
diff --git a/ceph/src/test/cls_lock/test_cls_lock.cc b/ceph/src/test/cls_lock/test_cls_lock.cc
new file mode 100644
index 00000000..ead03bb2
--- /dev/null
+++ b/ceph/src/test/cls_lock/test_cls_lock.cc
@@ -0,0 +1,300 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+
+#include "include/types.h"
+#include "common/Clock.h"
+#include "msg/msg_types.h"
+#include "include/rados/librados.hpp"
+
+#include "test/librados/test.h"
+#include "gtest/gtest.h"
+
+using namespace librados;
+
+#include "cls/lock/cls_lock_client.h"
+#include "cls/lock/cls_lock_ops.h"
+
+using namespace rados::cls::lock;
+
+void lock_info(IoCtx *ioctx, string& oid, string& name, map& lockers,
+	       ClsLockType *assert_type, string *assert_tag)
+{
+  ClsLockType lock_type = LOCK_NONE;
+  string tag;
+  lockers.clear();
+  ASSERT_EQ(0, get_lock_info(ioctx, oid, name, &lockers, &lock_type, &tag));
+  cout << "lock: " << name << std::endl;
+  cout << "  lock_type: " << cls_lock_type_str(lock_type) << std::endl;
+  cout << "  tag: " << tag << std::endl;
+  cout << "  lockers:" << std::endl;
+
+  if (assert_type)
+    ASSERT_EQ(*assert_type, lock_type);
+
+  if (assert_tag)
+    ASSERT_EQ(*assert_tag, tag);
+
+  map::iterator liter;
+  for (liter = lockers.begin(); liter != lockers.end(); ++liter) {
+    const locker_id_t& locker = liter->first;
+    cout << "    " << locker.locker << " expiration=" << liter->second.expiration
+         << " addr=" << liter->second.addr << " cookie=" << locker.cookie << std::endl;
+  }
+}
+
+void lock_info(IoCtx *ioctx, string& oid, string& name, map& lockers)
+{
+  lock_info(ioctx, oid, name, lockers, NULL, NULL);
+}
+
+TEST(ClsLock, TestMultiLocking) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+  ClsLockType lock_type_shared = LOCK_SHARED;
+  ClsLockType lock_type_exclusive = LOCK_EXCLUSIVE;
+
+
+  Rados cluster2;
+  IoCtx ioctx2;
+  ASSERT_EQ("", connect_cluster_pp(cluster2));
+  cluster2.ioctx_create(pool_name.c_str(), ioctx2);
+
+  string oid = "foo";
+  bufferlist bl;
+  string lock_name = "mylock";
+
+  ASSERT_EQ(0, ioctx.write(oid, bl, bl.length(), 0));
+
+  Lock l(lock_name);
+
+  /* test lock object */
+
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+
+  /* test exclusive lock */
+  ASSERT_EQ(-EEXIST, l.lock_exclusive(&ioctx, oid));
+
+  /* test idempotency */
+  l.set_renew(true);
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+
+  l.set_renew(false);
+
+  /* test second client */
+  Lock l2(lock_name);
+  ASSERT_EQ(-EBUSY, l2.lock_exclusive(&ioctx2, oid));
+  ASSERT_EQ(-EBUSY, l2.lock_shared(&ioctx2, oid));
+
+  list locks;
+  ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
+
+  ASSERT_EQ(1, (int)locks.size());
+  list::iterator iter = locks.begin();
+  map lockers;
+  lock_info(&ioctx, oid, *iter, lockers, &lock_type_exclusive, NULL);
+
+  ASSERT_EQ(1, (int)lockers.size());
+
+  /* test unlock */
+  ASSERT_EQ(0, l.unlock(&ioctx, oid));
+  locks.clear();
+  ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
+
+  /* test shared lock */
+  ASSERT_EQ(0, l2.lock_shared(&ioctx2, oid));
+  ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
+
+  locks.clear();
+  ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
+  ASSERT_EQ(1, (int)locks.size());
+  iter = locks.begin();
+  lock_info(&ioctx, oid, *iter, lockers, &lock_type_shared, NULL);
+  ASSERT_EQ(2, (int)lockers.size());
+
+  /* test break locks */
+  entity_name_t name = entity_name_t::CLIENT(cluster.get_instance_id());
+  entity_name_t name2 = entity_name_t::CLIENT(cluster2.get_instance_id());
+
+  l2.break_lock(&ioctx2, oid, name);
+  lock_info(&ioctx, oid, *iter, lockers);
+  ASSERT_EQ(1, (int)lockers.size());
+  map::iterator liter = lockers.begin();
+  const locker_id_t& id = liter->first;
+  ASSERT_EQ(name2, id.locker);
+
+  /* test lock tag */
+  Lock l_tag(lock_name);
+  l_tag.set_tag("non-default tag");
+  ASSERT_EQ(-EBUSY, l_tag.lock_shared(&ioctx, oid));
+
+
+  /* test modify description */
+  string description = "new description";
+  l.set_description(description);
+  ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsLock, TestMeta) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+
+  Rados cluster2;
+  IoCtx ioctx2;
+  ASSERT_EQ("", connect_cluster_pp(cluster2));
+  cluster2.ioctx_create(pool_name.c_str(), ioctx2);
+
+  string oid = "foo";
+  bufferlist bl;
+  string lock_name = "mylock";
+
+  ASSERT_EQ(0, ioctx.write(oid, bl, bl.length(), 0));
+
+  Lock l(lock_name);
+  ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
+
+  /* test lock tag */
+  Lock l_tag(lock_name);
+  l_tag.set_tag("non-default tag");
+  ASSERT_EQ(-EBUSY, l_tag.lock_shared(&ioctx2, oid));
+
+
+  ASSERT_EQ(0, l.unlock(&ioctx, oid));
+
+  /* test description */
+  Lock l2(lock_name);
+  string description = "new description";
+  l2.set_description(description);
+  ASSERT_EQ(0, l2.lock_shared(&ioctx2, oid));
+
+  map lockers;
+  lock_info(&ioctx, oid, lock_name, lockers, NULL, NULL);
+  ASSERT_EQ(1, (int)lockers.size());
+
+  map::iterator iter = lockers.begin();
+  locker_info_t locker = iter->second;
+  ASSERT_EQ("new description", locker.description);
+
+  ASSERT_EQ(0, l2.unlock(&ioctx2, oid));
+
+  /* check new tag */
+  string new_tag = "new_tag";
+  l.set_tag(new_tag);
+  l.set_renew(true);
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+  lock_info(&ioctx, oid, lock_name, lockers, NULL, &new_tag);
+  ASSERT_EQ(1, (int)lockers.size());
+  l.set_tag("");
+  ASSERT_EQ(-EBUSY, l.lock_exclusive(&ioctx, oid));
+  l.set_tag(new_tag);
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsLock, TestCookie) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  string oid = "foo";
+  string lock_name = "mylock";
+  Lock l(lock_name);
+
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+
+  /* new cookie */
+  string cookie = "new cookie";
+  l.set_cookie(cookie);
+  ASSERT_EQ(-EBUSY, l.lock_exclusive(&ioctx, oid));
+  ASSERT_EQ(-ENOENT, l.unlock(&ioctx, oid));
+  l.set_cookie("");
+  ASSERT_EQ(0, l.unlock(&ioctx, oid));
+
+  map lockers;
+  lock_info(&ioctx, oid, lock_name, lockers);
+  ASSERT_EQ(0, (int)lockers.size());
+
+  l.set_cookie(cookie);
+  ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
+  l.set_cookie("");
+  ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
+
+  lock_info(&ioctx, oid, lock_name, lockers);
+  ASSERT_EQ(2, (int)lockers.size());
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsLock, TestMultipleLocks) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  string oid = "foo";
+  Lock l("lock1");
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+
+  Lock l2("lock2");
+  ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid));
+
+  list locks;
+  ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
+
+  ASSERT_EQ(2, (int)locks.size());
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsLock, TestLockDuration) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  string oid = "foo";
+  Lock l("lock");
+  utime_t dur(5, 0);
+  l.set_duration(dur);
+  utime_t start = ceph_clock_now(NULL);
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+  int r = l.lock_exclusive(&ioctx, oid);
+  if (r == 0) {
+    // it's possible to get success if we were just really slow...
+    ASSERT_TRUE(ceph_clock_now(NULL) > start + dur);
+  } else {
+    ASSERT_EQ(-EEXIST, r);
+  }
+
+  sleep(dur.sec());
+  ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
diff --git a/ceph/src/test/cls_log/test_cls_log.cc b/ceph/src/test/cls_log/test_cls_log.cc
new file mode 100644
index 00000000..ce97025f
--- /dev/null
+++ b/ceph/src/test/cls_log/test_cls_log.cc
@@ -0,0 +1,334 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+#include "cls/log/cls_log_types.h"
+#include "cls/log/cls_log_client.h"
+
+#include "include/utime.h"
+#include "common/Clock.h"
+#include "global/global_context.h"
+
+#include "gtest/gtest.h"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+
+static librados::ObjectWriteOperation *new_op() {
+  return new librados::ObjectWriteOperation();
+}
+
+static librados::ObjectReadOperation *new_rop() {
+  return new librados::ObjectReadOperation();
+}
+
+static void reset_rop(librados::ObjectReadOperation **pop) {
+  delete *pop;
+  *pop = new_rop();
+}
+
+static int read_bl(bufferlist& bl, int *i)
+{
+  bufferlist::iterator iter = bl.begin();
+
+  try {
+    ::decode(*i, iter);
+  } catch (buffer::error& err) {
+    std::cout << "failed to decode buffer" << std::endl;
+    return -EIO;
+  }
+
+  return 0;
+}
+
+void add_log(librados::ObjectWriteOperation *op, utime_t& timestamp, string& section, string&name, int i)
+{
+  bufferlist bl;
+  ::encode(i, bl);
+
+  cls_log_add(*op, timestamp, section, name, bl);
+}
+
+
+string get_name(int i)
+{
+  string name_prefix = "data-source";
+
+  char buf[16];
+  snprintf(buf, sizeof(buf), "%d", i);
+  return name_prefix + buf;
+}
+
+void generate_log(librados::IoCtx& ioctx, string& oid, int max, utime_t& start_time, bool modify_time)
+{
+  string section = "global";
+
+  librados::ObjectWriteOperation *op = new_op();
+
+  int i;
+
+  for (i = 0; i < max; i++) {
+    uint32_t secs = start_time.sec();
+    if (modify_time)
+      secs += i;
+
+    utime_t ts(secs, start_time.nsec());
+    string name = get_name(i);
+
+    add_log(op, ts, section, name, i);
+  }
+
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  delete op;
+}
+
+utime_t get_time(utime_t& start_time, int i, bool modify_time)
+{
+  uint32_t secs = start_time.sec();
+  if (modify_time)
+    secs += i;
+  return utime_t(secs, start_time.nsec());
+}
+
+void check_entry(cls_log_entry& entry, utime_t& start_time, int i, bool modified_time)
+{
+  string section = "global";
+  string name = get_name(i);
+  utime_t ts = get_time(start_time, i, modified_time);
+
+  ASSERT_EQ(section, entry.section);
+  ASSERT_EQ(name, entry.name);
+  ASSERT_EQ(ts, entry.timestamp);
+}
+
+
+TEST(cls_rgw, test_log_add_same_time)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+  /* create object */
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  /* generate log */
+  utime_t start_time = ceph_clock_now(g_ceph_context);
+  generate_log(ioctx, oid, 10, start_time, false);
+
+  librados::ObjectReadOperation *rop = new_rop();
+
+  list entries;
+  bool truncated;
+
+  /* check list */
+
+  utime_t to_time = get_time(start_time, 1, true);
+
+  string marker;
+
+  cls_log_list(*rop, start_time, to_time, marker, 0, entries, &marker, &truncated);
+
+  bufferlist obl;
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+
+  ASSERT_EQ(10, (int)entries.size());
+  ASSERT_EQ(0, (int)truncated);
+
+  list::iterator iter;
+
+  /* need to sort returned entries, all were using the same time as key */
+  map check_ents;
+
+  for (iter = entries.begin(); iter != entries.end(); ++iter) {
+    cls_log_entry& entry = *iter;
+
+    int num;
+    ASSERT_EQ(0, read_bl(entry.data, &num));
+
+    check_ents[num] = entry;
+  }
+
+  ASSERT_EQ(10, (int)check_ents.size());
+
+  map::iterator ei;
+
+  /* verify entries are as expected */
+
+  int i;
+
+  for (i = 0, ei = check_ents.begin(); i < 10; i++, ++ei) {
+    cls_log_entry& entry = ei->second;
+
+    ASSERT_EQ(i, ei->first);
+    check_entry(entry, start_time, i, false);
+  }
+
+  reset_rop(&rop);
+
+  /* check list again, now want to be truncated*/
+
+  marker.clear();
+
+  cls_log_list(*rop, start_time, to_time, marker, 1, entries, &marker, &truncated);
+
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+
+  ASSERT_EQ(1, (int)entries.size());
+  ASSERT_EQ(1, (int)truncated);
+
+  delete rop;
+}
+
+TEST(cls_rgw, test_log_add_different_time)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+  /* create object */
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  /* generate log */
+  utime_t start_time = ceph_clock_now(g_ceph_context);
+  generate_log(ioctx, oid, 10, start_time, true);
+
+  librados::ObjectReadOperation *rop = new_rop();
+
+  list entries;
+  bool truncated;
+
+  utime_t to_time = utime_t(start_time.sec() + 10, start_time.nsec());
+
+  string marker;
+
+  /* check list */
+  cls_log_list(*rop, start_time, to_time, marker, 0, entries, &marker, &truncated);
+
+  bufferlist obl;
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+
+  ASSERT_EQ(10, (int)entries.size());
+  ASSERT_EQ(0, (int)truncated);
+
+  list::iterator iter;
+
+  /* returned entries should be sorted by time */
+  map check_ents;
+
+  int i;
+
+  for (i = 0, iter = entries.begin(); iter != entries.end(); ++iter, ++i) {
+    cls_log_entry& entry = *iter;
+
+    int num;
+
+    ASSERT_EQ(0, read_bl(entry.data, &num));
+
+    ASSERT_EQ(i, num);
+
+    check_entry(entry, start_time, i, true);
+  }
+
+  reset_rop(&rop);
+
+  /* check list again with shifted time */
+  utime_t next_time = get_time(start_time, 1, true);
+
+  marker.clear();
+
+  cls_log_list(*rop, next_time, to_time, marker, 0, entries, &marker, &truncated);
+
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+
+  ASSERT_EQ(9, (int)entries.size());
+  ASSERT_EQ(0, (int)truncated);
+
+  reset_rop(&rop);
+
+  marker.clear();
+
+  i = 0;
+  do {
+    bufferlist obl;
+    string old_marker = marker;
+    cls_log_list(*rop, start_time, to_time, old_marker, 1, entries, &marker, &truncated);
+
+    ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+    ASSERT_NE(old_marker, marker);
+    ASSERT_EQ(1, (int)entries.size());
+
+    ++i;
+    ASSERT_GE(10, i);
+  } while (truncated);
+
+  ASSERT_EQ(10, i);
+  delete rop;
+}
+
+TEST(cls_rgw, test_log_trim)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+  /* create object */
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  /* generate log */
+  utime_t start_time = ceph_clock_now(g_ceph_context);
+  generate_log(ioctx, oid, 10, start_time, true);
+
+  librados::ObjectReadOperation *rop = new_rop();
+
+  list entries;
+  bool truncated;
+
+  /* check list */
+
+  /* trim */
+  utime_t to_time = get_time(start_time, 10, true);
+
+  for (int i = 0; i < 10; i++) {
+    utime_t trim_time = get_time(start_time, i, true);
+
+    utime_t zero_time;
+    string start_marker, end_marker;
+
+    ASSERT_EQ(0, cls_log_trim(ioctx, oid, zero_time, trim_time, start_marker, end_marker));
+
+    string marker;
+
+    cls_log_list(*rop, start_time, to_time, marker, 0, entries, &marker, &truncated);
+
+    bufferlist obl;
+    ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+
+    ASSERT_EQ(9 - i, (int)entries.size());
+    ASSERT_EQ(0, (int)truncated);
+  }
+  delete rop;
+}
diff --git a/ceph/src/test/cls_rbd/test_cls_rbd.cc b/ceph/src/test/cls_rbd/test_cls_rbd.cc
new file mode 100644
index 00000000..48cfb33a
--- /dev/null
+++ b/ceph/src/test/cls_rbd/test_cls_rbd.cc
@@ -0,0 +1,919 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/snap_types.h"
+#include "include/encoding.h"
+#include "include/types.h"
+#include "include/rados/librados.h"
+#include "cls/rbd/cls_rbd.h"
+#include "cls/rbd/cls_rbd_client.h"
+
+#include "gtest/gtest.h"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+
+using namespace std;
+using ::librbd::cls_client::create_image;
+using ::librbd::cls_client::get_features;
+using ::librbd::cls_client::get_size;
+using ::librbd::cls_client::get_object_prefix;
+using ::librbd::cls_client::set_size;
+using ::librbd::cls_client::get_parent;
+using ::librbd::cls_client::set_parent;
+using ::librbd::cls_client::remove_parent;
+using ::librbd::cls_client::snapshot_add;
+using ::librbd::cls_client::snapshot_remove;
+using ::librbd::cls_client::add_child;
+using ::librbd::cls_client::remove_child;
+using ::librbd::cls_client::get_children;
+using ::librbd::cls_client::get_snapcontext;
+using ::librbd::cls_client::snapshot_list;
+using ::librbd::cls_client::copyup;
+using ::librbd::cls_client::get_id;
+using ::librbd::cls_client::set_id;
+using ::librbd::cls_client::dir_get_id;
+using ::librbd::cls_client::dir_get_name;
+using ::librbd::cls_client::dir_list;
+using ::librbd::cls_client::dir_add_image;
+using ::librbd::cls_client::dir_remove_image;
+using ::librbd::cls_client::dir_rename_image;
+using ::librbd::parent_info;
+using ::librbd::parent_spec;
+using ::librbd::cls_client::get_protection_status;
+using ::librbd::cls_client::set_protection_status;
+using ::librbd::cls_client::get_stripe_unit_count;
+using ::librbd::cls_client::set_stripe_unit_count;
+using ::librbd::cls_client::old_snapshot_add;
+
+static char *random_buf(size_t len)
+{
+  char *b = new char[len];
+  for (size_t i = 0; i < len; i++)
+    b[i] = (rand() % (128 - 32)) + 32;
+  return b;
+}
+
+TEST(cls_rbd, copyup)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  string oid = "rbd_copyup_test";
+  bufferlist inbl, outbl;
+
+  // copyup of 0-len nonexistent object should create new 0-len object
+  ioctx.remove(oid);
+  ASSERT_EQ(0, copyup(&ioctx, oid, inbl));
+  uint64_t size;
+  ASSERT_EQ(0, ioctx.stat(oid, &size, NULL));
+  ASSERT_EQ(0U, size);
+
+  // create some random data to write
+  size_t l = 4 << 20;
+  char *b = random_buf(l);
+  inbl.append(b, l);
+  delete b;
+  ASSERT_EQ(l, inbl.length());
+
+  // copyup to nonexistent object should create new object
+  ioctx.remove(oid);
+  ASSERT_EQ(-ENOENT, ioctx.remove(oid));
+  ASSERT_EQ(0, copyup(&ioctx, oid, inbl));
+  // and its contents should match
+  ASSERT_EQ(l, (size_t)ioctx.read(oid, outbl, l, 0));
+  ASSERT_TRUE(outbl.contents_equal(inbl));
+
+  // now send different data, but with a preexisting object
+  bufferlist inbl2;
+  b = random_buf(l);
+  inbl2.append(b, l);
+  delete b;
+  ASSERT_EQ(l, inbl2.length());
+
+  // should still succeed
+  ASSERT_EQ(0, copyup(&ioctx, oid, inbl));
+  ASSERT_EQ(l, (size_t)ioctx.read(oid, outbl, l, 0));
+  // but contents should not have changed
+  ASSERT_FALSE(outbl.contents_equal(inbl2));
+  ASSERT_TRUE(outbl.contents_equal(inbl));
+
+  ASSERT_EQ(0, ioctx.remove(oid));
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, get_and_set_id)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  string oid = "rbd_id_test";
+  string id;
+  string valid_id = "0123abcxyzZYXCBA";
+  string invalid_id = ".abc";
+  string empty_id;
+
+  ASSERT_EQ(-ENOENT, get_id(&ioctx, oid, &id));
+  ASSERT_EQ(-ENOENT, set_id(&ioctx, oid, valid_id));
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+  ASSERT_EQ(-EINVAL, set_id(&ioctx, oid, invalid_id));
+  ASSERT_EQ(-EINVAL, set_id(&ioctx, oid, empty_id));
+  ASSERT_EQ(-ENOENT, get_id(&ioctx, oid, &id));
+
+  ASSERT_EQ(0, set_id(&ioctx, oid, valid_id));
+  ASSERT_EQ(-EEXIST, set_id(&ioctx, oid, valid_id));
+  ASSERT_EQ(-EEXIST, set_id(&ioctx, oid, valid_id + valid_id));
+  ASSERT_EQ(0, get_id(&ioctx, oid, &id));
+  ASSERT_EQ(id, valid_id);
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, add_remove_child)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  string oid = "rbd_children_test";
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  string snapname = "parent_snap";
+  snapid_t snapid(10);
+  string parent_image = "parent_id";
+  setchildren;
+  parent_spec pspec(ioctx.get_id(), parent_image, snapid);
+
+  // nonexistent children cannot be listed or removed
+  ASSERT_EQ(-ENOENT, get_children(&ioctx, oid, pspec, children));
+  ASSERT_EQ(-ENOENT, remove_child(&ioctx, oid, pspec, "child1"));
+
+  // create the parent and snapshot
+  ASSERT_EQ(0, create_image(&ioctx, parent_image, 2<<20, 0,
+			    RBD_FEATURE_LAYERING, parent_image));
+  ASSERT_EQ(0, snapshot_add(&ioctx, parent_image, snapid, snapname));
+
+  // add child to it, verify it showed up
+  ASSERT_EQ(0, add_child(&ioctx, oid, pspec, "child1"));
+  ASSERT_EQ(0, get_children(&ioctx, oid, pspec, children));
+  ASSERT_TRUE(children.find("child1") != children.end());
+  // add another child to it, verify it showed up
+  ASSERT_EQ(0, add_child(&ioctx, oid, pspec, "child2"));
+  ASSERT_EQ(0, get_children(&ioctx, oid, pspec, children));
+  ASSERT_TRUE(children.find("child2") != children.end());
+  // add child2 again, expect -EEXIST
+  ASSERT_EQ(-EEXIST, add_child(&ioctx, oid, pspec, "child2"));
+  // remove first, verify it's gone
+  ASSERT_EQ(0, remove_child(&ioctx, oid, pspec, "child1"));
+  ASSERT_EQ(0, get_children(&ioctx, oid, pspec, children));
+  ASSERT_FALSE(children.find("child1") != children.end());
+  // remove second, verify list empty
+  ASSERT_EQ(0, remove_child(&ioctx, oid, pspec, "child2"));
+  ASSERT_EQ(-ENOENT, get_children(&ioctx, oid, pspec, children));
+  // try to remove again, validate -ENOENT to that as well
+  ASSERT_EQ(-ENOENT, remove_child(&ioctx, oid, pspec, "child2"));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, directory_methods)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  string oid = "rbd_id_test";
+  string id, name;
+  string imgname = "bar";
+  string imgname2 = "foo";
+  string imgname3 = "baz";
+  string valid_id = "0123abcxyzZYXCBA";
+  string valid_id2 = "5";
+  string invalid_id = ".abc";
+  string empty;
+
+  ASSERT_EQ(-ENOENT, dir_get_id(&ioctx, oid, imgname, &id));
+  ASSERT_EQ(-ENOENT, dir_get_name(&ioctx, oid, valid_id, &name));
+  ASSERT_EQ(-ENOENT, dir_remove_image(&ioctx, oid, imgname, valid_id));
+
+  ASSERT_EQ(-EINVAL, dir_add_image(&ioctx, oid, imgname, invalid_id));
+  ASSERT_EQ(-EINVAL, dir_add_image(&ioctx, oid, imgname, empty));
+  ASSERT_EQ(-EINVAL, dir_add_image(&ioctx, oid, empty, valid_id));
+
+  map images;
+  ASSERT_EQ(-ENOENT, dir_list(&ioctx, oid, "", 30, &images));
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+  ASSERT_EQ(0, dir_list(&ioctx, oid, "", 30, &images));
+  ASSERT_EQ(0u, images.size());
+  ASSERT_EQ(0, ioctx.remove(oid));
+
+  ASSERT_EQ(0, dir_add_image(&ioctx, oid, imgname, valid_id));
+  ASSERT_EQ(-EEXIST, dir_add_image(&ioctx, oid, imgname, valid_id2));
+  ASSERT_EQ(-EBADF, dir_add_image(&ioctx, oid, imgname2, valid_id));
+  ASSERT_EQ(0, dir_list(&ioctx, oid, "", 30, &images));
+  ASSERT_EQ(1u, images.size());
+  ASSERT_EQ(valid_id, images[imgname]);
+  ASSERT_EQ(0, dir_list(&ioctx, oid, "", 0, &images));
+  ASSERT_EQ(0u, images.size());
+  ASSERT_EQ(0, dir_get_name(&ioctx, oid, valid_id, &name));
+  ASSERT_EQ(imgname, name);
+  ASSERT_EQ(0, dir_get_id(&ioctx, oid, imgname, &id));
+  ASSERT_EQ(valid_id, id);
+
+  ASSERT_EQ(0, dir_add_image(&ioctx, oid, imgname2, valid_id2));
+  ASSERT_EQ(0, dir_list(&ioctx, oid, "", 30, &images));
+  ASSERT_EQ(2u, images.size());
+  ASSERT_EQ(valid_id, images[imgname]);
+  ASSERT_EQ(valid_id2, images[imgname2]);
+  ASSERT_EQ(0, dir_list(&ioctx, oid, imgname, 0, &images));
+  ASSERT_EQ(0u, images.size());
+  ASSERT_EQ(0, dir_list(&ioctx, oid, imgname, 2, &images));
+  ASSERT_EQ(1u, images.size());
+  ASSERT_EQ(valid_id2, images[imgname2]);
+  ASSERT_EQ(0, dir_get_name(&ioctx, oid, valid_id2, &name));
+  ASSERT_EQ(imgname2, name);
+  ASSERT_EQ(0, dir_get_id(&ioctx, oid, imgname2, &id));
+  ASSERT_EQ(valid_id2, id);
+
+  ASSERT_EQ(-ESTALE, dir_rename_image(&ioctx, oid, imgname, imgname2, valid_id2));
+  ASSERT_EQ(-ESTALE, dir_remove_image(&ioctx, oid, imgname, valid_id2));
+  ASSERT_EQ(-EEXIST, dir_rename_image(&ioctx, oid, imgname, imgname2, valid_id));
+  ASSERT_EQ(0, dir_get_id(&ioctx, oid, imgname, &id));
+  ASSERT_EQ(valid_id, id);
+  ASSERT_EQ(0, dir_get_name(&ioctx, oid, valid_id2, &name));
+  ASSERT_EQ(imgname2, name);
+
+  ASSERT_EQ(0, dir_rename_image(&ioctx, oid, imgname, imgname3, valid_id));
+  ASSERT_EQ(0, dir_get_id(&ioctx, oid, imgname3, &id));
+  ASSERT_EQ(valid_id, id);
+  ASSERT_EQ(0, dir_get_name(&ioctx, oid, valid_id, &name));
+  ASSERT_EQ(imgname3, name);
+  ASSERT_EQ(0, dir_rename_image(&ioctx, oid, imgname3, imgname, valid_id));
+
+  ASSERT_EQ(0, dir_remove_image(&ioctx, oid, imgname, valid_id));
+  ASSERT_EQ(0, dir_list(&ioctx, oid, "", 30, &images));
+  ASSERT_EQ(1u, images.size());
+  ASSERT_EQ(valid_id2, images[imgname2]);
+  ASSERT_EQ(0, dir_list(&ioctx, oid, imgname2, 30, &images));
+  ASSERT_EQ(0u, images.size());
+  ASSERT_EQ(0, dir_get_name(&ioctx, oid, valid_id2, &name));
+  ASSERT_EQ(imgname2, name);
+  ASSERT_EQ(0, dir_get_id(&ioctx, oid, imgname2, &id));
+  ASSERT_EQ(valid_id2, id);
+  ASSERT_EQ(-ENOENT, dir_get_name(&ioctx, oid, valid_id, &name));
+  ASSERT_EQ(-ENOENT, dir_get_id(&ioctx, oid, imgname, &id));
+
+  ASSERT_EQ(0, dir_add_image(&ioctx, oid, imgname, valid_id));
+  ASSERT_EQ(0, dir_list(&ioctx, oid, "", 30, &images));
+  ASSERT_EQ(2u, images.size());
+  ASSERT_EQ(valid_id, images[imgname]);
+  ASSERT_EQ(valid_id2, images[imgname2]);
+  ASSERT_EQ(0, dir_remove_image(&ioctx, oid, imgname, valid_id));
+  ASSERT_EQ(-ENOENT, dir_remove_image(&ioctx, oid, imgname, valid_id));
+  ASSERT_EQ(0, dir_remove_image(&ioctx, oid, imgname2, valid_id2));
+  ASSERT_EQ(0, dir_list(&ioctx, oid, "", 30, &images));
+  ASSERT_EQ(0u, images.size());
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, create)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  string oid = "testobj";
+  uint64_t size = 20ULL << 30;
+  uint64_t features = 0;
+  uint8_t order = 22;
+  string object_prefix = "foo";
+
+  ASSERT_EQ(0, create_image(&ioctx, oid, size, order,
+			    features, object_prefix));
+  ASSERT_EQ(-EEXIST, create_image(&ioctx, oid, size, order,
+				  features, object_prefix));
+  ASSERT_EQ(0, ioctx.remove(oid));
+
+  ASSERT_EQ(-EINVAL, create_image(&ioctx, oid, size, order,
+				  features, ""));
+  ASSERT_EQ(-ENOENT, ioctx.remove(oid));
+
+  ASSERT_EQ(0, create_image(&ioctx, oid, 0, order,
+			    features, object_prefix));
+  ASSERT_EQ(0, ioctx.remove(oid));
+
+  ASSERT_EQ(-ENOSYS, create_image(&ioctx, oid, size, order,
+				  -1, object_prefix));
+  ASSERT_EQ(-ENOENT, ioctx.remove(oid));
+
+  bufferlist inbl, outbl;
+  ASSERT_EQ(-EINVAL, ioctx.exec(oid, "rbd", "create", inbl, outbl));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, get_features)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  uint64_t features;
+  ASSERT_EQ(-ENOENT, get_features(&ioctx, "foo", CEPH_NOSNAP, &features));
+
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 0, 22, 0, "foo"));
+  ASSERT_EQ(0, get_features(&ioctx, "foo", CEPH_NOSNAP, &features));
+  ASSERT_EQ(0u, features);
+
+  ASSERT_EQ(-ENOENT, get_features(&ioctx, "foo", 1, &features));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, get_object_prefix)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  string object_prefix;
+  ASSERT_EQ(-ENOENT, get_object_prefix(&ioctx, "foo", &object_prefix));
+
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 0, 22, 0, "foo"));
+  ASSERT_EQ(0, get_object_prefix(&ioctx, "foo", &object_prefix));
+  ASSERT_EQ("foo", object_prefix);
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, get_size)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  uint64_t size;
+  uint8_t order;
+  ASSERT_EQ(-ENOENT, get_size(&ioctx, "foo", CEPH_NOSNAP, &size, &order));
+
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 0, 22, 0, "foo"));
+  ASSERT_EQ(0, get_size(&ioctx, "foo", CEPH_NOSNAP, &size, &order));
+  ASSERT_EQ(0u, size);
+  ASSERT_EQ(22, order);
+  ASSERT_EQ(0, ioctx.remove("foo"));
+
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 2 << 22, 0, 0, "foo"));
+  ASSERT_EQ(0, get_size(&ioctx, "foo", CEPH_NOSNAP, &size, &order));
+  ASSERT_EQ(2u << 22, size);
+  ASSERT_EQ(0, order);
+
+  ASSERT_EQ(-ENOENT, get_size(&ioctx, "foo", 1, &size, &order));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, set_size)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  ASSERT_EQ(-ENOENT, set_size(&ioctx, "foo", 5));
+
+  uint64_t size;
+  uint8_t order;
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 0, 22, 0, "foo"));
+  ASSERT_EQ(0, get_size(&ioctx, "foo", CEPH_NOSNAP, &size, &order));
+  ASSERT_EQ(0u, size);
+  ASSERT_EQ(22, order);
+
+  ASSERT_EQ(0, set_size(&ioctx, "foo", 0));
+  ASSERT_EQ(0, get_size(&ioctx, "foo", CEPH_NOSNAP, &size, &order));
+  ASSERT_EQ(0u, size);
+  ASSERT_EQ(22, order);
+
+  ASSERT_EQ(0, set_size(&ioctx, "foo", 3 << 22));
+  ASSERT_EQ(0, get_size(&ioctx, "foo", CEPH_NOSNAP, &size, &order));
+  ASSERT_EQ(3u << 22, size);
+  ASSERT_EQ(22, order);
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, protection_status)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  uint8_t status = RBD_PROTECTION_STATUS_UNPROTECTED;
+  ASSERT_EQ(-ENOENT, get_protection_status(&ioctx, "foo",
+					   CEPH_NOSNAP, &status));
+  ASSERT_EQ(-ENOENT, set_protection_status(&ioctx, "foo",
+					   CEPH_NOSNAP, status));
+
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 0, 22, RBD_FEATURE_LAYERING, "foo"));
+  ASSERT_EQ(0, create_image(&ioctx, "bar", 0, 22, 0, "foo"));
+  ASSERT_EQ(-EINVAL, get_protection_status(&ioctx, "bar",
+					   CEPH_NOSNAP, &status));
+  ASSERT_EQ(-ENOEXEC, set_protection_status(&ioctx, "bar",
+					   CEPH_NOSNAP, status));
+  ASSERT_EQ(-EINVAL, get_protection_status(&ioctx, "foo",
+					   CEPH_NOSNAP, &status));
+  ASSERT_EQ(-EINVAL, set_protection_status(&ioctx, "foo",
+					   CEPH_NOSNAP, status));
+  ASSERT_EQ(-ENOENT, get_protection_status(&ioctx, "foo",
+					   2, &status));
+  ASSERT_EQ(-ENOENT, set_protection_status(&ioctx, "foo",
+					   2, status));
+
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 10, "snap1"));
+  ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
+				     10, &status));
+  ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTED, status);
+
+  ASSERT_EQ(0, set_protection_status(&ioctx, "foo",
+				     10, RBD_PROTECTION_STATUS_PROTECTED));
+  ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
+				     10, &status));
+  ASSERT_EQ(+RBD_PROTECTION_STATUS_PROTECTED, status);
+  ASSERT_EQ(-EBUSY, snapshot_remove(&ioctx, "foo", 10));
+
+  ASSERT_EQ(0, set_protection_status(&ioctx, "foo",
+				     10, RBD_PROTECTION_STATUS_UNPROTECTING));
+  ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
+				     10, &status));
+  ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTING, status);
+  ASSERT_EQ(-EBUSY, snapshot_remove(&ioctx, "foo", 10));
+
+  ASSERT_EQ(-EINVAL, set_protection_status(&ioctx, "foo",
+					   10, RBD_PROTECTION_STATUS_LAST));
+  ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
+				     10, &status));
+  ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTING, status);
+
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 20, "snap2"));
+  ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
+				     20, &status));
+  ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTED, status);
+  ASSERT_EQ(0, set_protection_status(&ioctx, "foo",
+				     10, RBD_PROTECTION_STATUS_UNPROTECTED));
+  ASSERT_EQ(0, get_protection_status(&ioctx, "foo",
+				     10, &status));
+  ASSERT_EQ(+RBD_PROTECTION_STATUS_UNPROTECTED, status);
+
+  ASSERT_EQ(0, snapshot_remove(&ioctx, "foo", 10));
+  ASSERT_EQ(0, snapshot_remove(&ioctx, "foo", 20));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, parents)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  parent_spec pspec;
+  uint64_t size;
+
+  ASSERT_EQ(-ENOENT, get_parent(&ioctx, "doesnotexist", CEPH_NOSNAP, &pspec, &size));
+
+  // old image should fail
+  ASSERT_EQ(0, create_image(&ioctx, "old", 33<<20, 22, 0, "old_blk."));
+  // get nonexistent parent: succeed, return (-1, "", CEPH_NOSNAP), overlap 0
+  ASSERT_EQ(0, get_parent(&ioctx, "old", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, -1);
+  ASSERT_STREQ("", pspec.image_id.c_str());
+  ASSERT_EQ(pspec.snap_id, CEPH_NOSNAP);
+  ASSERT_EQ(size, 0ULL);
+  pspec = parent_spec(-1, "parent", 3);
+  ASSERT_EQ(-ENOEXEC, set_parent(&ioctx, "old", parent_spec(-1, "parent", 3), 10<<20));
+  ASSERT_EQ(-ENOEXEC, remove_parent(&ioctx, "old"));
+
+  // new image will work
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 33<<20, 22, RBD_FEATURE_LAYERING, "foo."));
+
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(-1, pspec.pool_id);
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 123, &pspec, &size));
+  ASSERT_EQ(-1, pspec.pool_id);
+
+  ASSERT_EQ(-EINVAL, set_parent(&ioctx, "foo", parent_spec(-1, "parent", 3), 10<<20));
+  ASSERT_EQ(-EINVAL, set_parent(&ioctx, "foo", parent_spec(1, "", 3), 10<<20));
+  ASSERT_EQ(-EINVAL, set_parent(&ioctx, "foo", parent_spec(1, "parent", CEPH_NOSNAP), 10<<20));
+  ASSERT_EQ(-EINVAL, set_parent(&ioctx, "foo", parent_spec(1, "parent", 3), 0));
+
+  pspec = parent_spec(1, "parent", 3);
+  ASSERT_EQ(0, set_parent(&ioctx, "foo", pspec, 10<<20));
+  ASSERT_EQ(-EEXIST, set_parent(&ioctx, "foo", pspec, 10<<20));
+  ASSERT_EQ(-EEXIST, set_parent(&ioctx, "foo", parent_spec(2, "parent", 34), 10<<20));
+
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+
+  ASSERT_EQ(0, remove_parent(&ioctx, "foo"));
+  ASSERT_EQ(-ENOENT, remove_parent(&ioctx, "foo"));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(-1, pspec.pool_id);
+
+  // snapshots
+  ASSERT_EQ(0, set_parent(&ioctx, "foo", parent_spec(1, "parent", 3), 10<<20));
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 10, "snap1"));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 10, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 10ull<<20);
+
+  ASSERT_EQ(0, remove_parent(&ioctx, "foo"));
+  ASSERT_EQ(0, set_parent(&ioctx, "foo", parent_spec(4, "parent2", 6), 5<<20));
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 11, "snap2"));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 10, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 10ull<<20);
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 11, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 4);
+  ASSERT_EQ(pspec.image_id, "parent2");
+  ASSERT_EQ(pspec.snap_id, snapid_t(6));
+  ASSERT_EQ(size, 5ull<<20);
+
+  ASSERT_EQ(0, remove_parent(&ioctx, "foo"));
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 12, "snap3"));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 10, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 10ull<<20);
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 11, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 4);
+  ASSERT_EQ(pspec.image_id, "parent2");
+  ASSERT_EQ(pspec.snap_id, snapid_t(6));
+  ASSERT_EQ(size, 5ull<<20);
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 12, &pspec, &size));
+  ASSERT_EQ(-1, pspec.pool_id);
+
+  // make sure set_parent takes min of our size and parent's size
+  ASSERT_EQ(0, set_parent(&ioctx, "foo", parent_spec(1, "parent", 3), 1<<20));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 1ull<<20);
+  ASSERT_EQ(0, remove_parent(&ioctx, "foo"));
+
+  ASSERT_EQ(0, set_parent(&ioctx, "foo", parent_spec(1, "parent", 3), 100<<20));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 33ull<<20);
+  ASSERT_EQ(0, remove_parent(&ioctx, "foo"));
+
+  // make sure resize adjust parent overlap
+  ASSERT_EQ(0, set_parent(&ioctx, "foo", parent_spec(1, "parent", 3), 10<<20));
+
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 14, "snap4"));
+  ASSERT_EQ(0, set_size(&ioctx, "foo", 3 << 20));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 3ull<<20);
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 14, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 10ull<<20);
+
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 15, "snap5"));
+  ASSERT_EQ(0, set_size(&ioctx, "foo", 30 << 20));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 3ull<<20);
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 14, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 10ull<<20);
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 15, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 3ull<<20);
+
+  ASSERT_EQ(0, set_size(&ioctx, "foo", 2 << 20));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", CEPH_NOSNAP, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 2ull<<20);
+
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 16, "snap6"));
+  ASSERT_EQ(0, get_parent(&ioctx, "foo", 16, &pspec, &size));
+  ASSERT_EQ(pspec.pool_id, 1);
+  ASSERT_EQ(pspec.image_id, "parent");
+  ASSERT_EQ(pspec.snap_id, snapid_t(3));
+  ASSERT_EQ(size, 2ull<<20);
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, snapshots)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  ASSERT_EQ(-ENOENT, snapshot_add(&ioctx, "foo", 0, "snap1"));
+
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 10, 22, 0, "foo"));
+
+  vector snap_names;
+  vector snap_sizes;
+  vector snap_features;
+  SnapContext snapc;
+  vector parents;
+  vector protection_status;
+
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(0u, snapc.snaps.size());
+  ASSERT_EQ(0u, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(0u, snap_names.size());
+  ASSERT_EQ(0u, snap_sizes.size());
+  ASSERT_EQ(0u, snap_features.size());
+
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 0, "snap1"));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(1u, snapc.snaps.size());
+  ASSERT_EQ(0u, snapc.snaps[0]);
+  ASSERT_EQ(0u, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(1u, snap_names.size());
+  ASSERT_EQ("snap1", snap_names[0]);
+  ASSERT_EQ(10u, snap_sizes[0]);
+  ASSERT_EQ(0u, snap_features[0]);
+
+  // snap with same id and name
+  ASSERT_EQ(-EEXIST, snapshot_add(&ioctx, "foo", 0, "snap1"));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(1u, snapc.snaps.size());
+  ASSERT_EQ(0u, snapc.snaps[0]);
+  ASSERT_EQ(0u, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(1u, snap_names.size());
+  ASSERT_EQ("snap1", snap_names[0]);
+  ASSERT_EQ(10u, snap_sizes[0]);
+  ASSERT_EQ(0u, snap_features[0]);
+
+  // snap with same id, different name
+  ASSERT_EQ(-EEXIST, snapshot_add(&ioctx, "foo", 0, "snap2"));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(1u, snapc.snaps.size());
+  ASSERT_EQ(0u, snapc.snaps[0]);
+  ASSERT_EQ(0u, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(1u, snap_names.size());
+  ASSERT_EQ("snap1", snap_names[0]);
+  ASSERT_EQ(10u, snap_sizes[0]);
+  ASSERT_EQ(0u, snap_features[0]);
+
+  // snap with different id, same name
+  ASSERT_EQ(-EEXIST, snapshot_add(&ioctx, "foo", 1, "snap1"));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(1u, snapc.snaps.size());
+  ASSERT_EQ(0u, snapc.snaps[0]);
+  ASSERT_EQ(0u, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(snap_names.size(), 1u);
+  ASSERT_EQ(snap_names[0], "snap1");
+  ASSERT_EQ(snap_sizes[0], 10u);
+  ASSERT_EQ(snap_features[0], 0u);
+
+  // snap with different id, different name
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", 1, "snap2"));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(2u, snapc.snaps.size());
+  ASSERT_EQ(1u, snapc.snaps[0]);
+  ASSERT_EQ(0u, snapc.snaps[1]);
+  ASSERT_EQ(1u, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(2u, snap_names.size());
+  ASSERT_EQ("snap2", snap_names[0]);
+  ASSERT_EQ(10u, snap_sizes[0]);
+  ASSERT_EQ(0u, snap_features[0]);
+  ASSERT_EQ("snap1", snap_names[1]);
+  ASSERT_EQ(10u, snap_sizes[1]);
+  ASSERT_EQ(0u, snap_features[1]);
+
+  ASSERT_EQ(0, snapshot_remove(&ioctx, "foo", 0));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(1u, snapc.snaps.size());
+  ASSERT_EQ(1u, snapc.snaps[0]);
+  ASSERT_EQ(1u, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(1u, snap_names.size());
+  ASSERT_EQ("snap2", snap_names[0]);
+  ASSERT_EQ(10u, snap_sizes[0]);
+  ASSERT_EQ(0u, snap_features[0]);
+
+  uint64_t size;
+  uint8_t order;
+  ASSERT_EQ(0, set_size(&ioctx, "foo", 0));
+  ASSERT_EQ(0, get_size(&ioctx, "foo", CEPH_NOSNAP, &size, &order));
+  ASSERT_EQ(0u, size);
+  ASSERT_EQ(22u, order);
+
+  uint64_t large_snap_id = 1ull << 63;
+  ASSERT_EQ(0, snapshot_add(&ioctx, "foo", large_snap_id, "snap3"));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(2u, snapc.snaps.size());
+  ASSERT_EQ(large_snap_id, snapc.snaps[0]);
+  ASSERT_EQ(1u, snapc.snaps[1]);
+  ASSERT_EQ(large_snap_id, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(2u, snap_names.size());
+  ASSERT_EQ("snap3", snap_names[0]);
+  ASSERT_EQ(0u, snap_sizes[0]);
+  ASSERT_EQ(0u, snap_features[0]);
+  ASSERT_EQ("snap2", snap_names[1]);
+  ASSERT_EQ(10u, snap_sizes[1]);
+  ASSERT_EQ(0u, snap_features[1]);
+
+  ASSERT_EQ(0, get_size(&ioctx, "foo", large_snap_id, &size, &order));
+  ASSERT_EQ(0u, size);
+  ASSERT_EQ(22u, order);
+
+  ASSERT_EQ(0, get_size(&ioctx, "foo", 1, &size, &order));
+  ASSERT_EQ(10u, size);
+  ASSERT_EQ(22u, order);
+
+  ASSERT_EQ(0, snapshot_remove(&ioctx, "foo", large_snap_id));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(1u, snapc.snaps.size());
+  ASSERT_EQ(1u, snapc.snaps[0]);
+  ASSERT_EQ(large_snap_id, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(1u, snap_names.size());
+  ASSERT_EQ("snap2", snap_names[0]);
+  ASSERT_EQ(10u, snap_sizes[0]);
+  ASSERT_EQ(0u, snap_features[0]);
+
+  ASSERT_EQ(-ENOENT, snapshot_remove(&ioctx, "foo", large_snap_id));
+  ASSERT_EQ(0, snapshot_remove(&ioctx, "foo", 1));
+  ASSERT_EQ(0, get_snapcontext(&ioctx, "foo", &snapc));
+  ASSERT_EQ(0u, snapc.snaps.size());
+  ASSERT_EQ(large_snap_id, snapc.seq);
+  ASSERT_EQ(0, snapshot_list(&ioctx, "foo", snapc.snaps, &snap_names,
+			     &snap_sizes, &snap_features, &parents,
+			     &protection_status));
+  ASSERT_EQ(0u, snap_names.size());
+  ASSERT_EQ(0u, snap_sizes.size());
+  ASSERT_EQ(0u, snap_features.size());
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, snapid_race)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  buffer::list bl;
+  buffer::ptr bp(4096);
+  bp.zero();
+  bl.append(bp);
+
+  string oid = "foo";
+  ASSERT_EQ(0, ioctx.write(oid, bl, 4096, 0));
+  ASSERT_EQ(0, old_snapshot_add(&ioctx, oid, 1, "test1"));
+  ASSERT_EQ(0, old_snapshot_add(&ioctx, oid, 3, "test3"));
+  ASSERT_EQ(-ESTALE, old_snapshot_add(&ioctx, oid, 2, "test2"));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rbd, stripingv2)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  ASSERT_EQ(0, create_image(&ioctx, "foo", 10, 22, 0, "foo"));
+
+  uint64_t su = 65536, sc = 12;
+  ASSERT_EQ(-ENOEXEC, get_stripe_unit_count(&ioctx, "foo", &su, &sc));
+  ASSERT_EQ(-ENOEXEC, set_stripe_unit_count(&ioctx, "foo", su, sc));
+  
+  ASSERT_EQ(0, create_image(&ioctx, "bar", 10, 22, RBD_FEATURE_STRIPINGV2, "bar"));
+  ASSERT_EQ(0, get_stripe_unit_count(&ioctx, "bar", &su, &sc));
+  ASSERT_EQ(1ull << 22, su);
+  ASSERT_EQ(1ull, sc);
+  su = 8192;
+  sc = 456;
+  ASSERT_EQ(0, set_stripe_unit_count(&ioctx, "bar", su, sc));
+  su = sc = 0;
+  ASSERT_EQ(0, get_stripe_unit_count(&ioctx, "bar", &su, &sc));
+  ASSERT_EQ(8192ull, su);
+  ASSERT_EQ(456ull, sc);
+
+  // su must not be larger than an object
+  ASSERT_EQ(-EINVAL, set_stripe_unit_count(&ioctx, "bar", 1 << 23, 1));
+  // su must be a factor of object size
+  ASSERT_EQ(-EINVAL, set_stripe_unit_count(&ioctx, "bar", 511, 1));
+  // su and sc must be non-zero
+  ASSERT_EQ(-EINVAL, set_stripe_unit_count(&ioctx, "bar", 0, 1));
+  ASSERT_EQ(-EINVAL, set_stripe_unit_count(&ioctx, "bar", 1, 0));
+  ASSERT_EQ(-EINVAL, set_stripe_unit_count(&ioctx, "bar", 0, 0));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
diff --git a/ceph/src/test/cls_refcount/test_cls_refcount.cc b/ceph/src/test/cls_refcount/test_cls_refcount.cc
new file mode 100644
index 00000000..79d9c942
--- /dev/null
+++ b/ceph/src/test/cls_refcount/test_cls_refcount.cc
@@ -0,0 +1,258 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+#include "cls/refcount/cls_refcount_client.h"
+
+#include "gtest/gtest.h"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+
+static librados::ObjectWriteOperation *new_op() {
+  return new librados::ObjectWriteOperation();
+}
+
+TEST(cls_rgw, test_implicit) /* test refcount using implicit referencing of newly created objects */
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+
+  /* create object */
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  /* read reference, should return a single wildcard entry */
+
+  list refs;
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs, true));
+  ASSERT_EQ(1, (int)refs.size());
+
+  string wildcard_tag;
+  string tag = refs.front();
+
+  ASSERT_EQ(wildcard_tag, tag);
+
+  /* take another reference, verify */
+
+  string oldtag = "oldtag";
+  string newtag = "newtag";
+
+  librados::ObjectWriteOperation *op = new_op();
+  cls_refcount_get(*op, newtag, true);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs, true));
+  ASSERT_EQ(2, (int)refs.size());
+
+  map refs_map;
+  for (list::iterator iter = refs.begin(); iter != refs.end(); ++iter) {
+    refs_map[*iter] = true;
+  }
+
+  ASSERT_EQ(1, (int)refs_map.count(wildcard_tag));
+  ASSERT_EQ(1, (int)refs_map.count(newtag));
+
+  delete op;
+
+  /* drop reference to oldtag */
+
+  op = new_op();
+  cls_refcount_put(*op, oldtag, true);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs, true));
+  ASSERT_EQ(1, (int)refs.size());
+
+  tag = refs.front();
+  ASSERT_EQ(newtag, tag);
+
+  delete op;
+
+  /* drop oldtag reference again, op should return success, wouldn't do anything */
+
+  op = new_op();
+  cls_refcount_put(*op, oldtag, true);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs, true));
+  ASSERT_EQ(1, (int)refs.size());
+
+  tag = refs.front();
+  ASSERT_EQ(newtag, tag);
+
+  delete op;
+
+  /* drop newtag reference, make sure object removed */
+  op = new_op();
+  cls_refcount_put(*op, newtag, true);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(-ENOENT, ioctx.stat(oid, NULL, NULL));
+
+  delete op;
+
+  /* remove pool */
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rgw, test_explicit) /* test refcount using implicit referencing of newly created objects */
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+
+  /* create object */
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  /* read reference, should return a single wildcard entry */
+
+  list refs;
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs));
+  ASSERT_EQ(0, (int)refs.size());
+
+
+  /* take first reference, verify */
+
+  string newtag = "newtag";
+
+  librados::ObjectWriteOperation *op = new_op();
+  cls_refcount_get(*op, newtag);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs));
+  ASSERT_EQ(1, (int)refs.size());
+
+  map refs_map;
+  for (list::iterator iter = refs.begin(); iter != refs.end(); ++iter) {
+    refs_map[*iter] = true;
+  }
+
+  ASSERT_EQ(1, (int)refs_map.count(newtag));
+
+  delete op;
+
+  /* try to drop reference to unexisting tag */
+
+  string nosuchtag = "nosuchtag";
+
+  op = new_op();
+  cls_refcount_put(*op, nosuchtag);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs));
+  ASSERT_EQ(1, (int)refs.size());
+
+  string tag = refs.front();
+  ASSERT_EQ(newtag, tag);
+
+  delete op;
+
+  /* drop newtag reference, make sure object removed */
+  op = new_op();
+  cls_refcount_put(*op, newtag);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(-ENOENT, ioctx.stat(oid, NULL, NULL));
+
+  delete op;
+
+  /* remove pool */
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(cls_rgw, set) /* test refcount using implicit referencing of newly created objects */
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+
+  /* create object */
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  /* read reference, should return a single wildcard entry */
+
+  list tag_refs, refs;
+
+#define TAGS_NUM 5
+  string tags[TAGS_NUM];
+
+  char buf[16];
+  for (int i = 0; i < TAGS_NUM; i++) {
+    snprintf(buf, sizeof(buf), "tag%d", i);
+    tags[i] = buf;
+    tag_refs.push_back(tags[i]);
+  }
+
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs));
+  ASSERT_EQ(0, (int)refs.size());
+
+  /* set reference list, verify */
+
+  librados::ObjectWriteOperation *op = new_op();
+  cls_refcount_set(*op, tag_refs);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  refs.clear();
+  ASSERT_EQ(0, cls_refcount_read(ioctx, oid, &refs));
+  ASSERT_EQ(TAGS_NUM, (int)refs.size());
+
+  map refs_map;
+  for (list::iterator iter = refs.begin(); iter != refs.end(); ++iter) {
+    refs_map[*iter] = true;
+  }
+
+  for (int i = 0; i < TAGS_NUM; i++) {
+    ASSERT_EQ(1, (int)refs_map.count(tags[i]));
+  }
+
+  delete op;
+
+  /* remove all refs */
+
+  for (int i = 0; i < TAGS_NUM; i++) {
+    op = new_op();
+    cls_refcount_put(*op, tags[i]);
+    ASSERT_EQ(0, ioctx.operate(oid, op));
+    delete op;
+  }
+
+  ASSERT_EQ(-ENOENT, ioctx.stat(oid, NULL, NULL));
+
+  /* remove pool */
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
diff --git a/ceph/src/test/cls_replica_log/test_cls_replica_log.cc b/ceph/src/test/cls_replica_log/test_cls_replica_log.cc
new file mode 100644
index 00000000..8c204cae
--- /dev/null
+++ b/ceph/src/test/cls_replica_log/test_cls_replica_log.cc
@@ -0,0 +1,151 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * Copyright 2013 Inktank
+ */
+
+#include "gtest/gtest.h"
+#include "test/librados/test.h"
+
+#include "cls/replica_log/cls_replica_log_client.h"
+#include "cls/replica_log/cls_replica_log_types.h"
+
+class cls_replica_log_Test : public ::testing::Test {
+public:
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name;
+  string oid;
+  string entity;
+  string marker;
+  utime_t time;
+  list > entries;
+  cls_replica_log_progress_marker progress;
+
+  void SetUp() {
+    pool_name = get_temp_pool_name();
+    ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+    ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+    oid = "obj";
+    ASSERT_EQ(0, ioctx.create(oid, true));
+  }
+
+  void add_marker() {
+    entity = "tester_entity";
+    marker = "tester_marker1";
+    time.set_from_double(10);
+    entries.push_back(make_pair("tester_obj1", time));
+    time.set_from_double(20);
+    cls_replica_log_prepare_marker(progress, entity, marker, time, &entries);
+    librados::ObjectWriteOperation opw;
+    cls_replica_log_update_bound(opw, progress);
+    ASSERT_EQ(0, ioctx.operate(oid, &opw));
+  }
+};
+
+TEST_F(cls_replica_log_Test, test_set_get_marker)
+{
+  add_marker();
+
+  string reply_position_marker;
+  utime_t reply_time;
+  list return_progress_list;
+  ASSERT_EQ(0, cls_replica_log_get_bounds(ioctx, oid, reply_position_marker,
+                                          reply_time, return_progress_list));
+
+  ASSERT_EQ(reply_position_marker, marker);
+  ASSERT_EQ((double)10, (double)reply_time);
+  string response_entity;
+  string response_marker;
+  utime_t response_time;
+  list > response_item_list;
+
+  cls_replica_log_extract_marker(return_progress_list.front(),
+                                 response_entity, response_marker,
+                                 response_time, response_item_list);
+  ASSERT_EQ(response_entity, entity);
+  ASSERT_EQ(response_marker, marker);
+  ASSERT_EQ(response_time, time);
+  ASSERT_EQ((unsigned)1, response_item_list.size());
+  ASSERT_EQ("tester_obj1", response_item_list.front().first);
+}
+
+TEST_F(cls_replica_log_Test, test_bad_update)
+{
+  add_marker();
+
+  time.set_from_double(15);
+  cls_replica_log_progress_marker bad_marker;
+  cls_replica_log_prepare_marker(bad_marker, entity, marker, time, &entries);
+  librados::ObjectWriteOperation badw;
+  cls_replica_log_update_bound(badw, bad_marker);
+  ASSERT_EQ(-EINVAL, ioctx.operate(oid, &badw));
+}
+
+TEST_F(cls_replica_log_Test, test_bad_delete)
+{
+  add_marker();
+
+  librados::ObjectWriteOperation badd;
+  cls_replica_log_delete_bound(badd, entity);
+  ASSERT_EQ(-ENOTEMPTY, ioctx.operate(oid, &badd));
+}
+
+TEST_F(cls_replica_log_Test, test_good_delete)
+{
+  add_marker();
+
+  librados::ObjectWriteOperation opc;
+  progress.items.clear();
+  cls_replica_log_update_bound(opc, progress);
+  ASSERT_EQ(0, ioctx.operate(oid, &opc));
+  librados::ObjectWriteOperation opd;
+  cls_replica_log_delete_bound(opd, entity);
+  ASSERT_EQ(0, ioctx.operate(oid, &opd));
+
+  string reply_position_marker;
+  utime_t reply_time;
+  list return_progress_list;
+  ASSERT_EQ(0, cls_replica_log_get_bounds(ioctx, oid, reply_position_marker,
+                                          reply_time, return_progress_list));
+  ASSERT_EQ((unsigned)0, return_progress_list.size());
+}
+
+TEST_F(cls_replica_log_Test, test_bad_get)
+{
+  string reply_position_marker;
+  utime_t reply_time;
+  list return_progress_list;
+  ASSERT_EQ(-ENOENT,
+            cls_replica_log_get_bounds(ioctx, oid, reply_position_marker,
+                                       reply_time, return_progress_list));
+}
+
+TEST_F(cls_replica_log_Test, test_double_delete)
+{
+  add_marker();
+
+  librados::ObjectWriteOperation opc;
+  progress.items.clear();
+  cls_replica_log_update_bound(opc, progress);
+  ASSERT_EQ(0, ioctx.operate(oid, &opc));
+  librados::ObjectWriteOperation opd;
+  cls_replica_log_delete_bound(opd, entity);
+  ASSERT_EQ(0, ioctx.operate(oid, &opd));
+
+  librados::ObjectWriteOperation opd2;
+  cls_replica_log_delete_bound(opd2, entity);
+  ASSERT_EQ(0, ioctx.operate(oid, &opd2));
+
+  string reply_position_marker;
+  utime_t reply_time;
+  list return_progress_list;
+  ASSERT_EQ(0, cls_replica_log_get_bounds(ioctx, oid, reply_position_marker,
+                                          reply_time, return_progress_list));
+  ASSERT_EQ((unsigned)0, return_progress_list.size());
+
+}
diff --git a/ceph/src/test/cls_rgw/test_cls_rgw.cc b/ceph/src/test/cls_rgw/test_cls_rgw.cc
new file mode 100644
index 00000000..44cb3030
--- /dev/null
+++ b/ceph/src/test/cls_rgw/test_cls_rgw.cc
@@ -0,0 +1,539 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+#include "cls/rgw/cls_rgw_client.h"
+
+#include "gtest/gtest.h"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+
+using namespace librados;
+
+librados::Rados rados;
+librados::IoCtx ioctx;
+string pool_name;
+
+
+/* must be the first test! */
+TEST(cls_rgw, init)
+{
+  pool_name = get_temp_pool_name();
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+}
+
+
+string str_int(string s, int i)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "-%d", i);
+  s.append(buf);
+
+  return s;
+}
+
+
+class OpMgr {
+  vector ops;
+
+public:
+  OpMgr() {}
+  ~OpMgr() {
+    vector::iterator iter;
+    for (iter = ops.begin(); iter != ops.end(); ++iter) {
+      ObjectOperation *op = *iter;
+      delete op;
+    }
+  }
+
+  ObjectReadOperation *read_op() {
+    ObjectReadOperation *op = new ObjectReadOperation;
+    ops.push_back(op);
+    return op;
+  }
+
+  ObjectWriteOperation *write_op() {
+    ObjectWriteOperation *op = new ObjectWriteOperation;
+    ops.push_back(op);
+    return op;
+  }
+};
+
+void test_stats(librados::IoCtx& ioctx, string& oid, int category, uint64_t num_entries, uint64_t total_size)
+{
+  rgw_bucket_dir_header header;
+  ASSERT_EQ(0, cls_rgw_get_dir_header(ioctx, oid, &header));
+
+  rgw_bucket_category_stats& stats = header.stats[category];
+  ASSERT_EQ(total_size, stats.total_size);
+  ASSERT_EQ(num_entries, stats.num_entries);
+}
+
+void index_prepare(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag, string& obj, string& loc)
+{
+  ObjectWriteOperation *op = mgr.write_op();
+  cls_rgw_bucket_prepare_op(*op, index_op, tag, obj, loc, true);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+}
+
+void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag, int epoch, string& obj, rgw_bucket_dir_entry_meta& meta)
+{
+  ObjectWriteOperation *op = mgr.write_op();
+  rgw_bucket_entry_ver ver;
+  ver.pool = ioctx.get_id();
+  ver.epoch = epoch;
+  cls_rgw_bucket_complete_op(*op, index_op, tag, ver, obj, meta, NULL, true);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+}
+
+TEST(cls_rgw, index_basic)
+{
+  string bucket_oid = str_int("bucket", 0);
+
+  OpMgr mgr;
+
+  ObjectWriteOperation *op = mgr.write_op();
+  cls_rgw_bucket_init(*op);
+  ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
+
+  uint64_t epoch = 1;
+
+  uint64_t obj_size = 1024;
+
+#define NUM_OBJS 10
+  for (int i = 0; i < NUM_OBJS; i++) {
+    string obj = str_int("obj", i);
+    string tag = str_int("tag", i);
+    string loc = str_int("loc", i);
+
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+    test_stats(ioctx, bucket_oid, 0, i, obj_size * i);
+
+    op = mgr.write_op();
+    rgw_bucket_dir_entry_meta meta;
+    meta.category = 0;
+    meta.size = obj_size;
+    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta);
+  }
+
+  test_stats(ioctx, bucket_oid, 0, NUM_OBJS, obj_size * NUM_OBJS);
+}
+
+TEST(cls_rgw, index_multiple_obj_writers)
+{
+  string bucket_oid = str_int("bucket", 1);
+
+  OpMgr mgr;
+
+  ObjectWriteOperation *op = mgr.write_op();
+  cls_rgw_bucket_init(*op);
+  ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
+
+  uint64_t obj_size = 1024;
+
+  string obj = str_int("obj", 0);
+  string loc = str_int("loc", 0);
+  /* multi prepare on a single object */
+  for (int i = 0; i < NUM_OBJS; i++) {
+    string tag = str_int("tag", i);
+
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+    test_stats(ioctx, bucket_oid, 0, 0, 0);
+  }
+
+  for (int i = NUM_OBJS; i > 0; i--) {
+    string tag = str_int("tag", i - 1);
+
+    rgw_bucket_dir_entry_meta meta;
+    meta.category = 0;
+    meta.size = obj_size * i;
+
+    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, i, obj, meta);
+
+    /* verify that object size doesn't change, as we went back with epoch */
+    test_stats(ioctx, bucket_oid, 0, 1, obj_size * NUM_OBJS);
+  }
+}
+
+TEST(cls_rgw, index_remove_object)
+{
+  string bucket_oid = str_int("bucket", 2);
+
+  OpMgr mgr;
+
+  ObjectWriteOperation *op = mgr.write_op();
+  cls_rgw_bucket_init(*op);
+  ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
+
+  uint64_t obj_size = 1024;
+  uint64_t total_size = 0;
+
+  int epoch = 0;
+
+  /* prepare multiple objects */
+  for (int i = 0; i < NUM_OBJS; i++) {
+    string obj = str_int("obj", i);
+    string tag = str_int("tag", i);
+    string loc = str_int("loc", i);
+
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+    test_stats(ioctx, bucket_oid, 0, i, total_size);
+
+    rgw_bucket_dir_entry_meta meta;
+    meta.category = 0;
+    meta.size = i * obj_size;
+    total_size += i * obj_size;
+
+    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta);
+
+    test_stats(ioctx, bucket_oid, 0, i + 1, total_size);
+  }
+
+  int i = NUM_OBJS / 2;
+  string tag_remove = "tag-rm";
+  string tag_modify = "tag-mod";
+  string obj = str_int("obj", i);
+  string loc = str_int("loc", i);
+
+  /* prepare both removal and modification on the same object */
+  index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc);
+  index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, obj, loc);
+
+  test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size);
+
+  rgw_bucket_dir_entry_meta meta;
+
+  /* complete object removal */
+  index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta);
+
+  /* verify stats correct */
+  total_size -= i * obj_size;
+  test_stats(ioctx, bucket_oid, 0, NUM_OBJS - 1, total_size);
+
+  meta.size = 512;
+  meta.category = 0;
+
+  /* complete object modification */
+  index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta);
+
+  /* verify stats correct */
+  total_size += meta.size;
+  test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size);
+
+
+  /* prepare both removal and modification on the same object, this time we'll
+   * first complete modification then remove*/
+  index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc);
+  index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_modify, obj, loc);
+
+  /* complete modification */
+  total_size -= meta.size;
+  meta.size = i * obj_size * 2;
+  meta.category = 0;
+
+  /* complete object modification */
+  index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta);
+
+  /* verify stats correct */
+  total_size += meta.size;
+  test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size);
+
+  /* complete object removal */
+  index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta);
+
+  /* verify stats correct */
+  total_size -= meta.size;
+  test_stats(ioctx, bucket_oid, 0, NUM_OBJS - 1, total_size);
+}
+
+TEST(cls_rgw, index_suggest)
+{
+  string bucket_oid = str_int("bucket", 3);
+
+  OpMgr mgr;
+
+  ObjectWriteOperation *op = mgr.write_op();
+  cls_rgw_bucket_init(*op);
+  ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
+
+  uint64_t total_size = 0;
+
+  int epoch = 0;
+
+  int num_objs = 100;
+
+  uint64_t obj_size = 1024;
+
+  /* create multiple objects */
+  for (int i = 0; i < num_objs; i++) {
+    string obj = str_int("obj", i);
+    string tag = str_int("tag", i);
+    string loc = str_int("loc", i);
+
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+    test_stats(ioctx, bucket_oid, 0, i, total_size);
+
+    rgw_bucket_dir_entry_meta meta;
+    meta.category = 0;
+    meta.size = obj_size;
+    total_size += meta.size;
+
+    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta);
+
+    test_stats(ioctx, bucket_oid, 0, i + 1, total_size);
+  }
+
+  /* prepare (without completion) some of the objects */
+  for (int i = 0; i < num_objs; i += 2) {
+    string obj = str_int("obj", i);
+    string tag = str_int("tag-prepare", i);
+    string loc = str_int("loc", i);
+
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+    test_stats(ioctx, bucket_oid, 0, num_objs, total_size);
+  }
+
+  int actual_num_objs = num_objs;
+  /* remove half of the objects */
+  for (int i = num_objs / 2; i < num_objs; i++) {
+    string obj = str_int("obj", i);
+    string tag = str_int("tag-rm", i);
+    string loc = str_int("loc", i);
+
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+    test_stats(ioctx, bucket_oid, 0, actual_num_objs, total_size);
+
+    rgw_bucket_dir_entry_meta meta;
+    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag, ++epoch, obj, meta);
+
+    total_size -= obj_size;
+    actual_num_objs--;
+    test_stats(ioctx, bucket_oid, 0, actual_num_objs, total_size);
+  }
+
+  bufferlist updates;
+
+  for (int i = 0; i < num_objs; i += 2) { 
+    string obj = str_int("obj", i);
+    string tag = str_int("tag-rm", i);
+    string loc = str_int("loc", i);
+
+    rgw_bucket_dir_entry dirent;
+    dirent.name = obj;
+    dirent.locator = loc;
+    dirent.exists = (i < num_objs / 2); // we removed half the objects
+    dirent.meta.size = 1024;
+
+    char suggest_op = (i < num_objs / 2 ? CEPH_RGW_UPDATE : CEPH_RGW_REMOVE);
+    cls_rgw_encode_suggestion(suggest_op, dirent, updates);
+  }
+
+  op = mgr.write_op();
+  cls_rgw_bucket_set_tag_timeout(*op, 1); // short tag timeout
+  ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
+
+  sleep(1);
+
+  /* suggest changes! */
+  op = mgr.write_op();
+  cls_rgw_suggest_changes(*op, updates);
+  ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
+
+  /* suggest changes twice! */
+  op = mgr.write_op();
+  cls_rgw_suggest_changes(*op, updates);
+  ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
+
+  test_stats(ioctx, bucket_oid, 0, num_objs / 2, total_size);
+}
+
+/* test garbage collection */
+static void create_obj(cls_rgw_obj& obj, int i, int j)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), "-%d.%d", i, j);
+  obj.pool = "pool";
+  obj.pool.append(buf);
+  obj.oid = "oid";
+  obj.oid.append(buf);
+  obj.key = "key";
+  obj.key.append(buf);
+}
+
+static bool cmp_objs(cls_rgw_obj& obj1, cls_rgw_obj& obj2)
+{
+  return (obj1.pool == obj2.pool) &&
+         (obj1.oid == obj2.oid) &&
+         (obj1.key == obj2.key);
+}
+
+
+TEST(cls_rgw, gc_set)
+{
+  /* add chains */
+  string oid = "obj";
+  for (int i = 0; i < 10; i++) {
+    char buf[32];
+    snprintf(buf, sizeof(buf), "chain-%d", i);
+    string tag = buf;
+    librados::ObjectWriteOperation op;
+    cls_rgw_gc_obj_info info;
+
+    cls_rgw_obj obj1, obj2;
+    create_obj(obj1, i, 1);
+    create_obj(obj2, i, 2);
+    info.chain.objs.push_back(obj1);
+    info.chain.objs.push_back(obj2);
+
+    op.create(false); // create object
+
+    info.tag = tag;
+    cls_rgw_gc_set_entry(op, 0, info);
+
+    ASSERT_EQ(0, ioctx.operate(oid, &op));
+  }
+
+  bool truncated;
+  list entries;
+  string marker;
+
+  /* list chains, verify truncated */
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated));
+  ASSERT_EQ(8, (int)entries.size());
+  ASSERT_EQ(1, truncated);
+
+  entries.clear();
+
+  /* list all chains, verify not truncated */
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, true, entries, &truncated));
+  ASSERT_EQ(10, (int)entries.size());
+  ASSERT_EQ(0, truncated);
+ 
+  /* verify all chains are valid */
+  list::iterator iter = entries.begin();
+  for (int i = 0; i < 10; i++, ++iter) {
+    cls_rgw_gc_obj_info& entry = *iter;
+
+    /* create expected chain name */
+    char buf[32];
+    snprintf(buf, sizeof(buf), "chain-%d", i);
+    string tag = buf;
+
+    /* verify chain name as expected */
+    ASSERT_EQ(entry.tag, tag);
+
+    /* verify expected num of objects in chain */
+    ASSERT_EQ(2, (int)entry.chain.objs.size());
+
+    list::iterator oiter = entry.chain.objs.begin();
+    cls_rgw_obj obj1, obj2;
+
+    /* create expected objects */
+    create_obj(obj1, i, 1);
+    create_obj(obj2, i, 2);
+
+    /* assign returned object names */
+    cls_rgw_obj& ret_obj1 = *oiter++;
+    cls_rgw_obj& ret_obj2 = *oiter;
+
+    /* verify objects are as expected */
+    ASSERT_EQ(1, (int)cmp_objs(obj1, ret_obj1));
+    ASSERT_EQ(1, (int)cmp_objs(obj2, ret_obj2));
+  }
+}
+
+TEST(cls_rgw, gc_defer)
+{
+  librados::IoCtx ioctx;
+  librados::Rados rados;
+
+  string gc_pool_name = get_temp_pool_name();
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(gc_pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(gc_pool_name.c_str(), ioctx));
+
+  string oid = "obj";
+  string tag = "mychain";
+
+  librados::ObjectWriteOperation op;
+  cls_rgw_gc_obj_info info;
+
+  op.create(false);
+
+  info.tag = tag;
+
+  /* create chain */
+  cls_rgw_gc_set_entry(op, 0, info);
+
+  ASSERT_EQ(0, ioctx.operate(oid, &op));
+
+  bool truncated;
+  list entries;
+  string marker;
+
+  /* list chains, verify num entries as expected */
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
+  ASSERT_EQ(1, (int)entries.size());
+  ASSERT_EQ(0, truncated);
+
+  librados::ObjectWriteOperation op2;
+
+  /* defer chain */
+  cls_rgw_gc_defer_entry(op2, 5, tag);
+  ASSERT_EQ(0, ioctx.operate(oid, &op2));
+
+  entries.clear();
+
+  /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
+  ASSERT_EQ(0, (int)entries.size());
+  ASSERT_EQ(0, truncated);
+
+  /* wait enough */
+  sleep(5);
+
+  /* verify list shows deferred entry */
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
+  ASSERT_EQ(1, (int)entries.size());
+  ASSERT_EQ(0, truncated);
+
+  librados::ObjectWriteOperation op3;
+  list tags;
+  tags.push_back(tag);
+
+  /* remove chain */
+  cls_rgw_gc_remove(op3, tags);
+  ASSERT_EQ(0, ioctx.operate(oid, &op3));
+
+  entries.clear();
+
+  /* verify entry was removed */
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
+  ASSERT_EQ(0, (int)entries.size());
+  ASSERT_EQ(0, truncated);
+
+  /* remove pool */
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(gc_pool_name, rados));
+}
+
+
+/* must be last test! */
+
+TEST(cls_rgw, finalize)
+{
+  /* remove pool */
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
diff --git a/ceph/src/test/cls_statelog/test_cls_statelog.cc b/ceph/src/test/cls_statelog/test_cls_statelog.cc
new file mode 100644
index 00000000..accab956
--- /dev/null
+++ b/ceph/src/test/cls_statelog/test_cls_statelog.cc
@@ -0,0 +1,211 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+#include "cls/statelog/cls_statelog_types.h"
+#include "cls/statelog/cls_statelog_client.h"
+
+#include "include/utime.h"
+#include "common/Clock.h"
+#include "global/global_context.h"
+
+#include "gtest/gtest.h"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+
+static librados::ObjectWriteOperation *new_op() {
+  return new librados::ObjectWriteOperation();
+}
+
+static librados::ObjectReadOperation *new_rop() {
+  return new librados::ObjectReadOperation();
+}
+
+static void reset_op(librados::ObjectWriteOperation **pop) {
+  delete *pop;
+  *pop = new_op();
+}
+static void reset_rop(librados::ObjectReadOperation **pop) {
+  delete *pop;
+  *pop = new_rop();
+}
+
+void add_log(librados::ObjectWriteOperation *op, const string& client_id, const string& op_id, string& obj, uint32_t state)
+{
+  bufferlist bl;
+  ::encode(state, bl);
+
+  utime_t ts = ceph_clock_now(g_ceph_context);
+
+  cls_statelog_add(*op, client_id, op_id, obj, ts, state, bl);
+}
+
+void next_op_id(string& op_id, int *id)
+{
+  char buf[16];
+  snprintf(buf, sizeof(buf), "%d", *id);
+  op_id = buf;
+  (*id)++;
+}
+
+static string get_obj_name(int num)
+{
+  char buf[16];
+  snprintf(buf, sizeof(buf), "obj-%d", num);
+  return string(buf);
+}
+
+static void get_entries_by_object(librados::IoCtx& ioctx, string& oid,
+                                  list& entries, string& object, string& op_id, int expected)
+{
+  /* search everything */
+  string empty_str, marker;
+
+  librados::ObjectReadOperation *rop = new_rop();
+  bufferlist obl;
+  bool truncated;
+  cls_statelog_list(*rop, empty_str, op_id, object, marker, 0, entries, &marker, &truncated);
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+  ASSERT_EQ(expected, (int)entries.size());
+  delete rop;
+}
+
+static void get_entries_by_client_id(librados::IoCtx& ioctx, string& oid,
+                                     list& entries, string& client_id, string& op_id, int expected)
+{
+  /* search everything */
+  string empty_str, marker;
+
+  librados::ObjectReadOperation *rop = new_rop();
+  bufferlist obl;
+  bool truncated;
+  cls_statelog_list(*rop, client_id, op_id, empty_str, marker, 0, entries, &marker, &truncated);
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+  ASSERT_EQ(expected, (int)entries.size());
+  delete rop;
+}
+
+static void get_all_entries(librados::IoCtx& ioctx, string& oid, list& entries, int expected)
+{
+  /* search everything */
+  string object, op_id;
+  get_entries_by_object(ioctx, oid, entries, object, op_id, expected);
+}
+
+TEST(cls_rgw, test_statelog_basic)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  string oid = "obj";
+
+  /* create object */
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  int id = 0;
+  string client_id[] = { "client-1", "client-2" };
+
+  const int num_ops = 10;
+  string op_ids[num_ops];
+
+  librados::ObjectWriteOperation *op = new_op();
+
+  for (int i = 0; i < num_ops; i++) {
+    next_op_id(op_ids[i], &id);
+    string obj = get_obj_name(i / 2);
+    string cid = client_id[i / (num_ops / 2)];
+    add_log(op, cid, op_ids[i], obj, i /* just for testing */); 
+  }
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  librados::ObjectReadOperation *rop = new_rop();
+
+  list entries;
+  bool truncated;
+
+  /* check list by client_id */
+
+  int total_count = 0;
+  for (int j = 0; j < 2; j++) {
+    string marker;
+    string obj;
+    string cid = client_id[j];
+    string op_id;
+
+    bufferlist obl;
+
+    cls_statelog_list(*rop, cid, op_id, obj, marker, 1, entries, &marker, &truncated);
+    ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+    ASSERT_EQ(1, (int)entries.size());
+
+    reset_rop(&rop);
+    marker.clear();
+    cls_statelog_list(*rop, cid, op_id, obj, marker, 0, entries, &marker, &truncated);
+    obl.clear();
+    ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+
+    ASSERT_EQ(5, (int)entries.size());
+    ASSERT_EQ(0, (int)truncated);
+
+    map emap;
+    for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
+      ASSERT_EQ(cid, iter->client_id);
+      emap[iter->op_id] = iter->object;
+    }
+    ASSERT_EQ(5, (int)emap.size());
+    /* verify correct object */
+    for (int i = 0; i < num_ops / 2; i++, total_count++) {
+      string ret_obj = emap[op_ids[total_count]];
+      string obj = get_obj_name(total_count / 2);
+      ASSERT_EQ(0, ret_obj.compare(obj));
+    }
+  }
+
+  entries.clear();
+  /* now search by object */
+  total_count = 0;
+  for (int i = 0; i < num_ops; i++) {
+    string marker;
+    string obj = get_obj_name(i / 2);
+    string cid;
+    string op_id;
+    bufferlist obl;
+
+    reset_rop(&rop);
+    cls_statelog_list(*rop, cid, op_id, obj, marker, 0, entries, &marker, &truncated);
+    ASSERT_EQ(0, ioctx.operate(oid, rop, &obl));
+    ASSERT_EQ(2, (int)entries.size());
+  }
+
+  /* search everything */
+
+  get_all_entries(ioctx, oid, entries, 10);
+
+  /* now remove an entry */
+  cls_statelog_entry e = entries.front();
+  entries.pop_front();
+
+  reset_op(&op);
+  cls_statelog_remove_by_client(*op, e.client_id, e.op_id);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  get_all_entries(ioctx, oid, entries, 9);
+
+  get_entries_by_object(ioctx, oid, entries, e.object, e.op_id, 0);
+  get_entries_by_client_id(ioctx, oid, entries, e.client_id, e.op_id, 0);
+
+  string empty_str;
+  get_entries_by_client_id(ioctx, oid, entries, e.client_id, empty_str, 4);
+  get_entries_by_object(ioctx, oid, entries, e.object, empty_str, 1);
+  delete op;
+  delete rop;
+}
+
diff --git a/ceph/src/test/cls_version/test_cls_version.cc b/ceph/src/test/cls_version/test_cls_version.cc
new file mode 100644
index 00000000..4c2d5950
--- /dev/null
+++ b/ceph/src/test/cls_version/test_cls_version.cc
@@ -0,0 +1,318 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+#include "cls/version/cls_version_types.h"
+#include "cls/version/cls_version_client.h"
+
+#include "gtest/gtest.h"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+
+static librados::ObjectWriteOperation *new_op() {
+  return new librados::ObjectWriteOperation();
+}
+
+static librados::ObjectReadOperation *new_rop() {
+  return new librados::ObjectReadOperation();
+}
+
+TEST(cls_rgw, test_version_inc_read)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+
+  /* create object */
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  obj_version ver;
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver));
+  ASSERT_EQ(0, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver.tag.size());
+  
+
+  /* inc version */
+  librados::ObjectWriteOperation *op = new_op();
+  cls_version_inc(*op);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver));
+  ASSERT_GT((long long)ver.ver, 0);
+  ASSERT_NE(0, (int)ver.tag.size());
+
+  /* inc version again! */
+  delete op;
+  op = new_op();
+  cls_version_inc(*op);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  obj_version ver2;
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver2));
+  ASSERT_GT((long long)ver2.ver, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver2.tag.compare(ver.tag));
+
+  delete op;
+
+  obj_version ver3;
+
+  librados::ObjectReadOperation *rop = new_rop();
+  cls_version_read(*rop, &ver3);
+  bufferlist outbl;
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &outbl));
+  ASSERT_EQ(ver2.ver, ver3.ver);
+  ASSERT_EQ(1, (long long)ver2.compare(&ver3));
+
+  delete rop;
+}
+
+
+TEST(cls_rgw, test_version_set)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+
+  /* create object */
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  obj_version ver;
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver));
+  ASSERT_EQ(0, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver.tag.size());
+
+
+  ver.ver = 123;
+  ver.tag = "foo";
+
+  /* set version */
+  librados::ObjectWriteOperation *op = new_op();
+  cls_version_set(*op, ver);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  /* read version */
+  obj_version ver2;
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver2));
+  ASSERT_EQ((long long)ver2.ver, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver2.tag.compare(ver.tag));
+
+  delete op;
+}
+
+TEST(cls_rgw, test_version_inc_cond)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+  /* create object */
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  obj_version ver;
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver));
+  ASSERT_EQ(0, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver.tag.size());
+  
+  /* inc version */
+  librados::ObjectWriteOperation *op = new_op();
+  cls_version_inc(*op);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver));
+  ASSERT_GT((long long)ver.ver, 0);
+  ASSERT_NE(0, (int)ver.tag.size());
+
+  obj_version cond_ver = ver;
+
+
+  /* inc version again! */
+  delete op;
+  op = new_op();
+  cls_version_inc(*op);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  obj_version ver2;
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver2));
+  ASSERT_GT((long long)ver2.ver, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver2.tag.compare(ver.tag));
+
+
+  /* now check various condition tests */
+  cls_version_inc(*op, cond_ver, VER_COND_NONE);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver2));
+  ASSERT_GT((long long)ver2.ver, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver2.tag.compare(ver.tag));
+
+  /* a bunch of conditions that should fail */
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, cond_ver, VER_COND_EQ);
+  ASSERT_EQ(-ECANCELED, ioctx.operate(oid, op));
+
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, cond_ver, VER_COND_LT);
+  ASSERT_EQ(-ECANCELED, ioctx.operate(oid, op));
+
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, cond_ver, VER_COND_LE);
+  ASSERT_EQ(-ECANCELED, ioctx.operate(oid, op));
+
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, cond_ver, VER_COND_TAG_NE);
+  ASSERT_EQ(-ECANCELED, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver2));
+  ASSERT_GT((long long)ver2.ver, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver2.tag.compare(ver.tag));
+
+  /* a bunch of conditions that should succeed */
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, ver2, VER_COND_EQ);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, cond_ver, VER_COND_GT);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, cond_ver, VER_COND_GE);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  delete op;
+  op = new_op();
+  cls_version_inc(*op, cond_ver, VER_COND_TAG_EQ);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  delete op;
+}
+
+TEST(cls_rgw, test_version_inc_check)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  /* create pool */
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  /* add chains */
+  string oid = "obj";
+
+
+  /* create object */
+
+  ASSERT_EQ(0, ioctx.create(oid, true));
+
+  obj_version ver;
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver));
+  ASSERT_EQ(0, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver.tag.size());
+  
+  /* inc version */
+  librados::ObjectWriteOperation *op = new_op();
+  cls_version_inc(*op);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver));
+  ASSERT_GT((long long)ver.ver, 0);
+  ASSERT_NE(0, (int)ver.tag.size());
+
+  obj_version cond_ver = ver;
+
+  /* a bunch of conditions that should succeed */
+  librados::ObjectReadOperation *rop = new_rop();
+  cls_version_check(*rop, cond_ver, VER_COND_EQ);
+  bufferlist bl;
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &bl));
+
+  delete rop;
+  rop = new_rop();
+  cls_version_check(*rop, cond_ver, VER_COND_GE);
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &bl));
+
+  delete rop;
+  rop = new_rop();
+  cls_version_check(*rop, cond_ver, VER_COND_LE);
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &bl));
+
+  delete rop;
+  rop = new_rop();
+  cls_version_check(*rop, cond_ver, VER_COND_TAG_EQ);
+  ASSERT_EQ(0, ioctx.operate(oid, rop, &bl));
+
+  obj_version ver2;
+
+  delete op;
+  op = new_op();
+  cls_version_inc(*op);
+  ASSERT_EQ(0, ioctx.operate(oid, op));
+
+  ASSERT_EQ(0, cls_version_read(ioctx, oid, &ver2));
+  ASSERT_GT((long long)ver2.ver, (long long)ver.ver);
+  ASSERT_EQ(0, (int)ver2.tag.compare(ver.tag));
+
+  delete op;
+
+  /* a bunch of conditions that should fail */
+  delete rop;
+  rop = new_rop();
+  cls_version_check(*rop, ver, VER_COND_LT);
+  ASSERT_EQ(-ECANCELED, ioctx.operate(oid, rop, &bl));
+
+  delete rop;
+  rop = new_rop();
+  cls_version_check(*rop, cond_ver, VER_COND_LE);
+  ASSERT_EQ(-ECANCELED, ioctx.operate(oid, rop, &bl));
+
+  delete rop;
+  rop = new_rop();
+  cls_version_check(*rop, cond_ver, VER_COND_TAG_NE);
+  ASSERT_EQ(-ECANCELED, ioctx.operate(oid, rop, &bl));
+
+  delete rop;
+}
diff --git a/ceph/src/test/common/ObjectContents.cc b/ceph/src/test/common/ObjectContents.cc
new file mode 100644
index 00000000..3b7e1f3e
--- /dev/null
+++ b/ceph/src/test/common/ObjectContents.cc
@@ -0,0 +1,128 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "ObjectContents.h"
+#include "include/buffer.h"
+#include 
+#include 
+
+bool test_object_contents()
+{
+  ObjectContents c, d;
+  assert(!c.exists());
+  c.debug(std::cerr);
+  c.write(10, 10, 10);
+  assert(c.exists());
+  assert(c.size() == 20);
+
+  c.debug(std::cerr);
+  bufferlist bl;
+  for (ObjectContents::Iterator iter = c.get_iterator();
+       iter.valid();
+       ++iter) {
+    bl.append(*iter);
+  }
+  assert(bl.length() == 20);
+
+  bufferlist bl2;
+  for (unsigned i = 0; i < 8; ++i) bl2.append(bl[i]);
+  c.write(10, 8, 4);
+  c.debug(std::cerr);
+  ObjectContents::Iterator iter = c.get_iterator();
+  iter.seek_to(8);
+  for (uint64_t i = 8;
+       i < 12;
+       ++i, ++iter) {
+    bl2.append(*iter);
+  }
+  for (unsigned i = 12; i < 20; ++i) bl2.append(bl[i]);
+  assert(bl2.length() == 20);
+
+  for (ObjectContents::Iterator iter3 = c.get_iterator();
+       iter.valid();
+       ++iter) {
+    assert(bl2[iter3.get_pos()] == *iter3);
+  }
+
+  assert(bl2[0] == '\0');
+  assert(bl2[7] == '\0');
+
+  interval_set to_clone;
+  to_clone.insert(5, 10);
+  d.clone_range(c, to_clone);
+  assert(d.size() == 15);
+
+  c.debug(std::cerr);
+  d.debug(std::cerr);
+
+  ObjectContents::Iterator iter2 = d.get_iterator();
+  iter2.seek_to(5);
+  for (uint64_t i = 5; i < 15; ++i, ++iter2) {
+    std::cerr << "i is " << i << std::endl;
+    assert(iter2.get_pos() == i);
+    assert(*iter2 == bl2[i]);
+  }
+  return true;
+}
+
+
+unsigned int ObjectContents::Iterator::get_state(uint64_t _pos)
+{
+  if (parent->seeds.count(_pos)) {
+    return parent->seeds[_pos];
+  }
+  seek_to(_pos - 1);
+  return current_state;
+}
+
+void ObjectContents::clone_range(ObjectContents &other,
+				 interval_set &intervals)
+{
+  interval_set written_to_clone;
+  written_to_clone.intersection_of(intervals, other.written);
+
+  interval_set zeroed = intervals;
+  zeroed.subtract(written_to_clone);
+
+  written.union_of(intervals);
+  written.subtract(zeroed);
+
+  for (interval_set::iterator i = written_to_clone.begin();
+       i != written_to_clone.end();
+       ++i) {
+    uint64_t start = i.get_start();
+    uint64_t len = i.get_len();
+
+    unsigned int seed = get_iterator().get_state(start+len);
+
+    seeds[start+len] = seed;
+    seeds.erase(seeds.lower_bound(start), seeds.lower_bound(start+len));
+
+    seeds[start] = other.get_iterator().get_state(start);
+    seeds.insert(other.seeds.upper_bound(start),
+		 other.seeds.lower_bound(start+len));
+  }
+
+  if (intervals.range_end() > _size)
+    _size = intervals.range_end();
+  _exists = true;
+  return;
+}
+
+void ObjectContents::write(unsigned int seed,
+			   uint64_t start,
+			   uint64_t len)
+{
+  _exists = true;
+  unsigned int _seed = get_iterator().get_state(start+len);
+  seeds[start+len] = _seed;
+  seeds.erase(seeds.lower_bound(start),
+	      seeds.lower_bound(start+len));
+  seeds[start] = seed;
+
+  interval_set to_write;
+  to_write.insert(start, len);
+  written.union_of(to_write);
+
+  if (start + len > _size)
+    _size = start + len;
+  return;
+}
diff --git a/ceph/src/test/common/ObjectContents.h b/ceph/src/test/common/ObjectContents.h
new file mode 100644
index 00000000..8ca410bd
--- /dev/null
+++ b/ceph/src/test/common/ObjectContents.h
@@ -0,0 +1,121 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "include/interval_set.h"
+#include "include/buffer.h"
+#include 
+
+#ifndef COMMON_OBJECT_H
+#define COMMON_OBJECT_H
+
+enum {
+  RANDOMWRITEFULL,
+  DELETED,
+  CLONERANGE
+};
+
+bool test_object_contents();
+
+class ObjectContents {
+  uint64_t _size;
+  map seeds;
+  interval_set written;
+  bool _exists;
+public:
+  class Iterator {
+    ObjectContents *parent;
+    map::iterator iter;
+    unsigned int current_state;
+    int current_val;
+    uint64_t pos;
+  private:
+    unsigned int get_state(uint64_t pos);
+  public:
+    Iterator(ObjectContents *parent) :
+      parent(parent), iter(parent->seeds.end()),
+      current_state(0), current_val(0), pos(-1) {
+      seek_to_first();
+    }
+    char operator*() {
+      return parent->written.contains(pos) ?
+	static_cast(current_val % 256) : '\0';
+    }
+    uint64_t get_pos() {
+      return pos;
+    }
+    void seek_to(uint64_t _pos) {
+      if (pos > _pos ||
+	  (iter != parent->seeds.end() && _pos >= iter->first)) {
+	iter = parent->seeds.upper_bound(_pos);
+	--iter;
+	current_state = iter->second;
+	current_val = rand_r(¤t_state);
+	pos = iter->first;
+	++iter;
+      }
+      while (pos < _pos) ++(*this);
+    }
+
+    void seek_to_first() {
+      seek_to(0);
+    }
+    Iterator &operator++() {
+      ++pos;
+      if (iter != parent->seeds.end() && pos >= iter->first) {
+	assert(pos == iter->first);
+	current_state = iter->second;
+	++iter;
+      }
+      current_val = rand_r(¤t_state);
+      return *this;
+    }
+    bool valid() {
+      return pos < parent->size();
+    }
+    friend class ObjectContents;
+  };
+
+  ObjectContents() : _size(0), _exists(false) {
+    seeds[0] = 0;
+  }
+
+  ObjectContents(bufferlist::iterator &bp) {
+    ::decode(_size, bp);
+    ::decode(seeds, bp);
+    ::decode(written, bp);
+    ::decode(_exists, bp);
+  }
+
+  void clone_range(ObjectContents &other,
+		   interval_set &intervals);
+  void write(unsigned int seed,
+	     uint64_t from,
+	     uint64_t len);
+  Iterator get_iterator() {
+    return Iterator(this);
+  }
+
+  uint64_t size() const { return _size; }
+
+  bool exists() { return _exists; }
+
+  void debug(std::ostream &out) {
+    out << "_size is " << _size << std::endl;
+    out << "seeds is: (";
+    for (map::iterator i = seeds.begin();
+	 i != seeds.end();
+	 ++i) {
+      out << "[" << i->first << "," << i->second << "], ";
+    }
+    out << ")" << std::endl;
+    out << "written is " << written << std::endl;
+    out << "_exists is " << _exists << std::endl;
+  }
+
+  void encode(bufferlist &bl) const {
+    ::encode(_size, bl);
+    ::encode(seeds, bl);
+    ::encode(written, bl);
+    ::encode(_exists, bl);
+  }
+};
+
+#endif
diff --git a/ceph/src/test/common/Throttle.cc b/ceph/src/test/common/Throttle.cc
new file mode 100644
index 00000000..0964cbef
--- /dev/null
+++ b/ceph/src/test/common/Throttle.cc
@@ -0,0 +1,261 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+#include "common/Mutex.h"
+#include "common/Thread.h"
+#include "common/Throttle.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include 
+
+class ThrottleTest : public ::testing::Test {
+protected:
+
+  class Thread_get : public Thread {
+  public:
+    Throttle &throttle;
+    int64_t count;
+    bool waited;
+
+    Thread_get(Throttle& _throttle, int64_t _count) : 
+      throttle(_throttle),
+      count(_count),
+      waited(false)
+    {
+    }
+    
+    virtual void *entry() {
+      waited = throttle.get(count);
+      throttle.put(count);
+      return NULL;
+    }
+  };
+
+};
+
+TEST_F(ThrottleTest, Throttle) {
+  ASSERT_THROW({
+      Throttle throttle(g_ceph_context, "throttle", -1);
+    }, FailedAssertion);
+
+  int64_t throttle_max = 10;
+  Throttle throttle(g_ceph_context, "throttle", throttle_max);
+  ASSERT_EQ(throttle.get_max(), throttle_max);
+  ASSERT_EQ(throttle.get_current(), 0);
+}
+
+TEST_F(ThrottleTest, take) {
+  int64_t throttle_max = 10;
+  Throttle throttle(g_ceph_context, "throttle", throttle_max);
+  ASSERT_THROW(throttle.take(-1), FailedAssertion);
+  ASSERT_EQ(throttle.take(throttle_max), throttle_max);
+  ASSERT_EQ(throttle.take(throttle_max), throttle_max * 2);
+}
+
+TEST_F(ThrottleTest, get) {
+  int64_t throttle_max = 10;
+  Throttle throttle(g_ceph_context, "throttle", throttle_max);
+  ASSERT_THROW(throttle.get(-1), FailedAssertion);
+  ASSERT_FALSE(throttle.get(5)); 
+  ASSERT_EQ(throttle.put(5), 0); 
+
+  ASSERT_FALSE(throttle.get(throttle_max)); 
+  ASSERT_FALSE(throttle.get_or_fail(1)); 
+  ASSERT_FALSE(throttle.get(1, throttle_max + 1)); 
+  ASSERT_EQ(throttle.put(throttle_max + 1), 0); 
+  ASSERT_FALSE(throttle.get(0, throttle_max)); 
+  ASSERT_FALSE(throttle.get(throttle_max)); 
+  ASSERT_FALSE(throttle.get_or_fail(1)); 
+  ASSERT_EQ(throttle.put(throttle_max), 0); 
+
+  useconds_t delay = 1;
+
+  bool waited;
+
+  do {
+    cout << "Trying (1) with delay " << delay << "us\n";
+
+    ASSERT_FALSE(throttle.get(throttle_max)); 
+    ASSERT_FALSE(throttle.get_or_fail(throttle_max));  
+
+    Thread_get t(throttle, 7);
+    t.create();
+    usleep(delay);
+    ASSERT_EQ(throttle.put(throttle_max), 0); 
+    t.join();
+
+    if (!(waited = t.waited))
+      delay *= 2;
+  } while(!waited);
+	  
+  do {
+    cout << "Trying (2) with delay " << delay << "us\n";
+
+    ASSERT_FALSE(throttle.get(throttle_max / 2));
+    ASSERT_FALSE(throttle.get_or_fail(throttle_max));  
+
+    Thread_get t(throttle, throttle_max);
+    t.create();
+    usleep(delay);
+
+    Thread_get u(throttle, 1);
+    u.create();
+    usleep(delay);
+
+    throttle.put(throttle_max / 2);
+
+    t.join();
+    u.join();
+
+    if (!(waited = t.waited && u.waited))
+      delay *= 2;
+  } while(!waited);
+	  
+}
+
+TEST_F(ThrottleTest, get_or_fail) {
+  {
+    Throttle throttle(g_ceph_context, "throttle");
+    
+    ASSERT_TRUE(throttle.get_or_fail(5));
+    ASSERT_TRUE(throttle.get_or_fail(5));
+  }
+
+  {
+    int64_t throttle_max = 10;
+    Throttle throttle(g_ceph_context, "throttle", throttle_max);
+
+    ASSERT_TRUE(throttle.get_or_fail(throttle_max));
+    ASSERT_EQ(throttle.put(throttle_max), 0);
+
+    ASSERT_TRUE(throttle.get_or_fail(throttle_max * 2));
+    ASSERT_FALSE(throttle.get_or_fail(1));
+    ASSERT_FALSE(throttle.get_or_fail(throttle_max * 2));
+    ASSERT_EQ(throttle.put(throttle_max * 2), 0);
+
+    ASSERT_TRUE(throttle.get_or_fail(throttle_max));  
+    ASSERT_FALSE(throttle.get_or_fail(1));  
+    ASSERT_EQ(throttle.put(throttle_max), 0);
+  }
+}
+
+TEST_F(ThrottleTest, wait) {
+  int64_t throttle_max = 10;
+  Throttle throttle(g_ceph_context, "throttle", throttle_max);
+
+  useconds_t delay = 1;
+
+  bool waited;
+
+  do {
+    cout << "Trying (3) with delay " << delay << "us\n";
+
+    ASSERT_FALSE(throttle.get(throttle_max / 2));
+    ASSERT_FALSE(throttle.get_or_fail(throttle_max));  
+
+    Thread_get t(throttle, throttle_max);
+    t.create();
+    usleep(delay);
+
+    //
+    // Throttle::_reset_max(int64_t m) used to contain a test
+    // that blocked the following statement, only if 
+    // the argument was greater than throttle_max. 
+    // Although a value lower than throttle_max would cover
+    // the same code in _reset_max, the throttle_max * 100
+    // value is left here to demonstrate that the problem
+    // has been solved.
+    //
+    throttle.wait(throttle_max * 100);
+    usleep(delay);
+    t.join();
+    ASSERT_EQ(throttle.get_current(), throttle_max / 2);
+
+    if (!(waited = t.waited)) {
+      delay *= 2;
+      // undo the changes we made
+      throttle.put(throttle_max / 2);
+      throttle.wait(throttle_max);
+    }
+  } while(!waited);
+}
+
+TEST_F(ThrottleTest, destructor) {
+  Thread_get *t;
+  {
+    int64_t throttle_max = 10;
+    Throttle *throttle = new Throttle(g_ceph_context, "throttle", throttle_max);
+
+    ASSERT_FALSE(throttle->get(5)); 
+
+    t = new Thread_get(*throttle, 7);
+    t->create();
+    bool blocked;
+    useconds_t delay = 1;
+    do {
+      usleep(delay);
+      if (throttle->get_or_fail(1)) {
+	throttle->put(1);
+	blocked = false;
+      } else {
+	blocked = true;
+      }
+      delay *= 2;
+    } while(!blocked);
+    delete throttle;
+  }
+  
+  { // 
+    // The thread is left hanging, otherwise it will abort().
+    // Deleting the Throttle on which it is waiting creates a
+    // inconsistency that will be detected: the Throttle object that
+    // it references no longer exists.
+    //
+    pthread_t id = t->get_thread_id();
+    ASSERT_EQ(pthread_kill(id, 0), 0);
+    delete t;
+    ASSERT_EQ(pthread_kill(id, 0), 0);
+  }
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; 
+ *   make unittest_throttle ; 
+ *   ./unittest_throttle # --gtest_filter=ThrottleTest.destructor \
+ *       --log-to-stderr=true --debug-filestore=20
+ * "
+ * End:
+ */
+
diff --git a/ceph/src/test/common/get_command_descriptions.cc b/ceph/src/test/common/get_command_descriptions.cc
new file mode 100644
index 00000000..c35e47ba
--- /dev/null
+++ b/ceph/src/test/common/get_command_descriptions.cc
@@ -0,0 +1,113 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+#include "mon/Monitor.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+
+static void usage(ostream &out)
+{
+  out << "usage: get_command_descriptions [options ...]" << std::endl;
+  out << "print on stdout the result of JSON formatted options\n";
+  out << "found in mon/MonCommands.h as produced by the\n";
+  out << "Monitor.cc::get_command_descriptions function.\n";
+  out << "Designed as a helper for ceph_argparse.py unit tests.\n";
+  out << "\n";
+  out << "  --all               all of mon/MonCommands.h \n";
+  out << "  --pull585           reproduce the bug fixed by #585\n";
+  out << "\n";
+  out << "Examples:\n";
+  out << "  get_command_descriptions --all\n";
+  out << "  get_command_descriptions --pull585\n";
+}
+
+static void json_print(const MonCommand *mon_commands, int size)
+{
+  bufferlist rdata;
+  Formatter *f = new_formatter("json");
+  Monitor::format_command_descriptions(mon_commands, size, f, &rdata);
+  delete f;
+  string data(rdata.c_str(), rdata.length());
+  cout << data << std::endl;
+}
+
+static void all()
+{
+#undef COMMAND
+  MonCommand mon_commands[] = {
+#define COMMAND(parsesig, helptext, modulename, req_perms, avail)	\
+    {parsesig, helptext, modulename, req_perms, avail},
+#include 
+  };
+
+  json_print(mon_commands, ARRAY_SIZE(mon_commands));
+}
+
+// syntax error https://github.com/ceph/ceph/pull/585
+static void pull585()
+{
+  MonCommand mon_commands[] = {
+    { "osd pool create "		       
+      "name=pool,type=CephPoolname " 
+      "name=pg_num,type=CephInt,range=0 " 
+      "name=pgp_num,type=CephInt,range=0,req=false" // !!! missing trailing space
+      "name=properties,type=CephString,n=N,req=false,goodchars=[A-Za-z0-9-_.=]", 
+      "create pool", "osd", "rw", "cli,rest" }
+  };
+
+  json_print(mon_commands, ARRAY_SIZE(mon_commands));
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  if (args.empty()) {
+    usage(cerr);
+    exit(1);
+  }
+  for (std::vector::iterator i = args.begin(); i != args.end(); ++i) {
+    string err;
+
+    if (*i == string("help") || *i == string("-h") || *i == string("--help")) {
+      usage(cout);
+      exit(0);
+    } else if (*i == string("--all")) {
+      all();
+    } else if (*i == string("--pull585")) {
+      pull585();
+    }
+  }  
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; 
+ *   make get_command_descriptions && 
+ *   ./get_command_descriptions --all --pull585"
+ * End:
+ */
+
diff --git a/ceph/src/test/common/histogram.cc b/ceph/src/test/common/histogram.cc
new file mode 100644
index 00000000..2fd3cfec
--- /dev/null
+++ b/ceph/src/test/common/histogram.cc
@@ -0,0 +1,126 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank 
+ *
+ * LGPL2.1 (see COPYING-LGPL2.1) or later
+ */
+
+#include 
+#include 
+
+#include "common/histogram.h"
+#include "include/stringify.h"
+
+TEST(Histogram, Basic) {
+  pow2_hist_t h;
+
+  h.add(0);
+  h.add(0);
+  h.add(0);
+  ASSERT_EQ(3, h.h[0]);
+  ASSERT_EQ(1u, h.h.size());
+
+  h.add(1);
+  ASSERT_EQ(3, h.h[0]);
+  ASSERT_EQ(1, h.h[1]);
+  ASSERT_EQ(2u, h.h.size());
+
+  h.add(2);
+  h.add(2);
+  ASSERT_EQ(3, h.h[0]);
+  ASSERT_EQ(1, h.h[1]);
+  ASSERT_EQ(2, h.h[2]);
+  ASSERT_EQ(3u, h.h.size());
+}
+
+TEST(Histogram, Set) {
+  pow2_hist_t h;
+  h.set_bin(0, 12);
+  h.set_bin(2, 12);
+  ASSERT_EQ(12, h.h[0]);
+  ASSERT_EQ(0, h.h[1]);
+  ASSERT_EQ(12, h.h[2]);
+  ASSERT_EQ(3u, h.h.size());
+}
+
+TEST(Histogram, Position) {
+  {
+    pow2_hist_t h;
+    uint64_t lb, ub;
+    h.add(0);
+    ASSERT_EQ(-1, h.get_position_micro(-20, &lb, &ub));
+  }
+  {
+    pow2_hist_t h;
+    h.add(0);
+    uint64_t lb, ub;
+    h.get_position_micro(0, &lb, &ub);
+    ASSERT_EQ(0u, lb);
+    ASSERT_EQ(1000000u, ub);
+    h.add(0);
+    h.add(0);
+    h.add(0);
+    h.get_position_micro(0, &lb, &ub);
+    ASSERT_EQ(0u, lb);
+    ASSERT_EQ(1000000u, ub);
+  }
+  {
+    pow2_hist_t h;
+    h.add(1);
+    h.add(1);
+    uint64_t lb, ub;
+    h.get_position_micro(0, &lb, &ub);
+    ASSERT_EQ(0u, lb);
+    ASSERT_EQ(0u, ub);
+    h.add(0);
+    h.get_position_micro(0, &lb, &ub);
+    ASSERT_EQ(0u, lb);
+    ASSERT_EQ(333333u, ub);
+    h.get_position_micro(1, &lb, &ub);
+    ASSERT_EQ(333333u, lb);
+    ASSERT_EQ(1000000u, ub);
+  }
+  {
+    pow2_hist_t h;
+    h.h.resize(10, 0);
+    h.h[0] = 1;
+    h.h[5] = 1;
+    uint64_t lb, ub;
+    h.get_position_micro(4, &lb, &ub);
+    ASSERT_EQ(500000u, lb);
+    ASSERT_EQ(500000u, ub);
+  }
+  {
+    pow2_hist_t h;
+    h.h.resize(10, 0);
+    h.h[0] = UINT_MAX;
+    h.h[5] = UINT_MAX;
+    uint64_t lb, ub;
+    ASSERT_EQ(500000u, lb);
+    ASSERT_EQ(500000u, ub);
+  }
+}
+
+TEST(Histogram, Decay) {
+  pow2_hist_t h;
+  h.set_bin(0, 123);
+  h.set_bin(3, 12);
+  h.set_bin(5, 1);
+  h.decay(1);
+  ASSERT_EQ(61, h.h[0]);
+  ASSERT_EQ(6, h.h[3]);
+  ASSERT_EQ(4u, h.h.size());
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ *   make unittest_histogram &&
+ *   valgrind --tool=memcheck --leak-check=full \
+ *      ./unittest_histogram
+ *   "
+ * End:
+ */
diff --git a/ceph/src/test/common/test_bloom_filter.cc b/ceph/src/test/common/test_bloom_filter.cc
new file mode 100644
index 00000000..cfd41305
--- /dev/null
+++ b/ceph/src/test/common/test_bloom_filter.cc
@@ -0,0 +1,289 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank 
+ *
+ * LGPL2.1 (see COPYING-LGPL2.1) or later
+ */
+
+#include 
+#include 
+
+#include "include/stringify.h"
+#include "common/bloom_filter.hpp"
+
+TEST(BloomFilter, Basic) {
+  bloom_filter bf(10, .1, 1);
+  bf.insert("foo");
+  bf.insert("bar");
+
+  ASSERT_TRUE(bf.contains("foo"));
+  ASSERT_TRUE(bf.contains("bar"));
+}
+
+TEST(BloomFilter, Empty) {
+  bloom_filter bf;
+  for (int i=0; i<100; ++i) {
+    ASSERT_FALSE(bf.contains(i));
+    ASSERT_FALSE(bf.contains(stringify(i)));
+  }
+}
+
+TEST(BloomFilter, Sweep) {
+  std::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
+  std::cout.precision(5);
+  std::cout << "# max\tfpp\tactual\tsize\tB/insert" << std::endl;
+  for (int ex = 3; ex < 12; ex += 2) {
+    for (float fpp = .001; fpp < .5; fpp *= 4.0) {
+      int max = 2 << ex;
+      bloom_filter bf(max, fpp, 1);
+      bf.insert("foo");
+      bf.insert("bar");
+
+      ASSERT_TRUE(bf.contains("foo"));
+      ASSERT_TRUE(bf.contains("bar"));
+
+      for (int n = 0; n < max; n++)
+	bf.insert("ok" + stringify(n));
+
+      int test = max * 100;
+      int hit = 0;
+      for (int n = 0; n < test; n++)
+	if (bf.contains("asdf" + stringify(n)))
+	  hit++;
+
+      ASSERT_TRUE(bf.contains("foo"));
+      ASSERT_TRUE(bf.contains("bar"));
+
+      double actual = (double)hit / (double)test;
+
+      bufferlist bl;
+      ::encode(bf, bl);
+
+      double byte_per_insert = (double)bl.length() / (double)max;
+
+      std::cout << max << "\t" << fpp << "\t" << actual << "\t" << bl.length() << "\t" << byte_per_insert << std::endl;
+      ASSERT_TRUE(actual < fpp * 10);
+
+    }
+  }
+}
+
+TEST(BloomFilter, SweepInt) {
+  std::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
+  std::cout.precision(5);
+  std::cout << "# max\tfpp\tactual\tsize\tB/insert\tdensity\tapprox_element_count" << std::endl;
+  for (int ex = 3; ex < 12; ex += 2) {
+    for (float fpp = .001; fpp < .5; fpp *= 4.0) {
+      int max = 2 << ex;
+      bloom_filter bf(max, fpp, 1);
+      bf.insert("foo");
+      bf.insert("bar");
+
+      ASSERT_TRUE(123);
+      ASSERT_TRUE(456);
+
+      for (int n = 0; n < max; n++)
+	bf.insert(n);
+
+      int test = max * 100;
+      int hit = 0;
+      for (int n = 0; n < test; n++)
+	if (bf.contains(100000 + n))
+	  hit++;
+
+      ASSERT_TRUE(123);
+      ASSERT_TRUE(456);
+
+      double actual = (double)hit / (double)test;
+
+      bufferlist bl;
+      ::encode(bf, bl);
+
+      double byte_per_insert = (double)bl.length() / (double)max;
+
+      std::cout << max << "\t" << fpp << "\t" << actual << "\t" << bl.length() << "\t" << byte_per_insert
+		<< "\t" << bf.density() << "\t" << bf.approx_unique_element_count() << std::endl;
+      ASSERT_TRUE(actual < fpp * 10);
+      ASSERT_TRUE(actual > fpp / 10);
+      ASSERT_TRUE(bf.density() > 0.40);
+      ASSERT_TRUE(bf.density() < 0.60);
+    }
+  }
+}
+
+
+TEST(BloomFilter, CompressibleSweep) {
+  std::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
+  std::cout.precision(5);
+  std::cout << "# max\tins\test ins\tafter\ttgtfpp\tactual\tsize\tb/elem\n";
+  float fpp = .01;
+  int max = 1024;
+  for (int div = 1; div < 10; div++) {
+    compressible_bloom_filter bf(max, fpp, 1);
+    int t = max/div;
+    for (int n = 0; n < t; n++)
+      bf.insert(n);
+
+    unsigned est = bf.approx_unique_element_count();
+    if (div > 1)
+      bf.compress(1.0 / div);
+
+    for (int n = 0; n < t; n++)
+      ASSERT_TRUE(bf.contains(n));
+
+    int test = max * 100;
+    int hit = 0;
+    for (int n = 0; n < test; n++)
+      if (bf.contains(100000 + n))
+	hit++;
+
+    double actual = (double)hit / (double)test;
+
+    bufferlist bl;
+    ::encode(bf, bl);
+
+    double byte_per_insert = (double)bl.length() / (double)max;
+    unsigned est_after = bf.approx_unique_element_count();
+    std::cout << max
+	      << "\t" << t
+	      << "\t" << est
+	      << "\t" << est_after
+	      << "\t" << fpp
+	      << "\t" << actual
+	      << "\t" << bl.length() << "\t" << byte_per_insert
+	      << std::endl;
+
+    ASSERT_TRUE(actual < fpp * 2.0);
+    ASSERT_TRUE(actual > fpp / 2.0);
+    ASSERT_TRUE(est_after < est * 2);
+    ASSERT_TRUE(est_after > est / 2);
+  }
+}
+
+
+
+TEST(BloomFilter, BinSweep) {
+  std::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
+  std::cout.precision(5);
+  int total_max = 16384;
+  float total_fpp = .01;
+  std::cout << "total_inserts " << total_max << " target-fpp " << total_fpp << std::endl;
+  for (int bins = 1; bins < 16; ++bins) {
+    int max = total_max / bins;
+    float fpp = total_fpp / bins;//pow(total_fpp, bins);
+
+    std::vector ls;
+    bufferlist bl;
+    for (int i=0; iinsert(10000 * (i+1) + j);
+      }
+      ::encode(*ls.front(), bl);
+    }
+
+    int hit = 0;
+    int test = max * 100;
+    for (int i=0; i::iterator j = ls.begin(); j != ls.end(); ++j) {
+	if ((*j)->contains(i * 732)) {  // note: sequential i does not work here; the intenral int hash is weak!!
+	  hit++;
+	  break;
+	}
+      }
+    }
+
+    double actual = (double)hit / (double)test;
+    std::cout << "bins " << bins << " bin-max " << max << " bin-fpp " << fpp
+	      << " actual-fpp " << actual
+	      << " total-size " << bl.length() << std::endl;
+  }
+}
+
+// disable these tests; doing dual insertions in consecutive filters
+// appears to be equivalent to doing a single insertion in a bloom
+// filter that is twice as big.
+#if 0
+
+// test the fpp over a sequence of bloom filters, each with unique
+// items inserted into it.
+//
+// we expect:  actual_fpp = num_filters * per_filter_fpp
+TEST(BloomFilter, Sequence) {
+
+  int max = 1024;
+  double fpp = .01;
+  for (int seq = 2; seq <= 128; seq *= 2) {
+    std::vector ls;
+    for (int i=0; iinsert("ok" + stringify(j) + "_" + stringify(i));
+	if (ls.size() > 1)
+	  ls[ls.size() - 2]->insert("ok" + stringify(j) + "_" + stringify(i));
+      }
+    }
+
+    int hit = 0;
+    int test = max * 100;
+    for (int i=0; i::iterator j = ls.begin(); j != ls.end(); ++j) {
+	if ((*j)->contains("bad" + stringify(i))) {
+	  hit++;
+	  break;
+	}
+      }
+    }
+
+    double actual = (double)hit / (double)test;
+    std::cout << "seq " << seq << " max " << max << " fpp " << fpp << " actual " << actual << std::endl;
+  }
+}
+
+// test the ffp over a sequence of bloom filters, where actual values
+// are always inserted into a consecutive pair of filters.  in order
+// to have a false positive, we need to falsely match two consecutive
+// filters.
+//
+// we expect:  actual_fpp = num_filters * per_filter_fpp^2
+TEST(BloomFilter, SequenceDouble) {
+  int max = 1024;
+  double fpp = .01;
+  for (int seq = 2; seq <= 128; seq *= 2) {
+    std::vector ls;
+    for (int i=0; iinsert("ok" + stringify(j) + "_" + stringify(i));
+	if (ls.size() > 1)
+	  ls[ls.size() - 2]->insert("ok" + stringify(j) + "_" + stringify(i));
+      }
+    }
+
+    int hit = 0;
+    int test = max * 100;
+    int run = 0;
+    for (int i=0; i::iterator j = ls.begin(); j != ls.end(); ++j) {
+	if ((*j)->contains("bad" + stringify(i))) {
+	  run++;
+	  if (run >= 2) {
+	    hit++;
+	    break;
+	  }
+	} else {
+	  run = 0;
+	}
+      }
+    }
+
+    double actual = (double)hit / (double)test;
+    std::cout << "seq " << seq << " max " << max << " fpp " << fpp << " actual " << actual
+	      << " expected " << (fpp*fpp*(double)seq) << std::endl;
+  }
+}
+
+#endif
diff --git a/ceph/src/test/common/test_config.cc b/ceph/src/test/common/test_config.cc
new file mode 100644
index 00000000..cffdc763
--- /dev/null
+++ b/ceph/src/test/common/test_config.cc
@@ -0,0 +1,187 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ *
+ */
+#include "common/config.h"
+#include "common/errno.h"
+#include "gtest/gtest.h"
+
+#define _STR(x) #x
+#define STRINGIFY(x) _STR(x)
+
+static struct config_option config_optionsp[] = {
+#define OPTION(name, type, def_val) \
+       { STRINGIFY(name), type, offsetof(struct md_config_t, name) },
+#define SUBSYS(name, log, gather)
+#define DEFAULT_SUBSYS(log, gather)
+#include "common/config_opts.h"
+#undef OPTION
+#undef SUBSYS
+#undef DEFAULT_SUBSYS
+};
+
+static const int NUM_CONFIG_OPTIONS = sizeof(config_optionsp) / sizeof(config_option);
+
+class test_md_config_t : public md_config_t, public ::testing::Test {
+public:
+  void test_expand_meta() {
+    Mutex::Locker l(lock);
+    // successfull meta expansion $run_dir and ${run_dir}
+    {
+      ostringstream oss;
+      std::string before = " BEFORE ";
+      std::string after = " AFTER ";
+      std::string val(before + "$run_dir${run_dir}" + after);
+      EXPECT_TRUE(expand_meta(val, &oss));
+      EXPECT_EQ(before + "/var/run/ceph/var/run/ceph" + after, val);
+      EXPECT_EQ("", oss.str());
+    }
+    // no meta expansion if variables are unknown
+    {
+      ostringstream oss;
+      std::string expected = "expect $foo and ${bar} to not expand";
+      std::string val = expected;
+      EXPECT_FALSE(expand_meta(val, &oss));
+      EXPECT_EQ(expected, val);
+      EXPECT_EQ("", oss.str());
+    }
+    // recursive variable expansion
+    {
+      std::string mon_host = "$cluster_network";
+      EXPECT_EQ(0, set_val("mon_host", mon_host.c_str(), false));
+
+      std::string lockdep = "true";
+      EXPECT_EQ(0, set_val("lockdep", lockdep.c_str(), false));
+
+      std::string cluster_network = "$public_network $public_network $lockdep $host";
+      EXPECT_EQ(0, set_val("cluster_network", cluster_network.c_str(), false));
+
+      std::string public_network = "NETWORK";
+      EXPECT_EQ(0, set_val("public_network", public_network.c_str(), false));
+
+      ostringstream oss;
+      std::string val = "$mon_host";
+      EXPECT_TRUE(expand_meta(val, &oss));
+      EXPECT_EQ(public_network + " " +
+                public_network + " " +
+                lockdep + " " +
+                "localhost", val);
+      EXPECT_EQ("", oss.str());
+    }
+    // variable expansion loops are non fatal
+    {
+      std::string mon_host = "$cluster_network";
+      EXPECT_EQ(0, set_val("mon_host", mon_host.c_str(), false));
+
+      std::string cluster_network = "$public_network";
+      EXPECT_EQ(0, set_val("cluster_network", cluster_network.c_str(), false));
+
+      std::string public_network = "$mon_host";
+      EXPECT_EQ(0, set_val("public_network", public_network.c_str(), false));
+
+      ostringstream oss;
+      std::string val = "$mon_host";
+      EXPECT_TRUE(expand_meta(val, &oss));
+      EXPECT_EQ("$cluster_network", val);
+      const char *expected_oss =
+        "variable expansion loop at mon_host=$cluster_network\n"
+        "expansion stack: \n"
+        "public_network=$mon_host\n"
+        "cluster_network=$public_network\n"
+        "mon_host=$cluster_network\n";
+      EXPECT_EQ(expected_oss, oss.str());
+    }
+  }
+
+  void test_expand_all_meta() {
+    Mutex::Locker l(lock);
+    int before_count = 0;
+    for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
+      config_option *opt = config_optionsp + i;
+      if (opt->type == OPT_STR) {
+        std::string *str = (std::string *)opt->conf_ptr(this);
+        if (str->find("$") != string::npos)
+          before_count++;
+      }
+    }
+    // if there are no meta variables in the default configuration,
+    // something must be done to check the expected side effect
+    // of expand_all_meta
+    ASSERT_LT(0, before_count);
+    expand_all_meta();
+    int after_count = 0;
+    for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
+      config_option *opt = config_optionsp + i;
+      if (opt->type == OPT_STR) {
+        std::string *str = (std::string *)opt->conf_ptr(this);
+        if (str->find("$") != string::npos)
+          after_count++;
+      }
+    }
+    ASSERT_EQ(0, after_count);
+  }
+};
+
+TEST_F(test_md_config_t, expand_meta)
+{
+  test_expand_meta();
+}
+
+TEST_F(test_md_config_t, expand_all_meta)
+{
+  test_expand_all_meta();
+}
+
+TEST(md_config_t, set_val)
+{
+  int buf_size = 1024;
+  md_config_t conf;
+  // disable meta variable expansion
+  {
+    char *buf = (char*)malloc(buf_size);
+    std::string expected = "$host";
+    EXPECT_EQ(0, conf.set_val("mon_host", expected.c_str(), false));
+    EXPECT_EQ(0, conf.get_val("mon_host", &buf, buf_size));
+    EXPECT_EQ(expected, buf);
+    free(buf);
+  }
+  // meta variable expansion is enabled by default
+  {
+    char *run_dir = (char*)malloc(buf_size);
+    EXPECT_EQ(0, conf.get_val("run_dir", &run_dir, buf_size));
+    EXPECT_EQ(0, conf.set_val("admin_socket", "$run_dir"));
+    char *admin_socket = (char*)malloc(buf_size);
+    EXPECT_EQ(0, conf.get_val("admin_socket", &admin_socket, buf_size));
+    EXPECT_EQ(std::string(run_dir), std::string(admin_socket));
+    free(run_dir);
+    free(admin_socket);
+  }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ *   make unittest_config &&
+ *    valgrind \
+ *    --max-stackframe=20000000 --tool=memcheck \
+ *   ./unittest_config # --gtest_filter=md_config_t.set_val
+ * "
+ * End:
+ */
diff --git a/ceph/src/test/common/test_context.cc b/ceph/src/test/common/test_context.cc
new file mode 100644
index 00000000..ca745c9a
--- /dev/null
+++ b/ceph/src/test/common/test_context.cc
@@ -0,0 +1,64 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ *
+ */
+#include "gtest/gtest.h"
+#include "include/types.h"
+#include "include/msgr.h"
+#include "common/ceph_context.h"
+#include "common/config.h"
+
+TEST(CephContext, do_command)
+{
+  CephContext *cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
+
+  string key("key");
+  string value("value");
+  cct->_conf->set_val(key.c_str(), value.c_str(), false);
+  cmdmap_t cmdmap;
+  cmdmap["var"] = key;
+
+  {
+    bufferlist out;
+    cct->do_command("config get", cmdmap, "xml", &out);
+    string s(out.c_str(), out.length());
+    EXPECT_EQ("" + value + "", s);
+  }
+
+  {
+    bufferlist out;
+    cct->do_command("config get", cmdmap, "UNSUPPORTED", &out);
+    string s(out.c_str(), out.length());
+    EXPECT_EQ("{ \"key\": \"value\"}", s);
+  }
+
+  cct->put();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ *   make unittest_context &&
+ *    valgrind \
+ *    --max-stackframe=20000000 --tool=memcheck \
+ *   ./unittest_context # --gtest_filter=CephContext.*
+ * "
+ * End:
+ */
diff --git a/ceph/src/test/common/test_crc32c.cc b/ceph/src/test/common/test_crc32c.cc
new file mode 100644
index 00000000..b4297c61
--- /dev/null
+++ b/ceph/src/test/common/test_crc32c.cc
@@ -0,0 +1,255 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include 
+#include 
+
+#include "include/types.h"
+#include "include/crc32c.h"
+#include "include/utime.h"
+#include "common/Clock.h"
+
+#include "gtest/gtest.h"
+
+#include "common/sctp_crc32.h"
+#include "common/crc32c_intel_baseline.h"
+
+TEST(Crc32c, Small) {
+  const char *a = "foo bar baz";
+  const char *b = "whiz bang boom";
+  ASSERT_EQ(4119623852u, ceph_crc32c(0, (unsigned char *)a, strlen(a)));
+  ASSERT_EQ(881700046u, ceph_crc32c(1234, (unsigned char *)a, strlen(a)));
+  ASSERT_EQ(2360230088u, ceph_crc32c(0, (unsigned char *)b, strlen(b)));
+  ASSERT_EQ(3743019208u, ceph_crc32c(5678, (unsigned char *)b, strlen(b)));
+}
+
+TEST(Crc32c, PartialWord) {
+  const char *a = (const char *)malloc(5);
+  const char *b = (const char *)malloc(35);
+  memset((void *)a, 1, 5);
+  memset((void *)b, 1, 35);
+  ASSERT_EQ(2715569182u, ceph_crc32c(0, (unsigned char *)a, 5));
+  ASSERT_EQ(440531800u, ceph_crc32c(0, (unsigned char *)b, 35));
+}
+
+TEST(Crc32c, Big) {
+  int len = 4096000;
+  char *a = (char *)malloc(len);
+  memset(a, 1, len);
+  ASSERT_EQ(31583199u, ceph_crc32c(0, (unsigned char *)a, len));
+  ASSERT_EQ(1400919119u, ceph_crc32c(1234, (unsigned char *)a, len));
+}
+
+TEST(Crc32c, Performance) {
+  int len = 1000 * 1024 * 1024;
+  char *a = (char *)malloc(len);
+  std::cout << "populating large buffer" << std::endl;
+  for (int i=0; i
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+#include "common/Thread.h"
+#include "common/sharedptr_registry.hpp"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include 
+
+using namespace std::tr1;
+
+class SharedPtrRegistryTest : public SharedPtrRegistry {
+public:
+  Mutex &get_lock() { return lock; }
+  map, int*> > &get_contents() {
+    return contents;
+  }
+};
+
+class SharedPtrRegistry_all : public ::testing::Test {
+public:
+
+  class Thread_wait : public Thread {
+  public:
+    SharedPtrRegistryTest ®istry;
+    unsigned int key;
+    int value;
+    shared_ptr ptr;
+    enum in_method_t { LOOKUP, LOOKUP_OR_CREATE } in_method;
+
+    Thread_wait(SharedPtrRegistryTest& _registry, unsigned int _key, int _value, in_method_t _in_method) : 
+      registry(_registry),
+      key(_key),
+      value(_value),
+      in_method(_in_method)
+    {
+    }
+    
+    virtual void *entry() {
+      switch(in_method) {
+      case LOOKUP_OR_CREATE:
+	if (value) 
+	  ptr = registry.lookup_or_create(key, value);
+	else
+	  ptr = registry.lookup_or_create(key);
+	break;
+      case LOOKUP:
+	ptr = shared_ptr(new int);
+	*ptr = value;
+	ptr = registry.lookup(key);
+	break;
+      }
+      return NULL;
+    }
+  };
+
+  static const useconds_t DELAY_MAX = 20 * 1000 * 1000;
+  static useconds_t delay;
+
+  bool wait_for(SharedPtrRegistryTest ®istry, int waiting) {
+    do {
+      //
+      // the delay variable is supposed to be initialized to zero. It would be fine
+      // to usleep(0) but we take this opportunity to test the loop. It will try 
+      // again and therefore show that the logic ( increasing the delay ) actually
+      // works. 
+      //
+      if (delay > 0)
+	usleep(delay);
+      {
+	Mutex::Locker l(registry.get_lock());
+	if (registry.waiting == waiting) 
+	  break;
+      }
+      if (delay > 0)
+	cout << "delay " << delay << "us, is not long enough, try again\n";
+    } while (( delay = delay * 2 + 1) < DELAY_MAX);
+    return delay < DELAY_MAX;
+  }
+};
+
+useconds_t SharedPtrRegistry_all::delay = 0;
+
+TEST_F(SharedPtrRegistry_all, lookup_or_create) {
+  SharedPtrRegistryTest registry;
+  unsigned int key = 1;
+  int value = 2;
+  shared_ptr ptr = registry.lookup_or_create(key);
+  *ptr = value;
+  ASSERT_EQ(value, *registry.lookup_or_create(key));
+}
+
+TEST_F(SharedPtrRegistry_all, wait_lookup_or_create) {
+  SharedPtrRegistryTest registry;
+
+  //
+  // simulate the following: The last reference to a shared_ptr goes
+  // out of scope and the shared_ptr object is about to be removed and
+  // marked as such. The weak_ptr stored in the registry will show
+  // that it has expired(). However, the SharedPtrRegistry::OnRemoval
+  // object has not yet been called and did not get a chance to
+  // acquire the lock. The lookup_or_create and lookup methods must
+  // detect that situation and wait until the weak_ptr is removed from
+  // the registry.
+  //
+  {
+    unsigned int key = 1;
+    {
+      shared_ptr ptr(new int);
+      registry.get_contents()[key] = make_pair(ptr, ptr.get());
+    }
+    EXPECT_FALSE(registry.get_contents()[key].first.lock());
+
+    Thread_wait t(registry, key, 0, Thread_wait::LOOKUP_OR_CREATE);
+    t.create();
+    ASSERT_TRUE(wait_for(registry, 1));
+    EXPECT_FALSE(t.ptr);
+    // waiting on a key does not block lookups on other keys
+    EXPECT_TRUE(registry.lookup_or_create(key + 12345));
+    registry.remove(key);
+    ASSERT_TRUE(wait_for(registry, 0));
+    t.join();
+    EXPECT_TRUE(t.ptr);
+  }
+  {
+    unsigned int key = 2;
+    int value = 3;
+    {
+      shared_ptr ptr(new int);
+      registry.get_contents()[key] = make_pair(ptr, ptr.get());
+    }
+    EXPECT_FALSE(registry.get_contents()[key].first.lock());
+
+    Thread_wait t(registry, key, value, Thread_wait::LOOKUP_OR_CREATE);
+    t.create();
+    ASSERT_TRUE(wait_for(registry, 1));
+    EXPECT_FALSE(t.ptr);
+    // waiting on a key does not block lookups on other keys
+    {
+      int other_value = value + 1;
+      unsigned int other_key = key + 1;
+      shared_ptr ptr = registry.lookup_or_create(other_key, other_value);
+      EXPECT_TRUE(ptr);
+      EXPECT_EQ(other_value, *ptr);
+    }
+    registry.remove(key);
+    ASSERT_TRUE(wait_for(registry, 0));
+    t.join();
+    EXPECT_TRUE(t.ptr);
+    EXPECT_EQ(value, *t.ptr);
+  }
+}
+
+TEST_F(SharedPtrRegistry_all, lookup) {
+  SharedPtrRegistryTest registry;
+  unsigned int key = 1;
+  int value = 2;
+  {
+    shared_ptr ptr = registry.lookup_or_create(key);
+    *ptr = value;
+    ASSERT_EQ(value, *registry.lookup(key));
+  }
+  ASSERT_FALSE(registry.lookup(key));
+}
+
+TEST_F(SharedPtrRegistry_all, wait_lookup) {
+  SharedPtrRegistryTest registry;
+
+  unsigned int key = 1;
+  int value = 2;
+  {
+    shared_ptr ptr(new int);
+    registry.get_contents()[key] = make_pair(ptr, ptr.get());
+  }
+  EXPECT_FALSE(registry.get_contents()[key].first.lock());
+
+  Thread_wait t(registry, key, value, Thread_wait::LOOKUP);
+  t.create();
+  ASSERT_TRUE(wait_for(registry, 1));
+  EXPECT_EQ(value, *t.ptr);
+  // waiting on a key does not block lookups on other keys
+  EXPECT_FALSE(registry.lookup(key + 12345));
+  registry.remove(key);
+  ASSERT_TRUE(wait_for(registry, 0));
+  t.join();
+  EXPECT_FALSE(t.ptr);
+}
+
+TEST_F(SharedPtrRegistry_all, get_next) {
+
+  {
+    SharedPtrRegistry registry;
+    const unsigned int key = 0;
+    pair i;
+    EXPECT_FALSE(registry.get_next(key, &i));
+  }
+  {
+    SharedPtrRegistryTest registry;
+
+    const unsigned int key2 = 333;
+    shared_ptr ptr2 = registry.lookup_or_create(key2);
+    const int value2 = *ptr2 = 400;
+
+    // entries with expired pointers are silentely ignored
+    const unsigned int key_gone = 222;
+    registry.get_contents()[key_gone] = make_pair(shared_ptr(), (int*)0);
+
+    const unsigned int key1 = 111;
+    shared_ptr ptr1 = registry.lookup_or_create(key1);
+    const int value1 = *ptr1 = 800;
+
+    pair i;
+    EXPECT_TRUE(registry.get_next(i.first, &i));
+    EXPECT_EQ(key1, i.first);
+    EXPECT_EQ(value1, i.second);
+
+    EXPECT_TRUE(registry.get_next(i.first, &i));
+    EXPECT_EQ(key2, i.first);
+    EXPECT_EQ(value2, i.second);
+
+    EXPECT_FALSE(registry.get_next(i.first, &i));
+  }
+  {
+    //
+    // http://tracker.ceph.com/issues/6117
+    // reproduce the issue.
+    //
+    SharedPtrRegistryTest registry;
+    const unsigned int key1 = 111;
+    shared_ptr *ptr1 = new shared_ptr(registry.lookup_or_create(key1));
+    const unsigned int key2 = 222;
+    shared_ptr ptr2 = registry.lookup_or_create(key2);
+    
+    pair > i;
+    EXPECT_TRUE(registry.get_next(i.first, &i));
+    EXPECT_EQ(key1, i.first);
+    delete ptr1;
+    EXPECT_TRUE(registry.get_next(i.first, &i));    
+    EXPECT_EQ(key2, i.first);
+  }
+}
+
+TEST_F(SharedPtrRegistry_all, remove) {
+  {
+    SharedPtrRegistryTest registry;
+    const unsigned int key1 = 1;
+    shared_ptr ptr1 = registry.lookup_or_create(key1);
+    *ptr1 = 400;
+    registry.remove(key1);
+
+    shared_ptr ptr2 = registry.lookup_or_create(key1);
+    *ptr2 = 500;
+
+    ptr1 = shared_ptr();
+    shared_ptr res = registry.lookup(key1);
+    assert(res);
+    assert(res == ptr2);
+    assert(*res == 500);
+  }
+  {
+    SharedPtrRegistryTest registry;
+    const unsigned int key1 = 1;
+    shared_ptr ptr1 = registry.lookup_or_create(key1, 400);
+    registry.remove(key1);
+
+    shared_ptr ptr2 = registry.lookup_or_create(key1, 500);
+
+    ptr1 = shared_ptr();
+    shared_ptr res = registry.lookup(key1);
+    assert(res);
+    assert(res == ptr2);
+    assert(*res == 500);
+  }
+}
+
+class SharedPtrRegistry_destructor : public ::testing::Test {
+public:
+
+  typedef enum { UNDEFINED, YES, NO } DieEnum;
+  static DieEnum died;
+
+  struct TellDie {
+    TellDie() { died = NO; }
+    ~TellDie() { died = YES; }
+    
+    int value;
+  };
+
+  virtual void SetUp() {
+    died = UNDEFINED;
+  }
+};
+
+SharedPtrRegistry_destructor::DieEnum SharedPtrRegistry_destructor::died = SharedPtrRegistry_destructor::UNDEFINED;
+
+TEST_F(SharedPtrRegistry_destructor, destructor) {
+  SharedPtrRegistry registry;
+  EXPECT_EQ(UNDEFINED, died);
+  int key = 101;
+  {
+    shared_ptr a = registry.lookup_or_create(key);
+    EXPECT_EQ(NO, died);
+    EXPECT_TRUE(a);
+  }
+  EXPECT_EQ(YES, died);
+  EXPECT_FALSE(registry.lookup(key));
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make unittest_sharedptr_registry && ./unittest_sharedptr_registry # --gtest_filter=*.* --log-to-stderr=true"
+// End:
diff --git a/ceph/src/test/common/test_sloppy_crc_map.cc b/ceph/src/test/common/test_sloppy_crc_map.cc
new file mode 100644
index 00000000..2650f4f9
--- /dev/null
+++ b/ceph/src/test/common/test_sloppy_crc_map.cc
@@ -0,0 +1,113 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/SloppyCRCMap.h"
+#include "common/Formatter.h"
+#include 
+
+void dump(const SloppyCRCMap& scm)
+{
+  Formatter *f = new_formatter("json-pretty");
+  f->open_object_section("map");
+  scm.dump(f);
+  f->close_section();
+  f->flush(cout);
+  delete f;
+}
+
+TEST(SloppyCRCMap, basic) {
+  SloppyCRCMap scm(4);
+
+  bufferlist a, b;
+  a.append("The quick brown fox jumped over a fence whose color I forget.");
+  b.append("asdf");
+
+  scm.write(0, a.length(), a);
+  if (0)
+    dump(scm);
+  ASSERT_EQ(0, scm.read(0, a.length(), a, &cout));
+
+  scm.write(12, b.length(), b);
+  if (0)
+    dump(scm);
+
+  ASSERT_EQ(0, scm.read(12, b.length(), b, &cout));
+  ASSERT_EQ(1, scm.read(0, a.length(), a, &cout));
+}
+
+TEST(SloppyCRCMap, truncate) {
+  SloppyCRCMap scm(4);
+
+  bufferlist a, b;
+  a.append("asdf");
+  b.append("qwer");
+
+  scm.write(0, a.length(), a);
+  scm.write(4, a.length(), a);
+  ASSERT_EQ(0, scm.read(4, 4, a, &cout));
+  ASSERT_EQ(1, scm.read(4, 4, b, &cout));
+  scm.truncate(4);
+  ASSERT_EQ(0, scm.read(4, 4, b, &cout));
+}
+
+TEST(SloppyCRCMap, zero) {
+  SloppyCRCMap scm(4);
+
+  bufferlist a, b;
+  a.append("asdf");
+  b.append("qwer");
+
+  scm.write(0, a.length(), a);
+  scm.write(4, a.length(), a);
+  ASSERT_EQ(0, scm.read(4, 4, a, &cout));
+  ASSERT_EQ(1, scm.read(4, 4, b, &cout));
+  scm.zero(4, 4);
+  ASSERT_EQ(1, scm.read(4, 4, a, &cout));
+  ASSERT_EQ(1, scm.read(4, 4, b, &cout));
+
+  bufferptr bp(4);
+  bp.zero();
+  bufferlist c;
+  c.append(bp);
+  ASSERT_EQ(0, scm.read(0, 4, a, &cout));
+  ASSERT_EQ(0, scm.read(4, 4, c, &cout));
+  scm.zero(0, 15);
+  ASSERT_EQ(1, scm.read(0, 4, a, &cout));
+  ASSERT_EQ(0, scm.read(0, 4, c, &cout));
+}
+
+TEST(SloppyCRCMap, clone_range) {
+  SloppyCRCMap src(4);
+  SloppyCRCMap dst(4);
+
+  bufferlist a, b;
+  a.append("asdfghjkl");
+  b.append("qwertyui");
+
+  src.write(0, a.length(), a);
+  src.write(8, a.length(), a);
+  src.write(16, a.length(), a);
+
+  dst.write(0, b.length(), b);
+  dst.clone_range(0, 8, 0, src);
+  ASSERT_EQ(2, dst.read(0, 8, b, &cout));
+  ASSERT_EQ(0, dst.read(8, 8, b, &cout));
+
+  dst.write(16, b.length(), b);
+  ASSERT_EQ(2, dst.read(16, 8, a, &cout));
+  dst.clone_range(16, 8, 16, src);
+  ASSERT_EQ(0, dst.read(16, 8, a, &cout));
+
+  dst.write(16, b.length(), b);
+  ASSERT_EQ(1, dst.read(16, 4, a, &cout));
+  dst.clone_range(16, 8, 2, src);
+  ASSERT_EQ(0, dst.read(16, 4, a, &cout));
+
+  dst.write(0, b.length(), b);
+  dst.write(8, b.length(), b);
+  ASSERT_EQ(2, dst.read(0, 8, a, &cout));
+  ASSERT_EQ(2, dst.read(8, 8, a, &cout));
+  dst.clone_range(2, 8, 0, src);
+  ASSERT_EQ(0, dst.read(0, 8, a, &cout));
+  ASSERT_EQ(0, dst.read(8, 4, a, &cout));
+}
diff --git a/ceph/src/test/common/test_str_map.cc b/ceph/src/test/common/test_str_map.cc
new file mode 100644
index 00000000..f60f0d8b
--- /dev/null
+++ b/ceph/src/test/common/test_str_map.cc
@@ -0,0 +1,70 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+#include 
+
+#include "include/str_map.h"
+
+using namespace std;
+
+TEST(str_map, json) {
+  map str_map;
+  stringstream ss;
+  // well formatted
+  ASSERT_EQ(0, get_str_map("{\"key\": \"value\"}", ss, &str_map));
+  ASSERT_EQ("value", str_map["key"]);
+  // well formatted but not a JSON object
+  ASSERT_EQ(-EINVAL, get_str_map("\"key\"", ss, &str_map));
+  ASSERT_NE(string::npos, ss.str().find("must be a JSON object"));
+}
+
+TEST(str_map, plaintext) {
+  stringstream ss;
+  {
+    map str_map;
+    ASSERT_EQ(0, get_str_map(" foo=bar\t\nfrob=nitz   yeah right=   \n\t",
+			     ss, &str_map));
+    ASSERT_EQ(4u, str_map.size());
+    ASSERT_EQ("bar", str_map["foo"]);
+    ASSERT_EQ("nitz", str_map["frob"]);
+    ASSERT_EQ("", str_map["yeah"]);
+    ASSERT_EQ("", str_map["right"]);
+  }
+  {
+    map str_map;
+    ASSERT_EQ(0, get_str_map("that", ss, &str_map));
+    ASSERT_EQ(1u, str_map.size());
+    ASSERT_EQ("", str_map["that"]);
+  }
+  {
+    map str_map;
+    ASSERT_EQ(0, get_str_map(" \t \n ", ss, &str_map));
+    ASSERT_EQ(0u, str_map.size());
+    ASSERT_EQ(0, get_str_map("", ss, &str_map));
+    ASSERT_EQ(0u, str_map.size());
+  }
+}
+
+/* 
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 && 
+ *   make unittest_str_map && 
+ *   valgrind --tool=memcheck --leak-check=full \
+ *      ./unittest_str_map
+ *   "
+ * End:
+ */
diff --git a/ceph/src/test/common/test_util.cc b/ceph/src/test/common/test_util.cc
new file mode 100644
index 00000000..cb22047c
--- /dev/null
+++ b/ceph/src/test/common/test_util.cc
@@ -0,0 +1,32 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/util.h"
+#include "gtest/gtest.h"
+
+#include 
+
+TEST(util, unit_to_bytesize)
+{
+  ASSERT_EQ(1234ll, unit_to_bytesize("1234", &cerr));
+  ASSERT_EQ(1024ll, unit_to_bytesize("1K", &cerr));
+  ASSERT_EQ(1024ll, unit_to_bytesize("1k", &cerr));
+  ASSERT_EQ(1048576ll, unit_to_bytesize("1M", &cerr));
+  ASSERT_EQ(1073741824ll, unit_to_bytesize("1G", &cerr));
+  ASSERT_EQ(1099511627776ll, unit_to_bytesize("1T", &cerr));
+  ASSERT_EQ(1125899906842624ll, unit_to_bytesize("1P", &cerr));
+  ASSERT_EQ(1152921504606846976ll, unit_to_bytesize("1E", &cerr));
+
+  ASSERT_EQ(65536ll, unit_to_bytesize(" 64K", &cerr));
+}
diff --git a/ceph/src/test/confutils.cc b/ceph/src/test/confutils.cc
new file mode 100644
index 00000000..61b92d44
--- /dev/null
+++ b/ceph/src/test/confutils.cc
@@ -0,0 +1,507 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "common/ConfUtils.h"
+#include "common/config.h"
+#include "common/errno.h"
+#include "gtest/gtest.h"
+#include "include/buffer.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "include/memory.h"
+
+using ceph::bufferlist;
+using std::cerr;
+using std::ostringstream;
+
+#define MAX_FILES_TO_DELETE 1000UL
+
+static size_t config_idx = 0;
+static size_t unlink_idx = 0;
+static char *to_unlink[MAX_FILES_TO_DELETE];
+
+static std::string get_temp_dir()
+{
+  static std::string temp_dir;
+
+  if (temp_dir.empty()) {
+    const char *tmpdir = getenv("TMPDIR");
+    if (!tmpdir)
+      tmpdir = "/tmp";
+    srand(time(NULL));
+    ostringstream oss;
+    oss << tmpdir << "/confutils_test_dir." << rand() << "." << getpid();
+    umask(022);
+    int res = mkdir(oss.str().c_str(), 01777);
+    if (res) {
+      cerr << "failed to create temp directory '" << temp_dir << "'" << std::endl;
+      return "";
+    }
+    temp_dir = oss.str();
+  }
+  return temp_dir;
+}
+
+static void unlink_all(void)
+{
+  for (size_t i = 0; i < unlink_idx; ++i) {
+    unlink(to_unlink[i]);
+  }
+  for (size_t i = 0; i < unlink_idx; ++i) {
+    free(to_unlink[i]);
+  }
+  rmdir(get_temp_dir().c_str());
+}
+
+static int create_tempfile(const std::string &fname, const char *text)
+{
+  FILE *fp = fopen(fname.c_str(), "w");
+  if (!fp) {
+    int err = errno;
+    cerr << "Failed to write file '" << fname << "' to temp directory '"
+	 << get_temp_dir() << "'. " << cpp_strerror(err) << std::endl;
+    return err;
+  }
+  ceph::shared_ptr fpp(fp, fclose);
+  if (unlink_idx >= MAX_FILES_TO_DELETE)
+    return -ENOBUFS;
+  if (unlink_idx == 0) {
+    memset(to_unlink, 0, sizeof(to_unlink));
+    atexit(unlink_all);
+  }
+  to_unlink[unlink_idx++] = strdup(fname.c_str());
+  size_t strlen_text = strlen(text);
+  size_t res = fwrite(text, 1, strlen_text, fp);
+  if (res != strlen_text) {
+    int err = errno;
+    cerr << "fwrite error while writing to " << fname 
+	 << ": " << cpp_strerror(err) << std::endl;
+    return err;
+  }
+  return 0;
+}
+
+static std::string next_tempfile(const char *text)
+{
+  ostringstream oss;
+  std::string temp_dir(get_temp_dir());
+  if (temp_dir.empty())
+    return "";
+  oss << temp_dir << "/test_config." << config_idx++ << ".config";
+  int ret = create_tempfile(oss.str(), text);
+  if (ret)
+    return "";
+  return oss.str();
+}
+
+const char * const trivial_conf_1 = "";
+
+const char * const trivial_conf_2 = "log dir = foobar";
+
+const char * const trivial_conf_3 = "log dir = barfoo\n";
+
+const char * const trivial_conf_4 = "log dir = \"barbaz\"\n";
+
+const char * const simple_conf_1 = "\
+; here's a comment\n\
+[global]\n\
+        keyring = .my_ceph_keyring\n\
+\n\
+[mds]\n\
+	log dir = out\n\
+	log per instance = true\n\
+	log sym history = 100\n\
+        profiling logger = true\n\
+	profiling logger dir = wowsers\n\
+	chdir = ""\n\
+	pid file = out/$name.pid\n\
+\n\
+        mds debug frag = true\n\
+[osd]\n\
+	pid file = out/$name.pid\n\
+        osd scrub load threshold = 5.0\n\
+\n\
+        lockdep = 1\n\
+[osd0]\n\
+        osd data = dev/osd0\n\
+        osd journal size = 100\n\
+[mds.a]\n\
+[mds.b]\n\
+[mds.c]\n\
+";
+
+// we can add whitespace at odd locations and it will get stripped out.
+const char * const simple_conf_2 = "\
+[mds.a]\n\
+	log dir = special_mds_a\n\
+[mds]\n\
+	log sym history = 100\n\
+	log dir = out # after a comment, anything # can ### happen ;;; right?\n\
+	log per instance = true\n\
+        profiling logger = true\n\
+	profiling                 logger dir = log\n\
+	chdir = ""\n\
+        pid file\t=\tfoo2\n\
+[osd0]\n\
+        keyring   =       osd_keyring          ; osd's keyring\n\
+\n\
+           \n\
+[global]\n\
+	# I like pound signs as comment markers.\n\
+	; Do you like pound signs as comment markers?\n\
+        keyring = shenanigans          ; The keyring of a leprechaun\n\
+\n\
+	# Let's just have a line with a lot of whitespace and nothing else.\n\
+                         \n\
+        lockdep = 1\n\
+";
+
+// test line-combining
+const char * const conf3 = "\
+[global]\n\
+	log file = /quite/a/long/path\\\n\
+/for/a/log/file\n\
+	pid file = \\\n\
+                           spork\\\n\
+\n\
+[mon] #nothing here \n\
+";
+
+const char * const escaping_conf_1 = "\
+[global]\n\
+	log file = the \"scare quotes\"\n\
+	pid file = a \\\n\
+pid file\n\
+[mon]\n\
+	keyring = \"nested \\\"quotes\\\"\"\n\
+";
+
+const char * const escaping_conf_2 = "\
+[apple \\]\\[]\n\
+	log file = floppy disk\n\
+[mon]\n\
+	keyring = \"backslash\\\\\"\n\
+";
+
+// illegal because it contains an invalid utf8 sequence.
+const char illegal_conf1[] = "\
+[global]\n\
+	log file = foo\n\
+	pid file = invalid-utf-\xe2\x28\xa1\n\
+[osd0]\n\
+        keyring = osd_keyring          ; osd's keyring\n\
+";
+
+// illegal because it contains a malformed section header.
+const char illegal_conf2[] = "\
+[global\n\
+	log file = foo\n\
+[osd0]\n\
+        keyring = osd_keyring          ; osd's keyring\n\
+";
+
+// illegal because it contains a line that doesn't parse
+const char illegal_conf3[] = "\
+[global]\n\
+        who_what_where\n\
+[osd0]\n\
+        keyring = osd_keyring          ; osd's keyring\n\
+";
+
+// illegal because it has unterminated quotes
+const char illegal_conf4[] = "\
+[global]\n\
+        keyring = \"unterminated quoted string\n\
+[osd0]\n\
+        keyring = osd_keyring          ; osd's keyring\n\
+";
+
+// illegal because it has a backslash at the very end
+const char illegal_conf5[] = "\
+[global]\n\
+        keyring = something awful\\\\\n\
+";
+
+// unicode config file
+const char unicode_config_1[] = "\
+[global]\n\
+        log file =           \x66\xd1\x86\xd1\x9d\xd3\xad\xd3\xae     \n\
+        pid file =           foo-bar\n\
+[osd0]\n\
+";
+
+const char override_config_1[] = "\
+[global]\n\
+        log file =           global_log\n\
+[mds]\n\
+        log file =           mds_log\n\
+[osd]\n\
+        log file =           osd_log\n\
+[osd.0]\n\
+        log file =           osd0_log\n\
+";
+
+const char dup_key_config_1[] = "\
+[mds.a]\n\
+        log_file = 1\n\
+        log_file = 3\n\
+";
+
+TEST(ConfUtils, Whitespace) {
+  std::string test0("");
+  ConfFile::trim_whitespace(test0, false);
+  ASSERT_EQ(test0, "");
+
+  std::string test0a("");
+  ConfFile::trim_whitespace(test0a, true);
+  ASSERT_EQ(test0a, "");
+
+  std::string test0b("          ");
+  ConfFile::trim_whitespace(test0b, false);
+  ASSERT_EQ(test0b, "");
+
+  std::string test0c("          ");
+  ConfFile::trim_whitespace(test0c, true);
+  ASSERT_EQ(test0c, "");
+
+  std::string test1(" abc             ");
+  ConfFile::trim_whitespace(test1, false);
+  ASSERT_EQ(test1, "abc");
+
+  std::string test2(" abc        d     ");
+  ConfFile::trim_whitespace(test2, true);
+  ASSERT_EQ(test2, "abc d");
+
+  std::string test3(" abc        d     ");
+  ConfFile::trim_whitespace(test3, false);
+  ASSERT_EQ(test3, "abc        d");
+
+  std::string test4("abcd");
+  ConfFile::trim_whitespace(test4, false);
+  ASSERT_EQ(test4, "abcd");
+
+  std::string test5("abcd");
+  ConfFile::trim_whitespace(test5, true);
+  ASSERT_EQ(test5, "abcd");
+}
+
+TEST(ConfUtils, ParseFiles0) {
+  std::deque err;
+  std::string val;
+  std::ostringstream warn;
+
+  std::string trivial_conf_1_f(next_tempfile(trivial_conf_1));
+  ConfFile cf1;
+  ASSERT_EQ(cf1.parse_file(trivial_conf_1_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+
+  std::string trivial_conf_2_f(next_tempfile(trivial_conf_2));
+  ConfFile cf2;
+  ASSERT_EQ(cf2.parse_file(trivial_conf_2_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 1U);
+
+  bufferlist bl3;
+  bl3.append(trivial_conf_3, strlen(trivial_conf_3));
+  ConfFile cf3;
+  ASSERT_EQ(cf3.parse_bufferlist(&bl3, &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(cf3.read("global", "log dir", val), 0);
+  ASSERT_EQ(val, "barfoo");
+
+  std::string trivial_conf_4_f(next_tempfile(trivial_conf_4));
+  ConfFile cf4;
+  ASSERT_EQ(cf4.parse_file(trivial_conf_4_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(cf4.read("global", "log dir", val), 0);
+  ASSERT_EQ(val, "barbaz");
+}
+
+TEST(ConfUtils, ParseFiles1) {
+  std::deque err;
+  std::ostringstream warn;
+  std::string simple_conf_1_f(next_tempfile(simple_conf_1));
+  ConfFile cf1;
+  ASSERT_EQ(cf1.parse_file(simple_conf_1_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+
+  std::string simple_conf_2_f(next_tempfile(simple_conf_1));
+  ConfFile cf2;
+  ASSERT_EQ(cf2.parse_file(simple_conf_2_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+
+  bufferlist bl3;
+  bl3.append(simple_conf_1, strlen(simple_conf_1));
+  ConfFile cf3;
+  ASSERT_EQ(cf3.parse_bufferlist(&bl3, &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+
+  bufferlist bl4;
+  bl4.append(simple_conf_2, strlen(simple_conf_2));
+  ConfFile cf4;
+  ASSERT_EQ(cf4.parse_bufferlist(&bl4, &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+}
+
+TEST(ConfUtils, ReadFiles1) {
+  std::deque err;
+  std::ostringstream warn;
+  std::string simple_conf_1_f(next_tempfile(simple_conf_1));
+  ConfFile cf1;
+  ASSERT_EQ(cf1.parse_file(simple_conf_1_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+
+  std::string val;
+  ASSERT_EQ(cf1.read("global", "keyring", val), 0);
+  ASSERT_EQ(val, ".my_ceph_keyring");
+
+  ASSERT_EQ(cf1.read("mds", "profiling logger dir", val), 0);
+  ASSERT_EQ(val, "wowsers");
+
+  ASSERT_EQ(cf1.read("mds", "something that does not exist", val), -ENOENT);
+
+  // exists in mds section, but not in global
+  ASSERT_EQ(cf1.read("global", "profiling logger dir", val), -ENOENT);
+
+  bufferlist bl2;
+  bl2.append(simple_conf_2, strlen(simple_conf_2));
+  ConfFile cf2;
+  ASSERT_EQ(cf2.parse_bufferlist(&bl2, &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(cf2.read("osd0", "keyring", val), 0);
+  ASSERT_EQ(val, "osd_keyring");
+
+  ASSERT_EQ(cf2.read("mds", "pid file", val), 0);
+  ASSERT_EQ(val, "foo2");
+  ASSERT_EQ(cf2.read("nonesuch", "keyring", val), -ENOENT);
+}
+
+TEST(ConfUtils, ReadFiles2) {
+  std::deque err;
+  std::ostringstream warn;
+  std::string conf3_f(next_tempfile(conf3));
+  ConfFile cf1;
+  std::string val;
+  ASSERT_EQ(cf1.parse_file(conf3_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(cf1.read("global", "log file", val), 0);
+  ASSERT_EQ(val, "/quite/a/long/path/for/a/log/file");
+  ASSERT_EQ(cf1.read("global", "pid file", val), 0);
+  ASSERT_EQ(val, "spork");
+
+  std::string unicode_config_1f(next_tempfile(unicode_config_1));
+  ConfFile cf2;
+  ASSERT_EQ(cf2.parse_file(unicode_config_1f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(cf2.read("global", "log file", val), 0);
+  ASSERT_EQ(val, "\x66\xd1\x86\xd1\x9d\xd3\xad\xd3\xae");
+}
+
+TEST(ConfUtils, IllegalFiles) {
+  std::deque err;
+  std::ostringstream warn;
+  std::string illegal_conf1_f(next_tempfile(illegal_conf1));
+  ConfFile cf1;
+  std::string val;
+  ASSERT_EQ(cf1.parse_file(illegal_conf1_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 1U);
+
+  bufferlist bl2;
+  bl2.append(illegal_conf2, strlen(illegal_conf2));
+  ConfFile cf2;
+  ASSERT_EQ(cf2.parse_bufferlist(&bl2, &err, &warn), 0);
+  ASSERT_EQ(err.size(), 1U);
+
+  std::string illegal_conf3_f(next_tempfile(illegal_conf3));
+  ConfFile cf3;
+  ASSERT_EQ(cf3.parse_file(illegal_conf3_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 1U);
+
+  std::string illegal_conf4_f(next_tempfile(illegal_conf4));
+  ConfFile cf4;
+  ASSERT_EQ(cf4.parse_file(illegal_conf4_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 1U);
+
+  std::string illegal_conf5_f(next_tempfile(illegal_conf5));
+  ConfFile cf5;
+  ASSERT_EQ(cf5.parse_file(illegal_conf5_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 1U);
+}
+
+TEST(ConfUtils, EscapingFiles) {
+  std::deque err;
+  std::ostringstream warn;
+  std::string escaping_conf_1_f(next_tempfile(escaping_conf_1));
+  ConfFile cf1;
+  std::string val;
+  ASSERT_EQ(cf1.parse_file(escaping_conf_1_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+
+  ASSERT_EQ(cf1.read("global", "log file", val), 0);
+  ASSERT_EQ(val, "the \"scare quotes\"");
+  ASSERT_EQ(cf1.read("global", "pid file", val), 0);
+  ASSERT_EQ(val, "a pid file");
+  ASSERT_EQ(cf1.read("mon", "keyring", val), 0);
+  ASSERT_EQ(val, "nested \"quotes\"");
+
+  std::string escaping_conf_2_f(next_tempfile(escaping_conf_2));
+  ConfFile cf2;
+  ASSERT_EQ(cf2.parse_file(escaping_conf_2_f.c_str(), &err, &warn), 0);
+  ASSERT_EQ(err.size(), 0U);
+
+  ASSERT_EQ(cf2.read("apple ][", "log file", val), 0);
+  ASSERT_EQ(val, "floppy disk");
+  ASSERT_EQ(cf2.read("mon", "keyring", val), 0);
+  ASSERT_EQ(val, "backslash\\");
+}
+
+TEST(ConfUtils, Overrides) {
+  md_config_t conf;
+  std::deque err;
+  std::ostringstream warn;
+  std::string override_conf_1_f(next_tempfile(override_config_1));
+
+  conf.name.set(CEPH_ENTITY_TYPE_MON, "0");
+  conf.parse_config_files(override_conf_1_f.c_str(), &err, &warn, 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(conf.log_file, "global_log");
+
+  conf.name.set(CEPH_ENTITY_TYPE_MDS, "a");
+  conf.parse_config_files(override_conf_1_f.c_str(), &err, &warn, 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(conf.log_file, "mds_log");
+
+  conf.name.set(CEPH_ENTITY_TYPE_OSD, "0");
+  conf.parse_config_files(override_conf_1_f.c_str(), &err, &warn, 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(conf.log_file, "osd0_log");
+}
+
+TEST(ConfUtils, DupKey) {
+  md_config_t conf;
+  std::deque err;
+  std::ostringstream warn;
+  std::string dup_key_config_f(next_tempfile(dup_key_config_1));
+
+  conf.name.set(CEPH_ENTITY_TYPE_MDS, "a");
+  conf.parse_config_files(dup_key_config_f.c_str(), &err, &warn, 0);
+  ASSERT_EQ(err.size(), 0U);
+  ASSERT_EQ(conf.log_file, string("3"));
+}
+
+
diff --git a/ceph/src/test/crush/TestCrushWrapper.cc b/ceph/src/test/crush/TestCrushWrapper.cc
new file mode 100644
index 00000000..34d6401a
--- /dev/null
+++ b/ceph/src/test/crush/TestCrushWrapper.cc
@@ -0,0 +1,638 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+
+#include "include/stringify.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+#include "include/Context.h"
+#include "osd/osd_types.h"
+
+#include "crush/CrushWrapper.h"
+
+TEST(CrushWrapper, get_immediate_parent) {
+  CrushWrapper *c = new CrushWrapper;
+  
+  const int ROOT_TYPE = 1;
+  c->set_type_name(ROOT_TYPE, "root");
+  const int OSD_TYPE = 0;
+  c->set_type_name(OSD_TYPE, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		ROOT_TYPE, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  int item = 0;
+
+  pair  loc;
+  int ret;
+  loc = c->get_immediate_parent(item, &ret);
+  EXPECT_EQ(-ENOENT, ret);
+
+  {
+    map loc;
+    loc["root"] = "default";
+
+    EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+				"osd.0", loc));
+  }
+
+  loc = c->get_immediate_parent(item, &ret);
+  EXPECT_EQ(0, ret);
+  EXPECT_EQ("root", loc.first);
+  EXPECT_EQ("default", loc.second);
+
+  delete c;
+}
+
+TEST(CrushWrapper, move_bucket) {
+  CrushWrapper *c = new CrushWrapper;
+
+  const int ROOT_TYPE = 2;
+  c->set_type_name(ROOT_TYPE, "root");
+  const int HOST_TYPE = 1;
+  c->set_type_name(HOST_TYPE, "host");
+  const int OSD_TYPE = 0;
+  c->set_type_name(OSD_TYPE, "osd");
+
+  int root0;
+  EXPECT_EQ(0, c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+			     ROOT_TYPE, 0, NULL, NULL, &root0));
+  EXPECT_EQ(0, c->set_item_name(root0, "root0"));
+
+  int item = 0;
+  {
+    map loc;
+    loc["root"] = "root0";
+    loc["host"] = "host0";
+
+    EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+				"osd.0", loc));
+  }
+  int host0 = c->get_item_id("host0");
+
+  int root1;
+  EXPECT_EQ(0, c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+			     ROOT_TYPE, 0, NULL, NULL, &root1));
+  EXPECT_EQ(0, c->set_item_name(root1, "root1"));
+
+  map loc;
+  loc["root"] = "root1";
+
+  // 0 is not a valid bucket number, must be negative
+  EXPECT_EQ(-EINVAL, c->move_bucket(g_ceph_context, 0, loc));
+  // -100 is not an existing bucket
+  EXPECT_EQ(-ENOENT, c->move_bucket(g_ceph_context, -100, loc));
+  // move host0 from root0 to root1
+  {
+    pair  loc;
+    int ret;
+    loc = c->get_immediate_parent(host0, &ret);
+    EXPECT_EQ(0, ret);
+    EXPECT_EQ("root", loc.first);
+    EXPECT_EQ("root0", loc.second);
+  }
+  EXPECT_EQ(0, c->move_bucket(g_ceph_context, host0, loc));
+  {
+    pair  loc;
+    int ret;
+    loc = c->get_immediate_parent(host0, &ret);
+    EXPECT_EQ(0, ret);
+    EXPECT_EQ("root", loc.first);
+    EXPECT_EQ("root1", loc.second);
+  }
+
+  delete c;
+}
+
+TEST(CrushWrapper, check_item_loc) {
+  CrushWrapper *c = new CrushWrapper;
+  int item = 0;
+  float expected_weight = 1.0;
+
+  // fail if loc is empty
+  {
+    float weight;
+    map loc;
+    EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  }
+
+  const int ROOT_TYPE = 2;
+  c->set_type_name(ROOT_TYPE, "root");
+  const int HOST_TYPE = 1;
+  c->set_type_name(HOST_TYPE, "host");
+  const int OSD_TYPE = 0;
+  c->set_type_name(OSD_TYPE, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		ROOT_TYPE, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  // fail because the item is not found at the specified location
+  {
+    float weight;
+    map loc;
+    loc["root"] = "default";
+    EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  }
+  // fail because the bucket name does not match an existing bucket
+  {
+    float weight;
+    map loc;
+    loc["root"] = "default";
+    const string HOST("host0");
+    loc["host"] = HOST;
+    EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  }
+  const string OSD("osd.0");
+  {
+    map loc;
+    loc["root"] = "default";
+    EXPECT_EQ(0, c->insert_item(g_ceph_context, item, expected_weight,
+				OSD, loc));
+  }
+  // fail because osd.0 is not a bucket and must not be in loc, in
+  // addition to being of the wrong type
+  {
+    float weight;
+    map loc;
+    loc["root"] = "osd.0";
+    EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  }
+  // succeed and retrieves the expected weight
+  {
+    float weight;
+    map loc;
+    loc["root"] = "default";
+    EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+    EXPECT_EQ(expected_weight, weight);
+  }
+
+  delete c;
+}
+
+TEST(CrushWrapper, update_item) {
+  CrushWrapper *c = new CrushWrapper;
+
+  const int ROOT_TYPE = 2;
+  c->set_type_name(ROOT_TYPE, "root");
+  const int HOST_TYPE = 1;
+  c->set_type_name(HOST_TYPE, "host");
+  const int OSD_TYPE = 0;
+  c->set_type_name(OSD_TYPE, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		ROOT_TYPE, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  const string HOST0("host0");
+  int host0;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		HOST_TYPE, 0, NULL, NULL, &host0);
+  c->set_item_name(host0, HOST0);
+
+  const string HOST1("host1");
+  int host1;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		HOST_TYPE, 0, NULL, NULL, &host1);
+  c->set_item_name(host1, HOST1);
+
+  int item = 0;
+
+  // fail if invalid names anywhere in loc
+  {
+    map loc;
+    loc["rack"] = "\001";
+    EXPECT_EQ(-EINVAL, c->update_item(g_ceph_context, item, 1.0,
+				      "osd." + stringify(item), loc));
+  }
+  // fail if invalid item name
+  {
+    map loc;
+    EXPECT_EQ(-EINVAL, c->update_item(g_ceph_context, item, 1.0,
+				      "\005", loc));
+  }
+  const string OSD0("osd.0");
+  const string OSD1("osd.1");
+  float original_weight = 1.0;
+  float modified_weight = 2.0;
+  float weight;
+
+  map loc;
+  loc["root"] = "default";
+  loc["host"] = HOST0;
+  EXPECT_GE(0.0, c->get_item_weightf(host0));
+  EXPECT_EQ(0, c->insert_item(g_ceph_context, item, original_weight,
+			      OSD0, loc));
+
+  // updating nothing changes nothing
+  EXPECT_EQ(OSD0, c->get_item_name(item));
+  EXPECT_EQ(original_weight, c->get_item_weightf(item));
+  EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  EXPECT_EQ(0, c->update_item(g_ceph_context, item, original_weight,
+			      OSD0, loc));
+  EXPECT_EQ(OSD0, c->get_item_name(item));
+  EXPECT_EQ(original_weight, c->get_item_weightf(item));
+  EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+
+  // update the name and weight of the item but not the location
+  EXPECT_EQ(OSD0, c->get_item_name(item));
+  EXPECT_EQ(original_weight, c->get_item_weightf(item));
+  EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  EXPECT_EQ(1, c->update_item(g_ceph_context, item, modified_weight,
+			      OSD1, loc));
+  EXPECT_EQ(OSD1, c->get_item_name(item));
+  EXPECT_EQ(modified_weight, c->get_item_weightf(item));
+  EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  c->set_item_name(item, OSD0);
+  c->adjust_item_weightf(g_ceph_context, item, original_weight);
+
+  // update the name and weight of the item and change its location
+  map other_loc;
+  other_loc["root"] = "default";
+  other_loc["host"] = HOST1;
+
+  EXPECT_EQ(OSD0, c->get_item_name(item));
+  EXPECT_EQ(original_weight, c->get_item_weightf(item));
+  EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, other_loc, &weight));
+  EXPECT_EQ(1, c->update_item(g_ceph_context, item, modified_weight,
+			      OSD1, other_loc));
+  EXPECT_EQ(OSD1, c->get_item_name(item));
+  EXPECT_EQ(modified_weight, c->get_item_weightf(item));
+  EXPECT_FALSE(c->check_item_loc(g_ceph_context, item, loc, &weight));
+  EXPECT_TRUE(c->check_item_loc(g_ceph_context, item, other_loc, &weight));
+
+  delete c;
+}
+
+TEST(CrushWrapper, insert_item) {
+  CrushWrapper *c = new CrushWrapper;
+
+  const int ROOT_TYPE = 2;
+  c->set_type_name(ROOT_TYPE, "root");
+  const int HOST_TYPE = 1;
+  c->set_type_name(HOST_TYPE, "host");
+  const int OSD_TYPE = 0;
+  c->set_type_name(OSD_TYPE, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		ROOT_TYPE, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  int item = 0;
+
+  // invalid names anywhere in loc trigger an error
+  {
+    map loc;
+    loc["host"] = "\001";
+    EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+				      "osd." + stringify(item), loc));
+  }
+
+  // insert an item in an existing bucket
+  {
+    map loc;
+    loc["root"] = "default";
+
+    item++;
+    EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+				"osd." + stringify(item), loc));
+    int another_item = item + 1;
+    EXPECT_EQ(-EEXIST, c->insert_item(g_ceph_context, another_item, 1.0,
+				      "osd." + stringify(item), loc));
+  }
+  // implicit creation of a bucket 
+  {
+    string name = "NAME";
+    map loc;
+    loc["root"] = "default";
+    loc["host"] = name;
+
+    item++;
+    EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+				"osd." + stringify(item), loc));
+  }
+  // adding to an existing item name that is not associated with a bucket
+  {
+    string name = "ITEM_WITHOUT_BUCKET";
+    map loc;
+    loc["root"] = "default";
+    loc["host"] = name;
+    item++;
+    c->set_item_name(item, name);
+
+    item++;
+    EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+				      "osd." + stringify(item), loc));
+  }
+  // 
+  //   When there is:
+  //
+  //   default --> host0 --> item
+  //
+  //   Trying to insert the same item higher in the hirarchy will fail
+  //   because it would create a loop.
+  //
+  //   default --> host0 --> item
+  //           |
+  //           +-> item 
+  //
+  {
+    item++;
+    {
+      map loc;
+      loc["root"] = "default";
+      loc["host"] = "host0";
+
+      EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+				  "osd." + stringify(item), loc));
+    }
+    {
+      map loc;
+      loc["root"] = "default";
+
+      EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+					"osd." + stringify(item), loc));
+    }
+  }
+  // 
+  //   When there is:
+  //
+  //   default --> host0
+  //
+  //   Trying to insert default under host0 must fail
+  //   because it would create a loop.
+  //
+  //   default --> host0 --> default
+  //
+  {
+    map loc;
+    loc["host"] = "host0";
+
+    EXPECT_EQ(-ELOOP, c->insert_item(g_ceph_context, rootno, 1.0,
+				     "default", loc));
+  }
+  // fail when mapping a bucket to the wrong type
+  {
+    // create an OSD bucket
+    int osdno;
+    int r = c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+			  10, 0, NULL, NULL, &osdno);
+    ASSERT_EQ(0, r);
+    c->set_item_name(osdno, "myosd");
+    map loc;
+    loc["root"] = "default";
+    // wrongfully pretend the osd is of type host
+    loc["host"] = "myosd";
+
+    item++;
+    EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+				      "osd." + stringify(item), loc));
+  }
+  // fail when no location 
+  {
+    map loc;
+    item++;
+    EXPECT_EQ(-EINVAL, c->insert_item(g_ceph_context, item, 1.0,
+				      "osd." + stringify(item), loc));
+  }
+
+  delete c;
+}
+
+TEST(CrushWrapper, item_bucket_names) {
+  CrushWrapper *c = new CrushWrapper;
+  int index = 123;
+  string name = "NAME";
+  EXPECT_EQ(-EINVAL, c->set_item_name(index, "\001"));
+  EXPECT_EQ(0, c->set_item_name(index, name));
+  EXPECT_TRUE(c->name_exists(name));
+  EXPECT_TRUE(c->item_exists(index));
+  EXPECT_EQ(index, c->get_item_id(name));
+  EXPECT_EQ(name, c->get_item_name(index));
+  delete c;
+}
+
+TEST(CrushWrapper, bucket_types) {
+  CrushWrapper *c = new CrushWrapper;
+  int index = 123;
+  string name = "NAME";
+  c->set_type_name(index, name);
+  EXPECT_EQ(1, c->get_num_type_names());
+  EXPECT_EQ(index, c->get_type_id(name));
+  EXPECT_EQ(name, c->get_type_name(index));
+  delete c;
+}
+
+TEST(CrushWrapper, is_valid_crush_name) {
+  EXPECT_TRUE(CrushWrapper::is_valid_crush_name("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012456789-_"));
+  EXPECT_FALSE(CrushWrapper::is_valid_crush_name(""));
+  EXPECT_FALSE(CrushWrapper::is_valid_crush_name("\001"));
+}
+
+TEST(CrushWrapper, is_valid_crush_loc) {
+  map loc;
+  EXPECT_TRUE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+  loc["good"] = "better";
+  EXPECT_TRUE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+  {
+    map loc;
+    loc["\005"] = "default";
+    EXPECT_FALSE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+  }
+  {
+    map loc;
+    loc["host"] = "\003";
+    EXPECT_FALSE(CrushWrapper::is_valid_crush_loc(g_ceph_context, loc));
+  }
+}
+
+TEST(CrushWrapper, dump_rules) {
+  CrushWrapper *c = new CrushWrapper;
+
+  const int ROOT_TYPE = 1;
+  c->set_type_name(ROOT_TYPE, "root");
+  const int OSD_TYPE = 0;
+  c->set_type_name(OSD_TYPE, "osd");
+
+  string failure_domain_type("osd");
+  string root_name("default");
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		ROOT_TYPE, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, root_name);
+
+  int item = 0;
+
+  pair  loc;
+  int ret;
+  loc = c->get_immediate_parent(item, &ret);
+  EXPECT_EQ(-ENOENT, ret);
+
+  {
+    map loc;
+    loc["root"] = root_name;
+
+    EXPECT_EQ(0, c->insert_item(g_ceph_context, item, 1.0,
+				"osd.0", loc));
+  }
+
+  // no ruleset by default
+  {
+    Formatter *f = new_formatter("json-pretty");
+    c->dump_rules(f);
+    stringstream ss;
+    f->flush(ss);
+    delete f;
+    EXPECT_EQ("", ss.str());
+  }
+
+  string name("NAME");
+  int ruleset = c->add_simple_ruleset(name, root_name, failure_domain_type,
+				      "firstn", pg_pool_t::TYPE_ERASURE);
+  EXPECT_EQ(0, ruleset);
+
+  {
+    Formatter *f = new_formatter("xml");
+    c->dump_rules(f);
+    stringstream ss;
+    f->flush(ss);
+    delete f;
+    EXPECT_EQ((unsigned)0, ss.str().find("0NAME"));
+  }
+
+  {
+    Formatter *f = new_formatter("xml");
+    c->dump_rule(ruleset, f);
+    stringstream ss;
+    f->flush(ss);
+    delete f;
+    EXPECT_EQ((unsigned)0, ss.str().find("0NAME"));
+    EXPECT_NE(string::npos,
+	      ss.str().find("default"));
+  }
+
+  map wm;
+  c->get_rule_weight_osd_map(0, &wm);
+  ASSERT_TRUE(wm.size() == 1);
+  ASSERT_TRUE(wm[0] == 1.0);
+
+  delete c;
+}
+
+TEST(CrushWrapper, distance) {
+  CrushWrapper c;
+  c.create();
+  c.set_type_name(1, "host");
+  c.set_type_name(2, "rack");
+  c.set_type_name(3, "root");
+  int bno;
+  int r = c.add_bucket(0, CRUSH_BUCKET_STRAW,
+		       CRUSH_HASH_DEFAULT, 3, 0, NULL,
+		       NULL, &bno);
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(-1, bno);
+  c.set_item_name(bno, "default");
+
+  c.set_max_devices(10);
+
+  //JSONFormatter jf(true);
+
+  map loc;
+  loc["host"] = "a1";
+  loc["rack"] = "a";
+  loc["root"] = "default";
+  c.insert_item(g_ceph_context, 0, 1, "osd.0", loc);
+
+  loc.clear();
+  loc["host"] = "a2";
+  loc["rack"] = "a";
+  loc["root"] = "default";
+  c.insert_item(g_ceph_context, 1, 1, "osd.1", loc);
+
+  loc.clear();
+  loc["host"] = "b1";
+  loc["rack"] = "b";
+  loc["root"] = "default";
+  c.insert_item(g_ceph_context, 2, 1, "osd.2", loc);
+
+  loc.clear();
+  loc["host"] = "b2";
+  loc["rack"] = "b";
+  loc["root"] = "default";
+  c.insert_item(g_ceph_context, 3, 1, "osd.3", loc);
+
+  vector > ol;
+  c.get_full_location_ordered(3, ol);
+  ASSERT_EQ(3u, ol.size());
+  ASSERT_EQ(make_pair(string("host"),string("b2")), ol[0]);
+  ASSERT_EQ(make_pair(string("rack"),string("b")), ol[1]);
+  ASSERT_EQ(make_pair(string("root"),string("default")), ol[2]);
+
+  //c.dump(&jf);
+  //jf.flush(cout);
+
+  multimap p;
+  p.insert(make_pair("host","b2"));
+  p.insert(make_pair("rack","b"));
+  p.insert(make_pair("root","default"));
+  ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 0, p));
+  ASSERT_EQ(3, c.get_common_ancestor_distance(g_ceph_context, 1, p));
+  ASSERT_EQ(2, c.get_common_ancestor_distance(g_ceph_context, 2, p));
+  ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
+  ASSERT_EQ(-ENOENT, c.get_common_ancestor_distance(g_ceph_context, 123, p));
+
+  // make sure a "multipath" location will reflect a minimal
+  // distance for both paths
+  p.insert(make_pair("host","b1"));
+  ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 2, p));
+  ASSERT_EQ(1, c.get_common_ancestor_distance(g_ceph_context, 3, p));
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  vector def_args;
+  def_args.push_back("--debug-crush=0");
+  global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 unittest_crush_wrapper && 
+ *    valgrind \
+ *    --max-stackframe=20000000 --tool=memcheck \
+ *    ./unittest_crush_wrapper --log-to-stderr=true --debug-crush=20 \
+ *        # --gtest_filter=CrushWrapper.insert_item"
+ * End:
+ */
diff --git a/ceph/src/test/crush/indep.cc b/ceph/src/test/crush/indep.cc
new file mode 100644
index 00000000..896e58fb
--- /dev/null
+++ b/ceph/src/test/crush/indep.cc
@@ -0,0 +1,262 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank 
+ *
+ * LGPL2.1 (see COPYING-LGPL2.1) or later
+ */
+
+#include 
+#include 
+
+#include "include/stringify.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+#include "crush/CrushWrapper.h"
+
+#include 
+
+CrushWrapper *build_indep_map(CephContext *cct, int num_rack, int num_host,
+			      int num_osd)
+{
+  CrushWrapper *c = new CrushWrapper;
+  c->create();
+
+  c->set_type_name(5, "root");
+  c->set_type_name(4, "row");
+  c->set_type_name(3, "rack");
+  c->set_type_name(2, "chasis");
+  c->set_type_name(1, "host");
+  c->set_type_name(0, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		5, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  map loc;
+  loc["root"] = "default";
+
+  int osd = 0;
+  for (int r=0; rinsert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
+      }
+    }
+  }
+
+  crush_rule *rule = crush_make_rule(4, 0, 123, 1, 20);
+  assert(rule);
+  crush_rule_set_step(rule, 0, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
+  crush_rule_set_step(rule, 1, CRUSH_RULE_TAKE, rootno, 0);
+  crush_rule_set_step(rule, 2,
+		      CRUSH_RULE_CHOOSELEAF_INDEP,
+		      CRUSH_CHOOSE_N,
+		      1);
+  crush_rule_set_step(rule, 3, CRUSH_RULE_EMIT, 0, 0);
+  int rno = crush_add_rule(c->crush, rule, -1);
+  c->set_rule_name(rno, "data");
+
+  if (false) {
+    Formatter *f = new_formatter("json-pretty");
+    f->open_object_section("crush_map");
+    c->dump(f);
+    f->close_section();
+    f->flush(cout);
+    delete f;
+  }
+
+  return c;
+}
+
+int get_num_dups(const vector& v)
+{
+  std::set s;
+  int dups = 0;
+  for (unsigned i=0; i weight(c->get_max_devices(), 0x10000);
+  c->dump_tree(weight, &cout, NULL);
+
+  for (int x = 0; x < 100; ++x) {
+    vector out;
+    c->do_rule(0, x, out, 5, weight);
+    cout << x << " -> " << out << std::endl;
+    int num_none = 0;
+    for (unsigned i=0; i weight(c->get_max_devices(), 0x10000);
+  c->dump_tree(weight, &cout, NULL);
+
+  for (int x = 0; x < 100; ++x) {
+    vector out;
+    c->do_rule(0, x, out, 5, weight);
+    cout << x << " -> " << out << std::endl;
+    int num_none = 0;
+    for (unsigned i=0; i weight(c->get_max_devices(), 0x10000);
+
+  // mark a bunch of osds out
+  int num = 3*3*3;
+  for (int i=0; idump_tree(weight, &cout, NULL);
+
+  // need more retries to get 9/9 hosts for x in 0..99
+  c->crush->choose_total_tries = 100;
+  for (int x = 0; x < 100; ++x) {
+    vector out;
+    c->do_rule(0, x, out, 9, weight);
+    cout << x << " -> " << out << std::endl;
+    int num_none = 0;
+    for (unsigned i=0; i weight(c->get_max_devices(), 0x10000);
+
+  // mark a bunch of osds out
+  int num = 3*3*3;
+  for (int i=0; idump_tree(weight, &cout, NULL);
+
+  c->crush->choose_total_tries = 100;
+  for (int x = 0; x < 100; ++x) {
+    vector out;
+    c->do_rule(0, x, out, 7, weight);
+    cout << x << " -> " << out << std::endl;
+    int num_none = 0;
+    for (unsigned i=0; icrush->choose_total_tries = 100;
+  vector<__u32> tweight(c->get_max_devices(), 0x10000);
+  c->dump_tree(tweight, &cout, NULL);
+
+  int tchanged = 0;
+  for (int x = 1; x < 5; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+    std::map pos;
+    vector prev;
+    for (unsigned i=0; i out;
+      c->do_rule(0, x, out, 7, weight);
+      cout << "(" << i << "/" << weight.size() << " out) "
+	   << x << " -> " << out << std::endl;
+      int num_none = 0;
+      for (unsigned k=0; k args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/crypto.cc b/ceph/src/test/crypto.cc
new file mode 100644
index 00000000..24d5c5a4
--- /dev/null
+++ b/ceph/src/test/crypto.cc
@@ -0,0 +1,148 @@
+#include 
+#include 
+
+#include "include/types.h"
+#include "auth/Crypto.h"
+#include "common/ceph_crypto.h"
+
+#include "test/unit.h"
+
+class CryptoEnvironment: public ::testing::Environment {
+public:
+  void SetUp() {
+    ceph::crypto::init(g_ceph_context);
+  }
+};
+
+::testing::Environment* const crypto_env = ::testing::AddGlobalTestEnvironment(new CryptoEnvironment);
+
+TEST(AES, ValidateSecret) {
+  CryptoHandler *h = g_ceph_context->get_crypto_handler(CEPH_CRYPTO_AES);
+  int l;
+
+  for (l=0; l<16; l++) {
+    bufferptr bp(l);
+    int err;
+    err = h->validate_secret(bp);
+    EXPECT_EQ(-EINVAL, err);
+  }
+
+  for (l=16; l<50; l++) {
+    bufferptr bp(l);
+    int err;
+    err = h->validate_secret(bp);
+    EXPECT_EQ(0, err);
+  }
+}
+
+TEST(AES, Encrypt) {
+  CryptoHandler *h = g_ceph_context->get_crypto_handler(CEPH_CRYPTO_AES);
+  char secret_s[] = {
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+  };
+  bufferptr secret(secret_s, sizeof(secret_s));
+
+  unsigned char plaintext_s[] = {
+    0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+    0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+  };
+  bufferlist plaintext;
+  plaintext.append((char *)plaintext_s, sizeof(plaintext_s));
+
+  bufferlist cipher;
+  std::string error;
+  h->encrypt(secret, plaintext, cipher, error);
+  ASSERT_EQ(error, "");
+
+  unsigned char want_cipher[] = {
+    0xb3, 0x8f, 0x5b, 0xc9, 0x35, 0x4c, 0xf8, 0xc6,
+    0x13, 0x15, 0x66, 0x6f, 0x37, 0xd7, 0x79, 0x3a,
+    0x11, 0x90, 0x7b, 0xe9, 0xd8, 0x3c, 0x35, 0x70,
+    0x58, 0x7b, 0x97, 0x9b, 0x03, 0xd2, 0xa5, 0x01,
+  };
+  char cipher_s[sizeof(want_cipher)];
+
+  ASSERT_EQ(sizeof(cipher_s), cipher.length());
+  cipher.copy(0, sizeof(cipher_s), &cipher_s[0]);
+
+  int err;
+  err = memcmp(cipher_s, want_cipher, sizeof(want_cipher));
+  ASSERT_EQ(0, err);
+}
+
+TEST(AES, Decrypt) {
+  CryptoHandler *h = g_ceph_context->get_crypto_handler(CEPH_CRYPTO_AES);
+  char secret_s[] = {
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+  };
+  bufferptr secret(secret_s, sizeof(secret_s));
+
+  unsigned char cipher_s[] = {
+    0xb3, 0x8f, 0x5b, 0xc9, 0x35, 0x4c, 0xf8, 0xc6,
+    0x13, 0x15, 0x66, 0x6f, 0x37, 0xd7, 0x79, 0x3a,
+    0x11, 0x90, 0x7b, 0xe9, 0xd8, 0x3c, 0x35, 0x70,
+    0x58, 0x7b, 0x97, 0x9b, 0x03, 0xd2, 0xa5, 0x01,
+  };
+  bufferlist cipher;
+  cipher.append((char *)cipher_s, sizeof(cipher_s));
+
+  unsigned char want_plaintext[] = {
+    0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+    0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+  };
+  char plaintext_s[sizeof(want_plaintext)];
+
+  std::string error;
+  bufferlist plaintext;
+  h->decrypt(secret, cipher, plaintext, error);
+  ASSERT_EQ(error, "");
+
+  ASSERT_EQ(sizeof(plaintext_s), plaintext.length());
+  plaintext.copy(0, sizeof(plaintext_s), &plaintext_s[0]);
+
+  int err;
+  err = memcmp(plaintext_s, want_plaintext, sizeof(want_plaintext));
+  ASSERT_EQ(0, err);
+}
+
+TEST(AES, Loop) {
+  int err;
+
+  char secret_s[16];
+  err = get_random_bytes(secret_s, sizeof(secret_s));
+  ASSERT_EQ(0, err);
+  bufferptr secret(secret_s, sizeof(secret_s));
+
+  char orig_plaintext_s[1024];
+  err = get_random_bytes(orig_plaintext_s, sizeof(orig_plaintext_s));
+  ASSERT_EQ(0, err);
+
+  bufferlist plaintext;
+  plaintext.append(orig_plaintext_s, sizeof(orig_plaintext_s));
+
+  for (int i=0; i<10000; i++) {
+    bufferlist cipher;
+    {
+      CryptoHandler *h = g_ceph_context->get_crypto_handler(CEPH_CRYPTO_AES);
+
+      std::string error;
+      h->encrypt(secret, plaintext, cipher, error);
+      ASSERT_EQ(error, "");
+    }
+    plaintext.clear();
+
+    {
+      CryptoHandler *h = g_ceph_context->get_crypto_handler(CEPH_CRYPTO_AES);
+      std::string error;
+      h->decrypt(secret, cipher, plaintext, error);
+      ASSERT_EQ(error, "");
+    }
+  }
+
+  char plaintext_s[sizeof(orig_plaintext_s)];
+  plaintext.copy(0, sizeof(plaintext_s), &plaintext_s[0]);
+  err = memcmp(plaintext_s, orig_plaintext_s, sizeof(orig_plaintext_s));
+  ASSERT_EQ(0, err);
+}
diff --git a/ceph/src/test/crypto_init.cc b/ceph/src/test/crypto_init.cc
new file mode 100644
index 00000000..d4b25621
--- /dev/null
+++ b/ceph/src/test/crypto_init.cc
@@ -0,0 +1,48 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/types.h"
+#include "common/code_environment.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "include/msgr.h"
+#include "gtest/gtest.h"
+#include "auth/Crypto.h"
+#include "common/ceph_crypto.h"
+
+#ifdef USE_NSS
+void *init_crypto(void *p) {
+  ceph::crypto::init(g_ceph_context);
+  return NULL;
+}
+
+// Tests for a race condition in libnss when calling crypto_init
+// multiple times simultaneously from different threads.
+TEST(CRYPTO_INIT, NSS_RACE) {
+  std::vector args;
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  // Most reliably reproduced with more threads than cores.
+  long n_thread = sysconf(_SC_NPROCESSORS_ONLN) * 2;
+  pthread_t *ts = (pthread_t*)malloc(n_thread * sizeof(pthread_t));
+  int i;
+  for (i = 0; i < n_thread; i++) {
+    pthread_create(&ts[i], NULL, init_crypto, NULL);
+  }
+  for (i = 0; i < n_thread; i++) {
+    int k;
+    void *p = (void*)&k;
+    pthread_join(ts[i], &p);
+  }
+  free(ts);
+}
+
+#endif
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/daemon_config.cc b/ceph/src/test/daemon_config.cc
new file mode 100644
index 00000000..3e6dc492
--- /dev/null
+++ b/ceph/src/test/daemon_config.cc
@@ -0,0 +1,336 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/rados/librados.h"
+#include "test/unit.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using std::string;
+
+TEST(DaemonConfig, SimpleSet) {
+  int ret;
+  ret = g_ceph_context->_conf->set_val("num_client", "21");
+  ASSERT_EQ(ret, 0);
+  g_ceph_context->_conf->apply_changes(NULL);
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  char *tmp = buf;
+  ret = g_ceph_context->_conf->get_val("num_client", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("21"), string(buf));
+}
+
+TEST(DaemonConfig, Substitution) {
+  int ret;
+  ret = g_ceph_context->_conf->set_val("internal_safe_to_start_threads", "false");
+  ret = g_ceph_context->_conf->set_val("host", "foo");
+  ASSERT_EQ(ret, 0);
+  ret = g_ceph_context->_conf->set_val("public_network", "bar$host.baz", false);
+  ASSERT_EQ(ret, 0);
+  g_ceph_context->_conf->apply_changes(NULL);
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  char *tmp = buf;
+  ret = g_ceph_context->_conf->get_val("public_network", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("barfoo.baz"), string(buf));
+}
+
+TEST(DaemonConfig, SubstitutionTrailing) {
+  int ret;
+  ret = g_ceph_context->_conf->set_val("internal_safe_to_start_threads", "false");
+  ret = g_ceph_context->_conf->set_val("host", "foo");
+  ASSERT_EQ(ret, 0);
+  ret = g_ceph_context->_conf->set_val("public_network", "bar$host", false);
+  ASSERT_EQ(ret, 0);
+  g_ceph_context->_conf->apply_changes(NULL);
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  char *tmp = buf;
+  ret = g_ceph_context->_conf->get_val("public_network", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("barfoo"), string(buf));
+}
+
+TEST(DaemonConfig, SubstitutionBraces) {
+  int ret;
+  ret = g_ceph_context->_conf->set_val("internal_safe_to_start_threads", "false");
+  ret = g_ceph_context->_conf->set_val("host", "foo");
+  ASSERT_EQ(ret, 0);
+  ret = g_ceph_context->_conf->set_val("public_network", "bar${host}baz", false);
+  ASSERT_EQ(ret, 0);
+  g_ceph_context->_conf->apply_changes(NULL);
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  char *tmp = buf;
+  ret = g_ceph_context->_conf->get_val("public_network", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("barfoobaz"), string(buf));
+}
+TEST(DaemonConfig, SubstitutionBracesTrailing) {
+  int ret;
+  ret = g_ceph_context->_conf->set_val("internal_safe_to_start_threads", "false");
+  ret = g_ceph_context->_conf->set_val("host", "foo");
+  ASSERT_EQ(ret, 0);
+  ret = g_ceph_context->_conf->set_val("public_network", "bar${host}", false);
+  ASSERT_EQ(ret, 0);
+  g_ceph_context->_conf->apply_changes(NULL);
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  char *tmp = buf;
+  ret = g_ceph_context->_conf->get_val("public_network", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("barfoo"), string(buf));
+}
+
+// config: variable substitution happen only once http://tracker.ceph.com/issues/7103
+TEST(DaemonConfig, SubstitutionMultiple) {
+  int ret;
+  ret = g_ceph_context->_conf->set_val("mon_host", "localhost", false);
+  ASSERT_EQ(ret, 0);
+  ret = g_ceph_context->_conf->set_val("keyring", "$mon_host/$cluster.keyring,$mon_host/$cluster.mon.keyring", false);
+  ASSERT_EQ(ret, 0);
+  g_ceph_context->_conf->apply_changes(NULL);
+  char buf[512];
+  memset(buf, 0, sizeof(buf));
+  char *tmp = buf;
+  ret = g_ceph_context->_conf->get_val("keyring", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("localhost/ceph.keyring,localhost/ceph.mon.keyring"), tmp);
+  ASSERT_TRUE(strchr(buf, '$') == NULL);
+}
+
+TEST(DaemonConfig, ArgV) {
+  ASSERT_EQ(0, g_ceph_context->_conf->set_val("internal_safe_to_start_threads",
+				       "false"));
+
+  int ret;
+  const char *argv[] = { "foo", "--num-client", "22",
+			 "--keyfile", "/tmp/my-keyfile", NULL };
+  size_t argc = (sizeof(argv) / sizeof(argv[0])) - 1;
+  vector args;
+  argv_to_vec(argc, argv, args);
+  g_ceph_context->_conf->parse_argv(args);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  char buf[128];
+  char *tmp = buf;
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("keyfile", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("/tmp/my-keyfile"), string(buf));
+
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("num_client", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("22"), string(buf));
+
+  ASSERT_EQ(0, g_ceph_context->_conf->set_val("internal_safe_to_start_threads",
+				       "true"));
+}
+
+TEST(DaemonConfig, InjectArgs) {
+  int ret;
+  std::string injection("--num-client 56 --max-open-files 42");
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, 0);
+
+  char buf[128];
+  char *tmp = buf;
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("max_open_files", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("42"), string(buf));
+
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("num_client", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("56"), string(buf));
+
+  injection = "--num-client 57";
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, 0);
+  ret = g_ceph_context->_conf->get_val("num_client", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("57"), string(buf));
+}
+
+TEST(DaemonConfig, InjectArgsReject) {
+  int ret;
+  char buf[128];
+  char *tmp = buf;
+  char buf2[128];
+  char *tmp2 = buf2;
+
+  // We should complain about the garbage in the input
+  std::string injection("--random-garbage-in-injectargs 26 --num-client 28");
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, -EINVAL); 
+
+  // But, debug should still be set...
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("num_client", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("28"), string(buf));
+
+  // What's the current value of osd_data?
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("osd_data", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+
+  // Injectargs shouldn't let us change this, since it is a string-valued
+  // variable and there isn't an observer for it.
+  std::string injection2("--osd_data /tmp/some-other-directory --num-client 4");
+  ret = g_ceph_context->_conf->injectargs(injection2, &cout);
+  ASSERT_EQ(ret, -ENOSYS); 
+
+  // It should be unchanged.
+  memset(buf2, 0, sizeof(buf2));
+  ret = g_ceph_context->_conf->get_val("osd_data", &tmp2, sizeof(buf2));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string(buf), string(buf2));
+}
+
+TEST(DaemonConfig, InjectArgsBooleans) {
+  int ret;
+  char buf[128];
+  char *tmp = buf;
+
+  // Change log_to_syslog
+  std::string injection("--log_to_syslog --num-client 28");
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, 0);
+
+  // log_to_syslog should be set...
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("log_to_syslog", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("true"), string(buf));
+
+  // Turn off log_to_syslog
+  injection = "--log_to_syslog=false --num-client 28";
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, 0);
+
+  // log_to_syslog should be cleared...
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("log_to_syslog", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("false"), string(buf));
+
+  // Turn on log_to_syslog
+  injection = "--num-client 1 --log_to_syslog=true --max-open-files 40";
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, 0);
+
+  // log_to_syslog should be set...
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("log_to_syslog", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("true"), string(buf));
+
+  // parse error
+  injection = "--num-client 1 --log_to_syslog=falsey --max-open-files 42";
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, -EINVAL);
+
+  // log_to_syslog should still be set...
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("log_to_syslog", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("true"), string(buf));
+
+  // debug-ms should still become 42...
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("max_open_files", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("42"), string(buf));
+}
+
+TEST(DaemonConfig, InjectArgsLogfile) {
+  int ret;
+  char tmpfile[PATH_MAX];
+  const char *tmpdir = getenv("TMPDIR");
+  if (!tmpdir)
+    tmpdir = "/tmp";
+  snprintf(tmpfile, sizeof(tmpfile), "%s/daemon_config_test.%d",
+	   tmpdir, getpid());
+  std::string injection("--log_file ");
+  injection += tmpfile;
+  // We're allowed to change log_file because there is an observer.
+  ret = g_ceph_context->_conf->injectargs(injection, &cout);
+  ASSERT_EQ(ret, 0);
+
+  // It should have taken effect.
+  char buf[128];
+  char *tmp = buf;
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("log_file", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string(buf), string(tmpfile));
+
+  // The logfile should exist.
+  ASSERT_EQ(access(tmpfile, R_OK), 0);
+
+  // Let's turn off the logfile.
+  ret = g_ceph_context->_conf->set_val("log_file", "");
+  ASSERT_EQ(ret, 0);
+  g_ceph_context->_conf->apply_changes(NULL);
+  ret = g_ceph_context->_conf->get_val("log_file", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string(""), string(buf));
+
+  // Clean up the garbage
+  unlink(tmpfile);
+}
+
+TEST(DaemonConfig, ThreadSafety1) {
+  int ret;
+  // Verify that we can't change this, since internal_safe_to_start_threads has
+  // been set.
+  ret = g_ceph_context->_conf->set_val("osd_data", "");
+  ASSERT_EQ(ret, -ENOSYS);
+
+  ASSERT_EQ(0, g_ceph_context->_conf->set_val("internal_safe_to_start_threads",
+				       "false"));
+
+  // Ok, now we can change this. Since this is just a test, and there are no
+  // OSD threads running, we know changing osd_data won't actually blow up the
+  // world.
+  ret = g_ceph_context->_conf->set_val("osd_data", "/tmp/crazydata");
+  ASSERT_EQ(ret, 0);
+
+  char buf[128];
+  char *tmp = buf;
+  memset(buf, 0, sizeof(buf));
+  ret = g_ceph_context->_conf->get_val("osd_data", &tmp, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("/tmp/crazydata"), string(buf));
+
+  ASSERT_EQ(0, g_ceph_context->_conf->set_val("internal_safe_to_start_threads",
+				       "false"));
+  ASSERT_EQ(ret, 0);
+}
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make unittest_daemon_config && ./unittest_daemon_config"
+ * End:
+ */
diff --git a/ceph/src/test/downloads/cram-0.5.0ceph.2011-01-14.tar.gz b/ceph/src/test/downloads/cram-0.5.0ceph.2011-01-14.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7d310fad4e8a9fca38195eaa330b5e8eeee7dccb
GIT binary patch
literal 23497
zcmV()K;OR~iwFpPmM~2M17mVwZ7nb^H7+n?WpHRNGB7bQEif@HF*Gi8VR8WMTzhxh
zHnPvZ<)=WLocbwB)Wde1I*sEvsn5o-kE68PMmZKmLK4>$$?{=I-QMqhe=`6nN>-dW
zkKXNGGm%W;F&GShc>ptTVtZ7nnJu&C$Y^NRYnD~1Srw~Uz4z6g8lIh2i~d?Wty=kS
z{@k;ymesVZ1~Tr|th%+`xF=fo{(>h!}u~0
zYeAUEEfGyk%G@%9kf~!ToPY5GHykH`9z|iCa@s-9^SyLRxFnHuy#7>S5JssN21#`!
zQ`@yuTcBmn1>&5|Ro}=oi>j%#V?Yh&pdN--GPVa2wNg2VZGy!aS>UA%e=*~Ysx2Pe
zuVzVH?Rh~}23Op1%5INCaS8Mj5%xtg*YF+_#P${GPyj!~AP%#L%5vJ4umcyQpn?)3
zHS5C>1}?!^yNcS!ebiNS@Epn*
z4_slUVvX8UNXb!8x-RB{7oePztec`e1n7}B7^WhYzh_=7F^eu~I5D;Wg5F{hF@=C2
za{IoVc$hbg=_$<{I~75gVz^OJ5(-h6B%YpH<>5@I-Lq*r47qtgr~d8uiwl{QplFf66!0
zZz=y9JU0sYzk_mCv)R0p|9{ETh5&we_G|m6)7N0#S6&4{Iyp{=QgYf!4E=;fi=K{``|5Mo=~&7$y-o
z@P+Mrb|Mp9trDQ7LP(caKlWF;&4&6)q3yqpKXA{(N+P@bUBV+{wAp56OKkCfht#G&qx$RN$pYXuP*`}iaoG=bL;cYER)2$_ppO+le4D0)-j
z%D$cXq+>ul1{G2@V%tf*E6I|6%Sb{o1~CF2T!>AekU@c%5-CX=QyNObR2X|HC{Rg;
z()ag^VZ;c(O+cwVHPWn~Rfc$f_cnXa87i&w?)0pEeERxL=e;0Oyy=+qPprjq9i^$l
zY4~30T6=k_0H`gri6$daJeV?LHQhLUBok_fG#ik>OI5v}sU8L~A+4cQFHiCU=p-Yh
zRLs}>ooYf2!357AHzfK-P!vd)?e`N^@=CYSgQi(-gVdz}ifF=+l^zTICDYYBs#YEg
zAxghM{E>{&A7==qLo3W8%+e^+2u`W~6wqgn9VBC@DU=9>hg87dGg65`OjReKIzcF|
z*Foa)Do0c$ie)0>D_J2ORMUZyQl7A&-7+?z0aPxfj4J6Ua=lo$Nh#>i2pX;(V}3%{
zbJ8&OiYjV$ogK7)nvJ$3jW&esAz5!N7!HXfT}`Qo`>)@g%^mWdTiHba&w-X$lF4@31zF4iX=h;^Q%G+Avw
z39%y53UwM~unv+8Jf6mPZ?32YW1Q+{!A(tq7dtuAgZc3!PZK}&v&lbI-GW-5(bdsafeHqzoDT3?;-qsIDE
z`;iNQZ`6@O%FMdZ3qxPp!6$T~Ja6sg3|kKv&5`|8US+GFIYkU&W7LwI*tGqlxO6@t
zB2hVqcBWOK71D)txkM-?(f4S_0L_3?idE*gs{$)gI@G1rfB(;_=sY%zbLb;X$h`WT
z`tf^_T)b5Kw&$C)l+DDvj}P7+fT;#zC*K*Vi{#RaBHG-r1QnMqm3n9CWd*Hl0ovkk
z!j@Xt==5s+)}vjg-rMQzI(Dn)^mcc5YqI&M-tTYsnyq@LXKv)9P33p&dc*D8yRDs_
zhO@J?-Q4YOTaRqF)_>&oTdiifJdf_OrkN~7*I8d96>W|-i~8IDV`}}Z7cHs63-bqB
z_zgrUQhHbmCiqHZCy8{+IZ$!ZDP{4?TuyMcyov(9r-@n48F>k9vdYk3UXw^gM8t7)Mrm_~jJ~yzC1>TbJyl;>WSP>i5#lEf${IEyOZ*Fd0&t`qF
zDDw2Fs90OpuGr-7r%y?@JY~HdTB3AG+meZ6M|jzzJk~vd8`~9qvD&HCIyI|Pt9NRR
zPOaIgwK}!!PHhK?lz|)z=@v39BwI+YBfU;F>YaeNe}#5-Lqb=G&II5lfHwia2_Q}Y
zasrqW0G$Bpq_*2h@!iA6!AIiL?@Z9v1Z_>Mx>y6c>({ZY1B*Jar~|(`FsTER1~6#=
zlLjG={08zH$ZsIOf&3=&n>z)TSU5SB=+gv!npnUL{hFX(6KfZrNBG$I06z=(SuQ@n
z(E<)L%;@tA#MwYwNGzm!whJR=0I8A85n4LKkhLygP*?3+i`}z2Fbu;^U=_I1(CbO4
z=-Vm%%wILA&C&o^dDAaE-={N*ko0GVP7V@DYB!sbVweS&IhHhq>b0$w*u?jCK80X!
zO0v2J8Es9&%7+0)#>3J{#K;%u7x)w(HGdTErmx?NKxrU~Q&>5t
zsQo=S#S(Tf&j_NAq6vF+FmH6+F^xN>amO@nWE!O7M0QGtzXNq%xN&Vgd<~|e8G_|L
zlC55=zSFN+{a(NBKI&PHRkz$)t!34Aoq5$^bhUJmUxQt!gx)nKqIb)R4N{siOZ1GK
zqL@vc03=-%e`+>zwVaK7SpBg9iO=46L)-K(U!TL
zI`Y|e&KxhA-Z7&)W^~7lz9BREn3;5Kb@9S>XFdS`;YL)O92WGRwd(AewazoD_@U0V
z6sMvS_kB|{oMqgmsHMF_T|S4dc;(|=KBD7-r31PL_xYG387c=Inzk#djH1$tu%3Kh
zSD6zp)d#Z0$=wjFR%}ib*z2g??F-#;!8ehx
zWOsEN+$!L9F@gSBZtn)X%^P9YO0XX)U%1YFQR$bV=Y-E;8qaxSE`ieQ`Q9J0?DL>$
z+x~4IbNMkX01CEf|E3%uo->Cp=cpu=d;03_$?G@!&H5}?C2{u4^P{s1?ba~|qMT7W
zS2$WY6V^l6sn9>nylOeSW2T*?m}Z?#`nRkMk)u?QMt5h+YKTq#UKrn%hp!G^zkEx6
zvMT*>A-nzCisDQ!PudF%peUmVq|1GfPS0Z@9;yerzwSGHN;c?;4ShbS`UNIX{IwKpuZg^)E7AZR0RM(z?_5{?I)
zBg+z`gD}m1}xFbv!G_e3xAeAh7e&z|ue3NhGq6|QPV)4YU3
z9;_%)SyF`au!I58_-@fFG@8D1-0*$LGZf;3okLe2uU;=c??gIRlzzv3ad8n%9oreo
ziwh1`Rv3z5I`W+>=3Gq<>nieEH{@;Kkhk-}yoPDrkXOH+
z$H7(~=%y;5=3_o8dp8vZC5yPQ@4qMd6+dIi8vH`49e^W4&Kbr7aa7;7_Cz`k^Ei{9
z>{Gyq7m$*}Q8E-lfP#%03c@6SWF$qvw}VU8&s1Jr0SV-o6A4Z_mK1Z51&)e9r#NFH
zlF=@DLw`t?4+1o{Pe^Q01N2o#I`Ec|xjN9S=&@l$)4mT@o)8HAy0jERZWi$M6AB9zz}dYpb(@Mk_+fadZfQsY(!Eo@DWiOXsv-&#weOjjDo!uX&?;Yv
zkt0PPyu+lS=)s{ZGW6!K7tBIrIo5)1tI%bDD;mqtaa9;7{sjGy-aU>IO
zoLzX*z1U;Ri30k*MnpCnlM7MV%BHzph_$M7bC@+fOGB;mjIR-HtzhYp>t%pH5_YYs
z@J55`({01Asn#s+S;Ht|pXs?x#8+NrSp@W#L_vR9B=qf~p}!y^`fEf*e{N*-XGTYV
zl?dt2jFSF>Na-()mNtkVsp-Pugs7NuvwM8m(TodIW#W$Au9VQK|QW+%qJx8y!
zmOJnil8S{Edh`~6yP;2qR0*_PzPO05B(mwyE9N_g_--H-*i1NlD%e2`-I&hMFqN(Y
zH;9q=n6snIPrXW^#iK}~E{Xz?^{qfRx5a2WAzv3or`8$@1y$xkVIx1&RiMaf$>9`z
zr3Hg7Udk!trVEM^YXD7c-VYklVH1FZKXvI^q6&+wl5xLEy}DYaXZBa8XnwMiFZWqox{`7yld44np0b#|m(TmYrq
zltyO+HJjt_bCk-!6#BuGui#=4uarq`Nh8}d#YLeO?ulWVMtjxjC{Hml*@?Zfoox=Z!5qIS5I=rkIZ)B>cAR@LF*s63YaK9#`-3+ml&jp1EAxQ;;>umcGtDgi7#^&
z)a@qVjYK;--wgYz*Wd~-!s0A+i%gWUx|WcM1hRL%RITEh!@MA5_(B4L6f1_np)mwf;2m3D-
zmdZ7NC8s0}SCzcMS!f1o&R3{s$Mp4w_8L0q1W%RkQ0Y9An^0$y{GfONZst&=45EX|
zZ|_T&K?(672cXEtHxuN?Zre8owU?F)iyn1~Id8$!99fXGq0#Lw8A^yc!}x`)ZE&4c
zx|5Q10|H9xMJ=^0W=ZO1%YS7$TbB0{5?(&}-KT1ACFJXR#6oko(kteNoUjDzP_Zra5aWHgbmTJ!5fdQ)&osNsrOpr*R4q}w&fR4_$p
zHRmg|xNb$1u3OPJ>WK0+I>xcuuSw|m$EMTitB|h~og-atrWX3=>|~q=w;gI*Qw@c>
z^d?gmwJLjt!I6kfwhGy!9YQc7>cSmBUuOTABBvD3#hhH_{V*e(UE888DT4SRc*-+H
zG1Qc8G_qk>gIZ>ODhrCORDE8dY(#BXu{GE2wolHUAD`1~!u!$5S*~<*jy4Fm!?S6_
zHm8hN6ISlYi=sR81~3&^ayn8#Umac?ym~bQo4X^;!$!|}L>Q&ti2EnAtd}S4B2#-A
zEtDWRk{4iKrDDX(?nd)X(mgpWvDEFbIrref!py&dMVEXMoryI12AmB&gzf)r@7vqj
zIFh~3zh|EJL(G!+l|+Qh#dZ?BB$2V~u))9qoU9jEGDrhVENQeel0m!gp3nZ)ud2GI
z=OXUO=GjHN4$@5DySnaGEekno;i|k=%;ZIvRj6%CxeUH-mXDK{V}o6Ce+{eonu>h(
zUg{kS=gdu264P67#TKIyPqP(Xq@HjzKiaHlFX#l>4+H|v8LZ+X^Ei5DFj?oMIGDt*
z+cQgBVIak|SDr3>Y!6dI6|b5=xCNYOuNo!`fKI1UhT^>$KD+@in2mpMMAj
zQQ!X|N~5r^qPX-UPGoj2+^efuf)0fyO^-Va&DaFDF}wC3@$e%*YYoHkQ+H9H`}IEs
z!FTC0Z!rf}>5j5)q4TV}!i-qq?QjMeuV5Ml?FLZM
zT2E3QIZ2Y|VIukET{8?*jtR{+6Q!Fz;Z%q@3pT|%kNAY(;3F1216+mXSXLda3+1%BQ)Ov0Eu>Y#wJOwXIW1mUZO1G-61bC)8-CKQt2!)t8f+76O3T!~
zvbe;gYqoeG2JZIfpH-d_j>J{f{`}X&-sc86(600(38!!@Ql;shYpm)C+(80>KlL>>
zM<6w7)TZcAg_kn1LHL9y_sey>S5>IYp>N!!t;es2P%QG_j$4cJDe3U4-YJBn<#
zClORq`XDZYgCj~14;7A1;&t#MQETDM8e#=^lsYt*vF
z96pMdpimVGnkDQ52%AytSW$c4Q8&%34!#`sNYql7$OJ}qX9mKNL?bpTolg2{ltN}!
z_ATgxQBDbkTT_?Lhwue#Eigr}#{nI$ZkVvXnYL{Ve8rb!MA>r#>y{W~hB4JGjB^iD
zSIcEB5?KKl4R}!;{t#3{gRje>|mAKqu3svjnhzQ@Ayh36zo$0N_DFze*(X
z=dLiHqI~`o4SqgW8;yvsSjHctL77S28;6|Fb2CoQnMq}ZZ=tbUF#E--+#YwrUD%dV
zJnGXE(cC*!X~j^b#hXzjdy7rj6DgO^2JP2XCRv5ElaTOS6yv<9F-f4y9$Migz=t#1
zlfz=H(s^XW0U>HM;(JL%YGQ8(+Vcc%asvkQlE|l5nHU*bFP(|`eib&%5cYEBU%aq+_RBoDZP_8Ai^!gjOLBjjqv}QCF3*TA)1`2w15jitz*>
zLd5b;encW}H}aiTqin0qDr>UDZZM7&fuwmy0+)Zy?)+hNTXgX$NHkWSIRRo3)0iWL
zlpKPLz>)#-vI6qb*Ea*?E8vQihJx*GzNxvvEZuT28$;qzXr+^|9|kG-M(;JB*b%$9
z=1e)p`Xu%jS}3*z_p;*bTAM};X>zVIPcNhKJ7;=E>wO)n(V!{ev{S=$n`ZbRiGIs_
z8;%WODmwW}Mf0>ZnjYsG$Q-j`v`ycNA
zYiRht4@%a8Xz{!aGa@NmCg{MWPRFADzO&zHYkeDweREk1J)2wWk-oAWpI>bq4T
zudHUG$L-1Z99I)$e7s+02L8Jo8{hEeP`KACSWB3VB1%bgGx7iTDL^XeiY7%aB$)Gq
ziT`SHx)ol$feL`W?m`ToZ;34tql5WzjmZ-_Q5)$+FoZ6u&s>{YL6I?~Y5)N-Pwc!n
z=l4SimpMk&wIy!xgSFm@#TYHKc~)#}fE96Srz(9+nhE^6q&MJ83FcTk+#U4%Jy}%t
z1*Qf#+&imC9eTt?@QovBeePtHh~D5071%l3UePjqtQaV9
zvU}*&M{|dt`%g0rG+ombhZGc+
zMC*mkWRNuj}mG9)5CwBHtslwY}Bx=l=TnoLlyi6~SxEi;EtPER;X-$6#uY
zRdC|y8>yA1O{Tz(U2NlrM4CC-y4R9YlaMrV&{e}RfaGdtL(Qk-c(k_6cOyB!&ZE=u
zMbsOg(+{~ujb3^lkV5e+`fENf)|~gBME`ewc)PH%^=A9mj{ja+e!l#o$p1ak|KI5I
zPVYEgP$j{tztmdvwt|61v3T}*6_9v)6GBbMvC-D72Dc9T`aT|n3nS(>W;tbghV#fS
zM$<Sk8PZ
zgXYZ3YbtwgBlixb^R1`dZG!5m$8!H5JJa0#gY8U9GlT9*Q@C{yUa5tL4azg0TyS1x
zpLYw^%j#Yqv}ZmugZJD54;93hwdn-&&Fbg|L48?O8{C(z!7YRQD)&Ky{$}n`J_yi!
zZg>4#Wz#c*10Nzt@L_@l?-n%p0KtO~8AN#BV8VL_6+UQi;XQ*4A0XK9A%YG|2V!;*
zVyWnDf)Xn#KTMFK`Q(EY@7CS5!HX8wW(P4=^!xxpjpj3x_})L*(R}U@^f+_1?-c}T
zJ`WNMX+94W6q#{-*91r2W}|K%ENNMrI|NPMce7>$QQD(%n_$YS!}0*Zl^IERh#*Vz
zxqYx@)%30izAP2LmATTbbl0Fvw~Yr4&U738ZGtpQi)p{u;LVJFs%5=mzaAo}vvObW
z9OPMQEf@4zoKy9@ZV(Jw$thO^hnC9SHdu5eKF#gk3?;!mf>JFJe30N&7t7v1NHs_8?-Q(ATD7|euU3Ml4Q6#V>lQ(+=5zbt
zR`Zz|hW^NbEoAKB)f_h6mJw%YN
zi-ztU?3+#DA%cE0+4R1_zcVTLJ%WKVA)*`J|k$kvhY3txL?q=KIz`;1z?qS5SCncrOQs
z&py@n2o}#XY_|*=pE;uY1(91nhMTcU4-;Ho(d>PL&FyC1Ht4)+igyn}zXBDOgVL|Z
zg||Y6*9WQF&Av~tdWHiZAZXouZ1DOG5Bcqb*)0ORaZq~|61;bid(J#|-(dHwvBMby
zzJCzB`8+@{d^X&N2#U8p8P~Ad_Y9VI27SMvc`G2^CwRW<1YaLSU$HKC4yvE!oN%`w
z`x%VJ0|ech&jSSCTQ$Sog79Y$(fbGGTfNfVgY#!{6}Ji2pWVoPgZItH1o2lJ_#QB*
z-);B$;QlM9eL2|wcGUiXg8waBbh9J?Rfqe2NddBB_yEZPEcSblBmpG?Qc4zZ)#RkDIx
zVufPzf-9UaoTLU7z6!IwHSU`1pwg0Y^W+C*D?@cMgvxH5?Y6zLM!+N*jsy~W!&9~~|(9xg5&E-oJ~t{g5tJ6wEzxcK65@k@G$7xZR{o-fgh
zC3?I>&zI@>GHNV~gOd0pxh*bJjb*B_Ol>Vwt!1jULbX%h2F2w`xScs
zjNU){Qd)#dllM~+XN<9^{?wfdxfr~h@78i|0858McBxvD)*N3hJho0wa#`D(sNYvH
zE>7U6P4X6r!#xr0&nLr=)R}knF59!0xm*1{&p(tazv}s4rZZnW|0~ZP&;M`n
z**C$dQ206e!8ui_&Dm@W(JURK_tj#h&X8Nkl!WXvRHYU4ZjhRj4>I-cf3^6SnCv$`
z|Lj3@9|UmS`FHp~bN*MB9`FDC-R%GH)4kCD_59!Bv%>yAgShax|G&X!{XFb`tX7b_
z{L|d$@PS0g{d(>pJ4%)xtG{IF#r|LL=&t<)fB^GbzEqME4i{^6uG+&BX%#Z^l6a~sX0{qv;P7ERrZwX`hDa?||V?7qW
z@1n3Yy8fM?|JHfb>xILo_d5T#lm7|-FF(rvzq9?9KX)Jn*V6yz%LV-Z
z;>+b1kNE#L=l|7v&|&#&sju7q$aepyo`0cazaIa$^yP~Z|G)J7G5_0d^4XKDZN`h%
z<@rQqS5(|(A&52q|M^`GgplY$zSZ+xDf@^vN$u1d{y#j)@#ISvQt=!ApQeb>>+R0c
zp_!3UDG6Yd@2z>&)V(E+s-SM0_YV(k4gViYi~khG+o8PpmR`*BW5bVmo8o(^M*GY{
zk1kltCWsICLsO!n*lfV77@*hDv?ikeeQ=%i;UzV3No`zGBbU_5B{g$N?Oakrm((alF{+%9PQu~+G{^ioL&p`qYtoWveX(%-C%Ox5L4gQh_f4TG({rj5!1@w;w
zv_u11>d`+M)Djk6hH`iQ@9rev_Wa-TXJ4*7;{Qkf@A30H;QvGm@+<29sL$g0r@xQq
z|F`&XW*P*^KOTk0^Iv`bT@B~2bN*L~{vR(^9__!s-T9x%VLu*^|KRh#zP|7N#u_~Lf9I=k&nf9L(H&5d=R{&lwYI-V48
z9P^i({+}kp&|muc>m`!7*W=MN;Z#WV^#(os>gy)I^4}z3m^sS6i6_GzM{zg(jbT?E
zw4Z;?p&Oy(%<-=mB5$vZa18JG;3AmvO-SA;
zEfdE}#D03t<&hU((q2U4mhZok;Ls#M;wd=AmmR0gWd|(78SiXr@N5z!0sRi;(xt1q
z!aL85d0~MgR*=+ol4$k~Go2{qvOG})FGZ4NIZ0bSI{|q!e7Yr)lQYNg`e;7c9F%w&
zl#t+=t>CaH$!w;Yuy`bH+y2ElvOEQR6-3JV#)wmXNvHy^UVSefgj#JDMCK;PKKvYl
zkM2q6ep&sY?Hl&hSseP&4Quo5ES7@JG-QQB#QJ<^Wu~3cRj3MjVOX
zl2gWY3*d|hWIkX`3u+E{A*|kkJZh&+AYsR$_a=k`0}LS?5>tkasQ?UpixSUl4bE^_
z1tm`Y^l>=C-3?0(rb|L5^Q(?T>hTNjcgeHM5gXcwjnfj1;lW1|3YroMM=zY58bd>HJzr~s@p+0ZQWt_~
zxVmM*SQ78?HZrnPn$D4Xv?b9n=r^eknG%Q$Ed-JF0n^eX?oGNf3??ErQwsdNPy5*V
zc$jO;Q+iZGNZLd)*9f@0i8y?|$(^|92}2s6bDIC5BqqaJj){cu?3Fl<@Olh7OK=C=
z${6HR97z;nFW}rOJPRUV1{^>cISnl@x}}_$L^SM0pQ7G`WBTJ0iO}axEI-qP*()U2
z?DANaaJy1#;j#{uo3p(o?N+JXejwdQoSx|gls>}41FoAdjf4;P0Gl*?^@pf
z@vdc#5qa6!eYfYgw_f|}+gqwZoGPrS6uMj_Unx|8|!U6
zz~GJhbOo!V;K++jGY~$8{in?hJaRpu7hq{Xdy#2FBd9qasX;sEF=7?{G?*%DJ0K2J
zqTEe30dHdIkF25LldIW_mMr_62GPyo;yzlqrpG*s9X^uxxr$Nr(!RuLeHi(
zZx9AZP$jnyIB(Df+NGgJp8~$}gXdEd)b~3#kYZAk#$CR^W?vQggkho}Az={OoQMf6L6=9!-BQ5GDEO-DKiBo;zeB1T*^&x#Q@THa5>
z;P@8Ek_p&tw3VV4LwD>XapeS~t0hU;iX(Uffz|M8+Dt8V`50`k%UpwPMlyjRZa(To
z6Jo?PvuDQtdRUrK)SbkWv@g8}Bf;eayU6y7gH8k{9-2Gbc#v)>FFR?9d2|xpeh>{L
zEsLT2FXieXgdz{e+GtcBhhY`px
z9p7s~Q!$85`|+Hd5>9E_D!WS}=`i!Ed<>!5^J$8-fC|@gn2oq5Jqd$1)m8%K6EzQ>
z@O+j%a`-P0-Af~x8>X2dXE}o$86`Ut
zSYujSURatVj<pn#Bni(l68RHVo@-m);QiAzwwz2{VIUnq_IsrB2bwOEN=8CN$m$O?k=J
zDu+Fbg*muvqWAl_SHE$sp$-T-rld!d7AZ&q77X=*3aU<P3
zFdvE>#`7yNo}|yx2eW0LI(S-=Uz&ykv+$v(K*nJ!G%|$+YuEV1J8XSu@?)IY^pD&^
z@xUk<4f}mVztEh|!fxz;io%NmPUKcH4zJ$1?1p;%X_Ut*a+#FTIhs8<(K|Rm>zJfl
zNRY)BEx|Ci(0VNWQUFgE)7iH{r4aQ7W)bPN|87SMD=A9xRM{
z5qJT5FE@LJs(8^%M_NUgp_xas$#h9GNzoZdW888-&F4?e_zp?oM5hmMh7MBY_Xf1a
zpF&A^Hq4zt*;#Nrd_>xUIFMhKfY}?
zNCowMp~G-0(ULZdP#6z$Sjfw-fGB`7b%@A$=2kmlhEuul
z&oB}mwl>jj43jxwY9ZQGmZT5FZ2x;n>H>cK7DqO({
z3a!achc{R^uS>1PgCL3eQ%Q=!v>aJ^W`R9V%yKta4x(y
z<;Fc;drxv`%Hm*;C4
z)WAX=#By4fM6U|lo17Txa3ZTutP41ooPq2#Ln@*Xld-T_P_|=WDW~2G@2RFXFPU))
z$d=f1I61u?v}ok)(U;ym?p?Yx_L$PRaMNbooAl+*AS1PqWY?dhESUsp8h0b35Thf2
zYc6{sa;n-3a7=l@!$~v}i!undXXqCO7xlACs3YfgK{NeqMl{RsB}aCXc{|Ts#TpuB
zwpp5wdtBJW2dbg^I9T)KeoUk4u}rO7tq!qW)P>qIt)f?EWH5Ies41GEVI=+@%+;D&KG=0a
z%@}2&ffDo?cAJ?|Zj4?`n9onWDz-9*C1Iohe>^!mcW}GSLCQ+b!H9$=$En0=q@XTx
z76VoT-+z|z41gk01c|za+fi)8fCZydQ
zd(7ipFgF;_?0$Cdz@M-y2@F=;f&nTLVIy;P&x&Dr_8gi81`EwbA&Xo^XC+A-)y&S@
z3kqR4G)E_B?Qh0&Axbj$8#^#QI=oE~Z-9%<07xE$p^N^1I_V2$=|=$xkIe2pm#s76
zohxkuBuC?d43*gb=YFKXMoDOXi`gu-WR4D>C
zR5X=&2wt$Om#^pvRFjf}
zj(HvhiR??-A`jbfW8K9
z&}xBGbyJ}7Hd4$snw|rN%nb87G9UnU>gxZ(cOCMmi1TBPgv4djafYcIhEBuPkB-S$
zIeptj;fyrJ+$Luk?Ivwzt$DKNWhy=US{Ul=*@5%rUJ
zN@Buvfn5rm!|m`1rWZnf!uH1Oh7(%@o0?c8hjo7{YJ2;O#7O2UXdW^}#ypbtOk6$z
z3^XdUynKO)CUMWxIWBerWC)9T0-|NrLA=RV3}1LS$LLd1NYPiMFE+P0u_xV74rRz(
zo{(N{0*A57@z~uc=}rbIV{O53CqX|$;i22Sa#(NCZ{RqYw8>x`qG>gq&sfRyfH4f0r<
zEhS&(JIAds!ZP-ABQ#b%^`yb1&k}9XxM5}jc5aJ>>2F&7suwx;9-OGvwrdH>h*k1f
zFyDX2`xwS>wYDxf-WmKlaIZ<%Tq$8BMHVMv!cW)+3Gcqrkiwp4pq){d$|UTKsQ4U7
z{($sTMx5-|(sg9EfS4t&^iq|mlaV#|vJ=(QUObcy=n?hlv1ci}TH!W8Q~)!`
zuwymj%aD-{Qru6hzl5SerG$cr&*O-B!u`UjaluqlsnbBw3$**};B=uv;t4H7_(=}H
zNm!;rT;;WW1t`+qZPu1(SfU%;~ai4(sz&MApV8k~psB{Wg$)Nvanpw77
zIbCpc#*UPHiGUSZ^h$H%D2oA%(Su%3FU6x}r|muqQG9gH7WCW{9j6zfYhsJ@1QFY*
zG{rSD80V|z!U05`Gh`-UK%#(`Erg(sNva*->M7h;CC51s>gwQPV)Ekz*BKBJV1(`r
z5jO>&S_Pw;A5Y@mv{Ie?b&EY=X1bzdrHuClZXIIlt!z1*nm9spb{
zjuZW*>Fr@|ZX0}f0GLGKBOrIA+#E$oWbc#W8!(X)PHISa1V)Iow;@;Pg#==KM)Bfs
z#9deqN-=W~AhHW2GuPS}whH=H=o_(xw8s;gH1MJ+A=me0{-H)T)=TY}d(M|F2@--T
z9fvxlv~xh-aMVgn3vATF7f$f6
z>xG)C{VsQc3rw{@8rz63Op=p~K6vUw5B5Eiq?I+aRKVIJT{L&i$``tubzWa`2uC~!
zamZ4S>48-;rq<(G!!$s|c-_r)odoi|Y%FjYKZ}DtpKiT<)_|R`1#)=q?bsC^j+x)xp
z=l#yszQ5DieYdf{Pp!TB;J0^nsPXoz&5pm>{)twz^XK)>&c6TC_nj?o8_oZ;LF3um
zZ==G-mjBc4#{S0ETW%1JxVsx~zu))2-`;%P*=29tr_?)F^mp33`x~7-kH+!i#_Rl~
z=GuES>^c9Zjs5Sp-|ySOV{+}S5B?7uTd$jbXM-E+{CQ`$vq$sysO^n+G~y1u+}K**
zeE*sqfnQMzTig4zPBb$b=l(V?jVWtdr@>JB-n-5&t>4yu`_;zg26YL~;Wr!mThs;n
z7q?~P>+d()G>P}SJKK95=x?xu)Cev3?#A8^etXZ;h5euRZQBs7F17Kly|vC8RoF;O
z*Z;8n9#n(oz4;miJW~J*)$w0<-gMUYH-4n;rIOUm-urhQUGKepUZBlQf2*@j<8AML
z@b@~qKW?mJ9lYJnPJ4rvA3oB%yJ%&5OK{6_3!6t!()kfE@P2C(v)%3d&wH9r1tg%2
z_FIBBET*&X-cK9UId-VH`Ax1uFS5=5Kwz}(ziWRG$LtRZ9yCnrxt)g!0*VYe+po5<
z2Cry58$2u;2o?le^t%17{kF5$^ehl?*Xm5(^!Ii;>l^q&?+Ns1uQz2?=oJ0Wd+Z`T
z&_?_=Z6~G%IMgk}83e3t85p6yi|4YQ^<98Jz{TeF9)OKH*>C$iV*2~4gVMX5Em|)=
zPVM#e_q%koPy#h*G<)yqux)Jdrg)eypXQC-*XBI&I{RCo}W9lAN=oWD_(V|So`&l8^Ao$k4I-=Z$sB)TU*d|WW>h7
zr!+q_UM1`##cB(?NML_3wZ`Qoo&Dmv+Nh#123h@ZP&G3VuJfzUu3umi`W
zoO06%1+~UvqdqhdaYiLO9tUc2%2*@o%p1GLQt&S<3Z=m*riKyQ+5=OV{rlL)gg0tq
z;X4mau&6|!h|eG~2%o~K+N_9qPnAu}oQ>Iq2TgGs={YM>n6Wd~e&*rlEITkK*CUHi
zIEq;UVpkmY#OT1$jiBqaNa_UnN318;Hy#7d0@PBz(N3E_A=%^9Cz2nCGf}`$
zz+U0sa9jCc>OaC0;T!6Kn*h~krso^!SFb;{vR`h$U)t!#+%60En+1o72j#fh-7~!E
ziYB;yIeeq_)|<^JWq}kDMs7OU%#2~&N-2RIL*GFKg*@+U6Lg0B0_XQ!)HUbm|QeD4$Ba=J)ES9>9x&4(&_vp^~eU2}~v0;zK-*d(&ZP&Nyh|
z$<%fv-a**__=s>m!J*qZHT1Is=QWte*kh26quiDabm7~*`Lx;i-+Gd{UzenlBmylNWiqnizf1Qq>+j6`jH?=^|~|AuGO-r
z-6+#=0Dj#4l?!5>I9QuS7Z(LE(Izo0)W9~JSi-7kJmVeC@tHvTdYul8k1`Ts
zG}yPx?a|Szr`9XQ)5eVoNsz*YaS}B_-92LHO#Dnmt_!a^*F{;i`It%R-!1=csfawrzg=6car#{DRCu3Z*Fex
zP5!c3j=M7V&8pLl++{7IlY@e&vd5;hWX$SYLPX{#@@kKXh^6KM@%TjRi61q;?VH3t
zBZNuV4rI$dnPy+l36=KEFKA)ycOtJ406lMs)q+%YroL>Uzh=MEs7A{Wbrf2CuSO)30H*YDb>eBT?@(8us|cmpX1&RE1L%D5BWp^$U9}uT4JZwd2FG}j8#%k^!W>idKF!J;k
zUUeMQ2!mMKx(NCoOBD*O!mkN@Kpl-VTEYo~zSL{2T18dyd><05Ov2Jj4CXYOVx0#w
zqY;rm{L^BW%%z>S5MQm&#|7H7IQbMU|L
zliJ~5YM2X`)iXln!t8)J4RAlQ7)V%Ohxc;7ybO4^z!H{rWCeNkkVz+_ksJWyu#G4S
zlD7~53BhHaA>iAV-_d(Yji8u_v9*>7+gIxdq;kQ2W{mW|M!MDC$QW;62kZYDJ
zYWhPu-6X#D!y(P9p>Y8zJ*?F*tb2*S2tE8sHrOc?X2RpD~^a3
z0)%i*{(9G}ltLJTFwFMI+t1z>%CD~Z|EOV;YO4$%v@=Oa-?_#=1*QZzD}hP@$#La$
z+IA7>N0H#|yKMOM5S!`OpQH_3YVpMX=bZ7ON-eaKT=-gRIPt}
zO)isb4j58`bquu-QKn;AcSH*`p(~>QoYQ$Y!MBt1$eAA4g$S&GRFu9riAF!}%E}*5
z1FKa3NN(vppf{`3$dTD2d_F7AACgI0HKqm&zy0_EZguA*CiWq-T@0P3#BoN9$52e4
zgyV~ly$yH({)Ffh+=|&zi?zl1QMa77UTHpjm3la_5p=7FDxHj|XKP%Aa>uOnfFs9X
z7tY!;?}%Z@jxR?hHGP3_;-)x2FdR+>i1rBUe~(7I39W1q2{YqCwv3Geurc8Je^|>D
z+s+|E7iFsXnP`qq!Jf%vCM0t*T-InZAu)!U)dpIWk2V^{Xfg9`F>pNvu<`b@8jho?
zp&5eRgVU;na35l8bS;&k)~lN%${Dg-lc?=9lIFy%1>g4xHLfLErB<>vBh7uCVP#c^tmQfn#JyJ$2=B53;|k6p5{#ME
z%!6BbC>2DH#j)pA0!t{}_X=2Q+yJ4*~zUR;xh`X1|)}$k%}f<+YZ==Dj~$Taohs
zv&UNP5DdJGI9V&~ZH8;t5jT~HyM}S+V`U|;{Y&+sP^aZy*dGU?)6boWkPWR`a
z@7#nNg1tGCG1X{HGeli;gF;Q3j5VbxW-rc?crwcNBB^i1(B@$&8?Cc&e9Q%?!+3-c
zrnEk6jthv8Ekkw#|*I`ndkxsV`kNb#$}pW(#hKIxpE$(~p*${x9_Rmt}g(>wBM1rKNtH^(^TX|f7AIBOOsMbN~fEm0jR%v>a$i=baytrP3zUl3WfwOf-B4qN>n
z5Q$bH7|~x62M!a61NegJuIx{`Ez$`>C+|v>|%S7Vhcd!GUVB>W9c-lE{~c
zGrp|RSxV2tet)gT%D42qmd#*yGL$=+i1?zCvgAMsnolv=_v99MM{F?tSpQt
za=l+J-(fKw1e@Lak$9pD2NB5#n!?iZJ4vPBGFeUTx}o9&m1;O|$9!ouXn!5$_dN4>GfVmu7(!esA=<<1jK*$Ec@
zYW}?blne%g9-l^f^(6h=XZUsA#>4FGH||@Zx%46?-jO$6P-$?I@;hy(MAm3rQ5hpSfKJ5D1)(todzfWV7)<5&x2*p9V
z%80YV>De=mK1^XW6Ne;Kh5Rp{!a5uDg2lv|
z33HG(wGr_~fG&vb?z|o+Nf>-g#Zlx?@{SxPvI4pDFB*Zxt8szsDRT-kOIaUhfw+!Y
znk!HP0Hy$yFe?bQQ`FTO#iM$o$eNmBe_+Zb@F!4S<~2=23f5Yc$1jYI-VudK6!o0I
zE5mVJ2F`PBkhD`84ocEc7O(A`27B-%J<DYiM
zDp#0<3XNIIZL51uoMipLwsB;*z^kess6iC5D5G)xZxfi5IqlUjy$E5PM1F)dV5*D_
zmRpSjt)$l*8?0l(exw05EHbuL!Y<7kzblTuloQ_+o>I~p8|xC@(mPFTCY4)$d+1L_
z(18YoH%M+o!6d^dLjonJ1~g!KnvmPjM#dT4zT=GkF05#oBfV&NB?iL4~-L#-&~
z1*xlLE-M?9GIH+v@mYP4o~;Qhfc)abJ(nJ=vEs0J0I8UqJsLugunJxb
z@mReKIJFIIk?`>1J)HGrFG|V_VYX&A2^3y)YMO%d^$I|vw^zcFARyMp6HlsGD<^X^
zi*_K~hGtkeJw3uT_k*6lFj_QYt`@Mk(Ha%IuYfgED!N|garH=f#9roz^IR&oT{q7z
z0?#qY6(FlA%TEiXa3>%tKeA&pOF_bvEKZ2r
zB?U@h6ef9R8W~Uqp**k*KCwRRRsdBHrxHdOcm*WO$`Yz$r&!6$CXtw~vpP%yrWMxkA>hUbBCBoa)3bd
z6w8ide6?w+Cy12McLkypeEAxdz&!XE
z5hX#<(*WB290Sd|5+vxVbf3DZLdDPxEeZ~SMv+IAB^@$~w1QqQ)2L+pIpI(NTOw(z
zOwNSAQg|sL{un4I7~7`k*D;VnfpY0e@^Rhdi0f)Us|OT&&%
z=5&^vyLf1M8c1aZAK_7^ZIw88dk$w4WFDL_p2eZq+OtLjx(!#O0_85xHQ$NZX;VAD
zEUx2|Gtut(%!yX0gqnu+X3cJq!#2uY#a#K;PUZHeD+WQw#Q`@Xv#B&d3thd)GvOx>
zLt3mFigJ38Jl)8()Iv*g`_}A+3SNZaCG@*>BZO32HK=eB4&qN?JtwSOz4MAWW>5SN
z7_OAMcDR^16Nu^oa(moJj$>+6fg^_zT-S)zqs&(cP2k+3Jw$mi!_W!mMUgCatxde27Bfz6JbD
zw`zt_X)wPvy~^j*EQIYk$7fzO6Nj`YeHZd7T8|l&<_>4+`g2u+dpjoI9wr@0PeH<$
zURXZ5U=+=Uj23W{NIn0E8b}RCp+1e<9L6gi^mXI!-*TT8Ot+O~#Bi%_zRXK=NCK-d
zQj_OPVyJi9@78!`ErhTn-B!-S@%R|Ir;m@-uW>#|&Ilgo=RaQHo3qC3q;xF){>L$r
zQd>I_pR%e^wytSC8wRnM#}eRt?LagW(b|kIuWcy@b6l|1W8*SF&G`+jutu*IJQO~L
zQ|aE6ZK-oZy>?tOp8GMTWPDQT
zt5IMu9Jo@Ua`!X>>er3hXzd0UZd8Xu@1z*3cIwXQacr_A)YRFaxVw&tD}>b=4g;-v
zoC3J84>GQ&jZ&a1kwyxu)Z%c`qBf_JM}S#|8>+_MGx-WtEKWkZD3}r&3WH(2hUwIh
zfqYs`Gc*w3N3_t%hXP#R%k|A$*kK8wURziIOUzk_%l94M|j=2R-=Mn
zn@Iw1GXq_idJfRGV!dR6m#XO(Udk!vcNOdbxa=G7zy{jP3FATMaTwvsV^gd?KX1M|
zqDF|5ly^R`fDSOPSsiE2Gaj`H77`Pb)ekT8W>e{a|IkS|I4Zv-HlFJ#WepvKSqah^
zmZ(6AmAJy72}gRCZ7Z}f3%niBq~)kX#YhbtLkT3dqlSrBAcF58agJ7y?5&6oMt%`v
zrohBiuOpe_*-yv03V{Q$^BNXtA9F-?k_Knt8pEc4u)V_>Y>s@mi|n;p5ZMraEbnVZ
z!RJto+QN^F#1{v66Z
zTaBG4@#ZRf_+)1VX7nzz%Io}CMJ9^B>M2Fip_3%ymYwD<$5W2m-b%=cLRliH?tLn~
zPs8+2shmcQc#~W3k;C*u!`NS
z2A~DpTDl;Lvs+G;?>j{VYRu8}k{&un>e1?{F0jI=*a`hsZ0#z+2jMuNk*|p)as>p3
zeGavlXdW4Ri7iC18N(3$)fMAdcKY9X|E|)t%8W{SWQ;Lm
zOpI|CD)!*!YoUppVm}L%mL)p%v62p$yN!@q}dqhh<2uy6?`EvOTjCk}p93pU?owM4IBrX$#n
z*|pscA|GrmH)X}km6m6MCS&l9HT&lip&L&mc6Bg;$134<&06*;bk!ZRc-NE%)~K{m
zFME1tt17wHMEG~%(V1~hKVd3}qNK)8Y9#|kwMwr%VbORkHL~kcu&xVHm2Z$U@T?a%
zti5z%MjWKVOisN~gc{ST|esP{?;0_W)
z)Q`&4JXk!MHAbN%IX?S#s#wmj!E=N^Ebwf48+K
zMZQ|BF7$S{vvaI`wP`GO%BEp1o>+ZFYEx;B4J(SXs)`LwF;~8`uSK6XGjzXfes^)7
zp-EtXGvm8aXIHu)DB5t$SZn-9-N!QvJU_KR6|#y}1~YT;L&q}0)v}TEGyezHyqEZ;
zX?DJ(@>PaUNd?kM24f<1byMDppz{M4k31c;+{ed|&yJ5($CX=E5$QLFD-qU<$rpyLi_*h`SRjp{-=Lk{-<)Dr7mB*V9+Dy2CKr6j|tWO
zea=5|7^4SC0C&y#U-|O+QilJZ;ry?Bxw!Or{{N-<-!|li-IPzho6;@LrD3xdk%9M0
zTisYM!ozC-5R%|>o;YI;X2&UgmUG}t{T_)H?8FJFnUy5`GxJXuk|f5^T4I7^k#tq9
zhpZHZ-T9ts*j^7L7?e`8JN&AFMKePi#|
z$*{?(%u~INN;PSPM&eTw9)^tIxVuTn5*IH{mwJYOjRWeF^pcW5N?;bUb;)@y1p#v_PgNz%0gyAZY4a_0go_V>g+dv(=cApmXnN9Tc~9Ika7
z8vvDArvn=IU2T4C!(8&#p3oq0UmKOv9;jcps7Z}W1-AD#a92bUBNaRSz9O-E{q4?X
z!);~@*^H^&W-vUP0H09(&hQKw_T6IJ_6qPlxyg>Z^*3m+h$my^eO6kprWl~S+wl8g
zbm{Mj$ssGfACG8+^xEd{ch{7j?~A@#>l`&rtUT|?+CL=qRIad#%>SO=Cz$
zaLa_E+8+{p_2Zy-JV^*V&d1}?>eHtVc(A1M)G{~(P*Sa6WDfQD!*J?I8%0JC#
zIL$_*b@XI8kfmBPIo*b5N_RJQz)n2^&xPA;GOY*CkX@i=Mhdmw9{T
zbhT;4R%^8?R*uR%a^{bp$Is*E@$>k3{5*ahKaZcs&*SIu^Z0rEJboTO|Dm7%54od+
IfB=vI0OQ8%(EtDd

literal 0
HcmV?d00001

diff --git a/ceph/src/test/encoding.cc b/ceph/src/test/encoding.cc
new file mode 100644
index 00000000..2662c240
--- /dev/null
+++ b/ceph/src/test/encoding.cc
@@ -0,0 +1,198 @@
+#include "common/config.h"
+#include "include/buffer.h"
+#include "include/encoding.h"
+
+#include "gtest/gtest.h"
+
+template < typename T >
+static void test_encode_and_decode(const T& src)
+{
+  bufferlist bl(1000000);
+  encode(src, bl);
+  T dst;
+  bufferlist::iterator i(bl.begin());
+  decode(dst, i);
+  ASSERT_EQ(src, dst) << "Encoding roundtrip changed the string: orig=" << src << ", but new=" << dst;
+}
+
+TEST(EncodingRoundTrip, StringSimple) {
+  string my_str("I am the very model of a modern major general");
+  test_encode_and_decode < std::string >(my_str);
+}
+
+TEST(EncodingRoundTrip, StringEmpty) {
+  string my_str("");
+  test_encode_and_decode < std::string >(my_str);
+}
+
+TEST(EncodingRoundTrip, StringNewline) {
+  string my_str("foo bar baz\n");
+  test_encode_and_decode < std::string >(my_str);
+}
+
+typedef std::multimap < int, std::string > multimap_t;
+typedef multimap_t::value_type my_val_ty;
+
+static std::ostream& operator<<(std::ostream& oss, const multimap_t &multimap)
+{
+  for (multimap_t::const_iterator m = multimap.begin();
+       m != multimap.end();
+       ++m)
+  {
+    oss << m->first << "->" << m->second << " ";
+  }
+  return oss;
+}
+
+TEST(EncodingRoundTrip, Multimap) {
+  multimap_t multimap;
+  multimap.insert( my_val_ty(1, "foo") );
+  multimap.insert( my_val_ty(2, "bar") );
+  multimap.insert( my_val_ty(2, "baz") );
+  multimap.insert( my_val_ty(3, "lucky number 3") );
+  multimap.insert( my_val_ty(10000, "large number") );
+
+  test_encode_and_decode < multimap_t >(multimap);
+}
+
+
+
+///////////////////////////////////////////////////////
+// ConstructorCounter
+///////////////////////////////////////////////////////
+template 
+class ConstructorCounter
+{
+public:
+  ConstructorCounter() : data(0)
+  {
+    default_ctor++;
+  }
+
+  ConstructorCounter(const T& data_)
+    : data(data_)
+  {
+    one_arg_ctor++;
+  }
+
+  ConstructorCounter(const ConstructorCounter &rhs)
+    : data(rhs.data)
+  {
+    copy_ctor++;
+  }
+
+  ConstructorCounter &operator=(const ConstructorCounter &rhs)
+  {
+    data = rhs.data;
+    assigns++;
+    return *this;
+  }
+
+  static void init(void)
+  {
+    default_ctor = 0;
+    one_arg_ctor = 0;
+    copy_ctor = 0;
+    assigns = 0;
+  }
+
+  static int get_default_ctor(void)
+  {
+    return default_ctor;
+  }
+
+  static int get_one_arg_ctor(void)
+  {
+    return one_arg_ctor;
+  }
+
+  static int get_copy_ctor(void)
+  {
+    return copy_ctor;
+  }
+
+  static int get_assigns(void)
+  {
+    return assigns;
+  }
+
+  bool operator<(const ConstructorCounter &rhs) const
+  {
+    return data < rhs.data;
+  }
+
+  bool operator==(const ConstructorCounter &rhs) const
+  {
+    return data == rhs.data;
+  }
+
+  friend void decode(ConstructorCounter &s, bufferlist::iterator& p)
+  {
+    ::decode(s.data, p);
+  }
+
+  friend void encode(const ConstructorCounter &s, bufferlist& p)
+  {
+    ::encode(s.data, p);
+  }
+
+  friend ostream& operator<<(ostream &oss, const ConstructorCounter &cc)
+  {
+    oss << cc.data;
+    return oss;
+  }
+
+  T data;
+private:
+  static int default_ctor;
+  static int one_arg_ctor;
+  static int copy_ctor;
+  static int assigns;
+};
+
+template class ConstructorCounter ;
+template class ConstructorCounter ;
+
+typedef ConstructorCounter  my_key_t;
+typedef ConstructorCounter  my_val_t;
+typedef std::multimap < my_key_t, my_val_t > multimap2_t;
+typedef multimap2_t::value_type val2_ty;
+
+template  int ConstructorCounter::default_ctor = 0;
+template  int ConstructorCounter::one_arg_ctor = 0;
+template  int ConstructorCounter::copy_ctor = 0;
+template  int ConstructorCounter::assigns = 0;
+
+static std::ostream& operator<<(std::ostream& oss, const multimap2_t &multimap)
+{
+  for (multimap2_t::const_iterator m = multimap.begin();
+       m != multimap.end();
+       ++m)
+  {
+    oss << m->first << "->" << m->second << " ";
+  }
+  return oss;
+}
+
+TEST(EncodingRoundTrip, MultimapConstructorCounter) {
+  multimap2_t multimap2;
+  multimap2.insert( val2_ty(my_key_t(1), my_val_t(10)) );
+  multimap2.insert( val2_ty(my_key_t(2), my_val_t(20)) );
+  multimap2.insert( val2_ty(my_key_t(2), my_val_t(30)) );
+  multimap2.insert( val2_ty(my_key_t(3), my_val_t(40)) );
+  multimap2.insert( val2_ty(my_key_t(10000), my_val_t(1)) );
+
+  my_key_t::init();
+  my_val_t::init();
+  test_encode_and_decode < multimap2_t >(multimap2);
+
+  EXPECT_EQ(my_key_t::get_default_ctor(), 5);
+  EXPECT_EQ(my_key_t::get_one_arg_ctor(), 0);
+  EXPECT_EQ(my_key_t::get_copy_ctor(), 10);
+  EXPECT_EQ(my_key_t::get_assigns(), 0);
+
+  EXPECT_EQ(my_val_t::get_default_ctor(), 5);
+  EXPECT_EQ(my_val_t::get_one_arg_ctor(), 0);
+  EXPECT_EQ(my_val_t::get_copy_ctor(), 10);
+  EXPECT_EQ(my_val_t::get_assigns(), 0);
+}
diff --git a/ceph/src/test/encoding/ceph_dencoder.cc b/ceph/src/test/encoding/ceph_dencoder.cc
new file mode 100644
index 00000000..8391f33c
--- /dev/null
+++ b/ceph/src/test/encoding/ceph_dencoder.cc
@@ -0,0 +1,411 @@
+#include 
+#include "include/types.h"
+#include "ceph_ver.h"
+#include "include/encoding.h"
+#include "include/ceph_features.h"
+#include "common/ceph_argparse.h"
+#include "common/Formatter.h"
+#include "common/errno.h"
+#include "msg/Message.h"
+#include "include/assert.h"
+
+#define TYPE(t)
+#define TYPEWITHSTRAYDATA(t)
+#define TYPE_FEATUREFUL(t)
+#define TYPE_NOCOPY(t)
+#define MESSAGE(t)
+#include "types.h"
+#undef TYPE
+#undef TYPEWITHSTRAYDATA
+#undef TYPE_FEATUREFUL
+#undef TYPE_NOCOPY
+#undef MESSAGE
+
+void usage(ostream &out)
+{
+  out << "usage: ceph-dencoder [commands ...]" << std::endl;
+  out << "\n";
+  out << "  version            print version string (to stdout)\n";
+  out << "\n";
+  out << "  import    read encoded data from encfile\n";
+  out << "  export     write encoded data to outfile\n";
+  out << "\n";
+  out << "  set_features   set feature bits used for encoding\n";
+  out << "  get_features        print feature bits (int) to stdout\n";
+  out << "\n";
+  out << "  list_types          list supported types\n";
+  out << "  type     select in-memory type\n";
+  out << "  decode              decode into in-memory object\n";
+  out << "  encode              encode in-memory object\n";
+  out << "  dump_json           dump in-memory object as json (to stdout)\n";
+  out << "\n";
+  out << "  copy                copy object (via operator=)\n";
+  out << "  copy_ctor           copy object (via copy ctor)\n";
+  out << "\n";
+  out << "  count_tests         print number of generated test objects (to stdout)\n";
+  out << "  select_test      select generated test object as in-memory object\n";
+}
+struct Dencoder {
+  virtual ~Dencoder() {}
+  virtual string decode(bufferlist bl, uint64_t seek) = 0;
+  virtual void encode(bufferlist& out, uint64_t features) = 0;
+  virtual void dump(ceph::Formatter *f) = 0;
+  virtual void copy() {
+    cerr << "copy operator= not supported" << std::endl;
+  }
+  virtual void copy_ctor() {
+    cerr << "copy ctor not supported" << std::endl;
+  }
+  virtual void generate() = 0;
+  virtual int num_generated() = 0;
+  virtual string select_generated(unsigned n) = 0;
+  //virtual void print(ostream& out) = 0;
+};
+
+template
+class DencoderBase : public Dencoder {
+protected:
+  T* m_object;
+  list m_list;
+  bool stray_okay;
+
+public:
+  DencoderBase(bool stray_okay) : m_object(new T), stray_okay(stray_okay) {}
+  ~DencoderBase() {
+    delete m_object;
+  }
+
+  string decode(bufferlist bl, uint64_t seek) {
+    bufferlist::iterator p = bl.begin();
+    p.seek(seek);
+    try {
+      m_object->decode(p);
+    }
+    catch (buffer::error& e) {
+      return e.what();
+    }
+    if (!stray_okay && !p.end()) {
+      ostringstream ss;
+      ss << "stray data at end of buffer, offset " << p.get_off();
+      return ss.str();
+    }
+    return string();
+  }
+
+  virtual void encode(bufferlist& out, uint64_t features) = 0;
+
+  void dump(ceph::Formatter *f) {
+    m_object->dump(f);
+  }
+  void generate() {
+    T::generate_test_instances(m_list);
+  }
+  int num_generated() {
+    return m_list.size();
+  }
+  string select_generated(unsigned i) {
+    // allow 0- or 1-based (by wrapping)
+    if (i == 0)
+      i = m_list.size();
+    if ((i == 0) || (i > m_list.size()))
+      return "invalid id for generated object";
+    typename list::iterator p = m_list.begin();
+    for (i--; i > 0 && p != m_list.end(); ++p, --i) ;
+    m_object = *p;
+    return string();
+  }
+};
+
+template
+class DencoderImplNoFeatureNoCopy : public DencoderBase {
+public:
+  DencoderImplNoFeatureNoCopy(bool stray_ok)
+    : DencoderBase(stray_ok) {}
+  virtual void encode(bufferlist& out, uint64_t features) {
+    out.clear();
+    this->m_object->encode(out);
+  }
+};
+
+template
+class DencoderImplNoFeature : public DencoderImplNoFeatureNoCopy {
+public:
+  DencoderImplNoFeature(bool stray_ok)
+    : DencoderImplNoFeatureNoCopy(stray_ok) {}
+  void copy() {
+    T *n = new T;
+    *n = *this->m_object;
+    delete this->m_object;
+    this->m_object = n;
+  }
+  void copy_ctor() {
+    T *n = new T(*this->m_object);
+    delete this->m_object;
+    this->m_object = n;
+  }
+};
+
+template
+class DencoderImplFeatureful : public DencoderBase {
+public:
+  DencoderImplFeatureful(bool stray_ok) : DencoderBase(stray_ok) {}
+  virtual void encode(bufferlist& out, uint64_t features) {
+    out.clear();
+    ::encode(*(this->m_object), out, features);
+  }
+};
+
+
+template
+class MessageDencoderImpl : public Dencoder {
+  T *m_object;
+  list m_list;
+
+public:
+  MessageDencoderImpl() {
+    m_object = new T;
+  }
+  ~MessageDencoderImpl() {
+    m_object->put();
+  }
+
+  string decode(bufferlist bl, uint64_t seek) {
+    bufferlist::iterator p = bl.begin();
+    p.seek(seek);
+    try {
+      Message *n = decode_message(g_ceph_context, p);
+      if (!n)
+	throw std::runtime_error("failed to decode");
+      if (n->get_type() != m_object->get_type()) {
+	stringstream ss;
+	ss << "decoded type " << n->get_type() << " instead of expected " << m_object->get_type();
+	throw std::runtime_error(ss.str());
+      }
+      m_object->put();
+      m_object = static_cast(n);
+    }
+    catch (buffer::error& e) {
+      return e.what();
+    }
+    if (!p.end()) {
+      ostringstream ss;
+      ss << "stray data at end of buffer, offset " << p.get_off();
+      return ss.str();
+    }
+    return string();
+  }
+
+  void encode(bufferlist& out, uint64_t features) {
+    out.clear();
+    encode_message(m_object, features, out);
+  }
+
+  void dump(ceph::Formatter *f) {
+    m_object->dump(f);
+  }
+  void generate() {
+    //T::generate_test_instances(m_list);
+  }
+  int num_generated() {
+    return m_list.size();
+  }
+  string select_generated(unsigned i) {
+    // allow 0- or 1-based (by wrapping)
+    if (i == 0)
+      i = m_list.size();
+    if ((i == 0) || (i > m_list.size()))
+      return "invalid id for generated object";
+    typename list::iterator p = m_list.begin();
+    for (i--; i > 0 && p != m_list.end(); ++p, --i) ;
+    m_object->put();
+    m_object = *p;
+    return string();
+  }
+
+  //void print(ostream& out) {
+  //out << m_object << std::endl;
+  //}
+};
+
+  
+
+int main(int argc, const char **argv)
+{
+  // dencoders
+  map dencoders;
+
+#define T_STR(x) #x
+#define T_STRINGIFY(x) T_STR(x)
+#define TYPE(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeature(false);
+#define TYPEWITHSTRAYDATA(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeature(true);
+#define TYPE_FEATUREFUL(t) dencoders[T_STRINGIFY(t)] = new DencoderImplFeatureful(false);
+#define TYPE_NOCOPY(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeatureNoCopy(false);
+#define MESSAGE(t) dencoders[T_STRINGIFY(t)] = new MessageDencoderImpl;
+#include "types.h"
+#undef TYPE
+#undef TYPEWITHSTRAYDATA
+#undef TYPE_FEATUREFUL
+#undef T_STR
+#undef T_STRINGIFY
+
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  Dencoder *den = NULL;
+  uint64_t features = CEPH_FEATURES_SUPPORTED_DEFAULT;
+  bufferlist encbl;
+  uint64_t skip = 0;
+
+  if (args.empty()) {
+    usage(cerr);
+    exit(1);
+  }
+  for (std::vector::iterator i = args.begin(); i != args.end(); ++i) {
+    string err;
+
+    if (*i == string("help") || *i == string("-h") || *i == string("--help")) {
+      usage(cout);
+      exit(0);
+    } else if (*i == string("version")) {
+      cout << CEPH_GIT_NICE_VER << std::endl;
+    } else if (*i == string("list_types")) {
+      for (map::iterator p = dencoders.begin();
+	   p != dencoders.end();
+	   ++p)
+	cout << p->first << std::endl;
+      exit(0);
+    } else if (*i == string("type")) {
+      ++i;
+      if (i == args.end()) {
+	usage(cerr);
+	exit(1);
+      }
+      string cname = *i;
+      if (!dencoders.count(cname)) {
+	cerr << "class '" << cname << "' unknown" << std::endl;
+	exit(1);
+      }
+      den = dencoders[cname];
+      den->generate();
+    } else if (*i == string("skip")) {
+      ++i;
+      if (i == args.end()) {
+	usage(cerr);
+	exit(1);
+      }
+      skip = atoi(*i);
+    } else if (*i == string("get_features")) {
+      cout << CEPH_FEATURES_SUPPORTED_DEFAULT << std::endl;
+      exit(0);
+    } else if (*i == string("set_features")) {
+      ++i;
+      if (i == args.end()) {
+	usage(cerr);
+	exit(1);
+      }
+      features = atoi(*i);
+
+    } else if (*i == string("encode")) {
+      if (!den) {
+	cerr << "must first select type with 'type '" << std::endl;
+	usage(cerr);
+	exit(1);
+      }
+      den->encode(encbl, features);
+    } else if (*i == string("decode")) {
+      if (!den) {
+	cerr << "must first select type with 'type '" << std::endl;
+	usage(cerr);
+	exit(1);
+      }
+      err = den->decode(encbl, skip);
+    } else if (*i == string("copy_ctor")) {
+      if (!den) {
+	cerr << "must first select type with 'type '" << std::endl;
+	usage(cerr);
+	exit(1);
+      }
+      den->copy_ctor();
+    } else if (*i == string("copy")) {
+      if (!den) {
+	cerr << "must first select type with 'type '" << std::endl;
+	usage(cerr);
+	exit(1);
+      }
+      den->copy();
+    } else if (*i == string("dump_json")) {
+      if (!den) {
+	cerr << "must first select type with 'type '" << std::endl;
+	usage(cerr);
+	exit(1);
+      }
+      JSONFormatter jf(true);
+      jf.open_object_section("object");
+      den->dump(&jf);
+      jf.close_section();
+      jf.flush(cout);
+      cout << std::endl;
+
+    } else if (*i == string("import")) {
+      ++i;
+      if (i == args.end()) {
+	usage(cerr);
+	exit(1);
+      }
+      int r = encbl.read_file(*i, &err);
+      if (r < 0) {
+        cerr << "error reading " << *i << ": " << err << std::endl;
+        exit(1);
+      }
+
+    } else if (*i == string("export")) {
+      ++i;
+      if (i == args.end()) {
+	usage(cerr);
+	exit(1);
+      }
+      int fd = ::open(*i, O_WRONLY|O_CREAT|O_TRUNC, 0644);
+      if (fd < 0) {
+	cerr << "error opening " << *i << " for write: " << cpp_strerror(errno) << std::endl;
+	exit(1);
+      }
+      int r = encbl.write_fd(fd);
+      if (r < 0) {
+	cerr << "error writing " << *i << ": " << cpp_strerror(errno) << std::endl;
+	exit(1);
+      }
+      ::close(fd);
+
+    } else if (*i == string("count_tests")) {
+      if (!den) {
+	cerr << "must first select type with 'type '" << std::endl;
+	usage(cerr);
+	exit(1);
+      }
+      cout << den->num_generated() << std::endl;
+    } else if (*i == string("select_test")) {
+      if (!den) {
+	cerr << "must first select type with 'type '" << std::endl;
+	usage(cerr);
+	exit(1);
+      }
+      ++i;
+      if (i == args.end()) {
+	usage(cerr);
+	exit(1);
+      }
+      int n = atoi(*i);
+      err = den->select_generated(n);      
+    } else {
+      cerr << "unknown option '" << *i << "'" << std::endl;
+      usage(cerr);
+      exit(1);
+    }      
+    if (err.length()) {
+      cerr << "error: " << err << std::endl;
+      exit(1);
+    }
+  }
+  return 0;
+}
diff --git a/ceph/src/test/encoding/check-generated.sh b/ceph/src/test/encoding/check-generated.sh
new file mode 100755
index 00000000..a429c4b5
--- /dev/null
+++ b/ceph/src/test/encoding/check-generated.sh
@@ -0,0 +1,77 @@
+#!/bin/sh -e
+
+dir=$1
+
+set -e
+
+tmp1=`mktemp /tmp/typ-XXXXXXXXX`
+tmp2=`mktemp /tmp/typ-XXXXXXXXX`
+tmp3=`mktemp /tmp/typ-XXXXXXXXX`
+tmp4=`mktemp /tmp/typ-XXXXXXXXX`
+
+failed=0
+numtests=0
+echo "checking ceph-dencoder generated test instances..."
+echo "numgen type"
+for type in `./ceph-dencoder list_types`; do
+    num=`./ceph-dencoder type $type count_tests`
+    echo "$num $type"
+    for n in `seq 1 1 $num 2>/dev/null`; do
+	if ! ./ceph-dencoder type $type select_test $n encode decode; then
+	    echo "**** $type test $n encode+decode check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n encode decode"
+	    failed=$(($failed + 3))
+	    continue
+	fi
+
+	./ceph-dencoder type $type select_test $n dump_json > $tmp1
+	./ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2
+	./ceph-dencoder type $type select_test $n copy dump_json > $tmp3
+	./ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp4
+
+	if ! cmp $tmp1 $tmp2; then
+	    echo "**** $type test $n dump_json check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+	    echo "   ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2"
+	    echo "   diff $tmp1 $tmp2"
+	    failed=$(($failed + 1))
+	fi
+
+	if ! cmp $tmp1 $tmp3; then
+	    echo "**** $type test $n copy dump_json check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+	    echo "   ceph-dencoder type $type select_test $n copy dump_json > $tmp2"
+	    echo "   diff $tmp1 $tmp2"
+	    failed=$(($failed + 1))
+	fi
+
+	if ! cmp $tmp1 $tmp4; then
+	    echo "**** $type test $n copy_ctor dump_json check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+	    echo "   ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp2"
+	    echo "   diff $tmp1 $tmp2"
+	    failed=$(($failed + 1))
+	fi
+
+	./ceph-dencoder type $type select_test $n encode export $tmp1
+	./ceph-dencoder type $type select_test $n encode decode encode export $tmp2
+	if ! cmp $tmp1 $tmp2; then
+	    echo "**** $type test $n binary reencode check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n encode export $tmp1"
+	    echo "   ceph-dencoder type $type select_test $n encode decode encode export $tmp2"
+	    echo "   cmp $tmp1 $tmp2"
+	    failed=$(($failed + 1))
+	fi
+
+
+	numtests=$(($numtests + 3))
+    done
+done
+
+rm -f $tmp1 $tmp2 $tmp3 $tmp4
+
+if [ $failed -gt 0 ]; then
+    echo "FAILED $failed / $numtests tests."
+    exit 1
+fi
+echo "passed $numtests tests."
diff --git a/ceph/src/test/encoding/readable.sh b/ceph/src/test/encoding/readable.sh
new file mode 100755
index 00000000..1f433a55
--- /dev/null
+++ b/ceph/src/test/encoding/readable.sh
@@ -0,0 +1,82 @@
+#!/bin/sh -e
+
+dir=$1
+
+set -e
+
+tmp1=`mktemp /tmp/typ-XXXXXXXXX`
+tmp2=`mktemp /tmp/typ-XXXXXXXXX`
+
+failed=0
+numtests=0
+
+myversion=`./ceph-dencoder version`
+
+for arversion in `ls -v $dir/archive`
+do
+    vdir="$dir/archive/$arversion"
+#    echo $vdir
+
+    if [ ! -d "$vdir/objects" ]; then
+	continue;
+    fi
+
+    for type in `ls $vdir/objects`
+    do
+	if ./ceph-dencoder type $type 2>/dev/null; then
+#	    echo "type $type";
+	    echo "        $vdir/objects/$type"
+
+	    # is there a fwd incompat change between $arversion and $version?
+	    incompat=""
+	    sawarversion=0
+	    for iv in `ls -v $dir/archive`
+	    do
+		if [ "$iv" = "$arversion" ]; then
+		    sawarversion=1
+		fi
+		if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then
+		    incompat="$iv"
+		fi
+		if [ "$iv" = "$version" ]; then
+		    break
+		fi
+	    done
+	    if [ -n "$incompat" ]; then
+		echo "skipping incompat $type version $arversion, changed at $iv < code $myversion"
+		continue
+	    fi
+
+	    for f in `ls $vdir/objects/$type`; do
+#		echo "\t$vdir/$type/$f"
+		if ! ./ceph-dencoder type $type import $vdir/objects/$type/$f decode dump_json > $tmp1; then
+		    echo "**** failed to decode $vdir/objects/$type/$f ****"
+		    failed=$(($failed + 1))
+		    continue	    
+		fi
+		if ! ./ceph-dencoder type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2; then
+		    echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****"
+		    failed=$(($failed + 1))
+		    continue
+		fi
+		if ! cmp $tmp1 $tmp2; then
+		    echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****"
+		    diff $tmp1 $tmp2
+		    failed=$(($failed + 1))
+	    	fi
+		numtests=$(($numtests + 1))
+	    done
+	else
+            echo "skipping unrecognized type $type"
+	fi
+    done
+done
+
+rm -f $tmp1 $tmp2
+
+if [ $failed -gt 0 ]; then
+    echo "FAILED $failed / $numtests tests."
+    exit 1
+fi
+echo "passed $numtests tests."
+
diff --git a/ceph/src/test/encoding/types.h b/ceph/src/test/encoding/types.h
new file mode 100644
index 00000000..7df2b6c1
--- /dev/null
+++ b/ceph/src/test/encoding/types.h
@@ -0,0 +1,528 @@
+#include "include/CompatSet.h"
+TYPE(CompatSet)
+
+#include "include/filepath.h"
+TYPE(filepath)
+
+#include "common/bloom_filter.hpp"
+TYPE(bloom_filter)
+TYPE(compressible_bloom_filter)
+
+#include "common/snap_types.h"
+TYPE(SnapContext)
+TYPE(SnapRealmInfo)
+
+#include "common/DecayCounter.h"
+TYPE(DecayCounter)
+
+#include "common/LogEntry.h"
+TYPE(LogEntryKey)
+TYPE(LogEntry)
+TYPE(LogSummary)
+
+#include "common/SloppyCRCMap.h"
+TYPE(SloppyCRCMap)
+
+#include "msg/msg_types.h"
+TYPE(entity_name_t)
+TYPE(entity_addr_t)
+
+#include "osd/OSDMap.h"
+TYPE(osd_info_t)
+TYPE(osd_xinfo_t)
+TYPEWITHSTRAYDATA(OSDMap)
+TYPEWITHSTRAYDATA(OSDMap::Incremental)
+
+#include "crush/CrushWrapper.h"
+TYPE_NOCOPY(CrushWrapper)
+
+#include "common/histogram.h"
+TYPE(pow2_hist_t)
+
+#include "osd/osd_types.h"
+TYPE(osd_reqid_t)
+TYPE(object_locator_t)
+TYPE(request_redirect_t)
+TYPE(pg_t)
+TYPE(coll_t)
+TYPE(objectstore_perf_stat_t)
+TYPE(osd_stat_t)
+TYPE(OSDSuperblock)
+TYPE_FEATUREFUL(pool_snap_info_t)
+TYPE_FEATUREFUL(pg_pool_t)
+TYPE(object_stat_sum_t)
+TYPE(object_stat_collection_t)
+TYPE(pg_stat_t)
+TYPE_FEATUREFUL(pool_stat_t)
+TYPE(pg_history_t)
+TYPE(pg_info_t)
+TYPE(pg_interval_t)
+TYPE_FEATUREFUL(pg_query_t)
+TYPE(pg_log_entry_t)
+TYPE(pg_log_t)
+TYPE(pg_missing_t::item)
+TYPE(pg_missing_t)
+TYPE(pg_ls_response_t)
+TYPE(object_copy_cursor_t)
+TYPE(object_copy_data_t)
+TYPE(pg_create_t)
+TYPE(watch_info_t)
+TYPE(object_info_t)
+TYPE(SnapSet)
+TYPE(ObjectRecoveryInfo)
+TYPE(ObjectRecoveryProgress)
+TYPE(ScrubMap::object)
+TYPE(ScrubMap)
+TYPE(pg_hit_set_info_t)
+TYPE(pg_hit_set_history_t)
+TYPE(osd_peer_stat_t)
+TYPE(clone_info)
+TYPE(obj_list_snap_response_t)
+TYPE(PullOp)
+TYPE(PushOp)
+TYPE(PushReplyOp)
+
+#include "osd/ECUtil.h"
+TYPE(ECUtil::HashInfo)
+
+#include "osd/ECMsgTypes.h"
+TYPE(ECSubWrite)
+TYPE(ECSubWriteReply)
+TYPE(ECSubRead)
+TYPE(ECSubReadReply)
+
+#include "osd/HitSet.h"
+TYPE(ExplicitHashHitSet)
+TYPE(ExplicitObjectHitSet)
+TYPE(BloomHitSet)
+TYPE(HitSet)
+TYPE(HitSet::Params)
+
+#include "os/ObjectStore.h"
+TYPE(ObjectStore::Transaction)
+
+#include "os/SequencerPosition.h"
+TYPE(SequencerPosition)
+
+#include "common/hobject.h"
+TYPE(hobject_t)
+TYPE(ghobject_t)
+
+#include "mon/AuthMonitor.h"
+TYPE(AuthMonitor::Incremental)
+
+#include "mon/PGMap.h"
+TYPE(PGMap::Incremental)
+TYPE(PGMap)
+
+#include "mon/MonitorDBStore.h"
+TYPE(MonitorDBStore::Transaction)
+TYPE(MonitorDBStore::Op)
+
+#include "mon/MonMap.h"
+TYPE_FEATUREFUL(MonMap)
+
+#include "mon/MonCap.h"
+TYPE(MonCap)
+
+#include "os/DBObjectMap.h"
+TYPE(DBObjectMap::_Header)
+TYPE(DBObjectMap::State)
+
+#include "osdc/Journaler.h"
+TYPE(Journaler::Header)
+
+#include "mds/Anchor.h"
+TYPE(Anchor)
+
+#include "mds/snap.h"
+TYPE(SnapInfo)
+TYPE(snaplink_t)
+TYPE(sr_t)
+
+#include "mds/mdstypes.h"
+TYPE(frag_info_t)
+TYPE(nest_info_t)
+TYPE(client_writeable_range_t)
+TYPE(inode_t)
+TYPE(old_inode_t)
+TYPE(fnode_t)
+TYPE(old_rstat_t)
+TYPE(session_info_t)
+TYPE(string_snap_t)
+TYPE(MDSCacheObjectInfo)
+TYPE(mds_table_pending_t)
+TYPE(inode_load_vec_t)
+TYPE(dirfrag_load_vec_t)
+TYPE(mds_load_t)
+TYPE(cap_reconnect_t)
+TYPE(inode_backtrace_t)
+TYPE(inode_backpointer_t)
+
+#include "mds/MDSMap.h"
+TYPE_FEATUREFUL(MDSMap)
+TYPE_FEATUREFUL(MDSMap::mds_info_t)
+
+#include "mds/Capability.h"
+TYPE_NOCOPY(Capability)
+
+#include "mds/AnchorServer.h"
+TYPEWITHSTRAYDATA(AnchorServer)
+
+#include "mds/InoTable.h"
+TYPE(InoTable)
+
+#include "mds/SnapServer.h"
+TYPEWITHSTRAYDATA(SnapServer)
+
+#include "mds/SessionMap.h"
+TYPE(SessionMap)
+
+#include "mds/events/ECommitted.h"
+TYPE(ECommitted)
+#include "mds/events/EExport.h"
+TYPE(EExport)
+#include "mds/events/EFragment.h"
+TYPE(EFragment)
+#include "mds/events/EImportFinish.h"
+TYPE(EImportFinish)
+#include "mds/events/EImportStart.h"
+TYPE(EImportStart)
+#include "mds/events/EMetaBlob.h"
+TYPE_NOCOPY(EMetaBlob::fullbit)
+TYPE(EMetaBlob::remotebit)
+TYPE(EMetaBlob::nullbit)
+TYPE(EMetaBlob::dirlump)
+TYPE(EMetaBlob)
+#include "mds/events/EOpen.h"
+TYPE(EOpen)
+#include "mds/events/EResetJournal.h"
+TYPE(EResetJournal)
+#include "mds/events/ESession.h"
+TYPE(ESession)
+#include "mds/events/ESessions.h"
+TYPE(ESessions)
+#include "mds/events/ESlaveUpdate.h"
+TYPE(link_rollback)
+TYPE(rmdir_rollback)
+TYPE(rename_rollback::drec)
+TYPE(rename_rollback)
+TYPE(ESlaveUpdate)
+#include "mds/events/ESubtreeMap.h"
+TYPE(ESubtreeMap)
+#include "mds/events/ETableClient.h"
+TYPE(ETableClient)
+#include "mds/events/ETableServer.h"
+TYPE(ETableServer)
+#include "mds/events/EUpdate.h"
+TYPE(EUpdate)
+
+#ifdef WITH_RADOSGW
+
+#include "rgw/rgw_rados.h"
+TYPE(RGWObjManifestPart);
+TYPE(RGWObjManifest);
+
+#include "rgw/rgw_acl.h"
+TYPE(ACLPermission)
+TYPE(ACLGranteeType)
+TYPE(ACLGrant)
+TYPE(RGWAccessControlList)
+TYPE(ACLOwner)
+TYPE(RGWAccessControlPolicy)
+
+#include "rgw/rgw_cache.h"
+TYPE(ObjectMetaInfo)
+TYPE(ObjectCacheInfo)
+TYPE(RGWCacheNotifyInfo)
+
+#include "cls/rgw/cls_rgw_types.h"
+TYPE(rgw_bucket_pending_info)
+TYPE(rgw_bucket_dir_entry_meta)
+TYPE(rgw_bucket_dir_entry)
+TYPE(rgw_bucket_category_stats)
+TYPE(rgw_bucket_dir_header)
+TYPE(rgw_bucket_dir)
+TYPE(rgw_bucket_entry_ver)
+
+#include "cls/rgw/cls_rgw_ops.h"
+TYPE(rgw_cls_obj_prepare_op)
+TYPE(rgw_cls_obj_complete_op)
+TYPE(rgw_cls_list_op)
+TYPE(rgw_cls_list_ret)
+TYPE(cls_rgw_gc_defer_entry_op)
+TYPE(cls_rgw_gc_list_op)
+TYPE(cls_rgw_gc_list_ret)
+TYPE(cls_rgw_gc_obj_info)
+TYPE(cls_rgw_gc_remove_op)
+TYPE(cls_rgw_gc_set_entry_op)
+TYPE(cls_rgw_obj)
+TYPE(cls_rgw_obj_chain)
+TYPE(rgw_cls_tag_timeout_op)
+TYPE(cls_rgw_bi_log_list_op)
+TYPE(cls_rgw_bi_log_trim_op)
+TYPE(cls_rgw_bi_log_list_ret)
+
+#include "cls/rgw/cls_rgw_client.h"
+TYPE(rgw_bi_log_entry)
+
+#include "cls/user/cls_user_types.h"
+TYPE(cls_user_bucket)
+TYPE(cls_user_bucket_entry)
+TYPE(cls_user_stats)
+TYPE(cls_user_header)
+
+#include "cls/user/cls_user_ops.h"
+TYPE(cls_user_set_buckets_op)
+TYPE(cls_user_remove_bucket_op)
+TYPE(cls_user_list_buckets_op)
+TYPE(cls_user_list_buckets_ret)
+TYPE(cls_user_get_header_op)
+TYPE(cls_user_get_header_ret)
+TYPE(cls_user_complete_stats_sync_op)
+
+#include "rgw/rgw_common.h"
+TYPE(RGWAccessKey);
+TYPE(RGWSubUser);
+TYPE(RGWUserInfo)
+TYPE(rgw_bucket)
+TYPE(RGWBucketInfo)
+TYPE(RGWBucketEnt)
+TYPE(RGWUploadPartInfo)
+TYPE(rgw_obj)
+
+#include "rgw/rgw_log.h"
+TYPE(rgw_log_entry)
+TYPE(rgw_intent_log_entry)
+
+#include "cls/rbd/cls_rbd.h"
+TYPE(cls_rbd_parent)
+TYPE(cls_rbd_snap)
+
+#endif
+
+#include "cls/lock/cls_lock_types.h"
+TYPE(rados::cls::lock::locker_id_t)
+TYPE(rados::cls::lock::locker_info_t)
+
+#include "cls/lock/cls_lock_ops.h"
+TYPE(cls_lock_lock_op)
+TYPE(cls_lock_unlock_op)
+TYPE(cls_lock_break_op)
+TYPE(cls_lock_get_info_op)
+TYPE(cls_lock_get_info_reply)
+TYPE(cls_lock_list_locks_reply)
+
+#include "cls/replica_log/cls_replica_log_types.h"
+TYPE(cls_replica_log_item_marker)
+TYPE(cls_replica_log_progress_marker)
+TYPE(cls_replica_log_bound)
+#include "cls/replica_log/cls_replica_log_ops.h"
+TYPE(cls_replica_log_delete_marker_op)
+TYPE(cls_replica_log_set_marker_op)
+TYPE(cls_replica_log_get_bounds_op)
+TYPE(cls_replica_log_get_bounds_ret)
+
+#include "cls/refcount/cls_refcount_ops.h"
+TYPE(cls_refcount_get_op)
+TYPE(cls_refcount_put_op)
+TYPE(cls_refcount_read_op)
+TYPE(cls_refcount_read_ret)
+TYPE(cls_refcount_set_op)
+
+
+// --- messages ---
+#include "messages/MAuth.h"
+MESSAGE(MAuth)
+#include "messages/MAuthReply.h"
+MESSAGE(MAuthReply)
+#include "messages/MCacheExpire.h"
+MESSAGE(MCacheExpire)
+#include "messages/MClientCapRelease.h"
+MESSAGE(MClientCapRelease)
+#include "messages/MClientCaps.h"
+MESSAGE(MClientCaps)
+#include "messages/MClientLease.h"
+MESSAGE(MClientLease)
+#include "messages/MClientReconnect.h"
+MESSAGE(MClientReconnect)
+#include "messages/MClientReply.h"
+MESSAGE(MClientReply)
+#include "messages/MClientRequest.h"
+MESSAGE(MClientRequest)
+#include "messages/MClientRequestForward.h"
+MESSAGE(MClientRequestForward)
+#include "messages/MClientSession.h"
+MESSAGE(MClientSession)
+#include "messages/MClientSnap.h"
+MESSAGE(MClientSnap)
+#include "messages/MCommand.h"
+MESSAGE(MCommand)
+#include "messages/MCommandReply.h"
+MESSAGE(MCommandReply)
+#include "messages/MDentryLink.h"
+MESSAGE(MDentryLink)
+#include "messages/MDentryUnlink.h"
+MESSAGE(MDentryUnlink)
+#include "messages/MDirUpdate.h"
+MESSAGE(MDirUpdate)
+#include "messages/MDiscover.h"
+MESSAGE(MDiscover)
+#include "messages/MDiscoverReply.h"
+MESSAGE(MDiscoverReply)
+#include "messages/MExportCaps.h"
+MESSAGE(MExportCaps)
+#include "messages/MExportCapsAck.h"
+MESSAGE(MExportCapsAck)
+#include "messages/MExportDir.h"
+MESSAGE(MExportDir)
+#include "messages/MExportDirAck.h"
+MESSAGE(MExportDirAck)
+#include "messages/MExportDirCancel.h"
+MESSAGE(MExportDirCancel)
+#include "messages/MExportDirDiscover.h"
+MESSAGE(MExportDirDiscover)
+#include "messages/MExportDirDiscoverAck.h"
+MESSAGE(MExportDirDiscoverAck)
+#include "messages/MExportDirFinish.h"
+MESSAGE(MExportDirFinish)
+#include "messages/MExportDirNotify.h"
+MESSAGE(MExportDirNotify)
+#include "messages/MExportDirNotifyAck.h"
+MESSAGE(MExportDirNotifyAck)
+#include "messages/MExportDirPrep.h"
+MESSAGE(MExportDirPrep)
+#include "messages/MExportDirPrepAck.h"
+MESSAGE(MExportDirPrepAck)
+#include "messages/MForward.h"
+MESSAGE(MForward)
+#include "messages/MGetPoolStats.h"
+MESSAGE(MGetPoolStats)
+#include "messages/MGetPoolStatsReply.h"
+MESSAGE(MGetPoolStatsReply)
+#include "messages/MHeartbeat.h"
+MESSAGE(MHeartbeat)
+#include "messages/MInodeFileCaps.h"
+MESSAGE(MInodeFileCaps)
+#include "messages/MLock.h"
+MESSAGE(MLock)
+#include "messages/MLog.h"
+MESSAGE(MLog)
+#include "messages/MLogAck.h"
+MESSAGE(MLogAck)
+#include "messages/MMDSBeacon.h"
+MESSAGE(MMDSBeacon)
+#include "messages/MMDSCacheRejoin.h"
+MESSAGE(MMDSCacheRejoin)
+#include "messages/MMDSFindIno.h"
+MESSAGE(MMDSFindIno)
+#include "messages/MMDSFindInoReply.h"
+MESSAGE(MMDSFindInoReply)
+#include "messages/MMDSFragmentNotify.h"
+MESSAGE(MMDSFragmentNotify)
+#include "messages/MMDSLoadTargets.h"
+MESSAGE(MMDSLoadTargets)
+#include "messages/MMDSMap.h"
+MESSAGE(MMDSMap)
+#include "messages/MMDSResolve.h"
+MESSAGE(MMDSResolve)
+#include "messages/MMDSResolveAck.h"
+MESSAGE(MMDSResolveAck)
+#include "messages/MMDSSlaveRequest.h"
+MESSAGE(MMDSSlaveRequest)
+#include "messages/MMDSTableRequest.h"
+MESSAGE(MMDSTableRequest)
+#include "messages/MMonCommand.h"
+MESSAGE(MMonCommand)
+#include "messages/MMonCommandAck.h"
+MESSAGE(MMonCommandAck)
+#include "messages/MMonElection.h"
+MESSAGE(MMonElection)
+#include "messages/MMonGetMap.h"
+MESSAGE(MMonGetMap)
+#include "messages/MMonGetVersion.h"
+MESSAGE(MMonGetVersion)
+#include "messages/MMonGetVersionReply.h"
+MESSAGE(MMonGetVersionReply)
+#include "messages/MMonGlobalID.h"
+MESSAGE(MMonGlobalID)
+#include "messages/MMonJoin.h"
+MESSAGE(MMonJoin)
+#include "messages/MMonMap.h"
+MESSAGE(MMonMap)
+#include "messages/MMonPaxos.h"
+MESSAGE(MMonPaxos)
+#include "messages/MMonProbe.h"
+MESSAGE(MMonProbe)
+#include "messages/MMonScrub.h"
+MESSAGE(MMonScrub)
+#include "messages/MMonSync.h"
+MESSAGE(MMonSync)
+#include "messages/MMonSubscribe.h"
+MESSAGE(MMonSubscribe)
+#include "messages/MMonSubscribeAck.h"
+MESSAGE(MMonSubscribeAck)
+#include "messages/MOSDAlive.h"
+MESSAGE(MOSDAlive)
+#include "messages/MOSDBoot.h"
+MESSAGE(MOSDBoot)
+#include "messages/MOSDFailure.h"
+MESSAGE(MOSDFailure)
+#include "messages/MOSDMap.h"
+MESSAGE(MOSDMap)
+#include "messages/MOSDOp.h"
+MESSAGE(MOSDOp)
+#include "messages/MOSDOpReply.h"
+MESSAGE(MOSDOpReply)
+#include "messages/MOSDPGBackfill.h"
+MESSAGE(MOSDPGBackfill)
+#include "messages/MOSDPGCreate.h"
+MESSAGE(MOSDPGCreate)
+#include "messages/MOSDPGInfo.h"
+MESSAGE(MOSDPGInfo)
+#include "messages/MOSDPGLog.h"
+MESSAGE(MOSDPGLog)
+#include "messages/MOSDPGMissing.h"
+MESSAGE(MOSDPGMissing)
+#include "messages/MOSDPGNotify.h"
+MESSAGE(MOSDPGNotify)
+#include "messages/MOSDPGQuery.h"
+MESSAGE(MOSDPGQuery)
+#include "messages/MOSDPGRemove.h"
+MESSAGE(MOSDPGRemove)
+#include "messages/MOSDPGScan.h"
+MESSAGE(MOSDPGScan)
+#include "messages/MOSDPGTemp.h"
+MESSAGE(MOSDPGTemp)
+#include "messages/MOSDPGTrim.h"
+MESSAGE(MOSDPGTrim)
+#include "messages/MOSDPing.h"
+MESSAGE(MOSDPing)
+#include "messages/MOSDRepScrub.h"
+MESSAGE(MOSDRepScrub)
+#include "messages/MOSDScrub.h"
+MESSAGE(MOSDScrub)
+#include "messages/MOSDSubOp.h"
+MESSAGE(MOSDSubOp)
+#include "messages/MOSDSubOpReply.h"
+MESSAGE(MOSDSubOpReply)
+#include "messages/MPGStats.h"
+MESSAGE(MPGStats)
+#include "messages/MPGStatsAck.h"
+MESSAGE(MPGStatsAck)
+#include "messages/MPing.h"
+MESSAGE(MPing)
+#include "messages/MPoolOp.h"
+MESSAGE(MPoolOp)
+#include "messages/MPoolOpReply.h"
+MESSAGE(MPoolOpReply)
+#include "messages/MRemoveSnaps.h"
+MESSAGE(MRemoveSnaps)
+#include "messages/MRoute.h"
+MESSAGE(MRoute)
+#include "messages/MStatfs.h"
+MESSAGE(MStatfs)
+#include "messages/MStatfsReply.h"
+MESSAGE(MStatfsReply)
+#include "messages/MWatchNotify.h"
+MESSAGE(MWatchNotify)
diff --git a/ceph/src/test/erasure-code/ErasureCodeExample.h b/ceph/src/test/erasure-code/ErasureCodeExample.h
new file mode 100644
index 00000000..2b77d51e
--- /dev/null
+++ b/ceph/src/test/erasure-code/ErasureCodeExample.h
@@ -0,0 +1,184 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#ifndef CEPH_ERASURE_CODE_EXAMPLE_H
+#define CEPH_ERASURE_CODE_EXAMPLE_H
+
+#include 
+#include 
+#include 
+#include 
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "erasure-code/ErasureCodeInterface.h"
+
+#define FIRST_DATA_CHUNK 0
+#define SECOND_DATA_CHUNK 1
+#define DATA_CHUNKS 2u
+
+#define CODING_CHUNK 2
+#define CODING_CHUNKS 1u
+
+#define MINIMUM_TO_RECOVER 2u
+
+class ErasureCodeExample : public ErasureCodeInterface {
+public:
+  virtual ~ErasureCodeExample() {}
+
+  virtual int create_ruleset(const string &name,
+			     CrushWrapper &crush,
+			     ostream *ss) const {
+    return crush.add_simple_ruleset(name, "default", "host",
+				    "indep", pg_pool_t::TYPE_ERASURE, ss);
+  }
+  
+  virtual int minimum_to_decode(const set &want_to_read,
+                                const set &available_chunks,
+                                set *minimum) {
+    if (includes(available_chunks.begin(), available_chunks.end(),
+		 want_to_read.begin(), want_to_read.end())) {
+      *minimum = want_to_read;
+      return 0;
+    } else if (available_chunks.size() >= MINIMUM_TO_RECOVER) {
+      *minimum = available_chunks;
+      return 0;
+    } else {
+      return -EIO;
+    }
+  }
+
+  virtual int minimum_to_decode_with_cost(const set &want_to_read,
+                                          const map &available,
+                                          set *minimum) {
+    //
+    // If one chunk is more expensive to fetch than the others,
+    // recover it instead. For instance, if the cost reflects the
+    // time it takes for a chunk to be retrieved from a remote
+    // OSD and if CPU is cheap, it could make sense to recover
+    // instead of fetching the chunk.
+    //
+    map c2c(available);
+    if (c2c.size() > DATA_CHUNKS) {
+      if (c2c[FIRST_DATA_CHUNK] > c2c[SECOND_DATA_CHUNK] &&
+	  c2c[FIRST_DATA_CHUNK] > c2c[CODING_CHUNK])
+	c2c.erase(FIRST_DATA_CHUNK);
+      else if(c2c[SECOND_DATA_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
+	      c2c[SECOND_DATA_CHUNK] > c2c[CODING_CHUNK])
+	c2c.erase(SECOND_DATA_CHUNK);
+      else if(c2c[CODING_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
+	      c2c[CODING_CHUNK] > c2c[SECOND_DATA_CHUNK])
+	c2c.erase(CODING_CHUNK);
+    }
+    set  available_chunks;
+    for (map::const_iterator i = c2c.begin();
+	 i != c2c.end();
+	 ++i)
+      available_chunks.insert(i->first);
+    return minimum_to_decode(want_to_read, available_chunks, minimum);
+  }
+
+  virtual unsigned int get_chunk_count() const {
+    return DATA_CHUNKS + CODING_CHUNKS;
+  }
+
+  virtual unsigned int get_data_chunk_count() const {
+    return DATA_CHUNKS;
+  }
+
+  virtual unsigned int get_chunk_size(unsigned int object_size) const {
+    return ( object_size / DATA_CHUNKS ) + 1;
+  }
+
+  virtual int encode(const set &want_to_encode,
+                     const bufferlist &in,
+                     map *encoded) {
+    //
+    // make sure all data chunks have the same length, allocating
+    // padding if necessary.
+    //
+    unsigned int chunk_length = get_chunk_size(in.length());
+    bufferlist out(in);
+    unsigned int width = get_chunk_count() * get_chunk_size(in.length());
+    bufferptr pad(width - in.length());
+    pad.zero(0, get_data_chunk_count());
+    out.push_back(pad);
+    //
+    // compute the coding chunk with first chunk ^ second chunk
+    //
+    char *p = out.c_str();
+    for (unsigned i = 0; i < chunk_length; i++)
+      p[i + CODING_CHUNK * chunk_length] =
+        p[i + FIRST_DATA_CHUNK * chunk_length] ^
+	p[i + SECOND_DATA_CHUNK * chunk_length];
+    //
+    // populate the bufferlist with bufferptr pointing
+    // to chunk boundaries
+    //
+    const bufferptr ptr = out.buffers().front();
+    for (set::iterator j = want_to_encode.begin();
+         j != want_to_encode.end();
+         ++j) {
+      bufferptr chunk(ptr, (*j) * chunk_length, chunk_length);
+      (*encoded)[*j].push_front(chunk);
+    }
+    return 0;
+  }
+
+  virtual int decode(const set &want_to_read,
+                     const map &chunks,
+                     map *decoded) {
+    //
+    // All chunks have the same size
+    //
+    unsigned chunk_length = (*chunks.begin()).second.length();
+    for (set::iterator i = want_to_read.begin();
+         i != want_to_read.end();
+         ++i) {
+      if (chunks.find(*i) != chunks.end()) {
+	//
+	// If the chunk is available, just copy the bufferptr pointer
+	// to the decoded argument.
+	//
+        (*decoded)[*i] = chunks.find(*i)->second;
+      } else if(chunks.size() != 2) {
+	//
+	// If a chunk is missing and there are not enough chunks
+	// to recover, abort.
+	//
+	return -ERANGE;
+      } else {
+	//
+	// No matter what the missing chunk is, XOR of the other
+	// two recovers it.
+	//
+        map::const_iterator k = chunks.begin();
+        const char *a = k->second.buffers().front().c_str();
+        ++k;
+        const char *b = k->second.buffers().front().c_str();
+        bufferptr chunk(chunk_length);
+	char *c = chunk.c_str();
+        for (unsigned j = 0; j < chunk_length; j++) {
+          c[j] = a[j] ^ b[j];
+        }
+        (*decoded)[*i].push_front(chunk);
+      }
+    }
+    return 0;
+  }
+};
+
+#endif
diff --git a/ceph/src/test/erasure-code/ErasureCodePluginExample.cc b/ceph/src/test/erasure-code/ErasureCodePluginExample.cc
new file mode 100644
index 00000000..c1b5b3c6
--- /dev/null
+++ b/ceph/src/test/erasure-code/ErasureCodePluginExample.cc
@@ -0,0 +1,36 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+
+#include "erasure-code/ErasureCodePlugin.h"
+#include "ErasureCodeExample.h"
+
+class ErasureCodePluginExample : public ErasureCodePlugin {
+public:
+  virtual int factory(const map ¶meters,
+                      ErasureCodeInterfaceRef *erasure_code)
+  {
+    *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample());
+    return 0;
+  }
+};
+
+int __erasure_code_init(char *plugin_name)
+{
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  return instance.add(plugin_name, new ErasureCodePluginExample());
+}
diff --git a/ceph/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc b/ceph/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc
new file mode 100644
index 00000000..d3043a43
--- /dev/null
+++ b/ceph/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc
@@ -0,0 +1,23 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+  return -ESRCH;
+}
diff --git a/ceph/src/test/erasure-code/ErasureCodePluginFailToRegister.cc b/ceph/src/test/erasure-code/ErasureCodePluginFailToRegister.cc
new file mode 100644
index 00000000..c26ac5b6
--- /dev/null
+++ b/ceph/src/test/erasure-code/ErasureCodePluginFailToRegister.cc
@@ -0,0 +1,22 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+  return 0;
+}
diff --git a/ceph/src/test/erasure-code/ErasureCodePluginHangs.cc b/ceph/src/test/erasure-code/ErasureCodePluginHangs.cc
new file mode 100644
index 00000000..01c2fa86
--- /dev/null
+++ b/ceph/src/test/erasure-code/ErasureCodePluginHangs.cc
@@ -0,0 +1,24 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+  sleep(1000);
+  return 0;
+}
diff --git a/ceph/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc b/ceph/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
new file mode 100644
index 00000000..4d8f1baf
--- /dev/null
+++ b/ceph/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
@@ -0,0 +1,4 @@
+// missing int __erasure_code_init(char *plugin_name) {}
+
+// avoid warnings about library containing no symbols
+int __this_is_an_used_variable_to_avoid_warnings;
diff --git a/ceph/src/test/erasure-code/Makefile.am b/ceph/src/test/erasure-code/Makefile.am
new file mode 100644
index 00000000..fdbe003f
--- /dev/null
+++ b/ceph/src/test/erasure-code/Makefile.am
@@ -0,0 +1,115 @@
+check_SCRIPTS += \
+	test/erasure-code/test-erasure-code.sh
+
+ceph_erasure_code_benchmark_SOURCES = \
+	test/erasure-code/ceph_erasure_code_benchmark.cc
+ceph_erasure_code_benchmark_LDADD = $(LIBOSD) $(LIBCOMMON) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
+if LINUX
+ceph_erasure_code_benchmark_LDADD += -ldl
+endif
+bin_DEBUGPROGRAMS += ceph_erasure_code_benchmark
+
+ceph_erasure_code_SOURCES = \
+	test/erasure-code/ceph_erasure_code.cc
+ceph_erasure_code_LDADD = $(LIBOSD) $(LIBCOMMON) $(BOOST_PROGRAM_OPTIONS_LIBS) $(CEPH_GLOBAL)
+if LINUX
+ceph_erasure_code_LDADD += -ldl
+endif
+bin_DEBUGPROGRAMS += ceph_erasure_code
+
+libec_example_la_SOURCES = test/erasure-code/ErasureCodePluginExample.cc
+libec_example_la_CFLAGS = ${AM_CFLAGS}
+libec_example_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_example.la
+
+libec_missing_entry_point_la_SOURCES = test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
+libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS}
+libec_missing_entry_point_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_missing_entry_point.la
+
+libec_hangs_la_SOURCES = test/erasure-code/ErasureCodePluginHangs.cc
+libec_hangs_la_CFLAGS = ${AM_CFLAGS}
+libec_hangs_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_hangs.la
+
+libec_fail_to_initialize_la_SOURCES = test/erasure-code/ErasureCodePluginFailToInitialize.cc
+libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_initialize_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_fail_to_initialize.la
+
+libec_fail_to_register_la_SOURCES = test/erasure-code/ErasureCodePluginFailToRegister.cc
+libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_register_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_fail_to_register.la
+
+libec_test_jerasure_sse4_la_SOURCES = test/erasure-code/TestJerasurePluginSSE4.cc
+libec_test_jerasure_sse4_la_CFLAGS = ${AM_CFLAGS}
+libec_test_jerasure_sse4_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_test_jerasure_sse4_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_test_jerasure_sse4_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_test_jerasure_sse4.la
+
+libec_test_jerasure_sse3_la_SOURCES = test/erasure-code/TestJerasurePluginSSE3.cc
+libec_test_jerasure_sse3_la_CFLAGS = ${AM_CFLAGS}
+libec_test_jerasure_sse3_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_test_jerasure_sse3_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_test_jerasure_sse3_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_test_jerasure_sse3.la
+
+libec_test_jerasure_generic_la_SOURCES = test/erasure-code/TestJerasurePluginGeneric.cc
+libec_test_jerasure_generic_la_CFLAGS = ${AM_CFLAGS}
+libec_test_jerasure_generic_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_test_jerasure_generic_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_test_jerasure_generic_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_test_jerasure_generic.la
+
+unittest_erasure_code_plugin_SOURCES = test/erasure-code/TestErasureCodePlugin.cc 
+unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_plugin_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_plugin
+
+unittest_erasure_code_jerasure_SOURCES = \
+	test/erasure-code/TestErasureCodeJerasure.cc \
+	${jerasure_sources}
+unittest_erasure_code_jerasure_CFLAGS = $(AM_CFLAGS) \
+	-Ierasure-code/jerasure/gf-complete/include \
+	-Ierasure-code/jerasure/jerasure/include
+unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS) \
+	-Ierasure-code/jerasure/gf-complete/include \
+	-Ierasure-code/jerasure/jerasure/include
+unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_jerasure_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_jerasure
+
+unittest_erasure_code_plugin_jerasure_SOURCES = \
+	test/erasure-code/TestErasureCodePluginJerasure.cc
+unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
+unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_plugin_jerasure_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_plugin_jerasure
+
+unittest_erasure_code_example_SOURCES = test/erasure-code/TestErasureCodeExample.cc 
+noinst_HEADERS += test/erasure-code/ErasureCodeExample.h
+unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_erasure_code_example
+
+noinst_HEADERS += \
+	test/erasure-code/ceph_erasure_code_benchmark.h
diff --git a/ceph/src/test/erasure-code/TestErasureCodeExample.cc b/ceph/src/test/erasure-code/TestErasureCodeExample.cc
new file mode 100644
index 00000000..4df07624
--- /dev/null
+++ b/ceph/src/test/erasure-code/TestErasureCodeExample.cc
@@ -0,0 +1,258 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include "include/stringify.h"
+#include "global/global_init.h"
+#include "ErasureCodeExample.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+TEST(ErasureCodeExample, chunk_size)
+{
+  ErasureCodeExample example;
+  EXPECT_EQ(3u, example.get_chunk_count());
+  EXPECT_EQ(11u, example.get_chunk_size(20));
+}
+
+TEST(ErasureCodeExample, minimum_to_decode)
+{
+  ErasureCodeExample example;
+  set available_chunks;
+  set want_to_read;
+  want_to_read.insert(1);
+  {
+    set minimum;
+    EXPECT_EQ(-EIO, example.minimum_to_decode(want_to_read,
+                                              available_chunks,
+                                              &minimum));
+  }
+  available_chunks.insert(0);
+  available_chunks.insert(2);
+  {
+    set minimum;
+    EXPECT_EQ(0, example.minimum_to_decode(want_to_read,
+                                           available_chunks,
+                                           &minimum));
+    EXPECT_EQ(available_chunks, minimum);
+    EXPECT_EQ(2u, minimum.size());
+    EXPECT_EQ(1u, minimum.count(0));
+    EXPECT_EQ(1u, minimum.count(2));
+  }
+  {
+    set minimum;
+    available_chunks.insert(1);
+    EXPECT_EQ(0, example.minimum_to_decode(want_to_read,
+                                           available_chunks,
+                                           &minimum));
+    EXPECT_EQ(1u, minimum.size());
+    EXPECT_EQ(1u, minimum.count(1));
+  }
+}
+
+TEST(ErasureCodeExample, minimum_to_decode_with_cost)
+{
+  ErasureCodeExample example;
+  map available;
+  set want_to_read;
+  want_to_read.insert(1);
+  {
+    set minimum;
+    EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read,
+							available,
+							&minimum));
+  }
+  available[0] = 1;
+  available[2] = 1;
+  {
+    set minimum;
+    EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+						     available,
+						     &minimum));
+    EXPECT_EQ(2u, minimum.size());
+    EXPECT_EQ(1u, minimum.count(0));
+    EXPECT_EQ(1u, minimum.count(2));
+  }
+  {
+    set minimum;
+    available[1] = 1;
+    EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+						     available,
+						     &minimum));
+    EXPECT_EQ(1u, minimum.size());
+    EXPECT_EQ(1u, minimum.count(1));
+  }
+  {
+    set minimum;
+    available[1] = 2;
+    EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+						     available,
+						     &minimum));
+    EXPECT_EQ(2u, minimum.size());
+    EXPECT_EQ(1u, minimum.count(0));
+    EXPECT_EQ(1u, minimum.count(2));
+  }
+}
+
+TEST(ErasureCodeExample, encode_decode)
+{
+  ErasureCodeExample example;
+
+  bufferlist in;
+  in.append("ABCDE");
+  set want_to_encode;
+  for(unsigned int i = 0; i < example.get_chunk_count(); i++)
+    want_to_encode.insert(i);
+  map encoded;
+  EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded));
+  EXPECT_EQ(example.get_chunk_count(), encoded.size());
+  EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length());
+  EXPECT_EQ('A', encoded[0][0]);
+  EXPECT_EQ('B', encoded[0][1]);
+  EXPECT_EQ('C', encoded[0][2]);
+  EXPECT_EQ('D', encoded[1][0]);
+  EXPECT_EQ('E', encoded[1][1]);
+  EXPECT_EQ('A'^'D', encoded[2][0]);
+  EXPECT_EQ('B'^'E', encoded[2][1]);
+  EXPECT_EQ('C'^0, encoded[2][2]);
+
+  // all chunks are available
+  {
+    int want_to_decode[] = { 0, 1 };
+    map decoded;
+    EXPECT_EQ(0, example.decode(set(want_to_decode, want_to_decode+2),
+                                encoded,
+                                &decoded));
+    EXPECT_EQ(2u, decoded.size());
+    EXPECT_EQ(3u, decoded[0].length());
+    EXPECT_EQ('A', decoded[0][0]);
+    EXPECT_EQ('B', decoded[0][1]);
+    EXPECT_EQ('C', decoded[0][2]);
+    EXPECT_EQ('D', decoded[1][0]);
+    EXPECT_EQ('E', decoded[1][1]);
+  }
+
+  // one chunk is missing 
+  {
+    map degraded = encoded;
+    degraded.erase(0);
+    EXPECT_EQ(2u, degraded.size());
+    int want_to_decode[] = { 0, 1 };
+    map decoded;
+    EXPECT_EQ(0, example.decode(set(want_to_decode, want_to_decode+2),
+                                degraded,
+                                &decoded));
+    EXPECT_EQ(2u, decoded.size());
+    EXPECT_EQ(3u, decoded[0].length());
+    EXPECT_EQ('A', decoded[0][0]);
+    EXPECT_EQ('B', decoded[0][1]);
+    EXPECT_EQ('C', decoded[0][2]);
+    EXPECT_EQ('D', decoded[1][0]);
+    EXPECT_EQ('E', decoded[1][1]);
+  }
+}
+
+TEST(ErasureCodeExample, decode)
+{
+  ErasureCodeExample example;
+
+#define LARGE_ENOUGH 2048
+  bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+  in_ptr.zero();
+  in_ptr.set_length(0);
+  const char *payload =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  in_ptr.append(payload, strlen(payload));
+  bufferlist in;
+  in.push_front(in_ptr);
+  int want_to_encode[] = { 0, 1, 2 };
+  map encoded;
+  EXPECT_EQ(0, example.encode(set(want_to_encode, want_to_encode+3),
+                              in,
+                              &encoded));
+  EXPECT_EQ(3u, encoded.size());
+
+  // successfull decode
+  bufferlist out;
+  EXPECT_EQ(0, example.decode_concat(encoded, &out));
+  bufferlist usable;
+  usable.substr_of(out, 0, in.length());
+  EXPECT_TRUE(usable == in);
+
+  // cannot recover
+  map degraded;  
+  degraded[0] = encoded[0];
+  EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
+}
+
+TEST(ErasureCodeExample, create_ruleset)
+{
+  CrushWrapper *c = new CrushWrapper;
+  c->create();
+  c->set_type_name(2, "root");
+  c->set_type_name(1, "host");
+  c->set_type_name(0, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		5, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  map loc;
+  loc["root"] = "default";
+
+  int num_host = 2;
+  int num_osd = 5;
+  int osd = 0;
+  for (int h=0; hinsert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+    }
+  }
+
+  stringstream ss;
+  ErasureCodeExample example;
+  EXPECT_EQ(0, example.create_ruleset("myrule", *c, &ss));
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; 
+ *   make -j4 && 
+ *   make unittest_erasure_code_example && 
+ *   valgrind  --leak-check=full --tool=memcheck \
+ *      ./unittest_erasure_code_example --gtest_filter=*.* \
+ *      --log-to-stderr=true --debug-osd=20
+ * "
+ * End:
+ */
+
diff --git a/ceph/src/test/erasure-code/TestErasureCodeJerasure.cc b/ceph/src/test/erasure-code/TestErasureCodeJerasure.cc
new file mode 100644
index 00000000..5c637dae
--- /dev/null
+++ b/ceph/src/test/erasure-code/TestErasureCodeJerasure.cc
@@ -0,0 +1,391 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+
+#include "crush/CrushWrapper.h"
+#include "include/stringify.h"
+#include "global/global_init.h"
+#include "erasure-code/jerasure/ErasureCodeJerasure.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+template 
+class ErasureCodeTest : public ::testing::Test {
+ public:
+};
+
+typedef ::testing::Types<
+  ErasureCodeJerasureReedSolomonVandermonde,
+  ErasureCodeJerasureReedSolomonRAID6,
+  ErasureCodeJerasureCauchyOrig,
+  ErasureCodeJerasureCauchyGood,
+  ErasureCodeJerasureLiberation,
+  ErasureCodeJerasureBlaumRoth,
+  ErasureCodeJerasureLiber8tion
+> JerasureTypes;
+TYPED_TEST_CASE(ErasureCodeTest, JerasureTypes);
+
+TYPED_TEST(ErasureCodeTest, encode_decode)
+{
+  TypeParam jerasure;
+  map parameters;
+  parameters["k"] = "2";
+  parameters["m"] = "2";
+  parameters["w"] = "7";
+  parameters["packetsize"] = "8";
+  jerasure.init(parameters);
+
+#define LARGE_ENOUGH 2048
+  bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+  in_ptr.zero();
+  in_ptr.set_length(0);
+  const char *payload =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  in_ptr.append(payload, strlen(payload));
+  bufferlist in;
+  in.push_front(in_ptr);
+  int want_to_encode[] = { 0, 1, 2, 3 };
+  map encoded;
+  EXPECT_EQ(0, jerasure.encode(set(want_to_encode, want_to_encode+4),
+                              in,
+                              &encoded));
+  EXPECT_EQ(4u, encoded.size());
+  unsigned length =  encoded[0].length();
+  EXPECT_EQ(0, strncmp(encoded[0].c_str(), in.c_str(), length));
+  EXPECT_EQ(0, strncmp(encoded[1].c_str(), in.c_str() + length,
+		       in.length() - length));
+
+
+  // all chunks are available
+  {
+    int want_to_decode[] = { 0, 1 };
+    map decoded;
+    EXPECT_EQ(0, jerasure.decode(set(want_to_decode, want_to_decode+2),
+                                encoded,
+                                &decoded));
+    EXPECT_EQ(2u, decoded.size()); 
+    EXPECT_EQ(length, decoded[0].length());
+    EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
+    EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
+			 in.length() - length));
+  }
+
+  // two chunks are missing 
+  {
+    map degraded = encoded;
+    degraded.erase(0);
+    degraded.erase(1);
+    EXPECT_EQ(2u, degraded.size());
+    int want_to_decode[] = { 0, 1 };
+    map decoded;
+    EXPECT_EQ(0, jerasure.decode(set(want_to_decode, want_to_decode+2),
+                                degraded,
+                                &decoded));
+    // always decode all, regardless of want_to_decode
+    EXPECT_EQ(4u, decoded.size()); 
+    EXPECT_EQ(length, decoded[0].length());
+    EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
+    EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
+			 in.length() - length));
+  }
+}
+
+TYPED_TEST(ErasureCodeTest, minimum_to_decode)
+{
+  TypeParam jerasure;
+  map parameters;
+  parameters["k"] = "2";
+  parameters["m"] = "2";
+  parameters["w"] = "7";
+  parameters["packetsize"] = "8";
+  jerasure.init(parameters);
+
+  //
+  // If trying to read nothing, the minimum is empty.
+  //
+  {
+    set want_to_read;
+    set available_chunks;
+    set minimum;
+
+    EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
+					    available_chunks,
+					    &minimum));
+    EXPECT_TRUE(minimum.empty());
+  }
+  //
+  // There is no way to read a chunk if none are available.
+  //
+  {
+    set want_to_read;
+    set available_chunks;
+    set minimum;
+
+    want_to_read.insert(0);
+
+    EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read,
+					       available_chunks,
+					       &minimum));
+  }
+  //
+  // Reading a subset of the available chunks is always possible.
+  //
+  {
+    set want_to_read;
+    set available_chunks;
+    set minimum;
+
+    want_to_read.insert(0);
+    available_chunks.insert(0);
+
+    EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
+					    available_chunks,
+					    &minimum));
+    EXPECT_EQ(want_to_read, minimum);
+  }
+  //
+  // There is no way to read a missing chunk if there is less than k
+  // chunks available.
+  //
+  {
+    set want_to_read;
+    set available_chunks;
+    set minimum;
+
+    want_to_read.insert(0);
+    want_to_read.insert(1);
+    available_chunks.insert(0);
+
+    EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read,
+					       available_chunks,
+					       &minimum));
+  }
+  //
+  // When chunks are not available, the minimum can be made of any
+  // chunks. For instance, to read 1 and 3 below the minimum could be
+  // 2 and 3 which may seem better because it contains one of the
+  // chunks to be read. But it won't be more efficient than retrieving
+  // 0 and 2 instead because, in both cases, the decode function will
+  // need to run the same recovery operation and use the same amount
+  // of CPU and memory.
+  //
+  {
+    set want_to_read;
+    set available_chunks;
+    set minimum;
+
+    want_to_read.insert(1);
+    want_to_read.insert(3);
+    available_chunks.insert(0);
+    available_chunks.insert(2);
+    available_chunks.insert(3);
+
+    EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
+					    available_chunks,
+					    &minimum));
+    EXPECT_EQ(2u, minimum.size());
+    EXPECT_EQ(0u, minimum.count(3));
+  }
+}
+
+TEST(ErasureCodeTest, encode)
+{
+  ErasureCodeJerasureReedSolomonVandermonde jerasure;
+  map parameters;
+  parameters["k"] = "2";
+  parameters["m"] = "2";
+  parameters["w"] = "8";
+  jerasure.init(parameters);
+
+  unsigned alignment = jerasure.get_alignment();
+  {
+    //
+    // When the input bufferlist needs to be padded because
+    // it is not properly aligned, it is padded with zeros.
+    //
+    bufferlist in;
+    map encoded;
+    int want_to_encode[] = { 0, 1, 2, 3 };
+    int trail_length = 10;
+    in.append(string(alignment + trail_length, 'X'));
+    EXPECT_EQ(0, jerasure.encode(set(want_to_encode, want_to_encode+4),
+				 in,
+				 &encoded));
+    EXPECT_EQ(4u, encoded.size());
+    for(int i = 0; i < 4; i++)
+      EXPECT_EQ(alignment, encoded[i].length());
+    char *last_chunk = encoded[1].c_str();
+    EXPECT_EQ('X', last_chunk[0]);
+    EXPECT_EQ('\0', last_chunk[trail_length]);
+  }
+
+  {
+    //
+    // When only the first chunk is required, the encoded map only
+    // contains the first chunk. Although the jerasure encode
+    // internally allocated a buffer because of padding requirements
+    // and also computes the coding chunks, they are released before
+    // the return of the method, as shown when running the tests thru
+    // valgrind (there is no leak).
+    //
+    bufferlist in;
+    map encoded;
+    set want_to_encode;
+    want_to_encode.insert(0);
+    int trail_length = 10;
+    in.append(string(alignment + trail_length, 'X'));
+    EXPECT_EQ(0, jerasure.encode(want_to_encode, in, &encoded));
+    EXPECT_EQ(1u, encoded.size());
+    EXPECT_EQ(alignment, encoded[0].length());
+  }
+}
+
+TEST(ErasureCodeTest, create_ruleset)
+{
+  CrushWrapper *c = new CrushWrapper;
+  c->create();
+  int root_type = 2;
+  c->set_type_name(root_type, "root");
+  int host_type = 1;
+  c->set_type_name(host_type, "host");
+  int osd_type = 0;
+  c->set_type_name(osd_type, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		root_type, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  map loc;
+  loc["root"] = "default";
+
+  int num_host = 4;
+  int num_osd = 5;
+  int osd = 0;
+  for (int h=0; hinsert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+    }
+  }
+
+  //
+  // The ruleid may be different from the ruleset when a crush rule is
+  // removed because the removed ruleid will be reused but the removed
+  // ruleset will not be reused. 
+  //
+  // This also asserts that the create_ruleset() method returns a
+  // ruleset and not a ruleid http://tracker.ceph.com/issues/9044
+  //
+  {
+    stringstream ss;
+    ErasureCodeJerasureReedSolomonVandermonde jerasure;
+    map parameters;
+    parameters["k"] = "2";
+    parameters["m"] = "2";
+    parameters["w"] = "8";
+    jerasure.init(parameters);
+    int FIRST = jerasure.create_ruleset("FIRST", *c, &ss);
+    int SECOND = jerasure.create_ruleset("SECOND", *c, &ss);
+    int FIRST_ruleid = c->get_rule_id("FIRST");
+    EXPECT_EQ(0, c->remove_rule(FIRST_ruleid));
+    int ruleset = jerasure.create_ruleset("myrule", *c, &ss);
+    EXPECT_NE(FIRST, ruleset);
+    EXPECT_NE(SECOND, ruleset);
+    EXPECT_NE(ruleset, c->get_rule_id("myrule"));
+    int SECOND_ruleid = c->get_rule_id("SECOND");
+    EXPECT_EQ(0, c->remove_rule(SECOND_ruleid));
+    int myrule_ruleid = c->get_rule_id("myrule");
+    EXPECT_EQ(0, c->remove_rule(myrule_ruleid));
+  }
+
+  {
+    stringstream ss;
+    ErasureCodeJerasureReedSolomonVandermonde jerasure;
+    map parameters;
+    parameters["k"] = "2";
+    parameters["m"] = "2";
+    parameters["w"] = "8";
+    jerasure.init(parameters);
+    int ruleset = jerasure.create_ruleset("myrule", *c, &ss);
+    EXPECT_EQ(0, ruleset);
+    EXPECT_EQ(-EEXIST, jerasure.create_ruleset("myrule", *c, &ss));
+    //
+    // the minimum that is expected from the created ruleset is to
+    // successfully map get_chunk_count() devices from the crushmap,
+    // at least once.
+    //
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector out;
+    int x = 0;
+    c->do_rule(ruleset, x, out, jerasure.get_chunk_count(), weight);
+    ASSERT_EQ(out.size(), jerasure.get_chunk_count());
+    for (unsigned i=0; i parameters;
+    parameters["k"] = "2";
+    parameters["m"] = "2";
+    parameters["w"] = "8";
+    parameters["ruleset-root"] = "BAD";
+    jerasure.init(parameters);
+    EXPECT_EQ(-ENOENT, jerasure.create_ruleset("otherrule", *c, &ss));
+    EXPECT_EQ("root item BAD does not exist", ss.str());
+  }
+  {
+    stringstream ss;
+    ErasureCodeJerasureReedSolomonVandermonde jerasure;
+    map parameters;
+    parameters["k"] = "2";
+    parameters["m"] = "2";
+    parameters["w"] = "8";
+    parameters["ruleset-failure-domain"] = "WORSE";
+    jerasure.init(parameters);
+    EXPECT_EQ(-EINVAL, jerasure.create_ruleset("otherrule", *c, &ss));
+    EXPECT_EQ("unknown type WORSE", ss.str());
+  }
+}
+
+int main(int argc, char **argv)
+{
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+/* 
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 && 
+ *   make unittest_erasure_code_jerasure && 
+ *   valgrind --tool=memcheck --leak-check=full \
+ *      ./unittest_erasure_code_jerasure \
+ *      --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/ceph/src/test/erasure-code/TestErasureCodePlugin.cc b/ceph/src/test/erasure-code/TestErasureCodePlugin.cc
new file mode 100644
index 00000000..a896e3f7
--- /dev/null
+++ b/ceph/src/test/erasure-code/TestErasureCodePlugin.cc
@@ -0,0 +1,110 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+#include 
+#include "common/Thread.h"
+#include "global/global_init.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+class ErasureCodePluginRegistryTest : public ::testing::Test {
+protected:
+
+  class Thread_factory : public Thread {
+  public:
+    virtual void *entry() {
+      map parameters;
+      parameters["directory"] = ".libs";
+      ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+      ErasureCodeInterfaceRef erasure_code;
+      stringstream ss;
+      instance.factory("hangs", parameters, &erasure_code, ss);
+      return NULL;
+    }
+  };
+
+};
+
+TEST_F(ErasureCodePluginRegistryTest, factory_mutex) {
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+
+  EXPECT_TRUE(instance.lock.TryLock());
+  instance.lock.Unlock();
+
+  // 
+  // Test that the loading of a plugin is protected by a mutex.
+  //
+  useconds_t delay = 0;
+  const useconds_t DELAY_MAX = 20 * 1000 * 1000;
+  Thread_factory sleep_forever;
+  sleep_forever.create();
+  do {
+    cout << "Trying (1) with delay " << delay << "us\n";
+    if (delay > 0)
+      usleep(delay);
+    if (!instance.loading)
+      delay = ( delay + 1 ) * 2;
+  } while(!instance.loading && delay < DELAY_MAX);
+  ASSERT_TRUE(delay < DELAY_MAX);
+
+  EXPECT_FALSE(instance.lock.TryLock());
+
+  EXPECT_EQ(0, pthread_cancel(sleep_forever.get_thread_id()));
+  EXPECT_EQ(0, sleep_forever.join());
+}
+
+TEST_F(ErasureCodePluginRegistryTest, all)
+{
+  map parameters;
+  parameters["directory"] = ".libs";
+  ErasureCodeInterfaceRef erasure_code;
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  stringstream ss;
+  EXPECT_FALSE(erasure_code);
+  EXPECT_EQ(-EIO, instance.factory("invalid", parameters, &erasure_code, ss));
+  EXPECT_FALSE(erasure_code);
+  EXPECT_EQ(-ENOENT, instance.factory("missing_entry_point", parameters,
+				      &erasure_code, ss));
+  EXPECT_FALSE(erasure_code);
+  EXPECT_EQ(-ESRCH, instance.factory("fail_to_initialize", parameters,
+				     &erasure_code, ss));
+  EXPECT_FALSE(erasure_code);
+  EXPECT_EQ(-EBADF, instance.factory("fail_to_register", parameters,
+				     &erasure_code, ss));
+  EXPECT_FALSE(erasure_code);
+  EXPECT_EQ(0, instance.factory("example", parameters, &erasure_code, ss));
+  EXPECT_TRUE(erasure_code);
+  ErasureCodePlugin *plugin = 0;
+  EXPECT_EQ(-EEXIST, instance.load("example", parameters, &plugin, ss));
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_plugin && valgrind  --leak-check=full --tool=memcheck ./unittest_erasure_code_plugin --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+// End:
diff --git a/ceph/src/test/erasure-code/TestErasureCodePluginJerasure.cc b/ceph/src/test/erasure-code/TestErasureCodePluginJerasure.cc
new file mode 100644
index 00000000..21f66157
--- /dev/null
+++ b/ceph/src/test/erasure-code/TestErasureCodePluginJerasure.cc
@@ -0,0 +1,235 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include 
+#include "arch/probe.h"
+#include "arch/intel.h"
+#include "global/global_init.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+TEST(ErasureCodePlugin, factory)
+{
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  map parameters;
+  parameters["directory"] = ".libs";
+  {
+    ErasureCodeInterfaceRef erasure_code;
+    EXPECT_FALSE(erasure_code);
+    EXPECT_EQ(-ENOENT, instance.factory("jerasure", parameters,
+                                        &erasure_code, cerr));
+    EXPECT_FALSE(erasure_code);
+  }
+  const char *techniques[] = {
+    "reed_sol_van",
+    "reed_sol_r6_op",
+    "cauchy_orig",
+    "cauchy_good",
+    "liberation",
+    "blaum_roth",
+    "liber8tion",
+    0
+  };
+  for(const char **technique = techniques; *technique; technique++) {
+    ErasureCodeInterfaceRef erasure_code;
+    parameters["technique"] = *technique;
+    EXPECT_FALSE(erasure_code);
+    EXPECT_EQ(0, instance.factory("jerasure", parameters,
+                                  &erasure_code, cerr));
+    EXPECT_TRUE(erasure_code);
+  }
+}
+
+TEST(ErasureCodePlugin, select)
+{
+  ceph_arch_probe();
+  // save probe results
+  int arch_intel_pclmul = ceph_arch_intel_pclmul;
+  int arch_intel_sse42  = ceph_arch_intel_sse42;
+  int arch_intel_sse41  = ceph_arch_intel_sse41;
+  int arch_intel_ssse3  = ceph_arch_intel_ssse3;
+  int arch_intel_sse3   = ceph_arch_intel_sse3;
+  int arch_intel_sse2   = ceph_arch_intel_sse2;
+
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  map parameters;
+  // load test plugins instead of actual plugins to assert the desired side effect
+  // happens
+  parameters["jerasure-name"] = "test_jerasure";
+  parameters["directory"] = ".libs";
+  parameters["technique"] = "reed_sol_van";
+
+  // all features are available, load the SSE4 plugin
+  {
+    ceph_arch_intel_pclmul = 1;
+    ceph_arch_intel_sse42  = 1;
+    ceph_arch_intel_sse41  = 1;
+    ceph_arch_intel_ssse3  = 1;
+    ceph_arch_intel_sse3   = 1;
+    ceph_arch_intel_sse2   = 1;
+
+    ErasureCodeInterfaceRef erasure_code;
+    int sse4_side_effect = -444;
+    EXPECT_EQ(sse4_side_effect, instance.factory("jerasure", parameters,
+                                                 &erasure_code, cerr));
+  }
+  // pclmul is missing, load the SSE3 plugin
+  {
+    ceph_arch_intel_pclmul = 0;
+    ceph_arch_intel_sse42  = 1;
+    ceph_arch_intel_sse41  = 1;
+    ceph_arch_intel_ssse3  = 1;
+    ceph_arch_intel_sse3   = 1;
+    ceph_arch_intel_sse2   = 1;
+
+    ErasureCodeInterfaceRef erasure_code;
+    int sse3_side_effect = -333;
+    EXPECT_EQ(sse3_side_effect, instance.factory("jerasure", parameters,
+                                                 &erasure_code, cerr));
+  }
+  // pclmul and sse3 are missing, load the generic plugin
+  {
+    ceph_arch_intel_pclmul = 0;
+    ceph_arch_intel_sse42  = 1;
+    ceph_arch_intel_sse41  = 1;
+    ceph_arch_intel_ssse3  = 1;
+    ceph_arch_intel_sse3   = 0;
+    ceph_arch_intel_sse2   = 1;
+
+    ErasureCodeInterfaceRef erasure_code;
+    int generic_side_effect = -111;
+    EXPECT_EQ(generic_side_effect, instance.factory("jerasure", parameters,
+                                                 &erasure_code, cerr));
+  }
+
+
+  // restore probe results
+  ceph_arch_intel_pclmul = arch_intel_pclmul;
+  ceph_arch_intel_sse42  = arch_intel_sse42;
+  ceph_arch_intel_sse41  = arch_intel_sse41;
+  ceph_arch_intel_ssse3  = arch_intel_ssse3;
+  ceph_arch_intel_sse3   = arch_intel_sse3;
+  ceph_arch_intel_sse2   = arch_intel_sse2;
+}
+
+TEST(ErasureCodePlugin, sse)
+{
+  ceph_arch_probe();
+  bool sse4 = ceph_arch_intel_pclmul &&
+    ceph_arch_intel_sse42 && ceph_arch_intel_sse41 &&
+    ceph_arch_intel_ssse3 && ceph_arch_intel_sse3 &&
+    ceph_arch_intel_sse2;
+  bool sse3 = ceph_arch_intel_ssse3 && ceph_arch_intel_sse3 &&
+    ceph_arch_intel_sse2;
+  vector sse_variants;
+  sse_variants.push_back("generic");
+  if (!sse3)
+    cerr << "SKIP sse3 plugin testing because CPU does not support it\n";
+  else
+    sse_variants.push_back("sse3");
+  if (!sse4)
+    cerr << "SKIP sse4 plugin testing because CPU does not support it\n";
+  else
+    sse_variants.push_back("sse4");
+
+#define LARGE_ENOUGH 2048
+  bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+  in_ptr.zero();
+  in_ptr.set_length(0);
+  const char *payload =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  in_ptr.append(payload, strlen(payload));
+  bufferlist in;
+  in.push_front(in_ptr);
+
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  map parameters;
+  parameters["directory"] = ".libs";
+  parameters["technique"] = "reed_sol_van";
+  parameters["k"] = "2";
+  parameters["m"] = "1";
+  for (vector::iterator sse_variant = sse_variants.begin();
+       sse_variant != sse_variants.end();
+       sse_variant++) {
+    //
+    // load the plugin variant
+    //
+    ErasureCodeInterfaceRef erasure_code;
+    EXPECT_FALSE(erasure_code);
+    EXPECT_EQ(0, instance.factory("jerasure_" + *sse_variant, parameters,
+                                  &erasure_code, cerr));
+    EXPECT_TRUE(erasure_code);
+
+    //
+    // encode
+    //
+    int want_to_encode[] = { 0, 1, 2 };
+    map encoded;
+    EXPECT_EQ(0, erasure_code->encode(set(want_to_encode, want_to_encode+3),
+                                      in,
+                                      &encoded));
+    EXPECT_EQ(3u, encoded.size());
+    unsigned length =  encoded[0].length();
+    EXPECT_EQ(0, strncmp(encoded[0].c_str(), in.c_str(), length));
+    EXPECT_EQ(0, strncmp(encoded[1].c_str(), in.c_str() + length,
+                         in.length() - length));
+
+    //
+    // decode with reconstruction
+    //
+    map degraded = encoded;
+    degraded.erase(1);
+    EXPECT_EQ(2u, degraded.size());
+    int want_to_decode[] = { 0, 1 };
+    map decoded;
+    EXPECT_EQ(0, erasure_code->decode(set(want_to_decode, want_to_decode+2),
+                                      degraded,
+                                      &decoded));
+    EXPECT_EQ(3u, decoded.size());
+    EXPECT_EQ(length, decoded[0].length());
+    EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
+    EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
+                         in.length() - length));
+
+  }
+}
+
+int main(int argc, char **argv)
+{
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ *   make unittest_erasure_code_plugin_jerasure &&
+ *   valgrind --tool=memcheck ./unittest_erasure_code_plugin_jerasure \
+ *      --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/ceph/src/test/erasure-code/TestJerasurePluginGeneric.cc b/ceph/src/test/erasure-code/TestJerasurePluginGeneric.cc
new file mode 100644
index 00000000..f273bd67
--- /dev/null
+++ b/ceph/src/test/erasure-code/TestJerasurePluginGeneric.cc
@@ -0,0 +1,23 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+  return -111;
+}
diff --git a/ceph/src/test/erasure-code/TestJerasurePluginSSE3.cc b/ceph/src/test/erasure-code/TestJerasurePluginSSE3.cc
new file mode 100644
index 00000000..9d74d52c
--- /dev/null
+++ b/ceph/src/test/erasure-code/TestJerasurePluginSSE3.cc
@@ -0,0 +1,23 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+  return -333;
+}
diff --git a/ceph/src/test/erasure-code/TestJerasurePluginSSE4.cc b/ceph/src/test/erasure-code/TestJerasurePluginSSE4.cc
new file mode 100644
index 00000000..7bb6f7e6
--- /dev/null
+++ b/ceph/src/test/erasure-code/TestJerasurePluginSSE4.cc
@@ -0,0 +1,23 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include 
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+  return -444;
+}
diff --git a/ceph/src/test/erasure-code/ceph_erasure_code.cc b/ceph/src/test/erasure-code/ceph_erasure_code.cc
new file mode 100644
index 00000000..f99e8454
--- /dev/null
+++ b/ceph/src/test/erasure-code/ceph_erasure_code.cc
@@ -0,0 +1,166 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "common/Clock.h"
+#include "include/utime.h"
+#include "erasure-code/ErasureCodePlugin.h"
+
+namespace po = boost::program_options;
+
+class ErasureCodeCommand {
+  po::variables_map vm;
+  map parameters;
+public:
+  int setup(int argc, char** argv);
+  int run();
+};
+
+int ErasureCodeCommand::setup(int argc, char** argv) {
+
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help,h", "produce help message")
+    ("all", "implies "
+     "--get_chunk_size 1024 "
+     "--get_data_chunk_count "
+     "--get_chunk_count ")
+    ("get_chunk_size", po::value(),
+     "display get_chunk_size()")
+    ("get_data_chunk_count", "display get_data_chunk_count()")
+    ("get_chunk_count", "display get_chunk_count()")
+    ("parameter,P", po::value >(),
+     "parameters")
+    ;
+
+  po::parsed_options parsed =
+    po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+  po::store(
+    parsed,
+    vm);
+  po::notify(vm);
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+    CODE_ENVIRONMENT_UTILITY,
+    CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  if (vm.count("parameter")) {
+    const vector &p = vm["parameter"].as< vector >();
+    for (vector::const_iterator i = p.begin();
+	 i != p.end();
+	 ++i) {
+      std::vector strs;
+      boost::split(strs, *i, boost::is_any_of("="));
+      if (strs.size() != 2) {
+	cerr << "--parameter " << *i
+	     << " ignored because it does not contain exactly one =" << endl;
+      } else {
+	parameters[strs[0]] = strs[1];
+      }
+    }
+  }
+
+  if (parameters.count("directory") == 0)
+    parameters["directory"] = ".libs";
+  if (parameters.count("plugin") == 0) {
+    cerr << "--parameter plugin= is mandatory" << endl;
+    return 1;
+  }
+
+  return 0;
+}
+
+int ErasureCodeCommand::run() {
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  instance.disable_dlclose = true;
+  ErasureCodeInterfaceRef erasure_code;
+  int code = instance.factory(parameters["plugin"],
+			      parameters,
+			      &erasure_code, cerr);
+  if (code)
+    return code;
+
+  if (vm.count("all") || vm.count("get_chunk_size")) {
+    unsigned int object_size = 1024;
+    if (vm.count("get_chunk_size"))
+      object_size = vm["get_chunk_size"].as();
+    cout << "get_chunk_size(" << object_size << ")\t"
+	 << erasure_code->get_chunk_size(object_size) << endl;
+  }
+  if (vm.count("all") || vm.count("get_data_chunk_count"))
+    cout << "get_data_chunk_count\t"
+      	 << erasure_code->get_data_chunk_count() << endl;
+  if (vm.count("all") || vm.count("get_chunk_count"))
+    cout << "get_chunk_count\t"
+      	 << erasure_code->get_chunk_count() << endl;
+  return 0;
+}
+
+int main(int argc, char** argv) {
+  ErasureCodeCommand eccommand;
+  int err = eccommand.setup(argc, argv);
+  if (err)
+    return err;
+  return eccommand.run();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ *   make -j4 ceph_erasure_code &&
+ *   valgrind --tool=memcheck --leak-check=full \
+ *      ./ceph_erasure_code \
+ *      --parameter plugin=jerasure \
+ *      --parameter directory=.libs \
+ *      --parameter technique=reed_sol_van \
+ *      --parameter k=2 \
+ *      --parameter m=2 \
+ *      --get_chunk_size 1024 \
+ *      --get_data_chunk_count \
+ *      --get_chunk_count \
+ * "
+ * End:
+ */
diff --git a/ceph/src/test/erasure-code/ceph_erasure_code_benchmark.cc b/ceph/src/test/erasure-code/ceph_erasure_code_benchmark.cc
new file mode 100644
index 00000000..9d702962
--- /dev/null
+++ b/ceph/src/test/erasure-code/ceph_erasure_code_benchmark.cc
@@ -0,0 +1,218 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "ceph_erasure_code_benchmark.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "common/Clock.h"
+#include "include/utime.h"
+#include "erasure-code/ErasureCodePlugin.h"
+
+namespace po = boost::program_options;
+
+int ErasureCodeBench::setup(int argc, char** argv) {
+
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help,h", "produce help message")
+    ("size,s", po::value()->default_value(1024 * 1024),
+     "size of the buffer to be encoded")
+    ("iterations,i", po::value()->default_value(1),
+     "number of encode/decode runs")
+    ("plugin,p", po::value()->default_value("jerasure"),
+     "erasure code plugin name")
+    ("workload,w", po::value()->default_value("encode"),
+     "run either encode or decode")
+    ("erasures,e", po::value()->default_value(1),
+     "number of erasures when decoding")
+    ("parameter,P", po::value >(),
+     "parameters")
+    ;
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+    po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+  po::store(
+    parsed,
+    vm);
+  po::notify(vm);
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+    CODE_ENVIRONMENT_UTILITY,
+    CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  if (vm.count("parameter")) {
+    const vector &p = vm["parameter"].as< vector >();
+    for (vector::const_iterator i = p.begin();
+	 i != p.end();
+	 ++i) {
+      std::vector strs;
+      boost::split(strs, *i, boost::is_any_of("="));
+      if (strs.size() != 2) {
+	cerr << "--parameter " << *i << " ignored because it does not contain exactly one =" << endl;
+      } else {
+	parameters[strs[0]] = strs[1];
+      }
+    }
+  }
+
+  if (parameters.count("directory") == 0)
+    parameters["directory"] = ".libs";
+
+  in_size = vm["size"].as();
+  max_iterations = vm["iterations"].as();
+  plugin = vm["plugin"].as();
+  workload = vm["workload"].as();
+  erasures = vm["erasures"].as();
+
+  return 0;
+}
+
+int ErasureCodeBench::run() {
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  instance.disable_dlclose = true;
+
+  if (workload == "encode")
+    return encode();
+  else
+    return decode();
+}
+
+int ErasureCodeBench::encode()
+{
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  ErasureCodeInterfaceRef erasure_code;
+  int code = instance.factory(plugin, parameters, &erasure_code, cerr);
+  if (code)
+    return code;
+  int k = atoi(parameters["k"].c_str());
+  int m = atoi(parameters["m"].c_str());
+
+  bufferlist in;
+  in.append(string(in_size, 'X'));
+  set want_to_encode;
+  for (int i = 0; i < k + m; i++) {
+    want_to_encode.insert(i);
+  }
+  utime_t begin_time = ceph_clock_now(g_ceph_context);
+  for (int i = 0; i < max_iterations; i++) {
+    map encoded;
+    code = erasure_code->encode(want_to_encode, in, &encoded);
+    if (code)
+      return code;
+  }
+  utime_t end_time = ceph_clock_now(g_ceph_context);
+  cout << (end_time - begin_time) << "\t" << (max_iterations * (in_size / 1024)) << endl;
+  return 0;
+}
+
+int ErasureCodeBench::decode()
+{
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  ErasureCodeInterfaceRef erasure_code;
+  int code = instance.factory(plugin, parameters, &erasure_code, cerr);
+  if (code)
+    return code;
+  int k = atoi(parameters["k"].c_str());
+  int m = atoi(parameters["m"].c_str());
+
+  bufferlist in;
+  in.append(string(in_size, 'X'));
+
+  set want_to_encode;
+  for (int i = 0; i < k + m; i++) {
+    want_to_encode.insert(i);
+  }
+
+  map encoded;
+  code = erasure_code->encode(want_to_encode, in, &encoded);
+  if (code)
+    return code;
+
+  set want_to_read = want_to_encode;
+
+  utime_t begin_time = ceph_clock_now(g_ceph_context);
+  for (int i = 0; i < max_iterations; i++) {
+    map chunks = encoded;
+    for (int j = 0; j < erasures; j++) {
+      int erasure;
+      do {
+	erasure = rand() % ( k + m );
+      } while(chunks.count(erasure) == 0);
+      chunks.erase(erasure);
+    }
+    map decoded;
+    code = erasure_code->decode(want_to_read, chunks, &decoded);
+    if (code)
+      return code;
+  }
+  utime_t end_time = ceph_clock_now(g_ceph_context);
+  cout << (end_time - begin_time) << "\t" << (max_iterations * (in_size / 1024)) << endl;
+  return 0;
+}
+
+int main(int argc, char** argv) {
+  ErasureCodeBench ecbench;
+  int err = ecbench.setup(argc, argv);
+  if (err)
+    return err;
+  return ecbench.run();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ *   make ceph_erasure_code_benchmark &&
+ *   valgrind --tool=memcheck --leak-check=full \
+ *      ./ceph_erasure_code_benchmark \
+ *      --plugin jerasure \
+ *      --parameter directory=.libs \
+ *      --parameter technique=reed_sol_van \
+ *      --parameter k=2 \
+ *      --parameter m=2 \
+ *      --iterations 1
+ * "
+ * End:
+ */
diff --git a/ceph/src/test/erasure-code/ceph_erasure_code_benchmark.h b/ceph/src/test/erasure-code/ceph_erasure_code_benchmark.h
new file mode 100644
index 00000000..df73aa75
--- /dev/null
+++ b/ceph/src/test/erasure-code/ceph_erasure_code_benchmark.h
@@ -0,0 +1,38 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#ifndef CEPH_ERASURE_CODE_BENCHMARK_H
+#define CEPH_ERASURE_CODE_BENCHMARK_H
+
+#include 
+
+using namespace std;
+
+class ErasureCodeBench {
+  int in_size;
+  int max_iterations;
+  string plugin;
+  int erasures;
+  string workload;
+  map parameters;
+public:
+  int setup(int argc, char** argv);
+  int run();
+  int decode();
+  int encode();
+};
+
+#endif
diff --git a/ceph/src/test/escape.cc b/ceph/src/test/escape.cc
new file mode 100644
index 00000000..cf09a5a2
--- /dev/null
+++ b/ceph/src/test/escape.cc
@@ -0,0 +1,95 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "common/escape.h"
+#include "gtest/gtest.h"
+#include 
+
+static std::string escape_xml_attrs(const char *str)
+{
+  int len = escape_xml_attr_len(str);
+  char out[len];
+  escape_xml_attr(str, out);
+  return out;
+}
+
+TEST(EscapeXml, PassThrough) {
+  ASSERT_EQ(escape_xml_attrs("simplicity itself"), "simplicity itself");
+  ASSERT_EQ(escape_xml_attrs(""), "");
+  ASSERT_EQ(escape_xml_attrs("simple examples please!"), "simple examples please!");
+}
+
+TEST(EscapeXml, EntityRefs1) {
+  ASSERT_EQ(escape_xml_attrs("The \"scare quotes\""), "The "scare quotes"");
+  ASSERT_EQ(escape_xml_attrs("I <3 XML"), "I <3 XML");
+  ASSERT_EQ(escape_xml_attrs("Some 'single' \"quotes\" here"),
+	    "Some 'single' "quotes" here");
+}
+
+TEST(EscapeXml, ControlChars) {
+  uint8_t cc1[] = { 0x01, 0x02, 0x03, 0x0 };
+  ASSERT_EQ(escape_xml_attrs((char*)cc1), "");
+
+  uint8_t cc2[] = { 0x61, 0x62, 0x63, 0x7f, 0x0 };
+  ASSERT_EQ(escape_xml_attrs((char*)cc2), "abc");
+}
+
+TEST(EscapeXml, Utf8) {
+  uint8_t cc1[] = { 0xe6, 0xb1, 0x89, 0xe5, 0xad, 0x97, 0x0a, 0x0 };
+  ASSERT_EQ(escape_xml_attrs((const char*)cc1), (const char*)cc1);
+
+  uint8_t cc2[] = { 0x3c, 0xe6, 0xb1, 0x89, 0xe5, 0xad, 0x97, 0x3e, 0x0a, 0x0 };
+  uint8_t cc2_out[] = { 0x26, 0x6c, 0x74, 0x3b, 0xe6, 0xb1, 0x89, 0xe5,
+			0xad, 0x97, 0x26, 0x67, 0x74, 0x3b, 0x0a, 0x0 };
+  ASSERT_EQ(escape_xml_attrs((const char*)cc2), (const char*)cc2_out);
+}
+
+static std::string escape_json_attrs(const char *str)
+{
+  int len = escape_json_attr_len(str);
+  char out[len];
+  escape_json_attr(str, out);
+  return out;
+}
+
+TEST(EscapeJson, PassThrough) {
+  ASSERT_EQ(escape_json_attrs("simplicity itself"), "simplicity itself");
+  ASSERT_EQ(escape_json_attrs(""), "");
+  ASSERT_EQ(escape_json_attrs("simple examples please!"), "simple examples please!");
+}
+
+TEST(EscapeJson, Escapes1) {
+  ASSERT_EQ(escape_json_attrs("The \"scare quotes\""),
+			     "The \\\"scare quotes\\\"");
+  ASSERT_EQ(escape_json_attrs("I <3 JSON"), "I <3 JSON");
+  ASSERT_EQ(escape_json_attrs(
+      "JSON calls a slash / backslash a solidus / reverse solidus"),
+      "JSON calls a slash \\/ backslash a solidus \\/ reverse solidus");
+  ASSERT_EQ(escape_json_attrs("Some 'single' \"quotes\" here"),
+      "Some 'single' \\\"quotes\\\" here");
+  ASSERT_EQ(escape_json_attrs("tabs\tand\tnewlines\n, oh my"),
+      "tabs\\tand\\tnewlines\\n, oh my");
+}
+
+TEST(EscapeJson, ControlChars) {
+  uint8_t cc1[] = { 0x01, 0x02, 0x03, 0x0 };
+  ASSERT_EQ(escape_json_attrs((char*)cc1), "\\u0001\\u0002\\u0003");
+
+  uint8_t cc2[] = { 0x61, 0x62, 0x63, 0x7f, 0x0 };
+  ASSERT_EQ(escape_json_attrs((char*)cc2), "abc\\u007f");
+}
+
+TEST(EscapeJson, Utf8) {
+  uint8_t cc1[] = { 0xe6, 0xb1, 0x89, 0xe5, 0xad, 0x97, 0x0a, 0x0 };
+  ASSERT_EQ(escape_xml_attrs((const char*)cc1), (const char*)cc1);
+}
diff --git a/ceph/src/test/formatter.cc b/ceph/src/test/formatter.cc
new file mode 100644
index 00000000..a02e895d
--- /dev/null
+++ b/ceph/src/test/formatter.cc
@@ -0,0 +1,180 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "test/unit.h"
+#include "common/Formatter.h"
+
+#include 
+#include 
+
+using std::ostringstream;
+
+TEST(JsonFormatter, Simple1) {
+  ostringstream oss;
+  JSONFormatter fmt(false);
+  fmt.open_object_section("foo");
+  fmt.dump_int("a", 1);
+  fmt.dump_int("b", 2);
+  fmt.dump_int("c", 3);
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "{\"a\":1,\"b\":2,\"c\":3}");
+}
+
+TEST(JsonFormatter, Simple2) {
+  ostringstream oss;
+  JSONFormatter fmt(false);
+  fmt.open_object_section("foo");
+  fmt.open_object_section("bar");
+  fmt.dump_int("int", 0xf00000000000ll);
+  fmt.dump_unsigned("unsigned", 0x8000000000000001llu);
+  fmt.dump_float("float", 1.234);
+  fmt.close_section();
+  fmt.dump_string("string", "str");
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "{\"bar\":{\"int\":263882790666240,\
+\"unsigned\":9223372036854775809,\"float\":\"1.234000\"},\
+\"string\":\"str\"}");
+}
+
+TEST(JsonFormatter, Empty) {
+  ostringstream oss;
+  JSONFormatter fmt(false);
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "");
+}
+
+TEST(XmlFormatter, Simple1) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+  fmt.open_object_section("foo");
+  fmt.dump_int("a", 1);
+  fmt.dump_int("b", 2);
+  fmt.dump_int("c", 3);
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "123");
+}
+
+TEST(XmlFormatter, Simple2) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+  fmt.open_object_section("foo");
+  fmt.open_object_section("bar");
+  fmt.dump_int("int", 0xf00000000000ll);
+  fmt.dump_unsigned("unsigned", 0x8000000000000001llu);
+  fmt.dump_float("float", 1.234);
+  fmt.close_section();
+  fmt.dump_string("string", "str");
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "\
+263882790666240\
+9223372036854775809\
+1.234\
+str\
+");
+}
+
+TEST(XmlFormatter, Empty) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "");
+}
+
+TEST(XmlFormatter, DumpStream1) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+  fmt.dump_stream("blah") << "hithere";
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "hithere");
+}
+
+TEST(XmlFormatter, DumpStream2) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+
+  fmt.open_array_section("foo");
+  fmt.dump_stream("blah") << "hithere";
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "hithere");
+}
+
+TEST(XmlFormatter, DumpStream3) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+
+  fmt.open_array_section("foo");
+  fmt.dump_stream("blah") << "hithere";
+  fmt.dump_float("pi", 3.14);
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), "hithere3.14");
+}
+
+TEST(XmlFormatter, DTD) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+
+  fmt.write_raw_data(XMLFormatter::XML_1_DTD);
+  fmt.open_array_section("foo");
+  fmt.dump_stream("blah") << "hithere";
+  fmt.dump_float("pi", 3.14);
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), ""
+    "hithere3.14");
+}
+
+TEST(XmlFormatter, Clear) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+
+  fmt.write_raw_data(XMLFormatter::XML_1_DTD);
+  fmt.open_array_section("foo");
+  fmt.dump_stream("blah") << "hithere";
+  fmt.dump_float("pi", 3.14);
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), ""
+    "hithere3.14");
+
+  ostringstream oss2;
+  fmt.flush(oss2);
+  ASSERT_EQ(oss2.str(), "");
+
+  ostringstream oss3;
+  fmt.reset();
+  fmt.flush(oss3);
+  ASSERT_EQ(oss3.str(), "");
+}
+
+TEST(XmlFormatter, NamespaceTest) {
+  ostringstream oss;
+  XMLFormatter fmt(false);
+
+  fmt.write_raw_data(XMLFormatter::XML_1_DTD);
+  fmt.open_array_section_in_ns("foo",
+			   "http://s3.amazonaws.com/doc/2006-03-01/");
+  fmt.dump_stream("blah") << "hithere";
+  fmt.dump_float("pi", 3.14);
+  fmt.close_section();
+  fmt.flush(oss);
+  ASSERT_EQ(oss.str(), ""
+    ""
+    "hithere3.14");
+}
diff --git a/ceph/src/test/gather.cc b/ceph/src/test/gather.cc
new file mode 100644
index 00000000..e067ceed
--- /dev/null
+++ b/ceph/src/test/gather.cc
@@ -0,0 +1,103 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 Greg Farnum 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include "include/Context.h"
+#include "test/unit.h"
+
+class C_Checker : public Context {
+public:
+  bool *finish_called;
+  int *result;
+  C_Checker(bool* _finish_called, int *r) :
+    finish_called(_finish_called), result(r) {}
+  void finish(int r) { *finish_called = true; *result = r; }
+};
+
+TEST(ContextGather, Constructor) {
+  C_GatherBuilder gather(g_ceph_context);
+  EXPECT_FALSE(gather.has_subs());
+  EXPECT_TRUE(gather.get() == NULL);
+}
+
+TEST(ContextGather, OneSub) {
+  C_GatherBuilder gather(g_ceph_context);
+  Context *sub = gather.new_sub();
+  EXPECT_EQ(1, gather.num_subs_created());
+  EXPECT_EQ(1, gather.num_subs_remaining());
+
+  bool finish_called = false;
+  int result = 0;
+  C_Checker *checker = new C_Checker(&finish_called, &result);
+  gather.set_finisher(checker);
+  gather.activate();
+  sub->complete(0);
+  EXPECT_TRUE(finish_called);
+  EXPECT_EQ(0, result);
+}
+
+TEST(ContextGather, ManySubs) {
+  bool finish_called = false;
+  int result = 0;
+  C_GatherBuilder gather(g_ceph_context, new C_Checker(&finish_called, &result));
+  int sub_count = 8;
+  Context* subs[sub_count];
+  //create subs and test
+  for (int i = 0; i < sub_count; ++i) {
+    subs[i] = gather.new_sub();
+    EXPECT_EQ(i+1, gather.num_subs_created());
+    EXPECT_EQ(i+1, gather.num_subs_remaining());
+  }
+  EXPECT_TRUE(gather.has_subs());
+  gather.activate();
+
+  //finish all except one sub
+  for (int j = 0; j < sub_count - 1; ++j) {
+    subs[j]->complete(0);
+    EXPECT_FALSE(finish_called);
+  }
+
+  //finish last one and check asserts
+  subs[sub_count-1]->complete(0);
+  EXPECT_TRUE(finish_called);
+}
+
+TEST(ContextGather, AlternatingSubCreateFinish) {
+  C_GatherBuilder gather(g_ceph_context);
+  int sub_count = 8;
+  bool finish_called = false;
+  int result = 0;
+  C_Checker *checker = new C_Checker(&finish_called, &result);
+  gather.set_finisher(checker);
+  Context* subs[sub_count];
+
+  //create half the subs
+  for (int i = 0; i < sub_count / 2; ++i) {
+    subs[i] = gather.new_sub();
+    EXPECT_EQ(i + 1, gather.num_subs_created());
+    EXPECT_EQ(i + 1, gather.num_subs_remaining());
+  }
+
+  //alternate finishing first half of subs and creating last half of subs
+  for (int j = 0; j < sub_count / 2; ++j) {
+    subs[j]->complete(0);
+    subs[sub_count / 2 + j] = gather.new_sub();
+  }
+  gather.activate();
+
+  //finish last half of subs
+  for (int k = sub_count / 2; k < sub_count; ++k) {
+    subs[k]->complete(0);
+  }
+
+  EXPECT_TRUE(finish_called);
+}
diff --git a/ceph/src/test/heartbeat_map.cc b/ceph/src/test/heartbeat_map.cc
new file mode 100644
index 00000000..9ba134af
--- /dev/null
+++ b/ceph/src/test/heartbeat_map.cc
@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/Mutex.h"
+#include "common/HeartbeatMap.h"
+#include "common/ceph_context.h"
+#include "test/unit.h"
+#include "common/config.h"
+
+using namespace ceph;
+
+TEST(HeartbeatMap, Healthy) {
+  HeartbeatMap hm(g_ceph_context);
+  heartbeat_handle_d *h = hm.add_worker("one");
+
+  hm.reset_timeout(h, 9, 18);
+  bool healthy = hm.is_healthy();
+  ASSERT_EQ(healthy, true);
+
+  hm.remove_worker(h);
+}
+
+TEST(HeartbeatMap, Unhealth) {
+  HeartbeatMap hm(g_ceph_context);
+  heartbeat_handle_d *h = hm.add_worker("one");
+
+  hm.reset_timeout(h, 1, 3);
+  sleep(2);
+  bool healthy = hm.is_healthy();
+  ASSERT_EQ(healthy, false);
+
+  hm.remove_worker(h);
+}
diff --git a/ceph/src/test/kv_store_bench.cc b/ceph/src/test/kv_store_bench.cc
new file mode 100644
index 00000000..698105c9
--- /dev/null
+++ b/ceph/src/test/kv_store_bench.cc
@@ -0,0 +1,560 @@
+/*
+ * KvStoreBench.cc
+ *
+ *  Created on: Aug 23, 2012
+ *      Author: eleanor
+ */
+
+#include "test/kv_store_bench.h"
+#include "key_value_store/key_value_structure.h"
+#include "key_value_store/kv_flat_btree_async.h"
+#include "include/rados/librados.hpp"
+#include "test/omap_bench.h"
+#include "common/ceph_argparse.h"
+
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+KvStoreBench::KvStoreBench()
+: entries(30),
+  ops(100),
+  clients(5),
+  key_size(5),
+  val_size(7),
+  max_ops_in_flight(8),
+  clear_first(false),
+  k(2),
+  cache_size(10),
+  cache_refresh(1),
+  client_name("admin"),
+  verbose(false),
+  kvs(NULL),
+  data_lock("data lock"),
+  ops_in_flight(0),
+  ops_in_flight_lock("KvStoreBench::ops_in_flight_lock"),
+  rados_id("admin"),
+  pool_name("data")
+{
+  probs[25] = 'i';
+  probs[50] = 'u';
+  probs[75] = 'd';
+  probs[100] = 'r';
+}
+
+KvStoreBench::~KvStoreBench()
+{
+  librados::ObjectWriteOperation owo;
+  owo.remove();
+  io_ctx.operate(client_name + ".done-setting", &owo);
+  delete kvs;
+}
+
+int KvStoreBench::setup(int argc, const char** argv) {
+  vector args;
+  argv_to_vec(argc,argv,args);
+  srand(time(NULL));
+
+  stringstream help;
+  help
+      << "Usage: KvStoreBench [options]\n"
+      << "Generate latency and throughput statistics for the key value store\n"
+      << "\n"
+      << "There are two sets of options - workload options affect the kind of\n"
+      << "test to run, while algorithm options affect how the key value\n"
+      << "store handles the workload.\n"
+      << "\n"
+      << "There are about entries / k objects in the store to begin with.\n"
+      << "Higher k values reduce the likelihood of splits and the likelihood\n"
+      << "multiple writers simultaneously faling to write because an object \n"
+      << "is full, but having a high k also means there will be more object\n"
+      << "contention.\n"
+      << "\n"
+      << "WORKLOAD OPTIONS\n"
+      << "   --name                           client name (default admin)\n"
+      << "   --entries                             number of key/value pairs to store initially\n"
+      << "                                                 (default " << entries << ")\n"
+      << "   --ops                                 number of operations to run\n"
+      << "   --keysize                             number of characters per key (default " << key_size << ")\n"
+      << "   --valsize                             number of characters per value (default " << val_size << ")\n"
+      << "   -t                                    number of operations in flight concurrently\n"
+      << "                                                 (default " << max_ops_in_flight << ")\n"
+      << "   --clients                             tells this instance how many total clients are. Note that\n"
+      << "                                                 changing this does not change the number of clients."
+      << "   -d              percent (1-100) of operations that should be of each type\n"
+      << "                                                 (default 25 25 25 25)\n"
+      << "   -r                                    random seed to use (default time(0))\n"
+      << "ALGORITHM OPTIONS\n"
+      << "   --kval                                        k, where each object has a number of entries\n"
+      << "                                                 >= k and <= 2k.\n"
+      << "   --cache-size                                  number of index entries to keep in cache\n"
+      << "                                                 (default " << cache_size << ")\n"
+      << "   --cache-refresh                               percent (1-100) of cache-size to read each \n"
+      << "                                                 time the index is read\n"
+      << "OTHER OPTIONS\n"
+      << "   --verbosity-on                                display debug output\n"
+      << "   --clear-first                                 delete all existing objects in the pool before running tests\n";
+  for (unsigned i = 0; i < args.size(); i++) {
+    if(i < args.size() - 1) {
+      if (strcmp(args[i], "--ops") == 0) {
+	ops = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--entries") == 0) {
+	entries = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--kval") == 0) {
+	k = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--keysize") == 0) {
+	key_size = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--valsize") == 0) {
+	val_size = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--cache-size") == 0) {
+	cache_size = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--cache-refresh") == 0) {
+	cache_refresh = 100 / atoi(args[i+1]);
+      } else if (strcmp(args[i], "-t") == 0) {
+	max_ops_in_flight = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--clients") == 0) {
+	clients = atoi(args[i+1]);
+      } else if (strcmp(args[i], "-d") == 0) {
+	if (i + 4 >= args.size()) {
+	  cout << "Invalid arguments after -d: there must be 4 of them."
+	      << std::endl;
+	  continue;
+	} else {
+	  probs.clear();
+	  int sum = atoi(args[i + 1]);
+	  probs[sum] = 'i';
+	  sum += atoi(args[i + 2]);
+	  probs[sum] = 'u';
+	  sum += atoi(args[i + 3]);
+	  probs[sum] = 'd';
+	  sum += atoi(args[i + 4]);
+	  probs[sum] = 'r';
+	  if (sum != 100) {
+	    cout << "Invalid arguments after -d: they must add to 100."
+		<< std::endl;
+	  }
+	}
+      } else if (strcmp(args[i], "--name") == 0) {
+	client_name = args[i+1];
+      } else if (strcmp(args[i], "-r") == 0) {
+	srand(atoi(args[i+1]));
+      }
+    } else if (strcmp(args[i], "--verbosity-on") == 0) {
+      verbose = true;
+    } else if (strcmp(args[i], "--clear-first") == 0) {
+      clear_first = true;
+    } else if (strcmp(args[i], "--help") == 0) {
+      cout << help.str() << std::endl;
+      exit(1);
+    }
+  }
+
+  KvFlatBtreeAsync * kvba = new KvFlatBtreeAsync(k, client_name, cache_size,
+      cache_refresh, verbose);
+  kvs = kvba;
+
+  int r = rados.init(rados_id.c_str());
+  if (r < 0) {
+    cout << "error during init" << std::endl;
+    return r;
+  }
+  r = rados.conf_parse_argv(argc, argv);
+  if (r < 0) {
+    cout << "error during parsing args" << std::endl;
+    return r;
+  }
+  r = rados.conf_parse_env(NULL);
+  if (r < 0) {
+    cout << "error during parsing env" << std::endl;
+    return r;
+  }
+  r = rados.conf_read_file(NULL);
+  if (r < 0) {
+    cout << "error during read file" << std::endl;
+    return r;
+  }
+  r = rados.connect();
+  if (r < 0) {
+    cout << "error during connect: " << r << std::endl;
+    return r;
+  }
+  r = rados.ioctx_create(pool_name.c_str(), io_ctx);
+  if (r < 0) {
+    cout << "error creating io ctx" << std::endl;
+    rados.shutdown();
+    return r;
+  }
+
+  if (clear_first) {
+    librados::ObjectIterator it;
+    for (it = io_ctx.objects_begin(); it != io_ctx.objects_end(); ++it) {
+      librados::ObjectWriteOperation rm;
+      rm.remove();
+      io_ctx.operate(it->first, &rm);
+    }
+  }
+
+  int err = kvs->setup(argc, argv);
+  if (err < 0 && err != -17) {
+    cout << "error during setup of kvs: " << err << std::endl;
+    return err;
+  }
+
+  return 0;
+}
+
+string KvStoreBench::random_string(int len) {
+  string ret;
+  string alphanum = "0123456789"
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+      "abcdefghijklmnopqrstuvwxyz";
+  for (int i = 0; i < len; ++i) {
+    ret.push_back(alphanum[rand() % (alphanum.size() - 1)]);
+  }
+
+  return ret;
+}
+
+pair KvStoreBench::rand_distr(bool new_elem) {
+  pair ret;
+  if (new_elem) {
+    ret = make_pair(random_string(key_size),
+	KvFlatBtreeAsync::to_bl(random_string(val_size)));
+    key_set.insert(ret.first);
+  } else {
+    if (key_set.size() == 0) {
+      return make_pair("",KvFlatBtreeAsync::to_bl(""));
+    }
+    string get_string = random_string(key_size);
+    std::set::iterator it = key_set.lower_bound(get_string);
+    if (it == key_set.end()) {
+      ret.first = *(key_set.rbegin());
+    } else {
+      ret.first = *it;
+    }
+    ret.second = KvFlatBtreeAsync::to_bl(random_string(val_size));
+  }
+  return ret;
+}
+
+int KvStoreBench::test_random_insertions() {
+  int err;
+  if (entries == 0) {
+    return 0;
+  }
+  stringstream prev_ss;
+  prev_ss << (atoi(client_name.c_str()) - 1);
+  string prev_rid = prev_ss.str();
+  stringstream last_ss;
+  if (client_name.size() > 1) {
+    last_ss << client_name.substr(0,client_name.size() - 2);
+  }
+  last_ss << clients - 1;
+  string last_rid = client_name == "admin" ? "admin" : last_ss.str();
+
+  map big_map;
+  for (int i = 0; i < entries; i++) {
+    bufferlist bfr;
+    bfr.append(random_string(7));
+    big_map[random_string(5)] = bfr;
+  }
+
+  uint64_t uint;
+  time_t t;
+  if (client_name[client_name.size() - 1] != '0' && client_name != "admin") {
+    do {
+      librados::ObjectReadOperation oro;
+      oro.stat(&uint, &t, &err);
+      err = io_ctx.operate(prev_rid + ".done-setting", &oro, NULL);
+      if (verbose) cout << "reading " << prev_rid << ": err = " << err
+	  << std::endl;
+    } while (err != 0);
+    cout << "detected " << prev_rid << ".done-setting" << std::endl;
+  }
+
+  cout << "testing random insertions";
+  err = kvs->set_many(big_map);
+  if (err < 0) {
+    cout << "error setting things" << std::endl;
+    return err;
+  }
+
+  librados::ObjectWriteOperation owo;
+  owo.create(true);
+  io_ctx.operate(client_name + ".done-setting", &owo);
+  cout << "created " << client_name + ".done-setting. waiting for "
+      << last_rid << ".done-setting" << std::endl;
+
+  do {
+    librados::ObjectReadOperation oro;
+    oro.stat(&uint, &t, &err);
+    err = io_ctx.operate(last_rid + ".done-setting", &oro, NULL);
+  } while (err != 0);
+  cout << "detected " << last_rid << ".done-setting" << std::endl;
+
+  return err;
+}
+
+void KvStoreBench::aio_callback_timed(int * err, void *arg) {
+  timed_args *args = reinterpret_cast(arg);
+  Mutex * ops_in_flight_lock = &args->kvsb->ops_in_flight_lock;
+  Mutex * data_lock = &args->kvsb->data_lock;
+  Cond * op_avail = &args->kvsb->op_avail;
+  int *ops_in_flight = &args->kvsb->ops_in_flight;
+  if (*err < 0 && *err != -61) {
+    cerr << "Error during " << args->op << " operation: " << *err << std::endl;
+  }
+
+  args->sw.stop_time();
+  double time = args->sw.get_time();
+  args->sw.clear();
+
+  data_lock->Lock();
+  //latency
+  args->kvsb->data.latency_jf.open_object_section("latency");
+  args->kvsb->data.latency_jf.dump_float(string(1, args->op).c_str(),
+      time);
+  args->kvsb->data.latency_jf.close_section();
+
+  //throughput
+  args->kvsb->data.throughput_jf.open_object_section("throughput");
+  args->kvsb->data.throughput_jf.dump_unsigned(string(1, args->op).c_str(),
+      ceph_clock_now(g_ceph_context));
+  args->kvsb->data.throughput_jf.close_section();
+
+  data_lock->Unlock();
+
+  ops_in_flight_lock->Lock();
+  (*ops_in_flight)--;
+  op_avail->Signal();
+  ops_in_flight_lock->Unlock();
+
+  delete args;
+}
+
+int KvStoreBench::test_teuthology_aio(next_gen_t distr,
+    const map &probs)
+{
+  int err = 0;
+  cout << "inserting initial entries..." << std::endl;
+  err = test_random_insertions();
+  if (err < 0) {
+    return err;
+  }
+  cout << "finished inserting initial entries. Waiting 10 seconds for everyone"
+      << " to catch up..." << std::endl;
+
+  sleep(10);
+
+  cout << "done waiting. Starting random operations..." << std::endl;
+
+  Mutex::Locker l(ops_in_flight_lock);
+  for (int i = 0; i < ops; i++) {
+    assert(ops_in_flight <= max_ops_in_flight);
+    if (ops_in_flight == max_ops_in_flight) {
+      int err = op_avail.Wait(ops_in_flight_lock);
+      if (err < 0) {
+	assert(false);
+	return err;
+      }
+      assert(ops_in_flight < max_ops_in_flight);
+    }
+    cout << "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" << i + 1 << " / "
+	<< ops << std::endl;
+    timed_args * cb_args = new timed_args(this);
+    pair kv;
+    int random = (rand() % 100);
+    cb_args->op = probs.lower_bound(random)->second;
+    switch (cb_args->op) {
+    case 'i':
+      kv = (((KvStoreBench *)this)->*distr)(true);
+      if (kv.first == "") {
+	i--;
+	delete cb_args;
+	continue;
+      }
+      ops_in_flight++;
+      cb_args->sw.start_time();
+      kvs->aio_set(kv.first, kv.second, false, aio_callback_timed,
+	  cb_args, &cb_args->err);
+      break;
+    case 'u':
+      kv = (((KvStoreBench *)this)->*distr)(false);
+      if (kv.first == "") {
+	i--;
+	delete cb_args;
+	continue;
+      }
+      ops_in_flight++;
+      cb_args->sw.start_time();
+      kvs->aio_set(kv.first, kv.second, true, aio_callback_timed,
+	  cb_args, &cb_args->err);
+      break;
+    case 'd':
+      kv = (((KvStoreBench *)this)->*distr)(false);
+      if (kv.first == "") {
+	i--;
+	delete cb_args;
+	continue;
+      }
+      key_set.erase(kv.first);
+      ops_in_flight++;
+      cb_args->sw.start_time();
+      kvs->aio_remove(kv.first, aio_callback_timed, cb_args, &cb_args->err);
+      break;
+    case 'r':
+      kv = (((KvStoreBench *)this)->*distr)(false);
+      if (kv.first == "") {
+	i--;
+	delete cb_args;
+	continue;
+      }
+      bufferlist val;
+      ops_in_flight++;
+      cb_args->sw.start_time();
+      kvs->aio_get(kv.first, &cb_args->val, aio_callback_timed,
+	  cb_args, &cb_args->err);
+      break;
+    }
+
+    if (cb_args) {
+      delete cb_args;
+    }
+  }
+
+  while(ops_in_flight > 0) {
+    op_avail.Wait(ops_in_flight_lock);
+  }
+
+  print_time_data();
+  return err;
+}
+
+int KvStoreBench::test_teuthology_sync(next_gen_t distr,
+    const map &probs)
+{
+  int err = 0;
+  err = test_random_insertions();
+  if (err < 0) {
+    return err;
+  }
+  sleep(10);
+  for (int i = 0; i < ops; i++) {
+    StopWatch sw;
+    pair d;
+    cout << "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" << i + 1 << " / "
+	<< ops << std::endl;
+    pair kv;
+    int random = (rand() % 100);
+    d.first = probs.lower_bound(random)->second;
+    switch (d.first) {
+    case 'i':
+      kv = (((KvStoreBench *)this)->*distr)(true);
+      if (kv.first == "") {
+	i--;
+	continue;
+      }
+      sw.start_time();
+      err = kvs->set(kv.first, kv.second, true);
+      sw.stop_time();
+      if (err < 0) {
+	cout << "Error setting " << kv << ": " << err << std::endl;
+	return err;
+      }
+      break;
+    case 'u':
+      kv = (((KvStoreBench *)this)->*distr)(false);
+      if (kv.first == "") {
+	i--;
+	continue;
+      }
+      sw.start_time();
+      err = kvs->set(kv.first, kv.second, true);
+      sw.stop_time();
+      if (err < 0 && err != -61) {
+	cout << "Error updating " << kv << ": " << err << std::endl;
+	return err;
+      }
+      break;
+    case 'd':
+      kv = (((KvStoreBench *)this)->*distr)(false);
+      if (kv.first == "") {
+	i--;
+	continue;
+      }
+      key_set.erase(kv.first);
+      sw.start_time();
+      err = kvs->remove(kv.first);
+      sw.stop_time();
+      if (err < 0 && err != -61) {
+	cout << "Error removing " << kv << ": " << err << std::endl;
+	return err;
+      }
+      break;
+    case 'r':
+      kv = (((KvStoreBench *)this)->*distr)(false);
+      if (kv.first == "") {
+	i--;
+	continue;
+      }
+      bufferlist val;
+      sw.start_time();
+      err = kvs->get(kv.first, &kv.second);
+      sw.stop_time();
+      if (err < 0 && err != -61) {
+	cout << "Error getting " << kv << ": " << err << std::endl;
+	return err;
+      }
+      break;
+    }
+
+    double time = sw.get_time();
+    d.second = time;
+    sw.clear();
+    //latency
+    data.latency_jf.open_object_section("latency");
+    data.latency_jf.dump_float(string(1, d.first).c_str(),
+        time);
+    data.latency_jf.close_section();
+  }
+
+  print_time_data();
+  return err;
+}
+
+void KvStoreBench::print_time_data() {
+  cout << "========================================================\n";
+  cout << "latency:" << std::endl;
+  data.latency_jf.flush(cout);
+  cout << std::endl;
+  cout << "throughput:" << std::endl;
+  data.throughput_jf.flush(cout);
+  cout << "\n========================================================"
+       << std::endl;
+}
+
+int KvStoreBench::teuthology_tests() {
+  int err = 0;
+  if (max_ops_in_flight > 1) {
+    test_teuthology_aio(&KvStoreBench::rand_distr, probs);
+  } else {
+    err = test_teuthology_sync(&KvStoreBench::rand_distr, probs);
+  }
+  return err;
+}
+
+int main(int argc, const char** argv) {
+  KvStoreBench kvsb;
+  int err = kvsb.setup(argc, argv);
+  if (err == 0) cout << "setup successful" << std::endl;
+  else{
+    cout << "error " << err << std::endl;
+    return err;
+  }
+  err = kvsb.teuthology_tests();
+  if (err < 0) return err;
+  return 0;
+};
diff --git a/ceph/src/test/kv_store_bench.h b/ceph/src/test/kv_store_bench.h
new file mode 100644
index 00000000..3ebb15fd
--- /dev/null
+++ b/ceph/src/test/kv_store_bench.h
@@ -0,0 +1,194 @@
+/*
+ * Benchmarking suite for key-value store
+ *
+ * September 2, 2012
+ * Eleanor Cawthon
+ * eleanor.cawthon@inktank.com
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef KVSTOREBENCH_H_
+#define KVSTOREBENCH_H_
+
+#include "key_value_store/key_value_structure.h"
+#include "key_value_store/kv_flat_btree_async.h"
+#include "common/Clock.h"
+#include "global/global_context.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using namespace std;
+using ceph::bufferlist;
+
+/**
+ * stores pairings from op type to time taken for that op (for latency), and to
+ * time that op completed to the nearest second (for throughput).
+ */
+struct kv_bench_data {
+  JSONFormatter throughput_jf;
+
+  JSONFormatter latency_jf;
+};
+
+class KvStoreBench;
+
+/**
+ * keeps track of the number of milliseconds between two events - used to
+ * measure latency
+ */
+struct StopWatch {
+  utime_t begin_time;
+  utime_t end_time;
+
+  void start_time() {
+    begin_time = ceph_clock_now(g_ceph_context);
+  }
+  void stop_time() {
+    end_time = ceph_clock_now(g_ceph_context);
+  }
+  double get_time() {
+    return (end_time - begin_time) * 1000;
+  }
+  void clear() {
+    begin_time = end_time = utime_t();
+  }
+};
+
+/**
+ * arguments passed to the callback method when the op is being timed
+ */
+struct timed_args {
+  StopWatch sw;
+  //kv_bench_data data;
+  KvStoreBench * kvsb;
+  bufferlist val;
+  int err;
+  char op;
+
+  timed_args ()
+  : kvsb(NULL),
+    err(0),
+    op(' ')
+  {};
+
+  timed_args (KvStoreBench * k)
+  : kvsb(k),
+    err(0),
+    op(' ')
+  {}
+};
+
+typedef pair (KvStoreBench::*next_gen_t)(bool new_elem);
+
+class KvStoreBench {
+
+protected:
+
+  //test setup variables set from command line
+  int entries; //the number of entries to write initially
+  int ops; //the number of operations to time
+  int clients; //the total number of clients running this test - used
+	       //in the aio test to coordinate the end of the initial sets
+  int key_size;//number of characters in keys to write
+  int val_size;//number of characters in values to write
+  int max_ops_in_flight;
+  bool clear_first;//if true, remove all objects in pool before starting tests
+
+  //variables passed to KeyValueStructure
+  int k;
+  int cache_size; //number of index entries to store in cache
+  double cache_refresh; //cache_size / cache_refresh entries are read each time
+			//the index is read
+  string client_name;
+  bool verbose;//if true, display debug output
+
+  //internal
+  map probs;//map of numbers from 1 to 100 to chars representing
+			//operation types - used to generate random operations
+  set key_set;//set of keys already in the data set
+  KeyValueStructure * kvs;
+  kv_bench_data data;//stores throughput and latency from completed tests
+  Mutex data_lock;
+  Cond op_avail;//signaled when an op completes
+  int ops_in_flight;//number of operations currently in progress
+  Mutex ops_in_flight_lock;
+  //these are used for cleanup and setup purposes - they are NOT passed to kvs!
+  librados::Rados rados;
+  string rados_id;
+  string pool_name;
+  librados::IoCtx io_ctx;
+
+  /**
+   * Prints JSON-formatted throughput and latency data.
+   *
+   * Throughput data is {'char representing the operation type':time the op
+   * completed to the nearest second}
+   * Latency is {'char representing the operation type':time taken by the op}
+   */
+  void print_time_data();
+
+public:
+
+  KvStoreBench();
+
+  //after this is called, objects created by the KeyValueStructure remain.
+  ~KvStoreBench();
+
+  /**
+   * parses command line arguments, sets up this rados instance, clears the
+   * pool if clear_first is true and calls kvs->setup.
+   */
+  int setup(int argc, const char** argv);
+
+  /**
+   * Returns a string of random characters of length len
+   */
+  string random_string(int len);
+
+  /**
+   * Inserts entries random keys and values asynchronously.
+   */
+  int test_random_insertions();
+
+  /**
+   * calls test_random_insertions, then does ops randomly chosen operations
+   * asynchronously, with max_ops_in_flight operations at a time.
+   */
+  int test_teuthology_aio(next_gen_t distr, const map &probs);
+
+  /**
+   * calls test_random_insertions, then does ops randomly chosen operations
+   * synchronously.
+   */
+  int test_teuthology_sync(next_gen_t distr, const map &probs);
+
+  /**
+   * returns a key-value pair. If new_elem is true, the key is randomly
+   * generated. If it is false, the key is selected from the keys currently in
+   * the key set.
+   */
+  pair rand_distr(bool new_elem);
+
+  /**
+   * Called when aio operations complete. Updates data.
+   */
+  static void aio_callback_timed(int * err, void *arg);
+
+  /**
+   * Calls test_ methods. Change to call, for example, multiple runs of a test
+   * with different settings. Currently just calls test_teuthology_aio.
+   */
+  int teuthology_tests();
+
+};
+
+#endif /* KVSTOREBENCH_H_ */
diff --git a/ceph/src/test/libcephfs/caps.cc b/ceph/src/test/libcephfs/caps.cc
new file mode 100644
index 00000000..e49c63c3
--- /dev/null
+++ b/ceph/src/test/libcephfs/caps.cc
@@ -0,0 +1,92 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/ceph_fs.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+TEST(Caps, ReadZero) {
+
+  int mypid = getpid();
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  int i = 0;
+  for(; i < 30; ++i) {
+
+    char c_path[1024];
+    sprintf(c_path, "/caps_rzfile_%d_%d", mypid, i);
+    int fd = ceph_open(cmount, c_path, O_CREAT|O_TRUNC|O_WRONLY, 0644);
+    ASSERT_LT(0, fd);
+
+    int expect = CEPH_CAP_FILE_EXCL | CEPH_CAP_FILE_WR | CEPH_CAP_FILE_BUFFER;
+    int caps = ceph_debug_get_fd_caps(cmount, fd);
+
+    ASSERT_EQ(expect, caps & expect);
+    ASSERT_EQ(0, ceph_close(cmount, fd));
+
+    caps = ceph_debug_get_file_caps(cmount, c_path);
+    ASSERT_EQ(expect, caps & expect);
+
+    char cw_path[1024];
+    sprintf(cw_path, "/caps_wzfile_%d_%d", mypid, i);
+    int wfd = ceph_open(cmount, cw_path, O_CREAT|O_TRUNC|O_WRONLY, 0644);
+    ASSERT_LT(0, wfd);
+
+    char wbuf[4096];
+    ASSERT_EQ(4096, ceph_write(cmount, wfd, wbuf, 4096, 0));
+
+    ASSERT_EQ(0, ceph_close(cmount, wfd));
+
+    struct stat st;
+    ASSERT_EQ(0, ceph_stat(cmount, c_path, &st));
+
+    caps = ceph_debug_get_file_caps(cmount, c_path);
+    ASSERT_EQ(expect, caps & expect);
+  }
+
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_debug_inject_tick_delay", "20"));
+
+  for(i = 0; i < 30; ++i) {
+
+    char c_path[1024];
+    sprintf(c_path, "/caps_rzfile_%d_%d", mypid, i);
+
+    int fd = ceph_open(cmount, c_path, O_RDONLY, 0);
+    ASSERT_LT(0, fd);
+    char buf[256];
+
+    int expect = CEPH_CAP_FILE_RD | CEPH_STAT_CAP_SIZE | CEPH_CAP_FILE_CACHE;
+    int caps = ceph_debug_get_fd_caps(cmount, fd);
+    ASSERT_EQ(expect, caps & expect);
+    ASSERT_EQ(0, ceph_read(cmount, fd, buf, 256, 0));
+
+    caps = ceph_debug_get_fd_caps(cmount, fd);
+    ASSERT_EQ(expect, caps & expect);
+    ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  }
+  ceph_shutdown(cmount);
+}
diff --git a/ceph/src/test/libcephfs/multiclient.cc b/ceph/src/test/libcephfs/multiclient.cc
new file mode 100644
index 00000000..78fdb9ab
--- /dev/null
+++ b/ceph/src/test/libcephfs/multiclient.cc
@@ -0,0 +1,97 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+TEST(LibCephFS, MulticlientSimple) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo.%d", getpid());
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+  int fdb = ceph_open(cb, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+
+  char bufa[4] = "foo";
+  char bufb[4];
+
+  for (int i=0; i<10; i++) {
+    strcpy(bufa, "foo");
+    ASSERT_EQ((int)sizeof(bufa), ceph_write(ca, fda, bufa, sizeof(bufa), i*6));
+    ASSERT_EQ((int)sizeof(bufa), ceph_read(cb, fdb, bufb, sizeof(bufa), i*6));
+    ASSERT_EQ(0, memcmp(bufa, bufb, sizeof(bufa)));
+    strcpy(bufb, "bar");
+    ASSERT_EQ((int)sizeof(bufb), ceph_write(cb, fdb, bufb, sizeof(bufb), i*6+3));
+    ASSERT_EQ((int)sizeof(bufb), ceph_read(ca, fda, bufa, sizeof(bufb), i*6+3));
+    ASSERT_EQ(0, memcmp(bufa, bufb, sizeof(bufa)));
+  }
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+TEST(LibCephFS, MulticlientHoleEOF) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo.%d", getpid());
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+  int fdb = ceph_open(cb, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+
+  ASSERT_EQ(3, ceph_write(ca, fda, "foo", 3, 0));
+  ASSERT_EQ(0, ceph_ftruncate(ca, fda, 1000000));
+
+  char buf[4];
+  ASSERT_EQ(2, ceph_read(cb, fdb, buf, sizeof(buf), 1000000-2));
+  ASSERT_EQ(0, buf[0]);
+  ASSERT_EQ(0, buf[1]);
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
diff --git a/ceph/src/test/libcephfs/readdir_r_cb.cc b/ceph/src/test/libcephfs/readdir_r_cb.cc
new file mode 100644
index 00000000..946a4055
--- /dev/null
+++ b/ceph/src/test/libcephfs/readdir_r_cb.cc
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include 
+#include 
+
+TEST(LibCephFS, ReaddirRCB) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char c_dir[256];
+  sprintf(c_dir, "/readdir_r_cb_tests_%d", getpid());
+  struct ceph_dir_result *dirp;
+  ASSERT_EQ(0, ceph_mkdirs(cmount, c_dir, 0777));
+  ASSERT_LE(0, ceph_opendir(cmount, c_dir, &dirp));
+
+  // dir is empty, check that it only contains . and ..
+  int buflen = 100;
+  char *buf = new char[buflen];
+  // . is 2, .. is 3 (for null terminators)
+  ASSERT_EQ(5, ceph_getdnames(cmount, dirp, buf, buflen));
+  char c_file[256];
+  sprintf(c_file, "/readdir_r_cb_tests_%d/foo", getpid());
+  int fd = ceph_open(cmount, c_file, O_CREAT, 0777);
+  ASSERT_LT(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  // check correctness with one entry
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_LE(0, ceph_opendir(cmount, c_dir, &dirp));
+  ASSERT_EQ(9, ceph_getdnames(cmount, dirp, buf, buflen)); // ., .., foo
+
+  // check correctness if buffer is too small
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_GE(0, ceph_opendir(cmount, c_dir, &dirp));
+  ASSERT_EQ(-ERANGE, ceph_getdnames(cmount, dirp, buf, 1));
+
+  //check correctness if it needs to split listing
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_LE(0, ceph_opendir(cmount, c_dir, &dirp));
+  ASSERT_EQ(5, ceph_getdnames(cmount, dirp, buf, 6));
+  ASSERT_EQ(4, ceph_getdnames(cmount, dirp, buf, 6));
+
+  // free cmount after finishing testing
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ASSERT_EQ(0, ceph_release(cmount));
+}
diff --git a/ceph/src/test/libcephfs/test.cc b/ceph/src/test/libcephfs/test.cc
new file mode 100644
index 00000000..9d917f51
--- /dev/null
+++ b/ceph/src/test/libcephfs/test.cc
@@ -0,0 +1,1163 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+TEST(LibCephFS, OpenEmptyComponent) {
+
+  pid_t mypid = getpid();
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char c_dir[1024];
+  sprintf(c_dir, "/open_test_%d", mypid);
+  struct ceph_dir_result *dirp;
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, c_dir, 0777));
+
+  ASSERT_EQ(0, ceph_opendir(cmount, c_dir, &dirp));
+
+  char c_path[1024];
+  sprintf(c_path, "/open_test_%d//created_file_%d", mypid, mypid);
+  int fd = ceph_open(cmount, c_path, O_RDONLY|O_CREAT, 0666);
+  ASSERT_LT(0, fd);
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_closedir(cmount, dirp));
+  ceph_shutdown(cmount);
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  fd = ceph_open(cmount, c_path, O_RDONLY, 0666);
+  ASSERT_LT(0, fd);
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MountNonExist) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_NE(0, ceph_mount(cmount, "/non-exist"));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MountDouble) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(-EISCONN, ceph_mount(cmount, "/"));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MountRemount) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+
+  CephContext *cct = ceph_get_mount_context(cmount);
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(cct, ceph_get_mount_context(cmount));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, UnmountUnmounted) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(-ENOTCONN, ceph_unmount(cmount));
+}
+
+TEST(LibCephFS, ReleaseUnmounted) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_release(cmount));
+}
+
+TEST(LibCephFS, ReleaseMounted) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(-EISCONN, ceph_release(cmount));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, UnmountRelease) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ASSERT_EQ(0, ceph_release(cmount));
+}
+
+TEST(LibCephFS, Mount) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ceph_shutdown(cmount);
+
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, OpenLayout) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  /* valid layout */
+  char test_layout_file[256];
+  sprintf(test_layout_file, "test_layout_%d_b", getpid());
+  int fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), NULL);
+  ASSERT_GT(fd, 0);
+  char poolname[80];
+  ASSERT_LT(0, ceph_get_file_pool_name(cmount, fd, poolname, sizeof(poolname)));
+  ASSERT_EQ(4, ceph_get_file_pool_name(cmount, fd, poolname, 0));
+  ASSERT_EQ(0, strcmp("data", poolname));
+  ceph_close(cmount, fd);
+
+  /* invalid layout */
+  sprintf(test_layout_file, "test_layout_%d_c", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 1, 19, NULL);
+  ASSERT_EQ(fd, -EINVAL);
+
+  /* with data pool */
+  sprintf(test_layout_file, "test_layout_%d_d", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), "data");
+  ASSERT_GT(fd, 0);
+  ASSERT_EQ(4, ceph_get_file_pool_name(cmount, fd, poolname, sizeof(poolname)));
+  ASSERT_EQ(0, strcmp("data", poolname));
+  ceph_close(cmount, fd);
+
+  /* with metadata pool (invalid) */
+  sprintf(test_layout_file, "test_layout_%d_e", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), "metadata");
+  ASSERT_EQ(fd, -EINVAL);
+
+  /* with metadata pool (does not exist) */
+  sprintf(test_layout_file, "test_layout_%d_f", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), "asdfjasdfjasdf");
+  ASSERT_EQ(fd, -EINVAL);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DirLs) {
+
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  struct ceph_dir_result *ls_dir = NULL;
+  char foostr[256];
+  sprintf(foostr, "dir_ls%d", mypid);
+  ASSERT_EQ(ceph_opendir(cmount, foostr, &ls_dir), -ENOENT);
+
+  ASSERT_EQ(ceph_mkdir(cmount, foostr, 0777), 0);
+  struct stat stbuf;
+  ASSERT_EQ(ceph_stat(cmount, foostr, &stbuf), 0);
+  ASSERT_NE(S_ISDIR(stbuf.st_mode), 0);
+
+  char barstr[256];
+  sprintf(barstr, "dir_ls2%d", mypid);
+  ASSERT_EQ(ceph_lstat(cmount, barstr, &stbuf), -ENOENT);
+
+  // insert files into directory and test open
+  char bazstr[256];
+  int i = 0, r = rand() % 4096;
+  if (getenv("LIBCEPHFS_RAND")) {
+    r = atoi(getenv("LIBCEPHFS_RAND"));
+  }
+  printf("rand: %d\n", r);
+  for(; i < r; ++i) {
+
+    sprintf(bazstr, "dir_ls%d/dirf%d", mypid, i);
+    int fd  = ceph_open(cmount, bazstr, O_CREAT|O_RDONLY, 0666);
+    ASSERT_GT(fd, 0);
+    ASSERT_EQ(ceph_close(cmount, fd), 0);
+
+    // set file sizes for readdirplus
+    ceph_truncate(cmount, bazstr, i);
+  }
+
+  ASSERT_EQ(ceph_opendir(cmount, foostr, &ls_dir), 0);
+
+  // not guaranteed to get . and .. first, but its a safe assumption in this case
+  struct dirent *result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  std::vector > entries;
+  // check readdir and capture stream order for future tests
+  for(i = 0; i < r; ++i) {
+
+    result = ceph_readdir(cmount, ls_dir);
+    ASSERT_TRUE(result != NULL);
+
+    int size;
+    sscanf(result->d_name, "dirf%d", &size);
+    entries.push_back(std::pair(strdup(result->d_name), size));
+  }
+
+  ASSERT_TRUE(ceph_readdir(cmount, ls_dir) == NULL);
+
+  // test rewinddir
+  ceph_rewinddir(cmount, ls_dir);
+
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  // check telldir
+  for(i = 0; i < r-1; ++i) {
+    int r = ceph_telldir(cmount, ls_dir);
+    ASSERT_GT(r, -1);
+    ceph_seekdir(cmount, ls_dir, r);
+    result = ceph_readdir(cmount, ls_dir);
+    ASSERT_TRUE(result != NULL);
+    ASSERT_STREQ(result->d_name, entries[i].first);
+  }
+
+  ceph_rewinddir(cmount, ls_dir);
+
+  int t = ceph_telldir(cmount, ls_dir);
+  ASSERT_GT(t, -1);
+
+  ASSERT_TRUE(ceph_readdir(cmount, ls_dir) != NULL);
+
+  // test seekdir - move back to the beginning
+  ceph_seekdir(cmount, ls_dir, t);
+
+  // test getdents
+  struct dirent *getdents_entries;
+  getdents_entries = (struct dirent *)malloc(r * sizeof(*getdents_entries));
+
+  int count = 0;
+  while (count < r) {
+    int len = ceph_getdents(cmount, ls_dir, (char *)getdents_entries, r * sizeof(*getdents_entries));
+    ASSERT_GT(len, 0);
+    ASSERT_TRUE((len % sizeof(*getdents_entries)) == 0);
+    int n = len / sizeof(*getdents_entries);
+    if (count == 0) {
+      ASSERT_STREQ(getdents_entries[0].d_name, ".");
+      ASSERT_STREQ(getdents_entries[1].d_name, "..");
+    }
+    int j;
+    i = count;
+    for(j = 2; j < n; ++i, ++j) {
+      ASSERT_STREQ(getdents_entries[j].d_name, entries[i].first);
+    }
+    count += n;
+  }
+
+  free(getdents_entries);
+
+  // test readdir_r
+  ceph_rewinddir(cmount, ls_dir);
+
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  for(i = 0; i < r; ++i) {
+    struct dirent rdent;
+    ASSERT_EQ(ceph_readdir_r(cmount, ls_dir, &rdent), 1);
+    ASSERT_STREQ(rdent.d_name, entries[i].first);
+  }
+
+  // test readdirplus
+  ceph_rewinddir(cmount, ls_dir);
+
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  for(i = 0; i < r; ++i) {
+    struct dirent rdent;
+    struct stat st;
+    int stmask;
+    ASSERT_EQ(ceph_readdirplus_r(cmount, ls_dir, &rdent, &st, &stmask), 1);
+    ASSERT_STREQ(rdent.d_name, entries[i].first);
+    ASSERT_EQ(st.st_size, entries[i].second);
+    ASSERT_EQ(st.st_ino, rdent.d_ino);
+    //ASSERT_EQ(st.st_mode, (mode_t)0666);
+  }
+
+  ASSERT_EQ(ceph_closedir(cmount, ls_dir), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ManyNestedDirs) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  const char *many_path = "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a";
+  ASSERT_EQ(ceph_mkdirs(cmount, many_path, 0755), 0);
+
+  int i = 0;
+
+  for(; i < 39; ++i) {
+    ASSERT_EQ(ceph_chdir(cmount, "a"), 0);
+
+    struct ceph_dir_result *dirp;
+    ASSERT_EQ(ceph_opendir(cmount, "a", &dirp), 0);
+    struct dirent *dent = ceph_readdir(cmount, dirp);
+    ASSERT_TRUE(dent != NULL);
+    ASSERT_STREQ(dent->d_name, ".");
+    dent = ceph_readdir(cmount, dirp);
+    ASSERT_TRUE(dent != NULL);
+    ASSERT_STREQ(dent->d_name, "..");
+    dent = ceph_readdir(cmount, dirp);
+    ASSERT_TRUE(dent != NULL);
+    ASSERT_STREQ(dent->d_name, "a");
+    ASSERT_EQ(ceph_closedir(cmount, dirp), 0);
+  }
+
+  ASSERT_STREQ(ceph_getcwd(cmount), "/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a");
+
+  ASSERT_EQ(ceph_chdir(cmount, "a/a/a"), 0);
+
+  for(i = 0; i < 39; ++i) {
+    ASSERT_EQ(ceph_chdir(cmount, ".."), 0);
+    ASSERT_EQ(ceph_rmdir(cmount, "a"), 0);
+  }
+
+  ASSERT_EQ(ceph_chdir(cmount, "/"), 0);
+
+  ASSERT_EQ(ceph_rmdir(cmount, "a/a/a"), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Xattrs) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_xattr_file[256];
+  sprintf(test_xattr_file, "test_xattr_%d", getpid());
+  int fd = ceph_open(cmount, test_xattr_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+
+  char i = 'a';
+  char xattrk[128];
+  char xattrv[128];
+  for(; i < 'a'+26; ++i) {
+    sprintf(xattrk, "user.test_xattr_%c", i);
+    int len = sprintf(xattrv, "testxattr%c", i);
+    ASSERT_EQ(ceph_setxattr(cmount, test_xattr_file, xattrk, (void *) xattrv, len, XATTR_CREATE), 0);
+  }
+
+  char xattrlist[128*26];
+  int len = ceph_listxattr(cmount, test_xattr_file, xattrlist, sizeof(xattrlist));
+  char *p = xattrlist;
+  char *n;
+  i = 'a';
+  while (len > 0) {
+    // skip/ignore the dir layout
+    if (strcmp(p, "ceph.dir.layout") == 0 ||
+	strcmp(p, "ceph.file.layout") == 0) {
+      len -= strlen(p) + 1;
+      p += strlen(p) + 1;
+      continue;
+    }
+
+    sprintf(xattrk, "user.test_xattr_%c", i);
+    ASSERT_STREQ(p, xattrk);
+
+    char gxattrv[128];
+    std::cout << "getting attr " << p << std::endl;
+    int alen = ceph_getxattr(cmount, test_xattr_file, p, (void *) gxattrv, 128);
+    ASSERT_GT(alen, 0);
+    sprintf(xattrv, "testxattr%c", i);
+    ASSERT_TRUE(!strncmp(xattrv, gxattrv, alen));
+
+    n = index(p, '\0');
+    n++;
+    len -= (n - p);
+    p = n;
+    ++i;
+  }
+
+  i = 'a';
+  for(i = 'a'; i < 'a'+26; ++i) {
+    sprintf(xattrk, "user.test_xattr_%c", i);
+    ASSERT_EQ(ceph_removexattr(cmount, test_xattr_file, xattrk), 0);
+  }
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+
+}
+
+TEST(LibCephFS, Xattrs_ll) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_xattr_file[256];
+  sprintf(test_xattr_file, "test_xattr_%d", getpid());
+  int fd = ceph_open(cmount, test_xattr_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  Inode *root = NULL;
+  Inode *existent_file_handle = NULL;
+  struct stat attr;
+
+  int res = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(res, 0);
+  res = ceph_ll_lookup(cmount, root, test_xattr_file, &attr, &existent_file_handle, 0, 0);
+  ASSERT_EQ(res, 0);
+
+  const char *valid_name = "user.attrname";
+  const char *value = "attrvalue";
+  char value_buf[256] = { 0 };
+
+  res = ceph_ll_setxattr(cmount, existent_file_handle, valid_name, value, strlen(value), 0, 0, 0);
+  ASSERT_EQ(res, 0);
+
+  res = ceph_ll_getxattr(cmount, existent_file_handle, valid_name, value_buf, 256, 0, 0);
+  ASSERT_EQ(res, (int)strlen(value));
+
+  value_buf[res] = '\0';
+  ASSERT_STREQ(value_buf, value);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LstatSlashdot) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  struct stat stbuf;
+  ASSERT_EQ(ceph_lstat(cmount, "/.", &stbuf), 0);
+  ASSERT_EQ(ceph_lstat(cmount, ".", &stbuf), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DoubleChmod) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_perms_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // write some stuff
+  const char *bytes = "foobarbaz";
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+
+  ceph_close(cmount, fd);
+
+  // set perms to read but can't write
+  ASSERT_EQ(ceph_chmod(cmount, test_file, 0400), 0);
+
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_EQ(fd, -EACCES);
+
+  fd = ceph_open(cmount, test_file, O_RDONLY, 0);
+  ASSERT_GT(fd, -1);
+
+  char buf[100];
+  int ret = ceph_read(cmount, fd, buf, 100, 0);
+  ASSERT_EQ(ret, (int)strlen(bytes));
+  buf[ret] = '\0';
+  ASSERT_STREQ(buf, bytes);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), -EBADF);
+
+  ceph_close(cmount, fd);
+
+  // reset back to writeable
+  ASSERT_EQ(ceph_chmod(cmount, test_file, 0600), 0);
+
+  // ensure perms are correct
+  struct stat stbuf;
+  ASSERT_EQ(ceph_lstat(cmount, test_file, &stbuf), 0);
+  ASSERT_EQ(stbuf.st_mode, 0100600U);
+
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_GT(fd, 0);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Fchmod) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_perms_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // write some stuff
+  const char *bytes = "foobarbaz";
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+
+  // set perms to read but can't write
+  ASSERT_EQ(ceph_fchmod(cmount, fd, 0400), 0);
+
+  char buf[100];
+  int ret = ceph_read(cmount, fd, buf, 100, 0);
+  ASSERT_EQ(ret, (int)strlen(bytes));
+  buf[ret] = '\0';
+  ASSERT_STREQ(buf, bytes);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+
+  ceph_close(cmount, fd);
+
+  ASSERT_EQ(ceph_open(cmount, test_file, O_RDWR, 0), -EACCES);
+
+  // reset back to writeable
+  ASSERT_EQ(ceph_chmod(cmount, test_file, 0600), 0);
+
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_GT(fd, 0);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Fchown) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_fchown_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // set perms to readable and writeable only by owner
+  ASSERT_EQ(ceph_fchmod(cmount, fd, 0600), 0);
+
+  // change ownership to nobody -- we assume nobody exists and id is always 65534
+  ASSERT_EQ(ceph_fchown(cmount, fd, 65534, 65534), 0);
+
+  ceph_close(cmount, fd);
+
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_EQ(fd, -EACCES);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Symlinks) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_symlinks_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  char test_symlink[256];
+  sprintf(test_symlink, "test_symlinks_sym_%d", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_file, test_symlink), 0);
+
+  // stat the original file
+  struct stat stbuf_orig;
+  ASSERT_EQ(ceph_stat(cmount, test_file, &stbuf_orig), 0);
+  // stat the symlink
+  struct stat stbuf_symlink_orig;
+  ASSERT_EQ(ceph_stat(cmount, test_symlink, &stbuf_symlink_orig), 0);
+  // ensure the stat bufs are equal
+  ASSERT_TRUE(!memcmp(&stbuf_orig, &stbuf_symlink_orig, sizeof(stbuf_orig)));
+
+  sprintf(test_file, "/test_symlinks_abs_%d", getpid());
+
+  fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  sprintf(test_symlink, "/test_symlinks_abs_sym_%d", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_file, test_symlink), 0);
+
+  // stat the original file
+  ASSERT_EQ(ceph_stat(cmount, test_file, &stbuf_orig), 0);
+  // stat the symlink
+  ASSERT_EQ(ceph_stat(cmount, test_symlink, &stbuf_symlink_orig), 0);
+  // ensure the stat bufs are equal
+  ASSERT_TRUE(!memcmp(&stbuf_orig, &stbuf_symlink_orig, sizeof(stbuf_orig)));
+
+  // test lstat
+  struct stat stbuf_symlink;
+  ASSERT_EQ(ceph_lstat(cmount, test_symlink, &stbuf_symlink), 0);
+  ASSERT_TRUE(S_ISLNK(stbuf_symlink.st_mode));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DirSyms) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_dir1[256];
+  sprintf(test_dir1, "dir1_symlinks_%d", getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir1, 0700), 0);
+
+  char test_symdir[256];
+  sprintf(test_symdir, "symdir_symlinks_%d", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_dir1, test_symdir), 0);
+
+  char test_file[256];
+  sprintf(test_file, "/symdir_symlinks_%d/test_symdir_file", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0600);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  struct stat stbuf;
+  ASSERT_EQ(ceph_lstat(cmount, test_file, &stbuf), 0);
+
+  // ensure that its a file not a directory we get back
+  ASSERT_TRUE(S_ISREG(stbuf.st_mode));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LoopSyms) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_dir1[256];
+  sprintf(test_dir1, "dir1_loopsym_%d", getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir1, 0700), 0);
+
+  char test_dir2[256];
+  sprintf(test_dir2, "/dir1_loopsym_%d/loop_dir", getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir2, 0700), 0);
+
+  // symlink it itself:  /path/to/mysym -> /path/to/mysym
+  char test_symdir[256];
+  sprintf(test_symdir, "/dir1_loopsym_%d/loop_dir/symdir", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_symdir, test_symdir), 0);
+
+  char test_file[256];
+  sprintf(test_file, "/dir1_loopsym_%d/loop_dir/symdir/test_loopsym_file", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0600);
+  ASSERT_EQ(fd, -ELOOP);
+
+  // loop: /a -> /b, /b -> /c, /c -> /a
+  char a[256], b[256], c[256];
+  sprintf(a, "/%s/a", test_dir1);
+  sprintf(b, "/%s/b", test_dir1);
+  sprintf(c, "/%s/c", test_dir1);
+  ASSERT_EQ(ceph_symlink(cmount, a, b), 0);
+  ASSERT_EQ(ceph_symlink(cmount, b, c), 0);
+  ASSERT_EQ(ceph_symlink(cmount, c, a), 0);
+  ASSERT_EQ(ceph_open(cmount, a, O_RDWR, 0), -ELOOP);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, HardlinkNoOriginal) {
+
+  int mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir[256];
+  sprintf(dir, "/test_rmdirfail%d", mypid);
+  ASSERT_EQ(ceph_mkdir(cmount, dir, 0777), 0);
+
+  ASSERT_EQ(ceph_chdir(cmount, dir), 0);
+
+  int fd = ceph_open(cmount, "f1", O_CREAT, 0644);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  // create hard link
+  ASSERT_EQ(ceph_link(cmount, "f1", "hardl1"), 0);
+
+  // remove file link points to
+  ASSERT_EQ(ceph_unlink(cmount, "f1"), 0);
+
+  ceph_shutdown(cmount);
+
+  // now cleanup
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ASSERT_EQ(ceph_chdir(cmount, dir), 0);
+  ASSERT_EQ(ceph_unlink(cmount, "hardl1"), 0);
+  ASSERT_EQ(ceph_rmdir(cmount, dir), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, BadFileDesc) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_EQ(ceph_fchmod(cmount, -1, 0655), -EBADF);
+  ASSERT_EQ(ceph_close(cmount, -1), -EBADF);
+  ASSERT_EQ(ceph_lseek(cmount, -1, 0, SEEK_SET), -EBADF);
+
+  char buf[0];
+  ASSERT_EQ(ceph_read(cmount, -1, buf, 0, 0), -EBADF);
+  ASSERT_EQ(ceph_write(cmount, -1, buf, 0, 0), -EBADF);
+
+  ASSERT_EQ(ceph_ftruncate(cmount, -1, 0), -EBADF);
+  ASSERT_EQ(ceph_fsync(cmount, -1, 0), -EBADF);
+
+  struct stat stat;
+  ASSERT_EQ(ceph_fstat(cmount, -1, &stat), -EBADF);
+
+  struct sockaddr_storage addr;
+  ASSERT_EQ(ceph_get_file_stripe_address(cmount, -1, 0, &addr, 1), -EBADF);
+
+  ASSERT_EQ(ceph_get_file_stripe_unit(cmount, -1), -EBADF);
+  ASSERT_EQ(ceph_get_file_pool(cmount, -1), -EBADF);
+  ASSERT_EQ(ceph_get_file_replication(cmount, -1), -EBADF);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ReadEmptyFile) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  // test the read_sync path in the client for zero files
+  ASSERT_EQ(ceph_conf_set(cmount, "client_debug_force_sync_read", "true"), 0);
+
+  int mypid = getpid();
+  char testf[256];
+
+  sprintf(testf, "test_reademptyfile%d", mypid);
+  int fd = ceph_open(cmount, testf, O_CREAT|O_TRUNC|O_WRONLY, 0644);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  fd = ceph_open(cmount, testf, O_RDONLY, 0);
+  ASSERT_GT(fd, 0);
+
+  char buf[4096];
+  ASSERT_EQ(ceph_read(cmount, fd, buf, 4096, 0), 0);
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, StripeUnitGran) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ASSERT_GT(ceph_get_stripe_unit_granularity(cmount), 0);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Rename) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  int mypid = getpid();
+  char path_src[256];
+  char path_dst[256];
+
+  /* make a source file */
+  sprintf(path_src, "test_rename_src%d", mypid);
+  int fd = ceph_open(cmount, path_src, O_CREAT|O_TRUNC|O_WRONLY, 0777);
+  ASSERT_GT(fd, 0);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  /* rename to a new dest path */
+  sprintf(path_dst, "test_rename_dst%d", mypid);
+  ASSERT_EQ(0, ceph_rename(cmount, path_src, path_dst));
+
+  /* test that dest path exists */
+  struct stat st;
+  ASSERT_EQ(0, ceph_lstat(cmount, path_dst, &st));
+
+  /* test that src path doesn't exist */
+  ASSERT_EQ(-ENOENT, ceph_lstat(cmount, path_src, &st));
+
+  /* rename with non-existent source path */
+  ASSERT_EQ(-ENOENT, ceph_rename(cmount, path_src, path_dst));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, path_dst));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, UseUnmounted) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+
+  struct statvfs stvfs;
+  EXPECT_EQ(-ENOTCONN, ceph_statfs(cmount, "/", &stvfs));
+  EXPECT_EQ(-ENOTCONN, ceph_get_local_osd(cmount));
+  EXPECT_EQ(-ENOTCONN, ceph_chdir(cmount, "/"));
+
+  struct ceph_dir_result *dirp;
+  EXPECT_EQ(-ENOTCONN, ceph_opendir(cmount, "/", &dirp));
+  EXPECT_EQ(-ENOTCONN, ceph_closedir(cmount, dirp));
+
+  ceph_readdir(cmount, dirp);
+  EXPECT_EQ(-ENOTCONN, errno);
+
+  struct dirent rdent;
+  EXPECT_EQ(-ENOTCONN, ceph_readdir_r(cmount, dirp, &rdent));
+
+  int stmask;
+  struct stat st;
+  EXPECT_EQ(-ENOTCONN, ceph_readdirplus_r(cmount, dirp, &rdent, &st, &stmask));
+  EXPECT_EQ(-ENOTCONN, ceph_getdents(cmount, dirp, NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_getdnames(cmount, dirp, NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_telldir(cmount, dirp));
+  EXPECT_EQ(-ENOTCONN, ceph_link(cmount, "/", "/link"));
+  EXPECT_EQ(-ENOTCONN, ceph_unlink(cmount, "/path"));
+  EXPECT_EQ(-ENOTCONN, ceph_rename(cmount, "/path", "/path"));
+  EXPECT_EQ(-ENOTCONN, ceph_mkdir(cmount, "/", 0655));
+  EXPECT_EQ(-ENOTCONN, ceph_mkdirs(cmount, "/", 0655));
+  EXPECT_EQ(-ENOTCONN, ceph_rmdir(cmount, "/path"));
+  EXPECT_EQ(-ENOTCONN, ceph_readlink(cmount, "/path", NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_symlink(cmount, "/path", "/path"));
+  EXPECT_EQ(-ENOTCONN, ceph_stat(cmount, "/path", &st));
+  EXPECT_EQ(-ENOTCONN, ceph_lstat(cmount, "/path", &st));
+  EXPECT_EQ(-ENOTCONN, ceph_setattr(cmount, "/path", &st, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_getxattr(cmount, "/path", "name", NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_lgetxattr(cmount, "/path", "name", NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_listxattr(cmount, "/path", NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_llistxattr(cmount, "/path", NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_removexattr(cmount, "/path", "name"));
+  EXPECT_EQ(-ENOTCONN, ceph_lremovexattr(cmount, "/path", "name"));
+  EXPECT_EQ(-ENOTCONN, ceph_setxattr(cmount, "/path", "name", NULL, 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_lsetxattr(cmount, "/path", "name", NULL, 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_chmod(cmount, "/path", 0));
+  EXPECT_EQ(-ENOTCONN, ceph_fchmod(cmount, 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_chown(cmount, "/path", 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_lchown(cmount, "/path", 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_fchown(cmount, 0, 0, 0));
+
+  struct utimbuf utb;
+  EXPECT_EQ(-ENOTCONN, ceph_utime(cmount, "/path", &utb));
+  EXPECT_EQ(-ENOTCONN, ceph_truncate(cmount, "/path", 0));
+  EXPECT_EQ(-ENOTCONN, ceph_mknod(cmount, "/path", 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_open(cmount, "/path", 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_open_layout(cmount, "/path", 0, 0, 0, 0, 0, "pool"));
+  EXPECT_EQ(-ENOTCONN, ceph_close(cmount, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_lseek(cmount, 0, 0, SEEK_SET));
+  EXPECT_EQ(-ENOTCONN, ceph_read(cmount, 0, NULL, 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_write(cmount, 0, NULL, 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_ftruncate(cmount, 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_fsync(cmount, 0, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_fstat(cmount, 0, &st));
+  EXPECT_EQ(-ENOTCONN, ceph_sync_fs(cmount));
+  EXPECT_EQ(-ENOTCONN, ceph_get_file_stripe_unit(cmount, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_get_file_pool(cmount, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_get_file_pool_name(cmount, 0, NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_get_file_replication(cmount, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_get_file_stripe_address(cmount, 0, 0, NULL, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_localize_reads(cmount, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_debug_get_fd_caps(cmount, 0));
+  EXPECT_EQ(-ENOTCONN, ceph_debug_get_file_caps(cmount, "/path"));
+  EXPECT_EQ(-ENOTCONN, ceph_get_stripe_unit_granularity(cmount));
+  EXPECT_EQ(-ENOTCONN, ceph_get_pool_id(cmount, "data"));
+  EXPECT_EQ(-ENOTCONN, ceph_get_pool_replication(cmount, 1));
+
+  ceph_release(cmount);
+}
+
+TEST(LibCephFS, GetPoolId) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_GE(ceph_get_pool_id(cmount, "data"), 0);
+  ASSERT_GE(ceph_get_pool_id(cmount, "metadata"), 0);
+  ASSERT_EQ(ceph_get_pool_id(cmount, "weflkjwelfjwlkejf"), -ENOENT);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetPoolReplication) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  /* negative pools */
+  ASSERT_EQ(ceph_get_pool_replication(cmount, -10), -ENOENT);
+
+  /* valid pool */
+  int pool_id = ceph_get_pool_id(cmount, "data");
+  ASSERT_GE(pool_id, 0);
+  ASSERT_GT(ceph_get_pool_replication(cmount, pool_id), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetExtentOsds) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+
+  EXPECT_EQ(-ENOTCONN, ceph_get_file_extent_osds(cmount, 0, 0, NULL, NULL, 0));
+
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  int stripe_unit = (1<<18);
+
+  /* make a file! */
+  char test_file[256];
+  sprintf(test_file, "test_extent_osds_%d", getpid());
+  int fd = ceph_open_layout(cmount, test_file, O_CREAT|O_RDWR, 0666,
+      stripe_unit, 2, stripe_unit*2, NULL);
+  ASSERT_GT(fd, 0);
+
+  /* get back how many osds > 0 */
+  int ret = ceph_get_file_extent_osds(cmount, fd, 0, NULL, NULL, 0);
+  EXPECT_GT(ret, 0);
+
+  int64_t len;
+  int osds[ret];
+
+  /* full stripe extent */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 0, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit);
+
+  /* half stripe extent */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, stripe_unit/2, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit/2);
+
+  /* 1.5 stripe unit offset -1 byte */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 3*stripe_unit/2-1, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit/2+1);
+
+  /* 1.5 stripe unit offset +1 byte */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 3*stripe_unit/2+1, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit/2-1);
+
+  /* only when more than 1 osd */
+  if (ret > 1)
+    EXPECT_EQ(-ERANGE, ceph_get_file_extent_osds(cmount, fd, 0, NULL, osds, 1));
+
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetOsdCrushLocation) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+
+  EXPECT_EQ(-ENOTCONN, ceph_get_osd_crush_location(cmount, 0, NULL, 0));
+
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, 0, NULL, 1), -EINVAL);
+
+  char path[256];
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, 9999999, path, 0), -ENOENT);
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, -1, path, 0), -EINVAL);
+
+  char test_file[256];
+  sprintf(test_file, "test_osds_loc_%d", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  /* get back how many osds > 0 */
+  int ret = ceph_get_file_extent_osds(cmount, fd, 0, NULL, NULL, 0);
+  EXPECT_GT(ret, 0);
+
+  /* full stripe extent */
+  int osds[ret];
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 0, NULL, osds, ret));
+
+  ASSERT_GT(ceph_get_osd_crush_location(cmount, 0, path, 0), 0);
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, 0, path, 1), -ERANGE);
+
+  for (int i = 0; i < ret; i++) {
+    int len = ceph_get_osd_crush_location(cmount, osds[i], path, sizeof(path));
+    ASSERT_GT(len, 0);
+    int pos = 0;
+    while (pos < len) {
+      std::string type(path + pos);
+      ASSERT_GT((int)type.size(), 0);
+      pos += type.size() + 1;
+
+      std::string name(path + pos);
+      ASSERT_GT((int)name.size(), 0);
+      pos += name.size() + 1;
+    }
+  }
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetOsdAddr) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+
+  EXPECT_EQ(-ENOTCONN, ceph_get_osd_addr(cmount, 0, NULL));
+
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_EQ(-EINVAL, ceph_get_osd_addr(cmount, 0, NULL));
+
+  struct sockaddr_storage addr;
+  ASSERT_EQ(-ENOENT, ceph_get_osd_addr(cmount, -1, &addr));
+  ASSERT_EQ(-ENOENT, ceph_get_osd_addr(cmount, 9999999, &addr));
+
+  ASSERT_EQ(0, ceph_get_osd_addr(cmount, 0, &addr));
+
+  ceph_shutdown(cmount);
+}
diff --git a/ceph/src/test/libcephfs_config.cc b/ceph/src/test/libcephfs_config.cc
new file mode 100644
index 00000000..0d908b8f
--- /dev/null
+++ b/ceph/src/test/libcephfs_config.cc
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+
+#include 
+#include 
+#include 
+
+using std::string;
+
+TEST(LibCephConfig, SimpleSet) {
+  struct ceph_mount_info *cmount;
+  int ret = ceph_create(&cmount, NULL);
+  ASSERT_EQ(ret, 0);
+
+  ret = ceph_conf_set(cmount, "max_open_files", "21");
+  ASSERT_EQ(ret, 0);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = ceph_conf_get(cmount, "max_open_files", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("21"), string(buf));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephConfig, ArgV) {
+  struct ceph_mount_info *cmount;
+  int ret = ceph_create(&cmount, NULL);
+  ASSERT_EQ(ret, 0);
+
+  const char *argv[] = { "foo", "--max-open-files", "2",
+			 "--keyfile", "/tmp/my-keyfile", NULL };
+  size_t argc = (sizeof(argv) / sizeof(argv[0])) - 1;
+  ceph_conf_parse_argv(cmount, argc, argv);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = ceph_conf_get(cmount, "keyfile", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("/tmp/my-keyfile"), string(buf));
+
+  memset(buf, 0, sizeof(buf));
+  ret = ceph_conf_get(cmount, "max_open_files", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("2"), string(buf));
+
+  ceph_shutdown(cmount);
+}
diff --git a/ceph/src/test/librados/TestCase.cc b/ceph/src/test/librados/TestCase.cc
new file mode 100644
index 00000000..7f072fd8
--- /dev/null
+++ b/ceph/src/test/librados/TestCase.cc
@@ -0,0 +1,239 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include 
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+using namespace librados;
+
+std::string RadosTest::pool_name;
+std::string RadosTest::nspace;
+rados_t RadosTest::s_cluster = NULL;
+
+void RadosTest::SetUpTestCase()
+{
+  pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTest::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTest::SetUp()
+{
+  cluster = RadosTest::s_cluster;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  nspace = get_temp_pool_name();
+  rados_ioctx_set_namespace(ioctx, nspace.c_str());
+  ASSERT_FALSE(rados_ioctx_pool_requires_alignment(ioctx));
+}
+
+void RadosTest::TearDown()
+{
+  cleanup_default_namespace(ioctx);
+  rados_ioctx_destroy(ioctx);
+}
+
+void RadosTest::cleanup_default_namespace(rados_ioctx_t ioctx)
+{
+  // remove all objects from the default namespace to avoid polluting
+  // other tests
+  rados_ioctx_set_namespace(ioctx, "");
+  rados_list_ctx_t list_ctx;
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &list_ctx));
+  int r;
+  const char *entry = NULL;
+  const char *key = NULL;
+  while ((r = rados_objects_list_next(list_ctx, &entry, &key)) != -ENOENT) {
+    ASSERT_EQ(0, r);
+    rados_ioctx_locator_set_key(ioctx, key);
+    ASSERT_EQ(0, rados_remove(ioctx, entry));
+  }
+  rados_objects_list_close(list_ctx);
+}
+
+std::string RadosTestPP::pool_name;
+Rados RadosTestPP::s_cluster;
+
+void RadosTestPP::SetUpTestCase()
+{
+  pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPP::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPP::SetUp()
+{
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  ns = get_temp_pool_name();
+  ioctx.set_namespace(ns);
+  ASSERT_FALSE(ioctx.pool_requires_alignment());
+}
+
+void RadosTestPP::TearDown()
+{
+  cleanup_default_namespace(ioctx);
+  ioctx.close();
+}
+
+void RadosTestPP::cleanup_default_namespace(librados::IoCtx ioctx)
+{
+  // remove all objects from the default namespace to avoid polluting
+  // other tests
+  ioctx.set_namespace("");
+  for (ObjectIterator it = ioctx.objects_begin();
+       it != ioctx.objects_end(); ++it) {
+    ioctx.locator_set_key(it->second);
+    ASSERT_EQ(0, ioctx.remove(it->first));
+  }
+}
+
+std::string RadosTestParamPP::pool_name;
+std::string RadosTestParamPP::cache_pool_name;
+Rados RadosTestParamPP::s_cluster;
+
+void RadosTestParamPP::SetUpTestCase()
+{
+  pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestParamPP::TearDownTestCase()
+{
+  if (cache_pool_name.length()) {
+    // tear down tiers
+    bufferlist inbl;
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+      "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd pool delete\", \"pool\": \"" + cache_pool_name +
+      "\", \"pool2\": \"" + cache_pool_name + "\", \"sure\": \"--yes-i-really-really-mean-it\"}",
+      inbl, NULL, NULL));
+    cache_pool_name = "";
+  }
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestParamPP::SetUp()
+{
+  if (strcmp(GetParam(), "cache") == 0 && cache_pool_name.empty()) {
+    cache_pool_name = get_temp_pool_name();
+    bufferlist inbl;
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd pool create\", \"pool\": \"" + cache_pool_name +
+      "\", \"pg_num\": 4}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name +
+      "\", \"force_nonempty\": \"--force-nonempty\" }",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+      "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+      "\", \"mode\": \"writeback\"}",
+      inbl, NULL, NULL));
+    cluster.wait_for_latest_osdmap();
+  }
+
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  ns = get_temp_pool_name();
+  ioctx.set_namespace(ns);
+  ASSERT_FALSE(ioctx.pool_requires_alignment());
+}
+
+void RadosTestParamPP::TearDown()
+{
+  cleanup_default_namespace(ioctx);
+  ioctx.close();
+}
+
+void RadosTestParamPP::cleanup_default_namespace(librados::IoCtx ioctx)
+{
+  // remove all objects from the default namespace to avoid polluting
+  // other tests
+  ioctx.set_namespace("");
+  for (ObjectIterator it = ioctx.objects_begin();
+       it != ioctx.objects_end(); ++it) {
+    ioctx.locator_set_key(it->second);
+    ASSERT_EQ(0, ioctx.remove(it->first));
+  }
+}
+
+std::string RadosTestEC::pool_name;
+rados_t RadosTestEC::s_cluster = NULL;
+
+void RadosTestEC::SetUpTestCase()
+{
+  pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_ec_pool(pool_name, &s_cluster));
+}
+
+void RadosTestEC::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_ec_pool(pool_name, &s_cluster));
+}
+
+void RadosTestEC::SetUp()
+{
+  cluster = RadosTestEC::s_cluster;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  std::string nspace = get_temp_pool_name();
+  rados_ioctx_set_namespace(ioctx, nspace.c_str());
+  ASSERT_TRUE(rados_ioctx_pool_requires_alignment(ioctx));
+  alignment = rados_ioctx_pool_required_alignment(ioctx);
+  ASSERT_NE((unsigned)0, alignment);
+}
+
+void RadosTestEC::TearDown()
+{
+  cleanup_default_namespace(ioctx);
+  rados_ioctx_destroy(ioctx);
+}
+
+std::string RadosTestECPP::pool_name;
+Rados RadosTestECPP::s_cluster;
+
+void RadosTestECPP::SetUpTestCase()
+{
+  pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestECPP::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestECPP::SetUp()
+{
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  ns = get_temp_pool_name();
+  ioctx.set_namespace(ns);
+  ASSERT_TRUE(ioctx.pool_requires_alignment());
+  alignment = ioctx.pool_required_alignment();
+  ASSERT_NE((unsigned)0, alignment);
+}
+
+void RadosTestECPP::TearDown()
+{
+  cleanup_default_namespace(ioctx);
+  ioctx.close();
+}
+
diff --git a/ceph/src/test/librados/TestCase.h b/ceph/src/test/librados/TestCase.h
new file mode 100644
index 00000000..4ede5e97
--- /dev/null
+++ b/ceph/src/test/librados/TestCase.h
@@ -0,0 +1,111 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_RADOS_TESTCASE_H
+#define CEPH_TEST_RADOS_TESTCASE_H
+
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "gtest/gtest.h"
+
+#include 
+
+/**
+ * These test cases create a temporary pool that lives as long as the
+ * test case.  Each test within a test case gets a new ioctx set to a
+ * unique namespace within the pool.
+ *
+ * Since pool creation and deletion is slow, this allows many tests to
+ * run faster.
+ */
+class RadosTest : public ::testing::Test {
+public:
+  RadosTest() {}
+  virtual ~RadosTest() {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static void cleanup_default_namespace(rados_ioctx_t ioctx);
+  static rados_t s_cluster;
+  static std::string pool_name;
+  static std::string nspace;
+
+  virtual void SetUp();
+  virtual void TearDown();
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+};
+
+class RadosTestPP : public ::testing::Test {
+public:
+  RadosTestPP() : cluster(s_cluster) {}
+  virtual ~RadosTestPP() {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static void cleanup_default_namespace(librados::IoCtx ioctx);
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+
+  virtual void SetUp();
+  virtual void TearDown();
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  std::string ns;
+};
+
+class RadosTestParamPP : public ::testing::TestWithParam {
+public:
+  RadosTestParamPP() : cluster(s_cluster) {}
+  virtual ~RadosTestParamPP() {}
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+protected:
+  static void cleanup_default_namespace(librados::IoCtx ioctx);
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+  static std::string cache_pool_name;
+
+  virtual void SetUp();
+  virtual void TearDown();
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  std::string ns;
+};
+
+class RadosTestEC : public RadosTest {
+public:
+  RadosTestEC() {}
+  virtual ~RadosTestEC() {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static rados_t s_cluster;
+  static std::string pool_name;
+
+  virtual void SetUp();
+  virtual void TearDown();
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  uint64_t alignment;
+};
+
+class RadosTestECPP : public RadosTestPP {
+public:
+  RadosTestECPP() : cluster(s_cluster) {};
+  virtual ~RadosTestECPP() {};
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+
+  virtual void SetUp();
+  virtual void TearDown();
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  std::string ns;
+  uint64_t alignment;
+};
+
+#endif
diff --git a/ceph/src/test/librados/aio.cc b/ceph/src/test/librados/aio.cc
new file mode 100644
index 00000000..218da3bc
--- /dev/null
+++ b/ceph/src/test/librados/aio.cc
@@ -0,0 +1,2865 @@
+#include "common/errno.h"
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+#include "include/types.h"
+
+#include "gtest/gtest.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using std::ostringstream;
+using namespace librados;
+using std::pair;
+
+class AioTestData
+{
+public:
+  AioTestData()
+    : m_cluster(NULL),
+      m_ioctx(NULL),
+      m_init(false),
+      m_complete(false),
+      m_safe(false)
+  {
+  }
+
+  ~AioTestData()
+  {
+    if (m_init) {
+      rados_ioctx_destroy(m_ioctx);
+      destroy_one_pool(m_pool_name, &m_cluster);
+      sem_destroy(&m_sem);
+    }
+  }
+
+  std::string init()
+  {
+    int ret;
+    if (sem_init(&m_sem, 0, 0)) {
+      int err = errno;
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "sem_init failed: " << cpp_strerror(err);
+      return oss.str();
+    }
+    m_pool_name = get_temp_pool_name();
+    std::string err = create_one_pool(m_pool_name, &m_cluster);
+    if (!err.empty()) {
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "create_one_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = rados_ioctx_create(m_cluster, m_pool_name.c_str(), &m_ioctx);
+    if (ret) {
+      sem_destroy(&m_sem);
+      destroy_one_pool(m_pool_name, &m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_init = true;
+    return "";
+  }
+
+  sem_t m_sem;
+  rados_t m_cluster;
+  rados_ioctx_t m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+  bool m_complete;
+  bool m_safe;
+};
+
+class AioTestDataPP
+{
+public:
+  AioTestDataPP()
+    : m_init(false),
+      m_complete(false),
+      m_safe(false)
+  {
+  }
+
+  ~AioTestDataPP()
+  {
+    if (m_init) {
+      m_ioctx.close();
+      destroy_one_pool_pp(m_pool_name, m_cluster);
+      sem_destroy(&m_sem);
+    }
+  }
+
+  std::string init()
+  {
+    int ret;
+    if (sem_init(&m_sem, 0, 0)) {
+      int err = errno;
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "sem_init failed: " << cpp_strerror(err);
+      return oss.str();
+    }
+    m_pool_name = get_temp_pool_name();
+    std::string err = create_one_pool_pp(m_pool_name, m_cluster);
+    if (!err.empty()) {
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "create_one_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = m_cluster.ioctx_create(m_pool_name.c_str(), m_ioctx);
+    if (ret) {
+      sem_destroy(&m_sem);
+      destroy_one_pool_pp(m_pool_name, m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_init = true;
+    return "";
+  }
+
+  sem_t m_sem;
+  Rados m_cluster;
+  IoCtx m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+  bool m_complete;
+  bool m_safe;
+};
+
+void set_completion_complete(rados_completion_t cb, void *arg)
+{
+  AioTestData *test = static_cast(arg);
+  test->m_complete = true;
+  sem_post(&test->m_sem);
+}
+
+void set_completion_safe(rados_completion_t cb, void *arg)
+{
+  AioTestData *test = static_cast(arg);
+  test->m_safe = true;
+  sem_post(&test->m_sem);
+}
+
+void set_completion_completePP(rados_completion_t cb, void *arg)
+{
+  AioTestDataPP *test = static_cast(arg);
+  test->m_complete = true;
+  sem_post(&test->m_sem);
+}
+
+void set_completion_safePP(rados_completion_t cb, void *arg)
+{
+  AioTestDataPP *test = static_cast(arg);
+  test->m_safe = true;
+  sem_post(&test->m_sem);
+}
+
+TEST(LibRadosAio, SimpleWrite) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAio, SimpleWritePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completePP, set_completion_safePP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo",
+			       my_completion, bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  delete my_completion;
+  }
+
+  {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  test_data.m_ioctx.set_namespace("nspace");
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completePP, set_completion_safePP);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo",
+			       my_completion, bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  delete my_completion;
+  }
+}
+
+TEST(LibRadosAio, WaitForSafe) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, rados_aio_wait_for_safe(my_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAio, WaitForSafePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completePP, set_completion_safePP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo",
+			       my_completion, bl1, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, my_completion->wait_for_safe());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  delete my_completion;
+}
+
+TEST(LibRadosAio, RoundTrip) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[256];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, RoundTrip2) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, RoundTripPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completePP, set_completion_safePP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completePP, set_completion_safePP);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo",
+			      my_completion2, &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAio, RoundTripPP2) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completePP, set_completion_safePP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completePP, set_completion_safePP);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo",
+			      my_completion2, &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_safe());
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAio, RoundTripAppend) {
+  AioTestData test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  char buf3[sizeof(buf) + sizeof(buf2)];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf3), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(buf3 + sizeof(buf), buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, RoundTripAppendPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append("foo", my_completion,
+					    bl1, sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  char buf2[128];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append("foo", my_completion2,
+					    bl2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  bufferlist bl3;
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo",
+			      my_completion3, &bl3, 2 * sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)(sizeof(buf) * 2), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf) * 2, bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(bl3.c_str() + sizeof(buf), buf2, sizeof(buf2)));
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+}
+
+TEST(LibRadosAio, IsComplete) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_complete.
+    while (true) {
+      int is_complete = rados_aio_is_complete(my_completion2);
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, IsCompletePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test is_complete.
+    while (true) {
+      int is_complete = my_completion2->is_complete();
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAio, IsSafe) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_safe = rados_aio_is_safe(my_completion);
+      if (is_safe)
+	break;
+    }
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, IsSafePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_safe = my_completion->is_safe();
+      if (is_safe)
+	break;
+    }
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  bufferlist bl2;
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAio, ReturnValue) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "nonexistent",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAio, ReturnValuePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  bufferlist bl1;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("nonexistent",
+			       my_completion, &bl1, 128, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion->get_return_value());
+  delete my_completion;
+}
+
+TEST(LibRadosAio, Flush) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush(test_data.m_ioctx));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, FlushPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAio, FlushAsync) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  rados_completion_t flush_completion;
+  ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &flush_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush_async(test_data.m_ioctx, flush_completion));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(flush_completion));
+    ASSERT_EQ(0, rados_aio_wait_for_safe(flush_completion));
+  }
+  ASSERT_EQ(1, rados_aio_is_complete(my_completion));
+  ASSERT_EQ(1, rados_aio_is_safe(my_completion));
+  ASSERT_EQ(1, rados_aio_is_complete(flush_completion));
+  ASSERT_EQ(1, rados_aio_is_safe(flush_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(flush_completion);
+}
+
+TEST(LibRadosAio, FlushAsyncPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *flush_completion =
+      test_data.m_cluster.aio_create_completion(NULL, NULL, NULL);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush_async(flush_completion));
+  {
+      TestAlarm alarm;
+      ASSERT_EQ(0, flush_completion->wait_for_complete());
+      ASSERT_EQ(0, flush_completion->wait_for_safe());
+  }
+  ASSERT_EQ(1, my_completion->is_complete());
+  ASSERT_EQ(1, my_completion->is_safe());
+  ASSERT_EQ(1, flush_completion->is_complete());
+  ASSERT_EQ(1, flush_completion->is_safe());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+  delete flush_completion;
+}
+
+TEST(LibRadosAio, RoundTripWriteFull) {
+  AioTestData test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write_full(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  char buf3[sizeof(buf) + sizeof(buf2)];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, RoundTripWriteFullPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write_full("foo", my_completion2, bl2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  bufferlist bl3;
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion3,
+					  &bl3, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf2), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf2), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+}
+
+
+TEST(LibRadosAio, SimpleStat) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, SimpleStatPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion2,
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAio, SimpleStatNS) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  rados_ioctx_set_namespace(test_data.m_ioctx, "");
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion3));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion3, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(sizeof(buf2), psize);
+
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, SimpleStatPPNS) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion2,
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAio, StatRemove) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion3));
+  ASSERT_EQ(0, rados_aio_remove(test_data.m_ioctx, "foo", my_completion3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  uint64_t psize2;
+  time_t pmtime2;
+  rados_completion_t my_completion4;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion4));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion4, &psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion4));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion4));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+  rados_aio_release(my_completion4);
+}
+
+TEST(LibRadosAio, StatRemovePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion2,
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  uint64_t psize2;
+  time_t pmtime2;
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_remove("foo", my_completion3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion3->get_return_value());
+
+  AioCompletion *my_completion4 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion4, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion4,
+			  		&psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion4->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion4->get_return_value());
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+  delete my_completion4;
+}
+
+using std::string;
+using std::map;
+using std::set;
+
+TEST(LibRadosAio, OmapPP) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  string header_str = "baz";
+  bufferptr bp(header_str.c_str(), header_str.size() + 1);
+  bufferlist header_to_set;
+  header_to_set.push_back(bp);
+  map to_set;
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectWriteOperation op;
+    to_set["foo"] = header_to_set;
+    to_set["foo2"] = header_to_set;
+    to_set["qfoo3"] = header_to_set;
+    op.omap_set(to_set);
+
+    op.omap_set_header(header_to_set);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectReadOperation op;
+    map > assertions;
+    bufferlist val;
+    val.append(string("bar"));
+    assertions["foo"] = pair(val, CEPH_OSD_CMPXATTR_OP_EQ);
+
+    int r;
+    op.omap_cmp(assertions, &r);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(-ECANCELED, my_completion->get_return_value());
+    ASSERT_EQ(-ECANCELED, r);
+  }
+
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectReadOperation op;
+
+    set set_got;
+    map map_got;
+
+    set to_get;
+    map got3;
+
+    map got4;
+
+    bufferlist header;
+
+    op.omap_get_keys("", 1, &set_got, 0);
+    op.omap_get_vals("foo", 1, &map_got, 0);
+
+    to_get.insert("foo");
+    to_get.insert("qfoo3");
+    op.omap_get_vals_by_keys(to_get, &got3, 0);
+
+    op.omap_get_header(&header, 0);
+
+    op.omap_get_vals("foo2", "q", 1, &got4, 0);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+
+    ASSERT_EQ(header.length(), header_to_set.length());
+    ASSERT_EQ(set_got.size(), (unsigned)1);
+    ASSERT_EQ(*set_got.begin(), "foo");
+    ASSERT_EQ(map_got.size(), (unsigned)1);
+    ASSERT_EQ(map_got.begin()->first, "foo2");
+    ASSERT_EQ(got3.size(), (unsigned)2);
+    ASSERT_EQ(got3.begin()->first, "foo");
+    ASSERT_EQ(got3.rbegin()->first, "qfoo3");
+    ASSERT_EQ(got4.size(), (unsigned)1);
+    ASSERT_EQ(got4.begin()->first, "qfoo3");
+  }
+
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectWriteOperation op;
+    set to_remove;
+    to_remove.insert("foo2");
+    op.omap_rm_keys(to_remove);
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectReadOperation op;
+
+    set set_got;
+    op.omap_get_keys("", -1, &set_got, 0);
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+    ASSERT_EQ(set_got.size(), (unsigned)2);
+  }
+
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectWriteOperation op;
+    op.omap_clear();
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectReadOperation op;
+
+    set set_got;
+    op.omap_get_keys("", -1, &set_got, 0);
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+    ASSERT_EQ(set_got.size(), (unsigned)0);
+  }
+
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAio, MultiWrite) {
+  AioTestData test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+
+  char buf3[(sizeof(buf) + sizeof(buf2)) * 3];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_complete, set_completion_safe, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)(sizeof(buf) + sizeof(buf2)), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(buf3 + sizeof(buf), buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, MultiWritePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion2,
+					   bl2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+
+  bufferlist bl3;
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_complete, set_completion_safe);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion3,
+					  &bl3, (sizeof(buf) + sizeof(buf2) * 3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)(sizeof(buf) + sizeof(buf2)), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf) + sizeof(buf2), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(bl3.c_str() + sizeof(buf), buf2, sizeof(buf2)));
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+}
+
+// EC test cases
+class AioTestDataEC
+{
+public:
+  AioTestDataEC()
+    : m_cluster(NULL),
+      m_ioctx(NULL),
+      m_init(false),
+      m_complete(false),
+      m_safe(false)
+  {
+  }
+
+  ~AioTestDataEC()
+  {
+    if (m_init) {
+      rados_ioctx_destroy(m_ioctx);
+      destroy_one_ec_pool(m_pool_name, &m_cluster);
+      sem_destroy(&m_sem);
+    }
+  }
+
+  std::string init()
+  {
+    int ret;
+    if (sem_init(&m_sem, 0, 0)) {
+      int err = errno;
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "sem_init failed: " << cpp_strerror(err);
+      return oss.str();
+    }
+    m_pool_name = get_temp_pool_name();
+    std::string err = create_one_ec_pool(m_pool_name, &m_cluster);
+    if (!err.empty()) {
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "create_one_ec_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = rados_ioctx_create(m_cluster, m_pool_name.c_str(), &m_ioctx);
+    if (ret) {
+      sem_destroy(&m_sem);
+      destroy_one_ec_pool(m_pool_name, &m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_init = true;
+    return "";
+  }
+
+  sem_t m_sem;
+  rados_t m_cluster;
+  rados_ioctx_t m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+  bool m_complete;
+  bool m_safe;
+};
+
+class AioTestDataECPP
+{
+public:
+  AioTestDataECPP()
+    : m_init(false),
+      m_complete(false),
+      m_safe(false)
+  {
+  }
+
+  ~AioTestDataECPP()
+  {
+    if (m_init) {
+      m_ioctx.close();
+      destroy_one_ec_pool_pp(m_pool_name, m_cluster);
+      sem_destroy(&m_sem);
+    }
+  }
+
+  std::string init()
+  {
+    int ret;
+    if (sem_init(&m_sem, 0, 0)) {
+      int err = errno;
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "sem_init failed: " << cpp_strerror(err);
+      return oss.str();
+    }
+    m_pool_name = get_temp_pool_name();
+    std::string err = create_one_ec_pool_pp(m_pool_name, m_cluster);
+    if (!err.empty()) {
+      sem_destroy(&m_sem);
+      ostringstream oss;
+      oss << "create_one_ec_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = m_cluster.ioctx_create(m_pool_name.c_str(), m_ioctx);
+    if (ret) {
+      sem_destroy(&m_sem);
+      destroy_one_ec_pool_pp(m_pool_name, m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_init = true;
+    return "";
+  }
+
+  sem_t m_sem;
+  Rados m_cluster;
+  IoCtx m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+  bool m_complete;
+  bool m_safe;
+};
+
+void set_completion_completeEC(rados_completion_t cb, void *arg)
+{
+  AioTestDataEC *test = static_cast(arg);
+  test->m_complete = true;
+  sem_post(&test->m_sem);
+}
+
+void set_completion_safeEC(rados_completion_t cb, void *arg)
+{
+  AioTestDataEC *test = static_cast(arg);
+  test->m_safe = true;
+  sem_post(&test->m_sem);
+}
+
+void set_completion_completeECPP(rados_completion_t cb, void *arg)
+{
+  AioTestDataECPP *test = static_cast(arg);
+  test->m_complete = true;
+  sem_post(&test->m_sem);
+}
+
+void set_completion_safeECPP(rados_completion_t cb, void *arg)
+{
+  AioTestDataECPP *test = static_cast(arg);
+  test->m_safe = true;
+  sem_post(&test->m_sem);
+}
+
+TEST(LibRadosAioEC, SimpleWrite) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAioEC, SimpleWritePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeECPP, set_completion_safeECPP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo",
+			       my_completion, bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  delete my_completion;
+  }
+
+  {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  test_data.m_ioctx.set_namespace("nspace");
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeECPP, set_completion_safeECPP);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo",
+			       my_completion, bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  delete my_completion;
+  }
+}
+
+TEST(LibRadosAioEC, WaitForSafe) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, rados_aio_wait_for_safe(my_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAioEC, WaitForSafePP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeECPP, set_completion_safeECPP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo",
+			       my_completion, bl1, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, my_completion->wait_for_safe());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  delete my_completion;
+}
+
+TEST(LibRadosAioEC, RoundTrip) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[256];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, RoundTrip2) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, RoundTripPP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeECPP, set_completion_safeECPP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeECPP, set_completion_safeECPP);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo",
+			      my_completion2, &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAioEC, RoundTripPP2) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeECPP, set_completion_safeECPP);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeECPP, set_completion_safeECPP);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo",
+			      my_completion2, &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_safe());
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAioEC, RoundTripAppend) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3, my_completion4;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  ASSERT_TRUE(rados_ioctx_pool_requires_alignment(test_data.m_ioctx));
+  uint64_t alignment = rados_ioctx_pool_required_alignment(test_data.m_ioctx);
+  ASSERT_NE((unsigned)0, alignment);
+
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion, buf, bsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  int hbsize = bsize / 2;
+  char *buf2 = (char *)new char[hbsize];
+  memset(buf2, 0xdd, hbsize);
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion3));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion3, buf2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  EXPECT_EQ(-EOPNOTSUPP, rados_aio_get_return_value(my_completion3));
+
+  int tbsize = bsize + hbsize;
+  char *buf3 = (char *)new char[tbsize];
+  memset(buf3, 0, tbsize);
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion4));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion4, buf3, bsize * 3, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion4));
+  }
+  ASSERT_EQ(tbsize, rados_aio_get_return_value(my_completion4));
+  ASSERT_EQ(0, memcmp(buf3, buf, bsize));
+  ASSERT_EQ(0, memcmp(buf3 + bsize, buf2, hbsize));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+}
+
+TEST(LibRadosAioEC, RoundTripAppendPP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  ASSERT_TRUE(test_data.m_ioctx.pool_requires_alignment());
+  uint64_t alignment = test_data.m_ioctx.pool_required_alignment();
+  ASSERT_NE((unsigned)0, alignment);
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append("foo", my_completion,
+					    bl1, bsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  int hbsize = bsize / 2;
+  char *buf2 = (char *)new char[hbsize];
+  memset(buf2, 0xdd, hbsize);
+  bufferlist bl2;
+  bl2.append(buf2, hbsize);
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append("foo", my_completion2,
+					    bl2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append("foo", my_completion3,
+					    bl2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  EXPECT_EQ(-EOPNOTSUPP, my_completion3->get_return_value());
+
+  bufferlist bl3;
+  AioCompletion *my_completion4 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion4, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo",
+			      my_completion4, &bl3, bsize * 3, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion4->wait_for_complete());
+  }
+  int tbsize = bsize + hbsize;
+  ASSERT_EQ(tbsize, my_completion4->get_return_value());
+  ASSERT_EQ((unsigned)tbsize, bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(0, memcmp(bl3.c_str() + bsize, buf2, hbsize));
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+  delete[] buf;
+  delete[] buf2;
+}
+
+TEST(LibRadosAioEC, IsComplete) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_complete.
+    while (true) {
+      int is_complete = rados_aio_is_complete(my_completion2);
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, IsCompletePP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test is_complete.
+    while (true) {
+      int is_complete = my_completion2->is_complete();
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAioEC, IsSafe) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_safe = rados_aio_is_safe(my_completion);
+      if (is_safe)
+	break;
+    }
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, IsSafePP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_safe = my_completion->is_safe();
+      if (is_safe)
+	break;
+    }
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  bufferlist bl2;
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAioEC, ReturnValue) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "nonexistent",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAioEC, ReturnValuePP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  bufferlist bl1;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("nonexistent",
+			       my_completion, &bl1, 128, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion->get_return_value());
+  delete my_completion;
+}
+
+TEST(LibRadosAioEC, Flush) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush(test_data.m_ioctx));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, FlushPP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAioEC, FlushAsync) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  rados_completion_t flush_completion;
+  ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &flush_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush_async(test_data.m_ioctx, flush_completion));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(flush_completion));
+    ASSERT_EQ(0, rados_aio_wait_for_safe(flush_completion));
+  }
+  ASSERT_EQ(1, rados_aio_is_complete(my_completion));
+  ASSERT_EQ(1, rados_aio_is_safe(my_completion));
+  ASSERT_EQ(1, rados_aio_is_complete(flush_completion));
+  ASSERT_EQ(1, rados_aio_is_safe(flush_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(flush_completion);
+}
+
+TEST(LibRadosAioEC, FlushAsyncPP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *flush_completion =
+      test_data.m_cluster.aio_create_completion(NULL, NULL, NULL);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush_async(flush_completion));
+  {
+      TestAlarm alarm;
+      ASSERT_EQ(0, flush_completion->wait_for_complete());
+      ASSERT_EQ(0, flush_completion->wait_for_safe());
+  }
+  ASSERT_EQ(1, my_completion->is_complete());
+  ASSERT_EQ(1, my_completion->is_safe());
+  ASSERT_EQ(1, flush_completion->is_complete());
+  ASSERT_EQ(1, flush_completion->is_safe());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion2,
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+  delete flush_completion;
+}
+
+TEST(LibRadosAioEC, RoundTripWriteFull) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write_full(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  char buf3[sizeof(buf) + sizeof(buf2)];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAioEC, RoundTripWriteFullPP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write_full("foo", my_completion2, bl2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  bufferlist bl3;
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion3,
+					  &bl3, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf2), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf2), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+}
+
+
+TEST(LibRadosAioEC, SimpleStat) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, SimpleStatPP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion2,
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAioEC, SimpleStatNS) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  rados_ioctx_set_namespace(test_data.m_ioctx, "");
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion3));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion3, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(sizeof(buf2), psize);
+
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAioEC, SimpleStatPPNS) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion2,
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  delete my_completion;
+  delete my_completion2;
+}
+
+TEST(LibRadosAioEC, StatRemove) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion3));
+  ASSERT_EQ(0, rados_aio_remove(test_data.m_ioctx, "foo", my_completion3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  uint64_t psize2;
+  time_t pmtime2;
+  rados_completion_t my_completion4;
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion4));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion4, &psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion4));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion4));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+  rados_aio_release(my_completion4);
+}
+
+TEST(LibRadosAioEC, StatRemovePP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    sem_wait(&test_data.m_sem);
+    sem_wait(&test_data.m_sem);
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion2,
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  uint64_t psize2;
+  time_t pmtime2;
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_remove("foo", my_completion3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion3->get_return_value());
+
+  AioCompletion *my_completion4 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion4, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat("foo", my_completion4,
+			  		&psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion4->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion4->get_return_value());
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+  delete my_completion4;
+}
+
+TEST(LibRadosAioEC, OmapPP) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_ec_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  string header_str = "baz";
+  bufferptr bp(header_str.c_str(), header_str.size() + 1);
+  bufferlist header_to_set;
+  header_to_set.push_back(bp);
+  map to_set;
+  {
+    boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0));
+    ObjectWriteOperation op;
+    to_set["foo"] = header_to_set;
+    to_set["foo2"] = header_to_set;
+    to_set["qfoo3"] = header_to_set;
+    op.omap_set(to_set);
+
+    op.omap_set_header(header_to_set);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(-EOPNOTSUPP, my_completion->get_return_value());
+  }
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAioEC, MultiWrite) {
+  AioTestDataEC test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(-EOPNOTSUPP, rados_aio_get_return_value(my_completion2));
+
+  char buf3[(sizeof(buf) + sizeof(buf2)) * 3];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+	      set_completion_completeEC, set_completion_safeEC, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAioEC, MultiWritePP) {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  AioCompletion *my_completion = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion,
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  AioCompletion *my_completion2 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion2, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion2,
+					   bl2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(-EOPNOTSUPP, my_completion2->get_return_value());
+
+  bufferlist bl3;
+  AioCompletion *my_completion3 = test_data.m_cluster.aio_create_completion(
+	  (void*)&test_data, set_completion_completeEC, set_completion_safeEC);
+  ASSERT_NE(my_completion3, my_completion_null);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", my_completion3,
+					  &bl3, (sizeof(buf) + sizeof(buf2) * 3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  delete my_completion;
+  delete my_completion2;
+  delete my_completion3;
+}
diff --git a/ceph/src/test/librados/c_read_operations.cc b/ceph/src/test/librados/c_read_operations.cc
new file mode 100644
index 00000000..3ca31f4c
--- /dev/null
+++ b/ceph/src/test/librados/c_read_operations.cc
@@ -0,0 +1,539 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// Tests for the C API coverage of atomic read operations
+
+#include 
+#include 
+
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+const char *data = "testdata";
+const char *obj = "testobj";
+const int len = strlen(data);
+
+class CReadOpsTest : public RadosTest {
+protected:
+  void write_object() {
+    // Create an object and write to it
+    ASSERT_EQ(0, rados_write(ioctx, obj, data, len, 0));
+  }
+  void remove_object() {
+    ASSERT_EQ(0, rados_remove(ioctx, obj));
+  }
+  int cmp_xattr(const char *xattr, const char *value, size_t value_len,
+		uint8_t cmp_op)
+  {
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_cmpxattr(op, xattr, cmp_op, value, value_len);
+    int r = rados_read_op_operate(op, ioctx, obj, 0);
+    rados_release_read_op(op);
+    return r;
+  }
+
+  void fetch_and_verify_omap_vals(char const* const* keys,
+				  char const* const* vals,
+				  const size_t *lens,
+				  size_t len)
+  {
+    rados_omap_iter_t iter_vals, iter_keys, iter_vals_by_key;
+    int r_vals, r_keys, r_vals_by_key;
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_omap_get_vals(op, NULL, NULL, 100, &iter_vals, &r_vals);
+    rados_read_op_omap_get_keys(op, NULL, 100, &iter_keys, &r_keys);
+    rados_read_op_omap_get_vals_by_keys(op, keys, len,
+					&iter_vals_by_key, &r_vals_by_key);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    rados_release_read_op(op);
+    ASSERT_EQ(0, r_vals);
+    ASSERT_EQ(0, r_keys);
+    ASSERT_EQ(0, r_vals_by_key);
+
+    const char *zeros[len];
+    size_t zero_lens[len];
+    memset(zeros, 0, len);
+    memset(zero_lens, 0, len * sizeof(size_t));
+    compare_omap_vals(keys, vals, lens, len, iter_vals);
+    compare_omap_vals(keys, zeros, zero_lens, len, iter_keys);
+    compare_omap_vals(keys, vals, lens, len, iter_vals_by_key);
+  }
+
+  void compare_omap_vals(char const* const* keys,
+			 char const* const* vals,
+			 const size_t *lens,
+			 size_t len,
+			 rados_omap_iter_t iter)
+  {
+    size_t i = 0;
+    char *key = NULL;
+    char *val = NULL;
+    size_t val_len = 0;
+    while (i < len) {
+      ASSERT_EQ(0, rados_omap_get_next(iter, &key, &val, &val_len));
+      if (len == 0 && key == NULL && val == NULL)
+	break;
+      if (key)
+	EXPECT_EQ(std::string(keys[i]), std::string(key));
+      else
+	EXPECT_EQ(keys[i], key);
+      ASSERT_EQ(0, memcmp(vals[i], val, val_len));
+      ASSERT_EQ(lens[i], val_len);
+      ++i;
+    }
+    ASSERT_EQ(i, len);
+    ASSERT_EQ(0, rados_omap_get_next(iter, &key, &val, &val_len));
+    ASSERT_EQ((char*)NULL, key);
+    ASSERT_EQ((char*)NULL, val);
+    ASSERT_EQ(0u, val_len);
+    rados_omap_get_end(iter);
+  }
+
+  void compare_xattrs(char const* const* keys,
+		      char const* const* vals,
+		      const size_t *lens,
+		      size_t len,
+		      rados_xattrs_iter_t iter)
+  {
+    size_t i = 0;
+    char *key = NULL;
+    char *val = NULL;
+    size_t val_len = 0;
+    while (i < len) {
+      ASSERT_EQ(0, rados_getxattrs_next(iter, (const char**) &key,
+					(const char**) &val, &val_len));
+      if (len == 0 && key == NULL && val == NULL)
+	break;
+      EXPECT_EQ(std::string(keys[i]), std::string(key));
+      EXPECT_EQ(0, memcmp(vals[i], val, val_len));
+      EXPECT_EQ(lens[i], val_len);
+      ++i;
+    }
+    ASSERT_EQ(i, len);
+    ASSERT_EQ(0, rados_getxattrs_next(iter, (const char**)&key,
+				      (const char**)&val, &val_len));
+    ASSERT_EQ((char*)NULL, key);
+    ASSERT_EQ((char*)NULL, val);
+    ASSERT_EQ(0u, val_len);
+    rados_getxattrs_end(iter);
+  }
+};
+
+TEST_F(CReadOpsTest, NewDelete) {
+  rados_read_op_t op = rados_create_read_op();
+  ASSERT_TRUE(op);
+  rados_release_read_op(op);
+}
+
+TEST_F(CReadOpsTest, SetOpFlags) {
+  write_object();
+
+  rados_read_op_t op = rados_create_read_op();
+  size_t bytes_read = 0;
+  char *out = NULL;
+  int rval = 0;
+  rados_read_op_exec(op, "rbd", "get_id", NULL, 0, &out,
+		     &bytes_read, &rval);
+  rados_read_op_set_flags(op, LIBRADOS_OP_FLAG_FAILOK);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(-EIO, rval);
+  EXPECT_EQ(0u, bytes_read);
+  EXPECT_EQ((char*)NULL, out);
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, AssertExists) {
+  rados_read_op_t op = rados_create_read_op();
+  rados_read_op_assert_exists(op);
+
+  ASSERT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  op = rados_create_read_op();
+  rados_read_op_assert_exists(op);
+
+  rados_completion_t completion;
+  ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &completion));
+  ASSERT_EQ(0, rados_aio_read_op_operate(op, ioctx, completion, obj, 0));
+  rados_aio_wait_for_complete(completion);
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(completion));
+  rados_release_read_op(op);
+
+  write_object();
+
+  op = rados_create_read_op();
+  rados_read_op_assert_exists(op);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, CmpXattr) {
+  write_object();
+
+  char buf[len];
+  memset(buf, 0xcc, sizeof(buf));
+
+  const char *xattr = "test";
+  rados_setxattr(ioctx, obj, xattr, buf, sizeof(buf));
+
+  // equal value
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LTE));
+
+  // < value
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_LTE));
+
+  // > value
+  memset(buf, 0xcd, sizeof(buf));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LTE));
+
+  // check that null bytes are compared correctly
+  rados_setxattr(ioctx, obj, xattr, "\0\0", 2);
+  buf[0] = '\0';
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LTE));
+
+  buf[1] = '\0';
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LTE));
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, Read) {
+  write_object();
+
+  char buf[len];
+  // check that using read_ops returns the same data with
+  // or without bytes_read and rval out params
+  {
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_read(op, 0, len, buf, NULL, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    int rval;
+    rados_read_op_read(op, 0, len, buf, NULL, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    rados_read_op_read(op, 0, len, buf, &bytes_read, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, (int)bytes_read);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    int rval;
+    rados_read_op_read(op, 0, len, buf, &bytes_read, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, (int)bytes_read);
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, ShortRead) {
+  write_object();
+
+  char buf[len * 2];
+  // check that using read_ops returns the same data with
+  // or without bytes_read and rval out params
+  {
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_read(op, 0, len * 2, buf, NULL, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    int rval;
+    rados_read_op_read(op, 0, len * 2, buf, NULL, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    rados_read_op_read(op, 0, len * 2, buf, &bytes_read, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, (int)bytes_read);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    int rval;
+    rados_read_op_read(op, 0, len * 2, buf, &bytes_read, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, (int)bytes_read);
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, Exec) {
+  // create object so we don't get -ENOENT
+  write_object();
+
+  rados_read_op_t op = rados_create_read_op();
+  ASSERT_TRUE(op);
+  size_t bytes_read = 0;
+  char *out = NULL;
+  int rval = 0;
+  rados_read_op_exec(op, "rbd", "get_all_features", NULL, 0, &out,
+		     &bytes_read, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+  EXPECT_EQ(0, rval);
+  EXPECT_TRUE(out);
+  uint64_t features;
+  EXPECT_EQ(sizeof(features), bytes_read);
+  // make sure buffer is at least as long as it claims
+  ASSERT_TRUE(out[bytes_read-1] == out[bytes_read-1]);
+  rados_buffer_free(out);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, ExecUserBuf) {
+  // create object so we don't get -ENOENT
+  write_object();
+
+  rados_read_op_t op = rados_create_read_op();
+  size_t bytes_read = 0;
+  uint64_t features;
+  char out[sizeof(features)];
+  int rval = 0;
+  rados_read_op_exec_user_buf(op, "rbd", "get_all_features", NULL, 0, out,
+			      sizeof(out), &bytes_read, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+  EXPECT_EQ(0, rval);
+  EXPECT_EQ(sizeof(features), bytes_read);
+
+  // buffer too short
+  bytes_read = 1024;
+  op = rados_create_read_op();
+  rados_read_op_exec_user_buf(op, "rbd", "get_all_features", NULL, 0, out,
+			      sizeof(features) - 1, &bytes_read, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0u, bytes_read);
+  EXPECT_EQ(-ERANGE, rval);
+
+  // input buffer and no rval or bytes_read
+  op = rados_create_read_op();
+  rados_read_op_exec_user_buf(op, "rbd", "get_all_features", out, sizeof(out),
+			      out, sizeof(out), NULL, NULL);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, Stat) {
+  rados_read_op_t op = rados_create_read_op();
+  uint64_t size = 1;
+  int rval;
+  rados_read_op_stat(op, &size, NULL, &rval);
+  EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(-EIO, rval);
+  EXPECT_EQ(1u, size);
+  rados_release_read_op(op);
+
+  write_object();
+
+  op = rados_create_read_op();
+  rados_read_op_stat(op, &size, NULL, &rval);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0, rval);
+  EXPECT_EQ(len, (int)size);
+  rados_release_read_op(op);
+
+  op = rados_create_read_op();
+  rados_read_op_stat(op, NULL, NULL, NULL);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+
+  op = rados_create_read_op();
+  rados_read_op_stat(op, NULL, NULL, NULL);
+  EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+}
+
+TEST_F(CReadOpsTest, Omap) {
+  char *keys[] = {(char*)"bar",
+		  (char*)"foo",
+		  (char*)"test1",
+		  (char*)"test2"};
+  char *vals[] = {(char*)"",
+		  (char*)"\0",
+		  (char*)"abc",
+		  (char*)"va\0lue"};
+  size_t lens[] = {0, 1, 3, 6};
+
+  // check for -ENOENT before the object exists and when it exists
+  // with no omap entries
+  rados_omap_iter_t iter_vals;
+  rados_read_op_t rop = rados_create_read_op();
+  rados_read_op_omap_get_vals(rop, "", "", 10, &iter_vals, NULL);
+  ASSERT_EQ(-ENOENT, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  compare_omap_vals(NULL, NULL, NULL, 0, iter_vals);
+
+  write_object();
+
+  fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+  // write and check for the k/v pairs
+  rados_write_op_t op = rados_create_write_op();
+  rados_write_op_omap_set(op, keys, vals, lens, 4);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals(keys, vals, lens, 4);
+
+  rados_omap_iter_t iter_keys;
+  int r_vals = -1, r_keys = -1;
+  rop = rados_create_read_op();
+  rados_read_op_omap_get_vals(rop, "", "test", 1, &iter_vals, &r_vals);
+  rados_read_op_omap_get_keys(rop, "test", 1, &iter_keys, &r_keys);
+  ASSERT_EQ(0, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  EXPECT_EQ(0, r_vals);
+  EXPECT_EQ(0, r_keys);
+
+  compare_omap_vals(&keys[2], &vals[2], &lens[2], 1, iter_vals);
+  compare_omap_vals(&keys[2], &vals[0], &lens[0], 1, iter_keys);
+
+  // check omap_cmp finds all expected values
+  rop = rados_create_read_op();
+  int rvals[4];
+  for (int i = 0; i < 4; ++i)
+    rados_read_op_omap_cmp(rop, keys[i], LIBRADOS_CMPXATTR_OP_EQ,
+			   vals[i], lens[i], &rvals[i]);
+  EXPECT_EQ(0, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  for (int i = 0; i < 4; ++i)
+    EXPECT_EQ(0, rvals[i]);
+
+  // try to remove keys with a guard that should fail
+  op = rados_create_write_op();
+  rados_write_op_omap_cmp(op, keys[2], LIBRADOS_CMPXATTR_OP_LT,
+			  vals[2], lens[2], &r_vals);
+  rados_write_op_omap_rm_keys(op, keys, 2);
+  EXPECT_EQ(-ECANCELED, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+  ASSERT_EQ(-ECANCELED, r_vals);
+
+  // verifying the keys are still there, and then remove them
+  op = rados_create_write_op();
+  rados_write_op_omap_cmp(op, keys[0], LIBRADOS_CMPXATTR_OP_EQ,
+			  vals[0], lens[0], NULL);
+  rados_write_op_omap_cmp(op, keys[1], LIBRADOS_CMPXATTR_OP_EQ,
+			  vals[1], lens[1], NULL);
+  rados_write_op_omap_rm_keys(op, keys, 2);
+  EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals(&keys[2], &vals[2], &lens[2], 2);
+
+  // clear the rest and check there are none left
+  op = rados_create_write_op();
+  rados_write_op_omap_clear(op);
+  EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, GetXattrs) {
+  write_object();
+
+  char *keys[] = {(char*)"bar",
+		  (char*)"foo",
+		  (char*)"test1",
+		  (char*)"test2"};
+  char *vals[] = {(char*)"",
+		  (char*)"\0",
+		  (char*)"abc",
+		  (char*)"va\0lue"};
+  size_t lens[] = {0, 1, 3, 6};
+
+  int rval = 1;
+  rados_read_op_t op = rados_create_read_op();
+  rados_xattrs_iter_t it;
+  rados_read_op_getxattrs(op, &it, &rval);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0, rval);
+  rados_release_read_op(op);
+  compare_xattrs(keys, vals, lens, 0, it);
+
+  for (int i = 0; i < 4; ++i)
+    rados_setxattr(ioctx, obj, keys[i], vals[i], lens[i]);
+
+  rval = 1;
+  op = rados_create_read_op();
+  rados_read_op_getxattrs(op, &it, &rval);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0, rval);
+  rados_release_read_op(op);
+  compare_xattrs(keys, vals, lens, 4, it);
+
+  remove_object();
+}
diff --git a/ceph/src/test/librados/c_write_operations.cc b/ceph/src/test/librados/c_write_operations.cc
new file mode 100644
index 00000000..4dede36a
--- /dev/null
+++ b/ceph/src/test/librados/c_write_operations.cc
@@ -0,0 +1,142 @@
+// Tests for the C API coverage of atomic write operations
+
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+#include "gtest/gtest.h"
+
+TEST(LibradosCWriteOps, NewDelete) {
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_release_write_op(op);
+}
+
+TEST(LibRadosCWriteOps, assertExists) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_assert_exists(op);
+
+  // -2, ENOENT
+  ASSERT_EQ(-2, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  rados_write_op_t op2 = rados_create_write_op();
+  ASSERT_TRUE(op2);
+  rados_write_op_assert_exists(op2);
+
+  rados_completion_t completion;
+  ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &completion));
+  ASSERT_EQ(0, rados_aio_write_op_operate(op2, ioctx, completion, "test", NULL, 0));
+  rados_aio_wait_for_complete(completion);
+  ASSERT_EQ(-2, rados_aio_get_return_value(completion));
+
+  rados_ioctx_destroy(ioctx);
+  rados_release_write_op(op2);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Xattrs) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  // Create an object with an xattr
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+  rados_write_op_setxattr(op, "key", "value", 5);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  // Check that xattr exists, if it does, delete it.
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_IDEMPOTENT, NULL);
+  rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
+  rados_write_op_rmxattr(op, "key");
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  // Check the xattr exits, if it does, add it again (will fail) with -125
+  // (ECANCELED)
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
+  rados_write_op_setxattr(op, "key", "value", 5);
+  ASSERT_EQ(-125, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+
+  rados_release_write_op(op);
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Write) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  // Create an object, write and write full to it
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+  rados_write_op_write(op, "four", 4, 0);
+  rados_write_op_write_full(op, "hi", 2);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  char hi[4];
+  ASSERT_EQ(2, rados_read(ioctx, "test", hi, 4, 0));
+  rados_release_write_op(op);
+
+  // Truncate and append
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_truncate(op, 1);
+  rados_write_op_append(op, "hi", 2);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  ASSERT_EQ(3, rados_read(ioctx, "test", hi, 4, 0));
+  rados_release_write_op(op);
+
+  // zero and remove
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_zero(op, 0, 3);
+  rados_write_op_remove(op);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  // ENOENT
+  ASSERT_EQ(-2, rados_read(ioctx, "test", hi, 4, 0));
+  rados_release_write_op(op);
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Exec) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  int rval = 1;
+  rados_write_op_t op = rados_create_write_op();
+  rados_write_op_exec(op, "hello", "record_hello", "test", 4, &rval);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+  ASSERT_EQ(0, rval);
+
+  char hi[100];
+  ASSERT_EQ(12, rados_read(ioctx, "test", hi, 100, 0));
+  hi[12] = '\0';
+  ASSERT_EQ(0, strcmp("Hello, test!", hi));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
diff --git a/ceph/src/test/librados/cls.cc b/ceph/src/test/librados/cls.cc
new file mode 100644
index 00000000..1f61664b
--- /dev/null
+++ b/ceph/src/test/librados/cls.cc
@@ -0,0 +1,37 @@
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+
+#include "gtest/gtest.h"
+#include 
+#include 
+#include 
+#include 
+
+using namespace librados;
+using ceph::buffer;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+TEST(LibRadosCls, DNE) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  // create an object
+  string oid = "foo";
+  bufferlist bl;
+  ASSERT_EQ(0, ioctx.write(oid, bl, bl.length(), 0));
+
+  // call a bogus class
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.exec(oid, "doesnotexistasdfasdf", "method", bl, bl));
+
+  // call a bogus method on existent class
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.exec(oid, "lock", "doesnotexistasdfasdfasdf", bl, bl));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
diff --git a/ceph/src/test/librados/cmd.cc b/ceph/src/test/librados/cmd.cc
new file mode 100644
index 00000000..4f327a0e
--- /dev/null
+++ b/ceph/src/test/librados/cmd.cc
@@ -0,0 +1,210 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "mds/mdstypes.h"
+#include "include/buffer.h"
+#include "include/rbd_types.h"
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "test/librados/test.h"
+
+#include "common/Cond.h"
+
+#include "gtest/gtest.h"
+#include 
+#include 
+#include 
+#include 
+
+using namespace librados;
+using ceph::buffer;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+TEST(LibRadosCmd, MonDescribe) {
+  rados_t cluster;
+  ASSERT_EQ("", connect_cluster(&cluster));
+
+  char *buf, *st;
+  size_t buflen, stlen;
+  char *cmd[2];
+
+  cmd[1] = NULL;
+
+  cmd[0] = (char *)"{\"prefix\":\"get_command_descriptions\"}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  ASSERT_LT(0u, buflen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"get_command_descriptions";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"asdfqwer";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "{}", 2, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{\"prefix\":\"mon_status\"}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  ASSERT_LT(0u, buflen);
+  //ASSERT_LT(0u, stlen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosCmd, MonDescribePP) {
+  Rados cluster;
+  ASSERT_EQ("", connect_cluster_pp(cluster));
+  bufferlist inbl, outbl;
+  string outs;
+  ASSERT_EQ(0, cluster.mon_command("{\"prefix\": \"get_command_descriptions\"}",
+				   inbl, &outbl, &outs));
+  ASSERT_LT(0u, outbl.length());
+  ASSERT_LE(0u, outs.length());
+  cluster.shutdown();
+}
+
+TEST(LibRadosCmd, OSDCmd) {
+  rados_t cluster;
+  ASSERT_EQ("", connect_cluster(&cluster));
+  int r;
+  char *buf, *st;
+  size_t buflen, stlen;
+  char *cmd[2];
+  cmd[1] = NULL;
+
+  // note: tolerate NXIO here in case the cluster is thrashing out underneath us.
+  cmd[0] = (char *)"asdfasdf";
+  r = rados_osd_command(cluster, 0, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+  cmd[0] = (char *)"version";
+  r = rados_osd_command(cluster, 0, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+  cmd[0] = (char *)"{\"prefix\":\"version\"}";
+  r = rados_osd_command(cluster, 0, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE((r == 0 && buflen > 0) || (r == -ENXIO && buflen == 0));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosCmd, PGCmd) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+
+  char *buf, *st;
+  size_t buflen, stlen;
+  char *cmd[2];
+  cmd[1] = NULL;
+
+  int64_t poolid = rados_pool_lookup(cluster, pool_name.c_str());
+  ASSERT_LT(0, poolid);
+
+  string pgid = stringify(poolid) + ".0";
+
+  cmd[0] = (char *)"asdfasdf";
+  // note: tolerate NXIO here in case the cluster is thrashing out underneath us.
+  int r = rados_pg_command(cluster, pgid.c_str(), (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+
+  // make sure the pg exists on the osd before we query it
+  rados_ioctx_t io;
+  rados_ioctx_create(cluster, pool_name.c_str(), &io);
+  for (int i=0; i<100; i++) {
+    string oid = "obj" + stringify(i);
+    ASSERT_EQ(-ENOENT, rados_stat(io, oid.c_str(), NULL, NULL));
+  }
+  rados_ioctx_destroy(io);
+
+  string qstr = "{\"prefix\":\"pg\", \"cmd\":\"query\", \"pgid\":\"" +  pgid + "\"}";
+  cmd[0] = (char *)qstr.c_str();
+  // note: tolerate ENOENT/ENXIO here if hte osd is thrashing out underneath us
+  r = rados_pg_command(cluster, pgid.c_str(), (const char **)cmd, 1, "", 0,  &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE(r == 0 || r == -ENOENT || r == -ENXIO);
+
+  ASSERT_LT(0u, buflen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+struct Log {
+  list log;
+  Cond cond;
+  Mutex lock;
+
+  Log() : lock("l::lock") {}
+
+  bool contains(string str) {
+    Mutex::Locker l(lock);
+    for (list::iterator p = log.begin(); p != log.end(); ++p) {
+      if (p->find(str) != std::string::npos)
+	return true;
+    }
+    return false;
+  }
+};
+
+void log_cb(void *arg,
+	     const char *line,
+	     const char *who, uint64_t stampsec, uint64_t stamp_nsec,
+	     uint64_t seq, const char *level,
+	     const char *msg) {
+  Log *l = static_cast(arg);
+  Mutex::Locker locker(l->lock);
+  l->log.push_back(line);
+  l->cond.Signal();
+  cout << "got: " << line << std::endl;
+}
+
+TEST(LibRadosCmd, WatchLog) {
+  rados_t cluster;
+  ASSERT_EQ("", connect_cluster(&cluster));
+  char *buf, *st;
+  char *cmd[2];
+  cmd[1] = NULL;
+  size_t buflen, stlen;
+  Log l;
+
+  ASSERT_EQ(0, rados_monitor_log(cluster, "info", log_cb, &l));
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"onexx\"]}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  for (int i=0; !l.contains("onexx"); i++) {
+    ASSERT_TRUE(i<100);
+    sleep(1);
+  }
+  ASSERT_TRUE(l.contains("onexx"));
+
+  /*
+    changing the subscribe level is currently broken.
+
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"twoxx\"]}";
+  ASSERT_EQ(0, rados_monitor_log(cluster, "err", log_cb, &l));
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  sleep(2);
+  ASSERT_FALSE(l.contains("twoxx"));
+  */
+
+  ASSERT_EQ(0, rados_monitor_log(cluster, "info", log_cb, &l));
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"threexx\"]}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  for (int i=0; !l.contains("threexx"); i++) {
+    ASSERT_TRUE(i<100);
+    sleep(1);
+  }
+
+  ASSERT_EQ(0, rados_monitor_log(cluster, "info", NULL, NULL));
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"fourxx\"]}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  sleep(2);
+  ASSERT_FALSE(l.contains("fourxx"));
+  rados_shutdown(cluster);
+}
diff --git a/ceph/src/test/librados/io.cc b/ceph/src/test/librados/io.cc
new file mode 100644
index 00000000..0bb805f0
--- /dev/null
+++ b/ceph/src/test/librados/io.cc
@@ -0,0 +1,966 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include 
+#include "gtest/gtest.h"
+
+using namespace librados;
+using std::string;
+
+typedef RadosTest LibRadosIo;
+typedef RadosTestEC LibRadosIoEC;
+typedef RadosTestPP LibRadosIoPP;
+typedef RadosTestECPP LibRadosIoECPP;
+
+TEST_F(LibRadosIo, SimpleWrite) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIo, ReadTimeout) {
+  char buf[128];
+  memset(buf, 'a', sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  {
+    // set up a second client
+    rados_t cluster;
+    rados_ioctx_t ioctx;
+    rados_create(&cluster, "admin");
+    rados_conf_read_file(cluster, NULL);
+    rados_conf_parse_env(cluster, NULL);
+    rados_conf_set(cluster, "rados_osd_op_timeout", "0.00001"); // use any small value that will result in a timeout
+    rados_connect(cluster);
+    rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+    rados_ioctx_set_namespace(ioctx, nspace.c_str());
+
+    // then we show that the buffer is changed after rados_read returned
+    // with a timeout
+    for (int i=0; i<5; i++) {
+      char buf2[sizeof(buf)];
+      memset(buf2, 0, sizeof(buf2));
+      int err = rados_read(ioctx, "foo", buf2, sizeof(buf2), 0);
+      if (err == -110) {
+	int startIndex = 0;
+	// find the index until which librados already read the object before the timeout occurred
+	for (unsigned b=0; b attrset;
+  ASSERT_EQ(0, ioctx.getxattrs("foo", attrset));
+  for (std::map::iterator i = attrset.begin();
+       i != attrset.end(); ++i) {
+    if (i->first == string(attr1)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr1_buf, sizeof(attr1_buf)));
+    }
+    else if (i->first == string(attr2)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr2_buf, sizeof(attr2_buf)));
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+}
+
+TEST_F(LibRadosIoEC, SimpleWrite) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoECPP, SimpleWritePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoECPP, ReadOpPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+
+  {
+      bufferlist op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl, op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist op_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl, op_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl1, read_bl2, op_bl;
+      int rval1 = 1000, rval2 = 1002;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl1, &rval1);
+      op.read(0, sizeof(buf), &read_bl2, &rval2);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl1.length());
+      ASSERT_EQ(sizeof(buf), read_bl2.length());
+      ASSERT_EQ(sizeof(buf) * 2, op_bl.length());
+      ASSERT_EQ(0, rval1);
+      ASSERT_EQ(0, rval2);
+      ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(op_bl.c_str() + sizeof(buf), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(0, rval);
+  }
+
+  {
+      bufferlist read_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl1, read_bl2;
+      int rval1 = 1000, rval2 = 1002;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl1, &rval1);
+      op.read(0, sizeof(buf), &read_bl2, &rval2);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl1.length());
+      ASSERT_EQ(sizeof(buf), read_bl2.length());
+      ASSERT_EQ(0, rval1);
+      ASSERT_EQ(0, rval2);
+      ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+  }
+}
+
+TEST_F(LibRadosIoEC, RoundTrip) {
+  char buf[128];
+  char buf2[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+}
+
+TEST_F(LibRadosIoECPP, RoundTripPP) {
+  char buf[128];
+  Rados cluster;
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  bufferlist cl;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", cl, sizeof(buf) * 3, 0));
+  ASSERT_EQ(0, memcmp(buf, cl.c_str(), sizeof(buf)));
+}
+
+TEST_F(LibRadosIoEC, OverlappingWriteRoundTrip) {
+  int bsize = alignment;
+  int dbsize = bsize * 2;
+  char *buf = (char *)new char[dbsize];
+  char *buf2 = (char *)new char[bsize];
+  char *buf3 = (char *)new char[dbsize];
+  memset(buf, 0xcc, dbsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, dbsize, 0));
+  memset(buf2, 0xdd, bsize);
+  ASSERT_EQ(-EOPNOTSUPP, rados_write(ioctx, "foo", buf2, bsize, 0));
+  memset(buf3, 0xdd, dbsize);
+  ASSERT_EQ(dbsize, rados_read(ioctx, "foo", buf3, dbsize, 0));
+  // Read the same as first write
+  ASSERT_EQ(0, memcmp(buf3, buf, dbsize));
+
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+}
+
+TEST_F(LibRadosIoECPP, OverlappingWriteRoundTripPP) {
+  int bsize = alignment;
+  int dbsize = bsize * 2;
+  char *buf = (char *)new char[dbsize];
+  char *buf2 = (char *)new char[bsize];
+  memset(buf, 0xcc, dbsize);
+  bufferlist bl1;
+  bl1.append(buf, dbsize);
+  ASSERT_EQ(0, ioctx.write("foo", bl1, dbsize, 0));
+  memset(buf2, 0xdd, bsize);
+  bufferlist bl2;
+  bl2.append(buf2, bsize);
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.write("foo", bl2, bsize, 0));
+  bufferlist bl3;
+  ASSERT_EQ(dbsize, ioctx.read("foo", bl3, dbsize, 0));
+  // Read the same as first write
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, dbsize));
+
+  delete[] buf;
+  delete[] buf2;
+}
+
+TEST_F(LibRadosIoEC, WriteFullRoundTrip) {
+  char buf[128];
+  char buf2[64];
+  char buf3[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+  memset(buf3, 0xee, sizeof(buf3));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIoECPP, WriteFullRoundTripPP) {
+  char buf[128];
+  char buf2[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write_full("foo", bl2));
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf2), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIoEC, AppendRoundTrip) {
+  char *buf = (char *)new char[alignment];
+  char *buf2 = (char *)new char[alignment];
+  char *buf3 = (char *)new char[alignment *2];
+  memset(buf, 0xde, alignment);
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, alignment));
+  memset(buf2, 0xad, alignment);
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf2, alignment));
+  memset(buf3, 0, alignment*2);
+  ASSERT_EQ((int)alignment*2, rados_read(ioctx, "foo", buf3, alignment*2, 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, alignment));
+  ASSERT_EQ(0, memcmp(buf3 + alignment, buf2, alignment));
+
+  int uasize = alignment/2;
+  char *unalignedbuf = (char *)new char[uasize];
+  ASSERT_EQ(0, rados_append(ioctx, "foo", unalignedbuf, uasize));
+  ASSERT_EQ(-EOPNOTSUPP, rados_append(ioctx, "foo", unalignedbuf, uasize));
+
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+  delete[] unalignedbuf;
+}
+
+TEST_F(LibRadosIoECPP, AppendRoundTripPP) {
+  char *buf = (char *)new char[alignment];
+  char *buf2 = (char *)new char[alignment];
+  memset(buf, 0xde, alignment);
+  bufferlist bl1;
+  bl1.append(buf, alignment);
+  ASSERT_EQ(0, ioctx.append("foo", bl1, alignment));
+  memset(buf2, 0xad, alignment);
+  bufferlist bl2;
+  bl2.append(buf2, alignment);
+  ASSERT_EQ(0, ioctx.append("foo", bl2, alignment));
+  bufferlist bl3;
+  ASSERT_EQ((int)(alignment * 2),
+	    ioctx.read("foo", bl3, (alignment * 4), 0));
+  const char *bl3_str = bl3.c_str();
+  ASSERT_EQ(0, memcmp(bl3_str, buf, alignment));
+  ASSERT_EQ(0, memcmp(bl3_str + alignment, buf2, alignment));
+
+  delete[] buf;
+  delete[] buf2;
+}
+
+TEST_F(LibRadosIoEC, TruncTest) {
+  char buf[128];
+  char buf2[sizeof(buf)];
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(-EOPNOTSUPP, rados_trunc(ioctx, "foo", sizeof(buf) / 2));
+  memset(buf2, 0, sizeof(buf2));
+  // Same size
+  ASSERT_EQ((int)sizeof(buf), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+  // No change
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+}
+
+TEST_F(LibRadosIoECPP, TruncTestPP) {
+  char buf[128];
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl, sizeof(buf)));
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.trunc("foo", sizeof(buf) / 2));
+  bufferlist bl2;
+  // Same size
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl2, sizeof(buf), 0));
+  // No change
+  ASSERT_EQ(0, memcmp(bl2.c_str(), buf, sizeof(buf)));
+}
+
+TEST_F(LibRadosIoEC, RemoveTest) {
+  char buf[128];
+  char buf2[sizeof(buf)];
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+}
+
+TEST_F(LibRadosIoECPP, RemoveTestPP) {
+  char buf[128];
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  ASSERT_EQ(0, ioctx.remove("foo"));
+  bufferlist bl2;
+  ASSERT_EQ(-ENOENT, ioctx.read("foo", bl2, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoEC, XattrsRoundTrip) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ((int)sizeof(attr1_buf),
+	    rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(attr1_buf, buf, sizeof(attr1_buf)));
+}
+
+TEST_F(LibRadosIoECPP, XattrsRoundTripPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  ASSERT_EQ(-ENODATA, ioctx.getxattr("foo", attr1, bl2));
+  bufferlist bl3;
+  bl3.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl3));
+  bufferlist bl4;
+  ASSERT_EQ((int)sizeof(attr1_buf),
+      ioctx.getxattr("foo", attr1, bl4));
+  ASSERT_EQ(0, memcmp(bl4.c_str(), attr1_buf, sizeof(attr1_buf)));
+}
+
+TEST_F(LibRadosIoEC, RmXattr) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0,
+      rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ(0, rados_rmxattr(ioctx, "foo", attr1));
+  ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+}
+
+TEST_F(LibRadosIoECPP, RmXattrPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl2));
+  ASSERT_EQ(0, ioctx.rmxattr("foo", attr1));
+  bufferlist bl3;
+  ASSERT_EQ(-ENODATA, ioctx.getxattr("foo", attr1, bl3));
+}
+
+TEST_F(LibRadosIoEC, XattrIter) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr2, attr2_buf, sizeof(attr2_buf)));
+  rados_xattrs_iter_t iter;
+  ASSERT_EQ(0, rados_getxattrs(ioctx, "foo", &iter));
+  int num_seen = 0;
+  while (true) {
+    const char *name;
+    const char *val;
+    size_t len;
+    ASSERT_EQ(0, rados_getxattrs_next(iter, &name, &val, &len));
+    if (name == NULL) {
+      break;
+    }
+    ASSERT_LT(num_seen, 2);
+    if ((strcmp(name, attr1) == 0) && (memcmp(val, attr1_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else if ((strcmp(name, attr2) == 0) && (memcmp(val, attr2_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+  rados_getxattrs_end(iter);
+}
+
+TEST_F(LibRadosIoECPP, XattrListPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl2));
+  bufferlist bl3;
+  bl3.append(attr2_buf, sizeof(attr2_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr2, bl3));
+  std::map attrset;
+  ASSERT_EQ(0, ioctx.getxattrs("foo", attrset));
+  for (std::map::iterator i = attrset.begin();
+       i != attrset.end(); ++i) {
+    if (i->first == string(attr1)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr1_buf, sizeof(attr1_buf)));
+    }
+    else if (i->first == string(attr2)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr2_buf, sizeof(attr2_buf)));
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+}
diff --git a/ceph/src/test/librados/librados.cc b/ceph/src/test/librados/librados.cc
new file mode 100644
index 00000000..c688724d
--- /dev/null
+++ b/ceph/src/test/librados/librados.cc
@@ -0,0 +1,13 @@
+//#include "common/config.h"
+#include "include/rados/librados.h"
+
+#include "gtest/gtest.h"
+
+TEST(Librados, CreateShutdown) {
+  rados_t cluster;
+  int err;
+  err = rados_create(&cluster, "someid");
+  EXPECT_EQ(err, 0);
+
+  rados_shutdown(cluster);
+}
diff --git a/ceph/src/test/librados/librados_config.cc b/ceph/src/test/librados/librados_config.cc
new file mode 100644
index 00000000..81c0465a
--- /dev/null
+++ b/ceph/src/test/librados/librados_config.cc
@@ -0,0 +1,98 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/rados/librados.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using std::string;
+
+TEST(LibRadosConfig, SimpleSet) {
+  rados_t cl;
+  int ret = rados_create(&cl, NULL);
+  ASSERT_EQ(ret, 0);
+
+  ret = rados_conf_set(cl, "max_open_files", "21");
+  ASSERT_EQ(ret, 0);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "max_open_files", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("21"), string(buf));
+
+  rados_shutdown(cl);
+}
+
+TEST(LibRadosConfig, ArgV) {
+  rados_t cl;
+  int ret = rados_create(&cl, NULL);
+  ASSERT_EQ(ret, 0);
+
+  const char *argv[] = { "foo", "--max-open-files", "2",
+			 "--keyfile", "/tmp/my-keyfile", NULL };
+  size_t argc = (sizeof(argv) / sizeof(argv[0])) - 1;
+  rados_conf_parse_argv(cl, argc, argv);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "keyfile", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("/tmp/my-keyfile"), string(buf));
+
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "max_open_files", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("2"), string(buf));
+
+  rados_shutdown(cl);
+}
+
+TEST(LibRadosConfig, DebugLevels) {
+  rados_t cl;
+  int ret = rados_create(&cl, NULL);
+  ASSERT_EQ(ret, 0);
+
+  ret = rados_conf_set(cl, "debug_rados", "3");
+  ASSERT_EQ(ret, 0);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "debug_rados", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(0, strncmp("3/", buf, 2));
+
+  ret = rados_conf_set(cl, "debug_rados", "7/8");
+  ASSERT_EQ(ret, 0);
+
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "debug_rados", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(0, strcmp("7/8", buf));
+
+  ret = rados_conf_set(cl, "debug_rados", "foo");
+  ASSERT_EQ(ret, -EINVAL);
+
+  ret = rados_conf_set(cl, "debug_asdkfasdjfajksdf", "foo");
+  ASSERT_EQ(ret, -ENOENT);
+
+  ret = rados_conf_get(cl, "debug_radfjadfsdados", buf, sizeof(buf));
+  ASSERT_EQ(ret, -ENOENT);
+
+  rados_shutdown(cl);
+}
diff --git a/ceph/src/test/librados/list.cc b/ceph/src/test/librados/list.cc
new file mode 100644
index 00000000..e3fe007f
--- /dev/null
+++ b/ceph/src/test/librados/list.cc
@@ -0,0 +1,633 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include "include/types.h"
+#include "gtest/gtest.h"
+#include 
+#include 
+
+using namespace librados;
+
+typedef RadosTest LibRadosList;
+typedef RadosTestPP LibRadosListPP;
+typedef RadosTestEC LibRadosListEC;
+typedef RadosTestECPP LibRadosListECPP;
+
+TEST_F(LibRadosList, ListObjects) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  const char *entry;
+  bool foundit = false;
+  while (rados_objects_list_next(ctx, &entry, NULL) != -ENOENT) {
+    foundit = true;
+    ASSERT_EQ(std::string(entry), "foo");
+  }
+  ASSERT_TRUE(foundit);
+  rados_objects_list_close(ctx);
+}
+
+TEST_F(LibRadosListPP, ListObjectsPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ObjectIterator iter(ioctx.objects_begin());
+  bool foundit = false;
+  while (iter != ioctx.objects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListPP, ListObjectsTwicePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ObjectIterator iter(ioctx.objects_begin());
+  bool foundit = false;
+  while (iter != ioctx.objects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.objects_end());
+  foundit = false;
+  iter.seek(0);
+  while (iter != ioctx.objects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListPP, ListObjectsCopyIterPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  // make sure this is still valid after the original iterators are gone
+  ObjectIterator iter3;
+  {
+    ObjectIterator iter(ioctx.objects_begin());
+    ObjectIterator iter2(iter);
+    iter3 = iter2;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.objects_end());
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.objects_end());
+
+    ASSERT_EQ(iter2->first, "foo");
+    ASSERT_EQ(iter3->first, "foo");
+    ++iter2;
+    ASSERT_TRUE(iter2 == ioctx.objects_end());
+  }
+
+  ASSERT_EQ(iter3->first, "foo");
+  iter3 = iter3;
+  ASSERT_EQ(iter3->first, "foo");
+  ++iter3;
+  ASSERT_TRUE(iter3 == ioctx.objects_end());
+}
+
+TEST_F(LibRadosListPP, ListObjectsEndIter) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  ObjectIterator iter(ioctx.objects_begin());
+  ObjectIterator iter_end(ioctx.objects_end());
+  ObjectIterator iter_end2 = ioctx.objects_end();
+  ASSERT_TRUE(iter_end == iter_end2);
+  ASSERT_TRUE(iter_end == ioctx.objects_end());
+  ASSERT_TRUE(iter_end2 == ioctx.objects_end());
+
+  ASSERT_EQ(iter->first, "foo");
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.objects_end());
+  ASSERT_TRUE(iter == iter_end);
+  ASSERT_TRUE(iter == iter_end2);
+  ObjectIterator iter2 = iter;
+  ASSERT_TRUE(iter2 == ioctx.objects_end());
+  ASSERT_TRUE(iter2 == iter_end);
+  ASSERT_TRUE(iter2 == iter_end2);
+}
+
+static void check_list(std::set& myset, rados_list_ctx_t& ctx)
+{
+  const char *entry;
+  std::set orig_set(myset);
+  /**
+   * During splitting, we might see duplicate items.
+   * We assert that every object returned is in myset and that
+   * we don't hit ENOENT until we have hit every item in myset
+   * at least once.
+   */
+  while (rados_objects_list_next(ctx, &entry, NULL) != -ENOENT) {
+    ASSERT_TRUE(orig_set.end() != orig_set.find(std::string(entry)));
+    myset.erase(std::string(entry));
+  }
+  ASSERT_TRUE(myset.empty());
+}
+
+TEST_F(LibRadosList, ListObjectsNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo3", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo4", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo5", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_write(ioctx, "foo6", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo7", buf, sizeof(buf), 0));
+
+  std::set def, ns1, ns2;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+
+  rados_list_ctx_t ctx;
+  // Check default namespace ""
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  check_list(def, ctx);
+  rados_objects_list_close(ctx);
+
+  // Check default namespace "ns1"
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  check_list(ns1, ctx);
+  rados_objects_list_close(ctx);
+
+  // Check default namespace "ns2"
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  check_list(ns2, ctx);
+  rados_objects_list_close(ctx);
+}
+
+static void check_listpp(std::set& myset, IoCtx& ioctx)
+{
+  ObjectIterator iter(ioctx.objects_begin());
+  std::set orig_set(myset);
+  /**
+   * During splitting, we might see duplicate items.
+   * We assert that every object returned is in myset and that
+   * we don't hit ENOENT until we have hit every item in myset
+   * at least once.
+   */
+  while (iter != ioctx.objects_end()) {
+    ASSERT_TRUE(orig_set.end() != orig_set.find(std::string((*iter).first)));
+    myset.erase(std::string((*iter).first));
+    ++iter;
+  }
+  ASSERT_TRUE(myset.empty());
+}
+
+TEST_F(LibRadosListPP, ListObjectsPPNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo2", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo3", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo4", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo5", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns2");
+  ASSERT_EQ(0, ioctx.write("foo6", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo7", bl1, sizeof(buf), 0));
+
+  std::set def, ns1, ns2;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+
+  ioctx.set_namespace("");
+  check_listpp(def, ioctx);
+
+  ioctx.set_namespace("ns1");
+  check_listpp(ns1, ioctx);
+
+  ioctx.set_namespace("ns2");
+  check_listpp(ns2, ioctx);
+}
+
+TEST_F(LibRadosListPP, ListObjectsManyPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<256; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::ObjectIterator it = ioctx.objects_begin();
+  std::set saw_obj;
+  std::set saw_pg;
+  for (; it != ioctx.objects_end(); ++it) {
+    std::cout << it->first
+	      << " " << it.get_pg_hash_position() << std::endl;
+    saw_obj.insert(it->first);
+    saw_pg.insert(it.get_pg_hash_position());
+  }
+  std::cout << "saw " << saw_pg.size() << " pgs " << std::endl;
+
+  // make sure they are 0..n
+  for (unsigned i = 0; i < saw_pg.size(); ++i)
+    ASSERT_TRUE(saw_pg.count(i));
+}
+
+TEST_F(LibRadosList, ListObjectsStart) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    string n = stringify(i);
+    ASSERT_EQ(0, rados_write(ioctx, n.c_str(), buf, sizeof(buf), 0));
+  }
+
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  std::map > pg_to_obj;
+  const char *entry;
+  while (rados_objects_list_next(ctx, &entry, NULL) == 0) {
+    uint32_t pos = rados_objects_list_get_pg_hash_position(ctx);
+    std::cout << entry << " " << pos << std::endl;
+    pg_to_obj[pos].insert(entry);
+  }
+  rados_objects_list_close(ctx);
+
+  std::map >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, rados_objects_list_seek(ctx, p->first));
+    ASSERT_EQ(0, rados_objects_list_next(ctx, &entry, NULL));
+    std::cout << "have " << entry << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(entry));
+    ++p;
+  }
+  rados_objects_list_close(ctx);
+}
+
+TEST_F(LibRadosListPP, ListObjectsStartPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::ObjectIterator it = ioctx.objects_begin();
+  std::map > pg_to_obj;
+  for (; it != ioctx.objects_end(); ++it) {
+    std::cout << it->first << " " << it.get_pg_hash_position() << std::endl;
+    pg_to_obj[it.get_pg_hash_position()].insert(it->first);
+  }
+
+  std::map >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  it = ioctx.objects_begin(p->first);
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, it.seek(p->first));
+    std::cout << "have " << it->first << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(it->first));
+    ++p;
+  }
+}
+
+TEST_F(LibRadosListEC, ListObjects) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  const char *entry;
+  bool foundit = false;
+  while (rados_objects_list_next(ctx, &entry, NULL) != -ENOENT) {
+    foundit = true;
+    ASSERT_EQ(std::string(entry), "foo");
+  }
+  ASSERT_TRUE(foundit);
+  rados_objects_list_close(ctx);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ObjectIterator iter(ioctx.objects_begin());
+  bool foundit = false;
+  while (iter != ioctx.objects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsTwicePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ObjectIterator iter(ioctx.objects_begin());
+  bool foundit = false;
+  while (iter != ioctx.objects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.objects_end());
+  foundit = false;
+  iter.seek(0);
+  while (iter != ioctx.objects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsCopyIterPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  // make sure this is still valid after the original iterators are gone
+  ObjectIterator iter3;
+  {
+    ObjectIterator iter(ioctx.objects_begin());
+    ObjectIterator iter2(iter);
+    iter3 = iter2;
+    ASSERT_EQ((*iter).first, "foo");
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.objects_end());
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.objects_end());
+
+    ASSERT_EQ(iter2->first, "foo");
+    ASSERT_EQ(iter3->first, "foo");
+    ++iter2;
+    ASSERT_TRUE(iter2 == ioctx.objects_end());
+  }
+
+  ASSERT_EQ(iter3->first, "foo");
+  iter3 = iter3;
+  ASSERT_EQ(iter3->first, "foo");
+  ++iter3;
+  ASSERT_TRUE(iter3 == ioctx.objects_end());
+}
+
+TEST_F(LibRadosListECPP, ListObjectsEndIter) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  ObjectIterator iter(ioctx.objects_begin());
+  ObjectIterator iter_end(ioctx.objects_end());
+  ObjectIterator iter_end2 = ioctx.objects_end();
+  ASSERT_TRUE(iter_end == iter_end2);
+  ASSERT_TRUE(iter_end == ioctx.objects_end());
+  ASSERT_TRUE(iter_end2 == ioctx.objects_end());
+
+  ASSERT_EQ(iter->first, "foo");
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.objects_end());
+  ASSERT_TRUE(iter == iter_end);
+  ASSERT_TRUE(iter == iter_end2);
+  ObjectIterator iter2 = iter;
+  ASSERT_TRUE(iter2 == ioctx.objects_end());
+  ASSERT_TRUE(iter2 == iter_end);
+  ASSERT_TRUE(iter2 == iter_end2);
+}
+
+TEST_F(LibRadosListEC, ListObjectsNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo3", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo4", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo5", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_write(ioctx, "foo6", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo7", buf, sizeof(buf), 0));
+
+  std::set def, ns1, ns2;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+
+  rados_list_ctx_t ctx;
+  // Check default namespace ""
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  check_list(def, ctx);
+  rados_objects_list_close(ctx);
+
+  // Check default namespace "ns1"
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  check_list(ns1, ctx);
+  rados_objects_list_close(ctx);
+
+  // Check default namespace "ns2"
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  check_list(ns2, ctx);
+  rados_objects_list_close(ctx);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsPPNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo2", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo3", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo4", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo5", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns2");
+  ASSERT_EQ(0, ioctx.write("foo6", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo7", bl1, sizeof(buf), 0));
+
+  std::set def, ns1, ns2;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+
+  ioctx.set_namespace("");
+  check_listpp(def, ioctx);
+
+  ioctx.set_namespace("ns1");
+  check_listpp(ns1, ioctx);
+
+  ioctx.set_namespace("ns2");
+  check_listpp(ns2, ioctx);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsManyPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<256; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::ObjectIterator it = ioctx.objects_begin();
+  std::set saw_obj;
+  std::set saw_pg;
+  for (; it != ioctx.objects_end(); ++it) {
+    std::cout << it->first
+	      << " " << it.get_pg_hash_position() << std::endl;
+    saw_obj.insert(it->first);
+    saw_pg.insert(it.get_pg_hash_position());
+  }
+  std::cout << "saw " << saw_pg.size() << " pgs " << std::endl;
+
+  // make sure they are 0..n
+  for (unsigned i = 0; i < saw_pg.size(); ++i)
+    ASSERT_TRUE(saw_pg.count(i));
+}
+
+TEST_F(LibRadosListEC, ListObjectsStart) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    string n = stringify(i);
+    ASSERT_EQ(0, rados_write(ioctx, n.c_str(), buf, sizeof(buf), 0));
+  }
+
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  std::map > pg_to_obj;
+  const char *entry;
+  while (rados_objects_list_next(ctx, &entry, NULL) == 0) {
+    uint32_t pos = rados_objects_list_get_pg_hash_position(ctx);
+    std::cout << entry << " " << pos << std::endl;
+    pg_to_obj[pos].insert(entry);
+  }
+  rados_objects_list_close(ctx);
+
+  std::map >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  ASSERT_EQ(0, rados_objects_list_open(ioctx, &ctx));
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, rados_objects_list_seek(ctx, p->first));
+    ASSERT_EQ(0, rados_objects_list_next(ctx, &entry, NULL));
+    std::cout << "have " << entry << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(entry));
+    ++p;
+  }
+  rados_objects_list_close(ctx);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsStartPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::ObjectIterator it = ioctx.objects_begin();
+  std::map > pg_to_obj;
+  for (; it != ioctx.objects_end(); ++it) {
+    std::cout << it->first << " " << it.get_pg_hash_position() << std::endl;
+    pg_to_obj[it.get_pg_hash_position()].insert(it->first);
+  }
+
+  std::map >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  it = ioctx.objects_begin(p->first);
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, it.seek(p->first));
+    std::cout << "have " << it->first << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(it->first));
+    ++p;
+  }
+}
diff --git a/ceph/src/test/librados/lock.cc b/ceph/src/test/librados/lock.cc
new file mode 100644
index 00000000..1a7c5838
--- /dev/null
+++ b/ceph/src/test/librados/lock.cc
@@ -0,0 +1,374 @@
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#include "cls/lock/cls_lock_client.h"
+
+#include 
+#include 
+#include "gtest/gtest.h"
+#include 
+
+using namespace librados;
+
+typedef RadosTest LibRadosLock;
+typedef RadosTestPP LibRadosLockPP;
+typedef RadosTestEC LibRadosLockEC;
+typedef RadosTestECPP LibRadosLockECPP;
+
+TEST_F(LibRadosLock, LockExclusive) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockPP, LockExclusivePP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLock, LockShared) {
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockPP, LockSharedPP) {
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLock, LockExclusiveDur) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", &tv,  0));
+  sleep(1);
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockPP, LockExclusiveDurPP) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", &tv,  0));
+  sleep(1);
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLock, LockSharedDur) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", &tv, 0));
+  sleep(1);
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockPP, LockSharedDurPP) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", &tv, 0));
+  sleep(1);
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLock, LockRenew) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, LOCK_FLAG_RENEW));
+}
+
+TEST_F(LibRadosLockPP, LockRenewPP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, LOCK_FLAG_RENEW));
+}
+
+TEST_F(LibRadosLock, Unlock) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock", "Cookie"));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL,  0));
+}
+
+TEST_F(LibRadosLockPP, UnlockPP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLock", "Cookie"));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLock, ListLockers) {
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock", "Cookie"));
+  ASSERT_EQ(0, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-34, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  tag_len = 1024;
+  clients_len = 1024;
+  cookies_len = 1024;
+  addresses_len = 1024;
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, exclusive);
+  ASSERT_EQ(0, strcmp(tag, "Tag"));
+  ASSERT_EQ(strlen("Tag") + 1, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+}
+
+TEST_F(LibRadosLockPP, ListLockersPP) {
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLock", "Cookie"));
+  {
+    int exclusive;
+    std::string tag;
+    std::list lockers;
+    ASSERT_EQ(0, ioctx.list_lockers("foo", "TestLock", &exclusive, &tag, &lockers));
+  }
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  {
+    int exclusive;
+    std::string tag;
+    std::list lockers;
+    ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLock", &exclusive, &tag, &lockers));
+    std::list::iterator it = lockers.begin();
+    ASSERT_FALSE(lockers.end() == it);
+    ASSERT_EQ(me, it->client);
+    ASSERT_EQ("Cookie", it->cookie);
+  }
+}
+
+TEST_F(LibRadosLock, BreakLock) {
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(1, exclusive);
+  ASSERT_EQ(0, strcmp(tag, ""));
+  ASSERT_EQ(1U, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+  ASSERT_EQ(0, rados_break_lock(ioctx, "foo", "TestLock", clients, "Cookie"));
+}
+
+TEST_F(LibRadosLockPP, BreakLockPP) {
+  int exclusive;
+  std::string tag;
+  std::list lockers;
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie",  "", NULL, 0));
+  ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLock", &exclusive, &tag, &lockers));
+  std::list::iterator it = lockers.begin();
+  ASSERT_FALSE(lockers.end() == it);
+  ASSERT_EQ(me, it->client);
+  ASSERT_EQ("Cookie", it->cookie);
+  ASSERT_EQ(0, ioctx.break_lock("foo", "TestLock", it->client, "Cookie"));
+}
+
+// EC testing
+TEST_F(LibRadosLockEC, LockExclusive) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockECPP, LockExclusivePP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockEC, LockShared) {
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockECPP, LockSharedPP) {
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockEC, LockExclusiveDur) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", &tv,  0));
+  sleep(1);
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockECPP, LockExclusiveDurPP) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", &tv,  0));
+  sleep(1);
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockEC, LockSharedDur) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", &tv, 0));
+  sleep(1);
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockECPP, LockSharedDurPP) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", &tv, 0));
+  sleep(1);
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockEC, LockRenew) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, LOCK_FLAG_RENEW));
+}
+
+TEST_F(LibRadosLockECPP, LockRenewPP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, LOCK_FLAG_RENEW));
+}
+
+TEST_F(LibRadosLockEC, Unlock) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock", "Cookie"));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL,  0));
+}
+
+TEST_F(LibRadosLockECPP, UnlockPP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLock", "Cookie"));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockEC, ListLockers) {
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock", "Cookie"));
+  ASSERT_EQ(0, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-34, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  tag_len = 1024;
+  clients_len = 1024;
+  cookies_len = 1024;
+  addresses_len = 1024;
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, exclusive);
+  ASSERT_EQ(0, strcmp(tag, "Tag"));
+  ASSERT_EQ(strlen("Tag") + 1, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+}
+
+TEST_F(LibRadosLockECPP, ListLockersPP) {
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLock", "Cookie"));
+  {
+    int exclusive;
+    std::string tag;
+    std::list lockers;
+    ASSERT_EQ(0, ioctx.list_lockers("foo", "TestLock", &exclusive, &tag, &lockers));
+  }
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLock", "Cookie", "Tag", "", NULL, 0));
+  {
+    int exclusive;
+    std::string tag;
+    std::list lockers;
+    ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLock", &exclusive, &tag, &lockers));
+    std::list::iterator it = lockers.begin();
+    ASSERT_FALSE(lockers.end() == it);
+    ASSERT_EQ(me, it->client);
+    ASSERT_EQ("Cookie", it->cookie);
+  }
+}
+
+TEST_F(LibRadosLockEC, BreakLock) {
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLock", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(1, exclusive);
+  ASSERT_EQ(0, strcmp(tag, ""));
+  ASSERT_EQ(1U, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+  ASSERT_EQ(0, rados_break_lock(ioctx, "foo", "TestLock", clients, "Cookie"));
+}
+
+TEST_F(LibRadosLockECPP, BreakLockPP) {
+  int exclusive;
+  std::string tag;
+  std::list lockers;
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLock", "Cookie",  "", NULL, 0));
+  ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLock", &exclusive, &tag, &lockers));
+  std::list::iterator it = lockers.begin();
+  ASSERT_FALSE(lockers.end() == it);
+  ASSERT_EQ(me, it->client);
+  ASSERT_EQ("Cookie", it->cookie);
+  ASSERT_EQ(0, ioctx.break_lock("foo", "TestLock", it->client, "Cookie"));
+}
diff --git a/ceph/src/test/librados/misc.cc b/ceph/src/test/librados/misc.cc
new file mode 100644
index 00000000..ea990b50
--- /dev/null
+++ b/ceph/src/test/librados/misc.cc
@@ -0,0 +1,621 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "gtest/gtest.h"
+
+#include "mds/mdstypes.h"
+#include "include/buffer.h"
+#include "include/rbd_types.h"
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/common_init.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using namespace librados;
+using ceph::buffer;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+typedef RadosTest LibRadosMisc;
+typedef RadosTestPP LibRadosMiscPP;
+
+TEST(LibRadosMiscVersion, Version) {
+  int major, minor, extra;
+  rados_version(&major, &minor, &extra);
+}
+
+TEST(LibRadosMiscVersion, VersionPP) {
+  int major, minor, extra;
+  Rados::version(&major, &minor, &extra);
+}
+
+TEST_F(LibRadosMisc, ClusterFSID) {
+  char fsid[37];
+  ASSERT_EQ(-ERANGE, rados_cluster_fsid(cluster, fsid, sizeof(fsid) - 1));
+  ASSERT_EQ(sizeof(fsid) - 1,
+            (size_t)rados_cluster_fsid(cluster, fsid, sizeof(fsid)));
+}
+
+TEST_F(LibRadosMiscPP, WaitOSDMapPP) {
+  ASSERT_EQ(0, cluster.wait_for_latest_osdmap());
+}
+
+static std::string read_key_from_tmap(IoCtx& ioctx, const std::string &obj,
+				      const std::string &key)
+{
+  bufferlist bl;
+  int r = ioctx.read(obj, bl, 0, 0);
+  if (r <= 0) {
+    ostringstream oss;
+    oss << "ioctx.read(" << obj << ", bl, 0, 0) returned " << r;
+    return oss.str();
+  }
+  bufferlist::iterator p = bl.begin();
+  bufferlist header;
+  map m;
+  ::decode(header, p);
+  ::decode(m, p);
+  map::iterator i = m.find(key);
+  if (i == m.end())
+    return "";
+  std::string retstring;
+  ::decode(retstring, i->second);
+  return retstring;
+}
+
+static std::string add_key_to_tmap(IoCtx &ioctx, const std::string &obj,
+	  const std::string &key, const std::string &val)
+{
+  __u8 c = CEPH_OSD_TMAP_SET;
+
+  bufferlist tmbl;
+  ::encode(c, tmbl);
+  ::encode(key, tmbl);
+  bufferlist blbl;
+  ::encode(val, blbl);
+  ::encode(blbl, tmbl);
+  int ret = ioctx.tmap_update(obj, tmbl);
+  if (ret) {
+    ostringstream oss;
+    oss << "ioctx.tmap_update(obj=" << obj << ", key="
+	<< key << ", val=" << val << ") failed with error " << ret;
+    return oss.str();
+  }
+  return "";
+}
+
+static int remove_key_from_tmap(IoCtx &ioctx, const std::string &obj,
+					const std::string &key)
+{
+  __u8 c = CEPH_OSD_TMAP_RM;
+
+  bufferlist tmbl;
+  ::encode(c, tmbl);
+  ::encode(key, tmbl);
+  int ret = ioctx.tmap_update(obj, tmbl);
+  if (ret) {
+    ostringstream oss;
+    oss << "ioctx.tmap_update(obj=" << obj << ", key="
+	<< key << ") failed with error " << ret;
+  }
+  return ret;
+}
+
+TEST_F(LibRadosMiscPP, TmapUpdatePP) {
+  // create tmap
+  {
+    __u8 c = CEPH_OSD_TMAP_CREATE;
+    std::string my_tmap("my_tmap");
+    bufferlist emptybl;
+
+    bufferlist tmbl;
+    ::encode(c, tmbl);
+    ::encode(my_tmap, tmbl);
+    ::encode(emptybl, tmbl);
+    ASSERT_EQ(0, ioctx.tmap_update("foo", tmbl));
+  }
+
+  ASSERT_EQ(string(""), add_key_to_tmap(ioctx, "foo", "key1", "val1"));
+
+  ASSERT_EQ(string(""), add_key_to_tmap(ioctx, "foo", "key2", "val2"));
+
+  // read key1 from the tmap
+  ASSERT_EQ(string("val1"), read_key_from_tmap(ioctx, "foo", "key1"));
+
+  // remove key1 from tmap
+  ASSERT_EQ(0, remove_key_from_tmap(ioctx, "foo", "key1"));
+  ASSERT_EQ(-ENOENT, remove_key_from_tmap(ioctx, "foo", "key1"));
+
+  // key should be removed
+  ASSERT_EQ(string(""), read_key_from_tmap(ioctx, "foo", "key1"));
+}
+
+TEST_F(LibRadosMiscPP, TmapUpdateMisorderedPP) {
+  // create tmap
+  {
+    __u8 c = CEPH_OSD_TMAP_CREATE;
+    std::string my_tmap("my_tmap");
+    bufferlist emptybl;
+
+    bufferlist tmbl;
+    ::encode(c, tmbl);
+    ::encode(my_tmap, tmbl);
+    ::encode(emptybl, tmbl);
+    ASSERT_EQ(0, ioctx.tmap_update("foo", tmbl));
+  }
+
+  // good update
+  {
+    __u8 c = CEPH_OSD_TMAP_SET;
+    bufferlist tmbl;
+    ::encode(c, tmbl);
+    ::encode("a", tmbl);
+    bufferlist blbl;
+    ::encode("old", blbl);
+    ::encode(blbl, tmbl);
+
+    ::encode(c, tmbl);
+    ::encode("b", tmbl);
+    ::encode(blbl, tmbl);
+
+    ::encode(c, tmbl);
+    ::encode("c", tmbl);
+    ::encode(blbl, tmbl);
+
+    ASSERT_EQ(0, ioctx.tmap_update("foo", tmbl));
+  }
+
+  // bad update
+  {
+    __u8 c = CEPH_OSD_TMAP_SET;
+    bufferlist tmbl;
+    ::encode(c, tmbl);
+    ::encode("b", tmbl);
+    bufferlist blbl;
+    ::encode("new", blbl);
+    ::encode(blbl, tmbl);
+
+    ::encode(c, tmbl);
+    ::encode("a", tmbl);
+    ::encode(blbl, tmbl);
+
+    ::encode(c, tmbl);
+    ::encode("c", tmbl);
+    ::encode(blbl, tmbl);
+
+    ASSERT_EQ(0, ioctx.tmap_update("foo", tmbl));
+  }
+
+  // check
+  ASSERT_EQ(string("new"), read_key_from_tmap(ioctx, "foo", "a"));
+  ASSERT_EQ(string("new"), read_key_from_tmap(ioctx, "foo", "b"));
+  ASSERT_EQ(string("new"), read_key_from_tmap(ioctx, "foo", "c"));
+
+  ASSERT_EQ(0, remove_key_from_tmap(ioctx, "foo", "a"));
+  ASSERT_EQ(string(""), read_key_from_tmap(ioctx, "foo", "a"));
+
+  ASSERT_EQ(0, remove_key_from_tmap(ioctx, "foo", "b"));
+  ASSERT_EQ(string(""), read_key_from_tmap(ioctx, "foo", "a"));
+}
+
+TEST_F(LibRadosMiscPP, TmapUpdateMisorderedPutPP) {
+  // create unsorted tmap
+  string h("header");
+  bufferlist bl;
+  ::encode(h, bl);
+  uint32_t n = 3;
+  ::encode(n, bl);
+  ::encode(string("b"), bl);
+  ::encode(string("bval"), bl);
+  ::encode(string("a"), bl);
+  ::encode(string("aval"), bl);
+  ::encode(string("c"), bl);
+  ::encode(string("cval"), bl);
+  bufferlist orig = bl;  // tmap_put steals bl content
+  ASSERT_EQ(0, ioctx.tmap_put("foo", bl));
+
+  // check
+  bufferlist newbl;
+  ioctx.read("foo", newbl, orig.length(), 0);
+  ASSERT_EQ(orig.contents_equal(newbl), false);
+}
+
+TEST_F(LibRadosMiscPP, Tmap2OmapPP) {
+  // create tmap
+  bufferlist hdr;
+  hdr.append("header");
+  map omap;
+  omap["1"].append("a");
+  omap["2"].append("b");
+  omap["3"].append("c");
+  {
+    bufferlist bl;
+    ::encode(hdr, bl);
+    ::encode(omap, bl);
+    ASSERT_EQ(0, ioctx.tmap_put("foo", bl));
+  }
+
+  // convert tmap to omap
+  ASSERT_EQ(0, ioctx.tmap_to_omap("foo", false));
+
+  // if tmap was truncated ?
+  {
+    uint64_t size;
+    time_t mtime;
+    ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+    ASSERT_EQ(0U, size);
+  }
+
+  // if 'nullok' works
+  ASSERT_EQ(0, ioctx.tmap_to_omap("foo", true));
+  ASSERT_LE(ioctx.tmap_to_omap("foo", false), 0);
+
+  {
+    // read omap
+    bufferlist got;
+    map m;
+    ObjectReadOperation o;
+    o.omap_get_header(&got, NULL);
+    o.omap_get_vals("", 1024, &m, NULL);
+    ASSERT_EQ(0, ioctx.operate("foo", &o, NULL));
+
+    // compare header
+    ASSERT_TRUE(hdr.contents_equal(got));
+
+    // compare values
+    ASSERT_EQ(omap.size(), m.size());
+    bool same = true;
+    for (map::iterator p = omap.begin(); p != omap.end(); ++p) {
+      map::iterator q = m.find(p->first);
+      if (q == m.end() || !p->second.contents_equal(q->second)) {
+	same = false;
+	break;
+      }
+    }
+    ASSERT_TRUE(same);
+  }
+}
+
+TEST_F(LibRadosMisc, Exec) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  char buf2[512];
+  int res = rados_exec(ioctx, "foo", "rbd", "get_all_features",
+			  NULL, 0, buf2, sizeof(buf2));
+  ASSERT_GT(res, 0);
+  bufferlist bl;
+  bl.append(buf2, res);
+  bufferlist::iterator iter = bl.begin();
+  uint64_t all_features;
+  ::decode(all_features, iter);
+  ASSERT_EQ(all_features, (uint64_t)RBD_FEATURES_ALL);
+}
+
+TEST_F(LibRadosMiscPP, ExecPP) {
+  bufferlist bl;
+  ASSERT_EQ(0, ioctx.write("foo", bl, 0, 0));
+  bufferlist bl2, out;
+  int r = ioctx.exec("foo", "rbd", "get_all_features", bl2, out);
+  ASSERT_EQ(0, r);
+  bufferlist::iterator iter = out.begin();
+  uint64_t all_features;
+  ::decode(all_features, iter);
+  ASSERT_EQ(all_features, (uint64_t)RBD_FEATURES_ALL);
+}
+
+TEST_F(LibRadosMiscPP, Operate1PP) {
+  ObjectWriteOperation o;
+  {
+    bufferlist bl;
+    o.write(0, bl);
+  }
+  std::string val1("val1");
+  {
+    bufferlist bl;
+    bl.append(val1.c_str(), val1.size() + 1);
+    o.setxattr("key1", bl);
+    o.omap_clear(); // shouldn't affect attrs!
+  }
+  ASSERT_EQ(0, ioctx.operate("foo", &o));
+
+  ObjectWriteOperation empty;
+  ASSERT_EQ(0, ioctx.operate("foo", &empty));
+
+  {
+    bufferlist bl;
+    ASSERT_GT(ioctx.getxattr("foo", "key1", bl), 0);
+    ASSERT_EQ(0, strcmp(bl.c_str(), val1.c_str()));
+  }
+  ObjectWriteOperation o2;
+  {
+    bufferlist bl;
+    bl.append(val1);
+    o2.cmpxattr("key1", CEPH_OSD_CMPXATTR_OP_EQ, bl);
+    o2.rmxattr("key1");
+  }
+  ASSERT_EQ(-ECANCELED, ioctx.operate("foo", &o2));
+  ObjectWriteOperation o3;
+  {
+    bufferlist bl;
+    bl.append(val1);
+    o3.cmpxattr("key1", CEPH_OSD_CMPXATTR_OP_EQ, bl);
+  }
+  ASSERT_EQ(-ECANCELED, ioctx.operate("foo", &o3));
+}
+
+TEST_F(LibRadosMiscPP, Operate2PP) {
+  ObjectWriteOperation o;
+  {
+    bufferlist bl;
+    bl.append("abcdefg");
+    o.write(0, bl);
+  }
+  std::string val1("val1");
+  {
+    bufferlist bl;
+    bl.append(val1.c_str(), val1.size() + 1);
+    o.setxattr("key1", bl);
+    o.truncate(0);
+  }
+  ASSERT_EQ(0, ioctx.operate("foo", &o));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(0U, size);
+}
+
+TEST_F(LibRadosMiscPP, BigObjectPP) {
+  bufferlist bl;
+  bl.append("abcdefg");
+  ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+
+  {
+    ObjectWriteOperation o;
+    o.truncate(500000000000ull);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+  {
+    ObjectWriteOperation o;
+    o.zero(500000000000ull, 1);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+  {
+    ObjectWriteOperation o;
+    o.zero(1, 500000000000ull);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+  {
+    ObjectWriteOperation o;
+    o.zero(500000000000ull, 500000000000ull);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+
+#ifdef __LP64__
+  // this test only works on 64-bit platforms
+  ASSERT_EQ(-EFBIG, ioctx.write("foo", bl, bl.length(), 500000000000ull));
+#endif
+}
+
+void set_completion_complete(rados_completion_t cb, void *arg)
+{
+  bool *my_aio_complete = (bool*)arg;
+  *my_aio_complete = true;
+}
+
+TEST_F(LibRadosMiscPP, AioOperatePP) {
+  bool my_aio_complete = false;
+  AioCompletion *my_completion = cluster.aio_create_completion(
+	  (void*)&my_aio_complete, set_completion_complete, NULL);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+
+  ObjectWriteOperation o;
+  {
+    bufferlist bl;
+    o.write(0, bl);
+  }
+  std::string val1("val1");
+  {
+    bufferlist bl;
+    bl.append(val1.c_str(), val1.size() + 1);
+    o.setxattr("key1", bl);
+    bufferlist bl2;
+    char buf2[1024];
+    memset(buf2, 0xdd, sizeof(buf2));
+    bl2.append(buf2, sizeof(buf2));
+    o.append(bl2);
+  }
+  ASSERT_EQ(0, ioctx.aio_operate("foo", my_completion, &o));
+  ASSERT_EQ(0, my_completion->wait_for_complete_and_cb());
+  ASSERT_EQ(my_aio_complete, true);
+
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(1024U, size);
+}
+
+TEST_F(LibRadosMiscPP, CloneRangePP) {
+  char buf[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ioctx.locator_set_key("foo");
+  ASSERT_EQ(0, ioctx.clone_range("bar", 0, "foo", 0, sizeof(buf)));
+  bufferlist bl2;
+  ASSERT_EQ(sizeof(buf), (size_t)ioctx.read("bar", bl2, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST_F(LibRadosMisc, CloneRange) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "src", buf, sizeof(buf), 0));
+  rados_ioctx_locator_set_key(ioctx, "src");
+  ASSERT_EQ(0, rados_clone_range(ioctx, "dst", 0, "src", 0, sizeof(buf)));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "dst", buf2, sizeof(buf2), 0));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+}
+
+TEST_F(LibRadosMiscPP, AssertExistsPP) {
+  char buf[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  ObjectWriteOperation op;
+  op.assert_exists();
+  op.write(0, bl);
+  ASSERT_EQ(-ENOENT, ioctx.operate("asdffoo", &op));
+  ASSERT_EQ(0, ioctx.create("asdffoo", true));
+  ASSERT_EQ(0, ioctx.operate("asdffoo", &op));
+  ASSERT_EQ(-EEXIST, ioctx.create("asdffoo", true));
+}
+
+TEST_F(LibRadosMiscPP, BigAttrPP) {
+  char buf[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  ASSERT_EQ(0, ioctx.create("foo", true));
+
+  bufferlist got;
+
+  cout << "osd_max_attr_size = " << g_conf->osd_max_attr_size << std::endl;
+  if (g_conf->osd_max_attr_size) {
+    bl.clear();
+    got.clear();
+    bl.append(buffer::create(g_conf->osd_max_attr_size));
+    ASSERT_EQ(0, ioctx.setxattr("foo", "one", bl));
+    ASSERT_EQ((int)bl.length(), ioctx.getxattr("foo", "one", got));
+    ASSERT_TRUE(bl.contents_equal(got));
+
+    bl.clear();
+    bl.append(buffer::create(g_conf->osd_max_attr_size+1));
+    ASSERT_EQ(-EFBIG, ioctx.setxattr("foo", "one", bl));
+  } else {
+    cout << "osd_max_attr_size == 0; skipping test" << std::endl;
+  }
+
+  for (int i=0; i<1000; i++) {
+    bl.clear();
+    got.clear();
+    bl.append(buffer::create(MIN(g_conf->osd_max_attr_size, 1024)));
+    char n[10];
+    snprintf(n, sizeof(n), "a%d", i);
+    ASSERT_EQ(0, ioctx.setxattr("foo", n, bl));
+    ASSERT_EQ((int)bl.length(), ioctx.getxattr("foo", n, got));
+    ASSERT_TRUE(bl.contents_equal(got));
+  }
+}
+
+TEST_F(LibRadosMiscPP, CopyPP) {
+  bufferlist bl, x;
+  bl.append("hi there");
+  x.append("bar");
+
+  // small object
+  bufferlist blc = bl;
+  bufferlist xc = x;
+  ASSERT_EQ(0, ioctx.write_full("foo", blc));
+  ASSERT_EQ(0, ioctx.setxattr("foo", "myattr", xc));
+
+  version_t uv = ioctx.get_last_version();
+  {
+    // pass future version
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, uv + 1);
+    ASSERT_EQ(-EOVERFLOW, ioctx.operate("foo.copy", &op));
+  }
+  {
+    // pass old version
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, uv - 1);
+    ASSERT_EQ(-ERANGE, ioctx.operate("foo.copy", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, uv);
+    ASSERT_EQ(0, ioctx.operate("foo.copy", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("foo.copy", bl2, 10000, 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+
+  // small object without a version
+  {
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, 0);
+    ASSERT_EQ(0, ioctx.operate("foo.copy2", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("foo.copy2", bl2, 10000, 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy2", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+
+  // do a big object
+  bl.append(buffer::create(g_conf->osd_copyfrom_max_chunk * 3));
+  bl.zero();
+  bl.append("tail");
+  blc = bl;
+  xc = x;
+  ASSERT_EQ(0, ioctx.write_full("big", blc));
+  ASSERT_EQ(0, ioctx.setxattr("big", "myattr", xc));
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("big", ioctx, ioctx.get_last_version());
+    ASSERT_EQ(0, ioctx.operate("big.copy", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("big.copy", bl2, bl.length(), 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("big", ioctx, 0);
+    ASSERT_EQ(0, ioctx.operate("big.copy2", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("big.copy2", bl2, bl.length(), 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy2", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+}
+
+int main(int argc, char **argv)
+{
+  ::testing::InitGoogleTest(&argc, argv);
+
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/librados/pool.cc b/ceph/src/test/librados/pool.cc
new file mode 100644
index 00000000..04286fcf
--- /dev/null
+++ b/ceph/src/test/librados/pool.cc
@@ -0,0 +1,106 @@
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+
+#include "gtest/gtest.h"
+#include 
+#include 
+
+#define POOL_LIST_BUF_SZ 32768
+
+TEST(LibRadosPools, PoolList) {
+  char pool_list_buf[POOL_LIST_BUF_SZ];
+  char *buf = pool_list_buf;
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_LT(rados_pool_list(cluster, buf, POOL_LIST_BUF_SZ), POOL_LIST_BUF_SZ);
+
+  bool found_pool = false;
+  while (buf[0] != '\0') {
+    if ((found_pool == false) && (strcmp(buf, pool_name.c_str()) == 0)) {
+      found_pool = true;
+    }
+    buf += strlen(buf) + 1;
+  }
+  ASSERT_EQ(found_pool, true);
+
+  // make sure we honor the buffer size limit
+  buf = pool_list_buf;
+  memset(buf, 0, POOL_LIST_BUF_SZ);
+  ASSERT_LT(rados_pool_list(cluster, buf, 20), POOL_LIST_BUF_SZ);
+  ASSERT_NE(0, buf[0]);  // include at least one pool name
+  ASSERT_EQ(0, buf[20]);  // but don't touch the stopping point
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+int64_t rados_pool_lookup(rados_t cluster, const char *pool_name);
+
+TEST(LibRadosPools, PoolLookup) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_LT(0, rados_pool_lookup(cluster, pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolLookup2) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  int64_t pool_id = rados_pool_lookup(cluster, pool_name.c_str());
+  ASSERT_GT(pool_id, 0);
+  rados_ioctx_t ioctx;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  int64_t pool_id2 = rados_ioctx_get_id(ioctx);
+  ASSERT_EQ(pool_id, pool_id2);
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolDelete) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_EQ(0, rados_pool_delete(cluster, pool_name.c_str()));
+  ASSERT_GT(0, rados_pool_lookup(cluster, pool_name.c_str()));
+  ASSERT_EQ(0, rados_pool_create(cluster, pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolCreateDelete) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+
+  std::string n = pool_name + "abc123";
+  ASSERT_EQ(0, rados_pool_create(cluster, n.c_str()));
+  ASSERT_EQ(-EEXIST, rados_pool_create(cluster, n.c_str()));
+  ASSERT_EQ(0, rados_pool_delete(cluster, n.c_str()));
+  ASSERT_EQ(-ENOENT, rados_pool_delete(cluster, n.c_str()));
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolCreateWithCrushRule) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+
+  std::string pool2_name = get_temp_pool_name();
+  ASSERT_EQ(0, rados_pool_create_with_crush_rule(cluster,
+			    pool2_name.c_str(), 0));
+  ASSERT_EQ(0, rados_pool_delete(cluster, pool2_name.c_str()));
+
+  std::string pool3_name = get_temp_pool_name();
+  ASSERT_EQ(0, rados_pool_create_with_all(cluster, pool3_name.c_str(),
+					  456ull, 0));
+  rados_ioctx_t ioctx;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool3_name.c_str(), &ioctx));
+  uint64_t auid;
+  ASSERT_EQ(0, rados_ioctx_pool_get_auid(ioctx, &auid));
+  ASSERT_EQ(456ull, auid);
+  ASSERT_EQ(0, rados_pool_delete(cluster, pool3_name.c_str()));
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
diff --git a/ceph/src/test/librados/snapshots.cc b/ceph/src/test/librados/snapshots.cc
new file mode 100644
index 00000000..020af111
--- /dev/null
+++ b/ceph/src/test/librados/snapshots.cc
@@ -0,0 +1,798 @@
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include 
+#include 
+#include "gtest/gtest.h"
+#include 
+
+using namespace librados;
+using std::string;
+
+typedef RadosTest LibRadosSnapshots;
+typedef RadosTest LibRadosSnapshotsSelfManaged;
+typedef RadosTestPP LibRadosSnapshotsPP;
+typedef RadosTestPP LibRadosSnapshotsSelfManagedPP;
+typedef RadosTestEC LibRadosSnapshotsEC;
+typedef RadosTestEC LibRadosSnapshotsSelfManagedEC;
+typedef RadosTestECPP LibRadosSnapshotsECPP;
+typedef RadosTestECPP LibRadosSnapshotsSelfManagedECPP;
+
+const int bufsize = 128;
+
+TEST_F(LibRadosSnapshots, SnapList) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t snaps[10];
+  EXPECT_EQ(1, rados_ioctx_snap_list(ioctx, snaps,
+				     sizeof(snaps) / sizeof(snaps[0])));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshotsPP, SnapListPP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  std::vector snaps;
+  EXPECT_EQ(0, ioctx.snap_list(&snaps));
+  EXPECT_EQ(1U, snaps.size());
+  snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+}
+
+TEST_F(LibRadosSnapshots, SnapRemove) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  ASSERT_EQ(-EEXIST, rados_ioctx_snap_create(ioctx, "snap1"));
+  ASSERT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+  ASSERT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshotsPP, SnapRemovePP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  ASSERT_EQ(0, ioctx.snap_remove("snap1"));
+  ASSERT_EQ(-ENOENT, ioctx.snap_lookup("snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshots, Rollback) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  EXPECT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+  EXPECT_EQ(0, rados_ioctx_snap_rollback(ioctx, "foo", "snap1"));
+  char buf3[sizeof(buf)];
+  EXPECT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  EXPECT_EQ(0, memcmp(buf, buf3, sizeof(buf)));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshotsPP, RollbackPP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  EXPECT_EQ(0, ioctx.write_full("foo", bl2));
+  EXPECT_EQ(0, ioctx.snap_rollback("foo", "snap1"));
+  bufferlist bl3;
+  EXPECT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  EXPECT_EQ(0, memcmp(buf, bl3.c_str(), sizeof(buf)));
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+}
+
+TEST_F(LibRadosSnapshots, SnapGetName) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snapbar", &rid));
+  char name[128];
+  memset(name, 0, sizeof(name));
+  EXPECT_EQ(0, rados_ioctx_snap_get_name(ioctx, rid, name, sizeof(name)));
+  time_t snaptime;
+  EXPECT_EQ(0, rados_ioctx_snap_get_stamp(ioctx, rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name, "snapfoo"));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsPP, SnapGetNamePP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, ioctx.snap_lookup("snapbar", &rid));
+  std::string name;
+  EXPECT_EQ(0, ioctx.snap_get_name(rid, &name));
+  time_t snaptime;
+  EXPECT_EQ(0, ioctx.snap_get_stamp(rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name.c_str(), "snapfoo"));
+  EXPECT_EQ(0, ioctx.snap_remove("snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManaged, Snap) {
+  std::vector my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]-1);
+  char buf3[sizeof(buf)];
+  ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]);
+  ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManaged, Rollback) {
+  std::vector my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+  rados_ioctx_selfmanaged_snap_rollback(ioctx, "foo", my_snaps[1]);
+  char buf3[sizeof(buf)];
+  ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, SnapPP) {
+  std::vector my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+
+  ioctx.snap_set_read(my_snaps[1]);
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+  ASSERT_EQ(0, ioctx.remove("foo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, RollbackPP) {
+  std::vector my_snaps;
+  IoCtx readioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+  readioctx.set_namespace(ns);
+  readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  //Write 3 consecutive buffers
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*2));
+
+  snap_set_t ss;
+
+  snap_t head = SNAP_HEAD;
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(1u, ss.clones.size());
+  ASSERT_EQ(head, ss.clones[0].cloneid);
+  ASSERT_EQ(0u, ss.clones[0].snaps.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap.size());
+  ASSERT_EQ(384u, ss.clones[0].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  //Change the middle buffer
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize));
+  //Add another after
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*3));
+
+  ASSERT_EQ(-EINVAL, ioctx.list_snaps("foo", &ss));
+  ObjectReadOperation o;
+  o.list_snaps(&ss, NULL);
+  ASSERT_EQ(-EINVAL, ioctx.operate("foo", &o, NULL));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(2u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(2u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[0].overlap[1].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[1].second);
+  ASSERT_EQ(384u, ss.clones[0].size);
+  ASSERT_EQ(head, ss.clones[1].cloneid);
+  ASSERT_EQ(0u, ss.clones[1].snaps.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap.size());
+  ASSERT_EQ(512u, ss.clones[1].size);
+
+  ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]);
+
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), bufsize));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), bufsize*2));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ((int)0, ioctx.read("foo", bl3, sizeof(buf), bufsize*3));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  readioctx.close();
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, SnapOverlapPP) {
+  std::vector my_snaps;
+  IoCtx readioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+  readioctx.set_namespace(ns);
+  readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*2));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*4));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*6));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*8));
+
+  snap_set_t ss;
+  snap_t head = SNAP_HEAD;
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(1u, ss.clones.size());
+  ASSERT_EQ(head, ss.clones[0].cloneid);
+  ASSERT_EQ(0u, ss.clones[0].snaps.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap.size());
+  ASSERT_EQ(1152u, ss.clones[0].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*1));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*3));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*5));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*7));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*9));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(2u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(5u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[0].overlap[1].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[1].second);
+  ASSERT_EQ(512u, ss.clones[0].overlap[2].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[2].second);
+  ASSERT_EQ(768u, ss.clones[0].overlap[3].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[3].second);
+  ASSERT_EQ(1024u, ss.clones[0].overlap[4].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[4].second);
+  ASSERT_EQ(1152u, ss.clones[0].size);
+  ASSERT_EQ(head, ss.clones[1].cloneid);
+  ASSERT_EQ(0u, ss.clones[1].snaps.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap.size());
+  ASSERT_EQ(1280u, ss.clones[1].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+
+  char buf3[sizeof(buf)];
+  memset(buf3, 0xee, sizeof(buf3));
+  bufferlist bl4;
+  bl4.append(buf3, sizeof(buf3));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*1));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*4));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*5));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*8));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(3u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(5u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[0].overlap[1].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[1].second);
+  ASSERT_EQ(512u, ss.clones[0].overlap[2].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[2].second);
+  ASSERT_EQ(768u, ss.clones[0].overlap[3].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[3].second);
+  ASSERT_EQ(1024u, ss.clones[0].overlap[4].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[4].second);
+  ASSERT_EQ(1152u, ss.clones[0].size);
+
+  ASSERT_EQ(my_snaps[2], ss.clones[1].cloneid);
+  ASSERT_EQ(1u, ss.clones[1].snaps.size());
+  ASSERT_EQ(my_snaps[2], ss.clones[1].snaps[0]);
+  ASSERT_EQ(4u, ss.clones[1].overlap.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[1].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[1].overlap[1].first);
+  ASSERT_EQ(256u, ss.clones[1].overlap[1].second);
+  ASSERT_EQ(768u, ss.clones[1].overlap[2].first);
+  ASSERT_EQ(256u, ss.clones[1].overlap[2].second);
+  ASSERT_EQ(1152u, ss.clones[1].overlap[3].first);
+  ASSERT_EQ(128u, ss.clones[1].overlap[3].second);
+  ASSERT_EQ(1280u, ss.clones[1].size);
+
+  ASSERT_EQ(head, ss.clones[2].cloneid);
+  ASSERT_EQ(0u, ss.clones[2].snaps.size());
+  ASSERT_EQ(0u, ss.clones[2].overlap.size());
+  ASSERT_EQ(1280u, ss.clones[2].size);
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  readioctx.close();
+}
+
+// EC testing
+TEST_F(LibRadosSnapshotsEC, SnapList) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t snaps[10];
+  EXPECT_EQ(1, rados_ioctx_snap_list(ioctx, snaps,
+				     sizeof(snaps) / sizeof(snaps[0])));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshotsECPP, SnapListPP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  std::vector snaps;
+  EXPECT_EQ(0, ioctx.snap_list(&snaps));
+  EXPECT_EQ(1U, snaps.size());
+  snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+}
+
+TEST_F(LibRadosSnapshotsEC, SnapRemove) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  ASSERT_EQ(-EEXIST, rados_ioctx_snap_create(ioctx, "snap1"));
+  ASSERT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+  ASSERT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshotsECPP, SnapRemovePP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  ASSERT_EQ(0, ioctx.snap_remove("snap1"));
+  ASSERT_EQ(-ENOENT, ioctx.snap_lookup("snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshotsEC, Rollback) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  EXPECT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+  EXPECT_EQ(0, rados_ioctx_snap_rollback(ioctx, "foo", "snap1"));
+  char buf3[sizeof(buf)];
+  EXPECT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  EXPECT_EQ(0, memcmp(buf, buf3, sizeof(buf)));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshotsECPP, RollbackPP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  EXPECT_EQ(0, ioctx.write_full("foo", bl2));
+  EXPECT_EQ(0, ioctx.snap_rollback("foo", "snap1"));
+  bufferlist bl3;
+  EXPECT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  EXPECT_EQ(0, memcmp(buf, bl3.c_str(), sizeof(buf)));
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+}
+
+TEST_F(LibRadosSnapshotsEC, SnapGetName) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snapbar", &rid));
+  char name[128];
+  memset(name, 0, sizeof(name));
+  EXPECT_EQ(0, rados_ioctx_snap_get_name(ioctx, rid, name, sizeof(name)));
+  time_t snaptime;
+  EXPECT_EQ(0, rados_ioctx_snap_get_stamp(ioctx, rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name, "snapfoo"));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsECPP, SnapGetNamePP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, ioctx.snap_lookup("snapbar", &rid));
+  std::string name;
+  EXPECT_EQ(0, ioctx.snap_get_name(rid, &name));
+  time_t snaptime;
+  EXPECT_EQ(0, ioctx.snap_get_stamp(rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name.c_str(), "snapfoo"));
+  EXPECT_EQ(0, ioctx.snap_remove("snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedEC, Snap) {
+  std::vector my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, bsize, 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, bsize, bsize));
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]-1);
+  char *buf3 = (char *)new char[bsize*2];
+  ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf3, bsize*2, 0));
+
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]);
+  ASSERT_EQ(bsize, rados_read(ioctx, "foo", buf3, bsize*2, 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, bsize));
+
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedEC, Rollback) {
+  std::vector my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, bsize, 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, bsize, bsize));
+  rados_ioctx_selfmanaged_snap_rollback(ioctx, "foo", my_snaps[1]);
+  char *buf3 = (char *)new char[bsize*2];
+  ASSERT_EQ(bsize, rados_read(ioctx, "foo", buf3, bsize*2, 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, bsize));
+
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedECPP, SnapPP) {
+  std::vector my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+  bufferlist bl2;
+  bl2.append(buf2, bsize);
+  // Add another aligned buffer
+  ASSERT_EQ(0, ioctx.write("foo", bl2, bsize, bsize));
+
+  ioctx.snap_set_read(my_snaps[1]);
+  bufferlist bl3;
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize*3, 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+  ASSERT_EQ(0, ioctx.remove("foo"));
+  delete[] buf;
+  delete[] buf2;
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedECPP, RollbackPP) {
+  std::vector my_snaps;
+  IoCtx readioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+  readioctx.set_namespace(ns);
+  readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  //Write 3 consecutive buffers
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, 0));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, bsize));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, bsize*2));
+
+  snap_set_t ss;
+
+  snap_t head = SNAP_HEAD;
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(1u, ss.clones.size());
+  ASSERT_EQ(head, ss.clones[0].cloneid);
+  ASSERT_EQ(0u, ss.clones[0].snaps.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap.size());
+  ASSERT_EQ((unsigned)(bsize*3), ss.clones[0].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+  bufferlist bl2;
+  bl2.append(buf2, bsize);
+  //Change the middle buffer
+  //ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize));
+  //Add another after
+  ASSERT_EQ(0, ioctx.write("foo", bl2, bsize, bsize*3));
+
+  ASSERT_EQ(-EINVAL, ioctx.list_snaps("foo", &ss));
+  ObjectReadOperation o;
+  o.list_snaps(&ss, NULL);
+  ASSERT_EQ(-EINVAL, ioctx.operate("foo", &o, NULL));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(2u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(1u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ((unsigned)bsize*3, ss.clones[0].overlap[0].second);
+  ASSERT_EQ((unsigned)bsize*3, ss.clones[0].size);
+  ASSERT_EQ(head, ss.clones[1].cloneid);
+  ASSERT_EQ(0u, ss.clones[1].snaps.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap.size());
+  ASSERT_EQ((unsigned)bsize*4, ss.clones[1].size);
+
+  ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]);
+
+  bufferlist bl3;
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize, 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize, bsize));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize, bsize*2));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(0, ioctx.read("foo", bl3, bsize, bsize*3));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  readioctx.close();
+
+  delete[] buf;
+  delete[] buf2;
+}
+
diff --git a/ceph/src/test/librados/stat.cc b/ceph/src/test/librados/stat.cc
new file mode 100644
index 00000000..5757b30e
--- /dev/null
+++ b/ceph/src/test/librados/stat.cc
@@ -0,0 +1,247 @@
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include 
+#include 
+#include "gtest/gtest.h"
+
+using namespace librados;
+
+typedef RadosTest LibRadosStat;
+typedef RadosTestPP LibRadosStatPP;
+typedef RadosTestEC LibRadosStatEC;
+typedef RadosTestECPP LibRadosStatECPP;
+
+TEST_F(LibRadosStat, Stat) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStatPP, StatPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStat, StatNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xcc, sizeof(buf2));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+
+  uint64_t size;
+  time_t mtime;
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "foo2", &size, &mtime));
+}
+
+TEST_F(LibRadosStatPP, StatPPNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo2", bl, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+
+  uint64_t size;
+  time_t mtime;
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, ioctx.stat("foo2", &size, &mtime));
+}
+
+TEST_F(LibRadosStat, ClusterStat) {
+  struct rados_cluster_stat_t result;
+  ASSERT_EQ(0, rados_cluster_stat(cluster, &result));
+}
+
+TEST_F(LibRadosStatPP, ClusterStatPP) {
+  cluster_stat_t cstat;
+  ASSERT_EQ(0, cluster.cluster_stat(cstat));
+}
+
+TEST_F(LibRadosStat, PoolStat) {
+  char buf[128];
+  char actual_pool_name[80];
+  unsigned l = rados_ioctx_get_pool_name(ioctx, actual_pool_name, sizeof(actual_pool_name));
+  ASSERT_EQ(strlen(actual_pool_name), l);
+  ASSERT_EQ(0, strcmp(actual_pool_name, pool_name.c_str()));
+  memset(buf, 0xff, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  struct rados_pool_stat_t stats;
+  memset(&stats, 0, sizeof(stats));
+  ASSERT_EQ(0, rados_ioctx_pool_stat(ioctx, &stats));
+}
+
+TEST_F(LibRadosStatPP, PoolStatPP) {
+  std::string n = ioctx.get_pool_name();
+  ASSERT_EQ(n, pool_name);
+  char buf[128];
+  memset(buf, 0xff, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  std::list v;
+  std::map stats;
+  ASSERT_EQ(0, cluster.get_pool_stats(v, stats));
+}
+
+TEST_F(LibRadosStatEC, Stat) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStatECPP, StatPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStatEC, StatNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xcc, sizeof(buf2));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+
+  uint64_t size;
+  time_t mtime;
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "foo2", &size, &mtime));
+}
+
+TEST_F(LibRadosStatECPP, StatPPNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo2", bl, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+
+  uint64_t size;
+  time_t mtime;
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, ioctx.stat("foo2", &size, &mtime));
+}
+
+TEST_F(LibRadosStatEC, ClusterStat) {
+  struct rados_cluster_stat_t result;
+  ASSERT_EQ(0, rados_cluster_stat(cluster, &result));
+}
+
+TEST_F(LibRadosStatECPP, ClusterStatPP) {
+  cluster_stat_t cstat;
+  ASSERT_EQ(0, cluster.cluster_stat(cstat));
+}
+
+TEST_F(LibRadosStatEC, PoolStat) {
+  char buf[128];
+  char actual_pool_name[80];
+  unsigned l = rados_ioctx_get_pool_name(ioctx, actual_pool_name, sizeof(actual_pool_name));
+  ASSERT_EQ(strlen(actual_pool_name), l);
+  ASSERT_EQ(0, strcmp(actual_pool_name, pool_name.c_str()));
+  memset(buf, 0xff, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  struct rados_pool_stat_t stats;
+  memset(&stats, 0, sizeof(stats));
+  ASSERT_EQ(0, rados_ioctx_pool_stat(ioctx, &stats));
+}
+
+TEST_F(LibRadosStatECPP, PoolStatPP) {
+  std::string n = ioctx.get_pool_name();
+  ASSERT_EQ(n, pool_name);
+  char buf[128];
+  memset(buf, 0xff, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  std::list v;
+  std::map stats;
+  ASSERT_EQ(0, cluster.get_pool_stats(v, stats));
+}
diff --git a/ceph/src/test/librados/test.cc b/ceph/src/test/librados/test.cc
new file mode 100644
index 00000000..f8a92a2f
--- /dev/null
+++ b/ceph/src/test/librados/test.cc
@@ -0,0 +1,257 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace librados;
+
+std::string get_temp_pool_name()
+{
+  char hostname[80];
+  char out[80];
+  memset(hostname, 0, sizeof(hostname));
+  memset(out, 0, sizeof(out));
+  gethostname(hostname, sizeof(hostname)-1);
+  static int num = 1;
+  sprintf(out, "%s-%d-%d", hostname, getpid(), num);
+  num++;
+  std::string prefix("test-rados-api-");
+  prefix += out;
+  return prefix;
+}
+
+std::string create_one_pool(const std::string &pool_name, rados_t *cluster)
+{
+  std::string err = connect_cluster(cluster);
+  if (err.length())
+    return err;
+  int ret = rados_pool_create(*cluster, pool_name.c_str());
+  if (ret) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "rados_pool_create(" << pool_name << ") failed with error " << ret;
+    return oss.str();
+  }
+  return "";
+}
+
+std::string create_one_ec_pool(const std::string &pool_name, rados_t *cluster)
+{
+  std::string err = connect_cluster(cluster);
+  if (err.length())
+    return err;
+
+  char *cmd[2];
+
+  cmd[1] = NULL;
+
+  cmd[0] = (char *)"{\"prefix\": \"osd erasure-code-profile set\", \"name\": \"testprofile\", \"profile\": [ \"k=2\", \"m=1\", \"ruleset-failure-domain=osd\"]}";
+  int ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL, NULL, NULL, NULL);
+  if (ret) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "rados_mon_command erasure-code-profile set name:testprofile failed with error " << ret;
+    return oss.str();
+  }
+    
+  std::string cmdstr = "{\"prefix\": \"osd pool create\", \"pool\": \"" +
+     pool_name + "\", \"pool_type\":\"erasure\", \"pg_num\":8, \"pgp_num\":8, \"erasure_code_profile\":\"testprofile\"}";
+  cmd[0] = (char *)cmdstr.c_str();
+  ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0);
+  if (ret) {
+    std::ostringstream oss;
+
+    cmd[0] = (char *)"{\"prefix\": \"osd erasure-code-profile rm\", \"name\": \"testprofile\"}";
+    int ret2 = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0);
+    if (ret2)
+      oss << "rados_mon_command osd erasure-code-profile rm name:testprofile failed with error " << ret2 << std::endl;
+
+    rados_shutdown(*cluster);
+    oss << "rados_mon_command erasure-code-profile set name:testprofile failed with error " << ret;
+    return oss.str();
+  }
+
+  rados_wait_for_latest_osdmap(*cluster);
+  return "";
+}
+
+std::string create_one_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+  std::string err = connect_cluster_pp(cluster);
+  if (err.length())
+    return err;
+  int ret = cluster.pool_create(pool_name.c_str());
+  if (ret) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "cluster.pool_create(" << pool_name << ") failed with error " << ret;
+    return oss.str();
+  }
+  return "";
+}
+
+std::string create_one_ec_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+  std::string err = connect_cluster_pp(cluster);
+  if (err.length())
+    return err;
+
+  bufferlist inbl;
+  int ret = cluster.mon_command(
+    "{\"prefix\": \"osd erasure-code-profile set\", \"name\": \"testprofile\", \"profile\": [ \"k=2\", \"m=1\", \"ruleset-failure-domain=osd\"]}",
+    inbl, NULL, NULL);
+  if (ret) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "mon_command erasure-code-profile set name:testprofile failed with error " << ret;
+    return oss.str();
+  }
+    
+  ret = cluster.mon_command(
+    "{\"prefix\": \"osd pool create\", \"pool\": \"" + pool_name + "\", \"pool_type\":\"erasure\", \"pg_num\":8, \"pgp_num\":8, \"erasure_code_profile\":\"testprofile\"}",
+    inbl, NULL, NULL);
+  if (ret) {
+    std::ostringstream oss;
+    bufferlist inbl;
+    int ret2 = cluster.mon_command(
+      "{\"prefix\": \"osd erasure-code-profile rm\", \"name\": \"testprofile\"}",
+      inbl, NULL, NULL);
+    if (ret2)
+      oss << "mon_command osd erasure-code-profile rm name:testprofile failed with error " << ret2 << std::endl;
+
+    cluster.shutdown();
+    oss << "mon_command osd pool create pool:" << pool_name << " pool_type:erasure failed with error " << ret;
+    return oss.str();
+  }
+
+  cluster.wait_for_latest_osdmap();
+  return "";
+}
+
+std::string connect_cluster(rados_t *cluster)
+{
+  char *id = getenv("CEPH_CLIENT_ID");
+  if (id) std::cerr << "Client id is: " << id << std::endl;
+
+  int ret;
+  ret = rados_create(cluster, NULL);
+  if (ret) {
+    std::ostringstream oss;
+    oss << "rados_create failed with error " << ret;
+    return oss.str();
+  }
+  ret = rados_conf_read_file(*cluster, NULL);
+  if (ret) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "rados_conf_read_file failed with error " << ret;
+    return oss.str();
+  }
+  rados_conf_parse_env(*cluster, NULL);
+  ret = rados_connect(*cluster);
+  if (ret) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "rados_connect failed with error " << ret;
+    return oss.str();
+  }
+  return "";
+}
+
+std::string connect_cluster_pp(Rados &cluster)
+{
+  char *id = getenv("CEPH_CLIENT_ID");
+  if (id) std::cerr << "Client id is: " << id << std::endl;
+
+  int ret;
+  ret = cluster.init(id);
+  if (ret) {
+    std::ostringstream oss;
+    oss << "cluster.init failed with error " << ret;
+    return oss.str();
+  }
+  ret = cluster.conf_read_file(NULL);
+  if (ret) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "cluster.conf_read_file failed with error " << ret;
+    return oss.str();
+  }
+  cluster.conf_parse_env(NULL);
+  ret = cluster.connect();
+  if (ret) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "cluster.connect failed with error " << ret;
+    return oss.str();
+  }
+  return "";
+}
+
+int destroy_one_pool(const std::string &pool_name, rados_t *cluster)
+{
+  int ret = rados_pool_delete(*cluster, pool_name.c_str());
+  if (ret) {
+    rados_shutdown(*cluster);
+    return ret;
+  }
+  rados_shutdown(*cluster);
+  return 0;
+}
+
+int destroy_one_ec_pool(const std::string &pool_name, rados_t *cluster)
+{
+  int ret = rados_pool_delete(*cluster, pool_name.c_str());
+  if (ret == 0) {
+    char *cmd[2];
+
+    cmd[1] = NULL;
+
+    cmd[0] = (char *)"{\"prefix\": \"osd erasure-code-profile rm\", \"name\": \"testprofile\"}";
+    int ret2 = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0);
+    if (ret2) {
+      rados_shutdown(*cluster);
+      return ret2;
+    }
+    rados_wait_for_latest_osdmap(*cluster);
+  }
+  rados_shutdown(*cluster);
+  return ret;
+}
+
+int destroy_one_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+  int ret = cluster.pool_delete(pool_name.c_str());
+  if (ret) {
+    cluster.shutdown();
+    return ret;
+  }
+  cluster.shutdown();
+  return 0;
+}
+
+int destroy_one_ec_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+  int ret = cluster.pool_delete(pool_name.c_str());
+  bufferlist inbl;
+  if (ret == 0) {
+    int ret2 = cluster.mon_command(
+      "{\"prefix\": \"osd erasure-code-profile rm\", \"name\": \"testprofile\"}",
+      inbl, NULL, NULL);
+    if (ret2) {
+      cluster.shutdown();
+      return ret2;
+    }
+    cluster.wait_for_latest_osdmap();
+  }
+  cluster.shutdown();
+  return ret;
+}
diff --git a/ceph/src/test/librados/test.h b/ceph/src/test/librados/test.h
new file mode 100644
index 00000000..6cf522de
--- /dev/null
+++ b/ceph/src/test/librados/test.h
@@ -0,0 +1,50 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_TEST_RADOS_API_TEST_H
+#define CEPH_TEST_RADOS_API_TEST_H
+
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+
+#include 
+#include 
+
+std::string get_temp_pool_name();
+
+std::string create_one_pool(const std::string &pool_name, rados_t *cluster);
+std::string create_one_ec_pool(const std::string &pool_name, rados_t *cluster);
+std::string create_one_pool_pp(const std::string &pool_name,
+			    librados::Rados &cluster);
+std::string create_one_ec_pool_pp(const std::string &pool_name,
+			    librados::Rados &cluster);
+std::string connect_cluster(rados_t *cluster);
+std::string connect_cluster_pp(librados::Rados &cluster);
+int destroy_one_pool(const std::string &pool_name, rados_t *cluster);
+int destroy_one_ec_pool(const std::string &pool_name, rados_t *cluster);
+int destroy_one_pool_pp(const std::string &pool_name, librados::Rados &cluster);
+int destroy_one_ec_pool_pp(const std::string &pool_name, librados::Rados &cluster);
+
+class TestAlarm
+{
+public:
+  TestAlarm() {
+    alarm(360);
+  }
+  ~TestAlarm() {
+    alarm(0);
+  }
+};
+
+#endif
diff --git a/ceph/src/test/librados/tier.cc b/ceph/src/test/librados/tier.cc
new file mode 100644
index 00000000..42673895
--- /dev/null
+++ b/ceph/src/test/librados/tier.cc
@@ -0,0 +1,3957 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "gtest/gtest.h"
+
+#include "mds/mdstypes.h"
+#include "include/buffer.h"
+#include "include/rbd_types.h"
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "include/types.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/common_init.h"
+#include "common/Cond.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#include "json_spirit/json_spirit.h"
+
+#include "osd/HitSet.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using namespace librados;
+using ceph::buffer;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+typedef RadosTestPP LibRadosTierPP;
+typedef RadosTestECPP LibRadosTierECPP;
+
+void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
+{
+  bufferlist inbl;
+  cache_ioctx.set_namespace("");
+  for (ObjectIterator it = cache_ioctx.objects_begin();
+       it != cache_ioctx.objects_end(); ++it) {
+    cache_ioctx.locator_set_key(it->second);
+    {
+      ObjectReadOperation op;
+      op.cache_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      cache_ioctx.aio_operate(
+        it->first, completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL);
+      completion->wait_for_safe();
+      completion->get_return_value();
+      completion->release();
+    }
+    {
+      ObjectReadOperation op;
+      op.cache_evict();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      cache_ioctx.aio_operate(
+        it->first, completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL);
+      completion->wait_for_safe();
+      completion->get_return_value();
+      completion->release();
+    }
+  }
+}
+
+class LibRadosTwoPoolsPP : public RadosTestPP
+{
+public:
+  LibRadosTwoPoolsPP() {};
+  virtual ~LibRadosTwoPoolsPP() {};
+protected:
+  static void SetUpTestCase() {
+    pool_name = get_temp_pool_name();
+    ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+    cache_pool_name = get_temp_pool_name();
+    ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
+  }
+  static void TearDownTestCase() {
+    ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
+    ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+  }
+  static std::string cache_pool_name;
+
+  virtual void SetUp() {
+    RadosTestPP::SetUp();
+    ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+    cache_ioctx.set_namespace(ns);
+  }
+  virtual void TearDown() {
+    RadosTestPP::TearDown();
+
+    // flush + evict cache
+    flush_evict_all(cluster, cache_ioctx);
+
+    bufferlist inbl;
+    // tear down tiers
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+      "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+    // wait for maps to settle before next test
+    cluster.wait_for_latest_osdmap();
+
+    cleanup_default_namespace(cache_ioctx);
+
+    cache_ioctx.close();
+  }
+  librados::IoCtx cache_ioctx;
+};
+
+std::string LibRadosTwoPoolsPP::cache_pool_name;
+
+TEST_F(LibRadosTierPP, Dirty) {
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
+  }
+  {
+    ObjectWriteOperation op;
+    op.create(true);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));  // still 0 if already clean
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  {
+    ObjectWriteOperation op;
+    op.truncate(0);  // still a write even tho it is a no-op
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Overlay) {
+  // create objects
+  {
+    bufferlist bl;
+    bl.append("base");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("cache");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // by default, the overlay sends us to cache pool
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // unless we say otherwise
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(0, 1, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+    ASSERT_EQ('b', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Promote) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, PromoteSnap) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // read foo snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read bar snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read baz snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+
+  // read foo
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read bar
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read baz
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, PromoteSnapScrub) {
+  int num = 100;
+
+  // create objects
+  for (int i=0; i my_snaps;
+  for (int snap=0; snap<4; ++snap) {
+    // create a snapshot, clone
+    vector ns(1);
+    ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
+    my_snaps.swap(ns);
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+    cout << "my_snaps " << my_snaps << std::endl;
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+						      my_snaps));
+    for (int i=0; i num - 3) {
+      bufferlist bl;
+      ASSERT_EQ(1, ioctx.read(string("foo") + stringify(i), bl, 1, 0));
+      ASSERT_EQ('c', bl[0]);
+    }
+  }
+
+  for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
+    cout << "promoting from clones for snap " << my_snaps[snap] << std::endl;
+    ioctx.snap_set_read(my_snaps[snap]);
+
+    // read some snaps, semi-randomly
+    for (int i=0; i<50; ++i) {
+      bufferlist bl;
+      string o = string("foo") + stringify((snap * i * 137) % 80);
+      //cout << o << std::endl;
+      ASSERT_EQ(1, ioctx.read(o, bl, 1, 0));
+    }
+  }
+
+  // ok, stop and scrub this pool (to make sure scrub can handle
+  // missing clones in the cache tier).
+  {
+    IoCtx cache_ioctx;
+    ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+    for (int i=0; i<10; ++i) {
+      ostringstream ss;
+      ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
+	 << cache_ioctx.get_id() << "." << i
+	 << "\"}";
+      cluster.mon_command(ss.str(), inbl, NULL, NULL);
+    }
+
+    // give it a few seconds to go.  this is sloppy but is usually enough time
+    cout << "waiting for scrubs..." << std::endl;
+    sleep(30);
+    cout << "done waiting" << std::endl;
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+}
+
+
+TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // delete the snap
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // read foo snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Whiteout) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create some whiteouts, verify they behave
+  ASSERT_EQ(0, ioctx.remove("foo"));
+
+  ASSERT_EQ(-ENOENT, ioctx.remove("bar"));
+  ASSERT_EQ(-ENOENT, ioctx.remove("bar"));
+
+  // verify the whiteouts are there in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
+
+  // recreate an object and verify we can read it
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Evict) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout, and a dirty object
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+					 librados::OPERATION_IGNORE_CACHE,
+					 NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "fooberdoodle", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // evict bam
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+
+  // read foo snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict foo snap
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // snap is gone...
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+  // head is still there...
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // promote head + snap of bar
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict bar head (fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict bar snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // ...and then head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, TryFlush) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it != ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Flush) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  uint64_t user_version = 0;
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+    user_version = cache_ioctx.get_last_version();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it != ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // read it again and verify the version is consistent
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ(user_version, cache_ioctx.get_last_version());
+  }
+
+  // erase it
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush whiteout
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+  // or base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("a");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("b");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("c");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // flush on head (should fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on recent snap (should fail)
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on oldest snap
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on next oldest snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify i can read the snaps from the cache pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  // remove overlay
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+
+  // verify i can read the snaps from the base pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+}
+
+TEST_F(LibRadosTierPP, FlushWriteRaces) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  std::string cache_pool_name = pool_name + "-cache";
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+  IoCtx cache_ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+  IoCtx ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  bufferlist bl;
+  bl.append("hi there");
+  {
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + write
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectWriteOperation op2;
+    op2.write_full(bl);
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion2, &op2, 0));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  int tries = 1000;
+  do {
+    // create/dirty object
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + write
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectWriteOperation op2;
+      op2.write_full(bl);
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
+
+      completion->wait_for_safe();
+      completion2->wait_for_safe();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+
+  ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  int tries = 1000;
+  do {
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + flush
+    //  (flush will not piggyback on try-flush)
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectReadOperation op2;
+      op2.cache_flush();
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion2, &op2,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+      completion->wait_for_safe();
+      completion2->wait_for_safe();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // try-flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+}
+
+
+IoCtx *read_ioctx = 0;
+Mutex test_lock("FlushReadRaces::lock");
+Cond cond;
+int max_reads = 100;
+int num_reads = 0; // in progress
+
+void flush_read_race_cb(completion_t cb, void *arg);
+
+void start_flush_read()
+{
+  //cout << " starting read" << std::endl;
+  ObjectReadOperation op;
+  op.stat(NULL, NULL, NULL);
+  librados::AioCompletion *completion =
+    librados::Rados::aio_create_completion();
+  completion->set_complete_callback(0, flush_read_race_cb);
+  read_ioctx->aio_operate("foo", completion, &op, NULL);
+}
+
+void flush_read_race_cb(completion_t cb, void *arg)
+{
+  //cout << " finished read" << std::endl;
+  test_lock.Lock();
+  if (num_reads > max_reads) {
+    num_reads--;
+    cond.Signal();
+  } else {
+    start_flush_read();
+  }
+  // fixme: i'm leaking cb...
+  test_lock.Unlock();
+}
+
+TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    bufferptr bp(4000000);  // make it big!
+    bp.zero();
+    bl.append(bp);
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // start a continuous stream of reads
+  read_ioctx = &ioctx;
+  test_lock.Lock();
+  for (int i = 0; i < max_reads; ++i) {
+    start_flush_read();
+    num_reads++;
+  }
+  test_lock.Unlock();
+
+  // try-flush
+  ObjectReadOperation op;
+  op.cache_try_flush();
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+  completion->wait_for_safe();
+  ASSERT_EQ(0, completion->get_return_value());
+  completion->release();
+
+  // stop reads
+  test_lock.Lock();
+  max_reads = 0;
+  while (num_reads > 0)
+    cond.Wait(test_lock);
+  test_lock.Unlock();
+}
+
+TEST_F(LibRadosTierPP, HitSetNone) {
+  {
+    list< pair > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
+    c->wait_for_complete();
+    ASSERT_EQ(0, c->get_return_value());
+    ASSERT_TRUE(ls.empty());
+    c->release();
+  }
+  {
+    bufferlist bl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
+    c->wait_for_complete();
+    ASSERT_EQ(-ENOENT, c->get_return_value());
+    c->release();
+  }
+}
+
+string set_pool_str(string pool, string var, string val)
+{
+  return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
+    + string("\",\"var\": \"") + var + string("\",\"val\": \"")
+    + val + string("\"}");
+}
+
+string set_pool_str(string pool, string var, int val)
+{
+  return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
+    + string("\",\"var\": \"") + var + string("\",\"val\": \"")
+    + stringify(val) + string("\"}");
+}
+
+TEST_F(LibRadosTwoPoolsPP, HitSetRead) {
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
+						"explicit_object"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  // keep reading until we see our object appear in the HitSet
+  utime_t start = ceph_clock_now(NULL);
+  utime_t hard_stop = start + utime_t(600, 0);
+
+  while (true) {
+    utime_t now = ceph_clock_now(NULL);
+    ASSERT_TRUE(now < hard_stop);
+
+    string name = "foo";
+    uint32_t hash = cache_ioctx.get_object_hash_position(name);
+    hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
+		  cluster.pool_lookup(cache_pool_name.c_str()), "");
+
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
+
+    bufferlist hbl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
+    c->wait_for_complete();
+    c->release();
+
+    if (hbl.length()) {
+      bufferlist::iterator p = hbl.begin();
+      HitSet hs;
+      ::decode(hs, p);
+      if (hs.contains(oid)) {
+	cout << "ok, hit_set contains " << oid << std::endl;
+	break;
+      }
+      cout << "hmm, not in HitSet yet" << std::endl;
+    } else {
+      cout << "hmm, no HitSet yet" << std::endl;
+    }
+
+    sleep(1);
+  }
+}
+
+static int _get_pg_num(Rados& cluster, string pool_name)
+{
+  bufferlist inbl;
+  string cmd = string("{\"prefix\": \"osd pool get\",\"pool\":\"")
+    + pool_name
+    + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
+  bufferlist outbl;
+  int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
+  assert(r >= 0);
+  string outstr(outbl.c_str(), outbl.length());
+  json_spirit::Value v;
+  if (!json_spirit::read(outstr, v)) {
+    cerr <<" unable to parse json " << outstr << std::endl;
+    return -1;
+  }
+
+  json_spirit::Object& o = v.get_obj();
+  for (json_spirit::Object::size_type i=0; i 0);
+
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 8),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
+						"explicit_hash"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  int num = 200;
+
+  // do a bunch of writes
+  for (int i=0; i hitsets;
+  for (int i=0; i > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_list(i, c, &ls));
+    c->wait_for_complete();
+    c->release();
+    std::cout << "pg " << i << " ls " << ls << std::endl;
+    ASSERT_FALSE(ls.empty());
+
+    // get the latest
+    c = librados::Rados::aio_create_completion();
+    bufferlist bl;
+    ASSERT_EQ(0, cache_ioctx.hit_set_get(i, c, ls.back().first, &bl));
+    c->wait_for_complete();
+    c->release();
+
+    //std::cout << "bl len is " << bl.length() << "\n";
+    //bl.hexdump(std::cout);
+    //std::cout << std::endl;
+
+    bufferlist::iterator p = bl.begin();
+    ::decode(hitsets[i], p);
+
+    // cope with racing splits by refreshing pg_num
+    if (i == num_pg - 1)
+      num_pg = _get_pg_num(cluster, cache_pool_name);
+  }
+
+  for (int i=0; i > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
+    c->wait_for_complete();
+    c->release();
+
+    ASSERT_TRUE(ls.size() <= count + 1);
+    cout << " got ls " << ls << std::endl;
+    if (!ls.empty()) {
+      if (!first) {
+	first = ls.front().first;
+	cout << "first is " << first << std::endl;
+      } else {
+	if (ls.front().first != first) {
+	  cout << "first now " << ls.front().first << ", trimmed" << std::endl;
+	  break;
+	}
+      }
+    }
+
+    utime_t now = ceph_clock_now(NULL);
+    ASSERT_TRUE(now < hard_stop);
+
+    sleep(1);
+  }
+}
+
+class LibRadosTwoPoolsECPP : public RadosTestECPP
+{
+public:
+  LibRadosTwoPoolsECPP() {};
+  virtual ~LibRadosTwoPoolsECPP() {};
+protected:
+  static void SetUpTestCase() {
+    pool_name = get_temp_pool_name();
+    ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
+    cache_pool_name = get_temp_pool_name();
+    ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
+  }
+  static void TearDownTestCase() {
+    ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
+    ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
+  }
+  static std::string cache_pool_name;
+
+  virtual void SetUp() {
+    RadosTestECPP::SetUp();
+    ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+    cache_ioctx.set_namespace(ns);
+  }
+  virtual void TearDown() {
+    RadosTestECPP::TearDown();
+
+    // flush + evict cache
+    flush_evict_all(cluster, cache_ioctx);
+
+    bufferlist inbl;
+    // tear down tiers
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+      "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+    // wait for maps to settle before next test
+    cluster.wait_for_latest_osdmap();
+
+    cleanup_default_namespace(cache_ioctx);
+
+    cache_ioctx.close();
+  }
+
+  librados::IoCtx cache_ioctx;
+};
+
+std::string LibRadosTwoPoolsECPP::cache_pool_name;
+
+TEST_F(LibRadosTierECPP, Dirty) {
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
+  }
+  {
+    ObjectWriteOperation op;
+    op.create(true);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));  // still 0 if already clean
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  //{
+  //  ObjectWriteOperation op;
+  //  op.truncate(0);  // still a write even tho it is a no-op
+  //  ASSERT_EQ(0, ioctx.operate("foo", &op));
+  //}
+  //{
+  //  bool dirty = false;
+  //  int r = -1;
+  //  ObjectReadOperation op;
+  //  op.is_dirty(&dirty, &r);
+  //  ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+  //  ASSERT_TRUE(dirty);
+  //  ASSERT_EQ(0, r);
+  //}
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Overlay) {
+  // create objects
+  {
+    bufferlist bl;
+    bl.append("base");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("cache");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // by default, the overlay sends us to cache pool
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // unless we say otherwise
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(0, 1, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+    ASSERT_EQ('b', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Promote) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, PromoteSnap) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // stop and scrub this pg (to make sure scrub can handle missing
+  // clones in the cache tier)
+  // This test requires cache tier and base tier to have the same pg_num/pgp_num
+  {
+    for (int tries = 0; tries < 5; ++tries) {
+      IoCtx cache_ioctx;
+      ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+      ostringstream ss;
+      ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
+	 << cache_ioctx.get_id() << "."
+	 << ioctx.get_object_pg_hash_position("foo")
+	 << "\"}";
+      int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
+      if (r == -EAGAIN)
+	continue;
+      ASSERT_EQ(0, r);
+      break;
+    }
+    // give it a few seconds to go.  this is sloppy but is usually enough time
+    cout << "waiting for scrub..." << std::endl;
+    sleep(15);
+    cout << "done waiting" << std::endl;
+  }
+
+  // read foo snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read bar snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read baz snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+
+  // read foo
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read bar
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read baz
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, PromoteSnapTrimRace) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // delete the snap
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // read foo snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Whiteout) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create some whiteouts, verify they behave
+  ASSERT_EQ(0, ioctx.remove("foo"));
+
+  ASSERT_EQ(-ENOENT, ioctx.remove("bar"));
+  ASSERT_EQ(-ENOENT, ioctx.remove("bar"));
+
+  // verify the whiteouts are there in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
+
+  // recreate an object and verify we can read it
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Evict) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout, and a dirty object
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->first == string("foo") || it->first == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+					 librados::OPERATION_IGNORE_CACHE,
+					 NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "fooberdoodle", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // evict bam
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+
+  // read foo snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict foo snap
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // snap is gone...
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+  // head is still there...
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // promote head + snap of bar
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict bar head (fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict bar snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // ...and then head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it != ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Flush) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  uint64_t user_version = 0;
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+    user_version = cache_ioctx.get_last_version();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it != ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // read it again and verify the version is consistent
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ(user_version, cache_ioctx.get_last_version());
+  }
+
+  // erase it
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush whiteout
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+  // or base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("a");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("b");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("c");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    ObjectIterator it = cache_ioctx.objects_begin();
+    ASSERT_TRUE(it != cache_ioctx.objects_end());
+    ASSERT_TRUE(it->first == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.objects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    ObjectIterator it = ioctx.objects_begin();
+    ASSERT_TRUE(it == ioctx.objects_end());
+  }
+
+  // flush on head (should fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on recent snap (should fail)
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on oldest snap
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on next oldest snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify i can read the snaps from the cache pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // verify i can read the snaps from the base pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+}
+
+TEST_F(LibRadosTierECPP, FlushWriteRaces) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  std::string cache_pool_name = pool_name + "-cache";
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+  IoCtx cache_ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+  IoCtx ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  bufferlist bl;
+  bl.append("hi there");
+  {
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + write
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectWriteOperation op2;
+    op2.write_full(bl);
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion2, &op2, 0));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  int tries = 1000;
+  do {
+    // create/dirty object
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + write
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectWriteOperation op2;
+      op2.write_full(bl);
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
+
+      completion->wait_for_safe();
+      completion2->wait_for_safe();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+
+  ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  int tries = 1000;
+  do {
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + flush
+    //  (flush will not piggyback on try-flush)
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectReadOperation op2;
+      op2.cache_flush();
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion2, &op2,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+      completion->wait_for_safe();
+      completion2->wait_for_safe();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // try-flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_safe();
+    completion2->wait_for_safe();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    bufferptr bp(4000000);  // make it big!
+    bp.zero();
+    bl.append(bp);
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // start a continuous stream of reads
+  read_ioctx = &ioctx;
+  test_lock.Lock();
+  for (int i = 0; i < max_reads; ++i) {
+    start_flush_read();
+    num_reads++;
+  }
+  test_lock.Unlock();
+
+  // try-flush
+  ObjectReadOperation op;
+  op.cache_try_flush();
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+  completion->wait_for_safe();
+  ASSERT_EQ(0, completion->get_return_value());
+  completion->release();
+
+  // stop reads
+  test_lock.Lock();
+  max_reads = 0;
+  while (num_reads > 0)
+    cond.Wait(test_lock);
+  test_lock.Unlock();
+}
+
+TEST_F(LibRadosTierECPP, HitSetNone) {
+  {
+    list< pair > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
+    c->wait_for_complete();
+    ASSERT_EQ(0, c->get_return_value());
+    ASSERT_TRUE(ls.empty());
+    c->release();
+  }
+  {
+    bufferlist bl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
+    c->wait_for_complete();
+    ASSERT_EQ(-ENOENT, c->get_return_value());
+    c->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, HitSetRead) {
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
+						"explicit_object"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  // keep reading until we see our object appear in the HitSet
+  utime_t start = ceph_clock_now(NULL);
+  utime_t hard_stop = start + utime_t(600, 0);
+
+  while (true) {
+    utime_t now = ceph_clock_now(NULL);
+    ASSERT_TRUE(now < hard_stop);
+
+    string name = "foo";
+    uint32_t hash = cache_ioctx.get_object_hash_position(name);
+    hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
+		  cluster.pool_lookup(cache_pool_name.c_str()), "");
+
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
+
+    bufferlist hbl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
+    c->wait_for_complete();
+    c->release();
+
+    if (hbl.length()) {
+      bufferlist::iterator p = hbl.begin();
+      HitSet hs;
+      ::decode(hs, p);
+      if (hs.contains(oid)) {
+	cout << "ok, hit_set contains " << oid << std::endl;
+	break;
+      }
+      cout << "hmm, not in HitSet yet" << std::endl;
+    } else {
+      cout << "hmm, no HitSet yet" << std::endl;
+    }
+
+    sleep(1);
+  }
+}
+
+// disable this test until hitset-get reliably works on EC pools
+#if 0
+TEST_F(LibRadosTierECPP, HitSetWrite) {
+  int num_pg = _get_pg_num(cluster, pool_name);
+  assert(num_pg > 0);
+
+  // enable hitset tracking for this pool
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 8),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
+						"explicit_hash"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  ioctx.set_namespace("");
+
+  // do a bunch of writes
+  for (int i=0; i<1000; ++i) {
+    bufferlist bl;
+    bl.append("a");
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, 1, 0));
+  }
+
+  // get HitSets
+  std::map hitsets;
+  for (int i=0; i > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_list(i, c, &ls));
+    c->wait_for_complete();
+    c->release();
+    std::cout << "pg " << i << " ls " << ls << std::endl;
+    ASSERT_FALSE(ls.empty());
+
+    // get the latest
+    c = librados::Rados::aio_create_completion();
+    bufferlist bl;
+    ASSERT_EQ(0, ioctx.hit_set_get(i, c, ls.back().first, &bl));
+    c->wait_for_complete();
+    c->release();
+
+    //std::cout << "bl len is " << bl.length() << "\n";
+    //bl.hexdump(std::cout);
+    //std::cout << std::endl;
+
+    bufferlist::iterator p = bl.begin();
+    ::decode(hitsets[i], p);
+
+    // cope with racing splits by refreshing pg_num
+    if (i == num_pg - 1)
+      num_pg = _get_pg_num(cluster, pool_name);
+  }
+
+  for (int i=0; i<1000; ++i) {
+    string n = stringify(i);
+    uint32_t hash = ioctx.get_object_hash_position(n);
+    hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
+		  cluster.pool_lookup(pool_name.c_str()), "");
+    std::cout << "checking for " << oid << std::endl;
+    bool found = false;
+    for (int p=0; p > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
+    c->wait_for_complete();
+    c->release();
+
+    ASSERT_TRUE(ls.size() <= count + 1);
+    cout << " got ls " << ls << std::endl;
+    if (!ls.empty()) {
+      if (!first) {
+	first = ls.front().first;
+	cout << "first is " << first << std::endl;
+      } else {
+	if (ls.front().first != first) {
+	  cout << "first now " << ls.front().first << ", trimmed" << std::endl;
+	  break;
+	}
+      }
+    }
+
+    utime_t now = ceph_clock_now(NULL);
+    ASSERT_TRUE(now < hard_stop);
+
+    sleep(1);
+  }
+  delete[] buf;
+}
+
+int main(int argc, char **argv)
+{
+  ::testing::InitGoogleTest(&argc, argv);
+
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/librados/watch_notify.cc b/ceph/src/test/librados/watch_notify.cc
new file mode 100644
index 00000000..d92c1b8e
--- /dev/null
+++ b/ceph/src/test/librados/watch_notify.cc
@@ -0,0 +1,139 @@
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/rados/rados_types.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include 
+#include 
+#include "gtest/gtest.h"
+
+using namespace librados;
+
+typedef RadosTest LibRadosWatchNotify;
+typedef RadosTestParamPP LibRadosWatchNotifyPP;
+typedef RadosTestEC LibRadosWatchNotifyEC;
+typedef RadosTestECPP LibRadosWatchNotifyECPP;
+
+static sem_t sem;
+
+static void watch_notify_test_cb(uint8_t opcode, uint64_t ver, void *arg)
+{
+  sem_post(&sem);
+}
+
+class WatchNotifyTestCtx : public WatchCtx
+{
+public:
+    void notify(uint8_t opcode, uint64_t ver, bufferlist& bl)
+    {
+      sem_post(&sem);
+    }
+};
+
+TEST_F(LibRadosWatchNotify, WatchNotifyTest) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+      rados_watch(ioctx, "foo", 0, &handle, watch_notify_test_cb, NULL));
+  ASSERT_EQ(0, rados_notify(ioctx, "foo", 0, NULL, 0));
+  TestAlarm alarm;
+  sem_wait(&sem);
+  rados_unwatch(ioctx, "foo", handle);
+  sem_destroy(&sem);
+}
+
+TEST_P(LibRadosWatchNotifyPP, WatchNotifyTestPP) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  std::list watches;
+  ASSERT_EQ(0, ioctx.list_watchers("foo", &watches));
+  ASSERT_EQ(watches.size(), 1u);
+  bufferlist bl2;
+  ASSERT_EQ(0, ioctx.notify("foo", 0, bl2));
+  TestAlarm alarm;
+  sem_wait(&sem);
+  ioctx.unwatch("foo", handle);
+  sem_destroy(&sem);
+}
+TEST_P(LibRadosWatchNotifyPP, WatchNotifyTimeoutTestPP) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  ioctx.set_notify_timeout(1);
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  sem_destroy(&sem);
+}
+
+TEST_F(LibRadosWatchNotifyEC, WatchNotifyTest) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+      rados_watch(ioctx, "foo", 0, &handle, watch_notify_test_cb, NULL));
+  ASSERT_EQ(0, rados_notify(ioctx, "foo", 0, NULL, 0));
+  TestAlarm alarm;
+  sem_wait(&sem);
+  rados_unwatch(ioctx, "foo", handle);
+  sem_destroy(&sem);
+}
+
+TEST_F(LibRadosWatchNotifyECPP, WatchNotifyTestPP) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  std::list watches;
+  ASSERT_EQ(0, ioctx.list_watchers("foo", &watches));
+  ASSERT_EQ(watches.size(), 1u);
+  bufferlist bl2;
+  ASSERT_EQ(0, ioctx.notify("foo", 0, bl2));
+  TestAlarm alarm;
+  sem_wait(&sem);
+  ioctx.unwatch("foo", handle);
+  sem_destroy(&sem);
+}
+
+TEST_F(LibRadosWatchNotifyECPP, WatchNotifyTimeoutTestPP) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  ioctx.set_notify_timeout(1);
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  sem_destroy(&sem);
+}
+
+
+INSTANTIATE_TEST_CASE_P(LibRadosWatchNotifyPPTests, LibRadosWatchNotifyPP,
+			::testing::Values("", "cache"));
diff --git a/ceph/src/test/librbd/fsx.c b/ceph/src/test/librbd/fsx.c
new file mode 100644
index 00000000..97feb4c9
--- /dev/null
+++ b/ceph/src/test/librbd/fsx.c
@@ -0,0 +1,1622 @@
+// -*- mode:C; tab-width:8; c-basic-offset:8; indent-tabs-mode:t -*- 
+/*
+ *	Copyright (C) 1991, NeXT Computer, Inc.  All Rights Reserverd.
+ *
+ *	File:	fsx.c
+ *	Author:	Avadis Tevanian, Jr.
+ *
+ *	File system exerciser. 
+ *
+ *	Rewritten 8/98 by Conrad Minshall.
+ *
+ *	Small changes to work under Linux -- davej.
+ *
+ *	Checks for mmap last-page zero fill.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#ifdef HAVE_ERR_H
+#include 
+#endif
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/rados/librados.h"
+#include "include/rbd/librbd.h"
+
+#define NUMPRINTCOLUMNS 32	/* # columns of data to print on each line */
+
+/*
+ *	A log entry is an operation and a bunch of arguments.
+ */
+
+struct log_entry {
+	int	operation;
+	int	args[3];
+};
+
+#define	LOGSIZE	1000
+
+struct log_entry	oplog[LOGSIZE];	/* the log */
+int			logptr = 0;	/* current position in log */
+int			logcount = 0;	/* total ops */
+
+/*
+ * The operation matrix is complex due to conditional execution of different
+ * features. Hence when we come to deciding what operation to run, we need to
+ * be careful in how we select the different operations. The active operations
+ * are mapped to numbers as follows:
+ *
+ *		lite	!lite
+ * READ:	0	0
+ * WRITE:	1	1
+ * MAPREAD:	2	2
+ * MAPWRITE:	3	3
+ * TRUNCATE:	-	4
+ * FALLOCATE:	-	5
+ * PUNCH HOLE:	-	6
+ *
+ * When mapped read/writes are disabled, they are simply converted to normal
+ * reads and writes. When fallocate/fpunch calls are disabled, they are
+ * converted to OP_SKIPPED. Hence OP_SKIPPED needs to have a number higher than
+ * the operation selction matrix, as does the OP_CLOSEOPEN which is an
+ * operation modifier rather than an operation in itself.
+ *
+ * Because of the "lite" version, we also need to have different "maximum
+ * operation" defines to allow the ops to be selected correctly based on the
+ * mode being run.
+ */
+
+/* common operations */
+#define	OP_READ		0
+#define OP_WRITE	1
+#define OP_MAPREAD	2
+#define OP_MAPWRITE	3
+#define OP_MAX_LITE	4
+
+/* !lite operations */
+#define OP_TRUNCATE	4
+#define OP_FALLOCATE	5
+#define OP_PUNCH_HOLE	6
+/* rbd-specific operations */
+#define OP_CLONE        7
+#define OP_FLATTEN	8
+#define OP_MAX_FULL	9
+
+/* operation modifiers */
+#define OP_CLOSEOPEN	100
+#define OP_SKIPPED	101
+
+#undef PAGE_SIZE
+#define PAGE_SIZE       getpagesize()
+#undef PAGE_MASK
+#define PAGE_MASK       (PAGE_SIZE - 1)
+
+char	*original_buf;			/* a pointer to the original data */
+char	*good_buf;			/* a pointer to the correct data */
+char	*temp_buf;			/* a pointer to the current data */
+char	*pool;				/* name of the pool our test image is in */
+char	*iname;				/* name of our test image */
+rados_t cluster;                        /* handle for our test cluster */
+rados_ioctx_t	ioctx;			/* handle for our test pool */
+rbd_image_t	image;			/* handle for our test image */
+
+char	dirpath[1024];
+
+off_t		file_size = 0;
+off_t		biggest = 0;
+char		state[256];
+unsigned long	testcalls = 0;		/* calls to function "test" */
+
+unsigned long	simulatedopcount = 0;	/* -b flag */
+int	closeprob = 0;			/* -c flag */
+int	debug = 0;			/* -d flag */
+unsigned long	debugstart = 0;		/* -D flag */
+int	flush = 0;			/* -f flag */
+int	do_fsync = 0;			/* -y flag */
+unsigned long	maxfilelen = 256 * 1024;	/* -l flag */
+int	sizechecks = 1;			/* -n flag disables them */
+int	maxoplen = 64 * 1024;		/* -o flag */
+int	quiet = 0;			/* -q flag */
+unsigned long progressinterval = 0;	/* -p flag */
+int	readbdy = 1;			/* -r flag */
+int	style = 0;			/* -s flag */
+int	prealloc = 0;			/* -x flag */
+int	truncbdy = 1;			/* -t flag */
+int	writebdy = 1;			/* -w flag */
+long	monitorstart = -1;		/* -m flag */
+long	monitorend = -1;		/* -m flag */
+int	lite = 0;			/* -L flag */
+long	numops = -1;			/* -N flag */
+int	randomoplen = 1;		/* -O flag disables it */
+int	seed = 1;			/* -S flag */
+int     mapped_writes = 0;              /* -W flag disables */
+int     fallocate_calls = 0;            /* -F flag disables */
+int     punch_hole_calls = 1;           /* -H flag disables */
+int	clone_calls = 1;                /* -C flag disables */
+int     randomize_striping = 1;
+int 	mapped_reads = 0;		/* -R flag disables it */
+int	fsxgoodfd = 0;
+int	o_direct;			/* -Z */
+int	aio = 0;
+
+int num_clones = 0;
+
+int page_size;
+int page_mask;
+int mmap_mask;
+#ifdef AIO
+int aio_rw(int rw, int fd, char *buf, unsigned len, unsigned offset);
+#define READ 0
+#define WRITE 1
+#define fsxread(a,b,c,d)	aio_rw(READ, a,b,c,d)
+#define fsxwrite(a,b,c,d)	aio_rw(WRITE, a,b,c,d)
+#else
+#define fsxread(a,b,c,d)	read(a,b,c)
+#define fsxwrite(a,b,c,d)	write(a,b,c)
+#endif
+
+FILE *	fsxlogf = NULL;
+int badoff = -1;
+int closeopen = 0;
+
+static void *round_ptr_up(void *ptr, unsigned long align, unsigned long offset)
+{
+	unsigned long ret = (unsigned long)ptr;
+
+	ret = ((ret + align - 1) & ~(align - 1));
+	ret += offset;
+	return (void *)ret;
+}
+
+void
+vwarnc(int code, const char *fmt, va_list ap) {
+  fprintf(stderr, "fsx: ");
+  if (fmt != NULL) {
+	vfprintf(stderr, fmt, ap);
+	fprintf(stderr, ": ");
+  }
+  fprintf(stderr, "%s\n", strerror(code));
+}
+
+void
+warn(const char * fmt, ...)  {
+	va_list ap;
+	va_start(ap, fmt);
+	vwarnc(errno, fmt, ap);
+	va_end(ap);
+}
+
+#define BUF_SIZE 1024
+
+void
+prt(char *fmt, ...)
+{
+	va_list args;
+	char buffer[BUF_SIZE];
+
+	va_start(args, fmt);
+	vsnprintf(buffer, BUF_SIZE, fmt, args);
+	va_end(args);
+	fprintf(stdout, "%s", buffer);
+	if (fsxlogf)
+		fprintf(fsxlogf, "%s", buffer);
+}
+
+void
+prterr(char *prefix)
+{
+	prt("%s%s%s\n", prefix, prefix ? ": " : "", strerror(errno));
+}
+
+void
+log4(int operation, int arg0, int arg1, int arg2)
+{
+	struct log_entry *le;
+
+	le = &oplog[logptr];
+	le->operation = operation;
+	if (closeopen)
+		le->operation = ~ le->operation;
+	le->args[0] = arg0;
+	le->args[1] = arg1;
+	le->args[2] = arg2;
+	logptr++;
+	logcount++;
+	if (logptr >= LOGSIZE)
+		logptr = 0;
+}
+
+void
+simple_err(const char *msg, int err)
+{
+    fprintf(stderr, "%s: %s\n", msg, strerror(-err));
+}
+
+void
+logdump(void)
+{
+	int	i, count, down;
+	struct log_entry	*lp;
+	char *falloc_type[3] = {"PAST_EOF", "EXTENDING", "INTERIOR"};
+
+	prt("LOG DUMP (%d total operations):\n", logcount);
+	if (logcount < LOGSIZE) {
+		i = 0;
+		count = logcount;
+	} else {
+		i = logptr;
+		count = LOGSIZE;
+	}
+	for ( ; count > 0; count--) {
+		int opnum;
+
+		opnum = i+1 + (logcount/LOGSIZE)*LOGSIZE;
+		prt("%d(%3d mod 256): ", opnum, opnum%256);
+		lp = &oplog[i];
+		if ((closeopen = lp->operation < 0))
+			lp->operation = ~ lp->operation;
+			
+		switch (lp->operation) {
+		case OP_MAPREAD:
+			prt("MAPREAD  0x%x thru 0x%x\t(0x%x bytes)",
+			    lp->args[0], lp->args[0] + lp->args[1] - 1,
+			    lp->args[1]);
+			if (badoff >= lp->args[0] && badoff <
+						     lp->args[0] + lp->args[1])
+				prt("\t***RRRR***");
+			break;
+		case OP_MAPWRITE:
+			prt("MAPWRITE 0x%x thru 0x%x\t(0x%x bytes)",
+			    lp->args[0], lp->args[0] + lp->args[1] - 1,
+			    lp->args[1]);
+			if (badoff >= lp->args[0] && badoff <
+						     lp->args[0] + lp->args[1])
+				prt("\t******WWWW");
+			break;
+		case OP_READ:
+			prt("READ     0x%x thru 0x%x\t(0x%x bytes)",
+			    lp->args[0], lp->args[0] + lp->args[1] - 1,
+			    lp->args[1]);
+			if (badoff >= lp->args[0] &&
+			    badoff < lp->args[0] + lp->args[1])
+				prt("\t***RRRR***");
+			break;
+		case OP_WRITE:
+			prt("WRITE    0x%x thru 0x%x\t(0x%x bytes)",
+			    lp->args[0], lp->args[0] + lp->args[1] - 1,
+			    lp->args[1]);
+			if (lp->args[0] > lp->args[2])
+				prt(" HOLE");
+			else if (lp->args[0] + lp->args[1] > lp->args[2])
+				prt(" EXTEND");
+			if ((badoff >= lp->args[0] || badoff >=lp->args[2]) &&
+			    badoff < lp->args[0] + lp->args[1])
+				prt("\t***WWWW");
+			break;
+		case OP_TRUNCATE:
+			down = lp->args[0] < lp->args[1];
+			prt("TRUNCATE %s\tfrom 0x%x to 0x%x",
+			    down ? "DOWN" : "UP", lp->args[1], lp->args[0]);
+			if (badoff >= lp->args[!down] &&
+			    badoff < lp->args[!!down])
+				prt("\t******WWWW");
+			break;
+		case OP_FALLOCATE:
+			/* 0: offset 1: length 2: where alloced */
+			prt("FALLOC   0x%x thru 0x%x\t(0x%x bytes) %s",
+				lp->args[0], lp->args[0] + lp->args[1],
+				lp->args[1], falloc_type[lp->args[2]]);
+			if (badoff >= lp->args[0] &&
+			    badoff < lp->args[0] + lp->args[1])
+				prt("\t******FFFF");
+			break;
+		case OP_PUNCH_HOLE:
+			prt("PUNCH    0x%x thru 0x%x\t(0x%x bytes)",
+			    lp->args[0], lp->args[0] + lp->args[1] - 1,
+			    lp->args[1]);
+			if (badoff >= lp->args[0] && badoff <
+						     lp->args[0] + lp->args[1])
+				prt("\t******PPPP");
+			break;
+		case OP_CLONE:
+			prt("CLONE");
+			break;
+		case OP_FLATTEN:
+			prt("FLATTEN");
+			break;
+		case OP_SKIPPED:
+			prt("SKIPPED (no operation)");
+			break;
+		default:
+			prt("BOGUS LOG ENTRY (operation code = %d)!",
+			    lp->operation);
+		}
+		if (closeopen)
+			prt("\n\t\tCLOSE/OPEN");
+		prt("\n");
+		i++;
+		if (i == LOGSIZE)
+			i = 0;
+	}
+}
+
+void
+prterrcode(char *prefix, int code)
+{
+	prt("%s%s%s\n", prefix, prefix ? ": " : "", strerror(-code));
+}
+
+void
+save_buffer(char *buffer, off_t bufferlength, int fd)
+{
+	off_t ret;
+	ssize_t byteswritten;
+
+	if (fd <= 0 || bufferlength == 0)
+		return;
+
+	if (bufferlength > SSIZE_MAX) {
+		prt("fsx flaw: overflow in save_buffer\n");
+		exit(67);
+	}
+
+	ret = lseek(fd, (off_t)0, SEEK_SET);
+	if (ret == (off_t)-1)
+		prterr("save_buffer: lseek 0");
+	
+	byteswritten = write(fd, buffer, (size_t)bufferlength);
+	if (byteswritten != bufferlength) {
+		if (byteswritten == -1)
+			prterr("save_buffer write");
+		else
+			warn("save_buffer: short write, 0x%x bytes instead of 0x%llx\n",
+			     (unsigned)byteswritten,
+			     (unsigned long long)bufferlength);
+	}
+}
+
+
+void
+report_failure(int status)
+{
+	logdump();
+	
+	if (fsxgoodfd) {
+		if (good_buf) {
+			save_buffer(good_buf, file_size, fsxgoodfd);
+			prt("Correct content saved for comparison\n");
+			prt("(maybe hexdump \"%s\" vs \"%s.fsxgood\")\n",
+			    iname, iname);
+		}
+		close(fsxgoodfd);
+	}
+	sleep(3);   // so the log can flush to disk.  KLUDGEY!
+	exit(status);
+}
+
+#define short_at(cp) ((unsigned short)((*((unsigned char *)(cp)) << 8) | \
+				        *(((unsigned char *)(cp)) + 1)))
+
+void
+check_buffers(char *good_buf, char *temp_buf, unsigned offset, unsigned size)
+{
+	unsigned char c, t;
+	unsigned i = 0;
+	unsigned n = 0;
+	unsigned op = 0;
+	unsigned bad = 0;
+
+	if (memcmp(good_buf + offset, temp_buf, size) != 0) {
+		prt("READ BAD DATA: offset = 0x%x, size = 0x%x, fname = %s\n",
+		    offset, size, iname);
+		prt("OFFSET\tGOOD\tBAD\tRANGE\n");
+		while (size > 0) {
+			c = good_buf[offset];
+			t = temp_buf[i];
+			if (c != t) {
+			        if (n < 16) {
+					bad = short_at(&temp_buf[i]);
+				        prt("0x%5x\t0x%04x\t0x%04x", offset,
+				            short_at(&good_buf[offset]), bad);
+					op = temp_buf[offset & 1 ? i+1 : i];
+				        prt("\t0x%5x\n", n);
+					if (op)
+						prt("operation# (mod 256) for "
+						  "the bad data may be %u\n",
+						((unsigned)op & 0xff));
+					else
+						prt("operation# (mod 256) for "
+						  "the bad data unknown, check"
+						  " HOLE and EXTEND ops\n");
+				}
+				n++;
+				badoff = offset;
+			}
+			offset++;
+			i++;
+			size--;
+		}
+		report_failure(110);
+	}
+}
+
+
+void
+check_size(void)
+{
+	rbd_image_info_t statbuf;
+	int ret;
+
+	if ((ret = rbd_stat(image, &statbuf, sizeof(statbuf))) < 0) {
+		prterrcode("check_size: fstat", ret);
+	}
+	if ((uint64_t)file_size != statbuf.size) {
+		prt("Size error: expected 0x%llx stat 0x%llx\n",
+		    (unsigned long long)file_size,
+		    (unsigned long long)statbuf.size);
+		report_failure(120);
+	}
+}
+
+
+void
+check_trunc_hack(void)
+{
+	rbd_image_info_t statbuf;
+
+	rbd_resize(image, (off_t)0);
+	rbd_resize(image, (off_t)100000);
+	rbd_stat(image, &statbuf, sizeof(statbuf));
+	if (statbuf.size != (off_t)100000) {
+ 		prt("no extend on truncate! not posix!\n");
+ 		exit(130);
+ 	}
+	rbd_resize(image, (off_t)0);
+}
+
+int
+create_image()
+{
+	int r;
+	int order = 0;
+	r = rados_create(&cluster, NULL);
+	if (r < 0) {
+		simple_err("Could not create cluster handle", r);
+		return r;
+	}
+	rados_conf_parse_env(cluster, NULL);
+	r = rados_conf_read_file(cluster, NULL);
+	if (r < 0) {
+		simple_err("Error reading ceph config file", r);
+		goto failed_shutdown;
+	}
+	r = rados_connect(cluster);
+	if (r < 0) {
+		simple_err("Error connecting to cluster", r);
+		goto failed_shutdown;
+	}
+	r = rados_pool_create(cluster, pool);
+	if (r < 0 && r != -EEXIST) {
+		simple_err("Error creating pool", r);
+		goto failed_shutdown;
+	}
+	r = rados_ioctx_create(cluster, pool, &ioctx);
+	if (r < 0) {
+		simple_err("Error creating ioctx", r);
+		goto failed_shutdown;
+	}
+	if (clone_calls) {
+		r = rbd_create2(ioctx, iname, 0, RBD_FEATURE_LAYERING, &order);
+	} else {
+		r = rbd_create(ioctx, iname, 0, &order);
+	}
+	if (r < 0) {
+		simple_err("Error creating image", r);
+		goto failed_open;
+	}
+
+	return 0;
+
+ failed_open:
+	rados_ioctx_destroy(ioctx);
+ failed_shutdown:
+	rados_shutdown(cluster);
+	return r;
+}
+
+void
+doflush(unsigned offset, unsigned size)
+{
+	if (o_direct == O_DIRECT)
+		return;
+
+	rbd_flush(image);
+}
+
+void
+doread(unsigned offset, unsigned size)
+{
+	int ret;
+
+	offset -= offset % readbdy;
+	if (o_direct)
+		size -= size % readbdy;
+	if (size == 0) {
+		if (!quiet && testcalls > simulatedopcount && !o_direct)
+			prt("skipping zero size read\n");
+		log4(OP_SKIPPED, OP_READ, offset, size);
+		return;
+	}
+	if (size + offset > file_size) {
+		if (!quiet && testcalls > simulatedopcount)
+			prt("skipping seek/read past end of file\n");
+		log4(OP_SKIPPED, OP_READ, offset, size);
+		return;
+	}
+
+	log4(OP_READ, offset, size, 0);
+
+	if (testcalls <= simulatedopcount)
+		return;
+
+	if (!quiet &&
+		((progressinterval && testcalls % progressinterval == 0)  ||
+		(debug &&
+		       (monitorstart == -1 ||
+			(offset + size > monitorstart &&
+			(monitorend == -1 || offset <= monitorend))))))
+		prt("%lu read\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
+		    offset, offset + size - 1, size);
+	ret = rbd_read(image, offset, size, temp_buf);
+	if (ret != (int)size) {
+		if (ret < 0)
+			prterrcode("doread: read", ret);
+		else
+			prt("short read: 0x%x bytes instead of 0x%x\n",
+			    ret, size);
+		report_failure(141);
+	}
+	check_buffers(good_buf, temp_buf, offset, size);
+}
+
+
+void
+check_eofpage(char *s, unsigned offset, char *p, int size)
+{
+	unsigned long last_page, should_be_zero;
+
+	if (offset + size <= (file_size & ~page_mask))
+		return;
+	/*
+	 * we landed in the last page of the file
+	 * test to make sure the VM system provided 0's 
+	 * beyond the true end of the file mapping
+	 * (as required by mmap def in 1996 posix 1003.1)
+	 */
+	last_page = ((unsigned long)p + (offset & page_mask) + size) & ~page_mask;
+
+	for (should_be_zero = last_page + (file_size & page_mask);
+	     should_be_zero < last_page + page_size;
+	     should_be_zero++)
+		if (*(char *)should_be_zero) {
+			prt("Mapped %s: non-zero data past EOF (0x%llx) page offset 0x%x is 0x%04x\n",
+			    s, file_size - 1, should_be_zero & page_mask,
+			    short_at(should_be_zero));
+			report_failure(205);
+		}
+}
+
+
+void
+gendata(char *original_buf, char *good_buf, unsigned offset, unsigned size)
+{
+	while (size--) {
+		good_buf[offset] = testcalls % 256; 
+		if (offset % 2)
+			good_buf[offset] += original_buf[offset];
+		offset++;
+	}
+}
+
+
+void
+dowrite(unsigned offset, unsigned size)
+{
+	ssize_t ret;
+	off_t newsize;
+
+	offset -= offset % writebdy;
+	if (o_direct)
+		size -= size % writebdy;
+	if (size == 0) {
+		if (!quiet && testcalls > simulatedopcount && !o_direct)
+			prt("skipping zero size write\n");
+		log4(OP_SKIPPED, OP_WRITE, offset, size);
+		return;
+	}
+
+	log4(OP_WRITE, offset, size, file_size);
+
+	gendata(original_buf, good_buf, offset, size);
+	if (file_size < offset + size) {
+		newsize = ceil(((double)offset + size) / truncbdy) * truncbdy;
+		if (file_size < newsize)
+			memset(good_buf + file_size, '\0', newsize - file_size);
+		file_size = newsize;
+		if (lite) {
+			warn("Lite file size bug in fsx!");
+			report_failure(149);
+		}
+		ret = rbd_resize(image, newsize);
+		if (ret < 0) {
+			prterrcode("dowrite: resize", ret);
+			report_failure(150);
+		}
+	}
+
+	if (testcalls <= simulatedopcount)
+		return;
+
+	if (!quiet &&
+		((progressinterval && testcalls % progressinterval == 0) ||
+		       (debug &&
+		       (monitorstart == -1 ||
+			(offset + size > monitorstart &&
+			(monitorend == -1 || offset <= monitorend))))))
+		prt("%lu write\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
+		    offset, offset + size - 1, size);
+
+	ret = rbd_write(image, offset, size, good_buf + offset);
+	if (ret != size) {
+		if (ret < 0)
+			prterrcode("dowrite: rbd_write", ret);
+		else
+			prt("short write: 0x%x bytes instead of 0x%x\n",
+			    ret, size);
+		report_failure(151);
+	}
+	if (flush) {
+		doflush(offset, size);
+	}
+}
+
+
+void
+dotruncate(unsigned size)
+{
+	int oldsize = file_size;
+	int ret;
+
+	size -= size % truncbdy;
+	if (size > biggest) {
+		biggest = size;
+		if (!quiet && testcalls > simulatedopcount)
+			prt("truncating to largest ever: 0x%x\n", size);
+	}
+
+	log4(OP_TRUNCATE, size, (unsigned)file_size, 0);
+
+	if (size > file_size)
+		memset(good_buf + file_size, '\0', size - file_size);
+	else if (size < file_size)
+		memset(good_buf + size, '\0', file_size - size);
+	file_size = size;
+
+	if (testcalls <= simulatedopcount)
+		return;
+	
+	if ((progressinterval && testcalls % progressinterval == 0) ||
+	    (debug && (monitorstart == -1 || monitorend == -1 ||
+		      size <= monitorend)))
+		prt("%lu trunc\tfrom 0x%x to 0x%x\n", testcalls, oldsize, size);
+	if ((ret = rbd_resize(image, size)) < 0) {
+		prt("rbd_resize: %x\n", size);
+		prterrcode("dotruncate: ftruncate", ret);
+		report_failure(160);
+	}
+}
+
+void
+do_punch_hole(unsigned offset, unsigned length)
+{
+	unsigned end_offset;
+	int max_offset = 0;
+	int max_len = 0;
+	int ret;
+
+	if (length == 0) {
+		if (!quiet && testcalls > simulatedopcount)
+			prt("skipping zero length punch hole\n");
+			log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, length);
+		return;
+	}
+
+	if (file_size <= (loff_t)offset) {
+		if (!quiet && testcalls > simulatedopcount)
+			prt("skipping hole punch off the end of the file\n");
+			log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, length);
+		return;
+	}
+
+	end_offset = offset + length;
+
+	log4(OP_PUNCH_HOLE, offset, length, 0);
+
+	if (testcalls <= simulatedopcount)
+		return;
+
+	if ((progressinterval && testcalls % progressinterval == 0) ||
+	    (debug && (monitorstart == -1 || monitorend == -1 ||
+		      end_offset <= monitorend))) {
+		prt("%lu punch\tfrom 0x%x to 0x%x, (0x%x bytes)\n", testcalls,
+			offset, offset+length, length);
+	}
+	if ((ret = rbd_discard(image, (unsigned long long) offset,
+			       (unsigned long long) length)) < 0) {
+		prt("%punch hole: %x to %x\n", offset, length);
+		prterrcode("do_punch_hole: discard", ret);
+		report_failure(161);
+	}
+
+
+	max_offset = offset < file_size ? offset : file_size;
+	max_len = max_offset + length <= file_size ? length :
+			file_size - max_offset;
+	memset(good_buf + max_offset, '\0', max_len);
+}
+
+void clone_filename(char *buf, size_t len, int clones)
+{
+	snprintf(buf, len, "%s/fsx-%s-parent%d",
+		 dirpath, iname, clones);
+}
+
+void clone_imagename(char *buf, size_t len, int clones)
+{
+	if (clones > 0)
+		snprintf(buf, len, "%s-clone%d", iname, clones);
+	else
+		strncpy(buf, iname, len);
+}
+
+void check_clone(int clonenum);
+
+void
+do_clone()
+{
+	char filename[1024];
+	char imagename[1024];
+	char lastimagename[1024];
+	int ret, fd;
+	int order = 0, stripe_unit = 0, stripe_count = 0;
+
+	if (randomize_striping) {
+		order = 18 + rand() % 8;
+		stripe_unit = 1ull << (order - 1 - (rand() % 8));
+		stripe_count = 2 + rand() % 14;
+	}
+
+	log4(OP_CLONE, 0, 0, 0);
+	++num_clones;
+	prt("%lu clone\t%d order %d su %d sc %d\n", testcalls, num_clones, order, stripe_unit, stripe_count);
+
+	clone_filename(filename, sizeof(filename), num_clones);
+	if ((fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) {
+		simple_err("do_clone: open", -errno);
+		exit(162);
+	}
+	save_buffer(good_buf, file_size, fd);
+	if ((ret = close(fd)) < 0) {
+		simple_err("do_clone: close", -errno);
+		exit(163);
+	}
+
+	clone_imagename(imagename, sizeof(imagename), num_clones);
+	clone_imagename(lastimagename, sizeof(lastimagename),
+			num_clones - 1);
+
+	if ((ret = rbd_snap_create(image, "snap")) < 0) {
+		simple_err("do_clone: rbd create snap", ret);
+		exit(164);
+	}
+
+	if ((ret = rbd_snap_protect(image, "snap")) < 0) {
+		simple_err("do_clone: rbd protect snap", ret);
+		exit(164);
+	}
+
+	ret = rbd_clone2(ioctx, lastimagename, "snap", ioctx, imagename,
+			 RBD_FEATURES_ALL, &order, stripe_unit, stripe_count);
+	if (ret < 0) {
+		simple_err("do_clone: rbd clone", ret);
+		exit(165);
+	}
+
+	if ((ret = rbd_close(image)) < 0) {
+		simple_err("do_clone: rbd close", ret);
+		exit(174);
+	}
+
+	if ((ret = rbd_open(ioctx, imagename, &image, NULL)) < 0) {
+		simple_err("do_clone: rbd open", ret);
+		exit(166);
+	}
+
+	if (num_clones > 1)
+		check_clone(num_clones - 2);
+}
+
+void
+check_clone(int clonenum)
+{
+	char filename[128];
+	char imagename[128];
+	int ret, fd;
+	rbd_image_t cur_image;
+	struct stat file_info;
+	char *good_buf, *temp_buf;
+
+	clone_imagename(imagename, sizeof(imagename), clonenum);
+	if ((ret = rbd_open(ioctx, imagename, &cur_image, NULL)) < 0) {
+		simple_err("check_clone: rbd open", ret);
+		exit(167);
+	}
+
+	clone_filename(filename, sizeof(filename), clonenum + 1);
+	if ((fd = open(filename, O_RDONLY)) < 0) {
+		simple_err("check_clone: open", -errno);
+		exit(168);
+	}
+
+	prt("checking clone #%d, image %s against file %s\n",
+	    clonenum, imagename, filename);
+	if ((ret = fstat(fd, &file_info)) < 0) {
+		simple_err("check_clone: fstat", -errno);
+		exit(169);
+	}
+
+	good_buf = malloc(file_info.st_size);
+	temp_buf = malloc(file_info.st_size);
+
+	if ((ret = pread(fd, good_buf, file_info.st_size, 0)) < 0) {
+		simple_err("check_clone: pread", -errno);
+		exit(170);
+	}
+	if ((ret = rbd_read(cur_image, 0, file_info.st_size, temp_buf)) < 0) {
+		simple_err("check_clone: rbd_read", ret);
+		exit(171);
+	}
+	close(fd);
+	if ((ret = rbd_close(cur_image)) < 0) {
+		simple_err("check_clone: rbd close", ret);
+		exit(174);
+	}
+	check_buffers(good_buf, temp_buf, 0, file_info.st_size);
+
+	unlink(filename);
+
+	free(good_buf);
+	free(temp_buf);
+}
+
+void
+writefileimage()
+{
+	ssize_t ret;
+
+	ret = rbd_write(image, 0, file_size, good_buf);
+	if (ret != file_size) {
+		if (ret < 0)
+			prterrcode("writefileimage: write", ret);
+		else
+			prt("short write: 0x%x bytes instead of 0x%llx\n",
+			    ret, (unsigned long long)file_size);
+		report_failure(172);
+	}
+	if (lite ? 0 : (ret = rbd_resize(image, file_size)) < 0) {
+		prt("rbd_resize: %llx\n", (unsigned long long)file_size);
+		prterrcode("writefileimage: rbd_resize", ret);
+		report_failure(173);
+	}
+}
+
+void
+do_flatten()
+{
+	int ret;
+
+	if (num_clones == 0 ||
+	    (rbd_get_parent_info(image, NULL, 0, NULL, 0, NULL, 0)
+	    == -ENOENT)) {
+		log4(OP_SKIPPED, OP_FLATTEN, 0, 0);
+		return;
+	}
+	log4(OP_FLATTEN, 0, 0, 0);
+	prt("%lu flatten\n", testcalls);
+
+	if ((ret = rbd_flatten(image)) < 0) {
+		simple_err("do_flatten: rbd flatten", ret);
+		exit(177);
+	}
+}
+
+void
+docloseopen(void)
+{
+	int ret;
+
+	if (testcalls <= simulatedopcount)
+		return;
+
+	if (debug)
+		prt("%lu close/open\n", testcalls);
+	if ((ret = rbd_close(image)) < 0) {
+		prterrcode("docloseopen: close", ret);
+		report_failure(180);
+	}
+	ret = rbd_open(ioctx, iname, &image, NULL);
+	if (ret < 0) {
+		prterrcode("docloseopen: open", ret);
+		report_failure(181);
+	}
+}
+
+#define TRIM_OFF_LEN(off, len, size)	\
+do {					\
+	if (size)			\
+		(off) %= (size);	\
+	else				\
+		(off) = 0;		\
+	if ((unsigned)(off) + (unsigned)(len) > (unsigned)(size))	\
+		(len) = (size) - (off);	\
+} while (0)
+
+void
+test(void)
+{
+	unsigned long	offset;
+	unsigned long	size = maxoplen;
+	unsigned long	rv = random();
+	unsigned long	op;
+
+	if (simulatedopcount > 0 && testcalls == simulatedopcount)
+		writefileimage();
+
+	testcalls++;
+
+	if (closeprob)
+		closeopen = (rv >> 3) < (1u << 28) / (unsigned)closeprob;
+
+	if (debugstart > 0 && testcalls >= debugstart)
+		debug = 1;
+
+	if (!quiet && testcalls < simulatedopcount && testcalls % 100000 == 0)
+		prt("%lu...\n", testcalls);
+
+	offset = random();
+	if (randomoplen)
+		size = random() % (maxoplen + 1);
+
+	/* calculate appropriate op to run */
+	if (lite)
+		op = rv % OP_MAX_LITE;
+	else
+		op = rv % OP_MAX_FULL;
+
+	switch (op) {
+	case OP_MAPREAD:
+		if (!mapped_reads)
+			op = OP_READ;
+		break;
+	case OP_MAPWRITE:
+		if (!mapped_writes)
+			op = OP_WRITE;
+		break;
+	case OP_FALLOCATE:
+		if (!fallocate_calls) {
+			log4(OP_SKIPPED, OP_FALLOCATE, offset, size);
+			goto out;
+		}
+		break;
+	case OP_PUNCH_HOLE:
+		if (!punch_hole_calls) {
+			log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, size);
+			goto out;
+		}
+		break;
+	case OP_CLONE:
+		if (!clone_calls || random() % 100 > 5 || file_size == 0) {
+			log4(OP_SKIPPED, OP_CLONE, 0, 0);
+			goto out;
+		}
+		break;
+	}
+
+	switch (op) {
+	case OP_READ:
+		TRIM_OFF_LEN(offset, size, file_size);
+		doread(offset, size);
+		break;
+
+	case OP_WRITE:
+		TRIM_OFF_LEN(offset, size, maxfilelen);
+		dowrite(offset, size);
+		break;
+
+	case OP_MAPREAD:
+		TRIM_OFF_LEN(offset, size, file_size);
+		exit(183);
+		break;
+
+	case OP_MAPWRITE:
+		TRIM_OFF_LEN(offset, size, maxfilelen);
+		exit(182);
+		break;
+
+	case OP_TRUNCATE:
+		if (!style)
+			size = random() % maxfilelen;
+		dotruncate(size);
+		break;
+
+	case OP_PUNCH_HOLE:
+		TRIM_OFF_LEN(offset, size, file_size);
+		do_punch_hole(offset, size);
+		break;
+
+	case OP_CLONE:
+		do_clone();
+		break;
+
+	case OP_FLATTEN:
+		do_flatten();
+		break;
+
+	default:
+		prterr("test: unknown operation");
+		report_failure(42);
+		break;
+	}
+
+out:
+	if (sizechecks && testcalls > simulatedopcount)
+		check_size();
+	if (closeopen)
+		docloseopen();
+}
+
+
+void
+cleanup(sig)
+	int	sig;
+{
+	if (sig)
+		prt("signal %d\n", sig);
+	prt("testcalls = %lu\n", testcalls);
+	exit(sig);
+}
+
+
+void
+usage(void)
+{
+	fprintf(stdout, "usage: %s",
+		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] pname iname\n\
+	-b opnum: beginning operation number (default 1)\n\
+	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
+	-d: debug output for all operations\n\
+	-f flush and invalidate cache after I/O\n\
+	-l flen: the upper bound on file size (default 262144)\n\
+	-m startop:endop: monitor (print debug output) specified byte range (default 0:infinity)\n\
+	-n: no verifications of file size\n\
+	-o oplen: the upper bound on operation size (default 65536)\n\
+	-p progressinterval: debug output at specified operation interval\n\
+	-q: quieter operation\n\
+	-r readbdy: 4096 would make reads page aligned (default 1)\n\
+	-s style: 1 gives smaller truncates (default 0)\n\
+	-t truncbdy: 4096 would make truncates page aligned (default 1)\n\
+	-w writebdy: 4096 would make writes page aligned (default 1)\n\
+	-x: preallocate file space before starting, XFS only (default 0)\n\
+	-y synchronize changes to a file\n"
+
+#ifdef AIO
+"	-A: Use the AIO system calls\n"
+#endif
+"	-D startingop: debug output starting at specified operation\n"
+#ifdef FALLOCATE
+"	-F: Do not use fallocate (preallocation) calls\n"
+#endif
+"        -H: Do not use punch hole calls\n"
+"        -C: Do not use clone calls\n"
+"	-L: fsxLite - no file creations & no file size changes\n\
+	-N numops: total # operations to do (default infinity)\n\
+	-O: use oplen (see -o flag) for every op (default random)\n\
+	-P: save .fsxlog and .fsxgood files in dirpath (default ./)\n\
+	-S seed: for random # generator (default 1) 0 gets timestamp\n\
+	-W: mapped write operations DISabled\n\
+        -R: read() system calls only (mapped reads disabled)\n\
+        -Z: O_DIRECT (use -R, -W, -r and -w too)\n\
+	poolname: this is REQUIRED (no default)\n\
+	imagename: this is REQUIRED (no default)\n");
+	exit(89);
+}
+
+
+int
+getnum(char *s, char **e)
+{
+	int ret;
+
+	*e = (char *) 0;
+	ret = strtol(s, e, 0);
+	if (*e)
+		switch (**e) {
+		case 'b':
+		case 'B':
+			ret *= 512;
+			*e = *e + 1;
+			break;
+		case 'k':
+		case 'K':
+			ret *= 1024;
+			*e = *e + 1;
+			break;
+		case 'm':
+		case 'M':
+			ret *= 1024*1024;
+			*e = *e + 1;
+			break;
+		case 'w':
+		case 'W':
+			ret *= 4;
+			*e = *e + 1;
+			break;
+		}
+	return (ret);
+}
+
+#ifdef AIO
+
+#define QSZ     1024
+io_context_t	io_ctx;
+struct iocb 	iocb;
+
+int aio_setup()
+{
+	int ret;
+	ret = io_queue_init(QSZ, &io_ctx);
+	if (ret != 0) {
+		fprintf(stderr, "aio_setup: io_queue_init failed: %s\n",
+                        strerror(ret));
+                return(-1);
+        }
+        return(0);
+}
+
+int
+__aio_rw(int rw, int fd, char *buf, unsigned len, unsigned offset)
+{
+	struct io_event event;
+	static struct timespec ts;
+	struct iocb *iocbs[] = { &iocb };
+	int ret;
+	long res;
+
+	if (rw == READ) {
+		io_prep_pread(&iocb, fd, buf, len, offset);
+	} else {
+		io_prep_pwrite(&iocb, fd, buf, len, offset);
+	}
+
+	ts.tv_sec = 30;
+	ts.tv_nsec = 0;
+	ret = io_submit(io_ctx, 1, iocbs);
+	if (ret != 1) {
+		fprintf(stderr, "errcode=%d\n", ret);
+		fprintf(stderr, "aio_rw: io_submit failed: %s\n",
+				strerror(ret));
+		goto out_error;
+	}
+
+	ret = io_getevents(io_ctx, 1, 1, &event, &ts);
+	if (ret != 1) {
+		if (ret == 0)
+			fprintf(stderr, "aio_rw: no events available\n");
+		else {
+			fprintf(stderr, "errcode=%d\n", -ret);
+			fprintf(stderr, "aio_rw: io_getevents failed: %s\n",
+				 	strerror(-ret));
+		}
+		goto out_error;
+	}
+	if (len != event.res) {
+		/*
+		 * The b0rked libaio defines event.res as unsigned.
+		 * However the kernel strucuture has it signed,
+		 * and it's used to pass negated error value.
+		 * Till the library is fixed use the temp var.
+		 */
+		res = (long)event.res;
+		if (res >= 0)
+			fprintf(stderr, "bad io length: %lu instead of %u\n",
+					res, len);
+		else {
+			fprintf(stderr, "errcode=%ld\n", -res);
+			fprintf(stderr, "aio_rw: async io failed: %s\n",
+					strerror(-res));
+			ret = res;
+			goto out_error;
+		}
+
+	}
+	return event.res;
+
+out_error:
+	/*
+	 * The caller expects error return in traditional libc
+	 * convention, i.e. -1 and the errno set to error.
+	 */
+	errno = -ret;
+	return -1;
+}
+
+int aio_rw(int rw, int fd, char *buf, unsigned len, unsigned offset)
+{
+	int ret;
+
+	if (aio) {
+		ret = __aio_rw(rw, fd, buf, len, offset);
+	} else {
+		if (rw == READ)
+			ret = read(fd, buf, len);
+		else
+			ret = write(fd, buf, len);
+	}
+	return ret;
+}
+
+#endif
+
+void
+test_fallocate()
+{
+#ifdef FALLOCATE
+	if (!lite && fallocate_calls) {
+		if (fallocate(fd, 0, 0, 1) && errno == EOPNOTSUPP) {
+			if(!quiet)
+				warn("main: filesystem does not support fallocate, disabling\n");
+			fallocate_calls = 0;
+		} else {
+			ftruncate(fd, 0);
+		}
+	}
+#else /* ! FALLOCATE */
+	fallocate_calls = 0;
+#endif
+
+}
+
+int
+main(int argc, char **argv)
+{
+	int	i, style, ch, ret;
+	char	*endp;
+	char goodfile[1024];
+	char logfile[1024];
+
+	goodfile[0] = 0;
+	logfile[0] = 0;
+
+	page_size = getpagesize();
+	page_mask = page_size - 1;
+	mmap_mask = page_mask;
+
+	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
+
+	while ((ch = getopt(argc, argv, "b:c:dfl:m:no:p:qr:s:t:w:xyACD:FHLN:OP:RS:WZ"))
+	       != EOF)
+		switch (ch) {
+		case 'b':
+			simulatedopcount = getnum(optarg, &endp);
+			if (!quiet)
+				fprintf(stdout, "Will begin at operation %ld\n",
+					simulatedopcount);
+			if (simulatedopcount == 0)
+				usage();
+			simulatedopcount -= 1;
+			break;
+		case 'c':
+			closeprob = getnum(optarg, &endp);
+			if (!quiet)
+				fprintf(stdout,
+					"Chance of close/open is 1 in %d\n",
+					closeprob);
+			if (closeprob <= 0)
+				usage();
+			break;
+		case 'd':
+			debug = 1;
+			break;
+		case 'f':
+			flush = 1;
+			break;
+		case 'l':
+			maxfilelen = getnum(optarg, &endp);
+			if (maxfilelen <= 0)
+				usage();
+			break;
+		case 'm':
+			monitorstart = getnum(optarg, &endp);
+			if (monitorstart < 0)
+				usage();
+			if (!endp || *endp++ != ':')
+				usage();
+			monitorend = getnum(endp, &endp);
+			if (monitorend < 0)
+				usage();
+			if (monitorend == 0)
+				monitorend = -1; /* aka infinity */
+			debug = 1;
+			break;
+		case 'n':
+			sizechecks = 0;
+			break;
+		case 'o':
+			maxoplen = getnum(optarg, &endp);
+			if (maxoplen <= 0)
+				usage();
+			break;
+		case 'p':
+			progressinterval = getnum(optarg, &endp);
+			if (progressinterval == 0)
+				usage();
+			break;
+		case 'q':
+			quiet = 1;
+			break;
+		case 'r':
+			readbdy = getnum(optarg, &endp);
+			if (readbdy <= 0)
+				usage();
+			break;
+		case 's':
+			style = getnum(optarg, &endp);
+			if (style < 0 || style > 1)
+				usage();
+			break;
+		case 't':
+			truncbdy = getnum(optarg, &endp);
+			if (truncbdy <= 0)
+				usage();
+			break;
+		case 'w':
+			writebdy = getnum(optarg, &endp);
+			if (writebdy <= 0)
+				usage();
+			break;
+		case 'x':
+			prealloc = 1;
+			break;
+		case 'y':
+			do_fsync = 1;
+			break;
+		case 'A':
+		        aio = 1;
+			break;
+		case 'C':
+			clone_calls = 0;
+			break;
+		case 'D':
+			debugstart = getnum(optarg, &endp);
+			if (debugstart < 1)
+				usage();
+			break;
+		case 'F':
+			fallocate_calls = 0;
+			break;
+		case 'H':
+			punch_hole_calls = 0;
+			break;
+		case 'L':
+			prt("lite mode not supported for rbd\n");
+			exit(1);
+			break;
+		case 'N':
+			numops = getnum(optarg, &endp);
+			if (numops < 0)
+				usage();
+			break;
+		case 'O':
+			randomoplen = 0;
+			break;
+		case 'P':
+			strncpy(dirpath, optarg, sizeof(dirpath));
+			strncpy(goodfile, dirpath, sizeof(goodfile));
+			strcat(goodfile, "/");
+			strncpy(logfile, dirpath, sizeof(logfile));
+			strcat(logfile, "/");
+			break;
+                case 'R':
+                        mapped_reads = 0;
+                        break;
+		case 'S':
+                        seed = getnum(optarg, &endp);
+			if (seed == 0)
+				seed = time(0) % 10000;
+			if (!quiet)
+				fprintf(stdout, "Seed set to %d\n", seed);
+			if (seed < 0)
+				usage();
+			break;
+		case 'W':
+		        mapped_writes = 0;
+			if (!quiet)
+				fprintf(stdout, "mapped writes DISABLED\n");
+			break;
+		case 'Z':
+			o_direct = O_DIRECT;
+			break;
+		default:
+			usage();
+			/* NOTREACHED */
+		}
+	argc -= optind;
+	argv += optind;
+	if (argc != 2)
+		usage();
+	pool = argv[0];
+	iname = argv[1];
+
+	signal(SIGHUP,	cleanup);
+	signal(SIGINT,	cleanup);
+	signal(SIGPIPE,	cleanup);
+	signal(SIGALRM,	cleanup);
+	signal(SIGTERM,	cleanup);
+	signal(SIGXCPU,	cleanup);
+	signal(SIGXFSZ,	cleanup);
+	signal(SIGVTALRM,	cleanup);
+	signal(SIGUSR1,	cleanup);
+	signal(SIGUSR2,	cleanup);
+
+	initstate(seed, state, 256);
+	setstate(state);
+
+	ret = create_image();
+	if (ret < 0) {
+		prterrcode(iname, ret);
+		exit(90);
+	}
+	ret = rbd_open(ioctx, iname, &image, NULL);
+	if (ret < 0) {
+		simple_err("Error opening image", ret);
+		exit(91);
+	}
+	if (!dirpath[0])
+		strcat(dirpath, ".");
+	strncat(goodfile, iname, 256);
+	strcat (goodfile, ".fsxgood");
+	fsxgoodfd = open(goodfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	if (fsxgoodfd < 0) {
+		prterr(goodfile);
+		exit(92);
+	}
+	strncat(logfile, iname, 256);
+	strcat (logfile, ".fsxlog");
+	fsxlogf = fopen(logfile, "w");
+	if (fsxlogf == NULL) {
+		prterr(logfile);
+		exit(93);
+	}
+
+#ifdef AIO
+	if (aio) 
+		aio_setup();
+#endif
+
+	original_buf = (char *) malloc(maxfilelen);
+	for (i = 0; i < (int)maxfilelen; i++)
+		original_buf[i] = random() % 256;
+	good_buf = (char *) malloc(maxfilelen + writebdy);
+	good_buf = round_ptr_up(good_buf, writebdy, 0);
+	memset(good_buf, '\0', maxfilelen);
+	temp_buf = (char *) malloc(maxfilelen + writebdy);
+	temp_buf = round_ptr_up(temp_buf, writebdy, 0);
+	memset(temp_buf, '\0', maxfilelen);
+	if (lite) {	/* zero entire existing file */
+		ssize_t written;
+
+		written = rbd_write(image, 0, (size_t)maxfilelen, good_buf);
+		if (written != (ssize_t)maxfilelen) {
+			if (written < 0) {
+				prterrcode(iname, written);
+				warn("main: error on write");
+			} else
+				warn("main: short write, 0x%x bytes instead "
+					"of 0x%lx\n",
+					(unsigned)written,
+					maxfilelen);
+			exit(98);
+		}
+	} else
+		check_trunc_hack();
+
+	//test_fallocate();
+
+	while (numops == -1 || numops--)
+		test();
+
+	if ((ret = rbd_close(image)) < 0) {
+		prterrcode("rbd_close", ret);
+		report_failure(99);
+	}
+
+	if (num_clones > 0)
+		check_clone(num_clones - 1);
+
+	while (num_clones >= 0) {
+		static int first = 1;
+		char clonename[128];
+		char errmsg[128];
+		clone_imagename(clonename, 128, num_clones);
+		if ((ret = rbd_open(ioctx, clonename, &image, NULL)) < 0) {
+			sprintf(errmsg, "rbd_open %s", clonename);
+			prterrcode(errmsg, ret);
+			report_failure(101);
+		}
+		if (!first) {
+			if ((ret = rbd_snap_unprotect(image, "snap")) < 0) {
+				sprintf(errmsg, "rbd_snap_unprotect %s@snap",
+					clonename);
+				prterrcode(errmsg, ret);
+			report_failure(102);
+			}
+			if ((ret = rbd_snap_remove(image, "snap")) < 0) {
+				sprintf(errmsg, "rbd_snap_remove %s@snap",
+					clonename);
+				prterrcode(errmsg, ret);
+				report_failure(103);
+			}
+		}
+		if ((ret = rbd_close(image)) < 0) {
+			sprintf(errmsg, "rbd_close %s", clonename);
+			prterrcode(errmsg, ret);
+			report_failure(104);
+		}
+
+		if ((ret = rbd_remove(ioctx, clonename)) < 0) {
+			sprintf(errmsg, "rbd_remove %s", clonename);
+			prterrcode(errmsg, ret);
+			report_failure(105);
+		}
+
+		first = 0;
+		num_clones--;
+	}
+
+	rados_ioctx_destroy(ioctx);
+	rados_shutdown(cluster);
+
+	free(original_buf);
+	free(good_buf);
+	free(temp_buf);
+
+	prt("All operations completed A-OK!\n");
+	fclose(fsxlogf);
+
+	exit(0);
+	return 0;
+}
diff --git a/ceph/src/test/librbd/test_librbd.cc b/ceph/src/test/librbd/test_librbd.cc
new file mode 100644
index 00000000..7f354187
--- /dev/null
+++ b/ceph/src/test/librbd/test_librbd.cc
@@ -0,0 +1,1873 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/int_types.h"
+#include "include/rados/librados.h"
+#include "include/rbd_types.h"
+#include "include/rbd/librbd.h"
+#include "include/rbd/librbd.hpp"
+
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+
+#include "gtest/gtest.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "test/librados/test.h"
+#include "common/errno.h"
+#include "include/interval_set.h"
+#include "include/stringify.h"
+
+using namespace std;
+
+static int get_features(bool *old_format, uint64_t *features)
+{
+  const char *c = getenv("RBD_FEATURES");
+  if (c) {
+    stringstream ss;
+    ss << c;
+    ss >> *features;
+    if (ss.fail())
+      return -EINVAL;
+    *old_format = false;
+    cout << "using new format!" << std::endl;
+  } else {
+    *old_format = true;
+    cout << "using old format" << std::endl;
+  }
+
+  return 0;
+}
+
+static int create_image_full(rados_ioctx_t ioctx, const char *name,
+			      uint64_t size, int *order, int old_format,
+			      uint64_t features)
+{
+  if (old_format) {
+    return rbd_create(ioctx, name, size, order);
+  } else {
+    return rbd_create2(ioctx, name, size, features, order);
+  }
+}
+
+static int create_image(rados_ioctx_t ioctx, const char *name,
+			uint64_t size, int *order)
+{
+  bool old_format;
+  uint64_t features;
+
+  int r = get_features(&old_format, &features);
+  if (r < 0)
+    return r;
+  return create_image_full(ioctx, name, size, order, old_format, features);
+}
+
+static int create_image_pp(librbd::RBD &rbd,
+			   librados::IoCtx &ioctx,
+			   const char *name,
+			   uint64_t size, int *order) {
+  bool old_format;
+  uint64_t features;
+  int r = get_features(&old_format, &features);
+  if (r < 0)
+    return r;
+  if (old_format) {
+    return rbd.create(ioctx, name, size, order);
+  } else {
+    return rbd.create2(ioctx, name, size, features, order);
+  }
+}
+
+TEST(LibRBD, CreateAndStat)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+
+  rbd_image_info_t info;
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+  ASSERT_EQ(0, rbd_stat(image, &info, sizeof(info)));
+  printf("image has size %llu and order %d\n", (unsigned long long) info.size, info.order);
+  ASSERT_EQ(info.size, size);
+  ASSERT_EQ(info.order, order);
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRBD, CreateAndStatPP)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::image_info_t info;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 2 << 20;
+    
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+    ASSERT_EQ(0, image.stat(info, sizeof(info)));
+    ASSERT_EQ(info.size, size);
+    ASSERT_EQ(info.order, order);
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(LibRBD, ResizeAndStat)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_info_t info;
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  ASSERT_EQ(0, rbd_resize(image, size * 4));
+  ASSERT_EQ(0, rbd_stat(image, &info, sizeof(info)));
+  ASSERT_EQ(info.size, size * 4);
+
+  ASSERT_EQ(0, rbd_resize(image, size / 2));
+  ASSERT_EQ(0, rbd_stat(image, &info, sizeof(info)));
+  ASSERT_EQ(info.size, size / 2);
+  
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRBD, ResizeAndStatPP)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::image_info_t info;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 2 << 20;
+    
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+    
+    ASSERT_EQ(0, image.resize(size * 4));
+    ASSERT_EQ(0, image.stat(info, sizeof(info)));
+    ASSERT_EQ(info.size, size * 4);
+    
+    ASSERT_EQ(0, image.resize(size / 2));
+    ASSERT_EQ(0, image.stat(info, sizeof(info)));
+    ASSERT_EQ(info.size, size / 2);
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+int test_ls(rados_ioctx_t io_ctx, size_t num_expected, ...)
+{
+  int num_images, i, j;
+  char *names, *cur_name;
+  va_list ap;
+  size_t max_size = 1024;
+
+  names = (char *) malloc(sizeof(char *) * 1024);
+  int len = rbd_list(io_ctx, names, &max_size);
+
+  for (i = 0, num_images = 0, cur_name = names; cur_name < names + len; i++) {
+    printf("image: %s\n", cur_name);
+    cur_name += strlen(cur_name) + 1;
+    num_images++;
+  }
+
+  va_start(ap, num_expected);
+  for (i = num_expected; i > 0; i--) {
+    char *expected = va_arg(ap, char *);
+    printf("expected = %s\n", expected);
+    int found = 0;
+    for (j = 0, cur_name = names; j < num_images; j++) {
+      if (cur_name[0] == '_') {
+	cur_name += strlen(cur_name) + 1;
+	continue;
+      }
+      if (strcmp(cur_name, expected) == 0) {
+	printf("found %s\n", cur_name);
+	cur_name[0] = '_';
+	found = 1;
+	break;
+      }
+    }
+    assert(found);
+  }
+  va_end(ap);
+
+  for (i = 0, cur_name = names; cur_name < names + len; i++) {
+    assert(cur_name[0] == '_');
+    cur_name += strlen(cur_name) + 1;
+  }
+  free(names);
+
+  return num_images;
+}
+
+TEST(LibRBD, TestCreateLsDelete)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  int order = 0;
+  const char *name = "testimg";
+  const char *name2 = "testimg2";
+  uint64_t size = 2 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(1, test_ls(ioctx, 1, name));
+  ASSERT_EQ(0, create_image(ioctx, name2, size, &order));
+  ASSERT_EQ(2, test_ls(ioctx, 2, name, name2));
+  ASSERT_EQ(0, rbd_remove(ioctx, name));
+  ASSERT_EQ(1, test_ls(ioctx, 1, name2));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+int test_ls_pp(librbd::RBD& rbd, librados::IoCtx& io_ctx, size_t num_expected, ...)
+{
+  int r;
+  size_t i;
+  va_list ap;
+  vector names;
+  r = rbd.list(io_ctx, names);
+  if (r == -ENOENT)
+    r = 0;
+  assert(r >= 0);
+  cout << "num images is: " << names.size() << endl
+	    << "expected: " << num_expected << endl;
+  int num = names.size();
+
+  for (i = 0; i < names.size(); i++) {
+    cout << "image: " << names[i] << endl;
+  }
+
+  va_start(ap, num_expected);
+  for (i = num_expected; i > 0; i--) {
+    char *expected = va_arg(ap, char *);
+    cout << "expected = " << expected << endl;
+    vector::iterator listed_name = find(names.begin(), names.end(), string(expected));
+    assert(listed_name != names.end());
+    names.erase(listed_name);
+  }
+  va_end(ap);
+
+  assert(names.empty());
+
+  return num;
+}
+
+TEST(LibRBD, TestCreateLsDeletePP)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    const char *name2 = "testimg2";
+    uint64_t size = 2 << 20;  
+    
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(1, test_ls_pp(rbd, ioctx, 1, name));
+    ASSERT_EQ(0, rbd.create(ioctx, name2, size, &order));
+    ASSERT_EQ(2, test_ls_pp(rbd, ioctx, 2, name, name2));
+    ASSERT_EQ(0, rbd.remove(ioctx, name));
+    ASSERT_EQ(1, test_ls_pp(rbd, ioctx, 1, name2));
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+
+static int print_progress_percent(uint64_t offset, uint64_t src_size,
+				     void *data)
+{
+  float percent = ((float)offset * 100) / src_size;
+  printf("%3.2f%% done\n", percent);
+  return 0; 
+}
+
+TEST(LibRBD, TestCopy)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  const char *name2 = "testimg2";
+  const char *name3 = "testimg3";
+
+  uint64_t size = 2 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+  ASSERT_EQ(1, test_ls(ioctx, 1, name));
+  ASSERT_EQ(0, rbd_copy(image, ioctx, name2));
+  ASSERT_EQ(2, test_ls(ioctx, 2, name, name2));
+  ASSERT_EQ(0, rbd_copy_with_progress(image, ioctx, name3, print_progress_percent, NULL));
+  ASSERT_EQ(3, test_ls(ioctx, 3, name, name2, name3));
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+class PrintProgress : public librbd::ProgressContext
+{
+public:
+  int update_progress(uint64_t offset, uint64_t src_size)
+  {
+    float percent = ((float)offset * 100) / src_size;
+    printf("%3.2f%% done\n", percent);
+    return 0;
+  }
+};
+
+TEST(LibRBD, TestCopyPP)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+  
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    const char *name2 = "testimg2";
+    const char *name3 = "testimg3";
+    uint64_t size = 2 << 20;
+    PrintProgress pp;
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+    ASSERT_EQ(1, test_ls_pp(rbd, ioctx, 1, name));
+    ASSERT_EQ(0, image.copy(ioctx, name2));
+    ASSERT_EQ(2, test_ls_pp(rbd, ioctx, 2, name, name2));
+    ASSERT_EQ(0, image.copy_with_progress(ioctx, name3, pp));
+    ASSERT_EQ(3, test_ls_pp(rbd, ioctx, 3, name, name2, name3));
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+int test_ls_snaps(rbd_image_t image, int num_expected, ...)
+{
+  int num_snaps, i, j, max_size = 10;
+  va_list ap;
+  rbd_snap_info_t snaps[max_size];
+  num_snaps = rbd_snap_list(image, snaps, &max_size);
+  printf("num snaps is: %d\nexpected: %d\n", num_snaps, num_expected);
+
+  for (i = 0; i < num_snaps; i++) {
+    printf("snap: %s\n", snaps[i].name);
+  }
+
+  va_start(ap, num_expected);
+  for (i = num_expected; i > 0; i--) {
+    char *expected = va_arg(ap, char *);
+    uint64_t expected_size = va_arg(ap, uint64_t);
+    int found = 0;
+    for (j = 0; j < num_snaps; j++) {
+      if (snaps[j].name == NULL)
+	continue;
+      if (strcmp(snaps[j].name, expected) == 0) {
+	printf("found %s with size %llu\n", snaps[j].name, (unsigned long long) snaps[j].size);
+	assert(snaps[j].size == expected_size);
+	free((void *) snaps[j].name);
+	snaps[j].name = NULL;
+	found = 1;
+	break;
+      }
+    }
+    assert(found);
+  }
+  va_end(ap);
+
+  for (i = 0; i < num_snaps; i++) {
+    assert(snaps[i].name == NULL);
+  }
+
+  return num_snaps;
+}
+
+TEST(LibRBD, TestCreateLsDeleteSnap)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+  uint64_t size2 = 4 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
+  ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
+  ASSERT_EQ(0, rbd_resize(image, size2));
+  ASSERT_EQ(0, rbd_snap_create(image, "snap2"));
+  ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
+  ASSERT_EQ(0, rbd_snap_remove(image, "snap1"));
+  ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2", size2));
+  ASSERT_EQ(0, rbd_snap_remove(image, "snap2"));
+  ASSERT_EQ(0, test_ls_snaps(image, 0));
+  
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+int test_ls_snaps(librbd::Image& image, size_t num_expected, ...)
+{
+  int r;
+  size_t i, j;
+  va_list ap;
+  vector snaps;
+  r = image.snap_list(snaps);
+  assert(r >= 0);
+  cout << "num snaps is: " << snaps.size() << endl
+	    << "expected: " << num_expected << endl;
+
+  for (i = 0; i < snaps.size(); i++) {
+    cout << "snap: " << snaps[i].name << endl;
+  }
+
+  va_start(ap, num_expected);
+  for (i = num_expected; i > 0; i--) {
+    char *expected = va_arg(ap, char *);
+    uint64_t expected_size = va_arg(ap, uint64_t);
+    int found = 0;
+    for (j = 0; j < snaps.size(); j++) {
+      if (snaps[j].name == "")
+	continue;
+      if (strcmp(snaps[j].name.c_str(), expected) == 0) {
+	cout << "found " << snaps[j].name << " with size " << snaps[j].size << endl;
+	assert(snaps[j].size == expected_size);
+	snaps[j].name = "";
+	found = 1;
+	break;
+      }
+    }
+    assert(found);
+  }
+  va_end(ap);
+
+  for (i = 0; i < snaps.size(); i++) {
+    assert(snaps[i].name == "");
+  }
+
+  return snaps.size();
+}
+
+TEST(LibRBD, TestCreateLsDeleteSnapPP)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 2 << 20;
+    uint64_t size2 = 4 << 20;
+    
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+
+    ASSERT_FALSE(image.snap_exists("snap1"));
+    ASSERT_EQ(0, image.snap_create("snap1"));
+    ASSERT_TRUE(image.snap_exists("snap1"));
+    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap1", size));
+    ASSERT_EQ(0, image.resize(size2));
+    ASSERT_FALSE(image.snap_exists("snap2"));
+    ASSERT_EQ(0, image.snap_create("snap2"));
+    ASSERT_TRUE(image.snap_exists("snap2"));
+    ASSERT_EQ(2, test_ls_snaps(image, 2, "snap1", size, "snap2", size2));
+    ASSERT_EQ(0, image.snap_remove("snap1"));
+    ASSERT_FALSE(image.snap_exists("snap1"));
+    ASSERT_EQ(1, test_ls_snaps(image, 1, "snap2", size2));
+    ASSERT_EQ(0, image.snap_remove("snap2"));
+    ASSERT_FALSE(image.snap_exists("snap2"));
+    ASSERT_EQ(0, test_ls_snaps(image, 0));
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+
+
+#define TEST_IO_SIZE 512
+#define TEST_IO_TO_SNAP_SIZE 80
+
+void simple_write_cb(rbd_completion_t cb, void *arg)
+{
+  printf("write completion cb called!\n");
+}
+
+void simple_read_cb(rbd_completion_t cb, void *arg)
+{
+  printf("read completion cb called!\n");
+}
+
+void aio_write_test_data(rbd_image_t image, const char *test_data, uint64_t off, size_t len)
+{
+  rbd_completion_t comp;
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
+  printf("created completion\n");
+  rbd_aio_write(image, off, len, test_data, comp);
+  printf("started write\n");
+  rbd_aio_wait_for_complete(comp);
+  int r = rbd_aio_get_return_value(comp);
+  printf("return value is: %d\n", r);
+  assert(r == 0);
+  printf("finished write\n");
+  rbd_aio_release(comp);
+}
+
+void write_test_data(rbd_image_t image, const char *test_data, uint64_t off, size_t len)
+{
+  ssize_t written;
+  written = rbd_write(image, off, len, test_data);
+  printf("wrote: %d\n", (int) written);
+  assert(written == (ssize_t)len);
+}
+
+void aio_discard_test_data(rbd_image_t image, uint64_t off, uint64_t len)
+{
+  rbd_completion_t comp;
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_write_cb, &comp);
+  rbd_aio_discard(image, off, len, comp);
+  rbd_aio_wait_for_complete(comp);
+  int r = rbd_aio_get_return_value(comp);
+  assert(r == 0);
+  printf("aio discard: %d~%d = %d\n", (int)off, (int)len, (int)r);
+  rbd_aio_release(comp);
+}
+
+void discard_test_data(rbd_image_t image, uint64_t off, size_t len)
+{
+  ssize_t written;
+  written = rbd_discard(image, off, len);
+  printf("discard: %d~%d = %d\n", (int)off, (int)len, (int)written);
+  assert(written == (ssize_t)len);
+}
+
+void aio_read_test_data(rbd_image_t image, const char *expected, uint64_t off, size_t len)
+{
+  rbd_completion_t comp;
+  char *result = (char *)malloc(len + 1);
+
+  assert(result);
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
+  printf("created completion\n");
+  rbd_aio_read(image, off, len, result, comp);
+  printf("started read\n");
+  rbd_aio_wait_for_complete(comp);
+  int r = rbd_aio_get_return_value(comp);
+  printf("return value is: %d\n", r);
+  assert(r == (ssize_t)len);
+  rbd_aio_release(comp);
+  if (memcmp(result, expected, len)) {
+    printf("read: %s\nexpected: %s\n", result, expected);
+    assert(memcmp(result, expected, len) == 0);
+  }
+  free(result);
+}
+
+void read_test_data(rbd_image_t image, const char *expected, uint64_t off, size_t len)
+{
+  ssize_t read;
+  char *result = (char *)malloc(len + 1);
+
+  assert(result);
+  read = rbd_read(image, off, len, result);
+  printf("read: %d\n", (int) read);
+  assert(read == (ssize_t)len);
+  result[len] = '\0';
+  if (memcmp(result, expected, len)) {
+    printf("read: %s\nexpected: %s\n", result, expected);
+    assert(memcmp(result, expected, len) == 0);
+  }
+  free(result);
+}
+
+TEST(LibRBD, TestIO)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  char test_data[TEST_IO_SIZE + 1];
+  char zero_data[TEST_IO_SIZE + 1];
+  int i;
+
+  for (i = 0; i < TEST_IO_SIZE; ++i) {
+    test_data[i] = (char) (rand() % (126 - 33) + 33);
+  }
+  test_data[TEST_IO_SIZE] = '\0';
+  memset(zero_data, 0, sizeof(zero_data));
+
+  for (i = 0; i < 5; ++i)
+    write_test_data(image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
+
+  for (i = 5; i < 10; ++i)
+    aio_write_test_data(image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
+
+  for (i = 0; i < 5; ++i)
+    read_test_data(image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
+
+  for (i = 5; i < 10; ++i)
+    aio_read_test_data(image, test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
+
+  // discard 2nd, 4th sections.
+  discard_test_data(image, TEST_IO_SIZE, TEST_IO_SIZE);
+  aio_discard_test_data(image, TEST_IO_SIZE*3, TEST_IO_SIZE);
+
+  read_test_data(image, test_data,  0, TEST_IO_SIZE);
+  read_test_data(image,  zero_data, TEST_IO_SIZE, TEST_IO_SIZE);
+  read_test_data(image, test_data,  TEST_IO_SIZE*2, TEST_IO_SIZE);
+  read_test_data(image,  zero_data, TEST_IO_SIZE*3, TEST_IO_SIZE);
+  read_test_data(image, test_data,  TEST_IO_SIZE*4, TEST_IO_SIZE);
+  
+  rbd_image_info_t info;
+  rbd_completion_t comp;
+  ASSERT_EQ(0, rbd_stat(image, &info, sizeof(info)));
+  // can't read or write starting past end
+  ASSERT_EQ(-EINVAL, rbd_write(image, info.size, 1, test_data));
+  ASSERT_EQ(-EINVAL, rbd_read(image, info.size, 1, test_data));
+  // reading through end returns amount up to end
+  ASSERT_EQ(10, rbd_read(image, info.size - 10, 100, test_data));
+  // writing through end returns amount up to end
+  ASSERT_EQ(10, rbd_write(image, info.size - 10, 100, test_data));
+
+  rbd_aio_create_completion(NULL, (rbd_callback_t) simple_read_cb, &comp);
+  ASSERT_EQ(-EINVAL, rbd_aio_write(image, info.size, 1, test_data, comp));
+  ASSERT_EQ(-EINVAL, rbd_aio_read(image, info.size, 1, test_data, comp));
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRBD, TestEmptyDiscard)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 20 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  aio_discard_test_data(image, 0, 1*1024*1024);
+  aio_discard_test_data(image, 0, 4*1024*1024);
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+
+void simple_write_cb_pp(librbd::completion_t cb, void *arg)
+{
+  cout << "write completion cb called!" << endl;
+}
+
+void simple_read_cb_pp(librbd::completion_t cb, void *arg)
+{
+  cout << "read completion cb called!" << endl;
+}
+
+void aio_write_test_data(librbd::Image& image, const char *test_data, off_t off)
+{
+  ceph::bufferlist bl;
+  bl.append(test_data, strlen(test_data));
+  librbd::RBD::AioCompletion *comp = new librbd::RBD::AioCompletion(NULL, (librbd::callback_t) simple_write_cb_pp);
+  printf("created completion\n");
+  image.aio_write(off, strlen(test_data), bl, comp);
+  printf("started write\n");
+  comp->wait_for_complete();
+  int r = comp->get_return_value();
+  printf("return value is: %d\n", r);
+  assert(r >= 0);
+  printf("finished write\n");
+  comp->release();
+}
+
+void aio_discard_test_data(librbd::Image& image, off_t off, size_t len)
+{
+  librbd::RBD::AioCompletion *comp = new librbd::RBD::AioCompletion(NULL, (librbd::callback_t) simple_write_cb_pp);
+  image.aio_discard(off, len, comp);
+  comp->wait_for_complete();
+  int r = comp->get_return_value();
+  assert(r >= 0);
+  comp->release();
+}
+
+void write_test_data(librbd::Image& image, const char *test_data, off_t off)
+{
+  size_t written;
+  size_t len = strlen(test_data);
+  ceph::bufferlist bl;
+  bl.append(test_data, len);
+  written = image.write(off, len, bl);
+  printf("wrote: %u\n", (unsigned int) written);
+  assert(written == bl.length());
+}
+
+void discard_test_data(librbd::Image& image, off_t off, size_t len)
+{
+  size_t written;
+  written = image.discard(off, len);
+  printf("discard: %u~%u\n", (unsigned)off, (unsigned)len);
+  assert(written == len);
+}
+
+void aio_read_test_data(librbd::Image& image, const char *expected, off_t off, size_t expected_len)
+{
+  librbd::RBD::AioCompletion *comp = new librbd::RBD::AioCompletion(NULL, (librbd::callback_t) simple_read_cb_pp);
+  ceph::bufferlist bl;
+  printf("created completion\n");
+  image.aio_read(off, expected_len, bl, comp);
+  printf("started read\n");
+  comp->wait_for_complete();
+  int r = comp->get_return_value();
+  printf("return value is: %d\n", r);
+  assert(r == TEST_IO_SIZE);
+  assert(strncmp(expected, bl.c_str(), TEST_IO_SIZE) == 0);
+  printf("finished read\n");
+  comp->release();
+}
+
+void read_test_data(librbd::Image& image, const char *expected, off_t off, size_t expected_len)
+{
+  int read, total_read = 0;
+  size_t len = expected_len;
+  ceph::bufferlist bl;
+  read = image.read(off + total_read, len, bl);
+  assert(read >= 0);
+  printf("read: %u\n", (unsigned int) read);
+  printf("read: %s\nexpected: %s\n", bl.c_str(), expected);
+  assert(strncmp(bl.c_str(), expected, expected_len) == 0);
+}
+
+TEST(LibRBD, TestIOPP) 
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 2 << 20;
+    
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+
+    char test_data[TEST_IO_SIZE + 1];
+    char zero_data[TEST_IO_SIZE + 1];
+    int i;
+    
+    srand(time(0));
+    for (i = 0; i < TEST_IO_SIZE; ++i) {
+      test_data[i] = (char) (rand() % (126 - 33) + 33);
+    }
+    test_data[TEST_IO_SIZE] = '\0';
+    memset(zero_data, 0, sizeof(zero_data));
+
+    for (i = 0; i < 5; ++i)
+      write_test_data(image, test_data, strlen(test_data) * i);
+    
+    for (i = 5; i < 10; ++i)
+      aio_write_test_data(image, test_data, strlen(test_data) * i);
+    
+    for (i = 0; i < 5; ++i)
+      read_test_data(image, test_data, strlen(test_data) * i, TEST_IO_SIZE);
+    
+    for (i = 5; i < 10; ++i)
+      aio_read_test_data(image, test_data, strlen(test_data) * i, TEST_IO_SIZE);
+
+    // discard 2nd, 4th sections.
+    discard_test_data(image, TEST_IO_SIZE, TEST_IO_SIZE);
+    aio_discard_test_data(image, TEST_IO_SIZE*3, TEST_IO_SIZE);
+    
+    read_test_data(image, test_data,  0, TEST_IO_SIZE);
+    read_test_data(image,  zero_data, TEST_IO_SIZE, TEST_IO_SIZE);
+    read_test_data(image, test_data,  TEST_IO_SIZE*2, TEST_IO_SIZE);
+    read_test_data(image,  zero_data, TEST_IO_SIZE*3, TEST_IO_SIZE);
+    read_test_data(image, test_data,  TEST_IO_SIZE*4, TEST_IO_SIZE);
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+
+TEST(LibRBD, TestIOToSnapshot)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t isize = 2 << 20;
+  
+  ASSERT_EQ(0, create_image(ioctx, name, isize, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  int i, r;
+  rbd_image_t image_at_snap;
+  char orig_data[TEST_IO_TO_SNAP_SIZE + 1];
+  char test_data[TEST_IO_TO_SNAP_SIZE + 1];
+
+  for (i = 0; i < TEST_IO_TO_SNAP_SIZE; ++i)
+    test_data[i] = (char) (i + 48);
+  test_data[TEST_IO_TO_SNAP_SIZE] = '\0';
+  orig_data[TEST_IO_TO_SNAP_SIZE] = '\0';
+
+  r = rbd_read(image, 0, TEST_IO_TO_SNAP_SIZE, orig_data);
+  ASSERT_EQ(r, TEST_IO_TO_SNAP_SIZE);
+
+  ASSERT_EQ(0, test_ls_snaps(image, 0));
+  ASSERT_EQ(0, rbd_snap_create(image, "orig"));
+  ASSERT_EQ(1, test_ls_snaps(image, 1, "orig", isize));
+  read_test_data(image, orig_data, 0, TEST_IO_TO_SNAP_SIZE);
+
+  printf("write test data!\n");
+  write_test_data(image, test_data, 0, TEST_IO_TO_SNAP_SIZE);
+  ASSERT_EQ(0, rbd_snap_create(image, "written"));
+  ASSERT_EQ(2, test_ls_snaps(image, 2, "orig", isize, "written", isize));
+
+  read_test_data(image, test_data, 0, TEST_IO_TO_SNAP_SIZE);
+
+  rbd_snap_set(image, "orig");
+  read_test_data(image, orig_data, 0, TEST_IO_TO_SNAP_SIZE);
+
+  rbd_snap_set(image, "written");
+  read_test_data(image, test_data, 0, TEST_IO_TO_SNAP_SIZE);
+
+  rbd_snap_set(image, "orig");
+
+  r = rbd_write(image, 0, TEST_IO_TO_SNAP_SIZE, test_data);
+  printf("write to snapshot returned %d\n", r);
+  ASSERT_LT(r, 0);
+  cout << cpp_strerror(-r) << std::endl;
+
+  read_test_data(image, orig_data, 0, TEST_IO_TO_SNAP_SIZE);
+  rbd_snap_set(image, "written");
+  read_test_data(image, test_data, 0, TEST_IO_TO_SNAP_SIZE);
+
+  r = rbd_snap_rollback(image, "orig");
+  ASSERT_EQ(r, -EROFS);
+
+  r = rbd_snap_set(image, NULL);
+  ASSERT_EQ(r, 0);
+  r = rbd_snap_rollback(image, "orig");
+  ASSERT_EQ(r, 0);
+
+  write_test_data(image, test_data, 0, TEST_IO_TO_SNAP_SIZE);
+
+  rbd_flush(image);
+
+  printf("opening testimg@orig\n");
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image_at_snap, "orig"));
+  read_test_data(image_at_snap, orig_data, 0, TEST_IO_TO_SNAP_SIZE);
+  r = rbd_write(image_at_snap, 0, TEST_IO_TO_SNAP_SIZE, test_data);
+  printf("write to snapshot returned %d\n", r);
+  ASSERT_LT(r, 0);
+  cout << cpp_strerror(-r) << std::endl;
+  ASSERT_EQ(0, rbd_close(image_at_snap));
+
+  ASSERT_EQ(2, test_ls_snaps(image, 2, "orig", isize, "written", isize));
+  ASSERT_EQ(0, rbd_snap_remove(image, "written"));
+  ASSERT_EQ(1, test_ls_snaps(image, 1, "orig", isize));
+  ASSERT_EQ(0, rbd_snap_remove(image, "orig"));
+  ASSERT_EQ(0, test_ls_snaps(image, 0));
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRBD, TestClone)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  rbd_image_info_t pinfo, cinfo;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  int features = RBD_FEATURE_LAYERING;
+  rbd_image_t parent, child;
+  int order = 0;
+
+  // make a parent to clone from
+  ASSERT_EQ(0, create_image_full(ioctx, "parent", 4<<20, &order, false, features));
+  ASSERT_EQ(0, rbd_open(ioctx, "parent", &parent, NULL));
+  printf("made parent image \"parent\"\n");
+
+  char *data = (char *)"testdata";
+  ASSERT_EQ((ssize_t)strlen(data), rbd_write(parent, 0, strlen(data), data));
+
+  // can't clone a non-snapshot, expect failure
+  EXPECT_NE(0, rbd_clone(ioctx, "parent", NULL, ioctx, "child", features, &order));
+
+  // verify that there is no parent info on "parent"
+  char ppool[1], pname[1], psnapname[1];
+  ASSERT_EQ(-ENOENT, rbd_get_parent_info(parent, ppool, sizeof(ppool),
+	    pname, sizeof(pname), psnapname, sizeof(psnapname)));
+  printf("parent has no parent info\n");
+
+  // create a snapshot, reopen as the parent we're interested in
+  ASSERT_EQ(0, rbd_snap_create(parent, "parent_snap"));
+  printf("made snapshot \"parent@parent_snap\"\n");
+  ASSERT_EQ(0, rbd_close(parent));
+  ASSERT_EQ(0, rbd_open(ioctx, "parent", &parent, "parent_snap"));
+
+  ASSERT_EQ(-EINVAL, rbd_clone(ioctx, "parent", "parent_snap", ioctx, "child",
+	    features, &order));
+
+  // unprotected image should fail unprotect
+  ASSERT_EQ(-EINVAL, rbd_snap_unprotect(parent, "parent_snap"));
+  printf("can't unprotect an unprotected snap\n");
+
+  ASSERT_EQ(0, rbd_snap_protect(parent, "parent_snap"));
+  // protecting again should fail
+  ASSERT_EQ(-EBUSY, rbd_snap_protect(parent, "parent_snap"));
+  printf("can't protect a protected snap\n");
+
+  // This clone and open should work
+  ASSERT_EQ(0, rbd_clone(ioctx, "parent", "parent_snap", ioctx, "child",
+	    features, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, "child", &child, NULL));
+  printf("made and opened clone \"child\"\n");
+
+  // check read
+  read_test_data(child, data, 0, strlen(data));
+
+  // check write
+  ASSERT_EQ((ssize_t)strlen(data), rbd_write(child, 20, strlen(data), data));
+  read_test_data(child, data, 20, strlen(data));
+  read_test_data(child, data, 0, strlen(data));
+
+  // check attributes
+  ASSERT_EQ(0, rbd_stat(parent, &pinfo, sizeof(pinfo)));
+  ASSERT_EQ(0, rbd_stat(child, &cinfo, sizeof(cinfo)));
+  EXPECT_EQ(cinfo.size, pinfo.size);
+  uint64_t overlap;
+  rbd_get_overlap(child, &overlap);
+  EXPECT_EQ(overlap, pinfo.size);
+  EXPECT_EQ(cinfo.obj_size, pinfo.obj_size);
+  EXPECT_EQ(cinfo.order, pinfo.order);
+  printf("sizes and overlaps are good between parent and child\n");
+
+  // sizing down child results in changing overlap and size, not parent size
+  ASSERT_EQ(0, rbd_resize(child, 2UL<<20));
+  ASSERT_EQ(0, rbd_stat(child, &cinfo, sizeof(cinfo)));
+  rbd_get_overlap(child, &overlap);
+  ASSERT_EQ(overlap, 2UL<<20);
+  ASSERT_EQ(cinfo.size, 2UL<<20);
+  ASSERT_EQ(0, rbd_resize(child, 4UL<<20));
+  ASSERT_EQ(0, rbd_stat(child, &cinfo, sizeof(cinfo)));
+  rbd_get_overlap(child, &overlap);
+  ASSERT_EQ(overlap, 2UL<<20);
+  ASSERT_EQ(cinfo.size, 4UL<<20);
+  printf("sized down clone, changed overlap\n");
+
+  // sizing back up doesn't change that
+  ASSERT_EQ(0, rbd_resize(child, 5UL<<20));
+  ASSERT_EQ(0, rbd_stat(child, &cinfo, sizeof(cinfo)));
+  rbd_get_overlap(child, &overlap);
+  ASSERT_EQ(overlap, 2UL<<20);
+  ASSERT_EQ(cinfo.size, 5UL<<20);
+  ASSERT_EQ(0, rbd_stat(parent, &pinfo, sizeof(pinfo)));
+  printf("parent info: size %lld obj_size %lld parent_pool %lld\n",
+	 (unsigned long long)pinfo.size, (unsigned long long)pinfo.obj_size,
+	 (unsigned long long)pinfo.parent_pool);
+  ASSERT_EQ(pinfo.size, 4UL<<20);
+  printf("sized up clone, changed size but not overlap or parent's size\n");
+  
+  ASSERT_EQ(0, rbd_close(child));
+
+  ASSERT_EQ(-EBUSY, rbd_snap_remove(parent, "parent_snap"));
+  printf("can't remove parent while child still exists\n");
+  ASSERT_EQ(0, rbd_remove(ioctx, "child"));
+  ASSERT_EQ(-EBUSY, rbd_snap_remove(parent, "parent_snap"));
+  printf("can't remove parent while still protected\n");
+  ASSERT_EQ(0, rbd_snap_unprotect(parent, "parent_snap"));
+  ASSERT_EQ(0, rbd_snap_remove(parent, "parent_snap"));
+  printf("removed parent snap after unprotecting\n");
+
+  ASSERT_EQ(0, rbd_close(parent));
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRBD, TestClone2)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  int features = RBD_FEATURE_LAYERING;
+  rbd_image_t parent, child;
+  int order = 0;
+
+  // make a parent to clone from
+  ASSERT_EQ(0, create_image_full(ioctx, "parent", 4<<20, &order, false, features));
+  ASSERT_EQ(0, rbd_open(ioctx, "parent", &parent, NULL));
+  printf("made parent image \"parent\"\n");
+
+  char *data = (char *)"testdata";
+  char *childata = (char *)"childata";
+  ASSERT_EQ((ssize_t)strlen(data), rbd_write(parent, 0, strlen(data), data));
+  ASSERT_EQ((ssize_t)strlen(data), rbd_write(parent, 12, strlen(data), data));
+
+  // can't clone a non-snapshot, expect failure
+  EXPECT_NE(0, rbd_clone(ioctx, "parent", NULL, ioctx, "child", features, &order));
+
+  // verify that there is no parent info on "parent"
+  char ppool[1], pname[1], psnapname[1];
+  ASSERT_EQ(-ENOENT, rbd_get_parent_info(parent, ppool, sizeof(ppool),
+	    pname, sizeof(pname), psnapname, sizeof(psnapname)));
+  printf("parent has no parent info\n");
+
+  // create a snapshot, reopen as the parent we're interested in
+  ASSERT_EQ(0, rbd_snap_create(parent, "parent_snap"));
+  printf("made snapshot \"parent@parent_snap\"\n");
+  ASSERT_EQ(0, rbd_close(parent));
+  ASSERT_EQ(0, rbd_open(ioctx, "parent", &parent, "parent_snap"));
+
+  ASSERT_EQ(-EINVAL, rbd_clone(ioctx, "parent", "parent_snap", ioctx, "child",
+	    features, &order));
+
+  // unprotected image should fail unprotect
+  ASSERT_EQ(-EINVAL, rbd_snap_unprotect(parent, "parent_snap"));
+  printf("can't unprotect an unprotected snap\n");
+
+  ASSERT_EQ(0, rbd_snap_protect(parent, "parent_snap"));
+  // protecting again should fail
+  ASSERT_EQ(-EBUSY, rbd_snap_protect(parent, "parent_snap"));
+  printf("can't protect a protected snap\n");
+
+  // This clone and open should work
+  ASSERT_EQ(0, rbd_clone(ioctx, "parent", "parent_snap", ioctx, "child",
+	    features, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, "child", &child, NULL));
+  printf("made and opened clone \"child\"\n");
+
+  // write something in
+  ASSERT_EQ((ssize_t)strlen(childata), rbd_write(child, 20, strlen(childata), childata));
+
+  char test[strlen(data) * 2];
+  ASSERT_EQ((ssize_t)strlen(data), rbd_read(child, 20, strlen(data), test));
+  ASSERT_EQ(0, memcmp(test, childata, strlen(childata)));
+
+  // overlap
+  ASSERT_EQ((ssize_t)sizeof(test), rbd_read(child, 20 - strlen(data), sizeof(test), test));
+  ASSERT_EQ(0, memcmp(test, data, strlen(data)));
+  ASSERT_EQ(0, memcmp(test + strlen(data), childata, strlen(childata)));
+
+  // all parent
+  ASSERT_EQ((ssize_t)sizeof(test), rbd_read(child, 0, sizeof(test), test));
+  ASSERT_EQ(0, memcmp(test, data, strlen(data)));
+
+  ASSERT_EQ(0, rbd_close(child));
+  ASSERT_EQ(0, rbd_close(parent));
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+static void test_list_children(rbd_image_t image, ssize_t num_expected, ...)
+{
+  va_list ap;
+  va_start(ap, num_expected);
+  size_t pools_len = 100;
+  size_t children_len = 100;
+  char *pools = NULL;
+  char *children = NULL;
+  ssize_t num_children;
+
+  do {
+    free(pools);
+    free(children);
+    pools = (char *) malloc(pools_len);
+    children = (char *) malloc(children_len);
+    num_children = rbd_list_children(image, pools, &pools_len,
+				     children, &children_len);
+  } while (num_children == -ERANGE);
+
+  ASSERT_EQ(num_expected, num_children);
+  for (ssize_t i = num_expected; i > 0; --i) {
+    char *expected_pool = va_arg(ap, char *);
+    char *expected_image = va_arg(ap, char *);
+    char *pool = pools;
+    char *image = children;
+    bool found = 0;
+    printf("\ntrying to find %s/%s\n", expected_pool, expected_image);
+    for (ssize_t j = 0; j < num_children; ++j) {
+      printf("checking %s/%s\n", pool, image);
+      if (strcmp(expected_pool, pool) == 0 &&
+	  strcmp(expected_image, image) == 0) {
+	printf("found child %s/%s\n\n", pool, image);
+	found = 1;
+	break;
+      }
+      pool += strlen(pool) + 1;
+      image += strlen(image) + 1;
+      if (j == num_children - 1) {
+	ASSERT_EQ(pool - pools - 1, (ssize_t) pools_len);
+	ASSERT_EQ(image - children - 1, (ssize_t) children_len);
+      }
+    }
+    ASSERT_TRUE(found);
+  }
+  va_end(ap);
+
+  if (pools)
+    free(pools);
+  if (children)
+    free(children);
+}
+
+TEST(LibRBD, ListChildren)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx1, ioctx2;
+  string pool_name1 = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name1, &cluster));
+  string pool_name2 = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name2, &cluster));
+  rados_ioctx_create(cluster, pool_name1.c_str(), &ioctx1);
+  rados_ioctx_create(cluster, pool_name2.c_str(), &ioctx2);
+
+  int features = RBD_FEATURE_LAYERING;
+  rbd_image_t parent;
+  int order = 0;
+
+  // make a parent to clone from
+  ASSERT_EQ(0, create_image_full(ioctx1, "parent", 4<<20, &order,
+				 false, features));
+  ASSERT_EQ(0, rbd_open(ioctx1, "parent", &parent, NULL));
+  // create a snapshot, reopen as the parent we're interested in
+  ASSERT_EQ(0, rbd_snap_create(parent, "parent_snap"));
+  ASSERT_EQ(0, rbd_snap_set(parent, "parent_snap"));
+  ASSERT_EQ(0, rbd_snap_protect(parent, "parent_snap"));
+
+  ASSERT_EQ(0, rbd_close(parent));
+  ASSERT_EQ(0, rbd_open(ioctx1, "parent", &parent, "parent_snap"));
+
+  ASSERT_EQ(0, rbd_clone(ioctx1, "parent", "parent_snap", ioctx2, "child1",
+	    features, &order));
+  test_list_children(parent, 1, pool_name2.c_str(), "child1");
+
+  ASSERT_EQ(0, rbd_clone(ioctx1, "parent", "parent_snap", ioctx1, "child2",
+	    features, &order));
+  test_list_children(parent, 2, pool_name2.c_str(), "child1",
+		     pool_name1.c_str(), "child2");
+
+  ASSERT_EQ(0, rbd_clone(ioctx1, "parent", "parent_snap", ioctx2, "child3",
+	    features, &order));
+  test_list_children(parent, 3, pool_name2.c_str(), "child1",
+		     pool_name1.c_str(), "child2",
+		     pool_name2.c_str(), "child3");
+
+  ASSERT_EQ(0, rbd_clone(ioctx1, "parent", "parent_snap", ioctx2, "child4",
+	    features, &order));
+  test_list_children(parent, 4, pool_name2.c_str(), "child1",
+		     pool_name1.c_str(), "child2",
+		     pool_name2.c_str(), "child3",
+		     pool_name2.c_str(), "child4");
+
+  ASSERT_EQ(0, rbd_remove(ioctx2, "child1"));
+  test_list_children(parent, 3,
+		     pool_name1.c_str(), "child2",
+		     pool_name2.c_str(), "child3",
+		     pool_name2.c_str(), "child4");
+
+  ASSERT_EQ(0, rbd_remove(ioctx2, "child3"));
+  test_list_children(parent, 2,
+		     pool_name1.c_str(), "child2",
+		     pool_name2.c_str(), "child4");
+
+  ASSERT_EQ(0, rbd_remove(ioctx2, "child4"));
+  test_list_children(parent, 1,
+		     pool_name1.c_str(), "child2");
+
+  ASSERT_EQ(0, rbd_remove(ioctx1, "child2"));
+  test_list_children(parent, 0);
+
+  ASSERT_EQ(0, rbd_snap_unprotect(parent, "parent_snap"));
+  ASSERT_EQ(0, rbd_snap_remove(parent, "parent_snap"));
+  ASSERT_EQ(0, rbd_close(parent));
+  ASSERT_EQ(0, rbd_remove(ioctx1, "parent"));
+  rados_ioctx_destroy(ioctx1);
+  rados_ioctx_destroy(ioctx2);
+  // destroy_one_pool also closes the cluster; do this one step at a time
+  ASSERT_EQ(0, rados_pool_delete(cluster, pool_name1.c_str()));
+  ASSERT_EQ(0, destroy_one_pool(pool_name2, &cluster));
+}
+
+TEST(LibRBD, LockingPP)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 2 << 20;
+    std::string cookie1 = "foo";
+    std::string cookie2 = "bar";
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+
+    // no lockers initially
+    std::list lockers;
+    std::string tag;
+    bool exclusive;
+    ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag));
+    ASSERT_EQ(0u, lockers.size());
+    ASSERT_EQ("", tag);
+
+    // exclusive lock is exclusive
+    ASSERT_EQ(0, image.lock_exclusive(cookie1));
+    ASSERT_EQ(-EEXIST, image.lock_exclusive(cookie1));
+    ASSERT_EQ(-EBUSY, image.lock_exclusive(""));
+    ASSERT_EQ(-EEXIST, image.lock_shared(cookie1, ""));
+    ASSERT_EQ(-EBUSY, image.lock_shared(cookie1, "test"));
+    ASSERT_EQ(-EBUSY, image.lock_shared("", "test"));
+    ASSERT_EQ(-EBUSY, image.lock_shared("", ""));
+
+    // list exclusive
+    ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag));
+    ASSERT_TRUE(exclusive);
+    ASSERT_EQ("", tag);
+    ASSERT_EQ(1u, lockers.size());
+    ASSERT_EQ(cookie1, lockers.front().cookie);
+
+    // unlock
+    ASSERT_EQ(-ENOENT, image.unlock(""));
+    ASSERT_EQ(-ENOENT, image.unlock(cookie2));
+    ASSERT_EQ(0, image.unlock(cookie1));
+    ASSERT_EQ(-ENOENT, image.unlock(cookie1));
+    ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag));
+    ASSERT_EQ(0u, lockers.size());
+
+    ASSERT_EQ(0, image.lock_shared(cookie1, ""));
+    ASSERT_EQ(-EEXIST, image.lock_shared(cookie1, ""));
+    ASSERT_EQ(0, image.lock_shared(cookie2, ""));
+    ASSERT_EQ(-EEXIST, image.lock_shared(cookie2, ""));
+    ASSERT_EQ(-EEXIST, image.lock_exclusive(cookie1));
+    ASSERT_EQ(-EEXIST, image.lock_exclusive(cookie2));
+    ASSERT_EQ(-EBUSY, image.lock_exclusive(""));
+    ASSERT_EQ(-EBUSY, image.lock_exclusive("test"));
+
+    // list shared
+    ASSERT_EQ(0, image.list_lockers(&lockers, &exclusive, &tag));
+    ASSERT_EQ(2u, lockers.size());
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(LibRBD, FlushAio)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+  size_t num_aios = 256;
+
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  char test_data[TEST_IO_SIZE + 1];
+  size_t i;
+  for (i = 0; i < TEST_IO_SIZE; ++i) {
+    test_data[i] = (char) (rand() % (126 - 33) + 33);
+  }
+
+  rbd_completion_t write_comps[num_aios];
+  for (i = 0; i < num_aios; ++i) {
+    ASSERT_EQ(0, rbd_aio_create_completion(NULL, NULL, &write_comps[i]));
+    uint64_t offset = rand() % (size - TEST_IO_SIZE);
+    ASSERT_EQ(0, rbd_aio_write(image, offset, TEST_IO_SIZE, test_data,
+			       write_comps[i]));
+  }
+
+  rbd_completion_t flush_comp;
+  ASSERT_EQ(0, rbd_aio_create_completion(NULL, NULL, &flush_comp));
+  ASSERT_EQ(0, rbd_aio_flush(image, flush_comp));
+  ASSERT_EQ(0, rbd_aio_wait_for_complete(flush_comp));
+  ASSERT_EQ(1, rbd_aio_is_complete(flush_comp));
+  rbd_aio_release(flush_comp);
+
+  for (i = 0; i < num_aios; ++i) {
+    ASSERT_EQ(1, rbd_aio_is_complete(write_comps[i]));
+    rbd_aio_release(write_comps[i]);
+  }
+
+  ASSERT_EQ(0, rbd_close(image));
+  ASSERT_EQ(0, rbd_remove(ioctx, name));
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRBD, FlushAioPP)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 2 << 20;
+    size_t num_aios = 256;
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+
+    char test_data[TEST_IO_SIZE + 1];
+    size_t i;
+    for (i = 0; i < TEST_IO_SIZE; ++i) {
+      test_data[i] = (char) (rand() % (126 - 33) + 33);
+    }
+
+    librbd::RBD::AioCompletion *write_comps[num_aios];
+    for (i = 0; i < num_aios; ++i) {
+      ceph::bufferlist bl;
+      bl.append(test_data, strlen(test_data));
+      write_comps[i] = new librbd::RBD::AioCompletion(NULL, NULL);
+      uint64_t offset = rand() % (size - TEST_IO_SIZE);
+      ASSERT_EQ(0, image.aio_write(offset, TEST_IO_SIZE, bl,
+				   write_comps[i]));
+    }
+
+    librbd::RBD::AioCompletion *flush_comp =
+      new librbd::RBD::AioCompletion(NULL, NULL);
+    ASSERT_EQ(0, image.aio_flush(flush_comp));
+    ASSERT_EQ(0, flush_comp->wait_for_complete());
+    ASSERT_EQ(1, flush_comp->is_complete());
+    delete flush_comp;
+
+    for (i = 0; i < num_aios; ++i) {
+      librbd::RBD::AioCompletion *comp = write_comps[i];
+      ASSERT_EQ(1, comp->is_complete());
+      delete comp;
+    }
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+
+int iterate_cb(uint64_t off, size_t len, int exists, void *arg)
+{
+  //cout << "iterate_cb " << off << "~" << len << std::endl;
+  interval_set *diff = static_cast *>(arg);
+  diff->insert(off, len);
+  return 0;
+}
+
+void scribble(librbd::Image& image, int n, int max, interval_set *exists, interval_set *what)
+{
+  uint64_t size;
+  image.size(&size);
+  interval_set exists_at_start = *exists;
+  for (int i=0; i w;      
+      w.insert(off, len);
+
+      // the zeroed bit no longer exists...
+      w.intersection_of(*exists); 
+      exists->subtract(w);
+
+      // the bits we discarded are no long written...
+      interval_set w2 = w;
+      w2.intersection_of(*what);
+      what->subtract(w2);
+
+      // except for the extents that existed at the start that we overwrote.
+      interval_set w3;
+      w3.insert(off, len);
+      w3.intersection_of(exists_at_start);
+      what->union_of(w3);
+
+    } else {
+      bufferlist bl;
+      bl.append(buffer::create(len));
+      bl.zero();
+      ASSERT_EQ((int)len, image.write(off, len, bl));
+      interval_set w;
+      w.insert(off, len);
+      what->union_of(w);
+      exists->union_of(w);
+    }
+  }
+}
+
+TEST(LibRBD, DiffIterate)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  int seed = getpid();
+  cout << "seed " << seed << std::endl;
+  srand(seed);
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 20 << 20;
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+
+    interval_set exists;
+    interval_set one, two;
+    scribble(image, 10, 102400, &exists, &one);
+    cout << " wrote " << one << std::endl;
+    ASSERT_EQ(0, image.snap_create("one"));
+    scribble(image, 10, 102400, &exists, &two);
+    cout << " wrote " << two << std::endl;
+
+    interval_set diff;
+    ASSERT_EQ(0, image.diff_iterate("one", 0, size, iterate_cb, (void *)&diff));
+    cout << " diff was " << diff << std::endl;
+    if (!two.subset_of(diff)) {
+      interval_set i;
+      i.intersection_of(two, diff);
+      interval_set l = two;
+      l.subtract(i);
+      cout << " ... two - (two*diff) = " << l << std::endl;     
+    }
+    ASSERT_TRUE(two.subset_of(diff));
+  }
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+struct diff_extent {
+  diff_extent(uint64_t offset, uint64_t length, bool exists) :
+    offset(offset), length(length), exists(exists) {}
+  uint64_t offset;
+  uint64_t length;
+  bool exists;
+  bool operator==(const diff_extent& o) const {
+    return offset == o.offset && length == o.length && exists == o.exists;
+  }
+};
+
+ostream& operator<<(ostream & o, const diff_extent& e) {
+  return o << '(' << e.offset << '~' << e.length << ' ' << (e.exists ? "true" : "false") << ')';
+}
+
+int vector_iterate_cb(uint64_t off, size_t len, int exists, void *arg)
+{
+  cout << "iterate_cb " << off << "~" << len << std::endl;
+  vector *diff = static_cast *>(arg);
+  diff->push_back(diff_extent(off, len, exists));
+  return 0;
+}
+
+TEST(LibRBD, DiffIterateDiscard)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  int seed = getpid();
+  cout << "seed " << seed << std::endl;
+  srand(seed);
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 20 << 20;
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+
+    vector extents;
+    ceph::bufferlist bl;
+
+    ASSERT_EQ(0, image.diff_iterate(NULL, 0, size,
+				    vector_iterate_cb, (void *) &extents));
+    ASSERT_EQ(0u, extents.size());
+
+    char data[256];
+    memset(data, 1, sizeof(data));
+    bl.append(data, 256);
+    ASSERT_EQ(256, image.write(0, 256, bl));
+    ASSERT_EQ(0, image.diff_iterate(NULL, 0, size,
+				    vector_iterate_cb, (void *) &extents));
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 256, true), extents[0]);
+
+    int obj_ofs = 256;
+    ASSERT_EQ(obj_ofs, image.discard(0, obj_ofs));
+
+    extents.clear();
+    ASSERT_EQ(0, image.diff_iterate(NULL, 0, size,
+				    vector_iterate_cb, (void *) &extents));
+    ASSERT_EQ(0u, extents.size());
+
+    ASSERT_EQ(0, image.snap_create("snap1"));
+    ASSERT_EQ(256, image.write(0, 256, bl));
+    ASSERT_EQ(0, image.diff_iterate(NULL, 0, size,
+				    vector_iterate_cb, (void *) &extents));
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 256, true), extents[0]);
+    ASSERT_EQ(0, image.snap_create("snap2"));
+
+    ASSERT_EQ(obj_ofs, image.discard(0, obj_ofs));
+
+    extents.clear();
+    ASSERT_EQ(0, image.snap_set("snap2"));
+    ASSERT_EQ(0, image.diff_iterate("snap1", 0, size,
+				    vector_iterate_cb, (void *) &extents));
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 256, true), extents[0]);
+
+    ASSERT_EQ(0, image.snap_set(NULL));
+    ASSERT_EQ(1 << order, image.discard(0, 1 << order));
+    ASSERT_EQ(0, image.snap_create("snap3"));
+    ASSERT_EQ(0, image.snap_set("snap3"));
+
+    extents.clear();
+    ASSERT_EQ(0, image.diff_iterate("snap1", 0, size,
+				    vector_iterate_cb, (void *) &extents));
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 256, false), extents[0]);
+  }
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(LibRBD, DiffIterateStress)
+{
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  string pool_name = get_temp_pool_name();
+
+  ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+  ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+
+  int seed = getpid();
+  cout << "seed " << seed << std::endl;
+  srand(seed);
+
+  {
+    librbd::RBD rbd;
+    librbd::Image image;
+    int order = 0;
+    const char *name = "testimg";
+    uint64_t size = 400 << 20;
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name, size, &order));
+    ASSERT_EQ(0, rbd.open(ioctx, image, name, NULL));
+
+    interval_set curexists;
+    vector > wrote;
+    vector > exists;
+    vector snap;
+    int n = 20;
+    for (int i=0; i w;
+      scribble(image, 10, 8192000, &curexists, &w);
+      cout << " i=" << i << " exists " << curexists << " wrote " << w << std::endl;
+      string s = "snap" + stringify(i);
+      ASSERT_EQ(0, image.snap_create(s.c_str()));
+      wrote.push_back(w);
+      exists.push_back(curexists);
+      snap.push_back(s);
+    }
+
+    for (int i=0; i diff, actual, uex;
+	for (int k=i+1; k<=j; k++)
+	  diff.union_of(wrote[k]);
+	cout << "from " << i << " to " << j << " diff " << diff << std::endl;
+
+	// limit to extents that exists both at the beginning and at the end
+	uex.union_of(exists[i], exists[j]);
+	diff.intersection_of(uex);
+	cout << "  limited diff " << diff << std::endl;
+
+	image.snap_set(snap[j].c_str());
+	ASSERT_EQ(0, image.diff_iterate(snap[i].c_str(), 0, size, iterate_cb, (void *)&actual));
+	cout << " actual was " << actual << std::endl;
+	if (!diff.subset_of(actual)) {
+	  interval_set i;
+	  i.intersection_of(diff, actual);
+	  interval_set l = diff;
+	  l.subtract(i);
+	  cout << " ... diff - (actual*diff) = " << l << std::endl;     
+	}
+	ASSERT_TRUE(diff.subset_of(actual));
+      }
+    }
+
+  }
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+}
+
+TEST(LibRBD, ZeroLengthWrite)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  char read_data[1];
+  ASSERT_EQ(0, rbd_write(image, 0, 0, NULL));
+  ASSERT_EQ(1, rbd_read(image, 0, 1, read_data));
+  ASSERT_EQ('\0', read_data[0]);
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+
+TEST(LibRBD, ZeroLengthDiscard)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  const char *data = "blah";
+  char read_data[strlen(data)];
+  ASSERT_EQ((int)strlen(data), rbd_write(image, 0, strlen(data), data));
+  ASSERT_EQ(0, rbd_discard(image, 0, 0));
+  ASSERT_EQ((int)strlen(data), rbd_read(image, 0, strlen(data), read_data));
+  ASSERT_EQ(0, memcmp(data, read_data, strlen(data)));
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRBD, ZeroLengthRead)
+{
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rbd_image_t image;
+  int order = 0;
+  const char *name = "testimg";
+  uint64_t size = 2 << 20;
+
+  ASSERT_EQ(0, create_image(ioctx, name, size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name, &image, NULL));
+
+  char read_data[1];
+  ASSERT_EQ(0, rbd_read(image, 0, 0, read_data));
+
+  ASSERT_EQ(0, rbd_close(image));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+int main(int argc, char **argv)
+{
+  ::testing::InitGoogleTest(&argc, argv);
+
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/mime.cc b/ceph/src/test/mime.cc
new file mode 100644
index 00000000..29c73e5a
--- /dev/null
+++ b/ceph/src/test/mime.cc
@@ -0,0 +1,150 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "common/mime.h"
+#include "gtest/gtest.h"
+
+#include 
+#include 
+
+using std::string;
+
+TEST(MimeTests, SimpleEncode) {
+  char output[256];
+  memset(output, 0, sizeof(output));
+  int len;
+
+  len = mime_encode_as_qp("abc", NULL, 0);
+  ASSERT_EQ(len, 4);
+  len = mime_encode_as_qp("abc", output, 4);
+  ASSERT_EQ(len, 4);
+  ASSERT_EQ(string("abc"), string(output));
+
+  len = mime_encode_as_qp("a=b", NULL, 0);
+  ASSERT_EQ(len, 6);
+  len = mime_encode_as_qp("a=b", output, 6);
+  ASSERT_EQ(len, 6);
+  ASSERT_EQ(string("a=3Db"), string(output));
+
+  len = mime_encode_as_qp("Libert\xc3\xa9", NULL, 0);
+  ASSERT_EQ(len, 13);
+  len = mime_encode_as_qp("Libert\xc3\xa9", output, 13);
+  ASSERT_EQ(len, 13);
+  ASSERT_EQ(string("Libert=C3=A9"), string(output));
+}
+
+TEST(MimeTests, EncodeOutOfSpace) {
+  char output[256];
+  memset(output, 0, sizeof(output));
+  int len;
+
+  len = mime_encode_as_qp("abcdefg", NULL, 0);
+  ASSERT_EQ(len, 8);
+  len = mime_encode_as_qp("abcdefg", output, 4);
+  ASSERT_EQ(len, 8);
+  ASSERT_EQ(string("abc"), string(output));
+  len = mime_encode_as_qp("abcdefg", output, 1);
+  ASSERT_EQ(len, 8);
+  ASSERT_EQ(string(""), string(output));
+
+  len = mime_encode_as_qp("a=b", output, 2);
+  ASSERT_EQ(len, 6);
+  ASSERT_EQ(string("a"), string(output));
+  len = mime_encode_as_qp("a=b", output, 3);
+  ASSERT_EQ(len, 6);
+  ASSERT_EQ(string("a"), string(output));
+}
+
+TEST(MimeTests, SimpleDecode) {
+  char output[256];
+  memset(output, 0, sizeof(output));
+  int len;
+
+  len = mime_decode_from_qp("abc", NULL, 0);
+  ASSERT_EQ(len, 4);
+  len = mime_decode_from_qp("abc", output, 4);
+  ASSERT_EQ(len, 4);
+  ASSERT_EQ(string("abc"), string(output));
+
+  len = mime_decode_from_qp("a=3Db", NULL, 0);
+  ASSERT_EQ(len, 4);
+  len = mime_decode_from_qp("a=3Db", output, 4);
+  ASSERT_EQ(len, 4);
+  ASSERT_EQ(string("a=b"), string(output));
+
+  len = mime_decode_from_qp("Libert=C3=A9", NULL, 0);
+  ASSERT_EQ(len, 9);
+  len = mime_decode_from_qp("Libert=C3=A9", output, 9);
+  ASSERT_EQ(len, 9);
+  ASSERT_EQ(string("Libert\xc3\xa9"), string(output));
+}
+
+TEST(MimeTests, LowercaseDecode) {
+  char output[256];
+  memset(output, 0, sizeof(output));
+  int len;
+
+  len = mime_decode_from_qp("Libert=c3=a9", NULL, 0);
+  ASSERT_EQ(len, 9);
+  len = mime_decode_from_qp("Libert=c3=a9", output, 9);
+  ASSERT_EQ(len, 9);
+  ASSERT_EQ(string("Libert\xc3\xa9"), string(output));
+}
+
+TEST(MimeTests, DecodeOutOfSpace) {
+  char output[256];
+  memset(output, 0, sizeof(output));
+  int len;
+
+  len = mime_decode_from_qp("abcdefg", NULL, 0);
+  ASSERT_EQ(len, 8);
+  len = mime_decode_from_qp("abcdefg", output, 4);
+  ASSERT_EQ(len, 8);
+  ASSERT_EQ(string("abc"), string(output));
+  len = mime_decode_from_qp("abcdefg", output, 1);
+  ASSERT_EQ(len, 8);
+  ASSERT_EQ(string(""), string(output));
+
+  len = mime_decode_from_qp("a=3Db", output, 2);
+  ASSERT_EQ(len, 4);
+  ASSERT_EQ(string("a"), string(output));
+  len = mime_decode_from_qp("a=3Db", output, 3);
+  ASSERT_EQ(len, 4);
+  ASSERT_EQ(string("a="), string(output));
+}
+
+TEST(MimeTests, DecodeErrors) {
+  char output[128];
+  memset(output, 0, sizeof(output));
+  int len;
+
+  // incomplete escape sequence
+  len = mime_decode_from_qp("boo=", output, sizeof(output));
+  ASSERT_LT(len, 0);
+
+  // invalid escape sequences
+  len = mime_decode_from_qp("boo=gg", output, sizeof(output));
+  ASSERT_LT(len, 0);
+  len = mime_decode_from_qp("boo=g", output, sizeof(output));
+  ASSERT_LT(len, 0);
+  len = mime_decode_from_qp("boo==", output, sizeof(output));
+  ASSERT_LT(len, 0);
+  len = mime_decode_from_qp("boo=44bar=z", output, sizeof(output));
+  ASSERT_LT(len, 0);
+
+  // high bit should not be set in quoted-printable mime output
+  unsigned char bad_input2[] = { 0x81, 0x6a, 0x0 };
+  len = mime_decode_from_qp(reinterpret_cast(bad_input2),
+			    output, sizeof(output));
+  ASSERT_LT(len, 0);
+}
diff --git a/ceph/src/test/mon/PGMap.cc b/ceph/src/test/mon/PGMap.cc
new file mode 100644
index 00000000..9f7a6b2d
--- /dev/null
+++ b/ceph/src/test/mon/PGMap.cc
@@ -0,0 +1,96 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 Inktank 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include "mon/PGMap.h"
+#include "gtest/gtest.h"
+
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+TEST(pgmap, min_last_epoch_clean)
+{
+  PGMap pg_map;
+  PGMap::Incremental inc;
+  osd_stat_t os;
+  pg_stat_t ps;
+
+  ps.last_epoch_clean = 999;
+  inc.pg_stat_updates[pg_t(9,9)] = ps;
+  inc.version = 1;
+  inc.update_stat(0, 123, os);
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(123u, pg_map.get_min_last_epoch_clean());
+
+  inc = PGMap::Incremental();
+  inc.version = 2;
+  inc.update_stat(1, 222, os);
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(123u, pg_map.get_min_last_epoch_clean());
+
+  inc = PGMap::Incremental();
+  inc.version = 3;
+  inc.update_stat(0, 222, os);
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(222u, pg_map.get_min_last_epoch_clean());
+
+  inc = PGMap::Incremental();
+  inc.version = 4;
+  inc.update_stat(0, 333, os);
+  inc.update_stat(1, 333, os);
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(333u, pg_map.get_min_last_epoch_clean());
+
+  ps.last_epoch_clean = 222;
+  inc = PGMap::Incremental();
+  inc.version = 5;
+  inc.pg_stat_updates[pg_t(1,1)] = ps;
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(222u, pg_map.get_min_last_epoch_clean());
+
+  ps.last_epoch_clean = 223;
+  inc = PGMap::Incremental();
+  inc.version = 6;
+  inc.pg_stat_updates[pg_t(1,1)] = ps;
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(223u, pg_map.get_min_last_epoch_clean());
+
+  ps.last_epoch_clean = 224;
+  inc = PGMap::Incremental();
+  inc.version = 7;
+  inc.pg_stat_updates[pg_t(2,2)] = ps;
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(223u, pg_map.get_min_last_epoch_clean());
+
+  ps.last_epoch_clean = 225;
+  inc = PGMap::Incremental();
+  inc.version = 8;
+  inc.pg_stat_updates[pg_t(1,1)] = ps;
+  pg_map.apply_incremental(g_ceph_context, inc);
+  ASSERT_EQ(224u, pg_map.get_min_last_epoch_clean());
+
+}
+
+
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+  env_to_vec(args);
+
+  vector def_args;
+  global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/mon/mon-test-helpers.sh b/ceph/src/test/mon/mon-test-helpers.sh
new file mode 100644
index 00000000..d228569e
--- /dev/null
+++ b/ceph/src/test/mon/mon-test-helpers.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+#
+# Copyright (C) 2013,2014 Cloudwatt 
+#
+# Author: Loic Dachary 
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+function setup() {
+    local dir=$1
+    teardown $dir
+    mkdir $dir
+}
+
+function teardown() {
+    local dir=$1
+    kill_daemons $dir
+    rm -fr $dir
+}
+
+function run_mon() {
+    local dir=$1
+    shift
+    local id=$1
+    shift
+    dir+=/$id
+    
+    ./ceph-mon \
+        --id $id \
+        --mkfs \
+        --mon-data=$dir --run-dir=$dir \
+        "$@"
+
+    ./ceph-mon \
+        --id $id \
+        --paxos-propose-interval=0.1 \
+        --osd-pool-default-erasure-code-directory=.libs \
+        --debug-mon 20 \
+        --debug-ms 20 \
+        --debug-paxos 20 \
+        --mon-advanced-debug-mode \
+        --chdir= \
+        --mon-data=$dir \
+        --log-file=$dir/log \
+        --mon-cluster-log-file=$dir/log \
+        --run-dir=$dir \
+        --pid-file=$dir/pidfile \
+        "$@"
+}
+
+function kill_daemons() {
+    local dir=$1
+    for pidfile in $(find $dir | grep pidfile) ; do
+        for try in 0 1 1 1 2 3 ; do
+            kill -9 $(cat $pidfile 2> /dev/null) 2> /dev/null || break
+            sleep $try
+        done
+    done
+}
+
+function call_TEST_functions() {
+    local dir=$1
+    shift
+    local id=$2
+    shift
+
+    setup $dir || return 1
+    run_mon $dir $id "$@"
+    SHARE_MON_FUNCTIONS=${SHARE_MON_FUNCTIONS:-$(set | sed -n -e 's/^\(SHARE_MON_TEST_[0-9a-z_]*\) .*/\1/p')}
+    for TEST_function in $SHARE_MON_FUNCTIONS ; do
+        if ! $TEST_function $dir $id ; then
+            cat $dir/$id/log
+            return 1
+        fi
+    done
+    teardown $dir || return 1
+
+    FUNCTIONS=${FUNCTIONS:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
+    for TEST_function in $FUNCTIONS ; do
+        setup $dir || return 1
+        $TEST_function $dir || return 1
+        teardown $dir || return 1
+    done
+}
+
+function main() {
+    local dir=$1
+
+    export PATH=:$PATH # make sure program from sources are prefered
+
+    PS4='${FUNCNAME[0]}: $LINENO: '
+    export CEPH_CONF=/dev/null
+    unset CEPH_ARGS
+
+    set -x
+    setup $dir || return 1
+    local code
+    if run $dir ; then
+        code=0
+    else
+        code=1
+    fi
+    teardown $dir || return 1
+    return $code
+}
diff --git a/ceph/src/test/mon/moncap.cc b/ceph/src/test/mon/moncap.cc
new file mode 100644
index 00000000..d381319f
--- /dev/null
+++ b/ceph/src/test/mon/moncap.cc
@@ -0,0 +1,222 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+
+#include "include/stringify.h"
+#include "mon/MonCap.h"
+
+#include "gtest/gtest.h"
+
+const char *parse_good[] = {
+
+  // MonCapMatch
+  "allow *",
+  "allow r",
+  "allow rwx",
+  "allow  r",
+  " allow rwx",
+  "allow rwx ",
+  " allow rwx ",
+  " allow\t   rwx ",
+  "\tallow\nrwx\t",
+  "allow service=foo x",
+  "allow service=\"froo\" x",
+  "allow profile osd",
+  "allow profile osd-bootstrap",
+  "allow profile \"mds-bootstrap\", allow *",
+  "allow command \"a b c\"",
+  "allow command abc",
+  "allow command abc with arg=foo",
+  "allow command abc with arg=foo arg2=bar",
+  "allow command abc with arg=foo arg2=bar",
+  "allow command abc with arg=foo arg2 prefix bar arg3 prefix baz",
+  "allow command abc with arg=foo arg2 prefix \"bar bingo\" arg3 prefix baz",
+  "allow service foo x",
+  "allow service foo x; allow service bar x",
+  "allow service foo w ;allow service bar x",
+  "allow service foo  w , allow service bar x",
+  "allow service foo r , allow service bar x",
+  "allow service foo_foo r, allow service bar r",
+  "allow service foo-foo r, allow service bar r",
+  "allow service \" foo \" w, allow service bar r",
+  "allow command abc with arg=foo arg2=bar, allow service foo r",
+  "allow command abc.def with arg=foo arg2=bar, allow service foo r",
+  "allow command \"foo bar\" with arg=\"baz\"",
+  "allow command \"foo bar\" with arg=\"baz.xx\"",
+  0
+};
+
+TEST(MonCap, ParseGood) {
+  for (int i=0; parse_good[i]; ++i) {
+    string str = parse_good[i];
+    MonCap cap;
+    std::cout << "Testing good input: '" << str << "'" << std::endl;
+    ASSERT_TRUE(cap.parse(str, &cout));
+    std::cout << "                                         -> " << cap << std::endl;
+  }
+}
+
+// these should stringify to the input value
+const char *parse_identity[] = {
+  "allow *",
+  "allow r",
+  "allow rwx",
+  "allow service foo x",
+  "allow profile osd",
+  "allow profile osd-bootstrap",
+  "allow profile mds-bootstrap, allow *",
+  "allow profile \"foo bar\", allow *",
+  "allow command abc",
+  "allow command \"a b c\"",
+  "allow command abc with arg=foo",
+  "allow command abc with arg=foo arg2=bar",
+  "allow command abc with arg=foo arg2=bar",
+  "allow command abc with arg=foo arg2 prefix bar arg3 prefix baz",
+  "allow command abc with arg=foo arg2 prefix \"bar bingo\" arg3 prefix baz",
+  "allow service foo x",
+  "allow service foo x, allow service bar x",
+  "allow service foo w, allow service bar x",
+  "allow service foo r, allow service bar x",
+  "allow service foo_foo r, allow service bar r",
+  "allow service foo-foo r, allow service bar r",
+  "allow service \" foo \" w, allow service bar r",
+  "allow command abc with arg=foo arg2=bar, allow service foo r",
+  0
+};
+
+TEST(MonCap, ParseIdentity)
+{
+  for (int i=0; parse_identity[i]; ++i) {
+    string str = parse_identity[i];
+    MonCap cap;
+    std::cout << "Testing good input: '" << str << "'" << std::endl;
+    ASSERT_TRUE(cap.parse(str, &cout));
+    string out = stringify(cap);
+    ASSERT_EQ(out, str);
+  }
+}
+
+const char *parse_bad[] = {
+  "allow r foo",
+  "allow*",
+  "foo allow *",
+  "allow profile foo rwx",
+  "allow profile",
+  "allow profile foo bar rwx",
+  "allow service bar",
+  "allow command baz x",
+  "allow r w",
+  "ALLOW r",
+  "allow rwx,",
+  "allow rwx x",
+  "allow r pool foo r",
+  "allow wwx pool taco",
+  "allow wwx pool taco^funny&chars",
+  "allow rwx pool 'weird name''",
+  "allow rwx object_prefix \"beforepool\" pool weird",
+  "allow rwx auid 123 pool asdf",
+  "allow command foo a prefix b",
+  "allow command foo with a prefixb",
+  "allow command foo with a = prefix b",
+  "allow command foo with a prefix b c",
+  0
+};
+
+TEST(MonCap, ParseBad) {
+  for (int i=0; parse_bad[i]; ++i) {
+    string str = parse_bad[i];
+    MonCap cap;
+    std::cout << "Testing bad input: '" << str << "'" << std::endl;
+    ASSERT_FALSE(cap.parse(str, &cout));
+  }
+}
+
+TEST(MonCap, AllowAll) {
+  MonCap cap;
+  ASSERT_FALSE(cap.is_allow_all());
+
+  ASSERT_TRUE(cap.parse("allow r", NULL));
+  ASSERT_FALSE(cap.is_allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow w", NULL));
+  ASSERT_FALSE(cap.is_allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow x", NULL));
+  ASSERT_FALSE(cap.is_allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow rwx", NULL));
+  ASSERT_FALSE(cap.is_allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow rw", NULL));
+  ASSERT_FALSE(cap.is_allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow rx", NULL));
+  ASSERT_FALSE(cap.is_allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow wx", NULL));
+  ASSERT_FALSE(cap.is_allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow *", NULL));
+  ASSERT_TRUE(cap.is_allow_all());
+  ASSERT_TRUE(cap.is_capable(NULL, entity_name_t::CLIENT(0),
+			     "foo", "asdf", map(), true, true, true));
+
+  MonCap cap2;
+  ASSERT_FALSE(cap2.is_allow_all());
+  cap2.set_allow_all();
+  ASSERT_TRUE(cap2.is_allow_all());
+}
+
+TEST(MonCap, ProfileOSD) {
+  MonCap cap;
+  bool r = cap.parse("allow profile osd", NULL);
+  ASSERT_TRUE(r);
+
+  entity_name_t name = entity_name_t::OSD(123);
+  map ca;
+
+  ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, false, false));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, true, false));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, true, true));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, true, true));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "mon", "", ca, true, false,false));
+
+  ASSERT_FALSE(cap.is_capable(NULL, name, "mds", "", ca, true, true, true));
+  ASSERT_FALSE(cap.is_capable(NULL, name, "mon", "", ca, true, true, true));
+
+  ca.clear();
+  ASSERT_FALSE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true));
+  ca["key"] = "daemon-private/osd.123";
+  ASSERT_FALSE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true));
+  ca["key"] = "daemon-private/osd.12/asdf";
+  ASSERT_FALSE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true));
+  ca["key"] = "daemon-private/osd.123/";
+  ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true));
+  ca["key"] = "daemon-private/osd.123/foo";
+  ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key put", ca, true, true, true));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key exists", ca, true, true, true));
+  ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key delete", ca, true, true, true));
+}
+
diff --git a/ceph/src/test/mon/test_mon_workloadgen.cc b/ceph/src/test/mon/test_mon_workloadgen.cc
new file mode 100644
index 00000000..3c6ff562
--- /dev/null
+++ b/ceph/src/test/mon/test_mon_workloadgen.cc
@@ -0,0 +1,1114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include "acconfig.h"
+
+#ifdef HAVE_SYS_MOUNT_H
+#include 
+#endif
+
+#ifdef HAVE_SYS_PARAM_H
+#include 
+#endif
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+
+#include "osd/osd_types.h"
+#include "osd/OSD.h"
+#include "osd/OSDMap.h"
+#include "osdc/Objecter.h"
+#include "mon/MonClient.h"
+#include "msg/Dispatcher.h"
+#include "msg/Messenger.h"
+#include "common/Timer.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/signal_handler.h"
+#include "common/config.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/Cond.h"
+#include "common/Mutex.h"
+#include "common/strtol.h"
+#include "common/LogEntry.h"
+#include "auth/KeyRing.h"
+#include "auth/AuthAuthorizeHandler.h"
+#include "include/uuid.h"
+#include "include/assert.h"
+
+#include "messages/MOSDBoot.h"
+#include "messages/MOSDAlive.h"
+#include "messages/MOSDPGCreate.h"
+#include "messages/MOSDPGRemove.h"
+#include "messages/MOSDMap.h"
+#include "messages/MPGStats.h"
+#include "messages/MLog.h"
+#include "messages/MOSDPGTemp.h"
+
+using namespace std;
+
+#define dout_subsys ceph_subsys_
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, get_name())
+static ostream& _prefix(std::ostream *_dout, string n) {
+  return *_dout << " stub(" << n << ") ";
+}
+
+
+typedef boost::mt11213b rngen_t;
+typedef boost::scoped_ptr MessengerRef;
+typedef boost::scoped_ptr ObjecterRef;
+
+class TestStub : public Dispatcher
+{
+ protected:
+  MessengerRef messenger;
+  MonClient monc;
+
+  Mutex lock;
+  Cond cond;
+  SafeTimer timer;
+
+  bool do_shutdown;
+  double tick_seconds;
+
+  struct C_Tick : public Context {
+    TestStub *s;
+    C_Tick(TestStub *stub) : s(stub) {}
+    void finish(int r) {
+      generic_dout(20) << "C_Tick::" << __func__ << dendl;
+      if (r == -ECANCELED) {
+	generic_dout(20) << "C_Tick::" << __func__
+			<< " shutdown" << dendl;
+	return;
+      }
+      s->tick();
+    }
+  };
+
+  virtual bool ms_dispatch(Message *m) = 0;
+  virtual void ms_handle_connect(Connection *con) = 0;
+  virtual void ms_handle_remote_reset(Connection *con) = 0;
+  virtual int _shutdown() = 0;
+  // courtesy method to be implemented by the stubs at their
+  // own discretion
+  virtual void _tick() { }
+  // different stubs may have different needs; if a stub needs
+  // to tick, then it must call this function.
+  void start_ticking(double t=1.0) {
+    tick_seconds = t;
+    if (t <= 0) {
+      stop_ticking();
+      return;
+    }
+    dout(20) << __func__ << " adding tick timer" << dendl;
+    timer.add_event_after(tick_seconds, new C_Tick(this));
+  }
+  // If we have a function to start ticking that the stubs can
+  // use at their own discretion, then we should also have a
+  // function to disable said ticking to be used the same way.
+  // Just in case.
+  // For simplicity's sake, we don't cancel the tick right off
+  // the bat; instead, we wait for the next tick to kick in and
+  // disable itself.
+  void stop_ticking() {
+    dout(20) << __func__ << " disable tick" << dendl;
+    tick_seconds = 0;
+  }
+
+ public:
+  void tick() {
+    std::cout << __func__ << std::endl;
+    if (do_shutdown || (tick_seconds <= 0)) {
+      std::cout << __func__ << " "
+		<< (do_shutdown ? "shutdown" : "stop ticking")
+		<< std::endl;
+      return;
+    }
+    _tick();
+    timer.add_event_after(tick_seconds, new C_Tick(this));
+  }
+
+  virtual const string get_name() = 0;
+  virtual int init() = 0;
+
+  virtual int shutdown() {
+    Mutex::Locker l(lock);
+    do_shutdown = true;
+    int r = _shutdown();
+    if (r < 0) {
+      dout(10) << __func__ << " error shutting down: "
+	       << cpp_strerror(-r) << dendl;
+      return r;
+    }
+    monc.shutdown();
+    timer.shutdown();
+    messenger->shutdown();
+    return 0;
+  }
+
+  virtual void print(ostream &out) {
+    out << "stub(" << get_name() << ")";
+  }
+
+  void wait() {
+    if (messenger != NULL)
+      messenger->wait();
+  }
+
+  TestStub(CephContext *cct, string who)
+    : Dispatcher(cct),
+      monc(cct),
+      lock(who.append("::lock").c_str()),
+      timer(cct, lock),
+      do_shutdown(false),
+      tick_seconds(0.0) { }
+};
+
+class ClientStub : public TestStub
+{
+  OSDMap osdmap;
+  ObjecterRef objecter;
+  rngen_t gen;
+
+ protected:
+  bool ms_dispatch(Message *m) {
+    Mutex::Locker l(lock);
+    dout(1) << "client::" << __func__ << " " << *m << dendl;
+    switch (m->get_type()) {
+    case CEPH_MSG_OSD_MAP:
+      objecter->handle_osd_map((MOSDMap*)m);
+      cond.Signal();
+      break;
+    }
+    return true;
+  }
+
+  void ms_handle_connect(Connection *con) {
+    dout(1) << "client::" << __func__ << " " << con << dendl;
+    Mutex::Locker l(lock);
+    objecter->ms_handle_connect(con);
+  }
+
+  void ms_handle_remote_reset(Connection *con) {
+    dout(1) << "client::" << __func__ << " " << con << dendl;
+    Mutex::Locker l(lock);
+    objecter->ms_handle_remote_reset(con);
+  }
+
+  bool ms_handle_reset(Connection *con) {
+    dout(1) << "client::" << __func__ << dendl;
+    Mutex::Locker l(lock);
+    objecter->ms_handle_reset(con);
+    return false;
+  }
+
+  const string get_name() {
+    return "client";
+  }
+
+  virtual int _shutdown() {
+    if (objecter) {
+      objecter->shutdown_locked();
+      objecter->shutdown_unlocked();
+    }
+    return 0;
+  }
+
+ public:
+  ClientStub(CephContext *cct)
+    : TestStub(cct, "client"),
+      gen((int) time(NULL))
+  { }
+
+  int init() {
+    int err;
+    err = monc.build_initial_monmap();
+    if (err < 0) {
+      derr << "ClientStub::" << __func__ << " ERROR: build initial monmap: "
+	   << cpp_strerror(err) << dendl;
+      return err;
+    }
+
+    messenger.reset(Messenger::create(cct, entity_name_t::CLIENT(-1),
+				      "stubclient", getpid()));
+    assert(messenger.get() != NULL);
+
+    messenger->set_default_policy(
+	Messenger::Policy::lossy_client(0, CEPH_FEATURE_OSDREPLYMUX));
+    dout(10) << "ClientStub::" << __func__ << " starting messenger at "
+	    << messenger->get_myaddr() << dendl;
+
+    objecter.reset(new Objecter(cct, messenger.get(), &monc, &osdmap,
+				lock, timer, 0, 0));
+    assert(objecter.get() != NULL);
+    objecter->set_balanced_budget();
+
+    monc.set_messenger(messenger.get());
+    messenger->add_dispatcher_head(this);
+    messenger->start();
+    monc.set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD);
+
+    err = monc.init();
+    if (err < 0) {
+      derr << "ClientStub::" << __func__ << " monc init error: "
+	   << cpp_strerror(-err) << dendl;
+      return err;
+    }
+
+    err = monc.authenticate();
+    if (err < 0) {
+      derr << "ClientStub::" << __func__ << " monc authenticate error: "
+	   << cpp_strerror(-err) << dendl;
+      monc.shutdown();
+      return err;
+    }
+    monc.wait_auth_rotating(30.0);
+
+    objecter->init_unlocked();
+
+    lock.Lock();
+    timer.init();
+    objecter->set_client_incarnation(0);
+    objecter->init_locked();
+    monc.renew_subs();
+
+    while (osdmap.get_epoch() == 0) {
+      dout(1) << "ClientStub::" << __func__ << " waiting for osdmap" << dendl;
+      cond.Wait(lock);
+    }
+
+    lock.Unlock();
+    dout(10) << "ClientStub::" << __func__ << " done" << dendl;
+
+    return 0;
+  }
+};
+
+typedef boost::scoped_ptr AuthHandlerRef;
+class OSDStub : public TestStub
+{
+  AuthHandlerRef auth_handler_registry;
+  int whoami;
+  OSDSuperblock sb;
+  OSDMap osdmap;
+  osd_stat_t osd_stat;
+
+  map pgs;
+  set pgs_changes;
+
+  rngen_t gen;
+  boost::uniform_int<> mon_osd_rng;
+
+  utime_t last_boot_attempt;
+  static const double STUB_BOOT_INTERVAL;
+
+
+ public:
+
+  enum {
+    STUB_MON_OSD_ALIVE	  = 1,
+    STUB_MON_OSD_PGTEMP	  = 2,
+    STUB_MON_OSD_FAILURE  = 3,
+    STUB_MON_OSD_PGSTATS  = 4,
+    STUB_MON_LOG	  = 5,
+
+    STUB_MON_OSD_FIRST	  = STUB_MON_OSD_ALIVE,
+    STUB_MON_OSD_LAST	  = STUB_MON_LOG,
+  };
+
+  struct C_CreatePGs : public Context {
+    OSDStub *s;
+    C_CreatePGs(OSDStub *stub) : s(stub) {}
+    void finish(int r) {
+      if (r == -ECANCELED) {
+	generic_dout(20) << "C_CreatePGs::" << __func__
+			<< " shutdown" << dendl;
+	return;
+      }
+      generic_dout(20) << "C_CreatePGs::" << __func__ << dendl;
+      s->auto_create_pgs();
+    }
+  };
+
+
+  OSDStub(int whoami, CephContext *cct)
+    : TestStub(cct, "osd"),
+      auth_handler_registry(new AuthAuthorizeHandlerRegistry(
+				  cct,
+				  cct->_conf->auth_cluster_required.length() ?
+				  cct->_conf->auth_cluster_required :
+				  cct->_conf->auth_supported)),
+      whoami(whoami),
+      gen(whoami),
+      mon_osd_rng(STUB_MON_OSD_FIRST, STUB_MON_OSD_LAST)
+  {
+    dout(20) << __func__ << " auth supported: "
+	     << cct->_conf->auth_supported << dendl;
+    stringstream ss;
+    ss << "client-osd" << whoami;
+    messenger.reset(Messenger::create(cct, entity_name_t::OSD(whoami),
+				      ss.str().c_str(), getpid()));
+
+    Throttle throttler(g_ceph_context, "osd_client_bytes",
+	g_conf->osd_client_message_size_cap);
+    uint64_t supported =
+      CEPH_FEATURE_UID |
+      CEPH_FEATURE_NOSRCADDR |
+      CEPH_FEATURE_PGID64;
+
+    messenger->set_default_policy(
+	Messenger::Policy::stateless_server(supported, 0));
+    messenger->set_policy_throttlers(entity_name_t::TYPE_CLIENT,
+				    &throttler, NULL);
+    messenger->set_policy(entity_name_t::TYPE_MON,
+	Messenger::Policy::lossy_client(supported, CEPH_FEATURE_UID |
+	  CEPH_FEATURE_PGID64 |
+	  CEPH_FEATURE_OSDENC));
+    messenger->set_policy(entity_name_t::TYPE_OSD,
+	Messenger::Policy::stateless_server(0,0));
+
+    dout(10) << __func__ << " public addr " << g_conf->public_addr << dendl;
+    int err = messenger->bind(g_conf->public_addr);
+    if (err < 0)
+      exit(1);
+
+    if (monc.build_initial_monmap() < 0)
+      exit(1);
+
+    messenger->start();
+    monc.set_messenger(messenger.get());
+  }
+
+  int init() {
+    dout(10) << __func__ << dendl;
+    Mutex::Locker l(lock);
+
+    dout(1) << __func__ << " fsid " << monc.monmap.fsid
+	    << " osd_fsid " << g_conf->osd_uuid << dendl;
+    dout(1) << __func__ << " name " << g_conf->name << dendl;
+
+    timer.init();
+    messenger->add_dispatcher_head(this);
+    monc.set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD);
+
+    int err = monc.init();
+    if (err < 0) {
+      derr << __func__ << " monc init error: "
+	   << cpp_strerror(-err) << dendl;
+      return err;
+    }
+
+    err = monc.authenticate();
+    if (err < 0) {
+      derr << __func__ << " monc authenticate error: "
+	   << cpp_strerror(-err) << dendl;
+      monc.shutdown();
+      return err;
+    }
+    assert(!monc.get_fsid().is_zero());
+
+    monc.wait_auth_rotating(30.0);
+
+
+    dout(10) << __func__ << " creating osd superblock" << dendl;
+    sb.cluster_fsid = monc.monmap.fsid;
+    sb.osd_fsid.generate_random();
+    sb.whoami = whoami;
+    sb.compat_features = CompatSet();
+    dout(20) << __func__ << " " << sb << dendl;
+    dout(20) << __func__ << " osdmap " << osdmap << dendl;
+
+    update_osd_stat();
+
+    start_ticking();
+    // give a chance to the mons to inform us of what PGs we should create
+    timer.add_event_after(30.0, new C_CreatePGs(this));
+
+    return 0;
+  }
+
+  int _shutdown() {
+
+    return 0;
+  }
+
+  void boot() {
+    dout(1) << __func__ << " boot?" << dendl;
+
+    utime_t now = ceph_clock_now(messenger->cct);
+    if ((last_boot_attempt > 0.0)
+	&& ((now - last_boot_attempt)) <= STUB_BOOT_INTERVAL) {
+      dout(1) << __func__ << " backoff and try again later." << dendl;
+      return;
+    }
+
+    dout(1) << __func__ << " boot!" << dendl;
+    MOSDBoot *mboot = new MOSDBoot;
+    mboot->sb = sb;
+    last_boot_attempt = now;
+    monc.send_mon_message(mboot);
+  }
+
+  void add_pg(pg_t pgid, epoch_t epoch, pg_t parent) {
+
+    utime_t now = ceph_clock_now(messenger->cct);
+
+    pg_stat_t s;
+    s.created = epoch;
+    s.last_epoch_clean = epoch;
+    s.parent = parent;
+    s.state |= PG_STATE_CLEAN | PG_STATE_ACTIVE;
+    s.last_fresh = now;
+    s.last_change = now;
+    s.last_clean = now;
+    s.last_active = now;
+    s.last_unstale = now;
+
+    pgs[pgid] = s;
+    pgs_changes.insert(pgid);
+  }
+
+  void auto_create_pgs() {
+    bool has_pgs = !pgs.empty();
+    dout(10) << __func__
+	     << ": " << (has_pgs ? "has pgs; ignore" : "create pgs") << dendl;
+    if (has_pgs)
+      return;
+
+    if (!osdmap.get_epoch()) {
+      dout(1) << __func__
+	      << " still don't have osdmap; reschedule pg creation" << dendl;
+      timer.add_event_after(10.0, new C_CreatePGs(this));
+      return;
+    }
+
+    const map &osdmap_pools = osdmap.get_pools();
+    map::const_iterator pit;
+    for (pit = osdmap_pools.begin(); pit != osdmap_pools.end(); ++pit) {
+      const int64_t pool_id = pit->first;
+      const pg_pool_t &pool = pit->second;
+      int ruleno = pool.get_crush_ruleset();
+
+      if (!osdmap.crush->rule_exists(ruleno)) {
+	dout(20) << __func__
+		 << " no crush rule for pool id " << pool_id
+		 << " rule no " << ruleno << dendl;
+	continue;
+      }
+
+      epoch_t pool_epoch = pool.get_last_change();
+      dout(20) << __func__
+	       << " pool num pgs " << pool.get_pg_num()
+	       << " epoch " << pool_epoch << dendl;
+
+      for (ps_t ps = 0; ps < pool.get_pg_num(); ++ps) {
+	pg_t pgid(ps, pool_id, -1);
+	pg_t parent;
+	dout(20) << __func__
+		 << " pgid " << pgid << " parent " << parent << dendl;
+	add_pg(pgid, pool_epoch, parent);
+      }
+    }
+  }
+
+  void update_osd_stat() {
+    struct statfs stbuf;
+    int ret = statfs(".", &stbuf);
+    if (ret < 0) {
+      ret = -errno;
+      dout(0) << __func__ 
+              << " cannot statfs ." << cpp_strerror(ret) << dendl;
+      return;
+    }
+
+    osd_stat.kb = stbuf.f_blocks * stbuf.f_bsize / 1024;
+    osd_stat.kb_used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize / 1024;
+    osd_stat.kb_avail = stbuf.f_bavail * stbuf.f_bsize / 1024;
+  }
+
+  void send_pg_stats() {
+    dout(10) << __func__
+	     << " pgs " << pgs.size() << " osdmap " << osdmap << dendl;
+    utime_t now = ceph_clock_now(messenger->cct);
+    MPGStats *mstats = new MPGStats(monc.get_fsid(), osdmap.get_epoch(), now);
+
+    mstats->set_tid(1);
+    mstats->osd_stat = osd_stat;
+
+    set::iterator it;
+    for (it = pgs_changes.begin(); it != pgs_changes.end(); ++it) {
+      pg_t pgid = (*it);
+      if (pgs.count(pgid) == 0) {
+	derr << __func__
+	     << " pgid " << pgid << " not on our map" << dendl;
+	assert(0 == "pgid not on our map");
+      }
+      pg_stat_t &s = pgs[pgid];
+      mstats->pg_stat[pgid] = s;
+
+      JSONFormatter f(true);
+      s.dump(&f);
+      dout(20) << __func__
+	       << " pg " << pgid << " stats:\n";
+      f.flush(*_dout);
+      *_dout << dendl;
+
+    }
+    dout(10) << __func__ << " send " << *mstats << dendl;
+    monc.send_mon_message(mstats);
+  }
+
+  void modify_pg(pg_t pgid) {
+    dout(10) << __func__ << " pg " << pgid << dendl;
+    assert(pgs.count(pgid) > 0);
+
+    pg_stat_t &s = pgs[pgid];
+    utime_t now = ceph_clock_now(messenger->cct);
+
+    if (now - s.last_change < 10.0) {
+      dout(10) << __func__
+	       << " pg " << pgid << " changed in the last 10s" << dendl;
+      return;
+    }
+
+    s.state ^= PG_STATE_CLEAN;
+    if (s.state & PG_STATE_CLEAN)
+      s.last_clean = now;
+    s.last_change = now;
+    s.reported_seq++;
+
+    pgs_changes.insert(pgid);
+  }
+
+  void modify_pgs() {
+    dout(10) << __func__ << dendl;
+
+    if (pgs.empty()) {
+      dout(1) << __func__
+	      << " no pgs available! don't attempt to modify." << dendl;
+      return;
+    }
+
+    boost::uniform_int<> pg_rng(0, pgs.size()-1);
+    set pgs_pos;
+
+    int num_pgs = pg_rng(gen);
+    while ((int)pgs_pos.size() < num_pgs)
+      pgs_pos.insert(pg_rng(gen));
+
+    map::iterator it = pgs.begin();
+    set::iterator pos_it = pgs_pos.begin();
+
+    int pgs_at = 0;
+    while (pos_it != pgs_pos.end()) {
+      int at = *pos_it;
+      dout(20) << __func__ << " pg at pos " << at << dendl;
+      while ((pgs_at != at) && (it != pgs.end())) {
+	++it;
+	++pgs_at;
+      }
+      assert(it != pgs.end());
+      dout(20) << __func__
+	       << " pg at pos " << at << ": " << it->first << dendl;
+      modify_pg(it->first);
+      ++pos_it;
+    }
+  }
+
+  void op_alive() {
+    dout(10) << __func__ << dendl;
+    if (!osdmap.exists(whoami)) {
+      dout(0) << __func__ << " I'm not in the osdmap!!\n";
+      JSONFormatter f(true);
+      osdmap.dump(&f);
+      f.flush(*_dout);
+      *_dout << dendl;
+    }
+    if (osdmap.get_epoch() == 0) {
+      dout(1) << __func__ << " wait for osdmap" << dendl;
+      return;
+    }
+    epoch_t up_thru = osdmap.get_up_thru(whoami);
+    dout(10) << __func__ << "up_thru: " << osdmap.get_up_thru(whoami) << dendl;
+
+    monc.send_mon_message(new MOSDAlive(osdmap.get_epoch(), up_thru));
+  }
+
+  void op_pgtemp() {
+    if (osdmap.get_epoch() == 0) {
+      dout(1) << __func__ << " wait for osdmap" << dendl;
+      return;
+    }
+    dout(10) << __func__ << dendl;
+    MOSDPGTemp *m = new MOSDPGTemp(osdmap.get_epoch());
+    monc.send_mon_message(m);
+  }
+
+  void op_failure() {
+    dout(10) << __func__ << dendl;
+  }
+
+  void op_pgstats() {
+    dout(10) << __func__ << dendl;
+
+    modify_pgs();
+    if (!pgs_changes.empty())
+      send_pg_stats();
+    monc.sub_want("osd_pg_creates", 0, CEPH_SUBSCRIBE_ONETIME);
+    monc.renew_subs();
+
+    dout(20) << __func__ << " pg pools:\n";
+
+    JSONFormatter f(true);
+    f.open_array_section("pools");
+    const map &osdmap_pools = osdmap.get_pools();
+    map::const_iterator pit;
+    for (pit = osdmap_pools.begin(); pit != osdmap_pools.end(); ++pit) {
+      const int64_t pool_id = pit->first;
+      const pg_pool_t &pool = pit->second;
+      f.open_object_section("pool");
+      f.dump_int("pool_id", pool_id);
+      f.open_object_section("pool_dump");
+      pool.dump(&f);
+      f.close_section();
+      f.close_section();
+    }
+    f.close_section();
+    f.flush(*_dout);
+    *_dout << dendl;
+  }
+
+  void op_log() {
+    dout(10) << __func__ << dendl;
+
+    MLog *m = new MLog(monc.get_fsid());
+
+    boost::uniform_int<> log_rng(1, 10);
+    size_t num_entries = log_rng(gen);
+    dout(10) << __func__
+	     << " send " << num_entries << " log messages" << dendl;
+
+    utime_t now = ceph_clock_now(messenger->cct);
+    int seq = 0;
+    for (; num_entries > 0; --num_entries) {
+      LogEntry e;
+      e.who = messenger->get_myinst();
+      e.stamp = now;
+      e.seq = seq++;
+      e.type = CLOG_DEBUG;
+      e.msg = "OSDStub::op_log";
+      m->entries.push_back(e);
+    }
+
+    monc.send_mon_message(m);
+  }
+
+  void _tick() {
+    if (!osdmap.exists(whoami)) {
+      std::cout << __func__ << " not in the cluster; boot!" << std::endl;
+      boot();
+      return;
+    }
+
+    update_osd_stat();
+
+    boost::uniform_int<> op_rng(STUB_MON_OSD_FIRST, STUB_MON_OSD_LAST);
+    int op = op_rng(gen);
+    switch (op) {
+    case STUB_MON_OSD_ALIVE:
+      op_alive();
+      break;
+    case STUB_MON_OSD_PGTEMP:
+      op_pgtemp();
+      break;
+    case STUB_MON_OSD_FAILURE:
+      op_failure();
+      break;
+    case STUB_MON_OSD_PGSTATS:
+      op_pgstats();
+      break;
+    case STUB_MON_LOG:
+      op_log();
+      break;
+    }
+  }
+
+  void handle_pg_create(MOSDPGCreate *m) {
+    assert(m != NULL);
+    if (m->epoch < osdmap.get_epoch()) {
+      std::cout << __func__ << " epoch " << m->epoch << " < "
+	       << osdmap.get_epoch() << "; dropping" << std::endl;
+      m->put();
+      return;
+    }
+
+    for (map::iterator it = m->mkpg.begin();
+	 it != m->mkpg.end(); ++it) {
+      pg_create_t &c = it->second;
+      std::cout << __func__ << " pg " << it->first
+	      << " created " << c.created
+	      << " parent " << c.parent << std::endl;
+      if (pgs.count(it->first)) {
+	std::cout << __func__ << " pg " << it->first
+		 << " exists; skipping" << std::endl;
+	continue;
+      }
+
+      pg_t pgid = it->first;
+      add_pg(pgid, c.created, c.parent);
+    }
+    send_pg_stats();
+  }
+
+  void handle_osd_map(MOSDMap *m) {
+    dout(1) << __func__ << dendl;
+    if (m->fsid != monc.get_fsid()) {
+      dout(0) << __func__
+              << " message fsid " << m->fsid << " != " << monc.get_fsid()
+              << dendl;
+      dout(0) << __func__ << " " << m
+              << " from " << m->get_source_inst()
+              << dendl;
+      dout(0) << monc.get_monmap() << dendl;
+    }
+    assert(m->fsid == monc.get_fsid());
+
+    epoch_t first = m->get_first();
+    epoch_t last = m->get_last();
+    dout(5) << __func__
+	    << " epochs [" << first << "," << last << "]"
+	    << " current " << osdmap.get_epoch() << dendl;
+
+    if (last <= osdmap.get_epoch()) {
+      dout(5) << __func__ << " no new maps here; dropping" << dendl;
+      m->put();
+      return;
+    }
+
+    if (first > osdmap.get_epoch() + 1) {
+      dout(5) << __func__
+	      << osdmap.get_epoch() + 1 << ".." << (first-1) << dendl;
+      if ((m->oldest_map < first && osdmap.get_epoch() == 0) ||
+	  m->oldest_map <= osdmap.get_epoch()) {
+	monc.sub_want("osdmap", osdmap.get_epoch()+1,
+		       CEPH_SUBSCRIBE_ONETIME);
+	monc.renew_subs();
+	m->put();
+	return;
+      }
+    }
+
+    epoch_t start_full = MAX(osdmap.get_epoch() + 1, first);
+
+    if (m->maps.size() > 0) {
+      map::reverse_iterator rit;
+      rit = m->maps.rbegin();
+      if (start_full <= rit->first) {
+	start_full = rit->first;
+	dout(5) << __func__
+		<< " full epoch " << start_full << dendl;
+	bufferlist &bl = rit->second;
+	bufferlist::iterator p = bl.begin();
+	osdmap.decode(p);
+      }
+    }
+
+    for (epoch_t e = start_full; e <= last; e++) {
+      map::iterator it;
+      it = m->incremental_maps.find(e);
+      if (it == m->incremental_maps.end())
+	continue;
+
+      dout(20) << __func__
+	       << " incremental epoch " << e
+	       << " on full epoch " << start_full << dendl;
+      OSDMap::Incremental inc;
+      bufferlist &bl = it->second;
+      bufferlist::iterator p = bl.begin();
+      inc.decode(p);
+
+      int err = osdmap.apply_incremental(inc);
+      if (err < 0) {
+	derr << "osd." << whoami << "::" << __func__
+	     << "** ERROR: applying incremental: "
+	     << cpp_strerror(err) << dendl;
+	assert(0 == "error applying incremental");
+      }
+    }
+    dout(30) << __func__ << "\nosdmap:\n";
+    JSONFormatter f(true);
+    osdmap.dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    if (osdmap.is_up(whoami) &&
+	osdmap.get_addr(whoami) == messenger->get_myaddr()) {
+      dout(1) << __func__
+	      << " got into the osdmap and we're up!" << dendl;
+    }
+
+    if (m->newest_map && m->newest_map > last) {
+      dout(1) << __func__
+	      << " they have more maps; requesting them!" << dendl;
+      monc.sub_want("osdmap", osdmap.get_epoch()+1, CEPH_SUBSCRIBE_ONETIME);
+      monc.renew_subs();
+    }
+
+    dout(10) << __func__ << " done" << dendl;
+    m->put();
+  }
+
+  bool ms_dispatch(Message *m) {
+    dout(1) << __func__ << " " << *m << dendl;
+
+    switch (m->get_type()) {
+    case MSG_OSD_PG_CREATE:
+      handle_pg_create((MOSDPGCreate*)m);
+      break;
+    case CEPH_MSG_OSD_MAP:
+      handle_osd_map((MOSDMap*)m);
+      break;
+    default:
+      m->put();
+      break;
+    }
+    return true;
+  }
+
+  void ms_handle_connect(Connection *con) {
+    dout(1) << __func__ << " " << con << dendl;
+    if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+      dout(10) << __func__ << " on mon" << dendl;
+    }
+  }
+
+  void ms_handle_remote_reset(Connection *con) {}
+
+  bool ms_handle_reset(Connection *con) {
+    dout(1) << __func__ << dendl;
+    OSD::Session *session = (OSD::Session *)con->get_priv();
+    if (!session)
+      return false;
+    session->put();
+    return true;
+  }
+
+  const string get_name() {
+    stringstream ss;
+    ss << "osd." << whoami;
+    return ss.str();
+  }
+};
+
+double const OSDStub::STUB_BOOT_INTERVAL = 10.0;
+
+#undef dout_prefix
+#define dout_prefix *_dout << "main "
+
+const char *our_name = NULL;
+vector stubs;
+Mutex shutdown_lock("main::shutdown_lock");
+Cond shutdown_cond;
+Context *shutdown_cb = NULL;
+SafeTimer *shutdown_timer = NULL;
+
+struct C_Shutdown : public Context
+{
+  void finish(int r) {
+    generic_dout(10) << "main::shutdown time has ran out" << dendl;
+    shutdown_cond.Signal();
+  }
+};
+
+void handle_test_signal(int signum)
+{
+  if ((signum != SIGINT) && (signum != SIGTERM))
+    return;
+
+  std::cerr << "*** Got signal " << sys_siglist[signum] << " ***" << std::endl;
+  Mutex::Locker l(shutdown_lock);
+  if (shutdown_timer) {
+    shutdown_timer->cancel_all_events();
+    shutdown_cond.Signal();
+  }
+}
+
+void usage() {
+  assert(our_name != NULL);
+
+  std::cout << "usage: " << our_name
+	    << " <--stub-id ID> [--stub-id ID...]"
+	    << std::endl;
+  std::cout << "\n\
+Global Options:\n\
+  -c FILE                   Read configuration from FILE\n\
+  --keyring FILE            Read keyring from FILE\n\
+  --help                    This message\n\
+\n\
+Test-specific Options:\n\
+  --stub-id ID1..ID2        Interval of OSD ids for multiple stubs to mimic.\n\
+  --stub-id ID              OSD id a stub will mimic to be\n\
+                            (same as --stub-id ID..ID)\n\
+" << std::endl;
+}
+
+int get_id_interval(int &first, int &last, string &str)
+{
+  size_t found = str.find("..");
+  string first_str, last_str;
+  if (found == string::npos) {
+    first_str = last_str = str;
+  } else {
+    first_str = str.substr(0, found);
+    last_str = str.substr(found+2);
+  }
+
+  string err;
+  first = strict_strtol(first_str.c_str(), 10, &err);
+  if ((first == 0) && (!err.empty())) {
+    std::cerr << err << std::endl;
+    return -1;
+  }
+
+  last = strict_strtol(last_str.c_str(), 10, &err);
+  if ((last == 0) && (!err.empty())) {
+    std::cerr << err << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int main(int argc, const char *argv[])
+{
+  vector def_args;
+  vector args;
+  our_name = argv[0];
+  argv_to_vec(argc, argv, args);
+
+  global_init(&def_args, args,
+	      CEPH_ENTITY_TYPE_OSD, CODE_ENVIRONMENT_UTILITY,
+	      0);
+
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  set stub_ids;
+  double duration = 300.0;
+
+  for (std::vector::iterator i = args.begin(); i != args.end();) {
+    string val;
+
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--stub-id", (char*) NULL)) {
+      int first = -1, last = -1;
+      if (get_id_interval(first, last, val) < 0) {
+	std::cerr << "** error parsing stub id '" << val << "'" << std::endl;
+	exit(1);
+      }
+
+      for (; first <= last; ++first)
+	stub_ids.insert(first);
+    } else if (ceph_argparse_witharg(args, i, &val,
+	"--duration", (char*) NULL)) {
+      string err;
+      duration = (double) strict_strtol(val.c_str(), 10, &err);
+      if ((duration == 0) && (!err.empty())) {
+	std::cerr << "** error parsing '--duration " << val << "': '"
+		  << err << std::endl;
+	exit(1);
+      }
+    } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
+      usage();
+      exit(0);
+    } else {
+      std::cerr << "unknown argument '" << *i << "'" << std::endl;
+      return 1;
+    }
+  }
+
+  if (stub_ids.empty()) {
+    std::cerr << "** error: must specify at least one '--stub-id '"
+         << std::endl;
+    usage();
+    return 1;
+  }
+
+  for (set::iterator i = stub_ids.begin(); i != stub_ids.end(); ++i) {
+    int whoami = *i;
+
+    std::cout << __func__ << " starting stub." << whoami << std::endl;
+    OSDStub *stub = new OSDStub(whoami, g_ceph_context);
+    int err = stub->init();
+    if (err < 0) {
+      std::cerr << "** osd stub error: " << cpp_strerror(-err) << std::endl;
+      return 1;
+    }
+    stubs.push_back(stub);
+  }
+
+  std::cout << __func__ << " starting client stub" << std::endl;
+  ClientStub *cstub = new ClientStub(g_ceph_context);
+  int err = cstub->init();
+  if (err < 0) {
+    std::cerr << "** client stub error: " << cpp_strerror(-err) << std::endl;
+    return 1;
+  }
+  stubs.push_back(cstub);
+
+  init_async_signal_handler();
+  register_async_signal_handler_oneshot(SIGINT, handle_test_signal);
+  register_async_signal_handler_oneshot(SIGTERM, handle_test_signal);
+
+  shutdown_lock.Lock();
+  shutdown_timer = new SafeTimer(g_ceph_context, shutdown_lock);
+  shutdown_timer->init();
+  if (duration != 0) {
+    std::cout << __func__
+	    << " run test for " << duration << " seconds" << std::endl;
+    shutdown_timer->add_event_after((double) duration, new C_Shutdown);
+  }
+  shutdown_cond.Wait(shutdown_lock);
+
+  shutdown_timer->shutdown();
+  delete shutdown_timer;
+  shutdown_timer = NULL;
+  shutdown_lock.Unlock();
+
+  unregister_async_signal_handler(SIGINT, handle_test_signal);
+  unregister_async_signal_handler(SIGTERM, handle_test_signal);
+
+  std::cout << __func__ << " waiting for stubs to finish" << std::endl;
+  vector::iterator it;
+  int i;
+  for (i = 0, it = stubs.begin(); it != stubs.end(); ++it, ++i) {
+    if (*it != NULL) {
+      (*it)->shutdown();
+      (*it)->wait();
+      std::cout << __func__ << " finished " << (*it)->get_name() << std::endl;
+      delete (*it);
+      (*it) = NULL;
+    }
+  }
+
+  return 0;
+}
diff --git a/ceph/src/test/multi_stress_watch.cc b/ceph/src/test/multi_stress_watch.cc
new file mode 100644
index 00000000..6203d7b1
--- /dev/null
+++ b/ceph/src/test/multi_stress_watch.cc
@@ -0,0 +1,160 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace librados;
+using ceph::buffer;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+static sem_t sem;
+
+class WatchNotifyTestCtx : public WatchCtx
+{
+public:
+    void notify(uint8_t opcode, uint64_t ver, bufferlist& bl)
+    {
+      sem_post(&sem);
+    }
+};
+
+void
+test_loop(Rados &cluster, std::string pool_name, std::string obj_name)
+{
+  int ret;
+  IoCtx ioctx;
+  ret = cluster.ioctx_create(pool_name.c_str(), ioctx);
+  if (ret < 0) {
+    std::cerr << "ioctx_create " << pool_name << " failed with " << ret << std::endl;
+    exit(1);
+  }
+
+  ret = ioctx.create(obj_name, false);
+  if (ret < 0) {
+    std::cerr << "create failed with " << ret << std::endl;
+    exit(1);
+  }
+
+  for (int i = 0; i < 10000; ++i) {
+    std::cerr << "Iteration " << i << std::endl;
+    uint64_t handle;
+    WatchNotifyTestCtx ctx;
+    ret = ioctx.watch(obj_name, 0, &handle, &ctx);
+    assert(!ret);
+    bufferlist bl2;
+    ret = ioctx.notify(obj_name, 0, bl2);
+    assert(!ret);
+    TestAlarm alarm;
+    sem_wait(&sem);
+    ioctx.unwatch(obj_name, handle);
+  }
+
+  ioctx.close();
+}
+
+void
+test_replicated(Rados &cluster, std::string pool_name, std::string obj_name)
+{
+  // May already exist
+  cluster.pool_create(pool_name.c_str());
+
+  test_loop(cluster, pool_name, obj_name);
+}
+
+void
+test_erasure(Rados &cluster, std::string pool_name, std::string obj_name)
+{
+  string outs;
+  bufferlist inbl;
+  int ret;
+  ret = cluster.mon_command(
+    "{\"prefix\": \"osd erasure-code-profile set\", \"name\": \"testprofile\", \"profile\": [ \"k=2\", \"m=1\", \"ruleset-failure-domain=osd\"]}",
+    inbl, NULL, &outs);
+  if (ret < 0) {
+    std::cerr << "mon_command erasure-code-profile set failed with " << ret << std::endl;
+    exit(1);
+  }
+  //std::cout << outs << std::endl;
+
+  outs.clear();
+  ret = cluster.mon_command(
+    "{\"prefix\": \"osd pool create\", \"pool\": \"" + pool_name + "\", \"pool_type\":\"erasure\", \"pg_num\":12, \"pgp_num\":12, \"erasure_code_profile\":\"testprofile\"}",
+    inbl, NULL, &outs);
+  if (ret < 0) {
+    std::cerr << outs << std::endl;
+    std::cerr << "mon_command create pool failed with " << ret << std::endl;
+    exit(1);
+  }
+  //std::cout << outs << std::endl;
+
+  cluster.wait_for_latest_osdmap();
+  test_loop(cluster, pool_name, obj_name);
+  return;
+}
+
+int main(int args, char **argv)
+{
+  if (args != 3 && args != 4) {
+    std::cerr << "Error: " << argv[0] << " [ec|rep] pool_name obj_name" << std::endl;
+    return 1;
+  }
+
+  std::string pool_name, obj_name, type;
+  // For backward compatibility with unmodified teuthology version
+  if (args == 3) {
+    type = "rep";
+    pool_name = argv[1];
+    obj_name = argv[2];
+  } else {
+    type = argv[1];
+    pool_name = argv[2];
+    obj_name = argv[3];
+  }
+  std::cerr << "Test type " << type << std::endl;
+  std::cerr << "pool_name, obj_name are " << pool_name << ", " << obj_name << std::endl;
+
+  if (type != "ec" && type != "rep") {
+    std::cerr << "Error: " << argv[0] << " Invalid arg must be 'ec' or 'rep' saw " << type << std::endl;
+    return 1;
+  }
+
+  char *id = getenv("CEPH_CLIENT_ID");
+  if (id) std::cerr << "Client id is: " << id << std::endl;
+  Rados cluster;
+  int ret;
+  ret = cluster.init(id);
+  if (ret) {
+    std::cerr << "Error " << ret << " in cluster.init" << std::endl;
+    return ret;
+  }
+  ret = cluster.conf_read_file(NULL);
+  if (ret) {
+    std::cerr << "Error " << ret << " in cluster.conf_read_file" << std::endl;
+    return ret;
+  }
+  ret = cluster.conf_parse_env(NULL);
+  if (ret) {
+    std::cerr << "Error " << ret << " in cluster.conf_read_env" << std::endl;
+    return ret;
+  }
+  cluster.connect();
+
+  if (type == "rep")
+    test_replicated(cluster, pool_name, obj_name);
+  else if (type == "ec")
+    test_erasure(cluster, pool_name, obj_name);
+
+  sem_destroy(&sem);
+  return 0;
+}
diff --git a/ceph/src/test/objectstore/DeterministicOpSequence.cc b/ceph/src/test/objectstore/DeterministicOpSequence.cc
new file mode 100644
index 00000000..09c9f5e7
--- /dev/null
+++ b/ceph/src/test/objectstore/DeterministicOpSequence.cc
@@ -0,0 +1,526 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "os/ObjectStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include 
+#include 
+
+#include "DeterministicOpSequence.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "deterministic_seq "
+
+DeterministicOpSequence::DeterministicOpSequence(ObjectStore *store,
+						 std::string status)
+  : TestObjectStoreState(store),
+    txn(0),
+    m_osr("OSR")
+{
+  txn_coll = coll_t("meta");
+  txn_object = hobject_t(sobject_t("txn", CEPH_NOSNAP));
+
+  if (!status.empty())
+    m_status.open(status.c_str());
+}
+
+DeterministicOpSequence::~DeterministicOpSequence()
+{
+  // TODO Auto-generated destructor stub
+}
+
+bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen)
+{
+  bool ok = false;
+  switch (op) {
+  case DSOP_TOUCH:
+    ok = do_touch(gen);
+    break;
+  case DSOP_WRITE:
+    ok = do_write(gen);
+    break;
+  case DSOP_CLONE:
+    ok = do_clone(gen);
+    break;
+  case DSOP_CLONE_RANGE:
+    ok = do_clone_range(gen);
+    break;
+  case DSOP_OBJ_REMOVE:
+    ok = do_remove(gen);
+    break;
+  case DSOP_COLL_ADD:
+    ok = do_coll_add(gen);
+    break;
+  case DSOP_COLL_RENAME:
+    //do_coll_rename(gen);
+    break;
+  case DSOP_SET_ATTRS:
+    ok = do_set_attrs(gen);
+    break;
+  default:
+    assert(0 == "bad op");
+  }
+  return ok;
+}
+
+void DeterministicOpSequence::generate(int seed, int num_txs)
+{
+  std::ostringstream ss;
+  ss << "generate run " << num_txs << " --seed " << seed;
+
+  if (m_status.is_open()) {
+    m_status << ss.str() << std::endl;
+    m_status.flush();
+  }
+
+  dout(0) << ss.str() << dendl;
+
+  rngen_t gen(seed);
+  boost::uniform_int<> op_rng(DSOP_FIRST, DSOP_LAST);
+
+  for (txn = 1; txn <= num_txs; ) {
+    int op = op_rng(gen);
+    _print_status(txn, op);
+    dout(0) << "generate seq " << txn << " op " << op << dendl;
+    if (run_one_op(op, gen))
+      txn++;
+  }
+}
+
+void DeterministicOpSequence::_print_status(int seq, int op)
+{
+  if (!m_status.is_open())
+    return;
+  m_status << seq << " " << op << std::endl;
+  m_status.flush();
+}
+
+int DeterministicOpSequence::_gen_coll_id(rngen_t& gen)
+{
+  boost::uniform_int<> coll_rng(0, m_collections_ids.size()-1);
+  return coll_rng(gen);
+}
+
+int DeterministicOpSequence::_gen_obj_id(rngen_t& gen)
+{
+  boost::uniform_int<> obj_rng(0, m_num_objects - 1);
+  return obj_rng(gen);
+}
+
+void DeterministicOpSequence::note_txn(ObjectStore::Transaction *t)
+{
+  bufferlist bl;
+  ::encode(txn, bl);
+  t->truncate(txn_coll, txn_object, 0);
+  t->write(txn_coll, txn_object, 0, bl.length(), bl);
+  dout(10) << __func__ << " " << txn << dendl;
+}
+
+bool DeterministicOpSequence::do_touch(rngen_t& gen)
+{
+  int coll_id = _gen_coll_id(gen);
+  int obj_id = _gen_obj_id(gen);
+
+  coll_entry_t *entry = get_coll_at(coll_id);
+  ceph_assert(entry != NULL);
+
+  // Don't care about other collections if already exists
+  if (!entry->check_for_obj(obj_id)) {
+    bool other_found = false;
+    map::iterator it = m_collections.begin();
+    for (; it != m_collections.end(); ++it) {
+      if (it->second->check_for_obj(obj_id)) {
+        ceph_assert(it->first != coll_id);
+        other_found = true;
+      }
+    }
+    if (other_found) {
+      dout(0) << "do_touch new object in collection and exists in another" << dendl;
+      return false;
+    }
+  }
+  hobject_t *obj = entry->touch_obj(obj_id);
+
+  dout(0) << "do_touch " << entry->m_coll.to_str() << "/" << obj->oid.name << dendl;
+
+  _do_touch(entry->m_coll, *obj);
+  return true;
+}
+
+bool DeterministicOpSequence::do_remove(rngen_t& gen)
+{
+  int coll_id = _gen_coll_id(gen);
+
+  coll_entry_t *entry = get_coll_at(coll_id);
+  ceph_assert(entry != NULL);
+
+  if (entry->m_objects.size() == 0) {
+    dout(0) << "do_remove no objects in collection" << dendl;
+    return false;
+  }
+  int obj_id = entry->get_random_obj_id(gen);
+  hobject_t *obj = entry->touch_obj(obj_id);
+  ceph_assert(obj);
+
+  dout(0) << "do_remove " << entry->m_coll.to_str() << "/" << obj->oid.name << dendl;
+
+  _do_remove(entry->m_coll, *obj);
+  hobject_t *rmobj = entry->remove_obj(obj_id);
+  ceph_assert(rmobj);
+  delete rmobj;
+  return true;
+}
+
+static void _gen_random(rngen_t& gen,
+			size_t size, bufferlist& bl) {
+
+  static const char alphanum[] = "0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    "abcdefghijklmnopqrstuvwxyz";
+
+  boost::uniform_int<> char_rng(0, sizeof(alphanum));
+  bufferptr bp(size);
+  for (unsigned int i = 0; i < size - 1; i++) {
+    bp[i] = alphanum[char_rng(gen)];
+  }
+  bp[size - 1] = '\0';
+  bl.append(bp);
+}
+
+static void gen_attrs(rngen_t &gen,
+		      map *out) {
+  boost::uniform_int<> num_rng(10, 30);
+  boost::uniform_int<> key_size_rng(5, 10);
+  boost::uniform_int<> val_size_rng(100, 1000);
+  size_t num_attrs = static_cast(num_rng(gen));
+  for (size_t i = 0; i < num_attrs; ++i) {
+    size_t key_size = static_cast(num_rng(gen));
+    size_t val_size = static_cast(num_rng(gen));
+    bufferlist keybl;
+    _gen_random(gen, key_size, keybl);
+    string key(keybl.c_str(), keybl.length());
+    _gen_random(gen, val_size, (*out)[key]);
+  }
+}
+
+bool DeterministicOpSequence::do_set_attrs(rngen_t& gen) 
+{
+  int coll_id = _gen_coll_id(gen);
+
+  coll_entry_t *entry = get_coll_at(coll_id);
+  ceph_assert(entry != NULL);
+
+  if (entry->m_objects.size() == 0) {
+    dout(0) << "do_set_attrs no objects in collection" << dendl;
+    return false;
+  }
+  int obj_id = entry->get_random_obj_id(gen);
+  hobject_t *obj = entry->touch_obj(obj_id);
+  ceph_assert(obj);
+
+  map out;
+  gen_attrs(gen, &out);
+
+  dout(0) << "do_set_attrs " << out.size() << " entries" << dendl;
+  _do_set_attrs(entry->m_coll, *obj, out);
+  return true;
+}
+
+bool DeterministicOpSequence::do_write(rngen_t& gen)
+{
+  int coll_id = _gen_coll_id(gen);
+
+  coll_entry_t *entry = get_coll_at(coll_id);
+  ceph_assert(entry != NULL);
+
+  if (entry->m_objects.size() == 0) {
+    dout(0) << "do_write no objects in collection" << dendl;
+    return false;
+  }
+  int obj_id = entry->get_random_obj_id(gen);
+  hobject_t *obj = entry->touch_obj(obj_id);
+  ceph_assert(obj);
+
+  boost::uniform_int<> size_rng(100, (2 << 19));
+  size_t size = (size_t) size_rng(gen);
+  bufferlist bl;
+  _gen_random(gen, size, bl);
+
+  dout(0) << "do_write " << entry->m_coll.to_str() << "/" << obj->oid.name
+	  << " 0~" << size << dendl;
+
+  _do_write(entry->m_coll, *obj, 0, bl.length(), bl);
+  return true;
+}
+
+bool DeterministicOpSequence::_prepare_clone(rngen_t& gen,
+					     coll_t& coll_ret, hobject_t& orig_obj_ret, hobject_t& new_obj_ret)
+{
+  int coll_id = _gen_coll_id(gen);
+
+  coll_entry_t *entry = get_coll_at(coll_id);
+  ceph_assert(entry != NULL);
+
+  if (entry->m_objects.size() >= 2) {
+    dout(0) << "_prepare_clone coll " << entry->m_coll.to_str()
+	    << " doesn't have 2 or more objects" << dendl;
+    return false;
+  }
+
+  int orig_obj_id = entry->get_random_obj_id(gen);
+  hobject_t *orig_obj = entry->touch_obj(orig_obj_id);
+  ceph_assert(orig_obj);
+
+  int id;
+  do {
+    id = entry->get_random_obj_id(gen);
+  } while (id == orig_obj_id);
+  hobject_t *new_obj = entry->touch_obj(id);
+  ceph_assert(new_obj);
+
+  coll_ret = entry->m_coll;
+  orig_obj_ret = *orig_obj;
+  new_obj_ret = *new_obj;
+
+  return true;
+}
+
+bool DeterministicOpSequence::do_clone(rngen_t& gen)
+{
+  coll_t coll;
+  hobject_t orig_obj, new_obj;
+  if (!_prepare_clone(gen, coll, orig_obj, new_obj)) {
+    return false;
+  }
+
+  dout(0) << "do_clone " << coll.to_str() << "/" << orig_obj.oid.name
+      << " => " << coll.to_str() << "/" << new_obj.oid.name << dendl;
+
+  _do_clone(coll, orig_obj, new_obj);
+  return true;
+}
+
+bool DeterministicOpSequence::do_clone_range(rngen_t& gen)
+{
+  coll_t coll;
+  hobject_t orig_obj, new_obj;
+  if (!_prepare_clone(gen, coll, orig_obj, new_obj)) {
+    return false;
+  }
+
+  /* Whenever we have to make a clone_range() operation, just write to the
+   * object first, so we know we have something to clone in the said range.
+   * This may not be the best solution ever, but currently we're not keeping
+   * track of the written-to objects, and until we know for sure we really
+   * need to, let's just focus on the task at hand.
+   */
+
+  boost::uniform_int<> write_size_rng(100, (2 << 19));
+  size_t size = (size_t) write_size_rng(gen);
+  bufferlist bl;
+  _gen_random(gen, size, bl);
+
+  boost::uniform_int<> clone_len(1, bl.length());
+  size = (size_t) clone_len(gen);
+
+  dout(0) << "do_clone_range " << coll.to_str() << "/" << orig_obj.oid.name
+      << " (0~" << size << ")"
+      << " => " << coll.to_str() << "/" << new_obj.oid.name
+      << " (0)" << dendl;
+  _do_write_and_clone_range(coll, orig_obj, new_obj, 0, size, 0, bl);
+  return true;
+}
+
+bool DeterministicOpSequence::_prepare_colls(rngen_t& gen,
+					     coll_entry_t* &orig_coll, coll_entry_t* &new_coll)
+{
+  ceph_assert(m_collections_ids.size() > 1);
+  int orig_coll_id = _gen_coll_id(gen);
+  int new_coll_id;
+  do {
+    new_coll_id = _gen_coll_id(gen);
+  } while (new_coll_id == orig_coll_id);
+
+  dout(0) << "_prepare_colls from coll id " << orig_coll_id
+      << " to coll id " << new_coll_id << dendl;
+
+  orig_coll = get_coll_at(orig_coll_id);
+  ceph_assert(orig_coll != NULL);
+  new_coll = get_coll_at(new_coll_id);
+  ceph_assert(new_coll != NULL);
+
+  if (!orig_coll->m_objects.size()) {
+    dout(0) << "_prepare_colls coll " << orig_coll->m_coll.to_str()
+        << " has no objects to use" << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+
+bool DeterministicOpSequence::do_coll_rename(rngen_t& gen)
+{
+  int coll_pos = _gen_coll_id(gen);
+  dout(0) << "do_coll_rename coll pos #" << coll_pos << dendl;
+
+  coll_entry_t *coll_entry = get_coll_at(coll_pos);
+  if (!coll_entry) {
+    dout(0) << "do_coll_rename no collection at pos #" << coll_pos << dendl;
+    return false;
+  }
+
+  coll_t orig_coll = coll_entry->m_coll;
+  char buf[100];
+  memset(buf, 0, 100);
+  snprintf(buf, 100, "0.%d_head", m_next_coll_nr++);
+  coll_t new_coll(buf);
+  coll_entry->m_coll = new_coll;
+
+  dout(0) << "do_coll_rename " << orig_coll.to_str()
+      << " => " << new_coll.to_str() << dendl;
+  _do_coll_rename(orig_coll, new_coll);
+  return true;
+}
+
+bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
+{
+  coll_entry_t *orig_coll = NULL, *new_coll = NULL;
+  if (!_prepare_colls(gen, orig_coll, new_coll))
+    return false;
+
+  ceph_assert(orig_coll && new_coll);
+
+  boost::uniform_int<> obj_rng(0, orig_coll->m_objects.size()-1);
+  int obj_pos = obj_rng(gen);
+  int obj_key = -1;
+  hobject_t *obj = orig_coll->get_obj_at(obj_pos, &obj_key);
+  if (!obj) {
+    dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
+        << " has no object as pos #" << obj_pos << " (key " << obj_key << ")"
+        << dendl;
+    return false;
+  }
+  if (new_coll->check_for_obj(obj_key)) {
+    dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
+        << " already has object as pos #" << obj_pos << " (key " << obj_key << ")"
+        << dendl;
+    return false;
+  }
+  dout(0) << "do_coll_add " << orig_coll->m_coll.to_str() << "/" << obj->oid.name
+        << " => " << new_coll->m_coll.to_str() << "/" << obj->oid.name << dendl;
+  new_coll->touch_obj(obj_key);
+
+  _do_coll_add(orig_coll->m_coll, new_coll->m_coll, *obj);
+  return true;
+}
+
+void DeterministicOpSequence::_do_touch(coll_t coll, hobject_t& obj)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.touch(coll, obj);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_remove(coll_t coll, hobject_t& obj)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.remove(coll, obj);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_set_attrs(coll_t coll,
+					    hobject_t &obj,
+					    const map &attrs)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.omap_setkeys(coll, obj, attrs);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_write(coll_t coll, hobject_t& obj,
+					uint64_t off, uint64_t len, const bufferlist& data)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.write(coll, obj, off, len, data);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_clone(coll_t coll, hobject_t& orig_obj,
+					hobject_t& new_obj)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.clone(coll, orig_obj, new_obj);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_clone_range(coll_t coll,
+					      hobject_t& orig_obj, hobject_t& new_obj, uint64_t srcoff,
+					      uint64_t srclen, uint64_t dstoff)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.clone_range(coll, orig_obj, new_obj, srcoff, srclen, dstoff);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_write_and_clone_range(coll_t coll,
+                                                        hobject_t& orig_obj,
+                                                        hobject_t& new_obj,
+                                                        uint64_t srcoff,
+                                                        uint64_t srclen,
+                                                        uint64_t dstoff,
+                                                        bufferlist& bl)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.write(coll, orig_obj, srcoff, bl.length(), bl);
+  t.clone_range(coll, orig_obj, new_obj, srcoff, srclen, dstoff);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_coll_add(coll_t orig_coll, coll_t new_coll,
+					   hobject_t& obj)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.remove(new_coll, obj);
+  t.collection_add(new_coll, orig_coll, obj);
+  m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_coll_rename(coll_t orig_coll, coll_t new_coll)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.collection_rename(orig_coll, new_coll);
+  m_store->apply_transaction(t);
+}
diff --git a/ceph/src/test/objectstore/DeterministicOpSequence.h b/ceph/src/test/objectstore/DeterministicOpSequence.h
new file mode 100644
index 00000000..1980c98c
--- /dev/null
+++ b/ceph/src/test/objectstore/DeterministicOpSequence.h
@@ -0,0 +1,98 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef FILESTORE_DTRMNSTC_SEQ_OPS_H_
+#define FILESTORE_DTRMNSTC_SEQ_OPS_H_
+
+#include 
+#include 
+#include 
+#include "os/ObjectStore.h"
+#include 
+#include 
+#include 
+
+#include "TestObjectStoreState.h"
+
+typedef boost::mt11213b rngen_t;
+
+class DeterministicOpSequence : public TestObjectStoreState {
+ public:
+  DeterministicOpSequence(ObjectStore *store, std::string status = std::string());
+  virtual ~DeterministicOpSequence();
+
+  virtual void generate(int seed, int num_txs);
+
+ protected:
+  enum {
+    DSOP_TOUCH = 0,
+    DSOP_WRITE = 1,
+    DSOP_CLONE = 2,
+    DSOP_CLONE_RANGE = 3,
+    DSOP_OBJ_REMOVE = 4,
+    DSOP_COLL_RENAME = 5,
+    DSOP_COLL_ADD = 6,
+    DSOP_SET_ATTRS = 7,
+
+    DSOP_FIRST = DSOP_TOUCH,
+    DSOP_LAST = DSOP_SET_ATTRS,
+  };
+
+  int32_t txn;
+
+  coll_t txn_coll;
+  hobject_t txn_object;
+
+  ObjectStore::Sequencer m_osr;
+  std::ofstream m_status;
+
+  bool run_one_op(int op, rngen_t& gen);
+
+  void note_txn(ObjectStore::Transaction *t);
+  bool do_touch(rngen_t& gen);
+  bool do_remove(rngen_t& gen);
+  bool do_write(rngen_t& gen);
+  bool do_clone(rngen_t& gen);
+  bool do_clone_range(rngen_t& gen);
+  bool do_coll_rename(rngen_t& gen);
+  bool do_coll_add(rngen_t& gen);
+  bool do_set_attrs(rngen_t& gen);
+
+  virtual void _do_touch(coll_t coll, hobject_t& obj);
+  virtual void _do_remove(coll_t coll, hobject_t& obj);
+  virtual void _do_write(coll_t coll, hobject_t& obj, uint64_t off,
+      uint64_t len, const bufferlist& data);
+  virtual void _do_set_attrs(coll_t coll,
+			     hobject_t &obj,
+			     const map &attrs);
+  virtual void _do_clone(coll_t coll, hobject_t& orig_obj, hobject_t& new_obj);
+  virtual void _do_clone_range(coll_t coll, hobject_t& orig_obj,
+      hobject_t& new_obj, uint64_t srcoff, uint64_t srclen, uint64_t dstoff);
+  virtual void _do_write_and_clone_range(coll_t coll, hobject_t& orig_obj,
+      hobject_t& new_obj, uint64_t srcoff, uint64_t srclen,
+      uint64_t dstoff, bufferlist& bl);
+  virtual void _do_coll_add(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
+  virtual void _do_coll_rename(coll_t orig_coll, coll_t new_coll);
+
+  int _gen_coll_id(rngen_t& gen);
+  int _gen_obj_id(rngen_t& gen);
+  void _print_status(int seq, int op);
+
+ private:
+  bool _prepare_clone(rngen_t& gen, coll_t& coll_ret,
+      hobject_t& orig_obj_ret, hobject_t& new_obj_ret);
+  bool _prepare_colls(rngen_t& gen,
+      coll_entry_t* &orig_coll, coll_entry_t* &new_coll);
+};
+
+
+#endif /* FILESTORE_DTRMNSTC_SEQ_OPS_H_ */
diff --git a/ceph/src/test/objectstore/FileStoreDiff.cc b/ceph/src/test/objectstore/FileStoreDiff.cc
new file mode 100644
index 00000000..40c0b32d
--- /dev/null
+++ b/ceph/src/test/objectstore/FileStoreDiff.cc
@@ -0,0 +1,319 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include "common/debug.h"
+#include "os/FileStore.h"
+#include "common/config.h"
+
+#include "FileStoreDiff.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "filestore_diff "
+
+FileStoreDiff::FileStoreDiff(FileStore *a, FileStore *b)
+    : a_store(a), b_store(b)
+{
+  int err;
+  err = a_store->mount();
+  ceph_assert(err == 0);
+
+  err = b_store->mount();
+  ceph_assert(err == 0);
+}
+
+FileStoreDiff::~FileStoreDiff()
+{
+  a_store->umount();
+  b_store->umount();
+}
+
+
+bool FileStoreDiff::diff_attrs(std::map& b,
+    std::map& a)
+{
+  bool ret = false;
+  std::map::iterator b_it = b.begin();
+  std::map::iterator a_it = a.begin();
+  for (; b_it != b.end(); ++b_it, ++a_it) {
+    if (b_it->first != a_it->first) {
+      dout(0) << "diff_attrs name mismatch (verify: " << b_it->first
+          << ", store: " << a_it->first << ")" << dendl;
+      ret = true;
+      continue;
+    }
+
+    if (!b_it->second.cmp(a_it->second)) {
+      dout(0) << "diff_attrs contents mismatch on attr " << b_it->first << dendl;
+      ret = true;
+      continue;
+    }
+  }
+  return ret;
+}
+
+static bool diff_omap(std::map& b,
+		      std::map& a)
+{
+  bool ret = false;
+  std::map::iterator b_it = b.begin();
+  std::map::iterator a_it = a.begin();
+  for (; b_it != b.end(); ++b_it, ++a_it) {
+    if (b_it->first != a_it->first) {
+      dout(0) << "diff_attrs name mismatch (verify: " << b_it->first
+          << ", store: " << a_it->first << ")" << dendl;
+      ret = true;
+      continue;
+    }
+
+    if (!(b_it->second == a_it->second)) {
+      dout(0) << "diff_attrs contents mismatch on attr " << b_it->first << dendl;
+      ret = true;
+      continue;
+    }
+  }
+  return ret;
+}
+
+bool FileStoreDiff::diff_objects_stat(struct stat& a, struct stat& b)
+{
+  bool ret = false;
+
+  if (a.st_uid != b.st_uid) {
+    dout(0) << "diff_objects_stat uid mismatch (A: "
+        << a.st_uid << " != B: " << b.st_uid << ")" << dendl;
+    ret = true;
+  }
+
+  if (a.st_gid != b.st_gid) {
+    dout(0) << "diff_objects_stat gid mismatch (A: "
+        << a.st_gid << " != B: " << b.st_gid << ")" << dendl;
+    ret = true;
+  }
+
+  if (a.st_mode != b.st_mode) {
+    dout(0) << "diff_objects_stat mode mismatch (A: "
+        << a.st_mode << " != B: " << b.st_mode << ")" << dendl;
+    ret = true;
+  }
+
+  if (a.st_nlink != b.st_nlink) {
+    dout(0) << "diff_objects_stat nlink mismatch (A: "
+        << a.st_nlink << " != B: " << b.st_nlink << ")" << dendl;
+    ret = true;
+  }
+
+  if (a.st_size != b.st_size) {
+    dout(0) << "diff_objects_stat size mismatch (A: "
+        << a.st_size << " != B: " << b.st_size << ")" << dendl;
+    ret = true;
+  }
+  return ret;
+}
+
+bool FileStoreDiff::diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll)
+{
+  dout(2) << __func__ << " coll "  << coll << dendl;
+
+  bool ret = false;
+
+  int err;
+  std::vector b_objects, a_objects;
+  err = b_store->collection_list(coll, b_objects);
+  if (err < 0) {
+    dout(0) << "diff_objects list on verify coll " << coll.to_str()
+	    << " returns " << err << dendl;
+    return true;
+  }
+  err = a_store->collection_list(coll, a_objects);
+  if (err < 0) {
+    dout(0) << "diff_objects list on store coll " << coll.to_str()
+              << " returns " << err << dendl;
+    return true;
+  }
+
+  if (b_objects.size() != a_objects.size()) {
+    dout(0) << "diff_objects num objs mismatch (A: " << a_objects.size()
+        << ", B: " << b_objects.size() << ")" << dendl;
+    ret = true;
+  }
+
+  std::vector::iterator b_it = b_objects.begin();
+  std::vector::iterator a_it = b_objects.begin();
+  for (; b_it != b_objects.end(); ++b_it, ++a_it) {
+    ghobject_t b_obj = *b_it, a_obj = *a_it;
+    if (b_obj.hobj.oid.name != a_obj.hobj.oid.name) {
+      dout(0) << "diff_objects name mismatch on A object "
+          << coll << "/" << a_obj << " and B object "
+          << coll << "/" << b_obj << dendl;
+      ret = true;
+      continue;
+    }
+
+    struct stat b_stat, a_stat;
+    err = b_store->stat(coll, b_obj, &b_stat);
+    if (err < 0) {
+      dout(0) << "diff_objects error stating B object "
+	      << coll.to_str() << "/" << b_obj.hobj.oid.name << dendl;
+      ret = true;
+    }
+    err = a_store->stat(coll, a_obj, &a_stat);
+    if (err < 0) {
+      dout(0) << "diff_objects error stating A object "
+          << coll << "/" << a_obj << dendl;
+      ret = true;
+    }
+
+    if (diff_objects_stat(a_stat, b_stat)) {
+      dout(0) << "diff_objects stat mismatch on "
+          << coll << "/" << b_obj << dendl;
+      ret = true;
+    }
+
+    bufferlist a_obj_bl, b_obj_bl;
+    b_store->read(coll, b_obj, 0, b_stat.st_size, b_obj_bl);
+    a_store->read(coll, a_obj, 0, a_stat.st_size, a_obj_bl);
+
+    if (!a_obj_bl.contents_equal(b_obj_bl)) {
+      dout(0) << "diff_objects content mismatch on "
+          << coll << "/" << b_obj << dendl;
+      ret = true;
+    }
+
+    std::map a_obj_attrs_map, b_obj_attrs_map;
+    err = a_store->getattrs(coll, a_obj, a_obj_attrs_map);
+    if (err < 0) {
+      dout(0) << "diff_objects getattrs on A object " << coll << "/" << a_obj
+              << " returns " << err << dendl;
+      ret = true;
+    }
+    err = b_store->getattrs(coll, b_obj, b_obj_attrs_map);
+    if (err < 0) {
+      dout(0) << "diff_objects getattrs on B object " << coll << "/" << b_obj
+              << "returns " << err << dendl;
+      ret = true;
+    }
+
+    if (diff_attrs(b_obj_attrs_map, a_obj_attrs_map)) {
+      dout(0) << "diff_objects attrs mismatch on A object "
+          << coll << "/" << a_obj << " and B object "
+          << coll << "/" << b_obj << dendl;
+      ret = true;
+    }
+
+    std::map a_obj_omap, b_obj_omap;
+    std::set a_omap_keys, b_omap_keys;
+    err = a_store->omap_get_keys(coll, a_obj, &a_omap_keys);
+    if (err < 0) {
+      dout(0) << "diff_objects getomap on A object " << coll << "/" << a_obj
+              << " returns " << err << dendl;
+      ret = true;
+    }
+    err = a_store->omap_get_values(coll, a_obj, a_omap_keys, &a_obj_omap);
+    if (err < 0) {
+      dout(0) << "diff_objects getomap on A object " << coll << "/" << a_obj
+              << " returns " << err << dendl;
+      ret = true;
+    }
+    err = b_store->omap_get_keys(coll, b_obj, &b_omap_keys);
+    if (err < 0) {
+      dout(0) << "diff_objects getomap on A object " << coll << "/" << b_obj
+              << " returns " << err << dendl;
+      ret = true;
+    }
+    err = b_store->omap_get_values(coll, b_obj, b_omap_keys, &b_obj_omap);
+    if (err < 0) {
+      dout(0) << "diff_objects getomap on A object " << coll << "/" << b_obj
+              << " returns " << err << dendl;
+      ret = true;
+    }
+    if (diff_omap(a_obj_omap, b_obj_omap)) {
+      dout(0) << "diff_objects omap mismatch on A object "
+	      << coll << "/" << a_obj << " and B object "
+	      << coll << "/" << b_obj << dendl;
+      ret = true;
+    }
+  }
+
+  return ret;
+}
+
+bool FileStoreDiff::diff_coll_attrs(FileStore *a_store, FileStore *b_store, coll_t coll)
+{
+  bool ret = false;
+
+  int err;
+  std::map b_coll_attrs, a_coll_attrs;
+  err = b_store->collection_getattrs(coll, b_coll_attrs);
+  if (err < 0) {
+    dout(0) << "diff_attrs getattrs on verify coll " << coll.to_str()
+        << "returns " << err << dendl;
+    ret = true;
+  }
+  err = a_store->collection_getattrs(coll, a_coll_attrs);
+  if (err < 0) {
+    dout(0) << "diff_attrs getattrs on A coll " << coll.to_str()
+              << "returns " << err << dendl;
+    ret = true;
+  }
+
+  if (b_coll_attrs.size() != a_coll_attrs.size()) {
+    dout(0) << "diff_attrs size mismatch (A: " << a_coll_attrs.size()
+        << ", B: " << a_coll_attrs.size() << ")" << dendl;
+    ret = true;
+  }
+
+  return diff_attrs(b_coll_attrs, a_coll_attrs) || ret;
+}
+
+bool FileStoreDiff::diff()
+{
+  bool ret = false;
+
+  std::vector a_coll_list, b_coll_list;
+  a_store->list_collections(a_coll_list);
+  b_store->list_collections(b_coll_list);
+
+  std::vector::iterator it = b_coll_list.begin();
+  for (; it != b_coll_list.end(); ++it) {
+    coll_t b_coll = *it;
+    if (!a_store->collection_exists(b_coll)) {
+      dout(0) << "diff B coll " << b_coll.to_str() << " DNE on A" << dendl;
+      ret = true;
+      continue;
+    }
+    for (std::vector::iterator j = a_coll_list.begin();
+	 j != a_coll_list.end(); ++j) {
+      if (*j == *it) {
+	a_coll_list.erase(j);
+	break;
+      }
+    }
+
+    if (diff_coll_attrs(a_store, b_store, b_coll))
+      ret = true;
+
+    if (diff_objects(a_store, b_store, b_coll))
+      ret = true;
+  }
+  for (std::vector::iterator it = a_coll_list.begin();
+       it != a_coll_list.end(); ++it) {
+    dout(0) << "diff A coll " << *it << " DNE on B" << dendl;
+    ret = true;
+  }
+
+  return ret;
+}
diff --git a/ceph/src/test/objectstore/FileStoreDiff.h b/ceph/src/test/objectstore/FileStoreDiff.h
new file mode 100644
index 00000000..cacd3ce8
--- /dev/null
+++ b/ceph/src/test/objectstore/FileStoreDiff.h
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef FILESTORE_DIFF_H_
+#define FILESTORE_DIFF_H_
+
+#include 
+#include 
+#include 
+#include 
+#include "common/debug.h"
+#include "os/FileStore.h"
+#include "common/config.h"
+
+class FileStoreDiff {
+
+ private:
+  FileStore *a_store;
+  FileStore *b_store;
+
+  bool diff_coll_attrs(FileStore *a_store, FileStore *b_store, coll_t coll);
+  bool diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll);
+  bool diff_objects_stat(struct stat& a, struct stat& b);
+  bool diff_attrs(std::map& b,
+      std::map& a);
+
+public:
+  FileStoreDiff(FileStore *a, FileStore *b);
+  virtual ~FileStoreDiff();
+
+  bool diff();
+};
+
+#endif /* FILESTOREDIFF_H_ */
diff --git a/ceph/src/test/objectstore/FileStoreTracker.cc b/ceph/src/test/objectstore/FileStoreTracker.cc
new file mode 100644
index 00000000..afdc31ba
--- /dev/null
+++ b/ceph/src/test/objectstore/FileStoreTracker.cc
@@ -0,0 +1,452 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "FileStoreTracker.h"
+#include 
+#include 
+#include 
+#include "include/Context.h"
+#include "common/Mutex.h"
+
+class OnApplied : public Context {
+  FileStoreTracker *tracker;
+  list, uint64_t> > in_flight;
+  ObjectStore::Transaction *t;
+public:
+  OnApplied(FileStoreTracker *tracker,
+	    list, uint64_t> > in_flight,
+	    ObjectStore::Transaction *t)
+    : tracker(tracker), in_flight(in_flight), t(t) {}
+
+  void finish(int r) {
+    for (list, uint64_t> >::iterator i =
+	   in_flight.begin();
+	 i != in_flight.end();
+	 ++i) {
+      tracker->applied(i->first, i->second);
+    }
+    delete t;
+  }
+};
+
+class OnCommitted : public Context {
+  FileStoreTracker *tracker;
+  list, uint64_t> > in_flight;
+public:
+  OnCommitted(FileStoreTracker *tracker,
+	      list, uint64_t> > in_flight)
+    : tracker(tracker), in_flight(in_flight) {}
+
+  void finish(int r) {
+    for (list, uint64_t> >::iterator i =
+	   in_flight.begin();
+	 i != in_flight.end();
+	 ++i) {
+      tracker->committed(i->first, i->second);
+    }
+  }
+};
+
+int FileStoreTracker::init()
+{
+  set to_get;
+  to_get.insert("STATUS");
+  map got;
+  db->get("STATUS", to_get, &got);
+  restart_seq = 0;
+  if (!got.empty()) {
+    bufferlist::iterator bp = got.begin()->second.begin();
+    ::decode(restart_seq, bp);
+  }
+  ++restart_seq;
+  KeyValueDB::Transaction t = db->get_transaction();
+  got.clear();
+  ::encode(restart_seq, got["STATUS"]);
+  t->set("STATUS", got);
+  db->submit_transaction(t);
+  return 0;
+}
+
+void FileStoreTracker::submit_transaction(Transaction &t)
+{
+  list, uint64_t> > in_flight;
+  OutTransaction out;
+  out.t = new ObjectStore::Transaction;
+  out.in_flight = &in_flight;
+  for (list::iterator i = t.ops.begin();
+       i != t.ops.end();
+       ++i) {
+    (**i)(this, &out);
+  }
+  store->queue_transaction(
+    0, out.t,
+    new OnApplied(this, in_flight, out.t),
+    new OnCommitted(this, in_flight));
+}
+
+void FileStoreTracker::write(const pair &obj,
+			     OutTransaction *out)
+{
+  Mutex::Locker l(lock);
+  std::cerr << "Writing " << obj << std::endl;
+  ObjectContents contents = get_current_content(obj);
+
+  uint64_t offset = rand() % (SIZE/2);
+  uint64_t len = rand() % (SIZE/2);
+  if (!len) len = 10;
+  contents.write(rand(), offset, len);
+
+  bufferlist to_write;
+  ObjectContents::Iterator iter = contents.get_iterator();
+  iter.seek_to(offset);
+  for (uint64_t i = offset;
+       i < offset + len;
+       ++i, ++iter) {
+    assert(iter.valid());
+    to_write.append(*iter);
+  }
+  out->t->write(coll_t(obj.first),
+		hobject_t(sobject_t(obj.second, CEPH_NOSNAP)),
+		offset,
+		len,
+		to_write);
+  out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
+}
+
+void FileStoreTracker::remove(const pair &obj,
+			      OutTransaction *out)
+{
+  std::cerr << "Deleting " << obj << std::endl;
+  Mutex::Locker l(lock);
+  ObjectContents old_contents = get_current_content(obj);
+  if (!old_contents.exists())
+    return;
+  out->t->remove(coll_t(obj.first),
+		 hobject_t(sobject_t(obj.second, CEPH_NOSNAP)));
+  ObjectContents contents;
+  out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
+}
+
+void FileStoreTracker::clone_range(const pair &from,
+				   const pair &to,
+				   OutTransaction *out) {
+  Mutex::Locker l(lock);
+  std::cerr << "CloningRange " << from << " to " << to << std::endl;
+  assert(from.first == to.first);
+  ObjectContents from_contents = get_current_content(from);
+  ObjectContents to_contents = get_current_content(to);
+  if (!from_contents.exists()) {
+    return;
+  }
+  if (from.second == to.second) {
+    return;
+  }
+
+  uint64_t new_size = from_contents.size();
+  interval_set interval_to_clone;
+  uint64_t offset = rand() % (new_size/2);
+  uint64_t len = rand() % (new_size/2);
+  if (!len) len = 10;
+  interval_to_clone.insert(offset, len);
+  to_contents.clone_range(from_contents, interval_to_clone);
+  out->t->clone_range(coll_t(from.first),
+		      hobject_t(sobject_t(from.second, CEPH_NOSNAP)),
+		      hobject_t(sobject_t(to.second, CEPH_NOSNAP)),
+		      offset,
+		      len,
+		      offset);
+  out->in_flight->push_back(make_pair(to, set_content(to, to_contents)));
+}
+
+void FileStoreTracker::clone(const pair &from,
+			     const pair &to,
+			     OutTransaction *out) {
+  Mutex::Locker l(lock);
+  std::cerr << "Cloning " << from << " to " << to << std::endl;
+  assert(from.first == to.first);
+  if (from.second == to.second) {
+    return;
+  }
+  ObjectContents from_contents = get_current_content(from);
+  ObjectContents to_contents = get_current_content(to);
+  if (!from_contents.exists()) {
+    return;
+  }
+
+  if (to_contents.exists())
+    out->t->remove(coll_t(to.first),
+		   hobject_t(sobject_t(to.second, CEPH_NOSNAP)));
+  out->t->clone(coll_t(from.first),
+		hobject_t(sobject_t(from.second, CEPH_NOSNAP)),
+		hobject_t(sobject_t(to.second, CEPH_NOSNAP)));
+  out->in_flight->push_back(make_pair(to, set_content(to, from_contents)));
+}
+
+
+string obj_to_prefix(const pair &obj) {
+  string sep;
+  sep.push_back('^');
+  return obj.first + sep + obj.second + "_CONTENTS_";
+}
+
+string obj_to_meta_prefix(const pair &obj) {
+  string sep;
+  sep.push_back('^');
+  return obj.first + sep + obj.second;
+}
+
+string seq_to_key(uint64_t seq) {
+  char buf[50];
+  snprintf(buf, sizeof(buf), "%*llu", 20, (unsigned long long int)seq);
+  return string(buf);
+}
+
+struct ObjStatus {
+  uint64_t last_applied;
+  uint64_t last_committed;
+  uint64_t restart_seq;
+  ObjStatus() : last_applied(0), last_committed(0), restart_seq(0) {}
+
+  uint64_t get_last_applied(uint64_t seq) const {
+    if (seq > restart_seq)
+      return last_committed;
+    else
+      return last_applied;
+  }
+  void set_last_applied(uint64_t _last_applied, uint64_t seq) {
+    last_applied = _last_applied;
+    restart_seq = seq;
+  }
+  uint64_t trim_to() const {
+    return last_applied < last_committed ?
+      last_applied : last_committed;
+  }
+};
+void encode(const ObjStatus &obj, bufferlist &bl) {
+  ::encode(obj.last_applied, bl);
+  ::encode(obj.last_committed, bl);
+  ::encode(obj.restart_seq, bl);
+}
+void decode(ObjStatus &obj, bufferlist::iterator &bl) {
+  ::decode(obj.last_applied, bl);
+  ::decode(obj.last_committed, bl);
+  ::decode(obj.restart_seq, bl);
+}
+
+
+ObjStatus get_obj_status(const pair &obj,
+			 KeyValueDB *db)
+{
+  set to_get;
+  to_get.insert("META");
+  map got;
+  db->get(obj_to_meta_prefix(obj), to_get, &got);
+  ObjStatus retval;
+  if (!got.empty()) {
+    bufferlist::iterator bp = got.begin()->second.begin();
+    ::decode(retval, bp);
+  }
+  return retval;
+}
+
+void set_obj_status(const pair &obj,
+		    const ObjStatus &status,
+		    KeyValueDB::Transaction t)
+{
+  map to_set;
+  ::encode(status, to_set["META"]);
+  t->set(obj_to_meta_prefix(obj), to_set);
+}
+
+void _clean_forward(const pair &obj,
+		    uint64_t last_valid,
+		    KeyValueDB *db)
+{
+  KeyValueDB::Transaction t = db->get_transaction();
+  KeyValueDB::Iterator i = db->get_iterator(obj_to_prefix(obj));
+  set to_remove;
+  i->upper_bound(seq_to_key(last_valid));
+  for (; i->valid(); i->next()) {
+    to_remove.insert(i->key());
+  }
+  t->rmkeys(obj_to_prefix(obj), to_remove);
+  db->submit_transaction(t);
+}
+
+
+void FileStoreTracker::verify(const string &coll, const string &obj,
+			      bool on_start) {
+  Mutex::Locker l(lock);
+  std::cerr << "Verifying " << make_pair(coll, obj) << std::endl;
+
+  pair valid_reads = get_valid_reads(make_pair(coll, obj));
+  std::cerr << "valid_reads is " << valid_reads << std::endl;
+  bufferlist contents;
+  int r = store->read(coll_t(coll),
+		      hobject_t(sobject_t(obj, CEPH_NOSNAP)),
+		      0,
+		      2*SIZE,
+		      contents);
+  std::cerr << "exists: " << r << std::endl;
+
+
+  for (uint64_t i = valid_reads.first;
+       i < valid_reads.second;
+       ++i) {
+    ObjectContents old_contents = get_content(make_pair(coll, obj), i);
+
+    std::cerr << "old_contents exists " << old_contents.exists() << std::endl;
+    if (!old_contents.exists() && (r == -ENOENT))
+      return;
+
+    if (old_contents.exists() && (r == -ENOENT))
+      continue;
+
+    if (!old_contents.exists() && (r != -ENOENT))
+      continue;
+
+    if (contents.length() != old_contents.size()) {
+      std::cerr << "old_contents.size() is "
+		<< old_contents.size() << std::endl;
+      continue;
+    }
+
+    bufferlist::iterator bp = contents.begin();
+    ObjectContents::Iterator iter = old_contents.get_iterator();
+    iter.seek_to_first();
+    bool matches = true;
+    uint64_t pos = 0;
+    for (; !bp.end() && iter.valid();
+	 ++iter, ++bp, ++pos) {
+      if (*iter != *bp) {
+	std::cerr << "does not match at pos " << pos << std::endl;
+	matches = false;
+	break;
+      }
+    }
+    if (matches) {
+      if (on_start)
+	_clean_forward(make_pair(coll, obj), i, db);
+      return;
+    }
+  }
+  std::cerr << "Verifying " << make_pair(coll, obj) << " failed " << std::endl;
+  assert(0);
+}
+
+ObjectContents FileStoreTracker::get_current_content(
+  const pair &obj)
+{
+  KeyValueDB::Iterator iter = db->get_iterator(
+    obj_to_prefix(obj));
+  iter->seek_to_last();
+  if (iter->valid()) {
+    bufferlist bl = iter->value();
+    bufferlist::iterator bp = bl.begin();
+    pair val;
+    ::decode(val, bp);
+    assert(seq_to_key(val.first) == iter->key());
+    bp = val.second.begin();
+    return ObjectContents(bp);
+  }
+  return ObjectContents();
+}
+
+ObjectContents FileStoreTracker::get_content(
+  const pair &obj, uint64_t version)
+{
+  set to_get;
+  map got;
+  to_get.insert(seq_to_key(version));
+  db->get(obj_to_prefix(obj), to_get, &got);
+  if (got.empty())
+    return ObjectContents();
+  pair val;
+  bufferlist::iterator bp = got.begin()->second.begin();
+  ::decode(val, bp);
+  bp = val.second.begin();
+  assert(val.first == version);
+  return ObjectContents(bp);
+}
+
+pair FileStoreTracker::get_valid_reads(
+  const pair &obj)
+{
+  pair bounds = make_pair(0,1);
+  KeyValueDB::Iterator iter = db->get_iterator(
+    obj_to_prefix(obj));
+  iter->seek_to_last();
+  if (iter->valid()) {
+    pair val;
+    bufferlist bl = iter->value();
+    bufferlist::iterator bp = bl.begin();
+    ::decode(val, bp);
+    bounds.second = val.first + 1;
+  }
+
+  ObjStatus obj_status = get_obj_status(obj, db);
+  bounds.first = obj_status.get_last_applied(restart_seq);
+  return bounds;
+}
+
+void clear_obsolete(const pair &obj,
+		    const ObjStatus &status,
+		    KeyValueDB *db,
+		    KeyValueDB::Transaction t)
+{
+  KeyValueDB::Iterator iter = db->get_iterator(obj_to_prefix(obj));
+  set to_remove;
+  iter->seek_to_first();
+  for (; iter->valid() && iter->key() < seq_to_key(status.trim_to());
+       iter->next())
+    to_remove.insert(iter->key());
+  t->rmkeys(obj_to_prefix(obj), to_remove);
+}
+
+void FileStoreTracker::committed(const pair &obj,
+				 uint64_t seq) {
+  Mutex::Locker l(lock);
+  ObjStatus status = get_obj_status(obj, db);
+  assert(status.last_committed < seq);
+  status.last_committed = seq;
+  KeyValueDB::Transaction t = db->get_transaction();
+  clear_obsolete(obj, status, db, t);
+  set_obj_status(obj, status, t);
+  db->submit_transaction(t);
+}
+
+void FileStoreTracker::applied(const pair &obj,
+			       uint64_t seq) {
+  Mutex::Locker l(lock);
+  std::cerr << "Applied " << obj << " version " << seq << std::endl;
+  ObjStatus status = get_obj_status(obj, db);
+  assert(status.last_applied < seq);
+  status.set_last_applied(seq, restart_seq);
+  KeyValueDB::Transaction t = db->get_transaction();
+  clear_obsolete(obj, status, db, t);
+  set_obj_status(obj, status, t);
+  db->submit_transaction(t);
+}
+
+
+uint64_t FileStoreTracker::set_content(const pair &obj,
+				       ObjectContents &content) {
+  KeyValueDB::Transaction t = db->get_transaction();
+  KeyValueDB::Iterator iter = db->get_iterator(
+    obj_to_prefix(obj));
+  iter->seek_to_last();
+  uint64_t most_recent = 0;
+  if (iter->valid()) {
+    pair val;
+    bufferlist bl = iter->value();
+    bufferlist::iterator bp = bl.begin();
+    ::decode(val, bp);
+    most_recent = val.first;
+  }
+  bufferlist buf_content;
+  content.encode(buf_content);
+  map to_set;
+  ::encode(make_pair(most_recent + 1, buf_content),
+	   to_set[seq_to_key(most_recent + 1)]);
+  t->set(obj_to_prefix(obj), to_set);
+  db->submit_transaction(t);
+  return most_recent + 1;
+}
diff --git a/ceph/src/test/objectstore/FileStoreTracker.h b/ceph/src/test/objectstore/FileStoreTracker.h
new file mode 100644
index 00000000..d70e54a1
--- /dev/null
+++ b/ceph/src/test/objectstore/FileStoreTracker.h
@@ -0,0 +1,138 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef FILESTORE_TRACKER_H
+#define FILESTORE_TRACKER_H
+#include "test/common/ObjectContents.h"
+#include "os/FileStore.h"
+#include "os/KeyValueDB.h"
+#include 
+#include 
+#include 
+#include "common/Mutex.h"
+
+class FileStoreTracker {
+  const static uint64_t SIZE = 4 * 1024;
+  ObjectStore *store;
+  KeyValueDB *db;
+  Mutex lock;
+  uint64_t restart_seq;
+
+  struct OutTransaction {
+    list, uint64_t> > *in_flight;
+    ObjectStore::Transaction *t;
+  };
+public:
+  FileStoreTracker(ObjectStore *store, KeyValueDB *db)
+    : store(store), db(db),
+      lock("Tracker Lock"), restart_seq(0) {}
+
+  class Transaction {
+    class Op {
+    public:
+      virtual void operator()(FileStoreTracker *harness,
+			      OutTransaction *out) = 0;
+      virtual ~Op() {};
+    };
+    list ops;
+    class Write : public Op {
+    public:
+      string coll;
+      string oid;
+      Write(const string &coll,
+	    const string &oid)
+	: coll(coll), oid(oid) {}
+      void operator()(FileStoreTracker *harness,
+		      OutTransaction *out) {
+	harness->write(make_pair(coll, oid), out);
+      }
+    };
+    class CloneRange : public Op {
+    public:
+      string coll;
+      string from;
+      string to;
+      CloneRange(const string &coll,
+		 const string &from,
+		 const string &to)
+	: coll(coll), from(from), to(to) {}
+      void operator()(FileStoreTracker *harness,
+		      OutTransaction *out) {
+	harness->clone_range(make_pair(coll, from), make_pair(coll, to),
+			     out);
+      }
+    };
+    class Clone : public Op {
+    public:
+      string coll;
+      string from;
+      string to;
+      Clone(const string &coll,
+		 const string &from,
+		 const string &to)
+	: coll(coll), from(from), to(to) {}
+      void operator()(FileStoreTracker *harness,
+		      OutTransaction *out) {
+	harness->clone(make_pair(coll, from), make_pair(coll, to),
+			     out);
+      }
+    };
+    class Remove: public Op {
+    public:
+      string coll;
+      string obj;
+      Remove(const string &coll,
+	     const string &obj)
+	: coll(coll), obj(obj) {}
+      void operator()(FileStoreTracker *harness,
+		      OutTransaction *out) {
+	harness->remove(make_pair(coll, obj),
+			out);
+      }
+    };
+  public:
+    void write(const string &coll, const string &oid) {
+      ops.push_back(new Write(coll, oid));
+    }
+    void clone_range(const string &coll, const string &from,
+		     const string &to) {
+      ops.push_back(new CloneRange(coll, from, to));
+    }
+    void clone(const string &coll, const string &from,
+	       const string &to) {
+      ops.push_back(new Clone(coll, from, to));
+    }
+    void remove(const string &coll, const string &oid) {
+      ops.push_back(new Remove(coll, oid));
+    }
+    friend class FileStoreTracker;
+  };
+
+  int init();
+  void submit_transaction(Transaction &t);
+  void verify(const string &coll,
+	      const string &from,
+	      bool on_start = false);
+
+private:
+  ObjectContents get_current_content(const pair &obj);
+  pair get_valid_reads(const pair &obj);
+  ObjectContents get_content(const pair &obj, uint64_t version);
+
+  void committed(const pair &obj, uint64_t seq);
+  void applied(const pair &obj, uint64_t seq);
+  uint64_t set_content(const pair &obj, ObjectContents &content);
+
+  // ObjectContents Operations
+  void write(const pair &obj, OutTransaction *out);
+  void remove(const pair &obj, OutTransaction *out);
+  void clone_range(const pair &from,
+		   const pair &to,
+		   OutTransaction *out);
+  void clone(const pair &from,
+	     const pair &to,
+	     OutTransaction *out);
+  friend class OnApplied;
+  friend class OnCommitted;
+};
+
+#endif
diff --git a/ceph/src/test/objectstore/TestObjectStoreState.cc b/ceph/src/test/objectstore/TestObjectStoreState.cc
new file mode 100644
index 00000000..37f99c7d
--- /dev/null
+++ b/ceph/src/test/objectstore/TestObjectStoreState.cc
@@ -0,0 +1,296 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "os/ObjectStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include 
+#include 
+#include "TestObjectStoreState.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "ceph_test_objectstore_state "
+
+const coll_t TestObjectStoreState::META_COLL("meta");
+const coll_t TestObjectStoreState::TEMP_COLL("temp");
+
+void TestObjectStoreState::init(int colls, int objs)
+{
+  dout(5) << "init " << colls << " colls " << objs << " objs" << dendl;
+
+  ObjectStore::Transaction *t;
+  t = new ObjectStore::Transaction;
+
+  t->create_collection(META_COLL);
+  t->create_collection(TEMP_COLL);
+  m_store->apply_transaction(*t);
+
+  wait_for_ready();
+
+  int baseid = 0;
+  for (int i = 0; i < colls; i++) {
+    int coll_id = i;
+    coll_entry_t *entry = coll_create(coll_id);
+    dout(5) << "init create collection " << entry->m_coll.to_str()
+        << " meta " << entry->m_meta_obj.oid.name << dendl;
+
+    t = new ObjectStore::Transaction;
+    t->create_collection(entry->m_coll);
+    t->touch(META_COLL, entry->m_meta_obj);
+
+    for (int i = 0; i < objs; i++) {
+      hobject_t *obj = entry->touch_obj(i + baseid);
+      t->touch(entry->m_coll, *obj);
+      ceph_assert(i + baseid == m_num_objects);
+      m_num_objects++;
+    }
+    baseid += objs;
+
+    m_store->queue_transaction(&(entry->m_osr), t,
+        new C_OnFinished(this, t));
+    inc_in_flight();
+
+    m_collections.insert(make_pair(coll_id, entry));
+    m_collections_ids.push_back(coll_id);
+    m_next_coll_nr++;
+  }
+  dout(5) << "init has " << m_in_flight.read() << "in-flight transactions" << dendl;
+  wait_for_done();
+  dout(5) << "init finished" << dendl;
+}
+
+TestObjectStoreState::coll_entry_t *TestObjectStoreState::coll_create(int id)
+{
+  char buf[100];
+  char meta_buf[100];
+  memset(buf, 0, 100);
+  memset(meta_buf, 0, 100);
+  snprintf(buf, 100, "0.%d_head", id);
+  snprintf(meta_buf, 100, "pglog_0.%d_head", id);
+  return (new coll_entry_t(id, buf, meta_buf));
+}
+
+TestObjectStoreState::coll_entry_t*
+TestObjectStoreState::get_coll(int key, bool erase)
+{
+  dout(5) << "get_coll id " << key << dendl;
+
+  coll_entry_t *entry = NULL;
+  map::iterator it = m_collections.find(key);
+  if (it != m_collections.end()) {
+    entry = it->second;
+    if (erase) {
+      m_collections.erase(it);
+      vector::iterator cid_it = m_collections_ids.begin()+(entry->m_id);
+      dout(20) << __func__ << " removing key " << key << " coll_id " << entry->m_id
+	      << " iterator's entry id " << (*cid_it) << dendl;
+      m_collections_ids.erase(cid_it);
+    }
+  }
+
+  dout(5) << "get_coll id " << key;
+  if (!entry)
+    *_dout << " non-existent";
+  else
+    *_dout << " name " << entry->m_coll.to_str();
+  *_dout << dendl;
+  return entry;
+}
+
+TestObjectStoreState::coll_entry_t*
+TestObjectStoreState::get_coll_at(int pos, bool erase)
+{
+  dout(5) << "get_coll_at pos " << pos << dendl;
+
+  if (m_collections.empty())
+    return NULL;
+
+  assert((size_t) pos < m_collections_ids.size());
+
+  int coll_id = m_collections_ids[pos];
+  coll_entry_t *entry = m_collections[coll_id];
+
+  if (entry == NULL) {
+    dout(5) << "get_coll_at pos " << pos << " non-existent" << dendl;
+    return NULL;
+  }
+
+  if (erase) {
+    m_collections.erase(coll_id);
+    vector::iterator it = m_collections_ids.begin()+(pos);
+    dout(20) << __func__ << " removing pos " << pos << " coll_id " << coll_id
+	    << " iterator's entry id " << (*it) << dendl;
+    m_collections_ids.erase(it);
+  }
+
+  dout(5) << "get_coll_at pos " << pos << ": "
+      << entry->m_coll << "(removed: " << erase << ")" << dendl;
+
+  return entry;
+}
+
+TestObjectStoreState::coll_entry_t::~coll_entry_t()
+{
+  if (m_objects.size() > 0) {
+    map::iterator it = m_objects.begin();
+    for (; it != m_objects.end(); ++it) {
+      hobject_t *obj = it->second;
+      if (obj) {
+        delete obj;
+      }
+    }
+    m_objects.clear();
+  }
+}
+
+bool TestObjectStoreState::coll_entry_t::check_for_obj(int id)
+{
+  if (m_objects.count(id))
+    return true;
+  return false;
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::touch_obj(int id)
+{
+  map::iterator it = m_objects.find(id);
+  if (it != m_objects.end()) {
+    dout(5) << "touch_obj coll id " << m_id
+        << " name " << it->second->oid.name << dendl;
+    return it->second;
+  }
+
+  char buf[100];
+  memset(buf, 0, 100);
+  snprintf(buf, 100, "obj%d", id);
+
+  hobject_t *obj = new hobject_t(sobject_t(object_t(buf), CEPH_NOSNAP));
+  m_objects.insert(make_pair(id, obj));
+
+  dout(5) << "touch_obj coll id " << m_id << " name " << buf << dendl;
+  return obj;
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj(int id)
+{
+  return get_obj(id, false);
+}
+
+/**
+ * remove_obj - Removes object without freeing it.
+ * @param id Object's id in the map.
+ * @return The object or NULL in case of error.
+ */
+hobject_t *TestObjectStoreState::coll_entry_t::remove_obj(int id)
+{
+  return get_obj(id, true);
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj(int id, bool remove)
+{
+  map::iterator it = m_objects.find(id);
+  if (it == m_objects.end()) {
+    dout(5) << "get_obj coll " << m_coll.to_str()
+        << " obj #" << id << " non-existent" << dendl;
+    return NULL;
+  }
+
+  hobject_t *obj = it->second;
+  if (remove)
+    m_objects.erase(it);
+
+  dout(5) << "get_obj coll " << m_coll.to_str() << " id " << id
+      << ": " << obj->oid.name << "(removed: " << remove << ")" << dendl;
+
+  return obj;
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj_at(int pos, int *key)
+{
+  return get_obj_at(pos, false, key);
+}
+
+/**
+ * remove_obj_at - Removes object without freeing it.
+ * @param pos The map's position in which the object lies.
+ * @return The object or NULL in case of error.
+ */
+hobject_t *TestObjectStoreState::coll_entry_t::remove_obj_at(int pos, int *key)
+{
+  return get_obj_at(pos, true, key);
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj_at(int pos,
+    bool remove, int *key)
+{
+  if (m_objects.empty()) {
+    dout(5) << "get_obj_at coll " << m_coll.to_str() << " pos " << pos
+        << " in an empty collection" << dendl;
+    return NULL;
+  }
+
+  hobject_t *ret = NULL;
+  map::iterator it = m_objects.begin();
+  for (int i = 0; it != m_objects.end(); ++it, i++) {
+    if (i == pos) {
+      ret = it->second;
+      break;
+    }
+  }
+
+  if (ret == NULL) {
+    dout(5) << "get_obj_at coll " << m_coll.to_str() << " pos " << pos
+        << " non-existent" << dendl;
+    return NULL;
+  }
+
+  if (key != NULL)
+    *key = it->first;
+
+  if (remove)
+    m_objects.erase(it);
+
+  dout(5) << "get_obj_at coll id " << m_id << " pos " << pos
+      << ": " << ret->oid.name << "(removed: " << remove << ")" << dendl;
+
+  return ret;
+}
+
+hobject_t*
+TestObjectStoreState::coll_entry_t::replace_obj(int id, hobject_t *obj) {
+  hobject_t *old_obj = remove_obj(id);
+  m_objects.insert(make_pair(id, obj));
+  return old_obj;
+}
+
+int TestObjectStoreState::coll_entry_t::get_random_obj_id(rngen_t& gen)
+{
+  ceph_assert(!m_objects.empty());
+
+  boost::uniform_int<> orig_obj_rng(0, m_objects.size()-1);
+  int pos = orig_obj_rng(gen);
+  map::iterator it = m_objects.begin();
+  for (int i = 0; it != m_objects.end(); ++it, i++) {
+    if (i == pos) {
+      return it->first;
+    }
+  }
+  ceph_assert(0 == "INTERNAL ERROR");
+}
diff --git a/ceph/src/test/objectstore/TestObjectStoreState.h b/ceph/src/test/objectstore/TestObjectStoreState.h
new file mode 100644
index 00000000..dad2aab7
--- /dev/null
+++ b/ceph/src/test/objectstore/TestObjectStoreState.h
@@ -0,0 +1,148 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef TEST_OBJECTSTORE_STATE_H_
+#define TEST_OBJECTSTORE_STATE_H_
+
+#include "os/ObjectStore.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+typedef boost::mt11213b rngen_t;
+
+class TestObjectStoreState {
+public:
+  struct coll_entry_t {
+    int m_id;
+    coll_t m_coll;
+    hobject_t m_meta_obj;
+    ObjectStore::Sequencer m_osr;
+    map m_objects;
+    int m_next_object_id;
+
+    coll_entry_t(int i, char *coll_buf, char *meta_obj_buf)
+    : m_id(i), m_coll(coll_buf),
+      m_meta_obj(sobject_t(object_t(meta_obj_buf), CEPH_NOSNAP)),
+      m_osr(coll_buf), m_next_object_id(0) {
+    }
+    ~coll_entry_t();
+
+    hobject_t *touch_obj(int id);
+    bool check_for_obj(int id);
+    hobject_t *get_obj(int id);
+    hobject_t *remove_obj(int id);
+    hobject_t *get_obj_at(int pos, int *key = NULL);
+    hobject_t *remove_obj_at(int pos, int *key = NULL);
+    hobject_t *replace_obj(int id, hobject_t *obj);
+    int get_random_obj_id(rngen_t& gen);
+
+   private:
+    hobject_t *get_obj(int id, bool remove);
+    hobject_t *get_obj_at(int pos, bool remove, int *key = NULL);
+  };
+
+  /* kept in upper case for consistency with coll_t's */
+  static const coll_t META_COLL;
+  static const coll_t TEMP_COLL;
+
+ protected:
+  boost::shared_ptr m_store;
+  map m_collections;
+  vector m_collections_ids;
+  int m_next_coll_nr;
+  int m_num_objs_per_coll;
+  int m_num_objects;
+
+  int m_max_in_flight;
+  atomic_t m_in_flight;
+  Mutex m_finished_lock;
+  Cond m_finished_cond;
+
+  void wait_for_ready() {
+    Mutex::Locker locker(m_finished_lock);
+    while ((m_max_in_flight > 0) && ((int)m_in_flight.read() >= m_max_in_flight))
+      m_finished_cond.Wait(m_finished_lock);
+  }
+
+  void wait_for_done() {
+    Mutex::Locker locker(m_finished_lock);
+    while (m_in_flight.read())
+      m_finished_cond.Wait(m_finished_lock);
+  }
+
+  void set_max_in_flight(int max) {
+    m_max_in_flight = max;
+  }
+  void set_num_objs_per_coll(int val) {
+    m_num_objs_per_coll = val;
+  }
+
+  coll_entry_t *get_coll(int key, bool erase = false);
+  coll_entry_t *get_coll_at(int pos, bool erase = false);
+
+ private:
+  static const int m_default_num_colls = 30;
+
+ public:
+  TestObjectStoreState(ObjectStore *store) :
+    m_next_coll_nr(0), m_num_objs_per_coll(10), m_num_objects(0),
+    m_max_in_flight(0), m_finished_lock("Finished Lock") {
+    m_in_flight.set(0);
+    m_store.reset(store);
+  }
+  ~TestObjectStoreState() { 
+    map::iterator it = m_collections.begin();
+    while (it != m_collections.end()) {
+      if (it->second)
+	delete it->second;
+      m_collections.erase(it++);
+    }
+  }
+
+  void init(int colls, int objs);
+  void init() {
+    init(m_default_num_colls, 0);
+  }
+
+  int inc_in_flight() {
+    return ((int) m_in_flight.inc());
+  }
+
+  int dec_in_flight() {
+    return ((int) m_in_flight.dec());
+  }
+
+  coll_entry_t *coll_create(int id);
+
+  class C_OnFinished: public Context {
+   protected:
+    TestObjectStoreState *m_state;
+    ObjectStore::Transaction *m_tx;
+
+   public:
+    C_OnFinished(TestObjectStoreState *state,
+        ObjectStore::Transaction *t) : m_state(state), m_tx(t) { }
+
+    void finish(int r) {
+      Mutex::Locker locker(m_state->m_finished_lock);
+      m_state->dec_in_flight();
+      m_state->m_finished_cond.Signal();
+
+      delete m_tx;
+    }
+  };
+};
+
+#endif /* TEST_OBJECTSTORE_STATE_H_ */
diff --git a/ceph/src/test/objectstore/chain_xattr.cc b/ceph/src/test/objectstore/chain_xattr.cc
new file mode 100644
index 00000000..8346c02b
--- /dev/null
+++ b/ceph/src/test/objectstore/chain_xattr.cc
@@ -0,0 +1,217 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+#include "os/chain_xattr.h"
+#include "include/Context.h"
+#include "common/errno.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include 
+
+#define LARGE_BLOCK_LEN CHAIN_XATTR_MAX_BLOCK_LEN + 1024
+
+TEST(chain_xattr, get_and_set) {
+  const char* file = "testfile";
+  ::unlink(file);
+  int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
+  const string user("user.");
+  
+  {
+    const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+    const string x(LARGE_BLOCK_LEN, 'X');
+
+    {
+      char y[LARGE_BLOCK_LEN];
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN));
+      ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
+      ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
+    }
+
+    {
+      char y[LARGE_BLOCK_LEN];
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN));
+      ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
+      ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
+    }
+  }
+
+  //
+  // when chain_setxattr is used to store value that is
+  // CHAIN_XATTR_MAX_BLOCK_LEN * 2 + 10 bytes long it 
+  //
+  // add user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+  // add user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+  // add user.foo@2 => 10 bytes
+  //
+  // then ( no chain_removexattr in between ) when it is used to
+  // override with a value that is exactly CHAIN_XATTR_MAX_BLOCK_LEN
+  // bytes long it will
+  //
+  // replace user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+  // remove user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+  // leak user.foo@2 => 10 bytes
+  // 
+  // see http://marc.info/?l=ceph-devel&m=136027076615853&w=4 for the 
+  // discussion
+  //
+  {
+    const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+    const string x(LARGE_BLOCK_LEN, 'X');
+
+    {
+      char y[CHAIN_XATTR_MAX_NAME_LEN];
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+      ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
+      ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
+      ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+      ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
+      ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+    }
+
+    {
+      char y[CHAIN_XATTR_MAX_BLOCK_LEN];
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+      ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
+      ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
+      ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+      ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
+      ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+    }
+  }
+
+  {
+    int x = 0;
+    ASSERT_EQ(-ENOENT, chain_setxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
+    ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", 0, 0));
+    ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
+    ASSERT_EQ(-ENOENT, chain_removexattr("UNLIKELY_TO_EXIST", "NAME"));
+    int unlikely_to_be_a_valid_fd = 400;
+    ASSERT_EQ(-EBADF, chain_fsetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
+    ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", 0, 0));
+    ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
+    ASSERT_EQ(-EBADF, chain_fremovexattr(unlikely_to_be_a_valid_fd, "NAME"));
+  }
+
+  {
+    int x;
+    const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN * 2, '@');
+    ASSERT_THROW(chain_setxattr(file, name.c_str(), &x, sizeof(x)), FailedAssertion);
+    ASSERT_THROW(chain_fsetxattr(fd, name.c_str(), &x, sizeof(x)), FailedAssertion);
+  }
+
+  {
+    const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+    const string x(LARGE_BLOCK_LEN, 'X');
+    {
+      char y[LARGE_BLOCK_LEN];
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+      ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN - 1));
+      ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+      ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
+    }
+
+    {
+      char y[LARGE_BLOCK_LEN];
+      ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+      ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN - 1));
+      ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+      ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
+    }
+  }
+
+  ::close(fd);
+  ::unlink(file);
+}
+
+TEST(chain_xattr, listxattr) {
+  const char* file = "testfile";
+  ::unlink(file);
+  int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
+  const string user("user.");
+  const string name1 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '1');
+  const string name2 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+  const string x(LARGE_BLOCK_LEN, 'X');
+  const int y = 1234;
+  
+  ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name1.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+  ASSERT_EQ((int)sizeof(y), chain_setxattr(file, name2.c_str(), &y, sizeof(y)));
+
+  int buffer_size = name1.size() + sizeof('\0') + name2.size() + sizeof('\0');
+  char* expected = (char*)malloc(buffer_size);
+  ::strcpy(expected, name1.c_str());
+  ::strcpy(expected + name1.size() + 1, name2.c_str());
+  char* actual = (char*)calloc(1, buffer_size);
+  ASSERT_LT(buffer_size, chain_listxattr(file, NULL, 0)); // size evaluation is conservative
+  chain_listxattr(file, actual, buffer_size);
+  ::memset(actual, '\0', buffer_size);
+  chain_flistxattr(fd, actual, buffer_size);
+  ASSERT_EQ(0, ::memcmp(expected, actual, buffer_size));
+
+  int unlikely_to_be_a_valid_fd = 400;
+  ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, 0));
+  ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, buffer_size));
+  ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, 0));
+  ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, buffer_size));
+  ASSERT_EQ(-ERANGE, chain_listxattr(file, actual, 1));
+  ASSERT_EQ(-ERANGE, chain_flistxattr(fd, actual, 1));
+
+  ASSERT_EQ(0, chain_removexattr(file, name1.c_str()));
+  ASSERT_EQ(0, chain_removexattr(file, name2.c_str()));
+
+  ::unlink(file);
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->set_val("err_to_stderr", "false");
+  g_ceph_context->_conf->set_val("log_to_stderr", "false");
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  const char* file = "testfile";
+  int x = 1234;
+  int y = 0;
+  int tmpfd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
+  int ret = ::ceph_os_fsetxattr(tmpfd, "user.test", &x, sizeof(x));
+  if (ret >= 0)
+    ret = ::ceph_os_fgetxattr(tmpfd, "user.test", &y, sizeof(y));
+  ::close(tmpfd);
+  ::unlink(file);
+  if ((ret < 0) || (x != y)) {
+    cerr << "SKIP all tests because extended attributes don't appear to work in the file system in which the tests are run: " << cpp_strerror(ret) << std::endl;
+  } else {
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+  }
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make unittest_chain_xattr ; valgrind --tool=memcheck ./unittest_chain_xattr # --gtest_filter=chain_xattr.get_and_set"
+// End:
diff --git a/ceph/src/test/objectstore/store_test.cc b/ceph/src/test/objectstore/store_test.cc
new file mode 100644
index 00000000..7c5dc583
--- /dev/null
+++ b/ceph/src/test/objectstore/store_test.cc
@@ -0,0 +1,1318 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "os/ObjectStore.h"
+#include "os/FileStore.h"
+#include "os/KeyValueStore.h"
+#include "include/Context.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/errno.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/unordered_map.h"
+typedef boost::mt11213b gen_type;
+
+#if GTEST_HAS_PARAM_TEST
+
+class StoreTest : public ::testing::TestWithParam {
+public:
+  boost::scoped_ptr store;
+
+  StoreTest() : store(0) {}
+  virtual void SetUp() {
+    int r = ::mkdir("store_test_temp_dir", 0777);
+    if (r < 0 && errno != EEXIST) {
+      r = -errno;
+      cerr << __func__ << ": unable to create store_test_temp_dir" << ": " << cpp_strerror(r) << std::endl;
+      return;
+    }
+
+    ObjectStore *store_ = ObjectStore::create(g_ceph_context,
+                                              string(GetParam()),
+                                              string("store_test_temp_dir"),
+                                              string("store_test_temp_journal"));
+    store.reset(store_);
+    EXPECT_EQ(store->mkfs(), 0);
+    EXPECT_EQ(store->mount(), 0);
+  }
+
+  virtual void TearDown() {
+    store->umount();
+  }
+};
+
+bool sorted(const vector &in) {
+  ghobject_t start;
+  for (vector::const_iterator i = in.begin();
+       i != in.end();
+       ++i) {
+    if (start > *i) return false;
+    start = *i;
+  }
+  return true;
+}
+
+TEST_P(StoreTest, SimpleColTest) {
+  coll_t cid = coll_t("initial");
+  int r = 0;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    cerr << "create collection" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.remove_collection(cid);
+    cerr << "remove collection" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    cerr << "add collection" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.remove_collection(cid);
+    cerr << "remove collection" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+}
+
+TEST_P(StoreTest, SimpleObjectTest) {
+  int r;
+  coll_t cid = coll_t("coll");
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    cerr << "Creating collection " << cid << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
+  {
+    ObjectStore::Transaction t;
+    t.touch(cid, hoid);
+    cerr << "Creating object " << hoid << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, hoid);
+    t.remove_collection(cid);
+    cerr << "Cleaning" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+}
+
+TEST_P(StoreTest, SimpleObjectLongnameTest) {
+  int r;
+  coll_t cid = coll_t("coll");
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    cerr << "Creating collection " << cid << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP)));
+  {
+    ObjectStore::Transaction t;
+    t.touch(cid, hoid);
+    cerr << "Creating object " << hoid << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, hoid);
+    t.remove_collection(cid);
+    cerr << "Cleaning" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+}
+
+TEST_P(StoreTest, ManyObjectTest) {
+  int NUM_OBJS = 2000;
+  int r = 0;
+  coll_t cid("blah");
+  string base = "";
+  for (int i = 0; i < 100; ++i) base.append("aaaaa");
+  set created;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  for (int i = 0; i < NUM_OBJS; ++i) {
+    if (!(i % 5)) {
+      cerr << "Object " << i << std::endl;
+    }
+    ObjectStore::Transaction t;
+    char buf[100];
+    snprintf(buf, sizeof(buf), "%d", i);
+    ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP)));
+    t.touch(cid, hoid);
+    created.insert(hoid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  for (set::iterator i = created.begin();
+       i != created.end();
+       ++i) {
+    struct stat buf;
+    ASSERT_TRUE(!store->stat(cid, *i, &buf));
+  }
+
+  set listed;
+  vector objects;
+  r = store->collection_list(cid, objects);
+  ASSERT_EQ(r, 0);
+
+  cerr << "objects.size() is " << objects.size() << std::endl;
+  for (vector ::iterator i = objects.begin();
+       i != objects.end();
+       ++i) {
+    listed.insert(*i);
+    ASSERT_TRUE(created.count(*i));
+  }
+  ASSERT_TRUE(listed.size() == created.size());
+
+  ghobject_t start, next;
+  objects.clear();
+  r = store->collection_list_partial(
+    cid,
+    ghobject_t::get_max(),
+    50,
+    60,
+    0,
+    &objects,
+    &next
+    );
+  ASSERT_EQ(r, 0);
+  ASSERT_TRUE(objects.empty());
+
+  objects.clear();
+  listed.clear();
+  while (1) {
+    r = store->collection_list_partial(cid, start,
+				       50,
+				       60,
+				       0,
+				       &objects,
+				       &next);
+    ASSERT_TRUE(sorted(objects));
+    ASSERT_EQ(r, 0);
+    listed.insert(objects.begin(), objects.end());
+    if (objects.size() < 50) {
+      ASSERT_TRUE(next.is_max());
+      break;
+    }
+    objects.clear();
+    start = next;
+  }
+  cerr << "listed.size() is " << listed.size() << std::endl;
+  ASSERT_TRUE(listed.size() == created.size());
+  for (set::iterator i = listed.begin();
+       i != listed.end();
+       ++i) {
+    ASSERT_TRUE(created.count(*i));
+  }
+
+  for (set::iterator i = created.begin();
+       i != created.end();
+       ++i) {
+    ObjectStore::Transaction t;
+    t.remove(cid, *i);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  cerr << "cleaning up" << std::endl;
+  {
+    ObjectStore::Transaction t;
+    t.remove_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+}
+
+
+class ObjectGenerator {
+public:
+  virtual ghobject_t create_object(gen_type *gen) = 0;
+  virtual ~ObjectGenerator() {}
+};
+
+class MixedGenerator : public ObjectGenerator {
+public:
+  unsigned seq;
+  MixedGenerator() : seq(0) {}
+  ghobject_t create_object(gen_type *gen) {
+    char buf[100];
+    snprintf(buf, sizeof(buf), "%u", seq);
+
+    boost::uniform_int<> true_false(0, 1);
+    string name(buf);
+    if (true_false(*gen)) {
+      // long
+      for (int i = 0; i < 100; ++i) name.append("aaaaa");
+    } else if (true_false(*gen)) {
+      name = "DIR_" + name;
+    }
+
+    // hash
+    //boost::binomial_distribution bin(0xFFFFFF, 0.5);
+    ++seq;
+    return ghobject_t(hobject_t(name, string(), rand() & 2 ? CEPH_NOSNAP : rand(), rand() & 0xFF, 0, ""));
+  }
+};
+
+class SyntheticWorkloadState {
+public:
+  static const unsigned max_in_flight = 16;
+  static const unsigned max_objects = 3000;
+  static const unsigned max_object_len = 1024 * 20;
+  coll_t cid;
+  unsigned in_flight;
+  map contents;
+  set available_objects;
+  set in_flight_objects;
+  ObjectGenerator *object_gen;
+  gen_type *rng;
+  ObjectStore *store;
+  ObjectStore::Sequencer *osr;
+
+  Mutex lock;
+  Cond cond;
+
+  class C_SyntheticOnReadable : public Context {
+  public:
+    SyntheticWorkloadState *state;
+    ObjectStore::Transaction *t;
+    ghobject_t hoid;
+    C_SyntheticOnReadable(SyntheticWorkloadState *state,
+                          ObjectStore::Transaction *t, ghobject_t hoid)
+      : state(state), t(t), hoid(hoid) {}
+
+    void finish(int r) {
+      Mutex::Locker locker(state->lock);
+      ASSERT_TRUE(state->in_flight_objects.count(hoid));
+      ASSERT_EQ(r, 0);
+      state->in_flight_objects.erase(hoid);
+      if (state->contents.count(hoid))
+        state->available_objects.insert(hoid);
+      --(state->in_flight);
+      state->cond.Signal();
+    }
+  };
+
+  static void filled_byte_array(bufferlist& bl, size_t size)
+  {
+    static const char alphanum[] = "0123456789"
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+      "abcdefghijklmnopqrstuvwxyz";
+
+    bufferptr bp(size);
+    for (unsigned int i = 0; i < size - 1; i++) {
+      bp[i] = alphanum[rand() % sizeof(alphanum)];
+    }
+    bp[size - 1] = '\0';
+
+    bl.append(bp);
+  }
+
+  SyntheticWorkloadState(ObjectStore *store,
+			 ObjectGenerator *gen,
+			 gen_type *rng,
+			 ObjectStore::Sequencer *osr,
+			 coll_t cid)
+    : cid(cid), in_flight(0), object_gen(gen), rng(rng), store(store), osr(osr),
+      lock("State lock") {}
+
+  int init() {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    return store->apply_transaction(t);
+  }
+
+  ghobject_t get_uniform_random_object() {
+    while (in_flight >= max_in_flight || available_objects.empty())
+      cond.Wait(lock);
+    boost::uniform_int<> choose(0, available_objects.size() - 1);
+    int index = choose(*rng);
+    set::iterator i = available_objects.begin();
+    for ( ; index > 0; --index, ++i) ;
+    ghobject_t ret = *i;
+    return ret;
+  }
+
+  void wait_for_ready() {
+    while (in_flight >= max_in_flight)
+      cond.Wait(lock);
+  }
+
+  void wait_for_done() {
+    Mutex::Locker locker(lock);
+    while (in_flight)
+      cond.Wait(lock);
+  }
+
+  bool can_create() {
+    return (available_objects.size() + in_flight_objects.size()) < max_objects;
+  }
+
+  bool can_unlink() {
+    return (available_objects.size() + in_flight_objects.size()) > 0;
+  }
+
+  int touch() {
+    Mutex::Locker locker(lock);
+    if (!can_create())
+      return -ENOSPC;
+    wait_for_ready();
+    ghobject_t new_obj = object_gen->create_object(rng);
+    available_objects.erase(new_obj);
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    t->touch(cid, new_obj);
+    ++in_flight;
+    in_flight_objects.insert(new_obj);
+    if (!contents.count(new_obj))
+      contents[new_obj] = bufferlist();
+    return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, new_obj));
+  }
+
+  int write() {
+    Mutex::Locker locker(lock);
+    if (!can_unlink())
+      return -ENOENT;
+    wait_for_ready();
+
+    ghobject_t new_obj = get_uniform_random_object();
+    available_objects.erase(new_obj);
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+
+    boost::uniform_int<> u1(0, max_object_len/2);
+    boost::uniform_int<> u2(0, max_object_len);
+    uint64_t offset = u1(*rng);
+    uint64_t len = u2(*rng);
+    bufferlist bl;
+    if (offset > len)
+      swap(offset, len);
+
+    filled_byte_array(bl, len);
+
+    if (contents[new_obj].length() <= offset) {
+      contents[new_obj].append_zero(offset-contents[new_obj].length());
+      contents[new_obj].append(bl);
+    } else {
+      bufferlist value;
+      contents[new_obj].copy(0, offset, value);
+      value.append(bl);
+      if (value.length() < contents[new_obj].length())
+        contents[new_obj].copy(value.length(), contents[new_obj].length()-value.length(), value);
+      value.swap(contents[new_obj]);
+    }
+
+    t->write(cid, new_obj, offset, len, bl);
+    ++in_flight;
+    in_flight_objects.insert(new_obj);
+    return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, new_obj));
+  }
+
+  void read() {
+    boost::uniform_int<> u1(0, max_object_len/2);
+    boost::uniform_int<> u2(0, max_object_len);
+    uint64_t offset = u1(*rng);
+    uint64_t len = u2(*rng);
+    if (offset > len)
+      swap(offset, len);
+
+    ghobject_t obj;
+    int r;
+    {
+      Mutex::Locker locker(lock);
+      if (!can_unlink())
+        return ;
+      wait_for_ready();
+
+      obj = get_uniform_random_object();
+    }
+    bufferlist bl, result;
+    r = store->read(cid, obj, offset, len, result);
+    if (offset >= contents[obj].length()) {
+      ASSERT_EQ(r, 0);
+    } else {
+      size_t max_len = contents[obj].length() - offset;
+      if (len > max_len)
+        len = max_len;
+      ASSERT_EQ(len, result.length());
+      contents[obj].copy(offset, len, bl);
+      ASSERT_EQ(r, (int)len);
+      ASSERT_TRUE(result.contents_equal(bl));
+    }
+  }
+
+  int truncate() {
+    Mutex::Locker locker(lock);
+    if (!can_unlink())
+      return -ENOENT;
+    wait_for_ready();
+
+    ghobject_t obj = get_uniform_random_object();
+    available_objects.erase(obj);
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+
+    boost::uniform_int<> choose(0, max_object_len);
+    size_t len = choose(*rng);
+    bufferlist bl;
+
+    t->truncate(cid, obj, len);
+    ++in_flight;
+    in_flight_objects.insert(obj);
+    if (contents[obj].length() <= len)
+      contents[obj].append_zero(len - contents[obj].length());
+    else {
+      contents[obj].copy(0, len, bl);
+      bl.swap(contents[obj]);
+    }
+
+    return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, obj));
+  }
+
+  void scan() {
+    Mutex::Locker locker(lock);
+    while (in_flight)
+      cond.Wait(lock);
+    vector objects;
+    set objects_set, objects_set2;
+    ghobject_t next, current;
+    while (1) {
+      cerr << "scanning..." << std::endl;
+      int r = store->collection_list_partial(cid, current, 50, 100,
+					     0, &objects, &next);
+      ASSERT_EQ(r, 0);
+      ASSERT_TRUE(sorted(objects));
+      objects_set.insert(objects.begin(), objects.end());
+      objects.clear();
+      if (next.is_max()) break;
+      current = next;
+    }
+    ASSERT_EQ(objects_set.size(), available_objects.size());
+    for (set::iterator i = objects_set.begin();
+	 i != objects_set.end();
+	 ++i) {
+      ASSERT_GT(available_objects.count(*i), (unsigned)0);
+    }
+
+    int r = store->collection_list(cid, objects);
+    ASSERT_EQ(r, 0);
+    objects_set2.insert(objects.begin(), objects.end());
+    ASSERT_EQ(objects_set2.size(), available_objects.size());
+    for (set::iterator i = objects_set2.begin();
+	 i != objects_set2.end();
+	 ++i) {
+      ASSERT_GT(available_objects.count(*i), (unsigned)0);
+    }
+  }
+
+  void stat() {
+    ghobject_t hoid;
+    {
+      Mutex::Locker locker(lock);
+      if (!can_unlink())
+        return ;
+      hoid = get_uniform_random_object();
+      in_flight_objects.insert(hoid);
+      available_objects.erase(hoid);
+      ++in_flight;
+    }
+    struct stat buf;
+    int r = store->stat(cid, hoid, &buf);
+    ASSERT_EQ(0, r);
+    ASSERT_TRUE(buf.st_size == contents[hoid].length());
+    {
+      Mutex::Locker locker(lock);
+      --in_flight;
+      cond.Signal();
+      in_flight_objects.erase(hoid);
+      available_objects.insert(hoid);
+    }
+  }
+
+  int unlink() {
+    Mutex::Locker locker(lock);
+    if (!can_unlink())
+      return -ENOENT;
+    ghobject_t to_remove = get_uniform_random_object();
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    t->remove(cid, to_remove);
+    ++in_flight;
+    available_objects.erase(to_remove);
+    in_flight_objects.insert(to_remove);
+    contents.erase(to_remove);
+    return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, to_remove));
+  }
+
+  void print_internal_state() {
+    Mutex::Locker locker(lock);
+    cerr << "available_objects: " << available_objects.size()
+	 << " in_flight_objects: " << in_flight_objects.size()
+	 << " total objects: " << in_flight_objects.size() + available_objects.size()
+	 << " in_flight " << in_flight << std::endl;
+  }
+};
+
+TEST_P(StoreTest, Synthetic) {
+  ObjectStore::Sequencer osr("test");
+  MixedGenerator gen;
+  gen_type rng(time(NULL));
+  coll_t cid("synthetic_1");
+
+  SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid);
+  test_obj.init();
+  for (int i = 0; i < 1000; ++i) {
+    if (!(i % 10)) cerr << "seeding object " << i << std::endl;
+    test_obj.touch();
+  }
+  for (int i = 0; i < 10000; ++i) {
+    if (!(i % 10)) {
+      cerr << "Op " << i << std::endl;
+      test_obj.print_internal_state();
+    }
+    boost::uniform_int<> true_false(0, 99);
+    int val = true_false(rng);
+    if (val > 97) {
+      test_obj.scan();
+    } else if (val > 90) {
+      test_obj.stat();
+    } else if (val > 85) {
+      test_obj.unlink();
+    } else if (val > 50) {
+      test_obj.write();
+    } else if (val > 10) {
+      test_obj.read();
+    } else {
+      test_obj.truncate();
+    }
+  }
+  test_obj.wait_for_done();
+}
+
+TEST_P(StoreTest, HashCollisionTest) {
+  coll_t cid("blah");
+  int r;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  string base = "";
+  for (int i = 0; i < 100; ++i) base.append("aaaaa");
+  set created;
+  for (int n = 0; n < 10; ++n) {
+    char nbuf[100];
+    sprintf(nbuf, "n%d", n);
+  for (int i = 0; i < 1000; ++i) {
+    char buf[100];
+    sprintf(buf, "%d", i);
+    if (!(i % 5)) {
+      cerr << "Object n" << n << " "<< i << std::endl;
+    }
+    ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, 0, string(nbuf)));
+    {
+      ObjectStore::Transaction t;
+      t.touch(cid, hoid);
+      r = store->apply_transaction(t);
+      ASSERT_EQ(r, 0);
+    }
+    created.insert(hoid);
+  }
+  }
+  vector objects;
+  r = store->collection_list(cid, objects);
+  ASSERT_EQ(r, 0);
+  set listed(objects.begin(), objects.end());
+  cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl;
+  ASSERT_TRUE(listed.size() == created.size());
+  objects.clear();
+  listed.clear();
+  ghobject_t current, next;
+  while (1) {
+    r = store->collection_list_partial(cid, current, 50, 60,
+				       0, &objects, &next);
+    ASSERT_EQ(r, 0);
+    ASSERT_TRUE(sorted(objects));
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i) {
+      if (listed.count(*i))
+	cerr << *i << " repeated" << std::endl;
+      listed.insert(*i);
+    }
+    if (objects.size() < 50) {
+      ASSERT_TRUE(next.is_max());
+      break;
+    }
+    objects.clear();
+    current = next;
+  }
+  cerr << "listed.size() is " << listed.size() << std::endl;
+  ASSERT_TRUE(listed.size() == created.size());
+  for (set::iterator i = listed.begin();
+       i != listed.end();
+       ++i) {
+    ASSERT_TRUE(created.count(*i));
+  }
+
+  for (set::iterator i = created.begin();
+       i != created.end();
+       ++i) {
+    ObjectStore::Transaction t;
+    t.collection_remove(cid, *i);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  ObjectStore::Transaction t;
+  t.remove_collection(cid);
+  store->apply_transaction(t);
+}
+
+TEST_P(StoreTest, OMapTest) {
+  coll_t cid("blah");
+  ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
+  int r;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  map attrs;
+  {
+    ObjectStore::Transaction t;
+    t.touch(cid, hoid);
+    t.omap_clear(cid, hoid);
+    map start_set;
+    t.omap_setkeys(cid, hoid, start_set);
+    store->apply_transaction(t);
+  }
+
+  for (int i = 0; i < 100; i++) {
+    if (!(i%5)) {
+      std::cout << "On iteration " << i << std::endl;
+    }
+    ObjectStore::Transaction t;
+    bufferlist bl;
+    map cur_attrs;
+    r = store->omap_get(cid, hoid, &bl, &cur_attrs);
+    ASSERT_EQ(r, 0);
+    for (map::iterator j = attrs.begin();
+	 j != attrs.end();
+	 ++j) {
+      bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
+      if (!correct) {
+	std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
+	if (cur_attrs.count(j->first) > 0) {
+	  std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
+	}
+      }
+      ASSERT_EQ(correct, true);
+    }
+    ASSERT_EQ(attrs.size(), cur_attrs.size());
+
+    char buf[100];
+    snprintf(buf, sizeof(buf), "%d", i);
+    bl.clear();
+    bufferptr bp(buf, strlen(buf) + 1);
+    bl.append(bp);
+    map to_add;
+    to_add.insert(pair("key-" + string(buf), bl));
+    attrs.insert(pair("key-" + string(buf), bl));
+    t.omap_setkeys(cid, hoid, to_add);
+    store->apply_transaction(t);
+  }
+
+  int i = 0;
+  while (attrs.size()) {
+    if (!(i%5)) {
+      std::cout << "removal: On iteration " << i << std::endl;
+    }
+    ObjectStore::Transaction t;
+    bufferlist bl;
+    map cur_attrs;
+    r = store->omap_get(cid, hoid, &bl, &cur_attrs);
+    ASSERT_EQ(r, 0);
+    for (map::iterator j = attrs.begin();
+	 j != attrs.end();
+	 ++j) {
+      bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
+      if (!correct) {
+	std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
+	if (cur_attrs.count(j->first) > 0) {
+	  std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
+	}
+      }
+      ASSERT_EQ(correct, true);
+    }
+
+    string to_remove = attrs.begin()->first;
+    set keys_to_remove;
+    keys_to_remove.insert(to_remove);
+    t.omap_rmkeys(cid, hoid, keys_to_remove);
+    store->apply_transaction(t);
+
+    attrs.erase(to_remove);
+
+    ++i;
+  }
+
+  {
+    bufferlist bl1;
+    bl1.append("omap_header");
+    ObjectStore::Transaction t;
+    t.omap_setheader(cid, hoid, bl1);
+    store->apply_transaction(t);
+
+    bufferlist bl2;
+    bl2.append("value");
+    map to_add;
+    to_add.insert(pair("key", bl2));
+    t.omap_setkeys(cid, hoid, to_add);
+    store->apply_transaction(t);
+
+    bufferlist bl3;
+    map cur_attrs;
+    r = store->omap_get(cid, hoid, &bl3, &cur_attrs);
+    ASSERT_EQ(r, 0);
+    ASSERT_EQ(cur_attrs.size(), size_t(1));
+    ASSERT_TRUE(bl3.contents_equal(bl1));
+  }
+
+  ObjectStore::Transaction t;
+  t.remove(cid, hoid);
+  t.remove_collection(cid);
+  store->apply_transaction(t);
+}
+
+TEST_P(StoreTest, XattrTest) {
+  coll_t cid("blah");
+  ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
+  bufferlist big;
+  for (unsigned i = 0; i < 10000; ++i) {
+    big.append('\0');
+  }
+  bufferlist small;
+  for (unsigned i = 0; i < 10; ++i) {
+    small.append('\0');
+  }
+  int r;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    t.touch(cid, hoid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  map attrs;
+  {
+    ObjectStore::Transaction t;
+    t.setattr(cid, hoid, "attr1", small);
+    attrs["attr1"] = small;
+    t.setattr(cid, hoid, "attr2", big);
+    attrs["attr2"] = big;
+    t.setattr(cid, hoid, "attr3", small);
+    attrs["attr3"] = small;
+    t.setattr(cid, hoid, "attr1", small);
+    attrs["attr1"] = small;
+    t.setattr(cid, hoid, "attr4", big);
+    attrs["attr4"] = big;
+    t.setattr(cid, hoid, "attr3", big);
+    attrs["attr3"] = big;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  map aset;
+  store->getattrs(cid, hoid, aset);
+  ASSERT_EQ(aset.size(), attrs.size());
+  for (map::iterator i = aset.begin();
+       i != aset.end();
+       ++i) {
+    bufferlist bl;
+    bl.push_back(i->second);
+    ASSERT_TRUE(attrs[i->first] == bl);
+  }
+
+  {
+    ObjectStore::Transaction t;
+    t.rmattr(cid, hoid, "attr2");
+    attrs.erase("attr2");
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  aset.clear();
+  store->getattrs(cid, hoid, aset);
+  ASSERT_EQ(aset.size(), attrs.size());
+  for (map::iterator i = aset.begin();
+       i != aset.end();
+       ++i) {
+    bufferlist bl;
+    bl.push_back(i->second);
+    ASSERT_TRUE(attrs[i->first] == bl);
+  }
+
+  bufferptr bp;
+  r = store->getattr(cid, hoid, "attr2", bp);
+  ASSERT_EQ(r, -ENODATA);
+
+  r = store->getattr(cid, hoid, "attr3", bp);
+  ASSERT_GE(r, 0);
+  bufferlist bl2;
+  bl2.push_back(bp);
+  ASSERT_TRUE(bl2 == attrs["attr3"]);
+}
+
+void colsplittest(
+  ObjectStore *store,
+  unsigned num_objects,
+  unsigned common_suffix_size
+  ) {
+  coll_t cid("from");
+  coll_t tid("to");
+  int r = 0;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    for (uint32_t i = 0; i < 2*num_objects; ++i) {
+      stringstream objname;
+      objname << "obj" << i;
+      t.touch(cid, ghobject_t(hobject_t(
+	  objname.str(),
+	  "",
+	  CEPH_NOSNAP,
+	  i<apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(tid);
+    t.split_collection(cid, common_suffix_size+1, 0, tid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  ObjectStore::Transaction t;
+  vector objects;
+  r = store->collection_list(cid, objects);
+  ASSERT_EQ(r, 0);
+  ASSERT_EQ(objects.size(), num_objects);
+  for (vector::iterator i = objects.begin();
+       i != objects.end();
+       ++i) {
+    ASSERT_EQ(!(i->hobj.hash & (1<collection_list(tid, objects);
+  ASSERT_EQ(r, 0);
+  ASSERT_EQ(objects.size(), num_objects);
+  for (vector::iterator i = objects.begin();
+       i != objects.end();
+       ++i) {
+    ASSERT_EQ(i->hobj.hash & (1<apply_transaction(t);
+  ASSERT_EQ(r, 0);
+}
+
+TEST_P(StoreTest, ColSplitTest1) {
+  colsplittest(store.get(), 10000, 11);
+}
+TEST_P(StoreTest, ColSplitTest2) {
+  colsplittest(store.get(), 100, 7);
+}
+
+#if 0
+TEST_P(StoreTest, ColSplitTest3) {
+  colsplittest(store.get(), 100000, 25);
+}
+#endif
+
+/**
+ * This test tests adding two different groups
+ * of objects, each with 1 common prefix and 1
+ * different prefix.  We then remove half
+ * in order to verify that the merging correctly
+ * stops at the common prefix subdir.  See bug
+ * #5273 */
+TEST_P(StoreTest, TwoHash) {
+  coll_t cid("asdf");
+  int r;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  std::cout << "Making objects" << std::endl;
+  for (int i = 0; i < 360; ++i) {
+    ObjectStore::Transaction t;
+    ghobject_t o;
+    if (i < 8) {
+      o.hobj.hash = (i << 16) | 0xA1;
+      t.touch(cid, o);
+    }
+    o.hobj.hash = (i << 16) | 0xB1;
+    t.touch(cid, o);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  std::cout << "Removing half" << std::endl;
+  for (int i = 1; i < 8; ++i) {
+    ObjectStore::Transaction t;
+    ghobject_t o;
+    o.hobj.hash = (i << 16) | 0xA1;
+    t.remove(cid, o);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  std::cout << "Checking" << std::endl;
+  for (int i = 1; i < 8; ++i) {
+    ObjectStore::Transaction t;
+    ghobject_t o;
+    o.hobj.hash = (i << 16) | 0xA1;
+    bool exists = store->exists(cid, o);
+    ASSERT_EQ(exists, false);
+  }
+  {
+    ghobject_t o;
+    o.hobj.hash = 0xA1;
+    bool exists = store->exists(cid, o);
+    ASSERT_EQ(exists, true);
+  }
+  std::cout << "Cleanup" << std::endl;
+  for (int i = 0; i < 360; ++i) {
+    ObjectStore::Transaction t;
+    ghobject_t o;
+    o.hobj.hash = (i << 16) | 0xA1;
+    t.remove(cid, o);
+    o.hobj.hash = (i << 16) | 0xB1;
+    t.remove(cid, o);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  ObjectStore::Transaction t;
+  t.remove_collection(cid);
+  r = store->apply_transaction(t);
+  ASSERT_EQ(r, 0);
+}
+
+TEST_P(StoreTest, MoveRename) {
+  coll_t temp_cid("mytemp");
+  hobject_t temp_oid("tmp_oid", "", CEPH_NOSNAP, 0, 0, "");
+  coll_t cid("dest");
+  hobject_t oid("dest_oid", "", CEPH_NOSNAP, 0, 0, "");
+  int r;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    t.touch(cid, oid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  ASSERT_TRUE(store->exists(cid, oid));
+  bufferlist data, attr;
+  map omap;
+  data.append("data payload");
+  attr.append("attr value");
+  omap["omap_key"].append("omap value");
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(temp_cid);
+    t.touch(temp_cid, temp_oid);
+    t.write(temp_cid, temp_oid, 0, data.length(), data);
+    t.setattr(temp_cid, temp_oid, "attr", attr);
+    t.omap_setkeys(temp_cid, temp_oid, omap);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  ASSERT_TRUE(store->exists(temp_cid, temp_oid));
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, oid);
+    t.collection_move_rename(temp_cid, temp_oid, cid, oid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  ASSERT_TRUE(store->exists(cid, oid));
+  ASSERT_FALSE(store->exists(temp_cid, temp_oid));
+  {
+    bufferlist newdata;
+    r = store->read(cid, oid, 0, 1000, newdata);
+    ASSERT_GE(r, 0);
+    ASSERT_TRUE(newdata.contents_equal(data));
+    bufferlist newattr;
+    r = store->getattr(cid, oid, "attr", newattr);
+    ASSERT_GE(r, 0);
+    ASSERT_TRUE(newattr.contents_equal(attr));
+    set keys;
+    keys.insert("omap_key");
+    map newomap;
+    r = store->omap_get_values(cid, oid, keys, &newomap);
+    ASSERT_GE(r, 0);
+    ASSERT_EQ(1u, newomap.size());
+    ASSERT_TRUE(newomap.count("omap_key"));
+    ASSERT_TRUE(newomap["omap_key"].contents_equal(omap["omap_key"]));
+  }
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, oid);
+    t.remove_collection(cid);
+    t.remove_collection(temp_cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+}
+
+TEST_P(StoreTest, BigRGWObjectName) {
+  store->set_allow_sharded_objects();
+  store->sync_and_flush();
+  coll_t temp_cid("mytemp");
+  hobject_t temp_oid("tmp_oid", "", CEPH_NOSNAP, 0, 0, "");
+  coll_t cid("dest");
+  ghobject_t oid(
+    hobject_t(
+      "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "",
+      CEPH_NOSNAP,
+      0x81920472,
+      3,
+      ""),
+    15,
+    shard_id_t(1));
+  ghobject_t oid2(oid);
+  oid2.generation = 17;
+  ghobject_t oidhead(oid);
+  oidhead.generation = ghobject_t::NO_GEN;
+
+  int r;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    t.touch(cid, oidhead);
+    t.collection_move_rename(cid, oidhead, cid, oid);
+    t.touch(cid, oidhead);
+    t.collection_move_rename(cid, oidhead, cid, oid2);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, oid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+
+  {
+    vector objects;
+    r = store->collection_list(cid, objects);
+    ASSERT_EQ(r, 0);
+    ASSERT_EQ(objects.size(), 1u);
+    ASSERT_EQ(objects[0], oid2);
+  }
+
+  ASSERT_FALSE(store->exists(cid, oid));
+
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, oid2);
+    t.remove_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+
+  }
+}
+
+TEST_P(StoreTest, SetAllocHint) {
+  coll_t cid("alloc_hint");
+  ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, 0, ""));
+  int r;
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    t.touch(cid, hoid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.remove(cid, hoid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    ObjectStore::Transaction t;
+    t.remove_collection(cid);
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(
+  ObjectStore,
+  StoreTest,
+  ::testing::Values("filestore", "keyvaluestore-dev"));
+
+#else
+
+// Google Test may not support value-parameterized tests with some
+// compilers. If we use conditional compilation to compile out all
+// code referring to the gtest_main library, MSVC linker will not link
+// that library at all and consequently complain about missing entry
+// point defined in that library (fatal error LNK1561: entry point
+// must be defined). This dummy test keeps gtest_main linked in.
+TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {}
+
+#endif
+
+
+//
+// support tests for qa/workunits/filestore/filestore.sh
+//
+TEST(EXT4StoreTest, _detect_fs) {
+  if (::getenv("DISK") == NULL || ::getenv("MOUNTPOINT") == NULL) {
+    cerr << "SKIP because DISK and MOUNTPOINT environment variables are not set. It is meant to run from qa/workunits/filestore/filestore.sh " << std::endl;
+    return;
+  }
+  const string disk(::getenv("DISK"));
+  EXPECT_LT((unsigned)0, disk.size());
+  const string mnt(::getenv("MOUNTPOINT"));
+  EXPECT_LT((unsigned)0, mnt.size());
+  ::umount(mnt.c_str());
+
+  const string dir("store_test_temp_dir");
+  const string journal("store_test_temp_journal");
+
+  //
+  // without user_xattr, ext4 fails
+  //
+  {
+    g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "true");
+    EXPECT_EQ(::system((string("mount -o loop,nouser_xattr ") + disk + " " + mnt).c_str()), 0);
+    EXPECT_EQ(::chdir(mnt.c_str()), 0);
+    EXPECT_EQ(::mkdir(dir.c_str(), 0755), 0);
+    FileStore store(dir, journal);
+    EXPECT_EQ(store._detect_fs(), -ENOTSUP);
+    EXPECT_EQ(::chdir(".."), 0);
+    EXPECT_EQ(::umount(mnt.c_str()), 0);
+  }
+  //
+  // mounted with user_xattr, ext4 fails if filestore_xattr_use_omap is false
+  //
+  {
+    g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "false");
+    EXPECT_EQ(::system((string("mount -o loop,user_xattr ") + disk + " " + mnt).c_str()), 0);
+    EXPECT_EQ(::chdir(mnt.c_str()), 0);
+    FileStore store(dir, journal);
+    EXPECT_EQ(store._detect_fs(), -ENOTSUP);
+    EXPECT_EQ(::chdir(".."), 0);
+    EXPECT_EQ(::umount(mnt.c_str()), 0);
+  }
+  //
+  // mounted with user_xattr, ext4 succeeds if filestore_xattr_use_omap is true
+  //
+  {
+    g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "true");
+    EXPECT_EQ(::system((string("mount -o loop,user_xattr ") + disk + " " + mnt).c_str()), 0);
+    EXPECT_EQ(::chdir(mnt.c_str()), 0);
+    FileStore store(dir, journal);
+    EXPECT_EQ(store._detect_fs(), 0);
+    EXPECT_EQ(::chdir(".."), 0);
+    EXPECT_EQ(::umount(mnt.c_str()), 0);
+  }
+}
+
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->set_val("osd_journal_size", "400");
+  g_ceph_context->_conf->set_val("filestore_index_retry_probability", "0.5");
+  g_ceph_context->_conf->set_val("filestore_op_thread_timeout", "1000");
+  g_ceph_context->_conf->set_val("filestore_op_thread_suicide_timeout", "10000");
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make ceph_test_objectstore ; ./ceph_test_objectstore --gtest_filter=StoreTest.* --log-to-stderr=true --debug-filestore=20"
+// End:
diff --git a/ceph/src/test/objectstore/test_idempotent.cc b/ceph/src/test/objectstore/test_idempotent.cc
new file mode 100644
index 00000000..4cfb8d11
--- /dev/null
+++ b/ceph/src/test/objectstore/test_idempotent.cc
@@ -0,0 +1,113 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include "os/FileStore.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+#include "test/common/ObjectContents.h"
+#include "FileStoreTracker.h"
+#include "os/LevelDBStore.h"
+#include "os/KeyValueDB.h"
+#include "os/ObjectStore.h"
+
+void usage(const string &name) {
+  std::cerr << "Usage: " << name << " [new|continue] store_path store_journal db_path"
+	    << std::endl;
+}
+
+template 
+typename T::iterator rand_choose(T &cont) {
+  if (cont.size() == 0) {
+    return cont.end();
+  }
+  int index = rand() % cont.size();
+  typename T::iterator retval = cont.begin();
+
+  for (; index > 0; --index) ++retval;
+  return retval;
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  std::cerr << "args: " << args << std::endl;
+  if (args.size() < 4) {
+    usage(argv[0]);
+    return 1;
+  }
+
+  string store_path(args[1]);
+  string store_dev(args[2]);
+  string db_path(args[3]);
+
+  bool start_new = false;
+  if (string(args[0]) == string("new")) start_new = true;
+
+  LevelDBStore *_db = new LevelDBStore(g_ceph_context, db_path);
+  assert(!_db->create_and_open(std::cerr));
+  boost::scoped_ptr db(_db);
+  boost::scoped_ptr store(new FileStore(store_path, store_dev));
+
+
+  if (start_new) {
+    std::cerr << "mkfs" << std::endl;
+    assert(!store->mkfs());
+    ObjectStore::Transaction t;
+    assert(!store->mount());
+    t.create_collection(coll_t("coll"));
+    store->apply_transaction(t);
+  } else {
+    assert(!store->mount());
+  }
+
+  FileStoreTracker tracker(store.get(), db.get());
+
+  set objects;
+  for (unsigned i = 0; i < 10; ++i) {
+    stringstream stream;
+    stream << "Object_" << i;
+    tracker.verify("coll", stream.str(), true);
+    objects.insert(stream.str());
+  }
+
+  while (1) {
+    FileStoreTracker::Transaction t;
+    for (unsigned j = 0; j < 100; ++j) {
+      int val = rand() % 100;
+      if (val < 30) {
+	t.write("coll", *rand_choose(objects));
+      } else if (val < 60) {
+	t.clone("coll", *rand_choose(objects),
+		*rand_choose(objects));
+      } else if (val < 70) {
+	t.remove("coll", *rand_choose(objects));
+      } else {
+	t.clone_range("coll", *rand_choose(objects),
+		      *rand_choose(objects));
+      }
+    }
+    tracker.submit_transaction(t);
+    tracker.verify("coll", *rand_choose(objects));
+  }
+  return 0;
+}
diff --git a/ceph/src/test/objectstore/test_idempotent_sequence.cc b/ceph/src/test/objectstore/test_idempotent_sequence.cc
new file mode 100644
index 00000000..3ef2c799
--- /dev/null
+++ b/ceph/src/test/objectstore/test_idempotent_sequence.cc
@@ -0,0 +1,247 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include "os/FileStore.h"
+
+#include "DeterministicOpSequence.h"
+#include "FileStoreDiff.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_
+#undef dout_prefix
+#define dout_prefix *_dout << "test_idempotent_sequence "
+
+void usage(const char *name, std::string command = "") {
+  assert(name != NULL);
+
+  std::string more = "cmd ";
+  std::string diff = "diff    ";
+  std::string get_last_op = "get-last-op  ";
+  std::string run_seq_to = "run-sequence-to   ";
+
+  if (!command.empty()) {
+    if (command == "diff")
+      more = diff;
+    else if (command == "get-last-op")
+      more = get_last_op;
+    else if (command == "run-sequence-to")
+      more = run_seq_to;
+  }
+  std::cout << "usage: " << name << " " << more << " [options]" << std::endl;
+
+  std::cout << "\n\
+Commands:\n\
+  " << diff << "\n\
+  " << get_last_op << "\n\
+  " << run_seq_to << "\n\
+\n\
+Global Options:\n\
+  -c FILE                             Read configuration from FILE\n\
+  --osd-data PATH                     Set OSD Data path\n\
+  --osd-journal PATH                  Set OSD Journal path\n\
+  --osd-journal-size VAL              Set Journal size\n\
+  --help                              This message\n\
+\n\
+Test-specific Options:\n\
+  --test-seed VAL                     Seed to run the test\n\
+  --test-status-file PATH             Path to keep the status file\n\
+  --test-num-colls VAL                Number of collections to create on init\n\
+  --test-num-objs VAL                 Number of objects to create on init\n\
+" << std::endl;
+}
+
+const char *our_name = NULL;
+int seed = 0, num_txs = 100, num_colls = 30, num_objs = 0;
+bool is_seed_set = false;
+int verify_at = 0;
+std::string status_file;
+
+int run_diff(std::string& a_path, std::string& a_journal,
+	      std::string& b_path, std::string& b_journal)
+{
+  FileStore *a = new FileStore(a_path, a_journal, "a");
+  FileStore *b = new FileStore(b_path, b_journal, "b");
+
+  int ret = 0;
+  {
+    FileStoreDiff fsd(a, b);
+    if (fsd.diff()) {
+      dout(0) << "diff found an difference" << dendl;
+      ret = -1;
+    } else {
+      dout(0) << "no diff" << dendl;
+    }
+  }
+
+  delete a;
+  delete b;
+  return ret;
+}
+
+int run_get_last_op(std::string& filestore_path, std::string& journal_path)
+{
+  FileStore *store = new FileStore(filestore_path, journal_path);
+
+  int err = store->mount();
+  if (err) {
+    store->umount();
+    delete store;
+    return err;
+  }
+
+  coll_t txn_coll("meta");
+  hobject_t txn_object(sobject_t("txn", CEPH_NOSNAP));
+  bufferlist bl;
+  store->read(txn_coll, txn_object, 0, 100, bl);
+  int32_t txn = 0;
+  if (bl.length()) {
+    bufferlist::iterator p = bl.begin();
+    ::decode(txn, p);
+  }
+
+  store->umount();
+  delete store;
+
+  cout << txn << std::endl;
+  return 0;
+}
+
+int run_sequence_to(int val, std::string& filestore_path,
+		     std::string& journal_path)
+{
+  num_txs = val;
+
+  if (!is_seed_set)
+    seed = (int) time(NULL);
+
+  FileStore *store = new FileStore(filestore_path, journal_path);
+
+  int err;
+
+  // mkfs iff directory dne
+  err = ::mkdir(filestore_path.c_str(), 0755);
+  if (err) {
+    cerr << filestore_path << " already exists" << std::endl;
+    store->umount();
+    delete store;
+    return err;
+  }
+  
+  err = store->mkfs();
+  ceph_assert(err == 0);
+
+  err = store->mount();
+  ceph_assert(err == 0);
+
+  DeterministicOpSequence op_sequence(store, status_file);
+  op_sequence.init(num_colls, num_objs);
+  op_sequence.generate(seed, num_txs);
+  store->umount();
+  return 0;
+}
+
+int run_command(std::string& command, std::vector& args)
+{
+  if (command.empty()) {
+    usage(our_name);
+    exit(0);
+  }
+
+  /* We'll have a class that will handle the options, the command
+   * and its arguments. For the time being, and so we can move on, let's
+   * tolerate this big, ugly code.
+   */
+  if (command == "diff") {
+    /* expect 4 arguments: (filestore path + journal path)*2 */
+    if (args.size() == 4) {
+      return run_diff(args[0], args[1], args[2], args[3]);
+    }
+  } else if (command == "get-last-op") {
+    /* expect 2 arguments: a filestore path + journal */
+    if (args.size() == 2) {
+      return run_get_last_op(args[0], args[1]);
+    }
+  } else if (command == "run-sequence-to") {
+    /* expect 3 arguments: # of operations and a filestore path + journal. */
+    if (args.size() == 3) {
+      return run_sequence_to(strtoll(args[0].c_str(), NULL, 10), args[1], args[2]);
+    }
+  } else {
+    std::cout << "unknown command " << command << std::endl;
+    usage(our_name);
+    exit(1);
+  }
+
+  usage(our_name, command);
+  exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+  vector def_args;
+  vector args;
+  our_name = argv[0];
+  argv_to_vec(argc, argv, args);
+
+  global_init(&def_args, args,
+	      CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  std::string command;
+  std::vector command_args;
+
+  for (std::vector::iterator i = args.begin(); i != args.end();) {
+    string val;
+
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-seed", (char*) NULL)) {
+      seed = strtoll(val.c_str(), NULL, 10);
+      is_seed_set = true;
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-num-colls", (char*) NULL)) {
+      num_colls = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-num-objs", (char*) NULL)) {
+      num_objs = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-status-file", (char*) NULL)) {
+      status_file = val;
+    } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
+      usage(our_name);
+      exit(0);
+    } else {
+      if (command.empty())
+        command = *i++;
+      else
+        command_args.push_back(string(*i++));
+    }
+  }
+
+  int ret = run_command(command, command_args);
+
+  return ret;
+}
diff --git a/ceph/src/test/objectstore/workload_generator.cc b/ceph/src/test/objectstore/workload_generator.cc
new file mode 100644
index 00000000..acf0fc14
--- /dev/null
+++ b/ceph/src/test/objectstore/workload_generator.cc
@@ -0,0 +1,575 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "os/ObjectStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include 
+#include 
+#include "workload_generator.h"
+#include "include/assert.h"
+
+#include "TestObjectStoreState.h"
+
+static const char *our_name = NULL;
+void usage();
+
+boost::scoped_ptr wrkldgen;
+
+#define dout_subsys ceph_subsys_
+
+
+WorkloadGenerator::WorkloadGenerator(vector args)
+  : TestObjectStoreState(NULL),
+    m_max_in_flight(def_max_in_flight),
+    m_num_ops(-1),
+    m_destroy_coll_every_nr_runs(def_destroy_coll_every_nr_runs),
+    m_num_colls(def_num_colls),
+    m_write_data_bytes(0), m_write_xattr_obj_bytes(0),
+    m_write_xattr_coll_bytes(0), m_write_pglog_bytes(0),
+    m_suppress_write_data(false), m_suppress_write_xattr_obj(false),
+    m_suppress_write_xattr_coll(false), m_suppress_write_log(false),
+    m_do_stats(false),
+    m_stats_finished_txs(0),
+    m_stats_lock("WorldloadGenerator::m_stats_lock"),
+    m_stats_show_secs(5),
+    m_stats_total_written(0),
+    m_stats_begin()
+{
+  int err = 0;
+
+  m_nr_runs.set(0);
+
+  init_args(args);
+  dout(0) << "data            = " << g_conf->osd_data << dendl;
+  dout(0) << "journal         = " << g_conf->osd_journal << dendl;
+  dout(0) << "journal size    = " << g_conf->osd_journal_size << dendl;
+
+  err = ::mkdir(g_conf->osd_data.c_str(), 0755);
+  ceph_assert(err == 0 || (err < 0 && errno == EEXIST));
+  ObjectStore *store_ptr = ObjectStore::create(g_ceph_context,
+                                               g_conf->osd_objectstore,
+                                               g_conf->osd_data,
+                                               g_conf->osd_journal);
+  m_store.reset(store_ptr);
+  err = m_store->mkfs();
+  ceph_assert(err == 0);
+  err = m_store->mount();
+  ceph_assert(err == 0);
+
+  set_max_in_flight(m_max_in_flight);
+  set_num_objs_per_coll(def_num_obj_per_coll);
+
+  init(m_num_colls, 0);
+
+  dout(0) << "#colls          = " << m_num_colls << dendl;
+  dout(0) << "#objs per coll  = " << m_num_objs_per_coll << dendl;
+  dout(0) << "#txs per destr  = " << m_destroy_coll_every_nr_runs << dendl;
+
+}
+
+size_t WorkloadGenerator::_parse_size_or_die(std::string& val)
+{
+  size_t s = 0;
+  int multiplier = 0;
+  size_t i = 0;
+
+  if (val.empty()) // this should never happen, but catch it anyway.
+    goto die;
+
+
+  for (i = 0; i < val.length(); i++) {
+    if (!isdigit(val[i])) {
+      if (isalpha(val[i])) {
+        val[i] = tolower(val[i]);
+        switch (val[i]) {
+        case 'b': break;
+        case 'k': multiplier = 10; break;
+        case 'm': multiplier = 20; break;
+        case 'g': multiplier = 30; break;
+        default:
+          goto die;
+        }
+        val[i] = '\0';
+        break;
+      } else {
+        goto die;
+      }
+    }
+  }
+
+  s = strtoll(val.c_str(), NULL, 10) * (1 << multiplier);
+  return s;
+
+die:
+  usage();
+  exit(1);
+}
+
+void WorkloadGenerator::_suppress_ops_or_die(std::string& val)
+{
+  for (size_t i = 0; i < val.length(); i++) {
+    switch (val[i]) {
+    case 'c': m_suppress_write_xattr_coll = true; break;
+    case 'o': m_suppress_write_xattr_obj = true; break;
+    case 'l': m_suppress_write_log = true; break;
+    case 'd': m_suppress_write_data = true; break;
+    default:
+      usage();
+      exit(1);
+    }
+  }
+}
+
+void WorkloadGenerator::init_args(vector args)
+{
+  for (std::vector::iterator i = args.begin(); i != args.end();) {
+    string val;
+
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-num-colls", (char*) NULL)) {
+      m_num_colls = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-objs-per-coll", (char*) NULL)) {
+      m_num_objs_per_coll = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-destroy-coll-per-N-trans", (char*) NULL)) {
+      m_destroy_coll_every_nr_runs = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-num-ops", (char*) NULL)) {
+      m_num_ops = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-max-in-flight", (char*) NULL)) {
+      m_max_in_flight = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-write-data-size", (char*) NULL)) {
+      m_write_data_bytes = _parse_size_or_die(val);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-write-xattr-obj-size", (char*) NULL)) {
+      m_write_xattr_obj_bytes = _parse_size_or_die(val);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-write-xattr-coll-size", (char*) NULL)) {
+      m_write_xattr_coll_bytes = _parse_size_or_die(val);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-write-pglog-size", (char*) NULL)) {
+      m_write_pglog_bytes = _parse_size_or_die(val);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-suppress-ops", (char*) NULL)) {
+      _suppress_ops_or_die(val);
+    } else if (ceph_argparse_witharg(args, i, &val,
+        "--test-show-stats-period", (char*) NULL)) {
+      m_stats_show_secs = strtoll(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_flag(args, i, "--test-show-stats", (char*) NULL)) {
+      m_do_stats = true;
+    } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
+      usage();
+      exit(0);
+    }
+  }
+}
+
+int WorkloadGenerator::get_uniform_random_value(int min, int max)
+{
+  boost::uniform_int<> value(min, max);
+  return value(m_rng);
+}
+
+TestObjectStoreState::coll_entry_t *WorkloadGenerator::get_rnd_coll_entry(bool erase = false)
+{
+  int index = get_uniform_random_value(0, m_collections_ids.size()-1);
+  coll_entry_t *entry = get_coll_at(index, erase);
+  return entry;
+}
+
+hobject_t *WorkloadGenerator::get_rnd_obj(coll_entry_t *entry)
+{
+  assert(entry != NULL);
+
+  bool create =
+      (get_uniform_random_value(0,100) < 50 || !entry->m_objects.size());
+
+  if (create && ((int) entry->m_objects.size() < m_num_objs_per_coll)) {
+    return (entry->touch_obj(entry->m_next_object_id++));
+  }
+
+  int idx = get_uniform_random_value(0, entry->m_objects.size()-1);
+  return entry->get_obj_at(idx);
+}
+
+/**
+ * We'll generate a random amount of bytes, ranging from a single byte up to
+ * a couple of MB.
+ */
+size_t WorkloadGenerator::get_random_byte_amount(size_t min, size_t max)
+{
+  size_t diff = max - min;
+  return (size_t) (min + (rand() % diff));
+}
+
+void WorkloadGenerator::get_filled_byte_array(bufferlist& bl, size_t size)
+{
+  static const char alphanum[] = "0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    "abcdefghijklmnopqrstuvwxyz";
+
+  bufferptr bp(size);
+  if (false) {
+    for (unsigned int i = 0; i < size - 1; i++) {
+      bp[i] = alphanum[rand() % sizeof(alphanum)];
+    }
+    bp[size - 1] = '\0';
+  } else {
+    bp.zero();
+  }
+  bl.append(bp);
+}
+
+void WorkloadGenerator::do_write_object(ObjectStore::Transaction *t,
+					coll_t coll, hobject_t obj,
+					C_StatState *stat)
+{
+  if (m_suppress_write_data) {
+    dout(5) << __func__ << " suppressed" << dendl;
+    return;
+  }
+
+  size_t size = m_write_data_bytes;
+  if (!size)
+    size = get_random_byte_amount(min_write_bytes, max_write_bytes);
+
+  bufferlist bl;
+  get_filled_byte_array(bl, size);
+
+  dout(2) << __func__ << " " << coll << "/" << obj
+	  << " size " << bl.length() << dendl;
+
+  if (m_do_stats && (stat != NULL))
+    stat->written_data += bl.length();
+
+  t->write(coll, obj, 0, bl.length(), bl);
+}
+
+void WorkloadGenerator::do_setattr_object(ObjectStore::Transaction *t,
+					  coll_t coll, hobject_t obj,
+					  C_StatState *stat)
+{
+  if (m_suppress_write_xattr_obj) {
+    dout(5) << __func__ << " suppressed" << dendl;
+    return;
+  }
+
+  size_t size = m_write_xattr_obj_bytes;
+  if (!size)
+    size = get_random_byte_amount(min_xattr_obj_bytes, max_xattr_obj_bytes);
+
+  bufferlist bl;
+  get_filled_byte_array(bl, size);
+
+  dout(2) << __func__ << " " << coll << "/" << obj << " size " << size << dendl;
+
+  if (m_do_stats && (stat != NULL))
+      stat->written_data += bl.length();
+
+  t->setattr(coll, obj, "objxattr", bl);
+}
+
+void WorkloadGenerator::do_setattr_collection(ObjectStore::Transaction *t,
+					      coll_t coll, C_StatState *stat)
+{
+  if (m_suppress_write_xattr_coll) {
+    dout(5) << __func__ << " suppressed" << dendl;
+    return;
+  }
+
+  size_t size = m_write_xattr_coll_bytes;
+  if (!size)
+    size = get_random_byte_amount(min_xattr_coll_bytes, max_xattr_coll_bytes);
+
+  bufferlist bl;
+  get_filled_byte_array(bl, size);
+  dout(2) << __func__ << " coll " << coll << " size " << size << dendl;
+
+  if (m_do_stats && (stat != NULL))
+      stat->written_data += bl.length();
+
+  t->collection_setattr(coll, "collxattr", bl);
+}
+
+void WorkloadGenerator::do_append_log(ObjectStore::Transaction *t,
+                                      coll_entry_t *entry, C_StatState *stat)
+{
+  if (m_suppress_write_log) {
+    dout(5) << __func__ << " suppressed" << dendl;
+    return;
+  }
+
+  size_t size = (m_write_pglog_bytes ? m_write_pglog_bytes : log_append_bytes);
+
+  bufferlist bl;
+  get_filled_byte_array(bl, size);
+  hobject_t log_obj = entry->m_meta_obj;
+
+  dout(2) << __func__ << " coll " << entry->m_coll << " "
+      << META_COLL << " /" << log_obj << " (" << bl.length() << ")" << dendl;
+
+  if (m_do_stats && (stat != NULL))
+      stat->written_data += bl.length();
+
+  uint64_t s = pg_log_size[entry->m_coll];
+  t->write(META_COLL, log_obj, s, bl.length(), bl);
+  pg_log_size[entry->m_coll] += bl.length();
+}
+
+void WorkloadGenerator::do_destroy_collection(ObjectStore::Transaction *t,
+					      coll_entry_t *entry,
+					      C_StatState *stat)
+{  
+  m_nr_runs.set(0);
+  entry->m_osr.flush();
+  vector ls;
+  m_store->collection_list(entry->m_coll, ls);
+  dout(2) << __func__ << " coll " << entry->m_coll
+      << " (" << ls.size() << " objects)" << dendl;
+
+  for (vector::iterator it = ls.begin(); it < ls.end(); ++it) {
+    t->remove(entry->m_coll, *it);
+  }
+
+  t->remove_collection(entry->m_coll);
+  t->remove(META_COLL, entry->m_meta_obj);
+}
+
+TestObjectStoreState::coll_entry_t
+*WorkloadGenerator::do_create_collection(ObjectStore::Transaction *t,
+                                         C_StatState *stat)
+{
+  coll_entry_t *entry = coll_create(m_next_coll_nr++);
+  if (!entry) {
+    dout(0) << __func__ << " failed to create coll id "
+        << m_next_coll_nr << dendl;
+    return NULL;
+  }
+  m_collections.insert(make_pair(entry->m_id, entry));
+
+  dout(2) << __func__ << " id " << entry->m_id << " coll " << entry->m_coll << dendl;
+  t->create_collection(entry->m_coll);
+  dout(2) << __func__ << " meta " << META_COLL << "/" << entry->m_meta_obj << dendl;
+  t->touch(META_COLL, entry->m_meta_obj);
+  return entry;
+}
+
+void WorkloadGenerator::do_stats()
+{
+  utime_t now = ceph_clock_now(NULL);
+  m_stats_lock.Lock();
+
+  utime_t duration = (now - m_stats_begin);
+
+  // when cast to double, a utime_t behaves properly
+  double throughput = (m_stats_total_written / ((double) duration));
+  double tx_throughput (m_stats_finished_txs / ((double) duration));
+
+  dout(0) << __func__
+	  << " written: " << m_stats_total_written
+	  << " duration: " << duration << " sec"
+	  << " bandwidth: " << prettybyte_t(throughput) << "/s"
+	  << " iops: " << tx_throughput << "/s"
+	  << dendl;
+
+  m_stats_lock.Unlock();
+}
+
+void WorkloadGenerator::run()
+{
+  bool create_coll = false;
+  int ops_run = 0;
+
+  utime_t stats_interval(m_stats_show_secs, 0);
+  utime_t now = ceph_clock_now(NULL);
+  utime_t stats_time = now;
+  m_stats_begin = now;
+
+  do {
+    C_StatState *stat_state = NULL;
+
+    if (m_num_ops && (ops_run == m_num_ops))
+      break;
+
+    if (!create_coll && !m_collections.size()) {
+      dout(0) << "We ran out of collections!" << dendl;
+      break;
+    }
+
+    dout(5) << __func__
+        << " m_finished_lock is-locked: " << m_finished_lock.is_locked()
+        << " in-flight: " << m_in_flight.read()
+        << dendl;
+
+    wait_for_ready();
+
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    Context *c;
+    bool destroy_collection = false;
+    TestObjectStoreState::coll_entry_t *entry = NULL;
+
+
+    if (m_do_stats) {
+      utime_t now = ceph_clock_now(NULL);
+      utime_t elapsed = now - stats_time;
+      if (elapsed >= stats_interval) {
+	do_stats();
+	stats_time = now;
+      }
+      stat_state = new C_StatState(this, now);
+    }
+
+    if (create_coll) {
+      create_coll = false;
+
+      entry = do_create_collection(t, stat_state);
+      if (!entry) {
+        dout(0) << __func__ << " something went terribly wrong creating coll" << dendl;
+        break;
+      }
+
+      c = new C_OnReadable(this, t);
+      goto queue_tx;
+    }
+
+    destroy_collection = should_destroy_collection();
+    entry = get_rnd_coll_entry(destroy_collection);
+    assert(entry != NULL);
+
+    if (destroy_collection) {
+      do_destroy_collection(t, entry, stat_state);
+      c = new C_OnDestroyed(this, t, entry);
+      if (!m_num_ops)
+        create_coll = true;
+    } else {
+      hobject_t *obj = get_rnd_obj(entry);
+
+      do_write_object(t, entry->m_coll, *obj, stat_state);
+      do_setattr_object(t, entry->m_coll, *obj, stat_state);
+      do_setattr_collection(t, entry->m_coll, stat_state);
+      do_append_log(t, entry, stat_state);
+
+      c = new C_OnReadable(this, t);
+    }
+
+queue_tx:
+
+    if (m_do_stats) {
+      Context *tmp = c;
+      c = new C_StatWrapper(stat_state, tmp);
+    }
+
+    m_store->queue_transaction(&(entry->m_osr), t, c);
+
+    inc_in_flight();
+
+    ops_run ++;
+
+  } while (true);
+
+  dout(2) << __func__ << " waiting for "
+	  << m_in_flight.read() << " in-flight transactions" << dendl;
+
+  wait_for_done();
+
+  do_stats();
+
+  dout(0) << __func__ << " finishing" << dendl;
+}
+
+void usage()
+{
+  cout << "usage: " << our_name << "[options]" << std::endl;
+
+  cout << "\
+\n\
+Global Options:\n\
+  -c FILE                             Read configuration from FILE\n\
+  --osd-objectstore TYPE              Set OSD ObjectStore type\n\
+  --osd-data PATH                     Set OSD Data path\n\
+  --osd-journal PATH                  Set OSD Journal path\n\
+  --osd-journal-size VAL              Set Journal size\n\
+  --help                              This message\n\
+\n\
+Test-specific Options:\n\
+  --test-num-colls VAL                Set the number of collections\n\
+  --test-num-objs-per-coll VAL        Set the number of objects per collection\n\
+  --test-destroy-coll-per-N-trans VAL Set how many transactions to run before\n\
+                                      destroying a collection.\n\
+  --test-num-ops VAL                  Run a certain number of operations\n\
+                                      (a VAL of 0 runs the test forever)\n\
+   --test-max-in-flight VAL           Maximum number of in-flight transactions\n\
+                                      (default: 50)\n\
+   --test-suppress-ops OPS            Suppress ops specified in OPS\n\
+   --test-write-data-size SIZE        Specify SIZE for all data writes\n\
+   --test-write-xattr-obj-size SIZE   Specify SIZE for all xattrs on objects\n\
+   --test-write-xattr-coll-size SIZE  Specify SIZE for all xattrs on colls\n\
+   --test-write-pglog-size SIZE       Specify SIZE for all pglog writes\n\
+   --test-show-stats                  Show stats as we go\n\
+   --test-show-stats-period SECS      Show stats every SECS (default: 5)\n\
+\n\
+   SIZE is a numeric value that can be assumed as being bytes, or may be any\n\
+   other unit if specified: B or b, K or k, M or m, G or g.\n\
+      e.g., 1G = 1024M = 1048576k = 1073741824\n\
+\n\
+   OPS can be one or more of the following options:\n\
+      c    writes on collection's xattrs\n\
+      o    writes on object's xattr\n\
+      l    writes on pglog\n\
+      d    data writes on objects\n\
+\n\
+" << std::endl;
+}
+
+int main(int argc, const char *argv[])
+{
+  vector def_args;
+  vector args;
+
+  our_name = argv[0];
+
+  def_args.push_back("--osd-journal-size");
+  def_args.push_back("400");
+//  def_args.push_back("--osd-data");
+//  def_args.push_back("workload_gen_dir");
+//  def_args.push_back("--osd-journal");
+//  def_args.push_back("workload_gen_dir/journal");
+  argv_to_vec(argc, argv, args);
+
+  global_init(&def_args, args,
+      CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  WorkloadGenerator *wrkldgen_ptr = new WorkloadGenerator(args);
+  wrkldgen.reset(wrkldgen_ptr);
+  wrkldgen->run();
+  return 0;
+}
diff --git a/ceph/src/test/objectstore/workload_generator.h b/ceph/src/test/objectstore/workload_generator.h
new file mode 100644
index 00000000..3235fe86
--- /dev/null
+++ b/ceph/src/test/objectstore/workload_generator.h
@@ -0,0 +1,184 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+#ifndef WORKLOAD_GENERATOR_H_
+#define WORKLOAD_GENERATOR_H_
+
+#include "os/ObjectStore.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "TestObjectStoreState.h"
+
+typedef boost::mt11213b rngen_t;
+
+class WorkloadGenerator : public TestObjectStoreState {
+ public:
+  static const int def_max_in_flight = 50;
+
+  static const int def_destroy_coll_every_nr_runs = 100;
+  static const int def_num_obj_per_coll = 6000;
+  static const int def_num_colls = 30;
+
+  static const size_t min_write_bytes = 1;
+  static const size_t max_write_mb = 5;
+  static const size_t max_write_bytes = (max_write_mb * 1024 * 1024);
+
+  static const size_t min_xattr_obj_bytes = 2;
+  static const size_t max_xattr_obj_bytes = 300;
+  static const size_t min_xattr_coll_bytes = 4;
+  static const size_t max_xattr_coll_bytes = 600;
+
+  static const size_t log_append_bytes = 1024;
+
+  struct C_StatState {
+    utime_t start;
+    unsigned int written_data;
+    WorkloadGenerator *wrkldgen;
+
+    C_StatState(WorkloadGenerator *state, utime_t s)
+      : start(s), written_data(0), wrkldgen(state) { }
+  };
+
+
+ protected:
+  int m_max_in_flight;
+  int m_num_ops;
+  int m_destroy_coll_every_nr_runs;
+  atomic_t m_nr_runs;
+
+  int m_num_colls;
+
+  rngen_t m_rng;
+
+  map pg_log_size;
+
+  size_t m_write_data_bytes;
+  size_t m_write_xattr_obj_bytes;
+  size_t m_write_xattr_coll_bytes;
+  size_t m_write_pglog_bytes;
+
+  bool m_suppress_write_data;
+  bool m_suppress_write_xattr_obj;
+  bool m_suppress_write_xattr_coll;
+  bool m_suppress_write_log;
+
+  bool m_do_stats;
+
+  int m_stats_finished_txs;
+  Mutex m_stats_lock;
+  int m_stats_show_secs;
+
+  size_t m_stats_total_written;
+  utime_t m_stats_begin;
+
+ private:
+
+  void _suppress_ops_or_die(std::string& val);
+  size_t _parse_size_or_die(std::string& val);
+  void init_args(vector args);
+
+  int get_uniform_random_value(int min, int max);
+  coll_entry_t *get_rnd_coll_entry(bool erase);
+  hobject_t *get_rnd_obj(coll_entry_t *entry);
+  int get_random_collection_nr();
+  int get_random_object_nr(int coll_nr);
+
+  size_t get_random_byte_amount(size_t min, size_t max);
+  void get_filled_byte_array(bufferlist& bl, size_t size);
+
+  void do_write_object(ObjectStore::Transaction *t,
+      coll_t coll, hobject_t obj, C_StatState *stat);
+  void do_setattr_object(ObjectStore::Transaction *t,
+      coll_t coll, hobject_t obj, C_StatState *stat);
+  void do_setattr_collection(ObjectStore::Transaction *t, coll_t coll,
+      C_StatState *stat);
+  void do_append_log(ObjectStore::Transaction *t, coll_entry_t *entry,
+      C_StatState *stat);
+
+  bool should_destroy_collection() {
+    return ((m_destroy_coll_every_nr_runs > 0) &&
+        ((int)m_nr_runs.read() >= m_destroy_coll_every_nr_runs));
+  }
+  void do_destroy_collection(ObjectStore::Transaction *t, coll_entry_t *entry,
+      C_StatState *stat);
+  coll_entry_t *do_create_collection(ObjectStore::Transaction *t,
+      C_StatState *stat);
+
+  void do_stats();
+
+public:
+  WorkloadGenerator(vector args);
+  ~WorkloadGenerator() {
+    m_store->umount();
+  }
+
+  class C_OnReadable: public TestObjectStoreState::C_OnFinished {
+    WorkloadGenerator *wrkldgen_state;
+
+  public:
+    C_OnReadable(WorkloadGenerator *state,
+                                  ObjectStore::Transaction *t)
+     :TestObjectStoreState::C_OnFinished(state, t), wrkldgen_state(state) { }
+
+    void finish(int r)
+    {
+      TestObjectStoreState::C_OnFinished::finish(r);
+      wrkldgen_state->m_nr_runs.inc();
+    }
+  };
+
+  class C_OnDestroyed: public C_OnReadable {
+    coll_entry_t *m_entry;
+
+  public:
+    C_OnDestroyed(WorkloadGenerator *state,
+        ObjectStore::Transaction *t, coll_entry_t *entry) :
+          C_OnReadable(state, t), m_entry(entry) {}
+
+    void finish(int r) {
+      C_OnReadable::finish(r);
+      delete m_entry;
+    }
+  };
+
+  class C_StatWrapper : public Context {
+    C_StatState *stat_state;
+    Context *ctx;
+
+   public:
+    C_StatWrapper(C_StatState *state, Context *context)
+      : stat_state(state), ctx(context) { }
+
+    void finish(int r) {
+      ctx->complete(r);
+
+      stat_state->wrkldgen->m_stats_lock.Lock();
+
+      stat_state->wrkldgen->m_stats_total_written += stat_state->written_data;
+      stat_state->wrkldgen->m_stats_finished_txs ++;
+      stat_state->wrkldgen->m_stats_lock.Unlock();
+    }
+  };
+
+  void run(void);
+};
+
+bool operator<(const WorkloadGenerator::coll_entry_t& l,
+    const WorkloadGenerator::coll_entry_t& r) {
+      return (l.m_id < r.m_id);
+}
+
+#endif /* WORKLOAD_GENERATOR_H_ */
diff --git a/ceph/src/test/omap_bench.cc b/ceph/src/test/omap_bench.cc
new file mode 100644
index 00000000..4cb15a1b
--- /dev/null
+++ b/ceph/src/test/omap_bench.cc
@@ -0,0 +1,439 @@
+/*
+ * Generate latency statistics for a configurable number of write
+ * operations of configurable size.
+ *
+ *  Created on: May 21, 2012
+ *      Author: Eleanor Cawthon
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include "include/rados/librados.hpp"
+#include "include/Context.h"
+#include "common/ceph_context.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "include/utime.h"
+#include "global/global_context.h"
+#include "common/ceph_argparse.h"
+#include "test/omap_bench.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace std;
+using ceph::bufferlist;
+
+int OmapBench::setup(int argc, const char** argv) {
+  //parse key_value_store_bench args
+  vector args;
+  argv_to_vec(argc,argv,args);
+  for (unsigned i = 0; i < args.size(); i++) {
+    if(i < args.size() - 1) {
+      if (strcmp(args[i], "-t") == 0) {
+	threads = atoi(args[i+1]);
+      } else if (strcmp(args[i], "-o") == 0) {
+	objects = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--entries") == 0) {
+	entries_per_omap = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--keysize") == 0) {
+	key_size = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--valsize") == 0) {
+	value_size = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--inc") == 0) {
+	increment = atoi(args[i+1]);
+      } else if (strcmp(args[i], "--omaptype") == 0) {
+	if(strcmp("rand",args[i+1]) == 0) {
+	  omap_generator = OmapBench::generate_non_uniform_omap;
+	}
+	else if (strcmp("uniform", args[i+1]) == 0) {
+	  omap_generator = OmapBench::generate_uniform_omap;
+	}
+      } else if (strcmp(args[i], "--name") == 0) {
+	rados_id = args[i+1];
+      }
+    } else if (strcmp(args[i], "--help") == 0) {
+      cout << "\nUsage: ostorebench [options]\n"
+	   << "Generate latency statistics for a configurable number of "
+	   << "key value pair operations of\n"
+	   << "configurable size.\n\n"
+	   << "OPTIONS\n"
+	   << "	-t              number of threads to use (default "<data_lock.Lock();
+  name << omap_bench->prefix << ++(ob->data.started_ops);
+  ob->data_lock.Unlock();
+  oid = name.str();
+}
+void Writer::start_time() {
+  begin_time = ceph_clock_now(g_ceph_context);
+}
+void Writer::stop_time() {
+  end_time = ceph_clock_now(g_ceph_context);
+}
+double Writer::get_time() {
+  return (end_time - begin_time) * 1000;
+}
+string Writer::get_oid() {
+  return oid;
+}
+std::map & Writer::get_omap() {
+  return omap;
+}
+
+//AioWriter functions
+AioWriter::AioWriter(OmapBench *ob) : Writer(ob) {
+  aioc = NULL;
+}
+AioWriter::~AioWriter() {
+  if(aioc) aioc->release();
+}
+librados::AioCompletion * AioWriter::get_aioc() {
+  return aioc;
+}
+void AioWriter::set_aioc(librados::callback_t complete,
+    librados::callback_t safe) {
+  aioc = ob->rados.aio_create_completion(this, complete, safe);
+}
+
+
+//Helper methods
+void OmapBench::aio_is_safe(rados_completion_t c, void *arg) {
+  AioWriter *aiow = reinterpret_cast(arg);
+  aiow->stop_time();
+  Mutex * data_lock = &aiow->ob->data_lock;
+  Mutex * thread_is_free_lock = &aiow->ob->thread_is_free_lock;
+  Cond * thread_is_free = &aiow->ob->thread_is_free;
+  int &busythreads_count = aiow->ob->busythreads_count;
+  o_bench_data &data = aiow->ob->data;
+  int INCREMENT = aiow->ob->increment;
+  int err = aiow->get_aioc()->get_return_value();
+  if (err < 0) {
+    cout << "error writing AioCompletion";
+    return;
+  }
+  double time = aiow->get_time();
+  delete aiow;
+  data_lock->Lock();
+  data.avg_latency = (data.avg_latency * data.completed_ops + time)
+      / (data.completed_ops + 1);
+  data.completed_ops++;
+  if (time < data.min_latency) {
+    data.min_latency = time;
+  }
+  if (time > data.max_latency) {
+    data.max_latency = time;
+  }
+  data.total_latency += time;
+  ++(data.freq_map[time / INCREMENT]);
+  if(data.freq_map[time/INCREMENT] > data.mode.second) {
+    data.mode.first = time/INCREMENT;
+    data.mode.second = data.freq_map[time/INCREMENT];
+  }
+  data_lock->Unlock();
+
+  thread_is_free_lock->Lock();
+  busythreads_count--;
+  thread_is_free->Signal();
+  thread_is_free_lock->Unlock();
+}
+
+string OmapBench::random_string(int len) {
+  string ret;
+  string alphanum = "0123456789"
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+      "abcdefghijklmnopqrstuvwxyz";
+
+  for (int i = 0; i < len; ++i) {
+    ret.push_back(alphanum[rand() % (alphanum.size() - 1)]);
+  }
+
+  return ret;
+}
+
+int OmapBench::run() {
+  return (((OmapBench *)this)->*OmapBench::test)(omap_generator);
+}
+
+int OmapBench::print_written_omap() {
+  for (int i = 1; i <= objects; i++) {
+    int err = 0;
+    librados::ObjectReadOperation key_read;
+    set out_keys;
+    map out_vals;
+    std::stringstream objstrm;
+    objstrm << prefix;
+    objstrm << i;
+    cout << "\nPrinting omap for "<::iterator iter = out_keys.begin();
+	iter != out_keys.end(); ++iter) {
+      cout << *iter << "\t" << (out_vals)[*iter] << std::endl;
+    }
+  }
+  return 0;
+}
+
+void OmapBench::print_results() {
+  cout << "========================================================";
+  cout << "\nNumber of kvmaps written:\t" << objects;
+  cout << "\nNumber of ops at once:\t" << threads;
+  cout << "\nEntries per kvmap:\t\t" << entries_per_omap;
+  cout << "\nCharacters per key:\t" << key_size;
+  cout << "\nCharacters per val:\t" << value_size;
+  cout << std::endl;
+  cout << std::endl;
+  cout << "Average latency:\t" << data.avg_latency;
+  cout << "ms\nMinimum latency:\t" << data.min_latency;
+  cout << "ms\nMaximum latency:\t" << data.max_latency;
+  cout << "ms\nMode latency:\t\t"<<"between "<= "<< i * increment;
+    cout << "ms";
+    int spaces;
+    if (i == 0) spaces = 4;
+    else spaces = 3 - floor(log10(i));
+    for (int j = 0; j < spaces; j++) {
+      cout << " ";
+    }
+    cout << "[";
+    for(int j = 0; j < ((data.freq_map)[i])*45/(data.mode.second); j++) {
+      cout << "*";
+    }
+    cout << std::endl;
+  }
+  cout << "\n========================================================"
+       << std::endl;
+}
+
+int OmapBench::write_omap_asynchronously(AioWriter *aiow,
+    const std::map &omap) {
+  librados::ObjectWriteOperation owo;
+  owo.create(false);
+  owo.omap_clear();
+  owo.omap_set(omap);
+  aiow->start_time();
+  int err = io_ctx.aio_operate(aiow->get_oid(), aiow->get_aioc(), &owo);
+  if (err < 0) {
+    cout << "writing omap failed with code "< * out_omap) {
+  bufferlist bl;
+
+  //setup omap
+  for (int i = 0; i < omap_entries; i++) {
+    bufferlist omap_val;
+    omap_val.append(random_string(value_size));
+    string key = random_string(key_size);
+    (*out_omap)[key]= omap_val;
+  }
+  return 0;
+}
+
+int OmapBench::generate_non_uniform_omap(const int omap_entries,
+    const int key_size, const int value_size,
+    std::map * out_omap) {
+  bufferlist bl;
+
+  int num_entries = rand() % omap_entries + 1;
+  int key_len = rand() % key_size +1;
+  int val_len = rand() % value_size +1;
+
+  //setup omap
+  for (int i = 0; i < num_entries; i++) {
+    bufferlist omap_val;
+    omap_val.append(random_string(val_len));
+    string key = random_string(key_len);
+    (*out_omap)[key] = omap_val;
+  }
+  return 0;
+}
+
+int OmapBench::generate_small_non_random_omap(const int omap_entries,
+    const int key_size, const int value_size,
+    std::map * out_omap) {
+  bufferlist bl;
+  stringstream key;
+
+  //setup omap
+  for (int i = 0; i < omap_entries; i++) {
+    bufferlist omap_val;
+    omap_val.append("Value ");
+    omap_val.append(i);
+    key << "Key " << i;
+    (*out_omap)[key.str()]= omap_val;
+  }
+  return 0;
+}
+
+//tests
+int OmapBench::test_write_objects_in_parallel(omap_generator_t omap_gen) {
+  comp = NULL;
+  AioWriter *this_aio_writer;
+
+  Mutex::Locker l(thread_is_free_lock);
+  for (int i = 0; i < objects; i++) {
+    assert(busythreads_count <= threads);
+    //wait for a writer to be free
+    if (busythreads_count == threads) {
+      int err = thread_is_free.Wait(thread_is_free_lock);
+      assert(busythreads_count < threads);
+      if (err < 0) {
+	return err;
+      }
+    }
+
+    //set up the write
+    this_aio_writer = new AioWriter(this);
+    this_aio_writer->set_aioc(NULL,safe);
+
+    //perform the write
+    busythreads_count++;
+    int err = omap_gen(entries_per_omap, key_size, value_size,
+	& this_aio_writer->get_omap());
+    if (err < 0) {
+      return err;
+    }
+    err = OmapBench::write_omap_asynchronously(this_aio_writer,
+	(this_aio_writer->get_omap()));
+
+
+    if (err < 0) {
+      return err;
+    }
+  }
+  while(busythreads_count > 0) {
+    thread_is_free.Wait(thread_is_free_lock);
+  }
+
+  return 0;
+}
+
+/**
+ * runs the specified test with the specified parameters and generates
+ * a histogram of latencies
+ */
+int main(int argc, const char** argv) {
+  OmapBench ob;
+  int err = ob.setup(argc, argv);
+  if (err<0) {
+    cout << "error during setup: "<
+#include 
+#include 
+
+using ceph::bufferlist;
+
+struct o_bench_data {
+  double avg_latency;
+  double min_latency;
+  double max_latency;
+  double total_latency;
+  int started_ops;
+  int completed_ops;
+  std::map freq_map;
+  pair mode;
+  o_bench_data()
+  : avg_latency(0.0), min_latency(DBL_MAX), max_latency(0.0),
+    total_latency(0.0),
+    started_ops(0), completed_ops(0)
+  {}
+};
+
+class OmapBench;
+
+typedef int (*omap_generator_t)(const int omap_entries, const int key_size,
+				const int value_size,
+				std::map *out_omap);
+typedef int (OmapBench::*test_t)(omap_generator_t omap_gen);
+
+
+class Writer{
+protected:
+  string oid;
+  utime_t begin_time;
+  utime_t end_time;
+  std::map omap;
+  OmapBench *ob;
+  friend class OmapBench;
+public:
+  Writer(OmapBench *omap_bench);
+  virtual ~Writer(){};
+  virtual void start_time();
+  virtual void stop_time();
+  virtual double get_time();
+  virtual string get_oid();
+  virtual std::map & get_omap();
+};
+
+class AioWriter : public Writer{
+protected:
+  librados::AioCompletion * aioc;
+  friend class OmapBench;
+
+public:
+  AioWriter(OmapBench *omap_bench);
+  ~AioWriter();
+  virtual librados::AioCompletion * get_aioc();
+  virtual void set_aioc(librados::callback_t complete,
+      librados::callback_t safe);
+};
+
+class OmapBench{
+protected:
+  librados::IoCtx io_ctx;
+  librados::Rados rados;
+  struct o_bench_data data;
+  test_t test;
+  omap_generator_t omap_generator;
+
+  //aio things
+  Cond thread_is_free;
+  Mutex thread_is_free_lock;
+  Mutex  data_lock;
+  int busythreads_count;
+  librados::callback_t comp;
+  librados::callback_t safe;
+
+  string pool_name;
+  string rados_id;
+  string prefix;
+  int threads;
+  int objects;
+  int entries_per_omap;
+  int key_size;
+  int value_size;
+  double increment;
+
+  friend class Writer;
+  friend class AioWriter;
+
+public:
+  OmapBench()
+    : test(&OmapBench::test_write_objects_in_parallel),
+      omap_generator(generate_uniform_omap),
+      thread_is_free_lock("OmapBench::thread_is_free_lock"),
+      data_lock("OmapBench::data_lock"),
+      busythreads_count(0),
+      comp(NULL), safe(aio_is_safe),
+      pool_name("data"),
+      rados_id("admin"),
+      prefix(rados_id+".obj."),
+      threads(3), objects(100), entries_per_omap(10), key_size(10),
+      value_size(100), increment(10)
+  {}
+  /**
+   * Parses command line args, initializes rados and ioctx
+   */
+  int setup(int argc, const char** argv);
+
+  /**
+   * Callback for when an AioCompletion (called from an AioWriter)
+   * is safe. deletes the AioWriter that called it,
+   * Updates data, updates busythreads, and signals thread_is_free.
+   *
+   * @param c provided by aio_write - not used
+   * @param arg the AioWriter that contains this AioCompletion
+   */
+  static void aio_is_safe(rados_completion_t c, void *arg);
+
+  /**
+   * Generates a random string len characters long
+   */
+  static string random_string(int len);
+
+  /*
+   * runs the test specified by test using the omap generator specified by
+   * omap_generator
+   *
+   * @return error code
+   */
+  int run();
+
+  /*
+   * Prints all keys and values for all omap entries for all objects
+   */
+  int print_written_omap();
+
+  /*
+   * Displays relevant constants and the histogram generated through a test
+   */
+  void print_results();
+
+  /**
+   * Writes an object with the specified AioWriter.
+   *
+   * @param aiow the AioWriter to write with
+   * @param omap the omap to write
+   * @post: an asynchronous omap_set is launched
+   */
+  int write_omap_asynchronously(AioWriter *aiow,
+      const std::map &map);
+
+
+  /**
+   * Generates an omap with omap_entries entries, each with keys key_size
+   * characters long and with string values value_size characters long.
+   *
+   * @param out_map pointer to the map to be created
+   * @return error code
+   */
+  static int generate_uniform_omap(const int omap_entries, const int key_size,
+      const int value_size, std::map * out_omap);
+
+  /**
+   * The same as generate_uniform_omap except that string lengths are picked
+   * randomly between 1 and the int arguments
+   */
+  static int generate_non_uniform_omap(const int omap_entries,
+      const int key_size,
+      const int value_size, std::map * out_omap);
+
+  static int generate_small_non_random_omap(const int omap_entries,
+      const int key_size, const int value_size,
+      std::map * out_omap);
+
+  /*
+   * Uses aio_write to write omaps generated by omap_gen to OBJECTS objects
+   * using THREADS AioWriters at a time.
+   *
+   * @param omap_gen the method used to generate the omaps.
+   */
+  int test_write_objects_in_parallel(omap_generator_t omap_gen);
+
+};
+
+
+
+#endif /* OMAP_BENCH_HPP_ */
+
diff --git a/ceph/src/test/on_exit.cc b/ceph/src/test/on_exit.cc
new file mode 100644
index 00000000..26f357ef
--- /dev/null
+++ b/ceph/src/test/on_exit.cc
@@ -0,0 +1,113 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "include/on_exit.h"
+
+#ifndef MAP_ANONYMOUS
+# ifdef MAP_ANON
+#  define MAP_ANONYMOUS MAP_ANON
+# else
+#  error "Don't know how to create anonymous mmap"
+# endif
+#endif
+
+static int func_scope_val;
+
+static void add(void *incp)
+{
+  func_scope_val += *((int*)incp);
+}
+
+static void func_scope(void)
+{
+  OnExitManager mgr;
+
+  int *inc_1 = (int*)malloc(sizeof(*inc_1));
+  *inc_1 = 5;
+  mgr.add_callback(add, inc_1);
+
+  int *inc_2 = (int*)malloc(sizeof(*inc_2));
+  *inc_2 = 3;
+  mgr.add_callback(add, inc_2);
+}
+
+// shared between processes
+static int *shared_val;
+
+#define MAIN_SCOPE_VAL 0x1111111
+static OnExitManager main_scope_mgr;
+static void main_scope_cb(void *val)
+{
+  *shared_val = *((int*)val);
+}
+
+#define EXIT_FUNC_VAL 0x22222222
+static OnExitManager exit_func_mgr;
+static void exit_func_cb(void *val)
+{
+  *shared_val = *((int*)val);
+}
+
+static void call_exit()
+{
+  exit(3);
+}
+
+int main(int argc, char **argv)
+{
+  // test basic function scope behavior
+  assert(func_scope_val == 0);
+  func_scope();
+  assert(func_scope_val == 8);
+
+  // shared mem for exit tests
+  shared_val = (int*)mmap(NULL, sizeof(int),
+      PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, 0, 0);
+  assert(shared_val != MAP_FAILED);
+
+  // test normal exit returning from main
+  *shared_val = 0;
+  int pid = fork();
+  assert(pid >= 0);
+  if (pid) {
+    int status;
+    int ret = waitpid(pid, &status, 0);
+    assert(ret == pid); // should be our child
+    assert(status == 0);
+    assert(*shared_val == MAIN_SCOPE_VAL);
+  } else {
+    // child adds a callback to the static scope callback manager and then
+    // exits by returning from main. The parent checks the value after the
+    // child exits via the memory map.
+    assert(*shared_val == 0);
+    int *new_val = (int*)malloc(sizeof(*new_val));
+    *new_val = MAIN_SCOPE_VAL;
+    main_scope_mgr.add_callback(main_scope_cb, new_val);
+    return 0;
+  }
+
+  // test exit via exit()
+  *shared_val = 0;
+  pid = fork();
+  assert(pid >= 0);
+  if (pid) {
+    int status;
+    int ret = waitpid(pid, &status, 0);
+    assert(ret == pid); // should be our child
+    assert(WEXITSTATUS(status) == 3);
+    assert(*shared_val == EXIT_FUNC_VAL);
+  } else {
+    // child adds a callback to the static scope callback manager and then
+    // exits via exit().
+    assert(*shared_val == 0);
+    int *new_val = (int*)malloc(sizeof(*new_val));
+    *new_val = EXIT_FUNC_VAL;
+    exit_func_mgr.add_callback(exit_func_cb, new_val);
+    call_exit();
+    assert(0);
+  }
+
+  return 0;
+}
diff --git a/ceph/src/test/os/TestFlatIndex.cc b/ceph/src/test/os/TestFlatIndex.cc
new file mode 100644
index 00000000..7bb2d42f
--- /dev/null
+++ b/ceph/src/test/os/TestFlatIndex.cc
@@ -0,0 +1,143 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+#include "os/FlatIndex.h"
+#include "os/CollectionIndex.h"
+#include "os/chain_xattr.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include 
+
+TEST(FlatIndex, FlatIndex) {
+  coll_t collection("ABC");
+  const std::string base_path("PATH");
+  FlatIndex index(collection, base_path);
+  EXPECT_EQ(collection, index.coll());
+  EXPECT_EQ((unsigned)0, index.collection_version());
+  //
+  // checking placeholders
+  //
+  EXPECT_EQ(0, index.init());
+  EXPECT_EQ(0, index.cleanup());
+}
+
+#ifdef GTEST_HAS_DEATH_TEST
+TEST(FlatIndex, collection) {
+  coll_t collection("ABC");
+  const std::string base_path("PATH");
+  FlatIndex index(collection, base_path);
+  const std::string key("KEY");
+  uint64_t hash = 111;
+  uint64_t pool = 222;
+  const std::string object_name(10, 'A');
+  ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, ""));
+  vector ls;
+  ASSERT_DEATH(index.collection_list_partial(hoid, 0, 0, 0, &ls, &hoid), "0");
+}
+#endif //GTEST_HAS_DEATH_TEST
+
+TEST(FlatIndex, created_unlink) {
+  coll_t collection("ABC");
+  const std::string base_path("PATH");
+  EXPECT_EQ(0, ::system("rm -fr PATH"));
+  EXPECT_EQ(0, ::mkdir("PATH", 0700));
+  ceph::shared_ptr index(new FlatIndex(collection, base_path));
+  const std::string key("KEY");
+  uint64_t hash = 111;
+  uint64_t pool = 222;
+  //
+  // short object name
+  //
+  {
+    CollectionIndex::IndexedPath indexed_path;
+    index->set_ref(index);
+    const std::string object_name(10, 'A');
+    ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, ""));
+    int exists;
+    EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_EQ(0, ::close(::creat(indexed_path->path(), 0600)));
+    EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
+    EXPECT_EQ(1, exists);
+    EXPECT_EQ(0, index->unlink(hoid));
+    EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
+    EXPECT_EQ(0, exists);
+  }
+  //
+  // long object name
+  //
+  {
+    CollectionIndex::IndexedPath indexed_path;
+    index->set_ref(index);
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, ""));
+    int exists;
+    EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_EQ(0, ::close(::creat(indexed_path->path(), 0600)));
+    EXPECT_EQ(0, index->created(hoid, indexed_path->path()));
+    EXPECT_EQ(0, index->unlink(hoid));
+    EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
+    EXPECT_EQ(0, exists);
+  }
+  EXPECT_EQ(0, ::system("rm -fr PATH"));
+}
+
+TEST(FlatIndex, collection_list) {
+  coll_t collection("ABC");
+  const std::string base_path("PATH");
+  EXPECT_EQ(0, ::system("rm -fr PATH"));
+  EXPECT_EQ(0, ::mkdir("PATH", 0700));
+  const std::string object_name("ABC");
+  const std::string filename("PATH/" + object_name + "_head");
+  EXPECT_EQ(0, ::close(::creat(filename.c_str(), 0600)));
+  ceph::shared_ptr index(new FlatIndex(collection, base_path));
+  vector ls;
+  index->collection_list(&ls);
+  EXPECT_EQ((unsigned)1, ls.size());
+  EXPECT_EQ(object_name, ls[0].hobj.oid.name);
+  EXPECT_EQ(0, ::system("rm -fr PATH"));
+}
+
+int main(int argc, char **argv) {
+  int fd = ::creat("detect", 0600);
+  int ret = chain_fsetxattr(fd, "user.test", "A", 1);
+  ::close(fd);
+  ::unlink("detect");
+  if (ret < 0) {
+    cerr << "SKIP FlatIndex because unable to test for xattr" << std::endl;
+  } else {
+    vector args;
+    argv_to_vec(argc, (const char **)argv, args);
+
+    global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+    common_init_finish(g_ceph_context);
+
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+  }
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make unittest_flatindex ; ./unittest_flatindex # --gtest_filter=FlatIndexTest.FlatIndex --log-to-stderr=true --debug-filestore=20"
+// End:
diff --git a/ceph/src/test/os/TestLFNIndex.cc b/ceph/src/test/os/TestLFNIndex.cc
new file mode 100644
index 00000000..74fdf66d
--- /dev/null
+++ b/ceph/src/test/os/TestLFNIndex.cc
@@ -0,0 +1,467 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+#include "os/LFNIndex.h"
+#include "os/chain_xattr.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include 
+
+class TestWrapLFNIndex : public LFNIndex {
+public:
+  TestWrapLFNIndex(coll_t collection,
+		   const char *base_path,
+		   uint32_t index_version) : LFNIndex(collection, base_path, index_version) {}
+
+  virtual uint32_t collection_version() {
+    return index_version;
+  }
+
+  int cleanup() { return 0; }
+
+  virtual int _split(
+		     uint32_t match,                           
+		     uint32_t bits,                            
+		     ceph::shared_ptr dest
+		     ) { return 0; }
+
+  void test_generate_and_parse(const ghobject_t &hoid, const std::string &mangled_expected) {
+    const std::string mangled_name = lfn_generate_object_name(hoid);
+    EXPECT_EQ(mangled_expected, mangled_name);
+    ghobject_t hoid_parsed;
+    EXPECT_TRUE(lfn_parse_object_name(mangled_name, &hoid_parsed));
+    EXPECT_EQ(hoid, hoid_parsed);
+  }
+
+protected:
+  virtual int _init() { return 0; }
+
+  virtual int _created(
+		       const vector &path,
+		       const ghobject_t &hoid,
+		       const string &mangled_name 
+		       ) { return 0; }
+
+  virtual int _remove(
+		      const vector &path,
+		      const ghobject_t &hoid,
+		      const string &mangled_name
+		      ) { return 0; }
+
+  virtual int _lookup(
+		      const ghobject_t &hoid,
+		      vector *path,
+		      string *mangled_name,
+		      int *exists		 
+		      ) { return 0; }
+
+  virtual int _collection_list(
+			       vector *ls
+			       ) { return 0; }
+
+  virtual int _collection_list_partial(
+				       const ghobject_t &start,
+				       int min_count,
+				       int max_count,
+				       snapid_t seq,
+				       vector *ls,
+				       ghobject_t *next
+				       ) { return 0; }
+};
+
+class TestHASH_INDEX_TAG : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestHASH_INDEX_TAG() : TestWrapLFNIndex(coll_t("ABC"), "PATH", CollectionIndex::HASH_INDEX_TAG) {
+  }
+};
+
+TEST_F(TestHASH_INDEX_TAG, generate_and_parse_name) {
+  const vector path;
+  const std::string key;
+  uint64_t hash = 0xABABABAB;
+  uint64_t pool = -1;
+
+  test_generate_and_parse(ghobject_t(hobject_t(object_t(".A/B_\\C.D"), key, CEPH_NOSNAP, hash, pool, "")),
+			  "\\.A\\sB_\\\\C.D_head_ABABABAB");
+  test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
+			  "\\dA_head_ABABABAB");
+}
+
+class TestHASH_INDEX_TAG_2 : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestHASH_INDEX_TAG_2() : TestWrapLFNIndex(coll_t("ABC"), "PATH", CollectionIndex::HASH_INDEX_TAG_2) {
+  }
+};
+
+TEST_F(TestHASH_INDEX_TAG_2, generate_and_parse_name) {
+  const vector path;
+  std::string mangled_name;
+  const std::string key("KEY");
+  uint64_t hash = 0xABABABAB;
+  uint64_t pool = -1;
+
+  {
+    std::string name(".XA/B_\\C.D");
+    name[1] = '\0';
+    ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
+
+    test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB");
+  }
+  test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
+			  "\\dA_KEY_head_ABABABAB");
+}
+
+class TestHOBJECT_WITH_POOL : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestHOBJECT_WITH_POOL() : TestWrapLFNIndex(coll_t("ABC"), "PATH", CollectionIndex::HOBJECT_WITH_POOL) {
+  }
+};
+
+TEST_F(TestHOBJECT_WITH_POOL, generate_and_parse_name) {
+  const vector path;
+  std::string mangled_name;
+  const std::string key("KEY");
+  uint64_t hash = 0xABABABAB;
+  uint64_t pool = 0xCDCDCDCD;
+  int64_t gen = 0xefefefefef;
+  int8_t shard_id = 0xb;
+
+  {
+    std::string name(".XA/B_\\C.D");
+    name[1] = '\0';
+    ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
+  }
+  {
+    ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""));
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
+  }
+  {
+    std::string name(".XA/B_\\C.D");
+    name[1] = '\0';
+    ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
+  }
+  {
+    ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
+  }
+}
+
+class TestLFNIndex : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestLFNIndex() : TestWrapLFNIndex(coll_t("ABC"), "PATH", CollectionIndex::HOBJECT_WITH_POOL) {
+  }
+
+  virtual void SetUp() {
+    ::chmod("PATH", 0700);
+    ASSERT_EQ(0, ::system("rm -fr PATH"));
+    ASSERT_EQ(0, ::mkdir("PATH", 0700));
+  }
+
+  virtual void TearDown() {
+    ASSERT_EQ(0, ::system("rm -fr PATH"));
+  }
+};
+
+TEST_F(TestLFNIndex, remove_object) {
+  const vector path;
+
+  //
+  // small object name removal
+  //
+  {
+    std::string mangled_name;
+    int exists = 666;
+    ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
+
+    EXPECT_EQ(0, ::chmod("PATH", 0000));
+    EXPECT_EQ(-EACCES, remove_object(path, hoid));
+    EXPECT_EQ(0, ::chmod("PATH", 0700));
+    EXPECT_EQ(-ENOENT, remove_object(path, hoid));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    const std::string pathname("PATH/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+  }
+  //
+  // long object name removal of a single file
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    std::string pathname("PATH/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+  }
+
+  //
+  // long object name removal of the last file 
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    //
+    //   PATH/AAA..._0_long => does not match long object name
+    //
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    std::string pathname("PATH/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+    string LFN_ATTR = "user.cephos.lfn";
+    if (index_version != HASH_INDEX_TAG) {
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%d", index_version);
+      LFN_ATTR += string(buf);
+    }
+    const std::string object_name_1 = object_name + "SUFFIX";
+    EXPECT_EQ(object_name_1.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_1.c_str(), object_name_1.size()));
+
+    //
+    //   PATH/AAA..._1_long => matches long object name
+    //
+    std::string mangled_name_1;
+    exists = 666;
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_1, &exists));
+    EXPECT_NE(std::string::npos, mangled_name_1.find("1_long"));
+    EXPECT_EQ(0, exists);
+    std::string pathname_1("PATH/" + mangled_name_1);
+    EXPECT_EQ(0, ::close(::creat(pathname_1.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname_1.c_str()));
+
+    //
+    // remove_object skips PATH/AAA..._0_long and removes PATH/AAA..._1_long
+    //
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(0, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+  }
+
+  //
+  // long object name removal of a file in the middle of the list
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    //
+    //   PATH/AAA..._0_long => matches long object name
+    //
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    std::string pathname("PATH/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+    //
+    //   PATH/AAA..._1_long => matches long object name
+    //
+    std::string mangled_name_1 = mangled_name;
+    mangled_name_1.replace(mangled_name_1.find("0_long"), 6, "1_long");
+    const std::string pathname_1("PATH/" + mangled_name_1);
+    const std::string cmd("cp --preserve=xattr " + pathname + " " + pathname_1);
+    EXPECT_EQ(0, ::system(cmd.c_str()));
+    const string ATTR = "user.MARK";
+    EXPECT_EQ((unsigned)1, (unsigned)chain_setxattr(pathname_1.c_str(), ATTR.c_str(), "Y", 1));
+
+    //
+    // remove_object replaces the file to be removed with the last from the
+    // collision list. In this case it replaces
+    //    PATH/AAA..._0_long 
+    // with
+    //    PATH/AAA..._1_long 
+    //
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(0, ::access(pathname.c_str(), 0));
+    char buffer[1] = { 0, };
+    EXPECT_EQ((unsigned)1, (unsigned)chain_getxattr(pathname.c_str(), ATTR.c_str(), buffer, 1));
+    EXPECT_EQ('Y', buffer[0]);
+    EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+  }
+}
+
+TEST_F(TestLFNIndex, get_mangled_name) {
+  const vector path;
+
+  //
+  // small object name
+  //
+  {
+    std::string mangled_name;
+    int exists = 666;
+    ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
+
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
+    EXPECT_EQ(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(0, exists);
+    const std::string pathname("PATH/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
+    EXPECT_EQ(1, exists);
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+  }
+  //
+  // long object name
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    //
+    // long version of the mangled name and no matching
+    // file exists 
+    //
+    mangled_name.clear();
+    exists = 666;
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(0, exists);
+
+    const std::string pathname("PATH/" + mangled_name);
+
+    //
+    // if a file by the same name exists but does not have the
+    // expected extended attribute, it is silently removed 
+    //
+    mangled_name.clear();
+    exists = 666;
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(0, exists);
+    EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+
+    //
+    // if a file by the same name exists but does not have the
+    // expected extended attribute, and cannot be removed, 
+    // return on error
+    //
+    mangled_name.clear();
+    exists = 666;
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, ::chmod("PATH", 0500));
+    EXPECT_EQ(-EACCES, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_EQ("", mangled_name);
+    EXPECT_EQ(666, exists);
+    EXPECT_EQ(0, ::chmod("PATH", 0700));
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+
+    //
+    // long version of the mangled name and a file
+    // exists by that name and contains the long object name
+    //
+    mangled_name.clear();
+    exists = 666;
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(1, exists);
+    EXPECT_EQ(0, ::access(pathname.c_str(), 0));
+
+    //
+    // long version of the mangled name and a file exists by that name
+    // and contains a long object name with the same prefix but they
+    // are not identical and it so happens that their SHA1 is
+    // identical : a collision number is used to differentiate them
+    //
+    string LFN_ATTR = "user.cephos.lfn";
+    if (index_version != HASH_INDEX_TAG) {
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%d", index_version);
+      LFN_ATTR += string(buf);
+    }
+    const std::string object_name_same_prefix = object_name + "SUFFIX";
+    EXPECT_EQ(object_name_same_prefix.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_same_prefix.c_str(), object_name_same_prefix.size()));
+    std::string mangled_name_same_prefix;
+    exists = 666;
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_same_prefix, &exists));
+    EXPECT_NE(std::string::npos, mangled_name_same_prefix.find("1_long"));
+    EXPECT_EQ(0, exists);
+    
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+  }
+}
+
+int main(int argc, char **argv) {
+  int fd = ::creat("detect", 0600);
+  int ret = chain_fsetxattr(fd, "user.test", "A", 1);
+  ::close(fd);
+  ::unlink("detect");
+  if (ret < 0) {
+    cerr << "SKIP LFNIndex because unable to test for xattr" << std::endl;
+  } else {
+    vector args;
+    argv_to_vec(argc, (const char **)argv, args);
+
+    global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+    common_init_finish(g_ceph_context);
+
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+  }
+}
+
+/* 
+ * Local Variables:
+ * compile-command: "cd ../.. ; 
+ *   make unittest_lfnindex && 
+ *   valgrind --tool=memcheck ./unittest_lfnindex \
+ *   # --gtest_filter=TestLFNIndex.* --log-to-stderr=true --debug-filestore=20"
+ * End:
+ */
diff --git a/ceph/src/test/osd/Object.cc b/ceph/src/test/osd/Object.cc
new file mode 100644
index 00000000..c98d62d2
--- /dev/null
+++ b/ceph/src/test/osd/Object.cc
@@ -0,0 +1,182 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+#include "include/interval_set.h"
+#include "include/buffer.h"
+#include 
+#include 
+#include 
+
+#include "Object.h"
+
+void ContDesc::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  ::encode(objnum, bl);
+  ::encode(cursnap, bl);
+  ::encode(seqnum, bl);
+  ::encode(prefix, bl);
+  ::encode(oid, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ContDesc::decode(bufferlist::iterator &bl)
+{
+  DECODE_START(1, bl);
+  ::decode(objnum, bl);
+  ::decode(cursnap, bl);
+  ::decode(seqnum, bl);
+  ::decode(prefix, bl);
+  ::decode(oid, bl);
+  DECODE_FINISH(bl);
+}
+
+ostream &operator<<(ostream &out, const ContDesc &rhs)
+{
+  return out << "(ObjNum " << rhs.objnum
+	     << " snap " << rhs.cursnap
+	     << " seq_num " << rhs.seqnum
+    //<< " prefix " << rhs.prefix
+	     << ")";
+}
+
+void AppendGenerator::get_ranges_map(
+  const ContDesc &cont, map &out) {
+  RandWrap rand(cont.seqnum);
+  uint64_t pos = off;
+  uint64_t limit = off + get_append_size(cont);
+  while (pos < limit) {
+    uint64_t segment_length = round_up(
+      rand() % (max_append_size - min_append_size),
+      alignment) + min_append_size;
+    assert(segment_length >= min_append_size);
+    if (segment_length + pos > limit) {
+      segment_length = limit - pos;
+    }
+    if (alignment)
+      assert(segment_length % alignment == 0);
+    out.insert(make_pair(pos, segment_length));
+    pos += segment_length;
+  }
+}
+
+void VarLenGenerator::get_ranges_map(
+  const ContDesc &cont, map &out) {
+  RandWrap rand(cont.seqnum);
+  uint64_t pos = 0;
+  uint64_t limit = get_length(cont);
+  bool include = false;
+  while (pos < limit) {
+    uint64_t segment_length = (rand() % (max_stride_size - min_stride_size)) + min_stride_size;
+    assert(segment_length < max_stride_size);
+    assert(segment_length >= min_stride_size);
+    if (segment_length + pos > limit) {
+      segment_length = limit - pos;
+    }
+    if (include) {
+      out.insert(make_pair(pos, segment_length));
+      include = false;
+    } else {
+      include = true;
+    }
+    pos += segment_length;
+  }
+  // make sure we write up to the limit
+  if (limit > 0 && (
+	out.empty() ||
+	(out.rbegin()->first + out.rbegin()->second < limit)))
+    out[limit-1] = 1;
+}
+
+ObjectDesc::iterator &ObjectDesc::iterator::advance(bool init) {
+  assert(pos < limit);
+  assert(!end());
+  if (!init) {
+    pos++;
+  }
+  if (end()) {
+    return *this;
+  }
+  while (pos == limit) {
+    cur_cont = stack.begin()->first;
+    limit = stack.begin()->second;
+    stack.pop_front();
+  }
+
+  if (cur_cont == obj.layers.end()) {
+    return *this;
+  }
+
+  interval_set ranges;
+  cur_cont->first->get_ranges(cur_cont->second, ranges);
+  while (!ranges.contains(pos)) {
+    stack.push_front(make_pair(cur_cont, limit));
+    uint64_t length = cur_cont->first->get_length(cur_cont->second);
+    uint64_t next;
+    if (pos >= length) {
+      next = limit;
+      cur_cont = obj.layers.end();
+    } else if (ranges.empty() || pos >= ranges.range_end()) {
+      next = length;
+      ++cur_cont;
+    } else {
+      next = ranges.start_after(pos);
+      ++cur_cont;
+    }
+    if (next < limit) {
+      limit = next;
+    }
+    if (cur_cont == obj.layers.end()) {
+      break;
+    }
+
+    ranges.clear();
+    cur_cont->first->get_ranges(cur_cont->second, ranges);
+  }
+
+  if (cur_cont == obj.layers.end()) {
+    return *this;
+  }
+
+  if (!cont_iters.count(cur_cont->second)) {
+    cont_iters.insert(pair(
+			cur_cont->second,
+			cur_cont->first->get_iterator(cur_cont->second)));
+  }
+  map::iterator j = cont_iters.find(
+    cur_cont->second);
+  assert(j != cont_iters.end());
+  j->second.seek(pos);
+  return *this;
+}
+
+const ContDesc &ObjectDesc::most_recent() {
+  return layers.begin()->second;
+}
+
+void ObjectDesc::update(ContentsGenerator *gen, const ContDesc &next) {
+  layers.push_front(make_pair(gen, next));
+  return;
+}
+
+bool ObjectDesc::check(bufferlist &to_check) {
+  iterator i = begin();
+  uint64_t pos = 0;
+  for (bufferlist::iterator p = to_check.begin();
+       !p.end();
+       ++p, ++i, ++pos) {
+    if (i.end()) {
+      std::cout << "reached end of iterator first" << std::endl;
+      return false;
+    }
+    if (*i != *p) {
+      std::cout << "incorrect buffer at pos " << pos << std::endl;
+      return false;
+    }
+  }
+  uint64_t size = layers.empty() ? 0 : 
+    most_recent_gen()->get_length(most_recent());
+  if (pos != size) {
+    std::cout << "only read " << pos << " out of size " << size << std::endl;
+    return false;
+  }
+  return true;
+}
diff --git a/ceph/src/test/osd/Object.h b/ceph/src/test/osd/Object.h
new file mode 100644
index 00000000..d39d36c6
--- /dev/null
+++ b/ceph/src/test/osd/Object.h
@@ -0,0 +1,367 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+#include "include/interval_set.h"
+#include "include/buffer.h"
+#include "include/encoding.h"
+#include 
+#include 
+#include 
+
+#ifndef OBJECT_H
+#define OBJECT_H
+
+class ContDesc {
+public:
+  int objnum;
+  int cursnap;
+  unsigned seqnum;
+  string prefix;
+  string oid;
+
+  ContDesc() :
+    objnum(0), cursnap(0),
+    seqnum(0), prefix("") {}
+
+  ContDesc(int objnum,
+	   int cursnap,
+	   unsigned seqnum,
+	   const string &prefix) :
+    objnum(objnum), cursnap(cursnap),
+    seqnum(seqnum), prefix(prefix) {}
+
+  bool operator==(const ContDesc &rhs) {
+    return (rhs.objnum == objnum &&
+	    rhs.cursnap == cursnap &&
+	    rhs.seqnum == seqnum &&
+	    rhs.prefix == prefix &&
+	    rhs.oid == oid);
+  }
+
+  bool operator<(const ContDesc &rhs) const {
+    return seqnum < rhs.seqnum;
+  }
+
+  bool operator!=(const ContDesc &rhs) {
+    return !((*this) == rhs);
+  }
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::iterator &bp);
+};
+WRITE_CLASS_ENCODER(ContDesc)
+
+ostream &operator<<(ostream &out, const ContDesc &rhs);
+
+class ContentsGenerator {
+public:
+
+  class iterator_impl {
+  public:
+    virtual char operator*() = 0;
+    virtual iterator_impl &operator++() = 0;
+    virtual void seek(uint64_t pos) = 0;
+    virtual bool end() = 0;
+    virtual ContDesc get_cont() const = 0;
+    virtual uint64_t get_pos() const = 0;
+    virtual ~iterator_impl() {};
+  };
+
+  class iterator {
+  public:
+    ContentsGenerator *parent;
+    iterator_impl *impl;
+    char operator *() { return **impl; }
+    iterator &operator++() { ++(*impl); return *this; };
+    void seek(uint64_t pos) { impl->seek(pos); }
+    bool end() { return impl->end(); }
+    ~iterator() { parent->put_iterator_impl(impl); }
+    iterator(const iterator &rhs) : parent(rhs.parent) {
+      impl = parent->dup_iterator_impl(rhs.impl);
+    }
+    iterator &operator=(const iterator &rhs) {
+      iterator new_iter(rhs);
+      swap(new_iter);
+      return *this;
+    }
+    void swap(iterator &other) {
+      ContentsGenerator *otherparent = other.parent;
+      other.parent = parent;
+      parent = otherparent;
+
+      iterator_impl *otherimpl = other.impl;
+      other.impl = impl;
+      impl = otherimpl;
+    }
+    iterator(ContentsGenerator *parent, iterator_impl *impl) :
+      parent(parent), impl(impl) {}
+  };
+
+  virtual uint64_t get_length(const ContDesc &in) = 0;
+
+  virtual void get_ranges_map(
+    const ContDesc &cont, map &out) = 0;
+  void get_ranges(const ContDesc &cont, interval_set &out) {
+    map ranges;
+    get_ranges_map(cont, ranges);
+    for (map::iterator i = ranges.begin();
+	 i != ranges.end();
+	 ++i) {
+      out.insert(i->first, i->second);
+    }
+  }
+
+
+  virtual iterator_impl *get_iterator_impl(const ContDesc &in) = 0;
+
+  virtual iterator_impl *dup_iterator_impl(const iterator_impl *in) = 0;
+
+  virtual void put_iterator_impl(iterator_impl *in) = 0;
+
+  virtual ~ContentsGenerator() {};
+
+  iterator get_iterator(const ContDesc &in) {
+    return iterator(this, get_iterator_impl(in));
+  }
+};
+
+class RandGenerator : public ContentsGenerator {
+public:
+  class RandWrap {
+  public:
+    unsigned int state;
+    RandWrap(unsigned int seed)
+    {
+      state = seed;
+    }
+
+    int operator()()
+    {
+      return rand_r(&state);
+    }
+  };
+
+  class iterator_impl : public ContentsGenerator::iterator_impl {
+  public:
+    uint64_t pos;
+    ContDesc cont;
+    RandWrap rand;
+    RandGenerator *cont_gen;
+    char current;
+    iterator_impl(const ContDesc &cont, RandGenerator *cont_gen) : 
+      pos(0), cont(cont), rand(cont.seqnum), cont_gen(cont_gen) {
+      current = rand();
+    }
+
+    ContDesc get_cont() const { return cont; }
+    uint64_t get_pos() const { return pos; }
+
+    iterator_impl &operator++() {
+      pos++;
+      current = rand();
+      return *this;
+    }
+
+    char operator*() {
+      return current;
+    }
+
+    void seek(uint64_t _pos) {
+      if (_pos < pos) {
+	iterator_impl begin = iterator_impl(cont, cont_gen);
+	begin.seek(_pos);
+	*this = begin;
+      }
+      while (pos < _pos) {
+	++(*this);
+      }
+    }
+
+    bool end() {
+      return pos >= cont_gen->get_length(cont);
+    }
+  };
+
+  ContentsGenerator::iterator_impl *get_iterator_impl(const ContDesc &in) {
+    RandGenerator::iterator_impl *i = new iterator_impl(in, this);
+    return i;
+  }
+
+  void put_iterator_impl(ContentsGenerator::iterator_impl *in) {
+    delete in;
+  }
+
+  ContentsGenerator::iterator_impl *dup_iterator_impl(
+    const ContentsGenerator::iterator_impl *in) {
+    ContentsGenerator::iterator_impl *retval = get_iterator_impl(in->get_cont());
+    retval->seek(in->get_pos());
+    return retval;
+  }
+};
+
+class VarLenGenerator : public RandGenerator {
+  uint64_t max_length;
+  uint64_t min_stride_size;
+  uint64_t max_stride_size;
+public:
+  VarLenGenerator(
+    uint64_t length, uint64_t min_stride_size, uint64_t max_stride_size) :
+    max_length(length),
+    min_stride_size(min_stride_size),
+    max_stride_size(max_stride_size) {}
+  void get_ranges_map(
+    const ContDesc &cont, map &out);
+  uint64_t get_length(const ContDesc &in) {
+    RandWrap rand(in.seqnum);
+    return (rand() % max_length);
+  }
+};
+
+class AttrGenerator : public RandGenerator {
+  uint64_t max_len;
+public:
+  AttrGenerator(uint64_t max_len) : max_len(max_len) {}
+  void get_ranges_map(
+    const ContDesc &cont, map &out) {
+    out.insert(make_pair(0, get_length(cont)));
+  }
+  uint64_t get_length(const ContDesc &in) {
+    RandWrap rand(in.seqnum);
+    return (rand() % max_len);
+  }
+  bufferlist gen_bl(const ContDesc &in) {
+    bufferlist bl;
+    for (iterator i = get_iterator(in); !i.end(); ++i) {
+      bl.append(*i);
+    }
+    assert(bl.length() < max_len);
+    return bl;
+  }
+};
+
+class AppendGenerator : public RandGenerator {
+  uint64_t off;
+  uint64_t alignment;
+  uint64_t min_append_size;
+  uint64_t max_append_size;
+  uint64_t max_append_total;
+
+  uint64_t round_up(uint64_t in, uint64_t by) {
+    if (by)
+      in += (by - (in % by));
+    return in;
+  }
+
+public:
+  AppendGenerator(
+    uint64_t off,
+    uint64_t alignment,
+    uint64_t min_append_size,
+    uint64_t _max_append_size,
+    uint64_t max_append_multiple) :
+    off(off), alignment(alignment),
+    min_append_size(round_up(min_append_size, alignment)),
+    max_append_size(round_up(_max_append_size, alignment)) {
+    if (_max_append_size == min_append_size)
+      max_append_size += alignment;
+    max_append_total = max_append_multiple * max_append_size;
+  }
+  uint64_t get_append_size(const ContDesc &in) {
+    RandWrap rand(in.seqnum);
+    return round_up(rand() % max_append_total, alignment);
+  }
+  uint64_t get_length(const ContDesc &in) {
+    return off + get_append_size(in);
+  }
+  void get_ranges_map(
+    const ContDesc &cont, map &out);
+};
+
+class ObjectDesc {
+public:
+  ObjectDesc()
+    : exists(false), dirty(false),
+      version(0) {}
+  ObjectDesc(const ContDesc &init, ContentsGenerator *cont_gen)
+    : exists(false), dirty(false),
+      version(0) {
+    layers.push_front(make_pair(cont_gen, init));
+  }
+
+  class iterator {
+  public:
+    uint64_t pos;
+    ObjectDesc &obj;
+    list,
+			ContDesc> >::iterator,
+	      uint64_t> > stack;
+    map cont_iters;
+    uint64_t limit;
+    list,
+	      ContDesc> >::iterator cur_cont;
+    
+    iterator(ObjectDesc &obj) :
+      pos(0), obj(obj) {
+      limit = obj.layers.begin()->first->get_length(obj.layers.begin()->second);
+      cur_cont = obj.layers.begin();
+      advance(true);
+    }
+
+    iterator &advance(bool init);
+    iterator &operator++() {
+      return advance(false);
+    }
+
+    char operator*() {
+      if (cur_cont == obj.layers.end()) {
+	return '\0';
+      } else {
+	map::iterator j = cont_iters.find(
+	  cur_cont->second);
+	assert(j != cont_iters.end());
+	return *(j->second);
+      }
+    }
+
+    bool end() {
+      return pos >= obj.layers.begin()->first->get_length(
+	obj.layers.begin()->second);
+    }
+
+    void seek(uint64_t _pos) {
+      if (_pos < pos) {
+	assert(0);
+      }
+      while (pos < _pos) {
+	++(*this);
+      }
+    }
+  };
+    
+  iterator begin() {
+    return iterator(*this);
+  }
+
+  bool deleted() {
+    return !exists;
+  }
+
+  bool has_contents() {
+    return layers.size();
+  }
+
+  // takes ownership of gen
+  void update(ContentsGenerator *gen, const ContDesc &next);
+  bool check(bufferlist &to_check);
+  const ContDesc &most_recent();
+  ContentsGenerator *most_recent_gen() {
+    return layers.begin()->first.get();
+  }
+  map attrs; // Both omap and xattrs
+  bufferlist header;
+  bool exists;
+  bool dirty;
+
+  uint64_t version;
+private:
+  list, ContDesc> > layers;
+};
+
+#endif
diff --git a/ceph/src/test/osd/RadosModel.cc b/ceph/src/test/osd/RadosModel.cc
new file mode 100644
index 00000000..efde283f
--- /dev/null
+++ b/ceph/src/test/osd/RadosModel.cc
@@ -0,0 +1,37 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+#include "include/interval_set.h"
+#include "include/buffer.h"
+#include 
+#include 
+#include 
+#include "include/rados/librados.h"
+#include "RadosModel.h"
+#include "TestOpStat.h"
+
+
+void TestOp::begin()
+{
+  //if (stat) stat->begin(this);
+  _begin();
+}
+
+void TestOp::finish(TestOp::CallbackInfo *info)
+{
+  _finish(info);
+  //if (stat && finished()) stat->end(this);
+}
+
+void read_callback(librados::completion_t comp, void *arg) {
+  TestOp* op = static_cast(arg);
+  op->finish(NULL);
+}
+
+void write_callback(librados::completion_t comp, void *arg) {
+  std::pair *args =
+    static_cast *>(arg);
+  TestOp* op = args->first;
+  TestOp::CallbackInfo *info = args->second;
+  op->finish(info);
+  delete args;
+  delete info;
+}
diff --git a/ceph/src/test/osd/RadosModel.h b/ceph/src/test/osd/RadosModel.h
new file mode 100644
index 00000000..e81699da
--- /dev/null
+++ b/ceph/src/test/osd/RadosModel.h
@@ -0,0 +1,1996 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+#include "include/int_types.h"
+
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "include/rados/librados.hpp"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "Object.h"
+#include "TestOpStat.h"
+#include "test/librados/test.h"
+#include "include/memory.h"
+#include "common/sharedptr_registry.hpp"
+#include "common/errno.h"
+#include "osd/HitSet.h"
+
+#ifndef RADOSMODEL_H
+#define RADOSMODEL_H
+
+using namespace std;
+
+class RadosTestContext;
+class TestOpStat;
+
+template 
+typename T::iterator rand_choose(T &cont) {
+  if (cont.size() == 0) {
+    return cont.end();
+  }
+  int index = rand() % cont.size();
+  typename T::iterator retval = cont.begin();
+
+  for (; index > 0; --index) ++retval;
+  return retval;
+}
+
+enum TestOpType {
+  TEST_OP_READ,
+  TEST_OP_WRITE,
+  TEST_OP_DELETE,
+  TEST_OP_SNAP_CREATE,
+  TEST_OP_SNAP_REMOVE,
+  TEST_OP_ROLLBACK,
+  TEST_OP_SETATTR,
+  TEST_OP_RMATTR,
+  TEST_OP_WATCH,
+  TEST_OP_COPY_FROM,
+  TEST_OP_HIT_SET_LIST,
+  TEST_OP_UNDIRTY,
+  TEST_OP_IS_DIRTY,
+  TEST_OP_CACHE_FLUSH,
+  TEST_OP_CACHE_TRY_FLUSH,
+  TEST_OP_CACHE_EVICT,
+  TEST_OP_APPEND
+};
+
+class TestWatchContext : public librados::WatchCtx {
+  TestWatchContext(const TestWatchContext&);
+public:
+  Cond cond;
+  uint64_t handle;
+  bool waiting;
+  Mutex lock;
+  TestWatchContext() : handle(0), waiting(false),
+		       lock("watch lock") {}
+  void notify(uint8_t opcode, uint64_t ver, bufferlist &bl) {
+    Mutex::Locker l(lock);
+    waiting = false;
+    cond.SignalAll();
+  }
+  void start() {
+    Mutex::Locker l(lock);
+    waiting = true;
+  }
+  void wait() {
+    Mutex::Locker l(lock);
+    while (waiting)
+      cond.Wait(lock);
+  }
+  uint64_t &get_handle() {
+    return handle;
+  }
+};
+
+class TestOp {
+public:
+  int num;
+  RadosTestContext *context;
+  TestOpStat *stat;
+  bool done;
+  TestOp(int n, RadosTestContext *context,
+	 TestOpStat *stat = 0)
+    : num(n),
+      context(context),
+      stat(stat),
+      done(0)
+  {}
+
+  virtual ~TestOp() {};
+
+  /**
+   * This struct holds data to be passed by a callback
+   * to a TestOp::finish method.
+   */
+  struct CallbackInfo {
+    uint64_t id;
+    CallbackInfo(uint64_t id) : id(id) {}
+    virtual ~CallbackInfo() {};
+  };
+
+  virtual void _begin() = 0;
+
+  /**
+   * Called when the operation completes.
+   * This should be overridden by asynchronous operations.
+   *
+   * @param info information stored by a callback, or NULL -
+   *             useful for multi-operation TestOps
+   */
+  virtual void _finish(CallbackInfo *info)
+  {
+    return;
+  }
+  virtual string getType() = 0;
+  virtual bool finished()
+  {
+    return true;
+  }
+
+  void begin();
+  void finish(CallbackInfo *info);
+  virtual bool must_quiesce_other_ops() { return false; }
+};
+
+class TestOpGenerator {
+public:
+  virtual ~TestOpGenerator() {};
+  virtual TestOp *next(RadosTestContext &context) = 0;
+};
+
+class RadosTestContext {
+public:
+  Mutex state_lock;
+  Cond wait_cond;
+  map > pool_obj_cont;
+  set oid_in_use;
+  set oid_not_in_use;
+  set oid_flushing;
+  set oid_not_flushing;
+  SharedPtrRegistry snaps_in_use;
+  int current_snap;
+  string pool_name;
+  librados::IoCtx io_ctx;
+  librados::Rados rados;
+  int next_oid;
+  string prefix;
+  int errors;
+  int max_in_flight;
+  int seq_num;
+  map snaps;
+  uint64_t seq;
+  const char *rados_id;
+  bool initialized;
+  map watches;
+  const uint64_t max_size;
+  const uint64_t min_stride_size;
+  const uint64_t max_stride_size;
+  AttrGenerator attr_gen;
+  const bool no_omap;
+  bool pool_snaps;
+  int snapname_num;
+
+  RadosTestContext(const string &pool_name, 
+		   int max_in_flight,
+		   uint64_t max_size,
+		   uint64_t min_stride_size,
+		   uint64_t max_stride_size,
+		   bool no_omap,
+		   bool pool_snaps,
+		   const char *id = 0) :
+    state_lock("Context Lock"),
+    pool_obj_cont(),
+    current_snap(0),
+    pool_name(pool_name),
+    next_oid(0),
+    errors(0),
+    max_in_flight(max_in_flight),
+    seq_num(0), seq(0),
+    rados_id(id), initialized(false),
+    max_size(max_size), 
+    min_stride_size(min_stride_size), max_stride_size(max_stride_size),
+    attr_gen(2000),
+    no_omap(no_omap),
+    pool_snaps(pool_snaps),
+    snapname_num(0)
+  {
+  }
+
+  int init()
+  {
+    int r = rados.init(rados_id);
+    if (r < 0)
+      return r;
+    r = rados.conf_read_file(NULL);
+    if (r < 0)
+      return r;
+    r = rados.conf_parse_env(NULL);
+    if (r < 0)
+      return r;
+    r = rados.connect();
+    if (r < 0)
+      return r;
+    r = rados.ioctx_create(pool_name.c_str(), io_ctx);
+    if (r < 0) {
+      rados.shutdown();
+      return r;
+    }
+    char hostname_cstr[100];
+    gethostname(hostname_cstr, 100);
+    stringstream hostpid;
+    hostpid << hostname_cstr << getpid() << "-";
+    prefix = hostpid.str();
+    assert(!initialized);
+    initialized = true;
+    return 0;
+  }
+
+  void shutdown()
+  {
+    if (initialized) {
+      rados.shutdown();
+    }
+  }
+
+  void loop(TestOpGenerator *gen)
+  {
+    assert(initialized);
+    list inflight;
+    state_lock.Lock();
+
+    TestOp *next = gen->next(*this);
+    TestOp *waiting = NULL;
+
+    while (next || !inflight.empty()) {
+      if (next && next->must_quiesce_other_ops() && !inflight.empty()) {
+	waiting = next;
+	next = NULL;   // Force to wait for inflight to drain
+      }
+      if (next) {
+	inflight.push_back(next);
+      }
+      state_lock.Unlock();
+      if (next) {
+	(*inflight.rbegin())->begin();
+      }
+      state_lock.Lock();
+      while (1) {
+	for (list::iterator i = inflight.begin();
+	     i != inflight.end();) {
+	  if ((*i)->finished()) {
+	    cout << (*i)->num << ": done (" << (inflight.size()-1) << " left)" << std::endl;
+	    delete *i;
+	    inflight.erase(i++);
+	  } else {
+	    ++i;
+	  }
+	}
+	
+	if (inflight.size() >= (unsigned) max_in_flight || (!next && !inflight.empty())) {
+	  cout << " waiting on " << inflight.size() << std::endl;
+	  wait();
+	} else {
+	  break;
+	}
+      }
+      if (waiting) {
+	next = waiting;
+	waiting = NULL;
+      } else {
+	next = gen->next(*this);
+      }
+    }
+    state_lock.Unlock();
+  }
+
+  void wait()
+  {
+    wait_cond.Wait(state_lock);
+  }
+
+  void kick()
+  {
+    wait_cond.Signal();
+  }
+
+  TestWatchContext *get_watch_context(const string &oid) {
+    return watches.count(oid) ? watches[oid] : 0;
+  }
+
+  TestWatchContext *watch(const string &oid) {
+    assert(!watches.count(oid));
+    return (watches[oid] = new TestWatchContext);
+  }
+
+  void unwatch(const string &oid) {
+    assert(watches.count(oid));
+    delete watches[oid];
+    watches.erase(oid);
+  }
+
+  ObjectDesc get_most_recent(const string &oid) {
+    ObjectDesc new_obj;
+    for (map >::reverse_iterator i =
+	   pool_obj_cont.rbegin();
+	 i != pool_obj_cont.rend();
+	 ++i) {
+      map::iterator j = i->second.find(oid);
+      if (j != i->second.end()) {
+	new_obj = j->second;
+	break;
+      }
+    }
+    return new_obj;
+  }
+
+  void rm_object_attrs(const string &oid, const set &attrs)
+  {
+    ObjectDesc new_obj = get_most_recent(oid);
+    for (set::const_iterator i = attrs.begin();
+	 i != attrs.end();
+	 ++i) {
+      new_obj.attrs.erase(*i);
+    }
+    new_obj.dirty = true;
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, new_obj));
+  }
+
+  void remove_object_header(const string &oid)
+  {
+    ObjectDesc new_obj = get_most_recent(oid);
+    new_obj.header = bufferlist();
+    new_obj.dirty = true;
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, new_obj));
+  }
+
+
+  void update_object_header(const string &oid, const bufferlist &bl)
+  {
+    ObjectDesc new_obj = get_most_recent(oid);
+    new_obj.header = bl;
+    new_obj.exists = true;
+    new_obj.dirty = true;
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, new_obj));
+  }
+
+  void update_object_attrs(const string &oid, const map &attrs)
+  {
+    ObjectDesc new_obj = get_most_recent(oid);
+    for (map::const_iterator i = attrs.begin();
+	 i != attrs.end();
+	 ++i) {
+      new_obj.attrs[i->first] = i->second;
+    }
+    new_obj.exists = true;
+    new_obj.dirty = true;
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, new_obj));
+  }
+
+  void update_object(ContentsGenerator *cont_gen,
+		     const string &oid, const ContDesc &contents)
+  {
+    ObjectDesc new_obj = get_most_recent(oid);
+    new_obj.exists = true;
+    new_obj.dirty = true;
+    new_obj.update(cont_gen,
+		   contents);
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, new_obj));
+  }
+
+  void update_object_full(const string &oid, const ObjectDesc &contents)
+  {
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, contents));
+    pool_obj_cont[current_snap][oid].dirty = true;
+  }
+
+  void update_object_undirty(const string &oid)
+  {
+    ObjectDesc new_obj = get_most_recent(oid);
+    new_obj.dirty = false;
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, new_obj));
+  }
+
+  void update_object_version(const string &oid, uint64_t version,
+			     int snap = -1)
+  {
+    for (map >::reverse_iterator i = 
+	   pool_obj_cont.rbegin();
+	 i != pool_obj_cont.rend();
+	 ++i) {
+      if (snap != -1 && snap < i->first)
+	continue;
+      map::iterator j = i->second.find(oid);
+      if (j != i->second.end()) {
+	if (version)
+	  j->second.version = version;
+	cout << __func__ << " oid " << oid
+	     << " v " << version << " " << j->second.most_recent()
+	     << " " << (j->second.dirty ? "dirty" : "clean")
+	     << " " << (j->second.exists ? "exists" : "dne")
+	     << std::endl;
+	break;
+      }
+    }
+  }
+
+  void remove_object(const string &oid)
+  {
+    assert(!get_watch_context(oid));
+    ObjectDesc new_obj;
+    pool_obj_cont[current_snap].erase(oid);
+    pool_obj_cont[current_snap].insert(pair(oid, new_obj));
+  }
+
+  bool find_object(const string &oid, ObjectDesc *contents, int snap = -1) const
+  {
+    for (map >::const_reverse_iterator i = 
+	   pool_obj_cont.rbegin();
+	 i != pool_obj_cont.rend();
+	 ++i) {
+      if (snap != -1 && snap < i->first) continue;
+      if (i->second.count(oid) != 0) {
+	*contents = i->second.find(oid)->second;
+	return true;
+      }
+    }
+    return false;
+  }
+
+  void remove_snap(int snap)
+  {
+    map >::iterator next_iter = pool_obj_cont.find(snap);
+    assert(next_iter != pool_obj_cont.end());
+    map >::iterator current_iter = next_iter++;
+    assert(current_iter != pool_obj_cont.end());
+    map ¤t = current_iter->second;
+    map &next = next_iter->second;
+    for (map::iterator i = current.begin();
+	 i != current.end();
+	 ++i) {
+      if (next.count(i->first) == 0) {
+	next.insert(pair(i->first, i->second));
+      }
+    }
+    pool_obj_cont.erase(current_iter);
+    snaps.erase(snap);
+  }
+
+  void add_snap(uint64_t snap)
+  {
+    snaps[current_snap] = snap;
+    current_snap++;
+    pool_obj_cont[current_snap];
+    seq = snap;
+  }
+
+  void roll_back(const string &oid, int snap)
+  {
+    assert(!get_watch_context(oid));
+    ObjectDesc contents;
+    find_object(oid, &contents, snap);
+    contents.dirty = true;
+    pool_obj_cont.rbegin()->second.erase(oid);
+    pool_obj_cont.rbegin()->second.insert(pair(oid, contents));
+  }
+};
+
+void read_callback(librados::completion_t comp, void *arg);
+void write_callback(librados::completion_t comp, void *arg);
+
+class RemoveAttrsOp : public TestOp {
+public:
+  string oid;
+  librados::ObjectWriteOperation op;
+  librados::AioCompletion *comp;
+  bool done;
+  RemoveAttrsOp(int n, RadosTestContext *context,
+	       const string &oid,
+	       TestOpStat *stat)
+    : TestOp(n, context, stat), oid(oid), comp(NULL), done(false)
+  {}
+
+  void _begin()
+  {
+    ContDesc cont;
+    set to_remove;
+    {
+      Mutex::Locker l(context->state_lock);
+      ObjectDesc obj;
+      if (!context->find_object(oid, &obj)) {
+	context->kick();
+	done = true;
+	return;
+      }
+      cont = ContDesc(context->seq_num, context->current_snap,
+		      context->seq_num, "");
+      context->oid_in_use.insert(oid);
+      context->oid_not_in_use.erase(oid);
+
+      if (rand() % 30) {
+	ContentsGenerator::iterator iter = context->attr_gen.get_iterator(cont);
+	for (map::iterator i = obj.attrs.begin();
+	     i != obj.attrs.end();
+	     ++i, ++iter) {
+	  if (!(*iter % 3)) {
+	    //op.rmxattr(i->first.c_str());
+	    to_remove.insert(i->first);
+	    op.rmxattr(i->first.c_str());
+	  }
+	}
+	if (to_remove.empty()) {
+	  context->kick();
+	  context->oid_in_use.erase(oid);
+	  context->oid_not_in_use.insert(oid);
+	  done = true;
+	  return;
+	}
+	if (!context->no_omap) {
+	  op.omap_rm_keys(to_remove);
+	}
+      } else {
+	if (!context->no_omap) {
+	  op.omap_clear();
+	}
+	for (map::iterator i = obj.attrs.begin();
+	     i != obj.attrs.end();
+	     ++i) {
+	  op.rmxattr(i->first.c_str());
+	  to_remove.insert(i->first);
+	}
+	context->remove_object_header(oid);
+      }
+      context->rm_object_attrs(oid, to_remove);
+    }
+
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+						&write_callback);
+    context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    Mutex::Locker l(context->state_lock);
+    done = true;
+    context->update_object_version(oid, comp->get_version64());
+    context->oid_in_use.erase(oid);
+    context->oid_not_in_use.insert(oid);
+    context->kick();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "RemoveAttrsOp";
+  }
+};
+
+class SetAttrsOp : public TestOp {
+public:
+  string oid;
+  librados::ObjectWriteOperation op;
+  librados::AioCompletion *comp;
+  bool done;
+  SetAttrsOp(int n,
+	     RadosTestContext *context,
+	     const string &oid,
+	     TestOpStat *stat)
+    : TestOp(n, context, stat),
+      oid(oid), comp(NULL), done(false)
+  {}
+
+  void _begin()
+  {
+    ContDesc cont;
+    {
+      Mutex::Locker l(context->state_lock);
+      cont = ContDesc(context->seq_num, context->current_snap,
+		      context->seq_num, "");
+      context->oid_in_use.insert(oid);
+      context->oid_not_in_use.erase(oid);
+    }
+
+    map omap_contents;
+    map omap;
+    bufferlist header;
+    ContentsGenerator::iterator keygen = context->attr_gen.get_iterator(cont);
+    op.create(false);
+    while (!*keygen) ++keygen;
+    while (*keygen) {
+      if (*keygen != '_')
+	header.append(*keygen);
+      ++keygen;
+    }
+    for (int i = 0; i < 20; ++i) {
+      string key;
+      while (!*keygen) ++keygen;
+      while (*keygen && key.size() < 40) {
+	key.push_back((*keygen % 20) + 'a');
+	++keygen;
+      }
+      ContDesc val(cont);
+      val.seqnum += (unsigned)(*keygen);
+      val.prefix = ("oid: " + oid);
+      omap[key] = val;
+      bufferlist val_buffer = context->attr_gen.gen_bl(val);
+      omap_contents[key] = val_buffer;
+      op.setxattr(key.c_str(), val_buffer);
+    }
+    if (!context->no_omap) {
+      op.omap_set_header(header);
+      op.omap_set(omap_contents);
+    }
+
+    {
+      Mutex::Locker l(context->state_lock);
+      context->update_object_header(oid, header);
+      context->update_object_attrs(oid, omap);
+    }
+
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+						&write_callback);
+    context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    Mutex::Locker l(context->state_lock);
+    int r;
+    if ((r = comp->get_return_value())) {
+      cerr << "err " << r << std::endl;
+      assert(0);
+    }
+    done = true;
+    context->update_object_version(oid, comp->get_version64());
+    context->oid_in_use.erase(oid);
+    context->oid_not_in_use.insert(oid);
+    context->kick();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "SetAttrsOp";
+  }
+};
+
+class WriteOp : public TestOp {
+public:
+  string oid;
+  ContDesc cont;
+  set waiting;
+  librados::AioCompletion *rcompletion;
+  uint64_t waiting_on;
+  uint64_t last_acked_tid;
+
+  librados::ObjectReadOperation read_op;
+  librados::ObjectWriteOperation write_op;
+  bufferlist rbuffer;
+
+  bool do_append;
+
+  WriteOp(int n,
+	  RadosTestContext *context,
+	  const string &oid,
+	  bool do_append,
+	  TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      oid(oid), waiting_on(0), last_acked_tid(0), do_append(do_append)
+  {}
+		
+  void _begin()
+  {
+    context->state_lock.Lock();
+    done = 0;
+    stringstream acc;
+    acc << context->prefix << "OID: " << oid << " snap " << context->current_snap << std::endl;
+    string prefix = acc.str();
+
+    cont = ContDesc(context->seq_num, context->current_snap, context->seq_num, prefix);
+
+    ContentsGenerator *cont_gen;
+    if (do_append) {
+      ObjectDesc old_value;
+      bool found = context->find_object(oid, &old_value);
+      uint64_t prev_length = found && old_value.has_contents() ?
+	old_value.most_recent_gen()->get_length(old_value.most_recent()) :
+	0;
+      cont_gen = new AppendGenerator(
+	prev_length,
+	(context->io_ctx.pool_requires_alignment() ?
+	 context->io_ctx.pool_required_alignment() : 0),
+	context->min_stride_size,
+	context->max_stride_size,
+	3);
+    } else {
+      cont_gen = new VarLenGenerator(
+	context->max_size, context->min_stride_size, context->max_stride_size);
+    }
+    context->update_object(cont_gen, oid, cont);
+
+    context->oid_in_use.insert(oid);
+    context->oid_not_in_use.erase(oid);
+
+    map ranges;
+
+    cont_gen->get_ranges_map(cont, ranges);
+    std::cout << num << ":  seq_num " << context->seq_num << " ranges " << ranges << std::endl;
+    context->seq_num++;
+
+    waiting_on = ranges.size();
+    //cout << " waiting_on = " << waiting_on << std::endl;
+    ContentsGenerator::iterator gen_pos = cont_gen->get_iterator(cont);
+    uint64_t tid = 1;
+    for (map::iterator i = ranges.begin(); 
+	 i != ranges.end();
+	 ++i, ++tid) {
+      bufferlist to_write;
+      gen_pos.seek(i->first);
+      for (uint64_t k = 0; k != i->second; ++k, ++gen_pos) {
+	to_write.append(*gen_pos);
+      }
+      assert(to_write.length() == i->second);
+      assert(to_write.length() > 0);
+      std::cout << num << ":  writing " << context->prefix+oid
+		<< " from " << i->first
+		<< " to " << i->first + i->second << " tid " << tid << std::endl;
+      pair *cb_arg =
+	new pair(this,
+						 new TestOp::CallbackInfo(tid));
+      librados::AioCompletion *completion =
+	context->rados.aio_create_completion((void*) cb_arg, NULL,
+					     &write_callback);
+      waiting.insert(completion);
+      librados::ObjectWriteOperation op;
+      if (do_append) {
+	op.append(to_write);
+      } else {
+	op.write(i->first, to_write);
+      }
+      context->io_ctx.aio_operate(
+	context->prefix+oid, completion,
+	&op);
+    }
+
+    bufferlist contbl;
+    ::encode(cont, contbl);
+    pair *cb_arg =
+      new pair(
+	this,
+	new TestOp::CallbackInfo(++tid));
+    librados::AioCompletion *completion = context->rados.aio_create_completion(
+      (void*) cb_arg, NULL, &write_callback);
+    waiting.insert(completion);
+    waiting_on++;
+    write_op.setxattr("_header", contbl);
+    if (!do_append) {
+      write_op.truncate(cont_gen->get_length(cont));
+    }
+    context->io_ctx.aio_operate(
+      context->prefix+oid, completion, &write_op);
+
+    cb_arg =
+      new pair(
+	this,
+	new TestOp::CallbackInfo(++tid));
+    rcompletion = context->rados.aio_create_completion(
+         (void*) cb_arg, NULL, &write_callback);
+    waiting_on++;
+    read_op.read(0, 1, &rbuffer, 0);
+    context->io_ctx.aio_operate(
+      context->prefix+oid, rcompletion,
+      &read_op,
+      librados::OPERATION_ORDER_READS_WRITES,  // order wrt previous write/update
+      0);
+    context->state_lock.Unlock();
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    assert(info);
+    context->state_lock.Lock();
+    uint64_t tid = info->id;
+
+    cout << num << ":  finishing write tid " << tid << " to " << context->prefix + oid << std::endl;
+
+    if (tid <= last_acked_tid) {
+      cerr << "Error: finished tid " << tid
+	   << " when last_acked_tid was " << last_acked_tid << std::endl;
+      assert(0);
+    }
+    last_acked_tid = tid;
+
+    assert(!done);
+    waiting_on--;
+    if (waiting_on == 0) {
+      uint64_t version = 0;
+      for (set::iterator i = waiting.begin();
+	   i != waiting.end();
+	   ) {
+	assert((*i)->is_complete());
+	if (int err = (*i)->get_return_value()) {
+	  cerr << "Error: oid " << oid << " write returned error code "
+	       << err << std::endl;
+	}
+	if ((*i)->get_version64() > version)
+	  version = (*i)->get_version64();
+	(*i)->release();
+	waiting.erase(i++);
+      }
+      
+      context->update_object_version(oid, version);
+      if (rcompletion->get_version64() != version) {
+	cerr << "Error: racing read on " << oid << " returned version "
+	     << rcompletion->get_version64() << " rather than version "
+	     << version << std::endl;
+	assert(0 == "racing read got wrong version");
+      }
+      rcompletion->release();
+      context->oid_in_use.erase(oid);
+      context->oid_not_in_use.insert(oid);
+      context->kick();
+      done = true;
+    }
+    context->state_lock.Unlock();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "WriteOp";
+  }
+};
+
+class DeleteOp : public TestOp {
+public:
+  string oid;
+
+  DeleteOp(int n,
+	   RadosTestContext *context,
+	   const string &oid,
+	   TestOpStat *stat = 0)
+    : TestOp(n, context, stat), oid(oid)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+    if (context->get_watch_context(oid)) {
+      context->kick();
+      context->state_lock.Unlock();
+      return;
+    }
+
+    ObjectDesc contents;
+    context->find_object(oid, &contents);
+    bool present = !contents.deleted();
+
+    context->oid_in_use.insert(oid);
+    context->oid_not_in_use.erase(oid);
+    context->seq_num++;
+
+    context->remove_object(oid);
+
+    interval_set ranges;
+    context->state_lock.Unlock();
+
+    int r = context->io_ctx.remove(context->prefix+oid);
+    if (r && !(r == -ENOENT && !present)) {
+      cerr << "r is " << r << " while deleting " << oid << " and present is " << present << std::endl;
+      assert(0);
+    }
+
+    context->state_lock.Lock();
+    context->oid_in_use.erase(oid);
+    context->oid_not_in_use.insert(oid);
+    context->kick();
+    context->state_lock.Unlock();
+  }
+
+  string getType()
+  {
+    return "DeleteOp";
+  }
+};
+
+class ReadOp : public TestOp {
+public:
+  librados::AioCompletion *completion;
+  librados::ObjectReadOperation op;
+  string oid;
+  ObjectDesc old_value;
+  int snap;
+
+  ceph::shared_ptr in_use;
+
+  bufferlist result;
+  int retval;
+
+  map attrs;
+  int attrretval;
+
+  set omap_requested_keys;
+  map omap_returned_values;
+  set omap_keys;
+  map omap;
+  bufferlist header;
+
+  map xattrs;
+  ReadOp(int n,
+	 RadosTestContext *context,
+	 const string &oid,
+	 TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      completion(NULL),
+      oid(oid),
+      snap(0),
+      retval(0),
+      attrretval(0)
+  {}
+		
+  void _begin()
+  {
+    context->state_lock.Lock();
+    if (!(rand() % 4) && !context->snaps.empty()) {
+      snap = rand_choose(context->snaps)->first;
+      in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+    } else {
+      snap = -1;
+    }
+    std::cout << num << ": read oid " << oid << " snap " << snap << std::endl;
+    done = 0;
+    completion = context->rados.aio_create_completion((void *) this, &read_callback, 0);
+
+    context->oid_in_use.insert(oid);
+    context->oid_not_in_use.erase(oid);
+    assert(context->find_object(oid, &old_value, snap));
+
+    TestWatchContext *ctx = context->get_watch_context(oid);
+    context->state_lock.Unlock();
+    if (ctx) {
+      assert(old_value.exists);
+      TestAlarm alarm;
+      std::cerr << num << ":  about to start" << std::endl;
+      ctx->start();
+      std::cerr << num << ":  started" << std::endl;
+      bufferlist bl;
+      context->io_ctx.set_notify_timeout(600);
+      int r = context->io_ctx.notify(context->prefix+oid, 0, bl);
+      if (r < 0) {
+	std::cerr << "r is " << r << std::endl;
+	assert(0);
+      }
+      std::cerr << num << ":  notified, waiting" << std::endl;
+      ctx->wait();
+    }
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(context->snaps[snap]);
+    }
+
+    op.read(0,
+	    !old_value.has_contents() ? 0 :
+	    old_value.most_recent_gen()->get_length(old_value.most_recent()),
+	    &result,
+	    &retval);
+
+    for (map::iterator i = old_value.attrs.begin();
+	 i != old_value.attrs.end();
+	 ++i) {
+      if (rand() % 2) {
+	string key = i->first;
+	if (rand() % 2)
+	  key.push_back((rand() % 26) + 'a');
+	omap_requested_keys.insert(key);
+      }
+    }
+    if (!context->no_omap) {
+      op.omap_get_vals_by_keys(omap_requested_keys, &omap_returned_values, 0);
+
+      op.omap_get_keys("", -1, &omap_keys, 0);
+      op.omap_get_vals("", -1, &omap, 0);
+      op.omap_get_header(&header, 0);
+    }
+    op.getxattrs(&xattrs, 0);
+    assert(!context->io_ctx.aio_operate(context->prefix+oid, completion, &op, 0));
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(0);
+    }
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    context->state_lock.Lock();
+    assert(!done);
+    context->oid_in_use.erase(oid);
+    context->oid_not_in_use.insert(oid);
+    assert(completion->is_complete());
+    uint64_t version = completion->get_version64();
+    if (int err = completion->get_return_value()) {
+      if (!(err == -ENOENT && old_value.deleted())) {
+	cerr << num << ": Error: oid " << oid << " read returned error code "
+	     << err << std::endl;
+	assert(0);
+      }
+    } else {
+      map::iterator iter = xattrs.find("_header");
+      bufferlist headerbl;
+      if (iter == xattrs.end()) {
+	if (old_value.has_contents()) {
+	  cerr << num << ": Error: did not find header attr, has_contents: "
+	       << old_value.has_contents()
+	       << std::endl;
+	  assert(!old_value.has_contents());
+	}
+      } else {
+	headerbl = iter->second;
+	xattrs.erase(iter);
+      }
+      cout << num << ":  expect " << old_value.most_recent() << std::endl;
+      assert(!old_value.deleted());
+      if (old_value.has_contents()) {
+	ContDesc to_check;
+	bufferlist::iterator p = headerbl.begin();
+	::decode(to_check, p);
+	if (to_check != old_value.most_recent()) {
+	  cerr << num << ": oid " << oid << " found incorrect object contents " << to_check
+	       << ", expected " << old_value.most_recent() << std::endl;
+	  context->errors++;
+	}
+	if (!old_value.check(result)) {
+	  cerr << num << ": oid " << oid << " contents " << to_check << " corrupt" << std::endl;
+	  context->errors++;
+	}
+	if (context->errors) assert(0);
+      }
+
+      // Attributes
+      if (!context->no_omap) {
+	if (!(old_value.header == header)) {
+	  cerr << num << ": oid " << oid << " header does not match, old size: "
+	       << old_value.header.length() << " new size " << header.length()
+	       << std::endl;
+	  assert(old_value.header == header);
+	}
+	if (omap.size() != old_value.attrs.size()) {
+	  cerr << num << ": oid " << oid << " omap.size() is " << omap.size()
+	       << " and old is " << old_value.attrs.size() << std::endl;
+	  assert(omap.size() == old_value.attrs.size());
+	}
+	if (omap_keys.size() != old_value.attrs.size()) {
+	  cerr << num << ": oid " << oid << " omap.size() is " << omap_keys.size()
+	       << " and old is " << old_value.attrs.size() << std::endl;
+	  assert(omap_keys.size() == old_value.attrs.size());
+	}
+      }
+      if (xattrs.size() != old_value.attrs.size()) {
+	cerr << num << ": oid " << oid << " xattrs.size() is " << xattrs.size()
+	     << " and old is " << old_value.attrs.size() << std::endl;
+	assert(xattrs.size() == old_value.attrs.size());
+      }
+      if (version != old_value.version) {
+	cerr << num << ": oid " << oid << " version is " << version
+	     << " and expected " << old_value.version << std::endl;
+	assert(version == old_value.version);
+      }
+      for (map::iterator iter = old_value.attrs.begin();
+	   iter != old_value.attrs.end();
+	   ++iter) {
+	bufferlist bl = context->attr_gen.gen_bl(
+	  iter->second);
+	if (!context->no_omap) {
+	  map::iterator omap_iter = omap.find(iter->first);
+	  assert(omap_iter != omap.end());
+	  assert(bl.length() == omap_iter->second.length());
+	  bufferlist::iterator k = bl.begin();
+	  for(bufferlist::iterator l = omap_iter->second.begin();
+	      !k.end() && !l.end();
+	      ++k, ++l) {
+	    assert(*l == *k);
+	  }
+	}
+	map::iterator xattr_iter = xattrs.find(iter->first);
+	assert(xattr_iter != xattrs.end());
+	assert(bl.length() == xattr_iter->second.length());
+	bufferlist::iterator k = bl.begin();
+	for (bufferlist::iterator j = xattr_iter->second.begin();
+	     !k.end() && !j.end();
+	     ++j, ++k) {
+	  assert(*j == *k);
+	}
+      }
+      if (!context->no_omap) {
+	for (set::iterator i = omap_requested_keys.begin();
+	     i != omap_requested_keys.end();
+	     ++i) {
+	  if (!omap_returned_values.count(*i))
+	    assert(!old_value.attrs.count(*i));
+	  if (!old_value.attrs.count(*i))
+	    assert(!omap_returned_values.count(*i));
+	}
+	for (map::iterator i = omap_returned_values.begin();
+	     i != omap_returned_values.end();
+	     ++i) {
+	  assert(omap_requested_keys.count(i->first));
+	  assert(omap.count(i->first));
+	  assert(old_value.attrs.count(i->first));
+	  assert(i->second == omap[i->first]);
+	}
+      }
+    }
+    context->kick();
+    done = true;
+    context->state_lock.Unlock();
+  }
+
+  bool finished()
+  {
+    return done && completion->is_complete();
+  }
+
+  string getType()
+  {
+    return "ReadOp";
+  }
+};
+
+class SnapCreateOp : public TestOp {
+public:
+  SnapCreateOp(int n,
+	       RadosTestContext *context,
+	       TestOpStat *stat = 0)
+    : TestOp(n, context, stat)
+  {}
+
+  void _begin()
+  {
+    uint64_t snap;
+    string snapname;
+
+    if (context->pool_snaps) {
+      stringstream ss;
+
+      ss << context->prefix << "snap" << ++context->snapname_num;
+      snapname = ss.str();
+
+      int ret = context->io_ctx.snap_create(snapname.c_str());
+      if (ret) {
+	cerr << "snap_create returned " << ret << std::endl;
+	assert(0);
+      }
+      assert(!context->io_ctx.snap_lookup(snapname.c_str(), &snap));
+
+    } else {
+      assert(!context->io_ctx.selfmanaged_snap_create(&snap));
+    }
+
+    context->state_lock.Lock();
+    context->add_snap(snap);
+
+    if (context->pool_snaps) {
+      context->state_lock.Unlock();
+    } else {
+      vector snapset(context->snaps.size());
+
+      int j = 0;
+      for (map::reverse_iterator i = context->snaps.rbegin();
+	   i != context->snaps.rend();
+	   ++i, ++j) {
+	snapset[j] = i->second;
+      }
+
+      context->state_lock.Unlock();
+
+      int r = context->io_ctx.selfmanaged_snap_set_write_ctx(context->seq, snapset);
+      if (r) {
+	cerr << "r is " << r << " snapset is " << snapset << " seq is " << context->seq << std::endl;
+	assert(0);
+      }
+    }
+  }
+
+  string getType()
+  {
+    return "SnapCreateOp";
+  }
+  bool must_quiesce_other_ops() { return context->pool_snaps; }
+};
+
+class SnapRemoveOp : public TestOp {
+public:
+  int to_remove;
+  SnapRemoveOp(int n, RadosTestContext *context,
+	       int snap,
+	       TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      to_remove(snap)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+    uint64_t snap = context->snaps[to_remove];
+    context->remove_snap(to_remove);
+    context->state_lock.Unlock();
+
+    if (context->pool_snaps) {
+      string snapname;
+
+      assert(!context->io_ctx.snap_get_name(snap, &snapname));
+      assert(!context->io_ctx.snap_remove(snapname.c_str()));
+     } else {
+      assert(!context->io_ctx.selfmanaged_snap_remove(snap));
+
+      vector snapset(context->snaps.size());
+      int j = 0;
+      for (map::reverse_iterator i = context->snaps.rbegin();
+	   i != context->snaps.rend();
+	   ++i, ++j) {
+	snapset[j] = i->second;
+      }
+
+      int r = context->io_ctx.selfmanaged_snap_set_write_ctx(context->seq, snapset);
+      if (r) {
+	cerr << "r is " << r << " snapset is " << snapset << " seq is " << context->seq << std::endl;
+	assert(0);
+      }
+    }
+  }
+
+  string getType()
+  {
+    return "SnapRemoveOp";
+  }
+};
+
+class WatchOp : public TestOp {
+  string oid;
+public:
+  WatchOp(int n,
+	  RadosTestContext *context,
+	  const string &_oid,
+	  TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      oid(_oid)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+    ObjectDesc contents;
+    context->find_object(oid, &contents);
+    if (contents.deleted()) {
+      context->kick();
+      context->state_lock.Unlock();
+      return;
+    }
+    context->oid_in_use.insert(oid);
+    context->oid_not_in_use.erase(oid);
+
+    TestWatchContext *ctx = context->get_watch_context(oid);
+    context->state_lock.Unlock();
+    int r;
+    if (!ctx) {
+      {
+	Mutex::Locker l(context->state_lock);
+	ctx = context->watch(oid);
+      }
+
+      r = context->io_ctx.watch(context->prefix+oid,
+				0,
+				&ctx->get_handle(),
+				ctx);
+    } else {
+      r = context->io_ctx.unwatch(context->prefix+oid,
+				  ctx->get_handle());
+      {
+	Mutex::Locker l(context->state_lock);
+	context->unwatch(oid);
+      }
+    }
+
+    if (r) {
+      cerr << "r is " << r << std::endl;
+      assert(0);
+    }
+
+    {
+      Mutex::Locker l(context->state_lock);
+      context->oid_in_use.erase(oid);
+      context->oid_not_in_use.insert(oid);
+    }
+  }
+
+  string getType()
+  {
+    return "WatchOp";
+  }
+};
+
+class RollbackOp : public TestOp {
+public:
+  string oid;
+  int roll_back_to;
+  bool done;
+  librados::ObjectWriteOperation op;
+  librados::AioCompletion *comp;
+  ceph::shared_ptr in_use;
+
+  RollbackOp(int n,
+	     RadosTestContext *context,
+	     const string &_oid,
+	     TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      oid(_oid), roll_back_to(-1), 
+      done(false), comp(NULL)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+    if (context->get_watch_context(oid)) {
+      context->kick();
+      context->state_lock.Unlock();
+      return;
+    }
+
+    if (context->snaps.empty()) {
+      context->kick();
+      context->state_lock.Unlock();
+      done = true;
+      return;
+    }
+
+    context->oid_in_use.insert(oid);
+    context->oid_not_in_use.erase(oid);
+
+    roll_back_to = rand_choose(context->snaps)->first;
+    in_use = context->snaps_in_use.lookup_or_create(
+      roll_back_to,
+      roll_back_to);
+
+
+    cout << "rollback oid " << oid << " to " << roll_back_to << std::endl;
+
+    context->roll_back(oid, roll_back_to);
+    uint64_t snap = context->snaps[roll_back_to];
+
+    context->state_lock.Unlock();
+
+    if (context->pool_snaps) {
+      op.snap_rollback(snap);
+    } else {
+      op.selfmanaged_snap_rollback(snap);
+    }
+
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+						&write_callback);
+    context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    Mutex::Locker l(context->state_lock);
+    int r;
+    if ((r = comp->get_return_value())) {
+      cerr << "err " << r << std::endl;
+      assert(0);
+    }
+    done = true;
+    context->update_object_version(oid, comp->get_version64());
+    context->oid_in_use.erase(oid);
+    context->oid_not_in_use.insert(oid);
+    in_use = ceph::shared_ptr();
+    context->kick();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "RollBackOp";
+  }
+};
+
+class CopyFromOp : public TestOp {
+public:
+  string oid, oid_src;
+  ObjectDesc src_value;
+  librados::ObjectWriteOperation op;
+  librados::ObjectReadOperation rd_op;
+  librados::AioCompletion *comp;
+  librados::AioCompletion *comp_racing_read;
+  ceph::shared_ptr in_use;
+  int snap;
+  int done;
+  uint64_t version;
+  int r;
+  CopyFromOp(int n,
+	     RadosTestContext *context,
+	     const string &oid,
+	     const string &oid_src,
+	     TestOpStat *stat)
+    : TestOp(n, context, stat),
+      oid(oid), oid_src(oid_src),
+      comp(NULL), snap(-1), done(0), 
+      version(0), r(0)
+  {}
+
+  void _begin()
+  {
+    ContDesc cont;
+    {
+      Mutex::Locker l(context->state_lock);
+      cont = ContDesc(context->seq_num, context->current_snap,
+		      context->seq_num, "");
+      context->oid_in_use.insert(oid);
+      context->oid_not_in_use.erase(oid);
+      context->oid_in_use.insert(oid_src);
+      context->oid_not_in_use.erase(oid_src);
+
+      // choose source snap
+      if (0 && !(rand() % 4) && !context->snaps.empty()) {
+	snap = rand_choose(context->snaps)->first;
+	in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+      } else {
+	snap = -1;
+      }
+      context->find_object(oid_src, &src_value, snap);
+      if (!src_value.deleted())
+	context->update_object_full(oid, src_value);
+    }
+
+    string src = context->prefix+oid_src;
+    op.copy_from(src.c_str(), context->io_ctx, src_value.version);
+
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    comp = context->rados.aio_create_completion((void*) cb_arg, NULL,
+						&write_callback);
+    context->io_ctx.aio_operate(context->prefix+oid, comp, &op);
+
+    // queue up a racing read, too.
+    pair *read_cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(1));
+    comp_racing_read = context->rados.aio_create_completion((void*) read_cb_arg, NULL, &write_callback);
+    rd_op.stat(NULL, NULL, NULL);
+    context->io_ctx.aio_operate(context->prefix+oid, comp_racing_read, &rd_op,
+				librados::OPERATION_ORDER_READS_WRITES,  // order wrt previous write/update
+				NULL);
+
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    Mutex::Locker l(context->state_lock);
+
+    // note that the read can (and atm will) come back before the
+    // write reply, but will reflect the update and the versions will
+    // match.
+
+    if (info->id == 0) {
+      // copy_from
+      assert(comp->is_complete());
+      cout << num << ":  finishing copy_from to " << context->prefix + oid << std::endl;
+      if ((r = comp->get_return_value())) {
+	if (r == -ENOENT && src_value.deleted()) {
+	  cout << num << ":  got expected ENOENT (src dne)" << std::endl;
+	} else {
+	  cerr << "Error: oid " << oid << " copy_from " << oid_src << " returned error code "
+	       << r << std::endl;
+	  assert(0);
+	}
+      } else {
+	assert(!version || comp->get_version64() == version);
+	version = comp->get_version64();
+	context->update_object_version(oid, comp->get_version64());
+      }
+    } else if (info->id == 1) {
+      // racing read
+      assert(comp_racing_read->is_complete());
+      cout << num << ":  finishing copy_from racing read to " << context->prefix + oid << std::endl;
+      if ((r = comp_racing_read->get_return_value())) {
+	if (!(r == -ENOENT && src_value.deleted())) {
+	  cerr << "Error: oid " << oid << " copy_from " << oid_src << " returned error code "
+	       << r << std::endl;
+	}
+      } else {
+	assert(comp_racing_read->get_return_value() == 0);
+	assert(!version || comp_racing_read->get_version64() == version);
+	version = comp_racing_read->get_version64();
+      }
+    }
+    if (++done == 2) {
+      context->oid_in_use.erase(oid);
+      context->oid_not_in_use.insert(oid);
+      context->oid_in_use.erase(oid_src);
+      context->oid_not_in_use.insert(oid_src);
+      context->kick();
+    }
+  }
+
+  bool finished()
+  {
+    return done == 2;
+  }
+
+  string getType()
+  {
+    return "CopyFromOp";
+  }
+};
+
+class HitSetListOp : public TestOp {
+  bool done;
+  librados::AioCompletion *comp1, *comp2;
+  uint32_t hash;
+  std::list< std::pair > ls;
+  bufferlist bl;
+
+public:
+  HitSetListOp(int n,
+	       RadosTestContext *context,
+	       uint32_t hash,
+	       TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      done(false), comp1(NULL), comp2(NULL),
+      hash(hash)
+  {}
+
+  void _begin()
+  {
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    comp1 = context->rados.aio_create_completion((void*) cb_arg, NULL,
+						 &write_callback);
+    int r = context->io_ctx.hit_set_list(hash, comp1, &ls);
+    assert(r == 0);
+  }
+
+  void _finish(CallbackInfo *info) {
+    Mutex::Locker l(context->state_lock);
+    if (!comp2) {
+      if (ls.empty()) {
+	cerr << num << ": no hitsets" << std::endl;
+	done = true;
+      } else {
+	cerr << num << ": hitsets are " << ls << std::endl;
+	int r = rand() % ls.size();
+	std::list >::iterator p = ls.begin();
+	while (r--)
+	  ++p;
+	pair *cb_arg =
+	  new pair(this,
+						   new TestOp::CallbackInfo(0));
+	comp2 = context->rados.aio_create_completion((void*) cb_arg, NULL,
+						     &write_callback);
+	r = context->io_ctx.hit_set_get(hash, comp2, p->second, &bl);
+	assert(r == 0);
+      }
+    } else {
+      int r = comp2->get_return_value();
+      if (r == 0) {
+	HitSet hitset;
+	bufferlist::iterator p = bl.begin();
+	::decode(hitset, p);
+	cout << num << ": got hitset of type " << hitset.get_type_name()
+	     << " size " << bl.length()
+	     << std::endl;
+      } else {
+	// FIXME: we could verify that we did in fact race with a trim...
+	assert(r == -ENOENT);
+      }
+      done = true;
+    }
+
+    context->kick();
+  }
+
+  bool finished() {
+    return done;
+  }
+
+  string getType() {
+    return "HitSetListOp";
+  }
+};
+
+class UndirtyOp : public TestOp {
+public:
+  librados::AioCompletion *completion;
+  librados::ObjectWriteOperation op;
+  string oid;
+
+  UndirtyOp(int n,
+	    RadosTestContext *context,
+	    const string &oid,
+	    TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      completion(NULL),
+      oid(oid)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+						      &write_callback);
+
+    context->oid_in_use.insert(oid);
+    context->oid_not_in_use.erase(oid);
+    context->update_object_undirty(oid);
+    context->state_lock.Unlock();
+
+    op.undirty();
+    int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+					&op, 0);
+    assert(!r);
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    context->state_lock.Lock();
+    assert(!done);
+    assert(completion->is_complete());
+    context->oid_in_use.erase(oid);
+    context->oid_not_in_use.insert(oid);
+    context->update_object_version(oid, completion->get_version64());
+    context->kick();
+    done = true;
+    context->state_lock.Unlock();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "UndirtyOp";
+  }
+};
+
+class IsDirtyOp : public TestOp {
+public:
+  librados::AioCompletion *completion;
+  librados::ObjectReadOperation op;
+  string oid;
+  bool dirty;
+  ObjectDesc old_value;
+  int snap;
+  ceph::shared_ptr in_use;
+
+  IsDirtyOp(int n,
+	    RadosTestContext *context,
+	    const string &oid,
+	    TestOpStat *stat = 0)
+    : TestOp(n, context, stat),
+      completion(NULL),
+      oid(oid),
+      dirty(false)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+
+    if (!(rand() % 4) && !context->snaps.empty()) {
+      snap = rand_choose(context->snaps)->first;
+      in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+    } else {
+      snap = -1;
+    }
+    std::cout << num << ": is_dirty oid " << oid << " snap " << snap
+	      << std::endl;
+
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+						      &write_callback);
+
+    context->oid_in_use.insert(oid);
+    context->oid_not_in_use.erase(oid);
+    context->state_lock.Unlock();
+
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(context->snaps[snap]);
+    }
+
+    op.is_dirty(&dirty, NULL);
+    int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+					&op, 0);
+    assert(!r);
+
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(0);
+    }
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    context->state_lock.Lock();
+    assert(!done);
+    assert(completion->is_complete());
+    context->oid_in_use.erase(oid);
+    context->oid_not_in_use.insert(oid);
+
+    assert(context->find_object(oid, &old_value, snap));
+
+    int r = completion->get_return_value();
+    if (r == 0) {
+      cout << num << ":  " << (dirty ? "dirty" : "clean") << std::endl;
+      assert(!old_value.deleted());
+      assert(dirty == old_value.dirty);
+    } else {
+      cout << num << ":  got " << r << std::endl;
+      assert(r == -ENOENT);
+      assert(old_value.deleted());
+    }
+    context->kick();
+    done = true;
+    context->state_lock.Unlock();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "IsDirtyOp";
+  }
+};
+
+
+
+class CacheFlushOp : public TestOp {
+public:
+  librados::AioCompletion *completion;
+  librados::ObjectReadOperation op;
+  string oid;
+  bool blocking;
+  int snap;
+  bool can_fail;
+  ceph::shared_ptr in_use;
+
+  CacheFlushOp(int n,
+	       RadosTestContext *context,
+	       const string &oid,
+	       TestOpStat *stat,
+	       bool b)
+    : TestOp(n, context, stat),
+      completion(NULL),
+      oid(oid),
+      blocking(b),
+      snap(0),
+      can_fail(false)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+
+    if (!(rand() % 4) && !context->snaps.empty()) {
+      snap = rand_choose(context->snaps)->first;
+      in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+    } else {
+      snap = -1;
+    }
+    // not being particularly specific here about knowing which
+    // flushes are on the oldest clean snap and which ones are not.
+    can_fail = !blocking || !context->snaps.empty();
+    // FIXME: we can could fail if we've ever removed a snap due to
+    // the async snap trimming.
+    can_fail = true;
+    cout << num << ": " << (blocking ? "cache_flush" : "cache_try_flush")
+	 << " oid " << oid << " snap " << snap << std::endl;
+
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(context->snaps[snap]);
+    }
+
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+						      &write_callback);
+    // leave object in unused list so that we race with other operations
+    //context->oid_in_use.insert(oid);
+    //context->oid_not_in_use.erase(oid);
+    context->oid_flushing.insert(oid);
+    context->oid_not_flushing.erase(oid);
+    context->state_lock.Unlock();
+
+    unsigned flags = librados::OPERATION_IGNORE_CACHE;
+    if (blocking) {
+      op.cache_flush();
+    } else {
+      op.cache_try_flush();
+      flags = librados::OPERATION_SKIPRWLOCKS;
+    }
+    int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+					&op, flags, NULL);
+    assert(!r);
+
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(0);
+    }
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    context->state_lock.Lock();
+    assert(!done);
+    assert(completion->is_complete());
+    //context->oid_in_use.erase(oid);
+    //context->oid_not_in_use.insert(oid);
+    context->oid_flushing.erase(oid);
+    context->oid_not_flushing.insert(oid);
+    int r = completion->get_return_value();
+    cout << num << ":  got " << cpp_strerror(r) << std::endl;
+    if (r == 0) {
+      context->update_object_version(oid, 0, snap);
+    } else if (r == -EBUSY) {
+      assert(can_fail);
+    } else if (r == -EINVAL) {
+      // caching not enabled?
+    } else if (r == -ENOENT) {
+      // may have raced with a remove?
+    } else {
+      assert(0 == "shouldn't happen");
+    }
+    context->kick();
+    done = true;
+    context->state_lock.Unlock();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "CacheFlushOp";
+  }
+};
+
+class CacheEvictOp : public TestOp {
+public:
+  librados::AioCompletion *completion;
+  librados::ObjectReadOperation op;
+  string oid;
+  ceph::shared_ptr in_use;
+
+  CacheEvictOp(int n,
+	       RadosTestContext *context,
+	       const string &oid,
+	       TestOpStat *stat)
+    : TestOp(n, context, stat),
+      completion(NULL),
+      oid(oid)
+  {}
+
+  void _begin()
+  {
+    context->state_lock.Lock();
+
+    int snap;
+    if (!(rand() % 4) && !context->snaps.empty()) {
+      snap = rand_choose(context->snaps)->first;
+      in_use = context->snaps_in_use.lookup_or_create(snap, snap);
+    } else {
+      snap = -1;
+    }
+    cout << num << ": cache_evict oid " << oid << " snap " << snap << std::endl;
+
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(context->snaps[snap]);
+    }
+
+    pair *cb_arg =
+      new pair(this,
+					       new TestOp::CallbackInfo(0));
+    completion = context->rados.aio_create_completion((void *) cb_arg, NULL,
+						      &write_callback);
+    // leave object in unused list so that we race with other operations
+    //context->oid_in_use.insert(oid);
+    //context->oid_not_in_use.erase(oid);
+    context->state_lock.Unlock();
+
+    op.cache_evict();
+    int r = context->io_ctx.aio_operate(context->prefix+oid, completion,
+					&op, librados::OPERATION_IGNORE_CACHE,
+					NULL);
+    assert(!r);
+
+    if (snap >= 0) {
+      context->io_ctx.snap_set_read(0);
+    }
+  }
+
+  void _finish(CallbackInfo *info)
+  {
+    context->state_lock.Lock();
+    assert(!done);
+    assert(completion->is_complete());
+    //context->oid_in_use.erase(oid);
+    //context->oid_not_in_use.insert(oid);
+    int r = completion->get_return_value();
+    cout << num << ":  got " << cpp_strerror(r) << std::endl;
+    if (r == 0) {
+      // yay!
+    } else if (r == -EBUSY) {
+      // raced with something that dirtied the object
+    } else if (r == -EINVAL) {
+      // caching not enabled?
+    } else if (r == -ENOENT) {
+      // may have raced with a remove?
+    } else {
+      assert(0 == "shouldn't happen");
+    }
+    context->kick();
+    done = true;
+    context->state_lock.Unlock();
+  }
+
+  bool finished()
+  {
+    return done;
+  }
+
+  string getType()
+  {
+    return "CacheEvictOp";
+  }
+};
+
+
+#endif
diff --git a/ceph/src/test/osd/TestECBackend.cc b/ceph/src/test/osd/TestECBackend.cc
new file mode 100644
index 00000000..affff369
--- /dev/null
+++ b/ceph/src/test/osd/TestECBackend.cc
@@ -0,0 +1,60 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include "osd/ECBackend.h"
+#include "gtest/gtest.h"
+
+TEST(ECUtil, stripe_info_t)
+{
+  const uint64_t swidth = 4096;
+  const uint64_t ssize = 4;
+
+  ECUtil::stripe_info_t s(ssize, swidth);
+  ASSERT_EQ(s.get_stripe_width(), swidth);
+
+  ASSERT_EQ(s.logical_to_next_chunk_offset(0), 0u);
+  ASSERT_EQ(s.logical_to_next_chunk_offset(1), s.get_chunk_size());
+  ASSERT_EQ(s.logical_to_next_chunk_offset(swidth - 1),
+	    s.get_chunk_size());
+
+  ASSERT_EQ(s.logical_to_prev_chunk_offset(0), 0u);
+  ASSERT_EQ(s.logical_to_prev_chunk_offset(swidth), s.get_chunk_size());
+  ASSERT_EQ(s.logical_to_prev_chunk_offset((swidth * 2) - 1),
+	    s.get_chunk_size());
+
+  ASSERT_EQ(s.logical_to_next_stripe_offset(0), 0u);
+  ASSERT_EQ(s.logical_to_next_stripe_offset(swidth - 1),
+	    s.get_stripe_width());
+
+  ASSERT_EQ(s.logical_to_prev_stripe_offset(swidth), s.get_stripe_width());
+  ASSERT_EQ(s.logical_to_prev_stripe_offset(swidth), s.get_stripe_width());
+  ASSERT_EQ(s.logical_to_prev_stripe_offset((swidth * 2) - 1),
+	    s.get_stripe_width());
+
+  ASSERT_EQ(s.aligned_logical_offset_to_chunk_offset(2*swidth),
+	    2*s.get_chunk_size());
+  ASSERT_EQ(s.aligned_chunk_offset_to_logical_offset(2*s.get_chunk_size()),
+	    2*s.get_stripe_width());
+
+  ASSERT_EQ(s.aligned_offset_len_to_chunk(make_pair(swidth, 10*swidth)),
+	    make_pair(s.get_chunk_size(), 10*s.get_chunk_size()));
+
+  ASSERT_EQ(s.offset_len_to_stripe_bounds(make_pair(swidth-10, (uint64_t)20)),
+            make_pair((uint64_t)0, 2*swidth));
+}
+
diff --git a/ceph/src/test/osd/TestOSDMap.cc b/ceph/src/test/osd/TestOSDMap.cc
new file mode 100644
index 00000000..451b6b22
--- /dev/null
+++ b/ceph/src/test/osd/TestOSDMap.cc
@@ -0,0 +1,401 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "gtest/gtest.h"
+#include "osd/OSDMap.h"
+
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/common_init.h"
+
+#include 
+
+using namespace std;
+
+int main(int argc, char **argv) {
+  std::vector preargs;
+  std::vector args(argv, argv+argc);
+  global_init(&preargs, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+              CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  // make sure we have 3 copies, or some tests won't work
+  g_ceph_context->_conf->set_val("osd_pool_default_size", "3", false);
+  // our map is flat, so just try and split across OSDs, not hosts or whatever
+  g_ceph_context->_conf->set_val("osd_crush_chooseleaf_type", "0", false);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+class OSDMapTest : public testing::Test {
+  const static int num_osds = 6;
+public:
+  OSDMap osdmap;
+  OSDMapTest() {}
+
+  void set_up_map() {
+    uuid_d fsid;
+    osdmap.build_simple(g_ceph_context, 0, fsid, num_osds, 6, 6);
+    OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+    pending_inc.fsid = osdmap.get_fsid();
+    entity_addr_t sample_addr;
+    uuid_d sample_uuid;
+    for (int i = 0; i < num_osds; ++i) {
+      sample_uuid.uuid[i] = i;
+      sample_addr.nonce = i;
+      pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW;
+      pending_inc.new_up_client[i] = sample_addr;
+      pending_inc.new_up_cluster[i] = sample_addr;
+      pending_inc.new_hb_back_up[i] = sample_addr;
+      pending_inc.new_hb_front_up[i] = sample_addr;
+      pending_inc.new_weight[i] = CEPH_OSD_IN;
+      pending_inc.new_uuid[i] = sample_uuid;
+    }
+    osdmap.apply_incremental(pending_inc);
+
+    // Create an EC ruleset and a pool using it
+    int r = osdmap.crush->add_simple_ruleset("erasure", "default", "osd",
+					     "indep", pg_pool_t::TYPE_ERASURE,
+					     &cerr);
+
+    OSDMap::Incremental new_pool_inc(osdmap.get_epoch() + 1);
+    new_pool_inc.new_pool_max = osdmap.get_pool_max();
+    new_pool_inc.fsid = osdmap.get_fsid();
+    pg_pool_t empty;
+    uint64_t pool_id = ++new_pool_inc.new_pool_max;
+    pg_pool_t *p = new_pool_inc.get_new_pool(pool_id, &empty);
+    p->size = 3;
+    p->set_pg_num(64);
+    p->set_pgp_num(64);
+    p->type = pg_pool_t::TYPE_ERASURE;
+    p->crush_ruleset = r;
+    new_pool_inc.new_pool_names[pool_id] = "ec";
+    osdmap.apply_incremental(new_pool_inc);
+  }
+  unsigned int get_num_osds() { return num_osds; }
+
+  void test_mappings(int pool,
+		     int num,
+		     vector *any,
+		     vector *first,
+		     vector *primary) {
+    for (int i=0; i o;
+      int p;
+      pg_t pgid(i, pool);
+      osdmap.pg_to_acting_osds(pgid, &o, &p);
+      for (unsigned j=0; j= 0)
+	(*primary)[p]++;
+    }
+  }
+};
+
+TEST_F(OSDMapTest, Create) {
+  set_up_map();
+  ASSERT_EQ(get_num_osds(), (unsigned)osdmap.get_max_osd());
+  ASSERT_EQ(get_num_osds(), osdmap.get_num_in_osds());
+}
+
+TEST_F(OSDMapTest, Features) {
+  // with EC pool
+  set_up_map();
+  uint64_t features = osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2);
+  ASSERT_FALSE(features & CEPH_FEATURE_CRUSH_TUNABLES3);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_V2);
+  ASSERT_TRUE(features & CEPH_FEATURE_OSD_ERASURE_CODES);
+  ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL);
+  ASSERT_FALSE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY);
+
+  // clients have a slightly different view
+  features = osdmap.get_features(CEPH_ENTITY_TYPE_CLIENT, NULL);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2);
+  ASSERT_FALSE(features & CEPH_FEATURE_CRUSH_TUNABLES3);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_V2);
+  ASSERT_FALSE(features & CEPH_FEATURE_OSD_ERASURE_CODES);  // dont' need this
+  ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL);
+  ASSERT_FALSE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY);
+
+  // remove teh EC pool, but leave the rule.  add primary affinity.
+  {
+    OSDMap::Incremental new_pool_inc(osdmap.get_epoch() + 1);
+    new_pool_inc.old_pools.insert(osdmap.lookup_pg_pool_name("ec"));
+    new_pool_inc.new_primary_affinity[0] = 0x8000;
+    osdmap.apply_incremental(new_pool_inc);
+  }
+
+  features = osdmap.get_features(CEPH_ENTITY_TYPE_MON, NULL);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2);
+  ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES3); // shared bit with primary affinity
+  ASSERT_FALSE(features & CEPH_FEATURE_CRUSH_V2);
+  ASSERT_FALSE(features & CEPH_FEATURE_OSD_ERASURE_CODES);
+  ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL);
+  ASSERT_TRUE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY);
+
+  // FIXME: test tiering feature bits
+}
+
+TEST_F(OSDMapTest, MapPG) {
+  set_up_map();
+
+  pg_t rawpg(0, 0, -1);
+  pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+  vector up_osds, acting_osds;
+  int up_primary, acting_primary;
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+
+  vector old_up_osds, old_acting_osds;
+  osdmap.pg_to_up_acting_osds(pgid, old_up_osds, old_acting_osds);
+  ASSERT_EQ(old_up_osds, up_osds);
+  ASSERT_EQ(old_acting_osds, acting_osds);
+
+  ASSERT_EQ(osdmap.get_pg_pool(0)->get_size(), up_osds.size());
+}
+
+TEST_F(OSDMapTest, MapFunctionsMatch) {
+  // TODO: make sure pg_to_up_acting_osds and pg_to_acting_osds match
+  set_up_map();
+
+  pg_t rawpg(0, 0, -1);
+  pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+  vector up_osds, acting_osds;
+  int up_primary, acting_primary;
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+
+  vector up_osds_two, acting_osds_two;
+
+  osdmap.pg_to_up_acting_osds(pgid, up_osds_two, acting_osds_two);
+
+  ASSERT_EQ(up_osds, up_osds_two);
+  ASSERT_EQ(acting_osds, acting_osds_two);
+
+  int acting_primary_two;
+  osdmap.pg_to_acting_osds(pgid, &acting_osds_two, &acting_primary_two);
+  EXPECT_EQ(acting_osds, acting_osds_two);
+  EXPECT_EQ(acting_primary, acting_primary_two);
+  osdmap.pg_to_acting_osds(pgid, acting_osds_two);
+  EXPECT_EQ(acting_osds, acting_osds_two);
+}
+
+/** This test must be removed or modified appropriately when we allow
+ * other ways to specify a primary. */
+TEST_F(OSDMapTest, PrimaryIsFirst) {
+  set_up_map();
+
+  pg_t rawpg(0, 0, -1);
+  pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+  vector up_osds, acting_osds;
+  int up_primary, acting_primary;
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+  EXPECT_EQ(up_osds[0], up_primary);
+  EXPECT_EQ(acting_osds[0], acting_primary);
+}
+
+TEST_F(OSDMapTest, PGTempRespected) {
+  set_up_map();
+
+  pg_t rawpg(0, 0, -1);
+  pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+  vector up_osds, acting_osds;
+  int up_primary, acting_primary;
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+
+  // copy and swap first and last element in acting_osds
+  vector new_acting_osds(acting_osds);
+  int first = new_acting_osds[0];
+  new_acting_osds[0] = *new_acting_osds.rbegin();
+  *new_acting_osds.rbegin() = first;
+
+  // apply pg_temp to osdmap
+  OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+  pgtemp_map.new_pg_temp[pgid] = new_acting_osds;
+  osdmap.apply_incremental(pgtemp_map);
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+  EXPECT_EQ(new_acting_osds, acting_osds);
+}
+
+TEST_F(OSDMapTest, PrimaryTempRespected) {
+  set_up_map();
+
+  pg_t rawpg(0, 0, -1);
+  pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+  vector up_osds, acting_osds;
+  int up_primary, acting_primary;
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+
+  // make second OSD primary via incremental
+  OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+  pgtemp_map.new_primary_temp[pgid] = acting_osds[1];
+  osdmap.apply_incremental(pgtemp_map);
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+  EXPECT_EQ(acting_primary, acting_osds[1]);
+}
+
+TEST_F(OSDMapTest, RemovesRedundantTemps) {
+  set_up_map();
+
+  pg_t rawpg(0, 0, -1);
+  pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+  vector up_osds, acting_osds;
+  int up_primary, acting_primary;
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+
+  // stick calculated values in to temps
+  OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+  pgtemp_map.new_pg_temp[pgid] = up_osds;
+  pgtemp_map.new_primary_temp[pgid] = up_primary;
+  osdmap.apply_incremental(pgtemp_map);
+
+  OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+  OSDMap::remove_redundant_temporaries(g_ceph_context, osdmap, &pending_inc);
+
+  EXPECT_TRUE(pending_inc.new_pg_temp.count(pgid) &&
+	      pending_inc.new_pg_temp[pgid].size() == 0);
+  EXPECT_EQ(-1, pending_inc.new_primary_temp[pgid]);
+}
+
+TEST_F(OSDMapTest, KeepsNecessaryTemps) {
+  set_up_map();
+
+  pg_t rawpg(0, 0, -1);
+  pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+  vector up_osds, acting_osds;
+  int up_primary, acting_primary;
+
+  osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
+                              &acting_osds, &acting_primary);
+
+  // find unused OSD and stick it in there
+  OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
+  // find an unused osd and put it in place of the first one
+  int i = 0;
+  for(; i != (int)get_num_osds(); ++i) {
+    bool in_use = false;
+    for (vector::iterator osd_it = up_osds.begin();
+	 osd_it != up_osds.end();
+	 ++osd_it) {
+      if (i == *osd_it) {
+	in_use = true;
+        break;
+      }
+    }
+    if (!in_use) {
+      up_osds[1] = i;
+      break;
+    }
+  }
+  if (i == (int)get_num_osds())
+    ASSERT_EQ(0, "did not find unused OSD for temp mapping");
+
+  pgtemp_map.new_pg_temp[pgid] = up_osds;
+  pgtemp_map.new_primary_temp[pgid] = up_osds[1];
+  osdmap.apply_incremental(pgtemp_map);
+
+  OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+
+  OSDMap::remove_redundant_temporaries(g_ceph_context, osdmap, &pending_inc);
+  EXPECT_FALSE(pending_inc.new_pg_temp.count(pgid));
+  EXPECT_FALSE(pending_inc.new_primary_temp.count(pgid));
+}
+
+TEST_F(OSDMapTest, PrimaryAffinity) {
+  set_up_map();
+
+  /*
+  osdmap.print(cout);
+  Formatter *f = new_formatter("json-pretty");
+  f->open_object_section("CRUSH");
+  osdmap.crush->dump(f);
+  f->close_section();
+  f->flush(cout);
+  delete f;
+  */
+
+  int n = get_num_osds();
+  for (map::const_iterator p = osdmap.get_pools().begin();
+       p != osdmap.get_pools().end();
+       ++p) {
+    int pool = p->first;
+    cout << "pool " << pool << std::endl;
+    {
+      vector any(n, 0);
+      vector first(n, 0);
+      vector primary(n, 0);
+      test_mappings(0, 10000, &any, &first, &primary);
+      for (int i=0; i any(n, 0);
+      vector first(n, 0);
+      vector primary(n, 0);
+      test_mappings(pool, 10000, &any, &first, &primary);
+      for (int i=0; i= 2) {
+	  ASSERT_LT(0, first[i]);
+	  ASSERT_LT(0, primary[i]);
+	} else {
+	  if (p->second.is_replicated())
+	    ASSERT_EQ(0, first[i]);
+	  ASSERT_EQ(0, primary[i]);
+	}
+      }
+    }
+
+    osdmap.set_primary_affinity(0, 0x8000);
+    osdmap.set_primary_affinity(1, 0);
+    {
+      vector any(n, 0);
+      vector first(n, 0);
+      vector primary(n, 0);
+      test_mappings(pool, 10000, &any, &first, &primary);
+      for (int i=0; i= 2) {
+	  ASSERT_LT(0, first[i]);
+	  ASSERT_LT(0, primary[i]);
+	} else if (i == 1) {
+	  if (p->second.is_replicated())
+	    ASSERT_EQ(0, first[i]);
+	  ASSERT_EQ(0, primary[i]);
+	} else {
+	  ASSERT_LT(10000/6/4, primary[0]);
+	  ASSERT_GT(10000/6/4*3, primary[0]);
+	}
+      }
+    }
+
+    osdmap.set_primary_affinity(0, 0x10000);
+    osdmap.set_primary_affinity(1, 0x10000);
+  }
+}
diff --git a/ceph/src/test/osd/TestOpStat.cc b/ceph/src/test/osd/TestOpStat.cc
new file mode 100644
index 00000000..db6a82e2
--- /dev/null
+++ b/ceph/src/test/osd/TestOpStat.cc
@@ -0,0 +1,61 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+#include "include/interval_set.h"
+#include "include/buffer.h"
+#include 
+#include 
+#include 
+#include "RadosModel.h"
+#include "TestOpStat.h"
+
+void TestOpStat::begin(TestOp *in) {
+  stat_lock.Lock();
+  stats[in->getType()].begin(in);
+  stat_lock.Unlock();
+}
+
+void TestOpStat::end(TestOp *in) {
+  stat_lock.Lock();
+  stats[in->getType()].end(in);
+  stat_lock.Unlock();
+}
+
+void TestOpStat::TypeStatus::export_latencies(map &in) const
+{
+  map::iterator i = in.begin();
+  multiset::iterator j = latencies.begin();
+  int count = 0;
+  while (j != latencies.end() && i != in.end()) {
+    count++;
+    if ((((double)count)/((double)latencies.size())) * 100 >= i->first) {
+      i->second = *j;
+      ++i;
+    }
+    ++j;
+  }
+}
+  
+std::ostream & operator<<(std::ostream &out, TestOpStat &rhs)
+{
+  rhs.stat_lock.Lock();
+  for (map::iterator i = rhs.stats.begin();
+       i != rhs.stats.end();
+       ++i) {
+    map latency;
+    latency[10] = 0;
+    latency[50] = 0;
+    latency[90] = 0;
+    latency[99] = 0;
+    i->second.export_latencies(latency);
+    
+    out << i->first << " latency: " << std::endl;
+    for (map::iterator j = latency.begin();
+	 j != latency.end();
+	 ++j) {
+      if (j->second == 0) break;
+      out << "\t" << j->first << "th percentile: " 
+	  << j->second / 1000 << "ms" << std::endl;
+    }
+  }
+  rhs.stat_lock.Unlock();
+  return out;
+}
diff --git a/ceph/src/test/osd/TestOpStat.h b/ceph/src/test/osd/TestOpStat.h
new file mode 100644
index 00000000..3d70ece7
--- /dev/null
+++ b/ceph/src/test/osd/TestOpStat.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "include/rados/librados.hpp"
+
+#ifndef TESTOPSTAT_H
+#define TESTOPSTAT_H
+
+class TestOp;
+
+class TestOpStat {
+public:
+  Mutex stat_lock;
+
+  TestOpStat() : stat_lock("TestOpStat lock") {}
+    
+  static uint64_t gettime()
+  {
+    timeval t;
+    gettimeofday(&t,0);
+    return (1000000*t.tv_sec) + t.tv_usec;
+  }
+
+  class TypeStatus {
+  public:
+    map inflight;
+    multiset latencies;
+    void begin(TestOp *in)
+    {
+      assert(!inflight.count(in));
+      inflight[in] = gettime();
+    }
+
+    void end(TestOp *in)
+    {
+      assert(inflight.count(in));
+      uint64_t curtime = gettime();
+      latencies.insert(curtime - inflight[in]);
+      inflight.erase(in);
+    }
+
+    void export_latencies(map &in) const;
+  };
+  map stats;
+
+  void begin(TestOp *in);
+  void end(TestOp *in);
+  friend std::ostream & operator<<(std::ostream &, TestOpStat&);
+};
+
+std::ostream & operator<<(std::ostream &out, TestOpStat &rhs);
+
+#endif
diff --git a/ceph/src/test/osd/TestPGLog.cc b/ceph/src/test/osd/TestPGLog.cc
new file mode 100644
index 00000000..c2063b89
--- /dev/null
+++ b/ceph/src/test/osd/TestPGLog.cc
@@ -0,0 +1,1878 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include 
+#include 
+#include "osd/PGLog.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include 
+
+class PGLogTest : public ::testing::Test, protected PGLog {
+public:
+  virtual void SetUp() { }
+
+  virtual void TearDown() {
+    clear();
+  }
+
+  static hobject_t mk_obj(unsigned id) {
+    hobject_t hoid;
+    stringstream ss;
+    ss << "obj_" << id;
+    hoid.oid = ss.str();
+    hoid.hash = id;
+    return hoid;
+  }
+  static eversion_t mk_evt(unsigned ep, unsigned v) {
+    return eversion_t(ep, v);
+  }
+  static pg_log_entry_t mk_ple_mod(
+    const hobject_t &hoid, eversion_t v, eversion_t pv) {
+    pg_log_entry_t e;
+    e.mod_desc.mark_unrollbackable();
+    e.op = pg_log_entry_t::MODIFY;
+    e.soid = hoid;
+    e.version = v;
+    e.prior_version = pv;
+    return e;
+  }
+  static pg_log_entry_t mk_ple_dt(
+    const hobject_t &hoid, eversion_t v, eversion_t pv) {
+    pg_log_entry_t e;
+    e.mod_desc.mark_unrollbackable();
+    e.op = pg_log_entry_t::DELETE;
+    e.soid = hoid;
+    e.version = v;
+    e.prior_version = pv;
+    return e;
+  }
+  static pg_log_entry_t mk_ple_mod_rb(
+    const hobject_t &hoid, eversion_t v, eversion_t pv) {
+    pg_log_entry_t e;
+    e.op = pg_log_entry_t::MODIFY;
+    e.soid = hoid;
+    e.version = v;
+    e.prior_version = pv;
+    return e;
+  }
+  static pg_log_entry_t mk_ple_dt_rb(
+    const hobject_t &hoid, eversion_t v, eversion_t pv) {
+    pg_log_entry_t e;
+    e.op = pg_log_entry_t::DELETE;
+    e.soid = hoid;
+    e.version = v;
+    e.prior_version = pv;
+    return e;
+  }
+
+  struct TestCase {
+    list base;
+    list auth;
+    list div;
+
+    pg_missing_t init;
+    pg_missing_t final;
+
+    set toremove;
+    list torollback;
+
+  private:
+    IndexedLog fullauth;
+    IndexedLog fulldiv;
+    pg_info_t authinfo;
+    pg_info_t divinfo;
+  public:
+    void setup() {
+      fullauth.log.insert(fullauth.log.end(), base.begin(), base.end());
+      fullauth.log.insert(fullauth.log.end(), auth.begin(), auth.end());
+      fulldiv.log.insert(fulldiv.log.end(), base.begin(), base.end());
+      fulldiv.log.insert(fulldiv.log.end(), div.begin(), div.end());
+
+      fullauth.head = authinfo.last_update = fullauth.log.rbegin()->version;
+      authinfo.last_complete = fullauth.log.rbegin()->version;
+      authinfo.log_tail = fullauth.log.begin()->version;
+      authinfo.log_tail.version--;
+      fullauth.tail = authinfo.log_tail;
+      authinfo.last_backfill = hobject_t::get_max();
+
+      fulldiv.head = divinfo.last_update = fulldiv.log.rbegin()->version;
+      divinfo.last_complete = eversion_t();
+      divinfo.log_tail = fulldiv.log.begin()->version;
+      divinfo.log_tail.version--;
+      fulldiv.tail = divinfo.log_tail;
+      divinfo.last_backfill = hobject_t::get_max();
+
+      if (init.missing.empty()) {
+	divinfo.last_complete = divinfo.last_update;
+      } else {
+	eversion_t fmissing = init.missing[init.rmissing.begin()->second].need;
+	for (list::const_iterator i = fulldiv.log.begin();
+	     i != fulldiv.log.end();
+	     ++i) {
+	  if (i->version < fmissing)
+	    divinfo.last_complete = i->version;
+	  else
+	    break;
+	}
+      }
+
+      fullauth.index();
+      fulldiv.index();
+    }
+    const IndexedLog &get_fullauth() const { return fullauth; }
+    const IndexedLog &get_fulldiv() const { return fulldiv; }
+    const pg_info_t &get_authinfo() const { return authinfo; }
+    const pg_info_t &get_divinfo() const { return divinfo; }
+  };
+
+  struct LogHandler : public PGLog::LogEntryHandler {
+    set removed;
+    list rolledback;
+    
+    void rollback(
+      const pg_log_entry_t &entry) {
+      rolledback.push_back(entry);
+    }
+    void remove(
+      const hobject_t &hoid) {
+      removed.insert(hoid);
+    }
+    void trim(
+      const pg_log_entry_t &entry) {}
+  };
+
+  void verify_missing(
+    const TestCase &tcase,
+    const pg_missing_t &missing) {
+    ASSERT_EQ(tcase.final.missing.size(), missing.missing.size());
+    for (map::const_iterator i =
+	   missing.missing.begin();
+	 i != missing.missing.end();
+	 ++i) {
+      EXPECT_TRUE(tcase.final.missing.count(i->first));
+      EXPECT_EQ(tcase.final.missing.find(i->first)->second.need, i->second.need);
+      EXPECT_EQ(tcase.final.missing.find(i->first)->second.have, i->second.have);
+    }
+  }
+
+  void verify_sideeffects(
+    const TestCase &tcase,
+    const LogHandler &handler) {
+    ASSERT_EQ(tcase.toremove.size(), handler.removed.size());
+    ASSERT_EQ(tcase.torollback.size(), handler.rolledback.size());
+
+    {
+      list::const_iterator titer = tcase.torollback.begin();
+      list::const_iterator hiter = handler.rolledback.begin();
+      for (; titer != tcase.torollback.end(); ++titer, ++hiter) {
+	EXPECT_EQ(titer->version, hiter->version);
+      }
+    }
+
+    {
+      set::const_iterator titer = tcase.toremove.begin();
+      set::const_iterator hiter = handler.removed.begin();
+      for (; titer != tcase.toremove.end(); ++titer, ++hiter) {
+	EXPECT_EQ(*titer, *hiter);
+      }
+    }
+  }
+
+  void test_merge_log(const TestCase &tcase) {
+    clear();
+    ObjectStore::Transaction t;
+    log = tcase.get_fulldiv();
+    pg_info_t info = tcase.get_divinfo();
+
+    missing = tcase.init;
+
+    IndexedLog olog;
+    olog = tcase.get_fullauth();
+    pg_info_t oinfo = tcase.get_authinfo();
+
+    LogHandler h;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+    merge_log(
+      t, oinfo, olog, pg_shard_t(1, 0), info,
+      &h, dirty_info, dirty_big_info);
+
+    ASSERT_EQ(info.last_update, oinfo.last_update);
+    verify_missing(tcase, missing);
+    verify_sideeffects(tcase, h);
+  };
+  void test_proc_replica_log(const TestCase &tcase) {
+    clear();
+    ObjectStore::Transaction t;
+    log = tcase.get_fullauth();
+    pg_info_t info = tcase.get_authinfo();
+
+    pg_missing_t omissing = tcase.init;
+
+    IndexedLog olog;
+    olog = tcase.get_fulldiv();
+    pg_info_t oinfo = tcase.get_divinfo();
+
+    proc_replica_log(
+      t, oinfo, olog, omissing, pg_shard_t(1, 0));
+
+    if (!tcase.base.empty()) {
+      ASSERT_EQ(tcase.base.rbegin()->version, oinfo.last_update);
+    }
+
+    for (list::const_iterator i = tcase.auth.begin();
+	 i != tcase.auth.end();
+	 ++i) {
+      omissing.add_next_event(*i);
+    }
+    verify_missing(tcase, omissing);
+  }
+  void run_test_case(const TestCase &tcase) {
+    test_merge_log(tcase);
+    test_proc_replica_log(tcase);
+  }
+};
+
+struct TestHandler : public PGLog::LogEntryHandler {
+  list &removed;
+  TestHandler(list &removed) : removed(removed) {}
+
+  void rollback(
+    const pg_log_entry_t &entry) {}
+  void remove(
+    const hobject_t &hoid) {
+    removed.push_back(hoid);
+  }
+  void cant_rollback(const pg_log_entry_t &entry) {}
+  void trim(
+    const pg_log_entry_t &entry) {}
+};
+
+TEST_F(PGLogTest, rewind_divergent_log) {
+  // newhead > log.tail : throw an assert
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    log.tail = eversion_t(2, 1);
+    TestHandler h(remove_snap);
+    EXPECT_THROW(rewind_divergent_log(t, eversion_t(1, 1), info, &h,
+				      dirty_info, dirty_big_info),
+		 FailedAssertion);
+  }
+
+  /*        +----------------+
+            |  log           |
+            +--------+-------+
+            |        |object |
+            |version | hash  |
+            |        |       |
+       tail > (1,1)  |  x5   |
+            |        |       |
+            |        |       |
+            | (1,4)  |  x9   < newhead
+            | MODIFY |       |
+            |        |       |
+       head > (1,5)  |  x9   |
+            | DELETE |       |
+            |        |       |
+            +--------+-------+
+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    hobject_t divergent_object;
+    eversion_t divergent_version;
+    eversion_t newhead;
+
+    hobject_t divergent;
+    divergent.hash = 0x9;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = newhead = eversion_t(1, 4);
+      e.soid = divergent;
+      e.op = pg_log_entry_t::MODIFY;
+      log.log.push_back(e);
+      e.version = divergent_version = eversion_t(1, 5);
+      e.prior_version = eversion_t(1, 4);
+      e.soid = divergent;
+      divergent_object = e.soid;
+      e.op = pg_log_entry_t::DELETE;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      info.last_update = log.head;
+      info.last_complete = log.head;
+    }
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(3U, log.log.size());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_EQ(log.head, info.last_update);
+    EXPECT_EQ(log.head, info.last_complete);
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+
+    TestHandler h(remove_snap);
+    rewind_divergent_log(t, newhead, info, &h,
+			 dirty_info, dirty_big_info);
+
+    EXPECT_TRUE(log.objects.count(divergent));
+    EXPECT_TRUE(missing.is_missing(divergent_object));
+    EXPECT_EQ(1U, log.objects.count(divergent_object));
+    EXPECT_EQ(2U, log.log.size());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_EQ(newhead, info.last_update);
+    EXPECT_EQ(newhead, info.last_complete);
+    EXPECT_TRUE(is_dirty());
+    EXPECT_TRUE(dirty_info);
+    EXPECT_TRUE(dirty_big_info);
+  }
+
+  /*        +----------------+
+            |  log           |
+            +--------+-------+
+            |        |object |
+            |version | hash  |
+            |        |       |
+       tail > (1,1)  | NULL  |
+            |        |       |
+            | (1,4)  | NULL  < newhead
+            |        |       |
+       head > (1,5)  |  x9   |
+            |        |       |
+            +--------+-------+
+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    hobject_t divergent_object;
+    eversion_t divergent_version;
+    eversion_t prior_version;
+    eversion_t newhead;
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      info.log_tail = log.tail = eversion_t(1, 1);
+      newhead = eversion_t(1, 3);
+      e.version = divergent_version = eversion_t(1, 5);
+      e.soid.hash = 0x9;
+      divergent_object = e.soid;
+      e.op = pg_log_entry_t::DELETE;
+      e.prior_version = prior_version = eversion_t(0, 2);
+      log.log.push_back(e);
+      log.head = e.version;
+    }
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(1U, log.log.size());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+
+    TestHandler h(remove_snap);
+    rewind_divergent_log(t, newhead, info, &h,
+			 dirty_info, dirty_big_info);
+
+    EXPECT_TRUE(missing.is_missing(divergent_object));
+    EXPECT_EQ(0U, log.objects.count(divergent_object));
+    EXPECT_TRUE(log.empty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(is_dirty());
+    EXPECT_TRUE(dirty_info);
+    EXPECT_TRUE(dirty_big_info);
+  }
+}
+
+TEST_F(PGLogTest, merge_old_entry) {
+  // entries > last_backfill are silently ignored
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_entry_t oe;
+    oe.mod_desc.mark_unrollbackable();
+    pg_info_t info;
+    list remove_snap;
+
+    info.last_backfill = hobject_t();
+    info.last_backfill.hash = 1;
+    oe.soid.hash = 2;
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_TRUE(log.empty());
+
+    TestHandler h(remove_snap);
+    merge_old_entry(t, oe, info, &h);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_TRUE(log.empty());
+  }
+
+  // the new entry (from the logs) has a version that is higher than
+  // the old entry (from the log entry given in argument) : do
+  // nothing and return false
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_info_t info;
+    list remove_snap;
+
+    pg_log_entry_t ne;
+    ne.mod_desc.mark_unrollbackable();
+    ne.version = eversion_t(2,1);
+    log.add(ne);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(1U, log.log.size());
+    EXPECT_EQ(ne.version, log.log.front().version);
+
+    // the newer entry ( from the logs ) can be DELETE
+    {
+      log.log.front().op = pg_log_entry_t::DELETE;
+      pg_log_entry_t oe;
+      oe.mod_desc.mark_unrollbackable();
+      oe.version = eversion_t(1,1);
+
+      TestHandler h(remove_snap);
+      merge_old_entry(t, oe, info, &h);
+    }
+
+    // if the newer entry is not DELETE, the object must be in missing
+    {
+      pg_log_entry_t &ne = log.log.front();
+      ne.op = pg_log_entry_t::MODIFY;
+      missing.add_next_event(ne);
+      pg_log_entry_t oe;
+      oe.mod_desc.mark_unrollbackable();
+      oe.version = eversion_t(1,1);
+
+      TestHandler h(remove_snap);
+      merge_old_entry(t, oe, info, &h);
+
+      missing.rm(ne.soid, ne.version);
+    }
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(1U, log.log.size());
+    EXPECT_EQ(ne.version, log.log.front().version);
+
+  }
+
+  // the new entry (from the logs) has a version that is lower than
+  // the old entry (from the log entry given in argument) and
+  // old and new are delete : do nothing and return false
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_entry_t oe;
+    oe.mod_desc.mark_unrollbackable();
+    pg_info_t info;
+    list remove_snap;
+
+    pg_log_entry_t ne;
+    ne.mod_desc.mark_unrollbackable();
+    ne.version = eversion_t(1,1);
+    ne.op = pg_log_entry_t::DELETE;
+    log.add(ne);
+
+    oe.version = eversion_t(2,1);
+    oe.op = pg_log_entry_t::DELETE;
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(1U, log.log.size());
+
+    TestHandler h(remove_snap);
+    merge_old_entry(t, oe, info, &h);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(1U, log.log.size());
+  }
+
+  // the new entry (from the logs) has a version that is lower than
+  // the old entry (from the log entry given in argument) and
+  // old is update and new is DELETE :
+  // if the object is in missing, it is removed
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_entry_t oe;
+    oe.mod_desc.mark_unrollbackable();
+    pg_info_t info;
+    list remove_snap;
+
+    pg_log_entry_t ne;
+    ne.mod_desc.mark_unrollbackable();
+    ne.version = eversion_t(1,1);
+    ne.op = pg_log_entry_t::DELETE;
+    log.add(ne);
+
+    oe.version = eversion_t(2,1);
+    oe.op = pg_log_entry_t::MODIFY;
+    missing.add_next_event(oe);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(missing.is_missing(oe.soid));
+    EXPECT_EQ(1U, log.log.size());
+
+    TestHandler h(remove_snap);
+    merge_old_entry(t, oe, info, &h);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.size() > 0);
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(1U, log.log.size());
+  }
+
+  // there is no new entry (from the logs) and
+  // the old entry (from the log entry given in argument) is not a CLONE and
+  // the old entry prior_version is greater than the tail of the log :
+  // do nothing and return false
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_entry_t oe;
+    oe.mod_desc.mark_unrollbackable();
+    pg_info_t info;
+    list remove_snap;
+
+    info.log_tail = eversion_t(1,1);
+    oe.op = pg_log_entry_t::MODIFY;
+    oe.prior_version = eversion_t(2,1);
+    missing_add(oe.soid, oe.prior_version, eversion_t());
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(log.empty());
+
+    TestHandler h(remove_snap);
+    merge_old_entry(t, oe, info, &h);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(log.empty());
+  }
+
+  // there is no new entry (from the logs) and
+  // the old entry (from the log entry given in argument) is not a CLONE and
+  // the old entry (from the log entry given in argument) is not a DELETE and
+  // the old entry prior_version is lower than the tail of the log :
+  //   add the old object to the remove_snap list and 
+  //   add the old object to divergent priors and
+  //   add or update the prior_version of the object to missing and
+  //   return false
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_entry_t oe;
+    oe.mod_desc.mark_unrollbackable();
+    pg_info_t info;
+    list remove_snap;
+
+    info.log_tail = eversion_t(2,1);
+    oe.soid.hash = 1;
+    oe.op = pg_log_entry_t::MODIFY;
+    oe.prior_version = eversion_t(1,1);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_TRUE(log.empty());
+
+    TestHandler h(remove_snap);
+    merge_old_entry(t, oe, info, &h);
+
+    EXPECT_TRUE(is_dirty());
+    EXPECT_EQ(oe.soid, remove_snap.front());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(missing.is_missing(oe.soid));
+    EXPECT_TRUE(log.empty());
+    EXPECT_EQ(oe.soid, divergent_priors[oe.prior_version]);
+  }
+
+  // there is no new entry (from the logs) and
+  // the old entry (from the log entry given in argument) is not a CLONE and
+  // the old entry (from the log entry given in argument) is a DELETE and
+  // the old entry prior_version is lower than the tail of the log :
+  //   add the old object to divergent priors and
+  //   add or update the prior_version of the object to missing and
+  //   return false
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_entry_t oe;
+    oe.mod_desc.mark_unrollbackable();
+    pg_info_t info;
+    list remove_snap;
+
+    info.log_tail = eversion_t(2,1);
+    oe.soid.hash = 1;
+    oe.op = pg_log_entry_t::DELETE;
+    oe.prior_version = eversion_t(1,1);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_TRUE(log.empty());
+
+    TestHandler h(remove_snap);
+    merge_old_entry(t, oe, info, &h);
+
+    EXPECT_TRUE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(missing.is_missing(oe.soid));
+    EXPECT_TRUE(log.empty());
+    EXPECT_EQ(oe.soid, divergent_priors[oe.prior_version]);
+  }
+
+
+  // there is no new entry (from the logs) and
+  // the old entry (from the log entry given in argument) is not a CLONE and
+  // the old entry (from the log entry given in argument) is not a DELETE and
+  // the old entry prior_version is eversion_t() :
+  //   add the old object to the remove_snap list and
+  //   remove the prior_version of the object from missing, if any and
+  //   return false
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_entry_t oe;
+    oe.mod_desc.mark_unrollbackable();
+    pg_info_t info;
+    list remove_snap;
+
+    info.log_tail = eversion_t(10,1);
+    oe.soid.hash = 1;
+    oe.op = pg_log_entry_t::MODIFY;
+    oe.prior_version = eversion_t();
+
+    missing.add(oe.soid, eversion_t(1,1), eversion_t());
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(missing.is_missing(oe.soid));
+    EXPECT_TRUE(log.empty());
+
+    TestHandler h(remove_snap);
+    merge_old_entry(t, oe, info, &h);
+
+    EXPECT_FALSE(is_dirty());
+    EXPECT_EQ(oe.soid, remove_snap.front());
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_TRUE(log.empty());
+  }
+
+}
+
+TEST_F(PGLogTest, merge_log) {
+  // head and tail match, last_backfill is set:
+  // noop
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_shard_t fromosd;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    hobject_t last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
+    info.last_backfill = last_backfill;
+    eversion_t stat_version(10, 1);
+    info.stats.version = stat_version;
+    log.tail = olog.tail = eversion_t(1, 1);
+    log.head = olog.head = eversion_t(2, 1);
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(0U, log.log.size());
+    EXPECT_EQ(stat_version, info.stats.version);
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_EQ(last_backfill, info.last_backfill);
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+
+    TestHandler h(remove_snap);
+    merge_log(t, oinfo, olog, fromosd, info, &h,
+              dirty_info, dirty_big_info);
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(0U, log.log.size());
+    EXPECT_EQ(stat_version, info.stats.version);
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+  }
+
+  // head and tail match, last_backfill is not set: info.stats is
+  // copied from oinfo.stats but info.stats.reported_* is guaranteed to
+  // never be replaced by a lower version
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_shard_t fromosd;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    eversion_t stat_version(10, 1);
+    oinfo.stats.version = stat_version;
+    info.stats.reported_seq = 1;
+    info.stats.reported_epoch = 10;
+    oinfo.stats.reported_seq = 1;
+    oinfo.stats.reported_epoch = 1;
+    log.tail = olog.tail = eversion_t(1, 1);
+    log.head = olog.head = eversion_t(2, 1);
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(0U, log.log.size());
+    EXPECT_EQ(eversion_t(), info.stats.version);
+    EXPECT_EQ(1ull, info.stats.reported_seq);
+    EXPECT_EQ(10u, info.stats.reported_epoch);
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(info.last_backfill.is_max());
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+
+    TestHandler h(remove_snap);
+    merge_log(t, oinfo, olog, fromosd, info, &h,
+              dirty_info, dirty_big_info);
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(0U, log.log.size());
+    EXPECT_EQ(stat_version, info.stats.version);
+    EXPECT_EQ(1ull, info.stats.reported_seq);
+    EXPECT_EQ(10u, info.stats.reported_epoch);
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+  }
+
+  /*         Before
+            +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+            |        |       |         |
+            |        |  x5   |  (1,1)  < tail
+            |        |       |         |
+            |        |       |         |
+       tail > (1,4)  |  x7   |         |
+            |        |       |         |
+            |        |       |         |
+       head > (1,5)  |  x9   |  (1,5)  < head
+            |        |       |         |
+            |        |       |         |
+            +--------+-------+---------+
+
+             After
+            +-----------------
+            |  log           |
+            +--------+-------+
+            |        |object |
+            |version | hash  |
+            |        |       |
+       tail > (1,1)  |  x5   |
+            |        |       |
+            |        |       |
+            | (1,4)  |  x7   |
+            |        |       |
+            |        |       |
+       head > (1,5)  |  x9   |
+            |        |       |
+            |        |       |
+            +--------+-------+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_shard_t fromosd;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 4);
+      e.soid.hash = 0x5;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = eversion_t(1, 5);
+      e.soid.hash = 0x9;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      info.last_update = log.head;
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      olog.tail = e.version;
+      olog.log.push_back(e);
+      e.version = eversion_t(1, 5);
+      e.soid.hash = 0x9;
+      olog.log.push_back(e);
+      olog.head = e.version;
+    }
+
+    hobject_t last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
+    info.last_backfill = last_backfill;
+    eversion_t stat_version(10, 1);
+    info.stats.version = stat_version;
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(2U, log.log.size());
+    EXPECT_EQ(stat_version, info.stats.version);
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_EQ(last_backfill, info.last_backfill);
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+
+    TestHandler h(remove_snap);
+    merge_log(t, oinfo, olog, fromosd, info, &h,
+              dirty_info, dirty_big_info);
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(3U, log.log.size());
+    EXPECT_EQ(stat_version, info.stats.version);
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_TRUE(is_dirty());
+    EXPECT_TRUE(dirty_info);
+    EXPECT_TRUE(dirty_big_info);
+  }
+
+  /*        +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+            |        |       |         |
+       tail > (1,1)  |  x5   |  (1,1)  < tail
+            |        |       |         |
+            |        |       |         |
+            | (1,2)  |  x3   |  (1,2)  < lower_bound
+            |        |       |         |
+            |        |       |         |
+       head > (1,3)  |  x9   |         |
+            | DELETE |       |         |
+            |        |       |         |
+            |        |  x9   |  (2,3)  |
+            |        |       |  MODIFY |
+            |        |       |         |
+            |        |  x7   |  (2,4)  < head
+            |        |       |  DELETE |
+            +--------+-------+---------+
+
+      The log entry (1,3) deletes the object x9 but the olog entry (2,3) modifies
+      it and is authoritative : the log entry (1,3) is divergent.
+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_shard_t fromosd;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    hobject_t divergent_object;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = eversion_t(1, 2);
+      e.soid.hash = 0x3;
+      log.log.push_back(e);
+      e.version = eversion_t(1,3);
+      e.soid.hash = 0x9;
+      divergent_object = e.soid;
+      e.op = pg_log_entry_t::DELETE;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      info.last_update = log.head;
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      olog.tail = e.version;
+      olog.log.push_back(e);
+      e.version = eversion_t(1, 2);
+      e.soid.hash = 0x3;
+      olog.log.push_back(e);
+      e.version = eversion_t(2, 3);
+      e.soid.hash = 0x9;
+      e.op = pg_log_entry_t::MODIFY;
+      olog.log.push_back(e);
+      e.version = eversion_t(2, 4);
+      e.soid.hash = 0x7;
+      e.op = pg_log_entry_t::DELETE;
+      olog.log.push_back(e);
+      olog.head = e.version;
+    }
+
+    snapid_t purged_snap(1);
+    {
+      oinfo.last_update = olog.head;
+      oinfo.purged_snaps.insert(purged_snap);
+    }
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(1U, log.objects.count(divergent_object));
+    EXPECT_EQ(3U, log.log.size());
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_EQ(log.head, info.last_update);
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+
+    TestHandler h(remove_snap);
+    merge_log(t, oinfo, olog, fromosd, info, &h,
+              dirty_info, dirty_big_info);
+
+    /* When the divergent entry is a DELETE and the authoritative
+       entry is a MODIFY, the object will be added to missing : it is
+       a verifiable side effect proving the entry was identified
+       to be divergent.
+    */
+    EXPECT_TRUE(missing.is_missing(divergent_object));
+    EXPECT_EQ(1U, log.objects.count(divergent_object));
+    EXPECT_EQ(4U, log.log.size());
+    /* DELETE entries from olog that are appended to the hed of the
+       log are also added to remove_snap.
+     */
+    EXPECT_EQ(0x7U, remove_snap.front().hash);
+    EXPECT_TRUE(t.empty());
+    EXPECT_EQ(log.head, info.last_update);
+    EXPECT_TRUE(info.purged_snaps.contains(purged_snap));
+    EXPECT_TRUE(is_dirty());
+    EXPECT_TRUE(dirty_info);
+    EXPECT_TRUE(dirty_big_info);
+  }
+
+  /*        +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+            |        |       |         |
+       tail > (1,1)  |  x5   |  (1,1)  < tail
+            |        |       |         |
+            |        |       |         |
+            | (1,4)  |  x7   |  (1,4)  < head
+            |        |       |         |
+            |        |       |         |
+       head > (1,5)  |  x9   |         |
+            |        |       |         |
+            |        |       |         |
+            +--------+-------+---------+
+
+      The head of the log entry (1,5) is divergent because it is greater than the
+      head of olog.
+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_shard_t fromosd;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = eversion_t(1, 4);
+      e.soid.hash = 0x7;
+      log.log.push_back(e);
+      e.version = eversion_t(1, 5);
+      e.soid.hash = 0x9;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      info.last_update = log.head;
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      olog.tail = e.version;
+      olog.log.push_back(e);
+      e.version = eversion_t(1, 4);
+      e.soid.hash = 0x7;
+      olog.log.push_back(e);
+      olog.head = e.version;
+    }
+
+    hobject_t last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
+    info.last_backfill = last_backfill;
+    eversion_t stat_version(10, 1);
+    info.stats.version = stat_version;
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(3U, log.log.size());
+    EXPECT_EQ(stat_version, info.stats.version);
+    EXPECT_TRUE(remove_snap.empty());
+    EXPECT_TRUE(t.empty());
+    EXPECT_EQ(last_backfill, info.last_backfill);
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_FALSE(is_dirty());
+    EXPECT_FALSE(dirty_info);
+    EXPECT_FALSE(dirty_big_info);
+
+    TestHandler h(remove_snap);
+    merge_log(t, oinfo, olog, fromosd, info, &h,
+              dirty_info, dirty_big_info);
+
+    EXPECT_FALSE(missing.have_missing());
+    EXPECT_EQ(2U, log.log.size());
+    EXPECT_EQ(stat_version, info.stats.version);
+    EXPECT_EQ(0x9U, remove_snap.front().hash);
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(info.purged_snaps.empty());
+    EXPECT_TRUE(is_dirty());
+    EXPECT_TRUE(dirty_info);
+    EXPECT_TRUE(dirty_big_info);
+  }
+
+  // If our log is empty, the incoming log needs to have not been trimmed.
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_shard_t fromosd;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    // olog has been trimmed
+    olog.tail = eversion_t(1, 1);
+
+    TestHandler h(remove_snap);
+    ASSERT_THROW(merge_log(t, oinfo, olog, fromosd, info, &h,
+                           dirty_info, dirty_big_info), FailedAssertion);
+  }
+
+  /*        +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+       tail > (0,0)  |       |         |
+            | (1,1)  |  x5   |         |
+            |        |       |         |
+            |        |       |         |
+       head > (1,2)  |  x3   |         |
+            |        |       |         |
+            |        |       |  (2,3)  < tail
+            |        |  x9   |  (2,4)  |
+            |        |       |         |
+            |        |  x5   |  (2,5)  < head
+            |        |       |         |
+            +--------+-------+---------+
+
+      If the logs do not overlap, throw an exception.
+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_shard_t fromosd;
+    pg_info_t info;
+    list remove_snap;
+    bool dirty_info = false;
+    bool dirty_big_info = false;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      log.tail = eversion_t();
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      log.log.push_back(e);
+      e.version = eversion_t(1, 2);
+      e.soid.hash = 0x3;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      info.last_update = log.head;
+
+      olog.tail = eversion_t(2, 3);
+      e.version = eversion_t(2, 4);
+      e.soid.hash = 0x9;
+      olog.log.push_back(e);
+      e.version = eversion_t(2, 5);
+      e.soid.hash = 0x5;
+      olog.log.push_back(e);
+      olog.head = e.version;
+    }
+
+    TestHandler h(remove_snap);
+    EXPECT_THROW(merge_log(t, oinfo, olog, fromosd, info, &h,
+                           dirty_info, dirty_big_info), FailedAssertion);
+  }
+}
+
+TEST_F(PGLogTest, proc_replica_log) {
+  // empty log : no side effect
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_missing_t omissing;
+    pg_shard_t from;
+
+    eversion_t last_update(1, 1);
+    oinfo.last_update = last_update;
+    eversion_t last_complete(2, 1);
+    oinfo.last_complete = last_complete;
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+    EXPECT_EQ(last_update, oinfo.last_update);
+    EXPECT_EQ(last_complete, oinfo.last_complete);
+
+    proc_replica_log(t, oinfo, olog, omissing, from);
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+    EXPECT_EQ(last_update, oinfo.last_update);
+    EXPECT_EQ(last_update, oinfo.last_complete);
+  }
+
+  /*        +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+            |        |       |         |
+            |        |  x3   |  (1,1)  < tail
+            |        |       |         |
+            |        |       |         |
+       tail > (1,2)  |  x5   |         |
+            |        |       |         |
+            |        |       |         |
+       head > (1,3)  |  x9   |         |
+            | DELETE |       |         |
+            |        |       |         |
+            |        |  x9   |  (2,3)  < head
+            |        |       |  DELETE |
+            |        |       |         |
+            +--------+-------+---------+
+	    
+      The log entry (1,3) deletes the object x9 and the olog entry
+      (2,3) also deletes it : do nothing. The olog tail is ignored
+      because it is before the log tail.
+      
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_missing_t omissing;
+    pg_shard_t from;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 2);
+      e.soid.hash = 0x5;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = eversion_t(1, 3);
+      e.soid.hash = 0x9;
+      e.op = pg_log_entry_t::DELETE;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x3;
+      olog.tail = e.version;
+      olog.log.push_back(e);
+      e.version = eversion_t(2, 3);
+      e.soid.hash = 0x9;
+      e.op = pg_log_entry_t::DELETE;
+      olog.log.push_back(e);
+      olog.head = e.version;
+
+      oinfo.last_update = olog.head;
+      oinfo.last_complete = olog.head;
+    }
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+    EXPECT_EQ(olog.head, oinfo.last_update);
+    EXPECT_EQ(olog.head, oinfo.last_complete);
+
+    proc_replica_log(t, oinfo, olog, omissing, from);
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+  }
+
+ {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_missing_t omissing;
+    pg_shard_t from;
+
+    hobject_t divergent_object;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      {
+	e.soid = divergent_object;
+	e.soid.hash = 0x1;
+	e.version = eversion_t(1, 1);
+	log.tail = e.version;
+	log.log.push_back(e);
+
+	e.soid = divergent_object;
+	e.prior_version = eversion_t(1, 1);
+	e.version = eversion_t(1, 2);
+	log.tail = e.version;
+	log.log.push_back(e);
+
+	e.soid.hash = 0x3;
+	e.version = eversion_t(1, 4);
+	log.log.push_back(e);
+
+	e.soid.hash = 0x7;
+	e.version = eversion_t(1, 5);
+	log.log.push_back(e);
+
+	e.soid.hash = 0x8;
+	e.version = eversion_t(1, 6);
+	log.log.push_back(e);
+
+	e.soid.hash = 0x9;
+	e.op = pg_log_entry_t::DELETE;
+	e.version = eversion_t(2, 7);
+	log.log.push_back(e);
+
+	e.soid.hash = 0xa;
+	e.version = eversion_t(2, 8);
+	log.head = e.version;
+	log.log.push_back(e);
+      }
+      log.index();
+
+      {
+	e.soid = divergent_object;
+	e.soid.hash = 0x1;
+	e.version = eversion_t(1, 1);
+	olog.tail = e.version;
+	olog.log.push_back(e);
+
+	e.soid = divergent_object;
+	e.prior_version = eversion_t(1, 1);
+	e.version = eversion_t(1, 2);
+	olog.log.push_back(e);
+
+	e.prior_version = eversion_t(0, 0);
+	e.soid.hash = 0x3;
+	e.version = eversion_t(1, 4);
+	olog.log.push_back(e);
+
+	e.soid.hash = 0x7;
+	e.version = eversion_t(1, 5);
+	olog.log.push_back(e);
+
+	e.soid.hash = 0x8;
+	e.version = eversion_t(1, 6);
+	olog.log.push_back(e);
+
+	e.soid.hash = 0x9; // should not be added to missing, create
+	e.op = pg_log_entry_t::MODIFY;
+	e.version = eversion_t(1, 7);
+	olog.log.push_back(e);
+
+	e.soid = divergent_object; // should be added to missing at 1,2
+	e.op = pg_log_entry_t::MODIFY;
+	e.version = eversion_t(1, 8);
+	e.prior_version = eversion_t(1, 2);
+	olog.log.push_back(e);
+	olog.head = e.version;
+      }
+      oinfo.last_update = olog.head;
+      oinfo.last_complete = olog.head;
+    }
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+    EXPECT_EQ(olog.head, oinfo.last_update);
+    EXPECT_EQ(olog.head, oinfo.last_complete);
+
+    proc_replica_log(t, oinfo, olog, omissing, from);
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(omissing.have_missing());
+    EXPECT_TRUE(omissing.is_missing(divergent_object));
+    EXPECT_EQ(eversion_t(1, 2), omissing.missing[divergent_object].need);
+    EXPECT_EQ(eversion_t(1, 6), oinfo.last_update);
+    EXPECT_EQ(eversion_t(1, 1), oinfo.last_complete);
+  }
+
+  /*        +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+            |        |       |         |
+       tail > (1,1)  |  x5   |  (1,1)  < tail
+            |        |       |         |
+            |        |       |         |
+            | (1,2)  |  x3   |  (1,2)  |
+            |        |       |         |
+            |        |       |         |
+       head > (1,3)  |  x9   |         |
+            | DELETE |       |         |
+            |        |       |         |
+            |        |  x9   |  (2,3)  < head
+            |        |       |  DELETE |
+            |        |       |         |
+            +--------+-------+---------+
+
+      The log entry (1,3) deletes the object x9 and the olog entry
+      (2,3) also deletes it : do nothing.
+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_missing_t omissing;
+    pg_shard_t from;
+
+    eversion_t last_update(1, 2);
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = last_update;
+      e.soid.hash = 0x3;
+      log.log.push_back(e);
+      e.version = eversion_t(1,3);
+      e.soid.hash = 0x9;
+      e.op = pg_log_entry_t::DELETE;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      olog.tail = e.version;
+      olog.log.push_back(e);
+      e.version = last_update;
+      e.soid.hash = 0x3;
+      olog.log.push_back(e);
+      e.version = eversion_t(2, 3);
+      e.soid.hash = 0x9;
+      e.op = pg_log_entry_t::DELETE;
+      olog.log.push_back(e);
+      olog.head = e.version;
+
+      oinfo.last_update = olog.head;
+      oinfo.last_complete = olog.head;
+    }
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+    EXPECT_EQ(olog.head, oinfo.last_update);
+    EXPECT_EQ(olog.head, oinfo.last_complete);
+
+    proc_replica_log(t, oinfo, olog, omissing, from);
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+    EXPECT_EQ(last_update, oinfo.last_update);
+    EXPECT_EQ(last_update, oinfo.last_complete);
+  }
+
+  /*        +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+            |        |       |         |
+       tail > (1,1)  |  x5   |  (1,1)  < tail
+            |        |       |         |
+            |        |       |         |
+            | (1,2)  |  x3   |  (1,2)  |
+            |        |       |         |
+            |        |       |         |
+       head > (1,3)  |  x9   |         |
+            | DELETE |       |         |
+            |        |       |         |
+            |        |  x9   |  (2,3)  < head
+            |        |       |  MODIFY |
+            |        |       |         |
+            +--------+-------+---------+
+
+      The log entry (1,3) deletes the object x9 but the olog entry
+      (2,3) modifies it : remove it from omissing.
+
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_missing_t omissing;
+    pg_shard_t from;
+
+    eversion_t last_update(1, 2);
+    hobject_t divergent_object;
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = last_update;
+      e.soid.hash = 0x3;
+      log.log.push_back(e);
+      e.version = eversion_t(1, 3);
+      e.soid.hash = 0x9;
+      e.op = pg_log_entry_t::DELETE;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x5;
+      olog.tail = e.version;
+      olog.log.push_back(e);
+      e.version = last_update;
+      e.soid.hash = 0x3;
+      olog.log.push_back(e);
+      e.version = eversion_t(2, 3);
+      e.soid.hash = 0x9;
+      divergent_object = e.soid;
+      omissing.add(divergent_object, e.version, eversion_t());
+      e.op = pg_log_entry_t::MODIFY;
+      olog.log.push_back(e);
+      olog.head = e.version;
+
+      oinfo.last_update = olog.head;
+      oinfo.last_complete = olog.head;
+    }
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(omissing.have_missing());
+    EXPECT_TRUE(omissing.is_missing(divergent_object));
+    EXPECT_EQ(eversion_t(2, 3), omissing.missing[divergent_object].need);
+    EXPECT_EQ(olog.head, oinfo.last_update);
+    EXPECT_EQ(olog.head, oinfo.last_complete);
+
+    proc_replica_log(t, oinfo, olog, omissing, from);
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_FALSE(omissing.have_missing());
+    EXPECT_EQ(last_update, oinfo.last_update);
+    EXPECT_EQ(last_update, oinfo.last_complete);
+  }
+
+  /*        +--------------------------+
+            |  log              olog   |
+            +--------+-------+---------+
+            |        |object |         |
+            |version | hash  | version |
+            |        |       |         |
+       tail > (1,1)  |  x9   |  (1,1)  < tail
+            |        |       |         |
+            |        |       |         |
+            | (1,2)  |  x3   |  (1,2)  |
+            |        |       |         |
+            |        |       |         |
+            |        |  x9   |  (1,3)  < head
+            |        |       |  MODIFY |
+            |        |       |         |
+       head > (2,3)  |  x9   |         |
+            | DELETE |       |         |
+            |        |       |         |
+            +--------+-------+---------+
+
+      The log entry (2,3) deletes the object x9 but the olog entry
+      (1,3) modifies it : proc_replica_log should adjust missing to
+      1,1 for that object until add_next_event in PG::activate processes
+      the delete.
+  */
+  {
+    clear();
+
+    ObjectStore::Transaction t;
+    pg_log_t olog;
+    pg_info_t oinfo;
+    pg_missing_t omissing;
+    pg_shard_t from;
+
+    eversion_t last_update(1, 2);
+    hobject_t divergent_object;
+    eversion_t new_version(2, 3);
+    eversion_t divergent_version(1, 3);
+
+    {
+      pg_log_entry_t e;
+      e.mod_desc.mark_unrollbackable();
+
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x9;
+      log.tail = e.version;
+      log.log.push_back(e);
+      e.version = last_update;
+      e.soid.hash = 0x3;
+      log.log.push_back(e);
+      e.version = new_version;
+      e.prior_version = eversion_t(1, 1);
+      e.soid.hash = 0x9;
+      e.op = pg_log_entry_t::DELETE;
+      log.log.push_back(e);
+      log.head = e.version;
+      log.index();
+
+      e.op = pg_log_entry_t::MODIFY;
+      e.version = eversion_t(1, 1);
+      e.soid.hash = 0x9;
+      olog.tail = e.version;
+      olog.log.push_back(e);
+      e.version = last_update;
+      e.soid.hash = 0x3;
+      olog.log.push_back(e);
+      e.version = divergent_version;
+      e.prior_version = eversion_t(1, 1);
+      e.soid.hash = 0x9;
+      divergent_object = e.soid;
+      omissing.add(divergent_object, e.version, eversion_t());
+      e.op = pg_log_entry_t::MODIFY;
+      olog.log.push_back(e);
+      olog.head = e.version;
+
+      oinfo.last_update = olog.head;
+      oinfo.last_complete = olog.head;
+    }
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(omissing.have_missing());
+    EXPECT_TRUE(omissing.is_missing(divergent_object));
+    EXPECT_EQ(divergent_version, omissing.missing[divergent_object].need);
+    EXPECT_EQ(olog.head, oinfo.last_update);
+    EXPECT_EQ(olog.head, oinfo.last_complete);
+
+    proc_replica_log(t, oinfo, olog, omissing, from);
+
+    EXPECT_TRUE(t.empty());
+    EXPECT_TRUE(omissing.have_missing());
+    EXPECT_TRUE(omissing.missing.begin()->second.need == eversion_t(1, 1));
+    EXPECT_EQ(last_update, oinfo.last_update);
+    EXPECT_EQ(eversion_t(0, 0), oinfo.last_complete);
+  }
+
+}
+
+TEST_F(PGLogTest, merge_log_1) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+
+  t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0));
+
+  t.toremove.insert(mk_obj(1));
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_2) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+  t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+  t.torollback.insert(
+    t.torollback.begin(), t.div.rbegin(), t.div.rend());
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_3) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+  t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+  t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0));
+
+  t.toremove.insert(mk_obj(1));
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_4) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+  t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+  t.init.add(mk_obj(1), mk_evt(10, 102), mk_evt(0, 0));
+  t.final.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0));
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_5) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+  t.div.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
+
+  t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+  t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(0, 0));
+
+  t.toremove.insert(mk_obj(1));
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_6) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+  t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100));
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_7) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+  t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80));
+  t.final.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80));
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_8) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.auth.push_back(mk_ple_dt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
+
+  t.init.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80));
+
+  t.toremove.insert(mk_obj(1));
+
+  t.setup();
+  run_test_case(t);
+}
+
+TEST_F(PGLogTest, merge_log_prior_version_have) {
+  TestCase t;
+  t.base.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
+
+  t.div.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
+
+  t.init.add(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100));
+
+  t.setup();
+  run_test_case(t);
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make unittest_pglog ; ./unittest_pglog --log-to-stderr=true  --debug-osd=20 # --gtest_filter=*.* "
+// End:
diff --git a/ceph/src/test/osd/TestRados.cc b/ceph/src/test/osd/TestRados.cc
new file mode 100644
index 00000000..b61ceb3d
--- /dev/null
+++ b/ceph/src/test/osd/TestRados.cc
@@ -0,0 +1,403 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/errno.h"
+#include "common/version.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "test/osd/RadosModel.h"
+
+
+using namespace std;
+
+class WeightedTestGenerator : public TestOpGenerator
+{
+public:
+
+  WeightedTestGenerator(int ops,
+			int objects,
+			map op_weights,
+			TestOpStat *stats,
+			int max_seconds,
+			bool ec_pool) :
+    m_nextop(NULL), m_op(0), m_ops(ops), m_seconds(max_seconds),
+    m_objects(objects), m_stats(stats),
+    m_total_weight(0),
+    m_ec_pool(ec_pool)
+  {
+    m_start = time(0);
+    for (map::const_iterator it = op_weights.begin();
+	 it != op_weights.end();
+	 ++it) {
+      m_total_weight += it->second;
+      m_weight_sums.insert(pair(it->first,
+							  m_total_weight));
+    }
+  }
+
+  TestOp *next(RadosTestContext &context)
+  {
+    TestOp *retval = NULL;
+
+    ++m_op;
+    if (m_op <= m_objects) {
+      stringstream oid;
+      oid << m_op;
+      cout << m_op << ": write initial oid " << oid.str() << std::endl;
+      context.oid_not_flushing.insert(oid.str());
+      if (m_ec_pool) {
+	return new WriteOp(m_op, &context, oid.str(), true);
+      } else {
+	return new WriteOp(m_op, &context, oid.str(), false);
+      }
+    } else if (m_op >= m_ops) {
+      return NULL;
+    }
+
+    if (m_nextop) {
+      retval = m_nextop;
+      m_nextop = NULL;
+      return retval;
+    }
+
+    while (retval == NULL) {
+      unsigned int rand_val = rand() % m_total_weight;
+
+      time_t now = time(0);
+      if (m_seconds && now - m_start > m_seconds)
+	break;
+
+      for (map::const_iterator it = m_weight_sums.begin();
+	   it != m_weight_sums.end();
+	   ++it) {
+	if (rand_val < it->second) {
+	  retval = gen_op(context, it->first);
+	  break;
+	}
+      }
+    }
+    return retval;
+  }
+
+private:
+
+  TestOp *gen_op(RadosTestContext &context, TestOpType type)
+  {
+    string oid, oid2;
+    //cout << "oids not in use " << context.oid_not_in_use.size() << std::endl;
+    assert(context.oid_not_in_use.size());
+
+    switch (type) {
+    case TEST_OP_READ:
+      oid = *(rand_choose(context.oid_not_in_use));
+      return new ReadOp(m_op, &context, oid, m_stats);
+
+    case TEST_OP_WRITE:
+      oid = *(rand_choose(context.oid_not_in_use));
+      cout << m_op << ": " << "write oid " << oid << " current snap is "
+	   << context.current_snap << std::endl;
+      return new WriteOp(m_op, &context, oid, false, m_stats);
+
+    case TEST_OP_DELETE:
+      oid = *(rand_choose(context.oid_not_in_use));
+      cout << m_op << ": " << "delete oid " << oid << " current snap is "
+	   << context.current_snap << std::endl;
+      return new DeleteOp(m_op, &context, oid, m_stats);
+
+    case TEST_OP_SNAP_CREATE:
+      cout << m_op << ": " << "snap_create" << std::endl;
+      return new SnapCreateOp(m_op, &context, m_stats);
+
+    case TEST_OP_SNAP_REMOVE:
+      if (context.snaps.size() <= context.snaps_in_use.size()) {
+	return NULL;
+      }
+      while (true) {
+	int snap = rand_choose(context.snaps)->first;
+	if (context.snaps_in_use.lookup(snap))
+	  continue;  // in use; try again!
+	cout << m_op << ": " << "snap_remove snap " << snap << std::endl;
+	return new SnapRemoveOp(m_op, &context, snap, m_stats);
+      }
+
+    case TEST_OP_ROLLBACK:
+      {
+	string oid = *(rand_choose(context.oid_not_in_use));
+	cout << m_op << ": " << "rollback oid " << oid << " current snap is "
+	     << context.current_snap << std::endl;
+	return new RollbackOp(m_op, &context, oid);
+      }
+
+    case TEST_OP_SETATTR:
+      oid = *(rand_choose(context.oid_not_in_use));
+      cout << m_op << ": " << "setattr oid " << oid
+	   << " current snap is " << context.current_snap << std::endl;
+      return new SetAttrsOp(m_op, &context, oid, m_stats);
+
+    case TEST_OP_RMATTR:
+      oid = *(rand_choose(context.oid_not_in_use));
+      cout << m_op << ": " << "rmattr oid " << oid
+	   << " current snap is " << context.current_snap << std::endl;
+      return new RemoveAttrsOp(m_op, &context, oid, m_stats);
+
+    case TEST_OP_WATCH:
+      oid = *(rand_choose(context.oid_not_in_use));
+      cout << m_op << ": " << "watch oid " << oid
+	   << " current snap is " << context.current_snap << std::endl;
+      return new WatchOp(m_op, &context, oid, m_stats);
+
+    case TEST_OP_COPY_FROM:
+      oid = *(rand_choose(context.oid_not_in_use));
+      do {
+	oid2 = *(rand_choose(context.oid_not_in_use));
+      } while (oid == oid2);
+      cout << m_op << ": " << "copy_from oid " << oid << " from oid " << oid2
+	   << " current snap is " << context.current_snap << std::endl;
+      return new CopyFromOp(m_op, &context, oid, oid2, m_stats);
+
+    case TEST_OP_HIT_SET_LIST:
+      {
+	uint32_t hash = rjhash32(rand());
+	cout << m_op << ": " << "hit_set_list " << hash << std::endl;
+	return new HitSetListOp(m_op, &context, hash, m_stats);
+      }
+
+    case TEST_OP_UNDIRTY:
+      {
+	oid = *(rand_choose(context.oid_not_in_use));
+	cout << m_op << ": " << "undirty oid " << oid << std::endl;
+	return new UndirtyOp(m_op, &context, oid, m_stats);
+      }
+
+    case TEST_OP_IS_DIRTY:
+      {
+	oid = *(rand_choose(context.oid_not_flushing));
+	return new IsDirtyOp(m_op, &context, oid, m_stats);
+      }
+
+    case TEST_OP_CACHE_FLUSH:
+      {
+	oid = *(rand_choose(context.oid_not_in_use));
+	return new CacheFlushOp(m_op, &context, oid, m_stats, true);
+      }
+
+    case TEST_OP_CACHE_TRY_FLUSH:
+      {
+	oid = *(rand_choose(context.oid_not_in_use));
+	return new CacheFlushOp(m_op, &context, oid, m_stats, false);
+      }
+
+    case TEST_OP_CACHE_EVICT:
+      {
+	oid = *(rand_choose(context.oid_not_in_use));
+	return new CacheEvictOp(m_op, &context, oid, m_stats);
+      }
+
+    case TEST_OP_APPEND:
+      oid = *(rand_choose(context.oid_not_in_use));
+      cout << "append oid " << oid << " current snap is "
+	   << context.current_snap << std::endl;
+      return new WriteOp(m_op, &context, oid, true, m_stats);
+
+    default:
+      cerr << m_op << ": Invalid op type " << type << std::endl;
+      assert(0);
+    }
+  }
+
+  TestOp *m_nextop;
+  int m_op;
+  int m_ops;
+  int m_seconds;
+  int m_objects;
+  time_t m_start;
+  TestOpStat *m_stats;
+  map m_weight_sums;
+  unsigned int m_total_weight;
+  bool m_ec_pool;
+};
+
+int main(int argc, char **argv)
+{
+  int ops = 1000;
+  int objects = 50;
+  int max_in_flight = 16;
+  int64_t size = 4000000; // 4 MB
+  int64_t min_stride_size = -1, max_stride_size = -1;
+  int max_seconds = 0;
+  bool pool_snaps = false;
+
+  struct {
+    TestOpType op;
+    const char *name;
+    bool ec_pool_valid;
+  } op_types[] = {
+    { TEST_OP_READ, "read", true },
+    { TEST_OP_WRITE, "write", false },
+    { TEST_OP_DELETE, "delete", true },
+    { TEST_OP_SNAP_CREATE, "snap_create", true },
+    { TEST_OP_SNAP_REMOVE, "snap_remove", true },
+    { TEST_OP_ROLLBACK, "rollback", true },
+    { TEST_OP_SETATTR, "setattr", true },
+    { TEST_OP_RMATTR, "rmattr", true },
+    { TEST_OP_WATCH, "watch", true },
+    { TEST_OP_COPY_FROM, "copy_from", true },
+    { TEST_OP_HIT_SET_LIST, "hit_set_list", true },
+    { TEST_OP_IS_DIRTY, "is_dirty", true },
+    { TEST_OP_UNDIRTY, "undirty", true },
+    { TEST_OP_CACHE_FLUSH, "cache_flush", true },
+    { TEST_OP_CACHE_TRY_FLUSH, "cache_try_flush", true },
+    { TEST_OP_CACHE_EVICT, "cache_evict", true },
+    { TEST_OP_APPEND, "append", true },
+    { TEST_OP_READ /* grr */, NULL },
+  };
+
+  map op_weights;
+  string pool_name = "data";
+  bool ec_pool = false;
+  bool no_omap = false;
+
+  for (int i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "--max-ops") == 0)
+      ops = atoi(argv[++i]);
+    else if (strcmp(argv[i], "--pool") == 0)
+      pool_name = argv[++i];
+    else if (strcmp(argv[i], "--max-seconds") == 0)
+      max_seconds = atoi(argv[++i]);
+    else if (strcmp(argv[i], "--objects") == 0)
+      objects = atoi(argv[++i]);
+    else if (strcmp(argv[i], "--max-in-flight") == 0)
+      max_in_flight = atoi(argv[++i]);
+    else if (strcmp(argv[i], "--size") == 0)
+      size = atoi(argv[++i]);
+    else if (strcmp(argv[i], "--min-stride-size") == 0)
+      min_stride_size = atoi(argv[++i]);
+    else if (strcmp(argv[i], "--max-stride-size") == 0)
+      max_stride_size = atoi(argv[++i]);
+    else if (strcmp(argv[i], "--no-omap") == 0)
+      no_omap = true;
+    else if (strcmp(argv[i], "--pool-snaps") == 0)
+      pool_snaps = true;
+    else if (strcmp(argv[i], "--ec-pool") == 0) {
+      if (!op_weights.empty()) {
+	cerr << "--ec-pool must be specified prior to any ops" << std::endl;
+	exit(1);
+      }
+      ec_pool = true;
+      no_omap = true;
+    } else if (strcmp(argv[i], "--op") == 0) {
+      i++;
+      if (i == argc) {
+        cerr << "Missing op after --op" << std::endl;
+        return 1;
+      }
+      int j;
+      for (j = 0; op_types[j].name; ++j) {
+	if (strcmp(op_types[j].name, argv[i]) == 0) {
+	  break;
+	}
+      }
+      if (!op_types[j].name) {
+	cerr << "unknown op " << argv[i] << std::endl;
+	exit(1);
+      }
+      i++;
+      if (i == argc) {
+	cerr << "Weight unspecified." << std::endl;
+	return 1;
+      }
+      int weight = atoi(argv[i]);
+      if (weight < 0) {
+	cerr << "Weights must be nonnegative." << std::endl;
+	return 1;
+      } else if (weight > 0) {
+	if (ec_pool && !op_types[j].ec_pool_valid) {
+	  cerr << "Error: cannot use op type " << op_types[j].name
+	       << " with --ec-pool" << std::endl;
+	  exit(1);
+	}
+	cout << "adding op weight " << op_types[j].name << " -> " << weight << std::endl;
+	op_weights.insert(pair(op_types[j].op, weight));
+      }
+    } else {
+      cerr << "unknown arg " << argv[i] << std::endl;
+      //usage();
+      exit(1);
+    }
+  }
+
+  if (op_weights.empty()) {
+    cerr << "No operations specified" << std::endl;
+    //usage();
+    exit(1);
+  }
+
+  if (min_stride_size < 0)
+    min_stride_size = size / 10;
+  if (max_stride_size < 0)
+    max_stride_size = size / 5;
+
+  cout << pretty_version_to_str() << std::endl;
+  cout << "Configuration:" << std::endl
+       << "\tNumber of operations: " << ops << std::endl
+       << "\tNumber of objects: " << objects << std::endl
+       << "\tMax in flight operations: " << max_in_flight << std::endl
+       << "\tObject size (in bytes): " << size << std::endl
+       << "\tWrite stride min: " << min_stride_size << std::endl
+       << "\tWrite stride max: " << max_stride_size << std::endl;
+
+  if (min_stride_size > max_stride_size) {
+    cerr << "Error: min_stride_size cannot be more than max_stride_size"
+	 << std::endl;
+    return 1;
+  }
+
+  if (min_stride_size > size || max_stride_size > size) {
+    cerr << "Error: min_stride_size and max_stride_size must be "
+	 << "smaller than object size" << std::endl;
+    return 1;
+  }
+
+  if (max_in_flight * 2 > objects) {
+    cerr << "Error: max_in_flight must be <= than the number of objects / 2"
+	 << std::endl;
+    return 1;
+  }
+
+  char *id = getenv("CEPH_CLIENT_ID");
+  RadosTestContext context(
+    pool_name,
+    max_in_flight,
+    size,
+    min_stride_size,
+    max_stride_size,
+    no_omap,
+    pool_snaps,
+    id);
+
+  TestOpStat stats;
+  WeightedTestGenerator gen = WeightedTestGenerator(
+    ops, objects,
+    op_weights, &stats, max_seconds,
+    ec_pool);
+  int r = context.init();
+  if (r < 0) {
+    cerr << "Error initializing rados test context: "
+	 << cpp_strerror(r) << std::endl;
+    exit(1);
+  }
+  context.loop(&gen);
+
+  context.shutdown();
+  cerr << context.errors << " errors." << std::endl;
+  cerr << stats << std::endl;
+  return 0;
+}
diff --git a/ceph/src/test/osd/hitset.cc b/ceph/src/test/osd/hitset.cc
new file mode 100644
index 00000000..9e8ee5e0
--- /dev/null
+++ b/ceph/src/test/osd/hitset.cc
@@ -0,0 +1,197 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ * Copyright 2013 Inktank
+ */
+
+#include "gtest/gtest.h"
+#include "osd/HitSet.h"
+#include 
+
+class HitSetTestStrap {
+public:
+  HitSet *hitset;
+
+  HitSetTestStrap(HitSet *h) : hitset(h) {}
+
+  void fill(unsigned count) {
+    char buf[50];
+    for (unsigned i = 0; i < count; ++i) {
+      sprintf(buf, "hitsettest_%d", i);
+      hobject_t obj(object_t(buf), "", 0, i, 0, "");
+      hitset->insert(obj);
+    }
+    EXPECT_EQ(count, hitset->insert_count());
+  }
+  void verify_fill(unsigned count) {
+    char buf[50];
+    for (unsigned i = 0; i < count; ++i) {
+      sprintf(buf, "hitsettest_%d", i);
+      hobject_t obj(object_t(buf), "", 0, i, 0, "");
+      EXPECT_TRUE(hitset->contains(obj));
+    }
+  }
+
+};
+
+class BloomHitSetTest : public testing::Test, public HitSetTestStrap {
+public:
+
+  BloomHitSetTest() : HitSetTestStrap(new HitSet(new BloomHitSet)) {}
+
+  void rebuild(double fp, uint64_t target, uint64_t seed) {
+    BloomHitSet::Params *bparams = new BloomHitSet::Params(fp, target, seed);
+    HitSet::Params param(bparams);
+    HitSet new_set(param);
+    *hitset = new_set;
+  }
+
+  BloomHitSet *get_hitset() { return static_cast(hitset->impl.get()); }
+};
+
+TEST_F(BloomHitSetTest, Params) {
+  BloomHitSet::Params params(0.01, 100, 5);
+  EXPECT_EQ(.01, params.get_fpp());
+  EXPECT_EQ((unsigned)100, params.target_size);
+  EXPECT_EQ((unsigned)5, params.seed);
+  params.set_fpp(0.1);
+  EXPECT_EQ(0.1, params.get_fpp());
+
+  bufferlist bl;
+  params.encode(bl);
+  BloomHitSet::Params p2;
+  bufferlist::iterator iter = bl.begin();
+  p2.decode(iter);
+  EXPECT_EQ(0.1, p2.get_fpp());
+  EXPECT_EQ((unsigned)100, p2.target_size);
+  EXPECT_EQ((unsigned)5, p2.seed);
+}
+
+TEST_F(BloomHitSetTest, Construct) {
+  ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_BLOOM);
+  // success!
+}
+
+TEST_F(BloomHitSetTest, Rebuild) {
+  rebuild(0.1, 100, 1);
+  ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_BLOOM);
+}
+
+TEST_F(BloomHitSetTest, InsertsMatch) {
+  rebuild(0.1, 100, 1);
+  fill(50);
+  /*
+   *  the approx unique count is atrocious on bloom filters. Empirical
+   *  evidence suggests the current test will produce a value of 62
+   *  regardless of hitset size
+   */
+  EXPECT_TRUE(hitset->approx_unique_insert_count() >= 50 &&
+              hitset->approx_unique_insert_count() <= 62);
+  verify_fill(50);
+  EXPECT_FALSE(hitset->is_full());
+}
+
+TEST_F(BloomHitSetTest, FillsUp) {
+  rebuild(0.1, 20, 1);
+  fill(20);
+  verify_fill(20);
+  EXPECT_TRUE(hitset->is_full());
+}
+
+TEST_F(BloomHitSetTest, RejectsNoMatch) {
+  rebuild(0.001, 100, 1);
+  fill(100);
+  verify_fill(100);
+  EXPECT_TRUE(hitset->is_full());
+
+  char buf[50];
+  int matches = 0;
+  for (int i = 100; i < 200; ++i) {
+    sprintf(buf, "hitsettest_%d", i);
+    hobject_t obj(object_t(buf), "", 0, i, 0, "");
+    if (hitset->contains(obj))
+      ++matches;
+  }
+  // we set a 1 in 1000 false positive; allow one in our 100
+  EXPECT_LT(matches, 2);
+}
+
+class ExplicitHashHitSetTest : public testing::Test, public HitSetTestStrap {
+public:
+
+  ExplicitHashHitSetTest() : HitSetTestStrap(new HitSet(new ExplicitHashHitSet)) {}
+
+  ExplicitHashHitSet *get_hitset() { return static_cast(hitset->impl.get()); }
+};
+
+TEST_F(ExplicitHashHitSetTest, Construct) {
+  ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_EXPLICIT_HASH);
+  // success!
+}
+
+TEST_F(ExplicitHashHitSetTest, InsertsMatch) {
+  fill(50);
+  verify_fill(50);
+  EXPECT_EQ((unsigned)50, hitset->approx_unique_insert_count());
+  EXPECT_FALSE(hitset->is_full());
+}
+
+TEST_F(ExplicitHashHitSetTest, RejectsNoMatch) {
+  fill(100);
+  verify_fill(100);
+  EXPECT_FALSE(hitset->is_full());
+
+  char buf[50];
+  int matches = 0;
+  for (int i = 100; i < 200; ++i) {
+    sprintf(buf, "hitsettest_%d", i);
+    hobject_t obj(object_t(buf), "", 0, i, 0, "");
+    if (hitset->contains(obj)) {
+      ++matches;
+    }
+  }
+  EXPECT_EQ(matches, 0);
+}
+
+class ExplicitObjectHitSetTest : public testing::Test, public HitSetTestStrap {
+public:
+
+  ExplicitObjectHitSetTest() : HitSetTestStrap(new HitSet(new ExplicitObjectHitSet)) {}
+
+  ExplicitObjectHitSet *get_hitset() { return static_cast(hitset->impl.get()); }
+};
+
+TEST_F(ExplicitObjectHitSetTest, Construct) {
+  ASSERT_EQ(hitset->impl->get_type(), HitSet::TYPE_EXPLICIT_OBJECT);
+  // success!
+}
+
+TEST_F(ExplicitObjectHitSetTest, InsertsMatch) {
+  fill(50);
+  verify_fill(50);
+  EXPECT_EQ((unsigned)50, hitset->approx_unique_insert_count());
+  EXPECT_FALSE(hitset->is_full());
+}
+
+TEST_F(ExplicitObjectHitSetTest, RejectsNoMatch) {
+  fill(100);
+  verify_fill(100);
+  EXPECT_FALSE(hitset->is_full());
+
+  char buf[50];
+  int matches = 0;
+  for (int i = 100; i < 200; ++i) {
+    sprintf(buf, "hitsettest_%d", i);
+    hobject_t obj(object_t(buf), "", 0, i, 0, "");
+    if (hitset->contains(obj)) {
+      ++matches;
+    }
+  }
+  EXPECT_EQ(matches, 0);
+}
diff --git a/ceph/src/test/osd/osd-test-helpers.sh b/ceph/src/test/osd/osd-test-helpers.sh
new file mode 100644
index 00000000..1ea17dd1
--- /dev/null
+++ b/ceph/src/test/osd/osd-test-helpers.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Copyright (C) 2014 Cloudwatt 
+#
+# Author: Loic Dachary 
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+
+function run_osd() {
+    local dir=$1
+    shift
+    local id=$1
+    shift
+    local osd_data=$dir/$id
+
+    local ceph_disk_args
+    ceph_disk_args+=" --statedir=$dir"
+    ceph_disk_args+=" --sysconfdir=$dir"
+    ceph_disk_args+=" --prepend-to-path="
+    ceph_disk_args+=" --verbose"
+
+    touch $dir/ceph.conf
+
+    ./ceph-disk $ceph_disk_args \
+        prepare $osd_data || return 1
+
+    local ceph_args="$CEPH_ARGS"
+    ceph_args+=" --osd-journal-size=100"
+    ceph_args+=" --osd-data=$osd_data"
+    ceph_args+=" --chdir="
+    ceph_args+=" --osd-pool-default-erasure-code-directory=.libs"
+    ceph_args+=" --run-dir=$dir"
+    ceph_args+=" --debug-osd=20"
+    ceph_args+=" --log-file=$dir/osd-\$id.log"
+    ceph_args+=" --pid-file=$dir/osd-\$id.pidfile"
+    ceph_args+=" "
+    ceph_args+="$@"
+    CEPH_ARGS="$ceph_args" ./ceph-disk $ceph_disk_args \
+        activate \
+        --mark-init=none \
+        $osd_data || return 1
+
+    [ "$id" = "$(cat $osd_data/whoami)" ] || return 1
+
+    ./ceph osd crush create-or-move "$id" 1 root=default host=localhost
+}
diff --git a/ceph/src/test/osd/osdcap.cc b/ceph/src/test/osd/osdcap.cc
new file mode 100644
index 00000000..a9aefd72
--- /dev/null
+++ b/ceph/src/test/osd/osdcap.cc
@@ -0,0 +1,590 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+
+#include "include/stringify.h"
+#include "osd/OSDCap.h"
+
+#include "gtest/gtest.h"
+
+const char *parse_good[] = {
+  "allow *",
+  "allow r",
+  "allow rwx",
+  "allow r pool foo ",
+  "allow r pool=foo",
+  "allow wx pool taco",
+  "allow pool foo r",
+  "allow pool taco wx",
+  "allow wx pool taco object_prefix obj",
+  "allow wx pool taco object_prefix obj_with_underscores_and_no_quotes",
+  "allow pool taco object_prefix obj wx",
+  "allow pool taco object_prefix obj_with_underscores_and_no_quotes wx",
+  "allow rwx pool 'weird name'",
+  "allow rwx pool \"weird name with ''s\"",
+  "allow rwx auid 123",
+  "allow rwx pool foo, allow r pool bar",
+  "allow rwx pool foo ; allow r pool bar",
+  "allow rwx pool foo ;allow r pool bar",
+  "allow rwx pool foo; allow r pool bar",
+  "allow auid 123 rwx",
+  "allow pool foo rwx, allow pool bar r",
+  "allow pool foo.froo.foo rwx, allow pool bar r",
+  "allow pool foo rwx ; allow pool bar r",
+  "allow pool foo rwx ;allow pool bar r",
+  "allow pool foo rwx; allow pool bar r",
+  "allow pool data rw, allow pool rbd rwx, allow pool images class rbd foo",
+  "allow class-read",
+  "allow class-write",
+  "allow class-read class-write",
+  "allow r class-read pool foo",
+  "allow rw class-read class-write pool foo",
+  "allow r class-read pool foo",
+  "allow pool bar rwx; allow pool baz r class-read",
+  "allow class foo",
+  "allow class clsname \"clsthingidon'tunderstand\"",
+  "  allow rwx pool foo; allow r pool bar  ",
+  "  allow   rwx   pool foo; allow r pool bar  ",
+  "  allow pool foo rwx; allow pool bar r  ",
+  "  allow     pool foo rwx; allow pool bar r  ",
+  " allow wx pool taco",
+  "\tallow\nwx\tpool \n taco\t",
+  "allow r   pool    foo    object_prefix   blah   ;   allow   w   auid  5",
+  "allow class-read object_prefix rbd_children, allow pool libvirt-pool-test rwx",
+  "allow class-read object_prefix rbd-children, allow pool libvirt_pool_test rwx",
+  "allow pool foo namespace nfoo rwx, allow pool bar namespace=nbar r",
+  "allow pool foo namespace=nfoo rwx ; allow pool bar namespace=nbar r",
+  "allow pool foo namespace nfoo rwx ;allow pool bar namespace nbar r",
+  "allow pool foo namespace=nfoo rwx; allow pool bar namespace nbar object_prefix rbd r",
+  "allow pool foo namespace=\"\" rwx; allow pool bar namespace='' object_prefix rbd r",
+  "allow pool foo namespace \"\" rwx; allow pool bar namespace '' object_prefix rbd r",
+  0
+};
+
+TEST(OSDCap, ParseGood) {
+  for (int i=0; parse_good[i]; i++) {
+    string str = parse_good[i];
+    OSDCap cap;
+    std::cout << "Testing good input: '" << str << "'" << std::endl;
+    ASSERT_TRUE(cap.parse(str, &cout));
+  }
+}
+
+const char *parse_bad[] = {
+  "allow r poolfoo",
+  "allow r w",
+  "ALLOW r",
+  "allow rwx,",
+  "allow rwx x",
+  "allow r pool foo r",
+  "allow wwx pool taco",
+  "allow wwx pool taco^funny&chars",
+  "allow rwx pool 'weird name''",
+  "allow rwx object_prefix \"beforepool\" pool weird",
+  "allow rwx auid 123 pool asdf",
+  "allow xrwx pool foo,, allow r pool bar",
+  ";allow rwx pool foo rwx ; allow r pool bar",
+  "allow rwx pool foo ;allow r pool bar gibberish",
+  "allow rwx auid 123 pool asdf namespace=foo",
+  "allow rwx auid 123 namespace",
+  "allow rwx namespace",
+  "allow namespace",
+  "allow namespace=foo",
+  "allow rwx auid 123 namespace asdf",
+  "allow wwx pool ''",
+  0
+};
+
+TEST(OSDCap, ParseBad) {
+  for (int i=0; parse_bad[i]; i++) {
+    string str = parse_bad[i];
+    OSDCap cap;
+    std::cout << "Testing bad input: '" << str << "'" << std::endl;
+    ASSERT_FALSE(cap.parse(str, &cout));
+  }
+}
+
+TEST(OSDCap, AllowAll) {
+  OSDCap cap;
+  ASSERT_FALSE(cap.allow_all());
+
+  ASSERT_TRUE(cap.parse("allow r", NULL));
+  ASSERT_FALSE(cap.allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow w", NULL));
+  ASSERT_FALSE(cap.allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow x", NULL));
+  ASSERT_FALSE(cap.allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow rwx", NULL));
+  ASSERT_FALSE(cap.allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow rw", NULL));
+  ASSERT_FALSE(cap.allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow rx", NULL));
+  ASSERT_FALSE(cap.allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow wx", NULL));
+  ASSERT_FALSE(cap.allow_all());
+  cap.grants.clear();
+
+  ASSERT_TRUE(cap.parse("allow *", NULL));
+  ASSERT_TRUE(cap.allow_all());
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "asdf", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "anamespace", 0, "asdf", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "asdf", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "anamespace", 0, "asdf", true, true, true, true));
+}
+
+TEST(OSDCap, AllowPool) {
+  OSDCap cap;
+  bool r = cap.parse("allow rwx pool foo", NULL);
+  ASSERT_TRUE(r);
+
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "", true, true, true, true));
+}
+
+TEST(OSDCap, AllowPools) {
+  OSDCap cap;
+  bool r = cap.parse("allow rwx pool foo, allow r pool bar", NULL);
+  ASSERT_TRUE(r);
+
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "ns", 0, "", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("baz", "ns", 0, "", true, false, false, false));
+}
+
+TEST(OSDCap, AllowPools2) {
+  OSDCap cap;
+  bool r = cap.parse("allow r, allow rwx pool foo", NULL);
+  ASSERT_TRUE(r);
+
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "", true, false, false, false));
+}
+
+TEST(OSDCap, ObjectPrefix) {
+  OSDCap cap;
+  bool r = cap.parse("allow rwx object_prefix foo", NULL);
+  ASSERT_TRUE(r);
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "food", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo_bar", true, true, true, true));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "_foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, " foo ", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "fo", true, true, true, true));
+}
+
+TEST(OSDCap, ObjectPoolAndPrefix) {
+  OSDCap cap;
+  bool r = cap.parse("allow rwx pool bar object_prefix foo", NULL);
+  ASSERT_TRUE(r);
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "food", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo_bar", true, true, true, true));
+
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "food", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "fo", true, true, true, true));
+}
+
+TEST(OSDCap, BasicR) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow r", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+}
+
+TEST(OSDCap, BasicW) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow w", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+}
+
+TEST(OSDCap, BasicX) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow x", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, true));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+}
+
+TEST(OSDCap, BasicRW) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow rw", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+}
+
+TEST(OSDCap, BasicRX) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow rx", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, true, true));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+}
+
+TEST(OSDCap, BasicWX) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow wx", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, true, true));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+}
+
+TEST(OSDCap, BasicRWX) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow rwx", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, false));
+}
+
+TEST(OSDCap, BasicRWClassRClassW) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow rw class-read class-write", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, false));
+}
+
+TEST(OSDCap, ClassR) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow class-read", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, false, false, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+}
+
+TEST(OSDCap, ClassW) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow class-write", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, false, true));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+}
+
+TEST(OSDCap, ClassRW) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow class-read class-write", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, true));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+}
+
+TEST(OSDCap, BasicRClassR) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow r class-read", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+
+  ASSERT_TRUE(cap.is_capable("bar", "any", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "any", 0, "foo", true, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "any", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "any", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "any", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "any", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "any", 0, "foo", true, true, false, false));
+}
+
+TEST(OSDCap, PoolClassR) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow pool bar r class-read, allow pool foo rwx", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+
+  ASSERT_TRUE(cap.is_capable("bar", "ns", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "ns", 0, "foo", true, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "ns", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "foo", true, true, false, false));
+
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", false, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", false, false, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", false, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, false, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, true, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, true, true, false));
+
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", false, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", false, false, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", false, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, false, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, true, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, true, true, false));
+
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", false, false, false, false));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", false, false, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, false, false, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, true, false, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, true, true, false));
+}
+
+TEST(OSDCap, PoolClassRNS) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow pool bar namespace='' r class-read, allow pool foo namespace=ns rwx", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, true, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "", 0, "foo", true, true, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "foo", false, false, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "foo", true, false, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "ns", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "other", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "other", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "other", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("bar", "other", 0, "foo", true, true, false, false));
+
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", false, false, false, false));
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", false, false, true, true));
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", true, false, false, true));
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", true, true, false, true));
+  ASSERT_FALSE(cap.is_capable("foo", "", 0, "foo", true, true, true, false));
+
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", false, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", false, false, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", false, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, false, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, true, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "ns", 0, "foo", true, true, true, false));
+
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", false, false, false, false));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", false, false, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", false, true, true, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, false, false, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, true, false, true));
+  ASSERT_FALSE(cap.is_capable("baz", "", 0, "foo", true, true, true, false));
+}
+
+TEST(OSDCap, NSClassR) {
+  OSDCap cap;
+  ASSERT_TRUE(cap.parse("allow namespace '' rw class-read class-write, allow namespace test r", NULL));
+
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, false, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", false, true, true, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, false, true));
+  ASSERT_TRUE(cap.is_capable("bar", "", 0, "foo", true, true, true, false));
+
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", false, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", false, false, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", false, true, true, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, false, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, false, false, false));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, true, false, true));
+  ASSERT_TRUE(cap.is_capable("foo", "", 0, "foo", true, true, true, false));
+
+  ASSERT_TRUE(cap.is_capable("bar", "test", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("bar", "test", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("bar", "test", 0, "foo", true, false, true, false));
+  ASSERT_FALSE(cap.is_capable("bar", "test", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("bar", "test", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("bar", "test", 0, "foo", true, true, false, false));
+
+  ASSERT_TRUE(cap.is_capable("foo", "test", 0, "foo", true, false, false, false));
+
+  ASSERT_FALSE(cap.is_capable("foo", "test", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("foo", "test", 0, "foo", true, false, true, false));
+  ASSERT_FALSE(cap.is_capable("foo", "test", 0, "foo", true, true, true, true));
+  ASSERT_FALSE(cap.is_capable("foo", "test", 0, "foo", false, true, false, true));
+  ASSERT_FALSE(cap.is_capable("foo", "test", 0, "foo", true, true, false, false));
+
+  ASSERT_FALSE(cap.is_capable("foo", "bad", 0, "foo", true, false, false, false));
+  ASSERT_FALSE(cap.is_capable("foo", "bad", 0, "foo", false, true, false, false));
+  ASSERT_FALSE(cap.is_capable("foo", "bad", 0, "foo", false, false, true, false));
+  ASSERT_FALSE(cap.is_capable("foo", "bad", 0, "foo", false, false, false, true));
+}
+
+TEST(OSDCap, OutputParsed)
+{
+  struct CapsTest {
+    const char *input;
+    const char *output;
+  };
+  CapsTest test_values[] = {
+    {"allow *",
+     "osdcap[grant(*)]"},
+    {"allow r",
+     "osdcap[grant(r)]"},
+    {"allow rx",
+     "osdcap[grant(rx)]"},
+    {"allow rwx",
+     "osdcap[grant(rwx)]"},
+    {"allow rw class-read class-write",
+     "osdcap[grant(rwx)]"},
+    {"allow rw class-read",
+     "osdcap[grant(rw class-read)]"},
+    {"allow rw class-write",
+     "osdcap[grant(rw class-write)]"},
+    {"allow rwx pool images",
+     "osdcap[grant(pool images rwx)]"},
+    {"allow r pool images",
+     "osdcap[grant(pool images r)]"},
+    {"allow pool images rwx",
+     "osdcap[grant(pool images rwx)]"},
+    {"allow pool images r",
+     "osdcap[grant(pool images r)]"},
+    {"allow pool images w",
+     "osdcap[grant(pool images w)]"},
+    {"allow pool images x",
+     "osdcap[grant(pool images x)]"},
+    {"allow r pool images namespace ''",
+     "osdcap[grant(pool images namespace \"\" r)]"},
+    {"allow r pool images namespace foo",
+     "osdcap[grant(pool images namespace foo r)]"},
+    {"allow r pool images namespace \"\"",
+     "osdcap[grant(pool images namespace \"\" r)]"},
+    {"allow r namespace foo",
+     "osdcap[grant(namespace foo r)]"},
+    {"allow pool images r; allow pool rbd rwx",
+     "osdcap[grant(pool images r),grant(pool rbd rwx)]"},
+    {"allow pool images r, allow pool rbd rwx",
+     "osdcap[grant(pool images r),grant(pool rbd rwx)]"},
+    {"allow class-read object_prefix rbd_children, allow pool libvirt-pool-test rwx",
+     "osdcap[grant(object_prefix rbd_children  class-read),grant(pool libvirt-pool-test rwx)]"}
+  };
+
+  size_t num_tests = sizeof(test_values) / sizeof(*test_values);
+  for (size_t i = 0; i < num_tests; ++i) {
+    OSDCap cap;
+    std::cout << "Testing input '" << test_values[i].input << "'" << std::endl;
+    ASSERT_TRUE(cap.parse(test_values[i].input));
+    ASSERT_EQ(test_values[i].output, stringify(cap));
+  }
+}
diff --git a/ceph/src/test/osd/types.cc b/ceph/src/test/osd/types.cc
new file mode 100644
index 00000000..a04f2cba
--- /dev/null
+++ b/ceph/src/test/osd/types.cc
@@ -0,0 +1,1304 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ * Copyright (C) 2013 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/types.h"
+#include "osd/osd_types.h"
+#include "osd/OSDMap.h"
+#include "gtest/gtest.h"
+#include "common/Thread.h"
+
+#include 
+
+TEST(hobject, prefixes0)
+{
+  uint32_t mask = 0xE947FA20;
+  uint32_t bits = 12;
+  int64_t pool = 0;
+
+  set prefixes_correct;
+  prefixes_correct.insert(string("0000000000000000.02A"));
+
+  set prefixes_out(hobject_t::get_prefixes(bits, mask, pool));
+  ASSERT_EQ(prefixes_out, prefixes_correct);
+}
+
+TEST(hobject, prefixes1)
+{
+  uint32_t mask = 0x0000000F;
+  uint32_t bits = 6;
+  int64_t pool = 20;
+
+  set prefixes_correct;
+  prefixes_correct.insert(string("0000000000000014.F0"));
+  prefixes_correct.insert(string("0000000000000014.F4"));
+  prefixes_correct.insert(string("0000000000000014.F8"));
+  prefixes_correct.insert(string("0000000000000014.FC"));
+
+  set prefixes_out(hobject_t::get_prefixes(bits, mask, pool));
+  ASSERT_EQ(prefixes_out, prefixes_correct);
+}
+
+TEST(hobject, prefixes2)
+{
+  uint32_t mask = 0xDEADBEAF;
+  uint32_t bits = 25;
+  int64_t pool = 0;
+
+  set prefixes_correct;
+  prefixes_correct.insert(string("0000000000000000.FAEBDA0"));
+  prefixes_correct.insert(string("0000000000000000.FAEBDA2"));
+  prefixes_correct.insert(string("0000000000000000.FAEBDA4"));
+  prefixes_correct.insert(string("0000000000000000.FAEBDA6"));
+  prefixes_correct.insert(string("0000000000000000.FAEBDA8"));
+  prefixes_correct.insert(string("0000000000000000.FAEBDAA"));
+  prefixes_correct.insert(string("0000000000000000.FAEBDAC"));
+  prefixes_correct.insert(string("0000000000000000.FAEBDAE"));
+
+  set prefixes_out(hobject_t::get_prefixes(bits, mask, pool));
+  ASSERT_EQ(prefixes_out, prefixes_correct);
+}
+
+TEST(hobject, prefixes3)
+{
+  uint32_t mask = 0xE947FA20;
+  uint32_t bits = 32;
+  int64_t pool = 0x23;
+
+  set prefixes_correct;
+  prefixes_correct.insert(string("0000000000000023.02AF749E"));
+
+  set prefixes_out(hobject_t::get_prefixes(bits, mask, pool));
+  ASSERT_EQ(prefixes_out, prefixes_correct);
+}
+
+TEST(hobject, prefixes4)
+{
+  uint32_t mask = 0xE947FA20;
+  uint32_t bits = 0;
+  int64_t pool = 0x23;
+
+  set prefixes_correct;
+  prefixes_correct.insert(string("0000000000000023."));
+
+  set prefixes_out(hobject_t::get_prefixes(bits, mask, pool));
+  ASSERT_EQ(prefixes_out, prefixes_correct);
+}
+
+TEST(hobject, prefixes5)
+{
+  uint32_t mask = 0xDEADBEAF;
+  uint32_t bits = 1;
+  int64_t pool = 0x34AC5D00;
+
+  set prefixes_correct;
+  prefixes_correct.insert(string("0000000034AC5D00.1"));
+  prefixes_correct.insert(string("0000000034AC5D00.3"));
+  prefixes_correct.insert(string("0000000034AC5D00.5"));
+  prefixes_correct.insert(string("0000000034AC5D00.7"));
+  prefixes_correct.insert(string("0000000034AC5D00.9"));
+  prefixes_correct.insert(string("0000000034AC5D00.B"));
+  prefixes_correct.insert(string("0000000034AC5D00.D"));
+  prefixes_correct.insert(string("0000000034AC5D00.F"));
+
+  set prefixes_out(hobject_t::get_prefixes(bits, mask, pool));
+  ASSERT_EQ(prefixes_out, prefixes_correct);
+}
+
+TEST(pg_interval_t, check_new_interval)
+{
+  //
+  // Create a situation where osdmaps are the same so that
+  // each test case can diverge from it using minimal code.
+  //
+  int osd_id = 1;
+  epoch_t epoch = 40;
+  ceph::shared_ptr osdmap(new OSDMap());
+  osdmap->set_max_osd(10);
+  osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
+  osdmap->set_epoch(epoch);
+  ceph::shared_ptr lastmap(new OSDMap());
+  lastmap->set_max_osd(10);
+  lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
+  lastmap->set_epoch(epoch);
+  epoch_t same_interval_since = epoch;
+  epoch_t last_epoch_clean = same_interval_since;
+  int64_t pool_id = 200;
+  int pg_num = 4;
+  __u8 min_size = 2;
+  {
+    OSDMap::Incremental inc(epoch + 1);
+    inc.new_pools[pool_id].min_size = min_size;
+    inc.new_pools[pool_id].set_pg_num(pg_num);
+    inc.new_up_thru[osd_id] = epoch + 1;
+    osdmap->apply_incremental(inc);
+    lastmap->apply_incremental(inc);
+  }
+  vector new_acting;
+  new_acting.push_back(osd_id);
+  new_acting.push_back(osd_id + 1);
+  vector old_acting = new_acting;
+  int old_primary = osd_id;
+  int new_primary = osd_id;
+  vector new_up;
+  new_up.push_back(osd_id);
+  int old_up_primary = osd_id;
+  int new_up_primary = osd_id;
+  vector old_up = new_up;
+  pg_t pgid;
+  pgid.set_pool(pool_id);
+
+  //
+  // Do nothing if there are no modifications in
+  // acting, up or pool size and that the pool is not
+  // being split
+  //
+  {
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_FALSE(pg_interval_t::check_new_interval(old_primary,
+						   new_primary,
+						   old_acting,
+						   new_acting,
+						   old_up_primary,
+						   new_up_primary,
+						   old_up,
+						   new_up,
+						   same_interval_since,
+						   last_epoch_clean,
+						   osdmap,
+						   lastmap,
+						   pool_id,
+						   pgid,
+						   &past_intervals));
+    ASSERT_TRUE(past_intervals.empty());
+  }
+
+  //
+  // pool did not exist in the old osdmap
+  //
+  {
+    ceph::shared_ptr lastmap(new OSDMap());
+    lastmap->set_max_osd(10);
+    lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
+    lastmap->set_epoch(epoch);
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_EQ(same_interval_since, past_intervals[same_interval_since].first);
+    ASSERT_EQ(osdmap->get_epoch() - 1, past_intervals[same_interval_since].last);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].acting[0]);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].up[0]);
+  }
+
+  //
+  // The acting set has changed
+  //
+  {
+    vector new_acting;
+    int _new_primary = osd_id + 1;
+    new_acting.push_back(_new_primary);
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals));
+    old_primary = new_primary;
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_EQ(same_interval_since, past_intervals[same_interval_since].first);
+    ASSERT_EQ(osdmap->get_epoch() - 1, past_intervals[same_interval_since].last);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].acting[0]);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].up[0]);
+  }
+
+  //
+  // The up set has changed
+  //
+  {
+    vector new_up;
+    int _new_primary = osd_id + 1;
+    new_up.push_back(_new_primary);
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_EQ(same_interval_since, past_intervals[same_interval_since].first);
+    ASSERT_EQ(osdmap->get_epoch() - 1, past_intervals[same_interval_since].last);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].acting[0]);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].up[0]);
+  }
+
+  //
+  // The up primary has changed
+  //
+  {
+    vector new_up;
+    int _new_up_primary = osd_id + 1;
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  _new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_EQ(same_interval_since, past_intervals[same_interval_since].first);
+    ASSERT_EQ(osdmap->get_epoch() - 1, past_intervals[same_interval_since].last);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].acting[0]);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].up[0]);
+  }
+
+  //
+  // PG is splitting
+  //
+  {
+    ceph::shared_ptr osdmap(new OSDMap());
+    osdmap->set_max_osd(10);
+    osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
+    osdmap->set_epoch(epoch);
+    int new_pg_num = pg_num ^ 2;
+    OSDMap::Incremental inc(epoch + 1);
+    inc.new_pools[pool_id].min_size = min_size;
+    inc.new_pools[pool_id].set_pg_num(new_pg_num);
+    osdmap->apply_incremental(inc);
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_EQ(same_interval_since, past_intervals[same_interval_since].first);
+    ASSERT_EQ(osdmap->get_epoch() - 1, past_intervals[same_interval_since].last);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].acting[0]);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].up[0]);
+  }
+
+  //
+  // PG size has changed
+  //
+  {
+    ceph::shared_ptr osdmap(new OSDMap());
+    osdmap->set_max_osd(10);
+    osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
+    osdmap->set_epoch(epoch);
+    OSDMap::Incremental inc(epoch + 1);
+    __u8 new_min_size = min_size + 1;
+    inc.new_pools[pool_id].min_size = new_min_size;
+    inc.new_pools[pool_id].set_pg_num(pg_num);
+    osdmap->apply_incremental(inc);
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_EQ(same_interval_since, past_intervals[same_interval_since].first);
+    ASSERT_EQ(osdmap->get_epoch() - 1, past_intervals[same_interval_since].last);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].acting[0]);
+    ASSERT_EQ(osd_id, past_intervals[same_interval_since].up[0]);
+  }
+
+  //
+  // The old acting set was empty : the previous interval could not
+  // have been rw
+  //
+  {
+    vector old_acting;
+
+    map past_intervals;
+
+    ostringstream out;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals,
+						  &out));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_FALSE(past_intervals[same_interval_since].maybe_went_rw);
+    ASSERT_NE(string::npos, out.str().find("acting set is too small"));
+  }
+
+  //
+  // The old acting set did not have enough osd : it could
+  // not have been rw
+  //
+  {
+    vector old_acting;
+    old_acting.push_back(osd_id);
+
+    //
+    // see http://tracker.ceph.com/issues/5780
+    // the size of the old acting set should be compared
+    // with the min_size of the old osdmap
+    //
+    // The new osdmap is created so that it triggers the
+    // bug.
+    //
+    ceph::shared_ptr osdmap(new OSDMap());
+    osdmap->set_max_osd(10);
+    osdmap->set_state(osd_id, CEPH_OSD_EXISTS);
+    osdmap->set_epoch(epoch);
+    OSDMap::Incremental inc(epoch + 1);
+    __u8 new_min_size = old_acting.size();
+    inc.new_pools[pool_id].min_size = new_min_size;
+    inc.new_pools[pool_id].set_pg_num(pg_num);
+    osdmap->apply_incremental(inc);
+
+    ostringstream out;
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals,
+						  &out));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_FALSE(past_intervals[same_interval_since].maybe_went_rw);
+    ASSERT_NE(string::npos, out.str().find("acting set is too small"));
+  }
+
+  //
+  // The acting set changes. The old acting set primary was up during the
+  // previous interval and may have been rw.
+  //
+  {
+    vector new_acting;
+    new_acting.push_back(osd_id + 4);
+    new_acting.push_back(osd_id + 5);
+
+    ostringstream out;
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals,
+						  &out));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_TRUE(past_intervals[same_interval_since].maybe_went_rw);
+    ASSERT_NE(string::npos, out.str().find("includes interval"));
+  }
+  //
+  // The acting set changes. The old acting set primary was not up
+  // during the old interval but last_epoch_clean is in the
+  // old interval and it may have been rw.
+  //
+  {
+    vector new_acting;
+    new_acting.push_back(osd_id + 4);
+    new_acting.push_back(osd_id + 5);
+
+    ceph::shared_ptr lastmap(new OSDMap());
+    lastmap->set_max_osd(10);
+    lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
+    lastmap->set_epoch(epoch);
+    OSDMap::Incremental inc(epoch + 1);
+    inc.new_pools[pool_id].min_size = min_size;
+    inc.new_pools[pool_id].set_pg_num(pg_num);
+    inc.new_up_thru[osd_id] = epoch - 10;
+    lastmap->apply_incremental(inc);
+
+    ostringstream out;
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals,
+						  &out));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_TRUE(past_intervals[same_interval_since].maybe_went_rw);
+    ASSERT_NE(string::npos, out.str().find("presumed to have been rw"));
+  }
+
+  //
+  // The acting set changes. The old acting set primary was not up
+  // during the old interval and last_epoch_clean is before the
+  // old interval : the previous interval could not possibly have
+  // been rw.
+  //
+  {
+    vector new_acting;
+    new_acting.push_back(osd_id + 4);
+    new_acting.push_back(osd_id + 5);
+
+    epoch_t last_epoch_clean = epoch - 10;
+
+    ceph::shared_ptr lastmap(new OSDMap());
+    lastmap->set_max_osd(10);
+    lastmap->set_state(osd_id, CEPH_OSD_EXISTS);
+    lastmap->set_epoch(epoch);
+    OSDMap::Incremental inc(epoch + 1);
+    inc.new_pools[pool_id].min_size = min_size;
+    inc.new_pools[pool_id].set_pg_num(pg_num);
+    inc.new_up_thru[osd_id] = last_epoch_clean;
+    lastmap->apply_incremental(inc);
+
+    ostringstream out;
+
+    map past_intervals;
+
+    ASSERT_TRUE(past_intervals.empty());
+    ASSERT_TRUE(pg_interval_t::check_new_interval(old_primary,
+						  new_primary,
+						  old_acting,
+						  new_acting,
+						  old_up_primary,
+						  new_up_primary,
+						  old_up,
+						  new_up,
+						  same_interval_since,
+						  last_epoch_clean,
+						  osdmap,
+						  lastmap,
+						  pool_id,
+						  pgid,
+						  &past_intervals,
+						  &out));
+    ASSERT_EQ((unsigned int)1, past_intervals.size());
+    ASSERT_FALSE(past_intervals[same_interval_since].maybe_went_rw);
+    ASSERT_NE(string::npos, out.str().find("does not include interval"));
+  }
+}
+
+TEST(pg_t, get_ancestor)
+{
+  ASSERT_EQ(pg_t(0, 0, -1), pg_t(16, 0, -1).get_ancestor(16));
+  ASSERT_EQ(pg_t(1, 0, -1), pg_t(17, 0, -1).get_ancestor(16));
+  ASSERT_EQ(pg_t(0, 0, -1), pg_t(16, 0, -1).get_ancestor(8));
+  ASSERT_EQ(pg_t(16, 0, -1), pg_t(16, 0, -1).get_ancestor(80));
+  ASSERT_EQ(pg_t(16, 0, -1), pg_t(16, 0, -1).get_ancestor(83));
+  ASSERT_EQ(pg_t(1, 0, -1), pg_t(1321, 0, -1).get_ancestor(123).get_ancestor(8));
+  ASSERT_EQ(pg_t(3, 0, -1), pg_t(1323, 0, -1).get_ancestor(123).get_ancestor(8));
+  ASSERT_EQ(pg_t(3, 0, -1), pg_t(1323, 0, -1).get_ancestor(8));
+}
+
+TEST(pg_t, split)
+{
+  pg_t pgid(0, 0, -1);
+  set s;
+  bool b;
+
+  s.clear();
+  b = pgid.is_split(1, 1, &s);
+  ASSERT_TRUE(!b);
+
+  s.clear();
+  b = pgid.is_split(2, 4, NULL);
+  ASSERT_TRUE(b);
+  b = pgid.is_split(2, 4, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(1u, s.size());
+  ASSERT_TRUE(s.count(pg_t(2, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(2, 8, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(3u, s.size());
+  ASSERT_TRUE(s.count(pg_t(2, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(4, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(6, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(3, 8, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(1u, s.size());
+  ASSERT_TRUE(s.count(pg_t(4, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(6, 8, NULL);
+  ASSERT_TRUE(!b);
+  b = pgid.is_split(6, 8, &s);
+  ASSERT_TRUE(!b);
+  ASSERT_EQ(0u, s.size());
+
+  pgid = pg_t(1, 0, -1);
+
+  s.clear();
+  b = pgid.is_split(2, 4, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(1u, s.size());
+  ASSERT_TRUE(s.count(pg_t(3, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(2, 6, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(2u, s.size());
+  ASSERT_TRUE(s.count(pg_t(3, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(5, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(2, 8, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(3u, s.size());
+  ASSERT_TRUE(s.count(pg_t(3, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(5, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(7, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(4, 8, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(1u, s.size());
+  ASSERT_TRUE(s.count(pg_t(5, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(3, 8, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(3u, s.size());
+  ASSERT_TRUE(s.count(pg_t(3, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(5, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(7, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(6, 8, &s);
+  ASSERT_TRUE(!b);
+  ASSERT_EQ(0u, s.size());
+
+  pgid = pg_t(3, 0, -1);
+
+  s.clear();
+  b = pgid.is_split(7, 8, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(1u, s.size());
+  ASSERT_TRUE(s.count(pg_t(7, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(7, 12, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(2u, s.size());
+  ASSERT_TRUE(s.count(pg_t(7, 0, -1)));
+  ASSERT_TRUE(s.count(pg_t(11, 0, -1)));
+
+  s.clear();
+  b = pgid.is_split(7, 11, &s);
+  ASSERT_TRUE(b);
+  ASSERT_EQ(1u, s.size());
+  ASSERT_TRUE(s.count(pg_t(7, 0, -1)));
+
+}
+
+TEST(pg_missing_t, constructor)
+{
+  pg_missing_t missing;
+  EXPECT_EQ((unsigned int)0, missing.num_missing());
+  EXPECT_FALSE(missing.have_missing());
+}
+
+TEST(pg_missing_t, have_missing)
+{
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  pg_missing_t missing;
+  EXPECT_FALSE(missing.have_missing());
+  missing.add(oid, eversion_t(), eversion_t());
+  EXPECT_TRUE(missing.have_missing());
+}
+
+TEST(pg_missing_t, swap)
+{
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  pg_missing_t missing;
+  EXPECT_FALSE(missing.have_missing());
+  missing.add(oid, eversion_t(), eversion_t());
+  EXPECT_TRUE(missing.have_missing());
+
+  pg_missing_t other;
+  EXPECT_FALSE(other.have_missing());
+
+  other.swap(missing);
+  EXPECT_FALSE(missing.have_missing());
+  EXPECT_TRUE(other.have_missing());
+}
+
+TEST(pg_missing_t, is_missing)
+{
+  // pg_missing_t::is_missing(const hobject_t& oid) const
+  {
+    hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+    pg_missing_t missing;
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add(oid, eversion_t(), eversion_t());
+    EXPECT_TRUE(missing.is_missing(oid));
+  }
+
+  // bool pg_missing_t::is_missing(const hobject_t& oid, eversion_t v) const
+  {
+    hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+    pg_missing_t missing;
+    eversion_t need(10,5);
+    EXPECT_FALSE(missing.is_missing(oid, eversion_t()));
+    missing.add(oid, need, eversion_t());
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_FALSE(missing.is_missing(oid, eversion_t()));
+    EXPECT_TRUE(missing.is_missing(oid, need));
+  }
+}
+
+TEST(pg_missing_t, have_old)
+{
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  pg_missing_t missing;
+  EXPECT_EQ(eversion_t(), missing.have_old(oid));
+  missing.add(oid, eversion_t(), eversion_t());
+  EXPECT_EQ(eversion_t(), missing.have_old(oid));
+  eversion_t have(1,1);
+  missing.revise_have(oid, have);
+  EXPECT_EQ(have, missing.have_old(oid));
+}
+
+TEST(pg_missing_t, add_next_event)
+{
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  hobject_t oid_other(object_t("other"), "key", 9123, 9456, 0, "");
+  eversion_t version(10,5);
+  eversion_t prior_version(3,4);
+  pg_log_entry_t sample_e(pg_log_entry_t::DELETE, oid, version, prior_version,
+			  0, osd_reqid_t(entity_name_t::CLIENT(777), 8, 999),
+			  utime_t(8,9));
+
+  // new object (MODIFY)
+  {
+    pg_missing_t missing;
+    pg_log_entry_t e = sample_e;
+
+    e.op = pg_log_entry_t::MODIFY;
+    e.prior_version = eversion_t();
+    EXPECT_TRUE(e.is_update());
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add_next_event(e);
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_EQ(eversion_t(), missing.missing[oid].have);
+    EXPECT_EQ(oid, missing.rmissing[e.version.version]);
+    EXPECT_EQ(1U, missing.num_missing());
+    EXPECT_EQ(1U, missing.rmissing.size());
+
+    // adding the same object replaces the previous one
+    missing.add_next_event(e);
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_EQ(1U, missing.num_missing());
+    EXPECT_EQ(1U, missing.rmissing.size());
+  }
+
+  // new object (CLONE)
+  {
+    pg_missing_t missing;
+    pg_log_entry_t e = sample_e;
+
+    e.op = pg_log_entry_t::CLONE;
+    e.prior_version = eversion_t();
+    EXPECT_TRUE(e.is_clone());
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add_next_event(e);
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_EQ(eversion_t(), missing.missing[oid].have);
+    EXPECT_EQ(oid, missing.rmissing[e.version.version]);
+    EXPECT_EQ(1U, missing.num_missing());
+    EXPECT_EQ(1U, missing.rmissing.size());
+
+    // adding the same object replaces the previous one
+    missing.add_next_event(e);
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_EQ(1U, missing.num_missing());
+    EXPECT_EQ(1U, missing.rmissing.size());
+  }
+
+  // existing object (MODIFY)
+  {
+    pg_missing_t missing;
+    pg_log_entry_t e = sample_e;
+
+    e.op = pg_log_entry_t::MODIFY;
+    e.prior_version = eversion_t();
+    EXPECT_TRUE(e.is_update());
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add_next_event(e);
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_EQ(eversion_t(), missing.missing[oid].have);
+    EXPECT_EQ(oid, missing.rmissing[e.version.version]);
+    EXPECT_EQ(1U, missing.num_missing());
+    EXPECT_EQ(1U, missing.rmissing.size());
+
+    // adding the same object with a different version
+    e.prior_version = prior_version;
+    missing.add_next_event(e);
+    EXPECT_EQ(eversion_t(), missing.missing[oid].have);
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_EQ(1U, missing.num_missing());
+    EXPECT_EQ(1U, missing.rmissing.size());
+  }
+
+  // object with prior version (MODIFY)
+  {
+    pg_missing_t missing;
+    pg_log_entry_t e = sample_e;
+
+    e.op = pg_log_entry_t::MODIFY;
+    EXPECT_TRUE(e.is_update());
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add_next_event(e);
+    EXPECT_TRUE(missing.is_missing(oid));
+    EXPECT_EQ(prior_version, missing.missing[oid].have);
+    EXPECT_EQ(version, missing.missing[oid].need);
+    EXPECT_EQ(oid, missing.rmissing[e.version.version]);
+    EXPECT_EQ(1U, missing.num_missing());
+    EXPECT_EQ(1U, missing.rmissing.size());
+  }
+
+  // obsolete (BACKLOG)
+  {
+    pg_missing_t missing;
+    pg_log_entry_t e = sample_e;
+
+    e.op = pg_log_entry_t::BACKLOG;
+    EXPECT_TRUE(e.is_backlog());
+    EXPECT_FALSE(missing.is_missing(oid));
+    EXPECT_THROW(missing.add_next_event(e), FailedAssertion);
+  }
+
+  // adding a DELETE matching an existing event
+  {
+    pg_missing_t missing;
+    pg_log_entry_t e = sample_e;
+
+    e.op = pg_log_entry_t::MODIFY;
+    EXPECT_TRUE(e.is_update());
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add_next_event(e);
+    EXPECT_TRUE(missing.is_missing(oid));
+
+    e.op = pg_log_entry_t::DELETE;
+    EXPECT_TRUE(e.is_delete());
+    missing.add_next_event(e);
+    EXPECT_FALSE(missing.have_missing());
+  }
+}
+
+TEST(pg_missing_t, revise_need)
+{
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  pg_missing_t missing;
+  // create a new entry
+  EXPECT_FALSE(missing.is_missing(oid));
+  eversion_t need(10,10);
+  missing.revise_need(oid, need);
+  EXPECT_TRUE(missing.is_missing(oid));
+  EXPECT_EQ(eversion_t(), missing.missing[oid].have);
+  EXPECT_EQ(need, missing.missing[oid].need);
+  // update an existing entry and preserve have
+  eversion_t have(1,1);
+  missing.revise_have(oid, have);
+  eversion_t new_need(10,12);
+  EXPECT_EQ(have, missing.missing[oid].have);
+  missing.revise_need(oid, new_need);
+  EXPECT_EQ(have, missing.missing[oid].have);
+  EXPECT_EQ(new_need, missing.missing[oid].need);
+}
+
+TEST(pg_missing_t, revise_have)
+{
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  pg_missing_t missing;
+  // a non existing entry means noop
+  EXPECT_FALSE(missing.is_missing(oid));
+  eversion_t have(1,1);
+  missing.revise_have(oid, have);
+  EXPECT_FALSE(missing.is_missing(oid));
+  // update an existing entry
+  eversion_t need(10,12);
+  missing.add(oid, need, have);
+  EXPECT_TRUE(missing.is_missing(oid));
+  eversion_t new_have(2,2);
+  EXPECT_EQ(have, missing.missing[oid].have);
+  missing.revise_have(oid, new_have);
+  EXPECT_EQ(new_have, missing.missing[oid].have);
+  EXPECT_EQ(need, missing.missing[oid].need);
+}
+
+TEST(pg_missing_t, add)
+{
+  hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+  pg_missing_t missing;
+  EXPECT_FALSE(missing.is_missing(oid));
+  eversion_t have(1,1);
+  eversion_t need(10,10);
+  missing.add(oid, need, have);
+  EXPECT_TRUE(missing.is_missing(oid));
+  EXPECT_EQ(have, missing.missing[oid].have);
+  EXPECT_EQ(need, missing.missing[oid].need);
+}
+
+TEST(pg_missing_t, rm)
+{
+  // void pg_missing_t::rm(const hobject_t& oid, eversion_t v)
+  {
+    hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+    pg_missing_t missing;
+    EXPECT_FALSE(missing.is_missing(oid));
+    epoch_t epoch = 10;
+    eversion_t need(epoch,10);
+    missing.add(oid, need, eversion_t());
+    EXPECT_TRUE(missing.is_missing(oid));
+    // rm of an older version is a noop
+    missing.rm(oid, eversion_t(epoch / 2,20));
+    EXPECT_TRUE(missing.is_missing(oid));
+    // rm of a later version removes the object
+    missing.rm(oid, eversion_t(epoch * 2,20));
+    EXPECT_FALSE(missing.is_missing(oid));
+  }
+  // void pg_missing_t::rm(const std::map::iterator &m)
+  {
+    hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+    pg_missing_t missing;
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add(oid, eversion_t(), eversion_t());
+    EXPECT_TRUE(missing.is_missing(oid));
+    const std::map::iterator m = missing.missing.find(oid);
+    missing.rm(m);
+    EXPECT_FALSE(missing.is_missing(oid));
+  }
+}
+
+TEST(pg_missing_t, got)
+{
+  // void pg_missing_t::got(const hobject_t& oid, eversion_t v)
+  {
+    hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+    pg_missing_t missing;
+    // assert if the oid does not exist
+    EXPECT_THROW(missing.got(oid, eversion_t()), FailedAssertion);
+    EXPECT_FALSE(missing.is_missing(oid));
+    epoch_t epoch = 10;
+    eversion_t need(epoch,10);
+    missing.add(oid, need, eversion_t());
+    EXPECT_TRUE(missing.is_missing(oid));
+    // assert if that the version to be removed is lower than the version of the object
+    EXPECT_THROW(missing.got(oid, eversion_t(epoch / 2,20)), FailedAssertion);
+    // remove of a later version removes the object
+    missing.got(oid, eversion_t(epoch * 2,20));
+    EXPECT_FALSE(missing.is_missing(oid));
+  }
+  // void pg_missing_t::got(const std::map::iterator &m)
+  {
+    hobject_t oid(object_t("objname"), "key", 123, 456, 0, "");
+    pg_missing_t missing;
+    EXPECT_FALSE(missing.is_missing(oid));
+    missing.add(oid, eversion_t(), eversion_t());
+    EXPECT_TRUE(missing.is_missing(oid));
+    const std::map::iterator m = missing.missing.find(oid);
+    missing.got(m);
+    EXPECT_FALSE(missing.is_missing(oid));
+  }
+}
+
+TEST(pg_missing_t, split_into)
+{
+  uint32_t hash1 = 1;
+  hobject_t oid1(object_t("objname"), "key1", 123, hash1, 0, "");
+  uint32_t hash2 = 2;
+  hobject_t oid2(object_t("objname"), "key2", 123, hash2, 0, "");
+  pg_missing_t missing;
+  missing.add(oid1, eversion_t(), eversion_t());
+  missing.add(oid2, eversion_t(), eversion_t());
+  pg_t child_pgid;
+  child_pgid.m_seed = 1;
+  pg_missing_t child;
+  unsigned split_bits = 1;
+  missing.split_into(child_pgid, split_bits, &child);
+  EXPECT_TRUE(child.is_missing(oid1));
+  EXPECT_FALSE(child.is_missing(oid2));
+  EXPECT_FALSE(missing.is_missing(oid1));
+  EXPECT_TRUE(missing.is_missing(oid2));
+}
+
+class ObjectContextTest : public ::testing::Test {
+protected:
+
+  static const useconds_t DELAY_MAX = 20 * 1000 * 1000;
+
+  class Thread_read_lock : public Thread {
+  public:
+    ObjectContext &obc;
+
+    Thread_read_lock(ObjectContext& _obc) :
+      obc(_obc)
+    {
+    }
+
+    virtual void *entry() {
+      obc.ondisk_read_lock();
+      return NULL;
+    }
+  };
+
+  class Thread_write_lock : public Thread {
+  public:
+    ObjectContext &obc;
+
+    Thread_write_lock(ObjectContext& _obc) :
+      obc(_obc)
+    {
+    }
+
+    virtual void *entry() {
+      obc.ondisk_write_lock();
+      return NULL;
+    }
+  };
+
+};
+
+TEST_F(ObjectContextTest, read_write_lock)
+{
+  {
+    ObjectContext obc;
+
+    //
+    // write_lock
+    // write_lock
+    // write_unlock
+    // write_unlock
+    //
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    obc.ondisk_write_lock();
+
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(1, obc.unstable_writes);
+
+    obc.ondisk_write_lock();
+
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(2, obc.unstable_writes);
+
+    obc.ondisk_write_unlock();
+
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(1, obc.unstable_writes);
+
+    obc.ondisk_write_unlock();
+
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+  }
+
+  useconds_t delay = 0;
+
+  {
+    ObjectContext obc;
+
+    //
+    // write_lock
+    // read_lock => wait
+    // write_unlock => signal
+    // read_unlock
+    //
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(0, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    obc.ondisk_write_lock();
+
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(0, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(1, obc.unstable_writes);
+
+    Thread_read_lock t(obc);
+    t.create();
+
+    do {
+      cout << "Trying (1) with delay " << delay << "us\n";
+      usleep(delay);
+    } while (obc.readers_waiting == 0 &&
+	     ( delay = delay * 2 + 1) < DELAY_MAX);
+
+    EXPECT_EQ(1, obc.readers_waiting);
+    EXPECT_EQ(0, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(1, obc.unstable_writes);
+
+    obc.ondisk_write_unlock();
+
+    do {
+      cout << "Trying (2) with delay " << delay << "us\n";
+      usleep(delay);
+    } while ((obc.readers == 0 || obc.readers_waiting == 1) &&
+	     ( delay = delay * 2 + 1) < DELAY_MAX);
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(1, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    obc.ondisk_read_unlock();
+
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(0, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    t.join();
+  }
+
+  {
+    ObjectContext obc;
+
+    //
+    // read_lock
+    // write_lock => wait
+    // read_unlock => signal
+    // write_unlock
+    //
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(0, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    obc.ondisk_read_lock();
+
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(1, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    Thread_write_lock t(obc);
+    t.create();
+
+    do {
+      cout << "Trying (3) with delay " << delay << "us\n";
+      usleep(delay);
+    } while ((obc.writers_waiting == 0) &&
+	     ( delay = delay * 2 + 1) < DELAY_MAX);
+
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(1, obc.readers);
+    EXPECT_EQ(1, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    obc.ondisk_read_unlock();
+
+    do {
+      cout << "Trying (4) with delay " << delay << "us\n";
+      usleep(delay);
+    } while ((obc.unstable_writes == 0 || obc.writers_waiting == 1) &&
+	     ( delay = delay * 2 + 1) < DELAY_MAX);
+
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(0, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(1, obc.unstable_writes);
+
+    obc.ondisk_write_unlock();
+
+    EXPECT_EQ(0, obc.readers_waiting);
+    EXPECT_EQ(0, obc.readers);
+    EXPECT_EQ(0, obc.writers_waiting);
+    EXPECT_EQ(0, obc.unstable_writes);
+
+    t.join();
+  }
+
+}
+
+TEST(pg_pool_t_test, get_pg_num_divisor) {
+  pg_pool_t p;
+  p.set_pg_num(16);
+  p.set_pgp_num(16);
+
+  for (int i = 0; i < 16; ++i)
+    ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(i, 1)));
+
+  p.set_pg_num(12);
+  p.set_pgp_num(12);
+  //cout << "num " << p.get_pg_num()
+  //     << " mask " << p.get_pg_num_mask() << std::endl;
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(0, 1)));
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(1, 1)));
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(2, 1)));
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(3, 1)));
+  ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(4, 1)));
+  ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(5, 1)));
+  ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(6, 1)));
+  ASSERT_EQ(8u, p.get_pg_num_divisor(pg_t(7, 1)));
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(8, 1)));
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(9, 1)));
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(10, 1)));
+  ASSERT_EQ(16u, p.get_pg_num_divisor(pg_t(11, 1)));
+}
+
+TEST(pg_pool_t_test, get_random_pg_position) {
+  srand(getpid());
+  for (int i = 0; i < 100; ++i) {
+    pg_pool_t p;
+    p.set_pg_num(1 + (rand() % 1000));
+    p.set_pgp_num(p.get_pg_num());
+    pg_t pgid(rand() % p.get_pg_num(), 1);
+    uint32_t h = p.get_random_pg_position(pgid, rand());
+    uint32_t ps = p.raw_hash_to_pg(h);
+    cout << p.get_pg_num() << " " << pgid << ": "
+	 << h << " -> " << pg_t(ps, 1) << std::endl;
+    ASSERT_EQ(pgid.ps(), ps);
+  }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd .. ;
+ *   make unittest_osd_types ;
+ *   ./unittest_osd_types # --gtest_filter=pg_missing_t.constructor
+ * "
+ * End:
+ */
diff --git a/ceph/src/test/osdc/FakeWriteback.cc b/ceph/src/test/osdc/FakeWriteback.cc
new file mode 100644
index 00000000..7c678cf4
--- /dev/null
+++ b/ceph/src/test/osdc/FakeWriteback.cc
@@ -0,0 +1,87 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include 
+#include 
+
+#include "common/debug.h"
+#include "common/Cond.h"
+#include "common/Finisher.h"
+#include "common/Mutex.h"
+#include "include/assert.h"
+#include "include/utime.h"
+
+#include "FakeWriteback.h"
+
+#define dout_subsys ceph_subsys_objectcacher
+#undef dout_prefix
+#define dout_prefix *_dout << "FakeWriteback(" << this << ") "
+
+class C_Delay : public Context {
+  CephContext *m_cct;
+  Context *m_con;
+  utime_t m_delay;
+  Mutex *m_lock;
+  bufferlist *m_bl;
+  uint64_t m_off;
+
+public:
+  C_Delay(CephContext *cct, Context *c, Mutex *lock, uint64_t off,
+	  bufferlist *pbl, uint64_t delay_ns=0)
+    : m_cct(cct), m_con(c), m_delay(0, delay_ns), m_lock(lock), m_bl(pbl), m_off(off) {}
+  void finish(int r) {
+    struct timespec delay;
+    m_delay.to_timespec(&delay);
+    nanosleep(&delay, NULL);
+    if (m_bl) {
+      buffer::ptr bp(r);
+      bp.zero();
+      m_bl->append(bp);
+      ldout(m_cct, 20) << "finished read " << m_off << "~" << r << dendl;
+    }
+    m_lock->Lock();
+    m_con->complete(r);
+    m_lock->Unlock();
+  }
+};
+
+FakeWriteback::FakeWriteback(CephContext *cct, Mutex *lock, uint64_t delay_ns)
+  : m_cct(cct), m_lock(lock), m_delay_ns(delay_ns)
+{
+  m_finisher = new Finisher(cct);
+  m_finisher->start();
+}
+
+FakeWriteback::~FakeWriteback()
+{
+  m_finisher->stop();
+  delete m_finisher;
+}
+
+void FakeWriteback::read(const object_t& oid,
+			 const object_locator_t& oloc,
+			 uint64_t off, uint64_t len, snapid_t snapid,
+			 bufferlist *pbl, uint64_t trunc_size,
+			 __u32 trunc_seq, Context *onfinish)
+{
+  C_Delay *wrapper = new C_Delay(m_cct, onfinish, m_lock, off, pbl, m_delay_ns);
+  m_finisher->queue(wrapper, len);
+}
+
+ceph_tid_t FakeWriteback::write(const object_t& oid,
+			   const object_locator_t& oloc,
+			   uint64_t off, uint64_t len,
+			   const SnapContext& snapc,
+			   const bufferlist &bl, utime_t mtime,
+			   uint64_t trunc_size, __u32 trunc_seq,
+			   Context *oncommit)
+{
+  C_Delay *wrapper = new C_Delay(m_cct, oncommit, m_lock, off, NULL, m_delay_ns);;
+  m_finisher->queue(wrapper, 0);
+  return m_tid.inc();
+}
+
+bool FakeWriteback::may_copy_on_write(const object_t&, uint64_t, uint64_t, snapid_t)
+{
+  return false;
+}
diff --git a/ceph/src/test/osdc/FakeWriteback.h b/ceph/src/test/osdc/FakeWriteback.h
new file mode 100644
index 00000000..2b7fbd67
--- /dev/null
+++ b/ceph/src/test/osdc/FakeWriteback.h
@@ -0,0 +1,40 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_TEST_OSDC_FAKEWRITEBACK_H
+#define CEPH_TEST_OSDC_FAKEWRITEBACK_H
+
+#include "include/atomic.h"
+#include "include/Context.h"
+#include "include/types.h"
+#include "osd/osd_types.h"
+#include "osdc/WritebackHandler.h"
+
+class Finisher;
+class Mutex;
+
+class FakeWriteback : public WritebackHandler {
+public:
+  FakeWriteback(CephContext *cct, Mutex *lock, uint64_t delay_ns);
+  virtual ~FakeWriteback();
+
+  virtual void read(const object_t& oid, const object_locator_t& oloc,
+		    uint64_t off, uint64_t len, snapid_t snapid,
+		    bufferlist *pbl, uint64_t trunc_size,  __u32 trunc_seq,
+		    Context *onfinish);
+
+  virtual ceph_tid_t write(const object_t& oid, const object_locator_t& oloc,
+		           uint64_t off, uint64_t len,
+			   const SnapContext& snapc, const bufferlist &bl,
+			   utime_t mtime, uint64_t trunc_size,
+			   __u32 trunc_seq, Context *oncommit);
+
+  virtual bool may_copy_on_write(const object_t&, uint64_t, uint64_t, snapid_t);
+private:
+  CephContext *m_cct;
+  Mutex *m_lock;
+  uint64_t m_delay_ns;
+  atomic_t m_tid;
+  Finisher *m_finisher;
+};
+
+#endif
diff --git a/ceph/src/test/osdc/object_cacher_stress.cc b/ceph/src/test/osdc/object_cacher_stress.cc
new file mode 100644
index 00000000..4f6fffe0
--- /dev/null
+++ b/ceph/src/test/osdc/object_cacher_stress.cc
@@ -0,0 +1,233 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/ceph_argparse.h"
+#include "common/common_init.h"
+#include "common/config.h"
+#include "common/Mutex.h"
+#include "common/snap_types.h"
+#include "global/global_init.h"
+#include "include/atomic.h"
+#include "include/buffer.h"
+#include "include/Context.h"
+#include "include/stringify.h"
+#include "osdc/ObjectCacher.h"
+
+#include "FakeWriteback.h"
+
+// XXX: Only tests default namespace
+struct op_data {
+  op_data(std::string oid, uint64_t offset, uint64_t len, bool read)
+    : extent(oid, 0, offset, len, 0), is_read(read)
+  {
+    extent.oloc.pool = 0;
+    extent.buffer_extents.push_back(make_pair(0, len));
+  }
+
+  ObjectExtent extent;
+  bool is_read;
+  ceph::bufferlist result;
+  atomic_t done;
+};
+
+class C_Count : public Context {
+  op_data *m_op;
+  atomic_t *m_outstanding;
+public:
+  C_Count(op_data *op, atomic_t *outstanding)
+    : m_op(op), m_outstanding(outstanding) {}
+  void finish(int r) {
+    m_op->done.inc();
+    assert(m_outstanding->read() > 0);
+    m_outstanding->dec();
+  }
+};
+
+int stress_test(uint64_t num_ops, uint64_t num_objs,
+		uint64_t max_obj_size, uint64_t delay_ns,
+		uint64_t max_op_len, float percent_reads)
+{
+  Mutex lock("object_cacher_stress::object_cacher");
+  FakeWriteback writeback(g_ceph_context, &lock, delay_ns);
+
+  ObjectCacher obc(g_ceph_context, "test", writeback, lock, NULL, NULL,
+		   g_conf->client_oc_size,
+		   g_conf->client_oc_max_objects,
+		   g_conf->client_oc_max_dirty,
+		   g_conf->client_oc_target_dirty,
+		   g_conf->client_oc_max_dirty_age,
+		   true);
+  obc.start();
+
+  atomic_t outstanding_reads;
+  vector > ops;
+  ObjectCacher::ObjectSet object_set(NULL, 0, 0);
+  SnapContext snapc;
+  ceph::buffer::ptr bp(max_op_len);
+  ceph::bufferlist bl;
+  bp.zero();
+  bl.append(bp);
+
+  // schedule ops
+  std::cout << "Test configuration:\n\n"
+	    << setw(10) << "ops: " << num_ops << "\n"
+	    << setw(10) << "objects: " << num_objs << "\n"
+	    << setw(10) << "obj size: " << max_obj_size << "\n"
+	    << setw(10) << "delay: " << delay_ns << "\n"
+	    << setw(10) << "max op len: " << max_op_len << "\n"
+	    << setw(10) << "percent reads: " << percent_reads << "\n\n";
+
+  for (uint64_t i = 0; i < num_ops; ++i) {
+    uint64_t offset = random() % max_obj_size;
+    uint64_t max_len = MIN(max_obj_size - offset, max_op_len);
+    // no zero-length operations
+    uint64_t length = random() % (MAX(max_len - 1, 1)) + 1;
+    std::string oid = "test" + stringify(random() % num_objs);
+    bool is_read = random() < percent_reads * RAND_MAX;
+    ceph::shared_ptr op(new op_data(oid, offset, length, is_read));
+    ops.push_back(op);
+    std::cout << "op " << i << " " << (is_read ? "read" : "write")
+	      << " " << op->extent << "\n";
+    if (op->is_read) {
+      ObjectCacher::OSDRead *rd = obc.prepare_read(CEPH_NOSNAP, &op->result, 0);
+      rd->extents.push_back(op->extent);
+      outstanding_reads.inc();
+      Context *completion = new C_Count(op.get(), &outstanding_reads);
+      lock.Lock();
+      int r = obc.readx(rd, &object_set, completion);
+      lock.Unlock();
+      assert(r >= 0);
+      if ((uint64_t)r == length)
+	completion->complete(r);
+      else
+	assert(r == 0);
+    } else {
+      ObjectCacher::OSDWrite *wr = obc.prepare_write(snapc, bl, utime_t(), 0);
+      wr->extents.push_back(op->extent);
+      lock.Lock();
+      obc.writex(wr, &object_set, lock, NULL);
+      lock.Unlock();
+    }
+  }
+
+  // check that all reads completed
+  for (uint64_t i = 0; i < num_ops; ++i) {
+    if (!ops[i]->is_read)
+      continue;
+    std::cout << "waiting for read " << i << ops[i]->extent << std::endl;
+    uint64_t done = 0;
+    while (done == 0) {
+      done = ops[i]->done.read();
+      if (!done) {
+	usleep(500);
+      }
+    }
+    if (done > 1) {
+      std::cout << "completion called more than once!\n" << std::endl;
+      return EXIT_FAILURE;
+    }
+  }
+
+  lock.Lock();
+  obc.release_set(&object_set);
+  lock.Unlock();
+
+  int r = 0;
+  Mutex mylock("librbd::ImageCtx::flush_cache");
+  Cond cond;
+  bool done;
+  Context *onfinish = new C_SafeCond(&mylock, &cond, &done, &r);
+  lock.Lock();
+  bool already_flushed = obc.flush_set(&object_set, onfinish);
+  std::cout << "already flushed = " << already_flushed << std::endl;
+  lock.Unlock();
+  mylock.Lock();
+  while (!done) {
+    cond.Wait(mylock);
+  }
+  mylock.Unlock();
+
+  lock.Lock();
+  bool unclean = obc.release_set(&object_set);
+  lock.Unlock();
+
+  if (unclean) {
+    std::cout << "unclean buffers left over!" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  obc.stop();
+
+  std::cout << "Test completed successfully." << std::endl;
+
+  return EXIT_SUCCESS;
+}
+
+int main(int argc, const char **argv)
+{
+  std::vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+
+  long long delay_ns = 0;
+  long long num_ops = 1000;
+  long long obj_bytes = 4 << 20;
+  long long max_len = 128 << 10;
+  long long num_objs = 10;
+  float percent_reads = 0.90;
+  int seed = time(0) % 100000;
+  std::ostringstream err;
+  std::vector::iterator i;
+  for (i = args.begin(); i != args.end();) {
+    if (ceph_argparse_withlonglong(args, i, &delay_ns, &err, "--delay-ns", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << argv[0] << ": " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_withlonglong(args, i, &num_ops, &err, "--ops", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << argv[0] << ": " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_withlonglong(args, i, &num_objs, &err, "--objects", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << argv[0] << ": " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_withlonglong(args, i, &obj_bytes, &err, "--obj-size", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << argv[0] << ": " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_withlonglong(args, i, &max_len, &err, "--max-op-size", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << argv[0] << ": " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_withfloat(args, i, &percent_reads, &err, "--percent-read", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << argv[0] << ": " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else if (ceph_argparse_withint(args, i, &seed, &err, "--seed", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << argv[0] << ": " << err.str() << std::endl;
+	return EXIT_FAILURE;
+      }
+    } else {
+      cerr << "unknown option " << *i << std::endl;
+      return EXIT_FAILURE;
+    }
+  }
+
+  srandom(seed);
+  return stress_test(num_ops, num_objs, obj_bytes, delay_ns, max_len, percent_reads);
+}
diff --git a/ceph/src/test/perf_counters.cc b/ceph/src/test/perf_counters.cc
new file mode 100644
index 00000000..c44a15ef
--- /dev/null
+++ b/ceph/src/test/perf_counters.cc
@@ -0,0 +1,165 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+#include "include/types.h" // FIXME: ordering shouldn't be important, but right 
+                           // now, this include has to come before the others.
+
+
+#include "common/perf_counters.h"
+#include "common/admin_socket_client.h"
+#include "common/ceph_context.h"
+#include "common/config.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+
+#include "common/code_environment.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "include/msgr.h" // for CEPH_ENTITY_TYPE_CLIENT
+#include "gtest/gtest.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/common_init.h"
+
+int main(int argc, char **argv) {
+  std::vector preargs;
+  preargs.push_back("--admin-socket");
+  preargs.push_back(get_rand_socket_path());
+  std::vector args;
+  global_init(&preargs, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+TEST(PerfCounters, SimpleTest) {
+  AdminSocketClient client(get_rand_socket_path());
+  std::string message;
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\" }", &message));
+  ASSERT_EQ("{}", message);
+}
+
+enum {
+  TEST_PERFCOUNTERS1_ELEMENT_FIRST = 200,
+  TEST_PERFCOUNTERS1_ELEMENT_1,
+  TEST_PERFCOUNTERS1_ELEMENT_2,
+  TEST_PERFCOUNTERS1_ELEMENT_3,
+  TEST_PERFCOUNTERS1_ELEMENT_LAST,
+};
+
+std::string sd(const char *c)
+{
+  std::string ret(c);
+  std::string::size_type sz = ret.size();
+  for (std::string::size_type i = 0; i < sz; ++i) {
+    if (ret[i] == '\'') {
+      ret[i] = '\"';
+    }
+  }
+  return ret;
+}
+
+static PerfCounters* setup_test_perfcounters1(CephContext *cct)
+{
+  PerfCountersBuilder bld(cct, "test_perfcounter_1",
+	  TEST_PERFCOUNTERS1_ELEMENT_FIRST, TEST_PERFCOUNTERS1_ELEMENT_LAST);
+  bld.add_u64(TEST_PERFCOUNTERS1_ELEMENT_1, "element1");
+  bld.add_time(TEST_PERFCOUNTERS1_ELEMENT_2, "element2");
+  bld.add_time_avg(TEST_PERFCOUNTERS1_ELEMENT_3, "element3");
+  return bld.create_perf_counters();
+}
+
+TEST(PerfCounters, SinglePerfCounters) {
+  PerfCountersCollection *coll = g_ceph_context->get_perfcounters_collection();
+  PerfCounters* fake_pf = setup_test_perfcounters1(g_ceph_context);
+  coll->add(fake_pf);
+  AdminSocketClient client(get_rand_socket_path());
+  std::string msg;
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":0,"
+	    "\"element2\":0.000000000,\"element3\":{\"avgcount\":0,\"sum\":0.000000000}}}"), msg);
+  fake_pf->inc(TEST_PERFCOUNTERS1_ELEMENT_1);
+  fake_pf->tset(TEST_PERFCOUNTERS1_ELEMENT_2, utime_t(0, 500000000));
+  fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(100, 0));
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1,"
+	    "\"element2\":0.500000000,\"element3\":{\"avgcount\":1,\"sum\":100.000000000}}}"), msg);
+  fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t());
+  fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(25,0));
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1,\"element2\":0.500000000,"
+	    "\"element3\":{\"avgcount\":3,\"sum\":125.000000000}}}"), msg);
+}
+
+enum {
+  TEST_PERFCOUNTERS2_ELEMENT_FIRST = 400,
+  TEST_PERFCOUNTERS2_ELEMENT_FOO,
+  TEST_PERFCOUNTERS2_ELEMENT_BAR,
+  TEST_PERFCOUNTERS2_ELEMENT_LAST,
+};
+
+static PerfCounters* setup_test_perfcounter2(CephContext *cct)
+{
+  PerfCountersBuilder bld(cct, "test_perfcounter_2",
+	  TEST_PERFCOUNTERS2_ELEMENT_FIRST, TEST_PERFCOUNTERS2_ELEMENT_LAST);
+  bld.add_u64(TEST_PERFCOUNTERS2_ELEMENT_FOO, "foo");
+  bld.add_time(TEST_PERFCOUNTERS2_ELEMENT_BAR, "bar");
+  return bld.create_perf_counters();
+}
+
+TEST(PerfCounters, MultiplePerfCounters) {
+  PerfCountersCollection *coll = g_ceph_context->get_perfcounters_collection();
+  coll->clear();
+  PerfCounters* fake_pf1 = setup_test_perfcounters1(g_ceph_context);
+  PerfCounters* fake_pf2 = setup_test_perfcounter2(g_ceph_context);
+  coll->add(fake_pf1);
+  coll->add(fake_pf2);
+  AdminSocketClient client(get_rand_socket_path());
+  std::string msg;
+
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":0,\"element2\":0.000000000,\"element3\":"
+	    "{\"avgcount\":0,\"sum\":0.000000000}},\"test_perfcounter_2\":{\"foo\":0,\"bar\":0.000000000}}"), msg);
+
+  fake_pf1->inc(TEST_PERFCOUNTERS1_ELEMENT_1);
+  fake_pf1->inc(TEST_PERFCOUNTERS1_ELEMENT_1, 5);
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":6,\"element2\":0.000000000,\"element3\":"
+	    "{\"avgcount\":0,\"sum\":0.000000000}},\"test_perfcounter_2\":{\"foo\":0,\"bar\":0.000000000}}"), msg);
+
+  coll->remove(fake_pf2);
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":6,\"element2\":0.000000000,"
+	    "\"element3\":{\"avgcount\":0,\"sum\":0.000000000}}}"), msg);
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perf schema\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":{\"type\":2},"
+	       "\"element2\":{\"type\":1},\"element3\":{\"type\":5}}}"), msg);
+  coll->clear();
+  ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg));
+  ASSERT_EQ("{}", msg);
+}
diff --git a/ceph/src/test/rgw/test_rgw_manifest.cc b/ceph/src/test/rgw/test_rgw_manifest.cc
new file mode 100644
index 00000000..502eaf9d
--- /dev/null
+++ b/ceph/src/test/rgw/test_rgw_manifest.cc
@@ -0,0 +1,227 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include 
+#include "global/global_init.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#define GTEST
+#ifdef GTEST
+#include 
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+using namespace std;
+
+static void init_bucket(rgw_bucket *bucket, const char *name)
+{
+  *bucket = rgw_bucket(name, ".data-pool", ".index-pool", "marker.", "bucket-id", NULL);
+}
+
+void append_head(list *objs, rgw_obj& head)
+{
+  objs->push_back(head);
+}
+
+void append_stripes(list *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size)
+{
+  string prefix = manifest.get_prefix();
+  rgw_bucket bucket = manifest.get_head().bucket;
+
+  int i = 0;
+  for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) {
+    char buf[16];
+    snprintf(buf, sizeof(buf), "%d", ++i);
+    string oid = prefix + buf;
+  cout << "oid=" << oid << std::endl;
+    rgw_obj obj;
+    obj.init_ns(bucket, oid, "shadow");
+    objs->push_back(obj);
+  }
+}
+
+static void gen_obj(uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
+                    RGWObjManifest *manifest, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen,
+                    list *test_objs)
+{
+  manifest->set_trivial_rule(head_max_size, stripe_size);
+
+  init_bucket(bucket, "buck");
+
+  *head = rgw_obj(*bucket, "oid");
+  gen->create_begin(g_ceph_context, manifest, *bucket, *head);
+
+  append_head(test_objs, *head);
+  cout << "test_objs.size()=" << test_objs->size() << std::endl;
+  append_stripes(test_objs, *manifest, obj_size, stripe_size);
+
+  cout << "test_objs.size()=" << test_objs->size() << std::endl;
+
+  ASSERT_EQ((int)manifest->get_obj_size(), 0);
+  ASSERT_EQ((int)manifest->get_head_size(), 0);
+  ASSERT_EQ(manifest->has_tail(), false);
+
+  uint64_t ofs = 0;
+  list::iterator iter = test_objs->begin();
+
+  while (ofs < obj_size) {
+    rgw_obj obj = gen->get_cur_obj();
+cout << "obj=" << obj << std::endl;
+    ASSERT_TRUE(obj == *iter);
+
+    ofs = MIN(ofs + gen->cur_stripe_max_size(), obj_size);
+    gen->create_next(ofs);
+
+  cout << "obj=" << obj << " *iter=" << *iter << std::endl;
+  cout << "test_objs.size()=" << test_objs->size() << std::endl;
+    ++iter;
+
+  }
+
+  if (manifest->has_tail()) {
+    rgw_obj obj = gen->get_cur_obj();
+    ASSERT_TRUE(obj == *iter);
+    ++iter;
+  }
+  ASSERT_TRUE(iter == test_objs->end());
+  ASSERT_EQ(manifest->get_obj_size(), obj_size);
+  ASSERT_EQ(manifest->get_head_size(), MIN(obj_size, head_max_size));
+  ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size));
+}
+
+TEST(TestRGWManifest, head_only_obj) {
+  RGWObjManifest manifest;
+  rgw_bucket bucket;
+  rgw_obj head;
+  RGWObjManifest::generator gen;
+
+  int obj_size = 256 * 1024;
+
+  list objs;
+
+  gen_obj(obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, &bucket, &head, &gen, &objs);
+
+  cout <<  " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl;
+  cout <<  " manifest.get_head_size()=" << manifest.get_head_size() << std::endl;
+  list::iterator liter;
+
+  RGWObjManifest::obj_iterator iter;
+  for (iter = manifest.obj_begin(), liter = objs.begin();
+       iter != manifest.obj_end() && liter != objs.end();
+       ++iter, ++liter) {
+    ASSERT_TRUE(*liter == iter.get_location());
+  }
+
+  ASSERT_TRUE(iter == manifest.obj_end());
+  ASSERT_TRUE(liter == objs.end());
+
+  iter = manifest.obj_find(100 * 1024);
+  ASSERT_TRUE(iter.get_location() == head);
+  ASSERT_EQ((int)iter.get_stripe_size(), obj_size);
+}
+
+TEST(TestRGWManifest, obj_with_head_and_tail) {
+  RGWObjManifest manifest;
+  rgw_bucket bucket;
+  rgw_obj head;
+  RGWObjManifest::generator gen;
+
+  list objs;
+
+  int obj_size = 21 * 1024 * 1024 + 1000;
+  int stripe_size = 4 * 1024 * 1024;
+  int head_size = 512 * 1024;
+
+  gen_obj(obj_size, head_size, stripe_size, &manifest, &bucket, &head, &gen, &objs);
+
+  list::iterator liter;
+
+  rgw_obj last_obj;
+
+  RGWObjManifest::obj_iterator iter;
+  for (iter = manifest.obj_begin(), liter = objs.begin();
+       iter != manifest.obj_end() && liter != objs.end();
+       ++iter, ++liter) {
+    cout << "*liter=" << *liter << " iter.get_location()=" << iter.get_location() << std::endl;
+    ASSERT_TRUE(*liter == iter.get_location());
+
+    last_obj = iter.get_location();
+  }
+
+  ASSERT_TRUE(iter == manifest.obj_end());
+  ASSERT_TRUE(liter == objs.end());
+
+  iter = manifest.obj_find(100 * 1024);
+  ASSERT_TRUE(iter.get_location() == head);
+  ASSERT_EQ((int)iter.get_stripe_size(), head_size);
+
+  uint64_t ofs = 20 * 1024 * 1024 + head_size;
+  iter = manifest.obj_find(ofs + 100);
+
+  ASSERT_TRUE(iter.get_location() == last_obj);
+  ASSERT_EQ(iter.get_stripe_ofs(), ofs);
+  ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs);
+}
+
+TEST(TestRGWManifest, multipart) {
+  int num_parts = 16;
+  RGWObjManifest pm[num_parts];
+  rgw_bucket bucket;
+  uint64_t part_size = 10 * 1024 * 1024;
+  uint64_t stripe_size = 4 * 1024 * 1024;
+
+  string upload_id = "abc123";
+
+  for (int i = 0; i < num_parts; ++i) {
+    RGWObjManifest& manifest = pm[i];
+    RGWObjManifest::generator gen;
+    manifest.set_prefix(upload_id);
+
+    manifest.set_multipart_part_rule(stripe_size, i + 1);
+
+    uint64_t ofs;
+    rgw_obj head;
+    for (ofs = 0; ofs < part_size; ofs += stripe_size) {
+      if (ofs == 0) {
+        int r = gen.create_begin(g_ceph_context, &manifest, bucket, head);
+        ASSERT_EQ(r, 0);
+        continue;
+      }
+      gen.create_next(ofs);
+    }
+
+    if (ofs > part_size) {
+      gen.create_next(part_size);
+    }
+  }
+
+  RGWObjManifest m;
+
+  for (int i = 0; i < num_parts; i++) {
+    m.append(pm[i]);
+  }
+  RGWObjManifest::obj_iterator iter;
+  for (iter = m.obj_begin(); iter != m.obj_end(); ++iter) {
+    RGWObjManifest::obj_iterator fiter = m.obj_find(iter.get_ofs());
+    ASSERT_TRUE(fiter.get_location() == iter.get_location());
+  }
+
+  ASSERT_EQ(m.get_obj_size(), num_parts * part_size);
+}
+
diff --git a/ceph/src/test/run-cli-tests b/ceph/src/test/run-cli-tests
new file mode 100755
index 00000000..fec60dea
--- /dev/null
+++ b/ceph/src/test/run-cli-tests
@@ -0,0 +1,57 @@
+#!/bin/sh
+set -e
+
+if ! command -v virtualenv >/dev/null; then
+    echo "$0: virtualenv not installed, skipping python-using tests." 1>&2
+    exit 1
+fi
+
+SRCDIR="$(dirname "$0")"
+
+# build directory, if different, can be passed as an argument;
+# it is expected to point to the equivalent subdirectory of the
+# tree as where this script is stored
+BUILDDIR="$SRCDIR"
+case "$1" in
+    ''|-*)
+	# not set or looks like a flag to cram
+        ;;
+    *)
+	# looks like the builddir
+	BUILDDIR="$1"
+	shift
+	;;
+esac
+
+VENV="$BUILDDIR/virtualenv"
+CRAM_BIN="$VENV/bin/cram"
+if [ ! -e "$CRAM_BIN" ]; then
+    # With "make distcheck", the source directory must be read-only. I
+    # patched cram to support that. See upstream ticket at
+    # https://bitbucket.org/brodie/cram/issue/9/allow-read-only-directories-for-t
+    # -- tv@inktank.com
+    virtualenv "$VENV" && $VENV/bin/pip install "$SRCDIR/downloads/cram-0.5.0ceph.2011-01-14.tar.gz"
+fi
+
+SRCDIR_ABS="$(readlink -f "$SRCDIR")"
+BUILDDIR_ABS="$(readlink -f "$BUILDDIR")"
+
+# cram doesn't like seeing the same foo.t basename twice on the same
+# run, so run it once per directory
+FAILED=0
+for tool in "$SRCDIR"/cli/*; do
+    toolname="$(basename "$tool")"
+    install -d -m0755 -- "$BUILDDIR/cli/$toolname"
+    if ! env -i \
+	PATH="$BUILDDIR_ABS/..:$SRCDIR_ABS/..:$PATH" \
+	CEPH_CONF=/dev/null \
+	CCACHE_DIR="$CCACHE_DIR" \
+	CC="$CC" \
+	CXX="$CXX" \
+	"$SRCDIR/run-cli-tests-maybe-unset-ccache" \
+	"$CRAM_BIN" -v "$@"  --error-dir="$BUILDDIR/cli/$toolname" -- "$tool"/*.t; then
+	FAILED=1
+    fi
+done
+
+exit "$FAILED"
diff --git a/ceph/src/test/run-cli-tests-maybe-unset-ccache b/ceph/src/test/run-cli-tests-maybe-unset-ccache
new file mode 100755
index 00000000..ba72b062
--- /dev/null
+++ b/ceph/src/test/run-cli-tests-maybe-unset-ccache
@@ -0,0 +1,20 @@
+#!/bin/sh
+set -e
+
+# ccache breaks if it sees CCACHE_DIR="", yet due to clumsiness of
+# /usr/bin/env, it's hard to avoid setting env vars for the parent;
+# unset them if they're empty
+
+if [ -z "$CCACHE_DIR" ]; then
+   unset CCACHE_DIR
+fi
+
+if [ -z "$CC" ]; then
+   unset CC
+fi
+
+if [ -z "$CXX" ]; then
+   unset CXX
+fi
+
+exec "$@"
diff --git a/ceph/src/test/run_cmd.cc b/ceph/src/test/run_cmd.cc
new file mode 100644
index 00000000..ada624e8
--- /dev/null
+++ b/ceph/src/test/run_cmd.cc
@@ -0,0 +1,26 @@
+#include "common/config.h"
+#include "common/run_cmd.h"
+
+#include "gtest/gtest.h"
+
+#include 
+#include 
+
+TEST(RunCommand, StringSimple)
+{
+  char temp_file_name[] = "run_cmd_temp_file_XXXXXX";
+
+  int fd = ::mkstemp(temp_file_name);
+  ASSERT_GE(fd, 0);
+  ::close(fd);
+
+  std::string ret = run_cmd("touch", temp_file_name, (char*)NULL);
+  ASSERT_EQ(ret, "");
+
+  ASSERT_EQ(access(temp_file_name, R_OK), 0);
+
+  ret = run_cmd("rm", "-f", temp_file_name, (char*)NULL);
+  ASSERT_EQ(ret, "");
+
+  ASSERT_NE(access(temp_file_name, R_OK), 0);
+}
diff --git a/ceph/src/test/signals.cc b/ceph/src/test/signals.cc
new file mode 100644
index 00000000..2c7f4d32
--- /dev/null
+++ b/ceph/src/test/signals.cc
@@ -0,0 +1,142 @@
+#include "common/config.h"
+#include "common/signal.h"
+#include "global/signal_handler.h"
+
+#include "test/unit.h"
+
+#include 
+#include 
+#include 
+#include 
+
+static volatile sig_atomic_t got_sigusr1 = 0;
+
+static void handle_sigusr1(int signo)
+{
+  got_sigusr1 = 1;
+}
+
+TEST(SignalApi, SimpleInstall)
+{
+  install_sighandler(SIGPIPE, handle_sigusr1, 0);
+}
+
+TEST(SignalApi, SimpleInstallAndTest)
+{
+  install_sighandler(SIGPIPE, handle_sigusr1, 0);
+
+  // SIGPIPE starts out blocked
+  int ret = kill(getpid(), SIGPIPE);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(got_sigusr1, 0);
+
+  // handle SIGPIPE
+  sigset_t mask;
+  sigemptyset(&mask);
+  ret = sigsuspend(&mask);
+  if (ret == -1)
+    ret = errno;
+
+  // we should have gotten it
+  ASSERT_EQ(ret, EINTR);
+  ASSERT_EQ(got_sigusr1, 1);
+}
+
+TEST(SignalEffects, ErrnoTest1)
+{
+}
+
+bool usr1 = false;
+bool usr2 = false;
+
+void reset()
+{
+  usr1 = false;
+  usr2 = false;
+}
+
+void testhandler(int signal)
+{
+  switch (signal) {
+  case SIGUSR1:
+    usr1 = true;
+    break;
+  case SIGUSR2:
+    usr2 = true;
+    break;
+  default:
+    assert(0 == "unexpected signal");
+  }
+}
+
+TEST(SignalHandler, Single)
+{
+  reset();
+  init_async_signal_handler();
+  register_async_signal_handler(SIGUSR1, testhandler);
+  ASSERT_TRUE(usr1 == false);
+
+  int ret = kill(getpid(), SIGUSR1);
+  ASSERT_EQ(ret, 0);
+
+  sleep(1);
+  ASSERT_TRUE(usr1 == true);
+
+  unregister_async_signal_handler(SIGUSR1, testhandler);
+  shutdown_async_signal_handler();
+}
+
+TEST(SignalHandler, Multiple)
+{
+  int ret;
+
+  reset();
+  init_async_signal_handler();
+  register_async_signal_handler(SIGUSR1, testhandler);
+  register_async_signal_handler(SIGUSR2, testhandler);
+  ASSERT_TRUE(usr1 == false);
+  ASSERT_TRUE(usr2 == false);
+
+  ret = kill(getpid(), SIGUSR1);
+  ASSERT_EQ(ret, 0);
+  ret = kill(getpid(), SIGUSR2);
+  ASSERT_EQ(ret, 0);
+
+  sleep(1);
+  ASSERT_TRUE(usr1 == true);
+  ASSERT_TRUE(usr2 == true);
+
+  unregister_async_signal_handler(SIGUSR1, testhandler);
+  unregister_async_signal_handler(SIGUSR2, testhandler);
+  shutdown_async_signal_handler();
+}
+
+/*
+TEST(SignalHandler, MultipleBigFd)
+{
+  int ret;
+
+  for (int i = 0; i < 1500; i++)
+    ::open(".", O_RDONLY);
+
+  reset();
+  init_async_signal_handler();
+  register_async_signal_handler(SIGUSR1, testhandler);
+  register_async_signal_handler(SIGUSR2, testhandler);
+  ASSERT_TRUE(usr1 == false);
+  ASSERT_TRUE(usr2 == false);
+
+  ret = kill(getpid(), SIGUSR1);
+  ASSERT_EQ(ret, 0);
+  ret = kill(getpid(), SIGUSR2);
+  ASSERT_EQ(ret, 0);
+
+  sleep(1);
+  ASSERT_TRUE(usr1 == true);
+  ASSERT_TRUE(usr2 == true);
+
+  unregister_async_signal_handler(SIGUSR1, testhandler);
+  unregister_async_signal_handler(SIGUSR2, testhandler);
+  shutdown_async_signal_handler();
+}
+*/
diff --git a/ceph/src/test/simple_spin.cc b/ceph/src/test/simple_spin.cc
new file mode 100644
index 00000000..63e09a45
--- /dev/null
+++ b/ceph/src/test/simple_spin.cc
@@ -0,0 +1,39 @@
+#include "gtest/gtest.h"
+
+#include "common/simple_spin.h"
+
+TEST(SimpleSpin, Test0)
+{
+  simple_spinlock_t lock0 = SIMPLE_SPINLOCK_INITIALIZER;
+  simple_spin_lock(&lock0);
+  simple_spin_unlock(&lock0);
+}
+
+static simple_spinlock_t lock = SIMPLE_SPINLOCK_INITIALIZER;
+static uint32_t counter = 0;
+
+static void* mythread(void *v)
+{
+  for (int j = 0; j < 1000000; ++j) {
+    simple_spin_lock(&lock);
+    counter++;
+    simple_spin_unlock(&lock);
+  }
+  return NULL;
+}
+
+TEST(SimpleSpin, Test1)
+{
+  int ret;
+  pthread_t thread1;
+  pthread_t thread2;
+  ret = pthread_create(&thread1, NULL, mythread, NULL);
+  ASSERT_EQ(ret, 0);
+  ret = pthread_create(&thread2, NULL, mythread, NULL);
+  ASSERT_EQ(ret, 0);
+  ret = pthread_join(thread1, NULL);
+  ASSERT_EQ(ret, 0);
+  ret = pthread_join(thread2, NULL);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(counter, 2000000U);
+}
diff --git a/ceph/src/test/streamtest.cc b/ceph/src/test/streamtest.cc
new file mode 100644
index 00000000..d4ccadf1
--- /dev/null
+++ b/ceph/src/test/streamtest.cc
@@ -0,0 +1,191 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include "os/FileStore.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+
+#undef dout_prefix
+#define dout_prefix *_dout
+
+struct io {
+  utime_t start, ack, commit;
+  bool done() {
+    return ack.sec() && commit.sec();
+  }
+};
+map writes;
+Cond cond;
+Mutex test_lock("streamtest.cc lock");
+
+unsigned concurrent = 1;
+void throttle()
+{ 
+  Mutex::Locker l(test_lock);
+  while (writes.size() >= concurrent) {
+    //generic_dout(0) << "waiting" << dendl;
+    cond.Wait(test_lock);
+  }
+}
+
+double total_ack = 0;
+double total_commit = 0;
+int total_num = 0;
+
+void pr(off_t off)
+{
+  io &i = writes[off];
+  if (false) cout << off << "\t" 
+       << (i.ack - i.start) << "\t"
+       << (i.commit - i.start) << std::endl;
+  total_num++;
+  total_ack += (i.ack - i.start);
+  total_commit += (i.commit - i.start);
+  writes.erase(off);
+  cond.Signal();
+}
+
+void set_start(off_t off, utime_t t)
+{
+  Mutex::Locker l(test_lock);
+  writes[off].start = t;
+}
+
+void set_ack(off_t off, utime_t t)
+{
+  Mutex::Locker l(test_lock);
+  //generic_dout(0) << "ack " << off << dendl;
+  writes[off].ack = t;
+  if (writes[off].done())
+    pr(off);
+}
+
+void set_commit(off_t off, utime_t t)
+{
+  Mutex::Locker l(test_lock);
+  //generic_dout(0) << "commit " << off << dendl;
+  writes[off].commit = t;
+  if (writes[off].done())
+    pr(off);
+}
+
+
+struct C_Ack : public Context {
+  off_t off;
+  C_Ack(off_t o) : off(o) {}
+  void finish(int r) {
+    set_ack(off, ceph_clock_now(g_ceph_context));
+  }
+};
+struct C_Commit : public Context {
+  off_t off;
+  C_Commit(off_t o) : off(o) {}
+  void finish(int r) {
+    set_commit(off, ceph_clock_now(g_ceph_context));
+  }
+};
+
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  // args
+  if (args.size() < 3) return -1;
+  const char *filename = args[0];
+  int seconds = atoi(args[1]);
+  int bytes = atoi(args[2]);
+  const char *journal = 0;
+  if (args.size() >= 4)
+    journal = args[3];
+  if (args.size() >= 5)
+    concurrent = atoi(args[4]);
+
+  cout << "concurrent = " << concurrent << std::endl;
+
+  buffer::ptr bp(bytes);
+  bp.zero();
+  bufferlist bl;
+  bl.push_back(bp);
+
+  //float interval = 1.0 / 1000;
+  
+  cout << "#dev " << filename
+       << ", " << seconds << " seconds, " << bytes << " bytes per write" << std::endl;
+
+  ObjectStore *fs = new FileStore(filename, journal);
+  
+  if (fs->mkfs() < 0) {
+    cout << "mkfs failed" << std::endl;
+    return -1;
+  }
+  
+  if (fs->mount() < 0) {
+    cout << "mount failed" << std::endl;
+    return -1;
+  }
+
+  ObjectStore::Transaction ft;
+  ft.create_collection(coll_t());
+  fs->apply_transaction(ft);
+
+  utime_t now = ceph_clock_now(g_ceph_context);
+  utime_t start = now;
+  utime_t end = now;
+  end += seconds;
+  off_t pos = 0;
+  //cout << "stop at " << end << std::endl;
+  cout << "# offset\tack\tcommit" << std::endl;
+  while (now < end) {
+    sobject_t poid(object_t("streamtest"), 0);
+
+    set_start(pos, ceph_clock_now(g_ceph_context));
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    t->write(coll_t(), hobject_t(poid), pos, bytes, bl);
+    fs->queue_transaction(NULL, t, new C_Ack(pos), new C_Commit(pos));
+    pos += bytes;
+
+    throttle();
+
+    now = ceph_clock_now(g_ceph_context);
+
+    // wait?
+    /*
+    utime_t next = start;
+    next += interval;
+    if (now < next) {
+      float s = next - now;
+      s *= 1000 * 1000;  // s -> us
+      //cout << "sleeping for " << s << " us" << std::endl;
+      usleep((int)s);
+    }
+    */
+  }
+
+  cout << "total num " << total_num << std::endl;
+  cout << "avg ack\t" << (total_ack / (double)total_num) << std::endl;
+  cout << "avg commit\t" << (total_commit / (double)total_num) << std::endl;
+  cout << "tput\t" << prettybyte_t((double)(total_num * bytes) / (double)(end-start)) << "/sec" << std::endl;
+
+  fs->umount();
+
+}
+
diff --git a/ceph/src/test/strtol.cc b/ceph/src/test/strtol.cc
new file mode 100644
index 00000000..08ba081a
--- /dev/null
+++ b/ceph/src/test/strtol.cc
@@ -0,0 +1,211 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/strtol.h"
+#include 
+#include 
+
+#include "gtest/gtest.h"
+
+static void test_strict_strtoll(const char *str, long long expected)
+{
+  std::string err;
+  long long val = strict_strtoll(str, 10, &err);
+  if (!err.empty()) {
+    ASSERT_EQ(err, "");
+  }
+  else {
+    ASSERT_EQ(val, expected);
+  }
+}
+
+static void test_strict_strtol(const char *str, long expected)
+{
+  std::string err;
+  long val = strict_strtol(str, 10, &err);
+  if (!err.empty()) {
+    ASSERT_EQ(err, "");
+  }
+  else {
+    ASSERT_EQ(val, expected);
+  }
+}
+
+static void test_strict_strtod(const char *str, double expected)
+{
+  std::string err;
+  double val = strict_strtod(str, &err);
+  if (!err.empty()) {
+    ASSERT_EQ(err, "");
+  }
+  else {
+    // when comparing floats, use a margin of error
+    if ((expected - 0.001 > val) || (expected + 0.001 < val)) {
+      ASSERT_EQ(val, expected);
+    }
+  }
+}
+
+static void test_strict_strtof(const char *str, float expected)
+{
+  std::string err;
+  float val = strict_strtof(str, &err);
+  if (!err.empty()) {
+    ASSERT_EQ(err, "");
+  }
+  else {
+    // when comparing floats, use a margin of error
+    if ((expected - 0.001 > val) || (expected + 0.001 < val)) {
+      ASSERT_EQ(val, expected);
+    }
+  }
+}
+
+TEST(StrToL, Simple1) {
+  test_strict_strtoll("123", 123);
+  test_strict_strtoll("0", 0);
+  test_strict_strtoll("-123", -123);
+  test_strict_strtoll("8796093022208", 8796093022208LL);
+  test_strict_strtoll("-8796093022208", -8796093022208LL);
+
+  test_strict_strtol("208", 208);
+  test_strict_strtol("-4", -4);
+  test_strict_strtol("0", 0);
+  test_strict_strtol("2147483646", 2147483646);
+
+  test_strict_strtof("0.05", 0.05);
+  test_strict_strtof("0", 0.0);
+  test_strict_strtof("-0", 0.0);
+  test_strict_strtof("10000000.5", 10000000.5);
+
+  test_strict_strtod("-0.2", -0.2);
+  test_strict_strtod("0.1", 0.1);
+  test_strict_strtod("0", 0.0);
+}
+
+static void test_strict_strtoll_err(const char *str)
+{
+  std::string err;
+  strict_strtoll(str, 10, &err);
+  ASSERT_NE(err, "");
+}
+
+static void test_strict_strtol_err(const char *str)
+{
+  std::string err;
+  strict_strtol(str, 10, &err);
+  ASSERT_NE(err, "");
+}
+
+static void test_strict_strtod_err(const char *str)
+{
+  std::string err;
+  strict_strtod(str, &err);
+  ASSERT_NE(err, "");
+}
+
+static void test_strict_strtof_err(const char *str)
+{
+  std::string err;
+  strict_strtof(str, &err);
+  ASSERT_NE(err, "");
+}
+
+TEST(StrToL, Error1) {
+  test_strict_strtoll_err("604462909807314587353088"); // overflow
+  test_strict_strtoll_err("aw shucks"); // invalid
+  test_strict_strtoll_err("343245 aw shucks"); // invalid chars at end
+
+  test_strict_strtol_err("35 aw shucks"); // invalid chars at end
+  test_strict_strtol_err("--0");
+
+  test_strict_strtod_err("345345.0-");
+  test_strict_strtod_err("34.0 garbo");
+
+  test_strict_strtof_err("0.05.0");
+}
+
+
+static void test_strict_sistrtoll(const char *str)
+{
+  std::string err;
+  strict_sistrtoll(str, &err);
+  ASSERT_EQ(err, "");
+}
+
+static void test_strict_sistrtoll_units(const std::string& foo,
+                                      char u, const int m)
+{
+  std::string s(foo);
+  s.push_back(u);
+  const char *str = s.c_str();
+  std::string err;
+  uint64_t r = strict_sistrtoll(str, &err);
+  ASSERT_EQ(err, "");
+
+  str = foo.c_str();
+  std::string err2;
+  long long tmp = strict_strtoll(str, 10, &err2);
+  ASSERT_EQ(err2, "");
+  tmp = (tmp << m);
+  ASSERT_EQ(tmp, (long long)r);
+}
+
+TEST(SIStrToLL, WithUnits) {
+  std::map units;
+  units['B'] = 0;
+  units['K'] = 10;
+  units['M'] = 20;
+  units['G'] = 30;
+  units['T'] = 40;
+  units['P'] = 50;
+  units['E'] = 60;
+
+  for (std::map::iterator p = units.begin();
+       p != units.end(); ++p) {
+    test_strict_sistrtoll_units("1024", p->first, p->second);
+    test_strict_sistrtoll_units("1", p->first, p->second);
+    test_strict_sistrtoll_units("0", p->first, p->second);
+  }
+}
+
+TEST(SIStrToLL, WithoutUnits) {
+  test_strict_sistrtoll("1024");
+  test_strict_sistrtoll("1152921504606846976");
+  test_strict_sistrtoll("0");
+}
+
+static void test_strict_sistrtoll_err(const char *str)
+{
+  std::string err;
+  strict_sistrtoll(str, &err);
+  ASSERT_NE(err, "");
+}
+
+TEST(SIStrToLL, Error) {
+  test_strict_sistrtoll_err("1024F");
+  test_strict_sistrtoll_err("QDDSA");
+  test_strict_sistrtoll_err("1b");
+  test_strict_sistrtoll_err("100k");
+  test_strict_sistrtoll_err("1000m");
+  test_strict_sistrtoll_err("1g");
+  test_strict_sistrtoll_err("20t");
+  test_strict_sistrtoll_err("100p");
+  test_strict_sistrtoll_err("1000e");
+  test_strict_sistrtoll_err("B");
+  test_strict_sistrtoll_err("M");
+  test_strict_sistrtoll_err("BM");
+  test_strict_sistrtoll_err("B0wef");
+  test_strict_sistrtoll_err("0m");
+}
diff --git a/ceph/src/test/system/cross_process_sem.cc b/ceph/src/test/system/cross_process_sem.cc
new file mode 100644
index 00000000..667ba1cc
--- /dev/null
+++ b/ceph/src/test/system/cross_process_sem.cc
@@ -0,0 +1,107 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+
+#include 
+#include 
+#include 
+#include 
+
+/* We put our cross-process semaphore into a page of memory mapped with mmap. */
+struct cross_process_sem_data_t
+{
+  sem_t sem;
+};
+
+/* A factory function is a good choice here because we want to be able to
+ * return an error code. It does force heap allocation, but that is the
+ * easiest way to use synchronization primitives anyway. Most programmers don't
+ * care about destroying semaphores before the process finishes. It's pretty
+ * difficult to get it right and there is usually no benefit.
+ */
+int CrossProcessSem::
+create(int initial_val, CrossProcessSem** res)
+{
+  struct cross_process_sem_data_t *data = static_cast < cross_process_sem_data_t*> (
+    mmap(NULL, sizeof(struct cross_process_sem_data_t),
+       PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, 0, 0));
+  if (data == MAP_FAILED) {
+    int err = errno;
+    return err;
+  }
+  int ret = sem_init(&data->sem, 1, initial_val);
+  if (ret) {
+    return ret;
+  }
+  *res = new CrossProcessSem(data);
+  return 0;
+}
+
+CrossProcessSem::
+~CrossProcessSem()
+{
+  munmap(m_data, sizeof(struct cross_process_sem_data_t));
+  m_data = NULL;
+}
+
+void CrossProcessSem::
+wait()
+{
+  while(true) {
+    int ret = sem_wait(&m_data->sem);
+    if (ret == 0)
+      return;
+    int err = errno;
+    if (err == -EINTR)
+      continue;
+    abort();
+  }
+}
+
+void CrossProcessSem::
+post()
+{
+  int ret = sem_post(&m_data->sem);
+  if (ret == -1) {
+    abort();
+  }
+}
+
+int CrossProcessSem::
+reinit(int dval)
+{
+  if (dval < 0)
+    return -EINVAL;
+  int cval;
+  if (sem_getvalue(&m_data->sem, &cval) == -1)
+    return errno;
+  if (cval < dval) {
+    int diff = dval - cval;
+    for (int i = 0; i < diff; ++i)
+      sem_post(&m_data->sem);
+  }
+  else {
+    int diff = cval - dval;
+    for (int i = 0; i < diff; ++i)
+      sem_wait(&m_data->sem);
+  }
+  return 0;
+}
+
+CrossProcessSem::
+CrossProcessSem(struct cross_process_sem_data_t *data)
+  : m_data(data)
+{
+}
diff --git a/ceph/src/test/system/cross_process_sem.h b/ceph/src/test/system/cross_process_sem.h
new file mode 100644
index 00000000..d087d292
--- /dev/null
+++ b/ceph/src/test/system/cross_process_sem.h
@@ -0,0 +1,40 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+struct cross_process_sem_data_t;
+
+class CrossProcessSem
+{
+public:
+  static int create(int initial_val, CrossProcessSem** ret);
+  ~CrossProcessSem();
+
+  /* Initialize the semaphore. Must be called before any operations */
+  int init();
+
+  /* Semaphore wait */
+  void wait();
+
+  /* Semaphore post */
+  void post();
+
+  /* Reinitialize the semaphore to the desired value.
+   * NOT thread-safe if it is in use at the time!
+   */ 
+  int reinit(int dval);
+
+private:
+  CrossProcessSem(struct cross_process_sem_data_t *data);
+  struct cross_process_sem_data_t *m_data;
+};
diff --git a/ceph/src/test/system/rados_delete_pools_parallel.cc b/ceph/src/test/system/rados_delete_pools_parallel.cc
new file mode 100644
index 00000000..5347c53c
--- /dev/null
+++ b/ceph/src/test/system/rados_delete_pools_parallel.cc
@@ -0,0 +1,110 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "st_rados_delete_pool.h"
+#include "st_rados_list_objects.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+static int g_num_objects = 50;
+
+/*
+ * rados_delete_pools_parallel
+ *
+ * This tests creation and deletion races.
+ *
+ * EXPECT:            * can delete a pool while another user is using it
+ *                    * operations on pools return error codes after the pools
+ *                      are deleted
+ *
+ * DO NOT EXPECT      * hangs, crashes
+ */
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  const char *num_objects = getenv("NUM_OBJECTS");
+  std::string pool = "foo";
+  if (num_objects) {
+    g_num_objects = atoi(num_objects); 
+    if (g_num_objects == 0)
+      return 100;
+  }
+
+  CrossProcessSem *pool_setup_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem));
+  CrossProcessSem *delete_pool_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &delete_pool_sem));
+
+  // first test: create a pool, then delete that pool
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, 50, ".obj");
+    StRadosDeletePool r2(argc, argv, pool_setup_sem, NULL, pool);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test1: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // second test: create a pool, the list objects in that pool while it's
+  // being deleted.
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(delete_pool_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosDeletePool r2(argc, argv, delete_pool_sem, NULL, pool);
+    StRadosListObjects r3(argc, argv, pool, true, g_num_objects / 2,
+			  pool_setup_sem, NULL, delete_pool_sem);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test2: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  printf("******* SUCCESS **********\n"); 
+  return EXIT_SUCCESS;
+}
diff --git a/ceph/src/test/system/rados_list_parallel.cc b/ceph/src/test/system/rados_list_parallel.cc
new file mode 100644
index 00000000..7af86e41
--- /dev/null
+++ b/ceph/src/test/system/rados_list_parallel.cc
@@ -0,0 +1,340 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "include/stringify.h"
+#include "st_rados_create_pool.h"
+#include "st_rados_list_objects.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+static int g_num_objects = 50;
+
+static CrossProcessSem *pool_setup_sem = NULL;
+static CrossProcessSem *modify_sem = NULL;
+
+class RadosDeleteObjectsR : public SysTestRunnable
+{
+public:
+  RadosDeleteObjectsR(int argc, const char **argv,
+		      const std::string &pool_name)
+    : SysTestRunnable(argc, argv), m_pool_name(pool_name)
+  {
+  }
+
+  ~RadosDeleteObjectsR()
+  {
+  }
+
+  int run(void)
+  {
+    rados_t cl;
+    RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+    rados_conf_parse_argv(cl, m_argc, m_argv);
+    RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+    rados_conf_parse_env(cl, NULL);
+    std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+    if (!log_name.empty())
+      rados_conf_set(cl, "log_file", log_name.c_str());
+    RETURN1_IF_NONZERO(rados_connect(cl));
+    pool_setup_sem->wait();
+    pool_setup_sem->post();
+
+    rados_ioctx_t io_ctx;
+    rados_pool_create(cl, m_pool_name.c_str());
+    RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+    std::map  to_delete;
+    for (int i = 0; i < g_num_objects; ++i) {
+      char oid[128];
+      snprintf(oid, sizeof(oid), "%d.obj", i);
+      to_delete[i] = oid;
+    }
+
+    int removed = 0;
+    while (true) {
+      if (to_delete.empty())
+	break;
+      int r = rand() % to_delete.size();
+      std::map ::iterator d = to_delete.begin();
+      for (int i = 0; i < r; ++i)
+	++d;
+      if (d == to_delete.end()) {
+	return -EDOM;
+      }
+      std::string oid(d->second);
+      to_delete.erase(d);
+      int ret = rados_remove(io_ctx, oid.c_str());
+      if (ret != 0) {
+	printf("%s: rados_remove(%s) failed with error %d\n",
+	       get_id_str(), oid.c_str(), ret);
+	return ret;
+      }
+      ++removed;
+      if ((removed % 25) == 0) {
+	printf("%s: removed %d objects...\n", get_id_str(), removed);
+      }
+      if (removed == g_num_objects / 2) {
+	printf("%s: removed half of the objects\n", get_id_str());
+	modify_sem->post();
+      }
+    }
+
+    printf("%s: removed %d objects\n", get_id_str(), removed);
+
+    rados_ioctx_destroy(io_ctx);
+    rados_shutdown(cl);
+
+    return 0;
+  }
+private:
+  std::string m_pool_name;
+};
+
+class RadosAddObjectsR : public SysTestRunnable
+{
+public:
+  RadosAddObjectsR(int argc, const char **argv,
+		   const std::string &pool_name,
+		   const std::string &suffix)
+    : SysTestRunnable(argc, argv),
+      m_pool_name(pool_name),
+      m_suffix(suffix)
+  {
+  }
+
+  ~RadosAddObjectsR()
+  {
+  }
+
+  int run(void)
+  {
+    rados_t cl;
+    RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+    rados_conf_parse_argv(cl, m_argc, m_argv);
+    RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+    rados_conf_parse_env(cl, NULL);
+    std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+    if (!log_name.empty())
+      rados_conf_set(cl, "log_file", log_name.c_str());
+    RETURN1_IF_NONZERO(rados_connect(cl));
+    pool_setup_sem->wait();
+    pool_setup_sem->post();
+
+    rados_ioctx_t io_ctx;
+    rados_pool_create(cl, m_pool_name.c_str());
+    RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+    std::map  to_add;
+    for (int i = 0; i < g_num_objects; ++i) {
+      char oid[128];
+      snprintf(oid, sizeof(oid), "%d%s", i, m_suffix.c_str());
+      to_add[i] = oid;
+    }
+
+    int added = 0;
+    while (true) {
+      if (to_add.empty())
+	break;
+      int r = rand() % to_add.size();
+      std::map ::iterator d = to_add.begin();
+      for (int i = 0; i < r; ++i)
+	++d;
+      if (d == to_add.end()) {
+	return -EDOM;
+      }
+      std::string oid(d->second);
+      to_add.erase(d);
+
+      std::string buf(StRadosCreatePool::get_random_buf(256));
+      int ret = rados_write(io_ctx, oid.c_str(), buf.c_str(), buf.size(), 0);
+      if (ret != 0) {
+	printf("%s: rados_write(%s) failed with error %d\n",
+	       get_id_str(), oid.c_str(), ret);
+	return ret;
+      }
+      ++added;
+      if ((added % 25) == 0) {
+	printf("%s: added %d objects...\n", get_id_str(), added);
+      }
+      if (added == g_num_objects / 2) {
+	printf("%s: added half of the objects\n", get_id_str());
+	modify_sem->post();
+      }
+    }
+
+    printf("%s: added %d objects\n", get_id_str(), added);
+
+    rados_ioctx_destroy(io_ctx);
+    rados_shutdown(cl);
+
+    return 0;
+  }
+private:
+  std::string m_pool_name;
+  std::string m_suffix;
+};
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  const char *num_objects = getenv("NUM_OBJECTS");
+  std::string pool = "foo." + stringify(getpid());
+  if (num_objects) {
+    g_num_objects = atoi(num_objects); 
+    if (g_num_objects == 0)
+      return 100;
+  }
+
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem));
+  RETURN1_IF_NONZERO(CrossProcessSem::create(1, &modify_sem));
+
+  std::string error;
+
+  // Test 1... list objects
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects,
+			  pool_setup_sem, modify_sem, NULL);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 2... list objects while they're being deleted
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    RadosDeleteObjectsR r3(argc, argv, pool);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 3... list objects while others are being added
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    RadosAddObjectsR r3(argc, argv, pool, ".obj2");
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 4... list objects while others are being added and deleted
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    RadosAddObjectsR r3(argc, argv, pool, ".obj2");
+    RadosAddObjectsR r4(argc, argv, pool, ".obj3");
+    RadosDeleteObjectsR r5(argc, argv, pool);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    vec.push_back(&r5);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 5... list objects while they are being modified
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    // AddObjects with the same 'suffix' as used in StRadosCreatePool
+    RadosAddObjectsR r3(argc, argv, pool, ".obj");
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  rados_t cl;
+  rados_create(&cl, NULL);
+  rados_conf_parse_argv(cl, argc, argv);
+  rados_conf_parse_argv(cl, argc, argv);
+  rados_conf_read_file(cl, NULL);
+  rados_conf_parse_env(cl, NULL);
+  rados_connect(cl);
+  rados_pool_delete(cl, pool.c_str());
+
+  printf("******* SUCCESS **********\n"); 
+  return EXIT_SUCCESS;
+}
diff --git a/ceph/src/test/system/rados_open_pools_parallel.cc b/ceph/src/test/system/rados_open_pools_parallel.cc
new file mode 100644
index 00000000..82c71207
--- /dev/null
+++ b/ceph/src/test/system/rados_open_pools_parallel.cc
@@ -0,0 +1,136 @@
+
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+/*
+ * rados_open_pools_parallel
+ *
+ * This tests creating a pool in one Runnable, and then opening an io context
+ * based on that pool in another.
+ *
+ * EXPECT:            * can't create the same pool twice
+ *                    * one Runnable can use the pool after the other one creates it
+ *
+ * DO NOT EXPECT      * hangs, crashes
+ */
+class StRadosOpenPool : public SysTestRunnable
+{
+public:
+  StRadosOpenPool(int argc, const char **argv,
+		  CrossProcessSem *pool_setup_sem, CrossProcessSem *open_pool_sem)
+    : SysTestRunnable(argc, argv),
+      m_pool_setup_sem(pool_setup_sem), m_open_pool_sem(open_pool_sem)
+  {
+  }
+
+  ~StRadosOpenPool()
+  {
+  }
+
+  int run()
+  {
+    rados_t cl;
+    RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+    rados_conf_parse_argv(cl, m_argc, m_argv);
+    std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+    if (!log_name.empty())
+      rados_conf_set(cl, "log_file", log_name.c_str());
+    RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+    rados_conf_parse_env(cl, NULL);
+    RETURN1_IF_NONZERO(rados_connect(cl));
+    if (m_pool_setup_sem)
+      m_pool_setup_sem->wait();
+
+    printf("%s: rados_pool_create.\n", get_id_str());
+    rados_pool_create(cl, "foo");
+    rados_ioctx_t io_ctx;
+    printf("%s: rados_ioctx_create.\n", get_id_str());
+    RETURN1_IF_NOT_VAL(0, rados_ioctx_create(cl, "foo", &io_ctx));
+    if (m_open_pool_sem)
+      m_open_pool_sem->post();
+    rados_ioctx_destroy(io_ctx);
+    rados_shutdown(cl);
+    return 0;
+  }
+
+private:
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_open_pool_sem;
+};
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  // first test: create a pool, shut down the client, access that 
+  // pool in a different process.
+  CrossProcessSem *pool_setup_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem));
+  StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+					   "foo", 50, ".obj");
+  StRadosOpenPool r2(argc, argv, pool_setup_sem, NULL);
+  vector < SysTestRunnable* > vec;
+  vec.push_back(&r1);
+  vec.push_back(&r2);
+  std::string error = SysTestRunnable::run_until_finished(vec);
+  if (!error.empty()) {
+    printf("test1: got error: %s\n", error.c_str());
+    return EXIT_FAILURE;
+  }
+
+  // second test: create a pool, access that 
+  // pool in a different process, THEN shut down the first client.
+  CrossProcessSem *pool_setup_sem2 = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem2));
+  CrossProcessSem *open_pool_sem2 = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &open_pool_sem2));
+  StRadosCreatePool r3(argc, argv, NULL, pool_setup_sem2, open_pool_sem2,
+					   "foo", 50, ".obj");
+  StRadosOpenPool r4(argc, argv, pool_setup_sem2, open_pool_sem2);
+  vector < SysTestRunnable* > vec2;
+  vec2.push_back(&r3);
+  vec2.push_back(&r4);
+  error = SysTestRunnable::run_until_finished(vec2);
+  if (!error.empty()) {
+    printf("test2: got error: %s\n", error.c_str());
+    return EXIT_FAILURE;
+  }
+
+  printf("******* SUCCESS **********\n"); 
+  return EXIT_SUCCESS;
+}
diff --git a/ceph/src/test/system/rados_watch_notify.cc b/ceph/src/test/system/rados_watch_notify.cc
new file mode 100644
index 00000000..6517f33e
--- /dev/null
+++ b/ceph/src/test/system/rados_watch_notify.cc
@@ -0,0 +1,191 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "st_rados_delete_pool.h"
+#include "st_rados_delete_objs.h"
+#include "st_rados_watch.h"
+#include "st_rados_notify.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+#include "include/stringify.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+/*
+ * rados_watch_notify
+ *
+ * This tests watch/notify with pool and object deletion.
+ *
+ * EXPECT:            * notifies to a deleted object or pool are not received
+ *                    * notifies to existing objects are received
+ *
+ * DO NOT EXPECT      * hangs, crashes
+ */
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  std::string pool = "foo." + stringify(getpid());
+  CrossProcessSem *setup_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &setup_sem));
+  CrossProcessSem *watch_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &watch_sem));
+  CrossProcessSem *notify_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, ¬ify_sem));
+
+  // create a pool and an object, watch the object, notify.
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, notify_sem,
+		    1, 0, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     0, pool, "0.obj");
+    vector vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test1: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  RETURN1_IF_NONZERO(setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(watch_sem->reinit(0));
+  RETURN1_IF_NONZERO(notify_sem->reinit(0));
+
+  // create a pool and an object, watch a non-existent object,
+  // notify non-existent object.watch
+  pool += ".";
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 0, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, notify_sem,
+		    0, -ENOENT, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     -ENOENT, pool, "0.obj");
+    vector vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test2: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  RETURN1_IF_NONZERO(setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(watch_sem->reinit(0));
+  RETURN1_IF_NONZERO(notify_sem->reinit(0));
+
+  CrossProcessSem *finished_notifies_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &finished_notifies_sem));
+  CrossProcessSem *deleted_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &deleted_sem));
+  CrossProcessSem *second_pool_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &second_pool_sem));
+
+  // create a pool and an object, watch the object, notify,
+  // then delete the pool.
+  // Create a new pool and write to it to make the osd get the updated map,
+  // then try notifying on the deleted pool.
+  pool += ".";
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, finished_notifies_sem,
+		    1, 0, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     0, pool, "0.obj");
+    StRadosDeletePool r4(argc, argv, notify_sem, deleted_sem, pool);
+    StRadosCreatePool r5(argc, argv, deleted_sem, second_pool_sem, NULL,
+			 "bar", 1, ".obj");
+    StRadosNotify r6(argc, argv, second_pool_sem, NULL, finished_notifies_sem,
+		     0, "bar", "0.obj");
+    StRadosDeletePool r7(argc, argv, finished_notifies_sem, NULL, "bar");
+    vector vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    vec.push_back(&r5);
+    vec.push_back(&r6);
+    vec.push_back(&r7);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test3: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  RETURN1_IF_NONZERO(setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(watch_sem->reinit(0));
+  RETURN1_IF_NONZERO(notify_sem->reinit(0));
+  RETURN1_IF_NONZERO(finished_notifies_sem->reinit(0));
+  RETURN1_IF_NONZERO(deleted_sem->reinit(0));
+
+  // create a pool and an object, watch the object, notify,
+  // then delete the object, notify
+  if (false) {
+    // this test is currently broken, pending the resolution of bug #2339
+  pool += ".";
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, finished_notifies_sem,
+		    1, 0, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     0, pool, "0.obj");
+    StRadosDeleteObjs r4(argc, argv, notify_sem, deleted_sem, 1, pool, ".obj");
+    StRadosNotify r5(argc, argv, setup_sem, deleted_sem, finished_notifies_sem,
+		     -ENOENT, pool, "0.obj");
+
+    vector vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    vec.push_back(&r5);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test4: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+  }
+
+  printf("******* SUCCESS **********\n");
+  return EXIT_SUCCESS;
+}
diff --git a/ceph/src/test/system/st_rados_create_pool.cc b/ceph/src/test/system/st_rados_create_pool.cc
new file mode 100644
index 00000000..6bbe628e
--- /dev/null
+++ b/ceph/src/test/system/st_rados_create_pool.cc
@@ -0,0 +1,110 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using std::ostringstream;
+
+std::string StRadosCreatePool::
+get_random_buf(int sz)
+{
+  ostringstream oss;
+  int size = rand() % sz; // yep, it's not very random
+  for (int i = 0; i < size; ++i) {
+    oss << ".";
+  }
+  return oss.str();
+}
+
+StRadosCreatePool::
+StRadosCreatePool(int argc, const char **argv,
+		  CrossProcessSem *setup_sem,
+		  CrossProcessSem *pool_setup_sem,
+		  CrossProcessSem *close_create_pool,
+		  const std::string &pool_name,
+		  int num_objects,
+		  const std::string &suffix)
+  : SysTestRunnable(argc, argv),
+    m_setup_sem(setup_sem),
+    m_pool_setup_sem(pool_setup_sem),
+    m_close_create_pool(close_create_pool),
+    m_pool_name(pool_name),
+    m_num_objects(num_objects),
+    m_suffix(suffix)
+{
+}
+
+StRadosCreatePool::
+~StRadosCreatePool()
+{
+}
+
+int StRadosCreatePool::
+run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+  if (!log_name.empty())
+    rados_conf_set(cl, "log_file", log_name.c_str());
+  rados_conf_parse_env(cl, NULL);
+
+  if (m_setup_sem) {
+    m_setup_sem->wait();
+    m_setup_sem->post();
+  }
+
+  RETURN1_IF_NONZERO(rados_connect(cl));
+
+  printf("%s: creating pool %s\n", get_id_str(), m_pool_name.c_str());
+  rados_pool_create(cl, m_pool_name.c_str());
+  rados_ioctx_t io_ctx;
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+  for (int i = 0; i < m_num_objects; ++i) {
+    char oid[128];
+    snprintf(oid, sizeof(oid), "%d%s", i, m_suffix.c_str());
+    std::string buf(get_random_buf(256));
+    int ret = rados_write(io_ctx, oid, buf.c_str(), buf.size(), 0);
+    if (ret != 0) {
+      printf("%s: rados_write error %d\n", get_id_str(), ret);
+      return ret;
+    }
+    if (((i % 25) == 0) || (i == m_num_objects - 1)) {
+      printf("%s: created object %d...\n", get_id_str(), i);
+    }
+  }
+  printf("%s: finishing.\n", get_id_str());
+  if (m_pool_setup_sem)
+    m_pool_setup_sem->post();
+  if (m_close_create_pool)
+    m_close_create_pool->wait();
+  rados_ioctx_destroy(io_ctx);
+  rados_shutdown(cl);
+  return 0;
+}
diff --git a/ceph/src/test/system/st_rados_create_pool.h b/ceph/src/test/system/st_rados_create_pool.h
new file mode 100644
index 00000000..f0f8a3bc
--- /dev/null
+++ b/ceph/src/test/system/st_rados_create_pool.h
@@ -0,0 +1,51 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_CREATE_POOL_H
+#define TEST_SYSTEM_ST_RADOS_CREATE_POOL_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_create_pool
+ *
+ * Waits, then posts to setup_sem.
+ * Creates a pool and populates it with some objects.
+ * Then, calls pool_setup_sem->post()
+ */
+class StRadosCreatePool : public SysTestRunnable
+{
+public:
+  static std::string get_random_buf(int sz);
+  StRadosCreatePool(int argc, const char **argv,
+		    CrossProcessSem *setup_sem,
+		    CrossProcessSem *pool_setup_sem,
+		    CrossProcessSem *close_create_pool_sem,
+		    const std::string &pool_name,
+		    int num_objects,
+		    const std::string &suffix);
+  ~StRadosCreatePool();
+  virtual int run();
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_close_create_pool;
+  std::string m_pool_name;
+  int m_num_objects;
+  std::string m_suffix;
+};
+
+#endif
diff --git a/ceph/src/test/system/st_rados_delete_objs.cc b/ceph/src/test/system/st_rados_delete_objs.cc
new file mode 100644
index 00000000..b43ffdda
--- /dev/null
+++ b/ceph/src/test/system/st_rados_delete_objs.cc
@@ -0,0 +1,71 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_delete_objs.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+
+StRadosDeleteObjs::StRadosDeleteObjs(int argc, const char **argv,
+				     CrossProcessSem *setup_sem,
+				     CrossProcessSem *deleted_sem,
+				     int num_objs,
+				     const std::string &pool_name,
+				     const std::string &suffix)
+  : SysTestRunnable(argc, argv),
+    m_setup_sem(setup_sem),
+    m_deleted_sem(deleted_sem),
+    m_num_objs(num_objs),
+    m_pool_name(pool_name),
+    m_suffix(suffix)
+{
+}
+
+StRadosDeleteObjs::~StRadosDeleteObjs()
+{
+}
+
+int StRadosDeleteObjs::run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  m_setup_sem->wait();
+  m_setup_sem->post();
+
+  rados_ioctx_t io_ctx;
+  rados_pool_create(cl, m_pool_name.c_str());
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+  for (int i = 0; i < m_num_objs; ++i) {
+    char oid[128];
+    snprintf(oid, sizeof(oid), "%d%s", i, m_suffix.c_str());
+    RETURN1_IF_NONZERO(rados_remove(io_ctx, oid));
+    if (((i % 25) == 0) || (i == m_num_objs - 1)) {
+      printf("%s: deleted object %d...\n", get_id_str(), i);
+    }
+  }
+
+  rados_ioctx_destroy(io_ctx);
+  if (m_deleted_sem)
+    m_deleted_sem->post();
+  rados_shutdown(cl);
+  return 0;
+}
diff --git a/ceph/src/test/system/st_rados_delete_objs.h b/ceph/src/test/system/st_rados_delete_objs.h
new file mode 100644
index 00000000..718f88a6
--- /dev/null
+++ b/ceph/src/test/system/st_rados_delete_objs.h
@@ -0,0 +1,48 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_DELETE_OBJS_H
+#define TEST_SYSTEM_ST_RADOS_DELETE_OBJS_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_delete_objs
+ *
+ * Waits on setup_sem, posts to it,
+ * deletes num_objs objects from the pool,
+ * and posts to deleted_sem.
+ */
+class StRadosDeleteObjs : public SysTestRunnable
+{
+public:
+  StRadosDeleteObjs(int argc, const char **argv,
+		    CrossProcessSem *setup_sem,
+		    CrossProcessSem *deleted_sem,
+		    int num_objs,
+		    const std::string &pool_name,
+		    const std::string &suffix);
+  ~StRadosDeleteObjs();
+  virtual int run();
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_deleted_sem;
+  int m_num_objs;
+  std::string m_pool_name;
+  std::string m_suffix;
+};
+
+#endif
diff --git a/ceph/src/test/system/st_rados_delete_pool.cc b/ceph/src/test/system/st_rados_delete_pool.cc
new file mode 100644
index 00000000..de553e98
--- /dev/null
+++ b/ceph/src/test/system/st_rados_delete_pool.cc
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_delete_pool.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+
+StRadosDeletePool::StRadosDeletePool(int argc, const char **argv,
+				     CrossProcessSem *pool_setup_sem,
+				     CrossProcessSem *delete_pool_sem,
+				     const std::string &pool_name)
+    : SysTestRunnable(argc, argv),
+      m_pool_setup_sem(pool_setup_sem),
+      m_delete_pool_sem(delete_pool_sem),
+      m_pool_name(pool_name)
+{
+}
+
+StRadosDeletePool::~StRadosDeletePool()
+{
+}
+
+int StRadosDeletePool::run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  m_pool_setup_sem->wait();
+  m_pool_setup_sem->post();
+
+  rados_ioctx_t io_ctx;
+  rados_pool_create(cl, m_pool_name.c_str());
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+  rados_ioctx_destroy(io_ctx);
+  printf("%s: deleting pool %s\n", get_id_str(), m_pool_name.c_str());
+  RETURN1_IF_NONZERO(rados_pool_delete(cl, m_pool_name.c_str()));
+  if (m_delete_pool_sem)
+    m_delete_pool_sem->post();
+  rados_shutdown(cl);
+  return 0;
+}
diff --git a/ceph/src/test/system/st_rados_delete_pool.h b/ceph/src/test/system/st_rados_delete_pool.h
new file mode 100644
index 00000000..5aeb9513
--- /dev/null
+++ b/ceph/src/test/system/st_rados_delete_pool.h
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_DELETE_POOL_H
+#define TEST_SYSTEM_ST_RADOS_DELETE_POOL_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_delete_pool
+ *
+ * Waits on pool_setup_sem, posts to it,
+ * deletes a pool, and posts to delete_pool_sem.
+ */
+class StRadosDeletePool : public SysTestRunnable
+{
+public:
+  StRadosDeletePool(int argc, const char **argv,
+		    CrossProcessSem *pool_setup_sem,
+		    CrossProcessSem *delete_pool_sem,
+		    const std::string &pool_name);
+  ~StRadosDeletePool();
+  virtual int run();
+private:
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_delete_pool_sem;
+  std::string m_pool_name;
+};
+
+#endif
diff --git a/ceph/src/test/system/st_rados_list_objects.cc b/ceph/src/test/system/st_rados_list_objects.cc
new file mode 100644
index 00000000..be6ead64
--- /dev/null
+++ b/ceph/src/test/system/st_rados_list_objects.cc
@@ -0,0 +1,103 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_list_objects.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using std::ostringstream;
+
+StRadosListObjects::
+StRadosListObjects(int argc, const char **argv,
+		   const std::string &pool_name,
+		   bool accept_list_errors,
+		   int midway_cnt,
+		   CrossProcessSem *pool_setup_sem,
+		   CrossProcessSem *midway_sem_wait,
+		   CrossProcessSem *midway_sem_post)
+  : SysTestRunnable(argc, argv),
+    m_accept_list_errors(accept_list_errors),
+    m_midway_cnt(midway_cnt),
+    m_pool_setup_sem(pool_setup_sem),
+    m_midway_sem_wait(midway_sem_wait),
+    m_midway_sem_post(midway_sem_post)
+{
+}
+
+StRadosListObjects::
+~StRadosListObjects()
+{
+}
+
+int StRadosListObjects::
+run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  m_pool_setup_sem->wait();
+  m_pool_setup_sem->post();
+
+  rados_ioctx_t io_ctx;
+  rados_pool_create(cl, "foo");
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, "foo", &io_ctx));
+
+  int saw = 0;
+  const char *obj_name;
+  rados_list_ctx_t h;
+  printf("%s: listing objects.\n", get_id_str());
+  RETURN1_IF_NONZERO(rados_objects_list_open(io_ctx, &h));
+  while (true) {
+    int ret = rados_objects_list_next(h, &obj_name, NULL);
+    if (ret == -ENOENT) {
+      break;
+    }
+    else if (ret != 0) {
+      if (m_accept_list_errors && (!m_midway_sem_post || saw > m_midway_cnt))
+	break;
+      printf("%s: rados_objects_list_next error: %d\n", get_id_str(), ret);
+      return ret;
+    }
+    if ((saw % 25) == 0) {
+      printf("%s: listed object %d...\n", get_id_str(), saw);
+    }
+    ++saw;
+    if (saw == m_midway_cnt) {
+      if (m_midway_sem_wait)
+	m_midway_sem_wait->wait();
+      if (m_midway_sem_post)
+	m_midway_sem_post->post();
+    }
+  }
+  rados_objects_list_close(h);
+
+  printf("%s: saw %d objects\n", get_id_str(), saw);
+
+  rados_ioctx_destroy(io_ctx);
+  rados_shutdown(cl);
+
+  return 0;
+}
diff --git a/ceph/src/test/system/st_rados_list_objects.h b/ceph/src/test/system/st_rados_list_objects.h
new file mode 100644
index 00000000..ddb1342e
--- /dev/null
+++ b/ceph/src/test/system/st_rados_list_objects.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_LIST_OBJECTS_H
+#define TEST_SYSTEM_ST_RADOS_LIST_OBJECTS_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_list_objects
+ *
+ * 1. calls pool_setup_sem->wait()
+ * 2. calls pool_setup_sem->post()
+ * 3. list some objects
+ * 4. modify_sem->wait()
+ * 5. list some objects
+ */
+class StRadosListObjects : public SysTestRunnable
+{
+public:
+  static std::string get_random_buf(int sz);
+  StRadosListObjects(int argc, const char **argv,
+		     const std::string &pool_name,
+		     bool accept_list_errors,
+		     int midway_cnt,
+		     CrossProcessSem *pool_setup_sem,
+		     CrossProcessSem *midway_sem_wait,
+		     CrossProcessSem *midway_sem_post);
+  ~StRadosListObjects();
+  virtual int run();
+private:
+  std::string m_pool_name;
+  bool m_accept_list_errors;
+  int m_midway_cnt;
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_midway_sem_wait;
+  CrossProcessSem *m_midway_sem_post;
+};
+
+#endif
diff --git a/ceph/src/test/system/st_rados_notify.cc b/ceph/src/test/system/st_rados_notify.cc
new file mode 100644
index 00000000..e74711cb
--- /dev/null
+++ b/ceph/src/test/system/st_rados_notify.cc
@@ -0,0 +1,74 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_notify.h"
+#include "systest_runnable.h"
+
+StRadosNotify::StRadosNotify(int argc, const char **argv,
+			     CrossProcessSem *setup_sem,
+			     CrossProcessSem *notify_sem,
+			     CrossProcessSem *notified_sem,
+			     int notify_retcode,
+			     const std::string &pool_name,
+			     const std::string &obj_name)
+  : SysTestRunnable(argc, argv),
+    m_setup_sem(setup_sem),
+    m_notify_sem(notify_sem),
+    m_notified_sem(notified_sem),
+    m_notify_retcode(notify_retcode),
+    m_pool_name(pool_name),
+    m_obj_name(obj_name)
+{
+}
+
+StRadosNotify::~StRadosNotify()
+{
+}
+
+int StRadosNotify::run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+
+  if (m_setup_sem) {
+    m_setup_sem->wait();
+    m_setup_sem->post();
+  }
+
+  rados_ioctx_t io_ctx;
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+  if (m_notify_sem) {
+    m_notify_sem->wait();
+    m_notify_sem->post();
+  }
+
+  printf("%s: notifying object %s\n", get_id_str(), m_obj_name.c_str());
+  RETURN1_IF_NOT_VAL(m_notify_retcode,
+		     rados_notify(io_ctx, m_obj_name.c_str(), 0, NULL, 0));
+  if (m_notified_sem) {
+    m_notified_sem->post();
+  }
+
+  rados_ioctx_destroy(io_ctx);
+  rados_shutdown(cl);
+
+  return 0;
+}
diff --git a/ceph/src/test/system/st_rados_notify.h b/ceph/src/test/system/st_rados_notify.h
new file mode 100644
index 00000000..7c65efb6
--- /dev/null
+++ b/ceph/src/test/system/st_rados_notify.h
@@ -0,0 +1,52 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_NOTIFY_H
+#define TEST_SYSTEM_ST_RADOS_NOTIFY_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_notify
+ *
+ * 1. waits on and then posts to setup_sem
+ * 2. connects and opens the pool
+ * 3. waits on and then posts to notify_sem
+ * 4. notifies on the object
+ * 5. posts to notified_sem
+ */
+class StRadosNotify : public SysTestRunnable
+{
+public:
+  StRadosNotify(int argc, const char **argv,
+		CrossProcessSem *setup_sem,
+		CrossProcessSem *notify_sem,
+		CrossProcessSem *notified_sem,
+		int notify_retcode,
+		const std::string &pool_name,
+		const std::string &obj_name);
+  ~StRadosNotify();
+  virtual int run();
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_notify_sem;
+  CrossProcessSem *m_notified_sem;
+  int m_notify_retcode;
+  std::string m_pool_name;
+  std::string m_obj_name;
+};
+
+#endif
diff --git a/ceph/src/test/system/st_rados_watch.cc b/ceph/src/test/system/st_rados_watch.cc
new file mode 100644
index 00000000..38b7b5ee
--- /dev/null
+++ b/ceph/src/test/system/st_rados_watch.cc
@@ -0,0 +1,96 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_watch.h"
+#include "systest_runnable.h"
+
+void notify_cb(uint8_t opcode, uint64_t ver, void *arg)
+{
+  int *notifies = reinterpret_cast(arg);
+  ++(*notifies);
+}
+
+StRadosWatch::StRadosWatch(int argc, const char **argv,
+			   CrossProcessSem *setup_sem,
+			   CrossProcessSem *watch_sem,
+			   CrossProcessSem *notify_sem,
+			   int num_notifies,
+			   int watch_retcode,
+			   const std::string &pool_name,
+			   const std::string &obj_name)
+  : SysTestRunnable(argc, argv),
+    m_setup_sem(setup_sem),
+    m_watch_sem(watch_sem),
+    m_notify_sem(notify_sem),
+    m_num_notifies(num_notifies),
+    m_watch_retcode(watch_retcode),
+    m_pool_name(pool_name),
+    m_obj_name(obj_name)
+{
+}
+
+StRadosWatch::
+~StRadosWatch()
+{
+}
+
+int StRadosWatch::
+run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+
+  if (m_setup_sem) {
+    m_setup_sem->wait();
+    m_setup_sem->post();
+  }
+
+  rados_ioctx_t io_ctx;
+  uint64_t handle;
+  int num_notifies = 0;
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+  printf("%s: watching object %s\n", get_id_str(), m_obj_name.c_str());
+
+  RETURN1_IF_NOT_VAL(
+    rados_watch(io_ctx, m_obj_name.c_str(), 0, &handle,
+		reinterpret_cast(notify_cb),
+		reinterpret_cast(&num_notifies)),
+    m_watch_retcode
+    );
+  if (m_watch_sem) {
+    m_watch_sem->post();
+  }
+
+  m_notify_sem->wait();
+  m_notify_sem->post();
+
+  int r = 0;
+  if (num_notifies < m_num_notifies) {
+    printf("Received fewer notifies than expected: %d < %d\n",
+	   num_notifies, m_num_notifies);
+    r = 1;
+  }
+
+  rados_unwatch(io_ctx, m_obj_name.c_str(), handle);
+  rados_ioctx_destroy(io_ctx);
+  rados_shutdown(cl);
+
+  return r;
+}
diff --git a/ceph/src/test/system/st_rados_watch.h b/ceph/src/test/system/st_rados_watch.h
new file mode 100644
index 00000000..e3e78c0c
--- /dev/null
+++ b/ceph/src/test/system/st_rados_watch.h
@@ -0,0 +1,56 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_WATCH_H
+#define TEST_SYSTEM_ST_RADOS_WATCH_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_watch
+ *
+ * 1. waits on setup_sem
+ * 2. posts to setup_sem
+ * 3. watches an object
+ * 4. posts to watch_sem
+ * 5. waits on notify_sem
+ * 6. posts to notify_sem
+ * 7. checks that the correct number of notifies were received
+ */
+class StRadosWatch : public SysTestRunnable
+{
+public:
+  StRadosWatch(int argc, const char **argv,
+	       CrossProcessSem *setup_sem,
+	       CrossProcessSem *watch_sem,
+	       CrossProcessSem *notify_sem,
+	       int num_notifies,
+	       int watch_retcode,
+	       const std::string &pool_name,
+	       const std::string &obj_name);
+  ~StRadosWatch();
+  virtual int run();
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_watch_sem;
+  CrossProcessSem *m_notify_sem;
+  int m_num_notifies;
+  int m_watch_retcode;
+  std::string m_pool_name;
+  std::string m_obj_name;
+};
+
+#endif
diff --git a/ceph/src/test/system/systest_runnable.cc b/ceph/src/test/system/systest_runnable.cc
new file mode 100644
index 00000000..ec9b823d
--- /dev/null
+++ b/ceph/src/test/system/systest_runnable.cc
@@ -0,0 +1,248 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/errno.h"
+#include "include/atomic.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#if defined(__FreeBSD__)
+#include 
+#endif
+
+using std::ostringstream;
+using std::string;
+
+static pid_t do_gettid(void)
+{
+#if defined(__linux__)
+  return static_cast < pid_t >(syscall(SYS_gettid));
+#else
+  return static_cast < pid_t >(pthread_getthreadid_np());
+#endif
+}
+
+ceph::atomic_t m_highest_id(0);
+
+SysTestRunnable::
+SysTestRunnable(int argc, const char **argv)
+  : m_argc(0),
+    m_argv(NULL),
+    m_argv_orig(NULL)
+{
+  m_started = false;
+  m_id = m_highest_id.inc();
+  memset(&m_pthread, 0, sizeof(m_pthread));
+  m_pid = 0;
+  update_id_str(false);
+  set_argv(argc, argv);
+}
+
+SysTestRunnable::
+~SysTestRunnable()
+{
+  set_argv(0, NULL);
+}
+
+const char* SysTestRunnable::
+get_id_str(void) const
+{
+  return m_id_str;
+}
+
+int SysTestRunnable::
+start()
+{
+  if (m_started) {
+    return -EDOM;
+  }
+  bool use_threads = SysTestSettings::inst().use_threads();
+  if (use_threads) {
+    int ret = pthread_create(&m_pthread, NULL, systest_runnable_pthread_helper,
+			     static_cast(this));
+    if (ret)
+      return ret;
+    m_started = true;
+    return 0;
+  }
+  else {
+    pid_t pid = fork();
+    if (pid == -1) {
+      int err = errno;
+      return -err;
+    }
+    else if (pid == 0) {
+      m_started = true;
+      m_pid = getpid();
+      void *retptr = systest_runnable_pthread_helper(static_cast(this));
+      exit((int)(uintptr_t)retptr);
+    }
+    else {
+      m_started = true;
+      m_pid = pid;
+      return 0;
+    }
+  }
+}
+
+std::string SysTestRunnable::
+join()
+{
+  if (!m_started) {
+    return "SysTestRunnable was never started.";
+  }
+  bool use_threads = SysTestSettings::inst().use_threads();
+  if (use_threads) {
+    void *ptrretval;
+    int ret = pthread_join(m_pthread, &ptrretval);
+    if (ret) {
+      ostringstream oss;
+      oss << "pthread_join failed with error " << ret;
+      return oss.str();
+    }
+    int retval = (int)(uintptr_t)ptrretval;
+    if (retval != 0) {
+      ostringstream oss;
+      oss << "ERROR " << retval;
+      return oss.str();
+    }
+    return "";
+  }
+  else {
+    int status;
+    printf("waitpid(%d)\n", m_pid);
+    pid_t pid = waitpid(m_pid, &status, 0);
+    if (pid == -1) {
+      int err = errno;
+      ostringstream oss;
+      oss << get_id_str() << " waitpid error: " << cpp_strerror(err);
+      return oss.str();
+    }
+    else if (WIFSIGNALED(status)) {
+      ostringstream oss;
+      oss << get_id_str() << " exited with a signal";
+      return oss.str();
+    }
+    else if (!WIFEXITED(status)) {
+      ostringstream oss;
+      oss << get_id_str() << " did not exit normally";
+      return oss.str();
+    }
+    else {
+      int exit_status = WEXITSTATUS(status);
+      if (exit_status != 0) {
+	ostringstream oss;
+	oss << get_id_str() << " returned exit_status " << exit_status;
+	return oss.str();
+      }
+      return "";
+    }
+  }
+}
+
+std::string SysTestRunnable::
+run_until_finished(std::vector < SysTestRunnable * > &runnables)
+{
+  int index = 0;
+  for (std::vector < SysTestRunnable * >::const_iterator r = runnables.begin();
+      r != runnables.end(); ++r) {
+    int ret = (*r)->start();
+    if (ret) {
+      ostringstream oss;
+      oss << "run_until_finished: got error " << ret
+	  << " when starting runnable " << index;
+      return oss.str();
+    }
+    ++index;
+  }
+
+  for (std::vector < SysTestRunnable * >::const_iterator r = runnables.begin();
+      r != runnables.end(); ++r) {
+    std::string rstr = (*r)->join();
+    if (!rstr.empty()) {
+      ostringstream oss;
+      oss << "run_until_finished: runnable " << (*r)->get_id_str() 
+	  << ": got error: " << rstr;
+      return oss.str();
+    }
+  }
+  printf("*******************************\n");
+  return "";
+}
+
+void *systest_runnable_pthread_helper(void *arg)
+{
+  SysTestRunnable *st = static_cast < SysTestRunnable * >(arg);
+  st->update_id_str(true);
+  int ret = st->run();
+  return (void*)(uintptr_t)ret;
+}
+
+void SysTestRunnable::
+update_id_str(bool started)
+{
+  bool use_threads = SysTestSettings::inst().use_threads();
+  char extra[128];
+  extra[0] = '\0';
+
+  if (started) {
+    if (use_threads)
+      snprintf(extra, sizeof(extra), "_[%d]", do_gettid());
+    else
+      snprintf(extra, sizeof(extra), "_[%d]", getpid());
+  }
+  if (use_threads)
+    snprintf(m_id_str, SysTestRunnable::ID_STR_SZ, "thread_%d%s", m_id, extra);
+  else
+    snprintf(m_id_str, SysTestRunnable::ID_STR_SZ, "process_%d%s", m_id, extra);
+}
+
+// Copy argv so that if some fiend decides to modify it, it's ok.
+void SysTestRunnable::
+set_argv(int argc, const char **argv)
+{
+  if (m_argv_orig != NULL) {
+    for (int i = 0; i < m_argc; ++i)
+      free((void*)(m_argv_orig[i]));
+    delete[] m_argv_orig;
+    m_argv_orig = NULL;
+    delete[] m_argv;
+    m_argv = NULL;
+    m_argc = 0;
+  }
+  if (argv == NULL)
+    return;
+  m_argc = argc;
+  m_argv_orig = new const char*[m_argc+1];
+  for (int i = 0; i < m_argc; ++i)
+    m_argv_orig[i] = strdup(argv[i]);
+  m_argv_orig[argc] = NULL;
+  m_argv = new const char*[m_argc+1];
+  for (int i = 0; i <= m_argc; ++i)
+    m_argv[i] = m_argv_orig[i];
+}
diff --git a/ceph/src/test/system/systest_runnable.h b/ceph/src/test/system/systest_runnable.h
new file mode 100644
index 00000000..8fb59f4c
--- /dev/null
+++ b/ceph/src/test/system/systest_runnable.h
@@ -0,0 +1,88 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_SYSTEM_TEST_H
+#define CEPH_SYSTEM_TEST_H
+
+#include 
+#include 
+#include 
+#include 
+
+#define RETURN1_IF_NOT_VAL(expected, expr) \
+  do {\
+    int _rinv_ret = expr;\
+    if (_rinv_ret != expected) {\
+      printf("%s: file %s, line %d: expected %d, got %d\n",\
+	     get_id_str(), __FILE__, __LINE__, expected, _rinv_ret);\
+      return 1; \
+    }\
+  } while(0);
+
+#define RETURN1_IF_NONZERO(expr) \
+  RETURN1_IF_NOT_VAL(0, expr)
+
+extern void* systest_runnable_pthread_helper(void *arg);
+
+/* Represents a single test thread / process.
+ *
+ * Inherit from this class and implement the test body in run().
+*/
+class SysTestRunnable
+{
+public:
+  static const int ID_STR_SZ = 128;
+
+  SysTestRunnable(int argc, const char **argv);
+  virtual ~SysTestRunnable();
+
+  /* Returns 0 on success; error code otherwise. */
+  virtual int run() = 0;
+
+  /* Return a string identifying the runnable. */ 
+  const char* get_id_str(void) const;
+
+  /* Start the Runnable */
+  int start();
+
+  /* Wait until the Runnable is finished. Returns an error string on failure. */
+  std::string join();
+
+  /* Starts a bunch of SystemTestRunnables and waits until they're done.
+   *
+   * Returns an error string on failure. */
+  static std::string run_until_finished(std::vector < SysTestRunnable * >&
+					runnables);
+
+protected:
+  int m_argc;
+  const char **m_argv;
+
+private:
+  SysTestRunnable(const SysTestRunnable &rhs);
+  SysTestRunnable& operator=(const SysTestRunnable &rhs);
+  void update_id_str(bool started);
+  void set_argv(int argc, const char **argv);
+
+  friend void* systest_runnable_pthread_helper(void *arg);
+
+  const char **m_argv_orig;
+  bool m_started;
+  int m_id;
+  pthread_t m_pthread;
+  int m_pid;
+  char m_id_str[ID_STR_SZ];
+};
+
+#endif
diff --git a/ceph/src/test/system/systest_settings.cc b/ceph/src/test/system/systest_settings.cc
new file mode 100644
index 00000000..b1d14a25
--- /dev/null
+++ b/ceph/src/test/system/systest_settings.cc
@@ -0,0 +1,64 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "systest_settings.h"
+
+#include 
+#include 
+#include 
+
+pthread_mutex_t g_system_test_settings_lock = PTHREAD_MUTEX_INITIALIZER;
+
+SysTestSettings& SysTestSettings::
+inst()
+{
+  pthread_mutex_lock(&g_system_test_settings_lock);
+  if (!m_inst)
+    m_inst = new SysTestSettings();
+  pthread_mutex_unlock(&g_system_test_settings_lock);
+  return *m_inst;
+}
+
+bool SysTestSettings::
+use_threads() const
+{
+  return m_use_threads;
+}
+
+std::string SysTestSettings::
+get_log_name(const std::string &suffix) const
+{
+  if (m_log_file_base.empty())
+    return "";
+  std::ostringstream oss;
+  oss << m_log_file_base << "." << suffix;
+  return oss.str();
+}
+
+SysTestSettings* SysTestSettings::
+m_inst = NULL;
+
+SysTestSettings::
+SysTestSettings()
+{
+  m_use_threads = !!getenv("USE_THREADS");
+  const char *lfb = getenv("LOG_FILE_BASE");
+  if (lfb)
+    m_log_file_base.assign(lfb);
+}
+
+SysTestSettings::
+~SysTestSettings()
+{
+}
diff --git a/ceph/src/test/system/systest_settings.h b/ceph/src/test/system/systest_settings.h
new file mode 100644
index 00000000..4c32143f
--- /dev/null
+++ b/ceph/src/test/system/systest_settings.h
@@ -0,0 +1,36 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_SYSTEM_TEST_SETTINGS_H
+#define CEPH_SYSTEM_TEST_SETTINGS_H
+
+#include 
+
+/* Singleton with settings grabbed from environment variables */
+class SysTestSettings
+{
+public:
+  static SysTestSettings& inst();
+  bool use_threads() const;
+  std::string get_log_name(const std::string &suffix) const;
+private:
+  static SysTestSettings* m_inst;
+  SysTestSettings();
+  ~SysTestSettings();
+
+  bool m_use_threads;
+  std::string m_log_file_base;
+};
+
+#endif
diff --git a/ceph/src/test/test_addrs.cc b/ceph/src/test/test_addrs.cc
new file mode 100644
index 00000000..8cb3d4d7
--- /dev/null
+++ b/ceph/src/test/test_addrs.cc
@@ -0,0 +1,64 @@
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License version 2, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/types.h"
+#include "msg/msg_types.h"
+#include "gtest/gtest.h"
+
+#include 
+
+// input, parsed+printed addr output, leftover
+// if the parse fails, output + leftover should both be blank.
+const char *addr_checks[][3] = {
+  { "127.0.0.1", "127.0.0.1:0/0", "" },
+  { "127.0.0.1 foo", "127.0.0.1:0/0", " foo" },
+  { "127.0.0.1:1234 foo", "127.0.0.1:1234/0", " foo" },
+  { "127.0.0.1:1234/5678 foo", "127.0.0.1:1234/5678", " foo" },
+  { "1.2.3:4 a", "", "" },
+  { "2607:f298:4:2243::5522", "[2607:f298:4:2243::5522]:0/0", "" },
+  { "[2607:f298:4:2243::5522]", "[2607:f298:4:2243::5522]:0/0", "" },
+  { "2607:f298:4:2243::5522a", "", "" },
+  { "[2607:f298:4:2243::5522]a", "[2607:f298:4:2243::5522]:0/0", "a" },
+  { "[2607:f298:4:2243::5522]:1234a", "[2607:f298:4:2243::5522]:1234/0", "a" },
+  { "2001:0db8:85a3:0000:0000:8a2e:0370:7334", "[2001:db8:85a3::8a2e:370:7334]:0/0", "" },
+  { "2001:2db8:85a3:4334:4324:8a2e:1370:7334", "[2001:2db8:85a3:4334:4324:8a2e:1370:7334]:0/0", "" },
+  { "::", "[::]:0/0", "" },
+  { "::zz", "[::]:0/0", "zz" },
+  { ":: 12:34", "[::]:0/0", " 12:34" },
+  { NULL, NULL, NULL },
+};
+
+
+TEST(Msgr, TestAddrParsing)
+{
+
+  for (unsigned i = 0; addr_checks[i][0]; ++i) {
+    entity_addr_t a;
+    const char *end = "";
+    bool ok = a.parse(addr_checks[i][0], &end);
+    string in = addr_checks[i][0];
+    string out;
+    if (ok) {
+      stringstream ss;
+      ss << a;
+      getline(ss, out);
+    }
+    string left = end;
+      
+    cout << "'" << addr_checks[i][0] << "' -> '" << out << "' + '" << left << "'" << std::endl;
+
+    ASSERT_EQ(out, addr_checks[i][1]);
+    ASSERT_EQ(left, addr_checks[i][2]);
+  }
+}
diff --git a/ceph/src/test/test_arch.cc b/ceph/src/test/test_arch.cc
new file mode 100644
index 00000000..7b7ffe42
--- /dev/null
+++ b/ceph/src/test/test_arch.cc
@@ -0,0 +1,20 @@
+
+#include 
+#include 
+#include 
+
+#include "arch/probe.h"
+#include "arch/intel.h"
+#include "arch/neon.h"
+
+int main(int argc, char **argv)
+{
+	ceph_arch_probe();
+	assert(ceph_arch_probed);
+
+	printf("ceph_arch_intel_sse42 = %d\n", ceph_arch_intel_sse42);
+        printf("ceph_arch_intel_sse2 = %d\n", ceph_arch_intel_sse2);
+	printf("ceph_arch_neon = %d\n", ceph_arch_neon);
+
+	return 0;
+}
diff --git a/ceph/src/test/test_c_headers.c b/ceph/src/test/test_c_headers.c
new file mode 100644
index 00000000..0f41966a
--- /dev/null
+++ b/ceph/src/test/test_c_headers.c
@@ -0,0 +1,22 @@
+#include "include/cephfs/libcephfs.h"
+#include "include/rados/librados.h"
+
+#ifdef __cplusplus
+#error "test invalid: only use C mode"
+#endif
+
+int main(int argc, char **argv)
+{
+	int ret;
+	(void)ret; // squash unused warning
+
+	/* librados.h */
+	rados_t cluster;
+	ret = rados_create(&cluster, NULL);
+
+	/* libcephfs.h */
+	struct ceph_mount_info *cmount;
+	ret = ceph_create(&cmount, NULL);
+
+	return 0;
+}
diff --git a/ceph/src/test/test_cfuse_cache_invalidate.cc b/ceph/src/test/test_cfuse_cache_invalidate.cc
new file mode 100644
index 00000000..1389e5d7
--- /dev/null
+++ b/ceph/src/test/test_cfuse_cache_invalidate.cc
@@ -0,0 +1,53 @@
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define REGION 1048576
+int main(int argc, char *argv[]) {
+
+  pid_t p = fork();
+  char buf[REGION];
+  memset(buf, 0, sizeof(buf));
+
+  if (p != 0) {
+    int done = 0;
+    int fd = open(argv[1], O_RDWR|O_CREAT, 0644);
+    if (fd < 0) {
+      perror(argv[1]);
+      return 1;
+    }
+
+    int i = 0;
+    while(!done) {
+      printf("writing %d\n", i++);
+      assert(pwrite(fd, buf, REGION, 0) == REGION);
+      int status;
+      int ret = waitpid(p, &status, WNOHANG);
+      assert(ret >= 0);
+      if (ret > 0) {
+	done = 1;
+      }
+    }
+    close(fd);
+  } else {
+    sleep(1);
+    int fd = open(argv[2], O_RDONLY, 0644);
+    if (fd < 0) {
+      perror(argv[2]);
+      return 1;
+    }
+
+    printf("reading\n");
+    assert(pread(fd, buf, REGION, 0) == REGION);
+    close(fd);
+  }
+
+  return 0;
+}
diff --git a/ceph/src/test/test_cors.cc b/ceph/src/test/test_cors.cc
new file mode 100644
index 00000000..4d385bc0
--- /dev/null
+++ b/ceph/src/test/test_cors.cc
@@ -0,0 +1,909 @@
+#include 
+#include 
+#include 
+#include 
+extern "C"{
+#include 
+}
+#include "common/ceph_crypto.h"
+#include 
+#include 
+#define S3_BUCKET_NAME "s3testgw.fcgi"
+#define SWIFT_BUCKET_NAME "swift3testgw.fcgi"
+#define BUCKET_URL \
+  ((g_test->get_key_type() == KEY_TYPE_S3)?(string("/"S3_BUCKET_NAME)):(string("/swift/v1/"SWIFT_BUCKET_NAME)))
+#define GTEST
+#ifdef GTEST
+#include 
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+#include "common/code_environment.h"
+#include "common/ceph_argparse.h"
+#include "common/Finisher.h"
+#include "global/global_init.h"
+#include "rgw/rgw_cors.h"
+#include "rgw/rgw_cors_s3.h"
+
+using namespace std;
+
+#define CURL_VERBOSE 0
+#define HTTP_RESPONSE_STR "RespCode"
+#define CEPH_CRYPTO_HMACSHA1_DIGESTSIZE 20
+
+extern "C" int ceph_armor(char *dst, const char *dst_end, 
+                          const char *src, const char *end);
+enum key_type {
+  KEY_TYPE_UNDEFINED = 0,
+  KEY_TYPE_SWIFT,
+  KEY_TYPE_S3
+};
+
+static void print_usage(char *exec){
+  cout << "Usage: " << exec << " \n";
+  cout << "Options:\n"
+          "-g  - The ip address of the gateway\n"
+          "-p  - The port number of the gateway\n"
+          "-k  - The key type, either SWIFT or S3\n"
+          "-s3  - Only, if the key type is S3, gives S3 credentials\n"
+          "-swift  - Only if the key type is SWIFT, and gives the SWIFT credentials\n";
+}
+class test_cors_helper {
+  private:
+    string host;
+    string port;
+    string creds;
+    CURL *curl_inst;
+    map response;
+    list extra_hdrs;
+    string *resp_data;
+    unsigned resp_code;
+    key_type kt;
+  public:
+    test_cors_helper() : resp_data(NULL), kt(KEY_TYPE_UNDEFINED){
+      curl_global_init(CURL_GLOBAL_ALL);
+    }
+    ~test_cors_helper(){
+      curl_global_cleanup();
+    }
+    int send_request(string method, string uri, 
+                     size_t (*function)(void *,size_t,size_t,void *) = 0,
+                     void *ud = 0, size_t length = 0);
+    int extract_input(unsigned argc, char *argv[]);
+    string& get_response(string hdr){
+      return response[hdr];
+    }
+    void set_extra_header(string hdr){
+      extra_hdrs.push_back(hdr);
+    }
+    void set_response(char *val);
+    void set_response_data(char *data, size_t len){
+      if(resp_data) delete resp_data;
+      resp_data = new string(data, len);
+      /*cout << resp_data->c_str() << "\n";*/
+    }
+    const string *get_response_data(){return resp_data;}
+    unsigned get_resp_code(){return resp_code;}
+    key_type get_key_type(){return kt;}
+};
+
+int test_cors_helper::extract_input(unsigned argc, char *argv[]){
+#define ERR_CHECK_NEXT_PARAM(o) \
+  if((loop + 1) >= argc)return -1; \
+  else o = argv[loop+1];
+
+  for(unsigned loop = 1;loop < argc; loop += 2){
+    if(strcmp(argv[loop], "-g") == 0){
+      ERR_CHECK_NEXT_PARAM(host);
+    }else if(strcmp(argv[loop], "-k") == 0){
+      string type;
+      ERR_CHECK_NEXT_PARAM(type);
+      if(type.compare("S3") == 0)kt = KEY_TYPE_S3;
+      else if(type.compare("SWIFT") == 0)kt = KEY_TYPE_SWIFT;
+    }else if(strcmp(argv[loop],"-s3") == 0){
+      ERR_CHECK_NEXT_PARAM(creds);
+    }else if(strcmp(argv[loop],"-swift") == 0){
+      ERR_CHECK_NEXT_PARAM(creds);
+    }else if(strcmp(argv[loop],"-p") == 0){
+      ERR_CHECK_NEXT_PARAM(port);
+    }else return -1;
+  }
+  if(host.length() <= 0 ||
+     creds.length() <= 0)
+    return -1;
+  return 0;
+}
+
+void test_cors_helper::set_response(char *r){
+  string sr(r), h, v;
+  size_t off = sr.find(": ");
+  if(off != string::npos){
+    h.assign(sr, 0, off);
+    v.assign(sr, off + 2, sr.find("\r\n") - (off+2));
+  }else{
+    /*Could be the status code*/
+    if(sr.find("HTTP/") != string::npos){
+      h.assign(HTTP_RESPONSE_STR);
+      off = sr.find(" ");
+      v.assign(sr, off + 1, sr.find("\r\n") - (off + 1));
+      resp_code = atoi((v.substr(0, 3)).c_str());
+    }
+  }
+  response[h] = v;
+}
+
+size_t write_header(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_cors_helper *h = static_cast(ud);
+  h->set_response((char *)ptr);
+  return size*nmemb;
+}
+
+size_t write_data(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_cors_helper *h = static_cast(ud);
+  h->set_response_data((char *)ptr, size*nmemb);
+  return size*nmemb;
+}
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+static void calc_hmac_sha1(const char *key, int key_len,
+                    const char *msg, int msg_len, char *dest)
+/* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
+{
+  ceph::crypto::HMACSHA1 hmac((const unsigned char *)key, key_len);
+  hmac.Update((const unsigned char *)msg, msg_len);
+  hmac.Final((unsigned char *)dest);
+  
+  char hex_str[(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2) + 1];
+  buf_to_hex((unsigned char *)dest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE, hex_str);
+}
+
+static int get_s3_auth(string method, string creds, string date, string res, string& out){
+  string aid, secret, auth_hdr;
+  size_t off = creds.find(":");
+  out = "";
+  if(off != string::npos){
+    aid.assign(creds, 0, off);
+    secret.assign(creds, off + 1, string::npos);
+
+    /*sprintf(auth_hdr, "%s\n\n\n%s\n%s", req_type, date, res);*/
+    char hmac_sha1[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+    char b64[65]; /* 64 is really enough */
+    auth_hdr.append(method + string("\n\n\n") + date + string("\n") + res);
+    calc_hmac_sha1(secret.c_str(), secret.length(), auth_hdr.c_str(), auth_hdr.length(), hmac_sha1);
+    int ret = ceph_armor(b64, b64 + 64, hmac_sha1,
+                         hmac_sha1 + CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+    if (ret < 0) {
+      cout << "ceph_armor failed\n";
+      return -1;
+    }
+    b64[ret] = 0;
+    out.append(aid + string(":") + b64);
+  }else return -1;
+  return 0;
+}
+
+void get_date(string& d){
+  struct timeval tv;
+  char date[64];
+  struct tm tm;
+  char *days[] = {(char *)"Sun", (char *)"Mon", (char *)"Tue",
+                  (char *)"Wed", (char *)"Thu", (char *)"Fri", 
+                  (char *)"Sat"};
+  char *months[] = {(char *)"Jan", (char *)"Feb", (char *)"Mar", 
+                    (char *)"Apr", (char *)"May", (char *)"Jun",
+                    (char *)"Jul",(char *) "Aug", (char *)"Sep", 
+                    (char *)"Oct", (char *)"Nov", (char *)"Dec"};
+  gettimeofday(&tv, NULL);
+  gmtime_r(&tv.tv_sec, &tm);
+  sprintf(date, "%s, %d %s %d %d:%d:%d GMT", 
+          days[tm.tm_wday], 
+          tm.tm_mday, months[tm.tm_mon], 
+          tm.tm_year + 1900,
+          tm.tm_hour, tm.tm_min, 0 /*tm.tm_sec*/);
+  d = date;
+}
+
+int test_cors_helper::send_request(string method, string res, 
+                                   size_t (*read_function)( void *,size_t,size_t,void *),
+                                   void *ud,
+                                   size_t length){
+  string url;
+  string auth, date;
+  url.append(string("http://") + host);
+  if(port.length() > 0)url.append(string(":") + port);
+  url.append(res);
+  curl_inst = curl_easy_init();
+  if(curl_inst){
+    curl_easy_setopt(curl_inst, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_CUSTOMREQUEST, method.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_VERBOSE, CURL_VERBOSE);
+    curl_easy_setopt(curl_inst, CURLOPT_HEADERFUNCTION, write_header);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEHEADER, (void *)this);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEFUNCTION, write_data);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEDATA, (void *)this);
+    if(read_function){
+      curl_easy_setopt(curl_inst, CURLOPT_READFUNCTION, read_function);
+      curl_easy_setopt(curl_inst, CURLOPT_READDATA, (void *)ud);
+      curl_easy_setopt(curl_inst, CURLOPT_UPLOAD, 1L);
+      curl_easy_setopt(curl_inst, CURLOPT_INFILESIZE_LARGE, (curl_off_t)length);
+    }
+
+    get_date(date);
+    string http_date;
+    http_date.append(string("Date: ") + date);
+    if(kt == KEY_TYPE_S3){
+      string s3auth;
+      if(get_s3_auth(method, creds, date, res, s3auth) < 0)return -1;
+      auth.append(string("Authorization: AWS ") + s3auth);
+    } else if(kt == KEY_TYPE_SWIFT){
+      auth.append(string("X-Auth-Token: ") + creds);
+    } else {
+      cout << "Unknown state (" << kt << ")\n";
+      return -1;
+    }
+
+    struct curl_slist *slist = NULL;
+    slist = curl_slist_append(slist, auth.c_str());
+    slist = curl_slist_append(slist, http_date.c_str());
+    for(list::iterator it = extra_hdrs.begin();
+        it != extra_hdrs.end(); ++it){
+      slist = curl_slist_append(slist, (*it).c_str());
+    }
+    if(read_function)
+      curl_slist_append(slist, "Expect:");
+    curl_easy_setopt(curl_inst, CURLOPT_HTTPHEADER, slist); 
+
+    response.erase(response.begin(), response.end());
+    extra_hdrs.erase(extra_hdrs.begin(), extra_hdrs.end());
+    CURLcode res = curl_easy_perform(curl_inst);
+    if(res != CURLE_OK){
+      cout << "Curl perform failed for " << url << ", res: " << 
+        curl_easy_strerror(res) << "\n";
+      return -1;
+    }
+    curl_slist_free_all(slist);
+  }
+  curl_easy_cleanup(curl_inst);
+  return 0;
+}
+
+test_cors_helper *g_test;
+Finisher *finisher;
+
+static int create_bucket(void){
+  if(g_test->get_key_type() == KEY_TYPE_S3){
+    g_test->send_request(string("PUT"), string("/"S3_BUCKET_NAME));
+    if(g_test->get_resp_code() != 200U){
+      cout << "Error creating bucket, http code " << g_test->get_resp_code();
+      return -1;
+    }
+  }else if(g_test->get_key_type() == KEY_TYPE_SWIFT){
+    g_test->send_request(string("PUT"), string("/swift/v1/"SWIFT_BUCKET_NAME));
+    if(g_test->get_resp_code() != 201U){
+      cout << "Error creating bucket, http code " << g_test->get_resp_code();
+      return -1;
+    }
+  }else return -1;
+  return 0;
+}
+
+static int delete_bucket(void){
+  if(g_test->get_key_type() == KEY_TYPE_S3){
+    g_test->send_request(string("DELETE"), string("/"S3_BUCKET_NAME));
+    if(g_test->get_resp_code() != 204U){
+      cout << "Error deleting bucket, http code " << g_test->get_resp_code();
+      return -1;
+    }
+  }else if(g_test->get_key_type() == KEY_TYPE_SWIFT){
+    g_test->send_request(string("DELETE"), string("/swift/v1/"SWIFT_BUCKET_NAME));
+    if(g_test->get_resp_code() != 204U){
+      cout << "Error deleting bucket, http code " << g_test->get_resp_code();
+      return -1;
+    }
+  }else return -1;
+  return 0;
+}
+
+RGWCORSRule *xml_to_cors_rule(string s){
+  RGWCORSConfiguration_S3 *cors_config;
+  RGWCORSXMLParser_S3 parser(g_ceph_context);
+  const string *data = g_test->get_response_data();
+  if (!parser.init()) {
+    return NULL;
+  }
+  if (!parser.parse(data->c_str(), data->length(), 1)) {
+    return NULL;
+  }
+  cors_config = (RGWCORSConfiguration_S3 *)parser.find_first("CORSConfiguration");
+  if (!cors_config) {
+    return NULL;
+  }
+  return cors_config->host_name_rule(s.c_str());
+}
+
+size_t cors_read_xml(void *ptr, size_t s, size_t n, void *ud){
+  stringstream *ss = (stringstream *)ud;
+  size_t len = ss->str().length();
+  if(s*n < len){
+    cout << "Cannot copy xml data, as len is not enough\n";
+    return 0;
+  }
+  memcpy(ptr, (void *)ss->str().c_str(), len);
+  return len;
+}
+
+void send_cors(set o, set h,
+               list e, uint8_t flags, 
+               unsigned max_age){
+  if(g_test->get_key_type() == KEY_TYPE_S3){
+    RGWCORSRule rule(o, h, e, flags, max_age);
+    RGWCORSConfiguration config;
+    config.stack_rule(rule);
+    stringstream ss;
+    RGWCORSConfiguration_S3 *s3;
+    s3 = static_cast(&config);
+    s3->to_xml(ss);
+
+    g_test->send_request(string("PUT"), string("/"S3_BUCKET_NAME"?cors"), cors_read_xml, 
+                         (void *)&ss, ss.str().length());
+  }else if(g_test->get_key_type() == KEY_TYPE_SWIFT){
+    set::iterator it;
+    string a_o;
+    for(it = o.begin(); it != o.end(); ++it){
+      if(a_o.length() > 0)a_o.append(" ");
+      a_o.append(*it);
+    }
+    g_test->set_extra_header(string("X-Container-Meta-Access-Control-Allow-Origin: ") + a_o);
+
+    if(!h.empty()){
+      string a_h;
+      for(it = h.begin(); it != h.end(); ++it){
+        if(a_h.length() > 0)a_h.append(" ");
+        a_h.append(*it);
+      }
+      g_test->set_extra_header(string("X-Container-Meta-Access-Control-Allow-Headers: ") + a_h);
+    }
+    if(!e.empty()){
+      string e_h;
+      for(list::iterator lit = e.begin(); lit != e.end(); ++lit){
+        if(e_h.length() > 0)e_h.append(" ");
+        e_h.append(*lit);
+      }
+      g_test->set_extra_header(string("X-Container-Meta-Access-Control-Expose-Headers: ") + e_h);
+    }
+    if(max_age != CORS_MAX_AGE_INVALID){
+      char age[32];
+      sprintf(age, "%u", max_age);
+      g_test->set_extra_header(string("X-Container-Meta-Access-Control-Max-Age: ") + string(age));
+    }
+    //const char *data = "1";
+    stringstream ss;
+    ss << "1";
+    g_test->send_request(string("POST"), string("/swift/v1/"SWIFT_BUCKET_NAME), cors_read_xml, 
+                         (void *)&ss, 1);
+  }
+}
+
+TEST(TestCORS, getcors_firsttime){
+  if(g_test->get_key_type() == KEY_TYPE_SWIFT)return;
+  ASSERT_EQ(0, create_bucket());
+  g_test->send_request(string("GET"), string("/"S3_BUCKET_NAME"?cors"));
+  EXPECT_EQ(404U, g_test->get_resp_code());
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, putcors_firsttime){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "example.com");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+
+  /*Now get the CORS and check if its fine*/
+  if(g_test->get_key_type() == KEY_TYPE_S3){
+    g_test->send_request(string("GET"), string("/"S3_BUCKET_NAME"?cors"));
+    EXPECT_EQ(200U, g_test->get_resp_code());
+
+    RGWCORSRule *r = xml_to_cors_rule(string("example.com"));
+    EXPECT_TRUE(r != NULL);
+    if(!r)return;
+
+    EXPECT_TRUE((r->get_allowed_methods() & (RGW_CORS_GET | RGW_CORS_PUT))
+                == (RGW_CORS_GET | RGW_CORS_PUT));
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, putcors_invalid_hostname){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "*.example.*");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ((400U), g_test->get_resp_code());
+  origins.erase(origins.begin(), origins.end());
+ 
+  if((g_test->get_key_type() != KEY_TYPE_SWIFT)){
+    origins.insert(origins.end(), "");
+    send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+    EXPECT_EQ((400U), g_test->get_resp_code());
+    origins.erase(origins.begin(), origins.end());
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, putcors_invalid_headers){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "www.example.com");
+  h.insert(h.end(), "*-Header-*");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ((400U), g_test->get_resp_code());
+  h.erase(h.begin(), h.end());
+ 
+  if((g_test->get_key_type() != KEY_TYPE_SWIFT)){
+    h.insert(h.end(), "");
+    flags = RGW_CORS_GET | RGW_CORS_PUT;
+    send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+    EXPECT_EQ((400U), g_test->get_resp_code());
+    h.erase(h.begin(), h.end());
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, optionscors_test_options_1){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "*.example.com");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+
+  g_test->set_extra_header(string("Origin: a.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->set_extra_header(string("Access-Control-Allow-Headers: SomeHeader"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("a.example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0U, s.length());
+    s = g_test->get_response(string("Access-Control-Max-Age"));
+    EXPECT_EQ(0U, s.length());
+    s = g_test->get_response(string("Access-Control-Expose-Headers"));
+    EXPECT_EQ(0U, s.length());
+  }
+
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, optionscors_test_options_2){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "*.example.com");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT | RGW_CORS_DELETE | RGW_CORS_HEAD;
+  
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+
+  g_test->set_extra_header(string("Origin: a.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: HEAD"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("a.example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("HEAD"));
+  }
+
+  g_test->set_extra_header(string("Origin: foo.bar.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: HEAD"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("foo.bar.example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("HEAD"));
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, optionscors_test_options_3){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "*");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT | RGW_CORS_DELETE | RGW_CORS_HEAD;
+  
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+
+  /*Check for HEAD in Access-Control-Allow-Methods*/
+  g_test->set_extra_header(string("Origin: a.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: HEAD"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("a.example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("HEAD"));
+  }
+
+  /*Check for DELETE in Access-Control-Allow-Methods*/
+  g_test->set_extra_header(string("Origin: foo.bar"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: DELETE"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("foo.bar"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("DELETE"));
+  }
+
+  /*Check for PUT in Access-Control-Allow-Methods*/
+  g_test->set_extra_header(string("Origin: foo.bar"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: PUT"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("foo.bar"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("PUT"));
+  }
+
+  /*Check for POST in Access-Control-Allow-Methods*/
+  g_test->set_extra_header(string("Origin: foo.bar"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: POST"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("foo.bar"));
+    if(g_test->get_key_type() == KEY_TYPE_S3){
+      s = g_test->get_response(string("Access-Control-Allow-Methods"));
+      EXPECT_EQ(0U, s.length());
+    }else{
+      s = g_test->get_response(string("Access-Control-Allow-Methods"));
+      EXPECT_EQ(0, s.compare("POST"));
+    }
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, optionscors_test_options_4){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "example.com");
+  h.insert(h.end(), "Header1");
+  h.insert(h.end(), "Header2");
+  h.insert(h.end(), "*");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+  
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0U, s.length());
+  }
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->set_extra_header(string("Access-Control-Allow-Headers: Header1"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0, s.compare("Header1"));
+  }
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->set_extra_header(string("Access-Control-Allow-Headers: Header2"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0, s.compare("Header2"));
+  }
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->set_extra_header(string("Access-Control-Allow-Headers: Header2, Header1"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0, s.compare("Header2,Header1"));
+  }
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->set_extra_header(string("Access-Control-Allow-Headers: Header1, Header2"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0, s.compare("Header1,Header2"));
+  }
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->set_extra_header(string("Access-Control-Allow-Headers: Header1, Header2, Header3"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0, s.compare("Header1,Header2,Header3"));
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, optionscors_test_options_5){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "example.com");
+  e.insert(e.end(), "Expose1");
+  e.insert(e.end(), "Expose2");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+  
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Expose-Headers"));
+    EXPECT_EQ(0, s.compare("Expose1,Expose2"));
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, optionscors_test_options_6){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+  unsigned err = (g_test->get_key_type() == KEY_TYPE_SWIFT)?401U:403U;
+
+  origins.insert(origins.end(), "http://www.example.com");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+  
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(err, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: http://example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(err, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: www.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(err, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: http://www.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  origins.erase(origins.begin(), origins.end());
+  origins.insert(origins.end(), "*.example.com");
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+
+  g_test->set_extra_header(string("Origin: .example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: http://example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(err, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: www.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: http://www.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  g_test->set_extra_header(string("Origin: https://www.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  origins.erase(origins.begin(), origins.end());
+  origins.insert(origins.end(), "https://example*.com");
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+
+  g_test->set_extra_header(string("Origin: https://example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: http://example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(err, g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: www.example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(err, g_test->get_resp_code());
+
+  g_test->set_extra_header(string("Origin: https://example.a.b.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, optionscors_test_options_7){
+  ASSERT_EQ(0, create_bucket());
+  set origins, h;
+  list e;
+
+  origins.insert(origins.end(), "example.com");
+  h.insert(h.end(), "Header*");
+  h.insert(h.end(), "Hdr-*-Length");
+  h.insert(h.end(), "*-Length");
+  h.insert(h.end(), "foo*foo");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+  
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+  
+  g_test->set_extra_header(string("Origin: example.com"));
+  g_test->set_extra_header(string("Access-Control-Request-Method: GET"));
+  g_test->set_extra_header(string("Access-Control-Allow-Headers: Header1, Header2, Header3, "
+                                  "Hdr--Length, Hdr-1-Length, Header-Length, Content-Length, foofoofoo"));
+  g_test->send_request(string("OPTIONS"), BUCKET_URL);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  if(g_test->get_resp_code() == 200){
+    string s = g_test->get_response(string("Access-Control-Allow-Origin"));
+    EXPECT_EQ(0, s.compare("example.com"));
+    s = g_test->get_response(string("Access-Control-Allow-Methods"));
+    EXPECT_EQ(0, s.compare("GET"));
+    s = g_test->get_response(string("Access-Control-Allow-Headers"));
+    EXPECT_EQ(0, s.compare("Header1,Header2,Header3,"
+                           "Hdr--Length,Hdr-1-Length,Header-Length,Content-Length,foofoofoo"));
+  }
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, deletecors_firsttime){
+  if(g_test->get_key_type() == KEY_TYPE_SWIFT)return;
+  ASSERT_EQ(0, create_bucket());
+  g_test->send_request("DELETE", "/"S3_BUCKET_NAME"?cors");
+  EXPECT_EQ(204U, g_test->get_resp_code());
+  ASSERT_EQ(0, delete_bucket());
+}
+
+TEST(TestCORS, deletecors_test){
+  set origins, h;
+  list e;
+  if(g_test->get_key_type() == KEY_TYPE_SWIFT)return;
+  ASSERT_EQ(0, create_bucket());
+  origins.insert(origins.end(), "example.com");
+  uint8_t flags = RGW_CORS_GET | RGW_CORS_PUT;
+  
+  send_cors(origins, h, e, flags, CORS_MAX_AGE_INVALID);
+  EXPECT_EQ(((g_test->get_key_type() == KEY_TYPE_SWIFT)?202U:200U), g_test->get_resp_code());
+
+  g_test->send_request("GET", "/"S3_BUCKET_NAME"?cors");
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  g_test->send_request("DELETE", "/"S3_BUCKET_NAME"?cors");
+  EXPECT_EQ(204U, g_test->get_resp_code());
+  g_test->send_request("GET", "/"S3_BUCKET_NAME"?cors");
+  EXPECT_EQ(404U, g_test->get_resp_code());
+  ASSERT_EQ(0, delete_bucket());
+}
+
+int main(int argc, char *argv[]){
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_test = new test_cors_helper();
+  finisher = new Finisher(g_ceph_context);
+#ifdef GTEST
+  ::testing::InitGoogleTest(&argc, argv);
+#endif
+  finisher->start();
+
+  if(g_test->extract_input((unsigned)argc, argv) < 0){
+    print_usage(argv[0]);
+    return -1;
+  }
+#ifdef GTEST
+  int r = RUN_ALL_TESTS();
+  if (r >= 0) {
+    cout << "There are failures in the test case\n";
+    return -1;
+  }
+#endif
+  finisher->stop();
+  return 0;
+}
+
diff --git a/ceph/src/test/test_filejournal.cc b/ceph/src/test/test_filejournal.cc
new file mode 100644
index 00000000..e50e5dce
--- /dev/null
+++ b/ceph/src/test/test_filejournal.cc
@@ -0,0 +1,546 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include 
+#include 
+#include 
+
+#include "common/ceph_argparse.h"
+#include "common/common_init.h"
+#include "global/global_init.h"
+#include "common/config.h"
+#include "common/Finisher.h"
+#include "os/FileJournal.h"
+#include "include/Context.h"
+#include "common/Mutex.h"
+#include "common/safe_io.h"
+
+Finisher *finisher;
+Cond sync_cond;
+char path[200];
+uuid_d fsid;
+bool directio = false;
+bool aio = false;
+
+// ----
+Cond cond;
+Mutex wait_lock("lock");
+bool done;
+
+void wait()
+{
+  wait_lock.Lock();
+  while (!done)
+    cond.Wait(wait_lock);
+  wait_lock.Unlock();
+}
+
+// ----
+class C_Sync {
+public:
+  Cond cond;
+  Mutex lock;
+  bool done;
+  C_SafeCond *c;
+
+  C_Sync()
+    : lock("C_Sync::lock"), done(false) {
+    c = new C_SafeCond(&lock, &cond, &done);
+  }
+  ~C_Sync() {
+    lock.Lock();
+    //cout << "wait" << std::endl;
+    while (!done)
+      cond.Wait(lock);
+    //cout << "waited" << std::endl;
+    lock.Unlock();
+  }
+};
+
+unsigned size_mb = 200;
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  char mb[10];
+  sprintf(mb, "%u", size_mb);
+  g_ceph_context->_conf->set_val("osd_journal_size", mb);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  finisher = new Finisher(g_ceph_context);
+  
+  if (!args.empty()) {
+    size_t copy_len = std::min(sizeof(path)-1, strlen(args[0]));
+    strncpy(path, args[0], copy_len);
+    path[copy_len] = '\0';
+  } else {
+    srand(getpid()+time(0));
+    snprintf(path, sizeof(path), "/tmp/ceph_test_filejournal.tmp.%d", rand());
+  }
+  cout << "path " << path << std::endl;
+
+  ::testing::InitGoogleTest(&argc, argv);
+
+  finisher->start();
+
+  cout << "DIRECTIO OFF  AIO OFF" << std::endl;
+  directio = false;
+  aio = false;
+  int r = RUN_ALL_TESTS();
+  if (r >= 0) {
+    cout << "DIRECTIO ON  AIO OFF" << std::endl;
+    directio = true;
+    r = RUN_ALL_TESTS();
+
+    if (r >= 0) {
+      cout << "DIRECTIO ON  AIO ON" << std::endl;
+      aio = true;
+      r = RUN_ALL_TESTS();
+    }
+  }
+  
+  finisher->stop();
+
+  unlink(path);
+  
+  return r;
+}
+
+TEST(TestFileJournal, Create) {
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+}
+
+TEST(TestFileJournal, WriteSmall) {
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  bufferlist bl;
+  bl.append("small");
+  j.submit_entry(1, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
+  wait();
+
+  j.close();
+}
+
+TEST(TestFileJournal, WriteBig) {
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  bufferlist bl;
+  while (bl.length() < size_mb*1000/2) {
+    char foo[1024*1024];
+    memset(foo, 1, sizeof(foo));
+    bl.append(foo, sizeof(foo));
+  }
+  j.submit_entry(1, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
+  wait();
+
+  j.close();
+}
+
+TEST(TestFileJournal, WriteMany) {
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
+  
+  bufferlist bl;
+  bl.append("small");
+  uint64_t seq = 1;
+  for (int i=0; i<100; i++) {
+    bl.append("small");
+    j.submit_entry(seq++, bl, 0, gb.new_sub());
+  }
+
+  gb.activate();
+
+  wait();
+
+  j.close();
+}
+
+TEST(TestFileJournal, WriteManyVecs) {
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
+
+  bufferlist first;
+  first.append("small");
+  j.submit_entry(1, first, 0, gb.new_sub());
+
+  bufferlist bl;
+  for (int i=0; i= 2) {
+	cout << "replacing at offset " << o << std::endl;
+	memcpy(buf+o, newneedle, strlen(newneedle));
+      } else {
+	cout << "leaving at offset " << o << std::endl;
+      }
+      n++;
+    }
+  }
+  ASSERT_EQ(n, 4);
+  close(fd);
+  fd = open(path, O_WRONLY);
+  ASSERT_GE(fd, 0);
+  r = safe_write(fd, buf, sizeof(buf));
+  ASSERT_EQ(r, 0);
+  close(fd);
+
+  j.open(1);
+
+  bufferlist inbl;
+  string v;
+  uint64_t seq = 0;
+  ASSERT_EQ(true, j.read_entry(inbl, seq));
+  ASSERT_EQ(seq, 2ull);
+  inbl.copy(0, inbl.length(), v);
+  ASSERT_EQ(needle, v);
+  inbl.clear();
+  v.clear();
+  ASSERT_TRUE(!j.read_entry(inbl, seq));
+
+  j.make_writeable();
+  j.close();
+}
+
+TEST(TestFileJournal, WriteTrim) {
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  list ls;
+  
+  bufferlist bl;
+  char foo[1024*1024];
+  memset(foo, 1, sizeof(foo));
+
+  uint64_t seq = 1, committed = 0;
+
+  for (unsigned i=0; ic);
+
+    while (ls.size() > size_mb/2) {
+      delete ls.front();
+      ls.pop_front();
+      committed++;
+      j.committed_thru(committed);
+    }
+  }
+
+  while (ls.size()) {
+    delete ls.front();
+    ls.pop_front();
+    j.committed_thru(committed);
+  }
+
+  j.close();
+}
+
+TEST(TestFileJournal, WriteTrimSmall) {
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  list ls;
+  
+  bufferlist bl;
+  char foo[1024*1024];
+  memset(foo, 1, sizeof(foo));
+
+  uint64_t seq = 1, committed = 0;
+
+  for (unsigned i=0; ic);
+
+    while (ls.size() > size_mb/2) {
+      delete ls.front();
+      ls.pop_front();
+      committed++;
+      j.committed_thru(committed);
+    }
+  }
+
+  while (ls.size()) {
+    delete ls.front();
+    ls.pop_front();
+    j.committed_thru(committed);
+  }
+
+  j.close();
+}
+
+TEST(TestFileJournal, ReplayDetectCorruptFooterMagic) {
+  g_ceph_context->_conf->set_val("journal_ignore_corruption", "true");
+  g_ceph_context->_conf->set_val("journal_write_header_frequency", "1");
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
+
+  const char *needle =    "i am a needle";
+  for (unsigned i = 1; i <= 4; ++i) {
+    bufferlist bl;
+    bl.append(needle);
+    j.submit_entry(i, bl, 0, gb.new_sub());
+  }
+  gb.activate();
+  wait();
+
+  bufferlist bl;
+  bl.append("needle");
+  j.submit_entry(5, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
+  wait();
+
+  j.close();
+  int fd = open(path, O_WRONLY);
+
+  cout << "corrupting journal" << std::endl;
+  j.open(0);
+  j.corrupt_footer_magic(fd, 2);
+
+  uint64_t seq = 0;
+  bl.clear();
+  bool corrupt = false;
+  bool result = j.read_entry(bl, seq, &corrupt);
+  ASSERT_TRUE(result);
+  ASSERT_EQ(seq, 1UL);
+  ASSERT_FALSE(corrupt);
+
+  result = j.read_entry(bl, seq, &corrupt);
+  ASSERT_FALSE(result);
+  ASSERT_TRUE(corrupt);
+
+  j.make_writeable();
+  j.close();
+  ::close(fd);
+}
+
+TEST(TestFileJournal, ReplayDetectCorruptPayload) {
+  g_ceph_context->_conf->set_val("journal_ignore_corruption", "true");
+  g_ceph_context->_conf->set_val("journal_write_header_frequency", "1");
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
+
+  const char *needle =    "i am a needle";
+  for (unsigned i = 1; i <= 4; ++i) {
+    bufferlist bl;
+    bl.append(needle);
+    j.submit_entry(i, bl, 0, gb.new_sub());
+  }
+  gb.activate();
+  wait();
+
+  bufferlist bl;
+  bl.append("needle");
+  j.submit_entry(5, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
+  wait();
+
+  j.close();
+  int fd = open(path, O_WRONLY);
+
+  cout << "corrupting journal" << std::endl;
+  j.open(0);
+  j.corrupt_payload(fd, 2);
+
+  uint64_t seq = 0;
+  bl.clear();
+  bool corrupt = false;
+  bool result = j.read_entry(bl, seq, &corrupt);
+  ASSERT_TRUE(result);
+  ASSERT_EQ(seq, 1UL);
+  ASSERT_FALSE(corrupt);
+
+  result = j.read_entry(bl, seq, &corrupt);
+  ASSERT_FALSE(result);
+  ASSERT_TRUE(corrupt);
+
+  j.make_writeable();
+  j.close();
+  ::close(fd);
+}
+
+TEST(TestFileJournal, ReplayDetectCorruptHeader) {
+  g_ceph_context->_conf->set_val("journal_ignore_corruption", "true");
+  g_ceph_context->_conf->set_val("journal_write_header_frequency", "1");
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  fsid.generate_random();
+  FileJournal j(fsid, finisher, &sync_cond, path, directio, aio);
+  ASSERT_EQ(0, j.create());
+  j.make_writeable();
+
+  C_GatherBuilder gb(g_ceph_context, new C_SafeCond(&wait_lock, &cond, &done));
+
+  const char *needle =    "i am a needle";
+  for (unsigned i = 1; i <= 4; ++i) {
+    bufferlist bl;
+    bl.append(needle);
+    j.submit_entry(i, bl, 0, gb.new_sub());
+  }
+  gb.activate();
+  wait();
+
+  bufferlist bl;
+  bl.append("needle");
+  j.submit_entry(5, bl, 0, new C_SafeCond(&wait_lock, &cond, &done));
+  wait();
+
+  j.close();
+  int fd = open(path, O_WRONLY);
+
+  cout << "corrupting journal" << std::endl;
+  j.open(0);
+  j.corrupt_header_magic(fd, 2);
+
+  uint64_t seq = 0;
+  bl.clear();
+  bool corrupt = false;
+  bool result = j.read_entry(bl, seq, &corrupt);
+  ASSERT_TRUE(result);
+  ASSERT_EQ(seq, 1UL);
+  ASSERT_FALSE(corrupt);
+
+  result = j.read_entry(bl, seq, &corrupt);
+  ASSERT_FALSE(result);
+  ASSERT_TRUE(corrupt);
+
+  j.make_writeable();
+  j.close();
+  ::close(fd);
+}
diff --git a/ceph/src/test/test_get_blkdev_size.cc b/ceph/src/test/test_get_blkdev_size.cc
new file mode 100644
index 00000000..ba28f1cd
--- /dev/null
+++ b/ceph/src/test/test_get_blkdev_size.cc
@@ -0,0 +1,35 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "common/blkdev.h"
+
+int main(int argc, char **argv)
+{
+	int fd, ret;
+	int64_t size;
+
+	if (argc != 2) {
+		fprintf(stderr, "usage: %s \n", argv[0]);
+		return -1;
+	}
+
+	fd = open(argv[1], O_RDONLY);
+	if (fd < 0) {
+		perror("open");
+		return -1;
+	}
+
+	ret = get_block_device_size(fd, &size);
+	if (ret < 0) {
+		fprintf(stderr, "get_block_device_size: %s\n", strerror(-ret));
+		return -1;
+	}
+
+	fprintf(stdout, "%" PRId64, size);
+
+	return 0;
+}
diff --git a/ceph/src/test/test_ipaddr.cc b/ceph/src/test/test_ipaddr.cc
new file mode 100644
index 00000000..82008214
--- /dev/null
+++ b/ceph/src/test/test_ipaddr.cc
@@ -0,0 +1,478 @@
+#include "include/ipaddr.h"
+#include "gtest/gtest.h"
+
+#include 
+
+static void ipv4(struct sockaddr_in *addr, const char *s) {
+  int err;
+
+  addr->sin_family = AF_INET;
+  err = inet_pton(AF_INET, s, &addr->sin_addr);
+  ASSERT_EQ(1, err);
+}
+
+static void ipv6(struct sockaddr_in6 *addr, const char *s) {
+  int err;
+
+  addr->sin6_family = AF_INET6;
+  err = inet_pton(AF_INET6, s, &addr->sin6_addr);
+  ASSERT_EQ(1, err);
+}
+
+TEST(CommonIPAddr, TestNotFound)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in a_one;
+  struct sockaddr_in6 a_two;
+  struct sockaddr_in net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv4(&a_one, "10.11.12.13");
+  ipv6(&a_two, "2001:1234:5678:90ab::cdef");
+  ipv4(&net, "10.11.234.56");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 24);
+  ASSERT_EQ(0, result);
+}
+
+TEST(CommonIPAddr, TestV4_Simple)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in a_one;
+  struct sockaddr_in6 a_two;
+  struct sockaddr_in net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv4(&a_one, "10.11.12.13");
+  ipv6(&a_two, "2001:1234:5678:90ab::cdef");
+  ipv4(&net, "10.11.12.42");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 24);
+  ASSERT_EQ((struct sockaddr*)&a_one, result);
+}
+
+TEST(CommonIPAddr, TestV4_Prefix25)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in a_one;
+  struct sockaddr_in a_two;
+  struct sockaddr_in net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv4(&a_one, "10.11.12.13");
+  ipv4(&a_two, "10.11.12.129");
+  ipv4(&net, "10.11.12.128");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 25);
+  ASSERT_EQ((struct sockaddr*)&a_two, result);
+}
+
+TEST(CommonIPAddr, TestV4_Prefix16)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in a_one;
+  struct sockaddr_in a_two;
+  struct sockaddr_in net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv4(&a_one, "10.1.1.2");
+  ipv4(&a_two, "10.2.1.123");
+  ipv4(&net, "10.2.0.0");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 16);
+  ASSERT_EQ((struct sockaddr*)&a_two, result);
+}
+
+TEST(CommonIPAddr, TestV4_PrefixTooLong)
+{
+  struct ifaddrs one;
+  struct sockaddr_in a_one;
+  struct sockaddr_in net;
+  const struct sockaddr *result;
+
+  one.ifa_next = NULL;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  ipv4(&a_one, "10.11.12.13");
+  ipv4(&net, "10.11.12.12");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 42);
+  ASSERT_EQ(0, result);
+}
+
+TEST(CommonIPAddr, TestV4_PrefixZero)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in6 a_one;
+  struct sockaddr_in a_two;
+  struct sockaddr_in net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv6(&a_one, "2001:1234:5678:900F::cdef");
+  ipv4(&a_two, "10.1.2.3");
+  ipv4(&net, "255.0.1.2");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 0);
+  ASSERT_EQ((struct sockaddr*)&a_two, result);
+}
+
+TEST(CommonIPAddr, TestV6_Simple)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in a_one;
+  struct sockaddr_in6 a_two;
+  struct sockaddr_in6 net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv4(&a_one, "10.11.12.13");
+  ipv6(&a_two, "2001:1234:5678:90ab::cdef");
+  ipv6(&net, "2001:1234:5678:90ab::dead:beef");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 64);
+  ASSERT_EQ((struct sockaddr*)&a_two, result);
+}
+
+TEST(CommonIPAddr, TestV6_Prefix57)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in6 a_one;
+  struct sockaddr_in6 a_two;
+  struct sockaddr_in6 net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv6(&a_one, "2001:1234:5678:900F::cdef");
+  ipv6(&a_two, "2001:1234:5678:90ab::cdef");
+  ipv6(&net, "2001:1234:5678:90ab::dead:beef");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 57);
+  ASSERT_EQ((struct sockaddr*)&a_two, result);
+}
+
+TEST(CommonIPAddr, TestV6_PrefixTooLong)
+{
+  struct ifaddrs one;
+  struct sockaddr_in6 a_one;
+  struct sockaddr_in6 net;
+  const struct sockaddr *result;
+
+  one.ifa_next = NULL;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  ipv6(&a_one, "2001:1234:5678:900F::cdef");
+  ipv6(&net, "2001:1234:5678:900F::cdee");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 9000);
+  ASSERT_EQ(0, result);
+}
+
+TEST(CommonIPAddr, TestV6_PrefixZero)
+{
+  struct ifaddrs one, two;
+  struct sockaddr_in a_one;
+  struct sockaddr_in6 a_two;
+  struct sockaddr_in6 net;
+  const struct sockaddr *result;
+
+  one.ifa_next = &two;
+  one.ifa_addr = (struct sockaddr*)&a_one;
+
+  two.ifa_next = NULL;
+  two.ifa_addr = (struct sockaddr*)&a_two;
+
+  ipv4(&a_one, "10.2.3.4");
+  ipv6(&a_two, "2001:f00b::1");
+  ipv6(&net, "ff00::1");
+
+  result = find_ip_in_subnet(&one, (struct sockaddr*)&net, 0);
+  ASSERT_EQ((struct sockaddr*)&a_two, result);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Empty)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_Junk)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("foo", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_SlashNum)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("/24", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_Slash)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("/", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv4)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv4Slash)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123/", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv4SlashNegative)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123/-3", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv4SlashJunk)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123/foo", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv6)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv6Slash)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef/", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv6SlashNegative)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef/-3", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_Bad_IPv6SlashJunk)
+{
+  struct sockaddr network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef/foo", &network, &prefix_len);
+  ASSERT_EQ(ok, false);
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv4_0)
+{
+  struct sockaddr_in network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123/0", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(0U, prefix_len);
+  ASSERT_EQ(AF_INET, network.sin_family);
+  ASSERT_EQ(0, network.sin_port);
+  struct sockaddr_in want;
+  ipv4(&want, "123.123.123.123");
+  ASSERT_EQ(want.sin_addr.s_addr, network.sin_addr.s_addr);
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv4_13)
+{
+  struct sockaddr_in network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123/13", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(13U, prefix_len);
+  ASSERT_EQ(AF_INET, network.sin_family);
+  ASSERT_EQ(0, network.sin_port);
+  struct sockaddr_in want;
+  ipv4(&want, "123.123.123.123");
+  ASSERT_EQ(want.sin_addr.s_addr, network.sin_addr.s_addr);
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv4_32)
+{
+  struct sockaddr_in network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123/32", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(32U, prefix_len);
+  ASSERT_EQ(AF_INET, network.sin_family);
+  ASSERT_EQ(0, network.sin_port);
+  struct sockaddr_in want;
+  ipv4(&want, "123.123.123.123");
+  ASSERT_EQ(want.sin_addr.s_addr, network.sin_addr.s_addr);
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv4_42)
+{
+  struct sockaddr_in network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("123.123.123.123/42", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(42U, prefix_len);
+  ASSERT_EQ(AF_INET, network.sin_family);
+  ASSERT_EQ(0, network.sin_port);
+  struct sockaddr_in want;
+  ipv4(&want, "123.123.123.123");
+  ASSERT_EQ(want.sin_addr.s_addr, network.sin_addr.s_addr);
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv6_0)
+{
+  struct sockaddr_in6 network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef/0", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(0U, prefix_len);
+  ASSERT_EQ(AF_INET6, network.sin6_family);
+  ASSERT_EQ(0, network.sin6_port);
+  struct sockaddr_in6 want;
+  ipv6(&want, "2001:1234:5678:90ab::dead:beef");
+  ASSERT_EQ(0, memcmp(want.sin6_addr.s6_addr, network.sin6_addr.s6_addr, sizeof(network.sin6_addr.s6_addr)));
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv6_67)
+{
+  struct sockaddr_in6 network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef/67", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(67U, prefix_len);
+  ASSERT_EQ(AF_INET6, network.sin6_family);
+  ASSERT_EQ(0, network.sin6_port);
+  struct sockaddr_in6 want;
+  ipv6(&want, "2001:1234:5678:90ab::dead:beef");
+  ASSERT_EQ(0, memcmp(want.sin6_addr.s6_addr, network.sin6_addr.s6_addr, sizeof(network.sin6_addr.s6_addr)));
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv6_128)
+{
+  struct sockaddr_in6 network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef/128", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(128U, prefix_len);
+  ASSERT_EQ(AF_INET6, network.sin6_family);
+  ASSERT_EQ(0, network.sin6_port);
+  struct sockaddr_in6 want;
+  ipv6(&want, "2001:1234:5678:90ab::dead:beef");
+  ASSERT_EQ(0, memcmp(want.sin6_addr.s6_addr, network.sin6_addr.s6_addr, sizeof(network.sin6_addr.s6_addr)));
+}
+
+TEST(CommonIPAddr, ParseNetwork_IPv6_9000)
+{
+  struct sockaddr_in6 network;
+  unsigned int prefix_len;
+  bool ok;
+
+  ok = parse_network("2001:1234:5678:90ab::dead:beef/9000", (struct sockaddr*)&network, &prefix_len);
+  ASSERT_EQ(ok, true);
+  ASSERT_EQ(9000U, prefix_len);
+  ASSERT_EQ(AF_INET6, network.sin6_family);
+  ASSERT_EQ(0, network.sin6_port);
+  struct sockaddr_in6 want;
+  ipv6(&want, "2001:1234:5678:90ab::dead:beef");
+  ASSERT_EQ(0, memcmp(want.sin6_addr.s6_addr, network.sin6_addr.s6_addr, sizeof(network.sin6_addr.s6_addr)));
+}
diff --git a/ceph/src/test/test_mutate.cc b/ceph/src/test/test_mutate.cc
new file mode 100644
index 00000000..b9e0d717
--- /dev/null
+++ b/ceph/src/test/test_mutate.cc
@@ -0,0 +1,110 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+/*
+ * Test Ioctx::operate
+ */
+
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+#include "common/config.h"
+#include "global/global_init.h"
+#include "include/rados/librados.hpp"
+#include "include/types.h"
+
+#include 
+#include 
+#include 
+
+using std::cerr;
+using std::string;
+
+using namespace librados;
+
+static void usage(void)
+{
+  cerr << "--oid           set object id to 'operate' on" << std::endl;
+  cerr << "--pool          set pool to 'operate' on" << std::endl;
+}
+
+int main(int argc, const char **argv)
+{
+  int ret = 0;
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  string val;
+  string oid("ceph_test_object");
+  string pool_name("test_pool");
+  for (std::vector::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    }
+    else if (ceph_argparse_witharg(args, i, &val, "--oid", "-o", (char*)NULL)) {
+      oid = val;
+    }
+    else if (ceph_argparse_witharg(args, i, &val, "--pool", "-p", (char*)NULL)) {
+      pool_name = val;
+    }
+    else {
+      cerr << "unknown command line option: " << *i << std::endl;
+      cerr << std::endl;
+      usage();
+      return 2;
+    }
+  }
+
+  Rados rados;
+  if (rados.init_with_context(g_ceph_context) < 0) {
+     cerr << "couldn't initialize rados!" << std::endl;
+     return 1;
+  }
+  if (rados.conf_read_file(NULL) < 0) {
+     cerr << "failed to read rados configuration file!" << std::endl;
+     return 1;
+  }
+  if (rados.connect() < 0) {
+     cerr << "couldn't connect to cluster!" << std::endl;
+     return 1;
+  }
+
+  librados::ObjectWriteOperation o;
+  IoCtx ioctx;
+  if (rados.pool_lookup(pool_name.c_str()) <= 0) {
+    ret = rados.pool_create(pool_name.c_str());
+    if (ret) {
+       cerr << "failed to create pool named '" << pool_name
+	    << "': error " << ret << std::endl;
+       return 1;
+    }
+  }
+  ret = rados.ioctx_create(pool_name.c_str(), ioctx);
+  if (ret) {
+     cerr << "failed to create ioctx for pool '" << pool_name
+	  << "': error " << ret << std::endl;
+     return 1;
+  }
+  librados::ObjectWriteOperation op;
+  op.create(true);
+  ret = ioctx.operate(oid, &op);
+  if (ret) {
+     cerr << "ioctx.operate failed: ret = " << ret << std::endl;
+     return 1;
+  }
+
+  return 0;
+}
diff --git a/ceph/src/test/test_prebufferedstreambuf.cc b/ceph/src/test/test_prebufferedstreambuf.cc
new file mode 100644
index 00000000..489a985d
--- /dev/null
+++ b/ceph/src/test/test_prebufferedstreambuf.cc
@@ -0,0 +1,95 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/PrebufferedStreambuf.h"
+#include "gtest/gtest.h"
+
+TEST(PrebufferedStreambuf, Empty)
+{
+  char buf[10];
+  PrebufferedStreambuf sb(buf, sizeof(buf));
+
+  std::istream is(&sb);
+  std::string out;
+  getline(is, out);
+  ASSERT_EQ("", out);
+}
+
+TEST(PrebufferedStreambuf, Simple)
+{
+  char buf[10];
+  PrebufferedStreambuf sb(buf, sizeof(buf));
+
+  std::ostream os(&sb);
+  os << "test";
+
+  std::istream is(&sb);
+  std::string out;
+  getline(is, out);
+  ASSERT_EQ("test", out);
+}
+
+TEST(PrebufferedStreambuf, Multiline)
+{
+  char buf[10];
+  PrebufferedStreambuf sb(buf, sizeof(buf));
+
+  std::ostream os(&sb);
+  const char *s = "this is a line\nanother line\nand a third\nwhee!\n";
+  os << s;
+
+  std::istream is(&sb);
+  std::string out;
+  getline(is, out, is.widen(0));
+  ASSERT_EQ(s, out);
+}
+
+TEST(PrebufferedStreambuf, Withnull)
+{
+  char buf[10];
+  PrebufferedStreambuf sb(buf, sizeof(buf));
+
+  std::ostream os(&sb);
+  std::string s("null \0 and more", 15);
+  os << s;
+
+  std::istream is(&sb);
+  std::string out;
+  getline(is, out);
+  ASSERT_EQ(s, out);
+}
+
+TEST(PrebufferedStreambuf, SimpleOverflow)
+{
+  char buf[10];
+  PrebufferedStreambuf sb(buf, sizeof(buf));
+
+  std::ostream os(&sb);
+  const char *s = "hello, this is longer than buf[10]";
+  os << s;
+
+  ASSERT_EQ(s, sb.get_str());
+
+  std::istream is(&sb);
+  std::string out;
+  getline(is, out);
+  ASSERT_EQ(s, out);
+}
+
+TEST(PrebufferedStreambuf, ManyOverflow)
+{
+  char buf[10];
+  PrebufferedStreambuf sb(buf, sizeof(buf));
+
+  std::ostream os(&sb);
+  const char *s = "hello, this way way way way way way way way way way way way way way way way way way way way way way way way way _way_ longer than buf[10]";
+  os << s;
+
+  ASSERT_EQ(s, sb.get_str());
+
+  std::istream is(&sb);
+  std::string out;
+  getline(is, out);
+  ASSERT_EQ(s, out);
+}
+
diff --git a/ceph/src/test/test_rewrite_latency.cc b/ceph/src/test/test_rewrite_latency.cc
new file mode 100644
index 00000000..67e7c675
--- /dev/null
+++ b/ceph/src/test/test_rewrite_latency.cc
@@ -0,0 +1,47 @@
+
+#include 
+#include 
+#include 
+
+#include "include/utime.h"
+#include "common/Clock.h"
+#include "common/errno.h"
+
+using namespace std;
+
+int main(int argc, const char **argv)
+{
+  const char *fn = argv[1];
+  multimap latency;
+  unsigned max = 10;
+
+  int fd = ::open(fn, O_CREAT|O_RDWR, 0644);
+  if (fd < 1) {
+    int err = errno;
+    cerr << "failed to open " << fn << " with " << cpp_strerror(err) << std::endl;
+    return -1;
+  }
+
+  while (true) {
+    utime_t now = ceph_clock_now(NULL);
+    int r = ::pwrite(fd, fn, strlen(fn), 0);
+    assert(r >= 0);
+    utime_t lat = ceph_clock_now(NULL);
+    lat -= now;
+    utime_t oldmin;
+    if (!latency.empty())
+      oldmin = latency.begin()->first;
+    latency.insert(make_pair(lat, now));
+    utime_t newmin = latency.begin()->first;
+    while (latency.size() > max)
+      latency.erase(latency.begin());
+    if (oldmin == newmin) {
+      cout << "latency\tat" << std::endl;
+      for (multimap::reverse_iterator p = latency.rbegin();
+	   p != latency.rend();
+	   ++p) {
+	cout << p->first << "\t" << p->second << std::endl;
+      }
+    }
+  }
+}
diff --git a/ceph/src/test/test_rgw_admin_log.cc b/ceph/src/test/test_rgw_admin_log.cc
new file mode 100644
index 00000000..f49a107d
--- /dev/null
+++ b/ceph/src/test/test_rgw_admin_log.cc
@@ -0,0 +1,1608 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+extern "C"{
+#include 
+}
+#include "common/ceph_crypto.h"
+#include "include/str_list.h"
+#include "common/ceph_json.h"
+#include "common/code_environment.h"
+#include "common/ceph_argparse.h"
+#include "common/Finisher.h"
+#include "global/global_init.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_bucket.h"
+#include "rgw/rgw_rados.h"
+#include "include/utime.h"
+#include "include/object.h"
+#define GTEST
+#ifdef GTEST
+#include 
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+using namespace std;
+
+#define CURL_VERBOSE 0
+#define HTTP_RESPONSE_STR "RespCode"
+#define CEPH_CRYPTO_HMACSHA1_DIGESTSIZE 20
+#define RGW_ADMIN_RESP_PATH "/tmp/.test_rgw_admin_resp"
+#define TEST_BUCKET_NAME "test_bucket"
+#define TEST_BUCKET_OBJECT "test_object"
+#define TEST_BUCKET_OBJECT_1 "test_object1"
+#define TEST_BUCKET_OBJECT_SIZE 1024
+
+static string uid = "ceph";
+static string display_name = "CEPH";
+
+extern "C" int ceph_armor(char *dst, const char *dst_end, 
+                          const char *src, const char *end);
+static void print_usage(char *exec){
+  cout << "Usage: " << exec << " \n";
+  cout << "Options:\n"
+          "-g  - The ip address of the gateway\n"
+          "-p  - The port number of the gateway\n"
+          "-c  - Absolute path of ceph config file\n"
+          "-rgw-admin  - radosgw-admin absolute path\n";
+}
+
+namespace admin_log {
+class test_helper {
+  private:
+    string host;
+    string port;
+    string creds;
+    string rgw_admin_path;
+    string conf_path;
+    CURL *curl_inst;
+    map response;
+    list extra_hdrs;
+    string *resp_data;
+    unsigned resp_code;
+  public:
+    test_helper() : resp_data(NULL){
+      curl_global_init(CURL_GLOBAL_ALL);
+    }
+    ~test_helper(){
+      curl_global_cleanup();
+    }
+    int send_request(string method, string uri, 
+                     size_t (*function)(void *,size_t,size_t,void *) = 0,
+                     void *ud = 0, size_t length = 0);
+    int extract_input(int argc, char *argv[]);
+    string& get_response(string hdr){
+      return response[hdr];
+    }
+    void set_extra_header(string hdr){
+      extra_hdrs.push_back(hdr);
+    }
+    void set_response(char *val);
+    void set_response_data(char *data, size_t len){
+      if(resp_data) delete resp_data;
+      resp_data = new string(data, len);
+    }
+    string& get_rgw_admin_path() {
+      return rgw_admin_path;
+    }
+    string& get_ceph_conf_path() {
+      return conf_path;
+    }
+    void set_creds(string& c) {
+      creds = c;
+    }
+    const string *get_response_data(){return resp_data;}
+    unsigned get_resp_code(){return resp_code;}
+};
+
+int test_helper::extract_input(int argc, char *argv[]){
+#define ERR_CHECK_NEXT_PARAM(o) \
+  if(((int)loop + 1) >= argc)return -1;		\
+  else o = argv[loop+1];
+
+  for(unsigned loop = 1;loop < (unsigned)argc; loop += 2){
+    if(strcmp(argv[loop], "-g") == 0){
+      ERR_CHECK_NEXT_PARAM(host);
+    }else if(strcmp(argv[loop],"-p") == 0){
+      ERR_CHECK_NEXT_PARAM(port);
+    }else if(strcmp(argv[loop], "-c") == 0){
+      ERR_CHECK_NEXT_PARAM(conf_path);
+    }else if(strcmp(argv[loop], "-rgw-admin") == 0){
+      ERR_CHECK_NEXT_PARAM(rgw_admin_path);
+    }else return -1;
+  }
+  if(host.length() <= 0 ||
+     rgw_admin_path.length() <= 0)
+    return -1;
+  return 0;
+}
+
+void test_helper::set_response(char *r){
+  string sr(r), h, v;
+  size_t off = sr.find(": ");
+  if(off != string::npos){
+    h.assign(sr, 0, off);
+    v.assign(sr, off + 2, sr.find("\r\n") - (off+2));
+  }else{
+    /*Could be the status code*/
+    if(sr.find("HTTP/") != string::npos){
+      h.assign(HTTP_RESPONSE_STR);
+      off = sr.find(" ");
+      v.assign(sr, off + 1, sr.find("\r\n") - (off + 1));
+      resp_code = atoi((v.substr(0, 3)).c_str());
+    }
+  }
+  response[h] = v;
+}
+
+size_t write_header(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = static_cast(ud);
+  h->set_response((char *)ptr);
+  return size*nmemb;
+}
+
+size_t write_data(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = static_cast(ud);
+  h->set_response_data((char *)ptr, size*nmemb);
+  return size*nmemb;
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+static void calc_hmac_sha1(const char *key, int key_len,
+                    const char *msg, int msg_len, char *dest)
+/* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
+{
+  ceph::crypto::HMACSHA1 hmac((const unsigned char *)key, key_len);
+  hmac.Update((const unsigned char *)msg, msg_len);
+  hmac.Final((unsigned char *)dest);
+  
+  char hex_str[(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2) + 1];
+  admin_log::buf_to_hex((unsigned char *)dest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE, hex_str);
+}
+
+static int get_s3_auth(string method, string creds, string date, string res, string& out){
+  string aid, secret, auth_hdr;
+  string tmp_res;
+  size_t off = creds.find(":");
+  out = "";
+  if(off != string::npos){
+    aid.assign(creds, 0, off);
+    secret.assign(creds, off + 1, string::npos);
+
+    /*sprintf(auth_hdr, "%s\n\n\n%s\n%s", req_type, date, res);*/
+    char hmac_sha1[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+    char b64[65]; /* 64 is really enough */
+    size_t off = res.find("?");
+    if(off == string::npos)
+      tmp_res = res;
+    else
+      tmp_res.assign(res, 0, off);
+    auth_hdr.append(method + string("\n\n\n") + date + string("\n") + tmp_res);
+    admin_log::calc_hmac_sha1(secret.c_str(), secret.length(), 
+                               auth_hdr.c_str(), auth_hdr.length(), hmac_sha1);
+    int ret = ceph_armor(b64, b64 + 64, hmac_sha1,
+                         hmac_sha1 + CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+    if (ret < 0) {
+      cout << "ceph_armor failed\n";
+      return -1;
+    }
+    b64[ret] = 0;
+    out.append(aid + string(":") + b64);
+  }else return -1;
+  return 0;
+}
+
+void get_date(string& d){
+  struct timeval tv;
+  char date[64];
+  struct tm tm;
+  char *days[] = {(char *)"Sun", (char *)"Mon", (char *)"Tue",
+                  (char *)"Wed", (char *)"Thu", (char *)"Fri", 
+                  (char *)"Sat"};
+  char *months[] = {(char *)"Jan", (char *)"Feb", (char *)"Mar", 
+                    (char *)"Apr", (char *)"May", (char *)"Jun",
+                    (char *)"Jul",(char *) "Aug", (char *)"Sep", 
+                    (char *)"Oct", (char *)"Nov", (char *)"Dec"};
+  gettimeofday(&tv, NULL);
+  gmtime_r(&tv.tv_sec, &tm);
+  sprintf(date, "%s, %d %s %d %d:%d:%d GMT", 
+          days[tm.tm_wday], 
+          tm.tm_mday, months[tm.tm_mon], 
+          tm.tm_year + 1900,
+          tm.tm_hour, tm.tm_min, tm.tm_sec);
+  d = date;
+}
+
+int test_helper::send_request(string method, string res, 
+                                   size_t (*read_function)( void *,size_t,size_t,void *),
+                                   void *ud,
+                                   size_t length){
+  string url;
+  string auth, date;
+  url.append(string("http://") + host);
+  if(port.length() > 0)url.append(string(":") + port);
+  url.append(res);
+  curl_inst = curl_easy_init();
+  if(curl_inst){
+    curl_easy_setopt(curl_inst, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_CUSTOMREQUEST, method.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_VERBOSE, CURL_VERBOSE);
+    curl_easy_setopt(curl_inst, CURLOPT_HEADERFUNCTION, admin_log::write_header);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEHEADER, (void *)this);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEFUNCTION, admin_log::write_data);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEDATA, (void *)this);
+    if(read_function){
+      curl_easy_setopt(curl_inst, CURLOPT_READFUNCTION, read_function);
+      curl_easy_setopt(curl_inst, CURLOPT_READDATA, (void *)ud);
+      curl_easy_setopt(curl_inst, CURLOPT_UPLOAD, 1L);
+      curl_easy_setopt(curl_inst, CURLOPT_INFILESIZE_LARGE, (curl_off_t)length);
+    }
+
+    get_date(date);
+    string http_date;
+    http_date.append(string("Date: ") + date);
+
+    string s3auth;
+    if (admin_log::get_s3_auth(method, creds, date, res, s3auth) < 0)
+      return -1;
+    auth.append(string("Authorization: AWS ") + s3auth);
+
+    struct curl_slist *slist = NULL;
+    slist = curl_slist_append(slist, auth.c_str());
+    slist = curl_slist_append(slist, http_date.c_str());
+    for(list::iterator it = extra_hdrs.begin();
+        it != extra_hdrs.end(); ++it){
+      slist = curl_slist_append(slist, (*it).c_str());
+    }
+    if(read_function)
+      curl_slist_append(slist, "Expect:");
+    curl_easy_setopt(curl_inst, CURLOPT_HTTPHEADER, slist); 
+
+    response.erase(response.begin(), response.end());
+    extra_hdrs.erase(extra_hdrs.begin(), extra_hdrs.end());
+    CURLcode res = curl_easy_perform(curl_inst);
+    if(res != CURLE_OK){
+      cout << "Curl perform failed for " << url << ", res: " << 
+        curl_easy_strerror(res) << "\n";
+      return -1;
+    }
+    curl_slist_free_all(slist);
+  }
+  curl_easy_cleanup(curl_inst);
+  return 0;
+}
+};
+
+admin_log::test_helper *g_test;
+Finisher *finisher;
+
+int run_rgw_admin(string& cmd, string& resp) {
+  pid_t pid;
+  pid = fork();
+  if (pid == 0) {
+    /* child */
+    list l;
+    get_str_list(cmd, " \t", l);
+    char *argv[l.size()];
+    unsigned loop = 1;
+
+    argv[0] = (char *)"radosgw-admin";
+    for (list::iterator it = l.begin(); 
+         it != l.end(); ++it) {
+      argv[loop++] = (char *)(*it).c_str();
+    }
+    argv[loop] = NULL;
+    close(1);
+    stdout = fopen(RGW_ADMIN_RESP_PATH, "w+");
+    if (!stdout) {
+      cout << "Unable to open stdout file" << std::endl;
+    }
+    execv((g_test->get_rgw_admin_path()).c_str(), argv); 
+  } else if (pid > 0) {
+    int status;
+    waitpid(pid, &status, 0);
+    if (WIFEXITED(status)) {
+      if(WEXITSTATUS(status) != 0) {
+        cout << "Child exited with status " << WEXITSTATUS(status) << std::endl;
+        return -1;
+      }
+    }
+    ifstream in;
+    struct stat st;
+
+    if (stat(RGW_ADMIN_RESP_PATH, &st) < 0) {
+      cout << "Error stating the admin response file, errno " << errno << std::endl;
+      return -1;
+    } else {
+      char *data = (char *)malloc(st.st_size + 1);
+      in.open(RGW_ADMIN_RESP_PATH);
+      in.read(data, st.st_size);
+      in.close();
+      data[st.st_size] = 0;
+      resp = data;
+      free(data);
+      unlink(RGW_ADMIN_RESP_PATH);
+      /* cout << "radosgw-admin " << cmd << ": " << resp << std::endl; */
+    }
+  } else 
+    return -1;
+  return 0;
+}
+
+int get_creds(string& json, string& creds) {
+  JSONParser parser;
+  if(!parser.parse(json.c_str(), json.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+
+  RGWUserInfo info;
+  decode_json_obj(info, &parser);
+  creds = "";
+  for(map::iterator it = info.access_keys.begin();
+      it != info.access_keys.end(); ++it) {
+    RGWAccessKey _k = it->second;
+    /*cout << "accesskeys [ " << it->first << " ] = " << 
+      "{ " << _k.id << ", " << _k.key << ", " << _k.subuser << "}" << std::endl;*/
+    creds.append(it->first + string(":") + _k.key);
+    break;
+  }
+  return 0;
+}
+
+int user_create(string& uid, string& display_name, bool set_creds = true) {
+  stringstream ss;
+  string creds;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user create --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  get_creds(out, creds);
+  if(set_creds)
+    g_test->set_creds(creds);
+  return 0;
+}
+
+int user_info(string& uid, string& display_name, RGWUserInfo& uinfo) {
+  stringstream ss;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user info --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error reading user information" << std::endl;
+    return -1;
+  }
+  JSONParser parser;
+  if(!parser.parse(out.c_str(), out.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  decode_json_obj(uinfo, &parser);
+  return 0;
+}
+
+int user_rm(string& uid, string& display_name) {
+  stringstream ss;
+  ss << "-c " << g_test->get_ceph_conf_path() << 
+    " metadata rm --metadata-key=user:" << uid;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error removing user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int caps_add(const char * name, const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps add --caps=" <<
+     name << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int caps_rm(const char * name, const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps rm --caps=" <<
+     name << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+static int create_bucket(void){
+  g_test->send_request(string("PUT"), string("/"TEST_BUCKET_NAME));
+  if(g_test->get_resp_code() != 200U){
+    cout << "Error creating bucket, http code " << g_test->get_resp_code();
+    return -1;
+  }
+  return 0;
+}
+
+static int delete_bucket(void){
+  g_test->send_request(string("DELETE"), string("/"TEST_BUCKET_NAME));
+  if(g_test->get_resp_code() != 204U){
+    cout << "Error deleting bucket, http code " << g_test->get_resp_code();
+    return -1;
+  }
+  return 0;
+}
+
+size_t read_dummy_post(void *ptr, size_t s, size_t n, void *ud) {
+  int dummy = 0;
+  memcpy(ptr, &dummy, sizeof(dummy));
+  return sizeof(dummy);
+}
+
+size_t read_bucket_object(void *ptr, size_t s, size_t n, void *ud) {
+  memcpy(ptr, ud, TEST_BUCKET_OBJECT_SIZE);
+  return TEST_BUCKET_OBJECT_SIZE;
+}
+
+static int put_bucket_obj(const char *obj_name, char *data, unsigned len) {
+  string req = "/"TEST_BUCKET_NAME"/";
+  req.append(obj_name);
+  g_test->send_request(string("PUT"), req,
+                       read_bucket_object, (void *)data, (size_t)len);
+  if (g_test->get_resp_code() != 200U) {
+    cout << "Errror sending object to the bucket, http_code " << g_test->get_resp_code();
+    return -1;
+  }
+  return 0;
+}
+
+static int read_bucket_obj(const char *obj_name) {
+  string req = "/"TEST_BUCKET_NAME"/";
+  req.append(obj_name);
+  g_test->send_request(string("GET"), req);
+  if (g_test->get_resp_code() != 200U) {
+    cout << "Errror sending object to the bucket, http_code " << g_test->get_resp_code();
+    return -1;
+  }
+  return 0;
+}
+
+static int delete_obj(const char *obj_name) {
+  string req = "/"TEST_BUCKET_NAME"/";
+  req.append(obj_name);
+  g_test->send_request(string("DELETE"), req);
+  if (g_test->get_resp_code() != 204U) {
+    cout << "Errror deleting object from bucket, http_code " << g_test->get_resp_code();
+    return -1;
+  }
+  return 0;
+}
+
+int get_formatted_time(string& ret) {
+  struct tm *tm = NULL;
+  char str_time[200];
+  const char *format = "%Y-%m-%d%%20%H:%M:%S";
+  time_t t;
+
+  t = time(NULL);
+  tm = gmtime(&t);
+  if(!tm) {
+    cerr << "Error returned by gmtime\n";
+    return -1;
+  }
+  if (strftime(str_time, sizeof(str_time), format, tm) == 0) {
+    cerr << "Error returned by strftime\n";
+    return -1;
+  }
+  ret = str_time;
+  return 0;
+}
+
+int parse_json_resp(JSONParser &parser) {
+  string *resp;
+  resp = (string *)g_test->get_response_data();
+  if(!resp)
+    return -1;
+  if(!parser.parse(resp->c_str(), resp->length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+struct RGWMetadataLogData {
+  obj_version read_version;
+  obj_version write_version;
+  string status;
+};
+
+struct cls_log_entry_json {
+  string section;
+  string name;
+  utime_t timestamp;
+  RGWMetadataLogData log_data;
+};
+
+static int decode_json(JSONObj *obj, RGWMetadataLogData &data) {
+  JSONObj *jo;
+
+  jo = obj->find_obj("read_version");
+  if (!jo)
+    return -1;
+  data.read_version.decode_json(obj);
+  data.write_version.decode_json(obj);
+
+  jo = obj->find_obj("status");
+  if (!jo)
+    return -1;
+  JSONDecoder::decode_json("status", data.status, jo);
+  return 0;
+}
+
+static int decode_json(JSONObj *obj, cls_log_entry_json& ret) {
+  JSONDecoder::decode_json("section", ret.section, obj);
+  JSONDecoder::decode_json("name", ret.name, obj);
+  JSONObj *jo = obj->find_obj("data");
+  if(!jo) 
+    return 0;
+  return decode_json(jo, ret.log_data);
+}
+
+static int get_log_list(list &entries) {
+  JSONParser parser;
+  if (parse_json_resp(parser) != 0)
+    return -1;
+  if (!parser.is_array()) 
+    return -1;
+
+  vector l;
+  l = parser.get_array_elements();
+  int loop = 0;
+  for(vector::iterator it = l.begin();
+      it != l.end(); ++it, loop++) {
+    JSONParser jp;
+    cls_log_entry_json entry;
+
+    if(!jp.parse((*it).c_str(), (*it).length())) {
+      cerr << "Error parsing log json object" << std::endl;
+      return -1;
+    }
+    EXPECT_EQ(decode_json((JSONObj *)&jp, entry), 0);
+    entries.push_back(entry);
+  }
+  return 0;
+}
+
+struct cls_bilog_entry {
+  string op_id;
+  string op_tag;
+  string op;
+  string object;
+  string status;
+  unsigned index_ver;
+};
+
+static int decode_json(JSONObj *obj, cls_bilog_entry& ret) {
+  JSONDecoder::decode_json("op_id", ret.op_id, obj);
+  JSONDecoder::decode_json("op_tag", ret.op_tag, obj);
+  JSONDecoder::decode_json("op", ret.op, obj);
+  JSONDecoder::decode_json("object", ret.object, obj);
+  JSONDecoder::decode_json("state", ret.status, obj);
+  JSONDecoder::decode_json("index_ver", ret.index_ver, obj);
+  return 0;
+}
+
+static int get_bilog_list(list &entries) {
+  JSONParser parser;
+  if (parse_json_resp(parser) != 0)
+    return -1;
+  if (!parser.is_array()) 
+    return -1;
+
+  vector l;
+  l = parser.get_array_elements();
+  int loop = 0;
+  for(vector::iterator it = l.begin();
+      it != l.end(); ++it, loop++) {
+    JSONParser jp;
+    cls_bilog_entry entry;
+
+    if(!jp.parse((*it).c_str(), (*it).length())) {
+      cerr << "Error parsing log json object" << std::endl;
+      return -1;
+    }
+    EXPECT_EQ(decode_json((JSONObj *)&jp, entry), 0);
+    entries.push_back(entry);
+  }
+  return 0;
+}
+
+static int decode_json(JSONObj *obj, rgw_data_change& ret) {
+  string entity;
+
+  JSONDecoder::decode_json("entity_type", entity, obj);
+  if (entity.compare("bucket") == 0)
+    ret.entity_type = ENTITY_TYPE_BUCKET;
+  JSONDecoder::decode_json("key", ret.key, obj);
+  return 0;
+}
+
+static int get_datalog_list(list &entries) {
+  JSONParser parser;
+
+  if (parse_json_resp(parser) != 0)
+    return -1;
+  if (!parser.is_array()) 
+    return -1;
+
+  vector l;
+  l = parser.get_array_elements();
+  int loop = 0;
+  for(vector::iterator it = l.begin();
+      it != l.end(); ++it, loop++) {
+    JSONParser jp;
+    rgw_data_change entry;
+
+    if(!jp.parse((*it).c_str(), (*it).length())) {
+      cerr << "Error parsing log json object" << std::endl;
+      return -1;
+    }
+    EXPECT_EQ(decode_json((JSONObj *)&jp, entry), 0);
+    entries.push_back(entry);
+  }
+  return 0;
+}
+
+unsigned get_mdlog_shard_id(string& key, int max_shards) {
+  string section = "user";
+  uint32_t val = ceph_str_hash_linux(key.c_str(), key.size());
+  val ^= ceph_str_hash_linux(section.c_str(), section.size());
+  return (unsigned)(val % max_shards);
+}
+
+unsigned get_datalog_shard_id(const char *bucket_name, int max_shards) {
+  uint32_t r = ceph_str_hash_linux(bucket_name, strlen(bucket_name)) % max_shards;
+  return (int)r;
+}
+
+TEST(TestRGWAdmin, datalog_list) {
+  string start_time, 
+         end_time,
+         start_time_2;
+  const char *cname = "datalog",
+             *perm = "*";
+  string rest_req;
+  unsigned shard_id = get_datalog_shard_id(TEST_BUCKET_NAME, g_ceph_context->_conf->rgw_data_log_num_shards);
+  stringstream ss;
+  list entries;
+
+  ASSERT_EQ(get_formatted_time(start_time), 0);
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  rest_req = "/admin/log?type=data";
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  JSONParser parser;
+  int num_objects;
+  EXPECT_EQ (parse_json_resp(parser), 0);
+  JSONDecoder::decode_json("num_objects", num_objects, (JSONObj *)&parser);
+  ASSERT_EQ(num_objects,g_ceph_context->_conf->rgw_data_log_num_shards);
+ 
+  sleep(1);
+  ASSERT_EQ(0, create_bucket());
+  
+  char *bucket_obj = (char *)malloc(TEST_BUCKET_OBJECT_SIZE);
+  ASSERT_TRUE(bucket_obj != NULL);
+  EXPECT_EQ(put_bucket_obj(TEST_BUCKET_OBJECT, bucket_obj, TEST_BUCKET_OBJECT_SIZE), 0);
+  free(bucket_obj);
+  sleep(1); 
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_EQ(1U, entries.size());
+  if (entries.size() == 1) {
+    rgw_data_change entry = *(entries.begin());
+    EXPECT_EQ(entry.entity_type, ENTITY_TYPE_BUCKET);
+    EXPECT_EQ(entry.key.compare(TEST_BUCKET_NAME), 0);
+  }
+  ASSERT_EQ(0, delete_obj(TEST_BUCKET_OBJECT));
+  sleep(1);
+  ASSERT_EQ(get_formatted_time(end_time), 0);
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_EQ(1U, entries.size());
+  if (entries.size() == 1) {
+    list::iterator it = (entries.begin());
+    EXPECT_EQ((*it).entity_type, ENTITY_TYPE_BUCKET);
+    EXPECT_EQ((*it).key.compare(TEST_BUCKET_NAME), 0);
+  }
+
+  sleep(1);
+  EXPECT_EQ(put_bucket_obj(TEST_BUCKET_OBJECT, bucket_obj, TEST_BUCKET_OBJECT_SIZE), 0);
+  sleep(20);
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_EQ(2U, entries.size());
+  if (entries.size() == 2) {
+    list::iterator it = (entries.begin());
+    EXPECT_EQ((*it).entity_type, ENTITY_TYPE_BUCKET);
+    EXPECT_EQ((*it).key.compare(TEST_BUCKET_NAME), 0);
+    it++; 
+    EXPECT_EQ((*it).entity_type, ENTITY_TYPE_BUCKET);
+    EXPECT_EQ((*it).key.compare(TEST_BUCKET_NAME), 0);
+  }
+
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time
+    << "&max-entries=1";
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_EQ(1U, entries.size());
+  
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_EQ(1U, entries.size());
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "read";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(403U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, delete_obj(TEST_BUCKET_OBJECT));
+  ASSERT_EQ(0, delete_bucket());
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, datalog_lock_unlock) {
+  const char *cname = "datalog",
+             *perm = "*";
+  string rest_req;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  rest_req = "/admin/log?type=data&lock&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=data&lock&id=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=data&lock&length=3&id=1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=data&lock&length=3&id=1&locker-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  rest_req = "/admin/log?type=data&unlock&id=1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  rest_req = "/admin/log?type=data&unlock&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=data&unlock&locker-id=ceph&id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&unlock&id=1&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  utime_t sleep_time(3, 0);
+
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(500U, g_test->get_resp_code()); 
+
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph1&zone-id=2";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(500U, g_test->get_resp_code()); 
+
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  sleep_time.sleep();
+
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&unlock&id=1&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "read";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "write";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  rest_req = "/admin/log?type=data&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=data&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, datalog_trim) {
+  string start_time, 
+         end_time;
+  const char *cname = "datalog",
+             *perm = "*";
+  string rest_req;
+  unsigned shard_id = get_datalog_shard_id(TEST_BUCKET_NAME, g_ceph_context->_conf->rgw_data_log_num_shards);
+  stringstream ss;
+  list entries;
+
+  ASSERT_EQ(get_formatted_time(start_time), 0);
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  rest_req = "/admin/log?type=data";
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  ss.str("");
+  ss << "/admin/log?type=data&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+ 
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  ASSERT_EQ(0, create_bucket());
+
+  char *bucket_obj = (char *)malloc(TEST_BUCKET_OBJECT_SIZE);
+  ASSERT_TRUE(bucket_obj != NULL);
+  EXPECT_EQ(put_bucket_obj(TEST_BUCKET_OBJECT, bucket_obj, TEST_BUCKET_OBJECT_SIZE), 0);
+  ASSERT_EQ(0, delete_obj(TEST_BUCKET_OBJECT));
+  sleep(1);
+  EXPECT_EQ(put_bucket_obj(TEST_BUCKET_OBJECT, bucket_obj, TEST_BUCKET_OBJECT_SIZE), 0);
+  ASSERT_EQ(0, delete_obj(TEST_BUCKET_OBJECT));
+  sleep(20);
+  free(bucket_obj);
+
+  ASSERT_EQ(get_formatted_time(end_time), 0);
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_TRUE(entries.size() > 0);
+
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_TRUE(entries.size() == 0);
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "write";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(403U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, delete_bucket());
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, mdlog_list) {
+  string start_time, 
+         end_time,
+         start_time_2;
+  const char *cname = "mdlog",
+             *perm = "*";
+  string rest_req;
+  unsigned shard_id = get_mdlog_shard_id(uid, g_ceph_context->_conf->rgw_md_log_max_shards);
+  stringstream ss;
+
+  sleep(2);
+  ASSERT_EQ(get_formatted_time(start_time), 0);
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  rest_req = "/admin/log?type=metadata";
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  JSONParser parser;
+  int num_objects;
+  EXPECT_EQ (parse_json_resp(parser), 0);
+  JSONDecoder::decode_json("num_objects", num_objects, (JSONObj *)&parser);
+  ASSERT_EQ(num_objects,g_ceph_context->_conf->rgw_md_log_max_shards);
+
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  
+  list entries;
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 4U);
+
+  if(entries.size() == 4) {
+    list::iterator it = entries.begin();
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("write") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("complete") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("write") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("complete") == 0);
+  }
+
+  sleep(1); /*To get a modified time*/
+  ASSERT_EQ(get_formatted_time(start_time_2), 0);
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm="read";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time_2;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+ 
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 4U);
+
+  if(entries.size() == 4) {
+    list::iterator it = entries.begin();
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("write") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("complete") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("write") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("complete") == 0);
+  }
+
+  sleep(1);
+  ASSERT_EQ(get_formatted_time(start_time_2), 0);
+  ASSERT_EQ(0, user_rm(uid, display_name));
+  
+  ASSERT_EQ(0, user_create(uid, display_name));
+  perm = "*";
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time_2;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 6U);
+  if(entries.size() == 6) {
+    list::iterator it = entries.begin();
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("remove") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("write") == 0);
+    it++;
+    EXPECT_TRUE(it->section.compare("user") == 0);
+    EXPECT_TRUE(it->name.compare(uid) == 0);
+    EXPECT_TRUE(it->log_data.status.compare("complete") == 0);
+  }
+
+  sleep(1);
+  ASSERT_EQ(get_formatted_time(end_time), 0);
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 14U);
+
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time 
+    << "&max-entries=" << 1;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time 
+    << "&max-entries=" << 6;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 6U);
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(403U, g_test->get_resp_code());
+
+  /*cleanup*/
+  ASSERT_EQ(0, caps_add(cname, perm));
+  sleep(1);
+  ASSERT_EQ(get_formatted_time(end_time), 0);
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time 
+    << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, mdlog_trim) {
+  string start_time, 
+         end_time,
+         start_time_2;
+  const char *cname = "mdlog",
+             *perm = "*";
+  string rest_req;
+  list entries;
+  unsigned shard_id = get_mdlog_shard_id(uid, g_ceph_context->_conf->rgw_md_log_max_shards);
+  ostringstream ss;
+
+  sleep(1);
+  ASSERT_EQ(get_formatted_time(start_time), 0);
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 4U);
+
+  sleep(1);
+  ASSERT_EQ(get_formatted_time(end_time), 0);
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 0U);
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm="write";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  ASSERT_EQ(get_formatted_time(end_time), 0);
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time << "&end-time=" << end_time;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(403U, g_test->get_resp_code());
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, mdlog_lock_unlock) {
+  const char *cname = "mdlog",
+             *perm = "*";
+  string rest_req;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  rest_req = "/admin/log?type=metadata&lock&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=metadata&lock&id=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=metadata&lock&length=3&id=1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  rest_req = "/admin/log?type=metadata&lock&id=3&locker-id=ceph&length=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=metadata&unlock&id=1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  rest_req = "/admin/log?type=metadata&unlock&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=metadata&unlock&locker-id=ceph&id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&unlock&id=1&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  utime_t sleep_time(3, 0);
+
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(500U, g_test->get_resp_code()); 
+
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph&zone-id=2";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(500U, g_test->get_resp_code()); 
+
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  sleep_time.sleep();
+
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&unlock&id=1&locker-id=ceph1&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "read";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "write";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  rest_req = "/admin/log?type=metadata&lock&id=1&length=3&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/log?type=metadata&unlock&id=1&locker-id=ceph&zone-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, bilog_list) {
+  const char *cname = "bilog",
+             *perm = "*";
+  string rest_req;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  ASSERT_EQ(0, create_bucket());
+
+  char *bucket_obj = (char *)malloc(TEST_BUCKET_OBJECT_SIZE);
+  ASSERT_TRUE(bucket_obj != NULL);
+  EXPECT_EQ(put_bucket_obj(TEST_BUCKET_OBJECT, bucket_obj, TEST_BUCKET_OBJECT_SIZE), 0);
+  free(bucket_obj);
+  
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  list entries;
+  get_bilog_list(entries);
+  EXPECT_EQ(2U, entries.size());
+  if (entries.size() == 2) {
+    list::iterator it = entries.begin();
+    EXPECT_EQ(it->op.compare("write"), 0);
+    EXPECT_EQ(it->object.compare(TEST_BUCKET_OBJECT), 0);
+    EXPECT_EQ(it->status.compare("pending"), 0);
+    EXPECT_EQ(it->index_ver, 1U);
+    it++;
+    EXPECT_EQ(it->op.compare("write"), 0);
+    EXPECT_EQ(it->object.compare(TEST_BUCKET_OBJECT), 0);
+    EXPECT_EQ(it->status.compare("complete"), 0);
+    EXPECT_EQ(it->index_ver, 2U);
+  }
+  EXPECT_EQ(read_bucket_obj(TEST_BUCKET_OBJECT), 0);
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_bilog_list(entries);
+  EXPECT_EQ(2U, entries.size());
+
+  bucket_obj = (char *)malloc(TEST_BUCKET_OBJECT_SIZE);
+  ASSERT_TRUE(bucket_obj != NULL);
+  EXPECT_EQ(put_bucket_obj(TEST_BUCKET_OBJECT_1, bucket_obj, TEST_BUCKET_OBJECT_SIZE), 0);
+  free(bucket_obj);
+  
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_bilog_list(entries);
+  EXPECT_EQ(4U, entries.size());
+  if (entries.size() == 4) {
+    list::iterator it = entries.begin();
+
+    it++; it++;
+    EXPECT_EQ(it->op.compare("write"), 0);
+    EXPECT_EQ(it->object.compare(TEST_BUCKET_OBJECT_1), 0);
+    EXPECT_EQ(it->status.compare("pending"), 0);
+    EXPECT_EQ(it->index_ver, 3U);
+    it++;
+    EXPECT_EQ(it->op.compare("write"), 0);
+    EXPECT_EQ(it->object.compare(TEST_BUCKET_OBJECT_1), 0);
+    EXPECT_EQ(it->status.compare("complete"), 0);
+    EXPECT_EQ(it->index_ver, 4U);
+  }
+
+  ASSERT_EQ(0, delete_obj(TEST_BUCKET_OBJECT));
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_bilog_list(entries);
+
+  EXPECT_EQ(6U, entries.size());
+  string marker;
+  if (entries.size() == 6) {
+    list::iterator it = entries.begin();
+    
+    it++; it++; it++; it++;
+    marker = it->op_id;
+    EXPECT_EQ(it->op.compare("del"), 0);
+    EXPECT_EQ(it->object.compare(TEST_BUCKET_OBJECT), 0);
+    EXPECT_EQ(it->status.compare("pending"), 0);
+    EXPECT_EQ(it->index_ver, 5U);
+    it++;
+    EXPECT_EQ(it->op.compare("del"), 0);
+    EXPECT_EQ(it->object.compare(TEST_BUCKET_OBJECT), 0);
+    EXPECT_EQ(it->status.compare("complete"), 0);
+    EXPECT_EQ(it->index_ver, 6U);
+  }
+
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  rest_req.append("&marker=");
+  rest_req.append(marker);
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_bilog_list(entries);
+  EXPECT_EQ(2U, entries.size());
+  if (entries.size() == 2U) {
+    list::iterator it = entries.begin();
+    EXPECT_EQ(it->index_ver, 5U);
+    it++;
+    EXPECT_EQ(it->index_ver, 6U);
+    EXPECT_EQ(it->op.compare("del"), 0);
+  }
+
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  rest_req.append("&marker=");
+  rest_req.append(marker);
+  rest_req.append("&max-entries=1");
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_bilog_list(entries);
+  EXPECT_EQ(1U, entries.size());
+  EXPECT_EQ((entries.begin())->index_ver, 5U);
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "read";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, caps_rm(cname, perm));
+  perm = "write";
+  ASSERT_EQ(0, caps_add(cname, perm));
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(403U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, delete_obj(TEST_BUCKET_OBJECT_1));
+  ASSERT_EQ(0, delete_bucket());
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, bilog_trim) {
+  const char *cname = "bilog",
+             *perm = "*";
+  string rest_req, start_marker, end_marker;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  ASSERT_EQ(0, create_bucket());
+
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  char *bucket_obj = (char *)malloc(TEST_BUCKET_OBJECT_SIZE);
+  ASSERT_TRUE(bucket_obj != NULL);
+  EXPECT_EQ(put_bucket_obj(TEST_BUCKET_OBJECT, bucket_obj, TEST_BUCKET_OBJECT_SIZE), 0);
+  free(bucket_obj);
+  
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  list entries;
+  get_bilog_list(entries);
+  EXPECT_EQ(2U, entries.size());
+
+  list::iterator it = entries.begin();
+  start_marker = it->op_id;
+  it++;
+  end_marker = it->op_id;
+
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  rest_req.append("&start-marker=");
+  rest_req.append(start_marker);
+  rest_req.append("&end-marker=");
+  rest_req.append(end_marker);
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  rest_req = "/admin/log?type=bucket-index&bucket="TEST_BUCKET_NAME;
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_bilog_list(entries);
+  EXPECT_EQ(0U, entries.size());
+  
+  ASSERT_EQ(0, delete_obj(TEST_BUCKET_OBJECT));
+  ASSERT_EQ(0, delete_bucket());
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+int main(int argc, char *argv[]){
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_test = new admin_log::test_helper();
+  finisher = new Finisher(g_ceph_context);
+#ifdef GTEST
+  ::testing::InitGoogleTest(&argc, argv);
+#endif
+  finisher->start();
+
+  if(g_test->extract_input(argc, argv) < 0){
+    print_usage(argv[0]);
+    return -1;
+  }
+#ifdef GTEST
+  int r = RUN_ALL_TESTS();
+  if (r >= 0) {
+    cout << "There are no failures in the test case\n";
+  } else {
+    cout << "There are some failures\n";
+  }
+#endif
+  finisher->stop();
+  return 0;
+}
diff --git a/ceph/src/test/test_rgw_admin_meta.cc b/ceph/src/test/test_rgw_admin_meta.cc
new file mode 100644
index 00000000..fb9cf863
--- /dev/null
+++ b/ceph/src/test/test_rgw_admin_meta.cc
@@ -0,0 +1,936 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+extern "C"{
+#include 
+}
+#include "common/ceph_crypto.h"
+#include "include/str_list.h"
+#include "common/ceph_json.h"
+#include "common/code_environment.h"
+#include "common/ceph_argparse.h"
+#include "common/Finisher.h"
+#include "global/global_init.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#define GTEST
+#ifdef GTEST
+#include 
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+using namespace std;
+
+#define CURL_VERBOSE 0
+#define HTTP_RESPONSE_STR "RespCode"
+#define CEPH_CRYPTO_HMACSHA1_DIGESTSIZE 20
+#define RGW_ADMIN_RESP_PATH "/tmp/.test_rgw_admin_resp"
+#define CEPH_UID  "ceph"
+
+static string uid = CEPH_UID;
+static string display_name = "CEPH";
+static string meta_caps = "metadata";
+
+extern "C" int ceph_armor(char *dst, const char *dst_end, 
+                          const char *src, const char *end);
+static void print_usage(char *exec){
+  cout << "Usage: " << exec << " \n";
+  cout << "Options:\n"
+          "-g  - The ip address of the gateway\n"
+          "-p  - The port number of the gateway\n"
+          "-c  - Absolute path of ceph config file\n"
+          "-rgw-admin  - radosgw-admin absolute path\n";
+}
+
+namespace admin_meta {
+class test_helper {
+  private:
+    string host;
+    string port;
+    string creds;
+    string rgw_admin_path;
+    string conf_path;
+    CURL *curl_inst;
+    map response;
+    list extra_hdrs;
+    string *resp_data;
+    unsigned resp_code;
+  public:
+    test_helper() : resp_data(NULL){
+      curl_global_init(CURL_GLOBAL_ALL);
+    }
+    ~test_helper(){
+      curl_global_cleanup();
+    }
+    int send_request(string method, string uri, 
+                     size_t (*function)(void *,size_t,size_t,void *) = 0,
+                     void *ud = 0, size_t length = 0);
+    int extract_input(int argc, char *argv[]);
+    string& get_response(string hdr){
+      return response[hdr];
+    }
+    void set_extra_header(string hdr){
+      extra_hdrs.push_back(hdr);
+    }
+    void set_response(char *val);
+    void set_response_data(char *data, size_t len){
+      if(resp_data) delete resp_data;
+      resp_data = new string(data, len);
+    }
+    string& get_rgw_admin_path() {
+      return rgw_admin_path;
+    }
+    string& get_ceph_conf_path() {
+      return conf_path;
+    }
+    void set_creds(string& c) {
+      creds = c;
+    }
+    const string *get_response_data(){return resp_data;}
+    unsigned get_resp_code(){return resp_code;}
+};
+
+int test_helper::extract_input(int argc, char *argv[]){
+#define ERR_CHECK_NEXT_PARAM(o) \
+  if(((int)loop + 1) >= argc)return -1;		\
+  else o = argv[loop+1];
+
+  for(unsigned loop = 1;loop < (unsigned)argc; loop += 2){
+    if(strcmp(argv[loop], "-g") == 0){
+      ERR_CHECK_NEXT_PARAM(host);
+    }else if(strcmp(argv[loop],"-p") == 0){
+      ERR_CHECK_NEXT_PARAM(port);
+    }else if(strcmp(argv[loop], "-c") == 0){
+      ERR_CHECK_NEXT_PARAM(conf_path);
+    }else if(strcmp(argv[loop], "-rgw-admin") == 0){
+      ERR_CHECK_NEXT_PARAM(rgw_admin_path);
+    }else return -1;
+  }
+  if(host.length() <= 0 ||
+     rgw_admin_path.length() <= 0)
+    return -1;
+  return 0;
+}
+
+void test_helper::set_response(char *r){
+  string sr(r), h, v;
+  size_t off = sr.find(": ");
+  if(off != string::npos){
+    h.assign(sr, 0, off);
+    v.assign(sr, off + 2, sr.find("\r\n") - (off+2));
+  }else{
+    /*Could be the status code*/
+    if(sr.find("HTTP/") != string::npos){
+      h.assign(HTTP_RESPONSE_STR);
+      off = sr.find(" ");
+      v.assign(sr, off + 1, sr.find("\r\n") - (off + 1));
+      resp_code = atoi((v.substr(0, 3)).c_str());
+    }
+  }
+  response[h] = v;
+}
+
+size_t write_header(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = static_cast(ud);
+  h->set_response((char *)ptr);
+  return size*nmemb;
+}
+
+size_t write_data(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = static_cast(ud);
+  h->set_response_data((char *)ptr, size*nmemb);
+  return size*nmemb;
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+static void calc_hmac_sha1(const char *key, int key_len,
+                    const char *msg, int msg_len, char *dest)
+/* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
+{
+  ceph::crypto::HMACSHA1 hmac((const unsigned char *)key, key_len);
+  hmac.Update((const unsigned char *)msg, msg_len);
+  hmac.Final((unsigned char *)dest);
+  
+  char hex_str[(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2) + 1];
+  admin_meta::buf_to_hex((unsigned char *)dest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE, hex_str);
+}
+
+static int get_s3_auth(string method, string creds, string date, string res, string& out){
+  string aid, secret, auth_hdr;
+  string tmp_res;
+  size_t off = creds.find(":");
+  out = "";
+  if(off != string::npos){
+    aid.assign(creds, 0, off);
+    secret.assign(creds, off + 1, string::npos);
+
+    /*sprintf(auth_hdr, "%s\n\n\n%s\n%s", req_type, date, res);*/
+    char hmac_sha1[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+    char b64[65]; /* 64 is really enough */
+    size_t off = res.find("?");
+    if(off == string::npos)
+      tmp_res = res;
+    else
+      tmp_res.assign(res, 0, off);
+    auth_hdr.append(method + string("\n\n\n") + date + string("\n") + tmp_res);
+    admin_meta::calc_hmac_sha1(secret.c_str(), secret.length(), 
+                               auth_hdr.c_str(), auth_hdr.length(), hmac_sha1);
+    int ret = ceph_armor(b64, b64 + 64, hmac_sha1,
+                         hmac_sha1 + CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+    if (ret < 0) {
+      cout << "ceph_armor failed\n";
+      return -1;
+    }
+    b64[ret] = 0;
+    out.append(aid + string(":") + b64);
+  }else return -1;
+  return 0;
+}
+
+void get_date(string& d){
+  struct timeval tv;
+  char date[64];
+  struct tm tm;
+  char *days[] = {(char *)"Sun", (char *)"Mon", (char *)"Tue",
+                  (char *)"Wed", (char *)"Thu", (char *)"Fri", 
+                  (char *)"Sat"};
+  char *months[] = {(char *)"Jan", (char *)"Feb", (char *)"Mar", 
+                    (char *)"Apr", (char *)"May", (char *)"Jun",
+                    (char *)"Jul",(char *) "Aug", (char *)"Sep", 
+                    (char *)"Oct", (char *)"Nov", (char *)"Dec"};
+  gettimeofday(&tv, NULL);
+  gmtime_r(&tv.tv_sec, &tm);
+  sprintf(date, "%s, %d %s %d %d:%d:%d GMT", 
+          days[tm.tm_wday], 
+          tm.tm_mday, months[tm.tm_mon], 
+          tm.tm_year + 1900,
+          tm.tm_hour, tm.tm_min, 0 /*tm.tm_sec*/);
+  d = date;
+}
+
+int test_helper::send_request(string method, string res, 
+                                   size_t (*read_function)( void *,size_t,size_t,void *),
+                                   void *ud,
+                                   size_t length){
+  string url;
+  string auth, date;
+  url.append(string("http://") + host);
+  if(port.length() > 0)url.append(string(":") + port);
+  url.append(res);
+  curl_inst = curl_easy_init();
+  if(curl_inst){
+    curl_easy_setopt(curl_inst, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_CUSTOMREQUEST, method.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_VERBOSE, CURL_VERBOSE);
+    curl_easy_setopt(curl_inst, CURLOPT_HEADERFUNCTION, admin_meta::write_header);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEHEADER, (void *)this);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEFUNCTION, admin_meta::write_data);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEDATA, (void *)this);
+    if(read_function){
+      curl_easy_setopt(curl_inst, CURLOPT_READFUNCTION, read_function);
+      curl_easy_setopt(curl_inst, CURLOPT_READDATA, (void *)ud);
+      curl_easy_setopt(curl_inst, CURLOPT_UPLOAD, 1L);
+      curl_easy_setopt(curl_inst, CURLOPT_INFILESIZE_LARGE, (curl_off_t)length);
+    }
+
+    get_date(date);
+    string http_date;
+    http_date.append(string("Date: ") + date);
+
+    string s3auth;
+    if (admin_meta::get_s3_auth(method, creds, date, res, s3auth) < 0)
+      return -1;
+    auth.append(string("Authorization: AWS ") + s3auth);
+
+    struct curl_slist *slist = NULL;
+    slist = curl_slist_append(slist, auth.c_str());
+    slist = curl_slist_append(slist, http_date.c_str());
+    for(list::iterator it = extra_hdrs.begin();
+        it != extra_hdrs.end(); ++it){
+      slist = curl_slist_append(slist, (*it).c_str());
+    }
+    if(read_function)
+      curl_slist_append(slist, "Expect:");
+    curl_easy_setopt(curl_inst, CURLOPT_HTTPHEADER, slist); 
+
+    response.erase(response.begin(), response.end());
+    extra_hdrs.erase(extra_hdrs.begin(), extra_hdrs.end());
+    CURLcode res = curl_easy_perform(curl_inst);
+    if(res != CURLE_OK){
+      cout << "Curl perform failed for " << url << ", res: " << 
+        curl_easy_strerror(res) << "\n";
+      return -1;
+    }
+    curl_slist_free_all(slist);
+  }
+  curl_easy_cleanup(curl_inst);
+  return 0;
+}
+};
+
+admin_meta::test_helper *g_test;
+Finisher *finisher;
+
+int run_rgw_admin(string& cmd, string& resp) {
+  pid_t pid;
+  pid = fork();
+  if (pid == 0) {
+    /* child */
+    list l;
+    get_str_list(cmd, " \t", l);
+    char *argv[l.size()];
+    unsigned loop = 1;
+
+    argv[0] = (char *)"radosgw-admin";
+    for (list::iterator it = l.begin(); 
+         it != l.end(); ++it) {
+      argv[loop++] = (char *)(*it).c_str();
+    }
+    argv[loop] = NULL;
+    close(1);
+    stdout = fopen(RGW_ADMIN_RESP_PATH, "w+");
+    if (!stdout) {
+      cout << "Unable to open stdout file" << std::endl;
+    }
+    execv((g_test->get_rgw_admin_path()).c_str(), argv); 
+  } else if (pid > 0) {
+    int status;
+    waitpid(pid, &status, 0);
+    if (WIFEXITED(status)) {
+      if(WEXITSTATUS(status) != 0) {
+        cout << "Child exited with status " << WEXITSTATUS(status) << std::endl;
+        return -1;
+      }
+    }
+    ifstream in;
+    struct stat st;
+
+    if (stat(RGW_ADMIN_RESP_PATH, &st) < 0) {
+      cout << "Error stating the admin response file, errno " << errno << std::endl;
+      return -1;
+    } else {
+      char *data = (char *)malloc(st.st_size + 1);
+      in.open(RGW_ADMIN_RESP_PATH);
+      in.read(data, st.st_size);
+      in.close();
+      data[st.st_size] = 0;
+      resp = data;
+      free(data);
+      unlink(RGW_ADMIN_RESP_PATH);
+      /* cout << "radosgw-admin " << cmd << ": " << resp << std::endl;*/
+    }
+  } else 
+    return -1;
+  return 0;
+}
+
+int get_creds(string& json, string& creds) {
+  JSONParser parser;
+  if(!parser.parse(json.c_str(), json.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+
+  RGWUserInfo info;
+  decode_json_obj(info, &parser);
+  creds = "";
+  for(map::iterator it = info.access_keys.begin();
+      it != info.access_keys.end(); ++it) {
+    RGWAccessKey _k = it->second;
+    /*cout << "accesskeys [ " << it->first << " ] = " << 
+      "{ " << _k.id << ", " << _k.key << ", " << _k.subuser << "}" << std::endl;*/
+    creds.append(it->first + string(":") + _k.key);
+    break;
+  }
+  return 0;
+}
+
+int user_create(string& uid, string& display_name, bool set_creds = true) {
+  stringstream ss;
+  string creds;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user create --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  get_creds(out, creds);
+  if(set_creds)
+    g_test->set_creds(creds);
+  return 0;
+}
+
+int user_info(string& uid, string& display_name, RGWUserInfo& uinfo) {
+  stringstream ss;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user info --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error reading user information" << std::endl;
+    return -1;
+  }
+  JSONParser parser;
+  if(!parser.parse(out.c_str(), out.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  decode_json_obj(uinfo, &parser);
+  return 0;
+}
+
+int user_rm(string& uid, string& display_name) {
+  stringstream ss;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user rm --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error removing user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int meta_caps_add(const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps add --caps=" <<
+     meta_caps << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int meta_caps_rm(const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps rm --caps=" <<
+     meta_caps << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int compare_access_keys(RGWAccessKey& k1, RGWAccessKey& k2) {
+  if (k1.id.compare(k2.id) != 0)
+    return -1;
+  if (k1.key.compare(k2.key) != 0) 
+    return -1;
+  if (k1.subuser.compare(k2.subuser) != 0) 
+    return -1;
+
+  return 0;
+}
+
+int compare_user_info(RGWUserInfo& i1, RGWUserInfo& i2) {
+  int rv;
+
+  if ((rv = i1.user_id.compare(i2.user_id)) != 0)
+    return rv;
+  if ((rv = i1.display_name.compare(i2.display_name)) != 0)
+    return rv;
+  if ((rv = i1.user_email.compare(i2.user_email)) != 0)
+    return rv;
+  if (i1.access_keys.size() != i2.access_keys.size())
+    return -1;
+  for (map::iterator it = i1.access_keys.begin();
+       it != i1.access_keys.end(); ++it) {
+    RGWAccessKey k1, k2;
+    k1 = it->second;
+    if (i2.access_keys.count(it->first) == 0)
+      return -1;
+    k2 = i2.access_keys[it->first];
+    if (compare_access_keys(k1, k2) != 0)
+      return -1;
+  }
+  if (i1.swift_keys.size() != i2.swift_keys.size())
+    return -1;
+  for (map::iterator it = i1.swift_keys.begin();
+       it != i1.swift_keys.end(); ++it) {
+    RGWAccessKey k1, k2;
+    k1 = it->second;
+    if (i2.swift_keys.count(it->first) == 0)
+      return -1;
+    k2 = i2.swift_keys[it->first];
+    if (compare_access_keys(k1, k2) != 0)
+      return -1;
+  }
+  if (i1.subusers.size() != i2.subusers.size()) 
+    return -1;
+  for (map::iterator it = i1.subusers.begin();
+       it != i1.subusers.end(); ++it) {
+    RGWSubUser k1, k2;
+    k1 = it->second;
+    if (!i2.subusers.count(it->first))
+      return -1;
+    k2 = i2.subusers[it->first];
+    if (k1.name.compare(k2.name) != 0) 
+      return -1;
+    if (k1.perm_mask != k2.perm_mask)
+      return -1;
+  }
+  if (i1.suspended != i2.suspended)
+    return -1;
+  if (i1.max_buckets != i2.max_buckets)
+    return -1;
+  uint32_t p1, p2;
+  p1 = p2 = RGW_CAP_ALL;
+  if (i1.caps.check_cap(meta_caps, p1) != 0)
+    return -1;
+  if (i2.caps.check_cap(meta_caps, p2) != 0)
+    return -1;
+  return 0;
+}
+
+size_t read_dummy_post(void *ptr, size_t s, size_t n, void *ud) {
+  int dummy = 0;
+  memcpy(ptr, &dummy, sizeof(dummy));
+  return sizeof(dummy);
+}
+
+
+int parse_json_resp(JSONParser &parser) {
+  string *resp;
+  resp = (string *)g_test->get_response_data();
+  if(!resp)
+    return -1;
+  if(!parser.parse(resp->c_str(), resp->length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+size_t meta_read_json(void *ptr, size_t s, size_t n, void *ud){
+  stringstream *ss = (stringstream *)ud;
+  size_t len = ss->str().length();
+  if(s*n < len){
+    cout << "Cannot copy json data, as len is not enough\n";
+    return 0;
+  }
+  memcpy(ptr, (void *)ss->str().c_str(), len);
+  return len;
+}
+
+TEST(TestRGWAdmin, meta_list){
+  JSONParser parser;
+  bool found = false;
+  const char *perm = "*";
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+
+  /*Check the sections*/
+  g_test->send_request(string("GET"), string("/admin/metadata/"));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  EXPECT_TRUE(parser.is_array());
+  
+  vector l;
+  l = parser.get_array_elements();
+  for(vector::iterator it = l.begin();
+      it != l.end(); it++) {
+    if((*it).compare("\"user\"") == 0) {
+      found = true;
+      break;
+    }
+  }
+  EXPECT_TRUE(found);
+
+  /*Check with a wrong section*/
+  g_test->send_request(string("GET"), string("/admin/metadata/users"));
+  EXPECT_EQ(404U, g_test->get_resp_code());
+
+  /*Check the list of keys*/
+  g_test->send_request(string("GET"), string("/admin/metadata/user"));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  EXPECT_TRUE(parser.is_array());
+  
+  l = parser.get_array_elements();
+  EXPECT_EQ(1U, l.size());
+  for(vector::iterator it = l.begin();
+      it != l.end(); it++) {
+    if((*it).compare(string("\"") + uid + string("\"")) == 0) {
+      found = true;
+      break;
+    }
+  }
+  EXPECT_TRUE(found);
+
+  /*Check with second user*/
+  string uid2 = "ceph1", display_name2 = "CEPH1";
+  ASSERT_EQ(0, user_create(uid2, display_name2, false));
+  /*Check the list of keys*/
+  g_test->send_request(string("GET"), string("/admin/metadata/user"));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  EXPECT_TRUE(parser.is_array());
+  
+  l = parser.get_array_elements();
+  EXPECT_EQ(2U, l.size());
+  bool found2 = false;
+  for(vector::iterator it = l.begin();
+      it != l.end(); it++) {
+    if((*it).compare(string("\"") + uid + string("\"")) == 0) {
+      found = true;
+    }
+    if((*it).compare(string("\"") + uid2 + string("\"")) == 0) {
+      found2 = true;
+    }
+  }
+  EXPECT_TRUE(found && found2);
+  ASSERT_EQ(0, user_rm(uid2, display_name2));
+
+  /*Remove the metadata caps*/
+  int rv = meta_caps_rm(perm);
+  EXPECT_EQ(0, rv);
+  
+  if(rv == 0) {
+    g_test->send_request(string("GET"), string("/admin/metadata/"));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+
+    g_test->send_request(string("GET"), string("/admin/metadata/user"));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+  }
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, meta_get){
+  JSONParser parser;
+  const char *perm = "*";
+  RGWUserInfo info;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+
+  ASSERT_EQ(0, user_info(uid, display_name, info));
+ 
+  g_test->send_request(string("GET"), string("/admin/metadata/user?key=test"));
+  EXPECT_EQ(404U, g_test->get_resp_code());
+
+  g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  RGWObjVersionTracker objv_tracker;
+  string metadata_key;
+
+  obj_version *objv = &objv_tracker.read_version;
+     
+  JSONDecoder::decode_json("key", metadata_key, &parser);
+  JSONDecoder::decode_json("ver", *objv, &parser);
+  JSONObj *jo = parser.find_obj("data");
+  ASSERT_TRUE(jo);
+  string exp_meta_key = "user:";
+  exp_meta_key.append(uid);
+  EXPECT_TRUE(metadata_key.compare(exp_meta_key) == 0);
+
+  RGWUserInfo obt_info;
+  decode_json_obj(obt_info, jo);
+
+  EXPECT_TRUE(compare_user_info(info, obt_info) == 0);
+
+  /*Make a modification and check if its reflected*/
+  ASSERT_EQ(0, meta_caps_rm(perm));
+  perm = "read";
+  ASSERT_EQ(0, meta_caps_add(perm));
+  
+  JSONParser parser1;
+  g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser1) == 0);
+ 
+  RGWObjVersionTracker objv_tracker1;
+  obj_version *objv1 = &objv_tracker1.read_version;
+
+  JSONDecoder::decode_json("key", metadata_key, &parser1);
+  JSONDecoder::decode_json("ver", *objv1, &parser1);
+  jo = parser1.find_obj("data");
+  ASSERT_TRUE(jo);
+
+  decode_json_obj(obt_info, jo);
+  uint32_t p1, p2;
+  p1 = RGW_CAP_ALL;
+  p2 = RGW_CAP_READ;
+  EXPECT_TRUE (info.caps.check_cap(meta_caps, p1) == 0);
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, p2) == 0);
+  p2 = RGW_CAP_WRITE;
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, p2) != 0);
+
+  /*Version and tag infromation*/
+  EXPECT_TRUE(objv1->ver > objv->ver);
+  EXPECT_EQ(objv1->tag, objv->tag);
+  
+  int rv = meta_caps_rm(perm);
+  EXPECT_EQ(0, rv);
+  
+  if(rv == 0) {
+    g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+  }
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, meta_put){
+  JSONParser parser;
+  const char *perm = "*";
+  RGWUserInfo info;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+  
+  g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  RGWObjVersionTracker objv_tracker;
+  string metadata_key;
+
+  obj_version *objv = &objv_tracker.read_version;
+     
+  JSONDecoder::decode_json("key", metadata_key, &parser);
+  JSONDecoder::decode_json("ver", *objv, &parser);
+  JSONObj *jo = parser.find_obj("data");
+  ASSERT_TRUE(jo);
+  string exp_meta_key = "user:";
+  exp_meta_key.append(uid);
+  EXPECT_TRUE(metadata_key.compare(exp_meta_key) == 0);
+
+  RGWUserInfo obt_info;
+  decode_json_obj(obt_info, jo);
+
+  /*Change the cap and PUT */
+  RGWUserCaps caps;
+  string new_cap;
+  Formatter *f = new JSONFormatter();
+
+  new_cap = meta_caps + string("=write");
+  caps.add_from_string(new_cap);
+  obt_info.caps = caps;
+  f->open_object_section("metadata_info");
+  ::encode_json("key", metadata_key, f);
+  ::encode_json("ver", *objv, f);
+  ::encode_json("data", obt_info, f);
+  f->close_section();
+  std::stringstream ss;
+  f->flush(ss);
+
+  g_test->send_request(string("PUT"), (string("/admin/metadata/user?key=") + uid), 
+                       meta_read_json,
+                       (void *)&ss, ss.str().length());
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, user_info(uid, display_name, obt_info));
+  uint32_t cp;
+  cp = RGW_CAP_WRITE;
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, cp) == 0);
+  cp = RGW_CAP_READ;
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, cp) != 0);
+  
+  int rv = meta_caps_rm("write");
+  EXPECT_EQ(0, rv);
+  if(rv == 0) {
+    g_test->send_request(string("PUT"), (string("/admin/metadata/user?key=") + uid));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+  }
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, meta_lock_unlock) {
+  const char *perm = "*";
+  string rest_req;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/metadata/user?lock&length=3&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&unlock";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+
+  rest_req = "/admin/metadata/user?unlock&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Bad request*/
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&unlock&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&unlock&lock_id=ceph1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  utime_t sleep_time(3, 0);
+
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(500U, g_test->get_resp_code()); 
+
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(409U, g_test->get_resp_code()); 
+  sleep_time.sleep();
+
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&unlock&lock_id=ceph1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+
+  ASSERT_EQ(0, meta_caps_rm(perm));
+  perm = "read";
+  ASSERT_EQ(0, meta_caps_add(perm));
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&unlock&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, meta_caps_rm(perm));
+  perm = "write";
+  ASSERT_EQ(0, meta_caps_add(perm));
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&unlock&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, meta_caps_rm(perm));
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&lock&length=3&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  rest_req = "/admin/metadata/user?key="CEPH_UID"&unlock&lock_id=ceph";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(403U, g_test->get_resp_code()); 
+  
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, meta_delete){
+  JSONParser parser;
+  const char *perm = "*";
+  RGWUserInfo info;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+
+  g_test->send_request(string("DELETE"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(user_info(uid, display_name, info) != 0);
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  perm = "read";
+  ASSERT_EQ(0, meta_caps_add(perm));
+  
+  g_test->send_request(string("DELETE"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(403U, g_test->get_resp_code());
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+int main(int argc, char *argv[]){
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_test = new admin_meta::test_helper();
+  finisher = new Finisher(g_ceph_context);
+#ifdef GTEST
+  ::testing::InitGoogleTest(&argc, argv);
+#endif
+  finisher->start();
+
+  if(g_test->extract_input(argc, argv) < 0){
+    print_usage(argv[0]);
+    return -1;
+  }
+#ifdef GTEST
+  int r = RUN_ALL_TESTS();
+  if (r >= 0) {
+    cout << "There are no failures in the test case\n";
+  } else {
+    cout << "There are some failures\n";
+  }
+#endif
+  finisher->stop();
+  return 0;
+}
diff --git a/ceph/src/test/test_rgw_admin_opstate.cc b/ceph/src/test/test_rgw_admin_opstate.cc
new file mode 100644
index 00000000..9c8835c0
--- /dev/null
+++ b/ceph/src/test/test_rgw_admin_opstate.cc
@@ -0,0 +1,834 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+extern "C"{
+#include 
+}
+#include "common/ceph_crypto.h"
+#include "include/str_list.h"
+#include "common/ceph_json.h"
+#include "common/code_environment.h"
+#include "common/ceph_argparse.h"
+#include "common/Finisher.h"
+#include "global/global_init.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_bucket.h"
+#include "rgw/rgw_rados.h"
+#include "include/utime.h"
+#include "include/object.h"
+#include "cls/statelog/cls_statelog_types.h"
+#define GTEST
+#ifdef GTEST
+#include 
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+using namespace std;
+
+#define CURL_VERBOSE 0
+#define HTTP_RESPONSE_STR "RespCode"
+#define CEPH_CRYPTO_HMACSHA1_DIGESTSIZE 20
+#define RGW_ADMIN_RESP_PATH "/tmp/.test_rgw_admin_resp"
+
+static string uid = "ceph";
+static string display_name = "CEPH";
+
+extern "C" int ceph_armor(char *dst, const char *dst_end, 
+                          const char *src, const char *end);
+static void print_usage(char *exec){
+  cout << "Usage: " << exec << " \n";
+  cout << "Options:\n"
+          "-g  - The ip address of the gateway\n"
+          "-p  - The port number of the gateway\n"
+          "-c  - Absolute path of ceph config file\n"
+          "-rgw-admin  - radosgw-admin absolute path\n";
+}
+
+namespace admin_log {
+class test_helper {
+  private:
+    string host;
+    string port;
+    string creds;
+    string rgw_admin_path;
+    string conf_path;
+    CURL *curl_inst;
+    map response;
+    list extra_hdrs;
+    string *resp_data;
+    unsigned resp_code;
+  public:
+    test_helper() : resp_data(NULL){
+      curl_global_init(CURL_GLOBAL_ALL);
+    }
+    ~test_helper(){
+      curl_global_cleanup();
+    }
+    int send_request(string method, string uri, 
+                     size_t (*function)(void *,size_t,size_t,void *) = 0,
+                     void *ud = 0, size_t length = 0);
+    int extract_input(int argc, char *argv[]);
+    string& get_response(string hdr){
+      return response[hdr];
+    }
+    void set_extra_header(string hdr){
+      extra_hdrs.push_back(hdr);
+    }
+    void set_response(char *val);
+    void set_response_data(char *data, size_t len){
+      if(resp_data) delete resp_data;
+      resp_data = new string(data, len);
+    }
+    string& get_rgw_admin_path() {
+      return rgw_admin_path;
+    }
+    string& get_ceph_conf_path() {
+      return conf_path;
+    }
+    void set_creds(string& c) {
+      creds = c;
+    }
+    const string *get_response_data(){return resp_data;}
+    unsigned get_resp_code(){return resp_code;}
+};
+
+int test_helper::extract_input(int argc, char *argv[]){
+#define ERR_CHECK_NEXT_PARAM(o) \
+  if(((int)loop + 1) >= argc)return -1;		\
+  else o = argv[loop+1];
+
+  for(unsigned loop = 1;loop < (unsigned)argc; loop += 2){
+    if(strcmp(argv[loop], "-g") == 0){
+      ERR_CHECK_NEXT_PARAM(host);
+    }else if(strcmp(argv[loop],"-p") == 0){
+      ERR_CHECK_NEXT_PARAM(port);
+    }else if(strcmp(argv[loop], "-c") == 0){
+      ERR_CHECK_NEXT_PARAM(conf_path);
+    }else if(strcmp(argv[loop], "-rgw-admin") == 0){
+      ERR_CHECK_NEXT_PARAM(rgw_admin_path);
+    }else return -1;
+  }
+  if(host.length() <= 0 ||
+     rgw_admin_path.length() <= 0)
+    return -1;
+  return 0;
+}
+
+void test_helper::set_response(char *r){
+  string sr(r), h, v;
+  size_t off = sr.find(": ");
+  if(off != string::npos){
+    h.assign(sr, 0, off);
+    v.assign(sr, off + 2, sr.find("\r\n") - (off+2));
+  }else{
+    /*Could be the status code*/
+    if(sr.find("HTTP/") != string::npos){
+      h.assign(HTTP_RESPONSE_STR);
+      off = sr.find(" ");
+      v.assign(sr, off + 1, sr.find("\r\n") - (off + 1));
+      resp_code = atoi((v.substr(0, 3)).c_str());
+    }
+  }
+  response[h] = v;
+}
+
+size_t write_header(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = static_cast(ud);
+  h->set_response((char *)ptr);
+  return size*nmemb;
+}
+
+size_t write_data(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = static_cast(ud);
+  h->set_response_data((char *)ptr, size*nmemb);
+  return size*nmemb;
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+static void calc_hmac_sha1(const char *key, int key_len,
+                    const char *msg, int msg_len, char *dest)
+/* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
+{
+  ceph::crypto::HMACSHA1 hmac((const unsigned char *)key, key_len);
+  hmac.Update((const unsigned char *)msg, msg_len);
+  hmac.Final((unsigned char *)dest);
+  
+  char hex_str[(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2) + 1];
+  admin_log::buf_to_hex((unsigned char *)dest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE, hex_str);
+}
+
+static int get_s3_auth(string method, string creds, string date, string res, string& out){
+  string aid, secret, auth_hdr;
+  string tmp_res;
+  size_t off = creds.find(":");
+  out = "";
+  if(off != string::npos){
+    aid.assign(creds, 0, off);
+    secret.assign(creds, off + 1, string::npos);
+
+    /*sprintf(auth_hdr, "%s\n\n\n%s\n%s", req_type, date, res);*/
+    char hmac_sha1[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+    char b64[65]; /* 64 is really enough */
+    size_t off = res.find("?");
+    if(off == string::npos)
+      tmp_res = res;
+    else
+      tmp_res.assign(res, 0, off);
+    auth_hdr.append(method + string("\n\n\n") + date + string("\n") + tmp_res);
+    admin_log::calc_hmac_sha1(secret.c_str(), secret.length(), 
+                               auth_hdr.c_str(), auth_hdr.length(), hmac_sha1);
+    int ret = ceph_armor(b64, b64 + 64, hmac_sha1,
+                         hmac_sha1 + CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+    if (ret < 0) {
+      cout << "ceph_armor failed\n";
+      return -1;
+    }
+    b64[ret] = 0;
+    out.append(aid + string(":") + b64);
+  }else return -1;
+  return 0;
+}
+
+void get_date(string& d){
+  struct timeval tv;
+  char date[64];
+  struct tm tm;
+  char *days[] = {(char *)"Sun", (char *)"Mon", (char *)"Tue",
+                  (char *)"Wed", (char *)"Thu", (char *)"Fri", 
+                  (char *)"Sat"};
+  char *months[] = {(char *)"Jan", (char *)"Feb", (char *)"Mar", 
+                    (char *)"Apr", (char *)"May", (char *)"Jun",
+                    (char *)"Jul",(char *) "Aug", (char *)"Sep", 
+                    (char *)"Oct", (char *)"Nov", (char *)"Dec"};
+  gettimeofday(&tv, NULL);
+  gmtime_r(&tv.tv_sec, &tm);
+  sprintf(date, "%s, %d %s %d %d:%d:%d GMT", 
+          days[tm.tm_wday], 
+          tm.tm_mday, months[tm.tm_mon], 
+          tm.tm_year + 1900,
+          tm.tm_hour, tm.tm_min, tm.tm_sec);
+  d = date;
+}
+
+int test_helper::send_request(string method, string res, 
+                                   size_t (*read_function)( void *,size_t,size_t,void *),
+                                   void *ud,
+                                   size_t length){
+  string url;
+  string auth, date;
+  url.append(string("http://") + host);
+  if(port.length() > 0)url.append(string(":") + port);
+  url.append(res);
+  curl_inst = curl_easy_init();
+  if(curl_inst){
+    curl_easy_setopt(curl_inst, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_CUSTOMREQUEST, method.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_VERBOSE, CURL_VERBOSE);
+    curl_easy_setopt(curl_inst, CURLOPT_HEADERFUNCTION, admin_log::write_header);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEHEADER, (void *)this);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEFUNCTION, admin_log::write_data);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEDATA, (void *)this);
+    if(read_function){
+      curl_easy_setopt(curl_inst, CURLOPT_READFUNCTION, read_function);
+      curl_easy_setopt(curl_inst, CURLOPT_READDATA, (void *)ud);
+      curl_easy_setopt(curl_inst, CURLOPT_UPLOAD, 1L);
+      curl_easy_setopt(curl_inst, CURLOPT_INFILESIZE_LARGE, (curl_off_t)length);
+    }
+
+    get_date(date);
+    string http_date;
+    http_date.append(string("Date: ") + date);
+
+    string s3auth;
+    if (admin_log::get_s3_auth(method, creds, date, res, s3auth) < 0)
+      return -1;
+    auth.append(string("Authorization: AWS ") + s3auth);
+
+    struct curl_slist *slist = NULL;
+    slist = curl_slist_append(slist, auth.c_str());
+    slist = curl_slist_append(slist, http_date.c_str());
+    for(list::iterator it = extra_hdrs.begin();
+        it != extra_hdrs.end(); ++it){
+      slist = curl_slist_append(slist, (*it).c_str());
+    }
+    if(read_function)
+      curl_slist_append(slist, "Expect:");
+    curl_easy_setopt(curl_inst, CURLOPT_HTTPHEADER, slist); 
+
+    response.erase(response.begin(), response.end());
+    extra_hdrs.erase(extra_hdrs.begin(), extra_hdrs.end());
+    CURLcode res = curl_easy_perform(curl_inst);
+    if(res != CURLE_OK){
+      cout << "Curl perform failed for " << url << ", res: " << 
+        curl_easy_strerror(res) << "\n";
+      return -1;
+    }
+    curl_slist_free_all(slist);
+  }
+  curl_easy_cleanup(curl_inst);
+  return 0;
+}
+};
+
+admin_log::test_helper *g_test;
+Finisher *finisher;
+RGWRados *store;
+
+int run_rgw_admin(string& cmd, string& resp) {
+  pid_t pid;
+  pid = fork();
+  if (pid == 0) {
+    /* child */
+    list l;
+    get_str_list(cmd, " \t", l);
+    char *argv[l.size()];
+    unsigned loop = 1;
+
+    argv[0] = (char *)"radosgw-admin";
+    for (list::iterator it = l.begin(); 
+         it != l.end(); ++it) {
+      argv[loop++] = (char *)(*it).c_str();
+    }
+    argv[loop] = NULL;
+    close(1);
+    stdout = fopen(RGW_ADMIN_RESP_PATH, "w+");
+    if (!stdout) {
+      cout << "Unable to open stdout file" << std::endl;
+    }
+    execv((g_test->get_rgw_admin_path()).c_str(), argv); 
+  } else if (pid > 0) {
+    int status;
+    waitpid(pid, &status, 0);
+    if (WIFEXITED(status)) {
+      if(WEXITSTATUS(status) != 0) {
+        cout << "Child exited with status " << WEXITSTATUS(status) << std::endl;
+        return -1;
+      }
+    }
+    ifstream in;
+    struct stat st;
+
+    if (stat(RGW_ADMIN_RESP_PATH, &st) < 0) {
+      cout << "Error stating the admin response file, errno " << errno << std::endl;
+      return -1;
+    } else {
+      char *data = (char *)malloc(st.st_size + 1);
+      in.open(RGW_ADMIN_RESP_PATH);
+      in.read(data, st.st_size);
+      in.close();
+      data[st.st_size] = 0;
+      resp = data;
+      free(data);
+      unlink(RGW_ADMIN_RESP_PATH);
+      /* cout << "radosgw-admin " << cmd << ": " << resp << std::endl; */
+    }
+  } else 
+    return -1;
+  return 0;
+}
+
+int get_creds(string& json, string& creds) {
+  JSONParser parser;
+  if(!parser.parse(json.c_str(), json.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+
+  RGWUserInfo info;
+  decode_json_obj(info, &parser);
+  creds = "";
+  for(map::iterator it = info.access_keys.begin();
+      it != info.access_keys.end(); ++it) {
+    RGWAccessKey _k = it->second;
+    /*cout << "accesskeys [ " << it->first << " ] = " << 
+      "{ " << _k.id << ", " << _k.key << ", " << _k.subuser << "}" << std::endl;*/
+    creds.append(it->first + string(":") + _k.key);
+    break;
+  }
+  return 0;
+}
+
+int user_create(string& uid, string& display_name, bool set_creds = true) {
+  stringstream ss;
+  string creds;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user create --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  get_creds(out, creds);
+  if(set_creds)
+    g_test->set_creds(creds);
+  return 0;
+}
+
+int user_info(string& uid, string& display_name, RGWUserInfo& uinfo) {
+  stringstream ss;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user info --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error reading user information" << std::endl;
+    return -1;
+  }
+  JSONParser parser;
+  if(!parser.parse(out.c_str(), out.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  decode_json_obj(uinfo, &parser);
+  return 0;
+}
+
+int user_rm(string& uid, string& display_name) {
+  stringstream ss;
+  ss << "-c " << g_test->get_ceph_conf_path() << 
+    " metadata rm --metadata-key=user:" << uid;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error removing user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int caps_add(const char * name, const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps add --caps=" <<
+     name << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int caps_rm(const char * name, const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps rm --caps=" <<
+     name << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+size_t read_dummy_post(void *ptr, size_t s, size_t n, void *ud) {
+  int dummy = 0;
+  memcpy(ptr, &dummy, sizeof(dummy));
+  return sizeof(dummy);
+}
+
+int parse_json_resp(JSONParser &parser) {
+  string *resp;
+  resp = (string *)g_test->get_response_data();
+  if(!resp)
+    return -1;
+  if(!parser.parse(resp->c_str(), resp->length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+static int decode_json(JSONObj *obj, cls_statelog_entry& ret) {
+  JSONDecoder::decode_json("op_id", ret.op_id, obj);
+  JSONDecoder::decode_json("client_id", ret.client_id, obj);
+  JSONDecoder::decode_json("object", ret.object, obj);
+  string state;
+  JSONDecoder::decode_json("state", state, obj);
+  RGWOpState oc(store);
+  return oc.state_from_str(state, (RGWOpState::OpState *)&ret.state);
+}
+
+static int get_opstate_list(list &entries) {
+  JSONParser parser;
+  if (parse_json_resp(parser) != 0)
+    return -1;
+  if (!parser.is_array()) 
+    return -1;
+
+  vector l;
+  l = parser.get_array_elements();
+  int loop = 0;
+  for(vector::iterator it = l.begin();
+      it != l.end(); ++it, loop++) {
+    JSONParser jp;
+    cls_statelog_entry entry;
+
+    if(!jp.parse((*it).c_str(), (*it).length())) {
+      cerr << "Error parsing log json object" << std::endl;
+      return -1;
+    }
+    EXPECT_EQ(decode_json((JSONObj *)&jp, entry), 0);
+    entries.push_back(entry);
+  }
+  return 0;
+}
+
+
+TEST(TestRGWAdmin, opstate_set_list_delete) {
+  const char *cname = "opstate",
+             *perm = "*";
+  string rest_req;
+  stringstream ss;
+  list entries;
+  string cid_1 = "cid_1", cid_2 = "cid_2";
+  string oid_1 = "operation", oid_2 = "operation";
+  string obj_1 = "obj1", obj_2 = "obj2";
+  string state;
+
+  ASSERT_EQ(user_create(uid, display_name), 0);
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  rest_req = "/admin/opstate?client-id=1&op-id=1&object=o";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing state*/
+
+  rest_req = "/admin/opstate?client-id=1&op-id=1&state=in-progress";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing object*/
+
+  rest_req = "/admin/opstate?client-id=1&state=in-progress&object=o";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing op-id*/
+
+  rest_req = "/admin/opstate?state=in-progress&op-id=1&object=o";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing client-id*/
+
+  rest_req = "/admin/opstate?state=invalid&op-id=1&object=o&client-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Invalid state*/
+
+  state = "in-progress";
+  entries.clear();
+  ss << "/admin/opstate?client-id=" << cid_1 << "&op-id=" << oid_1 
+    << "&object=" << obj_1 << "&state=" << state;
+  rest_req = ss.str();
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ss.str("");
+  entries.clear();
+  rest_req = "/admin/opstate";
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+
+  if (entries.size() == 1U) {
+    list::iterator it = entries.begin();
+
+    EXPECT_TRUE((*it).client_id.compare(cid_1) == 0);
+    EXPECT_TRUE((*it).op_id.compare(oid_1) == 0);
+    EXPECT_TRUE((*it).object.compare(obj_1) == 0);
+    EXPECT_EQ((*it).state, (uint32_t)RGWOpState::OPSTATE_IN_PROGRESS);
+  }
+
+  state = "complete";
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate?client-id=" << cid_1 << "&op-id=" << oid_1 
+    << "&object=" << obj_1 << "&state=" << state;
+  rest_req = ss.str();
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  entries.clear();
+  rest_req = "/admin/opstate";
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+
+  if (entries.size() == 1U) {
+    list::iterator it = entries.begin();
+
+    EXPECT_TRUE((*it).client_id.compare(cid_1) == 0);
+    EXPECT_TRUE((*it).op_id.compare(oid_1) == 0);
+    EXPECT_TRUE((*it).object.compare(obj_1) == 0);
+    EXPECT_EQ((*it).state, (uint32_t)RGWOpState::OPSTATE_COMPLETE);
+  }
+
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate?client-id=" << cid_2 << "&op-id=" << oid_2 
+    << "&object=" << obj_2 << "&state=" << state;
+  rest_req = ss.str();
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  entries.clear();
+  rest_req = "/admin/opstate";
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 2U);
+
+  if (entries.size() == 1U) {
+    list::iterator it = entries.begin();
+
+    EXPECT_TRUE((*it).client_id.compare(cid_1) == 0);
+    EXPECT_TRUE((*it).op_id.compare(oid_1) == 0);
+    EXPECT_TRUE((*it).object.compare(obj_1) == 0);
+    EXPECT_EQ((*it).state, (uint32_t)RGWOpState::OPSTATE_COMPLETE);
+
+    it++;
+    EXPECT_TRUE((*it).client_id.compare(cid_2) == 0);
+    EXPECT_TRUE((*it).op_id.compare(oid_2) == 0);
+    EXPECT_TRUE((*it).object.compare(obj_2) == 0);
+    EXPECT_EQ((*it).state, (uint32_t)RGWOpState::OPSTATE_COMPLETE);
+  }
+
+  entries.clear();
+  rest_req = "/admin/opstate?max-entries=1";
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+  
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate?client-id=" << cid_2 << "&op-id=" << oid_2 << "&object=" << obj_2;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+  
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate?client-id=" << cid_2;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate?op-id=" << oid_2;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 2U);
+
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate?op-id=" << oid_2 << "&object=" << obj_1 << "&client-id=" << cid_2 ;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 0U);
+
+  rest_req = "/admin/opstate?client-id=1&op-id=1";
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing object*/
+
+  rest_req = "/admin/opstate?client-id=1&object=1";
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing op-id*/
+
+  rest_req = "/admin/opstate?object=1&op-id=1";
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing client-id*/
+  
+  rest_req = "/admin/opstate?object=1&op-id=1&client-id=1";
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(404U, g_test->get_resp_code());
+
+  ss.str("");
+  ss << "/admin/opstate?client-id=" << cid_1 << "&op-id=" << oid_1 
+    << "&object=" << obj_1;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate";
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+
+  ss.str("");
+  ss << "/admin/opstate?client-id=" << cid_2 << "&op-id=" << oid_2 
+    << "&object=" << obj_2;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ss.str("");
+  entries.clear();
+  ss << "/admin/opstate";
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  EXPECT_EQ(get_opstate_list(entries), 0);
+  EXPECT_EQ(entries.size(), 0U);
+
+  ASSERT_EQ(caps_rm(cname, perm), 0);
+  ASSERT_EQ(user_rm(uid, display_name), 0);
+}
+
+TEST(TestRGWAdmin, opstate_renew) {
+  const char *cname = "opstate",
+             *perm = "*";
+  string rest_req;
+  stringstream ss;
+  string cid_1 = "cid_1";
+  string oid_1 = "operation";
+  string obj_1 = "obj1";
+  string state;
+
+  ASSERT_EQ(user_create(uid, display_name), 0);
+  ASSERT_EQ(0, caps_add(cname, perm));
+
+  rest_req = "/admin/opstate?renew&client-id=1&op-id=1&object=o";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing state*/
+
+  rest_req = "/admin/opstate?renew&client-id=1&op-id=1&state=in-progress";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing object*/
+
+  rest_req = "/admin/opstate?renew&client-id=1&state=in-progress&object=o";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing op-id*/
+
+  rest_req = "/admin/opstate?renew&state=in-progress&op-id=1&object=o";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Missing client-id*/
+
+  rest_req = "/admin/opstate?&renew&state=invalid&op-id=1&object=o&client-id=1";
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(400U, g_test->get_resp_code()); /*Invalid state*/
+
+  state = "complete";
+  ss.str("");
+  ss << "/admin/opstate?client-id=" << cid_1 << "&op-id=" << oid_1 
+    << "&object=" << obj_1 << "&state=" << state;
+  rest_req = ss.str();
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  rest_req = "/admin/opstate";
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ss.str("");
+  ss << "/admin/opstate?renew&client-id=" << cid_1 << "&op-id=1" 
+    << "&object=" << obj_1 << "&state=" << state;
+  rest_req = ss.str();
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(404U, g_test->get_resp_code());
+  
+  ss.str("");
+  ss << "/admin/opstate?renew&client-id=" << cid_1 << "&op-id=" << oid_1 
+    << "&object=" << obj_1 << "&state=" << state;
+  rest_req = ss.str();
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ss.str("");
+  ss << "/admin/opstate?renew&client-id=" << cid_1 << "&op-id=" << oid_1 
+    << "&object=" << obj_1 << "&state=in-progress";
+  rest_req = ss.str();
+  g_test->send_request(string("POST"), rest_req, read_dummy_post, NULL, sizeof(int));
+  EXPECT_EQ(500U, g_test->get_resp_code());
+
+  ss.str("");
+  ss << "/admin/opstate?client-id=" << cid_1 << "&op-id=" << oid_1 
+    << "&object=" << obj_1;
+  rest_req = ss.str();
+  g_test->send_request(string("DELETE"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(caps_rm(cname, perm), 0);
+  ASSERT_EQ(user_rm(uid, display_name), 0);
+}
+
+int main(int argc, char *argv[]){
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  store = RGWStoreManager::get_storage(g_ceph_context, false, false);
+  g_test = new admin_log::test_helper();
+  finisher = new Finisher(g_ceph_context);
+#ifdef GTEST
+  ::testing::InitGoogleTest(&argc, argv);
+#endif
+  finisher->start();
+
+  if(g_test->extract_input(argc, argv) < 0){
+    print_usage(argv[0]);
+    return -1;
+  }
+#ifdef GTEST
+  int r = RUN_ALL_TESTS();
+  if (r >= 0) {
+    cout << "There are no failures in the test case\n";
+  } else {
+    cout << "There are some failures\n";
+  }
+#endif
+  finisher->stop();
+  return 0;
+}
diff --git a/ceph/src/test/test_snap_mapper.cc b/ceph/src/test/test_snap_mapper.cc
new file mode 100644
index 00000000..04db762b
--- /dev/null
+++ b/ceph/src/test/test_snap_mapper.cc
@@ -0,0 +1,670 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "include/memory.h"
+#include 
+#include 
+#include 
+#include 
+
+#include "include/buffer.h"
+#include "common/map_cacher.hpp"
+#include "osd/SnapMapper.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+
+#include "gtest/gtest.h"
+#include "stdlib.h"
+
+using namespace std;
+
+template 
+typename T::iterator rand_choose(T &cont) {
+  if (cont.size() == 0) {
+    return cont.end();
+  }
+  int index = rand() % cont.size();
+  typename T::iterator retval = cont.begin();
+
+  for (; index > 0; --index) ++retval;
+  return retval;
+}
+
+string random_string(size_t size)
+{
+  string name;
+  for (size_t j = 0; j < size; ++j) {
+    name.push_back('a' + (rand() % 26));
+  }
+  return name;
+}
+
+class PausyAsyncMap : public MapCacher::StoreDriver {
+  struct _Op {
+    virtual void operate(map *store) = 0;
+    virtual ~_Op() {}
+  };
+  typedef ceph::shared_ptr<_Op> Op;
+  struct Remove : public _Op {
+    set to_remove;
+    Remove(const set &to_remove) : to_remove(to_remove) {}
+    void operate(map *store) {
+      for (set::iterator i = to_remove.begin();
+	   i != to_remove.end();
+	   ++i) {
+	store->erase(*i);
+      }
+    }
+  };
+  struct Insert : public _Op {
+    map to_insert;
+    Insert(const map &to_insert) : to_insert(to_insert) {}
+    void operate(map *store) {
+      for (map::iterator i = to_insert.begin();
+	   i != to_insert.end();
+	   ++i) {
+	store->erase(i->first);
+	store->insert(*i);
+      }
+    }
+  };
+  struct Callback : public _Op {
+    Context *context;
+    Callback(Context *c) : context(c) {}
+    void operate(map *store) {
+      context->complete(0);
+    }
+  };
+public:
+  class Transaction : public MapCacher::Transaction {
+    friend class PausyAsyncMap;
+    list ops;
+    list callbacks;
+  public:
+    void set_keys(const map &i) {
+      ops.push_back(Op(new Insert(i)));
+    }
+    void remove_keys(const set &r) {
+      ops.push_back(Op(new Remove(r)));
+    }
+    void add_callback(Context *c) {
+      callbacks.push_back(Op(new Callback(c)));
+    }
+  };
+private:
+
+  Mutex lock;
+  map store;
+
+  class Doer : public Thread {
+    static const size_t MAX_SIZE = 100;
+    PausyAsyncMap *parent;
+    Mutex lock;
+    Cond cond;
+    int stopping;
+    bool paused;
+    list queue;
+  public:
+    Doer(PausyAsyncMap *parent) :
+      parent(parent), lock("Doer lock"), stopping(0), paused(false) {}
+    virtual void *entry() {
+      while (1) {
+	list ops;
+	{
+	  Mutex::Locker l(lock);
+	  while (!stopping && (queue.empty() || paused))
+	    cond.Wait(lock);
+	  if (stopping && queue.empty()) {
+	    stopping = 2;
+	    cond.Signal();
+	    return 0;
+	  }
+	  assert(!queue.empty());
+	  assert(!paused);
+	  ops.swap(queue);
+	  cond.Signal();
+	}
+	assert(!ops.empty());
+
+	for (list::iterator i = ops.begin();
+	     i != ops.end();
+	     ops.erase(i++)) {
+	  if (!(rand()%3))
+	    usleep(1+(rand() % 5000));
+	  Mutex::Locker l(parent->lock);
+	  (*i)->operate(&(parent->store));
+	}
+      }
+    }
+
+    void pause() {
+      Mutex::Locker l(lock);
+      paused = true;
+      cond.Signal();
+    }
+
+    void resume() {
+      Mutex::Locker l(lock);
+      paused = false;
+      cond.Signal();
+    }
+
+    void submit(list &in) {
+      Mutex::Locker l(lock);
+      while (queue.size() >= MAX_SIZE)
+	cond.Wait(lock);
+      queue.splice(queue.end(), in, in.begin(), in.end());
+      cond.Signal();
+    }
+
+    void stop() {
+      Mutex::Locker l(lock);
+      stopping = 1;
+      cond.Signal();
+      while (stopping != 2)
+	cond.Wait(lock);
+      cond.Signal();
+    }
+  } doer;
+
+public:
+  PausyAsyncMap() : lock("PausyAsyncMap"), doer(this) {
+    doer.create();
+  }
+  ~PausyAsyncMap() {
+    doer.join();
+  }
+  int get_keys(
+    const set &keys,
+    map *out) {
+    Mutex::Locker l(lock);
+    for (set::const_iterator i = keys.begin();
+	 i != keys.end();
+	 ++i) {
+      map::iterator j = store.find(*i);
+      if (j != store.end())
+	out->insert(*j);
+    }
+    return 0;
+  }
+  int get_next(
+    const string &key,
+    pair *next) {
+    Mutex::Locker l(lock);
+    map::iterator j = store.upper_bound(key);
+    if (j != store.end()) {
+      if (next)
+	*next = *j;
+      return 0;
+    } else {
+      return -ENOENT;
+    }
+  }
+  void submit(Transaction *t) {
+    doer.submit(t->ops);
+    doer.submit(t->callbacks);
+  }
+
+  void flush() {
+    Mutex lock("flush lock");
+    Cond cond;
+    bool done = false;
+
+    class OnFinish : public Context {
+      Mutex *lock;
+      Cond *cond;
+      bool *done;
+    public:
+      OnFinish(Mutex *lock, Cond *cond, bool *done)
+	: lock(lock), cond(cond), done(done) {}
+      void finish(int) {
+	Mutex::Locker l(*lock);
+	*done = true;
+	cond->Signal();
+      }
+    };
+    Transaction t;
+    t.add_callback(new OnFinish(&lock, &cond, &done));
+    submit(&t);
+    {
+      Mutex::Locker l(lock);
+      while (!done)
+	cond.Wait(lock);
+    }
+  }
+
+  void pause() {
+    doer.pause();
+  }
+  void resume() {
+    doer.resume();
+  }
+  void stop() {
+    doer.stop();
+  }
+
+};
+
+class MapCacherTest : public ::testing::Test {
+protected:
+  boost::scoped_ptr< PausyAsyncMap > driver;
+  boost::scoped_ptr > cache;
+  map truth;
+  set names;
+public:
+  void assert_bl_eq(bufferlist &bl1, bufferlist &bl2) {
+    ASSERT_EQ(bl1.length(), bl2.length());
+    bufferlist::iterator j = bl2.begin();
+    for (bufferlist::iterator i = bl1.begin();
+	 !i.end();
+	 ++i, ++j) {
+      ASSERT_TRUE(!j.end());
+      ASSERT_EQ(*i, *j);
+    }
+  }
+  void assert_bl_map_eq(map &m1,
+			map &m2) {
+    ASSERT_EQ(m1.size(), m2.size());
+    map::iterator j = m2.begin();
+    for (map::iterator i = m1.begin();
+	 i != m1.end();
+	 ++i, ++j) {
+      ASSERT_TRUE(j != m2.end());
+      ASSERT_EQ(i->first, j->first);
+      assert_bl_eq(i->second, j->second);
+    }
+
+  }
+  size_t random_num() {
+    return random() % 10;
+  }
+  size_t random_size() {
+    return random() % 1000;
+  }
+  void random_bl(size_t size, bufferlist *bl) {
+    for (size_t i = 0; i < size; ++i) {
+      bl->append(rand());
+    }
+  }
+  void do_set() {
+    size_t set_size = random_num();
+    map to_set;
+    for (size_t i = 0; i < set_size; ++i) {
+      bufferlist bl;
+      random_bl(random_size(), &bl);
+      string key = *rand_choose(names);
+      to_set.insert(
+	make_pair(key, bl));
+    }
+    for (map::iterator i = to_set.begin();
+	 i != to_set.end();
+	 ++i) {
+      truth.erase(i->first);
+      truth.insert(*i);
+    }
+    {
+      PausyAsyncMap::Transaction t;
+      cache->set_keys(to_set, &t);
+      driver->submit(&t);
+    }
+  }
+  void remove() {
+    size_t remove_size = random_num();
+    set to_remove;
+    for (size_t i = 0; i < remove_size ; ++i) {
+      to_remove.insert(*rand_choose(names));
+    }
+    for (set::iterator i = to_remove.begin();
+	 i != to_remove.end();
+	 ++i) {
+      truth.erase(*i);
+    }
+    {
+      PausyAsyncMap::Transaction t;
+      cache->remove_keys(to_remove, &t);
+      driver->submit(&t);
+    }
+  }
+  void get() {
+    set to_get;
+    size_t get_size = random_num();
+    for (size_t i = 0; i < get_size; ++i) {
+      to_get.insert(*rand_choose(names));
+    }
+
+    map got_truth;
+    for (set::iterator i = to_get.begin();
+	 i != to_get.end();
+	 ++i) {
+      map::iterator j = truth.find(*i);
+      if (j != truth.end())
+	got_truth.insert(*j);
+    }
+
+    map got;
+    cache->get_keys(to_get, &got);
+
+    assert_bl_map_eq(got, got_truth);
+  }
+
+  void get_next() {
+    string cur;
+    while (true) {
+      pair next;
+      int r = cache->get_next(cur, &next);
+
+      pair next_truth;
+      map::iterator i = truth.upper_bound(cur);
+      int r_truth = (i == truth.end()) ? -ENOENT : 0;
+      if (i != truth.end())
+	next_truth = *i;
+
+      ASSERT_EQ(r, r_truth);
+      if (r == -ENOENT)
+	break;
+
+      ASSERT_EQ(next.first, next_truth.first);
+      assert_bl_eq(next.second, next_truth.second);
+      cur = next.first;
+    }
+  }
+  virtual void SetUp() {
+    driver.reset(new PausyAsyncMap());
+    cache.reset(new MapCacher::MapCacher(driver.get()));
+    names.clear();
+    truth.clear();
+    size_t names_size(random_num() + 10);
+    for (size_t i = 0; i < names_size; ++i) {
+      names.insert(random_string(1 + (random_size() % 10)));
+    }
+  }
+  virtual void TearDown() {
+    driver->stop();
+    cache.reset();
+    driver.reset();
+  }
+
+};
+
+TEST_F(MapCacherTest, Simple)
+{
+  driver->pause();
+  map truth;
+  set truth_keys;
+  string blah("asdf");
+  bufferlist bl;
+  ::encode(blah, bl);
+  truth[string("asdf")] = bl;
+  truth_keys.insert(truth.begin()->first);
+  {
+    PausyAsyncMap::Transaction t;
+    cache->set_keys(truth, &t);
+    driver->submit(&t);
+    cache->set_keys(truth, &t);
+    driver->submit(&t);
+  }
+
+  map got;
+  cache->get_keys(truth_keys, &got);
+  assert_bl_map_eq(got, truth);
+
+  driver->resume();
+  sleep(1);
+
+  got.clear();
+  cache->get_keys(truth_keys, &got);
+  assert_bl_map_eq(got, truth);
+}
+
+TEST_F(MapCacherTest, Random)
+{
+  for (size_t i = 0; i < 5000; ++i) {
+    if (!(i % 50)) {
+      std::cout << "On iteration " << i << std::endl;
+    }
+    switch (rand() % 4) {
+    case 0:
+      get();
+      break;
+    case 1:
+      do_set();
+      break;
+    case 2:
+      get_next();
+      break;
+    case 3:
+      remove();
+      break;
+    }
+  }
+}
+
+class MapperVerifier {
+  PausyAsyncMap *driver;
+  boost::scoped_ptr< SnapMapper > mapper;
+  map > snap_to_hobject;
+  map > hobject_to_snap;
+  snapid_t next;
+  uint32_t mask;
+  uint32_t bits;
+  Mutex lock;
+public:
+
+  MapperVerifier(
+    PausyAsyncMap *driver,
+    uint32_t mask,
+    uint32_t bits)
+    : driver(driver),
+      mapper(new SnapMapper(driver, mask, bits, 0, 1)),
+             mask(mask), bits(bits),
+      lock("lock") {}
+
+  hobject_t random_hobject() {
+    return hobject_t(
+      random_string(1+(rand() % 16)),
+      random_string(1+(rand() % 16)),
+      snapid_t(rand() % 1000),
+      (rand() & ((~0)< *snaps) {
+    assert(snaps);
+    assert(!snap_to_hobject.empty());
+    for (int i = 0; i < num || snaps->empty(); ++i) {
+      snaps->insert(rand_choose(snap_to_hobject)->first);
+    }
+  }
+
+  void create_snap() {
+    snap_to_hobject[next];
+    ++next;
+  }
+
+  void create_object() {
+    Mutex::Locker l(lock);
+    if (snap_to_hobject.empty())
+      return;
+    hobject_t obj;
+    do {
+      obj = random_hobject();
+    } while (hobject_to_snap.count(obj));
+
+    set &snaps = hobject_to_snap[obj];
+    choose_random_snaps(1 + (rand() % 20), &snaps);
+    for (set::iterator i = snaps.begin();
+	 i != snaps.end();
+	 ++i) {
+      map >::iterator j = snap_to_hobject.find(*i);
+      assert(j != snap_to_hobject.end());
+      j->second.insert(obj);
+    }
+    {
+      PausyAsyncMap::Transaction t;
+      mapper->add_oid(obj, snaps, &t);
+      driver->submit(&t);
+    }
+  }
+
+  void trim_snap() {
+    Mutex::Locker l(lock);
+    if (snap_to_hobject.empty())
+      return;
+    map >::iterator snap =
+      rand_choose(snap_to_hobject);
+    set hobjects = snap->second;
+
+    hobject_t hoid;
+    while (mapper->get_next_object_to_trim(snap->first, &hoid) == 0) {
+      assert(!hoid.is_max());
+      assert(hobjects.count(hoid));
+      hobjects.erase(hoid);
+
+      map >::iterator j =
+	hobject_to_snap.find(hoid);
+      assert(j->second.count(snap->first));
+      set old_snaps(j->second);
+      j->second.erase(snap->first);
+
+      {
+	PausyAsyncMap::Transaction t;
+	mapper->update_snaps(
+	  hoid,
+	  j->second,
+	  &old_snaps,
+	  &t);
+	driver->submit(&t);
+      }
+      if (j->second.empty()) {
+	hobject_to_snap.erase(j);
+      }
+      hoid = hobject_t::get_max();
+    }
+    assert(hobjects.empty());
+
+    snap_to_hobject.erase(snap);
+  }
+
+  void remove_oid() {
+    Mutex::Locker l(lock);
+    if (hobject_to_snap.empty())
+      return;
+    map >::iterator obj =
+      rand_choose(hobject_to_snap);
+    for (set::iterator i = obj->second.begin();
+	 i != obj->second.end();
+	 ++i) {
+      map >::iterator j =
+	snap_to_hobject.find(*i);
+      assert(j->second.count(obj->first));
+      j->second.erase(obj->first);
+    }
+    {
+      PausyAsyncMap::Transaction t;
+      mapper->remove_oid(
+	obj->first,
+	&t);
+      driver->submit(&t);
+    }
+    hobject_to_snap.erase(obj);
+  }
+
+  void check_oid() {
+    Mutex::Locker l(lock);
+    if (hobject_to_snap.empty())
+      return;
+    map >::iterator obj =
+      rand_choose(hobject_to_snap);
+    set snaps;
+    int r = mapper->get_snaps(obj->first, &snaps);
+    assert(r == 0);
+    ASSERT_EQ(snaps, obj->second);
+  }
+};
+
+class SnapMapperTest : public ::testing::Test {
+protected:
+  boost::scoped_ptr< PausyAsyncMap > driver;
+  map > mappers;
+  uint32_t pgnum;
+
+  virtual void SetUp() {
+    driver.reset(new PausyAsyncMap());
+    pgnum = 0;
+  }
+
+  virtual void TearDown() {
+    driver->stop();
+    mappers.clear();
+    driver.reset();
+  }
+
+  MapperVerifier &get_tester() {
+    //return *(mappers.begin()->second);
+    return *(rand_choose(mappers)->second);
+  }
+
+  void init(uint32_t to_set) {
+    pgnum = to_set;
+    for (uint32_t i = 0; i < pgnum; ++i) {
+      pg_t pgid(i, 0, -1);
+      mappers[pgid].reset(
+	new MapperVerifier(
+	  driver.get(),
+	  i,
+	  pgid.get_split_bits(pgnum)
+	  )
+	);
+    }
+  }
+
+  void run() {
+    for (int i = 0; i < 5000; ++i) {
+      if (!(i % 50))
+	std::cout << i << std::endl;
+      switch (rand() % 5) {
+      case 0:
+	get_tester().create_snap();
+	break;
+      case 1:
+	get_tester().create_object();
+	break;
+      case 2:
+	get_tester().trim_snap();
+	break;
+      case 3:
+	get_tester().check_oid();
+	break;
+      case 4:
+	get_tester().remove_oid();
+	break;
+      }
+    }
+  }
+};
+
+TEST_F(SnapMapperTest, Simple) {
+  init(1);
+  get_tester().create_snap();
+  get_tester().create_object();
+  get_tester().trim_snap();
+}
+
+TEST_F(SnapMapperTest, More) {
+  init(1);
+  run();
+}
+
+TEST_F(SnapMapperTest, MultiPG) {
+  init(50);
+  run();
+}
+
+int main(int argc, char **argv)
+{
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/test_str_list.cc b/ceph/src/test/test_str_list.cc
new file mode 100644
index 00000000..7b92e518
--- /dev/null
+++ b/ceph/src/test/test_str_list.cc
@@ -0,0 +1,37 @@
+
+#include "include/types.h"
+#include "include/str_list.h"
+
+#include 
+#include 
+
+#include "gtest/gtest.h"
+
+
+const char *tests[][10] = {
+  { "foo,bar", "foo", "bar", 0 },
+  { "foo", "foo", 0 },
+  { "foo;bar", "foo", "bar", 0 },
+  { "foo bar", "foo", "bar", 0 },
+  { " foo bar", "foo", "bar", 0 },
+  { " foo bar ", "foo", "bar", 0 },
+  { "a,b,c", "a", "b", "c", 0 },
+  { " a\tb\tc\t", "a", "b", "c", 0 },
+  { "a, b, c", "a", "b", "c", 0 },
+  { "a b c", "a", "b", "c", 0 },
+  { 0 },
+};
+
+TEST(StrList, All)
+{
+  for (unsigned i=0; tests[i][0]; ++i) {
+    std::string src = tests[i][0];
+    std::list expected;
+    for (unsigned j=1; tests[i][j]; ++j)
+      expected.push_back(tests[i][j]);
+    std::list actual;
+    get_str_list(src, actual);
+    std::cout << "'" << src << "' -> " << actual << std::endl;
+    ASSERT_EQ(actual, expected);
+  }
+}
diff --git a/ceph/src/test/test_stress_watch.cc b/ceph/src/test/test_stress_watch.cc
new file mode 100644
index 00000000..52b480b4
--- /dev/null
+++ b/ceph/src/test/test_stress_watch.cc
@@ -0,0 +1,118 @@
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/atomic.h"
+#include "include/utime.h"
+#include "common/Thread.h"
+#include "common/Clock.h"
+#include "test/librados/test.h"
+
+#include "gtest/gtest.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "test/librados/TestCase.h"
+
+
+using namespace librados;
+using ceph::buffer;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+static sem_t sem;
+static atomic_t stop_flag;
+
+class WatchNotifyTestCtx : public WatchCtx
+{
+public:
+    void notify(uint8_t opcode, uint64_t ver, bufferlist& bl)
+    {
+      sem_post(&sem);
+    }
+};
+
+struct WatcherUnwatcher : public Thread {
+  string pool;
+  WatcherUnwatcher(string& _pool) : pool(_pool) {}
+
+  void *entry() {
+    Rados cluster;
+    connect_cluster_pp(cluster);
+    while (!stop_flag.read()) {
+      IoCtx ioctx;
+      cluster.ioctx_create(pool.c_str(), ioctx);
+
+      uint64_t handle;
+      WatchNotifyTestCtx watch_ctx;
+      ioctx.watch("foo", 0, &handle, &watch_ctx);
+      bufferlist bl;
+      ioctx.unwatch("foo", handle);
+      ioctx.close();
+    }
+    return NULL;
+  }
+};
+
+typedef RadosTestParamPP WatchStress;
+
+INSTANTIATE_TEST_CASE_P(WatchStressTests, WatchStress,
+			::testing::Values("", "cache"));
+
+TEST_P(WatchStress, Stress1) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  Rados ncluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, ncluster));
+  IoCtx nioctx;
+  ncluster.ioctx_create(pool_name.c_str(), nioctx);
+  WatchNotifyTestCtx ctx;
+
+  WatcherUnwatcher *thr = new WatcherUnwatcher(pool_name);
+  thr->create();
+  ASSERT_EQ(0, nioctx.create("foo", false));
+
+  for (unsigned i = 0; i < 75; ++i) {
+    std::cerr << "Iteration " << i << std::endl;
+    uint64_t handle;
+    Rados cluster;
+    IoCtx ioctx;
+    WatchNotifyTestCtx ctx;
+
+    connect_cluster_pp(cluster);
+    cluster.ioctx_create(pool_name.c_str(), ioctx);
+    ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+
+    bool do_blacklist = i % 2;
+    if (do_blacklist) {
+      cluster.test_blacklist_self(true);
+      std::cerr << "blacklisted" << std::endl;
+      sleep(1);
+    }
+
+    bufferlist bl2;
+    ASSERT_EQ(0, nioctx.notify("foo", 0, bl2));
+
+    if (do_blacklist) {
+      sleep(1); // Give a change to see an incorrect notify
+    } else {
+      TestAlarm alarm;
+      sem_wait(&sem);
+    }
+
+    if (do_blacklist) {
+      cluster.test_blacklist_self(false);
+    }
+
+    ioctx.unwatch("foo", handle);
+    ioctx.close();
+  }
+  stop_flag.set(1);
+  thr->join();
+  nioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, ncluster));
+  sem_destroy(&sem);
+}
diff --git a/ceph/src/test/test_striper.cc b/ceph/src/test/test_striper.cc
new file mode 100644
index 00000000..9946142d
--- /dev/null
+++ b/ceph/src/test/test_striper.cc
@@ -0,0 +1,74 @@
+#include "gtest/gtest.h"
+#include "global/global_context.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/common_init.h"
+
+#include "osdc/Striper.h"
+
+TEST(Striper, Stripe1)
+{
+  ceph_file_layout l;
+  memset(&l, 0, sizeof(l));
+
+  l.fl_object_size = 262144;
+  l.fl_stripe_unit = 4096;
+  l.fl_stripe_count = 3;
+
+  vector ex;
+  Striper::file_to_extents(g_ceph_context, 1, &l, 5006035, 46419, 5006035, ex);
+
+  cout << "result " << ex << std::endl;
+
+  ASSERT_EQ(3u, ex.size());
+  ASSERT_EQ(98304u, ex[0].truncate_size);
+  ASSERT_EQ(ex[1].offset, ex[1].truncate_size);
+  ASSERT_EQ(94208u, ex[2].truncate_size);
+}
+
+TEST(Striper, EmptyPartialResult)
+{
+  ceph_file_layout l;
+  memset(&l, 0, sizeof(l));
+
+  l.fl_object_size = 4194304;
+  l.fl_stripe_unit = 4194304;
+  l.fl_stripe_count = 1;
+
+  vector ex;
+  Striper::file_to_extents(g_ceph_context, 1, &l, 725549056, 131072, 72554905600, ex);
+  cout << "ex " << ex << std::endl;
+  ASSERT_EQ(2u, ex.size());
+
+  Striper::StripedReadResult r;
+
+  bufferlist bl;
+  r.add_partial_result(g_ceph_context, bl, ex[1].buffer_extents);
+
+  bufferptr bp(65536);
+  bp.zero();
+  bl.append(bp);
+
+  r.add_partial_result(g_ceph_context, bl, ex[0].buffer_extents);
+
+  bufferlist outbl;
+  r.assemble_result(g_ceph_context, outbl, false);
+
+  ASSERT_EQ(65536u, outbl.length());
+}
+
+
+
+int main(int argc, char **argv)
+{
+  ::testing::InitGoogleTest(&argc, argv);
+
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/test_texttable.cc b/ceph/src/test/test_texttable.cc
new file mode 100644
index 00000000..03ae3c8e
--- /dev/null
+++ b/ceph/src/test/test_texttable.cc
@@ -0,0 +1,77 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/TextTable.h"
+#include 
+#include "gtest/gtest.h"
+
+TEST(TextTable, Alignment) {
+  TextTable t;
+
+  // test alignment
+  // 3 5-character columns
+  t.define_column("HEAD1", TextTable::LEFT, TextTable::LEFT);
+  t.define_column("HEAD2", TextTable::LEFT, TextTable::CENTER);
+  t.define_column("HEAD3", TextTable::LEFT, TextTable::RIGHT);
+
+  t << "1" << 2 << 3 << TextTable::endrow;
+  std::ostringstream oss;
+  oss << t;
+  ASSERT_STREQ("HEAD1 HEAD2 HEAD3 \n1       2       3 \n", oss.str().c_str());
+}
+
+TEST(TextTable, WidenAndClearShrink) {
+  TextTable t;
+
+  t.define_column("1", TextTable::LEFT, TextTable::LEFT);
+
+  // default column size is 1, widen to 5
+  t << "wider";
+
+  // validate wide output
+  std::ostringstream oss;
+  oss << t;
+  ASSERT_STREQ("1     \nwider \n", oss.str().c_str());
+  oss.str("");
+
+  // reset, validate single-char width output
+  t.clear();
+  t << "s";
+  oss << t;
+  ASSERT_STREQ("1 \ns \n", oss.str().c_str());
+}
+
+TEST(TextTable, Indent) {
+  TextTable t;
+
+  t.define_column("1", TextTable::LEFT, TextTable::LEFT);
+  t.set_indent(10);
+  t << "s";
+  std::ostringstream oss;
+  oss << t;
+  ASSERT_STREQ("          1 \n          s \n", oss.str().c_str());
+}
+
+
+TEST(TextTable, TooManyItems) {
+  TextTable t;
+
+  t.define_column("1", TextTable::LEFT, TextTable::LEFT);
+  t.define_column("2", TextTable::LEFT, TextTable::LEFT);
+  t.define_column("3", TextTable::LEFT, TextTable::LEFT);
+
+  // expect assertion failure on this, which throws FailedAssertion
+  ASSERT_THROW((t << "1" << "2" << "3" << "4" << TextTable::endrow),
+	       FailedAssertion);
+}
diff --git a/ceph/src/test/test_trans.cc b/ceph/src/test/test_trans.cc
new file mode 100644
index 00000000..43821c13
--- /dev/null
+++ b/ceph/src/test/test_trans.cc
@@ -0,0 +1,77 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+#include "os/FileStore.h"
+#include "global/global_init.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout
+
+struct Foo : public Thread {
+  void *entry() {
+    dout(0) << "foo started" << dendl;
+    sleep(1);
+    dout(0) << "foo asserting 0" << dendl;
+    assert(0);
+  }
+} foo;
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  // args
+  if (args.size() < 2) return -1;
+  const char *filename = args[0];
+  int mb = atoi(args[1]);
+
+  cout << "#dev " << filename << std::endl;
+  cout << "#mb " << mb << std::endl;
+
+  ObjectStore *fs = new FileStore(filename, NULL);
+  if (fs->mount() < 0) {
+    cout << "mount failed" << std::endl;
+    return -1;
+  }
+
+  ObjectStore::Transaction t;
+  char buf[1 << 20];
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  t.create_collection(coll_t());
+
+  for (int i=0; iapply_transaction(t);
+
+}
+
diff --git a/ceph/src/test/test_workqueue.cc b/ceph/src/test/test_workqueue.cc
new file mode 100644
index 00000000..602a0ec1
--- /dev/null
+++ b/ceph/src/test/test_workqueue.cc
@@ -0,0 +1,72 @@
+#include "gtest/gtest.h"
+
+#include "common/WorkQueue.h"
+#include "global/global_context.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/common_init.h"
+
+TEST(WorkQueue, StartStop)
+{
+  ThreadPool tp(g_ceph_context, "foo", 10, "");
+  
+  tp.start();
+  tp.pause();
+  tp.pause_new();
+  tp.unpause();
+  tp.unpause();
+  tp.drain();
+  tp.stop();
+}
+
+TEST(WorkQueue, Resize)
+{
+  ThreadPool tp(g_ceph_context, "bar", 2, "osd_op_threads");
+  
+  tp.start();
+
+  sleep(1);
+  ASSERT_EQ(2, tp.get_num_threads());
+
+  g_conf->set_val("osd op threads", "5");
+  g_conf->apply_changes(&cout);
+  sleep(1);
+  ASSERT_EQ(5, tp.get_num_threads());
+
+  g_conf->set_val("osd op threads", "3");
+  g_conf->apply_changes(&cout);
+  sleep(1);
+  ASSERT_EQ(3, tp.get_num_threads());
+
+  g_conf->set_val("osd op threads", "15");
+  g_conf->apply_changes(&cout);
+  sleep(1);
+  ASSERT_EQ(15, tp.get_num_threads());
+
+  g_conf->set_val("osd op threads", "0");
+  g_conf->apply_changes(&cout);
+  sleep(1);
+  ASSERT_EQ(15, tp.get_num_threads());
+
+  g_conf->set_val("osd op threads", "-1");
+  g_conf->apply_changes(&cout);
+  sleep(1);
+  ASSERT_EQ(15, tp.get_num_threads());
+
+  sleep(1);
+  tp.stop();
+}
+
+
+int main(int argc, char **argv)
+{
+  ::testing::InitGoogleTest(&argc, argv);
+
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  return RUN_ALL_TESTS();
+}
diff --git a/ceph/src/test/testcrypto.cc b/ceph/src/test/testcrypto.cc
new file mode 100644
index 00000000..0b7a9d54
--- /dev/null
+++ b/ceph/src/test/testcrypto.cc
@@ -0,0 +1,56 @@
+#include "auth/Crypto.h"
+#include "common/Clock.h"
+
+#include "common/config.h"
+#include "common/debug.h"
+
+#define dout_subsys ceph_subsys_auth
+
+#define AES_KEY_LEN	16
+
+int main(int argc, char *argv[])
+{
+  char aes_key[AES_KEY_LEN];
+  memset(aes_key, 0x77, sizeof(aes_key));
+  bufferptr keybuf(aes_key, sizeof(aes_key));
+  CryptoKey key(CEPH_CRYPTO_AES, ceph_clock_now(g_ceph_context), keybuf);
+
+  const char *msg="hello! this is a message\n";
+  char pad[16];
+  memset(pad, 0, 16);
+  bufferptr ptr(msg, strlen(msg));
+  bufferlist enc_in;
+  enc_in.append(ptr);
+  enc_in.append(msg, strlen(msg));
+
+  bufferlist enc_out;
+  std::string error;
+  key.encrypt(g_ceph_context, enc_in, enc_out, error);
+  if (!error.empty()) {
+    dout(0) << "couldn't encode! error " << error << dendl;
+    exit(1);
+  }
+
+  const char *enc_buf = enc_out.c_str();
+  for (unsigned i=0; i args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  KeyRing extra;
+  KeyServer server(g_ceph_context, &extra);
+
+  generic_dout(0) << "server created" << dendl;
+
+  getchar();
+
+#if 0
+  char aes_key[AES_KEY_LEN];
+  memset(aes_key, 0x77, sizeof(aes_key));
+  bufferptr keybuf(aes_key, sizeof(aes_key));
+  CryptoKey key(CEPH_CRYPTO_AES, ceph_clock_now(g_ceph_context), keybuf);
+
+  const char *msg="hello! this is a message\n";
+  char pad[16];
+  memset(pad, 0, 16);
+  bufferptr ptr(msg, strlen(msg));
+  bufferlist enc_in;
+  enc_in.append(ptr);
+  enc_in.append(msg, strlen(msg));
+
+  bufferlist enc_out;
+  if (key.encrypt(enc_in, enc_out) < 0) {
+    derr(0) << "couldn't encode!" << dendl;
+    exit(1);
+  }
+
+  const char *enc_buf = enc_out.c_str();
+  for (unsigned i=0; i
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+using namespace std;
+
+#include "common/config.h"
+
+#include "mon/MonMap.h"
+#include "mon/MonClient.h"
+#include "msg/Messenger.h"
+#include "messages/MPing.h"
+
+#include "common/Timer.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+
+#ifndef DARWIN
+#include 
+#endif // DARWIN
+
+#include 
+#include 
+
+#define dout_subsys ceph_subsys_ms
+
+Messenger *messenger = 0;
+
+Mutex test_lock("mylock");
+Cond cond;
+
+uint64_t received = 0;
+
+class Admin : public Dispatcher {
+public:
+  Admin() 
+    : Dispatcher(g_ceph_context)
+  {
+  }
+private:
+  bool ms_dispatch(Message *m) {
+
+    //cerr << "got ping from " << m->get_source() << std::endl;
+    dout(0) << "got ping from " << m->get_source() << dendl;
+    test_lock.Lock();
+    ++received;
+    cond.Signal();
+    test_lock.Unlock();
+
+    m->put();
+    return true;
+  }
+
+  bool ms_handle_reset(Connection *con) { return false; }
+  void ms_handle_remote_reset(Connection *con) {}
+
+} dispatcher;
+
+
+int main(int argc, const char **argv, const char *envp[]) {
+
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  dout(0) << "i am mon " << args[0] << dendl;
+
+  // get monmap
+  MonClient mc(g_ceph_context);
+  if (mc.build_initial_monmap() < 0)
+    return -1;
+  
+  // start up network
+  int whoami = mc.monmap.get_rank(args[0]);
+  assert(whoami >= 0);
+  ostringstream ss;
+  ss << mc.monmap.get_addr(whoami);
+  std::string sss(ss.str());
+  g_ceph_context->_conf->set_val("public_addr", sss.c_str());
+  g_ceph_context->_conf->apply_changes(NULL);
+  Messenger *rank = Messenger::create(g_ceph_context,
+				      entity_name_t::MON(whoami), "tester",
+				      getpid());
+  int err = rank->bind(g_ceph_context->_conf->public_addr);
+  if (err < 0)
+    return 1;
+
+  // start monitor
+  messenger = rank;
+  messenger->set_default_send_priority(CEPH_MSG_PRIO_HIGH);
+  messenger->add_dispatcher_head(&dispatcher);
+
+  rank->start();
+  
+  int isend = 0;
+  if (whoami == 0)
+    isend = 100;
+
+  test_lock.Lock();
+  uint64_t sent = 0;
+  while (1) {
+    while (received + isend <= sent) {
+      //cerr << "wait r " << received << " s " << sent << " is " << isend << std::endl;
+      dout(0) << "wait r " << received << " s " << sent << " is " << isend << dendl;
+      cond.Wait(test_lock);
+    }
+
+    int t = rand() % mc.get_num_mon();
+    if (t == whoami)
+      continue;
+    
+    if (rand() % 10 == 0) {
+      //cerr << "mark_down " << t << std::endl;
+      dout(0) << "mark_down " << t << dendl;
+      messenger->mark_down(mc.get_mon_addr(t));
+    } 
+    //cerr << "pinging " << t << std::endl;
+    dout(0) << "pinging " << t << dendl;
+    messenger->send_message(new MPing, mc.get_mon_inst(t));
+    cerr << isend << "\t" << ++sent << "\t" << received << "\r";
+  }
+  test_lock.Unlock();
+
+  // wait for messenger to finish
+  rank->wait();
+  
+  return 0;
+}
+
diff --git a/ceph/src/test/unit.h b/ceph/src/test/unit.h
new file mode 100644
index 00000000..b1fb373e
--- /dev/null
+++ b/ceph/src/test/unit.h
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_UNIT_TEST_H
+#define CEPH_UNIT_TEST_H
+
+#include "include/types.h" // FIXME: ordering shouldn't be important, but right 
+                           // now, this include has to come before the others.
+
+#include "common/code_environment.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "include/msgr.h" // for CEPH_ENTITY_TYPE_CLIENT
+#include "gtest/gtest.h"
+
+#include 
+
+/*
+ * You only need to include this file if you are testing Ceph internal code. If
+ * you are testing library code, the library init() interfaces will handle
+ * initialization for you.
+ */
+int main(int argc, char **argv) {
+  std::vector args;
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+#endif
diff --git a/ceph/src/test/utf8.cc b/ceph/src/test/utf8.cc
new file mode 100644
index 00000000..8e22e524
--- /dev/null
+++ b/ceph/src/test/utf8.cc
@@ -0,0 +1,66 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "common/utf8.h"
+#include "gtest/gtest.h"
+#include 
+
+TEST(IsValidUtf8, SimpleAscii) {
+  ASSERT_EQ(0, check_utf8_cstr("Ascii ftw."));
+  ASSERT_EQ(0, check_utf8_cstr(""));
+  ASSERT_EQ(0, check_utf8_cstr("B"));
+  ASSERT_EQ(0, check_utf8_cstr("Badgers badgers badgers badgers "
+			       "mushroom mushroom"));
+  ASSERT_EQ(0, check_utf8("foo", strlen("foo")));
+}
+
+TEST(IsValidUtf8, ControlChars) {
+  // Sadly, control characters are valid utf8...
+  uint8_t control_chars[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 
+			      0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d };
+  ASSERT_EQ(0, check_utf8((char*)control_chars, sizeof(control_chars)));
+}
+
+TEST(IsValidUtf8, SimpleUtf8) {
+  uint8_t funkystr[] = { 0x66, 0xd1, 0x86, 0xd1, 0x9d, 0xd2, 0xa0, 0xd3,
+		       0xad, 0xd3, 0xae, 0x0a };
+  ASSERT_EQ(0, check_utf8((char*)funkystr, sizeof(funkystr)));
+
+  uint8_t valid2[] = { 0xc3, 0xb1 };
+  ASSERT_EQ(0, check_utf8((char*)valid2, sizeof(valid2)));
+}
+
+TEST(IsValidUtf8, InvalidUtf8) {
+  uint8_t inval[] = { 0xe2, 0x28, 0xa1 };
+  ASSERT_NE(0, check_utf8((char*)inval, sizeof(inval)));
+
+  uint8_t invalid2[] = { 0xc3, 0x28 };
+  ASSERT_NE(0, check_utf8((char*)invalid2, sizeof(invalid2)));
+}
+
+TEST(HasControlChars, HasControlChars1) {
+  uint8_t has_control_chars[] = { 0x41, 0x01, 0x00 };
+  ASSERT_NE(0, check_for_control_characters_cstr((const char*)has_control_chars));
+  uint8_t has_control_chars2[] = { 0x7f, 0x41, 0x00 };
+  ASSERT_NE(0, check_for_control_characters_cstr((const char*)has_control_chars2));
+
+  char has_newline[] = "blah   blah\n";
+  ASSERT_NE(0, check_for_control_characters_cstr(has_newline));
+
+  char no_control_chars[] = "blah   blah";
+  ASSERT_EQ(0, check_for_control_characters_cstr(no_control_chars));
+
+  uint8_t validutf[] = { 0x66, 0xd1, 0x86, 0xd1, 0x9d, 0xd2, 0xa0, 0xd3,
+		       0xad, 0xd3, 0xae, 0x0 };
+  ASSERT_EQ(0, check_for_control_characters_cstr((const char*)validutf));
+}
diff --git a/ceph/src/test/xattr_bench.cc b/ceph/src/test/xattr_bench.cc
new file mode 100644
index 00000000..2b8d09ff
--- /dev/null
+++ b/ceph/src/test/xattr_bench.cc
@@ -0,0 +1,200 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "os/FileStore.h"
+#include "include/Context.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/unordered_map.h"
+
+void usage(const string &name) {
+  std::cerr << "Usage: " << name << " [xattr|omap] store_path store_journal"
+	    << std::endl;
+}
+
+const int THREADS = 5;
+
+template 
+typename T::iterator rand_choose(T &cont) {
+  if (cont.size() == 0) {
+    return cont.end();
+  }
+  int index = rand() % cont.size();
+  typename T::iterator retval = cont.begin();
+
+  for (; index > 0; --index) ++retval;
+  return retval;
+}
+
+class OnApplied : public Context {
+public:
+  Mutex *lock;
+  Cond *cond;
+  int *in_progress;
+  ObjectStore::Transaction *t;
+  OnApplied(Mutex *lock,
+	    Cond *cond,
+	    int *in_progress,
+	    ObjectStore::Transaction *t)
+    : lock(lock), cond(cond),
+      in_progress(in_progress), t(t) {
+    Mutex::Locker l(*lock);
+    (*in_progress)++;
+  }
+
+  void finish(int r) {
+    Mutex::Locker l(*lock);
+    (*in_progress)--;
+    cond->Signal();
+  }
+};
+
+uint64_t get_time() {
+  time_t start;
+  time(&start);
+  return start * 1000;
+}
+
+double print_time(uint64_t ms) {
+  return ((double)ms)/1000;
+}
+
+uint64_t do_run(ObjectStore *store, int attrsize, int numattrs,
+		int run,
+		int transsize, int ops,
+		ostream &out) {
+  Mutex lock("lock");
+  Cond cond;
+  int in_flight = 0;
+  ObjectStore::Transaction t;
+  map, ObjectStore::Sequencer*> > collections;
+  for (int i = 0; i < 3*THREADS; ++i) {
+    stringstream coll_str;
+    coll_str << "coll_" << i << "_" << run;
+    t.create_collection(coll_t(coll_str.str()));
+    set objects;
+    for (int i = 0; i < transsize; ++i) {
+      stringstream obj_str;
+      obj_str << i;
+      t.touch(coll_t(coll_str.str()),
+	      hobject_t(sobject_t(obj_str.str(), CEPH_NOSNAP)));
+      objects.insert(obj_str.str());
+    }
+    collections[coll_str.str()] = make_pair(objects, new ObjectStore::Sequencer(coll_str.str()));
+  }
+  store->apply_transaction(t);
+
+  bufferlist bl;
+  for (int i = 0; i < attrsize; ++i) {
+    bl.append('\0');
+  }
+
+  uint64_t start = get_time();
+  for (int i = 0; i < ops; ++i) {
+    {
+      Mutex::Locker l(lock);
+      while (in_flight >= THREADS)
+	cond.Wait(lock);
+    }
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    map, ObjectStore::Sequencer*> >::iterator iter =
+      rand_choose(collections);
+    for (set::iterator obj = iter->second.first.begin();
+	 obj != iter->second.first.end();
+	 ++obj) {
+      for (int j = 0; j < numattrs; ++j) {
+	stringstream ss;
+	ss << i << ", " << j << ", " << *obj;
+	t->setattr(coll_t(iter->first),
+		   hobject_t(sobject_t(*obj, CEPH_NOSNAP)),
+		   ss.str().c_str(),
+		   bl);
+      }
+    }
+    store->queue_transaction(iter->second.second, t,
+			     new OnApplied(&lock, &cond, &in_flight,
+					   t));
+  }
+  {
+    Mutex::Locker l(lock);
+    while (in_flight)
+      cond.Wait(lock);
+  }
+  return get_time() - start;
+}
+
+int main(int argc, char **argv) {
+  vector args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(0, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  if (args[0] == string("omap")) {
+    std::cerr << "using omap xattrs" << std::endl;
+    g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "true");
+  } else {
+    std::cerr << "not using omap xattrs" << std::endl;
+    g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "false");
+  }
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  std::cerr << "args: " << args << std::endl;
+  if (args.size() < 3) {
+    usage(argv[0]);
+    return 1;
+  }
+
+  string store_path(args[1]);
+  string store_dev(args[2]);
+
+  boost::scoped_ptr store(new FileStore(store_path, store_dev));
+
+  std::cerr << "mkfs starting" << std::endl;
+  assert(!store->mkfs());
+  assert(!store->mount());
+  std::cerr << "mounted" << std::endl;
+
+  std::cerr << "attrsize\tnumattrs\ttranssize\tops\ttime" << std::endl;
+  int runs = 0;
+  int total_size = 11;
+  for (int i = 6; i < total_size; ++i) {
+    for (int j = (total_size - i); j >= 0; --j) {
+      std::cerr << "starting run " << runs << std::endl;
+      ++runs;
+      uint64_t time = do_run(store.get(), (1 << i), (1 << j), runs,
+			     10,
+			     1000, std::cout);
+      std::cout << (1 << i) << "\t"
+		<< (1 << j) << "\t"
+		<< 10 << "\t"
+		<< 1000 << "\t"
+		<< print_time(time) << std::endl;
+    }
+  }
+  store->umount();
+  return 0;
+}
diff --git a/ceph/src/tools/Makefile.am b/ceph/src/tools/Makefile.am
new file mode 100644
index 00000000..6078eb47
--- /dev/null
+++ b/ceph/src/tools/Makefile.am
@@ -0,0 +1,104 @@
+ceph_osdomap_tool_SOURCES = tools/ceph_osdomap_tool.cc
+ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
+bin_DEBUGPROGRAMS += ceph-osdomap-tool
+
+ceph_monstore_tool_SOURCES = tools/ceph_monstore_tool.cc
+ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
+bin_DEBUGPROGRAMS += ceph-monstore-tool
+
+ceph_kvstore_tool_SOURCES = tools/ceph_kvstore_tool.cc
+ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph-kvstore-tool
+
+
+ceph_filestore_tool_SOURCES = tools/ceph_filestore_tool.cc
+ceph_filestore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
+if LINUX
+ceph_filestore_tool_LDADD += -ldl
+endif # LINUX
+bin_PROGRAMS += ceph_filestore_tool
+
+ceph_filestore_dump_SOURCES = tools/ceph_filestore_dump.cc
+ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
+if LINUX
+ceph_filestore_dump_LDADD += -ldl
+endif # LINUX
+bin_PROGRAMS += ceph_filestore_dump
+
+monmaptool_SOURCES = tools/monmaptool.cc
+monmaptool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
+bin_PROGRAMS += monmaptool
+
+crushtool_SOURCES = tools/crushtool.cc
+crushtool_LDADD = $(CEPH_GLOBAL)
+bin_PROGRAMS += crushtool
+
+osdmaptool_SOURCES = tools/osdmaptool.cc
+osdmaptool_LDADD = $(CEPH_GLOBAL)
+bin_PROGRAMS += osdmaptool
+
+ceph_scratchtool_SOURCES = tools/scratchtool.c
+ceph_scratchtool_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_scratchtool
+
+ceph_scratchtoolpp_SOURCES = tools/scratchtoolpp.cc
+ceph_scratchtoolpp_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_scratchtoolpp
+
+ceph_psim_SOURCES = tools/psim.cc
+ceph_psim_LDADD = $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_psim
+
+ceph_dupstore_SOURCES = tools/dupstore.cc
+ceph_dupstore_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_dupstore
+
+ceph_radosacl_SOURCES = tools/radosacl.cc
+ceph_radosacl_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_radosacl
+
+ceph_client_debug_SOURCES = tools/ceph-client-debug.cc
+ceph_client_debug_LDADD = $(LIBCEPHFS) $(CEPH_GLOBAL) $(LIBCOMMON)
+bin_DEBUGPROGRAMS += ceph-client-debug
+
+rados_SOURCES = \
+	tools/rados/rados.cc \
+	tools/rados/rados_import.cc \
+	tools/rados/rados_export.cc \
+	tools/rados/rados_sync.cc
+rados_SOURCES += common/obj_bencher.cc # needs cleanup so it can go in libcommon.la
+rados_LDADD = libcls_lock_client.la $(LIBRADOS) $(CEPH_GLOBAL)
+bin_PROGRAMS += rados
+
+if WITH_REST_BENCH
+rest_bench_SOURCES = tools/rest_bench.cc
+rest_bench_SOURCES += common/obj_bencher.cc # needs cleanup so it can go in libcommon.la
+rest_bench_LDADD = $(CEPH_GLOBAL)
+bin_PROGRAMS += rest-bench
+
+if WITH_SYSTEM_LIBS3
+rest_bench_LDADD += -ls3
+else
+rest_bench_LDADD += libs3/build/lib/libs3.a -lcurl -lxml2
+rest_bench_CXXFLAGS = ${AM_CXXFLAGS} -I$(top_srcdir)/src/libs3/inc
+SUBDIRS += libs3
+endif # WITH_SYSTEM_LIBS3
+endif # WITH_REST_BENCH
+
+ceph_conf_SOURCES = tools/ceph_conf.cc
+ceph_conf_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
+bin_PROGRAMS += ceph-conf
+
+ceph_authtool_SOURCES = tools/ceph_authtool.cc
+ceph_authtool_LDADD = $(CEPH_GLOBAL) $(LIBCOMMON)
+bin_PROGRAMS += ceph-authtool
+
+ceph_mon_store_converter_SOURCES = tools/mon_store_converter.cc
+ceph_mon_store_converter_LDADD = $(LIBMON) $(LIBOS) $(CEPH_GLOBAL)
+bin_PROGRAMS += ceph_mon_store_converter
+
+noinst_HEADERS += \
+	tools/rados/rados_sync.h \
+	tools/common.h
+
diff --git a/ceph/src/tools/ceph-client-debug.cc b/ceph/src/tools/ceph-client-debug.cc
new file mode 100644
index 00000000..2ed93326
--- /dev/null
+++ b/ceph/src/tools/ceph-client-debug.cc
@@ -0,0 +1,179 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/Formatter.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "client/Inode.h"
+#include "client/Dentry.h"
+#include "client/Dir.h"
+#include "include/cephfs/libcephfs.h"
+
+#define dout_subsys ceph_subsys_client
+
+void usage()
+{
+  std::cout << "Usage: ceph-client-debug [options] " << std::endl;
+  generic_client_usage();
+}
+
+
+/**
+ * Given an inode, look up the path from the Client cache: assumes
+ * client cache is fully populated.
+ */
+void traverse_dentries(Inode *ino, std::vector &parts)
+{
+  if (ino->dn_set.empty()) {
+    return;
+  }
+  
+  Dentry* dn = *(ino->dn_set.begin());
+  parts.push_back(dn);
+  traverse_dentries(dn->dir->parent_inode, parts);
+}
+
+
+/**
+ * Given an inode, send lookup requests to the MDS for
+ * all its ancestors, such that the full trace will be
+ * populated in client cache.
+ */
+int lookup_trace(ceph_mount_info *client, inodeno_t const ino)
+{
+  Inode *inode;
+  int r = ceph_ll_lookup_inode(client, ino, &inode);
+  if (r != 0) {
+    return r;
+  } else {
+    if (!inode->dn_set.empty()) {
+      Dentry *dn = *(inode->dn_set.begin());
+      assert(dn->dir);
+      assert(dn->dir->parent_inode);
+      r = lookup_trace(client, dn->dir->parent_inode->ino);
+      if (r) {
+        return r;
+      }
+    } else {
+      // We reached the root of the tree
+      assert(inode->ino == CEPH_INO_ROOT);
+    }
+  }
+
+  return r;
+}
+
+
+int main(int argc, const char **argv)
+{
+  // Argument handling
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS);
+  
+  common_init_finish(g_ceph_context);
+
+  // Expect exactly one positional argument (inode number)
+  if (args.size() != 1) {
+    usage();
+  }
+  char const *inode_str = args[0];
+  inodeno_t inode = strtoll(inode_str, NULL, 0);
+  if (inode <= 0) {
+    derr << "Invalid inode: " << inode_str << dendl;
+    return -1;
+  }
+
+  // Initialize filesystem client
+  struct ceph_mount_info *client;
+  int r = ceph_create_with_context(&client, g_ceph_context);
+  if (r) {
+    derr << "Error initializing libcephfs: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  r = ceph_mount(client, "/");
+  if (r) {
+    derr << "Error mounting: " << cpp_strerror(r) << dendl;
+    ceph_shutdown(client);
+    return r;
+  }
+
+
+  // Populate client cache with inode of interest & ancestors
+  r = lookup_trace(client, inode);
+  if (r) {
+    derr << "Error looking up inode " << std::hex << inode << std::dec <<
+      ": " << cpp_strerror(r) << dendl;
+    return -1;
+  }
+
+  // Retrieve inode of interest
+  struct vinodeno_t vinode;
+  vinode.ino = inode;
+  vinode.snapid = CEPH_NOSNAP;
+  Inode *ino = ceph_ll_get_inode(client, vinode);
+
+  // Retrieve dentry trace
+  std::vector path;
+  traverse_dentries(ino, path);
+  
+  // Print inode and path as a JSON object
+  JSONFormatter jf(true);
+  jf.open_object_section("client_debug");
+  {
+    jf.open_object_section("inode");
+    {
+      ino->dump(&jf);
+    }
+    jf.close_section(); // inode
+    jf.open_array_section("path");
+    {
+      for (std::vector::reverse_iterator p = path.rbegin(); p != path.rend(); ++p) {
+        jf.open_object_section("dentry");
+        {
+          (*p)->dump(&jf);
+        }
+        jf.close_section(); // dentry
+      }
+    }
+    jf.close_section(); // path
+  }
+  jf.close_section(); // client_debug
+  jf.flush(std::cout);
+  std::cout << std::endl;
+
+  // Release Inode references
+  ceph_ll_forget(client, ino, 1);
+  for (std::vector::reverse_iterator p = path.rbegin(); p != path.rend(); ++p) {
+    ceph_ll_forget(client, (*p)->inode, 1);
+  }
+  ino = NULL;
+  path.clear();  
+
+  // Shut down
+  r = ceph_unmount(client);
+  if (r) {
+    derr << "Error mounting: " << cpp_strerror(r) << dendl;
+  }
+  ceph_shutdown(client);
+  
+  return r;
+}
diff --git a/ceph/src/tools/ceph_authtool.cc b/ceph/src/tools/ceph_authtool.cc
new file mode 100644
index 00000000..f66a3c66
--- /dev/null
+++ b/ceph/src/tools/ceph_authtool.cc
@@ -0,0 +1,277 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2009 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "common/config.h"
+#include "common/strtol.h"
+
+#include "common/ConfUtils.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "auth/Crypto.h"
+#include "auth/Auth.h"
+#include "auth/KeyRing.h"
+
+#include 
+
+void usage()
+{
+  cout << "usage: ceph-authtool keyringfile [OPTIONS]...\n"
+       << "where the options are:\n"
+       << "  -l, --list                    will list all keys and capabilities present in\n"
+       << "                                the keyring\n"
+       << "  -p, --print-key               will print an encoded key for the specified\n"
+       << "                                entityname. This is suitable for the\n"
+       << "                                'mount -o secret=..' argument\n"
+       << "  -C, --create-keyring          will create a new keyring, overwriting any\n"
+       << "                                existing keyringfile\n"
+       << "  -g, --gen-key                 will generate a new secret key for the\n"
+       << "                                specified entityname\n"
+       << "  --gen-print-key               will generate a new secret key without set it\n"
+       << "                                to the keyringfile, prints the secret to stdout\n"
+       << "  --import-keyring              will import the content of a given keyring\n"
+       << "                                into the keyringfile\n"
+       << "  -u, --set-uid                 sets the auid (authenticated user id) for the\n"
+       << "                                specified entityname\n"
+       << "  -a, --add-key                 will add an encoded key to the keyring\n"
+       << "  --cap subsystem capability    will set the capability for given subsystem\n"
+       << "  --caps capsfile               will set all of capabilities associated with a\n"
+       << "                                given key, for all subsystems"
+       << std::endl;
+  exit(1);
+}
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  bool gen_key = false;
+  bool gen_print_key = false;
+  std::string add_key;
+  bool list = false;
+  bool print_key = false;
+  bool create_keyring = false;
+  std::string caps_fn;
+  std::string import_keyring;
+  bool set_auid = false;
+  uint64_t auid = CEPH_AUTH_UID_DEFAULT;
+  map caps;
+  std::string fn;
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  std::vector::iterator i;
+  for (i = args.begin(); i != args.end(); ) {
+    std::string val;
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-g", "--gen-key", (char*)NULL)) {
+      gen_key = true;
+    } else if (ceph_argparse_flag(args, i, "--gen-print-key", (char*)NULL)) {
+      gen_print_key = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "-a", "--add-key", (char*)NULL)) {
+      add_key = val;
+    } else if (ceph_argparse_flag(args, i, &val, "-l", "--list", (char*)NULL)) {
+      list = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
+      caps_fn = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--cap", (char*)NULL)) {
+      std::string my_key = val;
+      if (i == args.end()) {
+	cerr << "must give two arguments to --cap: key and val." << std::endl;
+	exit(1);
+      }
+      std::string my_val = *i;
+      ++i;
+      ::encode(my_val, caps[my_key]);
+    } else if (ceph_argparse_flag(args, i, "-p", "--print-key", (char*)NULL)) {
+      print_key = true;
+    } else if (ceph_argparse_flag(args, i, "-C", "--create-keyring", (char*)NULL)) {
+      create_keyring = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--import-keyring", (char*)NULL)) {
+      import_keyring = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-u", "--set-uid", (char*)NULL)) {
+      std::string err;
+      auid = strict_strtoll(val.c_str(), 10, &err);
+      if (!err.empty()) {
+	cerr << "error parsing UID: " << err << std::endl;
+	exit(1);
+      }
+      set_auid = true;
+    } else if (fn.empty()) {
+      fn = *i++;
+    } else {
+      cerr << argv[0] << ": unexpected '" << *i << "'" << std::endl;
+      usage();
+    }
+  }
+  if (fn.empty() && !gen_print_key) {
+    cerr << argv[0] << ": must specify filename" << std::endl;
+    usage();
+  }
+  if (!(gen_key ||
+	gen_print_key ||
+	!add_key.empty() ||
+	list ||
+	!caps_fn.empty() ||
+	!caps.empty() ||
+	set_auid ||
+	print_key ||
+	create_keyring ||
+	!import_keyring.empty())) {
+    cerr << "no command specified" << std::endl;
+    usage();
+  }
+  if (gen_key && (!add_key.empty())) {
+    cerr << "can't both gen_key and add_key" << std::endl;
+    usage();
+  }	
+
+  common_init_finish(g_ceph_context);
+  EntityName ename(g_conf->name);
+
+  if (gen_print_key) {
+    CryptoKey key;
+    key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    cout << key << std::endl;    
+    return 0;
+  }
+
+  // keyring --------
+  bool modified = false;
+  KeyRing keyring;
+
+  bufferlist bl;
+  int r = 0;
+  if (create_keyring) {
+    cout << "creating " << fn << std::endl;
+    modified = true;
+  } else {
+    std::string err;
+    r = bl.read_file(fn.c_str(), &err);
+    if (r >= 0) {
+      try {
+	bufferlist::iterator iter = bl.begin();
+	::decode(keyring, iter);
+      } catch (const buffer::error &err) {
+	cerr << "error reading file " << fn << std::endl;
+	exit(1);
+      }
+    } else {
+      cerr << "can't open " << fn << ": " << err << std::endl;
+      exit(1);
+    }
+  }
+
+  // write commands
+  if (!import_keyring.empty()) {
+    KeyRing other;
+    bufferlist obl;
+    std::string err;
+    int r = obl.read_file(import_keyring.c_str(), &err);
+    if (r >= 0) {
+      try {
+	bufferlist::iterator iter = obl.begin();
+	::decode(other, iter);
+      } catch (const buffer::error &err) {
+	cerr << "error reading file " << import_keyring << std::endl;
+	exit(1);
+      }
+      
+      cout << "importing contents of " << import_keyring << " into " << fn << std::endl;
+      //other.print(cout);
+      keyring.import(g_ceph_context, other);
+      modified = true;
+    } else {
+      cerr << "can't open " << import_keyring << ": " << err << std::endl;
+      exit(1);
+    }
+  }
+  if (gen_key) {
+    EntityAuth eauth;
+    eauth.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    keyring.add(ename, eauth);
+    modified = true;
+  }
+  if (!add_key.empty()) {
+    EntityAuth eauth;
+    try {
+      eauth.key.decode_base64(add_key);
+    } catch (const buffer::error &err) {
+      cerr << "can't decode key '" << add_key << "'" << std::endl;
+      exit(1);
+    }
+    keyring.add(ename, eauth);
+    modified = true;
+    cout << "added entity " << ename << " auth " << eauth << std::endl;
+  }
+  if (!caps_fn.empty()) {
+    ConfFile cf;
+    std::deque parse_errors;
+    if (cf.parse_file(caps_fn, &parse_errors, &cerr) != 0) {
+      cerr << "could not parse caps file " << caps_fn << std::endl;
+      exit(1);
+    }
+    complain_about_parse_errors(g_ceph_context, &parse_errors);
+    map caps;
+    const char *key_names[] = { "mon", "osd", "mds", NULL };
+    for (int i=0; key_names[i]; i++) {
+      std::string val;
+      if (cf.read("global", key_names[i], val) == 0) {
+        bufferlist bl;
+        ::encode(val, bl);
+        string s(key_names[i]);
+        caps[s] = bl; 
+      }
+    }
+    keyring.set_caps(ename, caps);
+    modified = true;
+  }
+  if (!caps.empty()) {
+    keyring.set_caps(ename, caps);
+    modified = true;
+  }
+  if (set_auid) {
+    keyring.set_uid(ename, auid);
+    modified = true;
+  }
+
+  // read commands
+  if (list) {
+    keyring.print(cout);
+  }
+  if (print_key) {
+    CryptoKey key;
+    if (keyring.get_secret(ename, key)) {
+      cout << key << std::endl;
+    } else {
+      cerr << "entity " << ename << " not found" << std::endl;
+    }
+  }
+
+  // write result?
+  if (modified) {
+    bufferlist bl;
+    keyring.encode_plaintext(bl);
+    r = bl.write_file(fn.c_str(), 0600);
+    if (r < 0) {
+      cerr << "could not write " << fn << std::endl;
+    }
+    //cout << "wrote " << bl.length() << " bytes to " << fn << std::endl;
+  }
+
+  return 0;
+}
diff --git a/ceph/src/tools/ceph_conf.cc b/ceph/src/tools/ceph_conf.cc
new file mode 100644
index 00000000..09bbc561
--- /dev/null
+++ b/ceph/src/tools/ceph_conf.cc
@@ -0,0 +1,229 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2010 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "mon/AuthMonitor.h"
+#include "common/ConfUtils.h"
+#include "global/global_init.h"
+#include "common/entity_name.h"
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "include/str_list.h"
+
+using std::deque;
+using std::string;
+
+static void usage()
+{
+  // TODO: add generic_usage once cerr/derr issues are resolved
+  cerr << "Ceph configuration query tool\n\
+\n\
+USAGE\n\
+ceph-conf  \n\
+\n\
+ACTIONS\n\
+  -L|--list-all-sections          List all sections\n\
+  -l|--list-sections      List sections with the given prefix\n\
+  --filter-key               Filter section list to only include sections\n\
+                                  with given key defined.\n\
+  --filter-key-value =  Filter section list to only include sections\n\
+                                  with given key/value pair.\n\
+  --lookup                   Print a configuration setting to stdout.\n\
+                                  Returns 0 (success) if the configuration setting is\n\
+                                  found; 1 otherwise.\n\
+  -r|--resolve-search             search for the first file that exists and\n\
+                                  can be opened in the resulted comma\n\
+                                  delimited search list.\n\
+\n\
+FLAGS\n\
+  --name name                     Set type.id\n\
+  [-s 
]                  Add to list of sections to search\n\
+\n\
+If there is no action given, the action will default to --lookup.\n\
+\n\
+EXAMPLES\n\
+$ ceph-conf --name mon.0 -c /etc/ceph/ceph.conf 'mon addr'\n\
+Find out what the value of 'mon add' is for monitor 0.\n\
+\n\
+$ ceph-conf -l mon\n\
+List sections beginning with 'mon'.\n\
+\n\
+RETURN CODE\n\
+Return code will be 0 on success; error code otherwise.\n\
+";
+  exit(1);
+}
+
+static int list_sections(const std::string &prefix,
+			 const std::list& filter_key,
+			 const std::map& filter_key_value)
+{
+  std::vector  sections;
+  int ret = g_conf->get_all_sections(sections);
+  if (ret)
+    return 2;
+  for (std::vector::const_iterator p = sections.begin();
+       p != sections.end(); ++p) {
+    if (strncmp(prefix.c_str(), p->c_str(), prefix.size()))
+      continue;
+
+    std::vector sec;
+    sec.push_back(*p);
+
+    int r = 0;
+    for (std::list::const_iterator q = filter_key.begin(); q != filter_key.end(); ++q) {
+      string v;
+      r = g_conf->get_val_from_conf_file(sec, q->c_str(), v, false);
+      if (r < 0)
+	break;
+    }
+    if (r < 0)
+      continue;
+
+    for (std::map::const_iterator q = filter_key_value.begin();
+	 q != filter_key_value.end();
+	 ++q) {
+      string v;
+      r = g_conf->get_val_from_conf_file(sec, q->first.c_str(), v, false);
+      if (r < 0 || v != q->second) {
+	r = -1;
+	break;
+      }
+    }
+    if (r < 0)
+      continue;
+    
+    cout << *p << std::endl;
+  }
+  return 0;
+}
+
+static int lookup(const std::deque §ions,
+		  const std::string &key, bool resolve_search)
+{
+  std::vector  my_sections;
+  for (deque::const_iterator s = sections.begin(); s != sections.end(); ++s) {
+    my_sections.push_back(*s);
+  }
+  g_conf->get_my_sections(my_sections);
+  std::string val;
+  int ret = g_conf->get_val_from_conf_file(my_sections, key.c_str(), val, true);
+  if (ret == -ENOENT)
+    return 1;
+  else if (ret == 0) {
+    if (resolve_search) {
+      string result;
+      if (ceph_resolve_file_search(val, result))
+	puts(result.c_str());
+    }
+    else {
+      puts(val.c_str());
+    }
+    return 0;
+  }
+  else {
+    cerr << "error looking up '" << key << "': error " << ret << std::endl;
+    return 2;
+  }
+}
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  deque sections;
+  bool resolve_search = false;
+  std::string action;
+  std::string lookup_key;
+  std::string section_list_prefix;
+  std::list filter_key;
+  std::map filter_key_value;
+
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+  vector orig_args = args;
+
+  global_pre_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_DAEMON,
+		  CINIT_FLAG_NO_DAEMON_ACTIONS);
+
+  // do not common_init_finish(); do not start threads; do not do any of thing
+  // wonky things the daemon whose conf we are examining would do (like initialize
+  // the admin socket).
+  //common_init_finish(g_ceph_context);
+
+  std::string val;
+  for (std::vector::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_witharg(args, i, &val, "-s", "--section", (char*)NULL)) {
+      sections.push_back(val);
+    } else if (ceph_argparse_flag(args, i, "-r", "--resolve_search", (char*)NULL)) {
+      resolve_search = true;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      action = "help";
+    } else if (ceph_argparse_witharg(args, i, &val, "--lookup", (char*)NULL)) {
+      action = "lookup";
+      lookup_key = val;
+    } else if (ceph_argparse_flag(args, i, "-L", "--list_all_sections", (char*)NULL)) {
+      action = "list-sections";
+      section_list_prefix = "";
+    } else if (ceph_argparse_witharg(args, i, &val, "-l", "--list_sections", (char*)NULL)) {
+      action = "list-sections";
+      section_list_prefix = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--filter_key", (char*)NULL)) {
+      filter_key.push_back(val);
+    } else if (ceph_argparse_witharg(args, i, &val, "--filter_key_value", (char*)NULL)) {
+      size_t pos = val.find_first_of('=');
+      if (pos == string::npos) {
+	cerr << "expecting argument like 'key=value' for --filter-key-value (not '" << val << "')" << std::endl;
+	usage();
+	exit(1);
+      } 
+      string key(val, 0, pos);
+      string value(val, pos+1);
+      filter_key_value[key] = value;
+    } else {
+      if (((action == "lookup") || (action == "")) && (lookup_key.empty())) {
+	action = "lookup";
+	lookup_key = *i++;
+      } else {
+	cerr << "unable to parse option: '" << *i << "'" << std::endl;
+	cerr << "args:";
+	for (std::vector::iterator ci = orig_args.begin(); ci != orig_args.end(); ++ci) {
+	  cerr << " '" << *ci << "'";
+	}
+	cerr << std::endl;
+	usage();
+	exit(1);
+      }
+    }
+  }
+
+  if (action == "help") {
+    usage();
+    exit(0);
+  } else if (action == "list-sections") {
+    return list_sections(section_list_prefix, filter_key, filter_key_value);
+  } else if (action == "lookup") {
+    return lookup(sections, lookup_key, resolve_search);
+  } else {
+    cerr << "You must give an action, such as --lookup or --list-all-sections." << std::endl;
+    cerr << "Pass --help for more help." << std::endl;
+    exit(1);
+  }
+}
diff --git a/ceph/src/tools/ceph_filestore_dump.cc b/ceph/src/tools/ceph_filestore_dump.cc
new file mode 100644
index 00000000..87a83873
--- /dev/null
+++ b/ceph/src/tools/ceph_filestore_dump.cc
@@ -0,0 +1,1429 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "global/global_init.h"
+#include "os/ObjectStore.h"
+#include "os/FileStore.h"
+#include "common/perf_counters.h"
+#include "common/errno.h"
+#include "osd/PGLog.h"
+#include "osd/OSD.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+enum {
+    TYPE_NONE = 0,
+    TYPE_PG_BEGIN,
+    TYPE_PG_END,
+    TYPE_OBJECT_BEGIN,
+    TYPE_OBJECT_END,
+    TYPE_DATA,
+    TYPE_ATTRS,
+    TYPE_OMAP_HDR,
+    TYPE_OMAP,
+    TYPE_PG_METADATA,
+    END_OF_TYPES,	//Keep at the end
+};
+
+//#define INTERNAL_TEST
+//#define INTERNAL_TEST2
+
+#ifdef INTERNAL_TEST
+CompatSet get_test_compat_set() {
+  CompatSet::FeatureSet ceph_osd_feature_compat;
+  CompatSet::FeatureSet ceph_osd_feature_ro_compat;
+  CompatSet::FeatureSet ceph_osd_feature_incompat;
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_PGINFO);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_OLOC);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEC);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_CATEGORIES);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_HOBJECTPOOL);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BIGINFO);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBINFO);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBLOG);
+#ifdef INTERNAL_TEST2
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SNAPMAPPER);
+  ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
+#endif
+  return CompatSet(ceph_osd_feature_compat, ceph_osd_feature_ro_compat,
+		   ceph_osd_feature_incompat);
+}
+#endif
+
+typedef uint8_t sectiontype_t;
+typedef uint32_t mymagic_t;
+typedef int64_t mysize_t;
+const ssize_t max_read = 1024 * 1024;
+const uint16_t shortmagic = 0xffce;	//goes into stream as "ceff"
+//endmagic goes into stream as "ceff ffec"
+const mymagic_t endmagic = (0xecff << 16) | shortmagic;
+const int fd_none = INT_MIN;
+
+//The first FIXED_LENGTH bytes are a fixed
+//portion of the export output.  This includes the overall
+//version number, and size of header and footer.
+//THIS STRUCTURE CAN ONLY BE APPENDED TO.  If it needs to expand,
+//the version can be bumped and then anything
+//can be added to the export format.
+struct super_header {
+  static const uint32_t super_magic = (shortmagic << 16) | shortmagic;
+  static const uint32_t super_ver = 2;
+  static const uint32_t FIXED_LENGTH = 16;
+  uint32_t magic;
+  uint32_t version;
+  uint32_t header_size;
+  uint32_t footer_size;
+
+  super_header() : magic(0), version(0), header_size(0), footer_size(0) { }
+  int read_super();
+
+  void encode(bufferlist& bl) const {
+    ::encode(magic, bl);
+    ::encode(version, bl);
+    ::encode(header_size, bl);
+    ::encode(footer_size, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(magic, bl);
+    ::decode(version, bl);
+    ::decode(header_size, bl);
+    ::decode(footer_size, bl);
+  }
+};
+
+struct header {
+  sectiontype_t type;
+  mysize_t size;
+  header(sectiontype_t type, mysize_t size) :
+    type(type), size(size) { }
+  header(): type(0), size(0) { }
+
+  int get_header();
+
+  void encode(bufferlist& bl) const {
+    uint32_t debug_type = (type << 24) | (type << 16) | shortmagic;
+    ENCODE_START(1, 1, bl);
+    ::encode(debug_type, bl);
+    ::encode(size, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    uint32_t debug_type;
+    DECODE_START(1, bl);
+    ::decode(debug_type, bl);
+    type = debug_type >> 24;
+    ::decode(size, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+struct footer {
+  mymagic_t magic;
+  footer() : magic(endmagic) { }
+
+  int get_footer();
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(magic, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(magic, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+struct pg_begin {
+  spg_t pgid;
+  OSDSuperblock superblock;
+
+  pg_begin(spg_t pg, const OSDSuperblock& sb):
+    pgid(pg), superblock(sb) { }
+  pg_begin() { }
+
+  void encode(bufferlist& bl) const {
+    // If superblock doesn't include CEPH_FS_FEATURE_INCOMPAT_SHARDS then
+    // shard will be NO_SHARD for a replicated pool.  This means
+    // that we allow the decode by struct_v 2.
+    ENCODE_START(3, 2, bl);
+    ::encode(pgid.pgid, bl);
+    ::encode(superblock, bl);
+    ::encode(pgid.shard, bl);
+    ENCODE_FINISH(bl);
+  }
+  // NOTE: New super_ver prevents decode from ver 1
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(3, bl);
+    ::decode(pgid.pgid, bl);
+    if (struct_v > 1) {
+      ::decode(superblock, bl);
+    }
+    if (struct_v > 2) {
+      ::decode(pgid.shard, bl);
+    } else {
+      pgid.shard = ghobject_t::NO_SHARD;
+    }
+    DECODE_FINISH(bl);
+  }
+};
+
+struct object_begin {
+  ghobject_t hoid;
+  object_begin(const ghobject_t &hoid): hoid(hoid) { }
+  object_begin() { }
+
+  // If superblock doesn't include CEPH_FS_FEATURE_INCOMPAT_SHARDS then
+  // generation will be NO_GEN, shard_id will be NO_SHARD for a replicated
+  // pool.  This means we will allow the decode by struct_v 1.
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 1, bl);
+    ::encode(hoid.hobj, bl);
+    ::encode(hoid.generation, bl);
+    ::encode(hoid.shard_id, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(2, bl);
+    ::decode(hoid.hobj, bl);
+    if (struct_v > 1) {
+      ::decode(hoid.generation, bl);
+      ::decode(hoid.shard_id, bl);
+    } else {
+      hoid.generation = ghobject_t::NO_GEN;
+      hoid.shard_id = ghobject_t::NO_SHARD;
+    }
+    DECODE_FINISH(bl);
+  }
+};
+
+struct data_section {
+  uint64_t offset;
+  uint64_t len;
+  bufferlist databl;
+  data_section(uint64_t offset, uint64_t len, bufferlist bl):
+     offset(offset), len(len), databl(bl) { }
+  data_section(): offset(0), len(0) { }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(offset, bl);
+    ::encode(len, bl);
+    ::encode(databl, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(offset, bl);
+    ::decode(len, bl);
+    ::decode(databl, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+struct attr_section {
+  map data;
+  attr_section(const map &data) : data(data) { }
+  attr_section() { }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(data, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(data, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+struct omap_hdr_section {
+  bufferlist hdr;
+  omap_hdr_section(bufferlist hdr) : hdr(hdr) { }
+  omap_hdr_section() { }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(hdr, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(hdr, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+struct omap_section {
+  map omap;
+  omap_section(const map &omap) :
+    omap(omap) { }
+  omap_section() { }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(omap, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(omap, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+struct metadata_section {
+  __u8 struct_ver;
+  epoch_t map_epoch;
+  pg_info_t info;
+  pg_log_t log;
+
+  metadata_section(__u8 struct_ver, epoch_t map_epoch, const pg_info_t &info,
+		   const pg_log_t &log)
+    : struct_ver(struct_ver),
+      map_epoch(map_epoch),
+      info(info),
+      log(log) { }
+  metadata_section()
+    : struct_ver(0),
+      map_epoch(0) { }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(struct_ver, bl);
+    ::encode(map_epoch, bl);
+    ::encode(info, bl);
+    ::encode(log, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(struct_ver, bl);
+    ::decode(map_epoch, bl);
+    ::decode(info, bl);
+    ::decode(log, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+hobject_t infos_oid = OSD::make_infos_oid();
+hobject_t biginfo_oid, log_oid;
+
+int file_fd = fd_none;
+bool debug = false;
+super_header sh;
+
+template 
+int write_section(sectiontype_t type, const T& obj, int fd) {
+  bufferlist blhdr, bl, blftr;
+  obj.encode(bl);
+  header hdr(type, bl.length());
+  hdr.encode(blhdr);
+  footer ft;
+  ft.encode(blftr);
+
+  int ret = blhdr.write_fd(fd);
+  if (ret) return ret;
+  ret = bl.write_fd(fd);
+  if (ret) return ret;
+  ret = blftr.write_fd(fd);
+  return ret;
+}
+
+int write_simple(sectiontype_t type, int fd)
+{
+  bufferlist hbl;
+
+  header hdr(type, 0);
+  hdr.encode(hbl);
+  return hbl.write_fd(fd);
+}
+
+static void invalid_path(string &path)
+{
+  cout << "Invalid path to osd store specified: " << path << "\n";
+  exit(1);
+}
+
+int get_log(ObjectStore *fs, coll_t coll, spg_t pgid, const pg_info_t &info,
+   PGLog::IndexedLog &log, pg_missing_t &missing)
+{ 
+  map divergent_priors;
+  try {
+    ostringstream oss;
+    PGLog::read_log(fs, coll, log_oid, info, divergent_priors, log, missing, oss);
+    if (debug && oss.str().size())
+      cerr << oss.str() << std::endl;
+  }
+  catch (const buffer::error &e) {
+    cout << "read_log threw exception error " << e.what() << std::endl;
+    return 1;
+  }
+  return 0;
+}
+
+//Based on RemoveWQ::_process()
+void remove_coll(ObjectStore *store, const coll_t &coll)
+{
+  spg_t pg;
+  coll.is_pg_prefix(pg);
+  OSDriver driver(
+    store,
+    coll_t(),
+    OSD::make_snapmapper_oid());
+  SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
+
+  vector objects;
+  ghobject_t next;
+  int r = 0;
+  int64_t num = 0;
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  cout << "remove_coll " << coll << std::endl;
+  while (!next.is_max()) {
+    r = store->collection_list_partial(coll, next, 200, 300, 0,
+      &objects, &next);
+    if (r < 0)
+      goto out;
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i, ++num) {
+
+      OSDriver::OSTransaction _t(driver.get_transaction(t));
+      cout << "remove " << *i << std::endl;
+      int r = mapper.remove_oid(i->hobj, &_t);
+      if (r != 0 && r != -ENOENT) {
+        assert(0);
+      }
+
+      t->remove(coll, *i);
+      if (num >= 30) {
+        store->apply_transaction(*t);
+        delete t;
+        t = new ObjectStore::Transaction;
+        num = 0;
+      }
+    }
+  }
+  t->remove_collection(coll);
+  store->apply_transaction(*t);
+out:
+  delete t;
+}
+
+//Based on part of OSD::load_pgs()
+int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
+{
+  vector ls;
+  int r = store->list_collections(ls);
+  if (r < 0) {
+    cout << "finish_remove_pgs: failed to list pgs: " << cpp_strerror(-r)
+      << std::endl;
+    return r;
+  }
+
+  for (vector::iterator it = ls.begin();
+       it != ls.end();
+       ++it) {
+    spg_t pgid;
+    snapid_t snap;
+
+    if (it->is_temp(pgid)) {
+      cout << "finish_remove_pgs " << *it << " clearing temp" << std::endl;
+      OSD::recursive_remove_collection(store, *it);
+      continue;
+    }
+
+    if (it->is_pg(pgid, snap)) {
+      continue;
+    }
+
+    uint64_t seq;
+    if (it->is_removal(&seq, &pgid)) {
+      if (seq >= *next_removal_seq)
+	*next_removal_seq = seq + 1;
+      cout << "finish_remove_pgs removing " << *it << ", seq is "
+	       << seq << " pgid is " << pgid << std::endl;
+      remove_coll(store, *it);
+      continue;
+    }
+
+    //cout << "finish_remove_pgs ignoring unrecognized " << *it << std::endl;
+  }
+  return 0;
+}
+
+int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid,
+    uint64_t *next_removal_seq)
+{
+  ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
+
+  if (store->collection_exists(coll_t(r_pgid))) {
+      coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++, r_pgid);
+      cout << "collection rename " << coll_t(r_pgid)
+	   << " to " << to_remove
+        << std::endl;
+      rmt->collection_rename(coll_t(r_pgid), to_remove);
+  } else {
+    delete rmt;
+    return ENOENT;
+  }
+
+  cout << "remove " << coll_t::META_COLL << " " << log_oid.oid << std::endl;
+  rmt->remove(coll_t::META_COLL, log_oid);
+  cout << "remove " << coll_t::META_COLL << " " << biginfo_oid.oid << std::endl;
+  rmt->remove(coll_t::META_COLL, biginfo_oid);
+
+  store->apply_transaction(*rmt);
+
+  return 0;
+}
+
+int header::get_header()
+{
+  bufferlist ebl;
+  bufferlist::iterator ebliter = ebl.begin();
+  ssize_t bytes;
+
+  bytes = ebl.read_fd(file_fd, sh.header_size);
+  if ((size_t)bytes != sh.header_size) {
+    cout << "Unexpected EOF" << std::endl;
+    return EFAULT;
+  }
+
+  decode(ebliter);
+
+  return 0;
+}
+
+int footer::get_footer()
+{
+  bufferlist ebl;
+  bufferlist::iterator ebliter = ebl.begin();
+  ssize_t bytes;
+
+  bytes = ebl.read_fd(file_fd, sh.footer_size);
+  if ((size_t)bytes != sh.footer_size) {
+    cout << "Unexpected EOF" << std::endl;
+    return EFAULT;
+  }
+
+  decode(ebliter);
+
+  if (magic != endmagic) {
+    cout << "Bad footer magic" << std::endl;
+    return EFAULT;
+  }
+
+  return 0;
+}
+
+int write_info(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
+    __u8 struct_ver)
+{
+  //Empty for this
+  interval_set snap_collections; // obsolete
+  map past_intervals;
+  coll_t coll(info.pgid);
+
+  int ret = PG::_write_info(t, epoch,
+    info, coll,
+    past_intervals,
+    snap_collections,
+    infos_oid,
+    struct_ver,
+    true, true);
+  if (ret < 0) ret = -ret;
+  if (ret) cout << "Failed to write info" << std::endl;
+  return ret;
+}
+
+void write_log(ObjectStore::Transaction &t, pg_log_t &log)
+{
+  map divergent_priors;
+  PGLog::write_log(t, log, log_oid, divergent_priors);
+}
+
+int write_pg(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
+    pg_log_t &log, __u8 struct_ver)
+{
+  int ret = write_info(t, epoch, info, struct_ver);
+  if (ret) return ret;
+  write_log(t, log);
+  return 0;
+}
+
+int export_file(ObjectStore *store, coll_t cid, ghobject_t &obj)
+{
+  struct stat st;
+  mysize_t total;
+  footer ft;
+
+  int ret = store->stat(cid, obj, &st);
+  if (ret < 0)
+    return ret;
+
+  if (file_fd != STDOUT_FILENO)
+    cout << "read " << obj << std::endl;
+
+  total = st.st_size;
+  if (debug && file_fd != STDOUT_FILENO)
+    cout << "size=" << total << std::endl;
+
+  object_begin objb(obj);
+  ret = write_section(TYPE_OBJECT_BEGIN, objb, file_fd);
+  if (ret < 0)
+    return ret;
+
+  uint64_t offset = 0;
+  bufferlist rawdatabl, databl;
+  while(total > 0) {
+    rawdatabl.clear();
+    databl.clear();
+    mysize_t len = max_read;
+    if (len > total)
+      len = total;
+
+    ret = store->read(cid, obj, offset, len, rawdatabl);
+    if (ret < 0)
+      return ret;
+    if (ret == 0)
+      return -EINVAL;
+
+    data_section dblock(offset, len, rawdatabl);
+    total -= ret;
+    offset += ret;
+
+    if (debug && file_fd != STDOUT_FILENO)
+      cout << "data section offset=" << offset << " len=" << len << std::endl;
+
+    ret = write_section(TYPE_DATA, dblock, file_fd);
+    if (ret) return ret;
+  }
+
+  //Handle attrs for this object
+  map aset;
+  ret = store->getattrs(cid, obj, aset, false);
+  if (ret) return ret;
+  attr_section as(aset);
+  ret = write_section(TYPE_ATTRS, as, file_fd);
+  if (ret)
+    return ret;
+
+  if (debug && file_fd != STDOUT_FILENO) {
+    cout << "attrs size " << aset.size() << std::endl;
+  }
+
+  //Handle omap information
+  databl.clear();
+  bufferlist hdrbuf;
+  map out;
+  ret = store->omap_get(cid, obj, &hdrbuf, &out);
+  if (ret < 0)
+    return ret;
+
+  omap_hdr_section ohs(hdrbuf);
+  ret = write_section(TYPE_OMAP_HDR, ohs, file_fd);
+  if (ret)
+    return ret;
+
+  if (!out.empty()) {
+    omap_section oms(out);
+    ret = write_section(TYPE_OMAP, oms, file_fd);
+    if (ret)
+      return ret;
+
+    if (debug && file_fd != STDOUT_FILENO)
+      cout << "omap map size " << out.size() << std::endl;
+  }
+
+  ret = write_simple(TYPE_OBJECT_END, file_fd);
+  if (ret)
+    return ret;
+
+  return 0;
+}
+
+int export_files(ObjectStore *store, coll_t coll)
+{
+  vector objects;
+  ghobject_t next;
+
+  while (!next.is_max()) {
+    int r = store->collection_list_partial(coll, next, 200, 300, 0,
+      &objects, &next);
+    if (r < 0)
+      return r;
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i) {
+      r = export_file(store, coll, *i);
+      if (r < 0)
+        return r;
+    }
+  }
+  return 0;
+}
+
+//Write super_header with its fixed 16 byte length
+void write_super()
+{
+  bufferlist superbl;
+  super_header sh;
+  footer ft;
+
+  header hdr(TYPE_NONE, 0);
+  hdr.encode(superbl);
+
+  sh.magic = super_header::super_magic;
+  sh.version = super_header::super_ver;
+  sh.header_size = superbl.length();
+  superbl.clear();
+  ft.encode(superbl);
+  sh.footer_size = superbl.length();
+  superbl.clear();
+
+  sh.encode(superbl);
+  assert(super_header::FIXED_LENGTH == superbl.length());
+  superbl.write_fd(file_fd);
+}
+
+int do_export(ObjectStore *fs, coll_t coll, spg_t pgid, pg_info_t &info,
+    epoch_t map_epoch, __u8 struct_ver, const OSDSuperblock& superblock)
+{
+  PGLog::IndexedLog log;
+  pg_missing_t missing;
+
+  if (file_fd != STDOUT_FILENO)
+    cout << "Exporting " << pgid << std::endl;
+
+  int ret = get_log(fs, coll, pgid, info, log, missing);
+  if (ret > 0)
+      return ret;
+
+  write_super();
+
+  pg_begin pgb(pgid, superblock);
+  ret = write_section(TYPE_PG_BEGIN, pgb, file_fd);
+  if (ret)
+    return ret;
+
+  ret = export_files(fs, coll);
+  if (ret) {
+    if (file_fd != STDOUT_FILENO)
+      cout << "export_files error " << ret << std::endl;
+    return ret;
+  }
+
+  metadata_section ms(struct_ver, map_epoch, info, log);
+  ret = write_section(TYPE_PG_METADATA, ms, file_fd);
+  if (ret)
+    return ret;
+
+  ret = write_simple(TYPE_PG_END, file_fd);
+  if (ret)
+    return ret;
+
+  return 0;
+}
+
+int super_header::read_super()
+{
+  bufferlist ebl;
+  bufferlist::iterator ebliter = ebl.begin();
+  ssize_t bytes;
+
+  bytes = ebl.read_fd(file_fd, super_header::FIXED_LENGTH);
+  if ((size_t)bytes != super_header::FIXED_LENGTH) {
+    cout << "Unexpected EOF" << std::endl;
+    return EFAULT;
+  }
+
+  decode(ebliter);
+
+  return 0;
+}
+
+int read_section(int fd, sectiontype_t *type, bufferlist *bl)
+{
+  header hdr;
+  ssize_t bytes;
+
+  int ret = hdr.get_header();
+  if (ret)
+    return ret;
+
+  *type = hdr.type;
+
+  bl->clear();
+  bytes = bl->read_fd(fd, hdr.size);
+  if (bytes != hdr.size) {
+    cout << "Unexpected EOF" << std::endl;
+    return EFAULT;
+  }
+
+  if (hdr.size > 0) {
+    footer ft;
+    ret = ft.get_footer();
+    if (ret)
+      return ret;
+  }
+
+  return 0;
+}
+
+int get_data(ObjectStore *store, coll_t coll, ghobject_t hoid,
+    ObjectStore::Transaction *t, bufferlist &bl)
+{
+  bufferlist::iterator ebliter = bl.begin();
+  data_section ds;
+  ds.decode(ebliter);
+
+  if (debug)
+    cout << "\tdata: offset " << ds.offset << " len " << ds.len << std::endl;
+  t->write(coll, hoid, ds.offset, ds.len,  ds.databl);
+  return 0;
+}
+
+int get_attrs(ObjectStore *store, coll_t coll, ghobject_t hoid,
+    ObjectStore::Transaction *t, bufferlist &bl,
+    OSDriver &driver, SnapMapper &snap_mapper)
+{
+  bufferlist::iterator ebliter = bl.begin();
+  attr_section as;
+  as.decode(ebliter);
+
+  if (debug)
+    cout << "\tattrs: len " << as.data.size() << std::endl;
+  t->setattrs(coll, hoid, as.data);
+
+  if (hoid.hobj.snap < CEPH_MAXSNAP && hoid.generation == ghobject_t::NO_GEN) {
+    map::iterator mi = as.data.find(OI_ATTR);
+    if (mi != as.data.end()) {
+      bufferlist attr_bl;
+      attr_bl.push_back(mi->second);
+      object_info_t oi(attr_bl);
+  
+      if (debug)
+        cout << "object_info " << oi << std::endl;
+  
+      OSDriver::OSTransaction _t(driver.get_transaction(t));
+      set oi_snaps(oi.snaps.begin(), oi.snaps.end());
+      snap_mapper.add_oid(hoid.hobj, oi_snaps, &_t);
+    }
+  }
+
+  return 0;
+}
+
+int get_omap_hdr(ObjectStore *store, coll_t coll, ghobject_t hoid,
+    ObjectStore::Transaction *t, bufferlist &bl)
+{
+  bufferlist::iterator ebliter = bl.begin();
+  omap_hdr_section oh;
+  oh.decode(ebliter);
+
+  if (debug)
+    cout << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length())
+      << std::endl;
+  t->omap_setheader(coll, hoid, oh.hdr);
+  return 0;
+}
+
+int get_omap(ObjectStore *store, coll_t coll, ghobject_t hoid,
+    ObjectStore::Transaction *t, bufferlist &bl)
+{
+  bufferlist::iterator ebliter = bl.begin();
+  omap_section os;
+  os.decode(ebliter);
+
+  if (debug)
+    cout << "\tomap: size " << os.omap.size() << std::endl;
+  t->omap_setkeys(coll, hoid, os.omap);
+  return 0;
+}
+
+int get_object(ObjectStore *store, coll_t coll, bufferlist &bl)
+{
+  ObjectStore::Transaction tran;
+  ObjectStore::Transaction *t = &tran;
+  bufferlist::iterator ebliter = bl.begin();
+  object_begin ob;
+  ob.decode(ebliter);
+  OSDriver driver(
+    store,
+    coll_t(),
+    OSD::make_snapmapper_oid());
+  spg_t pg;
+  coll.is_pg_prefix(pg);
+  SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
+
+  t->touch(coll, ob.hoid);
+
+  cout << "Write " << ob.hoid << std::endl;
+
+  bufferlist ebl;
+  bool done = false;
+  while(!done) {
+    sectiontype_t type;
+    int ret = read_section(file_fd, &type, &ebl);
+    if (ret)
+      return ret;
+
+    //cout << "\tdo_object: Section type " << hex << type << dec << std::endl;
+    //cout << "\t\tsection size " << ebl.length() << std::endl;
+    if (type >= END_OF_TYPES) {
+      cout << "Skipping unknown object section type" << std::endl;
+      continue;
+    }
+    switch(type) {
+    case TYPE_DATA:
+      ret = get_data(store, coll, ob.hoid, t, ebl);
+      if (ret) return ret;
+      break;
+    case TYPE_ATTRS:
+      ret = get_attrs(store, coll, ob.hoid, t, ebl, driver, mapper);
+      if (ret) return ret;
+      break;
+    case TYPE_OMAP_HDR:
+      ret = get_omap_hdr(store, coll, ob.hoid, t, ebl);
+      if (ret) return ret;
+      break;
+    case TYPE_OMAP:
+      ret = get_omap(store, coll, ob.hoid, t, ebl);
+      if (ret) return ret;
+      break;
+    case TYPE_OBJECT_END:
+      done = true;
+      break;
+    default:
+      return EFAULT;
+    }
+  }
+  store->apply_transaction(*t);
+  return 0;
+}
+
+int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl)
+{
+  ObjectStore::Transaction tran;
+  ObjectStore::Transaction *t = &tran;
+  bufferlist::iterator ebliter = bl.begin();
+  metadata_section ms;
+  ms.decode(ebliter);
+
+#if DIAGNOSTIC
+  Formatter *formatter = new JSONFormatter(true);
+  cout << "struct_v " << (int)ms.struct_ver << std::endl;
+  cout << "epoch " << ms.map_epoch << std::endl;
+  formatter->open_object_section("info");
+  ms.info.dump(formatter);
+  formatter->close_section();
+  formatter->flush(cout);
+  cout << std::endl;
+  
+  formatter->open_object_section("log");
+  ms.log.dump(formatter);
+  formatter->close_section();
+  formatter->flush(cout);
+  cout << std::endl;
+#endif
+
+  coll_t newcoll(ms.info.pgid);
+  t->collection_rename(coll, newcoll);
+
+  int ret = write_pg(*t, ms.map_epoch, ms.info, ms.log, ms.struct_ver);
+  if (ret) return ret;
+
+  store->apply_transaction(*t);
+
+  return 0;
+}
+
+int do_import(ObjectStore *store, OSDSuperblock& sb)
+{
+  bufferlist ebl;
+  pg_info_t info;
+  PGLog::IndexedLog log;
+
+  uint64_t next_removal_seq = 0;	//My local seq
+  finish_remove_pgs(store, &next_removal_seq);
+
+  int ret = sh.read_super();
+  if (ret)
+    return ret;
+
+  if (sh.magic != super_header::super_magic) {
+    cout << "Invalid magic number" << std::endl;
+    return EFAULT;
+  }
+
+  if (sh.version > super_header::super_ver) {
+    cout << "Can't handle export format version=" << sh.version << std::endl;
+    return EINVAL;
+  }
+
+  //First section must be TYPE_PG_BEGIN
+  sectiontype_t type;
+  ret = read_section(file_fd, &type, &ebl);
+  if (ret)
+    return ret;
+  if (type != TYPE_PG_BEGIN) {
+    return EFAULT;
+  }
+
+  bufferlist::iterator ebliter = ebl.begin();
+  pg_begin pgb;
+  pgb.decode(ebliter);
+  spg_t pgid = pgb.pgid;
+
+  if (debug) {
+    cout << "Exported features: " << pgb.superblock.compat_features << std::endl;
+  }
+  if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) {
+    cout << "Export has incompatible features set "
+      << pgb.superblock.compat_features << std::endl;
+    return 1;
+  }
+
+  log_oid = OSD::make_pg_log_oid(pgid);
+  biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
+
+  //Check for PG already present.
+  coll_t coll(pgid);
+  if (store->collection_exists(coll)) {
+    cout << "pgid " << pgid << " already exists" << std::endl;
+    return 1;
+  }
+
+  //Switch to collection which will be removed automatically if
+  //this program is interupted.
+  coll_t rmcoll = coll_t::make_removal_coll(next_removal_seq, pgid);
+  ObjectStore::Transaction *t = new ObjectStore::Transaction;
+  t->create_collection(rmcoll);
+  store->apply_transaction(*t);
+  delete t;
+
+  cout << "Importing pgid " << pgid << std::endl;
+
+  bool done = false;
+  bool found_metadata = false;
+  while(!done) {
+    ret = read_section(file_fd, &type, &ebl);
+    if (ret)
+      return ret;
+
+    //cout << "do_import: Section type " << hex << type << dec << std::endl;
+    if (type >= END_OF_TYPES) {
+      cout << "Skipping unknown section type" << std::endl;
+      continue;
+    }
+    switch(type) {
+    case TYPE_OBJECT_BEGIN:
+      ret = get_object(store, rmcoll, ebl);
+      if (ret) return ret;
+      break;
+    case TYPE_PG_METADATA:
+      ret = get_pg_metadata(store, rmcoll, ebl);
+      if (ret) return ret;
+      found_metadata = true;
+      break;
+    case TYPE_PG_END:
+      done = true;
+      break;
+    default:
+      return EFAULT;
+    }
+  }
+
+  if (!found_metadata) {
+    cout << "Missing metadata section" << std::endl;
+    return EFAULT;
+  }
+
+  return 0;
+}
+
+int main(int argc, char **argv)
+{
+  string fspath, jpath, pgidstr, type, file;
+  Formatter *formatter = new JSONFormatter(true);
+
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("filestore-path", po::value(&fspath),
+     "path to filestore directory, mandatory")
+    ("journal-path", po::value(&jpath),
+     "path to journal, mandatory")
+    ("pgid", po::value(&pgidstr),
+     "PG id, mandatory except for import")
+    ("type", po::value(&type),
+     "Arg is one of [info, log, remove, export, or import], mandatory")
+    ("file", po::value(&file),
+     "path of file to export or import")
+    ("debug", "Enable diagnostic output to stderr")
+    ;
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+   po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+  po::store( parsed, vm);
+  try {
+    po::notify(vm);
+  }
+  catch(...) {
+    cout << desc << std::endl;
+    exit(1);
+  }
+     
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  if (!vm.count("filestore-path")) {
+    cout << "Must provide filestore-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  } 
+  if (!vm.count("journal-path")) {
+    cout << "Must provide journal-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  } 
+  if (!vm.count("type")) {
+    cout << "Must provide type (info, log, remove, export, import)"
+      << std::endl << desc << std::endl;
+    return 1;
+  } 
+  if (type != "import" && !vm.count("pgid")) {
+    cout << "Must provide pgid" << std::endl
+	 << desc << std::endl;
+    return 1;
+  } 
+
+  file_fd = fd_none;
+  if (type == "export") {
+    if (!vm.count("file")) {
+      file_fd = STDOUT_FILENO;
+    } else {
+      file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
+    }
+  } else if (type == "import") {
+    if (!vm.count("file")) {
+      file_fd = STDIN_FILENO;
+    } else {
+      file_fd = open(file.c_str(), O_RDONLY);
+    }
+  }
+
+  if (vm.count("file") && file_fd == fd_none) {
+    cout << "--file option only applies to import or export" << std::endl;
+    return 1;
+  }
+
+  if (file_fd != fd_none && file_fd < 0) {
+    perror("open");
+    return 1;
+  }
+  
+  if ((fspath.length() == 0 || jpath.length() == 0) ||
+      (type != "info" && type != "log" && type != "remove" && type != "export"
+        && type != "import") ||
+      (type != "import" && pgidstr.length() == 0)) {
+    cerr << "Invalid params" << std::endl;
+    exit(1);
+  }
+
+  if (type == "import" && pgidstr.length()) {
+    cerr << "--pgid option invalid with import" << std::endl;
+    exit(1);
+  }
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  //Suppress derr() output to stderr by default
+  if (!vm.count("debug")) {
+    close(STDERR_FILENO);
+    (void)open("/dev/null", O_WRONLY);
+    debug = false;
+  } else {
+    debug = true;
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
+    CODE_ENVIRONMENT_UTILITY, 0);
+    //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+  g_conf = g_ceph_context->_conf;
+
+  //Verify that fspath really is an osd store
+  struct stat st;
+  if (::stat(fspath.c_str(), &st) == -1) {
+     perror("fspath");
+     invalid_path(fspath);
+  }
+  if (!S_ISDIR(st.st_mode)) {
+    invalid_path(fspath);
+  }
+  string check = fspath + "/whoami";
+  if (::stat(check.c_str(), &st) == -1) {
+     perror("whoami");
+     invalid_path(fspath);
+  }
+  if (!S_ISREG(st.st_mode)) {
+    invalid_path(fspath);
+  }
+  check = fspath + "/current";
+  if (::stat(check.c_str(), &st) == -1) {
+     perror("current");
+     invalid_path(fspath);
+  }
+  if (!S_ISDIR(st.st_mode)) {
+    invalid_path(fspath);
+  }
+
+  spg_t pgid;
+  if (pgidstr.length() && !pgid.parse(pgidstr.c_str())) {
+    cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
+    exit(1);
+  }
+
+  ObjectStore *fs = new FileStore(fspath, jpath);
+  
+  int r = fs->mount();
+  if (r < 0) {
+    if (r == -EBUSY) {
+      cout << "OSD has the store locked" << std::endl;
+    } else {
+      cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl;
+    }
+    return 1;
+  }
+
+  bool fs_sharded_objects = fs->get_allow_sharded_objects();
+
+  int ret = 0;
+  vector ls;
+  vector::iterator it;
+  CompatSet supported;
+
+#ifdef INTERNAL_TEST
+  supported = get_test_compat_set();
+#else
+  supported = OSD::get_osd_compat_set();
+#endif
+
+  bufferlist bl;
+  OSDSuperblock superblock;
+  bufferlist::iterator p;
+  r = fs->read(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, 0, bl);
+  if (r < 0) {
+    cout << "Failure to read OSD superblock error= " << r << std::endl;
+    goto out;
+  }
+
+  p = bl.begin();
+  ::decode(superblock, p);
+
+#ifdef INTERNAL_TEST2
+  fs->set_allow_sharded_objects();
+  assert(fs->get_allow_sharded_objects());
+  fs_sharded_objects = true;
+  superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
+#endif
+
+  if (debug && file_fd != STDOUT_FILENO) {
+    cout << "Supported features: " << supported << std::endl;
+    cout << "On-disk features: " << superblock.compat_features << std::endl;
+  }
+  if (supported.compare(superblock.compat_features) == -1) {
+    cout << "On-disk OSD incompatible features set "
+      << superblock.compat_features << std::endl;
+    ret = EINVAL;
+    goto out;
+  }
+
+  // If there was a crash as an OSD was transitioning to sharded objects
+  // and hadn't completed a set_allow_sharded_objects().
+  // This utility does not want to attempt to finish that transition.
+  if (superblock.compat_features.incompat.contains(CEPH_OSD_FEATURE_INCOMPAT_SHARDS) != fs_sharded_objects) {
+    // An OSD should never have call set_allow_sharded_objects() before
+    // updating its own OSD features.
+    if (fs_sharded_objects)
+      cout << "FileStore sharded but OSD not set, Corruption?" << std::endl;
+    else
+      cout << "Found incomplete transition to sharded objects" << std::endl;
+    ret = EINVAL;
+    goto out;
+  }
+
+  if (type == "import") {
+
+    try {
+      ret = do_import(fs, superblock);
+    }
+    catch (const buffer::error &e) {
+      cout << "do_import threw exception error " << e.what() << std::endl;
+      ret = EFAULT;
+    }
+    if (ret == EFAULT) {
+      cout << "Corrupt input for import" << std::endl;
+    }
+    if (ret == 0)
+      cout << "Import successful" << std::endl;
+    goto out;
+  }
+
+  log_oid = OSD::make_pg_log_oid(pgid);
+  biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
+
+  if (type == "remove") {
+    uint64_t next_removal_seq = 0;	//My local seq
+    finish_remove_pgs(fs, &next_removal_seq);
+    int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq);
+    if (r) {
+      cout << "PG '" << pgid << "' not found" << std::endl;
+      ret = 1;
+      goto out;
+    }
+    finish_remove_pgs(fs, &next_removal_seq);
+    cout << "Remove successful" << std::endl;
+    goto out;
+  }
+
+  r = fs->list_collections(ls);
+  if (r < 0) {
+    cout << "failed to list pgs: " << cpp_strerror(-r) << std::endl;
+    exit(1);
+  }
+
+  for (it = ls.begin(); it != ls.end(); ++it) {
+    snapid_t snap;
+    spg_t tmppgid;
+
+    if (!it->is_pg(tmppgid, snap)) {
+      continue;
+    }
+
+    if (tmppgid != pgid) {
+      continue;
+    }
+    if (snap != CEPH_NOSNAP && debug) {
+      cerr << "skipping snapped dir " << *it
+	       << " (pg " << pgid << " snap " << snap << ")" << std::endl;
+      continue;
+    }
+
+    //Found!
+    break;
+  }
+
+  epoch_t map_epoch;
+  if (it != ls.end()) {
+  
+    coll_t coll = *it;
+  
+    bufferlist bl;
+    map_epoch = PG::peek_map_epoch(fs, coll, infos_oid, &bl);
+    if (debug)
+      cerr << "map_epoch " << map_epoch << std::endl;
+
+    pg_info_t info(pgid);
+    map past_intervals;
+    hobject_t biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
+    interval_set snap_collections;
+  
+    __u8 struct_ver;
+    r = PG::read_info(fs, coll, bl, info, past_intervals, biginfo_oid,
+      infos_oid, snap_collections, struct_ver);
+    if (r < 0) {
+      cout << "read_info error " << cpp_strerror(-r) << std::endl;
+      ret = 1;
+      goto out;
+    }
+    if (debug)
+      cerr << "struct_v " << (int)struct_ver << std::endl;
+
+    if (type == "export") {
+      ret = do_export(fs, coll, pgid, info, map_epoch, struct_ver, superblock);
+      if (ret == 0 && file_fd != STDOUT_FILENO)
+        cout << "Export successful" << std::endl;
+    } else if (type == "info") {
+      formatter->open_object_section("info");
+      info.dump(formatter);
+      formatter->close_section();
+      formatter->flush(cout);
+      cout << std::endl;
+    } else if (type == "log") {
+      PGLog::IndexedLog log;
+      pg_missing_t missing;
+      ret = get_log(fs, coll, pgid, info, log, missing);
+      if (ret > 0)
+          goto out;
+  
+      formatter->open_object_section("log");
+      log.dump(formatter);
+      formatter->close_section();
+      formatter->flush(cout);
+      cout << std::endl;
+      formatter->open_object_section("missing");
+      missing.dump(formatter);
+      formatter->close_section();
+      formatter->flush(cout);
+      cout << std::endl;
+    }
+  } else {
+    cout << "PG '" << pgid << "' not found" << std::endl;
+    ret = 1;
+  }
+
+out:
+  if (fs->umount() < 0) {
+    cout << "umount failed" << std::endl;
+    return 1;
+  }
+
+  return (ret != 0);
+}
+
diff --git a/ceph/src/tools/ceph_filestore_tool.cc b/ceph/src/tools/ceph_filestore_tool.cc
new file mode 100644
index 00000000..eb9f8dac
--- /dev/null
+++ b/ceph/src/tools/ceph_filestore_tool.cc
@@ -0,0 +1,260 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "common/Formatter.h"
+
+#include "global/global_init.h"
+#include "os/ObjectStore.h"
+#include "os/FileStore.h"
+#include "common/perf_counters.h"
+#include "common/errno.h"
+#include "osd/PGLog.h"
+#include "osd/osd_types.h"
+#include "osd/OSD.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+static void invalid_path(string &path)
+{
+  cout << "Invalid path to osd store specified: " << path << "\n";
+  exit(1);
+}
+
+int main(int argc, char **argv)
+{
+  string fspath, jpath, pgidstr;
+  bool list_lost_objects = false;
+  bool fix_lost_objects = false;
+  unsigned LIST_AT_A_TIME = 100;
+  unsigned scanned = 0;
+  
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("filestore-path", po::value(&fspath),
+     "path to filestore directory, mandatory")
+    ("journal-path", po::value(&jpath),
+     "path to journal, mandatory")
+    ("pgid", po::value(&pgidstr),
+     "PG id")
+    ("list-lost-objects", po::value(
+      &list_lost_objects)->default_value(false),
+     "list lost objects")
+    ("fix-lost-objects", po::value(
+      &fix_lost_objects)->default_value(false),
+     "fix lost objects")
+    ;
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+   po::command_line_parser(argc, argv).options(desc).
+    allow_unregistered().run();
+  po::store( parsed, vm);
+  try {
+    po::notify(vm);
+  }
+  catch(...) {
+    cout << desc << std::endl;
+    exit(1);
+  }
+     
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  if (!vm.count("filestore-path")) {
+    cerr << "Must provide filestore-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  } 
+  if (!vm.count("journal-path")) {
+    cerr << "Must provide journal-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  } 
+
+  if ((fspath.length() == 0 || jpath.length() == 0)) {
+    cerr << "Invalid params" << desc << std::endl;
+    exit(1);
+  }
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
+    CODE_ENVIRONMENT_UTILITY, 0);
+    //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+  g_conf = g_ceph_context->_conf;
+
+  //Verify that fspath really is an osd store
+  struct stat st;
+  if (::stat(fspath.c_str(), &st) == -1) {
+     perror("fspath");
+     invalid_path(fspath);
+  }
+  if (!S_ISDIR(st.st_mode)) {
+    invalid_path(fspath);
+  }
+  string check = fspath + "/whoami";
+  if (::stat(check.c_str(), &st) == -1) {
+     perror("whoami");
+     invalid_path(fspath);
+  }
+  if (!S_ISREG(st.st_mode)) {
+    invalid_path(fspath);
+  }
+  check = fspath + "/current";
+  if (::stat(check.c_str(), &st) == -1) {
+     perror("current");
+     invalid_path(fspath);
+  }
+  if (!S_ISDIR(st.st_mode)) {
+    invalid_path(fspath);
+  }
+
+  ObjectStore *fs = new FileStore(fspath, jpath);
+
+  int r = fs->mount();
+  if (r < 0) {
+    if (r == -EBUSY) {
+      cout << "OSD has the store locked" << std::endl;
+    } else {
+      cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl;
+    }
+    return 1;
+  }
+
+  vector colls_to_check;
+  if (pgidstr.length()) {
+    spg_t pgid;
+    if (!pgid.parse(pgidstr.c_str())) {
+      cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
+      exit(1);
+    }
+    colls_to_check.push_back(coll_t(pgid));
+  } else {
+    vector candidates;
+    r = fs->list_collections(candidates);
+    if (r < 0) {
+      cerr << "Error listing collections: " << cpp_strerror(r) << std::endl;
+      goto UMOUNT;
+    }
+    for (vector::iterator i = candidates.begin();
+	 i != candidates.end();
+	 ++i) {
+      spg_t pgid;
+      snapid_t snap;
+      if (i->is_pg(pgid, snap)) {
+	colls_to_check.push_back(*i);
+      }
+    }
+  }
+
+  cerr << colls_to_check.size() << " pgs to scan" << std::endl;
+  for (vector::iterator i = colls_to_check.begin();
+       i != colls_to_check.end();
+       ++i, ++scanned) {
+    cerr << "Scanning " << *i << ", " << scanned << "/"
+	 << colls_to_check.size() << " completed" << std::endl;
+    ghobject_t next;
+    while (!next.is_max()) {
+      vector list;
+      r = fs->collection_list_partial(
+	*i,
+	next,
+	LIST_AT_A_TIME,
+	LIST_AT_A_TIME,
+	CEPH_NOSNAP,
+	&list,
+	&next);
+      if (r < 0) {
+	cerr << "Error listing collection: " << *i << ", "
+	     << cpp_strerror(r) << std::endl;
+	goto UMOUNT;
+      }
+      for (vector::iterator obj = list.begin();
+	   obj != list.end();
+	   ++obj) {
+	bufferlist attr;
+	r = fs->getattr(*i, *obj, OI_ATTR, attr);
+	if (r < 0) {
+	  cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+	       << cpp_strerror(r) << std::endl;
+	  goto UMOUNT;
+	}
+	object_info_t oi;
+	bufferlist::iterator bp = attr.begin();
+	try {
+	  ::decode(oi, bp);
+	} catch (...) {
+	  r = -EINVAL;
+	  cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+	       << cpp_strerror(r) << std::endl;
+	  goto UMOUNT;
+	}
+	if (oi.is_lost()) {
+	  if (list_lost_objects) {
+	    cout << *i << "/" << *obj << " is lost" << std::endl;
+	  }
+	  if (fix_lost_objects) {
+	    cerr << *i << "/" << *obj << " is lost, fixing" << std::endl;
+	    oi.clear_flag(object_info_t::FLAG_LOST);
+	    bufferlist bl2;
+	    ::encode(oi, bl2);
+	    ObjectStore::Transaction t;
+	    t.setattr(*i, *obj, OI_ATTR, bl2);
+	    r = fs->apply_transaction(t);
+	    if (r < 0) {
+	      cerr << "Error getting fixing attr on : " << make_pair(*i, *obj)
+		   << ", "
+		   << cpp_strerror(r) << std::endl;
+	      goto UMOUNT;
+	    }
+	  }
+	}
+      }
+    }
+  }
+  cerr << "Completed" << std::endl;
+
+ UMOUNT:
+  fs->sync_and_flush();
+  fs->umount();
+  return r;
+}
diff --git a/ceph/src/tools/ceph_kvstore_tool.cc b/ceph/src/tools/ceph_kvstore_tool.cc
new file mode 100644
index 00000000..5a68d989
--- /dev/null
+++ b/ceph/src/tools/ceph_kvstore_tool.cc
@@ -0,0 +1,424 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "os/LevelDBStore.h"
+
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "common/config.h"
+#include "common/strtol.h"
+#include "include/stringify.h"
+
+using namespace std;
+
+class StoreTool
+{
+  boost::scoped_ptr db;
+  string store_path;
+
+  public:
+  StoreTool(const string &path) : store_path(path) {
+    LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path);
+    assert(!db_ptr->open(std::cerr));
+    db.reset(db_ptr);
+  }
+
+  uint32_t traverse(const string &prefix,
+                    const bool do_crc,
+                    ostream *out) {
+    KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
+
+    if (prefix.empty())
+      iter->seek_to_first();
+    else
+      iter->seek_to_first(prefix);
+
+    uint32_t crc = -1;
+
+    while (iter->valid()) {
+      pair rk = iter->raw_key();
+      if (!prefix.empty() && (rk.first != prefix))
+        break;
+
+      if (out)
+        *out << rk.first << ":" << rk.second;
+      if (do_crc) {
+        bufferlist bl;
+        bl.append(rk.first);
+        bl.append(rk.second);
+        bl.append(iter->value());
+
+        crc = bl.crc32c(crc);
+        if (out) {
+          *out << " (" << bl.crc32c(0) << ")";
+        }
+      }
+      if (out)
+        *out << std::endl;
+      iter->next();
+    }
+
+    return crc;
+  }
+
+  void list(const string &prefix, const bool do_crc) {
+    traverse(prefix, do_crc, &std::cout);
+  }
+
+  bool exists(const string &prefix) {
+    assert(!prefix.empty());
+    KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
+    iter->seek_to_first(prefix);
+    return (iter->valid() && (iter->raw_key().first == prefix));
+  }
+
+  bool exists(const string &prefix, const string &key) {
+    assert(!prefix.empty());
+
+    if (key.empty()) {
+      return exists(prefix);
+    }
+
+    bool exists = false;
+    get(prefix, key, exists);
+    return exists;
+  }
+
+  bufferlist get(const string &prefix, const string &key, bool &exists) {
+    assert(!prefix.empty() && !key.empty());
+
+    map result;
+    std::set keys;
+    keys.insert(key);
+    db->get(prefix, keys, &result);
+
+    if (result.count(key) > 0) {
+      exists = true;
+      return result[key];
+    }
+    exists = false;
+    return bufferlist();
+  }
+
+  uint64_t get_size() {
+    map extras;
+    uint64_t s = db->get_estimated_size(extras);
+    for (map::iterator p = extras.begin();
+         p != extras.end(); ++p) {
+      std::cout << p->first << " - " << p->second << std::endl;
+    }
+    std::cout << "total: " << s << std::endl;
+    return s;
+  }
+
+  bool set(const string &prefix, const string &key, bufferlist &val) {
+    assert(!prefix.empty());
+    assert(!key.empty());
+    assert(val.length() > 0);
+
+    KeyValueDB::Transaction tx = db->get_transaction();
+    tx->set(prefix, key, val);
+    int ret = db->submit_transaction_sync(tx);
+
+    return (ret == 0);
+  }
+
+  int copy_store_to(const string &other_path, const int num_keys_per_tx) {
+
+    if (num_keys_per_tx <= 0) {
+      std::cerr << "must specify a number of keys/tx > 0" << std::endl;
+      return -EINVAL;
+    }
+
+    // open or create a leveldb store at @p other_path
+    LevelDBStore other(g_ceph_context, other_path);
+    int err = other.create_and_open(std::cerr);
+    if (err < 0)
+      return err;
+
+    KeyValueDB::WholeSpaceIterator it = db->get_iterator();
+    it->seek_to_first();
+    uint64_t total_keys = 0;
+    uint64_t total_size = 0;
+    uint64_t total_txs = 0;
+
+    utime_t started_at = ceph_clock_now(g_ceph_context);
+
+    do {
+      int num_keys = 0;
+
+      KeyValueDB::Transaction tx = other.get_transaction();
+
+
+      while (it->valid() && num_keys < num_keys_per_tx) {
+        pair k = it->raw_key();
+        bufferlist v = it->value();
+        tx->set(k.first, k.second, v);
+
+        num_keys ++;
+        total_size += v.length();
+
+        it->next();
+      }
+
+      total_txs ++;
+      total_keys += num_keys;
+
+      if (num_keys > 0)
+        other.submit_transaction_sync(tx);
+
+      utime_t cur_duration = ceph_clock_now(g_ceph_context) - started_at;
+      std::cout << "ts = " << cur_duration << "s, copied " << total_keys
+                << " keys so far (" << stringify(si_t(total_size)) << ")"
+                << std::endl;
+
+    } while (it->valid());
+
+    utime_t time_taken = ceph_clock_now(g_ceph_context) - started_at;
+
+    std::cout << "summary:" << std::endl;
+    std::cout << "  copied " << total_keys << " keys" << std::endl;
+    std::cout << "  used " << total_txs << " transactions" << std::endl;
+    std::cout << "  total size " << stringify(si_t(total_size)) << std::endl;
+    std::cout << "  from '" << store_path << "' to '" << other_path << "'"
+              << std::endl;
+    std::cout << "  duration " << time_taken << " seconds" << std::endl;
+
+    return 0;
+  }
+};
+
+void usage(const char *pname)
+{
+  std::cerr << "Usage: " << pname << "  command [args...]\n"
+    << "\n"
+    << "Commands:\n"
+    << "  list [prefix]\n"
+    << "  list-crc [prefix]\n"
+    << "  exists  [key]\n"
+    << "  get   [out ]\n"
+    << "  crc  \n"
+    << "  get-size [ ]\n"
+    << "  set   [ver |in ]\n"
+    << "  store-copy  [num-keys-per-tx]\n"
+    << "  store-crc \n"
+    << std::endl;
+}
+
+int main(int argc, const char *argv[])
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(
+      NULL, args,
+      CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+
+  if (args.size() < 2) {
+    usage(argv[0]);
+    return 1;
+  }
+
+  string path(args[0]);
+  string cmd(args[1]);
+
+  StoreTool st(path);
+
+  if (cmd == "list" || cmd == "list-crc") {
+    string prefix;
+    if (argc > 3)
+      prefix = argv[3];
+
+    bool do_crc = (cmd == "list-crc");
+
+    st.list(prefix, do_crc);
+
+  } else if (cmd == "exists") {
+    string key;
+    if (argc < 4) {
+      usage(argv[0]);
+      return 1;
+    }
+    string prefix(argv[3]);
+    if (argc > 4)
+      key = argv[4];
+
+    bool ret = st.exists(prefix, key);
+    std::cout << "(" << prefix << ", " << key << ") "
+      << (ret ? "exists" : "does not exist")
+      << std::endl;
+    return (ret ? 0 : 1);
+
+  } else if (cmd == "get") {
+    if (argc < 5) {
+      usage(argv[0]);
+      return 1;
+    }
+    string prefix(argv[3]);
+    string key(argv[4]);
+
+    bool exists = false;
+    bufferlist bl = st.get(prefix, key, exists);
+    std::cout << "(" << prefix << ", " << key << ")";
+    if (!exists) {
+      std::cout << " does not exist" << std::endl;
+      return 1;
+    }
+    std::cout << std::endl;
+
+    if (argc >= 6) {
+      string subcmd(argv[5]);
+      string out(argv[6]);
+
+      if (subcmd != "out") {
+        std::cerr << "unrecognized subcmd '" << subcmd << "'"
+                  << std::endl;
+        return 1;
+      }
+
+      if (out.empty()) {
+        std::cerr << "unspecified out file" << std::endl;
+        return 1;
+      }
+
+      int err = bl.write_file(argv[6], 0644);
+      if (err < 0) {
+        std::cerr << "error writing value to '" << out << "': "
+                  << cpp_strerror(err) << std::endl;
+        return 1;
+      }
+    } else {
+      ostringstream os;
+      bl.hexdump(os);
+      std::cout << os.str() << std::endl;
+    }
+
+  } else if (cmd == "crc") {
+    if (argc < 5) {
+      usage(argv[0]);
+      return 1;
+    }
+    string prefix(argv[3]);
+    string key(argv[4]);
+
+    bool exists = false;
+    bufferlist bl = st.get(prefix, key, exists);
+    std::cout << "(" << prefix << ", " << key << ") ";
+    if (!exists) {
+      std::cout << " does not exist" << std::endl;
+      return 1;
+    }
+    std::cout << " crc " << bl.crc32c(0) << std::endl;
+
+  } else if (cmd == "get-size") {
+    std::cout << "estimated store size: " << st.get_size() << std::endl;
+
+    if (argc < 4)
+      return 0;
+
+    if (argc < 5) {
+      usage(argv[0]);
+      return 1;
+    }
+    string prefix(argv[3]);
+    string key(argv[4]);
+
+    bool exists = false;
+    bufferlist bl = st.get(prefix, key, exists);
+    if (!exists) {
+      std::cerr << "(" << prefix << "," << key
+                << ") does not exist" << std::endl;
+      return 1;
+    }
+    std::cout << "(" << prefix << "," << key
+              << ") size " << si_t(bl.length()) << std::endl;
+
+  } else if (cmd == "set") {
+    if (argc < 7) {
+      usage(argv[0]);
+      return 1;
+    }
+    string prefix(argv[3]);
+    string key(argv[4]);
+    string subcmd(argv[5]);
+
+    bufferlist val;
+    string errstr;
+    if (subcmd == "ver") {
+      version_t v = (version_t) strict_strtoll(argv[6], 10, &errstr);
+      if (!errstr.empty()) {
+        std::cerr << "error reading version: " << errstr << std::endl;
+        return 1;
+      }
+      ::encode(v, val);
+    } else if (subcmd == "in") {
+      int ret = val.read_file(argv[6], &errstr);
+      if (ret < 0 || !errstr.empty()) {
+        std::cerr << "error reading file: " << errstr << std::endl;
+        return 1;
+      }
+    } else {
+      std::cerr << "unrecognized subcommand '" << subcmd << "'" << std::endl;
+      usage(argv[0]);
+      return 1;
+    }
+
+    bool ret = st.set(prefix, key, val);
+    if (!ret) {
+      std::cerr << "error setting ("
+                << prefix << "," << key << ")" << std::endl;
+      return 1;
+    }
+  } else if (cmd == "store-copy") {
+    int num_keys_per_tx = 128; // magic number that just feels right.
+    if (argc < 4) {
+      usage(argv[0]);
+      return 1;
+    } else if (argc > 4) {
+      string err;
+      num_keys_per_tx = strict_strtol(argv[4], 10, &err);
+      if (!err.empty()) {
+        std::cerr << "invalid num_keys_per_tx: " << err << std::endl;
+        return 1;
+      }
+    }
+
+    int ret = st.copy_store_to(argv[3], num_keys_per_tx);
+    if (ret < 0) {
+      std::cerr << "error copying store to path '" << argv[3]
+                << "': " << cpp_strerror(ret) << std::endl;
+      return 1;
+    }
+
+  } else if (cmd == "store-crc") {
+    uint32_t crc = st.traverse(string(), true, NULL);
+    std::cout << "store at '" << path << "' crc " << crc << std::endl;
+
+  } else {
+    std::cerr << "Unrecognized command: " << cmd << std::endl;
+    return 1;
+  }
+
+  return 0;
+}
diff --git a/ceph/src/tools/ceph_monstore_tool.cc b/ceph/src/tools/ceph_monstore_tool.cc
new file mode 100644
index 00000000..231d257b
--- /dev/null
+++ b/ceph/src/tools/ceph_monstore_tool.cc
@@ -0,0 +1,420 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "global/global_init.h"
+#include "os/LevelDBStore.h"
+#include "mon/MonitorDBStore.h"
+#include "mon/Paxos.h"
+#include "common/Formatter.h"
+#include "include/stringify.h"
+#include "common/errno.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+class TraceIter {
+  int fd;
+  unsigned idx;
+  MonitorDBStore::Transaction t;
+public:
+  TraceIter(string fname) : fd(-1), idx(-1) {
+    fd = ::open(fname.c_str(), O_RDONLY);
+  }
+  bool valid() {
+    return fd != -1;
+  }
+  const MonitorDBStore::Transaction &cur() {
+    assert(valid());
+    return t;
+  }
+  unsigned num() { return idx; }
+  void next() {
+    ++idx;
+    bufferlist bl;
+    int r = bl.read_fd(fd, 6);
+    if (r < 0) {
+      std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd"
+		<< std::endl;
+      ::close(fd);
+      fd = -1;
+      return;
+    } else if ((unsigned)r < 6) {
+      std::cerr << "short read" << std::endl;
+      ::close(fd);
+      fd = -1;
+      return;
+    }
+    bufferlist::iterator bliter = bl.begin();
+    uint8_t ver, ver2;
+    ::decode(ver, bliter);
+    ::decode(ver2, bliter);
+    uint32_t len;
+    ::decode(len, bliter);
+    r = bl.read_fd(fd, len);
+    if (r < 0) {
+      std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd"
+		<< std::endl;
+      ::close(fd);
+      fd = -1;
+      return;
+    } else if ((unsigned)r < len) {
+      std::cerr << "short read" << std::endl;
+      ::close(fd);
+      fd = -1;
+      return;
+    }
+    bliter = bl.begin();
+    t.decode(bliter);
+  }
+  void init() {
+    next();
+  }
+  ~TraceIter() {
+    if (fd != -1) {
+      ::close(fd);
+      fd = -1;
+    }
+  }
+};
+
+int main(int argc, char **argv) {
+  po::options_description desc("Allowed options");
+  int version = -1;
+  string store_path, cmd, out_path, tfile;
+  unsigned dstart = 0;
+  unsigned dstop = ~0;
+  unsigned num_replays = 1;
+  unsigned tsize = 200;
+  unsigned tvalsize = 1024;
+  unsigned ntrans = 100;
+  desc.add_options()
+    ("help", "produce help message")
+    ("mon-store-path", po::value(&store_path),
+     "path to mon directory, mandatory")
+    ("out", po::value(&out_path),
+     "out path")
+    ("version", po::value(&version),
+     "version requested")
+    ("trace-file", po::value(&tfile),
+     "trace file")
+    ("dump-start", po::value(&dstart),
+     "transaction num to start dumping at")
+    ("dump-end", po::value(&dstop),
+     "transaction num to stop dumping at")
+    ("num-replays", po::value(&num_replays),
+     "number of times to replay")
+    ("trans-size", po::value(&tsize),
+     "keys to write in each transaction")
+    ("trans-val-size", po::value(&tvalsize),
+     "val to write in each key")
+    ("num-trans", po::value(&ntrans),
+     "number of transactions to run")
+    ("command", po::value(&cmd),
+     "command")
+    ;
+  po::positional_options_description p;
+  p.add("command", 1);
+  p.add("version", 1);
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+    po::command_line_parser(argc, argv).options(desc).positional(p).run();
+  po::store(
+    parsed,
+    vm);
+  try {
+    po::notify(vm);
+  } catch (...) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_MON,
+    CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+  g_conf = g_ceph_context->_conf;
+
+  if (vm.count("help")) {
+    std::cerr << desc << std::endl;
+    return 1;
+  }
+
+  int fd;
+  if (vm.count("out")) {
+    if ((fd = open(out_path.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) {
+      int _err = errno;
+      if (_err != EISDIR) {
+        std::cerr << "Couldn't open " << out_path << ": " << cpp_strerror(_err) << std::endl; 
+        return 1;
+      }
+    }
+  } else {
+    fd = STDOUT_FILENO;
+  }
+
+  if (fd < 0 && cmd != "store-copy") {
+    std::cerr << "error: '" << out_path << "' is a directory!" << std::endl;
+    return 1;
+  }
+
+  MonitorDBStore st(store_path);
+  if (store_path.size()) {
+    stringstream ss;
+    int r = st.open(ss);
+    if (r < 0) {
+      std::cerr << ss.str() << std::endl;
+      goto done;
+    }
+  }
+  if (cmd == "dump-keys") {
+    KeyValueDB::WholeSpaceIterator iter = st.get_iterator();
+    while (iter->valid()) {
+      pair key(iter->raw_key());
+      cout << key.first << " / " << key.second << std::endl;
+      iter->next();
+    }
+  } else if (cmd == "compact") {
+    st.compact();
+  } else if (cmd == "getmonmap") {
+    assert(fd >= 0);
+    if (!store_path.size()) {
+      std::cerr << "need mon store path" << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    version_t v;
+    if (version <= 0) {
+      v = st.get("monmap", "last_committed");
+    } else {
+      v = version;
+    }
+
+    bufferlist bl;
+    /// XXX: this is not ok, osdmap and full should be abstracted somewhere
+    int r = st.get("monmap", v, bl);
+    if (r < 0) {
+      std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl;
+      goto done;
+    }
+    bl.write_fd(fd);
+  } else if (cmd == "getosdmap") {
+    if (!store_path.size()) {
+      std::cerr << "need mon store path" << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    version_t v;
+    if (version == -1) {
+      v = st.get("osdmap", "last_committed");
+    } else {
+      v = version;
+    }
+
+    bufferlist bl;
+    /// XXX: this is not ok, osdmap and full should be abstracted somewhere
+    int r = st.get("osdmap", st.combine_strings("full", v), bl);
+    if (r < 0) {
+      std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl;
+      goto done;
+    }
+    bl.write_fd(fd);
+  } else if (cmd == "dump-paxos") {
+    for (version_t v = dstart; v <= dstop; ++v) {
+      bufferlist bl;
+      st.get("paxos", v, bl);
+      if (bl.length() == 0)
+	break;
+      cout << "\n--- " << v << " ---" << std::endl;
+      MonitorDBStore::Transaction tx;
+      Paxos::decode_append_transaction(tx, bl);
+      JSONFormatter f(true);
+      tx.dump(&f);
+      f.flush(cout);
+    }
+  } else if (cmd == "dump-trace") {
+    if (tfile.empty()) {
+      std::cerr << "Need trace_file" << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    TraceIter iter(tfile.c_str());
+    iter.init();
+    while (true) {
+      if (!iter.valid())
+	break;
+      if (iter.num() >= dstop) {
+	break;
+      }
+      if (iter.num() >= dstart) {
+	JSONFormatter f(true);
+	iter.cur().dump(&f, false);
+	f.flush(std::cout);
+	std::cout << std::endl;
+      }
+      iter.next();
+    }
+    std::cerr << "Read up to transaction " << iter.num() << std::endl;
+  } else if (cmd == "replay-trace") {
+    if (!store_path.size()) {
+      std::cerr << "need mon store path" << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    if (tfile.empty()) {
+      std::cerr << "Need trace_file" << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    unsigned num = 0;
+    for (unsigned i = 0; i < num_replays; ++i) {
+      TraceIter iter(tfile.c_str());
+      iter.init();
+      while (true) {
+	if (!iter.valid())
+	  break;
+	std::cerr << "Replaying trans num " << num << std::endl;
+	st.apply_transaction(iter.cur());
+	iter.next();
+	++num;
+      }
+      std::cerr << "Read up to transaction " << iter.num() << std::endl;
+    }
+  } else if (cmd == "random-gen") {
+    if (!store_path.size()) {
+      std::cerr << "need mon store path" << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    unsigned num = 0;
+    for (unsigned i = 0; i < ntrans; ++i) {
+      std::cerr << "Applying trans " << i << std::endl;
+      MonitorDBStore::Transaction t;
+      string prefix;
+      prefix.push_back((i%26)+'a');
+      for (unsigned j = 0; j < tsize; ++j) {
+	stringstream os;
+	os << num;
+	bufferlist bl;
+	for (unsigned k = 0; k < tvalsize; ++k) bl.append(rand());
+	t.put(prefix, os.str(), bl);
+	++num;
+      }
+      t.compact_prefix(prefix);
+      st.apply_transaction(t);
+    }
+  } else if (cmd == "store-copy") {
+    if (!store_path.size()) {
+      std::cerr << "need mon store path to copy from" << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    if (!out_path.size()) {
+      std::cerr << "need mon store path to copy to (--out )"
+                << std::endl;
+      std::cerr << desc << std::endl;
+      goto done;
+    }
+    if (fd > 0) {
+      std::cerr << "supplied out path '" << out_path << "' is not a directory"
+                << std::endl;
+      goto done;
+    }
+
+    MonitorDBStore out_store(out_path);
+    {
+      stringstream ss;
+      int r = out_store.create_and_open(ss);
+      if (r < 0) {
+        std::cerr << ss.str() << std::endl;
+        goto done;
+      }
+    }
+
+
+    KeyValueDB::WholeSpaceIterator it = st.get_iterator();
+    uint64_t total_keys = 0;
+    uint64_t total_size = 0;
+    uint64_t total_tx = 0;
+
+    do {
+      uint64_t num_keys = 0;
+
+      MonitorDBStore::Transaction tx;
+
+      while (it->valid() && num_keys < 128) {
+        pair k = it->raw_key();
+        bufferlist v = it->value();
+        tx.put(k.first, k.second, v);
+
+        num_keys ++;
+        total_tx ++;
+        total_size += v.length();
+
+        it->next();
+      }
+
+      total_keys += num_keys;
+
+      if (!tx.empty())
+        out_store.apply_transaction(tx);
+
+      std::cout << "copied " << total_keys << " keys so far ("
+                << stringify(si_t(total_size)) << ")" << std::endl;
+
+    } while (it->valid());
+
+    std::cout << "summary: copied " << total_keys << " keys, using "
+              << total_tx << " transactions, totalling "
+              << stringify(si_t(total_size)) << std::endl;
+    std::cout << "from '" << store_path << "' to '" << out_path << "'"
+              << std::endl;
+  } else {
+    std::cerr << "Unrecognized command: " << cmd << std::endl;
+    goto done;
+  }
+
+  done:
+  if (vm.count("out") && fd > 0) {
+    ::close(fd);
+  }
+  return 0;
+}
diff --git a/ceph/src/tools/ceph_osdomap_tool.cc b/ceph/src/tools/ceph_osdomap_tool.cc
new file mode 100644
index 00000000..f368a4b1
--- /dev/null
+++ b/ceph/src/tools/ceph_osdomap_tool.cc
@@ -0,0 +1,162 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License kkjversion 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "global/global_init.h"
+#include "os/LevelDBStore.h"
+#include "mon/MonitorDBStore.h"
+#include "os/DBObjectMap.h"
+#include "common/errno.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+int main(int argc, char **argv) {
+  po::options_description desc("Allowed options");
+  string store_path, cmd, out_path;
+  bool paranoid = false;
+  desc.add_options()
+    ("help", "produce help message")
+    ("omap-path", po::value(&store_path),
+     "path to mon directory, mandatory (current/omap usually)")
+    ("paranoid", po::value(¶noid),
+     "use paranoid checking")
+    ("command", po::value(&cmd),
+     "command")
+    ;
+  po::positional_options_description p;
+  p.add("command", 1);
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+    po::command_line_parser(argc, argv).options(desc).positional(p).run();
+  po::store(
+    parsed,
+    vm);
+  try {
+    po::notify(vm);
+  } catch (...) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  vector ceph_options, def_args;
+  vector ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
+    CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+  g_conf = g_ceph_context->_conf;
+
+  if (vm.count("help")) {
+    std::cerr << desc << std::endl;
+    return 1;
+  }
+
+  LevelDBStore* store(new LevelDBStore(g_ceph_context, store_path));
+  if (paranoid) {
+    std::cerr << "Enabling paranoid checks" << std::endl;
+    store->options.paranoid_checks = paranoid;
+  }
+  DBObjectMap omap(store);
+  stringstream out;
+  int r = store->open(out);
+  if (r < 0) {
+    std::cerr << "Store open got: " << cpp_strerror(r) << std::endl;
+    std::cerr << "Output: " << out.str() << std::endl;
+    goto done;
+  }
+  r = 0;
+
+
+  if (cmd == "dump-raw-keys") {
+    KeyValueDB::WholeSpaceIterator i = store->get_iterator();
+    for (i->seek_to_first(); i->valid(); i->next()) {
+      std::cout << i->raw_key() << std::endl;
+    }
+  } else if (cmd == "dump-raw-key-vals") {
+    KeyValueDB::WholeSpaceIterator i = store->get_iterator();
+    for (i->seek_to_first(); i->valid(); i->next()) {
+      std::cout << i->raw_key() << std::endl;
+      i->value().hexdump(std::cout);
+    }
+  } else if (cmd == "dump-objects") {
+    vector objects;
+    r = omap.list_objects(&objects);
+    if (r < 0) {
+      std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl;
+      goto done;
+    }
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i) {
+      std::cout << *i << std::endl;
+    }
+    r = 0;
+  } else if (cmd == "dump-objects-with-keys") {
+    vector objects;
+    r = omap.list_objects(&objects);
+    if (r < 0) {
+      std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl;
+      goto done;
+    }
+    for (vector::iterator i = objects.begin();
+	 i != objects.end();
+	 ++i) {
+      std::cout << "Object: " << *i << std::endl;
+      ObjectMap::ObjectMapIterator j = omap.get_iterator(i->hobj);
+      for (j->seek_to_first(); j->valid(); j->next()) {
+	std::cout << j->key() << std::endl;
+	j->value().hexdump(std::cout);
+      }
+    }
+  } else if (cmd == "check") {
+    r = omap.check(std::cout);
+    if (!r) {
+      std::cerr << "check got: " << cpp_strerror(r) << std::endl;
+      goto done;
+    }
+    std::cout << "check succeeded" << std::endl;
+  } else {
+    std::cerr << "Did not recognize command " << cmd << std::endl;
+    goto done;
+  }
+
+  done:
+  return r;
+}
diff --git a/ceph/src/tools/common.h b/ceph/src/tools/common.h
new file mode 100644
index 00000000..2a303675
--- /dev/null
+++ b/ceph/src/tools/common.h
@@ -0,0 +1,132 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef CEPH_TOOLS_COMMON_DOT_H
+#define CEPH_TOOLS_COMMON_DOT_H
+
+#include 
+#include 
+#include 
+
+#include "common/Cond.h"
+#include "common/Mutex.h"
+#include "mon/MonClient.h"
+#include "mon/PGMap.h"
+#include "mds/MDSMap.h"
+#include "osd/OSDMap.h"
+#include "common/Timer.h"
+
+
+#include "common/LogEntry.h"
+#include "mon/mon_types.h"
+#include "messages/MOSDMap.h"
+#include "messages/MLog.h"
+#include "messages/MCommand.h"
+#include "messages/MCommandReply.h"
+#include "messages/MMonCommandAck.h"
+
+
+#define OSD_MON_UPDATE	    (1<<0)
+#define MDS_MON_UPDATE	    (1<<1)
+#define PG_MON_UPDATE	    (1<<2)
+#define MON_MON_UPDATE	    (1<<3)
+#define EVERYTHING_UPDATE   0xffffffff
+
+class CephContext;
+class CephToolCtx;
+
+enum ceph_tool_mode_t {
+  CEPH_TOOL_MODE_CLI_INPUT = 0,
+  CEPH_TOOL_MODE_WATCH = 1,
+  CEPH_TOOL_MODE_STATUS = 2,
+  CEPH_TOOL_MODE_GUI = 3
+};
+
+struct Subscriptions {
+  CephToolCtx *ctx;
+  version_t last_known_version;
+  string name;
+
+  Subscriptions(CephToolCtx *c) : ctx(c), last_known_version(0) { }
+
+  void handle_log(MLog *m);
+};
+
+class Admin : public Dispatcher {
+private:
+  CephToolCtx *ctx;
+public:
+  Subscriptions subs;
+
+  Admin(CephToolCtx *ctx_)
+    : Dispatcher(g_ceph_context),
+      ctx(ctx_), subs(ctx_)
+  {
+  }
+
+  bool ms_dispatch(Message *m);
+  void ms_handle_connect(Connection *con);
+  bool ms_handle_reset(Connection *con);
+  void ms_handle_remote_reset(Connection *con) {}
+
+  bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new);
+};
+
+
+// tool/ceph.cc
+class CephToolCtx
+{
+public:
+  CephContext *cct;
+  PGMap pgmap;
+  MDSMap mdsmap;
+  OSDMap osdmap;
+  MonClient mc;
+
+  // Which aspects of the cluster have been updated recently?
+  uint32_t updates;
+
+  // The main log for ceph-tool
+  std::ostream *log;
+
+  // Used by the GUI to read from the log.
+  // NULL if there is no GUI active.
+  std::ostringstream *slog;
+
+  // The ceph-tool lock
+  Mutex lock;
+  SafeTimer timer;
+
+  // A condition variable used to wake up the GUI thread
+  Cond gui_cond;
+
+  bool concise;
+
+  Admin *dispatcher;
+
+  CephToolCtx(CephContext *cct_, bool concise_) :
+    cct(cct_),
+    mc(g_ceph_context),
+    updates(EVERYTHING_UPDATE),
+    log(&std::cout),
+    slog(NULL),
+    lock("ceph.cc lock"), timer(cct_, lock),
+    concise(concise_),
+    dispatcher(NULL)
+  {
+  }
+
+  ~CephToolCtx() {
+    delete dispatcher;
+  }
+};
+
+// tool/ceph.cc
+int ceph_tool_do_cli(CephToolCtx *data);
+int run_command(CephToolCtx *data, const char *line);
+CephToolCtx* ceph_tool_common_init(ceph_tool_mode_t mode, bool concise);
+int do_command(CephToolCtx *ctx,
+	       vector& cmd, bufferlist& bl, bufferlist& rbl);
+int ceph_tool_messenger_shutdown();
+int ceph_tool_common_shutdown(CephToolCtx *ctx);
+
+#endif
diff --git a/ceph/src/tools/crushtool.cc b/ceph/src/tools/crushtool.cc
new file mode 100644
index 00000000..8dcd79c3
--- /dev/null
+++ b/ceph/src/tools/crushtool.cc
@@ -0,0 +1,749 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ * Copyright (C) 2014 Cloudwatt 
+ *
+ * Author: Loic Dachary 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/config.h"
+
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "osd/OSDMap.h"
+#include "crush/CrushWrapper.h"
+#include "crush/CrushCompiler.h"
+#include "crush/CrushTester.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_crush
+
+using namespace std;
+
+const char *infn = "stdin";
+
+
+////////////////////////////////////////////////////////////////////////////
+
+void data_analysis_usage()
+{
+cout << "data output from testing routine ...\n";
+cout << "          absolute_weights\n";
+cout << "                the decimal weight of each OSD\n";
+cout << "                data layout: ROW MAJOR\n";
+cout << "                             OSD id (int), weight (int)\n";
+cout << "           batch_device_expected_utilization_all\n";
+cout << "                 the expected number of objects each OSD should receive per placement batch\n";
+cout << "                 which may be a decimal value\n";
+cout << "                 data layout: COLUMN MAJOR\n";
+cout << "                              round (int), objects expected on OSD 0...OSD n (float)\n";
+cout << "           batch_device_utilization_all\n";
+cout << "                 the number of objects stored on each OSD during each placement round\n";
+cout << "                 data layout: COLUMN MAJOR\n";
+cout << "                              round (int), objects stored on OSD 0...OSD n (int)\n";
+cout << "           device_utilization_all\n";
+cout << "                  the number of objects stored on each OSD at the end of placements\n";
+cout << "                  data_layout: ROW MAJOR\n";
+cout << "                               OSD id (int), objects stored (int), objects expected (float)\n";
+cout << "           device_utilization\n";
+cout << "                  the number of objects stored on each OSD marked 'up' at the end of placements\n";
+cout << "                  data_layout: ROW MAJOR\n";
+cout << "                               OSD id (int), objects stored (int), objects expected (float)\n";
+cout << "           placement_information\n";
+cout << "                  the map of input -> OSD\n";
+cout << "                  data_layout: ROW MAJOR\n";
+cout << "                               input (int), OSD's mapped (int)\n";
+cout << "           proportional_weights_all\n";
+cout << "                  the proportional weight of each OSD specified in the CRUSH map\n";
+cout << "                  data_layout: ROW MAJOR\n";
+cout << "                               OSD id (int), proportional weight (float)\n";
+cout << "           proportional_weights\n";
+cout << "                  the proportional weight of each 'up' OSD specified in the CRUSH map\n";
+cout << "                  data_layout: ROW MAJOR\n";
+cout << "                               OSD id (int), proportional weight (float)\n";
+}
+
+void usage()
+{
+  cout << "usage: crushtool ...\n";
+  cout << "   --decompile|-d map    decompile a crush map to source\n";
+  cout << "   --compile|-c map.txt  compile a map from source\n";
+  cout << "   [-o outfile [--clobber]]\n";
+  cout << "                         specify output for for (de)compilation\n";
+  cout << "   --build --num_osds N layer1 ...\n";
+  cout << "                         build a new map, where each 'layer' is\n";
+  cout << "                           'name (uniform|straw|list|tree) size'\n";
+  cout << "   -i mapfn --test       test a range of inputs on the map\n";
+  cout << "      [--min-x x] [--max-x x] [--x x]\n";
+  cout << "      [--min-rule r] [--max-rule r] [--rule r]\n";
+  cout << "      [--num-rep n]\n";
+  cout << "      [--batches b]      split the CRUSH mapping into b > 1 rounds\n";
+  cout << "      [--weight|-w devno weight]\n";
+  cout << "                         where weight is 0 to 1.0\n";
+  cout << "      [--simulate]       simulate placements using a random\n";
+  cout << "                         number generator in place of the CRUSH\n";
+  cout << "                         algorithm\n";
+  cout << "   -i mapfn --add-item id weight name [--loc type name ...]\n";
+  cout << "                         insert an item into the hierarchy at the\n";
+  cout << "                         given location\n";
+  cout << "   -i mapfn --update-item id weight name [--loc type name ...]\n";
+  cout << "                         insert or move an item into the hierarchy at the\n";
+  cout << "                         given location\n";
+  cout << "   -i mapfn --remove-item name\n"
+       << "                         remove the given item\n";
+  cout << "   -i mapfn --reweight-item name weight\n";
+  cout << "                         reweight a given item (and adjust ancestor\n"
+       << "                         weights as needed)\n";
+  cout << "   -i mapfn --reweight   recalculate all bucket weights\n";
+  cout << "   --show-utilization    show OSD usage\n";
+  cout << "   --show utilization-all\n";
+  cout << "                         include zero weight items\n";
+  cout << "   --show-statistics     show chi squared statistics\n";
+  cout << "   --show-bad-mappings   show bad mappings\n";
+  cout << "   --show-choose-tries   show choose tries histogram\n";
+  cout << "   --set-choose-local-tries N\n";
+  cout << "                         set choose local retries before re-descent\n";
+  cout << "   --set-choose-local-fallback-tries N\n";
+  cout << "                         set choose local retries using fallback\n";
+  cout << "                         permutation before re-descent\n";
+  cout << "   --set-choose-total-tries N\n";
+  cout << "                         set choose total descent attempts\n";
+  cout << "   --set-chooseleaf-descend-once <0|1>\n";
+  cout << "                         set chooseleaf to (not) retry the recursive descent\n";
+  cout << "   --set-chooseleaf-vary-r <0|1>\n";
+  cout << "                         set chooseleaf to (not) vary r based on parent\n";
+  cout << "   --output-name name\n";
+  cout << "                         prepend the data file(s) generated during the\n";
+  cout << "                         testing routine with name\n";
+  cout << "   --output-csv\n";
+  cout << "                         export select data generated during testing routine\n";
+  cout << "                         to CSV files for off-line post-processing\n";
+  cout << "                         use --help-output for more information\n";
+}
+
+struct bucket_types_t {
+  const char *name;
+  int type;
+} bucket_types[] = {
+  { "uniform", CRUSH_BUCKET_UNIFORM },
+  { "list", CRUSH_BUCKET_LIST },
+  { "straw", CRUSH_BUCKET_STRAW },
+  { "tree", CRUSH_BUCKET_TREE },
+  { 0, 0 },
+};
+
+struct layer_t {
+  const char *name;
+  const char *buckettype;
+  int size;
+};
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+
+  const char *me = argv[0];
+  std::string infn, srcfn, outfn, add_name, remove_name, reweight_name;
+  bool compile = false;
+  bool decompile = false;
+  bool test = false;
+  bool display = false;
+  bool write_to_file = false;
+  int verbose = 0;
+  bool unsafe_tunables = false;
+
+  bool reweight = false;
+  int add_item = -1;
+  bool update_item = false;
+  float add_weight = 0;
+  map add_loc;
+  float reweight_weight = 0;
+
+  bool adjust = false;
+
+  int build = 0;
+  int num_osds =0;
+  vector layers;
+
+  int choose_local_tries = -1;
+  int choose_local_fallback_tries = -1;
+  int choose_total_tries = -1;
+  int chooseleaf_descend_once = -1;
+  int chooseleaf_vary_r = -1;
+
+  CrushWrapper crush;
+
+  CrushTester tester(crush, cerr);
+
+  // we use -c, don't confuse the generic arg parsing
+  // only parse arguments from CEPH_ARGS, if in the environment
+  vector env_args;
+  env_to_vec(env_args);
+  global_init(NULL, env_args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+
+  int x;
+  float y;
+
+  std::string val;
+  std::ostringstream err;
+  int tmp;
+  for (std::vector::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+      exit(0);
+    } else if (ceph_argparse_witharg(args, i, &val, "-d", "--decompile", (char*)NULL)) {
+      infn = val;
+      decompile = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "-i", "--infn", (char*)NULL)) {
+      infn = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-o", "--outfn", (char*)NULL)) {
+      outfn = val;
+    } else if (ceph_argparse_flag(args, i, "-v", "--verbose", (char*)NULL)) {
+      verbose += 1;
+    } else if (ceph_argparse_flag(args, i, "--show_utilization", (char*)NULL)) {
+      display = true;
+      tester.set_output_utilization(true);
+    } else if (ceph_argparse_flag(args, i, "--show_utilization_all", (char*)NULL)) {
+      display = true;
+      tester.set_output_utilization_all(true);
+    } else if (ceph_argparse_flag(args, i, "--show_statistics", (char*)NULL)) {
+      display = true;
+      tester.set_output_statistics(true);
+    } else if (ceph_argparse_flag(args, i, "--show_bad_mappings", (char*)NULL)) {
+      display = true;
+      tester.set_output_bad_mappings(true);
+    } else if (ceph_argparse_flag(args, i, "--show_choose_tries", (char*)NULL)) {
+      display = true;
+      tester.set_output_choose_tries(true);
+    } else if (ceph_argparse_witharg(args, i, &val, "-c", "--compile", (char*)NULL)) {
+      srcfn = val;
+      compile = true;
+    } else if (ceph_argparse_flag(args, i, "-t", "--test", (char*)NULL)) {
+      test = true;
+    } else if (ceph_argparse_flag(args, i, "-s", "--simulate", (char*)NULL)) {
+      tester.set_random_placement();
+    } else if (ceph_argparse_flag(args, i, "--enable-unsafe-tunables", (char*)NULL)) {
+      unsafe_tunables = true;
+    } else if (ceph_argparse_withint(args, i, &choose_local_tries, &err,
+				     "--set_choose_local_tries", (char*)NULL)) {
+      adjust = true;
+    } else if (ceph_argparse_withint(args, i, &choose_local_fallback_tries, &err,
+				     "--set_choose_local_fallback_tries", (char*)NULL)) {
+      adjust = true;
+    } else if (ceph_argparse_withint(args, i, &choose_total_tries, &err,
+				     "--set_choose_total_tries", (char*)NULL)) {
+      adjust = true;
+    } else if (ceph_argparse_withint(args, i, &chooseleaf_descend_once, &err,
+				     "--set_chooseleaf_descend_once", (char*)NULL)) {
+      adjust = true;
+    } else if (ceph_argparse_withint(args, i, &chooseleaf_vary_r, &err,
+				     "--set_chooseleaf_vary_r", (char*)NULL)) {
+      adjust = true;
+    } else if (ceph_argparse_flag(args, i, "--reweight", (char*)NULL)) {
+      reweight = true;
+    } else if (ceph_argparse_withint(args, i, &add_item, &err, "--add_item", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --add-item" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      add_weight = atof(*i);
+      i = args.erase(i);
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --add-item" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      add_name.assign(*i);
+      i = args.erase(i);
+    } else if (ceph_argparse_withint(args, i, &add_item, &err, "--update_item", (char*)NULL)) {
+      update_item = true;
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --update-item" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      add_weight = atof(*i);
+      i = args.erase(i);
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --update-item" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      add_name.assign(*i);
+      i = args.erase(i);
+    } else if (ceph_argparse_witharg(args, i, &val, "--loc", (char*)NULL)) {
+      std::string type(val);
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --loc" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      std::string name(*i);
+      i = args.erase(i);
+      add_loc[type] = name;
+    } else if (ceph_argparse_flag(args, i, "--output-csv", (char*)NULL)) {
+      write_to_file = true;
+      tester.set_output_data_file(true);
+      tester.set_output_csv(true);
+    } else if (ceph_argparse_flag(args, i, "--help-output", (char*)NULL)) {
+      data_analysis_usage();
+      exit(0);
+    } else if (ceph_argparse_witharg(args, i, &val, "--output-name", (char*)NULL)) {
+      std::string name(val);
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --output-name" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      else {
+        tester.set_output_data_file_name(name + "-");
+      }
+    } else if (ceph_argparse_witharg(args, i, &val, "--remove_item", (char*)NULL)) {
+      remove_name = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--reweight_item", (char*)NULL)) {
+      reweight_name = val;
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --reweight-item" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      reweight_weight = atof(*i);
+      i = args.erase(i);
+    } else if (ceph_argparse_flag(args, i, "--build", (char*)NULL)) {
+      build = true;
+    } else if (ceph_argparse_withint(args, i, &num_osds, &err, "--num_osds", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--num_rep", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_num_rep(x);
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--max_x", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_max_x(x);
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--min_x", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_min_x(x);
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--x", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_x(x);
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--max_rule", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_max_rule(x);
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--min_rule", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_min_rule(x);
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--rule", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_rule(x);
+    } else if (ceph_argparse_withint(args, i, &x, &err, "--batches", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      tester.set_batches(x);
+    } else if (ceph_argparse_withfloat(args, i, &y, &err, "--mark-down-ratio", (char*)NULL)) {
+      if (!err.str().empty()) {
+        cerr << err.str() << std::endl;
+        exit(EXIT_FAILURE);
+      }
+      tester.set_device_down_ratio(y);
+    } else if (ceph_argparse_withfloat(args, i, &y, &err, "--mark-down-bucket-ratio", (char*)NULL)) {
+      if (!err.str().empty()) {
+        cerr << err.str() << std::endl;
+        exit(EXIT_FAILURE);
+      }
+      tester.set_bucket_down_ratio(y);
+    } else if (ceph_argparse_withint(args, i, &tmp, &err, "--weight", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      int dev = tmp;
+      if (i == args.end()) {
+	cerr << "expecting additional argument to --weight" << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      float f = atof(*i);
+      i = args.erase(i);
+      tester.set_device_weight(dev, f);
+    }
+    else {
+      ++i;
+    }
+  }
+
+  if (test && !display && !write_to_file) {
+    cerr << "WARNING: no output selected; use --output-csv or --show-X" << std::endl;
+    exit(EXIT_FAILURE);
+  }
+
+  if (decompile + compile + build > 1) {
+    cout << "cannot specify more than one of compile, decompile, and build" << std::endl;
+    exit(EXIT_FAILURE);
+  }
+  if (!compile && !decompile && !build && !test && !reweight && !adjust &&
+      add_item < 0 &&
+      remove_name.empty() && reweight_name.empty()) {
+    cout << "no action specified; -h for help" << std::endl;
+    exit(EXIT_FAILURE);
+  }
+  if ((!build) && (!args.empty())) {
+    cerr << "unrecognized arguments: " << args << std::endl;
+    exit(EXIT_FAILURE);
+  }
+  else {
+    if ((args.size() % 3) != 0U) {
+      cerr << "remaining args: " << args << std::endl;
+      cerr << "layers must be specified with 3-tuples of (name, buckettype, size)"
+    	   << std::endl;
+      exit(EXIT_FAILURE);
+    }
+    for (size_t j = 0; j < args.size(); j += 3) {
+      layer_t l;
+      l.name = args[j];
+      l.buckettype = args[j+1];
+      l.size = atoi(args[j+2]);
+      layers.push_back(l);
+    }
+  }
+
+  /*
+  if (outfn) cout << "outfn " << outfn << std::endl;
+  if (cinfn) cout << "cinfn " << cinfn << std::endl;
+  if (dinfn) cout << "dinfn " << dinfn << std::endl;
+  */
+
+  bool modified = false;
+
+  if (!infn.empty()) {
+    bufferlist bl;
+    std::string error;
+    int r = bl.read_file(infn.c_str(), &error);
+    if (r < 0) {
+      cerr << me << ": error reading '" << infn << "': " 
+	   << error << std::endl;
+      exit(1);
+    }
+    bufferlist::iterator p = bl.begin();
+    crush.decode(p);
+  }
+
+  if (decompile) {
+    CrushCompiler cc(crush, cerr, verbose);
+    if (!outfn.empty()) {
+      ofstream o;
+      o.open(outfn.c_str(), ios::out | ios::binary | ios::trunc);
+      if (!o.is_open()) {
+	cerr << me << ": error writing '" << outfn << "'" << std::endl;
+	exit(1);
+      }
+      cc.decompile(o);
+      o.close();
+    } else {
+      cc.decompile(cout);
+    }
+  }
+
+  if (compile) {
+    crush.create();
+
+    // read the file
+    ifstream in(srcfn.c_str());
+    if (!in.is_open()) {
+      cerr << "input file " << srcfn << " not found" << std::endl;
+      return -ENOENT;
+    }
+
+    CrushCompiler cc(crush, cerr, verbose);
+    if (unsafe_tunables)
+      cc.enable_unsafe_tunables();
+    int r = cc.compile(in, srcfn.c_str());
+    if (r < 0) 
+      exit(1);
+
+    modified = true;
+  }
+
+  if (build) {
+    if (layers.empty()) {
+      cerr << me << ": must specify at least one layer" << std::endl;
+      exit(1);
+    }
+
+    crush.create();
+
+    vector lower_items;
+    vector lower_weights;
+
+    for (int i=0; i::iterator p = layers.begin(); p != layers.end(); ++p, type++) {
+      layer_t &l = *p;
+
+      dout(2) << "layer " << type
+	      << "  " << l.name
+	      << "  bucket type " << l.buckettype
+	      << "  " << l.size 
+	      << dendl;
+
+      crush.set_type_name(type, l.name);
+
+      int buckettype = -1;
+      for (int i = 0; bucket_types[i].name; i++)
+	if (l.buckettype && strcmp(l.buckettype, bucket_types[i].name) == 0) {
+	  buckettype = bucket_types[i].type;
+	  break;
+	}
+      if (buckettype < 0) {
+	cerr << "unknown bucket type '" << l.buckettype << "'" << std::endl << std::endl;
+	exit(EXIT_FAILURE);
+      }
+
+      // build items
+      vector cur_items;
+      vector cur_weights;
+      unsigned lower_pos = 0;  // lower pos
+
+      dout(2) << "lower_items " << lower_items << dendl;
+      dout(2) << "lower_weights " << lower_weights << dendl;
+
+      int i = 0;
+      while (1) {
+	if (lower_pos == lower_items.size())
+	  break;
+
+	int items[num_osds];
+	int weights[num_osds];
+
+	int weight = 0;
+	int j;
+	for (j=0; j weights(crush.get_max_devices(), 0x10000);
+      crush.dump_tree(weights, &oss, NULL);
+      dout(1) << "\n" << oss.str() << dendl;
+    }
+
+    string root = layers.back().size == 0 ? layers.back().name :
+      string(layers.back().name) + "0";
+
+    {
+      set roots;
+      crush.find_roots(roots);
+      if (roots.size() > 1)
+	dout(1)	<< "The crush rulesets will use the root " << root << "\n"
+		<< "and ignore the others.\n"
+		<< "There are " << roots.size() << " roots, they can be\n"
+		<< "grouped into a single root by appending something like:\n"
+		<< "  root straw 0\n"
+		<< dendl;
+    }
+    
+    if (OSDMap::build_simple_crush_rulesets(g_ceph_context, crush, root, &cerr))
+      exit(EXIT_FAILURE);
+
+    modified = true;
+  }
+
+  if (!reweight_name.empty()) {
+    cout << me << " reweighting item " << reweight_name << " to " << reweight_weight << std::endl;
+    int r;
+    if (!crush.name_exists(reweight_name)) {
+      cerr << " name " << reweight_name << " dne" << std::endl;
+      r = -ENOENT;
+    } else {
+      int item = crush.get_item_id(reweight_name);
+      r = crush.adjust_item_weightf(g_ceph_context, item, reweight_weight);
+    }
+    if (r >= 0)
+      modified = true;
+    else {
+      cerr << me << " " << cpp_strerror(r) << std::endl;
+      return r;
+    }
+        
+  }
+  if (!remove_name.empty()) {
+    cout << me << " removing item " << remove_name << std::endl;
+    int r;
+    if (!crush.name_exists(remove_name)) {
+      cerr << " name " << remove_name << " dne" << std::endl;
+      r = -ENOENT;
+    } else {
+      int remove_item = crush.get_item_id(remove_name);
+      r = crush.remove_item(g_ceph_context, remove_item, false);
+    }
+    if (r == 0)
+      modified = true;
+    else {
+      cerr << me << " " << cpp_strerror(r) << std::endl;
+      return r;
+    }
+  }
+  if (add_item >= 0) {
+    int r;
+    if (update_item) {
+      r = crush.update_item(g_ceph_context, add_item, add_weight, add_name.c_str(), add_loc);
+    } else {
+      r = crush.insert_item(g_ceph_context, add_item, add_weight, add_name.c_str(), add_loc);
+    }
+    if (r >= 0) {
+      modified = true;
+    } else {
+      cerr << me << " " << cpp_strerror(r) << std::endl;
+      return r;
+    }
+  }
+  if (reweight) {
+    crush.reweight(g_ceph_context);
+    modified = true;
+  }
+
+  if (choose_local_tries >= 0) {
+    crush.set_choose_local_tries(choose_local_tries);
+    modified = true;
+  }
+  if (choose_local_fallback_tries >= 0) {
+    crush.set_choose_local_fallback_tries(choose_local_fallback_tries);
+    modified = true;
+  }
+  if (choose_total_tries >= 0) {
+    crush.set_choose_total_tries(choose_total_tries);
+    modified = true;
+  }
+  if (chooseleaf_descend_once >= 0) {
+    crush.set_chooseleaf_descend_once(chooseleaf_descend_once);
+    modified = true;
+  }
+  if (chooseleaf_vary_r >= 0) {
+    crush.set_chooseleaf_vary_r(chooseleaf_vary_r);
+    modified = true;
+  }
+  if (modified) {
+    crush.finalize();
+
+    if (outfn.empty()) {
+      cout << me << " successfully built or modified map.  Use '-o ' to write it out." << std::endl;
+    } else {
+      bufferlist bl;
+      crush.encode(bl);
+      int r = bl.write_file(outfn.c_str());
+      if (r < 0) {
+	cerr << me << ": error writing '" << outfn << "': " << cpp_strerror(r) << std::endl;
+	exit(1);
+      }
+      if (verbose)
+	cout << "wrote crush map to " << outfn << std::endl;
+    }
+  }
+
+  if (test) {
+    if (tester.get_output_utilization_all() ||
+	tester.get_output_utilization())
+      tester.set_output_statistics(true);
+
+    int r = tester.test();
+    if (r < 0)
+      exit(1);
+  }
+
+  return 0;
+}
+/*
+ * Local Variables:
+ * compile-command: "cd .. ; make crushtool && test/run-cli-tests"
+ * End:
+ */
diff --git a/ceph/src/tools/dupstore.cc b/ceph/src/tools/dupstore.cc
new file mode 100644
index 00000000..592b4b71
--- /dev/null
+++ b/ceph/src/tools/dupstore.cc
@@ -0,0 +1,113 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include "os/FileStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+
+#include "include/unordered_map.h"
+
+int dupstore(ObjectStore* src, ObjectStore* dst)
+{
+  if (src->mount() < 0) return 1;
+  if (dst->mkfs() < 0) return 1;
+  if (dst->mount() < 0) return 1;
+
+  // objects
+  ceph::unordered_map did_object;
+
+  // collections
+  vector collections;
+
+  int ret =  src->list_collections(collections);
+  if (ret < 0) {
+      cerr << "Error " << ret << " while listing collections" << std::endl;
+      return 1;
+  }
+
+  int num = collections.size();
+  cout << num << " collections" << std::endl;
+  int i = 1;
+  for (vector::iterator p = collections.begin();
+       p != collections.end();
+       ++p) {
+    cout << "collection " << i++ << "/" << num << " " << hex << *p << dec << std::endl;
+    {
+      ObjectStore::Transaction t;
+      t.create_collection(*p);
+      map attrs;
+      src->collection_getattrs(*p, attrs);
+      t.collection_setattrs(*p, attrs);
+      dst->apply_transaction(t);
+    }
+
+    vector o;
+    src->collection_list(*p, o);
+    int numo = o.size();
+    int j = 1;
+    for (vector::iterator q = o.begin(); q != o.end(); ++q) {
+      ObjectStore::Transaction t;
+      if (did_object.count(*q))
+	t.collection_add(*p, did_object[*q], *q);
+      else {
+	bufferlist bl;
+	src->read(*p, *q, 0, 0, bl);
+	cout << "object " << j++ << "/" << numo << " " << *q << " = " << bl.length() << " bytes" << std::endl;
+	t.write(*p, *q, 0, bl.length(), bl);
+	map attrs;
+	src->getattrs(*p, *q, attrs);
+	t.setattrs(*p, *q, attrs);
+	did_object[*q] = *p;
+      }
+      dst->apply_transaction(t);
+    }
+  }
+  
+  src->umount();
+  dst->umount();  
+  return 0;
+}
+
+void usage()
+{
+  cerr << "usage: ceph_dupstore filestore SRC filestore DST" << std::endl;
+  exit(0);
+}
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  // args
+  if (args.size() != 4) 
+    usage();
+
+  ObjectStore *src = 0, *dst = 0;
+
+  if (strcmp(args[0], "filestore") == 0) 
+    src = new FileStore(args[1], NULL);
+  else usage();
+
+  if (strcmp(args[2], "filestore") == 0) 
+    dst = new FileStore(args[3], NULL);
+  else usage();
+
+  return dupstore(src, dst);
+}
diff --git a/ceph/src/tools/mon_store_converter.cc b/ceph/src/tools/mon_store_converter.cc
new file mode 100644
index 00000000..1c0d3af9
--- /dev/null
+++ b/ceph/src/tools/mon_store_converter.cc
@@ -0,0 +1,336 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include "common/config.h"
+
+#include "mon/MonitorDBStore.h"
+#include "mon/MonitorStore.h"
+
+using namespace std;
+
+class MonitorStoreConverter {
+
+  boost::scoped_ptr db;
+  boost::scoped_ptr store;
+
+  set gvs;
+  version_t highest_last_pn;
+  version_t highest_accepted_pn;
+
+  static const int PAXOS_MAX_VERSIONS = 50;
+  string MONITOR_NAME;
+
+ public:
+  MonitorStoreConverter(string &store_path, string &db_store_path)
+    : db(0), store(0),
+      highest_last_pn(0), highest_accepted_pn(0),
+      MONITOR_NAME("monitor")
+  {
+    MonitorStore *store_ptr = new MonitorStore(store_path);
+    assert(!store_ptr->mount());
+    store.reset(store_ptr);
+
+    MonitorDBStore *db_ptr = new MonitorDBStore(db_store_path);
+    db.reset(db_ptr);
+  }
+
+  int convert() {
+    if (db->open(std::cerr) >= 0) {
+      std::cerr << "store already exists" << std::endl;
+      return EEXIST;
+    }
+    assert(!db->create_and_open(std::cerr));
+    if (db->exists("mon_convert", "on_going")) {
+      std::cout << __func__ << " found a mon store in mid-convertion; abort!"
+		<< std::endl;
+      return EEXIST;
+    }
+    _mark_convert_start();
+    _convert_monitor();
+    _convert_machines();
+    _mark_convert_finish();
+
+    std::cout << __func__ << " finished conversion" << std::endl;
+
+    return 0;
+  }
+
+  bool match() {
+    return true;
+  }
+
+ private:
+
+  set _get_machines_names() {
+    set names;
+    names.insert("auth");
+    names.insert("logm");
+    names.insert("mdsmap");
+    names.insert("monmap");
+    names.insert("osdmap");
+    names.insert("pgmap");
+
+    return names;
+  }
+
+  void _mark_convert_start() {
+    MonitorDBStore::Transaction tx;
+    tx.put("mon_convert", "on_going", 1);
+    db->apply_transaction(tx);
+  }
+
+  void _mark_convert_finish() {
+    MonitorDBStore::Transaction tx;
+    tx.erase("mon_convert", "on_going");
+    db->apply_transaction(tx);
+  }
+
+  void _convert_monitor() {
+
+    assert(store->exists_bl_ss("magic"));
+    assert(store->exists_bl_ss("keyring"));
+    assert(store->exists_bl_ss("feature_set"));
+    assert(store->exists_bl_ss("election_epoch"));
+
+    MonitorDBStore::Transaction tx;
+
+    if (store->exists_bl_ss("joined")) {
+      version_t joined = store->get_int("joined");
+      tx.put(MONITOR_NAME, "joined", joined);
+    }
+
+    vector keys;
+    keys.push_back("magic");
+    keys.push_back("feature_set");
+    keys.push_back("election_epoch");
+    keys.push_back("cluster_uuid");
+
+    vector::iterator it;
+    for (it = keys.begin(); it != keys.end(); ++it) {
+      if (!store->exists_bl_ss((*it).c_str()))
+	continue;
+
+      bufferlist bl;
+      int r = store->get_bl_ss(bl, (*it).c_str(), 0);
+      assert(r > 0);
+      tx.put(MONITOR_NAME, *it, bl);
+    }
+
+    assert(!tx.empty());
+    db->apply_transaction(tx);
+  }
+
+  void _convert_machines(string machine) {
+    std::cout << __func__ << " " << machine << std::endl;
+
+    version_t first_committed =
+      store->get_int(machine.c_str(), "first_committed");
+    version_t last_committed =
+      store->get_int(machine.c_str(), "last_committed");
+
+    version_t accepted_pn = store->get_int(machine.c_str(), "accepted_pn");
+    version_t last_pn = store->get_int(machine.c_str(), "last_pn");
+
+    if (accepted_pn > highest_accepted_pn)
+      highest_accepted_pn = accepted_pn;
+    if (last_pn > highest_last_pn)
+      highest_last_pn = last_pn;
+
+    string machine_gv(machine);
+    machine_gv.append("_gv");
+    bool has_gv = true;
+
+    if (!store->exists_bl_ss(machine_gv.c_str())) {
+      std::cerr << __func__ << " " << machine
+		<< " no gv dir '" << machine_gv << "'" << std::endl;
+      has_gv = false;
+    }
+
+    for (version_t ver = first_committed; ver <= last_committed; ver++) {
+      if (!store->exists_bl_sn(machine.c_str(), ver)) {
+	std::cerr << __func__ << " " << machine
+		  << " ver " << ver << " dne" << std::endl;
+	continue;
+      }
+
+      bufferlist bl;
+      int r = store->get_bl_sn(bl, machine.c_str(), ver);
+      assert(r >= 0);
+      std::cout << __func__ << " " << machine
+		<< " ver " << ver << " bl " << bl.length() << std::endl;
+
+      MonitorDBStore::Transaction tx;
+      tx.put(machine, ver, bl);
+      tx.put(machine, "last_committed", ver);
+
+      if (has_gv && store->exists_bl_sn(machine_gv.c_str(), ver)) {
+	stringstream s;
+	s << ver;
+	string ver_str = s.str();
+
+	version_t gv = store->get_int(machine_gv.c_str(), ver_str.c_str());
+	std::cerr << __func__ << " " << machine
+		  << " ver " << ver << " -> " << gv << std::endl;
+
+	if (gvs.count(gv) == 0) {
+	  gvs.insert(gv);
+	} else {
+	  std::cerr << __func__ << " " << machine
+		    << " gv " << gv << " already exists"
+		    << std::endl;
+	}
+
+	bufferlist tx_bl;
+	tx.encode(tx_bl);
+	tx.put("paxos", gv, tx_bl);
+      }
+      db->apply_transaction(tx);
+    }
+
+    version_t lc = db->get(machine, "last_committed");
+    assert(lc == last_committed);
+
+    MonitorDBStore::Transaction tx;
+    tx.put(machine, "first_committed", first_committed);
+    tx.put(machine, "last_committed", last_committed);
+
+    if (store->exists_bl_ss(machine.c_str(), "latest")) {
+      bufferlist latest_bl_raw;
+      int r = store->get_bl_ss(latest_bl_raw, machine.c_str(), "latest");
+      assert(r >= 0);
+      if (!latest_bl_raw.length()) {
+	std::cerr << __func__ << " machine " << machine
+		  << " skip latest with size 0" << std::endl;
+	goto out;
+      }
+
+      tx.put(machine, "latest", latest_bl_raw);
+
+      bufferlist::iterator lbl_it = latest_bl_raw.begin();
+      bufferlist latest_bl;
+      version_t latest_ver;
+      ::decode(latest_ver, lbl_it);
+      ::decode(latest_bl, lbl_it);
+
+      std::cout << __func__ << " machine " << machine
+		<< " latest ver " << latest_ver << std::endl;
+
+      tx.put(machine, "full_latest", latest_ver);
+      stringstream os;
+      os << "full_" << latest_ver;
+      tx.put(machine, os.str(), latest_bl);
+    }
+  out:
+    db->apply_transaction(tx);
+  }
+
+  void _convert_paxos() {
+    assert(!gvs.empty());
+
+    set::reverse_iterator rit = gvs.rbegin();
+    version_t highest_gv = *rit;
+    version_t last_gv = highest_gv;
+
+    int n = 0;
+    for (; (rit != gvs.rend()) && (n < PAXOS_MAX_VERSIONS); ++rit, ++n) {
+
+      version_t gv = *rit;
+
+      if (last_gv == gv)
+	continue;
+
+      if ((last_gv - gv) > 1) {
+	  // we are done; we found a gap and we are only interested in keeping
+	  // contiguous paxos versions.
+	  break;
+      }
+      last_gv = gv;
+    }
+
+    // erase all paxos versions between [first, last_gv[, with first being the
+    // first gv in the map.
+    MonitorDBStore::Transaction tx;
+    set::iterator it = gvs.begin();
+    std::cout << __func__ << " first gv " << (*it)
+	      << " last gv " << last_gv << std::endl;
+    for (; it != gvs.end() && (*it < last_gv); ++it) {
+      tx.erase("paxos", *it);
+    }
+    tx.put("paxos", "first_committed", last_gv);
+    tx.put("paxos", "last_committed", highest_gv);
+    tx.put("paxos", "accepted_pn", highest_accepted_pn);
+    tx.put("paxos", "last_pn", highest_last_pn);
+    db->apply_transaction(tx);
+  }
+
+  void _convert_machines() {
+
+    set machine_names = _get_machines_names();
+    set::iterator it = machine_names.begin();
+
+    std::cout << __func__ << std::endl;
+
+    for (; it != machine_names.end(); ++it) {
+      _convert_machines(*it);
+    }
+
+    _convert_paxos();
+  }
+};
+
+
+void usage(const char *pname)
+{
+  std::cerr << "Usage: " << pname << " \n"
+    << std::endl;
+}
+
+int main(int argc, const char *argv[])
+{
+  vector def_args;
+  vector args;
+  const char *our_name = argv[0];
+  argv_to_vec(argc, argv, args);
+
+  global_init(&def_args, args,
+	      CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+
+  if (args.empty()) {
+    usage(our_name);
+    return 1;
+  }
+  string store(args[0]);
+  string new_store(store);
+  MonitorStoreConverter converter(store, new_store);
+  assert(!converter.convert());
+  assert(converter.match());
+
+  std::cout << "store successfully converted to new format" << std::endl;
+
+  return 0;
+}
diff --git a/ceph/src/tools/monmaptool.cc b/ceph/src/tools/monmaptool.cc
new file mode 100644
index 00000000..f11858c2
--- /dev/null
+++ b/ceph/src/tools/monmaptool.cc
@@ -0,0 +1,208 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+using namespace std;
+
+#include "common/config.h"
+#include "common/ceph_argparse.h"
+#include "common/errno.h"
+#include "global/global_init.h"
+#include "mon/MonMap.h"
+#include "include/str_list.h"
+
+void usage()
+{
+  cout << " usage: [--print] [--create [--clobber][--fsid uuid]] [--generate] [--set-initial-members] [--add name 1.2.3.4:567] [--rm name] " << std::endl;
+  exit(1);
+}
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+
+  const char *me = argv[0];
+
+  std::string fn;
+  bool print = false;
+  bool create = false;
+  bool clobber = false;
+  bool modified = false;
+  bool generate = false;
+  bool filter = false;
+  map add;
+  list rm;
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  std::string val;
+  for (std::vector::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+    } else if (ceph_argparse_flag(args, i, "-p", "--print", (char*)NULL)) {
+      print = true;
+    } else if (ceph_argparse_flag(args, i, "--create", (char*)NULL)) {
+      create = true;
+    } else if (ceph_argparse_flag(args, i, "--clobber", (char*)NULL)) {
+      clobber = true;
+    } else if (ceph_argparse_flag(args, i, "--generate", (char*)NULL)) {
+      generate = true;
+    } else if (ceph_argparse_flag(args, i, "--set-initial-members", (char*)NULL)) {
+      filter = true;
+    } else if (ceph_argparse_flag(args, i, "--add", (char*)NULL)) {
+      string name = *i;
+      i = args.erase(i);
+      if (i == args.end())
+	usage();
+      entity_addr_t addr;
+      if (!addr.parse(*i)) {
+	cerr << me << ": invalid ip:port '" << *i << "'" << std::endl;
+	return -1;
+      }
+      if (addr.get_port() == 0)
+	addr.set_port(CEPH_MON_PORT);
+      add[name] = addr;
+      modified = true;
+      i = args.erase(i);
+    } else if (ceph_argparse_witharg(args, i, &val, "--rm", (char*)NULL)) {
+      rm.push_back(val);
+      modified = true;
+    } else {
+      ++i;
+    }
+  }
+  if (args.empty()) {
+    cerr << me << ": must specify monmap filename" << std::endl;
+    usage();
+  }
+  else if (args.size() > 1) {
+    cerr << me << ": too many arguments" << std::endl;
+    usage();
+  }
+  fn = args[0];
+  
+  MonMap monmap;
+
+  cout << me << ": monmap file " << fn << std::endl;
+
+  int r = 0;
+  if (!(create && clobber)) {
+    try {
+      r = monmap.read(fn.c_str());
+    } catch (...) {
+      cerr << me << ": unable to read monmap file" << std::endl;
+      return -1;
+    }
+  }
+
+  if (!create && r < 0) {
+    cerr << me << ": couldn't open " << fn << ": " << cpp_strerror(r) << std::endl;
+    return -1;
+  }    
+  else if (create && !clobber && r == 0) {
+    cerr << me << ": " << fn << " exists, --clobber to overwrite" << std::endl;
+    return -1;
+  }
+
+  if (create) {
+    monmap.epoch = 0;
+    monmap.created = ceph_clock_now(g_ceph_context);
+    monmap.last_changed = monmap.created;
+    srand(getpid() + time(0));
+    if (g_conf->fsid.is_zero()) {
+      monmap.generate_fsid();
+      cout << me << ": generated fsid " << monmap.fsid << std::endl;
+    }
+    modified = true;
+  }
+
+  if (generate) {
+    int r = monmap.build_initial(g_ceph_context, cerr);
+    if (r < 0)
+      return r;
+  }
+
+  if (filter) {
+    // apply initial members
+    list initial_members;
+    get_str_list(g_conf->mon_initial_members, initial_members);
+    if (!initial_members.empty()) {
+      cout << "initial_members " << initial_members << ", filtering seed monmap" << std::endl;
+      set removed;
+      monmap.set_initial_members(g_ceph_context, initial_members,
+				 string(), entity_addr_t(),
+				 &removed);
+      cout << "removed " << removed << std::endl;
+    }
+    modified = true;
+  }
+
+  if (!g_conf->fsid.is_zero()) {
+    monmap.fsid = g_conf->fsid;
+    cout << me << ": set fsid to " << monmap.fsid << std::endl;
+    modified = true;
+  }
+
+  for (map::iterator p = add.begin(); p != add.end(); ++p) {
+    if (monmap.contains(p->first)) {
+      cerr << me << ": map already contains mon." << p->first << std::endl;
+      usage();
+    }
+    if (monmap.contains(p->second)) {
+      cerr << me << ": map already contains " << p->second << std::endl;
+      usage();
+    }
+    monmap.add(p->first, p->second);
+  }
+  for (list::iterator p = rm.begin(); p != rm.end(); ++p) {
+    cout << me << ": removing " << *p << std::endl;
+    if (!monmap.contains(*p)) {
+      cerr << me << ": map does not contain " << *p << std::endl;
+      usage();
+    }
+    monmap.remove(*p);
+  }
+
+  if (!print && !modified)
+    usage();
+
+  if (print) 
+    monmap.print(cout);
+
+  if (modified) {
+    // write it out
+    cout << me << ": writing epoch " << monmap.epoch
+	 << " to " << fn
+	 << " (" << monmap.size() << " monitors)" 
+	 << std::endl;
+    int r = monmap.write(fn.c_str());
+    if (r < 0) {
+      cerr << "monmaptool: error writing to '" << fn << "': " << cpp_strerror(r) << std::endl;
+      return 1;
+    }
+  }
+  
+
+  return 0;
+}
diff --git a/ceph/src/tools/osdmaptool.cc b/ceph/src/tools/osdmaptool.cc
new file mode 100644
index 00000000..0db39da5
--- /dev/null
+++ b/ceph/src/tools/osdmaptool.cc
@@ -0,0 +1,488 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+using namespace std;
+
+#include "common/config.h"
+
+#include "common/errno.h"
+#include "osd/OSDMap.h"
+#include "mon/MonMap.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+
+void usage()
+{
+  cout << " usage: [--print] [--createsimple  [--clobber] [--pg_bits ]] " << std::endl;
+  cout << "   --export-crush    write osdmap's crush map to " << std::endl;
+  cout << "   --import-crush    replace osdmap's crush map with " << std::endl;
+  cout << "   --test-map-pgs [--pool ] map all pgs" << std::endl;
+  cout << "   --mark-up-in            mark osds up and in (but do not persist)" << std::endl;
+  cout << "   --clear-temp            clear pg_temp and primary_temp" << std::endl;
+  cout << "   --test-random           do random placements" << std::endl;
+  cout << "   --test-map-pg     map a pgid to osds" << std::endl;
+  cout << "   --test-map-object  [--pool ] map an object to osds"
+       << std::endl;
+  exit(1);
+}
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+	      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+
+  const char *me = argv[0];
+
+  std::string fn;
+  bool print = false;
+  bool print_json = false;
+  bool tree = false;
+  bool createsimple = false;
+  bool create_from_conf = false;
+  int num_osd = 0;
+  int pg_bits = g_conf->osd_pg_bits;
+  int pgp_bits = g_conf->osd_pgp_bits;
+  bool clobber = false;
+  bool modified = false;
+  std::string export_crush, import_crush, test_map_pg, test_map_object;
+  bool test_crush = false;
+  int range_first = -1;
+  int range_last = -1;
+  int pool = -1;
+  bool mark_up_in = false;
+  bool clear_temp = false;
+  bool test_map_pgs = false;
+  bool test_random = false;
+
+  std::string val;
+  std::ostringstream err;
+  for (std::vector::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+    } else if (ceph_argparse_flag(args, i, "-p", "--print", (char*)NULL)) {
+      print = true;
+    } else if (ceph_argparse_flag(args, i, "--dump-json", (char*)NULL)) {
+      print_json = true;
+    } else if (ceph_argparse_flag(args, i, "--tree", (char*)NULL)) {
+      tree = true;
+    } else if (ceph_argparse_withint(args, i, &num_osd, &err, "--createsimple", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+      createsimple = true;
+    } else if (ceph_argparse_flag(args, i, "--create-from-conf", (char*)NULL)) {
+      create_from_conf = true;
+    } else if (ceph_argparse_flag(args, i, "--mark-up-in", (char*)NULL)) {
+      mark_up_in = true;
+    } else if (ceph_argparse_flag(args, i, "--clear-temp", (char*)NULL)) {
+      clear_temp = true;
+    } else if (ceph_argparse_flag(args, i, "--test-map-pgs", (char*)NULL)) {
+      test_map_pgs = true;
+    } else if (ceph_argparse_flag(args, i, "--test-random", (char*)NULL)) {
+      test_random = true;
+    } else if (ceph_argparse_flag(args, i, "--clobber", (char*)NULL)) {
+      clobber = true;
+    } else if (ceph_argparse_withint(args, i, &pg_bits, &err, "--pg_bits", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+    } else if (ceph_argparse_withint(args, i, &pgp_bits, &err, "--pgp_bits", (char*)NULL)) {
+      if (!err.str().empty()) {
+	cerr << err.str() << std::endl;
+	exit(EXIT_FAILURE);
+      }
+    } else if (ceph_argparse_witharg(args, i, &val, "--export_crush", (char*)NULL)) {
+      export_crush = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--import_crush", (char*)NULL)) {
+      import_crush = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--test_map_pg", (char*)NULL)) {
+      test_map_pg = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--test_map_object", (char*)NULL)) {
+      test_map_object = val;
+    } else if (ceph_argparse_flag(args, i, "--test_crush", (char*)NULL)) {
+      test_crush = true;
+    } else if (ceph_argparse_withint(args, i, &range_first, &err, "--range_first", (char*)NULL)) {
+    } else if (ceph_argparse_withint(args, i, &range_last, &err, "--range_last", (char*)NULL)) {
+    } else if (ceph_argparse_withint(args, i, &pool, &err, "--pool", (char*)NULL)) {
+      if (!err.str().empty()) {
+        cerr << err.str() << std::endl;
+        exit(EXIT_FAILURE);
+      }
+    } else {
+      ++i;
+    }
+  }
+  if (args.empty()) {
+    cerr << me << ": must specify osdmap filename" << std::endl;
+    usage();
+  }
+  else if (args.size() > 1) {
+    cerr << me << ": too many arguments" << std::endl;
+    usage();
+  }
+  fn = args[0];
+
+  if (range_first >= 0 && range_last >= 0) {
+    set maps;
+    OSDMap *prev = NULL;
+    for (int i=range_first; i <= range_last; i++) {
+      ostringstream f;
+      f << fn << "/" << i;
+      bufferlist bl;
+      string error, s = f.str();
+      int r = bl.read_file(s.c_str(), &error);
+      if (r < 0) {
+	cerr << "unable to read " << s << ": " << cpp_strerror(r) << std::endl;
+	exit(1);
+      }
+      cout << s << " got " << bl.length() << " bytes" << std::endl;
+      OSDMap *o = new OSDMap;
+      o->decode(bl);
+      maps.insert(o);
+      if (prev)
+	OSDMap::dedup(prev, o);
+      prev = o;
+    }
+    exit(0);
+  }
+  
+  OSDMap osdmap;
+  bufferlist bl;
+
+  cerr << me << ": osdmap file '" << fn << "'" << std::endl;
+  
+  int r = 0;
+  struct stat st;
+  if (!createsimple && !create_from_conf && !clobber) {
+    std::string error;
+    r = bl.read_file(fn.c_str(), &error);
+    if (r == 0) {
+      try {
+	osdmap.decode(bl);
+      }
+      catch (const buffer::error &e) {
+	cerr << me << ": error decoding osdmap '" << fn << "'" << std::endl;
+	return -1;
+      }
+    }
+    else {
+      cerr << me << ": couldn't open " << fn << ": " << error << std::endl;
+      return -1;
+    }
+  }
+  else if ((createsimple || create_from_conf) && !clobber && ::stat(fn.c_str(), &st) == 0) {
+    cerr << me << ": " << fn << " exists, --clobber to overwrite" << std::endl;
+    return -1;
+  }
+
+  if (createsimple || create_from_conf) {
+    if (createsimple) {
+      if (num_osd < 1) {
+	cerr << me << ": osd count must be > 0" << std::endl;
+	exit(1);
+      }
+    } else {
+      num_osd = -1;
+    }
+    uuid_d fsid;
+    memset(&fsid, 0, sizeof(uuid_d));
+    osdmap.build_simple(g_ceph_context, 0, fsid, num_osd, pg_bits, pgp_bits);
+    modified = true;
+  }
+
+  if (mark_up_in) {
+    cout << "marking all OSDs up and in" << std::endl;
+    int n = osdmap.get_max_osd();
+    for (int i=0; iadjust_item_weightf(g_ceph_context, i, 1.0);
+    }
+  }
+  if (clear_temp) {
+    cout << "clearing pg/primary temp" << std::endl;
+    osdmap.clear_temp();
+  }
+
+  if (!import_crush.empty()) {
+    bufferlist cbl;
+    std::string error;
+    r = cbl.read_file(import_crush.c_str(), &error);
+    if (r) {
+      cerr << me << ": error reading crush map from " << import_crush
+	   << ": " << error << std::endl;
+      exit(1);
+    }
+
+    // validate
+    CrushWrapper cw;
+    bufferlist::iterator p = cbl.begin();
+    cw.decode(p);
+
+    if (cw.get_max_devices() > osdmap.get_max_osd()) {
+      cerr << me << ": crushmap max_devices " << cw.get_max_devices()
+	   << " > osdmap max_osd " << osdmap.get_max_osd() << std::endl;
+      exit(1);
+    }
+    
+    // apply
+    OSDMap::Incremental inc;
+    inc.fsid = osdmap.get_fsid();
+    inc.epoch = osdmap.get_epoch()+1;
+    inc.crush = cbl;
+    osdmap.apply_incremental(inc);
+    cout << me << ": imported " << cbl.length() << " byte crush map from " << import_crush << std::endl;
+    modified = true;
+  }
+
+  if (!export_crush.empty()) {
+    bufferlist cbl;
+    osdmap.crush->encode(cbl);
+    r = cbl.write_file(export_crush.c_str());
+    if (r < 0) {
+      cerr << me << ": error writing crush map to " << import_crush << std::endl;
+      exit(1);
+    }
+    cout << me << ": exported crush map to " << export_crush << std::endl;
+  }  
+
+  if (!test_map_object.empty()) {
+    object_t oid(test_map_object);
+    if (pool == -1) {
+      cout << me << ": assuming pool 0 (use --pool to override)" << std::endl;
+      pool = 0;
+    }
+    if (!osdmap.have_pg_pool(pool)) {
+      cerr << "There is no pool " << pool << std::endl;
+      exit(1);
+    }
+    object_locator_t loc(pool);
+    pg_t raw_pgid = osdmap.object_locator_to_pg(oid, loc);
+    pg_t pgid = osdmap.raw_pg_to_pg(raw_pgid);
+    
+    vector acting;
+    osdmap.pg_to_acting_osds(pgid, acting);
+    cout << " object '" << oid
+	 << "' -> " << pgid
+	 << " -> " << acting
+	 << std::endl;
+  }  
+  if (!test_map_pg.empty()) {
+    pg_t pgid;
+    if (!pgid.parse(test_map_pg.c_str())) {
+      cerr << me << ": failed to parse pg '" << test_map_pg
+	   << "', r = " << r << std::endl;
+      usage();
+    }
+    cout << " parsed '" << test_map_pg << "' -> " << pgid << std::endl;
+
+    vector raw, up, acting;
+    int calced_primary, up_primary, acting_primary;
+    osdmap.pg_to_osds(pgid, &raw, &calced_primary);
+    osdmap.pg_to_up_acting_osds(pgid, &up, &up_primary,
+                                &acting, &acting_primary);
+    cout << pgid << " raw (" << raw << ", p" << calced_primary
+         << ") up (" << up << ", p" << up_primary
+         << ") acting (" << acting << ", p" << acting_primary << ")"
+         << std::endl;
+  }
+  if (test_map_pgs) {
+    if (pool != -1 && !osdmap.have_pg_pool(pool)) {
+      cerr << "There is no pool " << pool << std::endl;
+      exit(1);
+    }
+    int n = osdmap.get_max_osd();
+    vector count(n, 0);
+    vector first_count(n, 0);
+    vector primary_count(n, 0);
+    vector size(30, 0);
+    if (test_random)
+      srand(getpid());
+    const map& pools = osdmap.get_pools();
+    for (map::const_iterator p = pools.begin();
+	 p != pools.end(); ++p) {
+      if (pool != -1 && p->first != pool)
+	continue;
+      cout << "pool " << p->first
+	   << " pg_num " << p->second.get_pg_num() << std::endl;
+      for (unsigned i = 0; i < p->second.get_pg_num(); ++i) {
+	pg_t pgid = pg_t(i, p->first);
+
+	vector osds;
+	int primary;
+	if (test_random) {
+	  osds.resize(p->second.size);
+	  for (unsigned i=0; i= 0)
+	  primary_count[primary]++;
+      }
+    }
+
+    uint64_t total = 0;
+    int in = 0;
+    int min_osd = -1;
+    int max_osd = -1;
+    cout << "#osd\tcount\tfirst\tprimary\tc wt\twt\n";
+    for (int i=0; iget_item_weight(i) <= 0)
+	continue;
+      in++;
+      cout << "osd." << i
+	   << "\t" << count[i]
+	   << "\t" << first_count[i]
+	   << "\t" << primary_count[i]
+	   << "\t" << osdmap.crush->get_item_weightf(i)
+	   << "\t" << osdmap.get_weightf(i)
+	   << std::endl;
+      total += count[i];
+      if (count[i] &&
+	  (min_osd < 0 ||
+	   count[i] < count[min_osd]))
+	min_osd = i;
+      if (count[i] &&
+	  (max_osd < 0 ||
+	   count[i] > count[max_osd]))
+	max_osd = i;
+
+    }
+    uint64_t avg = total / in;
+    double dev = 0;
+    for (int i=0; iget_item_weight(i) <= 0)
+	continue;
+      dev += (avg - count[i]) * (avg - count[i]);
+    }
+    dev /= in;
+    dev = sqrt(dev);
+
+    //double edev = sqrt(pgavg) * (double)avg / pgavg;
+    double edev = sqrt((double)total / (double)in * (1.0 - (1.0 / (double)in)));
+    cout << " in " << in << std::endl;
+    cout << " avg " << avg
+	 << " stddev " << dev
+	 << " (" << (dev/avg) << "x)"
+	 << " (expected " << edev << " " << (edev/avg) << "x))"
+	 << std::endl;
+
+    if (min_osd >= 0)
+      cout << " min osd." << min_osd << " " << count[min_osd] << std::endl;
+    if (max_osd >= 0)
+      cout << " max osd." << max_osd << " " << count[max_osd] << std::endl;
+
+    for (int i=0; i<4; i++) {
+      cout << "size " << i << "\t" << size[i] << std::endl;
+    }
+  }
+  if (test_crush) {
+    int pass = 0;
+    while (1) {
+      cout << "pass " << ++pass << std::endl;
+
+      ceph::unordered_map > m;
+      for (map::const_iterator p = osdmap.get_pools().begin();
+	   p != osdmap.get_pools().end();
+	   ++p) {
+	const pg_pool_t *pool = osdmap.get_pg_pool(p->first);
+	for (ps_t ps = 0; ps < pool->get_pg_num(); ps++) {
+	  pg_t pgid(ps, p->first, -1);
+	  for (int i=0; i<100; i++) {
+	    cout << pgid << " attempt " << i << std::endl;
+
+	    vector r;
+	    osdmap.pg_to_acting_osds(pgid, r);
+	    //cout << pgid << " " << r << std::endl;
+	    if (m.count(pgid)) {
+	      if (m[pgid] != r) {
+		cout << pgid << " had " << m[pgid] << " now " << r << std::endl;
+		assert(0);
+	      }
+	    } else
+	      m[pgid] = r;
+	  }
+	}
+      }
+    }
+  }
+
+  if (!print && !print_json && !tree && !modified && 
+      export_crush.empty() && import_crush.empty() && 
+      test_map_pg.empty() && test_map_object.empty() &&
+      !test_map_pgs) {
+    cerr << me << ": no action specified?" << std::endl;
+    usage();
+  }
+
+  if (modified)
+    osdmap.inc_epoch();
+
+  if (print) 
+    osdmap.print(cout);
+  if (print_json)
+    osdmap.dump_json(cout);
+  if (tree) 
+    osdmap.print_tree(&cout, NULL);
+
+  if (modified) {
+    bl.clear();
+    osdmap.encode(bl);
+
+    // write it out
+    cout << me << ": writing epoch " << osdmap.get_epoch()
+	 << " to " << fn
+	 << std::endl;
+    int r = bl.write_file(fn.c_str());
+    if (r) {
+      cerr << "osdmaptool: error writing to '" << fn << "': "
+	   << cpp_strerror(r) << std::endl;
+      return 1;
+    }
+  }
+  
+
+  return 0;
+}
diff --git a/ceph/src/tools/psim.cc b/ceph/src/tools/psim.cc
new file mode 100644
index 00000000..7f094b9b
--- /dev/null
+++ b/ceph/src/tools/psim.cc
@@ -0,0 +1,114 @@
+
+#include 
+
+#include "crush/CrushWrapper.h"
+#include "osd/OSDMap.h"
+#include "common/config.h"
+#include "include/buffer.h"
+
+int main(int argc, char **argv)
+{
+  /*
+   * you need to create a suitable osdmap first.  e.g., for 40 osds, 
+   * $ ./osdmaptool --createsimple 40 --clobber .ceph_osdmap
+   */
+  bufferlist bl;
+  std::string error;
+  if (bl.read_file(".ceph_osdmap", &error)) {
+    cout << argv[0] << ": error reading .ceph_osdmap: " << error << std::endl;
+    return 1;
+  }
+  OSDMap osdmap;
+  osdmap.decode(bl);
+
+  //osdmap.set_primary_affinity(0, 0x8000);
+  //osdmap.set_primary_affinity(3, 0);
+
+  int n = osdmap.get_max_osd();
+  int count[n];
+  int first_count[n];
+  int primary_count[n];
+  for (int i=0; itype = pg_pool_t::TYPE_ERASURE;
+
+  int size[4];
+  for (int i=0; i<4; i++)
+    size[i] = 0;
+
+  for (int n = 0; n < 10; n++) {   // namespaces
+    char nspace[20];
+    snprintf(nspace, sizeof(nspace), "n%d", n);
+  for (int f = 0; f < 5000; f++) {  // files
+    for (int b = 0; b < 4; b++) {   // blocks
+      char foo[20];
+      snprintf(foo, sizeof(foo), "%d.%d", f, b);
+      object_t oid(foo);
+      ceph_object_layout l = osdmap.make_object_layout(oid, 0, nspace);
+	//osdmap.file_to_object_layout(oid, g_default_file_layout);
+      vector osds;
+      pg_t pgid = pg_t(l.ol_pgid);
+      //pgid.u.ps = f * 4 + b;
+      int primary;
+      osdmap.pg_to_acting_osds(pgid, &osds, &primary);
+      size[osds.size()]++;
+#if 0
+      if (0) {
+	hash H;
+	int x = H(oid);
+	x = ceph_stable_mod(x, 1023, 1023);
+	int s = crush_hash32(x) % 15;
+	//cout << "ceph_psim: x = " << x << " s = " << s << std::endl;
+	//osds[0] = s;
+      }
+#endif
+      //osds[0] = crush_hash32(f) % n;
+      //cout << "oid " << oid << " pgid " << pgid << " on " << osds << std::endl;
+      for (unsigned i=0; i= 0)
+	primary_count[primary]++;
+    }
+  }
+  }
+
+  uint64_t avg = 0;
+  for (int i=0; iget_pg_num() / (double)n;
+  double edev = sqrt(pgavg) * (double)avg / pgavg;
+  cout << " avg " << avg
+       << " stddev " << dev
+       << " (expected " << edev << ")"
+       << " (indep object placement would be " << sqrt(avg) << ")" << std::endl;
+
+  for (int i=0; i<4; i++) {
+    cout << "size" << i << "\t" << size[i] << std::endl;
+  }
+  
+  return 0;
+}
diff --git a/ceph/src/tools/rados/rados.cc b/ceph/src/tools/rados/rados.cc
new file mode 100644
index 00000000..bd71433e
--- /dev/null
+++ b/ceph/src/tools/rados/rados.cc
@@ -0,0 +1,2657 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/types.h"
+
+#include "include/rados/librados.hpp"
+#include "include/rados/rados_types.hpp"
+#include "rados_sync.h"
+using namespace librados;
+
+#include "common/config.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/Cond.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/obj_bencher.h"
+#include "mds/inode_backtrace.h"
+#include "auth/Crypto.h"
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "cls/lock/cls_lock_client.h"
+#include "include/compat.h"
+
+int rados_tool_sync(const std::map < std::string, std::string > &opts,
+                             std::vector &args);
+
+// two steps seem to be necessary to do this right
+#define STR(x) _STR(x)
+#define _STR(x) #x
+
+void usage(ostream& out)
+{
+  out <<					\
+"usage: rados [options] [commands]\n"
+"POOL COMMANDS\n"
+"   lspools                          list pools\n"
+"   mkpool  [123[ 4]]     create pool '\n"
+"                                    [with auid 123[and using crush rule 4]]\n"
+"   cppool     copy content of a pool\n"
+"   rmpool  [ --yes-i-really-really-mean-it]\n"
+"                                    remove pool '\n"
+"   df                               show per-pool and total usage\n"
+"   ls                               list objects in pool\n\n"
+"   chown 123                        change the pool owner to auid 123\n"
+"\n"
+"OBJECT COMMANDS\n"
+"   get  [outfile]         fetch object\n"
+"   put  [infile]          write object\n"
+"   truncate  length       truncate object\n"
+"   create  [category]     create object\n"
+"   rm  ...                remove object(s)\n"
+"   cp  [target-obj]       copy object\n"
+"   clonedata      clone object data\n"
+"   listxattr \n"
+"   getxattr  attr\n"
+"   setxattr  attr val\n"
+"   rmxattr  attr\n"
+"   stat objname                     stat the named object\n"
+"   mapext \n"
+"   lssnap                           list snaps\n"
+"   mksnap                create snap \n"
+"   rmsnap                remove snap \n"
+"   rollback    roll back object to snap \n"
+"\n"
+"   listsnaps              list the snapshots of this object\n"
+"   bench  write|seq|rand [-t concurrent_operations] [--no-cleanup] [--run-name run_name]\n"
+"                                    default is 16 concurrent IOs and 4 MB ops\n"
+"                                    default is to clean up after write benchmark\n"
+"                                    default run-name is 'benchmark_last_metadata'\n"
+"   cleanup [--run-name run_name] [--prefix prefix]\n"
+"                                    clean up a previous benchmark operation\n"
+"                                    default run-name is 'benchmark_last_metadata'\n"
+"   load-gen [options]               generate load on the cluster\n"
+"   listomapkeys           list the keys in the object map\n"
+"   listomapvals           list the keys and vals in the object map \n"
+"   getomapval   [file] show the value for the specified key\n"
+"                                    in the object's object map\n"
+"   setomapval   \n"
+"   rmomapkey  \n"
+"   getomapheader  [file]\n"
+"   setomapheader  \n"
+"   tmap-to-omap           convert tmap keys/values to omap\n"
+"   listwatchers           list the watchers of this object\n"
+"   set-alloc-hint   \n"
+"                                    set allocation hint for an object\n"
+"\n"
+"IMPORT AND EXPORT\n"
+"   import [options]  \n"
+"       Upload  to \n"
+"   export [options] rados-pool> \n"
+"       Download  to \n"
+"   options:\n"
+"       -f / --force                 Copy everything, even if it hasn't changed.\n"
+"       -d / --delete-after          After synchronizing, delete unreferenced\n"
+"                                    files or objects from the target bucket\n"
+"                                    or directory.\n"
+"       --workers                    Number of worker threads to spawn \n"
+"                                    (default " STR(DEFAULT_NUM_RADOS_WORKER_THREADS) ")\n"
+"\n"
+"ADVISORY LOCKS\n"
+"   lock list \n"
+"       List all advisory locks on an object\n"
+"   lock get  \n"
+"       Try to acquire a lock\n"
+"   lock break   \n"
+"       Try to break a lock acquired by another client\n"
+"   lock info  \n"
+"       Show lock information\n"
+"   options:\n"
+"       --lock-tag                   Lock tag, all locks operation should use\n"
+"                                    the same tag\n"
+"       --lock-cookie                Locker cookie\n"
+"       --lock-description           Description of lock\n"
+"       --lock-duration              Lock duration (in seconds)\n"
+"       --lock-type                  Lock type (shared, exclusive)\n"
+"\n"
+"CACHE POOLS: (for testing/development only)\n"
+"   cache-flush            flush cache pool object (blocking)\n"
+"   cache-try-flush        flush cache pool object (non-blocking)\n"
+"   cache-evict            evict cache pool object\n"
+"   cache-flush-evict-all            flush+evict all objects\n"
+"   cache-try-flush-evict-all        try-flush+evict all objects\n"
+"\n"
+"GLOBAL OPTIONS:\n"
+"   --object_locator object_locator\n"
+"        set object_locator for operation\n"
+"   -p pool\n"
+"   --pool=pool\n"
+"        select given pool by name\n"
+"   --target-pool=pool\n"
+"        select target pool by name\n"
+"   -b op_size\n"
+"        set the size of write ops for put or benchmarking\n"
+"   -s name\n"
+"   --snap name\n"
+"        select given snap name for (read) IO\n"
+"   -i infile\n"
+"   --create\n"
+"        create the pool or directory that was specified\n"
+"   -N namespace\n"
+"   --namespace=namespace\n"
+"        specify the namespace to use for the object\n"
+"\n"
+"BENCH OPTIONS:\n"
+"   -t N\n"
+"   --concurrent-ios=N\n"
+"        Set number of concurrent I/O operations\n"
+"   --show-time\n"
+"        prefix output with date/time\n"
+"\n"
+"LOAD GEN OPTIONS:\n"
+"   --num-objects                    total number of objects\n"
+"   --min-object-size                min object size\n"
+"   --max-object-size                max object size\n"
+"   --min-ops                        min number of operations\n"
+"   --max-ops                        max number of operations\n"
+"   --max-backlog                    max backlog (in MB)\n"
+"   --percent                        percent of operations that are read\n"
+"   --target-throughput              target throughput (in MB)\n"
+"   --run-length                     total time (in seconds)\n"
+    ;
+}
+
+static void usage_exit()
+{
+  usage(cerr);
+  exit(1);
+}
+
+
+static int dump_data(std::string const &filename, bufferlist const &data)
+{
+  int fd;
+  if (filename == "-") {
+    fd = 1;
+  } else {
+    fd = TEMP_FAILURE_RETRY(::open(filename.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0644));
+    if (fd < 0) {
+      int err = errno;
+      cerr << "failed to open file: " << cpp_strerror(err) << std::endl;
+      return -err;
+    }
+  }
+
+  int r = data.write_fd(fd);
+
+  if (fd != 1) {
+    VOID_TEMP_FAILURE_RETRY(::close(fd));
+  }
+
+  return r;
+}
+
+
+static int do_get(IoCtx& io_ctx, const char *objname, const char *outfile, unsigned op_size)
+{
+  string oid(objname);
+
+  int fd;
+  if (strcmp(outfile, "-") == 0) {
+    fd = 1;
+  } else {
+    fd = TEMP_FAILURE_RETRY(::open(outfile, O_WRONLY|O_CREAT|O_TRUNC, 0644));
+    if (fd < 0) {
+      int err = errno;
+      cerr << "failed to open file: " << cpp_strerror(err) << std::endl;
+      return -err;
+    }
+  }
+
+  uint64_t offset = 0;
+  int ret;
+  while (true) {
+    bufferlist outdata;
+    ret = io_ctx.read(oid, outdata, op_size, offset);
+    if (ret <= 0) {
+      goto out;
+    }
+    ret = outdata.write_fd(fd);
+    if (ret < 0) {
+      cerr << "error writing to file: " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    if (outdata.length() < op_size)
+      break;
+    offset += outdata.length();
+  }
+  ret = 0;
+
+ out:
+  if (fd != 1)
+    VOID_TEMP_FAILURE_RETRY(::close(fd));
+  return ret;
+}
+
+static int do_copy(IoCtx& io_ctx, const char *objname, IoCtx& target_ctx, const char *target_obj)
+{
+  string oid(objname);
+  bufferlist outdata;
+  librados::ObjectReadOperation read_op;
+  string start_after;
+
+#define COPY_CHUNK_SIZE (4 * 1024 * 1024)
+  read_op.read(0, COPY_CHUNK_SIZE, &outdata, NULL);
+
+  map attrset;
+  read_op.getxattrs(&attrset, NULL);
+
+  bufferlist omap_header;
+  read_op.omap_get_header(&omap_header, NULL);
+
+#define OMAP_CHUNK 1000
+  map omap;
+  read_op.omap_get_vals(start_after, OMAP_CHUNK, &omap, NULL);
+
+  bufferlist opbl;
+  int ret = io_ctx.operate(oid, &read_op, &opbl);
+  if (ret < 0) {
+    return ret;
+  }
+
+  librados::ObjectWriteOperation write_op;
+  string target_oid(target_obj);
+
+  /* reset dest if exists */
+  write_op.create(false);
+  write_op.remove();
+
+  write_op.write_full(outdata);
+  write_op.omap_set_header(omap_header);
+
+  map::iterator iter;
+  for (iter = attrset.begin(); iter != attrset.end(); ++iter) {
+    write_op.setxattr(iter->first.c_str(), iter->second);
+  }
+  if (!omap.empty()) {
+    write_op.omap_set(omap);
+  }
+  ret = target_ctx.operate(target_oid, &write_op);
+  if (ret < 0) {
+    return ret;
+  }
+
+  uint64_t off = 0;
+
+  while (outdata.length() == COPY_CHUNK_SIZE) {
+    off += outdata.length();
+    outdata.clear();
+    ret = io_ctx.read(oid, outdata, COPY_CHUNK_SIZE, off); 
+    if (ret < 0)
+      goto err;
+
+    ret = target_ctx.write(target_oid, outdata, outdata.length(), off);
+    if (ret < 0)
+      goto err;
+  }
+
+  /* iterate through source omap and update target. This is not atomic */
+  while (omap.size() == OMAP_CHUNK) {
+    /* now start_after should point at the last entry */    
+    map::iterator iter = omap.end();
+    --iter;
+    start_after = iter->first;
+
+    omap.clear();
+    ret = io_ctx.omap_get_vals(oid, start_after, OMAP_CHUNK, &omap);
+    if (ret < 0)
+      goto err;
+
+    if (omap.empty())
+      break;
+
+    ret = target_ctx.omap_set(target_oid, omap);
+    if (ret < 0)
+      goto err;
+  }
+
+  return 0;
+
+err:
+  target_ctx.remove(target_oid);
+  return ret;
+}
+
+static int do_clone_data(IoCtx& io_ctx, const char *objname, IoCtx& target_ctx, const char *target_obj)
+{
+  string oid(objname);
+
+  // get size
+  uint64_t size;
+  int r = target_ctx.stat(oid, &size, NULL);
+  if (r < 0)
+    return r;
+
+  librados::ObjectWriteOperation write_op;
+  string target_oid(target_obj);
+
+  /* reset data stream only */
+  write_op.create(false);
+  write_op.truncate(0);
+  write_op.clone_range(0, oid, 0, size);
+  return target_ctx.operate(target_oid, &write_op);
+}
+
+static int do_copy_pool(Rados& rados, const char *src_pool, const char *target_pool)
+{
+  IoCtx src_ctx, target_ctx;
+  int ret = rados.ioctx_create(src_pool, src_ctx);
+  if (ret < 0) {
+    cerr << "cannot open source pool: " << src_pool << std::endl;
+    return ret;
+  }
+  ret = rados.ioctx_create(target_pool, target_ctx);
+  if (ret < 0) {
+    cerr << "cannot open target pool: " << target_pool << std::endl;
+    return ret;
+  }
+  librados::ObjectIterator i = src_ctx.objects_begin();
+  librados::ObjectIterator i_end = src_ctx.objects_end();
+  for (; i != i_end; ++i) {
+    string oid = i->first;
+    string locator = i->second;
+    if (i->second.size())
+      cout << src_pool << ":" << oid << "(@" << locator << ")" << " => "
+           << target_pool << ":" << oid << "(@" << locator << ")" << std::endl;
+    else
+      cout << src_pool << ":" << oid << " => "
+           << target_pool << ":" << oid << std::endl;
+
+
+    target_ctx.locator_set_key(locator);
+    ret = do_copy(src_ctx, oid.c_str(), target_ctx, oid.c_str());
+    if (ret < 0) {
+      cerr << "error copying object: " << cpp_strerror(errno) << std::endl;
+      return ret;
+    }
+  }
+
+  return 0;
+}
+
+static int do_put(IoCtx& io_ctx, const char *objname, const char *infile, int op_size)
+{
+  string oid(objname);
+  bufferlist indata;
+  bool stdio = false;
+  if (strcmp(infile, "-") == 0)
+    stdio = true;
+
+  int ret;
+  int fd = 0;
+  if (!stdio)
+    fd = open(infile, O_RDONLY);
+  if (fd < 0) {
+    cerr << "error reading input file " << infile << ": " << cpp_strerror(errno) << std::endl;
+    return 1;
+  }
+  char *buf = new char[op_size];
+  int count = op_size;
+  uint64_t offset = 0;
+  while (count != 0) {
+    count = read(fd, buf, op_size);
+    if (count < 0) {
+      ret = -errno;
+      cerr << "error reading input file " << infile << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    if (count == 0) {
+      if (!offset) {
+	ret = io_ctx.create(oid, true);
+	if (ret < 0) {
+	  cerr << "WARNING: could not create object: " << oid << std::endl;
+	  goto out;
+	}
+      }
+      continue;
+    }
+    indata.append(buf, count);
+    if (offset == 0)
+      ret = io_ctx.write_full(oid, indata);
+    else
+      ret = io_ctx.write(oid, indata, count, offset);
+    indata.clear();
+
+    if (ret < 0) {
+      goto out;
+    }
+    offset += count;
+  }
+  ret = 0;
+ out:
+  VOID_TEMP_FAILURE_RETRY(close(fd));
+  delete[] buf;
+  return ret;
+}
+
+class RadosWatchCtx : public librados::WatchCtx {
+  string name;
+public:
+  RadosWatchCtx(const char *imgname) : name(imgname) {}
+  virtual ~RadosWatchCtx() {}
+  virtual void notify(uint8_t opcode, uint64_t ver, bufferlist& bl) {
+    string s;
+    try {
+      bufferlist::iterator iter = bl.begin();
+      ::decode(s, iter);
+    } catch (buffer::error *err) {
+      cout << "could not decode bufferlist, buffer length=" << bl.length() << std::endl;
+    }
+    cout << name << " got notification opcode=" << (int)opcode << " ver=" << ver << " msg='" << s << "'" << std::endl;
+  }
+};
+
+static const char alphanum_table[]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
+
+int gen_rand_alphanumeric(char *dest, int size) /* size should be the required string size + 1 */
+{
+  int ret = get_random_bytes(dest, size);
+  if (ret < 0) {
+    cerr << "cannot get random bytes: " << cpp_strerror(ret) << std::endl;
+    return -1;
+  }
+
+  int i;
+  for (i=0; i objs;
+
+  utime_t start_time;
+
+  bool going_down;
+
+public:
+  int read_percent;
+  int num_objs;
+  size_t min_obj_len;
+  uint64_t max_obj_len;
+  size_t min_op_len;
+  size_t max_op_len;
+  size_t max_ops;
+  size_t max_backlog;
+  size_t target_throughput;
+  int run_length;
+
+  enum {
+    OP_READ,
+    OP_WRITE,
+  };
+
+  struct LoadGenOp {
+    int id;
+    int type;
+    string oid;
+    size_t off;
+    size_t len;
+    bufferlist bl;
+    LoadGen *lg;
+    librados::AioCompletion *completion;
+
+    LoadGenOp() {}
+    LoadGenOp(LoadGen *_lg) : lg(_lg), completion(NULL) {}
+  };
+
+  int max_op;
+
+  map pending_ops;
+
+  void gen_op(LoadGenOp *op);
+  uint64_t gen_next_op();
+  void run_op(LoadGenOp *op);
+
+  uint64_t cur_sent_rate() {
+    return total_sent / time_passed();
+  }
+
+  uint64_t cur_completed_rate() {
+    return total_completed / time_passed();
+  }
+
+  uint64_t total_expected() {
+    return target_throughput * time_passed();
+  }
+
+  float time_passed() {
+    utime_t now = ceph_clock_now(g_ceph_context);
+    now -= start_time;
+    uint64_t ns = now.nsec();
+    float total = ns / 1000000000;
+    total += now.sec();
+    return total;
+  }
+
+  Mutex lock;
+  Cond cond;
+
+  LoadGen(Rados *_rados) : rados(_rados), going_down(false), lock("LoadGen") {
+    read_percent = 80;
+    min_obj_len = 1024;
+    max_obj_len = 5ull * 1024ull * 1024ull * 1024ull;
+    min_op_len = 1024;
+    target_throughput = 5 * 1024 * 1024; // B/sec
+    max_op_len = 2 * 1024 * 1024;
+    max_backlog = target_throughput * 2;
+    run_length = 60;
+
+    total_sent = 0;
+    total_completed = 0;
+    num_objs = 200;
+    max_op = 16;
+  }
+  int bootstrap(const char *pool);
+  int run();
+  void cleanup();
+
+  void io_cb(completion_t c, LoadGenOp *op) {
+    total_completed += op->len;
+
+    Mutex::Locker l(lock);
+
+    double rate = (double)cur_completed_rate() / (1024 * 1024);
+    cout.precision(3);
+    cout << "op " << op->id << " completed, throughput=" << rate  << "MB/sec" << std::endl;
+
+    map::iterator iter = pending_ops.find(op->id);
+    if (iter != pending_ops.end())
+      pending_ops.erase(iter);
+
+    if (!going_down)
+      op->completion->release();
+
+    delete op;
+
+    cond.Signal();
+  }
+};
+
+static void _load_gen_cb(completion_t c, void *param)
+{
+  LoadGen::LoadGenOp *op = (LoadGen::LoadGenOp *)param;
+  op->lg->io_cb(c, op);
+}
+
+int LoadGen::bootstrap(const char *pool)
+{
+  char buf[128];
+  int i;
+
+  if (!pool) {
+    cerr << "ERROR: pool name was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  int ret = rados->ioctx_create(pool, io_ctx);
+  if (ret < 0) {
+    cerr << "error opening pool " << pool << ": " << cpp_strerror(ret) << std::endl;
+    return ret;
+  }
+
+  int buf_len = 1;
+  bufferptr p = buffer::create(buf_len);
+  bufferlist bl;
+  memset(p.c_str(), 0, buf_len);
+  bl.push_back(p);
+
+  list completions;
+  for (i = 0; i < num_objs; i++) {
+    obj_info info;
+    gen_rand_alphanumeric(buf, 16);
+    info.name = "obj-";
+    info.name.append(buf);
+    info.len = get_random(min_obj_len, max_obj_len);
+
+    // throttle...
+    while (completions.size() > max_ops) {
+      AioCompletion *c = completions.front();
+      c->wait_for_complete();
+      ret = c->get_return_value();
+      c->release();
+      completions.pop_front();
+      if (ret < 0) {
+	cerr << "aio_write failed" << std::endl;
+	return ret;
+      }
+    }
+
+    librados::AioCompletion *c = rados->aio_create_completion(NULL, NULL, NULL);
+    completions.push_back(c);
+    // generate object
+    ret = io_ctx.aio_write(info.name, c, bl, buf_len, info.len - buf_len);
+    if (ret < 0) {
+      cerr << "couldn't write obj: " << info.name << " ret=" << ret << std::endl;
+      return ret;
+    }
+    objs[i] = info;
+  }
+
+  list::iterator iter;
+  for (iter = completions.begin(); iter != completions.end(); ++iter) {
+    AioCompletion *c = *iter;
+    c->wait_for_complete();
+    ret = c->get_return_value();
+    c->release();
+    if (ret < 0) { // yes, we leak.
+      cerr << "aio_write failed" << std::endl;
+      return ret;
+    }
+  }
+  return 0;
+}
+
+void LoadGen::run_op(LoadGenOp *op)
+{
+  op->completion = rados->aio_create_completion(op, _load_gen_cb, NULL);
+
+  switch (op->type) {
+  case OP_READ:
+    io_ctx.aio_read(op->oid, op->completion, &op->bl, op->len, op->off);
+    break;
+  case OP_WRITE:
+    bufferptr p = buffer::create(op->len);
+    memset(p.c_str(), 0, op->len);
+    op->bl.push_back(p);
+    
+    io_ctx.aio_write(op->oid, op->completion, op->bl, op->len, op->off);
+    break;
+  }
+
+  total_sent += op->len;
+}
+
+void LoadGen::gen_op(LoadGenOp *op)
+{
+  int i = get_random(0, objs.size() - 1);
+  obj_info& info = objs[i];
+  op->oid = info.name;
+
+  size_t len = get_random(min_op_len, max_op_len);
+  if (len > info.len)
+    len = info.len;
+  size_t off = get_random(0, info.len);
+
+  if (off + len > info.len)
+    off = info.len - len;
+
+  op->off = off;
+  op->len = len;
+
+  i = get_random(1, 100);
+  if (i > read_percent)
+    op->type = OP_WRITE;
+  else
+    op->type = OP_READ;
+
+  cout << (op->type == OP_READ ? "READ" : "WRITE") << " : oid=" << op->oid << " off=" << op->off << " len=" << op->len << std::endl;
+}
+
+uint64_t LoadGen::gen_next_op()
+{
+  lock.Lock();
+
+  LoadGenOp *op = new LoadGenOp(this);
+  gen_op(op);
+  op->id = max_op++;
+  pending_ops[op->id] = op;
+
+  lock.Unlock();
+
+  run_op(op);
+
+  return op->len;
+}
+
+int LoadGen::run()
+{
+  start_time = ceph_clock_now(g_ceph_context);
+  utime_t end_time = start_time;
+  end_time += run_length;
+  utime_t stamp_time = start_time;
+  uint32_t total_sec = 0;
+
+  while (1) {
+    lock.Lock();
+    utime_t one_second(1, 0);
+    cond.WaitInterval(g_ceph_context, lock, one_second);
+    lock.Unlock();
+    utime_t now = ceph_clock_now(g_ceph_context);
+
+    if (now > end_time)
+      break;
+
+    uint64_t expected = total_expected();  
+    lock.Lock();
+    uint64_t sent = total_sent;
+    uint64_t completed = total_completed;
+    lock.Unlock();
+
+    if (now - stamp_time >= utime_t(1, 0)) {
+      double rate = (double)cur_completed_rate() / (1024 * 1024);
+      ++total_sec;
+      cout.precision(3);
+      cout << setw(5) << total_sec << ": throughput=" << rate  << "MB/sec" << " pending data=" << sent - completed << std::endl;
+      stamp_time = now; 
+    }
+
+    while (sent < expected &&
+           sent - completed < max_backlog &&
+	   pending_ops.size() < max_ops) {
+      sent += gen_next_op();
+    }
+  }
+
+  // get a reference to all pending requests
+  vector completions;
+  lock.Lock();
+  going_down = true;
+  map::iterator iter;
+  for (iter = pending_ops.begin(); iter != pending_ops.end(); ++iter) {
+    LoadGenOp *op = iter->second;
+    completions.push_back(op->completion);
+  }
+  lock.Unlock();
+
+  cout << "waiting for all operations to complete" << std::endl;
+
+  // now wait on all the pending requests
+  for (vector::iterator citer = completions.begin(); citer != completions.end(); ++citer) {
+    librados::AioCompletion *c = *citer;
+    c->wait_for_complete();
+    c->release();
+  }
+
+  return 0;
+}
+
+void LoadGen::cleanup()
+{
+  cout << "cleaning up objects" << std::endl;
+  map::iterator iter;
+  for (iter = objs.begin(); iter != objs.end(); ++iter) {
+    obj_info& info = iter->second;
+    int ret = io_ctx.remove(info.name);
+    if (ret < 0)
+      cerr << "couldn't remove obj: " << info.name << " ret=" << ret << std::endl;
+  }
+}
+
+
+class RadosBencher : public ObjBencher {
+  librados::AioCompletion **completions;
+  librados::Rados& rados;
+  librados::IoCtx& io_ctx;
+  librados::ObjectIterator oi;
+  bool iterator_valid;
+protected:
+  int completions_init(int concurrentios) {
+    completions = new librados::AioCompletion *[concurrentios];
+    return 0;
+  }
+  void completions_done() {
+    delete[] completions;
+    completions = NULL;
+  }
+  int create_completion(int slot, void (*cb)(void *, void*), void *arg) {
+    completions[slot] = rados.aio_create_completion((void *) arg, 0, cb);
+
+    if (!completions[slot])
+      return -EINVAL;
+
+    return 0;
+  }
+  void release_completion(int slot) {
+    completions[slot]->release();
+    completions[slot] = 0;
+  }
+
+  int aio_read(const std::string& oid, int slot, bufferlist *pbl, size_t len) {
+    return io_ctx.aio_read(oid, completions[slot], pbl, len, 0);
+  }
+
+  int aio_write(const std::string& oid, int slot, bufferlist& bl, size_t len) {
+    return io_ctx.aio_write(oid, completions[slot], bl, len, 0);
+  }
+
+  int aio_remove(const std::string& oid, int slot) {
+    return io_ctx.aio_remove(oid, completions[slot]);
+  }
+
+  int sync_read(const std::string& oid, bufferlist& bl, size_t len) {
+    return io_ctx.read(oid, bl, len, 0);
+  }
+  int sync_write(const std::string& oid, bufferlist& bl, size_t len) {
+    return io_ctx.write_full(oid, bl);
+  }
+
+  int sync_remove(const std::string& oid) {
+    return io_ctx.remove(oid);
+  }
+
+  bool completion_is_done(int slot) {
+    return completions[slot]->is_safe();
+  }
+
+  int completion_wait(int slot) {
+    return completions[slot]->wait_for_safe_and_cb();
+  }
+  int completion_ret(int slot) {
+    return completions[slot]->get_return_value();
+  }
+
+  bool get_objects(std::list* objects, int num) {
+    int count = 0;
+
+    if (!iterator_valid) {
+      oi = io_ctx.objects_begin();
+      iterator_valid = true;
+    }
+
+    librados::ObjectIterator ei = io_ctx.objects_end();
+
+    if (oi == ei) {
+      iterator_valid = false;
+      return false;
+    }
+
+    objects->clear();
+    for ( ; oi != ei && count < num; ++oi) {
+      objects->push_back(oi->first);
+      ++count;
+    }
+
+    return true;
+  }
+
+public:
+  RadosBencher(CephContext *cct_, librados::Rados& _r, librados::IoCtx& _i)
+    : ObjBencher(cct_), completions(NULL), rados(_r), io_ctx(_i), iterator_valid(false) {}
+  ~RadosBencher() { }
+};
+
+static int do_lock_cmd(std::vector &nargs,
+                       const std::map < std::string, std::string > &opts,
+                       IoCtx *ioctx,
+		       Formatter *formatter)
+{
+  if (nargs.size() < 3)
+    usage_exit();
+
+  string cmd(nargs[1]);
+  string oid(nargs[2]);
+
+  string lock_tag;
+  string lock_cookie;
+  string lock_description;
+  int lock_duration = 0;
+  ClsLockType lock_type = LOCK_EXCLUSIVE;
+
+  map::const_iterator i;
+  i = opts.find("lock-tag");
+  if (i != opts.end()) {
+    lock_tag = i->second;
+  }
+  i = opts.find("lock-cookie");
+  if (i != opts.end()) {
+    lock_cookie = i->second;
+  }
+  i = opts.find("lock-description");
+  if (i != opts.end()) {
+    lock_description = i->second;
+  }
+  i = opts.find("lock-duration");
+  if (i != opts.end()) {
+    lock_duration = strtol(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("lock-type");
+  if (i != opts.end()) {
+    const string& type_str = i->second;
+    if (type_str.compare("exclusive") == 0) {
+      lock_type = LOCK_EXCLUSIVE;
+    } else if (type_str.compare("shared") == 0) {
+      lock_type = LOCK_SHARED;
+    } else {
+      cerr << "unknown lock type was specified, aborting" << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  if (cmd.compare("list") == 0) {
+    list locks;
+    int ret = rados::cls::lock::list_locks(ioctx, oid, &locks);
+    if (ret < 0) {
+      cerr << "ERROR: rados_list_locks(): " << cpp_strerror(ret) << std::endl;
+      return ret;
+    }
+
+    formatter->open_object_section("object");
+    formatter->dump_string("objname", oid);
+    formatter->open_array_section("locks");
+    list::iterator iter;
+    for (iter = locks.begin(); iter != locks.end(); ++iter) {
+      formatter->open_object_section("lock");
+      formatter->dump_string("name", *iter);
+      formatter->close_section();
+    }
+    formatter->close_section();
+    formatter->close_section();
+    formatter->flush(cout);
+    return 0;
+  }
+
+  if (nargs.size() < 4)
+    usage_exit();
+
+  string lock_name(nargs[3]);
+
+  if (cmd.compare("info") == 0) {
+    map lockers;
+    ClsLockType type = LOCK_NONE;
+    string tag;
+    int ret = rados::cls::lock::get_lock_info(ioctx, oid, lock_name, &lockers, &type, &tag);
+    if (ret < 0) {
+      cerr << "ERROR: rados_lock_get_lock_info(): " << cpp_strerror(ret) << std::endl;
+      return ret;
+    }
+
+    formatter->open_object_section("lock");
+    formatter->dump_string("name", lock_name);
+    formatter->dump_string("type", cls_lock_type_str(type));
+    formatter->dump_string("tag", tag);
+    formatter->open_array_section("lockers");
+    map::iterator iter;
+    for (iter = lockers.begin(); iter != lockers.end(); ++iter) {
+      const rados::cls::lock::locker_id_t& id = iter->first;
+      const rados::cls::lock::locker_info_t& info = iter->second;
+      formatter->open_object_section("locker");
+      formatter->dump_stream("name") << id.locker;
+      formatter->dump_string("cookie", id.cookie);
+      formatter->dump_string("description", info.description);
+      formatter->dump_stream("expiration") << info.expiration;
+      formatter->dump_stream("addr") << info.addr;
+      formatter->close_section();
+    }
+    formatter->close_section();
+    formatter->close_section();
+    formatter->flush(cout);
+    
+    return ret;
+  } else if (cmd.compare("get") == 0) {
+    rados::cls::lock::Lock l(lock_name);
+    l.set_cookie(lock_cookie);
+    l.set_tag(lock_tag);
+    l.set_duration(utime_t(lock_duration, 0));
+    l.set_description(lock_description);
+    int ret;
+    switch (lock_type) {
+    case LOCK_SHARED:
+      ret = l.lock_shared(ioctx, oid);
+      break;
+    default:
+      ret = l.lock_exclusive(ioctx, oid);
+    }
+    if (ret < 0) {
+      cerr << "ERROR: failed locking: " << cpp_strerror(ret) << std::endl;
+      return ret;
+    }
+
+    return ret;
+  }
+
+  if (nargs.size() < 5)
+    usage_exit();
+
+  if (cmd.compare("break") == 0) {
+    string locker(nargs[4]);
+    rados::cls::lock::Lock l(lock_name);
+    l.set_cookie(lock_cookie);
+    l.set_tag(lock_tag);
+    entity_name_t name;
+    if (!name.parse(locker)) {
+      cerr << "ERROR: failed to parse locker name (" << locker << ")" << std::endl;
+      return -EINVAL;
+    }
+    int ret = l.break_lock(ioctx, oid, name);
+    if (ret < 0) {
+      cerr << "ERROR: failed breaking lock: " << cpp_strerror(ret) << std::endl;
+      return ret;
+    }
+  } else {
+    usage_exit();
+  }
+
+  return 0;
+}
+
+static int do_cache_flush(IoCtx& io_ctx, string oid)
+{
+  ObjectReadOperation op;
+  op.cache_flush();
+  librados::AioCompletion *completion =
+    librados::Rados::aio_create_completion();
+  io_ctx.aio_operate(oid.c_str(), completion, &op,
+		     librados::OPERATION_IGNORE_CACHE |
+		     librados::OPERATION_IGNORE_OVERLAY,
+		     NULL);
+  completion->wait_for_safe();
+  int r = completion->get_return_value();
+  completion->release();
+  return r;
+}
+
+static int do_cache_try_flush(IoCtx& io_ctx, string oid)
+{
+  ObjectReadOperation op;
+  op.cache_try_flush();
+  librados::AioCompletion *completion =
+    librados::Rados::aio_create_completion();
+  io_ctx.aio_operate(oid.c_str(), completion, &op,
+		     librados::OPERATION_IGNORE_CACHE |
+		     librados::OPERATION_IGNORE_OVERLAY |
+		     librados::OPERATION_SKIPRWLOCKS,
+		     NULL);
+  completion->wait_for_safe();
+  int r = completion->get_return_value();
+  completion->release();
+  return r;
+}
+
+static int do_cache_evict(IoCtx& io_ctx, string oid)
+{
+  ObjectReadOperation op;
+  op.cache_evict();
+  librados::AioCompletion *completion =
+    librados::Rados::aio_create_completion();
+  io_ctx.aio_operate(oid.c_str(), completion, &op,
+		     librados::OPERATION_IGNORE_CACHE |
+		     librados::OPERATION_IGNORE_OVERLAY |
+		     librados::OPERATION_SKIPRWLOCKS,
+		     NULL);
+  completion->wait_for_safe();
+  int r = completion->get_return_value();
+  completion->release();
+  return r;
+}
+
+static int do_cache_flush_evict_all(IoCtx& io_ctx, bool blocking)
+{
+  int r;
+  int errors = 0;
+  try {
+    librados::ObjectIterator i = io_ctx.objects_begin();
+    librados::ObjectIterator i_end = io_ctx.objects_end();
+    for (; i != i_end; ++i) {
+      cout << i->first << "\t" << i->second << std::endl;
+      if (i->second.size()) {
+	io_ctx.locator_set_key(i->second);
+      } else {
+	io_ctx.locator_set_key(string());
+      }
+      if (blocking)
+	r = do_cache_flush(io_ctx, i->first);
+      else
+	r = do_cache_try_flush(io_ctx, i->first);
+      if (r < 0) {
+	cerr << "failed to flush " << i->first << ": "
+	     << cpp_strerror(r) << std::endl;
+	++errors;
+	continue;
+      }
+      r = do_cache_evict(io_ctx, i->first);
+      if (r < 0) {
+	cerr << "failed to evict " << i->first << ": "
+	     << cpp_strerror(r) << std::endl;
+	++errors;
+	continue;
+      }
+    }
+  }
+  catch (const std::runtime_error& e) {
+    cerr << e.what() << std::endl;
+    return -1;
+  }
+  return errors ? -1 : 0;
+}
+
+/**********************************************
+
+**********************************************/
+static int rados_tool_common(const std::map < std::string, std::string > &opts,
+                             std::vector &nargs)
+{
+  int ret;
+  bool create_pool = false;
+  const char *pool_name = NULL;
+  const char *target_pool_name = NULL;
+  string oloc, target_oloc, nspace;
+  int concurrent_ios = 16;
+  int op_size = 1 << 22;
+  bool cleanup = true;
+  const char *snapname = NULL;
+  snap_t snapid = CEPH_NOSNAP;
+  std::map::const_iterator i;
+  std::string category;
+
+  uint64_t min_obj_len = 0;
+  uint64_t max_obj_len = 0;
+  uint64_t min_op_len = 0;
+  uint64_t max_op_len = 0;
+  uint64_t max_ops = 0;
+  uint64_t max_backlog = 0;
+  uint64_t target_throughput = 0;
+  int64_t read_percent = -1;
+  uint64_t num_objs = 0;
+  int run_length = 0;
+
+  bool show_time = false;
+
+  const char* run_name = NULL;
+  const char* prefix = NULL;
+
+  Formatter *formatter = NULL;
+  bool pretty_format = false;
+
+  Rados rados;
+  IoCtx io_ctx;
+
+  i = opts.find("create");
+  if (i != opts.end()) {
+    create_pool = true;
+  }
+  i = opts.find("pool");
+  if (i != opts.end()) {
+    pool_name = i->second.c_str();
+  }
+  i = opts.find("target_pool");
+  if (i != opts.end()) {
+    target_pool_name = i->second.c_str();
+  }
+  i = opts.find("object_locator");
+  if (i != opts.end()) {
+    oloc = i->second;
+  }
+  i = opts.find("target_locator");
+  if (i != opts.end()) {
+    target_oloc = i->second;
+  }
+  i = opts.find("category");
+  if (i != opts.end()) {
+    category = i->second;
+  }
+  i = opts.find("concurrent-ios");
+  if (i != opts.end()) {
+    concurrent_ios = strtol(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("run-name");
+  if (i != opts.end()) {
+    run_name = i->second.c_str();
+  }
+  i = opts.find("prefix");
+  if (i != opts.end()) {
+    prefix = i->second.c_str();
+  }
+  i = opts.find("block-size");
+  if (i != opts.end()) {
+    op_size = strtol(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("snap");
+  if (i != opts.end()) {
+    snapname = i->second.c_str();
+  }
+  i = opts.find("snapid");
+  if (i != opts.end()) {
+    snapid = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("min-object-size");
+  if (i != opts.end()) {
+    min_obj_len = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("max-object-size");
+  if (i != opts.end()) {
+    max_obj_len = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("min-op-len");
+  if (i != opts.end()) {
+    min_op_len = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("max-op-len");
+  if (i != opts.end()) {
+    max_op_len = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("max-ops");
+  if (i != opts.end()) {
+    max_ops = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("max-backlog");
+  if (i != opts.end()) {
+    max_backlog = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("target-throughput");
+  if (i != opts.end()) {
+    target_throughput = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("read-percent");
+  if (i != opts.end()) {
+    read_percent = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("num-objects");
+  if (i != opts.end()) {
+    num_objs = strtoll(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("run-length");
+  if (i != opts.end()) {
+    run_length = strtol(i->second.c_str(), NULL, 10);
+  }
+  i = opts.find("show-time");
+  if (i != opts.end()) {
+    show_time = true;
+  }
+  i = opts.find("no-cleanup");
+  if (i != opts.end()) {
+    cleanup = false;
+  }
+  i = opts.find("pretty-format");
+  if (i != opts.end()) {
+    pretty_format = true;
+  }
+  i = opts.find("format");
+  if (i != opts.end()) {
+    const char *format = i->second.c_str();
+    if (strcmp(format, "xml") == 0)
+      formatter = new XMLFormatter(pretty_format);
+    else if (strcmp(format, "json") == 0)
+      formatter = new JSONFormatter(pretty_format);
+    else {
+      cerr << "unrecognized format: " << format << std::endl;
+      return -EINVAL;
+    }
+  }
+  i = opts.find("namespace");
+  if (i != opts.end()) {
+    nspace = i->second;
+  }
+
+
+  // open rados
+  ret = rados.init_with_context(g_ceph_context);
+  if (ret) {
+     cerr << "couldn't initialize rados! error " << ret << std::endl;
+     ret = -1;
+     goto out;
+  }
+
+  ret = rados.connect();
+  if (ret) {
+     cerr << "couldn't connect to cluster! error " << ret << std::endl;
+     ret = -1;
+     goto out;
+  }
+
+  if (create_pool && !pool_name) {
+    cerr << "--create-pool requested but pool_name was not specified!" << std::endl;
+    usage_exit();
+  }
+
+  if (create_pool) {
+    ret = rados.pool_create(pool_name, 0, 0);
+    if (ret < 0) {
+      cerr << "error creating pool " << pool_name << ": "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  }
+
+  // open io context.
+  if (pool_name) {
+    ret = rados.ioctx_create(pool_name, io_ctx);
+    if (ret < 0) {
+      cerr << "error opening pool " << pool_name << ": "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+
+    // align op_size
+    if (io_ctx.pool_requires_alignment()) {
+      const uint64_t align = io_ctx.pool_required_alignment();
+      const bool wrn = (op_size != (1<<22));
+      op_size = uint64_t((op_size + align - 1) / align) * align;
+      if (wrn)
+	cerr << "INFO: op_size has been rounded to " << op_size << std::endl;
+    }
+  }
+
+  // snapname?
+  if (snapname) {
+    ret = io_ctx.snap_lookup(snapname, &snapid);
+    if (ret < 0) {
+      cerr << "error looking up snap '" << snapname << "': " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  }
+  if (oloc.size()) {
+    io_ctx.locator_set_key(oloc);
+  }
+  if (!nspace.empty()) {
+    io_ctx.set_namespace(nspace);
+  }
+  if (snapid != CEPH_NOSNAP) {
+    string name;
+    ret = io_ctx.snap_get_name(snapid, &name);
+    if (ret < 0) {
+      cerr << "snapid " << snapid << " doesn't exist in pool "
+	   << io_ctx.get_pool_name() << std::endl;
+      goto out;
+    }
+    io_ctx.snap_set_read(snapid);
+    cout << "selected snap " << snapid << " '" << snapname << "'" << std::endl;
+  }
+
+  assert(!nargs.empty());
+
+  // list pools?
+  if (strcmp(nargs[0], "lspools") == 0) {
+    list vec;
+    ret = rados.pool_list(vec);
+    if (ret < 0) {
+      cerr << "error listing pools: " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    for (list::iterator i = vec.begin(); i != vec.end(); ++i)
+      cout << *i << std::endl;
+  }
+  else if (strcmp(nargs[0], "df") == 0) {
+    // pools
+    list vec;
+
+    if (!pool_name) {
+      ret = rados.pool_list(vec);
+      if (ret < 0) {
+	cerr << "error listing pools: " << cpp_strerror(ret) << std::endl;
+	goto out;
+      }
+    } else {
+      vec.push_back(pool_name);
+    }
+
+    map > stats;
+    ret = rados.get_pool_stats(vec, category, stats);
+    if (ret < 0) {
+      cerr << "error fetching pool stats: " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+
+    if (!formatter) {
+      printf("%-15s %-15s"
+	     "%12s %12s %12s %12s "
+	     "%12s %12s %12s %12s %12s\n",
+	     "pool name",
+	     "category",
+	     "KB", "objects", "clones", "degraded",
+	     "unfound", "rd", "rd KB", "wr", "wr KB");
+    } else {
+      formatter->open_object_section("stats");
+      formatter->open_array_section("pools");
+    }
+    for (map::iterator c = stats.begin(); c != stats.end(); ++c) {
+      const char *pool_name = c->first.c_str();
+      stats_map& m = c->second;
+      if (formatter) {
+        formatter->open_object_section("pool");
+        int64_t pool_id = rados.pool_lookup(pool_name);
+        formatter->dump_string("name", pool_name);
+        if (pool_id >= 0)
+          formatter->dump_format("id", "%lld", pool_id);
+        else
+          cerr << "ERROR: lookup_pg_pool_name for name=" << pool_name << " returned " << pool_id << std::endl;
+        formatter->open_array_section("categories");
+      }
+      for (stats_map::iterator i = m.begin(); i != m.end(); ++i) {
+        const char *category = (i->first.size() ? i->first.c_str() : "");
+        pool_stat_t& s = i->second;
+        if (!formatter) {
+          if (!*category)
+            category = "-";
+          printf("%-15s "
+                 "%-15s "
+	         "%12lld %12lld %12lld %12lld"
+	         "%12lld %12lld %12lld %12lld %12lld\n",
+	         pool_name,
+                 category,
+	         (long long)s.num_kb,
+	         (long long)s.num_objects,
+	         (long long)s.num_object_clones,
+	         (long long)s.num_objects_degraded,
+	         (long long)s.num_objects_unfound,
+	         (long long)s.num_rd, (long long)s.num_rd_kb,
+	         (long long)s.num_wr, (long long)s.num_wr_kb);
+        } else {
+          formatter->open_object_section("category");
+          if (category)
+            formatter->dump_string("name", category);
+          formatter->dump_format("size_bytes", "%lld", s.num_bytes);
+          formatter->dump_format("size_kb", "%lld", s.num_kb);
+          formatter->dump_format("num_objects", "%lld", s.num_objects);
+          formatter->dump_format("num_object_clones", "%lld", s.num_object_clones);
+          formatter->dump_format("num_object_copies", "%lld", s.num_object_copies);
+          formatter->dump_format("num_objects_missing_on_primary", "%lld", s.num_objects_missing_on_primary);
+          formatter->dump_format("num_objects_unfound", "%lld", s.num_objects_unfound);
+          formatter->dump_format("num_objects_degraded", "%lld", s.num_objects_degraded);
+          formatter->dump_format("read_bytes", "%lld", s.num_rd);
+          formatter->dump_format("read_kb", "%lld", s.num_rd_kb);
+          formatter->dump_format("write_bytes", "%lld", s.num_wr);
+          formatter->dump_format("write_kb", "%lld", s.num_wr_kb);
+          formatter->flush(cout);
+        }
+        if (formatter) {
+          formatter->close_section();
+        }
+      }
+      if (formatter) {
+        formatter->close_section();
+        formatter->close_section();
+        formatter->flush(cout);
+      }
+    }
+
+    // total
+    cluster_stat_t tstats;
+    ret = rados.cluster_stat(tstats);
+    if (ret < 0) {
+      cerr << "error getting total cluster usage: " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    if (!formatter) {
+      printf("  total used    %12lld %12lld\n", (long long unsigned)tstats.kb_used,
+	     (long long unsigned)tstats.num_objects);
+      printf("  total avail   %12lld\n", (long long unsigned)tstats.kb_avail);
+      printf("  total space   %12lld\n", (long long unsigned)tstats.kb);
+    } else {
+      formatter->close_section();
+      formatter->dump_format("total_objects", "%lld", (long long unsigned)tstats.num_objects);
+      formatter->dump_format("total_used", "%lld", (long long unsigned)tstats.kb_used);
+      formatter->dump_format("total_avail", "%lld", (long long unsigned)tstats.kb_avail);
+      formatter->dump_format("total_space", "%lld", (long long unsigned)tstats.kb);
+      formatter->close_section();
+      formatter->flush(cout);
+    }
+  }
+
+  else if (strcmp(nargs[0], "ls") == 0) {
+    if (!pool_name) {
+      cerr << "pool name was not specified" << std::endl;
+      ret = -1;
+      goto out;
+    }
+
+    bool stdout = (nargs.size() < 2) || (strcmp(nargs[1], "-") == 0);
+    ostream *outstream;
+    if(stdout)
+      outstream = &cout;
+    else
+      outstream = new ofstream(nargs[1]);
+
+    {
+      try {
+	librados::ObjectIterator i = io_ctx.objects_begin();
+	librados::ObjectIterator i_end = io_ctx.objects_end();
+	for (; i != i_end; ++i) {
+	  if (i->second.size())
+	    *outstream << i->first << "\t" << i->second << std::endl;
+	  else
+	    *outstream << i->first << std::endl;
+	}
+      }
+      catch (const std::runtime_error& e) {
+	cerr << e.what() << std::endl;
+	ret = -1;
+	goto out;
+      }
+    }
+    if (!stdout)
+      delete outstream;
+  }
+  else if (strcmp(nargs[0], "chown") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    uint64_t new_auid = strtol(nargs[1], 0, 10);
+    ret = io_ctx.set_auid(new_auid);
+    if (ret < 0) {
+      cerr << "error changing auid on pool " << io_ctx.get_pool_name() << ':'
+	   << cpp_strerror(ret) << std::endl;
+    } else cerr << "changed auid on pool " << io_ctx.get_pool_name()
+		<< " to " << new_auid << std::endl;
+  }
+  else if (strcmp(nargs[0], "mapext") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+    std::map m;
+    ret = io_ctx.mapext(oid, 0, -1, m);
+    if (ret < 0) {
+      cerr << "mapext error on " << pool_name << "/" << oid << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    std::map::iterator iter;
+    for (iter = m.begin(); iter != m.end(); ++iter) {
+      cout << hex << iter->first << "\t" << iter->second << dec << std::endl;
+    }
+  }
+  else if (strcmp(nargs[0], "stat") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+    uint64_t size;
+    time_t mtime;
+    ret = io_ctx.stat(oid, &size, &mtime);
+    if (ret < 0) {
+      cerr << " error stat-ing " << pool_name << "/" << oid << ": "
+           << cpp_strerror(ret) << std::endl;
+      goto out;
+    } else {
+      cout << pool_name << "/" << oid
+           << " mtime " << mtime << ", size " << size << std::endl;
+    }
+  }
+  else if (strcmp(nargs[0], "get") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+    ret = do_get(io_ctx, nargs[1], nargs[2], op_size);
+    if (ret < 0) {
+      cerr << "error getting " << pool_name << "/" << nargs[1] << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  }
+  else if (strcmp(nargs[0], "put") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+    ret = do_put(io_ctx, nargs[1], nargs[2], op_size);
+    if (ret < 0) {
+      cerr << "error putting " << pool_name << "/" << nargs[1] << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  }
+  else if (strcmp(nargs[0], "truncate") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+
+    string oid(nargs[1]);
+    long size = atol(nargs[2]);
+    if (size < 0) {
+      cerr << "error, cannot truncate to negative value" << std::endl;
+      usage_exit();
+    }
+    ret = io_ctx.trunc(oid, size);
+    if (ret < 0) {
+      cerr << "error truncating oid "
+	   << oid << " to " << size << ": "
+	   << cpp_strerror(ret) << std::endl;
+    } else {
+      ret = 0;
+    }
+  }
+  else if (strcmp(nargs[0], "setxattr") == 0) {
+    if (!pool_name || nargs.size() < 4)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string attr_name(nargs[2]);
+    string attr_val(nargs[3]);
+
+    bufferlist bl;
+    bl.append(attr_val.c_str(), attr_val.length());
+
+    ret = io_ctx.setxattr(oid, attr_name.c_str(), bl);
+    if (ret < 0) {
+      cerr << "error setting xattr " << pool_name << "/" << oid << "/" << attr_name << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    else
+      ret = 0;
+  }
+  else if (strcmp(nargs[0], "getxattr") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string attr_name(nargs[2]);
+
+    bufferlist bl;
+    ret = io_ctx.getxattr(oid, attr_name.c_str(), bl);
+    if (ret < 0) {
+      cerr << "error getting xattr " << pool_name << "/" << oid << "/" << attr_name << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    else
+      ret = 0;
+    string s(bl.c_str(), bl.length());
+    cout << s << std::endl;
+  } else if (strcmp(nargs[0], "rmxattr") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string attr_name(nargs[2]);
+
+    ret = io_ctx.rmxattr(oid, attr_name.c_str());
+    if (ret < 0) {
+      cerr << "error removing xattr " << pool_name << "/" << oid << "/" << attr_name << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "listxattr") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    string oid(nargs[1]);
+    map attrset;
+    bufferlist bl;
+    ret = io_ctx.getxattrs(oid, attrset);
+    if (ret < 0) {
+      cerr << "error getting xattr set " << pool_name << "/" << oid << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+
+    for (map::iterator iter = attrset.begin();
+         iter != attrset.end(); ++iter) {
+      cout << iter->first << std::endl;
+    }
+  } else if (strcmp(nargs[0], "getomapheader") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string outfile;
+    if (nargs.size() >= 3) {
+      outfile = nargs[2];
+    }
+
+    bufferlist header;
+    ret = io_ctx.omap_get_header(oid, &header);
+    if (ret < 0) {
+      cerr << "error getting omap header " << pool_name << "/" << oid
+	   << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    } else {
+      if (!outfile.empty()) {
+	cerr << "Writing to " << outfile << std::endl;
+	dump_data(outfile, header);
+      } else {
+	cout << "header (" << header.length() << " bytes) :\n";
+	header.hexdump(cout);
+	cout << std::endl;
+      }
+      ret = 0;
+    }
+  } else if (strcmp(nargs[0], "setomapheader") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string val(nargs[2]);
+
+    bufferlist bl;
+    bl.append(val);
+
+    ret = io_ctx.omap_set_header(oid, bl);
+    if (ret < 0) {
+      cerr << "error setting omap value " << pool_name << "/" << oid
+	   << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    } else {
+      ret = 0;
+    }
+  } else if (strcmp(nargs[0], "setomapval") == 0) {
+    if (!pool_name || nargs.size() < 4)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string key(nargs[2]);
+    string val(nargs[3]);
+
+    map values;
+    bufferlist bl;
+    bl.append(val);
+    values[key] = bl;
+
+    ret = io_ctx.omap_set(oid, values);
+    if (ret < 0) {
+      cerr << "error setting omap value " << pool_name << "/" << oid << "/"
+	   << key << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    } else {
+      ret = 0;
+    }
+  } else if (strcmp(nargs[0], "getomapval") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string key(nargs[2]);
+    set keys;
+    keys.insert(key);
+
+    std::string outfile;
+    if (nargs.size() >= 4) {
+      outfile = nargs[3];
+    }
+
+    map values;
+    ret = io_ctx.omap_get_vals_by_keys(oid, keys, &values);
+    if (ret < 0) {
+      cerr << "error getting omap value " << pool_name << "/" << oid << "/"
+	   << key << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    } else {
+      ret = 0;
+    }
+
+    if (values.size() && values.begin()->first == key) {
+      cout << " (length " << values.begin()->second.length() << ") : ";
+      if (!outfile.empty()) {
+	cerr << "Writing to " << outfile << std::endl;
+	dump_data(outfile, values.begin()->second);
+      } else {
+	values.begin()->second.hexdump(cout);
+	cout << std::endl;
+      }
+      ret = 0;
+    } else {
+      cout << "No such key: " << pool_name << "/" << oid << "/" << key
+	   << std::endl;
+      ret = -1;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "rmomapkey") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string key(nargs[2]);
+    set keys;
+    keys.insert(key);
+
+    ret = io_ctx.omap_rm_keys(oid, keys);
+    if (ret < 0) {
+      cerr << "error removing omap key " << pool_name << "/" << oid << "/"
+	   << key << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    } else {
+      ret = 0;
+    }
+  } else if (strcmp(nargs[0], "listomapvals") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    string oid(nargs[1]);
+    string last_read = "";
+    int MAX_READ = 512;
+    do {
+      map values;
+      ret = io_ctx.omap_get_vals(oid, last_read, MAX_READ, &values);
+      if (ret < 0) {
+	cerr << "error getting omap keys " << pool_name << "/" << oid << ": "
+	     << cpp_strerror(ret) << std::endl;
+	return 1;
+      }
+      ret = values.size();
+      for (map::const_iterator it = values.begin();
+	   it != values.end(); ++it) {
+	last_read = it->first;
+	// dump key in hex if it contains nonprintable characters
+	if (std::count_if(it->first.begin(), it->first.end(),
+	    (int (*)(int))isprint) < (int)it->first.length()) {
+	  cout << "key: (" << it->first.length() << " bytes):\n";
+	  bufferlist keybl;
+	  keybl.append(it->first);
+	  keybl.hexdump(cout);
+	} else {
+	  cout << it->first;
+	}
+	cout << std::endl;
+	cout << "value: (" << it->second.length() << " bytes) :\n";
+	it->second.hexdump(cout);
+	cout << std::endl;
+      }
+    } while (ret == MAX_READ);
+    ret = 0;
+  }
+  else if (strcmp(nargs[0], "cp") == 0) {
+    if (!pool_name)
+      usage_exit();
+
+    if (nargs.size() < 2 || nargs.size() > 3)
+      usage_exit();
+
+    const char *target = target_pool_name;
+    if (!target)
+      target = pool_name;
+
+    const char *target_obj;
+    if (nargs.size() < 3) {
+      if (strcmp(target, pool_name) == 0) {
+        cerr << "cannot copy object into itself" << std::endl;
+	ret = -1;
+	goto out;
+      }
+      target_obj = nargs[1];
+    } else {
+      target_obj = nargs[2];
+    }
+
+    // open io context.
+    IoCtx target_ctx;
+    ret = rados.ioctx_create(target, target_ctx);
+    if (ret < 0) {
+      cerr << "error opening target pool " << target << ": "
+           << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    if (target_oloc.size()) {
+      target_ctx.locator_set_key(target_oloc);
+    }
+
+    ret = do_copy(io_ctx, nargs[1], target_ctx, target_obj);
+    if (ret < 0) {
+      cerr << "error copying " << pool_name << "/" << nargs[1] << " => " << target << "/" << target_obj << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  }
+  else if (strcmp(nargs[0], "clonedata") == 0) {
+    if (!pool_name)
+      usage_exit();
+
+    if (nargs.size() < 2 || nargs.size() > 3)
+      usage_exit();
+
+    const char *target = target_pool_name;
+    if (!target)
+      target = pool_name;
+
+    const char *target_obj;
+    if (nargs.size() < 3) {
+      if (strcmp(target, pool_name) == 0) {
+        cerr << "cannot copy object into itself" << std::endl;
+        ret = -1;
+	goto out;
+      }
+      target_obj = nargs[1];
+    } else {
+      target_obj = nargs[2];
+    }
+
+    // open io context.
+    IoCtx target_ctx;
+    ret = rados.ioctx_create(target, target_ctx);
+    if (ret < 0) {
+      cerr << "error opening target pool " << target << ": "
+           << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    if (oloc.size()) {
+      target_ctx.locator_set_key(oloc);
+    } else {
+      cerr << "must specify locator for clone" << std::endl;
+      ret = -1;
+      goto out;
+    }
+
+    ret = do_clone_data(io_ctx, nargs[1], target_ctx, target_obj);
+    if (ret < 0) {
+      cerr << "error cloning " << pool_name << "/" << nargs[1] << " => " << target << "/" << target_obj << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "rm") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    vector::iterator iter = nargs.begin();
+    ++iter;
+    for (; iter != nargs.end(); ++iter) {
+      const string & oid = *iter;
+      ret = io_ctx.remove(oid);
+      if (ret < 0) {
+        cerr << "error removing " << pool_name << "/" << oid << ": " << cpp_strerror(ret) << std::endl;
+        goto out;
+      }
+    }
+  }
+  else if (strcmp(nargs[0], "create") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+    if (nargs.size() > 2) {
+      string category(nargs[2]);
+      ret = io_ctx.create(oid, true, category);
+    } else {
+      ret = io_ctx.create(oid, true);
+    }
+    if (ret < 0) {
+      cerr << "error creating " << pool_name << "/" << oid << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  }
+
+  else if (strcmp(nargs[0], "tmap") == 0) {
+    if (nargs.size() < 3)
+      usage_exit();
+    if (strcmp(nargs[1], "dump") == 0) {
+      bufferlist outdata;
+      string oid(nargs[2]);
+      ret = io_ctx.read(oid, outdata, 0, 0);
+      if (ret < 0) {
+	cerr << "error reading " << pool_name << "/" << oid << ": " << cpp_strerror(ret) << std::endl;
+	goto out;
+      }
+      bufferlist::iterator p = outdata.begin();
+      bufferlist header;
+      map kv;
+      try {
+	::decode(header, p);
+	::decode(kv, p);
+      }
+      catch (buffer::error& e) {
+	cerr << "error decoding tmap " << pool_name << "/" << oid << std::endl;
+	ret = -EINVAL;
+	goto out;
+      }
+      cout << "header (" << header.length() << " bytes):\n";
+      header.hexdump(cout);
+      cout << "\n";
+      cout << kv.size() << " keys\n";
+      for (map::iterator q = kv.begin(); q != kv.end(); ++q) {
+	cout << "key '" << q->first << "' (" << q->second.length() << " bytes):\n";
+	q->second.hexdump(cout);
+	cout << "\n";
+      }
+    }
+    else if (strcmp(nargs[1], "set") == 0 ||
+	     strcmp(nargs[1], "create") == 0) {
+      if (nargs.size() < 5)
+	usage_exit();
+      string oid(nargs[2]);
+      string k(nargs[3]);
+      string v(nargs[4]);
+      bufferlist bl;
+      char c = (strcmp(nargs[1], "set") == 0) ? CEPH_OSD_TMAP_SET : CEPH_OSD_TMAP_CREATE;
+      ::encode(c, bl);
+      ::encode(k, bl);
+      ::encode(v, bl);
+      ret = io_ctx.tmap_update(oid, bl);
+    }
+  }
+
+  else if (strcmp(nargs[0], "tmap-to-omap") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+
+    bufferlist bl;
+    int r = io_ctx.tmap_get(oid, bl);
+    if (r < 0) {
+      ret = r;
+      cerr << "error reading tmap " << pool_name << "/" << oid
+	   << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    bufferlist hdr;
+    map kv;
+    bufferlist::iterator p = bl.begin();
+    try {
+      ::decode(hdr, p);
+      ::decode(kv, p);
+    }
+    catch (buffer::error& e) {
+      cerr << "error decoding tmap " << pool_name << "/" << oid << std::endl;
+      ret = -EINVAL;
+      goto out;
+    }
+    if (!p.end()) {
+      cerr << "error decoding tmap (stray trailing data) in " << pool_name << "/" << oid << std::endl;
+      ret = -EINVAL;
+      goto out;
+    }
+    librados::ObjectWriteOperation wr;
+    wr.omap_set_header(hdr);
+    wr.omap_set(kv);
+    wr.truncate(0);  // delete the old tmap data
+    r = io_ctx.operate(oid, &wr);
+    if (r < 0) {
+      ret = r;
+      cerr << "error writing tmap data as omap on " << pool_name << "/" << oid
+	   << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    ret = 0;
+  }
+
+  else if (strcmp(nargs[0], "mkpool") == 0) {
+    int auid = 0;
+    __u8 crush_rule = 0;
+    if (nargs.size() < 2)
+      usage_exit();
+    if (nargs.size() > 2) {
+      auid = strtol(nargs[2], 0, 10);
+      cerr << "setting auid:" << auid << std::endl;
+      if (nargs.size() > 3) {
+	crush_rule = (__u8)strtol(nargs[3], 0, 10);
+	cerr << "using crush rule " << (int)crush_rule << std::endl;
+      }
+    }
+    ret = rados.pool_create(nargs[1], auid, crush_rule);
+    if (ret < 0) {
+      cerr << "error creating pool " << nargs[1] << ": "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    cout << "successfully created pool " << nargs[1] << std::endl;
+  }
+  else if (strcmp(nargs[0], "cppool") == 0) {
+    if (nargs.size() != 3)
+      usage_exit();
+    const char *src_pool = nargs[1];
+    const char *target_pool = nargs[2];
+
+    if (strcmp(src_pool, target_pool) == 0) {
+      cerr << "cannot copy pool into itself" << std::endl;
+      ret = -1;
+      goto out;
+    }
+
+    ret = do_copy_pool(rados, src_pool, target_pool);
+    if (ret < 0) {
+      cerr << "error copying pool " << src_pool << " => " << target_pool << ": "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    cout << "successfully copied pool " << nargs[1] << std::endl;
+  }
+  else if (strcmp(nargs[0], "rmpool") == 0) {
+    if (nargs.size() < 2)
+      usage_exit();
+    if (nargs.size() < 4 ||
+	strcmp(nargs[1], nargs[2]) != 0 ||
+	strcmp(nargs[3], "--yes-i-really-really-mean-it") != 0) {
+      cerr << "WARNING:\n"
+	   << "  This will PERMANENTLY DESTROY an entire pool of objects with no way back.\n"
+	   << "  To confirm, pass the pool to remove twice, followed by\n"
+	   << "  --yes-i-really-really-mean-it" << std::endl;
+      ret = -1;
+      goto out;
+    }
+    ret = rados.pool_delete(nargs[1]);
+    if (ret >= 0) {
+      cout << "successfully deleted pool " << nargs[1] << std::endl;
+    } else { //error
+      cerr << "pool " << nargs[1] << " does not exist" << std::endl;
+    }
+  }
+  else if (strcmp(nargs[0], "lssnap") == 0) {
+    if (!pool_name || nargs.size() != 1)
+      usage_exit();
+
+    vector snaps;
+    io_ctx.snap_list(&snaps);
+    for (vector::iterator i = snaps.begin();
+	 i != snaps.end();
+	 ++i) {
+      string s;
+      time_t t;
+      if (io_ctx.snap_get_name(*i, &s) < 0)
+	continue;
+      if (io_ctx.snap_get_stamp(*i, &t) < 0)
+	continue;
+      struct tm bdt;
+      localtime_r(&t, &bdt);
+      cout << *i << "\t" << s << "\t";
+
+      cout.setf(std::ios::right);
+      cout.fill('0');
+      cout << std::setw(4) << (bdt.tm_year+1900)
+	   << '.' << std::setw(2) << (bdt.tm_mon+1)
+	   << '.' << std::setw(2) << bdt.tm_mday
+	   << ' '
+	   << std::setw(2) << bdt.tm_hour
+	   << ':' << std::setw(2) << bdt.tm_min
+	   << ':' << std::setw(2) << bdt.tm_sec
+	   << std::endl;
+      cout.unsetf(std::ios::right);
+    }
+    cout << snaps.size() << " snaps" << std::endl;
+  }
+
+  else if (strcmp(nargs[0], "mksnap") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    ret = io_ctx.snap_create(nargs[1]);
+    if (ret < 0) {
+      cerr << "error creating pool " << pool_name << " snapshot " << nargs[1]
+	   << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    cout << "created pool " << pool_name << " snap " << nargs[1] << std::endl;
+  }
+
+  else if (strcmp(nargs[0], "rmsnap") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    ret = io_ctx.snap_remove(nargs[1]);
+    if (ret < 0) {
+      cerr << "error removing pool " << pool_name << " snapshot " << nargs[1]
+	   << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    cout << "removed pool " << pool_name << " snap " << nargs[1] << std::endl;
+  }
+
+  else if (strcmp(nargs[0], "rollback") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+
+    ret = io_ctx.rollback(nargs[1], nargs[2]);
+    if (ret < 0) {
+      cerr << "error rolling back pool " << pool_name << " to snapshot " << nargs[1]
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    cout << "rolled back pool " << pool_name
+	 << " to snapshot " << nargs[2] << std::endl;
+  }
+  else if (strcmp(nargs[0], "bench") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+    int seconds = atoi(nargs[1]);
+    int operation = 0;
+    if (strcmp(nargs[2], "write") == 0)
+      operation = OP_WRITE;
+    else if (strcmp(nargs[2], "seq") == 0)
+      operation = OP_SEQ_READ;
+    else if (strcmp(nargs[2], "rand") == 0)
+      operation = OP_RAND_READ;
+    else
+      usage_exit();
+    RadosBencher bencher(g_ceph_context, rados, io_ctx);
+    bencher.set_show_time(show_time);
+    ret = bencher.aio_bench(operation, seconds, num_objs,
+			    concurrent_ios, op_size, cleanup, run_name);
+    if (ret != 0)
+      cerr << "error during benchmark: " << ret << std::endl;
+  }
+  else if (strcmp(nargs[0], "cleanup") == 0) {
+    if (!pool_name)
+      usage_exit();
+    RadosBencher bencher(g_ceph_context, rados, io_ctx);
+    ret = bencher.clean_up(prefix, concurrent_ios, run_name);
+    if (ret != 0)
+      cerr << "error during cleanup: " << ret << std::endl;
+  }
+  else if (strcmp(nargs[0], "watch") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+    RadosWatchCtx ctx(oid.c_str());
+    uint64_t cookie;
+    ret = io_ctx.watch(oid, 0, &cookie, &ctx);
+    if (ret != 0)
+      cerr << "error calling watch: " << ret << std::endl;
+    else {
+      cout << "press enter to exit..." << std::endl;
+      getchar();
+    }
+  }
+  else if (strcmp(nargs[0], "notify") == 0) {
+    if (!pool_name || nargs.size() < 3)
+      usage_exit();
+    string oid(nargs[1]);
+    string msg(nargs[2]);
+    bufferlist bl;
+    ::encode(msg, bl);
+    ret = io_ctx.notify(oid, 0, bl);
+    if (ret != 0)
+      cerr << "error calling notify: " << ret << std::endl;
+  } else if (strcmp(nargs[0], "set-alloc-hint") == 0) {
+    if (!pool_name || nargs.size() < 4)
+      usage_exit();
+    string err;
+    string oid(nargs[1]);
+    uint64_t expected_object_size = strict_strtoll(nargs[2], 10, &err);
+    if (!err.empty()) {
+      cerr << "couldn't parse expected_object_size: " << err << std::endl;
+      usage_exit();
+    }
+    uint64_t expected_write_size = strict_strtoll(nargs[3], 10, &err);
+    if (!err.empty()) {
+      cerr << "couldn't parse expected_write_size: " << err << std::endl;
+      usage_exit();
+    }
+    ret = io_ctx.set_alloc_hint(oid, expected_object_size, expected_write_size);
+    if (ret < 0) {
+      cerr << "error setting alloc-hint " << pool_name << "/" << oid << ": "
+           << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "load-gen") == 0) {
+    if (!pool_name) {
+      cerr << "error: must specify pool" << std::endl;
+      usage_exit();
+    }
+    LoadGen lg(&rados);
+    if (min_obj_len)
+      lg.min_obj_len = min_obj_len;
+    if (max_obj_len)
+      lg.max_obj_len = max_obj_len;
+    if (min_op_len)
+      lg.min_op_len = min_op_len;
+    if (max_op_len)
+      lg.max_op_len = max_op_len;
+    if (max_ops)
+      lg.max_ops = max_ops;
+    if (max_backlog)
+      lg.max_backlog = max_backlog;
+    if (target_throughput)
+      lg.target_throughput = target_throughput << 20;
+    if (read_percent >= 0)
+      lg.read_percent = read_percent;
+    if (num_objs)
+      lg.num_objs = num_objs;
+    if (run_length)
+      lg.run_length = run_length;
+
+    cout << "run length " << run_length << " seconds" << std::endl;
+    cout << "preparing " << lg.num_objs << " objects" << std::endl;
+    ret = lg.bootstrap(pool_name);
+    if (ret < 0) {
+      cerr << "load-gen bootstrap failed" << std::endl;
+      exit(1);
+    }
+    cout << "load-gen will run " << lg.run_length << " seconds" << std::endl;
+    lg.run();
+    lg.cleanup();
+  } else if (strcmp(nargs[0], "listomapkeys") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    librados::ObjectReadOperation read;
+    set out_keys;
+    read.omap_get_keys("", LONG_MAX, &out_keys, &ret);
+    io_ctx.operate(nargs[1], &read, NULL);
+    if (ret < 0) {
+      cerr << "error getting omap key set " << pool_name << "/"
+	   << nargs[1] << ": "  << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+
+    for (set::iterator iter = out_keys.begin();
+	 iter != out_keys.end(); ++iter) {
+      cout << *iter << std::endl;
+    }
+  } else if (strcmp(nargs[0], "lock") == 0) {
+    if (!pool_name)
+      usage_exit();
+
+    if (!formatter) {
+      formatter = new JSONFormatter(pretty_format);
+    }
+    ret = do_lock_cmd(nargs, opts, &io_ctx, formatter);
+  } else if (strcmp(nargs[0], "listwatchers") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    string oid(nargs[1]);
+    std::list lw;
+
+    ret = io_ctx.list_watchers(oid, &lw);
+    if (ret < 0) {
+      cerr << "error listing watchers " << pool_name << "/" << oid << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    else
+      ret = 0;
+    
+    for (std::list::iterator i = lw.begin(); i != lw.end(); ++i) {
+      cout << "watcher=" << i->addr << " client." << i->watcher_id << " cookie=" << i->cookie << std::endl;
+    }
+  } else if (strcmp(nargs[0], "listsnaps") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+
+    string oid(nargs[1]);
+    snap_set_t ls;
+
+    io_ctx.snap_set_read(LIBRADOS_SNAP_DIR);
+    ret = io_ctx.list_snaps(oid, &ls);
+    if (ret < 0) {
+      cerr << "error listing snap shots " << pool_name << "/" << oid << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    else
+      ret = 0;
+
+    map snamemap;
+    if (formatter || pretty_format) {
+      vector snaps;
+      io_ctx.snap_list(&snaps);
+      for (vector::iterator i = snaps.begin();
+          i != snaps.end(); ++i) {
+        string s;
+        if (io_ctx.snap_get_name(*i, &s) < 0)
+          continue;
+        snamemap.insert(pair(*i, s));
+      }
+    }
+
+    if (formatter) {
+      formatter->open_object_section("object");
+      formatter->dump_string("name", oid);
+      formatter->open_array_section("clones");
+    } else {
+      cout << oid << ":" << std::endl;
+      cout << "cloneid	snaps	size	overlap" << std::endl;
+    }
+
+    for (std::vector::iterator ci = ls.clones.begin();
+          ci != ls.clones.end(); ++ci) {
+
+      if (formatter) formatter->open_object_section("clone");
+
+      if (ci->cloneid == librados::SNAP_HEAD) {
+        if (formatter)
+          formatter->dump_string("id", "head");
+        else
+          cout << "head";
+      } else {
+        if (formatter)
+          formatter->dump_unsigned("id", ci->cloneid);
+        else
+          cout << ci->cloneid;
+      }
+
+      if (formatter)
+        formatter->open_array_section("snapshots");
+      else
+        cout << "\t";
+
+      if (!formatter && ci->snaps.empty()) {
+        cout << "-";
+      }
+      for (std::vector::const_iterator snapindex = ci->snaps.begin();
+          snapindex != ci->snaps.end(); ++snapindex) {
+
+        map::iterator si;
+
+        if (formatter || pretty_format) si = snamemap.find(*snapindex);
+
+        if (formatter) {
+          formatter->open_object_section("snapshot");
+          formatter->dump_unsigned("id", *snapindex);
+          if (si != snamemap.end())
+            formatter->dump_string("name", si->second);
+          formatter->close_section(); //snapshot
+        } else {
+          if (snapindex != ci->snaps.begin()) cout << ",";
+          if (!pretty_format || (si == snamemap.end()))
+            cout << *snapindex;
+          else
+            cout << si->second << "(" << *snapindex << ")";
+        }
+      }
+
+      if (formatter) {
+        formatter->close_section();	//Snapshots
+        formatter->dump_unsigned("size", ci->size);
+      } else {
+        cout << "\t" << ci->size;
+      }
+
+      if (ci->cloneid != librados::SNAP_HEAD) {
+        if (formatter)
+          formatter->open_array_section("overlaps");
+        else
+          cout << "\t[";
+
+        for (std::vector< std::pair >::iterator ovi = ci->overlap.begin();
+            ovi != ci->overlap.end(); ++ovi) {
+          if (formatter) {
+            formatter->open_object_section("section");
+            formatter->dump_unsigned("start", ovi->first);
+            formatter->dump_unsigned("length", ovi->second);
+            formatter->close_section(); //section
+          } else {
+            if (ovi != ci->overlap.begin()) cout << ",";
+            cout << ovi->first << "~" << ovi->second;
+          }
+        }
+        if (formatter)
+          formatter->close_section(); //overlaps
+        else
+          cout << "]" << std::endl;
+      }
+      if (formatter) formatter->close_section(); //clone
+    }
+    if (formatter) {
+      formatter->close_section(); //clones
+      formatter->close_section(); //object
+      formatter->flush(cout);
+    } else {
+      cout << std::endl;
+    }
+
+  } else if (strcmp(nargs[0], "cache-flush") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+    ret = do_cache_flush(io_ctx, oid);
+    if (ret < 0) {
+      cerr << "error from cache-flush " << oid << ": "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "cache-try-flush") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+    ret = do_cache_try_flush(io_ctx, oid);
+    if (ret < 0) {
+      cerr << "error from cache-try-flush " << oid << ": "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "cache-evict") == 0) {
+    if (!pool_name || nargs.size() < 2)
+      usage_exit();
+    string oid(nargs[1]);
+    ret = do_cache_evict(io_ctx, oid);
+    if (ret < 0) {
+      cerr << "error from cache-evict " << oid << ": "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "cache-flush-evict-all") == 0) {
+    if (!pool_name)
+      usage_exit();
+    ret = do_cache_flush_evict_all(io_ctx, true);
+    if (ret < 0) {
+      cerr << "error from cache-flush-evict-all: "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else if (strcmp(nargs[0], "cache-try-flush-evict-all") == 0) {
+    if (!pool_name)
+      usage_exit();
+    ret = do_cache_flush_evict_all(io_ctx, false);
+    if (ret < 0) {
+      cerr << "error from cache-try-flush-evict-all: "
+	   << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+  } else {
+    cerr << "unrecognized command " << nargs[0] << "; -h or --help for usage" << std::endl;
+    ret = -EINVAL;
+    goto out;
+  }
+
+  if (ret < 0)
+    cerr << "error " << (-ret) << ": " << cpp_strerror(ret) << std::endl;
+
+out:
+  delete formatter;
+  return (ret < 0) ? 1 : 0;
+}
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  std::map < std::string, std::string > opts;
+  std::vector::iterator i;
+  std::string val;
+  for (i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage(cout);
+      exit(0);
+    } else if (ceph_argparse_flag(args, i, "-f", "--force", (char*)NULL)) {
+      opts["force"] = "true";
+    } else if (ceph_argparse_flag(args, i, "-d", "--delete-after", (char*)NULL)) {
+      opts["delete-after"] = "true";
+    } else if (ceph_argparse_flag(args, i, "-C", "--create", "--create-pool",
+				  (char*)NULL)) {
+      opts["create"] = "true";
+    } else if (ceph_argparse_flag(args, i, "--pretty-format", (char*)NULL)) {
+      opts["pretty-format"] = "true";
+    } else if (ceph_argparse_flag(args, i, "--show-time", (char*)NULL)) {
+      opts["show-time"] = "true";
+    } else if (ceph_argparse_flag(args, i, "--no-cleanup", (char*)NULL)) {
+      opts["no-cleanup"] = "true";
+    } else if (ceph_argparse_witharg(args, i, &val, "--run-name", (char*)NULL)) {
+      opts["run-name"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--prefix", (char*)NULL)) {
+      opts["prefix"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-p", "--pool", (char*)NULL)) {
+      opts["pool"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--target-pool", (char*)NULL)) {
+      opts["target_pool"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--object-locator" , (char *)NULL)) {
+      opts["object_locator"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--target-locator" , (char *)NULL)) {
+      opts["target_locator"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--category", (char*)NULL)) {
+      opts["category"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-t", "--concurrent-ios", (char*)NULL)) {
+      opts["concurrent-ios"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--block-size", (char*)NULL)) {
+      opts["block-size"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-b", (char*)NULL)) {
+      opts["block-size"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-s", "--snap", (char*)NULL)) {
+      opts["snap"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-S", "--snapid", (char*)NULL)) {
+      opts["snapid"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--min-object-size", (char*)NULL)) {
+      opts["min-object-size"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-object-size", (char*)NULL)) {
+      opts["max-object-size"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--min-op-len", (char*)NULL)) {
+      opts["min-op-len"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-op-len", (char*)NULL)) {
+      opts["max-op-len"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-ops", (char*)NULL)) {
+      opts["max-ops"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--max-backlog", (char*)NULL)) {
+      opts["max-backlog"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--target-throughput", (char*)NULL)) {
+      opts["target-throughput"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--read-percent", (char*)NULL)) {
+      opts["read-percent"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--num-objects", (char*)NULL)) {
+      opts["num-objects"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--run-length", (char*)NULL)) {
+      opts["run-length"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--workers", (char*)NULL)) {
+      opts["workers"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--format", (char*)NULL)) {
+      opts["format"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--lock-tag", (char*)NULL)) {
+      opts["lock-tag"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--lock-cookie", (char*)NULL)) {
+      opts["lock-cookie"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--lock-description", (char*)NULL)) {
+      opts["lock-description"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--lock-duration", (char*)NULL)) {
+      opts["lock-duration"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--lock-type", (char*)NULL)) {
+      opts["lock-type"] = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "-N", "--namespace", (char*)NULL)) {
+      opts["namespace"] = val;
+    } else {
+      if (val[0] == '-')
+        usage_exit();
+      ++i;
+    }
+  }
+
+  if (args.empty()) {
+    cerr << "rados: you must give an action. Try --help" << std::endl;
+    return 1;
+  }
+  if ((strcmp(args[0], "import") == 0) || (strcmp(args[0], "export") == 0))
+    return rados_tool_sync(opts, args);
+  else
+    return rados_tool_common(opts, args);
+}
diff --git a/ceph/src/tools/rados/rados_export.cc b/ceph/src/tools/rados/rados_export.cc
new file mode 100644
index 00000000..bf665411
--- /dev/null
+++ b/ceph/src/tools/rados/rados_export.cc
@@ -0,0 +1,229 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+
+#include "rados_sync.h"
+#include "common/errno.h"
+#include "common/strtol.h"
+#include "include/rados/librados.hpp"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "include/compat.h"
+#include "common/xattr.h"
+
+using namespace librados;
+
+class ExportLocalFileWQ : public RadosSyncWQ {
+public:
+  ExportLocalFileWQ(IoCtxDistributor *io_ctx_dist, time_t ti,
+		    ThreadPool *tp, ExportDir *export_dir, bool force)
+    : RadosSyncWQ(io_ctx_dist, ti, 0, tp),
+      m_export_dir(export_dir),
+      m_force(force)
+  {
+  }
+private:
+  void _process(std::string *s) {
+    IoCtx &io_ctx(m_io_ctx_dist->get_ioctx());
+    int flags = 0;
+    auto_ptr  sobj;
+    auto_ptr  dobj;
+    const std::string &rados_name(*s);
+    std::list < std::string > only_in_a;
+    std::list < std::string > only_in_b;
+    std::list < std::string > diff;
+    int ret = BackedUpObject::from_rados(io_ctx, rados_name.c_str(), sobj);
+    if (ret) {
+      cerr << ERR_PREFIX << "couldn't get '" << rados_name << "' from rados: error "
+	   << ret << std::endl;
+      _exit(ret);
+    }
+    std::string obj_path(sobj->get_fs_path(m_export_dir));
+    if (m_force) {
+      flags |= (CHANGED_CONTENTS | CHANGED_XATTRS);
+    }
+    else {
+      ret = BackedUpObject::from_path(obj_path.c_str(), dobj);
+      if (ret == ENOENT) {
+	sobj->get_xattrs(only_in_a);
+	flags |= CHANGED_CONTENTS;
+      }
+      else if (ret) {
+	cerr << ERR_PREFIX << "BackedUpObject::from_path returned "
+	     << ret << std::endl;
+	_exit(ret);
+      }
+      else {
+	sobj->xattr_diff(dobj.get(), only_in_a, only_in_b, diff);
+	if ((sobj->get_rados_size() == dobj->get_rados_size()) &&
+	    (sobj->get_mtime() == dobj->get_mtime())) {
+	  flags |= CHANGED_CONTENTS;
+	}
+      }
+    }
+    if (flags & CHANGED_CONTENTS) {
+      ret = sobj->download(io_ctx, obj_path.c_str());
+      if (ret) {
+	cerr << ERR_PREFIX << "download error: " << ret << std::endl;
+	_exit(ret);
+      }
+    }
+    diff.splice(diff.begin(), only_in_a);
+    for (std::list < std::string >::const_iterator x = diff.begin();
+	 x != diff.end(); ++x) {
+      flags |= CHANGED_XATTRS;
+      const Xattr *xattr = sobj->get_xattr(*x);
+      if (xattr == NULL) {
+	cerr << ERR_PREFIX << "internal error on line: " << __LINE__ << std::endl;
+	_exit(ret);
+      }
+      std::string xattr_fs_name(USER_XATTR_PREFIX);
+      xattr_fs_name += x->c_str();
+      ret = ceph_os_setxattr(obj_path.c_str(), xattr_fs_name.c_str(),
+		     xattr->data, xattr->len);
+      if (ret) {
+	ret = errno;
+	cerr << ERR_PREFIX << "setxattr error: " << cpp_strerror(ret) << std::endl;
+	_exit(ret);
+      }
+    }
+    for (std::list < std::string >::const_iterator x = only_in_b.begin();
+	 x != only_in_b.end(); ++x) {
+      flags |= CHANGED_XATTRS;
+      ret = ceph_os_removexattr(obj_path.c_str(), x->c_str());
+      if (ret) {
+	ret = errno;
+	cerr << ERR_PREFIX << "removexattr error: " << cpp_strerror(ret) << std::endl;
+	_exit(ret);
+      }
+    }
+    if (m_force) {
+      cout << "[force]        " << rados_name << std::endl;
+    }
+    else if (flags & CHANGED_CONTENTS) {
+      cout << "[exported]     " << rados_name << std::endl;
+    }
+    else if (flags & CHANGED_XATTRS) {
+      cout << "[xattr]        " << rados_name << std::endl;
+    }
+  }
+  ExportDir *m_export_dir;
+  bool m_force;
+};
+
+class ExportValidateExistingWQ : public RadosSyncWQ {
+public:
+  ExportValidateExistingWQ(IoCtxDistributor *io_ctx_dist, time_t ti,
+			   ThreadPool *tp, const char *dir_name)
+    : RadosSyncWQ(io_ctx_dist, ti, 0, tp),
+      m_dir_name(dir_name)
+  {
+  }
+private:
+  void _process(std::string *s) {
+    IoCtx &io_ctx(m_io_ctx_dist->get_ioctx());
+    auto_ptr  lobj;
+    const std::string &local_name(*s);
+    int ret = BackedUpObject::from_file(local_name.c_str(), m_dir_name, lobj);
+    if (ret) {
+      cout << ERR_PREFIX << "BackedUpObject::from_file: delete loop: "
+	   << "got error " << ret << std::endl;
+      _exit(ret);
+    }
+    auto_ptr  robj;
+    ret = BackedUpObject::from_rados(io_ctx, lobj->get_rados_name(), robj);
+    if (ret == -ENOENT) {
+      // The entry doesn't exist on the remote server; delete it locally
+      char path[strlen(m_dir_name) + local_name.size() + 2];
+      snprintf(path, sizeof(path), "%s/%s", m_dir_name, local_name.c_str());
+      if (unlink(path)) {
+	ret = errno;
+	cerr << ERR_PREFIX << "error unlinking '" << path << "': "
+	     << cpp_strerror(ret) << std::endl;
+	_exit(ret);
+      }
+      cout << "[deleted]      " << "removed '" << local_name << "'" << std::endl;
+    }
+    else if (ret) {
+      cerr << ERR_PREFIX << "BackedUpObject::from_rados: delete loop: "
+	   << "got error " << ret << std::endl;
+      _exit(ret);
+    }
+  }
+  const char *m_dir_name;
+};
+
+int do_rados_export(ThreadPool *tp, IoCtx& io_ctx,
+      IoCtxDistributor *io_ctx_dist, const char *dir_name,
+      bool create, bool force, bool delete_after)
+{
+  librados::ObjectIterator oi = io_ctx.objects_begin();
+  librados::ObjectIterator oi_end = io_ctx.objects_end();
+  auto_ptr  export_dir;
+  export_dir.reset(ExportDir::create_for_writing(dir_name, 1, create));
+  if (!export_dir.get())
+    return -EIO;
+  ExportLocalFileWQ export_object_wq(io_ctx_dist, time(NULL),
+				     tp, export_dir.get(), force);
+  for (; oi != oi_end; ++oi) {
+    export_object_wq.queue(new std::string((*oi).first));
+  }
+  export_object_wq.drain();
+
+  if (delete_after) {
+    ExportValidateExistingWQ export_val_wq(io_ctx_dist, time(NULL),
+					   tp, dir_name);
+    DirHolder dh;
+    int err = dh.opendir(dir_name);
+    if (err) {
+      cerr << ERR_PREFIX << "opendir(" << dir_name << ") error: "
+	   << cpp_strerror(err) << std::endl;
+      return err;
+    }
+    while (true) {
+      struct dirent *de = readdir(dh.dp);
+      if (!de)
+	break;
+      if ((strcmp(de->d_name, ".") == 0) || (strcmp(de->d_name, "..") == 0))
+	continue;
+      if (is_suffix(de->d_name, RADOS_SYNC_TMP_SUFFIX)) {
+	char path[strlen(dir_name) + strlen(de->d_name) + 2];
+	snprintf(path, sizeof(path), "%s/%s", dir_name, de->d_name);
+	if (unlink(path)) {
+	  int ret = errno;
+	  cerr << ERR_PREFIX << "error unlinking temporary file '" << path << "': "
+	       << cpp_strerror(ret) << std::endl;
+	  return ret;
+	}
+	cout << "[deleted]      " << "removed temporary file '" << de->d_name << "'" << std::endl;
+	continue;
+      }
+      export_val_wq.queue(new std::string(de->d_name));
+    }
+    export_val_wq.drain();
+  }
+  cout << "[done]" << std::endl;
+  return 0;
+}
diff --git a/ceph/src/tools/rados/rados_import.cc b/ceph/src/tools/rados/rados_import.cc
new file mode 100644
index 00000000..a6a398d7
--- /dev/null
+++ b/ceph/src/tools/rados/rados_import.cc
@@ -0,0 +1,239 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "rados_sync.h"
+#include "common/errno.h"
+#include "common/strtol.h"
+#include "include/rados/librados.hpp"
+
+using namespace librados;
+using std::auto_ptr;
+
+class ImportLocalFileWQ : public RadosSyncWQ {
+public:
+  ImportLocalFileWQ(const char *dir_name, bool force,
+		    IoCtxDistributor *io_ctx_dist, time_t ti, ThreadPool *tp)
+    : RadosSyncWQ(io_ctx_dist, ti, 0, tp),
+      m_dir_name(dir_name),
+      m_force(force)
+  {
+  }
+private:
+  void _process(std::string *s) {
+    IoCtx &io_ctx(m_io_ctx_dist->get_ioctx());
+    const std::string &local_name(*s);
+    auto_ptr  sobj;
+    auto_ptr  dobj;
+    std::list < std::string > only_in_a;
+    std::list < std::string > only_in_b;
+    std::list < std::string > diff;
+    int flags = 0;
+
+    int ret = BackedUpObject::from_file(local_name.c_str(),
+					m_dir_name.c_str(), sobj);
+    if (ret) {
+      cerr << ERR_PREFIX << "BackedUpObject::from_file: got error "
+	   << ret << std::endl;
+      _exit(ret);
+    }
+    const char *rados_name(sobj->get_rados_name());
+    if (m_force) {
+      flags |= (CHANGED_CONTENTS | CHANGED_XATTRS);
+    }
+    else {
+      ret = BackedUpObject::from_rados(io_ctx, rados_name, dobj);
+      if (ret == -ENOENT) {
+	flags |= CHANGED_CONTENTS;
+	sobj->get_xattrs(only_in_a);
+      }
+      else if (ret) {
+	cerr << ERR_PREFIX << "BackedUpObject::from_rados returned "
+	     << ret << std::endl;
+	_exit(ret);
+      }
+      else {
+	sobj->xattr_diff(dobj.get(), only_in_a, only_in_b, diff);
+	if ((sobj->get_rados_size() == dobj->get_rados_size()) &&
+	    (sobj->get_mtime() == dobj->get_mtime())) {
+	  flags |= CHANGED_CONTENTS;
+	}
+      }
+    }
+    if (flags & CHANGED_CONTENTS) {
+      ret = sobj->upload(io_ctx, local_name.c_str(), m_dir_name.c_str());
+      if (ret) {
+	cerr << ERR_PREFIX << "upload error: " << ret << std::endl;
+	_exit(ret);
+      }
+    }
+    for (std::list < std::string >::const_iterator x = only_in_a.begin();
+	 x != only_in_a.end(); ++x) {
+      flags |= CHANGED_XATTRS;
+      const Xattr *xattr = sobj->get_xattr(*x);
+      if (xattr == NULL) {
+	cerr << ERR_PREFIX << "internal error on line: " << __LINE__ << std::endl;
+	_exit(ret);
+      }
+      bufferlist bl;
+      bl.append(xattr->data, xattr->len);
+      ret = io_ctx.setxattr(rados_name, x->c_str(), bl);
+      if (ret < 0) {
+	ret = errno;
+	cerr << ERR_PREFIX << "io_ctx.setxattr(rados_name='" << rados_name
+	     << "', xattr_name='" << x->c_str() << "'): " << cpp_strerror(ret)
+	     << std::endl;
+	_exit(ret);
+      }
+    }
+    for (std::list < std::string >::const_iterator x = diff.begin();
+	 x != diff.end(); ++x) {
+      flags |= CHANGED_XATTRS;
+      const Xattr *xattr = sobj->get_xattr(*x);
+      if (xattr == NULL) {
+	cerr << ERR_PREFIX << "internal error on line: " << __LINE__ << std::endl;
+	_exit(ret);
+      }
+      bufferlist bl;
+      bl.append(xattr->data, xattr->len);
+      ret = io_ctx.rmxattr(rados_name, x->c_str());
+      if (ret < 0) {
+	cerr << ERR_PREFIX << "io_ctx.rmxattr error2: " << cpp_strerror(ret)
+	     << std::endl;
+	_exit(ret);
+      }
+      ret = io_ctx.setxattr(rados_name, x->c_str(), bl);
+      if (ret < 0) {
+	ret = errno;
+	cerr << ERR_PREFIX << "io_ctx.setxattr(rados_name='" << rados_name
+	     << "', xattr='" << x->c_str() << "'): " << cpp_strerror(ret) << std::endl;
+	_exit(ret);
+      }
+    }
+    for (std::list < std::string >::const_iterator x = only_in_b.begin();
+	 x != only_in_b.end(); ++x) {
+      flags |= CHANGED_XATTRS;
+      ret = io_ctx.rmxattr(rados_name, x->c_str());
+      if (ret < 0) {
+	ret = errno;
+	cerr << ERR_PREFIX << "rmxattr error3: " << cpp_strerror(ret) << std::endl;
+	_exit(ret);
+      }
+    }
+    if (m_force) {
+      cout << "[force]        " << rados_name << std::endl;
+    }
+    else if (flags & CHANGED_CONTENTS) {
+      cout << "[imported]     " << rados_name << std::endl;
+    }
+    else if (flags & CHANGED_XATTRS) {
+      cout << "[xattr]        " << rados_name << std::endl;
+    }
+  }
+  std::string m_dir_name;
+  bool m_force;
+};
+
+class ImportValidateExistingWQ : public RadosSyncWQ {
+public:
+  ImportValidateExistingWQ(ExportDir *export_dir,
+		 IoCtxDistributor *io_ctx_dist, time_t ti, ThreadPool *tp)
+    : RadosSyncWQ(io_ctx_dist, ti, 0, tp),
+      m_export_dir(export_dir)
+  {
+  }
+private:
+  void _process(std::string *s) {
+    IoCtx &io_ctx(m_io_ctx_dist->get_ioctx());
+    const std::string &rados_name(*s);
+    auto_ptr  robj;
+    int ret = BackedUpObject::from_rados(io_ctx, rados_name.c_str(), robj);
+    if (ret) {
+      cerr << ERR_PREFIX << "BackedUpObject::from_rados in delete loop "
+	   << "returned " << ret << std::endl;
+      _exit(ret);
+    }
+    std::string obj_path(robj->get_fs_path(m_export_dir));
+    auto_ptr  lobj;
+    ret = BackedUpObject::from_path(obj_path.c_str(), lobj);
+    if (ret == ENOENT) {
+      ret = io_ctx.remove(rados_name);
+      if (ret && ret != -ENOENT) {
+	cerr << ERR_PREFIX << "io_ctx.remove(" << obj_path << ") failed "
+	    << "with error " << ret << std::endl;
+	_exit(ret);
+      }
+      cout << "[deleted]      " << "removed '" << rados_name << "'" << std::endl;
+    }
+    else if (ret) {
+      cerr << ERR_PREFIX << "BackedUpObject::from_path in delete loop "
+	   << "returned " << ret << std::endl;
+      _exit(ret);
+    }
+  }
+  ExportDir *m_export_dir;
+};
+
+int do_rados_import(ThreadPool *tp, IoCtx &io_ctx, IoCtxDistributor* io_ctx_dist,
+	   const char *dir_name, bool force, bool delete_after)
+{
+  auto_ptr  export_dir;
+  export_dir.reset(ExportDir::from_file_system(dir_name));
+  if (!export_dir.get())
+    return -EIO;
+  DirHolder dh;
+  int ret = dh.opendir(dir_name);
+  if (ret) {
+    cerr << ERR_PREFIX << "opendir(" << dir_name << ") error: "
+	 << cpp_strerror(ret) << std::endl;
+    return ret;
+  }
+  ImportLocalFileWQ import_file_wq(dir_name, force,
+				   io_ctx_dist, time(NULL), tp);
+  while (true) {
+    struct dirent *de = readdir(dh.dp);
+    if (!de)
+      break;
+    if ((strcmp(de->d_name, ".") == 0) || (strcmp(de->d_name, "..") == 0))
+      continue;
+    if (is_suffix(de->d_name, RADOS_SYNC_TMP_SUFFIX))
+      continue;
+    import_file_wq.queue(new std::string(de->d_name));
+  }
+  import_file_wq.drain();
+
+  if (delete_after) {
+    ImportValidateExistingWQ import_val_wq(export_dir.get(), io_ctx_dist,
+					   time(NULL), tp);
+    librados::ObjectIterator oi = io_ctx.objects_begin();
+    librados::ObjectIterator oi_end = io_ctx.objects_end();
+    for (; oi != oi_end; ++oi) {
+      import_val_wq.queue(new std::string((*oi).first));
+    }
+    import_val_wq.drain();
+  }
+  cout << "[done]" << std::endl;
+  return 0;
+}
diff --git a/ceph/src/tools/rados/rados_sync.cc b/ceph/src/tools/rados/rados_sync.cc
new file mode 100644
index 00000000..595df81d
--- /dev/null
+++ b/ceph/src/tools/rados/rados_sync.cc
@@ -0,0 +1,901 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "common/errno.h"
+#include "common/strtol.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "include/rados/librados.hpp"
+#include "rados_sync.h"
+#include "include/compat.h"
+
+#include "common/xattr.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace librados;
+using std::auto_ptr;
+
+static const char * const XATTR_RADOS_SYNC_VER = "user.rados_sync_ver";
+static const char * const XATTR_FULLNAME = "user.rados_full_name";
+const char USER_XATTR_PREFIX[] = "user.rados.";
+static const size_t USER_XATTR_PREFIX_LEN =
+  sizeof(USER_XATTR_PREFIX) / sizeof(USER_XATTR_PREFIX[0]) - 1;
+/* It's important that RADOS_SYNC_TMP_SUFFIX contain at least one character
+ * that we wouldn't normally alllow in a file name-- in this case, $ */
+const char RADOS_SYNC_TMP_SUFFIX[] = "$tmp";
+static const size_t RADOS_SYNC_TMP_SUFFIX_LEN =
+  sizeof(RADOS_SYNC_TMP_SUFFIX) / sizeof(RADOS_SYNC_TMP_SUFFIX[0]) - 1;
+
+std::string get_user_xattr_name(const char *fs_xattr_name)
+{
+  if (strncmp(fs_xattr_name, USER_XATTR_PREFIX, USER_XATTR_PREFIX_LEN))
+    return "";
+  return fs_xattr_name + USER_XATTR_PREFIX_LEN;
+}
+
+bool is_suffix(const char *str, const char *suffix)
+{
+  size_t strlen_str = strlen(str);
+  size_t strlen_suffix = strlen(suffix);
+  if (strlen_str < strlen_suffix)
+    return false;
+  return (strcmp(str + (strlen_str - strlen_suffix), suffix) == 0);
+}
+
+ExportDir* ExportDir::create_for_writing(const std::string &path, int version,
+				 bool create)
+{
+  if (access(path.c_str(), R_OK | W_OK) == 0) {
+    return ExportDir::from_file_system(path);
+  }
+  if (!create) {
+    cerr << ERR_PREFIX << "ExportDir: directory '"
+	 << path << "' does not exist. Use --create to create it."
+	 << std::endl;
+    return NULL;
+  }
+  int ret = mkdir(path.c_str(), 0700);
+  if (ret < 0) {
+    int err = errno;
+    if (err != EEXIST) {
+      cerr << ERR_PREFIX << "ExportDir: mkdir error: "
+	   << cpp_strerror(err) << std::endl;
+      return NULL;
+    }
+  }
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%d", version);
+  ret = ceph_os_setxattr(path.c_str(), XATTR_RADOS_SYNC_VER, buf, strlen(buf) + 1);
+  if (ret < 0) {
+    int err = errno;
+    cerr << ERR_PREFIX << "ExportDir: setxattr error :"
+	 << cpp_strerror(err) << std::endl;
+    return NULL;
+  }
+  return new ExportDir(path);
+}
+
+ExportDir* ExportDir::from_file_system(const std::string &path)
+{
+  if (access(path.c_str(), R_OK)) {
+      cerr << "ExportDir: source directory '" << path
+	   << "' appears to be inaccessible." << std::endl;
+      return NULL;
+  }
+  int ret;
+  char buf[32];
+  memset(buf, 0, sizeof(buf));
+  ret = ceph_os_getxattr(path.c_str(), XATTR_RADOS_SYNC_VER, buf, sizeof(buf) - 1);
+  if (ret < 0) {
+    ret = errno;
+    if (ret == ENODATA) {
+      cerr << ERR_PREFIX << "ExportDir: directory '" << path
+	   << "' does not appear to have been created by a rados "
+	   << "export operation." << std::endl;
+      return NULL;
+    }
+    cerr << ERR_PREFIX << "ExportDir: getxattr error :"
+	 << cpp_strerror(ret) << std::endl;
+    return NULL;
+  }
+  std::string err;
+  ret = strict_strtol(buf, 10, &err);
+  if (!err.empty()) {
+    cerr << ERR_PREFIX << "ExportDir: invalid value for "
+	 << XATTR_RADOS_SYNC_VER << ": " << buf << ". parse error: "
+	 << err << std::endl;
+    return NULL;
+  }
+  if (ret != 1) {
+    cerr << ERR_PREFIX << "ExportDir: can't handle any naming "
+	 << "convention besides version 1. You must upgrade this program to "
+	 << "handle the data in the new format." << std::endl;
+    return NULL;
+  }
+  return new ExportDir(path);
+}
+
+std::string ExportDir::get_fs_path(const std::string &rados_name) const
+{
+  static int HASH_LENGTH = 17;
+  size_t i;
+  size_t strlen_rados_name = strlen(rados_name.c_str());
+  size_t sz;
+  bool need_hash = false;
+  if (strlen_rados_name > 200) {
+    sz = 200;
+    need_hash = true;
+  }
+  else {
+    sz = strlen_rados_name;
+  }
+  char fs_path[sz + HASH_LENGTH + 1];
+  for (i = 0; i < sz; ++i) {
+    // Just replace anything that looks funny with an 'at' sign.
+    // Unicode also gets turned into 'at' signs.
+    signed char c = rados_name[i];
+    if (c < 0x20) {
+     // Since c is signed, this also eliminates bytes with the high bit set
+      c = '@';
+      need_hash = true;
+    }
+    else if (c == 0x7f) {
+      c = '@';
+      need_hash = true;
+    }
+    else if (c == '/') {
+      c = '@';
+      need_hash = true;
+    }
+    else if (c == '\\') {
+      c = '@';
+      need_hash = true;
+    }
+    else if (c == '$') {
+      c = '@';
+      need_hash = true;
+    }
+    else if (c == ' ') {
+      c = '_';
+      need_hash = true;
+    }
+    fs_path[i] = c;
+  }
+
+  if (need_hash) {
+    uint64_t hash = 17;
+    for (i = 0; i < strlen_rados_name; ++i) {
+      hash += (rados_name[i] * 33);
+    }
+    // The extra byte of length is because snprintf always NULL-terminates.
+    snprintf(fs_path + i, HASH_LENGTH + 1, "_%016" PRIx64, hash);
+  }
+  else {
+    // NULL-terminate.
+    fs_path[i] = '\0';
+  }
+
+  ostringstream oss;
+  oss << path << "/" << fs_path;
+  return oss.str();
+}
+
+ExportDir::ExportDir(const std::string &path_)
+   : path(path_)
+{
+}
+
+DirHolder::DirHolder()
+  : dp(NULL)
+{
+}
+
+DirHolder::~DirHolder() {
+  if (!dp)
+    return;
+  if (closedir(dp)) {
+    int err = errno;
+    cerr << ERR_PREFIX << "closedir failed: " << cpp_strerror(err) << std::endl;
+  }
+  dp = NULL;
+}
+
+int DirHolder::opendir(const char *dir_name) {
+  dp = ::opendir(dir_name);
+  if (!dp) {
+    int err = errno;
+    return err;
+  }
+  return 0;
+}
+
+static __thread int t_iod_idx = -1;
+
+static pthread_mutex_t io_ctx_distributor_lock = PTHREAD_MUTEX_INITIALIZER;
+
+IoCtxDistributor* IoCtxDistributor::instance() {
+  IoCtxDistributor *ret;
+  pthread_mutex_lock(&io_ctx_distributor_lock);
+  if (s_instance == NULL) {
+    s_instance = new IoCtxDistributor();
+  }
+  ret = s_instance;
+  pthread_mutex_unlock(&io_ctx_distributor_lock);
+  return ret;
+}
+
+int IoCtxDistributor::init(Rados &cluster, const char *pool_name,
+			   int num_ioctxes) {
+  m_io_ctxes.resize(num_ioctxes);
+  for (std::vector::iterator i = m_io_ctxes.begin();
+	 i != m_io_ctxes.end(); ++i) {
+    IoCtx &io_ctx(*i);
+    int ret = cluster.ioctx_create(pool_name, io_ctx);
+    if (ret) {
+      return ret;
+    }
+  }
+  m_highest_iod_idx.set(0);
+  return 0;
+}
+
+void IoCtxDistributor::clear() {
+  for (std::vector::iterator i = m_io_ctxes.begin();
+	 i != m_io_ctxes.end(); ++i) {
+    IoCtx &io_ctx(*i);
+    io_ctx.close();
+  }
+  m_io_ctxes.clear();
+  m_highest_iod_idx.set(0);
+}
+
+IoCtx& IoCtxDistributor::get_ioctx() {
+  if (t_iod_idx == -1) {
+    t_iod_idx = m_highest_iod_idx.inc() - 1;
+  }
+  if (m_io_ctxes.size() <= (unsigned int)t_iod_idx) {
+    cerr << ERR_PREFIX << "IoCtxDistributor: logic error on line "
+	 << __LINE__ << std::endl;
+    _exit(1);
+  }
+  return m_io_ctxes[t_iod_idx];
+}
+
+IoCtxDistributor *IoCtxDistributor::s_instance = NULL;
+
+IoCtxDistributor::IoCtxDistributor() {
+  clear();
+}
+
+IoCtxDistributor::~IoCtxDistributor() {
+  clear();
+}
+
+RadosSyncWQ::RadosSyncWQ(IoCtxDistributor *io_ctx_dist, time_t timeout, time_t suicide_timeout, ThreadPool *tp)
+  : ThreadPool::WorkQueue("FileStore::OpWQ", timeout, suicide_timeout, tp),
+    m_io_ctx_dist(io_ctx_dist)
+{
+}
+
+bool RadosSyncWQ::_enqueue(std::string *s) {
+  m_items.push_back(s);
+  return true;
+}
+
+void RadosSyncWQ::_dequeue(std::string *o) {
+  assert(0);
+}
+
+bool RadosSyncWQ::_empty() {
+  return m_items.empty();
+}
+
+std::string *RadosSyncWQ::_dequeue() {
+  if (m_items.empty())
+    return NULL;
+  std::string *ret = m_items.front();
+  m_items.pop_front();
+  return ret;
+}
+
+void RadosSyncWQ::_process_finish(std::string *s) {
+  delete s;
+}
+
+void RadosSyncWQ::_clear() {
+  for (std::deque::iterator i = m_items.begin();
+	 i != m_items.end(); ++i) {
+    delete *i;
+  }
+  m_items.clear();
+}
+
+Xattr::Xattr(char *data_, ssize_t len_)
+    : data(data_), len(len_)
+{
+}
+
+Xattr::~Xattr() {
+  free(data);
+}
+
+bool Xattr::operator==(const class Xattr &rhs) const {
+  if (len != rhs.len)
+    return false;
+  return (memcmp(data, rhs.data, len) == 0);
+}
+
+bool Xattr::operator!=(const class Xattr &rhs) const {
+  return !((*this) == rhs);
+}
+
+int BackedUpObject::from_file(const char *file_name, const char *dir_name,
+			  std::auto_ptr &obj)
+{
+  char obj_path[strlen(dir_name) + strlen(file_name) + 2];
+  snprintf(obj_path, sizeof(obj_path), "%s/%s", dir_name, file_name);
+  return BackedUpObject::from_path(obj_path, obj);
+}
+
+int BackedUpObject::from_path(const char *path, std::auto_ptr &obj)
+{
+  int ret;
+  FILE *fp = fopen(path, "r");
+  if (!fp) {
+    ret = errno;
+    if (ret != ENOENT) {
+      cerr << ERR_PREFIX << "BackedUpObject::from_path: error while trying to "
+	   << "open '" << path << "': " <<  cpp_strerror(ret) << std::endl;
+    }
+    return ret;
+  }
+  int fd = fileno(fp);
+  struct stat st_buf;
+  memset(&st_buf, 0, sizeof(st_buf));
+  ret = fstat(fd, &st_buf);
+  if (ret) {
+    ret = errno;
+    fclose(fp);
+    cerr << ERR_PREFIX << "BackedUpObject::from_path: error while trying "
+	 << "to stat '" << path << "': " <<  cpp_strerror(ret) << std::endl;
+    return ret;
+  }
+
+  // get fullname
+  ssize_t res = ceph_os_fgetxattr(fd, XATTR_FULLNAME, NULL, 0);
+  if (res <= 0) {
+    fclose(fp);
+    ret = errno;
+    if (res == 0) {
+      cerr << ERR_PREFIX << "BackedUpObject::from_path: found empty "
+	   << XATTR_FULLNAME << " attribute on '" << path
+	   << "'" << std::endl;
+      ret = ENODATA;
+    } else if (ret == ENODATA) {
+      cerr << ERR_PREFIX << "BackedUpObject::from_path: there was no "
+	   << XATTR_FULLNAME << " attribute found on '" << path
+	   << "'" << std::endl;
+    } else {
+      cerr << ERR_PREFIX << "getxattr error: " << cpp_strerror(ret) << std::endl;
+    }
+    return ret;
+  }
+  char rados_name_[res + 1];
+  memset(rados_name_, 0, sizeof(rados_name_));
+  res = ceph_os_fgetxattr(fd, XATTR_FULLNAME, rados_name_, res);
+  if (res < 0) {
+    ret = errno;
+    fclose(fp);
+    cerr << ERR_PREFIX << "BackedUpObject::getxattr(" << XATTR_FULLNAME
+	 << ") error: " << cpp_strerror(ret) << std::endl;
+    return ret;
+  }
+
+  BackedUpObject *o = new BackedUpObject(rados_name_,
+			     st_buf.st_size, st_buf.st_mtime);
+  if (!o) {
+    fclose(fp);
+    return ENOBUFS;
+  }
+  ret = o->read_xattrs_from_file(fileno(fp));
+  if (ret) {
+    fclose(fp);
+    cerr << ERR_PREFIX << "BackedUpObject::from_path(path = '"
+	 << path << "): read_xattrs_from_file returned " << ret << std::endl;
+    delete o;
+    return ret;
+  }
+
+  fclose(fp);
+  obj.reset(o);
+  return 0;
+}
+
+int BackedUpObject::from_rados(IoCtx& io_ctx, const char *rados_name_,
+		      auto_ptr &obj)
+{
+  uint64_t rados_size_ = 0;
+  time_t rados_time_ = 0;
+  int ret = io_ctx.stat(rados_name_, &rados_size_, &rados_time_);
+  if (ret == -ENOENT) {
+    // don't complain here about ENOENT
+    return ret;
+  } else if (ret < 0) {
+    cerr << ERR_PREFIX << "BackedUpObject::from_rados(rados_name_ = '"
+	 << rados_name_ << "'): stat failed with error " << ret << std::endl;
+    return ret;
+  }
+  BackedUpObject *o = new BackedUpObject(rados_name_, rados_size_, rados_time_);
+  ret = o->read_xattrs_from_rados(io_ctx);
+  if (ret) {
+    cerr << ERR_PREFIX << "BackedUpObject::from_rados(rados_name_ = '"
+	  << rados_name_ << "'): read_xattrs_from_rados returned "
+	  << ret << std::endl;
+    delete o;
+    return ret;
+  }
+  obj.reset(o);
+  return 0;
+}
+
+BackedUpObject::~BackedUpObject()
+{
+  for (std::map < std::string, Xattr* >::iterator x = xattrs.begin();
+	 x != xattrs.end(); ++x)
+  {
+    delete x->second;
+    x->second = NULL;
+  }
+  free(rados_name);
+}
+
+std::string BackedUpObject::get_fs_path(const ExportDir *export_dir) const
+{
+  return export_dir->get_fs_path(rados_name);
+}
+
+std::string BackedUpObject::xattrs_to_str() const
+{
+  ostringstream oss;
+  std::string prefix;
+  for (std::map < std::string, Xattr* >::const_iterator x = xattrs.begin();
+	 x != xattrs.end(); ++x)
+  {
+    char buf[x->second->len + 1];
+    memcpy(buf, x->second->data, x->second->len);
+    buf[x->second->len] = '\0';
+    oss << prefix << "{" << x->first << ":" << buf << "}";
+    prefix = ", ";
+  }
+  return oss.str();
+}
+
+void BackedUpObject::xattr_diff(const BackedUpObject *rhs,
+		std::list < std::string > &only_in_a,
+		std::list < std::string > &only_in_b,
+		std::list < std::string > &diff) const
+{
+  only_in_a.clear();
+  only_in_b.clear();
+  diff.clear();
+  for (std::map < std::string, Xattr* >::const_iterator x = xattrs.begin();
+	 x != xattrs.end(); ++x)
+  {
+    std::map < std::string, Xattr* >::const_iterator r = rhs->xattrs.find(x->first);
+    if (r == rhs->xattrs.end()) {
+      only_in_a.push_back(x->first);
+    }
+    else {
+      const Xattr &r_obj(*r->second);
+      const Xattr &x_obj(*x->second);
+      if (r_obj != x_obj)
+	diff.push_back(x->first);
+    }
+  }
+  for (std::map < std::string, Xattr* >::const_iterator r = rhs->xattrs.begin();
+	 r != rhs->xattrs.end(); ++r)
+  {
+    std::map < std::string, Xattr* >::const_iterator x = rhs->xattrs.find(r->first);
+    if (x == xattrs.end()) {
+      only_in_b.push_back(r->first);
+    }
+  }
+}
+
+void BackedUpObject::get_xattrs(std::list < std::string > &xattrs_) const
+{
+  for (std::map < std::string, Xattr* >::const_iterator r = xattrs.begin();
+	 r != xattrs.end(); ++r)
+  {
+    xattrs_.push_back(r->first);
+  }
+}
+
+const Xattr* BackedUpObject::get_xattr(const std::string &name) const
+{
+  std::map < std::string, Xattr* >::const_iterator x = xattrs.find(name);
+  if (x == xattrs.end())
+    return NULL;
+  else
+    return x->second;
+}
+
+const char *BackedUpObject::get_rados_name() const {
+  return rados_name;
+}
+
+uint64_t BackedUpObject::get_rados_size() const {
+  return rados_size;
+}
+
+time_t BackedUpObject::get_mtime() const {
+  return rados_time;
+}
+
+int BackedUpObject::download(IoCtx &io_ctx, const char *path)
+{
+  char tmp_path[strlen(path) + RADOS_SYNC_TMP_SUFFIX_LEN + 1];
+  snprintf(tmp_path, sizeof(tmp_path), "%s%s", path, RADOS_SYNC_TMP_SUFFIX);
+  FILE *fp = fopen(tmp_path, "w");
+  if (!fp) {
+    int err = errno;
+    cerr << ERR_PREFIX << "download: error opening '" << tmp_path << "':"
+	 <<  cpp_strerror(err) << std::endl;
+    return err;
+  }
+  int fd = fileno(fp);
+  uint64_t off = 0;
+  static const int CHUNK_SZ = 32765;
+  while (true) {
+    bufferlist bl;
+    int rlen = io_ctx.read(rados_name, bl, CHUNK_SZ, off);
+    if (rlen < 0) {
+      cerr << ERR_PREFIX << "download: io_ctx.read(" << rados_name << ") returned "
+	   << rlen << std::endl;
+      fclose(fp);
+      return rlen;
+    }
+    if (rlen < CHUNK_SZ)
+      off = 0;
+    else
+      off += rlen;
+    size_t flen = fwrite(bl.c_str(), 1, rlen, fp);
+    if (flen != (size_t)rlen) {
+      int err = errno;
+      cerr << ERR_PREFIX << "download: fwrite(" << tmp_path << ") error: "
+	   << cpp_strerror(err) << std::endl;
+      fclose(fp);
+      return err;
+    }
+    if (off == 0)
+      break;
+  }
+  size_t attr_sz = strlen(rados_name) + 1;
+  int res = ceph_os_fsetxattr(fd, XATTR_FULLNAME, rados_name, attr_sz);
+  if (res) {
+    int err = errno;
+    cerr << ERR_PREFIX << "download: fsetxattr(" << tmp_path << ") error: "
+	 << cpp_strerror(err) << std::endl;
+    fclose(fp);
+    return err;
+  }
+  if (fclose(fp)) {
+    int err = errno;
+    cerr << ERR_PREFIX << "download: fclose(" << tmp_path << ") error: "
+	 << cpp_strerror(err) << std::endl;
+    return err;
+  }
+  if (rename(tmp_path, path)) {
+    int err = errno;
+    cerr << ERR_PREFIX << "download: rename(" << tmp_path << ", "
+	 << path << ") error: " << cpp_strerror(err) << std::endl;
+    return err;
+  }
+  return 0;
+}
+
+int BackedUpObject::upload(IoCtx &io_ctx, const char *file_name, const char *dir_name)
+{
+  char path[strlen(file_name) + strlen(dir_name) + 2];
+  snprintf(path, sizeof(path), "%s/%s", dir_name, file_name);
+  FILE *fp = fopen(path, "r");
+  if (!fp) {
+    int err = errno;
+    cerr << ERR_PREFIX << "upload: error opening '" << path << "': "
+	 << cpp_strerror(err) << std::endl;
+    return err;
+  }
+  // Need to truncate RADOS object to size 0, in case there is
+  // already something there.
+  int ret = io_ctx.trunc(rados_name, 0);
+  if (ret) {
+    cerr << ERR_PREFIX << "upload: trunc failed with error " << ret << std::endl;
+    fclose(fp);
+    return ret;
+  }
+  uint64_t off = 0;
+  static const int CHUNK_SZ = 32765;
+  while (true) {
+    char buf[CHUNK_SZ];
+    int flen = fread(buf, 1, CHUNK_SZ, fp);
+    if (flen < 0) {
+      int err = errno;
+      cerr << ERR_PREFIX << "upload: fread(" << file_name << ") error: "
+	   << cpp_strerror(err) << std::endl;
+      fclose(fp);
+      return err;
+    }
+    if ((flen == 0) && (off != 0)) {
+      fclose(fp);
+      break;
+    }
+    // There must be a zero-copy way to do this?
+    bufferlist bl;
+    bl.append(buf, flen);
+    int rlen = io_ctx.write(rados_name, bl, flen, off);
+    if (rlen < 0) {
+      fclose(fp);
+      cerr << ERR_PREFIX << "upload: rados_write error: " << rlen << std::endl;
+      return rlen;
+    }
+    if (rlen != flen) {
+      fclose(fp);
+      cerr << ERR_PREFIX << "upload: rados_write error: short write" << std::endl;
+      return -EIO;
+    }
+    off += rlen;
+    if (flen < CHUNK_SZ) {
+      fclose(fp);
+      return 0;
+    }
+  }
+  return 0;
+}
+
+BackedUpObject::BackedUpObject(const char *rados_name_,
+			       uint64_t rados_size_, time_t rados_time_)
+  : rados_name(strdup(rados_name_)),
+    rados_size(rados_size_),
+    rados_time(rados_time_)
+{
+}
+
+int BackedUpObject::read_xattrs_from_file(int fd)
+{
+  ssize_t blen = ceph_os_flistxattr(fd, NULL, 0);
+  if (blen > 0x1000000) {
+    cerr << ERR_PREFIX << "BackedUpObject::read_xattrs_from_file: unwilling "
+	 << "to allocate a buffer of size " << blen << " on the stack for "
+	 << "flistxattr." << std::endl;
+    return ENOBUFS;
+  }
+  char buf[blen + 1];
+  memset(buf, 0, sizeof(buf));
+  ssize_t blen2 = ceph_os_flistxattr(fd, buf, blen);
+  if (blen != blen2) {
+    cerr << ERR_PREFIX << "BackedUpObject::read_xattrs_from_file: xattrs changed while "
+	 << "we were trying to "
+	 << "list them? First length was " << blen << ", but now it's " << blen2
+	 << std::endl;
+    return EDOM;
+  }
+  const char *b = buf;
+  while (*b) {
+    size_t bs = strlen(b);
+    std::string xattr_name = get_user_xattr_name(b);
+    if (!xattr_name.empty()) {
+      ssize_t attr_len = ceph_os_fgetxattr(fd, b, NULL, 0);
+      if (attr_len < 0) {
+	int err = errno;
+	cerr << ERR_PREFIX << "BackedUpObject::read_xattrs_from_file: "
+	     << "fgetxattr(rados_name = '" << rados_name << "', xattr_name='"
+	     << xattr_name << "') failed: " << cpp_strerror(err) << std::endl;
+	return EDOM;
+      }
+      char *attr = (char*)malloc(attr_len);
+      if (!attr) {
+	cerr << ERR_PREFIX << "BackedUpObject::read_xattrs_from_file: "
+	     << "malloc(" << attr_len << ") failed for xattr_name='"
+	     << xattr_name << "'" << std::endl;
+	return ENOBUFS;
+      }
+      ssize_t attr_len2 = ceph_os_fgetxattr(fd, b, attr, attr_len);
+      if (attr_len2 < 0) {
+	int err = errno;
+	cerr << ERR_PREFIX << "BackedUpObject::read_xattrs_from_file: "
+	     << "fgetxattr(rados_name = '" << rados_name << "', "
+	     << "xattr_name='" << xattr_name << "') failed: "
+	     << cpp_strerror(err) << std::endl;
+	free(attr);
+	return EDOM;
+      }
+      if (attr_len2 != attr_len) {
+	cerr << ERR_PREFIX << "BackedUpObject::read_xattrs_from_file: xattr "
+	     << "changed while we were trying to get it? "
+	     << "fgetxattr(rados_name = '"<< rados_name
+	     << "', xattr_name='" << xattr_name << "') returned a different length "
+	     << "than when we first called it! old_len = " << attr_len
+	     << "new_len = " << attr_len2 << std::endl;
+	free(attr);
+	return EDOM;
+      }
+      xattrs[xattr_name] = new Xattr(attr, attr_len);
+    }
+    b += (bs + 1);
+  }
+  return 0;
+}
+
+int BackedUpObject::read_xattrs_from_rados(IoCtx &io_ctx)
+{
+  map attrset;
+  int ret = io_ctx.getxattrs(rados_name, attrset);
+  if (ret) {
+    cerr << ERR_PREFIX << "BackedUpObject::read_xattrs_from_rados: "
+	 << "getxattrs failed with error code " << ret << std::endl;
+    return ret;
+  }
+  for (map::iterator i = attrset.begin();
+       i != attrset.end(); )
+  {
+    bufferlist& bl(i->second);
+    char *data = (char*)malloc(bl.length());
+    if (!data)
+      return ENOBUFS;
+    memcpy(data, bl.c_str(), bl.length());
+    Xattr *xattr = new Xattr(data, bl.length());
+    if (!xattr) {
+      free(data);
+      return ENOBUFS;
+    }
+    xattrs[i->first] = xattr;
+    attrset.erase(i++);
+  }
+  return 0;
+}
+
+int rados_tool_sync(const std::map < std::string, std::string > &opts,
+                             std::vector &args)
+{
+  int ret;
+  bool force = opts.count("force");
+  bool delete_after = opts.count("delete-after");
+  bool create = opts.count("create");
+
+  std::map < std::string, std::string >::const_iterator n = opts.find("workers");
+  int num_threads;
+  if (n == opts.end()) {
+    num_threads = DEFAULT_NUM_RADOS_WORKER_THREADS;
+  }
+  else {
+    std::string err;
+    num_threads = strict_strtol(n->second.c_str(), 10, &err);
+    if (!err.empty()) {
+      cerr << "rados: can't parse number of worker threads given: "
+	   << err << std::endl;
+      return 1;
+    }
+    if ((num_threads < 1) || (num_threads > 9000)) {
+      cerr << "rados: unreasonable value given for num_threads: "
+	   << num_threads << std::endl;
+      return 1;
+    }
+  }
+
+
+  std::string action, src, dst;
+  std::vector::iterator i = args.begin();
+  if ((i != args.end()) &&
+      ((strcmp(*i, "import") == 0) || (strcmp(*i, "export") == 0))) {
+    action = *i;
+    ++i;
+  }
+  else {
+    cerr << "rados" << ": You must specify either 'import' or 'export'.\n";
+    cerr << "Use --help to show help.\n";
+    exit(1);
+  }
+  if (i != args.end()) {
+    src = *i;
+    ++i;
+  }
+  else {
+    cerr << "rados" << ": You must give a source.\n";
+    cerr << "Use --help to show help.\n";
+    exit(1);
+  }
+  if (i != args.end()) {
+    dst = *i;
+    ++i;
+  }
+  else {
+    cerr << "rados" << ": You must give a destination.\n";
+    cerr << "Use --help to show help.\n";
+    exit(1);
+  }
+
+  // open rados
+  Rados rados;
+  if (rados.init_with_context(g_ceph_context) < 0) {
+     cerr << "rados" << ": failed to initialize Rados!" << std::endl;
+     exit(1);
+  }
+  if (rados.connect() < 0) {
+     cerr << "rados" << ": failed to connect to Rados cluster!" << std::endl;
+     exit(1);
+  }
+  IoCtx io_ctx;
+  std::string pool_name = (action == "import") ? dst : src;
+  ret = rados.ioctx_create(pool_name.c_str(), io_ctx);
+  if ((ret == -ENOENT) && (action == "import")) {
+    if (create) {
+      ret = rados.pool_create(pool_name.c_str());
+      if (ret) {
+	cerr << "rados" << ": pool_create failed with error " << ret
+	     << std::endl;
+	exit(ret);
+      }
+      ret = rados.ioctx_create(pool_name.c_str(), io_ctx);
+    }
+    else {
+      cerr << "rados" << ": pool '" << pool_name << "' does not exist. Use "
+	   << "--create to try to create it." << std::endl;
+      exit(ENOENT);
+    }
+  }
+  if (ret < 0) {
+    cerr << "rados" << ": error opening pool " << pool_name << ": "
+	 << cpp_strerror(ret) << std::endl;
+    exit(ret);
+  }
+
+  IoCtxDistributor *io_ctx_dist = IoCtxDistributor::instance();
+  ret = io_ctx_dist->init(rados, pool_name.c_str(), num_threads);
+  if (ret) {
+    cerr << ERR_PREFIX << "failed to initialize Rados io contexts."
+	 << std::endl;
+    _exit(ret);
+  }
+
+  ThreadPool thread_pool(g_ceph_context, "rados_sync_threadpool", num_threads);
+  thread_pool.start();
+
+  if (action == "import") {
+    ret = do_rados_import(&thread_pool, io_ctx, io_ctx_dist, src.c_str(),
+		     force, delete_after);
+    thread_pool.stop();
+    return ret;
+  }
+  else {
+    ret = do_rados_export(&thread_pool, io_ctx, io_ctx_dist, dst.c_str(),
+		     create, force, delete_after);
+    thread_pool.stop();
+    return ret;
+  }
+}
diff --git a/ceph/src/tools/rados/rados_sync.h b/ceph/src/tools/rados/rados_sync.h
new file mode 100644
index 00000000..d7624503
--- /dev/null
+++ b/ceph/src/tools/rados/rados_sync.h
@@ -0,0 +1,216 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_RADOS_SYNC_H
+#define CEPH_RADOS_SYNC_H
+
+#include 
+#include "include/atomic.h"
+#include "common/WorkQueue.h"
+
+#include 
+#include 
+
+namespace librados {
+  class IoCtx;
+  class Rados;
+}
+
+extern const char USER_XATTR_PREFIX[];
+extern const char RADOS_SYNC_TMP_SUFFIX[];
+#define ERR_PREFIX "[ERROR]        "
+#define DEFAULT_NUM_RADOS_WORKER_THREADS 5
+
+/* Linux seems to use ENODATA instead of ENOATTR when an extended attribute
+ * is missing */
+#ifndef ENOATTR
+#define ENOATTR ENODATA
+#endif
+
+enum {
+  CHANGED_XATTRS = 0x1,
+  CHANGED_CONTENTS = 0x2,
+};
+
+/** Given the name of an extended attribute from a file in the filesystem,
+ * returns an empty string if the extended attribute does not represent a rados
+ * user extended attribute. Otherwise, returns the name of the rados extended
+ * attribute.
+ *
+ * Rados user xattrs are prefixed with USER_XATTR_PREFIX.
+ */
+std::string get_user_xattr_name(const char *fs_xattr_name);
+
+/* Returns true if 'suffix' is a suffix of str */
+bool is_suffix(const char *str, const char *suffix);
+
+/** Represents a directory in the filesystem that we export rados objects to (or
+ * import them from.)
+ */
+class ExportDir
+{
+public:
+  static ExportDir* create_for_writing(const std::string &path, int version,
+					  bool create);
+  static ExportDir* from_file_system(const std::string &path);
+
+  /* Given a rados object name, return something which looks kind of like the
+   * first part of the name.
+   *
+   * The actual file name that the backed-up object is stored in is irrelevant
+   * to rados_sync. The only reason to make it human-readable at all is to make
+   * things easier on sysadmins.  The XATTR_FULLNAME extended attribute has the
+   * real, full object name.
+  *
+   * This function turns unicode into a bunch of 'at' signs. This could be
+   * fixed. If you try, be sure to handle all the multibyte characters
+   * correctly.
+   * I guess a better hash would be nice too.
+   */
+  std::string get_fs_path(const std::string &rados_name) const;
+
+private:
+  explicit ExportDir(const std::string &path_);
+
+  std::string path;
+};
+
+/** Smart pointer wrapper for a DIR*
+ */
+class DirHolder {
+public:
+  DirHolder();
+  ~DirHolder();
+  int opendir(const char *dir_name);
+  DIR *dp;
+};
+
+/** IoCtxDistributor is a singleton that distributes out IoCtx instances to
+ * different threads.
+ */
+class IoCtxDistributor
+{
+public:
+  static IoCtxDistributor* instance();
+  int init(librados::Rados &cluster, const char *pool_name, int num_ioctxes);
+  void clear();
+  librados::IoCtx& get_ioctx();
+private:
+  static IoCtxDistributor *s_instance;
+  IoCtxDistributor();
+  ~IoCtxDistributor();
+
+  ceph::atomic_t m_highest_iod_idx;
+
+  /* NB: there might be some false sharing here that we could optimize
+   * away in the future */
+  std::vector m_io_ctxes;
+};
+
+class RadosSyncWQ : public ThreadPool::WorkQueue {
+public:
+  RadosSyncWQ(IoCtxDistributor *io_ctx_dist, time_t timeout, time_t suicide_timeout, ThreadPool *tp);
+protected:
+  IoCtxDistributor *m_io_ctx_dist;
+private:
+  bool _enqueue(std::string *s);
+  void _dequeue(std::string *o);
+  bool _empty();
+  std::string *_dequeue();
+  void _process_finish(std::string *s);
+  void _clear();
+  std::deque m_items;
+};
+
+/* Stores a length and a chunk of malloc()ed data */
+class Xattr {
+public:
+  Xattr(char *data_, ssize_t len_);
+  ~Xattr();
+  bool operator==(const class Xattr &rhs) const;
+  bool operator!=(const class Xattr &rhs) const;
+
+  char *data;
+  ssize_t len;
+};
+
+/* Represents an object that we are backing up */
+class BackedUpObject
+{
+public:
+  static int from_file(const char *file_name, const char *dir_name,
+			    std::auto_ptr &obj);
+  static int from_path(const char *path, std::auto_ptr &obj);
+  static int from_rados(librados::IoCtx& io_ctx, const char *rados_name_,
+			auto_ptr &obj);
+  ~BackedUpObject();
+
+  /* Get the mangled name for this rados object. */
+  std::string get_fs_path(const ExportDir *export_dir) const;
+
+  /* Convert the xattrs on this BackedUpObject to a kind of JSON-like string.
+   * This is only used for debugging.
+   * Note that we're assuming we can just treat the xattr data as a
+   * null-terminated string, which isn't true. Again, this is just for debugging,
+   * so it doesn't matter.
+   */
+  std::string xattrs_to_str() const;
+
+  /* Diff the extended attributes on this BackedUpObject with those found on a
+   * different BackedUpObject
+   */
+  void xattr_diff(const BackedUpObject *rhs,
+		  std::list < std::string > &only_in_a,
+		  std::list < std::string > &only_in_b,
+		  std::list < std::string > &diff) const;
+
+  void get_xattrs(std::list < std::string > &xattrs_) const;
+
+  const Xattr* get_xattr(const std::string &name) const;
+
+  const char *get_rados_name() const;
+
+  uint64_t get_rados_size() const;
+
+  time_t get_mtime() const;
+
+  int download(librados::IoCtx &io_ctx, const char *path);
+
+  int upload(librados::IoCtx &io_ctx, const char *file_name, const char *dir_name);
+
+private:
+  BackedUpObject(const char *rados_name_, uint64_t rados_size_, time_t rados_time_);
+
+  int read_xattrs_from_file(int fd);
+
+  int read_xattrs_from_rados(librados::IoCtx &io_ctx);
+
+  // don't allow copying
+  BackedUpObject &operator=(const BackedUpObject &rhs);
+  BackedUpObject(const BackedUpObject &rhs);
+
+  char *rados_name;
+  uint64_t rados_size;
+  uint64_t rados_time;
+  std::map < std::string, Xattr* > xattrs;
+};
+
+extern int do_rados_import(ThreadPool *tp, librados::IoCtx &io_ctx,
+    IoCtxDistributor* io_ctx_dist, const char *dir_name,
+    bool force, bool delete_after);
+extern int do_rados_export(ThreadPool *tp, librados::IoCtx& io_ctx,
+    IoCtxDistributor *io_ctx_dist, const char *dir_name, 
+    bool create, bool force, bool delete_after);
+
+#endif
diff --git a/ceph/src/tools/radosacl.cc b/ceph/src/tools/radosacl.cc
new file mode 100644
index 00000000..d2f7ca5c
--- /dev/null
+++ b/ceph/src/tools/radosacl.cc
@@ -0,0 +1,202 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "include/types.h"
+#include "include/rados/librados.hpp"
+using namespace librados;
+
+#include 
+
+#include 
+#include 
+#include 
+
+void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  str[0] = '\0';
+  for (int i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+
+#define ID_SIZE 8
+
+#define ACL_RD	0x1
+#define ACL_WR	0x2
+
+struct ACLID {
+  char id[ID_SIZE + 1];
+
+  void encode(bufferlist& bl) const {
+    bl.append((const char *)id, ID_SIZE);
+  }
+  void decode(bufferlist::iterator& iter) {
+    iter.copy(ID_SIZE, (char *)id);
+  }
+};
+WRITE_CLASS_ENCODER(ACLID)
+
+typedef __u32 ACLFlags;
+
+
+inline bool operator<(const ACLID& l, const ACLID& r)
+{
+  return (memcmp(&l, &r, ID_SIZE) > 0);
+}
+
+struct ACLPair {
+  ACLID id;
+  ACLFlags flags;
+};
+
+class ObjectACLs {
+  map acls_map;
+
+public:
+
+  void encode(bufferlist& bl) const {
+    ::encode(acls_map, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(acls_map, bl);
+  }
+
+  int read_acl(ACLID& id, ACLFlags *flags);
+  void set_acl(ACLID& id, ACLFlags flags);
+};
+WRITE_CLASS_ENCODER(ObjectACLs)
+
+int ObjectACLs::read_acl(ACLID& id, ACLFlags *flags)
+{
+  if (!flags)
+    return -EINVAL;
+
+  map::iterator iter = acls_map.find(id);
+
+  if (iter == acls_map.end())
+    return -ENOENT;
+
+  *flags = iter->second;
+
+  return 0;
+}
+
+void ObjectACLs::set_acl(ACLID& id, ACLFlags flags)
+{
+  acls_map[id] = flags;
+}
+
+
+
+class ACLEntity
+{
+  string name;
+  map groups;
+};
+
+typedef map tACLIDEntityMap;
+
+static map users;
+static map groups;
+
+void get_user(ACLID& aclid, ACLEntity *entity)
+{
+  //users.find(aclid);
+}
+
+
+
+
+
+int main(int argc, const char **argv) 
+{
+  Rados rados;
+  if (rados.init(NULL) < 0) {
+     cerr << "couldn't initialize rados!" << std::endl;
+     exit(1);
+  }
+  if (rados.conf_read_file(NULL)) {
+     cerr << "couldn't read Ceph configuration file!" << std::endl;
+     exit(1);
+  }
+  if (rados.connect() < 0) {
+     cerr << "couldn't connect to cluster!" << std::endl;
+     exit(1);
+  }
+
+  time_t tm;
+  bufferlist bl, bl2;
+  char buf[128];
+
+  time(&tm);
+  snprintf(buf, 128, "%s", ctime(&tm));
+  bl.append(buf, strlen(buf));
+
+  const char *oid = "bar";
+
+  IoCtx io_ctx;
+  int r = rados.ioctx_create("data", io_ctx);
+  cout << "open io_ctx result = " << r << " pool = " << io_ctx.get_pool_name() << std::endl;
+
+  ACLID id;
+
+  snprintf(id.id, ID_SIZE + 1, "%.16x", 0x1234);
+  cout << "id=" << id.id << std::endl;
+
+  r = io_ctx.exec(oid, "acl", "get", bl, bl2);
+  cout << "exec returned " << r << " len=" << bl2.length() << std::endl;
+  ObjectACLs oa;
+  if (r >= 0) {
+    bufferlist::iterator iter = bl2.begin();
+    oa.decode(iter);
+  }
+
+  oa.set_acl(id, ACL_RD);
+  bl.clear();
+  oa.encode(bl);
+  r = io_ctx.exec(oid, "acl", "set", bl, bl2);
+
+  const unsigned char *md5 = (const unsigned char *)bl2.c_str();
+  char md5_str[bl2.length()*2 + 1];
+  buf_to_hex(md5, bl2.length(), md5_str);
+  cout << "md5 result=" << md5_str << std::endl;
+
+  int size = io_ctx.read(oid, bl2, 128, 0);
+  cout << "read result=" << bl2.c_str() << std::endl;
+  cout << "size=" << size << std::endl;
+
+#if 0
+  Rados::ListCtx ctx;
+  int entries;
+  do {
+    list vec;
+    r = rados.list(io_ctx, 2, vec, ctx);
+    entries = vec.size();
+    cout << "list result=" << r << " entries=" << entries << std::endl;
+    list::iterator iter;
+    for (iter = vec.begin(); iter != vec.end(); ++iter) {
+      cout << *iter << std::endl;
+    }
+  } while (entries);
+#endif
+#if 0
+  r = rados.remove(io_ctx, oid);
+  cout << "remove result=" << r << std::endl;
+  rados.close_io_ctx(io_ctx);
+#endif
+
+  return 0;
+}
+
diff --git a/ceph/src/tools/rest_bench.cc b/ceph/src/tools/rest_bench.cc
new file mode 100644
index 00000000..d617fe16
--- /dev/null
+++ b/ceph/src/tools/rest_bench.cc
@@ -0,0 +1,802 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/types.h"
+#include "include/atomic.h"
+
+#include "common/obj_bencher.h"
+#include "common/config.h"
+#include "common/debug.h"
+#include "common/ceph_argparse.h"
+#include "common/WorkQueue.h"
+#include "msg/Message.h"
+#include "global/global_init.h"
+
+#include "libs3.h"
+
+#include 
+
+#include 
+
+#define DEFAULT_USER_AGENT "rest-bench"
+#define DEFAULT_BUCKET "rest-bench-bucket"
+
+void usage(ostream& out)
+{
+  out <<					\
+"usage: rest-bench [options] \n"
+"       rest-bench [options] cleanup [--run-name run_name] [--prefix prefix]\n"
+"BENCHMARK OPTIONS\n"
+"   --seconds\n"
+"        benchmak length (default: 60)\n"
+"   -t concurrent_operations\n"
+"   --concurrent-ios=concurrent_operations\n"
+"        select bucket by name\n"
+"   -b op-size\n"
+"   --block-size=op-size\n"
+"        set the size of write ops for put or benchmarking\n"
+"   --show-time\n"
+"        prefix output lines with date and time\n"
+"   --no-cleanup\n"
+"        do not clean up data after write bench\n"
+"REST CONFIG OPTIONS\n"
+"   --api-host=bhost\n"
+"        host name\n"
+"   --bucket=bucket\n"
+"        select bucket by name\n"
+"   --access-key=access_key\n"
+"        access key to RESTful storage provider\n"
+"   --secret=secret_key\n"
+"        secret key for the specified access key\n"
+"   --protocol=\n"
+"        protocol to be used (default: http)\n"
+"   --uri_style=\n"
+"        uri style in requests (default: path)\n";
+}
+
+static void usage_exit()
+{
+  usage(cerr);
+  exit(1);
+}
+
+enum OpType {
+  OP_NONE    = 0,
+  OP_GET_OBJ = 1,
+  OP_PUT_OBJ = 2,
+  OP_DELETE_OBJ = 3,
+  OP_LIST_BUCKET = 4,
+  OP_CLEANUP = 5,
+};
+
+struct req_context : public RefCountedObject {
+  bool complete;
+  S3Status status;
+  S3RequestContext *ctx;
+  void (*cb)(void *, void *);
+  void *arg;
+  bufferlist *in_bl;
+  bufferlist out_bl;
+  uint64_t off;
+  uint64_t len;
+  const char *list_start;
+  std::list* list_objects;
+  int list_count;
+  string oid;
+  Mutex lock;
+  Cond cond;
+  S3BucketContext *bucket_ctx;
+
+  bool should_destroy_ctx;
+
+  OpType op;
+
+  bool used;
+
+  req_context() : complete(false), status(S3StatusOK), ctx(NULL), cb(NULL), arg(NULL), in_bl(NULL), off(0), len(0),
+                  lock("req_context"), bucket_ctx(NULL), should_destroy_ctx(false), op(OP_NONE), used(false) {}
+  ~req_context() {
+    if (should_destroy_ctx) {
+      S3_destroy_request_context(ctx);
+    }
+  }
+
+  int init_ctx() {
+    S3Status status = S3_create_request_context(&ctx);
+    if (status != S3StatusOK) {
+      cerr << "failed to create context: " << S3_get_status_name(status) << std::endl;
+      return -EINVAL;
+    }
+    should_destroy_ctx = true;
+
+    return 0;
+  }
+
+  int ret() {
+    if (status != S3StatusOK) {
+      return -EINVAL;
+    }
+    return 0;
+  }
+};
+
+static S3Status properties_callback(const S3ResponseProperties *properties, void *cb_data)
+{
+  return S3StatusOK;
+}
+
+static void complete_callback(S3Status status, const S3ErrorDetails *details, void *cb_data)
+{
+  if (!cb_data)
+    return;
+
+  struct req_context *ctx = (struct req_context *)cb_data;
+
+  ctx->lock.Lock();
+  ctx->status = status;
+  ctx->lock.Unlock();
+
+  if (ctx->cb) {
+    ctx->cb((void *)ctx->cb, ctx->arg);
+  }
+
+  ctx->put();
+}
+
+static S3Status get_obj_callback(int size, const char *buf,
+                                 void *cb_data)
+{
+  if (!cb_data)
+    return S3StatusOK;
+
+  struct req_context *ctx = (struct req_context *)cb_data;
+
+  ctx->in_bl->append(buf, size);
+
+  return S3StatusOK;
+}
+
+static int put_obj_callback(int size, char *buf,
+                            void *cb_data)
+{
+  if (!cb_data)
+    return 0;
+
+  struct req_context *ctx = (struct req_context *)cb_data;
+
+  int chunk = ctx->out_bl.length() - ctx->off;
+  if (!chunk)
+    return 0;
+
+  if (chunk > size)
+    chunk = size;
+
+  memcpy(buf, ctx->out_bl.c_str() + ctx->off, chunk);
+
+  ctx->off += chunk;
+
+  return chunk;
+}
+
+static S3Status list_bucket_callback(int is_truncated, const char *next_marker,
+                                int count, const S3ListBucketContent *objects,
+                                int prefix_count, const char **prefixes,
+                                void *cb_data)
+{
+  if (!cb_data)
+    return S3StatusOK;
+
+  struct req_context *ctx = (struct req_context *)cb_data;
+
+  ctx->list_start = next_marker;
+
+  for (int i = 0; i < count; ++i) {
+    ctx->list_objects->push_back(objects[i].key);
+  }
+
+  return S3StatusOK;
+}
+
+class RESTDispatcher {
+  deque m_req_queue;
+  ThreadPool m_tp;
+
+  S3ResponseHandler response_handler;
+  S3GetObjectHandler get_obj_handler;
+  S3PutObjectHandler put_obj_handler;
+  S3ListBucketHandler list_bucket_handler;
+
+  struct DispatcherWQ : public ThreadPool::WorkQueue {
+    RESTDispatcher *dispatcher;
+    DispatcherWQ(RESTDispatcher *p, time_t timeout, time_t suicide_timeout, ThreadPool *tp)
+      : ThreadPool::WorkQueue("REST", timeout, suicide_timeout, tp), dispatcher(p) {}
+
+    bool _enqueue(req_context *req) {
+      dispatcher->m_req_queue.push_back(req);
+      _dump_queue();
+      return true;
+    }
+    void _dequeue(req_context *req) {
+      assert(0);
+    }
+    bool _empty() {
+      return dispatcher->m_req_queue.empty();
+    }
+    req_context *_dequeue() {
+      if (dispatcher->m_req_queue.empty())
+	return NULL;
+      req_context *req = dispatcher->m_req_queue.front();
+      dispatcher->m_req_queue.pop_front();
+      _dump_queue();
+      return req;
+    }
+    void _process(req_context *req) {
+      dispatcher->process_context(req);
+    }
+    void _dump_queue() {
+      deque::iterator iter;
+      if (dispatcher->m_req_queue.empty()) {
+        generic_dout(20) << "DispatcherWQ: empty" << dendl;
+        return;
+      }
+      generic_dout(20) << "DispatcherWQ:" << dendl;
+      for (iter = dispatcher->m_req_queue.begin(); iter != dispatcher->m_req_queue.end(); ++iter) {
+        generic_dout(20) << "req: " << hex << *iter << dec << dendl;
+      }
+    }
+    void _clear() {
+      assert(dispatcher->m_req_queue.empty());
+    }
+  } req_wq;
+
+public:
+  CephContext *cct;
+  RESTDispatcher(CephContext *cct_, int num_threads)
+    : m_tp(cct_, "RESTDispatcher::m_tp", num_threads),
+      req_wq(this, cct_->_conf->rgw_op_thread_timeout,
+        cct_->_conf->rgw_op_thread_suicide_timeout, &m_tp),
+      cct(cct_) {
+
+
+    response_handler.propertiesCallback = properties_callback;
+    response_handler.completeCallback = complete_callback;
+
+    get_obj_handler.responseHandler = response_handler;
+    get_obj_handler.getObjectDataCallback = get_obj_callback;
+
+    put_obj_handler.responseHandler = response_handler;
+    put_obj_handler.putObjectDataCallback = put_obj_callback;
+
+    list_bucket_handler.responseHandler = response_handler;
+    list_bucket_handler.listBucketCallback = list_bucket_callback;
+
+  }
+  void process_context(req_context *ctx);
+  void get_obj(req_context *ctx);
+  void put_obj(req_context *ctx);
+  void delete_obj(req_context *ctx);
+  void list_bucket(req_context *ctx);
+
+  void queue(req_context *ctx) {
+    req_wq.queue(ctx);
+  }
+
+  void start() {
+    m_tp.start();
+  }
+};
+
+void RESTDispatcher::process_context(req_context *ctx)
+{
+  ctx->get();
+
+  switch (ctx->op) {
+    case OP_GET_OBJ:
+      get_obj(ctx);
+      break;
+    case OP_PUT_OBJ:
+      put_obj(ctx);
+      break;
+    case OP_DELETE_OBJ:
+      delete_obj(ctx);
+      break;
+    case OP_LIST_BUCKET:
+      list_bucket(ctx);
+      break;
+    default:
+      assert(0);
+  }
+
+  S3Status status = S3_runall_request_context(ctx->ctx);
+
+  if (status != S3StatusOK) {
+    cerr << "ERROR: S3_runall_request_context() returned " << S3_get_status_name(status) << std::endl;
+    ctx->status = status;
+  } else if (ctx->status != S3StatusOK) {
+    cerr << "ERROR: " << ctx->oid << ": " << S3_get_status_name(ctx->status) << std::endl;
+  }
+
+  ctx->lock.Lock();
+  ctx->complete = true;
+  ctx->cond.SignalAll();
+  ctx->lock.Unlock();
+
+  ctx->put();
+}
+
+void RESTDispatcher::put_obj(req_context *ctx)
+{
+  S3_put_object(ctx->bucket_ctx, ctx->oid.c_str(),
+                ctx->out_bl.length(),
+                NULL,
+                ctx->ctx,
+                &put_obj_handler, ctx);
+}
+
+void RESTDispatcher::get_obj(req_context *ctx)
+{
+  S3_get_object(ctx->bucket_ctx, ctx->oid.c_str(), NULL, 0, ctx->len, ctx->ctx,
+                &get_obj_handler, ctx);
+}
+
+void RESTDispatcher::delete_obj(req_context *ctx)
+{
+
+  S3_delete_object(ctx->bucket_ctx, ctx->oid.c_str(),
+                   ctx->ctx, &response_handler, ctx);
+}
+
+void RESTDispatcher::list_bucket(req_context *ctx)
+{
+  S3_list_bucket(ctx->bucket_ctx,
+                 NULL, ctx->list_start,
+                 NULL, ctx->list_count,
+                 ctx->ctx,
+                 &list_bucket_handler, ctx);
+}
+
+class RESTBencher : public ObjBencher {
+  RESTDispatcher *dispatcher;
+  struct req_context **completions;
+  struct S3RequestContext **handles;
+  S3BucketContext bucket_ctx;
+  const char *list_start;
+  bool bucket_list_done;
+  string user_agent;
+  string host;
+  string bucket;
+  S3Protocol protocol;
+  string access_key;
+  string secret;
+  int concurrentios;
+
+protected:
+  int rest_init() {
+    S3Status status = S3_initialize(user_agent.c_str(), S3_INIT_ALL, host.c_str());
+    if (status != S3StatusOK) {
+      cerr << "failed to init: " << S3_get_status_name(status) << std::endl;
+      return -EINVAL;
+    }
+
+
+    return 0;
+  }
+
+
+  int completions_init(int _concurrentios) {
+    concurrentios = _concurrentios;
+    completions = new req_context *[concurrentios];
+    handles = new S3RequestContext *[concurrentios];
+    for (int i = 0; i < concurrentios; i++) {
+      completions[i] = NULL;
+      S3Status status = S3_create_request_context(&handles[i]);
+      if (status != S3StatusOK) {
+        cerr << "failed to create context: " << S3_get_status_name(status) << std::endl;
+        return -EINVAL;
+      }
+    }
+    return 0;
+  }
+  void completions_done() {
+    delete[] completions;
+    completions = NULL;
+    for (int i = 0; i < concurrentios; i++) {
+      S3_destroy_request_context(handles[i]);
+    }
+    delete[] handles;
+    handles = NULL;
+  }
+  int create_completion(int slot, void (*cb)(void *, void*), void *arg) {
+    assert (!completions[slot]);
+
+    struct req_context *ctx = new req_context;
+    ctx->ctx = handles[slot];
+    assert (!ctx->used);
+    ctx->used = true;
+    ctx->cb = cb;
+    ctx->arg = arg;
+
+    completions[slot] = ctx;
+
+    return 0;
+  }
+  void release_completion(int slot) {
+    struct req_context *ctx = completions[slot];
+
+    ctx->used = false;
+
+    ctx->put();
+    completions[slot] = 0;
+  }
+
+  int aio_read(const std::string& oid, int slot, bufferlist *pbl, size_t len) {
+    struct req_context *ctx = completions[slot];
+
+    ctx->get();
+    ctx->in_bl = pbl;
+    ctx->oid = oid;
+    ctx->len = len;
+    ctx->bucket_ctx = &bucket_ctx;
+    ctx->op = OP_GET_OBJ;
+
+    dispatcher->queue(ctx);
+
+    return 0;
+  }
+
+  int aio_write(const std::string& oid, int slot, bufferlist& bl, size_t len) {
+    struct req_context *ctx = completions[slot];
+
+    ctx->get();
+    ctx->bucket_ctx = &bucket_ctx;
+    ctx->out_bl = bl;
+    ctx->oid = oid;
+    ctx->len = len;
+    ctx->op = OP_PUT_OBJ;
+
+    dispatcher->queue(ctx);
+    return 0;
+  }
+
+  int aio_remove(const std::string& oid, int slot) {
+    struct req_context *ctx = completions[slot];
+
+    ctx->get();
+    ctx->bucket_ctx = &bucket_ctx;
+    ctx->oid = oid;
+    ctx->op = OP_DELETE_OBJ;
+
+    dispatcher->queue(ctx);
+    return 0;
+  }
+
+  int sync_read(const std::string& oid, bufferlist& bl, size_t len) {
+    struct req_context *ctx = new req_context;
+    int ret = ctx->init_ctx();
+    if (ret < 0) {
+      return ret;
+    }
+    ctx->in_bl = &bl;
+    ctx->get();
+    ctx->bucket_ctx = &bucket_ctx;
+    ctx->oid = oid;
+    ctx->len = len;
+    ctx->op = OP_GET_OBJ;
+
+    dispatcher->process_context(ctx);
+    ret = ctx->ret();
+    ctx->put();
+    return bl.length();
+  }
+  int sync_write(const std::string& oid, bufferlist& bl, size_t len) {
+    struct req_context *ctx = new req_context;
+    int ret = ctx->init_ctx();
+    if (ret < 0) {
+      return ret;
+    }
+    ctx->get();
+    ctx->out_bl = bl;
+    ctx->bucket_ctx = &bucket_ctx;
+    ctx->oid = oid;
+    ctx->op = OP_PUT_OBJ;
+
+    dispatcher->process_context(ctx);
+    ret = ctx->ret();
+    ctx->put();
+    return ret;
+  }
+  int sync_remove(const std::string& oid) {
+    struct req_context *ctx = new req_context;
+    int ret = ctx->init_ctx();
+    if (ret < 0) {
+      return ret;
+    }
+    ctx->get();
+    ctx->bucket_ctx = &bucket_ctx;
+    ctx->oid = oid;
+    ctx->op = OP_DELETE_OBJ;
+
+    dispatcher->process_context(ctx);
+    ret = ctx->ret();
+    ctx->put();
+    return ret;
+  }
+
+  bool get_objects(std::list* objects, int num) {
+    if (bucket_list_done) {
+      bucket_list_done = false;
+      return false;
+    }
+
+    struct req_context *ctx = new req_context;
+    int ret = ctx->init_ctx();
+    if (ret < 0) {
+      return ret;
+    }
+    ctx->get();
+    ctx->bucket_ctx = &bucket_ctx;
+    ctx->list_start = list_start;
+    ctx->list_objects = objects;
+    ctx->list_count = num;
+    ctx->op = OP_LIST_BUCKET;
+
+    dispatcher->process_context(ctx);
+    ret = ctx->ret();
+
+    list_start = ctx->list_start;
+    if (list_start == NULL || strcmp(list_start, "") == 0) {
+      bucket_list_done = true;
+      list_start = NULL;
+    }
+
+    ctx->put();
+
+    return ret == 0;
+  }
+
+  bool completion_is_done(int slot) {
+    return completions[slot]->complete;
+  }
+
+  int completion_wait(int slot) {
+    req_context *ctx = completions[slot];
+
+    Mutex::Locker l(ctx->lock);
+
+    while (!ctx->complete) {
+      ctx->cond.Wait(ctx->lock);
+    }
+
+    return 0;
+  }
+
+  int completion_ret(int slot) {
+    S3Status status = completions[slot]->status;
+    if (status != S3StatusOK)
+      return -EIO;
+    return 0;
+  }
+
+public:
+  RESTBencher(RESTDispatcher *_dispatcher) :
+      ObjBencher(_dispatcher->cct),
+      dispatcher(_dispatcher),
+      completions(NULL),
+      list_start(NULL),
+      bucket_list_done(false)
+  {
+    dispatcher->start();
+  }
+  ~RESTBencher() { }
+
+  int init(string& _agent, string& _host, string& _bucket, S3Protocol _protocol,
+           S3UriStyle uri_style, string& _access_key, string& _secret) {
+    user_agent = _agent;
+    host = _host;
+    bucket = _bucket;
+    protocol = _protocol;
+    access_key = _access_key;
+    secret = _secret;
+
+    bucket_ctx.hostName = NULL; // host.c_str();
+    bucket_ctx.bucketName = bucket.c_str();
+    bucket_ctx.protocol =  protocol;
+    bucket_ctx.accessKeyId = access_key.c_str();
+    bucket_ctx.secretAccessKey = secret.c_str();
+    bucket_ctx.uriStyle = uri_style;
+    
+    struct req_context *ctx = new req_context;
+
+    int ret = rest_init();
+    if (ret < 0) {
+      return ret;
+    }
+
+    ret = ctx->init_ctx();
+    if (ret < 0) {
+      return ret;
+    }
+
+    ctx->get();
+
+    S3ResponseHandler response_handler;
+    response_handler.propertiesCallback = properties_callback;
+    response_handler.completeCallback = complete_callback;
+
+    S3_create_bucket(protocol, access_key.c_str(), secret.c_str(), NULL,
+                     bucket.c_str(), S3CannedAclPrivate,
+                     NULL, /* locationConstraint */
+                     NULL, /* requestContext */
+                     &response_handler, /* handler */
+                     (void *)ctx  /* callbackData */);
+
+    ret = ctx->ret();
+    if (ret < 0) {
+      cerr << "ERROR: failed to create bucket: " << S3_get_status_name(ctx->status) << std::endl;
+      return ret;
+    }
+
+    ctx->put();
+
+    return 0;
+  }
+};
+
+int main(int argc, const char **argv)
+{
+  vector args;
+  argv_to_vec(argc, argv, args);
+  env_to_vec(args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+
+  std::vector::iterator i;
+  std::string host;
+  std::string val;
+  std::string user_agent;
+  std::string access_key;
+  std::string secret;
+  std::string bucket = DEFAULT_BUCKET;
+  S3Protocol protocol = S3ProtocolHTTP;
+  S3UriStyle uri_style = S3UriStylePath;
+  std::string proto_str;
+  int concurrent_ios = 16;
+  int op_size = 1 << 22;
+  int seconds = 60;
+
+  bool show_time = false;
+  bool cleanup = true;
+  std::string run_name;
+  std::string prefix;
+
+
+  for (i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage(cout);
+      exit(0);
+    } else if (ceph_argparse_flag(args, i, "--show-time", (char*)NULL)) {
+      show_time = true;
+    } else if (ceph_argparse_flag(args, i, "--no-cleanup", (char*)NULL)) {
+      cleanup = false;
+    } else if (ceph_argparse_witharg(args, i, &user_agent, "--agent", (char*)NULL)) {
+      /* nothing */
+    } else if (ceph_argparse_witharg(args, i, &access_key, "--access-key", (char*)NULL)) {
+      /* nothing */
+    } else if (ceph_argparse_witharg(args, i, &secret, "--secret", (char*)NULL)) {
+      /* nothing */
+    } else if (ceph_argparse_witharg(args, i, &bucket, "--bucket", (char*)NULL)) {
+      /* nothing */
+    } else if (ceph_argparse_witharg(args, i, &host, "--api-host", (char*)NULL)) {
+      cerr << "host=" << host << std::endl;
+      /* nothing */
+    } else if (ceph_argparse_witharg(args, i, &proto_str, "--protocol", (char*)NULL)) {
+      if (strcasecmp(proto_str.c_str(), "http") == 0) {
+        protocol = S3ProtocolHTTP;
+      } else if (strcasecmp(proto_str.c_str(), "http") == 0) {
+        protocol = S3ProtocolHTTPS;
+      } else {
+        cerr << "bad protocol" << std::endl;
+        usage_exit();
+      }
+      /* nothing */
+    } else if (ceph_argparse_witharg(args, i, &proto_str, "--uri-style", (char*)NULL)) {
+      if (strcasecmp(proto_str.c_str(), "vhost") == 0) {
+        uri_style = S3UriStyleVirtualHost;
+      } else if (strcasecmp(proto_str.c_str(), "path") == 0) {
+        uri_style = S3UriStylePath;
+      } else {
+        cerr << "bad protocol" << std::endl;
+        usage_exit();
+      }
+    } else if (ceph_argparse_witharg(args, i, &val, "-t", "--concurrent-ios", (char*)NULL)) {
+      concurrent_ios = strtol(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val, "--run-name", (char*)NULL)) {
+      run_name = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--prefix", (char*)NULL)) {
+      prefix = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--seconds", (char*)NULL)) {
+      seconds = strtol(val.c_str(), NULL, 10);
+    } else if (ceph_argparse_witharg(args, i, &val, "-b", "--block-size", (char*)NULL)) {
+      op_size = strtol(val.c_str(), NULL, 10);
+    } else {
+      if (val[0] == '-')
+        usage_exit();
+      ++i;
+    }
+  }
+
+  if (bucket.empty()) {
+    cerr << "rest-bench: bucket not specified" << std::endl;
+    usage_exit();
+  }
+  if (args.empty())
+    usage_exit();
+  int operation = 0;
+  if (strcmp(args[0], "write") == 0)
+    operation = OP_WRITE;
+  else if (strcmp(args[0], "seq") == 0)
+    operation = OP_SEQ_READ;
+  else if (strcmp(args[0], "rand") == 0)
+    operation = OP_RAND_READ;
+  else if (strcmp(args[0], "cleanup") == 0) {
+    operation = OP_CLEANUP;
+  } else
+    usage_exit();
+
+  if (host.empty()) {
+    cerr << "rest-bench: api host not provided." << std::endl;
+    usage_exit();
+  }
+
+  if (access_key.empty() || secret.empty()) {
+    cerr << "rest-bench: access key or secret was not provided" << std::endl;
+    usage_exit();
+  }
+
+  if (bucket.empty()) {
+    bucket = DEFAULT_BUCKET;
+  }
+
+  if (user_agent.empty())
+    user_agent = DEFAULT_USER_AGENT;
+
+  RESTDispatcher dispatcher(g_ceph_context, concurrent_ios);
+
+  RESTBencher bencher(&dispatcher);
+  bencher.set_show_time(show_time);
+
+  int ret = bencher.init(user_agent, host, bucket, protocol, uri_style, access_key, secret);
+  if (ret < 0) {
+    cerr << "failed initializing benchmark" << std::endl;
+    exit(1);
+  }
+
+  if (operation == OP_CLEANUP) {
+    ret = bencher.clean_up(prefix.c_str(), concurrent_ios, run_name.c_str());
+    if (ret != 0)
+      cerr << "error during cleanup: " << ret << std::endl;
+  } else {
+    ret = bencher.aio_bench(operation, seconds, 0,
+			    concurrent_ios, op_size, cleanup, run_name.c_str());
+    if (ret != 0) {
+        cerr << "error during benchmark: " << ret << std::endl;
+    }
+  }
+
+  return 0;
+}
+
diff --git a/ceph/src/tools/scratchtool.c b/ceph/src/tools/scratchtool.c
new file mode 100644
index 00000000..22cf2bdf
--- /dev/null
+++ b/ceph/src/tools/scratchtool.c
@@ -0,0 +1,312 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil 
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/rados/librados.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+static int do_rados_setxattr(rados_ioctx_t io_ctx, const char *oid,
+			const char *key, const char *val)
+{
+	int ret = rados_setxattr(io_ctx, oid, key, val, strlen(val) + 1);
+	if (ret < 0) {
+		printf("rados_setxattr failed with error %d\n", ret);
+		return 1;
+	}
+	printf("rados_setxattr %s=%s\n", key, val);
+	return 0;
+}
+
+static int do_rados_getxattr(rados_ioctx_t io_ctx, const char *oid,
+			const char *key, const char *expected)
+{
+	size_t blen = strlen(expected) + 1;
+	char buf[blen];
+	memset(buf, 0, sizeof(buf));
+	int r = rados_getxattr(io_ctx, oid, key, buf, blen);
+	if (r < 0) {
+		printf("rados_getxattr(%s) failed with error %d\n", key, r);
+		return 1;
+	}
+	if (strcmp(buf, expected) != 0) {
+		printf("rados_getxattr(%s) got wrong result! "
+		       "expected: '%s'. got '%s'\n", key, expected, buf);
+		return 1;
+	}
+	printf("rados_getxattr %s=%s\n", key, buf);
+	return 0;
+}
+
+static int do_rados_getxattrs(rados_ioctx_t io_ctx, const char *oid,
+			const char **exkeys, const char **exvals)
+{
+	rados_xattrs_iter_t iter;
+	int nval = 0, i, nfound = 0, ret = 0;
+
+	for (i = 0; exvals[i]; ++i) {
+		++nval;
+	}
+	ret = rados_getxattrs(io_ctx, oid, &iter);
+	if (ret) {
+		printf("rados_getxattrs(%s) failed with error %d\n", oid, ret);
+		return 1;
+	}
+	while (1) {
+	        size_t len;
+	        const char *key, *val;
+		ret = rados_getxattrs_next(iter, &key, &val, &len);
+		if (ret) {
+			printf("rados_getxattrs(%s): rados_getxattrs_next "
+				"returned error %d\n", oid, ret);
+			return 1;
+		}
+		if (!key)
+			break;
+		for (i = 0; i < nval; ++i) {
+			if (strcmp(exkeys[i], key))
+				continue;
+			if ((len == strlen(exvals[i]) + 1) && (!strcmp(exvals[i], val))) {
+				nfound++;
+				break;
+			}
+			printf("rados_getxattrs(%s): got key %s, but the "
+				"value was %s rather than %s.\n",
+				oid, key, val, exvals[i]);
+			return 1;
+		}
+	}
+	if (nfound != nval) {
+		printf("rados_getxattrs(%s): only found %d extended attributes. "
+			"Expected %d\n", oid, nfound, nval);
+		return 1;
+	}
+	rados_getxattrs_end(iter);
+	printf("rados_getxattrs(%s)\n", oid);
+	return 0;
+}
+
+static int testrados(void)
+{
+	char tmp[32];
+	int i, r;
+	int ret = 1; //set 1 as error case
+	rados_t cl;
+
+	if (rados_create(&cl, NULL) < 0) {
+		printf("error initializing\n");
+		return 1;
+	}
+
+	if (rados_conf_read_file(cl, NULL)) {
+		printf("error reading configuration file\n");
+		goto out_err;
+	}
+
+	// Try to set a configuration option that doesn't exist.
+	// This should fail.
+	if (!rados_conf_set(cl, "config option that doesn't exist",
+			"some random value")) {
+		printf("error: succeeded in setting nonexistent config option\n");
+		goto out_err;
+	}
+
+	if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
+		printf("error: failed to read log_to_stderr from config\n");
+		goto out_err;
+	}
+
+	// Can we change it?
+	if (rados_conf_set(cl, "log to stderr", "true")) {
+		printf("error: error setting log_to_stderr\n");
+		goto out_err;
+	}
+	if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
+		printf("error: failed to read log_to_stderr from config\n");
+		goto out_err;
+	}
+	if (strcmp(tmp, "true")) {
+		printf("error: new setting for log_to_stderr failed to take effect.\n");
+		goto out_err;
+	}
+
+	if (rados_connect(cl)) {
+		printf("error connecting\n");
+		goto out_err;
+	}
+	if (rados_connect(cl) == 0) {
+		printf("second connect attempt didn't return an error\n");
+		goto out_err;
+	}
+
+	/* create an io_ctx */
+	r = rados_pool_create(cl, "foo");
+	printf("rados_pool_create = %d\n", r);
+
+	rados_ioctx_t io_ctx;
+	r = rados_ioctx_create(cl, "foo", &io_ctx);
+	printf("rados_ioctx_create = %d, io_ctx = %p\n", r, io_ctx);
+
+	/* list all pools */
+	{
+		int buf_sz = rados_pool_list(cl, NULL, 0);
+		printf("need buffer size of %d\n", buf_sz);
+		char buf[buf_sz];
+		int r = rados_pool_list(cl, buf, buf_sz);
+		if (r != buf_sz) {
+			printf("buffer size mismatch: got %d the first time, but %d "
+			"the second.\n", buf_sz, r);
+			goto out_err;
+		}
+		const char *b = buf;
+		printf("begin pools.\n");
+		while (1) {
+		if (b[0] == '\0')
+		break;
+		printf(" pool: '%s'\n", b);
+		b += strlen(b) + 1;
+		};
+		printf("end pools.\n");
+	}
+
+
+	/* stat */
+	struct rados_pool_stat_t st;
+	r = rados_ioctx_pool_stat(io_ctx, &st);
+	printf("rados_ioctx_pool_stat = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
+
+	/* snapshots */
+	r = rados_ioctx_snap_create(io_ctx, "snap1");
+	printf("rados_ioctx_snap_create snap1 = %d\n", r);
+	rados_snap_t snaps[10];
+	r = rados_ioctx_snap_list(io_ctx, snaps, 10);
+	for (i=0; i
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "include/types.h"
+#include "include/rados/librados.hpp"
+
+using namespace librados;
+
+#include 
+
+#include 
+#include 
+#include 
+
+void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  str[0] = '\0';
+  for (int i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+class C_Watch : public WatchCtx {
+public:
+  C_Watch() {}
+  void notify(uint8_t opcode, uint64_t ver, bufferlist& bl) {
+    cout << "C_Watch::notify() opcode=" << (int)opcode << " ver=" << ver << std::endl;
+  }
+};
+
+void testradospp_milestone(void)
+{
+  int c;
+  cout << "*** press enter to continue ***" << std::endl;
+  while ((c = getchar()) != EOF) {
+    if (c == '\n')
+      break;
+  }
+}
+
+int main(int argc, const char **argv) 
+{
+  Rados rados;
+  if (rados.init(NULL) < 0) {
+     cerr << "couldn't initialize rados!" << std::endl;
+     exit(1);
+  }
+
+  if (rados.conf_read_file(NULL)) {
+     cerr << "couldn't read configuration file." << std::endl;
+     exit(1);
+  }
+  rados.conf_parse_argv(argc, argv);
+
+  if (!rados.conf_set("config option that doesn't exist",
+                     "some random value")) {
+    printf("error: succeeded in setting nonexistent config option\n");
+    exit(1);
+  }
+  if (rados.conf_set("log to stderr", "true")) {
+    printf("error: error setting log_to_stderr\n");
+    exit(1);
+  }
+  std::string tmp;
+  if (rados.conf_get("log to stderr", tmp)) {
+    printf("error: failed to read log_to_stderr from config\n");
+    exit(1);
+  }
+  if (tmp != "true") {
+    printf("error: new setting for log_to_stderr failed to take effect.\n");
+    exit(1);
+  }
+
+  if (rados.connect()) {
+    printf("error connecting\n");
+    exit(1);
+  }
+
+  cout << "rados_initialize completed" << std::endl;
+  testradospp_milestone();
+
+  time_t tm;
+  bufferlist bl, bl2, blf;
+  char buf[128];
+
+  time(&tm);
+  snprintf(buf, 128, "%s", ctime(&tm));
+  bl.append(buf, strlen(buf));
+  blf.append(buf, 16);
+
+  const char *oid = "bar";
+
+  IoCtx io_ctx;
+  int r = rados.ioctx_create("data", io_ctx);
+  cout << "ioctx_create result = " << r << std::endl;
+
+  r = io_ctx.write(oid, bl, bl.length(), 0);
+  uint64_t objver = io_ctx.get_last_version();
+  assert(objver > 0);
+  cout << "io_ctx.write returned " << r << " last_ver=" << objver << std::endl;
+
+  uint64_t stat_size;
+  time_t stat_mtime;
+  r = io_ctx.stat(oid, &stat_size, &stat_mtime);
+  cout << "io_ctx.stat returned " << r << " size = " << stat_size << " mtime = " << stat_mtime << std::endl;
+
+  r = io_ctx.stat(oid, NULL, NULL);
+  cout << "io_ctx.stat(does_not_exist) = " << r << std::endl;
+
+  uint64_t handle;
+  C_Watch wc;
+  r = io_ctx.watch(oid, objver, &handle, &wc);
+  cout << "io_ctx.watch returned " << r << std::endl;
+
+  testradospp_milestone();
+  io_ctx.set_notify_timeout(7);
+  bufferlist notify_bl;
+  r = io_ctx.notify(oid, objver, notify_bl);
+  cout << "io_ctx.notify returned " << r << std::endl;
+  testradospp_milestone();
+
+  r = io_ctx.notify(oid, objver, notify_bl);
+  cout << "io_ctx.notify returned " << r << std::endl;
+  testradospp_milestone();
+
+  r = io_ctx.unwatch(oid, handle);
+  cout << "io_ctx.unwatch returned " << r << std::endl;
+  testradospp_milestone();
+
+  r = io_ctx.notify(oid, objver, notify_bl);
+  cout << "io_ctx.notify returned " << r << std::endl;
+  testradospp_milestone();
+  io_ctx.set_assert_version(objver);
+
+  r = io_ctx.write(oid, bl, bl.length() - 1, 0);
+  cout << "io_ctx.write returned " << r << std::endl;
+
+  r = io_ctx.write(oid, bl, bl.length() - 2, 0);
+  cout << "io_ctx.write returned " << r << std::endl;
+  r = io_ctx.write(oid, bl, bl.length() - 3, 0);
+  cout << "rados.write returned " << r << std::endl;
+  r = io_ctx.append(oid, bl, bl.length());
+  cout << "rados.write returned " << r << std::endl;
+  r = io_ctx.write_full(oid, blf);
+  cout << "rados.write_full returned " << r << std::endl;
+  r = io_ctx.read(oid, bl, bl.length(), 0);
+  cout << "rados.read returned " << r << std::endl;
+  r = io_ctx.trunc(oid, 8);
+  cout << "rados.trunc returned " << r << std::endl;
+  r = io_ctx.read(oid, bl, bl.length(), 0);
+  cout << "rados.read returned " << r << std::endl;
+  r = io_ctx.exec(oid, "crypto", "md5", bl, bl2);
+  cout << "exec returned " << r <<  " buf size=" << bl2.length() << std::endl;
+  const unsigned char *md5 = (const unsigned char *)bl2.c_str();
+  char md5_str[bl2.length()*2 + 1];
+  buf_to_hex(md5, bl2.length(), md5_str);
+  cout << "md5 result=" << md5_str << std::endl;
+
+  // test assert_version
+  r = io_ctx.read(oid, bl, 0, 1);
+  assert(r >= 0);
+  uint64_t v = io_ctx.get_last_version();
+  cout << oid << " version is " << v << std::endl;
+  assert(v > 0);
+  io_ctx.set_assert_version(v);
+  r = io_ctx.read(oid, bl, 0, 1);
+  assert(r >= 0);
+  io_ctx.set_assert_version(v - 1);
+  r = io_ctx.read(oid, bl, 0, 1);
+  assert(r == -ERANGE);
+  io_ctx.set_assert_version(v + 1);
+  r = io_ctx.read(oid, bl, 0, 1);
+  assert(r == -EOVERFLOW);
+
+  // test assert_src_version
+  r = io_ctx.read(oid, bl, 0, 1);
+  assert(r >= 0);
+  v = io_ctx.get_last_version();
+  cout << oid << " version is " << v << std::endl;
+  io_ctx.set_assert_src_version(oid, v);
+  
+  r = io_ctx.exec(oid, "crypto", "sha1", bl, bl2);
+  cout << "exec returned " << r << std::endl;
+  const unsigned char *sha1 = (const unsigned char *)bl2.c_str();
+  char sha1_str[bl2.length()*2 + 1];
+  buf_to_hex(sha1, bl2.length(), sha1_str);
+  cout << "sha1 result=" << sha1_str << std::endl;
+
+  r = io_ctx.exec(oid, "acl", "set", bl, bl2);
+  cout << "exec (set) returned " << r << std::endl;
+  r = io_ctx.exec(oid, "acl", "get", bl, bl2);
+  cout << "exec (get) returned " << r << std::endl;
+  if (bl2.length() > 0) {
+    cout << "attr=" << bl2.c_str() << std::endl;
+  }
+
+  int size = io_ctx.read(oid, bl2, 128, 0);
+  if (size <= 0) {
+    cout << "failed to read oid " << oid << "." << std::endl;
+    exit(1);
+  }
+  if (size > 4096) {
+    cout << "read too many bytes from oid " << oid << "." << std::endl;
+    exit(1);
+  }
+  char rbuf[size + 1];
+  memcpy(rbuf, bl2.c_str(), size);
+  rbuf[size] = '\0';
+  cout << "read result='" << rbuf << "'" << std::endl;
+  cout << "size=" << size << std::endl;
+
+  const char *oid2 = "jjj10.rbd";
+  r = io_ctx.exec(oid2, "rbd", "snap_list", bl, bl2);
+  cout << "snap_list result=" << r << std::endl;
+  r = io_ctx.exec(oid2, "rbd", "snap_add", bl, bl2);
+  cout << "snap_add result=" << r << std::endl;
+
+  if (r > 0) {
+    char *s = bl2.c_str();
+    for (int i=0; i= 0);
+  {
+    ObjectReadOperation o;
+    o.cmpxattr("foo", CEPH_OSD_CMPXATTR_OP_EQ, val);
+    r = io_ctx.operate(oid, &o, &bl2);
+    cout << " got " << r << " wanted >= 0" << std::endl;
+    assert(r >= 0);
+  }
+  val.append("...");
+  {
+    ObjectReadOperation o;
+    o.cmpxattr("foo", CEPH_OSD_CMPXATTR_OP_EQ, val);
+    r = io_ctx.operate(oid, &o, &bl2);
+    cout << " got " << r << " wanted " << -ECANCELED << " (-ECANCELED)" << std::endl;
+    assert(r == -ECANCELED);
+  }
+
+  cout << "src_cmpxattr" << std::endl;
+  const char *oidb = "bar-clone";
+  {
+    ObjectWriteOperation o;
+    o.src_cmpxattr(oid, "foo", CEPH_OSD_CMPXATTR_OP_EQ, val);
+    io_ctx.locator_set_key(oid);
+    o.write_full(val);
+    r = io_ctx.operate(oidb, &o);
+    cout << " got " << r << " wanted " << -ECANCELED << " (-ECANCELED)" << std::endl;
+    assert(r == -ECANCELED);
+  }
+  {
+    ObjectWriteOperation o;
+    o.src_cmpxattr(oid, "foo", CEPH_OSD_CMPXATTR_OP_NE, val);
+    io_ctx.locator_set_key(oid);
+    o.write_full(val);
+    r = io_ctx.operate(oidb, &o);
+    cout << " got " << r << " wanted >= 0" << std::endl;
+    assert(r >= 0);
+  }
+  io_ctx.locator_set_key(string());
+
+
+  cout << "iterating over objects..." << std::endl;
+  int num_objs = 0;
+  for (ObjectIterator iter = io_ctx.objects_begin();
+       iter != io_ctx.objects_end(); ++iter) {
+    num_objs++;
+    cout << "'" << *iter << "'" << std::endl;
+  }
+  cout << "iterated over " << num_objs << " objects." << std::endl;
+  map attrset;
+  io_ctx.getxattrs(oid, attrset);
+
+  map::iterator it;
+  for (it = attrset.begin(); it != attrset.end(); ++it) {
+    cout << "xattr: " << it->first << std::endl;
+  }
+
+  r = io_ctx.remove(oid);
+  cout << "remove result=" << r << std::endl;
+  rados.shutdown();
+
+  return 0;
+}
+
diff --git a/ceph/src/unittest_bufferlist.sh b/ceph/src/unittest_bufferlist.sh
new file mode 100755
index 00000000..8ddf24f8
--- /dev/null
+++ b/ceph/src/unittest_bufferlist.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# Ceph - scalable distributed file system
+#
+# Copyright (C) 2013 Cloudwatt 
+#
+# Author: Loic Dachary 
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+CEPH_BUFFER_TRACK=true ./unittest_bufferlist
diff --git a/ceph/src/upstart/ceph-all.conf b/ceph/src/upstart/ceph-all.conf
new file mode 100644
index 00000000..c0b0edd3
--- /dev/null
+++ b/ceph/src/upstart/ceph-all.conf
@@ -0,0 +1,4 @@
+description "Ceph"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
diff --git a/ceph/src/upstart/ceph-create-keys.conf b/ceph/src/upstart/ceph-create-keys.conf
new file mode 100644
index 00000000..7c79e692
--- /dev/null
+++ b/ceph/src/upstart/ceph-create-keys.conf
@@ -0,0 +1,8 @@
+description "Create Ceph client.admin key when possible"
+
+start on started ceph-mon
+stop on stopping ceph-mon
+
+task
+
+exec /usr/sbin/ceph-create-keys --cluster="${cluster:-ceph}" -i "${id:-$(hostname)}"
diff --git a/ceph/src/upstart/ceph-mds-all-starter.conf b/ceph/src/upstart/ceph-mds-all-starter.conf
new file mode 100644
index 00000000..5c1e02ef
--- /dev/null
+++ b/ceph/src/upstart/ceph-mds-all-starter.conf
@@ -0,0 +1,18 @@
+description "Ceph MDS (start all instances)"
+
+start on starting ceph-mds-all
+
+task
+
+script
+  set -e
+  # TODO what's the valid charset for cluster names and mds ids?
+  find -L /var/lib/ceph/mds/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+  | while read f; do
+    if [ -e "/var/lib/ceph/mds/$f/done" ] && [ -e "/var/lib/ceph/mds/$f/upstart" ] && [ ! -e "/var/lib/ceph/mds/$f/sysvinit" ]; then
+        cluster="${f%%-*}"
+        id="${f#*-}"
+        initctl emit ceph-mds cluster="$cluster" id="$id"
+    fi
+  done
+end script
diff --git a/ceph/src/upstart/ceph-mds-all.conf b/ceph/src/upstart/ceph-mds-all.conf
new file mode 100644
index 00000000..991019e1
--- /dev/null
+++ b/ceph/src/upstart/ceph-mds-all.conf
@@ -0,0 +1,4 @@
+description "Ceph MDS (all instances)"
+
+start on starting ceph-all
+stop on stopping ceph-all
diff --git a/ceph/src/upstart/ceph-mds.conf b/ceph/src/upstart/ceph-mds.conf
new file mode 100644
index 00000000..77841cdc
--- /dev/null
+++ b/ceph/src/upstart/ceph-mds.conf
@@ -0,0 +1,26 @@
+description "Ceph MDS"
+
+start on ceph-mds
+stop on runlevel [!2345] or stopping ceph-mds-all
+
+respawn
+respawn limit 5 30
+
+limit nofile 16384 16384
+
+pre-start script
+    set -e
+    test -x /usr/bin/ceph-mds || { stop; exit 0; }
+    test -d "/var/lib/ceph/mds/${cluster:-ceph}-$id" || { stop; exit 0; }
+
+    install -d -m0755 /var/run/ceph
+end script
+
+instance ${cluster:-ceph}/$id
+export cluster
+export id
+
+# this breaks oneiric
+#usage "cluster = name of cluster (defaults to 'ceph'); id = mds instance id"
+
+exec /usr/bin/ceph-mds --cluster="${cluster:-ceph}" -i "$id" -f
diff --git a/ceph/src/upstart/ceph-mon-all-starter.conf b/ceph/src/upstart/ceph-mon-all-starter.conf
new file mode 100644
index 00000000..9b9c8199
--- /dev/null
+++ b/ceph/src/upstart/ceph-mon-all-starter.conf
@@ -0,0 +1,19 @@
+description "Ceph MON (start all instances)"
+
+start on starting ceph-mon-all
+
+task
+
+script
+  set -e
+  # TODO what's the valid charset for cluster names and mon ids?
+  find -L /var/lib/ceph/mon/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+  | while read f; do
+    if [ -e "/var/lib/ceph/mon/$f/done" ] && [ -e "/var/lib/ceph/mon/$f/upstart" ] && [ ! -e "/var/lib/ceph/mon/$f/sysvinit" ]; then
+        cluster="${f%%-*}"
+        id="${f#*-}"
+
+	initctl emit ceph-mon cluster="$cluster" id="$id"
+    fi
+  done
+end script
diff --git a/ceph/src/upstart/ceph-mon-all.conf b/ceph/src/upstart/ceph-mon-all.conf
new file mode 100644
index 00000000..aea00c4d
--- /dev/null
+++ b/ceph/src/upstart/ceph-mon-all.conf
@@ -0,0 +1,4 @@
+description "Ceph monitor (all instances)"
+
+start on starting ceph-all
+stop on runlevel [!2345] or stopping ceph-all
diff --git a/ceph/src/upstart/ceph-mon.conf b/ceph/src/upstart/ceph-mon.conf
new file mode 100644
index 00000000..0279f15c
--- /dev/null
+++ b/ceph/src/upstart/ceph-mon.conf
@@ -0,0 +1,31 @@
+description "Ceph MON"
+
+start on ceph-mon
+stop on runlevel [!2345] or stopping ceph-mon-all
+
+respawn
+respawn limit 5 30
+
+limit nofile 16384 16384
+
+pre-start script
+    set -e
+    test -x /usr/bin/ceph-mon || { stop; exit 0; }
+    test -d "/var/lib/ceph/mon/${cluster:-ceph}-$id" || { stop; exit 0; }
+
+    install -d -m0755 /var/run/ceph
+end script
+
+instance ${cluster:-ceph}/$id
+export cluster
+export id
+
+# this breaks oneiric
+#usage "cluster = name of cluster (defaults to 'ceph'); id = monitor instance id"
+
+exec /usr/bin/ceph-mon --cluster="${cluster:-ceph}" -i "$id" -f
+
+post-stop script
+    # Cleanup socket in case of segfault
+    rm -f "/var/run/ceph/ceph-mon.$id.asok"
+end script
diff --git a/ceph/src/upstart/ceph-osd-all-starter.conf b/ceph/src/upstart/ceph-osd-all-starter.conf
new file mode 100644
index 00000000..94b99edb
--- /dev/null
+++ b/ceph/src/upstart/ceph-osd-all-starter.conf
@@ -0,0 +1,22 @@
+description "Ceph OSD (start all instances)"
+
+start on starting ceph-osd-all
+
+task
+
+script
+  set -e
+
+  # first activate any partitions
+  ceph-disk activate-all
+
+  # TODO what's the valid charset for cluster names and osd ids?
+  find -L /var/lib/ceph/osd/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+  | while read f; do
+    if [ -e "/var/lib/ceph/osd/$f/ready" ] && [ -e "/var/lib/ceph/osd/$f/upstart" ] && [ ! -e "/var/lib/ceph/osd/$f/sysvinit" ]; then
+        cluster="${f%%-*}"
+        id="${f#*-}"
+	initctl emit ceph-osd cluster="$cluster" id="$id"
+    fi
+  done
+end script
diff --git a/ceph/src/upstart/ceph-osd-all.conf b/ceph/src/upstart/ceph-osd-all.conf
new file mode 100644
index 00000000..73d3b37e
--- /dev/null
+++ b/ceph/src/upstart/ceph-osd-all.conf
@@ -0,0 +1,4 @@
+description "Ceph OSD (all instances)"
+
+start on starting ceph-all
+stop on runlevel [!2345] or stopping ceph-all
diff --git a/ceph/src/upstart/ceph-osd.conf b/ceph/src/upstart/ceph-osd.conf
new file mode 100644
index 00000000..7175c2d9
--- /dev/null
+++ b/ceph/src/upstart/ceph-osd.conf
@@ -0,0 +1,54 @@
+description "Ceph OSD"
+
+start on ceph-osd
+stop on runlevel [!2345] or stopping ceph-osd-all
+
+respawn
+respawn limit 5 30
+
+limit nofile 32768 32768
+
+pre-start script
+    set -e
+    test -x /usr/bin/ceph-osd || { stop; exit 0; }
+    test -d "/var/lib/ceph/osd/${cluster:-ceph}-$id" || { stop; exit 0; }
+
+    install -d -m0755 /var/run/ceph
+
+    update="$(ceph-conf --cluster=${cluster:-ceph} --name=osd.$id --lookup osd_crush_update_on_start || :)"
+    if [ "${update:-1}" = "1" -o "${update:-1}" = "true" ]; then
+        # update location in crush
+	hook="$(ceph-conf --cluster=${cluster:-ceph} --name=osd.$id --lookup osd_crush_location_hook || :)"
+	if [ -z "$hook" ]; then
+	    hook="/usr/bin/ceph-crush-location"
+	fi
+	location="$($hook --cluster ${cluster:-ceph} --id $id --type osd)"
+	weight="$(ceph-conf --cluster=${cluster:-ceph} --name=osd.$id --lookup osd_crush_initial_weight || :)"
+	defaultweight=`df -P -k /var/lib/ceph/osd/${cluster:-ceph}-$id/ | tail -1 | awk '{ d= $2/1073741824 ; r = sprintf("%.2f", d); print r }'`
+	ceph \
+            --cluster="${cluster:-ceph}" \
+            --name="osd.$id" \
+            --keyring="/var/lib/ceph/osd/${cluster:-ceph}-$id/keyring" \
+            osd crush create-or-move \
+	    -- \
+            "$id" \
+	    "${weight:-${defaultweight:-1}}" \
+	    $location
+    fi
+
+    journal="/var/lib/ceph/osd/${cluster:-ceph}-$id/journal"
+    if [ -L "$journal" -a ! -e "$journal" ]; then
+        udevadm settle --timeout=5 || :
+	if [ -L "$journal" -a ! -e "$journal" ]; then
+            echo "ceph-osd($UPSTART_INSTANCE): journal not present, not starting yet." 1>&2
+	    stop
+	    exit 0
+	fi
+    fi
+end script
+
+instance ${cluster:-ceph}/$id
+export cluster
+export id
+
+exec /usr/bin/ceph-osd --cluster="${cluster:-ceph}" -i "$id" -f
diff --git a/ceph/src/upstart/radosgw-all-starter.conf b/ceph/src/upstart/radosgw-all-starter.conf
new file mode 100644
index 00000000..8ab3c0d9
--- /dev/null
+++ b/ceph/src/upstart/radosgw-all-starter.conf
@@ -0,0 +1,18 @@
+description "Ceph radosgw (task to start all instances)"
+
+start on starting radosgw-all
+
+task
+
+script
+  set -e
+  # TODO what's the valid charset for cluster names and daemon ids?
+  find -L /var/lib/ceph/radosgw/ -mindepth 1 -maxdepth 1 -regextype posix-egrep -regex '.*/[A-Za-z0-9]+-[A-Za-z0-9._-]+' -printf '%P\n' \
+  | while read f; do
+    if [ -e "/var/lib/ceph/radosgw/$f/done" ]; then
+        cluster="${f%%-*}"
+        id="${f#*-}"
+        initctl emit radosgw cluster="$cluster" id="$id"
+    fi
+  done
+end script
diff --git a/ceph/src/upstart/radosgw-all.conf b/ceph/src/upstart/radosgw-all.conf
new file mode 100644
index 00000000..66107520
--- /dev/null
+++ b/ceph/src/upstart/radosgw-all.conf
@@ -0,0 +1,4 @@
+description "Ceph radosgw (all instances)"
+
+start on starting ceph-all
+stop on runlevel [!2345] or stopping ceph-all
diff --git a/ceph/src/upstart/radosgw.conf b/ceph/src/upstart/radosgw.conf
new file mode 100644
index 00000000..d1b5bc3b
--- /dev/null
+++ b/ceph/src/upstart/radosgw.conf
@@ -0,0 +1,26 @@
+description "Ceph radosgw"
+
+start on radosgw
+stop on runlevel [!2345] or stopping radosgw-all
+
+respawn
+respawn limit 5 30
+
+limit nofile 8096 65536
+
+pre-start script
+    set -e
+    test -x /usr/bin/radosgw || { stop; exit 0; }
+    test -d "/var/lib/ceph/radosgw/${cluster:-ceph}-$id" || { stop; exit 0; }
+
+    install -d -m0755 /var/run/ceph
+end script
+
+instance ${cluster:-ceph}/$id
+export cluster
+export id
+
+# this breaks oneiric
+#usage "cluster = name of cluster (defaults to 'ceph'); id = mds instance id"
+
+exec /usr/bin/radosgw --cluster="${cluster:-ceph}" --id "$id" -f
diff --git a/ceph/src/upstart/rbdmap.conf b/ceph/src/upstart/rbdmap.conf
new file mode 100644
index 00000000..eeefec3b
--- /dev/null
+++ b/ceph/src/upstart/rbdmap.conf
@@ -0,0 +1,48 @@
+# rbdmap - Ceph RBD Mapping
+#
+# This script does not manage mount and unmount fs which depends on rbd device.
+# You should use _netdev option in fstab to mount and umount in the correct order.
+
+description "ceph rbd mapping"
+
+start on (started networking
+    and remote-filesystems)
+stop on unmounted-remote-filesystems
+
+env RBDMAPFILE="/etc/ceph/rbdmap"
+
+pre-start script
+    if [ ! -f "$RBDMAPFILE" ]; then
+        exit 0
+    fi
+
+    while read DEV PARAMS; do
+        case "$DEV" in
+          ""|\#*)
+            continue
+            ;;
+          */*)
+            ;;
+          *)
+            DEV=rbd/$DEV
+            ;;
+        esac
+        for PARAM in $(echo $PARAMS | tr ',' '\n'); do
+            CMDPARAMS="$CMDPARAMS --$(echo $PARAM | tr '=' ' ')"
+        done
+        if [ ! -b /dev/rbd/$DEV ]; then
+            echo "rbd map $DEV"
+            rbd map $DEV $CMDPARAMS
+        fi
+    done < $RBDMAPFILE
+end script
+
+post-stop script
+    if ls /dev/rbd[0-9]* >/dev/null 2>&1; then
+        for DEV in /dev/rbd[0-9]*; do
+            echo "rbd unmap $DEV"
+            rbd unmap $DEV
+        done
+    fi
+end script
+
diff --git a/ceph/src/verify-mds-journal.sh b/ceph/src/verify-mds-journal.sh
new file mode 100755
index 00000000..22ebac06
--- /dev/null
+++ b/ceph/src/verify-mds-journal.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+while [ 1 ]
+do
+  ./ceph-mds -f --debug_mds 20 --debug_ms 1 --standby_replay_for 0 || exit 1
+  echo replay ok, sleeping
+  sleep 30
+done
\ No newline at end of file
diff --git a/ceph/src/vstart.sh b/ceph/src/vstart.sh
new file mode 100755
index 00000000..5349c496
--- /dev/null
+++ b/ceph/src/vstart.sh
@@ -0,0 +1,653 @@
+#!/bin/sh
+
+export PYTHONPATH=./pybind
+export LD_LIBRARY_PATH=.libs
+export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
+
+
+# abort on failure
+set -e
+
+[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON"
+[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD"
+[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS"
+[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW"
+
+[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3
+[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=1
+[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3
+[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=1
+
+[ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD/"
+[ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev"
+[ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out"
+[ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000
+
+extra_conf=""
+new=0
+standby=0
+debug=0
+start_all=1
+start_mon=0
+start_mds=0
+start_osd=0
+start_rgw=0
+ip=""
+nodaemon=0
+smallmds=0
+hitset=""
+overwrite_conf=1
+cephx=1 #turn cephx on by default
+cache=""
+memstore=0
+
+MON_ADDR=""
+
+conf="$CEPH_DIR/ceph.conf"
+
+keyring_fn="$CEPH_DIR/keyring"
+osdmap_fn="/tmp/ceph_osdmap.$$"
+monmap_fn="/tmp/ceph_monmap.$$"
+
+usage="usage: $0 [option]... [mon] [mds] [osd]\n"
+usage=$usage"options:\n"
+usage=$usage"\t-d, --debug\n"
+usage=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n"
+usage=$usage"\t-l, --localhost: use localhost instead of hostname\n"
+usage=$usage"\t-i : bind to specific ip\n"
+usage=$usage"\t-r start radosgw (needs ceph compiled with --radosgw and apache2 with mod_fastcgi)\n"
+usage=$usage"\t-n, --new\n"
+usage=$usage"\t--valgrind[_{osd,mds,mon}] 'toolname args...'\n"
+usage=$usage"\t--nodaemon: use ceph-run as wrapper for mon/osd/mds\n"
+usage=$usage"\t--smallmds: limit mds cache size\n"
+usage=$usage"\t-m ip:port\t\tspecify monitor address\n"
+usage=$usage"\t-k keep old configuration files\n"
+usage=$usage"\t-x enable cephx (on by default)\n"
+usage=$usage"\t-X disable cephx\n"
+usage=$usage"\t--hitset  : enable hitset tracking\n"
+usage=$usage"\t-o config\t\t add extra config parameters to mds section\n"
+
+usage_exit() {
+	printf "$usage"
+	exit
+}
+
+while [ $# -ge 1 ]; do
+case $1 in
+    -d | --debug )
+	    debug=1
+	    ;;
+    -s | --standby_mds)
+	    standby=1
+	    ;;
+    -l | --localhost )
+	    ip="127.0.0.1"
+	    ;;
+    -i )
+	    [ -z "$2" ] && usage_exit
+	    ip="$2"
+	    shift
+	    ;;
+    -r )
+	    start_rgw=1
+	    ;;
+    --new | -n )
+	    new=1
+	    ;;
+    --valgrind )
+	    [ -z "$2" ] && usage_exit
+	    valgrind=$2
+	    shift
+	    ;;
+    --valgrind_mds )
+	    [ -z "$2" ] && usage_exit
+	    valgrind_mds=$2
+	    shift
+	    ;;
+    --valgrind_osd )
+	    [ -z "$2" ] && usage_exit
+	    valgrind_osd=$2
+	    shift
+	    ;;
+    --valgrind_mon )
+	    [ -z "$2" ] && usage_exit
+	    valgrind_mon=$2
+	    shift
+	    ;;
+    --nodaemon )
+	    nodaemon=1
+	    ;;
+    --smallmds )
+	    smallmds=1
+	    ;;
+    mon )
+	    start_mon=1
+	    start_all=0
+	    ;;
+    mds )
+	    start_mds=1
+	    start_all=0
+	    ;;
+    osd )
+	    start_osd=1
+	    start_all=0
+	    ;;
+    -m )
+	    [ -z "$2" ] && usage_exit
+	    MON_ADDR=$2
+	    shift
+	    ;;
+    -x )
+	    cephx=1 # this is on be default, flag exists for historical consistency
+	    ;;
+    -X )
+	    cephx=0
+	    ;;
+    -k )
+	    overwrite_conf=0
+	    ;;
+    --memstore )
+	    memstore=1
+	    ;;
+    --hitset )
+	    hitset="$hitset $2 $3"
+	    shift
+	    shift
+	    ;;
+    -o )
+	    extra_conf="$extra_conf	$2
+"
+	    shift
+	    ;;
+    --cache )
+	    if [ -z "$cache" ]; then
+		cache="$2"
+	    else
+		cache="$cache $2"
+	    fi
+	    shift
+	    ;;
+    * )
+	    usage_exit
+esac
+shift
+done
+
+if [ "$start_all" -eq 1 ]; then
+	start_mon=1
+	start_mds=1
+	start_osd=1
+fi
+
+ARGS="-c $conf"
+
+run() {
+    type=$1
+    shift
+    eval "valg=\$valgrind_$type"
+    [ -z "$valg" ] && valg="$valgrind"
+
+    if [ -n "$valg" ]; then
+	echo "valgrind --tool=$valg $* -f &"
+	valgrind --tool=$valg $* -f &
+	sleep 1
+    else
+	if [ "$nodaemon" -eq 0 ]; then
+	    echo "$*"
+	    $*
+	else
+	    echo "ceph-run $* -f &"
+	    ./ceph-run $* -f &
+	fi
+    fi
+}
+
+if [ "$debug" -eq 0 ]; then
+    CMONDEBUG='
+	debug mon = 10
+        debug ms = 1'
+    COSDDEBUG='
+        debug ms = 1'
+    CMDSDEBUG='
+        debug ms = 1'
+else
+    echo "** going verbose **"
+    CMONDEBUG='
+	debug mon = 20
+        debug paxos = 20
+        debug auth = 20
+        debug ms = 1'
+    COSDDEBUG='
+        debug ms = 1
+        debug osd = 25
+        debug objecter = 20
+        debug monc = 20
+        debug journal = 20
+        debug filestore = 20
+        debug objclass = 20'
+    CMDSDEBUG='
+        debug ms = 1
+        debug mds = 20
+        debug auth = 20
+        debug monc = 20
+        mds debug scatterstat = true
+        mds verify scatter = true
+        mds log max segments = 2'
+fi
+
+if [ -n "$MON_ADDR" ]; then
+	CMON_ARGS=" -m "$MON_ADDR
+	COSD_ARGS=" -m "$MON_ADDR
+	CMDS_ARGS=" -m "$MON_ADDR
+fi
+
+if [ "$memstore" -eq 1 ]; then
+    COSDMEMSTORE='
+	osd objectstore = memstore'
+fi
+
+# lockdep everywhere?
+# export CEPH_ARGS="--lockdep 1"
+
+[ -z "$CEPH_BIN" ] && CEPH_BIN=.
+[ -z "$CEPH_PORT" ] && CEPH_PORT=6789
+
+
+# sudo if btrfs
+test -d $CEPH_DEV_DIR/osd0/. && test -e $CEPH_DEV_DIR/sudo && SUDO="sudo"
+
+if [ "$start_all" -eq 1 ]; then
+    $SUDO $CEPH_BIN/init-ceph stop
+fi
+$SUDO rm -f core*
+
+test -d $CEPH_OUT_DIR || mkdir $CEPH_OUT_DIR
+test -d $CEPH_DEV_DIR || mkdir $CEPH_DEV_DIR
+$SUDO rm -rf $CEPH_OUT_DIR/*
+test -d gmon && $SUDO rm -rf gmon/*
+
+[ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && test -e $keyring_fn && rm $keyring_fn
+
+
+# figure machine's ip
+HOSTNAME=`hostname`
+if [ -n "$ip" ]; then
+    IP="$ip"
+else
+    echo hostname $HOSTNAME
+    RAW_IP=`hostname -I`
+    # filter out IPv6 and localhost addresses
+    IP="$(echo "$RAW_IP"|tr ' ' '\012'|grep -v :|grep -v '^127\.'|head -n1)"
+    # if that left nothing, then try to use the raw thing, it might work
+    if [ -z "$IP" ]; then IP="$RAW_IP"; fi
+    echo ip $IP
+fi
+echo "ip $IP"
+
+
+
+if [ "$cephx" -eq 1 ]; then
+    CEPH_ADM="$CEPH_BIN/ceph -c $conf -k $keyring_fn"
+else
+    CEPH_ADM="$CEPH_BIN/ceph -c $conf"
+fi
+
+MONS=""
+count=0
+for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
+do
+    if [ -z "$MONS" ];
+    then
+	MONS="$f"
+    else
+	MONS="$MONS $f"
+    fi
+    count=$(($count + 1))
+    [ $count -eq $CEPH_NUM_MON ] && break;
+done
+
+DAEMONOPTS="
+	log file = $CEPH_OUT_DIR/\$name.log
+        admin socket = $CEPH_OUT_DIR/\$name.asok
+	chdir = \"\"
+	pid file = $CEPH_OUT_DIR/\$name.pid
+        heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
+"
+
+
+if [ "$start_mon" -eq 1 ]; then
+
+	if [ "$new" -eq 1 ]; then
+		if [ $overwrite_conf -eq 1 ]; then
+		        cat < $conf
+; generated by vstart.sh on `date`
+[global]
+        fsid = $(uuidgen)
+        osd pg bits = 3
+        osd pgp bits = 5  ; (invalid, but ceph should cope!)
+        osd crush chooseleaf type = 0
+        osd pool default min size = 1
+        osd pool default erasure code directory = .libs
+        osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd
+        run dir = $CEPH_OUT_DIR
+EOF
+if [ "$cephx" -eq 1 ] ; then
+cat <> $conf
+        auth supported = cephx
+EOF
+else
+cat <> $conf
+	auth cluster required = none
+	auth service required = none
+	auth client required = none
+EOF
+fi
+			cat <> $conf
+
+[client]
+        keyring = $keyring_fn
+        log file = $CEPH_OUT_DIR/\$name.\$pid.log
+
+[mds]
+$DAEMONOPTS
+$CMDSDEBUG
+        mds debug frag = true
+        mds debug auth pins = true
+        mds debug subtrees = true
+        mds data = $CEPH_DEV_DIR/mds.\$id
+$extra_conf
+[osd]
+$DAEMONOPTS
+        osd data = $CEPH_DEV_DIR/osd\$id
+        osd journal = $CEPH_DEV_DIR/osd\$id.journal
+        osd journal size = 100
+        osd class tmp = out
+        osd class dir = .libs
+        osd scrub load threshold = 5.0
+        osd debug op order = true
+$COSDDEBUG
+$COSDMEMSTORE
+$extra_conf
+[mon]
+        mon pg warn min per osd = 10
+        mon osd allow primary affinity = true
+$DAEMONOPTS
+$CMONDEBUG
+$extra_conf
+        mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log
+[global]
+$extra_conf
+EOF
+		fi
+
+		if [ `echo $IP | grep '^127\\.'` ]
+		then
+			echo
+			echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
+			echo "  connect.  either adjust /etc/hosts, or edit this script to use your"
+			echo "  machine's real IP."
+			echo
+		fi
+
+	        $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mon. $keyring_fn --cap mon 'allow *'
+	        $SUDO $CEPH_BIN/ceph-authtool --gen-key --name=client.admin --set-uid=0 \
+		    --cap mon 'allow *' \
+		    --cap osd 'allow *' \
+		    --cap mds allow \
+		    $keyring_fn
+
+		# build a fresh fs monmap, mon fs
+		str="$CEPH_BIN/monmaptool --create --clobber"
+		count=0
+		for f in $MONS
+		do
+			str=$str" --add $f $IP:$(($CEPH_PORT+$count))"
+			if [ $overwrite_conf -eq 1 ]; then
+				cat <> $conf
+[mon.$f]
+        host = $HOSTNAME
+        mon data = $CEPH_DEV_DIR/mon.$f
+        mon addr = $IP:$(($CEPH_PORT+$count))
+EOF
+			fi
+			count=$(($count + 1))
+		done
+		str=$str" --print $monmap_fn"
+		echo $str
+		$str
+
+		for f in $MONS
+		do
+		    cmd="rm -rf $CEPH_DEV_DIR/mon.$f"
+		    echo $cmd
+		    $cmd
+                    cmd="mkdir -p $CEPH_DEV_DIR/mon.$f"
+                    echo $cmd
+                    $cmd
+		    cmd="$CEPH_BIN/ceph-mon --mkfs -c $conf -i $f --monmap=$monmap_fn"
+		    cmd="$cmd --keyring=$keyring_fn"
+		    echo $cmd
+		    $cmd
+		done
+
+		rm $monmap_fn
+	fi
+
+	# start monitors
+	if [ "$start_mon" -ne 0 ]; then
+		for f in $MONS
+		do
+		    run 'mon' $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
+		done
+	fi
+fi
+
+#osd
+if [ "$start_osd" -eq 1 ]; then
+    for osd in `seq 0 $((CEPH_NUM_OSD-1))`
+    do
+	if [ "$new" -eq 1 ]; then
+	    if [ $overwrite_conf -eq 1 ]; then
+		    cat <> $conf
+[osd.$osd]
+        host = $HOSTNAME
+EOF
+		    rm -rf $CEPH_DEV_DIR/osd$osd || true
+		    for f in $CEPH_DEV_DIR/osd$osd/* ; do btrfs sub delete $f || true ; done || true
+		    mkdir -p $CEPH_DEV_DIR/osd$osd
+	    fi
+
+	    uuid=`uuidgen`
+	    echo "add osd$osd $uuid"
+	    $SUDO $CEPH_ADM osd create $uuid
+	    $SUDO $CEPH_ADM osd crush add osd.$osd 1.0 host=$HOSTNAME root=default
+	    $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid
+
+	    key_fn=$CEPH_DEV_DIR/osd$osd/keyring
+	    echo adding osd$osd key to auth repository
+	    $SUDO $CEPH_ADM -i $key_fn auth add osd.$osd osd "allow *" mon "allow profile osd"
+	fi
+	echo start osd$osd
+	run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS
+    done
+fi
+
+# mds
+if [ "$smallmds" -eq 1 ]; then
+    cat <> $conf
+[mds]
+	mds log max segments = 2
+	mds cache size = 10000
+EOF
+fi
+
+if [ "$start_mds" -eq 1 -a "$CEPH_NUM_MDS" -gt 0 ]; then
+    mds=0
+    for name in a b c d e f g h i j k l m n o p
+    do
+	if [ "$new" -eq 1 ]; then
+	    mkdir -p $CEPH_DEV_DIR/mds.$name
+	    key_fn=$CEPH_DEV_DIR/mds.$name/keyring
+	    if [ $overwrite_conf -eq 1 ]; then
+	    	cat <> $conf
+[mds.$name]
+        host = $HOSTNAME
+EOF
+		if [ "$standby" -eq 1 ]; then
+		    mkdir -p $CEPH_DEV_DIR/mds.${name}s
+		    cat <> $conf
+       mds standby for rank = $mds
+[mds.${name}s]
+        mds standby replay = true
+        mds standby for name = ${name}
+EOF
+		fi
+	    fi
+	    $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mds.$name $key_fn
+	    $SUDO $CEPH_ADM -i $key_fn auth add mds.$name mon 'allow profile mds' osd 'allow *' mds 'allow'
+	    if [ "$standby" -eq 1 ]; then
+		    $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mds.${name}s \
+			$CEPH_DEV_DIR/mds.${name}s/keyring
+                    $SUDO $CEPH_ADM -i $CEPH_DEV_DIR/mds.${name}s/keyring auth add mds.${name}s \
+			mon 'allow *' osd 'allow *' mds 'allow'
+	    fi
+	fi
+	
+	run 'mds' $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
+	if [ "$standby" -eq 1 ]; then
+	    run 'mds' $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
+	fi
+	
+	mds=$(($mds + 1))
+	[ $mds -eq $CEPH_NUM_MDS ] && break
+
+#valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
+#$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
+#$CEPH_ADM mds set max_mds 2
+    done
+    cmd="$CEPH_ADM mds set max_mds $CEPH_NUM_MDS"
+    echo $cmd
+    $cmd
+fi
+
+
+# rgw
+if [ "$start_rgw" -eq 1 ]; then
+    for rgw in `seq 0 $((CEPH_NUM_RGW-1))`
+    do
+	rgwport=$(( $CEPH_RGW_PORT + $rgw ))
+	if [ "$new" -eq 1 ]; then
+	    if [ $overwrite_conf -eq 1 ]; then
+		    dnsname=`hostname -f`
+		    cat <> $conf
+[client.radosgw.rgw$rgw]
+        host = $HOSTNAME
+$DAEMONOPTS
+        keyring = $CEPH_OUT_DIR/keyring.client.radosgw.rgw$rgw
+        rgw socket path = $CEPH_OUT_DIR/sock.client.radosgw.rgw$rgw
+        rgw dns name = $dnsname
+EOF
+		    mkdir -p $CEPH_OUT_DIR/htdocs
+		    mkdir -p $CEPH_OUT_DIR/fastcgi_sock
+		    APACHE2_MODULE_PATH="/usr/lib/apache2/modules"
+		    APACHE2_EXTRA_MODULES_NAME="mpm_prefork authz_core"
+		    for module in $APACHE2_EXTRA_MODULES_NAME
+		    do
+			    if [ -f "${APACHE2_MODULE_PATH}/mod_${module}.so" ]; then
+				    APACHE2_EXTRA_MODULES="${APACHE2_EXTRA_MODULES}LoadModule ${module}_module ${APACHE2_MODULE_PATH}/mod_${module}.so
+"
+			    fi 
+		    done
+		    echo $APACHE2_EXTRA_MODULES
+		    cat < $CEPH_OUT_DIR/apache.conf
+LoadModule env_module /usr/lib/apache2/modules/mod_env.so
+LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
+LoadModule fastcgi_module /usr/lib/apache2/modules/mod_fastcgi.so
+$APACHE2_EXTRA_MODULES
+
+Listen $rgwport
+ServerName rgwtest.example.com
+
+ServerRoot $CEPH_OUT_DIR
+ErrorLog $CEPH_OUT_DIR/apache.error.log
+LogFormat "%h l %u %t \"%r\" %>s %b \"{Referer}i\" \"%{User-agent}i\"" combined
+CustomLog $CEPH_OUT_DIR/apache.access.log combined
+PidFile $CEPH_OUT_DIR/apache.pid
+DocumentRoot $CEPH_OUT_DIR/htdocs
+FastCgiIPCDir $CEPH_OUT_DIR/fastcgi_sock
+FastCgiExternalServer $CEPH_OUT_DIR/htdocs/rgw.fcgi -socket $CEPH_OUT_DIR/sock.client.radosgw.rgw$rgw
+RewriteEngine On
+
+RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /rgw.fcgi?page=$1¶ms=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+# Set fastcgi environment variables.
+# Note that this is separate from Unix environment variables!
+SetEnv RGW_LOG_LEVEL 20
+SetEnv RGW_PRINT_CONTINUE yes
+SetEnv RGW_SHOULD_LOG yes
+
+
+  Options +ExecCGI
+  AllowOverride All
+  SetHandler fastcgi-script
+
+
+AllowEncodedSlashes On
+ServerSignature Off
+EOF
+		    $SUDO $CEPH_ADM auth get-or-create client.radosgw.rgw$rgw osd 'allow rwx' mon 'allow r' -o $CEPH_OUT_DIR/keyring.client.radosgw.rgw$rgw
+
+		    #akey=`echo $$ | md5sum | cut -c 1-20`
+		    #skey=`dd if=/dev/urandom of=/tmp/random.$$ bs=1 count=40 2>/dev/null ; base64 < /tmp/random.$$ ; rm /tmp/random.$$`
+		    akey='0555b35654ad1656d804'
+		    skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
+		    echo access key $akey
+		    echo secret key $skey
+		    $CEPH_BIN/radosgw-admin user create --uid tester --access-key $akey --secret $skey --display-name 'M. Tester' --email tester@ceph.com -c $conf
+	    fi
+	fi
+	echo start rgw$rgw on http://localhost:$rgwport
+	run 'rgw' $SUDO $CEPH_BIN/radosgw -n client.radosgw.rgw$rgw $ARGS
+	run 'apache2' $SUDO apache2 -f $CEPH_OUT_DIR/apache.conf
+    done
+fi
+
+echo "started.  stop.sh to stop.  see out/* (e.g. 'tail -f out/????') for debug output."
+
+do_cache() {
+    while [ -n "$*" ]; do
+	p="$1"
+	shift
+	echo "creating cache for pool $p ..."
+	$SUDO $CEPH_ADM <
+# 	old-postinst abort-upgrade 
+# 	conflictor's-postinst abort-remove in-favour  
+# 	postinst abort-remove
+# 	deconfigured's-postinst abort-deconfigure in-favour   [ ]
+#
+
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+	configure)
+		if [ -x /sbin/start ]; then
+			invoke-rc.d ceph-mds-all start || {
+				RESULT=$?
+				if [ $RESULT != 100 ]; then
+					exit $RESULT
+				fi
+			}
+		fi
+	;;
+	abort-upgrade|abort-remove|abort-deconfigure)
+		:
+	;;
+	*)
+		echo "postinst called with unknown argument \`$1'" >&2
+		exit 1
+	;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/ceph-mds.prerm b/debian/ceph-mds.prerm
new file mode 100644
index 00000000..e952a3f3
--- /dev/null
+++ b/debian/ceph-mds.prerm
@@ -0,0 +1,29 @@
+#!/bin/sh
+# vim: set noet ts=8:
+
+set -e
+
+case "$1" in
+	remove)
+		if [ -x /sbin/stop ]; then
+			invoke-rc.d ceph-mds-all stop || {
+				RESULT=$?
+				if [ $RESULT != 100 ]; then
+					exit $RESULT
+				fi
+			}
+		fi
+		invoke-rc.d ceph stop mds || {
+			RESULT=$?
+			if [ $RESULT != 100 ]; then
+				exit $RESULT
+			fi
+		}
+	;;
+	*)
+	;;
+esac
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/ceph-resource-agents.install b/debian/ceph-resource-agents.install
new file mode 100644
index 00000000..47e75161
--- /dev/null
+++ b/debian/ceph-resource-agents.install
@@ -0,0 +1 @@
+usr/lib/ocf/resource.d/ceph
diff --git a/debian/ceph-test.install b/debian/ceph-test.install
new file mode 100644
index 00000000..1d37f45f
--- /dev/null
+++ b/debian/ceph-test.install
@@ -0,0 +1,28 @@
+usr/bin/ceph-coverage /usr/lib/ceph/bin
+usr/bin/ceph-kvstore-tool /usr/lib/ceph/bin
+usr/bin/ceph-monstore-tool /usr/lib/ceph/bin
+usr/bin/ceph-osdomap-tool /usr/lib/ceph/bin
+usr/bin/ceph_bench_log /usr/lib/ceph/bin
+usr/bin/ceph_dupstore /usr/lib/ceph/bin
+usr/bin/ceph_erasure_code /usr/lib/ceph/bin
+usr/bin/ceph_erasure_code_benchmark /usr/lib/ceph/bin
+usr/bin/ceph_filestore_dump /usr/lib/ceph/bin
+usr/bin/ceph_filestore_tool /usr/lib/ceph/bin
+usr/bin/ceph_kvstorebench /usr/lib/ceph/bin
+usr/bin/ceph_multi_stress_watch /usr/lib/ceph/bin
+usr/bin/ceph_omapbench /usr/lib/ceph/bin
+usr/bin/ceph_psim /usr/lib/ceph/bin
+usr/bin/ceph_radosacl /usr/lib/ceph/bin
+usr/bin/ceph_rgw_jsonparser /usr/lib/ceph/bin
+usr/bin/ceph_rgw_multiparser /usr/lib/ceph/bin
+usr/bin/ceph_scratchtool /usr/lib/ceph/bin
+usr/bin/ceph_scratchtoolpp /usr/lib/ceph/bin
+usr/bin/ceph_smalliobench /usr/lib/ceph/bin
+usr/bin/ceph_smalliobenchdumb /usr/lib/ceph/bin
+usr/bin/ceph_smalliobenchfs /usr/lib/ceph/bin
+usr/bin/ceph_smalliobenchrbd /usr/lib/ceph/bin
+usr/bin/ceph_streamtest /usr/lib/ceph/bin
+usr/bin/ceph_test_* /usr/lib/ceph/bin
+usr/bin/ceph_tpbench /usr/lib/ceph/bin
+usr/bin/ceph_xattr_bench /usr/lib/ceph/bin
+usr/share/java/libcephfs-test.jar
diff --git a/debian/ceph.dirs b/debian/ceph.dirs
new file mode 100644
index 00000000..ca7a8806
--- /dev/null
+++ b/debian/ceph.dirs
@@ -0,0 +1,8 @@
+etc/ceph
+var/log/ceph
+var/lib/ceph/tmp
+var/lib/ceph/mon
+var/lib/ceph/osd
+var/lib/ceph/mds
+var/lib/ceph/bootstrap-osd
+var/lib/ceph/bootstrap-mds
diff --git a/debian/ceph.install b/debian/ceph.install
new file mode 100644
index 00000000..bc443130
--- /dev/null
+++ b/debian/ceph.install
@@ -0,0 +1,32 @@
+../../src/rbdmap etc/ceph
+../../udev/60-ceph-partuuid-workaround.rules lib/udev/rules.d
+../../udev/95-ceph-osd.rules lib/udev/rules.d
+etc/bash_completion.d/ceph
+sbin/mkcephfs
+usr/bin/ceph-clsinfo
+usr/bin/ceph-debugpack
+usr/bin/ceph-mon
+usr/bin/ceph-osd
+usr/bin/ceph-run
+usr/bin/ceph_mon_store_converter
+usr/bin/crushtool
+usr/bin/monmaptool
+usr/bin/osdmaptool
+usr/lib/*/ceph/ceph_common.sh
+usr/lib/*/ceph/erasure-code/libec_jerasure*.so
+usr/lib/*/rados-classes/*.so
+usr/sbin/ceph-create-keys
+usr/sbin/ceph-disk
+usr/sbin/ceph-disk-activate
+usr/sbin/ceph-disk-prepare
+usr/share/doc/ceph/sample.ceph.conf
+usr/share/doc/ceph/sample.fetch_config
+usr/share/man/man8/ceph-clsinfo.8
+usr/share/man/man8/ceph-debugpack.8
+usr/share/man/man8/ceph-mon.8
+usr/share/man/man8/ceph-osd.8
+usr/share/man/man8/ceph-run.8
+usr/share/man/man8/crushtool.8
+usr/share/man/man8/mkcephfs.8
+usr/share/man/man8/monmaptool.8
+usr/share/man/man8/osdmaptool.8
diff --git a/debian/ceph.lintian-overrides b/debian/ceph.lintian-overrides
new file mode 100644
index 00000000..2d71ebc9
--- /dev/null
+++ b/debian/ceph.lintian-overrides
@@ -0,0 +1,26 @@
+# Ceph upstart configurations don't have init.d equivalents
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-create-keys
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-create-keys
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-mon
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-mon
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-osd-all
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-osd-all
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-mon-all
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-mon-all
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-osd
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-osd
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-all
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-all
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-mon-all-starter
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-mon-all-starter
+ceph: init.d-script-not-marked-as-conffile etc/init.d/ceph-osd-all-starter
+ceph: init.d-script-not-included-in-package etc/init.d/ceph-osd-all-starter
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-mon-all-starter
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-osd
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-osd-all
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-mon
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-create-keys
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-mon-all
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/rbdmap
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-all
+ceph: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/ceph-osd-all-starter
diff --git a/debian/ceph.postinst b/debian/ceph.postinst
new file mode 100644
index 00000000..8292e642
--- /dev/null
+++ b/debian/ceph.postinst
@@ -0,0 +1,54 @@
+#!/bin/sh
+# vim: set noet ts=8:
+# postinst script for ceph
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#
+# 	postinst configure 
+# 	old-postinst abort-upgrade 
+# 	conflictor's-postinst abort-remove in-favour  
+# 	postinst abort-remove
+# 	deconfigured's-postinst abort-deconfigure in-favour   [ ]
+#
+# The current action is to simply remove the mistakenly-added
+# /etc/init/ceph.conf file; this could be done in any of these cases,
+# although technically it will leave the system in a different state
+# than the original install that included that file.  So instead we
+# only remove on "configure", since that's the only time we know we're
+# successful in installing a newer package than the erroneous version.
+
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+	configure)
+		rm -f /etc/init/ceph.conf
+		if [ -x /sbin/start ]; then
+			invoke-rc.d ceph-all start || {
+				RESULT=$?
+				if [ $RESULT != 100 ]; then
+					exit $RESULT
+				fi
+			}
+		fi
+	;;
+	abort-upgrade|abort-remove|abort-deconfigure)
+		:
+	;;
+	*)
+		echo "postinst called with unknown argument \`$1'" >&2
+		exit 1
+	;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/ceph.postrm b/debian/ceph.postrm
new file mode 100644
index 00000000..48e4853c
--- /dev/null
+++ b/debian/ceph.postrm
@@ -0,0 +1,47 @@
+#!/bin/sh
+# postrm script for ceph
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#        *  `remove'
+#        *  `purge'
+#        *  `upgrade' 
+#        *  `failed-upgrade' 
+#        *  `abort-install'
+#        *  `abort-install' 
+#        *  `abort-upgrade' 
+#        *  `disappear' 
+#          
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+    remove)
+    ;;
+
+    purge)
+	rm -rf /var/log/ceph 
+	rm -rf /etc/ceph
+    ;;
+
+    upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
+    ;;
+
+    *)
+        echo "postrm called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
+
+
diff --git a/debian/ceph.prerm b/debian/ceph.prerm
new file mode 100644
index 00000000..be7f715c
--- /dev/null
+++ b/debian/ceph.prerm
@@ -0,0 +1,29 @@
+#!/bin/sh
+# vim: set noet ts=8:
+
+set -e
+
+case "$1" in
+	remove)
+		if [ -x /sbin/stop ]; then
+			invoke-rc.d ceph-all stop || {
+				RESULT=$?
+				if [ $RESULT != 100 ]; then
+					exit $RESULT
+				fi
+			}
+		fi
+		invoke-rc.d ceph stop || {
+			RESULT=$?
+			if [ $RESULT != 100 ]; then
+				exit $RESULT
+			fi
+		}
+	;;
+	*)
+	;;
+esac
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 00000000..7943e3a7
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,313 @@
+ceph (0.80.7-0ubuntu0.14.04.1~mos6.1+1) trusty; urgency=low
+
+  * Sync with Ubuntu 14.04 sources.
+  * Apply upstream bugfixes from
+    - http://tracker.ceph.com/issues/9877
+    - http://tracker.ceph.com/issues/9226
+  * Bump version according to MOS versioning policy.
+
+ -- Alexei Sheplyakov   Sun, 01 Feb 2015 16:36:35 +0300
+
+ceph (0.80.7-0ubuntu0.14.04.1) trusty; urgency=medium
+
+  * New upstream stable point release (LP: #1381410).
+
+ -- James Page   Wed, 12 Nov 2014 11:31:46 +0000
+
+ceph (0.80.5-0ubuntu0.14.04.1) trusty; urgency=medium
+
+  * New upstream stable point release (LP: #1346166).
+
+ -- James Page   Mon, 04 Aug 2014 08:09:39 +0100
+
+ceph (0.80.4-0ubuntu0.14.04.1) trusty; urgency=medium
+
+  * New upstream stable point release (LP: #1346166):
+    - d/p/bug-8428.patch: Dropped, included upstream.
+
+ -- James Page   Mon, 21 Jul 2014 12:04:25 +0100
+
+ceph (0.80.1-0ubuntu1.1) trusty; urgency=medium
+
+  * Fix issue setting Swift ACL's on existing containers (LP: #1322498):
+    - d/p/bug-8428.patch: Cherry pick proposed fix from upstream VCS.
+
+ -- James Page   Mon, 02 Jun 2014 09:09:37 +0100
+
+ceph (0.80.1-0ubuntu1) trusty; urgency=medium
+
+  * New upstream release stable point release (LP: #1278466).
+
+ -- James Page   Wed, 14 May 2014 11:09:37 -0400
+
+ceph (0.80-0ubuntu1) trusty; urgency=medium
+
+  * New upstream release stable release (LP: #1278466).
+
+ -- James Page   Sun, 11 May 2014 09:54:31 -0400
+
+ceph (0.79-0ubuntu1) trusty; urgency=medium
+
+  * New upstream release (LP: #1278466):
+    - d/p/modules.patch: Refreshed.
+    - d/ceph.install: Install all jerasure modules.
+
+ -- James Page   Wed, 09 Apr 2014 11:14:03 +0100
+
+ceph (0.78-0ubuntu1) trusty; urgency=medium
+
+  * New upstream release:
+    - d/control: Add xfslib-dev to BD's.
+    - d/*: Sync relevant packaging changes from upstream.
+    - d/p/*: Drop upstreamed patches.
+    - d/p/modules.patch: Mark libcls_user.so and libec_jerasure.so as modules.
+    - d/ceph.install: Only install libec_jerasure.so.
+  * d/ceph-test.install: Install test binaries to /usr/lib/ceph/bin; they
+    really don't need to be installed on the default path.
+  * d/{ceph|radosgw|ceph-mds}.lintian-overrides: Add overrides for intentional
+    difference in naming and structure between upstart configurations and
+    init.d scripts.
+
+ -- James Page   Sat, 22 Mar 2014 18:27:40 +0000
+
+ceph (0.72.2-2) unstable; urgency=medium
+
+  * d/radosgw.{postinst,postrm,preinst}: Handle renaming of radosgw
+    upstart configuration on upgrade@0.72.1-3.
+  * d/{ceph|ceph-mds|radosgw}.{postinst|prerm}: Check to ensure that system
+    is running upstart before trying to start/stop upstart configurations
+    (Closes: #734241, #738845, #738845).
+
+ -- James Page   Sat, 08 Mar 2014 16:48:28 +0000
+
+ceph (0.72.2-1) unstable; urgency=medium
+
+  * New upstream release.
+
+ -- James Page   Wed, 01 Jan 2014 09:32:03 +0000
+
+ceph (0.72.1-3) unstable; urgency=low
+
+  * d/rules,ceph.install: Correct install paths for ceph-* helpers. 
+  * d/p/modules: Mark libcls_kvs.so as module.
+  * d/rules: Rename radosgw upstart configuration to radosgw-instance to
+    avoid namespace conflict with init script which breaks backwards
+    compatibility (LP: #1255464).
+
+ -- James Page   Wed, 27 Nov 2013 10:52:48 +0000
+
+ceph (0.72.1-2) unstable; urgency=low
+
+  * Fix upgrade failures from ceph < 0.67.3-1 (Closes: #728164):
+    - d/control: ceph-mds Breaks/Replaces ceph (<< 0.67.3-1).
+    - d/control: ceph-fs-common Breaks/Replaces ceph-common (<< 0.67.3-1).
+  * d/rules,control: Use google-perftools on armhf and powerpc archs.
+
+ -- James Page   Mon, 25 Nov 2013 10:13:19 +0000
+
+ceph (0.72.1-1) unstable; urgency=low
+
+  * New upstream stable release:
+    - d/ceph-test.install: Add new ceph_filestore_tool, ceph-kvstore-tool
+      and ceph_test_cls_hello binaries, drop ceph_test_store_tool.
+    - d/ceph-common.install: Add new ceph-post-file binary and manpage.
+    - d/ceph.install: Tweaked install path /usr/sbin -> /sbin.
+    - d/control: Add new BD's on python-nose and yasm.
+    - d/copyright: Updates inline with changes in codebase.
+    - d/ceph.install,rules: Install rbdmap init file using dh_installinit.
+    - Refresh patches.
+  * d/control,rules: Disable unit testing; it requires a forked version of
+    cram and is still trying to download dependencies using virtualenv.
+
+ -- James Page   Fri, 22 Nov 2013 13:02:29 +0000
+
+ceph (0.67.3-1) unstable; urgency=low
+
+  [ Laszlo Boszormenyi ]
+  * New upstream release (Closes: #693866, #705262).
+  * Update debian/copyright.
+  * Sync with Ubuntu.
+
+  [ James Page ]
+  * d/control,rules,libcephfs-{java,jni}: Enable Java CephFS library,
+    add new BD's on javahelper and default-jdk, add dbg package.
+  * d/control: Add new BD on libboost-thread-dev for RADOS Gateway
+    keystone integration.
+  * d/{control,obsync.install}: Drop obsync package inline with
+    upstream.
+  * d/librbd-dev.install: Pickup new features.h file.
+  * Remove manual calls to ldconfig:
+    - d/lib{rados2|rbd1|cephfs1}.post*: Dropped - all these do is call
+      ldconfig which will automatically be done. 
+    - d/rules: Let dh_makeshlibs do its magic with postinst/postrm.
+  * d/tests/*: Added autopkgtests for librbd, librados, python-ceph
+    and the ceph CLI.
+  * d/control: Fix versions of librbd1, librados2 and libcephfs1 for
+    python-ceph as it requires an exact version match.
+  * d/ceph.docs: Drop - README from upstream is only useful for developers
+    (Closes: #722957).
+  * d/rules: Drop --upstart-only from dh_installinit calls for upstart
+    configurations; this is deprecated in Ubuntu and not support in Debian.
+  * d/rules: Exclude jni package from shlibs generation to avoid pointless
+    ldconfig calls in maintainer scripts.
+
+  [ Bastian Blank ]
+  * Use debhelper 9.
+  * Use dh-autoreconf.
+  * Install files from source tree if possible.
+  * Run test-suite:
+    - Build-depend on python-virtualenv.
+    - Ask virtualenv to never download anything.
+  * Fix clean target.
+  * Properly mark library modules:
+    - Don't longer exclude them from stripping.
+  * Drop all libtool .la files.
+  * Generate python dependencies.
+  * Don't exclude stuff from shlibs generation.
+
+ -- Laszlo Boszormenyi (GCS)   Tue, 01 Oct 2013 02:29:08 +0200
+
+ceph (0.48-1) unstable; urgency=low
+
+  * New upstream release, the first with long-term support.
+  * As gceph dropped by upstream, remove it from packaging.
+  * Build with hardening enabled and build-conflict with libcryptopp not to
+    mix up with libnss.
+  * Use symbol versioning (closes: #679686).
+  * Update debian/watch to GitHub tags.
+
+ -- Laszlo Boszormenyi (GCS)   Sat, 07 Jul 2012 07:53:40 +0200
+
+ceph (0.47.2-1) unstable; urgency=low
+
+  * New upstream release.
+  * Use system leveldb (closes: #667907).
+  * Remove librgw1 , librgw-dev and librgw1-dbg and add rest-bench and
+    rest-bench-dbg packages.
+  * Backport leveldb build fixes from upstream git as
+    fix_leveldb_dep_for_system_library_case.patch and
+    fix_leveldb_includes_for_system_library_case.patch .
+  * Update packaging.
+  * Sync with Ubuntu: switch build-dependency from libcryptopp to libnss as
+    libcryptopp is not seeded.
+
+ -- Laszlo Boszormenyi (GCS)   Sun, 03 Jun 2012 13:37:52 +0200
+
+ceph (0.44.1-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Laszlo Boszormenyi (GCS)   Fri, 06 Apr 2012 01:10:15 +0200
+
+ceph (0.43-1) unstable; urgency=low
+
+  * New upstream release, now creates /var/run/ceph on each start
+    (closes: #660238).
+  * Update debian/copyright .
+
+ -- Laszlo Boszormenyi (GCS)   Sun, 26 Feb 2012 04:07:02 +0100
+
+ceph (0.41-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Laszlo Boszormenyi (GCS)   Sun, 05 Feb 2012 10:07:38 +0100
+
+ceph (0.40-1) unstable; urgency=low
+
+  * New upstream release (closes: #652037).
+  * Adjust copyright to match upstream source changes.
+
+ -- Laszlo Boszormenyi (GCS)   Sat, 14 Jan 2012 12:01:30 +0100
+
+ceph (0.38-1) unstable; urgency=low
+
+  * New upstream release (closes: #647764), missingok is now part of logrotate
+    directives (closes: #645651).
+  * Rename ceph-client-tools package to ceph-common , libceph-dev to
+    libcephfs-dev and libceph1{,-dbg} ones to libcephfs1{,-dbg} respectively.
+  * Update upstream VCS locations.
+
+ -- Laszlo Boszormenyi (GCS)   Sun, 27 Nov 2011 21:40:52 +0100
+
+ceph (0.35-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Laszlo Boszormenyi (GCS)   Sat, 24 Sep 2011 16:51:57 +0200
+
+ceph (0.34-1) unstable; urgency=low
+
+  * New upstream release (closes: #638714).
+  * Make librbd-dev depends on librados-dev as it uses headers from the latter
+    (closes: #636845).
+  * Add new binary packages, gceph, gceph-dbg and obsync . The libcrush ones
+    removed.
+  * Change to quilt source format and tune packaging.
+
+ -- Laszlo Boszormenyi (GCS)   Sun, 28 Aug 2011 15:56:16 +0200
+
+ceph (0.27-1.1) unstable; urgency=low
+
+  * Non-maintainer upload.
+  * Remove references to other libraries from dependency_libs field
+    (closes: #621208). 
+
+ -- Luk Claes   Sat, 28 May 2011 22:28:48 +0200
+
+ceph (0.27-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Laszlo Boszormenyi (GCS)   Mon, 25 Apr 2011 10:09:05 +0200
+
+ceph (0.25.2-1) unstable; urgency=low
+
+  * New upstream release.
+  * Make Ceph cross buildable (closes: #618939), thanks to Hector Oron.
+  * Disable libatomic-ops on ARMv4t (armel) archs to prevent FTBFS
+    (closes: #615235), thanks go to Hector Oron again.
+  * Rename librados1{,-dbg,-dev} packages to librados2{,-dbg,-dev} ones;
+    conflict with and replace the former ones.
+  * Add librbd1 and librbd-dev packages.
+
+ -- Laszlo Boszormenyi (GCS)   Sun, 27 Mar 2011 15:51:23 +0200
+
+ceph (0.24.3-2) unstable; urgency=low
+
+  * Make Ceph Linux only and build on all Linux archs (closes: #614890).
+  * Support parallel building via DEB_BUILD_OPTIONS .
+  * Add watch file, thanks to Clint Byrum (closes: #615021).
+  * Tune packaging.
+
+ -- Laszlo Boszormenyi (GCS)   Fri, 25 Feb 2011 15:17:26 +0100
+
+ceph (0.24.3-1) unstable; urgency=low
+
+  * New upstream bugfix release.
+
+ -- Laszlo Boszormenyi (GCS)   Sat, 19 Feb 2011 12:25:43 +0100
+
+ceph (0.24.2-1) unstable; urgency=low
+
+  * New upstream bugfix release.
+
+ -- Laszlo Boszormenyi (GCS)   Sat, 29 Jan 2011 15:25:14 +0100
+
+ceph (0.24.1-1) unstable; urgency=low
+
+  * New upstream bugfix release.
+
+ -- Laszlo Boszormenyi (GCS)   Tue, 11 Jan 2011 22:23:18 +0100
+
+ceph (0.24-1) unstable; urgency=low
+
+  * New upstream release.
+
+ -- Laszlo Boszormenyi (GCS)   Wed, 01 Dec 2010 09:26:25 -0800
+
+ceph (0.23.1-1) experimental; urgency=low
+
+  * Initial release (Closes: #506040)
+
+ -- Sage Weil   Sun, 21 Nov 2010 15:22:21 -0800
diff --git a/debian/compat b/debian/compat
new file mode 100644
index 00000000..ec635144
--- /dev/null
+++ b/debian/compat
@@ -0,0 +1 @@
+9
diff --git a/debian/control b/debian/control
new file mode 100644
index 00000000..5e94b902
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,468 @@
+Source: ceph
+Section: admin
+Priority: optional
+Maintainer: Ubuntu Developers 
+XSBC-Original-Maintainer: Ceph Maintainers 
+Uploaders: Laszlo Boszormenyi (GCS) ,
+           James Page 
+Homepage: http://ceph.com/
+Vcs-Git: git://anonscm.debian.org/pkg-ceph/ceph.git
+Vcs-Browser: http://anonscm.debian.org/gitweb/?p=pkg-ceph/ceph.git
+Build-Depends: debhelper (>= 9~),
+               default-jdk,
+               dh-autoreconf,
+               dpkg-dev (>= 1.16.1~),
+               javahelper,
+               junit4,
+               libaio-dev,
+               libatomic-ops-dev,
+               libblkid-dev (>= 2.17),
+               libboost-dev (>= 1.42),
+               libboost-program-options-dev (>= 1.42),
+               libboost-system-dev (>= 1.42),
+               libboost-thread-dev (>= 1.42),
+               libcurl4-gnutls-dev,
+               libedit-dev,
+               libexpat1-dev,
+               libfcgi-dev,
+               libfuse-dev,
+               libgoogle-perftools-dev [i386 amd64 powerpc armhf ppc64el],
+               libkeyutils-dev,
+               libleveldb-dev,
+               libnss3-dev,
+               libs3-dev,
+               libsnappy-dev,
+               libtool,
+               libxml2-dev,
+               pkg-config,
+               python-all (>= 2.6.6-3~),
+               python-nose,
+               uuid-dev,
+               uuid-runtime,
+               xfslibs-dev,
+               yasm
+Build-Conflicts: libcrypto++-dev
+Standards-Version: 3.9.4
+X-Python-Version: >= 2.6
+XS-Testsuite: autopkgtest
+
+Package: ceph
+Architecture: linux-any
+Depends: binutils,
+         ceph-common,
+         cryptsetup-bin | cryptsetup,
+         gdisk,
+         hdparm | sdparm,
+         parted,
+         uuid-runtime,
+         xfsprogs,
+         ${misc:Depends},
+         ${python:Depends},
+         ${shlibs:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Conflicts: gceph, librgw-dev, librgw1, librgw1-dbg, obsync
+Replaces: gceph, librgw-dev, librgw1, librgw1-dbg, obsync
+Recommends: libcephfs1, librados2, librbd1
+Description: distributed storage and file system
+ Ceph is a distributed storage system designed to provide excellent
+ performance, reliability, and scalability.
+ .
+ This package contains all server daemons and management tools for creating,
+ running, and administering a Ceph storage cluster, with the exception of the
+ metadata server, which is necessary for using the distributed file system
+ and is provided by the ceph-mds package.
+
+Package: ceph-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: ceph (= ${binary:Version}), ${misc:Depends}
+Conflicts: gceph-dbg
+Replaces: gceph-dbg
+Description: debugging symbols for ceph
+ Ceph is a distributed storage system designed to provide excellent
+ performance, reliability, and scalability.
+ .
+ This package contains the debugging symbols for ceph.
+
+Package: ceph-mds
+Architecture: linux-any
+Depends: ceph, ${misc:Depends}, ${shlibs:Depends}
+Recommends: ceph-fs-common, ceph-fuse, libcephfs1
+Breaks: ceph (<< 0.67.3-1)
+Replaces: ceph (<< 0.67.3-1)
+Description: metadata server for the ceph distributed file system
+ Ceph is a distributed storage and network file system designed to
+ provide excellent performance, reliability, and scalability.
+ .
+ This package contains the metadata server daemon, which is used to
+ create a distributed file system on top of the ceph storage cluster.
+
+Package: ceph-mds-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: ceph-mds (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for ceph-mds
+ Ceph is a distributed storage and network file system designed to provide
+ excellent performance, reliability, and scalability.
+ .
+ This package contains the debugging symbols for ceph-mds.
+
+Package: ceph-fuse
+Architecture: amd64
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Recommends: fuse
+Description: FUSE-based client for the Ceph distributed file system
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a
+ FUSE-based client that allows one to mount a Ceph file system without
+ root privileges.
+ .
+ Because the FUSE-based client has certain inherent performance
+ limitations, it is recommended that the native Linux kernel client
+ be used if possible.  If it is not practical to load a kernel module
+ (insufficient privileges, older kernel, etc.), then the FUSE client will
+ do.
+
+Package: ceph-fuse-dbg
+Architecture: amd64
+Section: debug
+Priority: extra
+Depends: ceph-fuse (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for ceph-fuse
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a
+ FUSE-based client that allows one to mount a Ceph file system without
+ root privileges.
+ .
+ This package contains the debugging symbols for ceph-fuse.
+
+Package: rbd-fuse
+Architecture: linux-any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Recommends: fuse
+Description: FUSE-based rbd client for the Ceph distributed file system
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a
+ FUSE-based client that allows one to map Ceph rbd images as files.
+ .
+ FUSE base client that allows one to map Ceph rbd images as files.
+
+Package: rbd-fuse-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: rbd-fuse (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for rbd-fuse
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a
+ FUSE-based client that allows one to map Ceph rbd images as files.
+ .
+ This package contains the debugging symbols for rbd-fuse.
+
+Package: ceph-common
+Architecture: linux-any
+Depends: librbd1 (= ${binary:Version}),
+         python-ceph (= ${binary:Version}),
+         ${misc:Depends},
+         ${python:Depends},
+         ${shlibs:Depends}
+Conflicts: ceph-client-tools
+Replaces: ceph-client-tools
+Suggests: ceph, ceph-mds
+Description: common utilities to mount and interact with a ceph storage cluster
+ Ceph is a distributed storage and file system designed to provide
+ excellent performance, reliability, and scalability.  This is a collection
+ of common tools that allow one to interact with and administer a Ceph cluster.
+
+Package: ceph-common-dbg
+Architecture: linux-any
+Depends: ceph-common (= ${binary:Version}), ${misc:Depends}
+Conflicts: ceph-client-tools-dbg
+Replaces: ceph-client-tools-dbg
+Section: debug
+Priority: extra
+Description: debugging symbols for ceph-common
+ Ceph is a distributed storage and file system designed to provide
+ excellent performance, reliability, and scalability.  This is a collection
+ of common tools that allow one to interact with and administer a Ceph cluster.
+ .
+ This package contains the debugging symbols for ceph-common.
+
+Package: ceph-fs-common
+Architecture: linux-any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Breaks: ceph-common (<< 0.67.3-1)
+Conflicts: ceph-client-tools
+Replaces: ceph-client-tools, ceph-common (<< 0.67.3-1)
+Suggests: ceph-mds
+Description: common utilities to mount and interact with a ceph file system
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a collection
+ of common tools, including the mount utility, that allows one to mount the
+ Ceph file system with the kernel client.
+
+Package: ceph-fs-common-dbg
+Architecture: linux-any
+Depends: ceph-fs-common (= ${binary:Version}), ${misc:Depends}
+Conflicts: ceph-client-tools-dbg
+Replaces: ceph-client-tools-dbg
+Section: debug
+Priority: extra
+Description: debugging symbols for ceph-fs-common
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a collection
+ of common tools, including the mount utility, that allows one to mount the
+ Ceph file system with the kernel client.
+ .
+ This package contains the debugging symbols for ceph-fs-common.
+
+Package: ceph-resource-agents
+Architecture: all
+Recommends: pacemaker
+Priority: extra
+Depends: ceph (>= ${binary:Version}), resource-agents, ${misc:Depends}
+Description: OCF-compliant resource agents for Ceph
+ Ceph is a distributed storage and network file system designed to provide
+ excellent performance, reliability, and scalability.
+ .
+ This package contains the resource agents (RAs) which integrate
+ Ceph with OCF-compliant cluster resource managers,
+ such as Pacemaker.
+
+Package: librados2
+Conflicts: libcrush, libcrush1, librados, librados1
+Replaces: libcrush, libcrush1, librados, librados1
+Architecture: linux-any
+Section: libs
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Description: RADOS distributed object store client library
+ RADOS is a reliable, autonomic distributed object storage cluster
+ developed as part of the Ceph distributed storage system.  This is a
+ shared library allowing applications to access the distributed object
+ store using a simple file-like interface.
+
+Package: librados2-dbg
+Conflicts: libcrush1-dbg, librados1-dbg
+Replaces: libcrush1-dbg, librados1-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: librados2 (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for librados2
+ RADOS is a reliable, autonomic distributed object storage cluster
+ developed as part of the Ceph distributed storage system.  This is a
+ shared library allowing applications to access the distributed object
+ store using a simple file-like interface.
+ .
+ This package contains debugging symbols for librados2.
+
+Package: librados-dev
+Architecture: linux-any
+Section: libdevel
+Depends: librados2 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}
+Conflicts: libcrush-dev, libcrush1-dev, librados1-dev, librados2-dev
+Replaces: libcrush-dev, libcrush1-dev, librados1-dev, librados2-dev
+Description: RADOS distributed object store client library (development files)
+ RADOS is a reliable, autonomic distributed object storage cluster
+ developed as part of the Ceph distributed storage system.  This is a
+ shared library allowing applications to access the distributed object
+ store using a simple file-like interface.
+ .
+ This package contains development files needed for building applications that
+ link against librados2.
+
+Package: librbd1
+Architecture: linux-any
+Section: libs
+Depends: librados2 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Description: RADOS block device client library
+ RBD is a block device striped across multiple distributed objects
+ in RADOS, a reliable, autonomic distributed object storage cluster
+ developed as part of the Ceph distributed storage system.  This is a
+ shared library allowing applications to manage these block devices.
+
+Package: librbd1-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: librbd1 (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for librbd1
+ RBD is a block device striped across multiple distributed objects
+ in RADOS, a reliable, autonomic distributed object storage cluster
+ developed as part of the Ceph distributed storage system.  This is a
+ shared library allowing applications to manage these block devices.
+ .
+ This package contains debugging symbols for librbd1.
+
+Package: librbd-dev
+Architecture: linux-any
+Section: libdevel
+Depends: librados-dev, librbd1 (= ${binary:Version}), ${misc:Depends}
+Conflicts: librbd1-dev
+Replaces: librbd1-dev
+Description: RADOS block device client library (development files)
+ RBD is a block device striped across multiple distributed objects
+ in RADOS, a reliable, autonomic distributed object storage cluster
+ developed as part of the Ceph distributed storage system.  This is a
+ shared library allowing applications to manage these block devices.
+ .
+ This package contains development files needed for building applications that
+ link against librbd1.
+
+Package: libcephfs1
+Conflicts: libceph, libceph1, libcephfs
+Replaces: libceph, libceph1, libcephfs
+Architecture: linux-any
+Section: libs
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Description: Ceph distributed file system client library
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a
+ shared library allowing applications to access a Ceph distributed
+ file system via a POSIX-like interface.
+
+Package: libcephfs1-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: libcephfs1 (= ${binary:Version}), ${misc:Depends}
+Conflicts: libceph1-dbg
+Replaces: libceph1-dbg
+Description: debugging symbols for libcephfs1
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a
+ shared library allowing applications to access a Ceph distributed
+ file system via a POSIX-like interface.
+ .
+ This package contains debugging symbols for libcephfs1.
+
+Package: libcephfs-dev
+Architecture: linux-any
+Section: libdevel
+Depends: libcephfs1 (= ${binary:Version}), ${misc:Depends}
+Conflicts: libceph-dev, libceph1-dev, libcephfs1-dev
+Replaces: libceph-dev, libceph1-dev, libcephfs1-dev
+Description: Ceph distributed file system client library (development files)
+ Ceph is a distributed network file system designed to provide
+ excellent performance, reliability, and scalability.  This is a
+ shared library allowing applications to access a Ceph distributed
+ file system via a POSIX-like interface.
+ .
+ This package contains development files needed for building applications that
+ link against libcephfs1.
+
+Package: radosgw
+Architecture: linux-any
+Depends: ceph-common (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}
+Description: REST gateway for RADOS distributed object store
+ RADOS is a distributed object store used by the Ceph distributed
+ storage system.  This package provides a REST gateway to the
+ object store that aims to implement a superset of Amazon's S3
+ service.
+ .
+ This package contains the proxy daemon and related tools only.
+
+Package: radosgw-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: radosgw (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for radosgw
+ RADOS is a distributed object store used by the Ceph distributed
+ storage system.  This package provides a REST gateway to the
+ object store that aims to implement a superset of Amazon's S3
+ service.
+ .
+ This package contains debugging symbols for radosgw.
+
+Package: rest-bench
+Architecture: linux-any
+Depends: ceph-common, curl, xml2, ${misc:Depends}, ${shlibs:Depends}
+Description: RESTful bencher that can be used to benchmark radosgw performance
+ Simple tool to benchmark radosgw (or S3) (based on 'rados-bench' command).
+
+Package: rest-bench-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: rest-bench (= ${binary:Version}), ${misc:Depends}
+Description: RESTful bencher that can be used to benchmark radosgw performance
+ Simple tool to benchmark radosgw (or S3) (based on 'rados-bench' command).
+ .
+ This package contains the debugging symbols for rest-bench.
+
+Package: ceph-test
+Architecture: linux-any
+Depends: ceph-common, curl, xml2, ${misc:Depends}, ${shlibs:Depends}
+Description: Ceph test and benchmarking tools
+ Ceph is a distributed storage system designed to provide excellent
+ performance, reliability, and scalability.
+ .
+ This package contains tools for testing and benchmarking Ceph.
+
+Package: ceph-test-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: ceph-common, curl, xml2, ${misc:Depends}, ${shlibs:Depends}
+Description: Debugging symbols for ceph-test
+ Ceph is a distributed storage system designed to provide excellent
+ performance, reliability, and scalability.
+ .
+ This package contains the debugging symbols for ceph-test.
+
+Package: python-ceph
+Architecture: linux-any
+Section: python
+Depends: libcephfs1 (= ${binary:Version}),
+         librados2 (= ${binary:Version}),
+         librbd1 (= ${binary:Version}),
+         python-flask,
+         python-requests,
+         ${misc:Depends},
+         ${python:Depends}
+Provides: ${python:Provides}
+Description: Python libraries for the Ceph distributed filesystem
+ Ceph is a distributed storage and network file system designed to provide
+ excellent performance, reliability, and scalability.
+ .
+ This package contains Python libraries for interacting with Ceph's
+ RADOS object storage, and RBD (RADOS block device).
+
+Package: libcephfs-java
+Section: java
+Architecture: all
+Depends: libcephfs-jni (>= ${binary:Version}), ${java:Depends}, ${misc:Depends}
+Description: Java library for the Ceph File System
+ Ceph is a distributed storage system designed to provide excellent
+ performance, reliability, and scalability.
+ .
+ This package contains the Java library for interacting with the Ceph
+ File System.
+
+Package: libcephfs-jni
+Architecture: linux-any
+Section: libs
+Depends: libcephfs1 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}
+Description: Java Native Interface library for CephFS Java bindings
+ Ceph is a distributed storage system designed to provide excellent
+ performance, reliability, and scalability.
+ .
+ This package contains the Java Native Interface library for interacting
+ with the Ceph File System.
+
+Package: libcephfs-jni-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: libcephfs-jni (= ${binary:Version}), ${misc:Depends}
+Description: Debugging symbols for libcephfs-jni
+ Ceph is a distributed storage system designed to provide excellent
+ performance, reliability, and scalability.
+ .
+ This package contains debugging symbols for libcephfs-jni.
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 00000000..9b8f0f5b
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,256 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: ceph
+Upstream-Contact: Sage Weil 
+Source: http://ceph.com/
+
+Files: *
+Copyright: 2004-2013 by Sage Weil 
+License: LGPL-2.1
+
+Files: src/mount/canonicalize.c
+Copyright: 1993 Rick Sladkey 
+License: LGPL-2+
+
+Files: src/os/btrfs_ioctl.h
+Copyright: 2007 Oracle.  All rights reserved.
+License: GPL-2
+
+Files: src/include/ceph_hash.cc
+Copyright: 1995-1997 Robert J. Jenkins Jr.
+License: public-domain
+  This file uses Robert Jenkin's hash function as detailed at:
+  .
+    http://burtleburtle.net/bob/hash/evahash.html
+  .
+  This is in the public domain.
+
+Files: src/common/bloom_filter.hpp
+Copyright: 2000 Arash Partow
+License: Boost Software License, Version 1.0
+ Permission is hereby granted, free of charge, to any person or organization
+ obtaining a copy of the software and accompanying documentation covered by
+ this license (the "Software") to use, reproduce, display, distribute,
+ execute, and transmit the Software, and to prepare derivative works of the
+ Software, and to permit third-parties to whom the Software is furnished to
+ do so, all subject to the following:
+ .
+   The copyright notices in the Software and this entire statement,
+   including the above license grant, this restriction and the following
+   disclaimer, must be included in all copies of the Software, in whole or
+   in part, and all derivative works of the Software, unless such copies
+   or derivative works are solely in the form of machine-executable object
+   code generated by a source language processor.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+ SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+ FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+
+Files: m4/acx_pthread.m4
+Copyright: Steven G. Johnson 
+License: GPLWithACException
+ This program is free software; you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 2 of the License, or (at your option) any later
+ version.
+ .
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see .
+ .
+ As a special exception, the respective Autoconf Macro’s copyright owner gives
+ unlimited permission to copy, distribute and modify the configure scripts that
+ are the output of Autoconf when processing the Macro. You need not follow the
+ terms of the GNU General Public License when using or distributing such
+ scripts, even though portions of the text of the Macro appear in them. The
+ GNU General Public License (GPL) does govern all other use of the material
+ that constitutes the Autoconf Macro.
+ .
+ This special exception to the GPL applies to versions of the Autoconf Macro
+ released by the Autoconf Archive. When you make and distribute a modified
+ version of the Autoconf Macro, you may extend this special exception to the
+ GPL to apply to your modified version as well.
+
+Files: src/common/crc32c_intel*
+Copyright: 2012-2013 Intel Corporation All Rights Reserved.
+License:
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ .
+ * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+ .
+ * Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in
+   the documentation and/or other materials provided with the
+   distribution.
+ .
+ * Neither the name of the Intel Corporation nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Files: src/common/sctp_crc32.c
+Copyright: 2001-2007, by Cisco Systems, Inc. All rights reserved,
+           2004-2006 Intel Corporation - All Rights Reserved
+License:
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ .
+ a) Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+ .
+ b) Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the distribution.
+ .
+ c) Neither the name of Cisco Systems, Inc. nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+Files: src/json_spirit/*
+Copyright: John W. Wilkinson 2007 - 2011
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+ .
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: src/test/common/Throttle.cc src/test/filestore/chain_xattr.cc
+Copyright: 2013 Cloudwatt 
+License: LGPL-2+
+
+Files: src/osd/ErasureCodePluginJerasure/*
+Copyright: 2011, James S. Plank 
+License:
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ .
+  - Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ .
+  - Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ .
+  - Neither the name of the University of Tennessee nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+ WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+
+Files: debian/*
+Copyright: 2004- Sage Weil ,
+           2010- Canonical, Ltd.,
+           2011- Laszlo Boszormenyi (GCS) 
+License: LGPL-2.1
+
+License: GPL-2
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ .
+ On Debian systems, the complete text of the GNU General Public License
+ version 2 can be found in `/usr/share/common-licenses/GPL-2' file.
+
+License: LGPL-2.1
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License version 2.1 as published by the Free Software Foundation.
+ .
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+ .
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ .
+ On Debian systems, the complete text of the GNU Lesser General
+ Public License can be found in `/usr/share/common-licenses/LGPL-2.1'.
+
+License: LGPL-2+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License version 2 (or later) as published by the Free Software Foundation.
+ .
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+ .
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ .
+ On Debian systems, the complete text of the GNU Lesser General
+ Public License 2 can be found in `/usr/share/common-licenses/LGPL-2'.
diff --git a/debian/gbp.conf b/debian/gbp.conf
new file mode 100644
index 00000000..fce89ec4
--- /dev/null
+++ b/debian/gbp.conf
@@ -0,0 +1,6 @@
+[DEFAULT]
+debian-branch = ubuntu
+pristine-tar = True
+
+[import-orig]
+filter = debian/*
diff --git a/debian/libcephfs-dev.install b/debian/libcephfs-dev.install
new file mode 100644
index 00000000..716d0035
--- /dev/null
+++ b/debian/libcephfs-dev.install
@@ -0,0 +1,3 @@
+usr/include/cephfs/libcephfs.h
+usr/lib/*/libcephfs.a
+usr/lib/*/libcephfs.so
diff --git a/debian/libcephfs-java.jlibs b/debian/libcephfs-java.jlibs
new file mode 100644
index 00000000..f59632af
--- /dev/null
+++ b/debian/libcephfs-java.jlibs
@@ -0,0 +1 @@
+src/java/libcephfs.jar
diff --git a/debian/libcephfs-jni.install b/debian/libcephfs-jni.install
new file mode 100644
index 00000000..15cb91d9
--- /dev/null
+++ b/debian/libcephfs-jni.install
@@ -0,0 +1 @@
+usr/lib/*/libcephfs_jni.so* usr/lib/jni
diff --git a/debian/libcephfs1.install b/debian/libcephfs1.install
new file mode 100644
index 00000000..f09e93ca
--- /dev/null
+++ b/debian/libcephfs1.install
@@ -0,0 +1 @@
+usr/lib/*/libcephfs.so.*
diff --git a/debian/librados-dev.install b/debian/librados-dev.install
new file mode 100644
index 00000000..cc8a7c0f
--- /dev/null
+++ b/debian/librados-dev.install
@@ -0,0 +1,12 @@
+usr/bin/librados-config
+usr/include/rados/buffer.h
+usr/include/rados/crc32c.h
+usr/include/rados/librados.h
+usr/include/rados/librados.hpp
+usr/include/rados/memory.h
+usr/include/rados/page.h
+usr/include/rados/rados_types.h
+usr/include/rados/rados_types.hpp
+usr/lib/*/librados.a
+usr/lib/*/librados.so
+usr/share/man/man8/librados-config.8
diff --git a/debian/librados2.install b/debian/librados2.install
new file mode 100644
index 00000000..a96bdb29
--- /dev/null
+++ b/debian/librados2.install
@@ -0,0 +1 @@
+usr/lib/*/librados.so.*
diff --git a/debian/librbd-dev.install b/debian/librbd-dev.install
new file mode 100644
index 00000000..a7ee137d
--- /dev/null
+++ b/debian/librbd-dev.install
@@ -0,0 +1,5 @@
+usr/include/rbd/features.h
+usr/include/rbd/librbd.h
+usr/include/rbd/librbd.hpp
+usr/lib/*/librbd.a
+usr/lib/*/librbd.so
diff --git a/debian/librbd1.install b/debian/librbd1.install
new file mode 100644
index 00000000..46bc2f70
--- /dev/null
+++ b/debian/librbd1.install
@@ -0,0 +1,4 @@
+../../udev/50-rbd.rules lib/udev/rules.d
+usr/bin/ceph-rbdnamer
+usr/lib/*/librbd.so.*
+usr/share/man/man8/ceph-rbdnamer.8
diff --git a/debian/patches/0001-rgw-RGWRados-get_obj-returns-wrong-len-if-len-0.patch b/debian/patches/0001-rgw-RGWRados-get_obj-returns-wrong-len-if-len-0.patch
new file mode 100644
index 00000000..f9b47e8f
--- /dev/null
+++ b/debian/patches/0001-rgw-RGWRados-get_obj-returns-wrong-len-if-len-0.patch
@@ -0,0 +1,31 @@
+From fe7bf06366adaf787816d1e68f5e3f68e8c91134 Mon Sep 17 00:00:00 2001
+From: Yehuda Sadeh 
+Date: Tue, 4 Nov 2014 22:05:03 -0800
+Subject: [PATCH] rgw: RGWRados::get_obj() returns wrong len if len == 0
+
+Fixes: #9877
+We only updated if len was > 0, should update it if r >= 0. This was the
+culprit for issue #9877.
+Backport: giant, firefly
+
+Signed-off-by: Yehuda Sadeh 
+---
+ src/rgw/rgw_rados.cc | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
+index 1d05509..709b802 100644
+--- a/src/rgw/rgw_rados.cc
++++ b/src/rgw/rgw_rados.cc
+@@ -4689,7 +4689,7 @@ int RGWRados::get_obj(void *ctx, RGWObjVersionTracker *objv_tracker, void **hand
+     bl.append(read_bl);
+ 
+ done:
+-  if (bl.length() > 0) {
++  if (r >= 0) {
+     r = bl.length();
+   }
+   if (r < 0 || !len || ((off_t)(ofs + len - 1) == end)) {
+-- 
+1.9.3 (Apple Git-50)
+
diff --git a/debian/patches/0001-rgw-fix-test-to-identify-whether-object-has-tail.patch b/debian/patches/0001-rgw-fix-test-to-identify-whether-object-has-tail.patch
new file mode 100644
index 00000000..a712ff98
--- /dev/null
+++ b/debian/patches/0001-rgw-fix-test-to-identify-whether-object-has-tail.patch
@@ -0,0 +1,41 @@
+From b8fa2ed60b6cce51701df972dbb6f5e02e0d84ba Mon Sep 17 00:00:00 2001
+From: Yehuda Sadeh 
+Date: Mon, 25 Aug 2014 10:38:42 -0700
+Subject: [PATCH] rgw: fix test to identify whether object has tail
+
+Fixes: #9226
+Reported-by: Sylvain Munaut 
+Backport: firefly
+
+We need to identify whether an object is just composed of a head, or
+also has a tail. Test for pre-firefly objects ("explicit objs") was
+broken as it was just looking at the number of explicit objs in the
+manifest. However, this is insufficient, as we might have empty head,
+and in this case it wouldn't appear, so we need to check whether the
+sole object is actually pointing at the head.
+
+Signed-off-by: Yehuda Sadeh 
+(cherry picked from commit 751b3e26532932a42ca34f9c062a0a3e29a58cff)
+---
+ src/rgw/rgw_rados.h | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
+index eea1137..dcaec83 100644
+--- a/src/rgw/rgw_rados.h
++++ b/src/rgw/rgw_rados.h
+@@ -298,6 +298,11 @@ public:
+ 
+   bool has_tail() {
+     if (explicit_objs) {
++      if (objs.size() == 1) {
++        map::iterator iter = objs.begin();
++        rgw_obj& obj = iter->second.loc;
++        return head_obj.object != obj.object;
++      }
+       return (objs.size() >= 2);
+     }
+     return (obj_size > head_size);
+-- 
+1.9.3 (Apple Git-50)
+
diff --git a/debian/patches/modules.patch b/debian/patches/modules.patch
new file mode 100644
index 00000000..b7866938
--- /dev/null
+++ b/debian/patches/modules.patch
@@ -0,0 +1,53 @@
+Description: Mark modules as actually being modules
+Author: James Page 
+Forwarded: no
+
+--- a/src/cls/Makefile.am
++++ b/src/cls/Makefile.am
+@@ -45,7 +45,7 @@ radoslib_LTLIBRARIES += libcls_replica_l
+ 
+ libcls_user_la_SOURCES = cls/user/cls_user.cc
+ libcls_user_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+-libcls_user_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__cls_.*'
++libcls_user_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
+ radoslib_LTLIBRARIES += libcls_user.la
+ 
+ libcls_rgw_la_SOURCES = \
+--- a/src/erasure-code/jerasure/Makefile.am
++++ b/src/erasure-code/jerasure/Makefile.am
+@@ -40,7 +40,7 @@ libec_jerasure_generic_la_CXXFLAGS= ${AM
+ 	-Ierasure-code/jerasure/gf-complete/include \
+ 	-Ierasure-code/jerasure/jerasure/include
+ libec_jerasure_generic_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+-libec_jerasure_generic_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0
++libec_jerasure_generic_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared
+ if LINUX
+ libec_jerasure_generic_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
+ endif
+@@ -63,7 +63,7 @@ libec_jerasure_sse3_la_CXXFLAGS= ${AM_CX
+ 	-Ierasure-code/jerasure/gf-complete/include \
+ 	-Ierasure-code/jerasure/jerasure/include
+ libec_jerasure_sse3_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+-libec_jerasure_sse3_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0
++libec_jerasure_sse3_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared
+ if LINUX
+ libec_jerasure_sse3_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
+ endif
+@@ -90,7 +90,7 @@ libec_jerasure_sse4_la_CXXFLAGS= ${AM_CX
+ 	-Ierasure-code/jerasure/gf-complete/include \
+ 	-Ierasure-code/jerasure/jerasure/include
+ libec_jerasure_sse4_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+-libec_jerasure_sse4_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0
++libec_jerasure_sse4_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared
+ if LINUX
+ libec_jerasure_sse4_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
+ endif
+@@ -102,7 +102,7 @@ libec_jerasure_la_SOURCES = \
+ libec_jerasure_la_CFLAGS = ${AM_CFLAGS} 
+ libec_jerasure_la_CXXFLAGS= ${AM_CXXFLAGS}
+ libec_jerasure_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+-libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:0:0
++libec_jerasure_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared
+ if LINUX
+ libec_jerasure_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
+ endif
diff --git a/debian/patches/series b/debian/patches/series
new file mode 100644
index 00000000..d7134e03
--- /dev/null
+++ b/debian/patches/series
@@ -0,0 +1,4 @@
+virtualenv-never-download
+modules.patch
+0001-rgw-RGWRados-get_obj-returns-wrong-len-if-len-0.patch
+0001-rgw-fix-test-to-identify-whether-object-has-tail.patch
diff --git a/debian/patches/virtualenv-never-download b/debian/patches/virtualenv-never-download
new file mode 100644
index 00000000..b95e96f0
--- /dev/null
+++ b/debian/patches/virtualenv-never-download
@@ -0,0 +1,16 @@
+Description: Ask virtualenv to never download anything
+Author: Bastian Blank 
+Origin: vendor
+Forwarded: no
+---
+--- a/src/test/run-cli-tests
++++ b/src/test/run-cli-tests
+@@ -30,7 +30,7 @@ if [ ! -e "$CRAM_BIN" ]; then
+     # patched cram to support that. See upstream ticket at
+     # https://bitbucket.org/brodie/cram/issue/9/allow-read-only-directories-for-t
+     # -- tv@inktank.com
+-    virtualenv "$VENV" && $VENV/bin/pip install "$SRCDIR/downloads/cram-0.5.0ceph.2011-01-14.tar.gz"
++    virtualenv --never-download "$VENV" && $VENV/bin/pip install "$SRCDIR/downloads/cram-0.5.0ceph.2011-01-14.tar.gz"
+ fi
+ 
+ SRCDIR_ABS="$(readlink -f "$SRCDIR")"
diff --git a/debian/python-ceph.install b/debian/python-ceph.install
new file mode 100644
index 00000000..607c0659
--- /dev/null
+++ b/debian/python-ceph.install
@@ -0,0 +1 @@
+usr/lib/python*
diff --git a/debian/radosgw.dirs b/debian/radosgw.dirs
new file mode 100644
index 00000000..f54add55
--- /dev/null
+++ b/debian/radosgw.dirs
@@ -0,0 +1,3 @@
+var/log/ceph
+var/log/radosgw
+var/lib/ceph/radosgw
diff --git a/debian/radosgw.install b/debian/radosgw.install
new file mode 100644
index 00000000..1ed620e0
--- /dev/null
+++ b/debian/radosgw.install
@@ -0,0 +1,5 @@
+etc/bash_completion.d/radosgw-admin
+usr/bin/radosgw
+usr/bin/radosgw-admin
+usr/share/man/man8/radosgw-admin.8
+usr/share/man/man8/radosgw.8
diff --git a/debian/radosgw.lintian-overrides b/debian/radosgw.lintian-overrides
new file mode 100644
index 00000000..b5c8c280
--- /dev/null
+++ b/debian/radosgw.lintian-overrides
@@ -0,0 +1,10 @@
+# Ceph upstart configuration's don't have init.d equivalents
+radosgw: init.d-script-not-marked-as-conffile etc/init.d/radosgw-all-starter
+radosgw: init.d-script-not-included-in-package etc/init.d/radosgw-all-starter
+radosgw: init.d-script-not-marked-as-conffile etc/init.d/radosgw-instance
+radosgw: init.d-script-not-included-in-package etc/init.d/radosgw-instance
+radosgw: init.d-script-not-marked-as-conffile etc/init.d/radosgw-all
+radosgw: init.d-script-not-included-in-package etc/init.d/radosgw-all
+radosgw: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/radosgw-all
+radosgw: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/radosgw-instance
+radosgw: postrm-does-not-call-updaterc.d-for-init.d-script etc/init.d/radosgw-all-starter
diff --git a/debian/radosgw.postinst b/debian/radosgw.postinst
new file mode 100644
index 00000000..ec8d86a7
--- /dev/null
+++ b/debian/radosgw.postinst
@@ -0,0 +1,62 @@
+#!/bin/sh
+# vim: set noet ts=8:
+# postinst script for radosgw
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#
+# 	postinst configure 
+# 	old-postinst abort-upgrade 
+# 	conflictor's-postinst abort-remove in-favour  
+# 	postinst abort-remove
+# 	deconfigured's-postinst abort-deconfigure in-favour   [ ]
+#
+# The current action is to simply remove the mistakenly-added
+# /etc/init/ceph.conf file; this could be done in any of these cases,
+# although technically it will leave the system in a different state
+# than the original install that included that file.  So instead we
+# only remove on "configure", since that's the only time we know we're
+# successful in installing a newer package than the erroneous version.
+
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+    configure)
+	if [ -x /sbin/start ]; then
+	        invoke-rc.d radosgw-all start || {
+	            RESULT=$?
+	            if [ $RESULT != 100 ]; then
+	                exit $RESULT
+        	    fi
+	        }
+	fi
+    ;;
+    abort-upgrade|abort-remove|abort-deconfigure)
+	:
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+if dpkg-maintscript-helper supports mv_conffile 2>/dev/null; then
+    dpkg-maintscript-helper mv_conffile \
+        /etc/init/radosgw.conf /etc/init/radosgw-instance.conf \
+        0.72.1-3~ radosgw -- "$@"
+fi
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
+
+
diff --git a/debian/radosgw.postrm b/debian/radosgw.postrm
new file mode 100644
index 00000000..3083c147
--- /dev/null
+++ b/debian/radosgw.postrm
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+if dpkg-maintscript-helper supports mv_conffile 2>/dev/null; then
+    dpkg-maintscript-helper mv_conffile \
+        /etc/init/radosgw.conf /etc/init/radosgw-instance.conf \
+        0.72.1-3~ radosgw -- "$@"
+fi
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/radosgw.preinst b/debian/radosgw.preinst
new file mode 100644
index 00000000..3083c147
--- /dev/null
+++ b/debian/radosgw.preinst
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+if dpkg-maintscript-helper supports mv_conffile 2>/dev/null; then
+    dpkg-maintscript-helper mv_conffile \
+        /etc/init/radosgw.conf /etc/init/radosgw-instance.conf \
+        0.72.1-3~ radosgw -- "$@"
+fi
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/radosgw.prerm b/debian/radosgw.prerm
new file mode 100644
index 00000000..baae2418
--- /dev/null
+++ b/debian/radosgw.prerm
@@ -0,0 +1,30 @@
+#!/bin/sh
+# vim: set noet ts=8:
+
+set -e
+
+case "$1" in
+    remove)
+	if [ -x /sbin/stop ]; then
+		invoke-rc.d radosgw-all stop || {
+	            RESULT=$?
+	            if [ $RESULT != 100 ]; then
+        	        exit $RESULT
+	            fi
+	        }
+	fi
+	invoke-rc.d radosgw stop || {
+	    RESULT=$?
+	    if [ $RESULT != 100 ]; then
+		exit $RESULT
+	    fi
+	}
+	;;
+
+    *)
+	;;
+esac
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/rbd-fuse.install b/debian/rbd-fuse.install
new file mode 100644
index 00000000..7b6b96fe
--- /dev/null
+++ b/debian/rbd-fuse.install
@@ -0,0 +1,2 @@
+usr/bin/rbd-fuse
+usr/share/man/man8/rbd-fuse.8
diff --git a/debian/rest-bench.install b/debian/rest-bench.install
new file mode 100644
index 00000000..8535f20d
--- /dev/null
+++ b/debian/rest-bench.install
@@ -0,0 +1 @@
+usr/bin/rest-bench
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 00000000..ef63946b
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,105 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+#export DH_VERBOSE=1
+
+export DESTDIR=$(CURDIR)/debian/tmp
+
+# Enable hardening
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+include /usr/share/dpkg/buildflags.mk
+
+export DEB_HOST_ARCH      ?= $(shell dpkg-architecture -qDEB_HOST_ARCH)
+
+extraopts += --disable-silent-rules
+extraopts += --with-ocf --with-rest-bench --with-nss
+extraopts += --with-debug
+extraopts += --enable-cephfs-java
+
+ifeq ($(DEB_HOST_ARCH), armel)
+  # armel supports ARMv4t or above instructions sets.
+  # libatomic-ops is only usable with Ceph for ARMv6 or above.
+  extraopts += --without-libatomic-ops
+endif
+
+# Only selected architectures support gperftools
+gperftools_archs = i386 amd64 powerpc armhf ppc64el
+ifneq (,$(filter $(DEB_HOST_ARCH), $(gperftools_archs)))
+  extraopts += --with-tcmalloc
+else
+  extraopts += --without-tcmalloc
+endif
+
+# Use system provided libs3
+extraopts += --with-system-libs3
+
+%:
+	dh $@ --with javahelper --with python2 --with autoreconf --parallel
+
+override_dh_auto_configure:
+	dh_auto_configure -- $(extraopts)
+
+override_dh_auto_build:
+	dh_auto_build
+	cp src/init-ceph debian/ceph.init
+	cp src/init-radosgw debian/radosgw.init
+	cp src/init-rbdmap debian/ceph.rbdmap.init
+	cp src/upstart/rbdmap.conf debian/ceph.rbdmap.upstart
+	cp src/logrotate.conf debian/ceph.logrotate
+	cp src/rgw/logrotate.conf debian/radosgw.logrotate
+
+override_dh_auto_clean:
+	dh_auto_clean
+	rm -f debian/ceph.init debian/radosgw.init debian/ceph.rbdmap.init debian/ceph.rbdmap.upstart
+	rm -f debian/ceph.logrotate debian/radosgw.logrotate
+	rm -f debian/*.upstart
+
+override_dh_auto_install:
+	dh_auto_install
+
+override_dh_installinit:
+	dh_installinit --no-start
+	dh_installinit -pceph --no-start --name=rbdmap
+	# Install upstart configurations using dh_installinit
+	for conf in `ls -1 src/upstart/ceph*.conf | grep -v mds`; do \
+        name=`basename $$conf | cut -d . -f 1`; \
+        cp $$conf debian/ceph.$$name.upstart; \
+        dh_installinit -pceph --no-start --name=$$name; \
+    done
+	for conf in `ls -1 src/upstart/ceph-mds*.conf`; do \
+        name=`basename $$conf | cut -d . -f 1`; \
+        cp $$conf debian/ceph-mds.$$name.upstart; \
+        dh_installinit -pceph-mds --no-start --name=$$name; \
+    done
+	for conf in `ls -1 src/upstart/radosgw*.conf`; do \
+        name=`basename $$conf | cut -d . -f 1`; \
+        [ $$name = "radosgw" ] && name="radosgw-instance";\
+        cp $$conf debian/radosgw.$$name.upstart; \
+        dh_installinit -pradosgw --no-start --name=$$name; \
+    done
+
+override_dh_strip:
+	dh_strip -pceph --dbg-package=ceph-dbg
+	dh_strip -pceph-mds --dbg-package=ceph-mds-dbg
+	dh_strip -pceph-fuse --dbg-package=ceph-fuse-dbg
+	dh_strip -prbd-fuse --dbg-package=rbd-fuse-dbg
+	dh_strip -pceph-common --dbg-package=ceph-common-dbg
+	dh_strip -pceph-fs-common --dbg-package=ceph-fs-common-dbg
+	dh_strip -plibrados2 --dbg-package=librados2-dbg
+	dh_strip -plibrbd1 --dbg-package=librbd1-dbg
+	dh_strip -plibcephfs1 --dbg-package=libcephfs1-dbg
+	dh_strip -pradosgw --dbg-package=radosgw-dbg
+	dh_strip -prest-bench --dbg-package=rest-bench-dbg
+	dh_strip -pceph-test --dbg-package=ceph-test-dbg
+	dh_strip -plibrados-dev
+	dh_strip -plibcephfs-jni --dbg-package=libcephfs-jni-dbg
+
+override_dh_makeshlibs:
+	# exclude jni libraries in libcephfs-jni to avoid pointless ldconfig
+	# calls in maintainer scripts
+	dh_makeshlibs -X/usr/lib/jni
+
+override_dh_auto_test:
+	# Skip tests as they rely on virtualenv
+	:
+
+.PHONY: override_dh_auto_configure override_dh_installinit override_dh_strip override_dh_makeshlibs override_dh_auto_test
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 00000000..163aaf8d
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/debian/tests/build-rados b/debian/tests/build-rados
new file mode 100755
index 00000000..c6299925
--- /dev/null
+++ b/debian/tests/build-rados
@@ -0,0 +1,31 @@
+#!/bin/sh
+# autopkgtest check: Build and run a program against librados2 to 
+# validate that headers are installed and libraries exists
+
+set -e
+
+WORKDIR=$(mktemp -d)
+trap "rm -rf $WORKDIR" 0 INT QUIT ABRT PIPE TERM
+cd $WORKDIR
+cat < radostest.c
+#include 
+
+int
+main(void)
+{
+    int err;
+    rados_t cluster;
+
+    err = rados_create(&cluster, NULL);
+    if (err < 0) {
+       return (1);
+    }
+    return(0);
+}
+EOF
+
+gcc -o radostest radostest.c -lrados
+echo "build: OK"
+[ -x radostest ]
+./radostest
+echo "run: OK"
diff --git a/debian/tests/build-rbd b/debian/tests/build-rbd
new file mode 100755
index 00000000..5ad6f882
--- /dev/null
+++ b/debian/tests/build-rbd
@@ -0,0 +1,24 @@
+#!/bin/sh
+# autopkgtest check: Build and run a program against librbd1 to 
+# validate that headers are installed and libraries exists
+
+set -e
+
+WORKDIR=$(mktemp -d)
+trap "rm -rf $WORKDIR" 0 INT QUIT ABRT PIPE TERM
+cd $WORKDIR
+cat < rbdtest.c
+#include 
+
+int
+main(void)
+{
+    return(0);
+}
+EOF
+
+gcc -o rbdtest rbdtest.c -lrbd
+echo "build: OK"
+[ -x rbdtest ]
+./rbdtest
+echo "run: OK"
diff --git a/debian/tests/ceph-client b/debian/tests/ceph-client
new file mode 100755
index 00000000..c693a569
--- /dev/null
+++ b/debian/tests/ceph-client
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+CLIENTS=('ceph')
+
+for client in "${CLIENTS[@]}"; do
+    echo -n "Testing client $client: "
+    $client -v 2>&1 > /dev/null
+    echo "OK"
+done
diff --git a/debian/tests/control b/debian/tests/control
new file mode 100644
index 00000000..9e6a190d
--- /dev/null
+++ b/debian/tests/control
@@ -0,0 +1,3 @@
+Tests: ceph-client build-rados build-rbd python-ceph
+Depends: ceph-common, librbd-dev, librados-dev, python-ceph, build-essential
+Restrictions: needs-root
diff --git a/debian/tests/python-ceph b/debian/tests/python-ceph
new file mode 100755
index 00000000..0af8f846
--- /dev/null
+++ b/debian/tests/python-ceph
@@ -0,0 +1,7 @@
+#!/usr/bin/python
+
+# Test that rbd and rados can be imported OK
+import rbd
+import rados
+
+print "python-ceph: OK"
diff --git a/debian/watch b/debian/watch
new file mode 100644
index 00000000..dae6427a
--- /dev/null
+++ b/debian/watch
@@ -0,0 +1,3 @@
+version=3
+opts="uversionmangle=s/-/~/" \
+ http://ceph.com/download/ceph-(\d.*)\.tar\.bz2
diff --git a/tests/integration_tests.conf b/tests/integration_tests.conf
new file mode 100644
index 00000000..70505fad
--- /dev/null
+++ b/tests/integration_tests.conf
@@ -0,0 +1 @@
+TEST_GROUP='ceph_multinode_compact'
-- 
2.45.2